From flang-commits at lists.llvm.org  Tue Jul  1 00:39:19 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 00:39:19 -0700 (PDT)
Subject: [flang-commits] [flang] 65cb0ea - [Flang][OpenMP] Add Semantics
 support for Nested OpenMPLoopConstructs (#145917)
Message-ID: <686390a7.170a0220.968f2.82bb@mx.google.com>


Author: Jack Styles
Date: 2025-07-01T08:39:15+01:00
New Revision: 65cb0eae58d2b668869f3e8f10cb79eb2b8c55ac

URL: https://github.com/llvm/llvm-project/commit/65cb0eae58d2b668869f3e8f10cb79eb2b8c55ac
DIFF: https://github.com/llvm/llvm-project/commit/65cb0eae58d2b668869f3e8f10cb79eb2b8c55ac.diff

LOG: [Flang][OpenMP] Add Semantics support for Nested OpenMPLoopConstructs (#145917)

In OpenMP Version 5.1, the tile and unroll directives were added. When
using these directives, it is possible to nest them within other OpenMP
Loop Constructs. This patch enables the semantics to allow for this
behaviour on these specific directives. Any nested loops will be stored
within the initial Loop Construct until reaching the DoConstruct itself.

Relevant tests have been added, and previous behaviour has been retained
with no changes.

See also, #110008

Added: 
    flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
    flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
    flang/test/Parser/OpenMP/loop-transformation-construct01.f90
    flang/test/Parser/OpenMP/loop-transformation-construct02.f90
    flang/test/Semantics/OpenMP/loop-transformation-construct01.f90

Modified: 
    flang/include/flang/Parser/parse-tree.h
    flang/lib/Lower/OpenMP/OpenMP.cpp
    flang/lib/Parser/unparse.cpp
    flang/lib/Semantics/canonicalize-omp.cpp
    flang/lib/Semantics/check-omp-structure.cpp
    flang/lib/Semantics/resolve-directives.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 61f97b855b0e5..7e752eeb4dfe4 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -267,6 +267,7 @@ struct AccEndCombinedDirective;
 struct OpenACCDeclarativeConstruct;
 struct OpenACCRoutineConstruct;
 struct OpenMPConstruct;
+struct OpenMPLoopConstruct;
 struct OpenMPDeclarativeConstruct;
 struct OmpEndLoopDirective;
 struct OmpMemoryOrderClause;
@@ -5021,11 +5022,13 @@ struct OpenMPBlockConstruct {
 };
 
 // OpenMP directives enclosing do loop
+using NestedConstruct =
+    std::variant<DoConstruct, common::Indirection<OpenMPLoopConstruct>>;
 struct OpenMPLoopConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPLoopConstruct);
   OpenMPLoopConstruct(OmpBeginLoopDirective &&a)
       : t({std::move(a), std::nullopt, std::nullopt}) {}
-  std::tuple<OmpBeginLoopDirective, std::optional<DoConstruct>,
+  std::tuple<OmpBeginLoopDirective, std::optional<NestedConstruct>,
       std::optional<OmpEndLoopDirective>>
       t;
 };

diff  --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 96daadfaf1b3b..0a56e888ac44b 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -3769,6 +3769,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   mlir::Location currentLocation =
       converter.genLocation(beginLoopDirective.source);
 
+  auto &optLoopCons =
+      std::get<std::optional<parser::NestedConstruct>>(loopConstruct.t);
+  if (optLoopCons.has_value()) {
+    if (auto *ompNestedLoopCons{
+            std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
+                &*optLoopCons)}) {
+      genOMP(converter, symTable, semaCtx, eval, ompNestedLoopCons->value());
+    }
+  }
+
   llvm::omp::Directive directive =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
   const parser::CharBlock &source =

diff  --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ed0f227fd5b98..903287515e559 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2926,7 +2926,8 @@ class UnparseVisitor {
     Walk(std::get<OmpBeginLoopDirective>(x.t));
     Put("\n");
     EndOpenMP();
-    Walk(std::get<std::optional<DoConstruct>>(x.t));
+    Walk(std::get<std::optional<std::variant<DoConstruct,
+            common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t));
     Walk(std::get<std::optional<OmpEndLoopDirective>>(x.t));
   }
   void Unparse(const BasedPointer &x) {

diff  --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 5164f1dc6faab..1edcb376596b0 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -8,6 +8,7 @@
 
 #include "canonicalize-omp.h"
 #include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
 
 // After Loop Canonicalization, rewrite OpenMP parse tree to make OpenMP
 // Constructs more structured which provide explicit scopes for later
@@ -125,6 +126,16 @@ class CanonicalizationOfOmp {
     parser::Block::iterator nextIt;
     auto &beginDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
     auto &dir{std::get<parser::OmpLoopDirective>(beginDir.t)};
+    auto missingDoConstruct = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
+    auto tileUnrollError = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "If a loop construct has been fully unrolled, it cannot then be tiled"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
 
     nextIt = it;
     while (++nextIt != block.end()) {
@@ -135,31 +146,95 @@ class CanonicalizationOfOmp {
       if (auto *doCons{GetConstructIf<parser::DoConstruct>(*nextIt)}) {
         if (doCons->GetLoopControl()) {
           // move DoConstruct
-          std::get<std::optional<parser::DoConstruct>>(x.t) =
+          std::get<std::optional<std::variant<parser::DoConstruct,
+              common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t) =
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if (nextIt != block.end()) {
-            if (auto *endDir{
-                    GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
-              std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
-                  std::move(*endDir);
-              block.erase(nextIt);
-            }
+          if (auto *endDir{
+                  GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
+            std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                std::move(*endDir);
+            nextIt = block.erase(nextIt);
           }
         } else {
           messages_.Say(dir.source,
               "DO loop after the %s directive must have loop control"_err_en_US,
               parser::ToUpperCaseLetters(dir.source.ToString()));
         }
+      } else if (auto *ompLoopCons{
+                     GetOmpIf<parser::OpenMPLoopConstruct>(*nextIt)}) {
+        // We should allow UNROLL and TILE constructs to be inserted between an
+        // OpenMP Loop Construct and the DO loop itself
+        auto &nestedBeginDirective =
+            std::get<parser::OmpBeginLoopDirective>(ompLoopCons->t);
+        auto &nestedBeginLoopDirective =
+            std::get<parser::OmpLoopDirective>(nestedBeginDirective.t);
+        if ((nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll ||
+                nestedBeginLoopDirective.v ==
+                    llvm::omp::Directive::OMPD_tile) &&
+            !(nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll &&
+                dir.v == llvm::omp::Directive::OMPD_tile)) {
+          // iterate through the remaining block items to find the end directive
+          // for the unroll/tile directive.
+          parser::Block::iterator endIt;
+          endIt = nextIt;
+          while (endIt != block.end()) {
+            if (auto *endDir{
+                    GetConstructIf<parser::OmpEndLoopDirective>(*endIt)}) {
+              auto &endLoopDirective =
+                  std::get<parser::OmpLoopDirective>(endDir->t);
+              if (endLoopDirective.v == dir.v) {
+                std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                    std::move(*endDir);
+                endIt = block.erase(endIt);
+                continue;
+              }
+            }
+            ++endIt;
+          }
+          RewriteOpenMPLoopConstruct(*ompLoopCons, block, nextIt);
+          auto &ompLoop = std::get<std::optional<parser::NestedConstruct>>(x.t);
+          ompLoop =
+              std::optional<parser::NestedConstruct>{parser::NestedConstruct{
+                  common::Indirection{std::move(*ompLoopCons)}}};
+          nextIt = block.erase(nextIt);
+        } else if (nestedBeginLoopDirective.v ==
+                llvm::omp::Directive::OMPD_unroll &&
+            dir.v == llvm::omp::Directive::OMPD_tile) {
+          // if a loop has been unrolled, the user can not then tile that loop
+          // as it has been unrolled
+          parser::OmpClauseList &unrollClauseList{
+              std::get<parser::OmpClauseList>(nestedBeginDirective.t)};
+          if (unrollClauseList.v.empty()) {
+            // if the clause list is empty for an unroll construct, we assume
+            // the loop is being fully unrolled
+            tileUnrollError(dir, messages_);
+          } else {
+            // parse the clauses for the unroll directive to find the full
+            // clause
+            for (auto clause{unrollClauseList.v.begin()};
+                clause != unrollClauseList.v.end(); ++clause) {
+              if (clause->Id() == llvm::omp::OMPC_full) {
+                tileUnrollError(dir, messages_);
+              }
+            }
+          }
+        } else {
+          messages_.Say(nestedBeginLoopDirective.source,
+              "Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs"_err_en_US,
+              parser::ToUpperCaseLetters(
+                  nestedBeginLoopDirective.source.ToString()));
+        }
       } else {
-        messages_.Say(dir.source,
-            "A DO loop must follow the %s directive"_err_en_US,
-            parser::ToUpperCaseLetters(dir.source.ToString()));
+        missingDoConstruct(dir, messages_);
       }
       // If we get here, we either found a loop, or issued an error message.
       return;
     }
+    if (nextIt == block.end()) {
+      missingDoConstruct(dir, messages_);
+    }
   }
 
   void RewriteOmpAllocations(parser::ExecutionPart &body) {

diff  --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 3abb5a304b00c..e080bce3cac3a 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -762,10 +762,13 @@ void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
   }
   SetLoopInfo(x);
 
-  if (const auto &doConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
-    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &doConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
+      CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+    }
   }
   CheckLoopItrVariableIsInt(x);
   CheckAssociatedLoopConstraints(x);
@@ -786,12 +789,15 @@ const parser::Name OmpStructureChecker::GetLoopIndex(
   return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
 }
 void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const parser::DoConstruct *loop{&*loopConstruct};
-    if (loop && loop->IsDoNormal()) {
-      const parser::Name &itrVal{GetLoopIndex(loop)};
-      SetLoopIv(itrVal.symbol);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const parser::DoConstruct *loop{&*loopConstruct};
+      if (loop && loop->IsDoNormal()) {
+        const parser::Name &itrVal{GetLoopIndex(loop)};
+        SetLoopIv(itrVal.symbol);
+      }
     }
   }
 }
@@ -857,27 +863,30 @@ void OmpStructureChecker::CheckIteratorModifier(const parser::OmpIterator &x) {
 
 void OmpStructureChecker::CheckLoopItrVariableIsInt(
     const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
 
-    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-      if (loop->IsDoNormal()) {
-        const parser::Name &itrVal{GetLoopIndex(loop)};
-        if (itrVal.symbol) {
-          const auto *type{itrVal.symbol->GetType()};
-          if (!type->IsNumeric(TypeCategory::Integer)) {
-            context_.Say(itrVal.source,
-                "The DO loop iteration"
-                " variable must be of the type integer."_err_en_US,
-                itrVal.ToString());
+      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+        if (loop->IsDoNormal()) {
+          const parser::Name &itrVal{GetLoopIndex(loop)};
+          if (itrVal.symbol) {
+            const auto *type{itrVal.symbol->GetType()};
+            if (!type->IsNumeric(TypeCategory::Integer)) {
+              context_.Say(itrVal.source,
+                  "The DO loop iteration"
+                  " variable must be of the type integer."_err_en_US,
+                  itrVal.ToString());
+            }
           }
         }
+        // Get the next DoConstruct if block is not empty.
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                 : nullptr;
       }
-      // Get the next DoConstruct if block is not empty.
-      const auto &block{std::get<parser::Block>(loop->t)};
-      const auto it{block.begin()};
-      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                               : nullptr;
     }
   }
 }
@@ -1077,25 +1086,28 @@ void OmpStructureChecker::CheckDistLinear(
 
     // Match the loop index variables with the collected symbols from linear
     // clauses.
-    if (const auto &loopConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-        if (loop->IsDoNormal()) {
-          const parser::Name &itrVal{GetLoopIndex(loop)};
-          if (itrVal.symbol) {
-            // Remove the symbol from the collected set
-            indexVars.erase(&itrVal.symbol->GetUltimate());
-          }
-          collapseVal--;
-          if (collapseVal == 0) {
-            break;
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &loopConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+          if (loop->IsDoNormal()) {
+            const parser::Name &itrVal{GetLoopIndex(loop)};
+            if (itrVal.symbol) {
+              // Remove the symbol from the collected set
+              indexVars.erase(&itrVal.symbol->GetUltimate());
+            }
+            collapseVal--;
+            if (collapseVal == 0) {
+              break;
+            }
           }
+          // Get the next DoConstruct if block is not empty.
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                   : nullptr;
         }
-        // Get the next DoConstruct if block is not empty.
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                 : nullptr;
       }
     }
 

diff  --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..8d741734601a7 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1796,10 +1796,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
 
   if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    if (const auto &doConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      if (doConstruct.value().IsDoWhile()) {
-        return true;
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &doConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        if (doConstruct->IsDoWhile()) {
+          return true;
+        }
       }
     }
   }
@@ -1962,48 +1965,69 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndexAndCheckLoopLevel(
   bool hasCollapseClause{
       clause ? (clause->Id() == llvm::omp::OMPC_collapse) : false};
 
-  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
-  if (outer.has_value()) {
-    for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-      if (loop->IsDoConcurrent()) {
-        // DO CONCURRENT is explicitly allowed for the LOOP construct so long as
-        // there isn't a COLLAPSE clause
-        if (isLoopConstruct) {
-          if (hasCollapseClause) {
-            // hasCollapseClause implies clause != nullptr
-            context_.Say(clause->source,
-                "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &outer{std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      for (const parser::DoConstruct *loop{&*outer}; loop && level > 0;
+          --level) {
+        if (loop->IsDoConcurrent()) {
+          // DO CONCURRENT is explicitly allowed for the LOOP construct so long
+          // as there isn't a COLLAPSE clause
+          if (isLoopConstruct) {
+            if (hasCollapseClause) {
+              // hasCollapseClause implies clause != nullptr
+              context_.Say(clause->source,
+                  "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+            }
+          } else {
+            auto &stmt =
+                std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
+            context_.Say(stmt.source,
+                "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
           }
-        } else {
-          auto &stmt =
-              std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
-          context_.Say(stmt.source,
-              "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
-        }
-      }
-      // go through all the nested do-loops and resolve index variables
-      const parser::Name *iv{GetLoopIndex(*loop)};
-      if (iv) {
-        if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
-          SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
-          iv->symbol = symbol; // adjust the symbol within region
-          AddToContextObjectWithDSA(*symbol, ivDSA);
         }
+        // go through all the nested do-loops and resolve index variables
+        const parser::Name *iv{GetLoopIndex(*loop)};
+        if (iv) {
+          if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
+            SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
+            iv->symbol = symbol; // adjust the symbol within region
+            AddToContextObjectWithDSA(*symbol, ivDSA);
+          }
 
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+        }
       }
+      CheckAssocLoopLevel(level, GetAssociatedClause());
+    } else if (const auto &loop{std::get_if<
+                   common::Indirection<parser::OpenMPLoopConstruct>>(
+                   &*optLoopCons)}) {
+      auto &beginDirective =
+          std::get<parser::OmpBeginLoopDirective>(loop->value().t);
+      auto &beginLoopDirective =
+          std::get<parser::OmpLoopDirective>(beginDirective.t);
+      if (beginLoopDirective.v != llvm::omp::Directive::OMPD_unroll &&
+          beginLoopDirective.v != llvm::omp::Directive::OMPD_tile) {
+        context_.Say(GetContext().directiveSource,
+            "Only UNROLL or TILE constructs are allowed between an OpenMP Loop Construct and a DO construct"_err_en_US,
+            parser::ToUpperCaseLetters(llvm::omp::getOpenMPDirectiveName(
+                GetContext().directive, version)
+                    .str()));
+      } else {
+        PrivatizeAssociatedLoopIndexAndCheckLoopLevel(loop->value());
+      }
+    } else {
+      context_.Say(GetContext().directiveSource,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
+                  .str()));
     }
-    CheckAssocLoopLevel(level, GetAssociatedClause());
-  } else {
-    context_.Say(GetContext().directiveSource,
-        "A DO loop must follow the %s directive"_err_en_US,
-        parser::ToUpperCaseLetters(
-            llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
-                .str()));
   }
 }
+
 void OmpAttributeVisitor::CheckAssocLoopLevel(
     std::int64_t level, const parser::OmpClause *clause) {
   if (clause && level != 0) {

diff  --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..a76e7e52100db
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for tile OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end do
+end subroutine
+
+!CHECK: not yet implemented: Unhandled loop directive (tile)

diff  --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
new file mode 100644
index 0000000000000..33b7c5a917619
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for unroll OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+program loop_transformation_construct
+  implicit none
+  integer, parameter :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do x = 1, I
+    y(x) = y(x) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end program loop_transformation_construct
+
+!CHECK: not yet implemented: Unhandled loop directive (unroll)

diff  --git a/flang/test/Parser/OpenMP/loop-transformation-construct01.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..baffc2f6e2f1e
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,74 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with 1 nested loop.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE

diff  --git a/flang/test/Parser/OpenMP/loop-transformation-construct02.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
new file mode 100644
index 0000000000000..b50e7183841cc
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
@@ -0,0 +1,85 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with multiple nested loops.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP TILE
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END TILE
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE

diff  --git a/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90 b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..f718efc32aabf
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,100 @@
+! Testing the Semantics of nested Loop Transformation Constructs
+
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=51
+
+subroutine loop_transformation_construct1
+  implicit none
+
+  !$omp do
+  !ERROR: A DO loop must follow the UNROLL directive
+  !$omp unroll
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !$omp end tile
+  !$omp end do
+  !ERROR: The END TILE directive must follow the DO loop associated with the loop construct
+  !$omp end tile
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs
+  !$omp parallel do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct3
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !ERROR: A DO loop must follow the TILE directive
+  !$omp tile
+end subroutine
+
+subroutine loop_transformation_construct4
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll full
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct5
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct6
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  !$omp unroll partial(2)
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine


        

From flang-commits at lists.llvm.org  Tue Jul  1 01:09:50 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 01 Jul 2025 01:09:50 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][OpenMP] Enable tiling
	(PR #143715)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143715@github.com>
Message-ID: <686397ce.170a0220.26efc5.0b04@mx.google.com>


================
@@ -131,20 +134,55 @@ class CanonicalizationOfOmp {
       // Ignore compiler directives.
       if (GetConstructIf<parser::CompilerDirective>(*nextIt))
         continue;
+      // Keep track of the loops to handle the end loop directives
+      llvm::SmallVector<parser::OpenMPLoopConstruct *> loops;
+      loops.push_back(&x);
+      if (auto *innerOmpLoop{GetOmpIf<parser::OpenMPLoopConstruct>(*nextIt)}) {
+        auto &innerBeginDir{
+            std::get<parser::OmpBeginLoopDirective>(innerOmpLoop->t)};
+        auto &innerDir{std::get<parser::OmpLoopDirective>(innerBeginDir.t)};
+        if (innerDir.v == llvm::omp::Directive::OMPD_tile) {
+          auto &innerLoop = std::get<
+              std::optional<common::Indirection<parser::OpenMPLoopConstruct>>>(
+              loops.back()->t);
+          innerLoop = std::move(*innerOmpLoop);
+          // Retrieveing the address so that DoConstruct or inner loop can be
+          // set later.
+          loops.push_back(&(innerLoop.value().value()));
+          nextIt = block.erase(nextIt);
+        }
+      }
 
       if (auto *doCons{GetConstructIf<parser::DoConstruct>(*nextIt)}) {
         if (doCons->GetLoopControl()) {
-          // move DoConstruct
-          std::get<std::optional<parser::DoConstruct>>(x.t) =
+          std::get<std::optional<parser::DoConstruct>>(loops.back()->t) =
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if (nextIt != block.end()) {
+          while (nextIt != block.end() && !loops.empty()) {
----------------
Stylie777 wrote:

Will resolve as #145917 is merged.

https://github.com/llvm/llvm-project/pull/143715

From flang-commits at lists.llvm.org  Tue Jul  1 01:36:29 2025
From: flang-commits at lists.llvm.org (Benjamin Maxwell via flang-commits)
Date: Tue, 01 Jul 2025 01:36:29 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <68639e0d.050a0220.11624b.2b0d@mx.google.com>


https://github.com/MacDue commented:

You may also want to change `Options.td` currently the note about needing GLIBC 2.40 is only displayed for Clang as it's under ` HelpTextForVariants<[ClangOption, CC1Option]`:
https://github.com/llvm/llvm-project/blob/cb806510914ed909b934d285062a9efb13b1cea4/clang/include/clang/Driver/Options.td#L3491-L3494

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 01:36:29 2025
From: flang-commits at lists.llvm.org (Benjamin Maxwell via flang-commits)
Date: Tue, 01 Jul 2025 01:36:29 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <68639e0d.170a0220.e85c6.8e2c@mx.google.com>


https://github.com/MacDue edited https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 01:36:29 2025
From: flang-commits at lists.llvm.org (Benjamin Maxwell via flang-commits)
Date: Tue, 01 Jul 2025 01:36:29 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <68639e0d.170a0220.378fe6.886b@mx.google.com>


================
@@ -34,6 +34,7 @@ page](https://llvm.org/releases/).
 
 * -floop-interchange is now recognized by flang.
 * -floop-interchange is enabled by default at -O2 and above.
+* -fveclib=libmvec is supported for AArch64 (same as Flang/x86 and Clang/AArch64)
----------------
MacDue wrote:

Is it worth noting the GLIBC 2.40 dependency here?

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 01:36:50 2025
From: flang-commits at lists.llvm.org (Benjamin Maxwell via flang-commits)
Date: Tue, 01 Jul 2025 01:36:50 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <68639e22.050a0220.1c69b6.0773@mx.google.com>


https://github.com/MacDue edited https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 01:37:00 2025
From: flang-commits at lists.llvm.org (Benjamin Maxwell via flang-commits)
Date: Tue, 01 Jul 2025 01:37:00 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <68639e2c.170a0220.2b25d9.db81@mx.google.com>


https://github.com/MacDue edited https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 02:27:00 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 02:27:00 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6863a9e4.170a0220.22acaf.8c08@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM with MacDue's comments. Thank you for contributing this!

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 02:37:23 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 02:37:23 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (PR #146359)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146359@github.com>
Message-ID: <6863ac53.050a0220.229596.1be9@mx.google.com>


https://github.com/tblah commented:

How bad is the impact on flang build time? From what I remember from when OpenMP lowering was split, the compilation file of each translation unit was so dominated by processing the headers that adding `n` translation units roughly increased compilation time of that part by a factor of `n`.

But that aside I think this is a good idea from a code design perspective.

Please could you add a comment at the top of each file documenting what should go in that file (just the text from the commit message would be fine).

https://github.com/llvm/llvm-project/pull/146359

From flang-commits at lists.llvm.org  Tue Jul  1 03:35:32 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 03:35:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863b9f4.050a0220.24e87f.3912@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 03:35:33 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 03:35:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863b9f5.050a0220.01ed.1992@mx.google.com>


================
@@ -41,6 +43,178 @@ namespace omp {
 using namespace Fortran::lower::omp;
 }
 
+namespace {
+// An example of a type that can be used to get the return value from
+// the visitor:
+//   visitor(type_identity<Xyz>) -> result_type
+using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
+
+struct GetProc
+    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
+                                false> {
+  using Result = const evaluate::ProcedureDesignator *;
+  using Base = evaluate::Traverse<GetProc, Result, false>;
+  GetProc() : Base(*this) {}
+
+  using Base::operator();
+
+  static Result Default() { return nullptr; }
+
+  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
+  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
+};
+
+struct WithType {
+  WithType(const evaluate::DynamicType &t) : type(t) {
+    assert(type.category() != common::TypeCategory::Derived &&
+           "Type cannot be a derived type");
+  }
+
+  template <typename VisitorTy> //
+  auto visit(VisitorTy &&visitor) const
+      -> std::invoke_result_t<VisitorTy, SomeArgType> {
+    switch (type.category()) {
+    case common::TypeCategory::Integer:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Unsigned:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Real:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Complex:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Logical:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
+      }
+      break;
+    case common::TypeCategory::Character:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
+      }
+      break;
+    case common::TypeCategory::Derived:
+      break;
+    }
+    llvm_unreachable("Unhandled type");
+  }
+
+  const evaluate::DynamicType &type;
+
+private:
+  // Shorter names.
+  static constexpr auto Character = common::TypeCategory::Character;
+  static constexpr auto Complex = common::TypeCategory::Complex;
+  static constexpr auto Derived = common::TypeCategory::Derived;
----------------
tblah wrote:

Unused

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 03:35:33 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 03:35:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863b9f5.170a0220.a10d7.90e9@mx.google.com>


================
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
+!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: } else {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: }
+!CHECK: omp.atomic.update {{.*}} : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
+!CHECK:   %[[V10:[a-z0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[IF_VAL]]
+!CHECK:   %[[V11:[a-z0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[IF_VAL]]
+!CHECK:   omp.yield(%[[V11]] : f32)
+!CHECK: }
+
+subroutine f00(a, x, y)
+  real :: a
+  real, optional :: x, y
+  !$omp atomic update
+  a = max(x, a, y)
----------------
tblah wrote:

Please also add a test for `min`

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 03:35:33 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 03:35:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863b9f5.170a0220.1f72e0.8985@mx.google.com>


https://github.com/tblah commented:

This is really nice. Just minor comments.

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 03:35:33 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 03:35:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863b9f5.170a0220.e1fcc.9579@mx.google.com>


================
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
+!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
+!CHECK:   fir.result {{.*}} : f32
----------------
tblah wrote:

Please could you add checks for the non-atomic max operation here so that it is clearer where that part goes

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 03:35:35 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 03:35:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863b9f7.170a0220.3d3cb2.0084@mx.google.com>


================
@@ -350,10 +603,28 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
 
   // This must exist by now.
-  semantics::SomeExpr input = *evaluate::GetConvertInput(assign.rhs);
-  std::vector<semantics::SomeExpr> args =
-      evaluate::GetTopLevelOperation(input).second;
+  semantics::SomeExpr rhs = assign.rhs;
+  semantics::SomeExpr input = *evaluate::GetConvertInput(rhs);
+  auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
+
+  const semantics::SomeExpr *atomArg = [&]() {
+    for (const semantics::SomeExpr &e : args) {
----------------
tblah wrote:

```
/home/USER/llvm-project/flang/lib/Lower/OpenMP/Atomic.cpp:612:41: error: captured structured bindings are a C++20 extension [-Werror,-Wc++20-extensions]
  612 |     for (const semantics::SomeExpr &e : args) {
      |                                         ^
/home/USER/llvm-project/flang/lib/Lower/OpenMP/Atomic.cpp:608:17: note: 'args' declared here
  608 |   auto [opcode, args] = evaluate::GetTopLevelOperation(input);
      |                 ^
1 error generated.
```

With clang 19

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 03:35:35 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 03:35:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863b9f7.170a0220.2c840b.789c@mx.google.com>


================
@@ -237,6 +411,85 @@ makeMemOrderAttr(lower::AbstractConverter &converter,
   return nullptr;
 }
 
+static bool replaceArgs(semantics::SomeExpr &expr,
+                        evaluate::ActualArguments &&newArgs) {
+  return ArgumentReplacer(std::move(newArgs))(expr);
+}
+
+static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
+                                    const evaluate::ProcedureDesignator &proc,
+                                    const evaluate::ActualArguments &args) {
+  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
+    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
+    return evaluate::AsGenericExpr(
+        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
+  });
+}
+
+static const evaluate::ProcedureDesignator &
+getProcedureDesignator(const semantics::SomeExpr &call) {
+  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
+  assert(proc && "Call has no procedure designator");
+  return *proc;
+}
+
+static semantics::SomeExpr //
+genReducedMinMax(const semantics::SomeExpr &orig,
+                 const semantics::SomeExpr *atomArg,
+                 const std::vector<semantics::SomeExpr> &args) {
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be atom (or a convert of atom).
----------------
tblah wrote:

ultra-nit
```suggestion
  // One of the a_i's, say a_t, must be atomArg (or a convert of atomArg).
```

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 04:25:43 2025
From: flang-commits at lists.llvm.org (KAWASHIMA Takahiro via flang-commits)
Date: Tue, 01 Jul 2025 04:25:43 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6863c5b7.170a0220.3a565.7971@mx.google.com>


https://github.com/kawashima-fj updated https://github.com/llvm/llvm-project/pull/146453

>From 158eb6d00a123e5d427b4aa52b37824ff3e20840 Mon Sep 17 00:00:00 2001
From: KAWASHIMA Takahiro <t-kawashima at fujitsu.com>
Date: Wed, 25 Jun 2025 16:34:21 +0900
Subject: [PATCH 1/2] [Flang][AArch64][VecLib] Add libmvec support for
 Flang/AArch64

`-fveclib=libmvec` for AArch64 (NEON and SVE) in Clang was supported
by #143696. This patch does the same for Flang.

Vector functions defined in `libmvec` are used for the following
Fortran operator and functions currently.

- Power operator (`**`)
- Fortran intrinsic functions listed below for `real(kind=4)` and
  `real(kind=8)`
  (including their coresponding specific intrinsic functions)
- Fortran intrinsic functions which are expanded using functions
  listed below
  (for example, `sin` for `complex(kind=8)`)

```
sin
tan
cos
asin
acos
atan (both atan(x) and atan(y, x))
atan2
cosh
tanh
asinh
acosh
atanh
erf
erfc
exp
log
log10
```

As with Clang/AArch64, glibc 2.40 or higher is required to use all
these functions.
---
 clang/lib/Driver/ToolChains/Flang.cpp | 9 ++++++++-
 flang/docs/ReleaseNotes.md            | 1 +
 flang/test/Driver/fveclib-codegen.f90 | 4 ++++
 flang/test/Driver/fveclib.f90         | 3 ++-
 4 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index e4e321ba1e195..75ba2af543c7a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -484,11 +484,18 @@ void Flang::addTargetOptions(const ArgList &Args,
           Triple.getArch() != llvm::Triple::x86_64)
         D.Diag(diag::err_drv_unsupported_opt_for_target)
             << Name << Triple.getArchName();
-    } else if (Name == "libmvec" || Name == "AMDLIBM") {
+    } else if (Name == "AMDLIBM") {
       if (Triple.getArch() != llvm::Triple::x86 &&
           Triple.getArch() != llvm::Triple::x86_64)
         D.Diag(diag::err_drv_unsupported_opt_for_target)
             << Name << Triple.getArchName();
+    } else if (Name == "libmvec") {
+      if (Triple.getArch() != llvm::Triple::x86 &&
+          Triple.getArch() != llvm::Triple::x86_64 &&
+          Triple.getArch() != llvm::Triple::aarch64 &&
+          Triple.getArch() != llvm::Triple::aarch64_be)
+        D.Diag(diag::err_drv_unsupported_opt_for_target)
+            << Name << Triple.getArchName();
     } else if (Name == "SLEEF" || Name == "ArmPL") {
       if (Triple.getArch() != llvm::Triple::aarch64 &&
           Triple.getArch() != llvm::Triple::aarch64_be)
diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index 35da8323e0a10..bd3a215323de2 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -34,6 +34,7 @@ page](https://llvm.org/releases/).
 
 * -floop-interchange is now recognized by flang.
 * -floop-interchange is enabled by default at -O2 and above.
+* -fveclib=libmvec is supported for AArch64 (same as Flang/x86 and Clang/AArch64)
 
 ## Windows Support
 
diff --git a/flang/test/Driver/fveclib-codegen.f90 b/flang/test/Driver/fveclib-codegen.f90
index 4cbb1e284f18e..88b31da40167e 100644
--- a/flang/test/Driver/fveclib-codegen.f90
+++ b/flang/test/Driver/fveclib-codegen.f90
@@ -1,6 +1,8 @@
 ! test that -fveclib= is passed to the backend
 ! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=SLEEF -o - %s | FileCheck %s --check-prefix=SLEEF %}
 ! RUN: %if x86-registered-target %{ %flang -S -Ofast -target x86_64-unknown-linux-gnu -fveclib=libmvec -o - %s | FileCheck %s %}
+! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=libmvec -march=armv8.2-a+nosve -o - %s | FileCheck %s --check-prefix=LIBMVEC-AARCH64-NEON %}
+! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=libmvec -march=armv8.2-a+sve -o - %s | FileCheck %s --check-prefix=LIBMVEC-AARCH64-SVE %}
 ! RUN: %if x86-registered-target %{ %flang -S -O3 -ffast-math -target x86_64-unknown-linux-gnu -fveclib=AMDLIBM -o - %s | FileCheck %s --check-prefix=AMDLIBM %}
 ! RUN: %flang -S -Ofast -fveclib=NoLibrary -o - %s | FileCheck %s --check-prefix=NOLIB
 
@@ -11,6 +13,8 @@ subroutine sb(a, b)
 ! check that we used a vectorized call to powf()
 ! CHECK: _ZGVbN4vv_powf
 ! SLEEF: _ZGVnN4vv_powf
+! LIBMVEC-AARCH64-NEON: _ZGVnN4vv_powf
+! LIBMVEC-AARCH64-SVE: _ZGVsMxvv_powf
 ! AMDLIBM: amd_vrs4_powf
 ! NOLIB: powf
     a(i) = a(i) ** b(i)
diff --git a/flang/test/Driver/fveclib.f90 b/flang/test/Driver/fveclib.f90
index 431a4bfc02522..d21e85e486f8d 100644
--- a/flang/test/Driver/fveclib.f90
+++ b/flang/test/Driver/fveclib.f90
@@ -1,6 +1,7 @@
 ! RUN: %flang -### -c -fveclib=none %s 2>&1 | FileCheck -check-prefix CHECK-NOLIB %s
 ! RUN: %flang -### -c -fveclib=Accelerate %s 2>&1 | FileCheck -check-prefix CHECK-ACCELERATE %s
 ! RUN: %flang -### -c --target=x86_64-unknown-linux-gnu -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-libmvec %s
+! RUN: %flang -### -c --target=aarch64-unknown-linux-gnu -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-libmvec %s
 ! RUN: %flang -### -c -fveclib=MASSV %s 2>&1 | FileCheck -check-prefix CHECK-MASSV %s
 ! RUN: %flang -### -c -fveclib=Darwin_libsystem_m %s 2>&1 | FileCheck -check-prefix CHECK-DARWIN_LIBSYSTEM_M %s
 ! RUN: %flang -### -c --target=aarch64-none-none -fveclib=SLEEF %s 2>&1 | FileCheck -check-prefix CHECK-SLEEF %s
@@ -23,7 +24,7 @@
 
 ! RUN: not %flang --target=x86-none-none -c -fveclib=SLEEF %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=x86-none-none -c -fveclib=ArmPL %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
-! RUN: not %flang --target=aarch64-none-none -c -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
+! RUN: not %flang --target=riscv64-none-none -c -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=aarch64-none-none -c -fveclib=SVML %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=aarch64-none-none -c -fveclib=AMDLIBM %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! CHECK-ERROR: unsupported option {{.*}} for target

>From 943347688a6fd43be367143aa1b7a19587da2573 Mon Sep 17 00:00:00 2001
From: KAWASHIMA Takahiro <t-kawashima at fujitsu.com>
Date: Tue, 1 Jul 2025 20:13:35 +0900
Subject: [PATCH 2/2] Mention the required GLIBC version in help and release
 note

---
 clang/include/clang/Driver/Options.td | 3 ++-
 flang/docs/ReleaseNotes.md            | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..b4b2ab5f916df 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3487,7 +3487,8 @@ def fno_experimental_isel : Flag<["-"], "fno-experimental-isel">, Group<f_clang_
   Alias<fno_global_isel>;
 def fveclib : Joined<["-"], "fveclib=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
-    HelpText<"Use the given vector functions library">,
+    HelpText<"Use the given vector functions library.\n"
+             "  Note: -fveclib=libmvec on AArch64 requires GLIBC 2.40 or newer.">,
     HelpTextForVariants<[ClangOption, CC1Option],
       "Use the given vector functions library.\n"
       "  Note: -fveclib={ArmPL,SLEEF,libmvec} implies -fno-math-errno.\n"
diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index bd3a215323de2..b2dbbcb5630f4 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -34,7 +34,8 @@ page](https://llvm.org/releases/).
 
 * -floop-interchange is now recognized by flang.
 * -floop-interchange is enabled by default at -O2 and above.
-* -fveclib=libmvec is supported for AArch64 (same as Flang/x86 and Clang/AArch64)
+* -fveclib=libmvec is supported for AArch64 (same as Flang/x86 and
+  Clang/AArch64) (requires GLIBC 2.40 or newer)
 
 ## Windows Support
 


From flang-commits at lists.llvm.org  Tue Jul  1 04:27:36 2025
From: flang-commits at lists.llvm.org (KAWASHIMA Takahiro via flang-commits)
Date: Tue, 01 Jul 2025 04:27:36 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6863c628.050a0220.2c8367.16ea@mx.google.com>


================
@@ -34,6 +34,7 @@ page](https://llvm.org/releases/).
 
 * -floop-interchange is now recognized by flang.
 * -floop-interchange is enabled by default at -O2 and above.
+* -fveclib=libmvec is supported for AArch64 (same as Flang/x86 and Clang/AArch64)
----------------
kawashima-fj wrote:

Thanks. I added the GLIBC version information in the help message and the release note.

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 05:34:53 2025
From: flang-commits at lists.llvm.org (Benjamin Maxwell via flang-commits)
Date: Tue, 01 Jul 2025 05:34:53 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6863d5ed.170a0220.232821.9c9d@mx.google.com>


https://github.com/MacDue approved this pull request.

Thanks! LGTM!

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 06:02:05 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:02:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863dc4d.170a0220.243cf1.71e0@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/146423

>From 4cfcf45d8250d648bd4ae0a9110b5034d2495149 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 20 Jun 2025 06:48:18 -0500
Subject: [PATCH 1/3] [flang][OpenMP] Rewrite min/max with more than 2
 arguments

Given an atomic operation `w = max(w, x1, x2, ...)` rewrite it as
`w = max(w, max(x1, x2, ...))`. This will avoid unnecessary non-atomic
comparisons inside of the atomic operation (min/max are expanded inline).

In particular, if some of the x_i's are optional dummy parameters in the
containing function, this will avoid any presence tests within the
atomic operation.

Fixes https://github.com/llvm/llvm-project/issues/144838
---
 flang/lib/Lower/OpenMP/Atomic.cpp             | 279 +++++++++++++++++-
 flang/test/Lower/OpenMP/atomic-update.f90     |  12 +-
 .../Lower/OpenMP/max-optional-parameters.f90  |  24 ++
 3 files changed, 302 insertions(+), 13 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/max-optional-parameters.f90

diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 33a743f8f9dda..b7a223eff80c5 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -11,6 +11,8 @@
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
@@ -41,6 +43,178 @@ namespace omp {
 using namespace Fortran::lower::omp;
 }
 
+namespace {
+// An example of a type that can be used to get the return value from
+// the visitor:
+//   visitor(type_identity<Xyz>) -> result_type
+using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
+
+struct GetProc
+    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
+                                false> {
+  using Result = const evaluate::ProcedureDesignator *;
+  using Base = evaluate::Traverse<GetProc, Result, false>;
+  GetProc() : Base(*this) {}
+
+  using Base::operator();
+
+  static Result Default() { return nullptr; }
+
+  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
+  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
+};
+
+struct WithType {
+  WithType(const evaluate::DynamicType &t) : type(t) {
+    assert(type.category() != common::TypeCategory::Derived &&
+           "Type cannot be a derived type");
+  }
+
+  template <typename VisitorTy> //
+  auto visit(VisitorTy &&visitor) const
+      -> std::invoke_result_t<VisitorTy, SomeArgType> {
+    switch (type.category()) {
+    case common::TypeCategory::Integer:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Unsigned:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Real:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Complex:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Logical:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
+      }
+      break;
+    case common::TypeCategory::Character:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
+      }
+      break;
+    case common::TypeCategory::Derived:
+      break;
+    }
+    llvm_unreachable("Unhandled type");
+  }
+
+  const evaluate::DynamicType &type;
+
+private:
+  // Shorter names.
+  static constexpr auto Character = common::TypeCategory::Character;
+  static constexpr auto Complex = common::TypeCategory::Complex;
+  static constexpr auto Derived = common::TypeCategory::Derived;
+  static constexpr auto Integer = common::TypeCategory::Integer;
+  static constexpr auto Logical = common::TypeCategory::Logical;
+  static constexpr auto Real = common::TypeCategory::Real;
+  static constexpr auto Unsigned = common::TypeCategory::Unsigned;
+};
+
+template <typename T, typename U = std::remove_const_t<T>>
+U AsRvalue(T &t) {
+  U copy{t};
+  return std::move(copy);
+}
+
+template <typename T>
+T &&AsRvalue(T &&t) {
+  return std::move(t);
+}
+
+struct ArgumentReplacer
+    : public evaluate::Traverse<ArgumentReplacer, bool, false> {
+  using Base = evaluate::Traverse<ArgumentReplacer, bool, false>;
+  using Result = bool;
+
+  Result Default() const { return false; }
+
+  ArgumentReplacer(evaluate::ActualArguments &&newArgs)
+      : Base(*this), args_(std::move(newArgs)) {}
+
+  using Base::operator();
+
+  template <typename T>
+  Result operator()(const evaluate::FunctionRef<T> &x) {
+    assert(!done_);
+    auto &mut = const_cast<evaluate::FunctionRef<T> &>(x);
+    mut.arguments() = args_;
+    done_ = true;
+    return true;
+  }
+
+  Result Combine(Result &&a, Result &&b) { return a || b; }
+
+private:
+  bool done_{false};
+  evaluate::ActualArguments &&args_;
+};
+} // namespace
+
 [[maybe_unused]] static void
 dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
   auto whatStr = [](int k) {
@@ -237,6 +411,85 @@ makeMemOrderAttr(lower::AbstractConverter &converter,
   return nullptr;
 }
 
+static bool replaceArgs(semantics::SomeExpr &expr,
+                        evaluate::ActualArguments &&newArgs) {
+  return ArgumentReplacer(std::move(newArgs))(expr);
+}
+
+static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
+                                    const evaluate::ProcedureDesignator &proc,
+                                    const evaluate::ActualArguments &args) {
+  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
+    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
+    return evaluate::AsGenericExpr(
+        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
+  });
+}
+
+static const evaluate::ProcedureDesignator &
+getProcedureDesignator(const semantics::SomeExpr &call) {
+  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
+  assert(proc && "Call has no procedure designator");
+  return *proc;
+}
+
+static semantics::SomeExpr //
+genReducedMinMax(const semantics::SomeExpr &orig,
+                 const semantics::SomeExpr *atomArg,
+                 const std::vector<semantics::SomeExpr> &args) {
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be atom (or a convert of atom).
+  // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
+  // call = min/max(a_t, tmp).
+  // Return "call".
+
+  // The min/max intrinsics have 2 mandatory arguments, the rest is optional.
+  // Make sure that the "tmp = min/max(...)" doesn't promote an optional
+  // argument to a non-optional position. This could happen if a_t is at
+  // position 0 or 1.
+  if (args.size() <= 2)
+    return orig;
+
+  evaluate::ActualArguments nonAtoms;
+
+  auto AsActual = [](const semantics::SomeExpr &x) {
+    semantics::SomeExpr copy = x;
+    return evaluate::ActualArgument(std::move(copy));
+  };
+  // Semantic checks guarantee that the "atom" shows exactly once in the
+  // argument list (with potential conversions around it).
+  // For the first two (non-optional) arguments, if "atom" is among them,
+  // replace it with another occurrence of the other non-optional argument.
+  if (atomArg == &args[0]) {
+    // (atom, x, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[1]));
+    nonAtoms.push_back(AsActual(args[1]));
+  } else if (atomArg == &args[1]) {
+    // (x, atom, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[0]));
+  } else {
+    // (x, y, z...) -> unchanged
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[1]));
+  }
+
+  // The rest of arguments are optional, so we can just skip "atom".
+  for (size_t i = 2, e = args.size(); i != e; ++i) {
+    if (atomArg != &args[i])
+      nonAtoms.push_back(AsActual(args[i]));
+  }
+
+  // The type of the intermediate min/max is the same as the type of its
+  // arguments, which may be different from the type of the original
+  // expression. The original expression may have additional coverts.
+  auto tmp =
+      makeCall(*atomArg->GetType(), getProcedureDesignator(orig), nonAtoms);
+  semantics::SomeExpr call = orig;
+  replaceArgs(call, {AsActual(*atomArg), AsActual(tmp)});
+  return call;
+}
+
 static mlir::Operation * //
 genAtomicRead(lower::AbstractConverter &converter,
               semantics::SemanticsContext &semaCtx, mlir::Location loc,
@@ -350,10 +603,28 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
 
   // This must exist by now.
-  semantics::SomeExpr input = *evaluate::GetConvertInput(assign.rhs);
-  std::vector<semantics::SomeExpr> args =
-      evaluate::GetTopLevelOperation(input).second;
+  semantics::SomeExpr rhs = assign.rhs;
+  semantics::SomeExpr input = *evaluate::GetConvertInput(rhs);
+  auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
+
+  const semantics::SomeExpr *atomArg = [&]() {
+    for (const semantics::SomeExpr &e : args) {
+      if (evaluate::IsSameOrConvertOf(e, atom))
+        return &e;
+    }
+    llvm_unreachable("Atomic variable not in argument list");
+  }();
+
+  if (opcode == evaluate::operation::Operator::Min ||
+      opcode == evaluate::operation::Operator::Max) {
+    // Min and max operations are expanded inline, so reduce them to
+    // operations with exactly two (non-optional) arguments.
+    rhs = genReducedMinMax(rhs, atomArg, args);
+    input = *evaluate::GetConvertInput(rhs);
+    std::tie(opcode, args) = evaluate::GetTopLevelOperation(input);
+    atomArg = nullptr; // No longer valid.
+  }
   for (auto &arg : args) {
     if (!evaluate::IsSameOrConvertOf(arg, atom)) {
       mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
@@ -372,7 +643,7 @@ genAtomicUpdate(lower::AbstractConverter &converter,
 
   converter.overrideExprValues(&overrides);
   mlir::Value updated =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+      fir::getBase(converter.genExprValue(rhs, stmtCtx, &loc));
   mlir::Value converted = builder.createConvert(loc, atomType, updated);
   builder.create<mlir::omp::YieldOp>(loc, converted);
   converter.resetExprOverrides();
diff --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90
index 3f840acefa6e8..f88bbea6fca85 100644
--- a/flang/test/Lower/OpenMP/atomic-update.f90
+++ b/flang/test/Lower/OpenMP/atomic-update.f90
@@ -107,8 +107,6 @@ program OmpAtomicUpdate
 !CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_Y_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK: ^bb0(%[[ARG:.*]]: i32):
 !CHECK:  {{.*}} = arith.cmpi sgt, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.select {{.*}}, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.cmpi sgt, {{.*}}
 !CHECK:  %[[TEMP:.*]] = arith.select {{.*}} : i32
 !CHECK:  omp.yield(%[[TEMP]] : i32)
 !CHECK: }
@@ -177,13 +175,9 @@ program OmpAtomicUpdate
 !CHECK:   %[[VAL_Z_LOADED:.*]] = fir.load %[[VAL_Z_DECLARE]]#0 : !fir.ref<i32>
 !CHECK:   omp.atomic.update %[[VAL_W_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG_W:.*]]: i32):
-!CHECK:     %[[WX_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WX_MIN:.*]] = arith.select %[[WX_CMP]], %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WXY_CMP:.*]] = arith.cmpi sgt, %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXY_MIN:.*]] = arith.select %[[WXY_CMP]], %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXYZ_CMP:.*]] = arith.cmpi sgt, %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     %[[WXYZ_MIN:.*]] = arith.select %[[WXYZ_CMP]], %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     omp.yield(%[[WXYZ_MIN]] : i32)
+!CHECK:     %[[W_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], {{.*}} : i32
+!CHECK:     %[[WXYZ_MAX:.*]] = arith.select %[[W_CMP]], %[[ARG_W]], {{.*}} : i32
+!CHECK:     omp.yield(%[[WXYZ_MAX]] : i32)
 !CHECK:   }
   !$omp atomic update
     w = max(w,x,y,z)
diff --git a/flang/test/Lower/OpenMP/max-optional-parameters.f90 b/flang/test/Lower/OpenMP/max-optional-parameters.f90
new file mode 100644
index 0000000000000..2bb6b110cb0ef
--- /dev/null
+++ b/flang/test/Lower/OpenMP/max-optional-parameters.f90
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
+!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: } else {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: }
+!CHECK: omp.atomic.update {{.*}} : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
+!CHECK:   %[[V10:[a-z0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[IF_VAL]]
+!CHECK:   %[[V11:[a-z0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[IF_VAL]]
+!CHECK:   omp.yield(%[[V11]] : f32)
+!CHECK: }
+
+subroutine f00(a, x, y)
+  real :: a
+  real, optional :: x, y
+  !$omp atomic update
+  a = max(x, a, y)
+end

>From 4302ed81021028baec7427028562802a516ea511 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 07:38:45 -0500
Subject: [PATCH 2/3] Address review comments

---
 flang/lib/Lower/OpenMP/Atomic.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index b7a223eff80c5..2ab91b239a3cc 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -157,6 +157,7 @@ struct WithType {
       }
       break;
     case common::TypeCategory::Derived:
+      (void)Derived;
       break;
     }
     llvm_unreachable("Unhandled type");
@@ -438,7 +439,7 @@ genReducedMinMax(const semantics::SomeExpr &orig,
                  const semantics::SomeExpr *atomArg,
                  const std::vector<semantics::SomeExpr> &args) {
   // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
-  // One of the a_i's, say a_t, must be atom (or a convert of atom).
+  // One of the a_i's, say a_t, must be atomArg.
   // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
   // call = min/max(a_t, tmp).
   // Return "call".
@@ -608,13 +609,14 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
 
-  const semantics::SomeExpr *atomArg = [&]() {
+  // Pass args as an argument to avoid capturing a structured binding.
+  const semantics::SomeExpr *atomArg = [&](auto &args) {
     for (const semantics::SomeExpr &e : args) {
       if (evaluate::IsSameOrConvertOf(e, atom))
         return &e;
     }
     llvm_unreachable("Atomic variable not in argument list");
-  }();
+  }(args);
 
   if (opcode == evaluate::operation::Operator::Min ||
       opcode == evaluate::operation::Operator::Max) {

>From 16df2e0eb775fce156db6f5fc62509026a0b2bc5 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 08:01:23 -0500
Subject: [PATCH 3/3] Check for non-atomic operations, add min

---
 .../Lower/OpenMP/max-optional-parameters.f90  | 58 ++++++++++++++++---
 1 file changed, 51 insertions(+), 7 deletions(-)

diff --git a/flang/test/Lower/OpenMP/max-optional-parameters.f90 b/flang/test/Lower/OpenMP/max-optional-parameters.f90
index 2bb6b110cb0ef..57a432eeeecec 100644
--- a/flang/test/Lower/OpenMP/max-optional-parameters.f90
+++ b/flang/test/Lower/OpenMP/max-optional-parameters.f90
@@ -3,16 +3,27 @@
 ! Check that the presence tests are done outside of the atomic update
 ! construct.
 
-!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
-!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
-!CHECK:   fir.result {{.*}} : f32
+!CHECK-LABEL: func.func @_QPf00
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<f32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpf ogt, %[[V4]], %[[V5]] fastmath<contract> : f32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : f32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (f32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<f32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpf ogt, %[[V8]], %[[V10]] fastmath<contract> : f32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : f32
+!CHECK:   fir.result %[[V12]] : f32
 !CHECK: } else {
-!CHECK:   fir.result {{.*}} : f32
+!CHECK:   fir.result %[[V8]] : f32
 !CHECK: }
-!CHECK: omp.atomic.update {{.*}} : !fir.ref<f32> {
+!CHECK: omp.atomic.update %[[VAL_A]]#0 : !fir.ref<f32> {
 !CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
-!CHECK:   %[[V10:[a-z0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[IF_VAL]]
-!CHECK:   %[[V11:[a-z0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[IF_VAL]]
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[V9]] fastmath<contract> : f32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : f32
 !CHECK:   omp.yield(%[[V11]] : f32)
 !CHECK: }
 
@@ -22,3 +33,36 @@ subroutine f00(a, x, y)
   !$omp atomic update
   a = max(x, a, y)
 end
+
+
+!CHECK-LABEL: func.func @_QPf01
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<i32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpi slt, %[[V4]], %[[V5]] : i32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : i32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (i32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<i32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpi slt, %[[V8]], %[[V10]] : i32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : i32
+!CHECK:   fir.result %[[V12]] : i32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : i32
+!CHECK: }
+!CHECK: omp.atomic.update %[[VAL_A]]#0 : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: i32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpi slt, %[[ARG]], %[[V9]] : i32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : i32
+!CHECK:   omp.yield(%[[V11]] : i32)
+!CHECK: }
+
+subroutine f01(a, x, y)
+  integer :: a
+  integer, optional :: x, y
+  !$omp atomic update
+  a = min(x, a, y)
+end
+


From flang-commits at lists.llvm.org  Tue Jul  1 06:04:04 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:04:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863dcc4.630a0220.1dacd2.a0c6@mx.google.com>


================
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
+!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: } else {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: }
+!CHECK: omp.atomic.update {{.*}} : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
+!CHECK:   %[[V10:[a-z0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[IF_VAL]]
+!CHECK:   %[[V11:[a-z0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[IF_VAL]]
+!CHECK:   omp.yield(%[[V11]] : f32)
+!CHECK: }
+
+subroutine f00(a, x, y)
+  real :: a
+  real, optional :: x, y
+  !$omp atomic update
+  a = max(x, a, y)
----------------
kparzysz wrote:

Done.

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 06:04:04 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:04:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863dcc4.170a0220.123bb4.6bc7@mx.google.com>


================
@@ -350,10 +603,28 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
 
   // This must exist by now.
-  semantics::SomeExpr input = *evaluate::GetConvertInput(assign.rhs);
-  std::vector<semantics::SomeExpr> args =
-      evaluate::GetTopLevelOperation(input).second;
+  semantics::SomeExpr rhs = assign.rhs;
+  semantics::SomeExpr input = *evaluate::GetConvertInput(rhs);
+  auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
+
+  const semantics::SomeExpr *atomArg = [&]() {
+    for (const semantics::SomeExpr &e : args) {
----------------
kparzysz wrote:

Fixed.

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 06:04:04 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:04:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863dcc4.170a0220.33ecb2.9c08@mx.google.com>


================
@@ -41,6 +43,178 @@ namespace omp {
 using namespace Fortran::lower::omp;
 }
 
+namespace {
+// An example of a type that can be used to get the return value from
+// the visitor:
+//   visitor(type_identity<Xyz>) -> result_type
+using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
+
+struct GetProc
+    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
+                                false> {
+  using Result = const evaluate::ProcedureDesignator *;
+  using Base = evaluate::Traverse<GetProc, Result, false>;
+  GetProc() : Base(*this) {}
+
+  using Base::operator();
+
+  static Result Default() { return nullptr; }
+
+  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
+  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
+};
+
+struct WithType {
+  WithType(const evaluate::DynamicType &t) : type(t) {
+    assert(type.category() != common::TypeCategory::Derived &&
+           "Type cannot be a derived type");
+  }
+
+  template <typename VisitorTy> //
+  auto visit(VisitorTy &&visitor) const
+      -> std::invoke_result_t<VisitorTy, SomeArgType> {
+    switch (type.category()) {
+    case common::TypeCategory::Integer:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Unsigned:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Real:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Complex:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Logical:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
+      }
+      break;
+    case common::TypeCategory::Character:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
+      }
+      break;
+    case common::TypeCategory::Derived:
+      break;
+    }
+    llvm_unreachable("Unhandled type");
+  }
+
+  const evaluate::DynamicType &type;
+
+private:
+  // Shorter names.
+  static constexpr auto Character = common::TypeCategory::Character;
+  static constexpr auto Complex = common::TypeCategory::Complex;
+  static constexpr auto Derived = common::TypeCategory::Derived;
----------------
kparzysz wrote:

Fixed.

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 06:04:21 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:04:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863dcd5.630a0220.13c6f5.9977@mx.google.com>


================
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
+!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
+!CHECK:   fir.result {{.*}} : f32
----------------
kparzysz wrote:

Done.

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 06:04:32 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:04:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863dce0.050a0220.27550.1be3@mx.google.com>


================
@@ -237,6 +411,85 @@ makeMemOrderAttr(lower::AbstractConverter &converter,
   return nullptr;
 }
 
+static bool replaceArgs(semantics::SomeExpr &expr,
+                        evaluate::ActualArguments &&newArgs) {
+  return ArgumentReplacer(std::move(newArgs))(expr);
+}
+
+static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
+                                    const evaluate::ProcedureDesignator &proc,
+                                    const evaluate::ActualArguments &args) {
+  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
+    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
+    return evaluate::AsGenericExpr(
+        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
+  });
+}
+
+static const evaluate::ProcedureDesignator &
+getProcedureDesignator(const semantics::SomeExpr &call) {
+  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
+  assert(proc && "Call has no procedure designator");
+  return *proc;
+}
+
+static semantics::SomeExpr //
+genReducedMinMax(const semantics::SomeExpr &orig,
+                 const semantics::SomeExpr *atomArg,
+                 const std::vector<semantics::SomeExpr> &args) {
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be atom (or a convert of atom).
----------------
kparzysz wrote:

Fixed.

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 06:06:44 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Tue, 01 Jul 2025 06:06:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863dd64.170a0220.20bd46.e8c5@mx.google.com>


kiranchandramohan wrote:

I have not gone through this in detail. But why is evaluate processing done during Lowering? Can this canonicalization be performed in the semantics stage?

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 06:49:35 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:49:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863e76f.170a0220.188e3a.ab91@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/146423

>From 4cfcf45d8250d648bd4ae0a9110b5034d2495149 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 20 Jun 2025 06:48:18 -0500
Subject: [PATCH 1/3] [flang][OpenMP] Rewrite min/max with more than 2
 arguments

Given an atomic operation `w = max(w, x1, x2, ...)` rewrite it as
`w = max(w, max(x1, x2, ...))`. This will avoid unnecessary non-atomic
comparisons inside of the atomic operation (min/max are expanded inline).

In particular, if some of the x_i's are optional dummy parameters in the
containing function, this will avoid any presence tests within the
atomic operation.

Fixes https://github.com/llvm/llvm-project/issues/144838
---
 flang/lib/Lower/OpenMP/Atomic.cpp             | 279 +++++++++++++++++-
 flang/test/Lower/OpenMP/atomic-update.f90     |  12 +-
 .../Lower/OpenMP/max-optional-parameters.f90  |  24 ++
 3 files changed, 302 insertions(+), 13 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/max-optional-parameters.f90

diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 33a743f8f9dda..b7a223eff80c5 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -11,6 +11,8 @@
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
@@ -41,6 +43,178 @@ namespace omp {
 using namespace Fortran::lower::omp;
 }
 
+namespace {
+// An example of a type that can be used to get the return value from
+// the visitor:
+//   visitor(type_identity<Xyz>) -> result_type
+using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
+
+struct GetProc
+    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
+                                false> {
+  using Result = const evaluate::ProcedureDesignator *;
+  using Base = evaluate::Traverse<GetProc, Result, false>;
+  GetProc() : Base(*this) {}
+
+  using Base::operator();
+
+  static Result Default() { return nullptr; }
+
+  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
+  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
+};
+
+struct WithType {
+  WithType(const evaluate::DynamicType &t) : type(t) {
+    assert(type.category() != common::TypeCategory::Derived &&
+           "Type cannot be a derived type");
+  }
+
+  template <typename VisitorTy> //
+  auto visit(VisitorTy &&visitor) const
+      -> std::invoke_result_t<VisitorTy, SomeArgType> {
+    switch (type.category()) {
+    case common::TypeCategory::Integer:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Unsigned:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Real:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Complex:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Logical:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
+      }
+      break;
+    case common::TypeCategory::Character:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
+      }
+      break;
+    case common::TypeCategory::Derived:
+      break;
+    }
+    llvm_unreachable("Unhandled type");
+  }
+
+  const evaluate::DynamicType &type;
+
+private:
+  // Shorter names.
+  static constexpr auto Character = common::TypeCategory::Character;
+  static constexpr auto Complex = common::TypeCategory::Complex;
+  static constexpr auto Derived = common::TypeCategory::Derived;
+  static constexpr auto Integer = common::TypeCategory::Integer;
+  static constexpr auto Logical = common::TypeCategory::Logical;
+  static constexpr auto Real = common::TypeCategory::Real;
+  static constexpr auto Unsigned = common::TypeCategory::Unsigned;
+};
+
+template <typename T, typename U = std::remove_const_t<T>>
+U AsRvalue(T &t) {
+  U copy{t};
+  return std::move(copy);
+}
+
+template <typename T>
+T &&AsRvalue(T &&t) {
+  return std::move(t);
+}
+
+struct ArgumentReplacer
+    : public evaluate::Traverse<ArgumentReplacer, bool, false> {
+  using Base = evaluate::Traverse<ArgumentReplacer, bool, false>;
+  using Result = bool;
+
+  Result Default() const { return false; }
+
+  ArgumentReplacer(evaluate::ActualArguments &&newArgs)
+      : Base(*this), args_(std::move(newArgs)) {}
+
+  using Base::operator();
+
+  template <typename T>
+  Result operator()(const evaluate::FunctionRef<T> &x) {
+    assert(!done_);
+    auto &mut = const_cast<evaluate::FunctionRef<T> &>(x);
+    mut.arguments() = args_;
+    done_ = true;
+    return true;
+  }
+
+  Result Combine(Result &&a, Result &&b) { return a || b; }
+
+private:
+  bool done_{false};
+  evaluate::ActualArguments &&args_;
+};
+} // namespace
+
 [[maybe_unused]] static void
 dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
   auto whatStr = [](int k) {
@@ -237,6 +411,85 @@ makeMemOrderAttr(lower::AbstractConverter &converter,
   return nullptr;
 }
 
+static bool replaceArgs(semantics::SomeExpr &expr,
+                        evaluate::ActualArguments &&newArgs) {
+  return ArgumentReplacer(std::move(newArgs))(expr);
+}
+
+static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
+                                    const evaluate::ProcedureDesignator &proc,
+                                    const evaluate::ActualArguments &args) {
+  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
+    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
+    return evaluate::AsGenericExpr(
+        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
+  });
+}
+
+static const evaluate::ProcedureDesignator &
+getProcedureDesignator(const semantics::SomeExpr &call) {
+  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
+  assert(proc && "Call has no procedure designator");
+  return *proc;
+}
+
+static semantics::SomeExpr //
+genReducedMinMax(const semantics::SomeExpr &orig,
+                 const semantics::SomeExpr *atomArg,
+                 const std::vector<semantics::SomeExpr> &args) {
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be atom (or a convert of atom).
+  // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
+  // call = min/max(a_t, tmp).
+  // Return "call".
+
+  // The min/max intrinsics have 2 mandatory arguments, the rest is optional.
+  // Make sure that the "tmp = min/max(...)" doesn't promote an optional
+  // argument to a non-optional position. This could happen if a_t is at
+  // position 0 or 1.
+  if (args.size() <= 2)
+    return orig;
+
+  evaluate::ActualArguments nonAtoms;
+
+  auto AsActual = [](const semantics::SomeExpr &x) {
+    semantics::SomeExpr copy = x;
+    return evaluate::ActualArgument(std::move(copy));
+  };
+  // Semantic checks guarantee that the "atom" shows exactly once in the
+  // argument list (with potential conversions around it).
+  // For the first two (non-optional) arguments, if "atom" is among them,
+  // replace it with another occurrence of the other non-optional argument.
+  if (atomArg == &args[0]) {
+    // (atom, x, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[1]));
+    nonAtoms.push_back(AsActual(args[1]));
+  } else if (atomArg == &args[1]) {
+    // (x, atom, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[0]));
+  } else {
+    // (x, y, z...) -> unchanged
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[1]));
+  }
+
+  // The rest of arguments are optional, so we can just skip "atom".
+  for (size_t i = 2, e = args.size(); i != e; ++i) {
+    if (atomArg != &args[i])
+      nonAtoms.push_back(AsActual(args[i]));
+  }
+
+  // The type of the intermediate min/max is the same as the type of its
+  // arguments, which may be different from the type of the original
+  // expression. The original expression may have additional coverts.
+  auto tmp =
+      makeCall(*atomArg->GetType(), getProcedureDesignator(orig), nonAtoms);
+  semantics::SomeExpr call = orig;
+  replaceArgs(call, {AsActual(*atomArg), AsActual(tmp)});
+  return call;
+}
+
 static mlir::Operation * //
 genAtomicRead(lower::AbstractConverter &converter,
               semantics::SemanticsContext &semaCtx, mlir::Location loc,
@@ -350,10 +603,28 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
 
   // This must exist by now.
-  semantics::SomeExpr input = *evaluate::GetConvertInput(assign.rhs);
-  std::vector<semantics::SomeExpr> args =
-      evaluate::GetTopLevelOperation(input).second;
+  semantics::SomeExpr rhs = assign.rhs;
+  semantics::SomeExpr input = *evaluate::GetConvertInput(rhs);
+  auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
+
+  const semantics::SomeExpr *atomArg = [&]() {
+    for (const semantics::SomeExpr &e : args) {
+      if (evaluate::IsSameOrConvertOf(e, atom))
+        return &e;
+    }
+    llvm_unreachable("Atomic variable not in argument list");
+  }();
+
+  if (opcode == evaluate::operation::Operator::Min ||
+      opcode == evaluate::operation::Operator::Max) {
+    // Min and max operations are expanded inline, so reduce them to
+    // operations with exactly two (non-optional) arguments.
+    rhs = genReducedMinMax(rhs, atomArg, args);
+    input = *evaluate::GetConvertInput(rhs);
+    std::tie(opcode, args) = evaluate::GetTopLevelOperation(input);
+    atomArg = nullptr; // No longer valid.
+  }
   for (auto &arg : args) {
     if (!evaluate::IsSameOrConvertOf(arg, atom)) {
       mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
@@ -372,7 +643,7 @@ genAtomicUpdate(lower::AbstractConverter &converter,
 
   converter.overrideExprValues(&overrides);
   mlir::Value updated =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+      fir::getBase(converter.genExprValue(rhs, stmtCtx, &loc));
   mlir::Value converted = builder.createConvert(loc, atomType, updated);
   builder.create<mlir::omp::YieldOp>(loc, converted);
   converter.resetExprOverrides();
diff --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90
index 3f840acefa6e8..f88bbea6fca85 100644
--- a/flang/test/Lower/OpenMP/atomic-update.f90
+++ b/flang/test/Lower/OpenMP/atomic-update.f90
@@ -107,8 +107,6 @@ program OmpAtomicUpdate
 !CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_Y_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK: ^bb0(%[[ARG:.*]]: i32):
 !CHECK:  {{.*}} = arith.cmpi sgt, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.select {{.*}}, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.cmpi sgt, {{.*}}
 !CHECK:  %[[TEMP:.*]] = arith.select {{.*}} : i32
 !CHECK:  omp.yield(%[[TEMP]] : i32)
 !CHECK: }
@@ -177,13 +175,9 @@ program OmpAtomicUpdate
 !CHECK:   %[[VAL_Z_LOADED:.*]] = fir.load %[[VAL_Z_DECLARE]]#0 : !fir.ref<i32>
 !CHECK:   omp.atomic.update %[[VAL_W_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG_W:.*]]: i32):
-!CHECK:     %[[WX_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WX_MIN:.*]] = arith.select %[[WX_CMP]], %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WXY_CMP:.*]] = arith.cmpi sgt, %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXY_MIN:.*]] = arith.select %[[WXY_CMP]], %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXYZ_CMP:.*]] = arith.cmpi sgt, %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     %[[WXYZ_MIN:.*]] = arith.select %[[WXYZ_CMP]], %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     omp.yield(%[[WXYZ_MIN]] : i32)
+!CHECK:     %[[W_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], {{.*}} : i32
+!CHECK:     %[[WXYZ_MAX:.*]] = arith.select %[[W_CMP]], %[[ARG_W]], {{.*}} : i32
+!CHECK:     omp.yield(%[[WXYZ_MAX]] : i32)
 !CHECK:   }
   !$omp atomic update
     w = max(w,x,y,z)
diff --git a/flang/test/Lower/OpenMP/max-optional-parameters.f90 b/flang/test/Lower/OpenMP/max-optional-parameters.f90
new file mode 100644
index 0000000000000..2bb6b110cb0ef
--- /dev/null
+++ b/flang/test/Lower/OpenMP/max-optional-parameters.f90
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
+!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: } else {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: }
+!CHECK: omp.atomic.update {{.*}} : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
+!CHECK:   %[[V10:[a-z0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[IF_VAL]]
+!CHECK:   %[[V11:[a-z0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[IF_VAL]]
+!CHECK:   omp.yield(%[[V11]] : f32)
+!CHECK: }
+
+subroutine f00(a, x, y)
+  real :: a
+  real, optional :: x, y
+  !$omp atomic update
+  a = max(x, a, y)
+end

>From 4302ed81021028baec7427028562802a516ea511 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 07:38:45 -0500
Subject: [PATCH 2/3] Address review comments

---
 flang/lib/Lower/OpenMP/Atomic.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index b7a223eff80c5..2ab91b239a3cc 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -157,6 +157,7 @@ struct WithType {
       }
       break;
     case common::TypeCategory::Derived:
+      (void)Derived;
       break;
     }
     llvm_unreachable("Unhandled type");
@@ -438,7 +439,7 @@ genReducedMinMax(const semantics::SomeExpr &orig,
                  const semantics::SomeExpr *atomArg,
                  const std::vector<semantics::SomeExpr> &args) {
   // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
-  // One of the a_i's, say a_t, must be atom (or a convert of atom).
+  // One of the a_i's, say a_t, must be atomArg.
   // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
   // call = min/max(a_t, tmp).
   // Return "call".
@@ -608,13 +609,14 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
 
-  const semantics::SomeExpr *atomArg = [&]() {
+  // Pass args as an argument to avoid capturing a structured binding.
+  const semantics::SomeExpr *atomArg = [&](auto &args) {
     for (const semantics::SomeExpr &e : args) {
       if (evaluate::IsSameOrConvertOf(e, atom))
         return &e;
     }
     llvm_unreachable("Atomic variable not in argument list");
-  }();
+  }(args);
 
   if (opcode == evaluate::operation::Operator::Min ||
       opcode == evaluate::operation::Operator::Max) {

>From 16df2e0eb775fce156db6f5fc62509026a0b2bc5 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 08:01:23 -0500
Subject: [PATCH 3/3] Check for non-atomic operations, add min

---
 .../Lower/OpenMP/max-optional-parameters.f90  | 58 ++++++++++++++++---
 1 file changed, 51 insertions(+), 7 deletions(-)

diff --git a/flang/test/Lower/OpenMP/max-optional-parameters.f90 b/flang/test/Lower/OpenMP/max-optional-parameters.f90
index 2bb6b110cb0ef..57a432eeeecec 100644
--- a/flang/test/Lower/OpenMP/max-optional-parameters.f90
+++ b/flang/test/Lower/OpenMP/max-optional-parameters.f90
@@ -3,16 +3,27 @@
 ! Check that the presence tests are done outside of the atomic update
 ! construct.
 
-!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
-!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
-!CHECK:   fir.result {{.*}} : f32
+!CHECK-LABEL: func.func @_QPf00
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<f32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpf ogt, %[[V4]], %[[V5]] fastmath<contract> : f32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : f32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (f32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<f32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpf ogt, %[[V8]], %[[V10]] fastmath<contract> : f32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : f32
+!CHECK:   fir.result %[[V12]] : f32
 !CHECK: } else {
-!CHECK:   fir.result {{.*}} : f32
+!CHECK:   fir.result %[[V8]] : f32
 !CHECK: }
-!CHECK: omp.atomic.update {{.*}} : !fir.ref<f32> {
+!CHECK: omp.atomic.update %[[VAL_A]]#0 : !fir.ref<f32> {
 !CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
-!CHECK:   %[[V10:[a-z0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[IF_VAL]]
-!CHECK:   %[[V11:[a-z0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[IF_VAL]]
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[V9]] fastmath<contract> : f32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : f32
 !CHECK:   omp.yield(%[[V11]] : f32)
 !CHECK: }
 
@@ -22,3 +33,36 @@ subroutine f00(a, x, y)
   !$omp atomic update
   a = max(x, a, y)
 end
+
+
+!CHECK-LABEL: func.func @_QPf01
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<i32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpi slt, %[[V4]], %[[V5]] : i32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : i32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (i32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<i32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpi slt, %[[V8]], %[[V10]] : i32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : i32
+!CHECK:   fir.result %[[V12]] : i32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : i32
+!CHECK: }
+!CHECK: omp.atomic.update %[[VAL_A]]#0 : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: i32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpi slt, %[[ARG]], %[[V9]] : i32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : i32
+!CHECK:   omp.yield(%[[V11]] : i32)
+!CHECK: }
+
+subroutine f01(a, x, y)
+  integer :: a
+  integer, optional :: x, y
+  !$omp atomic update
+  a = min(x, a, y)
+end
+


From flang-commits at lists.llvm.org  Tue Jul  1 06:52:05 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Tue, 01 Jul 2025 06:52:05 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6863e805.170a0220.300898.be3b@mx.google.com>


================
@@ -3487,7 +3487,8 @@ def fno_experimental_isel : Flag<["-"], "fno-experimental-isel">, Group<f_clang_
   Alias<fno_global_isel>;
 def fveclib : Joined<["-"], "fveclib=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
-    HelpText<"Use the given vector functions library">,
+    HelpText<"Use the given vector functions library.\n"
+             "  Note: -fveclib=libmvec on AArch64 requires GLIBC 2.40 or newer.">,
----------------
tarunprabhu wrote:

Is the constraint on `glibc` >=2.40 true for `clang` as well? What happens if the user attempts to build with an older `glibc`? Will they see a configure-time error or will the build fail?

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 06:54:26 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Tue, 01 Jul 2025 06:54:26 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6863e892.170a0220.10297b.b895@mx.google.com>


================
@@ -3487,7 +3487,8 @@ def fno_experimental_isel : Flag<["-"], "fno-experimental-isel">, Group<f_clang_
   Alias<fno_global_isel>;
 def fveclib : Joined<["-"], "fveclib=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
-    HelpText<"Use the given vector functions library">,
+    HelpText<"Use the given vector functions library.\n"
+             "  Note: -fveclib=libmvec on AArch64 requires GLIBC 2.40 or newer.">,
----------------
tarunprabhu wrote:

Ach. Please ignore the question about the constraint on glibc. The second question about failure modes with older glibc still holds though.

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 06:55:56 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:55:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (PR #146359)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146359@github.com>
Message-ID: <6863e8ec.170a0220.38aae.b5cd@mx.google.com>


kparzysz wrote:

I'm not sure about the compile time.  I'm guessing that with PCH enabled this shouldn't have any significant impact, but I'll try to see if I can get some measurements.

https://github.com/llvm/llvm-project/pull/146359

From flang-commits at lists.llvm.org  Tue Jul  1 06:58:52 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 06:58:52 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146532@github.com>
Message-ID: <6863e99c.170a0220.2bcad7.0f4c@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-fir-hlfir

@llvm/pr-subscribers-mlir

Author: Abid Qadeer (abidh)

<details>
<summary>Changes</summary>

This is combination of https://github.com/llvm/llvm-project/pull/138149 and https://github.com/llvm/llvm-project/pull/138039 which were opened separately for ease of reviewing. Only other change is adjustments in 2 tests which have gone in since. 

There are `DeclareOp` present for the variables mapped into target region. That  allow us to generate debug information for them. But the `TargetOp` is still part of parent function and those variables get the parent function's `DISubprogram` as a scope.
    
In `OMPIRBuilder`, a new function is created for the `TargetOp`. We also create a new `DISubprogram` for it. All the variables that were in the target region now have to be updated to have the correct scope. This after the fact updating of
debug information becomes very difficult in certain cases. Take the example of variable arrays. The type of those arrays depend on the artificial `DILocalVariable`(s) which hold the size(s) of the array. This new function will now require that we generate the new variable and and new types. Similar issue exist for character type variables too.
    
To avoid this after the fact updating, this PR generates a `DISubprogramAttr` for the `TargetOp` while generating the debug info in `flang`. Then we don't need to generate a `DISubprogram` in `OMPIRBuilder`.  This change is made a bit more complicated by the the fact that in new scheme, the debug location already points to the new `DISubprogram` by     the time it reaches `convertOmpTarget`. But we need some code generation in the parent function so we have to carefully manage the debug locations.
    
This fixes issue `#<!-- -->134991`.

---

Patch is 35.38 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/146532.diff


14 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/AddDebugInfo.cpp (+116-1) 
- (added) flang/test/Transforms/debug-omp-target-op-1.fir (+40) 
- (added) flang/test/Transforms/debug-omp-target-op-2.fir (+53) 
- (modified) llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp (+8-38) 
- (modified) mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp (+21) 
- (added) mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir (+27) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir (+3-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir (+5-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir (+4-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir (+8-5) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir (+8-5) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug.mlir (+6-2) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug2.mlir (+6-2) 
- (modified) mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir (+6-2) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 8fa2f38818c02..6eb914e67fd54 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -35,6 +35,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -104,6 +105,37 @@ bool debugInfoIsAlreadySet(mlir::Location loc) {
   return false;
 }
 
+// Generates the name for the artificial DISubprogram that we are going to
+// generate for omp::TargetOp. Its logic is borrowed from
+// getTargetEntryUniqueInfo and
+// TargetRegionEntryInfo::getTargetRegionEntryFnName to generate the same name.
+// But even if there was a slight mismatch, it is not a problem because this
+// name is artificial and not important to debug experience.
+mlir::StringAttr getTargetFunctionName(mlir::MLIRContext *context,
+                                       mlir::Location Loc,
+                                       llvm::StringRef parentName) {
+  auto fileLoc = Loc->findInstanceOf<mlir::FileLineColLoc>();
+
+  assert(fileLoc && "No file found from location");
+  llvm::StringRef fileName = fileLoc.getFilename().getValue();
+
+  llvm::sys::fs::UniqueID id;
+  uint64_t line = fileLoc.getLine();
+  size_t fileId;
+  size_t deviceId;
+  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {
+    fileId = llvm::hash_value(fileName.str());
+    deviceId = 0xdeadf17e;
+  } else {
+    fileId = id.getFile();
+    deviceId = id.getDevice();
+  }
+  return mlir::StringAttr::get(
+      context,
+      std::string(llvm::formatv("__omp_offloading_{0:x-}_{1:x-}_{2}_l{3}",
+                                deviceId, fileId, parentName, line)));
+}
+
 } // namespace
 
 bool AddDebugInfoPass::createCommonBlockGlobal(
@@ -446,6 +478,79 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
                                   line - 1, false);
   }
 
+  auto addTargetOpDISP = [&](bool lineTableOnly,
+                             llvm::ArrayRef<mlir::LLVM::DINodeAttr> entities) {
+    // When we process the DeclareOp inside the OpenMP target region, all the
+    // variables get the DISubprogram of the parent function of the target op as
+    // the scope. In the codegen (to llvm ir), OpenMP target op results in the
+    // creation of a separate function. As the variables in the debug info have
+    // the DISubprogram of the parent function as the scope, the variables
+    // need to be updated at codegen time to avoid verification failures.
+
+    // This updating after the fact becomes more and more difficult when types
+    // are dependent on local variables like in the case of variable size arrays
+    // or string. We not only have to generate new variables but also new types.
+    // We can avoid this problem by generating a DISubprogramAttr here for the
+    // target op and make sure that all the variables inside the target region
+    // get the correct scope in the first place.
+    funcOp.walk([&](mlir::omp::TargetOp targetOp) {
+      unsigned line = getLineFromLoc(targetOp.getLoc());
+      mlir::StringAttr name =
+          getTargetFunctionName(context, targetOp.getLoc(), funcOp.getName());
+      mlir::LLVM::DISubprogramFlags flags =
+          mlir::LLVM::DISubprogramFlags::Definition |
+          mlir::LLVM::DISubprogramFlags::LocalToUnit;
+      if (isOptimized)
+        flags = flags | mlir::LLVM::DISubprogramFlags::Optimized;
+
+      mlir::DistinctAttr id =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
+      types.push_back(mlir::LLVM::DINullTypeAttr::get(context));
+      for (auto arg : targetOp.getRegion().getArguments()) {
+        auto tyAttr = typeGen.convertType(fir::unwrapRefType(arg.getType()),
+                                          fileAttr, cuAttr, /*declOp=*/nullptr);
+        types.push_back(tyAttr);
+      }
+      CC = llvm::dwarf::getCallingConvention("DW_CC_normal");
+      mlir::LLVM::DISubroutineTypeAttr spTy =
+          mlir::LLVM::DISubroutineTypeAttr::get(context, CC, types);
+      if (lineTableOnly) {
+        auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+            context, id, compilationUnit, Scope, name, name, funcFileAttr, line,
+            line, flags, spTy, /*retainedNodes=*/{}, /*annotations=*/{});
+        targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+        return;
+      }
+      mlir::DistinctAttr recId =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/true, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, /*retainedNodes=*/{},
+          /*annotations=*/{});
+
+      // Make sure that information about the imported modules is copied in the
+      // new function.
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> opEntities;
+      for (mlir::LLVM::DINodeAttr N : entities) {
+        if (auto entity = mlir::dyn_cast<mlir::LLVM::DIImportedEntityAttr>(N)) {
+          auto importedEntity = mlir::LLVM::DIImportedEntityAttr::get(
+              context, llvm::dwarf::DW_TAG_imported_module, spAttr,
+              entity.getEntity(), fileAttr, /*line=*/1, /*name=*/nullptr,
+              /*elements*/ {});
+          opEntities.push_back(importedEntity);
+        }
+      }
+
+      id = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/false, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, opEntities,
+          /*annotations=*/{});
+      targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+    });
+  };
+
   // Don't process variables if user asked for line tables only.
   if (debugLevel == mlir::LLVM::DIEmissionKind::LineTablesOnly) {
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
@@ -453,6 +558,7 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
         line, line, subprogramFlags, subTypeAttr, /*retainedNodes=*/{},
         /*annotations=*/{});
     funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+    addTargetOpDISP(/*lineTableOnly=*/true, /*entities=*/{});
     return;
   }
 
@@ -510,9 +616,18 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
       funcName, fullName, funcFileAttr, line, line, subprogramFlags,
       subTypeAttr, entities, /*annotations=*/{});
   funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+  addTargetOpDISP(/*lineTableOnly=*/false, entities);
 
   funcOp.walk([&](fir::cg::XDeclareOp declOp) {
-    handleDeclareOp(declOp, fileAttr, spAttr, typeGen, symbolTable);
+    mlir::LLVM::DISubprogramAttr spTy = spAttr;
+    if (auto tOp = declOp->getParentOfType<mlir::omp::TargetOp>()) {
+      if (auto fusedLoc = llvm::dyn_cast<mlir::FusedLoc>(tOp.getLoc())) {
+        if (auto sp = llvm::dyn_cast<mlir::LLVM::DISubprogramAttr>(
+                fusedLoc.getMetadata()))
+          spTy = sp;
+      }
+    }
+    handleDeclareOp(declOp, fileAttr, spTy, typeGen, symbolTable);
   });
   // commonBlockMap ensures that we don't create multiple DICommonBlockAttr of
   // the same name in one function. But it is ok (rather required) to create
diff --git a/flang/test/Transforms/debug-omp-target-op-1.fir b/flang/test/Transforms/debug-omp-target-op-1.fir
new file mode 100644
index 0000000000000..6b895b732c42b
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-1.fir
@@ -0,0 +1,40 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info="debug-level=LineTablesOnly" --mlir-print-debuginfo %s | FileCheck %s --check-prefix=LINETABLE
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %c13_i32 = arith.constant 13 : i32
+    %c12_i32 = arith.constant 12 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c1_i32 = arith.constant 1 : i32
+    %c5_i32 = arith.constant 5 : i32
+    %0 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} loc(#loc1)
+    %1 = fircg.ext_declare %0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc1)
+    %2 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} loc(#loc2)
+    %3 = fircg.ext_declare %2 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc2)
+    %4 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "x"}
+    %5 = omp.map.info var_ptr(%3 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "y"}
+    omp.target map_entries(%4 -> %arg0, %5 -> %arg1 : !fir.ref<i32>, !fir.ref<i32>) {
+      %16 = fircg.ext_declare %arg0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc3)
+      %17 = fircg.ext_declare %arg1 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc4)
+      omp.terminator
+    } loc(#loc5)
+    return
+  }
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// CHECK: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "x"{{.*}}line = 1, type = #[[TY:.*]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "y"{{.*}}line = 3, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "x"{{.*}}line = 7, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "y"{{.*}}line = 8, type = #[[TY]]>
+
+// LINETABLE: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// LINETABLE: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// LINETABLE-NOT: #llvm.di_local_variable
diff --git a/flang/test/Transforms/debug-omp-target-op-2.fir b/flang/test/Transforms/debug-omp-target-op-2.fir
new file mode 100644
index 0000000000000..15dcf2389b21d
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-2.fir
@@ -0,0 +1,53 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @fn_(%arg0: !fir.ref<!fir.array<?x?xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+    %c1 = arith.constant 1 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.alloca i32
+    %1 = fir.alloca i32
+    %2 = fir.undefined !fir.dscope
+    %3 = fircg.ext_declare %arg1 dummy_scope %2 {uniq_name = "_QFfnEc"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc2)
+    %4 = fircg.ext_declare %arg2 dummy_scope %2 {uniq_name = "_QFfnEd"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc3)
+    %5 = fir.load %3 : !fir.ref<i32>
+    %6 = fir.convert %5 : (i32) -> index
+    %9 = fir.load %4 : !fir.ref<i32>
+    %10 = fir.convert %9 : (i32) -> index
+    %15 = fircg.ext_declare %arg0(%6, %10) dummy_scope %2 {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc4)
+    %16 = fircg.ext_embox %15(%6, %10) : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.box<!fir.array<?x?xi32>>
+    %17:3 = fir.box_dims %16, %c0 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %18 = arith.subi %17#1, %c1 : index
+    %19 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%18 : index) extent(%17#1 : index) stride(%17#2 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %20 = arith.muli %17#2, %17#1 : index
+    %21:3 = fir.box_dims %16, %c1 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %22 = arith.subi %21#1, %c1 : index
+    %23 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%22 : index) extent(%21#1 : index) stride(%20 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %24 = omp.map.info var_ptr(%15 : !fir.ref<!fir.array<?x?xi32>>, i32) map_clauses(tofrom) capture(ByRef) bounds(%19, %23) -> !fir.ref<!fir.array<?x?xi32>> {name = "b"}
+    %25 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    %26 = omp.map.info var_ptr(%0 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    omp.target map_entries(%24 -> %arg3, %25 -> %arg4, %26 -> %arg5 : !fir.ref<!fir.array<?x?xi32>>, !fir.ref<i32>, !fir.ref<i32>) {
+      %27 = fir.load %arg5 : !fir.ref<i32>
+      %28 = fir.load %arg4 : !fir.ref<i32>
+      %29 = fir.convert %27 : (i32) -> index
+      %31 = fir.convert %28 : (i32) -> index
+      %37 = fircg.ext_declare %arg3(%29, %31) {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc5)
+      omp.terminator
+    } loc(#loc6)
+    return
+  } loc(#loc7)
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+#loc6 = loc("test.f90":16:1)
+#loc7 = loc("test.f90":26:1)
+
+
+// Test that variable size arrays inside target regions get their own
+// compiler generated variables for size.
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_fn__l16"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb1"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb2"{{.*}}>
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 85451b1233f96..db792a3b52d24 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6891,23 +6891,19 @@ static void FixupDebugInfoForOutlinedFunction(
   if (!NewSP)
     return;
 
-  DenseMap<const MDNode *, MDNode *> Cache;
   SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
 
   auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
-    auto NewSP = Func->getSubprogram();
     DILocalVariable *&NewVar = RemappedVariables[OldVar];
     // Only use cached variable if the arg number matches. This is important
     // so that DIVariable created for privatized variables are not discarded.
     if (NewVar && (arg == NewVar->getArg()))
       return NewVar;
 
-    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
     NewVar = llvm::DILocalVariable::get(
-        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
-        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
-        OldVar->getAlignInBits(), OldVar->getAnnotations());
+        Builder.getContext(), OldVar->getScope(), OldVar->getName(),
+        OldVar->getFile(), OldVar->getLine(), OldVar->getType(), arg,
+        OldVar->getFlags(), OldVar->getAlignInBits(), OldVar->getAnnotations());
     return NewVar;
   };
 
@@ -6921,7 +6917,8 @@ static void FixupDebugInfoForOutlinedFunction(
         ArgNo = std::get<1>(Iter->second) + 1;
       }
     }
-    DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
+    if (ArgNo != 0)
+      DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
   };
 
   // The location and scope of variable intrinsics and records still point to
@@ -7000,36 +6997,9 @@ static Expected<Function *> createOutlinedFunction(
 
   // Save insert point.
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  // If there's a DISubprogram associated with current function, then
-  // generate one for the outlined function.
-  if (Function *ParentFunc = BB->getParent()) {
-    if (DISubprogram *SP = ParentFunc->getSubprogram()) {
-      DICompileUnit *CU = SP->getUnit();
-      DIBuilder DB(*M, true, CU);
-      DebugLoc DL = Builder.getCurrentDebugLocation();
-      if (DL) {
-        // TODO: We are using nullopt for arguments at the moment. This will
-        // need to be updated when debug data is being generated for variables.
-        DISubroutineType *Ty =
-            DB.createSubroutineType(DB.getOrCreateTypeArray({}));
-        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
-                                          DISubprogram::SPFlagOptimized |
-                                          DISubprogram::SPFlagLocalToUnit;
-
-        DISubprogram *OutlinedSP = DB.createFunction(
-            CU, FuncName, FuncName, SP->getFile(), DL.getLine(), Ty,
-            DL.getLine(), DINode::DIFlags::FlagArtificial, SPFlags);
-
-        // Attach subprogram to the function.
-        Func->setSubprogram(OutlinedSP);
-        // Update the CurrentDebugLocation in the builder so that right scope
-        // is used for things inside outlined function.
-        Builder.SetCurrentDebugLocation(
-            DILocation::get(Func->getContext(), DL.getLine(), DL.getCol(),
-                            OutlinedSP, DL.getInlinedAt()));
-      }
-    }
-  }
+  // We will generate the entries in the outlined function but the debug
+  // location may still be pointing to the parent function. Reset it now.
+  Builder.SetCurrentDebugLocation(llvm::DebugLoc());
 
   // Generate the region into the function.
   BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 3806db3ceab25..c1e1fec3ddef8 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5324,9 +5324,27 @@ static LogicalResult
 convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto targetOp = cast<omp::TargetOp>(opInst);
+  // The current debug location already has the DISubprogram for the outlined
+  // function that will be created for the target op. We save it here so that
+  // we can set it on the outlined function.
+  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
   if (failed(checkImplementationStatus(opInst)))
     return failure();
 
+  // During the handling of target op, we will generate instructions in the
+  // parent function like call to the oulined function or branch to a new
+  // BasicBlock. We set the debug location here to parent function so that those
+  // get the correct debug locations. For outlined functions, the normal MLIR op
+  // conversion will automatically pick the correct location.
+  llvm::BasicBlock *parentBB = builder.GetInsertBlock();
+  assert(parentBB && "No insert block is set for the builder");
+  llvm::Function *parentLLVMFn = parentBB->getParent();
+  assert(parentLLVMFn && "Parent Function must be valid");
+  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
+    builder.SetCurrentDebugLocation(llvm::DILocation::get(
+        parentLLVMFn->getContext(), outlinedFnLoc.getLine(),
+        outlinedFnLoc.getCol(), SP, outlinedFnLoc.getInlinedAt()));
+
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   bool isTargetDevice = ompBuilder->Config.isTargetDevice();
   bool isGPU = ompBuilder->Config.isGPU();
@@ -5420,6 +5438,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     assert(llvmParentFn && llvmOutlinedFn &&
            "Both parent and outlined functions must exist at this point");
 
+    if (outlinedFnLoc && llvmParentFn->getSubprogram())
+      llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());
+
     if (auto attr = llvmParentFn->getFnAttribute("target-cpu");
         attr.isStringAttribute())
       llvmOutlinedFn->addFnAttr(attr);
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
new file mode 100644
index 0000000000000..45e5d2612e2c2
---...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/146532

From flang-commits at lists.llvm.org  Tue Jul  1 06:58:52 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 06:58:52 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146532@github.com>
Message-ID: <6863e99c.050a0220.1eae93.49cf@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-mlir-llvm

Author: Abid Qadeer (abidh)

<details>
<summary>Changes</summary>

This is combination of https://github.com/llvm/llvm-project/pull/138149 and https://github.com/llvm/llvm-project/pull/138039 which were opened separately for ease of reviewing. Only other change is adjustments in 2 tests which have gone in since. 

There are `DeclareOp` present for the variables mapped into target region. That  allow us to generate debug information for them. But the `TargetOp` is still part of parent function and those variables get the parent function's `DISubprogram` as a scope.
    
In `OMPIRBuilder`, a new function is created for the `TargetOp`. We also create a new `DISubprogram` for it. All the variables that were in the target region now have to be updated to have the correct scope. This after the fact updating of
debug information becomes very difficult in certain cases. Take the example of variable arrays. The type of those arrays depend on the artificial `DILocalVariable`(s) which hold the size(s) of the array. This new function will now require that we generate the new variable and and new types. Similar issue exist for character type variables too.
    
To avoid this after the fact updating, this PR generates a `DISubprogramAttr` for the `TargetOp` while generating the debug info in `flang`. Then we don't need to generate a `DISubprogram` in `OMPIRBuilder`.  This change is made a bit more complicated by the the fact that in new scheme, the debug location already points to the new `DISubprogram` by     the time it reaches `convertOmpTarget`. But we need some code generation in the parent function so we have to carefully manage the debug locations.
    
This fixes issue `#<!-- -->134991`.

---

Patch is 35.38 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/146532.diff


14 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/AddDebugInfo.cpp (+116-1) 
- (added) flang/test/Transforms/debug-omp-target-op-1.fir (+40) 
- (added) flang/test/Transforms/debug-omp-target-op-2.fir (+53) 
- (modified) llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp (+8-38) 
- (modified) mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp (+21) 
- (added) mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir (+27) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir (+3-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir (+5-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir (+4-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir (+8-5) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir (+8-5) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug.mlir (+6-2) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug2.mlir (+6-2) 
- (modified) mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir (+6-2) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 8fa2f38818c02..6eb914e67fd54 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -35,6 +35,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -104,6 +105,37 @@ bool debugInfoIsAlreadySet(mlir::Location loc) {
   return false;
 }
 
+// Generates the name for the artificial DISubprogram that we are going to
+// generate for omp::TargetOp. Its logic is borrowed from
+// getTargetEntryUniqueInfo and
+// TargetRegionEntryInfo::getTargetRegionEntryFnName to generate the same name.
+// But even if there was a slight mismatch, it is not a problem because this
+// name is artificial and not important to debug experience.
+mlir::StringAttr getTargetFunctionName(mlir::MLIRContext *context,
+                                       mlir::Location Loc,
+                                       llvm::StringRef parentName) {
+  auto fileLoc = Loc->findInstanceOf<mlir::FileLineColLoc>();
+
+  assert(fileLoc && "No file found from location");
+  llvm::StringRef fileName = fileLoc.getFilename().getValue();
+
+  llvm::sys::fs::UniqueID id;
+  uint64_t line = fileLoc.getLine();
+  size_t fileId;
+  size_t deviceId;
+  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {
+    fileId = llvm::hash_value(fileName.str());
+    deviceId = 0xdeadf17e;
+  } else {
+    fileId = id.getFile();
+    deviceId = id.getDevice();
+  }
+  return mlir::StringAttr::get(
+      context,
+      std::string(llvm::formatv("__omp_offloading_{0:x-}_{1:x-}_{2}_l{3}",
+                                deviceId, fileId, parentName, line)));
+}
+
 } // namespace
 
 bool AddDebugInfoPass::createCommonBlockGlobal(
@@ -446,6 +478,79 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
                                   line - 1, false);
   }
 
+  auto addTargetOpDISP = [&](bool lineTableOnly,
+                             llvm::ArrayRef<mlir::LLVM::DINodeAttr> entities) {
+    // When we process the DeclareOp inside the OpenMP target region, all the
+    // variables get the DISubprogram of the parent function of the target op as
+    // the scope. In the codegen (to llvm ir), OpenMP target op results in the
+    // creation of a separate function. As the variables in the debug info have
+    // the DISubprogram of the parent function as the scope, the variables
+    // need to be updated at codegen time to avoid verification failures.
+
+    // This updating after the fact becomes more and more difficult when types
+    // are dependent on local variables like in the case of variable size arrays
+    // or string. We not only have to generate new variables but also new types.
+    // We can avoid this problem by generating a DISubprogramAttr here for the
+    // target op and make sure that all the variables inside the target region
+    // get the correct scope in the first place.
+    funcOp.walk([&](mlir::omp::TargetOp targetOp) {
+      unsigned line = getLineFromLoc(targetOp.getLoc());
+      mlir::StringAttr name =
+          getTargetFunctionName(context, targetOp.getLoc(), funcOp.getName());
+      mlir::LLVM::DISubprogramFlags flags =
+          mlir::LLVM::DISubprogramFlags::Definition |
+          mlir::LLVM::DISubprogramFlags::LocalToUnit;
+      if (isOptimized)
+        flags = flags | mlir::LLVM::DISubprogramFlags::Optimized;
+
+      mlir::DistinctAttr id =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
+      types.push_back(mlir::LLVM::DINullTypeAttr::get(context));
+      for (auto arg : targetOp.getRegion().getArguments()) {
+        auto tyAttr = typeGen.convertType(fir::unwrapRefType(arg.getType()),
+                                          fileAttr, cuAttr, /*declOp=*/nullptr);
+        types.push_back(tyAttr);
+      }
+      CC = llvm::dwarf::getCallingConvention("DW_CC_normal");
+      mlir::LLVM::DISubroutineTypeAttr spTy =
+          mlir::LLVM::DISubroutineTypeAttr::get(context, CC, types);
+      if (lineTableOnly) {
+        auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+            context, id, compilationUnit, Scope, name, name, funcFileAttr, line,
+            line, flags, spTy, /*retainedNodes=*/{}, /*annotations=*/{});
+        targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+        return;
+      }
+      mlir::DistinctAttr recId =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/true, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, /*retainedNodes=*/{},
+          /*annotations=*/{});
+
+      // Make sure that information about the imported modules is copied in the
+      // new function.
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> opEntities;
+      for (mlir::LLVM::DINodeAttr N : entities) {
+        if (auto entity = mlir::dyn_cast<mlir::LLVM::DIImportedEntityAttr>(N)) {
+          auto importedEntity = mlir::LLVM::DIImportedEntityAttr::get(
+              context, llvm::dwarf::DW_TAG_imported_module, spAttr,
+              entity.getEntity(), fileAttr, /*line=*/1, /*name=*/nullptr,
+              /*elements*/ {});
+          opEntities.push_back(importedEntity);
+        }
+      }
+
+      id = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/false, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, opEntities,
+          /*annotations=*/{});
+      targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+    });
+  };
+
   // Don't process variables if user asked for line tables only.
   if (debugLevel == mlir::LLVM::DIEmissionKind::LineTablesOnly) {
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
@@ -453,6 +558,7 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
         line, line, subprogramFlags, subTypeAttr, /*retainedNodes=*/{},
         /*annotations=*/{});
     funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+    addTargetOpDISP(/*lineTableOnly=*/true, /*entities=*/{});
     return;
   }
 
@@ -510,9 +616,18 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
       funcName, fullName, funcFileAttr, line, line, subprogramFlags,
       subTypeAttr, entities, /*annotations=*/{});
   funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+  addTargetOpDISP(/*lineTableOnly=*/false, entities);
 
   funcOp.walk([&](fir::cg::XDeclareOp declOp) {
-    handleDeclareOp(declOp, fileAttr, spAttr, typeGen, symbolTable);
+    mlir::LLVM::DISubprogramAttr spTy = spAttr;
+    if (auto tOp = declOp->getParentOfType<mlir::omp::TargetOp>()) {
+      if (auto fusedLoc = llvm::dyn_cast<mlir::FusedLoc>(tOp.getLoc())) {
+        if (auto sp = llvm::dyn_cast<mlir::LLVM::DISubprogramAttr>(
+                fusedLoc.getMetadata()))
+          spTy = sp;
+      }
+    }
+    handleDeclareOp(declOp, fileAttr, spTy, typeGen, symbolTable);
   });
   // commonBlockMap ensures that we don't create multiple DICommonBlockAttr of
   // the same name in one function. But it is ok (rather required) to create
diff --git a/flang/test/Transforms/debug-omp-target-op-1.fir b/flang/test/Transforms/debug-omp-target-op-1.fir
new file mode 100644
index 0000000000000..6b895b732c42b
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-1.fir
@@ -0,0 +1,40 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info="debug-level=LineTablesOnly" --mlir-print-debuginfo %s | FileCheck %s --check-prefix=LINETABLE
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %c13_i32 = arith.constant 13 : i32
+    %c12_i32 = arith.constant 12 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c1_i32 = arith.constant 1 : i32
+    %c5_i32 = arith.constant 5 : i32
+    %0 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} loc(#loc1)
+    %1 = fircg.ext_declare %0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc1)
+    %2 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} loc(#loc2)
+    %3 = fircg.ext_declare %2 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc2)
+    %4 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "x"}
+    %5 = omp.map.info var_ptr(%3 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "y"}
+    omp.target map_entries(%4 -> %arg0, %5 -> %arg1 : !fir.ref<i32>, !fir.ref<i32>) {
+      %16 = fircg.ext_declare %arg0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc3)
+      %17 = fircg.ext_declare %arg1 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc4)
+      omp.terminator
+    } loc(#loc5)
+    return
+  }
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// CHECK: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "x"{{.*}}line = 1, type = #[[TY:.*]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "y"{{.*}}line = 3, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "x"{{.*}}line = 7, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "y"{{.*}}line = 8, type = #[[TY]]>
+
+// LINETABLE: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// LINETABLE: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// LINETABLE-NOT: #llvm.di_local_variable
diff --git a/flang/test/Transforms/debug-omp-target-op-2.fir b/flang/test/Transforms/debug-omp-target-op-2.fir
new file mode 100644
index 0000000000000..15dcf2389b21d
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-2.fir
@@ -0,0 +1,53 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @fn_(%arg0: !fir.ref<!fir.array<?x?xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+    %c1 = arith.constant 1 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.alloca i32
+    %1 = fir.alloca i32
+    %2 = fir.undefined !fir.dscope
+    %3 = fircg.ext_declare %arg1 dummy_scope %2 {uniq_name = "_QFfnEc"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc2)
+    %4 = fircg.ext_declare %arg2 dummy_scope %2 {uniq_name = "_QFfnEd"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc3)
+    %5 = fir.load %3 : !fir.ref<i32>
+    %6 = fir.convert %5 : (i32) -> index
+    %9 = fir.load %4 : !fir.ref<i32>
+    %10 = fir.convert %9 : (i32) -> index
+    %15 = fircg.ext_declare %arg0(%6, %10) dummy_scope %2 {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc4)
+    %16 = fircg.ext_embox %15(%6, %10) : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.box<!fir.array<?x?xi32>>
+    %17:3 = fir.box_dims %16, %c0 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %18 = arith.subi %17#1, %c1 : index
+    %19 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%18 : index) extent(%17#1 : index) stride(%17#2 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %20 = arith.muli %17#2, %17#1 : index
+    %21:3 = fir.box_dims %16, %c1 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %22 = arith.subi %21#1, %c1 : index
+    %23 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%22 : index) extent(%21#1 : index) stride(%20 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %24 = omp.map.info var_ptr(%15 : !fir.ref<!fir.array<?x?xi32>>, i32) map_clauses(tofrom) capture(ByRef) bounds(%19, %23) -> !fir.ref<!fir.array<?x?xi32>> {name = "b"}
+    %25 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    %26 = omp.map.info var_ptr(%0 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    omp.target map_entries(%24 -> %arg3, %25 -> %arg4, %26 -> %arg5 : !fir.ref<!fir.array<?x?xi32>>, !fir.ref<i32>, !fir.ref<i32>) {
+      %27 = fir.load %arg5 : !fir.ref<i32>
+      %28 = fir.load %arg4 : !fir.ref<i32>
+      %29 = fir.convert %27 : (i32) -> index
+      %31 = fir.convert %28 : (i32) -> index
+      %37 = fircg.ext_declare %arg3(%29, %31) {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc5)
+      omp.terminator
+    } loc(#loc6)
+    return
+  } loc(#loc7)
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+#loc6 = loc("test.f90":16:1)
+#loc7 = loc("test.f90":26:1)
+
+
+// Test that variable size arrays inside target regions get their own
+// compiler generated variables for size.
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_fn__l16"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb1"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb2"{{.*}}>
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 85451b1233f96..db792a3b52d24 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6891,23 +6891,19 @@ static void FixupDebugInfoForOutlinedFunction(
   if (!NewSP)
     return;
 
-  DenseMap<const MDNode *, MDNode *> Cache;
   SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
 
   auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
-    auto NewSP = Func->getSubprogram();
     DILocalVariable *&NewVar = RemappedVariables[OldVar];
     // Only use cached variable if the arg number matches. This is important
     // so that DIVariable created for privatized variables are not discarded.
     if (NewVar && (arg == NewVar->getArg()))
       return NewVar;
 
-    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
     NewVar = llvm::DILocalVariable::get(
-        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
-        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
-        OldVar->getAlignInBits(), OldVar->getAnnotations());
+        Builder.getContext(), OldVar->getScope(), OldVar->getName(),
+        OldVar->getFile(), OldVar->getLine(), OldVar->getType(), arg,
+        OldVar->getFlags(), OldVar->getAlignInBits(), OldVar->getAnnotations());
     return NewVar;
   };
 
@@ -6921,7 +6917,8 @@ static void FixupDebugInfoForOutlinedFunction(
         ArgNo = std::get<1>(Iter->second) + 1;
       }
     }
-    DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
+    if (ArgNo != 0)
+      DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
   };
 
   // The location and scope of variable intrinsics and records still point to
@@ -7000,36 +6997,9 @@ static Expected<Function *> createOutlinedFunction(
 
   // Save insert point.
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  // If there's a DISubprogram associated with current function, then
-  // generate one for the outlined function.
-  if (Function *ParentFunc = BB->getParent()) {
-    if (DISubprogram *SP = ParentFunc->getSubprogram()) {
-      DICompileUnit *CU = SP->getUnit();
-      DIBuilder DB(*M, true, CU);
-      DebugLoc DL = Builder.getCurrentDebugLocation();
-      if (DL) {
-        // TODO: We are using nullopt for arguments at the moment. This will
-        // need to be updated when debug data is being generated for variables.
-        DISubroutineType *Ty =
-            DB.createSubroutineType(DB.getOrCreateTypeArray({}));
-        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
-                                          DISubprogram::SPFlagOptimized |
-                                          DISubprogram::SPFlagLocalToUnit;
-
-        DISubprogram *OutlinedSP = DB.createFunction(
-            CU, FuncName, FuncName, SP->getFile(), DL.getLine(), Ty,
-            DL.getLine(), DINode::DIFlags::FlagArtificial, SPFlags);
-
-        // Attach subprogram to the function.
-        Func->setSubprogram(OutlinedSP);
-        // Update the CurrentDebugLocation in the builder so that right scope
-        // is used for things inside outlined function.
-        Builder.SetCurrentDebugLocation(
-            DILocation::get(Func->getContext(), DL.getLine(), DL.getCol(),
-                            OutlinedSP, DL.getInlinedAt()));
-      }
-    }
-  }
+  // We will generate the entries in the outlined function but the debug
+  // location may still be pointing to the parent function. Reset it now.
+  Builder.SetCurrentDebugLocation(llvm::DebugLoc());
 
   // Generate the region into the function.
   BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 3806db3ceab25..c1e1fec3ddef8 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5324,9 +5324,27 @@ static LogicalResult
 convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto targetOp = cast<omp::TargetOp>(opInst);
+  // The current debug location already has the DISubprogram for the outlined
+  // function that will be created for the target op. We save it here so that
+  // we can set it on the outlined function.
+  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
   if (failed(checkImplementationStatus(opInst)))
     return failure();
 
+  // During the handling of target op, we will generate instructions in the
+  // parent function like call to the oulined function or branch to a new
+  // BasicBlock. We set the debug location here to parent function so that those
+  // get the correct debug locations. For outlined functions, the normal MLIR op
+  // conversion will automatically pick the correct location.
+  llvm::BasicBlock *parentBB = builder.GetInsertBlock();
+  assert(parentBB && "No insert block is set for the builder");
+  llvm::Function *parentLLVMFn = parentBB->getParent();
+  assert(parentLLVMFn && "Parent Function must be valid");
+  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
+    builder.SetCurrentDebugLocation(llvm::DILocation::get(
+        parentLLVMFn->getContext(), outlinedFnLoc.getLine(),
+        outlinedFnLoc.getCol(), SP, outlinedFnLoc.getInlinedAt()));
+
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   bool isTargetDevice = ompBuilder->Config.isTargetDevice();
   bool isGPU = ompBuilder->Config.isGPU();
@@ -5420,6 +5438,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     assert(llvmParentFn && llvmOutlinedFn &&
            "Both parent and outlined functions must exist at this point");
 
+    if (outlinedFnLoc && llvmParentFn->getSubprogram())
+      llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());
+
     if (auto attr = llvmParentFn->getFnAttribute("target-cpu");
         attr.isStringAttribute())
       llvmOutlinedFn->addFnAttr(attr);
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
new file mode 100644
index 0000000000000..45e5d2612e2c2
---...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/146532

From flang-commits at lists.llvm.org  Tue Jul  1 06:58:52 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 06:58:52 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146532@github.com>
Message-ID: <6863e99c.170a0220.1a32be.06de@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: Abid Qadeer (abidh)

<details>
<summary>Changes</summary>

This is combination of https://github.com/llvm/llvm-project/pull/138149 and https://github.com/llvm/llvm-project/pull/138039 which were opened separately for ease of reviewing. Only other change is adjustments in 2 tests which have gone in since. 

There are `DeclareOp` present for the variables mapped into target region. That  allow us to generate debug information for them. But the `TargetOp` is still part of parent function and those variables get the parent function's `DISubprogram` as a scope.
    
In `OMPIRBuilder`, a new function is created for the `TargetOp`. We also create a new `DISubprogram` for it. All the variables that were in the target region now have to be updated to have the correct scope. This after the fact updating of
debug information becomes very difficult in certain cases. Take the example of variable arrays. The type of those arrays depend on the artificial `DILocalVariable`(s) which hold the size(s) of the array. This new function will now require that we generate the new variable and and new types. Similar issue exist for character type variables too.
    
To avoid this after the fact updating, this PR generates a `DISubprogramAttr` for the `TargetOp` while generating the debug info in `flang`. Then we don't need to generate a `DISubprogram` in `OMPIRBuilder`.  This change is made a bit more complicated by the the fact that in new scheme, the debug location already points to the new `DISubprogram` by     the time it reaches `convertOmpTarget`. But we need some code generation in the parent function so we have to carefully manage the debug locations.
    
This fixes issue `#<!-- -->134991`.

---

Patch is 35.38 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/146532.diff


14 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/AddDebugInfo.cpp (+116-1) 
- (added) flang/test/Transforms/debug-omp-target-op-1.fir (+40) 
- (added) flang/test/Transforms/debug-omp-target-op-2.fir (+53) 
- (modified) llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp (+8-38) 
- (modified) mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp (+21) 
- (added) mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir (+27) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir (+3-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir (+5-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir (+4-1) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir (+8-5) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir (+8-5) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug.mlir (+6-2) 
- (modified) mlir/test/Target/LLVMIR/omptarget-debug2.mlir (+6-2) 
- (modified) mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir (+6-2) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 8fa2f38818c02..6eb914e67fd54 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -35,6 +35,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -104,6 +105,37 @@ bool debugInfoIsAlreadySet(mlir::Location loc) {
   return false;
 }
 
+// Generates the name for the artificial DISubprogram that we are going to
+// generate for omp::TargetOp. Its logic is borrowed from
+// getTargetEntryUniqueInfo and
+// TargetRegionEntryInfo::getTargetRegionEntryFnName to generate the same name.
+// But even if there was a slight mismatch, it is not a problem because this
+// name is artificial and not important to debug experience.
+mlir::StringAttr getTargetFunctionName(mlir::MLIRContext *context,
+                                       mlir::Location Loc,
+                                       llvm::StringRef parentName) {
+  auto fileLoc = Loc->findInstanceOf<mlir::FileLineColLoc>();
+
+  assert(fileLoc && "No file found from location");
+  llvm::StringRef fileName = fileLoc.getFilename().getValue();
+
+  llvm::sys::fs::UniqueID id;
+  uint64_t line = fileLoc.getLine();
+  size_t fileId;
+  size_t deviceId;
+  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {
+    fileId = llvm::hash_value(fileName.str());
+    deviceId = 0xdeadf17e;
+  } else {
+    fileId = id.getFile();
+    deviceId = id.getDevice();
+  }
+  return mlir::StringAttr::get(
+      context,
+      std::string(llvm::formatv("__omp_offloading_{0:x-}_{1:x-}_{2}_l{3}",
+                                deviceId, fileId, parentName, line)));
+}
+
 } // namespace
 
 bool AddDebugInfoPass::createCommonBlockGlobal(
@@ -446,6 +478,79 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
                                   line - 1, false);
   }
 
+  auto addTargetOpDISP = [&](bool lineTableOnly,
+                             llvm::ArrayRef<mlir::LLVM::DINodeAttr> entities) {
+    // When we process the DeclareOp inside the OpenMP target region, all the
+    // variables get the DISubprogram of the parent function of the target op as
+    // the scope. In the codegen (to llvm ir), OpenMP target op results in the
+    // creation of a separate function. As the variables in the debug info have
+    // the DISubprogram of the parent function as the scope, the variables
+    // need to be updated at codegen time to avoid verification failures.
+
+    // This updating after the fact becomes more and more difficult when types
+    // are dependent on local variables like in the case of variable size arrays
+    // or string. We not only have to generate new variables but also new types.
+    // We can avoid this problem by generating a DISubprogramAttr here for the
+    // target op and make sure that all the variables inside the target region
+    // get the correct scope in the first place.
+    funcOp.walk([&](mlir::omp::TargetOp targetOp) {
+      unsigned line = getLineFromLoc(targetOp.getLoc());
+      mlir::StringAttr name =
+          getTargetFunctionName(context, targetOp.getLoc(), funcOp.getName());
+      mlir::LLVM::DISubprogramFlags flags =
+          mlir::LLVM::DISubprogramFlags::Definition |
+          mlir::LLVM::DISubprogramFlags::LocalToUnit;
+      if (isOptimized)
+        flags = flags | mlir::LLVM::DISubprogramFlags::Optimized;
+
+      mlir::DistinctAttr id =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
+      types.push_back(mlir::LLVM::DINullTypeAttr::get(context));
+      for (auto arg : targetOp.getRegion().getArguments()) {
+        auto tyAttr = typeGen.convertType(fir::unwrapRefType(arg.getType()),
+                                          fileAttr, cuAttr, /*declOp=*/nullptr);
+        types.push_back(tyAttr);
+      }
+      CC = llvm::dwarf::getCallingConvention("DW_CC_normal");
+      mlir::LLVM::DISubroutineTypeAttr spTy =
+          mlir::LLVM::DISubroutineTypeAttr::get(context, CC, types);
+      if (lineTableOnly) {
+        auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+            context, id, compilationUnit, Scope, name, name, funcFileAttr, line,
+            line, flags, spTy, /*retainedNodes=*/{}, /*annotations=*/{});
+        targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+        return;
+      }
+      mlir::DistinctAttr recId =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/true, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, /*retainedNodes=*/{},
+          /*annotations=*/{});
+
+      // Make sure that information about the imported modules is copied in the
+      // new function.
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> opEntities;
+      for (mlir::LLVM::DINodeAttr N : entities) {
+        if (auto entity = mlir::dyn_cast<mlir::LLVM::DIImportedEntityAttr>(N)) {
+          auto importedEntity = mlir::LLVM::DIImportedEntityAttr::get(
+              context, llvm::dwarf::DW_TAG_imported_module, spAttr,
+              entity.getEntity(), fileAttr, /*line=*/1, /*name=*/nullptr,
+              /*elements*/ {});
+          opEntities.push_back(importedEntity);
+        }
+      }
+
+      id = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/false, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, opEntities,
+          /*annotations=*/{});
+      targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+    });
+  };
+
   // Don't process variables if user asked for line tables only.
   if (debugLevel == mlir::LLVM::DIEmissionKind::LineTablesOnly) {
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
@@ -453,6 +558,7 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
         line, line, subprogramFlags, subTypeAttr, /*retainedNodes=*/{},
         /*annotations=*/{});
     funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+    addTargetOpDISP(/*lineTableOnly=*/true, /*entities=*/{});
     return;
   }
 
@@ -510,9 +616,18 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
       funcName, fullName, funcFileAttr, line, line, subprogramFlags,
       subTypeAttr, entities, /*annotations=*/{});
   funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+  addTargetOpDISP(/*lineTableOnly=*/false, entities);
 
   funcOp.walk([&](fir::cg::XDeclareOp declOp) {
-    handleDeclareOp(declOp, fileAttr, spAttr, typeGen, symbolTable);
+    mlir::LLVM::DISubprogramAttr spTy = spAttr;
+    if (auto tOp = declOp->getParentOfType<mlir::omp::TargetOp>()) {
+      if (auto fusedLoc = llvm::dyn_cast<mlir::FusedLoc>(tOp.getLoc())) {
+        if (auto sp = llvm::dyn_cast<mlir::LLVM::DISubprogramAttr>(
+                fusedLoc.getMetadata()))
+          spTy = sp;
+      }
+    }
+    handleDeclareOp(declOp, fileAttr, spTy, typeGen, symbolTable);
   });
   // commonBlockMap ensures that we don't create multiple DICommonBlockAttr of
   // the same name in one function. But it is ok (rather required) to create
diff --git a/flang/test/Transforms/debug-omp-target-op-1.fir b/flang/test/Transforms/debug-omp-target-op-1.fir
new file mode 100644
index 0000000000000..6b895b732c42b
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-1.fir
@@ -0,0 +1,40 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info="debug-level=LineTablesOnly" --mlir-print-debuginfo %s | FileCheck %s --check-prefix=LINETABLE
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %c13_i32 = arith.constant 13 : i32
+    %c12_i32 = arith.constant 12 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c1_i32 = arith.constant 1 : i32
+    %c5_i32 = arith.constant 5 : i32
+    %0 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} loc(#loc1)
+    %1 = fircg.ext_declare %0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc1)
+    %2 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} loc(#loc2)
+    %3 = fircg.ext_declare %2 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc2)
+    %4 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "x"}
+    %5 = omp.map.info var_ptr(%3 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "y"}
+    omp.target map_entries(%4 -> %arg0, %5 -> %arg1 : !fir.ref<i32>, !fir.ref<i32>) {
+      %16 = fircg.ext_declare %arg0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc3)
+      %17 = fircg.ext_declare %arg1 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc4)
+      omp.terminator
+    } loc(#loc5)
+    return
+  }
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// CHECK: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "x"{{.*}}line = 1, type = #[[TY:.*]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "y"{{.*}}line = 3, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "x"{{.*}}line = 7, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "y"{{.*}}line = 8, type = #[[TY]]>
+
+// LINETABLE: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// LINETABLE: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// LINETABLE-NOT: #llvm.di_local_variable
diff --git a/flang/test/Transforms/debug-omp-target-op-2.fir b/flang/test/Transforms/debug-omp-target-op-2.fir
new file mode 100644
index 0000000000000..15dcf2389b21d
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-2.fir
@@ -0,0 +1,53 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @fn_(%arg0: !fir.ref<!fir.array<?x?xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+    %c1 = arith.constant 1 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.alloca i32
+    %1 = fir.alloca i32
+    %2 = fir.undefined !fir.dscope
+    %3 = fircg.ext_declare %arg1 dummy_scope %2 {uniq_name = "_QFfnEc"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc2)
+    %4 = fircg.ext_declare %arg2 dummy_scope %2 {uniq_name = "_QFfnEd"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc3)
+    %5 = fir.load %3 : !fir.ref<i32>
+    %6 = fir.convert %5 : (i32) -> index
+    %9 = fir.load %4 : !fir.ref<i32>
+    %10 = fir.convert %9 : (i32) -> index
+    %15 = fircg.ext_declare %arg0(%6, %10) dummy_scope %2 {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc4)
+    %16 = fircg.ext_embox %15(%6, %10) : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.box<!fir.array<?x?xi32>>
+    %17:3 = fir.box_dims %16, %c0 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %18 = arith.subi %17#1, %c1 : index
+    %19 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%18 : index) extent(%17#1 : index) stride(%17#2 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %20 = arith.muli %17#2, %17#1 : index
+    %21:3 = fir.box_dims %16, %c1 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %22 = arith.subi %21#1, %c1 : index
+    %23 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%22 : index) extent(%21#1 : index) stride(%20 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %24 = omp.map.info var_ptr(%15 : !fir.ref<!fir.array<?x?xi32>>, i32) map_clauses(tofrom) capture(ByRef) bounds(%19, %23) -> !fir.ref<!fir.array<?x?xi32>> {name = "b"}
+    %25 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    %26 = omp.map.info var_ptr(%0 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    omp.target map_entries(%24 -> %arg3, %25 -> %arg4, %26 -> %arg5 : !fir.ref<!fir.array<?x?xi32>>, !fir.ref<i32>, !fir.ref<i32>) {
+      %27 = fir.load %arg5 : !fir.ref<i32>
+      %28 = fir.load %arg4 : !fir.ref<i32>
+      %29 = fir.convert %27 : (i32) -> index
+      %31 = fir.convert %28 : (i32) -> index
+      %37 = fircg.ext_declare %arg3(%29, %31) {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc5)
+      omp.terminator
+    } loc(#loc6)
+    return
+  } loc(#loc7)
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+#loc6 = loc("test.f90":16:1)
+#loc7 = loc("test.f90":26:1)
+
+
+// Test that variable size arrays inside target regions get their own
+// compiler generated variables for size.
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_fn__l16"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb1"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb2"{{.*}}>
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 85451b1233f96..db792a3b52d24 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6891,23 +6891,19 @@ static void FixupDebugInfoForOutlinedFunction(
   if (!NewSP)
     return;
 
-  DenseMap<const MDNode *, MDNode *> Cache;
   SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
 
   auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
-    auto NewSP = Func->getSubprogram();
     DILocalVariable *&NewVar = RemappedVariables[OldVar];
     // Only use cached variable if the arg number matches. This is important
     // so that DIVariable created for privatized variables are not discarded.
     if (NewVar && (arg == NewVar->getArg()))
       return NewVar;
 
-    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
     NewVar = llvm::DILocalVariable::get(
-        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
-        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
-        OldVar->getAlignInBits(), OldVar->getAnnotations());
+        Builder.getContext(), OldVar->getScope(), OldVar->getName(),
+        OldVar->getFile(), OldVar->getLine(), OldVar->getType(), arg,
+        OldVar->getFlags(), OldVar->getAlignInBits(), OldVar->getAnnotations());
     return NewVar;
   };
 
@@ -6921,7 +6917,8 @@ static void FixupDebugInfoForOutlinedFunction(
         ArgNo = std::get<1>(Iter->second) + 1;
       }
     }
-    DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
+    if (ArgNo != 0)
+      DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
   };
 
   // The location and scope of variable intrinsics and records still point to
@@ -7000,36 +6997,9 @@ static Expected<Function *> createOutlinedFunction(
 
   // Save insert point.
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  // If there's a DISubprogram associated with current function, then
-  // generate one for the outlined function.
-  if (Function *ParentFunc = BB->getParent()) {
-    if (DISubprogram *SP = ParentFunc->getSubprogram()) {
-      DICompileUnit *CU = SP->getUnit();
-      DIBuilder DB(*M, true, CU);
-      DebugLoc DL = Builder.getCurrentDebugLocation();
-      if (DL) {
-        // TODO: We are using nullopt for arguments at the moment. This will
-        // need to be updated when debug data is being generated for variables.
-        DISubroutineType *Ty =
-            DB.createSubroutineType(DB.getOrCreateTypeArray({}));
-        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
-                                          DISubprogram::SPFlagOptimized |
-                                          DISubprogram::SPFlagLocalToUnit;
-
-        DISubprogram *OutlinedSP = DB.createFunction(
-            CU, FuncName, FuncName, SP->getFile(), DL.getLine(), Ty,
-            DL.getLine(), DINode::DIFlags::FlagArtificial, SPFlags);
-
-        // Attach subprogram to the function.
-        Func->setSubprogram(OutlinedSP);
-        // Update the CurrentDebugLocation in the builder so that right scope
-        // is used for things inside outlined function.
-        Builder.SetCurrentDebugLocation(
-            DILocation::get(Func->getContext(), DL.getLine(), DL.getCol(),
-                            OutlinedSP, DL.getInlinedAt()));
-      }
-    }
-  }
+  // We will generate the entries in the outlined function but the debug
+  // location may still be pointing to the parent function. Reset it now.
+  Builder.SetCurrentDebugLocation(llvm::DebugLoc());
 
   // Generate the region into the function.
   BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 3806db3ceab25..c1e1fec3ddef8 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5324,9 +5324,27 @@ static LogicalResult
 convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto targetOp = cast<omp::TargetOp>(opInst);
+  // The current debug location already has the DISubprogram for the outlined
+  // function that will be created for the target op. We save it here so that
+  // we can set it on the outlined function.
+  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
   if (failed(checkImplementationStatus(opInst)))
     return failure();
 
+  // During the handling of target op, we will generate instructions in the
+  // parent function like call to the oulined function or branch to a new
+  // BasicBlock. We set the debug location here to parent function so that those
+  // get the correct debug locations. For outlined functions, the normal MLIR op
+  // conversion will automatically pick the correct location.
+  llvm::BasicBlock *parentBB = builder.GetInsertBlock();
+  assert(parentBB && "No insert block is set for the builder");
+  llvm::Function *parentLLVMFn = parentBB->getParent();
+  assert(parentLLVMFn && "Parent Function must be valid");
+  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
+    builder.SetCurrentDebugLocation(llvm::DILocation::get(
+        parentLLVMFn->getContext(), outlinedFnLoc.getLine(),
+        outlinedFnLoc.getCol(), SP, outlinedFnLoc.getInlinedAt()));
+
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   bool isTargetDevice = ompBuilder->Config.isTargetDevice();
   bool isGPU = ompBuilder->Config.isGPU();
@@ -5420,6 +5438,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     assert(llvmParentFn && llvmOutlinedFn &&
            "Both parent and outlined functions must exist at this point");
 
+    if (outlinedFnLoc && llvmParentFn->getSubprogram())
+      llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());
+
     if (auto attr = llvmParentFn->getFnAttribute("target-cpu");
         attr.isStringAttribute())
       llvmOutlinedFn->addFnAttr(attr);
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
new file mode 100644
index 0000000000000..45e5d2612e2c2
---...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/146532

From flang-commits at lists.llvm.org  Tue Jul  1 07:00:53 2025
From: flang-commits at lists.llvm.org (Paul Walker via flang-commits)
Date: Tue, 01 Jul 2025 07:00:53 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6863ea15.050a0220.3c7d74.670f@mx.google.com>


================
@@ -3487,7 +3487,8 @@ def fno_experimental_isel : Flag<["-"], "fno-experimental-isel">, Group<f_clang_
   Alias<fno_global_isel>;
 def fveclib : Joined<["-"], "fveclib=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
-    HelpText<"Use the given vector functions library">,
+    HelpText<"Use the given vector functions library.\n"
+             "  Note: -fveclib=libmvec on AArch64 requires GLIBC 2.40 or newer.">,
----------------
paulwalker-arm wrote:

The toolchain user will see a link time failure when trying to resolve a call to a vector math routine. Note: the exact requirement depends on the function in question, but version 2.40 is what was used to decide which math functions to add mappings for.

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 07:01:01 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Tue, 01 Jul 2025 07:01:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][debug] Generate DISubprogramAttr
 for omp::TargetOp. (PR #138039)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/138039@github.com>
Message-ID: <6863ea1d.050a0220.122345.6a74@mx.google.com>


https://github.com/abidh closed https://github.com/llvm/llvm-project/pull/138039

From flang-commits at lists.llvm.org  Tue Jul  1 07:01:03 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Tue, 01 Jul 2025 07:01:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][debug] Generate DISubprogramAttr
 for omp::TargetOp. (PR #138039)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/138039@github.com>
Message-ID: <6863ea1f.170a0220.f9435.c8da@mx.google.com>


abidh wrote:

I have opened https://github.com/llvm/llvm-project/pull/146532 which is combination of https://github.com/llvm/llvm-project/pull/138039 and https://github.com/llvm/llvm-project/pull/138149. Closing it.

https://github.com/llvm/llvm-project/pull/138039

From flang-commits at lists.llvm.org  Tue Jul  1 07:01:48 2025
From: flang-commits at lists.llvm.org (Paul Walker via flang-commits)
Date: Tue, 01 Jul 2025 07:01:48 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6863ea4c.170a0220.dc008.c25e@mx.google.com>


https://github.com/paulwalker-arm edited https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Tue Jul  1 07:03:42 2025
From: flang-commits at lists.llvm.org (Kajetan Puchalski via flang-commits)
Date: Tue, 01 Jul 2025 07:03:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add -emit-final-mlir flag (PR
	#146533)
Message-ID: <llvm.org/llvm/llvm-project/pull/146533@github.com>


https://github.com/mrkajetanp created https://github.com/llvm/llvm-project/pull/146533

Add a flag to tco for emitting the final MLIR, prior to lowering to LLVM IR. This is intended to produce output that can be passed directly to mlir-translate.

>From 072a3cfd4fdfcb43f7cb292879b6daf5f8540de5 Mon Sep 17 00:00:00 2001
From: Kajetan Puchalski <kajetan.puchalski at arm.com>
Date: Tue, 1 Jul 2025 13:56:43 +0000
Subject: [PATCH] [flang][tco] Add -emit-final-mlir flag

Add a flag to tco for emitting the final MLIR, prior to lowering to LLVM
IR. This is intended to produce output that can be passed directly to
mlir-translate.

Signed-off-by: Kajetan Puchalski <kajetan.puchalski at arm.com>
---
 flang/tools/tco/tco.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/flang/tools/tco/tco.cpp b/flang/tools/tco/tco.cpp
index eaf4bae088454..806cd3148d09f 100644
--- a/flang/tools/tco/tco.cpp
+++ b/flang/tools/tco/tco.cpp
@@ -70,6 +70,11 @@ static cl::opt<bool> codeGenLLVM(
     cl::desc("Run only CodeGen passes and translate FIR to LLVM IR"),
     cl::init(false));
 
+static cl::opt<bool> emitFinalMLIR(
+    "emit-final-mlir",
+    cl::desc("Only translate FIR to MLIR, do not lower to LLVM IR"),
+    cl::init(false));
+
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
 
@@ -149,13 +154,15 @@ compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
       fir::registerDefaultInlinerPass(config);
       fir::createMLIRToLLVMPassPipeline(pm, config);
     }
-    fir::addLLVMDialectToLLVMPass(pm, out.os());
+    if (!emitFinalMLIR)
+      fir::addLLVMDialectToLLVMPass(pm, out.os());
   }
 
   // run the pass manager
   if (mlir::succeeded(pm.run(*owningRef))) {
     // passes ran successfully, so keep the output
-    if ((emitFir || passPipeline.hasAnyOccurrences()) && !codeGenLLVM)
+    if ((emitFir || passPipeline.hasAnyOccurrences() || emitFinalMLIR) &&
+        !codeGenLLVM)
       printModule(*owningRef, out.os());
     out.keep();
     return mlir::success();


From flang-commits at lists.llvm.org  Tue Jul  1 07:05:08 2025
From: flang-commits at lists.llvm.org (Kajetan Puchalski via flang-commits)
Date: Tue, 01 Jul 2025 07:05:08 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add -emit-final-mlir flag (PR
	#146533)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146533@github.com>
Message-ID: <6863eb14.050a0220.20a40c.4c6f@mx.google.com>


mrkajetanp wrote:

I can also rename to just `-emit-mlir` for brevity, but I wasn't sure whether that'd be confusing or not on account of it all technically being mlir.

https://github.com/llvm/llvm-project/pull/146533

From flang-commits at lists.llvm.org  Tue Jul  1 07:08:37 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Tue, 01 Jul 2025 07:08:37 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add ComplexTOROCDL pass (PR
	#144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <6863ebe5.170a0220.2e613c.b97a@mx.google.com>


TIFitis wrote:

Polite reminder for review, thanks.

https://github.com/llvm/llvm-project/pull/144926

From flang-commits at lists.llvm.org  Tue Jul  1 07:09:55 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Tue, 01 Jul 2025 07:09:55 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP] Add TargetAMDGPU support for
 Complex argument and return types (PR #144924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144924@github.com>
Message-ID: <6863ec33.050a0220.15ac0f.5414@mx.google.com>


TIFitis wrote:

Polite reminder for review, thanks.

https://github.com/llvm/llvm-project/pull/144924

From flang-commits at lists.llvm.org  Tue Jul  1 07:11:33 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 07:11:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add -emit-final-mlir flag (PR
	#146533)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146533@github.com>
Message-ID: <6863ec95.630a0220.86e02.9f13@mx.google.com>


https://github.com/tblah commented:

Great to see this, I'm sure I won't be the only one who can save time not having to dump mlir after every pass. I think the name is good.

Please could you add a test that uses this flag and checks that what it gets is LLVM dialect MLIR.

I'll add some other reviewers who often work on LLVMIR codegen as they may also find this useful.

https://github.com/llvm/llvm-project/pull/146533

From flang-commits at lists.llvm.org  Tue Jul  1 07:14:25 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 07:14:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863ed41.170a0220.3008c9.d840@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/146423

>From 4cfcf45d8250d648bd4ae0a9110b5034d2495149 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 20 Jun 2025 06:48:18 -0500
Subject: [PATCH 1/5] [flang][OpenMP] Rewrite min/max with more than 2
 arguments

Given an atomic operation `w = max(w, x1, x2, ...)` rewrite it as
`w = max(w, max(x1, x2, ...))`. This will avoid unnecessary non-atomic
comparisons inside of the atomic operation (min/max are expanded inline).

In particular, if some of the x_i's are optional dummy parameters in the
containing function, this will avoid any presence tests within the
atomic operation.

Fixes https://github.com/llvm/llvm-project/issues/144838
---
 flang/lib/Lower/OpenMP/Atomic.cpp             | 279 +++++++++++++++++-
 flang/test/Lower/OpenMP/atomic-update.f90     |  12 +-
 .../Lower/OpenMP/max-optional-parameters.f90  |  24 ++
 3 files changed, 302 insertions(+), 13 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/max-optional-parameters.f90

diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 33a743f8f9dda..b7a223eff80c5 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -11,6 +11,8 @@
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
@@ -41,6 +43,178 @@ namespace omp {
 using namespace Fortran::lower::omp;
 }
 
+namespace {
+// An example of a type that can be used to get the return value from
+// the visitor:
+//   visitor(type_identity<Xyz>) -> result_type
+using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
+
+struct GetProc
+    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
+                                false> {
+  using Result = const evaluate::ProcedureDesignator *;
+  using Base = evaluate::Traverse<GetProc, Result, false>;
+  GetProc() : Base(*this) {}
+
+  using Base::operator();
+
+  static Result Default() { return nullptr; }
+
+  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
+  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
+};
+
+struct WithType {
+  WithType(const evaluate::DynamicType &t) : type(t) {
+    assert(type.category() != common::TypeCategory::Derived &&
+           "Type cannot be a derived type");
+  }
+
+  template <typename VisitorTy> //
+  auto visit(VisitorTy &&visitor) const
+      -> std::invoke_result_t<VisitorTy, SomeArgType> {
+    switch (type.category()) {
+    case common::TypeCategory::Integer:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Unsigned:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Real:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Complex:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Logical:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
+      }
+      break;
+    case common::TypeCategory::Character:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
+      }
+      break;
+    case common::TypeCategory::Derived:
+      break;
+    }
+    llvm_unreachable("Unhandled type");
+  }
+
+  const evaluate::DynamicType &type;
+
+private:
+  // Shorter names.
+  static constexpr auto Character = common::TypeCategory::Character;
+  static constexpr auto Complex = common::TypeCategory::Complex;
+  static constexpr auto Derived = common::TypeCategory::Derived;
+  static constexpr auto Integer = common::TypeCategory::Integer;
+  static constexpr auto Logical = common::TypeCategory::Logical;
+  static constexpr auto Real = common::TypeCategory::Real;
+  static constexpr auto Unsigned = common::TypeCategory::Unsigned;
+};
+
+template <typename T, typename U = std::remove_const_t<T>>
+U AsRvalue(T &t) {
+  U copy{t};
+  return std::move(copy);
+}
+
+template <typename T>
+T &&AsRvalue(T &&t) {
+  return std::move(t);
+}
+
+struct ArgumentReplacer
+    : public evaluate::Traverse<ArgumentReplacer, bool, false> {
+  using Base = evaluate::Traverse<ArgumentReplacer, bool, false>;
+  using Result = bool;
+
+  Result Default() const { return false; }
+
+  ArgumentReplacer(evaluate::ActualArguments &&newArgs)
+      : Base(*this), args_(std::move(newArgs)) {}
+
+  using Base::operator();
+
+  template <typename T>
+  Result operator()(const evaluate::FunctionRef<T> &x) {
+    assert(!done_);
+    auto &mut = const_cast<evaluate::FunctionRef<T> &>(x);
+    mut.arguments() = args_;
+    done_ = true;
+    return true;
+  }
+
+  Result Combine(Result &&a, Result &&b) { return a || b; }
+
+private:
+  bool done_{false};
+  evaluate::ActualArguments &&args_;
+};
+} // namespace
+
 [[maybe_unused]] static void
 dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
   auto whatStr = [](int k) {
@@ -237,6 +411,85 @@ makeMemOrderAttr(lower::AbstractConverter &converter,
   return nullptr;
 }
 
+static bool replaceArgs(semantics::SomeExpr &expr,
+                        evaluate::ActualArguments &&newArgs) {
+  return ArgumentReplacer(std::move(newArgs))(expr);
+}
+
+static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
+                                    const evaluate::ProcedureDesignator &proc,
+                                    const evaluate::ActualArguments &args) {
+  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
+    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
+    return evaluate::AsGenericExpr(
+        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
+  });
+}
+
+static const evaluate::ProcedureDesignator &
+getProcedureDesignator(const semantics::SomeExpr &call) {
+  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
+  assert(proc && "Call has no procedure designator");
+  return *proc;
+}
+
+static semantics::SomeExpr //
+genReducedMinMax(const semantics::SomeExpr &orig,
+                 const semantics::SomeExpr *atomArg,
+                 const std::vector<semantics::SomeExpr> &args) {
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be atom (or a convert of atom).
+  // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
+  // call = min/max(a_t, tmp).
+  // Return "call".
+
+  // The min/max intrinsics have 2 mandatory arguments, the rest is optional.
+  // Make sure that the "tmp = min/max(...)" doesn't promote an optional
+  // argument to a non-optional position. This could happen if a_t is at
+  // position 0 or 1.
+  if (args.size() <= 2)
+    return orig;
+
+  evaluate::ActualArguments nonAtoms;
+
+  auto AsActual = [](const semantics::SomeExpr &x) {
+    semantics::SomeExpr copy = x;
+    return evaluate::ActualArgument(std::move(copy));
+  };
+  // Semantic checks guarantee that the "atom" shows exactly once in the
+  // argument list (with potential conversions around it).
+  // For the first two (non-optional) arguments, if "atom" is among them,
+  // replace it with another occurrence of the other non-optional argument.
+  if (atomArg == &args[0]) {
+    // (atom, x, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[1]));
+    nonAtoms.push_back(AsActual(args[1]));
+  } else if (atomArg == &args[1]) {
+    // (x, atom, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[0]));
+  } else {
+    // (x, y, z...) -> unchanged
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[1]));
+  }
+
+  // The rest of arguments are optional, so we can just skip "atom".
+  for (size_t i = 2, e = args.size(); i != e; ++i) {
+    if (atomArg != &args[i])
+      nonAtoms.push_back(AsActual(args[i]));
+  }
+
+  // The type of the intermediate min/max is the same as the type of its
+  // arguments, which may be different from the type of the original
+  // expression. The original expression may have additional coverts.
+  auto tmp =
+      makeCall(*atomArg->GetType(), getProcedureDesignator(orig), nonAtoms);
+  semantics::SomeExpr call = orig;
+  replaceArgs(call, {AsActual(*atomArg), AsActual(tmp)});
+  return call;
+}
+
 static mlir::Operation * //
 genAtomicRead(lower::AbstractConverter &converter,
               semantics::SemanticsContext &semaCtx, mlir::Location loc,
@@ -350,10 +603,28 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
 
   // This must exist by now.
-  semantics::SomeExpr input = *evaluate::GetConvertInput(assign.rhs);
-  std::vector<semantics::SomeExpr> args =
-      evaluate::GetTopLevelOperation(input).second;
+  semantics::SomeExpr rhs = assign.rhs;
+  semantics::SomeExpr input = *evaluate::GetConvertInput(rhs);
+  auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
+
+  const semantics::SomeExpr *atomArg = [&]() {
+    for (const semantics::SomeExpr &e : args) {
+      if (evaluate::IsSameOrConvertOf(e, atom))
+        return &e;
+    }
+    llvm_unreachable("Atomic variable not in argument list");
+  }();
+
+  if (opcode == evaluate::operation::Operator::Min ||
+      opcode == evaluate::operation::Operator::Max) {
+    // Min and max operations are expanded inline, so reduce them to
+    // operations with exactly two (non-optional) arguments.
+    rhs = genReducedMinMax(rhs, atomArg, args);
+    input = *evaluate::GetConvertInput(rhs);
+    std::tie(opcode, args) = evaluate::GetTopLevelOperation(input);
+    atomArg = nullptr; // No longer valid.
+  }
   for (auto &arg : args) {
     if (!evaluate::IsSameOrConvertOf(arg, atom)) {
       mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
@@ -372,7 +643,7 @@ genAtomicUpdate(lower::AbstractConverter &converter,
 
   converter.overrideExprValues(&overrides);
   mlir::Value updated =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+      fir::getBase(converter.genExprValue(rhs, stmtCtx, &loc));
   mlir::Value converted = builder.createConvert(loc, atomType, updated);
   builder.create<mlir::omp::YieldOp>(loc, converted);
   converter.resetExprOverrides();
diff --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90
index 3f840acefa6e8..f88bbea6fca85 100644
--- a/flang/test/Lower/OpenMP/atomic-update.f90
+++ b/flang/test/Lower/OpenMP/atomic-update.f90
@@ -107,8 +107,6 @@ program OmpAtomicUpdate
 !CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_Y_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK: ^bb0(%[[ARG:.*]]: i32):
 !CHECK:  {{.*}} = arith.cmpi sgt, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.select {{.*}}, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.cmpi sgt, {{.*}}
 !CHECK:  %[[TEMP:.*]] = arith.select {{.*}} : i32
 !CHECK:  omp.yield(%[[TEMP]] : i32)
 !CHECK: }
@@ -177,13 +175,9 @@ program OmpAtomicUpdate
 !CHECK:   %[[VAL_Z_LOADED:.*]] = fir.load %[[VAL_Z_DECLARE]]#0 : !fir.ref<i32>
 !CHECK:   omp.atomic.update %[[VAL_W_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG_W:.*]]: i32):
-!CHECK:     %[[WX_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WX_MIN:.*]] = arith.select %[[WX_CMP]], %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WXY_CMP:.*]] = arith.cmpi sgt, %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXY_MIN:.*]] = arith.select %[[WXY_CMP]], %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXYZ_CMP:.*]] = arith.cmpi sgt, %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     %[[WXYZ_MIN:.*]] = arith.select %[[WXYZ_CMP]], %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     omp.yield(%[[WXYZ_MIN]] : i32)
+!CHECK:     %[[W_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], {{.*}} : i32
+!CHECK:     %[[WXYZ_MAX:.*]] = arith.select %[[W_CMP]], %[[ARG_W]], {{.*}} : i32
+!CHECK:     omp.yield(%[[WXYZ_MAX]] : i32)
 !CHECK:   }
   !$omp atomic update
     w = max(w,x,y,z)
diff --git a/flang/test/Lower/OpenMP/max-optional-parameters.f90 b/flang/test/Lower/OpenMP/max-optional-parameters.f90
new file mode 100644
index 0000000000000..2bb6b110cb0ef
--- /dev/null
+++ b/flang/test/Lower/OpenMP/max-optional-parameters.f90
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
+!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: } else {
+!CHECK:   fir.result {{.*}} : f32
+!CHECK: }
+!CHECK: omp.atomic.update {{.*}} : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
+!CHECK:   %[[V10:[a-z0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[IF_VAL]]
+!CHECK:   %[[V11:[a-z0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[IF_VAL]]
+!CHECK:   omp.yield(%[[V11]] : f32)
+!CHECK: }
+
+subroutine f00(a, x, y)
+  real :: a
+  real, optional :: x, y
+  !$omp atomic update
+  a = max(x, a, y)
+end

>From 4302ed81021028baec7427028562802a516ea511 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 07:38:45 -0500
Subject: [PATCH 2/5] Address review comments

---
 flang/lib/Lower/OpenMP/Atomic.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index b7a223eff80c5..2ab91b239a3cc 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -157,6 +157,7 @@ struct WithType {
       }
       break;
     case common::TypeCategory::Derived:
+      (void)Derived;
       break;
     }
     llvm_unreachable("Unhandled type");
@@ -438,7 +439,7 @@ genReducedMinMax(const semantics::SomeExpr &orig,
                  const semantics::SomeExpr *atomArg,
                  const std::vector<semantics::SomeExpr> &args) {
   // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
-  // One of the a_i's, say a_t, must be atom (or a convert of atom).
+  // One of the a_i's, say a_t, must be atomArg.
   // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
   // call = min/max(a_t, tmp).
   // Return "call".
@@ -608,13 +609,14 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
 
-  const semantics::SomeExpr *atomArg = [&]() {
+  // Pass args as an argument to avoid capturing a structured binding.
+  const semantics::SomeExpr *atomArg = [&](auto &args) {
     for (const semantics::SomeExpr &e : args) {
       if (evaluate::IsSameOrConvertOf(e, atom))
         return &e;
     }
     llvm_unreachable("Atomic variable not in argument list");
-  }();
+  }(args);
 
   if (opcode == evaluate::operation::Operator::Min ||
       opcode == evaluate::operation::Operator::Max) {

>From 16df2e0eb775fce156db6f5fc62509026a0b2bc5 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 08:01:23 -0500
Subject: [PATCH 3/5] Check for non-atomic operations, add min

---
 .../Lower/OpenMP/max-optional-parameters.f90  | 58 ++++++++++++++++---
 1 file changed, 51 insertions(+), 7 deletions(-)

diff --git a/flang/test/Lower/OpenMP/max-optional-parameters.f90 b/flang/test/Lower/OpenMP/max-optional-parameters.f90
index 2bb6b110cb0ef..57a432eeeecec 100644
--- a/flang/test/Lower/OpenMP/max-optional-parameters.f90
+++ b/flang/test/Lower/OpenMP/max-optional-parameters.f90
@@ -3,16 +3,27 @@
 ! Check that the presence tests are done outside of the atomic update
 ! construct.
 
-!CHECK: %[[IS_PRESENT:[a-z0-9]+]] = fir.is_present
-!CHECK: %[[IF_VAL:[a-z0-9]+]] = fir.if %[[IS_PRESENT]] -> (f32) {
-!CHECK:   fir.result {{.*}} : f32
+!CHECK-LABEL: func.func @_QPf00
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<f32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpf ogt, %[[V4]], %[[V5]] fastmath<contract> : f32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : f32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (f32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<f32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpf ogt, %[[V8]], %[[V10]] fastmath<contract> : f32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : f32
+!CHECK:   fir.result %[[V12]] : f32
 !CHECK: } else {
-!CHECK:   fir.result {{.*}} : f32
+!CHECK:   fir.result %[[V8]] : f32
 !CHECK: }
-!CHECK: omp.atomic.update {{.*}} : !fir.ref<f32> {
+!CHECK: omp.atomic.update %[[VAL_A]]#0 : !fir.ref<f32> {
 !CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
-!CHECK:   %[[V10:[a-z0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[IF_VAL]]
-!CHECK:   %[[V11:[a-z0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[IF_VAL]]
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[V9]] fastmath<contract> : f32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : f32
 !CHECK:   omp.yield(%[[V11]] : f32)
 !CHECK: }
 
@@ -22,3 +33,36 @@ subroutine f00(a, x, y)
   !$omp atomic update
   a = max(x, a, y)
 end
+
+
+!CHECK-LABEL: func.func @_QPf01
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<i32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpi slt, %[[V4]], %[[V5]] : i32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : i32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (i32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<i32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpi slt, %[[V8]], %[[V10]] : i32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : i32
+!CHECK:   fir.result %[[V12]] : i32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : i32
+!CHECK: }
+!CHECK: omp.atomic.update %[[VAL_A]]#0 : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: i32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpi slt, %[[ARG]], %[[V9]] : i32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : i32
+!CHECK:   omp.yield(%[[V11]] : i32)
+!CHECK: }
+
+subroutine f01(a, x, y)
+  integer :: a
+  integer, optional :: x, y
+  !$omp atomic update
+  a = min(x, a, y)
+end
+

>From 6c45f7c32a4919ee950703377fa1b5c8f0257fc2 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 09:13:35 -0500
Subject: [PATCH 4/5] Fix test

---
 flang/test/Lower/OpenMP/max-optional-parameters.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Lower/OpenMP/max-optional-parameters.f90 b/flang/test/Lower/OpenMP/max-optional-parameters.f90
index 57a432eeeecec..418a3cad8cdaf 100644
--- a/flang/test/Lower/OpenMP/max-optional-parameters.f90
+++ b/flang/test/Lower/OpenMP/max-optional-parameters.f90
@@ -20,7 +20,7 @@
 !CHECK: } else {
 !CHECK:   fir.result %[[V8]] : f32
 !CHECK: }
-!CHECK: omp.atomic.update %[[VAL_A]]#0 : !fir.ref<f32> {
+!CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_A]]#0 : !fir.ref<f32> {
 !CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
 !CHECK:   %[[V10:[0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[V9]] fastmath<contract> : f32
 !CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : f32
@@ -52,7 +52,7 @@ subroutine f00(a, x, y)
 !CHECK: } else {
 !CHECK:   fir.result %[[V8]] : i32
 !CHECK: }
-!CHECK: omp.atomic.update %[[VAL_A]]#0 : !fir.ref<i32> {
+!CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_A]]#0 : !fir.ref<i32> {
 !CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: i32):
 !CHECK:   %[[V10:[0-9]+]] = arith.cmpi slt, %[[ARG]], %[[V9]] : i32
 !CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : i32

>From 3aa831fe921cb2ff40d24206dc2d98909f681e94 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 09:13:59 -0500
Subject: [PATCH 5/5] Rename test

---
 ...max-optional-parameters.f90 => minmax-optional-parameters.f90} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename flang/test/Lower/OpenMP/{max-optional-parameters.f90 => minmax-optional-parameters.f90} (100%)

diff --git a/flang/test/Lower/OpenMP/max-optional-parameters.f90 b/flang/test/Lower/OpenMP/minmax-optional-parameters.f90
similarity index 100%
rename from flang/test/Lower/OpenMP/max-optional-parameters.f90
rename to flang/test/Lower/OpenMP/minmax-optional-parameters.f90


From flang-commits at lists.llvm.org  Tue Jul  1 07:16:12 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 07:16:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863edac.170a0220.382880.d311@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM if Kiran is happy

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 07:52:17 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Tue, 01 Jul 2025 07:52:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863f621.050a0220.23a56e.8a20@mx.google.com>


kiranchandramohan wrote:

> Eventually both of these will be done in semantics.

OK.

> Meanwhile, there is another issue, this time with reassociation: the expression w = w .and. x .and. y, which is parsed as w = (w .and. x) .and. y, is rejected since w is not a top-level argument. 

There is https://github.com/llvm/llvm-project/issues/138748 assigned to me. If you are working on this, please feel free to assign to yourself.

Thanks for the explanations.

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 07:52:30 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Tue, 01 Jul 2025 07:52:30 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863f62e.630a0220.3835e0.fd8b@mx.google.com>


https://github.com/kiranchandramohan approved this pull request.

LG.

https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 07:55:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 07:55:02 -0700 (PDT)
Subject: [flang-commits] [flang] 86077c4 - [flang][OpenMP] Rewrite min/max
 with more than 2 arguments (#146423)
Message-ID: <6863f6c6.170a0220.2821cb.e75d@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-01T09:54:58-05:00
New Revision: 86077c41a7899fb3a3ce4654bdb373e7cd954f49

URL: https://github.com/llvm/llvm-project/commit/86077c41a7899fb3a3ce4654bdb373e7cd954f49
DIFF: https://github.com/llvm/llvm-project/commit/86077c41a7899fb3a3ce4654bdb373e7cd954f49.diff

LOG: [flang][OpenMP] Rewrite min/max with more than 2 arguments (#146423)

Given an atomic operation `w = max(w, x1, x2, ...)` rewrite it as `w =
max(w, max(x1, x2, ...))`. This will avoid unnecessary non-atomic
comparisons inside of the atomic operation (min/max are expanded
inline).

In particular, if some of the x_i's are optional dummy parameters in the
containing function, this will avoid any presence tests within the
atomic operation.

Fixes https://github.com/llvm/llvm-project/issues/144838

Added: 
    flang/test/Lower/OpenMP/minmax-optional-parameters.f90

Modified: 
    flang/lib/Lower/OpenMP/Atomic.cpp
    flang/test/Lower/OpenMP/atomic-update.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 33a743f8f9dda..2ab91b239a3cc 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -11,6 +11,8 @@
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
@@ -41,6 +43,179 @@ namespace omp {
 using namespace Fortran::lower::omp;
 }
 
+namespace {
+// An example of a type that can be used to get the return value from
+// the visitor:
+//   visitor(type_identity<Xyz>) -> result_type
+using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
+
+struct GetProc
+    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
+                                false> {
+  using Result = const evaluate::ProcedureDesignator *;
+  using Base = evaluate::Traverse<GetProc, Result, false>;
+  GetProc() : Base(*this) {}
+
+  using Base::operator();
+
+  static Result Default() { return nullptr; }
+
+  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
+  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
+};
+
+struct WithType {
+  WithType(const evaluate::DynamicType &t) : type(t) {
+    assert(type.category() != common::TypeCategory::Derived &&
+           "Type cannot be a derived type");
+  }
+
+  template <typename VisitorTy> //
+  auto visit(VisitorTy &&visitor) const
+      -> std::invoke_result_t<VisitorTy, SomeArgType> {
+    switch (type.category()) {
+    case common::TypeCategory::Integer:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Unsigned:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Real:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Complex:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Logical:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
+      }
+      break;
+    case common::TypeCategory::Character:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
+      }
+      break;
+    case common::TypeCategory::Derived:
+      (void)Derived;
+      break;
+    }
+    llvm_unreachable("Unhandled type");
+  }
+
+  const evaluate::DynamicType &type;
+
+private:
+  // Shorter names.
+  static constexpr auto Character = common::TypeCategory::Character;
+  static constexpr auto Complex = common::TypeCategory::Complex;
+  static constexpr auto Derived = common::TypeCategory::Derived;
+  static constexpr auto Integer = common::TypeCategory::Integer;
+  static constexpr auto Logical = common::TypeCategory::Logical;
+  static constexpr auto Real = common::TypeCategory::Real;
+  static constexpr auto Unsigned = common::TypeCategory::Unsigned;
+};
+
+template <typename T, typename U = std::remove_const_t<T>>
+U AsRvalue(T &t) {
+  U copy{t};
+  return std::move(copy);
+}
+
+template <typename T>
+T &&AsRvalue(T &&t) {
+  return std::move(t);
+}
+
+struct ArgumentReplacer
+    : public evaluate::Traverse<ArgumentReplacer, bool, false> {
+  using Base = evaluate::Traverse<ArgumentReplacer, bool, false>;
+  using Result = bool;
+
+  Result Default() const { return false; }
+
+  ArgumentReplacer(evaluate::ActualArguments &&newArgs)
+      : Base(*this), args_(std::move(newArgs)) {}
+
+  using Base::operator();
+
+  template <typename T>
+  Result operator()(const evaluate::FunctionRef<T> &x) {
+    assert(!done_);
+    auto &mut = const_cast<evaluate::FunctionRef<T> &>(x);
+    mut.arguments() = args_;
+    done_ = true;
+    return true;
+  }
+
+  Result Combine(Result &&a, Result &&b) { return a || b; }
+
+private:
+  bool done_{false};
+  evaluate::ActualArguments &&args_;
+};
+} // namespace
+
 [[maybe_unused]] static void
 dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
   auto whatStr = [](int k) {
@@ -237,6 +412,85 @@ makeMemOrderAttr(lower::AbstractConverter &converter,
   return nullptr;
 }
 
+static bool replaceArgs(semantics::SomeExpr &expr,
+                        evaluate::ActualArguments &&newArgs) {
+  return ArgumentReplacer(std::move(newArgs))(expr);
+}
+
+static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
+                                    const evaluate::ProcedureDesignator &proc,
+                                    const evaluate::ActualArguments &args) {
+  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
+    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
+    return evaluate::AsGenericExpr(
+        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
+  });
+}
+
+static const evaluate::ProcedureDesignator &
+getProcedureDesignator(const semantics::SomeExpr &call) {
+  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
+  assert(proc && "Call has no procedure designator");
+  return *proc;
+}
+
+static semantics::SomeExpr //
+genReducedMinMax(const semantics::SomeExpr &orig,
+                 const semantics::SomeExpr *atomArg,
+                 const std::vector<semantics::SomeExpr> &args) {
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be atomArg.
+  // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
+  // call = min/max(a_t, tmp).
+  // Return "call".
+
+  // The min/max intrinsics have 2 mandatory arguments, the rest is optional.
+  // Make sure that the "tmp = min/max(...)" doesn't promote an optional
+  // argument to a non-optional position. This could happen if a_t is at
+  // position 0 or 1.
+  if (args.size() <= 2)
+    return orig;
+
+  evaluate::ActualArguments nonAtoms;
+
+  auto AsActual = [](const semantics::SomeExpr &x) {
+    semantics::SomeExpr copy = x;
+    return evaluate::ActualArgument(std::move(copy));
+  };
+  // Semantic checks guarantee that the "atom" shows exactly once in the
+  // argument list (with potential conversions around it).
+  // For the first two (non-optional) arguments, if "atom" is among them,
+  // replace it with another occurrence of the other non-optional argument.
+  if (atomArg == &args[0]) {
+    // (atom, x, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[1]));
+    nonAtoms.push_back(AsActual(args[1]));
+  } else if (atomArg == &args[1]) {
+    // (x, atom, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[0]));
+  } else {
+    // (x, y, z...) -> unchanged
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[1]));
+  }
+
+  // The rest of arguments are optional, so we can just skip "atom".
+  for (size_t i = 2, e = args.size(); i != e; ++i) {
+    if (atomArg != &args[i])
+      nonAtoms.push_back(AsActual(args[i]));
+  }
+
+  // The type of the intermediate min/max is the same as the type of its
+  // arguments, which may be 
diff erent from the type of the original
+  // expression. The original expression may have additional coverts.
+  auto tmp =
+      makeCall(*atomArg->GetType(), getProcedureDesignator(orig), nonAtoms);
+  semantics::SomeExpr call = orig;
+  replaceArgs(call, {AsActual(*atomArg), AsActual(tmp)});
+  return call;
+}
+
 static mlir::Operation * //
 genAtomicRead(lower::AbstractConverter &converter,
               semantics::SemanticsContext &semaCtx, mlir::Location loc,
@@ -350,10 +604,29 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
 
   // This must exist by now.
-  semantics::SomeExpr input = *evaluate::GetConvertInput(assign.rhs);
-  std::vector<semantics::SomeExpr> args =
-      evaluate::GetTopLevelOperation(input).second;
+  semantics::SomeExpr rhs = assign.rhs;
+  semantics::SomeExpr input = *evaluate::GetConvertInput(rhs);
+  auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
+
+  // Pass args as an argument to avoid capturing a structured binding.
+  const semantics::SomeExpr *atomArg = [&](auto &args) {
+    for (const semantics::SomeExpr &e : args) {
+      if (evaluate::IsSameOrConvertOf(e, atom))
+        return &e;
+    }
+    llvm_unreachable("Atomic variable not in argument list");
+  }(args);
+
+  if (opcode == evaluate::operation::Operator::Min ||
+      opcode == evaluate::operation::Operator::Max) {
+    // Min and max operations are expanded inline, so reduce them to
+    // operations with exactly two (non-optional) arguments.
+    rhs = genReducedMinMax(rhs, atomArg, args);
+    input = *evaluate::GetConvertInput(rhs);
+    std::tie(opcode, args) = evaluate::GetTopLevelOperation(input);
+    atomArg = nullptr; // No longer valid.
+  }
   for (auto &arg : args) {
     if (!evaluate::IsSameOrConvertOf(arg, atom)) {
       mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
@@ -372,7 +645,7 @@ genAtomicUpdate(lower::AbstractConverter &converter,
 
   converter.overrideExprValues(&overrides);
   mlir::Value updated =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+      fir::getBase(converter.genExprValue(rhs, stmtCtx, &loc));
   mlir::Value converted = builder.createConvert(loc, atomType, updated);
   builder.create<mlir::omp::YieldOp>(loc, converted);
   converter.resetExprOverrides();

diff  --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90
index 3f840acefa6e8..f88bbea6fca85 100644
--- a/flang/test/Lower/OpenMP/atomic-update.f90
+++ b/flang/test/Lower/OpenMP/atomic-update.f90
@@ -107,8 +107,6 @@ program OmpAtomicUpdate
 !CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_Y_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK: ^bb0(%[[ARG:.*]]: i32):
 !CHECK:  {{.*}} = arith.cmpi sgt, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.select {{.*}}, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.cmpi sgt, {{.*}}
 !CHECK:  %[[TEMP:.*]] = arith.select {{.*}} : i32
 !CHECK:  omp.yield(%[[TEMP]] : i32)
 !CHECK: }
@@ -177,13 +175,9 @@ program OmpAtomicUpdate
 !CHECK:   %[[VAL_Z_LOADED:.*]] = fir.load %[[VAL_Z_DECLARE]]#0 : !fir.ref<i32>
 !CHECK:   omp.atomic.update %[[VAL_W_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG_W:.*]]: i32):
-!CHECK:     %[[WX_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WX_MIN:.*]] = arith.select %[[WX_CMP]], %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WXY_CMP:.*]] = arith.cmpi sgt, %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXY_MIN:.*]] = arith.select %[[WXY_CMP]], %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXYZ_CMP:.*]] = arith.cmpi sgt, %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     %[[WXYZ_MIN:.*]] = arith.select %[[WXYZ_CMP]], %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     omp.yield(%[[WXYZ_MIN]] : i32)
+!CHECK:     %[[W_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], {{.*}} : i32
+!CHECK:     %[[WXYZ_MAX:.*]] = arith.select %[[W_CMP]], %[[ARG_W]], {{.*}} : i32
+!CHECK:     omp.yield(%[[WXYZ_MAX]] : i32)
 !CHECK:   }
   !$omp atomic update
     w = max(w,x,y,z)

diff  --git a/flang/test/Lower/OpenMP/minmax-optional-parameters.f90 b/flang/test/Lower/OpenMP/minmax-optional-parameters.f90
new file mode 100644
index 0000000000000..418a3cad8cdaf
--- /dev/null
+++ b/flang/test/Lower/OpenMP/minmax-optional-parameters.f90
@@ -0,0 +1,68 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK-LABEL: func.func @_QPf00
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<f32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpf ogt, %[[V4]], %[[V5]] fastmath<contract> : f32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : f32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (f32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<f32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpf ogt, %[[V8]], %[[V10]] fastmath<contract> : f32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : f32
+!CHECK:   fir.result %[[V12]] : f32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : f32
+!CHECK: }
+!CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_A]]#0 : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[V9]] fastmath<contract> : f32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : f32
+!CHECK:   omp.yield(%[[V11]] : f32)
+!CHECK: }
+
+subroutine f00(a, x, y)
+  real :: a
+  real, optional :: x, y
+  !$omp atomic update
+  a = max(x, a, y)
+end
+
+
+!CHECK-LABEL: func.func @_QPf01
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<i32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpi slt, %[[V4]], %[[V5]] : i32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : i32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (i32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<i32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpi slt, %[[V8]], %[[V10]] : i32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : i32
+!CHECK:   fir.result %[[V12]] : i32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : i32
+!CHECK: }
+!CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_A]]#0 : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: i32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpi slt, %[[ARG]], %[[V9]] : i32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : i32
+!CHECK:   omp.yield(%[[V11]] : i32)
+!CHECK: }
+
+subroutine f01(a, x, y)
+  integer :: a
+  integer, optional :: x, y
+  !$omp atomic update
+  a = min(x, a, y)
+end
+


        

From flang-commits at lists.llvm.org  Tue Jul  1 07:55:07 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 07:55:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Rewrite min/max with more
 than 2 arguments (PR #146423)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146423@github.com>
Message-ID: <6863f6cb.050a0220.1f3361.aacc@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/146423

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:34 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:34 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f84e.630a0220.1cc7cc.fe2a@mx.google.com>


https://github.com/skatrak edited https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:34 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:34 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f84e.170a0220.e6823.00ef@mx.google.com>


https://github.com/skatrak commented:

Thank you Akash for this work, and excuse the delay!

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:35 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:35 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f84f.170a0220.115564.e706@mx.google.com>


================
@@ -3540,7 +3540,7 @@ WRAPPER_CLASS(OmpLocatorList, std::list<OmpLocator>);
 struct OmpMapperSpecifier {
   // Absent mapper-identifier is equivalent to DEFAULT.
   TUPLE_CLASS_BOILERPLATE(OmpMapperSpecifier);
-  std::tuple<std::optional<Name>, TypeSpec, Name> t;
+  std::tuple<std::string, TypeSpec, Name> t;
----------------
skatrak wrote:

Since we haven't figured out a better alternative for this situation, I think we can hopefully get away with making this PFT node owner of the identifier name for now.

However, this is a departure from the convention of only storing references to externally allocated strings, so I'd suggest adding a comment here to explain why we're making an exception (i.e. this identifier might be compiler-generated and not part of the source).

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:35 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:35 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f84f.050a0220.1d1bb5.bb50@mx.google.com>


================
@@ -1389,8 +1389,28 @@ TYPE_PARSER(
 TYPE_PARSER(sourced(construct<OpenMPDeclareTargetConstruct>(
     verbatim("DECLARE TARGET"_tok), Parser<OmpDeclareTargetSpecifier>{})))
 
+static OmpMapperSpecifier ConstructOmpMapperSpecifier(
----------------
skatrak wrote:

Nit: This name seems to follow other existing similar functions better, IMO. In any case, the first letter should be lowercase.
```suggestion
static OmpMapperSpecifier makeMapperSpecifier(
```

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:35 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:35 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f84f.170a0220.229da7.14f2@mx.google.com>


================
@@ -1,9 +1,11 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
+!CHECK: omp.declare_mapper @[[MAPPER1:_QQFmaptype_derived_implicit_allocatablescalar_and_array.omp.default.mapper]] : !fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
+!CHECK: omp.declare_mapper @[[MAPPER2:_QQFmaptype_derived_implicitscalar_and_array.omp.default.mapper]] : !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
----------------
skatrak wrote:

```suggestion
!CHECK: omp.declare_mapper @[[MAPPER2:_QQFmaptype_derived_implicitscalar_and_array\.omp\.default\.mapper]] : !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
```

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:35 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:35 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f84f.170a0220.123bb4.d4b7@mx.google.com>


================
@@ -1102,23 +1102,30 @@ void ClauseProcessor::processMapObjects(
 
   auto getDefaultMapperID = [&](const omp::Object &object,
                                 std::string &mapperIdName) {
-    if (!mlir::isa<mlir::omp::DeclareMapperOp>(
-            firOpBuilder.getRegion().getParentOp())) {
-      const semantics::DerivedTypeSpec *typeSpec = nullptr;
-
-      if (object.sym()->owner().IsDerivedType())
-        typeSpec = object.sym()->owner().derivedTypeSpec();
-      else if (object.sym()->GetType() &&
-               object.sym()->GetType()->category() ==
-                   semantics::DeclTypeSpec::TypeDerived)
-        typeSpec = &object.sym()->GetType()->derivedTypeSpec();
-
-      if (typeSpec) {
-        mapperIdName = typeSpec->name().ToString() + ".default";
+    const semantics::DerivedTypeSpec *typeSpec = nullptr;
+
+    if (object.sym()->GetType() && object.sym()->GetType()->category() ==
+                                       semantics::DeclTypeSpec::TypeDerived)
+      typeSpec = &object.sym()->GetType()->derivedTypeSpec();
+    else if (object.sym()->owner().IsDerivedType())
+      typeSpec = object.sym()->owner().derivedTypeSpec();
+
+    if (typeSpec) {
+      mapperIdName = typeSpec->name().ToString() + ".omp.default.mapper";
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+      else
         mapperIdName =
             converter.mangleName(mapperIdName, *typeSpec->GetScope());
-      }
     }
+
+    // Make sure we don't return a mapper to self
----------------
skatrak wrote:

```suggestion
    // Make sure we don't return a mapper to self.
```

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:36 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:36 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f850.170a0220.271039.fa4e@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
----------------
skatrak wrote:

Is `recType` not going to always match the `recordType` argument? This seems quite a complicated implementation if that's the case. `[fieldName, fieldTy]` would match `[memberName, memberType]` of the caller below.

Let me know if I'm missing something here.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:36 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:36 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f850.170a0220.36cb53.0bb4@mx.google.com>


================
@@ -171,3 +172,93 @@ subroutine declare_mapper_4
    a%num = 40
    !$omp end target
 end subroutine declare_mapper_4
+
+!--- omp-declare-mapper-5.f90
+program declare_mapper_5
+   implicit none
+
+   type :: mytype
+      integer :: x, y
+   end type
+
+   !CHECK: omp.declare_mapper @[[INNER_MAPPER_NAMED:_QQFFuse_innermy_mapper]] : [[MY_TYPE:!fir\.type<_QFTmytype\{x:i32,y:i32\}>]]
+   !CHECK: omp.declare_mapper @[[INNER_MAPPER_DEFAULT:_QQFFuse_innermytype.omp.default.mapper]] : [[MY_TYPE]]
----------------
skatrak wrote:

```suggestion
   !CHECK: omp.declare_mapper @[[INNER_MAPPER_DEFAULT:_QQFFuse_innermytype\.omp\.default\.mapper]] : [[MY_TYPE]]
```

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:36 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:36 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f850.170a0220.232821.0e00@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
+        fir::unwrapPassByRefType(rec.getType()));
+    auto [fieldName, fieldTy] = recType.getTypeList()[index];
+    mlir::Value field = firOpBuilder.create<fir::FieldIndexOp>(
+        loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
+        fir::getTypeParams(rec));
+    return {firOpBuilder.create<fir::CoordinateOp>(
+                loc, firOpBuilder.getRefType(fieldTy), rec, field),
+            fieldTy};
+  };
+
+  mlir::omp::DeclareMapperInfoOperands clauseOps;
+  llvm::SmallVector<llvm::SmallVector<int64_t>> memberPlacementIndices;
+  llvm::SmallVector<mlir::Value> memberMapOps;
+
+  llvm::omp::OpenMPOffloadMappingFlags mapFlag =
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+  mlir::omp::VariableCaptureKind captureKind =
+      mlir::omp::VariableCaptureKind::ByRef;
+  int64_t index = 0;
----------------
skatrak wrote:

Nit: Use `llvm::enumerate` instead of an outside index variable. That way we avoid letting it go out-of-sync in the future if we introduce any `continue` statements inside of the loop body.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:36 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:36 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f850.050a0220.23a5a8.8d08@mx.google.com>


================
@@ -1,9 +1,11 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
+!CHECK: omp.declare_mapper @[[MAPPER1:_QQFmaptype_derived_implicit_allocatablescalar_and_array.omp.default.mapper]] : !fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
----------------
skatrak wrote:

```suggestion
!CHECK: omp.declare_mapper @[[MAPPER1:_QQFmaptype_derived_implicit_allocatablescalar_and_array\.omp\.default\.mapper]] : !fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
```

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:36 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:36 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f850.170a0220.2e7d61.fd4e@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
+        fir::unwrapPassByRefType(rec.getType()));
+    auto [fieldName, fieldTy] = recType.getTypeList()[index];
+    mlir::Value field = firOpBuilder.create<fir::FieldIndexOp>(
+        loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
+        fir::getTypeParams(rec));
+    return {firOpBuilder.create<fir::CoordinateOp>(
+                loc, firOpBuilder.getRefType(fieldTy), rec, field),
+            fieldTy};
+  };
+
+  mlir::omp::DeclareMapperInfoOperands clauseOps;
+  llvm::SmallVector<llvm::SmallVector<int64_t>> memberPlacementIndices;
+  llvm::SmallVector<mlir::Value> memberMapOps;
+
+  llvm::omp::OpenMPOffloadMappingFlags mapFlag =
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+  mlir::omp::VariableCaptureKind captureKind =
+      mlir::omp::VariableCaptureKind::ByRef;
+  int64_t index = 0;
+
+  // Populate the declareMapper region with the map information.
+  for (const auto &[memberName, memberType] :
+       mlir::dyn_cast<fir::RecordType>(recordType).getTypeList()) {
+    auto [ref, type] = getFieldRef(declareOp.getBase(), index);
+    mlir::FlatSymbolRefAttr mapperId;
+    if (auto recType = mlir::dyn_cast<fir::RecordType>(memberType)) {
+      std::string mapperIdName =
+          recType.getName().str() + ".omp.default.mapper";
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+      else if (auto *sym = converter.getCurrentScope().FindSymbol(memberName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+
+      if (converter.getModuleOp().lookupSymbol(mapperIdName))
+        mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                                mapperIdName);
+      else
+        mapperId = genImplicitDefaultDeclareMapper(converter, loc, recType,
+                                                   mapperIdName);
+    }
+
+    llvm::SmallVector<mlir::Value> bounds;
+    genBoundsOps(ref, bounds);
+    mlir::Value mapOp = createMapInfoOp(
+        firOpBuilder, loc, ref, /*varPtrPtr=*/mlir::Value{}, "", bounds,
----------------
skatrak wrote:

Nit: Could the symbol name be passed to this function, so that the name here and below could be populated? I would imagine it might be populated as `"symName + "%" + memberName"` here (which could also be the `symName` passed to the recursive self call above) and just `symName` below.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:36 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:36 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f850.630a0220.1ddefc.fc3c@mx.google.com>


================
@@ -149,7 +150,7 @@ subroutine declare_mapper_4
       integer              :: num
    end type
 
-   !CHECK: omp.declare_mapper @[[MY_TYPE_MAPPER:_QQFdeclare_mapper_4my_type.default]] : [[MY_TYPE:!fir\.type<_QFdeclare_mapper_4Tmy_type\{num:i32\}>]]
+   !CHECK: omp.declare_mapper @[[MY_TYPE_MAPPER:_QQFdeclare_mapper_4my_type.omp.default.mapper]] : [[MY_TYPE:!fir\.type<_QFdeclare_mapper_4Tmy_type\{num:i32\}>]]
----------------
skatrak wrote:

```suggestion
   !CHECK: omp.declare_mapper @[[MY_TYPE_MAPPER:_QQFdeclare_mapper_4my_type\.omp\.default\.mapper]] : [[MY_TYPE:!fir\.type<_QFdeclare_mapper_4Tmy_type\{num:i32\}>]]
```

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:40 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:40 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f854.170a0220.2caeed.ea6e@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
----------------
skatrak wrote:

Is this exclusively needed for implicitly mapped variables into `target` regions or should it be called for explicit `map` clauses for derived types when they don't specify any mapper? In particular, in `target`, `declare mapper`, `target enter data` and `target exit data` directives.

Would we need this for any other map-like clause, like `use_device_ptr`, `use_device_addr` or `has_device_addr`?

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:40 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:40 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f854.170a0220.1357ad.0a39@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
+        fir::unwrapPassByRefType(rec.getType()));
+    auto [fieldName, fieldTy] = recType.getTypeList()[index];
+    mlir::Value field = firOpBuilder.create<fir::FieldIndexOp>(
+        loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
+        fir::getTypeParams(rec));
+    return {firOpBuilder.create<fir::CoordinateOp>(
+                loc, firOpBuilder.getRefType(fieldTy), rec, field),
+            fieldTy};
+  };
+
+  mlir::omp::DeclareMapperInfoOperands clauseOps;
+  llvm::SmallVector<llvm::SmallVector<int64_t>> memberPlacementIndices;
+  llvm::SmallVector<mlir::Value> memberMapOps;
+
+  llvm::omp::OpenMPOffloadMappingFlags mapFlag =
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+  mlir::omp::VariableCaptureKind captureKind =
+      mlir::omp::VariableCaptureKind::ByRef;
+  int64_t index = 0;
+
+  // Populate the declareMapper region with the map information.
+  for (const auto &[memberName, memberType] :
+       mlir::dyn_cast<fir::RecordType>(recordType).getTypeList()) {
+    auto [ref, type] = getFieldRef(declareOp.getBase(), index);
+    mlir::FlatSymbolRefAttr mapperId;
+    if (auto recType = mlir::dyn_cast<fir::RecordType>(memberType)) {
+      std::string mapperIdName =
+          recType.getName().str() + ".omp.default.mapper";
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+      else if (auto *sym = converter.getCurrentScope().FindSymbol(memberName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+
+      if (converter.getModuleOp().lookupSymbol(mapperIdName))
+        mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                                mapperIdName);
+      else
+        mapperId = genImplicitDefaultDeclareMapper(converter, loc, recType,
+                                                   mapperIdName);
----------------
skatrak wrote:

Nit: Since both calls to the new function are wrapped inside of the same logic, perhaps `genImplicitDefaultDeclareMapper` could be `getOrGenImplicitDefaultDeclareMapper` and do the lookup inside of the function. That way we ensure we won't create duplicated `DeclareMapperOp`s, if we decide to call this function from other spots in the future.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:43 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:43 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f857.170a0220.325eb1.7f86@mx.google.com>


================
@@ -171,3 +172,93 @@ subroutine declare_mapper_4
    a%num = 40
    !$omp end target
 end subroutine declare_mapper_4
+
+!--- omp-declare-mapper-5.f90
+program declare_mapper_5
+   implicit none
+
+   type :: mytype
+      integer :: x, y
+   end type
+
+   !CHECK: omp.declare_mapper @[[INNER_MAPPER_NAMED:_QQFFuse_innermy_mapper]] : [[MY_TYPE:!fir\.type<_QFTmytype\{x:i32,y:i32\}>]]
+   !CHECK: omp.declare_mapper @[[INNER_MAPPER_DEFAULT:_QQFFuse_innermytype.omp.default.mapper]] : [[MY_TYPE]]
+   !CHECK: omp.declare_mapper @[[OUTER_MAPPER_NAMED:_QQFmy_mapper]] : [[MY_TYPE]]
+   !CHECK: omp.declare_mapper @[[OUTER_MAPPER_DEFAULT:_QQFmytype.omp.default.mapper]] : [[MY_TYPE]]
----------------
skatrak wrote:

```suggestion
   !CHECK: omp.declare_mapper @[[OUTER_MAPPER_DEFAULT:_QQFmytype\.omp\.default\.mapper]] : [[MY_TYPE]]
```

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:01:37 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:01:37 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6863f851.050a0220.acf46.8176@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
----------------
skatrak wrote:

It's assumed that `recType != nullptr` below, so either `assert` this is the case by using `cast` or we need to handle that situation.
```suggestion
    auto recType = mlir::cast<fir::RecordType>(
```

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Tue Jul  1 08:02:41 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 08:02:41 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (PR #146359)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146359@github.com>
Message-ID: <6863f891.170a0220.1ecc5b.00de@mx.google.com>


kparzysz wrote:

I've done `make -C b/x86/tools/flang/lib/Semantics/ clean` followed by `time make -C b/x86/tools/flang/lib/Semantics/ all -j8`.

This PR: 4m55s
main: 4m51s

https://github.com/llvm/llvm-project/pull/146359

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:30 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:30 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c622.630a0220.2e2b82.8e94@mx.google.com>


https://github.com/skatrak edited https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:31 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c623.170a0220.22c045.07cf@mx.google.com>


================
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic control options for AMD gpu
----------------
skatrak wrote:

```suggestion
  /// Atomic control options
```

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:31 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c623.170a0220.f5d04.94b3@mx.google.com>


https://github.com/skatrak commented:

Thank you Anchu, I only have some small suggestions and a couple of questions.

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:31 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c623.170a0220.e1fcc.9845@mx.google.com>


================
@@ -58,6 +58,14 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and getters for atomic control options.
----------------
skatrak wrote:

Nit: All other getters/setters here are documented individually using doxygen-formatted comments. I'd suggest doing the same, even if a chunk of the description is going to repeat across multiple functions.

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:31 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c623.170a0220.39f238.1470@mx.google.com>


================
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic control options for AMD gpu
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
----------------
skatrak wrote:

Nit: I think it makes sense to include "atomic" within variables names and also associated getters/setters and attributes, to avoid confusion as to what these flags apply to.
```suggestion
  bool atomicIgnoreDenormalMode = false;
  bool atomicRemoteMemory = false;
  bool atomicFineGrainedMemory = false;
```

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:31 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c623.170a0220.38aae.9152@mx.google.com>


================
@@ -54,6 +54,21 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
----------------
skatrak wrote:

Nit: This definition should go before `DeclareTargetAttr`, since this file is alphabetically sorted.

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:31 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c623.170a0220.af388.8a39@mx.google.com>


================
@@ -88,6 +88,36 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
----------------
skatrak wrote:

Nit: Generally, dots are used to separate "scopes" in some hierarchy. I'd suggest following the `<dialect> <dot> <option name>` convention here.
```suggestion
    "fir.atomic_ignore_denormal_mode";
```

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:31 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c623.050a0220.e1bd8.2d1f@mx.google.com>


================
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          AtomicControlAttr:$atomic_control),
----------------
skatrak wrote:

Why have you made this attribute required? There's a clear default for it (all flags set to false) and, even in the unit tests you had to update because of this choice, the attribute is specified but empty.

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:31 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c623.170a0220.226bac.7529@mx.google.com>


================
@@ -88,6 +88,36 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+}
+
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
----------------
skatrak wrote:

```suggestion
static constexpr const char *fineGrainedMemoryName = "fir.atomic_fine_grained_memory";
```

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:34 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:34 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c626.050a0220.7d2ef.1ae8@mx.google.com>


================
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          AtomicControlAttr:$atomic_control),
----------------
skatrak wrote:

Does this new attribute only actually apply to `omp.atomic.update` or should we be adding it to the other atomic operations (`omp.atomic.read`, `omp.atomic.write` and `omp.atomic.capture`)?

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:33 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:33 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c625.170a0220.229da7.9c12@mx.google.com>


================
@@ -6654,6 +6654,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
+  if (targetOpts.ignoreDenormalMode)
+    fir::setIgnoreDenormalMode(*module);
----------------
skatrak wrote:

```suggestion
  fir::setIgnoreDenormalMode(*module, targetOpts.ignoreDenormalMode);
```

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:34 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:34 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c626.170a0220.325eb1.0dc4@mx.google.com>


================
@@ -58,6 +58,14 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and getters for atomic control options.
+void setIgnoreDenormalMode(mlir::ModuleOp mod);
----------------
skatrak wrote:

Setters should allow us to specify a value. Their implementation, when set to `false` would have to remove the associated unit attribute if it's present.
```suggestion
void setIgnoreDenormalMode(mlir::ModuleOp mod, bool value);
```

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 04:27:34 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 04:27:34 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6863c626.050a0220.3cd3d2.2256@mx.google.com>


================
@@ -88,6 +88,36 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+}
+
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
+void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
+}
+static constexpr const char *remoteMemoryName = "fir.remote.memory";
----------------
skatrak wrote:

```suggestion
static constexpr const char *remoteMemoryName = "fir.atomic_remote_memory";
```

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Tue Jul  1 06:46:03 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:46:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (PR #146359)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146359@github.com>
Message-ID: <6863e69b.630a0220.2d9aaf.a67c@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/146359

>From 251552277ca266b88e6bcbcf0ba59fd576495911 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 25 Jun 2025 15:25:45 -0500
Subject: [PATCH 1/4] [flang][OpenMP] Split check-omp-structure.cpp into
 smaller files, NFC

Create these new files in flang/lib/Semantics:
  openmp-utils.cpp/.h         - Common utilities
  check-omp-atomic.cpp        - Atomic-related checks
  check-omp-loop.cpp          - Loop constructs/clauses
  check-omp-metadirective.cpp - Metadirective-related checks

Update lists of included headers, std in particular.
---
 flang/lib/Semantics/CMakeLists.txt            |    8 +-
 flang/lib/Semantics/check-omp-atomic.cpp      | 1291 ++++++++
 flang/lib/Semantics/check-omp-loop.cpp        |  655 ++++
 .../lib/Semantics/check-omp-metadirective.cpp |  544 ++++
 flang/lib/Semantics/check-omp-structure.cpp   | 2818 +----------------
 flang/lib/Semantics/check-omp-structure.h     |    8 -
 flang/lib/Semantics/openmp-utils.cpp          |  389 +++
 flang/lib/Semantics/openmp-utils.h            |   62 +
 8 files changed, 3012 insertions(+), 2763 deletions(-)
 create mode 100644 flang/lib/Semantics/check-omp-atomic.cpp
 create mode 100644 flang/lib/Semantics/check-omp-loop.cpp
 create mode 100644 flang/lib/Semantics/check-omp-metadirective.cpp
 create mode 100644 flang/lib/Semantics/openmp-utils.cpp
 create mode 100644 flang/lib/Semantics/openmp-utils.h

diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index c0fda3631c01f..109bc2dbb8569 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -20,6 +20,9 @@ add_flang_library(FortranSemantics
   check-io.cpp
   check-namelist.cpp
   check-nullify.cpp
+  check-omp-atomic.cpp
+  check-omp-loop.cpp
+  check-omp-metadirective.cpp
   check-omp-structure.cpp
   check-purity.cpp
   check-return.cpp
@@ -34,12 +37,13 @@ add_flang_library(FortranSemantics
   mod-file.cpp
   openmp-dsa.cpp
   openmp-modifiers.cpp
+  openmp-utils.cpp
   pointer-assignment.cpp
   program-tree.cpp
-  resolve-labels.cpp
   resolve-directives.cpp
-  resolve-names-utils.cpp
+  resolve-labels.cpp
   resolve-names.cpp
+  resolve-names-utils.cpp
   rewrite-parse-tree.cpp
   runtime-type-info.cpp
   scope.cpp
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
new file mode 100644
index 0000000000000..19425a69523d3
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -0,0 +1,1291 @@
+//===-- lib/Semantics/check-omp-atomic.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/ErrorHandling.h"
+
+#include <cassert>
+#include <list>
+#include <optional>
+#include <string_view>
+#include <tuple>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+namespace operation = Fortran::evaluate::operation;
+
+template <typename T, typename U>
+static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
+  return !(e == f);
+}
+
+// There is no consistent way to get the source of a given ActionStmt, so
+// extract the source information from Statement<ActionStmt> when we can,
+// and keep it around for error reporting in further analyses.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+struct AnalyzedCondStmt {
+  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
+  parser::CharBlock source;
+  SourcedActionStmt ift, iff;
+};
+
+static SourcedActionStmt GetActionStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+static SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
+// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
+// is that the ActionStmt will be either an assignment or a pointer-assignment,
+// otherwise return std::nullopt.
+// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
+// the "typedAssignment" is unset. This can happen if there are semantic errors
+// in the purported assignment.
+static std::optional<evaluate::Assignment> GetEvaluateAssignment(
+    const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
+
+  return common::visit(
+      [](auto &&s) -> std::optional<evaluate::Assignment> {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>) {
+          const TypedAssignment &typed{s.value().typedAssignment};
+          // ForwardOwningPointer                 typedAssignment
+          // `- GenericAssignmentWrapper          ^.get()
+          //    `- std::optional<Assignment>      ^->v
+          return typed.get()->v;
+        } else {
+          return std::nullopt;
+        }
+      },
+      x->u);
+}
+
+static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  // Extract the evaluate::Expr from ScalarLogicalExpr.
+  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
+    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
+    const parser::Expr &expr{logical.thing.thing.value()};
+    return GetEvaluateExpr(expr);
+  }};
+
+  // Recognize either
+  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
+  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
+
+  if (auto &&action{GetActionStmt(x)}) {
+    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
+            &action.stmt->u)}) {
+      const parser::IfStmt &s{ifs->value()};
+      auto &&maybeCond{
+          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
+      auto &thenStmt{
+          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
+      if (maybeCond) {
+        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
+            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
+            SourcedActionStmt{}};
+      }
+    }
+    return std::nullopt;
+  }
+
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    if (auto *ifc{
+            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
+      using ElseBlock = parser::IfConstruct::ElseBlock;
+      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
+      const parser::IfConstruct &s{ifc->value()};
+
+      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
+        // Not expecting any else-if statements.
+        return std::nullopt;
+      }
+      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
+      auto &&maybeCond{getFromLogical(
+          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
+      if (!maybeCond) {
+        return std::nullopt;
+      }
+
+      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)),
+            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
+        if (result.ift.stmt && result.iff.stmt) {
+          return result;
+        }
+      } else {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
+        if (result.ift.stmt) {
+          return result;
+        }
+      }
+    }
+    return std::nullopt;
+  }
+
+  return std::nullopt;
+}
+
+static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
+    parser::CharBlock source) {
+  // Find => in the range, if not found, find = that is not a part of
+  // <=, >=, ==, or /=.
+  auto trim{[](std::string_view v) {
+    const char *begin{v.data()};
+    const char *end{begin + v.size()};
+    while (*begin == ' ' && begin != end) {
+      ++begin;
+    }
+    while (begin != end && end[-1] == ' ') {
+      --end;
+    }
+    assert(begin != end && "Source should not be empty");
+    return parser::CharBlock(begin, end - begin);
+  }};
+
+  std::string_view sv(source.begin(), source.size());
+
+  if (auto where{sv.find("=>")}; where != sv.npos) {
+    std::string_view lhs(sv.data(), where);
+    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
+    return std::make_pair(trim(lhs), trim(rhs));
+  }
+
+  // Go backwards, since all the exclusions above end with a '='.
+  for (size_t next{source.size()}; next > 1; --next) {
+    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
+      std::string_view lhs(sv.data(), next - 1);
+      std::string_view rhs(sv.data() + next, sv.size() - next);
+      return std::make_pair(trim(lhs), trim(rhs));
+    }
+  }
+  llvm_unreachable("Could not find assignment operator");
+}
+
+static bool IsCheckForAssociated(const SomeExpr &cond) {
+  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
+}
+
+static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
+  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
+  // for example.
+  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
+}
+
+static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
+  if (value) {
+    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
+        evaluate::GenericExprWrapper::Deleter);
+  }
+}
+
+static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
+    std::optional<evaluate::Assignment> value) {
+  if (value) {
+    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
+        evaluate::GenericAssignmentWrapper::Deleter);
+  }
+}
+
+static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
+    int what,
+    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
+  parser::OpenMPAtomicConstruct::Analysis::Op operation;
+  operation.what = what;
+  SetAssignment(operation.assign, maybeAssign);
+  return operation;
+}
+
+static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
+    const SomeExpr &atom, const MaybeExpr &cond,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
+  // Defined in flang/include/flang/Parser/parse-tree.h
+  //
+  // struct Analysis {
+  //   struct Kind {
+  //     static constexpr int None = 0;
+  //     static constexpr int Read = 1;
+  //     static constexpr int Write = 2;
+  //     static constexpr int Update = Read | Write;
+  //     static constexpr int Action = 3; // Bits containing N, R, W, U
+  //     static constexpr int IfTrue = 4;
+  //     static constexpr int IfFalse = 8;
+  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
+  //   };
+  //   struct Op {
+  //     int what;
+  //     TypedAssignment assign;
+  //   };
+  //   TypedExpr atom, cond;
+  //   Op op0, op1;
+  // };
+
+  parser::OpenMPAtomicConstruct::Analysis an;
+  SetExpr(an.atom, atom);
+  SetExpr(an.cond, cond);
+  an.op0 = std::move(op0);
+  an.op1 = std::move(op1);
+  return an;
+}
+
+/// Check if `expr` satisfies the following conditions for x and v:
+///
+/// [6.0:189:10-12]
+/// - x and v (as applicable) are either scalar variables or
+///   function references with scalar data pointer result of non-character
+///   intrinsic type or variables that are non-polymorphic scalar pointers
+///   and any length type parameter must be constant.
+void OmpStructureChecker::CheckAtomicType(
+    SymbolRef sym, parser::CharBlock source, std::string_view name) {
+  const DeclTypeSpec *typeSpec{sym->GetType()};
+  if (!typeSpec) {
+    return;
+  }
+
+  if (!IsPointer(sym)) {
+    using Category = DeclTypeSpec::Category;
+    Category cat{typeSpec->category()};
+    if (cat == Category::Character) {
+      context_.Say(source,
+          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
+    } else if (cat != Category::Numeric && cat != Category::Logical) {
+      context_.Say(source,
+          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
+    }
+    return;
+  }
+
+  // Variable is a pointer.
+  if (typeSpec->IsPolymorphic()) {
+    context_.Say(source,
+        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
+        name);
+    return;
+  }
+
+  // Go over all length parameters, if any, and check if they are
+  // explicit.
+  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
+    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
+          // "entry" is a map entry
+          return entry.second.isLen() && !entry.second.isExplicit();
+        })) {
+      context_.Say(source,
+          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
+          name);
+    }
+  }
+}
+
+void OmpStructureChecker::CheckAtomicVariable(
+    const SomeExpr &atom, parser::CharBlock source) {
+  if (atom.Rank() != 0) {
+    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
+        atom.AsFortran());
+  }
+
+  std::vector<SomeExpr> dsgs{GetAllDesignators(atom)};
+  assert(dsgs.size() == 1 && "Should have a single top-level designator");
+  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
+
+  CheckAtomicType(syms.back(), source, atom.AsFortran());
+
+  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
+    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
+        atom.AsFortran());
+  }
+}
+
+void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
+    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
+    parser::CharBlock source) {
+  if (auto *expr{HasStorageOverlap(base, exprs)}) {
+    context_.Say(source,
+        "Within atomic operation %s and %s access the same storage"_warn_en_US,
+        base.AsFortran(), expr->AsFortran());
+  }
+}
+
+void OmpStructureChecker::ErrorShouldBeVariable(
+    const MaybeExpr &expr, parser::CharBlock source) {
+  if (expr) {
+    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
+        expr->AsFortran());
+  } else {
+    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
+  }
+}
+
+std::pair<const parser::ExecutionPartConstruct *,
+    const parser::ExecutionPartConstruct *>
+OmpStructureChecker::CheckUpdateCapture(
+    const parser::ExecutionPartConstruct *ec1,
+    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
+  // Decide which statement is the atomic update and which is the capture.
+  //
+  // The two allowed cases are:
+  //   x = ...      atomic-var = ...
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  // or
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  //   x = ...      atomic-var = ...
+  //
+  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
+  // (which makes more sense, as it captures the original value of the atomic
+  // variable).
+  //
+  // If the two statements don't fit these criteria, return a pair of default-
+  // constructed values.
+  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
+      const parser::ExecutionPartConstruct *>;
+
+  SourcedActionStmt act1{GetActionStmt(ec1)};
+  SourcedActionStmt act2{GetActionStmt(ec2)};
+  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
+  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
+  if (!maybeAssign1 || !maybeAssign2) {
+    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
+      context_.Say(source,
+          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
+    }
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  auto as1{*maybeAssign1}, as2{*maybeAssign2};
+
+  auto isUpdateCapture{
+      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
+        return IsSameOrConvertOf(c.rhs, u.lhs);
+      }};
+
+  // Do some checks that narrow down the possible choices for the update
+  // and the capture statements. This will help to emit better diagnostics.
+  // 1. An assignment could be an update (cbu) if the left-hand side is a
+  //    subexpression of the right-hand side.
+  // 2. An assignment could be a capture (cbc) if the right-hand side is
+  //    a variable (or a function ref), with potential type conversions.
+  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
+  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
+  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
+  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
+
+  // We want to diagnose cases where both assignments cannot be an update,
+  // or both cannot be a capture, as well as cases where either assignment
+  // cannot be any of these two.
+  //
+  // If we organize these boolean values into a matrix
+  //   |cbu1 cbu2|
+  //   |cbc1 cbc2|
+  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
+  // row or column, including the case where everything is zero. All these
+  // cases correspond to the determinant of the matrix being 0, which suggests
+  // that checking the det may be a convenient diagnostic check. There is only
+  // one additional case where the det is 0, which is when the matrix is all 1
+  // ("true"). The "all true" case represents the situation where both
+  // assignments could be an update as well as a capture. On the other hand,
+  // whenever det != 0, the roles of the update and the capture can be
+  // unambiguously assigned to as1 and as2 [1].
+  //
+  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
+  // det = 0, leaving 6 cases where det != 0:
+  //   0 1   0 1   1 0   1 0   1 1   1 1
+  //   1 0   1 1   0 1   1 1   0 1   1 0
+  // In each case the classification is unambiguous.
+
+  //     |cbu1 cbu2|
+  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
+  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
+
+  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
+                                  const std::string &expr) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
+        expr);
+  }};
+
+  auto errorNeitherWorks{[&]() {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
+  }};
+
+  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
+    // If det != 0, then the checks unambiguously suggest a specific
+    // categorization.
+    // If det == 0, then this function should be called only if the
+    // checks haven't ruled out any possibility, i.e. when both assigments
+    // could still be either updates or captures.
+    if (det > 0) {
+      // as1 is update, as2 is capture
+      if (isUpdateCapture(as1, as2)) {
+        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
+      } else {
+        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else if (det < 0) {
+      // as2 is update, as1 is capture
+      if (isUpdateCapture(as2, as1)) {
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      } else {
+        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else {
+      bool updateFirst{isUpdateCapture(as1, as2)};
+      bool captureFirst{isUpdateCapture(as2, as1)};
+      if (updateFirst && captureFirst) {
+        // If both assignment could be the update and both could be the
+        // capture, emit a warning about the ambiguity.
+        context_.Say(act1.source,
+            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      }
+      if (updateFirst != captureFirst) {
+        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
+        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
+        return std::make_pair(upd, cap);
+      }
+      assert(!updateFirst && !captureFirst);
+      errorNeitherWorks();
+      return std::make_pair(nullptr, nullptr);
+    }
+  }};
+
+  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
+    return makeSelectionFromDet(det);
+  }
+  assert(det == 0 && "Prior checks should have covered det != 0");
+
+  // If neither of the statements is an RMW update, it could still be a
+  // "write" update. Pretty much any assignment can be a write update, so
+  // recompute det with cbu1 = cbu2 = true.
+  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
+    return makeSelectionFromDet(writeDet);
+  }
+
+  // It's only errors from here on.
+
+  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
+    errorNeitherWorks();
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // The remaining cases are that
+  // - no candidate for update, or for capture,
+  // - one of the assigments cannot be anything.
+
+  if (!cbu1 && !cbu2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  } else if (!cbc1 && !cbc2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
+    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
+    context_.Say(src,
+        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // All cases should have been covered.
+  llvm_unreachable("Unchecked condition");
+}
+
+void OmpStructureChecker::CheckAtomicCaptureAssignment(
+    const evaluate::Assignment &capture, const SomeExpr &atom,
+    parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &cap{capture.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, rsrc);
+    // This part should have been checked prior to calling this function.
+    assert(*GetConvertInput(capture.rhs) == atom &&
+        "This cannot be a capture assignment");
+    CheckStorageOverlap(atom, {cap}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicReadAssignment(
+    const evaluate::Assignment &read, parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+
+  if (auto maybe{GetConvertInput(read.rhs)}) {
+    const SomeExpr &atom{*maybe};
+
+    if (!IsVarOrFunctionRef(atom)) {
+      ErrorShouldBeVariable(atom, rsrc);
+    } else {
+      CheckAtomicVariable(atom, rsrc);
+      CheckStorageOverlap(atom, {read.lhs}, source);
+    }
+  } else {
+    ErrorShouldBeVariable(read.rhs, rsrc);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWriteAssignment(
+    const evaluate::Assignment &write, parser::CharBlock source) {
+  // [6.0:190:13-15]
+  // A write structured block is write-statement, a write statement that has
+  // one of the following forms:
+  //   x = expr
+  //   x => expr
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{write.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, lsrc);
+    CheckStorageOverlap(atom, {write.rhs}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateAssignment(
+    const evaluate::Assignment &update, parser::CharBlock source) {
+  // [6.0:191:1-7]
+  // An update structured block is update-statement, an update statement
+  // that has one of the following forms:
+  //   x = x operator expr
+  //   x = expr operator x
+  //   x = intrinsic-procedure-name (x)
+  //   x = intrinsic-procedure-name (x, expr-list)
+  //   x = intrinsic-procedure-name (expr-list, x)
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{update.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, lsrc);
+
+  std::pair<operation::Operator, std::vector<SomeExpr>> top{
+      operation::Operator::Unknown, {}};
+  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
+    top = GetTopLevelOperation(*maybeInput);
+  }
+  switch (top.first) {
+  case operation::Operator::Add:
+  case operation::Operator::Sub:
+  case operation::Operator::Mul:
+  case operation::Operator::Div:
+  case operation::Operator::And:
+  case operation::Operator::Or:
+  case operation::Operator::Eqv:
+  case operation::Operator::Neqv:
+  case operation::Operator::Min:
+  case operation::Operator::Max:
+  case operation::Operator::Identity:
+    break;
+  case operation::Operator::Call:
+    context_.Say(source,
+        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Convert:
+    context_.Say(source,
+        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Intrinsic:
+    context_.Say(source,
+        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Constant:
+  case operation::Operator::Unknown:
+    context_.Say(
+        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(source,
+        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
+        operation::ToString(top.first));
+    return;
+  }
+  // Check how many times `atom` occurs as an argument, if it's a subexpression
+  // of an argument, and collect the non-atom arguments.
+  std::vector<SomeExpr> nonAtom;
+  MaybeExpr subExpr;
+  auto atomCount{[&]() {
+    int count{0};
+    for (const SomeExpr &arg : top.second) {
+      if (IsSameOrConvertOf(arg, atom)) {
+        ++count;
+      } else {
+        if (!subExpr && IsSubexpressionOf(atom, arg)) {
+          subExpr = arg;
+        }
+        nonAtom.push_back(arg);
+      }
+    }
+    return count;
+  }()};
+
+  bool hasError{false};
+  if (subExpr) {
+    context_.Say(rsrc,
+        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
+        atom.AsFortran(), subExpr->AsFortran());
+    hasError = true;
+  }
+  if (top.first == operation::Operator::Identity) {
+    // This is "x = y".
+    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
+          atom.AsFortran());
+      hasError = true;
+    }
+  } else {
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    } else if (atomCount > 1) {
+      context_.Say(rsrc,
+          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    }
+  }
+
+  if (!hasError) {
+    CheckStorageOverlap(atom, nonAtom, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
+    const SomeExpr &cond, parser::CharBlock condSource,
+    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
+  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
+  const SomeExpr &atom{assign.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, arsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, alsrc);
+
+  auto top{GetTopLevelOperation(cond)};
+  // Missing arguments to operations would have been diagnosed by now.
+
+  switch (top.first) {
+  case operation::Operator::Associated:
+    if (atom != top.second.front()) {
+      context_.Say(assignSource,
+          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
+    }
+    break;
+  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
+  case operation::Operator::Eq:
+  case operation::Operator::Eqv:
+  // x ordop expr | expr ordop x
+  case operation::Operator::Lt:
+  case operation::Operator::Gt: {
+    const SomeExpr &arg0{top.second[0]};
+    const SomeExpr &arg1{top.second[1]};
+    if (IsSameOrConvertOf(arg0, atom)) {
+      CheckStorageOverlap(atom, {arg1}, condSource);
+    } else if (IsSameOrConvertOf(arg1, atom)) {
+      CheckStorageOverlap(atom, {arg0}, condSource);
+    } else {
+      assert(top.first != operation::Operator::Identity &&
+          "Handle this separately");
+      context_.Say(assignSource,
+          "An argument of the %s operator should be the target of the assignment"_err_en_US,
+          operation::ToString(top.first));
+    }
+    break;
+  }
+  case operation::Operator::Identity:
+  case operation::Operator::True:
+  case operation::Operator::False:
+    break;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(condSource,
+        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
+        operation::ToString(top.first));
+    break;
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
+    const AnalyzedCondStmt &update, parser::CharBlock source) {
+  // The condition/statements must be:
+  // - cond: x equalop e      ift: x =  d     iff: -
+  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
+  // - cond: associated(x)    ift: x => expr  iff: -
+  // - cond: associated(x, e) ift: x => expr  iff: -
+
+  // The if-true statement must be present, and must be an assignment.
+  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
+  if (!maybeAssign) {
+    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
+    } else {
+      context_.Say(
+          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
+    }
+    return;
+  }
+  const evaluate::Assignment assign{*maybeAssign};
+  const SomeExpr &atom{assign.lhs};
+
+  CheckAtomicConditionalUpdateAssignment(
+      update.cond, update.source, assign, update.ift.source);
+
+  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
+
+  if (update.iff) {
+    context_.Say(update.iff.source,
+        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateOnly(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeUpdate->lhs};
+      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdate(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // Allowable forms are (single-statement):
+  // - if ...
+  // - x = (... ? ... : x)
+  // and two-statement:
+  // - r = cond ; if (r) ...
+
+  const parser::ExecutionPartConstruct *ust{nullptr}; // update
+  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
+
+  if (body.size() == 1) {
+    ust = &body.front();
+  } else if (body.size() == 2) {
+    cst = &body.front();
+    ust = &body.back();
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
+    return;
+  }
+
+  // Flang doesn't support conditional-expr yet, so all update statements
+  // are if-statements.
+
+  // IfStmt:        if (...) ...
+  // IfConstruct:   if (...) then ... endif
+  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
+  if (!maybeUpdate) {
+    context_.Say(source,
+        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
+    return;
+  }
+
+  AnalyzedCondStmt &update{*maybeUpdate};
+
+  if (SourcedActionStmt action{GetActionStmt(cst)}) {
+    // The "condition" statement must be `r = cond`.
+    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
+      if (maybeCond->lhs != update.cond) {
+        context_.Say(update.source,
+            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
+            maybeCond->lhs.AsFortran());
+      } else {
+        // If it's "r = ...; if (r) ..." then put the original condition
+        // in `update`.
+        update.cond = maybeCond->rhs;
+      }
+    } else {
+      context_.Say(action.source,
+          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
+    }
+  }
+
+  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
+
+  CheckAtomicConditionalUpdateStmt(update, source);
+  if (IsCheckForAssociated(update.cond)) {
+    if (!IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
+    }
+  } else {
+    if (IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
+    }
+  }
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
+      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
+      MakeAtomicAnalysisOp(Analysis::None));
+}
+
+void OmpStructureChecker::CheckAtomicUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() != 2) {
+    context_.Say(source,
+        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
+    return;
+  }
+
+  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
+  if (!uec || !cec) {
+    // Diagnostics already emitted.
+    return;
+  }
+  SourcedActionStmt uact{GetActionStmt(uec)};
+  SourcedActionStmt cact{GetActionStmt(cec)};
+  // The "dereferences" of std::optional are guaranteed to be valid after
+  // CheckUpdateCapture.
+  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
+  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
+
+  const SomeExpr &atom{update.lhs};
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  int action;
+
+  if (IsMaybeAtomicWrite(update)) {
+    action = Analysis::Write;
+    CheckAtomicWriteAssignment(update, uact.source);
+  } else {
+    action = Analysis::Update;
+    CheckAtomicUpdateAssignment(update, uact.source);
+  }
+  CheckAtomicCaptureAssignment(capture, atom, cact.source);
+
+  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
+    context_.Say(cact.source,
+        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+    return;
+  }
+
+  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(action, update),
+        MakeAtomicAnalysisOp(Analysis::Read, capture));
+  } else {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(Analysis::Read, capture),
+        MakeAtomicAnalysisOp(action, update));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // There are two different variants of this:
+  // (1) conditional-update and capture separately:
+  //     This form only allows single-statement updates, i.e. the update
+  //     form "r = cond; if (r) ..." is not allowed.
+  // (2) conditional-update combined with capture in a single statement:
+  //     This form does allow the condition to be calculated separately,
+  //     i.e. "r = cond; if (r) ...".
+  // Regardless of what form it is, the actual update assignment is a
+  // proper write, i.e. "x = d", where d does not depend on x.
+
+  AnalyzedCondStmt update;
+  SourcedActionStmt capture;
+  bool captureAlways{true}, captureFirst{true};
+
+  auto extractCapture{[&]() {
+    capture = update.iff;
+    captureAlways = false;
+    update.iff = SourcedActionStmt{};
+  }};
+
+  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
+    // The non-update statement is either "r = cond" or the capture.
+    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
+      if (update.cond == maybeAssign->lhs) {
+        // If this is "r = cond; if (r) ...", then update the condition.
+        update.cond = maybeAssign->rhs;
+        update.source = action.source;
+        // In this form, the update and the capture are combined into
+        // an IF-THEN-ELSE statement.
+        extractCapture();
+      } else {
+        // Assume this is the capture-statement.
+        capture = action;
+      }
+    }
+  }};
+
+  if (body.size() == 2) {
+    // This could be
+    // - capture; conditional-update (in any order), or
+    // - r = cond; if (r) capture-update
+    const parser::ExecutionPartConstruct *st1{&body.front()};
+    const parser::ExecutionPartConstruct *st2{&body.back()};
+    // In either case, the conditional statement can be analyzed by
+    // AnalyzeConditionalStmt, whereas the other statement cannot.
+    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
+      update = *maybeUpdate1;
+      classifyNonUpdate(GetActionStmt(st2));
+      captureFirst = false;
+    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
+      update = *maybeUpdate2;
+      classifyNonUpdate(GetActionStmt(st1));
+    } else {
+      // None of the statements are conditional, this rules out the
+      // "r = cond; if (r) ..." and the "capture + conditional-update"
+      // variants. This could still be capture + write (which is classified
+      // as conditional-update-capture in the spec).
+      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
+      if (!uec || !cec) {
+        // Diagnostics already emitted.
+        return;
+      }
+      SourcedActionStmt uact{GetActionStmt(uec)};
+      SourcedActionStmt cact{GetActionStmt(cec)};
+      update.ift = uact;
+      capture = cact;
+      if (uec == st1) {
+        captureFirst = false;
+      }
+    }
+  } else if (body.size() == 1) {
+    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
+      update = *maybeUpdate;
+      // This is the form with update and capture combined into an IF-THEN-ELSE
+      // statement. The capture-statement is always the ELSE branch.
+      extractCapture();
+    } else {
+      goto invalid;
+    }
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
+    return;
+  invalid:
+    context_.Say(source,
+        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
+    return;
+  }
+
+  // The update must have a form `x = d` or `x => d`.
+  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
+    const SomeExpr &atom{maybeWrite->lhs};
+    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
+    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
+      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
+
+      if (IsPointerAssignment(*maybeWrite) !=
+          IsPointerAssignment(*maybeCapture)) {
+        context_.Say(capture.source,
+            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+        return;
+      }
+    } else {
+      if (!IsAssignment(capture.stmt)) {
+        context_.Say(capture.source,
+            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
+      }
+      return;
+    }
+  } else {
+    if (!IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
+    }
+    return;
+  }
+
+  // update.iff should be empty here, the capture statement should be
+  // stored in "capture".
+
+  // Fill out the analysis in the AST node.
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  bool condUnused{std::visit(
+      [](auto &&s) {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
+          return true;
+        } else {
+          return false;
+        }
+      },
+      update.cond.u)};
+
+  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
+  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
+
+  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
+  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
+
+  if (captureFirst) {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
+  } else {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicRead(
+    const parser::OpenMPAtomicConstruct &x) {
+  // [6.0:190:5-7]
+  // A read structured block is read-statement, a read statement that has one
+  // of the following forms:
+  //   v = x
+  //   v => x
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Read cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
+      CheckAtomicReadAssignment(*maybeRead, action.source);
+
+      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
+        const SomeExpr &atom{*maybe};
+        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
+            MakeAtomicAnalysisOp(Analysis::None));
+      }
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC READ operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWrite(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Write cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeWrite->lhs};
+      CheckAtomicWriteAssignment(*maybeWrite, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC WRITE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdate(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &block{std::get<parser::Block>(x.t)};
+
+  bool isConditional{x.IsCompare()};
+  bool isCapture{x.IsCapture()};
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (isConditional && isCapture) {
+    CheckAtomicConditionalUpdateCapture(x, body, x.source);
+  } else if (isConditional) {
+    CheckAtomicConditionalUpdate(x, body, x.source);
+  } else if (isCapture) {
+    CheckAtomicUpdateCapture(x, body, x.source);
+  } else { // update-only
+    CheckAtomicUpdateOnly(x, body, x.source);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
+  if (visitedAtomicSource_.empty())
+    visitedAtomicSource_ = x.source;
+
+  // All of the following groups have the "exclusive" property, i.e. at
+  // most one clause from each group is allowed.
+  // The exclusivity-checking code should eventually be unified for all
+  // clauses, with clause groups defined in OMP.td.
+  std::array atomic{llvm::omp::Clause::OMPC_read,
+      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
+  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
+      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
+      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
+
+  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
+                          std::string_view name,
+                          const parser::OmpClauseList &clauses) {
+    const parser::OmpClause *present{nullptr};
+    for (const parser::OmpClause &clause : clauses.v) {
+      llvm::omp::Clause id{clause.Id()};
+      if (!llvm::is_contained(group, id)) {
+        continue;
+      }
+      if (present == nullptr) {
+        present = &clause;
+        continue;
+      } else if (id == present->Id()) {
+        // Ignore repetitions of the same clause, those will be diagnosed
+        // separately.
+        continue;
+      }
+      parser::MessageFormattedText txt(
+          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
+          name.data());
+      parser::Message message(clause.source, txt);
+      message.Attach(present->source,
+          "Previous clause from this group provided here"_en_US);
+      context_.Say(std::move(message));
+      return;
+    }
+  }};
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
+  llvm::omp::Clause kind{x.GetKind()};
+
+  checkExclusive(atomic, "atomic", dirSpec.Clauses());
+  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
+
+  switch (kind) {
+  case llvm::omp::Clause::OMPC_read:
+    CheckAtomicRead(x);
+    break;
+  case llvm::omp::Clause::OMPC_write:
+    CheckAtomicWrite(x);
+    break;
+  case llvm::omp::Clause::OMPC_update:
+    CheckAtomicUpdate(x);
+    break;
+  default:
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
new file mode 100644
index 0000000000000..f2aad9efdd8de
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -0,0 +1,655 @@
+//===-- lib/Semantics/check-omp-loop.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <variant>
+
+namespace {
+using namespace Fortran;
+
+class AssociatedLoopChecker {
+public:
+  AssociatedLoopChecker(
+      semantics::SemanticsContext &context, std::int64_t level)
+      : context_{context}, level_{level} {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::DoConstruct &dc) {
+    level_--;
+    const auto &doStmt{
+        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
+    const auto &constructName{
+        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
+    if (constructName) {
+      constructNamesAndLevels_.emplace(
+          constructName.value().ToString(), level_);
+    }
+    if (level_ >= 0) {
+      if (dc.IsDoWhile()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
+      }
+      if (!dc.GetLoopControl()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
+      }
+    }
+    return true;
+  }
+
+  void Post(const parser::DoConstruct &dc) { level_++; }
+
+  bool Pre(const parser::CycleStmt &cyclestmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (cyclestmt.v) {
+      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second > 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ > 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::ExitStmt &exitStmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (exitStmt.v) {
+      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ >= 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "EXIT statement terminates associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
+    source_ = &actionstmt.source;
+    return true;
+  }
+
+private:
+  semantics::SemanticsContext &context_;
+  const parser::CharBlock *source_;
+  std::int64_t level_;
+  std::map<std::string, std::int64_t> constructNamesAndLevels_;
+};
+} // namespace
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::HasInvalidDistributeNesting(
+    const parser::OpenMPLoopConstruct &x) {
+  bool violation{false};
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
+    // `distribute` region has to be nested
+    if (!CurrentDirectiveIsNested()) {
+      violation = true;
+    } else {
+      // `distribute` region has to be strictly nested inside `teams`
+      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+        violation = true;
+      }
+    }
+  }
+  if (violation) {
+    context_.Say(beginDir.source,
+        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
+        "region."_err_en_US);
+  }
+}
+void OmpStructureChecker::HasInvalidLoopBinding(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
+    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+    for (const auto &clause : clauseList.v) {
+      if (const auto *bindClause{
+              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
+        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
+          context_.Say(beginDir.source, msg);
+        }
+      }
+    }
+  };
+
+  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
+      CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
+        "strictly nested inside a `TEAMS` region."_err_en_US);
+  }
+
+  if (OmpDirectiveSet{
+          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
+          .test(beginDir.v)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
+        "combined with a `TEAMS` construct."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
+  // Check the following:
+  //  The only OpenMP constructs that can be encountered during execution of
+  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
+  // construct and the `ordered` construct with the `simd` clause.
+
+  // Check if the parent context has the SIMD clause
+  // Please note that we use GetContext() instead of GetContextParent()
+  // because PushContextAndClauseSets() has not been called on the
+  // current context yet.
+  // TODO: Check for declare simd regions.
+  bool eligibleSIMD{false};
+  common::visit(
+      common::visitors{
+          // Allow `!$OMP ORDERED SIMD`
+          [&](const parser::OpenMPBlockConstruct &c) {
+            const auto &beginBlockDir{
+                std::get<parser::OmpBeginBlockDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
+            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
+              const auto &clauses{
+                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
+              for (const auto &clause : clauses.v) {
+                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
+                  eligibleSIMD = true;
+                  break;
+                }
+              }
+            }
+          },
+          [&](const parser::OpenMPStandaloneConstruct &c) {
+            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
+                    &c.u)}) {
+              llvm::omp::Directive dirId{ssc->v.DirId()};
+              if (dirId == llvm::omp::Directive::OMPD_ordered) {
+                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
+                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
+                    eligibleSIMD = true;
+                    break;
+                  }
+                }
+              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
+                eligibleSIMD = true;
+              }
+            }
+          },
+          // Allowing SIMD and loop construct
+          [&](const parser::OpenMPLoopConstruct &c) {
+            const auto &beginLoopDir{
+                std::get<parser::OmpBeginLoopDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
+              eligibleSIMD = true;
+            }
+          },
+          [&](const parser::OpenMPAtomicConstruct &c) {
+            // Allow `!$OMP ATOMIC`
+            eligibleSIMD = true;
+          },
+          [&](const auto &c) {},
+      },
+      c.u);
+  if (!eligibleSIMD) {
+    context_.Say(parser::FindSourceLocation(c),
+        "The only OpenMP constructs that can be encountered during execution "
+        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
+        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
+        "construct with the `SIMD` clause."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
+  loopStack_.push_back(&x);
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  PushContextAndClauseSets(beginDir.source, beginDir.v);
+
+  // check matching, End directive is optional
+  if (const auto &endLoopDir{
+          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
+    const auto &endDir{
+        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
+
+    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
+
+    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
+  }
+
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    EnterDirectiveNest(SIMDNest);
+  }
+
+  // Combined target loop constructs are target device constructs. Keep track of
+  // whether any such construct has been visited to later check that REQUIRES
+  // directives for target-related options don't appear after them.
+  if (llvm::omp::allTargetSet.test(beginDir.v)) {
+    deviceConstructFound_ = true;
+  }
+
+  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
+    // 2.7.1 do-clause -> private-clause |
+    //                    firstprivate-clause |
+    //                    lastprivate-clause |
+    //                    linear-clause |
+    //                    reduction-clause |
+    //                    schedule-clause |
+    //                    collapse-clause |
+    //                    ordered-clause
+
+    // nesting check
+    HasInvalidWorksharingNesting(
+        beginDir.source, llvm::omp::nestedWorkshareErrSet);
+  }
+  SetLoopInfo(x);
+
+  if (const auto &doConstruct{
+          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
+    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+  }
+  CheckLoopItrVariableIsInt(x);
+  CheckAssociatedLoopConstraints(x);
+  HasInvalidDistributeNesting(x);
+  HasInvalidLoopBinding(x);
+  if (CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
+  }
+  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
+      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
+    CheckDistLinear(x);
+  }
+}
+
+const parser::Name OmpStructureChecker::GetLoopIndex(
+    const parser::DoConstruct *x) {
+  using Bounds = parser::LoopControl::Bounds;
+  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
+}
+
+void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
+  if (const auto &loopConstruct{
+          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+    const parser::DoConstruct *loop{&*loopConstruct};
+    if (loop && loop->IsDoNormal()) {
+      const parser::Name &itrVal{GetLoopIndex(loop)};
+      SetLoopIv(itrVal.symbol);
+    }
+  }
+}
+
+void OmpStructureChecker::CheckLoopItrVariableIsInt(
+    const parser::OpenMPLoopConstruct &x) {
+  if (const auto &loopConstruct{
+          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+
+    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+      if (loop->IsDoNormal()) {
+        const parser::Name &itrVal{GetLoopIndex(loop)};
+        if (itrVal.symbol) {
+          const auto *type{itrVal.symbol->GetType()};
+          if (!type->IsNumeric(TypeCategory::Integer)) {
+            context_.Say(itrVal.source,
+                "The DO loop iteration"
+                " variable must be of the type integer."_err_en_US,
+                itrVal.ToString());
+          }
+        }
+      }
+      // Get the next DoConstruct if block is not empty.
+      const auto &block{std::get<parser::Block>(loop->t)};
+      const auto it{block.begin()};
+      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                               : nullptr;
+    }
+  }
+}
+
+std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+  std::int64_t orderedCollapseLevel{1};
+  std::int64_t orderedLevel{1};
+  std::int64_t collapseLevel{1};
+
+  for (const auto &clause : clauseList.v) {
+    if (const auto *collapseClause{
+            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
+      if (const auto v{GetIntValue(collapseClause->v)}) {
+        collapseLevel = *v;
+      }
+    }
+    if (const auto *orderedClause{
+            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
+      if (const auto v{GetIntValue(orderedClause->v)}) {
+        orderedLevel = *v;
+      }
+    }
+  }
+  if (orderedLevel >= collapseLevel) {
+    orderedCollapseLevel = orderedLevel;
+  } else {
+    orderedCollapseLevel = collapseLevel;
+  }
+  return orderedCollapseLevel;
+}
+
+void OmpStructureChecker::CheckAssociatedLoopConstraints(
+    const parser::OpenMPLoopConstruct &x) {
+  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
+  AssociatedLoopChecker checker{context_, ordCollapseLevel};
+  parser::Walk(x, checker);
+}
+
+void OmpStructureChecker::CheckDistLinear(
+    const parser::OpenMPLoopConstruct &x) {
+
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  SymbolSourceMap indexVars;
+
+  // Collect symbols of all the variables from linear clauses
+  for (auto &clause : clauses.v) {
+    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
+      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
+      GetSymbolsInObjectList(objects, indexVars);
+    }
+  }
+
+  if (!indexVars.empty()) {
+    // Get collapse level, if given, to find which loops are "associated."
+    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
+    // Include the top loop if no collapse is specified
+    if (collapseVal == 0) {
+      collapseVal = 1;
+    }
+
+    // Match the loop index variables with the collected symbols from linear
+    // clauses.
+    if (const auto &loopConstruct{
+            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+        if (loop->IsDoNormal()) {
+          const parser::Name &itrVal{GetLoopIndex(loop)};
+          if (itrVal.symbol) {
+            // Remove the symbol from the collected set
+            indexVars.erase(&itrVal.symbol->GetUltimate());
+          }
+          collapseVal--;
+          if (collapseVal == 0) {
+            break;
+          }
+        }
+        // Get the next DoConstruct if block is not empty.
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                 : nullptr;
+      }
+    }
+
+    // Show error for the remaining variables
+    for (auto &[symbol, source] : indexVars) {
+      const Symbol &root{GetAssociationRoot(*symbol)};
+      context_.Say(source,
+          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
+          root.name());
+    }
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  // A few semantic checks for InScan reduction are performed below as SCAN
+  // constructs inside LOOP may add the relevant information. Scan reduction is
+  // supported only in loop constructs, so same checks are not applicable to
+  // other directives.
+  using ReductionModifier = parser::OmpReductionModifier;
+  for (const auto &clause : clauseList.v) {
+    if (const auto *reductionClause{
+            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
+      auto &modifiers{OmpGetModifiers(reductionClause->v)};
+      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
+      if (maybeModifier &&
+          maybeModifier->v == ReductionModifier::Value::Inscan) {
+        const auto &objectList{
+            std::get<parser::OmpObjectList>(reductionClause->v.t)};
+        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
+          if (auto &symbol = name->symbol) {
+            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
+                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
+              context_.Say(name->source,
+                  "List item %s must appear in EXCLUSIVE or "
+                  "INCLUSIVE clause of an "
+                  "enclosed SCAN directive"_err_en_US,
+                  name->ToString());
+            }
+          }
+        };
+        for (const auto &ompObj : objectList.v) {
+          common::visit(
+              common::visitors{
+                  [&](const parser::Designator &designator) {
+                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
+                            designator)}) {
+                      checkReductionSymbolInScan(name);
+                    }
+                  },
+                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
+              },
+              ompObj.u);
+        }
+      }
+    }
+  }
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    ExitDirectiveNest(SIMDNest);
+  }
+  dirContext_.pop_back();
+
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top{loopStack_.back()};
+  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
+  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+}
+
+void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
+  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  // 2.7.1 end-do -> END DO [nowait-clause]
+  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
+  case llvm::omp::Directive::OMPD_do:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
+    break;
+  case llvm::omp::Directive::OMPD_do_simd:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
+    break;
+  default:
+    // no clauses are allowed
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
+    dirContext_.pop_back();
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
+  unsigned version{context_.langOptions().OpenMPVersion};
+  llvm::omp::Directive dir{GetContext().directive};
+  parser::CharBlock clauseSource{GetContext().clauseSource};
+  const parser::OmpLinearModifier *linearMod{nullptr};
+
+  SymbolSourceMap symbols;
+  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  CheckCrayPointee(objects, "LINEAR", false);
+  GetSymbolsInObjectList(objects, symbols);
+
+  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
+    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      context_.Say(source,
+          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
+          symbol->name(), desc.name.str());
+    }
+  }};
+
+  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(x.v)};
+    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
+    if (linearMod) {
+      // 2.7 Loop Construct Restriction
+      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
+        context_.Say(clauseSource,
+            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
+            ContextDirectiveAsFortran());
+        return;
+      }
+
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      for (auto &[symbol, source] : symbols) {
+        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
+          CheckIntegerNoRef(symbol, source);
+        } else {
+          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
+              !IsPolymorphic(*symbol)) {
+            context_.Say(source,
+                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
+                symbol->name(), desc.name.str());
+          }
+        }
+        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
+            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
+          if (!IsDummy(*symbol) || IsValue(*symbol)) {
+            context_.Say(source,
+                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
+                desc.name.str(), symbol->name());
+          }
+        }
+      } // for (symbol, source)
+
+      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
+        context_.Say(OmpGetModifierSource(modifiers, linearMod),
+            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
+            ThisVersion(version));
+      }
+    }
+  }
+
+  // OpenMP 5.2: Ordered clause restriction
+  if (const auto *clause{
+          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
+    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
+    if (orderedClause.v) {
+      return;
+    }
+  }
+
+  // OpenMP 5.2: Linear clause Restrictions
+  for (auto &[symbol, source] : symbols) {
+    if (!linearMod) {
+      // Already checked this with the modifier present.
+      CheckIntegerNoRef(symbol, source);
+    }
+    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
+      context_.Say(source,
+          "The list item `%s` must be a dummy argument"_err_en_US,
+          symbol->name());
+    }
+    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
+      context_.Say(source,
+          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
+          symbol->name());
+    }
+    if (FindCommonBlockContaining(*symbol)) {
+      context_.Say(source,
+          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
+          symbol->name());
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
+  Base::Enter(x);
+  loopStack_.push_back(&x);
+}
+
+void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top = loopStack_.back();
+  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
+  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+  Base::Leave(x);
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-metadirective.cpp b/flang/lib/Semantics/check-omp-metadirective.cpp
new file mode 100644
index 0000000000000..bf27797a7c44a
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-metadirective.cpp
@@ -0,0 +1,544 @@
+//===-- lib/Semantics/check-omp-metadirective.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <variant>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
+  OmpVerifyModifiers(
+      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
+}
+
+void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
+  EnterDirectiveNest(ContextSelectorNest);
+
+  using SetName = parser::OmpTraitSetSelectorName;
+  std::map<SetName::Value, const SetName *> visited;
+
+  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
+    auto &name{std::get<SetName>(traitSet.t)};
+    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait set name %s in a context specifier"_err_en_US,
+          showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait set %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+    CheckTraitSetSelector(traitSet);
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
+  ExitDirectiveNest(ContextSelectorNest);
+}
+
+const std::list<parser::OmpTraitProperty> &
+OmpStructureChecker::GetTraitPropertyList(
+    const parser::OmpTraitSelector &trait) {
+  static const std::list<parser::OmpTraitProperty> empty{};
+  auto &[_, maybeProps]{trait.t};
+  if (maybeProps) {
+    using PropertyList = std::list<parser::OmpTraitProperty>;
+    return std::get<PropertyList>(maybeProps->t);
+  } else {
+    return empty;
+  }
+}
+
+std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
+    const parser::OmpTraitProperty &property) {
+  using MaybeClause = std::optional<llvm::omp::Clause>;
+
+  // The parser for OmpClause will only succeed if the clause was
+  // given with all required arguments.
+  // If this is a string or complex extension with a clause name,
+  // treat it as a clause and let the trait checker deal with it.
+
+  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
+    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
+    if (id != llvm::omp::Clause::OMPC_unknown) {
+      return id;
+    } else {
+      return std::nullopt;
+    }
+  }};
+
+  return common::visit( //
+      common::visitors{
+          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
+            return getClauseFromString(x.v);
+          },
+          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
+            return x.value().Id();
+          },
+          [&](const parser::ScalarExpr &x) -> MaybeClause {
+            return std::nullopt;
+          },
+          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
+            using ExtProperty = parser::OmpTraitPropertyExtension;
+            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
+              return getClauseFromString(name->v);
+            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
+              return getClauseFromString(
+                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
+            }
+            return std::nullopt;
+          },
+      },
+      property.u);
+}
+
+void OmpStructureChecker::CheckTraitSelectorList(
+    const std::list<parser::OmpTraitSelector> &traits) {
+  // [6.0:322:20]
+  // Each trait-selector-name may only be specified once in a trait selector
+  // set.
+
+  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
+  using TraitName = parser::OmpTraitSelectorName;
+  using BareName = decltype(TraitName::u);
+  std::map<BareName, const TraitName *> visited;
+
+  for (const parser::OmpTraitSelector &trait : traits) {
+    auto &name{std::get<TraitName>(trait.t)};
+
+    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait name %s in a trait set"_err_en_US, showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSetSelector(
+    const parser::OmpTraitSetSelector &traitSet) {
+
+  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
+  //
+  // Construct      |     Simd, directive-name |      Yes |       No |    No |
+  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
+  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
+  //                |      Extension, Requires |          |          |       |
+  //                |                   Vendor |          |          |       |
+  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
+  //                |                Kind, Uid |          |          |       |
+  // User           |                Condition |       No |       No |   Yes |
+
+  struct TraitSetConfig {
+    std::set<parser::OmpTraitSelectorName::Value> allowed;
+    bool allowsDirectiveTraits;
+    bool allowsExtensionTraits;
+    bool allowsScore;
+  };
+
+  using SName = parser::OmpTraitSetSelectorName::Value;
+  using TName = parser::OmpTraitSelectorName::Value;
+
+  static const std::map<SName, TraitSetConfig> configs{
+      {SName::Construct, //
+          {{TName::Simd}, true, false, false}},
+      {SName::Device, //
+          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
+      {SName::Implementation, //
+          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
+               TName::Vendor},
+              false, true, true}},
+      {SName::Target_Device, //
+          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
+               TName::Uid},
+              false, true, false}},
+      {SName::User, //
+          {{TName::Condition}, false, false, true}},
+  };
+
+  auto checkTraitSet{[&](const TraitSetConfig &config) {
+    auto &[setName, traits]{traitSet.t};
+    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
+
+    // Check if there are any duplicate traits.
+    CheckTraitSelectorList(traits);
+
+    for (const parser::OmpTraitSelector &trait : traits) {
+      // Don't use structured bindings here, because they cannot be captured
+      // before C++20.
+      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
+      auto &maybeProps =
+          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
+              trait.t);
+
+      // Check allowed traits
+      common::visit( //
+          common::visitors{
+              [&](parser::OmpTraitSelectorName::Value v) {
+                if (!config.allowed.count(v)) {
+                  context_.Say(traitName.source,
+                      "%s is not a valid trait for %s trait set"_err_en_US,
+                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
+                }
+              },
+              [&](llvm::omp::Directive) {
+                if (!config.allowsDirectiveTraits) {
+                  context_.Say(traitName.source,
+                      "Directive name is not a valid trait for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+              [&](const std::string &) {
+                if (!config.allowsExtensionTraits) {
+                  context_.Say(traitName.source,
+                      "Extension traits are not valid for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+          },
+          traitName.u);
+
+      // Check score
+      if (maybeProps) {
+        auto &[maybeScore, _]{maybeProps->t};
+        if (maybeScore) {
+          CheckTraitScore(*maybeScore);
+        }
+      }
+
+      // Check the properties of the individual traits
+      CheckTraitSelector(traitSet, trait);
+    }
+  }};
+
+  checkTraitSet(
+      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
+}
+
+void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
+  // [6.0:322:23]
+  // A score-expression must be a non-negative constant integer expression.
+  if (auto value{GetIntValue(score)}; !value || value < 0) {
+    context_.Say(score.source,
+        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
+  }
+}
+
+bool OmpStructureChecker::VerifyTraitPropertyLists(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  using PropertyList = std::list<parser::OmpTraitProperty>;
+  auto &[traitName, maybeProps]{trait.t};
+
+  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
+                             const std::string &message) {
+    bool foundInvalid{false};
+    for (const parser::OmpTraitProperty &prop : properties) {
+      if (!isValid(prop)) {
+        if (foundInvalid) {
+          context_.Say(
+              prop.source, "More invalid properties are present"_err_en_US);
+          break;
+        }
+        context_.Say(prop.source, "%s"_err_en_US, message);
+        foundInvalid = true;
+      }
+    }
+    return !foundInvalid;
+  }};
+
+  bool invalid{false};
+
+  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
+    // Directive-name traits don't have properties.
+    if (maybeProps) {
+      context_.Say(trait.source,
+          "Directive-name traits cannot have properties"_err_en_US);
+      invalid = true;
+    }
+  }
+  // Ignore properties on extension traits.
+
+  // See `TraitSelectorParser` in openmp-parser.cpp
+  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
+    switch (*v) {
+    // name-list properties
+    case parser::OmpTraitSelectorName::Value::Arch:
+    case parser::OmpTraitSelectorName::Value::Extension:
+    case parser::OmpTraitSelectorName::Value::Isa:
+    case parser::OmpTraitSelectorName::Value::Kind:
+    case parser::OmpTraitSelectorName::Value::Uid:
+    case parser::OmpTraitSelectorName::Value::Vendor:
+      if (maybeProps) {
+        auto isName{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isName, "Trait property should be a name");
+      }
+      break;
+    // clause-list
+    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
+    case parser::OmpTraitSelectorName::Value::Requires:
+    case parser::OmpTraitSelectorName::Value::Simd:
+      if (maybeProps) {
+        auto isClause{[&](const parser::OmpTraitProperty &prop) {
+          return GetClauseFromProperty(prop).has_value();
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isClause, "Trait property should be a clause");
+      }
+      break;
+    // expr-list
+    case parser::OmpTraitSelectorName::Value::Condition:
+    case parser::OmpTraitSelectorName::Value::Device_Num:
+      if (maybeProps) {
+        auto isExpr{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::ScalarExpr>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isExpr, "Trait property should be a scalar expression");
+      }
+      break;
+    } // switch
+  }
+
+  return !invalid;
+}
+
+void OmpStructureChecker::CheckTraitSelector(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  auto &[traitName, maybeProps]{trait.t};
+
+  // Only do the detailed checks if the property lists are valid.
+  if (VerifyTraitPropertyLists(traitSet, trait)) {
+    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
+        std::holds_alternative<std::string>(traitName.u)) {
+      // No properties here: directives don't have properties, and
+      // we don't implement any extension traits now.
+      return;
+    }
+
+    // Specific traits we want to check.
+    // Limitations:
+    // (1) The properties for these traits are defined in "Additional
+    // Definitions for the OpenMP API Specification". It's not clear how
+    // to define them in a portable way, and how to verify their validity,
+    // especially if they get replaced by their integer values (in case
+    // they are defined as enums).
+    // (2) These are entirely implementation-defined, and at the moment
+    // there is no known schema to validate these values.
+    auto v{std::get<TraitName::Value>(traitName.u)};
+    switch (v) {
+    case TraitName::Value::Arch:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Atomic_Default_Mem_Order:
+      CheckTraitADMO(traitSet, trait);
+      break;
+    case TraitName::Value::Condition:
+      CheckTraitCondition(traitSet, trait);
+      break;
+    case TraitName::Value::Device_Num:
+      CheckTraitDeviceNum(traitSet, trait);
+      break;
+    case TraitName::Value::Extension:
+      // Ignore
+      break;
+    case TraitName::Value::Isa:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Kind:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Requires:
+      CheckTraitRequires(traitSet, trait);
+      break;
+    case TraitName::Value::Simd:
+      CheckTraitSimd(traitSet, trait);
+      break;
+    case TraitName::Value::Uid:
+      // Unchecked, TBD(2)
+      break;
+    case TraitName::Value::Vendor:
+      // Unchecked, TBD(1)
+      break;
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitADMO(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single clause property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto clauseId{*GetClauseFromProperty(property)};
+    // Check that the clause belongs to the memory-order clause-set.
+    // Clause sets will hopefully be autogenerated at some point.
+    switch (clauseId) {
+    case llvm::omp::Clause::OMPC_acq_rel:
+    case llvm::omp::Clause::OMPC_acquire:
+    case llvm::omp::Clause::OMPC_relaxed:
+    case llvm::omp::Clause::OMPC_release:
+    case llvm::omp::Clause::OMPC_seq_cst:
+      break;
+    default:
+      context_.Say(property.source,
+          "%s trait requires a clause from the memory-order clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitCondition(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
+
+    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
+    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
+      context_.Say(property.source,
+          "%s trait requires a single LOGICAL expression"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitDeviceNum(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  }
+  // No other checks at the moment.
+}
+
+void OmpStructureChecker::CheckTraitRequires(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_requires, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause from the requirement clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSimd(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_declare_simd, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()),
+          parser::ToUpperCaseLetters(
+              getDirectiveName(llvm::omp::OMPD_declare_simd)));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
+  EnterDirectiveNest(MetadirectiveNest);
+  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
+}
+
+void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
+  ExitDirectiveNest(MetadirectiveNest);
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 3abb5a304b00c..30eff01256c61 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -7,27 +7,56 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
 #include "definable.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
-#include "flang/Evaluate/check-expression.h"
-#include "flang/Evaluate/expression.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
 #include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/scope.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+#include "flang/Support/Fortran-features.h"
+
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <iterator>
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
 #include <variant>
 
 namespace Fortran::semantics {
 
-template <typename T, typename U>
-static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
-  return !(e == f);
-}
+using namespace Fortran::semantics::omp;
 
 // Use when clause falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
@@ -53,66 +82,6 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
     CheckAllowedClause(llvm::omp::Y); \
   }
 
-std::string ThisVersion(unsigned version) {
-  std::string tv{
-      std::to_string(version / 10) + "." + std::to_string(version % 10)};
-  return "OpenMP v" + tv;
-}
-
-std::string TryVersion(unsigned version) {
-  return "try -fopenmp-version=" + std::to_string(version);
-}
-
-static const parser::Designator *GetDesignatorFromObj(
-    const parser::OmpObject &object) {
-  return std::get_if<parser::Designator>(&object.u);
-}
-
-static const parser::DataRef *GetDataRefFromObj(
-    const parser::OmpObject &object) {
-  if (auto *desg{GetDesignatorFromObj(object)}) {
-    return std::get_if<parser::DataRef>(&desg->u);
-  }
-  return nullptr;
-}
-
-static const parser::ArrayElement *GetArrayElementFromObj(
-    const parser::OmpObject &object) {
-  if (auto *dataRef{GetDataRefFromObj(object)}) {
-    using ElementIndirection = common::Indirection<parser::ArrayElement>;
-    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
-      return &ind->value();
-    }
-  }
-  return nullptr;
-}
-
-static bool IsVarOrFunctionRef(const MaybeExpr &expr) {
-  if (expr) {
-    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
-        evaluate::IsVariable(*expr);
-  } else {
-    return false;
-  }
-}
-
-static std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
-  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
-  // ForwardOwningPointer           typedExpr
-  // `- GenericExprWrapper          ^.get()
-  //    `- std::optional<Expr>      ^->v
-  return typedExpr.get()->v;
-}
-
-static std::optional<evaluate::DynamicType> GetDynamicType(
-    const parser::Expr &parserExpr) {
-  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
-    return maybeExpr->GetType();
-  } else {
-    return std::nullopt;
-  }
-}
-
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
 // statements and the expressions enclosed in an OpenMP Workshare construct
 class OmpWorkshareBlockChecker {
@@ -172,85 +141,6 @@ class OmpWorkshareBlockChecker {
   parser::CharBlock source_;
 };
 
-class AssociatedLoopChecker {
-public:
-  AssociatedLoopChecker(SemanticsContext &context, std::int64_t level)
-      : context_{context}, level_{level} {}
-
-  template <typename T> bool Pre(const T &) { return true; }
-  template <typename T> void Post(const T &) {}
-
-  bool Pre(const parser::DoConstruct &dc) {
-    level_--;
-    const auto &doStmt{
-        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
-    const auto &constructName{
-        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
-    if (constructName) {
-      constructNamesAndLevels_.emplace(
-          constructName.value().ToString(), level_);
-    }
-    if (level_ >= 0) {
-      if (dc.IsDoWhile()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
-      }
-      if (!dc.GetLoopControl()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
-      }
-    }
-    return true;
-  }
-
-  void Post(const parser::DoConstruct &dc) { level_++; }
-
-  bool Pre(const parser::CycleStmt &cyclestmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (cyclestmt.v) {
-      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second > 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ > 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::ExitStmt &exitStmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (exitStmt.v) {
-      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ >= 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "EXIT statement terminates associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
-    source_ = &actionstmt.source;
-    return true;
-  }
-
-private:
-  SemanticsContext &context_;
-  const parser::CharBlock *source_;
-  std::int64_t level_;
-  std::map<std::string, std::int64_t> constructNamesAndLevels_;
-};
-
 // `OmpUnitedTaskDesignatorChecker` is used to check if the designator
 // can appear within the TASK construct
 class OmpUnitedTaskDesignatorChecker {
@@ -318,18 +208,6 @@ bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
   return CheckAllowed(clause);
 }
 
-bool OmpStructureChecker::IsCommonBlock(const Symbol &sym) {
-  return sym.detailsIf<CommonBlockDetails>() != nullptr;
-}
-
-bool OmpStructureChecker::IsVariableListItem(const Symbol &sym) {
-  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
-}
-
-bool OmpStructureChecker::IsExtendedListItem(const Symbol &sym) {
-  return IsVariableListItem(sym) || sym.IsSubprogram();
-}
-
 bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   // Definition of close nesting:
   //
@@ -371,60 +249,6 @@ bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   return false;
 }
 
-namespace {
-struct ContiguousHelper {
-  ContiguousHelper(SemanticsContext &context)
-      : fctx_(context.foldingContext()) {}
-
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
-    return Visit(x.value());
-  }
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Reference<Contained> &x) {
-    return Visit(x.get());
-  }
-  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
-    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
-  }
-  template <typename T>
-  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
-    return common::visit(
-        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
-  }
-  template <typename T> std::optional<bool> Visit(const T &) {
-    // Everything else.
-    return std::nullopt;
-  }
-
-private:
-  evaluate::FoldingContext &fctx_;
-};
-} // namespace
-
-// Return values:
-// - std::optional<bool>{true} if the object is known to be contiguous
-// - std::optional<bool>{false} if the object is known not to be contiguous
-// - std::nullopt if the object contiguity cannot be determined
-std::optional<bool> OmpStructureChecker::IsContiguous(
-    const parser::OmpObject &object) {
-  return common::visit( //
-      common::visitors{
-          [&](const parser::Name &x) {
-            // Any member of a common block must be contiguous.
-            return std::optional<bool>{true};
-          },
-          [&](const parser::Designator &x) {
-            evaluate::ExpressionAnalyzer ea{context_};
-            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
-              return ContiguousHelper{context_}.Visit(*maybeExpr);
-            }
-            return std::optional<bool>{};
-          },
-      },
-      object.u);
-}
-
 void OmpStructureChecker::CheckVariableListItem(
     const SymbolSourceMap &symbols) {
   for (auto &[symbol, source] : symbols) {
@@ -522,62 +346,6 @@ bool OmpStructureChecker::HasInvalidWorksharingNesting(
   return false;
 }
 
-void OmpStructureChecker::HasInvalidDistributeNesting(
-    const parser::OpenMPLoopConstruct &x) {
-  bool violation{false};
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
-    // `distribute` region has to be nested
-    if (!CurrentDirectiveIsNested()) {
-      violation = true;
-    } else {
-      // `distribute` region has to be strictly nested inside `teams`
-      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-        violation = true;
-      }
-    }
-  }
-  if (violation) {
-    context_.Say(beginDir.source,
-        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
-        "region."_err_en_US);
-  }
-}
-void OmpStructureChecker::HasInvalidLoopBinding(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
-    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-    for (const auto &clause : clauseList.v) {
-      if (const auto *bindClause{
-              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
-        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
-          context_.Say(beginDir.source, msg);
-        }
-      }
-    }
-  };
-
-  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
-      CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
-        "strictly nested inside a `TEAMS` region."_err_en_US);
-  }
-
-  if (OmpDirectiveSet{
-          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
-          .test(beginDir.v)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
-        "combined with a `TEAMS` construct."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::HasInvalidTeamsNesting(
     const llvm::omp::Directive &dir, const parser::CharBlock &source) {
   if (!llvm::omp::nestedTeamsAllowedSet.test(dir)) {
@@ -668,16 +436,6 @@ void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
-  EnterDirectiveNest(MetadirectiveNest);
-  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
-}
-
-void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
-  ExitDirectiveNest(MetadirectiveNest);
-  dirContext_.pop_back();
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
@@ -717,85 +475,6 @@ void OmpStructureChecker::AddEndDirectiveClauses(
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
-  loopStack_.push_back(&x);
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  PushContextAndClauseSets(beginDir.source, beginDir.v);
-
-  // check matching, End directive is optional
-  if (const auto &endLoopDir{
-          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
-    const auto &endDir{
-        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
-
-    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
-
-    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
-  }
-
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    EnterDirectiveNest(SIMDNest);
-  }
-
-  // Combined target loop constructs are target device constructs. Keep track of
-  // whether any such construct has been visited to later check that REQUIRES
-  // directives for target-related options don't appear after them.
-  if (llvm::omp::allTargetSet.test(beginDir.v)) {
-    deviceConstructFound_ = true;
-  }
-
-  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    // 2.7.1 do-clause -> private-clause |
-    //                    firstprivate-clause |
-    //                    lastprivate-clause |
-    //                    linear-clause |
-    //                    reduction-clause |
-    //                    schedule-clause |
-    //                    collapse-clause |
-    //                    ordered-clause
-
-    // nesting check
-    HasInvalidWorksharingNesting(
-        beginDir.source, llvm::omp::nestedWorkshareErrSet);
-  }
-  SetLoopInfo(x);
-
-  if (const auto &doConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
-    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
-  }
-  CheckLoopItrVariableIsInt(x);
-  CheckAssociatedLoopConstraints(x);
-  HasInvalidDistributeNesting(x);
-  HasInvalidLoopBinding(x);
-  if (CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
-  }
-  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
-      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
-    CheckDistLinear(x);
-  }
-}
-const parser::Name OmpStructureChecker::GetLoopIndex(
-    const parser::DoConstruct *x) {
-  using Bounds = parser::LoopControl::Bounds;
-  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
-}
-void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const parser::DoConstruct *loop{&*loopConstruct};
-    if (loop && loop->IsDoNormal()) {
-      const parser::Name &itrVal{GetLoopIndex(loop)};
-      SetLoopIv(itrVal.symbol);
-    }
-  }
-}
-
 void OmpStructureChecker::CheckIteratorRange(
     const parser::OmpIteratorSpecifier &x) {
   // Check:
@@ -855,108 +534,6 @@ void OmpStructureChecker::CheckIteratorModifier(const parser::OmpIterator &x) {
   }
 }
 
-void OmpStructureChecker::CheckLoopItrVariableIsInt(
-    const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-
-    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-      if (loop->IsDoNormal()) {
-        const parser::Name &itrVal{GetLoopIndex(loop)};
-        if (itrVal.symbol) {
-          const auto *type{itrVal.symbol->GetType()};
-          if (!type->IsNumeric(TypeCategory::Integer)) {
-            context_.Say(itrVal.source,
-                "The DO loop iteration"
-                " variable must be of the type integer."_err_en_US,
-                itrVal.ToString());
-          }
-        }
-      }
-      // Get the next DoConstruct if block is not empty.
-      const auto &block{std::get<parser::Block>(loop->t)};
-      const auto it{block.begin()};
-      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                               : nullptr;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
-  // Check the following:
-  //  The only OpenMP constructs that can be encountered during execution of
-  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
-  // construct and the `ordered` construct with the `simd` clause.
-
-  // Check if the parent context has the SIMD clause
-  // Please note that we use GetContext() instead of GetContextParent()
-  // because PushContextAndClauseSets() has not been called on the
-  // current context yet.
-  // TODO: Check for declare simd regions.
-  bool eligibleSIMD{false};
-  common::visit(
-      common::visitors{
-          // Allow `!$OMP ORDERED SIMD`
-          [&](const parser::OpenMPBlockConstruct &c) {
-            const auto &beginBlockDir{
-                std::get<parser::OmpBeginBlockDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
-            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
-              const auto &clauses{
-                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
-              for (const auto &clause : clauses.v) {
-                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
-                  eligibleSIMD = true;
-                  break;
-                }
-              }
-            }
-          },
-          [&](const parser::OpenMPStandaloneConstruct &c) {
-            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
-                    &c.u)}) {
-              llvm::omp::Directive dirId{ssc->v.DirId()};
-              if (dirId == llvm::omp::Directive::OMPD_ordered) {
-                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
-                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
-                    eligibleSIMD = true;
-                    break;
-                  }
-                }
-              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
-                eligibleSIMD = true;
-              }
-            }
-          },
-          // Allowing SIMD and loop construct
-          [&](const parser::OpenMPLoopConstruct &c) {
-            const auto &beginLoopDir{
-                std::get<parser::OmpBeginLoopDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
-              eligibleSIMD = true;
-            }
-          },
-          [&](const parser::OpenMPAtomicConstruct &c) {
-            // Allow `!$OMP ATOMIC`
-            eligibleSIMD = true;
-          },
-          [&](const auto &c) {},
-      },
-      c.u);
-  if (!eligibleSIMD) {
-    context_.Say(parser::FindSourceLocation(c),
-        "The only OpenMP constructs that can be encountered during execution "
-        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
-        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
-        "construct with the `SIMD` clause."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   // 2.12.5 Target Construct Restriction
   bool eligibleTarget{true};
@@ -1014,187 +591,6 @@ void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   }
 }
 
-std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-  std::int64_t orderedCollapseLevel{1};
-  std::int64_t orderedLevel{1};
-  std::int64_t collapseLevel{1};
-
-  for (const auto &clause : clauseList.v) {
-    if (const auto *collapseClause{
-            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
-      if (const auto v{GetIntValue(collapseClause->v)}) {
-        collapseLevel = *v;
-      }
-    }
-    if (const auto *orderedClause{
-            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
-      if (const auto v{GetIntValue(orderedClause->v)}) {
-        orderedLevel = *v;
-      }
-    }
-  }
-  if (orderedLevel >= collapseLevel) {
-    orderedCollapseLevel = orderedLevel;
-  } else {
-    orderedCollapseLevel = collapseLevel;
-  }
-  return orderedCollapseLevel;
-}
-
-void OmpStructureChecker::CheckAssociatedLoopConstraints(
-    const parser::OpenMPLoopConstruct &x) {
-  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
-  AssociatedLoopChecker checker{context_, ordCollapseLevel};
-  parser::Walk(x, checker);
-}
-
-void OmpStructureChecker::CheckDistLinear(
-    const parser::OpenMPLoopConstruct &x) {
-
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  SymbolSourceMap indexVars;
-
-  // Collect symbols of all the variables from linear clauses
-  for (auto &clause : clauses.v) {
-    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
-      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
-      GetSymbolsInObjectList(objects, indexVars);
-    }
-  }
-
-  if (!indexVars.empty()) {
-    // Get collapse level, if given, to find which loops are "associated."
-    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
-    // Include the top loop if no collapse is specified
-    if (collapseVal == 0) {
-      collapseVal = 1;
-    }
-
-    // Match the loop index variables with the collected symbols from linear
-    // clauses.
-    if (const auto &loopConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-        if (loop->IsDoNormal()) {
-          const parser::Name &itrVal{GetLoopIndex(loop)};
-          if (itrVal.symbol) {
-            // Remove the symbol from the collected set
-            indexVars.erase(&itrVal.symbol->GetUltimate());
-          }
-          collapseVal--;
-          if (collapseVal == 0) {
-            break;
-          }
-        }
-        // Get the next DoConstruct if block is not empty.
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                 : nullptr;
-      }
-    }
-
-    // Show error for the remaining variables
-    for (auto &[symbol, source] : indexVars) {
-      const Symbol &root{GetAssociationRoot(*symbol)};
-      context_.Say(source,
-          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
-          root.name());
-    }
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  // A few semantic checks for InScan reduction are performed below as SCAN
-  // constructs inside LOOP may add the relevant information. Scan reduction is
-  // supported only in loop constructs, so same checks are not applicable to
-  // other directives.
-  using ReductionModifier = parser::OmpReductionModifier;
-  for (const auto &clause : clauseList.v) {
-    if (const auto *reductionClause{
-            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
-      auto &modifiers{OmpGetModifiers(reductionClause->v)};
-      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
-      if (maybeModifier &&
-          maybeModifier->v == ReductionModifier::Value::Inscan) {
-        const auto &objectList{
-            std::get<parser::OmpObjectList>(reductionClause->v.t)};
-        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
-          if (auto &symbol = name->symbol) {
-            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
-                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
-              context_.Say(name->source,
-                  "List item %s must appear in EXCLUSIVE or "
-                  "INCLUSIVE clause of an "
-                  "enclosed SCAN directive"_err_en_US,
-                  name->ToString());
-            }
-          }
-        };
-        for (const auto &ompObj : objectList.v) {
-          common::visit(
-              common::visitors{
-                  [&](const parser::Designator &designator) {
-                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
-                            designator)}) {
-                      checkReductionSymbolInScan(name);
-                    }
-                  },
-                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
-              },
-              ompObj.u);
-        }
-      }
-    }
-  }
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    ExitDirectiveNest(SIMDNest);
-  }
-  dirContext_.pop_back();
-
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top{loopStack_.back()};
-  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
-  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
-  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  // 2.7.1 end-do -> END DO [nowait-clause]
-  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
-  case llvm::omp::Directive::OMPD_do:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
-    break;
-  case llvm::omp::Directive::OMPD_do_simd:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
-    dirContext_.pop_back();
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &endBlockDir{std::get<parser::OmpEndBlockDirective>(x.t)};
@@ -2659,1488 +2055,46 @@ void OmpStructureChecker::CheckCancellationNest(
           parser::ToUpperCaseLetters(typeName.str()));
       break;
     default:
-      // This is diagnosed later.
-      return;
-    }
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
-  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  case llvm::omp::Directive::OMPD_scope:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
-    break;
-  // 2.7.3 end-single-clause -> copyprivate-clause |
-  //                            nowait-clause
-  case llvm::omp::Directive::OMPD_single:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
-    break;
-  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
-  case llvm::omp::Directive::OMPD_workshare:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_workshare);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-// TODO: Verify the popping of dirContext requirement after nowait
-// implementation, as there is an implicit barrier at the end of the worksharing
-// constructs unless a nowait clause is specified. Only OMPD_end_single and
-// end_workshareare popped as they are pushed while entering the
-// EndBlockDirective.
-void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
-    dirContext_.pop_back();
-  }
-}
-
-/// parser::Block is a list of executable constructs, parser::BlockConstruct
-/// is Fortran's BLOCK/ENDBLOCK construct.
-/// Strip the outermost BlockConstructs, return the reference to the Block
-/// in the executable part of the innermost of the stripped constructs.
-/// Specifically, if the given `block` has a single entry (it's a list), and
-/// the entry is a BlockConstruct, get the Block contained within. Repeat
-/// this step as many times as possible.
-static const parser::Block &GetInnermostExecPart(const parser::Block &block) {
-  const parser::Block *iter{&block};
-  while (iter->size() == 1) {
-    const parser::ExecutionPartConstruct &ep{iter->front()};
-    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
-      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
-      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
-        iter = &std::get<parser::Block>(bc->value().t);
-        continue;
-      }
-    }
-    break;
-  }
-  return *iter;
-}
-
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
-struct AnalyzedCondStmt {
-  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
-  parser::CharBlock source;
-  SourcedActionStmt ift, iff;
-};
-
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
-// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
-// is that the ActionStmt will be either an assignment or a pointer-assignment,
-// otherwise return std::nullopt.
-// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
-// the "typedAssignment" is unset. This can happen if there are semantic errors
-// in the purported assignment.
-static std::optional<evaluate::Assignment> GetEvaluateAssignment(
-    const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
-
-  return common::visit(
-      [](auto &&s) -> std::optional<evaluate::Assignment> {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>) {
-          const TypedAssignment &typed{s.value().typedAssignment};
-          // ForwardOwningPointer                 typedAssignment
-          // `- GenericAssignmentWrapper          ^.get()
-          //    `- std::optional<Assignment>      ^->v
-          return typed.get()->v;
-        } else {
-          return std::nullopt;
-        }
-      },
-      x->u);
-}
-
-// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
-// to separate cases where the source has something that looks like an
-// assignment, but is semantically wrong (diagnosed by general semantic
-// checks), and where the source has some other statement (which we want
-// to report as "should be an assignment").
-static bool IsAssignment(const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return false;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-
-  return common::visit(
-      [](auto &&s) -> bool {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        return std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>;
-      },
-      x->u);
-}
-
-static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  // Extract the evaluate::Expr from ScalarLogicalExpr.
-  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
-    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
-    const parser::Expr &expr{logical.thing.thing.value()};
-    return GetEvaluateExpr(expr);
-  }};
-
-  // Recognize either
-  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
-  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
-
-  if (auto &&action{GetActionStmt(x)}) {
-    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
-            &action.stmt->u)}) {
-      const parser::IfStmt &s{ifs->value()};
-      auto &&maybeCond{
-          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
-      auto &thenStmt{
-          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
-      if (maybeCond) {
-        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
-            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
-            SourcedActionStmt{}};
-      }
-    }
-    return std::nullopt;
-  }
-
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    if (auto *ifc{
-            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
-      using ElseBlock = parser::IfConstruct::ElseBlock;
-      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
-      const parser::IfConstruct &s{ifc->value()};
-
-      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
-        // Not expecting any else-if statements.
-        return std::nullopt;
-      }
-      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
-      auto &&maybeCond{getFromLogical(
-          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
-      if (!maybeCond) {
-        return std::nullopt;
-      }
-
-      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)),
-            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
-        if (result.ift.stmt && result.iff.stmt) {
-          return result;
-        }
-      } else {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
-        if (result.ift.stmt) {
-          return result;
-        }
-      }
-    }
-    return std::nullopt;
-  }
-
-  return std::nullopt;
-}
-
-static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
-    parser::CharBlock source) {
-  // Find => in the range, if not found, find = that is not a part of
-  // <=, >=, ==, or /=.
-  auto trim{[](std::string_view v) {
-    const char *begin{v.data()};
-    const char *end{begin + v.size()};
-    while (*begin == ' ' && begin != end) {
-      ++begin;
-    }
-    while (begin != end && end[-1] == ' ') {
-      --end;
-    }
-    assert(begin != end && "Source should not be empty");
-    return parser::CharBlock(begin, end - begin);
-  }};
-
-  std::string_view sv(source.begin(), source.size());
-
-  if (auto where{sv.find("=>")}; where != sv.npos) {
-    std::string_view lhs(sv.data(), where);
-    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
-    return std::make_pair(trim(lhs), trim(rhs));
-  }
-
-  // Go backwards, since all the exclusions above end with a '='.
-  for (size_t next{source.size()}; next > 1; --next) {
-    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
-      std::string_view lhs(sv.data(), next - 1);
-      std::string_view rhs(sv.data() + next, sv.size() - next);
-      return std::make_pair(trim(lhs), trim(rhs));
-    }
-  }
-  llvm_unreachable("Could not find assignment operator");
-}
-
-namespace atomic {
-
-struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
-                                 std::vector<SomeExpr>, false> {
-  using Result = std::vector<SomeExpr>;
-  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
-  DesignatorCollector() : Base(*this) {}
-
-  Result Default() const { return {}; }
-
-  using Base::operator();
-
-  template <typename T> //
-  Result operator()(const evaluate::Designator<T> &x) const {
-    // Once in a designator, don't traverse it any further (i.e. only
-    // collect top-level designators).
-    auto copy{x};
-    return Result{AsGenericExpr(std::move(copy))};
-  }
-
-  template <typename... Rs> //
-  Result Combine(Result &&result, Rs &&...results) const {
-    Result v(std::move(result));
-    auto moveAppend{[](auto &accum, auto &&other) {
-      for (auto &&s : other) {
-        accum.push_back(std::move(s));
-      }
-    }};
-    (moveAppend(v, std::move(results)), ...);
-    return v;
-  }
-};
-
-struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
-  using Base = evaluate::AnyTraverse<VariableFinder>;
-  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
-
-  using Base::operator();
-
-  template <typename T>
-  bool operator()(const evaluate::Designator<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-  template <typename T>
-  bool operator()(const evaluate::FunctionRef<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-private:
-  const SomeExpr &var;
-};
-} // namespace atomic
-
-static bool IsPointerAssignment(const evaluate::Assignment &x) {
-  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
-      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
-}
-
-namespace operation = Fortran::evaluate::operation;
-
-static bool IsCheckForAssociated(const SomeExpr &cond) {
-  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
-}
-
-static bool HasCommonDesignatorSymbols(
-    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
-  // Compare the designators used in "other" with the designators whose
-  // symbols are given in baseSyms.
-  // This is a part of the check if these two expressions can access the same
-  // storage: if the designators used in them are different enough, then they
-  // will be assumed not to access the same memory.
-  //
-  // Consider an (array element) expression x%y(w%z), the corresponding symbol
-  // vector will be {x, y, w, z} (i.e. the symbols for these names).
-  // Check whether this exact sequence appears anywhere in any the symbol
-  // vector for "other". This will be true for x(y) and x(y+1), so this is
-  // not a sufficient condition, but can be used to eliminate candidates
-  // before doing more exhaustive checks.
-  //
-  // If any of the symbols in this sequence are function names, assume that
-  // there is no storage overlap, mostly because it would be impossible in
-  // general to determine what storage the function will access.
-  // Note: if f is pure, then two calls to f will access the same storage
-  // when called with the same arguments. This check is not done yet.
-
-  if (llvm::any_of(
-          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
-    // If there is a function symbol in the chain then we can't infer much
-    // about the accessed storage.
-    return false;
-  }
-
-  auto isSubsequence{// Is u a subsequence of v.
-      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
-        size_t us{u.size()}, vs{v.size()};
-        if (us > vs) {
-          return false;
-        }
-        for (size_t off{0}; off != vs - us + 1; ++off) {
-          bool same{true};
-          for (size_t i{0}; i != us; ++i) {
-            if (u[i] != v[off + i]) {
-              same = false;
-              break;
-            }
-          }
-          if (same) {
-            return true;
-          }
-        }
-        return false;
-      }};
-
-  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
-  return isSubsequence(baseSyms, otherSyms);
-}
-
-static bool HasCommonTopLevelDesignators(
-    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
-  // Compare designators directly as expressions. This will ensure
-  // that x(y) and x(y+1) are not flagged as overlapping, whereas
-  // the symbol vectors for both of these would be identical.
-  std::vector<SomeExpr> otherDsgs{atomic::DesignatorCollector{}(other)};
-
-  for (auto &s : baseDsgs) {
-    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
-      return true;
-    }
-  }
-  return false;
-}
-
-static const SomeExpr *HasStorageOverlap(
-    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
-  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
-  std::vector<SomeExpr> baseDsgs{atomic::DesignatorCollector{}(base)};
-
-  for (const SomeExpr &expr : exprs) {
-    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
-      continue;
-    }
-    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
-      return &expr;
-    }
-  }
-  return nullptr;
-}
-
-static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
-  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
-  // for example.
-  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
-}
-
-static bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
-  return atomic::VariableFinder{sub}(super);
-}
-
-static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
-  if (value) {
-    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
-        evaluate::GenericExprWrapper::Deleter);
-  }
-}
-
-static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
-    std::optional<evaluate::Assignment> value) {
-  if (value) {
-    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
-        evaluate::GenericAssignmentWrapper::Deleter);
-  }
-}
-
-static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
-    int what,
-    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
-  parser::OpenMPAtomicConstruct::Analysis::Op operation;
-  operation.what = what;
-  SetAssignment(operation.assign, maybeAssign);
-  return operation;
-}
-
-static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
-    const SomeExpr &atom, const MaybeExpr &cond,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
-  // Defined in flang/include/flang/Parser/parse-tree.h
-  //
-  // struct Analysis {
-  //   struct Kind {
-  //     static constexpr int None = 0;
-  //     static constexpr int Read = 1;
-  //     static constexpr int Write = 2;
-  //     static constexpr int Update = Read | Write;
-  //     static constexpr int Action = 3; // Bits containing N, R, W, U
-  //     static constexpr int IfTrue = 4;
-  //     static constexpr int IfFalse = 8;
-  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
-  //   };
-  //   struct Op {
-  //     int what;
-  //     TypedAssignment assign;
-  //   };
-  //   TypedExpr atom, cond;
-  //   Op op0, op1;
-  // };
-
-  parser::OpenMPAtomicConstruct::Analysis an;
-  SetExpr(an.atom, atom);
-  SetExpr(an.cond, cond);
-  an.op0 = std::move(op0);
-  an.op1 = std::move(op1);
-  return an;
-}
-
-void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
-    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
-    parser::CharBlock source) {
-  if (auto *expr{HasStorageOverlap(base, exprs)}) {
-    context_.Say(source,
-        "Within atomic operation %s and %s access the same storage"_warn_en_US,
-        base.AsFortran(), expr->AsFortran());
-  }
-}
-
-void OmpStructureChecker::ErrorShouldBeVariable(
-    const MaybeExpr &expr, parser::CharBlock source) {
-  if (expr) {
-    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
-        expr->AsFortran());
-  } else {
-    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
-  }
-}
-
-/// Check if `expr` satisfies the following conditions for x and v:
-///
-/// [6.0:189:10-12]
-/// - x and v (as applicable) are either scalar variables or
-///   function references with scalar data pointer result of non-character
-///   intrinsic type or variables that are non-polymorphic scalar pointers
-///   and any length type parameter must be constant.
-void OmpStructureChecker::CheckAtomicType(
-    SymbolRef sym, parser::CharBlock source, std::string_view name) {
-  const DeclTypeSpec *typeSpec{sym->GetType()};
-  if (!typeSpec) {
-    return;
-  }
-
-  if (!IsPointer(sym)) {
-    using Category = DeclTypeSpec::Category;
-    Category cat{typeSpec->category()};
-    if (cat == Category::Character) {
-      context_.Say(source,
-          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
-    } else if (cat != Category::Numeric && cat != Category::Logical) {
-      context_.Say(source,
-          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
-    }
-    return;
-  }
-
-  // Variable is a pointer.
-  if (typeSpec->IsPolymorphic()) {
-    context_.Say(source,
-        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
-        name);
-    return;
-  }
-
-  // Go over all length parameters, if any, and check if they are
-  // explicit.
-  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
-    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
-          // "entry" is a map entry
-          return entry.second.isLen() && !entry.second.isExplicit();
-        })) {
-      context_.Say(source,
-          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
-          name);
-    }
-  }
-}
-
-void OmpStructureChecker::CheckAtomicVariable(
-    const SomeExpr &atom, parser::CharBlock source) {
-  if (atom.Rank() != 0) {
-    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
-        atom.AsFortran());
-  }
-
-  std::vector<SomeExpr> dsgs{atomic::DesignatorCollector{}(atom)};
-  assert(dsgs.size() == 1 && "Should have a single top-level designator");
-  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
-
-  CheckAtomicType(syms.back(), source, atom.AsFortran());
-
-  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
-    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
-        atom.AsFortran());
-  }
-}
-
-std::pair<const parser::ExecutionPartConstruct *,
-    const parser::ExecutionPartConstruct *>
-OmpStructureChecker::CheckUpdateCapture(
-    const parser::ExecutionPartConstruct *ec1,
-    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
-  // Decide which statement is the atomic update and which is the capture.
-  //
-  // The two allowed cases are:
-  //   x = ...      atomic-var = ...
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  // or
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  //   x = ...      atomic-var = ...
-  //
-  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
-  // (which makes more sense, as it captures the original value of the atomic
-  // variable).
-  //
-  // If the two statements don't fit these criteria, return a pair of default-
-  // constructed values.
-  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
-      const parser::ExecutionPartConstruct *>;
-
-  SourcedActionStmt act1{GetActionStmt(ec1)};
-  SourcedActionStmt act2{GetActionStmt(ec2)};
-  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
-  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
-  if (!maybeAssign1 || !maybeAssign2) {
-    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
-      context_.Say(source,
-          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
-    }
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  auto as1{*maybeAssign1}, as2{*maybeAssign2};
-
-  auto isUpdateCapture{
-      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
-        return IsSameOrConvertOf(c.rhs, u.lhs);
-      }};
-
-  // Do some checks that narrow down the possible choices for the update
-  // and the capture statements. This will help to emit better diagnostics.
-  // 1. An assignment could be an update (cbu) if the left-hand side is a
-  //    subexpression of the right-hand side.
-  // 2. An assignment could be a capture (cbc) if the right-hand side is
-  //    a variable (or a function ref), with potential type conversions.
-  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
-  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
-  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
-  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
-
-  // We want to diagnose cases where both assignments cannot be an update,
-  // or both cannot be a capture, as well as cases where either assignment
-  // cannot be any of these two.
-  //
-  // If we organize these boolean values into a matrix
-  //   |cbu1 cbu2|
-  //   |cbc1 cbc2|
-  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
-  // row or column, including the case where everything is zero. All these
-  // cases correspond to the determinant of the matrix being 0, which suggests
-  // that checking the det may be a convenient diagnostic check. There is only
-  // one additional case where the det is 0, which is when the matrix is all 1
-  // ("true"). The "all true" case represents the situation where both
-  // assignments could be an update as well as a capture. On the other hand,
-  // whenever det != 0, the roles of the update and the capture can be
-  // unambiguously assigned to as1 and as2 [1].
-  //
-  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
-  // det = 0, leaving 6 cases where det != 0:
-  //   0 1   0 1   1 0   1 0   1 1   1 1
-  //   1 0   1 1   0 1   1 1   0 1   1 0
-  // In each case the classification is unambiguous.
-
-  //     |cbu1 cbu2|
-  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
-  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
-
-  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
-                                  const std::string &expr) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
-        expr);
-  }};
-
-  auto errorNeitherWorks{[&]() {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
-  }};
-
-  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
-    // If det != 0, then the checks unambiguously suggest a specific
-    // categorization.
-    // If det == 0, then this function should be called only if the
-    // checks haven't ruled out any possibility, i.e. when both assigments
-    // could still be either updates or captures.
-    if (det > 0) {
-      // as1 is update, as2 is capture
-      if (isUpdateCapture(as1, as2)) {
-        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
-      } else {
-        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else if (det < 0) {
-      // as2 is update, as1 is capture
-      if (isUpdateCapture(as2, as1)) {
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      } else {
-        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else {
-      bool updateFirst{isUpdateCapture(as1, as2)};
-      bool captureFirst{isUpdateCapture(as2, as1)};
-      if (updateFirst && captureFirst) {
-        // If both assignment could be the update and both could be the
-        // capture, emit a warning about the ambiguity.
-        context_.Say(act1.source,
-            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      }
-      if (updateFirst != captureFirst) {
-        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
-        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
-        return std::make_pair(upd, cap);
-      }
-      assert(!updateFirst && !captureFirst);
-      errorNeitherWorks();
-      return std::make_pair(nullptr, nullptr);
-    }
-  }};
-
-  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
-    return makeSelectionFromDet(det);
-  }
-  assert(det == 0 && "Prior checks should have covered det != 0");
-
-  // If neither of the statements is an RMW update, it could still be a
-  // "write" update. Pretty much any assignment can be a write update, so
-  // recompute det with cbu1 = cbu2 = true.
-  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
-    return makeSelectionFromDet(writeDet);
-  }
-
-  // It's only errors from here on.
-
-  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
-    errorNeitherWorks();
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // The remaining cases are that
-  // - no candidate for update, or for capture,
-  // - one of the assigments cannot be anything.
-
-  if (!cbu1 && !cbu2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  } else if (!cbc1 && !cbc2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
-    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
-    context_.Say(src,
-        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // All cases should have been covered.
-  llvm_unreachable("Unchecked condition");
-}
-
-void OmpStructureChecker::CheckAtomicCaptureAssignment(
-    const evaluate::Assignment &capture, const SomeExpr &atom,
-    parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &cap{capture.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, rsrc);
-    // This part should have been checked prior to calling this function.
-    assert(*GetConvertInput(capture.rhs) == atom &&
-        "This cannot be a capture assignment");
-    CheckStorageOverlap(atom, {cap}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicReadAssignment(
-    const evaluate::Assignment &read, parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-
-  if (auto maybe{GetConvertInput(read.rhs)}) {
-    const SomeExpr &atom{*maybe};
-
-    if (!IsVarOrFunctionRef(atom)) {
-      ErrorShouldBeVariable(atom, rsrc);
-    } else {
-      CheckAtomicVariable(atom, rsrc);
-      CheckStorageOverlap(atom, {read.lhs}, source);
-    }
-  } else {
-    ErrorShouldBeVariable(read.rhs, rsrc);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicWriteAssignment(
-    const evaluate::Assignment &write, parser::CharBlock source) {
-  // [6.0:190:13-15]
-  // A write structured block is write-statement, a write statement that has
-  // one of the following forms:
-  //   x = expr
-  //   x => expr
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{write.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, lsrc);
-    CheckStorageOverlap(atom, {write.rhs}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateAssignment(
-    const evaluate::Assignment &update, parser::CharBlock source) {
-  // [6.0:191:1-7]
-  // An update structured block is update-statement, an update statement
-  // that has one of the following forms:
-  //   x = x operator expr
-  //   x = expr operator x
-  //   x = intrinsic-procedure-name (x)
-  //   x = intrinsic-procedure-name (x, expr-list)
-  //   x = intrinsic-procedure-name (expr-list, x)
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{update.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, lsrc);
-
-  std::pair<operation::Operator, std::vector<SomeExpr>> top{
-      operation::Operator::Unknown, {}};
-  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
-    top = GetTopLevelOperation(*maybeInput);
-  }
-  switch (top.first) {
-  case operation::Operator::Add:
-  case operation::Operator::Sub:
-  case operation::Operator::Mul:
-  case operation::Operator::Div:
-  case operation::Operator::And:
-  case operation::Operator::Or:
-  case operation::Operator::Eqv:
-  case operation::Operator::Neqv:
-  case operation::Operator::Min:
-  case operation::Operator::Max:
-  case operation::Operator::Identity:
-    break;
-  case operation::Operator::Call:
-    context_.Say(source,
-        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Convert:
-    context_.Say(source,
-        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Intrinsic:
-    context_.Say(source,
-        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Constant:
-  case operation::Operator::Unknown:
-    context_.Say(
-        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(source,
-        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
-        operation::ToString(top.first));
-    return;
-  }
-  // Check how many times `atom` occurs as an argument, if it's a subexpression
-  // of an argument, and collect the non-atom arguments.
-  std::vector<SomeExpr> nonAtom;
-  MaybeExpr subExpr;
-  auto atomCount{[&]() {
-    int count{0};
-    for (const SomeExpr &arg : top.second) {
-      if (IsSameOrConvertOf(arg, atom)) {
-        ++count;
-      } else {
-        if (!subExpr && IsSubexpressionOf(atom, arg)) {
-          subExpr = arg;
-        }
-        nonAtom.push_back(arg);
-      }
-    }
-    return count;
-  }()};
-
-  bool hasError{false};
-  if (subExpr) {
-    context_.Say(rsrc,
-        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
-        atom.AsFortran(), subExpr->AsFortran());
-    hasError = true;
-  }
-  if (top.first == operation::Operator::Identity) {
-    // This is "x = y".
-    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
-          atom.AsFortran());
-      hasError = true;
-    }
-  } else {
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    } else if (atomCount > 1) {
-      context_.Say(rsrc,
-          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    }
-  }
-
-  if (!hasError) {
-    CheckStorageOverlap(atom, nonAtom, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
-    const SomeExpr &cond, parser::CharBlock condSource,
-    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
-  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
-  const SomeExpr &atom{assign.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, arsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, alsrc);
-
-  auto top{GetTopLevelOperation(cond)};
-  // Missing arguments to operations would have been diagnosed by now.
-
-  switch (top.first) {
-  case operation::Operator::Associated:
-    if (atom != top.second.front()) {
-      context_.Say(assignSource,
-          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
-    }
-    break;
-  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
-  case operation::Operator::Eq:
-  case operation::Operator::Eqv:
-  // x ordop expr | expr ordop x
-  case operation::Operator::Lt:
-  case operation::Operator::Gt: {
-    const SomeExpr &arg0{top.second[0]};
-    const SomeExpr &arg1{top.second[1]};
-    if (IsSameOrConvertOf(arg0, atom)) {
-      CheckStorageOverlap(atom, {arg1}, condSource);
-    } else if (IsSameOrConvertOf(arg1, atom)) {
-      CheckStorageOverlap(atom, {arg0}, condSource);
-    } else {
-      assert(top.first != operation::Operator::Identity &&
-          "Handle this separately");
-      context_.Say(assignSource,
-          "An argument of the %s operator should be the target of the assignment"_err_en_US,
-          operation::ToString(top.first));
-    }
-    break;
-  }
-  case operation::Operator::Identity:
-  case operation::Operator::True:
-  case operation::Operator::False:
-    break;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(condSource,
-        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
-        operation::ToString(top.first));
-    break;
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
-    const AnalyzedCondStmt &update, parser::CharBlock source) {
-  // The condition/statements must be:
-  // - cond: x equalop e      ift: x =  d     iff: -
-  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
-  // - cond: associated(x)    ift: x => expr  iff: -
-  // - cond: associated(x, e) ift: x => expr  iff: -
-
-  // The if-true statement must be present, and must be an assignment.
-  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
-  if (!maybeAssign) {
-    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
-    } else {
-      context_.Say(
-          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
-    }
-    return;
-  }
-  const evaluate::Assignment assign{*maybeAssign};
-  const SomeExpr &atom{assign.lhs};
-
-  CheckAtomicConditionalUpdateAssignment(
-      update.cond, update.source, assign, update.ift.source);
-
-  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
-
-  if (update.iff) {
-    context_.Say(update.iff.source,
-        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateOnly(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeUpdate->lhs};
-      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdate(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // Allowable forms are (single-statement):
-  // - if ...
-  // - x = (... ? ... : x)
-  // and two-statement:
-  // - r = cond ; if (r) ...
-
-  const parser::ExecutionPartConstruct *ust{nullptr}; // update
-  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
-
-  if (body.size() == 1) {
-    ust = &body.front();
-  } else if (body.size() == 2) {
-    cst = &body.front();
-    ust = &body.back();
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
-    return;
-  }
-
-  // Flang doesn't support conditional-expr yet, so all update statements
-  // are if-statements.
-
-  // IfStmt:        if (...) ...
-  // IfConstruct:   if (...) then ... endif
-  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
-  if (!maybeUpdate) {
-    context_.Say(source,
-        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
-    return;
-  }
-
-  AnalyzedCondStmt &update{*maybeUpdate};
-
-  if (SourcedActionStmt action{GetActionStmt(cst)}) {
-    // The "condition" statement must be `r = cond`.
-    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
-      if (maybeCond->lhs != update.cond) {
-        context_.Say(update.source,
-            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
-            maybeCond->lhs.AsFortran());
-      } else {
-        // If it's "r = ...; if (r) ..." then put the original condition
-        // in `update`.
-        update.cond = maybeCond->rhs;
-      }
-    } else {
-      context_.Say(action.source,
-          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
-    }
-  }
-
-  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
-
-  CheckAtomicConditionalUpdateStmt(update, source);
-  if (IsCheckForAssociated(update.cond)) {
-    if (!IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
-    }
-  } else {
-    if (IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
-    }
-  }
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
-      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
-      MakeAtomicAnalysisOp(Analysis::None));
-}
-
-void OmpStructureChecker::CheckAtomicUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() != 2) {
-    context_.Say(source,
-        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
-    return;
-  }
-
-  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
-  if (!uec || !cec) {
-    // Diagnostics already emitted.
-    return;
-  }
-  SourcedActionStmt uact{GetActionStmt(uec)};
-  SourcedActionStmt cact{GetActionStmt(cec)};
-  // The "dereferences" of std::optional are guaranteed to be valid after
-  // CheckUpdateCapture.
-  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
-  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
-
-  const SomeExpr &atom{update.lhs};
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  int action;
-
-  if (IsMaybeAtomicWrite(update)) {
-    action = Analysis::Write;
-    CheckAtomicWriteAssignment(update, uact.source);
-  } else {
-    action = Analysis::Update;
-    CheckAtomicUpdateAssignment(update, uact.source);
-  }
-  CheckAtomicCaptureAssignment(capture, atom, cact.source);
-
-  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
-    context_.Say(cact.source,
-        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-    return;
-  }
-
-  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(action, update),
-        MakeAtomicAnalysisOp(Analysis::Read, capture));
-  } else {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(Analysis::Read, capture),
-        MakeAtomicAnalysisOp(action, update));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // There are two different variants of this:
-  // (1) conditional-update and capture separately:
-  //     This form only allows single-statement updates, i.e. the update
-  //     form "r = cond; if (r) ..." is not allowed.
-  // (2) conditional-update combined with capture in a single statement:
-  //     This form does allow the condition to be calculated separately,
-  //     i.e. "r = cond; if (r) ...".
-  // Regardless of what form it is, the actual update assignment is a
-  // proper write, i.e. "x = d", where d does not depend on x.
-
-  AnalyzedCondStmt update;
-  SourcedActionStmt capture;
-  bool captureAlways{true}, captureFirst{true};
-
-  auto extractCapture{[&]() {
-    capture = update.iff;
-    captureAlways = false;
-    update.iff = SourcedActionStmt{};
-  }};
-
-  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
-    // The non-update statement is either "r = cond" or the capture.
-    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
-      if (update.cond == maybeAssign->lhs) {
-        // If this is "r = cond; if (r) ...", then update the condition.
-        update.cond = maybeAssign->rhs;
-        update.source = action.source;
-        // In this form, the update and the capture are combined into
-        // an IF-THEN-ELSE statement.
-        extractCapture();
-      } else {
-        // Assume this is the capture-statement.
-        capture = action;
-      }
-    }
-  }};
-
-  if (body.size() == 2) {
-    // This could be
-    // - capture; conditional-update (in any order), or
-    // - r = cond; if (r) capture-update
-    const parser::ExecutionPartConstruct *st1{&body.front()};
-    const parser::ExecutionPartConstruct *st2{&body.back()};
-    // In either case, the conditional statement can be analyzed by
-    // AnalyzeConditionalStmt, whereas the other statement cannot.
-    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
-      update = *maybeUpdate1;
-      classifyNonUpdate(GetActionStmt(st2));
-      captureFirst = false;
-    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
-      update = *maybeUpdate2;
-      classifyNonUpdate(GetActionStmt(st1));
-    } else {
-      // None of the statements are conditional, this rules out the
-      // "r = cond; if (r) ..." and the "capture + conditional-update"
-      // variants. This could still be capture + write (which is classified
-      // as conditional-update-capture in the spec).
-      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
-      if (!uec || !cec) {
-        // Diagnostics already emitted.
-        return;
-      }
-      SourcedActionStmt uact{GetActionStmt(uec)};
-      SourcedActionStmt cact{GetActionStmt(cec)};
-      update.ift = uact;
-      capture = cact;
-      if (uec == st1) {
-        captureFirst = false;
-      }
-    }
-  } else if (body.size() == 1) {
-    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
-      update = *maybeUpdate;
-      // This is the form with update and capture combined into an IF-THEN-ELSE
-      // statement. The capture-statement is always the ELSE branch.
-      extractCapture();
-    } else {
-      goto invalid;
-    }
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
-    return;
-  invalid:
-    context_.Say(source,
-        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
-    return;
-  }
-
-  // The update must have a form `x = d` or `x => d`.
-  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
-    const SomeExpr &atom{maybeWrite->lhs};
-    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
-    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
-      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
-
-      if (IsPointerAssignment(*maybeWrite) !=
-          IsPointerAssignment(*maybeCapture)) {
-        context_.Say(capture.source,
-            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-        return;
-      }
-    } else {
-      if (!IsAssignment(capture.stmt)) {
-        context_.Say(capture.source,
-            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
-      }
-      return;
-    }
-  } else {
-    if (!IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
-    }
-    return;
-  }
-
-  // update.iff should be empty here, the capture statement should be
-  // stored in "capture".
-
-  // Fill out the analysis in the AST node.
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  bool condUnused{std::visit(
-      [](auto &&s) {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
-          return true;
-        } else {
-          return false;
-        }
-      },
-      update.cond.u)};
-
-  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
-  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
-
-  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
-  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
-
-  if (captureFirst) {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
-  } else {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicRead(
-    const parser::OpenMPAtomicConstruct &x) {
-  // [6.0:190:5-7]
-  // A read structured block is read-statement, a read statement that has one
-  // of the following forms:
-  //   v = x
-  //   v => x
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Read cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
-      CheckAtomicReadAssignment(*maybeRead, action.source);
-
-      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
-        const SomeExpr &atom{*maybe};
-        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
-            MakeAtomicAnalysisOp(Analysis::None));
-      }
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+      // This is diagnosed later.
+      return;
     }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC READ operation should have a single statement"_err_en_US);
   }
 }
 
-void OmpStructureChecker::CheckAtomicWrite(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Write cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeWrite->lhs};
-      CheckAtomicWriteAssignment(*maybeWrite, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC WRITE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdate(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &block{std::get<parser::Block>(x.t)};
-
-  bool isConditional{x.IsCompare()};
-  bool isCapture{x.IsCapture()};
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (isConditional && isCapture) {
-    CheckAtomicConditionalUpdateCapture(x, body, x.source);
-  } else if (isConditional) {
-    CheckAtomicConditionalUpdate(x, body, x.source);
-  } else if (isCapture) {
-    CheckAtomicUpdateCapture(x, body, x.source);
-  } else { // update-only
-    CheckAtomicUpdateOnly(x, body, x.source);
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
-  if (visitedAtomicSource_.empty())
-    visitedAtomicSource_ = x.source;
-
-  // All of the following groups have the "exclusive" property, i.e. at
-  // most one clause from each group is allowed.
-  // The exclusivity-checking code should eventually be unified for all
-  // clauses, with clause groups defined in OMP.td.
-  std::array atomic{llvm::omp::Clause::OMPC_read,
-      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
-  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
-      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
-      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
-
-  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
-                          std::string_view name,
-                          const parser::OmpClauseList &clauses) {
-    const parser::OmpClause *present{nullptr};
-    for (const parser::OmpClause &clause : clauses.v) {
-      llvm::omp::Clause id{clause.Id()};
-      if (!llvm::is_contained(group, id)) {
-        continue;
-      }
-      if (present == nullptr) {
-        present = &clause;
-        continue;
-      } else if (id == present->Id()) {
-        // Ignore repetitions of the same clause, those will be diagnosed
-        // separately.
-        continue;
-      }
-      parser::MessageFormattedText txt(
-          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
-          name.data());
-      parser::Message message(clause.source, txt);
-      message.Attach(present->source,
-          "Previous clause from this group provided here"_en_US);
-      context_.Say(std::move(message));
-      return;
-    }
-  }};
-
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
-  llvm::omp::Clause kind{x.GetKind()};
-
-  checkExclusive(atomic, "atomic", dirSpec.Clauses());
-  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
-
-  switch (kind) {
-  case llvm::omp::Clause::OMPC_read:
-    CheckAtomicRead(x);
+void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
+  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  case llvm::omp::Directive::OMPD_scope:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
     break;
-  case llvm::omp::Clause::OMPC_write:
-    CheckAtomicWrite(x);
+  // 2.7.3 end-single-clause -> copyprivate-clause |
+  //                            nowait-clause
+  case llvm::omp::Directive::OMPD_single:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
     break;
-  case llvm::omp::Clause::OMPC_update:
-    CheckAtomicUpdate(x);
+  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
+  case llvm::omp::Directive::OMPD_workshare:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_workshare);
     break;
   default:
+    // no clauses are allowed
     break;
   }
 }
 
-void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
-  dirContext_.pop_back();
+// TODO: Verify the popping of dirContext requirement after nowait
+// implementation, as there is an implicit barrier at the end of the worksharing
+// constructs unless a nowait clause is specified. Only OMPD_end_single and
+// end_workshareare popped as they are pushed while entering the
+// EndBlockDirective.
+void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
+    dirContext_.pop_back();
+  }
 }
 
 // Clauses
@@ -5208,102 +3162,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
-  unsigned version{context_.langOptions().OpenMPVersion};
-  llvm::omp::Directive dir{GetContext().directive};
-  parser::CharBlock clauseSource{GetContext().clauseSource};
-  const parser::OmpLinearModifier *linearMod{nullptr};
-
-  SymbolSourceMap symbols;
-  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
-  CheckCrayPointee(objects, "LINEAR", false);
-  GetSymbolsInObjectList(objects, symbols);
-
-  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
-    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      context_.Say(source,
-          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
-          symbol->name(), desc.name.str());
-    }
-  }};
-
-  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
-    auto &modifiers{OmpGetModifiers(x.v)};
-    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
-    if (linearMod) {
-      // 2.7 Loop Construct Restriction
-      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
-        context_.Say(clauseSource,
-            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
-            ContextDirectiveAsFortran());
-        return;
-      }
-
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      for (auto &[symbol, source] : symbols) {
-        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
-          CheckIntegerNoRef(symbol, source);
-        } else {
-          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
-              !IsPolymorphic(*symbol)) {
-            context_.Say(source,
-                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
-                symbol->name(), desc.name.str());
-          }
-        }
-        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
-            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
-          if (!IsDummy(*symbol) || IsValue(*symbol)) {
-            context_.Say(source,
-                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
-                desc.name.str(), symbol->name());
-          }
-        }
-      } // for (symbol, source)
-
-      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
-        context_.Say(OmpGetModifierSource(modifiers, linearMod),
-            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
-            ThisVersion(version));
-      }
-    }
-  }
-
-  // OpenMP 5.2: Ordered clause restriction
-  if (const auto *clause{
-          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
-    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
-    if (orderedClause.v) {
-      return;
-    }
-  }
-
-  // OpenMP 5.2: Linear clause Restrictions
-  for (auto &[symbol, source] : symbols) {
-    if (!linearMod) {
-      // Already checked this with the modifier present.
-      CheckIntegerNoRef(symbol, source);
-    }
-    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
-      context_.Say(source,
-          "The list item `%s` must be a dummy argument"_err_en_US,
-          symbol->name());
-    }
-    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
-      context_.Say(source,
-          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
-          symbol->name());
-    }
-    if (FindCommonBlockContaining(*symbol)) {
-      context_.Say(source,
-          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
-          symbol->name());
-    }
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OmpClause::Detach &x) {
   unsigned version{context_.langOptions().OpenMPVersion};
   if (version >= 52) {
@@ -6012,503 +3870,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::OmpxBare &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
-  OmpVerifyModifiers(
-      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
-}
-
-void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
-  EnterDirectiveNest(ContextSelectorNest);
-
-  using SetName = parser::OmpTraitSetSelectorName;
-  std::map<SetName::Value, const SetName *> visited;
-
-  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
-    auto &name{std::get<SetName>(traitSet.t)};
-    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait set name %s in a context specifier"_err_en_US,
-          showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait set %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-    CheckTraitSetSelector(traitSet);
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
-  ExitDirectiveNest(ContextSelectorNest);
-}
-
-const std::list<parser::OmpTraitProperty> &
-OmpStructureChecker::GetTraitPropertyList(
-    const parser::OmpTraitSelector &trait) {
-  static const std::list<parser::OmpTraitProperty> empty{};
-  auto &[_, maybeProps]{trait.t};
-  if (maybeProps) {
-    using PropertyList = std::list<parser::OmpTraitProperty>;
-    return std::get<PropertyList>(maybeProps->t);
-  } else {
-    return empty;
-  }
-}
-
-std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
-    const parser::OmpTraitProperty &property) {
-  using MaybeClause = std::optional<llvm::omp::Clause>;
-
-  // The parser for OmpClause will only succeed if the clause was
-  // given with all required arguments.
-  // If this is a string or complex extension with a clause name,
-  // treat it as a clause and let the trait checker deal with it.
-
-  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
-    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
-    if (id != llvm::omp::Clause::OMPC_unknown) {
-      return id;
-    } else {
-      return std::nullopt;
-    }
-  }};
-
-  return common::visit( //
-      common::visitors{
-          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
-            return getClauseFromString(x.v);
-          },
-          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
-            return x.value().Id();
-          },
-          [&](const parser::ScalarExpr &x) -> MaybeClause {
-            return std::nullopt;
-          },
-          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
-            using ExtProperty = parser::OmpTraitPropertyExtension;
-            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
-              return getClauseFromString(name->v);
-            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
-              return getClauseFromString(
-                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
-            }
-            return std::nullopt;
-          },
-      },
-      property.u);
-}
-
-void OmpStructureChecker::CheckTraitSelectorList(
-    const std::list<parser::OmpTraitSelector> &traits) {
-  // [6.0:322:20]
-  // Each trait-selector-name may only be specified once in a trait selector
-  // set.
-
-  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
-  using TraitName = parser::OmpTraitSelectorName;
-  using BareName = decltype(TraitName::u);
-  std::map<BareName, const TraitName *> visited;
-
-  for (const parser::OmpTraitSelector &trait : traits) {
-    auto &name{std::get<TraitName>(trait.t)};
-
-    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait name %s in a trait set"_err_en_US, showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSetSelector(
-    const parser::OmpTraitSetSelector &traitSet) {
-
-  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
-  //
-  // Construct      |     Simd, directive-name |      Yes |       No |    No |
-  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
-  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
-  //                |      Extension, Requires |          |          |       |
-  //                |                   Vendor |          |          |       |
-  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
-  //                |                Kind, Uid |          |          |       |
-  // User           |                Condition |       No |       No |   Yes |
-
-  struct TraitSetConfig {
-    std::set<parser::OmpTraitSelectorName::Value> allowed;
-    bool allowsDirectiveTraits;
-    bool allowsExtensionTraits;
-    bool allowsScore;
-  };
-
-  using SName = parser::OmpTraitSetSelectorName::Value;
-  using TName = parser::OmpTraitSelectorName::Value;
-
-  static const std::map<SName, TraitSetConfig> configs{
-      {SName::Construct, //
-          {{TName::Simd}, true, false, false}},
-      {SName::Device, //
-          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
-      {SName::Implementation, //
-          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
-               TName::Vendor},
-              false, true, true}},
-      {SName::Target_Device, //
-          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
-               TName::Uid},
-              false, true, false}},
-      {SName::User, //
-          {{TName::Condition}, false, false, true}},
-  };
-
-  auto checkTraitSet{[&](const TraitSetConfig &config) {
-    auto &[setName, traits]{traitSet.t};
-    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
-
-    // Check if there are any duplicate traits.
-    CheckTraitSelectorList(traits);
-
-    for (const parser::OmpTraitSelector &trait : traits) {
-      // Don't use structured bindings here, because they cannot be captured
-      // before C++20.
-      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
-      auto &maybeProps =
-          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
-              trait.t);
-
-      // Check allowed traits
-      common::visit( //
-          common::visitors{
-              [&](parser::OmpTraitSelectorName::Value v) {
-                if (!config.allowed.count(v)) {
-                  context_.Say(traitName.source,
-                      "%s is not a valid trait for %s trait set"_err_en_US,
-                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
-                }
-              },
-              [&](llvm::omp::Directive) {
-                if (!config.allowsDirectiveTraits) {
-                  context_.Say(traitName.source,
-                      "Directive name is not a valid trait for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-              [&](const std::string &) {
-                if (!config.allowsExtensionTraits) {
-                  context_.Say(traitName.source,
-                      "Extension traits are not valid for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-          },
-          traitName.u);
-
-      // Check score
-      if (maybeProps) {
-        auto &[maybeScore, _]{maybeProps->t};
-        if (maybeScore) {
-          CheckTraitScore(*maybeScore);
-        }
-      }
-
-      // Check the properties of the individual traits
-      CheckTraitSelector(traitSet, trait);
-    }
-  }};
-
-  checkTraitSet(
-      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
-}
-
-void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
-  // [6.0:322:23]
-  // A score-expression must be a non-negative constant integer expression.
-  if (auto value{GetIntValue(score)}; !value || value < 0) {
-    context_.Say(score.source,
-        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
-  }
-}
-
-bool OmpStructureChecker::VerifyTraitPropertyLists(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  using PropertyList = std::list<parser::OmpTraitProperty>;
-  auto &[traitName, maybeProps]{trait.t};
-
-  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
-                             const std::string &message) {
-    bool foundInvalid{false};
-    for (const parser::OmpTraitProperty &prop : properties) {
-      if (!isValid(prop)) {
-        if (foundInvalid) {
-          context_.Say(
-              prop.source, "More invalid properties are present"_err_en_US);
-          break;
-        }
-        context_.Say(prop.source, "%s"_err_en_US, message);
-        foundInvalid = true;
-      }
-    }
-    return !foundInvalid;
-  }};
-
-  bool invalid{false};
-
-  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
-    // Directive-name traits don't have properties.
-    if (maybeProps) {
-      context_.Say(trait.source,
-          "Directive-name traits cannot have properties"_err_en_US);
-      invalid = true;
-    }
-  }
-  // Ignore properties on extension traits.
-
-  // See `TraitSelectorParser` in openmp-parser.cpp
-  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
-    switch (*v) {
-    // name-list properties
-    case parser::OmpTraitSelectorName::Value::Arch:
-    case parser::OmpTraitSelectorName::Value::Extension:
-    case parser::OmpTraitSelectorName::Value::Isa:
-    case parser::OmpTraitSelectorName::Value::Kind:
-    case parser::OmpTraitSelectorName::Value::Uid:
-    case parser::OmpTraitSelectorName::Value::Vendor:
-      if (maybeProps) {
-        auto isName{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isName, "Trait property should be a name");
-      }
-      break;
-    // clause-list
-    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
-    case parser::OmpTraitSelectorName::Value::Requires:
-    case parser::OmpTraitSelectorName::Value::Simd:
-      if (maybeProps) {
-        auto isClause{[&](const parser::OmpTraitProperty &prop) {
-          return GetClauseFromProperty(prop).has_value();
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isClause, "Trait property should be a clause");
-      }
-      break;
-    // expr-list
-    case parser::OmpTraitSelectorName::Value::Condition:
-    case parser::OmpTraitSelectorName::Value::Device_Num:
-      if (maybeProps) {
-        auto isExpr{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::ScalarExpr>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isExpr, "Trait property should be a scalar expression");
-      }
-      break;
-    } // switch
-  }
-
-  return !invalid;
-}
-
-void OmpStructureChecker::CheckTraitSelector(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  auto &[traitName, maybeProps]{trait.t};
-
-  // Only do the detailed checks if the property lists are valid.
-  if (VerifyTraitPropertyLists(traitSet, trait)) {
-    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
-        std::holds_alternative<std::string>(traitName.u)) {
-      // No properties here: directives don't have properties, and
-      // we don't implement any extension traits now.
-      return;
-    }
-
-    // Specific traits we want to check.
-    // Limitations:
-    // (1) The properties for these traits are defined in "Additional
-    // Definitions for the OpenMP API Specification". It's not clear how
-    // to define them in a portable way, and how to verify their validity,
-    // especially if they get replaced by their integer values (in case
-    // they are defined as enums).
-    // (2) These are entirely implementation-defined, and at the moment
-    // there is no known schema to validate these values.
-    auto v{std::get<TraitName::Value>(traitName.u)};
-    switch (v) {
-    case TraitName::Value::Arch:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Atomic_Default_Mem_Order:
-      CheckTraitADMO(traitSet, trait);
-      break;
-    case TraitName::Value::Condition:
-      CheckTraitCondition(traitSet, trait);
-      break;
-    case TraitName::Value::Device_Num:
-      CheckTraitDeviceNum(traitSet, trait);
-      break;
-    case TraitName::Value::Extension:
-      // Ignore
-      break;
-    case TraitName::Value::Isa:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Kind:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Requires:
-      CheckTraitRequires(traitSet, trait);
-      break;
-    case TraitName::Value::Simd:
-      CheckTraitSimd(traitSet, trait);
-      break;
-    case TraitName::Value::Uid:
-      // Unchecked, TBD(2)
-      break;
-    case TraitName::Value::Vendor:
-      // Unchecked, TBD(1)
-      break;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitADMO(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single clause property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto clauseId{*GetClauseFromProperty(property)};
-    // Check that the clause belongs to the memory-order clause-set.
-    // Clause sets will hopefully be autogenerated at some point.
-    switch (clauseId) {
-    case llvm::omp::Clause::OMPC_acq_rel:
-    case llvm::omp::Clause::OMPC_acquire:
-    case llvm::omp::Clause::OMPC_relaxed:
-    case llvm::omp::Clause::OMPC_release:
-    case llvm::omp::Clause::OMPC_seq_cst:
-      break;
-    default:
-      context_.Say(property.source,
-          "%s trait requires a clause from the memory-order clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitCondition(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
-
-    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
-    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
-      context_.Say(property.source,
-          "%s trait requires a single LOGICAL expression"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitDeviceNum(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  }
-  // No other checks at the moment.
-}
-
-void OmpStructureChecker::CheckTraitRequires(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_requires, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause from the requirement clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSimd(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_declare_simd, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()),
-          parser::ToUpperCaseLetters(
-              getDirectiveName(llvm::omp::OMPD_declare_simd)));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
 llvm::StringRef OmpStructureChecker::getClauseName(llvm::omp::Clause clause) {
   return llvm::omp::getOpenMPClauseName(clause);
 }
@@ -6519,39 +3880,6 @@ llvm::StringRef OmpStructureChecker::getDirectiveName(
   return llvm::omp::getOpenMPDirectiveName(directive, version);
 }
 
-const Symbol *OmpStructureChecker::GetObjectSymbol(
-    const parser::OmpObject &object) {
-  // Some symbols may be missing if the resolution failed, e.g. when an
-  // undeclared name is used with implicit none.
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    auto &last{GetLastName(*desg)};
-    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
-  }
-  return nullptr;
-}
-
-const Symbol *OmpStructureChecker::GetArgumentSymbol(
-    const parser::OmpArgument &argument) {
-  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
-    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
-      return GetObjectSymbol(*object);
-    }
-  }
-  return nullptr;
-}
-
-std::optional<parser::CharBlock> OmpStructureChecker::GetObjectSource(
-    const parser::OmpObject &object) {
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->source;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    return GetLastName(*desg).source;
-  }
-  return std::nullopt;
-}
-
 void OmpStructureChecker::CheckDependList(const parser::DataRef &d) {
   common::visit(
       common::visitors{
@@ -6861,7 +4189,7 @@ void OmpStructureChecker::CheckWorkshareBlockStmts(
 }
 
 void OmpStructureChecker::CheckIfContiguous(const parser::OmpObject &object) {
-  if (auto contig{IsContiguous(object)}; contig && !*contig) {
+  if (auto contig{IsContiguous(context_, object)}; contig && !*contig) {
     const parser::Name *name{GetObjectName(object)};
     assert(name && "Expecting name component");
     context_.Say(name->source,
@@ -6964,22 +4292,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::SelfMaps &x) {
   CheckAllowedRequiresClause(llvm::omp::Clause::OMPC_self_maps);
 }
 
-void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
-  Base::Enter(x);
-  loopStack_.push_back(&x);
-}
-
-void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top = loopStack_.back();
-  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
-  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-  Base::Leave(x);
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPInteropConstruct &x) {
   bool isDependClauseOccured{false};
   int targetCount{0}, targetSyncCount{0};
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index beb6e0528e814..2a3853335fd1c 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -162,10 +162,6 @@ class OmpStructureChecker
 
 private:
   bool CheckAllowedClause(llvmOmpClause clause);
-  bool IsVariableListItem(const Symbol &sym);
-  bool IsExtendedListItem(const Symbol &sym);
-  bool IsCommonBlock(const Symbol &sym);
-  std::optional<bool> IsContiguous(const parser::OmpObject &object);
   void CheckVariableListItem(const SymbolSourceMap &symbols);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
@@ -215,10 +211,6 @@ class OmpStructureChecker
       typename IterTy = decltype(std::declval<RangeTy>().begin())>
   std::optional<IterTy> FindDuplicate(RangeTy &&);
 
-  const Symbol *GetObjectSymbol(const parser::OmpObject &object);
-  const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
-  std::optional<parser::CharBlock> GetObjectSource(
-      const parser::OmpObject &object);
   void CheckDependList(const parser::DataRef &);
   void CheckDependArraySection(
       const common::Indirection<parser::ArrayElement> &, const parser::Name &);
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
new file mode 100644
index 0000000000000..055ff8e6b5a29
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -0,0 +1,389 @@
+//===-- lib/Semantics/openmp-utils.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Common/reference.h"
+#include "flang/Common/visit.h"
+#include "flang/Evaluate/check-expression.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
+#include "flang/Evaluate/variable.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/expression.h"
+#include "flang/Semantics/semantics.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+
+#include <optional>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics::omp {
+
+std::string ThisVersion(unsigned version) {
+  std::string tv{
+      std::to_string(version / 10) + "." + std::to_string(version % 10)};
+  return "OpenMP v" + tv;
+}
+
+std::string TryVersion(unsigned version) {
+  return "try -fopenmp-version=" + std::to_string(version);
+}
+
+const parser::Designator *GetDesignatorFromObj(
+    const parser::OmpObject &object) {
+  return std::get_if<parser::Designator>(&object.u);
+}
+
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object) {
+  if (auto *desg{GetDesignatorFromObj(object)}) {
+    return std::get_if<parser::DataRef>(&desg->u);
+  }
+  return nullptr;
+}
+
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object) {
+  if (auto *dataRef{GetDataRefFromObj(object)}) {
+    using ElementIndirection = common::Indirection<parser::ArrayElement>;
+    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
+      return &ind->value();
+    }
+  }
+  return nullptr;
+}
+
+const Symbol *GetObjectSymbol(const parser::OmpObject &object) {
+  // Some symbols may be missing if the resolution failed, e.g. when an
+  // undeclared name is used with implicit none.
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    auto &last{GetLastName(*desg)};
+    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
+  }
+  return nullptr;
+}
+
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument) {
+  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
+    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
+      return GetObjectSymbol(*object);
+    }
+  }
+  return nullptr;
+}
+
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object) {
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->source;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    return GetLastName(*desg).source;
+  }
+  return std::nullopt;
+}
+
+bool IsCommonBlock(const Symbol &sym) {
+  return sym.detailsIf<CommonBlockDetails>() != nullptr;
+}
+
+bool IsVariableListItem(const Symbol &sym) {
+  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
+}
+
+bool IsExtendedListItem(const Symbol &sym) {
+  return IsVariableListItem(sym) || sym.IsSubprogram();
+}
+
+bool IsVarOrFunctionRef(const MaybeExpr &expr) {
+  if (expr) {
+    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
+        evaluate::IsVariable(*expr);
+  } else {
+    return false;
+  }
+}
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
+  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
+  // ForwardOwningPointer           typedExpr
+  // `- GenericExprWrapper          ^.get()
+  //    `- std::optional<Expr>      ^->v
+  return typedExpr.get()->v;
+}
+
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr) {
+  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
+    return maybeExpr->GetType();
+  } else {
+    return std::nullopt;
+  }
+}
+
+namespace {
+struct ContiguousHelper {
+  ContiguousHelper(SemanticsContext &context)
+      : fctx_(context.foldingContext()) {}
+
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
+    return Visit(x.value());
+  }
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Reference<Contained> &x) {
+    return Visit(x.get());
+  }
+  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
+    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
+  }
+  template <typename T>
+  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
+    return common::visit(
+        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
+  }
+  template <typename T> std::optional<bool> Visit(const T &) {
+    // Everything else.
+    return std::nullopt;
+  }
+
+private:
+  evaluate::FoldingContext &fctx_;
+};
+} // namespace
+
+// Return values:
+// - std::optional<bool>{true} if the object is known to be contiguous
+// - std::optional<bool>{false} if the object is known not to be contiguous
+// - std::nullopt if the object contiguity cannot be determined
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object) {
+  return common::visit( //
+      common::visitors{
+          [&](const parser::Name &x) {
+            // Any member of a common block must be contiguous.
+            return std::optional<bool>{true};
+          },
+          [&](const parser::Designator &x) {
+            evaluate::ExpressionAnalyzer ea{semaCtx};
+            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
+              return ContiguousHelper{semaCtx}.Visit(*maybeExpr);
+            }
+            return std::optional<bool>{};
+          },
+      },
+      object.u);
+}
+
+struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
+                                 std::vector<SomeExpr>, false> {
+  using Result = std::vector<SomeExpr>;
+  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
+  DesignatorCollector() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <typename T> //
+  Result operator()(const evaluate::Designator<T> &x) const {
+    // Once in a designator, don't traverse it any further (i.e. only
+    // collect top-level designators).
+    auto copy{x};
+    return Result{AsGenericExpr(std::move(copy))};
+  }
+
+  template <typename... Rs> //
+  Result Combine(Result &&result, Rs &&...results) const {
+    Result v(std::move(result));
+    auto moveAppend{[](auto &accum, auto &&other) {
+      for (auto &&s : other) {
+        accum.push_back(std::move(s));
+      }
+    }};
+    (moveAppend(v, std::move(results)), ...);
+    return v;
+  }
+};
+
+struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
+  using Base = evaluate::AnyTraverse<VariableFinder>;
+  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
+
+  using Base::operator();
+
+  template <typename T>
+  bool operator()(const evaluate::Designator<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+  template <typename T>
+  bool operator()(const evaluate::FunctionRef<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+private:
+  const SomeExpr &var;
+};
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr) {
+  return DesignatorCollector{}(expr);
+}
+
+static bool HasCommonDesignatorSymbols(
+    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
+  // Compare the designators used in "other" with the designators whose
+  // symbols are given in baseSyms.
+  // This is a part of the check if these two expressions can access the same
+  // storage: if the designators used in them are different enough, then they
+  // will be assumed not to access the same memory.
+  //
+  // Consider an (array element) expression x%y(w%z), the corresponding symbol
+  // vector will be {x, y, w, z} (i.e. the symbols for these names).
+  // Check whether this exact sequence appears anywhere in any the symbol
+  // vector for "other". This will be true for x(y) and x(y+1), so this is
+  // not a sufficient condition, but can be used to eliminate candidates
+  // before doing more exhaustive checks.
+  //
+  // If any of the symbols in this sequence are function names, assume that
+  // there is no storage overlap, mostly because it would be impossible in
+  // general to determine what storage the function will access.
+  // Note: if f is pure, then two calls to f will access the same storage
+  // when called with the same arguments. This check is not done yet.
+
+  if (llvm::any_of(
+          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
+    // If there is a function symbol in the chain then we can't infer much
+    // about the accessed storage.
+    return false;
+  }
+
+  auto isSubsequence{// Is u a subsequence of v.
+      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
+        size_t us{u.size()}, vs{v.size()};
+        if (us > vs) {
+          return false;
+        }
+        for (size_t off{0}; off != vs - us + 1; ++off) {
+          bool same{true};
+          for (size_t i{0}; i != us; ++i) {
+            if (u[i] != v[off + i]) {
+              same = false;
+              break;
+            }
+          }
+          if (same) {
+            return true;
+          }
+        }
+        return false;
+      }};
+
+  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
+  return isSubsequence(baseSyms, otherSyms);
+}
+
+static bool HasCommonTopLevelDesignators(
+    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
+  // Compare designators directly as expressions. This will ensure
+  // that x(y) and x(y+1) are not flagged as overlapping, whereas
+  // the symbol vectors for both of these would be identical.
+  std::vector<SomeExpr> otherDsgs{GetAllDesignators(other)};
+
+  for (auto &s : baseDsgs) {
+    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
+      return true;
+    }
+  }
+  return false;
+}
+
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
+  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
+  std::vector<SomeExpr> baseDsgs{GetAllDesignators(base)};
+
+  for (const SomeExpr &expr : exprs) {
+    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
+      continue;
+    }
+    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
+      return &expr;
+    }
+  }
+  return nullptr;
+}
+
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
+  return VariableFinder{sub}(super);
+}
+
+// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
+// to separate cases where the source has something that looks like an
+// assignment, but is semantically wrong (diagnosed by general semantic
+// checks), and where the source has some other statement (which we want
+// to report as "should be an assignment").
+bool IsAssignment(const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return false;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+
+  return common::visit(
+      [](auto &&s) -> bool {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        return std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>;
+      },
+      x->u);
+}
+
+bool IsPointerAssignment(const evaluate::Assignment &x) {
+  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
+      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
+}
+
+/// parser::Block is a list of executable constructs, parser::BlockConstruct
+/// is Fortran's BLOCK/ENDBLOCK construct.
+/// Strip the outermost BlockConstructs, return the reference to the Block
+/// in the executable part of the innermost of the stripped constructs.
+/// Specifically, if the given `block` has a single entry (it's a list), and
+/// the entry is a BlockConstruct, get the Block contained within. Repeat
+/// this step as many times as possible.
+const parser::Block &GetInnermostExecPart(const parser::Block &block) {
+  const parser::Block *iter{&block};
+  while (iter->size() == 1) {
+    const parser::ExecutionPartConstruct &ep{iter->front()};
+    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
+      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
+      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
+        iter = &std::get<parser::Block>(bc->value().t);
+        continue;
+      }
+    }
+    break;
+  }
+  return *iter;
+}
+
+} // namespace Fortran::semantics::omp
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
new file mode 100644
index 0000000000000..fca0587a8bc84
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -0,0 +1,62 @@
+//===-- lib/Semantics/openmp-utils.h --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_OPENMP_UTILS_H
+#define FORTRAN_SEMANTICS_OPENMP_UTILS_H
+
+#include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include <optional>
+#include <string>
+
+namespace Fortran::semantics {
+class SemanticsContext;
+class Symbol;
+
+// Add this namespace to avoid potential conflicts
+namespace omp {
+std::string ThisVersion(unsigned version);
+std::string TryVersion(unsigned version);
+
+const parser::Designator *GetDesignatorFromObj(const parser::OmpObject &object);
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object);
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object);
+const Symbol *GetObjectSymbol(const parser::OmpObject &object);
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object);
+
+bool IsCommonBlock(const Symbol &sym);
+bool IsExtendedListItem(const Symbol &sym);
+bool IsVariableListItem(const Symbol &sym);
+bool IsVarOrFunctionRef(const MaybeExpr &expr);
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr);
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr);
+
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object);
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr);
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs);
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super);
+bool IsAssignment(const parser::ActionStmt *x);
+bool IsPointerAssignment(const evaluate::Assignment &x);
+const parser::Block &GetInnermostExecPart(const parser::Block &block);
+} // namespace omp
+} // namespace Fortran::semantics
+
+#endif // FORTRAN_SEMANTICS_OPENMP_UTILS_H

>From 2c3b718a05ee2b4e0e78ef45a6809dae6d04c912 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 08:12:57 -0500
Subject: [PATCH 2/4] Add short description to each added file

---
 flang/lib/Semantics/check-omp-atomic.cpp        | 5 +++++
 flang/lib/Semantics/check-omp-loop.cpp          | 5 +++++
 flang/lib/Semantics/check-omp-metadirective.cpp | 4 ++++
 flang/lib/Semantics/openmp-utils.cpp            | 4 ++++
 flang/lib/Semantics/openmp-utils.h              | 4 ++++
 5 files changed, 22 insertions(+)

diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index 19425a69523d3..e22c747621680 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -5,6 +5,11 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Semantic checks related to the ATOMIC construct.
+//
+//===----------------------------------------------------------------------===//
+
 
 #include "check-omp-structure.h"
 #include "openmp-utils.h"
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
index f2aad9efdd8de..ab40459f0e7bb 100644
--- a/flang/lib/Semantics/check-omp-loop.cpp
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -5,6 +5,11 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Semantic checks for constructs and clauses related to loops.
+//
+//===----------------------------------------------------------------------===//
+
 
 #include "check-omp-structure.h"
 
diff --git a/flang/lib/Semantics/check-omp-metadirective.cpp b/flang/lib/Semantics/check-omp-metadirective.cpp
index bf27797a7c44a..03487da64f1bf 100644
--- a/flang/lib/Semantics/check-omp-metadirective.cpp
+++ b/flang/lib/Semantics/check-omp-metadirective.cpp
@@ -5,6 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Semantic checks for METADIRECTIVE and related constructs/clauses.
+//
+//===----------------------------------------------------------------------===//
 
 #include "check-omp-structure.h"
 
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index 055ff8e6b5a29..fd9596a09cd52 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -5,6 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
 
 #include "openmp-utils.h"
 
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index fca0587a8bc84..dbb0565215357 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -5,6 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
 
 #ifndef FORTRAN_SEMANTICS_OPENMP_UTILS_H
 #define FORTRAN_SEMANTICS_OPENMP_UTILS_H

>From 420f15724336edadf3408cce236aaf388f2ac026 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 08:23:19 -0500
Subject: [PATCH 3/4] Port part of 65cb0eae

---
 flang/lib/Semantics/check-omp-atomic.cpp |   1 -
 flang/lib/Semantics/check-omp-loop.cpp   | 101 +++++++++++++----------
 2 files changed, 56 insertions(+), 46 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index e22c747621680..047c604693460 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "check-omp-structure.h"
 #include "openmp-utils.h"
 
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
index ab40459f0e7bb..b82e2f7342d85 100644
--- a/flang/lib/Semantics/check-omp-loop.cpp
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "check-omp-structure.h"
 
 #include "check-directive-structure.h"
@@ -301,10 +300,13 @@ void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
   }
   SetLoopInfo(x);
 
-  if (const auto &doConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
-    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &doConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
+      CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+    }
   }
   CheckLoopItrVariableIsInt(x);
   CheckAssociatedLoopConstraints(x);
@@ -327,39 +329,45 @@ const parser::Name OmpStructureChecker::GetLoopIndex(
 }
 
 void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const parser::DoConstruct *loop{&*loopConstruct};
-    if (loop && loop->IsDoNormal()) {
-      const parser::Name &itrVal{GetLoopIndex(loop)};
-      SetLoopIv(itrVal.symbol);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const parser::DoConstruct *loop{&*loopConstruct};
+      if (loop && loop->IsDoNormal()) {
+        const parser::Name &itrVal{GetLoopIndex(loop)};
+        SetLoopIv(itrVal.symbol);
+      }
     }
   }
 }
 
 void OmpStructureChecker::CheckLoopItrVariableIsInt(
     const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
 
-    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-      if (loop->IsDoNormal()) {
-        const parser::Name &itrVal{GetLoopIndex(loop)};
-        if (itrVal.symbol) {
-          const auto *type{itrVal.symbol->GetType()};
-          if (!type->IsNumeric(TypeCategory::Integer)) {
-            context_.Say(itrVal.source,
-                "The DO loop iteration"
-                " variable must be of the type integer."_err_en_US,
-                itrVal.ToString());
+      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+        if (loop->IsDoNormal()) {
+          const parser::Name &itrVal{GetLoopIndex(loop)};
+          if (itrVal.symbol) {
+            const auto *type{itrVal.symbol->GetType()};
+            if (!type->IsNumeric(TypeCategory::Integer)) {
+              context_.Say(itrVal.source,
+                  "The DO loop iteration"
+                  " variable must be of the type integer."_err_en_US,
+                  itrVal.ToString());
+            }
           }
         }
+        // Get the next DoConstruct if block is not empty.
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                 : nullptr;
       }
-      // Get the next DoConstruct if block is not empty.
-      const auto &block{std::get<parser::Block>(loop->t)};
-      const auto it{block.begin()};
-      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                               : nullptr;
     }
   }
 }
@@ -427,25 +435,28 @@ void OmpStructureChecker::CheckDistLinear(
 
     // Match the loop index variables with the collected symbols from linear
     // clauses.
-    if (const auto &loopConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-        if (loop->IsDoNormal()) {
-          const parser::Name &itrVal{GetLoopIndex(loop)};
-          if (itrVal.symbol) {
-            // Remove the symbol from the collected set
-            indexVars.erase(&itrVal.symbol->GetUltimate());
-          }
-          collapseVal--;
-          if (collapseVal == 0) {
-            break;
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &loopConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+          if (loop->IsDoNormal()) {
+            const parser::Name &itrVal{GetLoopIndex(loop)};
+            if (itrVal.symbol) {
+              // Remove the symbol from the collected set
+              indexVars.erase(&itrVal.symbol->GetUltimate());
+            }
+            collapseVal--;
+            if (collapseVal == 0) {
+              break;
+            }
           }
+          // Get the next DoConstruct if block is not empty.
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                   : nullptr;
         }
-        // Get the next DoConstruct if block is not empty.
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                 : nullptr;
       }
     }
 

>From 265bdc3cf759f117070f511738888c0116f1ec92 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 1 Jul 2025 08:39:15 +0100
Subject: [PATCH 4/4] Port PR145917

---
 flang/include/flang/Parser/parse-tree.h       |   5 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  10 ++
 flang/lib/Parser/unparse.cpp                  |   3 +-
 flang/lib/Semantics/canonicalize-omp.cpp      |  97 +++++++++++++++--
 flang/lib/Semantics/resolve-directives.cpp    | 102 +++++++++++-------
 ...nested-loop-transformation-construct01.f90 |  20 ++++
 ...nested-loop-transformation-construct02.f90 |  20 ++++
 .../loop-transformation-construct01.f90       |  74 +++++++++++++
 .../loop-transformation-construct02.f90       |  85 +++++++++++++++
 .../loop-transformation-construct01.f90       | 100 +++++++++++++++++
 10 files changed, 464 insertions(+), 52 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
 create mode 100644 flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
 create mode 100644 flang/test/Parser/OpenMP/loop-transformation-construct01.f90
 create mode 100644 flang/test/Parser/OpenMP/loop-transformation-construct02.f90
 create mode 100644 flang/test/Semantics/OpenMP/loop-transformation-construct01.f90

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 61f97b855b0e5..7e752eeb4dfe4 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -267,6 +267,7 @@ struct AccEndCombinedDirective;
 struct OpenACCDeclarativeConstruct;
 struct OpenACCRoutineConstruct;
 struct OpenMPConstruct;
+struct OpenMPLoopConstruct;
 struct OpenMPDeclarativeConstruct;
 struct OmpEndLoopDirective;
 struct OmpMemoryOrderClause;
@@ -5021,11 +5022,13 @@ struct OpenMPBlockConstruct {
 };
 
 // OpenMP directives enclosing do loop
+using NestedConstruct =
+    std::variant<DoConstruct, common::Indirection<OpenMPLoopConstruct>>;
 struct OpenMPLoopConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPLoopConstruct);
   OpenMPLoopConstruct(OmpBeginLoopDirective &&a)
       : t({std::move(a), std::nullopt, std::nullopt}) {}
-  std::tuple<OmpBeginLoopDirective, std::optional<DoConstruct>,
+  std::tuple<OmpBeginLoopDirective, std::optional<NestedConstruct>,
       std::optional<OmpEndLoopDirective>>
       t;
 };
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 96daadfaf1b3b..0a56e888ac44b 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -3769,6 +3769,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   mlir::Location currentLocation =
       converter.genLocation(beginLoopDirective.source);
 
+  auto &optLoopCons =
+      std::get<std::optional<parser::NestedConstruct>>(loopConstruct.t);
+  if (optLoopCons.has_value()) {
+    if (auto *ompNestedLoopCons{
+            std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
+                &*optLoopCons)}) {
+      genOMP(converter, symTable, semaCtx, eval, ompNestedLoopCons->value());
+    }
+  }
+
   llvm::omp::Directive directive =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
   const parser::CharBlock &source =
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ed0f227fd5b98..903287515e559 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2926,7 +2926,8 @@ class UnparseVisitor {
     Walk(std::get<OmpBeginLoopDirective>(x.t));
     Put("\n");
     EndOpenMP();
-    Walk(std::get<std::optional<DoConstruct>>(x.t));
+    Walk(std::get<std::optional<std::variant<DoConstruct,
+            common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t));
     Walk(std::get<std::optional<OmpEndLoopDirective>>(x.t));
   }
   void Unparse(const BasedPointer &x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 5164f1dc6faab..1edcb376596b0 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -8,6 +8,7 @@
 
 #include "canonicalize-omp.h"
 #include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
 
 // After Loop Canonicalization, rewrite OpenMP parse tree to make OpenMP
 // Constructs more structured which provide explicit scopes for later
@@ -125,6 +126,16 @@ class CanonicalizationOfOmp {
     parser::Block::iterator nextIt;
     auto &beginDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
     auto &dir{std::get<parser::OmpLoopDirective>(beginDir.t)};
+    auto missingDoConstruct = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
+    auto tileUnrollError = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "If a loop construct has been fully unrolled, it cannot then be tiled"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
 
     nextIt = it;
     while (++nextIt != block.end()) {
@@ -135,31 +146,95 @@ class CanonicalizationOfOmp {
       if (auto *doCons{GetConstructIf<parser::DoConstruct>(*nextIt)}) {
         if (doCons->GetLoopControl()) {
           // move DoConstruct
-          std::get<std::optional<parser::DoConstruct>>(x.t) =
+          std::get<std::optional<std::variant<parser::DoConstruct,
+              common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t) =
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if (nextIt != block.end()) {
-            if (auto *endDir{
-                    GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
-              std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
-                  std::move(*endDir);
-              block.erase(nextIt);
-            }
+          if (auto *endDir{
+                  GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
+            std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                std::move(*endDir);
+            nextIt = block.erase(nextIt);
           }
         } else {
           messages_.Say(dir.source,
               "DO loop after the %s directive must have loop control"_err_en_US,
               parser::ToUpperCaseLetters(dir.source.ToString()));
         }
+      } else if (auto *ompLoopCons{
+                     GetOmpIf<parser::OpenMPLoopConstruct>(*nextIt)}) {
+        // We should allow UNROLL and TILE constructs to be inserted between an
+        // OpenMP Loop Construct and the DO loop itself
+        auto &nestedBeginDirective =
+            std::get<parser::OmpBeginLoopDirective>(ompLoopCons->t);
+        auto &nestedBeginLoopDirective =
+            std::get<parser::OmpLoopDirective>(nestedBeginDirective.t);
+        if ((nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll ||
+                nestedBeginLoopDirective.v ==
+                    llvm::omp::Directive::OMPD_tile) &&
+            !(nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll &&
+                dir.v == llvm::omp::Directive::OMPD_tile)) {
+          // iterate through the remaining block items to find the end directive
+          // for the unroll/tile directive.
+          parser::Block::iterator endIt;
+          endIt = nextIt;
+          while (endIt != block.end()) {
+            if (auto *endDir{
+                    GetConstructIf<parser::OmpEndLoopDirective>(*endIt)}) {
+              auto &endLoopDirective =
+                  std::get<parser::OmpLoopDirective>(endDir->t);
+              if (endLoopDirective.v == dir.v) {
+                std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                    std::move(*endDir);
+                endIt = block.erase(endIt);
+                continue;
+              }
+            }
+            ++endIt;
+          }
+          RewriteOpenMPLoopConstruct(*ompLoopCons, block, nextIt);
+          auto &ompLoop = std::get<std::optional<parser::NestedConstruct>>(x.t);
+          ompLoop =
+              std::optional<parser::NestedConstruct>{parser::NestedConstruct{
+                  common::Indirection{std::move(*ompLoopCons)}}};
+          nextIt = block.erase(nextIt);
+        } else if (nestedBeginLoopDirective.v ==
+                llvm::omp::Directive::OMPD_unroll &&
+            dir.v == llvm::omp::Directive::OMPD_tile) {
+          // if a loop has been unrolled, the user can not then tile that loop
+          // as it has been unrolled
+          parser::OmpClauseList &unrollClauseList{
+              std::get<parser::OmpClauseList>(nestedBeginDirective.t)};
+          if (unrollClauseList.v.empty()) {
+            // if the clause list is empty for an unroll construct, we assume
+            // the loop is being fully unrolled
+            tileUnrollError(dir, messages_);
+          } else {
+            // parse the clauses for the unroll directive to find the full
+            // clause
+            for (auto clause{unrollClauseList.v.begin()};
+                clause != unrollClauseList.v.end(); ++clause) {
+              if (clause->Id() == llvm::omp::OMPC_full) {
+                tileUnrollError(dir, messages_);
+              }
+            }
+          }
+        } else {
+          messages_.Say(nestedBeginLoopDirective.source,
+              "Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs"_err_en_US,
+              parser::ToUpperCaseLetters(
+                  nestedBeginLoopDirective.source.ToString()));
+        }
       } else {
-        messages_.Say(dir.source,
-            "A DO loop must follow the %s directive"_err_en_US,
-            parser::ToUpperCaseLetters(dir.source.ToString()));
+        missingDoConstruct(dir, messages_);
       }
       // If we get here, we either found a loop, or issued an error message.
       return;
     }
+    if (nextIt == block.end()) {
+      missingDoConstruct(dir, messages_);
+    }
   }
 
   void RewriteOmpAllocations(parser::ExecutionPart &body) {
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..8d741734601a7 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1796,10 +1796,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
 
   if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    if (const auto &doConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      if (doConstruct.value().IsDoWhile()) {
-        return true;
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &doConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        if (doConstruct->IsDoWhile()) {
+          return true;
+        }
       }
     }
   }
@@ -1962,48 +1965,69 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndexAndCheckLoopLevel(
   bool hasCollapseClause{
       clause ? (clause->Id() == llvm::omp::OMPC_collapse) : false};
 
-  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
-  if (outer.has_value()) {
-    for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-      if (loop->IsDoConcurrent()) {
-        // DO CONCURRENT is explicitly allowed for the LOOP construct so long as
-        // there isn't a COLLAPSE clause
-        if (isLoopConstruct) {
-          if (hasCollapseClause) {
-            // hasCollapseClause implies clause != nullptr
-            context_.Say(clause->source,
-                "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &outer{std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      for (const parser::DoConstruct *loop{&*outer}; loop && level > 0;
+          --level) {
+        if (loop->IsDoConcurrent()) {
+          // DO CONCURRENT is explicitly allowed for the LOOP construct so long
+          // as there isn't a COLLAPSE clause
+          if (isLoopConstruct) {
+            if (hasCollapseClause) {
+              // hasCollapseClause implies clause != nullptr
+              context_.Say(clause->source,
+                  "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+            }
+          } else {
+            auto &stmt =
+                std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
+            context_.Say(stmt.source,
+                "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
           }
-        } else {
-          auto &stmt =
-              std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
-          context_.Say(stmt.source,
-              "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
-        }
-      }
-      // go through all the nested do-loops and resolve index variables
-      const parser::Name *iv{GetLoopIndex(*loop)};
-      if (iv) {
-        if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
-          SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
-          iv->symbol = symbol; // adjust the symbol within region
-          AddToContextObjectWithDSA(*symbol, ivDSA);
         }
+        // go through all the nested do-loops and resolve index variables
+        const parser::Name *iv{GetLoopIndex(*loop)};
+        if (iv) {
+          if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
+            SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
+            iv->symbol = symbol; // adjust the symbol within region
+            AddToContextObjectWithDSA(*symbol, ivDSA);
+          }
 
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+        }
       }
+      CheckAssocLoopLevel(level, GetAssociatedClause());
+    } else if (const auto &loop{std::get_if<
+                   common::Indirection<parser::OpenMPLoopConstruct>>(
+                   &*optLoopCons)}) {
+      auto &beginDirective =
+          std::get<parser::OmpBeginLoopDirective>(loop->value().t);
+      auto &beginLoopDirective =
+          std::get<parser::OmpLoopDirective>(beginDirective.t);
+      if (beginLoopDirective.v != llvm::omp::Directive::OMPD_unroll &&
+          beginLoopDirective.v != llvm::omp::Directive::OMPD_tile) {
+        context_.Say(GetContext().directiveSource,
+            "Only UNROLL or TILE constructs are allowed between an OpenMP Loop Construct and a DO construct"_err_en_US,
+            parser::ToUpperCaseLetters(llvm::omp::getOpenMPDirectiveName(
+                GetContext().directive, version)
+                    .str()));
+      } else {
+        PrivatizeAssociatedLoopIndexAndCheckLoopLevel(loop->value());
+      }
+    } else {
+      context_.Say(GetContext().directiveSource,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
+                  .str()));
     }
-    CheckAssocLoopLevel(level, GetAssociatedClause());
-  } else {
-    context_.Say(GetContext().directiveSource,
-        "A DO loop must follow the %s directive"_err_en_US,
-        parser::ToUpperCaseLetters(
-            llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
-                .str()));
   }
 }
+
 void OmpAttributeVisitor::CheckAssocLoopLevel(
     std::int64_t level, const parser::OmpClause *clause) {
   if (clause && level != 0) {
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..a76e7e52100db
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for tile OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end do
+end subroutine
+
+!CHECK: not yet implemented: Unhandled loop directive (tile)
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
new file mode 100644
index 0000000000000..33b7c5a917619
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for unroll OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+program loop_transformation_construct
+  implicit none
+  integer, parameter :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do x = 1, I
+    y(x) = y(x) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end program loop_transformation_construct
+
+!CHECK: not yet implemented: Unhandled loop directive (unroll)
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct01.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..baffc2f6e2f1e
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,74 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with 1 nested loop.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct02.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
new file mode 100644
index 0000000000000..b50e7183841cc
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
@@ -0,0 +1,85 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with multiple nested loops.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP TILE
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END TILE
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90 b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..f718efc32aabf
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,100 @@
+! Testing the Semantics of nested Loop Transformation Constructs
+
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=51
+
+subroutine loop_transformation_construct1
+  implicit none
+
+  !$omp do
+  !ERROR: A DO loop must follow the UNROLL directive
+  !$omp unroll
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !$omp end tile
+  !$omp end do
+  !ERROR: The END TILE directive must follow the DO loop associated with the loop construct
+  !$omp end tile
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs
+  !$omp parallel do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct3
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !ERROR: A DO loop must follow the TILE directive
+  !$omp tile
+end subroutine
+
+subroutine loop_transformation_construct4
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll full
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct5
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct6
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  !$omp unroll partial(2)
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine


From flang-commits at lists.llvm.org  Tue Jul  1 06:48:18 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 06:48:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (PR #146359)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146359@github.com>
Message-ID: <6863e722.170a0220.cdfa7.b387@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/146359

>From 251552277ca266b88e6bcbcf0ba59fd576495911 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 25 Jun 2025 15:25:45 -0500
Subject: [PATCH 1/4] [flang][OpenMP] Split check-omp-structure.cpp into
 smaller files, NFC

Create these new files in flang/lib/Semantics:
  openmp-utils.cpp/.h         - Common utilities
  check-omp-atomic.cpp        - Atomic-related checks
  check-omp-loop.cpp          - Loop constructs/clauses
  check-omp-metadirective.cpp - Metadirective-related checks

Update lists of included headers, std in particular.
---
 flang/lib/Semantics/CMakeLists.txt            |    8 +-
 flang/lib/Semantics/check-omp-atomic.cpp      | 1291 ++++++++
 flang/lib/Semantics/check-omp-loop.cpp        |  655 ++++
 .../lib/Semantics/check-omp-metadirective.cpp |  544 ++++
 flang/lib/Semantics/check-omp-structure.cpp   | 2818 +----------------
 flang/lib/Semantics/check-omp-structure.h     |    8 -
 flang/lib/Semantics/openmp-utils.cpp          |  389 +++
 flang/lib/Semantics/openmp-utils.h            |   62 +
 8 files changed, 3012 insertions(+), 2763 deletions(-)
 create mode 100644 flang/lib/Semantics/check-omp-atomic.cpp
 create mode 100644 flang/lib/Semantics/check-omp-loop.cpp
 create mode 100644 flang/lib/Semantics/check-omp-metadirective.cpp
 create mode 100644 flang/lib/Semantics/openmp-utils.cpp
 create mode 100644 flang/lib/Semantics/openmp-utils.h

diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index c0fda3631c01f..109bc2dbb8569 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -20,6 +20,9 @@ add_flang_library(FortranSemantics
   check-io.cpp
   check-namelist.cpp
   check-nullify.cpp
+  check-omp-atomic.cpp
+  check-omp-loop.cpp
+  check-omp-metadirective.cpp
   check-omp-structure.cpp
   check-purity.cpp
   check-return.cpp
@@ -34,12 +37,13 @@ add_flang_library(FortranSemantics
   mod-file.cpp
   openmp-dsa.cpp
   openmp-modifiers.cpp
+  openmp-utils.cpp
   pointer-assignment.cpp
   program-tree.cpp
-  resolve-labels.cpp
   resolve-directives.cpp
-  resolve-names-utils.cpp
+  resolve-labels.cpp
   resolve-names.cpp
+  resolve-names-utils.cpp
   rewrite-parse-tree.cpp
   runtime-type-info.cpp
   scope.cpp
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
new file mode 100644
index 0000000000000..19425a69523d3
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -0,0 +1,1291 @@
+//===-- lib/Semantics/check-omp-atomic.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/ErrorHandling.h"
+
+#include <cassert>
+#include <list>
+#include <optional>
+#include <string_view>
+#include <tuple>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+namespace operation = Fortran::evaluate::operation;
+
+template <typename T, typename U>
+static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
+  return !(e == f);
+}
+
+// There is no consistent way to get the source of a given ActionStmt, so
+// extract the source information from Statement<ActionStmt> when we can,
+// and keep it around for error reporting in further analyses.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+struct AnalyzedCondStmt {
+  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
+  parser::CharBlock source;
+  SourcedActionStmt ift, iff;
+};
+
+static SourcedActionStmt GetActionStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+static SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
+// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
+// is that the ActionStmt will be either an assignment or a pointer-assignment,
+// otherwise return std::nullopt.
+// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
+// the "typedAssignment" is unset. This can happen if there are semantic errors
+// in the purported assignment.
+static std::optional<evaluate::Assignment> GetEvaluateAssignment(
+    const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
+
+  return common::visit(
+      [](auto &&s) -> std::optional<evaluate::Assignment> {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>) {
+          const TypedAssignment &typed{s.value().typedAssignment};
+          // ForwardOwningPointer                 typedAssignment
+          // `- GenericAssignmentWrapper          ^.get()
+          //    `- std::optional<Assignment>      ^->v
+          return typed.get()->v;
+        } else {
+          return std::nullopt;
+        }
+      },
+      x->u);
+}
+
+static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  // Extract the evaluate::Expr from ScalarLogicalExpr.
+  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
+    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
+    const parser::Expr &expr{logical.thing.thing.value()};
+    return GetEvaluateExpr(expr);
+  }};
+
+  // Recognize either
+  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
+  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
+
+  if (auto &&action{GetActionStmt(x)}) {
+    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
+            &action.stmt->u)}) {
+      const parser::IfStmt &s{ifs->value()};
+      auto &&maybeCond{
+          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
+      auto &thenStmt{
+          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
+      if (maybeCond) {
+        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
+            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
+            SourcedActionStmt{}};
+      }
+    }
+    return std::nullopt;
+  }
+
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    if (auto *ifc{
+            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
+      using ElseBlock = parser::IfConstruct::ElseBlock;
+      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
+      const parser::IfConstruct &s{ifc->value()};
+
+      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
+        // Not expecting any else-if statements.
+        return std::nullopt;
+      }
+      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
+      auto &&maybeCond{getFromLogical(
+          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
+      if (!maybeCond) {
+        return std::nullopt;
+      }
+
+      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)),
+            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
+        if (result.ift.stmt && result.iff.stmt) {
+          return result;
+        }
+      } else {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
+        if (result.ift.stmt) {
+          return result;
+        }
+      }
+    }
+    return std::nullopt;
+  }
+
+  return std::nullopt;
+}
+
+static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
+    parser::CharBlock source) {
+  // Find => in the range, if not found, find = that is not a part of
+  // <=, >=, ==, or /=.
+  auto trim{[](std::string_view v) {
+    const char *begin{v.data()};
+    const char *end{begin + v.size()};
+    while (*begin == ' ' && begin != end) {
+      ++begin;
+    }
+    while (begin != end && end[-1] == ' ') {
+      --end;
+    }
+    assert(begin != end && "Source should not be empty");
+    return parser::CharBlock(begin, end - begin);
+  }};
+
+  std::string_view sv(source.begin(), source.size());
+
+  if (auto where{sv.find("=>")}; where != sv.npos) {
+    std::string_view lhs(sv.data(), where);
+    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
+    return std::make_pair(trim(lhs), trim(rhs));
+  }
+
+  // Go backwards, since all the exclusions above end with a '='.
+  for (size_t next{source.size()}; next > 1; --next) {
+    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
+      std::string_view lhs(sv.data(), next - 1);
+      std::string_view rhs(sv.data() + next, sv.size() - next);
+      return std::make_pair(trim(lhs), trim(rhs));
+    }
+  }
+  llvm_unreachable("Could not find assignment operator");
+}
+
+static bool IsCheckForAssociated(const SomeExpr &cond) {
+  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
+}
+
+static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
+  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
+  // for example.
+  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
+}
+
+static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
+  if (value) {
+    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
+        evaluate::GenericExprWrapper::Deleter);
+  }
+}
+
+static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
+    std::optional<evaluate::Assignment> value) {
+  if (value) {
+    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
+        evaluate::GenericAssignmentWrapper::Deleter);
+  }
+}
+
+static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
+    int what,
+    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
+  parser::OpenMPAtomicConstruct::Analysis::Op operation;
+  operation.what = what;
+  SetAssignment(operation.assign, maybeAssign);
+  return operation;
+}
+
+static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
+    const SomeExpr &atom, const MaybeExpr &cond,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
+  // Defined in flang/include/flang/Parser/parse-tree.h
+  //
+  // struct Analysis {
+  //   struct Kind {
+  //     static constexpr int None = 0;
+  //     static constexpr int Read = 1;
+  //     static constexpr int Write = 2;
+  //     static constexpr int Update = Read | Write;
+  //     static constexpr int Action = 3; // Bits containing N, R, W, U
+  //     static constexpr int IfTrue = 4;
+  //     static constexpr int IfFalse = 8;
+  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
+  //   };
+  //   struct Op {
+  //     int what;
+  //     TypedAssignment assign;
+  //   };
+  //   TypedExpr atom, cond;
+  //   Op op0, op1;
+  // };
+
+  parser::OpenMPAtomicConstruct::Analysis an;
+  SetExpr(an.atom, atom);
+  SetExpr(an.cond, cond);
+  an.op0 = std::move(op0);
+  an.op1 = std::move(op1);
+  return an;
+}
+
+/// Check if `expr` satisfies the following conditions for x and v:
+///
+/// [6.0:189:10-12]
+/// - x and v (as applicable) are either scalar variables or
+///   function references with scalar data pointer result of non-character
+///   intrinsic type or variables that are non-polymorphic scalar pointers
+///   and any length type parameter must be constant.
+void OmpStructureChecker::CheckAtomicType(
+    SymbolRef sym, parser::CharBlock source, std::string_view name) {
+  const DeclTypeSpec *typeSpec{sym->GetType()};
+  if (!typeSpec) {
+    return;
+  }
+
+  if (!IsPointer(sym)) {
+    using Category = DeclTypeSpec::Category;
+    Category cat{typeSpec->category()};
+    if (cat == Category::Character) {
+      context_.Say(source,
+          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
+    } else if (cat != Category::Numeric && cat != Category::Logical) {
+      context_.Say(source,
+          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
+    }
+    return;
+  }
+
+  // Variable is a pointer.
+  if (typeSpec->IsPolymorphic()) {
+    context_.Say(source,
+        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
+        name);
+    return;
+  }
+
+  // Go over all length parameters, if any, and check if they are
+  // explicit.
+  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
+    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
+          // "entry" is a map entry
+          return entry.second.isLen() && !entry.second.isExplicit();
+        })) {
+      context_.Say(source,
+          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
+          name);
+    }
+  }
+}
+
+void OmpStructureChecker::CheckAtomicVariable(
+    const SomeExpr &atom, parser::CharBlock source) {
+  if (atom.Rank() != 0) {
+    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
+        atom.AsFortran());
+  }
+
+  std::vector<SomeExpr> dsgs{GetAllDesignators(atom)};
+  assert(dsgs.size() == 1 && "Should have a single top-level designator");
+  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
+
+  CheckAtomicType(syms.back(), source, atom.AsFortran());
+
+  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
+    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
+        atom.AsFortran());
+  }
+}
+
+void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
+    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
+    parser::CharBlock source) {
+  if (auto *expr{HasStorageOverlap(base, exprs)}) {
+    context_.Say(source,
+        "Within atomic operation %s and %s access the same storage"_warn_en_US,
+        base.AsFortran(), expr->AsFortran());
+  }
+}
+
+void OmpStructureChecker::ErrorShouldBeVariable(
+    const MaybeExpr &expr, parser::CharBlock source) {
+  if (expr) {
+    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
+        expr->AsFortran());
+  } else {
+    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
+  }
+}
+
+std::pair<const parser::ExecutionPartConstruct *,
+    const parser::ExecutionPartConstruct *>
+OmpStructureChecker::CheckUpdateCapture(
+    const parser::ExecutionPartConstruct *ec1,
+    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
+  // Decide which statement is the atomic update and which is the capture.
+  //
+  // The two allowed cases are:
+  //   x = ...      atomic-var = ...
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  // or
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  //   x = ...      atomic-var = ...
+  //
+  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
+  // (which makes more sense, as it captures the original value of the atomic
+  // variable).
+  //
+  // If the two statements don't fit these criteria, return a pair of default-
+  // constructed values.
+  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
+      const parser::ExecutionPartConstruct *>;
+
+  SourcedActionStmt act1{GetActionStmt(ec1)};
+  SourcedActionStmt act2{GetActionStmt(ec2)};
+  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
+  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
+  if (!maybeAssign1 || !maybeAssign2) {
+    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
+      context_.Say(source,
+          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
+    }
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  auto as1{*maybeAssign1}, as2{*maybeAssign2};
+
+  auto isUpdateCapture{
+      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
+        return IsSameOrConvertOf(c.rhs, u.lhs);
+      }};
+
+  // Do some checks that narrow down the possible choices for the update
+  // and the capture statements. This will help to emit better diagnostics.
+  // 1. An assignment could be an update (cbu) if the left-hand side is a
+  //    subexpression of the right-hand side.
+  // 2. An assignment could be a capture (cbc) if the right-hand side is
+  //    a variable (or a function ref), with potential type conversions.
+  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
+  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
+  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
+  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
+
+  // We want to diagnose cases where both assignments cannot be an update,
+  // or both cannot be a capture, as well as cases where either assignment
+  // cannot be any of these two.
+  //
+  // If we organize these boolean values into a matrix
+  //   |cbu1 cbu2|
+  //   |cbc1 cbc2|
+  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
+  // row or column, including the case where everything is zero. All these
+  // cases correspond to the determinant of the matrix being 0, which suggests
+  // that checking the det may be a convenient diagnostic check. There is only
+  // one additional case where the det is 0, which is when the matrix is all 1
+  // ("true"). The "all true" case represents the situation where both
+  // assignments could be an update as well as a capture. On the other hand,
+  // whenever det != 0, the roles of the update and the capture can be
+  // unambiguously assigned to as1 and as2 [1].
+  //
+  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
+  // det = 0, leaving 6 cases where det != 0:
+  //   0 1   0 1   1 0   1 0   1 1   1 1
+  //   1 0   1 1   0 1   1 1   0 1   1 0
+  // In each case the classification is unambiguous.
+
+  //     |cbu1 cbu2|
+  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
+  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
+
+  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
+                                  const std::string &expr) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
+        expr);
+  }};
+
+  auto errorNeitherWorks{[&]() {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
+  }};
+
+  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
+    // If det != 0, then the checks unambiguously suggest a specific
+    // categorization.
+    // If det == 0, then this function should be called only if the
+    // checks haven't ruled out any possibility, i.e. when both assigments
+    // could still be either updates or captures.
+    if (det > 0) {
+      // as1 is update, as2 is capture
+      if (isUpdateCapture(as1, as2)) {
+        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
+      } else {
+        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else if (det < 0) {
+      // as2 is update, as1 is capture
+      if (isUpdateCapture(as2, as1)) {
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      } else {
+        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else {
+      bool updateFirst{isUpdateCapture(as1, as2)};
+      bool captureFirst{isUpdateCapture(as2, as1)};
+      if (updateFirst && captureFirst) {
+        // If both assignment could be the update and both could be the
+        // capture, emit a warning about the ambiguity.
+        context_.Say(act1.source,
+            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      }
+      if (updateFirst != captureFirst) {
+        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
+        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
+        return std::make_pair(upd, cap);
+      }
+      assert(!updateFirst && !captureFirst);
+      errorNeitherWorks();
+      return std::make_pair(nullptr, nullptr);
+    }
+  }};
+
+  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
+    return makeSelectionFromDet(det);
+  }
+  assert(det == 0 && "Prior checks should have covered det != 0");
+
+  // If neither of the statements is an RMW update, it could still be a
+  // "write" update. Pretty much any assignment can be a write update, so
+  // recompute det with cbu1 = cbu2 = true.
+  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
+    return makeSelectionFromDet(writeDet);
+  }
+
+  // It's only errors from here on.
+
+  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
+    errorNeitherWorks();
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // The remaining cases are that
+  // - no candidate for update, or for capture,
+  // - one of the assigments cannot be anything.
+
+  if (!cbu1 && !cbu2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  } else if (!cbc1 && !cbc2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
+    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
+    context_.Say(src,
+        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // All cases should have been covered.
+  llvm_unreachable("Unchecked condition");
+}
+
+void OmpStructureChecker::CheckAtomicCaptureAssignment(
+    const evaluate::Assignment &capture, const SomeExpr &atom,
+    parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &cap{capture.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, rsrc);
+    // This part should have been checked prior to calling this function.
+    assert(*GetConvertInput(capture.rhs) == atom &&
+        "This cannot be a capture assignment");
+    CheckStorageOverlap(atom, {cap}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicReadAssignment(
+    const evaluate::Assignment &read, parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+
+  if (auto maybe{GetConvertInput(read.rhs)}) {
+    const SomeExpr &atom{*maybe};
+
+    if (!IsVarOrFunctionRef(atom)) {
+      ErrorShouldBeVariable(atom, rsrc);
+    } else {
+      CheckAtomicVariable(atom, rsrc);
+      CheckStorageOverlap(atom, {read.lhs}, source);
+    }
+  } else {
+    ErrorShouldBeVariable(read.rhs, rsrc);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWriteAssignment(
+    const evaluate::Assignment &write, parser::CharBlock source) {
+  // [6.0:190:13-15]
+  // A write structured block is write-statement, a write statement that has
+  // one of the following forms:
+  //   x = expr
+  //   x => expr
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{write.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, lsrc);
+    CheckStorageOverlap(atom, {write.rhs}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateAssignment(
+    const evaluate::Assignment &update, parser::CharBlock source) {
+  // [6.0:191:1-7]
+  // An update structured block is update-statement, an update statement
+  // that has one of the following forms:
+  //   x = x operator expr
+  //   x = expr operator x
+  //   x = intrinsic-procedure-name (x)
+  //   x = intrinsic-procedure-name (x, expr-list)
+  //   x = intrinsic-procedure-name (expr-list, x)
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{update.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, lsrc);
+
+  std::pair<operation::Operator, std::vector<SomeExpr>> top{
+      operation::Operator::Unknown, {}};
+  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
+    top = GetTopLevelOperation(*maybeInput);
+  }
+  switch (top.first) {
+  case operation::Operator::Add:
+  case operation::Operator::Sub:
+  case operation::Operator::Mul:
+  case operation::Operator::Div:
+  case operation::Operator::And:
+  case operation::Operator::Or:
+  case operation::Operator::Eqv:
+  case operation::Operator::Neqv:
+  case operation::Operator::Min:
+  case operation::Operator::Max:
+  case operation::Operator::Identity:
+    break;
+  case operation::Operator::Call:
+    context_.Say(source,
+        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Convert:
+    context_.Say(source,
+        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Intrinsic:
+    context_.Say(source,
+        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Constant:
+  case operation::Operator::Unknown:
+    context_.Say(
+        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(source,
+        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
+        operation::ToString(top.first));
+    return;
+  }
+  // Check how many times `atom` occurs as an argument, if it's a subexpression
+  // of an argument, and collect the non-atom arguments.
+  std::vector<SomeExpr> nonAtom;
+  MaybeExpr subExpr;
+  auto atomCount{[&]() {
+    int count{0};
+    for (const SomeExpr &arg : top.second) {
+      if (IsSameOrConvertOf(arg, atom)) {
+        ++count;
+      } else {
+        if (!subExpr && IsSubexpressionOf(atom, arg)) {
+          subExpr = arg;
+        }
+        nonAtom.push_back(arg);
+      }
+    }
+    return count;
+  }()};
+
+  bool hasError{false};
+  if (subExpr) {
+    context_.Say(rsrc,
+        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
+        atom.AsFortran(), subExpr->AsFortran());
+    hasError = true;
+  }
+  if (top.first == operation::Operator::Identity) {
+    // This is "x = y".
+    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
+          atom.AsFortran());
+      hasError = true;
+    }
+  } else {
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    } else if (atomCount > 1) {
+      context_.Say(rsrc,
+          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    }
+  }
+
+  if (!hasError) {
+    CheckStorageOverlap(atom, nonAtom, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
+    const SomeExpr &cond, parser::CharBlock condSource,
+    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
+  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
+  const SomeExpr &atom{assign.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, arsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, alsrc);
+
+  auto top{GetTopLevelOperation(cond)};
+  // Missing arguments to operations would have been diagnosed by now.
+
+  switch (top.first) {
+  case operation::Operator::Associated:
+    if (atom != top.second.front()) {
+      context_.Say(assignSource,
+          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
+    }
+    break;
+  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
+  case operation::Operator::Eq:
+  case operation::Operator::Eqv:
+  // x ordop expr | expr ordop x
+  case operation::Operator::Lt:
+  case operation::Operator::Gt: {
+    const SomeExpr &arg0{top.second[0]};
+    const SomeExpr &arg1{top.second[1]};
+    if (IsSameOrConvertOf(arg0, atom)) {
+      CheckStorageOverlap(atom, {arg1}, condSource);
+    } else if (IsSameOrConvertOf(arg1, atom)) {
+      CheckStorageOverlap(atom, {arg0}, condSource);
+    } else {
+      assert(top.first != operation::Operator::Identity &&
+          "Handle this separately");
+      context_.Say(assignSource,
+          "An argument of the %s operator should be the target of the assignment"_err_en_US,
+          operation::ToString(top.first));
+    }
+    break;
+  }
+  case operation::Operator::Identity:
+  case operation::Operator::True:
+  case operation::Operator::False:
+    break;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(condSource,
+        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
+        operation::ToString(top.first));
+    break;
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
+    const AnalyzedCondStmt &update, parser::CharBlock source) {
+  // The condition/statements must be:
+  // - cond: x equalop e      ift: x =  d     iff: -
+  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
+  // - cond: associated(x)    ift: x => expr  iff: -
+  // - cond: associated(x, e) ift: x => expr  iff: -
+
+  // The if-true statement must be present, and must be an assignment.
+  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
+  if (!maybeAssign) {
+    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
+    } else {
+      context_.Say(
+          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
+    }
+    return;
+  }
+  const evaluate::Assignment assign{*maybeAssign};
+  const SomeExpr &atom{assign.lhs};
+
+  CheckAtomicConditionalUpdateAssignment(
+      update.cond, update.source, assign, update.ift.source);
+
+  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
+
+  if (update.iff) {
+    context_.Say(update.iff.source,
+        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateOnly(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeUpdate->lhs};
+      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdate(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // Allowable forms are (single-statement):
+  // - if ...
+  // - x = (... ? ... : x)
+  // and two-statement:
+  // - r = cond ; if (r) ...
+
+  const parser::ExecutionPartConstruct *ust{nullptr}; // update
+  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
+
+  if (body.size() == 1) {
+    ust = &body.front();
+  } else if (body.size() == 2) {
+    cst = &body.front();
+    ust = &body.back();
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
+    return;
+  }
+
+  // Flang doesn't support conditional-expr yet, so all update statements
+  // are if-statements.
+
+  // IfStmt:        if (...) ...
+  // IfConstruct:   if (...) then ... endif
+  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
+  if (!maybeUpdate) {
+    context_.Say(source,
+        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
+    return;
+  }
+
+  AnalyzedCondStmt &update{*maybeUpdate};
+
+  if (SourcedActionStmt action{GetActionStmt(cst)}) {
+    // The "condition" statement must be `r = cond`.
+    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
+      if (maybeCond->lhs != update.cond) {
+        context_.Say(update.source,
+            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
+            maybeCond->lhs.AsFortran());
+      } else {
+        // If it's "r = ...; if (r) ..." then put the original condition
+        // in `update`.
+        update.cond = maybeCond->rhs;
+      }
+    } else {
+      context_.Say(action.source,
+          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
+    }
+  }
+
+  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
+
+  CheckAtomicConditionalUpdateStmt(update, source);
+  if (IsCheckForAssociated(update.cond)) {
+    if (!IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
+    }
+  } else {
+    if (IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
+    }
+  }
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
+      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
+      MakeAtomicAnalysisOp(Analysis::None));
+}
+
+void OmpStructureChecker::CheckAtomicUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() != 2) {
+    context_.Say(source,
+        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
+    return;
+  }
+
+  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
+  if (!uec || !cec) {
+    // Diagnostics already emitted.
+    return;
+  }
+  SourcedActionStmt uact{GetActionStmt(uec)};
+  SourcedActionStmt cact{GetActionStmt(cec)};
+  // The "dereferences" of std::optional are guaranteed to be valid after
+  // CheckUpdateCapture.
+  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
+  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
+
+  const SomeExpr &atom{update.lhs};
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  int action;
+
+  if (IsMaybeAtomicWrite(update)) {
+    action = Analysis::Write;
+    CheckAtomicWriteAssignment(update, uact.source);
+  } else {
+    action = Analysis::Update;
+    CheckAtomicUpdateAssignment(update, uact.source);
+  }
+  CheckAtomicCaptureAssignment(capture, atom, cact.source);
+
+  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
+    context_.Say(cact.source,
+        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+    return;
+  }
+
+  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(action, update),
+        MakeAtomicAnalysisOp(Analysis::Read, capture));
+  } else {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(Analysis::Read, capture),
+        MakeAtomicAnalysisOp(action, update));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // There are two different variants of this:
+  // (1) conditional-update and capture separately:
+  //     This form only allows single-statement updates, i.e. the update
+  //     form "r = cond; if (r) ..." is not allowed.
+  // (2) conditional-update combined with capture in a single statement:
+  //     This form does allow the condition to be calculated separately,
+  //     i.e. "r = cond; if (r) ...".
+  // Regardless of what form it is, the actual update assignment is a
+  // proper write, i.e. "x = d", where d does not depend on x.
+
+  AnalyzedCondStmt update;
+  SourcedActionStmt capture;
+  bool captureAlways{true}, captureFirst{true};
+
+  auto extractCapture{[&]() {
+    capture = update.iff;
+    captureAlways = false;
+    update.iff = SourcedActionStmt{};
+  }};
+
+  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
+    // The non-update statement is either "r = cond" or the capture.
+    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
+      if (update.cond == maybeAssign->lhs) {
+        // If this is "r = cond; if (r) ...", then update the condition.
+        update.cond = maybeAssign->rhs;
+        update.source = action.source;
+        // In this form, the update and the capture are combined into
+        // an IF-THEN-ELSE statement.
+        extractCapture();
+      } else {
+        // Assume this is the capture-statement.
+        capture = action;
+      }
+    }
+  }};
+
+  if (body.size() == 2) {
+    // This could be
+    // - capture; conditional-update (in any order), or
+    // - r = cond; if (r) capture-update
+    const parser::ExecutionPartConstruct *st1{&body.front()};
+    const parser::ExecutionPartConstruct *st2{&body.back()};
+    // In either case, the conditional statement can be analyzed by
+    // AnalyzeConditionalStmt, whereas the other statement cannot.
+    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
+      update = *maybeUpdate1;
+      classifyNonUpdate(GetActionStmt(st2));
+      captureFirst = false;
+    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
+      update = *maybeUpdate2;
+      classifyNonUpdate(GetActionStmt(st1));
+    } else {
+      // None of the statements are conditional, this rules out the
+      // "r = cond; if (r) ..." and the "capture + conditional-update"
+      // variants. This could still be capture + write (which is classified
+      // as conditional-update-capture in the spec).
+      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
+      if (!uec || !cec) {
+        // Diagnostics already emitted.
+        return;
+      }
+      SourcedActionStmt uact{GetActionStmt(uec)};
+      SourcedActionStmt cact{GetActionStmt(cec)};
+      update.ift = uact;
+      capture = cact;
+      if (uec == st1) {
+        captureFirst = false;
+      }
+    }
+  } else if (body.size() == 1) {
+    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
+      update = *maybeUpdate;
+      // This is the form with update and capture combined into an IF-THEN-ELSE
+      // statement. The capture-statement is always the ELSE branch.
+      extractCapture();
+    } else {
+      goto invalid;
+    }
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
+    return;
+  invalid:
+    context_.Say(source,
+        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
+    return;
+  }
+
+  // The update must have a form `x = d` or `x => d`.
+  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
+    const SomeExpr &atom{maybeWrite->lhs};
+    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
+    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
+      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
+
+      if (IsPointerAssignment(*maybeWrite) !=
+          IsPointerAssignment(*maybeCapture)) {
+        context_.Say(capture.source,
+            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+        return;
+      }
+    } else {
+      if (!IsAssignment(capture.stmt)) {
+        context_.Say(capture.source,
+            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
+      }
+      return;
+    }
+  } else {
+    if (!IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
+    }
+    return;
+  }
+
+  // update.iff should be empty here, the capture statement should be
+  // stored in "capture".
+
+  // Fill out the analysis in the AST node.
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  bool condUnused{std::visit(
+      [](auto &&s) {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
+          return true;
+        } else {
+          return false;
+        }
+      },
+      update.cond.u)};
+
+  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
+  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
+
+  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
+  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
+
+  if (captureFirst) {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
+  } else {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicRead(
+    const parser::OpenMPAtomicConstruct &x) {
+  // [6.0:190:5-7]
+  // A read structured block is read-statement, a read statement that has one
+  // of the following forms:
+  //   v = x
+  //   v => x
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Read cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
+      CheckAtomicReadAssignment(*maybeRead, action.source);
+
+      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
+        const SomeExpr &atom{*maybe};
+        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
+            MakeAtomicAnalysisOp(Analysis::None));
+      }
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC READ operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWrite(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Write cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeWrite->lhs};
+      CheckAtomicWriteAssignment(*maybeWrite, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC WRITE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdate(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &block{std::get<parser::Block>(x.t)};
+
+  bool isConditional{x.IsCompare()};
+  bool isCapture{x.IsCapture()};
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (isConditional && isCapture) {
+    CheckAtomicConditionalUpdateCapture(x, body, x.source);
+  } else if (isConditional) {
+    CheckAtomicConditionalUpdate(x, body, x.source);
+  } else if (isCapture) {
+    CheckAtomicUpdateCapture(x, body, x.source);
+  } else { // update-only
+    CheckAtomicUpdateOnly(x, body, x.source);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
+  if (visitedAtomicSource_.empty())
+    visitedAtomicSource_ = x.source;
+
+  // All of the following groups have the "exclusive" property, i.e. at
+  // most one clause from each group is allowed.
+  // The exclusivity-checking code should eventually be unified for all
+  // clauses, with clause groups defined in OMP.td.
+  std::array atomic{llvm::omp::Clause::OMPC_read,
+      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
+  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
+      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
+      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
+
+  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
+                          std::string_view name,
+                          const parser::OmpClauseList &clauses) {
+    const parser::OmpClause *present{nullptr};
+    for (const parser::OmpClause &clause : clauses.v) {
+      llvm::omp::Clause id{clause.Id()};
+      if (!llvm::is_contained(group, id)) {
+        continue;
+      }
+      if (present == nullptr) {
+        present = &clause;
+        continue;
+      } else if (id == present->Id()) {
+        // Ignore repetitions of the same clause, those will be diagnosed
+        // separately.
+        continue;
+      }
+      parser::MessageFormattedText txt(
+          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
+          name.data());
+      parser::Message message(clause.source, txt);
+      message.Attach(present->source,
+          "Previous clause from this group provided here"_en_US);
+      context_.Say(std::move(message));
+      return;
+    }
+  }};
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
+  llvm::omp::Clause kind{x.GetKind()};
+
+  checkExclusive(atomic, "atomic", dirSpec.Clauses());
+  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
+
+  switch (kind) {
+  case llvm::omp::Clause::OMPC_read:
+    CheckAtomicRead(x);
+    break;
+  case llvm::omp::Clause::OMPC_write:
+    CheckAtomicWrite(x);
+    break;
+  case llvm::omp::Clause::OMPC_update:
+    CheckAtomicUpdate(x);
+    break;
+  default:
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
new file mode 100644
index 0000000000000..f2aad9efdd8de
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -0,0 +1,655 @@
+//===-- lib/Semantics/check-omp-loop.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <variant>
+
+namespace {
+using namespace Fortran;
+
+class AssociatedLoopChecker {
+public:
+  AssociatedLoopChecker(
+      semantics::SemanticsContext &context, std::int64_t level)
+      : context_{context}, level_{level} {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::DoConstruct &dc) {
+    level_--;
+    const auto &doStmt{
+        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
+    const auto &constructName{
+        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
+    if (constructName) {
+      constructNamesAndLevels_.emplace(
+          constructName.value().ToString(), level_);
+    }
+    if (level_ >= 0) {
+      if (dc.IsDoWhile()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
+      }
+      if (!dc.GetLoopControl()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
+      }
+    }
+    return true;
+  }
+
+  void Post(const parser::DoConstruct &dc) { level_++; }
+
+  bool Pre(const parser::CycleStmt &cyclestmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (cyclestmt.v) {
+      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second > 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ > 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::ExitStmt &exitStmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (exitStmt.v) {
+      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ >= 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "EXIT statement terminates associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
+    source_ = &actionstmt.source;
+    return true;
+  }
+
+private:
+  semantics::SemanticsContext &context_;
+  const parser::CharBlock *source_;
+  std::int64_t level_;
+  std::map<std::string, std::int64_t> constructNamesAndLevels_;
+};
+} // namespace
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::HasInvalidDistributeNesting(
+    const parser::OpenMPLoopConstruct &x) {
+  bool violation{false};
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
+    // `distribute` region has to be nested
+    if (!CurrentDirectiveIsNested()) {
+      violation = true;
+    } else {
+      // `distribute` region has to be strictly nested inside `teams`
+      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+        violation = true;
+      }
+    }
+  }
+  if (violation) {
+    context_.Say(beginDir.source,
+        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
+        "region."_err_en_US);
+  }
+}
+void OmpStructureChecker::HasInvalidLoopBinding(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
+    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+    for (const auto &clause : clauseList.v) {
+      if (const auto *bindClause{
+              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
+        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
+          context_.Say(beginDir.source, msg);
+        }
+      }
+    }
+  };
+
+  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
+      CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
+        "strictly nested inside a `TEAMS` region."_err_en_US);
+  }
+
+  if (OmpDirectiveSet{
+          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
+          .test(beginDir.v)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
+        "combined with a `TEAMS` construct."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
+  // Check the following:
+  //  The only OpenMP constructs that can be encountered during execution of
+  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
+  // construct and the `ordered` construct with the `simd` clause.
+
+  // Check if the parent context has the SIMD clause
+  // Please note that we use GetContext() instead of GetContextParent()
+  // because PushContextAndClauseSets() has not been called on the
+  // current context yet.
+  // TODO: Check for declare simd regions.
+  bool eligibleSIMD{false};
+  common::visit(
+      common::visitors{
+          // Allow `!$OMP ORDERED SIMD`
+          [&](const parser::OpenMPBlockConstruct &c) {
+            const auto &beginBlockDir{
+                std::get<parser::OmpBeginBlockDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
+            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
+              const auto &clauses{
+                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
+              for (const auto &clause : clauses.v) {
+                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
+                  eligibleSIMD = true;
+                  break;
+                }
+              }
+            }
+          },
+          [&](const parser::OpenMPStandaloneConstruct &c) {
+            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
+                    &c.u)}) {
+              llvm::omp::Directive dirId{ssc->v.DirId()};
+              if (dirId == llvm::omp::Directive::OMPD_ordered) {
+                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
+                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
+                    eligibleSIMD = true;
+                    break;
+                  }
+                }
+              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
+                eligibleSIMD = true;
+              }
+            }
+          },
+          // Allowing SIMD and loop construct
+          [&](const parser::OpenMPLoopConstruct &c) {
+            const auto &beginLoopDir{
+                std::get<parser::OmpBeginLoopDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
+              eligibleSIMD = true;
+            }
+          },
+          [&](const parser::OpenMPAtomicConstruct &c) {
+            // Allow `!$OMP ATOMIC`
+            eligibleSIMD = true;
+          },
+          [&](const auto &c) {},
+      },
+      c.u);
+  if (!eligibleSIMD) {
+    context_.Say(parser::FindSourceLocation(c),
+        "The only OpenMP constructs that can be encountered during execution "
+        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
+        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
+        "construct with the `SIMD` clause."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
+  loopStack_.push_back(&x);
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  PushContextAndClauseSets(beginDir.source, beginDir.v);
+
+  // check matching, End directive is optional
+  if (const auto &endLoopDir{
+          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
+    const auto &endDir{
+        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
+
+    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
+
+    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
+  }
+
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    EnterDirectiveNest(SIMDNest);
+  }
+
+  // Combined target loop constructs are target device constructs. Keep track of
+  // whether any such construct has been visited to later check that REQUIRES
+  // directives for target-related options don't appear after them.
+  if (llvm::omp::allTargetSet.test(beginDir.v)) {
+    deviceConstructFound_ = true;
+  }
+
+  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
+    // 2.7.1 do-clause -> private-clause |
+    //                    firstprivate-clause |
+    //                    lastprivate-clause |
+    //                    linear-clause |
+    //                    reduction-clause |
+    //                    schedule-clause |
+    //                    collapse-clause |
+    //                    ordered-clause
+
+    // nesting check
+    HasInvalidWorksharingNesting(
+        beginDir.source, llvm::omp::nestedWorkshareErrSet);
+  }
+  SetLoopInfo(x);
+
+  if (const auto &doConstruct{
+          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
+    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+  }
+  CheckLoopItrVariableIsInt(x);
+  CheckAssociatedLoopConstraints(x);
+  HasInvalidDistributeNesting(x);
+  HasInvalidLoopBinding(x);
+  if (CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
+  }
+  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
+      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
+    CheckDistLinear(x);
+  }
+}
+
+const parser::Name OmpStructureChecker::GetLoopIndex(
+    const parser::DoConstruct *x) {
+  using Bounds = parser::LoopControl::Bounds;
+  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
+}
+
+void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
+  if (const auto &loopConstruct{
+          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+    const parser::DoConstruct *loop{&*loopConstruct};
+    if (loop && loop->IsDoNormal()) {
+      const parser::Name &itrVal{GetLoopIndex(loop)};
+      SetLoopIv(itrVal.symbol);
+    }
+  }
+}
+
+void OmpStructureChecker::CheckLoopItrVariableIsInt(
+    const parser::OpenMPLoopConstruct &x) {
+  if (const auto &loopConstruct{
+          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+
+    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+      if (loop->IsDoNormal()) {
+        const parser::Name &itrVal{GetLoopIndex(loop)};
+        if (itrVal.symbol) {
+          const auto *type{itrVal.symbol->GetType()};
+          if (!type->IsNumeric(TypeCategory::Integer)) {
+            context_.Say(itrVal.source,
+                "The DO loop iteration"
+                " variable must be of the type integer."_err_en_US,
+                itrVal.ToString());
+          }
+        }
+      }
+      // Get the next DoConstruct if block is not empty.
+      const auto &block{std::get<parser::Block>(loop->t)};
+      const auto it{block.begin()};
+      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                               : nullptr;
+    }
+  }
+}
+
+std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+  std::int64_t orderedCollapseLevel{1};
+  std::int64_t orderedLevel{1};
+  std::int64_t collapseLevel{1};
+
+  for (const auto &clause : clauseList.v) {
+    if (const auto *collapseClause{
+            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
+      if (const auto v{GetIntValue(collapseClause->v)}) {
+        collapseLevel = *v;
+      }
+    }
+    if (const auto *orderedClause{
+            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
+      if (const auto v{GetIntValue(orderedClause->v)}) {
+        orderedLevel = *v;
+      }
+    }
+  }
+  if (orderedLevel >= collapseLevel) {
+    orderedCollapseLevel = orderedLevel;
+  } else {
+    orderedCollapseLevel = collapseLevel;
+  }
+  return orderedCollapseLevel;
+}
+
+void OmpStructureChecker::CheckAssociatedLoopConstraints(
+    const parser::OpenMPLoopConstruct &x) {
+  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
+  AssociatedLoopChecker checker{context_, ordCollapseLevel};
+  parser::Walk(x, checker);
+}
+
+void OmpStructureChecker::CheckDistLinear(
+    const parser::OpenMPLoopConstruct &x) {
+
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  SymbolSourceMap indexVars;
+
+  // Collect symbols of all the variables from linear clauses
+  for (auto &clause : clauses.v) {
+    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
+      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
+      GetSymbolsInObjectList(objects, indexVars);
+    }
+  }
+
+  if (!indexVars.empty()) {
+    // Get collapse level, if given, to find which loops are "associated."
+    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
+    // Include the top loop if no collapse is specified
+    if (collapseVal == 0) {
+      collapseVal = 1;
+    }
+
+    // Match the loop index variables with the collected symbols from linear
+    // clauses.
+    if (const auto &loopConstruct{
+            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+        if (loop->IsDoNormal()) {
+          const parser::Name &itrVal{GetLoopIndex(loop)};
+          if (itrVal.symbol) {
+            // Remove the symbol from the collected set
+            indexVars.erase(&itrVal.symbol->GetUltimate());
+          }
+          collapseVal--;
+          if (collapseVal == 0) {
+            break;
+          }
+        }
+        // Get the next DoConstruct if block is not empty.
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                 : nullptr;
+      }
+    }
+
+    // Show error for the remaining variables
+    for (auto &[symbol, source] : indexVars) {
+      const Symbol &root{GetAssociationRoot(*symbol)};
+      context_.Say(source,
+          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
+          root.name());
+    }
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  // A few semantic checks for InScan reduction are performed below as SCAN
+  // constructs inside LOOP may add the relevant information. Scan reduction is
+  // supported only in loop constructs, so same checks are not applicable to
+  // other directives.
+  using ReductionModifier = parser::OmpReductionModifier;
+  for (const auto &clause : clauseList.v) {
+    if (const auto *reductionClause{
+            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
+      auto &modifiers{OmpGetModifiers(reductionClause->v)};
+      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
+      if (maybeModifier &&
+          maybeModifier->v == ReductionModifier::Value::Inscan) {
+        const auto &objectList{
+            std::get<parser::OmpObjectList>(reductionClause->v.t)};
+        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
+          if (auto &symbol = name->symbol) {
+            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
+                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
+              context_.Say(name->source,
+                  "List item %s must appear in EXCLUSIVE or "
+                  "INCLUSIVE clause of an "
+                  "enclosed SCAN directive"_err_en_US,
+                  name->ToString());
+            }
+          }
+        };
+        for (const auto &ompObj : objectList.v) {
+          common::visit(
+              common::visitors{
+                  [&](const parser::Designator &designator) {
+                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
+                            designator)}) {
+                      checkReductionSymbolInScan(name);
+                    }
+                  },
+                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
+              },
+              ompObj.u);
+        }
+      }
+    }
+  }
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    ExitDirectiveNest(SIMDNest);
+  }
+  dirContext_.pop_back();
+
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top{loopStack_.back()};
+  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
+  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+}
+
+void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
+  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  // 2.7.1 end-do -> END DO [nowait-clause]
+  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
+  case llvm::omp::Directive::OMPD_do:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
+    break;
+  case llvm::omp::Directive::OMPD_do_simd:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
+    break;
+  default:
+    // no clauses are allowed
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
+    dirContext_.pop_back();
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
+  unsigned version{context_.langOptions().OpenMPVersion};
+  llvm::omp::Directive dir{GetContext().directive};
+  parser::CharBlock clauseSource{GetContext().clauseSource};
+  const parser::OmpLinearModifier *linearMod{nullptr};
+
+  SymbolSourceMap symbols;
+  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  CheckCrayPointee(objects, "LINEAR", false);
+  GetSymbolsInObjectList(objects, symbols);
+
+  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
+    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      context_.Say(source,
+          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
+          symbol->name(), desc.name.str());
+    }
+  }};
+
+  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(x.v)};
+    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
+    if (linearMod) {
+      // 2.7 Loop Construct Restriction
+      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
+        context_.Say(clauseSource,
+            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
+            ContextDirectiveAsFortran());
+        return;
+      }
+
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      for (auto &[symbol, source] : symbols) {
+        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
+          CheckIntegerNoRef(symbol, source);
+        } else {
+          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
+              !IsPolymorphic(*symbol)) {
+            context_.Say(source,
+                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
+                symbol->name(), desc.name.str());
+          }
+        }
+        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
+            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
+          if (!IsDummy(*symbol) || IsValue(*symbol)) {
+            context_.Say(source,
+                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
+                desc.name.str(), symbol->name());
+          }
+        }
+      } // for (symbol, source)
+
+      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
+        context_.Say(OmpGetModifierSource(modifiers, linearMod),
+            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
+            ThisVersion(version));
+      }
+    }
+  }
+
+  // OpenMP 5.2: Ordered clause restriction
+  if (const auto *clause{
+          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
+    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
+    if (orderedClause.v) {
+      return;
+    }
+  }
+
+  // OpenMP 5.2: Linear clause Restrictions
+  for (auto &[symbol, source] : symbols) {
+    if (!linearMod) {
+      // Already checked this with the modifier present.
+      CheckIntegerNoRef(symbol, source);
+    }
+    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
+      context_.Say(source,
+          "The list item `%s` must be a dummy argument"_err_en_US,
+          symbol->name());
+    }
+    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
+      context_.Say(source,
+          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
+          symbol->name());
+    }
+    if (FindCommonBlockContaining(*symbol)) {
+      context_.Say(source,
+          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
+          symbol->name());
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
+  Base::Enter(x);
+  loopStack_.push_back(&x);
+}
+
+void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top = loopStack_.back();
+  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
+  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+  Base::Leave(x);
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-metadirective.cpp b/flang/lib/Semantics/check-omp-metadirective.cpp
new file mode 100644
index 0000000000000..bf27797a7c44a
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-metadirective.cpp
@@ -0,0 +1,544 @@
+//===-- lib/Semantics/check-omp-metadirective.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <variant>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
+  OmpVerifyModifiers(
+      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
+}
+
+void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
+  EnterDirectiveNest(ContextSelectorNest);
+
+  using SetName = parser::OmpTraitSetSelectorName;
+  std::map<SetName::Value, const SetName *> visited;
+
+  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
+    auto &name{std::get<SetName>(traitSet.t)};
+    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait set name %s in a context specifier"_err_en_US,
+          showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait set %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+    CheckTraitSetSelector(traitSet);
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
+  ExitDirectiveNest(ContextSelectorNest);
+}
+
+const std::list<parser::OmpTraitProperty> &
+OmpStructureChecker::GetTraitPropertyList(
+    const parser::OmpTraitSelector &trait) {
+  static const std::list<parser::OmpTraitProperty> empty{};
+  auto &[_, maybeProps]{trait.t};
+  if (maybeProps) {
+    using PropertyList = std::list<parser::OmpTraitProperty>;
+    return std::get<PropertyList>(maybeProps->t);
+  } else {
+    return empty;
+  }
+}
+
+std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
+    const parser::OmpTraitProperty &property) {
+  using MaybeClause = std::optional<llvm::omp::Clause>;
+
+  // The parser for OmpClause will only succeed if the clause was
+  // given with all required arguments.
+  // If this is a string or complex extension with a clause name,
+  // treat it as a clause and let the trait checker deal with it.
+
+  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
+    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
+    if (id != llvm::omp::Clause::OMPC_unknown) {
+      return id;
+    } else {
+      return std::nullopt;
+    }
+  }};
+
+  return common::visit( //
+      common::visitors{
+          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
+            return getClauseFromString(x.v);
+          },
+          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
+            return x.value().Id();
+          },
+          [&](const parser::ScalarExpr &x) -> MaybeClause {
+            return std::nullopt;
+          },
+          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
+            using ExtProperty = parser::OmpTraitPropertyExtension;
+            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
+              return getClauseFromString(name->v);
+            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
+              return getClauseFromString(
+                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
+            }
+            return std::nullopt;
+          },
+      },
+      property.u);
+}
+
+void OmpStructureChecker::CheckTraitSelectorList(
+    const std::list<parser::OmpTraitSelector> &traits) {
+  // [6.0:322:20]
+  // Each trait-selector-name may only be specified once in a trait selector
+  // set.
+
+  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
+  using TraitName = parser::OmpTraitSelectorName;
+  using BareName = decltype(TraitName::u);
+  std::map<BareName, const TraitName *> visited;
+
+  for (const parser::OmpTraitSelector &trait : traits) {
+    auto &name{std::get<TraitName>(trait.t)};
+
+    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait name %s in a trait set"_err_en_US, showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSetSelector(
+    const parser::OmpTraitSetSelector &traitSet) {
+
+  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
+  //
+  // Construct      |     Simd, directive-name |      Yes |       No |    No |
+  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
+  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
+  //                |      Extension, Requires |          |          |       |
+  //                |                   Vendor |          |          |       |
+  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
+  //                |                Kind, Uid |          |          |       |
+  // User           |                Condition |       No |       No |   Yes |
+
+  struct TraitSetConfig {
+    std::set<parser::OmpTraitSelectorName::Value> allowed;
+    bool allowsDirectiveTraits;
+    bool allowsExtensionTraits;
+    bool allowsScore;
+  };
+
+  using SName = parser::OmpTraitSetSelectorName::Value;
+  using TName = parser::OmpTraitSelectorName::Value;
+
+  static const std::map<SName, TraitSetConfig> configs{
+      {SName::Construct, //
+          {{TName::Simd}, true, false, false}},
+      {SName::Device, //
+          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
+      {SName::Implementation, //
+          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
+               TName::Vendor},
+              false, true, true}},
+      {SName::Target_Device, //
+          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
+               TName::Uid},
+              false, true, false}},
+      {SName::User, //
+          {{TName::Condition}, false, false, true}},
+  };
+
+  auto checkTraitSet{[&](const TraitSetConfig &config) {
+    auto &[setName, traits]{traitSet.t};
+    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
+
+    // Check if there are any duplicate traits.
+    CheckTraitSelectorList(traits);
+
+    for (const parser::OmpTraitSelector &trait : traits) {
+      // Don't use structured bindings here, because they cannot be captured
+      // before C++20.
+      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
+      auto &maybeProps =
+          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
+              trait.t);
+
+      // Check allowed traits
+      common::visit( //
+          common::visitors{
+              [&](parser::OmpTraitSelectorName::Value v) {
+                if (!config.allowed.count(v)) {
+                  context_.Say(traitName.source,
+                      "%s is not a valid trait for %s trait set"_err_en_US,
+                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
+                }
+              },
+              [&](llvm::omp::Directive) {
+                if (!config.allowsDirectiveTraits) {
+                  context_.Say(traitName.source,
+                      "Directive name is not a valid trait for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+              [&](const std::string &) {
+                if (!config.allowsExtensionTraits) {
+                  context_.Say(traitName.source,
+                      "Extension traits are not valid for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+          },
+          traitName.u);
+
+      // Check score
+      if (maybeProps) {
+        auto &[maybeScore, _]{maybeProps->t};
+        if (maybeScore) {
+          CheckTraitScore(*maybeScore);
+        }
+      }
+
+      // Check the properties of the individual traits
+      CheckTraitSelector(traitSet, trait);
+    }
+  }};
+
+  checkTraitSet(
+      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
+}
+
+void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
+  // [6.0:322:23]
+  // A score-expression must be a non-negative constant integer expression.
+  if (auto value{GetIntValue(score)}; !value || value < 0) {
+    context_.Say(score.source,
+        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
+  }
+}
+
+bool OmpStructureChecker::VerifyTraitPropertyLists(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  using PropertyList = std::list<parser::OmpTraitProperty>;
+  auto &[traitName, maybeProps]{trait.t};
+
+  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
+                             const std::string &message) {
+    bool foundInvalid{false};
+    for (const parser::OmpTraitProperty &prop : properties) {
+      if (!isValid(prop)) {
+        if (foundInvalid) {
+          context_.Say(
+              prop.source, "More invalid properties are present"_err_en_US);
+          break;
+        }
+        context_.Say(prop.source, "%s"_err_en_US, message);
+        foundInvalid = true;
+      }
+    }
+    return !foundInvalid;
+  }};
+
+  bool invalid{false};
+
+  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
+    // Directive-name traits don't have properties.
+    if (maybeProps) {
+      context_.Say(trait.source,
+          "Directive-name traits cannot have properties"_err_en_US);
+      invalid = true;
+    }
+  }
+  // Ignore properties on extension traits.
+
+  // See `TraitSelectorParser` in openmp-parser.cpp
+  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
+    switch (*v) {
+    // name-list properties
+    case parser::OmpTraitSelectorName::Value::Arch:
+    case parser::OmpTraitSelectorName::Value::Extension:
+    case parser::OmpTraitSelectorName::Value::Isa:
+    case parser::OmpTraitSelectorName::Value::Kind:
+    case parser::OmpTraitSelectorName::Value::Uid:
+    case parser::OmpTraitSelectorName::Value::Vendor:
+      if (maybeProps) {
+        auto isName{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isName, "Trait property should be a name");
+      }
+      break;
+    // clause-list
+    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
+    case parser::OmpTraitSelectorName::Value::Requires:
+    case parser::OmpTraitSelectorName::Value::Simd:
+      if (maybeProps) {
+        auto isClause{[&](const parser::OmpTraitProperty &prop) {
+          return GetClauseFromProperty(prop).has_value();
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isClause, "Trait property should be a clause");
+      }
+      break;
+    // expr-list
+    case parser::OmpTraitSelectorName::Value::Condition:
+    case parser::OmpTraitSelectorName::Value::Device_Num:
+      if (maybeProps) {
+        auto isExpr{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::ScalarExpr>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isExpr, "Trait property should be a scalar expression");
+      }
+      break;
+    } // switch
+  }
+
+  return !invalid;
+}
+
+void OmpStructureChecker::CheckTraitSelector(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  auto &[traitName, maybeProps]{trait.t};
+
+  // Only do the detailed checks if the property lists are valid.
+  if (VerifyTraitPropertyLists(traitSet, trait)) {
+    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
+        std::holds_alternative<std::string>(traitName.u)) {
+      // No properties here: directives don't have properties, and
+      // we don't implement any extension traits now.
+      return;
+    }
+
+    // Specific traits we want to check.
+    // Limitations:
+    // (1) The properties for these traits are defined in "Additional
+    // Definitions for the OpenMP API Specification". It's not clear how
+    // to define them in a portable way, and how to verify their validity,
+    // especially if they get replaced by their integer values (in case
+    // they are defined as enums).
+    // (2) These are entirely implementation-defined, and at the moment
+    // there is no known schema to validate these values.
+    auto v{std::get<TraitName::Value>(traitName.u)};
+    switch (v) {
+    case TraitName::Value::Arch:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Atomic_Default_Mem_Order:
+      CheckTraitADMO(traitSet, trait);
+      break;
+    case TraitName::Value::Condition:
+      CheckTraitCondition(traitSet, trait);
+      break;
+    case TraitName::Value::Device_Num:
+      CheckTraitDeviceNum(traitSet, trait);
+      break;
+    case TraitName::Value::Extension:
+      // Ignore
+      break;
+    case TraitName::Value::Isa:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Kind:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Requires:
+      CheckTraitRequires(traitSet, trait);
+      break;
+    case TraitName::Value::Simd:
+      CheckTraitSimd(traitSet, trait);
+      break;
+    case TraitName::Value::Uid:
+      // Unchecked, TBD(2)
+      break;
+    case TraitName::Value::Vendor:
+      // Unchecked, TBD(1)
+      break;
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitADMO(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single clause property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto clauseId{*GetClauseFromProperty(property)};
+    // Check that the clause belongs to the memory-order clause-set.
+    // Clause sets will hopefully be autogenerated at some point.
+    switch (clauseId) {
+    case llvm::omp::Clause::OMPC_acq_rel:
+    case llvm::omp::Clause::OMPC_acquire:
+    case llvm::omp::Clause::OMPC_relaxed:
+    case llvm::omp::Clause::OMPC_release:
+    case llvm::omp::Clause::OMPC_seq_cst:
+      break;
+    default:
+      context_.Say(property.source,
+          "%s trait requires a clause from the memory-order clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitCondition(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
+
+    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
+    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
+      context_.Say(property.source,
+          "%s trait requires a single LOGICAL expression"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitDeviceNum(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  }
+  // No other checks at the moment.
+}
+
+void OmpStructureChecker::CheckTraitRequires(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_requires, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause from the requirement clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSimd(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_declare_simd, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()),
+          parser::ToUpperCaseLetters(
+              getDirectiveName(llvm::omp::OMPD_declare_simd)));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
+  EnterDirectiveNest(MetadirectiveNest);
+  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
+}
+
+void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
+  ExitDirectiveNest(MetadirectiveNest);
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 3abb5a304b00c..30eff01256c61 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -7,27 +7,56 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
 #include "definable.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
-#include "flang/Evaluate/check-expression.h"
-#include "flang/Evaluate/expression.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
 #include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/scope.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+#include "flang/Support/Fortran-features.h"
+
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <iterator>
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
 #include <variant>
 
 namespace Fortran::semantics {
 
-template <typename T, typename U>
-static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
-  return !(e == f);
-}
+using namespace Fortran::semantics::omp;
 
 // Use when clause falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
@@ -53,66 +82,6 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
     CheckAllowedClause(llvm::omp::Y); \
   }
 
-std::string ThisVersion(unsigned version) {
-  std::string tv{
-      std::to_string(version / 10) + "." + std::to_string(version % 10)};
-  return "OpenMP v" + tv;
-}
-
-std::string TryVersion(unsigned version) {
-  return "try -fopenmp-version=" + std::to_string(version);
-}
-
-static const parser::Designator *GetDesignatorFromObj(
-    const parser::OmpObject &object) {
-  return std::get_if<parser::Designator>(&object.u);
-}
-
-static const parser::DataRef *GetDataRefFromObj(
-    const parser::OmpObject &object) {
-  if (auto *desg{GetDesignatorFromObj(object)}) {
-    return std::get_if<parser::DataRef>(&desg->u);
-  }
-  return nullptr;
-}
-
-static const parser::ArrayElement *GetArrayElementFromObj(
-    const parser::OmpObject &object) {
-  if (auto *dataRef{GetDataRefFromObj(object)}) {
-    using ElementIndirection = common::Indirection<parser::ArrayElement>;
-    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
-      return &ind->value();
-    }
-  }
-  return nullptr;
-}
-
-static bool IsVarOrFunctionRef(const MaybeExpr &expr) {
-  if (expr) {
-    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
-        evaluate::IsVariable(*expr);
-  } else {
-    return false;
-  }
-}
-
-static std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
-  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
-  // ForwardOwningPointer           typedExpr
-  // `- GenericExprWrapper          ^.get()
-  //    `- std::optional<Expr>      ^->v
-  return typedExpr.get()->v;
-}
-
-static std::optional<evaluate::DynamicType> GetDynamicType(
-    const parser::Expr &parserExpr) {
-  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
-    return maybeExpr->GetType();
-  } else {
-    return std::nullopt;
-  }
-}
-
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
 // statements and the expressions enclosed in an OpenMP Workshare construct
 class OmpWorkshareBlockChecker {
@@ -172,85 +141,6 @@ class OmpWorkshareBlockChecker {
   parser::CharBlock source_;
 };
 
-class AssociatedLoopChecker {
-public:
-  AssociatedLoopChecker(SemanticsContext &context, std::int64_t level)
-      : context_{context}, level_{level} {}
-
-  template <typename T> bool Pre(const T &) { return true; }
-  template <typename T> void Post(const T &) {}
-
-  bool Pre(const parser::DoConstruct &dc) {
-    level_--;
-    const auto &doStmt{
-        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
-    const auto &constructName{
-        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
-    if (constructName) {
-      constructNamesAndLevels_.emplace(
-          constructName.value().ToString(), level_);
-    }
-    if (level_ >= 0) {
-      if (dc.IsDoWhile()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
-      }
-      if (!dc.GetLoopControl()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
-      }
-    }
-    return true;
-  }
-
-  void Post(const parser::DoConstruct &dc) { level_++; }
-
-  bool Pre(const parser::CycleStmt &cyclestmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (cyclestmt.v) {
-      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second > 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ > 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::ExitStmt &exitStmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (exitStmt.v) {
-      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ >= 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "EXIT statement terminates associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
-    source_ = &actionstmt.source;
-    return true;
-  }
-
-private:
-  SemanticsContext &context_;
-  const parser::CharBlock *source_;
-  std::int64_t level_;
-  std::map<std::string, std::int64_t> constructNamesAndLevels_;
-};
-
 // `OmpUnitedTaskDesignatorChecker` is used to check if the designator
 // can appear within the TASK construct
 class OmpUnitedTaskDesignatorChecker {
@@ -318,18 +208,6 @@ bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
   return CheckAllowed(clause);
 }
 
-bool OmpStructureChecker::IsCommonBlock(const Symbol &sym) {
-  return sym.detailsIf<CommonBlockDetails>() != nullptr;
-}
-
-bool OmpStructureChecker::IsVariableListItem(const Symbol &sym) {
-  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
-}
-
-bool OmpStructureChecker::IsExtendedListItem(const Symbol &sym) {
-  return IsVariableListItem(sym) || sym.IsSubprogram();
-}
-
 bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   // Definition of close nesting:
   //
@@ -371,60 +249,6 @@ bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   return false;
 }
 
-namespace {
-struct ContiguousHelper {
-  ContiguousHelper(SemanticsContext &context)
-      : fctx_(context.foldingContext()) {}
-
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
-    return Visit(x.value());
-  }
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Reference<Contained> &x) {
-    return Visit(x.get());
-  }
-  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
-    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
-  }
-  template <typename T>
-  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
-    return common::visit(
-        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
-  }
-  template <typename T> std::optional<bool> Visit(const T &) {
-    // Everything else.
-    return std::nullopt;
-  }
-
-private:
-  evaluate::FoldingContext &fctx_;
-};
-} // namespace
-
-// Return values:
-// - std::optional<bool>{true} if the object is known to be contiguous
-// - std::optional<bool>{false} if the object is known not to be contiguous
-// - std::nullopt if the object contiguity cannot be determined
-std::optional<bool> OmpStructureChecker::IsContiguous(
-    const parser::OmpObject &object) {
-  return common::visit( //
-      common::visitors{
-          [&](const parser::Name &x) {
-            // Any member of a common block must be contiguous.
-            return std::optional<bool>{true};
-          },
-          [&](const parser::Designator &x) {
-            evaluate::ExpressionAnalyzer ea{context_};
-            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
-              return ContiguousHelper{context_}.Visit(*maybeExpr);
-            }
-            return std::optional<bool>{};
-          },
-      },
-      object.u);
-}
-
 void OmpStructureChecker::CheckVariableListItem(
     const SymbolSourceMap &symbols) {
   for (auto &[symbol, source] : symbols) {
@@ -522,62 +346,6 @@ bool OmpStructureChecker::HasInvalidWorksharingNesting(
   return false;
 }
 
-void OmpStructureChecker::HasInvalidDistributeNesting(
-    const parser::OpenMPLoopConstruct &x) {
-  bool violation{false};
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
-    // `distribute` region has to be nested
-    if (!CurrentDirectiveIsNested()) {
-      violation = true;
-    } else {
-      // `distribute` region has to be strictly nested inside `teams`
-      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-        violation = true;
-      }
-    }
-  }
-  if (violation) {
-    context_.Say(beginDir.source,
-        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
-        "region."_err_en_US);
-  }
-}
-void OmpStructureChecker::HasInvalidLoopBinding(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
-    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-    for (const auto &clause : clauseList.v) {
-      if (const auto *bindClause{
-              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
-        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
-          context_.Say(beginDir.source, msg);
-        }
-      }
-    }
-  };
-
-  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
-      CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
-        "strictly nested inside a `TEAMS` region."_err_en_US);
-  }
-
-  if (OmpDirectiveSet{
-          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
-          .test(beginDir.v)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
-        "combined with a `TEAMS` construct."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::HasInvalidTeamsNesting(
     const llvm::omp::Directive &dir, const parser::CharBlock &source) {
   if (!llvm::omp::nestedTeamsAllowedSet.test(dir)) {
@@ -668,16 +436,6 @@ void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
-  EnterDirectiveNest(MetadirectiveNest);
-  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
-}
-
-void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
-  ExitDirectiveNest(MetadirectiveNest);
-  dirContext_.pop_back();
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
@@ -717,85 +475,6 @@ void OmpStructureChecker::AddEndDirectiveClauses(
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
-  loopStack_.push_back(&x);
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  PushContextAndClauseSets(beginDir.source, beginDir.v);
-
-  // check matching, End directive is optional
-  if (const auto &endLoopDir{
-          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
-    const auto &endDir{
-        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
-
-    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
-
-    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
-  }
-
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    EnterDirectiveNest(SIMDNest);
-  }
-
-  // Combined target loop constructs are target device constructs. Keep track of
-  // whether any such construct has been visited to later check that REQUIRES
-  // directives for target-related options don't appear after them.
-  if (llvm::omp::allTargetSet.test(beginDir.v)) {
-    deviceConstructFound_ = true;
-  }
-
-  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    // 2.7.1 do-clause -> private-clause |
-    //                    firstprivate-clause |
-    //                    lastprivate-clause |
-    //                    linear-clause |
-    //                    reduction-clause |
-    //                    schedule-clause |
-    //                    collapse-clause |
-    //                    ordered-clause
-
-    // nesting check
-    HasInvalidWorksharingNesting(
-        beginDir.source, llvm::omp::nestedWorkshareErrSet);
-  }
-  SetLoopInfo(x);
-
-  if (const auto &doConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
-    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
-  }
-  CheckLoopItrVariableIsInt(x);
-  CheckAssociatedLoopConstraints(x);
-  HasInvalidDistributeNesting(x);
-  HasInvalidLoopBinding(x);
-  if (CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
-  }
-  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
-      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
-    CheckDistLinear(x);
-  }
-}
-const parser::Name OmpStructureChecker::GetLoopIndex(
-    const parser::DoConstruct *x) {
-  using Bounds = parser::LoopControl::Bounds;
-  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
-}
-void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const parser::DoConstruct *loop{&*loopConstruct};
-    if (loop && loop->IsDoNormal()) {
-      const parser::Name &itrVal{GetLoopIndex(loop)};
-      SetLoopIv(itrVal.symbol);
-    }
-  }
-}
-
 void OmpStructureChecker::CheckIteratorRange(
     const parser::OmpIteratorSpecifier &x) {
   // Check:
@@ -855,108 +534,6 @@ void OmpStructureChecker::CheckIteratorModifier(const parser::OmpIterator &x) {
   }
 }
 
-void OmpStructureChecker::CheckLoopItrVariableIsInt(
-    const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-
-    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-      if (loop->IsDoNormal()) {
-        const parser::Name &itrVal{GetLoopIndex(loop)};
-        if (itrVal.symbol) {
-          const auto *type{itrVal.symbol->GetType()};
-          if (!type->IsNumeric(TypeCategory::Integer)) {
-            context_.Say(itrVal.source,
-                "The DO loop iteration"
-                " variable must be of the type integer."_err_en_US,
-                itrVal.ToString());
-          }
-        }
-      }
-      // Get the next DoConstruct if block is not empty.
-      const auto &block{std::get<parser::Block>(loop->t)};
-      const auto it{block.begin()};
-      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                               : nullptr;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
-  // Check the following:
-  //  The only OpenMP constructs that can be encountered during execution of
-  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
-  // construct and the `ordered` construct with the `simd` clause.
-
-  // Check if the parent context has the SIMD clause
-  // Please note that we use GetContext() instead of GetContextParent()
-  // because PushContextAndClauseSets() has not been called on the
-  // current context yet.
-  // TODO: Check for declare simd regions.
-  bool eligibleSIMD{false};
-  common::visit(
-      common::visitors{
-          // Allow `!$OMP ORDERED SIMD`
-          [&](const parser::OpenMPBlockConstruct &c) {
-            const auto &beginBlockDir{
-                std::get<parser::OmpBeginBlockDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
-            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
-              const auto &clauses{
-                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
-              for (const auto &clause : clauses.v) {
-                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
-                  eligibleSIMD = true;
-                  break;
-                }
-              }
-            }
-          },
-          [&](const parser::OpenMPStandaloneConstruct &c) {
-            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
-                    &c.u)}) {
-              llvm::omp::Directive dirId{ssc->v.DirId()};
-              if (dirId == llvm::omp::Directive::OMPD_ordered) {
-                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
-                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
-                    eligibleSIMD = true;
-                    break;
-                  }
-                }
-              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
-                eligibleSIMD = true;
-              }
-            }
-          },
-          // Allowing SIMD and loop construct
-          [&](const parser::OpenMPLoopConstruct &c) {
-            const auto &beginLoopDir{
-                std::get<parser::OmpBeginLoopDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
-              eligibleSIMD = true;
-            }
-          },
-          [&](const parser::OpenMPAtomicConstruct &c) {
-            // Allow `!$OMP ATOMIC`
-            eligibleSIMD = true;
-          },
-          [&](const auto &c) {},
-      },
-      c.u);
-  if (!eligibleSIMD) {
-    context_.Say(parser::FindSourceLocation(c),
-        "The only OpenMP constructs that can be encountered during execution "
-        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
-        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
-        "construct with the `SIMD` clause."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   // 2.12.5 Target Construct Restriction
   bool eligibleTarget{true};
@@ -1014,187 +591,6 @@ void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   }
 }
 
-std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-  std::int64_t orderedCollapseLevel{1};
-  std::int64_t orderedLevel{1};
-  std::int64_t collapseLevel{1};
-
-  for (const auto &clause : clauseList.v) {
-    if (const auto *collapseClause{
-            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
-      if (const auto v{GetIntValue(collapseClause->v)}) {
-        collapseLevel = *v;
-      }
-    }
-    if (const auto *orderedClause{
-            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
-      if (const auto v{GetIntValue(orderedClause->v)}) {
-        orderedLevel = *v;
-      }
-    }
-  }
-  if (orderedLevel >= collapseLevel) {
-    orderedCollapseLevel = orderedLevel;
-  } else {
-    orderedCollapseLevel = collapseLevel;
-  }
-  return orderedCollapseLevel;
-}
-
-void OmpStructureChecker::CheckAssociatedLoopConstraints(
-    const parser::OpenMPLoopConstruct &x) {
-  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
-  AssociatedLoopChecker checker{context_, ordCollapseLevel};
-  parser::Walk(x, checker);
-}
-
-void OmpStructureChecker::CheckDistLinear(
-    const parser::OpenMPLoopConstruct &x) {
-
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  SymbolSourceMap indexVars;
-
-  // Collect symbols of all the variables from linear clauses
-  for (auto &clause : clauses.v) {
-    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
-      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
-      GetSymbolsInObjectList(objects, indexVars);
-    }
-  }
-
-  if (!indexVars.empty()) {
-    // Get collapse level, if given, to find which loops are "associated."
-    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
-    // Include the top loop if no collapse is specified
-    if (collapseVal == 0) {
-      collapseVal = 1;
-    }
-
-    // Match the loop index variables with the collected symbols from linear
-    // clauses.
-    if (const auto &loopConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-        if (loop->IsDoNormal()) {
-          const parser::Name &itrVal{GetLoopIndex(loop)};
-          if (itrVal.symbol) {
-            // Remove the symbol from the collected set
-            indexVars.erase(&itrVal.symbol->GetUltimate());
-          }
-          collapseVal--;
-          if (collapseVal == 0) {
-            break;
-          }
-        }
-        // Get the next DoConstruct if block is not empty.
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                 : nullptr;
-      }
-    }
-
-    // Show error for the remaining variables
-    for (auto &[symbol, source] : indexVars) {
-      const Symbol &root{GetAssociationRoot(*symbol)};
-      context_.Say(source,
-          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
-          root.name());
-    }
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  // A few semantic checks for InScan reduction are performed below as SCAN
-  // constructs inside LOOP may add the relevant information. Scan reduction is
-  // supported only in loop constructs, so same checks are not applicable to
-  // other directives.
-  using ReductionModifier = parser::OmpReductionModifier;
-  for (const auto &clause : clauseList.v) {
-    if (const auto *reductionClause{
-            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
-      auto &modifiers{OmpGetModifiers(reductionClause->v)};
-      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
-      if (maybeModifier &&
-          maybeModifier->v == ReductionModifier::Value::Inscan) {
-        const auto &objectList{
-            std::get<parser::OmpObjectList>(reductionClause->v.t)};
-        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
-          if (auto &symbol = name->symbol) {
-            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
-                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
-              context_.Say(name->source,
-                  "List item %s must appear in EXCLUSIVE or "
-                  "INCLUSIVE clause of an "
-                  "enclosed SCAN directive"_err_en_US,
-                  name->ToString());
-            }
-          }
-        };
-        for (const auto &ompObj : objectList.v) {
-          common::visit(
-              common::visitors{
-                  [&](const parser::Designator &designator) {
-                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
-                            designator)}) {
-                      checkReductionSymbolInScan(name);
-                    }
-                  },
-                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
-              },
-              ompObj.u);
-        }
-      }
-    }
-  }
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    ExitDirectiveNest(SIMDNest);
-  }
-  dirContext_.pop_back();
-
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top{loopStack_.back()};
-  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
-  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
-  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  // 2.7.1 end-do -> END DO [nowait-clause]
-  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
-  case llvm::omp::Directive::OMPD_do:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
-    break;
-  case llvm::omp::Directive::OMPD_do_simd:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
-    dirContext_.pop_back();
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &endBlockDir{std::get<parser::OmpEndBlockDirective>(x.t)};
@@ -2659,1488 +2055,46 @@ void OmpStructureChecker::CheckCancellationNest(
           parser::ToUpperCaseLetters(typeName.str()));
       break;
     default:
-      // This is diagnosed later.
-      return;
-    }
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
-  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  case llvm::omp::Directive::OMPD_scope:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
-    break;
-  // 2.7.3 end-single-clause -> copyprivate-clause |
-  //                            nowait-clause
-  case llvm::omp::Directive::OMPD_single:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
-    break;
-  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
-  case llvm::omp::Directive::OMPD_workshare:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_workshare);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-// TODO: Verify the popping of dirContext requirement after nowait
-// implementation, as there is an implicit barrier at the end of the worksharing
-// constructs unless a nowait clause is specified. Only OMPD_end_single and
-// end_workshareare popped as they are pushed while entering the
-// EndBlockDirective.
-void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
-    dirContext_.pop_back();
-  }
-}
-
-/// parser::Block is a list of executable constructs, parser::BlockConstruct
-/// is Fortran's BLOCK/ENDBLOCK construct.
-/// Strip the outermost BlockConstructs, return the reference to the Block
-/// in the executable part of the innermost of the stripped constructs.
-/// Specifically, if the given `block` has a single entry (it's a list), and
-/// the entry is a BlockConstruct, get the Block contained within. Repeat
-/// this step as many times as possible.
-static const parser::Block &GetInnermostExecPart(const parser::Block &block) {
-  const parser::Block *iter{&block};
-  while (iter->size() == 1) {
-    const parser::ExecutionPartConstruct &ep{iter->front()};
-    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
-      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
-      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
-        iter = &std::get<parser::Block>(bc->value().t);
-        continue;
-      }
-    }
-    break;
-  }
-  return *iter;
-}
-
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
-struct AnalyzedCondStmt {
-  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
-  parser::CharBlock source;
-  SourcedActionStmt ift, iff;
-};
-
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
-// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
-// is that the ActionStmt will be either an assignment or a pointer-assignment,
-// otherwise return std::nullopt.
-// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
-// the "typedAssignment" is unset. This can happen if there are semantic errors
-// in the purported assignment.
-static std::optional<evaluate::Assignment> GetEvaluateAssignment(
-    const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
-
-  return common::visit(
-      [](auto &&s) -> std::optional<evaluate::Assignment> {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>) {
-          const TypedAssignment &typed{s.value().typedAssignment};
-          // ForwardOwningPointer                 typedAssignment
-          // `- GenericAssignmentWrapper          ^.get()
-          //    `- std::optional<Assignment>      ^->v
-          return typed.get()->v;
-        } else {
-          return std::nullopt;
-        }
-      },
-      x->u);
-}
-
-// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
-// to separate cases where the source has something that looks like an
-// assignment, but is semantically wrong (diagnosed by general semantic
-// checks), and where the source has some other statement (which we want
-// to report as "should be an assignment").
-static bool IsAssignment(const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return false;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-
-  return common::visit(
-      [](auto &&s) -> bool {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        return std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>;
-      },
-      x->u);
-}
-
-static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  // Extract the evaluate::Expr from ScalarLogicalExpr.
-  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
-    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
-    const parser::Expr &expr{logical.thing.thing.value()};
-    return GetEvaluateExpr(expr);
-  }};
-
-  // Recognize either
-  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
-  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
-
-  if (auto &&action{GetActionStmt(x)}) {
-    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
-            &action.stmt->u)}) {
-      const parser::IfStmt &s{ifs->value()};
-      auto &&maybeCond{
-          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
-      auto &thenStmt{
-          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
-      if (maybeCond) {
-        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
-            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
-            SourcedActionStmt{}};
-      }
-    }
-    return std::nullopt;
-  }
-
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    if (auto *ifc{
-            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
-      using ElseBlock = parser::IfConstruct::ElseBlock;
-      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
-      const parser::IfConstruct &s{ifc->value()};
-
-      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
-        // Not expecting any else-if statements.
-        return std::nullopt;
-      }
-      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
-      auto &&maybeCond{getFromLogical(
-          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
-      if (!maybeCond) {
-        return std::nullopt;
-      }
-
-      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)),
-            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
-        if (result.ift.stmt && result.iff.stmt) {
-          return result;
-        }
-      } else {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
-        if (result.ift.stmt) {
-          return result;
-        }
-      }
-    }
-    return std::nullopt;
-  }
-
-  return std::nullopt;
-}
-
-static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
-    parser::CharBlock source) {
-  // Find => in the range, if not found, find = that is not a part of
-  // <=, >=, ==, or /=.
-  auto trim{[](std::string_view v) {
-    const char *begin{v.data()};
-    const char *end{begin + v.size()};
-    while (*begin == ' ' && begin != end) {
-      ++begin;
-    }
-    while (begin != end && end[-1] == ' ') {
-      --end;
-    }
-    assert(begin != end && "Source should not be empty");
-    return parser::CharBlock(begin, end - begin);
-  }};
-
-  std::string_view sv(source.begin(), source.size());
-
-  if (auto where{sv.find("=>")}; where != sv.npos) {
-    std::string_view lhs(sv.data(), where);
-    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
-    return std::make_pair(trim(lhs), trim(rhs));
-  }
-
-  // Go backwards, since all the exclusions above end with a '='.
-  for (size_t next{source.size()}; next > 1; --next) {
-    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
-      std::string_view lhs(sv.data(), next - 1);
-      std::string_view rhs(sv.data() + next, sv.size() - next);
-      return std::make_pair(trim(lhs), trim(rhs));
-    }
-  }
-  llvm_unreachable("Could not find assignment operator");
-}
-
-namespace atomic {
-
-struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
-                                 std::vector<SomeExpr>, false> {
-  using Result = std::vector<SomeExpr>;
-  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
-  DesignatorCollector() : Base(*this) {}
-
-  Result Default() const { return {}; }
-
-  using Base::operator();
-
-  template <typename T> //
-  Result operator()(const evaluate::Designator<T> &x) const {
-    // Once in a designator, don't traverse it any further (i.e. only
-    // collect top-level designators).
-    auto copy{x};
-    return Result{AsGenericExpr(std::move(copy))};
-  }
-
-  template <typename... Rs> //
-  Result Combine(Result &&result, Rs &&...results) const {
-    Result v(std::move(result));
-    auto moveAppend{[](auto &accum, auto &&other) {
-      for (auto &&s : other) {
-        accum.push_back(std::move(s));
-      }
-    }};
-    (moveAppend(v, std::move(results)), ...);
-    return v;
-  }
-};
-
-struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
-  using Base = evaluate::AnyTraverse<VariableFinder>;
-  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
-
-  using Base::operator();
-
-  template <typename T>
-  bool operator()(const evaluate::Designator<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-  template <typename T>
-  bool operator()(const evaluate::FunctionRef<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-private:
-  const SomeExpr &var;
-};
-} // namespace atomic
-
-static bool IsPointerAssignment(const evaluate::Assignment &x) {
-  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
-      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
-}
-
-namespace operation = Fortran::evaluate::operation;
-
-static bool IsCheckForAssociated(const SomeExpr &cond) {
-  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
-}
-
-static bool HasCommonDesignatorSymbols(
-    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
-  // Compare the designators used in "other" with the designators whose
-  // symbols are given in baseSyms.
-  // This is a part of the check if these two expressions can access the same
-  // storage: if the designators used in them are different enough, then they
-  // will be assumed not to access the same memory.
-  //
-  // Consider an (array element) expression x%y(w%z), the corresponding symbol
-  // vector will be {x, y, w, z} (i.e. the symbols for these names).
-  // Check whether this exact sequence appears anywhere in any the symbol
-  // vector for "other". This will be true for x(y) and x(y+1), so this is
-  // not a sufficient condition, but can be used to eliminate candidates
-  // before doing more exhaustive checks.
-  //
-  // If any of the symbols in this sequence are function names, assume that
-  // there is no storage overlap, mostly because it would be impossible in
-  // general to determine what storage the function will access.
-  // Note: if f is pure, then two calls to f will access the same storage
-  // when called with the same arguments. This check is not done yet.
-
-  if (llvm::any_of(
-          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
-    // If there is a function symbol in the chain then we can't infer much
-    // about the accessed storage.
-    return false;
-  }
-
-  auto isSubsequence{// Is u a subsequence of v.
-      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
-        size_t us{u.size()}, vs{v.size()};
-        if (us > vs) {
-          return false;
-        }
-        for (size_t off{0}; off != vs - us + 1; ++off) {
-          bool same{true};
-          for (size_t i{0}; i != us; ++i) {
-            if (u[i] != v[off + i]) {
-              same = false;
-              break;
-            }
-          }
-          if (same) {
-            return true;
-          }
-        }
-        return false;
-      }};
-
-  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
-  return isSubsequence(baseSyms, otherSyms);
-}
-
-static bool HasCommonTopLevelDesignators(
-    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
-  // Compare designators directly as expressions. This will ensure
-  // that x(y) and x(y+1) are not flagged as overlapping, whereas
-  // the symbol vectors for both of these would be identical.
-  std::vector<SomeExpr> otherDsgs{atomic::DesignatorCollector{}(other)};
-
-  for (auto &s : baseDsgs) {
-    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
-      return true;
-    }
-  }
-  return false;
-}
-
-static const SomeExpr *HasStorageOverlap(
-    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
-  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
-  std::vector<SomeExpr> baseDsgs{atomic::DesignatorCollector{}(base)};
-
-  for (const SomeExpr &expr : exprs) {
-    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
-      continue;
-    }
-    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
-      return &expr;
-    }
-  }
-  return nullptr;
-}
-
-static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
-  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
-  // for example.
-  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
-}
-
-static bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
-  return atomic::VariableFinder{sub}(super);
-}
-
-static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
-  if (value) {
-    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
-        evaluate::GenericExprWrapper::Deleter);
-  }
-}
-
-static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
-    std::optional<evaluate::Assignment> value) {
-  if (value) {
-    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
-        evaluate::GenericAssignmentWrapper::Deleter);
-  }
-}
-
-static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
-    int what,
-    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
-  parser::OpenMPAtomicConstruct::Analysis::Op operation;
-  operation.what = what;
-  SetAssignment(operation.assign, maybeAssign);
-  return operation;
-}
-
-static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
-    const SomeExpr &atom, const MaybeExpr &cond,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
-  // Defined in flang/include/flang/Parser/parse-tree.h
-  //
-  // struct Analysis {
-  //   struct Kind {
-  //     static constexpr int None = 0;
-  //     static constexpr int Read = 1;
-  //     static constexpr int Write = 2;
-  //     static constexpr int Update = Read | Write;
-  //     static constexpr int Action = 3; // Bits containing N, R, W, U
-  //     static constexpr int IfTrue = 4;
-  //     static constexpr int IfFalse = 8;
-  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
-  //   };
-  //   struct Op {
-  //     int what;
-  //     TypedAssignment assign;
-  //   };
-  //   TypedExpr atom, cond;
-  //   Op op0, op1;
-  // };
-
-  parser::OpenMPAtomicConstruct::Analysis an;
-  SetExpr(an.atom, atom);
-  SetExpr(an.cond, cond);
-  an.op0 = std::move(op0);
-  an.op1 = std::move(op1);
-  return an;
-}
-
-void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
-    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
-    parser::CharBlock source) {
-  if (auto *expr{HasStorageOverlap(base, exprs)}) {
-    context_.Say(source,
-        "Within atomic operation %s and %s access the same storage"_warn_en_US,
-        base.AsFortran(), expr->AsFortran());
-  }
-}
-
-void OmpStructureChecker::ErrorShouldBeVariable(
-    const MaybeExpr &expr, parser::CharBlock source) {
-  if (expr) {
-    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
-        expr->AsFortran());
-  } else {
-    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
-  }
-}
-
-/// Check if `expr` satisfies the following conditions for x and v:
-///
-/// [6.0:189:10-12]
-/// - x and v (as applicable) are either scalar variables or
-///   function references with scalar data pointer result of non-character
-///   intrinsic type or variables that are non-polymorphic scalar pointers
-///   and any length type parameter must be constant.
-void OmpStructureChecker::CheckAtomicType(
-    SymbolRef sym, parser::CharBlock source, std::string_view name) {
-  const DeclTypeSpec *typeSpec{sym->GetType()};
-  if (!typeSpec) {
-    return;
-  }
-
-  if (!IsPointer(sym)) {
-    using Category = DeclTypeSpec::Category;
-    Category cat{typeSpec->category()};
-    if (cat == Category::Character) {
-      context_.Say(source,
-          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
-    } else if (cat != Category::Numeric && cat != Category::Logical) {
-      context_.Say(source,
-          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
-    }
-    return;
-  }
-
-  // Variable is a pointer.
-  if (typeSpec->IsPolymorphic()) {
-    context_.Say(source,
-        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
-        name);
-    return;
-  }
-
-  // Go over all length parameters, if any, and check if they are
-  // explicit.
-  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
-    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
-          // "entry" is a map entry
-          return entry.second.isLen() && !entry.second.isExplicit();
-        })) {
-      context_.Say(source,
-          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
-          name);
-    }
-  }
-}
-
-void OmpStructureChecker::CheckAtomicVariable(
-    const SomeExpr &atom, parser::CharBlock source) {
-  if (atom.Rank() != 0) {
-    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
-        atom.AsFortran());
-  }
-
-  std::vector<SomeExpr> dsgs{atomic::DesignatorCollector{}(atom)};
-  assert(dsgs.size() == 1 && "Should have a single top-level designator");
-  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
-
-  CheckAtomicType(syms.back(), source, atom.AsFortran());
-
-  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
-    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
-        atom.AsFortran());
-  }
-}
-
-std::pair<const parser::ExecutionPartConstruct *,
-    const parser::ExecutionPartConstruct *>
-OmpStructureChecker::CheckUpdateCapture(
-    const parser::ExecutionPartConstruct *ec1,
-    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
-  // Decide which statement is the atomic update and which is the capture.
-  //
-  // The two allowed cases are:
-  //   x = ...      atomic-var = ...
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  // or
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  //   x = ...      atomic-var = ...
-  //
-  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
-  // (which makes more sense, as it captures the original value of the atomic
-  // variable).
-  //
-  // If the two statements don't fit these criteria, return a pair of default-
-  // constructed values.
-  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
-      const parser::ExecutionPartConstruct *>;
-
-  SourcedActionStmt act1{GetActionStmt(ec1)};
-  SourcedActionStmt act2{GetActionStmt(ec2)};
-  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
-  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
-  if (!maybeAssign1 || !maybeAssign2) {
-    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
-      context_.Say(source,
-          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
-    }
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  auto as1{*maybeAssign1}, as2{*maybeAssign2};
-
-  auto isUpdateCapture{
-      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
-        return IsSameOrConvertOf(c.rhs, u.lhs);
-      }};
-
-  // Do some checks that narrow down the possible choices for the update
-  // and the capture statements. This will help to emit better diagnostics.
-  // 1. An assignment could be an update (cbu) if the left-hand side is a
-  //    subexpression of the right-hand side.
-  // 2. An assignment could be a capture (cbc) if the right-hand side is
-  //    a variable (or a function ref), with potential type conversions.
-  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
-  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
-  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
-  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
-
-  // We want to diagnose cases where both assignments cannot be an update,
-  // or both cannot be a capture, as well as cases where either assignment
-  // cannot be any of these two.
-  //
-  // If we organize these boolean values into a matrix
-  //   |cbu1 cbu2|
-  //   |cbc1 cbc2|
-  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
-  // row or column, including the case where everything is zero. All these
-  // cases correspond to the determinant of the matrix being 0, which suggests
-  // that checking the det may be a convenient diagnostic check. There is only
-  // one additional case where the det is 0, which is when the matrix is all 1
-  // ("true"). The "all true" case represents the situation where both
-  // assignments could be an update as well as a capture. On the other hand,
-  // whenever det != 0, the roles of the update and the capture can be
-  // unambiguously assigned to as1 and as2 [1].
-  //
-  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
-  // det = 0, leaving 6 cases where det != 0:
-  //   0 1   0 1   1 0   1 0   1 1   1 1
-  //   1 0   1 1   0 1   1 1   0 1   1 0
-  // In each case the classification is unambiguous.
-
-  //     |cbu1 cbu2|
-  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
-  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
-
-  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
-                                  const std::string &expr) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
-        expr);
-  }};
-
-  auto errorNeitherWorks{[&]() {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
-  }};
-
-  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
-    // If det != 0, then the checks unambiguously suggest a specific
-    // categorization.
-    // If det == 0, then this function should be called only if the
-    // checks haven't ruled out any possibility, i.e. when both assigments
-    // could still be either updates or captures.
-    if (det > 0) {
-      // as1 is update, as2 is capture
-      if (isUpdateCapture(as1, as2)) {
-        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
-      } else {
-        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else if (det < 0) {
-      // as2 is update, as1 is capture
-      if (isUpdateCapture(as2, as1)) {
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      } else {
-        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else {
-      bool updateFirst{isUpdateCapture(as1, as2)};
-      bool captureFirst{isUpdateCapture(as2, as1)};
-      if (updateFirst && captureFirst) {
-        // If both assignment could be the update and both could be the
-        // capture, emit a warning about the ambiguity.
-        context_.Say(act1.source,
-            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      }
-      if (updateFirst != captureFirst) {
-        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
-        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
-        return std::make_pair(upd, cap);
-      }
-      assert(!updateFirst && !captureFirst);
-      errorNeitherWorks();
-      return std::make_pair(nullptr, nullptr);
-    }
-  }};
-
-  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
-    return makeSelectionFromDet(det);
-  }
-  assert(det == 0 && "Prior checks should have covered det != 0");
-
-  // If neither of the statements is an RMW update, it could still be a
-  // "write" update. Pretty much any assignment can be a write update, so
-  // recompute det with cbu1 = cbu2 = true.
-  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
-    return makeSelectionFromDet(writeDet);
-  }
-
-  // It's only errors from here on.
-
-  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
-    errorNeitherWorks();
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // The remaining cases are that
-  // - no candidate for update, or for capture,
-  // - one of the assigments cannot be anything.
-
-  if (!cbu1 && !cbu2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  } else if (!cbc1 && !cbc2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
-    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
-    context_.Say(src,
-        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // All cases should have been covered.
-  llvm_unreachable("Unchecked condition");
-}
-
-void OmpStructureChecker::CheckAtomicCaptureAssignment(
-    const evaluate::Assignment &capture, const SomeExpr &atom,
-    parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &cap{capture.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, rsrc);
-    // This part should have been checked prior to calling this function.
-    assert(*GetConvertInput(capture.rhs) == atom &&
-        "This cannot be a capture assignment");
-    CheckStorageOverlap(atom, {cap}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicReadAssignment(
-    const evaluate::Assignment &read, parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-
-  if (auto maybe{GetConvertInput(read.rhs)}) {
-    const SomeExpr &atom{*maybe};
-
-    if (!IsVarOrFunctionRef(atom)) {
-      ErrorShouldBeVariable(atom, rsrc);
-    } else {
-      CheckAtomicVariable(atom, rsrc);
-      CheckStorageOverlap(atom, {read.lhs}, source);
-    }
-  } else {
-    ErrorShouldBeVariable(read.rhs, rsrc);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicWriteAssignment(
-    const evaluate::Assignment &write, parser::CharBlock source) {
-  // [6.0:190:13-15]
-  // A write structured block is write-statement, a write statement that has
-  // one of the following forms:
-  //   x = expr
-  //   x => expr
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{write.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, lsrc);
-    CheckStorageOverlap(atom, {write.rhs}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateAssignment(
-    const evaluate::Assignment &update, parser::CharBlock source) {
-  // [6.0:191:1-7]
-  // An update structured block is update-statement, an update statement
-  // that has one of the following forms:
-  //   x = x operator expr
-  //   x = expr operator x
-  //   x = intrinsic-procedure-name (x)
-  //   x = intrinsic-procedure-name (x, expr-list)
-  //   x = intrinsic-procedure-name (expr-list, x)
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{update.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, lsrc);
-
-  std::pair<operation::Operator, std::vector<SomeExpr>> top{
-      operation::Operator::Unknown, {}};
-  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
-    top = GetTopLevelOperation(*maybeInput);
-  }
-  switch (top.first) {
-  case operation::Operator::Add:
-  case operation::Operator::Sub:
-  case operation::Operator::Mul:
-  case operation::Operator::Div:
-  case operation::Operator::And:
-  case operation::Operator::Or:
-  case operation::Operator::Eqv:
-  case operation::Operator::Neqv:
-  case operation::Operator::Min:
-  case operation::Operator::Max:
-  case operation::Operator::Identity:
-    break;
-  case operation::Operator::Call:
-    context_.Say(source,
-        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Convert:
-    context_.Say(source,
-        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Intrinsic:
-    context_.Say(source,
-        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Constant:
-  case operation::Operator::Unknown:
-    context_.Say(
-        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(source,
-        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
-        operation::ToString(top.first));
-    return;
-  }
-  // Check how many times `atom` occurs as an argument, if it's a subexpression
-  // of an argument, and collect the non-atom arguments.
-  std::vector<SomeExpr> nonAtom;
-  MaybeExpr subExpr;
-  auto atomCount{[&]() {
-    int count{0};
-    for (const SomeExpr &arg : top.second) {
-      if (IsSameOrConvertOf(arg, atom)) {
-        ++count;
-      } else {
-        if (!subExpr && IsSubexpressionOf(atom, arg)) {
-          subExpr = arg;
-        }
-        nonAtom.push_back(arg);
-      }
-    }
-    return count;
-  }()};
-
-  bool hasError{false};
-  if (subExpr) {
-    context_.Say(rsrc,
-        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
-        atom.AsFortran(), subExpr->AsFortran());
-    hasError = true;
-  }
-  if (top.first == operation::Operator::Identity) {
-    // This is "x = y".
-    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
-          atom.AsFortran());
-      hasError = true;
-    }
-  } else {
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    } else if (atomCount > 1) {
-      context_.Say(rsrc,
-          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    }
-  }
-
-  if (!hasError) {
-    CheckStorageOverlap(atom, nonAtom, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
-    const SomeExpr &cond, parser::CharBlock condSource,
-    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
-  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
-  const SomeExpr &atom{assign.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, arsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, alsrc);
-
-  auto top{GetTopLevelOperation(cond)};
-  // Missing arguments to operations would have been diagnosed by now.
-
-  switch (top.first) {
-  case operation::Operator::Associated:
-    if (atom != top.second.front()) {
-      context_.Say(assignSource,
-          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
-    }
-    break;
-  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
-  case operation::Operator::Eq:
-  case operation::Operator::Eqv:
-  // x ordop expr | expr ordop x
-  case operation::Operator::Lt:
-  case operation::Operator::Gt: {
-    const SomeExpr &arg0{top.second[0]};
-    const SomeExpr &arg1{top.second[1]};
-    if (IsSameOrConvertOf(arg0, atom)) {
-      CheckStorageOverlap(atom, {arg1}, condSource);
-    } else if (IsSameOrConvertOf(arg1, atom)) {
-      CheckStorageOverlap(atom, {arg0}, condSource);
-    } else {
-      assert(top.first != operation::Operator::Identity &&
-          "Handle this separately");
-      context_.Say(assignSource,
-          "An argument of the %s operator should be the target of the assignment"_err_en_US,
-          operation::ToString(top.first));
-    }
-    break;
-  }
-  case operation::Operator::Identity:
-  case operation::Operator::True:
-  case operation::Operator::False:
-    break;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(condSource,
-        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
-        operation::ToString(top.first));
-    break;
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
-    const AnalyzedCondStmt &update, parser::CharBlock source) {
-  // The condition/statements must be:
-  // - cond: x equalop e      ift: x =  d     iff: -
-  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
-  // - cond: associated(x)    ift: x => expr  iff: -
-  // - cond: associated(x, e) ift: x => expr  iff: -
-
-  // The if-true statement must be present, and must be an assignment.
-  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
-  if (!maybeAssign) {
-    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
-    } else {
-      context_.Say(
-          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
-    }
-    return;
-  }
-  const evaluate::Assignment assign{*maybeAssign};
-  const SomeExpr &atom{assign.lhs};
-
-  CheckAtomicConditionalUpdateAssignment(
-      update.cond, update.source, assign, update.ift.source);
-
-  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
-
-  if (update.iff) {
-    context_.Say(update.iff.source,
-        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateOnly(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeUpdate->lhs};
-      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdate(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // Allowable forms are (single-statement):
-  // - if ...
-  // - x = (... ? ... : x)
-  // and two-statement:
-  // - r = cond ; if (r) ...
-
-  const parser::ExecutionPartConstruct *ust{nullptr}; // update
-  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
-
-  if (body.size() == 1) {
-    ust = &body.front();
-  } else if (body.size() == 2) {
-    cst = &body.front();
-    ust = &body.back();
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
-    return;
-  }
-
-  // Flang doesn't support conditional-expr yet, so all update statements
-  // are if-statements.
-
-  // IfStmt:        if (...) ...
-  // IfConstruct:   if (...) then ... endif
-  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
-  if (!maybeUpdate) {
-    context_.Say(source,
-        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
-    return;
-  }
-
-  AnalyzedCondStmt &update{*maybeUpdate};
-
-  if (SourcedActionStmt action{GetActionStmt(cst)}) {
-    // The "condition" statement must be `r = cond`.
-    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
-      if (maybeCond->lhs != update.cond) {
-        context_.Say(update.source,
-            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
-            maybeCond->lhs.AsFortran());
-      } else {
-        // If it's "r = ...; if (r) ..." then put the original condition
-        // in `update`.
-        update.cond = maybeCond->rhs;
-      }
-    } else {
-      context_.Say(action.source,
-          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
-    }
-  }
-
-  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
-
-  CheckAtomicConditionalUpdateStmt(update, source);
-  if (IsCheckForAssociated(update.cond)) {
-    if (!IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
-    }
-  } else {
-    if (IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
-    }
-  }
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
-      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
-      MakeAtomicAnalysisOp(Analysis::None));
-}
-
-void OmpStructureChecker::CheckAtomicUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() != 2) {
-    context_.Say(source,
-        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
-    return;
-  }
-
-  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
-  if (!uec || !cec) {
-    // Diagnostics already emitted.
-    return;
-  }
-  SourcedActionStmt uact{GetActionStmt(uec)};
-  SourcedActionStmt cact{GetActionStmt(cec)};
-  // The "dereferences" of std::optional are guaranteed to be valid after
-  // CheckUpdateCapture.
-  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
-  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
-
-  const SomeExpr &atom{update.lhs};
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  int action;
-
-  if (IsMaybeAtomicWrite(update)) {
-    action = Analysis::Write;
-    CheckAtomicWriteAssignment(update, uact.source);
-  } else {
-    action = Analysis::Update;
-    CheckAtomicUpdateAssignment(update, uact.source);
-  }
-  CheckAtomicCaptureAssignment(capture, atom, cact.source);
-
-  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
-    context_.Say(cact.source,
-        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-    return;
-  }
-
-  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(action, update),
-        MakeAtomicAnalysisOp(Analysis::Read, capture));
-  } else {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(Analysis::Read, capture),
-        MakeAtomicAnalysisOp(action, update));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // There are two different variants of this:
-  // (1) conditional-update and capture separately:
-  //     This form only allows single-statement updates, i.e. the update
-  //     form "r = cond; if (r) ..." is not allowed.
-  // (2) conditional-update combined with capture in a single statement:
-  //     This form does allow the condition to be calculated separately,
-  //     i.e. "r = cond; if (r) ...".
-  // Regardless of what form it is, the actual update assignment is a
-  // proper write, i.e. "x = d", where d does not depend on x.
-
-  AnalyzedCondStmt update;
-  SourcedActionStmt capture;
-  bool captureAlways{true}, captureFirst{true};
-
-  auto extractCapture{[&]() {
-    capture = update.iff;
-    captureAlways = false;
-    update.iff = SourcedActionStmt{};
-  }};
-
-  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
-    // The non-update statement is either "r = cond" or the capture.
-    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
-      if (update.cond == maybeAssign->lhs) {
-        // If this is "r = cond; if (r) ...", then update the condition.
-        update.cond = maybeAssign->rhs;
-        update.source = action.source;
-        // In this form, the update and the capture are combined into
-        // an IF-THEN-ELSE statement.
-        extractCapture();
-      } else {
-        // Assume this is the capture-statement.
-        capture = action;
-      }
-    }
-  }};
-
-  if (body.size() == 2) {
-    // This could be
-    // - capture; conditional-update (in any order), or
-    // - r = cond; if (r) capture-update
-    const parser::ExecutionPartConstruct *st1{&body.front()};
-    const parser::ExecutionPartConstruct *st2{&body.back()};
-    // In either case, the conditional statement can be analyzed by
-    // AnalyzeConditionalStmt, whereas the other statement cannot.
-    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
-      update = *maybeUpdate1;
-      classifyNonUpdate(GetActionStmt(st2));
-      captureFirst = false;
-    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
-      update = *maybeUpdate2;
-      classifyNonUpdate(GetActionStmt(st1));
-    } else {
-      // None of the statements are conditional, this rules out the
-      // "r = cond; if (r) ..." and the "capture + conditional-update"
-      // variants. This could still be capture + write (which is classified
-      // as conditional-update-capture in the spec).
-      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
-      if (!uec || !cec) {
-        // Diagnostics already emitted.
-        return;
-      }
-      SourcedActionStmt uact{GetActionStmt(uec)};
-      SourcedActionStmt cact{GetActionStmt(cec)};
-      update.ift = uact;
-      capture = cact;
-      if (uec == st1) {
-        captureFirst = false;
-      }
-    }
-  } else if (body.size() == 1) {
-    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
-      update = *maybeUpdate;
-      // This is the form with update and capture combined into an IF-THEN-ELSE
-      // statement. The capture-statement is always the ELSE branch.
-      extractCapture();
-    } else {
-      goto invalid;
-    }
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
-    return;
-  invalid:
-    context_.Say(source,
-        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
-    return;
-  }
-
-  // The update must have a form `x = d` or `x => d`.
-  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
-    const SomeExpr &atom{maybeWrite->lhs};
-    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
-    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
-      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
-
-      if (IsPointerAssignment(*maybeWrite) !=
-          IsPointerAssignment(*maybeCapture)) {
-        context_.Say(capture.source,
-            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-        return;
-      }
-    } else {
-      if (!IsAssignment(capture.stmt)) {
-        context_.Say(capture.source,
-            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
-      }
-      return;
-    }
-  } else {
-    if (!IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
-    }
-    return;
-  }
-
-  // update.iff should be empty here, the capture statement should be
-  // stored in "capture".
-
-  // Fill out the analysis in the AST node.
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  bool condUnused{std::visit(
-      [](auto &&s) {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
-          return true;
-        } else {
-          return false;
-        }
-      },
-      update.cond.u)};
-
-  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
-  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
-
-  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
-  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
-
-  if (captureFirst) {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
-  } else {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicRead(
-    const parser::OpenMPAtomicConstruct &x) {
-  // [6.0:190:5-7]
-  // A read structured block is read-statement, a read statement that has one
-  // of the following forms:
-  //   v = x
-  //   v => x
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Read cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
-      CheckAtomicReadAssignment(*maybeRead, action.source);
-
-      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
-        const SomeExpr &atom{*maybe};
-        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
-            MakeAtomicAnalysisOp(Analysis::None));
-      }
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+      // This is diagnosed later.
+      return;
     }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC READ operation should have a single statement"_err_en_US);
   }
 }
 
-void OmpStructureChecker::CheckAtomicWrite(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Write cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeWrite->lhs};
-      CheckAtomicWriteAssignment(*maybeWrite, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC WRITE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdate(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &block{std::get<parser::Block>(x.t)};
-
-  bool isConditional{x.IsCompare()};
-  bool isCapture{x.IsCapture()};
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (isConditional && isCapture) {
-    CheckAtomicConditionalUpdateCapture(x, body, x.source);
-  } else if (isConditional) {
-    CheckAtomicConditionalUpdate(x, body, x.source);
-  } else if (isCapture) {
-    CheckAtomicUpdateCapture(x, body, x.source);
-  } else { // update-only
-    CheckAtomicUpdateOnly(x, body, x.source);
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
-  if (visitedAtomicSource_.empty())
-    visitedAtomicSource_ = x.source;
-
-  // All of the following groups have the "exclusive" property, i.e. at
-  // most one clause from each group is allowed.
-  // The exclusivity-checking code should eventually be unified for all
-  // clauses, with clause groups defined in OMP.td.
-  std::array atomic{llvm::omp::Clause::OMPC_read,
-      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
-  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
-      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
-      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
-
-  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
-                          std::string_view name,
-                          const parser::OmpClauseList &clauses) {
-    const parser::OmpClause *present{nullptr};
-    for (const parser::OmpClause &clause : clauses.v) {
-      llvm::omp::Clause id{clause.Id()};
-      if (!llvm::is_contained(group, id)) {
-        continue;
-      }
-      if (present == nullptr) {
-        present = &clause;
-        continue;
-      } else if (id == present->Id()) {
-        // Ignore repetitions of the same clause, those will be diagnosed
-        // separately.
-        continue;
-      }
-      parser::MessageFormattedText txt(
-          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
-          name.data());
-      parser::Message message(clause.source, txt);
-      message.Attach(present->source,
-          "Previous clause from this group provided here"_en_US);
-      context_.Say(std::move(message));
-      return;
-    }
-  }};
-
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
-  llvm::omp::Clause kind{x.GetKind()};
-
-  checkExclusive(atomic, "atomic", dirSpec.Clauses());
-  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
-
-  switch (kind) {
-  case llvm::omp::Clause::OMPC_read:
-    CheckAtomicRead(x);
+void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
+  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  case llvm::omp::Directive::OMPD_scope:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
     break;
-  case llvm::omp::Clause::OMPC_write:
-    CheckAtomicWrite(x);
+  // 2.7.3 end-single-clause -> copyprivate-clause |
+  //                            nowait-clause
+  case llvm::omp::Directive::OMPD_single:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
     break;
-  case llvm::omp::Clause::OMPC_update:
-    CheckAtomicUpdate(x);
+  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
+  case llvm::omp::Directive::OMPD_workshare:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_workshare);
     break;
   default:
+    // no clauses are allowed
     break;
   }
 }
 
-void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
-  dirContext_.pop_back();
+// TODO: Verify the popping of dirContext requirement after nowait
+// implementation, as there is an implicit barrier at the end of the worksharing
+// constructs unless a nowait clause is specified. Only OMPD_end_single and
+// end_workshareare popped as they are pushed while entering the
+// EndBlockDirective.
+void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
+    dirContext_.pop_back();
+  }
 }
 
 // Clauses
@@ -5208,102 +3162,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
-  unsigned version{context_.langOptions().OpenMPVersion};
-  llvm::omp::Directive dir{GetContext().directive};
-  parser::CharBlock clauseSource{GetContext().clauseSource};
-  const parser::OmpLinearModifier *linearMod{nullptr};
-
-  SymbolSourceMap symbols;
-  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
-  CheckCrayPointee(objects, "LINEAR", false);
-  GetSymbolsInObjectList(objects, symbols);
-
-  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
-    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      context_.Say(source,
-          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
-          symbol->name(), desc.name.str());
-    }
-  }};
-
-  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
-    auto &modifiers{OmpGetModifiers(x.v)};
-    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
-    if (linearMod) {
-      // 2.7 Loop Construct Restriction
-      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
-        context_.Say(clauseSource,
-            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
-            ContextDirectiveAsFortran());
-        return;
-      }
-
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      for (auto &[symbol, source] : symbols) {
-        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
-          CheckIntegerNoRef(symbol, source);
-        } else {
-          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
-              !IsPolymorphic(*symbol)) {
-            context_.Say(source,
-                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
-                symbol->name(), desc.name.str());
-          }
-        }
-        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
-            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
-          if (!IsDummy(*symbol) || IsValue(*symbol)) {
-            context_.Say(source,
-                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
-                desc.name.str(), symbol->name());
-          }
-        }
-      } // for (symbol, source)
-
-      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
-        context_.Say(OmpGetModifierSource(modifiers, linearMod),
-            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
-            ThisVersion(version));
-      }
-    }
-  }
-
-  // OpenMP 5.2: Ordered clause restriction
-  if (const auto *clause{
-          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
-    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
-    if (orderedClause.v) {
-      return;
-    }
-  }
-
-  // OpenMP 5.2: Linear clause Restrictions
-  for (auto &[symbol, source] : symbols) {
-    if (!linearMod) {
-      // Already checked this with the modifier present.
-      CheckIntegerNoRef(symbol, source);
-    }
-    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
-      context_.Say(source,
-          "The list item `%s` must be a dummy argument"_err_en_US,
-          symbol->name());
-    }
-    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
-      context_.Say(source,
-          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
-          symbol->name());
-    }
-    if (FindCommonBlockContaining(*symbol)) {
-      context_.Say(source,
-          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
-          symbol->name());
-    }
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OmpClause::Detach &x) {
   unsigned version{context_.langOptions().OpenMPVersion};
   if (version >= 52) {
@@ -6012,503 +3870,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::OmpxBare &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
-  OmpVerifyModifiers(
-      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
-}
-
-void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
-  EnterDirectiveNest(ContextSelectorNest);
-
-  using SetName = parser::OmpTraitSetSelectorName;
-  std::map<SetName::Value, const SetName *> visited;
-
-  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
-    auto &name{std::get<SetName>(traitSet.t)};
-    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait set name %s in a context specifier"_err_en_US,
-          showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait set %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-    CheckTraitSetSelector(traitSet);
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
-  ExitDirectiveNest(ContextSelectorNest);
-}
-
-const std::list<parser::OmpTraitProperty> &
-OmpStructureChecker::GetTraitPropertyList(
-    const parser::OmpTraitSelector &trait) {
-  static const std::list<parser::OmpTraitProperty> empty{};
-  auto &[_, maybeProps]{trait.t};
-  if (maybeProps) {
-    using PropertyList = std::list<parser::OmpTraitProperty>;
-    return std::get<PropertyList>(maybeProps->t);
-  } else {
-    return empty;
-  }
-}
-
-std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
-    const parser::OmpTraitProperty &property) {
-  using MaybeClause = std::optional<llvm::omp::Clause>;
-
-  // The parser for OmpClause will only succeed if the clause was
-  // given with all required arguments.
-  // If this is a string or complex extension with a clause name,
-  // treat it as a clause and let the trait checker deal with it.
-
-  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
-    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
-    if (id != llvm::omp::Clause::OMPC_unknown) {
-      return id;
-    } else {
-      return std::nullopt;
-    }
-  }};
-
-  return common::visit( //
-      common::visitors{
-          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
-            return getClauseFromString(x.v);
-          },
-          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
-            return x.value().Id();
-          },
-          [&](const parser::ScalarExpr &x) -> MaybeClause {
-            return std::nullopt;
-          },
-          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
-            using ExtProperty = parser::OmpTraitPropertyExtension;
-            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
-              return getClauseFromString(name->v);
-            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
-              return getClauseFromString(
-                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
-            }
-            return std::nullopt;
-          },
-      },
-      property.u);
-}
-
-void OmpStructureChecker::CheckTraitSelectorList(
-    const std::list<parser::OmpTraitSelector> &traits) {
-  // [6.0:322:20]
-  // Each trait-selector-name may only be specified once in a trait selector
-  // set.
-
-  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
-  using TraitName = parser::OmpTraitSelectorName;
-  using BareName = decltype(TraitName::u);
-  std::map<BareName, const TraitName *> visited;
-
-  for (const parser::OmpTraitSelector &trait : traits) {
-    auto &name{std::get<TraitName>(trait.t)};
-
-    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait name %s in a trait set"_err_en_US, showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSetSelector(
-    const parser::OmpTraitSetSelector &traitSet) {
-
-  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
-  //
-  // Construct      |     Simd, directive-name |      Yes |       No |    No |
-  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
-  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
-  //                |      Extension, Requires |          |          |       |
-  //                |                   Vendor |          |          |       |
-  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
-  //                |                Kind, Uid |          |          |       |
-  // User           |                Condition |       No |       No |   Yes |
-
-  struct TraitSetConfig {
-    std::set<parser::OmpTraitSelectorName::Value> allowed;
-    bool allowsDirectiveTraits;
-    bool allowsExtensionTraits;
-    bool allowsScore;
-  };
-
-  using SName = parser::OmpTraitSetSelectorName::Value;
-  using TName = parser::OmpTraitSelectorName::Value;
-
-  static const std::map<SName, TraitSetConfig> configs{
-      {SName::Construct, //
-          {{TName::Simd}, true, false, false}},
-      {SName::Device, //
-          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
-      {SName::Implementation, //
-          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
-               TName::Vendor},
-              false, true, true}},
-      {SName::Target_Device, //
-          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
-               TName::Uid},
-              false, true, false}},
-      {SName::User, //
-          {{TName::Condition}, false, false, true}},
-  };
-
-  auto checkTraitSet{[&](const TraitSetConfig &config) {
-    auto &[setName, traits]{traitSet.t};
-    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
-
-    // Check if there are any duplicate traits.
-    CheckTraitSelectorList(traits);
-
-    for (const parser::OmpTraitSelector &trait : traits) {
-      // Don't use structured bindings here, because they cannot be captured
-      // before C++20.
-      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
-      auto &maybeProps =
-          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
-              trait.t);
-
-      // Check allowed traits
-      common::visit( //
-          common::visitors{
-              [&](parser::OmpTraitSelectorName::Value v) {
-                if (!config.allowed.count(v)) {
-                  context_.Say(traitName.source,
-                      "%s is not a valid trait for %s trait set"_err_en_US,
-                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
-                }
-              },
-              [&](llvm::omp::Directive) {
-                if (!config.allowsDirectiveTraits) {
-                  context_.Say(traitName.source,
-                      "Directive name is not a valid trait for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-              [&](const std::string &) {
-                if (!config.allowsExtensionTraits) {
-                  context_.Say(traitName.source,
-                      "Extension traits are not valid for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-          },
-          traitName.u);
-
-      // Check score
-      if (maybeProps) {
-        auto &[maybeScore, _]{maybeProps->t};
-        if (maybeScore) {
-          CheckTraitScore(*maybeScore);
-        }
-      }
-
-      // Check the properties of the individual traits
-      CheckTraitSelector(traitSet, trait);
-    }
-  }};
-
-  checkTraitSet(
-      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
-}
-
-void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
-  // [6.0:322:23]
-  // A score-expression must be a non-negative constant integer expression.
-  if (auto value{GetIntValue(score)}; !value || value < 0) {
-    context_.Say(score.source,
-        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
-  }
-}
-
-bool OmpStructureChecker::VerifyTraitPropertyLists(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  using PropertyList = std::list<parser::OmpTraitProperty>;
-  auto &[traitName, maybeProps]{trait.t};
-
-  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
-                             const std::string &message) {
-    bool foundInvalid{false};
-    for (const parser::OmpTraitProperty &prop : properties) {
-      if (!isValid(prop)) {
-        if (foundInvalid) {
-          context_.Say(
-              prop.source, "More invalid properties are present"_err_en_US);
-          break;
-        }
-        context_.Say(prop.source, "%s"_err_en_US, message);
-        foundInvalid = true;
-      }
-    }
-    return !foundInvalid;
-  }};
-
-  bool invalid{false};
-
-  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
-    // Directive-name traits don't have properties.
-    if (maybeProps) {
-      context_.Say(trait.source,
-          "Directive-name traits cannot have properties"_err_en_US);
-      invalid = true;
-    }
-  }
-  // Ignore properties on extension traits.
-
-  // See `TraitSelectorParser` in openmp-parser.cpp
-  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
-    switch (*v) {
-    // name-list properties
-    case parser::OmpTraitSelectorName::Value::Arch:
-    case parser::OmpTraitSelectorName::Value::Extension:
-    case parser::OmpTraitSelectorName::Value::Isa:
-    case parser::OmpTraitSelectorName::Value::Kind:
-    case parser::OmpTraitSelectorName::Value::Uid:
-    case parser::OmpTraitSelectorName::Value::Vendor:
-      if (maybeProps) {
-        auto isName{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isName, "Trait property should be a name");
-      }
-      break;
-    // clause-list
-    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
-    case parser::OmpTraitSelectorName::Value::Requires:
-    case parser::OmpTraitSelectorName::Value::Simd:
-      if (maybeProps) {
-        auto isClause{[&](const parser::OmpTraitProperty &prop) {
-          return GetClauseFromProperty(prop).has_value();
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isClause, "Trait property should be a clause");
-      }
-      break;
-    // expr-list
-    case parser::OmpTraitSelectorName::Value::Condition:
-    case parser::OmpTraitSelectorName::Value::Device_Num:
-      if (maybeProps) {
-        auto isExpr{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::ScalarExpr>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isExpr, "Trait property should be a scalar expression");
-      }
-      break;
-    } // switch
-  }
-
-  return !invalid;
-}
-
-void OmpStructureChecker::CheckTraitSelector(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  auto &[traitName, maybeProps]{trait.t};
-
-  // Only do the detailed checks if the property lists are valid.
-  if (VerifyTraitPropertyLists(traitSet, trait)) {
-    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
-        std::holds_alternative<std::string>(traitName.u)) {
-      // No properties here: directives don't have properties, and
-      // we don't implement any extension traits now.
-      return;
-    }
-
-    // Specific traits we want to check.
-    // Limitations:
-    // (1) The properties for these traits are defined in "Additional
-    // Definitions for the OpenMP API Specification". It's not clear how
-    // to define them in a portable way, and how to verify their validity,
-    // especially if they get replaced by their integer values (in case
-    // they are defined as enums).
-    // (2) These are entirely implementation-defined, and at the moment
-    // there is no known schema to validate these values.
-    auto v{std::get<TraitName::Value>(traitName.u)};
-    switch (v) {
-    case TraitName::Value::Arch:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Atomic_Default_Mem_Order:
-      CheckTraitADMO(traitSet, trait);
-      break;
-    case TraitName::Value::Condition:
-      CheckTraitCondition(traitSet, trait);
-      break;
-    case TraitName::Value::Device_Num:
-      CheckTraitDeviceNum(traitSet, trait);
-      break;
-    case TraitName::Value::Extension:
-      // Ignore
-      break;
-    case TraitName::Value::Isa:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Kind:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Requires:
-      CheckTraitRequires(traitSet, trait);
-      break;
-    case TraitName::Value::Simd:
-      CheckTraitSimd(traitSet, trait);
-      break;
-    case TraitName::Value::Uid:
-      // Unchecked, TBD(2)
-      break;
-    case TraitName::Value::Vendor:
-      // Unchecked, TBD(1)
-      break;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitADMO(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single clause property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto clauseId{*GetClauseFromProperty(property)};
-    // Check that the clause belongs to the memory-order clause-set.
-    // Clause sets will hopefully be autogenerated at some point.
-    switch (clauseId) {
-    case llvm::omp::Clause::OMPC_acq_rel:
-    case llvm::omp::Clause::OMPC_acquire:
-    case llvm::omp::Clause::OMPC_relaxed:
-    case llvm::omp::Clause::OMPC_release:
-    case llvm::omp::Clause::OMPC_seq_cst:
-      break;
-    default:
-      context_.Say(property.source,
-          "%s trait requires a clause from the memory-order clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitCondition(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
-
-    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
-    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
-      context_.Say(property.source,
-          "%s trait requires a single LOGICAL expression"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitDeviceNum(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  }
-  // No other checks at the moment.
-}
-
-void OmpStructureChecker::CheckTraitRequires(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_requires, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause from the requirement clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSimd(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_declare_simd, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()),
-          parser::ToUpperCaseLetters(
-              getDirectiveName(llvm::omp::OMPD_declare_simd)));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
 llvm::StringRef OmpStructureChecker::getClauseName(llvm::omp::Clause clause) {
   return llvm::omp::getOpenMPClauseName(clause);
 }
@@ -6519,39 +3880,6 @@ llvm::StringRef OmpStructureChecker::getDirectiveName(
   return llvm::omp::getOpenMPDirectiveName(directive, version);
 }
 
-const Symbol *OmpStructureChecker::GetObjectSymbol(
-    const parser::OmpObject &object) {
-  // Some symbols may be missing if the resolution failed, e.g. when an
-  // undeclared name is used with implicit none.
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    auto &last{GetLastName(*desg)};
-    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
-  }
-  return nullptr;
-}
-
-const Symbol *OmpStructureChecker::GetArgumentSymbol(
-    const parser::OmpArgument &argument) {
-  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
-    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
-      return GetObjectSymbol(*object);
-    }
-  }
-  return nullptr;
-}
-
-std::optional<parser::CharBlock> OmpStructureChecker::GetObjectSource(
-    const parser::OmpObject &object) {
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->source;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    return GetLastName(*desg).source;
-  }
-  return std::nullopt;
-}
-
 void OmpStructureChecker::CheckDependList(const parser::DataRef &d) {
   common::visit(
       common::visitors{
@@ -6861,7 +4189,7 @@ void OmpStructureChecker::CheckWorkshareBlockStmts(
 }
 
 void OmpStructureChecker::CheckIfContiguous(const parser::OmpObject &object) {
-  if (auto contig{IsContiguous(object)}; contig && !*contig) {
+  if (auto contig{IsContiguous(context_, object)}; contig && !*contig) {
     const parser::Name *name{GetObjectName(object)};
     assert(name && "Expecting name component");
     context_.Say(name->source,
@@ -6964,22 +4292,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::SelfMaps &x) {
   CheckAllowedRequiresClause(llvm::omp::Clause::OMPC_self_maps);
 }
 
-void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
-  Base::Enter(x);
-  loopStack_.push_back(&x);
-}
-
-void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top = loopStack_.back();
-  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
-  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-  Base::Leave(x);
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPInteropConstruct &x) {
   bool isDependClauseOccured{false};
   int targetCount{0}, targetSyncCount{0};
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index beb6e0528e814..2a3853335fd1c 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -162,10 +162,6 @@ class OmpStructureChecker
 
 private:
   bool CheckAllowedClause(llvmOmpClause clause);
-  bool IsVariableListItem(const Symbol &sym);
-  bool IsExtendedListItem(const Symbol &sym);
-  bool IsCommonBlock(const Symbol &sym);
-  std::optional<bool> IsContiguous(const parser::OmpObject &object);
   void CheckVariableListItem(const SymbolSourceMap &symbols);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
@@ -215,10 +211,6 @@ class OmpStructureChecker
       typename IterTy = decltype(std::declval<RangeTy>().begin())>
   std::optional<IterTy> FindDuplicate(RangeTy &&);
 
-  const Symbol *GetObjectSymbol(const parser::OmpObject &object);
-  const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
-  std::optional<parser::CharBlock> GetObjectSource(
-      const parser::OmpObject &object);
   void CheckDependList(const parser::DataRef &);
   void CheckDependArraySection(
       const common::Indirection<parser::ArrayElement> &, const parser::Name &);
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
new file mode 100644
index 0000000000000..055ff8e6b5a29
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -0,0 +1,389 @@
+//===-- lib/Semantics/openmp-utils.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Common/reference.h"
+#include "flang/Common/visit.h"
+#include "flang/Evaluate/check-expression.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
+#include "flang/Evaluate/variable.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/expression.h"
+#include "flang/Semantics/semantics.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+
+#include <optional>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics::omp {
+
+std::string ThisVersion(unsigned version) {
+  std::string tv{
+      std::to_string(version / 10) + "." + std::to_string(version % 10)};
+  return "OpenMP v" + tv;
+}
+
+std::string TryVersion(unsigned version) {
+  return "try -fopenmp-version=" + std::to_string(version);
+}
+
+const parser::Designator *GetDesignatorFromObj(
+    const parser::OmpObject &object) {
+  return std::get_if<parser::Designator>(&object.u);
+}
+
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object) {
+  if (auto *desg{GetDesignatorFromObj(object)}) {
+    return std::get_if<parser::DataRef>(&desg->u);
+  }
+  return nullptr;
+}
+
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object) {
+  if (auto *dataRef{GetDataRefFromObj(object)}) {
+    using ElementIndirection = common::Indirection<parser::ArrayElement>;
+    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
+      return &ind->value();
+    }
+  }
+  return nullptr;
+}
+
+const Symbol *GetObjectSymbol(const parser::OmpObject &object) {
+  // Some symbols may be missing if the resolution failed, e.g. when an
+  // undeclared name is used with implicit none.
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    auto &last{GetLastName(*desg)};
+    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
+  }
+  return nullptr;
+}
+
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument) {
+  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
+    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
+      return GetObjectSymbol(*object);
+    }
+  }
+  return nullptr;
+}
+
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object) {
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->source;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    return GetLastName(*desg).source;
+  }
+  return std::nullopt;
+}
+
+bool IsCommonBlock(const Symbol &sym) {
+  return sym.detailsIf<CommonBlockDetails>() != nullptr;
+}
+
+bool IsVariableListItem(const Symbol &sym) {
+  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
+}
+
+bool IsExtendedListItem(const Symbol &sym) {
+  return IsVariableListItem(sym) || sym.IsSubprogram();
+}
+
+bool IsVarOrFunctionRef(const MaybeExpr &expr) {
+  if (expr) {
+    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
+        evaluate::IsVariable(*expr);
+  } else {
+    return false;
+  }
+}
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
+  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
+  // ForwardOwningPointer           typedExpr
+  // `- GenericExprWrapper          ^.get()
+  //    `- std::optional<Expr>      ^->v
+  return typedExpr.get()->v;
+}
+
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr) {
+  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
+    return maybeExpr->GetType();
+  } else {
+    return std::nullopt;
+  }
+}
+
+namespace {
+struct ContiguousHelper {
+  ContiguousHelper(SemanticsContext &context)
+      : fctx_(context.foldingContext()) {}
+
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
+    return Visit(x.value());
+  }
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Reference<Contained> &x) {
+    return Visit(x.get());
+  }
+  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
+    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
+  }
+  template <typename T>
+  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
+    return common::visit(
+        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
+  }
+  template <typename T> std::optional<bool> Visit(const T &) {
+    // Everything else.
+    return std::nullopt;
+  }
+
+private:
+  evaluate::FoldingContext &fctx_;
+};
+} // namespace
+
+// Return values:
+// - std::optional<bool>{true} if the object is known to be contiguous
+// - std::optional<bool>{false} if the object is known not to be contiguous
+// - std::nullopt if the object contiguity cannot be determined
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object) {
+  return common::visit( //
+      common::visitors{
+          [&](const parser::Name &x) {
+            // Any member of a common block must be contiguous.
+            return std::optional<bool>{true};
+          },
+          [&](const parser::Designator &x) {
+            evaluate::ExpressionAnalyzer ea{semaCtx};
+            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
+              return ContiguousHelper{semaCtx}.Visit(*maybeExpr);
+            }
+            return std::optional<bool>{};
+          },
+      },
+      object.u);
+}
+
+struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
+                                 std::vector<SomeExpr>, false> {
+  using Result = std::vector<SomeExpr>;
+  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
+  DesignatorCollector() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <typename T> //
+  Result operator()(const evaluate::Designator<T> &x) const {
+    // Once in a designator, don't traverse it any further (i.e. only
+    // collect top-level designators).
+    auto copy{x};
+    return Result{AsGenericExpr(std::move(copy))};
+  }
+
+  template <typename... Rs> //
+  Result Combine(Result &&result, Rs &&...results) const {
+    Result v(std::move(result));
+    auto moveAppend{[](auto &accum, auto &&other) {
+      for (auto &&s : other) {
+        accum.push_back(std::move(s));
+      }
+    }};
+    (moveAppend(v, std::move(results)), ...);
+    return v;
+  }
+};
+
+struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
+  using Base = evaluate::AnyTraverse<VariableFinder>;
+  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
+
+  using Base::operator();
+
+  template <typename T>
+  bool operator()(const evaluate::Designator<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+  template <typename T>
+  bool operator()(const evaluate::FunctionRef<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+private:
+  const SomeExpr &var;
+};
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr) {
+  return DesignatorCollector{}(expr);
+}
+
+static bool HasCommonDesignatorSymbols(
+    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
+  // Compare the designators used in "other" with the designators whose
+  // symbols are given in baseSyms.
+  // This is a part of the check if these two expressions can access the same
+  // storage: if the designators used in them are different enough, then they
+  // will be assumed not to access the same memory.
+  //
+  // Consider an (array element) expression x%y(w%z), the corresponding symbol
+  // vector will be {x, y, w, z} (i.e. the symbols for these names).
+  // Check whether this exact sequence appears anywhere in any the symbol
+  // vector for "other". This will be true for x(y) and x(y+1), so this is
+  // not a sufficient condition, but can be used to eliminate candidates
+  // before doing more exhaustive checks.
+  //
+  // If any of the symbols in this sequence are function names, assume that
+  // there is no storage overlap, mostly because it would be impossible in
+  // general to determine what storage the function will access.
+  // Note: if f is pure, then two calls to f will access the same storage
+  // when called with the same arguments. This check is not done yet.
+
+  if (llvm::any_of(
+          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
+    // If there is a function symbol in the chain then we can't infer much
+    // about the accessed storage.
+    return false;
+  }
+
+  auto isSubsequence{// Is u a subsequence of v.
+      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
+        size_t us{u.size()}, vs{v.size()};
+        if (us > vs) {
+          return false;
+        }
+        for (size_t off{0}; off != vs - us + 1; ++off) {
+          bool same{true};
+          for (size_t i{0}; i != us; ++i) {
+            if (u[i] != v[off + i]) {
+              same = false;
+              break;
+            }
+          }
+          if (same) {
+            return true;
+          }
+        }
+        return false;
+      }};
+
+  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
+  return isSubsequence(baseSyms, otherSyms);
+}
+
+static bool HasCommonTopLevelDesignators(
+    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
+  // Compare designators directly as expressions. This will ensure
+  // that x(y) and x(y+1) are not flagged as overlapping, whereas
+  // the symbol vectors for both of these would be identical.
+  std::vector<SomeExpr> otherDsgs{GetAllDesignators(other)};
+
+  for (auto &s : baseDsgs) {
+    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
+      return true;
+    }
+  }
+  return false;
+}
+
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
+  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
+  std::vector<SomeExpr> baseDsgs{GetAllDesignators(base)};
+
+  for (const SomeExpr &expr : exprs) {
+    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
+      continue;
+    }
+    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
+      return &expr;
+    }
+  }
+  return nullptr;
+}
+
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
+  return VariableFinder{sub}(super);
+}
+
+// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
+// to separate cases where the source has something that looks like an
+// assignment, but is semantically wrong (diagnosed by general semantic
+// checks), and where the source has some other statement (which we want
+// to report as "should be an assignment").
+bool IsAssignment(const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return false;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+
+  return common::visit(
+      [](auto &&s) -> bool {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        return std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>;
+      },
+      x->u);
+}
+
+bool IsPointerAssignment(const evaluate::Assignment &x) {
+  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
+      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
+}
+
+/// parser::Block is a list of executable constructs, parser::BlockConstruct
+/// is Fortran's BLOCK/ENDBLOCK construct.
+/// Strip the outermost BlockConstructs, return the reference to the Block
+/// in the executable part of the innermost of the stripped constructs.
+/// Specifically, if the given `block` has a single entry (it's a list), and
+/// the entry is a BlockConstruct, get the Block contained within. Repeat
+/// this step as many times as possible.
+const parser::Block &GetInnermostExecPart(const parser::Block &block) {
+  const parser::Block *iter{&block};
+  while (iter->size() == 1) {
+    const parser::ExecutionPartConstruct &ep{iter->front()};
+    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
+      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
+      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
+        iter = &std::get<parser::Block>(bc->value().t);
+        continue;
+      }
+    }
+    break;
+  }
+  return *iter;
+}
+
+} // namespace Fortran::semantics::omp
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
new file mode 100644
index 0000000000000..fca0587a8bc84
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -0,0 +1,62 @@
+//===-- lib/Semantics/openmp-utils.h --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_OPENMP_UTILS_H
+#define FORTRAN_SEMANTICS_OPENMP_UTILS_H
+
+#include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include <optional>
+#include <string>
+
+namespace Fortran::semantics {
+class SemanticsContext;
+class Symbol;
+
+// Add this namespace to avoid potential conflicts
+namespace omp {
+std::string ThisVersion(unsigned version);
+std::string TryVersion(unsigned version);
+
+const parser::Designator *GetDesignatorFromObj(const parser::OmpObject &object);
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object);
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object);
+const Symbol *GetObjectSymbol(const parser::OmpObject &object);
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object);
+
+bool IsCommonBlock(const Symbol &sym);
+bool IsExtendedListItem(const Symbol &sym);
+bool IsVariableListItem(const Symbol &sym);
+bool IsVarOrFunctionRef(const MaybeExpr &expr);
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr);
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr);
+
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object);
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr);
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs);
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super);
+bool IsAssignment(const parser::ActionStmt *x);
+bool IsPointerAssignment(const evaluate::Assignment &x);
+const parser::Block &GetInnermostExecPart(const parser::Block &block);
+} // namespace omp
+} // namespace Fortran::semantics
+
+#endif // FORTRAN_SEMANTICS_OPENMP_UTILS_H

>From 2c3b718a05ee2b4e0e78ef45a6809dae6d04c912 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 08:12:57 -0500
Subject: [PATCH 2/4] Add short description to each added file

---
 flang/lib/Semantics/check-omp-atomic.cpp        | 5 +++++
 flang/lib/Semantics/check-omp-loop.cpp          | 5 +++++
 flang/lib/Semantics/check-omp-metadirective.cpp | 4 ++++
 flang/lib/Semantics/openmp-utils.cpp            | 4 ++++
 flang/lib/Semantics/openmp-utils.h              | 4 ++++
 5 files changed, 22 insertions(+)

diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index 19425a69523d3..e22c747621680 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -5,6 +5,11 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Semantic checks related to the ATOMIC construct.
+//
+//===----------------------------------------------------------------------===//
+
 
 #include "check-omp-structure.h"
 #include "openmp-utils.h"
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
index f2aad9efdd8de..ab40459f0e7bb 100644
--- a/flang/lib/Semantics/check-omp-loop.cpp
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -5,6 +5,11 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Semantic checks for constructs and clauses related to loops.
+//
+//===----------------------------------------------------------------------===//
+
 
 #include "check-omp-structure.h"
 
diff --git a/flang/lib/Semantics/check-omp-metadirective.cpp b/flang/lib/Semantics/check-omp-metadirective.cpp
index bf27797a7c44a..03487da64f1bf 100644
--- a/flang/lib/Semantics/check-omp-metadirective.cpp
+++ b/flang/lib/Semantics/check-omp-metadirective.cpp
@@ -5,6 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Semantic checks for METADIRECTIVE and related constructs/clauses.
+//
+//===----------------------------------------------------------------------===//
 
 #include "check-omp-structure.h"
 
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index 055ff8e6b5a29..fd9596a09cd52 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -5,6 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
 
 #include "openmp-utils.h"
 
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index fca0587a8bc84..dbb0565215357 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -5,6 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
 
 #ifndef FORTRAN_SEMANTICS_OPENMP_UTILS_H
 #define FORTRAN_SEMANTICS_OPENMP_UTILS_H

>From 420f15724336edadf3408cce236aaf388f2ac026 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 1 Jul 2025 08:23:19 -0500
Subject: [PATCH 3/4] Port part of 65cb0eae

---
 flang/lib/Semantics/check-omp-atomic.cpp |   1 -
 flang/lib/Semantics/check-omp-loop.cpp   | 101 +++++++++++++----------
 2 files changed, 56 insertions(+), 46 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index e22c747621680..047c604693460 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "check-omp-structure.h"
 #include "openmp-utils.h"
 
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
index ab40459f0e7bb..b82e2f7342d85 100644
--- a/flang/lib/Semantics/check-omp-loop.cpp
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "check-omp-structure.h"
 
 #include "check-directive-structure.h"
@@ -301,10 +300,13 @@ void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
   }
   SetLoopInfo(x);
 
-  if (const auto &doConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
-    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &doConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
+      CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+    }
   }
   CheckLoopItrVariableIsInt(x);
   CheckAssociatedLoopConstraints(x);
@@ -327,39 +329,45 @@ const parser::Name OmpStructureChecker::GetLoopIndex(
 }
 
 void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const parser::DoConstruct *loop{&*loopConstruct};
-    if (loop && loop->IsDoNormal()) {
-      const parser::Name &itrVal{GetLoopIndex(loop)};
-      SetLoopIv(itrVal.symbol);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const parser::DoConstruct *loop{&*loopConstruct};
+      if (loop && loop->IsDoNormal()) {
+        const parser::Name &itrVal{GetLoopIndex(loop)};
+        SetLoopIv(itrVal.symbol);
+      }
     }
   }
 }
 
 void OmpStructureChecker::CheckLoopItrVariableIsInt(
     const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
 
-    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-      if (loop->IsDoNormal()) {
-        const parser::Name &itrVal{GetLoopIndex(loop)};
-        if (itrVal.symbol) {
-          const auto *type{itrVal.symbol->GetType()};
-          if (!type->IsNumeric(TypeCategory::Integer)) {
-            context_.Say(itrVal.source,
-                "The DO loop iteration"
-                " variable must be of the type integer."_err_en_US,
-                itrVal.ToString());
+      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+        if (loop->IsDoNormal()) {
+          const parser::Name &itrVal{GetLoopIndex(loop)};
+          if (itrVal.symbol) {
+            const auto *type{itrVal.symbol->GetType()};
+            if (!type->IsNumeric(TypeCategory::Integer)) {
+              context_.Say(itrVal.source,
+                  "The DO loop iteration"
+                  " variable must be of the type integer."_err_en_US,
+                  itrVal.ToString());
+            }
           }
         }
+        // Get the next DoConstruct if block is not empty.
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                 : nullptr;
       }
-      // Get the next DoConstruct if block is not empty.
-      const auto &block{std::get<parser::Block>(loop->t)};
-      const auto it{block.begin()};
-      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                               : nullptr;
     }
   }
 }
@@ -427,25 +435,28 @@ void OmpStructureChecker::CheckDistLinear(
 
     // Match the loop index variables with the collected symbols from linear
     // clauses.
-    if (const auto &loopConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-        if (loop->IsDoNormal()) {
-          const parser::Name &itrVal{GetLoopIndex(loop)};
-          if (itrVal.symbol) {
-            // Remove the symbol from the collected set
-            indexVars.erase(&itrVal.symbol->GetUltimate());
-          }
-          collapseVal--;
-          if (collapseVal == 0) {
-            break;
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &loopConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+          if (loop->IsDoNormal()) {
+            const parser::Name &itrVal{GetLoopIndex(loop)};
+            if (itrVal.symbol) {
+              // Remove the symbol from the collected set
+              indexVars.erase(&itrVal.symbol->GetUltimate());
+            }
+            collapseVal--;
+            if (collapseVal == 0) {
+              break;
+            }
           }
+          // Get the next DoConstruct if block is not empty.
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                   : nullptr;
         }
-        // Get the next DoConstruct if block is not empty.
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                 : nullptr;
       }
     }
 

>From 265bdc3cf759f117070f511738888c0116f1ec92 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 1 Jul 2025 08:39:15 +0100
Subject: [PATCH 4/4] Port PR145917

---
 flang/include/flang/Parser/parse-tree.h       |   5 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  10 ++
 flang/lib/Parser/unparse.cpp                  |   3 +-
 flang/lib/Semantics/canonicalize-omp.cpp      |  97 +++++++++++++++--
 flang/lib/Semantics/resolve-directives.cpp    | 102 +++++++++++-------
 ...nested-loop-transformation-construct01.f90 |  20 ++++
 ...nested-loop-transformation-construct02.f90 |  20 ++++
 .../loop-transformation-construct01.f90       |  74 +++++++++++++
 .../loop-transformation-construct02.f90       |  85 +++++++++++++++
 .../loop-transformation-construct01.f90       | 100 +++++++++++++++++
 10 files changed, 464 insertions(+), 52 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
 create mode 100644 flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
 create mode 100644 flang/test/Parser/OpenMP/loop-transformation-construct01.f90
 create mode 100644 flang/test/Parser/OpenMP/loop-transformation-construct02.f90
 create mode 100644 flang/test/Semantics/OpenMP/loop-transformation-construct01.f90

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 61f97b855b0e5..7e752eeb4dfe4 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -267,6 +267,7 @@ struct AccEndCombinedDirective;
 struct OpenACCDeclarativeConstruct;
 struct OpenACCRoutineConstruct;
 struct OpenMPConstruct;
+struct OpenMPLoopConstruct;
 struct OpenMPDeclarativeConstruct;
 struct OmpEndLoopDirective;
 struct OmpMemoryOrderClause;
@@ -5021,11 +5022,13 @@ struct OpenMPBlockConstruct {
 };
 
 // OpenMP directives enclosing do loop
+using NestedConstruct =
+    std::variant<DoConstruct, common::Indirection<OpenMPLoopConstruct>>;
 struct OpenMPLoopConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPLoopConstruct);
   OpenMPLoopConstruct(OmpBeginLoopDirective &&a)
       : t({std::move(a), std::nullopt, std::nullopt}) {}
-  std::tuple<OmpBeginLoopDirective, std::optional<DoConstruct>,
+  std::tuple<OmpBeginLoopDirective, std::optional<NestedConstruct>,
       std::optional<OmpEndLoopDirective>>
       t;
 };
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 96daadfaf1b3b..0a56e888ac44b 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -3769,6 +3769,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   mlir::Location currentLocation =
       converter.genLocation(beginLoopDirective.source);
 
+  auto &optLoopCons =
+      std::get<std::optional<parser::NestedConstruct>>(loopConstruct.t);
+  if (optLoopCons.has_value()) {
+    if (auto *ompNestedLoopCons{
+            std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
+                &*optLoopCons)}) {
+      genOMP(converter, symTable, semaCtx, eval, ompNestedLoopCons->value());
+    }
+  }
+
   llvm::omp::Directive directive =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
   const parser::CharBlock &source =
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ed0f227fd5b98..903287515e559 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2926,7 +2926,8 @@ class UnparseVisitor {
     Walk(std::get<OmpBeginLoopDirective>(x.t));
     Put("\n");
     EndOpenMP();
-    Walk(std::get<std::optional<DoConstruct>>(x.t));
+    Walk(std::get<std::optional<std::variant<DoConstruct,
+            common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t));
     Walk(std::get<std::optional<OmpEndLoopDirective>>(x.t));
   }
   void Unparse(const BasedPointer &x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 5164f1dc6faab..1edcb376596b0 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -8,6 +8,7 @@
 
 #include "canonicalize-omp.h"
 #include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
 
 // After Loop Canonicalization, rewrite OpenMP parse tree to make OpenMP
 // Constructs more structured which provide explicit scopes for later
@@ -125,6 +126,16 @@ class CanonicalizationOfOmp {
     parser::Block::iterator nextIt;
     auto &beginDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
     auto &dir{std::get<parser::OmpLoopDirective>(beginDir.t)};
+    auto missingDoConstruct = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
+    auto tileUnrollError = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "If a loop construct has been fully unrolled, it cannot then be tiled"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
 
     nextIt = it;
     while (++nextIt != block.end()) {
@@ -135,31 +146,95 @@ class CanonicalizationOfOmp {
       if (auto *doCons{GetConstructIf<parser::DoConstruct>(*nextIt)}) {
         if (doCons->GetLoopControl()) {
           // move DoConstruct
-          std::get<std::optional<parser::DoConstruct>>(x.t) =
+          std::get<std::optional<std::variant<parser::DoConstruct,
+              common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t) =
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if (nextIt != block.end()) {
-            if (auto *endDir{
-                    GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
-              std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
-                  std::move(*endDir);
-              block.erase(nextIt);
-            }
+          if (auto *endDir{
+                  GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
+            std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                std::move(*endDir);
+            nextIt = block.erase(nextIt);
           }
         } else {
           messages_.Say(dir.source,
               "DO loop after the %s directive must have loop control"_err_en_US,
               parser::ToUpperCaseLetters(dir.source.ToString()));
         }
+      } else if (auto *ompLoopCons{
+                     GetOmpIf<parser::OpenMPLoopConstruct>(*nextIt)}) {
+        // We should allow UNROLL and TILE constructs to be inserted between an
+        // OpenMP Loop Construct and the DO loop itself
+        auto &nestedBeginDirective =
+            std::get<parser::OmpBeginLoopDirective>(ompLoopCons->t);
+        auto &nestedBeginLoopDirective =
+            std::get<parser::OmpLoopDirective>(nestedBeginDirective.t);
+        if ((nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll ||
+                nestedBeginLoopDirective.v ==
+                    llvm::omp::Directive::OMPD_tile) &&
+            !(nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll &&
+                dir.v == llvm::omp::Directive::OMPD_tile)) {
+          // iterate through the remaining block items to find the end directive
+          // for the unroll/tile directive.
+          parser::Block::iterator endIt;
+          endIt = nextIt;
+          while (endIt != block.end()) {
+            if (auto *endDir{
+                    GetConstructIf<parser::OmpEndLoopDirective>(*endIt)}) {
+              auto &endLoopDirective =
+                  std::get<parser::OmpLoopDirective>(endDir->t);
+              if (endLoopDirective.v == dir.v) {
+                std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                    std::move(*endDir);
+                endIt = block.erase(endIt);
+                continue;
+              }
+            }
+            ++endIt;
+          }
+          RewriteOpenMPLoopConstruct(*ompLoopCons, block, nextIt);
+          auto &ompLoop = std::get<std::optional<parser::NestedConstruct>>(x.t);
+          ompLoop =
+              std::optional<parser::NestedConstruct>{parser::NestedConstruct{
+                  common::Indirection{std::move(*ompLoopCons)}}};
+          nextIt = block.erase(nextIt);
+        } else if (nestedBeginLoopDirective.v ==
+                llvm::omp::Directive::OMPD_unroll &&
+            dir.v == llvm::omp::Directive::OMPD_tile) {
+          // if a loop has been unrolled, the user can not then tile that loop
+          // as it has been unrolled
+          parser::OmpClauseList &unrollClauseList{
+              std::get<parser::OmpClauseList>(nestedBeginDirective.t)};
+          if (unrollClauseList.v.empty()) {
+            // if the clause list is empty for an unroll construct, we assume
+            // the loop is being fully unrolled
+            tileUnrollError(dir, messages_);
+          } else {
+            // parse the clauses for the unroll directive to find the full
+            // clause
+            for (auto clause{unrollClauseList.v.begin()};
+                clause != unrollClauseList.v.end(); ++clause) {
+              if (clause->Id() == llvm::omp::OMPC_full) {
+                tileUnrollError(dir, messages_);
+              }
+            }
+          }
+        } else {
+          messages_.Say(nestedBeginLoopDirective.source,
+              "Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs"_err_en_US,
+              parser::ToUpperCaseLetters(
+                  nestedBeginLoopDirective.source.ToString()));
+        }
       } else {
-        messages_.Say(dir.source,
-            "A DO loop must follow the %s directive"_err_en_US,
-            parser::ToUpperCaseLetters(dir.source.ToString()));
+        missingDoConstruct(dir, messages_);
       }
       // If we get here, we either found a loop, or issued an error message.
       return;
     }
+    if (nextIt == block.end()) {
+      missingDoConstruct(dir, messages_);
+    }
   }
 
   void RewriteOmpAllocations(parser::ExecutionPart &body) {
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..8d741734601a7 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1796,10 +1796,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
 
   if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    if (const auto &doConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      if (doConstruct.value().IsDoWhile()) {
-        return true;
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &doConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        if (doConstruct->IsDoWhile()) {
+          return true;
+        }
       }
     }
   }
@@ -1962,48 +1965,69 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndexAndCheckLoopLevel(
   bool hasCollapseClause{
       clause ? (clause->Id() == llvm::omp::OMPC_collapse) : false};
 
-  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
-  if (outer.has_value()) {
-    for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-      if (loop->IsDoConcurrent()) {
-        // DO CONCURRENT is explicitly allowed for the LOOP construct so long as
-        // there isn't a COLLAPSE clause
-        if (isLoopConstruct) {
-          if (hasCollapseClause) {
-            // hasCollapseClause implies clause != nullptr
-            context_.Say(clause->source,
-                "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &outer{std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      for (const parser::DoConstruct *loop{&*outer}; loop && level > 0;
+          --level) {
+        if (loop->IsDoConcurrent()) {
+          // DO CONCURRENT is explicitly allowed for the LOOP construct so long
+          // as there isn't a COLLAPSE clause
+          if (isLoopConstruct) {
+            if (hasCollapseClause) {
+              // hasCollapseClause implies clause != nullptr
+              context_.Say(clause->source,
+                  "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+            }
+          } else {
+            auto &stmt =
+                std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
+            context_.Say(stmt.source,
+                "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
           }
-        } else {
-          auto &stmt =
-              std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
-          context_.Say(stmt.source,
-              "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
-        }
-      }
-      // go through all the nested do-loops and resolve index variables
-      const parser::Name *iv{GetLoopIndex(*loop)};
-      if (iv) {
-        if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
-          SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
-          iv->symbol = symbol; // adjust the symbol within region
-          AddToContextObjectWithDSA(*symbol, ivDSA);
         }
+        // go through all the nested do-loops and resolve index variables
+        const parser::Name *iv{GetLoopIndex(*loop)};
+        if (iv) {
+          if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
+            SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
+            iv->symbol = symbol; // adjust the symbol within region
+            AddToContextObjectWithDSA(*symbol, ivDSA);
+          }
 
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+        }
       }
+      CheckAssocLoopLevel(level, GetAssociatedClause());
+    } else if (const auto &loop{std::get_if<
+                   common::Indirection<parser::OpenMPLoopConstruct>>(
+                   &*optLoopCons)}) {
+      auto &beginDirective =
+          std::get<parser::OmpBeginLoopDirective>(loop->value().t);
+      auto &beginLoopDirective =
+          std::get<parser::OmpLoopDirective>(beginDirective.t);
+      if (beginLoopDirective.v != llvm::omp::Directive::OMPD_unroll &&
+          beginLoopDirective.v != llvm::omp::Directive::OMPD_tile) {
+        context_.Say(GetContext().directiveSource,
+            "Only UNROLL or TILE constructs are allowed between an OpenMP Loop Construct and a DO construct"_err_en_US,
+            parser::ToUpperCaseLetters(llvm::omp::getOpenMPDirectiveName(
+                GetContext().directive, version)
+                    .str()));
+      } else {
+        PrivatizeAssociatedLoopIndexAndCheckLoopLevel(loop->value());
+      }
+    } else {
+      context_.Say(GetContext().directiveSource,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
+                  .str()));
     }
-    CheckAssocLoopLevel(level, GetAssociatedClause());
-  } else {
-    context_.Say(GetContext().directiveSource,
-        "A DO loop must follow the %s directive"_err_en_US,
-        parser::ToUpperCaseLetters(
-            llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
-                .str()));
   }
 }
+
 void OmpAttributeVisitor::CheckAssocLoopLevel(
     std::int64_t level, const parser::OmpClause *clause) {
   if (clause && level != 0) {
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..a76e7e52100db
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for tile OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end do
+end subroutine
+
+!CHECK: not yet implemented: Unhandled loop directive (tile)
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
new file mode 100644
index 0000000000000..33b7c5a917619
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for unroll OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+program loop_transformation_construct
+  implicit none
+  integer, parameter :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do x = 1, I
+    y(x) = y(x) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end program loop_transformation_construct
+
+!CHECK: not yet implemented: Unhandled loop directive (unroll)
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct01.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..baffc2f6e2f1e
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,74 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with 1 nested loop.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct02.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
new file mode 100644
index 0000000000000..b50e7183841cc
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
@@ -0,0 +1,85 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with multiple nested loops.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP TILE
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END TILE
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90 b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..f718efc32aabf
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,100 @@
+! Testing the Semantics of nested Loop Transformation Constructs
+
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=51
+
+subroutine loop_transformation_construct1
+  implicit none
+
+  !$omp do
+  !ERROR: A DO loop must follow the UNROLL directive
+  !$omp unroll
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !$omp end tile
+  !$omp end do
+  !ERROR: The END TILE directive must follow the DO loop associated with the loop construct
+  !$omp end tile
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs
+  !$omp parallel do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct3
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !ERROR: A DO loop must follow the TILE directive
+  !$omp tile
+end subroutine
+
+subroutine loop_transformation_construct4
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll full
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct5
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct6
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  !$omp unroll partial(2)
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine


From flang-commits at lists.llvm.org  Tue Jul  1 06:58:15 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Tue, 01 Jul 2025 06:58:15 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
Message-ID: <llvm.org/llvm/llvm-project/pull/146532@github.com>


https://github.com/abidh created https://github.com/llvm/llvm-project/pull/146532

This is combination of https://github.com/llvm/llvm-project/pull/138149 and https://github.com/llvm/llvm-project/pull/138039 which were opened separately for ease of reviewing. Only other change is adjustments in 2 tests which have gone in since. 

There are `DeclareOp` present for the variables mapped into target region. That  allow us to generate debug information for them. But the `TargetOp` is still part of parent function and those variables get the parent function's `DISubprogram` as a scope.
    
In `OMPIRBuilder`, a new function is created for the `TargetOp`. We also create a new `DISubprogram` for it. All the variables that were in the target region now have to be updated to have the correct scope. This after the fact updating of
debug information becomes very difficult in certain cases. Take the example of variable arrays. The type of those arrays depend on the artificial `DILocalVariable`(s) which hold the size(s) of the array. This new function will now require that we generate the new variable and and new types. Similar issue exist for character type variables too.
    
To avoid this after the fact updating, this PR generates a `DISubprogramAttr` for the `TargetOp` while generating the debug info in `flang`. Then we don't need to generate a `DISubprogram` in `OMPIRBuilder`.  This change is made a bit more complicated by the the fact that in new scheme, the debug location already points to the new `DISubprogram` by     the time it reaches `convertOmpTarget`. But we need some code generation in the parent function so we have to carefully manage the debug locations.
    
This fixes issue `#134991`.

>From e476d57d80de8e121ec71c6bfe52f4b7cd2c0927 Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer at amd.com>
Date: Tue, 29 Apr 2025 12:30:42 +0100
Subject: [PATCH 1/3] [flang][debug] Generate DISubprogramAttr for
 omp::TargetOp.

There are DeclareOp present for the variables mapped into target region. That
allow us to generate debug information for them. Bu the TargetOp is still
part of parent function and those variables get the parent function's
DISubprogram as a scope.

In OMPIRBuilder, a new function is created for the TargetOp. We also create a
new DISubprogram for it. All the variables that were in the target region now
have to be updated to have the correct scope. This after the fact updating of
debug information becomes very difficult in certain cases. Take the example of
variable arrays. The type of those arrays depend on the artificial
DILocalVariable(s) which hold the size(s) of the array. This new function will
now require that we generate the new variable and and new types. Similar
issue exist for character type variables too.

To avoid this after the fact updating, this PR generates a DISubprogramAttr
for the TargetOp while generating the debug info in flang. This help us
avoid updating later.

This commit is flang side of the change.
---
 .../lib/Optimizer/Transforms/AddDebugInfo.cpp | 117 +++++++++++++++++-
 .../test/Transforms/debug-omp-target-op-1.fir |  40 ++++++
 .../test/Transforms/debug-omp-target-op-2.fir |  53 ++++++++
 3 files changed, 209 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Transforms/debug-omp-target-op-1.fir
 create mode 100644 flang/test/Transforms/debug-omp-target-op-2.fir

diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 8fa2f38818c02..6eb914e67fd54 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -35,6 +35,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -104,6 +105,37 @@ bool debugInfoIsAlreadySet(mlir::Location loc) {
   return false;
 }
 
+// Generates the name for the artificial DISubprogram that we are going to
+// generate for omp::TargetOp. Its logic is borrowed from
+// getTargetEntryUniqueInfo and
+// TargetRegionEntryInfo::getTargetRegionEntryFnName to generate the same name.
+// But even if there was a slight mismatch, it is not a problem because this
+// name is artificial and not important to debug experience.
+mlir::StringAttr getTargetFunctionName(mlir::MLIRContext *context,
+                                       mlir::Location Loc,
+                                       llvm::StringRef parentName) {
+  auto fileLoc = Loc->findInstanceOf<mlir::FileLineColLoc>();
+
+  assert(fileLoc && "No file found from location");
+  llvm::StringRef fileName = fileLoc.getFilename().getValue();
+
+  llvm::sys::fs::UniqueID id;
+  uint64_t line = fileLoc.getLine();
+  size_t fileId;
+  size_t deviceId;
+  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {
+    fileId = llvm::hash_value(fileName.str());
+    deviceId = 0xdeadf17e;
+  } else {
+    fileId = id.getFile();
+    deviceId = id.getDevice();
+  }
+  return mlir::StringAttr::get(
+      context,
+      std::string(llvm::formatv("__omp_offloading_{0:x-}_{1:x-}_{2}_l{3}",
+                                deviceId, fileId, parentName, line)));
+}
+
 } // namespace
 
 bool AddDebugInfoPass::createCommonBlockGlobal(
@@ -446,6 +478,79 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
                                   line - 1, false);
   }
 
+  auto addTargetOpDISP = [&](bool lineTableOnly,
+                             llvm::ArrayRef<mlir::LLVM::DINodeAttr> entities) {
+    // When we process the DeclareOp inside the OpenMP target region, all the
+    // variables get the DISubprogram of the parent function of the target op as
+    // the scope. In the codegen (to llvm ir), OpenMP target op results in the
+    // creation of a separate function. As the variables in the debug info have
+    // the DISubprogram of the parent function as the scope, the variables
+    // need to be updated at codegen time to avoid verification failures.
+
+    // This updating after the fact becomes more and more difficult when types
+    // are dependent on local variables like in the case of variable size arrays
+    // or string. We not only have to generate new variables but also new types.
+    // We can avoid this problem by generating a DISubprogramAttr here for the
+    // target op and make sure that all the variables inside the target region
+    // get the correct scope in the first place.
+    funcOp.walk([&](mlir::omp::TargetOp targetOp) {
+      unsigned line = getLineFromLoc(targetOp.getLoc());
+      mlir::StringAttr name =
+          getTargetFunctionName(context, targetOp.getLoc(), funcOp.getName());
+      mlir::LLVM::DISubprogramFlags flags =
+          mlir::LLVM::DISubprogramFlags::Definition |
+          mlir::LLVM::DISubprogramFlags::LocalToUnit;
+      if (isOptimized)
+        flags = flags | mlir::LLVM::DISubprogramFlags::Optimized;
+
+      mlir::DistinctAttr id =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
+      types.push_back(mlir::LLVM::DINullTypeAttr::get(context));
+      for (auto arg : targetOp.getRegion().getArguments()) {
+        auto tyAttr = typeGen.convertType(fir::unwrapRefType(arg.getType()),
+                                          fileAttr, cuAttr, /*declOp=*/nullptr);
+        types.push_back(tyAttr);
+      }
+      CC = llvm::dwarf::getCallingConvention("DW_CC_normal");
+      mlir::LLVM::DISubroutineTypeAttr spTy =
+          mlir::LLVM::DISubroutineTypeAttr::get(context, CC, types);
+      if (lineTableOnly) {
+        auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+            context, id, compilationUnit, Scope, name, name, funcFileAttr, line,
+            line, flags, spTy, /*retainedNodes=*/{}, /*annotations=*/{});
+        targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+        return;
+      }
+      mlir::DistinctAttr recId =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/true, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, /*retainedNodes=*/{},
+          /*annotations=*/{});
+
+      // Make sure that information about the imported modules is copied in the
+      // new function.
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> opEntities;
+      for (mlir::LLVM::DINodeAttr N : entities) {
+        if (auto entity = mlir::dyn_cast<mlir::LLVM::DIImportedEntityAttr>(N)) {
+          auto importedEntity = mlir::LLVM::DIImportedEntityAttr::get(
+              context, llvm::dwarf::DW_TAG_imported_module, spAttr,
+              entity.getEntity(), fileAttr, /*line=*/1, /*name=*/nullptr,
+              /*elements*/ {});
+          opEntities.push_back(importedEntity);
+        }
+      }
+
+      id = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/false, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, opEntities,
+          /*annotations=*/{});
+      targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+    });
+  };
+
   // Don't process variables if user asked for line tables only.
   if (debugLevel == mlir::LLVM::DIEmissionKind::LineTablesOnly) {
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
@@ -453,6 +558,7 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
         line, line, subprogramFlags, subTypeAttr, /*retainedNodes=*/{},
         /*annotations=*/{});
     funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+    addTargetOpDISP(/*lineTableOnly=*/true, /*entities=*/{});
     return;
   }
 
@@ -510,9 +616,18 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
       funcName, fullName, funcFileAttr, line, line, subprogramFlags,
       subTypeAttr, entities, /*annotations=*/{});
   funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+  addTargetOpDISP(/*lineTableOnly=*/false, entities);
 
   funcOp.walk([&](fir::cg::XDeclareOp declOp) {
-    handleDeclareOp(declOp, fileAttr, spAttr, typeGen, symbolTable);
+    mlir::LLVM::DISubprogramAttr spTy = spAttr;
+    if (auto tOp = declOp->getParentOfType<mlir::omp::TargetOp>()) {
+      if (auto fusedLoc = llvm::dyn_cast<mlir::FusedLoc>(tOp.getLoc())) {
+        if (auto sp = llvm::dyn_cast<mlir::LLVM::DISubprogramAttr>(
+                fusedLoc.getMetadata()))
+          spTy = sp;
+      }
+    }
+    handleDeclareOp(declOp, fileAttr, spTy, typeGen, symbolTable);
   });
   // commonBlockMap ensures that we don't create multiple DICommonBlockAttr of
   // the same name in one function. But it is ok (rather required) to create
diff --git a/flang/test/Transforms/debug-omp-target-op-1.fir b/flang/test/Transforms/debug-omp-target-op-1.fir
new file mode 100644
index 0000000000000..6b895b732c42b
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-1.fir
@@ -0,0 +1,40 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info="debug-level=LineTablesOnly" --mlir-print-debuginfo %s | FileCheck %s --check-prefix=LINETABLE
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %c13_i32 = arith.constant 13 : i32
+    %c12_i32 = arith.constant 12 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c1_i32 = arith.constant 1 : i32
+    %c5_i32 = arith.constant 5 : i32
+    %0 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} loc(#loc1)
+    %1 = fircg.ext_declare %0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc1)
+    %2 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} loc(#loc2)
+    %3 = fircg.ext_declare %2 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc2)
+    %4 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "x"}
+    %5 = omp.map.info var_ptr(%3 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "y"}
+    omp.target map_entries(%4 -> %arg0, %5 -> %arg1 : !fir.ref<i32>, !fir.ref<i32>) {
+      %16 = fircg.ext_declare %arg0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc3)
+      %17 = fircg.ext_declare %arg1 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc4)
+      omp.terminator
+    } loc(#loc5)
+    return
+  }
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// CHECK: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "x"{{.*}}line = 1, type = #[[TY:.*]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "y"{{.*}}line = 3, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "x"{{.*}}line = 7, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "y"{{.*}}line = 8, type = #[[TY]]>
+
+// LINETABLE: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// LINETABLE: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// LINETABLE-NOT: #llvm.di_local_variable
diff --git a/flang/test/Transforms/debug-omp-target-op-2.fir b/flang/test/Transforms/debug-omp-target-op-2.fir
new file mode 100644
index 0000000000000..15dcf2389b21d
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-2.fir
@@ -0,0 +1,53 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @fn_(%arg0: !fir.ref<!fir.array<?x?xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+    %c1 = arith.constant 1 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.alloca i32
+    %1 = fir.alloca i32
+    %2 = fir.undefined !fir.dscope
+    %3 = fircg.ext_declare %arg1 dummy_scope %2 {uniq_name = "_QFfnEc"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc2)
+    %4 = fircg.ext_declare %arg2 dummy_scope %2 {uniq_name = "_QFfnEd"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc3)
+    %5 = fir.load %3 : !fir.ref<i32>
+    %6 = fir.convert %5 : (i32) -> index
+    %9 = fir.load %4 : !fir.ref<i32>
+    %10 = fir.convert %9 : (i32) -> index
+    %15 = fircg.ext_declare %arg0(%6, %10) dummy_scope %2 {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc4)
+    %16 = fircg.ext_embox %15(%6, %10) : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.box<!fir.array<?x?xi32>>
+    %17:3 = fir.box_dims %16, %c0 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %18 = arith.subi %17#1, %c1 : index
+    %19 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%18 : index) extent(%17#1 : index) stride(%17#2 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %20 = arith.muli %17#2, %17#1 : index
+    %21:3 = fir.box_dims %16, %c1 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %22 = arith.subi %21#1, %c1 : index
+    %23 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%22 : index) extent(%21#1 : index) stride(%20 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %24 = omp.map.info var_ptr(%15 : !fir.ref<!fir.array<?x?xi32>>, i32) map_clauses(tofrom) capture(ByRef) bounds(%19, %23) -> !fir.ref<!fir.array<?x?xi32>> {name = "b"}
+    %25 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    %26 = omp.map.info var_ptr(%0 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    omp.target map_entries(%24 -> %arg3, %25 -> %arg4, %26 -> %arg5 : !fir.ref<!fir.array<?x?xi32>>, !fir.ref<i32>, !fir.ref<i32>) {
+      %27 = fir.load %arg5 : !fir.ref<i32>
+      %28 = fir.load %arg4 : !fir.ref<i32>
+      %29 = fir.convert %27 : (i32) -> index
+      %31 = fir.convert %28 : (i32) -> index
+      %37 = fircg.ext_declare %arg3(%29, %31) {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc5)
+      omp.terminator
+    } loc(#loc6)
+    return
+  } loc(#loc7)
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+#loc6 = loc("test.f90":16:1)
+#loc7 = loc("test.f90":26:1)
+
+
+// Test that variable size arrays inside target regions get their own
+// compiler generated variables for size.
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_fn__l16"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb1"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb2"{{.*}}>

>From 6980476ff9c1b8af2c22989035777166b0f6cd4e Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer at amd.com>
Date: Tue, 29 Apr 2025 12:31:00 +0100
Subject: [PATCH 2/3] [OpenMP] Improve debug info generation for outlined
 function.

In OMPIRBuilder, a new function is created for the TargetOp. We also
create a new DISubprogram for it. All the variables that were in the
target region now have to be updated to have the correct scope. This
after the fact updating of debug information becomes very difficult in
certain cases.

This PR is making the change that OMPIRBuilder will not generate a
DISubprogram. The responsibility has been shifted to the frontend. This
allows us to simplify the updating of debug records.

The commit is made a bit more complicated by the the fact that in new
scheme, the debug location already points to the new DISubprogram by
the time it reaches convertOmpTarget. But we need some code generation
in the parent function so we have to carefully manage the debug
locations.

This fixes issue `#134991`.
---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 46 ++++---------------
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 21 +++++++++
 .../Target/LLVMIR/omptarget-debug-empty.mlir  | 27 +++++++++++
 .../Target/LLVMIR/omptarget-debug-nowait.mlir |  5 +-
 .../Target/LLVMIR/omptarget-debug-var-1.mlir  | 13 ++++--
 .../Target/LLVMIR/omptarget-debug-var-2.mlir  | 13 ++++--
 mlir/test/Target/LLVMIR/omptarget-debug.mlir  |  8 +++-
 mlir/test/Target/LLVMIR/omptarget-debug2.mlir |  8 +++-
 .../LLVMIR/omptarget-parallel-llvm-debug.mlir |  8 +++-
 9 files changed, 94 insertions(+), 55 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 85451b1233f96..db792a3b52d24 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6891,23 +6891,19 @@ static void FixupDebugInfoForOutlinedFunction(
   if (!NewSP)
     return;
 
-  DenseMap<const MDNode *, MDNode *> Cache;
   SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
 
   auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
-    auto NewSP = Func->getSubprogram();
     DILocalVariable *&NewVar = RemappedVariables[OldVar];
     // Only use cached variable if the arg number matches. This is important
     // so that DIVariable created for privatized variables are not discarded.
     if (NewVar && (arg == NewVar->getArg()))
       return NewVar;
 
-    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
     NewVar = llvm::DILocalVariable::get(
-        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
-        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
-        OldVar->getAlignInBits(), OldVar->getAnnotations());
+        Builder.getContext(), OldVar->getScope(), OldVar->getName(),
+        OldVar->getFile(), OldVar->getLine(), OldVar->getType(), arg,
+        OldVar->getFlags(), OldVar->getAlignInBits(), OldVar->getAnnotations());
     return NewVar;
   };
 
@@ -6921,7 +6917,8 @@ static void FixupDebugInfoForOutlinedFunction(
         ArgNo = std::get<1>(Iter->second) + 1;
       }
     }
-    DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
+    if (ArgNo != 0)
+      DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
   };
 
   // The location and scope of variable intrinsics and records still point to
@@ -7000,36 +6997,9 @@ static Expected<Function *> createOutlinedFunction(
 
   // Save insert point.
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  // If there's a DISubprogram associated with current function, then
-  // generate one for the outlined function.
-  if (Function *ParentFunc = BB->getParent()) {
-    if (DISubprogram *SP = ParentFunc->getSubprogram()) {
-      DICompileUnit *CU = SP->getUnit();
-      DIBuilder DB(*M, true, CU);
-      DebugLoc DL = Builder.getCurrentDebugLocation();
-      if (DL) {
-        // TODO: We are using nullopt for arguments at the moment. This will
-        // need to be updated when debug data is being generated for variables.
-        DISubroutineType *Ty =
-            DB.createSubroutineType(DB.getOrCreateTypeArray({}));
-        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
-                                          DISubprogram::SPFlagOptimized |
-                                          DISubprogram::SPFlagLocalToUnit;
-
-        DISubprogram *OutlinedSP = DB.createFunction(
-            CU, FuncName, FuncName, SP->getFile(), DL.getLine(), Ty,
-            DL.getLine(), DINode::DIFlags::FlagArtificial, SPFlags);
-
-        // Attach subprogram to the function.
-        Func->setSubprogram(OutlinedSP);
-        // Update the CurrentDebugLocation in the builder so that right scope
-        // is used for things inside outlined function.
-        Builder.SetCurrentDebugLocation(
-            DILocation::get(Func->getContext(), DL.getLine(), DL.getCol(),
-                            OutlinedSP, DL.getInlinedAt()));
-      }
-    }
-  }
+  // We will generate the entries in the outlined function but the debug
+  // location may still be pointing to the parent function. Reset it now.
+  Builder.SetCurrentDebugLocation(llvm::DebugLoc());
 
   // Generate the region into the function.
   BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 3806db3ceab25..c1e1fec3ddef8 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5324,9 +5324,27 @@ static LogicalResult
 convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto targetOp = cast<omp::TargetOp>(opInst);
+  // The current debug location already has the DISubprogram for the outlined
+  // function that will be created for the target op. We save it here so that
+  // we can set it on the outlined function.
+  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
   if (failed(checkImplementationStatus(opInst)))
     return failure();
 
+  // During the handling of target op, we will generate instructions in the
+  // parent function like call to the oulined function or branch to a new
+  // BasicBlock. We set the debug location here to parent function so that those
+  // get the correct debug locations. For outlined functions, the normal MLIR op
+  // conversion will automatically pick the correct location.
+  llvm::BasicBlock *parentBB = builder.GetInsertBlock();
+  assert(parentBB && "No insert block is set for the builder");
+  llvm::Function *parentLLVMFn = parentBB->getParent();
+  assert(parentLLVMFn && "Parent Function must be valid");
+  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
+    builder.SetCurrentDebugLocation(llvm::DILocation::get(
+        parentLLVMFn->getContext(), outlinedFnLoc.getLine(),
+        outlinedFnLoc.getCol(), SP, outlinedFnLoc.getInlinedAt()));
+
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   bool isTargetDevice = ompBuilder->Config.isTargetDevice();
   bool isGPU = ompBuilder->Config.isGPU();
@@ -5420,6 +5438,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     assert(llvmParentFn && llvmOutlinedFn &&
            "Both parent and outlined functions must exist at this point");
 
+    if (outlinedFnLoc && llvmParentFn->getSubprogram())
+      llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());
+
     if (auto attr = llvmParentFn->getFnAttribute("target-cpu");
         attr.isStringAttribute())
       llvmOutlinedFn->addFnAttr(attr);
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
new file mode 100644
index 0000000000000..45e5d2612e2c2
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+module attributes {omp.is_target_device = false} {
+  llvm.func @test() {
+    omp.target {
+      omp.terminator
+    }  loc(#loc4)
+    llvm.return
+  }  loc(#loc3)
+}
+#file = #llvm.di_file<"target.f90" in "">
+#cu = #llvm.di_compile_unit<id = distinct[0]<>,
+ sourceLanguage = DW_LANG_Fortran95, file = #file, isOptimized = false,
+ emissionKind = Full>
+#sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
+#sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
+ name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
+#loc1 = loc("target.f90":1:1)
+#loc2 = loc("target.f90":46:3)
+#loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc2])
+
+// CHECK: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
+
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
index eaa88d9dd6053..3bd724f42e8ce 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
@@ -20,7 +20,7 @@ module attributes {omp.is_target_device = false} {
     ^bb3:  // pred: ^bb1
       llvm.store %13, %arg1 : i32, !llvm.ptr
       omp.terminator
-    }
+    } loc(#loc3)
     llvm.return
   } loc(#loc2)
 }
@@ -34,7 +34,10 @@ module attributes {omp.is_target_device = false} {
  types = #di_null_type>
 #sp = #llvm.di_subprogram<compileUnit = #cu, name = "main", file=#file,
  subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<compileUnit = #cu, name = "target", file=#file,
+ subprogramFlags = "Definition", type = #sp_ty>
 
 #loc1 = loc("test.f90":6:7)
 #loc2 = loc(fused<#sp>[#loc1])
+#loc3 = loc(fused<#sp1>[#loc1])
 
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
index ea92589bbd031..8f42995af23a8 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
@@ -19,11 +19,13 @@
 #g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
 #sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
   name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
-#var_arr = #llvm.di_local_variable<scope = #sp,
+#sp1 = #llvm.di_subprogram<id = distinct[3]<>, compileUnit = #cu, scope = #file,
+  name = "target", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp1,
   name = "arr", file = #file, line = 4, type = #array_ty>
-#var_i = #llvm.di_local_variable<scope = #sp,
+#var_i = #llvm.di_local_variable<scope = #sp1,
   name = "i", file = #file, line = 13, type = #int_ty>
-#var_x = #llvm.di_local_variable<scope = #sp,
+#var_x = #llvm.di_local_variable<scope = #sp1,
  name = "x", file = #file, line = 12, type = #real_ty>
 
 module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui32>>, llvm.target_triple = "amdgcn-amd-amdhsa", omp.is_target_device = true} {
@@ -47,7 +49,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
       llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
       omp.terminator
-    }
+    } loc(#loc5)
     llvm.return
   } loc(#loc3)
   llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
@@ -57,8 +59,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #loc2 = loc("target.f90":11:7)
 #loc3 = loc(fused<#sp>[#loc2])
 #loc4 = loc(fused<#g_var>[#loc1])
+#loc5 = loc(fused<#sp1>[#loc2])
 
-// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "target"{{.*}})
 // CHECK: !DILocalVariable(name: "dyn_ptr", arg: 1, scope: ![[SP]]{{.*}}flags: DIFlagArtificial)
 // CHECK: !DILocalVariable(name: "x", arg: 2, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "arr", arg: 3, scope: ![[SP]]{{.*}})
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
index 22db86fd85e2c..11a07dfd9a180 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
@@ -19,11 +19,13 @@
 #g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
 #sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
   name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
-#var_arr = #llvm.di_local_variable<scope = #sp,
+#sp1 = #llvm.di_subprogram<id = distinct[3]<>, compileUnit = #cu, scope = #file,
+  name = "target", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp1,
   name = "arr", file = #file, line = 4, type = #array_ty>
-#var_i = #llvm.di_local_variable<scope = #sp,
+#var_i = #llvm.di_local_variable<scope = #sp1,
   name = "i", file = #file, line = 13, type = #int_ty>
-#var_x = #llvm.di_local_variable<scope = #sp,
+#var_x = #llvm.di_local_variable<scope = #sp1,
  name = "x", file = #file, line = 12, type = #real_ty>
 
 module attributes {omp.is_target_device = false} {
@@ -45,7 +47,7 @@ module attributes {omp.is_target_device = false} {
       llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
       llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
       omp.terminator
-    }
+    }  loc(#loc5)
     llvm.return
   } loc(#loc3)
   llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
@@ -55,8 +57,9 @@ module attributes {omp.is_target_device = false} {
 #loc2 = loc("target.f90":11:7)
 #loc3 = loc(fused<#sp>[#loc2])
 #loc4 = loc(fused<#g_var>[#loc1])
+#loc5 = loc(fused<#sp1>[#loc2])
 
-// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "target"{{.*}})
 // CHECK: !DILocalVariable(name: "x", arg: 1, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "arr", arg: 2, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "i", arg: 3, scope: ![[SP]]{{.*}})
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug.mlir b/mlir/test/Target/LLVMIR/omptarget-debug.mlir
index 9c8344d69dc74..ab687f198b9b4 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug.mlir
@@ -10,7 +10,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       %13 = llvm.mlir.constant(1 : i32) : i32
       llvm.store %13, %arg0 : i32, !llvm.ptr loc(#loc2)
       omp.terminator
-    }
+    } loc(#loc4)
     llvm.return
   } loc(#loc3)
 }
@@ -21,9 +21,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
 #sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
  name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
 #loc1 = loc("target.f90":1:1)
 #loc2 = loc("target.f90":46:3)
 #loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc1])
 
-// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_{{.*}}"{{.*}})
+// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
 // CHECK-DAG: !DILocation(line: 46, column: 3, scope: ![[SP]])
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
index 78dc6e18a40a7..6cf75af38f916 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
@@ -11,7 +11,7 @@ module attributes {omp.is_target_device = false} {
       %13 = llvm.mlir.constant(1 : i32) : i32
       llvm.store %13, %arg0 : i32, !llvm.ptr loc(#loc2)
       omp.terminator
-    }
+    }  loc(#loc4)
     llvm.return
   } loc(#loc3)
 }
@@ -22,9 +22,13 @@ module attributes {omp.is_target_device = false} {
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
 #sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
  name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
 #loc1 = loc("target.f90":1:1)
 #loc2 = loc("target.f90":46:3)
 #loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc1])
 
-// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_{{.*}}"{{.*}})
+// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
 // CHECK-DAG: !DILocation(line: 46, column: 3, scope: ![[SP]])
diff --git a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
index 3c45f1f1c76fb..b18338ea35cc3 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
@@ -6,8 +6,10 @@
 #cu = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang", isOptimized = false, emissionKind = Full>
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 #sp = #llvm.di_subprogram<compileUnit = #cu, scope = #di_file, name = "test", file = #di_file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<compileUnit = #cu, scope = #di_file, name = "kernel", file = #di_file, subprogramFlags = "Definition", type = #sp_ty>
 #int_ty = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
 #var_x = #llvm.di_local_variable<scope = #sp, name = "x", file = #di_file, type = #int_ty>
+#var_x1 = #llvm.di_local_variable<scope = #sp1, name = "x", file = #di_file, type = #int_ty>
 module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, f128 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, f80 = dense<128> : vector<2xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr = dense<64> : vector<4xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, i1 = dense<8> : vector<2xi64>, i8 = dense<8> : vector<2xi64>, i16 = dense<16> : vector<2xi64>, "dlti.endianness" = "little", "dlti.stack_alignment" = 128 : i64, "dlti.mangling_mode" = "e">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", fir.target_cpu = "x86-64", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 21.0.0 (/home/haqadeer/work/src/aomp-llvm-project/flang 793f9220ab32f92fc3b253efec2e332c18090e53)", llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_gpu = false, omp.is_target_device = false, omp.requires = #omp<clause_requires none>, omp.target_triples = ["amdgcn-amd-amdhsa"], omp.version = #omp.version<version = 52>} {
   llvm.func @_QQmain() attributes {fir.bindc_name = "test", frame_pointer = #llvm.framePointerKind<all>, target_cpu = "x86-64"} {
     %0 = llvm.mlir.constant(1 : i64) : i64
@@ -16,7 +18,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>,
     %5 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "x"}
     omp.target map_entries(%5 -> %arg0 : !llvm.ptr) {
       %6 = llvm.mlir.constant(1 : i32) : i32
-      llvm.intr.dbg.declare #var_x = %arg0 : !llvm.ptr loc(#loc2)
+      llvm.intr.dbg.declare #var_x1 = %arg0 : !llvm.ptr loc(#loc3)
       omp.parallel {
         %7 = llvm.load %arg0 : !llvm.ptr -> i32
         %8 = llvm.add %7, %6 : i32
@@ -24,12 +26,14 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>,
         omp.terminator
       }
       omp.terminator
-    }
+    } loc(#loc4)
     llvm.return
   } loc(#loc10)
 }
 #loc1 = loc("target.f90":1:7)
 #loc2 = loc("target.f90":3:18)
+#loc3 = loc("target.f90":6:18)
+#loc4 = loc(fused<#sp1>[#loc3])
 #loc10 = loc(fused<#sp>[#loc1])
 
 

>From f5ecda01527f0dabe61a514a80918a637215e5e4 Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer at amd.com>
Date: Fri, 27 Jun 2025 12:25:04 +0100
Subject: [PATCH 3/3] Add di_subprogram to TargetOp in testcases.

---
 mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir     | 4 +++-
 mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
index a755cef98d7c4..c3320382f8d45 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
@@ -41,7 +41,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
         omp.terminator loc(#loc9)
       } loc(#loc9)
       omp.terminator loc(#loc9)
-    } loc(#loc9)
+    } loc(#loc15)
     llvm.return loc(#loc9)
   } loc(#loc14)
   llvm.mlir.global internal @_QFEarray() {addr_space = 0 : i32} : !llvm.array<16384 x i32> {
@@ -59,7 +59,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang", isOptimized = true, emissionKind = LineTablesOnly>
 #di_subroutine_type = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 #di_subprogram = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #di_compile_unit, scope = #di_file, name = "main", file = #di_file, subprogramFlags = "Definition|Optimized|MainSubprogram", type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit, name = "target", file = #di_file, subprogramFlags = "Definition", type = #di_subroutine_type>
 #loc14 = loc(fused<#di_subprogram>[#loc3])
+#loc15 = loc(fused<#di_subprogram1>[#loc9])
 
 
 // CHECK: call void @__kmpc_distribute_static{{.*}}!dbg
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
index 4495e1559a889..492610251769c 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
@@ -16,7 +16,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       %16 = llvm.load %arg1 : !llvm.ptr -> i32 loc(#loc5)
       llvm.store %16, %arg0 : i32, !llvm.ptr loc(#loc5)
       omp.terminator loc(#loc5)
-    } loc(#loc5)
+    } loc(#loc16)
     llvm.return loc(#loc6)
   } loc(#loc15)
 }
@@ -31,9 +31,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
   compileUnit = #di_compile_unit, scope = #di_file, name = "main",
   file = #di_file, subprogramFlags = "Definition|MainSubprogram",
   type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit,
+  name = "target", file = #di_file, subprogramFlags = "Definition",
+  type = #di_subroutine_type>
 #loc1 = loc("test.f90":3:18)
 #loc2 = loc("test.f90":7:7)
 #loc3 = loc("test.f90":9:18)
 #loc5 = loc("test.f90":11:7)
 #loc6 = loc("test.f90":12:7)
 #loc15 = loc(fused<#di_subprogram>[#loc2])
+#loc16 = loc(fused<#di_subprogram1>[#loc5])


From flang-commits at lists.llvm.org  Tue Jul  1 08:04:45 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:04:45 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add -emit-final-mlir flag (PR
	#146533)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146533@github.com>
Message-ID: <6863f90d.050a0220.01ed.952e@mx.google.com>


https://github.com/skatrak approved this pull request.

Thank you, LGTM!

https://github.com/llvm/llvm-project/pull/146533

From flang-commits at lists.llvm.org  Tue Jul  1 08:06:11 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 08:06:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
Message-ID: <llvm.org/llvm/llvm-project/pull/146543@github.com>


https://github.com/jeanPerier created https://github.com/llvm/llvm-project/pull/146543

The current DITypeAttr caching for derived type debug metadata generation strategy is not optimal. This turns out to be an issue for compile times in apps with very very complex derived types like [CP2K](https://github.com/cp2k/cp2k).

See the added debug-cyclic-derived-type-caching-simple.f90 test for more details about the duplication issue.

As a real world example justifying the new non trivial caching strategy, in CP2K, emitting debug type info for the [`swarm_worker_type` ](https://github.com/cp2k/cp2k/blob/c9ed8e0e9ce80a0104790ac3426f6468684713f5/src/swarm/swarm_worker.F#L41) caused 1,747,347 llvm debug metadata nodes to be emitted instead of 8023 after this patch (200x less) leading to noticeable compile time improvements (I measured 0.12s spent in `AddDebugInfo` pass instead of 7.5s prior to this patch).

The main idea is that caching is now associating to the cached DITypeAttr tree for a derived type a list of parent nodes being referred to recursively via indices in this DITypeAttr.

When leaving the context of a parent node, all types that were cached and linked to this parent node are cleared from the cache.
This allows more reusage in sub-trees while still fulfilling the MLIR requirements that DITypeAttr  types referring to a parent DITypeAttr  via integer id should only be used inside the DITypeAttr  of the parent.

Most of the complexity comes from computing the "list of parent nodes" by merging the ones from the components.

This is made is such a way that the extra cost for apps without recursive derived type is minimal because the extra data structure should not require extra dynamic allocations when they are no or little recursion.

>From 7830e4d137b8e62b469828ada053c2a79305fb48 Mon Sep 17 00:00:00 2001
From: Jean Perier <jperier at nvidia.com>
Date: Tue, 1 Jul 2025 07:32:04 -0700
Subject: [PATCH] [flang] improve DITypeAttr caching with recursive derived
 types

---
 .../Transforms/DebugTypeGenerator.cpp         | 236 ++++++++++--------
 .../Optimizer/Transforms/DebugTypeGenerator.h |  74 +++++-
 ...ug-cyclic-derived-type-caching-complex.f90 | 116 +++++++++
 ...bug-cyclic-derived-type-caching-simple.f90 |  39 +++
 4 files changed, 354 insertions(+), 111 deletions(-)
 create mode 100644 flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
 create mode 100644 flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index cdd30dce183dd..a848058486e2c 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -48,8 +48,7 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m,
                                        mlir::SymbolTable *symbolTable_,
                                        const mlir::DataLayout &dl)
     : module(m), symbolTable(symbolTable_), dataLayout{&dl},
-      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl),
-      derivedTypeDepth(0) {
+      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl) {
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 
   mlir::MLIRContext *context = module.getContext();
@@ -272,31 +271,127 @@ DebugTypeGenerator::getFieldSizeAndAlign(mlir::Type fieldTy) {
   return std::pair{byteSize, byteAlign};
 }
 
+mlir::LLVM::DITypeAttr DerivedTypeCache::lookup(mlir::Type type) {
+  auto iter = typeCache.find(type);
+  if (iter != typeCache.end()) {
+    if (iter->second.first) {
+      componentActiveRecursionLevels = iter->second.second;
+    }
+    return iter->second.first;
+  }
+  return nullptr;
+}
+
+DerivedTypeCache::ActiveLevels
+DerivedTypeCache::startTranslating(mlir::Type type,
+                                   mlir::LLVM::DITypeAttr placeHolder) {
+  derivedTypeDepth++;
+  if (!placeHolder)
+    return {};
+  typeCache[type] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      placeHolder, {derivedTypeDepth});
+  return {};
+}
+
+void DerivedTypeCache::preComponentVisitUpdate() {
+  componentActiveRecursionLevels.clear();
+}
+
+void DerivedTypeCache::postComponentVisitUpdate(
+    ActiveLevels &activeRecursionLevels) {
+  if (componentActiveRecursionLevels.empty())
+    return;
+  ActiveLevels oldLevels;
+  oldLevels.swap(activeRecursionLevels);
+  std::merge(componentActiveRecursionLevels.begin(),
+             componentActiveRecursionLevels.end(), oldLevels.begin(),
+             oldLevels.end(), std::back_inserter(activeRecursionLevels));
+}
+
+void DerivedTypeCache::finalize(mlir::Type ty, mlir::LLVM::DITypeAttr attr,
+                                ActiveLevels &&activeRecursionLevels) {
+  // If there is no nested recursion or if this type does not point to any type
+  // nodes above it, it is safe to cache it indefinitely (it can be used in any
+  // contexts).
+  if (activeRecursionLevels.empty() ||
+      (activeRecursionLevels[0] == derivedTypeDepth)) {
+    typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(attr, {});
+    componentActiveRecursionLevels.clear();
+    cleanUpCache(derivedTypeDepth);
+    --derivedTypeDepth;
+    return;
+  }
+  // Trim any recursion below the current type.
+  if (activeRecursionLevels.back() >= derivedTypeDepth) {
+    auto last = llvm::find_if(activeRecursionLevels, [&](std::int32_t depth) {
+      return depth >= derivedTypeDepth;
+    });
+    if (last != activeRecursionLevels.end()) {
+      activeRecursionLevels.erase(last, activeRecursionLevels.end());
+    }
+  }
+  componentActiveRecursionLevels = std::move(activeRecursionLevels);
+  typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      attr, componentActiveRecursionLevels);
+  cleanUpCache(derivedTypeDepth);
+  if (!componentActiveRecursionLevels.empty())
+    insertCacheCleanUp(ty, componentActiveRecursionLevels.back());
+  --derivedTypeDepth;
+}
+
+void DerivedTypeCache::insertCacheCleanUp(mlir::Type type, int32_t depth) {
+  auto iter = llvm::find_if(cacheCleanupList,
+                            [&](const auto &x) { return x.second >= depth; });
+  if (iter == cacheCleanupList.end()) {
+    cacheCleanupList.emplace_back(
+        std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+    return;
+  }
+  if (iter->second == depth) {
+    iter->first.push_back(type);
+    return;
+  }
+  cacheCleanupList.insert(
+      iter, std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+}
+
+void DerivedTypeCache::cleanUpCache(int32_t depth) {
+  if (cacheCleanupList.empty())
+    return;
+  // cleanups are done in the post actions when visiting a derived type
+  // tree. So if there is a clean-up for the current depth, it has to be
+  // the last one (deeper ones must have been done already).
+  if (cacheCleanupList.back().second == depth) {
+    for (mlir::Type type : cacheCleanupList.back().first)
+      typeCache[type].first = nullptr;
+    cacheCleanupList.pop_back_n(1);
+  }
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     fir::RecordType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
-  // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
 
-  bool canCacheThisType = true;
-  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
   mlir::MLIRContext *context = module.getContext();
-  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto [nameKind, sourceName] = fir::NameUniquer::deconstruct(Ty.getName());
+  if (nameKind != fir::NameUniquer::NameKind::DERIVED_TYPE)
+    return genPlaceholderType(context);
+
+  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   // Generate a place holder TypeAttr which will be used if a member
   // references the parent type.
-  auto comAttr = mlir::LLVM::DICompositeTypeAttr::get(
+  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto placeHolder = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/true, llvm::dwarf::DW_TAG_structure_type,
       mlir::StringAttr::get(context, ""), fileAttr, /*line=*/0, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = comAttr;
-
-  auto result = fir::NameUniquer::deconstruct(Ty.getName());
-  if (result.first != fir::NameUniquer::NameKind::DERIVED_TYPE)
-    return genPlaceholderType(context);
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty, placeHolder);
 
   fir::TypeInfoOp tiOp = symbolTable->lookup<fir::TypeInfoOp>(Ty.getName());
   unsigned line = (tiOp) ? getLineFromLoc(tiOp.getLoc()) : 1;
@@ -305,6 +400,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
   mlir::IntegerType intTy = mlir::IntegerType::get(context, 64);
   std::uint64_t offset = 0;
   for (auto [fieldName, fieldTy] : Ty.getTypeList()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     std::optional<llvm::ArrayRef<int64_t>> lowerBounds =
         fir::getComponentLowerBoundsIfNonDefault(Ty, fieldName, module,
@@ -317,7 +413,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     mlir::LLVM::DITypeAttr elemTy;
     if (lowerBounds && seqTy &&
         lowerBounds->size() == seqTy.getShape().size()) {
-      llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> arrayElements;
       for (auto [bound, dim] :
            llvm::zip_equal(*lowerBounds, seqTy.getShape())) {
         auto countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
@@ -325,14 +421,14 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
             context, countAttr, lowerAttr, /*upperBound=*/nullptr,
             /*stride=*/nullptr);
-        elements.push_back(subrangeTy);
+        arrayElements.push_back(subrangeTy);
       }
       elemTy = mlir::LLVM::DICompositeTypeAttr::get(
           context, llvm::dwarf::DW_TAG_array_type, /*name=*/nullptr,
           /*file=*/nullptr, /*line=*/0, /*scope=*/nullptr,
           convertType(seqTy.getEleTy(), fileAttr, scope, declOp),
           mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0, /*alignInBits=*/0,
-          elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
+          arrayElements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
           /*allocated=*/nullptr, /*associated=*/nullptr);
     } else
       elemTy = convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -344,80 +440,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
-
-    // Currently, the handling of recursive debug type in mlir has some
-    // limitations that were discussed at the end of the thread for following
-    // PR.
-    // https://github.com/llvm/llvm-project/pull/106571
-    //
-    // Problem could be explained with the following example code:
-    //  type t2
-    //   type(t1), pointer :: p1
-    // end type
-    // type t1
-    //   type(t2), pointer :: p2
-    // end type
-    // In the description below, type_self means a temporary type that is
-    // generated
-    // as a place holder while the members of that type are being processed.
-    //
-    // If we process t1 first then we will have the following structure after
-    // it has been processed.
-    // t1 -> t2 -> t1_self
-    // This is because when we started processing t2, we did not have the
-    // complete t1 but its place holder t1_self.
-    // Now if some entity requires t2, we will already have that in cache and
-    // will return it. But this t2 refers to t1_self and not to t1. In mlir
-    // handling, only those types are allowed to have _self reference which are
-    // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
-    // because the t1_self reference can be resolved by the outer t1. But
-    // standalone t2 is not because there will be no way to resolve it. Until
-    // this is fixed in mlir, we avoid caching such types. Please see
-    // DebugTranslation::translateRecursive for details on how mlir handles
-    // recursive types.
-    // The code below checks for situation where it will be unsafe to cache
-    // a type to avoid this problem. We do that in 2 situations.
-    // 1. If a member is record type, then its type would have been processed
-    // before reaching here. If it is not in the cache, it means that it was
-    // found to be unsafe to cache. So any type containing it will also not
-    // be cached
-    // 2. The type of the member is found in the cache but it is a place holder.
-    // In this case, its recID should match the recID of the type we are
-    // processing. This helps us to cache the following type.
-    // type t
-    //  type(t), allocatable :: p
-    // end type
-    mlir::Type baseTy = getDerivedType(fieldTy);
-    if (auto recTy = mlir::dyn_cast<fir::RecordType>(baseTy)) {
-      auto iter = typeCache.find(recTy);
-      if (iter == typeCache.end())
-        canCacheThisType = false;
-      else {
-        if (auto tyAttr =
-                mlir::dyn_cast<mlir::LLVM::DICompositeTypeAttr>(iter->second)) {
-          if (tyAttr.getIsRecSelf() && tyAttr.getRecId() != recId)
-            canCacheThisType = false;
-        }
-      }
-    }
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto finalAttr = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/false, llvm::dwarf::DW_TAG_structure_type,
-      mlir::StringAttr::get(context, result.second.name), fileAttr, line, scope,
+      mlir::StringAttr::get(context, sourceName.name), fileAttr, line, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
 
-  // derivedTypeDepth == 1 means that it is a top level type which is safe to
-  // cache.
-  if (canCacheThisType || derivedTypeDepth == 1) {
-    typeCache[Ty] = finalAttr;
-  } else {
-    auto iter = typeCache.find(Ty);
-    if (iter != typeCache.end())
-      typeCache.erase(iter);
-  }
+  derivedTypeCache.finalize(Ty, finalAttr, std::move(nestedRecursions));
+
   return finalAttr;
 }
 
@@ -425,15 +459,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
     mlir::TupleType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
   // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty);
 
   llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   mlir::MLIRContext *context = module.getContext();
 
   std::uint64_t offset = 0;
   for (auto fieldTy : Ty.getTypes()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     mlir::LLVM::DITypeAttr elemTy =
         convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -445,6 +482,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto typeAttr = mlir::LLVM::DICompositeTypeAttr::get(
@@ -453,7 +491,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = typeAttr;
+  derivedTypeCache.finalize(Ty, typeAttr, std::move(nestedRecursions));
   return typeAttr;
 }
 
@@ -667,27 +705,7 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
     return convertCharacterType(charTy, fileAttr, scope, declOp,
                                 /*hasDescriptor=*/false);
   } else if (auto recTy = mlir::dyn_cast_if_present<fir::RecordType>(Ty)) {
-    // For nested derived types like shown below, the call sequence of the
-    // convertRecordType will look something like as follows:
-    // convertRecordType (t1)
-    //  convertRecordType (t2)
-    //    convertRecordType (t3)
-    // We need to recognize when we are processing the top level type like t1
-    // to make caching decision. The variable `derivedTypeDepth` is used for
-    // this purpose and maintains the current depth of derived type processing.
-    //  type t1
-    //   type(t2), pointer :: p1
-    // end type
-    // type t2
-    //   type(t3), pointer :: p2
-    // end type
-    // type t2
-    //   integer a
-    // end type
-    derivedTypeDepth++;
-    auto result = convertRecordType(recTy, fileAttr, scope, declOp);
-    derivedTypeDepth--;
-    return result;
+    return convertRecordType(recTy, fileAttr, scope, declOp);
   } else if (auto tupleTy = mlir::dyn_cast_if_present<mlir::TupleType>(Ty)) {
     return convertTupleType(tupleTy, fileAttr, scope, declOp);
   } else if (auto refTy = mlir::dyn_cast_if_present<fir::ReferenceType>(Ty)) {
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index 93b9ac2d90fdf..854e4397ca32b 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -23,6 +23,77 @@
 
 namespace fir {
 
+/// Special cache to deal with the fact that mlir::LLVM::DITypeAttr for
+/// derived types may only be valid in specific nesting contexts in presence
+/// of derived type recursion and cannot be cached for the whole compilation.
+/// It is however still desirable to cache such mlir::LLVM::DITypeAttr as
+/// long as possible to avoid catastrophic compilation slow downs in very
+/// complex derived types where an intermediate type in a derived type cycle may
+/// indirectly appear hundreds of times under the top type of the derived type
+/// cycle. More details in the comment below.
+class DerivedTypeCache {
+public:
+  // Currently, the handling of recursive debug type in mlir has some
+  // limitations that were discussed at the end of the thread for following
+  // PR.
+  // https://github.com/llvm/llvm-project/pull/106571
+  //
+  // Problem could be explained with the following example code:
+  //  type t2
+  //   type(t1), pointer :: p1
+  // end type
+  // type t1
+  //   type(t2), pointer :: p2
+  // end type
+  // In the description below, type_self means a temporary type that is
+  // generated
+  // as a place holder while the members of that type are being processed.
+  //
+  // If we process t1 first then we will have the following structure after
+  // it has been processed.
+  // t1 -> t2 -> t1_self
+  // This is because when we started processing t2, we did not have the
+  // complete t1 but its place holder t1_self.
+  // Now if some entity requires t2, we will already have that in cache and
+  // will return it. But this t2 refers to t1_self and not to t1. In mlir
+  // handling, only those types are allowed to have _self reference which are
+  // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
+  // because the t1_self reference can be resolved by the outer t1. But
+  // standalone t2 is not because there will be no way to resolve it. Until
+  // this is fixed in mlir, we avoid caching such types. Please see
+  // DebugTranslation::translateRecursive for details on how mlir handles
+  // recursive types.
+  using ActiveLevels = llvm::SmallVector<int32_t, 1>;
+  mlir::LLVM::DITypeAttr lookup(mlir::Type);
+  ActiveLevels startTranslating(mlir::Type,
+                                mlir::LLVM::DITypeAttr placeHolder = nullptr);
+  void finalize(mlir::Type, mlir::LLVM::DITypeAttr, ActiveLevels &&);
+  void preComponentVisitUpdate();
+  void postComponentVisitUpdate(ActiveLevels &);
+
+private:
+  void insertCacheCleanUp(mlir::Type type, int32_t depth);
+  void cleanUpCache(int32_t depth);
+  // Current depth inside a top level derived type being converted.
+  int32_t derivedTypeDepth = 0;
+  // Cache for already translated derived types with the minimum depth where
+  // this cache entry is valid. Zero means the translation is always valid, "i"
+  // means the type depends its derived type tree parent node at depth "i". Such
+  // types should be cleaned-up from the cache in the post visit of node "i".
+  // Note that any new metadata created for a type with a component in the cache
+  // with validity of "i" shall not be added to the cache with a validity
+  // smaller than "i".
+  llvm::DenseMap<mlir::Type, std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>>
+      typeCache;
+  // List of parent nodes that are being recursively referred to in the
+  // component type that has just been computed.
+  ActiveLevels componentActiveRecursionLevels;
+  // Helper list that maintains the list of nodes that must be deleted from the
+  // cache when going back past listed parent depths.
+  llvm::SmallVector<std::pair<llvm::SmallVector<mlir::Type>, int32_t>>
+      cacheCleanupList;
+};
+
 /// This converts FIR/mlir type to DITypeAttr.
 class DebugTypeGenerator {
 public:
@@ -91,8 +162,7 @@ class DebugTypeGenerator {
   std::uint64_t lenOffset;
   std::uint64_t rankOffset;
   std::uint64_t rankSize;
-  int32_t derivedTypeDepth;
-  llvm::DenseMap<mlir::Type, mlir::LLVM::DITypeAttr> typeCache;
+  DerivedTypeCache derivedTypeCache;
 };
 
 } // namespace fir
diff --git a/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
new file mode 100644
index 0000000000000..72b7a78bef510
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
@@ -0,0 +1,116 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Test that debug metadata for derived types is not duplicated more than needed
+! when emitting debug info with non trivial cycles.
+
+
+! In the type graph below, G has a back edge to B, and F to D.
+! This causes C to be in the middle of B cycle, and E to be
+! both in B and D cycles.
+! C and E are used in several contexts, under B, under D, and outside
+! of it to test how metadata is generated for them.
+!
+! Without "local caching" of C and E when generating mlir::LLVM::DITypeAttr
+! for such derived types, many duplicate llvm metadata for the derived types
+! would be emitted, while with the right duplication of mlir::LLVM::DITypeAttr,
+! a lot more duplicate llvm IR metadata ends up emitted (19 DICompositeType
+! vs 71 before the patch that added this test).
+!
+!
+!  A -> B -> C -> D -> E -> F -> G -> B
+!  |    |         |         |
+!  |    |         |         | -> D
+!  |    |         |
+!  |    |         | -> H -> E
+!  |    |
+!  |    | -> I -> E
+!  |         | -> C
+!  |
+!  | -> C
+!  | -> E
+
+subroutine type_cycles_caching()
+  type g
+    type(b), pointer :: c_b
+  end type
+  type f
+    type(g) :: c_b
+    type(d), pointer :: c_d
+  end type
+  type e
+    type(f) :: c_f
+  end type
+    type h
+      ! Can reuse metadata of type(e) under 'd'.
+      type(e) :: c_e
+    end type
+  type d
+    type(e) :: c_e
+    type(h) :: c_h
+  end type
+  type c
+    type(d) :: c_d
+  end type
+    type i
+      ! Cannot reuse metadata of type(e) under 'd'.
+      type(e) :: c_e
+      ! Can reuse metadata of type(c) under 'b'.
+      type(c) :: c_c
+    end type
+  type b
+    type(c) :: c_c
+    type(i) :: c_i
+  end type
+  type a
+    type(b) :: c_b
+    ! Cannot reuse metadata of type(c) under 'b'
+    type(c) :: c_c
+    ! Cannot reuse metadata of type(e) under 'd', nor the one under 'b'
+    type(e) :: c_e
+  end type
+  type(a) :: xa
+  ! Can reuse metadata of type(c) for xa%c_c
+  type(c) :: xc
+  ! Can reuse metadata of type(c) for xa%c_e
+  type(e) :: xe
+  call bar(xa, xc, xe)
+end subroutine
+
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "a",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "b",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "c",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "d",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "g",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "h",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "i",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "c"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "d"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "g"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "h"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
diff --git a/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
new file mode 100644
index 0000000000000..c1f49d2f4a74a
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
@@ -0,0 +1,39 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Simple test that checks that metadata for `t0` is only duplicated once.
+
+! The difficulty is that at the mlir::LLVM::DITypeAttr, because of the
+! lack of MLIR attribute true recursion, the mlir::LLVM::DITypeAttr for
+! `t0` inside `t1` is special because it is only valid when found
+! in an mlir::LLVM::DITypeAttr tree under `base` (it will point to `base`
+! via an integer id that is only meanigfu;l when a node with such id has
+! been traversed).
+! A different node has to be created for `t0` usage in `x0` (will
+! point to the actual mlir::LLVM::DITypeAttr for `base` instead of
+! an integer id since the cycle is already "taken care of" in `base`
+! definition).
+! However, the same special `t0` node can be used for both `x1_1` and
+! `x1_2` components because they are both under `base` in the
+! mlir::LLVM::DITypeAttr tree definition. This used to not be the case,
+! leading to a lot of duplicate mlir::LLVM::DITypeAttr and actual LLVM IR
+! metadata, causing noticeable compilation slowdowns in apps with non trivial
+! derived types.
+
+subroutine duplicate_cycle_branch()
+  type t0
+    type(base), pointer :: x
+  end type
+  type t1
+    type(t0) :: x1_1
+    type(t0) :: x1_2
+  end type
+  type base
+    type(t1) :: x_base
+  end type
+  type(base) :: x
+  type(t0) :: x0
+  call bar(x, x0)
+end subroutine
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
+! CHECK-NOT: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",


From flang-commits at lists.llvm.org  Tue Jul  1 08:06:47 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 08:06:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146543@github.com>
Message-ID: <6863f987.050a0220.a756b.89a2@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: None (jeanPerier)

<details>
<summary>Changes</summary>

The current DITypeAttr caching for derived type debug metadata generation strategy is not optimal. This turns out to be an issue for compile times in apps with very very complex derived types like [CP2K](https://github.com/cp2k/cp2k).

See the added debug-cyclic-derived-type-caching-simple.f90 test for more details about the duplication issue.

As a real world example justifying the new non trivial caching strategy, in CP2K, emitting debug type info for the [`swarm_worker_type` ](https://github.com/cp2k/cp2k/blob/c9ed8e0e9ce80a0104790ac3426f6468684713f5/src/swarm/swarm_worker.F#L41) caused 1,747,347 llvm debug metadata nodes to be emitted instead of 8023 after this patch (200x less) leading to noticeable compile time improvements (I measured 0.12s spent in `AddDebugInfo` pass instead of 7.5s prior to this patch).

The main idea is that caching is now associating to the cached DITypeAttr tree for a derived type a list of parent nodes being referred to recursively via indices in this DITypeAttr.

When leaving the context of a parent node, all types that were cached and linked to this parent node are cleared from the cache.
This allows more reusage in sub-trees while still fulfilling the MLIR requirements that DITypeAttr  types referring to a parent DITypeAttr  via integer id should only be used inside the DITypeAttr  of the parent.

Most of the complexity comes from computing the "list of parent nodes" by merging the ones from the components.

This is made is such a way that the extra cost for apps without recursive derived type is minimal because the extra data structure should not require extra dynamic allocations when they are no or little recursion.

---

Patch is 25.80 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/146543.diff


4 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp (+127-109) 
- (modified) flang/lib/Optimizer/Transforms/DebugTypeGenerator.h (+72-2) 
- (added) flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90 (+116) 
- (added) flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90 (+39) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index cdd30dce183dd..a848058486e2c 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -48,8 +48,7 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m,
                                        mlir::SymbolTable *symbolTable_,
                                        const mlir::DataLayout &dl)
     : module(m), symbolTable(symbolTable_), dataLayout{&dl},
-      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl),
-      derivedTypeDepth(0) {
+      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl) {
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 
   mlir::MLIRContext *context = module.getContext();
@@ -272,31 +271,127 @@ DebugTypeGenerator::getFieldSizeAndAlign(mlir::Type fieldTy) {
   return std::pair{byteSize, byteAlign};
 }
 
+mlir::LLVM::DITypeAttr DerivedTypeCache::lookup(mlir::Type type) {
+  auto iter = typeCache.find(type);
+  if (iter != typeCache.end()) {
+    if (iter->second.first) {
+      componentActiveRecursionLevels = iter->second.second;
+    }
+    return iter->second.first;
+  }
+  return nullptr;
+}
+
+DerivedTypeCache::ActiveLevels
+DerivedTypeCache::startTranslating(mlir::Type type,
+                                   mlir::LLVM::DITypeAttr placeHolder) {
+  derivedTypeDepth++;
+  if (!placeHolder)
+    return {};
+  typeCache[type] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      placeHolder, {derivedTypeDepth});
+  return {};
+}
+
+void DerivedTypeCache::preComponentVisitUpdate() {
+  componentActiveRecursionLevels.clear();
+}
+
+void DerivedTypeCache::postComponentVisitUpdate(
+    ActiveLevels &activeRecursionLevels) {
+  if (componentActiveRecursionLevels.empty())
+    return;
+  ActiveLevels oldLevels;
+  oldLevels.swap(activeRecursionLevels);
+  std::merge(componentActiveRecursionLevels.begin(),
+             componentActiveRecursionLevels.end(), oldLevels.begin(),
+             oldLevels.end(), std::back_inserter(activeRecursionLevels));
+}
+
+void DerivedTypeCache::finalize(mlir::Type ty, mlir::LLVM::DITypeAttr attr,
+                                ActiveLevels &&activeRecursionLevels) {
+  // If there is no nested recursion or if this type does not point to any type
+  // nodes above it, it is safe to cache it indefinitely (it can be used in any
+  // contexts).
+  if (activeRecursionLevels.empty() ||
+      (activeRecursionLevels[0] == derivedTypeDepth)) {
+    typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(attr, {});
+    componentActiveRecursionLevels.clear();
+    cleanUpCache(derivedTypeDepth);
+    --derivedTypeDepth;
+    return;
+  }
+  // Trim any recursion below the current type.
+  if (activeRecursionLevels.back() >= derivedTypeDepth) {
+    auto last = llvm::find_if(activeRecursionLevels, [&](std::int32_t depth) {
+      return depth >= derivedTypeDepth;
+    });
+    if (last != activeRecursionLevels.end()) {
+      activeRecursionLevels.erase(last, activeRecursionLevels.end());
+    }
+  }
+  componentActiveRecursionLevels = std::move(activeRecursionLevels);
+  typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      attr, componentActiveRecursionLevels);
+  cleanUpCache(derivedTypeDepth);
+  if (!componentActiveRecursionLevels.empty())
+    insertCacheCleanUp(ty, componentActiveRecursionLevels.back());
+  --derivedTypeDepth;
+}
+
+void DerivedTypeCache::insertCacheCleanUp(mlir::Type type, int32_t depth) {
+  auto iter = llvm::find_if(cacheCleanupList,
+                            [&](const auto &x) { return x.second >= depth; });
+  if (iter == cacheCleanupList.end()) {
+    cacheCleanupList.emplace_back(
+        std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+    return;
+  }
+  if (iter->second == depth) {
+    iter->first.push_back(type);
+    return;
+  }
+  cacheCleanupList.insert(
+      iter, std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+}
+
+void DerivedTypeCache::cleanUpCache(int32_t depth) {
+  if (cacheCleanupList.empty())
+    return;
+  // cleanups are done in the post actions when visiting a derived type
+  // tree. So if there is a clean-up for the current depth, it has to be
+  // the last one (deeper ones must have been done already).
+  if (cacheCleanupList.back().second == depth) {
+    for (mlir::Type type : cacheCleanupList.back().first)
+      typeCache[type].first = nullptr;
+    cacheCleanupList.pop_back_n(1);
+  }
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     fir::RecordType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
-  // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
 
-  bool canCacheThisType = true;
-  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
   mlir::MLIRContext *context = module.getContext();
-  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto [nameKind, sourceName] = fir::NameUniquer::deconstruct(Ty.getName());
+  if (nameKind != fir::NameUniquer::NameKind::DERIVED_TYPE)
+    return genPlaceholderType(context);
+
+  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   // Generate a place holder TypeAttr which will be used if a member
   // references the parent type.
-  auto comAttr = mlir::LLVM::DICompositeTypeAttr::get(
+  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto placeHolder = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/true, llvm::dwarf::DW_TAG_structure_type,
       mlir::StringAttr::get(context, ""), fileAttr, /*line=*/0, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = comAttr;
-
-  auto result = fir::NameUniquer::deconstruct(Ty.getName());
-  if (result.first != fir::NameUniquer::NameKind::DERIVED_TYPE)
-    return genPlaceholderType(context);
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty, placeHolder);
 
   fir::TypeInfoOp tiOp = symbolTable->lookup<fir::TypeInfoOp>(Ty.getName());
   unsigned line = (tiOp) ? getLineFromLoc(tiOp.getLoc()) : 1;
@@ -305,6 +400,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
   mlir::IntegerType intTy = mlir::IntegerType::get(context, 64);
   std::uint64_t offset = 0;
   for (auto [fieldName, fieldTy] : Ty.getTypeList()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     std::optional<llvm::ArrayRef<int64_t>> lowerBounds =
         fir::getComponentLowerBoundsIfNonDefault(Ty, fieldName, module,
@@ -317,7 +413,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     mlir::LLVM::DITypeAttr elemTy;
     if (lowerBounds && seqTy &&
         lowerBounds->size() == seqTy.getShape().size()) {
-      llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> arrayElements;
       for (auto [bound, dim] :
            llvm::zip_equal(*lowerBounds, seqTy.getShape())) {
         auto countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
@@ -325,14 +421,14 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
             context, countAttr, lowerAttr, /*upperBound=*/nullptr,
             /*stride=*/nullptr);
-        elements.push_back(subrangeTy);
+        arrayElements.push_back(subrangeTy);
       }
       elemTy = mlir::LLVM::DICompositeTypeAttr::get(
           context, llvm::dwarf::DW_TAG_array_type, /*name=*/nullptr,
           /*file=*/nullptr, /*line=*/0, /*scope=*/nullptr,
           convertType(seqTy.getEleTy(), fileAttr, scope, declOp),
           mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0, /*alignInBits=*/0,
-          elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
+          arrayElements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
           /*allocated=*/nullptr, /*associated=*/nullptr);
     } else
       elemTy = convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -344,80 +440,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
-
-    // Currently, the handling of recursive debug type in mlir has some
-    // limitations that were discussed at the end of the thread for following
-    // PR.
-    // https://github.com/llvm/llvm-project/pull/106571
-    //
-    // Problem could be explained with the following example code:
-    //  type t2
-    //   type(t1), pointer :: p1
-    // end type
-    // type t1
-    //   type(t2), pointer :: p2
-    // end type
-    // In the description below, type_self means a temporary type that is
-    // generated
-    // as a place holder while the members of that type are being processed.
-    //
-    // If we process t1 first then we will have the following structure after
-    // it has been processed.
-    // t1 -> t2 -> t1_self
-    // This is because when we started processing t2, we did not have the
-    // complete t1 but its place holder t1_self.
-    // Now if some entity requires t2, we will already have that in cache and
-    // will return it. But this t2 refers to t1_self and not to t1. In mlir
-    // handling, only those types are allowed to have _self reference which are
-    // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
-    // because the t1_self reference can be resolved by the outer t1. But
-    // standalone t2 is not because there will be no way to resolve it. Until
-    // this is fixed in mlir, we avoid caching such types. Please see
-    // DebugTranslation::translateRecursive for details on how mlir handles
-    // recursive types.
-    // The code below checks for situation where it will be unsafe to cache
-    // a type to avoid this problem. We do that in 2 situations.
-    // 1. If a member is record type, then its type would have been processed
-    // before reaching here. If it is not in the cache, it means that it was
-    // found to be unsafe to cache. So any type containing it will also not
-    // be cached
-    // 2. The type of the member is found in the cache but it is a place holder.
-    // In this case, its recID should match the recID of the type we are
-    // processing. This helps us to cache the following type.
-    // type t
-    //  type(t), allocatable :: p
-    // end type
-    mlir::Type baseTy = getDerivedType(fieldTy);
-    if (auto recTy = mlir::dyn_cast<fir::RecordType>(baseTy)) {
-      auto iter = typeCache.find(recTy);
-      if (iter == typeCache.end())
-        canCacheThisType = false;
-      else {
-        if (auto tyAttr =
-                mlir::dyn_cast<mlir::LLVM::DICompositeTypeAttr>(iter->second)) {
-          if (tyAttr.getIsRecSelf() && tyAttr.getRecId() != recId)
-            canCacheThisType = false;
-        }
-      }
-    }
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto finalAttr = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/false, llvm::dwarf::DW_TAG_structure_type,
-      mlir::StringAttr::get(context, result.second.name), fileAttr, line, scope,
+      mlir::StringAttr::get(context, sourceName.name), fileAttr, line, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
 
-  // derivedTypeDepth == 1 means that it is a top level type which is safe to
-  // cache.
-  if (canCacheThisType || derivedTypeDepth == 1) {
-    typeCache[Ty] = finalAttr;
-  } else {
-    auto iter = typeCache.find(Ty);
-    if (iter != typeCache.end())
-      typeCache.erase(iter);
-  }
+  derivedTypeCache.finalize(Ty, finalAttr, std::move(nestedRecursions));
+
   return finalAttr;
 }
 
@@ -425,15 +459,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
     mlir::TupleType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
   // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty);
 
   llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   mlir::MLIRContext *context = module.getContext();
 
   std::uint64_t offset = 0;
   for (auto fieldTy : Ty.getTypes()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     mlir::LLVM::DITypeAttr elemTy =
         convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -445,6 +482,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto typeAttr = mlir::LLVM::DICompositeTypeAttr::get(
@@ -453,7 +491,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = typeAttr;
+  derivedTypeCache.finalize(Ty, typeAttr, std::move(nestedRecursions));
   return typeAttr;
 }
 
@@ -667,27 +705,7 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
     return convertCharacterType(charTy, fileAttr, scope, declOp,
                                 /*hasDescriptor=*/false);
   } else if (auto recTy = mlir::dyn_cast_if_present<fir::RecordType>(Ty)) {
-    // For nested derived types like shown below, the call sequence of the
-    // convertRecordType will look something like as follows:
-    // convertRecordType (t1)
-    //  convertRecordType (t2)
-    //    convertRecordType (t3)
-    // We need to recognize when we are processing the top level type like t1
-    // to make caching decision. The variable `derivedTypeDepth` is used for
-    // this purpose and maintains the current depth of derived type processing.
-    //  type t1
-    //   type(t2), pointer :: p1
-    // end type
-    // type t2
-    //   type(t3), pointer :: p2
-    // end type
-    // type t2
-    //   integer a
-    // end type
-    derivedTypeDepth++;
-    auto result = convertRecordType(recTy, fileAttr, scope, declOp);
-    derivedTypeDepth--;
-    return result;
+    return convertRecordType(recTy, fileAttr, scope, declOp);
   } else if (auto tupleTy = mlir::dyn_cast_if_present<mlir::TupleType>(Ty)) {
     return convertTupleType(tupleTy, fileAttr, scope, declOp);
   } else if (auto refTy = mlir::dyn_cast_if_present<fir::ReferenceType>(Ty)) {
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index 93b9ac2d90fdf..854e4397ca32b 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -23,6 +23,77 @@
 
 namespace fir {
 
+/// Special cache to deal with the fact that mlir::LLVM::DITypeAttr for
+/// derived types may only be valid in specific nesting contexts in presence
+/// of derived type recursion and cannot be cached for the whole compilation.
+/// It is however still desirable to cache such mlir::LLVM::DITypeAttr as
+/// long as possible to avoid catastrophic compilation slow downs in very
+/// complex derived types where an intermediate type in a derived type cycle may
+/// indirectly appear hundreds of times under the top type of the derived type
+/// cycle. More details in the comment below.
+class DerivedTypeCache {
+public:
+  // Currently, the handling of recursive debug type in mlir has some
+  // limitations that were discussed at the end of the thread for following
+  // PR.
+  // https://github.com/llvm/llvm-project/pull/106571
+  //
+  // Problem could be explained with the following example code:
+  //  type t2
+  //   type(t1), pointer :: p1
+  // end type
+  // type t1
+  //   type(t2), pointer :: p2
+  // end type
+  // In the description below, type_self means a temporary type that is
+  // generated
+  // as a place holder while the members of that type are being processed.
+  //
+  // If we process t1 first then we will have the following structure after
+  // it has been processed.
+  // t1 -> t2 -> t1_self
+  // This is because when we started processing t2, we did not have the
+  // complete t1 but its place holder t1_self.
+  // Now if some entity requires t2, we will already have that in cache and
+  // will return it. But this t2 refers to t1_self and not to t1. In mlir
+  // handling, only those types are allowed to have _self reference which are
+  // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
+  // because the t1_self reference can be resolved by the outer t1. But
+  // standalone t2 is not because there will be no way to resolve it. Until
+  // this is fixed in mlir, we avoid caching such types. Please see
+  // DebugTranslation::translateRecursive for details on how mlir handles
+  // recursive types.
+  using ActiveLevels = llvm::SmallVector<int32_t, 1>;
+  mlir::LLVM::DITypeAttr lookup(mlir::Type);
+  ActiveLevels startTranslating(mlir::Type,
+                                mlir::LLVM::DITypeAttr placeHolder = nullptr);
+  void finalize(mlir::Type, mlir::LLVM::DITypeAttr, ActiveLevels &&);
+  void preComponentVisitUpdate();
+  void postComponentVisitUpdate(ActiveLevels &);
+
+private:
+  void insertCacheCleanUp(mlir::Type type, int32_t depth);
+  void cleanUpCache(int32_t depth);
+  // Current depth inside a top level derived type being converted.
+  int32_t derivedTypeDepth = 0;
+  // Cache for already translated derived types with the minimum depth where
+  // this cache entry is valid. Zero means the translation is always valid, "i"
+  // means the type depends its derived type tree parent node at depth "i". Such
+  // types should be cleaned-up from the cache in the post visit of node "i".
+  // Note that any new metadata created for a type with a component in the cache
+  // with validity of "i" shall not be added to the cache with a validity
+  // smaller than "i".
+  llvm::DenseMap<mlir::Type, std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>>
+      typeCache;
+  // List of parent nodes that are being recursively referred to in the
+  // component type that has just been computed.
+  ActiveLevels componentActiveRecursionLevels;
+  // Helper list that maintains the list of nodes that must be deleted from the
+  // cache when going back past listed parent depths.
+  llvm::SmallVector<std::pair<llvm::SmallVector<mlir::Type>, int32_t>>
+      cacheCleanupList;
+};
+
 /// This converts FIR/mlir type to DITypeAttr.
 class DebugTypeGenerator {
 public:
@@ -91,8 +162,7 @@ class DebugTypeGenerator {
   std::uint64_t lenOffset;
   std::uint64_t rankOffset;
   std::uint64_t rankSize;
-  int32_t derivedTypeDepth;
-  llvm::DenseMap<mlir::Type, mlir::LLVM::DITypeAttr> typeCache;
+  DerivedTypeCache derivedTypeCache;
 };
 
 } // namespace fir
diff --git a/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
new file mode 100644
index 0000000000000..72b7a78bef510
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
@@ -0,0 +1,116 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Test that debug metadata for derived types is not duplicated more than needed
+! when emitting debug info with non trivial cycles.
+
+
+! In the type graph below, G has a back edge to B, and F to D.
+! This causes C to be in the middle of B cycle, and ...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/146543

From flang-commits at lists.llvm.org  Tue Jul  1 08:09:49 2025
From: flang-commits at lists.llvm.org (Kaviya Rajendiran via flang-commits)
Date: Tue, 01 Jul 2025 08:09:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add -emit-final-mlir flag (PR
	#146533)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146533@github.com>
Message-ID: <6863fa3d.050a0220.3aba87.a03e@mx.google.com>


kaviya2510 wrote:

Thanks for the patch, LGTM!

https://github.com/llvm/llvm-project/pull/146533

From flang-commits at lists.llvm.org  Tue Jul  1 08:12:39 2025
From: flang-commits at lists.llvm.org (Kaviya Rajendiran via flang-commits)
Date: Tue, 01 Jul 2025 08:12:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add -emit-final-mlir flag (PR
	#146533)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146533@github.com>
Message-ID: <6863fae7.630a0220.1635cf.04a1@mx.google.com>


https://github.com/kaviya2510 approved this pull request.


https://github.com/llvm/llvm-project/pull/146533

From flang-commits at lists.llvm.org  Tue Jul  1 08:22:40 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 08:22:40 -0700 (PDT)
Subject: [flang-commits] [flang] 69b69cb - [flang][tco] Add -emit-final-mlir
	flag (#146533)
Message-ID: <6863fd40.050a0220.a2691.c117@mx.google.com>


Author: Kajetan Puchalski
Date: 2025-07-01T16:22:36+01:00
New Revision: 69b69cbcb451ce48df24e1972e925fde2dcba511

URL: https://github.com/llvm/llvm-project/commit/69b69cbcb451ce48df24e1972e925fde2dcba511
DIFF: https://github.com/llvm/llvm-project/commit/69b69cbcb451ce48df24e1972e925fde2dcba511.diff

LOG: [flang][tco] Add -emit-final-mlir flag (#146533)

Add a flag to tco for emitting the final MLIR, prior to lowering to LLVM
IR. This is intended to produce output that can be passed directly to
mlir-translate.

---------

Signed-off-by: Kajetan Puchalski <kajetan.puchalski at arm.com>

Added: 
    flang/test/Driver/tco-emit-final-mlir.fir

Modified: 
    flang/tools/tco/tco.cpp

Removed: 
    


################################################################################
diff  --git a/flang/test/Driver/tco-emit-final-mlir.fir b/flang/test/Driver/tco-emit-final-mlir.fir
new file mode 100644
index 0000000000000..75f8f153127af
--- /dev/null
+++ b/flang/test/Driver/tco-emit-final-mlir.fir
@@ -0,0 +1,19 @@
+// Test tco's -emit-final-mlir option.
+
+// RUN: tco -emit-final-mlir %s 2>&1 | FileCheck %s
+
+// Check that the FIR file is translated into the LLVM IR
+// MLIR dialect, but not into LLVM IR itself.
+
+// CHECK-NOT: func.func
+// CHECK-LABEL: llvm.func @_QPfoo()
+// CHECK-NOT: fir.alloca
+// CHECK: %[[VAL_0:.*]] = llvm.mlir.constant(1 : i64) : i64
+// CHECK: %[[VAL_1:.*]] = llvm.alloca %[[VAL_0]] x i32 : (i64) -> !llvm.ptr
+// CHECK: llvm.return
+// CHECK-NOT: func.func
+
+func.func @_QPfoo() {
+  %1 = fir.alloca i32
+  return
+}

diff  --git a/flang/tools/tco/tco.cpp b/flang/tools/tco/tco.cpp
index eaf4bae088454..806cd3148d09f 100644
--- a/flang/tools/tco/tco.cpp
+++ b/flang/tools/tco/tco.cpp
@@ -70,6 +70,11 @@ static cl::opt<bool> codeGenLLVM(
     cl::desc("Run only CodeGen passes and translate FIR to LLVM IR"),
     cl::init(false));
 
+static cl::opt<bool> emitFinalMLIR(
+    "emit-final-mlir",
+    cl::desc("Only translate FIR to MLIR, do not lower to LLVM IR"),
+    cl::init(false));
+
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
 
@@ -149,13 +154,15 @@ compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
       fir::registerDefaultInlinerPass(config);
       fir::createMLIRToLLVMPassPipeline(pm, config);
     }
-    fir::addLLVMDialectToLLVMPass(pm, out.os());
+    if (!emitFinalMLIR)
+      fir::addLLVMDialectToLLVMPass(pm, out.os());
   }
 
   // run the pass manager
   if (mlir::succeeded(pm.run(*owningRef))) {
     // passes ran successfully, so keep the output
-    if ((emitFir || passPipeline.hasAnyOccurrences()) && !codeGenLLVM)
+    if ((emitFir || passPipeline.hasAnyOccurrences() || emitFinalMLIR) &&
+        !codeGenLLVM)
       printModule(*owningRef, out.os());
     out.keep();
     return mlir::success();


        

From flang-commits at lists.llvm.org  Tue Jul  1 08:22:43 2025
From: flang-commits at lists.llvm.org (Kajetan Puchalski via flang-commits)
Date: Tue, 01 Jul 2025 08:22:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add -emit-final-mlir flag (PR
	#146533)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146533@github.com>
Message-ID: <6863fd43.a70a0220.30810e.b461@mx.google.com>


https://github.com/mrkajetanp closed https://github.com/llvm/llvm-project/pull/146533

From flang-commits at lists.llvm.org  Tue Jul  1 08:42:38 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Tue, 01 Jul 2025 08:42:38 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Canonicalize divdc3 calls into
 arithmetic-based complex division (PR #146017)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146017@github.com>
Message-ID: <686401ee.170a0220.2e182f.1add@mx.google.com>


https://github.com/clementval commented:

Why not using the complex dialect directly? There has been discussion on the discourse about that.



https://github.com/llvm/llvm-project/pull/146017

From flang-commits at lists.llvm.org  Tue Jul  1 08:45:18 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 08:45:18 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Canonicalize divdc3 calls into
 arithmetic-based complex division (PR #146017)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146017@github.com>
Message-ID: <6864028e.170a0220.2b97a5.1ac0@mx.google.com>


jeanPerier wrote:

Hi @Hanwen-ece, thanks for the patch! I do have a preference using and improving mlir `complex.div` as discussed in https://discourse.llvm.org/t/optimization-of-complex-number-division/83468/18 instead of lowering the division directly at the FIR level because this duplicates complex div implementation and maintenance IMHO.

https://github.com/llvm/llvm-project/pull/146017

From flang-commits at lists.llvm.org  Tue Jul  1 08:46:23 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 08:46:23 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Canonicalize divdc3 calls into
 arithmetic-based complex division (PR #146017)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146017@github.com>
Message-ID: <686402cf.170a0220.2ad793.19f7@mx.google.com>


jeanPerier wrote:

Hi @Hanwen-ece, thanks for the patch! I do have a preference using and improving mlir `complex.div` as discussed in https://discourse.llvm.org/t/optimization-of-complex-number-division/83468/24 instead of lowering the division directly at the FIR level because this duplicates complex div implementation and maintenance IMHO.



https://github.com/llvm/llvm-project/pull/146017

From flang-commits at lists.llvm.org  Tue Jul  1 08:56:07 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 08:56:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Create unique reduction
 decls for different logical kinds (PR #146558)
Message-ID: <llvm.org/llvm/llvm-project/pull/146558@github.com>


https://github.com/tblah created https://github.com/llvm/llvm-project/pull/146558

Some Fujitsu tests showed incorrect results because we were sharing reduction declarations for different kinds for logical variables.

>From 14fa73cb0f758acf2a20a34d6f8c6f2f72f6c949 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Tue, 1 Jul 2025 15:47:50 +0000
Subject: [PATCH] [flang][OpenMP] Create new reduction delcls for different
 logical kinds

Some Fujitsu tests showed incorrect results because we were sharing
reduction declarations for different kinds for logical variables.
---
 flang/lib/Lower/OpenMP/ReductionProcessor.cpp | 20 ++++++++--------
 .../wsloop-reduction-logical-and-byref.f90    |  8 +++----
 .../OpenMP/wsloop-reduction-logical-and.f90   |  8 +++----
 .../wsloop-reduction-logical-eqv-byref.f90    |  8 +++----
 .../OpenMP/wsloop-reduction-logical-eqv.f90   |  8 +++----
 .../OpenMP/wsloop-reduction-logical-kinds.f90 | 24 +++++++++++++++++++
 .../wsloop-reduction-logical-neqv-byref.f90   |  8 +++----
 .../OpenMP/wsloop-reduction-logical-neqv.f90  |  8 +++----
 .../wsloop-reduction-logical-or-byref.f90     |  8 +++----
 .../OpenMP/wsloop-reduction-logical-or.f90    |  8 +++----
 10 files changed, 66 insertions(+), 42 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90

diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index d7f520e86e532..330cef7b54c74 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -149,13 +149,17 @@ std::string ReductionProcessor::getReductionName(
     reductionName = "multiply_reduction";
     break;
   case omp::clause::DefinedOperator::IntrinsicOperator::AND:
-    return "and_reduction";
+    reductionName = "and_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::EQV:
-    return "eqv_reduction";
+    reductionName = "eqv_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::OR:
-    return "or_reduction";
+    reductionName = "or_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:
-    return "neqv_reduction";
+    reductionName = "neqv_reduction";
+    break;
   default:
     reductionName = "other_reduction";
     break;
@@ -673,9 +677,6 @@ void ReductionProcessor::processReductionArguments(
     const auto &kindMap = firOpBuilder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
-    mlir::Type redNameTy = redType;
-    if (mlir::isa<fir::LogicalType>(redType.getEleTy()))
-      redNameTy = builder.getI1Type();
 
     if (const auto &redDefinedOp =
             std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
@@ -697,8 +698,7 @@ void ReductionProcessor::processReductionArguments(
         break;
       }
 
-      reductionName =
-          getReductionName(intrinsicOp, kindMap, redNameTy, isByRef);
+      reductionName = getReductionName(intrinsicOp, kindMap, redType, isByRef);
     } else if (const auto *reductionIntrinsic =
                    std::get_if<omp::clause::ProcedureDesignator>(
                        &redOperator.u)) {
@@ -709,7 +709,7 @@ void ReductionProcessor::processReductionArguments(
       redId = getReductionType(*reductionIntrinsic);
       reductionName =
           getReductionName(getRealName(*reductionIntrinsic).ToString(), kindMap,
-                           redNameTy, isByRef);
+                           redType, isByRef);
     } else {
       TODO(currentLocation, "Unexpected reduction type");
     }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 18e252a787d78..cf5c1b127c34d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -147,7 +147,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index 7f3e3922ab29b..eff97aa5d0db0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -139,7 +139,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index e7654edffec8b..bbe4bf04b6827 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -89,7 +89,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -146,7 +146,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index dffd6545b95bf..304c7eefd58b0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
new file mode 100644
index 0000000000000..f7fd2a52060e6
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
@@ -0,0 +1,24 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+! Check that logical reductions of different kinds do not end up using the same
+! reduction declaration
+
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l64 : !fir.logical<8> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
+
+subroutine test(a4, a8, sz)
+  integer :: sz
+  logical(4), dimension(sz) :: a4
+  logical(8), dimension(sz) :: a8
+
+  logical(4) :: res4 = .false.
+  logical(8) :: res8 = .false.
+  integer i
+
+! CHECK: omp.wsloop private({{.*}}) reduction(@or_reduction_l32 {{.*}}, @or_reduction_l64 {{.*}}) {
+  !$omp do reduction(.or.:res4, res8)
+  do i = 1,sz
+    res4 = res4 .or. a4(i)
+    res8 = res8 .or. a8(i)
+  enddo
+end subroutine
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index 366dd67591cfd..8c869952547c6 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @neqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @neqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -149,7 +149,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
index 66f624ff493fe..e0901bec6bc9c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @neqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @neqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant false
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -141,7 +141,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index a867d0a550bc7..bb09e2563e378 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @or_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -41,7 +41,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -88,7 +88,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -145,7 +145,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
index 592d4fa516bd7..5c9bcbf584d48 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @or_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant false
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)


From flang-commits at lists.llvm.org  Tue Jul  1 08:56:35 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 08:56:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Create unique reduction
 decls for different logical kinds (PR #146558)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146558@github.com>
Message-ID: <68640533.170a0220.22ac18.206d@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Tom Eccles (tblah)

<details>
<summary>Changes</summary>

Some Fujitsu tests showed incorrect results because we were sharing reduction declarations for different kinds for logical variables.

---

Patch is 37.46 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/146558.diff


10 Files Affected:

- (modified) flang/lib/Lower/OpenMP/ReductionProcessor.cpp (+10-10) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 (+4-4) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 (+4-4) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 (+4-4) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 (+4-4) 
- (added) flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90 (+24) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 (+4-4) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 (+4-4) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 (+4-4) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 (+4-4) 


``````````diff
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index d7f520e86e532..330cef7b54c74 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -149,13 +149,17 @@ std::string ReductionProcessor::getReductionName(
     reductionName = "multiply_reduction";
     break;
   case omp::clause::DefinedOperator::IntrinsicOperator::AND:
-    return "and_reduction";
+    reductionName = "and_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::EQV:
-    return "eqv_reduction";
+    reductionName = "eqv_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::OR:
-    return "or_reduction";
+    reductionName = "or_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:
-    return "neqv_reduction";
+    reductionName = "neqv_reduction";
+    break;
   default:
     reductionName = "other_reduction";
     break;
@@ -673,9 +677,6 @@ void ReductionProcessor::processReductionArguments(
     const auto &kindMap = firOpBuilder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
-    mlir::Type redNameTy = redType;
-    if (mlir::isa<fir::LogicalType>(redType.getEleTy()))
-      redNameTy = builder.getI1Type();
 
     if (const auto &redDefinedOp =
             std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
@@ -697,8 +698,7 @@ void ReductionProcessor::processReductionArguments(
         break;
       }
 
-      reductionName =
-          getReductionName(intrinsicOp, kindMap, redNameTy, isByRef);
+      reductionName = getReductionName(intrinsicOp, kindMap, redType, isByRef);
     } else if (const auto *reductionIntrinsic =
                    std::get_if<omp::clause::ProcedureDesignator>(
                        &redOperator.u)) {
@@ -709,7 +709,7 @@ void ReductionProcessor::processReductionArguments(
       redId = getReductionType(*reductionIntrinsic);
       reductionName =
           getReductionName(getRealName(*reductionIntrinsic).ToString(), kindMap,
-                           redNameTy, isByRef);
+                           redType, isByRef);
     } else {
       TODO(currentLocation, "Unexpected reduction type");
     }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 18e252a787d78..cf5c1b127c34d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -147,7 +147,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index 7f3e3922ab29b..eff97aa5d0db0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -139,7 +139,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index e7654edffec8b..bbe4bf04b6827 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -89,7 +89,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -146,7 +146,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index dffd6545b95bf..304c7eefd58b0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
new file mode 100644
index 0000000000000..f7fd2a52060e6
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
@@ -0,0 +1,24 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+! Check that logical reductions of different kinds do not end up using the same
+! reduction declaration
+
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l64 : !fir.logical<8> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
+
+subroutine test(a4, a8, sz)
+  integer :: sz
+  logical(4)...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/146558

From flang-commits at lists.llvm.org  Tue Jul  1 09:01:23 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 01 Jul 2025 09:01:23 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (PR #146359)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146359@github.com>
Message-ID: <68640653.630a0220.166f9c.181d@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks for checking!

https://github.com/llvm/llvm-project/pull/146359

From flang-commits at lists.llvm.org  Tue Jul  1 08:59:56 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Tue, 01 Jul 2025 08:59:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Create unique reduction
 decls for different logical kinds (PR #146558)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146558@github.com>
Message-ID: <686405fc.050a0220.36532f.b826@mx.google.com>


https://github.com/skatrak approved this pull request.

Thank you, this LGTM.

https://github.com/llvm/llvm-project/pull/146558

From flang-commits at lists.llvm.org  Tue Jul  1 09:12:08 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 01 Jul 2025 09:12:08 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (PR #146359)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146359@github.com>
Message-ID: <686408d8.630a0220.3639aa.2754@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/146359

From flang-commits at lists.llvm.org  Tue Jul  1 10:34:48 2025
From: flang-commits at lists.llvm.org (Kazu Hirata via flang-commits)
Date: Tue, 01 Jul 2025 10:34:48 -0700 (PDT)
Subject: [flang-commits] [flang] d79c284 - [flang] Fix a warning
Message-ID: <68641c38.630a0220.2be48.3d0d@mx.google.com>


Author: Kazu Hirata
Date: 2025-07-01T10:34:41-07:00
New Revision: d79c284a2fef5f9cb6b803b3d0cb4c1f857ece03

URL: https://github.com/llvm/llvm-project/commit/d79c284a2fef5f9cb6b803b3d0cb4c1f857ece03
DIFF: https://github.com/llvm/llvm-project/commit/d79c284a2fef5f9cb6b803b3d0cb4c1f857ece03.diff

LOG: [flang] Fix a warning

This patch fixes:

  flang/lib/Semantics/check-omp-structure.cpp:1747:41: error: lambda
  capture 'this' is not used [-Werror,-Wunused-lambda-capture]

Added: 
    

Modified: 
    flang/lib/Semantics/check-omp-structure.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 30eff01256c61..aa4cdb9d27019 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -1744,7 +1744,7 @@ void OmpStructureChecker::Enter(const parser::OpenMPFlushConstruct &x) {
 void OmpStructureChecker::Leave(const parser::OpenMPFlushConstruct &x) {
   auto &flushList{std::get<std::optional<parser::OmpArgumentList>>(x.v.t)};
 
-  auto isVariableListItemOrCommonBlock{[this](const Symbol &sym) {
+  auto isVariableListItemOrCommonBlock{[](const Symbol &sym) {
     return IsVariableListItem(sym) ||
         sym.detailsIf<semantics::CommonBlockDetails>();
   }};


        

From flang-commits at lists.llvm.org  Tue Jul  1 13:08:56 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Tue, 01 Jul 2025 13:08:56 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 '--offload-targets' for `-fopenmp-targets` (PR #146594)
Message-ID: <llvm.org/llvm/llvm-project/pull/146594@github.com>


https://github.com/jhuber6 created https://github.com/llvm/llvm-project/pull/146594

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.


>From 8dd774f8f7594e6bfbc1d970d79fbdb79c45e267 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Mon, 3 Feb 2025 12:46:27 -0600
Subject: [PATCH] [Clang] Introduce '--offload-targets' for `-fopenmp-targets`

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.
---
 clang/include/clang/Basic/DiagnosticDriverKinds.td | 2 ++
 clang/include/clang/Driver/Options.td              | 6 +++++-
 clang/lib/Driver/Driver.cpp                        | 8 ++++----
 clang/lib/Driver/ToolChain.cpp                     | 2 +-
 clang/lib/Driver/ToolChains/CommonArgs.cpp         | 2 +-
 clang/lib/Frontend/CompilerInvocation.cpp          | 6 +++---
 clang/test/Driver/amdgpu-openmp-sanitize-options.c | 4 ++--
 clang/test/Driver/hip-options.hip                  | 2 +-
 flang/lib/Frontend/CompilerInvocation.cpp          | 2 +-
 9 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 34b6c0d7a8acd..759ba0419bd45 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -116,6 +116,8 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
+def err_drv_mix_offload : Error<
+  "mixed %0 and %1 offloading compilation is not supported">;
 def err_drv_bad_target_id : Error<
   "invalid target ID '%0'; format is a processor name followed by an optional "
   "colon-delimited list of features followed by an enable/disable sign (e.g., "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..f359df55e0508 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : Joined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
 def offload_arch_EQ : Joined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
@@ -3678,7 +3682,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 775f3f029861c..930a678aa8923 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -991,7 +991,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1025,7 +1025,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1041,7 +1041,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1122,7 +1122,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..bdd77ac84913c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1404,7 +1404,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index f366e90945dac..c3fa27d9ff6d0 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3870,7 +3870,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4283,7 +4283,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4343,7 +4343,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..8e4c210306cb2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1248,7 +1248,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())


From flang-commits at lists.llvm.org  Tue Jul  1 13:09:28 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 13:09:28 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 '--offload-targets' for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <68644078.170a0220.140c11.58d4@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-clang

Author: Joseph Huber (jhuber6)

<details>
<summary>Changes</summary>

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.


---
Full diff: https://github.com/llvm/llvm-project/pull/146594.diff


9 Files Affected:

- (modified) clang/include/clang/Basic/DiagnosticDriverKinds.td (+2) 
- (modified) clang/include/clang/Driver/Options.td (+5-1) 
- (modified) clang/lib/Driver/Driver.cpp (+4-4) 
- (modified) clang/lib/Driver/ToolChain.cpp (+1-1) 
- (modified) clang/lib/Driver/ToolChains/CommonArgs.cpp (+1-1) 
- (modified) clang/lib/Frontend/CompilerInvocation.cpp (+3-3) 
- (modified) clang/test/Driver/amdgpu-openmp-sanitize-options.c (+2-2) 
- (modified) clang/test/Driver/hip-options.hip (+1-1) 
- (modified) flang/lib/Frontend/CompilerInvocation.cpp (+1-1) 


``````````diff
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 34b6c0d7a8acd..759ba0419bd45 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -116,6 +116,8 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
+def err_drv_mix_offload : Error<
+  "mixed %0 and %1 offloading compilation is not supported">;
 def err_drv_bad_target_id : Error<
   "invalid target ID '%0'; format is a processor name followed by an optional "
   "colon-delimited list of features followed by an enable/disable sign (e.g., "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..f359df55e0508 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : Joined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
 def offload_arch_EQ : Joined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
@@ -3678,7 +3682,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 775f3f029861c..930a678aa8923 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -991,7 +991,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1025,7 +1025,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1041,7 +1041,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1122,7 +1122,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..bdd77ac84913c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1404,7 +1404,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index f366e90945dac..c3fa27d9ff6d0 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3870,7 +3870,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4283,7 +4283,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4343,7 +4343,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..8e4c210306cb2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1248,7 +1248,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())

``````````

</details>


https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 13:09:29 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 13:09:29 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 '--offload-targets' for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <68644079.050a0220.3cd324.d40a@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-amdgpu

Author: Joseph Huber (jhuber6)

<details>
<summary>Changes</summary>

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.


---
Full diff: https://github.com/llvm/llvm-project/pull/146594.diff


9 Files Affected:

- (modified) clang/include/clang/Basic/DiagnosticDriverKinds.td (+2) 
- (modified) clang/include/clang/Driver/Options.td (+5-1) 
- (modified) clang/lib/Driver/Driver.cpp (+4-4) 
- (modified) clang/lib/Driver/ToolChain.cpp (+1-1) 
- (modified) clang/lib/Driver/ToolChains/CommonArgs.cpp (+1-1) 
- (modified) clang/lib/Frontend/CompilerInvocation.cpp (+3-3) 
- (modified) clang/test/Driver/amdgpu-openmp-sanitize-options.c (+2-2) 
- (modified) clang/test/Driver/hip-options.hip (+1-1) 
- (modified) flang/lib/Frontend/CompilerInvocation.cpp (+1-1) 


``````````diff
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 34b6c0d7a8acd..759ba0419bd45 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -116,6 +116,8 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
+def err_drv_mix_offload : Error<
+  "mixed %0 and %1 offloading compilation is not supported">;
 def err_drv_bad_target_id : Error<
   "invalid target ID '%0'; format is a processor name followed by an optional "
   "colon-delimited list of features followed by an enable/disable sign (e.g., "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..f359df55e0508 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : Joined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
 def offload_arch_EQ : Joined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
@@ -3678,7 +3682,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 775f3f029861c..930a678aa8923 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -991,7 +991,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1025,7 +1025,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1041,7 +1041,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1122,7 +1122,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..bdd77ac84913c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1404,7 +1404,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index f366e90945dac..c3fa27d9ff6d0 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3870,7 +3870,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4283,7 +4283,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4343,7 +4343,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..8e4c210306cb2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1248,7 +1248,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())

``````````

</details>


https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 13:12:21 2025
From: flang-commits at lists.llvm.org (Shilei Tian via flang-commits)
Date: Tue, 01 Jul 2025 13:12:21 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 '--offload-targets' for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <68644125.170a0220.33042a.5041@mx.google.com>


shiltian wrote:

Why not stick with `--offload-arch` for generic use? I'm not really sure about the motivation. 

https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 13:18:02 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Tue, 01 Jul 2025 13:18:02 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 '--offload-targets' for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <6864427a.a70a0220.32dbe7.0502@mx.google.com>


jhuber6 wrote:

> Why not stick with `--offload-arch` for generic use? I'm not really sure about the motivation.

This is the main problem I have with a lot of the offloading infrastructure. As far as I see it `--offload-arch=` is intended to behave like the `-mcpu` value but it's pulling double duty by also using *known* CPU names to *infer* the toolchain triple. We can still *do* that, I'm not planning on changing it, but I think it makes the infrastructure much easier to build on if we have a generic way to create an offloading toolchain in the same vein of `--target=`. I've had qualms with this since we do things like `--offload-arch=amdgcnspirv,gfx942` which implicitly creates two separate toolchains with incompatible triples.

I'm open to suggestions however.

https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 13:31:16 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Tue, 01 Jul 2025 13:31:16 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 '--offload-targets' for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <68644594.170a0220.e6823.4be3@mx.google.com>


https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/146594

>From c8e633a2384760c2c6d5b56f3de6cdf9b8ebde7e Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Mon, 3 Feb 2025 12:46:27 -0600
Subject: [PATCH] [Clang] Introduce '--offload-targets' for `-fopenmp-targets`

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.
---
 clang/include/clang/Basic/DiagnosticDriverKinds.td | 2 ++
 clang/include/clang/Driver/Options.td              | 6 +++++-
 clang/lib/Driver/Driver.cpp                        | 8 ++++----
 clang/lib/Driver/ToolChain.cpp                     | 2 +-
 clang/lib/Driver/ToolChains/CommonArgs.cpp         | 2 +-
 clang/lib/Frontend/CompilerInvocation.cpp          | 6 +++---
 clang/test/CodeGen/ibm128-unsupported.c            | 2 +-
 clang/test/Driver/amdgpu-openmp-sanitize-options.c | 4 ++--
 clang/test/Driver/hip-options.hip                  | 2 +-
 flang/lib/Frontend/CompilerInvocation.cpp          | 2 +-
 10 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 34b6c0d7a8acd..759ba0419bd45 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -116,6 +116,8 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
+def err_drv_mix_offload : Error<
+  "mixed %0 and %1 offloading compilation is not supported">;
 def err_drv_bad_target_id : Error<
   "invalid target ID '%0'; format is a processor name followed by an optional "
   "colon-delimited list of features followed by an enable/disable sign (e.g., "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..f359df55e0508 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : Joined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
 def offload_arch_EQ : Joined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
@@ -3678,7 +3682,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 775f3f029861c..930a678aa8923 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -991,7 +991,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1025,7 +1025,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1041,7 +1041,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1122,7 +1122,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..bdd77ac84913c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1404,7 +1404,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index f366e90945dac..c3fa27d9ff6d0 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3870,7 +3870,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4283,7 +4283,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4343,7 +4343,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
diff --git a/clang/test/CodeGen/ibm128-unsupported.c b/clang/test/CodeGen/ibm128-unsupported.c
index f4f0a16003e9e..5b4aaed7c5d7e 100644
--- a/clang/test/CodeGen/ibm128-unsupported.c
+++ b/clang/test/CodeGen/ibm128-unsupported.c
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple powerpc64le -emit-llvm-bc -fopenmp %s \
-// RUN:   -fopenmp-targets=powerpc64le,x86_64 -o %t-ppc-host.bc
+// RUN:   -fopenmp-targets=powerpc64le -o %t-ppc-host.bc
 // RUN: %clang_cc1 -verify -triple x86_64 -aux-triple powerpc64le -fopenmp \
 // RUN:   -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc %s \
 // RUN:   -fsyntax-only
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..8e4c210306cb2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1248,7 +1248,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())


From flang-commits at lists.llvm.org  Tue Jul  1 13:42:29 2025
From: flang-commits at lists.llvm.org (Shilei Tian via flang-commits)
Date: Tue, 01 Jul 2025 13:42:29 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <68644835.170a0220.1dfc7e.a086@mx.google.com>


https://github.com/shiltian edited https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 13:47:56 2025
From: flang-commits at lists.llvm.org (Shilei Tian via flang-commits)
Date: Tue, 01 Jul 2025 13:47:56 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <6864497c.170a0220.e85c6.557b@mx.google.com>


shiltian wrote:

I see. That makes sense. Worth a release note item.

https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 13:49:20 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Tue, 01 Jul 2025 13:49:20 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <686449d0.050a0220.2df6d9.f29e@mx.google.com>


jhuber6 wrote:

> I see. That makes sense. Worth a release note item.

This is just an alias, doesn't do what I want it to do yet. Struggling with how to maintain current semantics without hacking spaghetti code all over the place. Figured this was something I could pull out to make that smoother.

https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 15:53:34 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Tue, 01 Jul 2025 15:53:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (PR #146359)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146359@github.com>
Message-ID: <686466ee.170a0220.1a32be.a8cc@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `flang` at step 5 "build-unified-tree".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/32437

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 5 (build-unified-tree) failure: build (failure)
...
94.870 [38/13/6847] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Utils.cpp.o
94.871 [38/12/6848] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/PFTBuilder.cpp.o
94.872 [38/11/6849] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/PrivateReductionUtils.cpp.o
94.872 [38/10/6850] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExpr.cpp.o
94.873 [38/9/6851] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExprToHLFIR.cpp.o
94.874 [38/8/6852] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Clauses.cpp.o
94.875 [38/7/6853] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/OpenMP.cpp.o
94.876 [38/6/6854] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/Utils.cpp.o
101.263 [38/5/6855] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/openmp-utils.cpp.o
102.548 [38/4/6856] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-structure.cpp.o
FAILED: tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-structure.cpp.o 
ccache /home/buildbots/llvm-external-buildbots/clang.19.1.7/bin/clang++ -DFLANG_INCLUDE_TESTS=1 -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Semantics -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Semantics -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/llvm/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/../mlir/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/mlir/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/clang/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/llvm/../clang/include -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wc++98-compat-extra-semi -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Werror -Wno-deprecated-copy -Wno-string-conversion -Wno-ctad-maybe-unsupported -Wno-unused-command-line-argument -Wstring-conversion           -Wcovered-switch-default -Wno-nested-anon-types -Xclang -fno-pch-timestamp -O3 -DNDEBUG -std=c++17  -fno-exceptions -funwind-tables -fno-rtti -UNDEBUG -Winvalid-pch -Xclang -include-pch -Xclang /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/cmake_pch.hxx.pch -Xclang -include -Xclang /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/cmake_pch.hxx -MD -MT tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-structure.cpp.o -MF tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-structure.cpp.o.d -o tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-structure.cpp.o -c /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Semantics/check-omp-structure.cpp
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Semantics/check-omp-structure.cpp:1747:41: error: lambda capture 'this' is not used [-Werror,-Wunused-lambda-capture]
 1747 |   auto isVariableListItemOrCommonBlock{[this](const Symbol &sym) {
      |                                         ^~~~
1 error generated.
107.012 [38/3/6857] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-loop.cpp.o
130.883 [38/2/6858] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o
151.468 [38/1/6859] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/semantics.cpp.o
ninja: build stopped: subcommand failed.

```

</details>

https://github.com/llvm/llvm-project/pull/146359

From flang-commits at lists.llvm.org  Tue Jul  1 16:58:33 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Tue, 01 Jul 2025 16:58:33 -0700 (PDT)
Subject: [flang-commits] =?utf-8?q?=5Bflang=5D_=5Bflang=5D_Fix_handling_of?=
 =?utf-8?q?_identifier_in_column_1_of_free_form_continuat=E2=80=A6_=28PR_?=
 =?utf-8?q?=23146430=29?=
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146430@github.com>
Message-ID: <68647629.170a0220.3d3cb2.cacb@mx.google.com>


https://github.com/akuhlens approved this pull request.


https://github.com/llvm/llvm-project/pull/146430

From flang-commits at lists.llvm.org  Tue Jul  1 19:23:37 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 01 Jul 2025 19:23:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][prescanner] fix invalid check (PR
	#146613)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146613@github.com>
Message-ID: <68649829.170a0220.37c575.bc53@mx.google.com>


https://github.com/eugeneepshteyn approved this pull request.


https://github.com/llvm/llvm-project/pull/146613

From flang-commits at lists.llvm.org  Tue Jul  1 21:00:15 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Tue, 01 Jul 2025 21:00:15 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <6864aecf.170a0220.2de656.614e@mx.google.com>


================
@@ -116,6 +116,8 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
+def err_drv_mix_offload : Error<
----------------
jhuber6 wrote:

Oh that's leftover.

https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 21:01:00 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Tue, 01 Jul 2025 21:01:00 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <6864aefc.170a0220.e85c6.66d8@mx.google.com>


================
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : Joined<["--"], "offload-targets=">,
----------------
jhuber6 wrote:

Right, forgot that, thanks.

https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Tue Jul  1 21:03:05 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Tue, 01 Jul 2025 21:03:05 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <6864af79.050a0220.8cdf3.e1dc@mx.google.com>


https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/146594

>From 8db1344221d36408663b684cd5217a112bbe26fe Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Mon, 3 Feb 2025 12:46:27 -0600
Subject: [PATCH 1/2] [Clang] Introduce '--offload-targets' for
 `-fopenmp-targets`

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.
---
 clang/include/clang/Basic/DiagnosticDriverKinds.td | 2 ++
 clang/include/clang/Driver/Options.td              | 6 +++++-
 clang/lib/Driver/Driver.cpp                        | 8 ++++----
 clang/lib/Driver/ToolChain.cpp                     | 2 +-
 clang/lib/Driver/ToolChains/CommonArgs.cpp         | 2 +-
 clang/lib/Frontend/CompilerInvocation.cpp          | 6 +++---
 clang/test/CodeGen/ibm128-unsupported.c            | 2 +-
 clang/test/Driver/amdgpu-openmp-sanitize-options.c | 4 ++--
 clang/test/Driver/hip-options.hip                  | 2 +-
 flang/lib/Frontend/CompilerInvocation.cpp          | 2 +-
 flang/test/Driver/omp-driver-offload.f90           | 6 +++---
 11 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 34b6c0d7a8acd..759ba0419bd45 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -116,6 +116,8 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
+def err_drv_mix_offload : Error<
+  "mixed %0 and %1 offloading compilation is not supported">;
 def err_drv_bad_target_id : Error<
   "invalid target ID '%0'; format is a processor name followed by an optional "
   "colon-delimited list of features followed by an enable/disable sign (e.g., "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..f359df55e0508 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : Joined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
 def offload_arch_EQ : Joined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
@@ -3678,7 +3682,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 775f3f029861c..930a678aa8923 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -991,7 +991,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1025,7 +1025,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1041,7 +1041,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1122,7 +1122,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..bdd77ac84913c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1404,7 +1404,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index f366e90945dac..c3fa27d9ff6d0 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3870,7 +3870,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4283,7 +4283,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4343,7 +4343,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
diff --git a/clang/test/CodeGen/ibm128-unsupported.c b/clang/test/CodeGen/ibm128-unsupported.c
index f4f0a16003e9e..5b4aaed7c5d7e 100644
--- a/clang/test/CodeGen/ibm128-unsupported.c
+++ b/clang/test/CodeGen/ibm128-unsupported.c
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple powerpc64le -emit-llvm-bc -fopenmp %s \
-// RUN:   -fopenmp-targets=powerpc64le,x86_64 -o %t-ppc-host.bc
+// RUN:   -fopenmp-targets=powerpc64le -o %t-ppc-host.bc
 // RUN: %clang_cc1 -verify -triple x86_64 -aux-triple powerpc64le -fopenmp \
 // RUN:   -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc %s \
 // RUN:   -fsyntax-only
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..8e4c210306cb2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1248,7 +1248,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())
diff --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index 335bfad4b1884..f4f0c21dc1f5e 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -225,8 +225,8 @@
 ! RUN: | FileCheck %s --check-prefix=OFFLOAD-TARGETS
 
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"
 ! OFFLOAD-TARGETS-NEXT: "{{[^"]*}}flang" "-fc1" "-triple" "amdgcn-amd-amdhsa"
-! OFFLOAD-TARGETS-NOT: -fopenmp-targets
+! OFFLOAD-TARGETS-NOT: --offload-targets
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"

>From 325b18cde1f53d2e8aa04dc9cbae1ec46522e816 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Tue, 1 Jul 2025 23:02:56 -0500
Subject: [PATCH 2/2] Comments

---
 clang/include/clang/Basic/DiagnosticDriverKinds.td | 2 --
 clang/include/clang/Driver/Options.td              | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 759ba0419bd45..34b6c0d7a8acd 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -116,8 +116,6 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
-def err_drv_mix_offload : Error<
-  "mixed %0 and %1 offloading compilation is not supported">;
 def err_drv_bad_target_id : Error<
   "invalid target ID '%0'; format is a processor name followed by an optional "
   "colon-delimited list of features followed by an enable/disable sign (e.g., "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index f359df55e0508..1ba973cf2620c 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1142,7 +1142,7 @@ def offload_targets_EQ : Joined<["--"], "offload-targets=">,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify a list of target architectures to use for offloading.">;
 
-def offload_arch_EQ : Joined<["--"], "offload-arch=">,
+def offload_arch_EQ : CommaJoined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
            "If 'native' is used the compiler will detect locally installed architectures. "


From flang-commits at lists.llvm.org  Wed Jul  2 00:34:37 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Wed, 02 Jul 2025 00:34:37 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Canonicalize divdc3 calls into
 arithmetic-based complex division (PR #146017)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146017@github.com>
Message-ID: <6864e10d.630a0220.288d54.6015@mx.google.com>


s-watanabe314 wrote:

I've also submitted a patch as I was working on this independently and would like to share my approach. You can find my patch here: https://github.com/llvm/llvm-project/pull/146641

My patch takes a different approach to the `__divdc3` lowering. It allows us to choose whether to lower to a runtime function call or to a calculation using `complex.div` during the lowering phase. It also enables selection between the algebraic method and Smith's method for complex division by specifying a pass option in the conversion pass.

This implementation is based on the consensus reached in the Discourse discussion, and I believe it aligns well with the agreed-upon direction. It leverages the same driver options as Clang, which should contribute to easier maintenance and consistency. Furthermore, even if we need to change the method of calculating complex number division with other driver options like `-ffast-math`, we only need to modify the driver.

I'd be happy to discuss the differences between our approaches and see how we can best move forward. 


https://github.com/llvm/llvm-project/pull/146017

From flang-commits at lists.llvm.org  Wed Jul  2 01:47:00 2025
From: flang-commits at lists.llvm.org (Matt Arsenault via flang-commits)
Date: Wed, 02 Jul 2025 01:47:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <6864f204.170a0220.8c215.76d6@mx.google.com>


================
@@ -1867,6 +1867,20 @@ fir::factory::deduceOptimalExtents(mlir::ValueRange extents1,
   return extents;
 }
 
+uint64_t fir::factory::getGlobalAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getGlobalMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
----------------
arsenm wrote:

Why is there so much machinery? This should be direct read of constant 

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 02:04:29 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 02:04:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <6864f61d.170a0220.335cef.442a@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/146653

>From 569ad27f5ec86d5ef392ebb5f3b4a0dcaabf78b4 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 02:49:45 -0500
Subject: [PATCH 1/2] [flang] Emit `fir.global` in the global address space

Instead of emitting globals in the program/default address space, emit
them in the global address space. This also requires changes how address
of code-gen is handled, we need to cast to the default address space to
prevent code-gen issues.
---
 .../flang/Optimizer/Builder/FIRBuilder.h      |  4 ++
 .../flang/Optimizer/CodeGen/FIROpPatterns.h   |  3 +
 flang/lib/Optimizer/Builder/FIRBuilder.cpp    | 14 +++++
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 61 ++++++++++++++++---
 flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp | 15 +++++
 ...u-target-desc-cast-to-global-addrspace.f90 | 18 ++++++
 6 files changed, 106 insertions(+), 9 deletions(-)
 create mode 100644 flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90

diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index e1eaab3346901..f2c363e09e9e6 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -900,6 +900,10 @@ uint64_t getAllocaAddressSpace(const mlir::DataLayout *dataLayout);
 llvm::SmallVector<mlir::Value> deduceOptimalExtents(mlir::ValueRange extents1,
                                                     mlir::ValueRange extents2);
 
+uint64_t getGlobalAddressSpace(mlir::DataLayout *dataLayout);
+
+uint64_t getProgramAddressSpace(mlir::DataLayout *dataLayout);
+
 /// Given array extents generate code that sets them all to zeroes,
 /// if the array is empty, e.g.:
 ///   %false = arith.constant false
diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
index 7b1c14e4dfdc9..cbdcf6b501b8e 100644
--- a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
+++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
@@ -189,6 +189,9 @@ class ConvertFIRToLLVMPattern : public mlir::ConvertToLLVMPattern {
   unsigned
   getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) const;
 
+  unsigned
+  getGlobalAddressSpace(mlir::ConversionPatternRewriter &rewriter) const;
+
   const fir::FIRToLLVMPassOptions &options;
 
   using ConvertToLLVMPattern::matchAndRewrite;
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 6ac87067f6511..b5cabdb830e5c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -1867,6 +1867,20 @@ fir::factory::deduceOptimalExtents(mlir::ValueRange extents1,
   return extents;
 }
 
+uint64_t fir::factory::getGlobalAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getGlobalMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
+}
+
+uint64_t fir::factory::getProgramAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getProgramMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
+}
+
 llvm::SmallVector<mlir::Value> fir::factory::updateRuntimeExtentsForEmptyArrays(
     fir::FirOpBuilder &builder, mlir::Location loc, mlir::ValueRange extents) {
   if (extents.size() <= 1)
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 2b018912b40e4..09038f6e4d3ae 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -137,6 +137,38 @@ addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
 }
 
 namespace {
+
+mlir::Value replaceWithAddrOfOrASCast(mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Location loc,
+                                      std::uint64_t globalAS,
+                                      std::uint64_t programAS,
+                                      llvm::StringRef symName, mlir::Type type,
+                                      mlir::Operation *replaceOp = nullptr) {
+  if (mlir::isa<mlir::LLVM::LLVMPointerType>(type)) {
+    if (globalAS != programAS) {
+      auto llvmAddrOp = rewriter.create<mlir::LLVM::AddressOfOp>(
+          loc, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+      if (replaceOp)
+        return rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(
+            replaceOp, ::getLlvmPtrType(rewriter.getContext(), programAS),
+            llvmAddrOp);
+      return rewriter.create<mlir::LLVM::AddrSpaceCastOp>(
+          loc, getLlvmPtrType(rewriter.getContext(), programAS), llvmAddrOp);
+    }
+
+    if (replaceOp)
+      return rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
+          replaceOp, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+    return rewriter.create<mlir::LLVM::AddressOfOp>(
+        loc, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+  }
+
+  if (replaceOp)
+    return rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(replaceOp, type,
+                                                                symName);
+  return rewriter.create<mlir::LLVM::AddressOfOp>(loc, type, symName);
+}
+
 /// Lower `fir.address_of` operation to `llvm.address_of` operation.
 struct AddrOfOpConversion : public fir::FIROpConversion<fir::AddrOfOp> {
   using FIROpConversion::FIROpConversion;
@@ -144,9 +176,15 @@ struct AddrOfOpConversion : public fir::FIROpConversion<fir::AddrOfOp> {
   llvm::LogicalResult
   matchAndRewrite(fir::AddrOfOp addr, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
-    auto ty = convertType(addr.getType());
-    rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
-        addr, ty, addr.getSymbol().getRootReference().getValue());
+    auto global = addr->getParentOfType<mlir::ModuleOp>()
+                      .lookupSymbol<mlir::LLVM::GlobalOp>(addr.getSymbol());
+    replaceWithAddrOfOrASCast(
+        rewriter, addr->getLoc(),
+        global ? global.getAddrSpace() : getGlobalAddressSpace(rewriter),
+        getProgramAddressSpace(rewriter),
+        global ? global.getSymName()
+               : addr.getSymbol().getRootReference().getValue(),
+        convertType(addr.getType()), addr);
     return mlir::success();
   }
 };
@@ -1306,13 +1344,18 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
           ? fir::NameUniquer::getTypeDescriptorAssemblyName(recType.getName())
           : fir::NameUniquer::getTypeDescriptorName(recType.getName());
   mlir::Type llvmPtrTy = ::getLlvmPtrType(mod.getContext());
+  mlir::DataLayout dataLayout(mod);
   if (auto global = mod.template lookupSymbol<fir::GlobalOp>(name))
-    return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
-                                                    global.getSymName());
+    return replaceWithAddrOfOrASCast(
+        rewriter, loc, fir::factory::getGlobalAddressSpace(&dataLayout),
+        fir::factory::getProgramAddressSpace(&dataLayout), global.getSymName(),
+        llvmPtrTy);
   // The global may have already been translated to LLVM.
   if (auto global = mod.template lookupSymbol<mlir::LLVM::GlobalOp>(name))
-    return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
-                                                    global.getSymName());
+    return replaceWithAddrOfOrASCast(
+        rewriter, loc, global.getAddrSpace(),
+        fir::factory::getProgramAddressSpace(&dataLayout), global.getSymName(),
+        llvmPtrTy);
   // Type info derived types do not have type descriptors since they are the
   // types defining type descriptors.
   if (options.ignoreMissingTypeDescriptors ||
@@ -3130,8 +3173,8 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
     mlir::SymbolRefAttr comdat;
     llvm::ArrayRef<mlir::NamedAttribute> attrs;
     auto g = rewriter.create<mlir::LLVM::GlobalOp>(
-        loc, tyAttr, isConst, linkage, global.getSymName(), initAttr, 0, 0,
-        false, false, comdat, attrs, dbgExprs);
+        loc, tyAttr, isConst, linkage, global.getSymName(), initAttr, 0,
+        getGlobalAddressSpace(rewriter), false, false, comdat, attrs, dbgExprs);
 
     if (global.getAlignment() && *global.getAlignment() > 0)
       g.setAlignment(*global.getAlignment());
diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
index 12021deb4bd97..5c9c9c34caac1 100644
--- a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
+++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
@@ -365,4 +365,19 @@ unsigned ConvertFIRToLLVMPattern::getProgramAddressSpace(
   return defaultAddressSpace;
 }
 
+unsigned ConvertFIRToLLVMPattern::getGlobalAddressSpace(
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
+  assert(parentOp != nullptr &&
+         "expected insertion block to have parent operation");
+  auto module = mlir::isa<mlir::ModuleOp>(parentOp)
+                    ? mlir::cast<mlir::ModuleOp>(parentOp)
+                    : parentOp->getParentOfType<mlir::ModuleOp>();
+  if (module)
+    if (mlir::Attribute addrSpace =
+            mlir::DataLayout(module).getGlobalMemorySpace())
+      return llvm::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return defaultAddressSpace;
+}
+
 } // namespace fir
diff --git a/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90 b/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
new file mode 100644
index 0000000000000..249d403057b6c
--- /dev/null
+++ b/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
@@ -0,0 +1,18 @@
+!REQUIRES: amdgpu-registered-target
+
+!RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -target-cpu gfx908 %s -o - | FileCheck %s
+
+subroutine maintest
+  implicit none
+
+  type r1_t
+  end type r1_t
+
+  type(r1_t), pointer :: A
+end subroutine
+
+! CHECK: @[[TYPE_DESC:.*XdtXr1_t]] = linkonce_odr addrspace(1) constant %_QM__fortran_type_infoTderivedtype
+
+! CHECK: define void @maintest_() {{.*}} {
+! CHECK:   store { {{.*}} } { {{.*}}, ptr addrspacecast (ptr addrspace(1) @[[TYPE_DESC]] to ptr), {{.*}} }, {{.*}}
+! CHECK: }

>From d451d6640d955ae8a519976064a45af1330f44b5 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 04:04:14 -0500
Subject: [PATCH 2/2] simplify util

---
 flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
index 5c9c9c34caac1..75a54fd0ee14f 100644
--- a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
+++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/CodeGen/FIROpPatterns.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "llvm/Support/Debug.h"
 
@@ -370,14 +371,8 @@ unsigned ConvertFIRToLLVMPattern::getGlobalAddressSpace(
   mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
   assert(parentOp != nullptr &&
          "expected insertion block to have parent operation");
-  auto module = mlir::isa<mlir::ModuleOp>(parentOp)
-                    ? mlir::cast<mlir::ModuleOp>(parentOp)
-                    : parentOp->getParentOfType<mlir::ModuleOp>();
-  if (module)
-    if (mlir::Attribute addrSpace =
-            mlir::DataLayout(module).getGlobalMemorySpace())
-      return llvm::cast<mlir::IntegerAttr>(addrSpace).getUInt();
-  return defaultAddressSpace;
+  auto dataLayout = mlir::DataLayout::closest(parentOp);
+  return fir::factory::getGlobalAddressSpace(&dataLayout);
 }
 
 } // namespace fir


From flang-commits at lists.llvm.org  Wed Jul  2 02:07:02 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 02:07:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <6864f6b6.050a0220.181204.faed@mx.google.com>


================
@@ -1867,6 +1867,20 @@ fir::factory::deduceOptimalExtents(mlir::ValueRange extents1,
   return extents;
 }
 
+uint64_t fir::factory::getGlobalAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getGlobalMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
----------------
ergawy wrote:

This follows the same pattern as `getProgramAddressSpace` earlier in the file. I think `dataLayout` is made optional (through the pointer) to allow for easily testing when modules don't have a data layout attached.

I will see who wrote `getProgramAddressSpace` and add them to the review.

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 02:11:22 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 02:11:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <6864f7ba.170a0220.14041e.6b28@mx.google.com>


https://github.com/ergawy deleted https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 02:14:46 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 02:14:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <6864f886.a70a0220.754c2.f7cb@mx.google.com>


================
@@ -1867,6 +1867,20 @@ fir::factory::deduceOptimalExtents(mlir::ValueRange extents1,
   return extents;
 }
 
+uint64_t fir::factory::getGlobalAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getGlobalMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
----------------
ergawy wrote:

This follow the same pattern as `getAllocaAddressSpace` which I think was made this way to allow for testing modules that do not have data layout attached.

Added @tblah to verify if my understanding is correct or not (since it seems he wrote `getAllocaAddressSpace`).

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 02:15:50 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 02:15:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <6864f8c6.170a0220.6f701.7bd5@mx.google.com>


https://github.com/ergawy ready_for_review https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 02:25:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 02:25:46 -0700 (PDT)
Subject: [flang-commits] [flang] 1b7cbe1 - [flang][OpenMP] Create unique
 reduction decls for different logical kinds (#146558)
Message-ID: <6864fb1a.050a0220.264b0d.2480@mx.google.com>


Author: Tom Eccles
Date: 2025-07-02T10:25:43+01:00
New Revision: 1b7cbe1f871ebe572e383a2531257b88168eea1b

URL: https://github.com/llvm/llvm-project/commit/1b7cbe1f871ebe572e383a2531257b88168eea1b
DIFF: https://github.com/llvm/llvm-project/commit/1b7cbe1f871ebe572e383a2531257b88168eea1b.diff

LOG: [flang][OpenMP] Create unique reduction decls for different logical kinds (#146558)

Some Fujitsu tests showed incorrect results because we were sharing
reduction declarations for different kinds for logical variables.

Added: 
    flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90

Modified: 
    flang/lib/Lower/OpenMP/ReductionProcessor.cpp
    flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
    flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
    flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
    flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
    flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
    flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
    flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
    flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index d7f520e86e532..330cef7b54c74 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -149,13 +149,17 @@ std::string ReductionProcessor::getReductionName(
     reductionName = "multiply_reduction";
     break;
   case omp::clause::DefinedOperator::IntrinsicOperator::AND:
-    return "and_reduction";
+    reductionName = "and_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::EQV:
-    return "eqv_reduction";
+    reductionName = "eqv_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::OR:
-    return "or_reduction";
+    reductionName = "or_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:
-    return "neqv_reduction";
+    reductionName = "neqv_reduction";
+    break;
   default:
     reductionName = "other_reduction";
     break;
@@ -673,9 +677,6 @@ void ReductionProcessor::processReductionArguments(
     const auto &kindMap = firOpBuilder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
-    mlir::Type redNameTy = redType;
-    if (mlir::isa<fir::LogicalType>(redType.getEleTy()))
-      redNameTy = builder.getI1Type();
 
     if (const auto &redDefinedOp =
             std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
@@ -697,8 +698,7 @@ void ReductionProcessor::processReductionArguments(
         break;
       }
 
-      reductionName =
-          getReductionName(intrinsicOp, kindMap, redNameTy, isByRef);
+      reductionName = getReductionName(intrinsicOp, kindMap, redType, isByRef);
     } else if (const auto *reductionIntrinsic =
                    std::get_if<omp::clause::ProcedureDesignator>(
                        &redOperator.u)) {
@@ -709,7 +709,7 @@ void ReductionProcessor::processReductionArguments(
       redId = getReductionType(*reductionIntrinsic);
       reductionName =
           getReductionName(getRealName(*reductionIntrinsic).ToString(), kindMap,
-                           redNameTy, isByRef);
+                           redType, isByRef);
     } else {
       TODO(currentLocation, "Unexpected reduction type");
     }

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 18e252a787d78..cf5c1b127c34d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -147,7 +147,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index 7f3e3922ab29b..eff97aa5d0db0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -139,7 +139,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index e7654edffec8b..bbe4bf04b6827 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -89,7 +89,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -146,7 +146,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index dffd6545b95bf..304c7eefd58b0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
new file mode 100644
index 0000000000000..f7fd2a52060e6
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
@@ -0,0 +1,24 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+! Check that logical reductions of 
diff erent kinds do not end up using the same
+! reduction declaration
+
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l64 : !fir.logical<8> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
+
+subroutine test(a4, a8, sz)
+  integer :: sz
+  logical(4), dimension(sz) :: a4
+  logical(8), dimension(sz) :: a8
+
+  logical(4) :: res4 = .false.
+  logical(8) :: res8 = .false.
+  integer i
+
+! CHECK: omp.wsloop private({{.*}}) reduction(@or_reduction_l32 {{.*}}, @or_reduction_l64 {{.*}}) {
+  !$omp do reduction(.or.:res4, res8)
+  do i = 1,sz
+    res4 = res4 .or. a4(i)
+    res8 = res8 .or. a8(i)
+  enddo
+end subroutine

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index 366dd67591cfd..8c869952547c6 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @neqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @neqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -149,7 +149,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
index 66f624ff493fe..e0901bec6bc9c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @neqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @neqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant false
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -141,7 +141,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index a867d0a550bc7..bb09e2563e378 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @or_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -41,7 +41,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -88,7 +88,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -145,7 +145,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
index 592d4fa516bd7..5c9bcbf584d48 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @or_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant false
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)


        

From flang-commits at lists.llvm.org  Wed Jul  2 02:35:35 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 02 Jul 2025 02:35:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <6864fd67.170a0220.2223b5.7968@mx.google.com>


================
@@ -1867,6 +1867,20 @@ fir::factory::deduceOptimalExtents(mlir::ValueRange extents1,
   return extents;
 }
 
+uint64_t fir::factory::getGlobalAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getGlobalMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
----------------
tblah wrote:

Yes lots of flang lit tests do not have data layout strings. It would be inconvenient to add them.

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 03:00:35 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 02 Jul 2025 03:00:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <68650343.170a0220.3015b3.fc38@mx.google.com>


https://github.com/tblah commented:

The changes look sensible enough to me, but I am not that familiar with how this change might effect architectures using these different address spaces.

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 03:34:35 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 03:34:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Allocate `reduction` init
 temps on the stack for GPUs (PR #146667)
Message-ID: <llvm.org/llvm/llvm-project/pull/146667@github.com>


https://github.com/ergawy created https://github.com/llvm/llvm-project/pull/146667

Temps needed for the reduction init regions are now allocate on the heap all the time. However, this is performance killer for GPUs since malloc calls are prohibitively expensive. Therefore, we should do these allocations on the stack for GPU reductions.

>From 1f7d4677265ebf8ece574e33b11eabb44f3ba058 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 05:31:38 -0500
Subject: [PATCH] [flang][OpenMP] Allocate `reduction` init temps on the stack
 for GPUs

Temps needed for the reduction init regions are now allocate on the heap
all the time. However, this is performance killer for GPUs since malloc
calls are prohibitively expensive. Therefore, we should do these
allocations on the stack for GPU reductions.
---
 .../Lower/Support/PrivateReductionUtils.cpp   |  47 +++--
 .../Lower/OpenMP/parallel-reduction-array.f90 | 166 ++++++++++--------
 2 files changed, 119 insertions(+), 94 deletions(-)

diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index e878041d37c03..c3a5b6101ce00 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -502,22 +502,37 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
 
   // Allocating on the heap in case the whole reduction/privatization is nested
   // inside of a loop
-  auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
-  // if needsDealloc isn't statically false, add cleanup region. Always
-  // do this for allocatable boxes because they might have been re-allocated
-  // in the body of the loop/parallel region
-
-  std::optional<int64_t> cstNeedsDealloc = fir::getIntIfConstant(needsDealloc);
-  assert(cstNeedsDealloc.has_value() &&
-         "createTempFromMold decides this statically");
-  if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
-                        isDoConcurrent);
-  } else {
-    assert(!isAllocatableOrPointer &&
-           "Pointer-like arrays must be heap allocated");
-  }
+  auto temp = [&]() {
+    bool shouldAllocateOnStack = false;
+
+    // On the GPU, always allocate on the stack since heap allocatins are very
+    // expensive.
+    if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
+            *builder.getModule()))
+      shouldAllocateOnStack = offloadMod.getIsGPU();
+
+    if (shouldAllocateOnStack)
+      return createStackTempFromMold(loc, builder, source);
+
+    auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
+    // if needsDealloc isn't statically false, add cleanup region. Always
+    // do this for allocatable boxes because they might have been re-allocated
+    // in the body of the loop/parallel region
+
+    std::optional<int64_t> cstNeedsDealloc =
+        fir::getIntIfConstant(needsDealloc);
+    assert(cstNeedsDealloc.has_value() &&
+           "createTempFromMold decides this statically");
+    if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
+                          isDoConcurrent);
+    } else {
+      assert(!isAllocatableOrPointer &&
+             "Pointer-like arrays must be heap allocated");
+    }
+    return temp;
+  }();
 
   // Put the temporary inside of a box:
   // hlfir::genVariableBox doesn't handle non-default lower bounds
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 8e3de498f59c1..4f889d9a4e77f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -1,5 +1,8 @@
-! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+
+! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
 
 program reduce
 integer, dimension(3) :: i = 0
@@ -13,81 +16,88 @@ program reduce
 print *,i
 end program
 
-! CHECK-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
-! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK-LABEL:   } init {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
-! CHECK:           %[[TRUE:.*]]  = arith.constant true
+! CPU-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
+! CPU:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU-LABEL:   } init {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = arith.constant 0 : i32
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_4:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CPU:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
+! CPU:           %[[TRUE:.*]]  = arith.constant true
 !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
-! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         } combiner {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[C1_0:.*]] = arith.constant 1 : index
-! CHECK:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
-! CHECK:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
-! CHECK:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
-! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
-! CHECK:           }
-! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         }  cleanup {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
-! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
-! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
-! CHECK:           fir.if %[[VAL_5]] {
-! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
-! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
-! CHECK:           }
-! CHECK:           omp.yield
-! CHECK:         }
+! CPU:           %[[C0:.*]] = arith.constant 0 : index
+! CPU:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
+! CPU:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         } combiner {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[C1:.*]] = arith.constant 1 : index
+! CPU:           %[[C3:.*]] = arith.constant 3 : index
+! CPU:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[C1_0:.*]] = arith.constant 1 : index
+! CPU:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
+! CPU:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+! CPU:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
+! CPU:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
+! CPU:           }
+! CPU:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         }  cleanup {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
+! CPU:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CPU:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CPU:           fir.if %[[VAL_5]] {
+! CPU:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
+! CPU:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
+! CPU:           }
+! CPU:           omp.yield
+! CPU:         }
+
+! CPU-LABEL:   func.func @_QQmain()
+! CPU:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_1:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CPU:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
+! CPU:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:             %[[VAL_8:.*]] = arith.constant 1 : i32
+! CPU:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_10:.*]] = arith.constant 1 : index
+! CPU:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = arith.constant 2 : i32
+! CPU:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_14:.*]] = arith.constant 2 : index
+! CPU:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_16:.*]] = arith.constant 3 : i32
+! CPU:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_18:.*]] = arith.constant 3 : index
+! CPU:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
+! CPU:             omp.terminator
+! CPU:           }
 
-! CHECK-LABEL:   func.func @_QQmain()
-! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
-! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
-! CHECK:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : index
-! CHECK:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = arith.constant 2 : i32
-! CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_14:.*]] = arith.constant 2 : index
-! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_16:.*]] = arith.constant 3 : i32
-! CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_18:.*]] = arith.constant 3 : index
-! CHECK:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
-! CHECK:             omp.terminator
-! CHECK:           }
+! GPU: omp.declare_reduction {{.*}} alloc {
+! GPU: } init {
+! GPU-NOT: fir.allocmem {{.*}} {bindc_name = ".tmp", {{.*}}}
+! GPU:     fir.alloca {{.*}} {bindc_name = ".tmp"}
+! GPU: } combiner {
+! GPU: }


From flang-commits at lists.llvm.org  Wed Jul  2 03:35:04 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 03:35:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Allocate `reduction` init
 temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <68650b58.170a0220.16a0a7.6f8d@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-fir-hlfir

@llvm/pr-subscribers-flang-openmp

Author: Kareem Ergawy (ergawy)

<details>
<summary>Changes</summary>

Temps needed for the reduction init regions are now allocate on the heap all the time. However, this is performance killer for GPUs since malloc calls are prohibitively expensive. Therefore, we should do these allocations on the stack for GPU reductions.

---
Full diff: https://github.com/llvm/llvm-project/pull/146667.diff


2 Files Affected:

- (modified) flang/lib/Lower/Support/PrivateReductionUtils.cpp (+31-16) 
- (modified) flang/test/Lower/OpenMP/parallel-reduction-array.f90 (+88-78) 


``````````diff
diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index e878041d37c03..c3a5b6101ce00 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -502,22 +502,37 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
 
   // Allocating on the heap in case the whole reduction/privatization is nested
   // inside of a loop
-  auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
-  // if needsDealloc isn't statically false, add cleanup region. Always
-  // do this for allocatable boxes because they might have been re-allocated
-  // in the body of the loop/parallel region
-
-  std::optional<int64_t> cstNeedsDealloc = fir::getIntIfConstant(needsDealloc);
-  assert(cstNeedsDealloc.has_value() &&
-         "createTempFromMold decides this statically");
-  if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
-                        isDoConcurrent);
-  } else {
-    assert(!isAllocatableOrPointer &&
-           "Pointer-like arrays must be heap allocated");
-  }
+  auto temp = [&]() {
+    bool shouldAllocateOnStack = false;
+
+    // On the GPU, always allocate on the stack since heap allocatins are very
+    // expensive.
+    if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
+            *builder.getModule()))
+      shouldAllocateOnStack = offloadMod.getIsGPU();
+
+    if (shouldAllocateOnStack)
+      return createStackTempFromMold(loc, builder, source);
+
+    auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
+    // if needsDealloc isn't statically false, add cleanup region. Always
+    // do this for allocatable boxes because they might have been re-allocated
+    // in the body of the loop/parallel region
+
+    std::optional<int64_t> cstNeedsDealloc =
+        fir::getIntIfConstant(needsDealloc);
+    assert(cstNeedsDealloc.has_value() &&
+           "createTempFromMold decides this statically");
+    if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
+                          isDoConcurrent);
+    } else {
+      assert(!isAllocatableOrPointer &&
+             "Pointer-like arrays must be heap allocated");
+    }
+    return temp;
+  }();
 
   // Put the temporary inside of a box:
   // hlfir::genVariableBox doesn't handle non-default lower bounds
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 8e3de498f59c1..4f889d9a4e77f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -1,5 +1,8 @@
-! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+
+! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
 
 program reduce
 integer, dimension(3) :: i = 0
@@ -13,81 +16,88 @@ program reduce
 print *,i
 end program
 
-! CHECK-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
-! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK-LABEL:   } init {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
-! CHECK:           %[[TRUE:.*]]  = arith.constant true
+! CPU-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
+! CPU:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU-LABEL:   } init {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = arith.constant 0 : i32
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_4:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CPU:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
+! CPU:           %[[TRUE:.*]]  = arith.constant true
 !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
-! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         } combiner {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[C1_0:.*]] = arith.constant 1 : index
-! CHECK:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
-! CHECK:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
-! CHECK:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
-! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
-! CHECK:           }
-! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         }  cleanup {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
-! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
-! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
-! CHECK:           fir.if %[[VAL_5]] {
-! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
-! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
-! CHECK:           }
-! CHECK:           omp.yield
-! CHECK:         }
+! CPU:           %[[C0:.*]] = arith.constant 0 : index
+! CPU:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
+! CPU:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         } combiner {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[C1:.*]] = arith.constant 1 : index
+! CPU:           %[[C3:.*]] = arith.constant 3 : index
+! CPU:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[C1_0:.*]] = arith.constant 1 : index
+! CPU:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
+! CPU:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+! CPU:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
+! CPU:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
+! CPU:           }
+! CPU:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         }  cleanup {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
+! CPU:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CPU:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CPU:           fir.if %[[VAL_5]] {
+! CPU:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
+! CPU:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
+! CPU:           }
+! CPU:           omp.yield
+! CPU:         }
+
+! CPU-LABEL:   func.func @_QQmain()
+! CPU:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_1:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CPU:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
+! CPU:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:             %[[VAL_8:.*]] = arith.constant 1 : i32
+! CPU:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_10:.*]] = arith.constant 1 : index
+! CPU:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = arith.constant 2 : i32
+! CPU:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_14:.*]] = arith.constant 2 : index
+! CPU:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_16:.*]] = arith.constant 3 : i32
+! CPU:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_18:.*]] = arith.constant 3 : index
+! CPU:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
+! CPU:             omp.terminator
+! CPU:           }
 
-! CHECK-LABEL:   func.func @_QQmain()
-! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
-! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
-! CHECK:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : index
-! CHECK:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = arith.constant 2 : i32
-! CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_14:.*]] = arith.constant 2 : index
-! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_16:.*]] = arith.constant 3 : i32
-! CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_18:.*]] = arith.constant 3 : index
-! CHECK:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
-! CHECK:             omp.terminator
-! CHECK:           }
+! GPU: omp.declare_reduction {{.*}} alloc {
+! GPU: } init {
+! GPU-NOT: fir.allocmem {{.*}} {bindc_name = ".tmp", {{.*}}}
+! GPU:     fir.alloca {{.*}} {bindc_name = ".tmp"}
+! GPU: } combiner {
+! GPU: }

``````````

</details>


https://github.com/llvm/llvm-project/pull/146667

From flang-commits at lists.llvm.org  Wed Jul  2 03:53:25 2025
From: flang-commits at lists.llvm.org (Sergio Afonso via flang-commits)
Date: Wed, 02 Jul 2025 03:53:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <68650fa5.630a0220.244ad3.1316@mx.google.com>


skatrak wrote:

There's another pretty much equivalent approved PR here: #119585. @agozillon let us know if you intend to merge that one soon or if we should look into getting this one landed instead.

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 03:57:00 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 02 Jul 2025 03:57:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Allocate `reduction` init
 temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <6865107c.050a0220.20d41f.0fb7@mx.google.com>


https://github.com/tblah approved this pull request.

This looks okay to me, but you might need to do further work in OpenMPToLLVMIRConversion to be sure these allocas don't end up inlined inside of loops. This is complicated because we don't want to mix allocas and code in general so we can't just inline the init region at the allocaIP.

https://github.com/llvm/llvm-project/pull/146667

From flang-commits at lists.llvm.org  Wed Jul  2 03:59:40 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 03:59:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <6865111c.170a0220.3a4e43.741b@mx.google.com>


ergawy wrote:

> There's another pretty much equivalent approved PR here: #119585. @agozillon let us know if you intend to merge that one soon or if we should look into getting this one landed instead.

Thanks for bringing that to my attention, did not know about the PR.

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 04:18:24 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Wed, 02 Jul 2025 04:18:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Allocate `reduction` init
 temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <68651580.050a0220.2df6bb.202f@mx.google.com>


kiranchandramohan wrote:

> This looks okay to me, but you might need to do further work in OpenMPToLLVMIRConversion to be sure these allocas don't end up inlined inside of loops. This is complicated because we don't want to mix allocas and code in general so we can't just inline the init region at the allocaIP.

Fixed length allocas could be hoisted to the function entry but variable length allocas will need stacksave and restore.

https://github.com/llvm/llvm-project/pull/146667

From flang-commits at lists.llvm.org  Wed Jul  2 05:10:08 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 02 Jul 2025 05:10:08 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <686521a0.170a0220.3d3cb2.ea31@mx.google.com>


================
@@ -0,0 +1,39 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
+
+
+subroutine omp_unroll_heuristic01(lb, ub, inc)
+  integer res, i, lb, ub, inc
+
+  !$omp unroll
+  do i = lb, ub, inc
+    res = i
+  end do
+  !$omp end unroll
+
+end subroutine omp_unroll_heuristic01
+
+
+!CHECK-LABEL: func.func @_QPomp_unroll_heuristic01(
+!CHECK:      %c0_i32 = arith.constant 0 : i32
----------------
Meinersbur wrote:

This is not possible in MLIR: SSA value names are always automatically generated and cannot be user-provided. In this case there is a special handler for `arith.constant` that generates the name out of `c` (for constant), the value of the constant (`0`), and the type (`i32`). Without such a special handler, the value are just numbers. For values of type `omp.cli` I also defined such a special handler that generates names based on the nesting, see `NewCliOp::getAsmResultNames`.

https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Wed Jul  2 05:12:07 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 02 Jul 2025 05:12:07 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <68652217.050a0220.20d3ef.15f4@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Wed Jul  2 05:12:51 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 02 Jul 2025 05:12:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <68652243.630a0220.2166af.71a5@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Wed Jul  2 05:13:29 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 02 Jul 2025 05:13:29 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <68652269.170a0220.ef6cb.7796@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Wed Jul  2 05:14:19 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 02 Jul 2025 05:14:19 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <6865229b.630a0220.39b792.7759@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Wed Jul  2 05:19:47 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 02 Jul 2025 05:19:47 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <686523e3.170a0220.28ff68.7316@mx.google.com>


================
@@ -356,6 +357,212 @@ def SingleOp : OpenMP_Op<"single", traits = [
   let hasVerifier = 1;
 }
 
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Info Type
+//===---------------------------------------------------------------------===//
+
+def CanonicalLoopInfoType : OpenMP_Type<"CanonicalLoopInfo", "cli"> {
+  let summary = "Type for representing a reference to a canonical loop";
+  let description = [{
+    A variable of type CanonicalLoopInfo refers to an OpenMP-compatible
+    canonical loop in the same function. Values of this type are not
+    available at runtime and therefore cannot be used by the program itself,
+    i.e. an opaque type. It is similar to the transform dialect's
+    `!transform.interface` type, but instead of implementing an interface
+    for each transformation, the OpenMP dialect itself defines possible
+    operations on this type.
+
+    A value of type CanonicalLoopInfoType (in the following: CLI) value can be
+
+    1. created by omp.new_cli.
+    2. passed to omp.canonical_loop to associate the loop to that CLI. A CLI
+       can only be associated once.
+    3. passed to an omp loop transformation operation that modifies the loop
+       associated with the CLI. The CLI is the "applyee" and the operation is
+       the consumer. A CLI can only be consumed once.
+    4. passed to an omp loop transformation operation to associate the cli with
+       a result of that transformation. The CLI is the "generatee" and the
+       operation is the generator.
+
+    A CLI cannot
+
+    1. be returned from a function.
+    2. be passed to operations that are not specifically designed to take a
+       CanonicalLoopInfoType, including AnyType.
+
+    A CLI directly corresponds to an object of
+    OpenMPIRBuilder's CanonicalLoopInfo struct when lowering to LLVM-IR.
+  }];
+}
+
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Info Creation
+//===---------------------------------------------------------------------===//
+
+def NewCliOp : OpenMP_Op<"new_cli",
+    [DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>]> {
+  let summary = "Create a new Canonical Loop Info value.";
+  let description = [{
+    Create a new CLI that can be passed as an argument to a CanonicalLoopOp
+    and to loop transformation operations to handle dependencies between
+    loop transformation operations.
+  }];
+
+  let arguments = (ins );
+  let results = (outs CanonicalLoopInfoType:$result);
+  let assemblyFormat = [{
+      attr-dict
+  }];
+
+  let builders = [
+    OpBuilder<(ins )>,
+  ];
+
+  let hasVerifier = 1;
+}
+
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Operation
+//===---------------------------------------------------------------------===//
+def CanonicalLoopOp : OpenMPTransform_Op<"canonical_loop", 
+    [DeclareOpInterfaceMethods<OpAsmOpInterface, [ "getAsmBlockNames", "getAsmBlockArgumentNames"]>]> {
+  let summary = "OpenMP Canonical Loop Operation";
+  let description = [{
+    All loops that conform to OpenMP's definition of a canonical loop can be
+    simplified to a CanonicalLoopOp. In particular, there are no loop-carried
+    variables and the number of iterations it will execute is know before the
+    operation. This allows e.g. to determine the number of threads and chunks
+    the iterations space is split into before executing any iteration. More
+    restrictions may apply in cases such as (collapsed) loop nests, doacross
+    loops, etc.
+
+    In contrast to other loop operations such as `scf.for`, the number of
+    iterations is determined by only a single variable, the trip-count. The
+    induction variable value is the logical iteration number of that iteration,
+    which OpenMP defines to be between 0 and the trip-count (exclusive).
+    Loop representation having lower-bound, upper-bound, and step-size operands,
+    require passes to do more work than necessary, including handling special
+    cases such as upper-bound smaller than lower-bound, upper-bound equal to
+    the integer type's maximal value, negative step size, etc. This complexity
+    is better only handled once by the front-end and can apply its semantics
+    for such cases while still being able to represent any kind of loop, which
+    kind of the point of a mid-end intermediate representation. User-defined
+    types such as random-access iterators in C++ could not directly be
+    represented anyway.
+
+    The induction variable is always of the same type as the tripcount argument.
+    Since it can never be negative, tripcount is always interpreted as an
+    unsigned integer. It is the caller's responsibility to ensure the tripcount
+    is not negative when its interpretation is signed, i.e.
+    `%tripcount = max(0,%tripcount)`.
+
+    An optional argument to a omp.canonical_loop that can be passed in
+    is a CanonicalLoopInfo value that can be used to refer to the canonical
+    loop to apply transformations -- such as tiling, unrolling, or
+    work-sharing -- to the loop, similar to the transform dialect but
+    with OpenMP-specific semantics. Because it is optional, it has to be the
+    last of the operands, but appears first in the pretty format printing.
+
+    The pretty assembly format is inspired by python syntax, where `range(n)`
+    returns an iterator that runs from $0$ to $n-1$. The pretty assembly syntax
+    is one of:
+
+     omp.canonical_loop(%cli) %iv : !type in range(%tripcount)
+     omp.canonical_loop       %iv : !type in range(%tripcount)
+
+    A CanonicalLoopOp is lowered to LLVM-IR using
+    `OpenMPIRBuilder::createCanonicalLoop`.
+
+    #### Examples
+
+    Translation from lower-bound, upper-bound, step-size to trip-count.
+    ```c
+    for (int i = 3; i < 42; i+=2) {
+      B[i] = A[i];
+    }
+    ```
+
+    ```mlir
+    %lb = arith.constant 3 : i32
+    %ub = arith.constant 42 : i32
+    %step = arith.constant 2 : i32
+    %range = arith.sub %ub, %lb : i32
+    %tripcount = arith.div %range, %step : i32
+    omp.canonical_loop %iv : i32 in range(%tripcount) {
+      %offset = arith.mul %iv, %step : i32
+      %i = arith.add %offset, %lb : i32
+      %a = load %arrA[%i] : memref<?xf32>
+      store %a, %arrB[%i] : memref<?xf32>
+    }
+    ```
+
+    Nested canonical loop with transformation of the inner loop.
+    ```mlir
+    %outer = omp.new_cli : !omp.cli
+    %inner = omp.new_cli : !omp.cli
+    omp.canonical_loop(%outer) %iv1 : i32 in range(%tc1) {
+      omp.canonical_loop(%inner) %iv2 : i32 in range(%tc2) {
+        %a = load %arrA[%iv1, %iv2] : memref<?x?xf32>
+        store %a, %arrB[%iv1, %iv2] : memref<?x?xf32>
+      }
+    }
+    omp.unroll_full(%inner)
----------------
Meinersbur wrote:

You are correct, the position of `omp.unroll_full` is irrelevant as long as it is in the same function (except for now it must appear after `omp.canonical_loop(%inner)` because of how the simple version of MLIR-to-LLVM-IR lowering works, but nesting is still irrelevant).

https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Wed Jul  2 05:32:36 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Wed, 02 Jul 2025 05:32:36 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <686526e4.a70a0220.1e88aa.fdaf@mx.google.com>


https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/146594

>From 8db1344221d36408663b684cd5217a112bbe26fe Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Mon, 3 Feb 2025 12:46:27 -0600
Subject: [PATCH 1/2] [Clang] Introduce '--offload-targets' for
 `-fopenmp-targets`

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.
---
 clang/include/clang/Basic/DiagnosticDriverKinds.td | 2 ++
 clang/include/clang/Driver/Options.td              | 6 +++++-
 clang/lib/Driver/Driver.cpp                        | 8 ++++----
 clang/lib/Driver/ToolChain.cpp                     | 2 +-
 clang/lib/Driver/ToolChains/CommonArgs.cpp         | 2 +-
 clang/lib/Frontend/CompilerInvocation.cpp          | 6 +++---
 clang/test/CodeGen/ibm128-unsupported.c            | 2 +-
 clang/test/Driver/amdgpu-openmp-sanitize-options.c | 4 ++--
 clang/test/Driver/hip-options.hip                  | 2 +-
 flang/lib/Frontend/CompilerInvocation.cpp          | 2 +-
 flang/test/Driver/omp-driver-offload.f90           | 6 +++---
 11 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 34b6c0d7a8acd..759ba0419bd45 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -116,6 +116,8 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
+def err_drv_mix_offload : Error<
+  "mixed %0 and %1 offloading compilation is not supported">;
 def err_drv_bad_target_id : Error<
   "invalid target ID '%0'; format is a processor name followed by an optional "
   "colon-delimited list of features followed by an enable/disable sign (e.g., "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..f359df55e0508 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : Joined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
 def offload_arch_EQ : Joined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
@@ -3678,7 +3682,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 775f3f029861c..930a678aa8923 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -991,7 +991,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1025,7 +1025,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1041,7 +1041,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1122,7 +1122,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..bdd77ac84913c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1404,7 +1404,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index f366e90945dac..c3fa27d9ff6d0 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3870,7 +3870,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4283,7 +4283,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4343,7 +4343,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
diff --git a/clang/test/CodeGen/ibm128-unsupported.c b/clang/test/CodeGen/ibm128-unsupported.c
index f4f0a16003e9e..5b4aaed7c5d7e 100644
--- a/clang/test/CodeGen/ibm128-unsupported.c
+++ b/clang/test/CodeGen/ibm128-unsupported.c
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple powerpc64le -emit-llvm-bc -fopenmp %s \
-// RUN:   -fopenmp-targets=powerpc64le,x86_64 -o %t-ppc-host.bc
+// RUN:   -fopenmp-targets=powerpc64le -o %t-ppc-host.bc
 // RUN: %clang_cc1 -verify -triple x86_64 -aux-triple powerpc64le -fopenmp \
 // RUN:   -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc %s \
 // RUN:   -fsyntax-only
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..8e4c210306cb2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1248,7 +1248,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())
diff --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index 335bfad4b1884..f4f0c21dc1f5e 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -225,8 +225,8 @@
 ! RUN: | FileCheck %s --check-prefix=OFFLOAD-TARGETS
 
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"
 ! OFFLOAD-TARGETS-NEXT: "{{[^"]*}}flang" "-fc1" "-triple" "amdgcn-amd-amdhsa"
-! OFFLOAD-TARGETS-NOT: -fopenmp-targets
+! OFFLOAD-TARGETS-NOT: --offload-targets
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"

>From 11e7beaa06be602c571fbd037549c3dd2c930ea4 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Tue, 1 Jul 2025 23:02:56 -0500
Subject: [PATCH 2/2] Comments

---
 clang/include/clang/Basic/DiagnosticDriverKinds.td | 2 --
 clang/include/clang/Driver/Options.td              | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 759ba0419bd45..34b6c0d7a8acd 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -116,8 +116,6 @@ def err_drv_cuda_host_arch : Error<
   "unsupported architecture '%0' for host compilation">;
 def err_drv_mix_cuda_hip : Error<
   "mixed CUDA and HIP compilation is not supported">;
-def err_drv_mix_offload : Error<
-  "mixed %0 and %1 offloading compilation is not supported">;
 def err_drv_bad_target_id : Error<
   "invalid target ID '%0'; format is a processor name followed by an optional "
   "colon-delimited list of features followed by an enable/disable sign (e.g., "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index f359df55e0508..8ccb301a6362b 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,7 +1138,7 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
-def offload_targets_EQ : Joined<["--"], "offload-targets=">,
+def offload_targets_EQ : CommaJoined<["--"], "offload-targets=">,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify a list of target architectures to use for offloading.">;
 


From flang-commits at lists.llvm.org  Wed Jul  2 06:57:33 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 02 Jul 2025 06:57:33 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <68653acd.170a0220.2b159e.8cf6@mx.google.com>


================
@@ -15,14 +15,10 @@
 #ifndef MLIR_DIALECT_OPENMP_OPENMPCLAUSEOPERANDS_H_
 #define MLIR_DIALECT_OPENMP_OPENMPCLAUSEOPERANDS_H_
 
+#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h"
----------------
Meinersbur wrote:

@tblah Extracted into its own PR: #146684

https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Wed Jul  2 07:12:24 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 07:12:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <68653e48.170a0220.3c0999.92d9@mx.google.com>


agozillon wrote:

Feel free to land it! The PR stack that my version is tied to has been held up for a while due to some conflicts with the old variation of implicit mapping of allocatable derived type components; need to test it with Akash's new version, and I've been meaning to disentangle it for a while to land it separately but haven't found the time, so thank you for creating another PR :-) 

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 07:14:09 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 07:14:09 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <68653eb1.170a0220.1f4c92.9523@mx.google.com>


https://github.com/agozillon approved this pull request.

LGTM, we have this downstream currently and it's yet to blow anything up after half a year-ish and it solves several issues, so it's a step in the right direction even if it's not perfect. 

https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 07:14:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 07:14:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <68653ed6.630a0220.3639aa.98ca@mx.google.com>


https://github.com/agozillon edited https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Wed Jul  2 07:37:49 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 02 Jul 2025 07:37:49 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add ComplexTOROCDL pass (PR
	#144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <6865443d.050a0220.1e03c.4f78@mx.google.com>


https://github.com/TIFitis edited https://github.com/llvm/llvm-project/pull/144926

From flang-commits at lists.llvm.org  Wed Jul  2 07:40:53 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 02 Jul 2025 07:40:53 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add ComplexTOROCDL pass (PR
	#144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <686544f5.050a0220.7d9ae.5856@mx.google.com>


TIFitis wrote:

@ftynse Thanks for the suggestion, I've renamed the pass to ComplexToROCDLLibraryCalls. Please let me know if any other changes are needed.

Thanks

https://github.com/llvm/llvm-project/pull/144926

From flang-commits at lists.llvm.org  Wed Jul  2 08:01:11 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 02 Jul 2025 08:01:11 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add
	ComplexTOROCDLLibraryCalls pass (PR #144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <686549b7.050a0220.3ef44.5b71@mx.google.com>


https://github.com/TIFitis edited https://github.com/llvm/llvm-project/pull/144926

From flang-commits at lists.llvm.org  Wed Jul  2 08:15:26 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 08:15:26 -0700 (PDT)
Subject: [flang-commits] [flang] b177422 - [flang] Emit `fir.global` in the
 global address space (#146653)
Message-ID: <68654d0e.630a0220.330714.defd@mx.google.com>


Author: Kareem Ergawy
Date: 2025-07-02T17:15:22+02:00
New Revision: b1774222c761a7912cdbe0d0004ca12dae95f721

URL: https://github.com/llvm/llvm-project/commit/b1774222c761a7912cdbe0d0004ca12dae95f721
DIFF: https://github.com/llvm/llvm-project/commit/b1774222c761a7912cdbe0d0004ca12dae95f721.diff

LOG: [flang] Emit `fir.global` in the global address space (#146653)

Instead of emitting globals in the program/default address space, emit
them in the global address space. This also requires changes how address
of code-gen is handled, we need to cast to the default address space to
prevent code-gen issues.

Added: 
    flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90

Modified: 
    flang/include/flang/Optimizer/Builder/FIRBuilder.h
    flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
    flang/lib/Optimizer/Builder/FIRBuilder.cpp
    flang/lib/Optimizer/CodeGen/CodeGen.cpp
    flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index e1eaab3346901..f2c363e09e9e6 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -900,6 +900,10 @@ uint64_t getAllocaAddressSpace(const mlir::DataLayout *dataLayout);
 llvm::SmallVector<mlir::Value> deduceOptimalExtents(mlir::ValueRange extents1,
                                                     mlir::ValueRange extents2);
 
+uint64_t getGlobalAddressSpace(mlir::DataLayout *dataLayout);
+
+uint64_t getProgramAddressSpace(mlir::DataLayout *dataLayout);
+
 /// Given array extents generate code that sets them all to zeroes,
 /// if the array is empty, e.g.:
 ///   %false = arith.constant false

diff  --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
index 7b1c14e4dfdc9..cbdcf6b501b8e 100644
--- a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
+++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
@@ -189,6 +189,9 @@ class ConvertFIRToLLVMPattern : public mlir::ConvertToLLVMPattern {
   unsigned
   getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) const;
 
+  unsigned
+  getGlobalAddressSpace(mlir::ConversionPatternRewriter &rewriter) const;
+
   const fir::FIRToLLVMPassOptions &options;
 
   using ConvertToLLVMPattern::matchAndRewrite;

diff  --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 6ac87067f6511..b5cabdb830e5c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -1867,6 +1867,20 @@ fir::factory::deduceOptimalExtents(mlir::ValueRange extents1,
   return extents;
 }
 
+uint64_t fir::factory::getGlobalAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getGlobalMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
+}
+
+uint64_t fir::factory::getProgramAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getProgramMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
+}
+
 llvm::SmallVector<mlir::Value> fir::factory::updateRuntimeExtentsForEmptyArrays(
     fir::FirOpBuilder &builder, mlir::Location loc, mlir::ValueRange extents) {
   if (extents.size() <= 1)

diff  --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 2b018912b40e4..09038f6e4d3ae 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -137,6 +137,38 @@ addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
 }
 
 namespace {
+
+mlir::Value replaceWithAddrOfOrASCast(mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Location loc,
+                                      std::uint64_t globalAS,
+                                      std::uint64_t programAS,
+                                      llvm::StringRef symName, mlir::Type type,
+                                      mlir::Operation *replaceOp = nullptr) {
+  if (mlir::isa<mlir::LLVM::LLVMPointerType>(type)) {
+    if (globalAS != programAS) {
+      auto llvmAddrOp = rewriter.create<mlir::LLVM::AddressOfOp>(
+          loc, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+      if (replaceOp)
+        return rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(
+            replaceOp, ::getLlvmPtrType(rewriter.getContext(), programAS),
+            llvmAddrOp);
+      return rewriter.create<mlir::LLVM::AddrSpaceCastOp>(
+          loc, getLlvmPtrType(rewriter.getContext(), programAS), llvmAddrOp);
+    }
+
+    if (replaceOp)
+      return rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
+          replaceOp, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+    return rewriter.create<mlir::LLVM::AddressOfOp>(
+        loc, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+  }
+
+  if (replaceOp)
+    return rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(replaceOp, type,
+                                                                symName);
+  return rewriter.create<mlir::LLVM::AddressOfOp>(loc, type, symName);
+}
+
 /// Lower `fir.address_of` operation to `llvm.address_of` operation.
 struct AddrOfOpConversion : public fir::FIROpConversion<fir::AddrOfOp> {
   using FIROpConversion::FIROpConversion;
@@ -144,9 +176,15 @@ struct AddrOfOpConversion : public fir::FIROpConversion<fir::AddrOfOp> {
   llvm::LogicalResult
   matchAndRewrite(fir::AddrOfOp addr, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
-    auto ty = convertType(addr.getType());
-    rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
-        addr, ty, addr.getSymbol().getRootReference().getValue());
+    auto global = addr->getParentOfType<mlir::ModuleOp>()
+                      .lookupSymbol<mlir::LLVM::GlobalOp>(addr.getSymbol());
+    replaceWithAddrOfOrASCast(
+        rewriter, addr->getLoc(),
+        global ? global.getAddrSpace() : getGlobalAddressSpace(rewriter),
+        getProgramAddressSpace(rewriter),
+        global ? global.getSymName()
+               : addr.getSymbol().getRootReference().getValue(),
+        convertType(addr.getType()), addr);
     return mlir::success();
   }
 };
@@ -1306,13 +1344,18 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
           ? fir::NameUniquer::getTypeDescriptorAssemblyName(recType.getName())
           : fir::NameUniquer::getTypeDescriptorName(recType.getName());
   mlir::Type llvmPtrTy = ::getLlvmPtrType(mod.getContext());
+  mlir::DataLayout dataLayout(mod);
   if (auto global = mod.template lookupSymbol<fir::GlobalOp>(name))
-    return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
-                                                    global.getSymName());
+    return replaceWithAddrOfOrASCast(
+        rewriter, loc, fir::factory::getGlobalAddressSpace(&dataLayout),
+        fir::factory::getProgramAddressSpace(&dataLayout), global.getSymName(),
+        llvmPtrTy);
   // The global may have already been translated to LLVM.
   if (auto global = mod.template lookupSymbol<mlir::LLVM::GlobalOp>(name))
-    return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
-                                                    global.getSymName());
+    return replaceWithAddrOfOrASCast(
+        rewriter, loc, global.getAddrSpace(),
+        fir::factory::getProgramAddressSpace(&dataLayout), global.getSymName(),
+        llvmPtrTy);
   // Type info derived types do not have type descriptors since they are the
   // types defining type descriptors.
   if (options.ignoreMissingTypeDescriptors ||
@@ -3130,8 +3173,8 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
     mlir::SymbolRefAttr comdat;
     llvm::ArrayRef<mlir::NamedAttribute> attrs;
     auto g = rewriter.create<mlir::LLVM::GlobalOp>(
-        loc, tyAttr, isConst, linkage, global.getSymName(), initAttr, 0, 0,
-        false, false, comdat, attrs, dbgExprs);
+        loc, tyAttr, isConst, linkage, global.getSymName(), initAttr, 0,
+        getGlobalAddressSpace(rewriter), false, false, comdat, attrs, dbgExprs);
 
     if (global.getAlignment() && *global.getAlignment() > 0)
       g.setAlignment(*global.getAlignment());

diff  --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
index 12021deb4bd97..75a54fd0ee14f 100644
--- a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
+++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/CodeGen/FIROpPatterns.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "llvm/Support/Debug.h"
 
@@ -365,4 +366,13 @@ unsigned ConvertFIRToLLVMPattern::getProgramAddressSpace(
   return defaultAddressSpace;
 }
 
+unsigned ConvertFIRToLLVMPattern::getGlobalAddressSpace(
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
+  assert(parentOp != nullptr &&
+         "expected insertion block to have parent operation");
+  auto dataLayout = mlir::DataLayout::closest(parentOp);
+  return fir::factory::getGlobalAddressSpace(&dataLayout);
+}
+
 } // namespace fir

diff  --git a/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90 b/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
new file mode 100644
index 0000000000000..249d403057b6c
--- /dev/null
+++ b/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
@@ -0,0 +1,18 @@
+!REQUIRES: amdgpu-registered-target
+
+!RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -target-cpu gfx908 %s -o - | FileCheck %s
+
+subroutine maintest
+  implicit none
+
+  type r1_t
+  end type r1_t
+
+  type(r1_t), pointer :: A
+end subroutine
+
+! CHECK: @[[TYPE_DESC:.*XdtXr1_t]] = linkonce_odr addrspace(1) constant %_QM__fortran_type_infoTderivedtype
+
+! CHECK: define void @maintest_() {{.*}} {
+! CHECK:   store { {{.*}} } { {{.*}}, ptr addrspacecast (ptr addrspace(1) @[[TYPE_DESC]] to ptr), {{.*}} }, {{.*}}
+! CHECK: }


        

From flang-commits at lists.llvm.org  Wed Jul  2 08:15:29 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 08:15:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Emit `fir.global` in the global
	address space (PR #146653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146653@github.com>
Message-ID: <68654d11.170a0220.33b28c.f6bd@mx.google.com>


https://github.com/ergawy closed https://github.com/llvm/llvm-project/pull/146653

From flang-commits at lists.llvm.org  Tue Jul  1 09:12:04 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 01 Jul 2025 09:12:04 -0700 (PDT)
Subject: [flang-commits] [flang] ba116a8 - [flang][OpenMP] Split
 check-omp-structure.cpp into smaller files, NFC (#146359)
Message-ID: <686408d4.170a0220.14e15e.2dbe@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-01T11:12:00-05:00
New Revision: ba116a8bed9ae093575bd316bf8847bb016177dd

URL: https://github.com/llvm/llvm-project/commit/ba116a8bed9ae093575bd316bf8847bb016177dd
DIFF: https://github.com/llvm/llvm-project/commit/ba116a8bed9ae093575bd316bf8847bb016177dd.diff

LOG: [flang][OpenMP] Split check-omp-structure.cpp into smaller files, NFC (#146359)

Create these new files in flang/lib/Semantics:
  openmp-utils.cpp/.h         - Common utilities
  check-omp-atomic.cpp        - Atomic-related checks
  check-omp-loop.cpp          - Loop constructs/clauses
  check-omp-metadirective.cpp - Metadirective-related checks

Update lists of included headers, std in particular.

---------

Co-authored-by: Jack Styles <jack.styles at arm.com>

Added: 
    flang/lib/Semantics/check-omp-atomic.cpp
    flang/lib/Semantics/check-omp-loop.cpp
    flang/lib/Semantics/check-omp-metadirective.cpp
    flang/lib/Semantics/openmp-utils.cpp
    flang/lib/Semantics/openmp-utils.h

Modified: 
    flang/lib/Semantics/CMakeLists.txt
    flang/lib/Semantics/check-omp-structure.cpp
    flang/lib/Semantics/check-omp-structure.h

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index c0fda3631c01f..109bc2dbb8569 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -20,6 +20,9 @@ add_flang_library(FortranSemantics
   check-io.cpp
   check-namelist.cpp
   check-nullify.cpp
+  check-omp-atomic.cpp
+  check-omp-loop.cpp
+  check-omp-metadirective.cpp
   check-omp-structure.cpp
   check-purity.cpp
   check-return.cpp
@@ -34,12 +37,13 @@ add_flang_library(FortranSemantics
   mod-file.cpp
   openmp-dsa.cpp
   openmp-modifiers.cpp
+  openmp-utils.cpp
   pointer-assignment.cpp
   program-tree.cpp
-  resolve-labels.cpp
   resolve-directives.cpp
-  resolve-names-utils.cpp
+  resolve-labels.cpp
   resolve-names.cpp
+  resolve-names-utils.cpp
   rewrite-parse-tree.cpp
   runtime-type-info.cpp
   scope.cpp

diff  --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
new file mode 100644
index 0000000000000..047c604693460
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -0,0 +1,1295 @@
+//===-- lib/Semantics/check-omp-atomic.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks related to the ATOMIC construct.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/ErrorHandling.h"
+
+#include <cassert>
+#include <list>
+#include <optional>
+#include <string_view>
+#include <tuple>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+namespace operation = Fortran::evaluate::operation;
+
+template <typename T, typename U>
+static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
+  return !(e == f);
+}
+
+// There is no consistent way to get the source of a given ActionStmt, so
+// extract the source information from Statement<ActionStmt> when we can,
+// and keep it around for error reporting in further analyses.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+struct AnalyzedCondStmt {
+  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
+  parser::CharBlock source;
+  SourcedActionStmt ift, iff;
+};
+
+static SourcedActionStmt GetActionStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+static SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
+// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
+// is that the ActionStmt will be either an assignment or a pointer-assignment,
+// otherwise return std::nullopt.
+// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
+// the "typedAssignment" is unset. This can happen if there are semantic errors
+// in the purported assignment.
+static std::optional<evaluate::Assignment> GetEvaluateAssignment(
+    const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
+
+  return common::visit(
+      [](auto &&s) -> std::optional<evaluate::Assignment> {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>) {
+          const TypedAssignment &typed{s.value().typedAssignment};
+          // ForwardOwningPointer                 typedAssignment
+          // `- GenericAssignmentWrapper          ^.get()
+          //    `- std::optional<Assignment>      ^->v
+          return typed.get()->v;
+        } else {
+          return std::nullopt;
+        }
+      },
+      x->u);
+}
+
+static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  // Extract the evaluate::Expr from ScalarLogicalExpr.
+  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
+    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
+    const parser::Expr &expr{logical.thing.thing.value()};
+    return GetEvaluateExpr(expr);
+  }};
+
+  // Recognize either
+  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
+  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
+
+  if (auto &&action{GetActionStmt(x)}) {
+    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
+            &action.stmt->u)}) {
+      const parser::IfStmt &s{ifs->value()};
+      auto &&maybeCond{
+          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
+      auto &thenStmt{
+          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
+      if (maybeCond) {
+        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
+            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
+            SourcedActionStmt{}};
+      }
+    }
+    return std::nullopt;
+  }
+
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    if (auto *ifc{
+            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
+      using ElseBlock = parser::IfConstruct::ElseBlock;
+      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
+      const parser::IfConstruct &s{ifc->value()};
+
+      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
+        // Not expecting any else-if statements.
+        return std::nullopt;
+      }
+      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
+      auto &&maybeCond{getFromLogical(
+          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
+      if (!maybeCond) {
+        return std::nullopt;
+      }
+
+      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)),
+            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
+        if (result.ift.stmt && result.iff.stmt) {
+          return result;
+        }
+      } else {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
+        if (result.ift.stmt) {
+          return result;
+        }
+      }
+    }
+    return std::nullopt;
+  }
+
+  return std::nullopt;
+}
+
+static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
+    parser::CharBlock source) {
+  // Find => in the range, if not found, find = that is not a part of
+  // <=, >=, ==, or /=.
+  auto trim{[](std::string_view v) {
+    const char *begin{v.data()};
+    const char *end{begin + v.size()};
+    while (*begin == ' ' && begin != end) {
+      ++begin;
+    }
+    while (begin != end && end[-1] == ' ') {
+      --end;
+    }
+    assert(begin != end && "Source should not be empty");
+    return parser::CharBlock(begin, end - begin);
+  }};
+
+  std::string_view sv(source.begin(), source.size());
+
+  if (auto where{sv.find("=>")}; where != sv.npos) {
+    std::string_view lhs(sv.data(), where);
+    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
+    return std::make_pair(trim(lhs), trim(rhs));
+  }
+
+  // Go backwards, since all the exclusions above end with a '='.
+  for (size_t next{source.size()}; next > 1; --next) {
+    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
+      std::string_view lhs(sv.data(), next - 1);
+      std::string_view rhs(sv.data() + next, sv.size() - next);
+      return std::make_pair(trim(lhs), trim(rhs));
+    }
+  }
+  llvm_unreachable("Could not find assignment operator");
+}
+
+static bool IsCheckForAssociated(const SomeExpr &cond) {
+  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
+}
+
+static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
+  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
+  // for example.
+  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
+}
+
+static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
+  if (value) {
+    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
+        evaluate::GenericExprWrapper::Deleter);
+  }
+}
+
+static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
+    std::optional<evaluate::Assignment> value) {
+  if (value) {
+    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
+        evaluate::GenericAssignmentWrapper::Deleter);
+  }
+}
+
+static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
+    int what,
+    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
+  parser::OpenMPAtomicConstruct::Analysis::Op operation;
+  operation.what = what;
+  SetAssignment(operation.assign, maybeAssign);
+  return operation;
+}
+
+static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
+    const SomeExpr &atom, const MaybeExpr &cond,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
+  // Defined in flang/include/flang/Parser/parse-tree.h
+  //
+  // struct Analysis {
+  //   struct Kind {
+  //     static constexpr int None = 0;
+  //     static constexpr int Read = 1;
+  //     static constexpr int Write = 2;
+  //     static constexpr int Update = Read | Write;
+  //     static constexpr int Action = 3; // Bits containing N, R, W, U
+  //     static constexpr int IfTrue = 4;
+  //     static constexpr int IfFalse = 8;
+  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
+  //   };
+  //   struct Op {
+  //     int what;
+  //     TypedAssignment assign;
+  //   };
+  //   TypedExpr atom, cond;
+  //   Op op0, op1;
+  // };
+
+  parser::OpenMPAtomicConstruct::Analysis an;
+  SetExpr(an.atom, atom);
+  SetExpr(an.cond, cond);
+  an.op0 = std::move(op0);
+  an.op1 = std::move(op1);
+  return an;
+}
+
+/// Check if `expr` satisfies the following conditions for x and v:
+///
+/// [6.0:189:10-12]
+/// - x and v (as applicable) are either scalar variables or
+///   function references with scalar data pointer result of non-character
+///   intrinsic type or variables that are non-polymorphic scalar pointers
+///   and any length type parameter must be constant.
+void OmpStructureChecker::CheckAtomicType(
+    SymbolRef sym, parser::CharBlock source, std::string_view name) {
+  const DeclTypeSpec *typeSpec{sym->GetType()};
+  if (!typeSpec) {
+    return;
+  }
+
+  if (!IsPointer(sym)) {
+    using Category = DeclTypeSpec::Category;
+    Category cat{typeSpec->category()};
+    if (cat == Category::Character) {
+      context_.Say(source,
+          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
+    } else if (cat != Category::Numeric && cat != Category::Logical) {
+      context_.Say(source,
+          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
+    }
+    return;
+  }
+
+  // Variable is a pointer.
+  if (typeSpec->IsPolymorphic()) {
+    context_.Say(source,
+        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
+        name);
+    return;
+  }
+
+  // Go over all length parameters, if any, and check if they are
+  // explicit.
+  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
+    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
+          // "entry" is a map entry
+          return entry.second.isLen() && !entry.second.isExplicit();
+        })) {
+      context_.Say(source,
+          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
+          name);
+    }
+  }
+}
+
+void OmpStructureChecker::CheckAtomicVariable(
+    const SomeExpr &atom, parser::CharBlock source) {
+  if (atom.Rank() != 0) {
+    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
+        atom.AsFortran());
+  }
+
+  std::vector<SomeExpr> dsgs{GetAllDesignators(atom)};
+  assert(dsgs.size() == 1 && "Should have a single top-level designator");
+  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
+
+  CheckAtomicType(syms.back(), source, atom.AsFortran());
+
+  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
+    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
+        atom.AsFortran());
+  }
+}
+
+void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
+    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
+    parser::CharBlock source) {
+  if (auto *expr{HasStorageOverlap(base, exprs)}) {
+    context_.Say(source,
+        "Within atomic operation %s and %s access the same storage"_warn_en_US,
+        base.AsFortran(), expr->AsFortran());
+  }
+}
+
+void OmpStructureChecker::ErrorShouldBeVariable(
+    const MaybeExpr &expr, parser::CharBlock source) {
+  if (expr) {
+    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
+        expr->AsFortran());
+  } else {
+    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
+  }
+}
+
+std::pair<const parser::ExecutionPartConstruct *,
+    const parser::ExecutionPartConstruct *>
+OmpStructureChecker::CheckUpdateCapture(
+    const parser::ExecutionPartConstruct *ec1,
+    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
+  // Decide which statement is the atomic update and which is the capture.
+  //
+  // The two allowed cases are:
+  //   x = ...      atomic-var = ...
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  // or
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  //   x = ...      atomic-var = ...
+  //
+  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
+  // (which makes more sense, as it captures the original value of the atomic
+  // variable).
+  //
+  // If the two statements don't fit these criteria, return a pair of default-
+  // constructed values.
+  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
+      const parser::ExecutionPartConstruct *>;
+
+  SourcedActionStmt act1{GetActionStmt(ec1)};
+  SourcedActionStmt act2{GetActionStmt(ec2)};
+  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
+  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
+  if (!maybeAssign1 || !maybeAssign2) {
+    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
+      context_.Say(source,
+          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
+    }
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  auto as1{*maybeAssign1}, as2{*maybeAssign2};
+
+  auto isUpdateCapture{
+      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
+        return IsSameOrConvertOf(c.rhs, u.lhs);
+      }};
+
+  // Do some checks that narrow down the possible choices for the update
+  // and the capture statements. This will help to emit better diagnostics.
+  // 1. An assignment could be an update (cbu) if the left-hand side is a
+  //    subexpression of the right-hand side.
+  // 2. An assignment could be a capture (cbc) if the right-hand side is
+  //    a variable (or a function ref), with potential type conversions.
+  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
+  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
+  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
+  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
+
+  // We want to diagnose cases where both assignments cannot be an update,
+  // or both cannot be a capture, as well as cases where either assignment
+  // cannot be any of these two.
+  //
+  // If we organize these boolean values into a matrix
+  //   |cbu1 cbu2|
+  //   |cbc1 cbc2|
+  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
+  // row or column, including the case where everything is zero. All these
+  // cases correspond to the determinant of the matrix being 0, which suggests
+  // that checking the det may be a convenient diagnostic check. There is only
+  // one additional case where the det is 0, which is when the matrix is all 1
+  // ("true"). The "all true" case represents the situation where both
+  // assignments could be an update as well as a capture. On the other hand,
+  // whenever det != 0, the roles of the update and the capture can be
+  // unambiguously assigned to as1 and as2 [1].
+  //
+  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
+  // det = 0, leaving 6 cases where det != 0:
+  //   0 1   0 1   1 0   1 0   1 1   1 1
+  //   1 0   1 1   0 1   1 1   0 1   1 0
+  // In each case the classification is unambiguous.
+
+  //     |cbu1 cbu2|
+  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
+  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
+
+  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
+                                  const std::string &expr) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
+        expr);
+  }};
+
+  auto errorNeitherWorks{[&]() {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
+  }};
+
+  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
+    // If det != 0, then the checks unambiguously suggest a specific
+    // categorization.
+    // If det == 0, then this function should be called only if the
+    // checks haven't ruled out any possibility, i.e. when both assigments
+    // could still be either updates or captures.
+    if (det > 0) {
+      // as1 is update, as2 is capture
+      if (isUpdateCapture(as1, as2)) {
+        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
+      } else {
+        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else if (det < 0) {
+      // as2 is update, as1 is capture
+      if (isUpdateCapture(as2, as1)) {
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      } else {
+        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else {
+      bool updateFirst{isUpdateCapture(as1, as2)};
+      bool captureFirst{isUpdateCapture(as2, as1)};
+      if (updateFirst && captureFirst) {
+        // If both assignment could be the update and both could be the
+        // capture, emit a warning about the ambiguity.
+        context_.Say(act1.source,
+            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      }
+      if (updateFirst != captureFirst) {
+        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
+        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
+        return std::make_pair(upd, cap);
+      }
+      assert(!updateFirst && !captureFirst);
+      errorNeitherWorks();
+      return std::make_pair(nullptr, nullptr);
+    }
+  }};
+
+  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
+    return makeSelectionFromDet(det);
+  }
+  assert(det == 0 && "Prior checks should have covered det != 0");
+
+  // If neither of the statements is an RMW update, it could still be a
+  // "write" update. Pretty much any assignment can be a write update, so
+  // recompute det with cbu1 = cbu2 = true.
+  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
+    return makeSelectionFromDet(writeDet);
+  }
+
+  // It's only errors from here on.
+
+  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
+    errorNeitherWorks();
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // The remaining cases are that
+  // - no candidate for update, or for capture,
+  // - one of the assigments cannot be anything.
+
+  if (!cbu1 && !cbu2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  } else if (!cbc1 && !cbc2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
+    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
+    context_.Say(src,
+        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // All cases should have been covered.
+  llvm_unreachable("Unchecked condition");
+}
+
+void OmpStructureChecker::CheckAtomicCaptureAssignment(
+    const evaluate::Assignment &capture, const SomeExpr &atom,
+    parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &cap{capture.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, rsrc);
+    // This part should have been checked prior to calling this function.
+    assert(*GetConvertInput(capture.rhs) == atom &&
+        "This cannot be a capture assignment");
+    CheckStorageOverlap(atom, {cap}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicReadAssignment(
+    const evaluate::Assignment &read, parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+
+  if (auto maybe{GetConvertInput(read.rhs)}) {
+    const SomeExpr &atom{*maybe};
+
+    if (!IsVarOrFunctionRef(atom)) {
+      ErrorShouldBeVariable(atom, rsrc);
+    } else {
+      CheckAtomicVariable(atom, rsrc);
+      CheckStorageOverlap(atom, {read.lhs}, source);
+    }
+  } else {
+    ErrorShouldBeVariable(read.rhs, rsrc);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWriteAssignment(
+    const evaluate::Assignment &write, parser::CharBlock source) {
+  // [6.0:190:13-15]
+  // A write structured block is write-statement, a write statement that has
+  // one of the following forms:
+  //   x = expr
+  //   x => expr
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{write.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, lsrc);
+    CheckStorageOverlap(atom, {write.rhs}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateAssignment(
+    const evaluate::Assignment &update, parser::CharBlock source) {
+  // [6.0:191:1-7]
+  // An update structured block is update-statement, an update statement
+  // that has one of the following forms:
+  //   x = x operator expr
+  //   x = expr operator x
+  //   x = intrinsic-procedure-name (x)
+  //   x = intrinsic-procedure-name (x, expr-list)
+  //   x = intrinsic-procedure-name (expr-list, x)
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{update.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, lsrc);
+
+  std::pair<operation::Operator, std::vector<SomeExpr>> top{
+      operation::Operator::Unknown, {}};
+  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
+    top = GetTopLevelOperation(*maybeInput);
+  }
+  switch (top.first) {
+  case operation::Operator::Add:
+  case operation::Operator::Sub:
+  case operation::Operator::Mul:
+  case operation::Operator::Div:
+  case operation::Operator::And:
+  case operation::Operator::Or:
+  case operation::Operator::Eqv:
+  case operation::Operator::Neqv:
+  case operation::Operator::Min:
+  case operation::Operator::Max:
+  case operation::Operator::Identity:
+    break;
+  case operation::Operator::Call:
+    context_.Say(source,
+        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Convert:
+    context_.Say(source,
+        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Intrinsic:
+    context_.Say(source,
+        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Constant:
+  case operation::Operator::Unknown:
+    context_.Say(
+        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(source,
+        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
+        operation::ToString(top.first));
+    return;
+  }
+  // Check how many times `atom` occurs as an argument, if it's a subexpression
+  // of an argument, and collect the non-atom arguments.
+  std::vector<SomeExpr> nonAtom;
+  MaybeExpr subExpr;
+  auto atomCount{[&]() {
+    int count{0};
+    for (const SomeExpr &arg : top.second) {
+      if (IsSameOrConvertOf(arg, atom)) {
+        ++count;
+      } else {
+        if (!subExpr && IsSubexpressionOf(atom, arg)) {
+          subExpr = arg;
+        }
+        nonAtom.push_back(arg);
+      }
+    }
+    return count;
+  }()};
+
+  bool hasError{false};
+  if (subExpr) {
+    context_.Say(rsrc,
+        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
+        atom.AsFortran(), subExpr->AsFortran());
+    hasError = true;
+  }
+  if (top.first == operation::Operator::Identity) {
+    // This is "x = y".
+    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
+          atom.AsFortran());
+      hasError = true;
+    }
+  } else {
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    } else if (atomCount > 1) {
+      context_.Say(rsrc,
+          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    }
+  }
+
+  if (!hasError) {
+    CheckStorageOverlap(atom, nonAtom, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
+    const SomeExpr &cond, parser::CharBlock condSource,
+    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
+  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
+  const SomeExpr &atom{assign.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, arsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, alsrc);
+
+  auto top{GetTopLevelOperation(cond)};
+  // Missing arguments to operations would have been diagnosed by now.
+
+  switch (top.first) {
+  case operation::Operator::Associated:
+    if (atom != top.second.front()) {
+      context_.Say(assignSource,
+          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
+    }
+    break;
+  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
+  case operation::Operator::Eq:
+  case operation::Operator::Eqv:
+  // x ordop expr | expr ordop x
+  case operation::Operator::Lt:
+  case operation::Operator::Gt: {
+    const SomeExpr &arg0{top.second[0]};
+    const SomeExpr &arg1{top.second[1]};
+    if (IsSameOrConvertOf(arg0, atom)) {
+      CheckStorageOverlap(atom, {arg1}, condSource);
+    } else if (IsSameOrConvertOf(arg1, atom)) {
+      CheckStorageOverlap(atom, {arg0}, condSource);
+    } else {
+      assert(top.first != operation::Operator::Identity &&
+          "Handle this separately");
+      context_.Say(assignSource,
+          "An argument of the %s operator should be the target of the assignment"_err_en_US,
+          operation::ToString(top.first));
+    }
+    break;
+  }
+  case operation::Operator::Identity:
+  case operation::Operator::True:
+  case operation::Operator::False:
+    break;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(condSource,
+        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
+        operation::ToString(top.first));
+    break;
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
+    const AnalyzedCondStmt &update, parser::CharBlock source) {
+  // The condition/statements must be:
+  // - cond: x equalop e      ift: x =  d     iff: -
+  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
+  // - cond: associated(x)    ift: x => expr  iff: -
+  // - cond: associated(x, e) ift: x => expr  iff: -
+
+  // The if-true statement must be present, and must be an assignment.
+  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
+  if (!maybeAssign) {
+    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
+    } else {
+      context_.Say(
+          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
+    }
+    return;
+  }
+  const evaluate::Assignment assign{*maybeAssign};
+  const SomeExpr &atom{assign.lhs};
+
+  CheckAtomicConditionalUpdateAssignment(
+      update.cond, update.source, assign, update.ift.source);
+
+  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
+
+  if (update.iff) {
+    context_.Say(update.iff.source,
+        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateOnly(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeUpdate->lhs};
+      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdate(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // Allowable forms are (single-statement):
+  // - if ...
+  // - x = (... ? ... : x)
+  // and two-statement:
+  // - r = cond ; if (r) ...
+
+  const parser::ExecutionPartConstruct *ust{nullptr}; // update
+  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
+
+  if (body.size() == 1) {
+    ust = &body.front();
+  } else if (body.size() == 2) {
+    cst = &body.front();
+    ust = &body.back();
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
+    return;
+  }
+
+  // Flang doesn't support conditional-expr yet, so all update statements
+  // are if-statements.
+
+  // IfStmt:        if (...) ...
+  // IfConstruct:   if (...) then ... endif
+  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
+  if (!maybeUpdate) {
+    context_.Say(source,
+        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
+    return;
+  }
+
+  AnalyzedCondStmt &update{*maybeUpdate};
+
+  if (SourcedActionStmt action{GetActionStmt(cst)}) {
+    // The "condition" statement must be `r = cond`.
+    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
+      if (maybeCond->lhs != update.cond) {
+        context_.Say(update.source,
+            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
+            maybeCond->lhs.AsFortran());
+      } else {
+        // If it's "r = ...; if (r) ..." then put the original condition
+        // in `update`.
+        update.cond = maybeCond->rhs;
+      }
+    } else {
+      context_.Say(action.source,
+          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
+    }
+  }
+
+  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
+
+  CheckAtomicConditionalUpdateStmt(update, source);
+  if (IsCheckForAssociated(update.cond)) {
+    if (!IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
+    }
+  } else {
+    if (IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
+    }
+  }
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
+      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
+      MakeAtomicAnalysisOp(Analysis::None));
+}
+
+void OmpStructureChecker::CheckAtomicUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() != 2) {
+    context_.Say(source,
+        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
+    return;
+  }
+
+  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
+  if (!uec || !cec) {
+    // Diagnostics already emitted.
+    return;
+  }
+  SourcedActionStmt uact{GetActionStmt(uec)};
+  SourcedActionStmt cact{GetActionStmt(cec)};
+  // The "dereferences" of std::optional are guaranteed to be valid after
+  // CheckUpdateCapture.
+  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
+  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
+
+  const SomeExpr &atom{update.lhs};
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  int action;
+
+  if (IsMaybeAtomicWrite(update)) {
+    action = Analysis::Write;
+    CheckAtomicWriteAssignment(update, uact.source);
+  } else {
+    action = Analysis::Update;
+    CheckAtomicUpdateAssignment(update, uact.source);
+  }
+  CheckAtomicCaptureAssignment(capture, atom, cact.source);
+
+  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
+    context_.Say(cact.source,
+        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+    return;
+  }
+
+  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(action, update),
+        MakeAtomicAnalysisOp(Analysis::Read, capture));
+  } else {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(Analysis::Read, capture),
+        MakeAtomicAnalysisOp(action, update));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // There are two 
diff erent variants of this:
+  // (1) conditional-update and capture separately:
+  //     This form only allows single-statement updates, i.e. the update
+  //     form "r = cond; if (r) ..." is not allowed.
+  // (2) conditional-update combined with capture in a single statement:
+  //     This form does allow the condition to be calculated separately,
+  //     i.e. "r = cond; if (r) ...".
+  // Regardless of what form it is, the actual update assignment is a
+  // proper write, i.e. "x = d", where d does not depend on x.
+
+  AnalyzedCondStmt update;
+  SourcedActionStmt capture;
+  bool captureAlways{true}, captureFirst{true};
+
+  auto extractCapture{[&]() {
+    capture = update.iff;
+    captureAlways = false;
+    update.iff = SourcedActionStmt{};
+  }};
+
+  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
+    // The non-update statement is either "r = cond" or the capture.
+    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
+      if (update.cond == maybeAssign->lhs) {
+        // If this is "r = cond; if (r) ...", then update the condition.
+        update.cond = maybeAssign->rhs;
+        update.source = action.source;
+        // In this form, the update and the capture are combined into
+        // an IF-THEN-ELSE statement.
+        extractCapture();
+      } else {
+        // Assume this is the capture-statement.
+        capture = action;
+      }
+    }
+  }};
+
+  if (body.size() == 2) {
+    // This could be
+    // - capture; conditional-update (in any order), or
+    // - r = cond; if (r) capture-update
+    const parser::ExecutionPartConstruct *st1{&body.front()};
+    const parser::ExecutionPartConstruct *st2{&body.back()};
+    // In either case, the conditional statement can be analyzed by
+    // AnalyzeConditionalStmt, whereas the other statement cannot.
+    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
+      update = *maybeUpdate1;
+      classifyNonUpdate(GetActionStmt(st2));
+      captureFirst = false;
+    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
+      update = *maybeUpdate2;
+      classifyNonUpdate(GetActionStmt(st1));
+    } else {
+      // None of the statements are conditional, this rules out the
+      // "r = cond; if (r) ..." and the "capture + conditional-update"
+      // variants. This could still be capture + write (which is classified
+      // as conditional-update-capture in the spec).
+      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
+      if (!uec || !cec) {
+        // Diagnostics already emitted.
+        return;
+      }
+      SourcedActionStmt uact{GetActionStmt(uec)};
+      SourcedActionStmt cact{GetActionStmt(cec)};
+      update.ift = uact;
+      capture = cact;
+      if (uec == st1) {
+        captureFirst = false;
+      }
+    }
+  } else if (body.size() == 1) {
+    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
+      update = *maybeUpdate;
+      // This is the form with update and capture combined into an IF-THEN-ELSE
+      // statement. The capture-statement is always the ELSE branch.
+      extractCapture();
+    } else {
+      goto invalid;
+    }
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
+    return;
+  invalid:
+    context_.Say(source,
+        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
+    return;
+  }
+
+  // The update must have a form `x = d` or `x => d`.
+  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
+    const SomeExpr &atom{maybeWrite->lhs};
+    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
+    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
+      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
+
+      if (IsPointerAssignment(*maybeWrite) !=
+          IsPointerAssignment(*maybeCapture)) {
+        context_.Say(capture.source,
+            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+        return;
+      }
+    } else {
+      if (!IsAssignment(capture.stmt)) {
+        context_.Say(capture.source,
+            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
+      }
+      return;
+    }
+  } else {
+    if (!IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
+    }
+    return;
+  }
+
+  // update.iff should be empty here, the capture statement should be
+  // stored in "capture".
+
+  // Fill out the analysis in the AST node.
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  bool condUnused{std::visit(
+      [](auto &&s) {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
+          return true;
+        } else {
+          return false;
+        }
+      },
+      update.cond.u)};
+
+  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
+  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
+
+  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
+  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
+
+  if (captureFirst) {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
+  } else {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicRead(
+    const parser::OpenMPAtomicConstruct &x) {
+  // [6.0:190:5-7]
+  // A read structured block is read-statement, a read statement that has one
+  // of the following forms:
+  //   v = x
+  //   v => x
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Read cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
+      CheckAtomicReadAssignment(*maybeRead, action.source);
+
+      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
+        const SomeExpr &atom{*maybe};
+        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
+            MakeAtomicAnalysisOp(Analysis::None));
+      }
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC READ operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWrite(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Write cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeWrite->lhs};
+      CheckAtomicWriteAssignment(*maybeWrite, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC WRITE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdate(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &block{std::get<parser::Block>(x.t)};
+
+  bool isConditional{x.IsCompare()};
+  bool isCapture{x.IsCapture()};
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (isConditional && isCapture) {
+    CheckAtomicConditionalUpdateCapture(x, body, x.source);
+  } else if (isConditional) {
+    CheckAtomicConditionalUpdate(x, body, x.source);
+  } else if (isCapture) {
+    CheckAtomicUpdateCapture(x, body, x.source);
+  } else { // update-only
+    CheckAtomicUpdateOnly(x, body, x.source);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
+  if (visitedAtomicSource_.empty())
+    visitedAtomicSource_ = x.source;
+
+  // All of the following groups have the "exclusive" property, i.e. at
+  // most one clause from each group is allowed.
+  // The exclusivity-checking code should eventually be unified for all
+  // clauses, with clause groups defined in OMP.td.
+  std::array atomic{llvm::omp::Clause::OMPC_read,
+      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
+  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
+      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
+      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
+
+  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
+                          std::string_view name,
+                          const parser::OmpClauseList &clauses) {
+    const parser::OmpClause *present{nullptr};
+    for (const parser::OmpClause &clause : clauses.v) {
+      llvm::omp::Clause id{clause.Id()};
+      if (!llvm::is_contained(group, id)) {
+        continue;
+      }
+      if (present == nullptr) {
+        present = &clause;
+        continue;
+      } else if (id == present->Id()) {
+        // Ignore repetitions of the same clause, those will be diagnosed
+        // separately.
+        continue;
+      }
+      parser::MessageFormattedText txt(
+          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
+          name.data());
+      parser::Message message(clause.source, txt);
+      message.Attach(present->source,
+          "Previous clause from this group provided here"_en_US);
+      context_.Say(std::move(message));
+      return;
+    }
+  }};
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
+  llvm::omp::Clause kind{x.GetKind()};
+
+  checkExclusive(atomic, "atomic", dirSpec.Clauses());
+  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
+
+  switch (kind) {
+  case llvm::omp::Clause::OMPC_read:
+    CheckAtomicRead(x);
+    break;
+  case llvm::omp::Clause::OMPC_write:
+    CheckAtomicWrite(x);
+    break;
+  case llvm::omp::Clause::OMPC_update:
+    CheckAtomicUpdate(x);
+    break;
+  default:
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics

diff  --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
new file mode 100644
index 0000000000000..b82e2f7342d85
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -0,0 +1,671 @@
+//===-- lib/Semantics/check-omp-loop.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks for constructs and clauses related to loops.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <variant>
+
+namespace {
+using namespace Fortran;
+
+class AssociatedLoopChecker {
+public:
+  AssociatedLoopChecker(
+      semantics::SemanticsContext &context, std::int64_t level)
+      : context_{context}, level_{level} {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::DoConstruct &dc) {
+    level_--;
+    const auto &doStmt{
+        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
+    const auto &constructName{
+        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
+    if (constructName) {
+      constructNamesAndLevels_.emplace(
+          constructName.value().ToString(), level_);
+    }
+    if (level_ >= 0) {
+      if (dc.IsDoWhile()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
+      }
+      if (!dc.GetLoopControl()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
+      }
+    }
+    return true;
+  }
+
+  void Post(const parser::DoConstruct &dc) { level_++; }
+
+  bool Pre(const parser::CycleStmt &cyclestmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (cyclestmt.v) {
+      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second > 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ > 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::ExitStmt &exitStmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (exitStmt.v) {
+      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ >= 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "EXIT statement terminates associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
+    source_ = &actionstmt.source;
+    return true;
+  }
+
+private:
+  semantics::SemanticsContext &context_;
+  const parser::CharBlock *source_;
+  std::int64_t level_;
+  std::map<std::string, std::int64_t> constructNamesAndLevels_;
+};
+} // namespace
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::HasInvalidDistributeNesting(
+    const parser::OpenMPLoopConstruct &x) {
+  bool violation{false};
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
+    // `distribute` region has to be nested
+    if (!CurrentDirectiveIsNested()) {
+      violation = true;
+    } else {
+      // `distribute` region has to be strictly nested inside `teams`
+      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+        violation = true;
+      }
+    }
+  }
+  if (violation) {
+    context_.Say(beginDir.source,
+        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
+        "region."_err_en_US);
+  }
+}
+void OmpStructureChecker::HasInvalidLoopBinding(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
+    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+    for (const auto &clause : clauseList.v) {
+      if (const auto *bindClause{
+              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
+        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
+          context_.Say(beginDir.source, msg);
+        }
+      }
+    }
+  };
+
+  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
+      CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
+        "strictly nested inside a `TEAMS` region."_err_en_US);
+  }
+
+  if (OmpDirectiveSet{
+          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
+          .test(beginDir.v)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
+        "combined with a `TEAMS` construct."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
+  // Check the following:
+  //  The only OpenMP constructs that can be encountered during execution of
+  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
+  // construct and the `ordered` construct with the `simd` clause.
+
+  // Check if the parent context has the SIMD clause
+  // Please note that we use GetContext() instead of GetContextParent()
+  // because PushContextAndClauseSets() has not been called on the
+  // current context yet.
+  // TODO: Check for declare simd regions.
+  bool eligibleSIMD{false};
+  common::visit(
+      common::visitors{
+          // Allow `!$OMP ORDERED SIMD`
+          [&](const parser::OpenMPBlockConstruct &c) {
+            const auto &beginBlockDir{
+                std::get<parser::OmpBeginBlockDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
+            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
+              const auto &clauses{
+                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
+              for (const auto &clause : clauses.v) {
+                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
+                  eligibleSIMD = true;
+                  break;
+                }
+              }
+            }
+          },
+          [&](const parser::OpenMPStandaloneConstruct &c) {
+            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
+                    &c.u)}) {
+              llvm::omp::Directive dirId{ssc->v.DirId()};
+              if (dirId == llvm::omp::Directive::OMPD_ordered) {
+                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
+                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
+                    eligibleSIMD = true;
+                    break;
+                  }
+                }
+              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
+                eligibleSIMD = true;
+              }
+            }
+          },
+          // Allowing SIMD and loop construct
+          [&](const parser::OpenMPLoopConstruct &c) {
+            const auto &beginLoopDir{
+                std::get<parser::OmpBeginLoopDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
+              eligibleSIMD = true;
+            }
+          },
+          [&](const parser::OpenMPAtomicConstruct &c) {
+            // Allow `!$OMP ATOMIC`
+            eligibleSIMD = true;
+          },
+          [&](const auto &c) {},
+      },
+      c.u);
+  if (!eligibleSIMD) {
+    context_.Say(parser::FindSourceLocation(c),
+        "The only OpenMP constructs that can be encountered during execution "
+        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
+        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
+        "construct with the `SIMD` clause."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
+  loopStack_.push_back(&x);
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  PushContextAndClauseSets(beginDir.source, beginDir.v);
+
+  // check matching, End directive is optional
+  if (const auto &endLoopDir{
+          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
+    const auto &endDir{
+        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
+
+    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
+
+    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
+  }
+
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    EnterDirectiveNest(SIMDNest);
+  }
+
+  // Combined target loop constructs are target device constructs. Keep track of
+  // whether any such construct has been visited to later check that REQUIRES
+  // directives for target-related options don't appear after them.
+  if (llvm::omp::allTargetSet.test(beginDir.v)) {
+    deviceConstructFound_ = true;
+  }
+
+  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
+    // 2.7.1 do-clause -> private-clause |
+    //                    firstprivate-clause |
+    //                    lastprivate-clause |
+    //                    linear-clause |
+    //                    reduction-clause |
+    //                    schedule-clause |
+    //                    collapse-clause |
+    //                    ordered-clause
+
+    // nesting check
+    HasInvalidWorksharingNesting(
+        beginDir.source, llvm::omp::nestedWorkshareErrSet);
+  }
+  SetLoopInfo(x);
+
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &doConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
+      CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+    }
+  }
+  CheckLoopItrVariableIsInt(x);
+  CheckAssociatedLoopConstraints(x);
+  HasInvalidDistributeNesting(x);
+  HasInvalidLoopBinding(x);
+  if (CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
+  }
+  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
+      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
+    CheckDistLinear(x);
+  }
+}
+
+const parser::Name OmpStructureChecker::GetLoopIndex(
+    const parser::DoConstruct *x) {
+  using Bounds = parser::LoopControl::Bounds;
+  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
+}
+
+void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const parser::DoConstruct *loop{&*loopConstruct};
+      if (loop && loop->IsDoNormal()) {
+        const parser::Name &itrVal{GetLoopIndex(loop)};
+        SetLoopIv(itrVal.symbol);
+      }
+    }
+  }
+}
+
+void OmpStructureChecker::CheckLoopItrVariableIsInt(
+    const parser::OpenMPLoopConstruct &x) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+
+      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+        if (loop->IsDoNormal()) {
+          const parser::Name &itrVal{GetLoopIndex(loop)};
+          if (itrVal.symbol) {
+            const auto *type{itrVal.symbol->GetType()};
+            if (!type->IsNumeric(TypeCategory::Integer)) {
+              context_.Say(itrVal.source,
+                  "The DO loop iteration"
+                  " variable must be of the type integer."_err_en_US,
+                  itrVal.ToString());
+            }
+          }
+        }
+        // Get the next DoConstruct if block is not empty.
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                 : nullptr;
+      }
+    }
+  }
+}
+
+std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+  std::int64_t orderedCollapseLevel{1};
+  std::int64_t orderedLevel{1};
+  std::int64_t collapseLevel{1};
+
+  for (const auto &clause : clauseList.v) {
+    if (const auto *collapseClause{
+            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
+      if (const auto v{GetIntValue(collapseClause->v)}) {
+        collapseLevel = *v;
+      }
+    }
+    if (const auto *orderedClause{
+            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
+      if (const auto v{GetIntValue(orderedClause->v)}) {
+        orderedLevel = *v;
+      }
+    }
+  }
+  if (orderedLevel >= collapseLevel) {
+    orderedCollapseLevel = orderedLevel;
+  } else {
+    orderedCollapseLevel = collapseLevel;
+  }
+  return orderedCollapseLevel;
+}
+
+void OmpStructureChecker::CheckAssociatedLoopConstraints(
+    const parser::OpenMPLoopConstruct &x) {
+  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
+  AssociatedLoopChecker checker{context_, ordCollapseLevel};
+  parser::Walk(x, checker);
+}
+
+void OmpStructureChecker::CheckDistLinear(
+    const parser::OpenMPLoopConstruct &x) {
+
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  SymbolSourceMap indexVars;
+
+  // Collect symbols of all the variables from linear clauses
+  for (auto &clause : clauses.v) {
+    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
+      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
+      GetSymbolsInObjectList(objects, indexVars);
+    }
+  }
+
+  if (!indexVars.empty()) {
+    // Get collapse level, if given, to find which loops are "associated."
+    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
+    // Include the top loop if no collapse is specified
+    if (collapseVal == 0) {
+      collapseVal = 1;
+    }
+
+    // Match the loop index variables with the collected symbols from linear
+    // clauses.
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &loopConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+          if (loop->IsDoNormal()) {
+            const parser::Name &itrVal{GetLoopIndex(loop)};
+            if (itrVal.symbol) {
+              // Remove the symbol from the collected set
+              indexVars.erase(&itrVal.symbol->GetUltimate());
+            }
+            collapseVal--;
+            if (collapseVal == 0) {
+              break;
+            }
+          }
+          // Get the next DoConstruct if block is not empty.
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                   : nullptr;
+        }
+      }
+    }
+
+    // Show error for the remaining variables
+    for (auto &[symbol, source] : indexVars) {
+      const Symbol &root{GetAssociationRoot(*symbol)};
+      context_.Say(source,
+          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
+          root.name());
+    }
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  // A few semantic checks for InScan reduction are performed below as SCAN
+  // constructs inside LOOP may add the relevant information. Scan reduction is
+  // supported only in loop constructs, so same checks are not applicable to
+  // other directives.
+  using ReductionModifier = parser::OmpReductionModifier;
+  for (const auto &clause : clauseList.v) {
+    if (const auto *reductionClause{
+            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
+      auto &modifiers{OmpGetModifiers(reductionClause->v)};
+      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
+      if (maybeModifier &&
+          maybeModifier->v == ReductionModifier::Value::Inscan) {
+        const auto &objectList{
+            std::get<parser::OmpObjectList>(reductionClause->v.t)};
+        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
+          if (auto &symbol = name->symbol) {
+            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
+                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
+              context_.Say(name->source,
+                  "List item %s must appear in EXCLUSIVE or "
+                  "INCLUSIVE clause of an "
+                  "enclosed SCAN directive"_err_en_US,
+                  name->ToString());
+            }
+          }
+        };
+        for (const auto &ompObj : objectList.v) {
+          common::visit(
+              common::visitors{
+                  [&](const parser::Designator &designator) {
+                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
+                            designator)}) {
+                      checkReductionSymbolInScan(name);
+                    }
+                  },
+                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
+              },
+              ompObj.u);
+        }
+      }
+    }
+  }
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    ExitDirectiveNest(SIMDNest);
+  }
+  dirContext_.pop_back();
+
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top{loopStack_.back()};
+  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
+  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+}
+
+void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
+  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  // 2.7.1 end-do -> END DO [nowait-clause]
+  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
+  case llvm::omp::Directive::OMPD_do:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
+    break;
+  case llvm::omp::Directive::OMPD_do_simd:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
+    break;
+  default:
+    // no clauses are allowed
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
+    dirContext_.pop_back();
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
+  unsigned version{context_.langOptions().OpenMPVersion};
+  llvm::omp::Directive dir{GetContext().directive};
+  parser::CharBlock clauseSource{GetContext().clauseSource};
+  const parser::OmpLinearModifier *linearMod{nullptr};
+
+  SymbolSourceMap symbols;
+  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  CheckCrayPointee(objects, "LINEAR", false);
+  GetSymbolsInObjectList(objects, symbols);
+
+  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
+    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      context_.Say(source,
+          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
+          symbol->name(), desc.name.str());
+    }
+  }};
+
+  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(x.v)};
+    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
+    if (linearMod) {
+      // 2.7 Loop Construct Restriction
+      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
+        context_.Say(clauseSource,
+            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
+            ContextDirectiveAsFortran());
+        return;
+      }
+
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      for (auto &[symbol, source] : symbols) {
+        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
+          CheckIntegerNoRef(symbol, source);
+        } else {
+          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
+              !IsPolymorphic(*symbol)) {
+            context_.Say(source,
+                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
+                symbol->name(), desc.name.str());
+          }
+        }
+        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
+            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
+          if (!IsDummy(*symbol) || IsValue(*symbol)) {
+            context_.Say(source,
+                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
+                desc.name.str(), symbol->name());
+          }
+        }
+      } // for (symbol, source)
+
+      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
+        context_.Say(OmpGetModifierSource(modifiers, linearMod),
+            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
+            ThisVersion(version));
+      }
+    }
+  }
+
+  // OpenMP 5.2: Ordered clause restriction
+  if (const auto *clause{
+          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
+    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
+    if (orderedClause.v) {
+      return;
+    }
+  }
+
+  // OpenMP 5.2: Linear clause Restrictions
+  for (auto &[symbol, source] : symbols) {
+    if (!linearMod) {
+      // Already checked this with the modifier present.
+      CheckIntegerNoRef(symbol, source);
+    }
+    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
+      context_.Say(source,
+          "The list item `%s` must be a dummy argument"_err_en_US,
+          symbol->name());
+    }
+    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
+      context_.Say(source,
+          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
+          symbol->name());
+    }
+    if (FindCommonBlockContaining(*symbol)) {
+      context_.Say(source,
+          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
+          symbol->name());
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
+  Base::Enter(x);
+  loopStack_.push_back(&x);
+}
+
+void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top = loopStack_.back();
+  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
+  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+  Base::Leave(x);
+}
+
+} // namespace Fortran::semantics

diff  --git a/flang/lib/Semantics/check-omp-metadirective.cpp b/flang/lib/Semantics/check-omp-metadirective.cpp
new file mode 100644
index 0000000000000..03487da64f1bf
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-metadirective.cpp
@@ -0,0 +1,548 @@
+//===-- lib/Semantics/check-omp-metadirective.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks for METADIRECTIVE and related constructs/clauses.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <variant>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
+  OmpVerifyModifiers(
+      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
+}
+
+void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
+  EnterDirectiveNest(ContextSelectorNest);
+
+  using SetName = parser::OmpTraitSetSelectorName;
+  std::map<SetName::Value, const SetName *> visited;
+
+  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
+    auto &name{std::get<SetName>(traitSet.t)};
+    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait set name %s in a context specifier"_err_en_US,
+          showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait set %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+    CheckTraitSetSelector(traitSet);
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
+  ExitDirectiveNest(ContextSelectorNest);
+}
+
+const std::list<parser::OmpTraitProperty> &
+OmpStructureChecker::GetTraitPropertyList(
+    const parser::OmpTraitSelector &trait) {
+  static const std::list<parser::OmpTraitProperty> empty{};
+  auto &[_, maybeProps]{trait.t};
+  if (maybeProps) {
+    using PropertyList = std::list<parser::OmpTraitProperty>;
+    return std::get<PropertyList>(maybeProps->t);
+  } else {
+    return empty;
+  }
+}
+
+std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
+    const parser::OmpTraitProperty &property) {
+  using MaybeClause = std::optional<llvm::omp::Clause>;
+
+  // The parser for OmpClause will only succeed if the clause was
+  // given with all required arguments.
+  // If this is a string or complex extension with a clause name,
+  // treat it as a clause and let the trait checker deal with it.
+
+  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
+    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
+    if (id != llvm::omp::Clause::OMPC_unknown) {
+      return id;
+    } else {
+      return std::nullopt;
+    }
+  }};
+
+  return common::visit( //
+      common::visitors{
+          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
+            return getClauseFromString(x.v);
+          },
+          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
+            return x.value().Id();
+          },
+          [&](const parser::ScalarExpr &x) -> MaybeClause {
+            return std::nullopt;
+          },
+          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
+            using ExtProperty = parser::OmpTraitPropertyExtension;
+            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
+              return getClauseFromString(name->v);
+            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
+              return getClauseFromString(
+                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
+            }
+            return std::nullopt;
+          },
+      },
+      property.u);
+}
+
+void OmpStructureChecker::CheckTraitSelectorList(
+    const std::list<parser::OmpTraitSelector> &traits) {
+  // [6.0:322:20]
+  // Each trait-selector-name may only be specified once in a trait selector
+  // set.
+
+  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
+  using TraitName = parser::OmpTraitSelectorName;
+  using BareName = decltype(TraitName::u);
+  std::map<BareName, const TraitName *> visited;
+
+  for (const parser::OmpTraitSelector &trait : traits) {
+    auto &name{std::get<TraitName>(trait.t)};
+
+    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait name %s in a trait set"_err_en_US, showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSetSelector(
+    const parser::OmpTraitSetSelector &traitSet) {
+
+  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
+  //
+  // Construct      |     Simd, directive-name |      Yes |       No |    No |
+  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
+  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
+  //                |      Extension, Requires |          |          |       |
+  //                |                   Vendor |          |          |       |
+  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
+  //                |                Kind, Uid |          |          |       |
+  // User           |                Condition |       No |       No |   Yes |
+
+  struct TraitSetConfig {
+    std::set<parser::OmpTraitSelectorName::Value> allowed;
+    bool allowsDirectiveTraits;
+    bool allowsExtensionTraits;
+    bool allowsScore;
+  };
+
+  using SName = parser::OmpTraitSetSelectorName::Value;
+  using TName = parser::OmpTraitSelectorName::Value;
+
+  static const std::map<SName, TraitSetConfig> configs{
+      {SName::Construct, //
+          {{TName::Simd}, true, false, false}},
+      {SName::Device, //
+          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
+      {SName::Implementation, //
+          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
+               TName::Vendor},
+              false, true, true}},
+      {SName::Target_Device, //
+          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
+               TName::Uid},
+              false, true, false}},
+      {SName::User, //
+          {{TName::Condition}, false, false, true}},
+  };
+
+  auto checkTraitSet{[&](const TraitSetConfig &config) {
+    auto &[setName, traits]{traitSet.t};
+    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
+
+    // Check if there are any duplicate traits.
+    CheckTraitSelectorList(traits);
+
+    for (const parser::OmpTraitSelector &trait : traits) {
+      // Don't use structured bindings here, because they cannot be captured
+      // before C++20.
+      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
+      auto &maybeProps =
+          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
+              trait.t);
+
+      // Check allowed traits
+      common::visit( //
+          common::visitors{
+              [&](parser::OmpTraitSelectorName::Value v) {
+                if (!config.allowed.count(v)) {
+                  context_.Say(traitName.source,
+                      "%s is not a valid trait for %s trait set"_err_en_US,
+                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
+                }
+              },
+              [&](llvm::omp::Directive) {
+                if (!config.allowsDirectiveTraits) {
+                  context_.Say(traitName.source,
+                      "Directive name is not a valid trait for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+              [&](const std::string &) {
+                if (!config.allowsExtensionTraits) {
+                  context_.Say(traitName.source,
+                      "Extension traits are not valid for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+          },
+          traitName.u);
+
+      // Check score
+      if (maybeProps) {
+        auto &[maybeScore, _]{maybeProps->t};
+        if (maybeScore) {
+          CheckTraitScore(*maybeScore);
+        }
+      }
+
+      // Check the properties of the individual traits
+      CheckTraitSelector(traitSet, trait);
+    }
+  }};
+
+  checkTraitSet(
+      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
+}
+
+void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
+  // [6.0:322:23]
+  // A score-expression must be a non-negative constant integer expression.
+  if (auto value{GetIntValue(score)}; !value || value < 0) {
+    context_.Say(score.source,
+        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
+  }
+}
+
+bool OmpStructureChecker::VerifyTraitPropertyLists(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  using PropertyList = std::list<parser::OmpTraitProperty>;
+  auto &[traitName, maybeProps]{trait.t};
+
+  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
+                             const std::string &message) {
+    bool foundInvalid{false};
+    for (const parser::OmpTraitProperty &prop : properties) {
+      if (!isValid(prop)) {
+        if (foundInvalid) {
+          context_.Say(
+              prop.source, "More invalid properties are present"_err_en_US);
+          break;
+        }
+        context_.Say(prop.source, "%s"_err_en_US, message);
+        foundInvalid = true;
+      }
+    }
+    return !foundInvalid;
+  }};
+
+  bool invalid{false};
+
+  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
+    // Directive-name traits don't have properties.
+    if (maybeProps) {
+      context_.Say(trait.source,
+          "Directive-name traits cannot have properties"_err_en_US);
+      invalid = true;
+    }
+  }
+  // Ignore properties on extension traits.
+
+  // See `TraitSelectorParser` in openmp-parser.cpp
+  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
+    switch (*v) {
+    // name-list properties
+    case parser::OmpTraitSelectorName::Value::Arch:
+    case parser::OmpTraitSelectorName::Value::Extension:
+    case parser::OmpTraitSelectorName::Value::Isa:
+    case parser::OmpTraitSelectorName::Value::Kind:
+    case parser::OmpTraitSelectorName::Value::Uid:
+    case parser::OmpTraitSelectorName::Value::Vendor:
+      if (maybeProps) {
+        auto isName{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isName, "Trait property should be a name");
+      }
+      break;
+    // clause-list
+    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
+    case parser::OmpTraitSelectorName::Value::Requires:
+    case parser::OmpTraitSelectorName::Value::Simd:
+      if (maybeProps) {
+        auto isClause{[&](const parser::OmpTraitProperty &prop) {
+          return GetClauseFromProperty(prop).has_value();
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isClause, "Trait property should be a clause");
+      }
+      break;
+    // expr-list
+    case parser::OmpTraitSelectorName::Value::Condition:
+    case parser::OmpTraitSelectorName::Value::Device_Num:
+      if (maybeProps) {
+        auto isExpr{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::ScalarExpr>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isExpr, "Trait property should be a scalar expression");
+      }
+      break;
+    } // switch
+  }
+
+  return !invalid;
+}
+
+void OmpStructureChecker::CheckTraitSelector(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  auto &[traitName, maybeProps]{trait.t};
+
+  // Only do the detailed checks if the property lists are valid.
+  if (VerifyTraitPropertyLists(traitSet, trait)) {
+    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
+        std::holds_alternative<std::string>(traitName.u)) {
+      // No properties here: directives don't have properties, and
+      // we don't implement any extension traits now.
+      return;
+    }
+
+    // Specific traits we want to check.
+    // Limitations:
+    // (1) The properties for these traits are defined in "Additional
+    // Definitions for the OpenMP API Specification". It's not clear how
+    // to define them in a portable way, and how to verify their validity,
+    // especially if they get replaced by their integer values (in case
+    // they are defined as enums).
+    // (2) These are entirely implementation-defined, and at the moment
+    // there is no known schema to validate these values.
+    auto v{std::get<TraitName::Value>(traitName.u)};
+    switch (v) {
+    case TraitName::Value::Arch:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Atomic_Default_Mem_Order:
+      CheckTraitADMO(traitSet, trait);
+      break;
+    case TraitName::Value::Condition:
+      CheckTraitCondition(traitSet, trait);
+      break;
+    case TraitName::Value::Device_Num:
+      CheckTraitDeviceNum(traitSet, trait);
+      break;
+    case TraitName::Value::Extension:
+      // Ignore
+      break;
+    case TraitName::Value::Isa:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Kind:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Requires:
+      CheckTraitRequires(traitSet, trait);
+      break;
+    case TraitName::Value::Simd:
+      CheckTraitSimd(traitSet, trait);
+      break;
+    case TraitName::Value::Uid:
+      // Unchecked, TBD(2)
+      break;
+    case TraitName::Value::Vendor:
+      // Unchecked, TBD(1)
+      break;
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitADMO(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single clause property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto clauseId{*GetClauseFromProperty(property)};
+    // Check that the clause belongs to the memory-order clause-set.
+    // Clause sets will hopefully be autogenerated at some point.
+    switch (clauseId) {
+    case llvm::omp::Clause::OMPC_acq_rel:
+    case llvm::omp::Clause::OMPC_acquire:
+    case llvm::omp::Clause::OMPC_relaxed:
+    case llvm::omp::Clause::OMPC_release:
+    case llvm::omp::Clause::OMPC_seq_cst:
+      break;
+    default:
+      context_.Say(property.source,
+          "%s trait requires a clause from the memory-order clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitCondition(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
+
+    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
+    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
+      context_.Say(property.source,
+          "%s trait requires a single LOGICAL expression"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitDeviceNum(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  }
+  // No other checks at the moment.
+}
+
+void OmpStructureChecker::CheckTraitRequires(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_requires, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause from the requirement clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSimd(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_declare_simd, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()),
+          parser::ToUpperCaseLetters(
+              getDirectiveName(llvm::omp::OMPD_declare_simd)));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
+  EnterDirectiveNest(MetadirectiveNest);
+  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
+}
+
+void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
+  ExitDirectiveNest(MetadirectiveNest);
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics

diff  --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index e080bce3cac3a..30eff01256c61 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -7,27 +7,56 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
 #include "definable.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
-#include "flang/Evaluate/check-expression.h"
-#include "flang/Evaluate/expression.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
 #include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/scope.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+#include "flang/Support/Fortran-features.h"
+
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <iterator>
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
 #include <variant>
 
 namespace Fortran::semantics {
 
-template <typename T, typename U>
-static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
-  return !(e == f);
-}
+using namespace Fortran::semantics::omp;
 
 // Use when clause falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
@@ -53,66 +82,6 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
     CheckAllowedClause(llvm::omp::Y); \
   }
 
-std::string ThisVersion(unsigned version) {
-  std::string tv{
-      std::to_string(version / 10) + "." + std::to_string(version % 10)};
-  return "OpenMP v" + tv;
-}
-
-std::string TryVersion(unsigned version) {
-  return "try -fopenmp-version=" + std::to_string(version);
-}
-
-static const parser::Designator *GetDesignatorFromObj(
-    const parser::OmpObject &object) {
-  return std::get_if<parser::Designator>(&object.u);
-}
-
-static const parser::DataRef *GetDataRefFromObj(
-    const parser::OmpObject &object) {
-  if (auto *desg{GetDesignatorFromObj(object)}) {
-    return std::get_if<parser::DataRef>(&desg->u);
-  }
-  return nullptr;
-}
-
-static const parser::ArrayElement *GetArrayElementFromObj(
-    const parser::OmpObject &object) {
-  if (auto *dataRef{GetDataRefFromObj(object)}) {
-    using ElementIndirection = common::Indirection<parser::ArrayElement>;
-    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
-      return &ind->value();
-    }
-  }
-  return nullptr;
-}
-
-static bool IsVarOrFunctionRef(const MaybeExpr &expr) {
-  if (expr) {
-    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
-        evaluate::IsVariable(*expr);
-  } else {
-    return false;
-  }
-}
-
-static std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
-  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
-  // ForwardOwningPointer           typedExpr
-  // `- GenericExprWrapper          ^.get()
-  //    `- std::optional<Expr>      ^->v
-  return typedExpr.get()->v;
-}
-
-static std::optional<evaluate::DynamicType> GetDynamicType(
-    const parser::Expr &parserExpr) {
-  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
-    return maybeExpr->GetType();
-  } else {
-    return std::nullopt;
-  }
-}
-
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
 // statements and the expressions enclosed in an OpenMP Workshare construct
 class OmpWorkshareBlockChecker {
@@ -172,85 +141,6 @@ class OmpWorkshareBlockChecker {
   parser::CharBlock source_;
 };
 
-class AssociatedLoopChecker {
-public:
-  AssociatedLoopChecker(SemanticsContext &context, std::int64_t level)
-      : context_{context}, level_{level} {}
-
-  template <typename T> bool Pre(const T &) { return true; }
-  template <typename T> void Post(const T &) {}
-
-  bool Pre(const parser::DoConstruct &dc) {
-    level_--;
-    const auto &doStmt{
-        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
-    const auto &constructName{
-        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
-    if (constructName) {
-      constructNamesAndLevels_.emplace(
-          constructName.value().ToString(), level_);
-    }
-    if (level_ >= 0) {
-      if (dc.IsDoWhile()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
-      }
-      if (!dc.GetLoopControl()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
-      }
-    }
-    return true;
-  }
-
-  void Post(const parser::DoConstruct &dc) { level_++; }
-
-  bool Pre(const parser::CycleStmt &cyclestmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (cyclestmt.v) {
-      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second > 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ > 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::ExitStmt &exitStmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (exitStmt.v) {
-      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ >= 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "EXIT statement terminates associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
-    source_ = &actionstmt.source;
-    return true;
-  }
-
-private:
-  SemanticsContext &context_;
-  const parser::CharBlock *source_;
-  std::int64_t level_;
-  std::map<std::string, std::int64_t> constructNamesAndLevels_;
-};
-
 // `OmpUnitedTaskDesignatorChecker` is used to check if the designator
 // can appear within the TASK construct
 class OmpUnitedTaskDesignatorChecker {
@@ -318,18 +208,6 @@ bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
   return CheckAllowed(clause);
 }
 
-bool OmpStructureChecker::IsCommonBlock(const Symbol &sym) {
-  return sym.detailsIf<CommonBlockDetails>() != nullptr;
-}
-
-bool OmpStructureChecker::IsVariableListItem(const Symbol &sym) {
-  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
-}
-
-bool OmpStructureChecker::IsExtendedListItem(const Symbol &sym) {
-  return IsVariableListItem(sym) || sym.IsSubprogram();
-}
-
 bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   // Definition of close nesting:
   //
@@ -371,60 +249,6 @@ bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   return false;
 }
 
-namespace {
-struct ContiguousHelper {
-  ContiguousHelper(SemanticsContext &context)
-      : fctx_(context.foldingContext()) {}
-
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
-    return Visit(x.value());
-  }
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Reference<Contained> &x) {
-    return Visit(x.get());
-  }
-  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
-    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
-  }
-  template <typename T>
-  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
-    return common::visit(
-        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
-  }
-  template <typename T> std::optional<bool> Visit(const T &) {
-    // Everything else.
-    return std::nullopt;
-  }
-
-private:
-  evaluate::FoldingContext &fctx_;
-};
-} // namespace
-
-// Return values:
-// - std::optional<bool>{true} if the object is known to be contiguous
-// - std::optional<bool>{false} if the object is known not to be contiguous
-// - std::nullopt if the object contiguity cannot be determined
-std::optional<bool> OmpStructureChecker::IsContiguous(
-    const parser::OmpObject &object) {
-  return common::visit( //
-      common::visitors{
-          [&](const parser::Name &x) {
-            // Any member of a common block must be contiguous.
-            return std::optional<bool>{true};
-          },
-          [&](const parser::Designator &x) {
-            evaluate::ExpressionAnalyzer ea{context_};
-            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
-              return ContiguousHelper{context_}.Visit(*maybeExpr);
-            }
-            return std::optional<bool>{};
-          },
-      },
-      object.u);
-}
-
 void OmpStructureChecker::CheckVariableListItem(
     const SymbolSourceMap &symbols) {
   for (auto &[symbol, source] : symbols) {
@@ -522,62 +346,6 @@ bool OmpStructureChecker::HasInvalidWorksharingNesting(
   return false;
 }
 
-void OmpStructureChecker::HasInvalidDistributeNesting(
-    const parser::OpenMPLoopConstruct &x) {
-  bool violation{false};
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
-    // `distribute` region has to be nested
-    if (!CurrentDirectiveIsNested()) {
-      violation = true;
-    } else {
-      // `distribute` region has to be strictly nested inside `teams`
-      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-        violation = true;
-      }
-    }
-  }
-  if (violation) {
-    context_.Say(beginDir.source,
-        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
-        "region."_err_en_US);
-  }
-}
-void OmpStructureChecker::HasInvalidLoopBinding(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
-    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-    for (const auto &clause : clauseList.v) {
-      if (const auto *bindClause{
-              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
-        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
-          context_.Say(beginDir.source, msg);
-        }
-      }
-    }
-  };
-
-  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
-      CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
-        "strictly nested inside a `TEAMS` region."_err_en_US);
-  }
-
-  if (OmpDirectiveSet{
-          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
-          .test(beginDir.v)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
-        "combined with a `TEAMS` construct."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::HasInvalidTeamsNesting(
     const llvm::omp::Directive &dir, const parser::CharBlock &source) {
   if (!llvm::omp::nestedTeamsAllowedSet.test(dir)) {
@@ -668,16 +436,6 @@ void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
-  EnterDirectiveNest(MetadirectiveNest);
-  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
-}
-
-void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
-  ExitDirectiveNest(MetadirectiveNest);
-  dirContext_.pop_back();
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
@@ -717,91 +475,6 @@ void OmpStructureChecker::AddEndDirectiveClauses(
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
-  loopStack_.push_back(&x);
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  PushContextAndClauseSets(beginDir.source, beginDir.v);
-
-  // check matching, End directive is optional
-  if (const auto &endLoopDir{
-          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
-    const auto &endDir{
-        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
-
-    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
-
-    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
-  }
-
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    EnterDirectiveNest(SIMDNest);
-  }
-
-  // Combined target loop constructs are target device constructs. Keep track of
-  // whether any such construct has been visited to later check that REQUIRES
-  // directives for target-related options don't appear after them.
-  if (llvm::omp::allTargetSet.test(beginDir.v)) {
-    deviceConstructFound_ = true;
-  }
-
-  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    // 2.7.1 do-clause -> private-clause |
-    //                    firstprivate-clause |
-    //                    lastprivate-clause |
-    //                    linear-clause |
-    //                    reduction-clause |
-    //                    schedule-clause |
-    //                    collapse-clause |
-    //                    ordered-clause
-
-    // nesting check
-    HasInvalidWorksharingNesting(
-        beginDir.source, llvm::omp::nestedWorkshareErrSet);
-  }
-  SetLoopInfo(x);
-
-  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
-  if (optLoopCons.has_value()) {
-    if (const auto &doConstruct{
-            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
-      const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
-      CheckNoBranching(doBlock, beginDir.v, beginDir.source);
-    }
-  }
-  CheckLoopItrVariableIsInt(x);
-  CheckAssociatedLoopConstraints(x);
-  HasInvalidDistributeNesting(x);
-  HasInvalidLoopBinding(x);
-  if (CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
-  }
-  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
-      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
-    CheckDistLinear(x);
-  }
-}
-const parser::Name OmpStructureChecker::GetLoopIndex(
-    const parser::DoConstruct *x) {
-  using Bounds = parser::LoopControl::Bounds;
-  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
-}
-void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
-  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
-  if (optLoopCons.has_value()) {
-    if (const auto &loopConstruct{
-            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
-      const parser::DoConstruct *loop{&*loopConstruct};
-      if (loop && loop->IsDoNormal()) {
-        const parser::Name &itrVal{GetLoopIndex(loop)};
-        SetLoopIv(itrVal.symbol);
-      }
-    }
-  }
-}
-
 void OmpStructureChecker::CheckIteratorRange(
     const parser::OmpIteratorSpecifier &x) {
   // Check:
@@ -861,111 +534,6 @@ void OmpStructureChecker::CheckIteratorModifier(const parser::OmpIterator &x) {
   }
 }
 
-void OmpStructureChecker::CheckLoopItrVariableIsInt(
-    const parser::OpenMPLoopConstruct &x) {
-  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
-  if (optLoopCons.has_value()) {
-    if (const auto &loopConstruct{
-            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
-
-      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-        if (loop->IsDoNormal()) {
-          const parser::Name &itrVal{GetLoopIndex(loop)};
-          if (itrVal.symbol) {
-            const auto *type{itrVal.symbol->GetType()};
-            if (!type->IsNumeric(TypeCategory::Integer)) {
-              context_.Say(itrVal.source,
-                  "The DO loop iteration"
-                  " variable must be of the type integer."_err_en_US,
-                  itrVal.ToString());
-            }
-          }
-        }
-        // Get the next DoConstruct if block is not empty.
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                 : nullptr;
-      }
-    }
-  }
-}
-
-void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
-  // Check the following:
-  //  The only OpenMP constructs that can be encountered during execution of
-  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
-  // construct and the `ordered` construct with the `simd` clause.
-
-  // Check if the parent context has the SIMD clause
-  // Please note that we use GetContext() instead of GetContextParent()
-  // because PushContextAndClauseSets() has not been called on the
-  // current context yet.
-  // TODO: Check for declare simd regions.
-  bool eligibleSIMD{false};
-  common::visit(
-      common::visitors{
-          // Allow `!$OMP ORDERED SIMD`
-          [&](const parser::OpenMPBlockConstruct &c) {
-            const auto &beginBlockDir{
-                std::get<parser::OmpBeginBlockDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
-            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
-              const auto &clauses{
-                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
-              for (const auto &clause : clauses.v) {
-                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
-                  eligibleSIMD = true;
-                  break;
-                }
-              }
-            }
-          },
-          [&](const parser::OpenMPStandaloneConstruct &c) {
-            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
-                    &c.u)}) {
-              llvm::omp::Directive dirId{ssc->v.DirId()};
-              if (dirId == llvm::omp::Directive::OMPD_ordered) {
-                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
-                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
-                    eligibleSIMD = true;
-                    break;
-                  }
-                }
-              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
-                eligibleSIMD = true;
-              }
-            }
-          },
-          // Allowing SIMD and loop construct
-          [&](const parser::OpenMPLoopConstruct &c) {
-            const auto &beginLoopDir{
-                std::get<parser::OmpBeginLoopDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
-              eligibleSIMD = true;
-            }
-          },
-          [&](const parser::OpenMPAtomicConstruct &c) {
-            // Allow `!$OMP ATOMIC`
-            eligibleSIMD = true;
-          },
-          [&](const auto &c) {},
-      },
-      c.u);
-  if (!eligibleSIMD) {
-    context_.Say(parser::FindSourceLocation(c),
-        "The only OpenMP constructs that can be encountered during execution "
-        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
-        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
-        "construct with the `SIMD` clause."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   // 2.12.5 Target Construct Restriction
   bool eligibleTarget{true};
@@ -1023,190 +591,6 @@ void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   }
 }
 
-std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-  std::int64_t orderedCollapseLevel{1};
-  std::int64_t orderedLevel{1};
-  std::int64_t collapseLevel{1};
-
-  for (const auto &clause : clauseList.v) {
-    if (const auto *collapseClause{
-            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
-      if (const auto v{GetIntValue(collapseClause->v)}) {
-        collapseLevel = *v;
-      }
-    }
-    if (const auto *orderedClause{
-            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
-      if (const auto v{GetIntValue(orderedClause->v)}) {
-        orderedLevel = *v;
-      }
-    }
-  }
-  if (orderedLevel >= collapseLevel) {
-    orderedCollapseLevel = orderedLevel;
-  } else {
-    orderedCollapseLevel = collapseLevel;
-  }
-  return orderedCollapseLevel;
-}
-
-void OmpStructureChecker::CheckAssociatedLoopConstraints(
-    const parser::OpenMPLoopConstruct &x) {
-  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
-  AssociatedLoopChecker checker{context_, ordCollapseLevel};
-  parser::Walk(x, checker);
-}
-
-void OmpStructureChecker::CheckDistLinear(
-    const parser::OpenMPLoopConstruct &x) {
-
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  SymbolSourceMap indexVars;
-
-  // Collect symbols of all the variables from linear clauses
-  for (auto &clause : clauses.v) {
-    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
-      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
-      GetSymbolsInObjectList(objects, indexVars);
-    }
-  }
-
-  if (!indexVars.empty()) {
-    // Get collapse level, if given, to find which loops are "associated."
-    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
-    // Include the top loop if no collapse is specified
-    if (collapseVal == 0) {
-      collapseVal = 1;
-    }
-
-    // Match the loop index variables with the collected symbols from linear
-    // clauses.
-    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
-    if (optLoopCons.has_value()) {
-      if (const auto &loopConstruct{
-              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
-        for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-          if (loop->IsDoNormal()) {
-            const parser::Name &itrVal{GetLoopIndex(loop)};
-            if (itrVal.symbol) {
-              // Remove the symbol from the collected set
-              indexVars.erase(&itrVal.symbol->GetUltimate());
-            }
-            collapseVal--;
-            if (collapseVal == 0) {
-              break;
-            }
-          }
-          // Get the next DoConstruct if block is not empty.
-          const auto &block{std::get<parser::Block>(loop->t)};
-          const auto it{block.begin()};
-          loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                   : nullptr;
-        }
-      }
-    }
-
-    // Show error for the remaining variables
-    for (auto &[symbol, source] : indexVars) {
-      const Symbol &root{GetAssociationRoot(*symbol)};
-      context_.Say(source,
-          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
-          root.name());
-    }
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  // A few semantic checks for InScan reduction are performed below as SCAN
-  // constructs inside LOOP may add the relevant information. Scan reduction is
-  // supported only in loop constructs, so same checks are not applicable to
-  // other directives.
-  using ReductionModifier = parser::OmpReductionModifier;
-  for (const auto &clause : clauseList.v) {
-    if (const auto *reductionClause{
-            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
-      auto &modifiers{OmpGetModifiers(reductionClause->v)};
-      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
-      if (maybeModifier &&
-          maybeModifier->v == ReductionModifier::Value::Inscan) {
-        const auto &objectList{
-            std::get<parser::OmpObjectList>(reductionClause->v.t)};
-        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
-          if (auto &symbol = name->symbol) {
-            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
-                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
-              context_.Say(name->source,
-                  "List item %s must appear in EXCLUSIVE or "
-                  "INCLUSIVE clause of an "
-                  "enclosed SCAN directive"_err_en_US,
-                  name->ToString());
-            }
-          }
-        };
-        for (const auto &ompObj : objectList.v) {
-          common::visit(
-              common::visitors{
-                  [&](const parser::Designator &designator) {
-                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
-                            designator)}) {
-                      checkReductionSymbolInScan(name);
-                    }
-                  },
-                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
-              },
-              ompObj.u);
-        }
-      }
-    }
-  }
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    ExitDirectiveNest(SIMDNest);
-  }
-  dirContext_.pop_back();
-
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top{loopStack_.back()};
-  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
-  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
-  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  // 2.7.1 end-do -> END DO [nowait-clause]
-  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
-  case llvm::omp::Directive::OMPD_do:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
-    break;
-  case llvm::omp::Directive::OMPD_do_simd:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
-    dirContext_.pop_back();
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &endBlockDir{std::get<parser::OmpEndBlockDirective>(x.t)};
@@ -2671,1488 +2055,46 @@ void OmpStructureChecker::CheckCancellationNest(
           parser::ToUpperCaseLetters(typeName.str()));
       break;
     default:
-      // This is diagnosed later.
-      return;
-    }
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
-  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  case llvm::omp::Directive::OMPD_scope:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
-    break;
-  // 2.7.3 end-single-clause -> copyprivate-clause |
-  //                            nowait-clause
-  case llvm::omp::Directive::OMPD_single:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
-    break;
-  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
-  case llvm::omp::Directive::OMPD_workshare:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_workshare);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-// TODO: Verify the popping of dirContext requirement after nowait
-// implementation, as there is an implicit barrier at the end of the worksharing
-// constructs unless a nowait clause is specified. Only OMPD_end_single and
-// end_workshareare popped as they are pushed while entering the
-// EndBlockDirective.
-void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
-    dirContext_.pop_back();
-  }
-}
-
-/// parser::Block is a list of executable constructs, parser::BlockConstruct
-/// is Fortran's BLOCK/ENDBLOCK construct.
-/// Strip the outermost BlockConstructs, return the reference to the Block
-/// in the executable part of the innermost of the stripped constructs.
-/// Specifically, if the given `block` has a single entry (it's a list), and
-/// the entry is a BlockConstruct, get the Block contained within. Repeat
-/// this step as many times as possible.
-static const parser::Block &GetInnermostExecPart(const parser::Block &block) {
-  const parser::Block *iter{&block};
-  while (iter->size() == 1) {
-    const parser::ExecutionPartConstruct &ep{iter->front()};
-    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
-      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
-      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
-        iter = &std::get<parser::Block>(bc->value().t);
-        continue;
-      }
-    }
-    break;
-  }
-  return *iter;
-}
-
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
-struct AnalyzedCondStmt {
-  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
-  parser::CharBlock source;
-  SourcedActionStmt ift, iff;
-};
-
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
-// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
-// is that the ActionStmt will be either an assignment or a pointer-assignment,
-// otherwise return std::nullopt.
-// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
-// the "typedAssignment" is unset. This can happen if there are semantic errors
-// in the purported assignment.
-static std::optional<evaluate::Assignment> GetEvaluateAssignment(
-    const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
-
-  return common::visit(
-      [](auto &&s) -> std::optional<evaluate::Assignment> {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>) {
-          const TypedAssignment &typed{s.value().typedAssignment};
-          // ForwardOwningPointer                 typedAssignment
-          // `- GenericAssignmentWrapper          ^.get()
-          //    `- std::optional<Assignment>      ^->v
-          return typed.get()->v;
-        } else {
-          return std::nullopt;
-        }
-      },
-      x->u);
-}
-
-// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
-// to separate cases where the source has something that looks like an
-// assignment, but is semantically wrong (diagnosed by general semantic
-// checks), and where the source has some other statement (which we want
-// to report as "should be an assignment").
-static bool IsAssignment(const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return false;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-
-  return common::visit(
-      [](auto &&s) -> bool {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        return std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>;
-      },
-      x->u);
-}
-
-static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  // Extract the evaluate::Expr from ScalarLogicalExpr.
-  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
-    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
-    const parser::Expr &expr{logical.thing.thing.value()};
-    return GetEvaluateExpr(expr);
-  }};
-
-  // Recognize either
-  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
-  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
-
-  if (auto &&action{GetActionStmt(x)}) {
-    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
-            &action.stmt->u)}) {
-      const parser::IfStmt &s{ifs->value()};
-      auto &&maybeCond{
-          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
-      auto &thenStmt{
-          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
-      if (maybeCond) {
-        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
-            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
-            SourcedActionStmt{}};
-      }
-    }
-    return std::nullopt;
-  }
-
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    if (auto *ifc{
-            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
-      using ElseBlock = parser::IfConstruct::ElseBlock;
-      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
-      const parser::IfConstruct &s{ifc->value()};
-
-      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
-        // Not expecting any else-if statements.
-        return std::nullopt;
-      }
-      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
-      auto &&maybeCond{getFromLogical(
-          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
-      if (!maybeCond) {
-        return std::nullopt;
-      }
-
-      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)),
-            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
-        if (result.ift.stmt && result.iff.stmt) {
-          return result;
-        }
-      } else {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
-        if (result.ift.stmt) {
-          return result;
-        }
-      }
-    }
-    return std::nullopt;
-  }
-
-  return std::nullopt;
-}
-
-static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
-    parser::CharBlock source) {
-  // Find => in the range, if not found, find = that is not a part of
-  // <=, >=, ==, or /=.
-  auto trim{[](std::string_view v) {
-    const char *begin{v.data()};
-    const char *end{begin + v.size()};
-    while (*begin == ' ' && begin != end) {
-      ++begin;
-    }
-    while (begin != end && end[-1] == ' ') {
-      --end;
-    }
-    assert(begin != end && "Source should not be empty");
-    return parser::CharBlock(begin, end - begin);
-  }};
-
-  std::string_view sv(source.begin(), source.size());
-
-  if (auto where{sv.find("=>")}; where != sv.npos) {
-    std::string_view lhs(sv.data(), where);
-    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
-    return std::make_pair(trim(lhs), trim(rhs));
-  }
-
-  // Go backwards, since all the exclusions above end with a '='.
-  for (size_t next{source.size()}; next > 1; --next) {
-    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
-      std::string_view lhs(sv.data(), next - 1);
-      std::string_view rhs(sv.data() + next, sv.size() - next);
-      return std::make_pair(trim(lhs), trim(rhs));
-    }
-  }
-  llvm_unreachable("Could not find assignment operator");
-}
-
-namespace atomic {
-
-struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
-                                 std::vector<SomeExpr>, false> {
-  using Result = std::vector<SomeExpr>;
-  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
-  DesignatorCollector() : Base(*this) {}
-
-  Result Default() const { return {}; }
-
-  using Base::operator();
-
-  template <typename T> //
-  Result operator()(const evaluate::Designator<T> &x) const {
-    // Once in a designator, don't traverse it any further (i.e. only
-    // collect top-level designators).
-    auto copy{x};
-    return Result{AsGenericExpr(std::move(copy))};
-  }
-
-  template <typename... Rs> //
-  Result Combine(Result &&result, Rs &&...results) const {
-    Result v(std::move(result));
-    auto moveAppend{[](auto &accum, auto &&other) {
-      for (auto &&s : other) {
-        accum.push_back(std::move(s));
-      }
-    }};
-    (moveAppend(v, std::move(results)), ...);
-    return v;
-  }
-};
-
-struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
-  using Base = evaluate::AnyTraverse<VariableFinder>;
-  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
-
-  using Base::operator();
-
-  template <typename T>
-  bool operator()(const evaluate::Designator<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-  template <typename T>
-  bool operator()(const evaluate::FunctionRef<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-private:
-  const SomeExpr &var;
-};
-} // namespace atomic
-
-static bool IsPointerAssignment(const evaluate::Assignment &x) {
-  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
-      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
-}
-
-namespace operation = Fortran::evaluate::operation;
-
-static bool IsCheckForAssociated(const SomeExpr &cond) {
-  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
-}
-
-static bool HasCommonDesignatorSymbols(
-    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
-  // Compare the designators used in "other" with the designators whose
-  // symbols are given in baseSyms.
-  // This is a part of the check if these two expressions can access the same
-  // storage: if the designators used in them are 
diff erent enough, then they
-  // will be assumed not to access the same memory.
-  //
-  // Consider an (array element) expression x%y(w%z), the corresponding symbol
-  // vector will be {x, y, w, z} (i.e. the symbols for these names).
-  // Check whether this exact sequence appears anywhere in any the symbol
-  // vector for "other". This will be true for x(y) and x(y+1), so this is
-  // not a sufficient condition, but can be used to eliminate candidates
-  // before doing more exhaustive checks.
-  //
-  // If any of the symbols in this sequence are function names, assume that
-  // there is no storage overlap, mostly because it would be impossible in
-  // general to determine what storage the function will access.
-  // Note: if f is pure, then two calls to f will access the same storage
-  // when called with the same arguments. This check is not done yet.
-
-  if (llvm::any_of(
-          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
-    // If there is a function symbol in the chain then we can't infer much
-    // about the accessed storage.
-    return false;
-  }
-
-  auto isSubsequence{// Is u a subsequence of v.
-      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
-        size_t us{u.size()}, vs{v.size()};
-        if (us > vs) {
-          return false;
-        }
-        for (size_t off{0}; off != vs - us + 1; ++off) {
-          bool same{true};
-          for (size_t i{0}; i != us; ++i) {
-            if (u[i] != v[off + i]) {
-              same = false;
-              break;
-            }
-          }
-          if (same) {
-            return true;
-          }
-        }
-        return false;
-      }};
-
-  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
-  return isSubsequence(baseSyms, otherSyms);
-}
-
-static bool HasCommonTopLevelDesignators(
-    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
-  // Compare designators directly as expressions. This will ensure
-  // that x(y) and x(y+1) are not flagged as overlapping, whereas
-  // the symbol vectors for both of these would be identical.
-  std::vector<SomeExpr> otherDsgs{atomic::DesignatorCollector{}(other)};
-
-  for (auto &s : baseDsgs) {
-    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
-      return true;
-    }
-  }
-  return false;
-}
-
-static const SomeExpr *HasStorageOverlap(
-    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
-  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
-  std::vector<SomeExpr> baseDsgs{atomic::DesignatorCollector{}(base)};
-
-  for (const SomeExpr &expr : exprs) {
-    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
-      continue;
-    }
-    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
-      return &expr;
-    }
-  }
-  return nullptr;
-}
-
-static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
-  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
-  // for example.
-  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
-}
-
-static bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
-  return atomic::VariableFinder{sub}(super);
-}
-
-static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
-  if (value) {
-    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
-        evaluate::GenericExprWrapper::Deleter);
-  }
-}
-
-static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
-    std::optional<evaluate::Assignment> value) {
-  if (value) {
-    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
-        evaluate::GenericAssignmentWrapper::Deleter);
-  }
-}
-
-static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
-    int what,
-    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
-  parser::OpenMPAtomicConstruct::Analysis::Op operation;
-  operation.what = what;
-  SetAssignment(operation.assign, maybeAssign);
-  return operation;
-}
-
-static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
-    const SomeExpr &atom, const MaybeExpr &cond,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
-  // Defined in flang/include/flang/Parser/parse-tree.h
-  //
-  // struct Analysis {
-  //   struct Kind {
-  //     static constexpr int None = 0;
-  //     static constexpr int Read = 1;
-  //     static constexpr int Write = 2;
-  //     static constexpr int Update = Read | Write;
-  //     static constexpr int Action = 3; // Bits containing N, R, W, U
-  //     static constexpr int IfTrue = 4;
-  //     static constexpr int IfFalse = 8;
-  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
-  //   };
-  //   struct Op {
-  //     int what;
-  //     TypedAssignment assign;
-  //   };
-  //   TypedExpr atom, cond;
-  //   Op op0, op1;
-  // };
-
-  parser::OpenMPAtomicConstruct::Analysis an;
-  SetExpr(an.atom, atom);
-  SetExpr(an.cond, cond);
-  an.op0 = std::move(op0);
-  an.op1 = std::move(op1);
-  return an;
-}
-
-void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
-    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
-    parser::CharBlock source) {
-  if (auto *expr{HasStorageOverlap(base, exprs)}) {
-    context_.Say(source,
-        "Within atomic operation %s and %s access the same storage"_warn_en_US,
-        base.AsFortran(), expr->AsFortran());
-  }
-}
-
-void OmpStructureChecker::ErrorShouldBeVariable(
-    const MaybeExpr &expr, parser::CharBlock source) {
-  if (expr) {
-    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
-        expr->AsFortran());
-  } else {
-    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
-  }
-}
-
-/// Check if `expr` satisfies the following conditions for x and v:
-///
-/// [6.0:189:10-12]
-/// - x and v (as applicable) are either scalar variables or
-///   function references with scalar data pointer result of non-character
-///   intrinsic type or variables that are non-polymorphic scalar pointers
-///   and any length type parameter must be constant.
-void OmpStructureChecker::CheckAtomicType(
-    SymbolRef sym, parser::CharBlock source, std::string_view name) {
-  const DeclTypeSpec *typeSpec{sym->GetType()};
-  if (!typeSpec) {
-    return;
-  }
-
-  if (!IsPointer(sym)) {
-    using Category = DeclTypeSpec::Category;
-    Category cat{typeSpec->category()};
-    if (cat == Category::Character) {
-      context_.Say(source,
-          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
-    } else if (cat != Category::Numeric && cat != Category::Logical) {
-      context_.Say(source,
-          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
-    }
-    return;
-  }
-
-  // Variable is a pointer.
-  if (typeSpec->IsPolymorphic()) {
-    context_.Say(source,
-        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
-        name);
-    return;
-  }
-
-  // Go over all length parameters, if any, and check if they are
-  // explicit.
-  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
-    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
-          // "entry" is a map entry
-          return entry.second.isLen() && !entry.second.isExplicit();
-        })) {
-      context_.Say(source,
-          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
-          name);
-    }
-  }
-}
-
-void OmpStructureChecker::CheckAtomicVariable(
-    const SomeExpr &atom, parser::CharBlock source) {
-  if (atom.Rank() != 0) {
-    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
-        atom.AsFortran());
-  }
-
-  std::vector<SomeExpr> dsgs{atomic::DesignatorCollector{}(atom)};
-  assert(dsgs.size() == 1 && "Should have a single top-level designator");
-  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
-
-  CheckAtomicType(syms.back(), source, atom.AsFortran());
-
-  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
-    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
-        atom.AsFortran());
-  }
-}
-
-std::pair<const parser::ExecutionPartConstruct *,
-    const parser::ExecutionPartConstruct *>
-OmpStructureChecker::CheckUpdateCapture(
-    const parser::ExecutionPartConstruct *ec1,
-    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
-  // Decide which statement is the atomic update and which is the capture.
-  //
-  // The two allowed cases are:
-  //   x = ...      atomic-var = ...
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  // or
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  //   x = ...      atomic-var = ...
-  //
-  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
-  // (which makes more sense, as it captures the original value of the atomic
-  // variable).
-  //
-  // If the two statements don't fit these criteria, return a pair of default-
-  // constructed values.
-  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
-      const parser::ExecutionPartConstruct *>;
-
-  SourcedActionStmt act1{GetActionStmt(ec1)};
-  SourcedActionStmt act2{GetActionStmt(ec2)};
-  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
-  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
-  if (!maybeAssign1 || !maybeAssign2) {
-    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
-      context_.Say(source,
-          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
-    }
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  auto as1{*maybeAssign1}, as2{*maybeAssign2};
-
-  auto isUpdateCapture{
-      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
-        return IsSameOrConvertOf(c.rhs, u.lhs);
-      }};
-
-  // Do some checks that narrow down the possible choices for the update
-  // and the capture statements. This will help to emit better diagnostics.
-  // 1. An assignment could be an update (cbu) if the left-hand side is a
-  //    subexpression of the right-hand side.
-  // 2. An assignment could be a capture (cbc) if the right-hand side is
-  //    a variable (or a function ref), with potential type conversions.
-  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
-  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
-  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
-  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
-
-  // We want to diagnose cases where both assignments cannot be an update,
-  // or both cannot be a capture, as well as cases where either assignment
-  // cannot be any of these two.
-  //
-  // If we organize these boolean values into a matrix
-  //   |cbu1 cbu2|
-  //   |cbc1 cbc2|
-  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
-  // row or column, including the case where everything is zero. All these
-  // cases correspond to the determinant of the matrix being 0, which suggests
-  // that checking the det may be a convenient diagnostic check. There is only
-  // one additional case where the det is 0, which is when the matrix is all 1
-  // ("true"). The "all true" case represents the situation where both
-  // assignments could be an update as well as a capture. On the other hand,
-  // whenever det != 0, the roles of the update and the capture can be
-  // unambiguously assigned to as1 and as2 [1].
-  //
-  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
-  // det = 0, leaving 6 cases where det != 0:
-  //   0 1   0 1   1 0   1 0   1 1   1 1
-  //   1 0   1 1   0 1   1 1   0 1   1 0
-  // In each case the classification is unambiguous.
-
-  //     |cbu1 cbu2|
-  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
-  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
-
-  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
-                                  const std::string &expr) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
-        expr);
-  }};
-
-  auto errorNeitherWorks{[&]() {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
-  }};
-
-  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
-    // If det != 0, then the checks unambiguously suggest a specific
-    // categorization.
-    // If det == 0, then this function should be called only if the
-    // checks haven't ruled out any possibility, i.e. when both assigments
-    // could still be either updates or captures.
-    if (det > 0) {
-      // as1 is update, as2 is capture
-      if (isUpdateCapture(as1, as2)) {
-        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
-      } else {
-        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else if (det < 0) {
-      // as2 is update, as1 is capture
-      if (isUpdateCapture(as2, as1)) {
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      } else {
-        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else {
-      bool updateFirst{isUpdateCapture(as1, as2)};
-      bool captureFirst{isUpdateCapture(as2, as1)};
-      if (updateFirst && captureFirst) {
-        // If both assignment could be the update and both could be the
-        // capture, emit a warning about the ambiguity.
-        context_.Say(act1.source,
-            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      }
-      if (updateFirst != captureFirst) {
-        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
-        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
-        return std::make_pair(upd, cap);
-      }
-      assert(!updateFirst && !captureFirst);
-      errorNeitherWorks();
-      return std::make_pair(nullptr, nullptr);
-    }
-  }};
-
-  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
-    return makeSelectionFromDet(det);
-  }
-  assert(det == 0 && "Prior checks should have covered det != 0");
-
-  // If neither of the statements is an RMW update, it could still be a
-  // "write" update. Pretty much any assignment can be a write update, so
-  // recompute det with cbu1 = cbu2 = true.
-  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
-    return makeSelectionFromDet(writeDet);
-  }
-
-  // It's only errors from here on.
-
-  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
-    errorNeitherWorks();
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // The remaining cases are that
-  // - no candidate for update, or for capture,
-  // - one of the assigments cannot be anything.
-
-  if (!cbu1 && !cbu2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  } else if (!cbc1 && !cbc2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
-    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
-    context_.Say(src,
-        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // All cases should have been covered.
-  llvm_unreachable("Unchecked condition");
-}
-
-void OmpStructureChecker::CheckAtomicCaptureAssignment(
-    const evaluate::Assignment &capture, const SomeExpr &atom,
-    parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &cap{capture.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, rsrc);
-    // This part should have been checked prior to calling this function.
-    assert(*GetConvertInput(capture.rhs) == atom &&
-        "This cannot be a capture assignment");
-    CheckStorageOverlap(atom, {cap}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicReadAssignment(
-    const evaluate::Assignment &read, parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-
-  if (auto maybe{GetConvertInput(read.rhs)}) {
-    const SomeExpr &atom{*maybe};
-
-    if (!IsVarOrFunctionRef(atom)) {
-      ErrorShouldBeVariable(atom, rsrc);
-    } else {
-      CheckAtomicVariable(atom, rsrc);
-      CheckStorageOverlap(atom, {read.lhs}, source);
-    }
-  } else {
-    ErrorShouldBeVariable(read.rhs, rsrc);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicWriteAssignment(
-    const evaluate::Assignment &write, parser::CharBlock source) {
-  // [6.0:190:13-15]
-  // A write structured block is write-statement, a write statement that has
-  // one of the following forms:
-  //   x = expr
-  //   x => expr
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{write.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, lsrc);
-    CheckStorageOverlap(atom, {write.rhs}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateAssignment(
-    const evaluate::Assignment &update, parser::CharBlock source) {
-  // [6.0:191:1-7]
-  // An update structured block is update-statement, an update statement
-  // that has one of the following forms:
-  //   x = x operator expr
-  //   x = expr operator x
-  //   x = intrinsic-procedure-name (x)
-  //   x = intrinsic-procedure-name (x, expr-list)
-  //   x = intrinsic-procedure-name (expr-list, x)
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{update.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, lsrc);
-
-  std::pair<operation::Operator, std::vector<SomeExpr>> top{
-      operation::Operator::Unknown, {}};
-  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
-    top = GetTopLevelOperation(*maybeInput);
-  }
-  switch (top.first) {
-  case operation::Operator::Add:
-  case operation::Operator::Sub:
-  case operation::Operator::Mul:
-  case operation::Operator::Div:
-  case operation::Operator::And:
-  case operation::Operator::Or:
-  case operation::Operator::Eqv:
-  case operation::Operator::Neqv:
-  case operation::Operator::Min:
-  case operation::Operator::Max:
-  case operation::Operator::Identity:
-    break;
-  case operation::Operator::Call:
-    context_.Say(source,
-        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Convert:
-    context_.Say(source,
-        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Intrinsic:
-    context_.Say(source,
-        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Constant:
-  case operation::Operator::Unknown:
-    context_.Say(
-        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(source,
-        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
-        operation::ToString(top.first));
-    return;
-  }
-  // Check how many times `atom` occurs as an argument, if it's a subexpression
-  // of an argument, and collect the non-atom arguments.
-  std::vector<SomeExpr> nonAtom;
-  MaybeExpr subExpr;
-  auto atomCount{[&]() {
-    int count{0};
-    for (const SomeExpr &arg : top.second) {
-      if (IsSameOrConvertOf(arg, atom)) {
-        ++count;
-      } else {
-        if (!subExpr && IsSubexpressionOf(atom, arg)) {
-          subExpr = arg;
-        }
-        nonAtom.push_back(arg);
-      }
-    }
-    return count;
-  }()};
-
-  bool hasError{false};
-  if (subExpr) {
-    context_.Say(rsrc,
-        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
-        atom.AsFortran(), subExpr->AsFortran());
-    hasError = true;
-  }
-  if (top.first == operation::Operator::Identity) {
-    // This is "x = y".
-    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
-          atom.AsFortran());
-      hasError = true;
-    }
-  } else {
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    } else if (atomCount > 1) {
-      context_.Say(rsrc,
-          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    }
-  }
-
-  if (!hasError) {
-    CheckStorageOverlap(atom, nonAtom, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
-    const SomeExpr &cond, parser::CharBlock condSource,
-    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
-  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
-  const SomeExpr &atom{assign.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, arsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, alsrc);
-
-  auto top{GetTopLevelOperation(cond)};
-  // Missing arguments to operations would have been diagnosed by now.
-
-  switch (top.first) {
-  case operation::Operator::Associated:
-    if (atom != top.second.front()) {
-      context_.Say(assignSource,
-          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
-    }
-    break;
-  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
-  case operation::Operator::Eq:
-  case operation::Operator::Eqv:
-  // x ordop expr | expr ordop x
-  case operation::Operator::Lt:
-  case operation::Operator::Gt: {
-    const SomeExpr &arg0{top.second[0]};
-    const SomeExpr &arg1{top.second[1]};
-    if (IsSameOrConvertOf(arg0, atom)) {
-      CheckStorageOverlap(atom, {arg1}, condSource);
-    } else if (IsSameOrConvertOf(arg1, atom)) {
-      CheckStorageOverlap(atom, {arg0}, condSource);
-    } else {
-      assert(top.first != operation::Operator::Identity &&
-          "Handle this separately");
-      context_.Say(assignSource,
-          "An argument of the %s operator should be the target of the assignment"_err_en_US,
-          operation::ToString(top.first));
-    }
-    break;
-  }
-  case operation::Operator::Identity:
-  case operation::Operator::True:
-  case operation::Operator::False:
-    break;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(condSource,
-        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
-        operation::ToString(top.first));
-    break;
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
-    const AnalyzedCondStmt &update, parser::CharBlock source) {
-  // The condition/statements must be:
-  // - cond: x equalop e      ift: x =  d     iff: -
-  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
-  // - cond: associated(x)    ift: x => expr  iff: -
-  // - cond: associated(x, e) ift: x => expr  iff: -
-
-  // The if-true statement must be present, and must be an assignment.
-  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
-  if (!maybeAssign) {
-    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
-    } else {
-      context_.Say(
-          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
-    }
-    return;
-  }
-  const evaluate::Assignment assign{*maybeAssign};
-  const SomeExpr &atom{assign.lhs};
-
-  CheckAtomicConditionalUpdateAssignment(
-      update.cond, update.source, assign, update.ift.source);
-
-  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
-
-  if (update.iff) {
-    context_.Say(update.iff.source,
-        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateOnly(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeUpdate->lhs};
-      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdate(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // Allowable forms are (single-statement):
-  // - if ...
-  // - x = (... ? ... : x)
-  // and two-statement:
-  // - r = cond ; if (r) ...
-
-  const parser::ExecutionPartConstruct *ust{nullptr}; // update
-  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
-
-  if (body.size() == 1) {
-    ust = &body.front();
-  } else if (body.size() == 2) {
-    cst = &body.front();
-    ust = &body.back();
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
-    return;
-  }
-
-  // Flang doesn't support conditional-expr yet, so all update statements
-  // are if-statements.
-
-  // IfStmt:        if (...) ...
-  // IfConstruct:   if (...) then ... endif
-  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
-  if (!maybeUpdate) {
-    context_.Say(source,
-        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
-    return;
-  }
-
-  AnalyzedCondStmt &update{*maybeUpdate};
-
-  if (SourcedActionStmt action{GetActionStmt(cst)}) {
-    // The "condition" statement must be `r = cond`.
-    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
-      if (maybeCond->lhs != update.cond) {
-        context_.Say(update.source,
-            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
-            maybeCond->lhs.AsFortran());
-      } else {
-        // If it's "r = ...; if (r) ..." then put the original condition
-        // in `update`.
-        update.cond = maybeCond->rhs;
-      }
-    } else {
-      context_.Say(action.source,
-          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
-    }
-  }
-
-  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
-
-  CheckAtomicConditionalUpdateStmt(update, source);
-  if (IsCheckForAssociated(update.cond)) {
-    if (!IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
-    }
-  } else {
-    if (IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
-    }
-  }
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
-      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
-      MakeAtomicAnalysisOp(Analysis::None));
-}
-
-void OmpStructureChecker::CheckAtomicUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() != 2) {
-    context_.Say(source,
-        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
-    return;
-  }
-
-  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
-  if (!uec || !cec) {
-    // Diagnostics already emitted.
-    return;
-  }
-  SourcedActionStmt uact{GetActionStmt(uec)};
-  SourcedActionStmt cact{GetActionStmt(cec)};
-  // The "dereferences" of std::optional are guaranteed to be valid after
-  // CheckUpdateCapture.
-  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
-  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
-
-  const SomeExpr &atom{update.lhs};
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  int action;
-
-  if (IsMaybeAtomicWrite(update)) {
-    action = Analysis::Write;
-    CheckAtomicWriteAssignment(update, uact.source);
-  } else {
-    action = Analysis::Update;
-    CheckAtomicUpdateAssignment(update, uact.source);
-  }
-  CheckAtomicCaptureAssignment(capture, atom, cact.source);
-
-  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
-    context_.Say(cact.source,
-        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-    return;
-  }
-
-  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(action, update),
-        MakeAtomicAnalysisOp(Analysis::Read, capture));
-  } else {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(Analysis::Read, capture),
-        MakeAtomicAnalysisOp(action, update));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // There are two 
diff erent variants of this:
-  // (1) conditional-update and capture separately:
-  //     This form only allows single-statement updates, i.e. the update
-  //     form "r = cond; if (r) ..." is not allowed.
-  // (2) conditional-update combined with capture in a single statement:
-  //     This form does allow the condition to be calculated separately,
-  //     i.e. "r = cond; if (r) ...".
-  // Regardless of what form it is, the actual update assignment is a
-  // proper write, i.e. "x = d", where d does not depend on x.
-
-  AnalyzedCondStmt update;
-  SourcedActionStmt capture;
-  bool captureAlways{true}, captureFirst{true};
-
-  auto extractCapture{[&]() {
-    capture = update.iff;
-    captureAlways = false;
-    update.iff = SourcedActionStmt{};
-  }};
-
-  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
-    // The non-update statement is either "r = cond" or the capture.
-    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
-      if (update.cond == maybeAssign->lhs) {
-        // If this is "r = cond; if (r) ...", then update the condition.
-        update.cond = maybeAssign->rhs;
-        update.source = action.source;
-        // In this form, the update and the capture are combined into
-        // an IF-THEN-ELSE statement.
-        extractCapture();
-      } else {
-        // Assume this is the capture-statement.
-        capture = action;
-      }
-    }
-  }};
-
-  if (body.size() == 2) {
-    // This could be
-    // - capture; conditional-update (in any order), or
-    // - r = cond; if (r) capture-update
-    const parser::ExecutionPartConstruct *st1{&body.front()};
-    const parser::ExecutionPartConstruct *st2{&body.back()};
-    // In either case, the conditional statement can be analyzed by
-    // AnalyzeConditionalStmt, whereas the other statement cannot.
-    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
-      update = *maybeUpdate1;
-      classifyNonUpdate(GetActionStmt(st2));
-      captureFirst = false;
-    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
-      update = *maybeUpdate2;
-      classifyNonUpdate(GetActionStmt(st1));
-    } else {
-      // None of the statements are conditional, this rules out the
-      // "r = cond; if (r) ..." and the "capture + conditional-update"
-      // variants. This could still be capture + write (which is classified
-      // as conditional-update-capture in the spec).
-      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
-      if (!uec || !cec) {
-        // Diagnostics already emitted.
-        return;
-      }
-      SourcedActionStmt uact{GetActionStmt(uec)};
-      SourcedActionStmt cact{GetActionStmt(cec)};
-      update.ift = uact;
-      capture = cact;
-      if (uec == st1) {
-        captureFirst = false;
-      }
-    }
-  } else if (body.size() == 1) {
-    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
-      update = *maybeUpdate;
-      // This is the form with update and capture combined into an IF-THEN-ELSE
-      // statement. The capture-statement is always the ELSE branch.
-      extractCapture();
-    } else {
-      goto invalid;
-    }
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
-    return;
-  invalid:
-    context_.Say(source,
-        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
-    return;
-  }
-
-  // The update must have a form `x = d` or `x => d`.
-  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
-    const SomeExpr &atom{maybeWrite->lhs};
-    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
-    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
-      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
-
-      if (IsPointerAssignment(*maybeWrite) !=
-          IsPointerAssignment(*maybeCapture)) {
-        context_.Say(capture.source,
-            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-        return;
-      }
-    } else {
-      if (!IsAssignment(capture.stmt)) {
-        context_.Say(capture.source,
-            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
-      }
-      return;
-    }
-  } else {
-    if (!IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
-    }
-    return;
-  }
-
-  // update.iff should be empty here, the capture statement should be
-  // stored in "capture".
-
-  // Fill out the analysis in the AST node.
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  bool condUnused{std::visit(
-      [](auto &&s) {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
-          return true;
-        } else {
-          return false;
-        }
-      },
-      update.cond.u)};
-
-  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
-  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
-
-  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
-  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
-
-  if (captureFirst) {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
-  } else {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicRead(
-    const parser::OpenMPAtomicConstruct &x) {
-  // [6.0:190:5-7]
-  // A read structured block is read-statement, a read statement that has one
-  // of the following forms:
-  //   v = x
-  //   v => x
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Read cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
-      CheckAtomicReadAssignment(*maybeRead, action.source);
-
-      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
-        const SomeExpr &atom{*maybe};
-        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
-            MakeAtomicAnalysisOp(Analysis::None));
-      }
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+      // This is diagnosed later.
+      return;
     }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC READ operation should have a single statement"_err_en_US);
   }
 }
 
-void OmpStructureChecker::CheckAtomicWrite(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Write cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeWrite->lhs};
-      CheckAtomicWriteAssignment(*maybeWrite, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC WRITE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdate(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &block{std::get<parser::Block>(x.t)};
-
-  bool isConditional{x.IsCompare()};
-  bool isCapture{x.IsCapture()};
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (isConditional && isCapture) {
-    CheckAtomicConditionalUpdateCapture(x, body, x.source);
-  } else if (isConditional) {
-    CheckAtomicConditionalUpdate(x, body, x.source);
-  } else if (isCapture) {
-    CheckAtomicUpdateCapture(x, body, x.source);
-  } else { // update-only
-    CheckAtomicUpdateOnly(x, body, x.source);
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
-  if (visitedAtomicSource_.empty())
-    visitedAtomicSource_ = x.source;
-
-  // All of the following groups have the "exclusive" property, i.e. at
-  // most one clause from each group is allowed.
-  // The exclusivity-checking code should eventually be unified for all
-  // clauses, with clause groups defined in OMP.td.
-  std::array atomic{llvm::omp::Clause::OMPC_read,
-      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
-  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
-      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
-      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
-
-  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
-                          std::string_view name,
-                          const parser::OmpClauseList &clauses) {
-    const parser::OmpClause *present{nullptr};
-    for (const parser::OmpClause &clause : clauses.v) {
-      llvm::omp::Clause id{clause.Id()};
-      if (!llvm::is_contained(group, id)) {
-        continue;
-      }
-      if (present == nullptr) {
-        present = &clause;
-        continue;
-      } else if (id == present->Id()) {
-        // Ignore repetitions of the same clause, those will be diagnosed
-        // separately.
-        continue;
-      }
-      parser::MessageFormattedText txt(
-          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
-          name.data());
-      parser::Message message(clause.source, txt);
-      message.Attach(present->source,
-          "Previous clause from this group provided here"_en_US);
-      context_.Say(std::move(message));
-      return;
-    }
-  }};
-
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
-  llvm::omp::Clause kind{x.GetKind()};
-
-  checkExclusive(atomic, "atomic", dirSpec.Clauses());
-  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
-
-  switch (kind) {
-  case llvm::omp::Clause::OMPC_read:
-    CheckAtomicRead(x);
+void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
+  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  case llvm::omp::Directive::OMPD_scope:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
     break;
-  case llvm::omp::Clause::OMPC_write:
-    CheckAtomicWrite(x);
+  // 2.7.3 end-single-clause -> copyprivate-clause |
+  //                            nowait-clause
+  case llvm::omp::Directive::OMPD_single:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
     break;
-  case llvm::omp::Clause::OMPC_update:
-    CheckAtomicUpdate(x);
+  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
+  case llvm::omp::Directive::OMPD_workshare:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_workshare);
     break;
   default:
+    // no clauses are allowed
     break;
   }
 }
 
-void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
-  dirContext_.pop_back();
+// TODO: Verify the popping of dirContext requirement after nowait
+// implementation, as there is an implicit barrier at the end of the worksharing
+// constructs unless a nowait clause is specified. Only OMPD_end_single and
+// end_workshareare popped as they are pushed while entering the
+// EndBlockDirective.
+void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
+    dirContext_.pop_back();
+  }
 }
 
 // Clauses
@@ -5220,102 +3162,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
-  unsigned version{context_.langOptions().OpenMPVersion};
-  llvm::omp::Directive dir{GetContext().directive};
-  parser::CharBlock clauseSource{GetContext().clauseSource};
-  const parser::OmpLinearModifier *linearMod{nullptr};
-
-  SymbolSourceMap symbols;
-  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
-  CheckCrayPointee(objects, "LINEAR", false);
-  GetSymbolsInObjectList(objects, symbols);
-
-  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
-    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      context_.Say(source,
-          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
-          symbol->name(), desc.name.str());
-    }
-  }};
-
-  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
-    auto &modifiers{OmpGetModifiers(x.v)};
-    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
-    if (linearMod) {
-      // 2.7 Loop Construct Restriction
-      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
-        context_.Say(clauseSource,
-            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
-            ContextDirectiveAsFortran());
-        return;
-      }
-
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      for (auto &[symbol, source] : symbols) {
-        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
-          CheckIntegerNoRef(symbol, source);
-        } else {
-          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
-              !IsPolymorphic(*symbol)) {
-            context_.Say(source,
-                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
-                symbol->name(), desc.name.str());
-          }
-        }
-        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
-            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
-          if (!IsDummy(*symbol) || IsValue(*symbol)) {
-            context_.Say(source,
-                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
-                desc.name.str(), symbol->name());
-          }
-        }
-      } // for (symbol, source)
-
-      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
-        context_.Say(OmpGetModifierSource(modifiers, linearMod),
-            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
-            ThisVersion(version));
-      }
-    }
-  }
-
-  // OpenMP 5.2: Ordered clause restriction
-  if (const auto *clause{
-          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
-    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
-    if (orderedClause.v) {
-      return;
-    }
-  }
-
-  // OpenMP 5.2: Linear clause Restrictions
-  for (auto &[symbol, source] : symbols) {
-    if (!linearMod) {
-      // Already checked this with the modifier present.
-      CheckIntegerNoRef(symbol, source);
-    }
-    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
-      context_.Say(source,
-          "The list item `%s` must be a dummy argument"_err_en_US,
-          symbol->name());
-    }
-    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
-      context_.Say(source,
-          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
-          symbol->name());
-    }
-    if (FindCommonBlockContaining(*symbol)) {
-      context_.Say(source,
-          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
-          symbol->name());
-    }
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OmpClause::Detach &x) {
   unsigned version{context_.langOptions().OpenMPVersion};
   if (version >= 52) {
@@ -6024,503 +3870,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::OmpxBare &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
-  OmpVerifyModifiers(
-      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
-}
-
-void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
-  EnterDirectiveNest(ContextSelectorNest);
-
-  using SetName = parser::OmpTraitSetSelectorName;
-  std::map<SetName::Value, const SetName *> visited;
-
-  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
-    auto &name{std::get<SetName>(traitSet.t)};
-    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait set name %s in a context specifier"_err_en_US,
-          showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait set %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-    CheckTraitSetSelector(traitSet);
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
-  ExitDirectiveNest(ContextSelectorNest);
-}
-
-const std::list<parser::OmpTraitProperty> &
-OmpStructureChecker::GetTraitPropertyList(
-    const parser::OmpTraitSelector &trait) {
-  static const std::list<parser::OmpTraitProperty> empty{};
-  auto &[_, maybeProps]{trait.t};
-  if (maybeProps) {
-    using PropertyList = std::list<parser::OmpTraitProperty>;
-    return std::get<PropertyList>(maybeProps->t);
-  } else {
-    return empty;
-  }
-}
-
-std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
-    const parser::OmpTraitProperty &property) {
-  using MaybeClause = std::optional<llvm::omp::Clause>;
-
-  // The parser for OmpClause will only succeed if the clause was
-  // given with all required arguments.
-  // If this is a string or complex extension with a clause name,
-  // treat it as a clause and let the trait checker deal with it.
-
-  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
-    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
-    if (id != llvm::omp::Clause::OMPC_unknown) {
-      return id;
-    } else {
-      return std::nullopt;
-    }
-  }};
-
-  return common::visit( //
-      common::visitors{
-          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
-            return getClauseFromString(x.v);
-          },
-          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
-            return x.value().Id();
-          },
-          [&](const parser::ScalarExpr &x) -> MaybeClause {
-            return std::nullopt;
-          },
-          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
-            using ExtProperty = parser::OmpTraitPropertyExtension;
-            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
-              return getClauseFromString(name->v);
-            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
-              return getClauseFromString(
-                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
-            }
-            return std::nullopt;
-          },
-      },
-      property.u);
-}
-
-void OmpStructureChecker::CheckTraitSelectorList(
-    const std::list<parser::OmpTraitSelector> &traits) {
-  // [6.0:322:20]
-  // Each trait-selector-name may only be specified once in a trait selector
-  // set.
-
-  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
-  using TraitName = parser::OmpTraitSelectorName;
-  using BareName = decltype(TraitName::u);
-  std::map<BareName, const TraitName *> visited;
-
-  for (const parser::OmpTraitSelector &trait : traits) {
-    auto &name{std::get<TraitName>(trait.t)};
-
-    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait name %s in a trait set"_err_en_US, showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSetSelector(
-    const parser::OmpTraitSetSelector &traitSet) {
-
-  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
-  //
-  // Construct      |     Simd, directive-name |      Yes |       No |    No |
-  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
-  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
-  //                |      Extension, Requires |          |          |       |
-  //                |                   Vendor |          |          |       |
-  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
-  //                |                Kind, Uid |          |          |       |
-  // User           |                Condition |       No |       No |   Yes |
-
-  struct TraitSetConfig {
-    std::set<parser::OmpTraitSelectorName::Value> allowed;
-    bool allowsDirectiveTraits;
-    bool allowsExtensionTraits;
-    bool allowsScore;
-  };
-
-  using SName = parser::OmpTraitSetSelectorName::Value;
-  using TName = parser::OmpTraitSelectorName::Value;
-
-  static const std::map<SName, TraitSetConfig> configs{
-      {SName::Construct, //
-          {{TName::Simd}, true, false, false}},
-      {SName::Device, //
-          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
-      {SName::Implementation, //
-          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
-               TName::Vendor},
-              false, true, true}},
-      {SName::Target_Device, //
-          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
-               TName::Uid},
-              false, true, false}},
-      {SName::User, //
-          {{TName::Condition}, false, false, true}},
-  };
-
-  auto checkTraitSet{[&](const TraitSetConfig &config) {
-    auto &[setName, traits]{traitSet.t};
-    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
-
-    // Check if there are any duplicate traits.
-    CheckTraitSelectorList(traits);
-
-    for (const parser::OmpTraitSelector &trait : traits) {
-      // Don't use structured bindings here, because they cannot be captured
-      // before C++20.
-      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
-      auto &maybeProps =
-          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
-              trait.t);
-
-      // Check allowed traits
-      common::visit( //
-          common::visitors{
-              [&](parser::OmpTraitSelectorName::Value v) {
-                if (!config.allowed.count(v)) {
-                  context_.Say(traitName.source,
-                      "%s is not a valid trait for %s trait set"_err_en_US,
-                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
-                }
-              },
-              [&](llvm::omp::Directive) {
-                if (!config.allowsDirectiveTraits) {
-                  context_.Say(traitName.source,
-                      "Directive name is not a valid trait for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-              [&](const std::string &) {
-                if (!config.allowsExtensionTraits) {
-                  context_.Say(traitName.source,
-                      "Extension traits are not valid for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-          },
-          traitName.u);
-
-      // Check score
-      if (maybeProps) {
-        auto &[maybeScore, _]{maybeProps->t};
-        if (maybeScore) {
-          CheckTraitScore(*maybeScore);
-        }
-      }
-
-      // Check the properties of the individual traits
-      CheckTraitSelector(traitSet, trait);
-    }
-  }};
-
-  checkTraitSet(
-      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
-}
-
-void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
-  // [6.0:322:23]
-  // A score-expression must be a non-negative constant integer expression.
-  if (auto value{GetIntValue(score)}; !value || value < 0) {
-    context_.Say(score.source,
-        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
-  }
-}
-
-bool OmpStructureChecker::VerifyTraitPropertyLists(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  using PropertyList = std::list<parser::OmpTraitProperty>;
-  auto &[traitName, maybeProps]{trait.t};
-
-  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
-                             const std::string &message) {
-    bool foundInvalid{false};
-    for (const parser::OmpTraitProperty &prop : properties) {
-      if (!isValid(prop)) {
-        if (foundInvalid) {
-          context_.Say(
-              prop.source, "More invalid properties are present"_err_en_US);
-          break;
-        }
-        context_.Say(prop.source, "%s"_err_en_US, message);
-        foundInvalid = true;
-      }
-    }
-    return !foundInvalid;
-  }};
-
-  bool invalid{false};
-
-  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
-    // Directive-name traits don't have properties.
-    if (maybeProps) {
-      context_.Say(trait.source,
-          "Directive-name traits cannot have properties"_err_en_US);
-      invalid = true;
-    }
-  }
-  // Ignore properties on extension traits.
-
-  // See `TraitSelectorParser` in openmp-parser.cpp
-  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
-    switch (*v) {
-    // name-list properties
-    case parser::OmpTraitSelectorName::Value::Arch:
-    case parser::OmpTraitSelectorName::Value::Extension:
-    case parser::OmpTraitSelectorName::Value::Isa:
-    case parser::OmpTraitSelectorName::Value::Kind:
-    case parser::OmpTraitSelectorName::Value::Uid:
-    case parser::OmpTraitSelectorName::Value::Vendor:
-      if (maybeProps) {
-        auto isName{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isName, "Trait property should be a name");
-      }
-      break;
-    // clause-list
-    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
-    case parser::OmpTraitSelectorName::Value::Requires:
-    case parser::OmpTraitSelectorName::Value::Simd:
-      if (maybeProps) {
-        auto isClause{[&](const parser::OmpTraitProperty &prop) {
-          return GetClauseFromProperty(prop).has_value();
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isClause, "Trait property should be a clause");
-      }
-      break;
-    // expr-list
-    case parser::OmpTraitSelectorName::Value::Condition:
-    case parser::OmpTraitSelectorName::Value::Device_Num:
-      if (maybeProps) {
-        auto isExpr{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::ScalarExpr>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isExpr, "Trait property should be a scalar expression");
-      }
-      break;
-    } // switch
-  }
-
-  return !invalid;
-}
-
-void OmpStructureChecker::CheckTraitSelector(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  auto &[traitName, maybeProps]{trait.t};
-
-  // Only do the detailed checks if the property lists are valid.
-  if (VerifyTraitPropertyLists(traitSet, trait)) {
-    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
-        std::holds_alternative<std::string>(traitName.u)) {
-      // No properties here: directives don't have properties, and
-      // we don't implement any extension traits now.
-      return;
-    }
-
-    // Specific traits we want to check.
-    // Limitations:
-    // (1) The properties for these traits are defined in "Additional
-    // Definitions for the OpenMP API Specification". It's not clear how
-    // to define them in a portable way, and how to verify their validity,
-    // especially if they get replaced by their integer values (in case
-    // they are defined as enums).
-    // (2) These are entirely implementation-defined, and at the moment
-    // there is no known schema to validate these values.
-    auto v{std::get<TraitName::Value>(traitName.u)};
-    switch (v) {
-    case TraitName::Value::Arch:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Atomic_Default_Mem_Order:
-      CheckTraitADMO(traitSet, trait);
-      break;
-    case TraitName::Value::Condition:
-      CheckTraitCondition(traitSet, trait);
-      break;
-    case TraitName::Value::Device_Num:
-      CheckTraitDeviceNum(traitSet, trait);
-      break;
-    case TraitName::Value::Extension:
-      // Ignore
-      break;
-    case TraitName::Value::Isa:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Kind:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Requires:
-      CheckTraitRequires(traitSet, trait);
-      break;
-    case TraitName::Value::Simd:
-      CheckTraitSimd(traitSet, trait);
-      break;
-    case TraitName::Value::Uid:
-      // Unchecked, TBD(2)
-      break;
-    case TraitName::Value::Vendor:
-      // Unchecked, TBD(1)
-      break;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitADMO(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single clause property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto clauseId{*GetClauseFromProperty(property)};
-    // Check that the clause belongs to the memory-order clause-set.
-    // Clause sets will hopefully be autogenerated at some point.
-    switch (clauseId) {
-    case llvm::omp::Clause::OMPC_acq_rel:
-    case llvm::omp::Clause::OMPC_acquire:
-    case llvm::omp::Clause::OMPC_relaxed:
-    case llvm::omp::Clause::OMPC_release:
-    case llvm::omp::Clause::OMPC_seq_cst:
-      break;
-    default:
-      context_.Say(property.source,
-          "%s trait requires a clause from the memory-order clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitCondition(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
-
-    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
-    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
-      context_.Say(property.source,
-          "%s trait requires a single LOGICAL expression"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitDeviceNum(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  }
-  // No other checks at the moment.
-}
-
-void OmpStructureChecker::CheckTraitRequires(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_requires, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause from the requirement clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSimd(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_declare_simd, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()),
-          parser::ToUpperCaseLetters(
-              getDirectiveName(llvm::omp::OMPD_declare_simd)));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
 llvm::StringRef OmpStructureChecker::getClauseName(llvm::omp::Clause clause) {
   return llvm::omp::getOpenMPClauseName(clause);
 }
@@ -6531,39 +3880,6 @@ llvm::StringRef OmpStructureChecker::getDirectiveName(
   return llvm::omp::getOpenMPDirectiveName(directive, version);
 }
 
-const Symbol *OmpStructureChecker::GetObjectSymbol(
-    const parser::OmpObject &object) {
-  // Some symbols may be missing if the resolution failed, e.g. when an
-  // undeclared name is used with implicit none.
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    auto &last{GetLastName(*desg)};
-    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
-  }
-  return nullptr;
-}
-
-const Symbol *OmpStructureChecker::GetArgumentSymbol(
-    const parser::OmpArgument &argument) {
-  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
-    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
-      return GetObjectSymbol(*object);
-    }
-  }
-  return nullptr;
-}
-
-std::optional<parser::CharBlock> OmpStructureChecker::GetObjectSource(
-    const parser::OmpObject &object) {
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->source;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    return GetLastName(*desg).source;
-  }
-  return std::nullopt;
-}
-
 void OmpStructureChecker::CheckDependList(const parser::DataRef &d) {
   common::visit(
       common::visitors{
@@ -6873,7 +4189,7 @@ void OmpStructureChecker::CheckWorkshareBlockStmts(
 }
 
 void OmpStructureChecker::CheckIfContiguous(const parser::OmpObject &object) {
-  if (auto contig{IsContiguous(object)}; contig && !*contig) {
+  if (auto contig{IsContiguous(context_, object)}; contig && !*contig) {
     const parser::Name *name{GetObjectName(object)};
     assert(name && "Expecting name component");
     context_.Say(name->source,
@@ -6976,22 +4292,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::SelfMaps &x) {
   CheckAllowedRequiresClause(llvm::omp::Clause::OMPC_self_maps);
 }
 
-void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
-  Base::Enter(x);
-  loopStack_.push_back(&x);
-}
-
-void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top = loopStack_.back();
-  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
-  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-  Base::Leave(x);
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPInteropConstruct &x) {
   bool isDependClauseOccured{false};
   int targetCount{0}, targetSyncCount{0};

diff  --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index beb6e0528e814..2a3853335fd1c 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -162,10 +162,6 @@ class OmpStructureChecker
 
 private:
   bool CheckAllowedClause(llvmOmpClause clause);
-  bool IsVariableListItem(const Symbol &sym);
-  bool IsExtendedListItem(const Symbol &sym);
-  bool IsCommonBlock(const Symbol &sym);
-  std::optional<bool> IsContiguous(const parser::OmpObject &object);
   void CheckVariableListItem(const SymbolSourceMap &symbols);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
@@ -215,10 +211,6 @@ class OmpStructureChecker
       typename IterTy = decltype(std::declval<RangeTy>().begin())>
   std::optional<IterTy> FindDuplicate(RangeTy &&);
 
-  const Symbol *GetObjectSymbol(const parser::OmpObject &object);
-  const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
-  std::optional<parser::CharBlock> GetObjectSource(
-      const parser::OmpObject &object);
   void CheckDependList(const parser::DataRef &);
   void CheckDependArraySection(
       const common::Indirection<parser::ArrayElement> &, const parser::Name &);

diff  --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
new file mode 100644
index 0000000000000..fd9596a09cd52
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -0,0 +1,393 @@
+//===-- lib/Semantics/openmp-utils.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
+
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Common/reference.h"
+#include "flang/Common/visit.h"
+#include "flang/Evaluate/check-expression.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
+#include "flang/Evaluate/variable.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/expression.h"
+#include "flang/Semantics/semantics.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+
+#include <optional>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics::omp {
+
+std::string ThisVersion(unsigned version) {
+  std::string tv{
+      std::to_string(version / 10) + "." + std::to_string(version % 10)};
+  return "OpenMP v" + tv;
+}
+
+std::string TryVersion(unsigned version) {
+  return "try -fopenmp-version=" + std::to_string(version);
+}
+
+const parser::Designator *GetDesignatorFromObj(
+    const parser::OmpObject &object) {
+  return std::get_if<parser::Designator>(&object.u);
+}
+
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object) {
+  if (auto *desg{GetDesignatorFromObj(object)}) {
+    return std::get_if<parser::DataRef>(&desg->u);
+  }
+  return nullptr;
+}
+
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object) {
+  if (auto *dataRef{GetDataRefFromObj(object)}) {
+    using ElementIndirection = common::Indirection<parser::ArrayElement>;
+    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
+      return &ind->value();
+    }
+  }
+  return nullptr;
+}
+
+const Symbol *GetObjectSymbol(const parser::OmpObject &object) {
+  // Some symbols may be missing if the resolution failed, e.g. when an
+  // undeclared name is used with implicit none.
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    auto &last{GetLastName(*desg)};
+    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
+  }
+  return nullptr;
+}
+
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument) {
+  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
+    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
+      return GetObjectSymbol(*object);
+    }
+  }
+  return nullptr;
+}
+
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object) {
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->source;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    return GetLastName(*desg).source;
+  }
+  return std::nullopt;
+}
+
+bool IsCommonBlock(const Symbol &sym) {
+  return sym.detailsIf<CommonBlockDetails>() != nullptr;
+}
+
+bool IsVariableListItem(const Symbol &sym) {
+  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
+}
+
+bool IsExtendedListItem(const Symbol &sym) {
+  return IsVariableListItem(sym) || sym.IsSubprogram();
+}
+
+bool IsVarOrFunctionRef(const MaybeExpr &expr) {
+  if (expr) {
+    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
+        evaluate::IsVariable(*expr);
+  } else {
+    return false;
+  }
+}
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
+  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
+  // ForwardOwningPointer           typedExpr
+  // `- GenericExprWrapper          ^.get()
+  //    `- std::optional<Expr>      ^->v
+  return typedExpr.get()->v;
+}
+
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr) {
+  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
+    return maybeExpr->GetType();
+  } else {
+    return std::nullopt;
+  }
+}
+
+namespace {
+struct ContiguousHelper {
+  ContiguousHelper(SemanticsContext &context)
+      : fctx_(context.foldingContext()) {}
+
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
+    return Visit(x.value());
+  }
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Reference<Contained> &x) {
+    return Visit(x.get());
+  }
+  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
+    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
+  }
+  template <typename T>
+  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
+    return common::visit(
+        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
+  }
+  template <typename T> std::optional<bool> Visit(const T &) {
+    // Everything else.
+    return std::nullopt;
+  }
+
+private:
+  evaluate::FoldingContext &fctx_;
+};
+} // namespace
+
+// Return values:
+// - std::optional<bool>{true} if the object is known to be contiguous
+// - std::optional<bool>{false} if the object is known not to be contiguous
+// - std::nullopt if the object contiguity cannot be determined
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object) {
+  return common::visit( //
+      common::visitors{
+          [&](const parser::Name &x) {
+            // Any member of a common block must be contiguous.
+            return std::optional<bool>{true};
+          },
+          [&](const parser::Designator &x) {
+            evaluate::ExpressionAnalyzer ea{semaCtx};
+            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
+              return ContiguousHelper{semaCtx}.Visit(*maybeExpr);
+            }
+            return std::optional<bool>{};
+          },
+      },
+      object.u);
+}
+
+struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
+                                 std::vector<SomeExpr>, false> {
+  using Result = std::vector<SomeExpr>;
+  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
+  DesignatorCollector() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <typename T> //
+  Result operator()(const evaluate::Designator<T> &x) const {
+    // Once in a designator, don't traverse it any further (i.e. only
+    // collect top-level designators).
+    auto copy{x};
+    return Result{AsGenericExpr(std::move(copy))};
+  }
+
+  template <typename... Rs> //
+  Result Combine(Result &&result, Rs &&...results) const {
+    Result v(std::move(result));
+    auto moveAppend{[](auto &accum, auto &&other) {
+      for (auto &&s : other) {
+        accum.push_back(std::move(s));
+      }
+    }};
+    (moveAppend(v, std::move(results)), ...);
+    return v;
+  }
+};
+
+struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
+  using Base = evaluate::AnyTraverse<VariableFinder>;
+  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
+
+  using Base::operator();
+
+  template <typename T>
+  bool operator()(const evaluate::Designator<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+  template <typename T>
+  bool operator()(const evaluate::FunctionRef<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+private:
+  const SomeExpr &var;
+};
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr) {
+  return DesignatorCollector{}(expr);
+}
+
+static bool HasCommonDesignatorSymbols(
+    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
+  // Compare the designators used in "other" with the designators whose
+  // symbols are given in baseSyms.
+  // This is a part of the check if these two expressions can access the same
+  // storage: if the designators used in them are 
diff erent enough, then they
+  // will be assumed not to access the same memory.
+  //
+  // Consider an (array element) expression x%y(w%z), the corresponding symbol
+  // vector will be {x, y, w, z} (i.e. the symbols for these names).
+  // Check whether this exact sequence appears anywhere in any the symbol
+  // vector for "other". This will be true for x(y) and x(y+1), so this is
+  // not a sufficient condition, but can be used to eliminate candidates
+  // before doing more exhaustive checks.
+  //
+  // If any of the symbols in this sequence are function names, assume that
+  // there is no storage overlap, mostly because it would be impossible in
+  // general to determine what storage the function will access.
+  // Note: if f is pure, then two calls to f will access the same storage
+  // when called with the same arguments. This check is not done yet.
+
+  if (llvm::any_of(
+          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
+    // If there is a function symbol in the chain then we can't infer much
+    // about the accessed storage.
+    return false;
+  }
+
+  auto isSubsequence{// Is u a subsequence of v.
+      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
+        size_t us{u.size()}, vs{v.size()};
+        if (us > vs) {
+          return false;
+        }
+        for (size_t off{0}; off != vs - us + 1; ++off) {
+          bool same{true};
+          for (size_t i{0}; i != us; ++i) {
+            if (u[i] != v[off + i]) {
+              same = false;
+              break;
+            }
+          }
+          if (same) {
+            return true;
+          }
+        }
+        return false;
+      }};
+
+  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
+  return isSubsequence(baseSyms, otherSyms);
+}
+
+static bool HasCommonTopLevelDesignators(
+    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
+  // Compare designators directly as expressions. This will ensure
+  // that x(y) and x(y+1) are not flagged as overlapping, whereas
+  // the symbol vectors for both of these would be identical.
+  std::vector<SomeExpr> otherDsgs{GetAllDesignators(other)};
+
+  for (auto &s : baseDsgs) {
+    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
+      return true;
+    }
+  }
+  return false;
+}
+
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
+  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
+  std::vector<SomeExpr> baseDsgs{GetAllDesignators(base)};
+
+  for (const SomeExpr &expr : exprs) {
+    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
+      continue;
+    }
+    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
+      return &expr;
+    }
+  }
+  return nullptr;
+}
+
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
+  return VariableFinder{sub}(super);
+}
+
+// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
+// to separate cases where the source has something that looks like an
+// assignment, but is semantically wrong (diagnosed by general semantic
+// checks), and where the source has some other statement (which we want
+// to report as "should be an assignment").
+bool IsAssignment(const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return false;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+
+  return common::visit(
+      [](auto &&s) -> bool {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        return std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>;
+      },
+      x->u);
+}
+
+bool IsPointerAssignment(const evaluate::Assignment &x) {
+  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
+      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
+}
+
+/// parser::Block is a list of executable constructs, parser::BlockConstruct
+/// is Fortran's BLOCK/ENDBLOCK construct.
+/// Strip the outermost BlockConstructs, return the reference to the Block
+/// in the executable part of the innermost of the stripped constructs.
+/// Specifically, if the given `block` has a single entry (it's a list), and
+/// the entry is a BlockConstruct, get the Block contained within. Repeat
+/// this step as many times as possible.
+const parser::Block &GetInnermostExecPart(const parser::Block &block) {
+  const parser::Block *iter{&block};
+  while (iter->size() == 1) {
+    const parser::ExecutionPartConstruct &ep{iter->front()};
+    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
+      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
+      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
+        iter = &std::get<parser::Block>(bc->value().t);
+        continue;
+      }
+    }
+    break;
+  }
+  return *iter;
+}
+
+} // namespace Fortran::semantics::omp

diff  --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
new file mode 100644
index 0000000000000..dbb0565215357
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -0,0 +1,66 @@
+//===-- lib/Semantics/openmp-utils.h --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_OPENMP_UTILS_H
+#define FORTRAN_SEMANTICS_OPENMP_UTILS_H
+
+#include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include <optional>
+#include <string>
+
+namespace Fortran::semantics {
+class SemanticsContext;
+class Symbol;
+
+// Add this namespace to avoid potential conflicts
+namespace omp {
+std::string ThisVersion(unsigned version);
+std::string TryVersion(unsigned version);
+
+const parser::Designator *GetDesignatorFromObj(const parser::OmpObject &object);
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object);
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object);
+const Symbol *GetObjectSymbol(const parser::OmpObject &object);
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object);
+
+bool IsCommonBlock(const Symbol &sym);
+bool IsExtendedListItem(const Symbol &sym);
+bool IsVariableListItem(const Symbol &sym);
+bool IsVarOrFunctionRef(const MaybeExpr &expr);
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr);
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr);
+
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object);
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr);
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs);
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super);
+bool IsAssignment(const parser::ActionStmt *x);
+bool IsPointerAssignment(const evaluate::Assignment &x);
+const parser::Block &GetInnermostExecPart(const parser::Block &block);
+} // namespace omp
+} // namespace Fortran::semantics
+
+#endif // FORTRAN_SEMANTICS_OPENMP_UTILS_H


        

From flang-commits at lists.llvm.org  Wed Jul  2 02:25:49 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 02 Jul 2025 02:25:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Create unique reduction
 decls for different logical kinds (PR #146558)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146558@github.com>
Message-ID: <6864fb1d.170a0220.233758.7904@mx.google.com>


https://github.com/tblah closed https://github.com/llvm/llvm-project/pull/146558

From flang-commits at lists.llvm.org  Wed Jul  2 06:38:49 2025
From: flang-commits at lists.llvm.org (Anchu Rajendran S via flang-commits)
Date: Wed, 02 Jul 2025 06:38:49 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <68653669.050a0220.8fbbd.1860@mx.google.com>


================
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic control options for AMD gpu
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
----------------
anchuraj wrote:

I agree, Thank you! 

https://github.com/llvm/llvm-project/pull/143441

From flang-commits at lists.llvm.org  Wed Jul  2 07:35:41 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 02 Jul 2025 07:35:41 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add ComplexTOROCDL pass (PR
	#144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <686543bd.050a0220.3cd191.4111@mx.google.com>


https://github.com/TIFitis updated https://github.com/llvm/llvm-project/pull/144926

>From ba1230d545d045623c1ab3e202b7e2b836efe02f Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 19 Jun 2025 17:25:48 +0100
Subject: [PATCH 1/6] [MLIR] Add ComplexTOROCDL pass

This patch adds a new ComplexToROCDL pass to convert complex.abs operations to __ocml_cabs_f32/__ocml_cabs_f64 calls.
---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt    |  1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  5 +-
 .../ComplexToROCDL/ComplexToROCDL.h           | 19 ++++
 mlir/include/mlir/Conversion/Passes.h         |  1 +
 mlir/include/mlir/Conversion/Passes.td        | 12 +++
 mlir/lib/Conversion/CMakeLists.txt            |  1 +
 .../Conversion/ComplexToROCDL/CMakeLists.txt  | 18 ++++
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 95 +++++++++++++++++++
 .../ComplexToROCDL/complex-to-rocdl.mlir      | 13 +++
 9 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
 create mode 100644 mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
 create mode 100644 mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
 create mode 100644 mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 980307db315d9..8b4ac18fba527 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -40,6 +40,7 @@ add_flang_library(FIRCodeGen
   MLIRMathToLLVM
   MLIRMathToLibm
   MLIRMathToROCDL
+  MLIRComplexToROCDL
   MLIROpenMPToLLVM
   MLIROpenACCDialect
   MLIRBuiltinToLLVMIRTranslation
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..f721b6232b0fb 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -33,6 +33,7 @@
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
@@ -4105,8 +4106,10 @@ class FIRToLLVMLowering
     // GPU library calls, the rest can be converted to LLVM intrinsics, which
     // is handled in the mathToLLVM conversion. The lowering to libm calls is
     // not needed since all math operations are handled this way.
-    if (isAMDGCN)
+    if (isAMDGCN) {
       mathConvertionPM.addPass(mlir::createConvertMathToROCDL());
+      mathConvertionPM.addPass(mlir::createConvertComplexToROCDL());
+    }
 
     // Convert math::FPowI operations to inline implementation
     // only if the exponent's width is greater than 32, otherwise,
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
new file mode 100644
index 0000000000000..ed65be9980408
--- /dev/null
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -0,0 +1,19 @@
+#ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+class RewritePatternSet;
+
+#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDL
+#include "mlir/Conversion/Passes.h.inc"
+
+/// Populate the given list with patterns that convert from Complex to ROCDL
+/// calls.
+void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
+                                              PatternBenefit benefit);
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index c9d2a54433736..67e8f5b99b67b 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -23,6 +23,7 @@
 #include "mlir/Conversion/BufferizationToMemRef/BufferizationToMemRef.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
 #include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h"
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRVPass.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index b496ee0114910..8ad2341f93a15 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -312,6 +312,18 @@ def ConvertComplexToLibm : Pass<"convert-complex-to-libm", "ModuleOp"> {
   let dependentDialects = ["func::FuncDialect"];
 }
 
+//===----------------------------------------------------------------------===//
+// ComplexToROCDL
+//===----------------------------------------------------------------------===//
+
+def ConvertComplexToROCDL : Pass<"convert-complex-to-rocdl", "ModuleOp"> {
+  let summary = "Convert Complex dialect to ROCDL calls";
+  let description = [{
+    This pass converts supported Complex ops to calls to the AMD device library.
+  }];
+  let dependentDialects = ["func::FuncDialect"];
+}
+
 //===----------------------------------------------------------------------===//
 // ComplexToSPIRV
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index e4b4974600577..4ad81553a4fa8 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(AsyncToLLVM)
 add_subdirectory(BufferizationToMemRef)
 add_subdirectory(ComplexCommon)
 add_subdirectory(ComplexToLibm)
+add_subdirectory(ComplexToROCDL)
 add_subdirectory(ComplexToLLVM)
 add_subdirectory(ComplexToSPIRV)
 add_subdirectory(ComplexToStandard)
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
new file mode 100644
index 0000000000000..54607250083d7
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_conversion_library(MLIRComplexToROCDL
+  ComplexToROCDL.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDL
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRComplexDialect
+  MLIRFuncDialect
+  MLIRPass
+  MLIRTransformUtils
+  )
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
new file mode 100644
index 0000000000000..cdfe2a6dfe874
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -0,0 +1,95 @@
+//===-- ComplexToROCDL.cpp - conversion from Complex to ROCDL calls -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+
+#include "mlir/Dialect/Complex/IR/Complex.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include <optional>
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+
+namespace {
+struct FloatTypeResolver {
+  std::optional<bool> operator()(Type type) const {
+    auto elementType = cast<FloatType>(type);
+    if (!isa<Float32Type, Float64Type>(elementType))
+      return {};
+    return elementType.getIntOrFloatBitWidth() == 64;
+  }
+};
+
+template <typename Op, typename TypeResolver = FloatTypeResolver>
+struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
+  using OpRewritePattern<Op>::OpRewritePattern;
+  ScalarOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
+                      StringRef doubleFunc, PatternBenefit benefit)
+      : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
+        doubleFunc(doubleFunc) {}
+
+  LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
+    auto module = SymbolTable::getNearestSymbolTable(op);
+    auto isDouble = TypeResolver()(op.getType());
+    if (!isDouble.has_value())
+      return failure();
+
+    auto name = *isDouble ? doubleFunc : floatFunc;
+
+    auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
+        SymbolTable::lookupSymbolIn(module, name));
+    if (!opFunc) {
+      OpBuilder::InsertionGuard guard(rewriter);
+      rewriter.setInsertionPointToStart(&module->getRegion(0).front());
+      auto funcTy = FunctionType::get(
+          rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
+      opFunc =
+          rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name, funcTy);
+      opFunc.setPrivate();
+    }
+    rewriter.replaceOpWithNewOp<func::CallOp>(op, name, op.getType(),
+                                              op->getOperands());
+    return success();
+  }
+
+private:
+  std::string floatFunc, doubleFunc;
+};
+} // namespace
+
+void mlir::populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
+                                                    PatternBenefit benefit) {
+  patterns.add<ScalarOpToROCDLCall<complex::AbsOp>>(
+      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64", benefit);
+}
+
+namespace {
+struct ConvertComplexToROCDLPass
+    : public impl::ConvertComplexToROCDLBase<ConvertComplexToROCDLPass> {
+  void runOnOperation() override;
+};
+} // namespace
+
+void ConvertComplexToROCDLPass::runOnOperation() {
+  auto module = getOperation();
+
+  RewritePatternSet patterns(&getContext());
+  populateComplexToROCDLConversionPatterns(patterns, /*benefit=*/1);
+
+  ConversionTarget target(getContext());
+  target.addLegalDialect<func::FuncDialect>();
+  target.addIllegalOp<complex::AbsOp>();
+  if (failed(applyPartialConversion(module, target, std::move(patterns))))
+    signalPassFailure();
+}
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
new file mode 100644
index 0000000000000..618e9c238378c
--- /dev/null
+++ b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
@@ -0,0 +1,13 @@
+// RUN: mlir-opt %s -convert-complex-to-rocdl -canonicalize | FileCheck %s
+
+// CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
+// CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64
+
+func.func @abs_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
+  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%[[F:.*]])
+  %rf = complex.abs %f : complex<f32>
+  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%[[D:.*]])
+  %rd = complex.abs %d : complex<f64>
+  // CHECK: return %[[RF]], %[[RD]]
+  return %rf, %rd : f32, f64
+}

>From a10e50ad76963f65d3b84ff152159755a7efdcfe Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 19 Jun 2025 17:38:09 +0100
Subject: [PATCH 2/6] Added license to header.

---
 .../mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h       | 8 ++++++++
 mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp     | 1 -
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
index ed65be9980408..96d5a352f54c8 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -1,3 +1,11 @@
+//===-- ComplexToROCDL.h - conversion from Complex to ROCDL calls ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
 #ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
 #define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
 
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index cdfe2a6dfe874..7c7510b5c4e10 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
-
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"

>From bb88939bc53d9e88973749c63ba21dfa26279258 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Fri, 20 Jun 2025 16:22:07 +0100
Subject: [PATCH 3/6] Address reviewer changes. Add conversion for complex.exp.

---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 14 ++---
 .../ComplexToROCDL/ComplexToROCDL.h           |  3 +-
 .../Conversion/ComplexToROCDL/CMakeLists.txt  |  3 -
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 57 ++++++++++---------
 .../ComplexToROCDL/complex-to-rocdl.mlir      | 19 ++++++-
 5 files changed, 54 insertions(+), 42 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index f721b6232b0fb..b8c7cba80d863 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -4099,7 +4099,7 @@ class FIRToLLVMLowering
     // conversions that affect the ModuleOp, e.g. create new
     // function operations in it. We have to run such conversions
     // as passes here.
-    mlir::OpPassManager mathConvertionPM("builtin.module");
+    mlir::OpPassManager mathConversionPM("builtin.module");
 
     bool isAMDGCN = fir::getTargetTriple(mod).isAMDGCN();
     // If compiling for AMD target some math operations must be lowered to AMD
@@ -4107,8 +4107,8 @@ class FIRToLLVMLowering
     // is handled in the mathToLLVM conversion. The lowering to libm calls is
     // not needed since all math operations are handled this way.
     if (isAMDGCN) {
-      mathConvertionPM.addPass(mlir::createConvertMathToROCDL());
-      mathConvertionPM.addPass(mlir::createConvertComplexToROCDL());
+      mathConversionPM.addPass(mlir::createConvertMathToROCDL());
+      mathConversionPM.addPass(mlir::createConvertComplexToROCDL());
     }
 
     // Convert math::FPowI operations to inline implementation
@@ -4116,15 +4116,15 @@ class FIRToLLVMLowering
     // it will be lowered to LLVM intrinsic operation by a later conversion.
     mlir::ConvertMathToFuncsOptions mathToFuncsOptions{};
     mathToFuncsOptions.minWidthOfFPowIExponent = 33;
-    mathConvertionPM.addPass(
+    mathConversionPM.addPass(
         mlir::createConvertMathToFuncs(mathToFuncsOptions));
-    mathConvertionPM.addPass(mlir::createConvertComplexToStandardPass());
+    mathConversionPM.addPass(mlir::createConvertComplexToStandardPass());
     // Convert Math dialect operations into LLVM dialect operations.
     // There is no way to prefer MathToLLVM patterns over MathToLibm
     // patterns (applied below), so we have to run MathToLLVM conversion here.
-    mathConvertionPM.addNestedPass<mlir::func::FuncOp>(
+    mathConversionPM.addNestedPass<mlir::func::FuncOp>(
         mlir::createConvertMathToLLVMPass());
-    if (mlir::failed(runPipeline(mathConvertionPM, mod)))
+    if (mlir::failed(runPipeline(mathConversionPM, mod)))
       return signalPassFailure();
 
     std::optional<mlir::DataLayout> dl =
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
index 96d5a352f54c8..eb785080adab3 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -20,8 +20,7 @@ class RewritePatternSet;
 
 /// Populate the given list with patterns that convert from Complex to ROCDL
 /// calls.
-void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
-                                              PatternBenefit benefit);
+void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns);
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
index 54607250083d7..133809ac32f0f 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
+++ b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
@@ -7,9 +7,6 @@ add_mlir_conversion_library(MLIRComplexToROCDL
   DEPENDS
   MLIRConversionPassIncGen
 
-  LINK_COMPONENTS
-  Core
-
   LINK_LIBS PUBLIC
   MLIRComplexDialect
   MLIRFuncDialect
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index 7c7510b5c4e10..98adb9fb1f607 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -11,7 +11,6 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include <optional>
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
@@ -21,36 +20,38 @@ namespace mlir {
 using namespace mlir;
 
 namespace {
-struct FloatTypeResolver {
-  std::optional<bool> operator()(Type type) const {
-    auto elementType = cast<FloatType>(type);
-    if (!isa<Float32Type, Float64Type>(elementType))
-      return {};
-    return elementType.getIntOrFloatBitWidth() == 64;
-  }
-};
 
-template <typename Op, typename TypeResolver = FloatTypeResolver>
-struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
+template <typename Op>
+// Pattern to convert Complex ops to ROCDL function calls.
+struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ScalarOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
-                      StringRef doubleFunc, PatternBenefit benefit)
+  ComplexOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
+                       StringRef doubleFunc, PatternBenefit benefit = 1)
       : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
         doubleFunc(doubleFunc) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
-    auto module = SymbolTable::getNearestSymbolTable(op);
-    auto isDouble = TypeResolver()(op.getType());
-    if (!isDouble.has_value())
+    Operation *symTable = SymbolTable::getNearestSymbolTable(op);
+    Type resType = op.getType();
+    if (auto complexType = dyn_cast<ComplexType>(resType))
+      resType = complexType.getElementType();
+    FloatType floatTy = dyn_cast<FloatType>(resType);
+    if (!floatTy)
       return failure();
 
-    auto name = *isDouble ? doubleFunc : floatFunc;
+    StringRef name;
+    if (floatTy.isF64())
+      name = doubleFunc;
+    else if (floatTy.isF32())
+      name = floatFunc;
+    else
+      return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
-        SymbolTable::lookupSymbolIn(module, name));
+        SymbolTable::lookupSymbolIn(symTable, name));
     if (!opFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
-      rewriter.setInsertionPointToStart(&module->getRegion(0).front());
+      rewriter.setInsertionPointToStart(&symTable->getRegion(0).front());
       auto funcTy = FunctionType::get(
           rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
       opFunc =
@@ -67,10 +68,12 @@ struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
 };
 } // namespace
 
-void mlir::populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
-                                                    PatternBenefit benefit) {
-  patterns.add<ScalarOpToROCDLCall<complex::AbsOp>>(
-      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64", benefit);
+void mlir::populateComplexToROCDLConversionPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp>>(
+      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp>>(
+      patterns.getContext(), "__ocml_cexp_f32", "__ocml_cexp_f64");
 }
 
 namespace {
@@ -81,14 +84,14 @@ struct ConvertComplexToROCDLPass
 } // namespace
 
 void ConvertComplexToROCDLPass::runOnOperation() {
-  auto module = getOperation();
+  Operation *op = getOperation();
 
   RewritePatternSet patterns(&getContext());
-  populateComplexToROCDLConversionPatterns(patterns, /*benefit=*/1);
+  populateComplexToROCDLConversionPatterns(patterns);
 
   ConversionTarget target(getContext());
   target.addLegalDialect<func::FuncDialect>();
-  target.addIllegalOp<complex::AbsOp>();
-  if (failed(applyPartialConversion(module, target, std::move(patterns))))
+  target.addIllegalOp<complex::AbsOp, complex::ExpOp>();
+  if (failed(applyPartialConversion(op, target, std::move(patterns))))
     signalPassFailure();
 }
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
index 618e9c238378c..23631e25e4588 100644
--- a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
+++ b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
@@ -1,13 +1,26 @@
-// RUN: mlir-opt %s -convert-complex-to-rocdl -canonicalize | FileCheck %s
+// RUN: mlir-opt %s -convert-complex-to-rocdl | FileCheck %s
 
 // CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
 // CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64
+// CHECK-DAG: @__ocml_cexp_f32(complex<f32>) -> complex<f32>
+// CHECK-DAG: @__ocml_cexp_f64(complex<f64>) -> complex<f64>
 
+//CHECK-LABEL: @abs_caller
 func.func @abs_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
-  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%[[F:.*]])
+  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%{{.*}})
   %rf = complex.abs %f : complex<f32>
-  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%[[D:.*]])
+  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%{{.*}})
   %rd = complex.abs %d : complex<f64>
   // CHECK: return %[[RF]], %[[RD]]
   return %rf, %rd : f32, f64
 }
+
+//CHECK-LABEL: @exp_caller
+func.func @exp_caller(%f: complex<f32>, %d: complex<f64>) -> (complex<f32>, complex<f64>) {
+  // CHECK: %[[EF:.*]] = call @__ocml_cexp_f32(%{{.*}})
+  %ef = complex.exp %f : complex<f32>
+  // CHECK: %[[ED:.*]] = call @__ocml_cexp_f64(%{{.*}})
+  %ed = complex.exp %d : complex<f64>
+  // CHECK: return %[[EF]], %[[ED]]
+  return %ef, %ed : complex<f32>, complex<f64>
+}

>From d70bca22bae1d66f126b5a1832153810baf06ab4 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Fri, 20 Jun 2025 18:30:57 +0100
Subject: [PATCH 4/6] Correct alphabetical order for cmake.

---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 8b4ac18fba527..de8e1c5c3fa3f 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -34,13 +34,13 @@ add_flang_library(FIRCodeGen
 
   MLIR_LIBS
   MLIRComplexToLLVM
+  MLIRComplexToROCDL
   MLIRComplexToStandard
   MLIRGPUDialect
   MLIRMathToFuncs
   MLIRMathToLLVM
   MLIRMathToLibm
   MLIRMathToROCDL
-  MLIRComplexToROCDL
   MLIROpenMPToLLVM
   MLIROpenACCDialect
   MLIRBuiltinToLLVMIRTranslation

>From 0e9aef2910fee55664f7bbfd54ec7bbb1ee383f3 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Tue, 24 Jun 2025 16:34:11 +0100
Subject: [PATCH 5/6] Add FloatTy as a template parameter.

---
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 42 ++++++++-----------
 1 file changed, 18 insertions(+), 24 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index 98adb9fb1f607..cfad9f5f6fa19 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -21,59 +21,53 @@ using namespace mlir;
 
 namespace {
 
-template <typename Op>
+template <typename Op, typename Ty>
 // Pattern to convert Complex ops to ROCDL function calls.
 struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ComplexOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
-                       StringRef doubleFunc, PatternBenefit benefit = 1)
-      : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
-        doubleFunc(doubleFunc) {}
+  ComplexOpToROCDLCall(MLIRContext *context, StringRef funcName,
+                       PatternBenefit benefit = 1)
+      : OpRewritePattern<Op>(context, benefit), funcName(funcName) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
     Operation *symTable = SymbolTable::getNearestSymbolTable(op);
     Type resType = op.getType();
     if (auto complexType = dyn_cast<ComplexType>(resType))
       resType = complexType.getElementType();
-    FloatType floatTy = dyn_cast<FloatType>(resType);
-    if (!floatTy)
-      return failure();
-
-    StringRef name;
-    if (floatTy.isF64())
-      name = doubleFunc;
-    else if (floatTy.isF32())
-      name = floatFunc;
-    else
+    if (!isa<Ty>(resType))
       return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
-        SymbolTable::lookupSymbolIn(symTable, name));
+        SymbolTable::lookupSymbolIn(symTable, funcName));
     if (!opFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(&symTable->getRegion(0).front());
       auto funcTy = FunctionType::get(
           rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-      opFunc =
-          rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name, funcTy);
+      opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), funcName,
+                                             funcTy);
       opFunc.setPrivate();
     }
-    rewriter.replaceOpWithNewOp<func::CallOp>(op, name, op.getType(),
+    rewriter.replaceOpWithNewOp<func::CallOp>(op, funcName, op.getType(),
                                               op->getOperands());
     return success();
   }
 
 private:
-  std::string floatFunc, doubleFunc;
+  std::string funcName;
 };
 } // namespace
 
 void mlir::populateComplexToROCDLConversionPatterns(
     RewritePatternSet &patterns) {
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp>>(
-      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp>>(
-      patterns.getContext(), "__ocml_cexp_f32", "__ocml_cexp_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float32Type>>(
+      patterns.getContext(), "__ocml_cabs_f32");
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float64Type>>(
+      patterns.getContext(), "__ocml_cabs_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float32Type>>(
+      patterns.getContext(), "__ocml_cexp_f32");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float64Type>>(
+      patterns.getContext(), "__ocml_cexp_f64");
 }
 
 namespace {

>From 03e439e94408ae4f6dafe0cbc8f8dfad6a80dc54 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 2 Jul 2025 14:48:34 +0100
Subject: [PATCH 6/6] Rename to ComplexToROCDLLibraryCalls

---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt    |  2 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  4 +--
 .../ComplexToROCDLLibraryCalls.h}             | 14 ++++----
 mlir/include/mlir/Conversion/Passes.h         |  2 +-
 mlir/include/mlir/Conversion/Passes.td        |  6 ++--
 mlir/lib/Conversion/CMakeLists.txt            |  2 +-
 .../Conversion/ComplexToROCDL/CMakeLists.txt  | 15 --------
 .../ComplexToROCDLLibraryCalls/CMakeLists.txt | 18 ++++++++++
 .../ComplexToROCDLLibraryCalls.cpp}           | 35 ++++++++++---------
 .../complex-to-rocdl-library-calls.mlir}      |  2 +-
 10 files changed, 52 insertions(+), 48 deletions(-)
 rename mlir/include/mlir/Conversion/{ComplexToROCDL/ComplexToROCDL.h => ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h} (52%)
 delete mode 100644 mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
 create mode 100644 mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
 rename mlir/lib/Conversion/{ComplexToROCDL/ComplexToROCDL.cpp => ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp} (68%)
 rename mlir/test/Conversion/{ComplexToROCDL/complex-to-rocdl.mlir => ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir} (92%)

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index de8e1c5c3fa3f..16c7944a885a1 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -34,7 +34,7 @@ add_flang_library(FIRCodeGen
 
   MLIR_LIBS
   MLIRComplexToLLVM
-  MLIRComplexToROCDL
+  MLIRComplexToROCDLLibraryCalls
   MLIRComplexToStandard
   MLIRGPUDialect
   MLIRMathToFuncs
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index b8c7cba80d863..a536b3f13e997 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -33,7 +33,7 @@
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
@@ -4108,7 +4108,7 @@ class FIRToLLVMLowering
     // not needed since all math operations are handled this way.
     if (isAMDGCN) {
       mathConversionPM.addPass(mlir::createConvertMathToROCDL());
-      mathConversionPM.addPass(mlir::createConvertComplexToROCDL());
+      mathConversionPM.addPass(mlir::createConvertComplexToROCDLLibraryCalls());
     }
 
     // Convert math::FPowI operations to inline implementation
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
similarity index 52%
rename from mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
rename to mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
index eb785080adab3..409b87747f437 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
@@ -1,4 +1,4 @@
-//===-- ComplexToROCDL.h - conversion from Complex to ROCDL calls ---------===//
+//===- ComplexToROCDLLibraryCalls.h - convert from Complex to ROCDL calls -===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,21 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
-#define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#ifndef MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
+#define MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
 
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 class RewritePatternSet;
 
-#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDL
+#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDLLIBRARYCALLS
 #include "mlir/Conversion/Passes.h.inc"
 
 /// Populate the given list with patterns that convert from Complex to ROCDL
 /// calls.
-void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns);
+void populateComplexToROCDLLibraryCallsConversionPatterns(
+    RewritePatternSet &patterns);
 } // namespace mlir
 
-#endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#endif // MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 67e8f5b99b67b..896fff277679e 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -23,7 +23,7 @@
 #include "mlir/Conversion/BufferizationToMemRef/BufferizationToMemRef.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
 #include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h"
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRVPass.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 8ad2341f93a15..b85afb0ae97be 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -313,11 +313,11 @@ def ConvertComplexToLibm : Pass<"convert-complex-to-libm", "ModuleOp"> {
 }
 
 //===----------------------------------------------------------------------===//
-// ComplexToROCDL
+// ComplexToROCDLLibraryCalls
 //===----------------------------------------------------------------------===//
 
-def ConvertComplexToROCDL : Pass<"convert-complex-to-rocdl", "ModuleOp"> {
-  let summary = "Convert Complex dialect to ROCDL calls";
+def ConvertComplexToROCDLLibraryCalls : Pass<"convert-complex-to-rocdl-library-calls", "ModuleOp"> {
+  let summary = "Convert Complex dialect to ROCDL library calls";
   let description = [{
     This pass converts supported Complex ops to calls to the AMD device library.
   }];
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index 4ad81553a4fa8..b731662dcebc7 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -13,7 +13,7 @@ add_subdirectory(AsyncToLLVM)
 add_subdirectory(BufferizationToMemRef)
 add_subdirectory(ComplexCommon)
 add_subdirectory(ComplexToLibm)
-add_subdirectory(ComplexToROCDL)
+add_subdirectory(ComplexToROCDLLibraryCalls)
 add_subdirectory(ComplexToLLVM)
 add_subdirectory(ComplexToSPIRV)
 add_subdirectory(ComplexToStandard)
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
deleted file mode 100644
index 133809ac32f0f..0000000000000
--- a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-add_mlir_conversion_library(MLIRComplexToROCDL
-  ComplexToROCDL.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDL
-
-  DEPENDS
-  MLIRConversionPassIncGen
-
-  LINK_LIBS PUBLIC
-  MLIRComplexDialect
-  MLIRFuncDialect
-  MLIRPass
-  MLIRTransformUtils
-  )
diff --git a/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
new file mode 100644
index 0000000000000..695bb2dd0a82c
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_conversion_library(MLIRComplexToROCDLLibraryCalls
+  ComplexToROCDLLibraryCalls.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDLLibraryCalls
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRComplexDialect
+  MLIRFuncDialect
+  MLIRPass
+  MLIRTransformUtils
+  )
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
similarity index 68%
rename from mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
rename to mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
index cfad9f5f6fa19..99d5424aef79a 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
@@ -1,4 +1,4 @@
-//===-- ComplexToROCDL.cpp - conversion from Complex to ROCDL calls -------===//
+//=== ComplexToROCDLLibraryCalls.cpp - convert from Complex to ROCDL calls ===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
-#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
+#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDLLIBRARYCALLS
 #include "mlir/Conversion/Passes.h.inc"
 } // namespace mlir
 
@@ -21,12 +21,12 @@ using namespace mlir;
 
 namespace {
 
-template <typename Op, typename Ty>
+template <typename Op, typename FloatTy>
 // Pattern to convert Complex ops to ROCDL function calls.
-struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
+struct ComplexOpToROCDLLibraryCalls : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ComplexOpToROCDLCall(MLIRContext *context, StringRef funcName,
-                       PatternBenefit benefit = 1)
+  ComplexOpToROCDLLibraryCalls(MLIRContext *context, StringRef funcName,
+                               PatternBenefit benefit = 1)
       : OpRewritePattern<Op>(context, benefit), funcName(funcName) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
@@ -34,7 +34,7 @@ struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
     Type resType = op.getType();
     if (auto complexType = dyn_cast<ComplexType>(resType))
       resType = complexType.getElementType();
-    if (!isa<Ty>(resType))
+    if (!isa<FloatTy>(resType))
       return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
@@ -58,30 +58,31 @@ struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
 };
 } // namespace
 
-void mlir::populateComplexToROCDLConversionPatterns(
+void mlir::populateComplexToROCDLLibraryCallsConversionPatterns(
     RewritePatternSet &patterns) {
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float32Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float32Type>>(
       patterns.getContext(), "__ocml_cabs_f32");
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float64Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float64Type>>(
       patterns.getContext(), "__ocml_cabs_f64");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float32Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ExpOp, Float32Type>>(
       patterns.getContext(), "__ocml_cexp_f32");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float64Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ExpOp, Float64Type>>(
       patterns.getContext(), "__ocml_cexp_f64");
 }
 
 namespace {
-struct ConvertComplexToROCDLPass
-    : public impl::ConvertComplexToROCDLBase<ConvertComplexToROCDLPass> {
+struct ConvertComplexToROCDLLibraryCallsPass
+    : public impl::ConvertComplexToROCDLLibraryCallsBase<
+          ConvertComplexToROCDLLibraryCallsPass> {
   void runOnOperation() override;
 };
 } // namespace
 
-void ConvertComplexToROCDLPass::runOnOperation() {
+void ConvertComplexToROCDLLibraryCallsPass::runOnOperation() {
   Operation *op = getOperation();
 
   RewritePatternSet patterns(&getContext());
-  populateComplexToROCDLConversionPatterns(patterns);
+  populateComplexToROCDLLibraryCallsConversionPatterns(patterns);
 
   ConversionTarget target(getContext());
   target.addLegalDialect<func::FuncDialect>();
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
similarity index 92%
rename from mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
rename to mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
index 23631e25e4588..bae7c5986ef9e 100644
--- a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
+++ b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-complex-to-rocdl | FileCheck %s
+// RUN: mlir-opt %s -convert-complex-to-rocdl-library-calls | FileCheck %s
 
 // CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
 // CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64


From flang-commits at lists.llvm.org  Wed Jul  2 08:29:58 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 02 Jul 2025 08:29:58 -0700 (PDT)
Subject: [flang-commits] [flang] ab42c4a - [NFC] Replace
 "omp.default.mapper" with llvm::omp::OmpDefaultMapperName.
Message-ID: <68655076.170a0220.2acc1f.5ad5@mx.google.com>


Author: Akash Banerjee
Date: 2025-07-02T16:29:52+01:00
New Revision: ab42c4abe86c454a6812d837a7f2a8d5b9671fdf

URL: https://github.com/llvm/llvm-project/commit/ab42c4abe86c454a6812d837a7f2a8d5b9671fdf
DIFF: https://github.com/llvm/llvm-project/commit/ab42c4abe86c454a6812d837a7f2a8d5b9671fdf.diff

LOG: [NFC] Replace "omp.default.mapper" with llvm::omp::OmpDefaultMapperName.

Added: 
    

Modified: 
    flang/lib/Parser/openmp-parsers.cpp
    flang/lib/Parser/unparse.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..30b5d4f9513a7 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1553,7 +1553,7 @@ static OmpMapperSpecifier ConstructOmpMapperSpecifier(
         std::get<Name>(derived->t).ToString() + llvm::omp::OmpDefaultMapperName,
         std::move(typeSpec), std::move(varName)};
   }
-  return OmpMapperSpecifier{std::string("omp.default.mapper"),
+  return OmpMapperSpecifier{std::string(llvm::omp::OmpDefaultMapperName),
       std::move(typeSpec), std::move(varName)};
 }
 

diff  --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 903287515e559..dcd1ac165adc1 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2094,7 +2094,7 @@ class UnparseVisitor {
   }
   void Unparse(const OmpMapperSpecifier &x) {
     const auto &mapperName{std::get<std::string>(x.t)};
-    if (mapperName.find("omp.default.mapper") == std::string::npos) {
+    if (mapperName.find(llvm::omp::OmpDefaultMapperName) == std::string::npos) {
       Walk(mapperName);
       Put(":");
     }
@@ -2740,7 +2740,7 @@ class UnparseVisitor {
     Word("!$OMP DECLARE MAPPER (");
     const auto &spec{std::get<OmpMapperSpecifier>(z.t)};
     const auto &mapperName{std::get<std::string>(spec.t)};
-    if (mapperName.find("omp.default.mapper") == std::string::npos) {
+    if (mapperName.find(llvm::omp::OmpDefaultMapperName) == std::string::npos) {
       Walk(mapperName);
       Put(":");
     }


        

From flang-commits at lists.llvm.org  Wed Jul  2 08:35:22 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 08:35:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][do concurrent] Re-model `reduce` to
 match reductions are modelled in OpenMP and OpenACC (PR #145837)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145837@github.com>
Message-ID: <686551ba.a70a0220.31d786.c137@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/145837

>From bf252b5e85e8c09ddd70c5c4edd49d5215fb6ecc Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Thu, 12 Jun 2025 06:14:21 -0500
Subject: [PATCH 1/3] [flang][do concurrent] Re-model `reduce` to match
 reductions are modelled in OpenMP and OpenACC

This PR proposes re-modelling `reduce` specifiers to match OpenMP and
OpenACC. In particular, this PR includes the following:

* A new `fir` op: `fir.delcare_reduction` which is identical to OpenMP's
  `omp.declare_reduction` op.
* Updating the `reduce` clause on `fir.do_concurrent.loop` to use the
  new op.
* Re-uses the `ReductionProcessor` component to emit reductions for `do
  concurrent` just like we do for OpenMP. To do this, the
  `ReductionProcessor` had to be refactored to be more generalized.
* Upates mapping `do concurrent` to `fir.loop ... unordered` nests using
  the new reduction model.

Unfortunately, this is a big PR that would be difficult to divide up in
smaller parts because the bottom of the changes are the `fir` table-gen
changes to `do concurrent`. However, doing these MLIR changes cascades
to the other parts that have to be modified to not break things.
---
 .../flang/Optimizer/Dialect/FIRAttr.td        |   2 +-
 .../include/flang/Optimizer/Dialect/FIROps.td | 136 +++++++-
 flang/lib/Lower/Bridge.cpp                    |  85 +++--
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  79 ++++-
 flang/lib/Lower/OpenMP/ClauseProcessor.h      |   1 -
 flang/lib/Lower/OpenMP/Clauses.h              |   1 +
 flang/lib/Lower/OpenMP/OpenMP.cpp             |   1 -
 flang/lib/Lower/OpenMP/ReductionProcessor.cpp | 281 +++++++++--------
 flang/lib/Lower/OpenMP/ReductionProcessor.h   |  30 +-
 flang/lib/Lower/Support/Utils.cpp             |   4 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  48 +--
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 293 ++++++++++++------
 .../Transforms/SimplifyFIROperations.cpp      |  10 +-
 flang/test/Fir/do_concurrent.fir              |  63 +++-
 flang/test/Fir/invalid.fir                    |  18 +-
 flang/test/Lower/do_concurrent_reduce.f90     |  41 +++
 flang/test/Lower/loops.f90                    |   2 +-
 flang/test/Lower/loops3.f90                   |   2 +-
 .../do_concurrent-to-do_loop-unodered.fir     |   2 +-
 19 files changed, 780 insertions(+), 319 deletions(-)
 create mode 100644 flang/test/Lower/do_concurrent_reduce.f90

diff --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.td b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
index 2845080030b92..7bd96ac3ea631 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRAttr.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
@@ -112,7 +112,7 @@ def fir_ReduceOperationEnum : I32BitEnumAttr<"ReduceOperationEnum",
       I32BitEnumAttrCaseBit<"MIN", 7, "min">,
       I32BitEnumAttrCaseBit<"IAND", 8, "iand">,
       I32BitEnumAttrCaseBit<"IOR", 9, "ior">,
-      I32BitEnumAttrCaseBit<"EIOR", 10, "eior">
+      I32BitEnumAttrCaseBit<"IEOR", 10, "ieor">
     ]> {
   let separator = ", ";
   let cppNamespace = "::fir";
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index b96c0360316ad..b217f1433767f 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3518,7 +3518,7 @@ def fir_BoxTotalElementsOp
 
 def YieldOp : fir_Op<"yield",
     [Pure, ReturnLike, Terminator,
-     ParentOneOf<["LocalitySpecifierOp"]>]> {
+     ParentOneOf<["LocalitySpecifierOp", "DeclareReductionOp"]>]> {
   let summary = "loop yield and termination operation";
   let description = [{
     "fir.yield" yields SSA values from a fir dialect op region and
@@ -3662,6 +3662,103 @@ def fir_LocalitySpecifierOp : fir_Op<"local", [IsolatedFromAbove]> {
   let hasRegionVerifier = 1;
 }
 
+def fir_DeclareReductionOp : fir_Op<"declare_reduction", [IsolatedFromAbove,
+                                                         Symbol]> {
+  let summary = "declares a reduction kind";
+  let description = [{
+    Note: this operation is adapted from omp::DeclareReductionOp. There is a lot
+    duplication at the moment. TODO Combined both ops into one. See:
+    https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
+
+    Declares an `do concurrent` reduction. This requires two mandatory and three
+    optional regions.
+
+      1. The optional alloc region specifies how to allocate the thread-local
+         reduction value. This region should not contain control flow and all
+         IR should be suitable for inlining straight into an entry block. In
+         the common case this is expected to contain only allocas. It is
+         expected to `fir.yield` the allocated value on all control paths.
+         If allocation is conditional (e.g. only allocate if the mold is
+         allocated), this should be done in the initilizer region and this
+         region not included. The alloc region is not used for by-value
+         reductions (where allocation is implicit).
+      2. The initializer region specifies how to initialize the thread-local
+         reduction value. This is usually the neutral element of the reduction.
+         For convenience, the region has an argument that contains the value
+         of the reduction accumulator at the start of the reduction. If an alloc
+         region is specified, there is a second block argument containing the
+         address of the allocated memory. The initializer region is expected to
+         `fir.yield` the new value on all control flow paths.
+      3. The reduction region specifies how to combine two values into one, i.e.
+         the reduction operator. It accepts the two values as arguments and is
+         expected to `fir.yield` the combined value on all control flow paths.
+      4. The atomic reduction region is optional and specifies how two values
+         can be combined atomically given local accumulator variables. It is
+         expected to store the combined value in the first accumulator variable.
+      5. The cleanup region is optional and specifies how to clean up any memory
+         allocated by the initializer region. The region has an argument that
+         contains the value of the thread-local reduction accumulator. This will
+         be executed after the reduction has completed.
+
+    Note that the MLIR type system does not allow for type-polymorphic
+    reductions. Separate reduction declarations should be created for different
+    element and accumulator types.
+
+    For initializer and reduction regions, the operand to `fir.yield` must
+    match the parent operation's results.
+  }];
+
+  let arguments = (ins SymbolNameAttr:$sym_name,
+                       TypeAttr:$type);
+
+  let regions = (region MaxSizedRegion<1>:$allocRegion,
+                        AnyRegion:$initializerRegion,
+                        AnyRegion:$reductionRegion,
+                        AnyRegion:$atomicReductionRegion,
+                        AnyRegion:$cleanupRegion);
+
+  let assemblyFormat = "$sym_name `:` $type attr-dict-with-keyword "
+                       "( `alloc` $allocRegion^ )? "
+                       "`init` $initializerRegion "
+                       "`combiner` $reductionRegion "
+                       "( `atomic` $atomicReductionRegion^ )? "
+                       "( `cleanup` $cleanupRegion^ )? ";
+
+  let extraClassDeclaration = [{
+    mlir::BlockArgument getAllocMoldArg() {
+      auto &region = getAllocRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    mlir::BlockArgument getInitializerMoldArg() {
+      return getInitializerRegion().getArgument(0);
+    }
+    mlir::BlockArgument getInitializerAllocArg() {
+      return getAllocRegion().empty() ?
+          nullptr : getInitializerRegion().getArgument(1);
+    }
+    mlir::BlockArgument getReductionLhsArg() {
+      return getReductionRegion().getArgument(0);
+    }
+    mlir::BlockArgument getReductionRhsArg() {
+      return getReductionRegion().getArgument(1);
+    }
+    mlir::BlockArgument getAtomicReductionLhsArg() {
+      auto &region = getAtomicReductionRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    mlir::BlockArgument getAtomicReductionRhsArg() {
+      auto &region = getAtomicReductionRegion();
+      return region.empty() ? nullptr : region.getArgument(1);
+    }
+    mlir::BlockArgument getCleanupAllocArg() {
+      auto &region = getCleanupRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+  }];
+
+  let hasRegionVerifier = 1;
+}
+
 def fir_DoConcurrentOp : fir_Op<"do_concurrent",
     [SingleBlock, AutomaticAllocationScope]> {
   let summary = "do concurrent loop wrapper";
@@ -3700,6 +3797,25 @@ def fir_LocalSpecifier {
   );
 }
 
+def fir_ReduceSpecifier {
+  dag arguments = (ins
+    Variadic<AnyType>:$reduce_vars,
+    OptionalAttr<DenseBoolArrayAttr>:$reduce_byref,
+
+    // This introduces redundency in how reductions are modelled. In particular,
+    // a single reduction is represented by 2 attributes:
+    //
+    // 1. `$reduce_syms` which is a list of `DeclareReductionOp`s.
+    // 2. `$reduce_attrs` which is an array of `fir::ReduceAttr` values.
+    //
+    // The first makes it easier to map `do concurrent` to parallization models
+    // (e.g. OpenMP and OpenACC) while the second makes it easier to map it to
+    // nests of `fir.do_loop ... unodered` ops.
+    OptionalAttr<SymbolRefArrayAttr>:$reduce_syms,
+    OptionalAttr<ArrayAttr>:$reduce_attrs
+  );
+}
+
 def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     [AttrSizedOperandSegments, DeclareOpInterfaceMethods<LoopLikeOpInterface,
                                                          ["getLoopInductionVars"]>,
@@ -3709,7 +3825,7 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
   let description = [{
     An operation that models a Fortran `do concurrent` loop's header and block.
     This is a single-region single-block terminator op that is expected to
-    terminate the region of a `omp.do_concurrent` wrapper op.
+    terminate the region of a `fir.do_concurrent` wrapper op.
 
     This op borrows from both `scf.parallel` and `fir.do_loop` ops. Similar to
     `scf.parallel`, a loop nest takes 3 groups of SSA values as operands that
@@ -3747,8 +3863,6 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     - `lowerBound`: The group of SSA values for the nest's lower bounds.
     - `upperBound`: The group of SSA values for the nest's upper bounds.
     - `step`: The group of SSA values for the nest's steps.
-    - `reduceOperands`: The reduction SSA values, if any.
-    - `reduceAttrs`: Attributes to store reduction operations, if any.
     - `loopAnnotation`: Loop metadata to be passed down the compiler pipeline to
       LLVM.
   }];
@@ -3757,12 +3871,12 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     Variadic<Index>:$lowerBound,
     Variadic<Index>:$upperBound,
     Variadic<Index>:$step,
-    Variadic<AnyType>:$reduceOperands,
-    OptionalAttr<ArrayAttr>:$reduceAttrs,
     OptionalAttr<LoopAnnotationAttr>:$loopAnnotation
   );
 
-  let arguments = !con(opArgs, fir_LocalSpecifier.arguments);
+  let arguments = !con(opArgs,
+    fir_LocalSpecifier.arguments,
+    fir_ReduceSpecifier.arguments);
 
   let regions = (region SizedRegion<1>:$region);
 
@@ -3783,12 +3897,18 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
                                              getNumLocalOperands());
     }
 
+    mlir::Block::BlockArgListType getRegionReduceArgs() {
+      return getBody()->getArguments().slice(getNumInductionVars()
+                                               + getNumLocalOperands(),
+                                             getNumReduceOperands());
+    }
+
     /// Number of operands controlling the loop
     unsigned getNumControlOperands() { return getLowerBound().size() * 3; }
 
     // Get Number of reduction operands
     unsigned getNumReduceOperands() {
-      return getReduceOperands().size();
+      return getReduceVars().size();
     }
 
     mlir::Operation::operand_range getLocalOperands() {
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..e2dc8131331d3 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Lower/Bridge.h"
 
+#include "OpenMP/ReductionProcessor.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -127,9 +128,8 @@ struct IncrementLoopInfo {
   bool isConcurrent;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localSymList;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localInitSymList;
-  llvm::SmallVector<
-      std::pair<fir::ReduceOperationEnum, const Fortran::semantics::Symbol *>>
-      reduceSymList;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> reduceSymList;
+  llvm::SmallVector<fir::ReduceOperationEnum> reduceOperatorList;
   llvm::SmallVector<const Fortran::semantics::Symbol *> sharedSymList;
   mlir::Value loopVariable = nullptr;
 
@@ -1993,7 +1993,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     case Fortran::parser::ReductionOperator::Operator::Ior:
       return fir::ReduceOperationEnum::IOR;
     case Fortran::parser::ReductionOperator::Operator::Ieor:
-      return fir::ReduceOperationEnum::EIOR;
+      return fir::ReduceOperationEnum::IEOR;
     }
     llvm_unreachable("illegal reduction operator");
   }
@@ -2027,8 +2027,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
               std::get<Fortran::parser::ReductionOperator>(reduceList->t));
           for (const Fortran::parser::Name &x :
                std::get<std::list<Fortran::parser::Name>>(reduceList->t)) {
-            info.reduceSymList.push_back(
-                std::make_pair(reduce_operation, x.symbol));
+            info.reduceSymList.push_back(x.symbol);
+            info.reduceOperatorList.push_back(reduce_operation);
           }
         }
       }
@@ -2089,6 +2089,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         assign.u = Fortran::evaluate::Assignment::BoundsSpec{};
       genAssignment(assign);
     }
+
     for (const Fortran::semantics::Symbol *sym : info.sharedSymList) {
       const auto *hostDetails =
           sym->detailsIf<Fortran::semantics::HostAssocDetails>();
@@ -2112,6 +2113,46 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       }
     }
 
+    llvm::SmallVector<bool> reduceVarByRef;
+    llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
+    llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
+
+    for (const auto &reduceOp : info.reduceOperatorList) {
+      nestReduceAttrs.push_back(
+          fir::ReduceAttr::get(builder->getContext(), reduceOp));
+    }
+
+    llvm::SmallVector<mlir::Value> reduceVars;
+    Fortran::lower::omp::ReductionProcessor rp;
+    rp.processReductionArguments<fir::DeclareReductionOp>(
+        toLocation(), *this, info.reduceOperatorList, reduceVars,
+        reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
+
+    doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
+    doConcurrentLoopOp.setReduceSymsAttr(
+        reductionDeclSymbols.empty()
+            ? nullptr
+            : mlir::ArrayAttr::get(builder->getContext(),
+                                   reductionDeclSymbols));
+    doConcurrentLoopOp.setReduceAttrsAttr(
+        nestReduceAttrs.empty()
+            ? nullptr
+            : mlir::ArrayAttr::get(builder->getContext(), nestReduceAttrs));
+    doConcurrentLoopOp.setReduceByrefAttr(
+        reduceVarByRef.empty() ? nullptr
+                               : mlir::DenseBoolArrayAttr::get(
+                                     builder->getContext(), reduceVarByRef));
+
+    for (auto [sym, reduceVar] :
+         llvm::zip_equal(info.reduceSymList, reduceVars)) {
+      auto arg = doConcurrentLoopOp.getRegion().begin()->addArgument(
+          reduceVar.getType(), doConcurrentLoopOp.getLoc());
+      bindSymbol(*sym, hlfir::translateToExtendedValue(
+                           reduceVar.getLoc(), *builder, hlfir::Entity{arg},
+                           /*contiguousHint=*/true)
+                           .first);
+    }
+
     // Note that allocatable, types with ultimate components, and type
     // requiring finalization are forbidden in LOCAL/LOCAL_INIT (F2023 C1130),
     // so no clean-up needs to be generated for these entities.
@@ -2203,6 +2244,12 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       }
     }
 
+    // Introduce a `do concurrent` scope to bind symbols corresponding to local,
+    // local_init, and reduce region arguments.
+    if (!incrementLoopNestInfo.empty() &&
+        incrementLoopNestInfo.back().isConcurrent)
+      localSymbols.pushScope();
+
     // Increment loop begin code. (Infinite/while code was already generated.)
     if (!infiniteLoop && !whileCondition)
       genFIRIncrementLoopBegin(incrementLoopNestInfo, doStmtEval.dirs);
@@ -2226,6 +2273,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     // This call may generate a branch in some contexts.
     genFIR(endDoEval, unstructuredContext);
+
+    if (!incrementLoopNestInfo.empty() &&
+        incrementLoopNestInfo.back().isConcurrent)
+      localSymbols.popScope();
   }
 
   /// Generate FIR to evaluate loop control values (lower, upper and step).
@@ -2408,19 +2459,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         info.stepVariable = builder->createTemporary(loc, stepValue.getType());
         builder->create<fir::StoreOp>(loc, stepValue, info.stepVariable);
       }
-
-      if (genDoConcurrent && nestReduceOperands.empty()) {
-        // Create DO CONCURRENT reduce operands and attributes
-        for (const auto &reduceSym : info.reduceSymList) {
-          const fir::ReduceOperationEnum reduceOperation = reduceSym.first;
-          const Fortran::semantics::Symbol *sym = reduceSym.second;
-          fir::ExtendedValue exv = getSymbolExtendedValue(*sym, nullptr);
-          nestReduceOperands.push_back(fir::getBase(exv));
-          auto reduceAttr =
-              fir::ReduceAttr::get(builder->getContext(), reduceOperation);
-          nestReduceAttrs.push_back(reduceAttr);
-        }
-      }
     }
 
     for (auto [info, lowerValue, upperValue, stepValue] :
@@ -2518,11 +2556,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
-          loc, nestLBs, nestUBs, nestSts, nestReduceOperands,
-          nestReduceAttrs.empty()
-              ? nullptr
-              : mlir::ArrayAttr::get(builder->getContext(), nestReduceAttrs),
-          nullptr, /*local_vars=*/std::nullopt, /*local_syms=*/nullptr);
+          loc, nestLBs, nestUBs, nestSts, nullptr, /*local_vars=*/std::nullopt,
+          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
+          /*reduce_attrs=*/nullptr);
 
       llvm::SmallVector<mlir::Type> loopBlockArgTypes(
           incrementLoopNestInfo.size(), builder->getIndexType());
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 7bea427099a28..5aebfc901e8ac 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -12,6 +12,7 @@
 
 #include "ClauseProcessor.h"
 #include "Clauses.h"
+#include "ReductionProcessor.h"
 #include "Utils.h"
 
 #include "flang/Lower/ConvertExprToHLFIR.h"
@@ -25,6 +26,21 @@ namespace Fortran {
 namespace lower {
 namespace omp {
 
+using ReductionModifier =
+    Fortran::lower::omp::clause::Reduction::ReductionModifier;
+
+mlir::omp::ReductionModifier translateReductionModifier(ReductionModifier mod) {
+  switch (mod) {
+  case ReductionModifier::Default:
+    return mlir::omp::ReductionModifier::defaultmod;
+  case ReductionModifier::Inscan:
+    return mlir::omp::ReductionModifier::inscan;
+  case ReductionModifier::Task:
+    return mlir::omp::ReductionModifier::task;
+  }
+  return mlir::omp::ReductionModifier::defaultmod;
+}
+
 /// Check for unsupported map operand types.
 static void checkMapType(mlir::Location location, mlir::Type type) {
   if (auto refType = mlir::dyn_cast<fir::ReferenceType>(type))
@@ -1076,6 +1092,18 @@ bool ClauseProcessor::processIf(
   });
   return found;
 }
+
+template <typename T>
+void collectReductionSyms(
+    const T &reduction,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
+  const auto &objectList{std::get<omp::ObjectList>(reduction.t)};
+  for (const Object &object : objectList) {
+    const semantics::Symbol *symbol = object.sym();
+    reductionSyms.push_back(symbol);
+  }
+}
+
 bool ClauseProcessor::processInReduction(
     mlir::Location currentLocation, mlir::omp::InReductionClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &outReductionSyms) const {
@@ -1085,10 +1113,14 @@ bool ClauseProcessor::processInReduction(
         llvm::SmallVector<bool> inReduceVarByRef;
         llvm::SmallVector<mlir::Attribute> inReductionDeclSymbols;
         llvm::SmallVector<const semantics::Symbol *> inReductionSyms;
+        collectReductionSyms(clause, inReductionSyms);
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::InReduction>(
-            currentLocation, converter, clause, inReductionVars,
-            inReduceVarByRef, inReductionDeclSymbols, inReductionSyms);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
+            inReductionSyms);
 
         // Copy local lists into the output.
         llvm::copy(inReductionVars, std::back_inserter(result.inReductionVars));
@@ -1416,10 +1448,23 @@ bool ClauseProcessor::processReduction(
         llvm::SmallVector<bool> reduceVarByRef;
         llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
         llvm::SmallVector<const semantics::Symbol *> reductionSyms;
+        collectReductionSyms(clause, reductionSyms);
+
+        auto mod = std::get<std::optional<ReductionModifier>>(clause.t);
+        if (mod.has_value()) {
+          if (mod.value() == ReductionModifier::Task)
+            TODO(currentLocation, "Reduction modifier `task` is not supported");
+          else
+            result.reductionMod = mlir::omp::ReductionModifierAttr::get(
+                converter.getFirOpBuilder().getContext(),
+                translateReductionModifier(mod.value()));
+        }
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::Reduction>(
-            currentLocation, converter, clause, reductionVars, reduceVarByRef,
-            reductionDeclSymbols, reductionSyms, &result.reductionMod);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            reductionVars, reduceVarByRef, reductionDeclSymbols, reductionSyms);
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
@@ -1435,21 +1480,25 @@ bool ClauseProcessor::processTaskReduction(
   return findRepeatableClause<omp::clause::TaskReduction>(
       [&](const omp::clause::TaskReduction &clause, const parser::CharBlock &) {
         llvm::SmallVector<mlir::Value> taskReductionVars;
-        llvm::SmallVector<bool> TaskReduceVarByRef;
-        llvm::SmallVector<mlir::Attribute> TaskReductionDeclSymbols;
-        llvm::SmallVector<const semantics::Symbol *> TaskReductionSyms;
+        llvm::SmallVector<bool> taskReduceVarByRef;
+        llvm::SmallVector<mlir::Attribute> taskReductionDeclSymbols;
+        llvm::SmallVector<const semantics::Symbol *> taskReductionSyms;
+        collectReductionSyms(clause, taskReductionSyms);
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::TaskReduction>(
-            currentLocation, converter, clause, taskReductionVars,
-            TaskReduceVarByRef, TaskReductionDeclSymbols, TaskReductionSyms);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
+            taskReductionSyms);
         // Copy local lists into the output.
         llvm::copy(taskReductionVars,
                    std::back_inserter(result.taskReductionVars));
-        llvm::copy(TaskReduceVarByRef,
+        llvm::copy(taskReduceVarByRef,
                    std::back_inserter(result.taskReductionByref));
-        llvm::copy(TaskReductionDeclSymbols,
+        llvm::copy(taskReductionDeclSymbols,
                    std::back_inserter(result.taskReductionSyms));
-        llvm::copy(TaskReductionSyms, std::back_inserter(outReductionSyms));
+        llvm::copy(taskReductionSyms, std::back_inserter(outReductionSyms));
       });
 }
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 3d8c4a337a4a4..46b749fb66c86 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -14,7 +14,6 @@
 
 #include "ClauseFinder.h"
 #include "Clauses.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/Bridge.h"
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index d7ab21d428e32..7f317f05f67b7 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -179,6 +179,7 @@ using IteratorSpecifier = tomp::type::IteratorSpecifierT<TypeTy, IdTy, ExprTy>;
 using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
 using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
 using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
+using ReductionOperatorList = List<ReductionOperator>;
 using DependenceType = tomp::type::DependenceType;
 using Prescriptiveness = tomp::type::Prescriptiveness;
 
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0a56e888ac44b..65e852cbcc911 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -17,7 +17,6 @@
 #include "Clauses.h"
 #include "DataSharingProcessor.h"
 #include "Decomposer.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Lower/Bridge.h"
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index 330cef7b54c74..d14fc1f7a52da 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -12,6 +12,7 @@
 
 #include "ReductionProcessor.h"
 
+#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/ConvertType.h"
 #include "flang/Lower/Support/PrivateReductionUtils.h"
@@ -21,8 +22,6 @@
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
-#include "flang/Optimizer/Support/FatalError.h"
-#include "flang/Parser/tools.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "llvm/Support/CommandLine.h"
 #include <type_traits>
@@ -40,35 +39,35 @@ namespace lower {
 namespace omp {
 
 // explicit template declarations
-template void
-ReductionProcessor::processReductionArguments<omp::clause::Reduction>(
+template void ReductionProcessor::processReductionArguments<
+    mlir::omp::DeclareReductionOp, omp::clause::ReductionOperatorList>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::Reduction &reduction,
+    const omp::clause::ReductionOperatorList &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void
-ReductionProcessor::processReductionArguments<omp::clause::TaskReduction>(
+template void ReductionProcessor::processReductionArguments<
+    fir::DeclareReductionOp, llvm::SmallVector<fir::ReduceOperationEnum>>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::TaskReduction &reduction,
+    const llvm::SmallVector<fir::ReduceOperationEnum> &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void
-ReductionProcessor::processReductionArguments<omp::clause::InReduction>(
-    mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::InReduction &reduction,
-    llvm::SmallVectorImpl<mlir::Value> &reductionVars,
-    llvm::SmallVectorImpl<bool> &reduceVarByRef,
-    llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+template mlir::omp::DeclareReductionOp
+ReductionProcessor::createDeclareReduction<mlir::omp::DeclareReductionOp>(
+    AbstractConverter &converter, llvm::StringRef reductionOpName,
+    const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
+    bool isByRef);
+
+template fir::DeclareReductionOp
+ReductionProcessor::createDeclareReduction<fir::DeclareReductionOp>(
+    AbstractConverter &converter, llvm::StringRef reductionOpName,
+    const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
+    bool isByRef);
 
 ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(
     const omp::clause::ProcedureDesignator &pd) {
@@ -106,6 +105,37 @@ ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(
   }
 }
 
+ReductionProcessor::ReductionIdentifier
+ReductionProcessor::getReductionType(const fir::ReduceOperationEnum &redOp) {
+  switch (redOp) {
+  case fir::ReduceOperationEnum::Add:
+    return ReductionIdentifier::ADD;
+  case fir::ReduceOperationEnum::Multiply:
+    return ReductionIdentifier::MULTIPLY;
+
+  case fir::ReduceOperationEnum::AND:
+    return ReductionIdentifier::AND;
+  case fir::ReduceOperationEnum::OR:
+    return ReductionIdentifier::OR;
+
+  case fir::ReduceOperationEnum::EQV:
+    return ReductionIdentifier::EQV;
+  case fir::ReduceOperationEnum::NEQV:
+    return ReductionIdentifier::NEQV;
+
+  case fir::ReduceOperationEnum::IAND:
+    return ReductionIdentifier::IAND;
+  case fir::ReduceOperationEnum::IEOR:
+    return ReductionIdentifier::IEOR;
+  case fir::ReduceOperationEnum::IOR:
+    return ReductionIdentifier::IOR;
+  case fir::ReduceOperationEnum::MAX:
+    return ReductionIdentifier::MAX;
+  case fir::ReduceOperationEnum::MIN:
+    return ReductionIdentifier::MIN;
+  }
+}
+
 bool ReductionProcessor::supportedIntrinsicProcReduction(
     const omp::clause::ProcedureDesignator &pd) {
   semantics::Symbol *sym = pd.v.sym();
@@ -136,28 +166,29 @@ ReductionProcessor::getReductionName(llvm::StringRef name,
   return fir::getTypeAsString(ty, kindMap, (name + byrefAddition).str());
 }
 
-std::string ReductionProcessor::getReductionName(
-    omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp,
-    const fir::KindMapping &kindMap, mlir::Type ty, bool isByRef) {
+std::string
+ReductionProcessor::getReductionName(ReductionIdentifier redId,
+                                     const fir::KindMapping &kindMap,
+                                     mlir::Type ty, bool isByRef) {
   std::string reductionName;
 
-  switch (intrinsicOp) {
-  case omp::clause::DefinedOperator::IntrinsicOperator::Add:
+  switch (redId) {
+  case ReductionIdentifier::ADD:
     reductionName = "add_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::Multiply:
+  case ReductionIdentifier::MULTIPLY:
     reductionName = "multiply_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::AND:
+  case ReductionIdentifier::AND:
     reductionName = "and_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::EQV:
+  case ReductionIdentifier::EQV:
     reductionName = "eqv_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::OR:
+  case ReductionIdentifier::OR:
     reductionName = "or_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:
+  case ReductionIdentifier::NEQV:
     reductionName = "neqv_reduction";
     break;
   default:
@@ -334,8 +365,18 @@ mlir::Value ReductionProcessor::createScalarCombiner(
   return reductionOp;
 }
 
+template <typename ParentDeclOpType>
+static void genYield(fir::FirOpBuilder &builder, mlir::Location loc,
+                     mlir::Value yieldedValue) {
+  if constexpr (std::is_same_v<ParentDeclOpType, mlir::omp::DeclareReductionOp>)
+    builder.create<mlir::omp::YieldOp>(loc, yieldedValue);
+  else
+    builder.create<fir::YieldOp>(loc, yieldedValue);
+}
+
 /// Create reduction combiner region for reduction variables which are boxed
 /// arrays
+template <typename DeclRedOpType>
 static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
                            ReductionProcessor::ReductionIdentifier redId,
                            fir::BaseBoxType boxTy, mlir::Value lhs,
@@ -369,7 +410,7 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value result = ReductionProcessor::createScalarCombiner(
         builder, loc, redId, eleTy, lhs, rhs);
     builder.create<fir::StoreOp>(loc, result, lhsValAddr);
-    builder.create<mlir::omp::YieldOp>(loc, lhsAddr);
+    genYield<DeclRedOpType>(builder, loc, lhsAddr);
     return;
   }
 
@@ -408,10 +449,11 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
   builder.create<fir::StoreOp>(loc, scalarReduction, lhsEleAddr);
 
   builder.setInsertionPointAfter(nest.outerOp);
-  builder.create<mlir::omp::YieldOp>(loc, lhsAddr);
+  genYield<DeclRedOpType>(builder, loc, lhsAddr);
 }
 
 // generate combiner region for reduction operations
+template <typename DeclRedOpType>
 static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
                         ReductionProcessor::ReductionIdentifier redId,
                         mlir::Type ty, mlir::Value lhs, mlir::Value rhs,
@@ -426,15 +468,15 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
         builder, loc, redId, ty, lhsLoaded, rhsLoaded);
     if (isByRef) {
       builder.create<fir::StoreOp>(loc, result, lhs);
-      builder.create<mlir::omp::YieldOp>(loc, lhs);
+      genYield<DeclRedOpType>(builder, loc, lhs);
     } else {
-      builder.create<mlir::omp::YieldOp>(loc, result);
+      genYield<DeclRedOpType>(builder, loc, result);
     }
     return;
   }
   // all arrays should have been boxed
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {
-    genBoxCombiner(builder, loc, redId, boxTy, lhs, rhs);
+    genBoxCombiner<DeclRedOpType>(builder, loc, redId, boxTy, lhs, rhs);
     return;
   }
 
@@ -454,15 +496,13 @@ static mlir::Type unwrapSeqOrBoxedType(mlir::Type ty) {
   return ty;
 }
 
+template <typename OpType>
 static void createReductionAllocAndInitRegions(
-    AbstractConverter &converter, mlir::Location loc,
-    mlir::omp::DeclareReductionOp &reductionDecl,
+    AbstractConverter &converter, mlir::Location loc, OpType &reductionDecl,
     const ReductionProcessor::ReductionIdentifier redId, mlir::Type type,
     bool isByRef) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  auto yield = [&](mlir::Value ret) {
-    builder.create<mlir::omp::YieldOp>(loc, ret);
-  };
+  auto yield = [&](mlir::Value ret) { genYield<OpType>(builder, loc, ret); };
 
   mlir::Block *allocBlock = nullptr;
   mlir::Block *initBlock = nullptr;
@@ -512,7 +552,8 @@ static void createReductionAllocAndInitRegions(
   yield(boxAlloca);
 }
 
-mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
+template <typename OpType>
+OpType ReductionProcessor::createDeclareReduction(
     AbstractConverter &converter, llvm::StringRef reductionOpName,
     const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
     bool isByRef) {
@@ -522,8 +563,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
 
   assert(!reductionOpName.empty());
 
-  auto decl =
-      module.lookupSymbol<mlir::omp::DeclareReductionOp>(reductionOpName);
+  auto decl = module.lookupSymbol<OpType>(reductionOpName);
   if (decl)
     return decl;
 
@@ -532,8 +572,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
   if (!isByRef)
     type = valTy;
 
-  decl = modBuilder.create<mlir::omp::DeclareReductionOp>(loc, reductionOpName,
-                                                          type);
+  decl = modBuilder.create<OpType>(loc, reductionOpName, type);
   createReductionAllocAndInitRegions(converter, loc, decl, redId, type,
                                      isByRef);
 
@@ -544,7 +583,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
   builder.setInsertionPointToEnd(&decl.getReductionRegion().back());
   mlir::Value op1 = decl.getReductionRegion().front().getArgument(0);
   mlir::Value op2 = decl.getReductionRegion().front().getArgument(1);
-  genCombiner(builder, loc, redId, type, op1, op2, isByRef);
+  genCombiner<OpType>(builder, loc, redId, type, op1, op2, isByRef);
 
   return decl;
 }
@@ -563,64 +602,41 @@ static bool doReductionByRef(mlir::Value reductionVar) {
   return false;
 }
 
-mlir::omp::ReductionModifier translateReductionModifier(ReductionModifier mod) {
-  switch (mod) {
-  case ReductionModifier::Default:
-    return mlir::omp::ReductionModifier::defaultmod;
-  case ReductionModifier::Inscan:
-    return mlir::omp::ReductionModifier::inscan;
-  case ReductionModifier::Task:
-    return mlir::omp::ReductionModifier::task;
-  }
-  return mlir::omp::ReductionModifier::defaultmod;
-}
-
-template <class T>
+template <typename OpType, typename RedOperatorListTy>
 void ReductionProcessor::processReductionArguments(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const T &reduction, llvm::SmallVectorImpl<mlir::Value> &reductionVars,
+    const RedOperatorListTy &redOperatorList,
+    llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
-  if constexpr (std::is_same_v<T, omp::clause::Reduction>) {
-    auto mod = std::get<std::optional<ReductionModifier>>(reduction.t);
-    if (mod.has_value()) {
-      if (mod.value() == ReductionModifier::Task)
-        TODO(currentLocation, "Reduction modifier `task` is not supported");
-      else
-        *reductionMod = mlir::omp::ReductionModifierAttr::get(
-            firOpBuilder.getContext(), translateReductionModifier(mod.value()));
-    }
-  }
-
-  mlir::omp::DeclareReductionOp decl;
-  const auto &redOperatorList{
-      std::get<typename T::ReductionIdentifiers>(reduction.t)};
-  assert(redOperatorList.size() == 1 && "Expecting single operator");
-  const auto &redOperator = redOperatorList.front();
-  const auto &objectList{std::get<omp::ObjectList>(reduction.t)};
-
-  if (!std::holds_alternative<omp::clause::DefinedOperator>(redOperator.u)) {
-    if (const auto *reductionIntrinsic =
-            std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
-      if (!ReductionProcessor::supportedIntrinsicProcReduction(
-              *reductionIntrinsic)) {
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols) {
+  if constexpr (std::is_same_v<RedOperatorListTy,
+                               omp::clause::ReductionOperatorList>) {
+    // For OpenMP reduction clauses, check if the reduction operator is
+    // supported.
+    assert(redOperatorList.size() == 1 && "Expecting single operator");
+    const Fortran::lower::omp::clause::ReductionOperator &redOperator =
+        redOperatorList.front();
+
+    if (!std::holds_alternative<omp::clause::DefinedOperator>(redOperator.u)) {
+      if (const auto *reductionIntrinsic =
+              std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
+        if (!ReductionProcessor::supportedIntrinsicProcReduction(
+                *reductionIntrinsic)) {
+          return;
+        }
+      } else {
         return;
       }
-    } else {
-      return;
     }
   }
 
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
   // Reduction variable processing common to both intrinsic operators and
   // procedure designators
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  for (const Object &object : objectList) {
-    const semantics::Symbol *symbol = object.sym();
-    reductionSymbols.push_back(symbol);
+  for (const semantics::Symbol *symbol : reductionSymbols) {
     mlir::Value symVal = converter.getSymbolAddress(*symbol);
     mlir::Type eleType;
     auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
@@ -672,52 +688,63 @@ void ReductionProcessor::processReductionArguments(
     reduceVarByRef.push_back(doReductionByRef(symVal));
   }
 
+  unsigned idx = 0;
   for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
     auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
     const auto &kindMap = firOpBuilder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
 
-    if (const auto &redDefinedOp =
-            std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
-      const auto &intrinsicOp{
-          std::get<omp::clause::DefinedOperator::IntrinsicOperator>(
-              redDefinedOp->u)};
-      redId = getReductionType(intrinsicOp);
-      switch (redId) {
-      case ReductionIdentifier::ADD:
-      case ReductionIdentifier::MULTIPLY:
-      case ReductionIdentifier::AND:
-      case ReductionIdentifier::EQV:
-      case ReductionIdentifier::OR:
-      case ReductionIdentifier::NEQV:
-        break;
-      default:
-        TODO(currentLocation,
-             "Reduction of some intrinsic operators is not supported");
-        break;
-      }
-
-      reductionName = getReductionName(intrinsicOp, kindMap, redType, isByRef);
-    } else if (const auto *reductionIntrinsic =
-                   std::get_if<omp::clause::ProcedureDesignator>(
-                       &redOperator.u)) {
-      if (!ReductionProcessor::supportedIntrinsicProcReduction(
-              *reductionIntrinsic)) {
-        TODO(currentLocation, "Unsupported intrinsic proc reduction");
+    if constexpr (std::is_same_v<RedOperatorListTy,
+                                 omp::clause::ReductionOperatorList>) {
+      const Fortran::lower::omp::clause::ReductionOperator &redOperator =
+          redOperatorList.front();
+      if (const auto &redDefinedOp =
+              std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
+        const auto &intrinsicOp{
+            std::get<omp::clause::DefinedOperator::IntrinsicOperator>(
+                redDefinedOp->u)};
+        redId = getReductionType(intrinsicOp);
+        switch (redId) {
+        case ReductionIdentifier::ADD:
+        case ReductionIdentifier::MULTIPLY:
+        case ReductionIdentifier::AND:
+        case ReductionIdentifier::EQV:
+        case ReductionIdentifier::OR:
+        case ReductionIdentifier::NEQV:
+          break;
+        default:
+          TODO(currentLocation,
+               "Reduction of some intrinsic operators is not supported");
+          break;
+        }
+
+        reductionName = getReductionName(redId, kindMap, redType, isByRef);
+      } else if (const auto *reductionIntrinsic =
+                     std::get_if<omp::clause::ProcedureDesignator>(
+                         &redOperator.u)) {
+        if (!ReductionProcessor::supportedIntrinsicProcReduction(
+                *reductionIntrinsic)) {
+          TODO(currentLocation, "Unsupported intrinsic proc reduction");
+        }
+        redId = getReductionType(*reductionIntrinsic);
+        reductionName =
+            getReductionName(getRealName(*reductionIntrinsic).ToString(),
+                             kindMap, redType, isByRef);
+      } else {
+        TODO(currentLocation, "Unexpected reduction type");
       }
-      redId = getReductionType(*reductionIntrinsic);
-      reductionName =
-          getReductionName(getRealName(*reductionIntrinsic).ToString(), kindMap,
-                           redType, isByRef);
     } else {
-      TODO(currentLocation, "Unexpected reduction type");
+      // `do concurrent` reductions
+      redId = getReductionType(redOperatorList[idx]);
+      reductionName = getReductionName(redId, kindMap, redType, isByRef);
     }
 
-    decl = createDeclareReduction(converter, reductionName, redId, redType,
-                                  currentLocation, isByRef);
+    OpType decl = createDeclareReduction<OpType>(
+        converter, reductionName, redId, redType, currentLocation, isByRef);
     reductionDeclSymbols.push_back(
         mlir::SymbolRefAttr::get(firOpBuilder.getContext(), decl.getSymName()));
+    ++idx;
   }
 }
 
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/lib/Lower/OpenMP/ReductionProcessor.h
index a7198b48f6b4e..95b4b077bdc46 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.h
@@ -19,7 +19,6 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/type.h"
-#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Types.h"
 
@@ -65,6 +64,9 @@ class ReductionProcessor {
   static ReductionIdentifier
   getReductionType(omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp);
 
+  static ReductionIdentifier
+  getReductionType(const fir::ReduceOperationEnum &pd);
+
   static bool
   supportedIntrinsicProcReduction(const omp::clause::ProcedureDesignator &pd);
 
@@ -78,10 +80,9 @@ class ReductionProcessor {
                                       const fir::KindMapping &kindMap,
                                       mlir::Type ty, bool isByRef);
 
-  static std::string
-  getReductionName(omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp,
-                   const fir::KindMapping &kindMap, mlir::Type ty,
-                   bool isByRef);
+  static std::string getReductionName(ReductionIdentifier redId,
+                                      const fir::KindMapping &kindMap,
+                                      mlir::Type ty, bool isByRef);
 
   /// This function returns the identity value of the operator \p
   /// reductionOpName. For example:
@@ -113,22 +114,23 @@ class ReductionProcessor {
   /// symbol table. The declaration has a constant initializer with the neutral
   /// value `initValue`, and the reduction combiner carried over from `reduce`.
   /// TODO: add atomic region.
-  static mlir::omp::DeclareReductionOp
-  createDeclareReduction(AbstractConverter &builder,
-                         llvm::StringRef reductionOpName,
-                         const ReductionIdentifier redId, mlir::Type type,
-                         mlir::Location loc, bool isByRef);
+  template <typename OpType>
+  static OpType createDeclareReduction(AbstractConverter &builder,
+                                       llvm::StringRef reductionOpName,
+                                       const ReductionIdentifier redId,
+                                       mlir::Type type, mlir::Location loc,
+                                       bool isByRef);
 
   /// Creates a reduction declaration and associates it with an OpenMP block
   /// directive.
-  template <class T>
+  template <typename OpType, typename RedOperatorListTy>
   static void processReductionArguments(
       mlir::Location currentLocation, lower::AbstractConverter &converter,
-      const T &reduction, llvm::SmallVectorImpl<mlir::Value> &reductionVars,
+      const RedOperatorListTy &redOperatorList,
+      llvm::SmallVectorImpl<mlir::Value> &reductionVars,
       llvm::SmallVectorImpl<bool> &reduceVarByRef,
       llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-      llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-      mlir::omp::ReductionModifierAttr *reductionMod = nullptr);
+      const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 };
 
 template <typename FloatOp, typename IntegerOp>
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index c65f51ce6cacd..b9d2574a76ad0 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -668,9 +668,7 @@ void privatizeSymbol(
 
   const semantics::Symbol *sym =
       isDoConcurrent ? &symToPrivatize->GetUltimate() : symToPrivatize;
-  const lower::SymbolBox hsb = isDoConcurrent
-                                   ? converter.shallowLookupSymbol(*sym)
-                                   : converter.lookupOneLevelUpSymbol(*sym);
+  const lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
   assert(hsb && "Host symbol box not found");
 
   mlir::Location symLoc = hsb.getAddr().getLoc();
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 09038f6e4d3ae..87eef59fcf284 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -3341,26 +3341,26 @@ struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
   }
 };
 
-struct LocalitySpecifierOpConversion
-    : public fir::FIROpConversion<fir::LocalitySpecifierOp> {
-  using FIROpConversion::FIROpConversion;
+template <typename OpTy>
+struct DoConcurrentSpecifierOpConversion : public fir::FIROpConversion<OpTy> {
+  using fir::FIROpConversion<OpTy>::FIROpConversion;
   llvm::LogicalResult
-  matchAndRewrite(fir::LocalitySpecifierOp localizer, OpAdaptor adaptor,
+  matchAndRewrite(OpTy specifier, typename OpTy::Adaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
 #ifdef EXPENSIVE_CHECKS
     auto uses = mlir::SymbolTable::getSymbolUses(
-        localizer, localizer->getParentOfType<mlir::ModuleOp>());
+        specifier, specifier->getParentOfType<mlir::ModuleOp>());
 
-    // `fir.local` ops are not supposed to have any uses at this point (i.e.
-    // during lowering to LLVM). In case of serialization, the
-    // `fir.do_concurrent` users are expected to have been lowered to
+    // `fir.local|fir.declare_reduction` ops are not supposed to have any uses
+    // at this point (i.e. during lowering to LLVM). In case of serialization,
+    // the `fir.do_concurrent` users are expected to have been lowered to
     // `fir.do_loop` nests. In case of parallelization, the `fir.do_concurrent`
     // users are expected to have been lowered to the target parallel model
     // (e.g. OpenMP).
     assert(uses && uses->empty());
 #endif
 
-    rewriter.eraseOp(localizer);
+    rewriter.eraseOp(specifier);
     return mlir::success();
   }
 };
@@ -4315,20 +4315,22 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, VolatileCastOpConversion, ConvertOpConversion,
       CoordinateOpConversion, CopyOpConversion, DTEntryOpConversion,
-      DeclareOpConversion, DivcOpConversion, EmboxOpConversion,
-      EmboxCharOpConversion, EmboxProcOpConversion, ExtractValueOpConversion,
-      FieldIndexOpConversion, FirEndOpConversion, FreeMemOpConversion,
-      GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
-      IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
-      LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
-      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
-      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
-      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
-      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
-      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
-      XArrayCoorOpConversion, XEmboxOpConversion, XReboxOpConversion,
-      ZeroOpConversion>(converter, options);
+      DeclareOpConversion,
+      DoConcurrentSpecifierOpConversion<fir::LocalitySpecifierOp>,
+      DoConcurrentSpecifierOpConversion<fir::DeclareReductionOp>,
+      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
+      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
+      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
+      GlobalOpConversion, InsertOnRangeOpConversion, IsPresentOpConversion,
+      LenParamIndexOpConversion, LoadOpConversion, MulcOpConversion,
+      NegcOpConversion, NoReassocOpConversion, SelectCaseOpConversion,
+      SelectOpConversion, SelectRankOpConversion, SelectTypeOpConversion,
+      ShapeOpConversion, ShapeShiftOpConversion, ShiftOpConversion,
+      SliceOpConversion, StoreOpConversion, StringLitOpConversion,
+      SubcOpConversion, TypeDescOpConversion, TypeInfoOpConversion,
+      UnboxCharOpConversion, UnboxProcOpConversion, UndefOpConversion,
+      UnreachableOpConversion, XArrayCoorOpConversion, XEmboxOpConversion,
+      XReboxOpConversion, ZeroOpConversion>(converter, options);
 
   // Patterns that are populated without a type converter do not trigger
   // target materializations for the operands of the root op.
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..674129f546e94 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -5041,6 +5041,9 @@ void fir::BoxTotalElementsOp::getCanonicalizationPatterns(
 // LocalitySpecifierOp
 //===----------------------------------------------------------------------===//
 
+// TODO This is a copy of omp::PrivateClauseOp::verifiyRegions(). Once we find a
+// solution to merge both ops into one this duplication will not be needed. See:
+// https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
 llvm::LogicalResult fir::LocalitySpecifierOp::verifyRegions() {
   mlir::Type argType = getArgType();
   auto verifyTerminator = [&](mlir::Operation *terminator,
@@ -5136,6 +5139,84 @@ llvm::LogicalResult fir::LocalitySpecifierOp::verifyRegions() {
   return llvm::success();
 }
 
+// TODO This is a copy of omp::DeclareReductionOp::verifiyRegions(). Once we
+// find a solution to merge both ops into one this duplication will not be
+// needed.
+mlir::LogicalResult fir::DeclareReductionOp::verifyRegions() {
+  if (!getAllocRegion().empty()) {
+    for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
+      if (yieldOp.getResults().size() != 1 ||
+          yieldOp.getResults().getTypes()[0] != getType())
+        return emitOpError() << "expects alloc region to yield a value "
+                                "of the reduction type";
+    }
+  }
+
+  if (getInitializerRegion().empty())
+    return emitOpError() << "expects non-empty initializer region";
+  mlir::Block &initializerEntryBlock = getInitializerRegion().front();
+
+  if (initializerEntryBlock.getNumArguments() == 1) {
+    if (!getAllocRegion().empty())
+      return emitOpError() << "expects two arguments to the initializer region "
+                              "when an allocation region is used";
+  } else if (initializerEntryBlock.getNumArguments() == 2) {
+    if (getAllocRegion().empty())
+      return emitOpError() << "expects one argument to the initializer region "
+                              "when no allocation region is used";
+  } else {
+    return emitOpError()
+           << "expects one or two arguments to the initializer region";
+  }
+
+  for (mlir::Value arg : initializerEntryBlock.getArguments())
+    if (arg.getType() != getType())
+      return emitOpError() << "expects initializer region argument to match "
+                              "the reduction type";
+
+  for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
+    if (yieldOp.getResults().size() != 1 ||
+        yieldOp.getResults().getTypes()[0] != getType())
+      return emitOpError() << "expects initializer region to yield a value "
+                              "of the reduction type";
+  }
+
+  if (getReductionRegion().empty())
+    return emitOpError() << "expects non-empty reduction region";
+  mlir::Block &reductionEntryBlock = getReductionRegion().front();
+  if (reductionEntryBlock.getNumArguments() != 2 ||
+      reductionEntryBlock.getArgumentTypes()[0] !=
+          reductionEntryBlock.getArgumentTypes()[1] ||
+      reductionEntryBlock.getArgumentTypes()[0] != getType())
+    return emitOpError() << "expects reduction region with two arguments of "
+                            "the reduction type";
+  for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
+    if (yieldOp.getResults().size() != 1 ||
+        yieldOp.getResults().getTypes()[0] != getType())
+      return emitOpError() << "expects reduction region to yield a value "
+                              "of the reduction type";
+  }
+
+  if (!getAtomicReductionRegion().empty()) {
+    mlir::Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();
+    if (atomicReductionEntryBlock.getNumArguments() != 2 ||
+        atomicReductionEntryBlock.getArgumentTypes()[0] !=
+            atomicReductionEntryBlock.getArgumentTypes()[1])
+      return emitOpError() << "expects atomic reduction region with two "
+                              "arguments of the same type";
+  }
+
+  if (getCleanupRegion().empty())
+    return mlir::success();
+  mlir::Block &cleanupEntryBlock = getCleanupRegion().front();
+  if (cleanupEntryBlock.getNumArguments() != 1 ||
+      cleanupEntryBlock.getArgument(0).getType() != getType())
+    return emitOpError() << "expects cleanup region with one argument "
+                            "of the reduction type";
+
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // DoConcurrentOp
 //===----------------------------------------------------------------------===//
@@ -5157,6 +5238,86 @@ llvm::LogicalResult fir::DoConcurrentOp::verify() {
 // DoConcurrentLoopOp
 //===----------------------------------------------------------------------===//
 
+static mlir::ParseResult parseSpecifierList(
+    mlir::OpAsmParser &parser, mlir::OperationState &result,
+    llvm::StringRef specifierKeyword, llvm::StringRef symsAttrName,
+    llvm::SmallVectorImpl<mlir::OpAsmParser::Argument> &regionArgs,
+    llvm::SmallVectorImpl<mlir::Type> &regionArgTypes,
+    int32_t &numSpecifierOperands, bool isReduce = false) {
+  auto &builder = parser.getBuilder();
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> specifierOperands;
+
+  if (failed(parser.parseOptionalKeyword(specifierKeyword)))
+    return mlir::success();
+
+  std::size_t oldArgTypesSize = regionArgTypes.size();
+  if (failed(parser.parseLParen()))
+    return mlir::failure();
+
+  llvm::SmallVector<mlir::SymbolRefAttr> spceifierSymbolVec;
+  llvm::SmallVector<fir::ReduceAttr> attributes;
+
+  if (failed(parser.parseCommaSeparatedList([&]() {
+        if (failed(parser.parseAttribute(spceifierSymbolVec.emplace_back())))
+          return mlir::failure();
+
+        if (isReduce &&
+            failed(parser.parseAttribute(attributes.emplace_back()))) {
+          return mlir::failure();
+        }
+
+        if (parser.parseOperand(specifierOperands.emplace_back()) ||
+            parser.parseArrow() ||
+            parser.parseArgument(regionArgs.emplace_back()))
+          return mlir::failure();
+
+        return mlir::success();
+      })))
+    return mlir::failure();
+
+  if (failed(parser.parseColon()))
+    return mlir::failure();
+
+  if (failed(parser.parseCommaSeparatedList([&]() {
+        if (failed(parser.parseType(regionArgTypes.emplace_back())))
+          return mlir::failure();
+
+        return mlir::success();
+      })))
+    return mlir::failure();
+
+  if (regionArgs.size() != regionArgTypes.size())
+    return parser.emitError(parser.getNameLoc(), "mismatch in number of " +
+                                                     specifierKeyword.str() +
+                                                     " arg and types");
+
+  if (failed(parser.parseRParen()))
+    return mlir::failure();
+
+  for (auto operandType :
+       llvm::zip_equal(specifierOperands,
+                       llvm::drop_begin(regionArgTypes, oldArgTypesSize)))
+    if (parser.resolveOperand(std::get<0>(operandType),
+                              std::get<1>(operandType), result.operands))
+      return mlir::failure();
+
+  llvm::SmallVector<mlir::Attribute> symbolAttrs(spceifierSymbolVec.begin(),
+                                                 spceifierSymbolVec.end());
+  result.addAttribute(symsAttrName, builder.getArrayAttr(symbolAttrs));
+
+  if (isReduce) {
+    llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
+                                                 attributes.end());
+    result.addAttribute(
+        fir::DoConcurrentLoopOp::getReduceAttrsAttrName(result.name),
+        builder.getArrayAttr(arrayAttr));
+  }
+
+  numSpecifierOperands = specifierOperands.size();
+
+  return mlir::success();
+}
+
 mlir::ParseResult fir::DoConcurrentLoopOp::parse(mlir::OpAsmParser &parser,
                                                  mlir::OperationState &result) {
   auto &builder = parser.getBuilder();
@@ -5192,90 +5353,26 @@ mlir::ParseResult fir::DoConcurrentLoopOp::parse(mlir::OpAsmParser &parser,
       parser.resolveOperands(steps, builder.getIndexType(), result.operands))
     return mlir::failure();
 
-  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> reduceOperands;
-  llvm::SmallVector<mlir::Type> reduceArgTypes;
-  if (succeeded(parser.parseOptionalKeyword("reduce"))) {
-    // Parse reduction attributes and variables.
-    llvm::SmallVector<fir::ReduceAttr> attributes;
-    if (failed(parser.parseCommaSeparatedList(
-            mlir::AsmParser::Delimiter::Paren, [&]() {
-              if (parser.parseAttribute(attributes.emplace_back()) ||
-                  parser.parseArrow() ||
-                  parser.parseOperand(reduceOperands.emplace_back()) ||
-                  parser.parseColonType(reduceArgTypes.emplace_back()))
-                return mlir::failure();
-              return mlir::success();
-            })))
-      return mlir::failure();
-    // Resolve input operands.
-    for (auto operand_type : llvm::zip(reduceOperands, reduceArgTypes))
-      if (parser.resolveOperand(std::get<0>(operand_type),
-                                std::get<1>(operand_type), result.operands))
-        return mlir::failure();
-    llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
-                                                 attributes.end());
-    result.addAttribute(getReduceAttrsAttrName(result.name),
-                        builder.getArrayAttr(arrayAttr));
-  }
-
-  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> localOperands;
-  if (succeeded(parser.parseOptionalKeyword("local"))) {
-    std::size_t oldArgTypesSize = argTypes.size();
-    if (failed(parser.parseLParen()))
-      return mlir::failure();
-
-    llvm::SmallVector<mlir::SymbolRefAttr> localSymbolVec;
-    if (failed(parser.parseCommaSeparatedList([&]() {
-          if (failed(parser.parseAttribute(localSymbolVec.emplace_back())))
-            return mlir::failure();
-
-          if (parser.parseOperand(localOperands.emplace_back()) ||
-              parser.parseArrow() ||
-              parser.parseArgument(regionArgs.emplace_back()))
-            return mlir::failure();
-
-          return mlir::success();
-        })))
-      return mlir::failure();
-
-    if (failed(parser.parseColon()))
-      return mlir::failure();
-
-    if (failed(parser.parseCommaSeparatedList([&]() {
-          if (failed(parser.parseType(argTypes.emplace_back())))
-            return mlir::failure();
-
-          return mlir::success();
-        })))
-      return mlir::failure();
-
-    if (regionArgs.size() != argTypes.size())
-      return parser.emitError(parser.getNameLoc(),
-                              "mismatch in number of local arg and types");
-
-    if (failed(parser.parseRParen()))
-      return mlir::failure();
-
-    for (auto operandType : llvm::zip_equal(
-             localOperands, llvm::drop_begin(argTypes, oldArgTypesSize)))
-      if (parser.resolveOperand(std::get<0>(operandType),
-                                std::get<1>(operandType), result.operands))
-        return mlir::failure();
+  int32_t numLocalOperands = 0;
+  if (failed(parseSpecifierList(parser, result, "local",
+                                getLocalSymsAttrName(result.name), regionArgs,
+                                argTypes, numLocalOperands)))
+    return mlir::failure();
 
-    llvm::SmallVector<mlir::Attribute> symbolAttrs(localSymbolVec.begin(),
-                                                   localSymbolVec.end());
-    result.addAttribute(getLocalSymsAttrName(result.name),
-                        builder.getArrayAttr(symbolAttrs));
-  }
+  int32_t numReduceOperands = 0;
+  if (failed(parseSpecifierList(
+          parser, result, "reduce", getReduceSymsAttrName(result.name),
+          regionArgs, argTypes, numReduceOperands, /*isReduce=*/true)))
+    return mlir::failure();
 
   // Set `operandSegmentSizes` attribute.
-  result.addAttribute(DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
-                      builder.getDenseI32ArrayAttr(
-                          {static_cast<int32_t>(lower.size()),
-                           static_cast<int32_t>(upper.size()),
-                           static_cast<int32_t>(steps.size()),
-                           static_cast<int32_t>(reduceOperands.size()),
-                           static_cast<int32_t>(localOperands.size())}));
+  result.addAttribute(
+      DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
+      builder.getDenseI32ArrayAttr({static_cast<int32_t>(lower.size()),
+                                    static_cast<int32_t>(upper.size()),
+                                    static_cast<int32_t>(steps.size()),
+                                    static_cast<int32_t>(numLocalOperands),
+                                    static_cast<int32_t>(numReduceOperands)}));
 
   // Now parse the body.
   for (auto [arg, type] : llvm::zip_equal(regionArgs, argTypes))
@@ -5297,17 +5394,6 @@ void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
     << ") = (" << getLowerBound() << ") to (" << getUpperBound() << ") step ("
     << getStep() << ")";
 
-  if (!getReduceOperands().empty()) {
-    p << " reduce(";
-    auto attrs = getReduceAttrsAttr();
-    auto operands = getReduceOperands();
-    llvm::interleaveComma(llvm::zip(attrs, operands), p, [&](auto it) {
-      p << std::get<0>(it) << " -> " << std::get<1>(it) << " : "
-        << std::get<1>(it).getType();
-    });
-    p << ')';
-  }
-
   if (!getLocalVars().empty()) {
     p << " local(";
     llvm::interleaveComma(llvm::zip_equal(getLocalSymsAttr(), getLocalVars(),
@@ -5322,13 +5408,30 @@ void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
     p << ")";
   }
 
+  if (!getReduceVars().empty()) {
+    p << " reduce(";
+    llvm::interleaveComma(
+        llvm::zip_equal(getReduceSymsAttr(), getReduceAttrsAttr(),
+                        getReduceVars(), getRegionReduceArgs()),
+        p, [&](auto it) {
+          p << std::get<0>(it) << " " << std::get<1>(it) << " "
+            << std::get<2>(it) << " -> " << std::get<3>(it);
+        });
+    p << " : ";
+    llvm::interleaveComma(getReduceVars(), p,
+                          [&](auto it) { p << it.getType(); });
+    p << ")";
+  }
+
   p << ' ';
   p.printRegion(getRegion(), /*printEntryBlockArgs=*/false);
   p.printOptionalAttrDict(
       (*this)->getAttrs(),
       /*elidedAttrs=*/{DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
+                       DoConcurrentLoopOp::getLocalSymsAttrName(),
+                       DoConcurrentLoopOp::getReduceSymsAttrName(),
                        DoConcurrentLoopOp::getReduceAttrsAttrName(),
-                       DoConcurrentLoopOp::getLocalSymsAttrName()});
+                       DoConcurrentLoopOp::getReduceByrefAttrName()});
 }
 
 llvm::SmallVector<mlir::Region *> fir::DoConcurrentLoopOp::getLoopRegions() {
@@ -5340,6 +5443,7 @@ llvm::LogicalResult fir::DoConcurrentLoopOp::verify() {
   mlir::Operation::operand_range ubValues = getUpperBound();
   mlir::Operation::operand_range stepValues = getStep();
   mlir::Operation::operand_range localVars = getLocalVars();
+  mlir::Operation::operand_range reduceVars = getReduceVars();
 
   if (lbValues.empty())
     return emitOpError(
@@ -5353,7 +5457,8 @@ llvm::LogicalResult fir::DoConcurrentLoopOp::verify() {
   // Check that the body defines the same number of block arguments as the
   // number of tuple elements in step.
   mlir::Block *body = getBody();
-  unsigned numIndVarArgs = body->getNumArguments() - localVars.size();
+  unsigned numIndVarArgs =
+      body->getNumArguments() - localVars.size() - reduceVars.size();
 
   if (numIndVarArgs != stepValues.size())
     return emitOpError() << "expects the same number of induction variables: "
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index e440852b3103a..506c8e66dbdfa 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -234,6 +234,10 @@ class DoConcurrentConversion
       loop.setLocalSymsAttr(nullptr);
     }
 
+    for (auto [reduceVar, reduceArg] :
+         llvm::zip_equal(loop.getReduceVars(), loop.getRegionReduceArgs()))
+      rewriter.replaceAllUsesWith(reduceArg, reduceVar);
+
     // Collect iteration variable(s) allocations so that we can move them
     // outside the `fir.do_concurrent` wrapper.
     llvm::SmallVector<mlir::Operation *> opsToMove;
@@ -257,12 +261,16 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceOperands(),
+          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
     }
 
+    loop.getRegion().front().eraseArguments(loop.getNumInductionVars() +
+                                                loop.getNumLocalOperands(),
+                                            loop.getNumReduceOperands());
+
     rewriter.inlineBlockBefore(
         &loopBlock, innermostUnorderdLoop.getBody()->getTerminator(), ivArgs);
     rewriter.eraseOp(doConcurentOp);
diff --git a/flang/test/Fir/do_concurrent.fir b/flang/test/Fir/do_concurrent.fir
index cc1197ba56bd7..c207ea5dea825 100644
--- a/flang/test/Fir/do_concurrent.fir
+++ b/flang/test/Fir/do_concurrent.fir
@@ -63,7 +63,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
     %j = fir.alloca i32
     fir.do_concurrent.loop
       (%i_iv, %j_iv) = (%i_lb, %j_lb) to (%i_ub, %j_ub) step (%i_st, %j_st) 
-      reduce(#fir.reduce_attr<add> -> %sum : !fir.ref<i32>) {
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %sum -> %sum_arg : !fir.ref<i32>) {
       %0 = fir.convert %i_iv : (index) -> i32
       fir.store %0 to %i : !fir.ref<i32>
 
@@ -83,7 +83,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
 // CHECK:           %[[I:.*]] = fir.alloca i32
 // CHECK:           %[[J:.*]] = fir.alloca i32
 // CHECK:           fir.do_concurrent.loop
-// CHECK-SAME:        (%[[I_IV:.*]], %[[J_IV:.*]]) = (%[[I_LB]], %[[J_LB]]) to (%[[I_UB]], %[[J_UB]]) step (%[[I_ST]], %[[J_ST]]) reduce(#fir.reduce_attr<add> -> %[[SUM]] : !fir.ref<i32>) {
+// CHECK-SAME:        (%[[I_IV:.*]], %[[J_IV:.*]]) = (%[[I_LB]], %[[J_LB]]) to (%[[I_UB]], %[[J_UB]]) step (%[[I_ST]], %[[J_ST]]) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[SUM]] -> %{{.*}} : !fir.ref<i32>) {
 // CHECK:             %[[I_IV_CVT:.*]] = fir.convert %[[I_IV]] : (index) -> i32
 // CHECK:             fir.store %[[I_IV_CVT]] to %[[I]] : !fir.ref<i32>
 // CHECK:             %[[J_IV_CVT:.*]] = fir.convert %[[J_IV]] : (index) -> i32
@@ -161,3 +161,62 @@ func.func @do_concurrent_with_locality_specs() {
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @dc_reduce() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c1 = arith.constant 1 : index 
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(@add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1 : !fir.ref<i32>) {
+    }
+  }
+  return
+}
+
+// CHECK-LABEL:  func.func @dc_reduce() {
+// CHECK:          %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+// CHECK:          %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:          fir.do_concurrent {
+// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
+// CHECK:            }
+// CHECK:          }
+// CHECK:          return
+// CHECK:        }
+
+func.func @dc_reduce_2() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  %5 = fir.alloca i32 {bindc_name = "m", uniq_name = "dc_reduce"}
+  %6:2 = hlfir.declare %5 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  %c1 = arith.constant 1 : index 
+
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) 
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1,
+             @mul_reduction_i32 #fir.reduce_attr<multiply> %6#0 -> %arg2
+             : !fir.ref<i32>, !fir.ref<i32>) {
+    }
+  }
+
+  return
+}
+
+// CHECK-LABEL:  func.func @dc_reduce_2() {
+// CHECK:          %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+// CHECK:          %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+// CHECK:          %[[M_ALLOC:.*]] = fir.alloca i32 {bindc_name = "m", uniq_name = "dc_reduce"}
+// CHECK:          %[[M_DECL:.*]]:2 = hlfir.declare %[[M_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:          fir.do_concurrent {
+// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{[^[:space:]]+}}) 
+// CHECK-SAME:         reduce(
+// CHECK-SAME:           @add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:[^,]+]],
+// CHECK-SAME:           @mul_reduction_i32 #fir.reduce_attr<multiply> %[[M_DECL]]#0 -> %[[M_ARG:[^[:space:]]+]]
+// CHECK-SAME:           : !fir.ref<i32>, !fir.ref<i32>) {
+// CHECK:            }
+// CHECK:          }
+// CHECK:          return
+// CHECK:        }
+
diff --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index aca0ecc1abdc1..e32ea7ad3c729 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -1256,8 +1256,8 @@ func.func @dc_invalid_reduction(%arg0: index, %arg1: index) {
   %sum = fir.alloca i32
   // expected-error at +2 {{'fir.do_concurrent.loop' op mismatch in number of reduction variables and reduction attributes}}
   fir.do_concurrent {
-    "fir.do_concurrent.loop"(%arg0, %arg1, %arg0, %sum) <{operandSegmentSizes = array<i32: 1, 1, 1, 1, 0>}> ({
-    ^bb0(%arg3: index):
+    "fir.do_concurrent.loop"(%arg0, %arg1, %arg0, %sum) <{operandSegmentSizes = array<i32: 1, 1, 1, 0, 1>}> ({
+    ^bb0(%arg3: index, %sum_arg: i32):
       %tmp = "fir.alloca"() <{in_type = i32, operandSegmentSizes = array<i32: 0, 0>}> : () -> !fir.ref<i32>
     }) : (index, index, index, !fir.ref<i32>) -> ()
   }
@@ -1266,6 +1266,20 @@ func.func @dc_invalid_reduction(%arg0: index, %arg1: index) {
 
 // -----
 
+func.func @dc_reduce_no_attr() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c1 = arith.constant 1 : index 
+  // expected-error at +2 {{expected attribute value}}
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(@add_reduction_i32 %4#0 -> %arg1 : !fir.ref<i32>) {
+    }
+  }
+  return
+}
+
+// -----
+
 // Should fail when volatility changes from a fir.convert
 func.func @bad_convert_volatile(%arg0: !fir.ref<i32>) -> !fir.ref<i32, volatile> {
   // expected-error at +1 {{op this conversion does not preserve volatility}}
diff --git a/flang/test/Lower/do_concurrent_reduce.f90 b/flang/test/Lower/do_concurrent_reduce.f90
new file mode 100644
index 0000000000000..8591a21e2b9e0
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_reduce.f90
@@ -0,0 +1,41 @@
+! RUN: %flang_fc1 -emit-hlfir -mmlir --enable-delayed-privatization-staging=true -o - %s | FileCheck %s
+
+subroutine do_concurrent_reduce
+  implicit none
+  integer :: s, i
+
+  do concurrent (i=1:10) reduce(+:s)
+    s = s + 1
+  end do
+end
+
+! CHECK-LABEL:  fir.declare_reduction @add_reduction_i32 : i32 init {
+! CHECK:        ^bb0(%[[ARG0:.*]]: i32):
+! CHECK:          %[[VAL_0:.*]] = arith.constant 0 : i32
+! CHECK:          fir.yield(%[[VAL_0]] : i32)
+! CHECK:        } combiner {
+! CHECK:          ^bb0(%[[VAL_1:.*]]: i32, %[[VAL_2:.*]]: i32):
+! CHECK:          %[[VAL_3:.*]] = arith.addi %[[VAL_1]], %[[VAL_2]] : i32
+! CHECK:          fir.yield(%[[VAL_3]] : i32)
+! CHECK:        }
+
+! CHECK-LABEL:  func.func @_QPdo_concurrent_reduce() {
+! CHECK:           %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "_QFdo_concurrent_reduceEs"}
+! CHECK:           %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "_QFdo_concurrent_reduceEs"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+! CHECK:           fir.do_concurrent {
+! CHECK:             %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i"}
+! CHECK:             %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFdo_concurrent_reduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:             fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{[^[:space:]]+}}) 
+! CHECK-SAME:          reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
+
+! CHECK:               %[[S_ARG_DECL:.*]]:2 = hlfir.declare %[[S_ARG]] {uniq_name = "_QFdo_concurrent_reduceEs"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:               %[[S_ARG_VAL:.*]] = fir.load %[[S_ARG_DECL]]#0 : !fir.ref<i32>
+! CHECK:               %[[C1:.*]] = arith.constant 1 : i32
+! CHECK:               %[[RED_UPDATE:.*]] = arith.addi %[[S_ARG_VAL]], %[[C1]] : i32
+! CHECK:               hlfir.assign %[[RED_UPDATE]] to %[[S_ARG_DECL]]#0 : i32, !fir.ref<i32>
+
+! CHECK:             }
+! CHECK:           }
+! CHECK:           return
+! CHECK:        }
diff --git a/flang/test/Lower/loops.f90 b/flang/test/Lower/loops.f90
index 60df27a591dc3..64f14ff972272 100644
--- a/flang/test/Lower/loops.f90
+++ b/flang/test/Lower/loops.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
+! RUN: bbc -emit-fir -hlfir=false --enable-delayed-privatization=false -o - %s | FileCheck %s
 
 ! CHECK-LABEL: loop_test
 subroutine loop_test
diff --git a/flang/test/Lower/loops3.f90 b/flang/test/Lower/loops3.f90
index 84db1972cca16..2965b954b49a8 100644
--- a/flang/test/Lower/loops3.f90
+++ b/flang/test/Lower/loops3.f90
@@ -12,7 +12,7 @@ subroutine loop_test
 
   ! CHECK: %[[VAL_0:.*]] = fir.alloca f32 {bindc_name = "m", uniq_name = "_QFloop_testEm"}
   ! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QFloop_testEsum) : !fir.ref<i32>
-  ! CHECK: fir.do_concurrent.loop ({{.*}}) = ({{.*}}) to ({{.*}}) step ({{.*}}) reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
+  ! CHECK: fir.do_concurrent.loop ({{.*}}) = ({{.*}}) to ({{.*}}) step ({{.*}}) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[VAL_1]] -> %{{.*}}, @other_reduction_f32 #fir.reduce_attr<max> %[[VAL_0]] -> %{{.*}} : {{.*}}) {
   do concurrent (i=1:5, j=1:5, k=1:5) local(tmp) reduce(+:sum) reduce(max:m)
     tmp = i + j + k
     sum = tmp + sum
diff --git a/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir b/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
index d9ef36b175598..c550ab8a97d4c 100644
--- a/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
+++ b/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
@@ -86,7 +86,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
     %j = fir.alloca i32
     fir.do_concurrent.loop
       (%i_iv, %j_iv) = (%i_lb, %j_lb) to (%i_ub, %j_ub) step (%i_st, %j_st) 
-      reduce(#fir.reduce_attr<add> -> %sum : !fir.ref<i32>) {
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %sum -> %sum_arg : !fir.ref<i32>) {
       %0 = fir.convert %i_iv : (index) -> i32
       fir.store %0 to %i : !fir.ref<i32>
 

>From 7b1680e42a014ffbede691d7d8ba71e656a8dfd3 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Thu, 26 Jun 2025 23:17:37 -0500
Subject: [PATCH 2/3] review comments

---
 flang/include/flang/Optimizer/Dialect/FIROps.td | 4 ++--
 flang/lib/Lower/Bridge.cpp                      | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index b217f1433767f..41363234e5c6b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3667,10 +3667,10 @@ def fir_DeclareReductionOp : fir_Op<"declare_reduction", [IsolatedFromAbove,
   let summary = "declares a reduction kind";
   let description = [{
     Note: this operation is adapted from omp::DeclareReductionOp. There is a lot
-    duplication at the moment. TODO Combined both ops into one. See:
+    duplication at the moment. TODO Combine both ops into one. See:
     https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
 
-    Declares an `do concurrent` reduction. This requires two mandatory and three
+    Declares a `do concurrent` reduction. This requires two mandatory and three
     optional regions.
 
       1. The optional alloc region specifies how to allocate the thread-local
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index e2dc8131331d3..8e1c7becdba45 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2117,10 +2117,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
     llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
 
-    for (const auto &reduceOp : info.reduceOperatorList) {
+    for (const auto &reduceOp : info.reduceOperatorList)
       nestReduceAttrs.push_back(
           fir::ReduceAttr::get(builder->getContext(), reduceOp));
-    }
 
     llvm::SmallVector<mlir::Value> reduceVars;
     Fortran::lower::omp::ReductionProcessor rp;
@@ -2556,7 +2555,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
-          loc, nestLBs, nestUBs, nestSts, nullptr, /*local_vars=*/std::nullopt,
+          loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
+          /*local_vars=*/std::nullopt,
           /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
           /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
           /*reduce_attrs=*/nullptr);

>From 0095db8a5f72b58527c84d1ff8eb0cbf8aa7445f Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Fri, 27 Jun 2025 01:54:06 -0500
Subject: [PATCH 3/3] byref printing and parsing

---
 flang/lib/Optimizer/Dialect/FIROps.cpp | 27 ++++++++++++++++++++------
 flang/test/Fir/do_concurrent.fir       |  4 ++--
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 674129f546e94..6b40e7015fdd8 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -5254,17 +5254,21 @@ static mlir::ParseResult parseSpecifierList(
   if (failed(parser.parseLParen()))
     return mlir::failure();
 
+  llvm::SmallVector<bool> isByRefVec;
   llvm::SmallVector<mlir::SymbolRefAttr> spceifierSymbolVec;
   llvm::SmallVector<fir::ReduceAttr> attributes;
 
   if (failed(parser.parseCommaSeparatedList([&]() {
+        if (isReduce)
+          isByRefVec.push_back(
+              parser.parseOptionalKeyword("byref").succeeded());
+
         if (failed(parser.parseAttribute(spceifierSymbolVec.emplace_back())))
           return mlir::failure();
 
         if (isReduce &&
-            failed(parser.parseAttribute(attributes.emplace_back()))) {
+            failed(parser.parseAttribute(attributes.emplace_back())))
           return mlir::failure();
-        }
 
         if (parser.parseOperand(specifierOperands.emplace_back()) ||
             parser.parseArrow() ||
@@ -5301,6 +5305,13 @@ static mlir::ParseResult parseSpecifierList(
                               std::get<1>(operandType), result.operands))
       return mlir::failure();
 
+  if (isReduce)
+    result.addAttribute(
+        fir::DoConcurrentLoopOp::getReduceByrefAttrName(result.name),
+        isByRefVec.empty()
+            ? nullptr
+            : mlir::DenseBoolArrayAttr::get(builder.getContext(), isByRefVec));
+
   llvm::SmallVector<mlir::Attribute> symbolAttrs(spceifierSymbolVec.begin(),
                                                  spceifierSymbolVec.end());
   result.addAttribute(symsAttrName, builder.getArrayAttr(symbolAttrs));
@@ -5411,11 +5422,15 @@ void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
   if (!getReduceVars().empty()) {
     p << " reduce(";
     llvm::interleaveComma(
-        llvm::zip_equal(getReduceSymsAttr(), getReduceAttrsAttr(),
-                        getReduceVars(), getRegionReduceArgs()),
+        llvm::zip_equal(getReduceByrefAttr().asArrayRef(), getReduceSymsAttr(),
+                        getReduceAttrsAttr(), getReduceVars(),
+                        getRegionReduceArgs()),
         p, [&](auto it) {
-          p << std::get<0>(it) << " " << std::get<1>(it) << " "
-            << std::get<2>(it) << " -> " << std::get<3>(it);
+          if (std::get<0>(it))
+            p << "byref ";
+
+          p << std::get<1>(it) << " " << std::get<2>(it) << " "
+            << std::get<3>(it) << " -> " << std::get<4>(it);
         });
     p << " : ";
     llvm::interleaveComma(getReduceVars(), p,
diff --git a/flang/test/Fir/do_concurrent.fir b/flang/test/Fir/do_concurrent.fir
index c207ea5dea825..6e2173447855e 100644
--- a/flang/test/Fir/do_concurrent.fir
+++ b/flang/test/Fir/do_concurrent.fir
@@ -167,7 +167,7 @@ func.func @dc_reduce() {
   %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   %c1 = arith.constant 1 : index 
   fir.do_concurrent {
-    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(@add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1 : !fir.ref<i32>) {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(byref @add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1 : !fir.ref<i32>) {
     }
   }
   return
@@ -177,7 +177,7 @@ func.func @dc_reduce() {
 // CHECK:          %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
 // CHECK:          %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 // CHECK:          fir.do_concurrent {
-// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
+// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) reduce(byref @add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
 // CHECK:            }
 // CHECK:          }
 // CHECK:          return


From flang-commits at lists.llvm.org  Wed Jul  2 08:43:31 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 02 Jul 2025 08:43:31 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add
	ComplexTOROCDLLibraryCalls pass (PR #144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <686553a3.050a0220.276c9.88eb@mx.google.com>


https://github.com/TIFitis updated https://github.com/llvm/llvm-project/pull/144926

>From ba1230d545d045623c1ab3e202b7e2b836efe02f Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 19 Jun 2025 17:25:48 +0100
Subject: [PATCH 1/7] [MLIR] Add ComplexTOROCDL pass

This patch adds a new ComplexToROCDL pass to convert complex.abs operations to __ocml_cabs_f32/__ocml_cabs_f64 calls.
---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt    |  1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  5 +-
 .../ComplexToROCDL/ComplexToROCDL.h           | 19 ++++
 mlir/include/mlir/Conversion/Passes.h         |  1 +
 mlir/include/mlir/Conversion/Passes.td        | 12 +++
 mlir/lib/Conversion/CMakeLists.txt            |  1 +
 .../Conversion/ComplexToROCDL/CMakeLists.txt  | 18 ++++
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 95 +++++++++++++++++++
 .../ComplexToROCDL/complex-to-rocdl.mlir      | 13 +++
 9 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
 create mode 100644 mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
 create mode 100644 mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
 create mode 100644 mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 980307db315d9..8b4ac18fba527 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -40,6 +40,7 @@ add_flang_library(FIRCodeGen
   MLIRMathToLLVM
   MLIRMathToLibm
   MLIRMathToROCDL
+  MLIRComplexToROCDL
   MLIROpenMPToLLVM
   MLIROpenACCDialect
   MLIRBuiltinToLLVMIRTranslation
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..f721b6232b0fb 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -33,6 +33,7 @@
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
@@ -4105,8 +4106,10 @@ class FIRToLLVMLowering
     // GPU library calls, the rest can be converted to LLVM intrinsics, which
     // is handled in the mathToLLVM conversion. The lowering to libm calls is
     // not needed since all math operations are handled this way.
-    if (isAMDGCN)
+    if (isAMDGCN) {
       mathConvertionPM.addPass(mlir::createConvertMathToROCDL());
+      mathConvertionPM.addPass(mlir::createConvertComplexToROCDL());
+    }
 
     // Convert math::FPowI operations to inline implementation
     // only if the exponent's width is greater than 32, otherwise,
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
new file mode 100644
index 0000000000000..ed65be9980408
--- /dev/null
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -0,0 +1,19 @@
+#ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+class RewritePatternSet;
+
+#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDL
+#include "mlir/Conversion/Passes.h.inc"
+
+/// Populate the given list with patterns that convert from Complex to ROCDL
+/// calls.
+void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
+                                              PatternBenefit benefit);
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index c9d2a54433736..67e8f5b99b67b 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -23,6 +23,7 @@
 #include "mlir/Conversion/BufferizationToMemRef/BufferizationToMemRef.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
 #include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h"
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRVPass.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index b496ee0114910..8ad2341f93a15 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -312,6 +312,18 @@ def ConvertComplexToLibm : Pass<"convert-complex-to-libm", "ModuleOp"> {
   let dependentDialects = ["func::FuncDialect"];
 }
 
+//===----------------------------------------------------------------------===//
+// ComplexToROCDL
+//===----------------------------------------------------------------------===//
+
+def ConvertComplexToROCDL : Pass<"convert-complex-to-rocdl", "ModuleOp"> {
+  let summary = "Convert Complex dialect to ROCDL calls";
+  let description = [{
+    This pass converts supported Complex ops to calls to the AMD device library.
+  }];
+  let dependentDialects = ["func::FuncDialect"];
+}
+
 //===----------------------------------------------------------------------===//
 // ComplexToSPIRV
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index e4b4974600577..4ad81553a4fa8 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(AsyncToLLVM)
 add_subdirectory(BufferizationToMemRef)
 add_subdirectory(ComplexCommon)
 add_subdirectory(ComplexToLibm)
+add_subdirectory(ComplexToROCDL)
 add_subdirectory(ComplexToLLVM)
 add_subdirectory(ComplexToSPIRV)
 add_subdirectory(ComplexToStandard)
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
new file mode 100644
index 0000000000000..54607250083d7
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_conversion_library(MLIRComplexToROCDL
+  ComplexToROCDL.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDL
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRComplexDialect
+  MLIRFuncDialect
+  MLIRPass
+  MLIRTransformUtils
+  )
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
new file mode 100644
index 0000000000000..cdfe2a6dfe874
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -0,0 +1,95 @@
+//===-- ComplexToROCDL.cpp - conversion from Complex to ROCDL calls -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+
+#include "mlir/Dialect/Complex/IR/Complex.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include <optional>
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+
+namespace {
+struct FloatTypeResolver {
+  std::optional<bool> operator()(Type type) const {
+    auto elementType = cast<FloatType>(type);
+    if (!isa<Float32Type, Float64Type>(elementType))
+      return {};
+    return elementType.getIntOrFloatBitWidth() == 64;
+  }
+};
+
+template <typename Op, typename TypeResolver = FloatTypeResolver>
+struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
+  using OpRewritePattern<Op>::OpRewritePattern;
+  ScalarOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
+                      StringRef doubleFunc, PatternBenefit benefit)
+      : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
+        doubleFunc(doubleFunc) {}
+
+  LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
+    auto module = SymbolTable::getNearestSymbolTable(op);
+    auto isDouble = TypeResolver()(op.getType());
+    if (!isDouble.has_value())
+      return failure();
+
+    auto name = *isDouble ? doubleFunc : floatFunc;
+
+    auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
+        SymbolTable::lookupSymbolIn(module, name));
+    if (!opFunc) {
+      OpBuilder::InsertionGuard guard(rewriter);
+      rewriter.setInsertionPointToStart(&module->getRegion(0).front());
+      auto funcTy = FunctionType::get(
+          rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
+      opFunc =
+          rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name, funcTy);
+      opFunc.setPrivate();
+    }
+    rewriter.replaceOpWithNewOp<func::CallOp>(op, name, op.getType(),
+                                              op->getOperands());
+    return success();
+  }
+
+private:
+  std::string floatFunc, doubleFunc;
+};
+} // namespace
+
+void mlir::populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
+                                                    PatternBenefit benefit) {
+  patterns.add<ScalarOpToROCDLCall<complex::AbsOp>>(
+      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64", benefit);
+}
+
+namespace {
+struct ConvertComplexToROCDLPass
+    : public impl::ConvertComplexToROCDLBase<ConvertComplexToROCDLPass> {
+  void runOnOperation() override;
+};
+} // namespace
+
+void ConvertComplexToROCDLPass::runOnOperation() {
+  auto module = getOperation();
+
+  RewritePatternSet patterns(&getContext());
+  populateComplexToROCDLConversionPatterns(patterns, /*benefit=*/1);
+
+  ConversionTarget target(getContext());
+  target.addLegalDialect<func::FuncDialect>();
+  target.addIllegalOp<complex::AbsOp>();
+  if (failed(applyPartialConversion(module, target, std::move(patterns))))
+    signalPassFailure();
+}
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
new file mode 100644
index 0000000000000..618e9c238378c
--- /dev/null
+++ b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
@@ -0,0 +1,13 @@
+// RUN: mlir-opt %s -convert-complex-to-rocdl -canonicalize | FileCheck %s
+
+// CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
+// CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64
+
+func.func @abs_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
+  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%[[F:.*]])
+  %rf = complex.abs %f : complex<f32>
+  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%[[D:.*]])
+  %rd = complex.abs %d : complex<f64>
+  // CHECK: return %[[RF]], %[[RD]]
+  return %rf, %rd : f32, f64
+}

>From a10e50ad76963f65d3b84ff152159755a7efdcfe Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 19 Jun 2025 17:38:09 +0100
Subject: [PATCH 2/7] Added license to header.

---
 .../mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h       | 8 ++++++++
 mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp     | 1 -
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
index ed65be9980408..96d5a352f54c8 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -1,3 +1,11 @@
+//===-- ComplexToROCDL.h - conversion from Complex to ROCDL calls ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
 #ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
 #define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
 
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index cdfe2a6dfe874..7c7510b5c4e10 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
-
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"

>From bb88939bc53d9e88973749c63ba21dfa26279258 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Fri, 20 Jun 2025 16:22:07 +0100
Subject: [PATCH 3/7] Address reviewer changes. Add conversion for complex.exp.

---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 14 ++---
 .../ComplexToROCDL/ComplexToROCDL.h           |  3 +-
 .../Conversion/ComplexToROCDL/CMakeLists.txt  |  3 -
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 57 ++++++++++---------
 .../ComplexToROCDL/complex-to-rocdl.mlir      | 19 ++++++-
 5 files changed, 54 insertions(+), 42 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index f721b6232b0fb..b8c7cba80d863 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -4099,7 +4099,7 @@ class FIRToLLVMLowering
     // conversions that affect the ModuleOp, e.g. create new
     // function operations in it. We have to run such conversions
     // as passes here.
-    mlir::OpPassManager mathConvertionPM("builtin.module");
+    mlir::OpPassManager mathConversionPM("builtin.module");
 
     bool isAMDGCN = fir::getTargetTriple(mod).isAMDGCN();
     // If compiling for AMD target some math operations must be lowered to AMD
@@ -4107,8 +4107,8 @@ class FIRToLLVMLowering
     // is handled in the mathToLLVM conversion. The lowering to libm calls is
     // not needed since all math operations are handled this way.
     if (isAMDGCN) {
-      mathConvertionPM.addPass(mlir::createConvertMathToROCDL());
-      mathConvertionPM.addPass(mlir::createConvertComplexToROCDL());
+      mathConversionPM.addPass(mlir::createConvertMathToROCDL());
+      mathConversionPM.addPass(mlir::createConvertComplexToROCDL());
     }
 
     // Convert math::FPowI operations to inline implementation
@@ -4116,15 +4116,15 @@ class FIRToLLVMLowering
     // it will be lowered to LLVM intrinsic operation by a later conversion.
     mlir::ConvertMathToFuncsOptions mathToFuncsOptions{};
     mathToFuncsOptions.minWidthOfFPowIExponent = 33;
-    mathConvertionPM.addPass(
+    mathConversionPM.addPass(
         mlir::createConvertMathToFuncs(mathToFuncsOptions));
-    mathConvertionPM.addPass(mlir::createConvertComplexToStandardPass());
+    mathConversionPM.addPass(mlir::createConvertComplexToStandardPass());
     // Convert Math dialect operations into LLVM dialect operations.
     // There is no way to prefer MathToLLVM patterns over MathToLibm
     // patterns (applied below), so we have to run MathToLLVM conversion here.
-    mathConvertionPM.addNestedPass<mlir::func::FuncOp>(
+    mathConversionPM.addNestedPass<mlir::func::FuncOp>(
         mlir::createConvertMathToLLVMPass());
-    if (mlir::failed(runPipeline(mathConvertionPM, mod)))
+    if (mlir::failed(runPipeline(mathConversionPM, mod)))
       return signalPassFailure();
 
     std::optional<mlir::DataLayout> dl =
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
index 96d5a352f54c8..eb785080adab3 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -20,8 +20,7 @@ class RewritePatternSet;
 
 /// Populate the given list with patterns that convert from Complex to ROCDL
 /// calls.
-void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
-                                              PatternBenefit benefit);
+void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns);
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
index 54607250083d7..133809ac32f0f 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
+++ b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
@@ -7,9 +7,6 @@ add_mlir_conversion_library(MLIRComplexToROCDL
   DEPENDS
   MLIRConversionPassIncGen
 
-  LINK_COMPONENTS
-  Core
-
   LINK_LIBS PUBLIC
   MLIRComplexDialect
   MLIRFuncDialect
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index 7c7510b5c4e10..98adb9fb1f607 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -11,7 +11,6 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include <optional>
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
@@ -21,36 +20,38 @@ namespace mlir {
 using namespace mlir;
 
 namespace {
-struct FloatTypeResolver {
-  std::optional<bool> operator()(Type type) const {
-    auto elementType = cast<FloatType>(type);
-    if (!isa<Float32Type, Float64Type>(elementType))
-      return {};
-    return elementType.getIntOrFloatBitWidth() == 64;
-  }
-};
 
-template <typename Op, typename TypeResolver = FloatTypeResolver>
-struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
+template <typename Op>
+// Pattern to convert Complex ops to ROCDL function calls.
+struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ScalarOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
-                      StringRef doubleFunc, PatternBenefit benefit)
+  ComplexOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
+                       StringRef doubleFunc, PatternBenefit benefit = 1)
       : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
         doubleFunc(doubleFunc) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
-    auto module = SymbolTable::getNearestSymbolTable(op);
-    auto isDouble = TypeResolver()(op.getType());
-    if (!isDouble.has_value())
+    Operation *symTable = SymbolTable::getNearestSymbolTable(op);
+    Type resType = op.getType();
+    if (auto complexType = dyn_cast<ComplexType>(resType))
+      resType = complexType.getElementType();
+    FloatType floatTy = dyn_cast<FloatType>(resType);
+    if (!floatTy)
       return failure();
 
-    auto name = *isDouble ? doubleFunc : floatFunc;
+    StringRef name;
+    if (floatTy.isF64())
+      name = doubleFunc;
+    else if (floatTy.isF32())
+      name = floatFunc;
+    else
+      return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
-        SymbolTable::lookupSymbolIn(module, name));
+        SymbolTable::lookupSymbolIn(symTable, name));
     if (!opFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
-      rewriter.setInsertionPointToStart(&module->getRegion(0).front());
+      rewriter.setInsertionPointToStart(&symTable->getRegion(0).front());
       auto funcTy = FunctionType::get(
           rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
       opFunc =
@@ -67,10 +68,12 @@ struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
 };
 } // namespace
 
-void mlir::populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
-                                                    PatternBenefit benefit) {
-  patterns.add<ScalarOpToROCDLCall<complex::AbsOp>>(
-      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64", benefit);
+void mlir::populateComplexToROCDLConversionPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp>>(
+      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp>>(
+      patterns.getContext(), "__ocml_cexp_f32", "__ocml_cexp_f64");
 }
 
 namespace {
@@ -81,14 +84,14 @@ struct ConvertComplexToROCDLPass
 } // namespace
 
 void ConvertComplexToROCDLPass::runOnOperation() {
-  auto module = getOperation();
+  Operation *op = getOperation();
 
   RewritePatternSet patterns(&getContext());
-  populateComplexToROCDLConversionPatterns(patterns, /*benefit=*/1);
+  populateComplexToROCDLConversionPatterns(patterns);
 
   ConversionTarget target(getContext());
   target.addLegalDialect<func::FuncDialect>();
-  target.addIllegalOp<complex::AbsOp>();
-  if (failed(applyPartialConversion(module, target, std::move(patterns))))
+  target.addIllegalOp<complex::AbsOp, complex::ExpOp>();
+  if (failed(applyPartialConversion(op, target, std::move(patterns))))
     signalPassFailure();
 }
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
index 618e9c238378c..23631e25e4588 100644
--- a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
+++ b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
@@ -1,13 +1,26 @@
-// RUN: mlir-opt %s -convert-complex-to-rocdl -canonicalize | FileCheck %s
+// RUN: mlir-opt %s -convert-complex-to-rocdl | FileCheck %s
 
 // CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
 // CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64
+// CHECK-DAG: @__ocml_cexp_f32(complex<f32>) -> complex<f32>
+// CHECK-DAG: @__ocml_cexp_f64(complex<f64>) -> complex<f64>
 
+//CHECK-LABEL: @abs_caller
 func.func @abs_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
-  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%[[F:.*]])
+  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%{{.*}})
   %rf = complex.abs %f : complex<f32>
-  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%[[D:.*]])
+  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%{{.*}})
   %rd = complex.abs %d : complex<f64>
   // CHECK: return %[[RF]], %[[RD]]
   return %rf, %rd : f32, f64
 }
+
+//CHECK-LABEL: @exp_caller
+func.func @exp_caller(%f: complex<f32>, %d: complex<f64>) -> (complex<f32>, complex<f64>) {
+  // CHECK: %[[EF:.*]] = call @__ocml_cexp_f32(%{{.*}})
+  %ef = complex.exp %f : complex<f32>
+  // CHECK: %[[ED:.*]] = call @__ocml_cexp_f64(%{{.*}})
+  %ed = complex.exp %d : complex<f64>
+  // CHECK: return %[[EF]], %[[ED]]
+  return %ef, %ed : complex<f32>, complex<f64>
+}

>From d70bca22bae1d66f126b5a1832153810baf06ab4 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Fri, 20 Jun 2025 18:30:57 +0100
Subject: [PATCH 4/7] Correct alphabetical order for cmake.

---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 8b4ac18fba527..de8e1c5c3fa3f 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -34,13 +34,13 @@ add_flang_library(FIRCodeGen
 
   MLIR_LIBS
   MLIRComplexToLLVM
+  MLIRComplexToROCDL
   MLIRComplexToStandard
   MLIRGPUDialect
   MLIRMathToFuncs
   MLIRMathToLLVM
   MLIRMathToLibm
   MLIRMathToROCDL
-  MLIRComplexToROCDL
   MLIROpenMPToLLVM
   MLIROpenACCDialect
   MLIRBuiltinToLLVMIRTranslation

>From 0e9aef2910fee55664f7bbfd54ec7bbb1ee383f3 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Tue, 24 Jun 2025 16:34:11 +0100
Subject: [PATCH 5/7] Add FloatTy as a template parameter.

---
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 42 ++++++++-----------
 1 file changed, 18 insertions(+), 24 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index 98adb9fb1f607..cfad9f5f6fa19 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -21,59 +21,53 @@ using namespace mlir;
 
 namespace {
 
-template <typename Op>
+template <typename Op, typename Ty>
 // Pattern to convert Complex ops to ROCDL function calls.
 struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ComplexOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
-                       StringRef doubleFunc, PatternBenefit benefit = 1)
-      : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
-        doubleFunc(doubleFunc) {}
+  ComplexOpToROCDLCall(MLIRContext *context, StringRef funcName,
+                       PatternBenefit benefit = 1)
+      : OpRewritePattern<Op>(context, benefit), funcName(funcName) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
     Operation *symTable = SymbolTable::getNearestSymbolTable(op);
     Type resType = op.getType();
     if (auto complexType = dyn_cast<ComplexType>(resType))
       resType = complexType.getElementType();
-    FloatType floatTy = dyn_cast<FloatType>(resType);
-    if (!floatTy)
-      return failure();
-
-    StringRef name;
-    if (floatTy.isF64())
-      name = doubleFunc;
-    else if (floatTy.isF32())
-      name = floatFunc;
-    else
+    if (!isa<Ty>(resType))
       return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
-        SymbolTable::lookupSymbolIn(symTable, name));
+        SymbolTable::lookupSymbolIn(symTable, funcName));
     if (!opFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(&symTable->getRegion(0).front());
       auto funcTy = FunctionType::get(
           rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-      opFunc =
-          rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name, funcTy);
+      opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), funcName,
+                                             funcTy);
       opFunc.setPrivate();
     }
-    rewriter.replaceOpWithNewOp<func::CallOp>(op, name, op.getType(),
+    rewriter.replaceOpWithNewOp<func::CallOp>(op, funcName, op.getType(),
                                               op->getOperands());
     return success();
   }
 
 private:
-  std::string floatFunc, doubleFunc;
+  std::string funcName;
 };
 } // namespace
 
 void mlir::populateComplexToROCDLConversionPatterns(
     RewritePatternSet &patterns) {
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp>>(
-      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp>>(
-      patterns.getContext(), "__ocml_cexp_f32", "__ocml_cexp_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float32Type>>(
+      patterns.getContext(), "__ocml_cabs_f32");
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float64Type>>(
+      patterns.getContext(), "__ocml_cabs_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float32Type>>(
+      patterns.getContext(), "__ocml_cexp_f32");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float64Type>>(
+      patterns.getContext(), "__ocml_cexp_f64");
 }
 
 namespace {

>From 03e439e94408ae4f6dafe0cbc8f8dfad6a80dc54 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 2 Jul 2025 14:48:34 +0100
Subject: [PATCH 6/7] Rename to ComplexToROCDLLibraryCalls

---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt    |  2 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  4 +--
 .../ComplexToROCDLLibraryCalls.h}             | 14 ++++----
 mlir/include/mlir/Conversion/Passes.h         |  2 +-
 mlir/include/mlir/Conversion/Passes.td        |  6 ++--
 mlir/lib/Conversion/CMakeLists.txt            |  2 +-
 .../Conversion/ComplexToROCDL/CMakeLists.txt  | 15 --------
 .../ComplexToROCDLLibraryCalls/CMakeLists.txt | 18 ++++++++++
 .../ComplexToROCDLLibraryCalls.cpp}           | 35 ++++++++++---------
 .../complex-to-rocdl-library-calls.mlir}      |  2 +-
 10 files changed, 52 insertions(+), 48 deletions(-)
 rename mlir/include/mlir/Conversion/{ComplexToROCDL/ComplexToROCDL.h => ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h} (52%)
 delete mode 100644 mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
 create mode 100644 mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
 rename mlir/lib/Conversion/{ComplexToROCDL/ComplexToROCDL.cpp => ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp} (68%)
 rename mlir/test/Conversion/{ComplexToROCDL/complex-to-rocdl.mlir => ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir} (92%)

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index de8e1c5c3fa3f..16c7944a885a1 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -34,7 +34,7 @@ add_flang_library(FIRCodeGen
 
   MLIR_LIBS
   MLIRComplexToLLVM
-  MLIRComplexToROCDL
+  MLIRComplexToROCDLLibraryCalls
   MLIRComplexToStandard
   MLIRGPUDialect
   MLIRMathToFuncs
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index b8c7cba80d863..a536b3f13e997 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -33,7 +33,7 @@
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
@@ -4108,7 +4108,7 @@ class FIRToLLVMLowering
     // not needed since all math operations are handled this way.
     if (isAMDGCN) {
       mathConversionPM.addPass(mlir::createConvertMathToROCDL());
-      mathConversionPM.addPass(mlir::createConvertComplexToROCDL());
+      mathConversionPM.addPass(mlir::createConvertComplexToROCDLLibraryCalls());
     }
 
     // Convert math::FPowI operations to inline implementation
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
similarity index 52%
rename from mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
rename to mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
index eb785080adab3..409b87747f437 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
@@ -1,4 +1,4 @@
-//===-- ComplexToROCDL.h - conversion from Complex to ROCDL calls ---------===//
+//===- ComplexToROCDLLibraryCalls.h - convert from Complex to ROCDL calls -===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,21 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
-#define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#ifndef MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
+#define MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
 
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 class RewritePatternSet;
 
-#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDL
+#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDLLIBRARYCALLS
 #include "mlir/Conversion/Passes.h.inc"
 
 /// Populate the given list with patterns that convert from Complex to ROCDL
 /// calls.
-void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns);
+void populateComplexToROCDLLibraryCallsConversionPatterns(
+    RewritePatternSet &patterns);
 } // namespace mlir
 
-#endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#endif // MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 67e8f5b99b67b..896fff277679e 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -23,7 +23,7 @@
 #include "mlir/Conversion/BufferizationToMemRef/BufferizationToMemRef.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
 #include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h"
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRVPass.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 8ad2341f93a15..b85afb0ae97be 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -313,11 +313,11 @@ def ConvertComplexToLibm : Pass<"convert-complex-to-libm", "ModuleOp"> {
 }
 
 //===----------------------------------------------------------------------===//
-// ComplexToROCDL
+// ComplexToROCDLLibraryCalls
 //===----------------------------------------------------------------------===//
 
-def ConvertComplexToROCDL : Pass<"convert-complex-to-rocdl", "ModuleOp"> {
-  let summary = "Convert Complex dialect to ROCDL calls";
+def ConvertComplexToROCDLLibraryCalls : Pass<"convert-complex-to-rocdl-library-calls", "ModuleOp"> {
+  let summary = "Convert Complex dialect to ROCDL library calls";
   let description = [{
     This pass converts supported Complex ops to calls to the AMD device library.
   }];
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index 4ad81553a4fa8..b731662dcebc7 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -13,7 +13,7 @@ add_subdirectory(AsyncToLLVM)
 add_subdirectory(BufferizationToMemRef)
 add_subdirectory(ComplexCommon)
 add_subdirectory(ComplexToLibm)
-add_subdirectory(ComplexToROCDL)
+add_subdirectory(ComplexToROCDLLibraryCalls)
 add_subdirectory(ComplexToLLVM)
 add_subdirectory(ComplexToSPIRV)
 add_subdirectory(ComplexToStandard)
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
deleted file mode 100644
index 133809ac32f0f..0000000000000
--- a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-add_mlir_conversion_library(MLIRComplexToROCDL
-  ComplexToROCDL.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDL
-
-  DEPENDS
-  MLIRConversionPassIncGen
-
-  LINK_LIBS PUBLIC
-  MLIRComplexDialect
-  MLIRFuncDialect
-  MLIRPass
-  MLIRTransformUtils
-  )
diff --git a/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
new file mode 100644
index 0000000000000..695bb2dd0a82c
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_conversion_library(MLIRComplexToROCDLLibraryCalls
+  ComplexToROCDLLibraryCalls.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDLLibraryCalls
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRComplexDialect
+  MLIRFuncDialect
+  MLIRPass
+  MLIRTransformUtils
+  )
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
similarity index 68%
rename from mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
rename to mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
index cfad9f5f6fa19..99d5424aef79a 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
@@ -1,4 +1,4 @@
-//===-- ComplexToROCDL.cpp - conversion from Complex to ROCDL calls -------===//
+//=== ComplexToROCDLLibraryCalls.cpp - convert from Complex to ROCDL calls ===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
-#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
+#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDLLIBRARYCALLS
 #include "mlir/Conversion/Passes.h.inc"
 } // namespace mlir
 
@@ -21,12 +21,12 @@ using namespace mlir;
 
 namespace {
 
-template <typename Op, typename Ty>
+template <typename Op, typename FloatTy>
 // Pattern to convert Complex ops to ROCDL function calls.
-struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
+struct ComplexOpToROCDLLibraryCalls : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ComplexOpToROCDLCall(MLIRContext *context, StringRef funcName,
-                       PatternBenefit benefit = 1)
+  ComplexOpToROCDLLibraryCalls(MLIRContext *context, StringRef funcName,
+                               PatternBenefit benefit = 1)
       : OpRewritePattern<Op>(context, benefit), funcName(funcName) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
@@ -34,7 +34,7 @@ struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
     Type resType = op.getType();
     if (auto complexType = dyn_cast<ComplexType>(resType))
       resType = complexType.getElementType();
-    if (!isa<Ty>(resType))
+    if (!isa<FloatTy>(resType))
       return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
@@ -58,30 +58,31 @@ struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
 };
 } // namespace
 
-void mlir::populateComplexToROCDLConversionPatterns(
+void mlir::populateComplexToROCDLLibraryCallsConversionPatterns(
     RewritePatternSet &patterns) {
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float32Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float32Type>>(
       patterns.getContext(), "__ocml_cabs_f32");
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float64Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float64Type>>(
       patterns.getContext(), "__ocml_cabs_f64");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float32Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ExpOp, Float32Type>>(
       patterns.getContext(), "__ocml_cexp_f32");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float64Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ExpOp, Float64Type>>(
       patterns.getContext(), "__ocml_cexp_f64");
 }
 
 namespace {
-struct ConvertComplexToROCDLPass
-    : public impl::ConvertComplexToROCDLBase<ConvertComplexToROCDLPass> {
+struct ConvertComplexToROCDLLibraryCallsPass
+    : public impl::ConvertComplexToROCDLLibraryCallsBase<
+          ConvertComplexToROCDLLibraryCallsPass> {
   void runOnOperation() override;
 };
 } // namespace
 
-void ConvertComplexToROCDLPass::runOnOperation() {
+void ConvertComplexToROCDLLibraryCallsPass::runOnOperation() {
   Operation *op = getOperation();
 
   RewritePatternSet patterns(&getContext());
-  populateComplexToROCDLConversionPatterns(patterns);
+  populateComplexToROCDLLibraryCallsConversionPatterns(patterns);
 
   ConversionTarget target(getContext());
   target.addLegalDialect<func::FuncDialect>();
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
similarity index 92%
rename from mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
rename to mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
index 23631e25e4588..bae7c5986ef9e 100644
--- a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
+++ b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-complex-to-rocdl | FileCheck %s
+// RUN: mlir-opt %s -convert-complex-to-rocdl-library-calls | FileCheck %s
 
 // CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
 // CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64

>From ec59acc25579c0e961a71a20c3ca4ed192a1feb6 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 2 Jul 2025 16:43:10 +0100
Subject: [PATCH 7/7] Fix build error.

---
 .../ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
index 409b87747f437..daac2a99ed80f 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
@@ -10,6 +10,7 @@
 #define MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
 
 #include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 class RewritePatternSet;


From flang-commits at lists.llvm.org  Wed Jul  2 09:25:47 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Wed, 02 Jul 2025 09:25:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146543@github.com>
Message-ID: <68655d8b.170a0220.6ca89.0402@mx.google.com>


https://github.com/abidh edited https://github.com/llvm/llvm-project/pull/146543

From flang-commits at lists.llvm.org  Wed Jul  2 09:25:47 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Wed, 02 Jul 2025 09:25:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146543@github.com>
Message-ID: <68655d8b.170a0220.2bee0f.3c8e@mx.google.com>


https://github.com/abidh approved this pull request.

Thanks for the PR. It looks good to me apart from the following 2 points.

It took me sometime to get my head around it. It will be helpful for anybody working on in the future if you can elaborate a bit more how it works. May be take the comments in `debug-cyclic-derived-type-caching-complex.f90`, adds in some more details and make that part of the commit message.

Also when I looked at this part last year, I had a quite smallish patch that removed the limitation from the `mlir` about recursive types. I can't remember why I did not pursue it  further but I think this is one thing we need to explore a bit. It may be much simpler to remove that limitation in `mlir` than to workaround it in `flang`. 

https://github.com/llvm/llvm-project/pull/146543

From flang-commits at lists.llvm.org  Wed Jul  2 09:25:47 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Wed, 02 Jul 2025 09:25:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146543@github.com>
Message-ID: <68655d8b.050a0220.7f47b.a3fb@mx.google.com>


================
@@ -272,31 +271,127 @@ DebugTypeGenerator::getFieldSizeAndAlign(mlir::Type fieldTy) {
   return std::pair{byteSize, byteAlign};
 }
 
+mlir::LLVM::DITypeAttr DerivedTypeCache::lookup(mlir::Type type) {
+  auto iter = typeCache.find(type);
+  if (iter != typeCache.end()) {
+    if (iter->second.first) {
+      componentActiveRecursionLevels = iter->second.second;
+    }
+    return iter->second.first;
+  }
+  return nullptr;
+}
+
+DerivedTypeCache::ActiveLevels
+DerivedTypeCache::startTranslating(mlir::Type type,
+                                   mlir::LLVM::DITypeAttr placeHolder) {
+  derivedTypeDepth++;
+  if (!placeHolder)
+    return {};
+  typeCache[type] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      placeHolder, {derivedTypeDepth});
+  return {};
+}
----------------
abidh wrote:

This function always returns an empty list. 

https://github.com/llvm/llvm-project/pull/146543

From flang-commits at lists.llvm.org  Wed Jul  2 09:26:31 2025
From: flang-commits at lists.llvm.org (Anchu Rajendran S via flang-commits)
Date: Wed, 02 Jul 2025 09:26:31 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <68655db7.050a0220.2d1151.918b@mx.google.com>


https://github.com/anchuraj updated https://github.com/llvm/llvm-project/pull/143441

>From 55e513da2cb2d68dfb91e889bb66b7fdd58090ac Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 1/5] [flang][flang-driver] atomic control support

---
 clang/include/clang/Driver/Options.td         | 22 +++++-----
 flang/include/flang/Frontend/TargetOptions.h  |  5 +++
 .../Optimizer/Dialect/Support/FIRContext.h    | 14 +++++++
 flang/lib/Frontend/CompilerInvocation.cpp     | 12 ++++++
 flang/lib/Lower/Bridge.cpp                    |  6 +++
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 40 +++++++++++++++++++
 .../Lower/OpenMP/atomic-control-options.f90   | 37 +++++++++++++++++
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 14 +++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  8 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  9 +++++
 10 files changed, 153 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/atomic-control-options.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 0822df2640d23..300b9c0e27ba9 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2303,21 +2303,21 @@ def fsymbol_partition_EQ : Joined<["-"], "fsymbol-partition=">, Group<f_Group>,
 
 defm atomic_remote_memory : BoolFOption<"atomic-remote-memory",
   LangOpts<"AtomicRemoteMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on remote memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on remote memory">>;
 
 defm atomic_fine_grained_memory : BoolFOption<"atomic-fine-grained-memory",
   LangOpts<"AtomicFineGrainedMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on fine-grained memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on fine-grained memory">>;
 
 defm atomic_ignore_denormal_mode : BoolFOption<"atomic-ignore-denormal-mode",
   LangOpts<"AtomicIgnoreDenormalMode">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "Allow">,
-  NegFlag<SetFalse, [], [ClangOption], "Disallow">,
-  BothFlags<[], [ClangOption], " atomic operations to ignore denormal mode">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "Allow">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Disallow">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations to ignore denormal mode">>;
 
 defm memory_profile : OptInCC1FFlag<"memory-profile", "Enable", "Disable", " heap memory profiling">;
 def fmemory_profile_EQ : Joined<["-"], "fmemory-profile=">,
@@ -5305,9 +5305,9 @@ defm amdgpu_precise_memory_op
                   " precise memory mode (AMDGPU only)">;
 
 def munsafe_fp_atomics : Flag<["-"], "munsafe-fp-atomics">,
-  Visibility<[ClangOption, CC1Option]>, Alias<fatomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>, Alias<fatomic_ignore_denormal_mode>;
 def mno_unsafe_fp_atomics : Flag<["-"], "mno-unsafe-fp-atomics">,
-  Visibility<[ClangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, FlangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
 
 def faltivec : Flag<["-"], "faltivec">, Group<f_Group>;
 def fno_altivec : Flag<["-"], "fno-altivec">, Group<f_Group>;
diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 002d8d158abd4..26256fd775f11 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic Control Options for AMD GPU
+  bool amdgpuIgnoreDenormalMode = false;
+  bool amdgpuRemoteMemory = false;
+  bool amdgpuFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 2df14f83c11e1..ac563bcc402c7 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,10 +58,24 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and Getters for atomic control options
+void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
+bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 766131fd69d85..240b95a1fafb4 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -495,6 +495,18 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
+  if (llvm::Triple(opts.triple).isAMDGPU()) {
+    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
+        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+    opts.amdgpuFineGrainedMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_fine_grained_memory,
+        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+    opts.amdgpuRemoteMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_remote_memory,
+        clang::driver::options::OPT_fno_atomic_remote_memory, false);
+  }
+
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
     opts.cpu = a->getValue();
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 8506b9a984e58..f1bcb3c6e080c 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6654,6 +6654,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
+  if (targetOpts.amdgpuIgnoreDenormalMode)
+    fir::setAmdgpuIgnoreDenormalMode(*module);
+  if (targetOpts.amdgpuFineGrainedMemory)
+    fir::setAmdgpuFineGrainedMemory(*module);
+  if (targetOpts.amdgpuRemoteMemory)
+    fir::setAmdgpuRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 01c0be66d1ecc..b961793dbdfd5 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,6 +88,46 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *amdgpuIgnoreDenormalModeName =
+    "fir.amdgpu.ignore.denormal.mode";
+void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
+    return true;
+  return false;
+}
+
+static constexpr const char *amdgpuFineGrainedMemoryName =
+    "fir.amdgpu.fine.grained.memory";
+void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
+    return true;
+  return false;
+}
+static constexpr const char *amdgpuRemoteMemoryName =
+    "fir.amdgpu.remote.memory";
+void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
+    return true;
+  return false;
+}
+
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
   if (auto attr = mod->getAttrOfType<mlir::StringAttr>(tuneCpuName))
     return attr.getValue();
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
new file mode 100644
index 0000000000000..1eb0f617f365e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o - | FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o - | FileCheck -check-prefix=IGNORE-DENORMAL %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o - | FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o - | FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, B, threads
+    threads = 128
+    A = 0
+    B = 0
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic capture
+        A = A + B
+        B = A
+    !$omp end atomic
+    !$omp end target parallel
+end program test
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 704d0b2220e8a..84887dfd2c6f9 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -54,6 +54,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
+      clausesArgs);
 
   // Override region definition.
   let regions = (region SizedRegion<1>:$region);
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..e377b7aab3e43 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1562,6 +1562,15 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
     omp.yield(%newval : i32)
   }
 
+  // CHECK: omp.atomic.update %[[X]] : memref<i32> {
+  // CHECK-NEXT: (%[[XVAL:.*]]: i32):
+  // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
+  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  omp.atomic.update %x : memref<i32> {
+  ^bb0(%xval:i32):
+    omp.yield(%const:i32)
+  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+
   return
 }
 

>From 0d15bdb469d3be4764a79b32f05a7e2bdf66666a Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:58:07 -0500
Subject: [PATCH 2/5] R2: Making Atomic control non-optional

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   2 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |   2 +-
 mlir/test/Dialect/OpenMP/canonicalize.mlir    |  10 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  46 +++----
 mlir/test/Dialect/OpenMP/ops.mlir             | 127 +++++++++---------
 .../Target/LLVMIR/openmp-llvm-invalid.mlir    |   8 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  98 +++++++-------
 .../openmp-target-nesting-in-host-ops.mlir    |   2 +-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |   2 +-
 9 files changed, 148 insertions(+), 149 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 375e35a146f78..399f7fa6e78f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          OptionalAttr<AtomicControlAttr>:$atomic_control),
+          AtomicControlAttr:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index d69de998346b5..0b2445d0a0d18 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -133,7 +133,7 @@ func.func @atomic_update() {
     %1 = arith.constant 1 : i32
     %2 = arith.addi %arg0, %1  : i32
     omp.yield(%2 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 llvm.mlir.global internal @_QFsEc() : i32 {
diff --git a/mlir/test/Dialect/OpenMP/canonicalize.mlir b/mlir/test/Dialect/OpenMP/canonicalize.mlir
index de6c931ecc5fd..2492010c4d3c7 100644
--- a/mlir/test/Dialect/OpenMP/canonicalize.mlir
+++ b/mlir/test/Dialect/OpenMP/canonicalize.mlir
@@ -4,7 +4,7 @@ func.func @update_no_op(%x : memref<i32>) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -17,7 +17,7 @@ func.func @update_write_op(%x : memref<i32>, %value: i32) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%value : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -33,7 +33,7 @@ func.func @update_normal(%x : memref<i32>, %value: i32) {
   ^bb0(%xval : i32):
     %newval = arith.addi %xval, %value : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -50,7 +50,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.addi %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -65,7 +65,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.muli %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..afc3830286f30 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1007,7 +1007,7 @@ func.func @omp_atomic_update1(%x: memref<i32>, %expr: f32) {
   ^bb0(%xval: f32):
     %newval = llvm.fadd %xval, %expr : f32
     omp.yield (%newval : f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1020,7 +1020,7 @@ func.func @omp_atomic_update2(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.terminator
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1032,7 +1032,7 @@ func.func @omp_atomic_update3(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1044,7 +1044,7 @@ func.func @omp_atomic_update4(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1056,7 +1056,7 @@ func.func @omp_atomic_update5(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1068,7 +1068,7 @@ func.func @omp_atomic_update6(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval, %expr : i32, i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1080,7 +1080,7 @@ func.func @omp_atomic_update7(%x: memref<i32>, %expr: i32, %y: f32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%y: f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1092,7 +1092,7 @@ func.func @omp_atomic_update8(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32, %tmp: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1104,7 +1104,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1116,7 +1116,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1128,7 +1128,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1164,12 +1164,12 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.update %x : memref<i32> {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1197,7 +1197,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1212,7 +1212,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.write %x = %expr : memref<i32>, i32
     omp.terminator
   }
@@ -1240,7 +1240,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %y : memref<i32>, memref<i32>, i32
     omp.terminator
   }
@@ -1256,7 +1256,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
 }
@@ -1281,7 +1281,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1296,7 +1296,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1311,7 +1311,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1326,7 +1326,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1341,7 +1341,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1356,7 +1356,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x memory_order(seq_cst) : memref<i32>, memref<i32>, i32
   }
   return
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index e377b7aab3e43..5aa7befb2dc26 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
@@ -1380,57 +1380,57 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i1):
     %newval = llvm.and %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.shl %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.intr.smax(%[[XVAL]], %[[EXPR]]) : (i32, i32) -> i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.intr.smax(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.icmp "eq" %[[XVAL]], %[[EXPRBOOL]] : i1
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i1)
-  // }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %xBool : memref<i1> {
   ^bb0(%xval: i1):
     %newval = llvm.icmp "eq" %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%xval:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   %const = arith.constant 42 : i32
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1440,7 +1440,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1450,7 +1450,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1460,7 +1460,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1470,7 +1470,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1480,7 +1480,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1490,7 +1490,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1500,7 +1500,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1510,7 +1510,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1520,7 +1520,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1530,7 +1530,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(release) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1540,7 +1540,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(relaxed) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1550,7 +1550,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1560,17 +1560,16 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
-
+  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   return
 }
 
@@ -1582,7 +1581,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture{
@@ -1590,7 +1589,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   // CHECK: omp.atomic.capture {
@@ -1599,7 +1598,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: }
   omp.atomic.capture{
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
@@ -1607,7 +1606,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
@@ -1623,7 +1622,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(none) {
@@ -1631,7 +1630,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1640,7 +1639,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended) {
@@ -1648,7 +1647,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1657,7 +1656,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended) {
@@ -1665,7 +1664,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1674,7 +1673,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(nonspeculative) {
@@ -1682,7 +1681,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1691,7 +1690,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(speculative) {
@@ -1699,7 +1698,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1708,7 +1707,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, nonspeculative) {
@@ -1716,7 +1715,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1725,7 +1724,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, nonspeculative) {
@@ -1733,7 +1732,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1742,7 +1741,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, speculative) {
@@ -1750,7 +1749,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1759,7 +1758,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) {
@@ -1767,7 +1766,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1776,7 +1775,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(seq_cst) {
@@ -1784,7 +1783,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1793,7 +1792,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acq_rel) {
@@ -1801,7 +1800,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1810,7 +1809,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acquire) {
@@ -1818,7 +1817,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1827,7 +1826,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(release) {
@@ -1835,7 +1834,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1844,7 +1843,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(relaxed) {
@@ -1852,7 +1851,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1861,7 +1860,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) memory_order(seq_cst) {
@@ -1869,7 +1868,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -2653,14 +2652,14 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
   // CHECK: omp.atomic.update %[[x]] : !llvm.ptr {
   // CHECK-NEXT: ^{{[[:alnum:]]+}}(%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   %val = llvm.load %x : !llvm.ptr -> i32
   omp.atomic.write %v = %val : !llvm.ptr, i32
   omp.atomic.update %x : !llvm.ptr {
     ^bb0(%xval: i32):
       omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index 41bc5c4ba525f..00f53864b5fa7 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -9,7 +9,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
     %t2 = llvm.sdiv %t1, %expr : i32
     %newval = llvm.add %xval, %t2 : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -24,7 +24,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.mul %expr, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -41,7 +41,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
     ^bb0(%xval: i32):
       %newval = llvm.mul %expr, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
@@ -59,7 +59,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
       %t2 = llvm.sdiv %t1, %expr : i32
       %newval = llvm.add %xval, %t2 : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 32f0ba5b105ff..ab7f756bf95f9 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,13 +1462,13 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1563,7 +1563,7 @@ llvm.func @_QPomp_atomic_update_complex() {
       %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
       %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
       omp.yield(%16 : !llvm.struct<(f32, f32)>)
-    }
+    } {atomic_control = #omp.atomic_control<>}
    llvm.return
 }
 
@@ -1623,7 +1623,7 @@ llvm.func @_QPomp_atomic_capture_complex() {
         %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
         %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
         omp.yield(%20 : !llvm.struct<(f32, f32)>)
-      }
+      } {atomic_control = #omp.atomic_control<>}
       omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.ptr, !llvm.struct<(f32, f32)>
     }
     llvm.return
@@ -1665,7 +1665,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %expr, %xval : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1683,7 +1683,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1701,7 +1701,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
   // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
   // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
@@ -1710,7 +1710,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1732,7 +1732,7 @@ llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
     %2 = llvm.fadd %1, %0 : f32
     %3 = llvm.fptosi %2 : f32 to i32
     omp.yield(%3 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1751,7 +1751,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1763,7 +1763,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1775,7 +1775,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1787,7 +1787,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1799,7 +1799,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1814,7 +1814,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1829,7 +1829,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1844,7 +1844,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1859,7 +1859,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1874,7 +1874,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1889,7 +1889,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1904,7 +1904,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1919,7 +1919,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1934,7 +1934,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1949,7 +1949,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1964,7 +1964,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -1979,7 +1979,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -2001,7 +2001,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw sub ptr %[[x]], i32 %[[expr]] monotonic
@@ -2012,7 +2012,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw and ptr %[[x]], i32 %[[expr]] monotonic
@@ -2023,7 +2023,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw or ptr %[[x]], i32 %[[expr]] monotonic
@@ -2034,7 +2034,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw xor ptr %[[x]], i32 %[[expr]] monotonic
@@ -2045,7 +2045,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2060,7 +2060,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2075,7 +2075,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2090,7 +2090,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2105,7 +2105,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2120,7 +2120,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2135,7 +2135,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2150,7 +2150,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2165,7 +2165,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2180,7 +2180,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2195,7 +2195,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2211,7 +2211,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2227,7 +2227,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -2265,7 +2265,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acquire
@@ -2276,7 +2276,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] release
@@ -2287,7 +2287,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
@@ -2298,7 +2298,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acq_rel
@@ -2309,7 +2309,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -3055,7 +3055,7 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
index cbf273b887bc7..a9a4ffd533136 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
@@ -110,7 +110,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
 
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 97608ca3b4df1..2f294658edb3d 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -19,7 +19,7 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   ^bb0(%arg0: i32):
     %result = llvm.add %arg0, %expr : i32
     omp.yield(%result : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   llvm.return
 }

>From 0b6fd8405623d5192f9d7c22d2470c43be5b3bdf Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 3/5] R3: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h       |  2 +-
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  4 ++--
 flang/lib/Optimizer/Dialect/Support/FIRContext.cpp | 14 +++-----------
 flang/test/Fir/convert-to-llvm-openmp-and-fir.fir  |  4 ++--
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td |  3 ++-
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 26256fd775f11..dd29b4e3e8b08 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,7 +54,7 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic Control Options for AMD GPU
+  /// Atomic control options for AMD gpu
   bool amdgpuIgnoreDenormalMode = false;
   bool amdgpuRemoteMemory = false;
   bool amdgpuFineGrainedMemory = false;
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac563bcc402c7..ac5fd33874db8 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,7 +58,7 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and Getters for atomic control options
+// Setters and getters for atomic control options.
 void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
@@ -70,7 +70,7 @@ bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index b961793dbdfd5..297a4c7a39e44 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -96,10 +96,7 @@ void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
 }
 
 static constexpr const char *amdgpuFineGrainedMemoryName =
@@ -110,10 +107,7 @@ void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
 }
 static constexpr const char *amdgpuRemoteMemoryName =
     "fir.amdgpu.remote.memory";
@@ -123,9 +117,7 @@ void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 24e5cad84b709..47c00d489c200 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -619,7 +619,7 @@ func.func @_QPs() {
     %c1_i32 = arith.constant 1 : i32
     %1 = arith.addi %arg0, %c1_i32 : i32
     omp.yield(%1 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 fir.global internal @_QFsEc : i32 {
@@ -634,7 +634,7 @@ fir.global internal @_QFsEc : i32 {
 // CHECK:      %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:      %[[OUT_VAL:.*]] = llvm.add %[[IN_VAL]], %[[CONST_1]]  : i32
 // CHECK:      omp.yield(%[[OUT_VAL]] : i32)
-// CHECK:    }
+// CHECK:    } {atomic_control = #omp.atomic_control<>}
 // CHECK:    llvm.return
 // CHECK:  }
 // CHECK:  llvm.mlir.global internal @[[GLOBAL]]() {{.*}} : i32 {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 84887dfd2c6f9..61861173ae875 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -58,7 +58,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From fe9a7022ec1f1c1065c782408b54dff5868072ce Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Mon, 23 Jun 2025 12:14:19 -0500
Subject: [PATCH 4/5] R4: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  6 ++--
 .../Optimizer/Dialect/Support/FIRContext.h    | 18 ++++------
 flang/lib/Frontend/CompilerInvocation.cpp     | 20 +++++------
 flang/lib/Lower/Bridge.cpp                    | 12 +++----
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  8 ++++-
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 34 +++++++++----------
 .../Lower/OpenMP/atomic-control-options.f90   | 16 ++++-----
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     |  6 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  4 +--
 9 files changed, 60 insertions(+), 64 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index dd29b4e3e8b08..6ef5338acdc6e 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -55,9 +55,9 @@ class TargetOptions {
   bool asmVerbose = false;
 
   /// Atomic control options for AMD gpu
-  bool amdgpuIgnoreDenormalMode = false;
-  bool amdgpuRemoteMemory = false;
-  bool amdgpuFineGrainedMemory = false;
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac5fd33874db8..273b4683b6d9c 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -59,23 +59,17 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
 // Setters and getters for atomic control options.
-void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
-bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+void setIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getIgnoreDenormalMode(mlir::ModuleOp mod);
+void setFineGrainedMemory(mlir::ModuleOp mod);
+bool getFineGrainedMemory(mlir::ModuleOp mod);
+void setRemoteMemory(mlir::ModuleOp mod);
+bool getRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set atomic control options for AMD gpu.
-void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
-                                   bool amdgpuIgnoreDenormalMode,
-                                   bool amdgpuNoFineGrainedMemory,
-                                   bool amdgpuNoRemoteMemory);
-
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 240b95a1fafb4..ea3d2698f4af6 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -495,17 +495,15 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  if (llvm::Triple(opts.triple).isAMDGPU()) {
-    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
-        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
-        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-    opts.amdgpuFineGrainedMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_fine_grained_memory,
-        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-    opts.amdgpuRemoteMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_remote_memory,
-        clang::driver::options::OPT_fno_atomic_remote_memory, false);
-  }
+  opts.ignoreDenormalMode = args.hasFlag(
+      clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+      clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+  opts.fineGrainedMemory = args.hasFlag(
+      clang::driver::options::OPT_fatomic_fine_grained_memory,
+      clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+  opts.remoteMemory =
+      args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
+                   clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index f1bcb3c6e080c..459c7b84e4f87 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6654,12 +6654,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.amdgpuIgnoreDenormalMode)
-    fir::setAmdgpuIgnoreDenormalMode(*module);
-  if (targetOpts.amdgpuFineGrainedMemory)
-    fir::setAmdgpuFineGrainedMemory(*module);
-  if (targetOpts.amdgpuRemoteMemory)
-    fir::setAmdgpuRemoteMemory(*module);
+  if (targetOpts.ignoreDenormalMode)
+    fir::setIgnoreDenormalMode(*module);
+  if (targetOpts.fineGrainedMemory)
+    fir::setFineGrainedMemory(*module);
+  if (targetOpts.remoteMemory)
+    fir::setRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index e21d0effe2ef6..fd3ff8e262d0b 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2979,9 +2979,15 @@ genAtomicUpdate(lower::AbstractConverter &converter,
     }
   }
 
+  mlir::ModuleOp module = builder.getModule();
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      mlir::omp::AtomicControlAttr::get(
+          builder.getContext(), fir::getIgnoreDenormalMode(module),
+          fir::getFineGrainedMemory(module), fir::getRemoteMemory(module));
   builder.restoreInsertionPoint(atomicAt);
   auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+      loc, atomAddr, atomicControlAttr, hint,
+      makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 297a4c7a39e44..e3599204f5e2c 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,36 +88,34 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *amdgpuIgnoreDenormalModeName =
-    "fir.amdgpu.ignore.denormal.mode";
-void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
 }
 
-static constexpr const char *amdgpuFineGrainedMemoryName =
-    "fir.amdgpu.fine.grained.memory";
-void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
+void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
+bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
 }
-static constexpr const char *amdgpuRemoteMemoryName =
-    "fir.amdgpu.remote.memory";
-void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+static constexpr const char *remoteMemoryName = "fir.remote.memory";
+void fir::setRemoteMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
+bool fir::getRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
index 1eb0f617f365e..8f86a151b3713 100644
--- a/flang/test/Lower/OpenMP/atomic-control-options.f90
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -9,25 +9,25 @@ program test
     A = 0
     B = 0
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic
     A =  A + 1
     !$omp end target parallel
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic capture
         A = A + B
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 61861173ae875..0fefe7e017e94 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -62,9 +62,9 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 5aa7befb2dc26..9a72ddfc2989d 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1565,11 +1565,11 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   return
 }
 

>From 3a92297cddd1fa320656600701c7c09cc63f06f1 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 11:26:03 -0500
Subject: [PATCH 5/5] R5: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  8 +--
 .../Optimizer/Dialect/Support/FIRContext.h    | 19 ++++---
 flang/lib/Frontend/CompilerInvocation.cpp     |  6 +--
 flang/lib/Lower/Bridge.cpp                    |  9 ++--
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 53 ++++++++++++-------
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 30 +++++------
 6 files changed, 71 insertions(+), 54 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 6ef5338acdc6e..f6e5634d5a995 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,10 +54,10 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic control options for AMD gpu
-  bool ignoreDenormalMode = false;
-  bool remoteMemory = false;
-  bool fineGrainedMemory = false;
+  /// Atomic control options
+  bool atomicIgnoreDenormalMode = false;
+  bool atomicRemoteMemory = false;
+  bool atomicFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 273b4683b6d9c..c0b852ccaf83a 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,13 +58,18 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and getters for atomic control options.
-void setIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getIgnoreDenormalMode(mlir::ModuleOp mod);
-void setFineGrainedMemory(mlir::ModuleOp mod);
-bool getFineGrainedMemory(mlir::ModuleOp mod);
-void setRemoteMemory(mlir::ModuleOp mod);
-bool getRemoteMemory(mlir::ModuleOp mod);
+/// Sets that Denormal Mode can be ignored for lowering of floating point atomic operations.
+void setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value);
+/// Gets whether Denormal Mode can be ignored or not for lowering of floating point atomic operations.
+bool getAtomicIgnoreDenormalMode(mlir::ModuleOp mod);
+/// Sets that fine grained memory is used for lowering of atomic operations.
+void setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether fine grained memory is used or not for lowering of atomic operations.
+bool getAtomicFineGrainedMemory(mlir::ModuleOp mod);
+/// Sets that remote memory is used for lowering of atomic operations.
+void setAtomicRemoteMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether remote memory is used or not for lowering of atomic operations.
+bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index ea3d2698f4af6..024ad74a90522 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -495,13 +495,13 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  opts.ignoreDenormalMode = args.hasFlag(
+  opts.atomicIgnoreDenormalMode = args.hasFlag(
       clang::driver::options::OPT_fatomic_ignore_denormal_mode,
       clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-  opts.fineGrainedMemory = args.hasFlag(
+  opts.atomicFineGrainedMemory = args.hasFlag(
       clang::driver::options::OPT_fatomic_fine_grained_memory,
       clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-  opts.remoteMemory =
+  opts.atomicRemoteMemory =
       args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
                    clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 459c7b84e4f87..43fe26d90b080 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6654,12 +6654,9 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.ignoreDenormalMode)
-    fir::setIgnoreDenormalMode(*module);
-  if (targetOpts.fineGrainedMemory)
-    fir::setFineGrainedMemory(*module);
-  if (targetOpts.remoteMemory)
-    fir::setRemoteMemory(*module);
+  fir::setAtomicIgnoreDenormalMode(*module, targetOpts.atomicIgnoreDenormalMode);
+  fir::setAtomicFineGrainedMemory(*module, targetOpts.atomicFineGrainedMemory);
+  fir::setAtomicRemoteMemory(*module, targetOpts.atomicRemoteMemory);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index e3599204f5e2c..885efa653fa23 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,34 +88,49 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *ignoreDenormalModeName =
-    "fir.ignore.denormal.mode";
-void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicIgnoreDenormalModeName =
+    "fir.atomic_ignore_denormal_mode";
+void fir::setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicIgnoreDenormalModeName))
+      mod->removeAttr(atomicIgnoreDenormalModeName);
+  }
 }
 
-bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+bool fir::getAtomicIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicIgnoreDenormalModeName);
 }
 
-static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
-void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicFineGrainedMemoryName = "fir.atomic_fine_grained_memory";
+void fir::setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicFineGrainedMemoryName))
+      mod->removeAttr(atomicFineGrainedMemoryName);
+  }
 }
 
-bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
+bool fir::getAtomicFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicFineGrainedMemoryName);
 }
-static constexpr const char *remoteMemoryName = "fir.remote.memory";
-void fir::setRemoteMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicRemoteMemoryName = "fir.atomic_remote_memory";
+void fir::setAtomicRemoteMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicRemoteMemoryName))
+      mod->removeAttr(atomicRemoteMemoryName);
+  }
 }
 
-bool fir::getRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
+bool fir::getAtomicRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 0fefe7e017e94..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -22,6 +22,21 @@ class OpenMP_Attr<string name, string attrMnemonic, list<Trait> traits = [],
   let mnemonic = attrMnemonic;
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // DeclareTargetAttr
 //===----------------------------------------------------------------------===//
@@ -54,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//


From flang-commits at lists.llvm.org  Wed Jul  2 09:43:11 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 02 Jul 2025 09:43:11 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6865619f.050a0220.21fe7.d48d@mx.google.com>


https://github.com/TIFitis updated https://github.com/llvm/llvm-project/pull/140562

>From 74b5144ab52d8bbf03e21754ddc4fb1676f84358 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 8 May 2025 21:19:26 +0100
Subject: [PATCH 1/2] [OpenMP][Flang] Emit default declare mappers implicitly
 for derived types

This patch adds support to emit default declare mappers for implicit mapping of derived types when not supplied by user. This especially helps tackle mapping of allocatables of derived types.
This supports nested derived types as well.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  38 ++---
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 132 +++++++++++++++++-
 .../Optimizer/OpenMP/MapInfoFinalization.cpp  |   2 +-
 flang/test/Lower/OpenMP/derived-type-map.f90  |  22 ++-
 4 files changed, 174 insertions(+), 20 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 7bea427099a28..f534e37de32c2 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1167,24 +1167,28 @@ void ClauseProcessor::processMapObjects(
 
   auto getDefaultMapperID = [&](const omp::Object &object,
                                 std::string &mapperIdName) {
-    if (!mlir::isa<mlir::omp::DeclareMapperOp>(
-            firOpBuilder.getRegion().getParentOp())) {
-      const semantics::DerivedTypeSpec *typeSpec = nullptr;
-
-      if (object.sym()->owner().IsDerivedType())
-        typeSpec = object.sym()->owner().derivedTypeSpec();
-      else if (object.sym()->GetType() &&
-               object.sym()->GetType()->category() ==
-                   semantics::DeclTypeSpec::TypeDerived)
-        typeSpec = &object.sym()->GetType()->derivedTypeSpec();
-
-      if (typeSpec) {
-        mapperIdName =
-            typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;
-        if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
-          mapperIdName = converter.mangleName(mapperIdName, sym->owner());
-      }
+    const semantics::DerivedTypeSpec *typeSpec = nullptr;
+
+    if (object.sym()->GetType() && object.sym()->GetType()->category() ==
+                                       semantics::DeclTypeSpec::TypeDerived)
+      typeSpec = &object.sym()->GetType()->derivedTypeSpec();
+    else if (object.sym()->owner().IsDerivedType())
+      typeSpec = object.sym()->owner().derivedTypeSpec();
+
+    if (typeSpec) {
+      mapperIdName =
+          typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
     }
+
+    // Make sure we don't return a mapper to self
+    llvm::StringRef parentOpName;
+    if (auto declMapOp = mlir::dyn_cast<mlir::omp::DeclareMapperOp>(
+            firOpBuilder.getRegion().getParentOp()))
+      parentOpName = declMapOp.getSymName();
+    if (mapperIdName == parentOpName)
+      mapperIdName = "";
   };
 
   // Create the mapper symbol from its name, if specified.
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0a56e888ac44b..9ccccb96f7513 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2391,6 +2391,124 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  lower::StatementContext stmtCtx;
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
+        fir::unwrapPassByRefType(rec.getType()));
+    auto [fieldName, fieldTy] = recType.getTypeList()[index];
+    mlir::Value field = firOpBuilder.create<fir::FieldIndexOp>(
+        loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
+        fir::getTypeParams(rec));
+    return {firOpBuilder.create<fir::CoordinateOp>(
+                loc, firOpBuilder.getRefType(fieldTy), rec, field),
+            fieldTy};
+  };
+
+  mlir::omp::DeclareMapperInfoOperands clauseOps;
+  llvm::SmallVector<llvm::SmallVector<int64_t>> memberPlacementIndices;
+  llvm::SmallVector<mlir::Value> memberMapOps;
+
+  llvm::omp::OpenMPOffloadMappingFlags mapFlag =
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+  mlir::omp::VariableCaptureKind captureKind =
+      mlir::omp::VariableCaptureKind::ByRef;
+  int64_t index = 0;
+
+  // Populate the declareMapper region with the map information.
+  for (const auto &[memberName, memberType] :
+       mlir::dyn_cast<fir::RecordType>(recordType).getTypeList()) {
+    auto [ref, type] = getFieldRef(declareOp.getBase(), index);
+    mlir::FlatSymbolRefAttr mapperId;
+    if (auto recType = mlir::dyn_cast<fir::RecordType>(memberType)) {
+      std::string mapperIdName =
+          recType.getName().str() + ".omp.default.mapper";
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+      else if (auto *sym = converter.getCurrentScope().FindSymbol(memberName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+
+      if (converter.getModuleOp().lookupSymbol(mapperIdName))
+        mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                                mapperIdName);
+      else
+        mapperId = genImplicitDefaultDeclareMapper(converter, loc, recType,
+                                                   mapperIdName);
+    }
+
+    llvm::SmallVector<mlir::Value> bounds;
+    genBoundsOps(ref, bounds);
+    mlir::Value mapOp = createMapInfoOp(
+        firOpBuilder, loc, ref, /*varPtrPtr=*/mlir::Value{}, "", bounds,
+        /*members=*/{},
+        /*membersIndex=*/mlir::ArrayAttr{},
+        static_cast<
+            std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
+            mapFlag),
+        captureKind, ref.getType(), /*partialMap=*/false, mapperId);
+    memberMapOps.emplace_back(mapOp);
+    memberPlacementIndices.emplace_back(llvm::SmallVector<int64_t>{index++});
+  }
+
+  llvm::SmallVector<mlir::Value> bounds;
+  genBoundsOps(declareOp.getOriginalBase(), bounds);
+  mlir::omp::MapInfoOp mapOp = createMapInfoOp(
+      firOpBuilder, loc, declareOp.getOriginalBase(),
+      /*varPtrPtr=*/mlir::Value(), /*name=*/"", bounds, memberMapOps,
+      firOpBuilder.create2DI64ArrayAttr(memberPlacementIndices),
+      static_cast<std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
+          mapFlag),
+      captureKind, declareOp.getType(0),
+      /*partialMap=*/true);
+
+  clauseOps.mapVars.emplace_back(mapOp);
+
+  firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
+  // declMapperOp->dumpPretty();
+  return mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                      mapperNameStr);
+}
+
 static mlir::omp::TargetOp
 genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
             lower::StatementContext &stmtCtx,
@@ -2467,15 +2585,26 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       name << sym.name().ToString();
 
       mlir::FlatSymbolRefAttr mapperId;
-      if (sym.GetType()->category() == semantics::DeclTypeSpec::TypeDerived) {
+      if (sym.GetType()->category() == semantics::DeclTypeSpec::TypeDerived &&
+          defaultMaps.empty()) {
         auto &typeSpec = sym.GetType()->derivedTypeSpec();
         std::string mapperIdName =
             typeSpec.name().ToString() + llvm::omp::OmpDefaultMapperName;
         if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
           mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+        else
+          mapperIdName =
+              converter.mangleName(mapperIdName, *typeSpec.GetScope());
+
         if (converter.getModuleOp().lookupSymbol(mapperIdName))
           mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
                                                   mapperIdName);
+        else
+          mapperId = genImplicitDefaultDeclareMapper(
+              converter, loc,
+              mlir::cast<fir::RecordType>(
+                  converter.genType(sym.GetType()->derivedTypeSpec())),
+              mapperIdName);
       }
 
       fir::factory::AddrAndBoundsInfo info =
@@ -3442,6 +3571,7 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processMap(loc, stmtCtx, clauseOps);
   firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
+  // declMapperOp->dumpPretty();
 }
 
 static void
diff --git a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
index f052cf8d22d1e..2a11aff4befc0 100644
--- a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
@@ -433,7 +433,7 @@ class MapInfoFinalizationPass
                                    getDescriptorMapType(mapType, target)),
             op.getMapCaptureTypeAttr(), /*varPtrPtr=*/mlir::Value{}, newMembers,
             newMembersAttr, /*bounds=*/mlir::SmallVector<mlir::Value>{},
-            /*mapperId*/ mlir::FlatSymbolRefAttr(), op.getNameAttr(),
+            op.getMapperIdAttr(), op.getNameAttr(),
             /*partial_map=*/builder.getBoolAttr(false));
     op.replaceAllUsesWith(newDescParentMapOp.getResult());
     op->erase();
diff --git a/flang/test/Lower/OpenMP/derived-type-map.f90 b/flang/test/Lower/OpenMP/derived-type-map.f90
index 279cddec51fcf..1dd73229d8251 100644
--- a/flang/test/Lower/OpenMP/derived-type-map.f90
+++ b/flang/test/Lower/OpenMP/derived-type-map.f90
@@ -1,9 +1,11 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
+!CHECK: omp.declare_mapper @[[MAPPER1:_QQFmaptype_derived_implicit_allocatablescalar_and_array.omp.default.mapper]] : !fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
+!CHECK: omp.declare_mapper @[[MAPPER2:_QQFmaptype_derived_implicitscalar_and_array.omp.default.mapper]] : !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
 
 !CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_implicitEscalar_arr"}
 !CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_implicitEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
-!CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(implicit, tofrom) capture(ByRef) -> !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr"}
+!CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(implicit, tofrom) capture(ByRef) mapper(@[[MAPPER2]]) -> !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr"}
 !CHECK:     omp.target map_entries(%[[MAP]] -> %[[ARG0:.*]] : !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
 subroutine mapType_derived_implicit
     type :: scalar_and_array
@@ -18,6 +20,24 @@ subroutine mapType_derived_implicit
     !$omp end target
 end subroutine mapType_derived_implicit
 
+!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_implicit_allocatableEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFmaptype_derived_implicit_allocatableEscalar_arr"} : {{.*}}
+!CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : {{.*}}) map_clauses(implicit, to) capture(ByRef) mapper(@[[MAPPER1]])
+!CHECK:     omp.target map_entries(%[[MAP]] -> %[[ARG0:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>) {
+subroutine mapType_derived_implicit_allocatable
+    type :: scalar_and_array
+      real(4) :: real
+      integer(4) :: array(10)
+      integer(4) :: int
+    end type scalar_and_array
+    type(scalar_and_array), allocatable :: scalar_arr 
+
+    allocate (scalar_arr)
+    !$omp target
+       scalar_arr%int = 1
+    !$omp end target
+end subroutine mapType_derived_implicit_allocatable
+
 !CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_explicitEscalar_arr"}
 !CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_explicitEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
 !CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(tofrom) capture(ByRef) -> !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr"}

>From 138250f6eed33a589541c60c5290ea2b183c60c6 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 21 May 2025 16:48:27 +0100
Subject: [PATCH 2/2] Minor cleanup.

---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp | 33 +++++++++++++---------
 flang/lib/Lower/OpenMP/OpenMP.cpp          |  3 --
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index f534e37de32c2..f0a90887eeeec 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1167,19 +1167,26 @@ void ClauseProcessor::processMapObjects(
 
   auto getDefaultMapperID = [&](const omp::Object &object,
                                 std::string &mapperIdName) {
-    const semantics::DerivedTypeSpec *typeSpec = nullptr;
-
-    if (object.sym()->GetType() && object.sym()->GetType()->category() ==
-                                       semantics::DeclTypeSpec::TypeDerived)
-      typeSpec = &object.sym()->GetType()->derivedTypeSpec();
-    else if (object.sym()->owner().IsDerivedType())
-      typeSpec = object.sym()->owner().derivedTypeSpec();
-
-    if (typeSpec) {
-      mapperIdName =
-          typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;
-      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
-        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+    if (!mlir::isa<mlir::omp::DeclareMapperOp>(
+            firOpBuilder.getRegion().getParentOp())) {
+      const semantics::DerivedTypeSpec *typeSpec = nullptr;
+
+      if (object.sym()->owner().IsDerivedType())
+        typeSpec = object.sym()->owner().derivedTypeSpec();
+      else if (object.sym()->GetType() &&
+               object.sym()->GetType()->category() ==
+                   semantics::DeclTypeSpec::TypeDerived)
+        typeSpec = &object.sym()->GetType()->derivedTypeSpec();
+
+      if (typeSpec) {
+        mapperIdName =
+            typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;
+        if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+          mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+        else
+          mapperIdName =
+              converter.mangleName(mapperIdName, *typeSpec->GetScope());
+      }
     }
 
     // Make sure we don't return a mapper to self
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 9ccccb96f7513..9f5bb017a3e74 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2396,7 +2396,6 @@ genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
                                 mlir::Location loc, fir::RecordType recordType,
                                 llvm::StringRef mapperNameStr) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  lower::StatementContext stmtCtx;
 
   // Save current insertion point before moving to the module scope to create
   // the DeclareMapperOp
@@ -2504,7 +2503,6 @@ genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
   clauseOps.mapVars.emplace_back(mapOp);
 
   firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
-  // declMapperOp->dumpPretty();
   return mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
                                       mapperNameStr);
 }
@@ -3571,7 +3569,6 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processMap(loc, stmtCtx, clauseOps);
   firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
-  // declMapperOp->dumpPretty();
 }
 
 static void


From flang-commits at lists.llvm.org  Wed Jul  2 09:47:18 2025
From: flang-commits at lists.llvm.org (Anchu Rajendran S via flang-commits)
Date: Wed, 02 Jul 2025 09:47:18 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <68656296.050a0220.3dc0d9.accd@mx.google.com>


https://github.com/anchuraj updated https://github.com/llvm/llvm-project/pull/143441

>From 55e513da2cb2d68dfb91e889bb66b7fdd58090ac Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 1/5] [flang][flang-driver] atomic control support

---
 clang/include/clang/Driver/Options.td         | 22 +++++-----
 flang/include/flang/Frontend/TargetOptions.h  |  5 +++
 .../Optimizer/Dialect/Support/FIRContext.h    | 14 +++++++
 flang/lib/Frontend/CompilerInvocation.cpp     | 12 ++++++
 flang/lib/Lower/Bridge.cpp                    |  6 +++
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 40 +++++++++++++++++++
 .../Lower/OpenMP/atomic-control-options.f90   | 37 +++++++++++++++++
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 14 +++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  8 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  9 +++++
 10 files changed, 153 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/atomic-control-options.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 0822df2640d23..300b9c0e27ba9 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2303,21 +2303,21 @@ def fsymbol_partition_EQ : Joined<["-"], "fsymbol-partition=">, Group<f_Group>,
 
 defm atomic_remote_memory : BoolFOption<"atomic-remote-memory",
   LangOpts<"AtomicRemoteMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on remote memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on remote memory">>;
 
 defm atomic_fine_grained_memory : BoolFOption<"atomic-fine-grained-memory",
   LangOpts<"AtomicFineGrainedMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on fine-grained memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on fine-grained memory">>;
 
 defm atomic_ignore_denormal_mode : BoolFOption<"atomic-ignore-denormal-mode",
   LangOpts<"AtomicIgnoreDenormalMode">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "Allow">,
-  NegFlag<SetFalse, [], [ClangOption], "Disallow">,
-  BothFlags<[], [ClangOption], " atomic operations to ignore denormal mode">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "Allow">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Disallow">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations to ignore denormal mode">>;
 
 defm memory_profile : OptInCC1FFlag<"memory-profile", "Enable", "Disable", " heap memory profiling">;
 def fmemory_profile_EQ : Joined<["-"], "fmemory-profile=">,
@@ -5305,9 +5305,9 @@ defm amdgpu_precise_memory_op
                   " precise memory mode (AMDGPU only)">;
 
 def munsafe_fp_atomics : Flag<["-"], "munsafe-fp-atomics">,
-  Visibility<[ClangOption, CC1Option]>, Alias<fatomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>, Alias<fatomic_ignore_denormal_mode>;
 def mno_unsafe_fp_atomics : Flag<["-"], "mno-unsafe-fp-atomics">,
-  Visibility<[ClangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, FlangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
 
 def faltivec : Flag<["-"], "faltivec">, Group<f_Group>;
 def fno_altivec : Flag<["-"], "fno-altivec">, Group<f_Group>;
diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 002d8d158abd4..26256fd775f11 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic Control Options for AMD GPU
+  bool amdgpuIgnoreDenormalMode = false;
+  bool amdgpuRemoteMemory = false;
+  bool amdgpuFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 2df14f83c11e1..ac563bcc402c7 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,10 +58,24 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and Getters for atomic control options
+void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
+bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 766131fd69d85..240b95a1fafb4 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -495,6 +495,18 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
+  if (llvm::Triple(opts.triple).isAMDGPU()) {
+    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
+        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+    opts.amdgpuFineGrainedMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_fine_grained_memory,
+        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+    opts.amdgpuRemoteMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_remote_memory,
+        clang::driver::options::OPT_fno_atomic_remote_memory, false);
+  }
+
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
     opts.cpu = a->getValue();
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 8506b9a984e58..f1bcb3c6e080c 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6654,6 +6654,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
+  if (targetOpts.amdgpuIgnoreDenormalMode)
+    fir::setAmdgpuIgnoreDenormalMode(*module);
+  if (targetOpts.amdgpuFineGrainedMemory)
+    fir::setAmdgpuFineGrainedMemory(*module);
+  if (targetOpts.amdgpuRemoteMemory)
+    fir::setAmdgpuRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 01c0be66d1ecc..b961793dbdfd5 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,6 +88,46 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *amdgpuIgnoreDenormalModeName =
+    "fir.amdgpu.ignore.denormal.mode";
+void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
+    return true;
+  return false;
+}
+
+static constexpr const char *amdgpuFineGrainedMemoryName =
+    "fir.amdgpu.fine.grained.memory";
+void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
+    return true;
+  return false;
+}
+static constexpr const char *amdgpuRemoteMemoryName =
+    "fir.amdgpu.remote.memory";
+void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
+    return true;
+  return false;
+}
+
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
   if (auto attr = mod->getAttrOfType<mlir::StringAttr>(tuneCpuName))
     return attr.getValue();
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
new file mode 100644
index 0000000000000..1eb0f617f365e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o - | FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o - | FileCheck -check-prefix=IGNORE-DENORMAL %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o - | FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o - | FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, B, threads
+    threads = 128
+    A = 0
+    B = 0
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic capture
+        A = A + B
+        B = A
+    !$omp end atomic
+    !$omp end target parallel
+end program test
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 704d0b2220e8a..84887dfd2c6f9 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -54,6 +54,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
+      clausesArgs);
 
   // Override region definition.
   let regions = (region SizedRegion<1>:$region);
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..e377b7aab3e43 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1562,6 +1562,15 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
     omp.yield(%newval : i32)
   }
 
+  // CHECK: omp.atomic.update %[[X]] : memref<i32> {
+  // CHECK-NEXT: (%[[XVAL:.*]]: i32):
+  // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
+  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  omp.atomic.update %x : memref<i32> {
+  ^bb0(%xval:i32):
+    omp.yield(%const:i32)
+  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+
   return
 }
 

>From 0d15bdb469d3be4764a79b32f05a7e2bdf66666a Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:58:07 -0500
Subject: [PATCH 2/5] R2: Making Atomic control non-optional

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   2 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |   2 +-
 mlir/test/Dialect/OpenMP/canonicalize.mlir    |  10 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  46 +++----
 mlir/test/Dialect/OpenMP/ops.mlir             | 127 +++++++++---------
 .../Target/LLVMIR/openmp-llvm-invalid.mlir    |   8 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  98 +++++++-------
 .../openmp-target-nesting-in-host-ops.mlir    |   2 +-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |   2 +-
 9 files changed, 148 insertions(+), 149 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 375e35a146f78..399f7fa6e78f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          OptionalAttr<AtomicControlAttr>:$atomic_control),
+          AtomicControlAttr:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index d69de998346b5..0b2445d0a0d18 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -133,7 +133,7 @@ func.func @atomic_update() {
     %1 = arith.constant 1 : i32
     %2 = arith.addi %arg0, %1  : i32
     omp.yield(%2 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 llvm.mlir.global internal @_QFsEc() : i32 {
diff --git a/mlir/test/Dialect/OpenMP/canonicalize.mlir b/mlir/test/Dialect/OpenMP/canonicalize.mlir
index de6c931ecc5fd..2492010c4d3c7 100644
--- a/mlir/test/Dialect/OpenMP/canonicalize.mlir
+++ b/mlir/test/Dialect/OpenMP/canonicalize.mlir
@@ -4,7 +4,7 @@ func.func @update_no_op(%x : memref<i32>) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -17,7 +17,7 @@ func.func @update_write_op(%x : memref<i32>, %value: i32) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%value : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -33,7 +33,7 @@ func.func @update_normal(%x : memref<i32>, %value: i32) {
   ^bb0(%xval : i32):
     %newval = arith.addi %xval, %value : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -50,7 +50,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.addi %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -65,7 +65,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.muli %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..afc3830286f30 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1007,7 +1007,7 @@ func.func @omp_atomic_update1(%x: memref<i32>, %expr: f32) {
   ^bb0(%xval: f32):
     %newval = llvm.fadd %xval, %expr : f32
     omp.yield (%newval : f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1020,7 +1020,7 @@ func.func @omp_atomic_update2(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.terminator
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1032,7 +1032,7 @@ func.func @omp_atomic_update3(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1044,7 +1044,7 @@ func.func @omp_atomic_update4(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1056,7 +1056,7 @@ func.func @omp_atomic_update5(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1068,7 +1068,7 @@ func.func @omp_atomic_update6(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval, %expr : i32, i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1080,7 +1080,7 @@ func.func @omp_atomic_update7(%x: memref<i32>, %expr: i32, %y: f32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%y: f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1092,7 +1092,7 @@ func.func @omp_atomic_update8(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32, %tmp: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1104,7 +1104,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1116,7 +1116,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1128,7 +1128,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1164,12 +1164,12 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.update %x : memref<i32> {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1197,7 +1197,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1212,7 +1212,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.write %x = %expr : memref<i32>, i32
     omp.terminator
   }
@@ -1240,7 +1240,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %y : memref<i32>, memref<i32>, i32
     omp.terminator
   }
@@ -1256,7 +1256,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
 }
@@ -1281,7 +1281,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1296,7 +1296,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1311,7 +1311,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1326,7 +1326,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1341,7 +1341,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1356,7 +1356,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x memory_order(seq_cst) : memref<i32>, memref<i32>, i32
   }
   return
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index e377b7aab3e43..5aa7befb2dc26 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
@@ -1380,57 +1380,57 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i1):
     %newval = llvm.and %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.shl %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.intr.smax(%[[XVAL]], %[[EXPR]]) : (i32, i32) -> i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.intr.smax(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.icmp "eq" %[[XVAL]], %[[EXPRBOOL]] : i1
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i1)
-  // }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %xBool : memref<i1> {
   ^bb0(%xval: i1):
     %newval = llvm.icmp "eq" %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%xval:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   %const = arith.constant 42 : i32
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1440,7 +1440,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1450,7 +1450,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1460,7 +1460,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1470,7 +1470,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1480,7 +1480,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1490,7 +1490,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1500,7 +1500,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1510,7 +1510,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1520,7 +1520,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1530,7 +1530,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(release) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1540,7 +1540,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(relaxed) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1550,7 +1550,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1560,17 +1560,16 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
-
+  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   return
 }
 
@@ -1582,7 +1581,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture{
@@ -1590,7 +1589,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   // CHECK: omp.atomic.capture {
@@ -1599,7 +1598,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: }
   omp.atomic.capture{
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
@@ -1607,7 +1606,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
@@ -1623,7 +1622,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(none) {
@@ -1631,7 +1630,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1640,7 +1639,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended) {
@@ -1648,7 +1647,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1657,7 +1656,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended) {
@@ -1665,7 +1664,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1674,7 +1673,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(nonspeculative) {
@@ -1682,7 +1681,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1691,7 +1690,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(speculative) {
@@ -1699,7 +1698,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1708,7 +1707,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, nonspeculative) {
@@ -1716,7 +1715,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1725,7 +1724,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, nonspeculative) {
@@ -1733,7 +1732,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1742,7 +1741,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, speculative) {
@@ -1750,7 +1749,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1759,7 +1758,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) {
@@ -1767,7 +1766,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1776,7 +1775,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(seq_cst) {
@@ -1784,7 +1783,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1793,7 +1792,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acq_rel) {
@@ -1801,7 +1800,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1810,7 +1809,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acquire) {
@@ -1818,7 +1817,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1827,7 +1826,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(release) {
@@ -1835,7 +1834,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1844,7 +1843,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(relaxed) {
@@ -1852,7 +1851,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1861,7 +1860,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) memory_order(seq_cst) {
@@ -1869,7 +1868,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -2653,14 +2652,14 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
   // CHECK: omp.atomic.update %[[x]] : !llvm.ptr {
   // CHECK-NEXT: ^{{[[:alnum:]]+}}(%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   %val = llvm.load %x : !llvm.ptr -> i32
   omp.atomic.write %v = %val : !llvm.ptr, i32
   omp.atomic.update %x : !llvm.ptr {
     ^bb0(%xval: i32):
       omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index 41bc5c4ba525f..00f53864b5fa7 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -9,7 +9,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
     %t2 = llvm.sdiv %t1, %expr : i32
     %newval = llvm.add %xval, %t2 : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -24,7 +24,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.mul %expr, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -41,7 +41,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
     ^bb0(%xval: i32):
       %newval = llvm.mul %expr, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
@@ -59,7 +59,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
       %t2 = llvm.sdiv %t1, %expr : i32
       %newval = llvm.add %xval, %t2 : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 32f0ba5b105ff..ab7f756bf95f9 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,13 +1462,13 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1563,7 +1563,7 @@ llvm.func @_QPomp_atomic_update_complex() {
       %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
       %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
       omp.yield(%16 : !llvm.struct<(f32, f32)>)
-    }
+    } {atomic_control = #omp.atomic_control<>}
    llvm.return
 }
 
@@ -1623,7 +1623,7 @@ llvm.func @_QPomp_atomic_capture_complex() {
         %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
         %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
         omp.yield(%20 : !llvm.struct<(f32, f32)>)
-      }
+      } {atomic_control = #omp.atomic_control<>}
       omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.ptr, !llvm.struct<(f32, f32)>
     }
     llvm.return
@@ -1665,7 +1665,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %expr, %xval : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1683,7 +1683,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1701,7 +1701,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
   // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
   // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
@@ -1710,7 +1710,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1732,7 +1732,7 @@ llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
     %2 = llvm.fadd %1, %0 : f32
     %3 = llvm.fptosi %2 : f32 to i32
     omp.yield(%3 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1751,7 +1751,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1763,7 +1763,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1775,7 +1775,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1787,7 +1787,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1799,7 +1799,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1814,7 +1814,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1829,7 +1829,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1844,7 +1844,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1859,7 +1859,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1874,7 +1874,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1889,7 +1889,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1904,7 +1904,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1919,7 +1919,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1934,7 +1934,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1949,7 +1949,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1964,7 +1964,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -1979,7 +1979,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -2001,7 +2001,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw sub ptr %[[x]], i32 %[[expr]] monotonic
@@ -2012,7 +2012,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw and ptr %[[x]], i32 %[[expr]] monotonic
@@ -2023,7 +2023,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw or ptr %[[x]], i32 %[[expr]] monotonic
@@ -2034,7 +2034,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw xor ptr %[[x]], i32 %[[expr]] monotonic
@@ -2045,7 +2045,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2060,7 +2060,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2075,7 +2075,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2090,7 +2090,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2105,7 +2105,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2120,7 +2120,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2135,7 +2135,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2150,7 +2150,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2165,7 +2165,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2180,7 +2180,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2195,7 +2195,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2211,7 +2211,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2227,7 +2227,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -2265,7 +2265,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acquire
@@ -2276,7 +2276,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] release
@@ -2287,7 +2287,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
@@ -2298,7 +2298,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acq_rel
@@ -2309,7 +2309,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -3055,7 +3055,7 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
index cbf273b887bc7..a9a4ffd533136 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
@@ -110,7 +110,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
 
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 97608ca3b4df1..2f294658edb3d 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -19,7 +19,7 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   ^bb0(%arg0: i32):
     %result = llvm.add %arg0, %expr : i32
     omp.yield(%result : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   llvm.return
 }

>From 0b6fd8405623d5192f9d7c22d2470c43be5b3bdf Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 3/5] R3: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h       |  2 +-
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  4 ++--
 flang/lib/Optimizer/Dialect/Support/FIRContext.cpp | 14 +++-----------
 flang/test/Fir/convert-to-llvm-openmp-and-fir.fir  |  4 ++--
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td |  3 ++-
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 26256fd775f11..dd29b4e3e8b08 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,7 +54,7 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic Control Options for AMD GPU
+  /// Atomic control options for AMD gpu
   bool amdgpuIgnoreDenormalMode = false;
   bool amdgpuRemoteMemory = false;
   bool amdgpuFineGrainedMemory = false;
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac563bcc402c7..ac5fd33874db8 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,7 +58,7 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and Getters for atomic control options
+// Setters and getters for atomic control options.
 void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
@@ -70,7 +70,7 @@ bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index b961793dbdfd5..297a4c7a39e44 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -96,10 +96,7 @@ void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
 }
 
 static constexpr const char *amdgpuFineGrainedMemoryName =
@@ -110,10 +107,7 @@ void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
 }
 static constexpr const char *amdgpuRemoteMemoryName =
     "fir.amdgpu.remote.memory";
@@ -123,9 +117,7 @@ void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 24e5cad84b709..47c00d489c200 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -619,7 +619,7 @@ func.func @_QPs() {
     %c1_i32 = arith.constant 1 : i32
     %1 = arith.addi %arg0, %c1_i32 : i32
     omp.yield(%1 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 fir.global internal @_QFsEc : i32 {
@@ -634,7 +634,7 @@ fir.global internal @_QFsEc : i32 {
 // CHECK:      %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:      %[[OUT_VAL:.*]] = llvm.add %[[IN_VAL]], %[[CONST_1]]  : i32
 // CHECK:      omp.yield(%[[OUT_VAL]] : i32)
-// CHECK:    }
+// CHECK:    } {atomic_control = #omp.atomic_control<>}
 // CHECK:    llvm.return
 // CHECK:  }
 // CHECK:  llvm.mlir.global internal @[[GLOBAL]]() {{.*}} : i32 {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 84887dfd2c6f9..61861173ae875 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -58,7 +58,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From fe9a7022ec1f1c1065c782408b54dff5868072ce Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Mon, 23 Jun 2025 12:14:19 -0500
Subject: [PATCH 4/5] R4: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  6 ++--
 .../Optimizer/Dialect/Support/FIRContext.h    | 18 ++++------
 flang/lib/Frontend/CompilerInvocation.cpp     | 20 +++++------
 flang/lib/Lower/Bridge.cpp                    | 12 +++----
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  8 ++++-
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 34 +++++++++----------
 .../Lower/OpenMP/atomic-control-options.f90   | 16 ++++-----
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     |  6 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  4 +--
 9 files changed, 60 insertions(+), 64 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index dd29b4e3e8b08..6ef5338acdc6e 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -55,9 +55,9 @@ class TargetOptions {
   bool asmVerbose = false;
 
   /// Atomic control options for AMD gpu
-  bool amdgpuIgnoreDenormalMode = false;
-  bool amdgpuRemoteMemory = false;
-  bool amdgpuFineGrainedMemory = false;
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac5fd33874db8..273b4683b6d9c 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -59,23 +59,17 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
 // Setters and getters for atomic control options.
-void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
-bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+void setIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getIgnoreDenormalMode(mlir::ModuleOp mod);
+void setFineGrainedMemory(mlir::ModuleOp mod);
+bool getFineGrainedMemory(mlir::ModuleOp mod);
+void setRemoteMemory(mlir::ModuleOp mod);
+bool getRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set atomic control options for AMD gpu.
-void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
-                                   bool amdgpuIgnoreDenormalMode,
-                                   bool amdgpuNoFineGrainedMemory,
-                                   bool amdgpuNoRemoteMemory);
-
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 240b95a1fafb4..ea3d2698f4af6 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -495,17 +495,15 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  if (llvm::Triple(opts.triple).isAMDGPU()) {
-    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
-        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
-        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-    opts.amdgpuFineGrainedMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_fine_grained_memory,
-        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-    opts.amdgpuRemoteMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_remote_memory,
-        clang::driver::options::OPT_fno_atomic_remote_memory, false);
-  }
+  opts.ignoreDenormalMode = args.hasFlag(
+      clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+      clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+  opts.fineGrainedMemory = args.hasFlag(
+      clang::driver::options::OPT_fatomic_fine_grained_memory,
+      clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+  opts.remoteMemory =
+      args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
+                   clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index f1bcb3c6e080c..459c7b84e4f87 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6654,12 +6654,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.amdgpuIgnoreDenormalMode)
-    fir::setAmdgpuIgnoreDenormalMode(*module);
-  if (targetOpts.amdgpuFineGrainedMemory)
-    fir::setAmdgpuFineGrainedMemory(*module);
-  if (targetOpts.amdgpuRemoteMemory)
-    fir::setAmdgpuRemoteMemory(*module);
+  if (targetOpts.ignoreDenormalMode)
+    fir::setIgnoreDenormalMode(*module);
+  if (targetOpts.fineGrainedMemory)
+    fir::setFineGrainedMemory(*module);
+  if (targetOpts.remoteMemory)
+    fir::setRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index e21d0effe2ef6..fd3ff8e262d0b 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2979,9 +2979,15 @@ genAtomicUpdate(lower::AbstractConverter &converter,
     }
   }
 
+  mlir::ModuleOp module = builder.getModule();
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      mlir::omp::AtomicControlAttr::get(
+          builder.getContext(), fir::getIgnoreDenormalMode(module),
+          fir::getFineGrainedMemory(module), fir::getRemoteMemory(module));
   builder.restoreInsertionPoint(atomicAt);
   auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+      loc, atomAddr, atomicControlAttr, hint,
+      makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 297a4c7a39e44..e3599204f5e2c 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,36 +88,34 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *amdgpuIgnoreDenormalModeName =
-    "fir.amdgpu.ignore.denormal.mode";
-void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
 }
 
-static constexpr const char *amdgpuFineGrainedMemoryName =
-    "fir.amdgpu.fine.grained.memory";
-void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
+void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
+bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
 }
-static constexpr const char *amdgpuRemoteMemoryName =
-    "fir.amdgpu.remote.memory";
-void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+static constexpr const char *remoteMemoryName = "fir.remote.memory";
+void fir::setRemoteMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
+bool fir::getRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
index 1eb0f617f365e..8f86a151b3713 100644
--- a/flang/test/Lower/OpenMP/atomic-control-options.f90
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -9,25 +9,25 @@ program test
     A = 0
     B = 0
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic
     A =  A + 1
     !$omp end target parallel
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic capture
         A = A + B
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 61861173ae875..0fefe7e017e94 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -62,9 +62,9 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 5aa7befb2dc26..9a72ddfc2989d 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1565,11 +1565,11 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   return
 }
 

>From 25ceffef39b6d7c3c7a489152e4e216249125fcc Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 11:26:03 -0500
Subject: [PATCH 5/5] R5: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  8 +--
 .../Optimizer/Dialect/Support/FIRContext.h    | 19 ++++---
 flang/lib/Frontend/CompilerInvocation.cpp     |  6 +--
 flang/lib/Lower/Bridge.cpp                    |  9 ++--
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  4 +-
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 53 ++++++++++++-------
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 30 +++++------
 7 files changed, 73 insertions(+), 56 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 6ef5338acdc6e..f6e5634d5a995 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,10 +54,10 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic control options for AMD gpu
-  bool ignoreDenormalMode = false;
-  bool remoteMemory = false;
-  bool fineGrainedMemory = false;
+  /// Atomic control options
+  bool atomicIgnoreDenormalMode = false;
+  bool atomicRemoteMemory = false;
+  bool atomicFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 273b4683b6d9c..c0b852ccaf83a 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,13 +58,18 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and getters for atomic control options.
-void setIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getIgnoreDenormalMode(mlir::ModuleOp mod);
-void setFineGrainedMemory(mlir::ModuleOp mod);
-bool getFineGrainedMemory(mlir::ModuleOp mod);
-void setRemoteMemory(mlir::ModuleOp mod);
-bool getRemoteMemory(mlir::ModuleOp mod);
+/// Sets that Denormal Mode can be ignored for lowering of floating point atomic operations.
+void setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value);
+/// Gets whether Denormal Mode can be ignored or not for lowering of floating point atomic operations.
+bool getAtomicIgnoreDenormalMode(mlir::ModuleOp mod);
+/// Sets that fine grained memory is used for lowering of atomic operations.
+void setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether fine grained memory is used or not for lowering of atomic operations.
+bool getAtomicFineGrainedMemory(mlir::ModuleOp mod);
+/// Sets that remote memory is used for lowering of atomic operations.
+void setAtomicRemoteMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether remote memory is used or not for lowering of atomic operations.
+bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index ea3d2698f4af6..024ad74a90522 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -495,13 +495,13 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  opts.ignoreDenormalMode = args.hasFlag(
+  opts.atomicIgnoreDenormalMode = args.hasFlag(
       clang::driver::options::OPT_fatomic_ignore_denormal_mode,
       clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-  opts.fineGrainedMemory = args.hasFlag(
+  opts.atomicFineGrainedMemory = args.hasFlag(
       clang::driver::options::OPT_fatomic_fine_grained_memory,
       clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-  opts.remoteMemory =
+  opts.atomicRemoteMemory =
       args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
                    clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 459c7b84e4f87..43fe26d90b080 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6654,12 +6654,9 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.ignoreDenormalMode)
-    fir::setIgnoreDenormalMode(*module);
-  if (targetOpts.fineGrainedMemory)
-    fir::setFineGrainedMemory(*module);
-  if (targetOpts.remoteMemory)
-    fir::setRemoteMemory(*module);
+  fir::setAtomicIgnoreDenormalMode(*module, targetOpts.atomicIgnoreDenormalMode);
+  fir::setAtomicFineGrainedMemory(*module, targetOpts.atomicFineGrainedMemory);
+  fir::setAtomicRemoteMemory(*module, targetOpts.atomicRemoteMemory);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index fd3ff8e262d0b..c70ca9483fc3c 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2982,8 +2982,8 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::ModuleOp module = builder.getModule();
   mlir::omp::AtomicControlAttr atomicControlAttr =
       mlir::omp::AtomicControlAttr::get(
-          builder.getContext(), fir::getIgnoreDenormalMode(module),
-          fir::getFineGrainedMemory(module), fir::getRemoteMemory(module));
+          builder.getContext(), fir::getAtomicIgnoreDenormalMode(module),
+          fir::getAtomicFineGrainedMemory(module), fir::getAtomicRemoteMemory(module));
   builder.restoreInsertionPoint(atomicAt);
   auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
       loc, atomAddr, atomicControlAttr, hint,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index e3599204f5e2c..885efa653fa23 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,34 +88,49 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *ignoreDenormalModeName =
-    "fir.ignore.denormal.mode";
-void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicIgnoreDenormalModeName =
+    "fir.atomic_ignore_denormal_mode";
+void fir::setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicIgnoreDenormalModeName))
+      mod->removeAttr(atomicIgnoreDenormalModeName);
+  }
 }
 
-bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+bool fir::getAtomicIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicIgnoreDenormalModeName);
 }
 
-static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
-void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicFineGrainedMemoryName = "fir.atomic_fine_grained_memory";
+void fir::setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicFineGrainedMemoryName))
+      mod->removeAttr(atomicFineGrainedMemoryName);
+  }
 }
 
-bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
+bool fir::getAtomicFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicFineGrainedMemoryName);
 }
-static constexpr const char *remoteMemoryName = "fir.remote.memory";
-void fir::setRemoteMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicRemoteMemoryName = "fir.atomic_remote_memory";
+void fir::setAtomicRemoteMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicRemoteMemoryName))
+      mod->removeAttr(atomicRemoteMemoryName);
+  }
 }
 
-bool fir::getRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
+bool fir::getAtomicRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 0fefe7e017e94..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -22,6 +22,21 @@ class OpenMP_Attr<string name, string attrMnemonic, list<Trait> traits = [],
   let mnemonic = attrMnemonic;
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // DeclareTargetAttr
 //===----------------------------------------------------------------------===//
@@ -54,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//


From flang-commits at lists.llvm.org  Wed Jul  2 11:11:04 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 02 Jul 2025 11:11:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][prescanner] fix invalid check (PR
	#146613)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146613@github.com>
Message-ID: <68657638.050a0220.9e41.c404@mx.google.com>


https://github.com/akuhlens edited https://github.com/llvm/llvm-project/pull/146613

From flang-commits at lists.llvm.org  Wed Jul  2 11:37:26 2025
From: flang-commits at lists.llvm.org (Anchu Rajendran S via flang-commits)
Date: Wed, 02 Jul 2025 11:37:26 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <68657c66.170a0220.120e8a.5418@mx.google.com>


https://github.com/anchuraj updated https://github.com/llvm/llvm-project/pull/143441

>From 4816fb9bc0761d8761798300bb419db0072c9d04 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 1/5] [flang][flang-driver] atomic control support

---
 clang/include/clang/Driver/Options.td         | 22 +++++-----
 flang/include/flang/Frontend/TargetOptions.h  |  5 +++
 .../Optimizer/Dialect/Support/FIRContext.h    | 14 +++++++
 flang/lib/Frontend/CompilerInvocation.cpp     | 12 ++++++
 flang/lib/Lower/Bridge.cpp                    |  6 +++
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 40 +++++++++++++++++++
 .../Lower/OpenMP/atomic-control-options.f90   | 37 +++++++++++++++++
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 14 +++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  8 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  9 +++++
 10 files changed, 153 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/atomic-control-options.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8e38f0062140e..c1c7929b0c15d 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2303,21 +2303,21 @@ def fsymbol_partition_EQ : Joined<["-"], "fsymbol-partition=">, Group<f_Group>,
 
 defm atomic_remote_memory : BoolFOption<"atomic-remote-memory",
   LangOpts<"AtomicRemoteMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on remote memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on remote memory">>;
 
 defm atomic_fine_grained_memory : BoolFOption<"atomic-fine-grained-memory",
   LangOpts<"AtomicFineGrainedMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on fine-grained memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on fine-grained memory">>;
 
 defm atomic_ignore_denormal_mode : BoolFOption<"atomic-ignore-denormal-mode",
   LangOpts<"AtomicIgnoreDenormalMode">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "Allow">,
-  NegFlag<SetFalse, [], [ClangOption], "Disallow">,
-  BothFlags<[], [ClangOption], " atomic operations to ignore denormal mode">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "Allow">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Disallow">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations to ignore denormal mode">>;
 
 defm memory_profile : OptInCC1FFlag<"memory-profile", "Enable", "Disable", " heap memory profiling">;
 def fmemory_profile_EQ : Joined<["-"], "fmemory-profile=">,
@@ -5314,9 +5314,9 @@ defm amdgpu_precise_memory_op
                   " precise memory mode (AMDGPU only)">;
 
 def munsafe_fp_atomics : Flag<["-"], "munsafe-fp-atomics">,
-  Visibility<[ClangOption, CC1Option]>, Alias<fatomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>, Alias<fatomic_ignore_denormal_mode>;
 def mno_unsafe_fp_atomics : Flag<["-"], "mno-unsafe-fp-atomics">,
-  Visibility<[ClangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, FlangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
 
 def faltivec : Flag<["-"], "faltivec">, Group<f_Group>;
 def fno_altivec : Flag<["-"], "fno-altivec">, Group<f_Group>;
diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 002d8d158abd4..26256fd775f11 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic Control Options for AMD GPU
+  bool amdgpuIgnoreDenormalMode = false;
+  bool amdgpuRemoteMemory = false;
+  bool amdgpuFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 2df14f83c11e1..ac563bcc402c7 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,10 +58,24 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and Getters for atomic control options
+void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
+bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..00f854b905c72 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,6 +496,18 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
+  if (llvm::Triple(opts.triple).isAMDGPU()) {
+    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
+        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+    opts.amdgpuFineGrainedMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_fine_grained_memory,
+        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+    opts.amdgpuRemoteMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_remote_memory,
+        clang::driver::options::OPT_fno_atomic_remote_memory, false);
+  }
+
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
     opts.cpu = a->getValue();
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..acbfad589dfbb 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,6 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
+  if (targetOpts.amdgpuIgnoreDenormalMode)
+    fir::setAmdgpuIgnoreDenormalMode(*module);
+  if (targetOpts.amdgpuFineGrainedMemory)
+    fir::setAmdgpuFineGrainedMemory(*module);
+  if (targetOpts.amdgpuRemoteMemory)
+    fir::setAmdgpuRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 01c0be66d1ecc..b961793dbdfd5 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,6 +88,46 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *amdgpuIgnoreDenormalModeName =
+    "fir.amdgpu.ignore.denormal.mode";
+void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
+    return true;
+  return false;
+}
+
+static constexpr const char *amdgpuFineGrainedMemoryName =
+    "fir.amdgpu.fine.grained.memory";
+void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
+    return true;
+  return false;
+}
+static constexpr const char *amdgpuRemoteMemoryName =
+    "fir.amdgpu.remote.memory";
+void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
+    return true;
+  return false;
+}
+
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
   if (auto attr = mod->getAttrOfType<mlir::StringAttr>(tuneCpuName))
     return attr.getValue();
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
new file mode 100644
index 0000000000000..1eb0f617f365e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o - | FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o - | FileCheck -check-prefix=IGNORE-DENORMAL %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o - | FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o - | FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, B, threads
+    threads = 128
+    A = 0
+    B = 0
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic capture
+        A = A + B
+        B = A
+    !$omp end atomic
+    !$omp end target parallel
+end program test
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 704d0b2220e8a..84887dfd2c6f9 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -54,6 +54,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
+      clausesArgs);
 
   // Override region definition.
   let regions = (region SizedRegion<1>:$region);
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..e377b7aab3e43 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1562,6 +1562,15 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
     omp.yield(%newval : i32)
   }
 
+  // CHECK: omp.atomic.update %[[X]] : memref<i32> {
+  // CHECK-NEXT: (%[[XVAL:.*]]: i32):
+  // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
+  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  omp.atomic.update %x : memref<i32> {
+  ^bb0(%xval:i32):
+    omp.yield(%const:i32)
+  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+
   return
 }
 

>From 63d3b489ce5d1210172f8bc1d3b18d81ff52886d Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:58:07 -0500
Subject: [PATCH 2/5] R2: Making Atomic control non-optional

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   2 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |   2 +-
 mlir/test/Dialect/OpenMP/canonicalize.mlir    |  10 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  46 +++----
 mlir/test/Dialect/OpenMP/ops.mlir             | 127 +++++++++---------
 .../Target/LLVMIR/openmp-llvm-invalid.mlir    |   8 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  98 +++++++-------
 .../openmp-target-nesting-in-host-ops.mlir    |   2 +-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |   2 +-
 9 files changed, 148 insertions(+), 149 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 375e35a146f78..399f7fa6e78f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          OptionalAttr<AtomicControlAttr>:$atomic_control),
+          AtomicControlAttr:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index d69de998346b5..0b2445d0a0d18 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -133,7 +133,7 @@ func.func @atomic_update() {
     %1 = arith.constant 1 : i32
     %2 = arith.addi %arg0, %1  : i32
     omp.yield(%2 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 llvm.mlir.global internal @_QFsEc() : i32 {
diff --git a/mlir/test/Dialect/OpenMP/canonicalize.mlir b/mlir/test/Dialect/OpenMP/canonicalize.mlir
index de6c931ecc5fd..2492010c4d3c7 100644
--- a/mlir/test/Dialect/OpenMP/canonicalize.mlir
+++ b/mlir/test/Dialect/OpenMP/canonicalize.mlir
@@ -4,7 +4,7 @@ func.func @update_no_op(%x : memref<i32>) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -17,7 +17,7 @@ func.func @update_write_op(%x : memref<i32>, %value: i32) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%value : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -33,7 +33,7 @@ func.func @update_normal(%x : memref<i32>, %value: i32) {
   ^bb0(%xval : i32):
     %newval = arith.addi %xval, %value : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -50,7 +50,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.addi %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -65,7 +65,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.muli %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..afc3830286f30 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1007,7 +1007,7 @@ func.func @omp_atomic_update1(%x: memref<i32>, %expr: f32) {
   ^bb0(%xval: f32):
     %newval = llvm.fadd %xval, %expr : f32
     omp.yield (%newval : f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1020,7 +1020,7 @@ func.func @omp_atomic_update2(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.terminator
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1032,7 +1032,7 @@ func.func @omp_atomic_update3(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1044,7 +1044,7 @@ func.func @omp_atomic_update4(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1056,7 +1056,7 @@ func.func @omp_atomic_update5(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1068,7 +1068,7 @@ func.func @omp_atomic_update6(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval, %expr : i32, i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1080,7 +1080,7 @@ func.func @omp_atomic_update7(%x: memref<i32>, %expr: i32, %y: f32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%y: f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1092,7 +1092,7 @@ func.func @omp_atomic_update8(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32, %tmp: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1104,7 +1104,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1116,7 +1116,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1128,7 +1128,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1164,12 +1164,12 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.update %x : memref<i32> {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1197,7 +1197,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1212,7 +1212,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.write %x = %expr : memref<i32>, i32
     omp.terminator
   }
@@ -1240,7 +1240,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %y : memref<i32>, memref<i32>, i32
     omp.terminator
   }
@@ -1256,7 +1256,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
 }
@@ -1281,7 +1281,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1296,7 +1296,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1311,7 +1311,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1326,7 +1326,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1341,7 +1341,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1356,7 +1356,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x memory_order(seq_cst) : memref<i32>, memref<i32>, i32
   }
   return
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index e377b7aab3e43..5aa7befb2dc26 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
@@ -1380,57 +1380,57 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i1):
     %newval = llvm.and %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.shl %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.intr.smax(%[[XVAL]], %[[EXPR]]) : (i32, i32) -> i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.intr.smax(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.icmp "eq" %[[XVAL]], %[[EXPRBOOL]] : i1
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i1)
-  // }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %xBool : memref<i1> {
   ^bb0(%xval: i1):
     %newval = llvm.icmp "eq" %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%xval:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   %const = arith.constant 42 : i32
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1440,7 +1440,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1450,7 +1450,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1460,7 +1460,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1470,7 +1470,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1480,7 +1480,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1490,7 +1490,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1500,7 +1500,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1510,7 +1510,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1520,7 +1520,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1530,7 +1530,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(release) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1540,7 +1540,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(relaxed) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1550,7 +1550,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1560,17 +1560,16 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
-
+  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   return
 }
 
@@ -1582,7 +1581,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture{
@@ -1590,7 +1589,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   // CHECK: omp.atomic.capture {
@@ -1599,7 +1598,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: }
   omp.atomic.capture{
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
@@ -1607,7 +1606,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
@@ -1623,7 +1622,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(none) {
@@ -1631,7 +1630,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1640,7 +1639,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended) {
@@ -1648,7 +1647,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1657,7 +1656,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended) {
@@ -1665,7 +1664,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1674,7 +1673,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(nonspeculative) {
@@ -1682,7 +1681,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1691,7 +1690,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(speculative) {
@@ -1699,7 +1698,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1708,7 +1707,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, nonspeculative) {
@@ -1716,7 +1715,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1725,7 +1724,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, nonspeculative) {
@@ -1733,7 +1732,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1742,7 +1741,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, speculative) {
@@ -1750,7 +1749,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1759,7 +1758,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) {
@@ -1767,7 +1766,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1776,7 +1775,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(seq_cst) {
@@ -1784,7 +1783,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1793,7 +1792,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acq_rel) {
@@ -1801,7 +1800,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1810,7 +1809,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acquire) {
@@ -1818,7 +1817,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1827,7 +1826,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(release) {
@@ -1835,7 +1834,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1844,7 +1843,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(relaxed) {
@@ -1852,7 +1851,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1861,7 +1860,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) memory_order(seq_cst) {
@@ -1869,7 +1868,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -2653,14 +2652,14 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
   // CHECK: omp.atomic.update %[[x]] : !llvm.ptr {
   // CHECK-NEXT: ^{{[[:alnum:]]+}}(%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   %val = llvm.load %x : !llvm.ptr -> i32
   omp.atomic.write %v = %val : !llvm.ptr, i32
   omp.atomic.update %x : !llvm.ptr {
     ^bb0(%xval: i32):
       omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index 41bc5c4ba525f..00f53864b5fa7 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -9,7 +9,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
     %t2 = llvm.sdiv %t1, %expr : i32
     %newval = llvm.add %xval, %t2 : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -24,7 +24,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.mul %expr, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -41,7 +41,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
     ^bb0(%xval: i32):
       %newval = llvm.mul %expr, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
@@ -59,7 +59,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
       %t2 = llvm.sdiv %t1, %expr : i32
       %newval = llvm.add %xval, %t2 : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 32f0ba5b105ff..ab7f756bf95f9 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,13 +1462,13 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1563,7 +1563,7 @@ llvm.func @_QPomp_atomic_update_complex() {
       %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
       %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
       omp.yield(%16 : !llvm.struct<(f32, f32)>)
-    }
+    } {atomic_control = #omp.atomic_control<>}
    llvm.return
 }
 
@@ -1623,7 +1623,7 @@ llvm.func @_QPomp_atomic_capture_complex() {
         %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
         %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
         omp.yield(%20 : !llvm.struct<(f32, f32)>)
-      }
+      } {atomic_control = #omp.atomic_control<>}
       omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.ptr, !llvm.struct<(f32, f32)>
     }
     llvm.return
@@ -1665,7 +1665,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %expr, %xval : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1683,7 +1683,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1701,7 +1701,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
   // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
   // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
@@ -1710,7 +1710,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1732,7 +1732,7 @@ llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
     %2 = llvm.fadd %1, %0 : f32
     %3 = llvm.fptosi %2 : f32 to i32
     omp.yield(%3 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1751,7 +1751,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1763,7 +1763,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1775,7 +1775,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1787,7 +1787,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1799,7 +1799,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1814,7 +1814,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1829,7 +1829,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1844,7 +1844,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1859,7 +1859,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1874,7 +1874,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1889,7 +1889,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1904,7 +1904,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1919,7 +1919,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1934,7 +1934,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1949,7 +1949,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1964,7 +1964,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -1979,7 +1979,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -2001,7 +2001,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw sub ptr %[[x]], i32 %[[expr]] monotonic
@@ -2012,7 +2012,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw and ptr %[[x]], i32 %[[expr]] monotonic
@@ -2023,7 +2023,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw or ptr %[[x]], i32 %[[expr]] monotonic
@@ -2034,7 +2034,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw xor ptr %[[x]], i32 %[[expr]] monotonic
@@ -2045,7 +2045,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2060,7 +2060,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2075,7 +2075,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2090,7 +2090,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2105,7 +2105,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2120,7 +2120,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2135,7 +2135,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2150,7 +2150,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2165,7 +2165,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2180,7 +2180,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2195,7 +2195,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2211,7 +2211,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2227,7 +2227,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -2265,7 +2265,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acquire
@@ -2276,7 +2276,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] release
@@ -2287,7 +2287,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
@@ -2298,7 +2298,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acq_rel
@@ -2309,7 +2309,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -3055,7 +3055,7 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
index cbf273b887bc7..a9a4ffd533136 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
@@ -110,7 +110,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
 
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 29725a02c075a..0ec244bf28f33 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -19,7 +19,7 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   ^bb0(%arg0: i32):
     %result = llvm.add %arg0, %expr : i32
     omp.yield(%result : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   llvm.return
 }

>From 0b1b23ac98e9486d79b69092698623782ee96921 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 3/5] R3: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h       |  2 +-
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  4 ++--
 flang/lib/Optimizer/Dialect/Support/FIRContext.cpp | 14 +++-----------
 flang/test/Fir/convert-to-llvm-openmp-and-fir.fir  |  4 ++--
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td |  3 ++-
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 26256fd775f11..dd29b4e3e8b08 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,7 +54,7 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic Control Options for AMD GPU
+  /// Atomic control options for AMD gpu
   bool amdgpuIgnoreDenormalMode = false;
   bool amdgpuRemoteMemory = false;
   bool amdgpuFineGrainedMemory = false;
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac563bcc402c7..ac5fd33874db8 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,7 +58,7 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and Getters for atomic control options
+// Setters and getters for atomic control options.
 void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
@@ -70,7 +70,7 @@ bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index b961793dbdfd5..297a4c7a39e44 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -96,10 +96,7 @@ void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
 }
 
 static constexpr const char *amdgpuFineGrainedMemoryName =
@@ -110,10 +107,7 @@ void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
 }
 static constexpr const char *amdgpuRemoteMemoryName =
     "fir.amdgpu.remote.memory";
@@ -123,9 +117,7 @@ void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 24e5cad84b709..47c00d489c200 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -619,7 +619,7 @@ func.func @_QPs() {
     %c1_i32 = arith.constant 1 : i32
     %1 = arith.addi %arg0, %c1_i32 : i32
     omp.yield(%1 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 fir.global internal @_QFsEc : i32 {
@@ -634,7 +634,7 @@ fir.global internal @_QFsEc : i32 {
 // CHECK:      %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:      %[[OUT_VAL:.*]] = llvm.add %[[IN_VAL]], %[[CONST_1]]  : i32
 // CHECK:      omp.yield(%[[OUT_VAL]] : i32)
-// CHECK:    }
+// CHECK:    } {atomic_control = #omp.atomic_control<>}
 // CHECK:    llvm.return
 // CHECK:  }
 // CHECK:  llvm.mlir.global internal @[[GLOBAL]]() {{.*}} : i32 {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 84887dfd2c6f9..61861173ae875 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -58,7 +58,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From 565d104ba013acd7f7e64cc989e774f0f1f34e97 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Mon, 23 Jun 2025 12:14:19 -0500
Subject: [PATCH 4/5] R4: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  6 ++--
 .../Optimizer/Dialect/Support/FIRContext.h    | 18 ++++------
 flang/lib/Frontend/CompilerInvocation.cpp     | 20 +++++------
 flang/lib/Lower/Bridge.cpp                    | 12 +++----
 flang/lib/Lower/OpenMP/Atomic.cpp             |  9 ++++-
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 34 +++++++++----------
 .../Lower/OpenMP/atomic-control-options.f90   | 16 ++++-----
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     |  6 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  4 +--
 9 files changed, 61 insertions(+), 64 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index dd29b4e3e8b08..6ef5338acdc6e 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -55,9 +55,9 @@ class TargetOptions {
   bool asmVerbose = false;
 
   /// Atomic control options for AMD gpu
-  bool amdgpuIgnoreDenormalMode = false;
-  bool amdgpuRemoteMemory = false;
-  bool amdgpuFineGrainedMemory = false;
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac5fd33874db8..273b4683b6d9c 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -59,23 +59,17 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
 // Setters and getters for atomic control options.
-void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
-bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+void setIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getIgnoreDenormalMode(mlir::ModuleOp mod);
+void setFineGrainedMemory(mlir::ModuleOp mod);
+bool getFineGrainedMemory(mlir::ModuleOp mod);
+void setRemoteMemory(mlir::ModuleOp mod);
+bool getRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set atomic control options for AMD gpu.
-void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
-                                   bool amdgpuIgnoreDenormalMode,
-                                   bool amdgpuNoFineGrainedMemory,
-                                   bool amdgpuNoRemoteMemory);
-
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 00f854b905c72..63170a6df6960 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,17 +496,15 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  if (llvm::Triple(opts.triple).isAMDGPU()) {
-    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
-        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
-        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-    opts.amdgpuFineGrainedMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_fine_grained_memory,
-        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-    opts.amdgpuRemoteMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_remote_memory,
-        clang::driver::options::OPT_fno_atomic_remote_memory, false);
-  }
+  opts.ignoreDenormalMode = args.hasFlag(
+      clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+      clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+  opts.fineGrainedMemory = args.hasFlag(
+      clang::driver::options::OPT_fatomic_fine_grained_memory,
+      clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+  opts.remoteMemory =
+      args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
+                   clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index acbfad589dfbb..af6e72aa8cd10 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.amdgpuIgnoreDenormalMode)
-    fir::setAmdgpuIgnoreDenormalMode(*module);
-  if (targetOpts.amdgpuFineGrainedMemory)
-    fir::setAmdgpuFineGrainedMemory(*module);
-  if (targetOpts.amdgpuRemoteMemory)
-    fir::setAmdgpuRemoteMemory(*module);
+  if (targetOpts.ignoreDenormalMode)
+    fir::setIgnoreDenormalMode(*module);
+  if (targetOpts.fineGrainedMemory)
+    fir::setFineGrainedMemory(*module);
+  if (targetOpts.remoteMemory)
+    fir::setRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 2ab91b239a3cc..cbc850d67cc4d 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -634,9 +634,16 @@ genAtomicUpdate(lower::AbstractConverter &converter,
     }
   }
 
+  mlir::ModuleOp module = builder.getModule();
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      mlir::omp::AtomicControlAttr::get(
+          builder.getContext(), fir::getAtomicIgnoreDenormalMode(module),
+          fir::getAtomicFineGrainedMemory(module),
+          fir::getAtomicRemoteMemory(module));
   builder.restoreInsertionPoint(atomicAt);
   auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+      loc, atomAddr, atomicControlAttr, hint,
+      makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 297a4c7a39e44..e3599204f5e2c 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,36 +88,34 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *amdgpuIgnoreDenormalModeName =
-    "fir.amdgpu.ignore.denormal.mode";
-void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
 }
 
-static constexpr const char *amdgpuFineGrainedMemoryName =
-    "fir.amdgpu.fine.grained.memory";
-void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
+void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
+bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
 }
-static constexpr const char *amdgpuRemoteMemoryName =
-    "fir.amdgpu.remote.memory";
-void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+static constexpr const char *remoteMemoryName = "fir.remote.memory";
+void fir::setRemoteMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
+bool fir::getRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
index 1eb0f617f365e..8f86a151b3713 100644
--- a/flang/test/Lower/OpenMP/atomic-control-options.f90
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -9,25 +9,25 @@ program test
     A = 0
     B = 0
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic
     A =  A + 1
     !$omp end target parallel
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic capture
         A = A + B
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 61861173ae875..0fefe7e017e94 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -62,9 +62,9 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 5aa7befb2dc26..9a72ddfc2989d 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1565,11 +1565,11 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   return
 }
 

>From f5a469317a1446d17bc0fb6feb10849d70718368 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 11:26:03 -0500
Subject: [PATCH 5/5] R5: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  8 +--
 .../Optimizer/Dialect/Support/FIRContext.h    | 19 ++++---
 flang/lib/Frontend/CompilerInvocation.cpp     |  6 +--
 flang/lib/Lower/Bridge.cpp                    |  9 ++--
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 53 ++++++++++++-------
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 30 +++++------
 6 files changed, 71 insertions(+), 54 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 6ef5338acdc6e..f6e5634d5a995 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,10 +54,10 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic control options for AMD gpu
-  bool ignoreDenormalMode = false;
-  bool remoteMemory = false;
-  bool fineGrainedMemory = false;
+  /// Atomic control options
+  bool atomicIgnoreDenormalMode = false;
+  bool atomicRemoteMemory = false;
+  bool atomicFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 273b4683b6d9c..c0b852ccaf83a 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,13 +58,18 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and getters for atomic control options.
-void setIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getIgnoreDenormalMode(mlir::ModuleOp mod);
-void setFineGrainedMemory(mlir::ModuleOp mod);
-bool getFineGrainedMemory(mlir::ModuleOp mod);
-void setRemoteMemory(mlir::ModuleOp mod);
-bool getRemoteMemory(mlir::ModuleOp mod);
+/// Sets that Denormal Mode can be ignored for lowering of floating point atomic operations.
+void setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value);
+/// Gets whether Denormal Mode can be ignored or not for lowering of floating point atomic operations.
+bool getAtomicIgnoreDenormalMode(mlir::ModuleOp mod);
+/// Sets that fine grained memory is used for lowering of atomic operations.
+void setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether fine grained memory is used or not for lowering of atomic operations.
+bool getAtomicFineGrainedMemory(mlir::ModuleOp mod);
+/// Sets that remote memory is used for lowering of atomic operations.
+void setAtomicRemoteMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether remote memory is used or not for lowering of atomic operations.
+bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 63170a6df6960..39021207e897e 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,13 +496,13 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  opts.ignoreDenormalMode = args.hasFlag(
+  opts.atomicIgnoreDenormalMode = args.hasFlag(
       clang::driver::options::OPT_fatomic_ignore_denormal_mode,
       clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-  opts.fineGrainedMemory = args.hasFlag(
+  opts.atomicFineGrainedMemory = args.hasFlag(
       clang::driver::options::OPT_fatomic_fine_grained_memory,
       clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-  opts.remoteMemory =
+  opts.atomicRemoteMemory =
       args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
                    clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index af6e72aa8cd10..f8185f752b8e1 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,9 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.ignoreDenormalMode)
-    fir::setIgnoreDenormalMode(*module);
-  if (targetOpts.fineGrainedMemory)
-    fir::setFineGrainedMemory(*module);
-  if (targetOpts.remoteMemory)
-    fir::setRemoteMemory(*module);
+  fir::setAtomicIgnoreDenormalMode(*module, targetOpts.atomicIgnoreDenormalMode);
+  fir::setAtomicFineGrainedMemory(*module, targetOpts.atomicFineGrainedMemory);
+  fir::setAtomicRemoteMemory(*module, targetOpts.atomicRemoteMemory);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index e3599204f5e2c..885efa653fa23 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,34 +88,49 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *ignoreDenormalModeName =
-    "fir.ignore.denormal.mode";
-void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicIgnoreDenormalModeName =
+    "fir.atomic_ignore_denormal_mode";
+void fir::setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicIgnoreDenormalModeName))
+      mod->removeAttr(atomicIgnoreDenormalModeName);
+  }
 }
 
-bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+bool fir::getAtomicIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicIgnoreDenormalModeName);
 }
 
-static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
-void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicFineGrainedMemoryName = "fir.atomic_fine_grained_memory";
+void fir::setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicFineGrainedMemoryName))
+      mod->removeAttr(atomicFineGrainedMemoryName);
+  }
 }
 
-bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
+bool fir::getAtomicFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicFineGrainedMemoryName);
 }
-static constexpr const char *remoteMemoryName = "fir.remote.memory";
-void fir::setRemoteMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicRemoteMemoryName = "fir.atomic_remote_memory";
+void fir::setAtomicRemoteMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicRemoteMemoryName))
+      mod->removeAttr(atomicRemoteMemoryName);
+  }
 }
 
-bool fir::getRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
+bool fir::getAtomicRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 0fefe7e017e94..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -22,6 +22,21 @@ class OpenMP_Attr<string name, string attrMnemonic, list<Trait> traits = [],
   let mnemonic = attrMnemonic;
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // DeclareTargetAttr
 //===----------------------------------------------------------------------===//
@@ -54,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//


From flang-commits at lists.llvm.org  Wed Jul  2 11:54:16 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 02 Jul 2025 11:54:16 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][preprocessor] fix use of
 bitwise-and for logical-and (PR #146758)
Message-ID: <llvm.org/llvm/llvm-project/pull/146758@github.com>


https://github.com/akuhlens created https://github.com/llvm/llvm-project/pull/146758

The preprocessor used bitwise and to implement logical, this is a bug.

towards #146362

>From 3631e247920d8e1c5a1ef186e1abcd91d24dce9e Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Wed, 2 Jul 2025 11:51:05 -0700
Subject: [PATCH] initial commit

---
 flang/lib/Parser/preprocessor.cpp    |  9 +++++++--
 flang/test/Parser/issue-146362.2.f90 | 11 +++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Parser/issue-146362.2.f90

diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index ef24c704db885..d79b7d41da83c 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -1276,14 +1276,19 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
       left = right >= 64 ? 0 : left >> right;
       break;
     case BITAND:
-    case AND:
       left = left & right;
       break;
+    // (bool)(1 && 2) != (bool)(1 & 2), we can't use bitwise AND here.
+    case AND:
+      left = left && right;
+      break;
     case BITXOR:
       left = left ^ right;
       break;
-    case BITOR:
+    // (bool)(1 || 2) == (bool)(1 | 2), seems like we can use `|` here.
+    // But should we?
     case OR:
+    case BITOR:
       left = left | right;
       break;
     case LT:
diff --git a/flang/test/Parser/issue-146362.2.f90 b/flang/test/Parser/issue-146362.2.f90
new file mode 100644
index 0000000000000..db27115ec15d0
--- /dev/null
+++ b/flang/test/Parser/issue-146362.2.f90
@@ -0,0 +1,11 @@
+!RUN: %flang_fc1 -cpp -fdebug-unparse %s | FileCheck %s
+PROGRAM P
+#if (1 && 2)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+END PROGRAM
+


From flang-commits at lists.llvm.org  Wed Jul  2 11:54:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 11:54:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][preprocessor] fix use of
 bitwise-and for logical-and (PR #146758)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146758@github.com>
Message-ID: <68658076.630a0220.30a55b.2b31@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: Andre Kuhlenschmidt (akuhlens)

<details>
<summary>Changes</summary>

The preprocessor used bitwise and to implement logical, this is a bug.

towards #<!-- -->146362

---
Full diff: https://github.com/llvm/llvm-project/pull/146758.diff


2 Files Affected:

- (modified) flang/lib/Parser/preprocessor.cpp (+7-2) 
- (added) flang/test/Parser/issue-146362.2.f90 (+11) 


``````````diff
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index ef24c704db885..d79b7d41da83c 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -1276,14 +1276,19 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
       left = right >= 64 ? 0 : left >> right;
       break;
     case BITAND:
-    case AND:
       left = left & right;
       break;
+    // (bool)(1 && 2) != (bool)(1 & 2), we can't use bitwise AND here.
+    case AND:
+      left = left && right;
+      break;
     case BITXOR:
       left = left ^ right;
       break;
-    case BITOR:
+    // (bool)(1 || 2) == (bool)(1 | 2), seems like we can use `|` here.
+    // But should we?
     case OR:
+    case BITOR:
       left = left | right;
       break;
     case LT:
diff --git a/flang/test/Parser/issue-146362.2.f90 b/flang/test/Parser/issue-146362.2.f90
new file mode 100644
index 0000000000000..db27115ec15d0
--- /dev/null
+++ b/flang/test/Parser/issue-146362.2.f90
@@ -0,0 +1,11 @@
+!RUN: %flang_fc1 -cpp -fdebug-unparse %s | FileCheck %s
+PROGRAM P
+#if (1 && 2)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+END PROGRAM
+

``````````

</details>


https://github.com/llvm/llvm-project/pull/146758

From flang-commits at lists.llvm.org  Wed Jul  2 11:55:57 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 02 Jul 2025 11:55:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][preprocessor] fix use of
 bitwise-and for logical-and (PR #146758)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146758@github.com>
Message-ID: <686580bd.630a0220.307c59.26ce@mx.google.com>


================
@@ -1276,14 +1276,19 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
       left = right >= 64 ? 0 : left >> right;
       break;
     case BITAND:
-    case AND:
       left = left & right;
       break;
+    // (bool)(1 && 2) != (bool)(1 & 2), we can't use bitwise AND here.
+    case AND:
+      left = left && right;
+      break;
     case BITXOR:
       left = left ^ right;
       break;
-    case BITOR:
+    // (bool)(1 || 2) == (bool)(1 | 2), seems like we can use `|` here.
+    // But should we?
----------------
klausler wrote:

We should not.  The result has to be 1 or 0.

https://github.com/llvm/llvm-project/pull/146758

From flang-commits at lists.llvm.org  Wed Jul  2 12:03:09 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 02 Jul 2025 12:03:09 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
Message-ID: <llvm.org/llvm/llvm-project/pull/146759@github.com>


https://github.com/klausler created https://github.com/llvm/llvm-project/pull/146759

When using -fhermetic-module-files it's possible for a derived type to have multiple distinct definition sites that are being compared for being the same type, as in argument association.  Accept them as being the same type so long as they have the same names, the same module names, and identical definitions.

>From a75ff58742b3e7e209c54cda2b842b3689ff5828 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Wed, 2 Jul 2025 11:33:09 -0700
Subject: [PATCH] [flang] Unify derived types in distinct module files

When using -fhermetic-module-files it's possible for a derived type
to have multiple distinct definition sites that are being compared
for being the same type, as in argument association.  Accept
them as being the same type so long as they have the same names,
the same module names, and identical definitions.
---
 flang/lib/Evaluate/type.cpp      | 47 +++++++++++++++++++++-----------
 flang/test/Semantics/bug1092.F90 | 26 ++++++++++++++++++
 2 files changed, 57 insertions(+), 16 deletions(-)
 create mode 100644 flang/test/Semantics/bug1092.F90

diff --git a/flang/lib/Evaluate/type.cpp b/flang/lib/Evaluate/type.cpp
index 5b5f3c2cd0cf0..99dc8b1e5c676 100644
--- a/flang/lib/Evaluate/type.cpp
+++ b/flang/lib/Evaluate/type.cpp
@@ -299,13 +299,18 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &,
 
 // F2023 7.5.3.2
 static bool AreSameComponent(const semantics::Symbol &x,
-    const semantics::Symbol &y, bool ignoreSequence,
+    const semantics::Symbol &y, bool ignoreSequence, bool sameModuleName,
     SetOfDerivedTypePairs &inProgress) {
   if (x.attrs() != y.attrs()) {
     return false;
   }
-  if (x.attrs().test(semantics::Attr::PRIVATE)) {
-    return false;
+  if (x.attrs().test(semantics::Attr::PRIVATE) ||
+      y.attrs().test(semantics::Attr::PRIVATE)) {
+    if (!sameModuleName ||
+        x.attrs().test(semantics::Attr::PRIVATE) !=
+            y.attrs().test(semantics::Attr::PRIVATE)) {
+      return false;
+    }
   }
   if (x.size() && y.size()) {
     if (x.offset() != y.offset() || x.size() != y.size()) {
@@ -482,9 +487,20 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
           ySymbol.attrs().test(semantics::Attr::BIND_C)) {
     return false;
   }
-  if (!ignoreSequence && !(xDetails.sequence() && yDetails.sequence()) &&
-      !(xSymbol.attrs().test(semantics::Attr::BIND_C) &&
-          ySymbol.attrs().test(semantics::Attr::BIND_C))) {
+  bool sameModuleName{false};
+  const semantics::Scope &xOwner{xSymbol.owner()};
+  const semantics::Scope &yOwner{ySymbol.owner()};
+  if (xOwner.IsModule() && yOwner.IsModule()) {
+    if (auto xModuleName{xOwner.GetName()}) {
+      if (auto yModuleName{yOwner.GetName()}) {
+        if (*xModuleName == *yModuleName) {
+          sameModuleName = true;
+        }
+      }
+    }
+  }
+  if (!sameModuleName && !ignoreSequence && !xDetails.sequence() &&
+      !xSymbol.attrs().test(semantics::Attr::BIND_C)) {
     // PGI does not enforce this requirement; all other Fortran
     // compilers do with a hard error when violations are caught.
     return false;
@@ -502,9 +518,10 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
     const auto xLookup{xSymbol.scope()->find(*xComponentName)};
     const auto yLookup{ySymbol.scope()->find(*yComponentName)};
     if (xLookup == xSymbol.scope()->end() ||
-        yLookup == ySymbol.scope()->end() ||
-        !AreSameComponent(
-            *xLookup->second, *yLookup->second, ignoreSequence, inProgress)) {
+        yLookup == ySymbol.scope()->end()) {
+      return false;
+    } else if (!AreSameComponent(*xLookup->second, *yLookup->second,
+                   ignoreSequence, sameModuleName, inProgress)) {
       return false;
     }
   }
@@ -576,17 +593,15 @@ static bool AreCompatibleTypes(const DynamicType &x, const DynamicType &y,
     const auto yLen{y.knownLength()};
     return x.kind() == y.kind() &&
         (ignoreLengths || !xLen || !yLen || *xLen == *yLen);
-  } else if (x.category() != TypeCategory::Derived) {
-    if (x.IsTypelessIntrinsicArgument()) {
-      return y.IsTypelessIntrinsicArgument();
-    } else {
-      return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
-    }
-  } else {
+  } else if (x.category() == TypeCategory::Derived) {
     const auto *xdt{GetDerivedTypeSpec(x)};
     const auto *ydt{GetDerivedTypeSpec(y)};
     return AreCompatibleDerivedTypes(
         xdt, ydt, x.IsPolymorphic(), ignoreTypeParameterValues, false);
+  } else if (x.IsTypelessIntrinsicArgument()) {
+    return y.IsTypelessIntrinsicArgument();
+  } else {
+    return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
   }
 }
 
diff --git a/flang/test/Semantics/bug1092.F90 b/flang/test/Semantics/bug1092.F90
new file mode 100644
index 0000000000000..ef108733fd78b
--- /dev/null
+++ b/flang/test/Semantics/bug1092.F90
@@ -0,0 +1,26 @@
+!RUN: rm -rf %t && mkdir -p %t
+!RUN: %flang_fc1 -DWHICH=1 -fsyntax-only -J%t %s
+!RUN: %flang_fc1 -DWHICH=2 -fsyntax-only -fhermetic-module-files -I%t -J%t %s
+!RUN: %flang_fc1 -fsyntax-only -I%t %s 2>&1 | FileCheck --allow-empty %s
+!CHECK-NOT: error:
+
+#if WHICH == 1
+module bug1092a
+  type t
+  end type
+ contains
+  subroutine subr(x)
+    type(t) x
+  end
+end
+#elif WHICH == 2
+module bug1092b
+  use bug1092a, only: subr
+end
+#else
+use bug1092a, only: t
+use bug1092b, only: subr
+type(t) x
+call subr(x)
+end
+#endif


From flang-commits at lists.llvm.org  Wed Jul  2 12:03:42 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 12:03:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <6865828e.630a0220.e828.2d07@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Peter Klausler (klausler)

<details>
<summary>Changes</summary>

When using -fhermetic-module-files it's possible for a derived type to have multiple distinct definition sites that are being compared for being the same type, as in argument association.  Accept them as being the same type so long as they have the same names, the same module names, and identical definitions.

---
Full diff: https://github.com/llvm/llvm-project/pull/146759.diff


2 Files Affected:

- (modified) flang/lib/Evaluate/type.cpp (+31-16) 
- (added) flang/test/Semantics/bug1092.F90 (+26) 


``````````diff
diff --git a/flang/lib/Evaluate/type.cpp b/flang/lib/Evaluate/type.cpp
index 5b5f3c2cd0cf0..99dc8b1e5c676 100644
--- a/flang/lib/Evaluate/type.cpp
+++ b/flang/lib/Evaluate/type.cpp
@@ -299,13 +299,18 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &,
 
 // F2023 7.5.3.2
 static bool AreSameComponent(const semantics::Symbol &x,
-    const semantics::Symbol &y, bool ignoreSequence,
+    const semantics::Symbol &y, bool ignoreSequence, bool sameModuleName,
     SetOfDerivedTypePairs &inProgress) {
   if (x.attrs() != y.attrs()) {
     return false;
   }
-  if (x.attrs().test(semantics::Attr::PRIVATE)) {
-    return false;
+  if (x.attrs().test(semantics::Attr::PRIVATE) ||
+      y.attrs().test(semantics::Attr::PRIVATE)) {
+    if (!sameModuleName ||
+        x.attrs().test(semantics::Attr::PRIVATE) !=
+            y.attrs().test(semantics::Attr::PRIVATE)) {
+      return false;
+    }
   }
   if (x.size() && y.size()) {
     if (x.offset() != y.offset() || x.size() != y.size()) {
@@ -482,9 +487,20 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
           ySymbol.attrs().test(semantics::Attr::BIND_C)) {
     return false;
   }
-  if (!ignoreSequence && !(xDetails.sequence() && yDetails.sequence()) &&
-      !(xSymbol.attrs().test(semantics::Attr::BIND_C) &&
-          ySymbol.attrs().test(semantics::Attr::BIND_C))) {
+  bool sameModuleName{false};
+  const semantics::Scope &xOwner{xSymbol.owner()};
+  const semantics::Scope &yOwner{ySymbol.owner()};
+  if (xOwner.IsModule() && yOwner.IsModule()) {
+    if (auto xModuleName{xOwner.GetName()}) {
+      if (auto yModuleName{yOwner.GetName()}) {
+        if (*xModuleName == *yModuleName) {
+          sameModuleName = true;
+        }
+      }
+    }
+  }
+  if (!sameModuleName && !ignoreSequence && !xDetails.sequence() &&
+      !xSymbol.attrs().test(semantics::Attr::BIND_C)) {
     // PGI does not enforce this requirement; all other Fortran
     // compilers do with a hard error when violations are caught.
     return false;
@@ -502,9 +518,10 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
     const auto xLookup{xSymbol.scope()->find(*xComponentName)};
     const auto yLookup{ySymbol.scope()->find(*yComponentName)};
     if (xLookup == xSymbol.scope()->end() ||
-        yLookup == ySymbol.scope()->end() ||
-        !AreSameComponent(
-            *xLookup->second, *yLookup->second, ignoreSequence, inProgress)) {
+        yLookup == ySymbol.scope()->end()) {
+      return false;
+    } else if (!AreSameComponent(*xLookup->second, *yLookup->second,
+                   ignoreSequence, sameModuleName, inProgress)) {
       return false;
     }
   }
@@ -576,17 +593,15 @@ static bool AreCompatibleTypes(const DynamicType &x, const DynamicType &y,
     const auto yLen{y.knownLength()};
     return x.kind() == y.kind() &&
         (ignoreLengths || !xLen || !yLen || *xLen == *yLen);
-  } else if (x.category() != TypeCategory::Derived) {
-    if (x.IsTypelessIntrinsicArgument()) {
-      return y.IsTypelessIntrinsicArgument();
-    } else {
-      return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
-    }
-  } else {
+  } else if (x.category() == TypeCategory::Derived) {
     const auto *xdt{GetDerivedTypeSpec(x)};
     const auto *ydt{GetDerivedTypeSpec(y)};
     return AreCompatibleDerivedTypes(
         xdt, ydt, x.IsPolymorphic(), ignoreTypeParameterValues, false);
+  } else if (x.IsTypelessIntrinsicArgument()) {
+    return y.IsTypelessIntrinsicArgument();
+  } else {
+    return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
   }
 }
 
diff --git a/flang/test/Semantics/bug1092.F90 b/flang/test/Semantics/bug1092.F90
new file mode 100644
index 0000000000000..ef108733fd78b
--- /dev/null
+++ b/flang/test/Semantics/bug1092.F90
@@ -0,0 +1,26 @@
+!RUN: rm -rf %t && mkdir -p %t
+!RUN: %flang_fc1 -DWHICH=1 -fsyntax-only -J%t %s
+!RUN: %flang_fc1 -DWHICH=2 -fsyntax-only -fhermetic-module-files -I%t -J%t %s
+!RUN: %flang_fc1 -fsyntax-only -I%t %s 2>&1 | FileCheck --allow-empty %s
+!CHECK-NOT: error:
+
+#if WHICH == 1
+module bug1092a
+  type t
+  end type
+ contains
+  subroutine subr(x)
+    type(t) x
+  end
+end
+#elif WHICH == 2
+module bug1092b
+  use bug1092a, only: subr
+end
+#else
+use bug1092a, only: t
+use bug1092b, only: subr
+type(t) x
+call subr(x)
+end
+#endif

``````````

</details>


https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Wed Jul  2 12:56:35 2025
From: flang-commits at lists.llvm.org (Anchu Rajendran S via flang-commits)
Date: Wed, 02 Jul 2025 12:56:35 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [mlir]
 [mlir][OpenMP][llvm]Atomic control backend (PR #143769)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143769@github.com>
Message-ID: <68658ef3.170a0220.120e8a.58e2@mx.google.com>


https://github.com/anchuraj updated https://github.com/llvm/llvm-project/pull/143769

>From 4816fb9bc0761d8761798300bb419db0072c9d04 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 1/9] [flang][flang-driver] atomic control support

---
 clang/include/clang/Driver/Options.td         | 22 +++++-----
 flang/include/flang/Frontend/TargetOptions.h  |  5 +++
 .../Optimizer/Dialect/Support/FIRContext.h    | 14 +++++++
 flang/lib/Frontend/CompilerInvocation.cpp     | 12 ++++++
 flang/lib/Lower/Bridge.cpp                    |  6 +++
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 40 +++++++++++++++++++
 .../Lower/OpenMP/atomic-control-options.f90   | 37 +++++++++++++++++
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 14 +++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  8 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  9 +++++
 10 files changed, 153 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/atomic-control-options.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8e38f0062140e..c1c7929b0c15d 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2303,21 +2303,21 @@ def fsymbol_partition_EQ : Joined<["-"], "fsymbol-partition=">, Group<f_Group>,
 
 defm atomic_remote_memory : BoolFOption<"atomic-remote-memory",
   LangOpts<"AtomicRemoteMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on remote memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on remote memory">>;
 
 defm atomic_fine_grained_memory : BoolFOption<"atomic-fine-grained-memory",
   LangOpts<"AtomicFineGrainedMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on fine-grained memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on fine-grained memory">>;
 
 defm atomic_ignore_denormal_mode : BoolFOption<"atomic-ignore-denormal-mode",
   LangOpts<"AtomicIgnoreDenormalMode">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "Allow">,
-  NegFlag<SetFalse, [], [ClangOption], "Disallow">,
-  BothFlags<[], [ClangOption], " atomic operations to ignore denormal mode">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "Allow">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Disallow">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations to ignore denormal mode">>;
 
 defm memory_profile : OptInCC1FFlag<"memory-profile", "Enable", "Disable", " heap memory profiling">;
 def fmemory_profile_EQ : Joined<["-"], "fmemory-profile=">,
@@ -5314,9 +5314,9 @@ defm amdgpu_precise_memory_op
                   " precise memory mode (AMDGPU only)">;
 
 def munsafe_fp_atomics : Flag<["-"], "munsafe-fp-atomics">,
-  Visibility<[ClangOption, CC1Option]>, Alias<fatomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>, Alias<fatomic_ignore_denormal_mode>;
 def mno_unsafe_fp_atomics : Flag<["-"], "mno-unsafe-fp-atomics">,
-  Visibility<[ClangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, FlangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
 
 def faltivec : Flag<["-"], "faltivec">, Group<f_Group>;
 def fno_altivec : Flag<["-"], "fno-altivec">, Group<f_Group>;
diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 002d8d158abd4..26256fd775f11 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic Control Options for AMD GPU
+  bool amdgpuIgnoreDenormalMode = false;
+  bool amdgpuRemoteMemory = false;
+  bool amdgpuFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 2df14f83c11e1..ac563bcc402c7 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,10 +58,24 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and Getters for atomic control options
+void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
+bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..00f854b905c72 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,6 +496,18 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
+  if (llvm::Triple(opts.triple).isAMDGPU()) {
+    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
+        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+    opts.amdgpuFineGrainedMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_fine_grained_memory,
+        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+    opts.amdgpuRemoteMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_remote_memory,
+        clang::driver::options::OPT_fno_atomic_remote_memory, false);
+  }
+
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
     opts.cpu = a->getValue();
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..acbfad589dfbb 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,6 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
+  if (targetOpts.amdgpuIgnoreDenormalMode)
+    fir::setAmdgpuIgnoreDenormalMode(*module);
+  if (targetOpts.amdgpuFineGrainedMemory)
+    fir::setAmdgpuFineGrainedMemory(*module);
+  if (targetOpts.amdgpuRemoteMemory)
+    fir::setAmdgpuRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 01c0be66d1ecc..b961793dbdfd5 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,6 +88,46 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *amdgpuIgnoreDenormalModeName =
+    "fir.amdgpu.ignore.denormal.mode";
+void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
+    return true;
+  return false;
+}
+
+static constexpr const char *amdgpuFineGrainedMemoryName =
+    "fir.amdgpu.fine.grained.memory";
+void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
+    return true;
+  return false;
+}
+static constexpr const char *amdgpuRemoteMemoryName =
+    "fir.amdgpu.remote.memory";
+void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
+    return true;
+  return false;
+}
+
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
   if (auto attr = mod->getAttrOfType<mlir::StringAttr>(tuneCpuName))
     return attr.getValue();
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
new file mode 100644
index 0000000000000..1eb0f617f365e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o - | FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o - | FileCheck -check-prefix=IGNORE-DENORMAL %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o - | FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o - | FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, B, threads
+    threads = 128
+    A = 0
+    B = 0
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic capture
+        A = A + B
+        B = A
+    !$omp end atomic
+    !$omp end target parallel
+end program test
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 704d0b2220e8a..84887dfd2c6f9 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -54,6 +54,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
+      clausesArgs);
 
   // Override region definition.
   let regions = (region SizedRegion<1>:$region);
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..e377b7aab3e43 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1562,6 +1562,15 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
     omp.yield(%newval : i32)
   }
 
+  // CHECK: omp.atomic.update %[[X]] : memref<i32> {
+  // CHECK-NEXT: (%[[XVAL:.*]]: i32):
+  // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
+  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  omp.atomic.update %x : memref<i32> {
+  ^bb0(%xval:i32):
+    omp.yield(%const:i32)
+  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+
   return
 }
 

>From 63d3b489ce5d1210172f8bc1d3b18d81ff52886d Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:58:07 -0500
Subject: [PATCH 2/9] R2: Making Atomic control non-optional

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   2 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |   2 +-
 mlir/test/Dialect/OpenMP/canonicalize.mlir    |  10 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  46 +++----
 mlir/test/Dialect/OpenMP/ops.mlir             | 127 +++++++++---------
 .../Target/LLVMIR/openmp-llvm-invalid.mlir    |   8 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  98 +++++++-------
 .../openmp-target-nesting-in-host-ops.mlir    |   2 +-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |   2 +-
 9 files changed, 148 insertions(+), 149 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 375e35a146f78..399f7fa6e78f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          OptionalAttr<AtomicControlAttr>:$atomic_control),
+          AtomicControlAttr:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index d69de998346b5..0b2445d0a0d18 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -133,7 +133,7 @@ func.func @atomic_update() {
     %1 = arith.constant 1 : i32
     %2 = arith.addi %arg0, %1  : i32
     omp.yield(%2 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 llvm.mlir.global internal @_QFsEc() : i32 {
diff --git a/mlir/test/Dialect/OpenMP/canonicalize.mlir b/mlir/test/Dialect/OpenMP/canonicalize.mlir
index de6c931ecc5fd..2492010c4d3c7 100644
--- a/mlir/test/Dialect/OpenMP/canonicalize.mlir
+++ b/mlir/test/Dialect/OpenMP/canonicalize.mlir
@@ -4,7 +4,7 @@ func.func @update_no_op(%x : memref<i32>) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -17,7 +17,7 @@ func.func @update_write_op(%x : memref<i32>, %value: i32) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%value : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -33,7 +33,7 @@ func.func @update_normal(%x : memref<i32>, %value: i32) {
   ^bb0(%xval : i32):
     %newval = arith.addi %xval, %value : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -50,7 +50,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.addi %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -65,7 +65,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.muli %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..afc3830286f30 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1007,7 +1007,7 @@ func.func @omp_atomic_update1(%x: memref<i32>, %expr: f32) {
   ^bb0(%xval: f32):
     %newval = llvm.fadd %xval, %expr : f32
     omp.yield (%newval : f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1020,7 +1020,7 @@ func.func @omp_atomic_update2(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.terminator
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1032,7 +1032,7 @@ func.func @omp_atomic_update3(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1044,7 +1044,7 @@ func.func @omp_atomic_update4(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1056,7 +1056,7 @@ func.func @omp_atomic_update5(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1068,7 +1068,7 @@ func.func @omp_atomic_update6(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval, %expr : i32, i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1080,7 +1080,7 @@ func.func @omp_atomic_update7(%x: memref<i32>, %expr: i32, %y: f32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%y: f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1092,7 +1092,7 @@ func.func @omp_atomic_update8(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32, %tmp: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1104,7 +1104,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1116,7 +1116,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1128,7 +1128,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1164,12 +1164,12 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.update %x : memref<i32> {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1197,7 +1197,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1212,7 +1212,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.write %x = %expr : memref<i32>, i32
     omp.terminator
   }
@@ -1240,7 +1240,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %y : memref<i32>, memref<i32>, i32
     omp.terminator
   }
@@ -1256,7 +1256,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
 }
@@ -1281,7 +1281,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1296,7 +1296,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1311,7 +1311,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1326,7 +1326,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1341,7 +1341,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1356,7 +1356,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x memory_order(seq_cst) : memref<i32>, memref<i32>, i32
   }
   return
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index e377b7aab3e43..5aa7befb2dc26 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
@@ -1380,57 +1380,57 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i1):
     %newval = llvm.and %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.shl %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.intr.smax(%[[XVAL]], %[[EXPR]]) : (i32, i32) -> i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.intr.smax(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.icmp "eq" %[[XVAL]], %[[EXPRBOOL]] : i1
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i1)
-  // }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %xBool : memref<i1> {
   ^bb0(%xval: i1):
     %newval = llvm.icmp "eq" %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%xval:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   %const = arith.constant 42 : i32
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1440,7 +1440,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1450,7 +1450,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1460,7 +1460,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1470,7 +1470,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1480,7 +1480,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1490,7 +1490,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1500,7 +1500,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1510,7 +1510,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1520,7 +1520,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1530,7 +1530,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(release) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1540,7 +1540,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(relaxed) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1550,7 +1550,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1560,17 +1560,16 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
-
+  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   return
 }
 
@@ -1582,7 +1581,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture{
@@ -1590,7 +1589,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   // CHECK: omp.atomic.capture {
@@ -1599,7 +1598,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: }
   omp.atomic.capture{
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
@@ -1607,7 +1606,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
@@ -1623,7 +1622,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(none) {
@@ -1631,7 +1630,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1640,7 +1639,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended) {
@@ -1648,7 +1647,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1657,7 +1656,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended) {
@@ -1665,7 +1664,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1674,7 +1673,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(nonspeculative) {
@@ -1682,7 +1681,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1691,7 +1690,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(speculative) {
@@ -1699,7 +1698,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1708,7 +1707,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, nonspeculative) {
@@ -1716,7 +1715,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1725,7 +1724,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, nonspeculative) {
@@ -1733,7 +1732,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1742,7 +1741,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, speculative) {
@@ -1750,7 +1749,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1759,7 +1758,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) {
@@ -1767,7 +1766,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1776,7 +1775,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(seq_cst) {
@@ -1784,7 +1783,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1793,7 +1792,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acq_rel) {
@@ -1801,7 +1800,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1810,7 +1809,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acquire) {
@@ -1818,7 +1817,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1827,7 +1826,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(release) {
@@ -1835,7 +1834,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1844,7 +1843,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(relaxed) {
@@ -1852,7 +1851,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1861,7 +1860,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) memory_order(seq_cst) {
@@ -1869,7 +1868,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -2653,14 +2652,14 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
   // CHECK: omp.atomic.update %[[x]] : !llvm.ptr {
   // CHECK-NEXT: ^{{[[:alnum:]]+}}(%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   %val = llvm.load %x : !llvm.ptr -> i32
   omp.atomic.write %v = %val : !llvm.ptr, i32
   omp.atomic.update %x : !llvm.ptr {
     ^bb0(%xval: i32):
       omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index 41bc5c4ba525f..00f53864b5fa7 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -9,7 +9,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
     %t2 = llvm.sdiv %t1, %expr : i32
     %newval = llvm.add %xval, %t2 : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -24,7 +24,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.mul %expr, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -41,7 +41,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
     ^bb0(%xval: i32):
       %newval = llvm.mul %expr, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
@@ -59,7 +59,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
       %t2 = llvm.sdiv %t1, %expr : i32
       %newval = llvm.add %xval, %t2 : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 32f0ba5b105ff..ab7f756bf95f9 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,13 +1462,13 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1563,7 +1563,7 @@ llvm.func @_QPomp_atomic_update_complex() {
       %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
       %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
       omp.yield(%16 : !llvm.struct<(f32, f32)>)
-    }
+    } {atomic_control = #omp.atomic_control<>}
    llvm.return
 }
 
@@ -1623,7 +1623,7 @@ llvm.func @_QPomp_atomic_capture_complex() {
         %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
         %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
         omp.yield(%20 : !llvm.struct<(f32, f32)>)
-      }
+      } {atomic_control = #omp.atomic_control<>}
       omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.ptr, !llvm.struct<(f32, f32)>
     }
     llvm.return
@@ -1665,7 +1665,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %expr, %xval : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1683,7 +1683,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1701,7 +1701,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
   // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
   // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
@@ -1710,7 +1710,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1732,7 +1732,7 @@ llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
     %2 = llvm.fadd %1, %0 : f32
     %3 = llvm.fptosi %2 : f32 to i32
     omp.yield(%3 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1751,7 +1751,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1763,7 +1763,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1775,7 +1775,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1787,7 +1787,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1799,7 +1799,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1814,7 +1814,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1829,7 +1829,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1844,7 +1844,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1859,7 +1859,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1874,7 +1874,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1889,7 +1889,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1904,7 +1904,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1919,7 +1919,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1934,7 +1934,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1949,7 +1949,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1964,7 +1964,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -1979,7 +1979,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -2001,7 +2001,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw sub ptr %[[x]], i32 %[[expr]] monotonic
@@ -2012,7 +2012,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw and ptr %[[x]], i32 %[[expr]] monotonic
@@ -2023,7 +2023,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw or ptr %[[x]], i32 %[[expr]] monotonic
@@ -2034,7 +2034,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw xor ptr %[[x]], i32 %[[expr]] monotonic
@@ -2045,7 +2045,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2060,7 +2060,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2075,7 +2075,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2090,7 +2090,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2105,7 +2105,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2120,7 +2120,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2135,7 +2135,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2150,7 +2150,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2165,7 +2165,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2180,7 +2180,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2195,7 +2195,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2211,7 +2211,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2227,7 +2227,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -2265,7 +2265,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acquire
@@ -2276,7 +2276,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] release
@@ -2287,7 +2287,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
@@ -2298,7 +2298,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acq_rel
@@ -2309,7 +2309,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -3055,7 +3055,7 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
index cbf273b887bc7..a9a4ffd533136 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
@@ -110,7 +110,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
 
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 29725a02c075a..0ec244bf28f33 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -19,7 +19,7 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   ^bb0(%arg0: i32):
     %result = llvm.add %arg0, %expr : i32
     omp.yield(%result : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   llvm.return
 }

>From 0b1b23ac98e9486d79b69092698623782ee96921 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 3/9] R3: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h       |  2 +-
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  4 ++--
 flang/lib/Optimizer/Dialect/Support/FIRContext.cpp | 14 +++-----------
 flang/test/Fir/convert-to-llvm-openmp-and-fir.fir  |  4 ++--
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td |  3 ++-
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 26256fd775f11..dd29b4e3e8b08 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,7 +54,7 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic Control Options for AMD GPU
+  /// Atomic control options for AMD gpu
   bool amdgpuIgnoreDenormalMode = false;
   bool amdgpuRemoteMemory = false;
   bool amdgpuFineGrainedMemory = false;
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac563bcc402c7..ac5fd33874db8 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,7 +58,7 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and Getters for atomic control options
+// Setters and getters for atomic control options.
 void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
@@ -70,7 +70,7 @@ bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index b961793dbdfd5..297a4c7a39e44 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -96,10 +96,7 @@ void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
 }
 
 static constexpr const char *amdgpuFineGrainedMemoryName =
@@ -110,10 +107,7 @@ void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
 }
 static constexpr const char *amdgpuRemoteMemoryName =
     "fir.amdgpu.remote.memory";
@@ -123,9 +117,7 @@ void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 24e5cad84b709..47c00d489c200 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -619,7 +619,7 @@ func.func @_QPs() {
     %c1_i32 = arith.constant 1 : i32
     %1 = arith.addi %arg0, %c1_i32 : i32
     omp.yield(%1 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 fir.global internal @_QFsEc : i32 {
@@ -634,7 +634,7 @@ fir.global internal @_QFsEc : i32 {
 // CHECK:      %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:      %[[OUT_VAL:.*]] = llvm.add %[[IN_VAL]], %[[CONST_1]]  : i32
 // CHECK:      omp.yield(%[[OUT_VAL]] : i32)
-// CHECK:    }
+// CHECK:    } {atomic_control = #omp.atomic_control<>}
 // CHECK:    llvm.return
 // CHECK:  }
 // CHECK:  llvm.mlir.global internal @[[GLOBAL]]() {{.*}} : i32 {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 84887dfd2c6f9..61861173ae875 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -58,7 +58,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From 565d104ba013acd7f7e64cc989e774f0f1f34e97 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Mon, 23 Jun 2025 12:14:19 -0500
Subject: [PATCH 4/9] R4: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  6 ++--
 .../Optimizer/Dialect/Support/FIRContext.h    | 18 ++++------
 flang/lib/Frontend/CompilerInvocation.cpp     | 20 +++++------
 flang/lib/Lower/Bridge.cpp                    | 12 +++----
 flang/lib/Lower/OpenMP/Atomic.cpp             |  9 ++++-
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 34 +++++++++----------
 .../Lower/OpenMP/atomic-control-options.f90   | 16 ++++-----
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     |  6 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  4 +--
 9 files changed, 61 insertions(+), 64 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index dd29b4e3e8b08..6ef5338acdc6e 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -55,9 +55,9 @@ class TargetOptions {
   bool asmVerbose = false;
 
   /// Atomic control options for AMD gpu
-  bool amdgpuIgnoreDenormalMode = false;
-  bool amdgpuRemoteMemory = false;
-  bool amdgpuFineGrainedMemory = false;
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac5fd33874db8..273b4683b6d9c 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -59,23 +59,17 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
 // Setters and getters for atomic control options.
-void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
-bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+void setIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getIgnoreDenormalMode(mlir::ModuleOp mod);
+void setFineGrainedMemory(mlir::ModuleOp mod);
+bool getFineGrainedMemory(mlir::ModuleOp mod);
+void setRemoteMemory(mlir::ModuleOp mod);
+bool getRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set atomic control options for AMD gpu.
-void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
-                                   bool amdgpuIgnoreDenormalMode,
-                                   bool amdgpuNoFineGrainedMemory,
-                                   bool amdgpuNoRemoteMemory);
-
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 00f854b905c72..63170a6df6960 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,17 +496,15 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  if (llvm::Triple(opts.triple).isAMDGPU()) {
-    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
-        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
-        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-    opts.amdgpuFineGrainedMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_fine_grained_memory,
-        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-    opts.amdgpuRemoteMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_remote_memory,
-        clang::driver::options::OPT_fno_atomic_remote_memory, false);
-  }
+  opts.ignoreDenormalMode = args.hasFlag(
+      clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+      clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+  opts.fineGrainedMemory = args.hasFlag(
+      clang::driver::options::OPT_fatomic_fine_grained_memory,
+      clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+  opts.remoteMemory =
+      args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
+                   clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index acbfad589dfbb..af6e72aa8cd10 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.amdgpuIgnoreDenormalMode)
-    fir::setAmdgpuIgnoreDenormalMode(*module);
-  if (targetOpts.amdgpuFineGrainedMemory)
-    fir::setAmdgpuFineGrainedMemory(*module);
-  if (targetOpts.amdgpuRemoteMemory)
-    fir::setAmdgpuRemoteMemory(*module);
+  if (targetOpts.ignoreDenormalMode)
+    fir::setIgnoreDenormalMode(*module);
+  if (targetOpts.fineGrainedMemory)
+    fir::setFineGrainedMemory(*module);
+  if (targetOpts.remoteMemory)
+    fir::setRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 2ab91b239a3cc..cbc850d67cc4d 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -634,9 +634,16 @@ genAtomicUpdate(lower::AbstractConverter &converter,
     }
   }
 
+  mlir::ModuleOp module = builder.getModule();
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      mlir::omp::AtomicControlAttr::get(
+          builder.getContext(), fir::getAtomicIgnoreDenormalMode(module),
+          fir::getAtomicFineGrainedMemory(module),
+          fir::getAtomicRemoteMemory(module));
   builder.restoreInsertionPoint(atomicAt);
   auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+      loc, atomAddr, atomicControlAttr, hint,
+      makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 297a4c7a39e44..e3599204f5e2c 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,36 +88,34 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *amdgpuIgnoreDenormalModeName =
-    "fir.amdgpu.ignore.denormal.mode";
-void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
 }
 
-static constexpr const char *amdgpuFineGrainedMemoryName =
-    "fir.amdgpu.fine.grained.memory";
-void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
+void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
+bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
 }
-static constexpr const char *amdgpuRemoteMemoryName =
-    "fir.amdgpu.remote.memory";
-void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+static constexpr const char *remoteMemoryName = "fir.remote.memory";
+void fir::setRemoteMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
+bool fir::getRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
index 1eb0f617f365e..8f86a151b3713 100644
--- a/flang/test/Lower/OpenMP/atomic-control-options.f90
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -9,25 +9,25 @@ program test
     A = 0
     B = 0
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic
     A =  A + 1
     !$omp end target parallel
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic capture
         A = A + B
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 61861173ae875..0fefe7e017e94 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -62,9 +62,9 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 5aa7befb2dc26..9a72ddfc2989d 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1565,11 +1565,11 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   return
 }
 

>From f5a469317a1446d17bc0fb6feb10849d70718368 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 11:26:03 -0500
Subject: [PATCH 5/9] R5: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  8 +--
 .../Optimizer/Dialect/Support/FIRContext.h    | 19 ++++---
 flang/lib/Frontend/CompilerInvocation.cpp     |  6 +--
 flang/lib/Lower/Bridge.cpp                    |  9 ++--
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 53 ++++++++++++-------
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 30 +++++------
 6 files changed, 71 insertions(+), 54 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 6ef5338acdc6e..f6e5634d5a995 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,10 +54,10 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic control options for AMD gpu
-  bool ignoreDenormalMode = false;
-  bool remoteMemory = false;
-  bool fineGrainedMemory = false;
+  /// Atomic control options
+  bool atomicIgnoreDenormalMode = false;
+  bool atomicRemoteMemory = false;
+  bool atomicFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 273b4683b6d9c..c0b852ccaf83a 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,13 +58,18 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and getters for atomic control options.
-void setIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getIgnoreDenormalMode(mlir::ModuleOp mod);
-void setFineGrainedMemory(mlir::ModuleOp mod);
-bool getFineGrainedMemory(mlir::ModuleOp mod);
-void setRemoteMemory(mlir::ModuleOp mod);
-bool getRemoteMemory(mlir::ModuleOp mod);
+/// Sets that Denormal Mode can be ignored for lowering of floating point atomic operations.
+void setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value);
+/// Gets whether Denormal Mode can be ignored or not for lowering of floating point atomic operations.
+bool getAtomicIgnoreDenormalMode(mlir::ModuleOp mod);
+/// Sets that fine grained memory is used for lowering of atomic operations.
+void setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether fine grained memory is used or not for lowering of atomic operations.
+bool getAtomicFineGrainedMemory(mlir::ModuleOp mod);
+/// Sets that remote memory is used for lowering of atomic operations.
+void setAtomicRemoteMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether remote memory is used or not for lowering of atomic operations.
+bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 63170a6df6960..39021207e897e 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,13 +496,13 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  opts.ignoreDenormalMode = args.hasFlag(
+  opts.atomicIgnoreDenormalMode = args.hasFlag(
       clang::driver::options::OPT_fatomic_ignore_denormal_mode,
       clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-  opts.fineGrainedMemory = args.hasFlag(
+  opts.atomicFineGrainedMemory = args.hasFlag(
       clang::driver::options::OPT_fatomic_fine_grained_memory,
       clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-  opts.remoteMemory =
+  opts.atomicRemoteMemory =
       args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
                    clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index af6e72aa8cd10..f8185f752b8e1 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,9 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.ignoreDenormalMode)
-    fir::setIgnoreDenormalMode(*module);
-  if (targetOpts.fineGrainedMemory)
-    fir::setFineGrainedMemory(*module);
-  if (targetOpts.remoteMemory)
-    fir::setRemoteMemory(*module);
+  fir::setAtomicIgnoreDenormalMode(*module, targetOpts.atomicIgnoreDenormalMode);
+  fir::setAtomicFineGrainedMemory(*module, targetOpts.atomicFineGrainedMemory);
+  fir::setAtomicRemoteMemory(*module, targetOpts.atomicRemoteMemory);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index e3599204f5e2c..885efa653fa23 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,34 +88,49 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *ignoreDenormalModeName =
-    "fir.ignore.denormal.mode";
-void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicIgnoreDenormalModeName =
+    "fir.atomic_ignore_denormal_mode";
+void fir::setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicIgnoreDenormalModeName))
+      mod->removeAttr(atomicIgnoreDenormalModeName);
+  }
 }
 
-bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+bool fir::getAtomicIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicIgnoreDenormalModeName);
 }
 
-static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
-void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicFineGrainedMemoryName = "fir.atomic_fine_grained_memory";
+void fir::setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicFineGrainedMemoryName))
+      mod->removeAttr(atomicFineGrainedMemoryName);
+  }
 }
 
-bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
+bool fir::getAtomicFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicFineGrainedMemoryName);
 }
-static constexpr const char *remoteMemoryName = "fir.remote.memory";
-void fir::setRemoteMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicRemoteMemoryName = "fir.atomic_remote_memory";
+void fir::setAtomicRemoteMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicRemoteMemoryName))
+      mod->removeAttr(atomicRemoteMemoryName);
+  }
 }
 
-bool fir::getRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
+bool fir::getAtomicRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 0fefe7e017e94..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -22,6 +22,21 @@ class OpenMP_Attr<string name, string attrMnemonic, list<Trait> traits = [],
   let mnemonic = attrMnemonic;
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // DeclareTargetAttr
 //===----------------------------------------------------------------------===//
@@ -54,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//

>From 72a9c3adee1401f41493a2f8f2f6717fd2b6b17d Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 6/9] [flang][flang-driver] atomic control support

---
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  6 ++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td | 14 ++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index c0b852ccaf83a..9972fcc345cc5 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -75,6 +75,12 @@ bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 72ce4c6a21cb3..1d2118ffc13e7 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -69,6 +69,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//

>From 5c534945fe6aeb83b890691952e65523e91ae4ff Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 7/9] R3: Addressing review comments

---
 flang/include/flang/Optimizer/Dialect/Support/FIRContext.h | 2 +-
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td         | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 9972fcc345cc5..0e8451fe0b409 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -75,7 +75,7 @@ bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 1d2118ffc13e7..55761328bd109 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -73,7 +73,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From bc7e45218839bcfb429cf647d22c360271b7c898 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:24:46 -0500
Subject: [PATCH 8/9] backend changes

---
 flang/test/Driver/atomic-control-options.f90  | 19 ++++++++++++
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       | 14 +++++++--
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 31 +++++++++++++++----
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 31 ++++++++++++++++---
 4 files changed, 82 insertions(+), 13 deletions(-)
 create mode 100644 flang/test/Driver/atomic-control-options.f90

diff --git a/flang/test/Driver/atomic-control-options.f90 b/flang/test/Driver/atomic-control-options.f90
new file mode 100644
index 0000000000000..5cbecde6c521e
--- /dev/null
+++ b/flang/test/Driver/atomic-control-options.f90
@@ -0,0 +1,19 @@
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o -|FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o -|FileCheck -check-prefix=IGNORE-DENORMAL-MODE %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o -|FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o -|FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, threads
+    threads = 128
+    A = 0
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+end program test
+
+!UNSAFE-FP-ATOMICS: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.ignore.denormal.mode !{{.*}}, !amdgpu.no.fine.grained.memory !{{.*}}, !amdgpu.no.remote.memory !{{.*}}
+!IGNORE-DENORMAL-MODE: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.ignore.denormal.mode !{{.*}}, !amdgpu.no.fine.grained.memory !{{.*}}, !amdgpu.no.remote.memory !{{.*}}
+!FINE-GRAINED-MEMORY: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.no.remote.memory !{{.*}}
+!REMOTE-MEMORY: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.no.fine.grained.memory !{{.*}}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 19a4058b64382..9d9a90879a838 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -3282,7 +3282,9 @@ class OpenMPIRBuilder {
   emitAtomicUpdate(InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
                    AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
                    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX,
-                   bool IsXBinopExpr);
+                   bool IsXBinopExpr, bool IsAmdgpuIgnoreDenormalMode,
+                   bool IsAmdgpuNoFineGrainedMemory,
+                   bool IsAmdgpuNoRemoteMemory);
 
   /// Emit the binary op. described by \p RMWOp, using \p Src1 and \p Src2 .
   ///
@@ -3355,7 +3357,10 @@ class OpenMPIRBuilder {
   LLVM_ABI InsertPointOrErrorTy createAtomicUpdate(
       const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
       Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-      AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr);
+      AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
+      bool IsAmdgpuIgnoreDenormalMode = false,
+      bool IsAmdgpuNoFineGrainedMemory = false,
+      bool IsAmdgpuNoRemoteMemory = false);
 
   /// Emit atomic update for constructs: --- Only Scalar data types
   /// V = X; X = X BinOp Expr ,
@@ -3390,7 +3395,10 @@ class OpenMPIRBuilder {
       const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
       AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
       AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
-      bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr);
+      bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
+      bool IsAmdgpuIgnoreDenormalMode = false,
+      bool IsAmdgpuNoFineGrainedMemory = false,
+      bool IsAmdgpuNoRemoteMemory = false);
 
   /// Emit atomic compare for constructs: --- Only scalar data types
   /// cond-expr-stmt:
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 85451b1233f96..c00df4b2e24c5 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -8960,7 +8960,9 @@ OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
 OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
     const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
     Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
+    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
+    bool IsAmdgpuIgnoreDenormalMode, bool IsNoFineGrainedMemory,
+    bool IsNoRemoteMemory) {
   assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
   if (!updateToLocation(Loc))
     return Loc.IP;
@@ -8980,7 +8982,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
 
   Expected<std::pair<Value *, Value *>> AtomicResult =
       emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp,
-                       X.IsVolatile, IsXBinopExpr);
+                       X.IsVolatile, IsXBinopExpr, IsAmdgpuIgnoreDenormalMode,
+                       IsNoFineGrainedMemory, IsNoRemoteMemory);
   if (!AtomicResult)
     return AtomicResult.takeError();
   checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
@@ -9027,7 +9030,9 @@ Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
 Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
     InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
     AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) {
+    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,
+    bool IsAmdgpuIgnoreDenormalMode, bool IsAmdgpuNoFineGrainedMemory,
+    bool IsAmdgpuNoRemoteMemory) {
   // TODO: handle the case where XElemTy is not byte-sized or not a power of 2
   // or a complex datatype.
   bool emitRMWOp = false;
@@ -9050,7 +9055,18 @@ Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
 
   std::pair<Value *, Value *> Res;
   if (emitRMWOp) {
-    Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
+    AtomicRMWInst *atomicRMWInst =
+        Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
+    if (IsAmdgpuIgnoreDenormalMode)
+      atomicRMWInst->setMetadata("amdgpu.ignore.denormal.mode",
+                                 llvm::MDNode::get(Builder.getContext(), {}));
+    if (IsAmdgpuNoFineGrainedMemory)
+      atomicRMWInst->setMetadata("amdgpu.no.fine.grained.memory",
+                                 llvm::MDNode::get(Builder.getContext(), {}));
+    if (IsAmdgpuNoRemoteMemory)
+      atomicRMWInst->setMetadata("amdgpu.no.remote.memory",
+                                 llvm::MDNode::get(Builder.getContext(), {}));
+    Res.first = atomicRMWInst;
     // not needed except in case of postfix captures. Generate anyway for
     // consistency with the else part. Will be removed with any DCE pass.
     // AtomicRMWInst::Xchg does not have a coressponding instruction.
@@ -9182,7 +9198,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
     const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
     AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
     AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
-    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
+    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
+    bool IsAmdgpuIgnoreDenormalMode, bool IsAmdgpuNoFineGrainedMemory,
+    bool IsAmdgpuNoRemoteMemory) {
   if (!updateToLocation(Loc))
     return Loc.IP;
 
@@ -9203,7 +9221,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
   AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
   Expected<std::pair<Value *, Value *>> AtomicResult =
       emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
-                       X.IsVolatile, IsXBinopExpr);
+                       X.IsVolatile, IsXBinopExpr, IsAmdgpuIgnoreDenormalMode,
+                       IsAmdgpuNoFineGrainedMemory, IsAmdgpuNoRemoteMemory);
   if (!AtomicResult)
     return AtomicResult.takeError();
   Value *CapturedVal =
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b07243c5f843..28d09c1fa65f1 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3242,13 +3242,21 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      opInst.getAtomicControlAttr();
+  bool isAmdgpuIgnoreDenormalMode =
+      atomicControlAttr.getAmdgpuIgnoreDenormalMode();
+  bool isAmdgpuNoFineGrainedMemory =
+      !atomicControlAttr.getAmdgpuFineGrainedMemory();
+  bool isAmdgpuNoRemoteMemory = !atomicControlAttr.getAmdgpuRemoteMemory();
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
-      ompBuilder->createAtomicUpdate(ompLoc, allocaIP, llvmAtomicX, llvmExpr,
-                                     atomicOrdering, binop, updateFn,
-                                     isXBinopExpr);
+      ompBuilder->createAtomicUpdate(
+          ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop,
+          updateFn, isXBinopExpr, isAmdgpuIgnoreDenormalMode,
+          isAmdgpuNoFineGrainedMemory, isAmdgpuNoRemoteMemory);
 
   if (failed(handleError(afterIP, *opInst)))
     return failure();
@@ -3270,6 +3278,7 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
   llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
 
   omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();
+
   omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();
 
   assert((atomicUpdateOp || atomicWriteOp) &&
@@ -3337,13 +3346,27 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
+  bool isAmdgpuIgnoreDenormalMode = false;
+  bool isAmdgpuNoFineGrainedMemory = true;
+  bool isAmdgpuNoRemoteMemory = true;
+  if (atomicUpdateOp) {
+    mlir::omp::AtomicControlAttr atomicControlAttr =
+        atomicUpdateOp.getAtomicControlAttr();
+    isAmdgpuIgnoreDenormalMode =
+        atomicControlAttr.getAmdgpuIgnoreDenormalMode();
+    isAmdgpuNoFineGrainedMemory =
+        !atomicControlAttr.getAmdgpuFineGrainedMemory();
+    isAmdgpuNoRemoteMemory = !atomicControlAttr.getAmdgpuRemoteMemory();
+  }
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
       ompBuilder->createAtomicCapture(
           ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
-          binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr);
+          binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr,
+          isAmdgpuIgnoreDenormalMode, isAmdgpuNoFineGrainedMemory,
+          isAmdgpuNoRemoteMemory);
 
   if (failed(handleError(afterIP, *atomicCaptureOp)))
     return failure();

>From 028dd1093f55ec6bf5c7ba0be77e5fca5735b4a3 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 14:55:43 -0500
Subject: [PATCH 9/9] backend changes updates

---
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 15 --------
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 34 +++++++++----------
 2 files changed, 17 insertions(+), 32 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 55761328bd109..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -69,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 28d09c1fa65f1..f8da410dd8674 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3244,19 +3244,19 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
 
   mlir::omp::AtomicControlAttr atomicControlAttr =
       opInst.getAtomicControlAttr();
-  bool isAmdgpuIgnoreDenormalMode =
-      atomicControlAttr.getAmdgpuIgnoreDenormalMode();
-  bool isAmdgpuNoFineGrainedMemory =
-      !atomicControlAttr.getAmdgpuFineGrainedMemory();
-  bool isAmdgpuNoRemoteMemory = !atomicControlAttr.getAmdgpuRemoteMemory();
+  bool isIgnoreDenormalMode =
+      atomicControlAttr.getIgnoreDenormalMode();
+  bool isNoFineGrainedMemory =
+      !atomicControlAttr.getFineGrainedMemory();
+  bool isNoRemoteMemory = !atomicControlAttr.getRemoteMemory();
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
       ompBuilder->createAtomicUpdate(
           ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop,
-          updateFn, isXBinopExpr, isAmdgpuIgnoreDenormalMode,
-          isAmdgpuNoFineGrainedMemory, isAmdgpuNoRemoteMemory);
+          updateFn, isXBinopExpr, isIgnoreDenormalMode,
+          isNoFineGrainedMemory, isNoRemoteMemory);
 
   if (failed(handleError(afterIP, *opInst)))
     return failure();
@@ -3346,17 +3346,17 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
-  bool isAmdgpuIgnoreDenormalMode = false;
-  bool isAmdgpuNoFineGrainedMemory = true;
-  bool isAmdgpuNoRemoteMemory = true;
+  bool isIgnoreDenormalMode = false;
+  bool isNoFineGrainedMemory = true;
+  bool isNoRemoteMemory = true;
   if (atomicUpdateOp) {
     mlir::omp::AtomicControlAttr atomicControlAttr =
         atomicUpdateOp.getAtomicControlAttr();
-    isAmdgpuIgnoreDenormalMode =
-        atomicControlAttr.getAmdgpuIgnoreDenormalMode();
-    isAmdgpuNoFineGrainedMemory =
-        !atomicControlAttr.getAmdgpuFineGrainedMemory();
-    isAmdgpuNoRemoteMemory = !atomicControlAttr.getAmdgpuRemoteMemory();
+    isIgnoreDenormalMode =
+        atomicControlAttr.getIgnoreDenormalMode();
+    isNoFineGrainedMemory =
+        !atomicControlAttr.getFineGrainedMemory();
+    isNoRemoteMemory = !atomicControlAttr.getRemoteMemory();
   }
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
@@ -3365,8 +3365,8 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
       ompBuilder->createAtomicCapture(
           ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
           binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr,
-          isAmdgpuIgnoreDenormalMode, isAmdgpuNoFineGrainedMemory,
-          isAmdgpuNoRemoteMemory);
+          isIgnoreDenormalMode, isNoFineGrainedMemory,
+          isNoRemoteMemory);
 
   if (failed(handleError(afterIP, *atomicCaptureOp)))
     return failure();


From flang-commits at lists.llvm.org  Wed Jul  2 12:57:16 2025
From: flang-commits at lists.llvm.org (Anchu Rajendran S via flang-commits)
Date: Wed, 02 Jul 2025 12:57:16 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <68658f1c.a70a0220.173a04.abb1@mx.google.com>


https://github.com/anchuraj updated https://github.com/llvm/llvm-project/pull/143441

>From 4816fb9bc0761d8761798300bb419db0072c9d04 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 1/5] [flang][flang-driver] atomic control support

---
 clang/include/clang/Driver/Options.td         | 22 +++++-----
 flang/include/flang/Frontend/TargetOptions.h  |  5 +++
 .../Optimizer/Dialect/Support/FIRContext.h    | 14 +++++++
 flang/lib/Frontend/CompilerInvocation.cpp     | 12 ++++++
 flang/lib/Lower/Bridge.cpp                    |  6 +++
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 40 +++++++++++++++++++
 .../Lower/OpenMP/atomic-control-options.f90   | 37 +++++++++++++++++
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 14 +++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  8 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  9 +++++
 10 files changed, 153 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/atomic-control-options.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8e38f0062140e..c1c7929b0c15d 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2303,21 +2303,21 @@ def fsymbol_partition_EQ : Joined<["-"], "fsymbol-partition=">, Group<f_Group>,
 
 defm atomic_remote_memory : BoolFOption<"atomic-remote-memory",
   LangOpts<"AtomicRemoteMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on remote memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on remote memory">>;
 
 defm atomic_fine_grained_memory : BoolFOption<"atomic-fine-grained-memory",
   LangOpts<"AtomicFineGrainedMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on fine-grained memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on fine-grained memory">>;
 
 defm atomic_ignore_denormal_mode : BoolFOption<"atomic-ignore-denormal-mode",
   LangOpts<"AtomicIgnoreDenormalMode">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "Allow">,
-  NegFlag<SetFalse, [], [ClangOption], "Disallow">,
-  BothFlags<[], [ClangOption], " atomic operations to ignore denormal mode">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "Allow">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Disallow">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations to ignore denormal mode">>;
 
 defm memory_profile : OptInCC1FFlag<"memory-profile", "Enable", "Disable", " heap memory profiling">;
 def fmemory_profile_EQ : Joined<["-"], "fmemory-profile=">,
@@ -5314,9 +5314,9 @@ defm amdgpu_precise_memory_op
                   " precise memory mode (AMDGPU only)">;
 
 def munsafe_fp_atomics : Flag<["-"], "munsafe-fp-atomics">,
-  Visibility<[ClangOption, CC1Option]>, Alias<fatomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>, Alias<fatomic_ignore_denormal_mode>;
 def mno_unsafe_fp_atomics : Flag<["-"], "mno-unsafe-fp-atomics">,
-  Visibility<[ClangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, FlangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
 
 def faltivec : Flag<["-"], "faltivec">, Group<f_Group>;
 def fno_altivec : Flag<["-"], "fno-altivec">, Group<f_Group>;
diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 002d8d158abd4..26256fd775f11 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic Control Options for AMD GPU
+  bool amdgpuIgnoreDenormalMode = false;
+  bool amdgpuRemoteMemory = false;
+  bool amdgpuFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 2df14f83c11e1..ac563bcc402c7 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,10 +58,24 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and Getters for atomic control options
+void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
+bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..00f854b905c72 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,6 +496,18 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
+  if (llvm::Triple(opts.triple).isAMDGPU()) {
+    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
+        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+    opts.amdgpuFineGrainedMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_fine_grained_memory,
+        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+    opts.amdgpuRemoteMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_remote_memory,
+        clang::driver::options::OPT_fno_atomic_remote_memory, false);
+  }
+
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
     opts.cpu = a->getValue();
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..acbfad589dfbb 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,6 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
+  if (targetOpts.amdgpuIgnoreDenormalMode)
+    fir::setAmdgpuIgnoreDenormalMode(*module);
+  if (targetOpts.amdgpuFineGrainedMemory)
+    fir::setAmdgpuFineGrainedMemory(*module);
+  if (targetOpts.amdgpuRemoteMemory)
+    fir::setAmdgpuRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 01c0be66d1ecc..b961793dbdfd5 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,6 +88,46 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *amdgpuIgnoreDenormalModeName =
+    "fir.amdgpu.ignore.denormal.mode";
+void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
+    return true;
+  return false;
+}
+
+static constexpr const char *amdgpuFineGrainedMemoryName =
+    "fir.amdgpu.fine.grained.memory";
+void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
+    return true;
+  return false;
+}
+static constexpr const char *amdgpuRemoteMemoryName =
+    "fir.amdgpu.remote.memory";
+void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
+    return true;
+  return false;
+}
+
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
   if (auto attr = mod->getAttrOfType<mlir::StringAttr>(tuneCpuName))
     return attr.getValue();
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
new file mode 100644
index 0000000000000..1eb0f617f365e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o - | FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o - | FileCheck -check-prefix=IGNORE-DENORMAL %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o - | FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o - | FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, B, threads
+    threads = 128
+    A = 0
+    B = 0
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic capture
+        A = A + B
+        B = A
+    !$omp end atomic
+    !$omp end target parallel
+end program test
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 704d0b2220e8a..84887dfd2c6f9 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -54,6 +54,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
+      clausesArgs);
 
   // Override region definition.
   let regions = (region SizedRegion<1>:$region);
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..e377b7aab3e43 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1562,6 +1562,15 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
     omp.yield(%newval : i32)
   }
 
+  // CHECK: omp.atomic.update %[[X]] : memref<i32> {
+  // CHECK-NEXT: (%[[XVAL:.*]]: i32):
+  // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
+  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  omp.atomic.update %x : memref<i32> {
+  ^bb0(%xval:i32):
+    omp.yield(%const:i32)
+  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+
   return
 }
 

>From 63d3b489ce5d1210172f8bc1d3b18d81ff52886d Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:58:07 -0500
Subject: [PATCH 2/5] R2: Making Atomic control non-optional

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   2 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |   2 +-
 mlir/test/Dialect/OpenMP/canonicalize.mlir    |  10 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  46 +++----
 mlir/test/Dialect/OpenMP/ops.mlir             | 127 +++++++++---------
 .../Target/LLVMIR/openmp-llvm-invalid.mlir    |   8 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  98 +++++++-------
 .../openmp-target-nesting-in-host-ops.mlir    |   2 +-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |   2 +-
 9 files changed, 148 insertions(+), 149 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 375e35a146f78..399f7fa6e78f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          OptionalAttr<AtomicControlAttr>:$atomic_control),
+          AtomicControlAttr:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index d69de998346b5..0b2445d0a0d18 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -133,7 +133,7 @@ func.func @atomic_update() {
     %1 = arith.constant 1 : i32
     %2 = arith.addi %arg0, %1  : i32
     omp.yield(%2 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 llvm.mlir.global internal @_QFsEc() : i32 {
diff --git a/mlir/test/Dialect/OpenMP/canonicalize.mlir b/mlir/test/Dialect/OpenMP/canonicalize.mlir
index de6c931ecc5fd..2492010c4d3c7 100644
--- a/mlir/test/Dialect/OpenMP/canonicalize.mlir
+++ b/mlir/test/Dialect/OpenMP/canonicalize.mlir
@@ -4,7 +4,7 @@ func.func @update_no_op(%x : memref<i32>) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -17,7 +17,7 @@ func.func @update_write_op(%x : memref<i32>, %value: i32) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%value : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -33,7 +33,7 @@ func.func @update_normal(%x : memref<i32>, %value: i32) {
   ^bb0(%xval : i32):
     %newval = arith.addi %xval, %value : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -50,7 +50,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.addi %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -65,7 +65,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.muli %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..afc3830286f30 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1007,7 +1007,7 @@ func.func @omp_atomic_update1(%x: memref<i32>, %expr: f32) {
   ^bb0(%xval: f32):
     %newval = llvm.fadd %xval, %expr : f32
     omp.yield (%newval : f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1020,7 +1020,7 @@ func.func @omp_atomic_update2(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.terminator
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1032,7 +1032,7 @@ func.func @omp_atomic_update3(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1044,7 +1044,7 @@ func.func @omp_atomic_update4(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1056,7 +1056,7 @@ func.func @omp_atomic_update5(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1068,7 +1068,7 @@ func.func @omp_atomic_update6(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval, %expr : i32, i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1080,7 +1080,7 @@ func.func @omp_atomic_update7(%x: memref<i32>, %expr: i32, %y: f32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%y: f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1092,7 +1092,7 @@ func.func @omp_atomic_update8(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32, %tmp: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1104,7 +1104,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1116,7 +1116,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1128,7 +1128,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1164,12 +1164,12 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.update %x : memref<i32> {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1197,7 +1197,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1212,7 +1212,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.write %x = %expr : memref<i32>, i32
     omp.terminator
   }
@@ -1240,7 +1240,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %y : memref<i32>, memref<i32>, i32
     omp.terminator
   }
@@ -1256,7 +1256,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
 }
@@ -1281,7 +1281,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1296,7 +1296,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1311,7 +1311,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1326,7 +1326,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1341,7 +1341,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1356,7 +1356,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x memory_order(seq_cst) : memref<i32>, memref<i32>, i32
   }
   return
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index e377b7aab3e43..5aa7befb2dc26 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
@@ -1380,57 +1380,57 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i1):
     %newval = llvm.and %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.shl %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.intr.smax(%[[XVAL]], %[[EXPR]]) : (i32, i32) -> i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.intr.smax(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.icmp "eq" %[[XVAL]], %[[EXPRBOOL]] : i1
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i1)
-  // }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %xBool : memref<i1> {
   ^bb0(%xval: i1):
     %newval = llvm.icmp "eq" %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%xval:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   %const = arith.constant 42 : i32
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1440,7 +1440,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1450,7 +1450,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1460,7 +1460,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1470,7 +1470,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1480,7 +1480,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1490,7 +1490,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1500,7 +1500,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1510,7 +1510,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1520,7 +1520,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1530,7 +1530,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(release) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1540,7 +1540,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(relaxed) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1550,7 +1550,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1560,17 +1560,16 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
-
+  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   return
 }
 
@@ -1582,7 +1581,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture{
@@ -1590,7 +1589,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   // CHECK: omp.atomic.capture {
@@ -1599,7 +1598,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: }
   omp.atomic.capture{
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
@@ -1607,7 +1606,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
@@ -1623,7 +1622,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(none) {
@@ -1631,7 +1630,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1640,7 +1639,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended) {
@@ -1648,7 +1647,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1657,7 +1656,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended) {
@@ -1665,7 +1664,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1674,7 +1673,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(nonspeculative) {
@@ -1682,7 +1681,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1691,7 +1690,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(speculative) {
@@ -1699,7 +1698,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1708,7 +1707,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, nonspeculative) {
@@ -1716,7 +1715,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1725,7 +1724,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, nonspeculative) {
@@ -1733,7 +1732,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1742,7 +1741,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, speculative) {
@@ -1750,7 +1749,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1759,7 +1758,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) {
@@ -1767,7 +1766,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1776,7 +1775,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(seq_cst) {
@@ -1784,7 +1783,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1793,7 +1792,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acq_rel) {
@@ -1801,7 +1800,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1810,7 +1809,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acquire) {
@@ -1818,7 +1817,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1827,7 +1826,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(release) {
@@ -1835,7 +1834,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1844,7 +1843,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(relaxed) {
@@ -1852,7 +1851,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1861,7 +1860,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) memory_order(seq_cst) {
@@ -1869,7 +1868,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -2653,14 +2652,14 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
   // CHECK: omp.atomic.update %[[x]] : !llvm.ptr {
   // CHECK-NEXT: ^{{[[:alnum:]]+}}(%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   %val = llvm.load %x : !llvm.ptr -> i32
   omp.atomic.write %v = %val : !llvm.ptr, i32
   omp.atomic.update %x : !llvm.ptr {
     ^bb0(%xval: i32):
       omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index 41bc5c4ba525f..00f53864b5fa7 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -9,7 +9,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
     %t2 = llvm.sdiv %t1, %expr : i32
     %newval = llvm.add %xval, %t2 : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -24,7 +24,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.mul %expr, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -41,7 +41,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
     ^bb0(%xval: i32):
       %newval = llvm.mul %expr, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
@@ -59,7 +59,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
       %t2 = llvm.sdiv %t1, %expr : i32
       %newval = llvm.add %xval, %t2 : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 32f0ba5b105ff..ab7f756bf95f9 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,13 +1462,13 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1563,7 +1563,7 @@ llvm.func @_QPomp_atomic_update_complex() {
       %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
       %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
       omp.yield(%16 : !llvm.struct<(f32, f32)>)
-    }
+    } {atomic_control = #omp.atomic_control<>}
    llvm.return
 }
 
@@ -1623,7 +1623,7 @@ llvm.func @_QPomp_atomic_capture_complex() {
         %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
         %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
         omp.yield(%20 : !llvm.struct<(f32, f32)>)
-      }
+      } {atomic_control = #omp.atomic_control<>}
       omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.ptr, !llvm.struct<(f32, f32)>
     }
     llvm.return
@@ -1665,7 +1665,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %expr, %xval : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1683,7 +1683,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1701,7 +1701,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
   // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
   // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
@@ -1710,7 +1710,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1732,7 +1732,7 @@ llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
     %2 = llvm.fadd %1, %0 : f32
     %3 = llvm.fptosi %2 : f32 to i32
     omp.yield(%3 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1751,7 +1751,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1763,7 +1763,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1775,7 +1775,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1787,7 +1787,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1799,7 +1799,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1814,7 +1814,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1829,7 +1829,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1844,7 +1844,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1859,7 +1859,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1874,7 +1874,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1889,7 +1889,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1904,7 +1904,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1919,7 +1919,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1934,7 +1934,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1949,7 +1949,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1964,7 +1964,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -1979,7 +1979,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -2001,7 +2001,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw sub ptr %[[x]], i32 %[[expr]] monotonic
@@ -2012,7 +2012,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw and ptr %[[x]], i32 %[[expr]] monotonic
@@ -2023,7 +2023,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw or ptr %[[x]], i32 %[[expr]] monotonic
@@ -2034,7 +2034,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw xor ptr %[[x]], i32 %[[expr]] monotonic
@@ -2045,7 +2045,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2060,7 +2060,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2075,7 +2075,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2090,7 +2090,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2105,7 +2105,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2120,7 +2120,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2135,7 +2135,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2150,7 +2150,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2165,7 +2165,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2180,7 +2180,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2195,7 +2195,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2211,7 +2211,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2227,7 +2227,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -2265,7 +2265,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acquire
@@ -2276,7 +2276,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] release
@@ -2287,7 +2287,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
@@ -2298,7 +2298,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acq_rel
@@ -2309,7 +2309,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -3055,7 +3055,7 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
index cbf273b887bc7..a9a4ffd533136 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
@@ -110,7 +110,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
 
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 29725a02c075a..0ec244bf28f33 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -19,7 +19,7 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   ^bb0(%arg0: i32):
     %result = llvm.add %arg0, %expr : i32
     omp.yield(%result : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   llvm.return
 }

>From 0b1b23ac98e9486d79b69092698623782ee96921 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 3/5] R3: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h       |  2 +-
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  4 ++--
 flang/lib/Optimizer/Dialect/Support/FIRContext.cpp | 14 +++-----------
 flang/test/Fir/convert-to-llvm-openmp-and-fir.fir  |  4 ++--
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td |  3 ++-
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 26256fd775f11..dd29b4e3e8b08 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,7 +54,7 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic Control Options for AMD GPU
+  /// Atomic control options for AMD gpu
   bool amdgpuIgnoreDenormalMode = false;
   bool amdgpuRemoteMemory = false;
   bool amdgpuFineGrainedMemory = false;
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac563bcc402c7..ac5fd33874db8 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,7 +58,7 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and Getters for atomic control options
+// Setters and getters for atomic control options.
 void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
@@ -70,7 +70,7 @@ bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index b961793dbdfd5..297a4c7a39e44 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -96,10 +96,7 @@ void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
 }
 
 static constexpr const char *amdgpuFineGrainedMemoryName =
@@ -110,10 +107,7 @@ void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
 }
 static constexpr const char *amdgpuRemoteMemoryName =
     "fir.amdgpu.remote.memory";
@@ -123,9 +117,7 @@ void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 24e5cad84b709..47c00d489c200 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -619,7 +619,7 @@ func.func @_QPs() {
     %c1_i32 = arith.constant 1 : i32
     %1 = arith.addi %arg0, %c1_i32 : i32
     omp.yield(%1 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 fir.global internal @_QFsEc : i32 {
@@ -634,7 +634,7 @@ fir.global internal @_QFsEc : i32 {
 // CHECK:      %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:      %[[OUT_VAL:.*]] = llvm.add %[[IN_VAL]], %[[CONST_1]]  : i32
 // CHECK:      omp.yield(%[[OUT_VAL]] : i32)
-// CHECK:    }
+// CHECK:    } {atomic_control = #omp.atomic_control<>}
 // CHECK:    llvm.return
 // CHECK:  }
 // CHECK:  llvm.mlir.global internal @[[GLOBAL]]() {{.*}} : i32 {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 84887dfd2c6f9..61861173ae875 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -58,7 +58,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From 565d104ba013acd7f7e64cc989e774f0f1f34e97 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Mon, 23 Jun 2025 12:14:19 -0500
Subject: [PATCH 4/5] R4: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  6 ++--
 .../Optimizer/Dialect/Support/FIRContext.h    | 18 ++++------
 flang/lib/Frontend/CompilerInvocation.cpp     | 20 +++++------
 flang/lib/Lower/Bridge.cpp                    | 12 +++----
 flang/lib/Lower/OpenMP/Atomic.cpp             |  9 ++++-
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 34 +++++++++----------
 .../Lower/OpenMP/atomic-control-options.f90   | 16 ++++-----
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     |  6 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  4 +--
 9 files changed, 61 insertions(+), 64 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index dd29b4e3e8b08..6ef5338acdc6e 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -55,9 +55,9 @@ class TargetOptions {
   bool asmVerbose = false;
 
   /// Atomic control options for AMD gpu
-  bool amdgpuIgnoreDenormalMode = false;
-  bool amdgpuRemoteMemory = false;
-  bool amdgpuFineGrainedMemory = false;
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac5fd33874db8..273b4683b6d9c 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -59,23 +59,17 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
 // Setters and getters for atomic control options.
-void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
-bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+void setIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getIgnoreDenormalMode(mlir::ModuleOp mod);
+void setFineGrainedMemory(mlir::ModuleOp mod);
+bool getFineGrainedMemory(mlir::ModuleOp mod);
+void setRemoteMemory(mlir::ModuleOp mod);
+bool getRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set atomic control options for AMD gpu.
-void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
-                                   bool amdgpuIgnoreDenormalMode,
-                                   bool amdgpuNoFineGrainedMemory,
-                                   bool amdgpuNoRemoteMemory);
-
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 00f854b905c72..63170a6df6960 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,17 +496,15 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  if (llvm::Triple(opts.triple).isAMDGPU()) {
-    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
-        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
-        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-    opts.amdgpuFineGrainedMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_fine_grained_memory,
-        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-    opts.amdgpuRemoteMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_remote_memory,
-        clang::driver::options::OPT_fno_atomic_remote_memory, false);
-  }
+  opts.ignoreDenormalMode = args.hasFlag(
+      clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+      clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+  opts.fineGrainedMemory = args.hasFlag(
+      clang::driver::options::OPT_fatomic_fine_grained_memory,
+      clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+  opts.remoteMemory =
+      args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
+                   clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index acbfad589dfbb..af6e72aa8cd10 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.amdgpuIgnoreDenormalMode)
-    fir::setAmdgpuIgnoreDenormalMode(*module);
-  if (targetOpts.amdgpuFineGrainedMemory)
-    fir::setAmdgpuFineGrainedMemory(*module);
-  if (targetOpts.amdgpuRemoteMemory)
-    fir::setAmdgpuRemoteMemory(*module);
+  if (targetOpts.ignoreDenormalMode)
+    fir::setIgnoreDenormalMode(*module);
+  if (targetOpts.fineGrainedMemory)
+    fir::setFineGrainedMemory(*module);
+  if (targetOpts.remoteMemory)
+    fir::setRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 2ab91b239a3cc..cbc850d67cc4d 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -634,9 +634,16 @@ genAtomicUpdate(lower::AbstractConverter &converter,
     }
   }
 
+  mlir::ModuleOp module = builder.getModule();
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      mlir::omp::AtomicControlAttr::get(
+          builder.getContext(), fir::getAtomicIgnoreDenormalMode(module),
+          fir::getAtomicFineGrainedMemory(module),
+          fir::getAtomicRemoteMemory(module));
   builder.restoreInsertionPoint(atomicAt);
   auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+      loc, atomAddr, atomicControlAttr, hint,
+      makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 297a4c7a39e44..e3599204f5e2c 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,36 +88,34 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *amdgpuIgnoreDenormalModeName =
-    "fir.amdgpu.ignore.denormal.mode";
-void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
 }
 
-static constexpr const char *amdgpuFineGrainedMemoryName =
-    "fir.amdgpu.fine.grained.memory";
-void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
+void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
+bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
 }
-static constexpr const char *amdgpuRemoteMemoryName =
-    "fir.amdgpu.remote.memory";
-void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+static constexpr const char *remoteMemoryName = "fir.remote.memory";
+void fir::setRemoteMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
+bool fir::getRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
index 1eb0f617f365e..8f86a151b3713 100644
--- a/flang/test/Lower/OpenMP/atomic-control-options.f90
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -9,25 +9,25 @@ program test
     A = 0
     B = 0
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic
     A =  A + 1
     !$omp end target parallel
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic capture
         A = A + B
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 61861173ae875..0fefe7e017e94 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -62,9 +62,9 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 5aa7befb2dc26..9a72ddfc2989d 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1565,11 +1565,11 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   return
 }
 

>From 23d305883c3c456f094ebb9ec5f40237d5122cc8 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 11:26:03 -0500
Subject: [PATCH 5/5] R5: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  8 +--
 .../Optimizer/Dialect/Support/FIRContext.h    | 22 +++++---
 flang/lib/Frontend/CompilerInvocation.cpp     |  6 +-
 flang/lib/Lower/Bridge.cpp                    | 10 ++--
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 55 ++++++++++++-------
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 30 +++++-----
 6 files changed, 77 insertions(+), 54 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 6ef5338acdc6e..f6e5634d5a995 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,10 +54,10 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic control options for AMD gpu
-  bool ignoreDenormalMode = false;
-  bool remoteMemory = false;
-  bool fineGrainedMemory = false;
+  /// Atomic control options
+  bool atomicIgnoreDenormalMode = false;
+  bool atomicRemoteMemory = false;
+  bool atomicFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 273b4683b6d9c..fa61ab64e253d 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,13 +58,21 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and getters for atomic control options.
-void setIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getIgnoreDenormalMode(mlir::ModuleOp mod);
-void setFineGrainedMemory(mlir::ModuleOp mod);
-bool getFineGrainedMemory(mlir::ModuleOp mod);
-void setRemoteMemory(mlir::ModuleOp mod);
-bool getRemoteMemory(mlir::ModuleOp mod);
+/// Sets that Denormal Mode can be ignored for lowering of floating point atomic
+/// operations.
+void setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value);
+/// Gets whether Denormal Mode can be ignored or not for lowering of floating
+/// point atomic operations.
+bool getAtomicIgnoreDenormalMode(mlir::ModuleOp mod);
+/// Sets that fine grained memory is used for lowering of atomic operations.
+void setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether fine grained memory is used or not for lowering of atomic
+/// operations.
+bool getAtomicFineGrainedMemory(mlir::ModuleOp mod);
+/// Sets that remote memory is used for lowering of atomic operations.
+void setAtomicRemoteMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether remote memory is used or not for lowering of atomic operations.
+bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 63170a6df6960..39021207e897e 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,13 +496,13 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  opts.ignoreDenormalMode = args.hasFlag(
+  opts.atomicIgnoreDenormalMode = args.hasFlag(
       clang::driver::options::OPT_fatomic_ignore_denormal_mode,
       clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-  opts.fineGrainedMemory = args.hasFlag(
+  opts.atomicFineGrainedMemory = args.hasFlag(
       clang::driver::options::OPT_fatomic_fine_grained_memory,
       clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-  opts.remoteMemory =
+  opts.atomicRemoteMemory =
       args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
                    clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index af6e72aa8cd10..3795ce2e048b6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,10 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.ignoreDenormalMode)
-    fir::setIgnoreDenormalMode(*module);
-  if (targetOpts.fineGrainedMemory)
-    fir::setFineGrainedMemory(*module);
-  if (targetOpts.remoteMemory)
-    fir::setRemoteMemory(*module);
+  fir::setAtomicIgnoreDenormalMode(*module,
+                                   targetOpts.atomicIgnoreDenormalMode);
+  fir::setAtomicFineGrainedMemory(*module, targetOpts.atomicFineGrainedMemory);
+  fir::setAtomicRemoteMemory(*module, targetOpts.atomicRemoteMemory);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index e3599204f5e2c..c9c921008597d 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,34 +88,51 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *ignoreDenormalModeName =
-    "fir.ignore.denormal.mode";
-void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicIgnoreDenormalModeName =
+    "fir.atomic_ignore_denormal_mode";
+void fir::setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value) {
+  if (value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  } else {
+    if (mod->hasAttr(atomicIgnoreDenormalModeName))
+      mod->removeAttr(atomicIgnoreDenormalModeName);
+  }
 }
 
-bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+bool fir::getAtomicIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicIgnoreDenormalModeName);
 }
 
-static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
-void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicFineGrainedMemoryName =
+    "fir.atomic_fine_grained_memory";
+void fir::setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value) {
+  if (value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if (mod->hasAttr(atomicFineGrainedMemoryName))
+      mod->removeAttr(atomicFineGrainedMemoryName);
+  }
 }
 
-bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
+bool fir::getAtomicFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicFineGrainedMemoryName);
 }
-static constexpr const char *remoteMemoryName = "fir.remote.memory";
-void fir::setRemoteMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicRemoteMemoryName =
+    "fir.atomic_remote_memory";
+void fir::setAtomicRemoteMemory(mlir::ModuleOp mod, bool value) {
+  if (value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if (mod->hasAttr(atomicRemoteMemoryName))
+      mod->removeAttr(atomicRemoteMemoryName);
+  }
 }
 
-bool fir::getRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
+bool fir::getAtomicRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 0fefe7e017e94..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -22,6 +22,21 @@ class OpenMP_Attr<string name, string attrMnemonic, list<Trait> traits = [],
   let mnemonic = attrMnemonic;
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // DeclareTargetAttr
 //===----------------------------------------------------------------------===//
@@ -54,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//


From flang-commits at lists.llvm.org  Wed Jul  2 13:45:29 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 02 Jul 2025 13:45:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][preprocessor] fix use of
 bitwise-and for logical-and (PR #146758)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146758@github.com>
Message-ID: <68659a69.630a0220.2e2b82.2cb3@mx.google.com>


https://github.com/akuhlens updated https://github.com/llvm/llvm-project/pull/146758

>From 3631e247920d8e1c5a1ef186e1abcd91d24dce9e Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Wed, 2 Jul 2025 11:51:05 -0700
Subject: [PATCH 1/2] initial commit

---
 flang/lib/Parser/preprocessor.cpp    |  9 +++++++--
 flang/test/Parser/issue-146362.2.f90 | 11 +++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Parser/issue-146362.2.f90

diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index ef24c704db885..d79b7d41da83c 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -1276,14 +1276,19 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
       left = right >= 64 ? 0 : left >> right;
       break;
     case BITAND:
-    case AND:
       left = left & right;
       break;
+    // (bool)(1 && 2) != (bool)(1 & 2), we can't use bitwise AND here.
+    case AND:
+      left = left && right;
+      break;
     case BITXOR:
       left = left ^ right;
       break;
-    case BITOR:
+    // (bool)(1 || 2) == (bool)(1 | 2), seems like we can use `|` here.
+    // But should we?
     case OR:
+    case BITOR:
       left = left | right;
       break;
     case LT:
diff --git a/flang/test/Parser/issue-146362.2.f90 b/flang/test/Parser/issue-146362.2.f90
new file mode 100644
index 0000000000000..db27115ec15d0
--- /dev/null
+++ b/flang/test/Parser/issue-146362.2.f90
@@ -0,0 +1,11 @@
+!RUN: %flang_fc1 -cpp -fdebug-unparse %s | FileCheck %s
+PROGRAM P
+#if (1 && 2)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+END PROGRAM
+

>From 6abffad169a40a2a5744876964c11b4f676407bd Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Wed, 2 Jul 2025 13:44:49 -0700
Subject: [PATCH 2/2] remove using bitwise or for logical or

---
 flang/lib/Parser/preprocessor.cpp    | 13 ++++++-------
 flang/test/Parser/issue-146362.2.f90 |  7 +++++++
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index d79b7d41da83c..8ef9810463d5a 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -1278,19 +1278,18 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
     case BITAND:
       left = left & right;
       break;
-    // (bool)(1 && 2) != (bool)(1 & 2), we can't use bitwise AND here.
-    case AND:
-      left = left && right;
-      break;
     case BITXOR:
       left = left ^ right;
       break;
-    // (bool)(1 || 2) == (bool)(1 | 2), seems like we can use `|` here.
-    // But should we?
-    case OR:
     case BITOR:
       left = left | right;
       break;
+    case AND:
+      left = left && right;
+      break;
+    case OR:
+      left = left || right;
+      break;
     case LT:
       left = -(left < right);
       break;
diff --git a/flang/test/Parser/issue-146362.2.f90 b/flang/test/Parser/issue-146362.2.f90
index db27115ec15d0..d9ad8c7d547f2 100644
--- a/flang/test/Parser/issue-146362.2.f90
+++ b/flang/test/Parser/issue-146362.2.f90
@@ -7,5 +7,12 @@ PROGRAM P
   !CHECK-NOT: FALSE
   WRITE(*,*) 'FALSE'
 #endif
+#if ((1 || 2) != 3)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
 END PROGRAM
 


From flang-commits at lists.llvm.org  Wed Jul  2 13:50:28 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 02 Jul 2025 13:50:28 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][preprocessor] fix use of
 bitwise-and for logical-and (PR #146758)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146758@github.com>
Message-ID: <68659b94.170a0220.5fb88.5bd0@mx.google.com>


https://github.com/klausler approved this pull request.


https://github.com/llvm/llvm-project/pull/146758

From flang-commits at lists.llvm.org  Wed Jul  2 13:52:17 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 02 Jul 2025 13:52:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][preprocessor] fix use of
 bitwise-and for logical-and (PR #146758)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146758@github.com>
Message-ID: <68659c01.630a0220.288d54.1f88@mx.google.com>


https://github.com/eugeneepshteyn approved this pull request.


https://github.com/llvm/llvm-project/pull/146758

From flang-commits at lists.llvm.org  Wed Jul  2 13:54:26 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 02 Jul 2025 13:54:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <68659c82.170a0220.2d188a.6318@mx.google.com>


https://github.com/akuhlens approved this pull request.

LGTM otherwise!

https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Wed Jul  2 13:54:26 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 02 Jul 2025 13:54:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <68659c82.170a0220.d32c7.2def@mx.google.com>


https://github.com/akuhlens edited https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Wed Jul  2 13:54:26 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 02 Jul 2025 13:54:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <68659c82.170a0220.3008c9.36c6@mx.google.com>


================
@@ -482,9 +487,20 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
           ySymbol.attrs().test(semantics::Attr::BIND_C)) {
     return false;
   }
-  if (!ignoreSequence && !(xDetails.sequence() && yDetails.sequence()) &&
-      !(xSymbol.attrs().test(semantics::Attr::BIND_C) &&
-          ySymbol.attrs().test(semantics::Attr::BIND_C))) {
----------------
akuhlens wrote:

Is this check no longer needed?

https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Wed Jul  2 14:01:58 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 02 Jul 2025 14:01:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <68659e46.170a0220.21aa11.26ab@mx.google.com>


https://github.com/eugeneepshteyn edited https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Wed Jul  2 14:01:58 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 02 Jul 2025 14:01:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <68659e46.170a0220.277f41.7693@mx.google.com>


================
@@ -0,0 +1,26 @@
+!RUN: rm -rf %t && mkdir -p %t
+!RUN: %flang_fc1 -DWHICH=1 -fsyntax-only -J%t %s
----------------
eugeneepshteyn wrote:

It's interesting that `-J` option doesn't show up in either `flang --help` or `flang -fc1 --help`

https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Wed Jul  2 14:01:58 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 02 Jul 2025 14:01:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <68659e46.170a0220.2fdd1e.2e66@mx.google.com>


https://github.com/eugeneepshteyn approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Wed Jul  2 14:54:30 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 02 Jul 2025 14:54:30 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <6865aa96.630a0220.86e02.fe0b@mx.google.com>


================
@@ -482,9 +487,20 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
           ySymbol.attrs().test(semantics::Attr::BIND_C)) {
     return false;
   }
-  if (!ignoreSequence && !(xDetails.sequence() && yDetails.sequence()) &&
-      !(xSymbol.attrs().test(semantics::Attr::BIND_C) &&
-          ySymbol.attrs().test(semantics::Attr::BIND_C))) {
----------------
eugeneepshteyn wrote:

The remaining checks are on lines 502-503

https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Wed Jul  2 15:47:52 2025
From: flang-commits at lists.llvm.org (Katherine Rasmussen via flang-commits)
Date: Wed, 02 Jul 2025 15:47:52 -0700 (PDT)
Subject: [flang-commits] [flang] Parallel runtime library design doc (PRIF)
	(PR #76088)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/76088@github.com>
Message-ID: <6865b718.170a0220.7ccac.7053@mx.google.com>


ktras wrote:

Feedback received from @kiranchandramohan expressed a desire for a clear process on how PRIF will be updated in the future and how LLVM Flang developers can be involved. 

In response, we have formed the PRIF Specification Committee, with a [governance document](https://go.lbl.gov/prif-governance) detailing the process by which changes to PRIF are proposed and incorporated into the specification. The Committee meets in monthly virtual meetings and anyone is welcome to join. We have an open mailing list here, [prif-spec at googlegroups.com](mailto:prif-spec at googlegroups.com), which anyone can join or view. We believe this resolves the stated reviewer feedback.

https://github.com/llvm/llvm-project/pull/76088

From flang-commits at lists.llvm.org  Wed Jul  2 17:47:41 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 02 Jul 2025 17:47:41 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not create global for
 derived-type with allocatable device components (PR #146780)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146780@github.com>
Message-ID: <6865d32d.170a0220.17c8ca.321d@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `amdgpu-offload-ubuntu-22-cmake-build-only` running on `rocm-docker-ubu-22` while building `flang` at step 4 "annotate".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/203/builds/15409

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 4 (annotate) failure: '../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py --jobs=32' (failure)
...
[7504/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/pointer-assignment.cpp.o
[7505/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/openmp-utils.cpp.o
[7506/7899] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-integer.cpp.o
[7507/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/tools.cpp.o
[7508/7899] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o
[7509/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/scope.cpp.o
[7510/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/mod-file.cpp.o
[7511/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/symbol.cpp.o
[7512/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-names-utils.cpp.o
[7513/7899] Linking CXX shared library lib/libFortranEvaluate.so.21.0git
FAILED: lib/libFortranEvaluate.so.21.0git 
: && /usr/bin/c++ -fPIC -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough -Wno-uninitialized -Wno-nonnull -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wdelete-non-virtual-dtor -Wsuggest-override -Wno-comment -Wno-misleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Wno-deprecated-copy -Wno-ctad-maybe-unsupported -fno-semantic-interposition -fpch-preprocess -O3 -DNDEBUG -fno-semantic-interposition  -Wl,-z,defs -Wl,-z,nodelete   -Wl,-rpath-link,/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/./lib  -Wl,--gc-sections -shared -Wl,-soname,libFortranEvaluate.so.21.0git -o lib/libFortranEvaluate.so.21.0git tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/call.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/characteristics.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/check-expression.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/common.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/complex.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/constant.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/expression.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-character.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-complex.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-designator.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-integer.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-logical.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-real.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-reduction.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/formatting.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/host.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/initial-image.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/integer.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics-library.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/logical.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/real.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/shape.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/static-data.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/target.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/type.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/variable.cpp.o  -Wl,-rpath,"\$ORIGIN/../lib:/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib:"  lib/libFortranDecimal.so.21.0git  lib/libFortranParser.so.21.0git  -lquadmath  lib/libFortranSupport.so.21.0git  lib/libLLVMSupport.so.21.0git  -Wl,-rpath-link,/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib && :
/usr/bin/ld: tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o: in function `Fortran::semantics::CanCUDASymbolHaveSaveAttr(Fortran::semantics::Symbol const&) [clone .localalias]':
tools.cpp:(.text._ZN7Fortran9semantics25CanCUDASymbolHaveSaveAttrERKNS0_6SymbolE+0x76): undefined reference to `Fortran::semantics::FindCUDADeviceAllocatableUltimateComponent(Fortran::semantics::DerivedTypeSpec const&)'
collect2: error: ld returned 1 exit status
[7514/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/unparse-with-symbols.cpp.o
[7515/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-structure.cpp.o
[7516/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/runtime-type-info.cpp.o
[7517/7899] Building CXX object tools/flang/lib/FrontendTool/CMakeFiles/flangFrontendTool.dir/ExecuteCompilerInvocation.cpp.o
[7518/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Coarray.cpp.o
[7519/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ComponentPath.cpp.o
[7520/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/type.cpp.o
[7521/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CallInterface.cpp.o
[7522/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o
[7523/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Allocatable.cpp.o
[7524/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HlfirIntrinsics.cpp.o
[7525/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertType.cpp.o
[7526/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/LoweringOptions.cpp.o
[7527/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IterationSpace.cpp.o
[7528/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertProcedureDesignator.cpp.o
[7529/7899] Building CXX object tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/IntrinsicCall.cpp.o
[7530/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertCall.cpp.o
[7531/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertArrayConstructor.cpp.o
[7532/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertConstant.cpp.o
[7533/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CustomIntrinsicCall.cpp.o
[7534/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Mangler.cpp.o
[7535/7899] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/cmake_pch.hxx.gch
[7536/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IO.cpp.o
[7537/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/semantics.cpp.o
[7538/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Bridge.cpp.o
[7539/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HostAssociations.cpp.o
[7540/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o
[7541/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/expression.cpp.o
[7542/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExprToHLFIR.cpp.o
[7543/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Atomic.cpp.o
[7544/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-names.cpp.o
[7545/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenACC.cpp.o
[7546/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExpr.cpp.o
ninja: build stopped: subcommand failed.
Step 7 (build cmake config) failure: build cmake config (failure)
...
[7504/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/pointer-assignment.cpp.o
[7505/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/openmp-utils.cpp.o
[7506/7899] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-integer.cpp.o
[7507/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/tools.cpp.o
[7508/7899] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o
[7509/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/scope.cpp.o
[7510/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/mod-file.cpp.o
[7511/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/symbol.cpp.o
[7512/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-names-utils.cpp.o
[7513/7899] Linking CXX shared library lib/libFortranEvaluate.so.21.0git
FAILED: lib/libFortranEvaluate.so.21.0git 
: && /usr/bin/c++ -fPIC -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough -Wno-uninitialized -Wno-nonnull -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wdelete-non-virtual-dtor -Wsuggest-override -Wno-comment -Wno-misleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Wno-deprecated-copy -Wno-ctad-maybe-unsupported -fno-semantic-interposition -fpch-preprocess -O3 -DNDEBUG -fno-semantic-interposition  -Wl,-z,defs -Wl,-z,nodelete   -Wl,-rpath-link,/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/./lib  -Wl,--gc-sections -shared -Wl,-soname,libFortranEvaluate.so.21.0git -o lib/libFortranEvaluate.so.21.0git tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/call.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/characteristics.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/check-expression.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/common.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/complex.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/constant.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/expression.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-character.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-complex.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-designator.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-integer.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-logical.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-real.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-reduction.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/formatting.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/host.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/initial-image.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/integer.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics-library.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/logical.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/real.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/shape.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/static-data.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/target.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/type.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/variable.cpp.o  -Wl,-rpath,"\$ORIGIN/../lib:/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib:"  lib/libFortranDecimal.so.21.0git  lib/libFortranParser.so.21.0git  -lquadmath  lib/libFortranSupport.so.21.0git  lib/libLLVMSupport.so.21.0git  -Wl,-rpath-link,/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib && :
/usr/bin/ld: tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o: in function `Fortran::semantics::CanCUDASymbolHaveSaveAttr(Fortran::semantics::Symbol const&) [clone .localalias]':
tools.cpp:(.text._ZN7Fortran9semantics25CanCUDASymbolHaveSaveAttrERKNS0_6SymbolE+0x76): undefined reference to `Fortran::semantics::FindCUDADeviceAllocatableUltimateComponent(Fortran::semantics::DerivedTypeSpec const&)'
collect2: error: ld returned 1 exit status
[7514/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/unparse-with-symbols.cpp.o
[7515/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-structure.cpp.o
[7516/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/runtime-type-info.cpp.o
[7517/7899] Building CXX object tools/flang/lib/FrontendTool/CMakeFiles/flangFrontendTool.dir/ExecuteCompilerInvocation.cpp.o
[7518/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Coarray.cpp.o
[7519/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ComponentPath.cpp.o
[7520/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/type.cpp.o
[7521/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CallInterface.cpp.o
[7522/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o
[7523/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Allocatable.cpp.o
[7524/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HlfirIntrinsics.cpp.o
[7525/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertType.cpp.o
[7526/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/LoweringOptions.cpp.o
[7527/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IterationSpace.cpp.o
[7528/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertProcedureDesignator.cpp.o
[7529/7899] Building CXX object tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/IntrinsicCall.cpp.o
[7530/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertCall.cpp.o
[7531/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertArrayConstructor.cpp.o
[7532/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertConstant.cpp.o
[7533/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CustomIntrinsicCall.cpp.o
[7534/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Mangler.cpp.o
[7535/7899] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/cmake_pch.hxx.gch
[7536/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IO.cpp.o
[7537/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/semantics.cpp.o
[7538/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Bridge.cpp.o
[7539/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HostAssociations.cpp.o
[7540/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o
[7541/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/expression.cpp.o
[7542/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExprToHLFIR.cpp.o
[7543/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Atomic.cpp.o
[7544/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-names.cpp.o
[7545/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenACC.cpp.o
[7546/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExpr.cpp.o
ninja: build stopped: subcommand failed.

```

</details>

https://github.com/llvm/llvm-project/pull/146780

From flang-commits at lists.llvm.org  Wed Jul  2 17:51:39 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 02 Jul 2025 17:51:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not create global for
 derived-type with allocatable device components (PR #146780)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146780@github.com>
Message-ID: <6865d41b.170a0220.101122.3e18@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `amdgpu-offload-rhel-9-cmake-build-only` running on `rocm-docker-rhel-9` while building `flang` at step 4 "annotate".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/205/builds/14199

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 4 (annotate) failure: '../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py --jobs=32' (failure)
...
[7512/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertCall.cpp.o
[7513/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HlfirIntrinsics.cpp.o
[7514/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertType.cpp.o
[7515/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertProcedureDesignator.cpp.o
[7516/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertArrayConstructor.cpp.o
[7517/7899] Linking CXX shared library lib/libFortranParser.so.21.0git
[7518/7899] Creating library symlink lib/libFortranParser.so
[7519/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertConstant.cpp.o
[7520/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/LoweringOptions.cpp.o
[7521/7899] Linking CXX shared library lib/libFortranEvaluate.so.21.0git
FAILED: lib/libFortranEvaluate.so.21.0git 
: && /usr/bin/c++ -fPIC -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough -Wno-uninitialized -Wno-nonnull -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wdelete-non-virtual-dtor -Wsuggest-override -Wno-comment -Wno-misleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Wno-deprecated-copy -Wno-ctad-maybe-unsupported -fno-semantic-interposition -fpch-preprocess -O3 -DNDEBUG -fno-semantic-interposition  -Wl,-z,defs -Wl,-z,nodelete   -Wl,-rpath-link,/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/./lib  -Wl,--gc-sections -shared -Wl,-soname,libFortranEvaluate.so.21.0git -o lib/libFortranEvaluate.so.21.0git tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/call.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/characteristics.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/check-expression.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/common.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/complex.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/constant.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/expression.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-character.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-complex.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-designator.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-integer.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-logical.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-real.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-reduction.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/formatting.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/host.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/initial-image.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/integer.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics-library.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/logical.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/real.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/shape.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/static-data.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/target.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/type.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/variable.cpp.o  -Wl,-rpath,"\$ORIGIN/../lib:/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib:"  lib/libFortranDecimal.so.21.0git  lib/libFortranParser.so.21.0git  -lquadmath  lib/libFortranSupport.so.21.0git  lib/libLLVMSupport.so.21.0git  -Wl,-rpath-link,/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib && :
/usr/bin/ld: tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o: in function `Fortran::semantics::CanCUDASymbolHaveSaveAttr(Fortran::semantics::Symbol const&) [clone .localalias]':
tools.cpp:(.text._ZN7Fortran9semantics25CanCUDASymbolHaveSaveAttrERKNS0_6SymbolE+0x5e): undefined reference to `Fortran::semantics::FindCUDADeviceAllocatableUltimateComponent(Fortran::semantics::DerivedTypeSpec const&)'
collect2: error: ld returned 1 exit status
[7522/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IterationSpace.cpp.o
[7523/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CustomIntrinsicCall.cpp.o
[7524/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Bridge.cpp.o
[7525/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o
[7526/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HostAssociations.cpp.o
[7527/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Mangler.cpp.o
[7528/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Runtime.cpp.o
[7529/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o
[7530/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IO.cpp.o
[7531/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/PrivateReductionUtils.cpp.o
[7532/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/SymbolMap.cpp.o
[7533/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/ReductionProcessor.cpp.o
[7534/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExprToHLFIR.cpp.o
[7535/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/semantics.cpp.o
[7536/7899] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/CompilerInvocation.cpp.o
[7537/7899] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/CompilerInstance.cpp.o
[7538/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/expression.cpp.o
[7539/7899] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/CodeGenOptions.cpp.o
[7540/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Atomic.cpp.o
[7541/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Utils.cpp.o
[7542/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-names.cpp.o
[7543/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/VectorSubscripts.cpp.o
[7544/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/unparse-with-symbols.cpp.o
[7545/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/ClauseProcessor.cpp.o
[7546/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/type.cpp.o
[7547/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Decomposer.cpp.o
[7548/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/Utils.cpp.o
[7549/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenACC.cpp.o
[7550/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/OpenMP.cpp.o
[7551/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Clauses.cpp.o
[7552/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/PFTBuilder.cpp.o
[7553/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/DataSharingProcessor.cpp.o
[7554/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExpr.cpp.o
ninja: build stopped: subcommand failed.
Step 7 (build cmake config) failure: build cmake config (failure)
...
[7512/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertCall.cpp.o
[7513/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HlfirIntrinsics.cpp.o
[7514/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertType.cpp.o
[7515/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertProcedureDesignator.cpp.o
[7516/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertArrayConstructor.cpp.o
[7517/7899] Linking CXX shared library lib/libFortranParser.so.21.0git
[7518/7899] Creating library symlink lib/libFortranParser.so
[7519/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertConstant.cpp.o
[7520/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/LoweringOptions.cpp.o
[7521/7899] Linking CXX shared library lib/libFortranEvaluate.so.21.0git
FAILED: lib/libFortranEvaluate.so.21.0git 
: && /usr/bin/c++ -fPIC -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough -Wno-uninitialized -Wno-nonnull -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wdelete-non-virtual-dtor -Wsuggest-override -Wno-comment -Wno-misleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Wno-deprecated-copy -Wno-ctad-maybe-unsupported -fno-semantic-interposition -fpch-preprocess -O3 -DNDEBUG -fno-semantic-interposition  -Wl,-z,defs -Wl,-z,nodelete   -Wl,-rpath-link,/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/./lib  -Wl,--gc-sections -shared -Wl,-soname,libFortranEvaluate.so.21.0git -o lib/libFortranEvaluate.so.21.0git tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/call.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/characteristics.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/check-expression.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/common.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/complex.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/constant.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/expression.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-character.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-complex.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-designator.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-integer.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-logical.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-real.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-reduction.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/formatting.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/host.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/initial-image.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/integer.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics-library.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/logical.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/real.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/shape.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/static-data.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/target.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/type.cpp.o tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/variable.cpp.o  -Wl,-rpath,"\$ORIGIN/../lib:/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib:"  lib/libFortranDecimal.so.21.0git  lib/libFortranParser.so.21.0git  -lquadmath  lib/libFortranSupport.so.21.0git  lib/libLLVMSupport.so.21.0git  -Wl,-rpath-link,/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib && :
/usr/bin/ld: tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/tools.cpp.o: in function `Fortran::semantics::CanCUDASymbolHaveSaveAttr(Fortran::semantics::Symbol const&) [clone .localalias]':
tools.cpp:(.text._ZN7Fortran9semantics25CanCUDASymbolHaveSaveAttrERKNS0_6SymbolE+0x5e): undefined reference to `Fortran::semantics::FindCUDADeviceAllocatableUltimateComponent(Fortran::semantics::DerivedTypeSpec const&)'
collect2: error: ld returned 1 exit status
[7522/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IterationSpace.cpp.o
[7523/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CustomIntrinsicCall.cpp.o
[7524/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Bridge.cpp.o
[7525/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o
[7526/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HostAssociations.cpp.o
[7527/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Mangler.cpp.o
[7528/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Runtime.cpp.o
[7529/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o
[7530/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IO.cpp.o
[7531/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/PrivateReductionUtils.cpp.o
[7532/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/SymbolMap.cpp.o
[7533/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/ReductionProcessor.cpp.o
[7534/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExprToHLFIR.cpp.o
[7535/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/semantics.cpp.o
[7536/7899] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/CompilerInvocation.cpp.o
[7537/7899] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/CompilerInstance.cpp.o
[7538/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/expression.cpp.o
[7539/7899] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/CodeGenOptions.cpp.o
[7540/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Atomic.cpp.o
[7541/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Utils.cpp.o
[7542/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-names.cpp.o
[7543/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/VectorSubscripts.cpp.o
[7544/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/unparse-with-symbols.cpp.o
[7545/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/ClauseProcessor.cpp.o
[7546/7899] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/type.cpp.o
[7547/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Decomposer.cpp.o
[7548/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/Utils.cpp.o
[7549/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenACC.cpp.o
[7550/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/OpenMP.cpp.o
[7551/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Clauses.cpp.o
[7552/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/PFTBuilder.cpp.o
[7553/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/DataSharingProcessor.cpp.o
[7554/7899] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExpr.cpp.o
ninja: build stopped: subcommand failed.

```

</details>

https://github.com/llvm/llvm-project/pull/146780

From flang-commits at lists.llvm.org  Wed Jul  2 17:52:24 2025
From: flang-commits at lists.llvm.org (Valentin Clement via flang-commits)
Date: Wed, 02 Jul 2025 17:52:24 -0700 (PDT)
Subject: [flang-commits] [flang] a535078 - Revert "[flang][cuda] Allocate
 derived-type with CUDA componement in managed memory (#146797)"
Message-ID: <6865d448.050a0220.15ac0f.bd30@mx.google.com>


Author: Valentin Clement
Date: 2025-07-02T17:51:45-07:00
New Revision: a5350785db62f2082bd700fc68bbab9781d03489

URL: https://github.com/llvm/llvm-project/commit/a5350785db62f2082bd700fc68bbab9781d03489
DIFF: https://github.com/llvm/llvm-project/commit/a5350785db62f2082bd700fc68bbab9781d03489.diff

LOG: Revert "[flang][cuda] Allocate derived-type with CUDA componement in managed memory (#146797)"

This reverts commit 925588cd001a91d592b99e6e7c6bee9514f5a26e.

Added: 
    

Modified: 
    flang/lib/Lower/ConvertVariable.cpp
    flang/lib/Semantics/tools.cpp
    flang/test/Lower/CUDA/cuda-derived.cuf

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 44f534e7d569a..7ab3c43016bd9 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -702,29 +702,6 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
   mapSymbolAttributes(converter, var, symMap, stmtCtx, cast);
 }
 
-bool needCUDAAlloc(const Fortran::semantics::Symbol &sym) {
-  if (Fortran::semantics::IsDummy(sym))
-    return false;
-  if (const auto *details{
-          sym.GetUltimate()
-              .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
-    if (details->cudaDataAttr() &&
-        (*details->cudaDataAttr() == Fortran::common::CUDADataAttr::Device ||
-         *details->cudaDataAttr() == Fortran::common::CUDADataAttr::Managed ||
-         *details->cudaDataAttr() == Fortran::common::CUDADataAttr::Unified ||
-         *details->cudaDataAttr() == Fortran::common::CUDADataAttr::Shared ||
-         *details->cudaDataAttr() == Fortran::common::CUDADataAttr::Pinned))
-      return true;
-    const Fortran::semantics::DeclTypeSpec *type{details->type()};
-    const Fortran::semantics::DerivedTypeSpec *derived{type ? type->AsDerived()
-                                                            : nullptr};
-    if (derived)
-      if (FindCUDADeviceAllocatableUltimateComponent(*derived))
-        return true;
-  }
-  return false;
-}
-
 //===----------------------------------------------------------------===//
 // Local variables instantiation (not for alias)
 //===----------------------------------------------------------------===//
@@ -755,7 +732,7 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   if (ultimateSymbol.test(Fortran::semantics::Symbol::Flag::CrayPointee))
     return builder.create<fir::ZeroOp>(loc, fir::ReferenceType::get(ty));
 
-  if (needCUDAAlloc(ultimateSymbol)) {
+  if (Fortran::semantics::NeedCUDAAlloc(ultimateSymbol)) {
     cuf::DataAttributeAttr dataAttr =
         Fortran::lower::translateSymbolCUFDataAttribute(builder.getContext(),
                                                         ultimateSymbol);
@@ -1110,7 +1087,7 @@ static void instantiateLocal(Fortran::lower::AbstractConverter &converter,
     Fortran::lower::defaultInitializeAtRuntime(converter, var.getSymbol(),
                                                symMap);
   auto *builder = &converter.getFirOpBuilder();
-  if (needCUDAAlloc(var.getSymbol()) &&
+  if (Fortran::semantics::NeedCUDAAlloc(var.getSymbol()) &&
       !cuf::isCUDADeviceContext(builder->getRegion())) {
     cuf::DataAttributeAttr dataAttr =
         Fortran::lower::translateSymbolCUFDataAttribute(builder->getContext(),

diff  --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index ea37316e78273..498bbc18709ab 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -1095,20 +1095,9 @@ bool IsDeviceAllocatable(const Symbol &symbol) {
 }
 
 std::optional<common::CUDADataAttr> GetCUDADataAttr(const Symbol *symbol) {
-  const auto *details{
+  const auto *object{
       symbol ? symbol->detailsIf<ObjectEntityDetails>() : nullptr};
-  if (details) {
-    const Fortran::semantics::DeclTypeSpec *type{details->type()};
-    const Fortran::semantics::DerivedTypeSpec *derived{
-        type ? type->AsDerived() : nullptr};
-    if (derived) {
-      if (FindCUDADeviceAllocatableUltimateComponent(*derived)) {
-        return common::CUDADataAttr::Managed;
-      }
-    }
-    return details->cudaDataAttr();
-  }
-  return std::nullopt;
+  return object ? object->cudaDataAttr() : std::nullopt;
 }
 
 bool IsAccessible(const Symbol &original, const Scope &scope) {

diff  --git a/flang/test/Lower/CUDA/cuda-derived.cuf b/flang/test/Lower/CUDA/cuda-derived.cuf
index 96250d88d81c4..d280ac722d08f 100644
--- a/flang/test/Lower/CUDA/cuda-derived.cuf
+++ b/flang/test/Lower/CUDA/cuda-derived.cuf
@@ -7,16 +7,6 @@ module m1
 
   type t1; real, device, allocatable :: a(:); end type
   type t2; type(t1) :: b; end type
-contains
-  subroutine sub1()
-    type(ty_device) :: a
-  end subroutine
-
-! CHECK-LABEL: func.func @_QMm1Psub1()
-! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} -> !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
-! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>)
-! CHECK: cuf.free %[[DECL]]#0 : !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>> {data_attr = #cuf.cuda<managed>}
-
 end module
 
 program main
@@ -26,5 +16,5 @@ program main
 end
 
 ! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
-! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEa"}
-! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tt2{b:!fir.type<_QMm1Tt1{a:!fir.box<!fir.heap<!fir.array<?xf32>>>}>}> {bindc_name = "b", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEb"}
+! CHECK: %{{.*}} = fir.alloca !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", uniq_name = "_QFEa"}
+! CHECK: %{{.*}} = fir.alloca !fir.type<_QMm1Tt2{b:!fir.type<_QMm1Tt1{a:!fir.box<!fir.heap<!fir.array<?xf32>>>}>}> {bindc_name = "b", uniq_name = "_QFEb"}


        

From flang-commits at lists.llvm.org  Wed Jul  2 17:52:26 2025
From: flang-commits at lists.llvm.org (Valentin Clement via flang-commits)
Date: Wed, 02 Jul 2025 17:52:26 -0700 (PDT)
Subject: [flang-commits] [flang] e718ce0 - Revert "[flang][cuda] Do not
 create global for derived-type with allocatable device components
 (#146780)"
Message-ID: <6865d44a.050a0220.3aba87.da41@mx.google.com>


Author: Valentin Clement
Date: 2025-07-02T17:51:55-07:00
New Revision: e718ce00374db478f2387cfe6214d167ec8d309a

URL: https://github.com/llvm/llvm-project/commit/e718ce00374db478f2387cfe6214d167ec8d309a
DIFF: https://github.com/llvm/llvm-project/commit/e718ce00374db478f2387cfe6214d167ec8d309a.diff

LOG: Revert "[flang][cuda] Do not create global for derived-type with allocatable device components (#146780)"

This reverts commit e873ce31ae0e875081c8e5480c9c4925c97469ce.

Added: 
    

Modified: 
    flang/include/flang/Evaluate/tools.h
    flang/include/flang/Semantics/tools.h
    flang/lib/Evaluate/tools.cpp
    flang/lib/Semantics/tools.cpp

Removed: 
    flang/test/Lower/CUDA/cuda-derived.cuf


################################################################################
diff  --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 814fe4cccede6..cad1b634f8924 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1286,7 +1286,16 @@ bool CheckForCoindexedObject(parser::ContextualMessages &,
     const std::optional<ActualArgument> &, const std::string &procName,
     const std::string &argName);
 
-bool CanCUDASymbolHaveSaveAttr(const Symbol &sym);
+inline bool CanCUDASymbolHaveSaveAttr(const Symbol &sym) {
+  if (const auto *details =
+          sym.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()) {
+    if (details->cudaDataAttr() &&
+        *details->cudaDataAttr() != common::CUDADataAttr::Unified) {
+      return false;
+    }
+  }
+  return true;
+}
 
 inline bool IsCUDADeviceSymbol(const Symbol &sym) {
   if (const auto *details =

diff  --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index ca58e1065d3e5..f3cfa9b99fb4d 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -654,8 +654,6 @@ DirectComponentIterator::const_iterator FindAllocatableOrPointerDirectComponent(
     const DerivedTypeSpec &);
 PotentialComponentIterator::const_iterator
 FindPolymorphicAllocatablePotentialComponent(const DerivedTypeSpec &);
-UltimateComponentIterator::const_iterator
-FindCUDADeviceAllocatableUltimateComponent(const DerivedTypeSpec &);
 
 // The LabelEnforce class (given a set of labels) provides an error message if
 // there is a branch to a label which is not in the given set.

diff  --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index d912fc167a62e..fcacdb93d662b 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -2173,25 +2173,6 @@ bool IsAutomatic(const Symbol &original) {
   return false;
 }
 
-bool CanCUDASymbolHaveSaveAttr(const Symbol &sym) {
-  if (const auto *details{
-          sym.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()}) {
-    const Fortran::semantics::DeclTypeSpec *type{details->type()};
-    const Fortran::semantics::DerivedTypeSpec *derived{
-        type ? type->AsDerived() : nullptr};
-    if (derived) {
-      if (FindCUDADeviceAllocatableUltimateComponent(*derived)) {
-        return false;
-      }
-    }
-    if (details->cudaDataAttr() &&
-        *details->cudaDataAttr() != common::CUDADataAttr::Unified) {
-      return false;
-    }
-  }
-  return true;
-}
-
 bool IsSaved(const Symbol &original) {
   const Symbol &symbol{GetAssociationRoot(original)};
   const Scope &scope{symbol.owner()};
@@ -2214,7 +2195,7 @@ bool IsSaved(const Symbol &original) {
   } else if (scopeKind == Scope::Kind::Module ||
       (scopeKind == Scope::Kind::MainProgram &&
           (symbol.attrs().test(Attr::TARGET) || evaluate::IsCoarray(symbol)) &&
-          CanCUDASymbolHaveSaveAttr(symbol))) {
+          Fortran::evaluate::CanCUDASymbolHaveSaveAttr(symbol))) {
     // 8.5.16p4
     // In main programs, implied SAVE matters only for pointer
     // initialization targets and coarrays.
@@ -2224,7 +2205,7 @@ bool IsSaved(const Symbol &original) {
           (features.IsEnabled(
                common::LanguageFeature::SaveBigMainProgramVariables) &&
               symbol.size() > 32)) &&
-      CanCUDASymbolHaveSaveAttr(symbol)) {
+      Fortran::evaluate::CanCUDASymbolHaveSaveAttr(symbol)) {
     // With SaveBigMainProgramVariables, keeping all unsaved main program
     // variables of 32 bytes or less on the stack allows keeping numerical and
     // logical scalars, small scalar characters or derived, small arrays, and
@@ -2242,15 +2223,15 @@ bool IsSaved(const Symbol &original) {
   } else if (symbol.test(Symbol::Flag::InDataStmt)) {
     return true;
   } else if (const auto *object{symbol.detailsIf<ObjectEntityDetails>()};
-      object && object->init()) {
+             object && object->init()) {
     return true;
   } else if (IsProcedurePointer(symbol) && symbol.has<ProcEntityDetails>() &&
       symbol.get<ProcEntityDetails>().init()) {
     return true;
   } else if (scope.hasSAVE()) {
     return true; // bare SAVE statement
-  } else if (const Symbol *block{FindCommonBlockContaining(symbol)};
-      block && block->attrs().test(Attr::SAVE)) {
+  } else if (const Symbol * block{FindCommonBlockContaining(symbol)};
+             block && block->attrs().test(Attr::SAVE)) {
     return true; // in COMMON with SAVE
   } else {
     return false;

diff  --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 498bbc18709ab..d053179448c00 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -1081,19 +1081,6 @@ const Scope *FindCUDADeviceContext(const Scope *scope) {
   });
 }
 
-bool IsDeviceAllocatable(const Symbol &symbol) {
-  if (IsAllocatable(symbol)) {
-    if (const auto *details{
-            symbol.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()}) {
-      if (details->cudaDataAttr() &&
-          *details->cudaDataAttr() != common::CUDADataAttr::Pinned) {
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
 std::optional<common::CUDADataAttr> GetCUDADataAttr(const Symbol *symbol) {
   const auto *object{
       symbol ? symbol->detailsIf<ObjectEntityDetails>() : nullptr};
@@ -1439,12 +1426,6 @@ FindPolymorphicAllocatablePotentialComponent(const DerivedTypeSpec &derived) {
       potentials.begin(), potentials.end(), IsPolymorphicAllocatable);
 }
 
-UltimateComponentIterator::const_iterator
-FindCUDADeviceAllocatableUltimateComponent(const DerivedTypeSpec &derived) {
-  UltimateComponentIterator ultimates{derived};
-  return std::find_if(ultimates.begin(), ultimates.end(), IsDeviceAllocatable);
-}
-
 const Symbol *FindUltimateComponent(const DerivedTypeSpec &derived,
     const std::function<bool(const Symbol &)> &predicate) {
   UltimateComponentIterator ultimates{derived};
@@ -1807,4 +1788,4 @@ bool HadUseError(
   }
 }
 
-} // namespace Fortran::semantics
+} // namespace Fortran::semantics
\ No newline at end of file

diff  --git a/flang/test/Lower/CUDA/cuda-derived.cuf b/flang/test/Lower/CUDA/cuda-derived.cuf
deleted file mode 100644
index d280ac722d08f..0000000000000
--- a/flang/test/Lower/CUDA/cuda-derived.cuf
+++ /dev/null
@@ -1,20 +0,0 @@
-! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
-
-module m1
-  type ty_device
-    integer, device, allocatable, dimension(:) :: x
-  end type
-
-  type t1; real, device, allocatable :: a(:); end type
-  type t2; type(t1) :: b; end type
-end module
-
-program main
-  use m1
-  type(ty_device) :: a
-  type(t2) :: b
-end
-
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
-! CHECK: %{{.*}} = fir.alloca !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", uniq_name = "_QFEa"}
-! CHECK: %{{.*}} = fir.alloca !fir.type<_QMm1Tt2{b:!fir.type<_QMm1Tt1{a:!fir.box<!fir.heap<!fir.array<?xf32>>>}>}> {bindc_name = "b", uniq_name = "_QFEb"}


        

From flang-commits at lists.llvm.org  Wed Jul  2 21:29:43 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 21:29:43 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][OpenMP] Allocate `reduction`
 init temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <68660737.050a0220.25179e.0470@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/146667

>From 9a48bff311756ff1c3703f33d6acaecee195fe1e Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 05:31:38 -0500
Subject: [PATCH 1/2] [flang][OpenMP] Allocate `reduction` init temps on the
 stack for GPUs

Temps needed for the reduction init regions are now allocate on the heap
all the time. However, this is performance killer for GPUs since malloc
calls are prohibitively expensive. Therefore, we should do these
allocations on the stack for GPU reductions.
---
 .../Lower/Support/PrivateReductionUtils.cpp   |  47 +++--
 .../Lower/OpenMP/parallel-reduction-array.f90 | 166 ++++++++++--------
 2 files changed, 119 insertions(+), 94 deletions(-)

diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index e878041d37c03..c3a5b6101ce00 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -502,22 +502,37 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
 
   // Allocating on the heap in case the whole reduction/privatization is nested
   // inside of a loop
-  auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
-  // if needsDealloc isn't statically false, add cleanup region. Always
-  // do this for allocatable boxes because they might have been re-allocated
-  // in the body of the loop/parallel region
-
-  std::optional<int64_t> cstNeedsDealloc = fir::getIntIfConstant(needsDealloc);
-  assert(cstNeedsDealloc.has_value() &&
-         "createTempFromMold decides this statically");
-  if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
-                        isDoConcurrent);
-  } else {
-    assert(!isAllocatableOrPointer &&
-           "Pointer-like arrays must be heap allocated");
-  }
+  auto temp = [&]() {
+    bool shouldAllocateOnStack = false;
+
+    // On the GPU, always allocate on the stack since heap allocatins are very
+    // expensive.
+    if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
+            *builder.getModule()))
+      shouldAllocateOnStack = offloadMod.getIsGPU();
+
+    if (shouldAllocateOnStack)
+      return createStackTempFromMold(loc, builder, source);
+
+    auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
+    // if needsDealloc isn't statically false, add cleanup region. Always
+    // do this for allocatable boxes because they might have been re-allocated
+    // in the body of the loop/parallel region
+
+    std::optional<int64_t> cstNeedsDealloc =
+        fir::getIntIfConstant(needsDealloc);
+    assert(cstNeedsDealloc.has_value() &&
+           "createTempFromMold decides this statically");
+    if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
+                          isDoConcurrent);
+    } else {
+      assert(!isAllocatableOrPointer &&
+             "Pointer-like arrays must be heap allocated");
+    }
+    return temp;
+  }();
 
   // Put the temporary inside of a box:
   // hlfir::genVariableBox doesn't handle non-default lower bounds
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 8e3de498f59c1..4f889d9a4e77f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -1,5 +1,8 @@
-! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+
+! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
 
 program reduce
 integer, dimension(3) :: i = 0
@@ -13,81 +16,88 @@ program reduce
 print *,i
 end program
 
-! CHECK-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
-! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK-LABEL:   } init {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
-! CHECK:           %[[TRUE:.*]]  = arith.constant true
+! CPU-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
+! CPU:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU-LABEL:   } init {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = arith.constant 0 : i32
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_4:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CPU:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
+! CPU:           %[[TRUE:.*]]  = arith.constant true
 !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
-! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         } combiner {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[C1_0:.*]] = arith.constant 1 : index
-! CHECK:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
-! CHECK:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
-! CHECK:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
-! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
-! CHECK:           }
-! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         }  cleanup {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
-! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
-! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
-! CHECK:           fir.if %[[VAL_5]] {
-! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
-! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
-! CHECK:           }
-! CHECK:           omp.yield
-! CHECK:         }
+! CPU:           %[[C0:.*]] = arith.constant 0 : index
+! CPU:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
+! CPU:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         } combiner {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[C1:.*]] = arith.constant 1 : index
+! CPU:           %[[C3:.*]] = arith.constant 3 : index
+! CPU:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[C1_0:.*]] = arith.constant 1 : index
+! CPU:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
+! CPU:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+! CPU:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
+! CPU:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
+! CPU:           }
+! CPU:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         }  cleanup {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
+! CPU:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CPU:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CPU:           fir.if %[[VAL_5]] {
+! CPU:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
+! CPU:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
+! CPU:           }
+! CPU:           omp.yield
+! CPU:         }
+
+! CPU-LABEL:   func.func @_QQmain()
+! CPU:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_1:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CPU:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
+! CPU:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:             %[[VAL_8:.*]] = arith.constant 1 : i32
+! CPU:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_10:.*]] = arith.constant 1 : index
+! CPU:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = arith.constant 2 : i32
+! CPU:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_14:.*]] = arith.constant 2 : index
+! CPU:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_16:.*]] = arith.constant 3 : i32
+! CPU:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_18:.*]] = arith.constant 3 : index
+! CPU:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
+! CPU:             omp.terminator
+! CPU:           }
 
-! CHECK-LABEL:   func.func @_QQmain()
-! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
-! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
-! CHECK:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : index
-! CHECK:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = arith.constant 2 : i32
-! CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_14:.*]] = arith.constant 2 : index
-! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_16:.*]] = arith.constant 3 : i32
-! CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_18:.*]] = arith.constant 3 : index
-! CHECK:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
-! CHECK:             omp.terminator
-! CHECK:           }
+! GPU: omp.declare_reduction {{.*}} alloc {
+! GPU: } init {
+! GPU-NOT: fir.allocmem {{.*}} {bindc_name = ".tmp", {{.*}}}
+! GPU:     fir.alloca {{.*}} {bindc_name = ".tmp"}
+! GPU: } combiner {
+! GPU: }

>From c3777c9566273740206705d1c91262eca9995e02 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 23:29:23 -0500
Subject: [PATCH 2/2] add todo

---
 .../LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b07243c5f843..a55239ba1f28f 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -1291,6 +1291,11 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
     mapInitializationArgs(op, moduleTranslation, reductionDecls,
                           reductionVariableMap, i);
 
+    // TODO In some cases (specially on the GPU), the init regions may
+    // contains stack alloctaions. If the region is inlined in a loop, this is
+    // problematic. Instead of just inlining the region, handle allocations by
+    // hoisting fixed length allocations to the function entry and using
+    // stacksave and restore for variable length ones.
     if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
                                        "omp.reduction.neutral", builder,
                                        moduleTranslation, &phis)))


From flang-commits at lists.llvm.org  Wed Jul  2 21:30:24 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 02 Jul 2025 21:30:24 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][OpenMP] Allocate `reduction`
 init temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <68660760.170a0220.2e1482.75c9@mx.google.com>


ergawy wrote:

@tblah @kiranchandramohan thanks for taking a look at the PR, I added a TODO to handle allocations inside init regions as Kiran suggested.

https://github.com/llvm/llvm-project/pull/146667

From flang-commits at lists.llvm.org  Wed Jul  2 22:38:50 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 02 Jul 2025 22:38:50 -0700 (PDT)
Subject: [flang-commits] [flang] d67013a - [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (#146453)
Message-ID: <6866176a.170a0220.330c48.45ce@mx.google.com>


Author: KAWASHIMA Takahiro
Date: 2025-07-03T14:38:45+09:00
New Revision: d67013a2b44295e7558b6678f07c7f3a7ef9601c

URL: https://github.com/llvm/llvm-project/commit/d67013a2b44295e7558b6678f07c7f3a7ef9601c
DIFF: https://github.com/llvm/llvm-project/commit/d67013a2b44295e7558b6678f07c7f3a7ef9601c.diff

LOG: [Flang][AArch64][VecLib] Add libmvec support for Flang/AArch64 (#146453)

`-fveclib=libmvec` for AArch64 (NEON and SVE) in Clang was supported by
#143696. This patch does the same for Flang.

Vector functions defined in `libmvec` are used for the following Fortran
operator and functions currently.

- Power operator (`**`)
- Fortran intrinsic functions listed below for `real(kind=4)` and
`real(kind=8)` (including their coresponding specific intrinsic
functions)
- Fortran intrinsic functions which are expanded using functions listed
below (for example, `sin` for `complex(kind=8)`)

```
sin
tan
cos
asin
acos
atan (both atan(x) and atan(y, x))
atan2
cosh
tanh
asinh
acosh
atanh
erf
erfc
exp
log
log10
```

As with Clang/AArch64, glibc 2.40 or higher is required to use all these
functions.

Added: 
    

Modified: 
    clang/include/clang/Driver/Options.td
    clang/lib/Driver/ToolChains/Flang.cpp
    flang/docs/ReleaseNotes.md
    flang/test/Driver/fveclib-codegen.f90
    flang/test/Driver/fveclib.f90

Removed: 
    


################################################################################
diff  --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 5831ea73561ff..697117c06a6b2 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3489,7 +3489,8 @@ def fno_experimental_isel : Flag<["-"], "fno-experimental-isel">, Group<f_clang_
   Alias<fno_global_isel>;
 def fveclib : Joined<["-"], "fveclib=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
-    HelpText<"Use the given vector functions library">,
+    HelpText<"Use the given vector functions library.\n"
+             "  Note: -fveclib=libmvec on AArch64 requires GLIBC 2.40 or newer.">,
     HelpTextForVariants<[ClangOption, CC1Option],
       "Use the given vector functions library.\n"
       "  Note: -fveclib={ArmPL,SLEEF,libmvec} implies -fno-math-errno.\n"

diff  --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index e4e321ba1e195..75ba2af543c7a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -484,11 +484,18 @@ void Flang::addTargetOptions(const ArgList &Args,
           Triple.getArch() != llvm::Triple::x86_64)
         D.Diag(diag::err_drv_unsupported_opt_for_target)
             << Name << Triple.getArchName();
-    } else if (Name == "libmvec" || Name == "AMDLIBM") {
+    } else if (Name == "AMDLIBM") {
       if (Triple.getArch() != llvm::Triple::x86 &&
           Triple.getArch() != llvm::Triple::x86_64)
         D.Diag(diag::err_drv_unsupported_opt_for_target)
             << Name << Triple.getArchName();
+    } else if (Name == "libmvec") {
+      if (Triple.getArch() != llvm::Triple::x86 &&
+          Triple.getArch() != llvm::Triple::x86_64 &&
+          Triple.getArch() != llvm::Triple::aarch64 &&
+          Triple.getArch() != llvm::Triple::aarch64_be)
+        D.Diag(diag::err_drv_unsupported_opt_for_target)
+            << Name << Triple.getArchName();
     } else if (Name == "SLEEF" || Name == "ArmPL") {
       if (Triple.getArch() != llvm::Triple::aarch64 &&
           Triple.getArch() != llvm::Triple::aarch64_be)

diff  --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index 35da8323e0a10..b2dbbcb5630f4 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -34,6 +34,8 @@ page](https://llvm.org/releases/).
 
 * -floop-interchange is now recognized by flang.
 * -floop-interchange is enabled by default at -O2 and above.
+* -fveclib=libmvec is supported for AArch64 (same as Flang/x86 and
+  Clang/AArch64) (requires GLIBC 2.40 or newer)
 
 ## Windows Support
 

diff  --git a/flang/test/Driver/fveclib-codegen.f90 b/flang/test/Driver/fveclib-codegen.f90
index 4cbb1e284f18e..88b31da40167e 100644
--- a/flang/test/Driver/fveclib-codegen.f90
+++ b/flang/test/Driver/fveclib-codegen.f90
@@ -1,6 +1,8 @@
 ! test that -fveclib= is passed to the backend
 ! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=SLEEF -o - %s | FileCheck %s --check-prefix=SLEEF %}
 ! RUN: %if x86-registered-target %{ %flang -S -Ofast -target x86_64-unknown-linux-gnu -fveclib=libmvec -o - %s | FileCheck %s %}
+! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=libmvec -march=armv8.2-a+nosve -o - %s | FileCheck %s --check-prefix=LIBMVEC-AARCH64-NEON %}
+! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=libmvec -march=armv8.2-a+sve -o - %s | FileCheck %s --check-prefix=LIBMVEC-AARCH64-SVE %}
 ! RUN: %if x86-registered-target %{ %flang -S -O3 -ffast-math -target x86_64-unknown-linux-gnu -fveclib=AMDLIBM -o - %s | FileCheck %s --check-prefix=AMDLIBM %}
 ! RUN: %flang -S -Ofast -fveclib=NoLibrary -o - %s | FileCheck %s --check-prefix=NOLIB
 
@@ -11,6 +13,8 @@ subroutine sb(a, b)
 ! check that we used a vectorized call to powf()
 ! CHECK: _ZGVbN4vv_powf
 ! SLEEF: _ZGVnN4vv_powf
+! LIBMVEC-AARCH64-NEON: _ZGVnN4vv_powf
+! LIBMVEC-AARCH64-SVE: _ZGVsMxvv_powf
 ! AMDLIBM: amd_vrs4_powf
 ! NOLIB: powf
     a(i) = a(i) ** b(i)

diff  --git a/flang/test/Driver/fveclib.f90 b/flang/test/Driver/fveclib.f90
index 431a4bfc02522..d21e85e486f8d 100644
--- a/flang/test/Driver/fveclib.f90
+++ b/flang/test/Driver/fveclib.f90
@@ -1,6 +1,7 @@
 ! RUN: %flang -### -c -fveclib=none %s 2>&1 | FileCheck -check-prefix CHECK-NOLIB %s
 ! RUN: %flang -### -c -fveclib=Accelerate %s 2>&1 | FileCheck -check-prefix CHECK-ACCELERATE %s
 ! RUN: %flang -### -c --target=x86_64-unknown-linux-gnu -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-libmvec %s
+! RUN: %flang -### -c --target=aarch64-unknown-linux-gnu -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-libmvec %s
 ! RUN: %flang -### -c -fveclib=MASSV %s 2>&1 | FileCheck -check-prefix CHECK-MASSV %s
 ! RUN: %flang -### -c -fveclib=Darwin_libsystem_m %s 2>&1 | FileCheck -check-prefix CHECK-DARWIN_LIBSYSTEM_M %s
 ! RUN: %flang -### -c --target=aarch64-none-none -fveclib=SLEEF %s 2>&1 | FileCheck -check-prefix CHECK-SLEEF %s
@@ -23,7 +24,7 @@
 
 ! RUN: not %flang --target=x86-none-none -c -fveclib=SLEEF %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=x86-none-none -c -fveclib=ArmPL %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
-! RUN: not %flang --target=aarch64-none-none -c -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
+! RUN: not %flang --target=riscv64-none-none -c -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=aarch64-none-none -c -fveclib=SVML %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=aarch64-none-none -c -fveclib=AMDLIBM %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! CHECK-ERROR: unsupported option {{.*}} for target


        

From flang-commits at lists.llvm.org  Wed Jul  2 22:38:52 2025
From: flang-commits at lists.llvm.org (KAWASHIMA Takahiro via flang-commits)
Date: Wed, 02 Jul 2025 22:38:52 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <6866176c.a70a0220.280b69.c978@mx.google.com>


https://github.com/kawashima-fj closed https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Wed Jul  2 22:49:02 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 02 Jul 2025 22:49:02 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang][AArch64][VecLib] Add
 libmvec support for Flang/AArch64 (PR #146453)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146453@github.com>
Message-ID: <686619ce.170a0220.10395d.6fde@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `clang-hip-vega20` running on `hip-vega20-0` while building `clang,flang` at step 3 "annotate".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/123/builds/22559

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 3 (annotate) failure: '../llvm-zorg/zorg/buildbot/builders/annotated/hip-build.sh --jobs=' (failure)
...
[57/59] Linking CXX executable External/HIP/cmath-hip-6.3.0
[58/59] Building CXX object External/HIP/CMakeFiles/TheNextWeek-hip-6.3.0.dir/workload/ray-tracing/TheNextWeek/main.cc.o
[59/59] Linking CXX executable External/HIP/TheNextWeek-hip-6.3.0
+ build_step 'Testing HIP test-suite'
+ echo '@@@BUILD_STEP Testing HIP test-suite@@@'
@@@BUILD_STEP Testing HIP test-suite@@@
+ ninja check-hip-simple
[0/1] cd /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/External/HIP && /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/llvm/bin/llvm-lit -sv array-hip-6.3.0.test empty-hip-6.3.0.test with-fopenmp-hip-6.3.0.test saxpy-hip-6.3.0.test memmove-hip-6.3.0.test split-kernel-args-hip-6.3.0.test builtin-logb-scalbn-hip-6.3.0.test TheNextWeek-hip-6.3.0.test algorithm-hip-6.3.0.test cmath-hip-6.3.0.test complex-hip-6.3.0.test math_h-hip-6.3.0.test new-hip-6.3.0.test blender.test
-- Testing: 14 tests, 14 workers --
Testing:  0.. 10.. 20.. 30.. 40.. 50.. 60.. 70.. 80.. 90
FAIL: test-suite :: External/HIP/blender.test (14 of 14)
******************** TEST 'test-suite :: External/HIP/blender.test' FAILED ********************

/home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/tools/timeit-target --timeout 7200 --limit-core 0 --limit-cpu 7200 --limit-file-size 209715200 --limit-rss-size 838860800 --append-exitstatus --redirect-output /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/External/HIP/Output/blender.test.out --redirect-input /dev/null --summary /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/External/HIP/Output/blender.test.time /bin/bash test_blender.sh
/bin/bash verify_blender.sh /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/External/HIP/Output/blender.test.out
Begin Blender test.
TEST_SUITE_HIP_ROOT=/opt/botworker/llvm/External/hip
Render /opt/botworker/llvm/External/hip/Blender_Scenes/290skydemo_release.blend
Blender 4.1.1 (hash e1743a0317bc built 2024-04-15 23:47:45)
Read blend: "/opt/botworker/llvm/External/hip/Blender_Scenes/290skydemo_release.blend"
Could not open as Ogawa file from provided streams.
Unable to open /opt/botworker/llvm/External/hip/Blender_Scenes/290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.002", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.003", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.004", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.001", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
Could not open as Ogawa file from provided streams.
Unable to open /opt/botworker/llvm/External/hip/Blender_Scenes/290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.003", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.002", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.004", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.001", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
I0703 05:43:26.044307 3931561 device.cpp:39] HIPEW initialization succeeded
I0703 05:43:26.046325 3931561 device.cpp:45] Found HIPCC hipcc
I0703 05:43:26.125037 3931561 device.cpp:207] Device has compute preemption or is not used for display.
I0703 05:43:26.125093 3931561 device.cpp:211] Added device "" with id "HIP__0000:a3:00".
I0703 05:43:26.125188 3931561 device.cpp:568] Mapped host memory limit set to 536,444,985,344 bytes. (499.60G)
I0703 05:43:26.125452 3931561 device_impl.cpp:63] Using AVX2 CPU kernels.
Fra:1 Mem:524.00M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Eyepiece_rim
Fra:1 Mem:524.00M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Rivets.013
Fra:1 Mem:524.00M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Rivets.017
Fra:1 Mem:524.11M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Rivets.025
Fra:1 Mem:524.24M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Rivets.039
Fra:1 Mem:525.51M (Peak 525.51M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Hoses.003
Fra:1 Mem:531.49M (Peak 531.49M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Curve_Connectors
Fra:1 Mem:531.60M (Peak 531.60M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Curve_Connectors.004
Fra:1 Mem:531.64M (Peak 531.61M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Curve_Connectors.006
Step 12 (Testing HIP test-suite) failure: Testing HIP test-suite (failure)
@@@BUILD_STEP Testing HIP test-suite@@@
+ ninja check-hip-simple
[0/1] cd /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/External/HIP && /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/llvm/bin/llvm-lit -sv array-hip-6.3.0.test empty-hip-6.3.0.test with-fopenmp-hip-6.3.0.test saxpy-hip-6.3.0.test memmove-hip-6.3.0.test split-kernel-args-hip-6.3.0.test builtin-logb-scalbn-hip-6.3.0.test TheNextWeek-hip-6.3.0.test algorithm-hip-6.3.0.test cmath-hip-6.3.0.test complex-hip-6.3.0.test math_h-hip-6.3.0.test new-hip-6.3.0.test blender.test
-- Testing: 14 tests, 14 workers --
Testing:  0.. 10.. 20.. 30.. 40.. 50.. 60.. 70.. 80.. 90
FAIL: test-suite :: External/HIP/blender.test (14 of 14)
******************** TEST 'test-suite :: External/HIP/blender.test' FAILED ********************

/home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/tools/timeit-target --timeout 7200 --limit-core 0 --limit-cpu 7200 --limit-file-size 209715200 --limit-rss-size 838860800 --append-exitstatus --redirect-output /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/External/HIP/Output/blender.test.out --redirect-input /dev/null --summary /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/External/HIP/Output/blender.test.time /bin/bash test_blender.sh
/bin/bash verify_blender.sh /home/botworker/bbot/clang-hip-vega20/botworker/clang-hip-vega20/test-suite-build/External/HIP/Output/blender.test.out
Begin Blender test.
TEST_SUITE_HIP_ROOT=/opt/botworker/llvm/External/hip
Render /opt/botworker/llvm/External/hip/Blender_Scenes/290skydemo_release.blend
Blender 4.1.1 (hash e1743a0317bc built 2024-04-15 23:47:45)
Read blend: "/opt/botworker/llvm/External/hip/Blender_Scenes/290skydemo_release.blend"
Could not open as Ogawa file from provided streams.
Unable to open /opt/botworker/llvm/External/hip/Blender_Scenes/290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.002", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.003", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.004", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.001", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
Could not open as Ogawa file from provided streams.
Unable to open /opt/botworker/llvm/External/hip/Blender_Scenes/290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.003", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.002", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.004", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
WARN (bke.modifier): source/blender/blenkernel/intern/modifier.cc:425 BKE_modifier_set_error: Object: "GEO-flag.001", Modifier: "MeshSequenceCache", Could not create reader for file //290skydemo2_flags.abc
I0703 05:43:26.044307 3931561 device.cpp:39] HIPEW initialization succeeded
I0703 05:43:26.046325 3931561 device.cpp:45] Found HIPCC hipcc
I0703 05:43:26.125037 3931561 device.cpp:207] Device has compute preemption or is not used for display.
I0703 05:43:26.125093 3931561 device.cpp:211] Added device "" with id "HIP__0000:a3:00".
I0703 05:43:26.125188 3931561 device.cpp:568] Mapped host memory limit set to 536,444,985,344 bytes. (499.60G)
I0703 05:43:26.125452 3931561 device_impl.cpp:63] Using AVX2 CPU kernels.
Fra:1 Mem:524.00M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Eyepiece_rim
Fra:1 Mem:524.00M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Rivets.013
Fra:1 Mem:524.00M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Rivets.017
Fra:1 Mem:524.11M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Rivets.025
Fra:1 Mem:524.24M (Peak 524.70M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Rivets.039
Fra:1 Mem:525.51M (Peak 525.51M) | Time:00:00.69 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Hoses.003
Fra:1 Mem:531.49M (Peak 531.49M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Curve_Connectors
Fra:1 Mem:531.60M (Peak 531.60M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Curve_Connectors.004
Fra:1 Mem:531.64M (Peak 531.61M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Curve_Connectors.006
Fra:1 Mem:531.61M (Peak 531.61M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Curve_Connectors.009
Fra:1 Mem:532.23M (Peak 532.23M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | Cylinder.029
Fra:1 Mem:534.70M (Peak 534.70M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Pistons
Fra:1 Mem:539.57M (Peak 539.57M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Head_greeble.005
Fra:1 Mem:539.59M (Peak 539.59M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Head_greeble.007
Fra:1 Mem:539.90M (Peak 539.90M) | Time:00:00.70 | Mem:0.00M, Peak:0.00M | Scene, View Layer | Synchronizing object | GEO-Head_greeble.008

```

</details>

https://github.com/llvm/llvm-project/pull/146453

From flang-commits at lists.llvm.org  Wed Jul  2 15:47:13 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Wed, 02 Jul 2025 15:47:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <6865b6f1.170a0220.c0e7c.25b9@mx.google.com>


================
@@ -786,13 +797,59 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   mlir::Value shape = hlfir::genShape(loc, builder, lhs);
   llvm::SmallVector<mlir::Value> extents =
       hlfir::getIndexExtents(loc, builder, shape);
-  hlfir::LoopNest loopNest =
-      hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
-                         flangomp::shouldUseWorkshareLowering(assign));
-  builder.setInsertionPointToStart(loopNest.body);
-  auto arrayElement =
-      hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-  builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+
+  bool isArrayRef =
+      mlir::isa<fir::SequenceType>(fir::unwrapRefType(lhs.getType()));
+  if (lhs.isSimplyContiguous() && extents.size() > 1 &&
----------------
vzakhari wrote:

I believe `isSimplyContiguous` may also return true for `!fir.box<!fir.ptr<...`.  I do not think you need to explicitly handle all these different cases.  There is `hlfir::derefPointersAndAllocatables` call above.  If `lhs` is a box, you may use the box_addr operation to get the base address of the array, otherwise, `lhs` is the base address of the array already.

https://github.com/llvm/llvm-project/pull/146408

From flang-commits at lists.llvm.org  Thu Jul  3 00:37:31 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 00:37:31 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] fix skip-external-rtti-definition
	for ppc (PR #146826)
Message-ID: <llvm.org/llvm/llvm-project/pull/146826@github.com>


https://github.com/jeanPerier created https://github.com/llvm/llvm-project/pull/146826

PPC does not use comdat. There is no need to check for that in the test, just remove it.

Fix for https://lab.llvm.org/buildbot/#/builders/201/builds/5278

>From 3d85a279cb5591fd9a3ccdb497864052c691f7d3 Mon Sep 17 00:00:00 2001
From: Jean Perier <jperier at nvidia.com>
Date: Thu, 3 Jul 2025 00:33:24 -0700
Subject: [PATCH] [flang] fix skip-external-rtti-definition for ppc

---
 flang/test/Integration/skip-external-rtti-definition.F90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Integration/skip-external-rtti-definition.F90 b/flang/test/Integration/skip-external-rtti-definition.F90
index f86cc069b304d..35a5d559efa5e 100644
--- a/flang/test/Integration/skip-external-rtti-definition.F90
+++ b/flang/test/Integration/skip-external-rtti-definition.F90
@@ -33,9 +33,9 @@ subroutine needs_descriptor(x)
 
 #endif
 
-! LINKONCE-DAG: @_QMmodule_external_type_definitionEXnXt1 = linkonce_odr constant [2 x i8] c"t1", comdat
+! LINKONCE-DAG: @_QMmodule_external_type_definitionEXnXt1 = linkonce_odr constant [2 x i8] c"t1"
 ! LINKONCE-DAG: @_QMmodule_external_type_definitionEXdtXt1 = linkonce_odr constant {{.*}} {
-! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXnXt2 = linkonce_odr constant [2 x i8] c"t2", comdat
+! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXnXt2 = linkonce_odr constant [2 x i8] c"t2"
 ! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXdtXt2 = linkonce_odr constant {{.*}} {
 
 ! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1


From flang-commits at lists.llvm.org  Thu Jul  3 00:55:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 00:55:45 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68663781.050a0220.3d8f7a.dfd8@mx.google.com>


https://github.com/jeanPerier approved this pull request.

Thanks!

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 00:55:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 00:55:45 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68663781.050a0220.25e33.ca7d@mx.google.com>


https://github.com/jeanPerier edited https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 00:55:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 00:55:46 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68663782.170a0220.2ad8ba.7fcf@mx.google.com>


================
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 #define FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 
+#include "flang/Frontend/CodeGenOptions.h"
----------------
jeanPerier wrote:

Not requesting to do that here, but  I feel the CodeGenOptions should be defined in Codegen and used/set in Frontend rather than having Codegen depends on Frontend things I think.
This can be refactored independently and is not a huge deal for a header use without adding a library linking dependency.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 01:18:56 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 03 Jul 2025 01:18:56 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][OpenMP] Allocate `reduction`
 init temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <68663cf0.170a0220.1f4c92.43d7@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/146667

>From 839c85fc0d2040f7a75c437d68b0fc91136b7ab9 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 05:31:38 -0500
Subject: [PATCH 1/2] [flang][OpenMP] Allocate `reduction` init temps on the
 stack for GPUs

Temps needed for the reduction init regions are now allocate on the heap
all the time. However, this is performance killer for GPUs since malloc
calls are prohibitively expensive. Therefore, we should do these
allocations on the stack for GPU reductions.
---
 .../Lower/Support/PrivateReductionUtils.cpp   |  47 +++--
 .../Lower/OpenMP/parallel-reduction-array.f90 | 166 ++++++++++--------
 2 files changed, 119 insertions(+), 94 deletions(-)

diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index e878041d37c03..c3a5b6101ce00 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -502,22 +502,37 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
 
   // Allocating on the heap in case the whole reduction/privatization is nested
   // inside of a loop
-  auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
-  // if needsDealloc isn't statically false, add cleanup region. Always
-  // do this for allocatable boxes because they might have been re-allocated
-  // in the body of the loop/parallel region
-
-  std::optional<int64_t> cstNeedsDealloc = fir::getIntIfConstant(needsDealloc);
-  assert(cstNeedsDealloc.has_value() &&
-         "createTempFromMold decides this statically");
-  if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
-                        isDoConcurrent);
-  } else {
-    assert(!isAllocatableOrPointer &&
-           "Pointer-like arrays must be heap allocated");
-  }
+  auto temp = [&]() {
+    bool shouldAllocateOnStack = false;
+
+    // On the GPU, always allocate on the stack since heap allocatins are very
+    // expensive.
+    if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
+            *builder.getModule()))
+      shouldAllocateOnStack = offloadMod.getIsGPU();
+
+    if (shouldAllocateOnStack)
+      return createStackTempFromMold(loc, builder, source);
+
+    auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
+    // if needsDealloc isn't statically false, add cleanup region. Always
+    // do this for allocatable boxes because they might have been re-allocated
+    // in the body of the loop/parallel region
+
+    std::optional<int64_t> cstNeedsDealloc =
+        fir::getIntIfConstant(needsDealloc);
+    assert(cstNeedsDealloc.has_value() &&
+           "createTempFromMold decides this statically");
+    if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
+                          isDoConcurrent);
+    } else {
+      assert(!isAllocatableOrPointer &&
+             "Pointer-like arrays must be heap allocated");
+    }
+    return temp;
+  }();
 
   // Put the temporary inside of a box:
   // hlfir::genVariableBox doesn't handle non-default lower bounds
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 8e3de498f59c1..4f889d9a4e77f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -1,5 +1,8 @@
-! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+
+! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
 
 program reduce
 integer, dimension(3) :: i = 0
@@ -13,81 +16,88 @@ program reduce
 print *,i
 end program
 
-! CHECK-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
-! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK-LABEL:   } init {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
-! CHECK:           %[[TRUE:.*]]  = arith.constant true
+! CPU-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
+! CPU:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU-LABEL:   } init {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = arith.constant 0 : i32
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_4:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CPU:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
+! CPU:           %[[TRUE:.*]]  = arith.constant true
 !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
-! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         } combiner {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[C1_0:.*]] = arith.constant 1 : index
-! CHECK:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
-! CHECK:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
-! CHECK:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
-! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
-! CHECK:           }
-! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         }  cleanup {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
-! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
-! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
-! CHECK:           fir.if %[[VAL_5]] {
-! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
-! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
-! CHECK:           }
-! CHECK:           omp.yield
-! CHECK:         }
+! CPU:           %[[C0:.*]] = arith.constant 0 : index
+! CPU:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
+! CPU:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         } combiner {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[C1:.*]] = arith.constant 1 : index
+! CPU:           %[[C3:.*]] = arith.constant 3 : index
+! CPU:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[C1_0:.*]] = arith.constant 1 : index
+! CPU:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
+! CPU:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+! CPU:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
+! CPU:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
+! CPU:           }
+! CPU:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         }  cleanup {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
+! CPU:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CPU:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CPU:           fir.if %[[VAL_5]] {
+! CPU:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
+! CPU:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
+! CPU:           }
+! CPU:           omp.yield
+! CPU:         }
+
+! CPU-LABEL:   func.func @_QQmain()
+! CPU:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_1:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CPU:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
+! CPU:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:             %[[VAL_8:.*]] = arith.constant 1 : i32
+! CPU:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_10:.*]] = arith.constant 1 : index
+! CPU:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = arith.constant 2 : i32
+! CPU:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_14:.*]] = arith.constant 2 : index
+! CPU:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_16:.*]] = arith.constant 3 : i32
+! CPU:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_18:.*]] = arith.constant 3 : index
+! CPU:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
+! CPU:             omp.terminator
+! CPU:           }
 
-! CHECK-LABEL:   func.func @_QQmain()
-! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
-! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
-! CHECK:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : index
-! CHECK:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = arith.constant 2 : i32
-! CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_14:.*]] = arith.constant 2 : index
-! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_16:.*]] = arith.constant 3 : i32
-! CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_18:.*]] = arith.constant 3 : index
-! CHECK:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
-! CHECK:             omp.terminator
-! CHECK:           }
+! GPU: omp.declare_reduction {{.*}} alloc {
+! GPU: } init {
+! GPU-NOT: fir.allocmem {{.*}} {bindc_name = ".tmp", {{.*}}}
+! GPU:     fir.alloca {{.*}} {bindc_name = ".tmp"}
+! GPU: } combiner {
+! GPU: }

>From 9fe0e3f917eba67b694618aafe4bd19da6079617 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 23:29:23 -0500
Subject: [PATCH 2/2] add todo

---
 .../LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b07243c5f843..a55239ba1f28f 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -1291,6 +1291,11 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
     mapInitializationArgs(op, moduleTranslation, reductionDecls,
                           reductionVariableMap, i);
 
+    // TODO In some cases (specially on the GPU), the init regions may
+    // contains stack alloctaions. If the region is inlined in a loop, this is
+    // problematic. Instead of just inlining the region, handle allocations by
+    // hoisting fixed length allocations to the function entry and using
+    // stacksave and restore for variable length ones.
     if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
                                        "omp.reduction.neutral", builder,
                                        moduleTranslation, &phis)))


From flang-commits at lists.llvm.org  Thu Jul  3 01:48:58 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 01:48:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] fix skip-external-rtti-definition
	for ppc (PR #146826)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146826@github.com>
Message-ID: <686643fa.170a0220.288acb.7e34@mx.google.com>


jeanPerier wrote:

Note: CI failure is unrelated. `Semantics/windows.f90` is failing and this patch which is only touching an integration test has nothing to do with it.

https://github.com/llvm/llvm-project/pull/146826

From flang-commits at lists.llvm.org  Thu Jul  3 02:09:38 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 02:09:38 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Canonicalize divdc3 calls into
 arithmetic-based complex division (PR #146017)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146017@github.com>
Message-ID: <686648d2.170a0220.11a0cf.7af7@mx.google.com>


Hanwen-ece wrote:

> s-watanabe314

Dear Shunsuke,

Thank you for your thoughtful response and for sharing your approach to complex division optimization. We encountered similar performance issues while analyzing SPEC2017 workloads, particularly in the handling of complex arithmetic, and were motivated to propose a solution grounded in the current design conventions of LLVM Flang.

While our implementation aligns closely with existing patterns in the Flang complex runtime, we’ve found that your formulation provides a more principled and generalizable abstraction. It is both elegant and insightful, and indeed offers a more flexible foundation for future extensions in this area. In our benchmarks, we observed that the performance gains achieved by your approach are comparable to those of our implementation.

We have carefully studied your proposal and found it to be genuinely inspiring. Your contribution exemplifies the clarity and depth characteristic of the LLVM community — something we hold in high regard.

We look forward to learning more from your work and to further engaging with the community on this topic. Enjoy the rest of your day!

Warmest regards,

Hanwen (**@OpenXiangShan**)



https://github.com/llvm/llvm-project/pull/146017

From flang-commits at lists.llvm.org  Thu Jul  3 02:11:58 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 02:11:58 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <6866495e.170a0220.22acaf.47be@mx.google.com>


Hanwen-ece wrote:

Dear Shunsuke,

Thank you for your thoughtful response and for sharing your approach to complex division optimization. We encountered similar performance issues while analyzing SPEC2017 workloads, particularly in the handling of complex arithmetic, and were motivated to propose a solution grounded in the current design conventions of LLVM Flang.

While our implementation aligns closely with existing patterns in the Flang complex runtime, we’ve found that your formulation provides a more principled and generalizable abstraction. It is both elegant and insightful, and indeed offers a more flexible foundation for future extensions in this area. In our benchmarks, we observed that the performance gains achieved by your approach are comparable to those of our implementation.

We have carefully studied your proposal and found it to be genuinely inspiring. Your contribution exemplifies the clarity and depth characteristic of the LLVM community — something we hold in high regard.

We look forward to learning more from your work and to further engaging with the community on this topic. Enjoy the rest of your day!

Warmest regards,

Hanwen (https://github.com/OpenXiangShan)

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 02:31:00 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Thu, 03 Jul 2025 02:31:00 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68664dd4.170a0220.e6823.3fa7@mx.google.com>


https://github.com/kiranchandramohan approved this pull request.

LG. Thanks for all the work and experiments here.

Couple of nit/trivial comments.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 02:31:00 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Thu, 03 Jul 2025 02:31:00 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68664dd4.170a0220.36d396.4c13@mx.google.com>


================
@@ -1066,8 +1066,16 @@ struct BinaryOp<Fortran::evaluate::Divide<
     mlir::Type ty = Fortran::lower::getFIRType(
         builder.getContext(), Fortran::common::TypeCategory::Complex, KIND,
         /*params=*/{});
-    return hlfir::EntityWithAttributes{
-        fir::genDivC(builder, loc, ty, lhs, rhs)};
+
+    // TODO: Ideally, complex number division operations should always be
+    // lowered to MLIR. However, converting them to the runtime via MLIR causes
+    // ABI issues.
+    if (builder.getComplexDivisionToRuntimeFlag())
+      return hlfir::EntityWithAttributes{
+          fir::genDivC(builder, loc, ty, lhs, rhs)};
+    else
+      return hlfir::EntityWithAttributes{
+          builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
----------------
kiranchandramohan wrote:

Nit: braces

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 02:31:00 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Thu, 03 Jul 2025 02:31:00 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68664dd4.170a0220.144993.4bb7@mx.google.com>


================
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
----------------
kiranchandramohan wrote:

Is help text available for these options? If not could you add?

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 02:31:01 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Thu, 03 Jul 2025 02:31:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68664dd5.630a0220.236d1d.496f@mx.google.com>


================
@@ -0,0 +1,35 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_quad
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f128>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEa"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEb"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEc"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f128>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f128>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_RET:.*]] = fir.call @__divtc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f128, f128, f128, f128) -> complex<f128>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f128>, !fir.ref<complex<f128>>
+! CHECK: return
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
----------------
kiranchandramohan wrote:

Nit: end of line/newline

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 02:31:01 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Thu, 03 Jul 2025 02:31:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68664dd5.050a0220.3ef44.aebe@mx.google.com>


================
@@ -595,6 +595,30 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
+static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  default:
+    return "";
+  }
+}
+
+static std::string
+RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
+
----------------
kiranchandramohan wrote:

Can we share this code with similar code in Clang.cpp by moving to CommonArgs.cpp or a suitable place?

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 02:31:01 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Thu, 03 Jul 2025 02:31:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68664dd5.170a0220.17b81a.4a69@mx.google.com>


https://github.com/kiranchandramohan edited https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 02:35:24 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Thu, 03 Jul 2025 02:35:24 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146532@github.com>
Message-ID: <68664edc.170a0220.23479a.5642@mx.google.com>


================
@@ -6891,23 +6891,19 @@ static void FixupDebugInfoForOutlinedFunction(
   if (!NewSP)
     return;
 
-  DenseMap<const MDNode *, MDNode *> Cache;
   SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
 
   auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
-    auto NewSP = Func->getSubprogram();
     DILocalVariable *&NewVar = RemappedVariables[OldVar];
     // Only use cached variable if the arg number matches. This is important
     // so that DIVariable created for privatized variables are not discarded.
     if (NewVar && (arg == NewVar->getArg()))
       return NewVar;
 
-    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
     NewVar = llvm::DILocalVariable::get(
-        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
-        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
-        OldVar->getAlignInBits(), OldVar->getAnnotations());
+        Builder.getContext(), OldVar->getScope(), OldVar->getName(),
----------------
abidh wrote:

Using `OldVar->getScope()` is the right thing here.

https://github.com/llvm/llvm-project/pull/146532

From flang-commits at lists.llvm.org  Thu Jul  3 02:38:03 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Thu, 03 Jul 2025 02:38:03 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146532@github.com>
Message-ID: <68664f7b.170a0220.31ff0f.49db@mx.google.com>


================
@@ -5324,9 +5324,27 @@ static LogicalResult
 convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto targetOp = cast<omp::TargetOp>(opInst);
+  // The current debug location already has the DISubprogram for the outlined
+  // function that will be created for the target op. We save it here so that
+  // we can set it on the outlined function.
+  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
   if (failed(checkImplementationStatus(opInst)))
     return failure();
 
+  // During the handling of target op, we will generate instructions in the
+  // parent function like call to the oulined function or branch to a new
+  // BasicBlock. We set the debug location here to parent function so that those
+  // get the correct debug locations. For outlined functions, the normal MLIR op
+  // conversion will automatically pick the correct location.
+  llvm::BasicBlock *parentBB = builder.GetInsertBlock();
+  assert(parentBB && "No insert block is set for the builder");
+  llvm::Function *parentLLVMFn = parentBB->getParent();
+  assert(parentLLVMFn && "Parent Function must be valid");
+  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
+    builder.SetCurrentDebugLocation(llvm::DILocation::get(
----------------
abidh wrote:

The callback which generates the body of the outlined function uses the correct location.

https://github.com/llvm/llvm-project/pull/146532

From flang-commits at lists.llvm.org  Thu Jul  3 02:38:35 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Thu, 03 Jul 2025 02:38:35 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146532@github.com>
Message-ID: <68664f9b.170a0220.16c320.42e5@mx.google.com>


https://github.com/abidh closed https://github.com/llvm/llvm-project/pull/146532

From flang-commits at lists.llvm.org  Thu Jul  3 02:38:34 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 02:38:34 -0700 (PDT)
Subject: [flang-commits] [flang] d56c06e - [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (#146532)
Message-ID: <68664f9a.170a0220.1396a4.4550@mx.google.com>


Author: Abid Qadeer
Date: 2025-07-03T10:38:28+01:00
New Revision: d56c06e6c9f0cf03d75fd755c099a6844ecf38e6

URL: https://github.com/llvm/llvm-project/commit/d56c06e6c9f0cf03d75fd755c099a6844ecf38e6
DIFF: https://github.com/llvm/llvm-project/commit/d56c06e6c9f0cf03d75fd755c099a6844ecf38e6.diff

LOG: [flang][debug] Generate DISubprogramAttr for omp::TargetOp. (#146532)

This is combination of https://github.com/llvm/llvm-project/pull/138149
and https://github.com/llvm/llvm-project/pull/138039 which were opened
separately for ease of reviewing. Only other change is adjustments in 2
tests which have gone in since.

There are `DeclareOp` present for the variables mapped into target
region. That allow us to generate debug information for them. But the
`TargetOp` is still part of parent function and those variables get the
parent function's `DISubprogram` as a scope.
    
In `OMPIRBuilder`, a new function is created for the `TargetOp`. We also
create a new `DISubprogram` for it. All the variables that were in the
target region now have to be updated to have the correct scope. This
after the fact updating of
debug information becomes very difficult in certain cases. Take the
example of variable arrays. The type of those arrays depend on the
artificial `DILocalVariable`(s) which hold the size(s) of the array.
This new function will now require that we generate the new variable and
and new types. Similar issue exist for character type variables too.
    
To avoid this after the fact updating, this PR generates a
`DISubprogramAttr` for the `TargetOp` while generating the debug info in
`flang`. Then we don't need to generate a `DISubprogram` in
`OMPIRBuilder`. This change is made a bit more complicated by the the
fact that in new scheme, the debug location already points to the new
`DISubprogram` by the time it reaches `convertOmpTarget`. But we need
some code generation in the parent function so we have to carefully
manage the debug locations.
    
This fixes issue `#134991`.

Added: 
    flang/test/Transforms/debug-omp-target-op-1.fir
    flang/test/Transforms/debug-omp-target-op-2.fir
    mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir

Modified: 
    flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
    llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
    mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
    mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
    mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
    mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
    mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
    mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
    mlir/test/Target/LLVMIR/omptarget-debug.mlir
    mlir/test/Target/LLVMIR/omptarget-debug2.mlir
    mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 8fa2f38818c02..6eb914e67fd54 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -35,6 +35,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -104,6 +105,37 @@ bool debugInfoIsAlreadySet(mlir::Location loc) {
   return false;
 }
 
+// Generates the name for the artificial DISubprogram that we are going to
+// generate for omp::TargetOp. Its logic is borrowed from
+// getTargetEntryUniqueInfo and
+// TargetRegionEntryInfo::getTargetRegionEntryFnName to generate the same name.
+// But even if there was a slight mismatch, it is not a problem because this
+// name is artificial and not important to debug experience.
+mlir::StringAttr getTargetFunctionName(mlir::MLIRContext *context,
+                                       mlir::Location Loc,
+                                       llvm::StringRef parentName) {
+  auto fileLoc = Loc->findInstanceOf<mlir::FileLineColLoc>();
+
+  assert(fileLoc && "No file found from location");
+  llvm::StringRef fileName = fileLoc.getFilename().getValue();
+
+  llvm::sys::fs::UniqueID id;
+  uint64_t line = fileLoc.getLine();
+  size_t fileId;
+  size_t deviceId;
+  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {
+    fileId = llvm::hash_value(fileName.str());
+    deviceId = 0xdeadf17e;
+  } else {
+    fileId = id.getFile();
+    deviceId = id.getDevice();
+  }
+  return mlir::StringAttr::get(
+      context,
+      std::string(llvm::formatv("__omp_offloading_{0:x-}_{1:x-}_{2}_l{3}",
+                                deviceId, fileId, parentName, line)));
+}
+
 } // namespace
 
 bool AddDebugInfoPass::createCommonBlockGlobal(
@@ -446,6 +478,79 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
                                   line - 1, false);
   }
 
+  auto addTargetOpDISP = [&](bool lineTableOnly,
+                             llvm::ArrayRef<mlir::LLVM::DINodeAttr> entities) {
+    // When we process the DeclareOp inside the OpenMP target region, all the
+    // variables get the DISubprogram of the parent function of the target op as
+    // the scope. In the codegen (to llvm ir), OpenMP target op results in the
+    // creation of a separate function. As the variables in the debug info have
+    // the DISubprogram of the parent function as the scope, the variables
+    // need to be updated at codegen time to avoid verification failures.
+
+    // This updating after the fact becomes more and more 
diff icult when types
+    // are dependent on local variables like in the case of variable size arrays
+    // or string. We not only have to generate new variables but also new types.
+    // We can avoid this problem by generating a DISubprogramAttr here for the
+    // target op and make sure that all the variables inside the target region
+    // get the correct scope in the first place.
+    funcOp.walk([&](mlir::omp::TargetOp targetOp) {
+      unsigned line = getLineFromLoc(targetOp.getLoc());
+      mlir::StringAttr name =
+          getTargetFunctionName(context, targetOp.getLoc(), funcOp.getName());
+      mlir::LLVM::DISubprogramFlags flags =
+          mlir::LLVM::DISubprogramFlags::Definition |
+          mlir::LLVM::DISubprogramFlags::LocalToUnit;
+      if (isOptimized)
+        flags = flags | mlir::LLVM::DISubprogramFlags::Optimized;
+
+      mlir::DistinctAttr id =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
+      types.push_back(mlir::LLVM::DINullTypeAttr::get(context));
+      for (auto arg : targetOp.getRegion().getArguments()) {
+        auto tyAttr = typeGen.convertType(fir::unwrapRefType(arg.getType()),
+                                          fileAttr, cuAttr, /*declOp=*/nullptr);
+        types.push_back(tyAttr);
+      }
+      CC = llvm::dwarf::getCallingConvention("DW_CC_normal");
+      mlir::LLVM::DISubroutineTypeAttr spTy =
+          mlir::LLVM::DISubroutineTypeAttr::get(context, CC, types);
+      if (lineTableOnly) {
+        auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+            context, id, compilationUnit, Scope, name, name, funcFileAttr, line,
+            line, flags, spTy, /*retainedNodes=*/{}, /*annotations=*/{});
+        targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+        return;
+      }
+      mlir::DistinctAttr recId =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/true, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, /*retainedNodes=*/{},
+          /*annotations=*/{});
+
+      // Make sure that information about the imported modules is copied in the
+      // new function.
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> opEntities;
+      for (mlir::LLVM::DINodeAttr N : entities) {
+        if (auto entity = mlir::dyn_cast<mlir::LLVM::DIImportedEntityAttr>(N)) {
+          auto importedEntity = mlir::LLVM::DIImportedEntityAttr::get(
+              context, llvm::dwarf::DW_TAG_imported_module, spAttr,
+              entity.getEntity(), fileAttr, /*line=*/1, /*name=*/nullptr,
+              /*elements*/ {});
+          opEntities.push_back(importedEntity);
+        }
+      }
+
+      id = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/false, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, opEntities,
+          /*annotations=*/{});
+      targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+    });
+  };
+
   // Don't process variables if user asked for line tables only.
   if (debugLevel == mlir::LLVM::DIEmissionKind::LineTablesOnly) {
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
@@ -453,6 +558,7 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
         line, line, subprogramFlags, subTypeAttr, /*retainedNodes=*/{},
         /*annotations=*/{});
     funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+    addTargetOpDISP(/*lineTableOnly=*/true, /*entities=*/{});
     return;
   }
 
@@ -510,9 +616,18 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
       funcName, fullName, funcFileAttr, line, line, subprogramFlags,
       subTypeAttr, entities, /*annotations=*/{});
   funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+  addTargetOpDISP(/*lineTableOnly=*/false, entities);
 
   funcOp.walk([&](fir::cg::XDeclareOp declOp) {
-    handleDeclareOp(declOp, fileAttr, spAttr, typeGen, symbolTable);
+    mlir::LLVM::DISubprogramAttr spTy = spAttr;
+    if (auto tOp = declOp->getParentOfType<mlir::omp::TargetOp>()) {
+      if (auto fusedLoc = llvm::dyn_cast<mlir::FusedLoc>(tOp.getLoc())) {
+        if (auto sp = llvm::dyn_cast<mlir::LLVM::DISubprogramAttr>(
+                fusedLoc.getMetadata()))
+          spTy = sp;
+      }
+    }
+    handleDeclareOp(declOp, fileAttr, spTy, typeGen, symbolTable);
   });
   // commonBlockMap ensures that we don't create multiple DICommonBlockAttr of
   // the same name in one function. But it is ok (rather required) to create

diff  --git a/flang/test/Transforms/debug-omp-target-op-1.fir b/flang/test/Transforms/debug-omp-target-op-1.fir
new file mode 100644
index 0000000000000..6b895b732c42b
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-1.fir
@@ -0,0 +1,40 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info="debug-level=LineTablesOnly" --mlir-print-debuginfo %s | FileCheck %s --check-prefix=LINETABLE
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %c13_i32 = arith.constant 13 : i32
+    %c12_i32 = arith.constant 12 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c1_i32 = arith.constant 1 : i32
+    %c5_i32 = arith.constant 5 : i32
+    %0 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} loc(#loc1)
+    %1 = fircg.ext_declare %0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc1)
+    %2 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} loc(#loc2)
+    %3 = fircg.ext_declare %2 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc2)
+    %4 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "x"}
+    %5 = omp.map.info var_ptr(%3 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "y"}
+    omp.target map_entries(%4 -> %arg0, %5 -> %arg1 : !fir.ref<i32>, !fir.ref<i32>) {
+      %16 = fircg.ext_declare %arg0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc3)
+      %17 = fircg.ext_declare %arg1 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc4)
+      omp.terminator
+    } loc(#loc5)
+    return
+  }
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// CHECK: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "x"{{.*}}line = 1, type = #[[TY:.*]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "y"{{.*}}line = 3, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "x"{{.*}}line = 7, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "y"{{.*}}line = 8, type = #[[TY]]>
+
+// LINETABLE: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// LINETABLE: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// LINETABLE-NOT: #llvm.di_local_variable

diff  --git a/flang/test/Transforms/debug-omp-target-op-2.fir b/flang/test/Transforms/debug-omp-target-op-2.fir
new file mode 100644
index 0000000000000..15dcf2389b21d
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-2.fir
@@ -0,0 +1,53 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @fn_(%arg0: !fir.ref<!fir.array<?x?xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+    %c1 = arith.constant 1 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.alloca i32
+    %1 = fir.alloca i32
+    %2 = fir.undefined !fir.dscope
+    %3 = fircg.ext_declare %arg1 dummy_scope %2 {uniq_name = "_QFfnEc"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc2)
+    %4 = fircg.ext_declare %arg2 dummy_scope %2 {uniq_name = "_QFfnEd"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc3)
+    %5 = fir.load %3 : !fir.ref<i32>
+    %6 = fir.convert %5 : (i32) -> index
+    %9 = fir.load %4 : !fir.ref<i32>
+    %10 = fir.convert %9 : (i32) -> index
+    %15 = fircg.ext_declare %arg0(%6, %10) dummy_scope %2 {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc4)
+    %16 = fircg.ext_embox %15(%6, %10) : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.box<!fir.array<?x?xi32>>
+    %17:3 = fir.box_dims %16, %c0 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %18 = arith.subi %17#1, %c1 : index
+    %19 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%18 : index) extent(%17#1 : index) stride(%17#2 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %20 = arith.muli %17#2, %17#1 : index
+    %21:3 = fir.box_dims %16, %c1 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %22 = arith.subi %21#1, %c1 : index
+    %23 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%22 : index) extent(%21#1 : index) stride(%20 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %24 = omp.map.info var_ptr(%15 : !fir.ref<!fir.array<?x?xi32>>, i32) map_clauses(tofrom) capture(ByRef) bounds(%19, %23) -> !fir.ref<!fir.array<?x?xi32>> {name = "b"}
+    %25 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    %26 = omp.map.info var_ptr(%0 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    omp.target map_entries(%24 -> %arg3, %25 -> %arg4, %26 -> %arg5 : !fir.ref<!fir.array<?x?xi32>>, !fir.ref<i32>, !fir.ref<i32>) {
+      %27 = fir.load %arg5 : !fir.ref<i32>
+      %28 = fir.load %arg4 : !fir.ref<i32>
+      %29 = fir.convert %27 : (i32) -> index
+      %31 = fir.convert %28 : (i32) -> index
+      %37 = fircg.ext_declare %arg3(%29, %31) {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc5)
+      omp.terminator
+    } loc(#loc6)
+    return
+  } loc(#loc7)
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+#loc6 = loc("test.f90":16:1)
+#loc7 = loc("test.f90":26:1)
+
+
+// Test that variable size arrays inside target regions get their own
+// compiler generated variables for size.
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_fn__l16"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb1"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb2"{{.*}}>

diff  --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 85451b1233f96..db792a3b52d24 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6891,23 +6891,19 @@ static void FixupDebugInfoForOutlinedFunction(
   if (!NewSP)
     return;
 
-  DenseMap<const MDNode *, MDNode *> Cache;
   SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
 
   auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
-    auto NewSP = Func->getSubprogram();
     DILocalVariable *&NewVar = RemappedVariables[OldVar];
     // Only use cached variable if the arg number matches. This is important
     // so that DIVariable created for privatized variables are not discarded.
     if (NewVar && (arg == NewVar->getArg()))
       return NewVar;
 
-    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
     NewVar = llvm::DILocalVariable::get(
-        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
-        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
-        OldVar->getAlignInBits(), OldVar->getAnnotations());
+        Builder.getContext(), OldVar->getScope(), OldVar->getName(),
+        OldVar->getFile(), OldVar->getLine(), OldVar->getType(), arg,
+        OldVar->getFlags(), OldVar->getAlignInBits(), OldVar->getAnnotations());
     return NewVar;
   };
 
@@ -6921,7 +6917,8 @@ static void FixupDebugInfoForOutlinedFunction(
         ArgNo = std::get<1>(Iter->second) + 1;
       }
     }
-    DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
+    if (ArgNo != 0)
+      DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
   };
 
   // The location and scope of variable intrinsics and records still point to
@@ -7000,36 +6997,9 @@ static Expected<Function *> createOutlinedFunction(
 
   // Save insert point.
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  // If there's a DISubprogram associated with current function, then
-  // generate one for the outlined function.
-  if (Function *ParentFunc = BB->getParent()) {
-    if (DISubprogram *SP = ParentFunc->getSubprogram()) {
-      DICompileUnit *CU = SP->getUnit();
-      DIBuilder DB(*M, true, CU);
-      DebugLoc DL = Builder.getCurrentDebugLocation();
-      if (DL) {
-        // TODO: We are using nullopt for arguments at the moment. This will
-        // need to be updated when debug data is being generated for variables.
-        DISubroutineType *Ty =
-            DB.createSubroutineType(DB.getOrCreateTypeArray({}));
-        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
-                                          DISubprogram::SPFlagOptimized |
-                                          DISubprogram::SPFlagLocalToUnit;
-
-        DISubprogram *OutlinedSP = DB.createFunction(
-            CU, FuncName, FuncName, SP->getFile(), DL.getLine(), Ty,
-            DL.getLine(), DINode::DIFlags::FlagArtificial, SPFlags);
-
-        // Attach subprogram to the function.
-        Func->setSubprogram(OutlinedSP);
-        // Update the CurrentDebugLocation in the builder so that right scope
-        // is used for things inside outlined function.
-        Builder.SetCurrentDebugLocation(
-            DILocation::get(Func->getContext(), DL.getLine(), DL.getCol(),
-                            OutlinedSP, DL.getInlinedAt()));
-      }
-    }
-  }
+  // We will generate the entries in the outlined function but the debug
+  // location may still be pointing to the parent function. Reset it now.
+  Builder.SetCurrentDebugLocation(llvm::DebugLoc());
 
   // Generate the region into the function.
   BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);

diff  --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b07243c5f843..883d179580e0c 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5395,9 +5395,27 @@ static LogicalResult
 convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto targetOp = cast<omp::TargetOp>(opInst);
+  // The current debug location already has the DISubprogram for the outlined
+  // function that will be created for the target op. We save it here so that
+  // we can set it on the outlined function.
+  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
   if (failed(checkImplementationStatus(opInst)))
     return failure();
 
+  // During the handling of target op, we will generate instructions in the
+  // parent function like call to the oulined function or branch to a new
+  // BasicBlock. We set the debug location here to parent function so that those
+  // get the correct debug locations. For outlined functions, the normal MLIR op
+  // conversion will automatically pick the correct location.
+  llvm::BasicBlock *parentBB = builder.GetInsertBlock();
+  assert(parentBB && "No insert block is set for the builder");
+  llvm::Function *parentLLVMFn = parentBB->getParent();
+  assert(parentLLVMFn && "Parent Function must be valid");
+  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
+    builder.SetCurrentDebugLocation(llvm::DILocation::get(
+        parentLLVMFn->getContext(), outlinedFnLoc.getLine(),
+        outlinedFnLoc.getCol(), SP, outlinedFnLoc.getInlinedAt()));
+
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   bool isTargetDevice = ompBuilder->Config.isTargetDevice();
   bool isGPU = ompBuilder->Config.isGPU();
@@ -5491,6 +5509,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     assert(llvmParentFn && llvmOutlinedFn &&
            "Both parent and outlined functions must exist at this point");
 
+    if (outlinedFnLoc && llvmParentFn->getSubprogram())
+      llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());
+
     if (auto attr = llvmParentFn->getFnAttribute("target-cpu");
         attr.isStringAttribute())
       llvmOutlinedFn->addFnAttr(attr);

diff  --git a/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
new file mode 100644
index 0000000000000..45e5d2612e2c2
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+module attributes {omp.is_target_device = false} {
+  llvm.func @test() {
+    omp.target {
+      omp.terminator
+    }  loc(#loc4)
+    llvm.return
+  }  loc(#loc3)
+}
+#file = #llvm.di_file<"target.f90" in "">
+#cu = #llvm.di_compile_unit<id = distinct[0]<>,
+ sourceLanguage = DW_LANG_Fortran95, file = #file, isOptimized = false,
+ emissionKind = Full>
+#sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
+#sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
+ name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
+#loc1 = loc("target.f90":1:1)
+#loc2 = loc("target.f90":46:3)
+#loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc2])
+
+// CHECK: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
+

diff  --git a/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
index a755cef98d7c4..c3320382f8d45 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
@@ -41,7 +41,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
         omp.terminator loc(#loc9)
       } loc(#loc9)
       omp.terminator loc(#loc9)
-    } loc(#loc9)
+    } loc(#loc15)
     llvm.return loc(#loc9)
   } loc(#loc14)
   llvm.mlir.global internal @_QFEarray() {addr_space = 0 : i32} : !llvm.array<16384 x i32> {
@@ -59,7 +59,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang", isOptimized = true, emissionKind = LineTablesOnly>
 #di_subroutine_type = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 #di_subprogram = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #di_compile_unit, scope = #di_file, name = "main", file = #di_file, subprogramFlags = "Definition|Optimized|MainSubprogram", type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit, name = "target", file = #di_file, subprogramFlags = "Definition", type = #di_subroutine_type>
 #loc14 = loc(fused<#di_subprogram>[#loc3])
+#loc15 = loc(fused<#di_subprogram1>[#loc9])
 
 
 // CHECK: call void @__kmpc_distribute_static{{.*}}!dbg

diff  --git a/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
index 4495e1559a889..492610251769c 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
@@ -16,7 +16,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       %16 = llvm.load %arg1 : !llvm.ptr -> i32 loc(#loc5)
       llvm.store %16, %arg0 : i32, !llvm.ptr loc(#loc5)
       omp.terminator loc(#loc5)
-    } loc(#loc5)
+    } loc(#loc16)
     llvm.return loc(#loc6)
   } loc(#loc15)
 }
@@ -31,9 +31,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
   compileUnit = #di_compile_unit, scope = #di_file, name = "main",
   file = #di_file, subprogramFlags = "Definition|MainSubprogram",
   type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit,
+  name = "target", file = #di_file, subprogramFlags = "Definition",
+  type = #di_subroutine_type>
 #loc1 = loc("test.f90":3:18)
 #loc2 = loc("test.f90":7:7)
 #loc3 = loc("test.f90":9:18)
 #loc5 = loc("test.f90":11:7)
 #loc6 = loc("test.f90":12:7)
 #loc15 = loc(fused<#di_subprogram>[#loc2])
+#loc16 = loc(fused<#di_subprogram1>[#loc5])

diff  --git a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
index eaa88d9dd6053..3bd724f42e8ce 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
@@ -20,7 +20,7 @@ module attributes {omp.is_target_device = false} {
     ^bb3:  // pred: ^bb1
       llvm.store %13, %arg1 : i32, !llvm.ptr
       omp.terminator
-    }
+    } loc(#loc3)
     llvm.return
   } loc(#loc2)
 }
@@ -34,7 +34,10 @@ module attributes {omp.is_target_device = false} {
  types = #di_null_type>
 #sp = #llvm.di_subprogram<compileUnit = #cu, name = "main", file=#file,
  subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<compileUnit = #cu, name = "target", file=#file,
+ subprogramFlags = "Definition", type = #sp_ty>
 
 #loc1 = loc("test.f90":6:7)
 #loc2 = loc(fused<#sp>[#loc1])
+#loc3 = loc(fused<#sp1>[#loc1])
 

diff  --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
index ea92589bbd031..8f42995af23a8 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
@@ -19,11 +19,13 @@
 #g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
 #sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
   name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
-#var_arr = #llvm.di_local_variable<scope = #sp,
+#sp1 = #llvm.di_subprogram<id = distinct[3]<>, compileUnit = #cu, scope = #file,
+  name = "target", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp1,
   name = "arr", file = #file, line = 4, type = #array_ty>
-#var_i = #llvm.di_local_variable<scope = #sp,
+#var_i = #llvm.di_local_variable<scope = #sp1,
   name = "i", file = #file, line = 13, type = #int_ty>
-#var_x = #llvm.di_local_variable<scope = #sp,
+#var_x = #llvm.di_local_variable<scope = #sp1,
  name = "x", file = #file, line = 12, type = #real_ty>
 
 module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui32>>, llvm.target_triple = "amdgcn-amd-amdhsa", omp.is_target_device = true} {
@@ -47,7 +49,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
       llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
       omp.terminator
-    }
+    } loc(#loc5)
     llvm.return
   } loc(#loc3)
   llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
@@ -57,8 +59,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #loc2 = loc("target.f90":11:7)
 #loc3 = loc(fused<#sp>[#loc2])
 #loc4 = loc(fused<#g_var>[#loc1])
+#loc5 = loc(fused<#sp1>[#loc2])
 
-// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "target"{{.*}})
 // CHECK: !DILocalVariable(name: "dyn_ptr", arg: 1, scope: ![[SP]]{{.*}}flags: DIFlagArtificial)
 // CHECK: !DILocalVariable(name: "x", arg: 2, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "arr", arg: 3, scope: ![[SP]]{{.*}})

diff  --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
index 22db86fd85e2c..11a07dfd9a180 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
@@ -19,11 +19,13 @@
 #g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
 #sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
   name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
-#var_arr = #llvm.di_local_variable<scope = #sp,
+#sp1 = #llvm.di_subprogram<id = distinct[3]<>, compileUnit = #cu, scope = #file,
+  name = "target", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp1,
   name = "arr", file = #file, line = 4, type = #array_ty>
-#var_i = #llvm.di_local_variable<scope = #sp,
+#var_i = #llvm.di_local_variable<scope = #sp1,
   name = "i", file = #file, line = 13, type = #int_ty>
-#var_x = #llvm.di_local_variable<scope = #sp,
+#var_x = #llvm.di_local_variable<scope = #sp1,
  name = "x", file = #file, line = 12, type = #real_ty>
 
 module attributes {omp.is_target_device = false} {
@@ -45,7 +47,7 @@ module attributes {omp.is_target_device = false} {
       llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
       llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
       omp.terminator
-    }
+    }  loc(#loc5)
     llvm.return
   } loc(#loc3)
   llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
@@ -55,8 +57,9 @@ module attributes {omp.is_target_device = false} {
 #loc2 = loc("target.f90":11:7)
 #loc3 = loc(fused<#sp>[#loc2])
 #loc4 = loc(fused<#g_var>[#loc1])
+#loc5 = loc(fused<#sp1>[#loc2])
 
-// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "target"{{.*}})
 // CHECK: !DILocalVariable(name: "x", arg: 1, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "arr", arg: 2, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "i", arg: 3, scope: ![[SP]]{{.*}})

diff  --git a/mlir/test/Target/LLVMIR/omptarget-debug.mlir b/mlir/test/Target/LLVMIR/omptarget-debug.mlir
index 9c8344d69dc74..ab687f198b9b4 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug.mlir
@@ -10,7 +10,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       %13 = llvm.mlir.constant(1 : i32) : i32
       llvm.store %13, %arg0 : i32, !llvm.ptr loc(#loc2)
       omp.terminator
-    }
+    } loc(#loc4)
     llvm.return
   } loc(#loc3)
 }
@@ -21,9 +21,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
 #sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
  name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
 #loc1 = loc("target.f90":1:1)
 #loc2 = loc("target.f90":46:3)
 #loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc1])
 
-// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_{{.*}}"{{.*}})
+// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
 // CHECK-DAG: !DILocation(line: 46, column: 3, scope: ![[SP]])

diff  --git a/mlir/test/Target/LLVMIR/omptarget-debug2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
index 78dc6e18a40a7..6cf75af38f916 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
@@ -11,7 +11,7 @@ module attributes {omp.is_target_device = false} {
       %13 = llvm.mlir.constant(1 : i32) : i32
       llvm.store %13, %arg0 : i32, !llvm.ptr loc(#loc2)
       omp.terminator
-    }
+    }  loc(#loc4)
     llvm.return
   } loc(#loc3)
 }
@@ -22,9 +22,13 @@ module attributes {omp.is_target_device = false} {
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
 #sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
  name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
 #loc1 = loc("target.f90":1:1)
 #loc2 = loc("target.f90":46:3)
 #loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc1])
 
-// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_{{.*}}"{{.*}})
+// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
 // CHECK-DAG: !DILocation(line: 46, column: 3, scope: ![[SP]])

diff  --git a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
index 3c45f1f1c76fb..b18338ea35cc3 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
@@ -6,8 +6,10 @@
 #cu = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang", isOptimized = false, emissionKind = Full>
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 #sp = #llvm.di_subprogram<compileUnit = #cu, scope = #di_file, name = "test", file = #di_file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<compileUnit = #cu, scope = #di_file, name = "kernel", file = #di_file, subprogramFlags = "Definition", type = #sp_ty>
 #int_ty = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
 #var_x = #llvm.di_local_variable<scope = #sp, name = "x", file = #di_file, type = #int_ty>
+#var_x1 = #llvm.di_local_variable<scope = #sp1, name = "x", file = #di_file, type = #int_ty>
 module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, f128 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, f80 = dense<128> : vector<2xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr = dense<64> : vector<4xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, i1 = dense<8> : vector<2xi64>, i8 = dense<8> : vector<2xi64>, i16 = dense<16> : vector<2xi64>, "dlti.endianness" = "little", "dlti.stack_alignment" = 128 : i64, "dlti.mangling_mode" = "e">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", fir.target_cpu = "x86-64", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 21.0.0 (/home/haqadeer/work/src/aomp-llvm-project/flang 793f9220ab32f92fc3b253efec2e332c18090e53)", llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_gpu = false, omp.is_target_device = false, omp.requires = #omp<clause_requires none>, omp.target_triples = ["amdgcn-amd-amdhsa"], omp.version = #omp.version<version = 52>} {
   llvm.func @_QQmain() attributes {fir.bindc_name = "test", frame_pointer = #llvm.framePointerKind<all>, target_cpu = "x86-64"} {
     %0 = llvm.mlir.constant(1 : i64) : i64
@@ -16,7 +18,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>,
     %5 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "x"}
     omp.target map_entries(%5 -> %arg0 : !llvm.ptr) {
       %6 = llvm.mlir.constant(1 : i32) : i32
-      llvm.intr.dbg.declare #var_x = %arg0 : !llvm.ptr loc(#loc2)
+      llvm.intr.dbg.declare #var_x1 = %arg0 : !llvm.ptr loc(#loc3)
       omp.parallel {
         %7 = llvm.load %arg0 : !llvm.ptr -> i32
         %8 = llvm.add %7, %6 : i32
@@ -24,12 +26,14 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>,
         omp.terminator
       }
       omp.terminator
-    }
+    } loc(#loc4)
     llvm.return
   } loc(#loc10)
 }
 #loc1 = loc("target.f90":1:7)
 #loc2 = loc("target.f90":3:18)
+#loc3 = loc("target.f90":6:18)
+#loc4 = loc(fused<#sp1>[#loc3])
 #loc10 = loc(fused<#sp>[#loc1])
 
 


        

From flang-commits at lists.llvm.org  Thu Jul  3 02:44:38 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Thu, 03 Jul 2025 02:44:38 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146532@github.com>
Message-ID: <68665106.a70a0220.612e9.c4b3@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `lldb-x86_64-debian` running on `lldb-x86_64-debian` while building `flang,llvm,mlir` at step 6 "test".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/162/builds/25924

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (test) failure: build (failure)
...
PASS: lldb-api :: tools/lldb-dap/source/TestDAP_source.py (294 of 3071)
PASS: lldb-api :: functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/TestDataFormatterStdTuple.py (295 of 3071)
PASS: lldb-api :: functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py (296 of 3071)
PASS: lldb-api :: lang/c/vla/TestVLA.py (297 of 3071)
PASS: lldb-api :: tools/lldb-dap/save-core/TestDAP_save_core.py (298 of 3071)
PASS: lldb-api :: lang/c/conflicting-symbol/TestConflictingSymbol.py (299 of 3071)
PASS: lldb-api :: functionalities/gdb_remote_client/TestTargetXMLArch.py (300 of 3071)
PASS: lldb-api :: commands/expression/call-function/TestCallUserDefinedFunction.py (301 of 3071)
PASS: lldb-api :: functionalities/thread/concurrent_events/TestConcurrentManyCrash.py (302 of 3071)
PASS: lldb-api :: tools/lldb-server/signal-filtering/TestGdbRemote_QPassSignals.py (303 of 3071)
FAIL: lldb-api :: tools/lldb-dap/output/TestDAP_output.py (304 of 3071)
******************** TEST 'lldb-api :: tools/lldb-dap/output/TestDAP_output.py' FAILED ********************
Script:
--
/usr/bin/python3 /home/worker/2.0.1/lldb-x86_64-debian/llvm-project/lldb/test/API/dotest.py -u CXXFLAGS -u CFLAGS --env LLVM_LIBS_DIR=/home/worker/2.0.1/lldb-x86_64-debian/build/./lib --env LLVM_INCLUDE_DIR=/home/worker/2.0.1/lldb-x86_64-debian/build/include --env LLVM_TOOLS_DIR=/home/worker/2.0.1/lldb-x86_64-debian/build/./bin --arch x86_64 --build-dir /home/worker/2.0.1/lldb-x86_64-debian/build/lldb-test-build.noindex --lldb-module-cache-dir /home/worker/2.0.1/lldb-x86_64-debian/build/lldb-test-build.noindex/module-cache-lldb/lldb-api --clang-module-cache-dir /home/worker/2.0.1/lldb-x86_64-debian/build/lldb-test-build.noindex/module-cache-clang/lldb-api --executable /home/worker/2.0.1/lldb-x86_64-debian/build/./bin/lldb --compiler /home/worker/2.0.1/lldb-x86_64-debian/build/./bin/clang --dsymutil /home/worker/2.0.1/lldb-x86_64-debian/build/./bin/dsymutil --make /usr/bin/gmake --llvm-tools-dir /home/worker/2.0.1/lldb-x86_64-debian/build/./bin --lldb-obj-root /home/worker/2.0.1/lldb-x86_64-debian/build/tools/lldb --lldb-libs-dir /home/worker/2.0.1/lldb-x86_64-debian/build/./lib --cmake-build-type Release -t /home/worker/2.0.1/lldb-x86_64-debian/llvm-project/lldb/test/API/tools/lldb-dap/output -p TestDAP_output.py
--
Exit Code: 1

Command Output (stdout):
--
lldb version 21.0.0git (https://github.com/llvm/llvm-project.git revision d56c06e6c9f0cf03d75fd755c099a6844ecf38e6)
  clang revision d56c06e6c9f0cf03d75fd755c099a6844ecf38e6
  llvm revision d56c06e6c9f0cf03d75fd755c099a6844ecf38e6
Skipping the following test categories: ['libc++', 'dsym', 'gmodules', 'debugserver', 'objc']

--
Command Output (stderr):
--
Change dir to: /home/worker/2.0.1/lldb-x86_64-debian/llvm-project/lldb/test/API/tools/lldb-dap/output
runCmd: settings clear --all

output: 

runCmd: settings set symbols.enable-external-lookup false

output: 

runCmd: settings set target.inherit-tcc true

output: 

runCmd: settings set target.disable-aslr false

output: 

runCmd: settings set target.detach-on-error false

output: 


```

</details>

https://github.com/llvm/llvm-project/pull/146532

From flang-commits at lists.llvm.org  Thu Jul  3 02:53:50 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Thu, 03 Jul 2025 02:53:50 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][debug] Generate
 DISubprogramAttr for omp::TargetOp. (PR #146532)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146532@github.com>
Message-ID: <6866532e.050a0220.30a109.0496@mx.google.com>


abidh wrote:

> LLVM Buildbot has detected a new failure on builder `lldb-x86_64-debian` running on `lldb-x86_64-debian` while building `flang,llvm,mlir` at step 6 "test".
> 
> Full details are available at: https://lab.llvm.org/buildbot/#/builders/162/builds/25924
> 
> Here is the relevant piece of the build log for the reference

The Buildbot is green again. May be the failure was unrelated.

https://github.com/llvm/llvm-project/pull/146532

From flang-commits at lists.llvm.org  Thu Jul  3 03:03:56 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 03 Jul 2025 03:03:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] fix skip-external-rtti-definition
	for ppc (PR #146826)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146826@github.com>
Message-ID: <6866558c.050a0220.157535.da81@mx.google.com>


https://github.com/tblah approved this pull request.

Looks sensible to me. Normally I would say wait for @kkwli, but it seems worth fixing the buildbot failure asap

https://github.com/llvm/llvm-project/pull/146826

From flang-commits at lists.llvm.org  Thu Jul  3 03:39:35 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 03:39:35 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
Message-ID: <llvm.org/llvm/llvm-project/pull/146851@github.com>


https://github.com/EbinJose2002 created https://github.com/llvm/llvm-project/pull/146851

Solves #122823
An unnecessary convert instruction was generated in the case of reference to boxchar conversion. So excluded them in the condition. The output is generating correctly.

>From 6476107702e746e1fcd6a95a0e88fb8df6c4b97f Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Tue, 1 Jul 2025 18:11:05 +0530
Subject: [PATCH 1/3] Removed unnecessary handling of reference to boxchar
 conversion in BufferizeHLFIR.cpp

---
 flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 58f2b57712974..240b79138b0e1 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -446,7 +446,8 @@ struct AssociateOpConversion
             ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
               !mlir::isa<fir::BoxCharType>(assocType)))) {
           sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
-        } else {
+        } else if (!mlir::isa<fir::ReferenceType>(sourceVar.getType()) ||
+                   !mlir::isa<fir::BoxCharType>(assocType)) {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
         }
         return sourceVar;

>From ee425fb1685f8d783d0e50e76b15df59c921e3b2 Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Thu, 3 Jul 2025 15:58:59 +0530
Subject: [PATCH 2/3] Added test file for the issue

---
 flang/test/Lower/adjustr.f90 | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 flang/test/Lower/adjustr.f90

diff --git a/flang/test/Lower/adjustr.f90 b/flang/test/Lower/adjustr.f90
new file mode 100644
index 0000000000000..6add2f5456bbc
--- /dev/null
+++ b/flang/test/Lower/adjustr.f90
@@ -0,0 +1,14 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+! Checking no reference to boxchar conversion is in the emitted fir.
+! CHECK-NOT: (!fir.ref<!fir.char<1,4>>) -> !fir.boxchar<1>
+
+program main
+  logical         ,dimension(1):: mask = .true.
+  character(len=2),dimension(1):: d1 = "a "
+  character(len=4),dimension(1):: d4
+  where (mask)
+    d4 = adjustr(d1 // d1)
+  end where
+  write(6,*) "d4     =", d4
+end
\ No newline at end of file

>From 3928151da22c344c403585c6b4420a3097991ad0 Mon Sep 17 00:00:00 2001
From: Ebin-McW <ebin.jose at multicorewareinc.com>
Date: Thu, 3 Jul 2025 16:04:36 +0530
Subject: [PATCH 3/3] Update adjustr.f90

---
 flang/test/Lower/adjustr.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Lower/adjustr.f90 b/flang/test/Lower/adjustr.f90
index 6add2f5456bbc..4813834614bd6 100644
--- a/flang/test/Lower/adjustr.f90
+++ b/flang/test/Lower/adjustr.f90
@@ -11,4 +11,4 @@ program main
     d4 = adjustr(d1 // d1)
   end where
   write(6,*) "d4     =", d4
-end
\ No newline at end of file
+end


From flang-commits at lists.llvm.org  Thu Jul  3 03:40:08 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 03:40:08 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <68665e08.630a0220.1e5dc7.539e@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Ebin-McW (EbinJose2002)

<details>
<summary>Changes</summary>

Solves #<!-- -->122823
An unnecessary convert instruction was generated in the case of reference to boxchar conversion. So excluded them in the condition. The output is generating correctly.

---
Full diff: https://github.com/llvm/llvm-project/pull/146851.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp (+2-1) 
- (added) flang/test/Lower/adjustr.f90 (+14) 


``````````diff
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 58f2b57712974..240b79138b0e1 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -446,7 +446,8 @@ struct AssociateOpConversion
             ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
               !mlir::isa<fir::BoxCharType>(assocType)))) {
           sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
-        } else {
+        } else if (!mlir::isa<fir::ReferenceType>(sourceVar.getType()) ||
+                   !mlir::isa<fir::BoxCharType>(assocType)) {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
         }
         return sourceVar;
diff --git a/flang/test/Lower/adjustr.f90 b/flang/test/Lower/adjustr.f90
new file mode 100644
index 0000000000000..4813834614bd6
--- /dev/null
+++ b/flang/test/Lower/adjustr.f90
@@ -0,0 +1,14 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+! Checking no reference to boxchar conversion is in the emitted fir.
+! CHECK-NOT: (!fir.ref<!fir.char<1,4>>) -> !fir.boxchar<1>
+
+program main
+  logical         ,dimension(1):: mask = .true.
+  character(len=2),dimension(1):: d1 = "a "
+  character(len=4),dimension(1):: d4
+  where (mask)
+    d4 = adjustr(d1 // d1)
+  end where
+  write(6,*) "d4     =", d4
+end

``````````

</details>


https://github.com/llvm/llvm-project/pull/146851

From flang-commits at lists.llvm.org  Thu Jul  3 03:51:34 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 03 Jul 2025 03:51:34 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][OpenMP] Allocate `reduction`
 init temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <686660b6.050a0220.25d487.d513@mx.google.com>


ergawy wrote:

CI failure seems totally unrelated. The failing test is: `Semantics/windows.f90`.

https://github.com/llvm/llvm-project/pull/146667

From flang-commits at lists.llvm.org  Thu Jul  3 03:52:30 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 03 Jul 2025 03:52:30 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][OpenMP] Allocate `reduction`
 init temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <686660ee.170a0220.205d56.432a@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/146667

>From 860bd598def52f93e0d1f043b02153308ac0e1b4 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 05:31:38 -0500
Subject: [PATCH 1/2] [flang][OpenMP] Allocate `reduction` init temps on the
 stack for GPUs

Temps needed for the reduction init regions are now allocate on the heap
all the time. However, this is performance killer for GPUs since malloc
calls are prohibitively expensive. Therefore, we should do these
allocations on the stack for GPU reductions.
---
 .../Lower/Support/PrivateReductionUtils.cpp   |  47 +++--
 .../Lower/OpenMP/parallel-reduction-array.f90 | 166 ++++++++++--------
 2 files changed, 119 insertions(+), 94 deletions(-)

diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index e878041d37c03..c3a5b6101ce00 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -502,22 +502,37 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
 
   // Allocating on the heap in case the whole reduction/privatization is nested
   // inside of a loop
-  auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
-  // if needsDealloc isn't statically false, add cleanup region. Always
-  // do this for allocatable boxes because they might have been re-allocated
-  // in the body of the loop/parallel region
-
-  std::optional<int64_t> cstNeedsDealloc = fir::getIntIfConstant(needsDealloc);
-  assert(cstNeedsDealloc.has_value() &&
-         "createTempFromMold decides this statically");
-  if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
-                        isDoConcurrent);
-  } else {
-    assert(!isAllocatableOrPointer &&
-           "Pointer-like arrays must be heap allocated");
-  }
+  auto temp = [&]() {
+    bool shouldAllocateOnStack = false;
+
+    // On the GPU, always allocate on the stack since heap allocatins are very
+    // expensive.
+    if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
+            *builder.getModule()))
+      shouldAllocateOnStack = offloadMod.getIsGPU();
+
+    if (shouldAllocateOnStack)
+      return createStackTempFromMold(loc, builder, source);
+
+    auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
+    // if needsDealloc isn't statically false, add cleanup region. Always
+    // do this for allocatable boxes because they might have been re-allocated
+    // in the body of the loop/parallel region
+
+    std::optional<int64_t> cstNeedsDealloc =
+        fir::getIntIfConstant(needsDealloc);
+    assert(cstNeedsDealloc.has_value() &&
+           "createTempFromMold decides this statically");
+    if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
+                          isDoConcurrent);
+    } else {
+      assert(!isAllocatableOrPointer &&
+             "Pointer-like arrays must be heap allocated");
+    }
+    return temp;
+  }();
 
   // Put the temporary inside of a box:
   // hlfir::genVariableBox doesn't handle non-default lower bounds
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 8e3de498f59c1..4f889d9a4e77f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -1,5 +1,8 @@
-! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+
+! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
 
 program reduce
 integer, dimension(3) :: i = 0
@@ -13,81 +16,88 @@ program reduce
 print *,i
 end program
 
-! CHECK-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
-! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK-LABEL:   } init {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
-! CHECK:           %[[TRUE:.*]]  = arith.constant true
+! CPU-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
+! CPU:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU-LABEL:   } init {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = arith.constant 0 : i32
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_4:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CPU:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
+! CPU:           %[[TRUE:.*]]  = arith.constant true
 !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
-! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         } combiner {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[C1_0:.*]] = arith.constant 1 : index
-! CHECK:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
-! CHECK:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
-! CHECK:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
-! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
-! CHECK:           }
-! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         }  cleanup {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
-! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
-! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
-! CHECK:           fir.if %[[VAL_5]] {
-! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
-! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
-! CHECK:           }
-! CHECK:           omp.yield
-! CHECK:         }
+! CPU:           %[[C0:.*]] = arith.constant 0 : index
+! CPU:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
+! CPU:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         } combiner {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[C1:.*]] = arith.constant 1 : index
+! CPU:           %[[C3:.*]] = arith.constant 3 : index
+! CPU:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[C1_0:.*]] = arith.constant 1 : index
+! CPU:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
+! CPU:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+! CPU:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
+! CPU:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
+! CPU:           }
+! CPU:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         }  cleanup {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
+! CPU:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CPU:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CPU:           fir.if %[[VAL_5]] {
+! CPU:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
+! CPU:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
+! CPU:           }
+! CPU:           omp.yield
+! CPU:         }
+
+! CPU-LABEL:   func.func @_QQmain()
+! CPU:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_1:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CPU:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
+! CPU:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:             %[[VAL_8:.*]] = arith.constant 1 : i32
+! CPU:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_10:.*]] = arith.constant 1 : index
+! CPU:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = arith.constant 2 : i32
+! CPU:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_14:.*]] = arith.constant 2 : index
+! CPU:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_16:.*]] = arith.constant 3 : i32
+! CPU:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_18:.*]] = arith.constant 3 : index
+! CPU:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
+! CPU:             omp.terminator
+! CPU:           }
 
-! CHECK-LABEL:   func.func @_QQmain()
-! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
-! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
-! CHECK:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : index
-! CHECK:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = arith.constant 2 : i32
-! CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_14:.*]] = arith.constant 2 : index
-! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_16:.*]] = arith.constant 3 : i32
-! CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_18:.*]] = arith.constant 3 : index
-! CHECK:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
-! CHECK:             omp.terminator
-! CHECK:           }
+! GPU: omp.declare_reduction {{.*}} alloc {
+! GPU: } init {
+! GPU-NOT: fir.allocmem {{.*}} {bindc_name = ".tmp", {{.*}}}
+! GPU:     fir.alloca {{.*}} {bindc_name = ".tmp"}
+! GPU: } combiner {
+! GPU: }

>From 4cdd2577b62fc1e0c954d55ad8c1f8628878a76f Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 2 Jul 2025 23:29:23 -0500
Subject: [PATCH 2/2] add todo

---
 .../LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 883d179580e0c..ed88c19ab2c25 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -1291,6 +1291,11 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
     mapInitializationArgs(op, moduleTranslation, reductionDecls,
                           reductionVariableMap, i);
 
+    // TODO In some cases (specially on the GPU), the init regions may
+    // contains stack alloctaions. If the region is inlined in a loop, this is
+    // problematic. Instead of just inlining the region, handle allocations by
+    // hoisting fixed length allocations to the function entry and using
+    // stacksave and restore for variable length ones.
     if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
                                        "omp.reduction.neutral", builder,
                                        moduleTranslation, &phis)))


From flang-commits at lists.llvm.org  Thu Jul  3 03:54:40 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 03 Jul 2025 03:54:40 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][OpenMP] Allocate `reduction`
 init temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <68666170.170a0220.a1353.55f2@mx.google.com>


https://github.com/tblah approved this pull request.


https://github.com/llvm/llvm-project/pull/146667

From flang-commits at lists.llvm.org  Thu Jul  3 04:48:58 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 04:48:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146543@github.com>
Message-ID: <68666e2a.630a0220.3639aa.4fe9@mx.google.com>


https://github.com/jeanPerier edited https://github.com/llvm/llvm-project/pull/146543

From flang-commits at lists.llvm.org  Thu Jul  3 04:52:36 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 04:52:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146543@github.com>
Message-ID: <68666f04.170a0220.13362a.5417@mx.google.com>


jeanPerier wrote:

Thanks for the review!

>  It will be helpful for anybody working on in the future if you can elaborate a bit more how it works. May be take the comments in debug-cyclic-derived-type-caching-complex.f90, adds in some more details and make that part of the commit message.

I added an illustration in the description.

> Also when I looked at this part last year, I had a quite smallish patch that removed the limitation from the mlir about recursive types. I can't remember why I did not pursue it further but I think this is one thing we need to explore a bit. It may be much simpler to remove that limitation in mlir than to workaround it in flang.

That would be great! I will merge this one, but if you can get your hands on this patch, I agree that changing the handling in MLIR would be best if that is possible. Then we can go for a dumb map.

https://github.com/llvm/llvm-project/pull/146543

From flang-commits at lists.llvm.org  Thu Jul  3 04:53:04 2025
From: flang-commits at lists.llvm.org (Kelvin Li via flang-commits)
Date: Thu, 03 Jul 2025 04:53:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] fix skip-external-rtti-definition
	for ppc (PR #146826)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146826@github.com>
Message-ID: <68666f20.170a0220.161622.7e30@mx.google.com>


https://github.com/kkwli approved this pull request.

LG. Thanks for fixing it.

https://github.com/llvm/llvm-project/pull/146826

From flang-commits at lists.llvm.org  Thu Jul  3 05:06:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:06:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146543@github.com>
Message-ID: <68667256.170a0220.122df6.98b6@mx.google.com>


================
@@ -272,31 +271,127 @@ DebugTypeGenerator::getFieldSizeAndAlign(mlir::Type fieldTy) {
   return std::pair{byteSize, byteAlign};
 }
 
+mlir::LLVM::DITypeAttr DerivedTypeCache::lookup(mlir::Type type) {
+  auto iter = typeCache.find(type);
+  if (iter != typeCache.end()) {
+    if (iter->second.first) {
+      componentActiveRecursionLevels = iter->second.second;
+    }
+    return iter->second.first;
+  }
+  return nullptr;
+}
+
+DerivedTypeCache::ActiveLevels
+DerivedTypeCache::startTranslating(mlir::Type type,
+                                   mlir::LLVM::DITypeAttr placeHolder) {
+  derivedTypeDepth++;
+  if (!placeHolder)
+    return {};
+  typeCache[type] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      placeHolder, {derivedTypeDepth});
+  return {};
+}
----------------
jeanPerier wrote:

This is expected, at the starting point of the translation, it is not yet know which recursive references will be met directly or indirectly in components.

This list will be updated in postComponentVisitUpdate().

I added as a startTranslating results to enforce its creation at the right point.

https://github.com/llvm/llvm-project/pull/146543

From flang-commits at lists.llvm.org  Thu Jul  3 05:09:05 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:09:05 -0700 (PDT)
Subject: [flang-commits] [flang] 4868d66 - [flang] improve DITypeAttr
 caching with recursive derived types (#146543)
Message-ID: <686672e1.170a0220.3d8ce4.5acc@mx.google.com>


Author: jeanPerier
Date: 2025-07-03T14:09:01+02:00
New Revision: 4868d66282b231f22b464471e9a16a1ec2da015e

URL: https://github.com/llvm/llvm-project/commit/4868d66282b231f22b464471e9a16a1ec2da015e
DIFF: https://github.com/llvm/llvm-project/commit/4868d66282b231f22b464471e9a16a1ec2da015e.diff

LOG: [flang] improve DITypeAttr caching with recursive derived types (#146543)

The current DITypeAttr caching for derived type debug metadata
generation strategy is not optimal. This turns out to be an issue for
compile times in apps with very very complex derived types like CP2K

See the added debug-cyclic-derived-type-caching-simple.f90 test for more
details about the duplication issue.

As a real world example justifying the new non trivial caching strategy,
in CP2K, emitting debug type info for the swarm_worker_type` in swarm_worker.F
caused 1,747,347 llvm debug metadata nodes to be emitted instead of 8023
after this patch (200x less) leading to noticeable compile time
improvements (I measured 0.12s spent in `AddDebugInfo` pass instead of
7.5s prior to this patch).

The main idea is that caching is now associating to the cached
DITypeAttr tree for a derived type a list of parent nodes being referred
to recursively via indices in this DITypeAttr.

When leaving the context of a parent node, all types that were cached
and linked to this parent node are cleared from the cache.
This allows more reusage in sub-trees while still fulfilling the MLIR
requirements that DITypeAttr types referring to a parent DITypeAttr via
integer id should only be used inside the DITypeAttr of the parent.

Most of the complexity comes from computing the "list of parent nodes"
by merging the ones from the components.

This is made is such a way that the extra cost for apps without
recursive derived type is minimal because the extra data structure
should not require extra dynamic allocations when they are no or little
recursion.

Example:

Take the following type graph (Fortran source for it in the added
debug-cyclic-derived-type-caching-complex.f90).
A is the tope level types, and has direct components of types B, C, and
E.
There are cycles in the type tree introduced by type B and D.
Types `C` and `E` are of interest here because they are in the middle of
those cycles and appear in several places in the type tree. There
occurrences is labeled in brackets in the order of visit by the
DebugTypeGenerator.

```
 A -> B -> C [1] -> D -> E [1] -> F -> G -> B
 |   |              |             |
 |   |              |             | -> D
 |   |              |
 |   |              | -> H -> E [2] ->  F -> G -> B
 |   |                                  |
 |   |                                  |-> D
 |   |
 |   | -> I -> E [3] ->  F -> G -> B
 |   |                   |
 |   |                   |-> D
 |   | -> C [2]
 |
 | -> C [3] -> D
 | -> E [4] -> F -> G -> B
               |
               | -> D
```

With this patch, E[2] and E[3] can share the same DITypeAttr as well as
C[1] and C[2] while they previously all got there own nodes.

To be safe with regards to cycles in MLIR, a DITypeAttr created for a
node N2 under a node N1 being recursively referred to and above the
recursive reference to N1 shall not be used above N1 in the DITypeAttr
tree. It can however be used in several places under N1.

Hence here:
-E[2] cannot reuse E[1] DITypeAttr because D appears above and under
E[1].
-E[3] can reuse E[2] DITypeAttr because they are both under B and above
D.
-E[4] cannot reuse E[3] DITypeAttr  because it is above B.

This is achieved by this patch because when visiting A and reaching B,
the recursive reference to B is registered in the visit context. This
context is added D when going back-up in F. So when reaching back E[1]
with the information to build its DITypeAttr, its recursive references
are known and saved along the DITypeAttr in the cache.

When reaching back D, the cache for E is cleared because it is known it
depended on D. A new DITypeAttr is created after E[2], and this time it
only depends on B because the D under E[2] is not a recursive reference
(D is not above E[2]). Hence, when reaching E[3] it can be reused, and
the cache entry for E[2] is cleared when reaching B, which leads to a
new DITypeAttr to be created for E[4].

Added: 
    flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
    flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90

Modified: 
    flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
    flang/lib/Optimizer/Transforms/DebugTypeGenerator.h

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index cdd30dce183dd..a848058486e2c 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -48,8 +48,7 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m,
                                        mlir::SymbolTable *symbolTable_,
                                        const mlir::DataLayout &dl)
     : module(m), symbolTable(symbolTable_), dataLayout{&dl},
-      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl),
-      derivedTypeDepth(0) {
+      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl) {
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 
   mlir::MLIRContext *context = module.getContext();
@@ -272,31 +271,127 @@ DebugTypeGenerator::getFieldSizeAndAlign(mlir::Type fieldTy) {
   return std::pair{byteSize, byteAlign};
 }
 
+mlir::LLVM::DITypeAttr DerivedTypeCache::lookup(mlir::Type type) {
+  auto iter = typeCache.find(type);
+  if (iter != typeCache.end()) {
+    if (iter->second.first) {
+      componentActiveRecursionLevels = iter->second.second;
+    }
+    return iter->second.first;
+  }
+  return nullptr;
+}
+
+DerivedTypeCache::ActiveLevels
+DerivedTypeCache::startTranslating(mlir::Type type,
+                                   mlir::LLVM::DITypeAttr placeHolder) {
+  derivedTypeDepth++;
+  if (!placeHolder)
+    return {};
+  typeCache[type] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      placeHolder, {derivedTypeDepth});
+  return {};
+}
+
+void DerivedTypeCache::preComponentVisitUpdate() {
+  componentActiveRecursionLevels.clear();
+}
+
+void DerivedTypeCache::postComponentVisitUpdate(
+    ActiveLevels &activeRecursionLevels) {
+  if (componentActiveRecursionLevels.empty())
+    return;
+  ActiveLevels oldLevels;
+  oldLevels.swap(activeRecursionLevels);
+  std::merge(componentActiveRecursionLevels.begin(),
+             componentActiveRecursionLevels.end(), oldLevels.begin(),
+             oldLevels.end(), std::back_inserter(activeRecursionLevels));
+}
+
+void DerivedTypeCache::finalize(mlir::Type ty, mlir::LLVM::DITypeAttr attr,
+                                ActiveLevels &&activeRecursionLevels) {
+  // If there is no nested recursion or if this type does not point to any type
+  // nodes above it, it is safe to cache it indefinitely (it can be used in any
+  // contexts).
+  if (activeRecursionLevels.empty() ||
+      (activeRecursionLevels[0] == derivedTypeDepth)) {
+    typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(attr, {});
+    componentActiveRecursionLevels.clear();
+    cleanUpCache(derivedTypeDepth);
+    --derivedTypeDepth;
+    return;
+  }
+  // Trim any recursion below the current type.
+  if (activeRecursionLevels.back() >= derivedTypeDepth) {
+    auto last = llvm::find_if(activeRecursionLevels, [&](std::int32_t depth) {
+      return depth >= derivedTypeDepth;
+    });
+    if (last != activeRecursionLevels.end()) {
+      activeRecursionLevels.erase(last, activeRecursionLevels.end());
+    }
+  }
+  componentActiveRecursionLevels = std::move(activeRecursionLevels);
+  typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      attr, componentActiveRecursionLevels);
+  cleanUpCache(derivedTypeDepth);
+  if (!componentActiveRecursionLevels.empty())
+    insertCacheCleanUp(ty, componentActiveRecursionLevels.back());
+  --derivedTypeDepth;
+}
+
+void DerivedTypeCache::insertCacheCleanUp(mlir::Type type, int32_t depth) {
+  auto iter = llvm::find_if(cacheCleanupList,
+                            [&](const auto &x) { return x.second >= depth; });
+  if (iter == cacheCleanupList.end()) {
+    cacheCleanupList.emplace_back(
+        std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+    return;
+  }
+  if (iter->second == depth) {
+    iter->first.push_back(type);
+    return;
+  }
+  cacheCleanupList.insert(
+      iter, std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+}
+
+void DerivedTypeCache::cleanUpCache(int32_t depth) {
+  if (cacheCleanupList.empty())
+    return;
+  // cleanups are done in the post actions when visiting a derived type
+  // tree. So if there is a clean-up for the current depth, it has to be
+  // the last one (deeper ones must have been done already).
+  if (cacheCleanupList.back().second == depth) {
+    for (mlir::Type type : cacheCleanupList.back().first)
+      typeCache[type].first = nullptr;
+    cacheCleanupList.pop_back_n(1);
+  }
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     fir::RecordType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
-  // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
 
-  bool canCacheThisType = true;
-  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
   mlir::MLIRContext *context = module.getContext();
-  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto [nameKind, sourceName] = fir::NameUniquer::deconstruct(Ty.getName());
+  if (nameKind != fir::NameUniquer::NameKind::DERIVED_TYPE)
+    return genPlaceholderType(context);
+
+  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   // Generate a place holder TypeAttr which will be used if a member
   // references the parent type.
-  auto comAttr = mlir::LLVM::DICompositeTypeAttr::get(
+  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto placeHolder = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/true, llvm::dwarf::DW_TAG_structure_type,
       mlir::StringAttr::get(context, ""), fileAttr, /*line=*/0, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = comAttr;
-
-  auto result = fir::NameUniquer::deconstruct(Ty.getName());
-  if (result.first != fir::NameUniquer::NameKind::DERIVED_TYPE)
-    return genPlaceholderType(context);
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty, placeHolder);
 
   fir::TypeInfoOp tiOp = symbolTable->lookup<fir::TypeInfoOp>(Ty.getName());
   unsigned line = (tiOp) ? getLineFromLoc(tiOp.getLoc()) : 1;
@@ -305,6 +400,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
   mlir::IntegerType intTy = mlir::IntegerType::get(context, 64);
   std::uint64_t offset = 0;
   for (auto [fieldName, fieldTy] : Ty.getTypeList()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     std::optional<llvm::ArrayRef<int64_t>> lowerBounds =
         fir::getComponentLowerBoundsIfNonDefault(Ty, fieldName, module,
@@ -317,7 +413,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     mlir::LLVM::DITypeAttr elemTy;
     if (lowerBounds && seqTy &&
         lowerBounds->size() == seqTy.getShape().size()) {
-      llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> arrayElements;
       for (auto [bound, dim] :
            llvm::zip_equal(*lowerBounds, seqTy.getShape())) {
         auto countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
@@ -325,14 +421,14 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
             context, countAttr, lowerAttr, /*upperBound=*/nullptr,
             /*stride=*/nullptr);
-        elements.push_back(subrangeTy);
+        arrayElements.push_back(subrangeTy);
       }
       elemTy = mlir::LLVM::DICompositeTypeAttr::get(
           context, llvm::dwarf::DW_TAG_array_type, /*name=*/nullptr,
           /*file=*/nullptr, /*line=*/0, /*scope=*/nullptr,
           convertType(seqTy.getEleTy(), fileAttr, scope, declOp),
           mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0, /*alignInBits=*/0,
-          elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
+          arrayElements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
           /*allocated=*/nullptr, /*associated=*/nullptr);
     } else
       elemTy = convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -344,80 +440,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
-
-    // Currently, the handling of recursive debug type in mlir has some
-    // limitations that were discussed at the end of the thread for following
-    // PR.
-    // https://github.com/llvm/llvm-project/pull/106571
-    //
-    // Problem could be explained with the following example code:
-    //  type t2
-    //   type(t1), pointer :: p1
-    // end type
-    // type t1
-    //   type(t2), pointer :: p2
-    // end type
-    // In the description below, type_self means a temporary type that is
-    // generated
-    // as a place holder while the members of that type are being processed.
-    //
-    // If we process t1 first then we will have the following structure after
-    // it has been processed.
-    // t1 -> t2 -> t1_self
-    // This is because when we started processing t2, we did not have the
-    // complete t1 but its place holder t1_self.
-    // Now if some entity requires t2, we will already have that in cache and
-    // will return it. But this t2 refers to t1_self and not to t1. In mlir
-    // handling, only those types are allowed to have _self reference which are
-    // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
-    // because the t1_self reference can be resolved by the outer t1. But
-    // standalone t2 is not because there will be no way to resolve it. Until
-    // this is fixed in mlir, we avoid caching such types. Please see
-    // DebugTranslation::translateRecursive for details on how mlir handles
-    // recursive types.
-    // The code below checks for situation where it will be unsafe to cache
-    // a type to avoid this problem. We do that in 2 situations.
-    // 1. If a member is record type, then its type would have been processed
-    // before reaching here. If it is not in the cache, it means that it was
-    // found to be unsafe to cache. So any type containing it will also not
-    // be cached
-    // 2. The type of the member is found in the cache but it is a place holder.
-    // In this case, its recID should match the recID of the type we are
-    // processing. This helps us to cache the following type.
-    // type t
-    //  type(t), allocatable :: p
-    // end type
-    mlir::Type baseTy = getDerivedType(fieldTy);
-    if (auto recTy = mlir::dyn_cast<fir::RecordType>(baseTy)) {
-      auto iter = typeCache.find(recTy);
-      if (iter == typeCache.end())
-        canCacheThisType = false;
-      else {
-        if (auto tyAttr =
-                mlir::dyn_cast<mlir::LLVM::DICompositeTypeAttr>(iter->second)) {
-          if (tyAttr.getIsRecSelf() && tyAttr.getRecId() != recId)
-            canCacheThisType = false;
-        }
-      }
-    }
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto finalAttr = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/false, llvm::dwarf::DW_TAG_structure_type,
-      mlir::StringAttr::get(context, result.second.name), fileAttr, line, scope,
+      mlir::StringAttr::get(context, sourceName.name), fileAttr, line, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
 
-  // derivedTypeDepth == 1 means that it is a top level type which is safe to
-  // cache.
-  if (canCacheThisType || derivedTypeDepth == 1) {
-    typeCache[Ty] = finalAttr;
-  } else {
-    auto iter = typeCache.find(Ty);
-    if (iter != typeCache.end())
-      typeCache.erase(iter);
-  }
+  derivedTypeCache.finalize(Ty, finalAttr, std::move(nestedRecursions));
+
   return finalAttr;
 }
 
@@ -425,15 +459,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
     mlir::TupleType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
   // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty);
 
   llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   mlir::MLIRContext *context = module.getContext();
 
   std::uint64_t offset = 0;
   for (auto fieldTy : Ty.getTypes()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     mlir::LLVM::DITypeAttr elemTy =
         convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -445,6 +482,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto typeAttr = mlir::LLVM::DICompositeTypeAttr::get(
@@ -453,7 +491,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = typeAttr;
+  derivedTypeCache.finalize(Ty, typeAttr, std::move(nestedRecursions));
   return typeAttr;
 }
 
@@ -667,27 +705,7 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
     return convertCharacterType(charTy, fileAttr, scope, declOp,
                                 /*hasDescriptor=*/false);
   } else if (auto recTy = mlir::dyn_cast_if_present<fir::RecordType>(Ty)) {
-    // For nested derived types like shown below, the call sequence of the
-    // convertRecordType will look something like as follows:
-    // convertRecordType (t1)
-    //  convertRecordType (t2)
-    //    convertRecordType (t3)
-    // We need to recognize when we are processing the top level type like t1
-    // to make caching decision. The variable `derivedTypeDepth` is used for
-    // this purpose and maintains the current depth of derived type processing.
-    //  type t1
-    //   type(t2), pointer :: p1
-    // end type
-    // type t2
-    //   type(t3), pointer :: p2
-    // end type
-    // type t2
-    //   integer a
-    // end type
-    derivedTypeDepth++;
-    auto result = convertRecordType(recTy, fileAttr, scope, declOp);
-    derivedTypeDepth--;
-    return result;
+    return convertRecordType(recTy, fileAttr, scope, declOp);
   } else if (auto tupleTy = mlir::dyn_cast_if_present<mlir::TupleType>(Ty)) {
     return convertTupleType(tupleTy, fileAttr, scope, declOp);
   } else if (auto refTy = mlir::dyn_cast_if_present<fir::ReferenceType>(Ty)) {

diff  --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index 93b9ac2d90fdf..854e4397ca32b 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -23,6 +23,77 @@
 
 namespace fir {
 
+/// Special cache to deal with the fact that mlir::LLVM::DITypeAttr for
+/// derived types may only be valid in specific nesting contexts in presence
+/// of derived type recursion and cannot be cached for the whole compilation.
+/// It is however still desirable to cache such mlir::LLVM::DITypeAttr as
+/// long as possible to avoid catastrophic compilation slow downs in very
+/// complex derived types where an intermediate type in a derived type cycle may
+/// indirectly appear hundreds of times under the top type of the derived type
+/// cycle. More details in the comment below.
+class DerivedTypeCache {
+public:
+  // Currently, the handling of recursive debug type in mlir has some
+  // limitations that were discussed at the end of the thread for following
+  // PR.
+  // https://github.com/llvm/llvm-project/pull/106571
+  //
+  // Problem could be explained with the following example code:
+  //  type t2
+  //   type(t1), pointer :: p1
+  // end type
+  // type t1
+  //   type(t2), pointer :: p2
+  // end type
+  // In the description below, type_self means a temporary type that is
+  // generated
+  // as a place holder while the members of that type are being processed.
+  //
+  // If we process t1 first then we will have the following structure after
+  // it has been processed.
+  // t1 -> t2 -> t1_self
+  // This is because when we started processing t2, we did not have the
+  // complete t1 but its place holder t1_self.
+  // Now if some entity requires t2, we will already have that in cache and
+  // will return it. But this t2 refers to t1_self and not to t1. In mlir
+  // handling, only those types are allowed to have _self reference which are
+  // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
+  // because the t1_self reference can be resolved by the outer t1. But
+  // standalone t2 is not because there will be no way to resolve it. Until
+  // this is fixed in mlir, we avoid caching such types. Please see
+  // DebugTranslation::translateRecursive for details on how mlir handles
+  // recursive types.
+  using ActiveLevels = llvm::SmallVector<int32_t, 1>;
+  mlir::LLVM::DITypeAttr lookup(mlir::Type);
+  ActiveLevels startTranslating(mlir::Type,
+                                mlir::LLVM::DITypeAttr placeHolder = nullptr);
+  void finalize(mlir::Type, mlir::LLVM::DITypeAttr, ActiveLevels &&);
+  void preComponentVisitUpdate();
+  void postComponentVisitUpdate(ActiveLevels &);
+
+private:
+  void insertCacheCleanUp(mlir::Type type, int32_t depth);
+  void cleanUpCache(int32_t depth);
+  // Current depth inside a top level derived type being converted.
+  int32_t derivedTypeDepth = 0;
+  // Cache for already translated derived types with the minimum depth where
+  // this cache entry is valid. Zero means the translation is always valid, "i"
+  // means the type depends its derived type tree parent node at depth "i". Such
+  // types should be cleaned-up from the cache in the post visit of node "i".
+  // Note that any new metadata created for a type with a component in the cache
+  // with validity of "i" shall not be added to the cache with a validity
+  // smaller than "i".
+  llvm::DenseMap<mlir::Type, std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>>
+      typeCache;
+  // List of parent nodes that are being recursively referred to in the
+  // component type that has just been computed.
+  ActiveLevels componentActiveRecursionLevels;
+  // Helper list that maintains the list of nodes that must be deleted from the
+  // cache when going back past listed parent depths.
+  llvm::SmallVector<std::pair<llvm::SmallVector<mlir::Type>, int32_t>>
+      cacheCleanupList;
+};
+
 /// This converts FIR/mlir type to DITypeAttr.
 class DebugTypeGenerator {
 public:
@@ -91,8 +162,7 @@ class DebugTypeGenerator {
   std::uint64_t lenOffset;
   std::uint64_t rankOffset;
   std::uint64_t rankSize;
-  int32_t derivedTypeDepth;
-  llvm::DenseMap<mlir::Type, mlir::LLVM::DITypeAttr> typeCache;
+  DerivedTypeCache derivedTypeCache;
 };
 
 } // namespace fir

diff  --git a/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
new file mode 100644
index 0000000000000..72b7a78bef510
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
@@ -0,0 +1,116 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Test that debug metadata for derived types is not duplicated more than needed
+! when emitting debug info with non trivial cycles.
+
+
+! In the type graph below, G has a back edge to B, and F to D.
+! This causes C to be in the middle of B cycle, and E to be
+! both in B and D cycles.
+! C and E are used in several contexts, under B, under D, and outside
+! of it to test how metadata is generated for them.
+!
+! Without "local caching" of C and E when generating mlir::LLVM::DITypeAttr
+! for such derived types, many duplicate llvm metadata for the derived types
+! would be emitted, while with the right duplication of mlir::LLVM::DITypeAttr,
+! a lot more duplicate llvm IR metadata ends up emitted (19 DICompositeType
+! vs 71 before the patch that added this test).
+!
+!
+!  A -> B -> C -> D -> E -> F -> G -> B
+!  |    |         |         |
+!  |    |         |         | -> D
+!  |    |         |
+!  |    |         | -> H -> E
+!  |    |
+!  |    | -> I -> E
+!  |         | -> C
+!  |
+!  | -> C
+!  | -> E
+
+subroutine type_cycles_caching()
+  type g
+    type(b), pointer :: c_b
+  end type
+  type f
+    type(g) :: c_b
+    type(d), pointer :: c_d
+  end type
+  type e
+    type(f) :: c_f
+  end type
+    type h
+      ! Can reuse metadata of type(e) under 'd'.
+      type(e) :: c_e
+    end type
+  type d
+    type(e) :: c_e
+    type(h) :: c_h
+  end type
+  type c
+    type(d) :: c_d
+  end type
+    type i
+      ! Cannot reuse metadata of type(e) under 'd'.
+      type(e) :: c_e
+      ! Can reuse metadata of type(c) under 'b'.
+      type(c) :: c_c
+    end type
+  type b
+    type(c) :: c_c
+    type(i) :: c_i
+  end type
+  type a
+    type(b) :: c_b
+    ! Cannot reuse metadata of type(c) under 'b'
+    type(c) :: c_c
+    ! Cannot reuse metadata of type(e) under 'd', nor the one under 'b'
+    type(e) :: c_e
+  end type
+  type(a) :: xa
+  ! Can reuse metadata of type(c) for xa%c_c
+  type(c) :: xc
+  ! Can reuse metadata of type(c) for xa%c_e
+  type(e) :: xe
+  call bar(xa, xc, xe)
+end subroutine
+
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "a",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "b",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "c",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "d",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "g",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "h",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "i",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "c"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "d"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "g"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "h"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType

diff  --git a/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
new file mode 100644
index 0000000000000..c1f49d2f4a74a
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
@@ -0,0 +1,39 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Simple test that checks that metadata for `t0` is only duplicated once.
+
+! The 
diff iculty is that at the mlir::LLVM::DITypeAttr, because of the
+! lack of MLIR attribute true recursion, the mlir::LLVM::DITypeAttr for
+! `t0` inside `t1` is special because it is only valid when found
+! in an mlir::LLVM::DITypeAttr tree under `base` (it will point to `base`
+! via an integer id that is only meanigfu;l when a node with such id has
+! been traversed).
+! A 
diff erent node has to be created for `t0` usage in `x0` (will
+! point to the actual mlir::LLVM::DITypeAttr for `base` instead of
+! an integer id since the cycle is already "taken care of" in `base`
+! definition).
+! However, the same special `t0` node can be used for both `x1_1` and
+! `x1_2` components because they are both under `base` in the
+! mlir::LLVM::DITypeAttr tree definition. This used to not be the case,
+! leading to a lot of duplicate mlir::LLVM::DITypeAttr and actual LLVM IR
+! metadata, causing noticeable compilation slowdowns in apps with non trivial
+! derived types.
+
+subroutine duplicate_cycle_branch()
+  type t0
+    type(base), pointer :: x
+  end type
+  type t1
+    type(t0) :: x1_1
+    type(t0) :: x1_2
+  end type
+  type base
+    type(t1) :: x_base
+  end type
+  type(base) :: x
+  type(t0) :: x0
+  call bar(x, x0)
+end subroutine
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
+! CHECK-NOT: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",


        

From flang-commits at lists.llvm.org  Thu Jul  3 05:09:07 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:09:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] improve DITypeAttr caching with
 recursive derived types (PR #146543)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146543@github.com>
Message-ID: <686672e3.050a0220.1fed03.0d29@mx.google.com>


https://github.com/jeanPerier closed https://github.com/llvm/llvm-project/pull/146543

From flang-commits at lists.llvm.org  Thu Jul  3 05:10:34 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:10:34 -0700 (PDT)
Subject: [flang-commits] [flang] 8763ac3 - [flang] fix
	skip-external-rtti-definition for ppc (#146826)
Message-ID: <6866733a.170a0220.1dc2c8.4cf9@mx.google.com>


Author: jeanPerier
Date: 2025-07-03T14:10:31+02:00
New Revision: 8763ac3252f2f291db15c7adea9f72135a14c193

URL: https://github.com/llvm/llvm-project/commit/8763ac3252f2f291db15c7adea9f72135a14c193
DIFF: https://github.com/llvm/llvm-project/commit/8763ac3252f2f291db15c7adea9f72135a14c193.diff

LOG: [flang] fix skip-external-rtti-definition for ppc (#146826)

PPC does not use comdat. There is no need to check for that in the test,
just remove it.

Fix for https://lab.llvm.org/buildbot/#/builders/201/builds/5278

Added: 
    

Modified: 
    flang/test/Integration/skip-external-rtti-definition.F90

Removed: 
    


################################################################################
diff  --git a/flang/test/Integration/skip-external-rtti-definition.F90 b/flang/test/Integration/skip-external-rtti-definition.F90
index f86cc069b304d..35a5d559efa5e 100644
--- a/flang/test/Integration/skip-external-rtti-definition.F90
+++ b/flang/test/Integration/skip-external-rtti-definition.F90
@@ -33,9 +33,9 @@ subroutine needs_descriptor(x)
 
 #endif
 
-! LINKONCE-DAG: @_QMmodule_external_type_definitionEXnXt1 = linkonce_odr constant [2 x i8] c"t1", comdat
+! LINKONCE-DAG: @_QMmodule_external_type_definitionEXnXt1 = linkonce_odr constant [2 x i8] c"t1"
 ! LINKONCE-DAG: @_QMmodule_external_type_definitionEXdtXt1 = linkonce_odr constant {{.*}} {
-! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXnXt2 = linkonce_odr constant [2 x i8] c"t2", comdat
+! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXnXt2 = linkonce_odr constant [2 x i8] c"t2"
 ! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXdtXt2 = linkonce_odr constant {{.*}} {
 
 ! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1


        

From flang-commits at lists.llvm.org  Thu Jul  3 05:10:38 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:10:38 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] fix skip-external-rtti-definition
	for ppc (PR #146826)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146826@github.com>
Message-ID: <6866733e.170a0220.1ea674.4ead@mx.google.com>


https://github.com/jeanPerier closed https://github.com/llvm/llvm-project/pull/146826

From flang-commits at lists.llvm.org  Thu Jul  3 05:27:44 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:27:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix x86 REQUIRES in a couple of
	tests (PR #146869)
Message-ID: <llvm.org/llvm/llvm-project/pull/146869@github.com>


https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/146869

Many tests in Flang are looking for x86_64-registered-target, but this never exists because the target is just called x86.

These two pass with this corrected but the others I need to look into why they fail.

>From 746db3f46d3012e32cbdf21bf1656a89b9fa199b Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 12:25:53 +0000
Subject: [PATCH] [flang] Fix x86 REQUIRES in a couple of tests

Many tests in Flang are looking for x86_64-registered-target,
but this never exists because the target is just called x86.

These two pass with this corrected but the others I need
to look into why they fail.
---
 flang/test/Lower/Intrinsics/ieee_class_queries.f90 | 2 +-
 flang/test/Semantics/windows.f90                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Lower/Intrinsics/ieee_class_queries.f90 b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
index aae8071b6a8b5..8d20ff8f3178f 100644
--- a/flang/test/Lower/Intrinsics/ieee_class_queries.f90
+++ b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
@@ -1,5 +1,5 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir -o - %s | FileCheck %s
 
   ! CHECK-LABEL: func @_QQmain
diff --git a/flang/test/Semantics/windows.f90 b/flang/test/Semantics/windows.f90
index 705fb1b816895..799a34e6b89c5 100644
--- a/flang/test/Semantics/windows.f90
+++ b/flang/test/Semantics/windows.f90
@@ -1,4 +1,4 @@
-! RUN: %if x86_64-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
+! RUN: %if x86-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
 ! RUN: %if aarch64-registered-target %{ %python %S/test_errors.py %s %flang --target=aarch64-pc-windows-msvc -Werror %}
 
 subroutine uid


From flang-commits at lists.llvm.org  Thu Jul  3 05:28:18 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:28:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix x86 REQUIRES in a couple of
	tests (PR #146869)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146869@github.com>
Message-ID: <68667762.050a0220.70d52.ff9b@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: David Spickett (DavidSpickett)

<details>
<summary>Changes</summary>

Many tests in Flang are looking for x86_64-registered-target, but this never exists because the target is just called x86.

These two pass with this corrected but the others I need to look into why they fail.

---
Full diff: https://github.com/llvm/llvm-project/pull/146869.diff


2 Files Affected:

- (modified) flang/test/Lower/Intrinsics/ieee_class_queries.f90 (+1-1) 
- (modified) flang/test/Semantics/windows.f90 (+1-1) 


``````````diff
diff --git a/flang/test/Lower/Intrinsics/ieee_class_queries.f90 b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
index aae8071b6a8b5..8d20ff8f3178f 100644
--- a/flang/test/Lower/Intrinsics/ieee_class_queries.f90
+++ b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
@@ -1,5 +1,5 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir -o - %s | FileCheck %s
 
   ! CHECK-LABEL: func @_QQmain
diff --git a/flang/test/Semantics/windows.f90 b/flang/test/Semantics/windows.f90
index 705fb1b816895..799a34e6b89c5 100644
--- a/flang/test/Semantics/windows.f90
+++ b/flang/test/Semantics/windows.f90
@@ -1,4 +1,4 @@
-! RUN: %if x86_64-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
+! RUN: %if x86-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
 ! RUN: %if aarch64-registered-target %{ %python %S/test_errors.py %s %flang --target=aarch64-pc-windows-msvc -Werror %}
 
 subroutine uid

``````````

</details>


https://github.com/llvm/llvm-project/pull/146869

From flang-commits at lists.llvm.org  Thu Jul  3 05:28:18 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:28:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix x86 REQUIRES in a couple of
	tests (PR #146869)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146869@github.com>
Message-ID: <68667762.170a0220.25d67a.4e8e@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: David Spickett (DavidSpickett)

<details>
<summary>Changes</summary>

Many tests in Flang are looking for x86_64-registered-target, but this never exists because the target is just called x86.

These two pass with this corrected but the others I need to look into why they fail.

---
Full diff: https://github.com/llvm/llvm-project/pull/146869.diff


2 Files Affected:

- (modified) flang/test/Lower/Intrinsics/ieee_class_queries.f90 (+1-1) 
- (modified) flang/test/Semantics/windows.f90 (+1-1) 


``````````diff
diff --git a/flang/test/Lower/Intrinsics/ieee_class_queries.f90 b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
index aae8071b6a8b5..8d20ff8f3178f 100644
--- a/flang/test/Lower/Intrinsics/ieee_class_queries.f90
+++ b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
@@ -1,5 +1,5 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir -o - %s | FileCheck %s
 
   ! CHECK-LABEL: func @_QQmain
diff --git a/flang/test/Semantics/windows.f90 b/flang/test/Semantics/windows.f90
index 705fb1b816895..799a34e6b89c5 100644
--- a/flang/test/Semantics/windows.f90
+++ b/flang/test/Semantics/windows.f90
@@ -1,4 +1,4 @@
-! RUN: %if x86_64-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
+! RUN: %if x86-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
 ! RUN: %if aarch64-registered-target %{ %python %S/test_errors.py %s %flang --target=aarch64-pc-windows-msvc -Werror %}
 
 subroutine uid

``````````

</details>


https://github.com/llvm/llvm-project/pull/146869

From flang-commits at lists.llvm.org  Thu Jul  3 05:28:22 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:28:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix x86 REQUIRES in a couple of
	tests (PR #146869)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146869@github.com>
Message-ID: <68667766.050a0220.8cdf3.d203@mx.google.com>


DavidSpickett wrote:

>From a print in the lit config:
```
aarch64
amdgpu
arm
avr
bpf
hexagon
lanai
loongarch
mips
msp430
nvptx
powerpc
riscv
sparc
spirv
systemz
ve
webassembly
x86
xcore
```
Note x86 not x86_64.

https://github.com/llvm/llvm-project/pull/146869

From flang-commits at lists.llvm.org  Thu Jul  3 05:36:37 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:36:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix test REQUIRES and options
	(PR #146870)
Message-ID: <llvm.org/llvm/llvm-project/pull/146870@github.com>


https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/146870

This test should have been looking for the "x86" target, not "x86_64".

In the time it's not been running, -target must have been changed to -triple.

>From ebb936bcc0db22f62522ae33a503af1cf3a414a4 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 12:34:22 +0000
Subject: [PATCH] [flang][test] Fix test REQUIRES and options

This test should have been looking for the "x86" target,
not "x86_64".

In the time it's not been running, -target must have been
changed to -triple.
---
 flang/test/Lower/math-lowering/aint.f90 | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang/test/Lower/math-lowering/aint.f90 b/flang/test/Lower/math-lowering/aint.f90
index 6c7809f8ea1a3..b36fb421d7c8c 100644
--- a/flang/test/Lower/math-lowering/aint.f90
+++ b/flang/test/Lower/math-lowering/aint.f90
@@ -1,10 +1,10 @@
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
 
 function test_real4(x)
   real :: x, test_real4


From flang-commits at lists.llvm.org  Thu Jul  3 05:36:51 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:36:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix test REQUIRES and options
	for aint.f90 (PR #146870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146870@github.com>
Message-ID: <68667963.170a0220.15233c.a8df@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/146870

From flang-commits at lists.llvm.org  Thu Jul  3 05:37:12 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:37:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix test REQUIRES and options
	for aint.f90 (PR #146870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146870@github.com>
Message-ID: <68667978.050a0220.22c75f.db41@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: David Spickett (DavidSpickett)

<details>
<summary>Changes</summary>

This test should have been looking for the "x86" target, not "x86_64".

In the time it's not been running, -target must have been changed to -triple.

---
Full diff: https://github.com/llvm/llvm-project/pull/146870.diff


1 Files Affected:

- (modified) flang/test/Lower/math-lowering/aint.f90 (+4-4) 


``````````diff
diff --git a/flang/test/Lower/math-lowering/aint.f90 b/flang/test/Lower/math-lowering/aint.f90
index 6c7809f8ea1a3..b36fb421d7c8c 100644
--- a/flang/test/Lower/math-lowering/aint.f90
+++ b/flang/test/Lower/math-lowering/aint.f90
@@ -1,10 +1,10 @@
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
 
 function test_real4(x)
   real :: x, test_real4

``````````

</details>


https://github.com/llvm/llvm-project/pull/146870

From flang-commits at lists.llvm.org  Thu Jul  3 05:42:36 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:42:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
	sum.f90 (PR #146872)
Message-ID: <llvm.org/llvm/llvm-project/pull/146872@github.com>


https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/146872

This should have been looking for the "x86" target not "x64_64".

When run on AArch64 it failed because bbc tried to compile for AArch64. Add a target option to fix that, as this test is x86 specific.

>From 2f563f009d17c5e41442f0186b6a147b6d82e39c Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 12:40:24 +0000
Subject: [PATCH] [flang][test] Fix REQUIRES and options for sum.f90

This should have been looking for the "x86" target not "x64_64".

When run on AArch64 it failed because bbc tried to compile
for AArch64. Add a target option to fix that, as this test
is x86 specific.
---
 flang/test/Lower/Intrinsics/sum.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index d36b2b1263887..3167617b60457 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPsum_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32 {


From flang-commits at lists.llvm.org  Thu Jul  3 05:43:13 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 05:43:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
	sum.f90 (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <68667ae1.170a0220.2aaaf8.f531@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: David Spickett (DavidSpickett)

<details>
<summary>Changes</summary>

This should have been looking for the "x86" target not "x64_64".

When run on AArch64 it failed because bbc tried to compile for AArch64. Add a target option to fix that, as this test is x86 specific.

---
Full diff: https://github.com/llvm/llvm-project/pull/146872.diff


1 Files Affected:

- (modified) flang/test/Lower/Intrinsics/sum.f90 (+2-2) 


``````````diff
diff --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index d36b2b1263887..3167617b60457 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPsum_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32 {

``````````

</details>


https://github.com/llvm/llvm-project/pull/146872

From flang-commits at lists.llvm.org  Thu Jul  3 05:46:18 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:46:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
	sum.f90 (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <68667b9a.170a0220.9ecff.9dce@mx.google.com>


https://github.com/DavidSpickett updated https://github.com/llvm/llvm-project/pull/146872

>From 2f563f009d17c5e41442f0186b6a147b6d82e39c Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 12:40:24 +0000
Subject: [PATCH 1/2] [flang][test] Fix REQUIRES and options for sum.f90

This should have been looking for the "x86" target not "x64_64".

When run on AArch64 it failed because bbc tried to compile
for AArch64. Add a target option to fix that, as this test
is x86 specific.
---
 flang/test/Lower/Intrinsics/sum.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index d36b2b1263887..3167617b60457 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPsum_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32 {

>From b19991f4b6bcdb935d8a0e8ae475aa9f33368e35 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 12:46:02 +0000
Subject: [PATCH 2/2] product too

---
 flang/test/Lower/Intrinsics/product.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Lower/Intrinsics/product.f90 b/flang/test/Lower/Intrinsics/product.f90
index f5f7cff106cee..df7c1e4ce7eaa 100644
--- a/flang/test/Lower/Intrinsics/product.f90
+++ b/flang/test/Lower/Intrinsics/product.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPproduct_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32


From flang-commits at lists.llvm.org  Thu Jul  3 05:46:31 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:46:31 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
 sum.f90 and product.f90 (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <68667ba7.170a0220.ec273.5645@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/146872

From flang-commits at lists.llvm.org  Thu Jul  3 05:46:42 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:46:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
 sum.f90 and product.f90 (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <68667bb2.630a0220.2d9aaf.4b3d@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/146872

From flang-commits at lists.llvm.org  Thu Jul  3 05:46:53 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:46:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
 sum.f90 and product.f90 (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <68667bbd.050a0220.11c137.d857@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/146872

From flang-commits at lists.llvm.org  Thu Jul  3 05:50:59 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:50:59 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
 sum.f90 and product.f90 (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <68667cb3.170a0220.23f278.5a6a@mx.google.com>


https://github.com/DavidSpickett updated https://github.com/llvm/llvm-project/pull/146872

>From 2f563f009d17c5e41442f0186b6a147b6d82e39c Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 12:40:24 +0000
Subject: [PATCH 1/3] [flang][test] Fix REQUIRES and options for sum.f90

This should have been looking for the "x86" target not "x64_64".

When run on AArch64 it failed because bbc tried to compile
for AArch64. Add a target option to fix that, as this test
is x86 specific.
---
 flang/test/Lower/Intrinsics/sum.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index d36b2b1263887..3167617b60457 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPsum_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32 {

>From b19991f4b6bcdb935d8a0e8ae475aa9f33368e35 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 12:46:02 +0000
Subject: [PATCH 2/3] product too

---
 flang/test/Lower/Intrinsics/product.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Lower/Intrinsics/product.f90 b/flang/test/Lower/Intrinsics/product.f90
index f5f7cff106cee..df7c1e4ce7eaa 100644
--- a/flang/test/Lower/Intrinsics/product.f90
+++ b/flang/test/Lower/Intrinsics/product.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPproduct_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32

>From fa8c47d47ceb69c5ab8f7f976fef2adc350e7983 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 12:50:42 +0000
Subject: [PATCH 3/3] one more

---
 flang/test/Lower/complex-operations.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Lower/complex-operations.f90 b/flang/test/Lower/complex-operations.f90
index fdd13719c5b71..38954626e77ee 100644
--- a/flang/test/Lower/complex-operations.f90
+++ b/flang/test/Lower/complex-operations.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: @_QPadd_test
 subroutine add_test(a,b,c)


From flang-commits at lists.llvm.org  Thu Jul  3 05:51:17 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 05:51:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
 a few x86 specific tests (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <68667cc5.170a0220.10c2d8.96f1@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/146872

From flang-commits at lists.llvm.org  Thu Jul  3 06:04:16 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 06:04:16 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Enable kinds01/02/.f90 and
 update expected output (PR #146877)
Message-ID: <llvm.org/llvm/llvm-project/pull/146877@github.com>


https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/146877

These have not been run in a while because they required the "x86_64" target but it's not called that, it's just "x86".

After fixing that, 02 had some failures. I've just updated it to match, and reviewers will hopefully tell me if that's correct.

>From 1eca8a3de44ef91722a8e36fd9dfd7ce21bd44b6 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Thu, 3 Jul 2025 13:02:17 +0000
Subject: [PATCH] [flang][test] Enable kinds01/02/.f90 and update expected
 output

These have not been run in a while because they required the
"x86_64" target but it's not called that, it's just "x86".

After fixing that, 02 had some failures. I've just updated
it to match, and reviewers will hopefully tell me if that's correct.
---
 flang/test/Semantics/kinds01.f90 | 4 ++--
 flang/test/Semantics/kinds02.f90 | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/flang/test/Semantics/kinds01.f90 b/flang/test/Semantics/kinds01.f90
index 82c4d76da3710..ef822f5fbcbf0 100644
--- a/flang/test/Semantics/kinds01.f90
+++ b/flang/test/Semantics/kinds01.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: %python %S/test_symbols.py %s %flang_fc1
+! REQUIRES: x86-registered-target
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -triple x86_64-unknown-linux-gnu
  !DEF: /MainProgram1/jk1 ObjectEntity INTEGER(1)
  integer(kind=1) jk1
  !DEF: /MainProgram1/js1 ObjectEntity INTEGER(1)
diff --git a/flang/test/Semantics/kinds02.f90 b/flang/test/Semantics/kinds02.f90
index 02b1e6c8c310a..d018b69f81f76 100644
--- a/flang/test/Semantics/kinds02.f90
+++ b/flang/test/Semantics/kinds02.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: %python %S/test_errors.py %s %flang_fc1
+! REQUIRES: x86-registered-target
+! RUN: %python %S/test_errors.py %s %flang_fc1 -triple x86_64-unknown-linux-gnu
 ! C712 The value of scalar-int-constant-expr shall be nonnegative and 
 ! shall specify a representation method that exists on the processor.
 ! C714 The value of kind-param shall be nonnegative.
@@ -44,9 +44,9 @@
 complex(kind=32) :: z32
 !ERROR: COMPLEX*1 is not a supported type
 complex*1 :: zs1
-!ERROR: COMPLEX*2 is not a supported type
+!ERROR: COMPLEX(KIND=1) is not a supported type
 complex*2 :: zs2
-!ERROR: COMPLEX*64 is not a supported type
+!ERROR: COMPLEX(KIND=32) is not a supported type
 complex*64 :: zs64
 !ERROR: LOGICAL(KIND=0) is not a supported type
 logical(kind=0) :: l0


From flang-commits at lists.llvm.org  Thu Jul  3 06:04:24 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 06:04:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Enable kinds01/02.f90 and
 update expected output (PR #146877)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146877@github.com>
Message-ID: <68667fd8.050a0220.7d2ef.f0fc@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/146877

From flang-commits at lists.llvm.org  Thu Jul  3 06:04:49 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 06:04:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Enable kinds01/02.f90 and
 update expected output (PR #146877)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146877@github.com>
Message-ID: <68667ff1.050a0220.20d41f.fab3@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: David Spickett (DavidSpickett)

<details>
<summary>Changes</summary>

These have not been run in a while because they required the "x86_64" target but it's not called that, it's just "x86".

After fixing that, 02 had some failures. I've just updated it to match, and reviewers will hopefully tell me if that's correct.

---
Full diff: https://github.com/llvm/llvm-project/pull/146877.diff


2 Files Affected:

- (modified) flang/test/Semantics/kinds01.f90 (+2-2) 
- (modified) flang/test/Semantics/kinds02.f90 (+4-4) 


``````````diff
diff --git a/flang/test/Semantics/kinds01.f90 b/flang/test/Semantics/kinds01.f90
index 82c4d76da3710..ef822f5fbcbf0 100644
--- a/flang/test/Semantics/kinds01.f90
+++ b/flang/test/Semantics/kinds01.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: %python %S/test_symbols.py %s %flang_fc1
+! REQUIRES: x86-registered-target
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -triple x86_64-unknown-linux-gnu
  !DEF: /MainProgram1/jk1 ObjectEntity INTEGER(1)
  integer(kind=1) jk1
  !DEF: /MainProgram1/js1 ObjectEntity INTEGER(1)
diff --git a/flang/test/Semantics/kinds02.f90 b/flang/test/Semantics/kinds02.f90
index 02b1e6c8c310a..d018b69f81f76 100644
--- a/flang/test/Semantics/kinds02.f90
+++ b/flang/test/Semantics/kinds02.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: %python %S/test_errors.py %s %flang_fc1
+! REQUIRES: x86-registered-target
+! RUN: %python %S/test_errors.py %s %flang_fc1 -triple x86_64-unknown-linux-gnu
 ! C712 The value of scalar-int-constant-expr shall be nonnegative and 
 ! shall specify a representation method that exists on the processor.
 ! C714 The value of kind-param shall be nonnegative.
@@ -44,9 +44,9 @@
 complex(kind=32) :: z32
 !ERROR: COMPLEX*1 is not a supported type
 complex*1 :: zs1
-!ERROR: COMPLEX*2 is not a supported type
+!ERROR: COMPLEX(KIND=1) is not a supported type
 complex*2 :: zs2
-!ERROR: COMPLEX*64 is not a supported type
+!ERROR: COMPLEX(KIND=32) is not a supported type
 complex*64 :: zs64
 !ERROR: LOGICAL(KIND=0) is not a supported type
 logical(kind=0) :: l0

``````````

</details>


https://github.com/llvm/llvm-project/pull/146877

From flang-commits at lists.llvm.org  Thu Jul  3 06:05:04 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 03 Jul 2025 06:05:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Enable kinds01/02.f90 and
 update expected output (PR #146877)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146877@github.com>
Message-ID: <68668000.170a0220.1a3e8c.8e16@mx.google.com>


================
@@ -44,9 +44,9 @@
 complex(kind=32) :: z32
 !ERROR: COMPLEX*1 is not a supported type
 complex*1 :: zs1
-!ERROR: COMPLEX*2 is not a supported type
+!ERROR: COMPLEX(KIND=1) is not a supported type
 complex*2 :: zs2
-!ERROR: COMPLEX*64 is not a supported type
+!ERROR: COMPLEX(KIND=32) is not a supported type
----------------
DavidSpickett wrote:

This is what we get now, I don't know if that's actually correct though.

https://github.com/llvm/llvm-project/pull/146877

From flang-commits at lists.llvm.org  Thu Jul  3 07:01:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 07:01:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cmake] Separate FLANG_INCLUDE_TOOLS
 from FLANG_BUILD_TOOLS (PR #145005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145005@github.com>
Message-ID: <68668d55.170a0220.13186e.7749@mx.google.com>


parabola94 wrote:

> Thank you for the review! Could you merge this PR? It seems that I'm not granted to do that.

@mgorny @vzakhari would you mind if I ask you to do this?

https://github.com/llvm/llvm-project/pull/145005

From flang-commits at lists.llvm.org  Thu Jul  3 07:06:34 2025
From: flang-commits at lists.llvm.org (=?UTF-8?B?TWljaGHFgiBHw7Nybnk=?= via flang-commits)
Date: Thu, 03 Jul 2025 07:06:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cmake] Separate FLANG_INCLUDE_TOOLS
 from FLANG_BUILD_TOOLS (PR #145005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145005@github.com>
Message-ID: <68668e6a.170a0220.5cea8.a724@mx.google.com>


mgorny wrote:

Sure, sorry. I've started the workflows now, and will merge when they're done.

https://github.com/llvm/llvm-project/pull/145005

From flang-commits at lists.llvm.org  Thu Jul  3 07:33:49 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Thu, 03 Jul 2025 07:33:49 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686694cd.170a0220.37295a.9830@mx.google.com>


https://github.com/tarunprabhu edited https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 07:33:50 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Thu, 03 Jul 2025 07:33:50 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686694ce.170a0220.380180.a50e@mx.google.com>


================
@@ -484,6 +484,21 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
   }
 
   parseDoConcurrentMapping(opts, args, diags);
+
+  if (const auto *arg =
----------------
tarunprabhu wrote:

Could we use a concrete type instead of `auto` here?

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 07:33:50 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Thu, 03 Jul 2025 07:33:50 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686694ce.630a0220.3639aa.a0ad@mx.google.com>


https://github.com/tarunprabhu commented:

Thanks. I think we should share the `ComplexRangeKind` enum between `clang` and `flang`. I am ok with doing it in a separate PR.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 07:33:51 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Thu, 03 Jul 2025 07:33:51 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686694cf.a70a0220.31d786.6343@mx.google.com>


================
@@ -629,6 +655,20 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
     default:
       continue;
 
+    case options::OPT_fcomplex_arithmetic_EQ: {
----------------
tarunprabhu wrote:

I think this can be moved to `CommonArgs.cpp` as well.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 07:33:51 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Thu, 03 Jul 2025 07:33:51 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686694cf.170a0220.17e777.8f4b@mx.google.com>


================
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 #define FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 
+#include "flang/Frontend/CodeGenOptions.h"
----------------
tarunprabhu wrote:

I think the `CodeGenOptions.h` header is included because of the `ComplexRangeKind` enum. If that is moved to `llvm/Frontend/Driver/CodeGenOptions.h`, we don't have this issue. It may be worth doing it in this PR, but I am ok with moving it in a separate PR as well.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 07:33:51 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Thu, 03 Jul 2025 07:33:51 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686694cf.170a0220.279f7c.544b@mx.google.com>


================
@@ -192,6 +192,31 @@ class CodeGenOptions : public CodeGenOptionsBase {
     return getProfileUse() == llvm::driver::ProfileCSIRInstr;
   }
 
+  /// Controls the various implementations for complex division.
----------------
tarunprabhu wrote:

If this is exactly the same as the enum in `clang/Basic/LangOptions`, it could be moved to `llvm/Frontend/Driver/CodeGenOptions.h` and shared between clang and flang. There is precedence for this, most recently, [here](https://github.com/llvm/llvm-project/pull/142892).

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 07:33:52 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Thu, 03 Jul 2025 07:33:52 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686694d0.170a0220.b6d4b.9f30@mx.google.com>


================
@@ -609,6 +609,17 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return integerOverflowFlags;
   }
 
+  /// Set ComplexDivisionToRuntimeFlag value for whether complex number division
+  /// is lowered to a runtime function by this builder.
----------------
tarunprabhu wrote:

NIT: It's probably not necessary to repeat what the flag does here. If you would prefer to keep it, consider rephrasing

```suggestion
  /// Set ComplexDivisionToRuntimeFlag value. If set to true, complex number
  /// division is lowered to a runtime function by this builder.
```

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Thu Jul  3 07:45:50 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 07:45:50 -0700 (PDT)
Subject: [flang-commits] [flang] 2b49d36 - [flang][cmake] Separate
 FLANG_INCLUDE_TOOLS from FLANG_BUILD_TOOLS (#145005)
Message-ID: <6866979e.170a0220.e1fcc.b9ea@mx.google.com>


Author: parabola94
Date: 2025-07-03T16:45:47+02:00
New Revision: 2b49d36c081f4557cb17d176e17269c2c8843243

URL: https://github.com/llvm/llvm-project/commit/2b49d36c081f4557cb17d176e17269c2c8843243
DIFF: https://github.com/llvm/llvm-project/commit/2b49d36c081f4557cb17d176e17269c2c8843243.diff

LOG: [flang][cmake] Separate FLANG_INCLUDE_TOOLS from FLANG_BUILD_TOOLS (#145005)

If we disable `FLANG_BUILD_TOOLS`, not only building the tools but also
generating the targets for them is skipped now. On the other hand, llvm
separates them into `LLVM_BUILD_TOOLS` and `LLVM_INCLUDE_TOOLS`.
This patch introduces `FLANG_INCLUDE_TOOLS` for the distinction.

Added: 
    

Modified: 
    flang/CMakeLists.txt

Removed: 
    


################################################################################
diff  --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 068d134671db8..0bfada476348a 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -504,9 +504,10 @@ add_subdirectory(include)
 add_subdirectory(lib)
 add_subdirectory(cmake/modules)
 
+option(FLANG_INCLUDE_TOOLS "Generate build targets for the Flang tools." ON)
 option(FLANG_BUILD_TOOLS
   "Build the Flang tools. If OFF, just generate build targets." ON)
-if (FLANG_BUILD_TOOLS)
+if (FLANG_INCLUDE_TOOLS)
   add_subdirectory(tools)
 endif()
 


        

From flang-commits at lists.llvm.org  Thu Jul  3 07:45:53 2025
From: flang-commits at lists.llvm.org (=?UTF-8?B?TWljaGHFgiBHw7Nybnk=?= via flang-commits)
Date: Thu, 03 Jul 2025 07:45:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cmake] Separate FLANG_INCLUDE_TOOLS
 from FLANG_BUILD_TOOLS (PR #145005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145005@github.com>
Message-ID: <686697a1.630a0220.340a4b.b552@mx.google.com>


https://github.com/mgorny closed https://github.com/llvm/llvm-project/pull/145005

From flang-commits at lists.llvm.org  Thu Jul  3 07:46:11 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 07:46:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cmake] Separate FLANG_INCLUDE_TOOLS
 from FLANG_BUILD_TOOLS (PR #145005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145005@github.com>
Message-ID: <686697b3.170a0220.64104.ab89@mx.google.com>


github-actions[bot] wrote:

<!--LLVM BUILDBOT INFORMATION COMMENT-->

@parabola94 Congratulations on having your first Pull Request (PR) merged into the LLVM Project!

Your changes will be combined with recent changes from other authors, then tested by our [build bots](https://lab.llvm.org/buildbot/). If there is a problem with a build, you may receive a report in an email or a comment on this PR.

Please check whether problems have been caused by your change specifically, as the builds can include changes from many authors. It is not uncommon for your change to be included in a build that fails due to someone else's changes, or infrastructure issues.

How to do this, and the rest of the post-merge process, is covered in detail [here](https://llvm.org/docs/MyFirstTypoFix.html#myfirsttypofix-issues-after-landing-your-pr).

If your change does cause a problem, it may be reverted, or you can revert it yourself. This is a normal part of [LLVM development](https://llvm.org/docs/DeveloperPolicy.html#patch-reversion-policy). You can fix your changes and open a new PR to merge them again.

If you don't get any reports, no action is required from you. Your changes are working as expected, well done!


https://github.com/llvm/llvm-project/pull/145005

From flang-commits at lists.llvm.org  Thu Jul  3 08:36:39 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 08:36:39 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp_target_allocmem and omp_target_freemem fir ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6866a387.170a0220.a42d2.e21a@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions cpp -- flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp flang/lib/Optimizer/Dialect/FIROps.cpp mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index a04c5d7eb..14cc7bb51 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -142,8 +142,8 @@ static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
           /*isVarArg=*/false));
 }
 
-static mlir::Type
-convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
     return converter.convertBoxTypeAsStruct(boxTy);
   return converter.convertType(firType);
@@ -189,8 +189,9 @@ computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
 }
 
 static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
   return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
 }
 
@@ -224,8 +225,10 @@ genAllocationScaleSize(OP op, mlir::Type ity,
 }
 
 static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
-    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
-    mlir::Type ty, mlir::Value val, bool fold = false) {
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
   auto valTy = val.getType();
   // If the value was not yet lowered, lower its type so that it can
   // be used in getPrimitiveTypeSizeInBits.
@@ -261,11 +264,13 @@ struct TargetAllocMemOpConversion
     auto ity = lowerTy().indexType();
     mlir::Type dataTy = fir::unwrapRefType(heapTy);
     mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
-    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    mlir::Type llvmPtrTy =
+        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "omp.target_allocmem codegen of derived type with length "
                 "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy,
+                                          lowerTy().getDataLayout());
     if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands().drop_front())
@@ -281,7 +286,8 @@ struct TargetAllocMemOpConversion
         loc, llvmPtrTy,
         mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
         addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(
+        allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
     return mlir::success();
   }
 };

``````````

</details>


https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Thu Jul  3 08:45:37 2025
From: flang-commits at lists.llvm.org (Nick Sarnie via flang-commits)
Date: Thu, 03 Jul 2025 08:45:37 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <6866a5a1.050a0220.2ca3a0.a30e@mx.google.com>


================
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : CommaJoined<["--"], "offload-targets=">,
----------------
sarnex wrote:

I'm not sure if this is a realistic usecase or one we care about, but if we have a single option would we lose the ability to have different offload targets for different offloading languages at the same time? Like `-fopenmp -fopenmp-targets=amdgcn -fsycl  -fsycl-targets=nvptx64`. I guess if we really wanted to support that we could add some special optional syntax to offload-targets.

https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Thu Jul  3 08:59:08 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 08:59:08 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp_target_allocmem and omp_target_freemem fir ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6866a8cc.170a0220.5fb88.1da5@mx.google.com>


skc7 wrote:

> Thanks @skc7 for the PR! I have a few concerns about this approach, please let me know if I am missing anything:
> 
> 1. I am bit skeptical about leaking OpenMP specific ops to the `fir` dialect. The reason I see we have to do that is that we need to return a `fir_HeapType` from the `omp_target_allocmem` op, right? If there are not other reasons, why don't we introduce the op in the OpenMP dialect where it naturally fits, let it return an `Int64` type and use `fir.convert` to the target type?
> 2. In general though, why are we introducing a new op for the runtime API? Can't we directly emit these as call ops instead of having to introduce an op and then lower the op to a call? Are we planning to use these ops later somehow in some other way than lowering to the target call?
> 
> I am probably missing some context here though so my question might not be totally informed.

Hi @ergawy 

For implementing workdistribute lowering, in a new pass "lower-workdistribute" : [LINK](https://github.com/llvm/llvm-project/pull/140523), Need to move fir.allocmem nested in omp.target outside it (or hoist it outside omp.target)

Have updated the PR to implement the approach 1, to introduce omp.target_alloc_mem and omp.target_free_mem in openMP dialect. Also, their conversion to llvm ir has been implemented. 
But, inorder to properly calculate the size of allocation of fir types, had to implement TargetAllocMemOpConversion in CodeGenOpenMP, to lower omp.target_alloc_mem to LLVM dialect callOp of "omp_target_alloc". 

Approach 2, is to directly lower to a LLVM dialect, call to "omp_target_alloc".
Call to "omp_target_alloc" is to be made directly in the pass. It requires to calculate the actual size of allocation for fir type when it is lowered to llvm ir in the pass itself, which I think is bit complicated since it requires to move the utilities of size calculation (from Codegen fir to llvm) to with in the pass.

So, went ahead with approach 1. Please let me know your comments on this.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Thu Jul  3 09:08:42 2025
From: flang-commits at lists.llvm.org (Daniel Paoliello via flang-commits)
Date: Thu, 03 Jul 2025 09:08:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix x86 REQUIRES in a couple of
	tests (PR #146869)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146869@github.com>
Message-ID: <6866ab0a.050a0220.2c8367.7a14@mx.google.com>


https://github.com/dpaoliello approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/146869

From flang-commits at lists.llvm.org  Thu Jul  3 07:48:17 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Thu, 03 Jul 2025 07:48:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <68669831.170a0220.23ab8f.bb67@mx.google.com>


https://github.com/luporl updated https://github.com/llvm/llvm-project/pull/146408

>From c9a350f2c1099ba0c26539acf300b672f4b557a6 Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori at linaro.org>
Date: Mon, 9 Jun 2025 11:42:29 -0300
Subject: [PATCH 1/3] [flang] Optimize assignments of multidimensional arrays

Assignments of n-dimensional arrays, with trivial RHS, were
always being converted to n nested loops. For contiguous arrays,
it's possible to flatten them and use a single loop, that can
usually be better optimized by LLVM.

In a test program, using a 3-dimensional array and varying its
size, the resulting speedup was as follows (measured on Graviton4):

16K     1.09
64K     1.40
128K    1.90
256K    1.91
512K    1.00

For sizes above or equal to 512K no improvement was observed.
It looks like LLVM stops trying to perform aggressive loop
unrolling at a certain threshold and just uses nested loops
instead. Larger sizes won't fit on L1 and L2 caches too.

This was noticed while profiling 527.cam4_r. This optimization
makes aer_rad_props slightly faster, but unfortunately it
practically doesn't change 527.cam4_r total execution time.
---
 .../Transforms/OptimizedBufferization.cpp     | 43 ++++++++++++---
 flang/test/HLFIR/opt-scalar-assign.fir        | 53 ++++++++++++++-----
 2 files changed, 77 insertions(+), 19 deletions(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 91df8672c20d9..e88991b801415 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -786,13 +786,42 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   mlir::Value shape = hlfir::genShape(loc, builder, lhs);
   llvm::SmallVector<mlir::Value> extents =
       hlfir::getIndexExtents(loc, builder, shape);
-  hlfir::LoopNest loopNest =
-      hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
-                         flangomp::shouldUseWorkshareLowering(assign));
-  builder.setInsertionPointToStart(loopNest.body);
-  auto arrayElement =
-      hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-  builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+
+  if (lhs.isSimplyContiguous() && extents.size() > 1) {
+    // Flatten the array to use a single assign loop, that can be better
+    // optimized.
+    mlir::Value n = extents[0];
+    for (size_t i = 1; i < extents.size(); ++i)
+      n = builder.create<mlir::arith::MulIOp>(loc, n, extents[i]);
+    extents = {n};
+    shape = builder.genShape(loc, extents);
+    mlir::Type flatArrayType =
+        fir::ReferenceType::get(fir::SequenceType::get(eleTy, 1));
+    mlir::Value flatArray = lhs.getBase();
+    if (mlir::isa<fir::BoxType>(lhs.getType()))
+      flatArray = builder.create<fir::BoxAddrOp>(loc, flatArray);
+    flatArray = builder.createConvert(loc, flatArrayType, flatArray);
+
+    hlfir::LoopNest loopNest =
+        hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
+                           flangomp::shouldUseWorkshareLowering(assign));
+    builder.setInsertionPointToStart(loopNest.body);
+
+    mlir::Value coor = builder.create<fir::ArrayCoorOp>(
+        loc, fir::ReferenceType::get(eleTy), flatArray, shape,
+        /*slice=*/mlir::Value{}, loopNest.oneBasedIndices,
+        /*typeparams=*/mlir::ValueRange{});
+    builder.create<fir::StoreOp>(loc, rhs, coor);
+  } else {
+    hlfir::LoopNest loopNest =
+        hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
+                           flangomp::shouldUseWorkshareLowering(assign));
+    builder.setInsertionPointToStart(loopNest.body);
+    auto arrayElement =
+        hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
+    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+  }
+
   rewriter.eraseOp(assign);
   return mlir::success();
 }
diff --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 02ab02945b042..0f78d68f17ac8 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -12,18 +12,20 @@ func.func @_QPtest1() {
   return
 }
 // CHECK-LABEL:   func.func @_QPtest1() {
-// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[VAL_2:.*]] = arith.constant 11 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 13 : index
-// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
-// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_2]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_4]](%[[VAL_5]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
-// CHECK:           fir.do_loop %[[VAL_7:.*]] = %[[VAL_0]] to %[[VAL_3]] step %[[VAL_0]] unordered {
-// CHECK:             fir.do_loop %[[VAL_8:.*]] = %[[VAL_0]] to %[[VAL_2]] step %[[VAL_0]] unordered {
-// CHECK:               %[[VAL_9:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_8]], %[[VAL_7]])  : (!fir.ref<!fir.array<11x13xf32>>, index, index) -> !fir.ref<f32>
-// CHECK:               hlfir.assign %[[VAL_1]] to %[[VAL_9]] : f32, !fir.ref<f32>
-// CHECK:             }
+// CHECK:           %[[VAL_0:.*]] = arith.constant 143 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 11 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 13 : index
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_3]], %[[VAL_4]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
+
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.array<11x13xf32>>) -> !fir.ref<!fir.array<?xf32>>
+// CHECK:           fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_0]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_11:.*]] = fir.array_coor %[[VAL_9]](%[[VAL_8]]) %[[VAL_10]] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+// CHECK:             fir.store %[[VAL_2]] to %[[VAL_11]] : !fir.ref<f32>
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
@@ -129,3 +131,30 @@ func.func @_QPtest5(%arg0: !fir.ref<!fir.array<77xcomplex<f32>>> {fir.bindc_name
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @_QPtest6(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
+  %c0_i32 = arith.constant 0 : i32
+  %0:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+  hlfir.assign %c0_i32 to %0#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPtest6(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_1]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = arith.muli %[[VAL_6]]#1, %[[VAL_7]]#1 : index
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]] = fir.box_addr %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (!fir.heap<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+// CHECK:           fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_8]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_13:.*]] = fir.array_coor %[[VAL_11]](%[[VAL_9]]) %[[VAL_12]] : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
+// CHECK:             fir.store %[[VAL_3]] to %[[VAL_13]] : !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }

>From cc08a23fd65f7fcbb8597c499956f7045ea9f40b Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori at linaro.org>
Date: Wed, 2 Jul 2025 18:45:50 -0300
Subject: [PATCH 2/3] Use hlfir.designate and hlfir.assign

---
 .../Transforms/OptimizedBufferization.cpp     | 56 ++++++++++++++-----
 flang/test/HLFIR/opt-scalar-assign.fir        | 19 +++----
 2 files changed, 50 insertions(+), 25 deletions(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index e88991b801415..a6a2173af20db 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -21,6 +21,7 @@
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/HLFIR/Passes.h"
 #include "flang/Optimizer/OpenMP/Passes.h"
+#include "flang/Optimizer/Support/Utils.h"
 #include "flang/Optimizer/Transforms/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/Dominance.h"
@@ -758,6 +759,16 @@ class BroadcastAssignBufferization
                   mlir::PatternRewriter &rewriter) const override;
 };
 
+static bool isAllocatableArray(mlir::Type ty) {
+  auto boxTy = mlir::dyn_cast<fir::BoxType>(ty);
+  if (!boxTy)
+    return false;
+  auto heapTy = mlir::dyn_cast<fir::HeapType>(boxTy.getElementType());
+  if (!heapTy)
+    return false;
+  return mlir::isa<fir::SequenceType>(heapTy.getElementType());
+}
+
 llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
     hlfir::AssignOp assign, mlir::PatternRewriter &rewriter) const {
   // Since RHS is a scalar and LHS is an array, LHS must be allocated
@@ -787,31 +798,48 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   llvm::SmallVector<mlir::Value> extents =
       hlfir::getIndexExtents(loc, builder, shape);
 
-  if (lhs.isSimplyContiguous() && extents.size() > 1) {
+  bool isArrayRef =
+      mlir::isa<fir::SequenceType>(fir::unwrapRefType(lhs.getType()));
+  if (lhs.isSimplyContiguous() && extents.size() > 1 &&
+      (isArrayRef || isAllocatableArray(lhs.getType()))) {
     // Flatten the array to use a single assign loop, that can be better
     // optimized.
     mlir::Value n = extents[0];
     for (size_t i = 1; i < extents.size(); ++i)
       n = builder.create<mlir::arith::MulIOp>(loc, n, extents[i]);
-    extents = {n};
-    shape = builder.genShape(loc, extents);
-    mlir::Type flatArrayType =
-        fir::ReferenceType::get(fir::SequenceType::get(eleTy, 1));
+    llvm::SmallVector<mlir::Value> flatExtents = {n};
+
+    mlir::Type flatArrayType;
     mlir::Value flatArray = lhs.getBase();
-    if (mlir::isa<fir::BoxType>(lhs.getType()))
-      flatArray = builder.create<fir::BoxAddrOp>(loc, flatArray);
-    flatArray = builder.createConvert(loc, flatArrayType, flatArray);
+    if (isArrayRef) {
+      // Array references must have fixed shape, when used in assignments.
+      int64_t flatExtent = 1;
+      for (const mlir::Value &extent : extents) {
+        mlir::Operation *op = extent.getDefiningOp();
+        assert(op && "no defining operation for constant array extent");
+        flatExtent *= fir::toInt(mlir::cast<mlir::arith::ConstantOp>(*op));
+      }
+
+      flatArrayType =
+          fir::ReferenceType::get(fir::SequenceType::get({flatExtent}, eleTy));
+      flatArray = builder.createConvert(loc, flatArrayType, flatArray);
+    } else {
+      shape = builder.genShape(loc, flatExtents);
+      flatArrayType = fir::BoxType::get(
+          fir::HeapType::get(fir::SequenceType::get(eleTy, 1)));
+      flatArray = builder.create<fir::ReboxOp>(loc, flatArrayType, flatArray,
+                                               shape, /*slice=*/mlir::Value{});
+    }
 
     hlfir::LoopNest loopNest =
-        hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
+        hlfir::genLoopNest(loc, builder, flatExtents, /*isUnordered=*/true,
                            flangomp::shouldUseWorkshareLowering(assign));
     builder.setInsertionPointToStart(loopNest.body);
 
-    mlir::Value coor = builder.create<fir::ArrayCoorOp>(
-        loc, fir::ReferenceType::get(eleTy), flatArray, shape,
-        /*slice=*/mlir::Value{}, loopNest.oneBasedIndices,
-        /*typeparams=*/mlir::ValueRange{});
-    builder.create<fir::StoreOp>(loc, rhs, coor);
+    mlir::Value arrayElement =
+        builder.create<hlfir::DesignateOp>(loc, fir::ReferenceType::get(eleTy),
+                                           flatArray, loopNest.oneBasedIndices);
+    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
   } else {
     hlfir::LoopNest loopNest =
         hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
diff --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 0f78d68f17ac8..cbcdd579ff9e2 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -20,12 +20,10 @@ func.func @_QPtest1() {
 // CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
 // CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_3]], %[[VAL_4]] : (index, index) -> !fir.shape<2>
 // CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
-
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.array<11x13xf32>>) -> !fir.ref<!fir.array<?xf32>>
-// CHECK:           fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_0]] step %[[VAL_1]] unordered {
-// CHECK:             %[[VAL_11:.*]] = fir.array_coor %[[VAL_9]](%[[VAL_8]]) %[[VAL_10]] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
-// CHECK:             fir.store %[[VAL_2]] to %[[VAL_11]] : !fir.ref<f32>
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.array<11x13xf32>>) -> !fir.ref<!fir.array<143xf32>>
+// CHECK:           fir.do_loop %[[VAL_9:.*]] = %[[VAL_1]] to %[[VAL_0]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_10:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_9]]) : (!fir.ref<!fir.array<143xf32>>, index) -> !fir.ref<f32>
+// CHECK:             hlfir.assign %[[VAL_2]] to %[[VAL_10]] : f32, !fir.ref<f32>
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
@@ -150,11 +148,10 @@ func.func @_QPtest6(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {f
 // CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_1]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
 // CHECK:           %[[VAL_8:.*]] = arith.muli %[[VAL_6]]#1, %[[VAL_7]]#1 : index
 // CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_10:.*]] = fir.box_addr %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !fir.heap<!fir.array<?x?xi32>>
-// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (!fir.heap<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?xi32>>
-// CHECK:           fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_8]] step %[[VAL_1]] unordered {
-// CHECK:             %[[VAL_13:.*]] = fir.array_coor %[[VAL_11]](%[[VAL_9]]) %[[VAL_12]] : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
-// CHECK:             fir.store %[[VAL_3]] to %[[VAL_13]] : !fir.ref<i32>
+// CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_5]](%[[VAL_9]]) : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK:           fir.do_loop %[[VAL_11:.*]] = %[[VAL_1]] to %[[VAL_8]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_10]] (%[[VAL_11]]) : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_3]] to %[[VAL_12]] : i32, !fir.ref<i32>
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }

>From 2eb7b7e788be2d7b59ff7c58ec26087554c8a8eb Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori at linaro.org>
Date: Thu, 3 Jul 2025 11:34:13 -0300
Subject: [PATCH 3/3] Remoeve isAllocatableArray check

---
 .../Transforms/OptimizedBufferization.cpp     | 28 +++++--------------
 flang/test/HLFIR/opt-scalar-assign.fir        |  4 +--
 2 files changed, 9 insertions(+), 23 deletions(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index a6a2173af20db..54892ef99bf58 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -759,16 +759,6 @@ class BroadcastAssignBufferization
                   mlir::PatternRewriter &rewriter) const override;
 };
 
-static bool isAllocatableArray(mlir::Type ty) {
-  auto boxTy = mlir::dyn_cast<fir::BoxType>(ty);
-  if (!boxTy)
-    return false;
-  auto heapTy = mlir::dyn_cast<fir::HeapType>(boxTy.getElementType());
-  if (!heapTy)
-    return false;
-  return mlir::isa<fir::SequenceType>(heapTy.getElementType());
-}
-
 llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
     hlfir::AssignOp assign, mlir::PatternRewriter &rewriter) const {
   // Since RHS is a scalar and LHS is an array, LHS must be allocated
@@ -798,10 +788,7 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   llvm::SmallVector<mlir::Value> extents =
       hlfir::getIndexExtents(loc, builder, shape);
 
-  bool isArrayRef =
-      mlir::isa<fir::SequenceType>(fir::unwrapRefType(lhs.getType()));
-  if (lhs.isSimplyContiguous() && extents.size() > 1 &&
-      (isArrayRef || isAllocatableArray(lhs.getType()))) {
+  if (lhs.isSimplyContiguous() && extents.size() > 1) {
     // Flatten the array to use a single assign loop, that can be better
     // optimized.
     mlir::Value n = extents[0];
@@ -811,7 +798,12 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
 
     mlir::Type flatArrayType;
     mlir::Value flatArray = lhs.getBase();
-    if (isArrayRef) {
+    if (mlir::isa<fir::BoxType>(lhs.getType())) {
+      shape = builder.genShape(loc, flatExtents);
+      flatArrayType = fir::BoxType::get(fir::SequenceType::get(eleTy, 1));
+      flatArray = builder.create<fir::ReboxOp>(loc, flatArrayType, flatArray,
+                                               shape, /*slice=*/mlir::Value{});
+    } else {
       // Array references must have fixed shape, when used in assignments.
       int64_t flatExtent = 1;
       for (const mlir::Value &extent : extents) {
@@ -823,12 +815,6 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
       flatArrayType =
           fir::ReferenceType::get(fir::SequenceType::get({flatExtent}, eleTy));
       flatArray = builder.createConvert(loc, flatArrayType, flatArray);
-    } else {
-      shape = builder.genShape(loc, flatExtents);
-      flatArrayType = fir::BoxType::get(
-          fir::HeapType::get(fir::SequenceType::get(eleTy, 1)));
-      flatArray = builder.create<fir::ReboxOp>(loc, flatArrayType, flatArray,
-                                               shape, /*slice=*/mlir::Value{});
     }
 
     hlfir::LoopNest loopNest =
diff --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index cbcdd579ff9e2..74cdcd9622adb 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -148,9 +148,9 @@ func.func @_QPtest6(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {f
 // CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_1]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
 // CHECK:           %[[VAL_8:.*]] = arith.muli %[[VAL_6]]#1, %[[VAL_7]]#1 : index
 // CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_5]](%[[VAL_9]]) : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_5]](%[[VAL_9]]) : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
 // CHECK:           fir.do_loop %[[VAL_11:.*]] = %[[VAL_1]] to %[[VAL_8]] step %[[VAL_1]] unordered {
-// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_10]] (%[[VAL_11]]) : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_10]] (%[[VAL_11]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
 // CHECK:             hlfir.assign %[[VAL_3]] to %[[VAL_12]] : i32, !fir.ref<i32>
 // CHECK:           }
 // CHECK:           return


From flang-commits at lists.llvm.org  Thu Jul  3 07:51:42 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Thu, 03 Jul 2025 07:51:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <686698fe.170a0220.d7232.fc96@mx.google.com>


================
@@ -786,13 +797,59 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   mlir::Value shape = hlfir::genShape(loc, builder, lhs);
   llvm::SmallVector<mlir::Value> extents =
       hlfir::getIndexExtents(loc, builder, shape);
-  hlfir::LoopNest loopNest =
-      hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
-                         flangomp::shouldUseWorkshareLowering(assign));
-  builder.setInsertionPointToStart(loopNest.body);
-  auto arrayElement =
-      hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-  builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+
+  bool isArrayRef =
+      mlir::isa<fir::SequenceType>(fir::unwrapRefType(lhs.getType()));
+  if (lhs.isSimplyContiguous() && extents.size() > 1 &&
----------------
luporl wrote:

Done. Removing the `isAllocatableArray` check also made it easier to support `!fir.box<!fir.array>`.
Now all boxed arrays are converted to `!fir.box<!fir.array<?x type>>`

https://github.com/llvm/llvm-project/pull/146408

From flang-commits at lists.llvm.org  Thu Jul  3 08:34:07 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 08:34:07 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp_target_allocmem and omp_target_freemem fir ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6866a2ef.170a0220.384082.0ea5@mx.google.com>


https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/145464

>From 4915fb8f1d27847dc3d36899e233d5ac988f96c5 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 23 Jun 2025 16:39:55 +0530
Subject: [PATCH 1/4] [flang] Introduce omp_target_allocmem and
 omp_target_freemem fir ops.

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  58 ++++++++++
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +++++++++++++++++-
 2 files changed, 159 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 8ac847dd7dd0a..2dff0f05fade7 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,6 +517,64 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
+def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Creates a heap memory reference suitable for storing a value of the
+    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
+    The memory object is in an undefined state.  `omp_target_allocmem` operations must
+    be paired with `omp_target_freemem` operations to avoid memory leaks.
+
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyIntegerType>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyIntegerType>:$typeparams,
+    Variadic<AnyIntegerType>:$shape
+  );
+  let results = (outs fir_HeapType);
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+    bool hasLenParams() { return !getTypeparams().empty(); }
+    bool hasShapeOperands() { return !getShape().empty(); }
+    unsigned numLenParams() { return getTypeparams().size(); }
+    operand_range getLenParams() { return getTypeparams(); }
+    unsigned numShapeOperands() { return getShape().size(); }
+    operand_range getShapeOperands() { return getShape(); }
+    static mlir::Type getRefTy(mlir::Type ty);
+  }];
+}
+
+def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free a heap object";
+
+  let description = [{
+    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
+    The memory object that is deallocated is placed in an undefined state
+    after `fir.omp_target_freemem`.
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      ...
+      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyIntegerType, "", [MemFree]>:$device,
+  Arg<fir_HeapType, "", [MemFree]>:$heapref
+  );
+}
+
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..042ade6b1e0a1 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,6 +1168,105 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetAllocMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = heap.getType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
+    mlir::Location loc = heap.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(dataTy);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
+    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(loc, rewriter, mallocTy, size);
+    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+        heap, ::getLlvmPtrType(heap.getContext()),
+        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
+    return mlir::success();
+  }
+
+  /// Compute the allocation size in bytes of the element type of
+  /// \p llTy pointer type. The result is returned as a value of \p idxTy
+  /// integer type.
+  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy) const {
+    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+  }
+};
+} // namespace
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp freeFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
+    return freeFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_free",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMVoidType::get(module->getContext()),
+          {getLlvmPtrType(module->getContext()), i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetFreeMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
+    mlir::Location loc = freemem.getLoc();
+    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
+    rewriter.create<mlir::LLVM::CallOp>(
+        loc, mlir::TypeRange{},
+        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
+        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
+    rewriter.eraseOp(freemem);
+    return mlir::success();
+  }
+};
+} // namespace
+
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4274,7 +4373,8 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
+      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,

>From e41a2c76786538fc411e104542a0282cfebef4f7 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:20:36 +0530
Subject: [PATCH 2/4] [flang] Fix parsing and printing.

---
 .../include/flang/Optimizer/Dialect/FIROps.td | 13 ++-
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 90 ++++++++++++++++---
 flang/test/Fir/omp_target_allocmem.fir        | 28 ++++++
 flang/test/Fir/omp_target_freemem.fir         | 28 ++++++
 4 files changed, 145 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Fir/omp_target_allocmem.fir
 create mode 100644 flang/test/Fir/omp_target_freemem.fir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 2dff0f05fade7..666b66a8670d6 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -528,7 +528,8 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
     be paired with `omp_target_freemem` operations to avoid memory leaks.
 
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
     ```
   }];
 
@@ -542,6 +543,9 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
   );
   let results = (outs fir_HeapType);
 
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
   let extraClassDeclaration = [{
     mlir::Type getAllocatedType();
     bool hasLenParams() { return !getTypeparams().empty(); }
@@ -563,9 +567,9 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     The memory object that is deallocated is placed in an undefined state
     after `fir.omp_target_freemem`.
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
-      ...
-      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
     ```
   }];
 
@@ -573,6 +577,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   Arg<AnyIntegerType, "", [MemFree]>:$device,
   Arg<fir_HeapType, "", [MemFree]>:$heapref
   );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..9335a4b041ac8 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,24 +106,38 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca and Allocmem
-///
+/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
+/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
-                                            mlir::OpAsmParser &parser,
-                                            mlir::OperationState &result) {
+static mlir::ParseResult
+parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
+                   mlir::OperationState &result, bool isTargetOp = false) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+  // Parse device number as a new operand
+  if (isTargetOp) {
+    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+    mlir::Type deviceType;
+    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+      return mlir::failure();
+    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+      return mlir::failure();
+    if (parser.parseComma())
+      return mlir::failure();
+  }
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
-  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -147,13 +161,19 @@ static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
+
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
-                                                 {typeparamsSize, shapeSize}));
+  llvm::SmallVector<std::int32_t> segmentSizes;
+  if (isTargetOp)
+    segmentSizes.push_back(1);
+  segmentSizes.push_back(typeparamsSize);
+  segmentSizes.push_back(shapeSize);
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -385,6 +405,56 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// OmpTargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
+  return mlir::cast<fir::HeapType>(getType()).getEleTy();
+}
+
+mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
+  return fir::HeapType::get(ty);
+}
+
+mlir::ParseResult
+fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                mlir::OperationState &result) {
+  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
+}
+
+void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
+  llvm::SmallVector<llvm::StringRef> visited;
+  if (verifyInType(getInType(), visited, numShapeOperands()))
+    return emitOpError("invalid type for allocation");
+  if (verifyTypeParamCount(getInType(), numLenParams()))
+    return emitOpError("LEN params do not correspond to type");
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<fir::HeapType>(outType))
+    return emitOpError("must be a !fir.heap type");
+  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
+    return emitOpError("cannot allocate !fir.box of unknown rank or type");
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
new file mode 100644
index 0000000000000..5140c91c9510c
--- /dev/null
+++ b/flang/test/Fir/omp_target_allocmem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  return %1 : !fir.heap<!fir.array<3x3xi32>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_freemem.fir
new file mode 100644
index 0000000000000..02e136076a9cf
--- /dev/null
+++ b/flang/test/Fir/omp_target_freemem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  return
+}

>From a864094e5bd482499f82ef913c92c1f5b6bd070f Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:32:21 +0530
Subject: [PATCH 3/4] [flang] Fix doc in td

---
 flang/include/flang/Optimizer/Dialect/FIROps.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 666b66a8670d6..93d617027e30b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -560,7 +560,7 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
 
 def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object";
+  let summary = "free a heap object on an openmp device";
 
   let description = [{
     Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
@@ -569,7 +569,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     ```
       %device = arith.constant 0 : i32
       %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
     ```
   }];
 

>From 2f5b289464787d89ea00ab295ccb2ebee6b57329 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 12:22:55 +0530
Subject: [PATCH 4/4] [omp][mlir] Introduce TargetAllocMem and TargetFreeMem
 ops in openMP mlir dialect

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  63 -------
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +----------
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 161 ++++++++++++++++++
 flang/lib/Optimizer/Dialect/FIROps.cpp        |  88 +---------
 flang/test/Fir/omp_target_allocmem.fir        |  28 ---
 ...em.fir => omp_target_allocmem_freemem.fir} |  19 ++-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  64 +++++++
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 102 +++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  82 +++++++++
 .../ompenmp-target-allocmem-freemem.mlir      |  42 +++++
 10 files changed, 473 insertions(+), 278 deletions(-)
 delete mode 100644 flang/test/Fir/omp_target_allocmem.fir
 rename flang/test/Fir/{omp_target_freemem.fir => omp_target_allocmem_freemem.fir} (51%)
 create mode 100644 mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 93d617027e30b..8ac847dd7dd0a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,69 +517,6 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
-def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
-    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
-  let summary = "allocate storage on an openmp device for an object of a given type";
-
-  let description = [{
-    Creates a heap memory reference suitable for storing a value of the
-    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
-    The memory object is in an undefined state.  `omp_target_allocmem` operations must
-    be paired with `omp_target_freemem` operations to avoid memory leaks.
-
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-    ```
-  }];
-
-  let arguments = (ins
-    Arg<AnyIntegerType>:$device,
-    TypeAttr:$in_type,
-    OptionalAttr<StrAttr>:$uniq_name,
-    OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyIntegerType>:$typeparams,
-    Variadic<AnyIntegerType>:$shape
-  );
-  let results = (outs fir_HeapType);
-
-  let hasCustomAssemblyFormat = 1;
-  let hasVerifier = 1;
-
-  let extraClassDeclaration = [{
-    mlir::Type getAllocatedType();
-    bool hasLenParams() { return !getTypeparams().empty(); }
-    bool hasShapeOperands() { return !getShape().empty(); }
-    unsigned numLenParams() { return getTypeparams().size(); }
-    operand_range getLenParams() { return getTypeparams(); }
-    unsigned numShapeOperands() { return getShape().size(); }
-    operand_range getShapeOperands() { return getShape(); }
-    static mlir::Type getRefTy(mlir::Type ty);
-  }];
-}
-
-def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
-  [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object on an openmp device";
-
-  let description = [{
-    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
-    The memory object that is deallocated is placed in an undefined state
-    after `fir.omp_target_freemem`.
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
-    ```
-  }];
-
-  let arguments = (ins
-  Arg<AnyIntegerType, "", [MemFree]>:$device,
-  Arg<fir_HeapType, "", [MemFree]>:$heapref
-  );
-  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
-}
-
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 042ade6b1e0a1..a3de3ae9d116a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,105 +1168,6 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetAllocMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::Type heapTy = heap.getType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
-    mlir::Location loc = heap.getLoc();
-    auto ity = lowerTy().indexType();
-    mlir::Type dataTy = fir::unwrapRefType(heapTy);
-    mlir::Type llvmObjectTy = convertObjectType(dataTy);
-    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
-      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
-                "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
-      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands().drop_front())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
-    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
-    auto mallocTy =
-        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
-    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
-      size = integerCast(loc, rewriter, mallocTy, size);
-    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        heap, ::getLlvmPtrType(heap.getContext()),
-        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
-    return mlir::success();
-  }
-
-  /// Compute the allocation size in bytes of the element type of
-  /// \p llTy pointer type. The result is returned as a value of \p idxTy
-  /// integer type.
-  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
-  }
-};
-} // namespace
-
-static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp freeFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
-    return freeFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_free",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMVoidType::get(module->getContext()),
-          {getLlvmPtrType(module->getContext()), i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetFreeMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
-    mlir::Location loc = freemem.getLoc();
-    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
-    rewriter.create<mlir::LLVM::CallOp>(
-        loc, mlir::TypeRange{},
-        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
-        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
-    rewriter.eraseOp(freemem);
-    return mlir::success();
-  }
-};
-} // namespace
-
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4373,8 +4274,7 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
-      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 37f1c9f97e1ce..a04c5d7eb7ee7 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -125,10 +125,171 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type
+convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
+    mlir::Type ty, mlir::Value val, bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "omp.target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(lowerTy(), loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
+    allocmemOp->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    auto callOp = rewriter.create<mlir::LLVM::CallOp>(
+        loc, llvmPtrTy,
+        mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    return mlir::success();
+  }
+};
 } // namespace
 
 void fir::populateOpenMPFIRToLLVMConversionPatterns(
     const LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
   patterns.add<MapInfoOpConversion>(converter);
   patterns.add<PrivateClauseOpConversion>(converter);
+  patterns.add<TargetAllocMemOpConversion>(converter);
 }
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 9335a4b041ac8..f6c794b8fe9ae 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,38 +106,24 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// Parser shared by Alloca and Allocmem
 /// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
-/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
-///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
-///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult
-parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
-                   mlir::OperationState &result, bool isTargetOp = false) {
-  auto &builder = parser.getBuilder();
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
-  // Parse device number as a new operand
-  if (isTargetOp) {
-    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
-    mlir::Type deviceType;
-    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
-      return mlir::failure();
-    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
-      return mlir::failure();
-    if (parser.parseComma())
-      return mlir::failure();
-  }
+static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
+                                            mlir::OpAsmParser &parser,
+                                            mlir::OperationState &result) {
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
+  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -161,19 +147,13 @@ parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
-
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  llvm::SmallVector<std::int32_t> segmentSizes;
-  if (isTargetOp)
-    segmentSizes.push_back(1);
-  segmentSizes.push_back(typeparamsSize);
-  segmentSizes.push_back(shapeSize);
-  result.addAttribute("operandSegmentSizes",
-                      builder.getDenseI32ArrayAttr(segmentSizes));
+  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
+                                                 {typeparamsSize, shapeSize}));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -405,56 +385,6 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
-//===----------------------------------------------------------------------===//
-// OmpTargetAllocMemOp
-//===----------------------------------------------------------------------===//
-
-mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
-  return mlir::cast<fir::HeapType>(getType()).getEleTy();
-}
-
-mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
-  return fir::HeapType::get(ty);
-}
-
-mlir::ParseResult
-fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
-                                mlir::OperationState &result) {
-  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
-}
-
-void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
-  p << " ";
-  p.printOperand(getDevice());
-  p << " : ";
-  p << getDevice().getType();
-  p << ", ";
-  p << getInType();
-  if (!getTypeparams().empty()) {
-    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
-  }
-  for (auto sh : getShape()) {
-    p << ", ";
-    p.printOperand(sh);
-  }
-  p.printOptionalAttrDict((*this)->getAttrs(),
-                          {"in_type", "operandSegmentSizes"});
-}
-
-llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
-  llvm::SmallVector<llvm::StringRef> visited;
-  if (verifyInType(getInType(), visited, numShapeOperands()))
-    return emitOpError("invalid type for allocation");
-  if (verifyTypeParamCount(getInType(), numLenParams()))
-    return emitOpError("LEN params do not correspond to type");
-  mlir::Type outType = getType();
-  if (!mlir::dyn_cast<fir::HeapType>(outType))
-    return emitOpError("must be a !fir.heap type");
-  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
-    return emitOpError("cannot allocate !fir.box of unknown rank or type");
-  return mlir::success();
-}
-
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
deleted file mode 100644
index 5140c91c9510c..0000000000000
--- a/flang/test/Fir/omp_target_allocmem.fir
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
-// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
-func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  return %1 : !fir.heap<!fir.array<3x3xi32>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
-// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
-func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
-// CHECK-SAME: i32 %[[len:.*]])
-// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
-// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
-// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
-func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
-}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
similarity index 51%
rename from flang/test/Fir/omp_target_freemem.fir
rename to flang/test/Fir/omp_target_allocmem_freemem.fir
index 02e136076a9cf..9202202728454 100644
--- a/flang/test/Fir/omp_target_freemem.fir
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -1,28 +1,33 @@
 // RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_nonchar() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3xi32>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_char() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
 func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..bcc7c906041ac 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1887,4 +1887,68 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Allocates memory on the specified OpenMP device for an object of the given type.
+    Returns an integer value representing the device pointer to the allocated memory.
+    The memory is uninitialized after allocation. Operations must be paired with 
+    `omp.target_freemem` to avoid memory leaks.
+
+    ```mlir
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyInteger>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyInteger>:$typeparams,
+    Variadic<AnyInteger>:$shape
+  );
+  let results = (outs I64);
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// TargetFreeMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetFreeMemOp : OpenMP_Op<"target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free memory on an openmp device";
+
+  let description = [{
+    Deallocates memory on the specified OpenMP device that was previously 
+    allocated by an `omp.target_allocmem` operation. The memory is placed 
+    in an undefined state after deallocation.
+    ```
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+      omp.target_freemem %device, %ptr : i32, i64
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyInteger, "", [MemFree]>:$device,
+  Arg<I64, "", [MemFree]>:$heapref
+  );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..5e3ad6ccf3ffe 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3493,6 +3493,108 @@ LogicalResult ScanOp::verify() {
                    "reduction modifier");
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type omp::TargetAllocMemOp::getAllocatedType() {
+  return getInTypeAttr().getValue();
+}
+
+/// operation ::= %res = (`omp.target_alloc_mem`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
+static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+
+  // Parse device number as a new operand
+  mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+  mlir::Type deviceType;
+  if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+    return mlir::failure();
+  if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+    return mlir::failure();
+  if (parser.parseComma())
+    return mlir::failure();
+
+  mlir::Type intype;
+  if (parser.parseType(intype))
+    return mlir::failure();
+  result.addAttribute("in_type", mlir::TypeAttr::get(intype));
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
+  llvm::SmallVector<mlir::Type> typeVec;
+  if (!parser.parseOptionalLParen()) {
+    // parse the LEN params of the derived type. (<params> : <types>)
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
+        parser.parseColonTypeList(typeVec) || parser.parseRParen())
+      return mlir::failure();
+    typeparamsSize = operands.size();
+    hasOperands = true;
+  }
+  std::int32_t shapeSize = 0;
+  if (!parser.parseOptionalComma()) {
+    // parse size to scale by, vector of n dimensions of type index
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None))
+      return mlir::failure();
+    shapeSize = operands.size() - typeparamsSize;
+    auto idxTy = builder.getIndexType();
+    for (std::int32_t i = typeparamsSize, end = operands.size(); i != end; ++i)
+      typeVec.push_back(idxTy);
+    hasOperands = true;
+  }
+  if (hasOperands &&
+      parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
+                             result.operands))
+    return mlir::failure();
+
+  mlir::Type restype = builder.getIntegerType(64);
+  ;
+  if (!restype) {
+    parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
+    return mlir::failure();
+  }
+  llvm::SmallVector<std::int32_t> segmentSizes{1, typeparamsSize, shapeSize};
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
+  if (parser.parseOptionalAttrDict(result.attributes) ||
+      parser.addTypeToList(restype, result.types))
+    return mlir::failure();
+  return mlir::success();
+}
+
+mlir::ParseResult omp::TargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  return parseTargetAllocMemOp(parser, result);
+}
+
+void omp::TargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult omp::TargetAllocMemOp::verify() {
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<IntegerType>(outType))
+    return emitOpError("must be a integer type");
+  return mlir::success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index eece8573f00ec..63b41f3b13363 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5714,6 +5714,82 @@ static bool isTargetDeviceOp(Operation *op) {
   return false;
 }
 
+static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder,
+                                         llvm::Module *llvmModule) {
+  llvm::Type *i64Ty = builder.getInt64Ty();
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *returnType = builder.getPtrTy(0);
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(returnType, {i64Ty, i32Ty}, false);
+  llvm::Function *func = cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_alloc", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                        LLVM::ModuleTranslation &moduleTranslation) {
+  auto allocMemOp = cast<omp::TargetAllocMemOp>(opInst);
+  if (!allocMemOp)
+    return failure();
+
+  // Get "omp_target_alloc" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = allocMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the allocation size.
+  llvm::DataLayout dataLayout = llvmModule->getDataLayout();
+  mlir::Type heapTy = allocMemOp.getAllocatedType();
+  llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
+  llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
+  llvm::ConstantInt *allocSize = builder.getInt64(typeSize.getFixedValue());
+  // Create call to "omp_target_alloc" with the args as translated llvm values.
+  llvm::CallInst *call =
+      builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});
+  llvm::Value *resultI64 = builder.CreatePtrToInt(call, builder.getInt64Ty());
+
+  // Map the result
+  moduleTranslation.mapValue(allocMemOp.getResult(), resultI64);
+  return success();
+}
+
+static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder,
+                                        llvm::Module *llvmModule) {
+  llvm::Type *ptrTy = builder.getPtrTy(0);
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *voidTy = builder.getVoidTy();
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(voidTy, {ptrTy, i32Ty}, false);
+  llvm::Function *func = dyn_cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_free", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation) {
+  auto freeMemOp = cast<omp::TargetFreeMemOp>(opInst);
+  if (!freeMemOp)
+    return failure();
+
+  // Get "omp_target_free" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = freeMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the corresponding heapref value in llvm
+  mlir::Value heapref = freeMemOp.getHeapref();
+  llvm::Value *llvmHeapref = moduleTranslation.lookupValue(heapref);
+  // Convert heapref int to ptr and call "omp_target_free"
+  llvm::Value *intToPtr =
+      builder.CreateIntToPtr(llvmHeapref, builder.getPtrTy(0));
+  builder.CreateCall(ompTragetFreeFunc, {intToPtr, llvmDeviceNum});
+  return success();
+}
+
 /// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including
 /// OpenMP runtime calls).
 static LogicalResult
@@ -5871,6 +5947,12 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
                 // etc. and then discarded
                 return success();
               })
+          .Case([&](omp::TargetAllocMemOp) {
+            return convertTargetAllocMemOp(*op, builder, moduleTranslation);
+          })
+          .Case([&](omp::TargetFreeMemOp) {
+            return convertTargetFreeMemOp(*op, builder, moduleTranslation);
+          })
           .Default([&](Operation *inst) {
             return inst->emitError()
                    << "not yet implemented: " << inst->getName();
diff --git a/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
new file mode 100644
index 0000000000000..1bc97609ccff4
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt %s -convert-openmp-to-llvm | mlir-translate -mlir-to-llvmir | FileCheck %s
+
+// This file contains MLIR test cases for omp.target_allocmem and omp.target_freemem
+
+// CHECK-LABEL: test_alloc_free_i64
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 8, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_i64() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, i64
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_1d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 64, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_1d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_2d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 1024, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_2d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16x16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}


From flang-commits at lists.llvm.org  Thu Jul  3 08:36:32 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 08:36:32 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp_target_allocmem and omp_target_freemem fir ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6866a380.170a0220.161622.0c23@mx.google.com>


https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/145464

>From 4915fb8f1d27847dc3d36899e233d5ac988f96c5 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 23 Jun 2025 16:39:55 +0530
Subject: [PATCH 1/5] [flang] Introduce omp_target_allocmem and
 omp_target_freemem fir ops.

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  58 ++++++++++
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +++++++++++++++++-
 2 files changed, 159 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 8ac847dd7dd0a..2dff0f05fade7 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,6 +517,64 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
+def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Creates a heap memory reference suitable for storing a value of the
+    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
+    The memory object is in an undefined state.  `omp_target_allocmem` operations must
+    be paired with `omp_target_freemem` operations to avoid memory leaks.
+
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyIntegerType>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyIntegerType>:$typeparams,
+    Variadic<AnyIntegerType>:$shape
+  );
+  let results = (outs fir_HeapType);
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+    bool hasLenParams() { return !getTypeparams().empty(); }
+    bool hasShapeOperands() { return !getShape().empty(); }
+    unsigned numLenParams() { return getTypeparams().size(); }
+    operand_range getLenParams() { return getTypeparams(); }
+    unsigned numShapeOperands() { return getShape().size(); }
+    operand_range getShapeOperands() { return getShape(); }
+    static mlir::Type getRefTy(mlir::Type ty);
+  }];
+}
+
+def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free a heap object";
+
+  let description = [{
+    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
+    The memory object that is deallocated is placed in an undefined state
+    after `fir.omp_target_freemem`.
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      ...
+      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyIntegerType, "", [MemFree]>:$device,
+  Arg<fir_HeapType, "", [MemFree]>:$heapref
+  );
+}
+
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..042ade6b1e0a1 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,6 +1168,105 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetAllocMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = heap.getType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
+    mlir::Location loc = heap.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(dataTy);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
+    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(loc, rewriter, mallocTy, size);
+    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+        heap, ::getLlvmPtrType(heap.getContext()),
+        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
+    return mlir::success();
+  }
+
+  /// Compute the allocation size in bytes of the element type of
+  /// \p llTy pointer type. The result is returned as a value of \p idxTy
+  /// integer type.
+  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy) const {
+    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+  }
+};
+} // namespace
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp freeFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
+    return freeFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_free",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMVoidType::get(module->getContext()),
+          {getLlvmPtrType(module->getContext()), i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetFreeMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
+    mlir::Location loc = freemem.getLoc();
+    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
+    rewriter.create<mlir::LLVM::CallOp>(
+        loc, mlir::TypeRange{},
+        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
+        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
+    rewriter.eraseOp(freemem);
+    return mlir::success();
+  }
+};
+} // namespace
+
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4274,7 +4373,8 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
+      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,

>From e41a2c76786538fc411e104542a0282cfebef4f7 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:20:36 +0530
Subject: [PATCH 2/5] [flang] Fix parsing and printing.

---
 .../include/flang/Optimizer/Dialect/FIROps.td | 13 ++-
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 90 ++++++++++++++++---
 flang/test/Fir/omp_target_allocmem.fir        | 28 ++++++
 flang/test/Fir/omp_target_freemem.fir         | 28 ++++++
 4 files changed, 145 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Fir/omp_target_allocmem.fir
 create mode 100644 flang/test/Fir/omp_target_freemem.fir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 2dff0f05fade7..666b66a8670d6 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -528,7 +528,8 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
     be paired with `omp_target_freemem` operations to avoid memory leaks.
 
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
     ```
   }];
 
@@ -542,6 +543,9 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
   );
   let results = (outs fir_HeapType);
 
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
   let extraClassDeclaration = [{
     mlir::Type getAllocatedType();
     bool hasLenParams() { return !getTypeparams().empty(); }
@@ -563,9 +567,9 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     The memory object that is deallocated is placed in an undefined state
     after `fir.omp_target_freemem`.
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
-      ...
-      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
     ```
   }];
 
@@ -573,6 +577,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   Arg<AnyIntegerType, "", [MemFree]>:$device,
   Arg<fir_HeapType, "", [MemFree]>:$heapref
   );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..9335a4b041ac8 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,24 +106,38 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca and Allocmem
-///
+/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
+/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
-                                            mlir::OpAsmParser &parser,
-                                            mlir::OperationState &result) {
+static mlir::ParseResult
+parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
+                   mlir::OperationState &result, bool isTargetOp = false) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+  // Parse device number as a new operand
+  if (isTargetOp) {
+    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+    mlir::Type deviceType;
+    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+      return mlir::failure();
+    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+      return mlir::failure();
+    if (parser.parseComma())
+      return mlir::failure();
+  }
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
-  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -147,13 +161,19 @@ static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
+
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
-                                                 {typeparamsSize, shapeSize}));
+  llvm::SmallVector<std::int32_t> segmentSizes;
+  if (isTargetOp)
+    segmentSizes.push_back(1);
+  segmentSizes.push_back(typeparamsSize);
+  segmentSizes.push_back(shapeSize);
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -385,6 +405,56 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// OmpTargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
+  return mlir::cast<fir::HeapType>(getType()).getEleTy();
+}
+
+mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
+  return fir::HeapType::get(ty);
+}
+
+mlir::ParseResult
+fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                mlir::OperationState &result) {
+  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
+}
+
+void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
+  llvm::SmallVector<llvm::StringRef> visited;
+  if (verifyInType(getInType(), visited, numShapeOperands()))
+    return emitOpError("invalid type for allocation");
+  if (verifyTypeParamCount(getInType(), numLenParams()))
+    return emitOpError("LEN params do not correspond to type");
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<fir::HeapType>(outType))
+    return emitOpError("must be a !fir.heap type");
+  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
+    return emitOpError("cannot allocate !fir.box of unknown rank or type");
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
new file mode 100644
index 0000000000000..5140c91c9510c
--- /dev/null
+++ b/flang/test/Fir/omp_target_allocmem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  return %1 : !fir.heap<!fir.array<3x3xi32>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_freemem.fir
new file mode 100644
index 0000000000000..02e136076a9cf
--- /dev/null
+++ b/flang/test/Fir/omp_target_freemem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  return
+}

>From a864094e5bd482499f82ef913c92c1f5b6bd070f Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:32:21 +0530
Subject: [PATCH 3/5] [flang] Fix doc in td

---
 flang/include/flang/Optimizer/Dialect/FIROps.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 666b66a8670d6..93d617027e30b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -560,7 +560,7 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
 
 def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object";
+  let summary = "free a heap object on an openmp device";
 
   let description = [{
     Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
@@ -569,7 +569,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     ```
       %device = arith.constant 0 : i32
       %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
     ```
   }];
 

>From 2f5b289464787d89ea00ab295ccb2ebee6b57329 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 12:22:55 +0530
Subject: [PATCH 4/5] [omp][mlir] Introduce TargetAllocMem and TargetFreeMem
 ops in openMP mlir dialect

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  63 -------
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +----------
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 161 ++++++++++++++++++
 flang/lib/Optimizer/Dialect/FIROps.cpp        |  88 +---------
 flang/test/Fir/omp_target_allocmem.fir        |  28 ---
 ...em.fir => omp_target_allocmem_freemem.fir} |  19 ++-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  64 +++++++
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 102 +++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  82 +++++++++
 .../ompenmp-target-allocmem-freemem.mlir      |  42 +++++
 10 files changed, 473 insertions(+), 278 deletions(-)
 delete mode 100644 flang/test/Fir/omp_target_allocmem.fir
 rename flang/test/Fir/{omp_target_freemem.fir => omp_target_allocmem_freemem.fir} (51%)
 create mode 100644 mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 93d617027e30b..8ac847dd7dd0a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,69 +517,6 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
-def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
-    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
-  let summary = "allocate storage on an openmp device for an object of a given type";
-
-  let description = [{
-    Creates a heap memory reference suitable for storing a value of the
-    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
-    The memory object is in an undefined state.  `omp_target_allocmem` operations must
-    be paired with `omp_target_freemem` operations to avoid memory leaks.
-
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-    ```
-  }];
-
-  let arguments = (ins
-    Arg<AnyIntegerType>:$device,
-    TypeAttr:$in_type,
-    OptionalAttr<StrAttr>:$uniq_name,
-    OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyIntegerType>:$typeparams,
-    Variadic<AnyIntegerType>:$shape
-  );
-  let results = (outs fir_HeapType);
-
-  let hasCustomAssemblyFormat = 1;
-  let hasVerifier = 1;
-
-  let extraClassDeclaration = [{
-    mlir::Type getAllocatedType();
-    bool hasLenParams() { return !getTypeparams().empty(); }
-    bool hasShapeOperands() { return !getShape().empty(); }
-    unsigned numLenParams() { return getTypeparams().size(); }
-    operand_range getLenParams() { return getTypeparams(); }
-    unsigned numShapeOperands() { return getShape().size(); }
-    operand_range getShapeOperands() { return getShape(); }
-    static mlir::Type getRefTy(mlir::Type ty);
-  }];
-}
-
-def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
-  [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object on an openmp device";
-
-  let description = [{
-    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
-    The memory object that is deallocated is placed in an undefined state
-    after `fir.omp_target_freemem`.
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
-    ```
-  }];
-
-  let arguments = (ins
-  Arg<AnyIntegerType, "", [MemFree]>:$device,
-  Arg<fir_HeapType, "", [MemFree]>:$heapref
-  );
-  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
-}
-
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 042ade6b1e0a1..a3de3ae9d116a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,105 +1168,6 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetAllocMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::Type heapTy = heap.getType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
-    mlir::Location loc = heap.getLoc();
-    auto ity = lowerTy().indexType();
-    mlir::Type dataTy = fir::unwrapRefType(heapTy);
-    mlir::Type llvmObjectTy = convertObjectType(dataTy);
-    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
-      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
-                "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
-      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands().drop_front())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
-    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
-    auto mallocTy =
-        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
-    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
-      size = integerCast(loc, rewriter, mallocTy, size);
-    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        heap, ::getLlvmPtrType(heap.getContext()),
-        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
-    return mlir::success();
-  }
-
-  /// Compute the allocation size in bytes of the element type of
-  /// \p llTy pointer type. The result is returned as a value of \p idxTy
-  /// integer type.
-  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
-  }
-};
-} // namespace
-
-static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp freeFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
-    return freeFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_free",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMVoidType::get(module->getContext()),
-          {getLlvmPtrType(module->getContext()), i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetFreeMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
-    mlir::Location loc = freemem.getLoc();
-    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
-    rewriter.create<mlir::LLVM::CallOp>(
-        loc, mlir::TypeRange{},
-        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
-        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
-    rewriter.eraseOp(freemem);
-    return mlir::success();
-  }
-};
-} // namespace
-
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4373,8 +4274,7 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
-      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 37f1c9f97e1ce..a04c5d7eb7ee7 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -125,10 +125,171 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type
+convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
+    mlir::Type ty, mlir::Value val, bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "omp.target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(lowerTy(), loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
+    allocmemOp->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    auto callOp = rewriter.create<mlir::LLVM::CallOp>(
+        loc, llvmPtrTy,
+        mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    return mlir::success();
+  }
+};
 } // namespace
 
 void fir::populateOpenMPFIRToLLVMConversionPatterns(
     const LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
   patterns.add<MapInfoOpConversion>(converter);
   patterns.add<PrivateClauseOpConversion>(converter);
+  patterns.add<TargetAllocMemOpConversion>(converter);
 }
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 9335a4b041ac8..f6c794b8fe9ae 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,38 +106,24 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// Parser shared by Alloca and Allocmem
 /// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
-/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
-///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
-///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult
-parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
-                   mlir::OperationState &result, bool isTargetOp = false) {
-  auto &builder = parser.getBuilder();
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
-  // Parse device number as a new operand
-  if (isTargetOp) {
-    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
-    mlir::Type deviceType;
-    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
-      return mlir::failure();
-    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
-      return mlir::failure();
-    if (parser.parseComma())
-      return mlir::failure();
-  }
+static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
+                                            mlir::OpAsmParser &parser,
+                                            mlir::OperationState &result) {
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
+  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -161,19 +147,13 @@ parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
-
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  llvm::SmallVector<std::int32_t> segmentSizes;
-  if (isTargetOp)
-    segmentSizes.push_back(1);
-  segmentSizes.push_back(typeparamsSize);
-  segmentSizes.push_back(shapeSize);
-  result.addAttribute("operandSegmentSizes",
-                      builder.getDenseI32ArrayAttr(segmentSizes));
+  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
+                                                 {typeparamsSize, shapeSize}));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -405,56 +385,6 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
-//===----------------------------------------------------------------------===//
-// OmpTargetAllocMemOp
-//===----------------------------------------------------------------------===//
-
-mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
-  return mlir::cast<fir::HeapType>(getType()).getEleTy();
-}
-
-mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
-  return fir::HeapType::get(ty);
-}
-
-mlir::ParseResult
-fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
-                                mlir::OperationState &result) {
-  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
-}
-
-void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
-  p << " ";
-  p.printOperand(getDevice());
-  p << " : ";
-  p << getDevice().getType();
-  p << ", ";
-  p << getInType();
-  if (!getTypeparams().empty()) {
-    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
-  }
-  for (auto sh : getShape()) {
-    p << ", ";
-    p.printOperand(sh);
-  }
-  p.printOptionalAttrDict((*this)->getAttrs(),
-                          {"in_type", "operandSegmentSizes"});
-}
-
-llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
-  llvm::SmallVector<llvm::StringRef> visited;
-  if (verifyInType(getInType(), visited, numShapeOperands()))
-    return emitOpError("invalid type for allocation");
-  if (verifyTypeParamCount(getInType(), numLenParams()))
-    return emitOpError("LEN params do not correspond to type");
-  mlir::Type outType = getType();
-  if (!mlir::dyn_cast<fir::HeapType>(outType))
-    return emitOpError("must be a !fir.heap type");
-  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
-    return emitOpError("cannot allocate !fir.box of unknown rank or type");
-  return mlir::success();
-}
-
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
deleted file mode 100644
index 5140c91c9510c..0000000000000
--- a/flang/test/Fir/omp_target_allocmem.fir
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
-// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
-func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  return %1 : !fir.heap<!fir.array<3x3xi32>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
-// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
-func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
-// CHECK-SAME: i32 %[[len:.*]])
-// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
-// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
-// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
-func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
-}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
similarity index 51%
rename from flang/test/Fir/omp_target_freemem.fir
rename to flang/test/Fir/omp_target_allocmem_freemem.fir
index 02e136076a9cf..9202202728454 100644
--- a/flang/test/Fir/omp_target_freemem.fir
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -1,28 +1,33 @@
 // RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_nonchar() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3xi32>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_char() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
 func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..bcc7c906041ac 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1887,4 +1887,68 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Allocates memory on the specified OpenMP device for an object of the given type.
+    Returns an integer value representing the device pointer to the allocated memory.
+    The memory is uninitialized after allocation. Operations must be paired with 
+    `omp.target_freemem` to avoid memory leaks.
+
+    ```mlir
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyInteger>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyInteger>:$typeparams,
+    Variadic<AnyInteger>:$shape
+  );
+  let results = (outs I64);
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// TargetFreeMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetFreeMemOp : OpenMP_Op<"target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free memory on an openmp device";
+
+  let description = [{
+    Deallocates memory on the specified OpenMP device that was previously 
+    allocated by an `omp.target_allocmem` operation. The memory is placed 
+    in an undefined state after deallocation.
+    ```
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+      omp.target_freemem %device, %ptr : i32, i64
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyInteger, "", [MemFree]>:$device,
+  Arg<I64, "", [MemFree]>:$heapref
+  );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..5e3ad6ccf3ffe 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3493,6 +3493,108 @@ LogicalResult ScanOp::verify() {
                    "reduction modifier");
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type omp::TargetAllocMemOp::getAllocatedType() {
+  return getInTypeAttr().getValue();
+}
+
+/// operation ::= %res = (`omp.target_alloc_mem`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
+static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+
+  // Parse device number as a new operand
+  mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+  mlir::Type deviceType;
+  if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+    return mlir::failure();
+  if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+    return mlir::failure();
+  if (parser.parseComma())
+    return mlir::failure();
+
+  mlir::Type intype;
+  if (parser.parseType(intype))
+    return mlir::failure();
+  result.addAttribute("in_type", mlir::TypeAttr::get(intype));
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
+  llvm::SmallVector<mlir::Type> typeVec;
+  if (!parser.parseOptionalLParen()) {
+    // parse the LEN params of the derived type. (<params> : <types>)
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
+        parser.parseColonTypeList(typeVec) || parser.parseRParen())
+      return mlir::failure();
+    typeparamsSize = operands.size();
+    hasOperands = true;
+  }
+  std::int32_t shapeSize = 0;
+  if (!parser.parseOptionalComma()) {
+    // parse size to scale by, vector of n dimensions of type index
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None))
+      return mlir::failure();
+    shapeSize = operands.size() - typeparamsSize;
+    auto idxTy = builder.getIndexType();
+    for (std::int32_t i = typeparamsSize, end = operands.size(); i != end; ++i)
+      typeVec.push_back(idxTy);
+    hasOperands = true;
+  }
+  if (hasOperands &&
+      parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
+                             result.operands))
+    return mlir::failure();
+
+  mlir::Type restype = builder.getIntegerType(64);
+  ;
+  if (!restype) {
+    parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
+    return mlir::failure();
+  }
+  llvm::SmallVector<std::int32_t> segmentSizes{1, typeparamsSize, shapeSize};
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
+  if (parser.parseOptionalAttrDict(result.attributes) ||
+      parser.addTypeToList(restype, result.types))
+    return mlir::failure();
+  return mlir::success();
+}
+
+mlir::ParseResult omp::TargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  return parseTargetAllocMemOp(parser, result);
+}
+
+void omp::TargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult omp::TargetAllocMemOp::verify() {
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<IntegerType>(outType))
+    return emitOpError("must be a integer type");
+  return mlir::success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index eece8573f00ec..63b41f3b13363 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5714,6 +5714,82 @@ static bool isTargetDeviceOp(Operation *op) {
   return false;
 }
 
+static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder,
+                                         llvm::Module *llvmModule) {
+  llvm::Type *i64Ty = builder.getInt64Ty();
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *returnType = builder.getPtrTy(0);
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(returnType, {i64Ty, i32Ty}, false);
+  llvm::Function *func = cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_alloc", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                        LLVM::ModuleTranslation &moduleTranslation) {
+  auto allocMemOp = cast<omp::TargetAllocMemOp>(opInst);
+  if (!allocMemOp)
+    return failure();
+
+  // Get "omp_target_alloc" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = allocMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the allocation size.
+  llvm::DataLayout dataLayout = llvmModule->getDataLayout();
+  mlir::Type heapTy = allocMemOp.getAllocatedType();
+  llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
+  llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
+  llvm::ConstantInt *allocSize = builder.getInt64(typeSize.getFixedValue());
+  // Create call to "omp_target_alloc" with the args as translated llvm values.
+  llvm::CallInst *call =
+      builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});
+  llvm::Value *resultI64 = builder.CreatePtrToInt(call, builder.getInt64Ty());
+
+  // Map the result
+  moduleTranslation.mapValue(allocMemOp.getResult(), resultI64);
+  return success();
+}
+
+static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder,
+                                        llvm::Module *llvmModule) {
+  llvm::Type *ptrTy = builder.getPtrTy(0);
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *voidTy = builder.getVoidTy();
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(voidTy, {ptrTy, i32Ty}, false);
+  llvm::Function *func = dyn_cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_free", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation) {
+  auto freeMemOp = cast<omp::TargetFreeMemOp>(opInst);
+  if (!freeMemOp)
+    return failure();
+
+  // Get "omp_target_free" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = freeMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the corresponding heapref value in llvm
+  mlir::Value heapref = freeMemOp.getHeapref();
+  llvm::Value *llvmHeapref = moduleTranslation.lookupValue(heapref);
+  // Convert heapref int to ptr and call "omp_target_free"
+  llvm::Value *intToPtr =
+      builder.CreateIntToPtr(llvmHeapref, builder.getPtrTy(0));
+  builder.CreateCall(ompTragetFreeFunc, {intToPtr, llvmDeviceNum});
+  return success();
+}
+
 /// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including
 /// OpenMP runtime calls).
 static LogicalResult
@@ -5871,6 +5947,12 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
                 // etc. and then discarded
                 return success();
               })
+          .Case([&](omp::TargetAllocMemOp) {
+            return convertTargetAllocMemOp(*op, builder, moduleTranslation);
+          })
+          .Case([&](omp::TargetFreeMemOp) {
+            return convertTargetFreeMemOp(*op, builder, moduleTranslation);
+          })
           .Default([&](Operation *inst) {
             return inst->emitError()
                    << "not yet implemented: " << inst->getName();
diff --git a/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
new file mode 100644
index 0000000000000..1bc97609ccff4
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt %s -convert-openmp-to-llvm | mlir-translate -mlir-to-llvmir | FileCheck %s
+
+// This file contains MLIR test cases for omp.target_allocmem and omp.target_freemem
+
+// CHECK-LABEL: test_alloc_free_i64
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 8, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_i64() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, i64
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_1d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 64, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_1d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_2d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 1024, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_2d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16x16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}

>From 1e7a216d3d9e07146ebb364025983e0f5000eefa Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 21:05:35 +0530
Subject: [PATCH 5/5] Fix comments

---
 flang/lib/Optimizer/Dialect/FIROps.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index f6c794b8fe9ae..ba5c81d826e5e 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -107,7 +107,6 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
 }
 
 /// Parser shared by Alloca and Allocmem
-/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword


From flang-commits at lists.llvm.org  Thu Jul  3 09:05:27 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Thu, 03 Jul 2025 09:05:27 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <6866aa47.170a0220.3013d0.deb8@mx.google.com>


https://github.com/vzakhari approved this pull request.

Thank you! LGTM

https://github.com/llvm/llvm-project/pull/146408

From flang-commits at lists.llvm.org  Thu Jul  3 09:22:57 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Thu, 03 Jul 2025 09:22:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cmake] Separate FLANG_INCLUDE_TOOLS
 from FLANG_BUILD_TOOLS (PR #145005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145005@github.com>
Message-ID: <6866ae61.170a0220.128cac.2eb5@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `flang-arm64-windows-msvc` running on `linaro-armv8-windows-msvc-01` while building `flang` at step 7 "test-build-unified-tree-check-flang".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/207/builds/3321

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 7 (test-build-unified-tree-check-flang) failure: test (failure)
******************** TEST 'Flang :: Lower/OpenMP/wsloop-reduction-logical-kinds.f90' FAILED ********************
Exit Code: 2

Command Output (stdout):
--
# RUN: at line 1
c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe -fc1 -emit-hlfir -fopenmp -o - C:\Users\tcwg\llvm-worker\flang-arm64-windows-msvc\llvm-project\flang\test\Lower\OpenMP\wsloop-reduction-logical-kinds.f90 | c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\filecheck.exe C:\Users\tcwg\llvm-worker\flang-arm64-windows-msvc\llvm-project\flang\test\Lower\OpenMP\wsloop-reduction-logical-kinds.f90
# executed command: 'c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe' -fc1 -emit-hlfir -fopenmp -o - 'C:\Users\tcwg\llvm-worker\flang-arm64-windows-msvc\llvm-project\flang\test\Lower\OpenMP\wsloop-reduction-logical-kinds.f90'
# .---command stderr------------
# | PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
# | Stack dump:
# | 0.	Program arguments: c:\\users\\tcwg\\llvm-worker\\flang-arm64-windows-msvc\\build\\bin\\flang.exe -fc1 -emit-hlfir -fopenmp -o - C:\\Users\\tcwg\\llvm-worker\\flang-arm64-windows-msvc\\llvm-project\\flang\\test\\Lower\\OpenMP\\wsloop-reduction-logical-kinds.f90
# | Exception Code: 0xC0000005
# |  #0 0x00007ff6849f1d3c (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x1a91d3c)
# |  #1 0x00007ff6849efa24 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x1a8fa24)
# |  #2 0x00007ff6849ed8b8 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x1a8d8b8)
# |  #3 0x00007ff6849f35cc (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x1a935cc)
# |  #4 0x00007ff6849ed420 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x1a8d420)
# |  #5 0x00007ff68380c7a8 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x8ac7a8)
# |  #6 0x00007ff683855c08 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x8f5c08)
# |  #7 0x00007ff6838ff04c (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x99f04c)
# |  #8 0x00007ff683855044 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x8f5044)
# |  #9 0x00007ff682ff8ac0 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x98ac0)
# | #10 0x00007ff68300ea88 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0xaea88)
# | #11 0x00007ff682f634e8 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x34e8)
# | #12 0x00007ff682f620a4 (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x20a4)
# | #13 0x00007ff687c1dc30 mlir::detail::FallbackTypeIDResolver::registerImplicitTypeID(class llvm::StringRef) (c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\flang.exe+0x4cbdc30)
# | #14 0x2b08fff687c1dccc
# `-----------------------------
# error: command failed with exit status: 0xc0000005
# executed command: 'c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\filecheck.exe' 'C:\Users\tcwg\llvm-worker\flang-arm64-windows-msvc\llvm-project\flang\test\Lower\OpenMP\wsloop-reduction-logical-kinds.f90'
# .---command stderr------------
# | FileCheck error: '<stdin>' is empty.
# | FileCheck command line:  c:\users\tcwg\llvm-worker\flang-arm64-windows-msvc\build\bin\filecheck.exe C:\Users\tcwg\llvm-worker\flang-arm64-windows-msvc\llvm-project\flang\test\Lower\OpenMP\wsloop-reduction-logical-kinds.f90
# `-----------------------------
# error: command failed with exit status: 2

--

********************


```

</details>

https://github.com/llvm/llvm-project/pull/145005

From flang-commits at lists.llvm.org  Thu Jul  3 11:11:29 2025
From: flang-commits at lists.llvm.org (Daniel Paoliello via flang-commits)
Date: Thu, 03 Jul 2025 11:11:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Correctly handle
	-mframe-pointer=reserved (PR #146937)
Message-ID: <llvm.org/llvm/llvm-project/pull/146937@github.com>


https://github.com/dpaoliello created https://github.com/llvm/llvm-project/pull/146937

Fixes `#146802`

#146582 started using the `Reserved` Frame Pointer kind for Arm64 Windows, but this revealed a bug in Flang where it copied the `-mframe-pointer=reserved` flag from Clang, but didn't correctly handle it in its own command line parser and subsequent compilation pipeline.

This change adds support for `-mframe-pointer=reserved` and adds a test to make sure that functions are correctly marked when the flag is set.

>From 1cca70fa1b939b197137a7450b74e00fc5fa5970 Mon Sep 17 00:00:00 2001
From: Daniel Paoliello <danpao at microsoft.com>
Date: Thu, 3 Jul 2025 11:04:55 -0700
Subject: [PATCH] [flang] Correctly handle -mframe-pointer=reserved

---
 flang/lib/Frontend/CompilerInvocation.cpp | 3 ++-
 flang/lib/Optimizer/Passes/Pipelines.cpp  | 2 ++
 flang/test/Driver/func-attr.f90           | 2 ++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..c4d334c4b6de9 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -291,13 +291,14 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
 
   opts.AliasAnalysis = opts.OptimizationLevel > 0;
 
-  // -mframe-pointer=none/non-leaf/all option.
+  // -mframe-pointer=none/non-leaf/reserved/all option.
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_mframe_pointer_EQ)) {
     std::optional<llvm::FramePointerKind> val =
         llvm::StringSwitch<std::optional<llvm::FramePointerKind>>(a->getValue())
             .Case("none", llvm::FramePointerKind::None)
             .Case("non-leaf", llvm::FramePointerKind::NonLeaf)
+            .Case("reserved", llvm::FramePointerKind::Reserved)
             .Case("all", llvm::FramePointerKind::All)
             .Default(std::nullopt);
 
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 42d9e7ba2418f..dca176abe5ec5 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -358,6 +358,8 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::NonLeaf;
   else if (config.FramePointerKind == llvm::FramePointerKind::All)
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::All;
+  else if (config.FramePointerKind == llvm::FramePointerKind::Reserved)
+    framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::Reserved;
   else
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::None;
 
diff --git a/flang/test/Driver/func-attr.f90 b/flang/test/Driver/func-attr.f90
index 7442ac54dcaa5..e8cab4fca2226 100644
--- a/flang/test/Driver/func-attr.f90
+++ b/flang/test/Driver/func-attr.f90
@@ -4,6 +4,7 @@
 
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=none -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONEFP
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=non-leaf -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONLEAFFP
+! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=reserved -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-RESERVEDFP
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=all -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-ALLFP
 ! RUN: not %flang_fc1 -triple aarch64-none-none -mframe-pointer=wrongval -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-WRONGVALUEFP
 
@@ -20,6 +21,7 @@ end subroutine func
 
 ! CHECK-NONEFP-NOT: attributes #0 = { "frame-pointer"="{{.*}}" }
 ! CHECK-NONLEAFFP: attributes #0 = { "frame-pointer"="non-leaf" }
+! CHECK-RESERVEDFP: attributes #0 = { "frame-pointer"="reserved" }
 ! CHECK-ALLFP: attributes #0 = { "frame-pointer"="all" }
 
 ! CHECK-WRONGVALUEFP:error: invalid value 'wrongval' in '-mframe-pointer=wrongval'


From flang-commits at lists.llvm.org  Thu Jul  3 11:11:58 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 11:11:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Correctly handle
	-mframe-pointer=reserved (PR #146937)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146937@github.com>
Message-ID: <6866c7ee.170a0220.7903b.19cc@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Daniel Paoliello (dpaoliello)

<details>
<summary>Changes</summary>

Fixes `#<!-- -->146802`

#<!-- -->146582 started using the `Reserved` Frame Pointer kind for Arm64 Windows, but this revealed a bug in Flang where it copied the `-mframe-pointer=reserved` flag from Clang, but didn't correctly handle it in its own command line parser and subsequent compilation pipeline.

This change adds support for `-mframe-pointer=reserved` and adds a test to make sure that functions are correctly marked when the flag is set.

---
Full diff: https://github.com/llvm/llvm-project/pull/146937.diff


3 Files Affected:

- (modified) flang/lib/Frontend/CompilerInvocation.cpp (+2-1) 
- (modified) flang/lib/Optimizer/Passes/Pipelines.cpp (+2) 
- (modified) flang/test/Driver/func-attr.f90 (+2) 


``````````diff
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..c4d334c4b6de9 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -291,13 +291,14 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
 
   opts.AliasAnalysis = opts.OptimizationLevel > 0;
 
-  // -mframe-pointer=none/non-leaf/all option.
+  // -mframe-pointer=none/non-leaf/reserved/all option.
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_mframe_pointer_EQ)) {
     std::optional<llvm::FramePointerKind> val =
         llvm::StringSwitch<std::optional<llvm::FramePointerKind>>(a->getValue())
             .Case("none", llvm::FramePointerKind::None)
             .Case("non-leaf", llvm::FramePointerKind::NonLeaf)
+            .Case("reserved", llvm::FramePointerKind::Reserved)
             .Case("all", llvm::FramePointerKind::All)
             .Default(std::nullopt);
 
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 42d9e7ba2418f..dca176abe5ec5 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -358,6 +358,8 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::NonLeaf;
   else if (config.FramePointerKind == llvm::FramePointerKind::All)
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::All;
+  else if (config.FramePointerKind == llvm::FramePointerKind::Reserved)
+    framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::Reserved;
   else
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::None;
 
diff --git a/flang/test/Driver/func-attr.f90 b/flang/test/Driver/func-attr.f90
index 7442ac54dcaa5..e8cab4fca2226 100644
--- a/flang/test/Driver/func-attr.f90
+++ b/flang/test/Driver/func-attr.f90
@@ -4,6 +4,7 @@
 
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=none -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONEFP
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=non-leaf -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONLEAFFP
+! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=reserved -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-RESERVEDFP
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=all -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-ALLFP
 ! RUN: not %flang_fc1 -triple aarch64-none-none -mframe-pointer=wrongval -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-WRONGVALUEFP
 
@@ -20,6 +21,7 @@ end subroutine func
 
 ! CHECK-NONEFP-NOT: attributes #0 = { "frame-pointer"="{{.*}}" }
 ! CHECK-NONLEAFFP: attributes #0 = { "frame-pointer"="non-leaf" }
+! CHECK-RESERVEDFP: attributes #0 = { "frame-pointer"="reserved" }
 ! CHECK-ALLFP: attributes #0 = { "frame-pointer"="all" }
 
 ! CHECK-WRONGVALUEFP:error: invalid value 'wrongval' in '-mframe-pointer=wrongval'

``````````

</details>


https://github.com/llvm/llvm-project/pull/146937

From flang-commits at lists.llvm.org  Thu Jul  3 11:31:49 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Thu, 03 Jul 2025 11:31:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Correctly handle
	-mframe-pointer=reserved (PR #146937)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146937@github.com>
Message-ID: <6866cc95.170a0220.36e056.178e@mx.google.com>


https://github.com/tarunprabhu approved this pull request.

Thanks for this. This LGTM, but please wait for @DavidTruby

https://github.com/llvm/llvm-project/pull/146937

From flang-commits at lists.llvm.org  Thu Jul  3 12:36:36 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 12:36:36 -0700 (PDT)
Subject: [flang-commits] [flang] 67d6679 - [flang][prescanner] fix invalid
	check (#146613)
Message-ID: <6866dbc4.050a0220.2a764f.374f@mx.google.com>


Author: Andre Kuhlenschmidt
Date: 2025-07-03T12:36:34-07:00
New Revision: 67d6679c91e14e17981e4ea0340f91c79e9524da

URL: https://github.com/llvm/llvm-project/commit/67d6679c91e14e17981e4ea0340f91c79e9524da
DIFF: https://github.com/llvm/llvm-project/commit/67d6679c91e14e17981e4ea0340f91c79e9524da.diff

LOG: [flang][prescanner] fix invalid check (#146613)

`TokenSequence::pop_back()` had a check assumed that tokens are never
empty. Loosen this check since isn't true.

towards #146362

Added: 
    flang/test/Parser/issue-146362.1.f90

Modified: 
    flang/lib/Parser/token-sequence.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index 40a074eaf0a47..9deb513e4f64f 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -30,7 +30,8 @@ void TokenSequence::clear() {
 
 void TokenSequence::pop_back() {
   CHECK(!start_.empty());
-  CHECK(nextStart_ > start_.back());
+  // If the last token is empty then `nextStart_ == start_.back()`.
+  CHECK(nextStart_ >= start_.back());
   std::size_t bytes{nextStart_ - start_.back()};
   nextStart_ = start_.back();
   start_.pop_back();

diff  --git a/flang/test/Parser/issue-146362.1.f90 b/flang/test/Parser/issue-146362.1.f90
new file mode 100644
index 0000000000000..a37fd4b182a7f
--- /dev/null
+++ b/flang/test/Parser/issue-146362.1.f90
@@ -0,0 +1,5 @@
+! RUN: %flang_fc1 -fsyntax-only -cpp %s 2>&1
+#define UNITY(k) 1_ ## k
+PROGRAM REPRODUCER
+WRITE(*,*) UNITY(4)
+END PROGRAM REPRODUCER
\ No newline at end of file


        

From flang-commits at lists.llvm.org  Thu Jul  3 12:36:40 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Thu, 03 Jul 2025 12:36:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][prescanner] fix invalid check (PR
	#146613)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146613@github.com>
Message-ID: <6866dbc8.170a0220.185528.283b@mx.google.com>


https://github.com/akuhlens closed https://github.com/llvm/llvm-project/pull/146613

From flang-commits at lists.llvm.org  Thu Jul  3 12:36:51 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 12:36:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [acc][flang] Use SymbolRefAttr for
 func_name in ACC routine (PR #146951)
Message-ID: <llvm.org/llvm/llvm-project/pull/146951@github.com>


https://github.com/delaram-talaashrafi created https://github.com/llvm/llvm-project/pull/146951

Changed the type of the `func_name` attribute from SymbolNameAttr to SymbolRefAttr. SymbolNameAttr is typically used when defining a symbol (e.g., `sym_name`), while SymbolRefAttr is appropriate for referencing existing operations. This change ensures that MLIR can correctly track the link to the referenced `func.func` operation.

>From 6a28e458f6c7aaf703a2f6d78df39f9b339039b1 Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Thu, 3 Jul 2025 11:04:36 -0700
Subject: [PATCH] [acc][flang] Use SymbolRefAttr for func_name in ACC routine

Changed the type of the `func_name` attribute from SymbolNameAttr to SymbolRefAttr.
SymbolNameAttr is typically used when defining a symbol (e.g., `sym_name`), while
SymbolRefAttr is appropriate for referencing existing operations. This change ensures
that MLIR can correctly track the link to the referenced `func.func` operation.
---
 flang/lib/Lower/OpenACC.cpp                   |  7 +++++--
 .../mlir/Dialect/OpenACC/OpenACCOps.td        | 21 ++++++++-----------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 3ef3330cba2d6..e56d7f7ed9b6f 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4561,7 +4561,8 @@ void createOpenACCRoutineConstruct(
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes) {
 
   for (auto routineOp : mod.getOps<mlir::acc::RoutineOp>()) {
-    if (routineOp.getFuncName().str().compare(funcName) == 0) {
+    if (routineOp.getFuncName().getLeafReference().str().compare(funcName) ==
+        0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
       if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
@@ -4579,7 +4580,9 @@ void createOpenACCRoutineConstruct(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   modBuilder.create<mlir::acc::RoutineOp>(
-      loc, routineOpStr, funcName, getArrayAttrOrNull(builder, bindNames),
+      loc, routineOpStr,
+      mlir::SymbolRefAttr::get(builder.getContext(), funcName),
+      getArrayAttrOrNull(builder, bindNames),
       getArrayAttrOrNull(builder, bindNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 276b74bd43772..66378f116784e 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2771,18 +2771,15 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     must be modified as well.
   }];
 
-  let arguments = (ins SymbolNameAttr:$sym_name,
-                       SymbolNameAttr:$func_name,
-                       OptionalAttr<StrArrayAttr>:$bindName,
-                       OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
-                       OptionalAttr<DeviceTypeArrayAttr>:$worker,
-                       OptionalAttr<DeviceTypeArrayAttr>:$vector,
-                       OptionalAttr<DeviceTypeArrayAttr>:$seq,
-                       UnitAttr:$nohost,
-                       UnitAttr:$implicit,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gang,
-                       OptionalAttr<I64ArrayAttr>:$gangDim,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
+  let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
+      OptionalAttr<StrArrayAttr>:$bindName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$worker,
+      OptionalAttr<DeviceTypeArrayAttr>:$vector,
+      OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
+      UnitAttr:$implicit, OptionalAttr<DeviceTypeArrayAttr>:$gang,
+      OptionalAttr<I64ArrayAttr>:$gangDim,
+      OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
 
   let extraClassDeclaration = [{
     static StringRef getGangDimKeyword() { return "dim"; }


From flang-commits at lists.llvm.org  Thu Jul  3 12:37:09 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 12:37:09 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [acc][flang] Use SymbolRefAttr for
 func_name in ACC routine (PR #146951)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146951@github.com>
Message-ID: <6866dbe5.050a0220.10e156.2d35@mx.google.com>


github-actions[bot] wrote:

<!--LLVM NEW CONTRIBUTOR COMMENT-->

Thank you for submitting a Pull Request (PR) to the LLVM Project!

This PR will be automatically labeled and the relevant teams will be notified.

If you wish to, you can add reviewers by using the "Reviewers" section on this page.

If this is not working for you, it is probably because you do not have write permissions for the repository. In which case you can instead tag reviewers by name in a comment by using `@` followed by their GitHub username.

If you have received no comments on your PR for a week, you can request a review by "ping"ing the PR by adding a comment “Ping”. The common courtesy "ping" rate is once a week. Please remember that you are asking for valuable time from other developers.

If you have further questions, they may be answered by the [LLVM GitHub User Guide](https://llvm.org/docs/GitHub.html).

You can also ask questions in a comment on this PR, on the [LLVM Discord](https://discord.com/invite/xS7Z362) or on the [forums](https://discourse.llvm.org/).

https://github.com/llvm/llvm-project/pull/146951

From flang-commits at lists.llvm.org  Thu Jul  3 12:37:39 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 12:37:39 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [acc][flang] Use SymbolRefAttr for
 func_name in ACC routine (PR #146951)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146951@github.com>
Message-ID: <6866dc03.630a0220.19d87a.37ee@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: None (delaram-talaashrafi)

<details>
<summary>Changes</summary>

Changed the type of the `func_name` attribute from SymbolNameAttr to SymbolRefAttr. SymbolNameAttr is typically used when defining a symbol (e.g., `sym_name`), while SymbolRefAttr is appropriate for referencing existing operations. This change ensures that MLIR can correctly track the link to the referenced `func.func` operation.

---
Full diff: https://github.com/llvm/llvm-project/pull/146951.diff


2 Files Affected:

- (modified) flang/lib/Lower/OpenACC.cpp (+5-2) 
- (modified) mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td (+9-12) 


``````````diff
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 3ef3330cba2d6..e56d7f7ed9b6f 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4561,7 +4561,8 @@ void createOpenACCRoutineConstruct(
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes) {
 
   for (auto routineOp : mod.getOps<mlir::acc::RoutineOp>()) {
-    if (routineOp.getFuncName().str().compare(funcName) == 0) {
+    if (routineOp.getFuncName().getLeafReference().str().compare(funcName) ==
+        0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
       if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
@@ -4579,7 +4580,9 @@ void createOpenACCRoutineConstruct(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   modBuilder.create<mlir::acc::RoutineOp>(
-      loc, routineOpStr, funcName, getArrayAttrOrNull(builder, bindNames),
+      loc, routineOpStr,
+      mlir::SymbolRefAttr::get(builder.getContext(), funcName),
+      getArrayAttrOrNull(builder, bindNames),
       getArrayAttrOrNull(builder, bindNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 276b74bd43772..66378f116784e 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2771,18 +2771,15 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     must be modified as well.
   }];
 
-  let arguments = (ins SymbolNameAttr:$sym_name,
-                       SymbolNameAttr:$func_name,
-                       OptionalAttr<StrArrayAttr>:$bindName,
-                       OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
-                       OptionalAttr<DeviceTypeArrayAttr>:$worker,
-                       OptionalAttr<DeviceTypeArrayAttr>:$vector,
-                       OptionalAttr<DeviceTypeArrayAttr>:$seq,
-                       UnitAttr:$nohost,
-                       UnitAttr:$implicit,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gang,
-                       OptionalAttr<I64ArrayAttr>:$gangDim,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
+  let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
+      OptionalAttr<StrArrayAttr>:$bindName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$worker,
+      OptionalAttr<DeviceTypeArrayAttr>:$vector,
+      OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
+      UnitAttr:$implicit, OptionalAttr<DeviceTypeArrayAttr>:$gang,
+      OptionalAttr<I64ArrayAttr>:$gangDim,
+      OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
 
   let extraClassDeclaration = [{
     static StringRef getGangDimKeyword() { return "dim"; }

``````````

</details>


https://github.com/llvm/llvm-project/pull/146951

From flang-commits at lists.llvm.org  Thu Jul  3 12:37:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 12:37:57 -0700 (PDT)
Subject: [flang-commits] [flang] bc89380 - [flang][preprocessor] fix use of
 bitwise-and for logical-and (#146758)
Message-ID: <6866dc15.630a0220.1b98e0.3943@mx.google.com>


Author: Andre Kuhlenschmidt
Date: 2025-07-03T12:37:54-07:00
New Revision: bc89380179c4febb8d38948fdc71b9376ee1d6af

URL: https://github.com/llvm/llvm-project/commit/bc89380179c4febb8d38948fdc71b9376ee1d6af
DIFF: https://github.com/llvm/llvm-project/commit/bc89380179c4febb8d38948fdc71b9376ee1d6af.diff

LOG: [flang][preprocessor] fix use of bitwise-and for logical-and (#146758)

The preprocessor used bitwise and to implement logical, this is a bug.

towards #146362

Added: 
    flang/test/Parser/issue-146362.2.f90

Modified: 
    flang/lib/Parser/preprocessor.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index ef24c704db885..8ef9810463d5a 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -1276,16 +1276,20 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
       left = right >= 64 ? 0 : left >> right;
       break;
     case BITAND:
-    case AND:
       left = left & right;
       break;
     case BITXOR:
       left = left ^ right;
       break;
     case BITOR:
-    case OR:
       left = left | right;
       break;
+    case AND:
+      left = left && right;
+      break;
+    case OR:
+      left = left || right;
+      break;
     case LT:
       left = -(left < right);
       break;

diff  --git a/flang/test/Parser/issue-146362.2.f90 b/flang/test/Parser/issue-146362.2.f90
new file mode 100644
index 0000000000000..d9ad8c7d547f2
--- /dev/null
+++ b/flang/test/Parser/issue-146362.2.f90
@@ -0,0 +1,18 @@
+!RUN: %flang_fc1 -cpp -fdebug-unparse %s | FileCheck %s
+PROGRAM P
+#if (1 && 2)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+#if ((1 || 2) != 3)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+END PROGRAM
+


        

From flang-commits at lists.llvm.org  Thu Jul  3 12:38:01 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Thu, 03 Jul 2025 12:38:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][preprocessor] fix use of
 bitwise-and for logical-and (PR #146758)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146758@github.com>
Message-ID: <6866dc19.050a0220.2896c1.2ba9@mx.google.com>


https://github.com/akuhlens closed https://github.com/llvm/llvm-project/pull/146758

From flang-commits at lists.llvm.org  Thu Jul  3 12:55:56 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 03 Jul 2025 12:55:56 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][acc][flang] Use SymbolRefAttr
 for func_name in ACC routine (PR #146951)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146951@github.com>
Message-ID: <6866e04c.170a0220.1a83c4.2a24@mx.google.com>


https://github.com/clementval edited https://github.com/llvm/llvm-project/pull/146951

From flang-commits at lists.llvm.org  Thu Jul  3 12:56:03 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 03 Jul 2025 12:56:03 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][acc][flang] Use SymbolRefAttr
 for func_name in ACC routine (PR #146951)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146951@github.com>
Message-ID: <6866e053.170a0220.2b6fe5.2798@mx.google.com>


https://github.com/clementval approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/146951

From flang-commits at lists.llvm.org  Thu Jul  3 13:01:57 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Thu, 03 Jul 2025 13:01:57 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][acc][flang] Use SymbolRefAttr
 for func_name in ACC routine (PR #146951)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146951@github.com>
Message-ID: <6866e1b5.170a0220.fcddd.1fdb@mx.google.com>


https://github.com/razvanlupusoru approved this pull request.

This makes sense to me. Thank you for your contribution!

https://github.com/llvm/llvm-project/pull/146951

From flang-commits at lists.llvm.org  Thu Jul  3 14:32:41 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 14:32:41 -0700 (PDT)
Subject: [flang-commits] =?utf-8?q?=5Bflang=5D_dd3214d_-_=5Bflang=5D_Fix_h?=
 =?utf-8?q?andling_of_identifier_in_column_1_of_free_form_continuat?=
 =?utf-8?b?4oCmICgjMTQ2NDMwKQ==?=
Message-ID: <6866f6f9.630a0220.3c4bbe.3fe2@mx.google.com>


Author: Peter Klausler
Date: 2025-07-03T14:32:38-07:00
New Revision: dd3214d5a6b55c7ff515b1c795100ed627621dd4

URL: https://github.com/llvm/llvm-project/commit/dd3214d5a6b55c7ff515b1c795100ed627621dd4
DIFF: https://github.com/llvm/llvm-project/commit/dd3214d5a6b55c7ff515b1c795100ed627621dd4.diff

LOG: [flang] Fix handling of identifier in column 1 of free form continuat… (#146430)

…ion line

An obsolete flag ("insertASpace_") is being used to signal some cases in
the prescanner's implementation of continuation lines when a token
should be broken when it straddles a line break. It turns out that it's
sufficient to simply note these cases without ever actually inserting a
space, so don't do that (fixing the motivating bug). This leaves some
variables with obsolete names, so change them as well.

This patch handles the third of the three bugs reported in
https://github.com/llvm/llvm-project/issues/146362 .

Added: 
    flang/test/Preprocessing/bug1077.F90

Modified: 
    flang/lib/Parser/prescan.cpp
    flang/lib/Parser/prescan.h
    flang/test/Preprocessing/pp111.F90
    flang/test/Preprocessing/pp112.F90
    flang/test/Preprocessing/pp115.F90
    flang/test/Preprocessing/pp116.F90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index ed5184b0aa13d..ec894ab8513d2 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -593,13 +593,13 @@ bool Prescanner::SkipToNextSignificantCharacter() {
     return false;
   } else {
     auto anyContinuationLine{false};
-    bool mightNeedSpace{false};
+    bool atNewline{false};
     if (MustSkipToEndOfLine()) {
       SkipToEndOfLine();
     } else {
-      mightNeedSpace = *at_ == '\n';
+      atNewline = *at_ == '\n';
     }
-    for (; Continuation(mightNeedSpace); mightNeedSpace = false) {
+    for (; Continuation(atNewline); atNewline = false) {
       anyContinuationLine = true;
       ++continuationLines_;
       if (MustSkipToEndOfLine()) {
@@ -641,7 +641,7 @@ void Prescanner::SkipSpaces() {
   while (IsSpaceOrTab(at_)) {
     NextChar();
   }
-  insertASpace_ = false;
+  brokenToken_ = false;
 }
 
 const char *Prescanner::SkipWhiteSpace(const char *p) {
@@ -745,10 +745,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       }
     }
   }
-  if (insertASpace_) {
-    tokens.PutNextTokenChar(' ', spaceProvenance_);
-    insertASpace_ = false;
-  }
+  brokenToken_ = false;
   if (*at_ == '\n') {
     return false;
   }
@@ -808,7 +805,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
     bool anyDefined{false};
     bool hadContinuation{false};
     // Subtlety: When an identifier is split across continuation lines,
-    // its parts are kept as distinct pp-tokens if that macro replacement
+    // its parts are kept as distinct pp-tokens if macro replacement
     // should operate on them independently.  This trick accommodates the
     // historic practice of using line continuation for token pasting after
     // replacement.
@@ -822,6 +819,9 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       ++at_, ++column_;
       hadContinuation = SkipToNextSignificantCharacter();
       if (hadContinuation && IsLegalIdentifierStart(*at_)) {
+        if (brokenToken_) {
+          break;
+        }
         // Continued identifier
         tokens.CloseToken();
         ++parts;
@@ -1348,7 +1348,7 @@ bool Prescanner::SkipCommentLine(bool afterAmpersand) {
   return false;
 }
 
-const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
+const char *Prescanner::FixedFormContinuationLine(bool atNewline) {
   if (IsAtEnd()) {
     return nullptr;
   }
@@ -1381,8 +1381,8 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
       }
       const char *col6{nextLine_ + 5};
       if (*col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {
-        if (mightNeedSpace && !IsSpace(nextLine_ + 6)) {
-          insertASpace_ = true;
+        if (atNewline && !IsSpace(nextLine_ + 6)) {
+          brokenToken_ = true;
         }
         return nextLine_ + 6;
       }
@@ -1395,7 +1395,9 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
         nextLine_[4] == ' ' && IsCompilerDirectiveSentinel(&nextLine_[1], 1)) {
       if (const char *col6{nextLine_ + 5};
           *col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {
-        insertASpace_ |= mightNeedSpace && !IsSpace(nextLine_ + 6);
+        if (atNewline && !IsSpace(nextLine_ + 6)) {
+          brokenToken_ = true;
+        }
         return nextLine_ + 6;
       } else {
         return nullptr;
@@ -1464,7 +1466,7 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
     p = SkipWhiteSpace(p);
     if (*p == '&') {
       if (!ampersand) {
-        insertASpace_ = true;
+        brokenToken_ = true;
       }
       return p + 1;
     } else if (ampersand) {
@@ -1494,7 +1496,7 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
     } else if (p > lineStart && IsSpaceOrTab(p - 1)) {
       --p;
     } else {
-      insertASpace_ = true;
+      brokenToken_ = true;
     }
     return p;
   } else {
@@ -1502,14 +1504,14 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
   }
 }
 
-bool Prescanner::FixedFormContinuation(bool mightNeedSpace) {
+bool Prescanner::FixedFormContinuation(bool atNewline) {
   // N.B. We accept '&' as a continuation indicator in fixed form, too,
   // but not in a character literal.
   if (*at_ == '&' && inCharLiteral_) {
     return false;
   }
   do {
-    if (const char *cont{FixedFormContinuationLine(mightNeedSpace)}) {
+    if (const char *cont{FixedFormContinuationLine(atNewline)}) {
       BeginSourceLine(cont);
       column_ = 7;
       NextLine();

diff  --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index ec4c53cf3e0f2..f650d548e6eff 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -203,10 +203,10 @@ class Prescanner {
   std::optional<std::size_t> IsIncludeLine(const char *) const;
   void FortranInclude(const char *quote);
   const char *IsPreprocessorDirectiveLine(const char *) const;
-  const char *FixedFormContinuationLine(bool mightNeedSpace);
+  const char *FixedFormContinuationLine(bool atNewline);
   const char *FreeFormContinuationLine(bool ampersand);
   bool IsImplicitContinuation() const;
-  bool FixedFormContinuation(bool mightNeedSpace);
+  bool FixedFormContinuation(bool atNewline);
   bool FreeFormContinuation();
   bool Continuation(bool mightNeedFixedFormSpace);
   std::optional<LineClassification> IsFixedFormCompilerDirectiveLine(
@@ -256,10 +256,15 @@ class Prescanner {
   bool continuationInCharLiteral_{false};
   bool inPreprocessorDirective_{false};
 
-  // In some edge cases of compiler directive continuation lines, it
-  // is necessary to treat the line break as a space character by
-  // setting this flag, which is cleared by EmitChar().
-  bool insertASpace_{false};
+  // True after processing a continuation that can't be allowed
+  // to appear in the middle of an identifier token, but is fixed form,
+  // or is free form and doesn't have a space character handy to use as
+  // a separator when:
+  // a) (standard) doesn't begin with a leading '&' on the continuation
+  //     line, but has a non-blank in column 1, or
+  // b) (extension) does have a leading '&', but didn't have one
+  //    on the continued line.
+  bool brokenToken_{false};
 
   // When a free form continuation marker (&) appears at the end of a line
   // before a INCLUDE or #include, we delete it and omit the newline, so

diff  --git a/flang/test/Preprocessing/bug1077.F90 b/flang/test/Preprocessing/bug1077.F90
new file mode 100644
index 0000000000000..dd7391813a357
--- /dev/null
+++ b/flang/test/Preprocessing/bug1077.F90
@@ -0,0 +1,7 @@
+!RUN: %flang -E %s 2>&1 | FileCheck %s
+!CHECK: print *,((1)+(2)),4
+#define foo(x,y) ((x)+(y))
+print *,&
+foo(1,2)&
+,4
+end

diff  --git a/flang/test/Preprocessing/pp111.F90 b/flang/test/Preprocessing/pp111.F90
index 4da45ef35f5c0..bbf8709c3ab15 100644
--- a/flang/test/Preprocessing/pp111.F90
+++ b/flang/test/Preprocessing/pp111.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = IFLM(666)
 ! FLM call name split across continuation, no leading &, with & ! comment
       integer function IFLM(x)
         integer :: x

diff  --git a/flang/test/Preprocessing/pp112.F90 b/flang/test/Preprocessing/pp112.F90
index 16705527f68c3..a5244410f31af 100644
--- a/flang/test/Preprocessing/pp112.F90
+++ b/flang/test/Preprocessing/pp112.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = IFLM(666)
 ! ditto, but without & ! comment
       integer function IFLM(x)
         integer :: x

diff  --git a/flang/test/Preprocessing/pp115.F90 b/flang/test/Preprocessing/pp115.F90
index 4e4c621110ed8..eea42c53b936d 100644
--- a/flang/test/Preprocessing/pp115.F90
+++ b/flang/test/Preprocessing/pp115.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = ((666)+111)
 ! ditto, with & ! comment, no leading &
       integer function IFLM(x)
         integer :: x

diff  --git a/flang/test/Preprocessing/pp116.F90 b/flang/test/Preprocessing/pp116.F90
index e35a13cbf6489..39edf95763eab 100644
--- a/flang/test/Preprocessing/pp116.F90
+++ b/flang/test/Preprocessing/pp116.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = ((666)+111)
 ! FLM call split between name and (, no leading &
       integer function IFLM(x)
         integer :: x


        

From flang-commits at lists.llvm.org  Thu Jul  3 14:32:44 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Thu, 03 Jul 2025 14:32:44 -0700 (PDT)
Subject: [flang-commits] =?utf-8?q?=5Bflang=5D_=5Bflang=5D_Fix_handling_of?=
 =?utf-8?q?_identifier_in_column_1_of_free_form_continuat=E2=80=A6_=28PR_?=
 =?utf-8?q?=23146430=29?=
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146430@github.com>
Message-ID: <6866f6fc.630a0220.19e796.41da@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/146430

From flang-commits at lists.llvm.org  Thu Jul  3 14:34:19 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 14:34:19 -0700 (PDT)
Subject: [flang-commits] [flang] 2b7e3f6 - [flang] Unify derived types in
 distinct module files (#146759)
Message-ID: <6866f75b.170a0220.1bb57d.1340@mx.google.com>


Author: Peter Klausler
Date: 2025-07-03T14:34:16-07:00
New Revision: 2b7e3f6fa6fb1a45a2bf542788325db12666976e

URL: https://github.com/llvm/llvm-project/commit/2b7e3f6fa6fb1a45a2bf542788325db12666976e
DIFF: https://github.com/llvm/llvm-project/commit/2b7e3f6fa6fb1a45a2bf542788325db12666976e.diff

LOG: [flang] Unify derived types in distinct module files (#146759)

When using -fhermetic-module-files it's possible for a derived type to
have multiple distinct definition sites that are being compared for
being the same type, as in argument association. Accept them as being
the same type so long as they have the same names, the same module
names, and identical definitions.

Added: 
    flang/test/Semantics/bug1092.F90

Modified: 
    flang/lib/Evaluate/type.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Evaluate/type.cpp b/flang/lib/Evaluate/type.cpp
index 5b5f3c2cd0cf0..99dc8b1e5c676 100644
--- a/flang/lib/Evaluate/type.cpp
+++ b/flang/lib/Evaluate/type.cpp
@@ -299,13 +299,18 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &,
 
 // F2023 7.5.3.2
 static bool AreSameComponent(const semantics::Symbol &x,
-    const semantics::Symbol &y, bool ignoreSequence,
+    const semantics::Symbol &y, bool ignoreSequence, bool sameModuleName,
     SetOfDerivedTypePairs &inProgress) {
   if (x.attrs() != y.attrs()) {
     return false;
   }
-  if (x.attrs().test(semantics::Attr::PRIVATE)) {
-    return false;
+  if (x.attrs().test(semantics::Attr::PRIVATE) ||
+      y.attrs().test(semantics::Attr::PRIVATE)) {
+    if (!sameModuleName ||
+        x.attrs().test(semantics::Attr::PRIVATE) !=
+            y.attrs().test(semantics::Attr::PRIVATE)) {
+      return false;
+    }
   }
   if (x.size() && y.size()) {
     if (x.offset() != y.offset() || x.size() != y.size()) {
@@ -482,9 +487,20 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
           ySymbol.attrs().test(semantics::Attr::BIND_C)) {
     return false;
   }
-  if (!ignoreSequence && !(xDetails.sequence() && yDetails.sequence()) &&
-      !(xSymbol.attrs().test(semantics::Attr::BIND_C) &&
-          ySymbol.attrs().test(semantics::Attr::BIND_C))) {
+  bool sameModuleName{false};
+  const semantics::Scope &xOwner{xSymbol.owner()};
+  const semantics::Scope &yOwner{ySymbol.owner()};
+  if (xOwner.IsModule() && yOwner.IsModule()) {
+    if (auto xModuleName{xOwner.GetName()}) {
+      if (auto yModuleName{yOwner.GetName()}) {
+        if (*xModuleName == *yModuleName) {
+          sameModuleName = true;
+        }
+      }
+    }
+  }
+  if (!sameModuleName && !ignoreSequence && !xDetails.sequence() &&
+      !xSymbol.attrs().test(semantics::Attr::BIND_C)) {
     // PGI does not enforce this requirement; all other Fortran
     // compilers do with a hard error when violations are caught.
     return false;
@@ -502,9 +518,10 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
     const auto xLookup{xSymbol.scope()->find(*xComponentName)};
     const auto yLookup{ySymbol.scope()->find(*yComponentName)};
     if (xLookup == xSymbol.scope()->end() ||
-        yLookup == ySymbol.scope()->end() ||
-        !AreSameComponent(
-            *xLookup->second, *yLookup->second, ignoreSequence, inProgress)) {
+        yLookup == ySymbol.scope()->end()) {
+      return false;
+    } else if (!AreSameComponent(*xLookup->second, *yLookup->second,
+                   ignoreSequence, sameModuleName, inProgress)) {
       return false;
     }
   }
@@ -576,17 +593,15 @@ static bool AreCompatibleTypes(const DynamicType &x, const DynamicType &y,
     const auto yLen{y.knownLength()};
     return x.kind() == y.kind() &&
         (ignoreLengths || !xLen || !yLen || *xLen == *yLen);
-  } else if (x.category() != TypeCategory::Derived) {
-    if (x.IsTypelessIntrinsicArgument()) {
-      return y.IsTypelessIntrinsicArgument();
-    } else {
-      return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
-    }
-  } else {
+  } else if (x.category() == TypeCategory::Derived) {
     const auto *xdt{GetDerivedTypeSpec(x)};
     const auto *ydt{GetDerivedTypeSpec(y)};
     return AreCompatibleDerivedTypes(
         xdt, ydt, x.IsPolymorphic(), ignoreTypeParameterValues, false);
+  } else if (x.IsTypelessIntrinsicArgument()) {
+    return y.IsTypelessIntrinsicArgument();
+  } else {
+    return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
   }
 }
 

diff  --git a/flang/test/Semantics/bug1092.F90 b/flang/test/Semantics/bug1092.F90
new file mode 100644
index 0000000000000..ef108733fd78b
--- /dev/null
+++ b/flang/test/Semantics/bug1092.F90
@@ -0,0 +1,26 @@
+!RUN: rm -rf %t && mkdir -p %t
+!RUN: %flang_fc1 -DWHICH=1 -fsyntax-only -J%t %s
+!RUN: %flang_fc1 -DWHICH=2 -fsyntax-only -fhermetic-module-files -I%t -J%t %s
+!RUN: %flang_fc1 -fsyntax-only -I%t %s 2>&1 | FileCheck --allow-empty %s
+!CHECK-NOT: error:
+
+#if WHICH == 1
+module bug1092a
+  type t
+  end type
+ contains
+  subroutine subr(x)
+    type(t) x
+  end
+end
+#elif WHICH == 2
+module bug1092b
+  use bug1092a, only: subr
+end
+#else
+use bug1092a, only: t
+use bug1092b, only: subr
+type(t) x
+call subr(x)
+end
+#endif


        

From flang-commits at lists.llvm.org  Thu Jul  3 14:34:22 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Thu, 03 Jul 2025 14:34:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Unify derived types in distinct
	module files (PR #146759)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146759@github.com>
Message-ID: <6866f75e.630a0220.3947a9.3b50@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/146759

From flang-commits at lists.llvm.org  Thu Jul  3 14:55:52 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 14:55:52 -0700 (PDT)
Subject: [flang-commits] [flang] 1f7effc - [mlir][acc][flang] Use
 SymbolRefAttr for func_name in ACC routine (#146951)
Message-ID: <6866fc68.a70a0220.25c452.3b0b@mx.google.com>


Author: delaram-talaashrafi
Date: 2025-07-03T14:55:49-07:00
New Revision: 1f7effc88774eee0c957f0aa6c5092b5dae93069

URL: https://github.com/llvm/llvm-project/commit/1f7effc88774eee0c957f0aa6c5092b5dae93069
DIFF: https://github.com/llvm/llvm-project/commit/1f7effc88774eee0c957f0aa6c5092b5dae93069.diff

LOG: [mlir][acc][flang] Use SymbolRefAttr for func_name in ACC routine (#146951)

Changed the type of the `func_name` attribute from SymbolNameAttr to
SymbolRefAttr. SymbolNameAttr is typically used when defining a symbol
(e.g., `sym_name`), while SymbolRefAttr is appropriate for referencing
existing operations. This change ensures that MLIR can correctly track
the link to the referenced `func.func` operation.

Added: 
    

Modified: 
    flang/lib/Lower/OpenACC.cpp
    mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 3ef3330cba2d6..e56d7f7ed9b6f 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4561,7 +4561,8 @@ void createOpenACCRoutineConstruct(
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes) {
 
   for (auto routineOp : mod.getOps<mlir::acc::RoutineOp>()) {
-    if (routineOp.getFuncName().str().compare(funcName) == 0) {
+    if (routineOp.getFuncName().getLeafReference().str().compare(funcName) ==
+        0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
       if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
@@ -4579,7 +4580,9 @@ void createOpenACCRoutineConstruct(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   modBuilder.create<mlir::acc::RoutineOp>(
-      loc, routineOpStr, funcName, getArrayAttrOrNull(builder, bindNames),
+      loc, routineOpStr,
+      mlir::SymbolRefAttr::get(builder.getContext(), funcName),
+      getArrayAttrOrNull(builder, bindNames),
       getArrayAttrOrNull(builder, bindNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),

diff  --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 276b74bd43772..66378f116784e 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2771,18 +2771,15 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     must be modified as well.
   }];
 
-  let arguments = (ins SymbolNameAttr:$sym_name,
-                       SymbolNameAttr:$func_name,
-                       OptionalAttr<StrArrayAttr>:$bindName,
-                       OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
-                       OptionalAttr<DeviceTypeArrayAttr>:$worker,
-                       OptionalAttr<DeviceTypeArrayAttr>:$vector,
-                       OptionalAttr<DeviceTypeArrayAttr>:$seq,
-                       UnitAttr:$nohost,
-                       UnitAttr:$implicit,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gang,
-                       OptionalAttr<I64ArrayAttr>:$gangDim,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
+  let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
+      OptionalAttr<StrArrayAttr>:$bindName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$worker,
+      OptionalAttr<DeviceTypeArrayAttr>:$vector,
+      OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
+      UnitAttr:$implicit, OptionalAttr<DeviceTypeArrayAttr>:$gang,
+      OptionalAttr<I64ArrayAttr>:$gangDim,
+      OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
 
   let extraClassDeclaration = [{
     static StringRef getGangDimKeyword() { return "dim"; }


        

From flang-commits at lists.llvm.org  Thu Jul  3 14:55:55 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Thu, 03 Jul 2025 14:55:55 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][acc][flang] Use SymbolRefAttr
 for func_name in ACC routine (PR #146951)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146951@github.com>
Message-ID: <6866fc6b.050a0220.fb05f.3947@mx.google.com>


https://github.com/razvanlupusoru closed https://github.com/llvm/llvm-project/pull/146951

From flang-commits at lists.llvm.org  Thu Jul  3 14:56:11 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 14:56:11 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][acc][flang] Use SymbolRefAttr
 for func_name in ACC routine (PR #146951)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146951@github.com>
Message-ID: <6866fc7b.050a0220.35b89e.3003@mx.google.com>


github-actions[bot] wrote:

<!--LLVM BUILDBOT INFORMATION COMMENT-->

@delaram-talaashrafi Congratulations on having your first Pull Request (PR) merged into the LLVM Project!

Your changes will be combined with recent changes from other authors, then tested by our [build bots](https://lab.llvm.org/buildbot/). If there is a problem with a build, you may receive a report in an email or a comment on this PR.

Please check whether problems have been caused by your change specifically, as the builds can include changes from many authors. It is not uncommon for your change to be included in a build that fails due to someone else's changes, or infrastructure issues.

How to do this, and the rest of the post-merge process, is covered in detail [here](https://llvm.org/docs/MyFirstTypoFix.html#myfirsttypofix-issues-after-landing-your-pr).

If your change does cause a problem, it may be reverted, or you can revert it yourself. This is a normal part of [LLVM development](https://llvm.org/docs/DeveloperPolicy.html#patch-reversion-policy). You can fix your changes and open a new PR to merge them again.

If you don't get any reports, no action is required from you. Your changes are working as expected, well done!


https://github.com/llvm/llvm-project/pull/146951

From flang-commits at lists.llvm.org  Thu Jul  3 15:00:43 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Thu, 03 Jul 2025 15:00:43 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <6866fd8b.630a0220.1a4836.3b47@mx.google.com>


================
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : CommaJoined<["--"], "offload-targets=">,
----------------
jhuber6 wrote:

Yeah, I'm not sure if it will ever make sense to mix languages. The offloading kind is a function of the input, though that's muddled by arguments like `-fopenmp` and `-fsycl` which 'pretend' it's a flag when really it's like putting `-x c` in front of every input. *theoretically* we could have something similar like `-Xlang_sycl --offload-target=abc`,but I really don't know if anyone will ever care about that.

https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Thu Jul  3 15:44:57 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Thu, 03 Jul 2025 15:44:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add general symbol dependence
	collection utility (PR #146968)
Message-ID: <llvm.org/llvm/llvm-project/pull/146968@github.com>


https://github.com/klausler created https://github.com/llvm/llvm-project/pull/146968

Replace HarvestSymbolsNeededFromOtherModules() in mod-file.cpp with a general utility function in Semantics.  This new code will find other uses in further rework of hermetic module file generation as the means by which the necessary subsets of symbols in dependency modules are collected.

>From aee625a7a2b0b1d328823dd410522ab04cb8ee65 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Thu, 3 Jul 2025 15:17:33 -0700
Subject: [PATCH] [flang] Add general symbol dependence collection utility

Replace HarvestSymbolsNeededFromOtherModules() in mod-file.cpp with a
general utility function in Semantics.  This new code will find
other uses in further rework of hermetic module file generation
as the means by which the necessary subsets of symbols in dependency
modules are collected.
---
 .../flang/Semantics/symbol-dependence.h       |  36 ++
 flang/lib/Semantics/CMakeLists.txt            |   1 +
 flang/lib/Semantics/mod-file.cpp              |  76 +---
 flang/lib/Semantics/symbol-dependence.cpp     | 356 ++++++++++++++++++
 flang/test/Semantics/modfile44.f90            |   2 +
 flang/test/Semantics/modfile69.f90            |   2 +
 6 files changed, 406 insertions(+), 67 deletions(-)
 create mode 100644 flang/include/flang/Semantics/symbol-dependence.h
 create mode 100644 flang/lib/Semantics/symbol-dependence.cpp

diff --git a/flang/include/flang/Semantics/symbol-dependence.h b/flang/include/flang/Semantics/symbol-dependence.h
new file mode 100644
index 0000000000000..9bcff564a4c04
--- /dev/null
+++ b/flang/include/flang/Semantics/symbol-dependence.h
@@ -0,0 +1,36 @@
+//===-- include/flang/Semantics/symbol-dependence.h -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
+#define FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
+
+#include "flang/Semantics/symbol.h"
+
+namespace Fortran::semantics {
+
+// For a set or scope of symbols, computes the transitive closure of their
+// dependences due to their types, bounds, specific procedures, interfaces,
+// initialization, storage association, &c. Includes the original symbol
+// or members of the original set.  Does not include dependences from
+// subprogram definitions, only their interfaces.
+enum DependenceCollectionFlags {
+  NoDependenceCollectionFlags = 0,
+  IncludeOriginalSymbols = 1 << 0,
+  FollowUseAssociations = 1 << 1,
+  IncludeSpecificsOfGenerics = 1 << 2,
+  IncludeUsesOfGenerics = 1 << 3,
+  NotJustForOneModule = 1 << 4,
+};
+
+SymbolVector CollectAllDependences(const SymbolVector &,
+    int = NoDependenceCollectionFlags, const Scope * = nullptr);
+SymbolVector CollectAllDependences(
+    const Scope &, int = NoDependenceCollectionFlags);
+
+} // namespace Fortran::semantics
+#endif // FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 109bc2dbb8569..c711103e793d8 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -48,6 +48,7 @@ add_flang_library(FortranSemantics
   runtime-type-info.cpp
   scope.cpp
   semantics.cpp
+  symbol-dependence.cpp
   symbol.cpp
   tools.cpp
   type.cpp
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index 82c8536902eb2..ab851587bb49a 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -15,6 +15,7 @@
 #include "flang/Parser/unparse.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol-dependence.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/Support/FileSystem.h"
@@ -223,72 +224,13 @@ std::string ModFileWriter::GetAsString(const Symbol &symbol) {
 // Collect symbols from constant and specification expressions that are being
 // referenced directly from other modules; they may require new USE
 // associations.
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &, const Scope &);
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &set, const Symbol &symbol, const Scope &scope) {
-  auto HarvestBound{[&](const Bound &bound) {
-    if (const auto &expr{bound.GetExplicit()}) {
-      for (SymbolRef ref : evaluate::CollectSymbols(*expr)) {
-        set.emplace(*ref);
-      }
-    }
-  }};
-  auto HarvestShapeSpec{[&](const ShapeSpec &shapeSpec) {
-    HarvestBound(shapeSpec.lbound());
-    HarvestBound(shapeSpec.ubound());
-  }};
-  auto HarvestArraySpec{[&](const ArraySpec &arraySpec) {
-    for (const auto &shapeSpec : arraySpec) {
-      HarvestShapeSpec(shapeSpec);
-    }
-  }};
-
-  if (symbol.has<DerivedTypeDetails>()) {
-    if (symbol.scope()) {
-      HarvestSymbolsNeededFromOtherModules(set, *symbol.scope());
-    }
-  } else if (const auto &generic{symbol.detailsIf<GenericDetails>()};
-             generic && generic->derivedType()) {
-    const Symbol &dtSym{*generic->derivedType()};
-    if (dtSym.has<DerivedTypeDetails>()) {
-      if (dtSym.scope()) {
-        HarvestSymbolsNeededFromOtherModules(set, *dtSym.scope());
-      }
-    } else {
-      CHECK(dtSym.has<UseDetails>() || dtSym.has<UseErrorDetails>());
-    }
-  } else if (const auto *object{symbol.detailsIf<ObjectEntityDetails>()}) {
-    HarvestArraySpec(object->shape());
-    HarvestArraySpec(object->coshape());
-    if (IsNamedConstant(symbol) || scope.IsDerivedType()) {
-      if (object->init()) {
-        for (SymbolRef ref : evaluate::CollectSymbols(*object->init())) {
-          set.emplace(*ref);
-        }
-      }
-    }
-  } else if (const auto *proc{symbol.detailsIf<ProcEntityDetails>()}) {
-    if (proc->init() && *proc->init() && scope.IsDerivedType()) {
-      set.emplace(**proc->init());
-    }
-  } else if (const auto *subp{symbol.detailsIf<SubprogramDetails>()}) {
-    for (const Symbol *dummy : subp->dummyArgs()) {
-      if (dummy) {
-        HarvestSymbolsNeededFromOtherModules(set, *dummy, scope);
-      }
-    }
-    if (subp->isFunction()) {
-      HarvestSymbolsNeededFromOtherModules(set, subp->result(), scope);
-    }
-  }
-}
-
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &set, const Scope &scope) {
-  for (const auto &[_, symbol] : scope) {
-    HarvestSymbolsNeededFromOtherModules(set, *symbol, scope);
+static SourceOrderedSymbolSet HarvestSymbolsNeededFromOtherModules(
+    const Scope &scope) {
+  SourceOrderedSymbolSet set;
+  for (const Symbol &symbol : CollectAllDependences(scope)) {
+    set.insert(symbol);
   }
+  return set;
 }
 
 void ModFileWriter::PrepareRenamings(const Scope &scope) {
@@ -304,8 +246,8 @@ void ModFileWriter::PrepareRenamings(const Scope &scope) {
     }
   }
   // Collect symbols needed from other modules
-  SourceOrderedSymbolSet symbolsNeeded;
-  HarvestSymbolsNeededFromOtherModules(symbolsNeeded, scope);
+  SourceOrderedSymbolSet symbolsNeeded{
+      HarvestSymbolsNeededFromOtherModules(scope)};
   // Establish any necessary renamings of symbols in other modules
   // to their names in this scope, creating those new names when needed.
   auto &renamings{context_.moduleFileOutputRenamings()};
diff --git a/flang/lib/Semantics/symbol-dependence.cpp b/flang/lib/Semantics/symbol-dependence.cpp
new file mode 100644
index 0000000000000..2591f609f3d00
--- /dev/null
+++ b/flang/lib/Semantics/symbol-dependence.cpp
@@ -0,0 +1,356 @@
+//===-- lib/Semantics/symbol-dependence.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Semantics/symbol-dependence.h"
+#include "flang/Common/idioms.h"
+#include "flang/Common/restorer.h"
+#include "flang/Common/visit.h"
+#include <queue>
+
+static constexpr bool EnableDebugging{false};
+
+namespace Fortran::semantics {
+
+// Helper class that collects all of the symbol dependences for a
+// given symbol.
+class Collector {
+public:
+  explicit Collector(int flags) : flags_{flags} {}
+
+  void CollectSymbolDependences(const Symbol &);
+  UnorderedSymbolSet MustFollowDependences() { return std::move(dependences_); }
+  SymbolVector AllDependences() { return std::move(mentions_); }
+
+private:
+  // This symbol is depended upon and its declaration must precede
+  // the symbol of interest.
+  void MustFollow(const Symbol &x) {
+    if (!possibleImports_ || !DoesScopeContain(possibleImports_, x)) {
+      dependences_.insert(x);
+    }
+  }
+  // This symbol is depended upon, but is not necessarily a dependence
+  // that must precede the symbol of interest in the output of the
+  // topological sort.
+  void Need(const Symbol &x) {
+    if (mentioned_.insert(x).second) {
+      mentions_.emplace_back(x);
+    }
+  }
+  void Need(const Symbol *x) {
+    if (x) {
+      Need(*x);
+    }
+  }
+
+  // These overloads of Collect() are mutally recursive, so they're
+  // packaged as member functions of a class.
+  void Collect(const Symbol &x) {
+    Need(x);
+    const auto *subp{x.detailsIf<SubprogramDetails>()};
+    if ((subp && subp->isInterface()) || IsDummy(x) ||
+        x.has<CommonBlockDetails>() || x.has<NamelistDetails>()) {
+      // can be forward-referenced
+    } else {
+      MustFollow(x);
+    }
+  }
+  void Collect(SymbolRef x) { Collect(*x); }
+  template <typename A> void Collect(const std::optional<A> &x) {
+    if (x) {
+      Collect(*x);
+    }
+  }
+  template <typename A> void Collect(const A *x) {
+    if (x) {
+      Collect(*x);
+    }
+  }
+  void Collect(const UnorderedSymbolSet &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const SourceOrderedSymbolSet &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const SymbolVector &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const Scope &x) { Collect(x.GetSymbols()); }
+  template <typename T> void Collect(const evaluate::Expr<T> &x) {
+    UnorderedSymbolSet exprSyms{evaluate::CollectSymbols(x)};
+    for (const Symbol &sym : exprSyms) {
+      if (!sym.owner().IsDerivedType()) {
+        Collect(sym);
+      }
+    }
+  }
+  void Collect(const DeclTypeSpec &type) {
+    if (type.category() == DeclTypeSpec::Category::Character) {
+      Collect(type.characterTypeSpec().length());
+    } else {
+      Collect(type.AsDerived());
+    }
+  }
+  void Collect(const DerivedTypeSpec &type) {
+    const Symbol &typeSym{type.originalTypeSymbol()};
+    if (!derivedTypeReferenceCanBeForward_ || !type.parameters().empty()) {
+      MustFollow(typeSym);
+    }
+    Need(typeSym);
+    for (const auto &[_, value] : type.parameters()) {
+      Collect(value);
+    }
+  }
+  void Collect(const ParamValue &x) { Collect(x.GetExplicit()); }
+  void Collect(const Bound &x) { Collect(x.GetExplicit()); }
+  void Collect(const ShapeSpec &x) {
+    Collect(x.lbound());
+    Collect(x.ubound());
+  }
+  void Collect(const ArraySpec &x) {
+    for (const ShapeSpec &shapeSpec : x) {
+      Collect(shapeSpec);
+    }
+  }
+
+  UnorderedSymbolSet mentioned_, dependences_;
+  SymbolVector mentions_;
+  int flags_{NoDependenceCollectionFlags};
+  bool derivedTypeReferenceCanBeForward_{false};
+  const Scope *possibleImports_{nullptr};
+};
+
+void Collector::CollectSymbolDependences(const Symbol &symbol) {
+  if (symbol.has<ProcBindingDetails>() || symbol.has<SubprogramDetails>()) {
+    // type will be picked up later for the function result, if any
+  } else if (symbol.has<UseDetails>() || symbol.has<UseErrorDetails>() ||
+      symbol.has<HostAssocDetails>()) {
+  } else if (IsAllocatableOrPointer(symbol) && symbol.owner().IsDerivedType()) {
+    bool saveCanBeForward{derivedTypeReferenceCanBeForward_};
+    derivedTypeReferenceCanBeForward_ = true;
+    Collect(symbol.GetType());
+    derivedTypeReferenceCanBeForward_ = saveCanBeForward;
+  } else {
+    Collect(symbol.GetType());
+  }
+  common::visit(
+      common::visitors{
+          [this, &symbol](const ObjectEntityDetails &x) {
+            Collect(x.shape());
+            Collect(x.coshape());
+            if (IsNamedConstant(symbol) || symbol.owner().IsDerivedType()) {
+              Collect(x.init());
+            }
+            Need(x.commonBlock());
+            if (const auto *set{FindEquivalenceSet(symbol)}) {
+              for (const EquivalenceObject &equivObject : *set) {
+                Need(equivObject.symbol);
+              }
+            }
+          },
+          [this, &symbol](const ProcEntityDetails &x) {
+            Collect(x.rawProcInterface());
+            if (symbol.owner().IsDerivedType()) {
+              Collect(x.init());
+            }
+          },
+          [this](const ProcBindingDetails &x) { Need(x.symbol()); },
+          [this, &symbol](const SubprogramDetails &x) {
+            // Note dummy arguments & result symbol without dependence, unless
+            // the subprogram is an interface block that might need to IMPORT
+            // a type.
+            bool needImports{x.isInterface()};
+            auto restorer{common::ScopedSet(
+                possibleImports_, needImports ? symbol.scope() : nullptr)};
+            for (const Symbol *dummy : x.dummyArgs()) {
+              if (dummy) {
+                Need(*dummy);
+                if (needImports) {
+                  CollectSymbolDependences(*dummy);
+                }
+              }
+            }
+            if (x.isFunction()) {
+              Need(x.result());
+              if (needImports) {
+                CollectSymbolDependences(x.result());
+              }
+            }
+          },
+          [this, &symbol](const DerivedTypeDetails &x) {
+            Collect(symbol.scope());
+            for (const auto &[_, symbolRef] : x.finals()) {
+              Need(*symbolRef);
+            }
+          },
+          [this](const GenericDetails &x) {
+            Collect(x.derivedType());
+            Collect(x.specific());
+            if (flags_ & IncludeUsesOfGenerics) {
+              for (const Symbol &use : x.uses()) {
+                Collect(use);
+              }
+            }
+            if (flags_ & IncludeSpecificsOfGenerics) {
+              for (const Symbol &specific : x.specificProcs()) {
+                Collect(specific);
+              }
+            }
+          },
+          [this](const NamelistDetails &x) {
+            for (const Symbol &symbol : x.objects()) {
+              Collect(symbol);
+            }
+          },
+          [this](const CommonBlockDetails &x) {
+            for (auto ref : x.objects()) {
+              Collect(*ref);
+            }
+          },
+          [this](const UseDetails &x) {
+            if (flags_ & FollowUseAssociations) {
+              Need(x.symbol());
+            }
+          },
+          [this](const HostAssocDetails &x) { Need(x.symbol()); },
+          [](const auto &) {},
+      },
+      symbol.details());
+}
+
+SymbolVector CollectAllDependences(const Scope &scope, int flags) {
+  SymbolVector basis{scope.GetSymbols()};
+  return CollectAllDependences(basis, flags, &scope);
+}
+
+// Returns a vector of symbols, topologically sorted by dependence
+SymbolVector CollectAllDependences(
+    const SymbolVector &original, int flags, const Scope *forScope) {
+  std::queue<const Symbol *> work;
+  UnorderedSymbolSet enqueued;
+  for (const Symbol &symbol : original) {
+    if (!symbol.test(Symbol::Flag::CompilerCreated)) {
+      work.push(&symbol);
+      enqueued.insert(symbol);
+    }
+  }
+  // For each symbol, collect its dependences into "topology".
+  // The "visited" vector and "enqueued" set hold all of the
+  // symbols considered.
+  std::map<const Symbol *, UnorderedSymbolSet> topology;
+  std::vector<const Symbol *> visited;
+  visited.reserve(2 * original.size());
+  std::optional<SourceName> forModuleName;
+  if (forScope && !(flags & NotJustForOneModule)) {
+    if (const Scope *forModule{FindModuleContaining(*forScope)}) {
+      forModuleName = forModule->GetName();
+    }
+  }
+  while (!work.empty()) {
+    const Symbol &symbol{*work.front()};
+    work.pop();
+    visited.push_back(&symbol);
+    Collector collector{flags};
+    bool doCollection{true};
+    if (forModuleName) {
+      if (const Scope *symModule{FindModuleContaining(symbol.owner())}) {
+        if (auto symModName{symModule->GetName()}) {
+          doCollection = *forModuleName == *symModName;
+        }
+      }
+    }
+    if (doCollection) {
+      collector.CollectSymbolDependences(symbol);
+    }
+    auto dependences{collector.MustFollowDependences()};
+    auto mentions{collector.AllDependences()};
+    if constexpr (EnableDebugging) {
+      for (const Symbol &need : dependences) {
+        llvm::errs() << "symbol " << symbol << " must follow " << need << '\n';
+      }
+      for (const Symbol &need : mentions) {
+        llvm::errs() << "symbol " << symbol << " needs " << need << '\n';
+      }
+    }
+    CHECK(topology.find(&symbol) == topology.end());
+    topology.emplace(&symbol, std::move(dependences));
+    for (const Symbol &symbol : mentions) {
+      if (!symbol.test(Symbol::Flag::CompilerCreated)) {
+        if (enqueued.insert(symbol).second) {
+          work.push(&symbol);
+        }
+      }
+    }
+  }
+  CHECK(enqueued.size() == visited.size());
+  // Topological sorting
+  // Subtle: This inverted topology map uses a SymbolVector, not a set
+  // of symbols, so that the order of symbols in the final output remains
+  // deterministic.
+  std::map<const Symbol *, SymbolVector> invertedTopology;
+  for (const Symbol *symbol : visited) {
+    invertedTopology[symbol] = SymbolVector{};
+  }
+  std::map<const Symbol *, std::size_t> numWaitingFor;
+  for (const Symbol *symbol : visited) {
+    auto topoIter{topology.find(symbol)};
+    CHECK(topoIter != topology.end());
+    const auto &needs{topoIter->second};
+    if (needs.empty()) {
+      work.push(symbol);
+    } else {
+      numWaitingFor[symbol] = needs.size();
+      for (const Symbol &need : needs) {
+        invertedTopology[&need].push_back(*symbol);
+      }
+    }
+  }
+  CHECK(visited.size() == work.size() + numWaitingFor.size());
+  SymbolVector resultVector;
+  while (!work.empty()) {
+    const Symbol &symbol{*work.front()};
+    work.pop();
+    resultVector.push_back(symbol);
+    auto enqueuedIter{enqueued.find(symbol)};
+    CHECK(enqueuedIter != enqueued.end());
+    enqueued.erase(enqueuedIter);
+    if (auto invertedIter{invertedTopology.find(&symbol)};
+        invertedIter != invertedTopology.end()) {
+      for (const Symbol &neededBy : invertedIter->second) {
+        std::size_t stillAwaiting{numWaitingFor[&neededBy] - 1};
+        if (stillAwaiting == 0) {
+          work.push(&neededBy);
+        } else {
+          numWaitingFor[&neededBy] = stillAwaiting;
+        }
+      }
+    }
+  }
+  if constexpr (EnableDebugging) {
+    llvm::errs() << "Topological sort failed in CollectAllDependences\n";
+    for (const Symbol &remnant : enqueued) {
+      auto topoIter{topology.find(&remnant)};
+      CHECK(topoIter != topology.end());
+      llvm::errs() << "  remnant symbol " << remnant << " needs:\n";
+      for (const Symbol &n : topoIter->second) {
+        llvm::errs() << "   " << n << '\n';
+      }
+    }
+  }
+  CHECK(enqueued.empty());
+  CHECK(resultVector.size() == visited.size());
+  return resultVector;
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/test/Semantics/modfile44.f90 b/flang/test/Semantics/modfile44.f90
index 23d93b18a5a1f..48f6250cf6ce8 100644
--- a/flang/test/Semantics/modfile44.f90
+++ b/flang/test/Semantics/modfile44.f90
@@ -45,6 +45,8 @@ function foo(j) result(res)
 
 !Expect: m2.mod
 !module m2
+!use m1,only:xyz
+!private::xyz
 !contains
 !function foo(j) result(res)
 !use m1,only:xyz
diff --git a/flang/test/Semantics/modfile69.f90 b/flang/test/Semantics/modfile69.f90
index 6586e0524f5ea..f969573efdd78 100644
--- a/flang/test/Semantics/modfile69.f90
+++ b/flang/test/Semantics/modfile69.f90
@@ -36,6 +36,8 @@ subroutine sub(x)
 
 !Expect: m3.mod
 !module m3
+!use m2,only:bar
+!private::bar
 !contains
 !subroutine sub(x)
 !use m2,only:bar


From flang-commits at lists.llvm.org  Thu Jul  3 15:45:19 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Thu, 03 Jul 2025 15:45:19 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add general symbol dependence
	collection utility (PR #144618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144618@github.com>
Message-ID: <686707ff.630a0220.1b98e0.3ed3@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/144618

From flang-commits at lists.llvm.org  Thu Jul  3 15:45:30 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 15:45:30 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add general symbol dependence
	collection utility (PR #146968)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146968@github.com>
Message-ID: <6867080a.170a0220.3a583e.2f5b@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Peter Klausler (klausler)

<details>
<summary>Changes</summary>

Replace HarvestSymbolsNeededFromOtherModules() in mod-file.cpp with a general utility function in Semantics.  This new code will find other uses in further rework of hermetic module file generation as the means by which the necessary subsets of symbols in dependency modules are collected.

---
Full diff: https://github.com/llvm/llvm-project/pull/146968.diff


6 Files Affected:

- (added) flang/include/flang/Semantics/symbol-dependence.h (+36) 
- (modified) flang/lib/Semantics/CMakeLists.txt (+1) 
- (modified) flang/lib/Semantics/mod-file.cpp (+9-67) 
- (added) flang/lib/Semantics/symbol-dependence.cpp (+356) 
- (modified) flang/test/Semantics/modfile44.f90 (+2) 
- (modified) flang/test/Semantics/modfile69.f90 (+2) 


``````````diff
diff --git a/flang/include/flang/Semantics/symbol-dependence.h b/flang/include/flang/Semantics/symbol-dependence.h
new file mode 100644
index 0000000000000..9bcff564a4c04
--- /dev/null
+++ b/flang/include/flang/Semantics/symbol-dependence.h
@@ -0,0 +1,36 @@
+//===-- include/flang/Semantics/symbol-dependence.h -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
+#define FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
+
+#include "flang/Semantics/symbol.h"
+
+namespace Fortran::semantics {
+
+// For a set or scope of symbols, computes the transitive closure of their
+// dependences due to their types, bounds, specific procedures, interfaces,
+// initialization, storage association, &c. Includes the original symbol
+// or members of the original set.  Does not include dependences from
+// subprogram definitions, only their interfaces.
+enum DependenceCollectionFlags {
+  NoDependenceCollectionFlags = 0,
+  IncludeOriginalSymbols = 1 << 0,
+  FollowUseAssociations = 1 << 1,
+  IncludeSpecificsOfGenerics = 1 << 2,
+  IncludeUsesOfGenerics = 1 << 3,
+  NotJustForOneModule = 1 << 4,
+};
+
+SymbolVector CollectAllDependences(const SymbolVector &,
+    int = NoDependenceCollectionFlags, const Scope * = nullptr);
+SymbolVector CollectAllDependences(
+    const Scope &, int = NoDependenceCollectionFlags);
+
+} // namespace Fortran::semantics
+#endif // FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 109bc2dbb8569..c711103e793d8 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -48,6 +48,7 @@ add_flang_library(FortranSemantics
   runtime-type-info.cpp
   scope.cpp
   semantics.cpp
+  symbol-dependence.cpp
   symbol.cpp
   tools.cpp
   type.cpp
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index 82c8536902eb2..ab851587bb49a 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -15,6 +15,7 @@
 #include "flang/Parser/unparse.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol-dependence.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/Support/FileSystem.h"
@@ -223,72 +224,13 @@ std::string ModFileWriter::GetAsString(const Symbol &symbol) {
 // Collect symbols from constant and specification expressions that are being
 // referenced directly from other modules; they may require new USE
 // associations.
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &, const Scope &);
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &set, const Symbol &symbol, const Scope &scope) {
-  auto HarvestBound{[&](const Bound &bound) {
-    if (const auto &expr{bound.GetExplicit()}) {
-      for (SymbolRef ref : evaluate::CollectSymbols(*expr)) {
-        set.emplace(*ref);
-      }
-    }
-  }};
-  auto HarvestShapeSpec{[&](const ShapeSpec &shapeSpec) {
-    HarvestBound(shapeSpec.lbound());
-    HarvestBound(shapeSpec.ubound());
-  }};
-  auto HarvestArraySpec{[&](const ArraySpec &arraySpec) {
-    for (const auto &shapeSpec : arraySpec) {
-      HarvestShapeSpec(shapeSpec);
-    }
-  }};
-
-  if (symbol.has<DerivedTypeDetails>()) {
-    if (symbol.scope()) {
-      HarvestSymbolsNeededFromOtherModules(set, *symbol.scope());
-    }
-  } else if (const auto &generic{symbol.detailsIf<GenericDetails>()};
-             generic && generic->derivedType()) {
-    const Symbol &dtSym{*generic->derivedType()};
-    if (dtSym.has<DerivedTypeDetails>()) {
-      if (dtSym.scope()) {
-        HarvestSymbolsNeededFromOtherModules(set, *dtSym.scope());
-      }
-    } else {
-      CHECK(dtSym.has<UseDetails>() || dtSym.has<UseErrorDetails>());
-    }
-  } else if (const auto *object{symbol.detailsIf<ObjectEntityDetails>()}) {
-    HarvestArraySpec(object->shape());
-    HarvestArraySpec(object->coshape());
-    if (IsNamedConstant(symbol) || scope.IsDerivedType()) {
-      if (object->init()) {
-        for (SymbolRef ref : evaluate::CollectSymbols(*object->init())) {
-          set.emplace(*ref);
-        }
-      }
-    }
-  } else if (const auto *proc{symbol.detailsIf<ProcEntityDetails>()}) {
-    if (proc->init() && *proc->init() && scope.IsDerivedType()) {
-      set.emplace(**proc->init());
-    }
-  } else if (const auto *subp{symbol.detailsIf<SubprogramDetails>()}) {
-    for (const Symbol *dummy : subp->dummyArgs()) {
-      if (dummy) {
-        HarvestSymbolsNeededFromOtherModules(set, *dummy, scope);
-      }
-    }
-    if (subp->isFunction()) {
-      HarvestSymbolsNeededFromOtherModules(set, subp->result(), scope);
-    }
-  }
-}
-
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &set, const Scope &scope) {
-  for (const auto &[_, symbol] : scope) {
-    HarvestSymbolsNeededFromOtherModules(set, *symbol, scope);
+static SourceOrderedSymbolSet HarvestSymbolsNeededFromOtherModules(
+    const Scope &scope) {
+  SourceOrderedSymbolSet set;
+  for (const Symbol &symbol : CollectAllDependences(scope)) {
+    set.insert(symbol);
   }
+  return set;
 }
 
 void ModFileWriter::PrepareRenamings(const Scope &scope) {
@@ -304,8 +246,8 @@ void ModFileWriter::PrepareRenamings(const Scope &scope) {
     }
   }
   // Collect symbols needed from other modules
-  SourceOrderedSymbolSet symbolsNeeded;
-  HarvestSymbolsNeededFromOtherModules(symbolsNeeded, scope);
+  SourceOrderedSymbolSet symbolsNeeded{
+      HarvestSymbolsNeededFromOtherModules(scope)};
   // Establish any necessary renamings of symbols in other modules
   // to their names in this scope, creating those new names when needed.
   auto &renamings{context_.moduleFileOutputRenamings()};
diff --git a/flang/lib/Semantics/symbol-dependence.cpp b/flang/lib/Semantics/symbol-dependence.cpp
new file mode 100644
index 0000000000000..2591f609f3d00
--- /dev/null
+++ b/flang/lib/Semantics/symbol-dependence.cpp
@@ -0,0 +1,356 @@
+//===-- lib/Semantics/symbol-dependence.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Semantics/symbol-dependence.h"
+#include "flang/Common/idioms.h"
+#include "flang/Common/restorer.h"
+#include "flang/Common/visit.h"
+#include <queue>
+
+static constexpr bool EnableDebugging{false};
+
+namespace Fortran::semantics {
+
+// Helper class that collects all of the symbol dependences for a
+// given symbol.
+class Collector {
+public:
+  explicit Collector(int flags) : flags_{flags} {}
+
+  void CollectSymbolDependences(const Symbol &);
+  UnorderedSymbolSet MustFollowDependences() { return std::move(dependences_); }
+  SymbolVector AllDependences() { return std::move(mentions_); }
+
+private:
+  // This symbol is depended upon and its declaration must precede
+  // the symbol of interest.
+  void MustFollow(const Symbol &x) {
+    if (!possibleImports_ || !DoesScopeContain(possibleImports_, x)) {
+      dependences_.insert(x);
+    }
+  }
+  // This symbol is depended upon, but is not necessarily a dependence
+  // that must precede the symbol of interest in the output of the
+  // topological sort.
+  void Need(const Symbol &x) {
+    if (mentioned_.insert(x).second) {
+      mentions_.emplace_back(x);
+    }
+  }
+  void Need(const Symbol *x) {
+    if (x) {
+      Need(*x);
+    }
+  }
+
+  // These overloads of Collect() are mutally recursive, so they're
+  // packaged as member functions of a class.
+  void Collect(const Symbol &x) {
+    Need(x);
+    const auto *subp{x.detailsIf<SubprogramDetails>()};
+    if ((subp && subp->isInterface()) || IsDummy(x) ||
+        x.has<CommonBlockDetails>() || x.has<NamelistDetails>()) {
+      // can be forward-referenced
+    } else {
+      MustFollow(x);
+    }
+  }
+  void Collect(SymbolRef x) { Collect(*x); }
+  template <typename A> void Collect(const std::optional<A> &x) {
+    if (x) {
+      Collect(*x);
+    }
+  }
+  template <typename A> void Collect(const A *x) {
+    if (x) {
+      Collect(*x);
+    }
+  }
+  void Collect(const UnorderedSymbolSet &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const SourceOrderedSymbolSet &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const SymbolVector &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const Scope &x) { Collect(x.GetSymbols()); }
+  template <typename T> void Collect(const evaluate::Expr<T> &x) {
+    UnorderedSymbolSet exprSyms{evaluate::CollectSymbols(x)};
+    for (const Symbol &sym : exprSyms) {
+      if (!sym.owner().IsDerivedType()) {
+        Collect(sym);
+      }
+    }
+  }
+  void Collect(const DeclTypeSpec &type) {
+    if (type.category() == DeclTypeSpec::Category::Character) {
+      Collect(type.characterTypeSpec().length());
+    } else {
+      Collect(type.AsDerived());
+    }
+  }
+  void Collect(const DerivedTypeSpec &type) {
+    const Symbol &typeSym{type.originalTypeSymbol()};
+    if (!derivedTypeReferenceCanBeForward_ || !type.parameters().empty()) {
+      MustFollow(typeSym);
+    }
+    Need(typeSym);
+    for (const auto &[_, value] : type.parameters()) {
+      Collect(value);
+    }
+  }
+  void Collect(const ParamValue &x) { Collect(x.GetExplicit()); }
+  void Collect(const Bound &x) { Collect(x.GetExplicit()); }
+  void Collect(const ShapeSpec &x) {
+    Collect(x.lbound());
+    Collect(x.ubound());
+  }
+  void Collect(const ArraySpec &x) {
+    for (const ShapeSpec &shapeSpec : x) {
+      Collect(shapeSpec);
+    }
+  }
+
+  UnorderedSymbolSet mentioned_, dependences_;
+  SymbolVector mentions_;
+  int flags_{NoDependenceCollectionFlags};
+  bool derivedTypeReferenceCanBeForward_{false};
+  const Scope *possibleImports_{nullptr};
+};
+
+void Collector::CollectSymbolDependences(const Symbol &symbol) {
+  if (symbol.has<ProcBindingDetails>() || symbol.has<SubprogramDetails>()) {
+    // type will be picked up later for the function result, if any
+  } else if (symbol.has<UseDetails>() || symbol.has<UseErrorDetails>() ||
+      symbol.has<HostAssocDetails>()) {
+  } else if (IsAllocatableOrPointer(symbol) && symbol.owner().IsDerivedType()) {
+    bool saveCanBeForward{derivedTypeReferenceCanBeForward_};
+    derivedTypeReferenceCanBeForward_ = true;
+    Collect(symbol.GetType());
+    derivedTypeReferenceCanBeForward_ = saveCanBeForward;
+  } else {
+    Collect(symbol.GetType());
+  }
+  common::visit(
+      common::visitors{
+          [this, &symbol](const ObjectEntityDetails &x) {
+            Collect(x.shape());
+            Collect(x.coshape());
+            if (IsNamedConstant(symbol) || symbol.owner().IsDerivedType()) {
+              Collect(x.init());
+            }
+            Need(x.commonBlock());
+            if (const auto *set{FindEquivalenceSet(symbol)}) {
+              for (const EquivalenceObject &equivObject : *set) {
+                Need(equivObject.symbol);
+              }
+            }
+          },
+          [this, &symbol](const ProcEntityDetails &x) {
+            Collect(x.rawProcInterface());
+            if (symbol.owner().IsDerivedType()) {
+              Collect(x.init());
+            }
+          },
+          [this](const ProcBindingDetails &x) { Need(x.symbol()); },
+          [this, &symbol](const SubprogramDetails &x) {
+            // Note dummy arguments & result symbol without dependence, unless
+            // the subprogram is an interface block that might need to IMPORT
+            // a type.
+            bool needImports{x.isInterface()};
+            auto restorer{common::ScopedSet(
+                possibleImports_, needImports ? symbol.scope() : nullptr)};
+            for (const Symbol *dummy : x.dummyArgs()) {
+              if (dummy) {
+                Need(*dummy);
+                if (needImports) {
+                  CollectSymbolDependences(*dummy);
+                }
+              }
+            }
+            if (x.isFunction()) {
+              Need(x.result());
+              if (needImports) {
+                CollectSymbolDependences(x.result());
+              }
+            }
+          },
+          [this, &symbol](const DerivedTypeDetails &x) {
+            Collect(symbol.scope());
+            for (const auto &[_, symbolRef] : x.finals()) {
+              Need(*symbolRef);
+            }
+          },
+          [this](const GenericDetails &x) {
+            Collect(x.derivedType());
+            Collect(x.specific());
+            if (flags_ & IncludeUsesOfGenerics) {
+              for (const Symbol &use : x.uses()) {
+                Collect(use);
+              }
+            }
+            if (flags_ & IncludeSpecificsOfGenerics) {
+              for (const Symbol &specific : x.specificProcs()) {
+                Collect(specific);
+              }
+            }
+          },
+          [this](const NamelistDetails &x) {
+            for (const Symbol &symbol : x.objects()) {
+              Collect(symbol);
+            }
+          },
+          [this](const CommonBlockDetails &x) {
+            for (auto ref : x.objects()) {
+              Collect(*ref);
+            }
+          },
+          [this](const UseDetails &x) {
+            if (flags_ & FollowUseAssociations) {
+              Need(x.symbol());
+            }
+          },
+          [this](const HostAssocDetails &x) { Need(x.symbol()); },
+          [](const auto &) {},
+      },
+      symbol.details());
+}
+
+SymbolVector CollectAllDependences(const Scope &scope, int flags) {
+  SymbolVector basis{scope.GetSymbols()};
+  return CollectAllDependences(basis, flags, &scope);
+}
+
+// Returns a vector of symbols, topologically sorted by dependence
+SymbolVector CollectAllDependences(
+    const SymbolVector &original, int flags, const Scope *forScope) {
+  std::queue<const Symbol *> work;
+  UnorderedSymbolSet enqueued;
+  for (const Symbol &symbol : original) {
+    if (!symbol.test(Symbol::Flag::CompilerCreated)) {
+      work.push(&symbol);
+      enqueued.insert(symbol);
+    }
+  }
+  // For each symbol, collect its dependences into "topology".
+  // The "visited" vector and "enqueued" set hold all of the
+  // symbols considered.
+  std::map<const Symbol *, UnorderedSymbolSet> topology;
+  std::vector<const Symbol *> visited;
+  visited.reserve(2 * original.size());
+  std::optional<SourceName> forModuleName;
+  if (forScope && !(flags & NotJustForOneModule)) {
+    if (const Scope *forModule{FindModuleContaining(*forScope)}) {
+      forModuleName = forModule->GetName();
+    }
+  }
+  while (!work.empty()) {
+    const Symbol &symbol{*work.front()};
+    work.pop();
+    visited.push_back(&symbol);
+    Collector collector{flags};
+    bool doCollection{true};
+    if (forModuleName) {
+      if (const Scope *symModule{FindModuleContaining(symbol.owner())}) {
+        if (auto symModName{symModule->GetName()}) {
+          doCollection = *forModuleName == *symModName;
+        }
+      }
+    }
+    if (doCollection) {
+      collector.CollectSymbolDependences(symbol);
+    }
+    auto dependences{collector.MustFollowDependences()};
+    auto mentions{collector.AllDependences()};
+    if constexpr (EnableDebugging) {
+      for (const Symbol &need : dependences) {
+        llvm::errs() << "symbol " << symbol << " must follow " << need << '\n';
+      }
+      for (const Symbol &need : mentions) {
+        llvm::errs() << "symbol " << symbol << " needs " << need << '\n';
+      }
+    }
+    CHECK(topology.find(&symbol) == topology.end());
+    topology.emplace(&symbol, std::move(dependences));
+    for (const Symbol &symbol : mentions) {
+      if (!symbol.test(Symbol::Flag::CompilerCreated)) {
+        if (enqueued.insert(symbol).second) {
+          work.push(&symbol);
+        }
+      }
+    }
+  }
+  CHECK(enqueued.size() == visited.size());
+  // Topological sorting
+  // Subtle: This inverted topology map uses a SymbolVector, not a set
+  // of symbols, so that the order of symbols in the final output remains
+  // deterministic.
+  std::map<const Symbol *, SymbolVector> invertedTopology;
+  for (const Symbol *symbol : visited) {
+    invertedTopology[symbol] = SymbolVector{};
+  }
+  std::map<const Symbol *, std::size_t> numWaitingFor;
+  for (const Symbol *symbol : visited) {
+    auto topoIter{topology.find(symbol)};
+    CHECK(topoIter != topology.end());
+    const auto &needs{topoIter->second};
+    if (needs.empty()) {
+      work.push(symbol);
+    } else {
+      numWaitingFor[symbol] = needs.size();
+      for (const Symbol &need : needs) {
+        invertedTopology[&need].push_back(*symbol);
+      }
+    }
+  }
+  CHECK(visited.size() == work.size() + numWaitingFor.size());
+  SymbolVector resultVector;
+  while (!work.empty()) {
+    const Symbol &symbol{*work.front()};
+    work.pop();
+    resultVector.push_back(symbol);
+    auto enqueuedIter{enqueued.find(symbol)};
+    CHECK(enqueuedIter != enqueued.end());
+    enqueued.erase(enqueuedIter);
+    if (auto invertedIter{invertedTopology.find(&symbol)};
+        invertedIter != invertedTopology.end()) {
+      for (const Symbol &neededBy : invertedIter->second) {
+        std::size_t stillAwaiting{numWaitingFor[&neededBy] - 1};
+        if (stillAwaiting == 0) {
+          work.push(&neededBy);
+        } else {
+          numWaitingFor[&neededBy] = stillAwaiting;
+        }
+      }
+    }
+  }
+  if constexpr (EnableDebugging) {
+    llvm::errs() << "Topological sort failed in CollectAllDependences\n";
+    for (const Symbol &remnant : enqueued) {
+      auto topoIter{topology.find(&remnant)};
+      CHECK(topoIter != topology.end());
+      llvm::errs() << "  remnant symbol " << remnant << " needs:\n";
+      for (const Symbol &n : topoIter->second) {
+        llvm::errs() << "   " << n << '\n';
+      }
+    }
+  }
+  CHECK(enqueued.empty());
+  CHECK(resultVector.size() == visited.size());
+  return resultVector;
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/test/Semantics/modfile44.f90 b/flang/test/Semantics/modfile44.f90
index 23d93b18a5a1f..48f6250cf6ce8 100644
--- a/flang/test/Semantics/modfile44.f90
+++ b/flang/test/Semantics/modfile44.f90
@@ -45,6 +45,8 @@ function foo(j) result(res)
 
 !Expect: m2.mod
 !module m2
+!use m1,only:xyz
+!private::xyz
 !contains
 !function foo(j) result(res)
 !use m1,only:xyz
diff --git a/flang/test/Semantics/modfile69.f90 b/flang/test/Semantics/modfile69.f90
index 6586e0524f5ea..f969573efdd78 100644
--- a/flang/test/Semantics/modfile69.f90
+++ b/flang/test/Semantics/modfile69.f90
@@ -36,6 +36,8 @@ subroutine sub(x)
 
 !Expect: m3.mod
 !module m3
+!use m2,only:bar
+!private::bar
 !contains
 !subroutine sub(x)
 !use m2,only:bar

``````````

</details>


https://github.com/llvm/llvm-project/pull/146968

From flang-commits at lists.llvm.org  Thu Jul  3 21:02:24 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 21:02:24 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP][NFC] Remove flag toggling
 deprecated no hlfir flow in map-types-and-sizes.f90 (PR #146995)
Message-ID: <llvm.org/llvm/llvm-project/pull/146995@github.com>


https://github.com/agozillon created https://github.com/llvm/llvm-project/pull/146995

We no longer utilise the deprecated FIR only flow, so we should be testing for the current HLFIR flow that we support as opposed to the older that we no longer maintain.

>From 6b9b17282bb5dbce62f7e56ab59e0228c8203d8b Mon Sep 17 00:00:00 2001
From: agozillon <Andrew.Gozillon at amd.com>
Date: Thu, 3 Jul 2025 22:57:19 -0500
Subject: [PATCH] [Flang][OpenMP][NFC] Remove flag toggling deprecated no hlfir
 flow in map-types-and-sizes.f90

We no longer utilise the deprecated FIR only flow, so we should be testing for the current HLFIR flow that
we support as opposed to the older that we no longer maintain.
---
 .../OpenMP/map-types-and-sizes.f90            | 36 ++++++++++++-------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/flang/test/Integration/OpenMP/map-types-and-sizes.f90 b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
index 35eec823e30de..aa68c13558a7b 100644
--- a/flang/test/Integration/OpenMP/map-types-and-sizes.f90
+++ b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
@@ -6,7 +6,7 @@
 ! added to this directory and sub-directories.
 !===----------------------------------------------------------------------===!
 
-!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa %s -o - | FileCheck %s
 
 !===============================================================================
 ! Check MapTypes for target implicit captures
@@ -494,15 +494,18 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_single_member_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 1
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
-!CHECK: store ptr %[[MEMBER_ACCESS]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+!CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_multiple_nested_members_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 0
-!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS_0:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 0
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 1
 !CHECK: %[[ARR_END_OFF:.*]] = getelementptr float, ptr %[[MEMBER_ACCESS_2]], i64 1
 !CHECK: %[[ARR_END:.*]] = ptrtoint ptr %[[ARR_END_OFF]] to i64
 !CHECK: %[[FIRST_MEMBER:.*]] = ptrtoint ptr %[[MEMBER_ACCESS_1]] to i64
@@ -525,8 +528,9 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_member_with_bounds_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 2
-!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS]], i64 0, i64 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 2
+!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS_1]], i64 0, i64 1
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -574,19 +578,27 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [4 x i64], ptr %.offload_sizes, i32 0, i32 3
 !CHECK: store i64 %[[DTYPE_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 
+
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_alloca_derived_type_{{.*}}
 !CHECK: %{{.*}} = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_3:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, i64 1, align 8
-!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
-!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
-!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
 !CHECK: %[[ACCESS_DESC_MEMBER_UB:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[LOAD_DESC_MEMBER_UB:.*]] = load i64, ptr %[[ACCESS_DESC_MEMBER_UB]], align 8
 !CHECK: %[[OFFSET_MEMBER_UB:.*]] = sub i64 %[[LOAD_DESC_MEMBER_UB]], 1
+!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
+!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
+!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
 !CHECK: %[[DTYPE_ALLOCA_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ALLOCA_MEMBER_ACCESS]], i32 0, i32 0
+
 !CHECK: %[[DTYPE_BASE_ADDR_ACCESS_2:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA]], i32 0, i32 0
 !CHECK: %[[DTYPE_BASE_ADDR_LOAD_2:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS_2]], align 8
 !CHECK: %[[DTYPE_NONALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD_2]], i32 0, i32 5
@@ -715,11 +727,11 @@ end subroutine mapType_common_block_members
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_nested_derived_type_alloca{{.*}}
 !CHECK: %[[ALLOCATABLE_MEMBER_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_nested_derived_type_allocaTtop_layer, i64 1, align 8
+!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
+!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[ALLOCATABLE_MEMBER_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[ALLOCATABLE_MEMBER_ADDR_LOAD:.*]] = load i64, ptr %[[ALLOCATABLE_MEMBER_BASE_ADDR]], align 8
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_1:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_ADDR_LOAD]], 1
-!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
-!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[NESTED_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_2:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_1]], 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_3:.*]] = add i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_2]], 1


From flang-commits at lists.llvm.org  Thu Jul  3 21:02:54 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 21:02:54 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP][NFC] Remove flag toggling
 deprecated no hlfir flow in map-types-and-sizes.f90 (PR #146995)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146995@github.com>
Message-ID: <6867526e.170a0220.cfd1e.31e3@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: None (agozillon)

<details>
<summary>Changes</summary>

We no longer utilise the deprecated FIR only flow, so we should be testing for the current HLFIR flow that we support as opposed to the older that we no longer maintain.

---
Full diff: https://github.com/llvm/llvm-project/pull/146995.diff


1 Files Affected:

- (modified) flang/test/Integration/OpenMP/map-types-and-sizes.f90 (+24-12) 


``````````diff
diff --git a/flang/test/Integration/OpenMP/map-types-and-sizes.f90 b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
index 35eec823e30de..aa68c13558a7b 100644
--- a/flang/test/Integration/OpenMP/map-types-and-sizes.f90
+++ b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
@@ -6,7 +6,7 @@
 ! added to this directory and sub-directories.
 !===----------------------------------------------------------------------===!
 
-!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa %s -o - | FileCheck %s
 
 !===============================================================================
 ! Check MapTypes for target implicit captures
@@ -494,15 +494,18 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_single_member_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 1
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
-!CHECK: store ptr %[[MEMBER_ACCESS]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+!CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_multiple_nested_members_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 0
-!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS_0:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 0
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 1
 !CHECK: %[[ARR_END_OFF:.*]] = getelementptr float, ptr %[[MEMBER_ACCESS_2]], i64 1
 !CHECK: %[[ARR_END:.*]] = ptrtoint ptr %[[ARR_END_OFF]] to i64
 !CHECK: %[[FIRST_MEMBER:.*]] = ptrtoint ptr %[[MEMBER_ACCESS_1]] to i64
@@ -525,8 +528,9 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_member_with_bounds_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 2
-!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS]], i64 0, i64 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 2
+!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS_1]], i64 0, i64 1
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -574,19 +578,27 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [4 x i64], ptr %.offload_sizes, i32 0, i32 3
 !CHECK: store i64 %[[DTYPE_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 
+
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_alloca_derived_type_{{.*}}
 !CHECK: %{{.*}} = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_3:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, i64 1, align 8
-!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
-!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
-!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
 !CHECK: %[[ACCESS_DESC_MEMBER_UB:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[LOAD_DESC_MEMBER_UB:.*]] = load i64, ptr %[[ACCESS_DESC_MEMBER_UB]], align 8
 !CHECK: %[[OFFSET_MEMBER_UB:.*]] = sub i64 %[[LOAD_DESC_MEMBER_UB]], 1
+!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
+!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
+!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
 !CHECK: %[[DTYPE_ALLOCA_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ALLOCA_MEMBER_ACCESS]], i32 0, i32 0
+
 !CHECK: %[[DTYPE_BASE_ADDR_ACCESS_2:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA]], i32 0, i32 0
 !CHECK: %[[DTYPE_BASE_ADDR_LOAD_2:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS_2]], align 8
 !CHECK: %[[DTYPE_NONALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD_2]], i32 0, i32 5
@@ -715,11 +727,11 @@ end subroutine mapType_common_block_members
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_nested_derived_type_alloca{{.*}}
 !CHECK: %[[ALLOCATABLE_MEMBER_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_nested_derived_type_allocaTtop_layer, i64 1, align 8
+!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
+!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[ALLOCATABLE_MEMBER_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[ALLOCATABLE_MEMBER_ADDR_LOAD:.*]] = load i64, ptr %[[ALLOCATABLE_MEMBER_BASE_ADDR]], align 8
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_1:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_ADDR_LOAD]], 1
-!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
-!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[NESTED_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_2:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_1]], 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_3:.*]] = add i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_2]], 1

``````````

</details>


https://github.com/llvm/llvm-project/pull/146995

From flang-commits at lists.llvm.org  Thu Jul  3 21:57:45 2025
From: flang-commits at lists.llvm.org (Madhur Amilkanthwar via flang-commits)
Date: Thu, 03 Jul 2025 21:57:45 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] add -floop-fuse to clang and
	flang (PR #142686)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/142686@github.com>
Message-ID: <68675f49.050a0220.1410d4.425f@mx.google.com>


================
@@ -34,6 +34,7 @@ page](https://llvm.org/releases/).
 
 * -floop-interchange is now recognized by flang.
 * -floop-interchange is enabled by default at -O2 and above.
+* -fexperimental-fuse-loops is now recognized by flang.
----------------
madhur13490 wrote:

We should make a similar entry in `clang/docs/ReleaseNotes.md`.

https://github.com/llvm/llvm-project/pull/142686

From flang-commits at lists.llvm.org  Thu Jul  3 21:59:50 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 03 Jul 2025 21:59:50 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][OpenMP] Allocate `reduction`
 init temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <68675fc6.170a0220.2b7de1.39d3@mx.google.com>


ergawy wrote:

There is a Windows failure: https://lab.llvm.org/buildbot/#/builders/207/builds/3355. I will look into that a bit today ....

https://github.com/llvm/llvm-project/pull/146667

From flang-commits at lists.llvm.org  Thu Jul  3 22:04:47 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Thu, 03 Jul 2025 22:04:47 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][OpenMP] Allocate `reduction`
 init temps on the stack for GPUs (PR #146667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146667@github.com>
Message-ID: <686760ef.170a0220.21004f.3ad9@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `flang,mlir` at step 6 "test-build-unified-tree-check-flang".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/32718

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (test-build-unified-tree-check-flang) failure: test (failure)
******************** TEST 'Flang :: Lower/OpenMP/parallel-reduction-array.f90' FAILED ********************
Exit Code: 1

Command Output (stderr):
--
bbc -emit-hlfir -fopenmp -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 2>&1 | /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 --check-prefix=CPU # RUN: at line 1
+ bbc -emit-hlfir -fopenmp -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 --check-prefix=CPU
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -emit-hlfir -fopenmp -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 2>&1 | /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 --check-prefix=CPU # RUN: at line 2
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -emit-hlfir -fopenmp -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 --check-prefix=CPU
bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 2>&1 | /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 --check-prefix=GPU # RUN: at line 4
+ bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 --check-prefix=GPU
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 2>&1 | /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 --check-prefix=GPU # RUN: at line 5
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 --check-prefix=GPU
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90:98:8: error: GPU: expected string not found in input
! GPU: omp.declare_reduction {{.*}} alloc {
       ^
<stdin>:1:1: note: scanning from here
error: unable to create target: 'No available targets are compatible with triple "amdgcn-amd-amdhsa"'
^
<stdin>:1:15: note: possible intended match here
error: unable to create target: 'No available targets are compatible with triple "amdgcn-amd-amdhsa"'
              ^

Input file: <stdin>
Check file: /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90

-dump-input=help explains the following input dump.

Input was:
<<<<<<
            1: error: unable to create target: 'No available targets are compatible with triple "amdgcn-amd-amdhsa"' 
check:98'0     X~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ error: no match found
check:98'1                   ?                                                                                        possible intended match
>>>>>>

--

********************


```

</details>

https://github.com/llvm/llvm-project/pull/146667

From flang-commits at lists.llvm.org  Thu Jul  3 22:32:41 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 22:32:41 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solved an issue with usage of
 unlimited polymorphic in where construct (PR #147001)
Message-ID: <llvm.org/llvm/llvm-project/pull/147001@github.com>


https://github.com/EbinJose2002 created https://github.com/llvm/llvm-project/pull/147001

Fixes #133669

>From 4e93cc91262c4cb2beed9849df7f89e71fab9fb2 Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Fri, 27 Jun 2025 17:39:27 +0530
Subject: [PATCH 1/2] Solved the issue with polymorphic array declaration and
 where construct

---
 flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index 33f687db08f9a..2196f6a9dde0c 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -141,6 +141,13 @@ class AssignOpConversion : public mlir::OpRewritePattern<hlfir::AssignOp> {
         fir::runtime::genAssignTemporary(builder, loc, toMutableBox, from);
       else
         fir::runtime::genAssign(builder, loc, toMutableBox, from);
+    } else if (lhs.isPolymorphic() && rhs.isPolymorphic()) {
+      if (fir::isNoneOrSeqNone(fir::getElementTypeOf(lhsExv)) &&
+          fir::isNoneOrSeqNone(fir::getElementTypeOf(rhsExv))) {
+        mlir::Value to = fir::getBase(builder.createBox(loc, lhsExv));
+        mlir::Value from = fir::getBase(builder.createBox(loc, rhsExv));
+        fir::runtime::genAssignPolymorphic(builder, loc, to, from);
+      }
     } else {
       // TODO: use the type specification to see if IsFinalizable is set,
       // or propagate IsFinalizable attribute from lowering.

>From c3c542def2cdb43e4185e91e2389cf848ac9adc0 Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Fri, 4 Jul 2025 10:49:49 +0530
Subject: [PATCH 2/2] Wrote test to check polymorphic array handling in where
 construct

---
 flang/test/Lower/polymorphic-array.f90 | 27 ++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 flang/test/Lower/polymorphic-array.f90

diff --git a/flang/test/Lower/polymorphic-array.f90 b/flang/test/Lower/polymorphic-array.f90
new file mode 100644
index 0000000000000..6e6d9ae7187e5
--- /dev/null
+++ b/flang/test/Lower/polymorphic-array.f90
@@ -0,0 +1,27 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! Checks that no fir store is present with fir.class<none> as the first operand.
+! Regression test for bug: FIR lowering failure on polymorphic assignment.
+
+! CHECK-NOT: fir.store{{.*}}!fir.class<none>
+module m1
+  type x
+  end type x
+  logical,parameter::t=.true.,f=.false.
+  logical::mask(3)=[t,f,t]
+end module m1
+
+subroutine s1
+  use m1
+  class(*),allocatable::v(:),u(:)
+  allocate(x::v(3))
+  allocate(x::u(3))
+  where(mask)
+     u=v
+  end where
+end subroutine s1
+
+program main
+  call s1
+  print *,'pass'
+end program main


From flang-commits at lists.llvm.org  Thu Jul  3 22:33:10 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 22:33:10 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solved an issue with usage of
 unlimited polymorphic in where construct (PR #147001)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147001@github.com>
Message-ID: <68676796.a70a0220.8420a.405b@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Ebin-McW (EbinJose2002)

<details>
<summary>Changes</summary>

Fixes #<!-- -->133669

---
Full diff: https://github.com/llvm/llvm-project/pull/147001.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp (+7) 
- (added) flang/test/Lower/polymorphic-array.f90 (+27) 


``````````diff
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index 33f687db08f9a..2196f6a9dde0c 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -141,6 +141,13 @@ class AssignOpConversion : public mlir::OpRewritePattern<hlfir::AssignOp> {
         fir::runtime::genAssignTemporary(builder, loc, toMutableBox, from);
       else
         fir::runtime::genAssign(builder, loc, toMutableBox, from);
+    } else if (lhs.isPolymorphic() && rhs.isPolymorphic()) {
+      if (fir::isNoneOrSeqNone(fir::getElementTypeOf(lhsExv)) &&
+          fir::isNoneOrSeqNone(fir::getElementTypeOf(rhsExv))) {
+        mlir::Value to = fir::getBase(builder.createBox(loc, lhsExv));
+        mlir::Value from = fir::getBase(builder.createBox(loc, rhsExv));
+        fir::runtime::genAssignPolymorphic(builder, loc, to, from);
+      }
     } else {
       // TODO: use the type specification to see if IsFinalizable is set,
       // or propagate IsFinalizable attribute from lowering.
diff --git a/flang/test/Lower/polymorphic-array.f90 b/flang/test/Lower/polymorphic-array.f90
new file mode 100644
index 0000000000000..6e6d9ae7187e5
--- /dev/null
+++ b/flang/test/Lower/polymorphic-array.f90
@@ -0,0 +1,27 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! Checks that no fir store is present with fir.class<none> as the first operand.
+! Regression test for bug: FIR lowering failure on polymorphic assignment.
+
+! CHECK-NOT: fir.store{{.*}}!fir.class<none>
+module m1
+  type x
+  end type x
+  logical,parameter::t=.true.,f=.false.
+  logical::mask(3)=[t,f,t]
+end module m1
+
+subroutine s1
+  use m1
+  class(*),allocatable::v(:),u(:)
+  allocate(x::v(3))
+  allocate(x::u(3))
+  where(mask)
+     u=v
+  end where
+end subroutine s1
+
+program main
+  call s1
+  print *,'pass'
+end program main

``````````

</details>


https://github.com/llvm/llvm-project/pull/147001

From flang-commits at lists.llvm.org  Thu Jul  3 22:43:22 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 03 Jul 2025 22:43:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix Windows bot failure caused by
	#146667 (PR #147002)
Message-ID: <llvm.org/llvm/llvm-project/pull/147002@github.com>


https://github.com/ergawy created https://github.com/llvm/llvm-project/pull/147002

Fixes a Windows bot failure caused by #146667. Just run the test if an AMD GPU target is registered. Hopefully, the bot now passes.

Test quality is not reduced since `bbc` is still run on all platforms.

>From c5d0cef5916832c81e15cbfccff0d2611254e326 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Fri, 4 Jul 2025 00:40:53 -0500
Subject: [PATCH] [flang] Fix Windows bot failure caused by #146667

Fixes a Windows bot failure caused by #146667. Just run the test if an
AMD GPU target is registered. Hopefully, the bot now passes.

Test quality is not reduced since `bbc` is still run on all platforms.
---
 flang/test/Lower/OpenMP/parallel-reduction-array.f90 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 4f889d9a4e77f..d410714b1573e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -2,7 +2,8 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
 
 ! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
-! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %if amdgpu-registered-target %{ %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | \
+! RUN: FileCheck %s --check-prefix=GPU %}
 
 program reduce
 integer, dimension(3) :: i = 0


From flang-commits at lists.llvm.org  Thu Jul  3 22:43:51 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 03 Jul 2025 22:43:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix Windows bot failure caused by
	#146667 (PR #147002)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147002@github.com>
Message-ID: <68676a17.170a0220.379ec7.3a95@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/147002

From flang-commits at lists.llvm.org  Thu Jul  3 22:43:53 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 22:43:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix Windows bot failure caused by
	#146667 (PR #147002)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147002@github.com>
Message-ID: <68676a19.170a0220.3b1704.363f@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Kareem Ergawy (ergawy)

<details>
<summary>Changes</summary>

Fixes a Windows bot failure caused by #<!-- -->146667. Just run the test if an AMD GPU target is registered. Hopefully, the bot now passes.

Test quality is not reduced since `bbc` is still run on all platforms.

---
Full diff: https://github.com/llvm/llvm-project/pull/147002.diff


1 Files Affected:

- (modified) flang/test/Lower/OpenMP/parallel-reduction-array.f90 (+2-1) 


``````````diff
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 4f889d9a4e77f..d410714b1573e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -2,7 +2,8 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
 
 ! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
-! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %if amdgpu-registered-target %{ %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | \
+! RUN: FileCheck %s --check-prefix=GPU %}
 
 program reduce
 integer, dimension(3) :: i = 0

``````````

</details>


https://github.com/llvm/llvm-project/pull/147002

From flang-commits at lists.llvm.org  Thu Jul  3 22:43:54 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 22:43:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix Windows bot failure caused by
	#146667 (PR #147002)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147002@github.com>
Message-ID: <68676a1a.170a0220.21004f.3c93@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: Kareem Ergawy (ergawy)

<details>
<summary>Changes</summary>

Fixes a Windows bot failure caused by #<!-- -->146667. Just run the test if an AMD GPU target is registered. Hopefully, the bot now passes.

Test quality is not reduced since `bbc` is still run on all platforms.

---
Full diff: https://github.com/llvm/llvm-project/pull/147002.diff


1 Files Affected:

- (modified) flang/test/Lower/OpenMP/parallel-reduction-array.f90 (+2-1) 


``````````diff
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 4f889d9a4e77f..d410714b1573e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -2,7 +2,8 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
 
 ! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
-! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %if amdgpu-registered-target %{ %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | \
+! RUN: FileCheck %s --check-prefix=GPU %}
 
 program reduce
 integer, dimension(3) :: i = 0

``````````

</details>


https://github.com/llvm/llvm-project/pull/147002

From flang-commits at lists.llvm.org  Fri Jul  4 00:11:13 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Fri, 04 Jul 2025 00:11:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Move lowering of ATOMIC to
 separate file, NFC (PR #146225)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146225@github.com>
Message-ID: <68677e91.170a0220.2be3fb.3e9d@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64-flang-aix` running on `ppc64-flang-aix-test` while building `flang` at step 5 "build-unified-tree".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/201/builds/5291

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 5 (build-unified-tree) failure: build (failure)
...
5314.939 [3490/10/4714] Building CXX object lib/Target/X86/TargetInfo/CMakeFiles/LLVMX86Info.dir/X86TargetInfo.cpp.o
5315.076 [3489/10/4715] Building XCoreGenDAGISel.inc...
5315.213 [3488/10/4716] Building XCoreGenDisassemblerTables.inc...
5315.566 [3487/10/4717] Building XCoreGenInstrInfo.inc...
5315.632 [3486/10/4718] Building XCoreGenRegisterInfo.inc...
5315.773 [3485/10/4719] Building XCoreGenSDNodeInfo.inc...
5315.916 [3484/10/4720] Building XCoreGenSubtargetInfo.inc...
5316.613 [3483/10/4721] Building CXX object tools/clang/lib/StaticAnalyzer/Core/CMakeFiles/obj.clangStaticAnalyzerCore.dir/APSIntType.cpp.o
5319.376 [3482/10/4722] Building CXX object tools/clang/lib/StaticAnalyzer/Core/CMakeFiles/obj.clangStaticAnalyzerCore.dir/BlockCounter.cpp.o
5324.138 [3481/10/4723] Building CXX object tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/PPCIntrinsicCall.cpp.o
FAILED: tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/PPCIntrinsicCall.cpp.o 
/home/llvm/llvm-external-buildbots/clang.19.1.2/bin/clang++ -DFLANG_INCLUDE_TESTS=1 -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_LARGE_FILE_API -D_XOPEN_SOURCE=700 -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/tools/flang/lib/Optimizer/Builder -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/lib/Optimizer/Builder -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/tools/flang/include -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/include -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/llvm/include -isystem /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/../mlir/include -isystem /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/tools/mlir/include -isystem /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/tools/clang/include -isystem /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/llvm/../clang/include -mcmodel=large -fPIC -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wc++98-compat-extra-semi -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Werror -Wno-deprecated-copy -Wno-string-conversion -Wno-ctad-maybe-unsupported -Wno-unused-command-line-argument -Wstring-conversion           -Wcovered-switch-default -Wno-nested-anon-types -Xclang -fno-pch-timestamp -O3 -DNDEBUG -std=c++17  -fno-exceptions -funwind-tables -fno-rtti -UNDEBUG -pthread -MD -MT tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/PPCIntrinsicCall.cpp.o -MF tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/PPCIntrinsicCall.cpp.o.d -o tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/PPCIntrinsicCall.cpp.o -c /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp:16:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Optimizer/Builder/PPCIntrinsicCall.h:13:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Optimizer/Builder/IntrinsicCall.h:12:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Lower/AbstractConverter.h:18:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Lower/Support/Utils.h:17:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Lower/IterationSpace.h:18:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Lower/SymbolMap.h:17:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Optimizer/Builder/BoxValue.h:16:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Optimizer/Dialect/FIRType.h:18:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/../mlir/include/mlir/Interfaces/DataLayoutInterfaces.h:20:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/../mlir/include/mlir/IR/OpDefinition.h:22:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/../mlir/include/mlir/IR/Dialect.h:17:
In file included from /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/../mlir/include/mlir/IR/OperationSupport.h:23:
/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/../mlir/include/mlir/IR/TypeRange.h:51:19: error: 'ArrayRef' is deprecated: Use {} or ArrayRef<T>() instead [-Werror,-Wdeprecated-declarations]
   51 |       : TypeRange(ArrayRef<Type>(std::forward<Arg>(arg))) {}
      |                   ^
/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include/flang/Optimizer/Builder/IntrinsicCall.h:756:55: note: in instantiation of function template specialization 'mlir::TypeRange::TypeRange<const std::nullopt_t &, void>' requested here
  756 |     return mlir::FunctionType::get(context, argTypes, std::nullopt);
      |                                                       ^
/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp:1085:19: note: in instantiation of function template specialization 'fir::genFuncType<fir::ParamType<fir::ParamTypeId::Void, 0>, fir::ParamType<fir::ParamTypeId::Integer, 4>, fir::ParamType<fir::ParamTypeId::Integer, 4>>' requested here
 1085 |     libFuncType = genFuncType<Ty::Void, Ty::Integer<4>, Ty::Integer<4>>(
      |                   ^
/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp:508:62: note: in instantiation of function template specialization 'fir::PPCIntrinsicLibrary::genMtfsf<false>' requested here
  508 |      static_cast<IntrinsicLibrary::SubroutineGenerator>(&PI::genMtfsf<false>),
      |                                                              ^
/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/llvm/include/llvm/ADT/ArrayRef.h:70:18: note: 'ArrayRef' has been explicitly marked deprecated here
   70 |     /*implicit*/ LLVM_DEPRECATED("Use {} or ArrayRef<T>() instead", "{}")
      |                  ^
/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/llvm/include/llvm/Support/Compiler.h:249:50: note: expanded from macro 'LLVM_DEPRECATED'
  249 | #define LLVM_DEPRECATED(MSG, FIX) __attribute__((deprecated(MSG, FIX)))
      |                                                  ^
1 error generated.
5324.602 [3481/9/4724] Building CXX object lib/TextAPI/BinaryReader/CMakeFiles/LLVMTextAPIBinaryReader.dir/DylibReader.cpp.o
5326.645 [3481/8/4725] Building CXX object tools/clang/lib/StaticAnalyzer/Core/CMakeFiles/obj.clangStaticAnalyzerCore.dir/AnalyzerOptions.cpp.o
5327.784 [3481/7/4726] Building CXX object tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/IntrinsicCall.cpp.o
FAILED: tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/IntrinsicCall.cpp.o 
/home/llvm/llvm-external-buildbots/clang.19.1.2/bin/clang++ -DFLANG_INCLUDE_TESTS=1 -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_LARGE_FILE_API -D_XOPEN_SOURCE=700 -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/tools/flang/lib/Optimizer/Builder -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/lib/Optimizer/Builder -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/include -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/tools/flang/include -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/include -I/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/llvm/include -isystem /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/../mlir/include -isystem /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/tools/mlir/include -isystem /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/tools/clang/include -isystem /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/llvm/../clang/include -mcmodel=large -fPIC -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wc++98-compat-extra-semi -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Werror -Wno-deprecated-copy -Wno-string-conversion -Wno-ctad-maybe-unsupported -Wno-unused-command-line-argument -Wstring-conversion           -Wcovered-switch-default -Wno-nested-anon-types -Xclang -fno-pch-timestamp -O3 -DNDEBUG -std=c++17  -fno-exceptions -funwind-tables -fno-rtti -UNDEBUG -pthread -MD -MT tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/IntrinsicCall.cpp.o -MF tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/IntrinsicCall.cpp.o.d -o tools/flang/lib/Optimizer/Builder/CMakeFiles/FIRBuilder.dir/IntrinsicCall.cpp.o -c /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/lib/Optimizer/Builder/IntrinsicCall.cpp

```

</details>

https://github.com/llvm/llvm-project/pull/146225

From flang-commits at lists.llvm.org  Fri Jul  4 00:41:27 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 04 Jul 2025 00:41:27 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix Windows bot failure caused by
	#146667 (PR #147002)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147002@github.com>
Message-ID: <686785a7.050a0220.12c0df.4f7e@mx.google.com>


https://github.com/DavidSpickett approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/147002

From flang-commits at lists.llvm.org  Fri Jul  4 00:41:33 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 00:41:33 -0700 (PDT)
Subject: [flang-commits] [flang] 3e78aff - [flang] Fix Windows bot failure
	caused by #146667 (#147002)
Message-ID: <686785ad.170a0220.108950.437c@mx.google.com>


Author: Kareem Ergawy
Date: 2025-07-04T08:41:29+01:00
New Revision: 3e78afff0d8e3d982e4bcdaad623cbb12dff01d8

URL: https://github.com/llvm/llvm-project/commit/3e78afff0d8e3d982e4bcdaad623cbb12dff01d8
DIFF: https://github.com/llvm/llvm-project/commit/3e78afff0d8e3d982e4bcdaad623cbb12dff01d8.diff

LOG: [flang] Fix Windows bot failure caused by #146667 (#147002)

Fixes a Windows bot failure caused by #146667. Just run the test if an
AMD GPU target is registered. Hopefully, the bot now passes.

Test coverage is not reduced since `bbc` is still run on all platforms.

Added: 
    

Modified: 
    flang/test/Lower/OpenMP/parallel-reduction-array.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 4f889d9a4e77f..d410714b1573e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -2,7 +2,8 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
 
 ! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
-! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %if amdgpu-registered-target %{ %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | \
+! RUN: FileCheck %s --check-prefix=GPU %}
 
 program reduce
 integer, dimension(3) :: i = 0


        

From flang-commits at lists.llvm.org  Fri Jul  4 00:41:35 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 04 Jul 2025 00:41:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix Windows bot failure caused by
	#146667 (PR #147002)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147002@github.com>
Message-ID: <686785af.050a0220.2aae16.464b@mx.google.com>


https://github.com/DavidSpickett closed https://github.com/llvm/llvm-project/pull/147002

From flang-commits at lists.llvm.org  Fri Jul  4 00:44:00 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 00:44:00 -0700 (PDT)
Subject: [flang-commits] [flang] d84df61 - [flang] Fix x86 REQUIRES in a
	couple of tests (#146869)
Message-ID: <68678640.050a0220.3d71d7.48ff@mx.google.com>


Author: David Spickett
Date: 2025-07-04T08:43:57+01:00
New Revision: d84df61c001a9983bf445c2c0454edd581571a5f

URL: https://github.com/llvm/llvm-project/commit/d84df61c001a9983bf445c2c0454edd581571a5f
DIFF: https://github.com/llvm/llvm-project/commit/d84df61c001a9983bf445c2c0454edd581571a5f.diff

LOG: [flang] Fix x86 REQUIRES in a couple of tests (#146869)

Many tests in Flang are looking for x86_64-registered-target, but this
never exists because the target is just called x86.

These two pass with this corrected but the others I need to look into
why they fail.

Added: 
    

Modified: 
    flang/test/Lower/Intrinsics/ieee_class_queries.f90
    flang/test/Semantics/windows.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Lower/Intrinsics/ieee_class_queries.f90 b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
index aae8071b6a8b5..8d20ff8f3178f 100644
--- a/flang/test/Lower/Intrinsics/ieee_class_queries.f90
+++ b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
@@ -1,5 +1,5 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir -o - %s | FileCheck %s
 
   ! CHECK-LABEL: func @_QQmain

diff  --git a/flang/test/Semantics/windows.f90 b/flang/test/Semantics/windows.f90
index 705fb1b816895..799a34e6b89c5 100644
--- a/flang/test/Semantics/windows.f90
+++ b/flang/test/Semantics/windows.f90
@@ -1,4 +1,4 @@
-! RUN: %if x86_64-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
+! RUN: %if x86-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
 ! RUN: %if aarch64-registered-target %{ %python %S/test_errors.py %s %flang --target=aarch64-pc-windows-msvc -Werror %}
 
 subroutine uid


        

From flang-commits at lists.llvm.org  Fri Jul  4 00:44:03 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 04 Jul 2025 00:44:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix x86 REQUIRES in a couple of
	tests (PR #146869)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146869@github.com>
Message-ID: <68678643.170a0220.25d52c.4458@mx.google.com>


https://github.com/DavidSpickett closed https://github.com/llvm/llvm-project/pull/146869

From flang-commits at lists.llvm.org  Fri Jul  4 01:23:20 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Fri, 04 Jul 2025 01:23:20 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68678f78.050a0220.2e9039.547e@mx.google.com>


================
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
----------------
s-watanabe314 wrote:

I added help text. Could you please review it?

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Fri Jul  4 01:26:55 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Fri, 04 Jul 2025 01:26:55 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <6867904f.170a0220.286721.8216@mx.google.com>


================
@@ -595,6 +595,30 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
+static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  default:
+    return "";
+  }
+}
+
+static std::string
+RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
+
----------------
s-watanabe314 wrote:

I moved both functions to `CommonArgs.cpp` and removed them from `Clang.cpp` and `Flang.cpp`.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Fri Jul  4 02:06:44 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:06:44 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686799a4.050a0220.2cf4b6.54a6@mx.google.com>


https://github.com/skc7 edited https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Fri Jul  4 02:06:54 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Fri, 04 Jul 2025 02:06:54 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686799ae.050a0220.2b20ec.5675@mx.google.com>


================
@@ -192,6 +192,31 @@ class CodeGenOptions : public CodeGenOptionsBase {
     return getProfileUse() == llvm::driver::ProfileCSIRInstr;
   }
 
+  /// Controls the various implementations for complex division.
----------------
s-watanabe314 wrote:

Thank you for providing the example! I understand that your example is about moving Clang's `CodeGenOptions` to `llvm/Frontend/Driver/CodegenOptions.h`. Since Flang already defines `ComplexRangeKind` as `CodeGenOptions`, moving it to `llvm/Frontend/Driver/CodegenOptions.h` might not require significant modifications. However, in Clang, it's defined in `LangOptions`, so I'm unsure how to move it and how much modification would be necessary.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Fri Jul  4 02:08:21 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:08:21 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Implement
 workdistribute construct lowering (PR #140523)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140523@github.com>
Message-ID: <68679a05.050a0220.2cf4a7.564f@mx.google.com>


skc7 wrote:

> I am not sure the using rewrite patterns is suitable here - mainly because we have a list of transformations we know we need to do in what order so there is no need to involve patterns which add additional overhead and complexity (guarding against matching again etc, making sure the order is correct). That's how the LowerWorkshare pass is implemented.

Thanks @ivanradanov for feedback. Have updated PR, removing the rewrite patterns.

https://github.com/llvm/llvm-project/pull/140523

From flang-commits at lists.llvm.org  Fri Jul  4 02:08:53 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:08:53 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang-rt] Add Assign_omp RT call.
	(PR #145465)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145465@github.com>
Message-ID: <68679a25.170a0220.3092cd.769c@mx.google.com>


https://github.com/skc7 ready_for_review https://github.com/llvm/llvm-project/pull/145465

From flang-commits at lists.llvm.org  Fri Jul  4 02:09:32 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:09:32 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Add support for
 workdistribute construct in flang frontend (PR #146029)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146029@github.com>
Message-ID: <68679a4c.170a0220.360f59.45ba@mx.google.com>


https://github.com/skc7 ready_for_review https://github.com/llvm/llvm-project/pull/146029

From flang-commits at lists.llvm.org  Fri Jul  4 02:10:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:10:02 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Add support for
 workdistribute construct in flang frontend (PR #146029)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146029@github.com>
Message-ID: <68679a6a.a70a0220.25c452.54e4@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Chaitanya (skc7)

<details>
<summary>Changes</summary>

This PR introduces wordistribute construct support in flang frontend.
Also adds a workdistribute mlir op.

The work in this PR is C-P and updated from @<!-- -->ivanradanov commit from coexecute implementation:
[flang_workdistribute_iwomp_2024](https://github.com/ivanradanov/llvm-project/commits/flang_workdistribute_iwomp_2024)

---
Full diff: https://github.com/llvm/llvm-project/pull/146029.diff


10 Files Affected:

- (modified) flang/include/flang/Semantics/openmp-directive-sets.h (+14) 
- (modified) flang/lib/Lower/OpenMP/OpenMP.cpp (+25-1) 
- (modified) flang/lib/Parser/openmp-parsers.cpp (+6-1) 
- (modified) flang/lib/Semantics/resolve-directives.cpp (+7-1) 
- (added) flang/test/Lower/OpenMP/workdistribute.f90 (+59) 
- (modified) llvm/include/llvm/Frontend/OpenMP/OMP.td (+55) 
- (modified) mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td (+23) 
- (modified) mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp (+15) 
- (modified) mlir/test/Dialect/OpenMP/invalid.mlir (+21) 
- (modified) mlir/test/Dialect/OpenMP/ops.mlir (+13) 


``````````diff
diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index dd610c9702c28..7ced6ed9b44d6 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,6 +143,7 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
+    Directive::OMPD_target_teams_workdistribute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -172,6 +173,7 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
+    Directive::OMPD_teams_workdistribute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
@@ -187,9 +189,16 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
+        Directive::OMPD_target_teams_workdistribute,
     } | topTeamsSet,
 };
 
+static const OmpDirectiveSet allWorkdistributeSet{
+    Directive::OMPD_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_target_teams_workdistribute,
+};
+
 //===----------------------------------------------------------------------===//
 // Directive sets for groups of multiple directives
 //===----------------------------------------------------------------------===//
@@ -230,6 +239,9 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
+    Directive::OMPD_target_teams_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_workdistribute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -294,6 +306,7 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
+        Directive::OMPD_workdistribute,
     } | allDoSet,
 };
 
@@ -376,6 +389,7 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
+    Directive::OMPD_workdistribute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ebd1d038716e4..16d58b6be535f 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -585,6 +585,16 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
       cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
       break;
 
+    case OMPD_teams_workdistribute:
+      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      [[fallthrough]];
+    case OMPD_target_teams_workdistribute:
+      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      processSingleNestedIf([](Directive nestedDir) {
+        return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
+      });
+      break;
+
     // Standalone 'target' case.
     case OMPD_target: {
       processSingleNestedIf(
@@ -2682,6 +2692,17 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::omp::WorkdistributeOp genWorkdistributeOp(
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::Location loc, const ConstructQueue &queue,
+    ConstructQueue::const_iterator item) {
+  return genOpWithBody<mlir::omp::WorkdistributeOp>(
+      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
+                        llvm::omp::Directive::OMPD_workdistribute),
+      queue, item);
+}
+
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3302,7 +3323,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     TODO(loc, "Unhandled loop directive (" +
                   llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
   }
-  // case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_workdistribute:
+    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue,
+                                item);
+    break;
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
                            queue, item);
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..ad729932a5f00 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1492,12 +1492,17 @@ TYPE_PARSER(
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
+        "TARGET TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
+        "TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_teams_workdistribute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
-        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare))))
+        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
+        "WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_workdistribute))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..2e4e05f9e293b 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1656,10 +1656,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_teams_workdistribute:
     PushContext(beginDir.source, beginDir.v);
     break;
   default:
@@ -1689,9 +1692,12 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
-  case llvm::omp::Directive::OMPD_target_parallel: {
+  case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
+  case llvm::omp::Directive::OMPD_teams_workdistribute: {
     bool hasPrivate;
     for (const auto *allocName : allocateNames_) {
       hasPrivate = false;
diff --git a/flang/test/Lower/OpenMP/workdistribute.f90 b/flang/test/Lower/OpenMP/workdistribute.f90
new file mode 100644
index 0000000000000..924205bb72e5e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/workdistribute.f90
@@ -0,0 +1,59 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute
+subroutine target_teams_workdistribute()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end target teams workdistribute
+end subroutine target_teams_workdistribute
+
+! CHECK-LABEL: func @_QPteams_workdistribute
+subroutine teams_workdistribute()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end teams workdistribute
+end subroutine teams_workdistribute
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute_m
+subroutine target_teams_workdistribute_m()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+  !$omp end target
+end subroutine target_teams_workdistribute_m
+
+! CHECK-LABEL: func @_QPteams_workdistribute_m
+subroutine teams_workdistribute_m()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+end subroutine teams_workdistribute_m
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a87111cb5a11d..d1831db37fc46 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -1286,6 +1286,15 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
+def OMP_Workdistribute : Directive<[Spelling<"workdistribute">]> {
+  let association = AS_Block;
+  let category = CA_Executable;
+}
+def OMP_EndWorkdistribute : Directive<[Spelling<"end workdistribute">]> {
+  let leafConstructs = OMP_Workdistribute.leafConstructs;
+  let association = OMP_Workdistribute.association;
+  let category = OMP_Workdistribute.category;
+}
 
 //===----------------------------------------------------------------------===//
 // Definitions of OpenMP compound directives
@@ -2429,6 +2438,34 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_OMPX_DynCGroupMem>,
+    VersionedClause<OMPC_OMPX_Bare>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
@@ -2659,6 +2696,24 @@ def OMP_TeamsDistributeSimd : Directive<[Spelling<"teams distribute simd">]> {
   let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TeamsWorkdistribute : Directive<[Spelling<"teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_If, 52>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..a58e09d7bda71 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1887,4 +1887,27 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// workdistribute Construct
+//===----------------------------------------------------------------------===//
+
+def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
+  let summary = "workdistribute directive";
+  let description = [{
+    workdistribute divides execution of the enclosed structured block into
+    separate units of work, each executed only once by each
+    initial thread in the league.
+    ```
+    !$omp target teams
+        !$omp workdistribute
+        y = a * x + y 
+        !$omp end workdistribute
+    !$omp end target teams
+    ```
+  }];
+  let regions = (region AnyRegion:$region);
+  let hasVerifier = 1;
+  let assemblyFormat = "$region attr-dict";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..e2dd338829e76 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3493,6 +3493,21 @@ LogicalResult ScanOp::verify() {
                    "reduction modifier");
 }
 
+//===----------------------------------------------------------------------===//
+// WorkdistributeOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult WorkdistributeOp::verify() {
+  Region &region = getRegion();
+  if (!region.hasOneBlock())
+    return emitOpError("region must contain exactly one block");
+
+  Operation *parentOp = (*this)->getParentOp();
+  if (!llvm::dyn_cast<TeamsOp>(parentOp))
+    return emitOpError("workdistribute must be nested under teams");
+  return success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..522d20558a2b5 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -2960,3 +2960,24 @@ llvm.func @invalid_mapper(%0 : !llvm.ptr) {
   }
   llvm.return
 }
+
+func.func @invalid_workdistribute_with_multiple_blocks() {
+  // expected-error @below {{workdistribute must be nested under teams}}
+  omp.workdistribute {
+    omp.terminator
+  }
+  return
+}
+
+func.func @invalid_workdistribute_with_multiple_blocks() {
+  omp.teams {
+  // expected-error @below {{region must contain exactly one block}}
+  omp.workdistribute {
+    cf.br ^bb1
+  ^bb1:
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..af80284e53537 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -3197,3 +3197,16 @@ func.func @omp_workshare_loop_wrapper_attrs(%idx : index) {
   }
   return
 }
+
+// CHECK-LABEL: func @omp_workdistribute
+func.func @omp_workdistribute() {
+  // CHECK: omp.teams
+  omp.teams {
+  // CHECK: omp.workdistribute
+  omp.workdistribute {
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/146029

From flang-commits at lists.llvm.org  Fri Jul  4 02:10:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:10:02 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Add support for
 workdistribute construct in flang frontend (PR #146029)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146029@github.com>
Message-ID: <68679a6a.170a0220.137ae6.8694@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-parser

@llvm/pr-subscribers-flang-openmp

Author: Chaitanya (skc7)

<details>
<summary>Changes</summary>

This PR introduces wordistribute construct support in flang frontend.
Also adds a workdistribute mlir op.

The work in this PR is C-P and updated from @<!-- -->ivanradanov commit from coexecute implementation:
[flang_workdistribute_iwomp_2024](https://github.com/ivanradanov/llvm-project/commits/flang_workdistribute_iwomp_2024)

---
Full diff: https://github.com/llvm/llvm-project/pull/146029.diff


10 Files Affected:

- (modified) flang/include/flang/Semantics/openmp-directive-sets.h (+14) 
- (modified) flang/lib/Lower/OpenMP/OpenMP.cpp (+25-1) 
- (modified) flang/lib/Parser/openmp-parsers.cpp (+6-1) 
- (modified) flang/lib/Semantics/resolve-directives.cpp (+7-1) 
- (added) flang/test/Lower/OpenMP/workdistribute.f90 (+59) 
- (modified) llvm/include/llvm/Frontend/OpenMP/OMP.td (+55) 
- (modified) mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td (+23) 
- (modified) mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp (+15) 
- (modified) mlir/test/Dialect/OpenMP/invalid.mlir (+21) 
- (modified) mlir/test/Dialect/OpenMP/ops.mlir (+13) 


``````````diff
diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index dd610c9702c28..7ced6ed9b44d6 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,6 +143,7 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
+    Directive::OMPD_target_teams_workdistribute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -172,6 +173,7 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
+    Directive::OMPD_teams_workdistribute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
@@ -187,9 +189,16 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
+        Directive::OMPD_target_teams_workdistribute,
     } | topTeamsSet,
 };
 
+static const OmpDirectiveSet allWorkdistributeSet{
+    Directive::OMPD_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_target_teams_workdistribute,
+};
+
 //===----------------------------------------------------------------------===//
 // Directive sets for groups of multiple directives
 //===----------------------------------------------------------------------===//
@@ -230,6 +239,9 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
+    Directive::OMPD_target_teams_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_workdistribute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -294,6 +306,7 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
+        Directive::OMPD_workdistribute,
     } | allDoSet,
 };
 
@@ -376,6 +389,7 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
+    Directive::OMPD_workdistribute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ebd1d038716e4..16d58b6be535f 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -585,6 +585,16 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
       cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
       break;
 
+    case OMPD_teams_workdistribute:
+      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      [[fallthrough]];
+    case OMPD_target_teams_workdistribute:
+      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      processSingleNestedIf([](Directive nestedDir) {
+        return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
+      });
+      break;
+
     // Standalone 'target' case.
     case OMPD_target: {
       processSingleNestedIf(
@@ -2682,6 +2692,17 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::omp::WorkdistributeOp genWorkdistributeOp(
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::Location loc, const ConstructQueue &queue,
+    ConstructQueue::const_iterator item) {
+  return genOpWithBody<mlir::omp::WorkdistributeOp>(
+      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
+                        llvm::omp::Directive::OMPD_workdistribute),
+      queue, item);
+}
+
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3302,7 +3323,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     TODO(loc, "Unhandled loop directive (" +
                   llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
   }
-  // case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_workdistribute:
+    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue,
+                                item);
+    break;
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
                            queue, item);
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..ad729932a5f00 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1492,12 +1492,17 @@ TYPE_PARSER(
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
+        "TARGET TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
+        "TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_teams_workdistribute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
-        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare))))
+        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
+        "WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_workdistribute))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..2e4e05f9e293b 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1656,10 +1656,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_teams_workdistribute:
     PushContext(beginDir.source, beginDir.v);
     break;
   default:
@@ -1689,9 +1692,12 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
-  case llvm::omp::Directive::OMPD_target_parallel: {
+  case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
+  case llvm::omp::Directive::OMPD_teams_workdistribute: {
     bool hasPrivate;
     for (const auto *allocName : allocateNames_) {
       hasPrivate = false;
diff --git a/flang/test/Lower/OpenMP/workdistribute.f90 b/flang/test/Lower/OpenMP/workdistribute.f90
new file mode 100644
index 0000000000000..924205bb72e5e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/workdistribute.f90
@@ -0,0 +1,59 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute
+subroutine target_teams_workdistribute()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end target teams workdistribute
+end subroutine target_teams_workdistribute
+
+! CHECK-LABEL: func @_QPteams_workdistribute
+subroutine teams_workdistribute()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end teams workdistribute
+end subroutine teams_workdistribute
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute_m
+subroutine target_teams_workdistribute_m()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+  !$omp end target
+end subroutine target_teams_workdistribute_m
+
+! CHECK-LABEL: func @_QPteams_workdistribute_m
+subroutine teams_workdistribute_m()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+end subroutine teams_workdistribute_m
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a87111cb5a11d..d1831db37fc46 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -1286,6 +1286,15 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
+def OMP_Workdistribute : Directive<[Spelling<"workdistribute">]> {
+  let association = AS_Block;
+  let category = CA_Executable;
+}
+def OMP_EndWorkdistribute : Directive<[Spelling<"end workdistribute">]> {
+  let leafConstructs = OMP_Workdistribute.leafConstructs;
+  let association = OMP_Workdistribute.association;
+  let category = OMP_Workdistribute.category;
+}
 
 //===----------------------------------------------------------------------===//
 // Definitions of OpenMP compound directives
@@ -2429,6 +2438,34 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_OMPX_DynCGroupMem>,
+    VersionedClause<OMPC_OMPX_Bare>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
@@ -2659,6 +2696,24 @@ def OMP_TeamsDistributeSimd : Directive<[Spelling<"teams distribute simd">]> {
   let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TeamsWorkdistribute : Directive<[Spelling<"teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_If, 52>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..a58e09d7bda71 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1887,4 +1887,27 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// workdistribute Construct
+//===----------------------------------------------------------------------===//
+
+def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
+  let summary = "workdistribute directive";
+  let description = [{
+    workdistribute divides execution of the enclosed structured block into
+    separate units of work, each executed only once by each
+    initial thread in the league.
+    ```
+    !$omp target teams
+        !$omp workdistribute
+        y = a * x + y 
+        !$omp end workdistribute
+    !$omp end target teams
+    ```
+  }];
+  let regions = (region AnyRegion:$region);
+  let hasVerifier = 1;
+  let assemblyFormat = "$region attr-dict";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..e2dd338829e76 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3493,6 +3493,21 @@ LogicalResult ScanOp::verify() {
                    "reduction modifier");
 }
 
+//===----------------------------------------------------------------------===//
+// WorkdistributeOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult WorkdistributeOp::verify() {
+  Region &region = getRegion();
+  if (!region.hasOneBlock())
+    return emitOpError("region must contain exactly one block");
+
+  Operation *parentOp = (*this)->getParentOp();
+  if (!llvm::dyn_cast<TeamsOp>(parentOp))
+    return emitOpError("workdistribute must be nested under teams");
+  return success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..522d20558a2b5 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -2960,3 +2960,24 @@ llvm.func @invalid_mapper(%0 : !llvm.ptr) {
   }
   llvm.return
 }
+
+func.func @invalid_workdistribute_with_multiple_blocks() {
+  // expected-error @below {{workdistribute must be nested under teams}}
+  omp.workdistribute {
+    omp.terminator
+  }
+  return
+}
+
+func.func @invalid_workdistribute_with_multiple_blocks() {
+  omp.teams {
+  // expected-error @below {{region must contain exactly one block}}
+  omp.workdistribute {
+    cf.br ^bb1
+  ^bb1:
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..af80284e53537 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -3197,3 +3197,16 @@ func.func @omp_workshare_loop_wrapper_attrs(%idx : index) {
   }
   return
 }
+
+// CHECK-LABEL: func @omp_workdistribute
+func.func @omp_workdistribute() {
+  // CHECK: omp.teams
+  omp.teams {
+  // CHECK: omp.workdistribute
+  omp.workdistribute {
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/146029

From flang-commits at lists.llvm.org  Fri Jul  4 02:10:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:10:02 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Add support for
 workdistribute construct in flang frontend (PR #146029)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146029@github.com>
Message-ID: <68679a6a.050a0220.59462.4c81@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-mlir-openmp

Author: Chaitanya (skc7)

<details>
<summary>Changes</summary>

This PR introduces wordistribute construct support in flang frontend.
Also adds a workdistribute mlir op.

The work in this PR is C-P and updated from @<!-- -->ivanradanov commit from coexecute implementation:
[flang_workdistribute_iwomp_2024](https://github.com/ivanradanov/llvm-project/commits/flang_workdistribute_iwomp_2024)

---
Full diff: https://github.com/llvm/llvm-project/pull/146029.diff


10 Files Affected:

- (modified) flang/include/flang/Semantics/openmp-directive-sets.h (+14) 
- (modified) flang/lib/Lower/OpenMP/OpenMP.cpp (+25-1) 
- (modified) flang/lib/Parser/openmp-parsers.cpp (+6-1) 
- (modified) flang/lib/Semantics/resolve-directives.cpp (+7-1) 
- (added) flang/test/Lower/OpenMP/workdistribute.f90 (+59) 
- (modified) llvm/include/llvm/Frontend/OpenMP/OMP.td (+55) 
- (modified) mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td (+23) 
- (modified) mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp (+15) 
- (modified) mlir/test/Dialect/OpenMP/invalid.mlir (+21) 
- (modified) mlir/test/Dialect/OpenMP/ops.mlir (+13) 


``````````diff
diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index dd610c9702c28..7ced6ed9b44d6 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,6 +143,7 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
+    Directive::OMPD_target_teams_workdistribute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -172,6 +173,7 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
+    Directive::OMPD_teams_workdistribute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
@@ -187,9 +189,16 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
+        Directive::OMPD_target_teams_workdistribute,
     } | topTeamsSet,
 };
 
+static const OmpDirectiveSet allWorkdistributeSet{
+    Directive::OMPD_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_target_teams_workdistribute,
+};
+
 //===----------------------------------------------------------------------===//
 // Directive sets for groups of multiple directives
 //===----------------------------------------------------------------------===//
@@ -230,6 +239,9 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
+    Directive::OMPD_target_teams_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_workdistribute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -294,6 +306,7 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
+        Directive::OMPD_workdistribute,
     } | allDoSet,
 };
 
@@ -376,6 +389,7 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
+    Directive::OMPD_workdistribute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ebd1d038716e4..16d58b6be535f 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -585,6 +585,16 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
       cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
       break;
 
+    case OMPD_teams_workdistribute:
+      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      [[fallthrough]];
+    case OMPD_target_teams_workdistribute:
+      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      processSingleNestedIf([](Directive nestedDir) {
+        return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
+      });
+      break;
+
     // Standalone 'target' case.
     case OMPD_target: {
       processSingleNestedIf(
@@ -2682,6 +2692,17 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::omp::WorkdistributeOp genWorkdistributeOp(
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::Location loc, const ConstructQueue &queue,
+    ConstructQueue::const_iterator item) {
+  return genOpWithBody<mlir::omp::WorkdistributeOp>(
+      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
+                        llvm::omp::Directive::OMPD_workdistribute),
+      queue, item);
+}
+
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3302,7 +3323,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     TODO(loc, "Unhandled loop directive (" +
                   llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
   }
-  // case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_workdistribute:
+    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue,
+                                item);
+    break;
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
                            queue, item);
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..ad729932a5f00 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1492,12 +1492,17 @@ TYPE_PARSER(
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
+        "TARGET TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
+        "TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_teams_workdistribute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
-        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare))))
+        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
+        "WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_workdistribute))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..2e4e05f9e293b 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1656,10 +1656,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_teams_workdistribute:
     PushContext(beginDir.source, beginDir.v);
     break;
   default:
@@ -1689,9 +1692,12 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
-  case llvm::omp::Directive::OMPD_target_parallel: {
+  case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
+  case llvm::omp::Directive::OMPD_teams_workdistribute: {
     bool hasPrivate;
     for (const auto *allocName : allocateNames_) {
       hasPrivate = false;
diff --git a/flang/test/Lower/OpenMP/workdistribute.f90 b/flang/test/Lower/OpenMP/workdistribute.f90
new file mode 100644
index 0000000000000..924205bb72e5e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/workdistribute.f90
@@ -0,0 +1,59 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute
+subroutine target_teams_workdistribute()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end target teams workdistribute
+end subroutine target_teams_workdistribute
+
+! CHECK-LABEL: func @_QPteams_workdistribute
+subroutine teams_workdistribute()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end teams workdistribute
+end subroutine teams_workdistribute
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute_m
+subroutine target_teams_workdistribute_m()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+  !$omp end target
+end subroutine target_teams_workdistribute_m
+
+! CHECK-LABEL: func @_QPteams_workdistribute_m
+subroutine teams_workdistribute_m()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+end subroutine teams_workdistribute_m
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a87111cb5a11d..d1831db37fc46 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -1286,6 +1286,15 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
+def OMP_Workdistribute : Directive<[Spelling<"workdistribute">]> {
+  let association = AS_Block;
+  let category = CA_Executable;
+}
+def OMP_EndWorkdistribute : Directive<[Spelling<"end workdistribute">]> {
+  let leafConstructs = OMP_Workdistribute.leafConstructs;
+  let association = OMP_Workdistribute.association;
+  let category = OMP_Workdistribute.category;
+}
 
 //===----------------------------------------------------------------------===//
 // Definitions of OpenMP compound directives
@@ -2429,6 +2438,34 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_OMPX_DynCGroupMem>,
+    VersionedClause<OMPC_OMPX_Bare>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
@@ -2659,6 +2696,24 @@ def OMP_TeamsDistributeSimd : Directive<[Spelling<"teams distribute simd">]> {
   let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TeamsWorkdistribute : Directive<[Spelling<"teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_If, 52>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..a58e09d7bda71 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1887,4 +1887,27 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// workdistribute Construct
+//===----------------------------------------------------------------------===//
+
+def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
+  let summary = "workdistribute directive";
+  let description = [{
+    workdistribute divides execution of the enclosed structured block into
+    separate units of work, each executed only once by each
+    initial thread in the league.
+    ```
+    !$omp target teams
+        !$omp workdistribute
+        y = a * x + y 
+        !$omp end workdistribute
+    !$omp end target teams
+    ```
+  }];
+  let regions = (region AnyRegion:$region);
+  let hasVerifier = 1;
+  let assemblyFormat = "$region attr-dict";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..e2dd338829e76 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3493,6 +3493,21 @@ LogicalResult ScanOp::verify() {
                    "reduction modifier");
 }
 
+//===----------------------------------------------------------------------===//
+// WorkdistributeOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult WorkdistributeOp::verify() {
+  Region &region = getRegion();
+  if (!region.hasOneBlock())
+    return emitOpError("region must contain exactly one block");
+
+  Operation *parentOp = (*this)->getParentOp();
+  if (!llvm::dyn_cast<TeamsOp>(parentOp))
+    return emitOpError("workdistribute must be nested under teams");
+  return success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..522d20558a2b5 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -2960,3 +2960,24 @@ llvm.func @invalid_mapper(%0 : !llvm.ptr) {
   }
   llvm.return
 }
+
+func.func @invalid_workdistribute_with_multiple_blocks() {
+  // expected-error @below {{workdistribute must be nested under teams}}
+  omp.workdistribute {
+    omp.terminator
+  }
+  return
+}
+
+func.func @invalid_workdistribute_with_multiple_blocks() {
+  omp.teams {
+  // expected-error @below {{region must contain exactly one block}}
+  omp.workdistribute {
+    cf.br ^bb1
+  ^bb1:
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..af80284e53537 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -3197,3 +3197,16 @@ func.func @omp_workshare_loop_wrapper_attrs(%idx : index) {
   }
   return
 }
+
+// CHECK-LABEL: func @omp_workdistribute
+func.func @omp_workdistribute() {
+  // CHECK: omp.teams
+  omp.teams {
+  // CHECK: omp.workdistribute
+  omp.workdistribute {
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/146029

From flang-commits at lists.llvm.org  Fri Jul  4 02:10:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:10:02 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Add support for
 workdistribute construct in flang frontend (PR #146029)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146029@github.com>
Message-ID: <68679a6a.170a0220.144e3d.4653@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-mlir

Author: Chaitanya (skc7)

<details>
<summary>Changes</summary>

This PR introduces wordistribute construct support in flang frontend.
Also adds a workdistribute mlir op.

The work in this PR is C-P and updated from @<!-- -->ivanradanov commit from coexecute implementation:
[flang_workdistribute_iwomp_2024](https://github.com/ivanradanov/llvm-project/commits/flang_workdistribute_iwomp_2024)

---
Full diff: https://github.com/llvm/llvm-project/pull/146029.diff


10 Files Affected:

- (modified) flang/include/flang/Semantics/openmp-directive-sets.h (+14) 
- (modified) flang/lib/Lower/OpenMP/OpenMP.cpp (+25-1) 
- (modified) flang/lib/Parser/openmp-parsers.cpp (+6-1) 
- (modified) flang/lib/Semantics/resolve-directives.cpp (+7-1) 
- (added) flang/test/Lower/OpenMP/workdistribute.f90 (+59) 
- (modified) llvm/include/llvm/Frontend/OpenMP/OMP.td (+55) 
- (modified) mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td (+23) 
- (modified) mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp (+15) 
- (modified) mlir/test/Dialect/OpenMP/invalid.mlir (+21) 
- (modified) mlir/test/Dialect/OpenMP/ops.mlir (+13) 


``````````diff
diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index dd610c9702c28..7ced6ed9b44d6 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,6 +143,7 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
+    Directive::OMPD_target_teams_workdistribute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -172,6 +173,7 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
+    Directive::OMPD_teams_workdistribute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
@@ -187,9 +189,16 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
+        Directive::OMPD_target_teams_workdistribute,
     } | topTeamsSet,
 };
 
+static const OmpDirectiveSet allWorkdistributeSet{
+    Directive::OMPD_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_target_teams_workdistribute,
+};
+
 //===----------------------------------------------------------------------===//
 // Directive sets for groups of multiple directives
 //===----------------------------------------------------------------------===//
@@ -230,6 +239,9 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
+    Directive::OMPD_target_teams_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_workdistribute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -294,6 +306,7 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
+        Directive::OMPD_workdistribute,
     } | allDoSet,
 };
 
@@ -376,6 +389,7 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
+    Directive::OMPD_workdistribute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ebd1d038716e4..16d58b6be535f 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -585,6 +585,16 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
       cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
       break;
 
+    case OMPD_teams_workdistribute:
+      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      [[fallthrough]];
+    case OMPD_target_teams_workdistribute:
+      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      processSingleNestedIf([](Directive nestedDir) {
+        return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
+      });
+      break;
+
     // Standalone 'target' case.
     case OMPD_target: {
       processSingleNestedIf(
@@ -2682,6 +2692,17 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::omp::WorkdistributeOp genWorkdistributeOp(
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::Location loc, const ConstructQueue &queue,
+    ConstructQueue::const_iterator item) {
+  return genOpWithBody<mlir::omp::WorkdistributeOp>(
+      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
+                        llvm::omp::Directive::OMPD_workdistribute),
+      queue, item);
+}
+
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3302,7 +3323,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     TODO(loc, "Unhandled loop directive (" +
                   llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
   }
-  // case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_workdistribute:
+    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue,
+                                item);
+    break;
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
                            queue, item);
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..ad729932a5f00 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1492,12 +1492,17 @@ TYPE_PARSER(
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
+        "TARGET TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
+        "TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_teams_workdistribute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
-        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare))))
+        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
+        "WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_workdistribute))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..2e4e05f9e293b 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1656,10 +1656,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_teams_workdistribute:
     PushContext(beginDir.source, beginDir.v);
     break;
   default:
@@ -1689,9 +1692,12 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
-  case llvm::omp::Directive::OMPD_target_parallel: {
+  case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
+  case llvm::omp::Directive::OMPD_teams_workdistribute: {
     bool hasPrivate;
     for (const auto *allocName : allocateNames_) {
       hasPrivate = false;
diff --git a/flang/test/Lower/OpenMP/workdistribute.f90 b/flang/test/Lower/OpenMP/workdistribute.f90
new file mode 100644
index 0000000000000..924205bb72e5e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/workdistribute.f90
@@ -0,0 +1,59 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute
+subroutine target_teams_workdistribute()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end target teams workdistribute
+end subroutine target_teams_workdistribute
+
+! CHECK-LABEL: func @_QPteams_workdistribute
+subroutine teams_workdistribute()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end teams workdistribute
+end subroutine teams_workdistribute
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute_m
+subroutine target_teams_workdistribute_m()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+  !$omp end target
+end subroutine target_teams_workdistribute_m
+
+! CHECK-LABEL: func @_QPteams_workdistribute_m
+subroutine teams_workdistribute_m()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+end subroutine teams_workdistribute_m
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a87111cb5a11d..d1831db37fc46 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -1286,6 +1286,15 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
+def OMP_Workdistribute : Directive<[Spelling<"workdistribute">]> {
+  let association = AS_Block;
+  let category = CA_Executable;
+}
+def OMP_EndWorkdistribute : Directive<[Spelling<"end workdistribute">]> {
+  let leafConstructs = OMP_Workdistribute.leafConstructs;
+  let association = OMP_Workdistribute.association;
+  let category = OMP_Workdistribute.category;
+}
 
 //===----------------------------------------------------------------------===//
 // Definitions of OpenMP compound directives
@@ -2429,6 +2438,34 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_OMPX_DynCGroupMem>,
+    VersionedClause<OMPC_OMPX_Bare>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
@@ -2659,6 +2696,24 @@ def OMP_TeamsDistributeSimd : Directive<[Spelling<"teams distribute simd">]> {
   let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TeamsWorkdistribute : Directive<[Spelling<"teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_If, 52>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..a58e09d7bda71 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1887,4 +1887,27 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// workdistribute Construct
+//===----------------------------------------------------------------------===//
+
+def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
+  let summary = "workdistribute directive";
+  let description = [{
+    workdistribute divides execution of the enclosed structured block into
+    separate units of work, each executed only once by each
+    initial thread in the league.
+    ```
+    !$omp target teams
+        !$omp workdistribute
+        y = a * x + y 
+        !$omp end workdistribute
+    !$omp end target teams
+    ```
+  }];
+  let regions = (region AnyRegion:$region);
+  let hasVerifier = 1;
+  let assemblyFormat = "$region attr-dict";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..e2dd338829e76 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3493,6 +3493,21 @@ LogicalResult ScanOp::verify() {
                    "reduction modifier");
 }
 
+//===----------------------------------------------------------------------===//
+// WorkdistributeOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult WorkdistributeOp::verify() {
+  Region &region = getRegion();
+  if (!region.hasOneBlock())
+    return emitOpError("region must contain exactly one block");
+
+  Operation *parentOp = (*this)->getParentOp();
+  if (!llvm::dyn_cast<TeamsOp>(parentOp))
+    return emitOpError("workdistribute must be nested under teams");
+  return success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..522d20558a2b5 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -2960,3 +2960,24 @@ llvm.func @invalid_mapper(%0 : !llvm.ptr) {
   }
   llvm.return
 }
+
+func.func @invalid_workdistribute_with_multiple_blocks() {
+  // expected-error @below {{workdistribute must be nested under teams}}
+  omp.workdistribute {
+    omp.terminator
+  }
+  return
+}
+
+func.func @invalid_workdistribute_with_multiple_blocks() {
+  omp.teams {
+  // expected-error @below {{region must contain exactly one block}}
+  omp.workdistribute {
+    cf.br ^bb1
+  ^bb1:
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..af80284e53537 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -3197,3 +3197,16 @@ func.func @omp_workshare_loop_wrapper_attrs(%idx : index) {
   }
   return
 }
+
+// CHECK-LABEL: func @omp_workdistribute
+func.func @omp_workdistribute() {
+  // CHECK: omp.teams
+  omp.teams {
+  // CHECK: omp.workdistribute
+  omp.workdistribute {
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/146029

From flang-commits at lists.llvm.org  Fri Jul  4 02:10:44 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:10:44 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Implement
 workdistribute construct lowering (PR #140523)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140523@github.com>
Message-ID: <68679a94.050a0220.2e9039.5661@mx.google.com>


https://github.com/skc7 edited https://github.com/llvm/llvm-project/pull/140523

From flang-commits at lists.llvm.org  Fri Jul  4 02:11:05 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:11:05 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Implement
 workdistribute construct lowering (PR #140523)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140523@github.com>
Message-ID: <68679aa9.630a0220.a0b2b.59e4@mx.google.com>


https://github.com/skc7 ready_for_review https://github.com/llvm/llvm-project/pull/140523

From flang-commits at lists.llvm.org  Fri Jul  4 02:21:08 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:21:08 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] use set_union instead of merge in
 added DerivedTypeCache (PR #147024)
Message-ID: <llvm.org/llvm/llvm-project/pull/147024@github.com>


https://github.com/jeanPerier created https://github.com/llvm/llvm-project/pull/147024

When merging the list of recursive reference under two components, duplicates should be removed.
If the recursive reference to parents nodes (referred by depth of the parents node) are [1, 2, 5] and [4, 5], the new list should be [1,2,4,5].

With std::merge, the order was correct but 5 was duplicated use std::set_union instead that removes duplicates.

With this patch Fujitsu tests [0394_0030.f90](https://github.com/fujitsu/compiler-test-suite/blob/0d02267bb98b6bfdf46d1f6bbd92e9781c24356c/Fortran/0394/0394_0030.f90) and [0390_0230.f90](https://github.com/fujitsu/compiler-test-suite/blob/0d02267bb98b6bfdf46d1f6bbd92e9781c24356c/Fortran/0390/0390_0230.f90) finally compile with -g in about 10s. Their compilation was hanging before https://github.com/llvm/llvm-project/pull/146543, and they were now hitting an error "LLVM ERROR: SmallVector unable to grow" which is fixed by this patch.

>From f7a6224e3037b45a9ba3e24caeb4a489e15f1db6 Mon Sep 17 00:00:00 2001
From: Jean Perier <jperier at nvidia.com>
Date: Fri, 4 Jul 2025 02:08:49 -0700
Subject: [PATCH] [flang] use set_union instead of merge in added
 DerivedTypeCache

---
 flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index a848058486e2c..abad500d3f657 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -303,9 +303,9 @@ void DerivedTypeCache::postComponentVisitUpdate(
     return;
   ActiveLevels oldLevels;
   oldLevels.swap(activeRecursionLevels);
-  std::merge(componentActiveRecursionLevels.begin(),
-             componentActiveRecursionLevels.end(), oldLevels.begin(),
-             oldLevels.end(), std::back_inserter(activeRecursionLevels));
+  std::set_union(componentActiveRecursionLevels.begin(),
+                 componentActiveRecursionLevels.end(), oldLevels.begin(),
+                 oldLevels.end(), std::back_inserter(activeRecursionLevels));
 }
 
 void DerivedTypeCache::finalize(mlir::Type ty, mlir::LLVM::DITypeAttr attr,


From flang-commits at lists.llvm.org  Fri Jul  4 02:21:39 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:21:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] use set_union instead of merge in
 added DerivedTypeCache (PR #147024)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147024@github.com>
Message-ID: <68679d23.630a0220.31133f.2135@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: None (jeanPerier)

<details>
<summary>Changes</summary>

When merging the list of recursive reference under two components, duplicates should be removed.
If the recursive reference to parents nodes (referred by depth of the parents node) are [1, 2, 5] and [4, 5], the new list should be [1,2,4,5].

With std::merge, the order was correct but 5 was duplicated use std::set_union instead that removes duplicates.

With this patch Fujitsu tests [0394_0030.f90](https://github.com/fujitsu/compiler-test-suite/blob/0d02267bb98b6bfdf46d1f6bbd92e9781c24356c/Fortran/0394/0394_0030.f90) and [0390_0230.f90](https://github.com/fujitsu/compiler-test-suite/blob/0d02267bb98b6bfdf46d1f6bbd92e9781c24356c/Fortran/0390/0390_0230.f90) finally compile with -g in about 10s. Their compilation was hanging before https://github.com/llvm/llvm-project/pull/146543, and they were now hitting an error "LLVM ERROR: SmallVector unable to grow" which is fixed by this patch.

---
Full diff: https://github.com/llvm/llvm-project/pull/147024.diff


1 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp (+3-3) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index a848058486e2c..abad500d3f657 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -303,9 +303,9 @@ void DerivedTypeCache::postComponentVisitUpdate(
     return;
   ActiveLevels oldLevels;
   oldLevels.swap(activeRecursionLevels);
-  std::merge(componentActiveRecursionLevels.begin(),
-             componentActiveRecursionLevels.end(), oldLevels.begin(),
-             oldLevels.end(), std::back_inserter(activeRecursionLevels));
+  std::set_union(componentActiveRecursionLevels.begin(),
+                 componentActiveRecursionLevels.end(), oldLevels.begin(),
+                 oldLevels.end(), std::back_inserter(activeRecursionLevels));
 }
 
 void DerivedTypeCache::finalize(mlir::Type ty, mlir::LLVM::DITypeAttr attr,

``````````

</details>


https://github.com/llvm/llvm-project/pull/147024

From flang-commits at lists.llvm.org  Fri Jul  4 02:32:09 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Fri, 04 Jul 2025 02:32:09 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68679f99.a70a0220.248669.4bc1@mx.google.com>


================
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 #define FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 
+#include "flang/Frontend/CodeGenOptions.h"
----------------
s-watanabe314 wrote:

Thank you for the reviews!
> Not requesting to do that here, but I feel the CodeGenOptions should be defined in Codegen and used/set in Frontend rather than having Codegen depends on Frontend things I think.

Does this mean that `ComplexRangeKind` should be defined in `CodeGen.h` instead of `CodeGenOptions.h`, and that CodeGenOptions should reference it?

>I think the CodeGenOptions.h header is included because of the ComplexRangeKind enum. If that is moved to llvm/Frontend/Driver/CodeGenOptions.h, we don't have this issue. It may be worth doing it in this PR, but I am ok with moving it in a separate PR as well.

Yes, I'm including `CodeGenOptions.h` to use the `ComplexRangeKind` enum. I'll also try moving it to `llvm/Frontend/Driver/CodeGenOptions.h`,  but I think that would be a separate PR.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Fri Jul  4 02:35:56 2025
From: flang-commits at lists.llvm.org (Abid Qadeer via flang-commits)
Date: Fri, 04 Jul 2025 02:35:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] use set_union instead of merge in
 added DerivedTypeCache (PR #147024)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147024@github.com>
Message-ID: <6867a07c.630a0220.19d87a.57c7@mx.google.com>


https://github.com/abidh approved this pull request.

Thanks. This looks good to me.

https://github.com/llvm/llvm-project/pull/147024

From flang-commits at lists.llvm.org  Fri Jul  4 02:56:48 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Fri, 04 Jul 2025 02:56:48 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP][NFC] Remove flag toggling
 deprecated no hlfir flow in map-types-and-sizes.f90 (PR #146995)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146995@github.com>
Message-ID: <6867a560.170a0220.3c9954.4851@mx.google.com>


https://github.com/TIFitis approved this pull request.

LGTM 👍🏽

https://github.com/llvm/llvm-project/pull/146995

From flang-commits at lists.llvm.org  Fri Jul  4 03:47:05 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 04 Jul 2025 03:47:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Enable kinds01/02.f90 and
 update expected output (PR #146877)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146877@github.com>
Message-ID: <6867b129.170a0220.37348f.4ff3@mx.google.com>


DavidSpickett wrote:

kinds01 is failing CI but passing on my AArch64 Linux machine, though I would expect it to pass anywhere because of the -target.

https://github.com/llvm/llvm-project/actions/runs/16051206281?pr=146877

Perhaps this part of flang is still not cross compilation aware?

https://github.com/llvm/llvm-project/pull/146877

From flang-commits at lists.llvm.org  Fri Jul  4 03:48:09 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 03:48:09 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <6867b169.630a0220.279a5b.5efc@mx.google.com>


https://github.com/EbinJose2002 edited https://github.com/llvm/llvm-project/pull/146851

From flang-commits at lists.llvm.org  Fri Jul  4 04:49:54 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 04:49:54 -0700 (PDT)
Subject: [flang-commits] [flang] 0ba5958 - [flang] Optimize assignments of
 multidimensional arrays (#146408)
Message-ID: <6867bfe2.630a0220.1a1556.5cae@mx.google.com>


Author: Leandro Lupori
Date: 2025-07-04T08:49:51-03:00
New Revision: 0ba59587fa98849ed5107fee4134e810e84b69a3

URL: https://github.com/llvm/llvm-project/commit/0ba59587fa98849ed5107fee4134e810e84b69a3
DIFF: https://github.com/llvm/llvm-project/commit/0ba59587fa98849ed5107fee4134e810e84b69a3.diff

LOG: [flang] Optimize assignments of multidimensional arrays (#146408)

Assignments of n-dimensional arrays, with trivial RHS, were
always being converted to n nested loops. For contiguous arrays,
it's possible to flatten them and use a single loop, that can
usually be better optimized by LLVM.

In a test program, using a 3-dimensional array and varying its
size, the resulting speedup was as follows (measured on Graviton4):

16K     1.09
64K     1.40
128K    1.90
256K    1.91
512K    1.00

For sizes above or equal to 512K no improvement was observed.
It looks like LLVM stops trying to perform aggressive loop
unrolling at a certain threshold and just uses nested loops
instead. Larger sizes won't fit on L1 and L2 caches too.

This was noticed while profiling 527.cam4_r. This optimization
makes aer_rad_props_sw slightly faster, but unfortunately it
practically doesn't change 527.cam4_r total execution time.

Added: 
    

Modified: 
    flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
    flang/test/HLFIR/opt-scalar-assign.fir

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 91df8672c20d9..54892ef99bf58 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -21,6 +21,7 @@
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/HLFIR/Passes.h"
 #include "flang/Optimizer/OpenMP/Passes.h"
+#include "flang/Optimizer/Support/Utils.h"
 #include "flang/Optimizer/Transforms/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/Dominance.h"
@@ -786,13 +787,55 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   mlir::Value shape = hlfir::genShape(loc, builder, lhs);
   llvm::SmallVector<mlir::Value> extents =
       hlfir::getIndexExtents(loc, builder, shape);
-  hlfir::LoopNest loopNest =
-      hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
-                         flangomp::shouldUseWorkshareLowering(assign));
-  builder.setInsertionPointToStart(loopNest.body);
-  auto arrayElement =
-      hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-  builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+
+  if (lhs.isSimplyContiguous() && extents.size() > 1) {
+    // Flatten the array to use a single assign loop, that can be better
+    // optimized.
+    mlir::Value n = extents[0];
+    for (size_t i = 1; i < extents.size(); ++i)
+      n = builder.create<mlir::arith::MulIOp>(loc, n, extents[i]);
+    llvm::SmallVector<mlir::Value> flatExtents = {n};
+
+    mlir::Type flatArrayType;
+    mlir::Value flatArray = lhs.getBase();
+    if (mlir::isa<fir::BoxType>(lhs.getType())) {
+      shape = builder.genShape(loc, flatExtents);
+      flatArrayType = fir::BoxType::get(fir::SequenceType::get(eleTy, 1));
+      flatArray = builder.create<fir::ReboxOp>(loc, flatArrayType, flatArray,
+                                               shape, /*slice=*/mlir::Value{});
+    } else {
+      // Array references must have fixed shape, when used in assignments.
+      int64_t flatExtent = 1;
+      for (const mlir::Value &extent : extents) {
+        mlir::Operation *op = extent.getDefiningOp();
+        assert(op && "no defining operation for constant array extent");
+        flatExtent *= fir::toInt(mlir::cast<mlir::arith::ConstantOp>(*op));
+      }
+
+      flatArrayType =
+          fir::ReferenceType::get(fir::SequenceType::get({flatExtent}, eleTy));
+      flatArray = builder.createConvert(loc, flatArrayType, flatArray);
+    }
+
+    hlfir::LoopNest loopNest =
+        hlfir::genLoopNest(loc, builder, flatExtents, /*isUnordered=*/true,
+                           flangomp::shouldUseWorkshareLowering(assign));
+    builder.setInsertionPointToStart(loopNest.body);
+
+    mlir::Value arrayElement =
+        builder.create<hlfir::DesignateOp>(loc, fir::ReferenceType::get(eleTy),
+                                           flatArray, loopNest.oneBasedIndices);
+    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+  } else {
+    hlfir::LoopNest loopNest =
+        hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
+                           flangomp::shouldUseWorkshareLowering(assign));
+    builder.setInsertionPointToStart(loopNest.body);
+    auto arrayElement =
+        hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
+    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+  }
+
   rewriter.eraseOp(assign);
   return mlir::success();
 }

diff  --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 02ab02945b042..74cdcd9622adb 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -12,18 +12,18 @@ func.func @_QPtest1() {
   return
 }
 // CHECK-LABEL:   func.func @_QPtest1() {
-// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[VAL_2:.*]] = arith.constant 11 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 13 : index
-// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
-// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_2]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_4]](%[[VAL_5]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
-// CHECK:           fir.do_loop %[[VAL_7:.*]] = %[[VAL_0]] to %[[VAL_3]] step %[[VAL_0]] unordered {
-// CHECK:             fir.do_loop %[[VAL_8:.*]] = %[[VAL_0]] to %[[VAL_2]] step %[[VAL_0]] unordered {
-// CHECK:               %[[VAL_9:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_8]], %[[VAL_7]])  : (!fir.ref<!fir.array<11x13xf32>>, index, index) -> !fir.ref<f32>
-// CHECK:               hlfir.assign %[[VAL_1]] to %[[VAL_9]] : f32, !fir.ref<f32>
-// CHECK:             }
+// CHECK:           %[[VAL_0:.*]] = arith.constant 143 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 11 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 13 : index
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_3]], %[[VAL_4]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.array<11x13xf32>>) -> !fir.ref<!fir.array<143xf32>>
+// CHECK:           fir.do_loop %[[VAL_9:.*]] = %[[VAL_1]] to %[[VAL_0]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_10:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_9]]) : (!fir.ref<!fir.array<143xf32>>, index) -> !fir.ref<f32>
+// CHECK:             hlfir.assign %[[VAL_2]] to %[[VAL_10]] : f32, !fir.ref<f32>
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
@@ -129,3 +129,29 @@ func.func @_QPtest5(%arg0: !fir.ref<!fir.array<77xcomplex<f32>>> {fir.bindc_name
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @_QPtest6(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
+  %c0_i32 = arith.constant 0 : i32
+  %0:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+  hlfir.assign %c0_i32 to %0#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPtest6(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_1]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = arith.muli %[[VAL_6]]#1, %[[VAL_7]]#1 : index
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_5]](%[[VAL_9]]) : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           fir.do_loop %[[VAL_11:.*]] = %[[VAL_1]] to %[[VAL_8]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_10]] (%[[VAL_11]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_3]] to %[[VAL_12]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }


        

From flang-commits at lists.llvm.org  Fri Jul  4 05:27:50 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Fri, 04 Jul 2025 05:27:50 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][OpenMP] Enable tiling
	(PR #143715)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143715@github.com>
Message-ID: <6867c8c6.170a0220.61fb5.4df1@mx.google.com>


kparzysz wrote:

Is anybody working on resolving the conflicts?

https://github.com/llvm/llvm-project/pull/143715

From flang-commits at lists.llvm.org  Fri Jul  4 05:40:54 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Fri, 04 Jul 2025 05:40:54 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <6867cbd6.170a0220.a8b32.4ce6@mx.google.com>


================
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
----------------
kiranchandramohan wrote:

Looks Good. Would you mind adding some information to https://github.com/llvm/llvm-project/blob/main/flang/docs/ComplexOperations.md as well?

The Clang documentation in `clang/docs/UsersManual.rst` is more detailed.


https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Fri Jul  4 05:42:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 05:42:45 -0700 (PDT)
Subject: [flang-commits] [flang] 274e798 - [flang] use set_union instead of
 merge in added DerivedTypeCache (#147024)
Message-ID: <6867cc45.170a0220.12a3c7.4dcb@mx.google.com>


Author: jeanPerier
Date: 2025-07-04T14:42:42+02:00
New Revision: 274e798a9832b582b29b069009dbab123070a940

URL: https://github.com/llvm/llvm-project/commit/274e798a9832b582b29b069009dbab123070a940
DIFF: https://github.com/llvm/llvm-project/commit/274e798a9832b582b29b069009dbab123070a940.diff

LOG: [flang] use set_union instead of merge in added DerivedTypeCache (#147024)

When merging the list of recursive reference under two components,
duplicates should be removed.
If the recursive reference to parents nodes (referred by depth of the
parents node) are [1, 2, 5] and [4, 5], the new list should be [1,2,4,5].

With std::merge the order was correct but 5 was duplicated. Use
std::set_union instead that removes duplicates.

With this patch Fujitsu tests 0394_0030.f90 [1] and 0390_0230.f90 [2]()
finally compile with -g in about 10s. Their compilation was hanging
before #146543, and they were now hitting an error:
"LLVM ERROR: SmallVector unable to grow" which is fixed by this patch.

[1]: https://github.com/fujitsu/compiler-test-suite/blob/0d02267bb98b6bfdf46d1f6bbd92e9781c24356c/Fortran/0394/0394_0030.f90
[2]: https://github.com/fujitsu/compiler-test-suite/blob/0d02267bb98b6bfdf46d1f6bbd92e9781c24356c/Fortran/0390/0390_0230.f90

Added: 
    

Modified: 
    flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index a848058486e2c..abad500d3f657 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -303,9 +303,9 @@ void DerivedTypeCache::postComponentVisitUpdate(
     return;
   ActiveLevels oldLevels;
   oldLevels.swap(activeRecursionLevels);
-  std::merge(componentActiveRecursionLevels.begin(),
-             componentActiveRecursionLevels.end(), oldLevels.begin(),
-             oldLevels.end(), std::back_inserter(activeRecursionLevels));
+  std::set_union(componentActiveRecursionLevels.begin(),
+                 componentActiveRecursionLevels.end(), oldLevels.begin(),
+                 oldLevels.end(), std::back_inserter(activeRecursionLevels));
 }
 
 void DerivedTypeCache::finalize(mlir::Type ty, mlir::LLVM::DITypeAttr attr,


        

From flang-commits at lists.llvm.org  Fri Jul  4 05:42:48 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 05:42:48 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] use set_union instead of merge in
 added DerivedTypeCache (PR #147024)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147024@github.com>
Message-ID: <6867cc48.170a0220.3c2358.4c6c@mx.google.com>


https://github.com/jeanPerier closed https://github.com/llvm/llvm-project/pull/147024

From flang-commits at lists.llvm.org  Fri Jul  4 06:50:33 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 06:50:33 -0700 (PDT)
Subject: [flang-commits] [flang] ed17bf1 - [flang] Fix tests broken by
	#146734 (#147055)
Message-ID: <6867dc29.170a0220.1922f0.47b5@mx.google.com>


Author: Tom Eccles
Date: 2025-07-04T14:50:29+01:00
New Revision: ed17bf1e4c545bad277c8a0a01220124375810b1

URL: https://github.com/llvm/llvm-project/commit/ed17bf1e4c545bad277c8a0a01220124375810b1
DIFF: https://github.com/llvm/llvm-project/commit/ed17bf1e4c545bad277c8a0a01220124375810b1.diff

LOG: [flang] Fix tests broken by #146734 (#147055)

These tests referred to privatizers which were never declared

Added: 
    

Modified: 
    flang/test/Fir/convert-nontemporal-to-llvm.fir
    flang/test/Fir/simd-nontemporal.fir

Removed: 
    


################################################################################
diff  --git a/flang/test/Fir/convert-nontemporal-to-llvm.fir b/flang/test/Fir/convert-nontemporal-to-llvm.fir
index 6200ef1c621d7..e0a836ab0dad1 100644
--- a/flang/test/Fir/convert-nontemporal-to-llvm.fir
+++ b/flang/test/Fir/convert-nontemporal-to-llvm.fir
@@ -14,7 +14,7 @@
 // CHECK:    %[[VAL5:.*]] = llvm.alloca %[[CONST_VAL4]] x i32 {bindc_name = "a"} : (i64) -> !llvm.ptr
 // CHECK:    %[[CONST_VAL5:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:    %[[VAL6:.*]] = llvm.load %[[VAL1]] : !llvm.ptr -> i32
-// CHECK:    omp.simd nontemporal(%[[VAL5]], %[[VAL3]] : !llvm.ptr, !llvm.ptr) private(@_QFtestEi_private_i32 %[[VAL2]] -> %arg0 : !llvm.ptr) {
+// CHECK:    omp.simd nontemporal(%[[VAL5]], %[[VAL3]] : !llvm.ptr, !llvm.ptr) {
 // CHECK:      omp.loop_nest (%{{.*}}) : i32 = (%[[CONST_VAL5]]) to (%[[VAL6]]) inclusive step (%[[CONST_VAL5]]) {
 // CHECK:        llvm.store %{{.*}}, %{{.*}} : i32, !llvm.ptr
 // CHECK:        %[[VAL8:.*]] = llvm.load %[[VAL5]] {nontemporal} : !llvm.ptr -> i32
@@ -33,9 +33,9 @@
     %3 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtestEi"}
     %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFtestEn"}
     %5 = fir.load %4 : !fir.ref<i32>
-    omp.simd nontemporal(%0, %2 : !fir.ref<i32>, !fir.ref<i32>) private(@_QFtestEi_private_i32 %3 -> %arg0 : !fir.ref<i32>) {
+    omp.simd nontemporal(%0, %2 : !fir.ref<i32>, !fir.ref<i32>) {
       omp.loop_nest (%arg1) : i32 = (%c1_i32) to (%5) inclusive step (%c1_i32) {
-        fir.store %arg1 to %arg0 : !fir.ref<i32>
+        fir.store %arg1 to %3 : !fir.ref<i32>
         %6 = fir.load %0 {nontemporal}: !fir.ref<i32>
         %7 = fir.load %1 : !fir.ref<i32>
         %8 = arith.addi %6, %7 : i32
@@ -52,12 +52,12 @@
 // CHECK:    %[[IDX_VAL:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:    %[[CONST_VAL1:.*]] = llvm.mlir.constant(0 : index) : i64
 // CHECK:    %[[END_IDX:.*]] = llvm.mlir.constant(100 : i32) : i32
-// CHECK:    omp.simd nontemporal(%[[ARG0:.*]] : !llvm.ptr) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %[[ALLOCA2]] -> %[[ARG2:.*]] : !llvm.ptr) {
+// CHECK:    omp.simd nontemporal(%[[ARG0:.*]] : !llvm.ptr) {
 // CHECK:      omp.loop_nest (%[[ARG3:.*]]) : i32 = (%[[IDX_VAL]]) to (%[[END_IDX]]) inclusive step (%[[IDX_VAL]]) {
-// CHECK:        llvm.store %[[ARG3]], %[[ARG2]] : i32, !llvm.ptr
+// CHECK:        llvm.store %[[ARG3]], %{{.*}} : i32, !llvm.ptr
 // CHECK:        %[[CONST_VAL2:.*]] = llvm.mlir.constant(48 : i32) : i32
 // CHECK:        "llvm.intr.memcpy"(%[[ALLOCA1:.*]], %[[ARG0]], %[[CONST_VAL2]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
-// CHECK:        %[[VAL1:.*]] = llvm.load %[[ARG2]] : !llvm.ptr -> i32
+// CHECK:        %[[VAL1:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32
 // CHECK:        %[[VAL2:.*]] = llvm.sext %[[VAL1]] : i32 to i64
 // CHECK:        %[[VAL3:.*]] = llvm.getelementptr %[[ALLOCA1]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
 // CHECK:        %[[VAL4:.*]] = llvm.load %[[VAL3]] : !llvm.ptr -> !llvm.ptr
@@ -91,11 +91,11 @@
     %1 = fir.allocmem !fir.array<?xi32>, %c100 {fir.must_be_heap = true, uniq_name = "_QFsimd_nontemporal_allocatableEx.alloc"}
     %2 = fircg.ext_embox %1(%c100) : (!fir.heap<!fir.array<?xi32>>, index) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
     fir.store %2 to %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    omp.simd nontemporal(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %0 -> %arg2 : !fir.ref<i32>) {
+    omp.simd nontemporal(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
       omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
-        fir.store %arg3 to %arg2 : !fir.ref<i32>
+        fir.store %arg3 to %0 : !fir.ref<i32>
         %7 = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-        %8 = fir.load %arg2 : !fir.ref<i32>
+        %8 = fir.load %0 : !fir.ref<i32>
         %9 = fir.convert %8 : (i32) -> i64
         %10 = fir.box_addr %7 : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
         %11:3 = fir.box_dims %7, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)

diff  --git a/flang/test/Fir/simd-nontemporal.fir b/flang/test/Fir/simd-nontemporal.fir
index 31051ff52f9bd..a58bf01a60980 100644
--- a/flang/test/Fir/simd-nontemporal.fir
+++ b/flang/test/Fir/simd-nontemporal.fir
@@ -17,11 +17,11 @@ func.func @_QPsimd_with_nontemporal_clause(%arg0: !fir.ref<i32> {fir.bindc_name
     %8 = fir.declare %7 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
     %9 = fir.declare %arg0 dummy_scope %0 {uniq_name = "_QFsimd_with_nontemporal_clauseEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
     %10 = fir.load %9 : !fir.ref<i32>
-    // CHECK: omp.simd nontemporal(%[[A_DECL]], %[[C_DECL]] : !fir.ref<i32>, !fir.ref<i32>) private(@_QFsimd_with_nontemporal_clauseEi_private_i32 %8 -> %arg1 : !fir.ref<i32>) {
+    // CHECK: omp.simd nontemporal(%[[A_DECL]], %[[C_DECL]] : !fir.ref<i32>, !fir.ref<i32>) {
     // CHECK-NEXT: omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
-    omp.simd nontemporal(%2, %6 : !fir.ref<i32>, !fir.ref<i32>) private(@_QFsimd_with_nontemporal_clauseEi_private_i32 %8 -> %arg1 : !fir.ref<i32>) {
+    omp.simd nontemporal(%2, %6 : !fir.ref<i32>, !fir.ref<i32>) {
       omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%10) inclusive step (%c1_i32) {
-        %11 = fir.declare %arg1 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+        %11 = fir.declare %8 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
         fir.store %arg2 to %11 : !fir.ref<i32>
         // CHECK:  %[[LOAD:.*]] = fir.load %[[A_DECL]] {nontemporal} : !fir.ref<i32>
         %12 = fir.load %2 : !fir.ref<i32>
@@ -56,11 +56,11 @@ func.func @_QPsimd_nontemporal_allocatable(%arg0: !fir.ref<!fir.box<!fir.heap<!f
     %9 = fir.shape %7 : (index) -> !fir.shape<1>
     %10 = fir.embox %8(%9) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
     fir.store %10 to %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    // CHECK:  omp.simd nontemporal(%[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %2 -> %arg2 : !fir.ref<i32>) {
+    // CHECK:  omp.simd nontemporal(%[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
     // CHECK:   omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
-    omp.simd nontemporal(%3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %2 -> %arg2 : !fir.ref<i32>) {
+    omp.simd nontemporal(%3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
       omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
-        %16 = fir.declare %arg2 {uniq_name = "_QFsimd_nontemporal_allocatableEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+        %16 = fir.declare %2 {uniq_name = "_QFsimd_nontemporal_allocatableEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
         fir.store %arg3 to %16 : !fir.ref<i32>
         // CHECK:  %[[VAL1:.*]] = fir.load %[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
         %17 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>


        

From flang-commits at lists.llvm.org  Fri Jul  4 07:44:49 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Fri, 04 Jul 2025 07:44:49 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6867e8e1.630a0220.118ec0.18bf@mx.google.com>


================
@@ -3540,7 +3540,7 @@ WRAPPER_CLASS(OmpLocatorList, std::list<OmpLocator>);
 struct OmpMapperSpecifier {
   // Absent mapper-identifier is equivalent to DEFAULT.
   TUPLE_CLASS_BOILERPLATE(OmpMapperSpecifier);
-  std::tuple<std::optional<Name>, TypeSpec, Name> t;
+  std::tuple<std::string, TypeSpec, Name> t;
----------------
TIFitis wrote:

Sorry these are remnants from the PR stack. I've rebased this so the change is much smaller now, sorry for the delay in rebasing, it slipped my mind.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Fri Jul  4 08:04:42 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Fri, 04 Jul 2025 08:04:42 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6867ed8a.170a0220.e7e8.52cd@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
+        fir::unwrapPassByRefType(rec.getType()));
+    auto [fieldName, fieldTy] = recType.getTypeList()[index];
+    mlir::Value field = firOpBuilder.create<fir::FieldIndexOp>(
+        loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
+        fir::getTypeParams(rec));
+    return {firOpBuilder.create<fir::CoordinateOp>(
+                loc, firOpBuilder.getRefType(fieldTy), rec, field),
+            fieldTy};
+  };
+
+  mlir::omp::DeclareMapperInfoOperands clauseOps;
+  llvm::SmallVector<llvm::SmallVector<int64_t>> memberPlacementIndices;
+  llvm::SmallVector<mlir::Value> memberMapOps;
+
+  llvm::omp::OpenMPOffloadMappingFlags mapFlag =
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+  mlir::omp::VariableCaptureKind captureKind =
+      mlir::omp::VariableCaptureKind::ByRef;
+  int64_t index = 0;
+
+  // Populate the declareMapper region with the map information.
+  for (const auto &[memberName, memberType] :
+       mlir::dyn_cast<fir::RecordType>(recordType).getTypeList()) {
+    auto [ref, type] = getFieldRef(declareOp.getBase(), index);
+    mlir::FlatSymbolRefAttr mapperId;
+    if (auto recType = mlir::dyn_cast<fir::RecordType>(memberType)) {
+      std::string mapperIdName =
+          recType.getName().str() + ".omp.default.mapper";
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+      else if (auto *sym = converter.getCurrentScope().FindSymbol(memberName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+
+      if (converter.getModuleOp().lookupSymbol(mapperIdName))
+        mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                                mapperIdName);
+      else
+        mapperId = genImplicitDefaultDeclareMapper(converter, loc, recType,
+                                                   mapperIdName);
----------------
TIFitis wrote:

Done.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Fri Jul  4 08:05:02 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Fri, 04 Jul 2025 08:05:02 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Push context when parsing
 DECLARE VARIANT (PR #147075)
Message-ID: <llvm.org/llvm/llvm-project/pull/147075@github.com>


https://github.com/Stylie777 created https://github.com/llvm/llvm-project/pull/147075

Basic parsing and semantics support for Declare Variant was added in #130578. However, this did not include variant of `Pre` and `Post` within `OmpAttributeVisitor`. This meant that when a function in the class tried to get the context using `GetContext`, Flang would crash as the context was empty. To ensure this is possible, such as when resolving names as part of the `uniform` clause in the `simd` directive, the context is now pushed within `OmpAttributeVisitor` when parsing a `DECLARE VARIANT` directive.

Fixes #145222

>From 6ff15a5aa67d57d72112bec43a7920820bbe4927 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 4 Jul 2025 15:44:58 +0100
Subject: [PATCH] [Flang][OpenMP] Push context when parsing DECLARE VARIANT

Basic parsing and semantics support for Declare Variant was added in
PR 130578. However, this did not include variant of `Pre` and `Post`
within `OmpAttributeVisitor`. This meant that when a function in the
class tried to get the context using `GetContext`, Flang would crash
as the context was empty. To ensure this is possible, such as when
resolving names as part of the `uniform` clause in the `simd` directive,
the context is now pushed within `OmpAttributeVisitor` when parsing a
`DECLARE VARIANT` directive.

Fixes #145222
---
 flang/lib/Semantics/resolve-directives.cpp   |  6 +++++
 flang/test/Parser/OpenMP/declare-variant.f90 | 27 ++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 8d741734601a7..0d11b4a21ca3d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -495,6 +495,12 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   bool Pre(const parser::OpenMPAllocatorsConstruct &);
   void Post(const parser::OpenMPAllocatorsConstruct &);
 
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    PushContext(x.source, llvm::omp::Directive::OMPD_declare_variant);
+    return true;
+  }
+  void Post (const parser::OmpDeclareVariantDirective &) { PopContext(); };
+
   void Post(const parser::OmpObjectList &x) {
     // The objects from OMP clauses should have already been resolved,
     // except common blocks (the ResolveNamesVisitor does not visit
diff --git a/flang/test/Parser/OpenMP/declare-variant.f90 b/flang/test/Parser/OpenMP/declare-variant.f90
index 1b97733ea9525..3366b143e62e6 100644
--- a/flang/test/Parser/OpenMP/declare-variant.f90
+++ b/flang/test/Parser/OpenMP/declare-variant.f90
@@ -102,3 +102,30 @@ subroutine vsub(v1, v2)
     type(c_ptr), value :: v1, v2
   end
 end subroutine
+
+subroutine f
+  real :: x, y
+  y = 2
+  !omp simd
+  call f2(x, y)
+  !omp end simd 
+contains
+  subroutine f1 (x, y)
+    real :: x, y
+  end
+
+  subroutine f2 (x, y)
+    real :: x, y
+    !$omp declare variant (f1) match (construct={simd(uniform(y))})
+  end
+end subroutine
+!CHECK: !$OMP DECLARE VARIANT (f1) MATCH(CONSTRUCT={SIMD(UNIFORM(y))})
+!PARSE-TREE: | | | | DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpDeclareVariantDirective
+!PARSE-TREE-NEXT: | | | | | Verbatim
+!PARSE-TREE-NEXT: | | | | | Name = 'f1'
+!PARSE-TREE-NEXT: | | | | | OmpClauseList -> OmpClause -> Match -> OmpMatchClause -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSetSelectorName -> Value = Construct
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSelector
+!PARSE-TREE-NEXT: | | | | | | | OmpTraitSelectorName -> Value = Simd
+!PARSE-TREE-NEXT: | | | | | | | Properties
+!PARSE-TREE-NEXT: | | | | | | | | OmpTraitProperty -> OmpClause -> Uniform -> Name = 'y'


From flang-commits at lists.llvm.org  Fri Jul  4 08:05:32 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 08:05:32 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Push context when parsing
 DECLARE VARIANT (PR #147075)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147075@github.com>
Message-ID: <6867edbc.050a0220.2b0897.6210@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-parser

@llvm/pr-subscribers-flang-semantics

Author: Jack Styles (Stylie777)

<details>
<summary>Changes</summary>

Basic parsing and semantics support for Declare Variant was added in #<!-- -->130578. However, this did not include variant of `Pre` and `Post` within `OmpAttributeVisitor`. This meant that when a function in the class tried to get the context using `GetContext`, Flang would crash as the context was empty. To ensure this is possible, such as when resolving names as part of the `uniform` clause in the `simd` directive, the context is now pushed within `OmpAttributeVisitor` when parsing a `DECLARE VARIANT` directive.

Fixes #<!-- -->145222

---
Full diff: https://github.com/llvm/llvm-project/pull/147075.diff


2 Files Affected:

- (modified) flang/lib/Semantics/resolve-directives.cpp (+6) 
- (modified) flang/test/Parser/OpenMP/declare-variant.f90 (+27) 


``````````diff
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 8d741734601a7..0d11b4a21ca3d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -495,6 +495,12 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   bool Pre(const parser::OpenMPAllocatorsConstruct &);
   void Post(const parser::OpenMPAllocatorsConstruct &);
 
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    PushContext(x.source, llvm::omp::Directive::OMPD_declare_variant);
+    return true;
+  }
+  void Post (const parser::OmpDeclareVariantDirective &) { PopContext(); };
+
   void Post(const parser::OmpObjectList &x) {
     // The objects from OMP clauses should have already been resolved,
     // except common blocks (the ResolveNamesVisitor does not visit
diff --git a/flang/test/Parser/OpenMP/declare-variant.f90 b/flang/test/Parser/OpenMP/declare-variant.f90
index 1b97733ea9525..3366b143e62e6 100644
--- a/flang/test/Parser/OpenMP/declare-variant.f90
+++ b/flang/test/Parser/OpenMP/declare-variant.f90
@@ -102,3 +102,30 @@ subroutine vsub(v1, v2)
     type(c_ptr), value :: v1, v2
   end
 end subroutine
+
+subroutine f
+  real :: x, y
+  y = 2
+  !omp simd
+  call f2(x, y)
+  !omp end simd 
+contains
+  subroutine f1 (x, y)
+    real :: x, y
+  end
+
+  subroutine f2 (x, y)
+    real :: x, y
+    !$omp declare variant (f1) match (construct={simd(uniform(y))})
+  end
+end subroutine
+!CHECK: !$OMP DECLARE VARIANT (f1) MATCH(CONSTRUCT={SIMD(UNIFORM(y))})
+!PARSE-TREE: | | | | DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpDeclareVariantDirective
+!PARSE-TREE-NEXT: | | | | | Verbatim
+!PARSE-TREE-NEXT: | | | | | Name = 'f1'
+!PARSE-TREE-NEXT: | | | | | OmpClauseList -> OmpClause -> Match -> OmpMatchClause -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSetSelectorName -> Value = Construct
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSelector
+!PARSE-TREE-NEXT: | | | | | | | OmpTraitSelectorName -> Value = Simd
+!PARSE-TREE-NEXT: | | | | | | | Properties
+!PARSE-TREE-NEXT: | | | | | | | | OmpTraitProperty -> OmpClause -> Uniform -> Name = 'y'

``````````

</details>


https://github.com/llvm/llvm-project/pull/147075

From flang-commits at lists.llvm.org  Fri Jul  4 08:08:00 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 08:08:00 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Push context when parsing
 DECLARE VARIANT (PR #147075)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147075@github.com>
Message-ID: <6867ee50.170a0220.2398ac.492c@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions cpp -- flang/lib/Semantics/resolve-directives.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 0d11b4a21..299bb6ff8 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -499,7 +499,7 @@ public:
     PushContext(x.source, llvm::omp::Directive::OMPD_declare_variant);
     return true;
   }
-  void Post (const parser::OmpDeclareVariantDirective &) { PopContext(); };
+  void Post(const parser::OmpDeclareVariantDirective &) { PopContext(); };
 
   void Post(const parser::OmpObjectList &x) {
     // The objects from OMP clauses should have already been resolved,

``````````

</details>


https://github.com/llvm/llvm-project/pull/147075

From flang-commits at lists.llvm.org  Fri Jul  4 08:11:45 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Fri, 04 Jul 2025 08:11:45 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Push context when parsing
 DECLARE VARIANT (PR #147075)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147075@github.com>
Message-ID: <6867ef31.170a0220.f3b2e.9d65@mx.google.com>


https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/147075

>From 6ff15a5aa67d57d72112bec43a7920820bbe4927 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 4 Jul 2025 15:44:58 +0100
Subject: [PATCH 1/2] [Flang][OpenMP] Push context when parsing DECLARE VARIANT

Basic parsing and semantics support for Declare Variant was added in
PR 130578. However, this did not include variant of `Pre` and `Post`
within `OmpAttributeVisitor`. This meant that when a function in the
class tried to get the context using `GetContext`, Flang would crash
as the context was empty. To ensure this is possible, such as when
resolving names as part of the `uniform` clause in the `simd` directive,
the context is now pushed within `OmpAttributeVisitor` when parsing a
`DECLARE VARIANT` directive.

Fixes #145222
---
 flang/lib/Semantics/resolve-directives.cpp   |  6 +++++
 flang/test/Parser/OpenMP/declare-variant.f90 | 27 ++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 8d741734601a7..0d11b4a21ca3d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -495,6 +495,12 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   bool Pre(const parser::OpenMPAllocatorsConstruct &);
   void Post(const parser::OpenMPAllocatorsConstruct &);
 
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    PushContext(x.source, llvm::omp::Directive::OMPD_declare_variant);
+    return true;
+  }
+  void Post (const parser::OmpDeclareVariantDirective &) { PopContext(); };
+
   void Post(const parser::OmpObjectList &x) {
     // The objects from OMP clauses should have already been resolved,
     // except common blocks (the ResolveNamesVisitor does not visit
diff --git a/flang/test/Parser/OpenMP/declare-variant.f90 b/flang/test/Parser/OpenMP/declare-variant.f90
index 1b97733ea9525..3366b143e62e6 100644
--- a/flang/test/Parser/OpenMP/declare-variant.f90
+++ b/flang/test/Parser/OpenMP/declare-variant.f90
@@ -102,3 +102,30 @@ subroutine vsub(v1, v2)
     type(c_ptr), value :: v1, v2
   end
 end subroutine
+
+subroutine f
+  real :: x, y
+  y = 2
+  !omp simd
+  call f2(x, y)
+  !omp end simd 
+contains
+  subroutine f1 (x, y)
+    real :: x, y
+  end
+
+  subroutine f2 (x, y)
+    real :: x, y
+    !$omp declare variant (f1) match (construct={simd(uniform(y))})
+  end
+end subroutine
+!CHECK: !$OMP DECLARE VARIANT (f1) MATCH(CONSTRUCT={SIMD(UNIFORM(y))})
+!PARSE-TREE: | | | | DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpDeclareVariantDirective
+!PARSE-TREE-NEXT: | | | | | Verbatim
+!PARSE-TREE-NEXT: | | | | | Name = 'f1'
+!PARSE-TREE-NEXT: | | | | | OmpClauseList -> OmpClause -> Match -> OmpMatchClause -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSetSelectorName -> Value = Construct
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSelector
+!PARSE-TREE-NEXT: | | | | | | | OmpTraitSelectorName -> Value = Simd
+!PARSE-TREE-NEXT: | | | | | | | Properties
+!PARSE-TREE-NEXT: | | | | | | | | OmpTraitProperty -> OmpClause -> Uniform -> Name = 'y'

>From 3e246b0286b9d760a24a890d1b9c283b0762fa2a Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 4 Jul 2025 16:10:38 +0100
Subject: [PATCH 2/2] formatting fixes

---
 flang/lib/Semantics/resolve-directives.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 0d11b4a21ca3d..299bb6ff876e7 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -499,7 +499,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     PushContext(x.source, llvm::omp::Directive::OMPD_declare_variant);
     return true;
   }
-  void Post (const parser::OmpDeclareVariantDirective &) { PopContext(); };
+  void Post(const parser::OmpDeclareVariantDirective &) { PopContext(); };
 
   void Post(const parser::OmpObjectList &x) {
     // The objects from OMP clauses should have already been resolved,


From flang-commits at lists.llvm.org  Fri Jul  4 08:12:30 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Fri, 04 Jul 2025 08:12:30 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6867ef5e.170a0220.27745f.54e8@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
----------------
TIFitis wrote:

It shouldn't be used with use_device_ptr and other such clauses as they don't support a declare mapper.

At the moment it's only used when the user hasn't made any effort to map a derived type. This can't be used in tangent with explicit map clause as map-type composition can cause divergence from the user's intention.

This is only mean't to help map derived types, especially, allocatables of derived types and nested derived types when no other mapping information is available.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Fri Jul  4 08:19:58 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Fri, 04 Jul 2025 08:19:58 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6867f11e.170a0220.283564.5190@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
+        fir::unwrapPassByRefType(rec.getType()));
+    auto [fieldName, fieldTy] = recType.getTypeList()[index];
+    mlir::Value field = firOpBuilder.create<fir::FieldIndexOp>(
+        loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
+        fir::getTypeParams(rec));
+    return {firOpBuilder.create<fir::CoordinateOp>(
+                loc, firOpBuilder.getRefType(fieldTy), rec, field),
+            fieldTy};
+  };
+
+  mlir::omp::DeclareMapperInfoOperands clauseOps;
+  llvm::SmallVector<llvm::SmallVector<int64_t>> memberPlacementIndices;
+  llvm::SmallVector<mlir::Value> memberMapOps;
+
+  llvm::omp::OpenMPOffloadMappingFlags mapFlag =
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+  mlir::omp::VariableCaptureKind captureKind =
+      mlir::omp::VariableCaptureKind::ByRef;
+  int64_t index = 0;
+
+  // Populate the declareMapper region with the map information.
+  for (const auto &[memberName, memberType] :
+       mlir::dyn_cast<fir::RecordType>(recordType).getTypeList()) {
+    auto [ref, type] = getFieldRef(declareOp.getBase(), index);
+    mlir::FlatSymbolRefAttr mapperId;
+    if (auto recType = mlir::dyn_cast<fir::RecordType>(memberType)) {
+      std::string mapperIdName =
+          recType.getName().str() + ".omp.default.mapper";
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+      else if (auto *sym = converter.getCurrentScope().FindSymbol(memberName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+
+      if (converter.getModuleOp().lookupSymbol(mapperIdName))
+        mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                                mapperIdName);
+      else
+        mapperId = genImplicitDefaultDeclareMapper(converter, loc, recType,
+                                                   mapperIdName);
+    }
+
+    llvm::SmallVector<mlir::Value> bounds;
+    genBoundsOps(ref, bounds);
+    mlir::Value mapOp = createMapInfoOp(
+        firOpBuilder, loc, ref, /*varPtrPtr=*/mlir::Value{}, "", bounds,
----------------
TIFitis wrote:

The "Symbol" here is the `auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);` from above.

So there's no symname available for use here.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Fri Jul  4 08:20:17 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Fri, 04 Jul 2025 08:20:17 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][Doc] NFC: Minor fix for headings
	(PR #147077)
Message-ID: <llvm.org/llvm/llvm-project/pull/147077@github.com>


https://github.com/kiranchandramohan created https://github.com/llvm/llvm-project/pull/147077

Use a top level section to ensure that there is only one entry in the flang.llvm.org/docs page.

Also generate a table of contents.

>From f6ed8d7508b3edf6e2897d478c259c18d5bb8be5 Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan at arm.com>
Date: Fri, 4 Jul 2025 16:05:14 +0100
Subject: [PATCH] [Flang][Doc] NFC: Minor fix for headings

Use a top level section to ensure that there is only one entry in
the flang.llvm.org/docs page.

Also generate a table of contents.
---
 flang/docs/OpenMP-declare-target.md | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/flang/docs/OpenMP-declare-target.md b/flang/docs/OpenMP-declare-target.md
index 45062469007b6..478db2d004c87 100644
--- a/flang/docs/OpenMP-declare-target.md
+++ b/flang/docs/OpenMP-declare-target.md
@@ -7,7 +7,15 @@
 
 -->
 
-# Introduction to Declare Target
+# OpenMP Declare Target
+
+```{contents}
+---
+local:
+---
+```
+
+## Introduction to Declare Target
 
 In OpenMP `declare target` is a directive that can be applied to a function or
 variable (primarily global) to notate to the compiler that it should be 
@@ -55,7 +63,7 @@ in this case, leave off the `declare target` from the function and it
 would be implicitly marked `declare target any` (for both host and device), 
 as it's been utilised within a target region.
 
-# Declare Target as represented in the OpenMP Dialect
+## Declare Target as represented in the OpenMP Dialect
 
 In the OpenMP Dialect `declare target` is not represented by a specific 
 `operation`. Instead, it's an OpenMP dialect specific `attribute` that can be 
@@ -89,7 +97,7 @@ llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(Op.getOperation());
 declareTargetGlobal.isDeclareTarget();
 ```
 
-# Declare Target Fortran OpenMP Lowering
+## Declare Target Fortran OpenMP Lowering
 
 The initial lowering of `declare target` to MLIR for both use-cases is done
 inside of the usual OpenMP lowering in flang/lib/Lower/OpenMP.cpp. However, 
@@ -145,7 +153,7 @@ to the way `threadprivate` handles these cases, we raise and initialize the
 variable as an internal `GlobalOp` and apply the attribute. This occurs in the
 flang/lib/Lower/OpenMP.cpp function `genDeclareTargetIntGlobal`.
 
-# Declare Target Transformation Passes for Flang
+## Declare Target Transformation Passes for Flang
 
 There are currently two passes within Flang that are related to the processing 
 of `declare target`:
@@ -173,7 +181,7 @@ While this infrastructure could be generally applicable to more than just Flang,
 it is only utilised in the Flang frontend, so it resides there rather than in 
 the OpenMP dialect codebase. 
 
-# Declare Target OpenMP Dialect To LLVM-IR Lowering
+## Declare Target OpenMP Dialect To LLVM-IR Lowering
 
 The OpenMP dialect lowering of `declare target` is done through the 
 `amendOperation` flow, as it's not an `operation` but rather an 
@@ -228,7 +236,7 @@ place. It may be possible to store this information in the OMPIRBuilder and
 then perform this cleanup process on finalization, but this is open for 
 discussion and implementation still.
 
-# Current Support
+## Current Support
 
 For the moment, `declare target` should work for:
 * Marking functions/subroutines and function/subroutine interfaces for 


From flang-commits at lists.llvm.org  Fri Jul  4 08:24:55 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 08:24:55 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP][NFC] Remove flag toggling
 deprecated no hlfir flow in map-types-and-sizes.f90 (PR #146995)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146995@github.com>
Message-ID: <6867f247.050a0220.35188e.57d4@mx.google.com>


agozillon wrote:

Thank you for the quick review @TIFitis :-) !

https://github.com/llvm/llvm-project/pull/146995

From flang-commits at lists.llvm.org  Fri Jul  4 08:24:59 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 08:24:59 -0700 (PDT)
Subject: [flang-commits] [flang] fd5ed04 - [Flang][OpenMP][NFC] Remove flag
 toggling deprecated no hlfir flow in map-types-and-sizes.f90 (#146995)
Message-ID: <6867f24b.170a0220.f16d0.558a@mx.google.com>


Author: agozillon
Date: 2025-07-04T17:24:56+02:00
New Revision: fd5ed046fdca42603fef1cbc6f4e7e0f536dcff3

URL: https://github.com/llvm/llvm-project/commit/fd5ed046fdca42603fef1cbc6f4e7e0f536dcff3
DIFF: https://github.com/llvm/llvm-project/commit/fd5ed046fdca42603fef1cbc6f4e7e0f536dcff3.diff

LOG: [Flang][OpenMP][NFC] Remove flag toggling deprecated no hlfir flow in map-types-and-sizes.f90 (#146995)

We no longer utilise the deprecated FIR only flow, so we should be
testing for the current HLFIR flow that we support as opposed to the
older that we no longer maintain.

Added: 
    

Modified: 
    flang/test/Integration/OpenMP/map-types-and-sizes.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Integration/OpenMP/map-types-and-sizes.f90 b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
index 35eec823e30de..aa68c13558a7b 100644
--- a/flang/test/Integration/OpenMP/map-types-and-sizes.f90
+++ b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
@@ -6,7 +6,7 @@
 ! added to this directory and sub-directories.
 !===----------------------------------------------------------------------===!
 
-!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa %s -o - | FileCheck %s
 
 !===============================================================================
 ! Check MapTypes for target implicit captures
@@ -494,15 +494,18 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_single_member_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 1
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
-!CHECK: store ptr %[[MEMBER_ACCESS]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+!CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_multiple_nested_members_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 0
-!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS_0:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 0
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 1
 !CHECK: %[[ARR_END_OFF:.*]] = getelementptr float, ptr %[[MEMBER_ACCESS_2]], i64 1
 !CHECK: %[[ARR_END:.*]] = ptrtoint ptr %[[ARR_END_OFF]] to i64
 !CHECK: %[[FIRST_MEMBER:.*]] = ptrtoint ptr %[[MEMBER_ACCESS_1]] to i64
@@ -525,8 +528,9 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_member_with_bounds_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 2
-!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS]], i64 0, i64 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 2
+!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS_1]], i64 0, i64 1
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -574,19 +578,27 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [4 x i64], ptr %.offload_sizes, i32 0, i32 3
 !CHECK: store i64 %[[DTYPE_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 
+
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_alloca_derived_type_{{.*}}
 !CHECK: %{{.*}} = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_3:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, i64 1, align 8
-!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
-!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
-!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
 !CHECK: %[[ACCESS_DESC_MEMBER_UB:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[LOAD_DESC_MEMBER_UB:.*]] = load i64, ptr %[[ACCESS_DESC_MEMBER_UB]], align 8
 !CHECK: %[[OFFSET_MEMBER_UB:.*]] = sub i64 %[[LOAD_DESC_MEMBER_UB]], 1
+!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
+!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
+!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
 !CHECK: %[[DTYPE_ALLOCA_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ALLOCA_MEMBER_ACCESS]], i32 0, i32 0
+
 !CHECK: %[[DTYPE_BASE_ADDR_ACCESS_2:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA]], i32 0, i32 0
 !CHECK: %[[DTYPE_BASE_ADDR_LOAD_2:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS_2]], align 8
 !CHECK: %[[DTYPE_NONALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD_2]], i32 0, i32 5
@@ -715,11 +727,11 @@ end subroutine mapType_common_block_members
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_nested_derived_type_alloca{{.*}}
 !CHECK: %[[ALLOCATABLE_MEMBER_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_nested_derived_type_allocaTtop_layer, i64 1, align 8
+!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
+!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[ALLOCATABLE_MEMBER_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[ALLOCATABLE_MEMBER_ADDR_LOAD:.*]] = load i64, ptr %[[ALLOCATABLE_MEMBER_BASE_ADDR]], align 8
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_1:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_ADDR_LOAD]], 1
-!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
-!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[NESTED_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_2:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_1]], 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_3:.*]] = add i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_2]], 1


        

From flang-commits at lists.llvm.org  Fri Jul  4 08:25:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 08:25:02 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP][NFC] Remove flag toggling
 deprecated no hlfir flow in map-types-and-sizes.f90 (PR #146995)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146995@github.com>
Message-ID: <6867f24e.170a0220.3092cd.8526@mx.google.com>


https://github.com/agozillon closed https://github.com/llvm/llvm-project/pull/146995

From flang-commits at lists.llvm.org  Fri Jul  4 08:32:25 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Fri, 04 Jul 2025 08:32:25 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Push context when parsing
 DECLARE VARIANT (PR #147075)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147075@github.com>
Message-ID: <6867f409.050a0220.35188e.580a@mx.google.com>


https://github.com/kiranchandramohan approved this pull request.

LGTM. Thanks for the fix.

In future some name resolution will have to happen from `!$omp declare variant (f1)` to the subroutine `f1`.

https://github.com/llvm/llvm-project/pull/147075

From flang-commits at lists.llvm.org  Fri Jul  4 08:40:14 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Fri, 04 Jul 2025 08:40:14 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6867f5de.a70a0220.384b8.58a6@mx.google.com>


https://github.com/TIFitis updated https://github.com/llvm/llvm-project/pull/140562

>From 74b5144ab52d8bbf03e21754ddc4fb1676f84358 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 8 May 2025 21:19:26 +0100
Subject: [PATCH 1/3] [OpenMP][Flang] Emit default declare mappers implicitly
 for derived types

This patch adds support to emit default declare mappers for implicit mapping of derived types when not supplied by user. This especially helps tackle mapping of allocatables of derived types.
This supports nested derived types as well.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  38 ++---
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 132 +++++++++++++++++-
 .../Optimizer/OpenMP/MapInfoFinalization.cpp  |   2 +-
 flang/test/Lower/OpenMP/derived-type-map.f90  |  22 ++-
 4 files changed, 174 insertions(+), 20 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 7bea427099a28..f534e37de32c2 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1167,24 +1167,28 @@ void ClauseProcessor::processMapObjects(
 
   auto getDefaultMapperID = [&](const omp::Object &object,
                                 std::string &mapperIdName) {
-    if (!mlir::isa<mlir::omp::DeclareMapperOp>(
-            firOpBuilder.getRegion().getParentOp())) {
-      const semantics::DerivedTypeSpec *typeSpec = nullptr;
-
-      if (object.sym()->owner().IsDerivedType())
-        typeSpec = object.sym()->owner().derivedTypeSpec();
-      else if (object.sym()->GetType() &&
-               object.sym()->GetType()->category() ==
-                   semantics::DeclTypeSpec::TypeDerived)
-        typeSpec = &object.sym()->GetType()->derivedTypeSpec();
-
-      if (typeSpec) {
-        mapperIdName =
-            typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;
-        if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
-          mapperIdName = converter.mangleName(mapperIdName, sym->owner());
-      }
+    const semantics::DerivedTypeSpec *typeSpec = nullptr;
+
+    if (object.sym()->GetType() && object.sym()->GetType()->category() ==
+                                       semantics::DeclTypeSpec::TypeDerived)
+      typeSpec = &object.sym()->GetType()->derivedTypeSpec();
+    else if (object.sym()->owner().IsDerivedType())
+      typeSpec = object.sym()->owner().derivedTypeSpec();
+
+    if (typeSpec) {
+      mapperIdName =
+          typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
     }
+
+    // Make sure we don't return a mapper to self
+    llvm::StringRef parentOpName;
+    if (auto declMapOp = mlir::dyn_cast<mlir::omp::DeclareMapperOp>(
+            firOpBuilder.getRegion().getParentOp()))
+      parentOpName = declMapOp.getSymName();
+    if (mapperIdName == parentOpName)
+      mapperIdName = "";
   };
 
   // Create the mapper symbol from its name, if specified.
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0a56e888ac44b..9ccccb96f7513 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2391,6 +2391,124 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  lower::StatementContext stmtCtx;
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
+        fir::unwrapPassByRefType(rec.getType()));
+    auto [fieldName, fieldTy] = recType.getTypeList()[index];
+    mlir::Value field = firOpBuilder.create<fir::FieldIndexOp>(
+        loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
+        fir::getTypeParams(rec));
+    return {firOpBuilder.create<fir::CoordinateOp>(
+                loc, firOpBuilder.getRefType(fieldTy), rec, field),
+            fieldTy};
+  };
+
+  mlir::omp::DeclareMapperInfoOperands clauseOps;
+  llvm::SmallVector<llvm::SmallVector<int64_t>> memberPlacementIndices;
+  llvm::SmallVector<mlir::Value> memberMapOps;
+
+  llvm::omp::OpenMPOffloadMappingFlags mapFlag =
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
+      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+  mlir::omp::VariableCaptureKind captureKind =
+      mlir::omp::VariableCaptureKind::ByRef;
+  int64_t index = 0;
+
+  // Populate the declareMapper region with the map information.
+  for (const auto &[memberName, memberType] :
+       mlir::dyn_cast<fir::RecordType>(recordType).getTypeList()) {
+    auto [ref, type] = getFieldRef(declareOp.getBase(), index);
+    mlir::FlatSymbolRefAttr mapperId;
+    if (auto recType = mlir::dyn_cast<fir::RecordType>(memberType)) {
+      std::string mapperIdName =
+          recType.getName().str() + ".omp.default.mapper";
+      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+      else if (auto *sym = converter.getCurrentScope().FindSymbol(memberName))
+        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+
+      if (converter.getModuleOp().lookupSymbol(mapperIdName))
+        mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                                mapperIdName);
+      else
+        mapperId = genImplicitDefaultDeclareMapper(converter, loc, recType,
+                                                   mapperIdName);
+    }
+
+    llvm::SmallVector<mlir::Value> bounds;
+    genBoundsOps(ref, bounds);
+    mlir::Value mapOp = createMapInfoOp(
+        firOpBuilder, loc, ref, /*varPtrPtr=*/mlir::Value{}, "", bounds,
+        /*members=*/{},
+        /*membersIndex=*/mlir::ArrayAttr{},
+        static_cast<
+            std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
+            mapFlag),
+        captureKind, ref.getType(), /*partialMap=*/false, mapperId);
+    memberMapOps.emplace_back(mapOp);
+    memberPlacementIndices.emplace_back(llvm::SmallVector<int64_t>{index++});
+  }
+
+  llvm::SmallVector<mlir::Value> bounds;
+  genBoundsOps(declareOp.getOriginalBase(), bounds);
+  mlir::omp::MapInfoOp mapOp = createMapInfoOp(
+      firOpBuilder, loc, declareOp.getOriginalBase(),
+      /*varPtrPtr=*/mlir::Value(), /*name=*/"", bounds, memberMapOps,
+      firOpBuilder.create2DI64ArrayAttr(memberPlacementIndices),
+      static_cast<std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
+          mapFlag),
+      captureKind, declareOp.getType(0),
+      /*partialMap=*/true);
+
+  clauseOps.mapVars.emplace_back(mapOp);
+
+  firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
+  // declMapperOp->dumpPretty();
+  return mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                      mapperNameStr);
+}
+
 static mlir::omp::TargetOp
 genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
             lower::StatementContext &stmtCtx,
@@ -2467,15 +2585,26 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       name << sym.name().ToString();
 
       mlir::FlatSymbolRefAttr mapperId;
-      if (sym.GetType()->category() == semantics::DeclTypeSpec::TypeDerived) {
+      if (sym.GetType()->category() == semantics::DeclTypeSpec::TypeDerived &&
+          defaultMaps.empty()) {
         auto &typeSpec = sym.GetType()->derivedTypeSpec();
         std::string mapperIdName =
             typeSpec.name().ToString() + llvm::omp::OmpDefaultMapperName;
         if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
           mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+        else
+          mapperIdName =
+              converter.mangleName(mapperIdName, *typeSpec.GetScope());
+
         if (converter.getModuleOp().lookupSymbol(mapperIdName))
           mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
                                                   mapperIdName);
+        else
+          mapperId = genImplicitDefaultDeclareMapper(
+              converter, loc,
+              mlir::cast<fir::RecordType>(
+                  converter.genType(sym.GetType()->derivedTypeSpec())),
+              mapperIdName);
       }
 
       fir::factory::AddrAndBoundsInfo info =
@@ -3442,6 +3571,7 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processMap(loc, stmtCtx, clauseOps);
   firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
+  // declMapperOp->dumpPretty();
 }
 
 static void
diff --git a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
index f052cf8d22d1e..2a11aff4befc0 100644
--- a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
@@ -433,7 +433,7 @@ class MapInfoFinalizationPass
                                    getDescriptorMapType(mapType, target)),
             op.getMapCaptureTypeAttr(), /*varPtrPtr=*/mlir::Value{}, newMembers,
             newMembersAttr, /*bounds=*/mlir::SmallVector<mlir::Value>{},
-            /*mapperId*/ mlir::FlatSymbolRefAttr(), op.getNameAttr(),
+            op.getMapperIdAttr(), op.getNameAttr(),
             /*partial_map=*/builder.getBoolAttr(false));
     op.replaceAllUsesWith(newDescParentMapOp.getResult());
     op->erase();
diff --git a/flang/test/Lower/OpenMP/derived-type-map.f90 b/flang/test/Lower/OpenMP/derived-type-map.f90
index 279cddec51fcf..1dd73229d8251 100644
--- a/flang/test/Lower/OpenMP/derived-type-map.f90
+++ b/flang/test/Lower/OpenMP/derived-type-map.f90
@@ -1,9 +1,11 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
+!CHECK: omp.declare_mapper @[[MAPPER1:_QQFmaptype_derived_implicit_allocatablescalar_and_array.omp.default.mapper]] : !fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
+!CHECK: omp.declare_mapper @[[MAPPER2:_QQFmaptype_derived_implicitscalar_and_array.omp.default.mapper]] : !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {
 
 !CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_implicitEscalar_arr"}
 !CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_implicitEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
-!CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(implicit, tofrom) capture(ByRef) -> !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr"}
+!CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(implicit, tofrom) capture(ByRef) mapper(@[[MAPPER2]]) -> !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr"}
 !CHECK:     omp.target map_entries(%[[MAP]] -> %[[ARG0:.*]] : !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
 subroutine mapType_derived_implicit
     type :: scalar_and_array
@@ -18,6 +20,24 @@ subroutine mapType_derived_implicit
     !$omp end target
 end subroutine mapType_derived_implicit
 
+!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_implicit_allocatableEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFmaptype_derived_implicit_allocatableEscalar_arr"} : {{.*}}
+!CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : {{.*}}) map_clauses(implicit, to) capture(ByRef) mapper(@[[MAPPER1]])
+!CHECK:     omp.target map_entries(%[[MAP]] -> %[[ARG0:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>) {
+subroutine mapType_derived_implicit_allocatable
+    type :: scalar_and_array
+      real(4) :: real
+      integer(4) :: array(10)
+      integer(4) :: int
+    end type scalar_and_array
+    type(scalar_and_array), allocatable :: scalar_arr 
+
+    allocate (scalar_arr)
+    !$omp target
+       scalar_arr%int = 1
+    !$omp end target
+end subroutine mapType_derived_implicit_allocatable
+
 !CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_explicitEscalar_arr"}
 !CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_explicitEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
 !CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(tofrom) capture(ByRef) -> !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr"}

>From 138250f6eed33a589541c60c5290ea2b183c60c6 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 21 May 2025 16:48:27 +0100
Subject: [PATCH 2/3] Minor cleanup.

---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp | 33 +++++++++++++---------
 flang/lib/Lower/OpenMP/OpenMP.cpp          |  3 --
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index f534e37de32c2..f0a90887eeeec 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1167,19 +1167,26 @@ void ClauseProcessor::processMapObjects(
 
   auto getDefaultMapperID = [&](const omp::Object &object,
                                 std::string &mapperIdName) {
-    const semantics::DerivedTypeSpec *typeSpec = nullptr;
-
-    if (object.sym()->GetType() && object.sym()->GetType()->category() ==
-                                       semantics::DeclTypeSpec::TypeDerived)
-      typeSpec = &object.sym()->GetType()->derivedTypeSpec();
-    else if (object.sym()->owner().IsDerivedType())
-      typeSpec = object.sym()->owner().derivedTypeSpec();
-
-    if (typeSpec) {
-      mapperIdName =
-          typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;
-      if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
-        mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+    if (!mlir::isa<mlir::omp::DeclareMapperOp>(
+            firOpBuilder.getRegion().getParentOp())) {
+      const semantics::DerivedTypeSpec *typeSpec = nullptr;
+
+      if (object.sym()->owner().IsDerivedType())
+        typeSpec = object.sym()->owner().derivedTypeSpec();
+      else if (object.sym()->GetType() &&
+               object.sym()->GetType()->category() ==
+                   semantics::DeclTypeSpec::TypeDerived)
+        typeSpec = &object.sym()->GetType()->derivedTypeSpec();
+
+      if (typeSpec) {
+        mapperIdName =
+            typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;
+        if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
+          mapperIdName = converter.mangleName(mapperIdName, sym->owner());
+        else
+          mapperIdName =
+              converter.mangleName(mapperIdName, *typeSpec->GetScope());
+      }
     }
 
     // Make sure we don't return a mapper to self
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 9ccccb96f7513..9f5bb017a3e74 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2396,7 +2396,6 @@ genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
                                 mlir::Location loc, fir::RecordType recordType,
                                 llvm::StringRef mapperNameStr) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  lower::StatementContext stmtCtx;
 
   // Save current insertion point before moving to the module scope to create
   // the DeclareMapperOp
@@ -2504,7 +2503,6 @@ genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
   clauseOps.mapVars.emplace_back(mapOp);
 
   firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
-  // declMapperOp->dumpPretty();
   return mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
                                       mapperNameStr);
 }
@@ -3571,7 +3569,6 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processMap(loc, stmtCtx, clauseOps);
   firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
-  // declMapperOp->dumpPretty();
 }
 
 static void

>From c2ca1e8a3ab5ffd64f308c79e3a0931c5bcdfd9c Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Fri, 4 Jul 2025 16:39:47 +0100
Subject: [PATCH 3/3] Rebased. Addressed reviewer comments.

---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp |  4 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp          | 57 +++++++++++-----------
 2 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index f0a90887eeeec..58f3d9d5e62f3 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1189,7 +1189,7 @@ void ClauseProcessor::processMapObjects(
       }
     }
 
-    // Make sure we don't return a mapper to self
+    // Make sure we don't return a mapper to self.
     llvm::StringRef parentOpName;
     if (auto declMapOp = mlir::dyn_cast<mlir::omp::DeclareMapperOp>(
             firOpBuilder.getRegion().getParentOp()))
@@ -1299,7 +1299,7 @@ bool ClauseProcessor::processMap(
         llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
     std::string mapperIdName = "__implicit_mapper";
     // If the map type is specified, then process it else set the appropriate
-    // default value
+    // default value.
     Map::MapType type;
     if (directive == llvm::omp::Directive::OMPD_target_enter_data &&
         semaCtx.langOptions().OpenMPVersion >= 52)
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 9f5bb017a3e74..3b40f9dda8bb2 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2391,14 +2391,17 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
-static mlir::FlatSymbolRefAttr
-genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
-                                mlir::Location loc, fir::RecordType recordType,
-                                llvm::StringRef mapperNameStr) {
+static mlir::FlatSymbolRefAttr getOrGenImplicitDefaultDeclareMapper(
+    lower::AbstractConverter &converter, mlir::Location loc,
+    fir::RecordType recordType, llvm::StringRef mapperNameStr) {
+  if (converter.getModuleOp().lookupSymbol(mapperNameStr))
+    return mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
+                                        mapperNameStr);
+
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
   // Save current insertion point before moving to the module scope to create
-  // the DeclareMapperOp
+  // the DeclareMapperOp.
   mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
 
   firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
@@ -2429,11 +2432,8 @@ genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
   // Return a reference to the contents of a derived type with one field.
   // Also return the field type.
   const auto getFieldRef =
-      [&](mlir::Value rec,
-          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
-    auto recType = mlir::dyn_cast<fir::RecordType>(
-        fir::unwrapPassByRefType(rec.getType()));
-    auto [fieldName, fieldTy] = recType.getTypeList()[index];
+      [&](mlir::Value rec, llvm::StringRef fieldName, mlir::Type fieldTy,
+          mlir::Type recType) -> std::tuple<mlir::Value, mlir::Type> {
     mlir::Value field = firOpBuilder.create<fir::FieldIndexOp>(
         loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
         fir::getTypeParams(rec));
@@ -2452,33 +2452,32 @@ genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
       llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
   mlir::omp::VariableCaptureKind captureKind =
       mlir::omp::VariableCaptureKind::ByRef;
-  int64_t index = 0;
 
   // Populate the declareMapper region with the map information.
-  for (const auto &[memberName, memberType] :
-       mlir::dyn_cast<fir::RecordType>(recordType).getTypeList()) {
-    auto [ref, type] = getFieldRef(declareOp.getBase(), index);
+  for (const auto &entry : llvm::enumerate(
+           mlir::dyn_cast<fir::RecordType>(recordType).getTypeList())) {
+    const auto &memberName = entry.value().first;
+    const auto &memberType = entry.value().second;
+    auto [ref, type] =
+        getFieldRef(declareOp.getBase(), memberName, memberType, recordType);
     mlir::FlatSymbolRefAttr mapperId;
     if (auto recType = mlir::dyn_cast<fir::RecordType>(memberType)) {
       std::string mapperIdName =
-          recType.getName().str() + ".omp.default.mapper";
+          recType.getName().str() + llvm::omp::OmpDefaultMapperName;
       if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
         mapperIdName = converter.mangleName(mapperIdName, sym->owner());
       else if (auto *sym = converter.getCurrentScope().FindSymbol(memberName))
         mapperIdName = converter.mangleName(mapperIdName, sym->owner());
 
-      if (converter.getModuleOp().lookupSymbol(mapperIdName))
-        mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
-                                                mapperIdName);
-      else
-        mapperId = genImplicitDefaultDeclareMapper(converter, loc, recType,
-                                                   mapperIdName);
+      mapperId = getOrGenImplicitDefaultDeclareMapper(converter, loc, recType,
+                                                      mapperIdName);
     }
 
     llvm::SmallVector<mlir::Value> bounds;
     genBoundsOps(ref, bounds);
     mlir::Value mapOp = createMapInfoOp(
-        firOpBuilder, loc, ref, /*varPtrPtr=*/mlir::Value{}, "", bounds,
+        firOpBuilder, loc, ref, /*varPtrPtr=*/mlir::Value{}, /*name=*/"",
+        bounds,
         /*members=*/{},
         /*membersIndex=*/mlir::ArrayAttr{},
         static_cast<
@@ -2486,7 +2485,8 @@ genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
             mapFlag),
         captureKind, ref.getType(), /*partialMap=*/false, mapperId);
     memberMapOps.emplace_back(mapOp);
-    memberPlacementIndices.emplace_back(llvm::SmallVector<int64_t>{index++});
+    memberPlacementIndices.emplace_back(
+        llvm::SmallVector<int64_t>{(int64_t)entry.index()});
   }
 
   llvm::SmallVector<mlir::Value> bounds;
@@ -2597,12 +2597,11 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
         if (converter.getModuleOp().lookupSymbol(mapperIdName))
           mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
                                                   mapperIdName);
-        else
-          mapperId = genImplicitDefaultDeclareMapper(
-              converter, loc,
-              mlir::cast<fir::RecordType>(
-                  converter.genType(sym.GetType()->derivedTypeSpec())),
-              mapperIdName);
+        mapperId = getOrGenImplicitDefaultDeclareMapper(
+            converter, loc,
+            mlir::cast<fir::RecordType>(
+                converter.genType(sym.GetType()->derivedTypeSpec())),
+            mapperIdName);
       }
 
       fir::factory::AddrAndBoundsInfo info =


From flang-commits at lists.llvm.org  Fri Jul  4 08:41:05 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Fri, 04 Jul 2025 08:41:05 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][Doc] NFC: Minor fix for headings
	(PR #147077)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147077@github.com>
Message-ID: <6867f611.170a0220.2f65d2.58a2@mx.google.com>


https://github.com/kiranchandramohan updated https://github.com/llvm/llvm-project/pull/147077

>From 96c3a44650ff0dad6a75d5a2359326cf57c86c7f Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan at arm.com>
Date: Fri, 4 Jul 2025 16:05:14 +0100
Subject: [PATCH] [Flang][Doc] NFC: Minor fix for headings

Use a top level section to ensure that there is only one entry in
the flang.llvm.org/docs page.

Also generate a table of contents.
---
 flang/docs/OpenMP-declare-target.md        | 20 ++++++++++++++------
 flang/docs/OpenMP-descriptor-management.md | 11 +++++++++--
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/flang/docs/OpenMP-declare-target.md b/flang/docs/OpenMP-declare-target.md
index 45062469007b6..478db2d004c87 100644
--- a/flang/docs/OpenMP-declare-target.md
+++ b/flang/docs/OpenMP-declare-target.md
@@ -7,7 +7,15 @@
 
 -->
 
-# Introduction to Declare Target
+# OpenMP Declare Target
+
+```{contents}
+---
+local:
+---
+```
+
+## Introduction to Declare Target
 
 In OpenMP `declare target` is a directive that can be applied to a function or
 variable (primarily global) to notate to the compiler that it should be 
@@ -55,7 +63,7 @@ in this case, leave off the `declare target` from the function and it
 would be implicitly marked `declare target any` (for both host and device), 
 as it's been utilised within a target region.
 
-# Declare Target as represented in the OpenMP Dialect
+## Declare Target as represented in the OpenMP Dialect
 
 In the OpenMP Dialect `declare target` is not represented by a specific 
 `operation`. Instead, it's an OpenMP dialect specific `attribute` that can be 
@@ -89,7 +97,7 @@ llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(Op.getOperation());
 declareTargetGlobal.isDeclareTarget();
 ```
 
-# Declare Target Fortran OpenMP Lowering
+## Declare Target Fortran OpenMP Lowering
 
 The initial lowering of `declare target` to MLIR for both use-cases is done
 inside of the usual OpenMP lowering in flang/lib/Lower/OpenMP.cpp. However, 
@@ -145,7 +153,7 @@ to the way `threadprivate` handles these cases, we raise and initialize the
 variable as an internal `GlobalOp` and apply the attribute. This occurs in the
 flang/lib/Lower/OpenMP.cpp function `genDeclareTargetIntGlobal`.
 
-# Declare Target Transformation Passes for Flang
+## Declare Target Transformation Passes for Flang
 
 There are currently two passes within Flang that are related to the processing 
 of `declare target`:
@@ -173,7 +181,7 @@ While this infrastructure could be generally applicable to more than just Flang,
 it is only utilised in the Flang frontend, so it resides there rather than in 
 the OpenMP dialect codebase. 
 
-# Declare Target OpenMP Dialect To LLVM-IR Lowering
+## Declare Target OpenMP Dialect To LLVM-IR Lowering
 
 The OpenMP dialect lowering of `declare target` is done through the 
 `amendOperation` flow, as it's not an `operation` but rather an 
@@ -228,7 +236,7 @@ place. It may be possible to store this information in the OMPIRBuilder and
 then perform this cleanup process on finalization, but this is open for 
 discussion and implementation still.
 
-# Current Support
+## Current Support
 
 For the moment, `declare target` should work for:
 * Marking functions/subroutines and function/subroutine interfaces for 
diff --git a/flang/docs/OpenMP-descriptor-management.md b/flang/docs/OpenMP-descriptor-management.md
index cbc27f2a0fcb8..d0764c2df4642 100644
--- a/flang/docs/OpenMP-descriptor-management.md
+++ b/flang/docs/OpenMP-descriptor-management.md
@@ -6,8 +6,15 @@
 
 -->
 
-# OpenMP dialect: Fortran descriptor type mapping for offload
+# Fortran descriptor type mapping for OpenMP offload
+```{contents}
+---
+local:
+---
+```
+
 
+## Details
 The initial method for mapping Fortran types tied to descriptors for OpenMP offloading is to treat these types 
 as a special case of OpenMP record type (C/C++ structure/class, Fortran derived type etc.) mapping as far as the 
 runtime is concerned. Where the box (descriptor information) is the holding container and the underlying 
@@ -103,7 +110,7 @@ it is a little rigid in how the descriptor mappings are handled as there is no s
 to specialise the mappings for possible edge cases without polluting the dialect or lowering with further
 knowledge of Fortran and the FIR dialect.
 
-# OpenMP dialect differences from OpenACC dialect
+## Differences from OpenACC
 
 The descriptor mapping for OpenMP currently works differently to the planned direction for OpenACC, however, 
 it is possible and would likely be ideal to align the method with OpenACC in the future. 


From flang-commits at lists.llvm.org  Fri Jul  4 08:42:48 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Fri, 04 Jul 2025 08:42:48 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP][Flang] Emit default declare
 mappers implicitly for derived types (PR #140562)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140562@github.com>
Message-ID: <6867f678.170a0220.2d4506.5a45@mx.google.com>


================
@@ -2348,6 +2348,122 @@ genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::FlatSymbolRefAttr
+genImplicitDefaultDeclareMapper(lower::AbstractConverter &converter,
+                                mlir::Location loc, fir::RecordType recordType,
+                                llvm::StringRef mapperNameStr) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Save current insertion point before moving to the module scope to create
+  // the DeclareMapperOp
+  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
+
+  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
+  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+      loc, mapperNameStr, recordType);
+  auto &region = declMapperOp.getRegion();
+  firOpBuilder.createBlock(&region);
+  auto mapperArg = region.addArgument(firOpBuilder.getRefType(recordType), loc);
+
+  auto declareOp =
+      firOpBuilder.create<hlfir::DeclareOp>(loc, mapperArg, /*uniq_name=*/"");
+
+  const auto genBoundsOps = [&](mlir::Value mapVal,
+                                llvm::SmallVectorImpl<mlir::Value> &bounds) {
+    fir::ExtendedValue extVal =
+        hlfir::translateToExtendedValue(mapVal.getLoc(), firOpBuilder,
+                                        hlfir::Entity{mapVal},
+                                        /*contiguousHint=*/true)
+            .first;
+    fir::factory::AddrAndBoundsInfo info = fir::factory::getDataOperandBaseAddr(
+        firOpBuilder, mapVal, /*isOptional=*/false, mapVal.getLoc());
+    bounds = fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                                mlir::omp::MapBoundsType>(
+        firOpBuilder, info, extVal,
+        /*dataExvIsAssumedSize=*/false, mapVal.getLoc());
+  };
+
+  // Return a reference to the contents of a derived type with one field.
+  // Also return the field type.
+  const auto getFieldRef =
+      [&](mlir::Value rec,
+          unsigned index) -> std::tuple<mlir::Value, mlir::Type> {
+    auto recType = mlir::dyn_cast<fir::RecordType>(
----------------
TIFitis wrote:

You're right we should be reusing `memberName` and `memberType` from the callsite, thanks for catching this.

https://github.com/llvm/llvm-project/pull/140562

From flang-commits at lists.llvm.org  Fri Jul  4 10:02:03 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Fri, 04 Jul 2025 10:02:03 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <6868090b.170a0220.b566a.598c@mx.google.com>


https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/146594

>From 072e1d14c324ab501319d967bafe2588fff2d233 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Mon, 3 Feb 2025 12:46:27 -0600
Subject: [PATCH] [Clang] Introduce '--offload-targets' for `-fopenmp-targets`

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.
---
 clang/include/clang/Driver/Options.td              | 6 +++++-
 clang/lib/Driver/Driver.cpp                        | 8 ++++----
 clang/lib/Driver/ToolChain.cpp                     | 2 +-
 clang/lib/Driver/ToolChains/CommonArgs.cpp         | 2 +-
 clang/lib/Frontend/CompilerInvocation.cpp          | 6 +++---
 clang/test/Driver/amdgpu-openmp-sanitize-options.c | 4 ++--
 clang/test/Driver/hip-options.hip                  | 2 +-
 flang/lib/Frontend/CompilerInvocation.cpp          | 2 +-
 flang/test/Driver/omp-driver-offload.f90           | 6 +++---
 9 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 5831ea73561ff..0b98eaa436e65 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : CommaJoined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
 def offload_arch_EQ : CommaJoined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
@@ -3680,7 +3684,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 3b70310dd9004..55748c0bce918 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -996,7 +996,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1030,7 +1030,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1046,7 +1046,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1124,7 +1124,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 2fcf9b28dc746..9ae76fc307a56 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1415,7 +1415,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 0a9e3649b386b..7484ef5d746ec 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3891,7 +3891,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4322,7 +4322,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4382,7 +4382,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..8e4c210306cb2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1248,7 +1248,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())
diff --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index 335bfad4b1884..f4f0c21dc1f5e 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -225,8 +225,8 @@
 ! RUN: | FileCheck %s --check-prefix=OFFLOAD-TARGETS
 
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"
 ! OFFLOAD-TARGETS-NEXT: "{{[^"]*}}flang" "-fc1" "-triple" "amdgcn-amd-amdhsa"
-! OFFLOAD-TARGETS-NOT: -fopenmp-targets
+! OFFLOAD-TARGETS-NOT: --offload-targets
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"


From flang-commits at lists.llvm.org  Fri Jul  4 10:26:55 2025
From: flang-commits at lists.llvm.org (Jake Egan via flang-commits)
Date: Fri, 04 Jul 2025 10:26:55 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] add option to generate
 runtime type info as external (#145901) " (PR #146071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146071@github.com>
Message-ID: <68680edf.630a0220.abc15.6d1c@mx.google.com>


jakeegan wrote:

Hi, this new test fails on the [AIX bot](https://lab.llvm.org/buildbot/#/builders/201/builds/5294/steps/6/logs/FAIL__Flang__skip-external-rtti-definition_F90). Could you take a look, please?


https://github.com/llvm/llvm-project/pull/146071

From flang-commits at lists.llvm.org  Fri Jul  4 10:52:40 2025
From: flang-commits at lists.llvm.org (Kelvin Li via flang-commits)
Date: Fri, 04 Jul 2025 10:52:40 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] add option to generate
 runtime type info as external (#145901) " (PR #146071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146071@github.com>
Message-ID: <686814e8.170a0220.23297a.5955@mx.google.com>


kkwli wrote:

> Hi, this new test fails on the [AIX bot](https://lab.llvm.org/buildbot/#/builders/201/builds/5294/steps/6/logs/FAIL__Flang__skip-external-rtti-definition_F90). Could you take a look, please?

@jakeegan I think it was fixed in https://github.com/llvm/llvm-project/pull/146826

https://github.com/llvm/llvm-project/pull/146071

From flang-commits at lists.llvm.org  Fri Jul  4 14:20:58 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 14:20:58 -0700 (PDT)
Subject: [flang-commits] [flang] 6db02dc - [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (#146594)
Message-ID: <686845ba.050a0220.209a93.7161@mx.google.com>


Author: Joseph Huber
Date: 2025-07-04T16:20:53-05:00
New Revision: 6db02dc4311b3ee52ec6e7e5e5cafb2600e7b30f

URL: https://github.com/llvm/llvm-project/commit/6db02dc4311b3ee52ec6e7e5e5cafb2600e7b30f
DIFF: https://github.com/llvm/llvm-project/commit/6db02dc4311b3ee52ec6e7e5e5cafb2600e7b30f.diff

LOG: [Clang] Introduce `--offload-targets` for `-fopenmp-targets` (#146594)

Summary:
This patch is mostly an NFC that renames the existing `-fopenmp-targets`
into `--offload-targets`. Doing this early to simplify a follow-up patch
that will hopefully allow this syntax to be used more generically over
the existing `--offload` syntax (which I think is mostly unmaintained
now.). Following in the well-trodden path of trying to pull language
specific offload options into generic ones, but right now this is still
just OpenMP specific.

Added: 
    

Modified: 
    clang/include/clang/Driver/Options.td
    clang/lib/Driver/Driver.cpp
    clang/lib/Driver/ToolChain.cpp
    clang/lib/Driver/ToolChains/CommonArgs.cpp
    clang/lib/Frontend/CompilerInvocation.cpp
    clang/test/Driver/amdgpu-openmp-sanitize-options.c
    clang/test/Driver/hip-options.hip
    flang/lib/Frontend/CompilerInvocation.cpp
    flang/test/Driver/omp-driver-offload.f90

Removed: 
    


################################################################################
diff  --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 483516cf1b13d..54c71b066f9d4 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,6 +1138,10 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
+def offload_targets_EQ : CommaJoined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
 def offload_arch_EQ : CommaJoined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
@@ -3681,7 +3685,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,

diff  --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 3b70310dd9004..55748c0bce918 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -996,7 +996,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1030,7 +1030,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1046,7 +1046,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1124,7 +1124,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }

diff  --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }

diff  --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..bdd77ac84913c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1404,7 +1404,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();

diff  --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 9b3200bc8a091..6633e4a7787ee 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3891,7 +3891,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4322,7 +4322,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4382,7 +4382,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())

diff  --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}

diff  --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 

diff  --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..8e4c210306cb2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1248,7 +1248,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())

diff  --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index 335bfad4b1884..f4f0c21dc1f5e 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -225,8 +225,8 @@
 ! RUN: | FileCheck %s --check-prefix=OFFLOAD-TARGETS
 
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"
 ! OFFLOAD-TARGETS-NEXT: "{{[^"]*}}flang" "-fc1" "-triple" "amdgcn-amd-amdhsa"
-! OFFLOAD-TARGETS-NOT: -fopenmp-targets
+! OFFLOAD-TARGETS-NOT: --offload-targets
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"


        

From flang-commits at lists.llvm.org  Fri Jul  4 14:20:59 2025
From: flang-commits at lists.llvm.org (Joseph Huber via flang-commits)
Date: Fri, 04 Jul 2025 14:20:59 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Clang] Introduce
 `--offload-targets` for `-fopenmp-targets` (PR #146594)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146594@github.com>
Message-ID: <686845bb.050a0220.1f3fdf.67fd@mx.google.com>


https://github.com/jhuber6 closed https://github.com/llvm/llvm-project/pull/146594

From flang-commits at lists.llvm.org  Sun Jul  6 06:41:13 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Sun, 06 Jul 2025 06:41:13 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686a7cf9.170a0220.2f6859.9d50@mx.google.com>


ronlieb wrote:

seeing random 5 out of 30 compilations fail 

"flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu" "-emit-llvm-bc" "-mrelocation-model" "static" "-ffast-math" "-target-cpu" "x86-64" "-floop-interchange" "-vectorize-loops" "-vectorize-slp" "-fversion-loops-for-stride" "-fopenmp" "-fopenmp-targets=amdgcn-amd-amdhsa" "-mframe-pointer=none" "-O3" "-x" "f95" "update_halo_kernel-gfx90a-8cde79"



#4 0x000014937e341abe Fortran::semantics::CanonicalizationOfOmp::RewriteOpenMPLoopConstruct(Fortran::parser::OpenMPLoopConstruct&, std
::__cxx11::list<Fortran::parser::ExecutionPartConstruct, std::allocator<Fortran::parser::ExecutionPartConstruct>>&, std::_List_iterator
<Fortran::parser::ExecutionPartConstruct>) (/mnt/COD/2025-07-06/trunk_21.0-0/bin/../lib/../lib/libFortranSemantics.so.21.0git+0x1a8abe)
 #5 0x000014937e342040 Fortran::semantics::CanonicalizationOfOmp::Post(std::__cxx11::list<Fortran::parser::ExecutionPartConstruct, std:
:allocator<Fortran::parser::ExecutionPartConstruct>>&) (/mnt/COD/2025-07-06/trunk_21.0-0/bin/../lib/../lib/libFortranSemantics.so.21.0g
it+0x1a9040)
 #6 0x000014937e35151d void Fortran::parser::detail::ParseTreeVisitorLookupScope::Walk<Fortran::semantics::CanonicalizationOfOmp>(std::
__cxx11::list<Fortran::parser::ExecutionPartConstruct, std::allocator<Fortran::parser::ExecutionPartConstruct>>&, Fortran::semantics::C
anonicalizationOfOmp&) (/mnt/COD/2025-07-06/trunk_21.0-0/bin/../lib/../lib/libFortranSemantics.so.21.0git+0x1b851d)
 #7 0x000014937e35151d void Fortran::parser::detail::ParseTreeVisitorLookupScope::Walk<Fortran::semantics::CanonicalizationOfOmp>(std::
__cxx11::list<Fortran::parser::ExecutionPartConstruct, std::allocator<Fortran::parser::ExecutionPartConstruct>>&, Fortran::semantics::C
anonicalizationOfOmp&) (/mnt/COD/2025-07-06/trunk_21.0-0/bin/../lib/../lib/libFortranSemantics.so.21.0git+0x1b851d)




https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Sun Jul  6 11:54:47 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Sun, 06 Jul 2025 11:54:47 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][Doc] NFC: Minor fix for headings
	(PR #147077)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147077@github.com>
Message-ID: <686ac677.170a0220.1ea7c9.a7b8@mx.google.com>


https://github.com/tarunprabhu approved this pull request.

Thanks Kiran.

https://github.com/llvm/llvm-project/pull/147077

From flang-commits at lists.llvm.org  Sun Jul  6 16:26:35 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Sun, 06 Jul 2025 16:26:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add general symbol dependence
	collection utility (PR #146968)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146968@github.com>
Message-ID: <686b062b.050a0220.fb236.b106@mx.google.com>


https://github.com/clementval approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/146968

From flang-commits at lists.llvm.org  Thu Jul  3 10:31:17 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Thu, 03 Jul 2025 10:31:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Fix
 parallel-firstprivate-clause-scalar.f90 test (PR #146932)
Message-ID: <llvm.org/llvm/llvm-project/pull/146932@github.com>


https://github.com/luporl created https://github.com/llvm/llvm-project/pull/146932

Fix REQUIRES and references to declared variables.

Fixes #146875


>From 5854c130b05bde87255fb05b2b73a55c8f65ecd5 Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori at linaro.org>
Date: Thu, 3 Jul 2025 17:27:56 +0000
Subject: [PATCH] [flang][OpenMP] Fix parallel-firstprivate-clause-scalar.f90
 test

Fix REQUIRES and references to declared variables.

Fixes #146875
---
 .../parallel-firstprivate-clause-scalar.f90    | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 017b2a3f2edd2..6e72b1de9d2d2 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -1,8 +1,8 @@
 ! This test checks lowering of `FIRSTPRIVATE` clause for scalar types.
 
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! REQUIRES: shell
-! RUN: bbc -fopenmp -emit-hlfir %s -o - \
+! RUN: bbc -target x86_64-unknown-linux-gnu -fopenmp -emit-hlfir %s -o - \
 ! RUN: | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 !CHECK:  omp.private {type = firstprivate} @[[ARG2_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg2_firstprivate_l8]] : !fir.logical<1>
@@ -29,7 +29,7 @@
 !CHECK:   omp.parallel private(@[[ARG1_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG1_PVT:.*]], @[[ARG2_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG2_PVT:.*]] : {{.*}}) {
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
-!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
+!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -57,8 +57,8 @@ subroutine firstprivate_complex(arg1, arg2)
 !CHECK:    %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_integerEarg4"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_integerEarg5"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 !CHECK:    %[[ARG6_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG6_PVT]] {uniq_name = "_QFfirstprivate_integerEarg6"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
-!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1,
-!%[[ARG5_PVT_DECL]]#1, %[[ARG6_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
+!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0,
+!%[[ARG5_PVT_DECL]]#0, %[[ARG6_PVT_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
 !CHECK:    omp.terminator
 !CHECK:  }
 
@@ -88,7 +88,7 @@ subroutine firstprivate_integer(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg3"} : (!fir.ref<!fir.logical<2>>) -> (!fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<2>>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg4"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK:     %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg5"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
-!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1, %[[ARG5_PVT_DECL]]#1) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
+!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0, %[[ARG5_PVT_DECL]]#0) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -116,7 +116,7 @@ subroutine firstprivate_logical(arg1, arg2, arg3, arg4, arg5)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
+!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
@@ -152,7 +152,7 @@ subroutine firstprivate_real10(arg1)
 !CHECK-KIND16:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
 !CHECK-KIND16:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND16:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
-!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f128>) -> ()
+!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f128>) -> ()
 !CHECK-KIND16:     omp.terminator
 !CHECK-KIND16:   }
 subroutine firstprivate_real16(arg1)
@@ -171,7 +171,7 @@ subroutine firstprivate_real16(arg1)
 !CHECK:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[A_PRIV_ADDR:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[B_PRIV_ADDR:.*]] : {{.*}}) {
 !CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#1, %[[B_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#0, %[[B_PRIV_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 !CHECK:             omp.terminator
 !CHECK:           }
 !CHECK:           return


From flang-commits at lists.llvm.org  Thu Jul  3 10:31:48 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 03 Jul 2025 10:31:48 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Fix
 parallel-firstprivate-clause-scalar.f90 test (PR #146932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146932@github.com>
Message-ID: <6866be84.170a0220.15c39e.1ad2@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-fir-hlfir

@llvm/pr-subscribers-flang-openmp

Author: Leandro Lupori (luporl)

<details>
<summary>Changes</summary>

Fix REQUIRES and references to declared variables.

Fixes #<!-- -->146875


---
Full diff: https://github.com/llvm/llvm-project/pull/146932.diff


1 Files Affected:

- (modified) flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 (+9-9) 


``````````diff
diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 017b2a3f2edd2..6e72b1de9d2d2 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -1,8 +1,8 @@
 ! This test checks lowering of `FIRSTPRIVATE` clause for scalar types.
 
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! REQUIRES: shell
-! RUN: bbc -fopenmp -emit-hlfir %s -o - \
+! RUN: bbc -target x86_64-unknown-linux-gnu -fopenmp -emit-hlfir %s -o - \
 ! RUN: | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 !CHECK:  omp.private {type = firstprivate} @[[ARG2_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg2_firstprivate_l8]] : !fir.logical<1>
@@ -29,7 +29,7 @@
 !CHECK:   omp.parallel private(@[[ARG1_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG1_PVT:.*]], @[[ARG2_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG2_PVT:.*]] : {{.*}}) {
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
-!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
+!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -57,8 +57,8 @@ subroutine firstprivate_complex(arg1, arg2)
 !CHECK:    %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_integerEarg4"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_integerEarg5"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 !CHECK:    %[[ARG6_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG6_PVT]] {uniq_name = "_QFfirstprivate_integerEarg6"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
-!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1,
-!%[[ARG5_PVT_DECL]]#1, %[[ARG6_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
+!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0,
+!%[[ARG5_PVT_DECL]]#0, %[[ARG6_PVT_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
 !CHECK:    omp.terminator
 !CHECK:  }
 
@@ -88,7 +88,7 @@ subroutine firstprivate_integer(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg3"} : (!fir.ref<!fir.logical<2>>) -> (!fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<2>>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg4"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK:     %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg5"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
-!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1, %[[ARG5_PVT_DECL]]#1) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
+!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0, %[[ARG5_PVT_DECL]]#0) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -116,7 +116,7 @@ subroutine firstprivate_logical(arg1, arg2, arg3, arg4, arg5)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
+!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
@@ -152,7 +152,7 @@ subroutine firstprivate_real10(arg1)
 !CHECK-KIND16:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
 !CHECK-KIND16:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND16:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
-!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f128>) -> ()
+!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f128>) -> ()
 !CHECK-KIND16:     omp.terminator
 !CHECK-KIND16:   }
 subroutine firstprivate_real16(arg1)
@@ -171,7 +171,7 @@ subroutine firstprivate_real16(arg1)
 !CHECK:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[A_PRIV_ADDR:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[B_PRIV_ADDR:.*]] : {{.*}}) {
 !CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#1, %[[B_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#0, %[[B_PRIV_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 !CHECK:             omp.terminator
 !CHECK:           }
 !CHECK:           return

``````````

</details>


https://github.com/llvm/llvm-project/pull/146932

From flang-commits at lists.llvm.org  Thu Jul  3 11:44:20 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Thu, 03 Jul 2025 11:44:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Fix
 parallel-firstprivate-clause-scalar.f90 test (PR #146932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146932@github.com>
Message-ID: <6866cf84.170a0220.e138a.226a@mx.google.com>


https://github.com/luporl updated https://github.com/llvm/llvm-project/pull/146932

>From 5854c130b05bde87255fb05b2b73a55c8f65ecd5 Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori at linaro.org>
Date: Thu, 3 Jul 2025 17:27:56 +0000
Subject: [PATCH 1/2] [flang][OpenMP] Fix
 parallel-firstprivate-clause-scalar.f90 test

Fix REQUIRES and references to declared variables.

Fixes #146875
---
 .../parallel-firstprivate-clause-scalar.f90    | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 017b2a3f2edd2..6e72b1de9d2d2 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -1,8 +1,8 @@
 ! This test checks lowering of `FIRSTPRIVATE` clause for scalar types.
 
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! REQUIRES: shell
-! RUN: bbc -fopenmp -emit-hlfir %s -o - \
+! RUN: bbc -target x86_64-unknown-linux-gnu -fopenmp -emit-hlfir %s -o - \
 ! RUN: | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 !CHECK:  omp.private {type = firstprivate} @[[ARG2_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg2_firstprivate_l8]] : !fir.logical<1>
@@ -29,7 +29,7 @@
 !CHECK:   omp.parallel private(@[[ARG1_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG1_PVT:.*]], @[[ARG2_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG2_PVT:.*]] : {{.*}}) {
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
-!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
+!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -57,8 +57,8 @@ subroutine firstprivate_complex(arg1, arg2)
 !CHECK:    %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_integerEarg4"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_integerEarg5"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 !CHECK:    %[[ARG6_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG6_PVT]] {uniq_name = "_QFfirstprivate_integerEarg6"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
-!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1,
-!%[[ARG5_PVT_DECL]]#1, %[[ARG6_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
+!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0,
+!%[[ARG5_PVT_DECL]]#0, %[[ARG6_PVT_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
 !CHECK:    omp.terminator
 !CHECK:  }
 
@@ -88,7 +88,7 @@ subroutine firstprivate_integer(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg3"} : (!fir.ref<!fir.logical<2>>) -> (!fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<2>>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg4"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK:     %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg5"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
-!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1, %[[ARG5_PVT_DECL]]#1) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
+!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0, %[[ARG5_PVT_DECL]]#0) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -116,7 +116,7 @@ subroutine firstprivate_logical(arg1, arg2, arg3, arg4, arg5)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
+!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
@@ -152,7 +152,7 @@ subroutine firstprivate_real10(arg1)
 !CHECK-KIND16:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
 !CHECK-KIND16:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND16:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
-!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f128>) -> ()
+!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f128>) -> ()
 !CHECK-KIND16:     omp.terminator
 !CHECK-KIND16:   }
 subroutine firstprivate_real16(arg1)
@@ -171,7 +171,7 @@ subroutine firstprivate_real16(arg1)
 !CHECK:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[A_PRIV_ADDR:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[B_PRIV_ADDR:.*]] : {{.*}}) {
 !CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#1, %[[B_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#0, %[[B_PRIV_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 !CHECK:             omp.terminator
 !CHECK:           }
 !CHECK:           return

>From 2f87b306f2e29ce54a02ec2d2c9ee31aa83afb1a Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori at linaro.org>
Date: Thu, 3 Jul 2025 18:43:45 +0000
Subject: [PATCH 2/2] Fix CHECK-KIND10

---
 flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 6e72b1de9d2d2..2bb1036c4a7fa 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -136,7 +136,7 @@ subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK-KIND10:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>, !fir.dscope) -> (!fir.ref<f80>, !fir.ref<f80>)
 !CHECK-KIND10:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND10:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
-!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f80>) -> ()
+!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f80>) -> ()
 !CHECK-KIND10:     omp.terminator
 !CHECK-KIND10:   }
 subroutine firstprivate_real10(arg1)


From flang-commits at lists.llvm.org  Fri Jul  4 00:50:54 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 04 Jul 2025 00:50:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Fix
 parallel-firstprivate-clause-scalar.f90 test (PR #146932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146932@github.com>
Message-ID: <686787de.050a0220.2896c1.47e9@mx.google.com>


DavidSpickett wrote:

Thanks Leandro, I'll leave the review to the flang experts.

https://github.com/llvm/llvm-project/pull/146932

From flang-commits at lists.llvm.org  Fri Jul  4 01:19:27 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Fri, 04 Jul 2025 01:19:27 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <68678e8f.170a0220.2676cf.4488@mx.google.com>


https://github.com/s-watanabe314 updated https://github.com/llvm/llvm-project/pull/146641

>From baf87a6002d740868daeeba0f89d55c05e85b712 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 18 Apr 2025 09:04:07 +0900
Subject: [PATCH 1/4] [flang] Add -fcomplex-arithmetic= option and select
 complex division algorithm

This patch adds an option to select the method for computing complex
number division. It uses `LoweringOptions` to determine whether to lower
complex division to a runtime function call or to MLIR's `complex.div`,
and `CodeGenOptions` to select the computation algorithm for
`complex.div`. The available option values and their corresponding
algorithms are as follows:
- `full`: Lower to a runtime function call. (Default behavior)
- `improved`: Lower to `complex.div` and expand to Smith's algorithm.
- `basic`: Lower to `complex.div` and expand to the algebraic algorithm.

See also the discussion in the following discourse post:
https://discourse.llvm.org/t/optimization-of-complex-number-division/83468
---
 clang/include/clang/Driver/Options.td         |   4 +-
 clang/lib/Driver/ToolChains/Flang.cpp         |  47 ++
 .../include/flang/Frontend/CodeGenOptions.def |   1 +
 flang/include/flang/Frontend/CodeGenOptions.h |  25 +
 flang/include/flang/Lower/LoweringOptions.def |   4 +
 .../flang/Optimizer/Builder/FIRBuilder.h      |  15 +
 .../include/flang/Optimizer/CodeGen/CodeGen.h |   6 +
 flang/include/flang/Tools/CrossToolHelpers.h  |   3 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  21 +
 flang/lib/Frontend/FrontendActions.cpp        |   2 +
 flang/lib/Lower/Bridge.cpp                    |   2 +
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |  12 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  15 +-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |   1 +
 flang/test/Driver/complex-range.f90           |  23 +
 .../complex-div-to-llvm-kind10.f90            | 131 +++++
 .../complex-div-to-llvm-kind16.f90            | 131 +++++
 .../test/Integration/complex-div-to-llvm.f90  | 508 ++++++++++++++++++
 .../HLFIR/complex-div-to-hlfir-kind10.f90     |  34 ++
 .../HLFIR/complex-div-to-hlfir-kind16.f90     |  35 ++
 .../test/Lower/HLFIR/complex-div-to-hlfir.f90 |  94 ++++
 flang/tools/bbc/bbc.cpp                       |   8 +
 22 files changed, 1117 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Driver/complex-range.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm-kind10.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm-kind16.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..58209ceb5dc54 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
 
 def complex_range_EQ : Joined<["-"], "complex-range=">, Group<f_Group>,
-  Visibility<[CC1Option]>,
+  Visibility<[CC1Option, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>,
   MarshallingInfoEnum<LangOpts<"ComplexRange">, "CX_Full">;
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index e4e321ba1e195..ba5db7e3aba43 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -595,6 +595,30 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
+static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  default:
+    return "";
+  }
+}
+
+static std::string
+RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
+
 static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   StringRef FPContract;
@@ -605,6 +629,8 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
   bool AssociativeMath = false;
   bool ReciprocalMath = false;
 
+  LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_None;
+
   if (const Arg *A = Args.getLastArg(options::OPT_ffp_contract)) {
     const StringRef Val = A->getValue();
     if (Val == "fast" || Val == "off") {
@@ -629,6 +655,20 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
     default:
       continue;
 
+    case options::OPT_fcomplex_arithmetic_EQ: {
+      StringRef Val = A->getValue();
+      if (Val == "full")
+        Range = LangOptions::ComplexRangeKind::CX_Full;
+      else if (Val == "improved")
+        Range = LangOptions::ComplexRangeKind::CX_Improved;
+      else if (Val == "basic")
+        Range = LangOptions::ComplexRangeKind::CX_Basic;
+      else {
+        D.Diag(diag::err_drv_unsupported_option_argument)
+            << A->getSpelling() << Val;
+      }
+      break;
+    }
     case options::OPT_fhonor_infinities:
       HonorINFs = true;
       break;
@@ -699,6 +739,13 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
   if (!Recip.empty())
     CmdArgs.push_back(Args.MakeArgString("-mrecip=" + Recip));
 
+  if (Range != LangOptions::ComplexRangeKind::CX_None) {
+    std::string ComplexRangeStr = RenderComplexRangeOption(Range);
+    CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
+    CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" +
+                                         ComplexRangeKindToStr(Range)));
+  }
+
   if (!HonorINFs && !HonorNaNs && AssociativeMath && ReciprocalMath &&
       ApproxFunc && !SignedZeros &&
       (FPContract == "fast" || FPContract.empty())) {
diff --git a/flang/include/flang/Frontend/CodeGenOptions.def b/flang/include/flang/Frontend/CodeGenOptions.def
index ae12aec518108..cdeea93c9aecb 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.def
+++ b/flang/include/flang/Frontend/CodeGenOptions.def
@@ -52,6 +52,7 @@ ENUM_CODEGENOPT(RelocationModel, llvm::Reloc::Model, 3, llvm::Reloc::PIC_) ///<
 ENUM_CODEGENOPT(DebugInfo,  llvm::codegenoptions::DebugInfoKind, 4,  llvm::codegenoptions::NoDebugInfo) ///< Level of debug info to generate
 ENUM_CODEGENOPT(VecLib, llvm::driver::VectorLibrary, 4, llvm::driver::VectorLibrary::NoLibrary) ///< Vector functions library to use
 ENUM_CODEGENOPT(FramePointer, llvm::FramePointerKind, 2, llvm::FramePointerKind::None) ///< Enable the usage of frame pointers
+ENUM_CODEGENOPT(ComplexRange, ComplexRangeKind, 3, ComplexRangeKind::CX_Full) ///< Method for calculating complex number division
 
 ENUM_CODEGENOPT(DoConcurrentMapping, DoConcurrentMappingKind, 2, DoConcurrentMappingKind::DCMK_None) ///< Map `do concurrent` to OpenMP
 
diff --git a/flang/include/flang/Frontend/CodeGenOptions.h b/flang/include/flang/Frontend/CodeGenOptions.h
index bad17c8309eb8..df6063cc90340 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.h
+++ b/flang/include/flang/Frontend/CodeGenOptions.h
@@ -192,6 +192,31 @@ class CodeGenOptions : public CodeGenOptionsBase {
     return getProfileUse() == llvm::driver::ProfileCSIRInstr;
   }
 
+  /// Controls the various implementations for complex division.
+  enum ComplexRangeKind {
+    /// Implementation of complex division using a call to runtime library
+    /// functions. Overflow and non-finite values are handled by the library
+    /// implementation. This is the default value.
+    CX_Full,
+
+    /// Implementation of complex division offering an improved handling
+    /// for overflow in intermediate calculations. Overflow and non-finite
+    /// values are handled by MLIR's implementation of "complex.div", but this
+    /// may change in the future.
+    CX_Improved,
+
+    /// Implementation of complex division using algebraic formulas at source
+    /// precision. No special handling to avoid overflow. NaN and infinite
+    /// values are not handled.
+    CX_Basic,
+
+    /// No range rule is enabled.
+    CX_None
+
+    /// TODO: Implemention of other values as needed. In Clang, "CX_Promoted"
+    /// is implemented. (See clang/Basic/LangOptions.h)
+  };
+
   // Define accessors/mutators for code generation options of enumeration type.
 #define CODEGENOPT(Name, Bits, Default)
 #define ENUM_CODEGENOPT(Name, Type, Bits, Default)                             \
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index 3263ab129d076..8135704971aa4 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -70,5 +70,9 @@ ENUM_LOWERINGOPT(CUDARuntimeCheck, unsigned, 1, 0)
 /// derived types defined in other compilation units.
 ENUM_LOWERINGOPT(SkipExternalRttiDefinition, unsigned, 1, 0)
 
+/// If true, convert complex number division to runtime on the frontend.
+/// If false, lower to the complex dialect of MLIR.
+/// On by default.
+ENUM_LOWERINGOPT(ComplexDivisionToRuntime, unsigned, 1, 1)
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index e1eaab3346901..b1513850a9048 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -609,6 +609,17 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return integerOverflowFlags;
   }
 
+  /// Set ComplexDivisionToRuntimeFlag value for whether complex number division
+  /// is lowered to a runtime function by this builder.
+  void setComplexDivisionToRuntimeFlag(bool flag) {
+    complexDivisionToRuntimeFlag = flag;
+  }
+
+  /// Get current ComplexDivisionToRuntimeFlag value.
+  bool getComplexDivisionToRuntimeFlag() const {
+    return complexDivisionToRuntimeFlag;
+  }
+
   /// Dump the current function. (debug)
   LLVM_DUMP_METHOD void dumpFunc();
 
@@ -673,6 +684,10 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   /// mlir::arith::IntegerOverflowFlagsAttr.
   mlir::arith::IntegerOverflowFlags integerOverflowFlags{};
 
+  /// Flag to control whether complex number division is lowered to a runtime
+  /// function or to the MLIR complex dialect.
+  bool complexDivisionToRuntimeFlag = true;
+
   /// fir::GlobalOp and func::FuncOp symbol table to speed-up
   /// lookups.
   mlir::SymbolTable *symbolTable = nullptr;
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 93f07d8d5d4d9..e9a07a8dde5cd 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 #define FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 
+#include "flang/Frontend/CodeGenOptions.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassRegistry.h"
@@ -58,6 +59,11 @@ struct FIRToLLVMPassOptions {
   // the name of the global variable corresponding to a derived
   // type's descriptor.
   bool typeDescriptorsRenamedForAssembly = false;
+
+  // Specify the calculation method for complex number division used by the
+  // Conversion pass of the MLIR complex dialect.
+  Fortran::frontend::CodeGenOptions::ComplexRangeKind ComplexRange =
+      Fortran::frontend::CodeGenOptions::ComplexRangeKind::CX_Full;
 };
 
 /// Convert FIR to the LLVM IR dialect with default options.
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 337685c82af5f..df1da27058552 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -140,6 +140,9 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   std::string InstrumentFunctionExit =
       ""; ///< Name of the instrument-function that is called on each
           ///< function-exit
+  Fortran::frontend::CodeGenOptions::ComplexRangeKind ComplexRange =
+      Fortran::frontend::CodeGenOptions::ComplexRangeKind::
+          CX_Full; ///< Method for calculating complex number division
 };
 
 struct OffloadModuleOpts {
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..86ec410b1f70f 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -484,6 +484,21 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
   }
 
   parseDoConcurrentMapping(opts, args, diags);
+
+  if (const auto *arg =
+          args.getLastArg(clang::driver::options::OPT_complex_range_EQ)) {
+    llvm::StringRef argValue = llvm::StringRef(arg->getValue());
+    if (argValue == "full") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Full);
+    } else if (argValue == "improved") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Improved);
+    } else if (argValue == "basic") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Basic);
+    } else {
+      diags.Report(clang::diag::err_drv_invalid_value)
+          << arg->getAsString(args) << arg->getValue();
+    }
+  }
 }
 
 /// Parses all target input arguments and populates the target
@@ -1811,4 +1826,10 @@ void CompilerInvocation::setLoweringOptions() {
       .setNoSignedZeros(langOptions.NoSignedZeros)
       .setAssociativeMath(langOptions.AssociativeMath)
       .setReciprocalMath(langOptions.ReciprocalMath);
+
+  if (codegenOpts.getComplexRange() ==
+          CodeGenOptions::ComplexRangeKind::CX_Improved ||
+      codegenOpts.getComplexRange() ==
+          CodeGenOptions::ComplexRangeKind::CX_Basic)
+    loweringOpts.setComplexDivisionToRuntime(false);
 }
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index bf15def3f3b2e..b5f4f9421f633 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -750,6 +750,8 @@ void CodeGenAction::generateLLVMIR() {
   if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
     config.NSWOnLoopVarInc = false;
 
+  config.ComplexRange = opts.getComplexRange();
+
   // Create the pass pipeline
   fir::createMLIRToLLVMPassPipeline(pm, config, getCurrentFile());
   (void)mlir::applyPassManagerCLOptions(pm);
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..7e95c640e73b0 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5746,6 +5746,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     builder =
         new fir::FirOpBuilder(func, bridge.getKindMap(), &mlirSymbolTable);
     assert(builder && "FirOpBuilder did not instantiate");
+    builder->setComplexDivisionToRuntimeFlag(
+        bridge.getLoweringOptions().getComplexDivisionToRuntime());
     builder->setFastMathFlags(bridge.getLoweringOptions().getMathOptions());
     builder->setInsertionPointToStart(&func.front());
     if (funit.parent.isA<Fortran::lower::pft::FunctionLikeUnit>()) {
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index df8dfbc72c030..cb338618dbf3b 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1066,8 +1066,16 @@ struct BinaryOp<Fortran::evaluate::Divide<
     mlir::Type ty = Fortran::lower::getFIRType(
         builder.getContext(), Fortran::common::TypeCategory::Complex, KIND,
         /*params=*/{});
-    return hlfir::EntityWithAttributes{
-        fir::genDivC(builder, loc, ty, lhs, rhs)};
+
+    // TODO: Ideally, complex number division operations should always be
+    // lowered to MLIR. However, converting them to the runtime via MLIR causes
+    // ABI issues.
+    if (builder.getComplexDivisionToRuntimeFlag())
+      return hlfir::EntityWithAttributes{
+          fir::genDivC(builder, loc, ty, lhs, rhs)};
+    else
+      return hlfir::EntityWithAttributes{
+          builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
   }
 };
 
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 2b018912b40e4..4cf533f195e69 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -4119,7 +4119,20 @@ class FIRToLLVMLowering
     mathToFuncsOptions.minWidthOfFPowIExponent = 33;
     mathConvertionPM.addPass(
         mlir::createConvertMathToFuncs(mathToFuncsOptions));
-    mathConvertionPM.addPass(mlir::createConvertComplexToStandardPass());
+
+    mlir::ConvertComplexToStandardPassOptions complexToStandardOptions{};
+    if (options.ComplexRange ==
+        Fortran::frontend::CodeGenOptions::ComplexRangeKind::CX_Basic) {
+      complexToStandardOptions.complexRange =
+          mlir::complex::ComplexRangeFlags::basic;
+    } else if (options.ComplexRange == Fortran::frontend::CodeGenOptions::
+                                           ComplexRangeKind::CX_Improved) {
+      complexToStandardOptions.complexRange =
+          mlir::complex::ComplexRangeFlags::improved;
+    }
+    mathConvertionPM.addPass(
+        mlir::createConvertComplexToStandardPass(complexToStandardOptions));
+
     // Convert Math dialect operations into LLVM dialect operations.
     // There is no way to prefer MathToLLVM patterns over MathToLibm
     // patterns (applied below), so we have to run MathToLLVM conversion here.
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 42d9e7ba2418f..d940934f9f6ad 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -113,6 +113,7 @@ void addFIRToLLVMPass(mlir::PassManager &pm,
   options.forceUnifiedTBAATree = useOldAliasTags;
   options.typeDescriptorsRenamedForAssembly =
       !disableCompilerGeneratedNamesConversion;
+  options.ComplexRange = config.ComplexRange;
   addPassConditionally(pm, disableFirToLlvmIr,
                        [&]() { return fir::createFIRToLLVMPass(options); });
   // The dialect conversion framework may leave dead unrealized_conversion_cast
diff --git a/flang/test/Driver/complex-range.f90 b/flang/test/Driver/complex-range.f90
new file mode 100644
index 0000000000000..e5a1ba9068ac9
--- /dev/null
+++ b/flang/test/Driver/complex-range.f90
@@ -0,0 +1,23 @@
+! Test range options for complex multiplication and division.
+
+! RUN: %flang -### -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=RANGE
+
+! RUN: %flang -### -fcomplex-arithmetic=full -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=FULL
+
+! RUN: %flang -### -fcomplex-arithmetic=improved -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=IMPRVD
+
+! RUN: %flang -### -fcomplex-arithmetic=basic -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=BASIC
+
+! RUN: not %flang -### -fcomplex-arithmetic=foo -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=ERR
+
+! RANGE-NOT: -complex-range=
+! FULL: -complex-range=full
+! IMPRVD: -complex-range=improved
+! BASIC: -complex-range=basic
+
+! ERR: error: unsupported argument 'foo' to option '-fcomplex-arithmetic='
diff --git a/flang/test/Integration/complex-div-to-llvm-kind10.f90 b/flang/test/Integration/complex-div-to-llvm-kind10.f90
new file mode 100644
index 0000000000000..04d1f7ed9b024
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm-kind10.f90
@@ -0,0 +1,131 @@
+! Test lowering complex division to llvm ir according to options
+
+! REQUIRES: target=x86_64{{.*}}
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_extended
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { x86_fp80, x86_fp80 }, ptr %[[LHS]], align 16
+! CHECK: %[[LOAD_RHS:.*]] = load { x86_fp80, x86_fp80 }, ptr %[[RHS]], align 16
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract x86_fp80 %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract x86_fp80 %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract x86_fp80 %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract x86_fp80 %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract x86_fp80 %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract x86_fp80 %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract x86_fp80 %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract x86_fp80 %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract x86_fp80 %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract x86_fp80 %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract x86_fp80 %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract x86_fp80 %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq x86_fp80 %[[RHS_REAL_ABS]], 0xK00000000000000000000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq x86_fp80 %[[RHS_IMAG_ABS]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord x86_fp80 %[[LHS_REAL]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord x86_fp80 %[[LHS_IMAG]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 0xK7FFF8000000000000000, x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract x86_fp80 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract x86_fp80 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one x86_fp80 %[[RHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one x86_fp80 %[[RHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq x86_fp80 %[[LHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq x86_fp80 %[[LHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[LHS_REAL_IS_INF]], x86_fp80 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[LHS_IMAG_IS_INF]], x86_fp80 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract x86_fp80 %[[INF_MULTIPLICATOR_1]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract x86_fp80 %[[INF_MULTIPLICATOR_2]], 0xK7FFF8000000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one x86_fp80 %[[LHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one x86_fp80 %[[LHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq x86_fp80 %[[RHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq x86_fp80 %[[RHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[RHS_REAL_IS_INF]], x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[RHS_IMAG_IS_INF]], x86_fp80 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract x86_fp80 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract x86_fp80 %[[ZERO_MULTIPLICATOR_1]], 0xK00000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract x86_fp80 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract x86_fp80 %[[ZERO_MULTIPLICATOR_2]], 0xK00000000000000000000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt x86_fp80 %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], x86_fp80 %[[RESULT_REAL_1]], x86_fp80 %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], x86_fp80 %[[RESULT_IMAG_1]], x86_fp80 %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], x86_fp80 %[[RESULT_REAL_4]], x86_fp80 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], x86_fp80 %[[RESULT_IMAG_4]], x86_fp80 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], x86_fp80 %[[RESULT_REAL_3]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], x86_fp80 %[[RESULT_IMAG_3]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], x86_fp80 %[[INFINITY_RESULT_REAL]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], x86_fp80 %[[INFINITY_RESULT_IMAG]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno x86_fp80 %[[RESULT_REAL]], 0xK00000000000000000000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno x86_fp80 %[[RESULT_IMAG]], 0xK00000000000000000000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_1]], x86_fp80 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_1]], x86_fp80 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { x86_fp80, x86_fp80 } %[[RESULT_1]], x86_fp80 %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { x86_fp80, x86_fp80 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract x86_fp80 %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract x86_fp80 %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract x86_fp80 %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract x86_fp80 %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract x86_fp80 %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract x86_fp80 %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract x86_fp80 %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { x86_fp80, x86_fp80 } %[[RESULT_1]], x86_fp80 %[[IMAG]], 1
+! BASIC: store { x86_fp80, x86_fp80 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! CHECK: ret void
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
diff --git a/flang/test/Integration/complex-div-to-llvm-kind16.f90 b/flang/test/Integration/complex-div-to-llvm-kind16.f90
new file mode 100644
index 0000000000000..887a7972dc1be
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm-kind16.f90
@@ -0,0 +1,131 @@
+! Test lowering complex division to llvm ir according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_quad
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { fp128, fp128 }, ptr %[[LHS]], align 16
+! CHECK: %[[LOAD_RHS:.*]] = load { fp128, fp128 }, ptr %[[RHS]], align 16
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { fp128, fp128 } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { fp128, fp128 } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { fp128, fp128 } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { fp128, fp128 } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract fp128 %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract fp128 %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract fp128 %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract fp128 %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract fp128 %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract fp128 %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract fp128 %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract fp128 %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract fp128 %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract fp128 %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract fp128 %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract fp128 %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq fp128 %[[RHS_REAL_ABS]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord fp128 %[[LHS_REAL]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord fp128 %[[LHS_IMAG]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call fp128 @llvm.copysign.f128(fp128 0xL00000000000000007FFF000000000000, fp128 %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract fp128 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract fp128 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one fp128 %[[RHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq fp128 %[[LHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq fp128 %[[LHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[LHS_REAL_IS_INF]], fp128 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[LHS_IMAG_IS_INF]], fp128 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract fp128 %[[INF_MULTIPLICATOR_1]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract fp128 %[[INF_MULTIPLICATOR_2]], 0xL00000000000000007FFF000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one fp128 %[[LHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one fp128 %[[LHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq fp128 %[[RHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[RHS_REAL_IS_INF]], fp128 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[RHS_IMAG_IS_INF]], fp128 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract fp128 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract fp128 %[[ZERO_MULTIPLICATOR_1]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract fp128 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract fp128 %[[ZERO_MULTIPLICATOR_2]], 0xL00000000000000000000000000000000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt fp128 %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], fp128 %[[RESULT_REAL_1]], fp128 %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], fp128 %[[RESULT_IMAG_1]], fp128 %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], fp128 %[[RESULT_REAL_4]], fp128 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], fp128 %[[RESULT_IMAG_4]], fp128 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], fp128 %[[RESULT_REAL_3]], fp128 %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], fp128 %[[RESULT_IMAG_3]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], fp128 %[[INFINITY_RESULT_REAL]], fp128 %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], fp128 %[[INFINITY_RESULT_IMAG]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno fp128 %[[RESULT_REAL]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno fp128 %[[RESULT_IMAG]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], fp128 %[[RESULT_REAL_SPECIAL_CASE_1]], fp128 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_1]], fp128 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { fp128, fp128 } poison, fp128 %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { fp128, fp128 } %[[RESULT_1]], fp128 %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { fp128, fp128 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract fp128 %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract fp128 %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract fp128 %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract fp128 %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract fp128 %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract fp128 %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract fp128 %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { fp128, fp128 } poison, fp128 %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { fp128, fp128 } %[[RESULT_1]], fp128 %[[IMAG]], 1
+! BASIC: store { fp128, fp128 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! CHECK: ret void
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
diff --git a/flang/test/Integration/complex-div-to-llvm.f90 b/flang/test/Integration/complex-div-to-llvm.f90
new file mode 100644
index 0000000000000..01782a565f24c
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm.f90
@@ -0,0 +1,508 @@
+! Test lowering complex division to llvm ir according to options
+
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_half
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { half, half }, ptr %[[LHS]], align 2
+! CHECK: %[[LOAD_RHS:.*]] = load { half, half }, ptr %[[RHS]], align 2
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { half, half } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { half, half } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { half, half } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { half, half } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract half %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract half %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract half %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract half %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract half %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract half %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract half %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract half %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract half %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract half %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract half %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract half %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq half %[[RHS_REAL_ABS]], 0xH0000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq half %[[RHS_IMAG_ABS]], 0xH0000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord half %[[LHS_REAL]], 0xH0000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord half %[[LHS_IMAG]], 0xH0000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call half @llvm.copysign.f16(half 0xH7C00, half %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract half %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract half %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one half %[[RHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one half %[[RHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq half %[[LHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq half %[[LHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[LHS_REAL_IS_INF]], half %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[LHS_IMAG_IS_INF]], half %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract half %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract half %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract half %[[INF_MULTIPLICATOR_1]], 0xH7C00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract half %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract half %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract half %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract half %[[INF_MULTIPLICATOR_2]], 0xH7C00
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one half %[[LHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one half %[[LHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq half %[[RHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq half %[[RHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[RHS_REAL_IS_INF]], half %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[RHS_IMAG_IS_INF]], half %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract half %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract half %[[ZERO_MULTIPLICATOR_1]], 0xH0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract half %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract half %[[ZERO_MULTIPLICATOR_2]], 0xH0000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt half %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], half %[[RESULT_REAL_1]], half %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], half %[[RESULT_IMAG_1]], half %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], half %[[RESULT_REAL_4]], half %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], half %[[RESULT_IMAG_4]], half %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], half %[[RESULT_REAL_3]], half %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], half %[[RESULT_IMAG_3]], half %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], half %[[INFINITY_RESULT_REAL]], half %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], half %[[INFINITY_RESULT_IMAG]], half %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno half %[[RESULT_REAL]], 0xH0000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno half %[[RESULT_IMAG]], 0xH0000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], half %[[RESULT_REAL_SPECIAL_CASE_1]], half %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], half %[[RESULT_IMAG_SPECIAL_CASE_1]], half %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { half, half } poison, half %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { half, half } %[[RESULT_1]], half %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { half, half } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract half %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract half %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract half %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract half %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract half %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract half %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract half %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { half, half } poison, half %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { half, half } %[[RESULT_1]], half %[[IMAG]], 1
+! BASIC: store { half, half } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! CHECK: ret void
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+
+! CHECK-LABEL: @div_test_bfloat
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { bfloat, bfloat }, ptr %[[LHS]], align 2
+! CHECK: %[[LOAD_RHS:.*]] = load { bfloat, bfloat }, ptr %[[RHS]], align 2
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract bfloat %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract bfloat %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract bfloat %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract bfloat %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract bfloat %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract bfloat %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract bfloat %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract bfloat %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract bfloat %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract bfloat %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract bfloat %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract bfloat %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq bfloat %[[RHS_REAL_ABS]], 0xR0000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq bfloat %[[RHS_IMAG_ABS]], 0xR0000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord bfloat %[[LHS_REAL]], 0xR0000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord bfloat %[[LHS_IMAG]], 0xR0000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call bfloat @llvm.copysign.bf16(bfloat 0xR7F80, bfloat %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract bfloat %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract bfloat %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one bfloat %[[RHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one bfloat %[[RHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq bfloat %[[LHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq bfloat %[[LHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[LHS_REAL_IS_INF]], bfloat %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[LHS_IMAG_IS_INF]], bfloat %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract bfloat %[[INF_MULTIPLICATOR_1]], 0xR7F80
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract bfloat %[[INF_MULTIPLICATOR_2]], 0xR7F80
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one bfloat %[[LHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one bfloat %[[LHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq bfloat %[[RHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq bfloat %[[RHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[RHS_REAL_IS_INF]], bfloat %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[RHS_IMAG_IS_INF]], bfloat %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract bfloat %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract bfloat %[[ZERO_MULTIPLICATOR_1]], 0xR0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract bfloat %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract bfloat %[[ZERO_MULTIPLICATOR_2]], 0xR0000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt bfloat %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], bfloat %[[RESULT_REAL_1]], bfloat %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], bfloat %[[RESULT_IMAG_1]], bfloat %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], bfloat %[[RESULT_REAL_4]], bfloat %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], bfloat %[[RESULT_IMAG_4]], bfloat %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], bfloat %[[RESULT_REAL_3]], bfloat %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], bfloat %[[RESULT_IMAG_3]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], bfloat %[[INFINITY_RESULT_REAL]], bfloat %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], bfloat %[[INFINITY_RESULT_IMAG]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno bfloat %[[RESULT_REAL]], 0xR0000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno bfloat %[[RESULT_IMAG]], 0xR0000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], bfloat %[[RESULT_REAL_SPECIAL_CASE_1]], bfloat %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_1]], bfloat %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { bfloat, bfloat } poison, bfloat %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { bfloat, bfloat } %[[RESULT_1]], bfloat %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { bfloat, bfloat } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract bfloat %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract bfloat %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract bfloat %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract bfloat %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract bfloat %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract bfloat %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract bfloat %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { bfloat, bfloat } poison, bfloat %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { bfloat, bfloat } %[[RESULT_1]], bfloat %[[IMAG]], 1
+! BASIC: store { bfloat, bfloat } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! CHECK: ret void
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+
+! CHECK-LABEL: @div_test_single
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { float, float }, ptr %[[LHS]], align 4
+! CHECK: %[[LOAD_RHS:.*]] = load { float, float }, ptr %[[RHS]], align 4
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { float, float } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { float, float } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { float, float } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { float, float } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract float %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract float %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract float %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract float %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract float %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract float %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract float %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract float %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract float %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract float %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract float %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract float %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq float %[[RHS_REAL_ABS]], 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq float %[[RHS_IMAG_ABS]], 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord float %[[LHS_REAL]], 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord float %[[LHS_IMAG]], 0.000000e+00
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call float @llvm.copysign.f32(float 0x7FF0000000000000, float %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract float %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract float %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one float %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one float %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq float %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq float %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[LHS_REAL_IS_INF]], float %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[LHS_IMAG_IS_INF]], float %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract float %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract float %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract float %[[INF_MULTIPLICATOR_1]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract float %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract float %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract float %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract float %[[INF_MULTIPLICATOR_2]], 0x7FF0000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one float %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one float %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq float %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq float %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[RHS_REAL_IS_INF]], float %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[RHS_IMAG_IS_INF]], float %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract float %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract float %[[ZERO_MULTIPLICATOR_1]], 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract float %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract float %[[ZERO_MULTIPLICATOR_2]], 0.000000e+00
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt float %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], float %[[RESULT_REAL_1]], float %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], float %[[RESULT_IMAG_1]], float %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], float %[[RESULT_REAL_4]], float %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], float %[[RESULT_IMAG_4]], float %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], float %[[RESULT_REAL_3]], float %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], float %[[RESULT_IMAG_3]], float %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], float %[[INFINITY_RESULT_REAL]], float %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], float %[[INFINITY_RESULT_IMAG]], float %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno float %[[RESULT_REAL]], 0.000000e+00
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno float %[[RESULT_IMAG]], 0.000000e+00
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], float %[[RESULT_REAL_SPECIAL_CASE_1]], float %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], float %[[RESULT_IMAG_SPECIAL_CASE_1]], float %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { float, float } poison, float %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT_1]], float %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { float, float } %[[RESULT_2]], ptr %[[RET]], align 4
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract float %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract float %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract float %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract float %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract float %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract float %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract float %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { float, float } poison, float %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT_1]], float %[[IMAG]], 1
+! BASIC: store { float, float } %[[RESULT_2]], ptr %[[RET]], align 4
+
+! CHECK: ret void
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+
+! CHECK-LABEL: @div_test_double
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { double, double }, ptr %[[LHS]], align 8
+! CHECK: %[[LOAD_RHS:.*]] = load { double, double }, ptr %[[RHS]], align 8
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { double, double } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { double, double } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { double, double } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { double, double } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract double %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract double %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract double %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract double %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract double %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract double %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract double %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract double %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract double %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract double %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract double %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract double %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq double %[[RHS_REAL_ABS]], 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq double %[[RHS_IMAG_ABS]], 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord double %[[LHS_REAL]], 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord double %[[LHS_IMAG]], 0.000000e+00
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call double @llvm.copysign.f64(double 0x7FF0000000000000, double %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract double %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract double %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one double %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one double %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq double %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq double %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[LHS_REAL_IS_INF]], double %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[LHS_IMAG_IS_INF]], double %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract double %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract double %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract double %[[INF_MULTIPLICATOR_1]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract double %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract double %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract double %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract double %[[INF_MULTIPLICATOR_2]], 0x7FF0000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one double %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one double %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq double %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq double %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[RHS_REAL_IS_INF]], double %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[RHS_IMAG_IS_INF]], double %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract double %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract double %[[ZERO_MULTIPLICATOR_1]], 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract double %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract double %[[ZERO_MULTIPLICATOR_2]], 0.000000e+00
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt double %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], double %[[RESULT_REAL_1]], double %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], double %[[RESULT_IMAG_1]], double %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], double %[[RESULT_REAL_4]], double %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], double %[[RESULT_IMAG_4]], double %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], double %[[RESULT_REAL_3]], double %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], double %[[RESULT_IMAG_3]], double %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], double %[[INFINITY_RESULT_REAL]], double %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], double %[[INFINITY_RESULT_IMAG]], double %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno double %[[RESULT_REAL]], 0.000000e+00
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno double %[[RESULT_IMAG]], 0.000000e+00
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], double %[[RESULT_REAL_SPECIAL_CASE_1]], double %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], double %[[RESULT_IMAG_SPECIAL_CASE_1]], double %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { double, double } poison, double %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { double, double } %[[RESULT_1]], double %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { double, double } %[[RESULT_2]], ptr %[[RET]], align 8
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract double %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract double %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract double %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract double %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract double %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract double %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract double %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { double, double } poison, double %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { double, double } %[[RESULT_1]], double %[[IMAG]], 1
+! BASIC: store { double, double } %[[RESULT_2]], ptr %[[RET]], align 8
+
+! CHECK: ret void
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
new file mode 100644
index 0000000000000..0e219e30ec94b
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
@@ -0,0 +1,34 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: target=x86_64{{.*}}
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+! CHECK-LABEL: @_QPdiv_test_extended
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f80>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEa"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEb"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEc"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f80>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f80>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_RET:.*]] = fir.call @__divxc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f80, f80, f80, f80) -> complex<f80>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f80>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f80>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f80>, !fir.ref<complex<f80>>
+! CHECK: return
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
new file mode 100644
index 0000000000000..3e158b0bed998
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
@@ -0,0 +1,35 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_quad
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f128>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEa"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEb"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEc"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f128>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f128>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_RET:.*]] = fir.call @__divtc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f128, f128, f128, f128) -> complex<f128>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f128>, !fir.ref<complex<f128>>
+! CHECK: return
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
\ No newline at end of file
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90
new file mode 100644
index 0000000000000..b488bfde4ee85
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90
@@ -0,0 +1,94 @@
+! Test lowering of complex division according to options
+
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_half
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f16>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEa"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEb"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEc"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f16>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f16>>
+! CHECK: %[[VAL_9:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f16>
+! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : complex<f16>, !fir.ref<complex<f16>>
+! CHECK: return
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+
+! CHECK-LABEL: @_QPdiv_test_bfloat
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<bf16>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEa"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEb"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEc"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<bf16>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<bf16>>
+! CHECK: %[[VAL_9:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<bf16>
+! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : complex<bf16>, !fir.ref<complex<bf16>>
+! CHECK: return
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+
+! CHECK-LABEL: @_QPdiv_test_single
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f32>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEa"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEb"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEc"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f32>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_RET:.*]] = fir.call @__divsc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f32, f32, f32, f32) -> complex<f32>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f32>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f32>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f32>, !fir.ref<complex<f32>>
+! CHECK: return
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+
+! CHECK-LABEL: @_QPdiv_test_double
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f64>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEa"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEb"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEc"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f64>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f64>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_RET:.*]] = fir.call @__divdc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f64, f64, f64, f64) -> complex<f64>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f64>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f64>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f64>, !fir.ref<complex<f64>>
+! CHECK: return
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 59372a8eb58ed..2ccbe8b96d805 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -276,6 +276,12 @@ static llvm::cl::opt<bool>
                                      "leading dimension will be repacked"),
                       llvm::cl::init(true));
 
+static llvm::cl::opt<std::string> complexRange(
+    "complex-range",
+    llvm::cl::desc("Controls the various implementations for complex "
+                   "multiplication and division [full|improved|basic]"),
+    llvm::cl::init(""));
+
 #define FLANG_EXCLUDE_CODEGEN
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
@@ -437,6 +443,8 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setSkipExternalRttiDefinition(skipExternalRttiDefinition);
   if (enableCUDA)
     loweringOptions.setCUDARuntimeCheck(true);
+  if (complexRange == "improved" || complexRange == "basic")
+    loweringOptions.setComplexDivisionToRuntime(false);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   Fortran::frontend::TargetOptions targetOpts;
   Fortran::frontend::CodeGenOptions cgOpts;

>From 60e0cd9ece16f0f29ce2525d72e2ba37c0ae8447 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 10:19:12 +0900
Subject: [PATCH 2/4] Add help text

---
 clang/include/clang/Driver/Options.td | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 58209ceb5dc54..a1350e69a202e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1025,7 +1025,8 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
-  NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
+  NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>,
+  HelpText<"Controls the calculation methods of complex number multiplication and division.">;
 
 def complex_range_EQ : Joined<["-"], "complex-range=">, Group<f_Group>,
   Visibility<[CC1Option, FC1Option]>,

>From bddb2d33369ca4bdec50c3e927004555a7940d9a Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 12:01:16 +0900
Subject: [PATCH 3/4] Move common functions to CommonArgs.cpp

---
 clang/include/clang/Driver/CommonArgs.h    |  6 +++++
 clang/lib/Driver/ToolChains/Clang.cpp      | 27 ----------------------
 clang/lib/Driver/ToolChains/CommonArgs.cpp | 27 ++++++++++++++++++++++
 clang/lib/Driver/ToolChains/Flang.cpp      | 24 -------------------
 4 files changed, 33 insertions(+), 51 deletions(-)

diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index 26aa3ccf84786..cd1f3b2eb36ea 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -283,6 +283,12 @@ StringRef parseMPreferVectorWidthOption(clang::DiagnosticsEngine &Diags,
 StringRef parseMRecipOption(clang::DiagnosticsEngine &Diags,
                             const llvm::opt::ArgList &Args);
 
+// Convert ComplexRangeKind to a string that can be passed as a frontend option.
+std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range);
+
+// Render a frontend option corresponding to ComplexRangeKind.
+std::string RenderComplexRangeOption(LangOptions::ComplexRangeKind Range);
+
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index fea4ee909ff46..5d42ce56926d8 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2740,25 +2740,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
   }
 }
 
-static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
-  switch (Range) {
-  case LangOptions::ComplexRangeKind::CX_Full:
-    return "full";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Basic:
-    return "basic";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Improved:
-    return "improved";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Promoted:
-    return "promoted";
-    break;
-  default:
-    return "";
-  }
-}
-
 static std::string ComplexArithmeticStr(LangOptions::ComplexRangeKind Range) {
   return (Range == LangOptions::ComplexRangeKind::CX_None)
              ? ""
@@ -2772,14 +2753,6 @@ static void EmitComplexRangeDiag(const Driver &D, std::string str1,
   }
 }
 
-static std::string
-RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
-  if (!ComplexRangeStr.empty())
-    return "-complex-range=" + ComplexRangeStr;
-  return ComplexRangeStr;
-}
-
 static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
                                        bool OFastEnabled, const ArgList &Args,
                                        ArgStringList &CmdArgs,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..a6735e5e412d4 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -3409,3 +3409,30 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
 
   return Out;
 }
+
+std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Promoted:
+    return "promoted";
+    break;
+  default:
+    return "";
+  }
+}
+
+std::string
+tools::RenderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index ba5db7e3aba43..b5767918b86d1 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -595,30 +595,6 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
-static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
-  switch (Range) {
-  case LangOptions::ComplexRangeKind::CX_Full:
-    return "full";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Improved:
-    return "improved";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Basic:
-    return "basic";
-    break;
-  default:
-    return "";
-  }
-}
-
-static std::string
-RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
-  if (!ComplexRangeStr.empty())
-    return "-complex-range=" + ComplexRangeStr;
-  return ComplexRangeStr;
-}
-
 static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   StringRef FPContract;

>From 65c522ccd201fd48c481990ac0b1ec9b8be6d163 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 12:01:54 +0900
Subject: [PATCH 4/4] Minor fixes

---
 flang/lib/Frontend/CompilerInvocation.cpp              | 2 +-
 flang/lib/Lower/ConvertExprToHLFIR.cpp                 | 5 +++--
 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 86ec410b1f70f..910e0f5dcebd5 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -485,7 +485,7 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
 
   parseDoConcurrentMapping(opts, args, diags);
 
-  if (const auto *arg =
+  if (const llvm::opt::Arg *arg =
           args.getLastArg(clang::driver::options::OPT_complex_range_EQ)) {
     llvm::StringRef argValue = llvm::StringRef(arg->getValue());
     if (argValue == "full") {
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index cb338618dbf3b..9689f920840fb 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1070,12 +1070,13 @@ struct BinaryOp<Fortran::evaluate::Divide<
     // TODO: Ideally, complex number division operations should always be
     // lowered to MLIR. However, converting them to the runtime via MLIR causes
     // ABI issues.
-    if (builder.getComplexDivisionToRuntimeFlag())
+    if (builder.getComplexDivisionToRuntimeFlag()) {
       return hlfir::EntityWithAttributes{
           fir::genDivC(builder, loc, ty, lhs, rhs)};
-    else
+    } else {
       return hlfir::EntityWithAttributes{
           builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
+    }
   }
 };
 
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
index 3e158b0bed998..fe4a7256a4a16 100644
--- a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
@@ -32,4 +32,4 @@
 subroutine div_test_quad(a,b,c)
   complex(kind=16) :: a, b, c
   a = b / c
-end subroutine div_test_quad
\ No newline at end of file
+end subroutine div_test_quad


From flang-commits at lists.llvm.org  Fri Jul  4 02:06:23 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 04 Jul 2025 02:06:23 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Implement
 workdistribute construct lowering (PR #140523)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/140523@github.com>
Message-ID: <6867998f.050a0220.171fcb.4c5a@mx.google.com>


https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/140523

>From d80d54bf9178baecc46121f33eec861729544cd3 Mon Sep 17 00:00:00 2001
From: Ivan Radanov Ivanov <ivanov2 at llnl.gov>
Date: Mon, 4 Dec 2023 12:57:36 -0800
Subject: [PATCH 01/29] Add coexecute directives

---
 llvm/include/llvm/Frontend/OpenMP/OMP.td | 50 ++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a87111cb5a11d..696ec91083743 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -685,7 +685,7 @@ def OMP_CancellationPoint : Directive<[Spelling<"cancellation point">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_Critical : Directive<[Spelling<"critical">]> {
+def OMP_Critical : Directive<"critical"> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Hint>,
   ];
@@ -2206,8 +2206,34 @@ def OMP_TargetTeams : Directive<[Spelling<"target teams">]> {
   let leafConstructs = [OMP_Target, OMP_Teams];
   let category = CA_Executable;
 }
-def OMP_TargetTeamsDistribute
-    : Directive<[Spelling<"target teams distribute">]> {
+def OMP_TargetTeamsCoexecute : Directive<"target teams coexecute"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+  ];
+
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_OMPX_DynCGroupMem>,
+    VersionedClause<OMPC_OMX_Bare>,
+  ];
+}
+def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -2493,7 +2519,23 @@ def OMP_TaskLoopSimd : Directive<[Spelling<"taskloop simd">]> {
   let leafConstructs = [OMP_TaskLoop, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_TeamsDistribute : Directive<[Spelling<"teams distribute">]> {
+def OMP_TeamsCoexecute : Directive<"teams coexecute"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_If, 52>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>
+  ];
+}
+def OMP_TeamsDistribute : Directive<"teams distribute"> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,

>From 9235aaee4a4cb60fd85227681c3bbf1c99f6b01d Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Tue, 13 May 2025 11:01:45 +0530
Subject: [PATCH 02/29] [OpenMP] Fix Coexecute definitions

---
 llvm/include/llvm/Frontend/OpenMP/OMP.td | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 696ec91083743..9b325057a01b7 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -685,6 +685,15 @@ def OMP_CancellationPoint : Directive<[Spelling<"cancellation point">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
+def OMP_Coexecute : Directive<"coexecute"> {
+  let association = AS_Block;
+  let category = CA_Executable;
+}
+def OMP_EndCoexecute : Directive<"end coexecute"> {
+  let leafConstructs = OMP_Coexecute.leafConstructs;
+  let association = OMP_Coexecute.association;
+  let category = OMP_Coexecute.category;
+}
 def OMP_Critical : Directive<"critical"> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Hint>,
@@ -2230,8 +2239,10 @@ def OMP_TargetTeamsCoexecute : Directive<"target teams coexecute"> {
     VersionedClause<OMPC_NumTeams>,
     VersionedClause<OMPC_ThreadLimit>,
     VersionedClause<OMPC_OMPX_DynCGroupMem>,
-    VersionedClause<OMPC_OMX_Bare>,
+    VersionedClause<OMPC_OMPX_Bare>,
   ];
+  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Coexecute];
+  let category = CA_Executable;
 }
 def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> {
   let allowedClauses = [
@@ -2534,6 +2545,8 @@ def OMP_TeamsCoexecute : Directive<"teams coexecute"> {
     VersionedClause<OMPC_NumTeams>,
     VersionedClause<OMPC_ThreadLimit>
   ];
+  let leafConstructs = [OMP_Target, OMP_Teams];
+  let category = CA_Executable;
 }
 def OMP_TeamsDistribute : Directive<"teams distribute"> {
   let allowedClauses = [

>From e67e21ef820bdb506866345479bb74a51ae31de5 Mon Sep 17 00:00:00 2001
From: Ivan Radanov Ivanov <ivanov2 at llnl.gov>
Date: Mon, 4 Dec 2023 12:58:10 -0800
Subject: [PATCH 03/29] Add omp.coexecute op

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..bb61a46e13d6b 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -325,6 +325,41 @@ def SectionsOp : OpenMP_Op<"sections", traits = [
   let hasRegionVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// Coexecute Construct
+//===----------------------------------------------------------------------===//
+
+def CoexecuteOp : OpenMP_Op<"coexecute"> {
+  let summary = "coexecute directive";
+  let description = [{
+    The coexecute construct specifies that the teams from the teams directive
+    this is nested in shall cooperate to execute the computation in this region.
+    There is no implicit barrier at the end as specified in the standard.
+
+    TODO
+    We should probably change the defaut behaviour to have a barrier unless
+    nowait is specified, see below snippet.
+
+    ```
+    !$omp target teams
+        !$omp coexecute
+                tmp = matmul(x, y)
+        !$omp end coexecute
+        a = tmp(0, 0) ! there is no implicit barrier! the matmul hasnt completed!
+    !$omp end target teams coexecute
+    ```
+
+  }];
+
+  let arguments = (ins UnitAttr:$nowait);
+
+  let regions = (region AnyRegion:$region);
+
+  let assemblyFormat = [{
+    oilist(`nowait` $nowait) $region attr-dict
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // 2.8.2 Single Construct
 //===----------------------------------------------------------------------===//

>From 2fb623becf2b4ea69fe48a0a51a6ac40cac1e0c6 Mon Sep 17 00:00:00 2001
From: Ivan Radanov Ivanov <ivanov2 at llnl.gov>
Date: Mon, 4 Dec 2023 17:50:41 -0800
Subject: [PATCH 04/29] Initial frontend support for coexecute

---
 .../include/flang/Semantics/openmp-directive-sets.h | 13 +++++++++++++
 flang/lib/Lower/OpenMP/OpenMP.cpp                   | 12 ++++++++++++
 flang/lib/Parser/openmp-parsers.cpp                 |  5 ++++-
 flang/lib/Semantics/resolve-directives.cpp          |  6 ++++++
 4 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index dd610c9702c28..5c316e030c63f 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,6 +143,7 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
+    Directive::OMPD_target_teams_coexecute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -187,9 +188,16 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
+        Directive::OMPD_target_teams_coexecute,
     } | topTeamsSet,
 };
 
+static const OmpDirectiveSet allCoexecuteSet{
+    Directive::OMPD_coexecute,
+    Directive::OMPD_teams_coexecute,
+    Directive::OMPD_target_teams_coexecute,
+};
+
 //===----------------------------------------------------------------------===//
 // Directive sets for groups of multiple directives
 //===----------------------------------------------------------------------===//
@@ -230,6 +238,9 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
+    Directive::OMPD_target_teams_coexecute,
+    Directive::OMPD_teams_coexecute,
+    Directive::OMPD_coexecute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -294,6 +305,7 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
+        Directive::OMPD_coexecute,
     } | allDoSet,
 };
 
@@ -376,6 +388,7 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
+    Directive::OMPD_coexecute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ebd1d038716e4..390bba9cc5adf 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2682,6 +2682,15 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::omp::CoexecuteOp
+genCoexecuteOp(Fortran::lower::AbstractConverter &converter,
+               Fortran::lower::pft::Evaluation &eval,
+               mlir::Location currentLocation,
+               const Fortran::parser::OmpClauseList &clauseList) {
+  return genOpWithBody<mlir::omp::CoexecuteOp>(
+      converter, eval, currentLocation, /*outerCombined=*/false, &clauseList);
+}
+
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3296,6 +3305,9 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     newOp = genTeamsOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
                        item);
     break;
+  case llvm::omp::Directive::OMPD_coexecute:
+    newOp = genCoexecuteOp(converter, eval, currentLocation, beginClauseList);
+    break;
   case llvm::omp::Directive::OMPD_tile:
   case llvm::omp::Directive::OMPD_unroll: {
     unsigned version = semaCtx.langOptions().OpenMPVersion;
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..ebf2bb0c19bfd 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1492,12 +1492,15 @@ TYPE_PARSER(
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
+        "TARGET TEAMS COEXECUTE" >> pure(llvm::omp::Directive::OMPD_target_teams_coexecute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
+        "TEAMS COEXECUTE" >> pure(llvm::omp::Directive::OMPD_teams_coexecute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
-        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare))))
+        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
+        "COEXECUTE" >> pure(llvm::omp::Directive::OMPD_coexecute))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..133d4a6c18f17 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1656,6 +1656,9 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_coexecute:
+  case llvm::omp::Directive::OMPD_teams_coexecute:
+  case llvm::omp::Directive::OMPD_target_teams_coexecute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
@@ -1689,6 +1692,9 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_coexecute:
+  case llvm::omp::Directive::OMPD_teams_coexecute:
+  case llvm::omp::Directive::OMPD_target_teams_coexecute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
   case llvm::omp::Directive::OMPD_target_parallel: {

>From 21a93675e2b15a853fa69d80b9ecb4bc7b47156d Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Tue, 13 May 2025 15:09:45 +0530
Subject: [PATCH 05/29] [OpenMP] Fixes for coexecute definitions

---
 .../flang/Semantics/openmp-directive-sets.h   |  1 +
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 13 ++--
 flang/test/Lower/OpenMP/coexecute.f90         | 59 +++++++++++++++++++
 llvm/include/llvm/Frontend/OpenMP/OMP.td      | 33 +++++------
 4 files changed, 83 insertions(+), 23 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/coexecute.f90

diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index 5c316e030c63f..43f4e642b3d86 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -173,6 +173,7 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
+    Directive::OMPD_teams_coexecute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 390bba9cc5adf..0c436c15fd6c9 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2683,12 +2683,13 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 }
 
 static mlir::omp::CoexecuteOp
-genCoexecuteOp(Fortran::lower::AbstractConverter &converter,
-               Fortran::lower::pft::Evaluation &eval,
-               mlir::Location currentLocation,
-               const Fortran::parser::OmpClauseList &clauseList) {
+genCoexecuteOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+            mlir::Location loc, const ConstructQueue &queue,
+            ConstructQueue::const_iterator item) {
   return genOpWithBody<mlir::omp::CoexecuteOp>(
-      converter, eval, currentLocation, /*outerCombined=*/false, &clauseList);
+    OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
+                      llvm::omp::Directive::OMPD_coexecute), queue, item);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3306,7 +3307,7 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
                        item);
     break;
   case llvm::omp::Directive::OMPD_coexecute:
-    newOp = genCoexecuteOp(converter, eval, currentLocation, beginClauseList);
+    newOp = genCoexecuteOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_tile:
   case llvm::omp::Directive::OMPD_unroll: {
diff --git a/flang/test/Lower/OpenMP/coexecute.f90 b/flang/test/Lower/OpenMP/coexecute.f90
new file mode 100644
index 0000000000000..b14f71f9bbbfa
--- /dev/null
+++ b/flang/test/Lower/OpenMP/coexecute.f90
@@ -0,0 +1,59 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! CHECK-LABEL: func @_QPtarget_teams_coexecute
+subroutine target_teams_coexecute()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.coexecute
+  !$omp target teams coexecute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end target teams coexecute
+end subroutine target_teams_coexecute
+
+! CHECK-LABEL: func @_QPteams_coexecute
+subroutine teams_coexecute()
+  ! CHECK: omp.teams
+  ! CHECK: omp.coexecute
+  !$omp teams coexecute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end teams coexecute
+end subroutine teams_coexecute
+
+! CHECK-LABEL: func @_QPtarget_teams_coexecute_m
+subroutine target_teams_coexecute_m()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.coexecute
+  !$omp target
+  !$omp teams
+  !$omp coexecute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end coexecute
+  !$omp end teams
+  !$omp end target
+end subroutine target_teams_coexecute_m
+
+! CHECK-LABEL: func @_QPteams_coexecute_m
+subroutine teams_coexecute_m()
+  ! CHECK: omp.teams
+  ! CHECK: omp.coexecute
+  !$omp teams
+  !$omp coexecute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end coexecute
+  !$omp end teams
+end subroutine teams_coexecute_m
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 9b325057a01b7..f9d790a7bc987 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -2217,29 +2217,28 @@ def OMP_TargetTeams : Directive<[Spelling<"target teams">]> {
 }
 def OMP_TargetTeamsCoexecute : Directive<"target teams coexecute"> {
   let allowedClauses = [
-    VersionedClause<OMPC_If>,
-    VersionedClause<OMPC_Map>,
-    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_FirstPrivate>,
-    VersionedClause<OMPC_IsDevicePtr>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
     VersionedClause<OMPC_Reduction>,
-    VersionedClause<OMPC_Allocate>,
-    VersionedClause<OMPC_UsesAllocators, 50>,
     VersionedClause<OMPC_Shared>,
-    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
   ];
-
   let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DefaultMap>,
     VersionedClause<OMPC_Device>,
     VersionedClause<OMPC_NoWait>,
-    VersionedClause<OMPC_DefaultMap>,
-    VersionedClause<OMPC_Default>,
     VersionedClause<OMPC_NumTeams>,
-    VersionedClause<OMPC_ThreadLimit>,
     VersionedClause<OMPC_OMPX_DynCGroupMem>,
     VersionedClause<OMPC_OMPX_Bare>,
+    VersionedClause<OMPC_ThreadLimit>,
   ];
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Coexecute];
   let category = CA_Executable;
@@ -2532,20 +2531,20 @@ def OMP_TaskLoopSimd : Directive<[Spelling<"taskloop simd">]> {
 }
 def OMP_TeamsCoexecute : Directive<"teams coexecute"> {
   let allowedClauses = [
-    VersionedClause<OMPC_Private>,
-    VersionedClause<OMPC_FirstPrivate>,
-    VersionedClause<OMPC_Shared>,
-    VersionedClause<OMPC_Reduction>,
     VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
   ];
   let allowedOnceClauses = [
     VersionedClause<OMPC_Default>,
     VersionedClause<OMPC_If, 52>,
     VersionedClause<OMPC_NumTeams>,
-    VersionedClause<OMPC_ThreadLimit>
+    VersionedClause<OMPC_ThreadLimit>,
   ];
-  let leafConstructs = [OMP_Target, OMP_Teams];
+  let leafConstructs = [OMP_Teams, OMP_Coexecute];
   let category = CA_Executable;
 }
 def OMP_TeamsDistribute : Directive<"teams distribute"> {

>From 479f166e3836f485ef6d7143fecda9ef1d8ce84c Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 14 May 2025 14:48:52 +0530
Subject: [PATCH 06/29] [OpenMP] Use workdistribute instead of coexecute

---
 .../flang/Semantics/openmp-directive-sets.h   |  24 ++--
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  15 ++-
 flang/lib/Parser/openmp-parsers.cpp           |   6 +-
 flang/lib/Semantics/resolve-directives.cpp    |  12 +-
 flang/test/Lower/OpenMP/coexecute.f90         |  59 ----------
 flang/test/Lower/OpenMP/workdistribute.f90    |  59 ++++++++++
 llvm/include/llvm/Frontend/OpenMP/OMP.td      | 103 ++++++++++--------
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  28 ++---
 8 files changed, 153 insertions(+), 153 deletions(-)
 delete mode 100644 flang/test/Lower/OpenMP/coexecute.f90
 create mode 100644 flang/test/Lower/OpenMP/workdistribute.f90

diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index 43f4e642b3d86..7ced6ed9b44d6 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,7 +143,7 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
-    Directive::OMPD_target_teams_coexecute,
+    Directive::OMPD_target_teams_workdistribute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -173,7 +173,7 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
-    Directive::OMPD_teams_coexecute,
+    Directive::OMPD_teams_workdistribute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
@@ -189,14 +189,14 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
-        Directive::OMPD_target_teams_coexecute,
+        Directive::OMPD_target_teams_workdistribute,
     } | topTeamsSet,
 };
 
-static const OmpDirectiveSet allCoexecuteSet{
-    Directive::OMPD_coexecute,
-    Directive::OMPD_teams_coexecute,
-    Directive::OMPD_target_teams_coexecute,
+static const OmpDirectiveSet allWorkdistributeSet{
+    Directive::OMPD_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_target_teams_workdistribute,
 };
 
 //===----------------------------------------------------------------------===//
@@ -239,9 +239,9 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
-    Directive::OMPD_target_teams_coexecute,
-    Directive::OMPD_teams_coexecute,
-    Directive::OMPD_coexecute,
+    Directive::OMPD_target_teams_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_workdistribute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -306,7 +306,7 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
-        Directive::OMPD_coexecute,
+        Directive::OMPD_workdistribute,
     } | allDoSet,
 };
 
@@ -389,7 +389,7 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
-    Directive::OMPD_coexecute,
+    Directive::OMPD_workdistribute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0c436c15fd6c9..0133dbf912659 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2682,14 +2682,14 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
-static mlir::omp::CoexecuteOp
-genCoexecuteOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+static mlir::omp::WorkdistributeOp
+genWorkdistributeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
             semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
             mlir::Location loc, const ConstructQueue &queue,
             ConstructQueue::const_iterator item) {
-  return genOpWithBody<mlir::omp::CoexecuteOp>(
+  return genOpWithBody<mlir::omp::WorkdistributeOp>(
     OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                      llvm::omp::Directive::OMPD_coexecute), queue, item);
+                      llvm::omp::Directive::OMPD_workdistribute), queue, item);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3306,16 +3306,15 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     newOp = genTeamsOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
                        item);
     break;
-  case llvm::omp::Directive::OMPD_coexecute:
-    newOp = genCoexecuteOp(converter, symTable, semaCtx, eval, loc, queue, item);
-    break;
   case llvm::omp::Directive::OMPD_tile:
   case llvm::omp::Directive::OMPD_unroll: {
     unsigned version = semaCtx.langOptions().OpenMPVersion;
     TODO(loc, "Unhandled loop directive (" +
                   llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
   }
-  // case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_workdistribute:
+    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue, item);
+    break;
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
                            queue, item);
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index ebf2bb0c19bfd..cd0eccffa43f5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1492,15 +1492,15 @@ TYPE_PARSER(
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
-        "TARGET TEAMS COEXECUTE" >> pure(llvm::omp::Directive::OMPD_target_teams_coexecute),
+        "TARGET TEAMS WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
-        "TEAMS COEXECUTE" >> pure(llvm::omp::Directive::OMPD_teams_coexecute),
+        "TEAMS WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_teams_workdistribute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
         "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
-        "COEXECUTE" >> pure(llvm::omp::Directive::OMPD_coexecute))))
+        "WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_workdistribute))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 133d4a6c18f17..da3315ff1acfb 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1656,9 +1656,9 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
-  case llvm::omp::Directive::OMPD_coexecute:
-  case llvm::omp::Directive::OMPD_teams_coexecute:
-  case llvm::omp::Directive::OMPD_target_teams_coexecute:
+  case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_teams_workdistribute:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
@@ -1692,9 +1692,9 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
-  case llvm::omp::Directive::OMPD_coexecute:
-  case llvm::omp::Directive::OMPD_teams_coexecute:
-  case llvm::omp::Directive::OMPD_target_teams_coexecute:
+  case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_teams_workdistribute:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
   case llvm::omp::Directive::OMPD_target_parallel: {
diff --git a/flang/test/Lower/OpenMP/coexecute.f90 b/flang/test/Lower/OpenMP/coexecute.f90
deleted file mode 100644
index b14f71f9bbbfa..0000000000000
--- a/flang/test/Lower/OpenMP/coexecute.f90
+++ /dev/null
@@ -1,59 +0,0 @@
-! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
-
-! CHECK-LABEL: func @_QPtarget_teams_coexecute
-subroutine target_teams_coexecute()
-  ! CHECK: omp.target
-  ! CHECK: omp.teams
-  ! CHECK: omp.coexecute
-  !$omp target teams coexecute
-  ! CHECK: fir.call
-  call f1()
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  !$omp end target teams coexecute
-end subroutine target_teams_coexecute
-
-! CHECK-LABEL: func @_QPteams_coexecute
-subroutine teams_coexecute()
-  ! CHECK: omp.teams
-  ! CHECK: omp.coexecute
-  !$omp teams coexecute
-  ! CHECK: fir.call
-  call f1()
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  !$omp end teams coexecute
-end subroutine teams_coexecute
-
-! CHECK-LABEL: func @_QPtarget_teams_coexecute_m
-subroutine target_teams_coexecute_m()
-  ! CHECK: omp.target
-  ! CHECK: omp.teams
-  ! CHECK: omp.coexecute
-  !$omp target
-  !$omp teams
-  !$omp coexecute
-  ! CHECK: fir.call
-  call f1()
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  !$omp end coexecute
-  !$omp end teams
-  !$omp end target
-end subroutine target_teams_coexecute_m
-
-! CHECK-LABEL: func @_QPteams_coexecute_m
-subroutine teams_coexecute_m()
-  ! CHECK: omp.teams
-  ! CHECK: omp.coexecute
-  !$omp teams
-  !$omp coexecute
-  ! CHECK: fir.call
-  call f1()
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  !$omp end coexecute
-  !$omp end teams
-end subroutine teams_coexecute_m
diff --git a/flang/test/Lower/OpenMP/workdistribute.f90 b/flang/test/Lower/OpenMP/workdistribute.f90
new file mode 100644
index 0000000000000..924205bb72e5e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/workdistribute.f90
@@ -0,0 +1,59 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute
+subroutine target_teams_workdistribute()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end target teams workdistribute
+end subroutine target_teams_workdistribute
+
+! CHECK-LABEL: func @_QPteams_workdistribute
+subroutine teams_workdistribute()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end teams workdistribute
+end subroutine teams_workdistribute
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute_m
+subroutine target_teams_workdistribute_m()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+  !$omp end target
+end subroutine target_teams_workdistribute_m
+
+! CHECK-LABEL: func @_QPteams_workdistribute_m
+subroutine teams_workdistribute_m()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+end subroutine teams_workdistribute_m
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index f9d790a7bc987..b6d92d572206a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -1295,6 +1295,15 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
+def OMP_Workdistribute : Directive<"workdistribute"> {
+  let association = AS_Block;
+  let category = CA_Executable;
+}
+def OMP_EndWorkdistribute : Directive<"end workdistribute"> {
+  let leafConstructs = OMP_Workdistribute.leafConstructs;
+  let association = OMP_Workdistribute.association;
+  let category = OMP_Workdistribute.category;
+}
 
 //===----------------------------------------------------------------------===//
 // Definitions of OpenMP compound directives
@@ -2215,34 +2224,6 @@ def OMP_TargetTeams : Directive<[Spelling<"target teams">]> {
   let leafConstructs = [OMP_Target, OMP_Teams];
   let category = CA_Executable;
 }
-def OMP_TargetTeamsCoexecute : Directive<"target teams coexecute"> {
-  let allowedClauses = [
-    VersionedClause<OMPC_Allocate>,
-    VersionedClause<OMPC_Depend>,
-    VersionedClause<OMPC_FirstPrivate>,
-    VersionedClause<OMPC_HasDeviceAddr, 51>,
-    VersionedClause<OMPC_If>,
-    VersionedClause<OMPC_IsDevicePtr>,
-    VersionedClause<OMPC_Map>,
-    VersionedClause<OMPC_OMPX_Attribute>,
-    VersionedClause<OMPC_Private>,
-    VersionedClause<OMPC_Reduction>,
-    VersionedClause<OMPC_Shared>,
-    VersionedClause<OMPC_UsesAllocators, 50>,
-  ];
-  let allowedOnceClauses = [
-    VersionedClause<OMPC_Default>,
-    VersionedClause<OMPC_DefaultMap>,
-    VersionedClause<OMPC_Device>,
-    VersionedClause<OMPC_NoWait>,
-    VersionedClause<OMPC_NumTeams>,
-    VersionedClause<OMPC_OMPX_DynCGroupMem>,
-    VersionedClause<OMPC_OMPX_Bare>,
-    VersionedClause<OMPC_ThreadLimit>,
-  ];
-  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Coexecute];
-  let category = CA_Executable;
-}
 def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
@@ -2465,7 +2446,35 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
+def OMP_TargetTeamsWorkdistribute : Directive<"target teams workdistribute"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_OMPX_DynCGroupMem>,
+    VersionedClause<OMPC_OMPX_Bare>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
+def OMP_target_teams_loop : Directive<"target teams loop"> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_DefaultMap>,
@@ -2529,24 +2538,6 @@ def OMP_TaskLoopSimd : Directive<[Spelling<"taskloop simd">]> {
   let leafConstructs = [OMP_TaskLoop, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_TeamsCoexecute : Directive<"teams coexecute"> {
-  let allowedClauses = [
-    VersionedClause<OMPC_Allocate>,
-    VersionedClause<OMPC_FirstPrivate>,
-    VersionedClause<OMPC_OMPX_Attribute>,
-    VersionedClause<OMPC_Private>,
-    VersionedClause<OMPC_Reduction>,
-    VersionedClause<OMPC_Shared>,
-  ];
-  let allowedOnceClauses = [
-    VersionedClause<OMPC_Default>,
-    VersionedClause<OMPC_If, 52>,
-    VersionedClause<OMPC_NumTeams>,
-    VersionedClause<OMPC_ThreadLimit>,
-  ];
-  let leafConstructs = [OMP_Teams, OMP_Coexecute];
-  let category = CA_Executable;
-}
 def OMP_TeamsDistribute : Directive<"teams distribute"> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
@@ -2734,3 +2725,21 @@ def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let leafConstructs = [OMP_Teams, OMP_loop];
   let category = CA_Executable;
 }
+def OMP_TeamsWorkdistribute : Directive<"teams workdistribute"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_If, 52>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index bb61a46e13d6b..8d65b37330eb8 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -326,38 +326,30 @@ def SectionsOp : OpenMP_Op<"sections", traits = [
 }
 
 //===----------------------------------------------------------------------===//
-// Coexecute Construct
+// workdistribute Construct
 //===----------------------------------------------------------------------===//
 
-def CoexecuteOp : OpenMP_Op<"coexecute"> {
-  let summary = "coexecute directive";
+def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
+  let summary = "workdistribute directive";
   let description = [{
-    The coexecute construct specifies that the teams from the teams directive
-    this is nested in shall cooperate to execute the computation in this region.
-    There is no implicit barrier at the end as specified in the standard.
-
-    TODO
-    We should probably change the defaut behaviour to have a barrier unless
-    nowait is specified, see below snippet.
+    workdistribute divides execution of the enclosed structured block into
+    separate units of work, each executed only once by each
+    initial thread in the league.
 
     ```
     !$omp target teams
-        !$omp coexecute
+        !$omp workdistribute
                 tmp = matmul(x, y)
-        !$omp end coexecute
+        !$omp end workdistribute
         a = tmp(0, 0) ! there is no implicit barrier! the matmul hasnt completed!
-    !$omp end target teams coexecute
+    !$omp end target teams workdistribute
     ```
 
   }];
 
-  let arguments = (ins UnitAttr:$nowait);
-
   let regions = (region AnyRegion:$region);
 
-  let assemblyFormat = [{
-    oilist(`nowait` $nowait) $region attr-dict
-  }];
+  let assemblyFormat = "$region attr-dict";
 }
 
 //===----------------------------------------------------------------------===//

>From f7d6a3b9b30a82ca7849f47343bd94a430630efe Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 14 May 2025 16:17:14 +0530
Subject: [PATCH 07/29] [OpenMP] workdistribute trivial lowering

Lowering logic inspired from ivanradanov coexeute lowering
f56da1a207df4a40776a8570122a33f047074a3c
---
 .../include/flang/Optimizer/OpenMP/Passes.td  |   4 +
 flang/lib/Optimizer/OpenMP/CMakeLists.txt     |   1 +
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 101 ++++++++++++++++++
 .../OpenMP/lower-workdistribute.mlir          |  52 +++++++++
 4 files changed, 158 insertions(+)
 create mode 100644 flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
 create mode 100644 flang/test/Transforms/OpenMP/lower-workdistribute.mlir

diff --git a/flang/include/flang/Optimizer/OpenMP/Passes.td b/flang/include/flang/Optimizer/OpenMP/Passes.td
index 704faf0ccd856..743b6d381ed42 100644
--- a/flang/include/flang/Optimizer/OpenMP/Passes.td
+++ b/flang/include/flang/Optimizer/OpenMP/Passes.td
@@ -93,6 +93,10 @@ def LowerWorkshare : Pass<"lower-workshare", "::mlir::ModuleOp"> {
   let summary = "Lower workshare construct";
 }
 
+def LowerWorkdistribute : Pass<"lower-workdistribute", "::mlir::ModuleOp"> {
+  let summary = "Lower workdistribute construct";
+}
+
 def GenericLoopConversionPass
     : Pass<"omp-generic-loop-conversion", "mlir::func::FuncOp"> {
   let summary = "Converts OpenMP generic `omp.loop` to semantically "
diff --git a/flang/lib/Optimizer/OpenMP/CMakeLists.txt b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
index e31543328a9f9..cd746834741f9 100644
--- a/flang/lib/Optimizer/OpenMP/CMakeLists.txt
+++ b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
@@ -7,6 +7,7 @@ add_flang_library(FlangOpenMPTransforms
   MapsForPrivatizedSymbols.cpp
   MapInfoFinalization.cpp
   MarkDeclareTarget.cpp
+  LowerWorkdistribute.cpp
   LowerWorkshare.cpp
   LowerNontemporal.cpp
 
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
new file mode 100644
index 0000000000000..75c9d2b0d494e
--- /dev/null
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -0,0 +1,101 @@
+//===- LowerWorkshare.cpp - special cases for bufferization -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the lowering of omp.workdistribute.
+//
+//===----------------------------------------------------------------------===//
+
+#include <flang/Optimizer/Builder/FIRBuilder.h>
+#include <flang/Optimizer/Dialect/FIROps.h>
+#include <flang/Optimizer/Dialect/FIRType.h>
+#include <flang/Optimizer/HLFIR/HLFIROps.h>
+#include <flang/Optimizer/OpenMP/Passes.h>
+#include <llvm/ADT/BreadthFirstIterator.h>
+#include <llvm/ADT/STLExtras.h>
+#include <llvm/ADT/SmallVectorExtras.h>
+#include <llvm/ADT/iterator_range.h>
+#include <llvm/Support/ErrorHandling.h>
+#include <mlir/Dialect/Arith/IR/Arith.h>
+#include <mlir/Dialect/LLVMIR/LLVMTypes.h>
+#include <mlir/Dialect/OpenMP/OpenMPClauseOperands.h>
+#include <mlir/Dialect/OpenMP/OpenMPDialect.h>
+#include <mlir/Dialect/SCF/IR/SCF.h>
+#include <mlir/IR/BuiltinOps.h>
+#include <mlir/IR/IRMapping.h>
+#include <mlir/IR/OpDefinition.h>
+#include <mlir/IR/PatternMatch.h>
+#include <mlir/IR/Value.h>
+#include <mlir/IR/Visitors.h>
+#include <mlir/Interfaces/SideEffectInterfaces.h>
+#include <mlir/Support/LLVM.h>
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#include <variant>
+
+namespace flangomp {
+#define GEN_PASS_DEF_LOWERWORKDISTRIBUTE
+#include "flang/Optimizer/OpenMP/Passes.h.inc"
+} // namespace flangomp
+
+#define DEBUG_TYPE "lower-workdistribute"
+
+using namespace mlir;
+
+namespace {
+
+struct WorkdistributeToSingle : public mlir::OpRewritePattern<mlir::omp::WorkdistributeOp> {
+using OpRewritePattern::OpRewritePattern;
+mlir::LogicalResult
+    matchAndRewrite(mlir::omp::WorkdistributeOp workdistribute,
+                       mlir::PatternRewriter &rewriter) const override {
+        auto loc = workdistribute->getLoc();
+        auto teams = llvm::dyn_cast<mlir::omp::TeamsOp>(workdistribute->getParentOp());
+        if (!teams) {
+            mlir::emitError(loc, "workdistribute not nested in teams\n");
+            return mlir::failure();
+        }
+        if (workdistribute.getRegion().getBlocks().size() != 1) {
+            mlir::emitError(loc, "workdistribute with multiple blocks\n");
+            return mlir::failure();
+        }
+        if (teams.getRegion().getBlocks().size() != 1) {
+            mlir::emitError(loc, "teams with multiple blocks\n");
+           return mlir::failure();
+        }
+        if (teams.getRegion().getBlocks().front().getOperations().size() != 2) {
+            mlir::emitError(loc, "teams with multiple nested ops\n");
+            return mlir::failure();
+        }
+        mlir::Block *workdistributeBlock = &workdistribute.getRegion().front();
+        rewriter.eraseOp(workdistributeBlock->getTerminator());
+        rewriter.inlineBlockBefore(workdistributeBlock, teams);
+        rewriter.eraseOp(teams);
+        return mlir::success();
+    }
+};
+
+class LowerWorkdistributePass
+    : public flangomp::impl::LowerWorkdistributeBase<LowerWorkdistributePass> {
+public:
+  void runOnOperation() override {
+    mlir::MLIRContext &context = getContext();
+    mlir::RewritePatternSet patterns(&context);
+    mlir::GreedyRewriteConfig config;
+    // prevent the pattern driver form merging blocks
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
+  
+    patterns.insert<WorkdistributeToSingle>(&context);
+    mlir::Operation *op = getOperation();
+    if (mlir::failed(mlir::applyPatternsGreedily(op, std::move(patterns), config))) {
+      mlir::emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
+      signalPassFailure();
+    }
+  }
+};
+}
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute.mlir
new file mode 100644
index 0000000000000..34c8c3f01976d
--- /dev/null
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute.mlir
@@ -0,0 +1,52 @@
+// RUN: fir-opt --lower-workdistribute %s | FileCheck %s
+
+// CHECK-LABEL:   func.func @_QPtarget_simple() {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFtarget_simpleEa"}
+// CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "simple_var", uniq_name = "_QFtarget_simpleEsimple_var"}
+// CHECK:           %[[VAL_4:.*]] = fir.zero_bits !fir.heap<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_3]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+// CHECK:           hlfir.assign %[[VAL_0]] to %[[VAL_2]]#0 : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_7:.*]] = omp.map.info var_ptr(%[[VAL_2]]#1 : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32> {name = "a"}
+// CHECK:           omp.target map_entries(%[[VAL_7]] -> %[[VAL_8:.*]] : !fir.ref<i32>) private(@_QFtarget_simpleEsimple_var_private_ref_box_heap_i32 %[[VAL_6]]#0 -> %[[VAL_9:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>) {
+// CHECK:             %[[VAL_10:.*]] = arith.constant 10 : i32
+// CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_9]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+// CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
+// CHECK:             %[[VAL_14:.*]] = arith.addi %[[VAL_13]], %[[VAL_10]] : i32
+// CHECK:             hlfir.assign %[[VAL_14]] to %[[VAL_12]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
+// CHECK:             omp.terminator
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @_QPtarget_simple() {
+    %0 = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFtarget_simpleEa"}
+    %1:2 = hlfir.declare %0 {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+    %2 = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "simple_var", uniq_name = "_QFtarget_simpleEsimple_var"}
+    %3 = fir.zero_bits !fir.heap<i32>
+    %4 = fir.embox %3 : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+    fir.store %4 to %2 : !fir.ref<!fir.box<!fir.heap<i32>>>
+    %5:2 = hlfir.declare %2 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+    %c2_i32 = arith.constant 2 : i32
+    hlfir.assign %c2_i32 to %1#0 : i32, !fir.ref<i32>
+    %6 = omp.map.info var_ptr(%1#1 : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32> {name = "a"}
+    omp.target map_entries(%6 -> %arg0 : !fir.ref<i32>) private(@_QFtarget_simpleEsimple_var_private_ref_box_heap_i32 %5#0 -> %arg1 : !fir.ref<!fir.box<!fir.heap<i32>>>){
+        omp.teams {
+            omp.workdistribute {
+                %11:2 = hlfir.declare %arg0 {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+                %12:2 = hlfir.declare %arg1 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+                %c10_i32 = arith.constant 10 : i32
+                %13 = fir.load %11#0 : !fir.ref<i32>
+                %14 = arith.addi %c10_i32, %13 : i32
+                hlfir.assign %14 to %12#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
+                omp.terminator
+            }
+            omp.terminator
+        }
+        omp.terminator
+    }
+    return
+}
\ No newline at end of file

>From 46001010c8dca18dcb5d3c1edcb109b39709ce52 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 14 May 2025 19:29:33 +0530
Subject: [PATCH 08/29] [Flang][OpenMP] Add workdistribute lower pass to
 pipeline

---
 flang/lib/Optimizer/Passes/Pipelines.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 70f57bdeddd3f..c63e3799be650 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -288,8 +288,10 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
     addNestedPassToAllTopLevelOperations<PassConstructor>(
         pm, hlfir::createInlineHLFIRAssign);
   pm.addPass(hlfir::createConvertHLFIRtoFIR());
-  if (enableOpenMP)
+  if (enableOpenMP) {
     pm.addPass(flangomp::createLowerWorkshare());
+    pm.addPass(flangomp::createLowerWorkdistribute());
+  }
 }
 
 /// Create a pass pipeline for handling certain OpenMP transformations needed

>From 3ad6d57b2b68304012bb6917e881b3fafffe651b Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 15 May 2025 16:39:21 +0530
Subject: [PATCH 09/29] [Flang][OpenMP] Add FissionWorkdistribute lowering.

Fission logic inspired from ivanradanov implementation :
c97eca4010e460aac5a3d795614ca0980bce4565
---
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 233 ++++++++++++++----
 .../OpenMP/lower-workdistribute-fission.mlir  |  60 +++++
 ...ir => lower-workdistribute-to-single.mlir} |   2 +-
 3 files changed, 243 insertions(+), 52 deletions(-)
 create mode 100644 flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
 rename flang/test/Transforms/OpenMP/{lower-workdistribute.mlir => lower-workdistribute-to-single.mlir} (99%)

diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index 75c9d2b0d494e..f799202be2645 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -10,31 +10,26 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <flang/Optimizer/Builder/FIRBuilder.h>
-#include <flang/Optimizer/Dialect/FIROps.h>
-#include <flang/Optimizer/Dialect/FIRType.h>
-#include <flang/Optimizer/HLFIR/HLFIROps.h>
-#include <flang/Optimizer/OpenMP/Passes.h>
-#include <llvm/ADT/BreadthFirstIterator.h>
-#include <llvm/ADT/STLExtras.h>
-#include <llvm/ADT/SmallVectorExtras.h>
-#include <llvm/ADT/iterator_range.h>
-#include <llvm/Support/ErrorHandling.h>
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "flang/Optimizer/HLFIR/Passes.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/Value.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include <mlir/Dialect/Arith/IR/Arith.h>
 #include <mlir/Dialect/LLVMIR/LLVMTypes.h>
-#include <mlir/Dialect/OpenMP/OpenMPClauseOperands.h>
-#include <mlir/Dialect/OpenMP/OpenMPDialect.h>
-#include <mlir/Dialect/SCF/IR/SCF.h>
+#include <mlir/Dialect/Utils/IndexingUtils.h>
+#include <mlir/IR/BlockSupport.h>
 #include <mlir/IR/BuiltinOps.h>
+#include <mlir/IR/Diagnostics.h>
 #include <mlir/IR/IRMapping.h>
-#include <mlir/IR/OpDefinition.h>
 #include <mlir/IR/PatternMatch.h>
-#include <mlir/IR/Value.h>
-#include <mlir/IR/Visitors.h>
 #include <mlir/Interfaces/SideEffectInterfaces.h>
 #include <mlir/Support/LLVM.h>
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-
+#include <optional>
 #include <variant>
 
 namespace flangomp {
@@ -48,52 +43,188 @@ using namespace mlir;
 
 namespace {
 
-struct WorkdistributeToSingle : public mlir::OpRewritePattern<mlir::omp::WorkdistributeOp> {
-using OpRewritePattern::OpRewritePattern;
-mlir::LogicalResult
-    matchAndRewrite(mlir::omp::WorkdistributeOp workdistribute,
-                       mlir::PatternRewriter &rewriter) const override {
-        auto loc = workdistribute->getLoc();
-        auto teams = llvm::dyn_cast<mlir::omp::TeamsOp>(workdistribute->getParentOp());
-        if (!teams) {
-            mlir::emitError(loc, "workdistribute not nested in teams\n");
-            return mlir::failure();
-        }
-        if (workdistribute.getRegion().getBlocks().size() != 1) {
-            mlir::emitError(loc, "workdistribute with multiple blocks\n");
-            return mlir::failure();
+template <typename T>
+static T getPerfectlyNested(Operation *op) {
+  if (op->getNumRegions() != 1)
+    return nullptr;
+  auto &region = op->getRegion(0);
+  if (region.getBlocks().size() != 1)
+    return nullptr;
+  auto *block = &region.front();
+  auto *firstOp = &block->front();
+  if (auto nested = dyn_cast<T>(firstOp))
+    if (firstOp->getNextNode() == block->getTerminator())
+      return nested;
+  return nullptr;
+}
+
+/// This is the single source of truth about whether we should parallelize an
+/// operation nested in an omp.workdistribute region.
+static bool shouldParallelize(Operation *op) {
+    // Currently we cannot parallelize operations with results that have uses
+    if (llvm::any_of(op->getResults(),
+                     [](OpResult v) -> bool { return !v.use_empty(); }))
+      return false;
+    // We will parallelize unordered loops - these come from array syntax
+    if (auto loop = dyn_cast<fir::DoLoopOp>(op)) {
+      auto unordered = loop.getUnordered();
+      if (!unordered)
+        return false;
+      return *unordered;
+    }
+    if (auto callOp = dyn_cast<fir::CallOp>(op)) {
+      auto callee = callOp.getCallee();
+      if (!callee)
+        return false;
+      auto *func = op->getParentOfType<ModuleOp>().lookupSymbol(*callee);
+      // TODO need to insert a check here whether it is a call we can actually
+      // parallelize currently
+      if (func->getAttr(fir::FIROpsDialect::getFirRuntimeAttrName()))
+        return true;
+      return false;
+    }
+    // We cannot parallise anything else
+    return false;
+}
+
+struct WorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
+    using OpRewritePattern::OpRewritePattern;
+    LogicalResult matchAndRewrite(omp::TeamsOp teamsOp,
+                                    PatternRewriter &rewriter) const override {
+        auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
+        if (!workdistributeOp) {
+            LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " No workdistribute nested\n");
+            return failure();
         }
-        if (teams.getRegion().getBlocks().size() != 1) {
-            mlir::emitError(loc, "teams with multiple blocks\n");
-           return mlir::failure();
+      
+        Block *workdistributeBlock = &workdistributeOp.getRegion().front();
+        rewriter.eraseOp(workdistributeBlock->getTerminator());
+        rewriter.inlineBlockBefore(workdistributeBlock, teamsOp);
+        rewriter.eraseOp(teamsOp);
+        workdistributeOp.emitWarning("unable to parallelize coexecute");
+        return success();
+    }
+};
+
+/// If B() and D() are parallelizable,
+///
+/// omp.teams {
+///   omp.workdistribute {
+///     A()
+///     B()
+///     C()
+///     D()
+///     E()
+///   }
+/// }
+///
+/// becomes
+///
+/// A()
+/// omp.teams {
+///   omp.workdistribute {
+///     B()
+///   }
+/// }
+/// C()
+/// omp.teams {
+///   omp.workdistribute {
+///     D()
+///   }
+/// }
+/// E()
+
+struct FissionWorkdistribute
+    : public OpRewritePattern<omp::WorkdistributeOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult
+  matchAndRewrite(omp::WorkdistributeOp workdistribute,
+                  PatternRewriter &rewriter) const override {
+    auto loc = workdistribute->getLoc();
+    auto teams = dyn_cast<omp::TeamsOp>(workdistribute->getParentOp());
+    if (!teams) {
+      emitError(loc, "workdistribute not nested in teams\n");
+      return failure();
+    }
+    if (workdistribute.getRegion().getBlocks().size() != 1) {
+      emitError(loc, "workdistribute with multiple blocks\n");
+      return failure();
+    }
+    if (teams.getRegion().getBlocks().size() != 1) {
+      emitError(loc, "teams with multiple blocks\n");
+      return failure();
+    }
+    if (teams.getRegion().getBlocks().front().getOperations().size() != 2) {
+      emitError(loc, "teams with multiple nested ops\n");
+      return failure();
+    }
+
+    auto *teamsBlock = &teams.getRegion().front();
+
+    // While we have unhandled operations in the original workdistribute
+    auto *workdistributeBlock = &workdistribute.getRegion().front();
+    auto *terminator = workdistributeBlock->getTerminator();
+    bool changed = false;
+    while (&workdistributeBlock->front() != terminator) {
+      rewriter.setInsertionPoint(teams);
+      IRMapping mapping;
+      llvm::SmallVector<Operation *> hoisted;
+      Operation *parallelize = nullptr;
+      for (auto &op : workdistribute.getOps()) {
+        if (&op == terminator) {
+          break;
         }
-        if (teams.getRegion().getBlocks().front().getOperations().size() != 2) {
-            mlir::emitError(loc, "teams with multiple nested ops\n");
-            return mlir::failure();
+        if (shouldParallelize(&op)) {
+          parallelize = &op;
+          break;
+        } else {
+          rewriter.clone(op, mapping);
+          hoisted.push_back(&op);
+          changed = true;
         }
-        mlir::Block *workdistributeBlock = &workdistribute.getRegion().front();
-        rewriter.eraseOp(workdistributeBlock->getTerminator());
-        rewriter.inlineBlockBefore(workdistributeBlock, teams);
-        rewriter.eraseOp(teams);
-        return mlir::success();
+      }
+
+      for (auto *op : hoisted)
+        rewriter.replaceOp(op, mapping.lookup(op));
+
+      if (parallelize && hoisted.empty() &&
+          parallelize->getNextNode() == terminator)
+        break;
+      if (parallelize) {
+        auto newTeams = rewriter.cloneWithoutRegions(teams);
+        auto *newTeamsBlock = rewriter.createBlock(
+            &newTeams.getRegion(), newTeams.getRegion().begin(), {}, {});
+        for (auto arg : teamsBlock->getArguments())
+          newTeamsBlock->addArgument(arg.getType(), arg.getLoc());
+        auto newWorkdistribute = rewriter.create<omp::WorkdistributeOp>(loc);
+        rewriter.create<omp::TerminatorOp>(loc);
+        rewriter.createBlock(&newWorkdistribute.getRegion(),
+                            newWorkdistribute.getRegion().begin(), {}, {});
+        auto *cloned = rewriter.clone(*parallelize);
+        rewriter.replaceOp(parallelize, cloned);
+        rewriter.create<omp::TerminatorOp>(loc);
+        changed = true;
+      }
     }
+    return success(changed);
+  }
 };
 
 class LowerWorkdistributePass
     : public flangomp::impl::LowerWorkdistributeBase<LowerWorkdistributePass> {
 public:
   void runOnOperation() override {
-    mlir::MLIRContext &context = getContext();
-    mlir::RewritePatternSet patterns(&context);
-    mlir::GreedyRewriteConfig config;
+    MLIRContext &context = getContext();
+    RewritePatternSet patterns(&context);
+    GreedyRewriteConfig config;
     // prevent the pattern driver form merging blocks
     config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
+        GreedySimplifyRegionLevel::Disabled);
   
-    patterns.insert<WorkdistributeToSingle>(&context);
-    mlir::Operation *op = getOperation();
-    if (mlir::failed(mlir::applyPatternsGreedily(op, std::move(patterns), config))) {
-      mlir::emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
+    patterns.insert<FissionWorkdistribute, WorkdistributeToSingle>(&context);
+    Operation *op = getOperation();
+    if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
+      emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
       signalPassFailure();
     }
   }
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
new file mode 100644
index 0000000000000..ea03a10dd3d44
--- /dev/null
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
@@ -0,0 +1,60 @@
+// RUN: fir-opt --lower-workdistribute %s | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_fission_workdistribute({{.*}}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 9 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 5.000000e+00 : f32
+// CHECK:           fir.store %[[VAL_3]] to %[[ARG2:.*]] : !fir.ref<f32>
+// CHECK:           fir.do_loop %[[VAL_4:.*]] = %[[VAL_0]] to %[[VAL_2]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_5:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:             %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<f32>
+// CHECK:             %[[VAL_7:.*]] = fir.coordinate_of %[[ARG1:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:             fir.store %[[VAL_6]] to %[[VAL_7]] : !fir.ref<f32>
+// CHECK:           }
+// CHECK:           fir.call @regular_side_effect_func(%[[ARG2:.*]]) : (!fir.ref<f32>) -> ()
+// CHECK:           fir.call @my_fir_parallel_runtime_func(%[[ARG3:.*]]) : (!fir.ref<f32>) -> ()
+// CHECK:           fir.do_loop %[[VAL_8:.*]] = %[[VAL_0]] to %[[VAL_2]] step %[[VAL_1]] {
+// CHECK:             %[[VAL_9:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[VAL_8]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:             fir.store %[[VAL_3]] to %[[VAL_9]] : !fir.ref<f32>
+// CHECK:           }
+// CHECK:           %[[VAL_10:.*]] = fir.load %[[ARG2:.*]] : !fir.ref<f32>
+// CHECK:           fir.store %[[VAL_10]] to %[[ARG3:.*]] : !fir.ref<f32>
+// CHECK:           return
+// CHECK:         }
+module {
+func.func @regular_side_effect_func(%arg0: !fir.ref<f32>) {
+  return
+}
+func.func @my_fir_parallel_runtime_func(%arg0: !fir.ref<f32>) attributes {fir.runtime} {
+  return
+}
+func.func @test_fission_workdistribute(%arr1: !fir.ref<!fir.array<10xf32>>, %arr2: !fir.ref<!fir.array<10xf32>>, %scalar_ref1: !fir.ref<f32>, %scalar_ref2: !fir.ref<f32>) {
+  %c0_idx = arith.constant 0 : index
+  %c1_idx = arith.constant 1 : index
+  %c9_idx = arith.constant 9 : index
+  %float_val = arith.constant 5.0 : f32
+  omp.teams   {
+    omp.workdistribute   {
+      fir.store %float_val to %scalar_ref1 : !fir.ref<f32>
+      fir.do_loop %iv = %c0_idx to %c9_idx step %c1_idx unordered {
+        %elem_ptr_arr1 = fir.coordinate_of %arr1, %iv : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+        %loaded_val_loop1 = fir.load %elem_ptr_arr1 : !fir.ref<f32>
+        %elem_ptr_arr2 = fir.coordinate_of %arr2, %iv : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+        fir.store %loaded_val_loop1 to %elem_ptr_arr2 : !fir.ref<f32>
+      }
+      fir.call @regular_side_effect_func(%scalar_ref1) : (!fir.ref<f32>) -> ()
+      fir.call @my_fir_parallel_runtime_func(%scalar_ref2) : (!fir.ref<f32>) -> ()
+      fir.do_loop %jv = %c0_idx to %c9_idx step %c1_idx {
+        %elem_ptr_ordered_loop = fir.coordinate_of %arr1, %jv : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+        fir.store %float_val to %elem_ptr_ordered_loop : !fir.ref<f32>
+      }
+      %loaded_for_hoist = fir.load %scalar_ref1 : !fir.ref<f32>
+      fir.store %loaded_for_hoist to %scalar_ref2 : !fir.ref<f32>
+      omp.terminator  
+    }
+    omp.terminator
+  }
+  return
+}
+}
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-to-single.mlir
similarity index 99%
rename from flang/test/Transforms/OpenMP/lower-workdistribute.mlir
rename to flang/test/Transforms/OpenMP/lower-workdistribute-to-single.mlir
index 34c8c3f01976d..0cc2aeded2532 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-to-single.mlir
@@ -49,4 +49,4 @@ func.func @_QPtarget_simple() {
         omp.terminator
     }
     return
-}
\ No newline at end of file
+}

>From 7becfeede0bea64a137a5ef43e213213f3021462 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Sun, 18 May 2025 12:37:53 +0530
Subject: [PATCH 10/29] [OpenMP][Flang] Lower teams workdistribute do_loop to
 wsloop.

Logic inspired from ivanradanov commit
5682e9ea7fcba64693f7cfdc0f1970fab2d7d4ae
---
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 177 +++++++++++++++---
 .../OpenMP/lower-workdistribute-doloop.mlir   |  28 +++
 .../OpenMP/lower-workdistribute-fission.mlir  |  22 ++-
 3 files changed, 193 insertions(+), 34 deletions(-)
 create mode 100644 flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir

diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index f799202be2645..de208a8190650 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -6,18 +6,22 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements the lowering of omp.workdistribute.
+// This file implements the lowering and optimisations of omp.workdistribute.
 //
 //===----------------------------------------------------------------------===//
 
+#include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "flang/Optimizer/HLFIR/Passes.h"
+#include "flang/Optimizer/OpenMP/Utils.h"
+#include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Value.h"
+#include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include <mlir/Dialect/Arith/IR/Arith.h>
 #include <mlir/Dialect/LLVMIR/LLVMTypes.h>
@@ -29,6 +33,7 @@
 #include <mlir/IR/PatternMatch.h>
 #include <mlir/Interfaces/SideEffectInterfaces.h>
 #include <mlir/Support/LLVM.h>
+#include "mlir/Transforms/RegionUtils.h"
 #include <optional>
 #include <variant>
 
@@ -87,25 +92,6 @@ static bool shouldParallelize(Operation *op) {
     return false;
 }
 
-struct WorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
-    using OpRewritePattern::OpRewritePattern;
-    LogicalResult matchAndRewrite(omp::TeamsOp teamsOp,
-                                    PatternRewriter &rewriter) const override {
-        auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
-        if (!workdistributeOp) {
-            LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " No workdistribute nested\n");
-            return failure();
-        }
-      
-        Block *workdistributeBlock = &workdistributeOp.getRegion().front();
-        rewriter.eraseOp(workdistributeBlock->getTerminator());
-        rewriter.inlineBlockBefore(workdistributeBlock, teamsOp);
-        rewriter.eraseOp(teamsOp);
-        workdistributeOp.emitWarning("unable to parallelize coexecute");
-        return success();
-    }
-};
-
 /// If B() and D() are parallelizable,
 ///
 /// omp.teams {
@@ -210,22 +196,161 @@ struct FissionWorkdistribute
   }
 };
 
+static void
+genLoopNestClauseOps(mlir::Location loc,
+                     mlir::PatternRewriter &rewriter,
+                     fir::DoLoopOp loop,
+                     mlir::omp::LoopNestOperands &loopNestClauseOps) {
+  assert(loopNestClauseOps.loopLowerBounds.empty() &&
+         "Loop nest bounds were already emitted!");
+  loopNestClauseOps.loopLowerBounds.push_back(loop.getLowerBound());
+  loopNestClauseOps.loopUpperBounds.push_back(loop.getUpperBound());
+  loopNestClauseOps.loopSteps.push_back(loop.getStep());
+  loopNestClauseOps.loopInclusive = rewriter.getUnitAttr();
+}
+
+static void
+genWsLoopOp(mlir::PatternRewriter &rewriter,
+            fir::DoLoopOp doLoop,
+            const mlir::omp::LoopNestOperands &clauseOps) {
+
+  auto wsloopOp = rewriter.create<mlir::omp::WsloopOp>(doLoop.getLoc());
+  rewriter.createBlock(&wsloopOp.getRegion());
+
+  auto loopNestOp =
+      rewriter.create<mlir::omp::LoopNestOp>(doLoop.getLoc(), clauseOps);
+
+  // Clone the loop's body inside the loop nest construct using the
+  // mapped values.
+  rewriter.cloneRegionBefore(doLoop.getRegion(), loopNestOp.getRegion(),
+                             loopNestOp.getRegion().begin());
+  Block *clonedBlock = &loopNestOp.getRegion().back();
+  mlir::Operation *terminatorOp = clonedBlock->getTerminator();
+
+  // Erase fir.result op of do loop and create yield op.
+  if (auto resultOp = dyn_cast<fir::ResultOp>(terminatorOp)) {
+    rewriter.setInsertionPoint(terminatorOp);
+    rewriter.create<mlir::omp::YieldOp>(doLoop->getLoc());
+    rewriter.eraseOp(terminatorOp);
+  }
+  return;
+}
+
+/// If fir.do_loop id present inside teams workdistribute
+///
+/// omp.teams {
+///   omp.workdistribute {
+///     fir.do_loop unoredered {
+///       ...
+///     }
+///   }
+/// }
+///
+/// Then, its lowered to 
+///
+/// omp.teams {
+///   omp.workdistribute {
+///     omp.parallel {
+///       omp.wsloop {
+///         omp.loop_nest
+///           ...
+///         }
+///       }
+///     }
+///   }
+/// }
+
+struct TeamsWorkdistributeLowering : public OpRewritePattern<omp::TeamsOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(omp::TeamsOp teamsOp,
+                                PatternRewriter &rewriter) const override {
+    auto teamsLoc = teamsOp->getLoc();
+    auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
+    if (!workdistributeOp) {
+      LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " No workdistribute nested\n");
+      return failure();
+    }
+    assert(teamsOp.getReductionVars().empty());
+
+    auto doLoop = getPerfectlyNested<fir::DoLoopOp>(workdistributeOp);
+    if (doLoop && shouldParallelize(doLoop)) {
+
+      auto parallelOp = rewriter.create<mlir::omp::ParallelOp>(teamsLoc);
+      rewriter.createBlock(&parallelOp.getRegion());
+      rewriter.setInsertionPoint(rewriter.create<mlir::omp::TerminatorOp>(doLoop.getLoc()));
+
+      mlir::omp::LoopNestOperands loopNestClauseOps;
+      genLoopNestClauseOps(doLoop.getLoc(), rewriter, doLoop,
+                           loopNestClauseOps);
+
+      genWsLoopOp(rewriter, doLoop, loopNestClauseOps);
+      rewriter.setInsertionPoint(doLoop);
+      rewriter.eraseOp(doLoop);
+      return success();
+    }
+    return failure();
+  }
+};
+
+
+/// If A() and B () are present inside teams workdistribute
+///
+/// omp.teams {
+///   omp.workdistribute {
+///     A()
+///     B()
+///   }
+/// }
+///
+/// Then, its lowered to
+///
+/// A()
+/// B()
+///
+
+struct TeamsWorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(omp::TeamsOp teamsOp,
+                                  PatternRewriter &rewriter) const override {
+      auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
+      if (!workdistributeOp) {
+          LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " No workdistribute nested\n");
+          return failure();
+      }      
+      Block *workdistributeBlock = &workdistributeOp.getRegion().front();
+      rewriter.eraseOp(workdistributeBlock->getTerminator());
+      rewriter.inlineBlockBefore(workdistributeBlock, teamsOp);
+      rewriter.eraseOp(teamsOp);
+      return success();
+  }
+};
+
 class LowerWorkdistributePass
     : public flangomp::impl::LowerWorkdistributeBase<LowerWorkdistributePass> {
 public:
   void runOnOperation() override {
     MLIRContext &context = getContext();
-    RewritePatternSet patterns(&context);
     GreedyRewriteConfig config;
     // prevent the pattern driver form merging blocks
     config.setRegionSimplificationLevel(
         GreedySimplifyRegionLevel::Disabled);
-  
-    patterns.insert<FissionWorkdistribute, WorkdistributeToSingle>(&context);
+    
     Operation *op = getOperation();
-    if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
-      emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
-      signalPassFailure();
+    {
+      RewritePatternSet patterns(&context);
+      patterns.insert<FissionWorkdistribute, TeamsWorkdistributeLowering>(&context);
+      if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
+        emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
+        signalPassFailure();
+      }
+    }
+    {
+      RewritePatternSet patterns(&context);
+      patterns.insert<TeamsWorkdistributeLowering, TeamsWorkdistributeToSingle>(&context);
+      if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
+        emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
+        signalPassFailure();
+      }
     }
   }
 };
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
new file mode 100644
index 0000000000000..666bdb3ced647
--- /dev/null
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
@@ -0,0 +1,28 @@
+// RUN: fir-opt --lower-workdistribute %s | FileCheck %s
+
+// CHECK-LABEL:   func.func @x({{.*}})
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : index
+// CHECK:           omp.parallel {
+// CHECK:             omp.wsloop {
+// CHECK:               omp.loop_nest (%[[VAL_1:.*]]) : index = (%[[ARG0:.*]]) to (%[[ARG1:.*]]) inclusive step (%[[ARG2:.*]]) {
+// CHECK:                 fir.store %[[VAL_0]] to %[[ARG4:.*]] : !fir.ref<index>
+// CHECK:                 omp.yield
+// CHECK:               }
+// CHECK:             }
+// CHECK:             omp.terminator
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @x(%lb : index, %ub : index, %step : index, %b : i1, %addr : !fir.ref<index>) {
+  omp.teams {
+    omp.workdistribute { 
+      fir.do_loop %iv = %lb to %ub step %step unordered {
+        %zero = arith.constant 0 : index
+        fir.store %zero to %addr : !fir.ref<index>
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
\ No newline at end of file
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
index ea03a10dd3d44..cf50d135d01ec 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
@@ -6,20 +6,26 @@
 // CHECK:           %[[VAL_2:.*]] = arith.constant 9 : index
 // CHECK:           %[[VAL_3:.*]] = arith.constant 5.000000e+00 : f32
 // CHECK:           fir.store %[[VAL_3]] to %[[ARG2:.*]] : !fir.ref<f32>
-// CHECK:           fir.do_loop %[[VAL_4:.*]] = %[[VAL_0]] to %[[VAL_2]] step %[[VAL_1]] unordered {
-// CHECK:             %[[VAL_5:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
-// CHECK:             %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<f32>
-// CHECK:             %[[VAL_7:.*]] = fir.coordinate_of %[[ARG1:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
-// CHECK:             fir.store %[[VAL_6]] to %[[VAL_7]] : !fir.ref<f32>
+// CHECK:           omp.parallel {
+// CHECK:             omp.wsloop {
+// CHECK:               omp.loop_nest (%[[VAL_4:.*]]) : index = (%[[VAL_0]]) to (%[[VAL_2]]) inclusive step (%[[VAL_1]]) {
+// CHECK:                 %[[VAL_5:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_7:.*]] = fir.coordinate_of %[[ARG1:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:                 fir.store %[[VAL_6]] to %[[VAL_7]] : !fir.ref<f32>
+// CHECK:                 omp.yield
+// CHECK:               }
+// CHECK:             }
+// CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           fir.call @regular_side_effect_func(%[[ARG2:.*]]) : (!fir.ref<f32>) -> ()
 // CHECK:           fir.call @my_fir_parallel_runtime_func(%[[ARG3:.*]]) : (!fir.ref<f32>) -> ()
 // CHECK:           fir.do_loop %[[VAL_8:.*]] = %[[VAL_0]] to %[[VAL_2]] step %[[VAL_1]] {
-// CHECK:             %[[VAL_9:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[VAL_8]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:             %[[VAL_9:.*]] = fir.coordinate_of %[[ARG0]], %[[VAL_8]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
 // CHECK:             fir.store %[[VAL_3]] to %[[VAL_9]] : !fir.ref<f32>
 // CHECK:           }
-// CHECK:           %[[VAL_10:.*]] = fir.load %[[ARG2:.*]] : !fir.ref<f32>
-// CHECK:           fir.store %[[VAL_10]] to %[[ARG3:.*]] : !fir.ref<f32>
+// CHECK:           %[[VAL_10:.*]] = fir.load %[[ARG2]] : !fir.ref<f32>
+// CHECK:           fir.store %[[VAL_10]] to %[[ARG3]] : !fir.ref<f32>
 // CHECK:           return
 // CHECK:         }
 module {

>From b23969d36dc149e3ef86474400908c6079a5f15a Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 19 May 2025 15:33:53 +0530
Subject: [PATCH 11/29] clang format

---
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  18 +--
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 108 +++++++++---------
 flang/lib/Parser/openmp-parsers.cpp           |   6 +-
 .../OpenMP/lower-workdistribute-doloop.mlir   |   2 +-
 4 files changed, 67 insertions(+), 67 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0133dbf912659..a956498f6e521 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2682,14 +2682,15 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
-static mlir::omp::WorkdistributeOp
-genWorkdistributeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
-            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
-            mlir::Location loc, const ConstructQueue &queue,
-            ConstructQueue::const_iterator item) {
+static mlir::omp::WorkdistributeOp genWorkdistributeOp(
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::Location loc, const ConstructQueue &queue,
+    ConstructQueue::const_iterator item) {
   return genOpWithBody<mlir::omp::WorkdistributeOp>(
-    OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                      llvm::omp::Directive::OMPD_workdistribute), queue, item);
+      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
+                        llvm::omp::Directive::OMPD_workdistribute),
+      queue, item);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3313,7 +3314,8 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
                   llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
   }
   case llvm::omp::Directive::OMPD_workdistribute:
-    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue, item);
+    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue,
+                                item);
     break;
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index de208a8190650..f75d4d1988fd2 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -14,15 +14,16 @@
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
-#include "flang/Optimizer/Transforms/Passes.h"
 #include "flang/Optimizer/HLFIR/Passes.h"
 #include "flang/Optimizer/OpenMP/Utils.h"
+#include "flang/Optimizer/Transforms/Passes.h"
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "mlir/Transforms/RegionUtils.h"
 #include <mlir/Dialect/Arith/IR/Arith.h>
 #include <mlir/Dialect/LLVMIR/LLVMTypes.h>
 #include <mlir/Dialect/Utils/IndexingUtils.h>
@@ -33,7 +34,6 @@
 #include <mlir/IR/PatternMatch.h>
 #include <mlir/Interfaces/SideEffectInterfaces.h>
 #include <mlir/Support/LLVM.h>
-#include "mlir/Transforms/RegionUtils.h"
 #include <optional>
 #include <variant>
 
@@ -66,30 +66,30 @@ static T getPerfectlyNested(Operation *op) {
 /// This is the single source of truth about whether we should parallelize an
 /// operation nested in an omp.workdistribute region.
 static bool shouldParallelize(Operation *op) {
-    // Currently we cannot parallelize operations with results that have uses
-    if (llvm::any_of(op->getResults(),
-                     [](OpResult v) -> bool { return !v.use_empty(); }))
+  // Currently we cannot parallelize operations with results that have uses
+  if (llvm::any_of(op->getResults(),
+                   [](OpResult v) -> bool { return !v.use_empty(); }))
+    return false;
+  // We will parallelize unordered loops - these come from array syntax
+  if (auto loop = dyn_cast<fir::DoLoopOp>(op)) {
+    auto unordered = loop.getUnordered();
+    if (!unordered)
       return false;
-    // We will parallelize unordered loops - these come from array syntax
-    if (auto loop = dyn_cast<fir::DoLoopOp>(op)) {
-      auto unordered = loop.getUnordered();
-      if (!unordered)
-        return false;
-      return *unordered;
-    }
-    if (auto callOp = dyn_cast<fir::CallOp>(op)) {
-      auto callee = callOp.getCallee();
-      if (!callee)
-        return false;
-      auto *func = op->getParentOfType<ModuleOp>().lookupSymbol(*callee);
-      // TODO need to insert a check here whether it is a call we can actually
-      // parallelize currently
-      if (func->getAttr(fir::FIROpsDialect::getFirRuntimeAttrName()))
-        return true;
+    return *unordered;
+  }
+  if (auto callOp = dyn_cast<fir::CallOp>(op)) {
+    auto callee = callOp.getCallee();
+    if (!callee)
       return false;
-    }
-    // We cannot parallise anything else
+    auto *func = op->getParentOfType<ModuleOp>().lookupSymbol(*callee);
+    // TODO need to insert a check here whether it is a call we can actually
+    // parallelize currently
+    if (func->getAttr(fir::FIROpsDialect::getFirRuntimeAttrName()))
+      return true;
     return false;
+  }
+  // We cannot parallise anything else
+  return false;
 }
 
 /// If B() and D() are parallelizable,
@@ -120,12 +120,10 @@ static bool shouldParallelize(Operation *op) {
 /// }
 /// E()
 
-struct FissionWorkdistribute
-    : public OpRewritePattern<omp::WorkdistributeOp> {
+struct FissionWorkdistribute : public OpRewritePattern<omp::WorkdistributeOp> {
   using OpRewritePattern::OpRewritePattern;
-  LogicalResult
-  matchAndRewrite(omp::WorkdistributeOp workdistribute,
-                  PatternRewriter &rewriter) const override {
+  LogicalResult matchAndRewrite(omp::WorkdistributeOp workdistribute,
+                                PatternRewriter &rewriter) const override {
     auto loc = workdistribute->getLoc();
     auto teams = dyn_cast<omp::TeamsOp>(workdistribute->getParentOp());
     if (!teams) {
@@ -185,7 +183,7 @@ struct FissionWorkdistribute
         auto newWorkdistribute = rewriter.create<omp::WorkdistributeOp>(loc);
         rewriter.create<omp::TerminatorOp>(loc);
         rewriter.createBlock(&newWorkdistribute.getRegion(),
-                            newWorkdistribute.getRegion().begin(), {}, {});
+                             newWorkdistribute.getRegion().begin(), {}, {});
         auto *cloned = rewriter.clone(*parallelize);
         rewriter.replaceOp(parallelize, cloned);
         rewriter.create<omp::TerminatorOp>(loc);
@@ -197,8 +195,7 @@ struct FissionWorkdistribute
 };
 
 static void
-genLoopNestClauseOps(mlir::Location loc,
-                     mlir::PatternRewriter &rewriter,
+genLoopNestClauseOps(mlir::Location loc, mlir::PatternRewriter &rewriter,
                      fir::DoLoopOp loop,
                      mlir::omp::LoopNestOperands &loopNestClauseOps) {
   assert(loopNestClauseOps.loopLowerBounds.empty() &&
@@ -209,10 +206,8 @@ genLoopNestClauseOps(mlir::Location loc,
   loopNestClauseOps.loopInclusive = rewriter.getUnitAttr();
 }
 
-static void
-genWsLoopOp(mlir::PatternRewriter &rewriter,
-            fir::DoLoopOp doLoop,
-            const mlir::omp::LoopNestOperands &clauseOps) {
+static void genWsLoopOp(mlir::PatternRewriter &rewriter, fir::DoLoopOp doLoop,
+                        const mlir::omp::LoopNestOperands &clauseOps) {
 
   auto wsloopOp = rewriter.create<mlir::omp::WsloopOp>(doLoop.getLoc());
   rewriter.createBlock(&wsloopOp.getRegion());
@@ -236,7 +231,7 @@ genWsLoopOp(mlir::PatternRewriter &rewriter,
   return;
 }
 
-/// If fir.do_loop id present inside teams workdistribute
+/// If fir.do_loop is present inside teams workdistribute
 ///
 /// omp.teams {
 ///   omp.workdistribute {
@@ -246,7 +241,7 @@ genWsLoopOp(mlir::PatternRewriter &rewriter,
 ///   }
 /// }
 ///
-/// Then, its lowered to 
+/// Then, its lowered to
 ///
 /// omp.teams {
 ///   omp.workdistribute {
@@ -277,7 +272,8 @@ struct TeamsWorkdistributeLowering : public OpRewritePattern<omp::TeamsOp> {
 
       auto parallelOp = rewriter.create<mlir::omp::ParallelOp>(teamsLoc);
       rewriter.createBlock(&parallelOp.getRegion());
-      rewriter.setInsertionPoint(rewriter.create<mlir::omp::TerminatorOp>(doLoop.getLoc()));
+      rewriter.setInsertionPoint(
+          rewriter.create<mlir::omp::TerminatorOp>(doLoop.getLoc()));
 
       mlir::omp::LoopNestOperands loopNestClauseOps;
       genLoopNestClauseOps(doLoop.getLoc(), rewriter, doLoop,
@@ -292,7 +288,6 @@ struct TeamsWorkdistributeLowering : public OpRewritePattern<omp::TeamsOp> {
   }
 };
 
-
 /// If A() and B () are present inside teams workdistribute
 ///
 /// omp.teams {
@@ -311,17 +306,17 @@ struct TeamsWorkdistributeLowering : public OpRewritePattern<omp::TeamsOp> {
 struct TeamsWorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(omp::TeamsOp teamsOp,
-                                  PatternRewriter &rewriter) const override {
-      auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
-      if (!workdistributeOp) {
-          LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " No workdistribute nested\n");
-          return failure();
-      }      
-      Block *workdistributeBlock = &workdistributeOp.getRegion().front();
-      rewriter.eraseOp(workdistributeBlock->getTerminator());
-      rewriter.inlineBlockBefore(workdistributeBlock, teamsOp);
-      rewriter.eraseOp(teamsOp);
-      return success();
+                                PatternRewriter &rewriter) const override {
+    auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
+    if (!workdistributeOp) {
+      LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " No workdistribute nested\n");
+      return failure();
+    }
+    Block *workdistributeBlock = &workdistributeOp.getRegion().front();
+    rewriter.eraseOp(workdistributeBlock->getTerminator());
+    rewriter.inlineBlockBefore(workdistributeBlock, teamsOp);
+    rewriter.eraseOp(teamsOp);
+    return success();
   }
 };
 
@@ -332,13 +327,13 @@ class LowerWorkdistributePass
     MLIRContext &context = getContext();
     GreedyRewriteConfig config;
     // prevent the pattern driver form merging blocks
-    config.setRegionSimplificationLevel(
-        GreedySimplifyRegionLevel::Disabled);
-    
+    config.setRegionSimplificationLevel(GreedySimplifyRegionLevel::Disabled);
+
     Operation *op = getOperation();
     {
       RewritePatternSet patterns(&context);
-      patterns.insert<FissionWorkdistribute, TeamsWorkdistributeLowering>(&context);
+      patterns.insert<FissionWorkdistribute, TeamsWorkdistributeLowering>(
+          &context);
       if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
         emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
         signalPassFailure();
@@ -346,7 +341,8 @@ class LowerWorkdistributePass
     }
     {
       RewritePatternSet patterns(&context);
-      patterns.insert<TeamsWorkdistributeLowering, TeamsWorkdistributeToSingle>(&context);
+      patterns.insert<TeamsWorkdistributeLowering, TeamsWorkdistributeToSingle>(
+          &context);
       if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
         emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
         signalPassFailure();
@@ -354,4 +350,4 @@ class LowerWorkdistributePass
     }
   }
 };
-}
+} // namespace
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index cd0eccffa43f5..ad729932a5f00 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1492,12 +1492,14 @@ TYPE_PARSER(
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
-        "TARGET TEAMS WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
+        "TARGET TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
-        "TEAMS WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_teams_workdistribute),
+        "TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_teams_workdistribute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
         "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
         "WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_workdistribute))))
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
index 666bdb3ced647..9fb970246b90c 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
@@ -25,4 +25,4 @@ func.func @x(%lb : index, %ub : index, %step : index, %b : i1, %addr : !fir.ref<
     omp.terminator
   }
   return
-}
\ No newline at end of file
+}

>From 44709698f5baea865de0c197d06eabcf36bad66e Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Tue, 27 May 2025 16:24:26 +0530
Subject: [PATCH 12/29] update to workdistribute lowering

---
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 194 ++++++++++--------
 .../OpenMP/lower-workdistribute-doloop.mlir   |  19 +-
 .../OpenMP/lower-workdistribute-fission.mlir  |  31 +--
 3 files changed, 139 insertions(+), 105 deletions(-)

diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index f75d4d1988fd2..c9c7827ace217 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -48,25 +48,21 @@ using namespace mlir;
 
 namespace {
 
-template <typename T>
-static T getPerfectlyNested(Operation *op) {
-  if (op->getNumRegions() != 1)
-    return nullptr;
-  auto &region = op->getRegion(0);
-  if (region.getBlocks().size() != 1)
-    return nullptr;
-  auto *block = &region.front();
-  auto *firstOp = &block->front();
-  if (auto nested = dyn_cast<T>(firstOp))
-    if (firstOp->getNextNode() == block->getTerminator())
-      return nested;
-  return nullptr;
+static bool isRuntimeCall(Operation *op) {
+  if (auto callOp = dyn_cast<fir::CallOp>(op)) {
+    auto callee = callOp.getCallee();
+    if (!callee)
+      return false;
+    auto *func = op->getParentOfType<ModuleOp>().lookupSymbol(*callee);
+    if (func->getAttr(fir::FIROpsDialect::getFirRuntimeAttrName()))
+      return true;
+  }
+  return false;
 }
 
 /// This is the single source of truth about whether we should parallelize an
-/// operation nested in an omp.workdistribute region.
+/// operation nested in an omp.execute region.
 static bool shouldParallelize(Operation *op) {
-  // Currently we cannot parallelize operations with results that have uses
   if (llvm::any_of(op->getResults(),
                    [](OpResult v) -> bool { return !v.use_empty(); }))
     return false;
@@ -77,21 +73,28 @@ static bool shouldParallelize(Operation *op) {
       return false;
     return *unordered;
   }
-  if (auto callOp = dyn_cast<fir::CallOp>(op)) {
-    auto callee = callOp.getCallee();
-    if (!callee)
-      return false;
-    auto *func = op->getParentOfType<ModuleOp>().lookupSymbol(*callee);
-    // TODO need to insert a check here whether it is a call we can actually
-    // parallelize currently
-    if (func->getAttr(fir::FIROpsDialect::getFirRuntimeAttrName()))
-      return true;
-    return false;
+  if (isRuntimeCall(op)) {
+    return true;
   }
   // We cannot parallise anything else
   return false;
 }
 
+template <typename T>
+static T getPerfectlyNested(Operation *op) {
+  if (op->getNumRegions() != 1)
+    return nullptr;
+  auto &region = op->getRegion(0);
+  if (region.getBlocks().size() != 1)
+    return nullptr;
+  auto *block = &region.front();
+  auto *firstOp = &block->front();
+  if (auto nested = dyn_cast<T>(firstOp))
+    if (firstOp->getNextNode() == block->getTerminator())
+      return nested;
+  return nullptr;
+}
+
 /// If B() and D() are parallelizable,
 ///
 /// omp.teams {
@@ -138,17 +141,33 @@ struct FissionWorkdistribute : public OpRewritePattern<omp::WorkdistributeOp> {
       emitError(loc, "teams with multiple blocks\n");
       return failure();
     }
-    if (teams.getRegion().getBlocks().front().getOperations().size() != 2) {
-      emitError(loc, "teams with multiple nested ops\n");
-      return failure();
-    }
 
     auto *teamsBlock = &teams.getRegion().front();
+    bool changed = false;
+    // Move the ops inside teams and before workdistribute outside.
+    IRMapping irMapping;
+    llvm::SmallVector<Operation *> teamsHoisted;
+    for (auto &op : teams.getOps()) {
+      if (&op == workdistribute) {
+        break;
+      }
+      if (shouldParallelize(&op)) {
+        emitError(loc,
+                  "teams has parallelize ops before first workdistribute\n");
+        return failure();
+      } else {
+        rewriter.setInsertionPoint(teams);
+        rewriter.clone(op, irMapping);
+        teamsHoisted.push_back(&op);
+        changed = true;
+      }
+    }
+    for (auto *op : teamsHoisted)
+      rewriter.replaceOp(op, irMapping.lookup(op));
 
     // While we have unhandled operations in the original workdistribute
     auto *workdistributeBlock = &workdistribute.getRegion().front();
     auto *terminator = workdistributeBlock->getTerminator();
-    bool changed = false;
     while (&workdistributeBlock->front() != terminator) {
       rewriter.setInsertionPoint(teams);
       IRMapping mapping;
@@ -194,9 +213,51 @@ struct FissionWorkdistribute : public OpRewritePattern<omp::WorkdistributeOp> {
   }
 };
 
+/// If fir.do_loop is present inside teams workdistribute
+///
+/// omp.teams {
+///   omp.workdistribute {
+///     fir.do_loop unoredered {
+///       ...
+///     }
+///   }
+/// }
+///
+/// Then, its lowered to
+///
+/// omp.teams {
+///   omp.parallel {
+///     omp.distribute {
+///     omp.wsloop {
+///       omp.loop_nest
+///         ...
+///       }
+///     }
+///   }
+/// }
+
+static void genParallelOp(Location loc, PatternRewriter &rewriter,
+                          bool composite) {
+  auto parallelOp = rewriter.create<mlir::omp::ParallelOp>(loc);
+  parallelOp.setComposite(composite);
+  rewriter.createBlock(&parallelOp.getRegion());
+  rewriter.setInsertionPoint(rewriter.create<mlir::omp::TerminatorOp>(loc));
+  return;
+}
+
+static void genDistributeOp(Location loc, PatternRewriter &rewriter,
+                            bool composite) {
+  mlir::omp::DistributeOperands distributeClauseOps;
+  auto distributeOp =
+      rewriter.create<mlir::omp::DistributeOp>(loc, distributeClauseOps);
+  distributeOp.setComposite(composite);
+  auto distributeBlock = rewriter.createBlock(&distributeOp.getRegion());
+  rewriter.setInsertionPointToStart(distributeBlock);
+  return;
+}
+
 static void
-genLoopNestClauseOps(mlir::Location loc, mlir::PatternRewriter &rewriter,
-                     fir::DoLoopOp loop,
+genLoopNestClauseOps(mlir::PatternRewriter &rewriter, fir::DoLoopOp loop,
                      mlir::omp::LoopNestOperands &loopNestClauseOps) {
   assert(loopNestClauseOps.loopLowerBounds.empty() &&
          "Loop nest bounds were already emitted!");
@@ -207,9 +268,11 @@ genLoopNestClauseOps(mlir::Location loc, mlir::PatternRewriter &rewriter,
 }
 
 static void genWsLoopOp(mlir::PatternRewriter &rewriter, fir::DoLoopOp doLoop,
-                        const mlir::omp::LoopNestOperands &clauseOps) {
+                        const mlir::omp::LoopNestOperands &clauseOps,
+                        bool composite) {
 
   auto wsloopOp = rewriter.create<mlir::omp::WsloopOp>(doLoop.getLoc());
+  wsloopOp.setComposite(composite);
   rewriter.createBlock(&wsloopOp.getRegion());
 
   auto loopNestOp =
@@ -231,57 +294,20 @@ static void genWsLoopOp(mlir::PatternRewriter &rewriter, fir::DoLoopOp doLoop,
   return;
 }
 
-/// If fir.do_loop is present inside teams workdistribute
-///
-/// omp.teams {
-///   omp.workdistribute {
-///     fir.do_loop unoredered {
-///       ...
-///     }
-///   }
-/// }
-///
-/// Then, its lowered to
-///
-/// omp.teams {
-///   omp.workdistribute {
-///     omp.parallel {
-///       omp.wsloop {
-///         omp.loop_nest
-///           ...
-///         }
-///       }
-///     }
-///   }
-/// }
-
-struct TeamsWorkdistributeLowering : public OpRewritePattern<omp::TeamsOp> {
+struct WorkdistributeDoLower : public OpRewritePattern<omp::WorkdistributeOp> {
   using OpRewritePattern::OpRewritePattern;
-  LogicalResult matchAndRewrite(omp::TeamsOp teamsOp,
+  LogicalResult matchAndRewrite(omp::WorkdistributeOp workdistribute,
                                 PatternRewriter &rewriter) const override {
-    auto teamsLoc = teamsOp->getLoc();
-    auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
-    if (!workdistributeOp) {
-      LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " No workdistribute nested\n");
-      return failure();
-    }
-    assert(teamsOp.getReductionVars().empty());
-
-    auto doLoop = getPerfectlyNested<fir::DoLoopOp>(workdistributeOp);
+    auto doLoop = getPerfectlyNested<fir::DoLoopOp>(workdistribute);
+    auto wdLoc = workdistribute->getLoc();
     if (doLoop && shouldParallelize(doLoop)) {
-
-      auto parallelOp = rewriter.create<mlir::omp::ParallelOp>(teamsLoc);
-      rewriter.createBlock(&parallelOp.getRegion());
-      rewriter.setInsertionPoint(
-          rewriter.create<mlir::omp::TerminatorOp>(doLoop.getLoc()));
-
+      assert(doLoop.getReduceOperands().empty());
+      genParallelOp(wdLoc, rewriter, true);
+      genDistributeOp(wdLoc, rewriter, true);
       mlir::omp::LoopNestOperands loopNestClauseOps;
-      genLoopNestClauseOps(doLoop.getLoc(), rewriter, doLoop,
-                           loopNestClauseOps);
-
-      genWsLoopOp(rewriter, doLoop, loopNestClauseOps);
-      rewriter.setInsertionPoint(doLoop);
-      rewriter.eraseOp(doLoop);
+      genLoopNestClauseOps(rewriter, doLoop, loopNestClauseOps);
+      genWsLoopOp(rewriter, doLoop, loopNestClauseOps, true);
+      rewriter.eraseOp(workdistribute);
       return success();
     }
     return failure();
@@ -315,7 +341,7 @@ struct TeamsWorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
     Block *workdistributeBlock = &workdistributeOp.getRegion().front();
     rewriter.eraseOp(workdistributeBlock->getTerminator());
     rewriter.inlineBlockBefore(workdistributeBlock, teamsOp);
-    rewriter.eraseOp(teamsOp);
+    rewriter.eraseOp(workdistributeOp);
     return success();
   }
 };
@@ -332,8 +358,7 @@ class LowerWorkdistributePass
     Operation *op = getOperation();
     {
       RewritePatternSet patterns(&context);
-      patterns.insert<FissionWorkdistribute, TeamsWorkdistributeLowering>(
-          &context);
+      patterns.insert<FissionWorkdistribute, WorkdistributeDoLower>(&context);
       if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
         emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
         signalPassFailure();
@@ -341,8 +366,7 @@ class LowerWorkdistributePass
     }
     {
       RewritePatternSet patterns(&context);
-      patterns.insert<TeamsWorkdistributeLowering, TeamsWorkdistributeToSingle>(
-          &context);
+      patterns.insert<TeamsWorkdistributeToSingle>(&context);
       if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
         emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
         signalPassFailure();
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
index 9fb970246b90c..f8351bb64e6e8 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
@@ -2,13 +2,18 @@
 
 // CHECK-LABEL:   func.func @x({{.*}})
 // CHECK:           %[[VAL_0:.*]] = arith.constant 0 : index
-// CHECK:           omp.parallel {
-// CHECK:             omp.wsloop {
-// CHECK:               omp.loop_nest (%[[VAL_1:.*]]) : index = (%[[ARG0:.*]]) to (%[[ARG1:.*]]) inclusive step (%[[ARG2:.*]]) {
-// CHECK:                 fir.store %[[VAL_0]] to %[[ARG4:.*]] : !fir.ref<index>
-// CHECK:                 omp.yield
-// CHECK:               }
-// CHECK:             }
+// CHECK:           omp.teams {
+// CHECK:             omp.parallel {
+// CHECK:               omp.distribute {
+// CHECK:                 omp.wsloop {
+// CHECK:                   omp.loop_nest (%[[VAL_1:.*]]) : index = (%[[ARG0:.*]]) to (%[[ARG1:.*]]) inclusive step (%[[ARG2:.*]]) {
+// CHECK:                     fir.store %[[VAL_0]] to %[[ARG4:.*]] : !fir.ref<index>
+// CHECK:                     omp.yield
+// CHECK:                   }
+// CHECK:                 } {omp.composite}
+// CHECK:               } {omp.composite}
+// CHECK:               omp.terminator
+// CHECK:             } {omp.composite}
 // CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           return
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
index cf50d135d01ec..c562b7009664d 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-fission.mlir
@@ -1,21 +1,26 @@
 // RUN: fir-opt --lower-workdistribute %s | FileCheck %s
 
-// CHECK-LABEL:   func.func @test_fission_workdistribute({{.*}}) {
+// CHECK-LABEL:   func.func @test_fission_workdistribute(
 // CHECK:           %[[VAL_0:.*]] = arith.constant 0 : index
 // CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_2:.*]] = arith.constant 9 : index
 // CHECK:           %[[VAL_3:.*]] = arith.constant 5.000000e+00 : f32
 // CHECK:           fir.store %[[VAL_3]] to %[[ARG2:.*]] : !fir.ref<f32>
-// CHECK:           omp.parallel {
-// CHECK:             omp.wsloop {
-// CHECK:               omp.loop_nest (%[[VAL_4:.*]]) : index = (%[[VAL_0]]) to (%[[VAL_2]]) inclusive step (%[[VAL_1]]) {
-// CHECK:                 %[[VAL_5:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
-// CHECK:                 %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<f32>
-// CHECK:                 %[[VAL_7:.*]] = fir.coordinate_of %[[ARG1:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
-// CHECK:                 fir.store %[[VAL_6]] to %[[VAL_7]] : !fir.ref<f32>
-// CHECK:                 omp.yield
-// CHECK:               }
-// CHECK:             }
+// CHECK:           omp.teams {
+// CHECK:             omp.parallel {
+// CHECK:               omp.distribute {
+// CHECK:                 omp.wsloop {
+// CHECK:                   omp.loop_nest (%[[VAL_4:.*]]) : index = (%[[VAL_0]]) to (%[[VAL_2]]) inclusive step (%[[VAL_1]]) {
+// CHECK:                     %[[VAL_5:.*]] = fir.coordinate_of %[[ARG0:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:                     %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<f32>
+// CHECK:                     %[[VAL_7:.*]] = fir.coordinate_of %[[ARG1:.*]], %[[VAL_4]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:                     fir.store %[[VAL_6]] to %[[VAL_7]] : !fir.ref<f32>
+// CHECK:                     omp.yield
+// CHECK:                   }
+// CHECK:                 } {omp.composite}
+// CHECK:               } {omp.composite}
+// CHECK:               omp.terminator
+// CHECK:             } {omp.composite}
 // CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           fir.call @regular_side_effect_func(%[[ARG2:.*]]) : (!fir.ref<f32>) -> ()
@@ -24,8 +29,8 @@
 // CHECK:             %[[VAL_9:.*]] = fir.coordinate_of %[[ARG0]], %[[VAL_8]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
 // CHECK:             fir.store %[[VAL_3]] to %[[VAL_9]] : !fir.ref<f32>
 // CHECK:           }
-// CHECK:           %[[VAL_10:.*]] = fir.load %[[ARG2]] : !fir.ref<f32>
-// CHECK:           fir.store %[[VAL_10]] to %[[ARG3]] : !fir.ref<f32>
+// CHECK:           %[[VAL_10:.*]] = fir.load %[[ARG2:.*]] : !fir.ref<f32>
+// CHECK:           fir.store %[[VAL_10]] to %[[ARG3:.*]] : !fir.ref<f32>
 // CHECK:           return
 // CHECK:         }
 module {

>From 0166ceb0ac9c4b651ac2e969eb55224e83a02ec1 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 28 May 2025 21:41:25 +0530
Subject: [PATCH 13/29] Fix basic-program.fir test.

---
 flang/test/Fir/basic-program.fir | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/test/Fir/basic-program.fir b/flang/test/Fir/basic-program.fir
index 7ac8b92f48953..a611629eeb280 100644
--- a/flang/test/Fir/basic-program.fir
+++ b/flang/test/Fir/basic-program.fir
@@ -69,6 +69,7 @@ func.func @_QQmain() {
 // PASSES-NEXT:     InlineHLFIRAssign
 // PASSES-NEXT:   ConvertHLFIRtoFIR
 // PASSES-NEXT:   LowerWorkshare
+// PASSES-NEXT:   LowerWorkdistribute
 // PASSES-NEXT:   CSE
 // PASSES-NEXT:   (S) 0 num-cse'd - Number of operations CSE'd
 // PASSES-NEXT:   (S) 0 num-dce'd - Number of operations DCE'd

>From 7f8b3a29be467d998aa0591c77a75c68659f63db Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 30 May 2025 15:12:46 +0530
Subject: [PATCH 14/29] Wrap omp.target with omp.target_data

---
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 88 +++++++++++++++++++
 .../OpenMP/lower-workdistribute-target.mlir   | 36 ++++++++
 .../lower-workdistribute-to-single.mlir       | 52 -----------
 3 files changed, 124 insertions(+), 52 deletions(-)
 create mode 100644 flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
 delete mode 100644 flang/test/Transforms/OpenMP/lower-workdistribute-to-single.mlir

diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index c9c7827ace217..6509cc5014dd7 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -346,6 +346,85 @@ struct TeamsWorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
   }
 };
 
+static std::optional<std::tuple<Operation *, bool, bool>>
+getNestedOpToIsolate(omp::TargetOp targetOp) {
+  auto *targetBlock = &targetOp.getRegion().front();
+  for (auto &op : *targetBlock) {
+    bool first = &op == &*targetBlock->begin();
+    bool last = op.getNextNode() == targetBlock->getTerminator();
+    if (first && last)
+      return std::nullopt;
+
+    if (isa<omp::TeamsOp, omp::ParallelOp>(&op))
+      return {{&op, first, last}};
+  }
+  return std::nullopt;
+}
+
+struct SplitTargetResult {
+  omp::TargetOp targetOp;
+  omp::TargetDataOp dataOp;
+};
+
+/// If multiple coexecutes are nested in a target regions, we will need to split
+/// the target region, but we want to preserve the data semantics of the
+/// original data region and avoid unnecessary data movement at each of the
+/// subkernels - we split the target region into a target_data{target}
+/// nest where only the outer one moves the data
+std::optional<SplitTargetResult> splitTargetData(omp::TargetOp targetOp,
+                                                 RewriterBase &rewriter) {
+
+  auto loc = targetOp->getLoc();
+  if (targetOp.getMapVars().empty()) {
+    LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " target region has no data maps\n");
+    return std::nullopt;
+  }
+
+  // Collect all map_entries with capture(ByRef)
+  SmallVector<mlir::Value> byRefMapInfos;
+  SmallVector<omp::MapInfoOp> MapInfos;
+  for (auto opr : targetOp.getMapVars()) {
+    auto mapInfo = cast<omp::MapInfoOp>(opr.getDefiningOp());
+    MapInfos.push_back(mapInfo);
+    if (mapInfo.getMapCaptureType() == omp::VariableCaptureKind::ByRef)
+      byRefMapInfos.push_back(opr);
+  }
+
+  // Create the new omp.target_data op with these collected map_entries
+  auto targetLoc = targetOp.getLoc();
+  rewriter.setInsertionPoint(targetOp);
+  auto device = targetOp.getDevice();
+  auto ifExpr = targetOp.getIfExpr();
+  auto deviceAddrVars = targetOp.getHasDeviceAddrVars();
+  auto devicePtrVars = targetOp.getIsDevicePtrVars();
+  auto targetDataOp = rewriter.create<omp::TargetDataOp>(loc, device, ifExpr, 
+                                                          mlir::ValueRange{byRefMapInfos},
+                                                          deviceAddrVars,
+                                                          devicePtrVars);
+
+  auto taregtDataBlock = rewriter.createBlock(&targetDataOp.getRegion());
+  rewriter.create<mlir::omp::TerminatorOp>(loc);
+  rewriter.setInsertionPointToStart(taregtDataBlock);
+
+  // Clone mapInfo ops inside omp.target_data region
+  IRMapping mapping;
+  for (auto mapInfo : MapInfos) {
+    rewriter.clone(*mapInfo, mapping);
+  }
+  // Clone omp.target from exisiting targetOp inside target_data region.
+  auto newTargetOp = rewriter.clone(*targetOp, mapping);
+
+  // Erase TargetOp and its MapInfoOps
+  rewriter.eraseOp(targetOp);
+  
+  for (auto mapInfo : MapInfos) {
+    auto mapInfoRes = mapInfo.getResult();
+    if (mapInfoRes.getUsers().empty()) 
+      rewriter.eraseOp(mapInfo);
+  }
+  return SplitTargetResult{targetOp, targetDataOp};
+}                                                  
+
 class LowerWorkdistributePass
     : public flangomp::impl::LowerWorkdistributeBase<LowerWorkdistributePass> {
 public:
@@ -372,6 +451,15 @@ class LowerWorkdistributePass
         signalPassFailure();
       }
     }
+    {
+      SmallVector<omp::TargetOp> targetOps;
+      op->walk([&](omp::TargetOp targetOp) { targetOps.push_back(targetOp); });
+      IRRewriter rewriter(&context);
+      for (auto targetOp : targetOps) {
+        auto res = splitTargetData(targetOp, rewriter);
+      }
+    }
+
   }
 };
 } // namespace
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
new file mode 100644
index 0000000000000..e6ca98d3bf596
--- /dev/null
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
@@ -0,0 +1,36 @@
+// RUN: fir-opt --lower-workdistribute %s | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_nested_derived_type_map_operand_and_block_addition(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
+// CHECK:           %[[VAL_0:.*]] = fir.declare %[[ARG0]] {uniq_name = "_QFmaptype_derived_nested_explicit_multiple_membersEsa"} : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>
+// CHECK:           %[[VAL_1:.*]] = fir.coordinate_of %[[VAL_0]], n : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>
+// CHECK:           %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_1]], i : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_3:.*]] = omp.map.info var_ptr(%[[VAL_2]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "sa%[[VAL_4:.*]]%[[VAL_5:.*]]"}
+// CHECK:           %[[VAL_6:.*]] = fir.coordinate_of %[[VAL_0]], n : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>
+// CHECK:           %[[VAL_7:.*]] = fir.coordinate_of %[[VAL_6]], r : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>) -> !fir.ref<f32>
+// CHECK:           %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_7]] : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "sa%[[VAL_4]]%[[VAL_9:.*]]"}
+// CHECK:           %[[VAL_10:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
+// CHECK:           omp.target_data map_entries(%[[VAL_10]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
+// CHECK:             %[[VAL_11:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
+// CHECK:             omp.target map_entries(%[[VAL_11]] -> %[[VAL_12:.*]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
+// CHECK:               omp.terminator
+// CHECK:             }
+// CHECK:             omp.terminator
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+func.func @test_nested_derived_type_map_operand_and_block_addition(%arg0: !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {        
+  %0 = fir.declare %arg0 {uniq_name = "_QFmaptype_derived_nested_explicit_multiple_membersEsa"} : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>
+  %2 = fir.coordinate_of %0, n : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>
+  %4 = fir.coordinate_of %2, i : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>) -> !fir.ref<i32>
+  %5 = omp.map.info var_ptr(%4 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "sa%n%i"}
+  %7 = fir.coordinate_of %0, n : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>
+  %9 = fir.coordinate_of %7, r : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>) -> !fir.ref<f32>
+  %10 = omp.map.info var_ptr(%9 : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "sa%n%r"}
+  %11 = omp.map.info var_ptr(%0 : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%5, %10 : [1,0], [1,1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
+  omp.target map_entries(%11 -> %arg1 : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
+    omp.terminator
+  }
+  return
+}
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-to-single.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-to-single.mlir
deleted file mode 100644
index 0cc2aeded2532..0000000000000
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-to-single.mlir
+++ /dev/null
@@ -1,52 +0,0 @@
-// RUN: fir-opt --lower-workdistribute %s | FileCheck %s
-
-// CHECK-LABEL:   func.func @_QPtarget_simple() {
-// CHECK:           %[[VAL_0:.*]] = arith.constant 2 : i32
-// CHECK:           %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFtarget_simpleEa"}
-// CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "simple_var", uniq_name = "_QFtarget_simpleEsimple_var"}
-// CHECK:           %[[VAL_4:.*]] = fir.zero_bits !fir.heap<i32>
-// CHECK:           %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
-// CHECK:           fir.store %[[VAL_5]] to %[[VAL_3]] : !fir.ref<!fir.box<!fir.heap<i32>>>
-// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
-// CHECK:           hlfir.assign %[[VAL_0]] to %[[VAL_2]]#0 : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_7:.*]] = omp.map.info var_ptr(%[[VAL_2]]#1 : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32> {name = "a"}
-// CHECK:           omp.target map_entries(%[[VAL_7]] -> %[[VAL_8:.*]] : !fir.ref<i32>) private(@_QFtarget_simpleEsimple_var_private_ref_box_heap_i32 %[[VAL_6]]#0 -> %[[VAL_9:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>) {
-// CHECK:             %[[VAL_10:.*]] = arith.constant 10 : i32
-// CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_9]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
-// CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i32>
-// CHECK:             %[[VAL_14:.*]] = arith.addi %[[VAL_13]], %[[VAL_10]] : i32
-// CHECK:             hlfir.assign %[[VAL_14]] to %[[VAL_12]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
-// CHECK:             omp.terminator
-// CHECK:           }
-// CHECK:           return
-// CHECK:         }
-func.func @_QPtarget_simple() {
-    %0 = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFtarget_simpleEa"}
-    %1:2 = hlfir.declare %0 {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-    %2 = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "simple_var", uniq_name = "_QFtarget_simpleEsimple_var"}
-    %3 = fir.zero_bits !fir.heap<i32>
-    %4 = fir.embox %3 : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
-    fir.store %4 to %2 : !fir.ref<!fir.box<!fir.heap<i32>>>
-    %5:2 = hlfir.declare %2 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
-    %c2_i32 = arith.constant 2 : i32
-    hlfir.assign %c2_i32 to %1#0 : i32, !fir.ref<i32>
-    %6 = omp.map.info var_ptr(%1#1 : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32> {name = "a"}
-    omp.target map_entries(%6 -> %arg0 : !fir.ref<i32>) private(@_QFtarget_simpleEsimple_var_private_ref_box_heap_i32 %5#0 -> %arg1 : !fir.ref<!fir.box<!fir.heap<i32>>>){
-        omp.teams {
-            omp.workdistribute {
-                %11:2 = hlfir.declare %arg0 {uniq_name = "_QFtarget_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-                %12:2 = hlfir.declare %arg1 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtarget_simpleEsimple_var"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
-                %c10_i32 = arith.constant 10 : i32
-                %13 = fir.load %11#0 : !fir.ref<i32>
-                %14 = arith.addi %c10_i32, %13 : i32
-                hlfir.assign %14 to %12#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
-                omp.terminator
-            }
-            omp.terminator
-        }
-        omp.terminator
-    }
-    return
-}

>From c36df1081b6eb222d941a54e8b958bcae144e352 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Tue, 3 Jun 2025 15:47:08 +0530
Subject: [PATCH 15/29] Add fission of target region Logic inspired from
 ivanradanov llvm branch: flang_workdistribute_iwomp_2024 commit:
 a77451505dbd728a7a339f6c7c4c1382c709c502

---
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 437 +++++++++++++++++-
 .../lower-workdistribute-fission-target.mlir  | 104 +++++
 2 files changed, 521 insertions(+), 20 deletions(-)
 create mode 100644 flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir

diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index 6509cc5014dd7..8f2de92cfd186 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -34,6 +34,7 @@
 #include <mlir/IR/PatternMatch.h>
 #include <mlir/Interfaces/SideEffectInterfaces.h>
 #include <mlir/Support/LLVM.h>
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include <optional>
 #include <variant>
 
@@ -346,21 +347,6 @@ struct TeamsWorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
   }
 };
 
-static std::optional<std::tuple<Operation *, bool, bool>>
-getNestedOpToIsolate(omp::TargetOp targetOp) {
-  auto *targetBlock = &targetOp.getRegion().front();
-  for (auto &op : *targetBlock) {
-    bool first = &op == &*targetBlock->begin();
-    bool last = op.getNextNode() == targetBlock->getTerminator();
-    if (first && last)
-      return std::nullopt;
-
-    if (isa<omp::TeamsOp, omp::ParallelOp>(&op))
-      return {{&op, first, last}};
-  }
-  return std::nullopt;
-}
-
 struct SplitTargetResult {
   omp::TargetOp targetOp;
   omp::TargetDataOp dataOp;
@@ -371,8 +357,7 @@ struct SplitTargetResult {
 /// original data region and avoid unnecessary data movement at each of the
 /// subkernels - we split the target region into a target_data{target}
 /// nest where only the outer one moves the data
-std::optional<SplitTargetResult> splitTargetData(omp::TargetOp targetOp,
-                                                 RewriterBase &rewriter) {
+std::optional<SplitTargetResult> splitTargetData(omp::TargetOp targetOp, RewriterBase &rewriter) {
 
   auto loc = targetOp->getLoc();
   if (targetOp.getMapVars().empty()) {
@@ -391,7 +376,6 @@ std::optional<SplitTargetResult> splitTargetData(omp::TargetOp targetOp,
   }
 
   // Create the new omp.target_data op with these collected map_entries
-  auto targetLoc = targetOp.getLoc();
   rewriter.setInsertionPoint(targetOp);
   auto device = targetOp.getDevice();
   auto ifExpr = targetOp.getIfExpr();
@@ -422,8 +406,420 @@ std::optional<SplitTargetResult> splitTargetData(omp::TargetOp targetOp,
     if (mapInfoRes.getUsers().empty()) 
       rewriter.eraseOp(mapInfo);
   }
-  return SplitTargetResult{targetOp, targetDataOp};
-}                                                  
+  return SplitTargetResult{cast<omp::TargetOp>(newTargetOp), targetDataOp};
+}
+
+static std::optional<std::tuple<Operation *, bool, bool>>
+getNestedOpToIsolate(omp::TargetOp targetOp) {
+  if (targetOp.getRegion().empty())
+    return std::nullopt;
+  auto *targetBlock = &targetOp.getRegion().front();
+  for (auto &op : *targetBlock) {
+    bool first = &op == &*targetBlock->begin();
+    bool last = op.getNextNode() == targetBlock->getTerminator();
+    if (first && last)
+      return std::nullopt;
+
+    if (isa<omp::TeamsOp, omp::ParallelOp>(&op))
+      return {{&op, first, last}};
+  }
+  return std::nullopt;
+}
+
+struct TempOmpVar {
+  omp::MapInfoOp from, to;
+};
+
+static bool isPtr(Type ty) {
+  return isa<fir::ReferenceType>(ty) || isa<LLVM::LLVMPointerType>(ty);
+}
+
+static Type getPtrTypeForOmp(Type ty) {
+  if (isPtr(ty))
+    return LLVM::LLVMPointerType::get(ty.getContext());
+  else
+    return fir::LLVMPointerType::get(ty);
+}
+
+static TempOmpVar 
+allocateTempOmpVar(Location loc, Type ty, RewriterBase &rewriter) {
+  MLIRContext& ctx = *ty.getContext();
+  Value alloc;
+  Type allocType;
+  auto llvmPtrTy = LLVM::LLVMPointerType::get(&ctx);
+  if (isPtr(ty)) {
+    Type intTy = rewriter.getI32Type();
+    auto one = rewriter.create<LLVM::ConstantOp>(loc, intTy, 1);
+    allocType = llvmPtrTy;
+    alloc = rewriter.create<LLVM::AllocaOp>(loc, llvmPtrTy, allocType, one);
+    allocType = intTy;
+  }
+  else {
+    allocType = ty;
+    alloc = rewriter.create<fir::AllocaOp>(loc, allocType);
+  }
+  auto getMapInfo = [&](uint64_t mappingFlags, const char *name) {
+    return rewriter.create<omp::MapInfoOp>(
+      loc, alloc.getType(), alloc,
+      TypeAttr::get(allocType),
+      rewriter.getIntegerAttr(rewriter.getIntegerType(64, /*isSigned=*/false), mappingFlags),
+      rewriter.getAttr<omp::VariableCaptureKindAttr>(
+          omp::VariableCaptureKind::ByRef),
+      /*varPtrPtr=*/Value{},
+      /*members=*/SmallVector<Value>{},
+      /*member_index=*/mlir::ArrayAttr{},
+      /*bounds=*/ValueRange(),
+      /*mapperId=*/mlir::FlatSymbolRefAttr(), 
+      /*name=*/rewriter.getStringAttr(name),
+      rewriter.getBoolAttr(false));
+  };
+  uint64_t mapFrom = static_cast<std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM);
+  uint64_t mapTo = static_cast<std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO);
+  auto mapInfoFrom = getMapInfo(mapFrom, "__flang_workdistribute_from");
+  auto mapInfoTo = getMapInfo(mapTo, "__flang_workdistribute_to");
+  return TempOmpVar{mapInfoFrom, mapInfoTo};
+};
+
+static bool usedOutsideSplit(Value v, Operation *split) {
+  if (!split)
+    return false;
+  auto targetOp = cast<omp::TargetOp>(split->getParentOp());
+  auto *targetBlock = &targetOp.getRegion().front();
+  for (auto *user : v.getUsers()) {
+    while (user->getBlock() != targetBlock) {
+      user = user->getParentOp();
+    }
+    if (!user->isBeforeInBlock(split))
+      return true;
+  }
+  return false;
+};
+
+static bool isOpToBeCached(Operation *op) {
+  if (auto loadOp = dyn_cast<fir::LoadOp>(op)) {  
+    Value memref = loadOp.getMemref();  
+    if (auto blockArg = dyn_cast<BlockArgument>(memref)) {  
+      // 'op' is an operation within the targetOp that 'splitBefore' is also in.
+      Operation *parentOpOfLoadBlock = op->getBlock()->getParentOp();  
+      // Ensure the blockArg belongs to the entry block of this parent omp.TargetOp.  
+      // This implies the load is from a variable directly mapped into the target region.  
+      if (isa<omp::TargetOp>(parentOpOfLoadBlock) &&  
+          !parentOpOfLoadBlock->getRegions().empty()) {  
+        Block *targetOpEntryBlock = &parentOpOfLoadBlock->getRegions().front().front();  
+        if (blockArg.getOwner() == targetOpEntryBlock) {  
+          // This load is from a direct argument of the target op.  
+          // It's safe to recompute.
+          return false;  
+        }  
+      }  
+    }  
+  }
+  return true;
+}
+
+static bool isRecomputableAfterFission(Operation *op, Operation *splitBefore) {
+  if (isa<fir::DeclareOp>(op))
+    return true;
+
+  if (auto loadOp = dyn_cast<fir::LoadOp>(op)) {  
+    Value memref = loadOp.getMemref();  
+    if (auto blockArg = dyn_cast<BlockArgument>(memref)) {  
+      // 'op' is an operation within the targetOp that 'splitBefore' is also in.
+      Operation *parentOpOfLoadBlock = op->getBlock()->getParentOp();  
+      // Ensure the blockArg belongs to the entry block of this parent omp.TargetOp.  
+      // This implies the load is from a variable directly mapped into the target region.  
+      if (isa<omp::TargetOp>(parentOpOfLoadBlock) &&  
+          !parentOpOfLoadBlock->getRegions().empty()) {  
+        Block *targetOpEntryBlock = &parentOpOfLoadBlock->getRegions().front().front();  
+        if (blockArg.getOwner() == targetOpEntryBlock) {  
+          // This load is from a direct argument of the target op.  
+          // It's safe to recompute.
+          return true;  
+        }  
+      }  
+    }  
+  } 
+
+  llvm::SmallVector<MemoryEffects::EffectInstance> effects;
+  MemoryEffectOpInterface interface = dyn_cast<MemoryEffectOpInterface>(op);
+  if (!interface) {
+    return false;
+  }
+  interface.getEffects(effects);
+  if (effects.empty())
+    return true;
+  return false;
+}
+
+struct SplitResult {
+  omp::TargetOp preTargetOp;
+  omp::TargetOp isolatedTargetOp;
+  omp::TargetOp postTargetOp;
+};
+
+static void collectNonRecomputableDeps(Value& v,
+                                omp::TargetOp targetOp,
+                                SetVector<Operation *>& nonRecomputable,
+                                SetVector<Operation *>& toCache,
+                                SetVector<Operation *>& toRecompute) {
+  Operation *op = v.getDefiningOp();
+  if (!op) {
+    assert(cast<BlockArgument>(v).getOwner()->getParentOp() == targetOp);
+    return;
+  }
+  if (nonRecomputable.contains(op)) {
+    toCache.insert(op);
+    return;
+  }
+  toRecompute.insert(op);
+  for (auto opr : op->getOperands())
+    collectNonRecomputableDeps(opr, targetOp, nonRecomputable, toCache, toRecompute);
+}
+
+
+static void reloadCacheAndRecompute(Location loc, RewriterBase &rewriter,
+                        MLIRContext& ctx,
+                        IRMapping &mapping, Operation *splitBefore,
+                        Block *targetBlock, Block *newTargetBlock,
+                        SmallVector<Value>& allocs,
+                        SetVector<Operation *>& toRecompute) {
+  for (unsigned i = 0; i < targetBlock->getNumArguments(); i++) {
+    auto originalArg = targetBlock->getArgument(i);
+    auto newArg = newTargetBlock->addArgument(originalArg.getType(),
+                                              originalArg.getLoc());
+    mapping.map(originalArg, newArg);
+  }
+  auto llvmPtrTy = LLVM::LLVMPointerType::get(&ctx);
+  for (auto original : allocs) {
+    Value newArg = newTargetBlock->addArgument(
+      getPtrTypeForOmp(original.getType()), original.getLoc());
+    Value restored;
+    if (isPtr(original.getType())) {
+      restored = rewriter.create<LLVM::LoadOp>(loc, llvmPtrTy, newArg);
+      if (!isa<LLVM::LLVMPointerType>(original.getType()))
+        restored = rewriter.create<UnrealizedConversionCastOp>(loc, original.getType(), ValueRange(restored))
+                           .getResult(0);
+    } 
+    else {
+        restored = rewriter.create<fir::LoadOp>(loc, newArg);
+    }
+    mapping.map(original, restored);
+  }
+  for (auto it = targetBlock->begin(); it != splitBefore->getIterator(); it++) {
+    if (toRecompute.contains(&*it))
+      rewriter.clone(*it, mapping);
+  }
+}
+
+static SplitResult isolateOp(Operation *splitBeforeOp, bool splitAfter,
+                              RewriterBase &rewriter) {
+  auto targetOp = cast<omp::TargetOp>(splitBeforeOp->getParentOp());
+  MLIRContext& ctx = *targetOp.getContext();
+  assert(targetOp);
+  auto loc = targetOp.getLoc();
+  auto *targetBlock = &targetOp.getRegion().front();
+  rewriter.setInsertionPoint(targetOp);
+   
+  auto preMapOperands = SmallVector<Value>(targetOp.getMapVars());
+  auto postMapOperands = SmallVector<Value>(targetOp.getMapVars());
+
+  SmallVector<Value> requiredVals;
+  SetVector<Operation *> toCache;
+  SetVector<Operation *> toRecompute;
+  SetVector<Operation *> nonRecomputable;
+  SmallVector<Value> allocs;
+
+  for (auto it = targetBlock->begin(); it != splitBeforeOp->getIterator(); it++) {
+    for (auto res : it->getResults()) {
+      if (usedOutsideSplit(res, splitBeforeOp))
+        requiredVals.push_back(res);
+    }
+    if (!isRecomputableAfterFission(&*it, splitBeforeOp))
+        nonRecomputable.insert(&*it);
+  }
+
+  for (auto requiredVal : requiredVals)
+    collectNonRecomputableDeps(requiredVal, targetOp, nonRecomputable, toCache, toRecompute);
+  
+  for (Operation *op : toCache) {
+    for (auto res : op->getResults()) {
+      auto alloc = allocateTempOmpVar(targetOp.getLoc(), res.getType(), rewriter);
+      allocs.push_back(res);
+      preMapOperands.push_back(alloc.from);
+      postMapOperands.push_back(alloc.to);
+    }
+  }
+
+  rewriter.setInsertionPoint(targetOp);
+
+  auto preTargetOp = rewriter.create<omp::TargetOp>(
+        targetOp.getLoc(), targetOp.getAllocateVars(), targetOp.getAllocatorVars(),
+        targetOp.getBareAttr(), targetOp.getDependKindsAttr(),
+        targetOp.getDependVars(), targetOp.getDevice(),
+        targetOp.getHasDeviceAddrVars(), targetOp.getHostEvalVars(),
+        targetOp.getIfExpr(), targetOp.getInReductionVars(),
+        targetOp.getInReductionByrefAttr(), targetOp.getInReductionSymsAttr(),
+        targetOp.getIsDevicePtrVars(), preMapOperands,
+        targetOp.getNowaitAttr(), targetOp.getPrivateVars(),
+        targetOp.getPrivateSymsAttr(), targetOp.getThreadLimit(),
+        targetOp.getPrivateMapsAttr()); 
+  auto *preTargetBlock = rewriter.createBlock(
+      &preTargetOp.getRegion(), preTargetOp.getRegion().begin(), {}, {});
+  IRMapping preMapping;
+  for (unsigned i = 0; i < targetBlock->getNumArguments(); i++) {
+    auto originalArg = targetBlock->getArgument(i);
+    auto newArg = preTargetBlock->addArgument(originalArg.getType(),
+                                              originalArg.getLoc());
+    preMapping.map(originalArg, newArg);
+  }
+  for (auto it = targetBlock->begin(); it != splitBeforeOp->getIterator(); it++)
+    rewriter.clone(*it, preMapping);
+
+  auto llvmPtrTy = LLVM::LLVMPointerType::get(targetOp.getContext());
+
+
+  for (auto original : allocs) {
+    Value toStore = preMapping.lookup(original);
+    auto newArg = preTargetBlock->addArgument(
+        getPtrTypeForOmp(original.getType()), original.getLoc());
+    if (isPtr(original.getType())) {
+      if (!isa<LLVM::LLVMPointerType>(toStore.getType()))
+        toStore = rewriter.create<UnrealizedConversionCastOp>(loc, llvmPtrTy,
+                                                           ValueRange(toStore))
+                      .getResult(0);
+      rewriter.create<LLVM::StoreOp>(loc, toStore, newArg);
+    } else {
+      rewriter.create<fir::StoreOp>(loc, toStore, newArg);
+    }
+  }
+  rewriter.create<omp::TerminatorOp>(loc);
+
+  rewriter.setInsertionPoint(targetOp);
+
+  auto isolatedTargetOp = rewriter.create<omp::TargetOp>(
+      targetOp.getLoc(), targetOp.getAllocateVars(), targetOp.getAllocatorVars(),
+      targetOp.getBareAttr(), targetOp.getDependKindsAttr(),
+      targetOp.getDependVars(), targetOp.getDevice(),
+      targetOp.getHasDeviceAddrVars(), targetOp.getHostEvalVars(),
+      targetOp.getIfExpr(), targetOp.getInReductionVars(),
+      targetOp.getInReductionByrefAttr(), targetOp.getInReductionSymsAttr(),
+      targetOp.getIsDevicePtrVars(), postMapOperands,
+      targetOp.getNowaitAttr(), targetOp.getPrivateVars(),
+      targetOp.getPrivateSymsAttr(), targetOp.getThreadLimit(),
+      targetOp.getPrivateMapsAttr()); 
+
+  auto *isolatedTargetBlock =
+        rewriter.createBlock(&isolatedTargetOp.getRegion(),
+                             isolatedTargetOp.getRegion().begin(), {}, {});
+
+  IRMapping isolatedMapping;
+  reloadCacheAndRecompute(loc, rewriter, ctx, isolatedMapping, splitBeforeOp,
+                          targetBlock, isolatedTargetBlock,
+                          allocs, toRecompute);
+  rewriter.clone(*splitBeforeOp, isolatedMapping);
+  rewriter.create<omp::TerminatorOp>(loc);
+
+  omp::TargetOp postTargetOp = nullptr;
+  
+  if (splitAfter) {
+      rewriter.setInsertionPoint(targetOp);
+    postTargetOp = rewriter.create<omp::TargetOp>(
+        targetOp.getLoc(), targetOp.getAllocateVars(), targetOp.getAllocatorVars(),
+        targetOp.getBareAttr(), targetOp.getDependKindsAttr(),
+        targetOp.getDependVars(), targetOp.getDevice(),
+        targetOp.getHasDeviceAddrVars(), targetOp.getHostEvalVars(),
+        targetOp.getIfExpr(), targetOp.getInReductionVars(),
+        targetOp.getInReductionByrefAttr(), targetOp.getInReductionSymsAttr(),
+        targetOp.getIsDevicePtrVars(), postMapOperands,
+        targetOp.getNowaitAttr(), targetOp.getPrivateVars(),
+        targetOp.getPrivateSymsAttr(), targetOp.getThreadLimit(),
+        targetOp.getPrivateMapsAttr()); 
+    auto *postTargetBlock = rewriter.createBlock(
+          &postTargetOp.getRegion(), postTargetOp.getRegion().begin(), {}, {});
+    IRMapping postMapping;
+    reloadCacheAndRecompute(loc, rewriter, ctx, postMapping, splitBeforeOp, 
+                            targetBlock, postTargetBlock,
+                            allocs, toRecompute);
+
+    assert(splitBeforeOp->getNumResults() == 0 ||
+             llvm::all_of(splitBeforeOp->getResults(),
+                          [](Value result) { return result.use_empty(); }));
+
+    for (auto it = std::next(splitBeforeOp->getIterator());
+         it != targetBlock->end(); it++)
+      rewriter.clone(*it, postMapping);
+  }
+
+  rewriter.eraseOp(targetOp);
+  return SplitResult{preTargetOp, isolatedTargetOp, postTargetOp};
+}
+
+static void moveToHost(omp::TargetOp targetOp, RewriterBase &rewriter) {
+  OpBuilder::InsertionGuard guard(rewriter);
+  Block *targetBlock = &targetOp.getRegion().front();
+  assert(targetBlock == &targetOp.getRegion().back());
+  IRMapping mapping;
+  for (auto map :
+       zip_equal(targetOp.getMapVars(), targetBlock->getArguments())) {
+    Value mapInfo = std::get<0>(map);
+    BlockArgument arg = std::get<1>(map);
+    Operation *op = mapInfo.getDefiningOp();
+    assert(op);
+    auto mapInfoOp = cast<omp::MapInfoOp>(op);
+    mapping.map(arg, mapInfoOp.getVarPtr());
+  }
+  rewriter.setInsertionPoint(targetOp);
+  SmallVector<Operation *> opsToMove;
+  for (auto it = targetBlock->begin(), end = std::prev(targetBlock->end());
+       it != end; ++it) {
+    auto *op = &*it;
+    auto allocOp = dyn_cast<fir::AllocMemOp>(op);
+    auto freeOp = dyn_cast<fir::FreeMemOp>(op);
+    fir::CallOp runtimeCall = nullptr;
+    if (isRuntimeCall(op))
+      runtimeCall = cast<fir::CallOp>(op);
+
+    if (allocOp || freeOp || runtimeCall)
+        continue;
+    opsToMove.push_back(op);
+  }
+  // Move ops before targetOp and erase from region
+  for (Operation *op : opsToMove)
+    rewriter.clone(*op, mapping);
+  
+  rewriter.eraseOp(targetOp);
+}
+
+void fissionTarget(omp::TargetOp targetOp, RewriterBase &rewriter) {
+  auto tuple = getNestedOpToIsolate(targetOp);
+  if (!tuple) {
+    LLVM_DEBUG(llvm::dbgs() << " No op to isolate\n");
+    //moveToHost(targetOp, rewriter);
+    return;
+  }
+
+  Operation *toIsolate = std::get<0>(*tuple);
+  bool splitBefore = !std::get<1>(*tuple);
+  bool splitAfter = !std::get<2>(*tuple);
+
+  if (splitBefore && splitAfter) {
+    auto res = isolateOp(toIsolate, splitAfter, rewriter);
+    //moveToHost(res.preTargetOp, rewriter);
+    fissionTarget(res.postTargetOp, rewriter);
+    return;
+  }
+  if (splitBefore) {
+    auto res = isolateOp(toIsolate, splitAfter, rewriter);
+    //moveToHost(res.preTargetOp, rewriter);
+    return;
+  }
+  if (splitAfter) {
+    assert(false && "TODO");
+    auto res = isolateOp(toIsolate->getNextNode(), splitAfter, rewriter);
+    fissionTarget(res.postTargetOp, rewriter);
+    return;
+  }
+}
 
 class LowerWorkdistributePass
     : public flangomp::impl::LowerWorkdistributeBase<LowerWorkdistributePass> {
@@ -457,6 +853,7 @@ class LowerWorkdistributePass
       IRRewriter rewriter(&context);
       for (auto targetOp : targetOps) {
         auto res = splitTargetData(targetOp, rewriter);
+        if (res) fissionTarget(res->targetOp, rewriter);
       }
     }
 
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir
new file mode 100644
index 0000000000000..ed6c641f2e934
--- /dev/null
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir
@@ -0,0 +1,104 @@
+// RUN: fir-opt --lower-workdistribute %s | FileCheck %s
+
+// CHECK-LABEL:   func.func @x
+// CHECK:           %[[VAL_0:.*]] = fir.alloca index {bindc_name = "lb"}
+// CHECK:           fir.store %[[ARG0:.*]] to %[[VAL_0]] : !fir.ref<index>
+// CHECK:           %[[VAL_1:.*]] = fir.alloca index {bindc_name = "ub"}
+// CHECK:           fir.store %[[ARG1:.*]] to %[[VAL_1]] : !fir.ref<index>
+// CHECK:           %[[VAL_2:.*]] = fir.alloca index {bindc_name = "step"}
+// CHECK:           fir.store %[[ARG2:.*]] to %[[VAL_2]] : !fir.ref<index>
+// CHECK:           %[[VAL_3:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "lb"}
+// CHECK:           %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "ub"}
+// CHECK:           %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_2]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "step"}
+// CHECK:           %[[VAL_6:.*]] = omp.map.info var_ptr(%[[ARG3:.*]] : !fir.ref<index>, index) map_clauses(tofrom) capture(ByRef) -> !fir.ref<index> {name = "addr"}
+// CHECK:           omp.target_data map_entries(%[[VAL_3]], %[[VAL_4]], %[[VAL_5]], %[[VAL_6]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>) {
+// CHECK:             %[[VAL_7:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "lb"}
+// CHECK:             %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "ub"}
+// CHECK:             %[[VAL_9:.*]] = omp.map.info var_ptr(%[[VAL_2]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "step"}
+// CHECK:             %[[VAL_10:.*]] = omp.map.info var_ptr(%[[ARG3:.*]] : !fir.ref<index>, index) map_clauses(tofrom) capture(ByRef) -> !fir.ref<index> {name = "addr"}
+// CHECK:             %[[VAL_11:.*]] = fir.alloca !fir.heap<index>
+// CHECK:             %[[VAL_12:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.heap<index>>, !fir.heap<index>) map_clauses(from) capture(ByRef) -> !fir.ref<!fir.heap<index>> {name = "__flang_workdistribute_from"}
+// CHECK:             %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.heap<index>>, !fir.heap<index>) map_clauses(to) capture(ByRef) -> !fir.ref<!fir.heap<index>> {name = "__flang_workdistribute_to"}
+// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_14:.*]], %[[VAL_8]] -> %[[VAL_15:.*]], %[[VAL_9]] -> %[[VAL_16:.*]], %[[VAL_10]] -> %[[VAL_17:.*]], %[[VAL_12]] -> %[[VAL_18:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
+// CHECK:               %[[VAL_19:.*]] = arith.constant 1 : index
+// CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_14]] : !fir.ref<index>
+// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_15]] : !fir.ref<index>
+// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_16]] : !fir.ref<index>
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_21]], %[[VAL_21]] : index
+// CHECK:               %[[VAL_24:.*]] = fir.allocmem index, %[[VAL_19]] {uniq_name = "dev_buf"}
+// CHECK:               fir.store %[[VAL_24]] to %[[VAL_18]] : !fir.llvm_ptr<!fir.heap<index>>
+// CHECK:               omp.terminator
+// CHECK:             }
+// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_25:.*]], %[[VAL_8]] -> %[[VAL_26:.*]], %[[VAL_9]] -> %[[VAL_27:.*]], %[[VAL_10]] -> %[[VAL_28:.*]], %[[VAL_13]] -> %[[VAL_29:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
+// CHECK:               %[[VAL_30:.*]] = fir.load %[[VAL_29]] : !fir.llvm_ptr<!fir.heap<index>>
+// CHECK:               %[[VAL_31:.*]] = fir.load %[[VAL_25]] : !fir.ref<index>
+// CHECK:               %[[VAL_32:.*]] = fir.load %[[VAL_26]] : !fir.ref<index>
+// CHECK:               %[[VAL_33:.*]] = fir.load %[[VAL_27]] : !fir.ref<index>
+// CHECK:               %[[VAL_34:.*]] = arith.addi %[[VAL_32]], %[[VAL_32]] : index
+// CHECK:               omp.teams {
+// CHECK:                 omp.parallel {
+// CHECK:                   omp.distribute {
+// CHECK:                     omp.wsloop {
+// CHECK:                       omp.loop_nest (%[[VAL_35:.*]]) : index = (%[[VAL_31]]) to (%[[VAL_32]]) inclusive step (%[[VAL_33]]) {
+// CHECK:                         fir.store %[[VAL_34]] to %[[VAL_30]] : !fir.heap<index>
+// CHECK:                         omp.yield
+// CHECK:                       }
+// CHECK:                     } {omp.composite}
+// CHECK:                   } {omp.composite}
+// CHECK:                   omp.terminator
+// CHECK:                 } {omp.composite}
+// CHECK:                 omp.terminator
+// CHECK:               }
+// CHECK:               omp.terminator
+// CHECK:             }
+// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_36:.*]], %[[VAL_8]] -> %[[VAL_37:.*]], %[[VAL_9]] -> %[[VAL_38:.*]], %[[VAL_10]] -> %[[VAL_39:.*]], %[[VAL_13]] -> %[[VAL_40:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
+// CHECK:               %[[VAL_41:.*]] = fir.load %[[VAL_40]] : !fir.llvm_ptr<!fir.heap<index>>
+// CHECK:               %[[VAL_42:.*]] = fir.load %[[VAL_36]] : !fir.ref<index>
+// CHECK:               %[[VAL_43:.*]] = fir.load %[[VAL_37]] : !fir.ref<index>
+// CHECK:               %[[VAL_44:.*]] = fir.load %[[VAL_38]] : !fir.ref<index>
+// CHECK:               %[[VAL_45:.*]] = arith.addi %[[VAL_43]], %[[VAL_43]] : index
+// CHECK:               fir.store %[[VAL_42]] to %[[VAL_41]] : !fir.heap<index>
+// CHECK:               fir.freemem %[[VAL_41]] : !fir.heap<index>
+// CHECK:               omp.terminator
+// CHECK:             }
+// CHECK:             omp.terminator
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+func.func @x(%lb : index, %ub : index, %step : index, %addr : !fir.ref<index>) {
+  %lb_ref = fir.alloca index {bindc_name = "lb"}
+  fir.store %lb to %lb_ref : !fir.ref<index>
+  %ub_ref = fir.alloca index {bindc_name = "ub"}
+  fir.store %ub to %ub_ref : !fir.ref<index>
+  %step_ref = fir.alloca index {bindc_name = "step"}
+  fir.store %step to %step_ref : !fir.ref<index>
+
+  %lb_map = omp.map.info var_ptr(%lb_ref : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "lb"}
+  %ub_map = omp.map.info var_ptr(%ub_ref : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "ub"}
+  %step_map = omp.map.info var_ptr(%step_ref : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "step"}
+  %addr_map = omp.map.info var_ptr(%addr : !fir.ref<index>, index) map_clauses(tofrom) capture(ByRef) -> !fir.ref<index> {name = "addr"}
+
+  omp.target map_entries(%lb_map -> %arg0, %ub_map -> %arg1, %step_map -> %arg2, %addr_map -> %arg3 : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>) {
+    %lb_val = fir.load %arg0 : !fir.ref<index>
+    %ub_val = fir.load %arg1 : !fir.ref<index>
+    %step_val = fir.load %arg2 : !fir.ref<index>
+    %one = arith.constant 1 : index
+
+    %20 = arith.addi %ub_val, %ub_val : index
+    omp.teams {
+      omp.workdistribute {
+        %dev_mem = fir.allocmem index, %one {uniq_name = "dev_buf"}
+        fir.do_loop %iv = %lb_val to %ub_val step %step_val unordered {
+          fir.store %20 to %dev_mem : !fir.heap<index>
+        }
+        fir.store %lb_val to %dev_mem : !fir.heap<index>
+        fir.freemem %dev_mem : !fir.heap<index>
+        omp.terminator
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}

>From 611f8b9344502dcf1311b108ba18804fff89b53b Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 6 Jun 2025 14:20:08 +0530
Subject: [PATCH 16/29] Use fir.convert instead of unrealised cast

---
 flang/lib/Lower/OpenMP/OpenMP.cpp                      | 10 ++++++++++
 flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp     |  7 ++-----
 .../Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp       |  3 +++
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index a956498f6e521..1ae6366b3631a 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -565,6 +565,16 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
       });
       break;
 
+    case OMPD_teams_workdistribute:
+      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      [[fallthrough]];
+    case OMPD_target_teams_workdistribute:
+      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      processSingleNestedIf([](Directive nestedDir) {
+        return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
+      });
+      break;
+
     case OMPD_teams_distribute:
     case OMPD_teams_distribute_simd:
       cp.processThreadLimit(stmtCtx, hostInfo.ops);
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index 8f2de92cfd186..6d6de47f7741e 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -597,8 +597,7 @@ static void reloadCacheAndRecompute(Location loc, RewriterBase &rewriter,
     if (isPtr(original.getType())) {
       restored = rewriter.create<LLVM::LoadOp>(loc, llvmPtrTy, newArg);
       if (!isa<LLVM::LLVMPointerType>(original.getType()))
-        restored = rewriter.create<UnrealizedConversionCastOp>(loc, original.getType(), ValueRange(restored))
-                           .getResult(0);
+        restored = rewriter.create<fir::ConvertOp>(loc, original.getType(), restored);
     } 
     else {
         restored = rewriter.create<fir::LoadOp>(loc, newArg);
@@ -684,9 +683,7 @@ static SplitResult isolateOp(Operation *splitBeforeOp, bool splitAfter,
         getPtrTypeForOmp(original.getType()), original.getLoc());
     if (isPtr(original.getType())) {
       if (!isa<LLVM::LLVMPointerType>(toStore.getType()))
-        toStore = rewriter.create<UnrealizedConversionCastOp>(loc, llvmPtrTy,
-                                                           ValueRange(toStore))
-                      .getResult(0);
+        toStore = rewriter.create<fir::ConvertOp>(loc, llvmPtrTy, toStore);
       rewriter.create<LLVM::StoreOp>(loc, toStore, newArg);
     } else {
       rewriter.create<fir::StoreOp>(loc, toStore, newArg);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index eece8573f00ec..3fed83112dc97 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5246,6 +5246,9 @@ initTargetRuntimeAttrs(llvm::IRBuilderBase &builder,
   omp::LoopNestOp loopOp = castOrGetParentOfType<omp::LoopNestOp>(capturedOp);
   unsigned numLoops = loopOp ? loopOp.getNumLoops() : 0;
 
+  if (targetOp.getHostEvalVars().empty())
+    numLoops = 0;
+
   Value numThreads, numTeamsLower, numTeamsUpper, teamsThreadLimit;
   llvm::SmallVector<Value> lowerBounds(numLoops), upperBounds(numLoops),
       steps(numLoops);

>From 99b8d676648f20bd84634c6e6b1e06772c4bf9ad Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Tue, 10 Jun 2025 18:46:57 +0530
Subject: [PATCH 17/29] [Flang] Add fir omp target alloc and free ops

This commit is CP from ivanradanov commit
be860ac8baf24b8405e6f396c75d7f0d26375de5
---
 .../include/flang/Optimizer/Dialect/FIROps.td | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 8ac847dd7dd0a..466699cc4d476 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,6 +517,67 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
+def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Creates a heap memory reference suitable for storing a value of the
+    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
+    The memory object is in an undefined state.  `allocmem` operations must
+    be paired with `freemem` operations to avoid memory leaks.
+
+    ```
+      %0 = fir.omp_target_allocmem !fir.array<10 x f32>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyIntegerType>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyIntegerType>:$typeparams,
+    Variadic<AnyIntegerType>:$shape
+  );
+  let results = (outs fir_HeapType);
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+    bool hasLenParams() { return !getTypeparams().empty(); }
+    bool hasShapeOperands() { return !getShape().empty(); }
+    unsigned numLenParams() { return getTypeparams().size(); }
+    operand_range getLenParams() { return getTypeparams(); }
+    unsigned numShapeOperands() { return getShape().size(); }
+    operand_range getShapeOperands() { return getShape(); }
+    static mlir::Type getRefTy(mlir::Type ty);
+  }];
+}
+
+def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free a heap object";
+
+  let description = [{
+    Deallocates a heap memory reference that was allocated by an `allocmem`.
+    The memory object that is deallocated is placed in an undefined state
+    after `fir.freemem`.  Optimizations may treat the loading of an object
+    in the undefined state as undefined behavior.  This includes aliasing
+    references, such as the result of an `fir.embox`.Add commentMore actions
+
+    ```
+      %21 = fir.omp_target_allocmem !fir.type<ZT(p:i32){field:i32}>
+      ...
+      fir.omp_target_freemem %21 : !fir.heap<!fir.type<ZT>>
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyIntegerType, "", [MemFree]>:$device,
+  Arg<fir_HeapType, "", [MemFree]>:$heapref
+  );
+}
+
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//

>From cd7eb59ef83a4af943afeb3215f476a07762586a Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Tue, 10 Jun 2025 22:11:36 +0530
Subject: [PATCH 18/29] [Flang] Add fir omp target alloc and free ops This
 commit is C-P from ivaradanov commit be860ac8baf24b8405e6f396c75d7f0d26375de5

---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +++++++++++++++++-
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  |  81 ++++++++++++--
 2 files changed, 172 insertions(+), 11 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..f94ccbb91de57 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,6 +1168,105 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetAllocMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = heap.getType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
+    mlir::Location loc = heap.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(dataTy);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
+    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(loc, rewriter, mallocTy, size);
+    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+        heap, ::getLlvmPtrType(heap.getContext()),
+        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
+    return mlir::success();
+  }
+
+  /// Compute the allocation size in bytes of the element type of
+  /// \p llTy pointer type. The result is returned as a value of \p idxTy
+  /// integer type.
+  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy) const {
+    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+  }
+};
+} // namespace
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp freeFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
+    return freeFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_free",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMVoidType::get(module->getContext()),
+          {getLlvmPtrType(module->getContext()), i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetFreeMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
+    mlir::Location loc = freemem.getLoc();
+    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
+    rewriter.create<mlir::LLVM::CallOp>(
+        loc, mlir::TypeRange{},
+        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
+        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
+    rewriter.eraseOp(freemem);
+    return mlir::success();
+  }
+};
+} // namespace
+
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4274,7 +4373,8 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
+      OmpTargetFreeMemOpConversion,SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index 6d6de47f7741e..f0b4f24c2db5b 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -751,6 +751,15 @@ static SplitResult isolateOp(Operation *splitBeforeOp, bool splitAfter,
   return SplitResult{preTargetOp, isolatedTargetOp, postTargetOp};
 }
 
+static mlir::LLVM::ConstantOp
+genI32Constant(mlir::Location loc, mlir::RewriterBase &rewriter, int value) {
+  mlir::Type i32Ty = rewriter.getI32Type();
+  mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, i32Ty, attr);
+}
+
+static Type getOmpDeviceType(MLIRContext *c) { return IntegerType::get(c, 32); }
+
 static void moveToHost(omp::TargetOp targetOp, RewriterBase &rewriter) {
   OpBuilder::InsertionGuard guard(rewriter);
   Block *targetBlock = &targetOp.getRegion().front();
@@ -776,14 +785,66 @@ static void moveToHost(omp::TargetOp targetOp, RewriterBase &rewriter) {
     if (isRuntimeCall(op))
       runtimeCall = cast<fir::CallOp>(op);
 
-    if (allocOp || freeOp || runtimeCall)
-        continue;
-    opsToMove.push_back(op);
+    if (allocOp || freeOp || runtimeCall) {
+      Value device = targetOp.getDevice();
+      if (!device) {
+        device = genI32Constant(it->getLoc(), rewriter, 0);
+      }
+      if (allocOp) {
+        auto tmpAllocOp = rewriter.create<fir::OmpTargetAllocMemOp>(
+            allocOp.getLoc(), allocOp.getType(), device,
+            allocOp.getInTypeAttr(), allocOp.getUniqNameAttr(),
+            allocOp.getBindcNameAttr(), allocOp.getTypeparams(),
+            allocOp.getShape());
+        auto newAllocOp = cast<fir::OmpTargetAllocMemOp>(
+            rewriter.clone(*tmpAllocOp.getOperation(), mapping));
+        mapping.map(allocOp.getResult(), newAllocOp.getResult());
+        rewriter.eraseOp(tmpAllocOp);
+      } else if (freeOp) {
+        auto tmpFreeOp = rewriter.create<fir::OmpTargetFreeMemOp>(
+            freeOp.getLoc(), device, freeOp.getHeapref());
+        rewriter.clone(*tmpFreeOp.getOperation(), mapping);
+        rewriter.eraseOp(tmpFreeOp);
+      } else if (runtimeCall) {
+        auto module = runtimeCall->getParentOfType<ModuleOp>();
+        auto callee = cast<func::FuncOp>(
+            module.lookupSymbol(runtimeCall.getCalleeAttr()));
+        std::string newCalleeName = (callee.getName()).str();
+        mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+        func::FuncOp newCallee =
+            cast_or_null<func::FuncOp>(module.lookupSymbol(newCalleeName));
+        if (!newCallee) {
+          SmallVector<Type> argTypes(callee.getFunctionType().getInputs());
+          argTypes.push_back(getOmpDeviceType(rewriter.getContext()));
+          newCallee = moduleBuilder.create<func::FuncOp>(
+              callee->getLoc(), newCalleeName,
+              FunctionType::get(rewriter.getContext(), argTypes,
+                                callee.getFunctionType().getResults()));
+          if (callee.getArgAttrs())
+            newCallee.setArgAttrsAttr(*callee.getArgAttrs());
+          if (callee.getResAttrs())
+            newCallee.setResAttrsAttr(*callee.getResAttrs());
+          newCallee.setSymVisibility(callee.getSymVisibility());
+          newCallee->setDiscardableAttrs(
+              callee->getDiscardableAttrDictionary());
+        }
+        SmallVector<Value> operands = runtimeCall.getOperands();
+        operands.push_back(device);
+        auto tmpCall = rewriter.create<fir::CallOp>(
+            runtimeCall.getLoc(), runtimeCall.getResultTypes(),
+            SymbolRefAttr::get(newCallee), operands, nullptr, nullptr, nullptr,
+            runtimeCall.getFastmathAttr());
+        Operation *newCall = rewriter.clone(*tmpCall, mapping);
+        mapping.map(&*it, newCall);
+        rewriter.eraseOp(tmpCall);
+      }
+    } else {
+      Operation *clonedOp = rewriter.clone(*op, mapping);
+      for (unsigned i = 0; i < op->getNumResults(); ++i) {
+        mapping.map(op->getResult(i), clonedOp->getResult(i));
+      }
+    }
   }
-  // Move ops before targetOp and erase from region
-  for (Operation *op : opsToMove)
-    rewriter.clone(*op, mapping);
-  
   rewriter.eraseOp(targetOp);
 }
 
@@ -791,7 +852,7 @@ void fissionTarget(omp::TargetOp targetOp, RewriterBase &rewriter) {
   auto tuple = getNestedOpToIsolate(targetOp);
   if (!tuple) {
     LLVM_DEBUG(llvm::dbgs() << " No op to isolate\n");
-    //moveToHost(targetOp, rewriter);
+    moveToHost(targetOp, rewriter);
     return;
   }
 
@@ -801,13 +862,13 @@ void fissionTarget(omp::TargetOp targetOp, RewriterBase &rewriter) {
 
   if (splitBefore && splitAfter) {
     auto res = isolateOp(toIsolate, splitAfter, rewriter);
-    //moveToHost(res.preTargetOp, rewriter);
+    moveToHost(res.preTargetOp, rewriter);
     fissionTarget(res.postTargetOp, rewriter);
     return;
   }
   if (splitBefore) {
     auto res = isolateOp(toIsolate, splitAfter, rewriter);
-    //moveToHost(res.preTargetOp, rewriter);
+    moveToHost(res.preTargetOp, rewriter);
     return;
   }
   if (splitAfter) {

>From 7feb0199033ab970f4e39eaca40e3c56ef1b146b Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 13 Jun 2025 15:03:58 +0530
Subject: [PATCH 19/29] [Flang] Add Assign_omp fortran-rt function This commit
 is c-p from ivanradanov commit d7e44991415663f93cfc342b06e1c81c03161ed6

---
 flang-rt/lib/runtime/CMakeLists.txt           |  2 +
 flang-rt/lib/runtime/assign_omp.cpp           | 72 +++++++++++++++++++
 flang/include/flang/Runtime/assign.h          |  2 +
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  |  2 +-
 4 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 flang-rt/lib/runtime/assign_omp.cpp

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..5200b2b710a5e 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -21,6 +21,7 @@ set(supported_sources
   allocatable.cpp
   array-constructor.cpp
   assign.cpp
+  assign_omp.cpp
   buffer.cpp
   character.cpp
   connection.cpp
@@ -99,6 +100,7 @@ set(gpu_sources
   allocatable.cpp
   array-constructor.cpp
   assign.cpp
+  assign_omp.cpp
   buffer.cpp
   character.cpp
   connection.cpp
diff --git a/flang-rt/lib/runtime/assign_omp.cpp b/flang-rt/lib/runtime/assign_omp.cpp
new file mode 100644
index 0000000000000..e3680c59ed5a1
--- /dev/null
+++ b/flang-rt/lib/runtime/assign_omp.cpp
@@ -0,0 +1,72 @@
+//===-- lib/runtime/assign_omp.cpp ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Runtime/assign.h"
+#include "flang-rt/runtime/assign-impl.h"
+#include "flang-rt/runtime/derived.h"
+#include "flang-rt/runtime/descriptor.h"
+#include "flang-rt/runtime/stat.h"
+#include "flang-rt/runtime/terminator.h"
+#include "flang-rt/runtime/tools.h"
+#include "flang-rt/runtime/type-info.h"
+
+#include <omp.h>
+
+namespace Fortran::runtime {
+
+RT_API_ATTRS static void Assign(Descriptor &to, const Descriptor &from,
+    Terminator &terminator, int flags, int32_t omp_device) {
+  std::size_t toElementBytes{to.ElementBytes()};
+  std::size_t fromElementBytes{from.ElementBytes()};
+  std::size_t toElements{to.Elements()};
+  std::size_t fromElements{from.Elements()};
+
+  if (toElementBytes != fromElementBytes)
+    terminator.Crash("Assign: toElementBytes != fromElementBytes");
+  if (toElements != fromElements)
+    terminator.Crash("Assign: toElements != fromElements");
+
+  void *host_to_ptr = to.raw().base_addr;
+  void *host_from_ptr = from.raw().base_addr;
+  size_t length = toElements * toElementBytes;
+
+  printf("assign length: %zu\n", length);
+
+  if (!omp_target_is_present(host_to_ptr, omp_device))
+    terminator.Crash("Assign: !omp_target_is_present(host_to_ptr, omp_device)");
+  if (!omp_target_is_present(host_from_ptr, omp_device))
+    terminator.Crash(
+        "Assign: !omp_target_is_present(host_from_ptr, omp_device)");
+
+  printf("host_to_ptr: %p\n", host_to_ptr);
+#pragma omp target data use_device_ptr(host_to_ptr, host_from_ptr) device(omp_device)
+  {
+    printf("device_to_ptr: %p\n", host_to_ptr);
+    // TODO do we need to handle overlapping memory? does this function do that?
+    omp_target_memcpy(host_to_ptr, host_from_ptr, length, /*dst_offset*/ 0,
+        /*src_offset*/ 0, /*dst*/ omp_device, /*src*/ omp_device);
+  }
+
+  return;
+}
+
+extern "C" {
+RT_EXT_API_GROUP_BEGIN
+void RTDEF(Assign_omp)(Descriptor &to, const Descriptor &from,
+    const char *sourceFile, int sourceLine, int32_t omp_device) {
+  Terminator terminator{sourceFile, sourceLine};
+  // All top-level defined assignments can be recognized in semantics and
+  // will have been already been converted to calls, so don't check for
+  // defined assignment apart from components.
+  Assign(to, from, terminator,
+      MaybeReallocate | NeedFinalization | ComponentCanBeDefinedAssignment,
+      omp_device);
+}
+} // extern "C"
+
+}
\ No newline at end of file
diff --git a/flang/include/flang/Runtime/assign.h b/flang/include/flang/Runtime/assign.h
index 7d198bdcc9e89..0be52413e4814 100644
--- a/flang/include/flang/Runtime/assign.h
+++ b/flang/include/flang/Runtime/assign.h
@@ -56,6 +56,8 @@ extern "C" {
 // API for lowering assignment
 void RTDECL(Assign)(Descriptor &to, const Descriptor &from,
     const char *sourceFile = nullptr, int sourceLine = 0);
+void RTDECL(Assign_omp)(Descriptor &to, const Descriptor &from,
+    const char *sourceFile = nullptr, int sourceLine = 0, int32_t omp_device = 0);
 // This variant has no finalization, defined assignment, or allocatable
 // reallocation.
 void RTDECL(AssignTemporary)(Descriptor &to, const Descriptor &from,
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index f0b4f24c2db5b..d82a61705eae3 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -809,7 +809,7 @@ static void moveToHost(omp::TargetOp targetOp, RewriterBase &rewriter) {
         auto module = runtimeCall->getParentOfType<ModuleOp>();
         auto callee = cast<func::FuncOp>(
             module.lookupSymbol(runtimeCall.getCalleeAttr()));
-        std::string newCalleeName = (callee.getName()).str();
+        std::string newCalleeName = (callee.getName() + "_omp").str();
         mlir::OpBuilder moduleBuilder(module.getBodyRegion());
         func::FuncOp newCallee =
             cast_or_null<func::FuncOp>(module.lookupSymbol(newCalleeName));

>From 099274d9b93182bda46bb740608175519297a3a5 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 13 Jun 2025 17:37:28 +0530
Subject: [PATCH 20/29] [Flang] Fix workdistribute tests

---
 .../lower-workdistribute-fission-target.mlir  | 62 +++++++++----------
 .../OpenMP/lower-workdistribute-target.mlir   |  3 -
 2 files changed, 29 insertions(+), 36 deletions(-)

diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir
index ed6c641f2e934..53c09581f9816 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir
@@ -1,6 +1,6 @@
 // RUN: fir-opt --lower-workdistribute %s | FileCheck %s
 
-// CHECK-LABEL:   func.func @x
+// CHECK-LABEL:   func.func @x(
 // CHECK:           %[[VAL_0:.*]] = fir.alloca index {bindc_name = "lb"}
 // CHECK:           fir.store %[[ARG0:.*]] to %[[VAL_0]] : !fir.ref<index>
 // CHECK:           %[[VAL_1:.*]] = fir.alloca index {bindc_name = "ub"}
@@ -19,28 +19,26 @@
 // CHECK:             %[[VAL_11:.*]] = fir.alloca !fir.heap<index>
 // CHECK:             %[[VAL_12:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.heap<index>>, !fir.heap<index>) map_clauses(from) capture(ByRef) -> !fir.ref<!fir.heap<index>> {name = "__flang_workdistribute_from"}
 // CHECK:             %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.heap<index>>, !fir.heap<index>) map_clauses(to) capture(ByRef) -> !fir.ref<!fir.heap<index>> {name = "__flang_workdistribute_to"}
-// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_14:.*]], %[[VAL_8]] -> %[[VAL_15:.*]], %[[VAL_9]] -> %[[VAL_16:.*]], %[[VAL_10]] -> %[[VAL_17:.*]], %[[VAL_12]] -> %[[VAL_18:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
-// CHECK:               %[[VAL_19:.*]] = arith.constant 1 : index
-// CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_14]] : !fir.ref<index>
-// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_15]] : !fir.ref<index>
-// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_16]] : !fir.ref<index>
-// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_21]], %[[VAL_21]] : index
-// CHECK:               %[[VAL_24:.*]] = fir.allocmem index, %[[VAL_19]] {uniq_name = "dev_buf"}
-// CHECK:               fir.store %[[VAL_24]] to %[[VAL_18]] : !fir.llvm_ptr<!fir.heap<index>>
-// CHECK:               omp.terminator
-// CHECK:             }
-// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_25:.*]], %[[VAL_8]] -> %[[VAL_26:.*]], %[[VAL_9]] -> %[[VAL_27:.*]], %[[VAL_10]] -> %[[VAL_28:.*]], %[[VAL_13]] -> %[[VAL_29:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
-// CHECK:               %[[VAL_30:.*]] = fir.load %[[VAL_29]] : !fir.llvm_ptr<!fir.heap<index>>
-// CHECK:               %[[VAL_31:.*]] = fir.load %[[VAL_25]] : !fir.ref<index>
-// CHECK:               %[[VAL_32:.*]] = fir.load %[[VAL_26]] : !fir.ref<index>
-// CHECK:               %[[VAL_33:.*]] = fir.load %[[VAL_27]] : !fir.ref<index>
-// CHECK:               %[[VAL_34:.*]] = arith.addi %[[VAL_32]], %[[VAL_32]] : index
+// CHECK:             %[[VAL_14:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_15:.*]] = fir.load %[[VAL_0]] : !fir.ref<index>
+// CHECK:             %[[VAL_16:.*]] = fir.load %[[VAL_1]] : !fir.ref<index>
+// CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_2]] : !fir.ref<index>
+// CHECK:             %[[VAL_18:.*]] = arith.addi %[[VAL_16]], %[[VAL_16]] : index
+// CHECK:             %[[VAL_19:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:             %[[VAL_20:.*]] = "fir.omp_target_allocmem"(%[[VAL_19]], %[[VAL_14]]) <{in_type = index, operandSegmentSizes = array<i32: 1, 0, 1>, uniq_name = "dev_buf"}> : (i32, index) -> !fir.heap<index>
+// CHECK:             fir.store %[[VAL_20]] to %[[VAL_11]] : !fir.ref<!fir.heap<index>>
+// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_21:.*]], %[[VAL_8]] -> %[[VAL_22:.*]], %[[VAL_9]] -> %[[VAL_23:.*]], %[[VAL_10]] -> %[[VAL_24:.*]], %[[VAL_13]] -> %[[VAL_25:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.llvm_ptr<!fir.heap<index>>
+// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_21]] : !fir.ref<index>
+// CHECK:               %[[VAL_28:.*]] = fir.load %[[VAL_22]] : !fir.ref<index>
+// CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_23]] : !fir.ref<index>
+// CHECK:               %[[VAL_30:.*]] = arith.addi %[[VAL_28]], %[[VAL_28]] : index
 // CHECK:               omp.teams {
 // CHECK:                 omp.parallel {
 // CHECK:                   omp.distribute {
 // CHECK:                     omp.wsloop {
-// CHECK:                       omp.loop_nest (%[[VAL_35:.*]]) : index = (%[[VAL_31]]) to (%[[VAL_32]]) inclusive step (%[[VAL_33]]) {
-// CHECK:                         fir.store %[[VAL_34]] to %[[VAL_30]] : !fir.heap<index>
+// CHECK:                       omp.loop_nest (%[[VAL_31:.*]]) : index = (%[[VAL_27]]) to (%[[VAL_28]]) inclusive step (%[[VAL_29]]) {
+// CHECK:                         fir.store %[[VAL_30]] to %[[VAL_26]] : !fir.heap<index>
 // CHECK:                         omp.yield
 // CHECK:                       }
 // CHECK:                     } {omp.composite}
@@ -51,16 +49,14 @@
 // CHECK:               }
 // CHECK:               omp.terminator
 // CHECK:             }
-// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_36:.*]], %[[VAL_8]] -> %[[VAL_37:.*]], %[[VAL_9]] -> %[[VAL_38:.*]], %[[VAL_10]] -> %[[VAL_39:.*]], %[[VAL_13]] -> %[[VAL_40:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
-// CHECK:               %[[VAL_41:.*]] = fir.load %[[VAL_40]] : !fir.llvm_ptr<!fir.heap<index>>
-// CHECK:               %[[VAL_42:.*]] = fir.load %[[VAL_36]] : !fir.ref<index>
-// CHECK:               %[[VAL_43:.*]] = fir.load %[[VAL_37]] : !fir.ref<index>
-// CHECK:               %[[VAL_44:.*]] = fir.load %[[VAL_38]] : !fir.ref<index>
-// CHECK:               %[[VAL_45:.*]] = arith.addi %[[VAL_43]], %[[VAL_43]] : index
-// CHECK:               fir.store %[[VAL_42]] to %[[VAL_41]] : !fir.heap<index>
-// CHECK:               fir.freemem %[[VAL_41]] : !fir.heap<index>
-// CHECK:               omp.terminator
-// CHECK:             }
+// CHECK:             %[[VAL_32:.*]] = fir.load %[[VAL_11]] : !fir.ref<!fir.heap<index>>
+// CHECK:             %[[VAL_33:.*]] = fir.load %[[VAL_0]] : !fir.ref<index>
+// CHECK:             %[[VAL_34:.*]] = fir.load %[[VAL_1]] : !fir.ref<index>
+// CHECK:             %[[VAL_35:.*]] = fir.load %[[VAL_2]] : !fir.ref<index>
+// CHECK:             %[[VAL_36:.*]] = arith.addi %[[VAL_34]], %[[VAL_34]] : index
+// CHECK:             fir.store %[[VAL_33]] to %[[VAL_32]] : !fir.heap<index>
+// CHECK:             %[[VAL_37:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:             "fir.omp_target_freemem"(%[[VAL_37]], %[[VAL_32]]) : (i32, !fir.heap<index>) -> ()
 // CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           return
@@ -79,10 +75,10 @@ func.func @x(%lb : index, %ub : index, %step : index, %addr : !fir.ref<index>) {
   %step_map = omp.map.info var_ptr(%step_ref : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "step"}
   %addr_map = omp.map.info var_ptr(%addr : !fir.ref<index>, index) map_clauses(tofrom) capture(ByRef) -> !fir.ref<index> {name = "addr"}
 
-  omp.target map_entries(%lb_map -> %arg0, %ub_map -> %arg1, %step_map -> %arg2, %addr_map -> %arg3 : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>) {
-    %lb_val = fir.load %arg0 : !fir.ref<index>
-    %ub_val = fir.load %arg1 : !fir.ref<index>
-    %step_val = fir.load %arg2 : !fir.ref<index>
+  omp.target map_entries(%lb_map -> %ARG0, %ub_map -> %ARG1, %step_map -> %ARG2, %addr_map -> %ARG3 : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>) {
+    %lb_val = fir.load %ARG0 : !fir.ref<index>
+    %ub_val = fir.load %ARG1 : !fir.ref<index>
+    %step_val = fir.load %ARG2 : !fir.ref<index>
     %one = arith.constant 1 : index
 
     %20 = arith.addi %ub_val, %ub_val : index
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
index e6ca98d3bf596..ad2cd422d9533 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
@@ -12,9 +12,6 @@
 // CHECK:           %[[VAL_10:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
 // CHECK:           omp.target_data map_entries(%[[VAL_10]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
 // CHECK:             %[[VAL_11:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
-// CHECK:             omp.target map_entries(%[[VAL_11]] -> %[[VAL_12:.*]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
-// CHECK:               omp.terminator
-// CHECK:             }
 // CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           return

>From 73f1e0df3a010c72f855553eecc4c8b0cb0e381f Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 16 Jun 2025 13:40:27 +0530
Subject: [PATCH 21/29] [Flang] Fix omp target alloc mem lowering C-P from
 ivanradanov commit 73fd86537980fe1d9454d5a60642f1b71290cc55

---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index f94ccbb91de57..cf4ca5f1436b5 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1204,7 +1204,7 @@ struct OmpTargetAllocMemOpConversion
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
     if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands())
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
       size = rewriter.create<mlir::LLVM::MulOp>(
           loc, ity, size, integerCast(loc, rewriter, ity, opnd));
     auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();

>From 559df28e0db964e95072c319375db1aa7bb67fd4 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 18 Jun 2025 10:10:21 +0530
Subject: [PATCH 22/29] [Flang] Update assign_omp logic

---
 flang-rt/lib/runtime/assign_omp.cpp           | 56 ++++++-----
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 99 ++++++++++---------
 .../lower-workdistribute-fission-target.mlir  | 74 ++++++++------
 .../OpenMP/lower-workdistribute-target.mlir   |  2 +-
 4 files changed, 126 insertions(+), 105 deletions(-)

diff --git a/flang-rt/lib/runtime/assign_omp.cpp b/flang-rt/lib/runtime/assign_omp.cpp
index e3680c59ed5a1..dee912155829b 100644
--- a/flang-rt/lib/runtime/assign_omp.cpp
+++ b/flang-rt/lib/runtime/assign_omp.cpp
@@ -18,9 +18,23 @@
 #include <omp.h>
 
 namespace Fortran::runtime {
+namespace omp {
+
+typedef int32_t OMPDeviceTy;
+
+template <typename T> static T *getDevicePtr(T *anyPtr, OMPDeviceTy ompDevice) {
+  auto voidAnyPtr = reinterpret_cast<void *>(anyPtr);
+  // If not present on the device it should already be a device ptr
+  if (!omp_target_is_present(voidAnyPtr, ompDevice))
+    return anyPtr;
+  T *device_ptr = nullptr;
+#pragma omp target data use_device_ptr(anyPtr) device(ompDevice)
+  device_ptr = anyPtr;
+  return device_ptr;
+}
 
 RT_API_ATTRS static void Assign(Descriptor &to, const Descriptor &from,
-    Terminator &terminator, int flags, int32_t omp_device) {
+    Terminator &terminator, int flags, OMPDeviceTy omp_device) {
   std::size_t toElementBytes{to.ElementBytes()};
   std::size_t fromElementBytes{from.ElementBytes()};
   std::size_t toElements{to.Elements()};
@@ -31,42 +45,34 @@ RT_API_ATTRS static void Assign(Descriptor &to, const Descriptor &from,
   if (toElements != fromElements)
     terminator.Crash("Assign: toElements != fromElements");
 
-  void *host_to_ptr = to.raw().base_addr;
-  void *host_from_ptr = from.raw().base_addr;
+  // Get base addresses and calculate length
+  void *to_base = to.raw().base_addr;
+  void *from_base = from.raw().base_addr;
   size_t length = toElements * toElementBytes;
 
-  printf("assign length: %zu\n", length);
+  // Get device pointers after ensuring data is on device
+  void *to_ptr = getDevicePtr(to_base, omp_device);
+  void *from_ptr = getDevicePtr(from_base, omp_device);
 
-  if (!omp_target_is_present(host_to_ptr, omp_device))
-    terminator.Crash("Assign: !omp_target_is_present(host_to_ptr, omp_device)");
-  if (!omp_target_is_present(host_from_ptr, omp_device))
-    terminator.Crash(
-        "Assign: !omp_target_is_present(host_from_ptr, omp_device)");
-
-  printf("host_to_ptr: %p\n", host_to_ptr);
-#pragma omp target data use_device_ptr(host_to_ptr, host_from_ptr) device(omp_device)
-  {
-    printf("device_to_ptr: %p\n", host_to_ptr);
-    // TODO do we need to handle overlapping memory? does this function do that?
-    omp_target_memcpy(host_to_ptr, host_from_ptr, length, /*dst_offset*/ 0,
-        /*src_offset*/ 0, /*dst*/ omp_device, /*src*/ omp_device);
-  }
+  // Perform copy between device pointers
+  int result = omp_target_memcpy(to_ptr, from_ptr, length,
+      /*dst_offset*/ 0, /*src_offset*/ 0, omp_device, omp_device);
 
+  if (result != 0)
+    terminator.Crash("Assign: omp_target_memcpy failed");
   return;
 }
 
 extern "C" {
 RT_EXT_API_GROUP_BEGIN
 void RTDEF(Assign_omp)(Descriptor &to, const Descriptor &from,
-    const char *sourceFile, int sourceLine, int32_t omp_device) {
+    const char *sourceFile, int sourceLine, omp::OMPDeviceTy omp_device) {
   Terminator terminator{sourceFile, sourceLine};
-  // All top-level defined assignments can be recognized in semantics and
-  // will have been already been converted to calls, so don't check for
-  // defined assignment apart from components.
-  Assign(to, from, terminator,
+  omp::Assign(to, from, terminator,
       MaybeReallocate | NeedFinalization | ComponentCanBeDefinedAssignment,
       omp_device);
 }
-} // extern "C"
 
-}
\ No newline at end of file
+} // extern "C"
+} // namespace omp
+} // namespace Fortran::runtime
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index d82a61705eae3..5900d93c4e770 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -357,55 +357,77 @@ struct SplitTargetResult {
 /// original data region and avoid unnecessary data movement at each of the
 /// subkernels - we split the target region into a target_data{target}
 /// nest where only the outer one moves the data
-std::optional<SplitTargetResult> splitTargetData(omp::TargetOp targetOp, RewriterBase &rewriter) {
-
+std::optional<SplitTargetResult> splitTargetData(omp::TargetOp targetOp,
+                                                 RewriterBase &rewriter) {
   auto loc = targetOp->getLoc();
   if (targetOp.getMapVars().empty()) {
     LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " target region has no data maps\n");
     return std::nullopt;
   }
 
-  // Collect all map_entries with capture(ByRef)
-  SmallVector<mlir::Value> byRefMapInfos;
-  SmallVector<omp::MapInfoOp> MapInfos;
+  SmallVector<omp::MapInfoOp> mapInfos;
   for (auto opr : targetOp.getMapVars()) {
     auto mapInfo = cast<omp::MapInfoOp>(opr.getDefiningOp());
-    MapInfos.push_back(mapInfo);
-    if (mapInfo.getMapCaptureType() == omp::VariableCaptureKind::ByRef)
-      byRefMapInfos.push_back(opr);
+    mapInfos.push_back(mapInfo);
+  }
+
+  rewriter.setInsertionPoint(targetOp);
+  SmallVector<Value> innerMapInfos;
+  SmallVector<Value> outerMapInfos;
+
+  for (auto mapInfo : mapInfos) {
+    auto originalMapType =
+        (llvm::omp::OpenMPOffloadMappingFlags)(mapInfo.getMapType());
+    auto originalCaptureType = mapInfo.getMapCaptureType();
+    llvm::omp::OpenMPOffloadMappingFlags newMapType;
+    mlir::omp::VariableCaptureKind newCaptureType;
+
+    if (originalCaptureType == mlir::omp::VariableCaptureKind::ByCopy) {
+      newMapType = originalMapType;
+      newCaptureType = originalCaptureType;
+    } else if (originalCaptureType == mlir::omp::VariableCaptureKind::ByRef) {
+      newMapType = llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
+      newCaptureType = originalCaptureType;
+      outerMapInfos.push_back(mapInfo);
+    } else {
+      llvm_unreachable("Unhandled case");
+    }
+    auto innerMapInfo = cast<omp::MapInfoOp>(rewriter.clone(*mapInfo));
+    innerMapInfo.setMapTypeAttr(rewriter.getIntegerAttr(
+        rewriter.getIntegerType(64, false),
+        static_cast<
+            std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
+            newMapType)));
+    innerMapInfo.setMapCaptureType(newCaptureType);
+    innerMapInfos.push_back(innerMapInfo.getResult());
   }
 
-  // Create the new omp.target_data op with these collected map_entries
   rewriter.setInsertionPoint(targetOp);
   auto device = targetOp.getDevice();
   auto ifExpr = targetOp.getIfExpr();
   auto deviceAddrVars = targetOp.getHasDeviceAddrVars();
   auto devicePtrVars = targetOp.getIsDevicePtrVars();
-  auto targetDataOp = rewriter.create<omp::TargetDataOp>(loc, device, ifExpr, 
-                                                          mlir::ValueRange{byRefMapInfos},
-                                                          deviceAddrVars,
-                                                          devicePtrVars);
-
+  auto targetDataOp = rewriter.create<omp::TargetDataOp>(
+      loc, device, ifExpr, outerMapInfos, deviceAddrVars, devicePtrVars);
   auto taregtDataBlock = rewriter.createBlock(&targetDataOp.getRegion());
   rewriter.create<mlir::omp::TerminatorOp>(loc);
   rewriter.setInsertionPointToStart(taregtDataBlock);
 
-  // Clone mapInfo ops inside omp.target_data region
-  IRMapping mapping;
-  for (auto mapInfo : MapInfos) {
-    rewriter.clone(*mapInfo, mapping);
-  }
-  // Clone omp.target from exisiting targetOp inside target_data region.
-  auto newTargetOp = rewriter.clone(*targetOp, mapping);
+  auto newTargetOp = rewriter.create<omp::TargetOp>(
+      targetOp.getLoc(), targetOp.getAllocateVars(),
+      targetOp.getAllocatorVars(), targetOp.getBareAttr(),
+      targetOp.getDependKindsAttr(), targetOp.getDependVars(),
+      targetOp.getDevice(), targetOp.getHasDeviceAddrVars(),
+      targetOp.getHostEvalVars(), targetOp.getIfExpr(),
+      targetOp.getInReductionVars(), targetOp.getInReductionByrefAttr(),
+      targetOp.getInReductionSymsAttr(), targetOp.getIsDevicePtrVars(),
+      innerMapInfos, targetOp.getNowaitAttr(), targetOp.getPrivateVars(),
+      targetOp.getPrivateSymsAttr(), targetOp.getThreadLimit(),
+      targetOp.getPrivateMapsAttr());
+  rewriter.inlineRegionBefore(targetOp.getRegion(), newTargetOp.getRegion(),
+                              newTargetOp.getRegion().begin());
 
-  // Erase TargetOp and its MapInfoOps
-  rewriter.eraseOp(targetOp);
-  
-  for (auto mapInfo : MapInfos) {
-    auto mapInfoRes = mapInfo.getResult();
-    if (mapInfoRes.getUsers().empty()) 
-      rewriter.eraseOp(mapInfo);
-  }
+  rewriter.replaceOp(targetOp, newTargetOp);
   return SplitTargetResult{cast<omp::TargetOp>(newTargetOp), targetDataOp};
 }
 
@@ -521,25 +543,6 @@ static bool isRecomputableAfterFission(Operation *op, Operation *splitBefore) {
   if (isa<fir::DeclareOp>(op))
     return true;
 
-  if (auto loadOp = dyn_cast<fir::LoadOp>(op)) {  
-    Value memref = loadOp.getMemref();  
-    if (auto blockArg = dyn_cast<BlockArgument>(memref)) {  
-      // 'op' is an operation within the targetOp that 'splitBefore' is also in.
-      Operation *parentOpOfLoadBlock = op->getBlock()->getParentOp();  
-      // Ensure the blockArg belongs to the entry block of this parent omp.TargetOp.  
-      // This implies the load is from a variable directly mapped into the target region.  
-      if (isa<omp::TargetOp>(parentOpOfLoadBlock) &&  
-          !parentOpOfLoadBlock->getRegions().empty()) {  
-        Block *targetOpEntryBlock = &parentOpOfLoadBlock->getRegions().front().front();  
-        if (blockArg.getOwner() == targetOpEntryBlock) {  
-          // This load is from a direct argument of the target op.  
-          // It's safe to recompute.
-          return true;  
-        }  
-      }  
-    }  
-  } 
-
   llvm::SmallVector<MemoryEffects::EffectInstance> effects;
   MemoryEffectOpInterface interface = dyn_cast<MemoryEffectOpInterface>(op);
   if (!interface) {
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir
index 53c09581f9816..19bdb9ce10fbd 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-fission-target.mlir
@@ -11,34 +11,46 @@
 // CHECK:           %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "ub"}
 // CHECK:           %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_2]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "step"}
 // CHECK:           %[[VAL_6:.*]] = omp.map.info var_ptr(%[[ARG3:.*]] : !fir.ref<index>, index) map_clauses(tofrom) capture(ByRef) -> !fir.ref<index> {name = "addr"}
+// CHECK:           %[[VAL_7:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<index>, index) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> !fir.ref<index> {name = "lb"}
+// CHECK:           %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<index>, index) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> !fir.ref<index> {name = "ub"}
+// CHECK:           %[[VAL_9:.*]] = omp.map.info var_ptr(%[[VAL_2]] : !fir.ref<index>, index) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> !fir.ref<index> {name = "step"}
+// CHECK:           %[[VAL_10:.*]] = omp.map.info var_ptr(%[[ARG3:.*]] : !fir.ref<index>, index) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> !fir.ref<index> {name = "addr"}
 // CHECK:           omp.target_data map_entries(%[[VAL_3]], %[[VAL_4]], %[[VAL_5]], %[[VAL_6]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>) {
-// CHECK:             %[[VAL_7:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "lb"}
-// CHECK:             %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "ub"}
-// CHECK:             %[[VAL_9:.*]] = omp.map.info var_ptr(%[[VAL_2]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "step"}
-// CHECK:             %[[VAL_10:.*]] = omp.map.info var_ptr(%[[ARG3:.*]] : !fir.ref<index>, index) map_clauses(tofrom) capture(ByRef) -> !fir.ref<index> {name = "addr"}
-// CHECK:             %[[VAL_11:.*]] = fir.alloca !fir.heap<index>
-// CHECK:             %[[VAL_12:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.heap<index>>, !fir.heap<index>) map_clauses(from) capture(ByRef) -> !fir.ref<!fir.heap<index>> {name = "__flang_workdistribute_from"}
-// CHECK:             %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.heap<index>>, !fir.heap<index>) map_clauses(to) capture(ByRef) -> !fir.ref<!fir.heap<index>> {name = "__flang_workdistribute_to"}
-// CHECK:             %[[VAL_14:.*]] = arith.constant 1 : index
-// CHECK:             %[[VAL_15:.*]] = fir.load %[[VAL_0]] : !fir.ref<index>
-// CHECK:             %[[VAL_16:.*]] = fir.load %[[VAL_1]] : !fir.ref<index>
-// CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_2]] : !fir.ref<index>
-// CHECK:             %[[VAL_18:.*]] = arith.addi %[[VAL_16]], %[[VAL_16]] : index
-// CHECK:             %[[VAL_19:.*]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK:             %[[VAL_20:.*]] = "fir.omp_target_allocmem"(%[[VAL_19]], %[[VAL_14]]) <{in_type = index, operandSegmentSizes = array<i32: 1, 0, 1>, uniq_name = "dev_buf"}> : (i32, index) -> !fir.heap<index>
-// CHECK:             fir.store %[[VAL_20]] to %[[VAL_11]] : !fir.ref<!fir.heap<index>>
-// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_21:.*]], %[[VAL_8]] -> %[[VAL_22:.*]], %[[VAL_9]] -> %[[VAL_23:.*]], %[[VAL_10]] -> %[[VAL_24:.*]], %[[VAL_13]] -> %[[VAL_25:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
-// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.llvm_ptr<!fir.heap<index>>
-// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_21]] : !fir.ref<index>
-// CHECK:               %[[VAL_28:.*]] = fir.load %[[VAL_22]] : !fir.ref<index>
-// CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_23]] : !fir.ref<index>
-// CHECK:               %[[VAL_30:.*]] = arith.addi %[[VAL_28]], %[[VAL_28]] : index
+// CHECK:             %[[VAL_11:.*]] = fir.alloca index
+// CHECK:             %[[VAL_12:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<index>, index) map_clauses(from) capture(ByRef) -> !fir.ref<index> {name = "__flang_workdistribute_from"}
+// CHECK:             %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "__flang_workdistribute_to"}
+// CHECK:             %[[VAL_14:.*]] = fir.alloca index
+// CHECK:             %[[VAL_15:.*]] = omp.map.info var_ptr(%[[VAL_14]] : !fir.ref<index>, index) map_clauses(from) capture(ByRef) -> !fir.ref<index> {name = "__flang_workdistribute_from"}
+// CHECK:             %[[VAL_16:.*]] = omp.map.info var_ptr(%[[VAL_14]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "__flang_workdistribute_to"}
+// CHECK:             %[[VAL_17:.*]] = fir.alloca index
+// CHECK:             %[[VAL_18:.*]] = omp.map.info var_ptr(%[[VAL_17]] : !fir.ref<index>, index) map_clauses(from) capture(ByRef) -> !fir.ref<index> {name = "__flang_workdistribute_from"}
+// CHECK:             %[[VAL_19:.*]] = omp.map.info var_ptr(%[[VAL_17]] : !fir.ref<index>, index) map_clauses(to) capture(ByRef) -> !fir.ref<index> {name = "__flang_workdistribute_to"}
+// CHECK:             %[[VAL_20:.*]] = fir.alloca !fir.heap<index>
+// CHECK:             %[[VAL_21:.*]] = omp.map.info var_ptr(%[[VAL_20]] : !fir.ref<!fir.heap<index>>, !fir.heap<index>) map_clauses(from) capture(ByRef) -> !fir.ref<!fir.heap<index>> {name = "__flang_workdistribute_from"}
+// CHECK:             %[[VAL_22:.*]] = omp.map.info var_ptr(%[[VAL_20]] : !fir.ref<!fir.heap<index>>, !fir.heap<index>) map_clauses(to) capture(ByRef) -> !fir.ref<!fir.heap<index>> {name = "__flang_workdistribute_to"}
+// CHECK:             %[[VAL_23:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_0]] : !fir.ref<index>
+// CHECK:             %[[VAL_25:.*]] = fir.load %[[VAL_1]] : !fir.ref<index>
+// CHECK:             %[[VAL_26:.*]] = fir.load %[[VAL_2]] : !fir.ref<index>
+// CHECK:             %[[VAL_27:.*]] = arith.addi %[[VAL_25]], %[[VAL_25]] : index
+// CHECK:             %[[VAL_28:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:             %[[VAL_29:.*]] = "fir.omp_target_allocmem"(%[[VAL_28]], %[[VAL_23]]) <{in_type = index, operandSegmentSizes = array<i32: 1, 0, 1>, uniq_name = "dev_buf"}> : (i32, index) -> !fir.heap<index>
+// CHECK:             fir.store %[[VAL_24]] to %[[VAL_11]] : !fir.ref<index>
+// CHECK:             fir.store %[[VAL_25]] to %[[VAL_14]] : !fir.ref<index>
+// CHECK:             fir.store %[[VAL_26]] to %[[VAL_17]] : !fir.ref<index>
+// CHECK:             fir.store %[[VAL_29]] to %[[VAL_20]] : !fir.ref<!fir.heap<index>>
+// CHECK:             omp.target map_entries(%[[VAL_7]] -> %[[VAL_30:.*]], %[[VAL_8]] -> %[[VAL_31:.*]], %[[VAL_9]] -> %[[VAL_32:.*]], %[[VAL_10]] -> %[[VAL_33:.*]], %[[VAL_13]] -> %[[VAL_34:.*]], %[[VAL_16]] -> %[[VAL_35:.*]], %[[VAL_19]] -> %[[VAL_36:.*]], %[[VAL_22]] -> %[[VAL_37:.*]] : !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<index>, !fir.ref<!fir.heap<index>>) {
+// CHECK:               %[[VAL_38:.*]] = fir.load %[[VAL_34]] : !fir.llvm_ptr<index>
+// CHECK:               %[[VAL_39:.*]] = fir.load %[[VAL_35]] : !fir.llvm_ptr<index>
+// CHECK:               %[[VAL_40:.*]] = fir.load %[[VAL_36]] : !fir.llvm_ptr<index>
+// CHECK:               %[[VAL_41:.*]] = fir.load %[[VAL_37]] : !fir.llvm_ptr<!fir.heap<index>>
+// CHECK:               %[[VAL_42:.*]] = arith.addi %[[VAL_39]], %[[VAL_39]] : index
 // CHECK:               omp.teams {
 // CHECK:                 omp.parallel {
 // CHECK:                   omp.distribute {
 // CHECK:                     omp.wsloop {
-// CHECK:                       omp.loop_nest (%[[VAL_31:.*]]) : index = (%[[VAL_27]]) to (%[[VAL_28]]) inclusive step (%[[VAL_29]]) {
-// CHECK:                         fir.store %[[VAL_30]] to %[[VAL_26]] : !fir.heap<index>
+// CHECK:                       omp.loop_nest (%[[VAL_43:.*]]) : index = (%[[VAL_38]]) to (%[[VAL_39]]) inclusive step (%[[VAL_40]]) {
+// CHECK:                         fir.store %[[VAL_42]] to %[[VAL_41]] : !fir.heap<index>
 // CHECK:                         omp.yield
 // CHECK:                       }
 // CHECK:                     } {omp.composite}
@@ -49,14 +61,14 @@
 // CHECK:               }
 // CHECK:               omp.terminator
 // CHECK:             }
-// CHECK:             %[[VAL_32:.*]] = fir.load %[[VAL_11]] : !fir.ref<!fir.heap<index>>
-// CHECK:             %[[VAL_33:.*]] = fir.load %[[VAL_0]] : !fir.ref<index>
-// CHECK:             %[[VAL_34:.*]] = fir.load %[[VAL_1]] : !fir.ref<index>
-// CHECK:             %[[VAL_35:.*]] = fir.load %[[VAL_2]] : !fir.ref<index>
-// CHECK:             %[[VAL_36:.*]] = arith.addi %[[VAL_34]], %[[VAL_34]] : index
-// CHECK:             fir.store %[[VAL_33]] to %[[VAL_32]] : !fir.heap<index>
-// CHECK:             %[[VAL_37:.*]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK:             "fir.omp_target_freemem"(%[[VAL_37]], %[[VAL_32]]) : (i32, !fir.heap<index>) -> ()
+// CHECK:             %[[VAL_44:.*]] = fir.load %[[VAL_11]] : !fir.ref<index>
+// CHECK:             %[[VAL_45:.*]] = fir.load %[[VAL_14]] : !fir.ref<index>
+// CHECK:             %[[VAL_46:.*]] = fir.load %[[VAL_17]] : !fir.ref<index>
+// CHECK:             %[[VAL_47:.*]] = fir.load %[[VAL_20]] : !fir.ref<!fir.heap<index>>
+// CHECK:             %[[VAL_48:.*]] = arith.addi %[[VAL_45]], %[[VAL_45]] : index
+// CHECK:             fir.store %[[VAL_44]] to %[[VAL_47]] : !fir.heap<index>
+// CHECK:             %[[VAL_49:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:             "fir.omp_target_freemem"(%[[VAL_49]], %[[VAL_47]]) : (i32, !fir.heap<index>) -> ()
 // CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           return
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
index ad2cd422d9533..91e6d5b7201a7 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
@@ -10,8 +10,8 @@
 // CHECK:           %[[VAL_7:.*]] = fir.coordinate_of %[[VAL_6]], r : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>) -> !fir.ref<f32>
 // CHECK:           %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_7]] : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "sa%[[VAL_4]]%[[VAL_9:.*]]"}
 // CHECK:           %[[VAL_10:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
+// CHECK:           %[[VAL_11:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(exit_release_or_enter_alloc) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
 // CHECK:           omp.target_data map_entries(%[[VAL_10]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
-// CHECK:             %[[VAL_11:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
 // CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           return

>From 3b34aef42f52576fef93cdfb813abc57f3b8c9f9 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 18 Jun 2025 15:37:32 +0530
Subject: [PATCH 23/29] [Flang] Bail out if lower-workdistribute didn't
 patternmatch.

---
 flang-rt/lib/runtime/assign_omp.cpp           |  2 +-
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 47 +++++++++----------
 .../OpenMP/lower-workdistribute-target.mlir   |  3 +-
 3 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/flang-rt/lib/runtime/assign_omp.cpp b/flang-rt/lib/runtime/assign_omp.cpp
index dee912155829b..80c1c5cccb1ca 100644
--- a/flang-rt/lib/runtime/assign_omp.cpp
+++ b/flang-rt/lib/runtime/assign_omp.cpp
@@ -68,7 +68,7 @@ RT_EXT_API_GROUP_BEGIN
 void RTDEF(Assign_omp)(Descriptor &to, const Descriptor &from,
     const char *sourceFile, int sourceLine, omp::OMPDeviceTy omp_device) {
   Terminator terminator{sourceFile, sourceLine};
-  omp::Assign(to, from, terminator,
+  Fortran::runtime::omp::Assign(to, from, terminator,
       MaybeReallocate | NeedFinalization | ComponentCanBeDefinedAssignment,
       omp_device);
 }
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index 5900d93c4e770..3dfb574977036 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -125,6 +125,7 @@ static T getPerfectlyNested(Operation *op) {
 /// E()
 
 struct FissionWorkdistribute : public OpRewritePattern<omp::WorkdistributeOp> {
+  static bool fissionWorkdistributePatternMatched;
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(omp::WorkdistributeOp workdistribute,
                                 PatternRewriter &rewriter) const override {
@@ -210,9 +211,12 @@ struct FissionWorkdistribute : public OpRewritePattern<omp::WorkdistributeOp> {
         changed = true;
       }
     }
+    if (changed)
+      fissionWorkdistributePatternMatched = true;
     return success(changed);
   }
 };
+bool FissionWorkdistribute::fissionWorkdistributePatternMatched = false;
 
 /// If fir.do_loop is present inside teams workdistribute
 ///
@@ -296,6 +300,7 @@ static void genWsLoopOp(mlir::PatternRewriter &rewriter, fir::DoLoopOp doLoop,
 }
 
 struct WorkdistributeDoLower : public OpRewritePattern<omp::WorkdistributeOp> {
+  static bool workdistributeDoLowerPatternMatched;
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(omp::WorkdistributeOp workdistribute,
                                 PatternRewriter &rewriter) const override {
@@ -309,12 +314,15 @@ struct WorkdistributeDoLower : public OpRewritePattern<omp::WorkdistributeOp> {
       genLoopNestClauseOps(rewriter, doLoop, loopNestClauseOps);
       genWsLoopOp(rewriter, doLoop, loopNestClauseOps, true);
       rewriter.eraseOp(workdistribute);
+      workdistributeDoLowerPatternMatched = true;
       return success();
     }
     return failure();
   }
 };
 
+bool WorkdistributeDoLower::workdistributeDoLowerPatternMatched = false;
+
 /// If A() and B () are present inside teams workdistribute
 ///
 /// omp.teams {
@@ -331,6 +339,7 @@ struct WorkdistributeDoLower : public OpRewritePattern<omp::WorkdistributeOp> {
 ///
 
 struct TeamsWorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
+  static bool teamsWorkdistributeToSinglePatternMatched;
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(omp::TeamsOp teamsOp,
                                 PatternRewriter &rewriter) const override {
@@ -343,9 +352,12 @@ struct TeamsWorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
     rewriter.eraseOp(workdistributeBlock->getTerminator());
     rewriter.inlineBlockBefore(workdistributeBlock, teamsOp);
     rewriter.eraseOp(workdistributeOp);
+    teamsWorkdistributeToSinglePatternMatched = true;
     return success();
   }
 };
+bool TeamsWorkdistributeToSingle::teamsWorkdistributeToSinglePatternMatched =
+    false;
 
 struct SplitTargetResult {
   omp::TargetOp targetOp;
@@ -517,28 +529,6 @@ static bool usedOutsideSplit(Value v, Operation *split) {
   return false;
 };
 
-static bool isOpToBeCached(Operation *op) {
-  if (auto loadOp = dyn_cast<fir::LoadOp>(op)) {  
-    Value memref = loadOp.getMemref();  
-    if (auto blockArg = dyn_cast<BlockArgument>(memref)) {  
-      // 'op' is an operation within the targetOp that 'splitBefore' is also in.
-      Operation *parentOpOfLoadBlock = op->getBlock()->getParentOp();  
-      // Ensure the blockArg belongs to the entry block of this parent omp.TargetOp.  
-      // This implies the load is from a variable directly mapped into the target region.  
-      if (isa<omp::TargetOp>(parentOpOfLoadBlock) &&  
-          !parentOpOfLoadBlock->getRegions().empty()) {  
-        Block *targetOpEntryBlock = &parentOpOfLoadBlock->getRegions().front().front();  
-        if (blockArg.getOwner() == targetOpEntryBlock) {  
-          // This load is from a direct argument of the target op.  
-          // It's safe to recompute.
-          return false;  
-        }  
-      }  
-    }  
-  }
-  return true;
-}
-
 static bool isRecomputableAfterFission(Operation *op, Operation *splitBefore) {
   if (isa<fir::DeclareOp>(op))
     return true;
@@ -892,6 +882,7 @@ class LowerWorkdistributePass
     config.setRegionSimplificationLevel(GreedySimplifyRegionLevel::Disabled);
 
     Operation *op = getOperation();
+    bool anyPatternChanged = false;
     {
       RewritePatternSet patterns(&context);
       patterns.insert<FissionWorkdistribute, WorkdistributeDoLower>(&context);
@@ -899,16 +890,23 @@ class LowerWorkdistributePass
         emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
         signalPassFailure();
       }
+      anyPatternChanged |=
+          FissionWorkdistribute::fissionWorkdistributePatternMatched;
+      anyPatternChanged |=
+          WorkdistributeDoLower::workdistributeDoLowerPatternMatched;
     }
     {
       RewritePatternSet patterns(&context);
-      patterns.insert<TeamsWorkdistributeToSingle>(&context);
+      patterns.insert<WorkdistributeDoLower, TeamsWorkdistributeToSingle>(
+          &context);
       if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
         emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
         signalPassFailure();
       }
+      anyPatternChanged |= TeamsWorkdistributeToSingle::
+          teamsWorkdistributeToSinglePatternMatched;
     }
-    {
+    if (anyPatternChanged) {
       SmallVector<omp::TargetOp> targetOps;
       op->walk([&](omp::TargetOp targetOp) { targetOps.push_back(targetOp); });
       IRRewriter rewriter(&context);
@@ -917,7 +915,6 @@ class LowerWorkdistributePass
         if (res) fissionTarget(res->targetOp, rewriter);
       }
     }
-
   }
 };
 } // namespace
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
index 91e6d5b7201a7..d96068b26ca2f 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-target.mlir
@@ -10,8 +10,7 @@
 // CHECK:           %[[VAL_7:.*]] = fir.coordinate_of %[[VAL_6]], r : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>) -> !fir.ref<f32>
 // CHECK:           %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_7]] : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "sa%[[VAL_4]]%[[VAL_9:.*]]"}
 // CHECK:           %[[VAL_10:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
-// CHECK:           %[[VAL_11:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(exit_release_or_enter_alloc) capture(ByRef) members(%[[VAL_3]], %[[VAL_8]] : [1, 0], [1, 1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
-// CHECK:           omp.target_data map_entries(%[[VAL_10]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
+// CHECK:           omp.target map_entries(%[[VAL_10]] -> %[[VAL_11:.*]] : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
 // CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           return

>From 7ab0ba7cc714ae7094cd7ffa9ea5c9766e75e248 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 19 Jun 2025 12:20:40 +0530
Subject: [PATCH 24/29] [flang-rt] Use omp_get_mapped_ptr to get device ptrs.

---
 flang-rt/lib/runtime/assign_omp.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/flang-rt/lib/runtime/assign_omp.cpp b/flang-rt/lib/runtime/assign_omp.cpp
index 80c1c5cccb1ca..a214afea11380 100644
--- a/flang-rt/lib/runtime/assign_omp.cpp
+++ b/flang-rt/lib/runtime/assign_omp.cpp
@@ -27,9 +27,7 @@ template <typename T> static T *getDevicePtr(T *anyPtr, OMPDeviceTy ompDevice) {
   // If not present on the device it should already be a device ptr
   if (!omp_target_is_present(voidAnyPtr, ompDevice))
     return anyPtr;
-  T *device_ptr = nullptr;
-#pragma omp target data use_device_ptr(anyPtr) device(ompDevice)
-  device_ptr = anyPtr;
+  T *device_ptr = omp_get_mapped_ptr(anyPtr, ompDevice);
   return device_ptr;
 }
 

>From a5adfcc0c513b8f780b1d750a19fc0d8d94ea6a6 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 23 Jun 2025 12:44:12 +0530
Subject: [PATCH 25/29] [OMP] Update OMP_Workdistribute directive definition.

---
 llvm/include/llvm/Frontend/OpenMP/OMP.td | 25 ++++++++----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index b6d92d572206a..73fba71120af1 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -685,16 +685,7 @@ def OMP_CancellationPoint : Directive<[Spelling<"cancellation point">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_Coexecute : Directive<"coexecute"> {
-  let association = AS_Block;
-  let category = CA_Executable;
-}
-def OMP_EndCoexecute : Directive<"end coexecute"> {
-  let leafConstructs = OMP_Coexecute.leafConstructs;
-  let association = OMP_Coexecute.association;
-  let category = OMP_Coexecute.category;
-}
-def OMP_Critical : Directive<"critical"> {
+def OMP_Critical : Directive<[Spelling<"critical">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Hint>,
   ];
@@ -1295,11 +1286,11 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
-def OMP_Workdistribute : Directive<"workdistribute"> {
+def OMP_Workdistribute : Directive<[Spelling<"workdistribute">]> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_EndWorkdistribute : Directive<"end workdistribute"> {
+def OMP_EndWorkdistribute : Directive<[Spelling<"end workdistribute">]> {
   let leafConstructs = OMP_Workdistribute.leafConstructs;
   let association = OMP_Workdistribute.association;
   let category = OMP_Workdistribute.category;
@@ -2224,7 +2215,7 @@ def OMP_TargetTeams : Directive<[Spelling<"target teams">]> {
   let leafConstructs = [OMP_Target, OMP_Teams];
   let category = CA_Executable;
 }
-def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> {
+def OMP_TargetTeamsDistribute : Directive<[Spelling<"target teams distribute">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -2446,7 +2437,7 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_TargetTeamsWorkdistribute : Directive<"target teams workdistribute"> {
+def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistribute">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -2474,7 +2465,7 @@ def OMP_TargetTeamsWorkdistribute : Directive<"target teams workdistribute"> {
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
   let category = CA_Executable;
 }
-def OMP_target_teams_loop : Directive<"target teams loop"> {
+def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_DefaultMap>,
@@ -2538,7 +2529,7 @@ def OMP_TaskLoopSimd : Directive<[Spelling<"taskloop simd">]> {
   let leafConstructs = [OMP_TaskLoop, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_TeamsDistribute : Directive<"teams distribute"> {
+def OMP_TeamsDistribute : Directive<[Spelling<"teams distribute">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -2725,7 +2716,7 @@ def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let leafConstructs = [OMP_Teams, OMP_loop];
   let category = CA_Executable;
 }
-def OMP_TeamsWorkdistribute : Directive<"teams workdistribute"> {
+def OMP_TeamsWorkdistribute : Directive<[Spelling<"teams workdistribute">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_FirstPrivate>,

>From 8669a400cf1b2ab65ea0cc80fee10fcdd24ab8ce Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 23 Jun 2025 14:20:28 +0530
Subject: [PATCH 26/29] [Flang] Update omp::TargetOp calls

---
 .../lib/Optimizer/OpenMP/LowerWorkdistribute.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index 3dfb574977036..1fe2592d1a357 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -434,8 +434,8 @@ std::optional<SplitTargetResult> splitTargetData(omp::TargetOp targetOp,
       targetOp.getInReductionVars(), targetOp.getInReductionByrefAttr(),
       targetOp.getInReductionSymsAttr(), targetOp.getIsDevicePtrVars(),
       innerMapInfos, targetOp.getNowaitAttr(), targetOp.getPrivateVars(),
-      targetOp.getPrivateSymsAttr(), targetOp.getThreadLimit(),
-      targetOp.getPrivateMapsAttr());
+      targetOp.getPrivateSymsAttr(), targetOp.getPrivateNeedsBarrierAttr(),
+      targetOp.getThreadLimit(), targetOp.getPrivateMapsAttr());
   rewriter.inlineRegionBefore(targetOp.getRegion(), newTargetOp.getRegion(),
                               newTargetOp.getRegion().begin());
 
@@ -653,8 +653,8 @@ static SplitResult isolateOp(Operation *splitBeforeOp, bool splitAfter,
         targetOp.getInReductionByrefAttr(), targetOp.getInReductionSymsAttr(),
         targetOp.getIsDevicePtrVars(), preMapOperands,
         targetOp.getNowaitAttr(), targetOp.getPrivateVars(),
-        targetOp.getPrivateSymsAttr(), targetOp.getThreadLimit(),
-        targetOp.getPrivateMapsAttr()); 
+        targetOp.getPrivateSymsAttr(), targetOp.getPrivateNeedsBarrierAttr(),
+        targetOp.getThreadLimit(), targetOp.getPrivateMapsAttr());
   auto *preTargetBlock = rewriter.createBlock(
       &preTargetOp.getRegion(), preTargetOp.getRegion().begin(), {}, {});
   IRMapping preMapping;
@@ -695,8 +695,8 @@ static SplitResult isolateOp(Operation *splitBeforeOp, bool splitAfter,
       targetOp.getInReductionByrefAttr(), targetOp.getInReductionSymsAttr(),
       targetOp.getIsDevicePtrVars(), postMapOperands,
       targetOp.getNowaitAttr(), targetOp.getPrivateVars(),
-      targetOp.getPrivateSymsAttr(), targetOp.getThreadLimit(),
-      targetOp.getPrivateMapsAttr()); 
+      targetOp.getPrivateSymsAttr(), targetOp.getPrivateNeedsBarrierAttr(),
+      targetOp.getThreadLimit(), targetOp.getPrivateMapsAttr());
 
   auto *isolatedTargetBlock =
         rewriter.createBlock(&isolatedTargetOp.getRegion(),
@@ -722,8 +722,8 @@ static SplitResult isolateOp(Operation *splitBeforeOp, bool splitAfter,
         targetOp.getInReductionByrefAttr(), targetOp.getInReductionSymsAttr(),
         targetOp.getIsDevicePtrVars(), postMapOperands,
         targetOp.getNowaitAttr(), targetOp.getPrivateVars(),
-        targetOp.getPrivateSymsAttr(), targetOp.getThreadLimit(),
-        targetOp.getPrivateMapsAttr()); 
+        targetOp.getPrivateSymsAttr(), targetOp.getPrivateNeedsBarrierAttr(),
+        targetOp.getThreadLimit(), targetOp.getPrivateMapsAttr());
     auto *postTargetBlock = rewriter.createBlock(
           &postTargetOp.getRegion(), postTargetOp.getRegion().begin(), {}, {});
     IRMapping postMapping;

>From 002610d18a1268b23f3fa1ac76147b36179457f5 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 27 Jun 2025 12:45:22 +0530
Subject: [PATCH 27/29] Remove frontend and pre-requisite chanes, as they are
 in seperate PRs now.

---
 flang-rt/lib/runtime/CMakeLists.txt           |   2 -
 flang-rt/lib/runtime/assign_omp.cpp           |  76 -------------
 .../include/flang/Optimizer/Dialect/FIROps.td |  61 -----------
 flang/include/flang/Runtime/assign.h          |   2 -
 .../flang/Semantics/openmp-directive-sets.h   |  14 ---
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  26 +----
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +-----------------
 flang/lib/Parser/openmp-parsers.cpp           |   7 +-
 flang/lib/Semantics/resolve-directives.cpp    |   6 --
 flang/test/Lower/OpenMP/workdistribute.f90    |  59 ----------
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  55 ----------
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  27 -----
 12 files changed, 3 insertions(+), 434 deletions(-)
 delete mode 100644 flang-rt/lib/runtime/assign_omp.cpp
 delete mode 100644 flang/test/Lower/OpenMP/workdistribute.f90

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 5200b2b710a5e..332c0872e065f 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -21,7 +21,6 @@ set(supported_sources
   allocatable.cpp
   array-constructor.cpp
   assign.cpp
-  assign_omp.cpp
   buffer.cpp
   character.cpp
   connection.cpp
@@ -100,7 +99,6 @@ set(gpu_sources
   allocatable.cpp
   array-constructor.cpp
   assign.cpp
-  assign_omp.cpp
   buffer.cpp
   character.cpp
   connection.cpp
diff --git a/flang-rt/lib/runtime/assign_omp.cpp b/flang-rt/lib/runtime/assign_omp.cpp
deleted file mode 100644
index a214afea11380..0000000000000
--- a/flang-rt/lib/runtime/assign_omp.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-//===-- lib/runtime/assign_omp.cpp ----------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "flang/Runtime/assign.h"
-#include "flang-rt/runtime/assign-impl.h"
-#include "flang-rt/runtime/derived.h"
-#include "flang-rt/runtime/descriptor.h"
-#include "flang-rt/runtime/stat.h"
-#include "flang-rt/runtime/terminator.h"
-#include "flang-rt/runtime/tools.h"
-#include "flang-rt/runtime/type-info.h"
-
-#include <omp.h>
-
-namespace Fortran::runtime {
-namespace omp {
-
-typedef int32_t OMPDeviceTy;
-
-template <typename T> static T *getDevicePtr(T *anyPtr, OMPDeviceTy ompDevice) {
-  auto voidAnyPtr = reinterpret_cast<void *>(anyPtr);
-  // If not present on the device it should already be a device ptr
-  if (!omp_target_is_present(voidAnyPtr, ompDevice))
-    return anyPtr;
-  T *device_ptr = omp_get_mapped_ptr(anyPtr, ompDevice);
-  return device_ptr;
-}
-
-RT_API_ATTRS static void Assign(Descriptor &to, const Descriptor &from,
-    Terminator &terminator, int flags, OMPDeviceTy omp_device) {
-  std::size_t toElementBytes{to.ElementBytes()};
-  std::size_t fromElementBytes{from.ElementBytes()};
-  std::size_t toElements{to.Elements()};
-  std::size_t fromElements{from.Elements()};
-
-  if (toElementBytes != fromElementBytes)
-    terminator.Crash("Assign: toElementBytes != fromElementBytes");
-  if (toElements != fromElements)
-    terminator.Crash("Assign: toElements != fromElements");
-
-  // Get base addresses and calculate length
-  void *to_base = to.raw().base_addr;
-  void *from_base = from.raw().base_addr;
-  size_t length = toElements * toElementBytes;
-
-  // Get device pointers after ensuring data is on device
-  void *to_ptr = getDevicePtr(to_base, omp_device);
-  void *from_ptr = getDevicePtr(from_base, omp_device);
-
-  // Perform copy between device pointers
-  int result = omp_target_memcpy(to_ptr, from_ptr, length,
-      /*dst_offset*/ 0, /*src_offset*/ 0, omp_device, omp_device);
-
-  if (result != 0)
-    terminator.Crash("Assign: omp_target_memcpy failed");
-  return;
-}
-
-extern "C" {
-RT_EXT_API_GROUP_BEGIN
-void RTDEF(Assign_omp)(Descriptor &to, const Descriptor &from,
-    const char *sourceFile, int sourceLine, omp::OMPDeviceTy omp_device) {
-  Terminator terminator{sourceFile, sourceLine};
-  Fortran::runtime::omp::Assign(to, from, terminator,
-      MaybeReallocate | NeedFinalization | ComponentCanBeDefinedAssignment,
-      omp_device);
-}
-
-} // extern "C"
-} // namespace omp
-} // namespace Fortran::runtime
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 466699cc4d476..8ac847dd7dd0a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,67 +517,6 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
-def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
-    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
-  let summary = "allocate storage on an openmp device for an object of a given type";
-
-  let description = [{
-    Creates a heap memory reference suitable for storing a value of the
-    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
-    The memory object is in an undefined state.  `allocmem` operations must
-    be paired with `freemem` operations to avoid memory leaks.
-
-    ```
-      %0 = fir.omp_target_allocmem !fir.array<10 x f32>
-    ```
-  }];
-
-  let arguments = (ins
-    Arg<AnyIntegerType>:$device,
-    TypeAttr:$in_type,
-    OptionalAttr<StrAttr>:$uniq_name,
-    OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyIntegerType>:$typeparams,
-    Variadic<AnyIntegerType>:$shape
-  );
-  let results = (outs fir_HeapType);
-
-  let extraClassDeclaration = [{
-    mlir::Type getAllocatedType();
-    bool hasLenParams() { return !getTypeparams().empty(); }
-    bool hasShapeOperands() { return !getShape().empty(); }
-    unsigned numLenParams() { return getTypeparams().size(); }
-    operand_range getLenParams() { return getTypeparams(); }
-    unsigned numShapeOperands() { return getShape().size(); }
-    operand_range getShapeOperands() { return getShape(); }
-    static mlir::Type getRefTy(mlir::Type ty);
-  }];
-}
-
-def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
-  [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object";
-
-  let description = [{
-    Deallocates a heap memory reference that was allocated by an `allocmem`.
-    The memory object that is deallocated is placed in an undefined state
-    after `fir.freemem`.  Optimizations may treat the loading of an object
-    in the undefined state as undefined behavior.  This includes aliasing
-    references, such as the result of an `fir.embox`.Add commentMore actions
-
-    ```
-      %21 = fir.omp_target_allocmem !fir.type<ZT(p:i32){field:i32}>
-      ...
-      fir.omp_target_freemem %21 : !fir.heap<!fir.type<ZT>>
-    ```
-  }];
-
-  let arguments = (ins
-  Arg<AnyIntegerType, "", [MemFree]>:$device,
-  Arg<fir_HeapType, "", [MemFree]>:$heapref
-  );
-}
-
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/include/flang/Runtime/assign.h b/flang/include/flang/Runtime/assign.h
index 0be52413e4814..7d198bdcc9e89 100644
--- a/flang/include/flang/Runtime/assign.h
+++ b/flang/include/flang/Runtime/assign.h
@@ -56,8 +56,6 @@ extern "C" {
 // API for lowering assignment
 void RTDECL(Assign)(Descriptor &to, const Descriptor &from,
     const char *sourceFile = nullptr, int sourceLine = 0);
-void RTDECL(Assign_omp)(Descriptor &to, const Descriptor &from,
-    const char *sourceFile = nullptr, int sourceLine = 0, int32_t omp_device = 0);
 // This variant has no finalization, defined assignment, or allocatable
 // reallocation.
 void RTDECL(AssignTemporary)(Descriptor &to, const Descriptor &from,
diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index 7ced6ed9b44d6..dd610c9702c28 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,7 +143,6 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
-    Directive::OMPD_target_teams_workdistribute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -173,7 +172,6 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
-    Directive::OMPD_teams_workdistribute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
@@ -189,16 +187,9 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
-        Directive::OMPD_target_teams_workdistribute,
     } | topTeamsSet,
 };
 
-static const OmpDirectiveSet allWorkdistributeSet{
-    Directive::OMPD_workdistribute,
-    Directive::OMPD_teams_workdistribute,
-    Directive::OMPD_target_teams_workdistribute,
-};
-
 //===----------------------------------------------------------------------===//
 // Directive sets for groups of multiple directives
 //===----------------------------------------------------------------------===//
@@ -239,9 +230,6 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
-    Directive::OMPD_target_teams_workdistribute,
-    Directive::OMPD_teams_workdistribute,
-    Directive::OMPD_workdistribute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -306,7 +294,6 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
-        Directive::OMPD_workdistribute,
     } | allDoSet,
 };
 
@@ -389,7 +376,6 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
-    Directive::OMPD_workdistribute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 1ae6366b3631a..ebd1d038716e4 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -565,16 +565,6 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
       });
       break;
 
-    case OMPD_teams_workdistribute:
-      cp.processThreadLimit(stmtCtx, hostInfo.ops);
-      [[fallthrough]];
-    case OMPD_target_teams_workdistribute:
-      cp.processNumTeams(stmtCtx, hostInfo.ops);
-      processSingleNestedIf([](Directive nestedDir) {
-        return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
-      });
-      break;
-
     case OMPD_teams_distribute:
     case OMPD_teams_distribute_simd:
       cp.processThreadLimit(stmtCtx, hostInfo.ops);
@@ -2692,17 +2682,6 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
-static mlir::omp::WorkdistributeOp genWorkdistributeOp(
-    lower::AbstractConverter &converter, lower::SymMap &symTable,
-    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
-    mlir::Location loc, const ConstructQueue &queue,
-    ConstructQueue::const_iterator item) {
-  return genOpWithBody<mlir::omp::WorkdistributeOp>(
-      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_workdistribute),
-      queue, item);
-}
-
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3323,10 +3302,7 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     TODO(loc, "Unhandled loop directive (" +
                   llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
   }
-  case llvm::omp::Directive::OMPD_workdistribute:
-    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue,
-                                item);
-    break;
+  // case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
                            queue, item);
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index cf4ca5f1436b5..a3de3ae9d116a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,105 +1168,6 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetAllocMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::Type heapTy = heap.getType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
-    mlir::Location loc = heap.getLoc();
-    auto ity = lowerTy().indexType();
-    mlir::Type dataTy = fir::unwrapRefType(heapTy);
-    mlir::Type llvmObjectTy = convertObjectType(dataTy);
-    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
-      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
-                "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
-      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands().drop_front())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
-    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
-    auto mallocTy =
-        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
-    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
-      size = integerCast(loc, rewriter, mallocTy, size);
-    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        heap, ::getLlvmPtrType(heap.getContext()),
-        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
-    return mlir::success();
-  }
-
-  /// Compute the allocation size in bytes of the element type of
-  /// \p llTy pointer type. The result is returned as a value of \p idxTy
-  /// integer type.
-  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
-  }
-};
-} // namespace
-
-static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp freeFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
-    return freeFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_free",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMVoidType::get(module->getContext()),
-          {getLlvmPtrType(module->getContext()), i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetFreeMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
-    mlir::Location loc = freemem.getLoc();
-    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
-    rewriter.create<mlir::LLVM::CallOp>(
-        loc, mlir::TypeRange{},
-        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
-        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
-    rewriter.eraseOp(freemem);
-    return mlir::success();
-  }
-};
-} // namespace
-
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4373,8 +4274,7 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
-      OmpTargetFreeMemOpConversion,SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index ad729932a5f00..c55642d969503 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1492,17 +1492,12 @@ TYPE_PARSER(
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
-        "TARGET TEAMS WORKDISTRIBUTE" >>
-            pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
-        "TEAMS WORKDISTRIBUTE" >>
-            pure(llvm::omp::Directive::OMPD_teams_workdistribute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
-        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
-        "WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_workdistribute))))
+        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index da3315ff1acfb..885c02e6ec74b 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1656,9 +1656,6 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
-  case llvm::omp::Directive::OMPD_workdistribute:
-  case llvm::omp::Directive::OMPD_teams_workdistribute:
-  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
@@ -1692,9 +1689,6 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
-  case llvm::omp::Directive::OMPD_workdistribute:
-  case llvm::omp::Directive::OMPD_teams_workdistribute:
-  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
   case llvm::omp::Directive::OMPD_target_parallel: {
diff --git a/flang/test/Lower/OpenMP/workdistribute.f90 b/flang/test/Lower/OpenMP/workdistribute.f90
deleted file mode 100644
index 924205bb72e5e..0000000000000
--- a/flang/test/Lower/OpenMP/workdistribute.f90
+++ /dev/null
@@ -1,59 +0,0 @@
-! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
-
-! CHECK-LABEL: func @_QPtarget_teams_workdistribute
-subroutine target_teams_workdistribute()
-  ! CHECK: omp.target
-  ! CHECK: omp.teams
-  ! CHECK: omp.workdistribute
-  !$omp target teams workdistribute
-  ! CHECK: fir.call
-  call f1()
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  !$omp end target teams workdistribute
-end subroutine target_teams_workdistribute
-
-! CHECK-LABEL: func @_QPteams_workdistribute
-subroutine teams_workdistribute()
-  ! CHECK: omp.teams
-  ! CHECK: omp.workdistribute
-  !$omp teams workdistribute
-  ! CHECK: fir.call
-  call f1()
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  !$omp end teams workdistribute
-end subroutine teams_workdistribute
-
-! CHECK-LABEL: func @_QPtarget_teams_workdistribute_m
-subroutine target_teams_workdistribute_m()
-  ! CHECK: omp.target
-  ! CHECK: omp.teams
-  ! CHECK: omp.workdistribute
-  !$omp target
-  !$omp teams
-  !$omp workdistribute
-  ! CHECK: fir.call
-  call f1()
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  !$omp end workdistribute
-  !$omp end teams
-  !$omp end target
-end subroutine target_teams_workdistribute_m
-
-! CHECK-LABEL: func @_QPteams_workdistribute_m
-subroutine teams_workdistribute_m()
-  ! CHECK: omp.teams
-  ! CHECK: omp.workdistribute
-  !$omp teams
-  !$omp workdistribute
-  ! CHECK: fir.call
-  call f1()
-  ! CHECK: omp.terminator
-  ! CHECK: omp.terminator
-  !$omp end workdistribute
-  !$omp end teams
-end subroutine teams_workdistribute_m
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 73fba71120af1..c13215a1beb4d 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -1286,15 +1286,6 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
-def OMP_Workdistribute : Directive<[Spelling<"workdistribute">]> {
-  let association = AS_Block;
-  let category = CA_Executable;
-}
-def OMP_EndWorkdistribute : Directive<[Spelling<"end workdistribute">]> {
-  let leafConstructs = OMP_Workdistribute.leafConstructs;
-  let association = OMP_Workdistribute.association;
-  let category = OMP_Workdistribute.category;
-}
 
 //===----------------------------------------------------------------------===//
 // Definitions of OpenMP compound directives
@@ -2437,34 +2428,6 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistribute">]> {
-  let allowedClauses = [
-    VersionedClause<OMPC_Allocate>,
-    VersionedClause<OMPC_Depend>,
-    VersionedClause<OMPC_FirstPrivate>,
-    VersionedClause<OMPC_HasDeviceAddr, 51>,
-    VersionedClause<OMPC_If>,
-    VersionedClause<OMPC_IsDevicePtr>,
-    VersionedClause<OMPC_Map>,
-    VersionedClause<OMPC_OMPX_Attribute>,
-    VersionedClause<OMPC_Private>,
-    VersionedClause<OMPC_Reduction>,
-    VersionedClause<OMPC_Shared>,
-    VersionedClause<OMPC_UsesAllocators, 50>,
-  ];
-  let allowedOnceClauses = [
-    VersionedClause<OMPC_Default>,
-    VersionedClause<OMPC_DefaultMap>,
-    VersionedClause<OMPC_Device>,
-    VersionedClause<OMPC_NoWait>,
-    VersionedClause<OMPC_NumTeams>,
-    VersionedClause<OMPC_OMPX_DynCGroupMem>,
-    VersionedClause<OMPC_OMPX_Bare>,
-    VersionedClause<OMPC_ThreadLimit>,
-  ];
-  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
-  let category = CA_Executable;
-}
 def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
@@ -2716,21 +2679,3 @@ def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let leafConstructs = [OMP_Teams, OMP_loop];
   let category = CA_Executable;
 }
-def OMP_TeamsWorkdistribute : Directive<[Spelling<"teams workdistribute">]> {
-  let allowedClauses = [
-    VersionedClause<OMPC_Allocate>,
-    VersionedClause<OMPC_FirstPrivate>,
-    VersionedClause<OMPC_OMPX_Attribute>,
-    VersionedClause<OMPC_Private>,
-    VersionedClause<OMPC_Reduction>,
-    VersionedClause<OMPC_Shared>,
-  ];
-  let allowedOnceClauses = [
-    VersionedClause<OMPC_Default>,
-    VersionedClause<OMPC_If, 52>,
-    VersionedClause<OMPC_NumTeams>,
-    VersionedClause<OMPC_ThreadLimit>,
-  ];
-  let leafConstructs = [OMP_Teams, OMP_Workdistribute];
-  let category = CA_Executable;
-}
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 8d65b37330eb8..ac80926053a2d 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -325,33 +325,6 @@ def SectionsOp : OpenMP_Op<"sections", traits = [
   let hasRegionVerifier = 1;
 }
 
-//===----------------------------------------------------------------------===//
-// workdistribute Construct
-//===----------------------------------------------------------------------===//
-
-def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
-  let summary = "workdistribute directive";
-  let description = [{
-    workdistribute divides execution of the enclosed structured block into
-    separate units of work, each executed only once by each
-    initial thread in the league.
-
-    ```
-    !$omp target teams
-        !$omp workdistribute
-                tmp = matmul(x, y)
-        !$omp end workdistribute
-        a = tmp(0, 0) ! there is no implicit barrier! the matmul hasnt completed!
-    !$omp end target teams workdistribute
-    ```
-
-  }];
-
-  let regions = (region AnyRegion:$region);
-
-  let assemblyFormat = "$region attr-dict";
-}
-
 //===----------------------------------------------------------------------===//
 // 2.8.2 Single Construct
 //===----------------------------------------------------------------------===//

>From cd3ed43a9a2755ed8cda96f69641bb3293c40749 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 27 Jun 2025 12:47:41 +0530
Subject: [PATCH 28/29] Remove older changes

---
 llvm/include/llvm/Frontend/OpenMP/OMP.td | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index c13215a1beb4d..a87111cb5a11d 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -2206,7 +2206,8 @@ def OMP_TargetTeams : Directive<[Spelling<"target teams">]> {
   let leafConstructs = [OMP_Target, OMP_Teams];
   let category = CA_Executable;
 }
-def OMP_TargetTeamsDistribute : Directive<[Spelling<"target teams distribute">]> {
+def OMP_TargetTeamsDistribute
+    : Directive<[Spelling<"target teams distribute">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,

>From b6ca26a145c6e445599bdcfcd0de10f2e122c12a Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 4 Jul 2025 14:33:46 +0530
Subject: [PATCH 29/29] [Flang] Remove rewrite patterns in
 lower-workdistribute.

---
 .../Optimizer/OpenMP/LowerWorkdistribute.cpp  | 302 ++++++++----------
 .../OpenMP/lower-workdistribute-doloop.mlir   |   2 +-
 2 files changed, 141 insertions(+), 163 deletions(-)

diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
index 1fe2592d1a357..3f4116d524452 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkdistribute.cpp
@@ -124,99 +124,96 @@ static T getPerfectlyNested(Operation *op) {
 /// }
 /// E()
 
-struct FissionWorkdistribute : public OpRewritePattern<omp::WorkdistributeOp> {
-  static bool fissionWorkdistributePatternMatched;
-  using OpRewritePattern::OpRewritePattern;
-  LogicalResult matchAndRewrite(omp::WorkdistributeOp workdistribute,
-                                PatternRewriter &rewriter) const override {
-    auto loc = workdistribute->getLoc();
-    auto teams = dyn_cast<omp::TeamsOp>(workdistribute->getParentOp());
-    if (!teams) {
-      emitError(loc, "workdistribute not nested in teams\n");
-      return failure();
-    }
-    if (workdistribute.getRegion().getBlocks().size() != 1) {
-      emitError(loc, "workdistribute with multiple blocks\n");
-      return failure();
+static bool FissionWorkdistribute(omp::WorkdistributeOp workdistribute) {
+  OpBuilder rewriter(workdistribute);
+  auto loc = workdistribute->getLoc();
+  auto teams = dyn_cast<omp::TeamsOp>(workdistribute->getParentOp());
+  if (!teams) {
+    emitError(loc, "workdistribute not nested in teams\n");
+    return false;
+  }
+  if (workdistribute.getRegion().getBlocks().size() != 1) {
+    emitError(loc, "workdistribute with multiple blocks\n");
+    return false;
+  }
+  if (teams.getRegion().getBlocks().size() != 1) {
+    emitError(loc, "teams with multiple blocks\n");
+    return false;
+  }
+
+  auto *teamsBlock = &teams.getRegion().front();
+  bool changed = false;
+  // Move the ops inside teams and before workdistribute outside.
+  IRMapping irMapping;
+  llvm::SmallVector<Operation *> teamsHoisted;
+  for (auto &op : teams.getOps()) {
+    if (&op == workdistribute) {
+      break;
     }
-    if (teams.getRegion().getBlocks().size() != 1) {
-      emitError(loc, "teams with multiple blocks\n");
-      return failure();
+    if (shouldParallelize(&op)) {
+      emitError(loc, "teams has parallelize ops before first workdistribute\n");
+      return false;
+    } else {
+      rewriter.setInsertionPoint(teams);
+      rewriter.clone(op, irMapping);
+      teamsHoisted.push_back(&op);
+      changed = true;
     }
-
-    auto *teamsBlock = &teams.getRegion().front();
-    bool changed = false;
-    // Move the ops inside teams and before workdistribute outside.
-    IRMapping irMapping;
-    llvm::SmallVector<Operation *> teamsHoisted;
-    for (auto &op : teams.getOps()) {
-      if (&op == workdistribute) {
+  }
+  for (auto *op : llvm::reverse(teamsHoisted)) {
+    op->replaceAllUsesWith(irMapping.lookup(op));
+    op->erase();
+  }
+
+  // While we have unhandled operations in the original workdistribute
+  auto *workdistributeBlock = &workdistribute.getRegion().front();
+  auto *terminator = workdistributeBlock->getTerminator();
+  while (&workdistributeBlock->front() != terminator) {
+    rewriter.setInsertionPoint(teams);
+    IRMapping mapping;
+    llvm::SmallVector<Operation *> hoisted;
+    Operation *parallelize = nullptr;
+    for (auto &op : workdistribute.getOps()) {
+      if (&op == terminator) {
         break;
       }
       if (shouldParallelize(&op)) {
-        emitError(loc,
-                  "teams has parallelize ops before first workdistribute\n");
-        return failure();
+        parallelize = &op;
+        break;
       } else {
-        rewriter.setInsertionPoint(teams);
-        rewriter.clone(op, irMapping);
-        teamsHoisted.push_back(&op);
+        rewriter.clone(op, mapping);
+        hoisted.push_back(&op);
         changed = true;
       }
     }
-    for (auto *op : teamsHoisted)
-      rewriter.replaceOp(op, irMapping.lookup(op));
-
-    // While we have unhandled operations in the original workdistribute
-    auto *workdistributeBlock = &workdistribute.getRegion().front();
-    auto *terminator = workdistributeBlock->getTerminator();
-    while (&workdistributeBlock->front() != terminator) {
-      rewriter.setInsertionPoint(teams);
-      IRMapping mapping;
-      llvm::SmallVector<Operation *> hoisted;
-      Operation *parallelize = nullptr;
-      for (auto &op : workdistribute.getOps()) {
-        if (&op == terminator) {
-          break;
-        }
-        if (shouldParallelize(&op)) {
-          parallelize = &op;
-          break;
-        } else {
-          rewriter.clone(op, mapping);
-          hoisted.push_back(&op);
-          changed = true;
-        }
-      }
 
-      for (auto *op : hoisted)
-        rewriter.replaceOp(op, mapping.lookup(op));
+    for (auto *op : llvm::reverse(hoisted)) {
+      op->replaceAllUsesWith(mapping.lookup(op));
+      op->erase();
+    }
 
-      if (parallelize && hoisted.empty() &&
-          parallelize->getNextNode() == terminator)
-        break;
-      if (parallelize) {
-        auto newTeams = rewriter.cloneWithoutRegions(teams);
-        auto *newTeamsBlock = rewriter.createBlock(
-            &newTeams.getRegion(), newTeams.getRegion().begin(), {}, {});
-        for (auto arg : teamsBlock->getArguments())
-          newTeamsBlock->addArgument(arg.getType(), arg.getLoc());
-        auto newWorkdistribute = rewriter.create<omp::WorkdistributeOp>(loc);
-        rewriter.create<omp::TerminatorOp>(loc);
-        rewriter.createBlock(&newWorkdistribute.getRegion(),
-                             newWorkdistribute.getRegion().begin(), {}, {});
-        auto *cloned = rewriter.clone(*parallelize);
-        rewriter.replaceOp(parallelize, cloned);
-        rewriter.create<omp::TerminatorOp>(loc);
-        changed = true;
-      }
+    if (parallelize && hoisted.empty() &&
+        parallelize->getNextNode() == terminator)
+      break;
+    if (parallelize) {
+      auto newTeams = rewriter.cloneWithoutRegions(teams);
+      auto *newTeamsBlock = rewriter.createBlock(
+          &newTeams.getRegion(), newTeams.getRegion().begin(), {}, {});
+      for (auto arg : teamsBlock->getArguments())
+        newTeamsBlock->addArgument(arg.getType(), arg.getLoc());
+      auto newWorkdistribute = rewriter.create<omp::WorkdistributeOp>(loc);
+      rewriter.create<omp::TerminatorOp>(loc);
+      rewriter.createBlock(&newWorkdistribute.getRegion(),
+                           newWorkdistribute.getRegion().begin(), {}, {});
+      auto *cloned = rewriter.clone(*parallelize);
+      parallelize->replaceAllUsesWith(cloned);
+      parallelize->erase();
+      rewriter.create<omp::TerminatorOp>(loc);
+      changed = true;
     }
-    if (changed)
-      fissionWorkdistributePatternMatched = true;
-    return success(changed);
   }
-};
-bool FissionWorkdistribute::fissionWorkdistributePatternMatched = false;
+  return changed;
+}
 
 /// If fir.do_loop is present inside teams workdistribute
 ///
@@ -241,8 +238,7 @@ bool FissionWorkdistribute::fissionWorkdistributePatternMatched = false;
 ///   }
 /// }
 
-static void genParallelOp(Location loc, PatternRewriter &rewriter,
-                          bool composite) {
+static void genParallelOp(Location loc, OpBuilder &rewriter, bool composite) {
   auto parallelOp = rewriter.create<mlir::omp::ParallelOp>(loc);
   parallelOp.setComposite(composite);
   rewriter.createBlock(&parallelOp.getRegion());
@@ -250,8 +246,7 @@ static void genParallelOp(Location loc, PatternRewriter &rewriter,
   return;
 }
 
-static void genDistributeOp(Location loc, PatternRewriter &rewriter,
-                            bool composite) {
+static void genDistributeOp(Location loc, OpBuilder &rewriter, bool composite) {
   mlir::omp::DistributeOperands distributeClauseOps;
   auto distributeOp =
       rewriter.create<mlir::omp::DistributeOp>(loc, distributeClauseOps);
@@ -262,7 +257,7 @@ static void genDistributeOp(Location loc, PatternRewriter &rewriter,
 }
 
 static void
-genLoopNestClauseOps(mlir::PatternRewriter &rewriter, fir::DoLoopOp loop,
+genLoopNestClauseOps(OpBuilder &rewriter, fir::DoLoopOp loop,
                      mlir::omp::LoopNestOperands &loopNestClauseOps) {
   assert(loopNestClauseOps.loopLowerBounds.empty() &&
          "Loop nest bounds were already emitted!");
@@ -272,7 +267,7 @@ genLoopNestClauseOps(mlir::PatternRewriter &rewriter, fir::DoLoopOp loop,
   loopNestClauseOps.loopInclusive = rewriter.getUnitAttr();
 }
 
-static void genWsLoopOp(mlir::PatternRewriter &rewriter, fir::DoLoopOp doLoop,
+static void genWsLoopOp(mlir::OpBuilder &rewriter, fir::DoLoopOp doLoop,
                         const mlir::omp::LoopNestOperands &clauseOps,
                         bool composite) {
 
@@ -294,34 +289,28 @@ static void genWsLoopOp(mlir::PatternRewriter &rewriter, fir::DoLoopOp doLoop,
   if (auto resultOp = dyn_cast<fir::ResultOp>(terminatorOp)) {
     rewriter.setInsertionPoint(terminatorOp);
     rewriter.create<mlir::omp::YieldOp>(doLoop->getLoc());
-    rewriter.eraseOp(terminatorOp);
+    // rewriter.erase(terminatorOp);
+    terminatorOp->erase();
   }
   return;
 }
 
-struct WorkdistributeDoLower : public OpRewritePattern<omp::WorkdistributeOp> {
-  static bool workdistributeDoLowerPatternMatched;
-  using OpRewritePattern::OpRewritePattern;
-  LogicalResult matchAndRewrite(omp::WorkdistributeOp workdistribute,
-                                PatternRewriter &rewriter) const override {
-    auto doLoop = getPerfectlyNested<fir::DoLoopOp>(workdistribute);
-    auto wdLoc = workdistribute->getLoc();
-    if (doLoop && shouldParallelize(doLoop)) {
-      assert(doLoop.getReduceOperands().empty());
-      genParallelOp(wdLoc, rewriter, true);
-      genDistributeOp(wdLoc, rewriter, true);
-      mlir::omp::LoopNestOperands loopNestClauseOps;
-      genLoopNestClauseOps(rewriter, doLoop, loopNestClauseOps);
-      genWsLoopOp(rewriter, doLoop, loopNestClauseOps, true);
-      rewriter.eraseOp(workdistribute);
-      workdistributeDoLowerPatternMatched = true;
-      return success();
-    }
-    return failure();
+static bool WorkdistributeDoLower(omp::WorkdistributeOp workdistribute) {
+  OpBuilder rewriter(workdistribute);
+  auto doLoop = getPerfectlyNested<fir::DoLoopOp>(workdistribute);
+  auto wdLoc = workdistribute->getLoc();
+  if (doLoop && shouldParallelize(doLoop)) {
+    assert(doLoop.getReduceOperands().empty());
+    genParallelOp(wdLoc, rewriter, true);
+    genDistributeOp(wdLoc, rewriter, true);
+    mlir::omp::LoopNestOperands loopNestClauseOps;
+    genLoopNestClauseOps(rewriter, doLoop, loopNestClauseOps);
+    genWsLoopOp(rewriter, doLoop, loopNestClauseOps, true);
+    workdistribute.erase();
+    return true;
   }
-};
-
-bool WorkdistributeDoLower::workdistributeDoLowerPatternMatched = false;
+  return false;
+}
 
 /// If A() and B () are present inside teams workdistribute
 ///
@@ -338,34 +327,39 @@ bool WorkdistributeDoLower::workdistributeDoLowerPatternMatched = false;
 /// B()
 ///
 
-struct TeamsWorkdistributeToSingle : public OpRewritePattern<omp::TeamsOp> {
-  static bool teamsWorkdistributeToSinglePatternMatched;
-  using OpRewritePattern::OpRewritePattern;
-  LogicalResult matchAndRewrite(omp::TeamsOp teamsOp,
-                                PatternRewriter &rewriter) const override {
-    auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
-    if (!workdistributeOp) {
-      LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << " No workdistribute nested\n");
-      return failure();
-    }
-    Block *workdistributeBlock = &workdistributeOp.getRegion().front();
-    rewriter.eraseOp(workdistributeBlock->getTerminator());
-    rewriter.inlineBlockBefore(workdistributeBlock, teamsOp);
-    rewriter.eraseOp(workdistributeOp);
-    teamsWorkdistributeToSinglePatternMatched = true;
-    return success();
-  }
-};
-bool TeamsWorkdistributeToSingle::teamsWorkdistributeToSinglePatternMatched =
-    false;
+static bool TeamsWorkdistributeToSingleOp(omp::TeamsOp teamsOp) {
+  auto workdistributeOp = getPerfectlyNested<omp::WorkdistributeOp>(teamsOp);
+  if (!workdistributeOp)
+    return false;
+  // Get the block containing teamsOp (the parent block).
+  Block *parentBlock = teamsOp->getBlock();
+  Block &workdistributeBlock = *workdistributeOp.getRegion().begin();
+  auto insertPoint = Block::iterator(teamsOp);
+  // Get the range of operations to move (excluding the terminator).
+  auto workdistributeBegin = workdistributeBlock.begin();
+  auto workdistributeEnd = workdistributeBlock.getTerminator()->getIterator();
+  // Move the operations from workdistribute block to before teamsOp.
+  parentBlock->getOperations().splice(insertPoint,
+                                      workdistributeBlock.getOperations(),
+                                      workdistributeBegin, workdistributeEnd);
+  // Erase the now-empty workdistributeOp.
+  workdistributeOp.erase();
+  Block &teamsBlock = *teamsOp.getRegion().begin();
+  // Check if only the terminator remains and erase teams op.
+  if (teamsBlock.getOperations().size() == 1 &&
+      teamsBlock.getTerminator() != nullptr) {
+    teamsOp.erase();
+  }
+  return true;
+}
 
 struct SplitTargetResult {
   omp::TargetOp targetOp;
   omp::TargetDataOp dataOp;
 };
 
-/// If multiple coexecutes are nested in a target regions, we will need to split
-/// the target region, but we want to preserve the data semantics of the
+/// If multiple workdistribute are nested in a target regions, we will need to
+/// split the target region, but we want to preserve the data semantics of the
 /// original data region and avoid unnecessary data movement at each of the
 /// subkernels - we split the target region into a target_data{target}
 /// nest where only the outer one moves the data
@@ -877,38 +871,22 @@ class LowerWorkdistributePass
 public:
   void runOnOperation() override {
     MLIRContext &context = getContext();
-    GreedyRewriteConfig config;
-    // prevent the pattern driver form merging blocks
-    config.setRegionSimplificationLevel(GreedySimplifyRegionLevel::Disabled);
-
-    Operation *op = getOperation();
-    bool anyPatternChanged = false;
-    {
-      RewritePatternSet patterns(&context);
-      patterns.insert<FissionWorkdistribute, WorkdistributeDoLower>(&context);
-      if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
-        emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
-        signalPassFailure();
-      }
-      anyPatternChanged |=
-          FissionWorkdistribute::fissionWorkdistributePatternMatched;
-      anyPatternChanged |=
-          WorkdistributeDoLower::workdistributeDoLowerPatternMatched;
-    }
-    {
-      RewritePatternSet patterns(&context);
-      patterns.insert<WorkdistributeDoLower, TeamsWorkdistributeToSingle>(
-          &context);
-      if (failed(applyPatternsGreedily(op, std::move(patterns), config))) {
-        emitError(op->getLoc(), DEBUG_TYPE " pass failed\n");
-        signalPassFailure();
-      }
-      anyPatternChanged |= TeamsWorkdistributeToSingle::
-          teamsWorkdistributeToSinglePatternMatched;
-    }
-    if (anyPatternChanged) {
+    auto moduleOp = getOperation();
+    bool changed = false;
+    moduleOp->walk([&](mlir::omp::WorkdistributeOp workdistribute) {
+      changed |= FissionWorkdistribute(workdistribute);
+    });
+    moduleOp->walk([&](mlir::omp::WorkdistributeOp workdistribute) {
+      changed |= WorkdistributeDoLower(workdistribute);
+    });
+    moduleOp->walk([&](mlir::omp::TeamsOp teams) {
+      changed |= TeamsWorkdistributeToSingleOp(teams);
+    });
+
+    if (changed) {
       SmallVector<omp::TargetOp> targetOps;
-      op->walk([&](omp::TargetOp targetOp) { targetOps.push_back(targetOp); });
+      moduleOp->walk(
+          [&](omp::TargetOp targetOp) { targetOps.push_back(targetOp); });
       IRRewriter rewriter(&context);
       for (auto targetOp : targetOps) {
         auto res = splitTargetData(targetOp, rewriter);
diff --git a/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir b/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
index f8351bb64e6e8..00d10d6264ec9 100644
--- a/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workdistribute-doloop.mlir
@@ -1,12 +1,12 @@
 // RUN: fir-opt --lower-workdistribute %s | FileCheck %s
 
 // CHECK-LABEL:   func.func @x({{.*}})
-// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : index
 // CHECK:           omp.teams {
 // CHECK:             omp.parallel {
 // CHECK:               omp.distribute {
 // CHECK:                 omp.wsloop {
 // CHECK:                   omp.loop_nest (%[[VAL_1:.*]]) : index = (%[[ARG0:.*]]) to (%[[ARG1:.*]]) inclusive step (%[[ARG2:.*]]) {
+// CHECK:                     %[[VAL_0:.*]] = arith.constant 0 : index
 // CHECK:                     fir.store %[[VAL_0]] to %[[ARG4:.*]] : !fir.ref<index>
 // CHECK:                     omp.yield
 // CHECK:                   }


From flang-commits at lists.llvm.org  Fri Jul  4 04:49:36 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Fri, 04 Jul 2025 04:49:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <6867bfd0.170a0220.148c26.4a2e@mx.google.com>


luporl wrote:

Thanks for the review!

https://github.com/llvm/llvm-project/pull/146408

From flang-commits at lists.llvm.org  Fri Jul  4 04:49:58 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Fri, 04 Jul 2025 04:49:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <6867bfe6.170a0220.12d250.4d92@mx.google.com>


https://github.com/luporl closed https://github.com/llvm/llvm-project/pull/146408

From flang-commits at lists.llvm.org  Fri Jul  4 06:50:27 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 04 Jul 2025 06:50:27 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix tests broken by #146734 (PR
	#147055)
Message-ID: <llvm.org/llvm/llvm-project/pull/147055@github.com>


https://github.com/tblah created https://github.com/llvm/llvm-project/pull/147055

These tests referred to privatizers which were never declared

>From a115dff36c45bf7acbf3be2b1fb321e87bafef2f Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Fri, 4 Jul 2025 13:48:47 +0000
Subject: [PATCH] [flang] Fix tests broken by #146734

These tests referred to privatizers which were never declared
---
 flang/test/Fir/convert-nontemporal-to-llvm.fir | 18 +++++++++---------
 flang/test/Fir/simd-nontemporal.fir            | 12 ++++++------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/flang/test/Fir/convert-nontemporal-to-llvm.fir b/flang/test/Fir/convert-nontemporal-to-llvm.fir
index 6200ef1c621d7..e0a836ab0dad1 100644
--- a/flang/test/Fir/convert-nontemporal-to-llvm.fir
+++ b/flang/test/Fir/convert-nontemporal-to-llvm.fir
@@ -14,7 +14,7 @@
 // CHECK:    %[[VAL5:.*]] = llvm.alloca %[[CONST_VAL4]] x i32 {bindc_name = "a"} : (i64) -> !llvm.ptr
 // CHECK:    %[[CONST_VAL5:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:    %[[VAL6:.*]] = llvm.load %[[VAL1]] : !llvm.ptr -> i32
-// CHECK:    omp.simd nontemporal(%[[VAL5]], %[[VAL3]] : !llvm.ptr, !llvm.ptr) private(@_QFtestEi_private_i32 %[[VAL2]] -> %arg0 : !llvm.ptr) {
+// CHECK:    omp.simd nontemporal(%[[VAL5]], %[[VAL3]] : !llvm.ptr, !llvm.ptr) {
 // CHECK:      omp.loop_nest (%{{.*}}) : i32 = (%[[CONST_VAL5]]) to (%[[VAL6]]) inclusive step (%[[CONST_VAL5]]) {
 // CHECK:        llvm.store %{{.*}}, %{{.*}} : i32, !llvm.ptr
 // CHECK:        %[[VAL8:.*]] = llvm.load %[[VAL5]] {nontemporal} : !llvm.ptr -> i32
@@ -33,9 +33,9 @@
     %3 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtestEi"}
     %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFtestEn"}
     %5 = fir.load %4 : !fir.ref<i32>
-    omp.simd nontemporal(%0, %2 : !fir.ref<i32>, !fir.ref<i32>) private(@_QFtestEi_private_i32 %3 -> %arg0 : !fir.ref<i32>) {
+    omp.simd nontemporal(%0, %2 : !fir.ref<i32>, !fir.ref<i32>) {
       omp.loop_nest (%arg1) : i32 = (%c1_i32) to (%5) inclusive step (%c1_i32) {
-        fir.store %arg1 to %arg0 : !fir.ref<i32>
+        fir.store %arg1 to %3 : !fir.ref<i32>
         %6 = fir.load %0 {nontemporal}: !fir.ref<i32>
         %7 = fir.load %1 : !fir.ref<i32>
         %8 = arith.addi %6, %7 : i32
@@ -52,12 +52,12 @@
 // CHECK:    %[[IDX_VAL:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:    %[[CONST_VAL1:.*]] = llvm.mlir.constant(0 : index) : i64
 // CHECK:    %[[END_IDX:.*]] = llvm.mlir.constant(100 : i32) : i32
-// CHECK:    omp.simd nontemporal(%[[ARG0:.*]] : !llvm.ptr) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %[[ALLOCA2]] -> %[[ARG2:.*]] : !llvm.ptr) {
+// CHECK:    omp.simd nontemporal(%[[ARG0:.*]] : !llvm.ptr) {
 // CHECK:      omp.loop_nest (%[[ARG3:.*]]) : i32 = (%[[IDX_VAL]]) to (%[[END_IDX]]) inclusive step (%[[IDX_VAL]]) {
-// CHECK:        llvm.store %[[ARG3]], %[[ARG2]] : i32, !llvm.ptr
+// CHECK:        llvm.store %[[ARG3]], %{{.*}} : i32, !llvm.ptr
 // CHECK:        %[[CONST_VAL2:.*]] = llvm.mlir.constant(48 : i32) : i32
 // CHECK:        "llvm.intr.memcpy"(%[[ALLOCA1:.*]], %[[ARG0]], %[[CONST_VAL2]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
-// CHECK:        %[[VAL1:.*]] = llvm.load %[[ARG2]] : !llvm.ptr -> i32
+// CHECK:        %[[VAL1:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32
 // CHECK:        %[[VAL2:.*]] = llvm.sext %[[VAL1]] : i32 to i64
 // CHECK:        %[[VAL3:.*]] = llvm.getelementptr %[[ALLOCA1]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
 // CHECK:        %[[VAL4:.*]] = llvm.load %[[VAL3]] : !llvm.ptr -> !llvm.ptr
@@ -91,11 +91,11 @@
     %1 = fir.allocmem !fir.array<?xi32>, %c100 {fir.must_be_heap = true, uniq_name = "_QFsimd_nontemporal_allocatableEx.alloc"}
     %2 = fircg.ext_embox %1(%c100) : (!fir.heap<!fir.array<?xi32>>, index) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
     fir.store %2 to %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    omp.simd nontemporal(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %0 -> %arg2 : !fir.ref<i32>) {
+    omp.simd nontemporal(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
       omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
-        fir.store %arg3 to %arg2 : !fir.ref<i32>
+        fir.store %arg3 to %0 : !fir.ref<i32>
         %7 = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-        %8 = fir.load %arg2 : !fir.ref<i32>
+        %8 = fir.load %0 : !fir.ref<i32>
         %9 = fir.convert %8 : (i32) -> i64
         %10 = fir.box_addr %7 : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
         %11:3 = fir.box_dims %7, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
diff --git a/flang/test/Fir/simd-nontemporal.fir b/flang/test/Fir/simd-nontemporal.fir
index 31051ff52f9bd..a58bf01a60980 100644
--- a/flang/test/Fir/simd-nontemporal.fir
+++ b/flang/test/Fir/simd-nontemporal.fir
@@ -17,11 +17,11 @@ func.func @_QPsimd_with_nontemporal_clause(%arg0: !fir.ref<i32> {fir.bindc_name
     %8 = fir.declare %7 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
     %9 = fir.declare %arg0 dummy_scope %0 {uniq_name = "_QFsimd_with_nontemporal_clauseEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
     %10 = fir.load %9 : !fir.ref<i32>
-    // CHECK: omp.simd nontemporal(%[[A_DECL]], %[[C_DECL]] : !fir.ref<i32>, !fir.ref<i32>) private(@_QFsimd_with_nontemporal_clauseEi_private_i32 %8 -> %arg1 : !fir.ref<i32>) {
+    // CHECK: omp.simd nontemporal(%[[A_DECL]], %[[C_DECL]] : !fir.ref<i32>, !fir.ref<i32>) {
     // CHECK-NEXT: omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
-    omp.simd nontemporal(%2, %6 : !fir.ref<i32>, !fir.ref<i32>) private(@_QFsimd_with_nontemporal_clauseEi_private_i32 %8 -> %arg1 : !fir.ref<i32>) {
+    omp.simd nontemporal(%2, %6 : !fir.ref<i32>, !fir.ref<i32>) {
       omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%10) inclusive step (%c1_i32) {
-        %11 = fir.declare %arg1 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+        %11 = fir.declare %8 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
         fir.store %arg2 to %11 : !fir.ref<i32>
         // CHECK:  %[[LOAD:.*]] = fir.load %[[A_DECL]] {nontemporal} : !fir.ref<i32>
         %12 = fir.load %2 : !fir.ref<i32>
@@ -56,11 +56,11 @@ func.func @_QPsimd_nontemporal_allocatable(%arg0: !fir.ref<!fir.box<!fir.heap<!f
     %9 = fir.shape %7 : (index) -> !fir.shape<1>
     %10 = fir.embox %8(%9) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
     fir.store %10 to %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    // CHECK:  omp.simd nontemporal(%[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %2 -> %arg2 : !fir.ref<i32>) {
+    // CHECK:  omp.simd nontemporal(%[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
     // CHECK:   omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
-    omp.simd nontemporal(%3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %2 -> %arg2 : !fir.ref<i32>) {
+    omp.simd nontemporal(%3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
       omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
-        %16 = fir.declare %arg2 {uniq_name = "_QFsimd_nontemporal_allocatableEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+        %16 = fir.declare %2 {uniq_name = "_QFsimd_nontemporal_allocatableEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
         fir.store %arg3 to %16 : !fir.ref<i32>
         // CHECK:  %[[VAL1:.*]] = fir.load %[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
         %17 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>


From flang-commits at lists.llvm.org  Fri Jul  4 06:50:35 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 04 Jul 2025 06:50:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix tests broken by #146734 (PR
	#147055)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147055@github.com>
Message-ID: <6867dc2b.050a0220.32131b.56d0@mx.google.com>


https://github.com/tblah closed https://github.com/llvm/llvm-project/pull/147055

From flang-commits at lists.llvm.org  Mon Jul  7 00:08:53 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Mon, 07 Jul 2025 00:08:53 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ACOSD and ASIND (fixes issue
	#145593) (PR #145656)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145656@github.com>
Message-ID: <686b7285.630a0220.52af0.d0d4@mx.google.com>


mjklemm wrote:

> LG.
> 
> One question and a request. 
> - Why did you remove the precise test? 

I felt it was not needed any more, as the current checks seems sufficient.

> - Can you add some unit tests for these intrinsics in https://github.com/llvm/llvm-test-suite/tree/main/Fortran/UnitTests ?

Sure thing!  It would have been good to have those.  This bug would have been easy to catch then. :-)



https://github.com/llvm/llvm-project/pull/145656

From flang-commits at lists.llvm.org  Mon Jul  7 00:09:15 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 00:09:15 -0700 (PDT)
Subject: [flang-commits] [flang] 19afd27 - [Flang] Fix ACOSD and ASIND
	(fixes issue #145593) (#145656)
Message-ID: <686b729b.170a0220.124b47.1b7a@mx.google.com>


Author: Michael Klemm
Date: 2025-07-07T09:09:11+02:00
New Revision: 19afd27eb8efbe88392aea3294c2d7591eadc94f

URL: https://github.com/llvm/llvm-project/commit/19afd27eb8efbe88392aea3294c2d7591eadc94f
DIFF: https://github.com/llvm/llvm-project/commit/19afd27eb8efbe88392aea3294c2d7591eadc94f.diff

LOG: [Flang] Fix ACOSD and ASIND (fixes issue #145593) (#145656)

Original implementation converted DEG->RAD before calling ACOS/ASIN, but
the conversion needs to happen after invoking ACOS/ASIN.

Added: 
    

Modified: 
    flang/lib/Optimizer/Builder/IntrinsicCall.cpp
    flang/test/Lower/Intrinsics/acosd.f90
    flang/test/Lower/Intrinsics/asind.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index eb72dc9ebf89f..d32c1fde59f27 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -2648,16 +2648,18 @@ mlir::Value IntrinsicLibrary::genAbs(mlir::Type resultType,
 // ACOSD
 mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {
+  // maps ACOSD to ACOS * 180 / pi
   assert(args.size() == 1);
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value result =
+      getRuntimeCallGenerator("acos", ftype)(builder, loc, {args[0]});
   llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
   mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
+      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
-  return getRuntimeCallGenerator("acos", ftype)(builder, loc, {arg});
+  return builder.create<mlir::arith::MulFOp>(loc, result, factor);
 }
 
 // ADJUSTL & ADJUSTR
@@ -2799,16 +2801,18 @@ IntrinsicLibrary::genAny(mlir::Type resultType,
 // ASIND
 mlir::Value IntrinsicLibrary::genAsind(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {
+  // maps ASIND to ASIN * 180 / pi
   assert(args.size() == 1);
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value result =
+      getRuntimeCallGenerator("asin", ftype)(builder, loc, {args[0]});
   llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
   mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
+      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
-  return getRuntimeCallGenerator("asin", ftype)(builder, loc, {arg});
+  return builder.create<mlir::arith::MulFOp>(loc, result, factor);
 }
 
 // ATAND, ATAN2D

diff  --git a/flang/test/Lower/Intrinsics/acosd.f90 b/flang/test/Lower/Intrinsics/acosd.f90
index 3f558d66c35a1..7dfa28fd6494e 100644
--- a/flang/test/Lower/Intrinsics/acosd.f90
+++ b/flang/test/Lower/Intrinsics/acosd.f90
@@ -1,5 +1,3 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
-! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -8,10 +6,10 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 0.017453292519943295 : f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.acos %{{.*}} fastmath<contract> : f32
 ! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f32
-! CHECK-PRECISE: %{{.*}} = fir.call @acosf(%[[arg]]) fastmath<contract> : (f32) -> f32
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
   real(8) :: x, test_real8
@@ -19,6 +17,6 @@ function test_real8(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
-! CHECK: %[[factor:.*]] = arith.constant 0.017453292519943295 : f64
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f64
-! CHECK-PRECISE: %{{.*}} = fir.call @acos(%[[arg]]) fastmath<contract> : (f64) -> f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.acos %{{.*}} fastmath<contract> : f64
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[dfactor]] fastmath<contract> : f64

diff  --git a/flang/test/Lower/Intrinsics/asind.f90 b/flang/test/Lower/Intrinsics/asind.f90
index e03f2e5598555..564fa955fa969 100644
--- a/flang/test/Lower/Intrinsics/asind.f90
+++ b/flang/test/Lower/Intrinsics/asind.f90
@@ -1,5 +1,3 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
-! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -8,10 +6,10 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 0.017453292519943295 : f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.asin %{{.*}} fastmath<contract> : f32
 ! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f32
-! CHECK-PRECISE: %{{.*}} = fir.call @asinf(%[[arg]]) fastmath<contract> : (f32) -> f32
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
   real(8) :: x, test_real8
@@ -19,6 +17,6 @@ function test_real8(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
-! CHECK: %[[factor:.*]] = arith.constant 0.017453292519943295 : f64
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f64
-! CHECK-PRECISE: %{{.*}} = fir.call @asin(%[[arg]]) fastmath<contract> : (f64) -> f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.asin %{{.*}} fastmath<contract> : f64
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[dfactor]] fastmath<contract> : f64


        

From flang-commits at lists.llvm.org  Mon Jul  7 00:09:18 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Mon, 07 Jul 2025 00:09:18 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ACOSD and ASIND (fixes issue
	#145593) (PR #145656)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145656@github.com>
Message-ID: <686b729e.170a0220.f3500.bfe1@mx.google.com>


https://github.com/mjklemm closed https://github.com/llvm/llvm-project/pull/145656

From flang-commits at lists.llvm.org  Mon Jul  7 01:17:12 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Mon, 07 Jul 2025 01:17:12 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686b8288.a70a0220.25c452.d804@mx.google.com>


Stylie777 wrote:

Hi @ronlieb,
Can you provide a fortran reproducer for this issue? It's not something I hit when working on this and the output you've given does not indicate where in the function this is crashing.
Thanks.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Mon Jul  7 01:57:06 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 01:57:06 -0700 (PDT)
Subject: [flang-commits] [flang] 5271f9f - [Flang][Doc] NFC: Minor fix for
	headings (#147077)
Message-ID: <686b8be2.170a0220.35253b.c41c@mx.google.com>


Author: Kiran Chandramohan
Date: 2025-07-07T09:57:03+01:00
New Revision: 5271f9fba9dd8cbb173123a51772e631459f116a

URL: https://github.com/llvm/llvm-project/commit/5271f9fba9dd8cbb173123a51772e631459f116a
DIFF: https://github.com/llvm/llvm-project/commit/5271f9fba9dd8cbb173123a51772e631459f116a.diff

LOG: [Flang][Doc] NFC: Minor fix for headings (#147077)

Use a top level section to ensure that there is only one entry in the
flang.llvm.org/docs page.

Also generate a table of contents.

Added: 
    

Modified: 
    flang/docs/OpenMP-declare-target.md
    flang/docs/OpenMP-descriptor-management.md

Removed: 
    


################################################################################
diff  --git a/flang/docs/OpenMP-declare-target.md b/flang/docs/OpenMP-declare-target.md
index 45062469007b6..478db2d004c87 100644
--- a/flang/docs/OpenMP-declare-target.md
+++ b/flang/docs/OpenMP-declare-target.md
@@ -7,7 +7,15 @@
 
 -->
 
-# Introduction to Declare Target
+# OpenMP Declare Target
+
+```{contents}
+---
+local:
+---
+```
+
+## Introduction to Declare Target
 
 In OpenMP `declare target` is a directive that can be applied to a function or
 variable (primarily global) to notate to the compiler that it should be 
@@ -55,7 +63,7 @@ in this case, leave off the `declare target` from the function and it
 would be implicitly marked `declare target any` (for both host and device), 
 as it's been utilised within a target region.
 
-# Declare Target as represented in the OpenMP Dialect
+## Declare Target as represented in the OpenMP Dialect
 
 In the OpenMP Dialect `declare target` is not represented by a specific 
 `operation`. Instead, it's an OpenMP dialect specific `attribute` that can be 
@@ -89,7 +97,7 @@ llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(Op.getOperation());
 declareTargetGlobal.isDeclareTarget();
 ```
 
-# Declare Target Fortran OpenMP Lowering
+## Declare Target Fortran OpenMP Lowering
 
 The initial lowering of `declare target` to MLIR for both use-cases is done
 inside of the usual OpenMP lowering in flang/lib/Lower/OpenMP.cpp. However, 
@@ -145,7 +153,7 @@ to the way `threadprivate` handles these cases, we raise and initialize the
 variable as an internal `GlobalOp` and apply the attribute. This occurs in the
 flang/lib/Lower/OpenMP.cpp function `genDeclareTargetIntGlobal`.
 
-# Declare Target Transformation Passes for Flang
+## Declare Target Transformation Passes for Flang
 
 There are currently two passes within Flang that are related to the processing 
 of `declare target`:
@@ -173,7 +181,7 @@ While this infrastructure could be generally applicable to more than just Flang,
 it is only utilised in the Flang frontend, so it resides there rather than in 
 the OpenMP dialect codebase. 
 
-# Declare Target OpenMP Dialect To LLVM-IR Lowering
+## Declare Target OpenMP Dialect To LLVM-IR Lowering
 
 The OpenMP dialect lowering of `declare target` is done through the 
 `amendOperation` flow, as it's not an `operation` but rather an 
@@ -228,7 +236,7 @@ place. It may be possible to store this information in the OMPIRBuilder and
 then perform this cleanup process on finalization, but this is open for 
 discussion and implementation still.
 
-# Current Support
+## Current Support
 
 For the moment, `declare target` should work for:
 * Marking functions/subroutines and function/subroutine interfaces for 

diff  --git a/flang/docs/OpenMP-descriptor-management.md b/flang/docs/OpenMP-descriptor-management.md
index cbc27f2a0fcb8..d0764c2df4642 100644
--- a/flang/docs/OpenMP-descriptor-management.md
+++ b/flang/docs/OpenMP-descriptor-management.md
@@ -6,8 +6,15 @@
 
 -->
 
-# OpenMP dialect: Fortran descriptor type mapping for offload
+# Fortran descriptor type mapping for OpenMP offload
+```{contents}
+---
+local:
+---
+```
+
 
+## Details
 The initial method for mapping Fortran types tied to descriptors for OpenMP offloading is to treat these types 
 as a special case of OpenMP record type (C/C++ structure/class, Fortran derived type etc.) mapping as far as the 
 runtime is concerned. Where the box (descriptor information) is the holding container and the underlying 
@@ -103,7 +110,7 @@ it is a little rigid in how the descriptor mappings are handled as there is no s
 to specialise the mappings for possible edge cases without polluting the dialect or lowering with further
 knowledge of Fortran and the FIR dialect.
 
-# OpenMP dialect 
diff erences from OpenACC dialect
+## Differences from OpenACC
 
 The descriptor mapping for OpenMP currently works 
diff erently to the planned direction for OpenACC, however, 
 it is possible and would likely be ideal to align the method with OpenACC in the future. 


        

From flang-commits at lists.llvm.org  Mon Jul  7 01:57:08 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Mon, 07 Jul 2025 01:57:08 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][Doc] NFC: Minor fix for headings
	(PR #147077)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147077@github.com>
Message-ID: <686b8be4.050a0220.15583b.d5b1@mx.google.com>


https://github.com/kiranchandramohan closed https://github.com/llvm/llvm-project/pull/147077

From flang-commits at lists.llvm.org  Mon Jul  7 01:57:16 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 01:57:16 -0700 (PDT)
Subject: [flang-commits] [flang] 8a221a5 - [Flang][OpenMP] Push context when
 parsing DECLARE VARIANT (#147075)
Message-ID: <686b8bec.170a0220.158062.bf14@mx.google.com>


Author: Jack Styles
Date: 2025-07-07T09:57:13+01:00
New Revision: 8a221a585ca70dede85f191992a0364e43cfdbf0

URL: https://github.com/llvm/llvm-project/commit/8a221a585ca70dede85f191992a0364e43cfdbf0
DIFF: https://github.com/llvm/llvm-project/commit/8a221a585ca70dede85f191992a0364e43cfdbf0.diff

LOG: [Flang][OpenMP] Push context when parsing DECLARE VARIANT (#147075)

Basic parsing and semantics support for Declare Variant was added in
#130578. However, this did not include variant of `Pre` and `Post`
within `OmpAttributeVisitor`. This meant that when a function in the
class tried to get the context using `GetContext`, Flang would crash as
the context was empty. To ensure this is possible, such as when
resolving names as part of the `uniform` clause in the `simd` directive,
the context is now pushed within `OmpAttributeVisitor` when parsing a
`DECLARE VARIANT` directive.

Fixes #145222

Added: 
    

Modified: 
    flang/lib/Semantics/resolve-directives.cpp
    flang/test/Parser/OpenMP/declare-variant.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 8d741734601a7..299bb6ff876e7 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -495,6 +495,12 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   bool Pre(const parser::OpenMPAllocatorsConstruct &);
   void Post(const parser::OpenMPAllocatorsConstruct &);
 
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    PushContext(x.source, llvm::omp::Directive::OMPD_declare_variant);
+    return true;
+  }
+  void Post(const parser::OmpDeclareVariantDirective &) { PopContext(); };
+
   void Post(const parser::OmpObjectList &x) {
     // The objects from OMP clauses should have already been resolved,
     // except common blocks (the ResolveNamesVisitor does not visit

diff  --git a/flang/test/Parser/OpenMP/declare-variant.f90 b/flang/test/Parser/OpenMP/declare-variant.f90
index 1b97733ea9525..3366b143e62e6 100644
--- a/flang/test/Parser/OpenMP/declare-variant.f90
+++ b/flang/test/Parser/OpenMP/declare-variant.f90
@@ -102,3 +102,30 @@ subroutine vsub(v1, v2)
     type(c_ptr), value :: v1, v2
   end
 end subroutine
+
+subroutine f
+  real :: x, y
+  y = 2
+  !omp simd
+  call f2(x, y)
+  !omp end simd 
+contains
+  subroutine f1 (x, y)
+    real :: x, y
+  end
+
+  subroutine f2 (x, y)
+    real :: x, y
+    !$omp declare variant (f1) match (construct={simd(uniform(y))})
+  end
+end subroutine
+!CHECK: !$OMP DECLARE VARIANT (f1) MATCH(CONSTRUCT={SIMD(UNIFORM(y))})
+!PARSE-TREE: | | | | DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpDeclareVariantDirective
+!PARSE-TREE-NEXT: | | | | | Verbatim
+!PARSE-TREE-NEXT: | | | | | Name = 'f1'
+!PARSE-TREE-NEXT: | | | | | OmpClauseList -> OmpClause -> Match -> OmpMatchClause -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSetSelectorName -> Value = Construct
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSelector
+!PARSE-TREE-NEXT: | | | | | | | OmpTraitSelectorName -> Value = Simd
+!PARSE-TREE-NEXT: | | | | | | | Properties
+!PARSE-TREE-NEXT: | | | | | | | | OmpTraitProperty -> OmpClause -> Uniform -> Name = 'y'


        

From flang-commits at lists.llvm.org  Mon Jul  7 01:57:20 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Mon, 07 Jul 2025 01:57:20 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Push context when parsing
 DECLARE VARIANT (PR #147075)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147075@github.com>
Message-ID: <686b8bf0.050a0220.1f3f8c.d177@mx.google.com>


https://github.com/Stylie777 closed https://github.com/llvm/llvm-project/pull/147075

From flang-commits at lists.llvm.org  Mon Jul  7 02:20:40 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 07 Jul 2025 02:20:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix test REQUIRES and options
	for aint.f90 (PR #146870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146870@github.com>
Message-ID: <686b9168.170a0220.10395d.fc05@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks for the fix

https://github.com/llvm/llvm-project/pull/146870

From flang-commits at lists.llvm.org  Mon Jul  7 02:21:11 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 07 Jul 2025 02:21:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
 a few x86 specific tests (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <686b9187.170a0220.178c6.ca7c@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks for the fix

https://github.com/llvm/llvm-project/pull/146872

From flang-commits at lists.llvm.org  Mon Jul  7 02:25:44 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 02:25:44 -0700 (PDT)
Subject: [flang-commits] [flang] 3e934dd - [flang][test] Fix test REQUIRES
 and options for aint.f90 (#146870)
Message-ID: <686b9298.170a0220.12bdfd.de5d@mx.google.com>


Author: David Spickett
Date: 2025-07-07T10:25:41+01:00
New Revision: 3e934dded0d988bdfd0975c117a302721130082a

URL: https://github.com/llvm/llvm-project/commit/3e934dded0d988bdfd0975c117a302721130082a
DIFF: https://github.com/llvm/llvm-project/commit/3e934dded0d988bdfd0975c117a302721130082a.diff

LOG: [flang][test] Fix test REQUIRES and options for aint.f90 (#146870)

This test should have been looking for the "x86" target, not "x86_64".

In the time it's not been running, -target must have been changed to
-triple.

Added: 
    

Modified: 
    flang/test/Lower/math-lowering/aint.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Lower/math-lowering/aint.f90 b/flang/test/Lower/math-lowering/aint.f90
index 6c7809f8ea1a3..b36fb421d7c8c 100644
--- a/flang/test/Lower/math-lowering/aint.f90
+++ b/flang/test/Lower/math-lowering/aint.f90
@@ -1,10 +1,10 @@
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
 
 function test_real4(x)
   real :: x, test_real4


        

From flang-commits at lists.llvm.org  Mon Jul  7 02:25:47 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Mon, 07 Jul 2025 02:25:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix test REQUIRES and options
	for aint.f90 (PR #146870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146870@github.com>
Message-ID: <686b929b.170a0220.92836.c768@mx.google.com>


https://github.com/DavidSpickett closed https://github.com/llvm/llvm-project/pull/146870

From flang-commits at lists.llvm.org  Mon Jul  7 02:26:20 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 02:26:20 -0700 (PDT)
Subject: [flang-commits] [flang] b1a8c8a - [flang][test] Fix REQUIRES and
 options for a few x86 specific tests (#146872)
Message-ID: <686b92bc.170a0220.185528.c2a7@mx.google.com>


Author: David Spickett
Date: 2025-07-07T10:26:17+01:00
New Revision: b1a8c8a32c4b2453d526bcade145295966e90fda

URL: https://github.com/llvm/llvm-project/commit/b1a8c8a32c4b2453d526bcade145295966e90fda
DIFF: https://github.com/llvm/llvm-project/commit/b1a8c8a32c4b2453d526bcade145295966e90fda.diff

LOG: [flang][test] Fix REQUIRES and options for a few x86 specific tests (#146872)

These should have been looking for the "x86" target not "x64_64".

When run on AArch64 they failed because bbc tried to compile for
AArch64. Add a target option to fix that, as these tests are x86
specific.

Added: 
    

Modified: 
    flang/test/Lower/Intrinsics/product.f90
    flang/test/Lower/Intrinsics/sum.f90
    flang/test/Lower/complex-operations.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Lower/Intrinsics/product.f90 b/flang/test/Lower/Intrinsics/product.f90
index f5f7cff106cee..df7c1e4ce7eaa 100644
--- a/flang/test/Lower/Intrinsics/product.f90
+++ b/flang/test/Lower/Intrinsics/product.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPproduct_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32

diff  --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index d36b2b1263887..3167617b60457 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPsum_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32 {

diff  --git a/flang/test/Lower/complex-operations.f90 b/flang/test/Lower/complex-operations.f90
index fdd13719c5b71..38954626e77ee 100644
--- a/flang/test/Lower/complex-operations.f90
+++ b/flang/test/Lower/complex-operations.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: @_QPadd_test
 subroutine add_test(a,b,c)


        

From flang-commits at lists.llvm.org  Mon Jul  7 02:26:24 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Mon, 07 Jul 2025 02:26:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][test] Fix REQUIRES and options for
 a few x86 specific tests (PR #146872)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146872@github.com>
Message-ID: <686b92c0.050a0220.fb05f.d410@mx.google.com>


https://github.com/DavidSpickett closed https://github.com/llvm/llvm-project/pull/146872

From flang-commits at lists.llvm.org  Mon Jul  7 02:50:52 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Mon, 07 Jul 2025 02:50:52 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686b987c.050a0220.2b038f.e091@mx.google.com>


================
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
----------------
s-watanabe314 wrote:

I added content to `ComplexOperations.md` based on the clang documentation. Because I'm not a native English speaker, I would appreciate it if you could review it.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 03:32:01 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Mon, 07 Jul 2025 03:32:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686ba221.170a0220.33360d.fc2f@mx.google.com>


================
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
----------------
kiranchandramohan wrote:

It looks fine to me. Thanks for the additional content.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 03:32:36 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Mon, 07 Jul 2025 03:32:36 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686ba244.050a0220.257089.d90d@mx.google.com>


https://github.com/kiranchandramohan approved this pull request.

LG.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 04:01:23 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 07 Jul 2025 04:01:23 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement !DIR$ [NO]INLINE and
 FORCEINLINE directives (PR #134350)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/134350@github.com>
Message-ID: <686ba903.050a0220.28c530.cff1@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/134350

From flang-commits at lists.llvm.org  Mon Jul  7 04:01:24 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 07 Jul 2025 04:01:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement !DIR$ [NO]INLINE and
 FORCEINLINE directives (PR #134350)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/134350@github.com>
Message-ID: <686ba904.050a0220.2b0897.daa3@mx.google.com>


https://github.com/tblah approved this pull request.

Apologies it seems I was mistaken before and this is a valid call attribute after all. This works in my testing thank you for the patch.

https://github.com/llvm/llvm-project/pull/134350

From flang-commits at lists.llvm.org  Mon Jul  7 04:01:25 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 07 Jul 2025 04:01:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement !DIR$ [NO]INLINE and
 FORCEINLINE directives (PR #134350)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/134350@github.com>
Message-ID: <686ba905.170a0220.92836.cb7c@mx.google.com>


================
@@ -254,6 +254,13 @@ class ProcedureRef {
   bool IsElemental() const { return proc_.IsElemental(); }
   bool hasAlternateReturns() const { return hasAlternateReturns_; }
 
+  bool hasNoInline() const { return noInline_; }
+  void set_noInline(bool ni) { noInline_ = ni; }
+  bool hasAlwaysInline() const { return alwaysInline_; }
+  void set_alwaysInline(bool ai) { alwaysInline_ = ai; }
+  bool hasInlineHint() const { return inlineHint_; }
+  void set_inlineHint(bool ih) { inlineHint_ = ih; }
----------------
tblah wrote:

nit to match style
```suggestion
  void setNoInline(bool ni) { noInline_ = ni; }
  bool hasAlwaysInline() const { return alwaysInline_; }
  void setAlwaysInline(bool ai) { alwaysInline_ = ai; }
  bool hasInlineHint() const { return inlineHint_; }
  void setInlineHint(bool ih) { inlineHint_ = ih; }
```

https://github.com/llvm/llvm-project/pull/134350

From flang-commits at lists.llvm.org  Mon Jul  7 05:10:14 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 05:10:14 -0700 (PDT)
Subject: [flang-commits] [flang] 6855573 - [flang][OpenMP] Fix
 parallel-firstprivate-clause-scalar.f90 test (#146932)
Message-ID: <686bb926.170a0220.9339.caf0@mx.google.com>


Author: Leandro Lupori
Date: 2025-07-07T09:10:11-03:00
New Revision: 6855573700c3ee30ed19ac6eab914483a3019bd2

URL: https://github.com/llvm/llvm-project/commit/6855573700c3ee30ed19ac6eab914483a3019bd2
DIFF: https://github.com/llvm/llvm-project/commit/6855573700c3ee30ed19ac6eab914483a3019bd2.diff

LOG: [flang][OpenMP] Fix parallel-firstprivate-clause-scalar.f90 test (#146932)

Fix REQUIRES and references to declared variables.

Fixes #146875

Added: 
    

Modified: 
    flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 017b2a3f2edd2..2bb1036c4a7fa 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -1,8 +1,8 @@
 ! This test checks lowering of `FIRSTPRIVATE` clause for scalar types.
 
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! REQUIRES: shell
-! RUN: bbc -fopenmp -emit-hlfir %s -o - \
+! RUN: bbc -target x86_64-unknown-linux-gnu -fopenmp -emit-hlfir %s -o - \
 ! RUN: | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 !CHECK:  omp.private {type = firstprivate} @[[ARG2_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg2_firstprivate_l8]] : !fir.logical<1>
@@ -29,7 +29,7 @@
 !CHECK:   omp.parallel private(@[[ARG1_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG1_PVT:.*]], @[[ARG2_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG2_PVT:.*]] : {{.*}}) {
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
-!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
+!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -57,8 +57,8 @@ subroutine firstprivate_complex(arg1, arg2)
 !CHECK:    %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_integerEarg4"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_integerEarg5"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 !CHECK:    %[[ARG6_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG6_PVT]] {uniq_name = "_QFfirstprivate_integerEarg6"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
-!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1,
-!%[[ARG5_PVT_DECL]]#1, %[[ARG6_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
+!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0,
+!%[[ARG5_PVT_DECL]]#0, %[[ARG6_PVT_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
 !CHECK:    omp.terminator
 !CHECK:  }
 
@@ -88,7 +88,7 @@ subroutine firstprivate_integer(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg3"} : (!fir.ref<!fir.logical<2>>) -> (!fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<2>>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg4"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK:     %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg5"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
-!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1, %[[ARG5_PVT_DECL]]#1) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
+!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0, %[[ARG5_PVT_DECL]]#0) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -116,7 +116,7 @@ subroutine firstprivate_logical(arg1, arg2, arg3, arg4, arg5)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
+!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
@@ -136,7 +136,7 @@ subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK-KIND10:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>, !fir.dscope) -> (!fir.ref<f80>, !fir.ref<f80>)
 !CHECK-KIND10:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND10:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
-!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f80>) -> ()
+!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f80>) -> ()
 !CHECK-KIND10:     omp.terminator
 !CHECK-KIND10:   }
 subroutine firstprivate_real10(arg1)
@@ -152,7 +152,7 @@ subroutine firstprivate_real10(arg1)
 !CHECK-KIND16:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
 !CHECK-KIND16:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND16:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
-!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f128>) -> ()
+!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f128>) -> ()
 !CHECK-KIND16:     omp.terminator
 !CHECK-KIND16:   }
 subroutine firstprivate_real16(arg1)
@@ -171,7 +171,7 @@ subroutine firstprivate_real16(arg1)
 !CHECK:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[A_PRIV_ADDR:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[B_PRIV_ADDR:.*]] : {{.*}}) {
 !CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#1, %[[B_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#0, %[[B_PRIV_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 !CHECK:             omp.terminator
 !CHECK:           }
 !CHECK:           return


        

From flang-commits at lists.llvm.org  Mon Jul  7 05:10:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 05:10:57 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686bb951.170a0220.37e2af.ccbf@mx.google.com>


ronlieb wrote:

hi @Stylie777    did you try running the compilation 30 times ? its very intermittent for me

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Mon Jul  7 05:42:44 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Mon, 07 Jul 2025 05:42:44 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686bc0c4.170a0220.2a20c.cf94@mx.google.com>


Stylie777 wrote:

@ronlieb 
I did yes, I ran it 30 times against a test file(s) I have and all ended in errors stating that `tile` and `unroll` are unimplemented which is expected behviour. The CMD line is the same as you posted above, with the only change being the target file.

This is done with a build based on `1c8283a30c6e87640510f523dfc244bd9d2b991a` from this morning. 

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Mon Jul  7 06:00:49 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 07 Jul 2025 06:00:49 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686bc501.170a0220.1cb24e.e1c1@mx.google.com>


kparzysz wrote:

[This line](https://github.com/llvm/llvm-project/blob/main/flang/lib/Semantics/canonicalize-omp.cpp#L152) sets `nextIt` to `block.end()`, which makes the next if load some indeterminate value.  I'm trying to see if I can come up with a small testcase.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Mon Jul  7 06:14:25 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 07 Jul 2025 06:14:25 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <686bc831.170a0220.222bfa.ec84@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Mon Jul  7 06:22:51 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 07 Jul 2025 06:22:51 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686bca2b.a70a0220.2b70d.04eb@mx.google.com>


kparzysz wrote:

```
subroutine fred(x_min, x_max, a)
implicit none

integer :: x_min, x_max
real(kind=8), dimension(x_min:x_max, 2) :: a
integer :: i, j

!$omp target teams distribute parallel do collapse(2) private(i)
do i = x_min, x_max
  a(i, 1) = a(i, 0)
enddo

end
```

Command:
```
flang -fc1 -triple x86_64-unknown-linux-gnu -emit-llvm-bc -fopenmp -O3 h.f90
```

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Mon Jul  7 06:28:43 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Mon, 07 Jul 2025 06:28:43 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686bcb8b.050a0220.171fcb.0e39@mx.google.com>


Stylie777 wrote:

Thanks @kparzysz I will take a look.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Mon Jul  7 06:28:54 2025
From: flang-commits at lists.llvm.org (David Truby via flang-commits)
Date: Mon, 07 Jul 2025 06:28:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Correctly handle
	-mframe-pointer=reserved (PR #146937)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146937@github.com>
Message-ID: <686bcb96.170a0220.148c26.f944@mx.google.com>


https://github.com/DavidTruby approved this pull request.

LGTM too!

https://github.com/llvm/llvm-project/pull/146937

From flang-commits at lists.llvm.org  Mon Jul  7 07:33:55 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 07 Jul 2025 07:33:55 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686bdad3.170a0220.222bfa.3469@mx.google.com>


================
@@ -192,6 +192,31 @@ class CodeGenOptions : public CodeGenOptionsBase {
     return getProfileUse() == llvm::driver::ProfileCSIRInstr;
   }
 
+  /// Controls the various implementations for complex division.
----------------
tarunprabhu wrote:

I think that it should be moved, but since it is in `LangOptions` in clang, and not `CodeGenOptions`, we would need some reviewers from the `clang` as well. In the interest of not blocking this PR, let's leave things as they are, but it would be good to avoid this duplication.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 07:44:12 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 07 Jul 2025 07:44:12 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686bdd3c.170a0220.15c9a7.3fd1@mx.google.com>


https://github.com/tarunprabhu edited https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 07:44:13 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 07 Jul 2025 07:44:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686bdd3d.050a0220.184d2e.4675@mx.google.com>


================
@@ -3409,3 +3409,30 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
 
   return Out;
 }
+
+std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
----------------
tarunprabhu wrote:

The functions in `CommonArgs.cpp` use a lower-case letter as the first letter in the function name, unlike clang which often uses upper case. 

```suggestion
std::string tools::complexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
```

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 07:44:13 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 07 Jul 2025 07:44:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686bdd3d.170a0220.35778e.3ad6@mx.google.com>


https://github.com/tarunprabhu approved this pull request.

Aside from the slight change to the function names in `CommonArgs.cpp` and an optional tweak to the documentation text, this looks good to me. 

Thank you for PR and the clarifications to my comments.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 07:44:13 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 07 Jul 2025 07:44:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686bdd3d.170a0220.2398ac.3e10@mx.google.com>


================
@@ -74,4 +74,26 @@ The ComplexToStandard dialect does still call into libm for some floating
 point math operations, however these don't have the same ABI issues as the
 complex libm functions.
 
+The flang driver option `-fcomplex-arithmetic=` allows you to select whether
+complex number division is lowered to function calls or to the `complex.div`
+operation in the MLIR complex dialect. To avoid the ABI issues mentioned above,
+the selection between function calls and the `complex.div` is made during the
+lowering phase. The behavior of this option is as follows:
----------------
tarunprabhu wrote:

```suggestion
the choice of function calls or the `complex.div` operation is made during the
lowering phase. The behavior of this option is as follows:
```

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 07:44:14 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 07 Jul 2025 07:44:14 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686bdd3e.050a0220.fce42.5004@mx.google.com>


================
@@ -3409,3 +3409,30 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
 
   return Out;
 }
+
+std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Promoted:
+    return "promoted";
+    break;
+  default:
+    return "";
+  }
+}
+
+std::string
+tools::RenderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
----------------
tarunprabhu wrote:

```suggestion
tools::renderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
```

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Mon Jul  7 02:23:33 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 07 Jul 2025 02:23:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Fix
 parallel-firstprivate-clause-scalar.f90 test (PR #146932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146932@github.com>
Message-ID: <686b9215.170a0220.2b6fe5.c6e3@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks!

https://github.com/llvm/llvm-project/pull/146932

From flang-commits at lists.llvm.org  Mon Jul  7 02:43:41 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Mon, 07 Jul 2025 02:43:41 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686b96cd.050a0220.158668.d88b@mx.google.com>


https://github.com/s-watanabe314 updated https://github.com/llvm/llvm-project/pull/146641

>From baf87a6002d740868daeeba0f89d55c05e85b712 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 18 Apr 2025 09:04:07 +0900
Subject: [PATCH 1/6] [flang] Add -fcomplex-arithmetic= option and select
 complex division algorithm

This patch adds an option to select the method for computing complex
number division. It uses `LoweringOptions` to determine whether to lower
complex division to a runtime function call or to MLIR's `complex.div`,
and `CodeGenOptions` to select the computation algorithm for
`complex.div`. The available option values and their corresponding
algorithms are as follows:
- `full`: Lower to a runtime function call. (Default behavior)
- `improved`: Lower to `complex.div` and expand to Smith's algorithm.
- `basic`: Lower to `complex.div` and expand to the algebraic algorithm.

See also the discussion in the following discourse post:
https://discourse.llvm.org/t/optimization-of-complex-number-division/83468
---
 clang/include/clang/Driver/Options.td         |   4 +-
 clang/lib/Driver/ToolChains/Flang.cpp         |  47 ++
 .../include/flang/Frontend/CodeGenOptions.def |   1 +
 flang/include/flang/Frontend/CodeGenOptions.h |  25 +
 flang/include/flang/Lower/LoweringOptions.def |   4 +
 .../flang/Optimizer/Builder/FIRBuilder.h      |  15 +
 .../include/flang/Optimizer/CodeGen/CodeGen.h |   6 +
 flang/include/flang/Tools/CrossToolHelpers.h  |   3 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  21 +
 flang/lib/Frontend/FrontendActions.cpp        |   2 +
 flang/lib/Lower/Bridge.cpp                    |   2 +
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |  12 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  15 +-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |   1 +
 flang/test/Driver/complex-range.f90           |  23 +
 .../complex-div-to-llvm-kind10.f90            | 131 +++++
 .../complex-div-to-llvm-kind16.f90            | 131 +++++
 .../test/Integration/complex-div-to-llvm.f90  | 508 ++++++++++++++++++
 .../HLFIR/complex-div-to-hlfir-kind10.f90     |  34 ++
 .../HLFIR/complex-div-to-hlfir-kind16.f90     |  35 ++
 .../test/Lower/HLFIR/complex-div-to-hlfir.f90 |  94 ++++
 flang/tools/bbc/bbc.cpp                       |   8 +
 22 files changed, 1117 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Driver/complex-range.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm-kind10.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm-kind16.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..58209ceb5dc54 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
 
 def complex_range_EQ : Joined<["-"], "complex-range=">, Group<f_Group>,
-  Visibility<[CC1Option]>,
+  Visibility<[CC1Option, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>,
   MarshallingInfoEnum<LangOpts<"ComplexRange">, "CX_Full">;
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index e4e321ba1e195..ba5db7e3aba43 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -595,6 +595,30 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
+static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  default:
+    return "";
+  }
+}
+
+static std::string
+RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
+
 static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   StringRef FPContract;
@@ -605,6 +629,8 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
   bool AssociativeMath = false;
   bool ReciprocalMath = false;
 
+  LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_None;
+
   if (const Arg *A = Args.getLastArg(options::OPT_ffp_contract)) {
     const StringRef Val = A->getValue();
     if (Val == "fast" || Val == "off") {
@@ -629,6 +655,20 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
     default:
       continue;
 
+    case options::OPT_fcomplex_arithmetic_EQ: {
+      StringRef Val = A->getValue();
+      if (Val == "full")
+        Range = LangOptions::ComplexRangeKind::CX_Full;
+      else if (Val == "improved")
+        Range = LangOptions::ComplexRangeKind::CX_Improved;
+      else if (Val == "basic")
+        Range = LangOptions::ComplexRangeKind::CX_Basic;
+      else {
+        D.Diag(diag::err_drv_unsupported_option_argument)
+            << A->getSpelling() << Val;
+      }
+      break;
+    }
     case options::OPT_fhonor_infinities:
       HonorINFs = true;
       break;
@@ -699,6 +739,13 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
   if (!Recip.empty())
     CmdArgs.push_back(Args.MakeArgString("-mrecip=" + Recip));
 
+  if (Range != LangOptions::ComplexRangeKind::CX_None) {
+    std::string ComplexRangeStr = RenderComplexRangeOption(Range);
+    CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
+    CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" +
+                                         ComplexRangeKindToStr(Range)));
+  }
+
   if (!HonorINFs && !HonorNaNs && AssociativeMath && ReciprocalMath &&
       ApproxFunc && !SignedZeros &&
       (FPContract == "fast" || FPContract.empty())) {
diff --git a/flang/include/flang/Frontend/CodeGenOptions.def b/flang/include/flang/Frontend/CodeGenOptions.def
index ae12aec518108..cdeea93c9aecb 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.def
+++ b/flang/include/flang/Frontend/CodeGenOptions.def
@@ -52,6 +52,7 @@ ENUM_CODEGENOPT(RelocationModel, llvm::Reloc::Model, 3, llvm::Reloc::PIC_) ///<
 ENUM_CODEGENOPT(DebugInfo,  llvm::codegenoptions::DebugInfoKind, 4,  llvm::codegenoptions::NoDebugInfo) ///< Level of debug info to generate
 ENUM_CODEGENOPT(VecLib, llvm::driver::VectorLibrary, 4, llvm::driver::VectorLibrary::NoLibrary) ///< Vector functions library to use
 ENUM_CODEGENOPT(FramePointer, llvm::FramePointerKind, 2, llvm::FramePointerKind::None) ///< Enable the usage of frame pointers
+ENUM_CODEGENOPT(ComplexRange, ComplexRangeKind, 3, ComplexRangeKind::CX_Full) ///< Method for calculating complex number division
 
 ENUM_CODEGENOPT(DoConcurrentMapping, DoConcurrentMappingKind, 2, DoConcurrentMappingKind::DCMK_None) ///< Map `do concurrent` to OpenMP
 
diff --git a/flang/include/flang/Frontend/CodeGenOptions.h b/flang/include/flang/Frontend/CodeGenOptions.h
index bad17c8309eb8..df6063cc90340 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.h
+++ b/flang/include/flang/Frontend/CodeGenOptions.h
@@ -192,6 +192,31 @@ class CodeGenOptions : public CodeGenOptionsBase {
     return getProfileUse() == llvm::driver::ProfileCSIRInstr;
   }
 
+  /// Controls the various implementations for complex division.
+  enum ComplexRangeKind {
+    /// Implementation of complex division using a call to runtime library
+    /// functions. Overflow and non-finite values are handled by the library
+    /// implementation. This is the default value.
+    CX_Full,
+
+    /// Implementation of complex division offering an improved handling
+    /// for overflow in intermediate calculations. Overflow and non-finite
+    /// values are handled by MLIR's implementation of "complex.div", but this
+    /// may change in the future.
+    CX_Improved,
+
+    /// Implementation of complex division using algebraic formulas at source
+    /// precision. No special handling to avoid overflow. NaN and infinite
+    /// values are not handled.
+    CX_Basic,
+
+    /// No range rule is enabled.
+    CX_None
+
+    /// TODO: Implemention of other values as needed. In Clang, "CX_Promoted"
+    /// is implemented. (See clang/Basic/LangOptions.h)
+  };
+
   // Define accessors/mutators for code generation options of enumeration type.
 #define CODEGENOPT(Name, Bits, Default)
 #define ENUM_CODEGENOPT(Name, Type, Bits, Default)                             \
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index 3263ab129d076..8135704971aa4 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -70,5 +70,9 @@ ENUM_LOWERINGOPT(CUDARuntimeCheck, unsigned, 1, 0)
 /// derived types defined in other compilation units.
 ENUM_LOWERINGOPT(SkipExternalRttiDefinition, unsigned, 1, 0)
 
+/// If true, convert complex number division to runtime on the frontend.
+/// If false, lower to the complex dialect of MLIR.
+/// On by default.
+ENUM_LOWERINGOPT(ComplexDivisionToRuntime, unsigned, 1, 1)
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index e1eaab3346901..b1513850a9048 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -609,6 +609,17 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return integerOverflowFlags;
   }
 
+  /// Set ComplexDivisionToRuntimeFlag value for whether complex number division
+  /// is lowered to a runtime function by this builder.
+  void setComplexDivisionToRuntimeFlag(bool flag) {
+    complexDivisionToRuntimeFlag = flag;
+  }
+
+  /// Get current ComplexDivisionToRuntimeFlag value.
+  bool getComplexDivisionToRuntimeFlag() const {
+    return complexDivisionToRuntimeFlag;
+  }
+
   /// Dump the current function. (debug)
   LLVM_DUMP_METHOD void dumpFunc();
 
@@ -673,6 +684,10 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   /// mlir::arith::IntegerOverflowFlagsAttr.
   mlir::arith::IntegerOverflowFlags integerOverflowFlags{};
 
+  /// Flag to control whether complex number division is lowered to a runtime
+  /// function or to the MLIR complex dialect.
+  bool complexDivisionToRuntimeFlag = true;
+
   /// fir::GlobalOp and func::FuncOp symbol table to speed-up
   /// lookups.
   mlir::SymbolTable *symbolTable = nullptr;
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 93f07d8d5d4d9..e9a07a8dde5cd 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 #define FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 
+#include "flang/Frontend/CodeGenOptions.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassRegistry.h"
@@ -58,6 +59,11 @@ struct FIRToLLVMPassOptions {
   // the name of the global variable corresponding to a derived
   // type's descriptor.
   bool typeDescriptorsRenamedForAssembly = false;
+
+  // Specify the calculation method for complex number division used by the
+  // Conversion pass of the MLIR complex dialect.
+  Fortran::frontend::CodeGenOptions::ComplexRangeKind ComplexRange =
+      Fortran::frontend::CodeGenOptions::ComplexRangeKind::CX_Full;
 };
 
 /// Convert FIR to the LLVM IR dialect with default options.
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 337685c82af5f..df1da27058552 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -140,6 +140,9 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   std::string InstrumentFunctionExit =
       ""; ///< Name of the instrument-function that is called on each
           ///< function-exit
+  Fortran::frontend::CodeGenOptions::ComplexRangeKind ComplexRange =
+      Fortran::frontend::CodeGenOptions::ComplexRangeKind::
+          CX_Full; ///< Method for calculating complex number division
 };
 
 struct OffloadModuleOpts {
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..86ec410b1f70f 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -484,6 +484,21 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
   }
 
   parseDoConcurrentMapping(opts, args, diags);
+
+  if (const auto *arg =
+          args.getLastArg(clang::driver::options::OPT_complex_range_EQ)) {
+    llvm::StringRef argValue = llvm::StringRef(arg->getValue());
+    if (argValue == "full") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Full);
+    } else if (argValue == "improved") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Improved);
+    } else if (argValue == "basic") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Basic);
+    } else {
+      diags.Report(clang::diag::err_drv_invalid_value)
+          << arg->getAsString(args) << arg->getValue();
+    }
+  }
 }
 
 /// Parses all target input arguments and populates the target
@@ -1811,4 +1826,10 @@ void CompilerInvocation::setLoweringOptions() {
       .setNoSignedZeros(langOptions.NoSignedZeros)
       .setAssociativeMath(langOptions.AssociativeMath)
       .setReciprocalMath(langOptions.ReciprocalMath);
+
+  if (codegenOpts.getComplexRange() ==
+          CodeGenOptions::ComplexRangeKind::CX_Improved ||
+      codegenOpts.getComplexRange() ==
+          CodeGenOptions::ComplexRangeKind::CX_Basic)
+    loweringOpts.setComplexDivisionToRuntime(false);
 }
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index bf15def3f3b2e..b5f4f9421f633 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -750,6 +750,8 @@ void CodeGenAction::generateLLVMIR() {
   if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
     config.NSWOnLoopVarInc = false;
 
+  config.ComplexRange = opts.getComplexRange();
+
   // Create the pass pipeline
   fir::createMLIRToLLVMPassPipeline(pm, config, getCurrentFile());
   (void)mlir::applyPassManagerCLOptions(pm);
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..7e95c640e73b0 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5746,6 +5746,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     builder =
         new fir::FirOpBuilder(func, bridge.getKindMap(), &mlirSymbolTable);
     assert(builder && "FirOpBuilder did not instantiate");
+    builder->setComplexDivisionToRuntimeFlag(
+        bridge.getLoweringOptions().getComplexDivisionToRuntime());
     builder->setFastMathFlags(bridge.getLoweringOptions().getMathOptions());
     builder->setInsertionPointToStart(&func.front());
     if (funit.parent.isA<Fortran::lower::pft::FunctionLikeUnit>()) {
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index df8dfbc72c030..cb338618dbf3b 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1066,8 +1066,16 @@ struct BinaryOp<Fortran::evaluate::Divide<
     mlir::Type ty = Fortran::lower::getFIRType(
         builder.getContext(), Fortran::common::TypeCategory::Complex, KIND,
         /*params=*/{});
-    return hlfir::EntityWithAttributes{
-        fir::genDivC(builder, loc, ty, lhs, rhs)};
+
+    // TODO: Ideally, complex number division operations should always be
+    // lowered to MLIR. However, converting them to the runtime via MLIR causes
+    // ABI issues.
+    if (builder.getComplexDivisionToRuntimeFlag())
+      return hlfir::EntityWithAttributes{
+          fir::genDivC(builder, loc, ty, lhs, rhs)};
+    else
+      return hlfir::EntityWithAttributes{
+          builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
   }
 };
 
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 2b018912b40e4..4cf533f195e69 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -4119,7 +4119,20 @@ class FIRToLLVMLowering
     mathToFuncsOptions.minWidthOfFPowIExponent = 33;
     mathConvertionPM.addPass(
         mlir::createConvertMathToFuncs(mathToFuncsOptions));
-    mathConvertionPM.addPass(mlir::createConvertComplexToStandardPass());
+
+    mlir::ConvertComplexToStandardPassOptions complexToStandardOptions{};
+    if (options.ComplexRange ==
+        Fortran::frontend::CodeGenOptions::ComplexRangeKind::CX_Basic) {
+      complexToStandardOptions.complexRange =
+          mlir::complex::ComplexRangeFlags::basic;
+    } else if (options.ComplexRange == Fortran::frontend::CodeGenOptions::
+                                           ComplexRangeKind::CX_Improved) {
+      complexToStandardOptions.complexRange =
+          mlir::complex::ComplexRangeFlags::improved;
+    }
+    mathConvertionPM.addPass(
+        mlir::createConvertComplexToStandardPass(complexToStandardOptions));
+
     // Convert Math dialect operations into LLVM dialect operations.
     // There is no way to prefer MathToLLVM patterns over MathToLibm
     // patterns (applied below), so we have to run MathToLLVM conversion here.
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 42d9e7ba2418f..d940934f9f6ad 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -113,6 +113,7 @@ void addFIRToLLVMPass(mlir::PassManager &pm,
   options.forceUnifiedTBAATree = useOldAliasTags;
   options.typeDescriptorsRenamedForAssembly =
       !disableCompilerGeneratedNamesConversion;
+  options.ComplexRange = config.ComplexRange;
   addPassConditionally(pm, disableFirToLlvmIr,
                        [&]() { return fir::createFIRToLLVMPass(options); });
   // The dialect conversion framework may leave dead unrealized_conversion_cast
diff --git a/flang/test/Driver/complex-range.f90 b/flang/test/Driver/complex-range.f90
new file mode 100644
index 0000000000000..e5a1ba9068ac9
--- /dev/null
+++ b/flang/test/Driver/complex-range.f90
@@ -0,0 +1,23 @@
+! Test range options for complex multiplication and division.
+
+! RUN: %flang -### -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=RANGE
+
+! RUN: %flang -### -fcomplex-arithmetic=full -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=FULL
+
+! RUN: %flang -### -fcomplex-arithmetic=improved -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=IMPRVD
+
+! RUN: %flang -### -fcomplex-arithmetic=basic -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=BASIC
+
+! RUN: not %flang -### -fcomplex-arithmetic=foo -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=ERR
+
+! RANGE-NOT: -complex-range=
+! FULL: -complex-range=full
+! IMPRVD: -complex-range=improved
+! BASIC: -complex-range=basic
+
+! ERR: error: unsupported argument 'foo' to option '-fcomplex-arithmetic='
diff --git a/flang/test/Integration/complex-div-to-llvm-kind10.f90 b/flang/test/Integration/complex-div-to-llvm-kind10.f90
new file mode 100644
index 0000000000000..04d1f7ed9b024
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm-kind10.f90
@@ -0,0 +1,131 @@
+! Test lowering complex division to llvm ir according to options
+
+! REQUIRES: target=x86_64{{.*}}
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_extended
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { x86_fp80, x86_fp80 }, ptr %[[LHS]], align 16
+! CHECK: %[[LOAD_RHS:.*]] = load { x86_fp80, x86_fp80 }, ptr %[[RHS]], align 16
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract x86_fp80 %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract x86_fp80 %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract x86_fp80 %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract x86_fp80 %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract x86_fp80 %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract x86_fp80 %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract x86_fp80 %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract x86_fp80 %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract x86_fp80 %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract x86_fp80 %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract x86_fp80 %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract x86_fp80 %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq x86_fp80 %[[RHS_REAL_ABS]], 0xK00000000000000000000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq x86_fp80 %[[RHS_IMAG_ABS]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord x86_fp80 %[[LHS_REAL]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord x86_fp80 %[[LHS_IMAG]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 0xK7FFF8000000000000000, x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract x86_fp80 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract x86_fp80 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one x86_fp80 %[[RHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one x86_fp80 %[[RHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq x86_fp80 %[[LHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq x86_fp80 %[[LHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[LHS_REAL_IS_INF]], x86_fp80 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[LHS_IMAG_IS_INF]], x86_fp80 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract x86_fp80 %[[INF_MULTIPLICATOR_1]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract x86_fp80 %[[INF_MULTIPLICATOR_2]], 0xK7FFF8000000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one x86_fp80 %[[LHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one x86_fp80 %[[LHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq x86_fp80 %[[RHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq x86_fp80 %[[RHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[RHS_REAL_IS_INF]], x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[RHS_IMAG_IS_INF]], x86_fp80 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract x86_fp80 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract x86_fp80 %[[ZERO_MULTIPLICATOR_1]], 0xK00000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract x86_fp80 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract x86_fp80 %[[ZERO_MULTIPLICATOR_2]], 0xK00000000000000000000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt x86_fp80 %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], x86_fp80 %[[RESULT_REAL_1]], x86_fp80 %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], x86_fp80 %[[RESULT_IMAG_1]], x86_fp80 %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], x86_fp80 %[[RESULT_REAL_4]], x86_fp80 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], x86_fp80 %[[RESULT_IMAG_4]], x86_fp80 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], x86_fp80 %[[RESULT_REAL_3]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], x86_fp80 %[[RESULT_IMAG_3]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], x86_fp80 %[[INFINITY_RESULT_REAL]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], x86_fp80 %[[INFINITY_RESULT_IMAG]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno x86_fp80 %[[RESULT_REAL]], 0xK00000000000000000000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno x86_fp80 %[[RESULT_IMAG]], 0xK00000000000000000000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_1]], x86_fp80 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_1]], x86_fp80 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { x86_fp80, x86_fp80 } %[[RESULT_1]], x86_fp80 %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { x86_fp80, x86_fp80 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract x86_fp80 %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract x86_fp80 %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract x86_fp80 %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract x86_fp80 %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract x86_fp80 %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract x86_fp80 %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract x86_fp80 %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { x86_fp80, x86_fp80 } %[[RESULT_1]], x86_fp80 %[[IMAG]], 1
+! BASIC: store { x86_fp80, x86_fp80 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! CHECK: ret void
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
diff --git a/flang/test/Integration/complex-div-to-llvm-kind16.f90 b/flang/test/Integration/complex-div-to-llvm-kind16.f90
new file mode 100644
index 0000000000000..887a7972dc1be
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm-kind16.f90
@@ -0,0 +1,131 @@
+! Test lowering complex division to llvm ir according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_quad
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { fp128, fp128 }, ptr %[[LHS]], align 16
+! CHECK: %[[LOAD_RHS:.*]] = load { fp128, fp128 }, ptr %[[RHS]], align 16
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { fp128, fp128 } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { fp128, fp128 } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { fp128, fp128 } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { fp128, fp128 } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract fp128 %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract fp128 %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract fp128 %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract fp128 %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract fp128 %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract fp128 %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract fp128 %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract fp128 %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract fp128 %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract fp128 %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract fp128 %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract fp128 %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq fp128 %[[RHS_REAL_ABS]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord fp128 %[[LHS_REAL]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord fp128 %[[LHS_IMAG]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call fp128 @llvm.copysign.f128(fp128 0xL00000000000000007FFF000000000000, fp128 %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract fp128 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract fp128 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one fp128 %[[RHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq fp128 %[[LHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq fp128 %[[LHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[LHS_REAL_IS_INF]], fp128 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[LHS_IMAG_IS_INF]], fp128 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract fp128 %[[INF_MULTIPLICATOR_1]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract fp128 %[[INF_MULTIPLICATOR_2]], 0xL00000000000000007FFF000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one fp128 %[[LHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one fp128 %[[LHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq fp128 %[[RHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[RHS_REAL_IS_INF]], fp128 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[RHS_IMAG_IS_INF]], fp128 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract fp128 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract fp128 %[[ZERO_MULTIPLICATOR_1]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract fp128 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract fp128 %[[ZERO_MULTIPLICATOR_2]], 0xL00000000000000000000000000000000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt fp128 %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], fp128 %[[RESULT_REAL_1]], fp128 %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], fp128 %[[RESULT_IMAG_1]], fp128 %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], fp128 %[[RESULT_REAL_4]], fp128 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], fp128 %[[RESULT_IMAG_4]], fp128 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], fp128 %[[RESULT_REAL_3]], fp128 %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], fp128 %[[RESULT_IMAG_3]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], fp128 %[[INFINITY_RESULT_REAL]], fp128 %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], fp128 %[[INFINITY_RESULT_IMAG]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno fp128 %[[RESULT_REAL]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno fp128 %[[RESULT_IMAG]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], fp128 %[[RESULT_REAL_SPECIAL_CASE_1]], fp128 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_1]], fp128 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { fp128, fp128 } poison, fp128 %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { fp128, fp128 } %[[RESULT_1]], fp128 %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { fp128, fp128 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract fp128 %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract fp128 %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract fp128 %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract fp128 %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract fp128 %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract fp128 %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract fp128 %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { fp128, fp128 } poison, fp128 %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { fp128, fp128 } %[[RESULT_1]], fp128 %[[IMAG]], 1
+! BASIC: store { fp128, fp128 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! CHECK: ret void
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
diff --git a/flang/test/Integration/complex-div-to-llvm.f90 b/flang/test/Integration/complex-div-to-llvm.f90
new file mode 100644
index 0000000000000..01782a565f24c
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm.f90
@@ -0,0 +1,508 @@
+! Test lowering complex division to llvm ir according to options
+
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_half
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { half, half }, ptr %[[LHS]], align 2
+! CHECK: %[[LOAD_RHS:.*]] = load { half, half }, ptr %[[RHS]], align 2
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { half, half } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { half, half } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { half, half } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { half, half } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract half %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract half %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract half %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract half %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract half %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract half %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract half %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract half %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract half %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract half %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract half %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract half %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq half %[[RHS_REAL_ABS]], 0xH0000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq half %[[RHS_IMAG_ABS]], 0xH0000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord half %[[LHS_REAL]], 0xH0000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord half %[[LHS_IMAG]], 0xH0000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call half @llvm.copysign.f16(half 0xH7C00, half %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract half %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract half %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one half %[[RHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one half %[[RHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq half %[[LHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq half %[[LHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[LHS_REAL_IS_INF]], half %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[LHS_IMAG_IS_INF]], half %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract half %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract half %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract half %[[INF_MULTIPLICATOR_1]], 0xH7C00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract half %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract half %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract half %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract half %[[INF_MULTIPLICATOR_2]], 0xH7C00
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one half %[[LHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one half %[[LHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq half %[[RHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq half %[[RHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[RHS_REAL_IS_INF]], half %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[RHS_IMAG_IS_INF]], half %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract half %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract half %[[ZERO_MULTIPLICATOR_1]], 0xH0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract half %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract half %[[ZERO_MULTIPLICATOR_2]], 0xH0000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt half %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], half %[[RESULT_REAL_1]], half %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], half %[[RESULT_IMAG_1]], half %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], half %[[RESULT_REAL_4]], half %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], half %[[RESULT_IMAG_4]], half %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], half %[[RESULT_REAL_3]], half %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], half %[[RESULT_IMAG_3]], half %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], half %[[INFINITY_RESULT_REAL]], half %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], half %[[INFINITY_RESULT_IMAG]], half %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno half %[[RESULT_REAL]], 0xH0000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno half %[[RESULT_IMAG]], 0xH0000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], half %[[RESULT_REAL_SPECIAL_CASE_1]], half %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], half %[[RESULT_IMAG_SPECIAL_CASE_1]], half %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { half, half } poison, half %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { half, half } %[[RESULT_1]], half %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { half, half } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract half %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract half %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract half %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract half %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract half %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract half %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract half %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { half, half } poison, half %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { half, half } %[[RESULT_1]], half %[[IMAG]], 1
+! BASIC: store { half, half } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! CHECK: ret void
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+
+! CHECK-LABEL: @div_test_bfloat
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { bfloat, bfloat }, ptr %[[LHS]], align 2
+! CHECK: %[[LOAD_RHS:.*]] = load { bfloat, bfloat }, ptr %[[RHS]], align 2
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract bfloat %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract bfloat %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract bfloat %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract bfloat %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract bfloat %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract bfloat %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract bfloat %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract bfloat %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract bfloat %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract bfloat %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract bfloat %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract bfloat %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq bfloat %[[RHS_REAL_ABS]], 0xR0000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq bfloat %[[RHS_IMAG_ABS]], 0xR0000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord bfloat %[[LHS_REAL]], 0xR0000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord bfloat %[[LHS_IMAG]], 0xR0000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call bfloat @llvm.copysign.bf16(bfloat 0xR7F80, bfloat %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract bfloat %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract bfloat %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one bfloat %[[RHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one bfloat %[[RHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq bfloat %[[LHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq bfloat %[[LHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[LHS_REAL_IS_INF]], bfloat %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[LHS_IMAG_IS_INF]], bfloat %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract bfloat %[[INF_MULTIPLICATOR_1]], 0xR7F80
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract bfloat %[[INF_MULTIPLICATOR_2]], 0xR7F80
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one bfloat %[[LHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one bfloat %[[LHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq bfloat %[[RHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq bfloat %[[RHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[RHS_REAL_IS_INF]], bfloat %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[RHS_IMAG_IS_INF]], bfloat %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract bfloat %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract bfloat %[[ZERO_MULTIPLICATOR_1]], 0xR0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract bfloat %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract bfloat %[[ZERO_MULTIPLICATOR_2]], 0xR0000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt bfloat %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], bfloat %[[RESULT_REAL_1]], bfloat %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], bfloat %[[RESULT_IMAG_1]], bfloat %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], bfloat %[[RESULT_REAL_4]], bfloat %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], bfloat %[[RESULT_IMAG_4]], bfloat %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], bfloat %[[RESULT_REAL_3]], bfloat %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], bfloat %[[RESULT_IMAG_3]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], bfloat %[[INFINITY_RESULT_REAL]], bfloat %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], bfloat %[[INFINITY_RESULT_IMAG]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno bfloat %[[RESULT_REAL]], 0xR0000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno bfloat %[[RESULT_IMAG]], 0xR0000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], bfloat %[[RESULT_REAL_SPECIAL_CASE_1]], bfloat %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_1]], bfloat %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { bfloat, bfloat } poison, bfloat %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { bfloat, bfloat } %[[RESULT_1]], bfloat %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { bfloat, bfloat } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract bfloat %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract bfloat %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract bfloat %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract bfloat %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract bfloat %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract bfloat %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract bfloat %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { bfloat, bfloat } poison, bfloat %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { bfloat, bfloat } %[[RESULT_1]], bfloat %[[IMAG]], 1
+! BASIC: store { bfloat, bfloat } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! CHECK: ret void
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+
+! CHECK-LABEL: @div_test_single
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { float, float }, ptr %[[LHS]], align 4
+! CHECK: %[[LOAD_RHS:.*]] = load { float, float }, ptr %[[RHS]], align 4
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { float, float } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { float, float } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { float, float } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { float, float } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract float %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract float %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract float %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract float %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract float %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract float %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract float %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract float %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract float %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract float %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract float %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract float %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq float %[[RHS_REAL_ABS]], 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq float %[[RHS_IMAG_ABS]], 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord float %[[LHS_REAL]], 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord float %[[LHS_IMAG]], 0.000000e+00
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call float @llvm.copysign.f32(float 0x7FF0000000000000, float %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract float %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract float %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one float %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one float %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq float %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq float %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[LHS_REAL_IS_INF]], float %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[LHS_IMAG_IS_INF]], float %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract float %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract float %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract float %[[INF_MULTIPLICATOR_1]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract float %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract float %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract float %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract float %[[INF_MULTIPLICATOR_2]], 0x7FF0000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one float %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one float %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq float %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq float %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[RHS_REAL_IS_INF]], float %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[RHS_IMAG_IS_INF]], float %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract float %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract float %[[ZERO_MULTIPLICATOR_1]], 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract float %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract float %[[ZERO_MULTIPLICATOR_2]], 0.000000e+00
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt float %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], float %[[RESULT_REAL_1]], float %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], float %[[RESULT_IMAG_1]], float %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], float %[[RESULT_REAL_4]], float %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], float %[[RESULT_IMAG_4]], float %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], float %[[RESULT_REAL_3]], float %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], float %[[RESULT_IMAG_3]], float %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], float %[[INFINITY_RESULT_REAL]], float %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], float %[[INFINITY_RESULT_IMAG]], float %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno float %[[RESULT_REAL]], 0.000000e+00
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno float %[[RESULT_IMAG]], 0.000000e+00
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], float %[[RESULT_REAL_SPECIAL_CASE_1]], float %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], float %[[RESULT_IMAG_SPECIAL_CASE_1]], float %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { float, float } poison, float %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT_1]], float %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { float, float } %[[RESULT_2]], ptr %[[RET]], align 4
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract float %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract float %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract float %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract float %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract float %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract float %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract float %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { float, float } poison, float %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT_1]], float %[[IMAG]], 1
+! BASIC: store { float, float } %[[RESULT_2]], ptr %[[RET]], align 4
+
+! CHECK: ret void
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+
+! CHECK-LABEL: @div_test_double
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { double, double }, ptr %[[LHS]], align 8
+! CHECK: %[[LOAD_RHS:.*]] = load { double, double }, ptr %[[RHS]], align 8
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { double, double } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { double, double } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { double, double } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { double, double } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract double %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract double %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract double %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract double %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract double %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract double %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract double %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract double %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract double %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract double %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract double %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract double %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq double %[[RHS_REAL_ABS]], 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq double %[[RHS_IMAG_ABS]], 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord double %[[LHS_REAL]], 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord double %[[LHS_IMAG]], 0.000000e+00
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call double @llvm.copysign.f64(double 0x7FF0000000000000, double %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract double %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract double %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one double %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one double %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq double %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq double %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[LHS_REAL_IS_INF]], double %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[LHS_IMAG_IS_INF]], double %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract double %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract double %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract double %[[INF_MULTIPLICATOR_1]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract double %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract double %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract double %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract double %[[INF_MULTIPLICATOR_2]], 0x7FF0000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one double %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one double %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq double %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq double %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[RHS_REAL_IS_INF]], double %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[RHS_IMAG_IS_INF]], double %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract double %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract double %[[ZERO_MULTIPLICATOR_1]], 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract double %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract double %[[ZERO_MULTIPLICATOR_2]], 0.000000e+00
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt double %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], double %[[RESULT_REAL_1]], double %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], double %[[RESULT_IMAG_1]], double %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], double %[[RESULT_REAL_4]], double %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], double %[[RESULT_IMAG_4]], double %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], double %[[RESULT_REAL_3]], double %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], double %[[RESULT_IMAG_3]], double %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], double %[[INFINITY_RESULT_REAL]], double %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], double %[[INFINITY_RESULT_IMAG]], double %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno double %[[RESULT_REAL]], 0.000000e+00
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno double %[[RESULT_IMAG]], 0.000000e+00
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], double %[[RESULT_REAL_SPECIAL_CASE_1]], double %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], double %[[RESULT_IMAG_SPECIAL_CASE_1]], double %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { double, double } poison, double %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { double, double } %[[RESULT_1]], double %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { double, double } %[[RESULT_2]], ptr %[[RET]], align 8
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract double %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract double %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract double %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract double %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract double %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract double %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract double %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { double, double } poison, double %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { double, double } %[[RESULT_1]], double %[[IMAG]], 1
+! BASIC: store { double, double } %[[RESULT_2]], ptr %[[RET]], align 8
+
+! CHECK: ret void
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
new file mode 100644
index 0000000000000..0e219e30ec94b
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
@@ -0,0 +1,34 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: target=x86_64{{.*}}
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+! CHECK-LABEL: @_QPdiv_test_extended
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f80>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEa"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEb"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEc"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f80>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f80>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_RET:.*]] = fir.call @__divxc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f80, f80, f80, f80) -> complex<f80>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f80>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f80>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f80>, !fir.ref<complex<f80>>
+! CHECK: return
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
new file mode 100644
index 0000000000000..3e158b0bed998
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
@@ -0,0 +1,35 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_quad
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f128>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEa"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEb"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEc"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f128>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f128>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_RET:.*]] = fir.call @__divtc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f128, f128, f128, f128) -> complex<f128>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f128>, !fir.ref<complex<f128>>
+! CHECK: return
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
\ No newline at end of file
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90
new file mode 100644
index 0000000000000..b488bfde4ee85
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90
@@ -0,0 +1,94 @@
+! Test lowering of complex division according to options
+
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_half
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f16>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEa"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEb"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEc"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f16>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f16>>
+! CHECK: %[[VAL_9:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f16>
+! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : complex<f16>, !fir.ref<complex<f16>>
+! CHECK: return
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+
+! CHECK-LABEL: @_QPdiv_test_bfloat
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<bf16>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEa"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEb"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEc"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<bf16>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<bf16>>
+! CHECK: %[[VAL_9:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<bf16>
+! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : complex<bf16>, !fir.ref<complex<bf16>>
+! CHECK: return
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+
+! CHECK-LABEL: @_QPdiv_test_single
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f32>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEa"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEb"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEc"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f32>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_RET:.*]] = fir.call @__divsc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f32, f32, f32, f32) -> complex<f32>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f32>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f32>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f32>, !fir.ref<complex<f32>>
+! CHECK: return
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+
+! CHECK-LABEL: @_QPdiv_test_double
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f64>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEa"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEb"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEc"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f64>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f64>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_RET:.*]] = fir.call @__divdc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f64, f64, f64, f64) -> complex<f64>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f64>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f64>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f64>, !fir.ref<complex<f64>>
+! CHECK: return
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 59372a8eb58ed..2ccbe8b96d805 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -276,6 +276,12 @@ static llvm::cl::opt<bool>
                                      "leading dimension will be repacked"),
                       llvm::cl::init(true));
 
+static llvm::cl::opt<std::string> complexRange(
+    "complex-range",
+    llvm::cl::desc("Controls the various implementations for complex "
+                   "multiplication and division [full|improved|basic]"),
+    llvm::cl::init(""));
+
 #define FLANG_EXCLUDE_CODEGEN
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
@@ -437,6 +443,8 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setSkipExternalRttiDefinition(skipExternalRttiDefinition);
   if (enableCUDA)
     loweringOptions.setCUDARuntimeCheck(true);
+  if (complexRange == "improved" || complexRange == "basic")
+    loweringOptions.setComplexDivisionToRuntime(false);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   Fortran::frontend::TargetOptions targetOpts;
   Fortran::frontend::CodeGenOptions cgOpts;

>From 60e0cd9ece16f0f29ce2525d72e2ba37c0ae8447 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 10:19:12 +0900
Subject: [PATCH 2/6] Add help text

---
 clang/include/clang/Driver/Options.td | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 58209ceb5dc54..a1350e69a202e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1025,7 +1025,8 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
-  NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
+  NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>,
+  HelpText<"Controls the calculation methods of complex number multiplication and division.">;
 
 def complex_range_EQ : Joined<["-"], "complex-range=">, Group<f_Group>,
   Visibility<[CC1Option, FC1Option]>,

>From bddb2d33369ca4bdec50c3e927004555a7940d9a Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 12:01:16 +0900
Subject: [PATCH 3/6] Move common functions to CommonArgs.cpp

---
 clang/include/clang/Driver/CommonArgs.h    |  6 +++++
 clang/lib/Driver/ToolChains/Clang.cpp      | 27 ----------------------
 clang/lib/Driver/ToolChains/CommonArgs.cpp | 27 ++++++++++++++++++++++
 clang/lib/Driver/ToolChains/Flang.cpp      | 24 -------------------
 4 files changed, 33 insertions(+), 51 deletions(-)

diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index 26aa3ccf84786..cd1f3b2eb36ea 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -283,6 +283,12 @@ StringRef parseMPreferVectorWidthOption(clang::DiagnosticsEngine &Diags,
 StringRef parseMRecipOption(clang::DiagnosticsEngine &Diags,
                             const llvm::opt::ArgList &Args);
 
+// Convert ComplexRangeKind to a string that can be passed as a frontend option.
+std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range);
+
+// Render a frontend option corresponding to ComplexRangeKind.
+std::string RenderComplexRangeOption(LangOptions::ComplexRangeKind Range);
+
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index fea4ee909ff46..5d42ce56926d8 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2740,25 +2740,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
   }
 }
 
-static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
-  switch (Range) {
-  case LangOptions::ComplexRangeKind::CX_Full:
-    return "full";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Basic:
-    return "basic";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Improved:
-    return "improved";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Promoted:
-    return "promoted";
-    break;
-  default:
-    return "";
-  }
-}
-
 static std::string ComplexArithmeticStr(LangOptions::ComplexRangeKind Range) {
   return (Range == LangOptions::ComplexRangeKind::CX_None)
              ? ""
@@ -2772,14 +2753,6 @@ static void EmitComplexRangeDiag(const Driver &D, std::string str1,
   }
 }
 
-static std::string
-RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
-  if (!ComplexRangeStr.empty())
-    return "-complex-range=" + ComplexRangeStr;
-  return ComplexRangeStr;
-}
-
 static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
                                        bool OFastEnabled, const ArgList &Args,
                                        ArgStringList &CmdArgs,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..a6735e5e412d4 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -3409,3 +3409,30 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
 
   return Out;
 }
+
+std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Promoted:
+    return "promoted";
+    break;
+  default:
+    return "";
+  }
+}
+
+std::string
+tools::RenderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index ba5db7e3aba43..b5767918b86d1 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -595,30 +595,6 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
-static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
-  switch (Range) {
-  case LangOptions::ComplexRangeKind::CX_Full:
-    return "full";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Improved:
-    return "improved";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Basic:
-    return "basic";
-    break;
-  default:
-    return "";
-  }
-}
-
-static std::string
-RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
-  if (!ComplexRangeStr.empty())
-    return "-complex-range=" + ComplexRangeStr;
-  return ComplexRangeStr;
-}
-
 static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   StringRef FPContract;

>From 65c522ccd201fd48c481990ac0b1ec9b8be6d163 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 12:01:54 +0900
Subject: [PATCH 4/6] Minor fixes

---
 flang/lib/Frontend/CompilerInvocation.cpp              | 2 +-
 flang/lib/Lower/ConvertExprToHLFIR.cpp                 | 5 +++--
 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 86ec410b1f70f..910e0f5dcebd5 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -485,7 +485,7 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
 
   parseDoConcurrentMapping(opts, args, diags);
 
-  if (const auto *arg =
+  if (const llvm::opt::Arg *arg =
           args.getLastArg(clang::driver::options::OPT_complex_range_EQ)) {
     llvm::StringRef argValue = llvm::StringRef(arg->getValue());
     if (argValue == "full") {
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index cb338618dbf3b..9689f920840fb 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1070,12 +1070,13 @@ struct BinaryOp<Fortran::evaluate::Divide<
     // TODO: Ideally, complex number division operations should always be
     // lowered to MLIR. However, converting them to the runtime via MLIR causes
     // ABI issues.
-    if (builder.getComplexDivisionToRuntimeFlag())
+    if (builder.getComplexDivisionToRuntimeFlag()) {
       return hlfir::EntityWithAttributes{
           fir::genDivC(builder, loc, ty, lhs, rhs)};
-    else
+    } else {
       return hlfir::EntityWithAttributes{
           builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
+    }
   }
 };
 
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
index 3e158b0bed998..fe4a7256a4a16 100644
--- a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
@@ -32,4 +32,4 @@
 subroutine div_test_quad(a,b,c)
   complex(kind=16) :: a, b, c
   a = b / c
-end subroutine div_test_quad
\ No newline at end of file
+end subroutine div_test_quad

>From 528e1defac6714baad3f80a6e031fe02fb18011c Mon Sep 17 00:00:00 2001
From: Shunsuke Watanabe <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 17:41:45 +0900
Subject: [PATCH 5/6] Update flang/include/flang/Optimizer/Builder/FIRBuilder.h

Co-authored-by: Tarun Prabhu <tarunprabhu at gmail.com>
---
 flang/include/flang/Optimizer/Builder/FIRBuilder.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index b1513850a9048..004ad00b6f470 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -609,8 +609,8 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return integerOverflowFlags;
   }
 
-  /// Set ComplexDivisionToRuntimeFlag value for whether complex number division
-  /// is lowered to a runtime function by this builder.
+  /// Set ComplexDivisionToRuntimeFlag value. If set to true, complex number
+  /// division is lowered to a runtime function by this builder.
   void setComplexDivisionToRuntimeFlag(bool flag) {
     complexDivisionToRuntimeFlag = flag;
   }

>From eea4a5f402cddae02c31a1217af6930fa4981874 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Mon, 7 Jul 2025 18:27:04 +0900
Subject: [PATCH 6/6] Add lowering information to ComplexOperations.md

---
 flang/docs/ComplexOperations.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/flang/docs/ComplexOperations.md b/flang/docs/ComplexOperations.md
index 03566035c28a3..b7d2ecaa85e1c 100644
--- a/flang/docs/ComplexOperations.md
+++ b/flang/docs/ComplexOperations.md
@@ -74,4 +74,26 @@ The ComplexToStandard dialect does still call into libm for some floating
 point math operations, however these don't have the same ABI issues as the
 complex libm functions.
 
+The flang driver option `-fcomplex-arithmetic=` allows you to select whether
+complex number division is lowered to function calls or to the `complex.div`
+operation in the MLIR complex dialect. To avoid the ABI issues mentioned above,
+the selection between function calls and the `complex.div` is made during the
+lowering phase. The behavior of this option is as follows:
+
+- `basic`: Lowers to `complex.div` and is converted to algebraic formulas. No
+special handling to avoid overflow. NaN and infinite values are not handled.
+- `improved`: Lowers to `complex.div` and is converted to Smith's algorithm. See
+SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962). This
+value offers improved handling for overflow in intermediate calculations, but
+overflow may occur. NaN and infinite values are handled.
+- `full`: Lowers to a call to runtime library functions. Overflow and non-finite
+values are handled by the library implementation. This is the default value.
+
+While [the same option in clang][2] allows specifying `promoted`, this is not
+implemented in Flang. Also, in the case of `improved`, clang does not handle NaN
+and infinite values, but Flang does. These behavioral differences arise because
+the transformation of complex division calculations depends on the implementation
+of ComplexToStandard, which may change in the future.
+
 [1]: https://discourse.llvm.org/t/rfc-change-lowering-of-fortran-math-intrinsics/63971
+[2]: https://clang.llvm.org/docs/UsersManual.html#cmdoption-fcomplex-arithmetic


From flang-commits at lists.llvm.org  Mon Jul  7 05:10:18 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Mon, 07 Jul 2025 05:10:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Fix
 parallel-firstprivate-clause-scalar.f90 test (PR #146932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146932@github.com>
Message-ID: <686bb92a.050a0220.2384a2.d752@mx.google.com>


https://github.com/luporl closed https://github.com/llvm/llvm-project/pull/146932

From flang-commits at lists.llvm.org  Mon Jul  7 06:28:51 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Mon, 07 Jul 2025 06:28:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <686bcb93.050a0220.20a572.11ca@mx.google.com>


eugeneepshteyn wrote:

@luporl , this change seems to cause regression with a test in llvm-test-suite:
```
$ flang -c llvm-test-suite/Fortran/gfortran/regression/ISO_Fortran_binding_1.f90 -O1
flang: llvm-project/llvm/include/llvm/Support/Casting.h:572: decltype(auto) llvm::cast(From &) [To = mlir::arith::ConstantOp, From = mlir::Operation]: Assertion `isa<To>(Val) && "cast<Ty>() argument of incompatible type!"' failed.
```
The same failure can be observed in various apps that use C interop in similar fashion. 

The failure is gone if I revert this patch in my local workspace.

Could you please check?

https://github.com/llvm/llvm-project/pull/146408

From flang-commits at lists.llvm.org  Mon Jul  7 08:19:31 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 07 Jul 2025 08:19:31 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solved an issue with continuation
	lines (PR #146027)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146027@github.com>
Message-ID: <686be583.050a0220.2cf4a7.7157@mx.google.com>


https://github.com/klausler commented:

This would work only when the C comment is at the beginning of a line.

https://github.com/llvm/llvm-project/pull/146027

From flang-commits at lists.llvm.org  Mon Jul  7 08:20:15 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 07 Jul 2025 08:20:15 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solved an issue with continuation
	lines (PR #146027)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146027@github.com>
Message-ID: <686be5af.170a0220.178258.7028@mx.google.com>


klausler wrote:

@ceseo don't add me as a reviewer to patches

https://github.com/llvm/llvm-project/pull/146027

From flang-commits at lists.llvm.org  Mon Jul  7 08:45:13 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Mon, 07 Jul 2025 08:45:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid UB in CharBlock::Compare to C
	string (PR #147329)
Message-ID: <llvm.org/llvm/llvm-project/pull/147329@github.com>


https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/147329

The behaviour of strncmp is undefined if either string pointer is null (https://en.cppreference.com/w/cpp/string/byte/strncmp.html).

I've copied the logic over from Compare to another CharBlock, which had code to avoid UB in memcmp.

The test Preprocessing/kind-suffix.F90 was failing with UBSAN enabled, and now passes.

>From 1ac17232a1699f4e217bbd3c4ea36e42f810c9b8 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Mon, 7 Jul 2025 15:31:26 +0000
Subject: [PATCH] [flang] Avoid UB in CharBlock::Compare to C string

The behaviour of strncmp is undefined if either string pointer
is null (https://en.cppreference.com/w/cpp/string/byte/strncmp.html).

I've copied the logic over from Compare to another CharBlock,
which had code to avoid UB in memcmp.

The test Preprocessing/kind-suffix.F90 was failing
with UBSAN enabled, and now passes.
---
 flang/include/flang/Parser/char-block.h | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Parser/char-block.h b/flang/include/flang/Parser/char-block.h
index 38f4f7b82e1ea..b3b1f04034d3c 100644
--- a/flang/include/flang/Parser/char-block.h
+++ b/flang/include/flang/Parser/char-block.h
@@ -150,7 +150,12 @@ class CharBlock {
 
   int Compare(const char *that) const {
     std::size_t bytes{size()};
-    if (int cmp{std::strncmp(begin(), that, bytes)}) {
+    // strncmp is undefined if either pointer is null.
+    if (!bytes) {
+      return that == nullptr ? 0 : -1;
+    } else if (!that) {
+      return 1;
+    } else if (int cmp{std::strncmp(begin(), that, bytes)}) {
       return cmp;
     }
     return that[bytes] == '\0' ? 0 : -1;


From flang-commits at lists.llvm.org  Mon Jul  7 08:45:22 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Mon, 07 Jul 2025 08:45:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid UB in CharBlock Compare to C
	string (PR #147329)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147329@github.com>
Message-ID: <686beb92.630a0220.39ee07.7bd1@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/147329

From flang-commits at lists.llvm.org  Mon Jul  7 08:45:42 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 08:45:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid UB in CharBlock Compare to C
	string (PR #147329)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147329@github.com>
Message-ID: <686beba6.170a0220.35253b.5e4b@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: David Spickett (DavidSpickett)

<details>
<summary>Changes</summary>

The behaviour of strncmp is undefined if either string pointer is null (https://en.cppreference.com/w/cpp/string/byte/strncmp.html).

I've copied the logic over from Compare to another CharBlock, which had code to avoid UB in memcmp.

The test Preprocessing/kind-suffix.F90 was failing with UBSAN enabled, and now passes.

---
Full diff: https://github.com/llvm/llvm-project/pull/147329.diff


1 Files Affected:

- (modified) flang/include/flang/Parser/char-block.h (+6-1) 


``````````diff
diff --git a/flang/include/flang/Parser/char-block.h b/flang/include/flang/Parser/char-block.h
index 38f4f7b82e1ea..b3b1f04034d3c 100644
--- a/flang/include/flang/Parser/char-block.h
+++ b/flang/include/flang/Parser/char-block.h
@@ -150,7 +150,12 @@ class CharBlock {
 
   int Compare(const char *that) const {
     std::size_t bytes{size()};
-    if (int cmp{std::strncmp(begin(), that, bytes)}) {
+    // strncmp is undefined if either pointer is null.
+    if (!bytes) {
+      return that == nullptr ? 0 : -1;
+    } else if (!that) {
+      return 1;
+    } else if (int cmp{std::strncmp(begin(), that, bytes)}) {
       return cmp;
     }
     return that[bytes] == '\0' ? 0 : -1;

``````````

</details>


https://github.com/llvm/llvm-project/pull/147329

From flang-commits at lists.llvm.org  Mon Jul  7 08:48:41 2025
From: flang-commits at lists.llvm.org (Anchu Rajendran S via flang-commits)
Date: Mon, 07 Jul 2025 08:48:41 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [mlir]
 [mlir][OpenMP][llvm]Atomic control backend (PR #143769)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143769@github.com>
Message-ID: <686bec59.170a0220.14c894.931f@mx.google.com>


https://github.com/anchuraj updated https://github.com/llvm/llvm-project/pull/143769

>From 4816fb9bc0761d8761798300bb419db0072c9d04 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 1/9] [flang][flang-driver] atomic control support

---
 clang/include/clang/Driver/Options.td         | 22 +++++-----
 flang/include/flang/Frontend/TargetOptions.h  |  5 +++
 .../Optimizer/Dialect/Support/FIRContext.h    | 14 +++++++
 flang/lib/Frontend/CompilerInvocation.cpp     | 12 ++++++
 flang/lib/Lower/Bridge.cpp                    |  6 +++
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 40 +++++++++++++++++++
 .../Lower/OpenMP/atomic-control-options.f90   | 37 +++++++++++++++++
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 14 +++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  8 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  9 +++++
 10 files changed, 153 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/atomic-control-options.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8e38f0062140e..c1c7929b0c15d 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2303,21 +2303,21 @@ def fsymbol_partition_EQ : Joined<["-"], "fsymbol-partition=">, Group<f_Group>,
 
 defm atomic_remote_memory : BoolFOption<"atomic-remote-memory",
   LangOpts<"AtomicRemoteMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on remote memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on remote memory">>;
 
 defm atomic_fine_grained_memory : BoolFOption<"atomic-fine-grained-memory",
   LangOpts<"AtomicFineGrainedMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on fine-grained memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on fine-grained memory">>;
 
 defm atomic_ignore_denormal_mode : BoolFOption<"atomic-ignore-denormal-mode",
   LangOpts<"AtomicIgnoreDenormalMode">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "Allow">,
-  NegFlag<SetFalse, [], [ClangOption], "Disallow">,
-  BothFlags<[], [ClangOption], " atomic operations to ignore denormal mode">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "Allow">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Disallow">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations to ignore denormal mode">>;
 
 defm memory_profile : OptInCC1FFlag<"memory-profile", "Enable", "Disable", " heap memory profiling">;
 def fmemory_profile_EQ : Joined<["-"], "fmemory-profile=">,
@@ -5314,9 +5314,9 @@ defm amdgpu_precise_memory_op
                   " precise memory mode (AMDGPU only)">;
 
 def munsafe_fp_atomics : Flag<["-"], "munsafe-fp-atomics">,
-  Visibility<[ClangOption, CC1Option]>, Alias<fatomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>, Alias<fatomic_ignore_denormal_mode>;
 def mno_unsafe_fp_atomics : Flag<["-"], "mno-unsafe-fp-atomics">,
-  Visibility<[ClangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, FlangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
 
 def faltivec : Flag<["-"], "faltivec">, Group<f_Group>;
 def fno_altivec : Flag<["-"], "fno-altivec">, Group<f_Group>;
diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 002d8d158abd4..26256fd775f11 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic Control Options for AMD GPU
+  bool amdgpuIgnoreDenormalMode = false;
+  bool amdgpuRemoteMemory = false;
+  bool amdgpuFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 2df14f83c11e1..ac563bcc402c7 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,10 +58,24 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and Getters for atomic control options
+void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
+bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..00f854b905c72 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,6 +496,18 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
+  if (llvm::Triple(opts.triple).isAMDGPU()) {
+    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
+        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+    opts.amdgpuFineGrainedMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_fine_grained_memory,
+        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+    opts.amdgpuRemoteMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_remote_memory,
+        clang::driver::options::OPT_fno_atomic_remote_memory, false);
+  }
+
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
     opts.cpu = a->getValue();
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..acbfad589dfbb 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,6 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
+  if (targetOpts.amdgpuIgnoreDenormalMode)
+    fir::setAmdgpuIgnoreDenormalMode(*module);
+  if (targetOpts.amdgpuFineGrainedMemory)
+    fir::setAmdgpuFineGrainedMemory(*module);
+  if (targetOpts.amdgpuRemoteMemory)
+    fir::setAmdgpuRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 01c0be66d1ecc..b961793dbdfd5 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,6 +88,46 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *amdgpuIgnoreDenormalModeName =
+    "fir.amdgpu.ignore.denormal.mode";
+void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
+    return true;
+  return false;
+}
+
+static constexpr const char *amdgpuFineGrainedMemoryName =
+    "fir.amdgpu.fine.grained.memory";
+void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
+    return true;
+  return false;
+}
+static constexpr const char *amdgpuRemoteMemoryName =
+    "fir.amdgpu.remote.memory";
+void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
+    return true;
+  return false;
+}
+
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
   if (auto attr = mod->getAttrOfType<mlir::StringAttr>(tuneCpuName))
     return attr.getValue();
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
new file mode 100644
index 0000000000000..1eb0f617f365e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o - | FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o - | FileCheck -check-prefix=IGNORE-DENORMAL %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o - | FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o - | FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, B, threads
+    threads = 128
+    A = 0
+    B = 0
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic capture
+        A = A + B
+        B = A
+    !$omp end atomic
+    !$omp end target parallel
+end program test
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 704d0b2220e8a..84887dfd2c6f9 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -54,6 +54,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
+      clausesArgs);
 
   // Override region definition.
   let regions = (region SizedRegion<1>:$region);
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..e377b7aab3e43 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1562,6 +1562,15 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
     omp.yield(%newval : i32)
   }
 
+  // CHECK: omp.atomic.update %[[X]] : memref<i32> {
+  // CHECK-NEXT: (%[[XVAL:.*]]: i32):
+  // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
+  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  omp.atomic.update %x : memref<i32> {
+  ^bb0(%xval:i32):
+    omp.yield(%const:i32)
+  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+
   return
 }
 

>From 63d3b489ce5d1210172f8bc1d3b18d81ff52886d Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:58:07 -0500
Subject: [PATCH 2/9] R2: Making Atomic control non-optional

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   2 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |   2 +-
 mlir/test/Dialect/OpenMP/canonicalize.mlir    |  10 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  46 +++----
 mlir/test/Dialect/OpenMP/ops.mlir             | 127 +++++++++---------
 .../Target/LLVMIR/openmp-llvm-invalid.mlir    |   8 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  98 +++++++-------
 .../openmp-target-nesting-in-host-ops.mlir    |   2 +-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |   2 +-
 9 files changed, 148 insertions(+), 149 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 375e35a146f78..399f7fa6e78f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          OptionalAttr<AtomicControlAttr>:$atomic_control),
+          AtomicControlAttr:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index d69de998346b5..0b2445d0a0d18 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -133,7 +133,7 @@ func.func @atomic_update() {
     %1 = arith.constant 1 : i32
     %2 = arith.addi %arg0, %1  : i32
     omp.yield(%2 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 llvm.mlir.global internal @_QFsEc() : i32 {
diff --git a/mlir/test/Dialect/OpenMP/canonicalize.mlir b/mlir/test/Dialect/OpenMP/canonicalize.mlir
index de6c931ecc5fd..2492010c4d3c7 100644
--- a/mlir/test/Dialect/OpenMP/canonicalize.mlir
+++ b/mlir/test/Dialect/OpenMP/canonicalize.mlir
@@ -4,7 +4,7 @@ func.func @update_no_op(%x : memref<i32>) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -17,7 +17,7 @@ func.func @update_write_op(%x : memref<i32>, %value: i32) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%value : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -33,7 +33,7 @@ func.func @update_normal(%x : memref<i32>, %value: i32) {
   ^bb0(%xval : i32):
     %newval = arith.addi %xval, %value : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -50,7 +50,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.addi %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -65,7 +65,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.muli %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..afc3830286f30 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1007,7 +1007,7 @@ func.func @omp_atomic_update1(%x: memref<i32>, %expr: f32) {
   ^bb0(%xval: f32):
     %newval = llvm.fadd %xval, %expr : f32
     omp.yield (%newval : f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1020,7 +1020,7 @@ func.func @omp_atomic_update2(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.terminator
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1032,7 +1032,7 @@ func.func @omp_atomic_update3(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1044,7 +1044,7 @@ func.func @omp_atomic_update4(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1056,7 +1056,7 @@ func.func @omp_atomic_update5(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1068,7 +1068,7 @@ func.func @omp_atomic_update6(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval, %expr : i32, i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1080,7 +1080,7 @@ func.func @omp_atomic_update7(%x: memref<i32>, %expr: i32, %y: f32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%y: f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1092,7 +1092,7 @@ func.func @omp_atomic_update8(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32, %tmp: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1104,7 +1104,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1116,7 +1116,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1128,7 +1128,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1164,12 +1164,12 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.update %x : memref<i32> {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1197,7 +1197,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1212,7 +1212,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.write %x = %expr : memref<i32>, i32
     omp.terminator
   }
@@ -1240,7 +1240,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %y : memref<i32>, memref<i32>, i32
     omp.terminator
   }
@@ -1256,7 +1256,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
 }
@@ -1281,7 +1281,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1296,7 +1296,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1311,7 +1311,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1326,7 +1326,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1341,7 +1341,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1356,7 +1356,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x memory_order(seq_cst) : memref<i32>, memref<i32>, i32
   }
   return
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index e377b7aab3e43..5aa7befb2dc26 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
@@ -1380,57 +1380,57 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i1):
     %newval = llvm.and %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.shl %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.intr.smax(%[[XVAL]], %[[EXPR]]) : (i32, i32) -> i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.intr.smax(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.icmp "eq" %[[XVAL]], %[[EXPRBOOL]] : i1
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i1)
-  // }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %xBool : memref<i1> {
   ^bb0(%xval: i1):
     %newval = llvm.icmp "eq" %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%xval:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   %const = arith.constant 42 : i32
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1440,7 +1440,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1450,7 +1450,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1460,7 +1460,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1470,7 +1470,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1480,7 +1480,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1490,7 +1490,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1500,7 +1500,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1510,7 +1510,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1520,7 +1520,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1530,7 +1530,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(release) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1540,7 +1540,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(relaxed) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1550,7 +1550,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1560,17 +1560,16 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
-
+  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   return
 }
 
@@ -1582,7 +1581,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture{
@@ -1590,7 +1589,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   // CHECK: omp.atomic.capture {
@@ -1599,7 +1598,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: }
   omp.atomic.capture{
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
@@ -1607,7 +1606,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
@@ -1623,7 +1622,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(none) {
@@ -1631,7 +1630,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1640,7 +1639,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended) {
@@ -1648,7 +1647,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1657,7 +1656,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended) {
@@ -1665,7 +1664,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1674,7 +1673,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(nonspeculative) {
@@ -1682,7 +1681,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1691,7 +1690,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(speculative) {
@@ -1699,7 +1698,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1708,7 +1707,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, nonspeculative) {
@@ -1716,7 +1715,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1725,7 +1724,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, nonspeculative) {
@@ -1733,7 +1732,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1742,7 +1741,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, speculative) {
@@ -1750,7 +1749,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1759,7 +1758,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) {
@@ -1767,7 +1766,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1776,7 +1775,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(seq_cst) {
@@ -1784,7 +1783,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1793,7 +1792,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acq_rel) {
@@ -1801,7 +1800,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1810,7 +1809,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acquire) {
@@ -1818,7 +1817,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1827,7 +1826,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(release) {
@@ -1835,7 +1834,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1844,7 +1843,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(relaxed) {
@@ -1852,7 +1851,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1861,7 +1860,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) memory_order(seq_cst) {
@@ -1869,7 +1868,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -2653,14 +2652,14 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
   // CHECK: omp.atomic.update %[[x]] : !llvm.ptr {
   // CHECK-NEXT: ^{{[[:alnum:]]+}}(%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   %val = llvm.load %x : !llvm.ptr -> i32
   omp.atomic.write %v = %val : !llvm.ptr, i32
   omp.atomic.update %x : !llvm.ptr {
     ^bb0(%xval: i32):
       omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index 41bc5c4ba525f..00f53864b5fa7 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -9,7 +9,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
     %t2 = llvm.sdiv %t1, %expr : i32
     %newval = llvm.add %xval, %t2 : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -24,7 +24,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.mul %expr, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -41,7 +41,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
     ^bb0(%xval: i32):
       %newval = llvm.mul %expr, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
@@ -59,7 +59,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
       %t2 = llvm.sdiv %t1, %expr : i32
       %newval = llvm.add %xval, %t2 : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 32f0ba5b105ff..ab7f756bf95f9 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,13 +1462,13 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1563,7 +1563,7 @@ llvm.func @_QPomp_atomic_update_complex() {
       %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
       %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
       omp.yield(%16 : !llvm.struct<(f32, f32)>)
-    }
+    } {atomic_control = #omp.atomic_control<>}
    llvm.return
 }
 
@@ -1623,7 +1623,7 @@ llvm.func @_QPomp_atomic_capture_complex() {
         %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
         %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
         omp.yield(%20 : !llvm.struct<(f32, f32)>)
-      }
+      } {atomic_control = #omp.atomic_control<>}
       omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.ptr, !llvm.struct<(f32, f32)>
     }
     llvm.return
@@ -1665,7 +1665,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %expr, %xval : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1683,7 +1683,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1701,7 +1701,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
   // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
   // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
@@ -1710,7 +1710,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1732,7 +1732,7 @@ llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
     %2 = llvm.fadd %1, %0 : f32
     %3 = llvm.fptosi %2 : f32 to i32
     omp.yield(%3 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1751,7 +1751,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1763,7 +1763,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1775,7 +1775,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1787,7 +1787,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1799,7 +1799,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1814,7 +1814,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1829,7 +1829,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1844,7 +1844,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1859,7 +1859,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1874,7 +1874,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1889,7 +1889,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1904,7 +1904,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1919,7 +1919,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1934,7 +1934,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1949,7 +1949,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1964,7 +1964,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -1979,7 +1979,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -2001,7 +2001,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw sub ptr %[[x]], i32 %[[expr]] monotonic
@@ -2012,7 +2012,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw and ptr %[[x]], i32 %[[expr]] monotonic
@@ -2023,7 +2023,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw or ptr %[[x]], i32 %[[expr]] monotonic
@@ -2034,7 +2034,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw xor ptr %[[x]], i32 %[[expr]] monotonic
@@ -2045,7 +2045,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2060,7 +2060,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2075,7 +2075,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2090,7 +2090,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2105,7 +2105,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2120,7 +2120,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2135,7 +2135,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2150,7 +2150,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2165,7 +2165,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2180,7 +2180,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2195,7 +2195,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2211,7 +2211,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2227,7 +2227,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -2265,7 +2265,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acquire
@@ -2276,7 +2276,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] release
@@ -2287,7 +2287,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
@@ -2298,7 +2298,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acq_rel
@@ -2309,7 +2309,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -3055,7 +3055,7 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
index cbf273b887bc7..a9a4ffd533136 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
@@ -110,7 +110,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
 
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 29725a02c075a..0ec244bf28f33 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -19,7 +19,7 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   ^bb0(%arg0: i32):
     %result = llvm.add %arg0, %expr : i32
     omp.yield(%result : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   llvm.return
 }

>From 0b1b23ac98e9486d79b69092698623782ee96921 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 3/9] R3: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h       |  2 +-
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  4 ++--
 flang/lib/Optimizer/Dialect/Support/FIRContext.cpp | 14 +++-----------
 flang/test/Fir/convert-to-llvm-openmp-and-fir.fir  |  4 ++--
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td |  3 ++-
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 26256fd775f11..dd29b4e3e8b08 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,7 +54,7 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic Control Options for AMD GPU
+  /// Atomic control options for AMD gpu
   bool amdgpuIgnoreDenormalMode = false;
   bool amdgpuRemoteMemory = false;
   bool amdgpuFineGrainedMemory = false;
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac563bcc402c7..ac5fd33874db8 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,7 +58,7 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and Getters for atomic control options
+// Setters and getters for atomic control options.
 void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
@@ -70,7 +70,7 @@ bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index b961793dbdfd5..297a4c7a39e44 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -96,10 +96,7 @@ void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
 }
 
 static constexpr const char *amdgpuFineGrainedMemoryName =
@@ -110,10 +107,7 @@ void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
 }
 static constexpr const char *amdgpuRemoteMemoryName =
     "fir.amdgpu.remote.memory";
@@ -123,9 +117,7 @@ void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 24e5cad84b709..47c00d489c200 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -619,7 +619,7 @@ func.func @_QPs() {
     %c1_i32 = arith.constant 1 : i32
     %1 = arith.addi %arg0, %c1_i32 : i32
     omp.yield(%1 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 fir.global internal @_QFsEc : i32 {
@@ -634,7 +634,7 @@ fir.global internal @_QFsEc : i32 {
 // CHECK:      %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:      %[[OUT_VAL:.*]] = llvm.add %[[IN_VAL]], %[[CONST_1]]  : i32
 // CHECK:      omp.yield(%[[OUT_VAL]] : i32)
-// CHECK:    }
+// CHECK:    } {atomic_control = #omp.atomic_control<>}
 // CHECK:    llvm.return
 // CHECK:  }
 // CHECK:  llvm.mlir.global internal @[[GLOBAL]]() {{.*}} : i32 {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 84887dfd2c6f9..61861173ae875 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -58,7 +58,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From 565d104ba013acd7f7e64cc989e774f0f1f34e97 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Mon, 23 Jun 2025 12:14:19 -0500
Subject: [PATCH 4/9] R4: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  6 ++--
 .../Optimizer/Dialect/Support/FIRContext.h    | 18 ++++------
 flang/lib/Frontend/CompilerInvocation.cpp     | 20 +++++------
 flang/lib/Lower/Bridge.cpp                    | 12 +++----
 flang/lib/Lower/OpenMP/Atomic.cpp             |  9 ++++-
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 34 +++++++++----------
 .../Lower/OpenMP/atomic-control-options.f90   | 16 ++++-----
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     |  6 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  4 +--
 9 files changed, 61 insertions(+), 64 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index dd29b4e3e8b08..6ef5338acdc6e 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -55,9 +55,9 @@ class TargetOptions {
   bool asmVerbose = false;
 
   /// Atomic control options for AMD gpu
-  bool amdgpuIgnoreDenormalMode = false;
-  bool amdgpuRemoteMemory = false;
-  bool amdgpuFineGrainedMemory = false;
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac5fd33874db8..273b4683b6d9c 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -59,23 +59,17 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
 // Setters and getters for atomic control options.
-void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
-bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+void setIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getIgnoreDenormalMode(mlir::ModuleOp mod);
+void setFineGrainedMemory(mlir::ModuleOp mod);
+bool getFineGrainedMemory(mlir::ModuleOp mod);
+void setRemoteMemory(mlir::ModuleOp mod);
+bool getRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set atomic control options for AMD gpu.
-void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
-                                   bool amdgpuIgnoreDenormalMode,
-                                   bool amdgpuNoFineGrainedMemory,
-                                   bool amdgpuNoRemoteMemory);
-
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 00f854b905c72..63170a6df6960 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,17 +496,15 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  if (llvm::Triple(opts.triple).isAMDGPU()) {
-    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
-        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
-        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-    opts.amdgpuFineGrainedMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_fine_grained_memory,
-        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-    opts.amdgpuRemoteMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_remote_memory,
-        clang::driver::options::OPT_fno_atomic_remote_memory, false);
-  }
+  opts.ignoreDenormalMode = args.hasFlag(
+      clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+      clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+  opts.fineGrainedMemory = args.hasFlag(
+      clang::driver::options::OPT_fatomic_fine_grained_memory,
+      clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+  opts.remoteMemory =
+      args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
+                   clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index acbfad589dfbb..af6e72aa8cd10 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.amdgpuIgnoreDenormalMode)
-    fir::setAmdgpuIgnoreDenormalMode(*module);
-  if (targetOpts.amdgpuFineGrainedMemory)
-    fir::setAmdgpuFineGrainedMemory(*module);
-  if (targetOpts.amdgpuRemoteMemory)
-    fir::setAmdgpuRemoteMemory(*module);
+  if (targetOpts.ignoreDenormalMode)
+    fir::setIgnoreDenormalMode(*module);
+  if (targetOpts.fineGrainedMemory)
+    fir::setFineGrainedMemory(*module);
+  if (targetOpts.remoteMemory)
+    fir::setRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 2ab91b239a3cc..cbc850d67cc4d 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -634,9 +634,16 @@ genAtomicUpdate(lower::AbstractConverter &converter,
     }
   }
 
+  mlir::ModuleOp module = builder.getModule();
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      mlir::omp::AtomicControlAttr::get(
+          builder.getContext(), fir::getAtomicIgnoreDenormalMode(module),
+          fir::getAtomicFineGrainedMemory(module),
+          fir::getAtomicRemoteMemory(module));
   builder.restoreInsertionPoint(atomicAt);
   auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+      loc, atomAddr, atomicControlAttr, hint,
+      makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 297a4c7a39e44..e3599204f5e2c 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,36 +88,34 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *amdgpuIgnoreDenormalModeName =
-    "fir.amdgpu.ignore.denormal.mode";
-void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
 }
 
-static constexpr const char *amdgpuFineGrainedMemoryName =
-    "fir.amdgpu.fine.grained.memory";
-void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
+void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
+bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
 }
-static constexpr const char *amdgpuRemoteMemoryName =
-    "fir.amdgpu.remote.memory";
-void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+static constexpr const char *remoteMemoryName = "fir.remote.memory";
+void fir::setRemoteMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
+bool fir::getRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
index 1eb0f617f365e..8f86a151b3713 100644
--- a/flang/test/Lower/OpenMP/atomic-control-options.f90
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -9,25 +9,25 @@ program test
     A = 0
     B = 0
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic
     A =  A + 1
     !$omp end target parallel
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic capture
         A = A + B
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 61861173ae875..0fefe7e017e94 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -62,9 +62,9 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 5aa7befb2dc26..9a72ddfc2989d 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1565,11 +1565,11 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   return
 }
 

>From f5a469317a1446d17bc0fb6feb10849d70718368 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 11:26:03 -0500
Subject: [PATCH 5/9] R5: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  8 +--
 .../Optimizer/Dialect/Support/FIRContext.h    | 19 ++++---
 flang/lib/Frontend/CompilerInvocation.cpp     |  6 +--
 flang/lib/Lower/Bridge.cpp                    |  9 ++--
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 53 ++++++++++++-------
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 30 +++++------
 6 files changed, 71 insertions(+), 54 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 6ef5338acdc6e..f6e5634d5a995 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,10 +54,10 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic control options for AMD gpu
-  bool ignoreDenormalMode = false;
-  bool remoteMemory = false;
-  bool fineGrainedMemory = false;
+  /// Atomic control options
+  bool atomicIgnoreDenormalMode = false;
+  bool atomicRemoteMemory = false;
+  bool atomicFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 273b4683b6d9c..c0b852ccaf83a 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,13 +58,18 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and getters for atomic control options.
-void setIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getIgnoreDenormalMode(mlir::ModuleOp mod);
-void setFineGrainedMemory(mlir::ModuleOp mod);
-bool getFineGrainedMemory(mlir::ModuleOp mod);
-void setRemoteMemory(mlir::ModuleOp mod);
-bool getRemoteMemory(mlir::ModuleOp mod);
+/// Sets that Denormal Mode can be ignored for lowering of floating point atomic operations.
+void setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value);
+/// Gets whether Denormal Mode can be ignored or not for lowering of floating point atomic operations.
+bool getAtomicIgnoreDenormalMode(mlir::ModuleOp mod);
+/// Sets that fine grained memory is used for lowering of atomic operations.
+void setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether fine grained memory is used or not for lowering of atomic operations.
+bool getAtomicFineGrainedMemory(mlir::ModuleOp mod);
+/// Sets that remote memory is used for lowering of atomic operations.
+void setAtomicRemoteMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether remote memory is used or not for lowering of atomic operations.
+bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 63170a6df6960..39021207e897e 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,13 +496,13 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  opts.ignoreDenormalMode = args.hasFlag(
+  opts.atomicIgnoreDenormalMode = args.hasFlag(
       clang::driver::options::OPT_fatomic_ignore_denormal_mode,
       clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-  opts.fineGrainedMemory = args.hasFlag(
+  opts.atomicFineGrainedMemory = args.hasFlag(
       clang::driver::options::OPT_fatomic_fine_grained_memory,
       clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-  opts.remoteMemory =
+  opts.atomicRemoteMemory =
       args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
                    clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index af6e72aa8cd10..f8185f752b8e1 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,9 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.ignoreDenormalMode)
-    fir::setIgnoreDenormalMode(*module);
-  if (targetOpts.fineGrainedMemory)
-    fir::setFineGrainedMemory(*module);
-  if (targetOpts.remoteMemory)
-    fir::setRemoteMemory(*module);
+  fir::setAtomicIgnoreDenormalMode(*module, targetOpts.atomicIgnoreDenormalMode);
+  fir::setAtomicFineGrainedMemory(*module, targetOpts.atomicFineGrainedMemory);
+  fir::setAtomicRemoteMemory(*module, targetOpts.atomicRemoteMemory);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index e3599204f5e2c..885efa653fa23 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,34 +88,49 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *ignoreDenormalModeName =
-    "fir.ignore.denormal.mode";
-void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicIgnoreDenormalModeName =
+    "fir.atomic_ignore_denormal_mode";
+void fir::setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicIgnoreDenormalModeName))
+      mod->removeAttr(atomicIgnoreDenormalModeName);
+  }
 }
 
-bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+bool fir::getAtomicIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicIgnoreDenormalModeName);
 }
 
-static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
-void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicFineGrainedMemoryName = "fir.atomic_fine_grained_memory";
+void fir::setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicFineGrainedMemoryName))
+      mod->removeAttr(atomicFineGrainedMemoryName);
+  }
 }
 
-bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
+bool fir::getAtomicFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicFineGrainedMemoryName);
 }
-static constexpr const char *remoteMemoryName = "fir.remote.memory";
-void fir::setRemoteMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicRemoteMemoryName = "fir.atomic_remote_memory";
+void fir::setAtomicRemoteMemory(mlir::ModuleOp mod, bool value) {
+  if(value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if(mod->hasAttr(atomicRemoteMemoryName))
+      mod->removeAttr(atomicRemoteMemoryName);
+  }
 }
 
-bool fir::getRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
+bool fir::getAtomicRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 0fefe7e017e94..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -22,6 +22,21 @@ class OpenMP_Attr<string name, string attrMnemonic, list<Trait> traits = [],
   let mnemonic = attrMnemonic;
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // DeclareTargetAttr
 //===----------------------------------------------------------------------===//
@@ -54,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//

>From 72a9c3adee1401f41493a2f8f2f6717fd2b6b17d Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 6/9] [flang][flang-driver] atomic control support

---
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  6 ++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td | 14 ++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index c0b852ccaf83a..9972fcc345cc5 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -75,6 +75,12 @@ bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 72ce4c6a21cb3..1d2118ffc13e7 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -69,6 +69,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//

>From 5c534945fe6aeb83b890691952e65523e91ae4ff Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 7/9] R3: Addressing review comments

---
 flang/include/flang/Optimizer/Dialect/Support/FIRContext.h | 2 +-
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td         | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 9972fcc345cc5..0e8451fe0b409 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -75,7 +75,7 @@ bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 1d2118ffc13e7..55761328bd109 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -73,7 +73,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From bc7e45218839bcfb429cf647d22c360271b7c898 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:24:46 -0500
Subject: [PATCH 8/9] backend changes

---
 flang/test/Driver/atomic-control-options.f90  | 19 ++++++++++++
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       | 14 +++++++--
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 31 +++++++++++++++----
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 31 ++++++++++++++++---
 4 files changed, 82 insertions(+), 13 deletions(-)
 create mode 100644 flang/test/Driver/atomic-control-options.f90

diff --git a/flang/test/Driver/atomic-control-options.f90 b/flang/test/Driver/atomic-control-options.f90
new file mode 100644
index 0000000000000..5cbecde6c521e
--- /dev/null
+++ b/flang/test/Driver/atomic-control-options.f90
@@ -0,0 +1,19 @@
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o -|FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o -|FileCheck -check-prefix=IGNORE-DENORMAL-MODE %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o -|FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o -|FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, threads
+    threads = 128
+    A = 0
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+end program test
+
+!UNSAFE-FP-ATOMICS: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.ignore.denormal.mode !{{.*}}, !amdgpu.no.fine.grained.memory !{{.*}}, !amdgpu.no.remote.memory !{{.*}}
+!IGNORE-DENORMAL-MODE: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.ignore.denormal.mode !{{.*}}, !amdgpu.no.fine.grained.memory !{{.*}}, !amdgpu.no.remote.memory !{{.*}}
+!FINE-GRAINED-MEMORY: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.no.remote.memory !{{.*}}
+!REMOTE-MEMORY: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.no.fine.grained.memory !{{.*}}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 19a4058b64382..9d9a90879a838 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -3282,7 +3282,9 @@ class OpenMPIRBuilder {
   emitAtomicUpdate(InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
                    AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
                    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX,
-                   bool IsXBinopExpr);
+                   bool IsXBinopExpr, bool IsAmdgpuIgnoreDenormalMode,
+                   bool IsAmdgpuNoFineGrainedMemory,
+                   bool IsAmdgpuNoRemoteMemory);
 
   /// Emit the binary op. described by \p RMWOp, using \p Src1 and \p Src2 .
   ///
@@ -3355,7 +3357,10 @@ class OpenMPIRBuilder {
   LLVM_ABI InsertPointOrErrorTy createAtomicUpdate(
       const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
       Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-      AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr);
+      AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
+      bool IsAmdgpuIgnoreDenormalMode = false,
+      bool IsAmdgpuNoFineGrainedMemory = false,
+      bool IsAmdgpuNoRemoteMemory = false);
 
   /// Emit atomic update for constructs: --- Only Scalar data types
   /// V = X; X = X BinOp Expr ,
@@ -3390,7 +3395,10 @@ class OpenMPIRBuilder {
       const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
       AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
       AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
-      bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr);
+      bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
+      bool IsAmdgpuIgnoreDenormalMode = false,
+      bool IsAmdgpuNoFineGrainedMemory = false,
+      bool IsAmdgpuNoRemoteMemory = false);
 
   /// Emit atomic compare for constructs: --- Only scalar data types
   /// cond-expr-stmt:
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 85451b1233f96..c00df4b2e24c5 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -8960,7 +8960,9 @@ OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
 OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
     const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
     Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
+    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
+    bool IsAmdgpuIgnoreDenormalMode, bool IsNoFineGrainedMemory,
+    bool IsNoRemoteMemory) {
   assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
   if (!updateToLocation(Loc))
     return Loc.IP;
@@ -8980,7 +8982,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
 
   Expected<std::pair<Value *, Value *>> AtomicResult =
       emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp,
-                       X.IsVolatile, IsXBinopExpr);
+                       X.IsVolatile, IsXBinopExpr, IsAmdgpuIgnoreDenormalMode,
+                       IsNoFineGrainedMemory, IsNoRemoteMemory);
   if (!AtomicResult)
     return AtomicResult.takeError();
   checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
@@ -9027,7 +9030,9 @@ Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
 Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
     InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
     AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) {
+    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,
+    bool IsAmdgpuIgnoreDenormalMode, bool IsAmdgpuNoFineGrainedMemory,
+    bool IsAmdgpuNoRemoteMemory) {
   // TODO: handle the case where XElemTy is not byte-sized or not a power of 2
   // or a complex datatype.
   bool emitRMWOp = false;
@@ -9050,7 +9055,18 @@ Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
 
   std::pair<Value *, Value *> Res;
   if (emitRMWOp) {
-    Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
+    AtomicRMWInst *atomicRMWInst =
+        Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
+    if (IsAmdgpuIgnoreDenormalMode)
+      atomicRMWInst->setMetadata("amdgpu.ignore.denormal.mode",
+                                 llvm::MDNode::get(Builder.getContext(), {}));
+    if (IsAmdgpuNoFineGrainedMemory)
+      atomicRMWInst->setMetadata("amdgpu.no.fine.grained.memory",
+                                 llvm::MDNode::get(Builder.getContext(), {}));
+    if (IsAmdgpuNoRemoteMemory)
+      atomicRMWInst->setMetadata("amdgpu.no.remote.memory",
+                                 llvm::MDNode::get(Builder.getContext(), {}));
+    Res.first = atomicRMWInst;
     // not needed except in case of postfix captures. Generate anyway for
     // consistency with the else part. Will be removed with any DCE pass.
     // AtomicRMWInst::Xchg does not have a coressponding instruction.
@@ -9182,7 +9198,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
     const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
     AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
     AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
-    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
+    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
+    bool IsAmdgpuIgnoreDenormalMode, bool IsAmdgpuNoFineGrainedMemory,
+    bool IsAmdgpuNoRemoteMemory) {
   if (!updateToLocation(Loc))
     return Loc.IP;
 
@@ -9203,7 +9221,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
   AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
   Expected<std::pair<Value *, Value *>> AtomicResult =
       emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
-                       X.IsVolatile, IsXBinopExpr);
+                       X.IsVolatile, IsXBinopExpr, IsAmdgpuIgnoreDenormalMode,
+                       IsAmdgpuNoFineGrainedMemory, IsAmdgpuNoRemoteMemory);
   if (!AtomicResult)
     return AtomicResult.takeError();
   Value *CapturedVal =
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b07243c5f843..28d09c1fa65f1 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3242,13 +3242,21 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      opInst.getAtomicControlAttr();
+  bool isAmdgpuIgnoreDenormalMode =
+      atomicControlAttr.getAmdgpuIgnoreDenormalMode();
+  bool isAmdgpuNoFineGrainedMemory =
+      !atomicControlAttr.getAmdgpuFineGrainedMemory();
+  bool isAmdgpuNoRemoteMemory = !atomicControlAttr.getAmdgpuRemoteMemory();
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
-      ompBuilder->createAtomicUpdate(ompLoc, allocaIP, llvmAtomicX, llvmExpr,
-                                     atomicOrdering, binop, updateFn,
-                                     isXBinopExpr);
+      ompBuilder->createAtomicUpdate(
+          ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop,
+          updateFn, isXBinopExpr, isAmdgpuIgnoreDenormalMode,
+          isAmdgpuNoFineGrainedMemory, isAmdgpuNoRemoteMemory);
 
   if (failed(handleError(afterIP, *opInst)))
     return failure();
@@ -3270,6 +3278,7 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
   llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
 
   omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();
+
   omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();
 
   assert((atomicUpdateOp || atomicWriteOp) &&
@@ -3337,13 +3346,27 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
+  bool isAmdgpuIgnoreDenormalMode = false;
+  bool isAmdgpuNoFineGrainedMemory = true;
+  bool isAmdgpuNoRemoteMemory = true;
+  if (atomicUpdateOp) {
+    mlir::omp::AtomicControlAttr atomicControlAttr =
+        atomicUpdateOp.getAtomicControlAttr();
+    isAmdgpuIgnoreDenormalMode =
+        atomicControlAttr.getAmdgpuIgnoreDenormalMode();
+    isAmdgpuNoFineGrainedMemory =
+        !atomicControlAttr.getAmdgpuFineGrainedMemory();
+    isAmdgpuNoRemoteMemory = !atomicControlAttr.getAmdgpuRemoteMemory();
+  }
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
       ompBuilder->createAtomicCapture(
           ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
-          binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr);
+          binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr,
+          isAmdgpuIgnoreDenormalMode, isAmdgpuNoFineGrainedMemory,
+          isAmdgpuNoRemoteMemory);
 
   if (failed(handleError(afterIP, *atomicCaptureOp)))
     return failure();

>From 36f69702f07f8bd71766aa3be626029bbd611bff Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 14:55:43 -0500
Subject: [PATCH 9/9] backend changes updates

---
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 15 -------
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  2 +-
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 45 ++++++++++---------
 mlir/test/Dialect/OpenMP/ops.mlir             |  2 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  2 +-
 5 files changed, 28 insertions(+), 38 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 55761328bd109..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -69,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 399f7fa6e78f0..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          AtomicControlAttr:$atomic_control),
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 28d09c1fa65f1..6825df15ae518 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3223,7 +3223,6 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
 
   llvm::AtomicOrdering atomicOrdering =
       convertAtomicOrdering(opInst.getMemoryOrder());
-
   // Generate update code.
   auto updateFn =
       [&opInst, &moduleTranslation](
@@ -3242,21 +3241,27 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
-  mlir::omp::AtomicControlAttr atomicControlAttr =
-      opInst.getAtomicControlAttr();
-  bool isAmdgpuIgnoreDenormalMode =
-      atomicControlAttr.getAmdgpuIgnoreDenormalMode();
-  bool isAmdgpuNoFineGrainedMemory =
-      !atomicControlAttr.getAmdgpuFineGrainedMemory();
-  bool isAmdgpuNoRemoteMemory = !atomicControlAttr.getAmdgpuRemoteMemory();
+  bool isIgnoreDenormalMode = false;
+  bool isNoFineGrainedMemory = false;
+  bool isNoRemoteMemory = false;
+  if(opInst->hasAttr(opInst.getAtomicControlAttrName())) {
+    mlir::omp::AtomicControlAttr atomicControlAttr =
+        opInst.getAtomicControlAttr();
+    isIgnoreDenormalMode =
+        atomicControlAttr.getIgnoreDenormalMode();
+    isNoFineGrainedMemory =
+        !atomicControlAttr.getFineGrainedMemory();
+    isNoRemoteMemory = !atomicControlAttr.getRemoteMemory();
+    
+  }
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
       ompBuilder->createAtomicUpdate(
           ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop,
-          updateFn, isXBinopExpr, isAmdgpuIgnoreDenormalMode,
-          isAmdgpuNoFineGrainedMemory, isAmdgpuNoRemoteMemory);
+          updateFn, isXBinopExpr, isIgnoreDenormalMode,
+          isNoFineGrainedMemory, isNoRemoteMemory);
 
   if (failed(handleError(afterIP, *opInst)))
     return failure();
@@ -3346,17 +3351,17 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
-  bool isAmdgpuIgnoreDenormalMode = false;
-  bool isAmdgpuNoFineGrainedMemory = true;
-  bool isAmdgpuNoRemoteMemory = true;
+  bool isIgnoreDenormalMode = false;
+  bool isNoFineGrainedMemory = true;
+  bool isNoRemoteMemory = true;
   if (atomicUpdateOp) {
     mlir::omp::AtomicControlAttr atomicControlAttr =
         atomicUpdateOp.getAtomicControlAttr();
-    isAmdgpuIgnoreDenormalMode =
-        atomicControlAttr.getAmdgpuIgnoreDenormalMode();
-    isAmdgpuNoFineGrainedMemory =
-        !atomicControlAttr.getAmdgpuFineGrainedMemory();
-    isAmdgpuNoRemoteMemory = !atomicControlAttr.getAmdgpuRemoteMemory();
+    isIgnoreDenormalMode =
+        atomicControlAttr.getIgnoreDenormalMode();
+    isNoFineGrainedMemory =
+        !atomicControlAttr.getFineGrainedMemory();
+    isNoRemoteMemory = !atomicControlAttr.getRemoteMemory();
   }
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
@@ -3365,8 +3370,8 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
       ompBuilder->createAtomicCapture(
           ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
           binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr,
-          isAmdgpuIgnoreDenormalMode, isAmdgpuNoFineGrainedMemory,
-          isAmdgpuNoRemoteMemory);
+          isIgnoreDenormalMode, isNoFineGrainedMemory,
+          isNoRemoteMemory);
 
   if (failed(handleError(afterIP, *atomicCaptureOp)))
     return failure();
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 9a72ddfc2989d..c3547f8ca8f37 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index ab7f756bf95f9..704b47dabe499 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,7 +1462,7 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):


From flang-commits at lists.llvm.org  Mon Jul  7 09:05:56 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Mon, 07 Jul 2025 09:05:56 -0700 (PDT)
Subject: [flang-commits] [flang] Parallel runtime library design doc (PRIF)
	(PR #76088)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/76088@github.com>
Message-ID: <686bf064.050a0220.1b9caa.7955@mx.google.com>


https://github.com/kiranchandramohan approved this pull request.

LG. The governance model that allows others to participate and modify PRIF addresses my concerns.

Please reach out to others who had comments to check whether they are OK before submitting.

https://github.com/llvm/llvm-project/pull/76088

From flang-commits at lists.llvm.org  Mon Jul  7 09:15:50 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 09:15:50 -0700 (PDT)
Subject: [flang-commits] [flang] 71ffa2a - [flang] Correctly handle
	-mframe-pointer=reserved (#146937)
Message-ID: <686bf2b6.050a0220.c1c64.85ac@mx.google.com>


Author: Daniel Paoliello
Date: 2025-07-07T09:15:47-07:00
New Revision: 71ffa2a4d3c220c97fbffa6078a446cc17bbaada

URL: https://github.com/llvm/llvm-project/commit/71ffa2a4d3c220c97fbffa6078a446cc17bbaada
DIFF: https://github.com/llvm/llvm-project/commit/71ffa2a4d3c220c97fbffa6078a446cc17bbaada.diff

LOG: [flang] Correctly handle -mframe-pointer=reserved (#146937)

Fixes `#146802`

#146582 started using the `Reserved` Frame Pointer kind for Arm64
Windows, but this revealed a bug in Flang where it copied the
`-mframe-pointer=reserved` flag from Clang, but didn't correctly handle
it in its own command line parser and subsequent compilation pipeline.

This change adds support for `-mframe-pointer=reserved` and adds a test
to make sure that functions are correctly marked when the flag is set.

Added: 
    

Modified: 
    flang/lib/Frontend/CompilerInvocation.cpp
    flang/lib/Optimizer/Passes/Pipelines.cpp
    flang/test/Driver/func-attr.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 8e4c210306cb2..e5c6b1d5c61ad 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -291,13 +291,14 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
 
   opts.AliasAnalysis = opts.OptimizationLevel > 0;
 
-  // -mframe-pointer=none/non-leaf/all option.
+  // -mframe-pointer=none/non-leaf/reserved/all option.
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_mframe_pointer_EQ)) {
     std::optional<llvm::FramePointerKind> val =
         llvm::StringSwitch<std::optional<llvm::FramePointerKind>>(a->getValue())
             .Case("none", llvm::FramePointerKind::None)
             .Case("non-leaf", llvm::FramePointerKind::NonLeaf)
+            .Case("reserved", llvm::FramePointerKind::Reserved)
             .Case("all", llvm::FramePointerKind::All)
             .Default(std::nullopt);
 

diff  --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 42d9e7ba2418f..dca176abe5ec5 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -358,6 +358,8 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::NonLeaf;
   else if (config.FramePointerKind == llvm::FramePointerKind::All)
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::All;
+  else if (config.FramePointerKind == llvm::FramePointerKind::Reserved)
+    framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::Reserved;
   else
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::None;
 

diff  --git a/flang/test/Driver/func-attr.f90 b/flang/test/Driver/func-attr.f90
index 7442ac54dcaa5..e8cab4fca2226 100644
--- a/flang/test/Driver/func-attr.f90
+++ b/flang/test/Driver/func-attr.f90
@@ -4,6 +4,7 @@
 
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=none -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONEFP
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=non-leaf -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONLEAFFP
+! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=reserved -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-RESERVEDFP
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=all -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-ALLFP
 ! RUN: not %flang_fc1 -triple aarch64-none-none -mframe-pointer=wrongval -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-WRONGVALUEFP
 
@@ -20,6 +21,7 @@ end subroutine func
 
 ! CHECK-NONEFP-NOT: attributes #0 = { "frame-pointer"="{{.*}}" }
 ! CHECK-NONLEAFFP: attributes #0 = { "frame-pointer"="non-leaf" }
+! CHECK-RESERVEDFP: attributes #0 = { "frame-pointer"="reserved" }
 ! CHECK-ALLFP: attributes #0 = { "frame-pointer"="all" }
 
 ! CHECK-WRONGVALUEFP:error: invalid value 'wrongval' in '-mframe-pointer=wrongval'


        

From flang-commits at lists.llvm.org  Mon Jul  7 09:15:53 2025
From: flang-commits at lists.llvm.org (Daniel Paoliello via flang-commits)
Date: Mon, 07 Jul 2025 09:15:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Correctly handle
	-mframe-pointer=reserved (PR #146937)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146937@github.com>
Message-ID: <686bf2b9.630a0220.3593d2.7625@mx.google.com>


https://github.com/dpaoliello closed https://github.com/llvm/llvm-project/pull/146937

From flang-commits at lists.llvm.org  Mon Jul  7 09:21:04 2025
From: flang-commits at lists.llvm.org (Jeff Hammond via flang-commits)
Date: Mon, 07 Jul 2025 09:21:04 -0700 (PDT)
Subject: [flang-commits] [flang] Parallel runtime library design doc (PRIF)
	(PR #76088)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/76088@github.com>
Message-ID: <686bf3f0.630a0220.3af3f8.739e@mx.google.com>


jeffhammond wrote:

Shouldn't PRIF just become part of the LLVM project if it's a critical dependency? Wouldn't that address the governance issues automatically?

https://github.com/llvm/llvm-project/pull/76088

From flang-commits at lists.llvm.org  Mon Jul  7 09:33:12 2025
From: flang-commits at lists.llvm.org (Damian Rouson via flang-commits)
Date: Mon, 07 Jul 2025 09:33:12 -0700 (PDT)
Subject: [flang-commits] [flang] Parallel runtime library design doc (PRIF)
	(PR #76088)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/76088@github.com>
Message-ID: <686bf6c8.170a0220.2bb224.164d@mx.google.com>


rouson wrote:

@jeffhammond thanks for the suggestion. The goal is for PRIF to support multiple compilers, including ones that don't have LLVM as a dependency.  The [governance document] was drafted in response to the 23 January 2025 [comment] by @kiranchandramohan, who acknowledges above that the developed governance model addresses the stated concern.

[governance document]: https://go.lbl.gov/prif-governance
[comment]: https://github.com/llvm/llvm-project/pull/76088#issuecomment-2610415963

https://github.com/llvm/llvm-project/pull/76088

From flang-commits at lists.llvm.org  Mon Jul  7 09:56:45 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 07 Jul 2025 09:56:45 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid UB in CharBlock Compare to C
	string (PR #147329)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147329@github.com>
Message-ID: <686bfc4d.170a0220.e0973.851c@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks for the fix

https://github.com/llvm/llvm-project/pull/147329

From flang-commits at lists.llvm.org  Mon Jul  7 10:46:25 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Mon, 07 Jul 2025 10:46:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <686c07f1.170a0220.33e682.950d@mx.google.com>


eugeneepshteyn wrote:

It was detected by a custom run of a test from llvm-test-sute. 

Should probably check why ISO_Fortran_binding_1.f90 is disabled. Perhaps it could be enabled now.

Do buildbots also run tests at `-O1` and higher optimization levels?

https://github.com/llvm/llvm-project/pull/146408

From flang-commits at lists.llvm.org  Mon Jul  7 11:46:20 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Mon, 07 Jul 2025 11:46:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686c15fc.050a0220.c1c64.bd22@mx.google.com>


https://github.com/vzakhari approved this pull request.

Thank you!

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Mon Jul  7 13:04:18 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Mon, 07 Jul 2025 13:04:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686c2842.170a0220.21ffe.a499@mx.google.com>


eugeneepshteyn wrote:

Thank you for the quick fix!

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Mon Jul  7 13:06:01 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Mon, 07 Jul 2025 13:06:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize assignments of
	multidimensional arrays (PR #146408)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146408@github.com>
Message-ID: <686c28a9.170a0220.289ac9.045c@mx.google.com>


eugeneepshteyn wrote:

The comment for ISO_Fortran_binding_1.f90 may be wrong at this point. I'll look into enabling this test.

https://github.com/llvm/llvm-project/pull/146408

From flang-commits at lists.llvm.org  Mon Jul  7 14:55:11 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Mon, 07 Jul 2025 14:55:11 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <686c423f.a70a0220.5c110.be86@mx.google.com>


vzakhari wrote:

Thank you for the changes, but I think we cannot just avoid generating proper "conversion" here.  Maybe it is not the case currently, but imagine that both results of `hlfir.associate` actually have uses.  In this case, the users that expect `!fir.boxchar<1>` operand will receive `!fir.ref<...`.  I think we need to generate `fir.emboxchar` here to produce the new result-0 with the expected data type.

https://github.com/llvm/llvm-project/pull/146851

From flang-commits at lists.llvm.org  Mon Jul  7 14:55:54 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Mon, 07 Jul 2025 14:55:54 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <686c426a.a70a0220.9391c.bb4c@mx.google.com>


https://github.com/vzakhari requested changes to this pull request.

Please generate `fir.emboxchar` instead.

https://github.com/llvm/llvm-project/pull/146851

From flang-commits at lists.llvm.org  Mon Jul  7 16:50:51 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 07 Jul 2025 16:50:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <686c5d5b.170a0220.315562.a83d@mx.google.com>


https://github.com/Meinersbur converted_to_draft https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Mon Jul  7 16:49:56 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 07 Jul 2025 16:49:56 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic` (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <686c5d24.170a0220.25b600.a66d@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/144785

>From fed2aa77ab2cd5d2354d128fcfbe70bad8a4ec22 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Wed, 18 Jun 2025 21:55:47 +0200
Subject: [PATCH 1/5] [Flang][MLIR] Add `!$omp unroll` and
 `omp.unroll_heuristic`

---
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 164 +++++++-
 .../test/Lower/OpenMP/unroll-heuristic01.f90  |  39 ++
 .../test/Lower/OpenMP/unroll-heuristic02.f90  |  70 ++++
 flang/test/Parser/OpenMP/unroll-heuristic.f90 |  43 +++
 .../OpenMP/{unroll.f90 => unroll-partial.f90} |   0
 .../Dialect/OpenMP/OpenMPClauseOperands.h     |   6 +-
 .../mlir/Dialect/OpenMP/OpenMPDialect.h       |   8 +-
 .../mlir/Dialect/OpenMP/OpenMPInterfaces.h    |   1 +
 .../mlir/Dialect/OpenMP/OpenMPOpBase.td       |  11 +
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 207 ++++++++++
 .../mlir/Dialect/OpenMP/OpenMPOpsAttributes.h |  17 +
 .../mlir/Dialect/OpenMP/OpenMPOpsEnums.h      |  14 +
 .../Dialect/OpenMP/OpenMPOpsInterfaces.td     |  58 +++
 .../mlir/Target/LLVMIR/ModuleTranslation.h    |  43 +++
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |  10 +
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 361 ++++++++++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  78 ++++
 .../OpenMP/cli-canonical_loop-invalid.mlir    |  50 +++
 .../Dialect/OpenMP/cli-canonical_loop.mlir    | 157 ++++++++
 .../Dialect/OpenMP/cli-unroll-heuristic.mlir  |  59 +++
 .../LLVMIR/openmp-cli-canonical_loop.mlir     | 175 +++++++++
 .../LLVMIR/openmp-cli-unroll-heuristic01.mlir |  56 +++
 .../LLVMIR/openmp-cli-unroll-heuristic02.mlir |  93 +++++
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp   |  11 +
 24 files changed, 1718 insertions(+), 13 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/unroll-heuristic01.f90
 create mode 100644 flang/test/Lower/OpenMP/unroll-heuristic02.f90
 create mode 100644 flang/test/Parser/OpenMP/unroll-heuristic.f90
 rename flang/test/Parser/OpenMP/{unroll.f90 => unroll-partial.f90} (100%)
 create mode 100644 mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h
 create mode 100644 mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h
 create mode 100644 mlir/test/Dialect/OpenMP/cli-canonical_loop-invalid.mlir
 create mode 100644 mlir/test/Dialect/OpenMP/cli-canonical_loop.mlir
 create mode 100644 mlir/test/Dialect/OpenMP/cli-unroll-heuristic.mlir
 create mode 100644 mlir/test/Target/LLVMIR/openmp-cli-canonical_loop.mlir
 create mode 100644 mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic01.mlir
 create mode 100644 mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic02.mlir

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 82673f0948a5b..3a8c7dcb0690a 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2128,6 +2128,161 @@ genLoopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   return loopOp;
 }
 
+static mlir::omp::CanonicalLoopOp
+genCanonicalLoopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval, mlir::Location loc,
+                   const ConstructQueue &queue,
+                   ConstructQueue::const_iterator item,
+                   llvm::ArrayRef<const semantics::Symbol *> ivs,
+                   llvm::omp::Directive directive, DataSharingProcessor &dsp) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  assert(ivs.size() == 1 && "Nested loops not yet implemented");
+  const semantics::Symbol *iv = ivs[0];
+
+  auto &nestedEval = eval.getFirstNestedEvaluation();
+  if (nestedEval.getIf<parser::DoConstruct>()->IsDoConcurrent()) {
+    TODO(loc, "Do Concurrent in unroll construct");
+  }
+
+  // Get the loop bounds (and increment)
+  auto &doLoopEval = nestedEval.getFirstNestedEvaluation();
+  auto *doStmt = doLoopEval.getIf<parser::NonLabelDoStmt>();
+  assert(doStmt && "Expected do loop to be in the nested evaluation");
+  auto &loopControl = std::get<std::optional<parser::LoopControl>>(doStmt->t);
+  assert(loopControl.has_value());
+  auto *bounds = std::get_if<parser::LoopControl::Bounds>(&loopControl->u);
+  assert(bounds && "Expected bounds for canonical loop");
+  lower::StatementContext stmtCtx;
+  mlir::Value loopLBVar = fir::getBase(
+      converter.genExprValue(*semantics::GetExpr(bounds->lower), stmtCtx));
+  mlir::Value loopUBVar = fir::getBase(
+      converter.genExprValue(*semantics::GetExpr(bounds->upper), stmtCtx));
+  mlir::Value loopStepVar = [&]() {
+    if (bounds->step) {
+      return fir::getBase(
+          converter.genExprValue(*semantics::GetExpr(bounds->step), stmtCtx));
+    } else {
+      // If `step` is not present, assume it is `1`.
+      return firOpBuilder.createIntegerConstant(loc, firOpBuilder.getI32Type(),
+                                                1);
+    }
+  }();
+
+  // Get the integer kind for the loop variable and cast the loop bounds
+  size_t loopVarTypeSize = bounds->name.thing.symbol->GetUltimate().size();
+  mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
+  loopLBVar = firOpBuilder.createConvert(loc, loopVarType, loopLBVar);
+  loopUBVar = firOpBuilder.createConvert(loc, loopVarType, loopUBVar);
+  loopStepVar = firOpBuilder.createConvert(loc, loopVarType, loopStepVar);
+
+  // Start lowering
+  mlir::Value zero = firOpBuilder.createIntegerConstant(loc, loopVarType, 0);
+  mlir::Value one = firOpBuilder.createIntegerConstant(loc, loopVarType, 1);
+  mlir::Value isDownwards = firOpBuilder.create<mlir::arith::CmpIOp>(
+      loc, mlir::arith::CmpIPredicate::slt, loopStepVar, zero);
+
+  // Ensure we are counting upwards. If not, negate step and swap lb and ub.
+  mlir::Value negStep =
+      firOpBuilder.create<mlir::arith::SubIOp>(loc, zero, loopStepVar);
+  mlir::Value incr = firOpBuilder.create<mlir::arith::SelectOp>(
+      loc, isDownwards, negStep, loopStepVar);
+  mlir::Value lb = firOpBuilder.create<mlir::arith::SelectOp>(
+      loc, isDownwards, loopUBVar, loopLBVar);
+  mlir::Value ub = firOpBuilder.create<mlir::arith::SelectOp>(
+      loc, isDownwards, loopLBVar, loopUBVar);
+
+  // Compute the trip count assuming lb <= ub. This guarantees that the result
+  // is non-negative and we can use unsigned arithmetic.
+  mlir::Value span = firOpBuilder.create<mlir::arith::SubIOp>(
+      loc, ub, lb, ::mlir::arith::IntegerOverflowFlags::nuw);
+  mlir::Value tcMinusOne =
+      firOpBuilder.create<mlir::arith::DivUIOp>(loc, span, incr);
+  mlir::Value tcIfLooping = firOpBuilder.create<mlir::arith::AddIOp>(
+      loc, tcMinusOne, one, ::mlir::arith::IntegerOverflowFlags::nuw);
+
+  // Fall back to 0 if lb > ub
+  mlir::Value isZeroTC = firOpBuilder.create<mlir::arith::CmpIOp>(
+      loc, mlir::arith::CmpIPredicate::slt, ub, lb);
+  mlir::Value tripcount = firOpBuilder.create<mlir::arith::SelectOp>(
+      loc, isZeroTC, zero, tcIfLooping);
+
+  // Create the CLI handle.
+  auto newcli = firOpBuilder.create<mlir::omp::NewCliOp>(loc);
+  mlir::Value cli = newcli.getResult();
+
+  auto ivCallback = [&](mlir::Operation *op)
+      -> llvm::SmallVector<const Fortran::semantics::Symbol *> {
+    mlir::Region &region = op->getRegion(0);
+
+    // Create the op's region skeleton (BB taking the iv as argument)
+    firOpBuilder.createBlock(&region, {}, {loopVarType}, {loc});
+
+    // Compute the value of the loop variable from the logical iteration number.
+    mlir::Value natIterNum = fir::getBase(region.front().getArgument(0));
+    mlir::Value scaled =
+        firOpBuilder.create<mlir::arith::MulIOp>(loc, natIterNum, loopStepVar);
+    mlir::Value userVal =
+        firOpBuilder.create<mlir::arith::AddIOp>(loc, loopLBVar, scaled);
+
+    // The argument is not currently in memory, so make a temporary for the
+    // argument, and store it there, then bind that location to the argument.
+    mlir::Operation *storeOp =
+        createAndSetPrivatizedLoopVar(converter, loc, userVal, iv);
+
+    firOpBuilder.setInsertionPointAfter(storeOp);
+    return {iv};
+  };
+
+  // Create the omp.canonical_loop operation
+  auto canonLoop = genOpWithBody<mlir::omp::CanonicalLoopOp>(
+      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, nestedEval,
+                        directive)
+          .setClauses(&item->clauses)
+          .setDataSharingProcessor(&dsp)
+          .setGenRegionEntryCb(ivCallback),
+      queue, item, tripcount, cli);
+
+  firOpBuilder.setInsertionPointAfter(canonLoop);
+  return canonLoop;
+}
+
+static void genUnrollOp(Fortran::lower::AbstractConverter &converter,
+                        Fortran::lower::SymMap &symTable,
+                        lower::StatementContext &stmtCtx,
+                        Fortran::semantics::SemanticsContext &semaCtx,
+                        Fortran::lower::pft::Evaluation &eval,
+                        mlir::Location loc, const ConstructQueue &queue,
+                        ConstructQueue::const_iterator item) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  mlir::omp::LoopRelatedClauseOps loopInfo;
+  llvm::SmallVector<const semantics::Symbol *> iv;
+  collectLoopRelatedInfo(converter, loc, eval, item->clauses, loopInfo, iv);
+
+  // Clauses for unrolling not yet implemnted
+  ClauseProcessor cp(converter, semaCtx, item->clauses);
+  cp.processTODO<clause::Partial, clause::Full>(
+      loc, llvm::omp::Directive::OMPD_unroll);
+
+  // Even though unroll does not support data-sharing clauses, but this is
+  // required to fill the symbol table.
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
+                           /*shouldCollectPreDeterminedSymbols=*/true,
+                           /*useDelayedPrivatization=*/false, symTable);
+  dsp.processStep1();
+
+  // Emit the associated loop
+  auto canonLoop =
+      genCanonicalLoopOp(converter, symTable, semaCtx, eval, loc, queue, item,
+                         iv, llvm::omp::Directive::OMPD_unroll, dsp);
+
+  // Apply unrolling to it
+  auto cli = canonLoop.getCli();
+  firOpBuilder.create<mlir::omp::UnrollHeuristicOp>(loc, cli);
+}
+
 static mlir::omp::MaskedOp
 genMaskedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
             lower::StatementContext &stmtCtx,
@@ -3516,12 +3671,9 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     newOp = genTeamsOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
                        item);
     break;
-  case llvm::omp::Directive::OMPD_tile:
-  case llvm::omp::Directive::OMPD_unroll: {
-    unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(loc, "Unhandled loop directive (" +
-                  llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
-  }
+  case llvm::omp::Directive::OMPD_unroll:
+    genUnrollOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);
+    break;
   // case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
diff --git a/flang/test/Lower/OpenMP/unroll-heuristic01.f90 b/flang/test/Lower/OpenMP/unroll-heuristic01.f90
new file mode 100644
index 0000000000000..a5f5c003b8a7c
--- /dev/null
+++ b/flang/test/Lower/OpenMP/unroll-heuristic01.f90
@@ -0,0 +1,39 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
+
+
+subroutine omp_unroll_heuristic01(lb, ub, inc)
+  integer res, i, lb, ub, inc
+
+  !$omp unroll
+  do i = lb, ub, inc
+    res = i
+  end do
+  !$omp end unroll
+
+end subroutine omp_unroll_heuristic01
+
+
+!CHECK-LABEL: func.func @_QPomp_unroll_heuristic01(
+!CHECK:      %c0_i32 = arith.constant 0 : i32
+!CHECK-NEXT: %c1_i32 = arith.constant 1 : i32
+!CHECK-NEXT: %13 = arith.cmpi slt, %12, %c0_i32 : i32
+!CHECK-NEXT: %14 = arith.subi %c0_i32, %12 : i32
+!CHECK-NEXT: %15 = arith.select %13, %14, %12 : i32
+!CHECK-NEXT: %16 = arith.select %13, %11, %10 : i32
+!CHECK-NEXT: %17 = arith.select %13, %10, %11 : i32
+!CHECK-NEXT: %18 = arith.subi %17, %16 overflow<nuw> : i32
+!CHECK-NEXT: %19 = arith.divui %18, %15 : i32
+!CHECK-NEXT: %20 = arith.addi %19, %c1_i32 overflow<nuw> : i32
+!CHECK-NEXT: %21 = arith.cmpi slt, %17, %16 : i32
+!CHECK-NEXT: %22 = arith.select %21, %c0_i32, %20 : i32
+!CHECK-NEXT: %canonloop_s0 = omp.new_cli
+!CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%22) {
+!CHECK-NEXT:   %23 = arith.muli %iv, %12 : i32
+!CHECK-NEXT:   %24 = arith.addi %10, %23 : i32
+!CHECK-NEXT:   hlfir.assign %24 to %9#0 : i32, !fir.ref<i32>
+!CHECK-NEXT:   %25 = fir.load %9#0 : !fir.ref<i32>
+!CHECK-NEXT:   hlfir.assign %25 to %6#0 : i32, !fir.ref<i32>
+!CHECK-NEXT:   omp.terminator
+!CHECK-NEXT: }
+!CHECK-NEXT: omp.unroll_heuristic(%canonloop_s0)
+!CHECK-NEXT: return
diff --git a/flang/test/Lower/OpenMP/unroll-heuristic02.f90 b/flang/test/Lower/OpenMP/unroll-heuristic02.f90
new file mode 100644
index 0000000000000..669f185f910c4
--- /dev/null
+++ b/flang/test/Lower/OpenMP/unroll-heuristic02.f90
@@ -0,0 +1,70 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
+
+
+subroutine omp_unroll_heuristic_nested02(outer_lb, outer_ub, outer_inc, inner_lb, inner_ub, inner_inc)
+  integer res, i, j, inner_lb, inner_ub, inner_inc, outer_lb, outer_ub, outer_inc
+
+  !$omp unroll
+  do i = outer_lb, outer_ub, outer_inc
+    !$omp unroll
+    do j = inner_lb, inner_ub, inner_inc
+      res = i + j
+    end do
+    !$omp end unroll
+  end do
+  !$omp end unroll
+
+end subroutine omp_unroll_heuristic_nested02
+
+
+!CHECK-LABEL: func.func @_QPomp_unroll_heuristic_nested02(%arg0: !fir.ref<i32> {fir.bindc_name = "outer_lb"}, %arg1: !fir.ref<i32> {fir.bindc_name = "outer_ub"}, %arg2: !fir.ref<i32> {fir.bindc_name = "outer_inc"}, %arg3: !fir.ref<i32> {fir.bindc_name = "inner_lb"}, %arg4: !fir.ref<i32> {fir.bindc_name = "inner_ub"}, %arg5: !fir.ref<i32> {fir.bindc_name = "inner_inc"}) {
+!CHECK:      %c0_i32 = arith.constant 0 : i32
+!CHECK-NEXT: %c1_i32 = arith.constant 1 : i32
+!CHECK-NEXT: %18 = arith.cmpi slt, %17, %c0_i32 : i32
+!CHECK-NEXT: %19 = arith.subi %c0_i32, %17 : i32
+!CHECK-NEXT: %20 = arith.select %18, %19, %17 : i32
+!CHECK-NEXT: %21 = arith.select %18, %16, %15 : i32
+!CHECK-NEXT: %22 = arith.select %18, %15, %16 : i32
+!CHECK-NEXT: %23 = arith.subi %22, %21 overflow<nuw> : i32
+!CHECK-NEXT: %24 = arith.divui %23, %20 : i32
+!CHECK-NEXT: %25 = arith.addi %24, %c1_i32 overflow<nuw> : i32
+!CHECK-NEXT: %26 = arith.cmpi slt, %22, %21 : i32
+!CHECK-NEXT: %27 = arith.select %26, %c0_i32, %25 : i32
+!CHECK-NEXT: %canonloop_s0 = omp.new_cli
+!CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%27) {
+!CHECK-NEXT:   %28 = arith.muli %iv, %17 : i32
+!CHECK-NEXT:   %29 = arith.addi %15, %28 : i32
+!CHECK-NEXT:   hlfir.assign %29 to %14#0 : i32, !fir.ref<i32>
+!CHECK-NEXT:   %30 = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFomp_unroll_heuristic_nested02Ej"}
+!CHECK-NEXT:   %31:2 = hlfir.declare %30 {uniq_name = "_QFomp_unroll_heuristic_nested02Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK-NEXT:   %32 = fir.load %4#0 : !fir.ref<i32>
+!CHECK-NEXT:   %33 = fir.load %5#0 : !fir.ref<i32>
+!CHECK-NEXT:   %34 = fir.load %3#0 : !fir.ref<i32>
+!CHECK-NEXT:   %c0_i32_0 = arith.constant 0 : i32
+!CHECK-NEXT:   %c1_i32_1 = arith.constant 1 : i32
+!CHECK-NEXT:   %35 = arith.cmpi slt, %34, %c0_i32_0 : i32
+!CHECK-NEXT:   %36 = arith.subi %c0_i32_0, %34 : i32
+!CHECK-NEXT:   %37 = arith.select %35, %36, %34 : i32
+!CHECK-NEXT:   %38 = arith.select %35, %33, %32 : i32
+!CHECK-NEXT:   %39 = arith.select %35, %32, %33 : i32
+!CHECK-NEXT:   %40 = arith.subi %39, %38 overflow<nuw> : i32
+!CHECK-NEXT:   %41 = arith.divui %40, %37 : i32
+!CHECK-NEXT:   %42 = arith.addi %41, %c1_i32_1 overflow<nuw> : i32
+!CHECK-NEXT:   %43 = arith.cmpi slt, %39, %38 : i32
+!CHECK-NEXT:   %44 = arith.select %43, %c0_i32_0, %42 : i32
+!CHECK-NEXT:   %canonloop_s0_s0 = omp.new_cli
+!CHECK-NEXT:   omp.canonical_loop(%canonloop_s0_s0) %iv_2 : i32 in range(%44) {
+!CHECK-NEXT:     %45 = arith.muli %iv_2, %34 : i32
+!CHECK-NEXT:     %46 = arith.addi %32, %45 : i32
+!CHECK-NEXT:     hlfir.assign %46 to %31#0 : i32, !fir.ref<i32>
+!CHECK-NEXT:     %47 = fir.load %14#0 : !fir.ref<i32>
+!CHECK-NEXT:     %48 = fir.load %31#0 : !fir.ref<i32>
+!CHECK-NEXT:     %49 = arith.addi %47, %48 : i32
+!CHECK-NEXT:     hlfir.assign %49 to %12#0 : i32, !fir.ref<i32>
+!CHECK-NEXT:     omp.terminator
+!CHECK-NEXT:   }
+!CHECK-NEXT:   omp.unroll_heuristic(%canonloop_s0_s0)
+!CHECK-NEXT:   omp.terminator
+!CHECK-NEXT: }
+!CHECK-NEXT: omp.unroll_heuristic(%canonloop_s0)
+!CHECK-NEXT: return
diff --git a/flang/test/Parser/OpenMP/unroll-heuristic.f90 b/flang/test/Parser/OpenMP/unroll-heuristic.f90
new file mode 100644
index 0000000000000..2f589af0c83ca
--- /dev/null
+++ b/flang/test/Parser/OpenMP/unroll-heuristic.f90
@@ -0,0 +1,43 @@
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=51 %s -fdebug-unparse         | FileCheck --check-prefix=UNPARSE %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=51 %s -fdebug-dump-parse-tree | FileCheck --check-prefix=PTREE %s
+
+subroutine openmp_parse_unroll_heuristic
+  integer i
+
+  !$omp unroll
+  do i = 1, 100
+    call func(i)
+  end do
+  !$omp end unroll
+END subroutine openmp_parse_unroll_heuristic
+
+
+!UNPARSE:      !$OMP UNROLL
+!UNPARSE-NEXT: DO i=1_4,100_4
+!UNPARSE-NEXT:   CALL func(i)
+!UNPARSE-NEXT: END DO
+!UNPARSE-NEXT: !$OMP END UNROLL
+
+!PTREE:      OpenMPConstruct -> OpenMPLoopConstruct
+!PTREE-NEXT: | OmpBeginLoopDirective
+!PTREE-NEXT: | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!PTREE-NEXT: | | OmpClauseList ->
+!PTREE-NEXT: | DoConstruct
+!PTREE-NEXT: | | NonLabelDoStmt
+!PTREE-NEXT: | | | LoopControl -> LoopBounds
+!PTREE-NEXT: | | | | Scalar -> Name = 'i'
+!PTREE-NEXT: | | | | Scalar -> Expr = '1_4'
+!PTREE-NEXT: | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PTREE-NEXT: | | | | Scalar -> Expr = '100_4'
+!PTREE-NEXT: | | | | | LiteralConstant -> IntLiteralConstant = '100'
+!PTREE-NEXT: | | Block
+!PTREE-NEXT: | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> CallStmt = 'CALL func(i)'
+!PTREE-NEXT: | | | | | | Call
+!PTREE-NEXT: | | | | | ProcedureDesignator -> Name = 'func'
+!PTREE-NEXT: | | | | | ActualArgSpec
+!PTREE-NEXT: | | | | | | ActualArg -> Expr = 'i'
+!PTREE-NEXT: | | | | | | | Designator -> DataRef -> Name = 'i'
+!PTREE-NEXT: | | EndDoStmt ->
+!PTREE-NEXT: | OmpEndLoopDirective
+!PTREE-NEXT: | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!PTREE-NEXT: | | OmpClauseList ->
diff --git a/flang/test/Parser/OpenMP/unroll.f90 b/flang/test/Parser/OpenMP/unroll-partial.f90
similarity index 100%
rename from flang/test/Parser/OpenMP/unroll.f90
rename to flang/test/Parser/OpenMP/unroll-partial.f90
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
index f9a85626a3f14..faf820dcfdb29 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
@@ -15,14 +15,10 @@
 #ifndef MLIR_DIALECT_OPENMP_OPENMPCLAUSEOPERANDS_H_
 #define MLIR_DIALECT_OPENMP_OPENMPCLAUSEOPERANDS_H_
 
+#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "llvm/ADT/SmallVector.h"
 
-#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h.inc"
-
-#define GET_ATTRDEF_CLASSES
-#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h.inc"
-
 #include "mlir/Dialect/OpenMP/OpenMPClauseOps.h.inc"
 
 namespace mlir {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
index 248ac2eb72c61..0a844fc2380bf 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/AtomicInterfaces.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/OpenACCMPOpsInterfaces.h"
+#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -24,6 +25,11 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
 
+namespace mlir::omp {
+/// Find the omp.new_cli, generator, and consumer of a canonical loop info.
+std::tuple<NewCliOp, OpOperand *, OpOperand *> decodeCli(mlir::Value cli);
+} // namespace mlir::omp
+
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsTypes.h.inc"
 
@@ -33,8 +39,6 @@
 
 #include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.h.inc"
 
-#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
-
 #define GET_OP_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOps.h.inc"
 
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
index 989ab1710c211..bc9534974d21f 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
@@ -14,6 +14,7 @@
 #define MLIR_DIALECT_OPENMP_OPENMPINTERFACES_H_
 
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpBase.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpBase.td
index f3dd44d2c0717..bbcfb87fa03c6 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpBase.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpBase.td
@@ -204,4 +204,15 @@ class OpenMP_Op<string mnemonic, list<Trait> traits = [],
   let regions = !if(singleRegion, (region AnyRegion:$region), (region));
 }
 
+
+// Base class for OpenMP loop transformations (that either consume or generate
+// loops)
+//
+// Doesn't actually create a C++ base class (only defines default values for
+// tablegen classes that derive from this). Use LoopTransformationInterface
+// instead for common operations.
+class OpenMPTransform_Op<string mnemonic, list<Trait> traits = []> :
+      OpenMP_Op<mnemonic, !listconcat([DeclareOpInterfaceMethods<LoopTransformationInterface>], traits)  > {
+}
+
 #endif  // OPENMP_OP_BASE
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..8641c9b8150ee 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -22,6 +22,7 @@ include "mlir/Dialect/OpenMP/OpenMPOpBase.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 include "mlir/IR/EnumAttr.td"
+include "mlir/IR/OpAsmInterface.td"
 include "mlir/IR/OpBase.td"
 include "mlir/IR/SymbolInterfaces.td"
 
@@ -356,6 +357,212 @@ def SingleOp : OpenMP_Op<"single", traits = [
   let hasVerifier = 1;
 }
 
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Info Type
+//===---------------------------------------------------------------------===//
+
+def CanonicalLoopInfoType : OpenMP_Type<"CanonicalLoopInfo", "cli"> {
+  let summary = "Type for representing a reference to a canonical loop";
+  let description = [{
+    A variable of type CanonicalLoopInfo refers to an OpenMP-compatible
+    canonical loop in the same function. Values of this type are not
+    available at runtime and therefore cannot be used by the program itself,
+    i.e. an opaque type. It is similar to the transform dialect's
+    `!transform.interface` type, but instead of implementing an interface
+    for each transformation, the OpenMP dialect itself defines possible
+    operations on this type.
+
+    A value of type CanonicalLoopInfoType (in the following: CLI) value can be
+
+    1. created by omp.new_cli.
+    2. passed to omp.canonical_loop to associate the loop to that CLI. A CLI
+       can only be associated once.
+    3. passed to an omp loop transformation operation that modifies the loop
+       associated with the CLI. The CLI is the "applyee" and the operation is
+       the consumer. A CLI can only be consumed once.
+    4. passed to an omp loop transformation operation to associate the cli with
+       a result of that transformation. The CLI is the "generatee" and the
+       operation is the generator.
+
+    A CLI cannot
+
+    1. be returned from a function.
+    2. be passed to operations that are not specifically designed to take a
+       CanonicalLoopInfoType, including AnyType.
+
+    A CLI directly corresponds to an object of
+    OpenMPIRBuilder's CanonicalLoopInfo struct when lowering to LLVM-IR.
+  }];
+}
+
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Info Creation
+//===---------------------------------------------------------------------===//
+
+def NewCliOp : OpenMP_Op<"new_cli",
+    [DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>]> {
+  let summary = "Create a new Canonical Loop Info value.";
+  let description = [{
+    Create a new CLI that can be passed as an argument to a CanonicalLoopOp
+    and to loop transformation operations to handle dependencies between
+    loop transformation operations.
+  }];
+
+  let arguments = (ins );
+  let results = (outs CanonicalLoopInfoType:$result);
+  let assemblyFormat = [{
+      attr-dict
+  }];
+
+  let builders = [
+    OpBuilder<(ins )>,
+  ];
+
+  let hasVerifier = 1;
+}
+
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Operation
+//===---------------------------------------------------------------------===//
+def CanonicalLoopOp : OpenMPTransform_Op<"canonical_loop", 
+    [DeclareOpInterfaceMethods<OpAsmOpInterface, [ "getAsmBlockNames", "getAsmBlockArgumentNames"]>]> {
+  let summary = "OpenMP Canonical Loop Operation";
+  let description = [{
+    All loops that conform to OpenMP's definition of a canonical loop can be
+    simplified to a CanonicalLoopOp. In particular, there are no loop-carried
+    variables and the number of iterations it will execute is know before the
+    operation. This allows e.g. to determine the number of threads and chunks
+    the iterations space is split into before executing any iteration. More
+    restrictions may apply in cases such as (collapsed) loop nests, doacross
+    loops, etc.
+
+    In contrast to other loop operations such as `scf.for`, the number of
+    iterations is determined by only a single variable, the trip-count. The
+    induction variable value is the logical iteration number of that iteration,
+    which OpenMP defines to be between 0 and the trip-count (exclusive).
+    Loop representation having lower-bound, upper-bound, and step-size operands,
+    require passes to do more work than necessary, including handling special
+    cases such as upper-bound smaller than lower-bound, upper-bound equal to
+    the integer type's maximal value, negative step size, etc. This complexity
+    is better only handled once by the front-end and can apply its semantics
+    for such cases while still being able to represent any kind of loop, which
+    kind of the point of a mid-end intermediate representation. User-defined
+    types such as random-access iterators in C++ could not directly be
+    represented anyway.
+
+    The induction variable is always of the same type as the tripcount argument.
+    Since it can never be negative, tripcount is always interpreted as an
+    unsigned integer. It is the caller's responsibility to ensure the tripcount
+    is not negative when its interpretation is signed, i.e.
+    `%tripcount = max(0,%tripcount)`.
+
+    An optional argument to a omp.canonical_loop that can be passed in
+    is a CanonicalLoopInfo value that can be used to refer to the canonical
+    loop to apply transformations -- such as tiling, unrolling, or
+    work-sharing -- to the loop, similar to the transform dialect but
+    with OpenMP-specific semantics. Because it is optional, it has to be the
+    last of the operands, but appears first in the pretty format printing.
+
+    The pretty assembly format is inspired by python syntax, where `range(n)`
+    returns an iterator that runs from $0$ to $n-1$. The pretty assembly syntax
+    is one of:
+
+     omp.canonical_loop(%cli) %iv : !type in range(%tripcount)
+     omp.canonical_loop       %iv : !type in range(%tripcount)
+
+    A CanonicalLoopOp is lowered to LLVM-IR using
+    `OpenMPIRBuilder::createCanonicalLoop`.
+
+    #### Examples
+
+    Translation from lower-bound, upper-bound, step-size to trip-count.
+    ```c
+    for (int i = 3; i < 42; i+=2) {
+      B[i] = A[i];
+    }
+    ```
+
+    ```mlir
+    %lb = arith.constant 3 : i32
+    %ub = arith.constant 42 : i32
+    %step = arith.constant 2 : i32
+    %range = arith.sub %ub, %lb : i32
+    %tripcount = arith.div %range, %step : i32
+    omp.canonical_loop %iv : i32 in range(%tripcount) {
+      %offset = arith.mul %iv, %step : i32
+      %i = arith.add %offset, %lb : i32
+      %a = load %arrA[%i] : memref<?xf32>
+      store %a, %arrB[%i] : memref<?xf32>
+    }
+    ```
+
+    Nested canonical loop with transformation of the inner loop.
+    ```mlir
+    %outer = omp.new_cli : !omp.cli
+    %inner = omp.new_cli : !omp.cli
+    omp.canonical_loop(%outer) %iv1 : i32 in range(%tc1) {
+      omp.canonical_loop(%inner) %iv2 : i32 in range(%tc2) {
+        %a = load %arrA[%iv1, %iv2] : memref<?x?xf32>
+        store %a, %arrB[%iv1, %iv2] : memref<?x?xf32>
+      }
+    }
+    omp.unroll_full(%inner)
+    ```
+  }];
+
+
+  let arguments = (ins IntLikeType:$tripCount,
+                       Optional<CanonicalLoopInfoType>:$cli);
+  let regions = (region AnyRegion:$region);
+
+  let extraClassDeclaration = [{
+    ::mlir::Value getInductionVar();
+  }];
+
+  let builders = [
+    OpBuilder<(ins "::mlir::Value":$tripCount)>,
+    OpBuilder<(ins "::mlir::Value":$tripCount, "::mlir::Value":$cli)>,
+  ];
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// OpenMP unroll_heuristic operation
+//===----------------------------------------------------------------------===//
+
+def UnrollHeuristicOp : OpenMPTransform_Op<"unroll_heuristic", []> {
+  let summary = "OpenMP heuristic unroll operation";
+  let description = [{
+    Represents a `#pragma omp unroll` construct introduced in OpenMP 5.1.
+
+    The operation has one applyee and no generatees. The applyee is unrolled
+    according to implementation-defined heuristics. Implementations may choose
+    to not unroll the loop, partially unroll by a chosen factor, or fully
+    unroll it. Even if the implementation chooses to partially unroll the
+    applyee, the resulting unrolled loop is not accessible as a generatee. Use
+    omp.unroll_partial if a generatee is required.
+
+    The lowering is implemented using `OpenMPIRBuilder::unrollLoopHeuristic`,
+    which just attaches `llvm.loop.unroll.enable` metadata to the loop so the
+    unrolling is carried-out by LLVM's LoopUnroll pass. That is, unrolling only
+    actually performed in optimized builds.
+
+    Assembly formats:
+      omp.unroll_heuristic(%cli)
+      omp.unroll_heuristic(%cli) -> ()
+  }];
+
+  let arguments = (ins CanonicalLoopInfoType:$applyee);
+
+  let builders = [
+    OpBuilder<(ins "::mlir::Value":$cli)>,
+  ];
+
+  let hasCustomAssemblyFormat = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // 2.8.3 Workshare Construct
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h
new file mode 100644
index 0000000000000..9a653c4b557b5
--- /dev/null
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h
@@ -0,0 +1,17 @@
+//===- OpenMPOpsAttributes.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
+#define MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
+
+#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h.inc"
+
+#endif // MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h
new file mode 100644
index 0000000000000..0f6c41a179536
--- /dev/null
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h
@@ -0,0 +1,14 @@
+//===- OpenMPOpsEnums.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
+#define MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
+
+#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h.inc"
+
+#endif // MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index 92bf34ef3145f..02ec95c10d67d 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -551,4 +551,62 @@ def OffloadModuleInterface : OpInterface<"OffloadModuleInterface"> {
   ];
 }
 
+def LoopTransformationInterface : OpInterface<"LoopTransformationInterface"> {
+  let description = [{
+    Methods that are common for OpenMP loop transformation operations.
+  }];
+
+  let cppNamespace = "::mlir::omp";
+
+  let methods = [
+
+    InterfaceMethod<
+      /*description=*/[{
+        Get the indices for the arguments that represent CanonicalLoopInfo
+        applyees, i.e. loops that are transformed/consumed by this operation.
+      }],
+      /*returnType=*/ "std::pair<unsigned, unsigned>",
+      /*methodName=*/ "getApplyeesODSOperandIndexAndLength",
+      /*args=*/(ins)
+    >,
+
+    InterfaceMethod<
+      /*description=*/[{
+        Get the indices for the arguments that represent CanonicalLoopInfo
+        generatees, i.e. loops that created by this operation.
+      }],
+      /*returnType=*/ "std::pair<unsigned, unsigned>",
+      /*methodName=*/ "getGenerateesODSOperandIndexAndLength",
+      /*args=*/(ins)
+    >,
+
+    InterfaceMethod<
+      /*description=*/[{
+        Return the number of applyees of this loop transformation.
+      }],
+      /*returnType=*/ "unsigned",
+      /*methodName=*/ "getNumApplyees",
+      /*args=*/       (ins),
+      /*methodBody=*/ "",
+      /*defaultImpl=*/[{
+        return  $_op.getApplyeesODSOperandIndexAndLength().second;
+      }]
+    >,
+
+    InterfaceMethod<
+      /*description=*/[{
+        Return the number of generatees of this loop transformation.
+      }],
+      /*returnType=*/ "unsigned",
+      /*methodName=*/ "getNumGeneratees",
+      /*args=*/       (ins),
+      /*methodBody=*/ "",
+      /*defaultImpl=*/[{
+        return  $_op.getGenerateesODSOperandIndexAndLength().second;
+      }]
+    >,
+
+  ];
+}
+
 #endif // OPENMP_OPS_INTERFACES
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 97ae14aa0d6af..7de9021694438 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -15,6 +15,7 @@
 #define MLIR_TARGET_LLVMIR_MODULETRANSLATION_H
 
 #include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
@@ -23,6 +24,7 @@
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
 
 #include "llvm/ADT/SetVector.h"
+#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/FPEnv.h"
 
 namespace llvm {
@@ -107,6 +109,41 @@ class ModuleTranslation {
     return blockMapping.lookup(block);
   }
 
+  /// Find the LLVM-IR loop that represents an MLIR loop.
+  llvm::CanonicalLoopInfo *lookupOMPLoop(omp::NewCliOp mlir) const {
+    llvm::CanonicalLoopInfo *result = loopMapping.lookup(mlir);
+    assert(result && "attempt to get non-existing loop");
+    return result;
+  }
+
+  /// Find the LLVM-IR loop that represents an MLIR loop.
+  llvm::CanonicalLoopInfo *lookupOMPLoop(Value mlir) const {
+    return lookupOMPLoop(mlir.getDefiningOp<omp::NewCliOp>());
+  }
+
+  /// Mark an OpenMP loop as having been consumed.
+  void invalidateOmpLoop(omp::NewCliOp mlir) { loopMapping.erase(mlir); }
+
+  /// Mark an OpenMP loop as having been consumed.
+  void invalidateOmpLoop(Value mlir) {
+    invalidateOmpLoop(mlir.getDefiningOp<omp::NewCliOp>());
+  }
+
+  /// Map an MLIR OpenMP dialect CanonicalLoopInfo to its lowered LLVM-IR
+  /// OpenMPIRBuilder CanonicalLoopInfo
+  void mapOmpLoop(omp::NewCliOp mlir, llvm::CanonicalLoopInfo *llvm) {
+    assert(llvm && "argument must be non-null");
+    llvm::CanonicalLoopInfo *&cur = loopMapping[mlir];
+    assert(cur == nullptr && "attempting to map a loop that is already mapped");
+    cur = llvm;
+  }
+
+  /// Map an MLIR OpenMP dialect CanonicalLoopInfo to its lowered LLVM-IR
+  /// OpenMPIRBuilder CanonicalLoopInfo
+  void mapOmpLoop(Value mlir, llvm::CanonicalLoopInfo *llvm) {
+    mapOmpLoop(mlir.getDefiningOp<omp::NewCliOp>(), llvm);
+  }
+
   /// Stores the mapping between an MLIR operation with successors and a
   /// corresponding LLVM IR instruction.
   void mapBranch(Operation *mlir, llvm::Instruction *llvm) {
@@ -431,6 +468,12 @@ class ModuleTranslation {
   DenseMap<Value, llvm::Value *> valueMapping;
   DenseMap<Block *, llvm::BasicBlock *> blockMapping;
 
+  /// List of not yet consumed MLIR loop handles (represented by an omp.new_cli
+  /// operation which creates a value of type CanonicalLoopInfoType) and their
+  /// LLVM-IR representation as CanonicalLoopInfo which is managed by the
+  /// OpenMPIRBuilder.
+  DenseMap<omp::NewCliOp, llvm::CanonicalLoopInfo *> loopMapping;
+
   /// A mapping between MLIR LLVM dialect terminators and LLVM IR terminators
   /// they are converted to. This allows for connecting PHI nodes to the source
   /// values after all operations are converted.
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 7a0a7f86bc1e9..e77c4a0b94de9 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -42,6 +42,16 @@ template <typename T>
 struct OpenMPOpConversion : public ConvertOpToLLVMPattern<T> {
   using ConvertOpToLLVMPattern<T>::ConvertOpToLLVMPattern;
 
+  OpenMPOpConversion(LLVMTypeConverter &typeConverter,
+                     PatternBenefit benefit = 1)
+      : ConvertOpToLLVMPattern<T>(typeConverter, benefit) {
+    // Operations using CanonicalLoopInfoType are lowered only by
+    // mlir::translateModuleToLLVMIR() using the OpenMPIRBuilder. Until then,
+    // the type and operations using it must be preserved.
+    typeConverter.addConversion(
+        [&](::mlir::omp::CanonicalLoopInfoType type) { return type; });
+  }
+
   LogicalResult
   matchAndRewrite(T op, typename T::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..64b6dd32b0c37 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -24,6 +24,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallString.h"
@@ -3012,6 +3013,366 @@ void LoopNestOp::gatherWrappers(
   }
 }
 
+//===----------------------------------------------------------------------===//
+// OpenMP canonical loop handling
+//===----------------------------------------------------------------------===//
+
+std::tuple<NewCliOp, OpOperand *, OpOperand *>
+mlir::omp ::decodeCli(Value cli) {
+
+  // Defining a CLI for a generated loop is optional; if there is none then
+  // there is no followup-tranformation
+  if (!cli)
+    return {{}, nullptr, nullptr};
+
+  MLIRContext *ctx = cli.getContext();
+  assert(cli.getType() == CanonicalLoopInfoType::get(ctx) &&
+         "Unexpected type of cli");
+
+  NewCliOp create = cast<NewCliOp>(cli.getDefiningOp());
+  OpOperand *gen = nullptr;
+  OpOperand *cons = nullptr;
+  for (OpOperand &use : cli.getUses()) {
+    auto op = cast<LoopTransformationInterface>(use.getOwner());
+    auto applyees = op.getApplyeesODSOperandIndexAndLength();
+    auto generatees = op.getGenerateesODSOperandIndexAndLength();
+
+    unsigned opnum = use.getOperandNumber();
+    if (generatees.first <= opnum &&
+        opnum < generatees.first + generatees.second) {
+      assert(!gen && "Each CLI may have at most one consumer");
+      gen = &use;
+    } else if (applyees.first <= opnum &&
+               opnum < applyees.first + applyees.second) {
+      assert(!cons && "Each CLI may have at most one def");
+      cons = &use;
+    } else {
+      llvm_unreachable("Unexpected operand for a CLI");
+    }
+  }
+
+  return {create, gen, cons};
+}
+
+void NewCliOp::build(::mlir::OpBuilder &odsBuilder,
+                     ::mlir::OperationState &odsState) {
+  odsState.addTypes(CanonicalLoopInfoType::get(odsBuilder.getContext()));
+}
+
+void NewCliOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {
+  Value result = getResult();
+  auto [newCli, gen, cond] = decodeCli(result);
+
+  // Derive the CLI variable name from its generator:
+  //  * "canonloop" for omp.canonical_loop
+  //  * custom name for loop transformation generatees
+  //  * "cli" as fallback if no generator
+  //  * "_r<idx>" suffix for nested loops, where <idx> is the sequential order
+  //  at that level
+  //  * "_s<idx>" suffix for operations with multiple regions, where <idx> is
+  //  the index of that region
+  std::string cliName{"cli"};
+  if (gen) {
+    cliName =
+        TypeSwitch<Operation *, std::string>(gen->getOwner())
+            .Case([&](CanonicalLoopOp op) {
+              // Find the canonical loop nesting: For each ancestor add a
+              // "+_r<idx>" suffix (in reverse order)
+              SmallVector<std::string> components;
+              Operation *o = op.getOperation();
+              while (o) {
+                if (o->hasTrait<mlir::OpTrait::IsIsolatedFromAbove>())
+                  break;
+
+                Region *r = o->getParentRegion();
+                if (!r)
+                  break;
+
+                Operation *parent = r->getParentOp();
+                auto getSequentialIndex = [](Region *r, Operation *o) {
+                  llvm::ReversePostOrderTraversal<Block *> traversal(
+                      &r->getBlocks().front());
+                  size_t idx = 0;
+                  for (Block *b : traversal) {
+                    for (Operation &op : *b) {
+                      if (&op == o)
+                        return idx;
+                      // Only consider operations that are containers as
+                      // possible children
+                      if (!op.getRegions().empty())
+                        idx += 1;
+                    }
+                  }
+                  llvm_unreachable("Operation not part of the region");
+                };
+                size_t sequentialIdx = getSequentialIndex(r, o);
+                components.push_back(("s" + Twine(sequentialIdx)).str());
+
+                if (!parent)
+                  break;
+
+                // If the operation has more than one region, also count in
+                // which of the regions
+                if (parent->getRegions().size() > 1) {
+                  auto getRegionIndex = [](Operation *o, Region *r) {
+                    for (auto [idx, region] :
+                         llvm::enumerate(o->getRegions())) {
+                      if (&region == r)
+                        return idx;
+                    }
+                    llvm_unreachable("Region not child its parent operation");
+                  };
+                  size_t regionIdx = getRegionIndex(parent, r);
+                  components.push_back(("r" + Twine(regionIdx)).str());
+                }
+
+                // next parent
+                o = parent;
+              }
+
+              SmallString<64> Name("canonloop");
+              for (std::string s : reverse(components)) {
+                Name += '_';
+                Name += s;
+              }
+
+              return Name;
+            })
+            .Case([&](UnrollHeuristicOp op) -> std::string {
+              llvm_unreachable("heuristic unrolling does not generate a loop");
+            })
+            .Default([&](Operation *op) {
+              assert(!"TODO: Custom name for this operation");
+              return "transformed";
+            });
+  }
+
+  setNameFn(result, cliName);
+}
+
+LogicalResult NewCliOp::verify() {
+  Value cli = getResult();
+
+  MLIRContext *ctx = cli.getContext();
+  assert(cli.getType() == CanonicalLoopInfoType::get(ctx) &&
+         "Unexpected type of cli");
+
+  // Check that the CLI is used in at most generator and one consumer
+  OpOperand *gen = nullptr;
+  OpOperand *cons = nullptr;
+  for (mlir::OpOperand &use : cli.getUses()) {
+    auto op = cast<mlir::omp::LoopTransformationInterface>(use.getOwner());
+    auto applyees = op.getApplyeesODSOperandIndexAndLength();
+    auto generatees = op.getGenerateesODSOperandIndexAndLength();
+
+    unsigned opnum = use.getOperandNumber();
+    if (generatees.first <= opnum &&
+        opnum < generatees.first + generatees.second) {
+      if (gen) {
+        InFlightDiagnostic error =
+            emitOpError("CLI must have at most one generator");
+        error.attachNote(gen->getOwner()->getLoc())
+            .append("first generator here:");
+        error.attachNote(use.getOwner()->getLoc())
+            .append("second generator here:");
+        return error;
+      }
+
+      gen = &use;
+    } else if (applyees.first <= opnum &&
+               opnum < applyees.first + applyees.second) {
+      if (cons) {
+        InFlightDiagnostic error =
+            emitOpError("CLI must have at most one consumer");
+        error.attachNote(cons->getOwner()->getLoc())
+            .append("first consumer here:")
+            .appendOp(*cons->getOwner(),
+                      OpPrintingFlags().printGenericOpForm());
+        error.attachNote(use.getOwner()->getLoc())
+            .append("second consumer here:")
+            .appendOp(*use.getOwner(), OpPrintingFlags().printGenericOpForm());
+        return error;
+      }
+
+      cons = &use;
+    } else {
+      llvm_unreachable("Unexpected operand for a CLI");
+    }
+  }
+
+  // If the CLI is source of a transformation, it must have a generator
+  if (cons && !gen) {
+    InFlightDiagnostic error = emitOpError("CLI has no generator");
+    error.attachNote(cons->getOwner()->getLoc())
+        .append("see consumer here: ")
+        .appendOp(*cons->getOwner(), OpPrintingFlags().printGenericOpForm());
+    return error;
+  }
+
+  return success();
+}
+
+void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
+                            Value tripCount) {
+  odsState.addOperands(tripCount);
+  odsState.addOperands(Value());
+  (void)odsState.addRegion();
+}
+
+void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
+                            Value tripCount, ::mlir::Value cli) {
+  odsState.addOperands(tripCount);
+  odsState.addOperands(cli);
+  (void)odsState.addRegion();
+}
+
+void CanonicalLoopOp::getAsmBlockNames(OpAsmSetBlockNameFn setNameFn) {
+  setNameFn(&getRegion().front(), "body_entry");
+}
+
+void CanonicalLoopOp::getAsmBlockArgumentNames(Region &region,
+                                               OpAsmSetValueNameFn setNameFn) {
+  setNameFn(region.getArgument(0), "iv");
+}
+
+void CanonicalLoopOp::print(OpAsmPrinter &p) {
+  if (getCli())
+    p << '(' << getCli() << ')';
+  p << ' ' << getInductionVar() << " : " << getInductionVar().getType()
+    << " in range(" << getTripCount() << ") ";
+
+  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
+                /*printBlockTerminators=*/true);
+
+  p.printOptionalAttrDict((*this)->getAttrs());
+}
+
+mlir::ParseResult CanonicalLoopOp::parse(::mlir::OpAsmParser &parser,
+                                         ::mlir::OperationState &result) {
+  CanonicalLoopInfoType cliType =
+      CanonicalLoopInfoType::get(parser.getContext());
+
+  // Parse (optional) omp.cli identifier
+  OpAsmParser::UnresolvedOperand cli;
+  SmallVector<mlir::Value, 1> cliOperand;
+  if (!parser.parseOptionalLParen()) {
+    if (parser.parseOperand(cli) ||
+        parser.resolveOperand(cli, cliType, cliOperand) || parser.parseRParen())
+      return failure();
+  }
+
+  // We derive the type of tripCount from inductionVariable. MLIR requires the
+  // type of tripCount to be known when calling resolveOperand so we have parse
+  // the type before processing the inductionVariable.
+  OpAsmParser::Argument inductionVariable;
+  OpAsmParser::UnresolvedOperand tripcount;
+  if (parser.parseArgument(inductionVariable, /*allowType*/ true) ||
+      parser.parseKeyword("in") || parser.parseKeyword("range") ||
+      parser.parseLParen() || parser.parseOperand(tripcount) ||
+      parser.parseRParen() ||
+      parser.resolveOperand(tripcount, inductionVariable.type, result.operands))
+    return failure();
+
+  // Parse the loop body.
+  Region *region = result.addRegion();
+  if (parser.parseRegion(*region, {inductionVariable}))
+    return failure();
+
+  // We parsed the cli operand forst, but because it is optional, it must be
+  // last in the operand list.
+  result.operands.append(cliOperand);
+
+  // Parse the optional attribute list.
+  if (parser.parseOptionalAttrDict(result.attributes))
+    return failure();
+
+  return mlir::success();
+}
+
+LogicalResult CanonicalLoopOp::verify() {
+  // The region's entry must accept the induction variable
+  // It can also be empty if just created
+  if (!getRegion().empty()) {
+    Region &region = getRegion();
+    if (region.getNumArguments() != 1)
+      return emitOpError(
+          "Canonical loop region must have exactly one argument");
+
+    if (getInductionVar().getType() != getTripCount().getType())
+      return emitOpError(
+          "Region argument must be the same type as the trip count");
+  }
+
+  return success();
+}
+
+Value CanonicalLoopOp::getInductionVar() { return getRegion().getArgument(0); }
+
+std::pair<unsigned, unsigned>
+CanonicalLoopOp::getApplyeesODSOperandIndexAndLength() {
+  // No applyees
+  return {0, 0};
+}
+
+std::pair<unsigned, unsigned>
+CanonicalLoopOp::getGenerateesODSOperandIndexAndLength() {
+  return getODSOperandIndexAndLength(odsIndex_cli);
+}
+
+//===----------------------------------------------------------------------===//
+// UnrollHeuristicOp
+//===----------------------------------------------------------------------===//
+
+void UnrollHeuristicOp::build(::mlir::OpBuilder &odsBuilder,
+                              ::mlir::OperationState &odsState,
+                              ::mlir::Value cli) {
+  odsState.addOperands(cli);
+}
+
+void UnrollHeuristicOp::print(OpAsmPrinter &p) {
+  p << '(' << getApplyee() << ')';
+
+  p.printOptionalAttrDict((*this)->getAttrs());
+}
+
+mlir::ParseResult UnrollHeuristicOp::parse(::mlir::OpAsmParser &parser,
+                                           ::mlir::OperationState &result) {
+  auto cliType = CanonicalLoopInfoType::get(parser.getContext());
+
+  if (parser.parseLParen())
+    return failure();
+
+  OpAsmParser::UnresolvedOperand applyee;
+  if (parser.parseOperand(applyee) ||
+      parser.resolveOperand(applyee, cliType, result.operands))
+    return failure();
+
+  if (parser.parseRParen())
+    return failure();
+
+  // Optional output loop (full unrolling has none)
+  if (!parser.parseOptionalArrow()) {
+    if (parser.parseLParen() || parser.parseRParen())
+      return failure();
+  }
+
+  // Parse the optional attribute list.
+  if (parser.parseOptionalAttrDict(result.attributes))
+    return failure();
+
+  return mlir::success();
+}
+
+std::pair<unsigned, unsigned>
+UnrollHeuristicOp ::getApplyeesODSOperandIndexAndLength() {
+  return getODSOperandIndexAndLength(odsIndex_applyee);
+}
+
+std::pair<unsigned, unsigned>
+UnrollHeuristicOp::getGenerateesODSOperandIndexAndLength() {
+  return {0, 0};
+}
+
 //===----------------------------------------------------------------------===//
 // Critical construct (2.17.1)
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 6bccc1d6f5d30..d3cfdf26a344b 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3014,6 +3014,67 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
   return success();
 }
 
+/// Convert an omp.canonical_loop to LLVM-IR
+static LogicalResult
+convertOmpCanonicalLoopOp(omp::CanonicalLoopOp op, llvm::IRBuilderBase &builder,
+                          LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
+
+  llvm::OpenMPIRBuilder::LocationDescription loopLoc(builder);
+  Value loopIV = op.getInductionVar();
+  Value loopTC = op.getTripCount();
+
+  llvm::Value *llvmTC = moduleTranslation.lookupValue(loopTC);
+
+  llvm::Expected<llvm::CanonicalLoopInfo *> llvmOrError =
+      ompBuilder->createCanonicalLoop(
+          loopLoc,
+          [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *llvmIV) {
+            // Register the mapping of MLIR induction variable to LLVM-IR
+            // induction variable
+            moduleTranslation.mapValue(loopIV, llvmIV);
+
+            builder.restoreIP(ip);
+            llvm::Expected<llvm::BasicBlock *> bodyGenStatus =
+                convertOmpOpRegions(op.getRegion(), "omp.loop.region", builder,
+                                    moduleTranslation);
+
+            return bodyGenStatus.takeError();
+          },
+          llvmTC, "omp.loop");
+  if (!llvmOrError)
+    return op.emitError(llvm::toString(llvmOrError.takeError()));
+
+  llvm::CanonicalLoopInfo *llvmCLI = *llvmOrError;
+  llvm::IRBuilderBase::InsertPoint afterIP = llvmCLI->getAfterIP();
+  builder.restoreIP(afterIP);
+
+  // Register the mapping of MLIR loop to LLVM-IR OpenMPIRBuilder loop
+  if (Value cli = op.getCli())
+    moduleTranslation.mapOmpLoop(cli, llvmCLI);
+
+  return success();
+}
+
+/// Apply a `#pragma omp unroll` / "!$omp unroll" transformation using the
+/// OpenMPIRBuilder.
+static LogicalResult
+applyUnrollHeuristic(omp::UnrollHeuristicOp op, llvm::IRBuilderBase &builder,
+                     LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
+
+  Value applyee = op.getApplyee();
+  assert(applyee && "Loop to apply unrolling on required");
+
+  llvm::CanonicalLoopInfo *consBuilderCLI =
+      moduleTranslation.lookupOMPLoop(applyee);
+  llvm::OpenMPIRBuilder::LocationDescription loc(builder);
+  ompBuilder->unrollLoopHeuristic(loc.DL, consBuilderCLI);
+
+  moduleTranslation.invalidateOmpLoop(applyee);
+  return success();
+}
+
 /// Convert an Atomic Ordering attribute to llvm::AtomicOrdering.
 static llvm::AtomicOrdering
 convertAtomicOrdering(std::optional<omp::ClauseMemoryOrderKind> ao) {
@@ -5866,6 +5927,23 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
                 // etc. and then discarded
                 return success();
               })
+          .Case([&](omp::NewCliOp op) {
+            // Meta-operation: Doesn't do anything by itself, but used to
+            // identify a loop.
+            return success();
+          })
+          .Case([&](omp::CanonicalLoopOp op) {
+            return convertOmpCanonicalLoopOp(op, builder, moduleTranslation);
+          })
+          .Case([&](omp::UnrollHeuristicOp op) {
+            // FIXME: Handling omp.unroll_heuristic as an executable requires
+            // that the generator (e.g. omp.canonical_loop) has been seen first.
+            // For construct that require all codegen to occur inside a callback
+            // (e.g. OpenMPIRBilder::createParallel), all codegen of that
+            // contained region including their transformations must occur at
+            // the omp.canonical_loop.
+            return applyUnrollHeuristic(op, builder, moduleTranslation);
+          })
           .Default([&](Operation *inst) {
             return inst->emitError()
                    << "not yet implemented: " << inst->getName();
diff --git a/mlir/test/Dialect/OpenMP/cli-canonical_loop-invalid.mlir b/mlir/test/Dialect/OpenMP/cli-canonical_loop-invalid.mlir
new file mode 100644
index 0000000000000..345c53d2890b8
--- /dev/null
+++ b/mlir/test/Dialect/OpenMP/cli-canonical_loop-invalid.mlir
@@ -0,0 +1,50 @@
+// RUN: mlir-opt %s -verify-diagnostics
+
+
+func.func @omp_canonloop_block_arg(%tc : i32) -> () {
+  // expected-error at below {{Canonical loop region must have exactly one argument}}
+  "omp.canonical_loop" (%tc) ({
+    ^bb0(%iv: i32, %somearg: i32):
+      omp.terminator
+  }) : (i32) -> ()
+  return
+}
+
+
+func.func @omp_canonloop_multiple_generators(%tc : i32) -> () {
+  // expected-error at below {{'omp.new_cli' op CLI must have at most one generator}}
+  %cli = omp.new_cli
+  // expected-note at below {{second generator here}}
+  omp.canonical_loop(%cli) %iv1 : i32 in range(%tc) {
+    omp.terminator
+  }
+  // expected-note at below {{first generator here}}
+  omp.canonical_loop(%cli) %iv2 : i32 in range(%tc) {
+    omp.terminator
+  }
+  return
+}
+
+
+func.func @omp_canonloop_multiple_consumers() -> () {
+  // expected-error at below {{'omp.new_cli' op CLI must have at most one consumer}}
+  %cli = omp.new_cli
+  %tc = llvm.mlir.constant(4 : i32) : i32
+  omp.canonical_loop(%cli) %iv1 : i32 in range(%tc) {
+    omp.terminator
+  }
+  // expected-note at below {{second consumer here}}
+  omp.unroll_heuristic(%cli)
+  // expected-note at below {{first consumer here}}
+  omp.unroll_heuristic(%cli)
+  return
+}
+
+
+func.func @omp_canonloop_no_generator() -> () {
+  // expected-error at below {{'omp.new_cli' op CLI has no generator}}
+  %cli = omp.new_cli
+  // expected-note at below {{see consumer here}}
+  omp.unroll_heuristic(%cli)
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/cli-canonical_loop.mlir b/mlir/test/Dialect/OpenMP/cli-canonical_loop.mlir
new file mode 100644
index 0000000000000..adadb8bbac49d
--- /dev/null
+++ b/mlir/test/Dialect/OpenMP/cli-canonical_loop.mlir
@@ -0,0 +1,157 @@
+// RUN: mlir-opt %s | FileCheck %s
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+
+
+// CHECK-LABEL: @omp_canonloop_raw(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_raw(%tc : i32) -> () {
+  // CHECK: omp.canonical_loop %iv : i32 in range(%[[tc]]) {
+  "omp.canonical_loop" (%tc) ({
+    ^bb0(%iv: i32):
+      // CHECK-NEXT: = llvm.add %iv, %iv : i32
+      %newval = llvm.add %iv, %iv : i32
+      // CHECK-NEXT: omp.terminator
+      omp.terminator
+  // CHECK-NEXT: }
+  }) : (i32) -> ()
+  // CHECK-NEXT: return
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_sequential_raw(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_sequential_raw(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %canonloop_s0 = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  "omp.canonical_loop" (%tc, %canonloop_s0) ({
+    ^bb_first(%iv_first: i32):
+      // CHECK-NEXT: = llvm.add %iv, %iv : i32
+      %newval = llvm.add %iv_first, %iv_first : i32
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  // CHECK-NEXT: }
+  }) : (i32, !omp.cli) -> ()
+
+  // CHECK-NEXT: %canonloop_s1 = omp.new_cli
+  %canonloop_s1 = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT: omp.canonical_loop(%canonloop_s1) %iv : i32 in range(%[[tc]]) {
+  "omp.canonical_loop" (%tc, %canonloop_s1) ({
+    ^bb_second(%iv_second: i32):
+    // CHECK: omp.terminator
+    omp.terminator
+  // CHECK-NEXT: }
+  }) : (i32, !omp.cli) -> ()
+
+  // CHECK-NEXT: return
+  return
+}
+
+
+// CHECK-LABEL: @omp_nested_canonloop_raw(
+// CHECK-SAME: %[[tc_outer:.+]]: i32, %[[tc_inner:.+]]: i32)
+func.func @omp_nested_canonloop_raw(%tc_outer : i32, %tc_inner : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %outer = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT: %canonloop_s0_s0 = omp.new_cli
+  %inner = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc_outer]]) {
+  "omp.canonical_loop" (%tc_outer, %outer) ({
+    ^bb_outer(%iv_outer: i32):
+      // CHECK-NEXT: omp.canonical_loop(%canonloop_s0_s0) %iv_0 : i32 in range(%[[tc_inner]]) {
+      "omp.canonical_loop" (%tc_inner, %inner) ({
+        ^bb_inner(%iv_inner: i32):
+          // CHECK-NEXT: = llvm.add %iv, %iv_0 : i32
+          %newval = llvm.add %iv_outer, %iv_inner: i32
+          // CHECK-NEXT: omp.terminator
+          omp.terminator
+      }) : (i32, !omp.cli) -> ()
+      // CHECK: omp.terminator
+      omp.terminator
+  }) : (i32, !omp.cli) -> ()
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: omp.canonical_loop %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop %iv : i32 in range(%tc) {
+    // CHECK-NEXT: llvm.add %iv, %iv : i32
+    %newval = llvm.add %iv, %iv: i32
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_constant_pretty()
+func.func @omp_canonloop_constant_pretty() -> () {
+  // CHECK-NEXT:  %[[tc:.+]] = llvm.mlir.constant(4 : i32) : i32
+  %tc = llvm.mlir.constant(4 : i32) : i32
+  // CHECK-NEXT: omp.canonical_loop %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop %iv : i32 in range(%tc) {
+    // CHECK-NEXT: llvm.add %iv, %iv : i32
+    %newval = llvm.add %iv, %iv: i32
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_sequential_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_sequential_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %canonloop_s0 = omp.new_cli
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%tc) {
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  }
+
+  // CHECK: %canonloop_s1 = omp.new_cli
+  %canonloop_s1 = omp.new_cli
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s1) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%canonloop_s1) %iv_0 : i32 in range(%tc) {
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  }
+
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_nested_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_nested_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %canonloop_s0 = omp.new_cli
+  // CHECK-NEXT: %canonloop_s0_s0 = omp.new_cli
+  %canonloop_s0_s0 = omp.new_cli
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%tc) {
+    // CHECK-NEXT: omp.canonical_loop(%canonloop_s0_s0) %iv_0 : i32 in range(%[[tc]]) {
+    omp.canonical_loop(%canonloop_s0_s0) %iv_0 : i32 in range(%tc) {
+      // CHECK: omp.terminator
+      omp.terminator
+    }
+    // CHECK: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+
+// CHECK-LABEL: @omp_newcli_unused(
+// CHECK-SAME: )
+func.func @omp_newcli_unused() -> () {
+  // CHECK-NEXT:  %cli = omp.new_cli
+  %cli = omp.new_cli
+  // CHECK-NEXT: return
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/cli-unroll-heuristic.mlir b/mlir/test/Dialect/OpenMP/cli-unroll-heuristic.mlir
new file mode 100644
index 0000000000000..cda7d0b500166
--- /dev/null
+++ b/mlir/test/Dialect/OpenMP/cli-unroll-heuristic.mlir
@@ -0,0 +1,59 @@
+// RUN: mlir-opt %s            | FileCheck %s
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+
+
+// CHECK-LABEL: @omp_unroll_heuristic_raw(
+// CHECK-SAME: %[[tc:.+]]: i32) {
+func.func @omp_unroll_heuristic_raw(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %canonloop = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  "omp.canonical_loop" (%tc, %canonloop) ({
+    ^bb0(%iv: i32):
+      omp.terminator
+  }) : (i32, !omp.cli) -> ()
+  // CHECK: omp.unroll_heuristic(%canonloop_s0)
+  "omp.unroll_heuristic" (%canonloop) : (!omp.cli) -> ()
+  return
+}
+
+
+// CHECK-LABEL: @omp_unroll_heuristic_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32) {
+func.func @omp_unroll_heuristic_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: %[[CANONLOOP:.+]] = omp.new_cli
+  %canonloop = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%canonloop) %iv : i32 in range(%tc) {
+    omp.terminator
+  }
+  // CHECK: omp.unroll_heuristic(%canonloop_s0)
+  omp.unroll_heuristic(%canonloop)
+  return
+}
+
+
+// CHECK-LABEL: @omp_unroll_heuristic_nested_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32) {
+func.func @omp_unroll_heuristic_nested_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %cli_outer = omp.new_cli
+  // CHECK-NEXT: %canonloop_s0_s0 = omp.new_cli
+  %cli_inner = omp.new_cli
+  // CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%cli_outer) %iv_outer : i32 in range(%tc) {
+    // CHECK-NEXT: omp.canonical_loop(%canonloop_s0_s0) %iv_0 : i32 in range(%[[tc]]) {
+    omp.canonical_loop(%cli_inner) %iv_inner : i32 in range(%tc) {
+      // CHECK: omp.terminator
+      omp.terminator
+    }
+    // CHECK: omp.terminator
+    omp.terminator
+  }
+
+  // CHECK: omp.unroll_heuristic(%canonloop_s0)
+  omp.unroll_heuristic(%cli_outer)
+  // CHECK-NEXT: omp.unroll_heuristic(%canonloop_s0_s0)
+  omp.unroll_heuristic(%cli_inner)
+  return
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-canonical_loop.mlir b/mlir/test/Target/LLVMIR/openmp-cli-canonical_loop.mlir
new file mode 100644
index 0000000000000..9abef003d6183
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-cli-canonical_loop.mlir
@@ -0,0 +1,175 @@
+// Test lowering of standalone omp.canonical_loop
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK-LABEL: define void @anon_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]],
+// CHECK-SAME:    i32 %[[tc:.+]]) {
+// CHECK-NEXT:    br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.preheader:
+// CHECK-NEXT:    br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.header:
+// CHECK-NEXT:    %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:    br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.cond:
+// CHECK-NEXT:    %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[tc]]
+// CHECK-NEXT:    br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.body:
+// CHECK-NEXT:    br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp.loop.region:
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:    br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp.region.cont:
+// CHECK-NEXT:    br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.inc:
+// CHECK-NEXT:    %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:    br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.exit:
+// CHECK-NEXT:    br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.after:
+// CHECK-NEXT:    ret void
+// CHECK-NEXT:  }
+llvm.func @anon_loop(%ptr: !llvm.ptr, %tc : i32) -> () {
+  omp.canonical_loop %iv : i32 in range(%tc) {
+    %val = llvm.mlir.constant(42.0 : f32) : f32
+    llvm.store %val, %ptr : f32, !llvm.ptr
+    omp.terminator
+  }
+  llvm.return
+}
+
+
+
+// CHECK-LABEL: define void @trivial_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]],
+// CHECK-SAME:    i32 %[[tc:.+]]) {
+// CHECK-NEXT:    br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.preheader:
+// CHECK-NEXT:    br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.header:
+// CHECK-NEXT:    %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:    br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.cond:
+// CHECK-NEXT:    %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[tc]]
+// CHECK-NEXT:    br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.body:
+// CHECK-NEXT:    br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp.loop.region:
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:    br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp.region.cont:
+// CHECK-NEXT:    br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.inc:
+// CHECK-NEXT:    %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:    br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.exit:
+// CHECK-NEXT:    br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.after:
+// CHECK-NEXT:    ret void
+// CHECK-NEXT:  }
+llvm.func @trivial_loop(%ptr: !llvm.ptr, %tc : i32) -> () {
+  %cli = omp.new_cli
+  omp.canonical_loop(%cli) %iv : i32 in range(%tc) {
+    %val = llvm.mlir.constant(42.0 : f32) : f32
+    llvm.store %val, %ptr : f32, !llvm.ptr
+    omp.terminator
+  }
+  llvm.return
+}
+
+
+// CHECK-LABEL: define void @nested_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]], i32 %[[outer_tc:.+]], i32 %[[inner_tc:.+]]) {
+// CHECK-NEXT:  br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader:
+// CHECK-NEXT:  br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header:
+// CHECK-NEXT:  %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:  br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond:
+// CHECK-NEXT:  %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[outer_tc]]
+// CHECK-NEXT:  br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body:
+// CHECK-NEXT:  br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region:
+// CHECK-NEXT:  br label %omp_omp.loop.preheader1
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader1:
+// CHECK-NEXT:  br label %omp_omp.loop.header2
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header2:
+// CHECK-NEXT:  %omp_omp.loop.iv8 = phi i32 [ 0, %omp_omp.loop.preheader1 ], [ %omp_omp.loop.next10, %omp_omp.loop.inc5 ]
+// CHECK-NEXT:  br label %omp_omp.loop.cond3
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond3:
+// CHECK-NEXT:  %omp_omp.loop.cmp9 = icmp ult i32 %omp_omp.loop.iv8, %[[inner_tc]]
+// CHECK-NEXT:  br i1 %omp_omp.loop.cmp9, label %omp_omp.loop.body4, label %omp_omp.loop.exit6
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body4:
+// CHECK-NEXT:  br label %omp.loop.region12
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region12:
+// CHECK-NEXT:  store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:  br label %omp.region.cont11
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont11:
+// CHECK-NEXT:  br label %omp_omp.loop.inc5
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc5:
+// CHECK-NEXT:  %omp_omp.loop.next10 = add nuw i32 %omp_omp.loop.iv8, 1
+// CHECK-NEXT:  br label %omp_omp.loop.header2
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit6:
+// CHECK-NEXT:  br label %omp_omp.loop.after7
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after7:
+// CHECK-NEXT:  br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont:
+// CHECK-NEXT:  br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc:
+// CHECK-NEXT:  %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:  br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit:
+// CHECK-NEXT:  br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after:
+// CHECK-NEXT:  ret void
+// CHECK-NEXT: }
+llvm.func @nested_loop(%ptr: !llvm.ptr, %outer_tc : i32, %inner_tc : i32) -> () {
+  %outer_cli = omp.new_cli
+  %inner_cli = omp.new_cli
+  omp.canonical_loop(%outer_cli) %outer_iv : i32 in range(%outer_tc) {
+    omp.canonical_loop(%inner_cli) %inner_iv : i32 in range(%inner_tc) {
+      %val = llvm.mlir.constant(42.0 : f32) : f32
+      llvm.store %val, %ptr : f32, !llvm.ptr
+      omp.terminator
+    }
+    omp.terminator
+  }
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic01.mlir b/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic01.mlir
new file mode 100644
index 0000000000000..0f0448e15f983
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic01.mlir
@@ -0,0 +1,56 @@
+// Test lowering of the omp.unroll_heuristic
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+
+// CHECK-LABEL: define void @unroll_heuristic_trivial_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]], i32 %[[tc:.+]]) {
+// CHECK-NEXT:   br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader:
+// CHECK-NEXT:   br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header:
+// CHECK-NEXT:   %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:   br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond:
+// CHECK-NEXT:   %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[tc]]
+// CHECK-NEXT:   br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body:
+// CHECK-NEXT:   br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region:
+// CHECK-NEXT:   store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:   br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont:
+// CHECK-NEXT:   br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc:
+// CHECK-NEXT:   %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:   br label %omp_omp.loop.header, !llvm.loop ![[$MD1:[0-9]+]]
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit:
+// CHECK-NEXT:   br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after:
+// CHECK-NEXT:   ret void
+// CHECK-NEXT: }
+llvm.func @unroll_heuristic_trivial_loop(%ptr: !llvm.ptr, %tc: i32) -> () {
+  %literal_cli = omp.new_cli
+  omp.canonical_loop(%literal_cli) %iv : i32 in range(%tc) {
+    %val = llvm.mlir.constant(42.0 : f32) : f32
+    llvm.store %val, %ptr : f32, !llvm.ptr
+    omp.terminator
+  }
+  omp.unroll_heuristic(%literal_cli)
+  llvm.return
+}
+
+
+// Start of metadata
+// CHECK-LABEL: !llvm.module.flags
+
+// CHECK: ![[$MD1]] = distinct !{![[$MD1]], ![[$MD2:[0-9]+]]}
+// CHECK: ![[$MD2]] = !{!"llvm.loop.unroll.enable"}
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic02.mlir b/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic02.mlir
new file mode 100644
index 0000000000000..f82b4990e378e
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic02.mlir
@@ -0,0 +1,93 @@
+// Test lowering of the omp.unroll_heuristic
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+
+// CHECK-LABEL: define void @unroll_heuristic_nested_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]], i32 %[[outer_tc:.+]], i32 %[[inner_tc:.+]]) {
+// CHECK-NEXT:   br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader:
+// CHECK-NEXT:   br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header:
+// CHECK-NEXT:   %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:   br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond:
+// CHECK-NEXT:   %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[outer_tc]]
+// CHECK-NEXT:   br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body:
+// CHECK-NEXT:   br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region:
+// CHECK-NEXT:   br label %omp_omp.loop.preheader1
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader1:
+// CHECK-NEXT:   br label %omp_omp.loop.header2
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header2:
+// CHECK-NEXT:   %omp_omp.loop.iv8 = phi i32 [ 0, %omp_omp.loop.preheader1 ], [ %omp_omp.loop.next10, %omp_omp.loop.inc5 ]
+// CHECK-NEXT:   br label %omp_omp.loop.cond3
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond3:
+// CHECK-NEXT:   %omp_omp.loop.cmp9 = icmp ult i32 %omp_omp.loop.iv8, %[[inner_tc]]
+// CHECK-NEXT:   br i1 %omp_omp.loop.cmp9, label %omp_omp.loop.body4, label %omp_omp.loop.exit6
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body4:
+// CHECK-NEXT:   br label %omp.loop.region12
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region12:
+// CHECK-NEXT:   store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:   br label %omp.region.cont11
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont11:
+// CHECK-NEXT:   br label %omp_omp.loop.inc5
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc5:
+// CHECK-NEXT:   %omp_omp.loop.next10 = add nuw i32 %omp_omp.loop.iv8, 1
+// CHECK-NEXT:   br label %omp_omp.loop.header2, !llvm.loop ![[$MD1:[0-9]+]]
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit6:
+// CHECK-NEXT:   br label %omp_omp.loop.after7
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after7:
+// CHECK-NEXT:   br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont:
+// CHECK-NEXT:   br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc:
+// CHECK-NEXT:   %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:   br label %omp_omp.loop.header, !llvm.loop ![[$MD3:[0-9]+]]
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit:
+// CHECK-NEXT:   br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after:
+// CHECK-NEXT:   ret void
+// CHECK-NEXT: }
+llvm.func @unroll_heuristic_nested_loop(%ptr: !llvm.ptr, %outer_tc: i32, %inner_tc: i32) -> () {
+  %outer_cli = omp.new_cli
+  %inner_cli = omp.new_cli
+  omp.canonical_loop(%outer_cli) %outer_iv : i32 in range(%outer_tc) {
+    omp.canonical_loop(%inner_cli) %inner_iv : i32 in range(%inner_tc) {
+      %val = llvm.mlir.constant(42.0 : f32) : f32
+      llvm.store %val, %ptr : f32, !llvm.ptr
+      omp.terminator
+    }
+    omp.terminator
+  }
+  omp.unroll_heuristic(%outer_cli)
+  omp.unroll_heuristic(%inner_cli)
+  llvm.return
+}
+
+
+// Start of metadata
+// CHECK-LABEL: !llvm.module.flags
+
+// CHECK: ![[$MD1]] = distinct !{![[$MD1]], ![[$MD2:[0-9]+]]}
+// CHECK: ![[$MD2]] = !{!"llvm.loop.unroll.enable"}
+// CHECK: ![[$MD3]] = distinct !{![[$MD3]], ![[$MD2]]}
+
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 6008ed4673d1b..cbb4030f3adb4 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -2223,6 +2223,17 @@ generateNamedOperandGetters(const Operator &op, Class &opClass,
                     "'SameVariadicOperandSize' traits");
   }
 
+  // Print the ods names so they don't need to be hardcoded in the source.
+  for (int i = 0; i != numOperands; ++i) {
+    const auto &operand = op.getOperand(i);
+    if (operand.name.empty())
+      continue;
+
+    opClass.declare<Field>("static constexpr int", Twine("odsIndex_") +
+                                                       operand.name + " = " +
+                                                       Twine(i));
+  }
+
   // First emit a few "sink" getter methods upon which we layer all nicer named
   // getter methods.
   // If generating for an adaptor, the method is put into the non-templated

>From 4a18faad39910ea4d4b66185ba25488ea905e019 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 3 Jul 2025 10:51:55 +0200
Subject: [PATCH 2/5] Emit named operand indices

---
 mlir/test/mlir-tblgen/op-operand.td         | 11 +++++++++++
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp | 11 +++++++++++
 2 files changed, 22 insertions(+)

diff --git a/mlir/test/mlir-tblgen/op-operand.td b/mlir/test/mlir-tblgen/op-operand.td
index a2fa1f7046a97..ab8d721ed5427 100644
--- a/mlir/test/mlir-tblgen/op-operand.td
+++ b/mlir/test/mlir-tblgen/op-operand.td
@@ -13,6 +13,9 @@ def OpA : NS_Op<"one_normal_operand_op", []> {
   let arguments = (ins I32:$input);
 }
 
+// DECL-LABEL: class OpA : {{.*}} {
+// DECL: static constexpr int odsIndex_input = 0;
+
 // CHECK-LABEL: OpA definitions
 
 // CHECK:      void OpA::build
@@ -28,6 +31,9 @@ def OpB : NS_Op<"one_variadic_operand_op", []> {
   let arguments = (ins Variadic<I32>:$input);
 }
 
+// DECL-LABEL: class OpB : {{.*}} {
+// DECL: static constexpr int odsIndex_input = 0;
+
 // CHECK-LABEL: OpB::build
 // CHECK:         ::mlir::ValueRange input
 // CHECK-NOT:     assert
@@ -37,6 +43,11 @@ def OpD : NS_Op<"mix_variadic_and_normal_inputs_op", [SameVariadicOperandSize]>
   let arguments = (ins Variadic<AnyTensor>:$input1, AnyTensor:$input2, Variadic<AnyTensor>:$input3);
 }
 
+// DECL-LABEL: class OpD : {{.*}} {
+// DECL: static constexpr int odsIndex_input1 = 0;
+// DECL: static constexpr int odsIndex_input2 = 1;
+// DECL: static constexpr int odsIndex_input3 = 2;
+
 // DECL-LABEL: ::mlir::Operation::operand_range getInput1
 // DECL-NEXT: return getODSOperands(0);
 
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 6008ed4673d1b..cbb4030f3adb4 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -2223,6 +2223,17 @@ generateNamedOperandGetters(const Operator &op, Class &opClass,
                     "'SameVariadicOperandSize' traits");
   }
 
+  // Print the ods names so they don't need to be hardcoded in the source.
+  for (int i = 0; i != numOperands; ++i) {
+    const auto &operand = op.getOperand(i);
+    if (operand.name.empty())
+      continue;
+
+    opClass.declare<Field>("static constexpr int", Twine("odsIndex_") +
+                                                       operand.name + " = " +
+                                                       Twine(i));
+  }
+
   // First emit a few "sink" getter methods upon which we layer all nicer named
   // getter methods.
   // If generating for an adaptor, the method is put into the non-templated

>From c7ed06d92dc7e256bff5370eb3d74d3a43547625 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 4 Jul 2025 14:50:46 +0200
Subject: [PATCH 3/5] Add omp.canonical_loop and omp.unroll modelling

---
 .../mlir/Dialect/OpenMP/OpenMPDialect.h       |   5 +
 .../mlir/Dialect/OpenMP/OpenMPOpBase.td       |  11 +
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 207 ++++++++++
 .../Dialect/OpenMP/OpenMPOpsInterfaces.td     |  86 +++++
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 353 ++++++++++++++++++
 .../OpenMP/cli-canonical_loop-invalid.mlir    |  50 +++
 .../Dialect/OpenMP/cli-canonical_loop.mlir    | 157 ++++++++
 .../Dialect/OpenMP/cli-unroll-heuristic.mlir  |  59 +++
 8 files changed, 928 insertions(+)
 create mode 100644 mlir/test/Dialect/OpenMP/cli-canonical_loop-invalid.mlir
 create mode 100644 mlir/test/Dialect/OpenMP/cli-canonical_loop.mlir
 create mode 100644 mlir/test/Dialect/OpenMP/cli-unroll-heuristic.mlir

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
index ab11a6094e3e7..7cf738352ba47 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
@@ -37,4 +37,9 @@
 #define GET_OP_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOps.h.inc"
 
+namespace mlir::omp {
+/// Find the omp.new_cli, generator, and consumer of a canonical loop info.
+std::tuple<NewCliOp, OpOperand *, OpOperand *> decodeCli(mlir::Value cli);
+} // namespace mlir::omp
+
 #endif // MLIR_DIALECT_OPENMP_OPENMPDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpBase.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpBase.td
index f3dd44d2c0717..bbcfb87fa03c6 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpBase.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpBase.td
@@ -204,4 +204,15 @@ class OpenMP_Op<string mnemonic, list<Trait> traits = [],
   let regions = !if(singleRegion, (region AnyRegion:$region), (region));
 }
 
+
+// Base class for OpenMP loop transformations (that either consume or generate
+// loops)
+//
+// Doesn't actually create a C++ base class (only defines default values for
+// tablegen classes that derive from this). Use LoopTransformationInterface
+// instead for common operations.
+class OpenMPTransform_Op<string mnemonic, list<Trait> traits = []> :
+      OpenMP_Op<mnemonic, !listconcat([DeclareOpInterfaceMethods<LoopTransformationInterface>], traits)  > {
+}
+
 #endif  // OPENMP_OP_BASE
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..703384a0680d0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -22,6 +22,7 @@ include "mlir/Dialect/OpenMP/OpenMPOpBase.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 include "mlir/IR/EnumAttr.td"
+include "mlir/IR/OpAsmInterface.td"
 include "mlir/IR/OpBase.td"
 include "mlir/IR/SymbolInterfaces.td"
 
@@ -356,6 +357,212 @@ def SingleOp : OpenMP_Op<"single", traits = [
   let hasVerifier = 1;
 }
 
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Info Type
+//===---------------------------------------------------------------------===//
+
+def CanonicalLoopInfoType : OpenMP_Type<"CanonicalLoopInfo", "cli"> {
+  let summary = "Type for representing a reference to a canonical loop";
+  let description = [{
+    A variable of type CanonicalLoopInfo refers to an OpenMP-compatible
+    canonical loop in the same function. Values of this type are not
+    available at runtime and therefore cannot be used by the program itself,
+    i.e. an opaque type. It is similar to the transform dialect's
+    `!transform.interface` type, but instead of implementing an interface
+    for each transformation, the OpenMP dialect itself defines possible
+    operations on this type.
+
+    A value of type CanonicalLoopInfoType (in the following: CLI) value can be
+
+    1. created by omp.new_cli.
+    2. passed to omp.canonical_loop to associate the loop to that CLI. A CLI
+       can only be associated once.
+    3. passed to an omp loop transformation operation that modifies the loop
+       associated with the CLI. The CLI is the "applyee" and the operation is
+       the consumer. A CLI can only be consumed once.
+    4. passed to an omp loop transformation operation to associate the cli with
+       a result of that transformation. The CLI is the "generatee" and the
+       operation is the generator.
+
+    A CLI cannot
+
+    1. be returned from a function.
+    2. be passed to operations that are not specifically designed to take a
+       CanonicalLoopInfoType, including AnyType.
+
+    A CLI directly corresponds to an object of
+    OpenMPIRBuilder's CanonicalLoopInfo struct when lowering to LLVM-IR.
+  }];
+}
+
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Info Creation
+//===---------------------------------------------------------------------===//
+
+def NewCliOp : OpenMP_Op<"new_cli",
+    [DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>]> {
+  let summary = "Create a new Canonical Loop Info value.";
+  let description = [{
+    Create a new CLI that can be passed as an argument to a CanonicalLoopOp
+    and to loop transformation operations to handle dependencies between
+    loop transformation operations.
+  }];
+
+  let arguments = (ins );
+  let results = (outs CanonicalLoopInfoType:$result);
+  let assemblyFormat = [{
+      attr-dict
+  }];
+
+  let builders = [
+    OpBuilder<(ins )>,
+  ];
+
+  let hasVerifier = 1;
+}
+
+//===---------------------------------------------------------------------===//
+// OpenMP Canonical Loop Operation
+//===---------------------------------------------------------------------===//
+def CanonicalLoopOp : OpenMPTransform_Op<"canonical_loop", 
+    [DeclareOpInterfaceMethods<OpAsmOpInterface, [ "getAsmBlockNames", "getAsmBlockArgumentNames"]>]> {
+  let summary = "OpenMP Canonical Loop Operation";
+  let description = [{
+    All loops that conform to OpenMP's definition of a canonical loop can be
+    simplified to a CanonicalLoopOp. In particular, there are no loop-carried
+    variables and the number of iterations it will execute is known before the
+    operation. This allows e.g. to determine the number of threads and chunks
+    the iterations space is split into before executing any iteration. More
+    restrictions may apply in cases such as (collapsed) loop nests, doacross
+    loops, etc.
+
+    In contrast to other loop operations such as `scf.for`, the number of
+    iterations is determined by only a single variable, the trip-count. The
+    induction variable value is the logical iteration number of that iteration,
+    which OpenMP defines to be between 0 and the trip-count (exclusive).
+    Loop representation having lower-bound, upper-bound, and step-size operands,
+    require passes to do more work than necessary, including handling special
+    cases such as upper-bound smaller than lower-bound, upper-bound equal to
+    the integer type's maximal value, negative step size, etc. This complexity
+    is better only handled once by the front-end and can apply its semantics
+    for such cases while still being able to represent any kind of loop, which
+    kind of the point of a mid-end intermediate representation. User-defined
+    types such as random-access iterators in C++ could not directly be
+    represented anyway.
+
+    The induction variable is always of the same type as the tripcount argument.
+    Since it can never be negative, tripcount is always interpreted as an
+    unsigned integer. It is the caller's responsibility to ensure the tripcount
+    is not negative when its interpretation is signed, i.e.
+    `%tripcount = max(0,%tripcount)`.
+
+    An optional argument to a omp.canonical_loop that can be passed in
+    is a CanonicalLoopInfo value that can be used to refer to the canonical
+    loop to apply transformations -- such as tiling, unrolling, or
+    work-sharing -- to the loop, similar to the transform dialect but
+    with OpenMP-specific semantics. Because it is optional, it has to be the
+    last of the operands, but appears first in the pretty format printing.
+
+    The pretty assembly format is inspired by python syntax, where `range(n)`
+    returns an iterator that runs from $0$ to $n-1$. The pretty assembly syntax
+    is one of:
+
+     omp.canonical_loop(%cli) %iv : !type in range(%tripcount)
+     omp.canonical_loop       %iv : !type in range(%tripcount)
+
+    A CanonicalLoopOp is lowered to LLVM-IR using
+    `OpenMPIRBuilder::createCanonicalLoop`.
+
+    #### Examples
+
+    Translation from lower-bound, upper-bound, step-size to trip-count.
+    ```c
+    for (int i = 3; i < 42; i+=2) {
+      B[i] = A[i];
+    }
+    ```
+
+    ```mlir
+    %lb = arith.constant 3 : i32
+    %ub = arith.constant 42 : i32
+    %step = arith.constant 2 : i32
+    %range = arith.sub %ub, %lb : i32
+    %tripcount = arith.div %range, %step : i32
+    omp.canonical_loop %iv : i32 in range(%tripcount) {
+      %offset = arith.mul %iv, %step : i32
+      %i = arith.add %offset, %lb : i32
+      %a = load %arrA[%i] : memref<?xf32>
+      store %a, %arrB[%i] : memref<?xf32>
+    }
+    ```
+
+    Nested canonical loop with transformation of the inner loop.
+    ```mlir
+    %outer = omp.new_cli : !omp.cli
+    %inner = omp.new_cli : !omp.cli
+    omp.canonical_loop(%outer) %iv1 : i32 in range(%tc1) {
+      omp.canonical_loop(%inner) %iv2 : i32 in range(%tc2) {
+        %a = load %arrA[%iv1, %iv2] : memref<?x?xf32>
+        store %a, %arrB[%iv1, %iv2] : memref<?x?xf32>
+      }
+    }
+    omp.unroll_full(%inner)
+    ```
+  }];
+
+
+  let arguments = (ins IntLikeType:$tripCount,
+                       Optional<CanonicalLoopInfoType>:$cli);
+  let regions = (region AnyRegion:$region);
+
+  let extraClassDeclaration = [{
+    ::mlir::Value getInductionVar();
+  }];
+
+  let builders = [
+    OpBuilder<(ins "::mlir::Value":$tripCount)>,
+    OpBuilder<(ins "::mlir::Value":$tripCount, "::mlir::Value":$cli)>,
+  ];
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// OpenMP unroll_heuristic operation
+//===----------------------------------------------------------------------===//
+
+def UnrollHeuristicOp : OpenMPTransform_Op<"unroll_heuristic", []> {
+  let summary = "OpenMP heuristic unroll operation";
+  let description = [{
+    Represents a `#pragma omp unroll` construct introduced in OpenMP 5.1.
+
+    The operation has one applyee and no generatees. The applyee is unrolled
+    according to implementation-defined heuristics. Implementations may choose
+    to not unroll the loop, partially unroll by a chosen factor, or fully
+    unroll it. Even if the implementation chooses to partially unroll the
+    applyee, the resulting unrolled loop is not accessible as a generatee. Use
+    omp.unroll_partial if a generatee is required.
+
+    The lowering is implemented using `OpenMPIRBuilder::unrollLoopHeuristic`,
+    which just attaches `llvm.loop.unroll.enable` metadata to the loop so the
+    unrolling is carried-out by LLVM's LoopUnroll pass. That is, unrolling only
+    actually performed in optimized builds.
+
+    Assembly formats:
+      omp.unroll_heuristic(%cli)
+      omp.unroll_heuristic(%cli) -> ()
+  }];
+
+  let arguments = (ins CanonicalLoopInfoType:$applyee);
+
+  let builders = [
+    OpBuilder<(ins "::mlir::Value":$cli)>,
+  ];
+
+  let hasCustomAssemblyFormat = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // 2.8.3 Workshare Construct
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index 92bf34ef3145f..8d794918d57d5 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -551,4 +551,90 @@ def OffloadModuleInterface : OpInterface<"OffloadModuleInterface"> {
   ];
 }
 
+def LoopTransformationInterface : OpInterface<"LoopTransformationInterface"> {
+  let description = [{
+    Methods that are common for OpenMP loop transformation operations.
+  }];
+
+  let cppNamespace = "::mlir::omp";
+
+  let methods = [
+
+    InterfaceMethod<
+      /*description=*/[{
+        Get the indices for the arguments that represent CanonicalLoopInfo
+        applyees, i.e. loops that are transformed/consumed by this operation.
+      }],
+      /*returnType=*/ "std::pair<unsigned, unsigned>",
+      /*methodName=*/ "getApplyeesODSOperandIndexAndLength",
+      /*args=*/(ins)
+    >,
+
+    InterfaceMethod<
+      /*description=*/[{
+        Get the indices for the arguments that represent CanonicalLoopInfo
+        generatees, i.e. loops that created by this operation.
+      }],
+      /*returnType=*/ "std::pair<unsigned, unsigned>",
+      /*methodName=*/ "getGenerateesODSOperandIndexAndLength",
+      /*args=*/(ins)
+    >,
+
+    InterfaceMethod<
+      /*description=*/[{
+        Return the number of applyees of this loop transformation.
+      }],
+      /*returnType=*/ "unsigned",
+      /*methodName=*/ "getNumApplyees",
+      /*args=*/       (ins),
+      /*methodBody=*/ "",
+      /*defaultImpl=*/[{
+        return  $_op.getApplyeesODSOperandIndexAndLength().second;
+      }]
+    >,
+
+    InterfaceMethod<
+      /*description=*/[{
+        Return the number of generatees of this loop transformation.
+      }],
+      /*returnType=*/ "unsigned",
+      /*methodName=*/ "getNumGeneratees",
+      /*args=*/       (ins),
+      /*methodBody=*/ "",
+      /*defaultImpl=*/[{
+        return  $_op.getGenerateesODSOperandIndexAndLength().second;
+      }]
+    >,
+
+    InterfaceMethod<
+      /*description=*/[{
+        Return whether the provided operand is a generatee of this operation.
+      }],
+      /*returnType=*/ "unsigned",
+      /*methodName=*/ "isApplyee",
+      /*args=*/       (ins "unsigned":$opnum),
+      /*methodBody=*/ "",
+      /*defaultImpl=*/[{
+        auto applyeeArgs = $_op.getApplyeesODSOperandIndexAndLength();
+        return (applyeeArgs.first <= opnum && opnum < applyeeArgs.first + applyeeArgs.second) ;
+      }]
+    >,
+
+    InterfaceMethod<
+      /*description=*/[{
+        Return whether the provided operand is a generatee of this operation.
+      }],
+      /*returnType=*/ "unsigned",
+      /*methodName=*/ "isGeneratee",
+      /*args=*/       (ins "unsigned":$opnum),
+      /*methodBody=*/ "",
+      /*defaultImpl=*/[{
+        auto generateeArgs = $_op.getGenerateesODSOperandIndexAndLength();
+        return (generateeArgs.first <= opnum && opnum < generateeArgs.first + generateeArgs.second) ;
+      }]
+    >,
+
+  ];
+}
+
 #endif // OPENMP_OPS_INTERFACES
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..abd2e92e344ba 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -24,6 +24,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallString.h"
@@ -3012,6 +3013,358 @@ void LoopNestOp::gatherWrappers(
   }
 }
 
+//===----------------------------------------------------------------------===//
+// OpenMP canonical loop handling
+//===----------------------------------------------------------------------===//
+
+std::tuple<NewCliOp, OpOperand *, OpOperand *>
+mlir::omp ::decodeCli(Value cli) {
+
+  // Defining a CLI for a generated loop is optional; if there is none then
+  // there is no followup-tranformation
+  if (!cli)
+    return {{}, nullptr, nullptr};
+
+  MLIRContext *ctx = cli.getContext();
+  assert(cli.getType() == CanonicalLoopInfoType::get(ctx) &&
+         "Unexpected type of cli");
+
+  NewCliOp create = cast<NewCliOp>(cli.getDefiningOp());
+  OpOperand *gen = nullptr;
+  OpOperand *cons = nullptr;
+  for (OpOperand &use : cli.getUses()) {
+    auto op = cast<LoopTransformationInterface>(use.getOwner());
+
+    unsigned opnum = use.getOperandNumber();
+    if (op.isGeneratee(opnum)) {
+      assert(!gen && "Each CLI may have at most one def");
+      gen = &use;
+    } else if (op.isApplyee(opnum)) {
+      assert(!cons && "Each CLI may have at most one consumer");
+      cons = &use;
+    } else {
+      llvm_unreachable("Unexpected operand for a CLI");
+    }
+  }
+
+  return {create, gen, cons};
+}
+
+void NewCliOp::build(::mlir::OpBuilder &odsBuilder,
+                     ::mlir::OperationState &odsState) {
+  odsState.addTypes(CanonicalLoopInfoType::get(odsBuilder.getContext()));
+}
+
+void NewCliOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {
+  Value result = getResult();
+  auto [newCli, gen, cons] = decodeCli(result);
+
+  // Derive the CLI variable name from its generator:
+  //  * "canonloop" for omp.canonical_loop
+  //  * custom name for loop transformation generatees
+  //  * "cli" as fallback if no generator
+  //  * "_r<idx>" suffix for nested loops, where <idx> is the sequential order
+  //  at that level
+  //  * "_s<idx>" suffix for operations with multiple regions, where <idx> is
+  //  the index of that region
+  std::string cliName{"cli"};
+  if (gen) {
+    cliName =
+        TypeSwitch<Operation *, std::string>(gen->getOwner())
+            .Case([&](CanonicalLoopOp op) {
+              // Find the canonical loop nesting: For each ancestor add a
+              // "+_r<idx>" suffix (in reverse order)
+              SmallVector<std::string> components;
+              Operation *o = op.getOperation();
+              while (o) {
+                if (o->hasTrait<mlir::OpTrait::IsIsolatedFromAbove>())
+                  break;
+
+                Region *r = o->getParentRegion();
+                if (!r)
+                  break;
+
+                auto getSequentialIndex = [](Region *r, Operation *o) {
+                  llvm::ReversePostOrderTraversal<Block *> traversal(
+                      &r->getBlocks().front());
+                  size_t idx = 0;
+                  for (Block *b : traversal) {
+                    for (Operation &op : *b) {
+                      if (&op == o)
+                        return idx;
+                      // Only consider operations that are containers as
+                      // possible children
+                      if (!op.getRegions().empty())
+                        idx += 1;
+                    }
+                  }
+                  llvm_unreachable("Operation not part of the region");
+                };
+                size_t sequentialIdx = getSequentialIndex(r, o);
+                components.push_back(("s" + Twine(sequentialIdx)).str());
+
+                Operation *parent = r->getParentOp();
+                if (!parent)
+                  break;
+
+                // If the operation has more than one region, also count in
+                // which of the regions
+                if (parent->getRegions().size() > 1) {
+                  auto getRegionIndex = [](Operation *o, Region *r) {
+                    for (auto [idx, region] :
+                         llvm::enumerate(o->getRegions())) {
+                      if (&region == r)
+                        return idx;
+                    }
+                    llvm_unreachable("Region not child its parent operation");
+                  };
+                  size_t regionIdx = getRegionIndex(parent, r);
+                  components.push_back(("r" + Twine(regionIdx)).str());
+                }
+
+                // next parent
+                o = parent;
+              }
+
+              SmallString<64> Name("canonloop");
+              for (std::string s : reverse(components)) {
+                Name += '_';
+                Name += s;
+              }
+
+              return Name;
+            })
+            .Case([&](UnrollHeuristicOp op) -> std::string {
+              llvm_unreachable("heuristic unrolling does not generate a loop");
+            })
+            .Default([&](Operation *op) {
+              assert(!"TODO: Custom name for this operation");
+              return "transformed";
+            });
+  }
+
+  setNameFn(result, cliName);
+}
+
+LogicalResult NewCliOp::verify() {
+  Value cli = getResult();
+
+  MLIRContext *ctx = cli.getContext();
+  assert(cli.getType() == CanonicalLoopInfoType::get(ctx) &&
+         "Unexpected type of cli");
+
+  // Check that the CLI is used in at most generator and one consumer
+  OpOperand *gen = nullptr;
+  OpOperand *cons = nullptr;
+  for (mlir::OpOperand &use : cli.getUses()) {
+    auto op = cast<mlir::omp::LoopTransformationInterface>(use.getOwner());
+
+    unsigned opnum = use.getOperandNumber();
+    if (op.isGeneratee(opnum)) {
+      if (gen) {
+        InFlightDiagnostic error =
+            emitOpError("CLI must have at most one generator");
+        error.attachNote(gen->getOwner()->getLoc())
+            .append("first generator here:");
+        error.attachNote(use.getOwner()->getLoc())
+            .append("second generator here:");
+        return error;
+      }
+
+      gen = &use;
+    } else if (op.isApplyee(opnum)) {
+      if (cons) {
+        InFlightDiagnostic error =
+            emitOpError("CLI must have at most one consumer");
+        error.attachNote(cons->getOwner()->getLoc())
+            .append("first consumer here:")
+            .appendOp(*cons->getOwner(),
+                      OpPrintingFlags().printGenericOpForm());
+        error.attachNote(use.getOwner()->getLoc())
+            .append("second consumer here:")
+            .appendOp(*use.getOwner(), OpPrintingFlags().printGenericOpForm());
+        return error;
+      }
+
+      cons = &use;
+    } else {
+      llvm_unreachable("Unexpected operand for a CLI");
+    }
+  }
+
+  // If the CLI is source of a transformation, it must have a generator
+  if (cons && !gen) {
+    InFlightDiagnostic error = emitOpError("CLI has no generator");
+    error.attachNote(cons->getOwner()->getLoc())
+        .append("see consumer here: ")
+        .appendOp(*cons->getOwner(), OpPrintingFlags().printGenericOpForm());
+    return error;
+  }
+
+  return success();
+}
+
+void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
+                            Value tripCount) {
+  odsState.addOperands(tripCount);
+  odsState.addOperands(Value());
+  (void)odsState.addRegion();
+}
+
+void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
+                            Value tripCount, ::mlir::Value cli) {
+  odsState.addOperands(tripCount);
+  odsState.addOperands(cli);
+  (void)odsState.addRegion();
+}
+
+void CanonicalLoopOp::getAsmBlockNames(OpAsmSetBlockNameFn setNameFn) {
+  setNameFn(&getRegion().front(), "body_entry");
+}
+
+void CanonicalLoopOp::getAsmBlockArgumentNames(Region &region,
+                                               OpAsmSetValueNameFn setNameFn) {
+  setNameFn(region.getArgument(0), "iv");
+}
+
+void CanonicalLoopOp::print(OpAsmPrinter &p) {
+  if (getCli())
+    p << '(' << getCli() << ')';
+  p << ' ' << getInductionVar() << " : " << getInductionVar().getType()
+    << " in range(" << getTripCount() << ") ";
+
+  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
+                /*printBlockTerminators=*/true);
+
+  p.printOptionalAttrDict((*this)->getAttrs());
+}
+
+mlir::ParseResult CanonicalLoopOp::parse(::mlir::OpAsmParser &parser,
+                                         ::mlir::OperationState &result) {
+  CanonicalLoopInfoType cliType =
+      CanonicalLoopInfoType::get(parser.getContext());
+
+  // Parse (optional) omp.cli identifier
+  OpAsmParser::UnresolvedOperand cli;
+  SmallVector<mlir::Value, 1> cliOperand;
+  if (!parser.parseOptionalLParen()) {
+    if (parser.parseOperand(cli) ||
+        parser.resolveOperand(cli, cliType, cliOperand) || parser.parseRParen())
+      return failure();
+  }
+
+  // We derive the type of tripCount from inductionVariable. MLIR requires the
+  // type of tripCount to be known when calling resolveOperand so we have parse
+  // the type before processing the inductionVariable.
+  OpAsmParser::Argument inductionVariable;
+  OpAsmParser::UnresolvedOperand tripcount;
+  if (parser.parseArgument(inductionVariable, /*allowType*/ true) ||
+      parser.parseKeyword("in") || parser.parseKeyword("range") ||
+      parser.parseLParen() || parser.parseOperand(tripcount) ||
+      parser.parseRParen() ||
+      parser.resolveOperand(tripcount, inductionVariable.type, result.operands))
+    return failure();
+
+  // Parse the loop body.
+  Region *region = result.addRegion();
+  if (parser.parseRegion(*region, {inductionVariable}))
+    return failure();
+
+  // We parsed the cli operand forst, but because it is optional, it must be
+  // last in the operand list.
+  result.operands.append(cliOperand);
+
+  // Parse the optional attribute list.
+  if (parser.parseOptionalAttrDict(result.attributes))
+    return failure();
+
+  return mlir::success();
+}
+
+LogicalResult CanonicalLoopOp::verify() {
+  // The region's entry must accept the induction variable
+  // It can also be empty if just created
+  if (!getRegion().empty()) {
+    Region &region = getRegion();
+    if (region.getNumArguments() != 1)
+      return emitOpError(
+          "Canonical loop region must have exactly one argument");
+
+    if (getInductionVar().getType() != getTripCount().getType())
+      return emitOpError(
+          "Region argument must be the same type as the trip count");
+  }
+
+  return success();
+}
+
+Value CanonicalLoopOp::getInductionVar() { return getRegion().getArgument(0); }
+
+std::pair<unsigned, unsigned>
+CanonicalLoopOp::getApplyeesODSOperandIndexAndLength() {
+  // No applyees
+  return {0, 0};
+}
+
+std::pair<unsigned, unsigned>
+CanonicalLoopOp::getGenerateesODSOperandIndexAndLength() {
+  return getODSOperandIndexAndLength(odsIndex_cli);
+}
+
+//===----------------------------------------------------------------------===//
+// UnrollHeuristicOp
+//===----------------------------------------------------------------------===//
+
+void UnrollHeuristicOp::build(::mlir::OpBuilder &odsBuilder,
+                              ::mlir::OperationState &odsState,
+                              ::mlir::Value cli) {
+  odsState.addOperands(cli);
+}
+
+void UnrollHeuristicOp::print(OpAsmPrinter &p) {
+  p << '(' << getApplyee() << ')';
+
+  p.printOptionalAttrDict((*this)->getAttrs());
+}
+
+mlir::ParseResult UnrollHeuristicOp::parse(::mlir::OpAsmParser &parser,
+                                           ::mlir::OperationState &result) {
+  auto cliType = CanonicalLoopInfoType::get(parser.getContext());
+
+  if (parser.parseLParen())
+    return failure();
+
+  OpAsmParser::UnresolvedOperand applyee;
+  if (parser.parseOperand(applyee) ||
+      parser.resolveOperand(applyee, cliType, result.operands))
+    return failure();
+
+  if (parser.parseRParen())
+    return failure();
+
+  // Optional output loop (full unrolling has none)
+  if (!parser.parseOptionalArrow()) {
+    if (parser.parseLParen() || parser.parseRParen())
+      return failure();
+  }
+
+  // Parse the optional attribute list.
+  if (parser.parseOptionalAttrDict(result.attributes))
+    return failure();
+
+  return mlir::success();
+}
+
+std::pair<unsigned, unsigned>
+UnrollHeuristicOp ::getApplyeesODSOperandIndexAndLength() {
+  return getODSOperandIndexAndLength(odsIndex_applyee);
+}
+
+std::pair<unsigned, unsigned>
+UnrollHeuristicOp::getGenerateesODSOperandIndexAndLength() {
+  return {0, 0};
+}
+
 //===----------------------------------------------------------------------===//
 // Critical construct (2.17.1)
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/OpenMP/cli-canonical_loop-invalid.mlir b/mlir/test/Dialect/OpenMP/cli-canonical_loop-invalid.mlir
new file mode 100644
index 0000000000000..345c53d2890b8
--- /dev/null
+++ b/mlir/test/Dialect/OpenMP/cli-canonical_loop-invalid.mlir
@@ -0,0 +1,50 @@
+// RUN: mlir-opt %s -verify-diagnostics
+
+
+func.func @omp_canonloop_block_arg(%tc : i32) -> () {
+  // expected-error at below {{Canonical loop region must have exactly one argument}}
+  "omp.canonical_loop" (%tc) ({
+    ^bb0(%iv: i32, %somearg: i32):
+      omp.terminator
+  }) : (i32) -> ()
+  return
+}
+
+
+func.func @omp_canonloop_multiple_generators(%tc : i32) -> () {
+  // expected-error at below {{'omp.new_cli' op CLI must have at most one generator}}
+  %cli = omp.new_cli
+  // expected-note at below {{second generator here}}
+  omp.canonical_loop(%cli) %iv1 : i32 in range(%tc) {
+    omp.terminator
+  }
+  // expected-note at below {{first generator here}}
+  omp.canonical_loop(%cli) %iv2 : i32 in range(%tc) {
+    omp.terminator
+  }
+  return
+}
+
+
+func.func @omp_canonloop_multiple_consumers() -> () {
+  // expected-error at below {{'omp.new_cli' op CLI must have at most one consumer}}
+  %cli = omp.new_cli
+  %tc = llvm.mlir.constant(4 : i32) : i32
+  omp.canonical_loop(%cli) %iv1 : i32 in range(%tc) {
+    omp.terminator
+  }
+  // expected-note at below {{second consumer here}}
+  omp.unroll_heuristic(%cli)
+  // expected-note at below {{first consumer here}}
+  omp.unroll_heuristic(%cli)
+  return
+}
+
+
+func.func @omp_canonloop_no_generator() -> () {
+  // expected-error at below {{'omp.new_cli' op CLI has no generator}}
+  %cli = omp.new_cli
+  // expected-note at below {{see consumer here}}
+  omp.unroll_heuristic(%cli)
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/cli-canonical_loop.mlir b/mlir/test/Dialect/OpenMP/cli-canonical_loop.mlir
new file mode 100644
index 0000000000000..adadb8bbac49d
--- /dev/null
+++ b/mlir/test/Dialect/OpenMP/cli-canonical_loop.mlir
@@ -0,0 +1,157 @@
+// RUN: mlir-opt %s | FileCheck %s
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+
+
+// CHECK-LABEL: @omp_canonloop_raw(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_raw(%tc : i32) -> () {
+  // CHECK: omp.canonical_loop %iv : i32 in range(%[[tc]]) {
+  "omp.canonical_loop" (%tc) ({
+    ^bb0(%iv: i32):
+      // CHECK-NEXT: = llvm.add %iv, %iv : i32
+      %newval = llvm.add %iv, %iv : i32
+      // CHECK-NEXT: omp.terminator
+      omp.terminator
+  // CHECK-NEXT: }
+  }) : (i32) -> ()
+  // CHECK-NEXT: return
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_sequential_raw(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_sequential_raw(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %canonloop_s0 = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  "omp.canonical_loop" (%tc, %canonloop_s0) ({
+    ^bb_first(%iv_first: i32):
+      // CHECK-NEXT: = llvm.add %iv, %iv : i32
+      %newval = llvm.add %iv_first, %iv_first : i32
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  // CHECK-NEXT: }
+  }) : (i32, !omp.cli) -> ()
+
+  // CHECK-NEXT: %canonloop_s1 = omp.new_cli
+  %canonloop_s1 = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT: omp.canonical_loop(%canonloop_s1) %iv : i32 in range(%[[tc]]) {
+  "omp.canonical_loop" (%tc, %canonloop_s1) ({
+    ^bb_second(%iv_second: i32):
+    // CHECK: omp.terminator
+    omp.terminator
+  // CHECK-NEXT: }
+  }) : (i32, !omp.cli) -> ()
+
+  // CHECK-NEXT: return
+  return
+}
+
+
+// CHECK-LABEL: @omp_nested_canonloop_raw(
+// CHECK-SAME: %[[tc_outer:.+]]: i32, %[[tc_inner:.+]]: i32)
+func.func @omp_nested_canonloop_raw(%tc_outer : i32, %tc_inner : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %outer = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT: %canonloop_s0_s0 = omp.new_cli
+  %inner = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc_outer]]) {
+  "omp.canonical_loop" (%tc_outer, %outer) ({
+    ^bb_outer(%iv_outer: i32):
+      // CHECK-NEXT: omp.canonical_loop(%canonloop_s0_s0) %iv_0 : i32 in range(%[[tc_inner]]) {
+      "omp.canonical_loop" (%tc_inner, %inner) ({
+        ^bb_inner(%iv_inner: i32):
+          // CHECK-NEXT: = llvm.add %iv, %iv_0 : i32
+          %newval = llvm.add %iv_outer, %iv_inner: i32
+          // CHECK-NEXT: omp.terminator
+          omp.terminator
+      }) : (i32, !omp.cli) -> ()
+      // CHECK: omp.terminator
+      omp.terminator
+  }) : (i32, !omp.cli) -> ()
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: omp.canonical_loop %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop %iv : i32 in range(%tc) {
+    // CHECK-NEXT: llvm.add %iv, %iv : i32
+    %newval = llvm.add %iv, %iv: i32
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_constant_pretty()
+func.func @omp_canonloop_constant_pretty() -> () {
+  // CHECK-NEXT:  %[[tc:.+]] = llvm.mlir.constant(4 : i32) : i32
+  %tc = llvm.mlir.constant(4 : i32) : i32
+  // CHECK-NEXT: omp.canonical_loop %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop %iv : i32 in range(%tc) {
+    // CHECK-NEXT: llvm.add %iv, %iv : i32
+    %newval = llvm.add %iv, %iv: i32
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_sequential_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_sequential_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %canonloop_s0 = omp.new_cli
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%tc) {
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  }
+
+  // CHECK: %canonloop_s1 = omp.new_cli
+  %canonloop_s1 = omp.new_cli
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s1) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%canonloop_s1) %iv_0 : i32 in range(%tc) {
+    // CHECK-NEXT: omp.terminator
+    omp.terminator
+  }
+
+  return
+}
+
+
+// CHECK-LABEL: @omp_canonloop_nested_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32)
+func.func @omp_canonloop_nested_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %canonloop_s0 = omp.new_cli
+  // CHECK-NEXT: %canonloop_s0_s0 = omp.new_cli
+  %canonloop_s0_s0 = omp.new_cli
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%tc) {
+    // CHECK-NEXT: omp.canonical_loop(%canonloop_s0_s0) %iv_0 : i32 in range(%[[tc]]) {
+    omp.canonical_loop(%canonloop_s0_s0) %iv_0 : i32 in range(%tc) {
+      // CHECK: omp.terminator
+      omp.terminator
+    }
+    // CHECK: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+
+// CHECK-LABEL: @omp_newcli_unused(
+// CHECK-SAME: )
+func.func @omp_newcli_unused() -> () {
+  // CHECK-NEXT:  %cli = omp.new_cli
+  %cli = omp.new_cli
+  // CHECK-NEXT: return
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/cli-unroll-heuristic.mlir b/mlir/test/Dialect/OpenMP/cli-unroll-heuristic.mlir
new file mode 100644
index 0000000000000..cda7d0b500166
--- /dev/null
+++ b/mlir/test/Dialect/OpenMP/cli-unroll-heuristic.mlir
@@ -0,0 +1,59 @@
+// RUN: mlir-opt %s            | FileCheck %s
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+
+
+// CHECK-LABEL: @omp_unroll_heuristic_raw(
+// CHECK-SAME: %[[tc:.+]]: i32) {
+func.func @omp_unroll_heuristic_raw(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %canonloop = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  "omp.canonical_loop" (%tc, %canonloop) ({
+    ^bb0(%iv: i32):
+      omp.terminator
+  }) : (i32, !omp.cli) -> ()
+  // CHECK: omp.unroll_heuristic(%canonloop_s0)
+  "omp.unroll_heuristic" (%canonloop) : (!omp.cli) -> ()
+  return
+}
+
+
+// CHECK-LABEL: @omp_unroll_heuristic_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32) {
+func.func @omp_unroll_heuristic_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: %[[CANONLOOP:.+]] = omp.new_cli
+  %canonloop = "omp.new_cli" () : () -> (!omp.cli)
+  // CHECK-NEXT:  omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%canonloop) %iv : i32 in range(%tc) {
+    omp.terminator
+  }
+  // CHECK: omp.unroll_heuristic(%canonloop_s0)
+  omp.unroll_heuristic(%canonloop)
+  return
+}
+
+
+// CHECK-LABEL: @omp_unroll_heuristic_nested_pretty(
+// CHECK-SAME: %[[tc:.+]]: i32) {
+func.func @omp_unroll_heuristic_nested_pretty(%tc : i32) -> () {
+  // CHECK-NEXT: %canonloop_s0 = omp.new_cli
+  %cli_outer = omp.new_cli
+  // CHECK-NEXT: %canonloop_s0_s0 = omp.new_cli
+  %cli_inner = omp.new_cli
+  // CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%[[tc]]) {
+  omp.canonical_loop(%cli_outer) %iv_outer : i32 in range(%tc) {
+    // CHECK-NEXT: omp.canonical_loop(%canonloop_s0_s0) %iv_0 : i32 in range(%[[tc]]) {
+    omp.canonical_loop(%cli_inner) %iv_inner : i32 in range(%tc) {
+      // CHECK: omp.terminator
+      omp.terminator
+    }
+    // CHECK: omp.terminator
+    omp.terminator
+  }
+
+  // CHECK: omp.unroll_heuristic(%canonloop_s0)
+  omp.unroll_heuristic(%cli_outer)
+  // CHECK-NEXT: omp.unroll_heuristic(%canonloop_s0_s0)
+  omp.unroll_heuristic(%cli_inner)
+  return
+}

>From da2613d525deb4edcf0fac41e865ca0510c75210 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 4 Jul 2025 16:26:20 +0200
Subject: [PATCH 4/5] omp.canonical_loop and omp.unroll_heuristic lowering

---
 .../mlir/Target/LLVMIR/ModuleTranslation.h    |  43 +++++
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |  10 +
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  78 ++++++++
 .../LLVMIR/openmp-cli-canonical_loop.mlir     | 175 ++++++++++++++++++
 .../LLVMIR/openmp-cli-unroll-heuristic01.mlir |  56 ++++++
 .../LLVMIR/openmp-cli-unroll-heuristic02.mlir |  93 ++++++++++
 6 files changed, 455 insertions(+)
 create mode 100644 mlir/test/Target/LLVMIR/openmp-cli-canonical_loop.mlir
 create mode 100644 mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic01.mlir
 create mode 100644 mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic02.mlir

diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 79e8bb6add0da..5d52cf3f04b6a 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -15,6 +15,7 @@
 #define MLIR_TARGET_LLVMIR_MODULETRANSLATION_H
 
 #include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
@@ -24,6 +25,7 @@
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
 
 #include "llvm/ADT/SetVector.h"
+#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/FPEnv.h"
 
 namespace llvm {
@@ -108,6 +110,41 @@ class ModuleTranslation {
     return blockMapping.lookup(block);
   }
 
+  /// Find the LLVM-IR loop that represents an MLIR loop.
+  llvm::CanonicalLoopInfo *lookupOMPLoop(omp::NewCliOp mlir) const {
+    llvm::CanonicalLoopInfo *result = loopMapping.lookup(mlir);
+    assert(result && "attempt to get non-existing loop");
+    return result;
+  }
+
+  /// Find the LLVM-IR loop that represents an MLIR loop.
+  llvm::CanonicalLoopInfo *lookupOMPLoop(Value mlir) const {
+    return lookupOMPLoop(mlir.getDefiningOp<omp::NewCliOp>());
+  }
+
+  /// Mark an OpenMP loop as having been consumed.
+  void invalidateOmpLoop(omp::NewCliOp mlir) { loopMapping.erase(mlir); }
+
+  /// Mark an OpenMP loop as having been consumed.
+  void invalidateOmpLoop(Value mlir) {
+    invalidateOmpLoop(mlir.getDefiningOp<omp::NewCliOp>());
+  }
+
+  /// Map an MLIR OpenMP dialect CanonicalLoopInfo to its lowered LLVM-IR
+  /// OpenMPIRBuilder CanonicalLoopInfo
+  void mapOmpLoop(omp::NewCliOp mlir, llvm::CanonicalLoopInfo *llvm) {
+    assert(llvm && "argument must be non-null");
+    llvm::CanonicalLoopInfo *&cur = loopMapping[mlir];
+    assert(cur == nullptr && "attempting to map a loop that is already mapped");
+    cur = llvm;
+  }
+
+  /// Map an MLIR OpenMP dialect CanonicalLoopInfo to its lowered LLVM-IR
+  /// OpenMPIRBuilder CanonicalLoopInfo
+  void mapOmpLoop(Value mlir, llvm::CanonicalLoopInfo *llvm) {
+    mapOmpLoop(mlir.getDefiningOp<omp::NewCliOp>(), llvm);
+  }
+
   /// Stores the mapping between an MLIR operation with successors and a
   /// corresponding LLVM IR instruction.
   void mapBranch(Operation *mlir, llvm::Instruction *llvm) {
@@ -381,6 +418,12 @@ class ModuleTranslation {
   DenseMap<Value, llvm::Value *> valueMapping;
   DenseMap<Block *, llvm::BasicBlock *> blockMapping;
 
+  /// List of not yet consumed MLIR loop handles (represented by an omp.new_cli
+  /// operation which creates a value of type CanonicalLoopInfoType) and their
+  /// LLVM-IR representation as CanonicalLoopInfo which is managed by the
+  /// OpenMPIRBuilder.
+  DenseMap<omp::NewCliOp, llvm::CanonicalLoopInfo *> loopMapping;
+
   /// A mapping between MLIR LLVM dialect terminators and LLVM IR terminators
   /// they are converted to. This allows for connecting PHI nodes to the source
   /// values after all operations are converted.
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 7a0a7f86bc1e9..e77c4a0b94de9 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -42,6 +42,16 @@ template <typename T>
 struct OpenMPOpConversion : public ConvertOpToLLVMPattern<T> {
   using ConvertOpToLLVMPattern<T>::ConvertOpToLLVMPattern;
 
+  OpenMPOpConversion(LLVMTypeConverter &typeConverter,
+                     PatternBenefit benefit = 1)
+      : ConvertOpToLLVMPattern<T>(typeConverter, benefit) {
+    // Operations using CanonicalLoopInfoType are lowered only by
+    // mlir::translateModuleToLLVMIR() using the OpenMPIRBuilder. Until then,
+    // the type and operations using it must be preserved.
+    typeConverter.addConversion(
+        [&](::mlir::omp::CanonicalLoopInfoType type) { return type; });
+  }
+
   LogicalResult
   matchAndRewrite(T op, typename T::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b07243c5f843..6b54d957a30f3 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3090,6 +3090,67 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
   return success();
 }
 
+/// Convert an omp.canonical_loop to LLVM-IR
+static LogicalResult
+convertOmpCanonicalLoopOp(omp::CanonicalLoopOp op, llvm::IRBuilderBase &builder,
+                          LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
+
+  llvm::OpenMPIRBuilder::LocationDescription loopLoc(builder);
+  Value loopIV = op.getInductionVar();
+  Value loopTC = op.getTripCount();
+
+  llvm::Value *llvmTC = moduleTranslation.lookupValue(loopTC);
+
+  llvm::Expected<llvm::CanonicalLoopInfo *> llvmOrError =
+      ompBuilder->createCanonicalLoop(
+          loopLoc,
+          [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *llvmIV) {
+            // Register the mapping of MLIR induction variable to LLVM-IR
+            // induction variable
+            moduleTranslation.mapValue(loopIV, llvmIV);
+
+            builder.restoreIP(ip);
+            llvm::Expected<llvm::BasicBlock *> bodyGenStatus =
+                convertOmpOpRegions(op.getRegion(), "omp.loop.region", builder,
+                                    moduleTranslation);
+
+            return bodyGenStatus.takeError();
+          },
+          llvmTC, "omp.loop");
+  if (!llvmOrError)
+    return op.emitError(llvm::toString(llvmOrError.takeError()));
+
+  llvm::CanonicalLoopInfo *llvmCLI = *llvmOrError;
+  llvm::IRBuilderBase::InsertPoint afterIP = llvmCLI->getAfterIP();
+  builder.restoreIP(afterIP);
+
+  // Register the mapping of MLIR loop to LLVM-IR OpenMPIRBuilder loop
+  if (Value cli = op.getCli())
+    moduleTranslation.mapOmpLoop(cli, llvmCLI);
+
+  return success();
+}
+
+/// Apply a `#pragma omp unroll` / "!$omp unroll" transformation using the
+/// OpenMPIRBuilder.
+static LogicalResult
+applyUnrollHeuristic(omp::UnrollHeuristicOp op, llvm::IRBuilderBase &builder,
+                     LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
+
+  Value applyee = op.getApplyee();
+  assert(applyee && "Loop to apply unrolling on required");
+
+  llvm::CanonicalLoopInfo *consBuilderCLI =
+      moduleTranslation.lookupOMPLoop(applyee);
+  llvm::OpenMPIRBuilder::LocationDescription loc(builder);
+  ompBuilder->unrollLoopHeuristic(loc.DL, consBuilderCLI);
+
+  moduleTranslation.invalidateOmpLoop(applyee);
+  return success();
+}
+
 /// Convert an Atomic Ordering attribute to llvm::AtomicOrdering.
 static llvm::AtomicOrdering
 convertAtomicOrdering(std::optional<omp::ClauseMemoryOrderKind> ao) {
@@ -5961,6 +6022,23 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
                 // etc. and then discarded
                 return success();
               })
+          .Case([&](omp::NewCliOp op) {
+            // Meta-operation: Doesn't do anything by itself, but used to
+            // identify a loop.
+            return success();
+          })
+          .Case([&](omp::CanonicalLoopOp op) {
+            return convertOmpCanonicalLoopOp(op, builder, moduleTranslation);
+          })
+          .Case([&](omp::UnrollHeuristicOp op) {
+            // FIXME: Handling omp.unroll_heuristic as an executable requires
+            // that the generator (e.g. omp.canonical_loop) has been seen first.
+            // For construct that require all codegen to occur inside a callback
+            // (e.g. OpenMPIRBilder::createParallel), all codegen of that
+            // contained region including their transformations must occur at
+            // the omp.canonical_loop.
+            return applyUnrollHeuristic(op, builder, moduleTranslation);
+          })
           .Default([&](Operation *inst) {
             return inst->emitError()
                    << "not yet implemented: " << inst->getName();
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-canonical_loop.mlir b/mlir/test/Target/LLVMIR/openmp-cli-canonical_loop.mlir
new file mode 100644
index 0000000000000..9abef003d6183
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-cli-canonical_loop.mlir
@@ -0,0 +1,175 @@
+// Test lowering of standalone omp.canonical_loop
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK-LABEL: define void @anon_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]],
+// CHECK-SAME:    i32 %[[tc:.+]]) {
+// CHECK-NEXT:    br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.preheader:
+// CHECK-NEXT:    br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.header:
+// CHECK-NEXT:    %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:    br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.cond:
+// CHECK-NEXT:    %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[tc]]
+// CHECK-NEXT:    br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.body:
+// CHECK-NEXT:    br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp.loop.region:
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:    br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp.region.cont:
+// CHECK-NEXT:    br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.inc:
+// CHECK-NEXT:    %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:    br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.exit:
+// CHECK-NEXT:    br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.after:
+// CHECK-NEXT:    ret void
+// CHECK-NEXT:  }
+llvm.func @anon_loop(%ptr: !llvm.ptr, %tc : i32) -> () {
+  omp.canonical_loop %iv : i32 in range(%tc) {
+    %val = llvm.mlir.constant(42.0 : f32) : f32
+    llvm.store %val, %ptr : f32, !llvm.ptr
+    omp.terminator
+  }
+  llvm.return
+}
+
+
+
+// CHECK-LABEL: define void @trivial_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]],
+// CHECK-SAME:    i32 %[[tc:.+]]) {
+// CHECK-NEXT:    br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.preheader:
+// CHECK-NEXT:    br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.header:
+// CHECK-NEXT:    %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:    br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.cond:
+// CHECK-NEXT:    %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[tc]]
+// CHECK-NEXT:    br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.body:
+// CHECK-NEXT:    br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp.loop.region:
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:    br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp.region.cont:
+// CHECK-NEXT:    br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.inc:
+// CHECK-NEXT:    %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:    br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.exit:
+// CHECK-NEXT:    br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT:  omp_omp.loop.after:
+// CHECK-NEXT:    ret void
+// CHECK-NEXT:  }
+llvm.func @trivial_loop(%ptr: !llvm.ptr, %tc : i32) -> () {
+  %cli = omp.new_cli
+  omp.canonical_loop(%cli) %iv : i32 in range(%tc) {
+    %val = llvm.mlir.constant(42.0 : f32) : f32
+    llvm.store %val, %ptr : f32, !llvm.ptr
+    omp.terminator
+  }
+  llvm.return
+}
+
+
+// CHECK-LABEL: define void @nested_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]], i32 %[[outer_tc:.+]], i32 %[[inner_tc:.+]]) {
+// CHECK-NEXT:  br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader:
+// CHECK-NEXT:  br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header:
+// CHECK-NEXT:  %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:  br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond:
+// CHECK-NEXT:  %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[outer_tc]]
+// CHECK-NEXT:  br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body:
+// CHECK-NEXT:  br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region:
+// CHECK-NEXT:  br label %omp_omp.loop.preheader1
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader1:
+// CHECK-NEXT:  br label %omp_omp.loop.header2
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header2:
+// CHECK-NEXT:  %omp_omp.loop.iv8 = phi i32 [ 0, %omp_omp.loop.preheader1 ], [ %omp_omp.loop.next10, %omp_omp.loop.inc5 ]
+// CHECK-NEXT:  br label %omp_omp.loop.cond3
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond3:
+// CHECK-NEXT:  %omp_omp.loop.cmp9 = icmp ult i32 %omp_omp.loop.iv8, %[[inner_tc]]
+// CHECK-NEXT:  br i1 %omp_omp.loop.cmp9, label %omp_omp.loop.body4, label %omp_omp.loop.exit6
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body4:
+// CHECK-NEXT:  br label %omp.loop.region12
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region12:
+// CHECK-NEXT:  store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:  br label %omp.region.cont11
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont11:
+// CHECK-NEXT:  br label %omp_omp.loop.inc5
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc5:
+// CHECK-NEXT:  %omp_omp.loop.next10 = add nuw i32 %omp_omp.loop.iv8, 1
+// CHECK-NEXT:  br label %omp_omp.loop.header2
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit6:
+// CHECK-NEXT:  br label %omp_omp.loop.after7
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after7:
+// CHECK-NEXT:  br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont:
+// CHECK-NEXT:  br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc:
+// CHECK-NEXT:  %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:  br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit:
+// CHECK-NEXT:  br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after:
+// CHECK-NEXT:  ret void
+// CHECK-NEXT: }
+llvm.func @nested_loop(%ptr: !llvm.ptr, %outer_tc : i32, %inner_tc : i32) -> () {
+  %outer_cli = omp.new_cli
+  %inner_cli = omp.new_cli
+  omp.canonical_loop(%outer_cli) %outer_iv : i32 in range(%outer_tc) {
+    omp.canonical_loop(%inner_cli) %inner_iv : i32 in range(%inner_tc) {
+      %val = llvm.mlir.constant(42.0 : f32) : f32
+      llvm.store %val, %ptr : f32, !llvm.ptr
+      omp.terminator
+    }
+    omp.terminator
+  }
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic01.mlir b/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic01.mlir
new file mode 100644
index 0000000000000..0f0448e15f983
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic01.mlir
@@ -0,0 +1,56 @@
+// Test lowering of the omp.unroll_heuristic
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+
+// CHECK-LABEL: define void @unroll_heuristic_trivial_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]], i32 %[[tc:.+]]) {
+// CHECK-NEXT:   br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader:
+// CHECK-NEXT:   br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header:
+// CHECK-NEXT:   %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:   br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond:
+// CHECK-NEXT:   %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[tc]]
+// CHECK-NEXT:   br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body:
+// CHECK-NEXT:   br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region:
+// CHECK-NEXT:   store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:   br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont:
+// CHECK-NEXT:   br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc:
+// CHECK-NEXT:   %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:   br label %omp_omp.loop.header, !llvm.loop ![[$MD1:[0-9]+]]
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit:
+// CHECK-NEXT:   br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after:
+// CHECK-NEXT:   ret void
+// CHECK-NEXT: }
+llvm.func @unroll_heuristic_trivial_loop(%ptr: !llvm.ptr, %tc: i32) -> () {
+  %literal_cli = omp.new_cli
+  omp.canonical_loop(%literal_cli) %iv : i32 in range(%tc) {
+    %val = llvm.mlir.constant(42.0 : f32) : f32
+    llvm.store %val, %ptr : f32, !llvm.ptr
+    omp.terminator
+  }
+  omp.unroll_heuristic(%literal_cli)
+  llvm.return
+}
+
+
+// Start of metadata
+// CHECK-LABEL: !llvm.module.flags
+
+// CHECK: ![[$MD1]] = distinct !{![[$MD1]], ![[$MD2:[0-9]+]]}
+// CHECK: ![[$MD2]] = !{!"llvm.loop.unroll.enable"}
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic02.mlir b/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic02.mlir
new file mode 100644
index 0000000000000..f82b4990e378e
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-cli-unroll-heuristic02.mlir
@@ -0,0 +1,93 @@
+// Test lowering of the omp.unroll_heuristic
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+
+// CHECK-LABEL: define void @unroll_heuristic_nested_loop(
+// CHECK-SAME:    ptr %[[ptr:.+]], i32 %[[outer_tc:.+]], i32 %[[inner_tc:.+]]) {
+// CHECK-NEXT:   br label %omp_omp.loop.preheader
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader:
+// CHECK-NEXT:   br label %omp_omp.loop.header
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header:
+// CHECK-NEXT:   %omp_omp.loop.iv = phi i32 [ 0, %omp_omp.loop.preheader ], [ %omp_omp.loop.next, %omp_omp.loop.inc ]
+// CHECK-NEXT:   br label %omp_omp.loop.cond
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond:
+// CHECK-NEXT:   %omp_omp.loop.cmp = icmp ult i32 %omp_omp.loop.iv, %[[outer_tc]]
+// CHECK-NEXT:   br i1 %omp_omp.loop.cmp, label %omp_omp.loop.body, label %omp_omp.loop.exit
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body:
+// CHECK-NEXT:   br label %omp.loop.region
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region:
+// CHECK-NEXT:   br label %omp_omp.loop.preheader1
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.preheader1:
+// CHECK-NEXT:   br label %omp_omp.loop.header2
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.header2:
+// CHECK-NEXT:   %omp_omp.loop.iv8 = phi i32 [ 0, %omp_omp.loop.preheader1 ], [ %omp_omp.loop.next10, %omp_omp.loop.inc5 ]
+// CHECK-NEXT:   br label %omp_omp.loop.cond3
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.cond3:
+// CHECK-NEXT:   %omp_omp.loop.cmp9 = icmp ult i32 %omp_omp.loop.iv8, %[[inner_tc]]
+// CHECK-NEXT:   br i1 %omp_omp.loop.cmp9, label %omp_omp.loop.body4, label %omp_omp.loop.exit6
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.body4:
+// CHECK-NEXT:   br label %omp.loop.region12
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.loop.region12:
+// CHECK-NEXT:   store float 4.200000e+01, ptr %[[ptr]], align 4
+// CHECK-NEXT:   br label %omp.region.cont11
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont11:
+// CHECK-NEXT:   br label %omp_omp.loop.inc5
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc5:
+// CHECK-NEXT:   %omp_omp.loop.next10 = add nuw i32 %omp_omp.loop.iv8, 1
+// CHECK-NEXT:   br label %omp_omp.loop.header2, !llvm.loop ![[$MD1:[0-9]+]]
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit6:
+// CHECK-NEXT:   br label %omp_omp.loop.after7
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after7:
+// CHECK-NEXT:   br label %omp.region.cont
+// CHECK-EMPTY:
+// CHECK-NEXT: omp.region.cont:
+// CHECK-NEXT:   br label %omp_omp.loop.inc
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.inc:
+// CHECK-NEXT:   %omp_omp.loop.next = add nuw i32 %omp_omp.loop.iv, 1
+// CHECK-NEXT:   br label %omp_omp.loop.header, !llvm.loop ![[$MD3:[0-9]+]]
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.exit:
+// CHECK-NEXT:   br label %omp_omp.loop.after
+// CHECK-EMPTY:
+// CHECK-NEXT: omp_omp.loop.after:
+// CHECK-NEXT:   ret void
+// CHECK-NEXT: }
+llvm.func @unroll_heuristic_nested_loop(%ptr: !llvm.ptr, %outer_tc: i32, %inner_tc: i32) -> () {
+  %outer_cli = omp.new_cli
+  %inner_cli = omp.new_cli
+  omp.canonical_loop(%outer_cli) %outer_iv : i32 in range(%outer_tc) {
+    omp.canonical_loop(%inner_cli) %inner_iv : i32 in range(%inner_tc) {
+      %val = llvm.mlir.constant(42.0 : f32) : f32
+      llvm.store %val, %ptr : f32, !llvm.ptr
+      omp.terminator
+    }
+    omp.terminator
+  }
+  omp.unroll_heuristic(%outer_cli)
+  omp.unroll_heuristic(%inner_cli)
+  llvm.return
+}
+
+
+// Start of metadata
+// CHECK-LABEL: !llvm.module.flags
+
+// CHECK: ![[$MD1]] = distinct !{![[$MD1]], ![[$MD2:[0-9]+]]}
+// CHECK: ![[$MD2]] = !{!"llvm.loop.unroll.enable"}
+// CHECK: ![[$MD3]] = distinct !{![[$MD3]], ![[$MD2]]}
+

>From a12dca3b1d6976ff7c1b9647f4a9650a8ec3018b Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 8 Jul 2025 01:49:13 +0200
Subject: [PATCH 5/5] Emit error on unsupported compositionPost-merge
 fixAddress commentsIntroduce isApplyee/isGeneratee

---
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 46 +++++++++++++++++--
 ...nested-loop-transformation-construct01.f90 |  2 +-
 ...nested-loop-transformation-construct02.f90 |  4 +-
 .../mlir/Dialect/OpenMP/OpenMPDialect.h       |  5 --
 4 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 9d8cab02b2c9e..0f568b3626515 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -64,6 +64,28 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
                                    lower::pft::Evaluation &eval,
                                    mlir::Location loc);
 
+static llvm::omp::Directive
+getOpenMPDirectiveEnum(const parser::OmpLoopDirective &beginStatment) {
+  return beginStatment.v;
+}
+
+static llvm::omp::Directive getOpenMPDirectiveEnum(
+    const parser::OmpBeginLoopDirective &beginLoopDirective) {
+  return getOpenMPDirectiveEnum(
+      std::get<parser::OmpLoopDirective>(beginLoopDirective.t));
+}
+
+static llvm::omp::Directive
+getOpenMPDirectiveEnum(const parser::OpenMPLoopConstruct &ompLoopConstruct) {
+  return getOpenMPDirectiveEnum(
+      std::get<parser::OmpBeginLoopDirective>(ompLoopConstruct.t));
+}
+
+static llvm::omp::Directive getOpenMPDirectiveEnum(
+    const common::Indirection<parser::OpenMPLoopConstruct> &ompLoopConstruct) {
+  return getOpenMPDirectiveEnum(ompLoopConstruct.value());
+}
+
 namespace {
 /// Structure holding information that is needed to pass host-evaluated
 /// information to later lowering stages.
@@ -3489,6 +3511,11 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     newOp = genTeamsOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
                        item);
     break;
+  case llvm::omp::Directive::OMPD_tile: {
+    unsigned version = semaCtx.langOptions().OpenMPVersion;
+    TODO(loc, "Unhandled loop directive (" +
+                  llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+  }
   case llvm::omp::Directive::OMPD_unroll:
     genUnrollOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);
     break;
@@ -3927,12 +3954,25 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
     if (auto *ompNestedLoopCons{
             std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
                 &*optLoopCons)}) {
-      genOMP(converter, symTable, semaCtx, eval, ompNestedLoopCons->value());
+      llvm::omp::Directive nestedDirective =
+          getOpenMPDirectiveEnum(*ompNestedLoopCons);
+      switch (nestedDirective) {
+      case llvm::omp::Directive::OMPD_tile:
+        // Emit the omp.loop_nest with annotation for tiling
+        genOMP(converter, symTable, semaCtx, eval, ompNestedLoopCons->value());
+        break;
+      default: {
+        unsigned version = semaCtx.langOptions().OpenMPVersion;
+        TODO(currentLocation,
+             "Applying a loop-associated on the loop generated by the " +
+                 llvm::omp::getOpenMPDirectiveName(nestedDirective, version) +
+                 " construct");
+      }
+      }
     }
   }
 
-  llvm::omp::Directive directive =
-      std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
+  llvm::omp::Directive directive = getOpenMPDirectiveEnum(beginLoopDirective);
   const parser::CharBlock &source =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
   ConstructQueue queue{
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
index a76e7e52100db..17eba93a7405d 100644
--- a/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
@@ -1,6 +1,6 @@
 ! Test to ensure TODO message is emitted for tile OpenMP 5.1 Directives when they are nested.
 
-!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+!RUN: not %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
 
 subroutine loop_transformation_construct
   implicit none
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
index 33b7c5a917619..cdc628a3b2e64 100644
--- a/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
@@ -1,6 +1,6 @@
 ! Test to ensure TODO message is emitted for unroll OpenMP 5.1 Directives when they are nested.
 
-!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+!RUN: not %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
 
 program loop_transformation_construct
   implicit none
@@ -17,4 +17,4 @@ program loop_transformation_construct
   !$omp end do
 end program loop_transformation_construct
 
-!CHECK: not yet implemented: Unhandled loop directive (unroll)
+!CHECK: not yet implemented: Applying a loop-associated on the loop generated by the unroll construct
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
index 0233f9c19e726..7cf738352ba47 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
@@ -25,11 +25,6 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
 
-namespace mlir::omp {
-/// Find the omp.new_cli, generator, and consumer of a canonical loop info.
-std::tuple<NewCliOp, OpOperand *, OpOperand *> decodeCli(mlir::Value cli);
-} // namespace mlir::omp
-
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsTypes.h.inc"
 


From flang-commits at lists.llvm.org  Mon Jul  7 16:51:10 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 07 Jul 2025 16:51:10 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][core] NFC update builder
	create API usage (PR #147311)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147311@github.com>
Message-ID: <686c5d6e.050a0220.1c1b06.a886@mx.google.com>


https://github.com/makslevental updated https://github.com/llvm/llvm-project/pull/147311

>From 1e57cb84299e022230dc8443254d92083ea4816b Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental at gmail.com>
Date: Sat, 5 Jul 2025 20:26:55 -0400
Subject: [PATCH 1/5] [mlir][tblgen] add concrete create methods

---
 mlir/include/mlir/TableGen/Class.h          |  2 +
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp | 61 +++++++++++++++++++--
 2 files changed, 57 insertions(+), 6 deletions(-)

diff --git a/mlir/include/mlir/TableGen/Class.h b/mlir/include/mlir/TableGen/Class.h
index f750a34a3b2ba..69cefbbc43e0a 100644
--- a/mlir/include/mlir/TableGen/Class.h
+++ b/mlir/include/mlir/TableGen/Class.h
@@ -71,6 +71,8 @@ class MethodParameter {
   StringRef getName() const { return name; }
   /// Returns true if the parameter has a default value.
   bool hasDefaultValue() const { return !defaultValue.empty(); }
+  StringRef getDefaultValue() const { return defaultValue; }
+  bool isOptional() const { return optional; }
 
 private:
   /// The C++ type.
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index cbb4030f3adb4..20c483e52da04 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -230,6 +230,14 @@ static const char *const opCommentHeader = R"(
 
 )";
 
+static const char *const inlineCreateBody = R"(
+  ::mlir::OperationState __state__({0}, getOperationName());
+  build(builder, __state__{1});
+  auto __res__ = ::llvm::dyn_cast<{2}>(builder.create(__state__));
+  assert(__res__ && "builder didn't return the right type");
+  return __res__;
+)";
+
 //===----------------------------------------------------------------------===//
 // Utility structs and functions
 //===----------------------------------------------------------------------===//
@@ -665,6 +673,7 @@ class OpEmitter {
   // Generates the build() method that takes each operand/attribute
   // as a stand-alone parameter.
   void genSeparateArgParamBuilder();
+  void genInlineCreateBody(const SmallVector<MethodParameter> &paramList);
 
   // Generates the build() method that takes each operand/attribute as a
   // stand-alone parameter. The generated build() method uses first operand's
@@ -2568,6 +2577,39 @@ static bool canInferType(const Operator &op) {
   return op.getTrait("::mlir::InferTypeOpInterface::Trait");
 }
 
+void OpEmitter::genInlineCreateBody(
+    const SmallVector<MethodParameter> &paramList) {
+  SmallVector<MethodParameter> createParamList;
+  SmallVector<llvm::StringRef, 4> nonBuilderStateArgsList;
+  createParamList.emplace_back("::mlir::OpBuilder &", "builder");
+  std::string locParamName = "location";
+  while (llvm::find_if(paramList, [&locParamName](const MethodParameter &p) {
+           return p.getName() == locParamName;
+         }) != paramList.end()) {
+    locParamName += "_";
+  }
+  createParamList.emplace_back("::mlir::Location", locParamName);
+
+  for (auto &param : paramList) {
+    if (param.getType() == "::mlir::OpBuilder &" ||
+        param.getType() == "::mlir::OperationState &")
+      continue;
+    createParamList.emplace_back(param.getType(), param.getName(),
+                                 param.getDefaultValue(), param.isOptional());
+    nonBuilderStateArgsList.push_back(param.getName());
+  }
+  auto *c = opClass.addStaticMethod(opClass.getClassName(), "create",
+                                    createParamList);
+  std::string nonBuilderStateArgs = "";
+  if (!nonBuilderStateArgsList.empty()) {
+    llvm::raw_string_ostream nonBuilderStateArgsOS(nonBuilderStateArgs);
+    interleaveComma(nonBuilderStateArgsList, nonBuilderStateArgsOS);
+    nonBuilderStateArgs = ", " + nonBuilderStateArgs;
+  }
+  c->body() << llvm::formatv(inlineCreateBody, locParamName,
+                             nonBuilderStateArgs, opClass.getClassName());
+}
+
 void OpEmitter::genSeparateArgParamBuilder() {
   SmallVector<AttrParamKind, 2> attrBuilderType;
   attrBuilderType.push_back(AttrParamKind::WrappedAttr);
@@ -2584,10 +2626,12 @@ void OpEmitter::genSeparateArgParamBuilder() {
     buildParamList(paramList, inferredAttributes, resultNames, paramKind,
                    attrType);
 
-    auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+    auto *m = opClass.addStaticMethod("void", "build", paramList);
     // If the builder is redundant, skip generating the method.
     if (!m)
       return;
+    genInlineCreateBody(paramList);
+
     auto &body = m->body();
     genCodeForAddingArgAndRegionForBuilder(body, inferredAttributes,
                                            /*isRawValueAttr=*/attrType ==
@@ -2712,10 +2756,11 @@ void OpEmitter::genUseOperandAsResultTypeCollectiveParamBuilder(
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
-  auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+  auto *m = opClass.addStaticMethod("void", "build", paramList);
   // If the builder is redundant, skip generating the method
   if (!m)
     return;
+  genInlineCreateBody(paramList);
   auto &body = m->body();
 
   // Operands
@@ -2826,10 +2871,11 @@ void OpEmitter::genInferredTypeCollectiveParamBuilder(
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
-  auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+  auto *m = opClass.addStaticMethod("void", "build", paramList);
   // If the builder is redundant, skip generating the method
   if (!m)
     return;
+  genInlineCreateBody(paramList);
   auto &body = m->body();
 
   int numResults = op.getNumResults();
@@ -2906,10 +2952,11 @@ void OpEmitter::genUseOperandAsResultTypeSeparateParamBuilder() {
     buildParamList(paramList, inferredAttributes, resultNames,
                    TypeParamKind::None, attrType);
 
-    auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+    auto *m = opClass.addStaticMethod("void", "build", paramList);
     // If the builder is redundant, skip generating the method
     if (!m)
       return;
+    genInlineCreateBody(paramList);
     auto &body = m->body();
     genCodeForAddingArgAndRegionForBuilder(body, inferredAttributes,
                                            /*isRawValueAttr=*/attrType ==
@@ -2948,10 +2995,11 @@ void OpEmitter::genUseAttrAsResultTypeCollectiveParamBuilder(
                                  : "attributes";
   paramList.emplace_back("::llvm::ArrayRef<::mlir::NamedAttribute>",
                          attributesName, "{}");
-  auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+  auto *m = opClass.addStaticMethod("void", "build", paramList);
   // If the builder is redundant, skip generating the method
   if (!m)
     return;
+  genInlineCreateBody(paramList);
 
   auto &body = m->body();
 
@@ -3114,10 +3162,11 @@ void OpEmitter::genCollectiveParamBuilder(CollectiveBuilderKind kind) {
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
-  auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+  auto *m = opClass.addStaticMethod("void", "build", paramList);
   // If the builder is redundant, skip generating the method
   if (!m)
     return;
+  genInlineCreateBody(paramList);
   auto &body = m->body();
 
   // Operands

>From 25c06f35a4779f6f323debe4f77f1a90e804d998 Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 10:35:48 -0400
Subject: [PATCH 2/5] [mlir][core] update builder create API

---
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           | 176 ++---
 .../AffineToStandard/AffineToStandard.cpp     |  26 +-
 .../ArithToAMDGPU/ArithToAMDGPU.cpp           |  90 +--
 .../ArithToArmSME/ArithToArmSME.cpp           |   6 +-
 .../Conversion/ArithToEmitC/ArithToEmitC.cpp  |  44 +-
 .../Conversion/ArithToLLVM/ArithToLLVM.cpp    |  28 +-
 .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp  |  74 +-
 .../ArmNeon2dToIntr/ArmNeon2dToIntr.cpp       |   6 +-
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  | 122 +--
 .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp    |  68 +-
 .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp    |  66 +-
 .../BufferizationToMemRef.cpp                 |  26 +-
 .../ComplexCommon/DivisionConverter.cpp       | 392 ++++-----
 .../ComplexToLLVM/ComplexToLLVM.cpp           |  28 +-
 .../ComplexToLibm/ComplexToLibm.cpp           |   2 +-
 .../ComplexToStandard/ComplexToStandard.cpp   | 598 +++++++-------
 .../ControlFlowToLLVM/ControlFlowToLLVM.cpp   |   8 +-
 .../ControlFlowToSCF/ControlFlowToSCF.cpp     |  22 +-
 .../Conversion/FuncToEmitC/FuncToEmitC.cpp    |   2 +-
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp |  48 +-
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |  82 +-
 .../GPUCommon/GPUToLLVMConversion.cpp         |  78 +-
 .../GPUCommon/IndexIntrinsicsOpLowering.h     |  10 +-
 .../GPUCommon/OpToFuncCallLowering.h          |  12 +-
 .../Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp  |  20 +-
 .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp        |  40 +-
 .../Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp    |  40 +-
 .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp      |  50 +-
 mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp |  54 +-
 .../Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp  |   4 +-
 .../Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp  |   4 +-
 .../Conversion/IndexToLLVM/IndexToLLVM.cpp    |  68 +-
 .../Conversion/IndexToSPIRV/IndexToSPIRV.cpp  |  68 +-
 .../Conversion/LLVMCommon/MemRefBuilder.cpp   |  78 +-
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    |  62 +-
 .../Conversion/LLVMCommon/PrintCallHelper.cpp |   8 +-
 .../Conversion/LLVMCommon/StructBuilder.cpp   |   4 +-
 .../Conversion/LLVMCommon/TypeConverter.cpp   |  14 +-
 .../Conversion/LLVMCommon/VectorPattern.cpp   |   6 +-
 .../LinalgToStandard/LinalgToStandard.cpp     |   4 +-
 mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp   |  64 +-
 .../Conversion/MathToFuncs/MathToFuncs.cpp    | 180 ++---
 mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp |  38 +-
 mlir/lib/Conversion/MathToLibm/MathToLibm.cpp |  14 +-
 .../Conversion/MathToSPIRV/MathToSPIRV.cpp    |  82 +-
 .../MemRefToEmitC/MemRefToEmitC.cpp           |   6 +-
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  | 138 ++--
 .../MemRefToSPIRV/MemRefToSPIRV.cpp           |  18 +-
 mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp   | 204 ++---
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    | 158 ++--
 .../Conversion/OpenACCToSCF/OpenACCToSCF.cpp  |   2 +-
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |   2 +-
 .../PDLToPDLInterp/PDLToPDLInterp.cpp         | 112 +--
 .../SCFToControlFlow/SCFToControlFlow.cpp     |  36 +-
 mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp |  14 +-
 mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp     |  40 +-
 .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp    |  40 +-
 mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp |  40 +-
 .../ConvertLaunchFuncToLLVMCalls.cpp          |   8 +-
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    | 124 +--
 .../ConvertShapeConstraints.cpp               |   2 +-
 .../ShapeToStandard/ShapeToStandard.cpp       | 150 ++--
 .../TensorToSPIRV/TensorToSPIRV.cpp           |   6 +-
 .../Conversion/TosaToArith/TosaToArith.cpp    | 100 +--
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  | 742 +++++++++---------
 .../TosaToLinalg/TosaToLinalgNamed.cpp        | 218 ++---
 .../TosaToMLProgram/TosaToMLProgram.cpp       |   6 +-
 mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp   |  24 +-
 .../Conversion/TosaToTensor/TosaToTensor.cpp  |  20 +-
 .../VectorToArmSME/VectorToArmSME.cpp         |  56 +-
 .../Conversion/VectorToGPU/VectorToGPU.cpp    |  76 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      | 158 ++--
 .../Conversion/VectorToSCF/VectorToSCF.cpp    | 164 ++--
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  48 +-
 .../VectorToXeGPU/VectorToXeGPU.cpp           |  20 +-
 .../AMDGPU/Transforms/EmulateAtomics.cpp      |  20 +-
 .../AMDGPU/Transforms/MaskedloadToLoad.cpp    |  28 +-
 .../Transforms/ResolveStridedMetadata.cpp     |   6 +-
 mlir/lib/Dialect/AMX/IR/AMXDialect.cpp        |   8 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  34 +-
 .../Transforms/AffineDataCopyGeneration.cpp   |   2 +-
 .../Transforms/AffineExpandIndexOps.cpp       |  20 +-
 .../Affine/Transforms/DecomposeAffineOps.cpp  |   4 +-
 .../Dialect/Affine/Transforms/LoopFusion.cpp  |   2 +-
 .../Transforms/PipelineDataTransfer.cpp       |   6 +-
 .../Affine/Transforms/ReifyValueBounds.cpp    |   6 +-
 .../Affine/Transforms/SuperVectorize.cpp      |  24 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |  86 +-
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |  88 +--
 mlir/lib/Dialect/Arith/IR/ArithDialect.cpp    |   2 +-
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |  40 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |   2 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/EmulateUnsupportedFloats.cpp   |   4 +-
 .../Arith/Transforms/EmulateWideInt.cpp       | 176 ++---
 .../Dialect/Arith/Transforms/ExpandOps.cpp    | 228 +++---
 .../Transforms/IntRangeOptimizations.cpp      |  10 +-
 .../Arith/Transforms/ReifyValueBounds.cpp     |  16 +-
 .../Transforms/ShardingInterfaceImpl.cpp      |   2 +-
 mlir/lib/Dialect/Arith/Utils/Utils.cpp        |  90 +--
 .../LowerContractionToNeonI8MMPattern.cpp     |   8 +-
 mlir/lib/Dialect/ArmSME/IR/Utils.cpp          |  14 +-
 .../ArmSME/Transforms/OuterProductFusion.cpp  |   8 +-
 .../ArmSME/Transforms/TileAllocation.cpp      |   4 +-
 .../ArmSME/Transforms/VectorLegalization.cpp  | 110 +--
 .../Transforms/LegalizeForLLVMExport.cpp      |  16 +-
 .../Transforms/LegalizeVectorStorage.cpp      |  12 +-
 .../LowerContractionToSVEI8MMPattern.cpp      |  60 +-
 mlir/lib/Dialect/Async/IR/Async.cpp           |   2 +-
 .../Async/Transforms/AsyncParallelFor.cpp     | 194 ++---
 .../Transforms/AsyncRuntimeRefCounting.cpp    |  14 +-
 .../Async/Transforms/AsyncToAsyncRuntime.cpp  |  78 +-
 .../IR/BufferDeallocationOpInterface.cpp      |  10 +-
 .../IR/BufferizableOpInterface.cpp            |  10 +-
 .../Bufferization/IR/BufferizationOps.cpp     |  30 +-
 .../BufferizationTransformOps.cpp             |   4 +-
 .../BufferDeallocationSimplification.cpp      |  12 +-
 .../Transforms/BufferResultsToOutParams.cpp   |   6 +-
 .../Bufferization/Transforms/BufferUtils.cpp  |   2 +-
 .../Bufferization/Transforms/Bufferize.cpp    |   6 +-
 .../DropEquivalentBufferResults.cpp           |   4 +-
 .../Transforms/EmptyTensorElimination.cpp     |   2 +-
 .../FuncBufferizableOpInterfaceImpl.cpp       |   4 +-
 .../Transforms/LowerDeallocations.cpp         | 142 ++--
 .../OwnershipBasedBufferDeallocation.cpp      |  14 +-
 .../lib/Dialect/Complex/IR/ComplexDialect.cpp |   2 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.cpp |   6 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |  12 +-
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp           |   4 +-
 .../Dialect/EmitC/Transforms/Transforms.cpp   |   4 +-
 .../EmitC/Transforms/TypeConversions.cpp      |   2 +-
 .../EmitC/Transforms/WrapFuncInClass.cpp      |   8 +-
 .../Func/Extensions/InlinerExtension.cpp      |   2 +-
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |   2 +-
 .../Func/TransformOps/FuncTransformOps.cpp    |   2 +-
 .../Func/Transforms/FuncConversions.cpp       |   2 +-
 mlir/lib/Dialect/Func/Utils/Utils.cpp         |   6 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |  10 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |  10 +-
 mlir/lib/Dialect/GPU/TransformOps/Utils.cpp   |  24 +-
 .../GPU/Transforms/AllReduceLowering.cpp      |   2 +-
 .../GPU/Transforms/AsyncRegionRewriter.cpp    |   8 +-
 .../GPU/Transforms/DecomposeMemRefs.cpp       |   4 +-
 .../GPU/Transforms/GlobalIdRewriter.cpp       |   8 +-
 .../GPU/Transforms/KernelOutlining.cpp        |  10 +-
 .../GPU/Transforms/MemoryPromotion.cpp        |  16 +-
 .../Dialect/GPU/Transforms/ModuleToBinary.cpp |   2 +-
 .../GPU/Transforms/PromoteShuffleToAMDGPU.cpp |   4 +-
 .../GPU/Transforms/ShuffleRewriter.cpp        |  22 +-
 .../GPU/Transforms/SubgroupIdRewriter.cpp     |  22 +-
 .../GPU/Transforms/SubgroupReduceLowering.cpp |  70 +-
 .../Dialect/GPU/Utils/DistributionUtils.cpp   |   4 +-
 mlir/lib/Dialect/Index/IR/IndexOps.cpp        |   8 +-
 .../LLVMIR/IR/BasicPtxBuilderInterface.cpp    |   4 +-
 .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp   |   2 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |  16 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp |  46 +-
 .../Dialect/LLVMIR/Transforms/AddComdats.cpp  |   4 +-
 .../Transforms/InlinerInterfaceImpl.cpp       |  22 +-
 .../LLVMIR/Transforms/LegalizeForExport.cpp   |   2 +-
 .../LLVMIR/Transforms/OptimizeForNVVM.cpp     |  30 +-
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 202 ++---
 .../TransformOps/LinalgTransformOps.cpp       |  14 +-
 .../BufferizableOpInterfaceImpl.cpp           |   2 +-
 .../Transforms/ConvertConv2DToImg2Col.cpp     | 108 +--
 .../Transforms/ConvertToDestinationStyle.cpp  |  52 +-
 .../Transforms/DataLayoutPropagation.cpp      |  30 +-
 ...DecomposeGenericByUnfoldingPermutation.cpp |  10 +-
 .../Linalg/Transforms/DecomposeLinalgOps.cpp  |  14 +-
 .../Dialect/Linalg/Transforms/Detensorize.cpp |   6 +-
 .../Linalg/Transforms/DropUnitDims.cpp        |  22 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  40 +-
 .../Linalg/Transforms/ElementwiseToLinalg.cpp |   4 +-
 .../EraseUnusedOperandsAndResults.cpp         |   2 +-
 .../FusePadOpWithLinalgProducer.cpp           |   8 +-
 mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp |   2 +-
 .../Linalg/Transforms/Generalization.cpp      |   2 +-
 .../Linalg/Transforms/HoistPadding.cpp        |  20 +-
 .../Dialect/Linalg/Transforms/Hoisting.cpp    |   2 +-
 .../Transforms/InlineScalarOperands.cpp       |   6 +-
 .../Dialect/Linalg/Transforms/Interchange.cpp |   2 +-
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |   8 +-
 .../Transforms/MeshShardingInterfaceImpl.cpp  |  18 +-
 .../Linalg/Transforms/NamedOpConversions.cpp  |   8 +-
 .../Transforms/PackAndUnpackPatterns.cpp      |   6 +-
 .../Linalg/Transforms/PadTilingInterface.cpp  |   6 +-
 .../lib/Dialect/Linalg/Transforms/Padding.cpp |   6 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |  44 +-
 .../Transforms/RuntimeOpVerification.cpp      |   4 +-
 mlir/lib/Dialect/Linalg/Transforms/Split.cpp  |   2 +-
 .../Linalg/Transforms/SplitReduction.cpp      |  32 +-
 .../SwapExtractSliceWithFillPatterns.cpp      |   2 +-
 mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp |  16 +-
 .../Linalg/Transforms/TilingInterfaceImpl.cpp |  48 +-
 .../Dialect/Linalg/Transforms/Transforms.cpp  |  66 +-
 .../Linalg/Transforms/TransposeConv2D.cpp     |   6 +-
 .../Linalg/Transforms/TransposeMatmul.cpp     |  26 +-
 .../Linalg/Transforms/Vectorization.cpp       | 176 ++---
 .../Linalg/Transforms/WinogradConv2D.cpp      | 114 +--
 mlir/lib/Dialect/Linalg/Utils/Utils.cpp       |  18 +-
 .../BufferizableOpInterfaceImpl.cpp           |   2 +-
 mlir/lib/Dialect/MPI/IR/MPIOps.cpp            |   2 +-
 mlir/lib/Dialect/Math/IR/MathOps.cpp          |   2 +-
 .../Transforms/AlgebraicSimplification.cpp    |  20 +-
 .../Math/Transforms/ExpandPatterns.cpp        | 226 +++---
 .../Transforms/ExtendToSupportedTypes.cpp     |   4 +-
 .../Transforms/PolynomialApproximation.cpp    | 536 ++++++-------
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    |   6 +-
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |  22 +-
 .../TransformOps/MemRefTransformOps.cpp       |   2 +-
 .../Transforms/AllocationOpInterfaceImpl.cpp  |   8 +-
 .../MemRef/Transforms/ComposeSubView.cpp      |   2 +-
 .../MemRef/Transforms/EmulateNarrowType.cpp   |  36 +-
 .../Dialect/MemRef/Transforms/ExpandOps.cpp   |  14 +-
 .../MemRef/Transforms/ExpandRealloc.cpp       |  20 +-
 .../Transforms/ExpandStridedMetadata.cpp      |  24 +-
 .../Transforms/ExtractAddressComputations.cpp |  16 +-
 .../MemRef/Transforms/FlattenMemRefs.cpp      |  28 +-
 .../MemRef/Transforms/FoldMemRefAliasOps.cpp  |   6 +-
 .../Transforms/IndependenceTransforms.cpp     |   8 +-
 .../Dialect/MemRef/Transforms/MultiBuffer.cpp |   8 +-
 .../MemRef/Transforms/NormalizeMemRefs.cpp    |   2 +-
 .../MemRef/Transforms/ReifyResultShapes.cpp   |   4 +-
 .../ResolveShapedTypeResultDims.cpp           |   2 +-
 .../Transforms/RuntimeOpVerification.cpp      |  88 +--
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |  14 +-
 .../Mesh/Transforms/Simplifications.cpp       |   4 +-
 .../Dialect/Mesh/Transforms/Spmdization.cpp   |  14 +-
 .../Dialect/Mesh/Transforms/Transforms.cpp    |  36 +-
 .../NVGPU/TransformOps/NVGPUTransformOps.cpp  |  62 +-
 .../NVGPU/Transforms/CreateAsyncGroups.cpp    |  16 +-
 .../NVGPU/Transforms/OptimizeSharedMemory.cpp |  12 +-
 .../Quant/Transforms/LowerQuantOps.cpp        |  78 +-
 .../Quant/Transforms/StripFuncQuantTypes.cpp  |   2 +-
 mlir/lib/Dialect/SCF/IR/SCF.cpp               |  78 +-
 .../SCF/TransformOps/SCFTransformOps.cpp      |   4 +-
 .../BufferizableOpInterfaceImpl.cpp           |  18 +-
 .../lib/Dialect/SCF/Transforms/ForToWhile.cpp |   8 +-
 .../SCF/Transforms/ForallToParallel.cpp       |   2 +-
 .../Dialect/SCF/Transforms/LoopPipelining.cpp |  70 +-
 .../SCF/Transforms/LoopSpecialization.cpp     |  14 +-
 .../SCF/Transforms/ParallelLoopFusion.cpp     |   4 +-
 .../SCF/Transforms/ParallelLoopTiling.cpp     |  32 +-
 .../Transforms/StructuralTypeConversions.cpp  |   6 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  32 +-
 .../SCF/Transforms/UpliftWhileToFor.cpp       |  20 +-
 .../SCF/Transforms/WrapInZeroTripCheck.cpp    |  10 +-
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          |  94 +--
 mlir/lib/Dialect/SMT/IR/SMTDialect.cpp        |   4 +-
 mlir/lib/Dialect/SMT/IR/SMTOps.cpp            |   4 +-
 mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp  |  10 +-
 .../SPIRV/IR/SPIRVCanonicalization.cpp        |  20 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp    |   4 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp        |  22 +-
 .../Linking/ModuleCombiner/ModuleCombiner.cpp |   2 +-
 .../Transforms/LowerABIAttributesPass.cpp     |  18 +-
 .../SPIRV/Transforms/RewriteInsertsPass.cpp   |   2 +-
 .../SPIRV/Transforms/SPIRVConversion.cpp      |  62 +-
 .../Transforms/SPIRVWebGPUTransforms.cpp      |  32 +-
 .../Transforms/UnifyAliasedResourcePass.cpp   |  28 +-
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  30 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/OutlineShapeComputation.cpp    |   6 +-
 .../Shape/Transforms/ShapeToShapeLowering.cpp |   6 +-
 .../SparseTensor/IR/SparseTensorDialect.cpp   |   4 +-
 .../IR/SparseTensorInterfaces.cpp             |   2 +-
 .../Transforms/SparseAssembler.cpp            |  16 +-
 .../Transforms/SparseBufferRewriting.cpp      | 366 ++++-----
 .../Transforms/SparseGPUCodegen.cpp           | 178 ++---
 .../Transforms/SparseIterationToScf.cpp       |  26 +-
 .../Transforms/SparseReinterpretMap.cpp       |  20 +-
 .../Transforms/SparseSpaceCollapse.cpp        |   2 +-
 .../SparseStorageSpecifierToLLVM.cpp          |  10 +-
 .../Transforms/SparseTensorCodegen.cpp        | 138 ++--
 .../Transforms/SparseTensorConversion.cpp     |  44 +-
 .../Transforms/SparseTensorRewriting.cpp      | 212 ++---
 .../Transforms/SparseVectorization.cpp        |  50 +-
 .../Transforms/Sparsification.cpp             |  68 +-
 .../Transforms/StageSparseOperations.cpp      |   4 +-
 .../Transforms/Utils/CodegenUtils.cpp         |  74 +-
 .../Transforms/Utils/CodegenUtils.h           |  30 +-
 .../Transforms/Utils/LoopEmitter.cpp          |  54 +-
 .../Utils/SparseTensorDescriptor.cpp          |  10 +-
 .../Transforms/Utils/SparseTensorDescriptor.h |   2 +-
 .../Transforms/Utils/SparseTensorIterator.cpp |  76 +-
 .../lib/Dialect/SparseTensor/Utils/Merger.cpp | 116 +--
 .../Extensions/MeshShardingExtensions.cpp     |   8 +-
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      |  66 +-
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp   |  20 +-
 .../TransformOps/TensorTransformOps.cpp       |   4 +-
 .../BufferizableOpInterfaceImpl.cpp           |  44 +-
 .../Tensor/Transforms/EmptyOpPatterns.cpp     |   2 +-
 .../ExtractSliceFromReshapeUtils.cpp          |   8 +-
 .../Tensor/Transforms/FoldTensorSubsetOps.cpp |   2 +-
 .../Transforms/IndependenceTransforms.cpp     |   8 +-
 .../Tensor/Transforms/ReshapePatterns.cpp     |  10 +-
 .../Tensor/Transforms/RewriteAsConstant.cpp   |   2 +-
 .../Transforms/RuntimeOpVerification.cpp      |  40 +-
 .../SubsetInsertionOpInterfaceImpl.cpp        |   2 +-
 mlir/lib/Dialect/Tensor/Utils/Utils.cpp       |   6 +-
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |   4 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |  12 +-
 .../Transforms/TosaDecomposeDepthwise.cpp     |  16 +-
 .../Transforms/TosaDecomposeTransposeConv.cpp |   8 +-
 .../Tosa/Transforms/TosaInferShapes.cpp       |   2 +-
 .../Tosa/Transforms/TosaReduceTransposes.cpp  |   4 +-
 .../Tosa/Transforms/TosaTypeConverters.cpp    |   4 +-
 .../Dialect/Tosa/Utils/ConversionUtils.cpp    |  16 +-
 mlir/lib/Dialect/UB/IR/UBOps.cpp              |   2 +-
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  60 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Vector/Transforms/LowerVectorBitCast.cpp  |   8 +-
 .../Transforms/LowerVectorBroadcast.cpp       |  22 +-
 .../Vector/Transforms/LowerVectorContract.cpp | 128 +--
 .../Vector/Transforms/LowerVectorGather.cpp   |  36 +-
 .../Transforms/LowerVectorInterleave.cpp      |  20 +-
 .../Vector/Transforms/LowerVectorMask.cpp     |  22 +-
 .../Transforms/LowerVectorMultiReduction.cpp  |  38 +-
 .../Vector/Transforms/LowerVectorScan.cpp     |  16 +-
 .../Transforms/LowerVectorShapeCast.cpp       |  36 +-
 .../Vector/Transforms/LowerVectorTransfer.cpp |  26 +-
 .../Transforms/LowerVectorTranspose.cpp       |  26 +-
 .../Vector/Transforms/VectorDistribute.cpp    |  76 +-
 .../Transforms/VectorDropLeadUnitDim.cpp      |  34 +-
 .../VectorEmulateMaskedLoadStore.cpp          |  28 +-
 .../Transforms/VectorEmulateNarrowType.cpp    | 176 ++---
 ...sertExtractStridedSliceRewritePatterns.cpp |  30 +-
 .../Vector/Transforms/VectorLinearize.cpp     |   6 +-
 .../Transforms/VectorMaskElimination.cpp      |   2 +-
 .../Transforms/VectorTransferOpTransforms.cpp |  28 +-
 .../VectorTransferSplitRewritePatterns.cpp    |  72 +-
 .../Vector/Transforms/VectorTransforms.cpp    |  82 +-
 .../Vector/Transforms/VectorUnroll.cpp        |  32 +-
 mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp |   6 +-
 .../Dialect/X86Vector/IR/X86VectorDialect.cpp |   6 +-
 .../X86Vector/Transforms/AVXTranspose.cpp     |  26 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   4 +-
 .../Transforms/XeGPUSubgroupDistribute.cpp    |  28 +-
 .../Dialect/XeGPU/Transforms/XeGPUUnroll.cpp  |  38 +-
 .../Transforms/XeGPUWgToSgDistribute.cpp      |  22 +-
 mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp   |  12 +-
 mlir/lib/IR/BuiltinDialect.cpp                |   2 +-
 mlir/lib/Query/Query.cpp                      |   2 +-
 .../lib/Target/LLVMIR/LLVMImportInterface.cpp |   2 +-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |  76 +-
 .../SPIRV/Deserialization/DeserializeOps.cpp  |  20 +-
 .../SPIRV/Deserialization/Deserializer.cpp    |  36 +-
 mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp       |  44 +-
 .../Transforms/Utils/DialectConversion.cpp    |   2 +-
 349 files changed, 6988 insertions(+), 6988 deletions(-)

diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 910fe1b1d93c1..34e859bc9c86e 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -50,20 +50,20 @@ static Value convertUnsignedToI32(ConversionPatternRewriter &rewriter,
   if (i32 == valTy)
     return val;
   return valTy.getWidth() > 32
-             ? Value(rewriter.create<LLVM::TruncOp>(loc, i32, val))
-             : Value(rewriter.create<LLVM::ZExtOp>(loc, i32, val));
+             ? Value(LLVM::TruncOp::create(rewriter, loc, i32, val))
+             : Value(LLVM::ZExtOp::create(rewriter, loc, i32, val));
 }
 
 static Value createI32Constant(ConversionPatternRewriter &rewriter,
                                Location loc, int32_t value) {
   Type i32 = rewriter.getI32Type();
-  return rewriter.create<LLVM::ConstantOp>(loc, i32, value);
+  return LLVM::ConstantOp::create(rewriter, loc, i32, value);
 }
 
 static Value createI1Constant(ConversionPatternRewriter &rewriter, Location loc,
                               bool value) {
   Type llvmI1 = rewriter.getI1Type();
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmI1, value);
+  return LLVM::ConstantOp::create(rewriter, loc, llvmI1, value);
 }
 
 /// Returns the linear index used to access an element in the memref.
@@ -78,11 +78,11 @@ static Value getLinearIndexI32(ConversionPatternRewriter &rewriter,
           ShapedType::isDynamic(stride)
               ? convertUnsignedToI32(rewriter, loc,
                                      memRefDescriptor.stride(rewriter, loc, i))
-              : rewriter.create<LLVM::ConstantOp>(loc, i32, stride);
-      increment = rewriter.create<LLVM::MulOp>(loc, increment, strideValue);
+              : LLVM::ConstantOp::create(rewriter, loc, i32, stride);
+      increment = LLVM::MulOp::create(rewriter, loc, increment, strideValue);
     }
     index =
-        index ? rewriter.create<LLVM::AddOp>(loc, index, increment) : increment;
+        index ? LLVM::AddOp::create(rewriter, loc, index, increment) : increment;
   }
   return index ? index : createI32Constant(rewriter, loc, 0);
 }
@@ -110,14 +110,14 @@ static Value getNumRecords(ConversionPatternRewriter &rewriter, Location loc,
   for (uint32_t i = 0, e = memrefType.getRank(); i < e; ++i) {
     Value size = memrefDescriptor.size(rewriter, loc, i);
     Value stride = memrefDescriptor.stride(rewriter, loc, i);
-    Value maxThisDim = rewriter.create<LLVM::MulOp>(loc, size, stride);
+    Value maxThisDim = LLVM::MulOp::create(rewriter, loc, size, stride);
     maxIndex = maxIndex
-                   ? rewriter.create<LLVM::UMaxOp>(loc, maxIndex, maxThisDim)
+                   ? LLVM::UMaxOp::create(rewriter, loc, maxIndex, maxThisDim)
                    : maxThisDim;
   }
   Value maxIndexI32 = convertUnsignedToI32(rewriter, loc, maxIndex);
   Value byteWidthConst = createI32Constant(rewriter, loc, elementByteWidth);
-  return rewriter.create<LLVM::MulOp>(loc, maxIndexI32, byteWidthConst);
+  return LLVM::MulOp::create(rewriter, loc, maxIndexI32, byteWidthConst);
 }
 
 static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
@@ -132,13 +132,13 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
   Value stride;
   if (chipset.majorVersion == 9 && chipset >= kGfx942 && cacheSwizzleStride) {
     Value cacheStrideZext =
-        rewriter.create<LLVM::ZExtOp>(loc, i16, cacheSwizzleStride);
-    Value swizzleBit = rewriter.create<LLVM::ConstantOp>(
+        LLVM::ZExtOp::create(rewriter, loc, i16, cacheSwizzleStride);
+    Value swizzleBit = LLVM::ConstantOp::create(rewriter,
         loc, i16, rewriter.getI16IntegerAttr(1 << 14));
-    stride = rewriter.create<LLVM::OrOp>(loc, cacheStrideZext, swizzleBit,
+    stride = LLVM::OrOp::create(rewriter, loc, cacheStrideZext, swizzleBit,
                                          /*isDisjoint=*/true);
   } else {
-    stride = rewriter.create<LLVM::ConstantOp>(loc, i16,
+    stride = LLVM::ConstantOp::create(rewriter, loc, i16,
                                                rewriter.getI16IntegerAttr(0));
   }
   // Get the number of elements.
@@ -209,18 +209,18 @@ struct FatRawBufferCastLowering
             : descriptor.alignedPtr(rewriter, loc);
 
     Value offset = adaptor.getResetOffset()
-                       ? rewriter.create<LLVM::ConstantOp>(
+                       ? LLVM::ConstantOp::create(rewriter,
                              loc, getIndexType(), rewriter.getIndexAttr(0))
                        : descriptor.offset(rewriter, loc);
 
     bool hasSizes = memrefType.getRank() > 0;
     // No need to unpack() and pack() all the individual sizes and strides,
     // so we'll just extract the arrays.
-    Value sizes = hasSizes ? rewriter.create<LLVM::ExtractValueOp>(
+    Value sizes = hasSizes ? LLVM::ExtractValueOp::create(rewriter,
                                  loc, descriptor, kSizePosInMemRefDescriptor)
                            : Value{};
     Value strides = hasSizes
-                        ? rewriter.create<LLVM::ExtractValueOp>(
+                        ? LLVM::ExtractValueOp::create(rewriter,
                               loc, descriptor, kStridePosInMemRefDescriptor)
                         : Value{};
 
@@ -231,16 +231,16 @@ struct FatRawBufferCastLowering
     Value result = MemRefDescriptor::poison(
         rewriter, loc,
         getTypeConverter()->convertType(op.getResult().getType()));
-    result = rewriter.create<LLVM::InsertValueOp>(
+    result = LLVM::InsertValueOp::create(rewriter,
         loc, result, fatPtr, kAllocatedPtrPosInMemRefDescriptor);
-    result = rewriter.create<LLVM::InsertValueOp>(
+    result = LLVM::InsertValueOp::create(rewriter,
         loc, result, fatPtr, kAlignedPtrPosInMemRefDescriptor);
-    result = rewriter.create<LLVM::InsertValueOp>(loc, result, offset,
+    result = LLVM::InsertValueOp::create(rewriter, loc, result, offset,
                                                   kOffsetPosInMemRefDescriptor);
     if (hasSizes) {
-      result = rewriter.create<LLVM::InsertValueOp>(loc, result, sizes,
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, sizes,
                                                     kSizePosInMemRefDescriptor);
-      result = rewriter.create<LLVM::InsertValueOp>(
+      result = LLVM::InsertValueOp::create(rewriter,
           loc, result, strides, kStridePosInMemRefDescriptor);
     }
     rewriter.replaceOp(op, result);
@@ -343,7 +343,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     if (storeData) {
       if (llvmBufferValType != llvmWantedDataType) {
         Value castForStore =
-            rewriter.create<LLVM::BitcastOp>(loc, llvmBufferValType, storeData);
+            LLVM::BitcastOp::create(rewriter, loc, llvmBufferValType, storeData);
         args.push_back(castForStore);
       } else {
         args.push_back(storeData);
@@ -352,7 +352,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     if (atomicCmpData) {
       if (llvmBufferValType != llvmWantedDataType) {
-        Value castForCmp = rewriter.create<LLVM::BitcastOp>(
+        Value castForCmp = LLVM::BitcastOp::create(rewriter,
             loc, llvmBufferValType, atomicCmpData);
         args.push_back(castForCmp);
       } else {
@@ -383,17 +383,17 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
         indexOffset && *indexOffset > 0) {
       Value extraOffsetConst = createI32Constant(rewriter, loc, *indexOffset);
       voffset =
-          voffset ? rewriter.create<LLVM::AddOp>(loc, voffset, extraOffsetConst)
+          voffset ? LLVM::AddOp::create(rewriter, loc, voffset, extraOffsetConst)
                   : extraOffsetConst;
     }
-    voffset = rewriter.create<LLVM::MulOp>(loc, voffset, byteWidthConst);
+    voffset = LLVM::MulOp::create(rewriter, loc, voffset, byteWidthConst);
     args.push_back(voffset);
 
     // SGPR offset.
     Value sgprOffset = adaptor.getSgprOffset();
     if (!sgprOffset)
       sgprOffset = createI32Constant(rewriter, loc, 0);
-    sgprOffset = rewriter.create<LLVM::MulOp>(loc, sgprOffset, byteWidthConst);
+    sgprOffset = LLVM::MulOp::create(rewriter, loc, sgprOffset, byteWidthConst);
     args.push_back(sgprOffset);
 
     // bit 0: GLC = 0 (atomics drop value, less coherency)
@@ -403,12 +403,12 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
                                            llvmBufferValType);
-    Operation *lowered = rewriter.create<Intrinsic>(loc, resultTypes, args,
+    Operation *lowered = Intrinsic::create(rewriter, loc, resultTypes, args,
                                                     ArrayRef<NamedAttribute>());
     if (lowered->getNumResults() == 1) {
       Value replacement = lowered->getResult(0);
       if (llvmBufferValType != llvmWantedDataType) {
-        replacement = rewriter.create<LLVM::BitcastOp>(loc, llvmWantedDataType,
+        replacement = LLVM::BitcastOp::create(rewriter, loc, llvmWantedDataType,
                                                        replacement);
       }
       rewriter.replaceOp(gpuOp, replacement);
@@ -465,12 +465,12 @@ struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
                << chipset.majorVersion;
 
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::SWaitcntOp>(loc, ldsOnlyBits);
+      ROCDL::SWaitcntOp::create(rewriter, loc, ldsOnlyBits);
       rewriter.replaceOpWithNewOp<ROCDL::SBarrierOp>(op);
     } else {
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::WaitDscntOp>(loc, 0);
-      rewriter.create<ROCDL::BarrierSignalOp>(loc, -1);
+      ROCDL::WaitDscntOp::create(rewriter, loc, 0);
+      ROCDL::BarrierSignalOp::create(rewriter, loc, -1);
       rewriter.replaceOpWithNewOp<ROCDL::BarrierWaitOp>(op, -1);
     }
 
@@ -516,18 +516,18 @@ static Value convertMFMAVectorOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   if (auto vectorType = dyn_cast<VectorType>(inputType)) {
     if (vectorType.getElementType().isBF16() && !allowBf16)
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, vectorType.clone(rewriter.getI16Type()), input);
     if (vectorType.getElementType().isInteger(8) &&
         vectorType.getNumElements() <= 8)
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, rewriter.getIntegerType(vectorType.getNumElements() * 8), input);
     if (isa<IntegerType>(vectorType.getElementType()) &&
         vectorType.getElementTypeBitWidth() <= 8) {
       int64_t numWords = llvm::divideCeil(
           vectorType.getNumElements() * vectorType.getElementTypeBitWidth(),
           32);
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, VectorType::get(numWords, rewriter.getI32Type()), input);
     }
   }
@@ -549,8 +549,8 @@ static Value castMFMAScaleOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   Type outputType = rewriter.getI32Type();
   if (auto intType = dyn_cast<IntegerType>(inputType))
-    return rewriter.create<LLVM::ZExtOp>(loc, outputType, input);
-  return rewriter.create<LLVM::BitcastOp>(loc, outputType, input);
+    return LLVM::ZExtOp::create(rewriter, loc, outputType, input);
+  return LLVM::BitcastOp::create(rewriter, loc, outputType, input);
 }
 
 /// Push an input operand. If it is a float type, nothing to do. If it is
@@ -576,7 +576,7 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   Type elemType = vectorType.getElementType();
 
   if (elemType.isBF16())
-    llvmInput = rewriter.create<LLVM::BitcastOp>(
+    llvmInput = LLVM::BitcastOp::create(rewriter,
         loc, vectorType.clone(rewriter.getI16Type()), llvmInput);
   if (elemType.getIntOrFloatBitWidth() > 8) {
     operands.push_back(llvmInput);
@@ -613,7 +613,7 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   // (256 / 64) * 4 = 16 bits of input (on gfx12+) but take i32 arguments.
   // Add in the zeros here.
   if (numBits < 32)
-    castInput = rewriter.create<LLVM::ZExtOp>(loc, i32, castInput);
+    castInput = LLVM::ZExtOp::create(rewriter, loc, i32, castInput);
   operands.push_back(castInput);
 }
 
@@ -633,7 +633,7 @@ static void wmmaPushOutputOperand(ConversionPatternRewriter &rewriter,
   auto vectorType = dyn_cast<VectorType>(inputType);
   Type elemType = vectorType.getElementType();
   if (elemType.isBF16())
-    output = rewriter.create<LLVM::BitcastOp>(
+    output = LLVM::BitcastOp::create(rewriter,
         loc, vectorType.clone(rewriter.getI16Type()), output);
   operands.push_back(output);
   if (elemType.isF16() || elemType.isBF16() || elemType.isInteger(16)) {
@@ -992,7 +992,7 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
     };
     Value lowered = rewriter.create(loweredOp)->getResult(0);
     if (outType != intrinsicOutType)
-      lowered = rewriter.create<LLVM::BitcastOp>(loc, outType, lowered);
+      lowered = LLVM::BitcastOp::create(rewriter, loc, outType, lowered);
     rewriter.replaceOp(op, lowered);
     return success();
   }
@@ -1093,7 +1093,7 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
     Operation *maybeCastBack = lowered;
     if (rawOutType != outType)
       maybeCastBack =
-          rewriter.create<LLVM::BitcastOp>(loc, outType, lowered->getResult(0));
+          LLVM::BitcastOp::create(rewriter, loc, outType, lowered->getResult(0));
     rewriter.replaceOp(op, maybeCastBack->getResults());
 
     return success();
@@ -1144,21 +1144,21 @@ struct TransposeLoadOpLowering
     case 4: {
       assert(numElements == 16);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr4_b64>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr4_b64::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 6: {
       assert(numElements == 16);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr6_b96>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr6_b96::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 8: {
       assert(numElements == 8);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr8_b64>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr8_b64::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
@@ -1311,21 +1311,21 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
   Type sourceElemType = getElementTypeOrSelf(op.getSource());
   // Extend to a v4i8
   if (!sourceVecType || sourceVecType.getNumElements() < 4) {
-    Value longVec = rewriter.create<LLVM::UndefOp>(loc, v4i8);
+    Value longVec = LLVM::UndefOp::create(rewriter, loc, v4i8);
     if (!sourceVecType) {
-      longVec = rewriter.create<LLVM::InsertElementOp>(
+      longVec = LLVM::InsertElementOp::create(rewriter,
           loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
-        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        Value elem = LLVM::ExtractElementOp::create(rewriter, loc, source, idx);
         longVec =
-            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+            LLVM::InsertElementOp::create(rewriter, loc, longVec, elem, idx);
       }
     }
     source = longVec;
   }
-  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+  Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
   if (resultVecType) {
     if (typeIsExpectedBf8ForChipset(chipset, sourceElemType)) {
       rewriter.replaceOpWithNewOp<ROCDL::CvtPkF32Bf8Op>(op, f32, i32Source,
@@ -1377,21 +1377,21 @@ LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
 
   // Extend to a packedVectorType
   if (sourceVecType.getNumElements() < packedVecType.getNumElements()) {
-    Value longVec = rewriter.create<LLVM::ZeroOp>(loc, packedVecType);
+    Value longVec = LLVM::ZeroOp::create(rewriter, loc, packedVecType);
     if (!sourceVecType) {
-      longVec = rewriter.create<LLVM::InsertElementOp>(
+      longVec = LLVM::InsertElementOp::create(rewriter,
           loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
-        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        Value elem = LLVM::ExtractElementOp::create(rewriter, loc, source, idx);
         longVec =
-            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+            LLVM::InsertElementOp::create(rewriter, loc, longVec, elem, idx);
       }
     }
     source = longVec;
   }
-  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+  Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
 
   if (isa<Float8E5M2Type>(sourceElemType) && destElemType.isF32())
     rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Bf8Op>(
@@ -1449,53 +1449,53 @@ LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
   Value scale = adaptor.getScale();
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, intResultType, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, intResultType, existing);
   else
-    existing = rewriter.create<LLVM::ZeroOp>(loc, intResultType);
+    existing = LLVM::ZeroOp::create(rewriter, loc, intResultType);
 
   if (sourceVecType.getNumElements() < 2) {
     Value c0 = createI32Constant(rewriter, loc, 0);
-    Value elem0 = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
+    Value elem0 = LLVM::ExtractElementOp::create(rewriter, loc, source, c0);
     VectorType v2 = VectorType::get(2, sourceElemType);
-    source = rewriter.create<LLVM::ZeroOp>(loc, v2);
-    source = rewriter.create<LLVM::InsertElementOp>(loc, source, elem0, c0);
+    source = LLVM::ZeroOp::create(rewriter, loc, v2);
+    source = LLVM::InsertElementOp::create(rewriter, loc, source, elem0, c0);
   }
 
   Value sourceA, sourceB;
   if (sourceElemType.isF32()) {
     Value c0 = createI32Constant(rewriter, loc, 0);
     Value c1 = createI32Constant(rewriter, loc, 1);
-    sourceA = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
-    sourceB = rewriter.create<LLVM::ExtractElementOp>(loc, source, c1);
+    sourceA = LLVM::ExtractElementOp::create(rewriter, loc, source, c0);
+    sourceB = LLVM::ExtractElementOp::create(rewriter, loc, source, c1);
   }
 
   Value result;
   if (sourceElemType.isF32() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F32Op>(
+    result = ROCDL::CvtScaleF32PkBf8F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F16Op>(
+    result = ROCDL::CvtScaleF32PkBf8F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8Bf16Op>(
+    result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F32Op>(
+    result = ROCDL::CvtScaleF32PkFp8F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F16Op>(
+    result = ROCDL::CvtScaleF32PkFp8F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8Bf16Op>(
+    result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F32Op>(
+    result = ROCDL::CvtScaleF32PkFp4F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F16Op>(
+    result = ROCDL::CvtScaleF32PkFp4F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4Bf16Op>(
+    result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else
     return failure();
@@ -1521,19 +1521,19 @@ LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
   Value sourceA = adaptor.getSourceA();
   Value sourceB = adaptor.getSourceB();
   if (!sourceB)
-    sourceB = rewriter.create<LLVM::UndefOp>(loc, sourceA.getType());
+    sourceB = LLVM::UndefOp::create(rewriter, loc, sourceA.getType());
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, i32, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, i32, existing);
   else
-    existing = rewriter.create<LLVM::UndefOp>(loc, i32);
+    existing = LLVM::UndefOp::create(rewriter, loc, i32);
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtPkBf8F32Op>(loc, i32, sourceA, sourceB,
+    result = ROCDL::CvtPkBf8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
                                                    existing, op.getWordIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtPkFp8F32Op>(loc, i32, sourceA, sourceB,
+    result = ROCDL::CvtPkFp8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
                                                    existing, op.getWordIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
@@ -1558,16 +1558,16 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
   Value stoch = adaptor.getStochiasticParam();
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, i32, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, i32, existing);
   else
-    existing = rewriter.create<LLVM::UndefOp>(loc, i32);
+    existing = LLVM::UndefOp::create(rewriter, loc, i32);
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtSrBf8F32Op>(
+    result = ROCDL::CvtSrBf8F32Op::create(rewriter,
         loc, i32, source, stoch, existing, op.getStoreIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtSrFp8F32Op>(
+    result = ROCDL::CvtSrFp8F32Op::create(rewriter,
         loc, i32, source, stoch, existing, op.getStoreIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
@@ -1612,14 +1612,14 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
       if (operandType.getIntOrFloatBitWidth() <= 16) {
         if (llvm::isa<FloatType>(operandType)) {
           operand =
-              rewriter.create<LLVM::BitcastOp>(loc, llvmSrcIntType, operand);
+              LLVM::BitcastOp::create(rewriter, loc, llvmSrcIntType, operand);
         }
         auto llvmVecType = typeConverter->convertType(mlir::VectorType::get(
             32 / operandType.getIntOrFloatBitWidth(), llvmSrcIntType));
-        Value undefVec = rewriter.create<LLVM::UndefOp>(loc, llvmVecType);
-        operand = rewriter.create<LLVM::InsertElementOp>(
+        Value undefVec = LLVM::UndefOp::create(rewriter, loc, llvmVecType);
+        operand = LLVM::InsertElementOp::create(rewriter,
             loc, undefVec, operand, createI32Constant(rewriter, loc, 0));
-        operand = rewriter.create<LLVM::BitcastOp>(loc, llvmType, operand);
+        operand = LLVM::BitcastOp::create(rewriter, loc, llvmType, operand);
       }
       return operand;
     };
@@ -1706,14 +1706,14 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
     bool boundCtrl = DppOp->getAttrOfType<BoolAttr>("bound_ctrl").getValue();
 
     // create a ROCDL_DPPMovOp instruction with the appropriate attributes
-    auto dppMovOp = rewriter.create<ROCDL::DPPUpdateOp>(
+    auto dppMovOp = ROCDL::DPPUpdateOp::create(rewriter,
         loc, llvmType, old, src, DppCtrl, rowMask, bankMask, boundCtrl);
 
     Value result = dppMovOp.getRes();
     if (srcType.getIntOrFloatBitWidth() < 32) {
-      result = rewriter.create<LLVM::TruncOp>(loc, llvmSrcIntType, result);
+      result = LLVM::TruncOp::create(rewriter, loc, llvmSrcIntType, result);
       if (!llvm::isa<IntegerType>(srcType)) {
-        result = rewriter.create<LLVM::BitcastOp>(loc, srcType, result);
+        result = LLVM::BitcastOp::create(rewriter, loc, srcType, result);
       }
     }
 
@@ -1747,7 +1747,7 @@ struct AMDGPUSwizzleBitModeLowering
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
       Value res =
-          rewriter.create<ROCDL::DsSwizzleOp>(loc, v.getType(), v, maskValue);
+          ROCDL::DsSwizzleOp::create(rewriter, loc, v.getType(), v, maskValue);
       swizzled.emplace_back(res);
     }
 
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 3b143ca1ef9ce..5a9d946c2fe4e 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -50,9 +50,9 @@ static Value buildMinMaxReductionSeq(Location loc,
   Value value = *valueIt++;
   for (; valueIt != values.end(); ++valueIt) {
     if (predicate == arith::CmpIPredicate::sgt)
-      value = builder.create<arith::MaxSIOp>(loc, value, *valueIt);
+      value = arith::MaxSIOp::create(builder, loc, value, *valueIt);
     else
-      value = builder.create<arith::MinSIOp>(loc, value, *valueIt);
+      value = arith::MinSIOp::create(builder, loc, value, *valueIt);
   }
 
   return value;
@@ -154,8 +154,8 @@ class AffineForLowering : public OpRewritePattern<AffineForOp> {
     Value lowerBound = lowerAffineLowerBound(op, rewriter);
     Value upperBound = lowerAffineUpperBound(op, rewriter);
     Value step =
-        rewriter.create<arith::ConstantIndexOp>(loc, op.getStepAsInt());
-    auto scfForOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound,
+        arith::ConstantIndexOp::create(rewriter, loc, op.getStepAsInt());
+    auto scfForOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound,
                                                 step, op.getInits());
     rewriter.eraseBlock(scfForOp.getBody());
     rewriter.inlineRegionBefore(op.getRegion(), scfForOp.getRegion(),
@@ -197,7 +197,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     }
     steps.reserve(op.getSteps().size());
     for (int64_t step : op.getSteps())
-      steps.push_back(rewriter.create<arith::ConstantIndexOp>(loc, step));
+      steps.push_back(arith::ConstantIndexOp::create(rewriter, loc, step));
 
     // Get the terminator op.
     auto affineParOpTerminator =
@@ -205,7 +205,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     scf::ParallelOp parOp;
     if (op.getResults().empty()) {
       // Case with no reduction operations/return values.
-      parOp = rewriter.create<scf::ParallelOp>(loc, lowerBoundTuple,
+      parOp = scf::ParallelOp::create(rewriter, loc, lowerBoundTuple,
                                                upperBoundTuple, steps,
                                                /*bodyBuilderFn=*/nullptr);
       rewriter.eraseBlock(parOp.getBody());
@@ -233,7 +233,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       identityVals.push_back(
           arith::getIdentityValue(reductionOpValue, resultType, rewriter, loc));
     }
-    parOp = rewriter.create<scf::ParallelOp>(
+    parOp = scf::ParallelOp::create(rewriter,
         loc, lowerBoundTuple, upperBoundTuple, steps, identityVals,
         /*bodyBuilderFn=*/nullptr);
 
@@ -261,7 +261,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       Value reductionResult = arith::getReductionOp(
           reductionOpValue, rewriter, loc, reductionBody.getArgument(0),
           reductionBody.getArgument(1));
-      rewriter.create<scf::ReduceReturnOp>(loc, reductionResult);
+      scf::ReduceReturnOp::create(rewriter, loc, reductionResult);
     }
     rewriter.replaceOp(op, parOp.getResults());
     return success();
@@ -278,7 +278,7 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
 
     // Now we just have to handle the condition logic.
     auto integerSet = op.getIntegerSet();
-    Value zeroConstant = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zeroConstant = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value, 8> operands(op.getOperands());
     auto operandsRef = llvm::ArrayRef(operands);
 
@@ -298,17 +298,17 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
       auto pred =
           isEquality ? arith::CmpIPredicate::eq : arith::CmpIPredicate::sge;
       Value cmpVal =
-          rewriter.create<arith::CmpIOp>(loc, pred, affResult, zeroConstant);
+          arith::CmpIOp::create(rewriter, loc, pred, affResult, zeroConstant);
       cond = cond
-                 ? rewriter.create<arith::AndIOp>(loc, cond, cmpVal).getResult()
+                 ? arith::AndIOp::create(rewriter, loc, cond, cmpVal).getResult()
                  : cmpVal;
     }
     cond = cond ? cond
-                : rewriter.create<arith::ConstantIntOp>(loc, /*value=*/1,
+                : arith::ConstantIntOp::create(rewriter, loc, /*value=*/1,
                                                         /*width=*/1);
 
     bool hasElseRegion = !op.getElseRegion().empty();
-    auto ifOp = rewriter.create<scf::IfOp>(loc, op.getResultTypes(), cond,
+    auto ifOp = scf::IfOp::create(rewriter, loc, op.getResultTypes(), cond,
                                            hasElseRegion);
     rewriter.inlineRegionBefore(op.getThenRegion(),
                                 &ifOp.getThenRegion().back());
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index 3596b3235a631..d0155b12a6ec2 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -89,9 +89,9 @@ static Value castF32To(Type desType, Value f32, Location loc,
   if (elementType.isF32())
     return f32;
   if (elementType.getIntOrFloatBitWidth() < 32)
-    return rewriter.create<arith::TruncFOp>(loc, desType, f32);
+    return arith::TruncFOp::create(rewriter, loc, desType, f32);
   if (elementType.getIntOrFloatBitWidth() > 32)
-    return rewriter.create<arith::ExtFOp>(loc, desType, f32);
+    return arith::ExtFOp::create(rewriter, loc, desType, f32);
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
@@ -113,7 +113,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
   VectorType extResType = VectorType::get(2, rewriter.getF32Type());
   if (!inVecType) {
-    Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
+    Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
         loc, rewriter.getF32Type(), in, 0);
     Value result = castF32To(outElemType, asFloat, loc, rewriter);
     rewriter.replaceOp(op, result);
@@ -121,7 +121,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   }
   int64_t numElements = inVecType.getNumElements();
 
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   VectorType outType = cast<VectorType>(op.getOut().getType());
 
@@ -129,10 +129,10 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
     Value zerodSplat =
         rewriter.createOrFold<vector::SplatOp>(loc, outType, zero);
     Value scalarIn =
-        rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
+        vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     Value scalarExt =
-        rewriter.create<arith::ExtFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarExt, zerodSplat,
+        arith::ExtFOp::create(rewriter, loc, outElemType, scalarIn);
+    Value result = vector::InsertOp::create(rewriter, loc, scalarExt, zerodSplat,
                                                      ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
@@ -145,32 +145,32 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   if (inVecType.getRank() > 1) {
     inVecType = VectorType::get(SmallVector<int64_t>{numElements},
                                 inVecType.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVecType, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVecType, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
-    Value inSlice = rewriter.create<vector::ExtractStridedSliceOp>(
+    Value inSlice = vector::ExtractStridedSliceOp::create(rewriter,
         loc, in, i, elemsThisOp, 1);
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
       if (i + j + 1 < numElements) { // Convert two 8-bit elements
-        Value asFloats = rewriter.create<amdgpu::ExtPackedFp8Op>(
+        Value asFloats = amdgpu::ExtPackedFp8Op::create(rewriter,
             loc, extResType, inSlice, j / 2);
         Type desType = VectorType::get(2, outElemType);
         Value asType = castF32To(desType, asFloats, loc, rewriter);
-        result = rewriter.create<vector::InsertStridedSliceOp>(
+        result = vector::InsertStridedSliceOp::create(rewriter,
             loc, asType, result, i + j, 1);
       } else { // Convert a 8-bit element
-        Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
+        Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
             loc, rewriter.getF32Type(), inSlice, j / 2 * 2);
         Value asType = castF32To(outElemType, asFloat, loc, rewriter);
-        result = rewriter.create<vector::InsertOp>(loc, asType, result, i + j);
+        result = vector::InsertOp::create(rewriter, loc, asType, result, i + j);
       }
     }
   }
 
   if (inVecType.getRank() != outType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outType, result);
   }
 
   rewriter.replaceOp(op, result);
@@ -182,9 +182,9 @@ static Value castToF32(Value value, Location loc, PatternRewriter &rewriter) {
   if (type.isF32())
     return value;
   if (type.getIntOrFloatBitWidth() < 32)
-    return rewriter.create<arith::ExtFOp>(loc, rewriter.getF32Type(), value);
+    return arith::ExtFOp::create(rewriter, loc, rewriter.getF32Type(), value);
   if (type.getIntOrFloatBitWidth() > 32)
-    return rewriter.create<arith::TruncFOp>(loc, rewriter.getF32Type(), value);
+    return arith::TruncFOp::create(rewriter, loc, rewriter.getF32Type(), value);
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
@@ -224,13 +224,13 @@ static Value clampInput(PatternRewriter &rewriter, Location loc,
       loc, arith::CmpFPredicate::OEQ, source, negInf);
   Value isNan = rewriter.createOrFold<arith::CmpFOp>(
       loc, arith::CmpFPredicate::UNO, source, source);
-  Value isNonFinite = rewriter.create<arith::OrIOp>(
-      loc, rewriter.create<arith::OrIOp>(loc, isInf, isNegInf), isNan);
+  Value isNonFinite = arith::OrIOp::create(rewriter,
+      loc, arith::OrIOp::create(rewriter, loc, isInf, isNegInf), isNan);
 
-  Value clampedBelow = rewriter.create<arith::MaximumFOp>(loc, source, minCst);
-  Value clamped = rewriter.create<arith::MinimumFOp>(loc, clampedBelow, maxCst);
+  Value clampedBelow = arith::MaximumFOp::create(rewriter, loc, source, minCst);
+  Value clamped = arith::MinimumFOp::create(rewriter, loc, clampedBelow, maxCst);
   Value res =
-      rewriter.create<arith::SelectOp>(loc, isNonFinite, source, clamped);
+      arith::SelectOp::create(rewriter, loc, isNonFinite, source, clamped);
   return res;
 }
 
@@ -264,24 +264,24 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   VectorType truncResType = VectorType::get(4, outElemType);
   if (!inVectorTy) {
     Value asFloat = castToF32(in, loc, rewriter);
-    Value asF8s = rewriter.create<amdgpu::PackedTrunc2xFp8Op>(
+    Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
         loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
         /*existing=*/nullptr);
-    Value result = rewriter.create<vector::ExtractOp>(loc, asF8s, 0);
+    Value result = vector::ExtractOp::create(rewriter, loc, asF8s, 0);
     rewriter.replaceOp(op, result);
     return success();
   }
 
   int64_t numElements = outVecType.getNumElements();
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   if (outVecType.getShape().empty()) {
     Value scalarIn =
-        rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
+        vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     // Recurse to send the 0-D vector case to the 1-D vector case
     Value scalarTrunc =
-        rewriter.create<arith::TruncFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarTrunc, zero,
+        arith::TruncFOp::create(rewriter, loc, outElemType, scalarIn);
+    Value result = vector::InsertOp::create(rewriter, loc, scalarTrunc, zero,
                                                      ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
@@ -294,32 +294,32 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
                                  inVectorTy.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVectorTy, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVectorTy, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
     Value thisResult = nullptr;
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
-      Value elemA = rewriter.create<vector::ExtractOp>(loc, in, i + j);
+      Value elemA = vector::ExtractOp::create(rewriter, loc, in, i + j);
       Value asFloatA = castToF32(elemA, loc, rewriter);
       Value asFloatB = nullptr;
       if (j + 1 < elemsThisOp) {
-        Value elemB = rewriter.create<vector::ExtractOp>(loc, in, i + j + 1);
+        Value elemB = vector::ExtractOp::create(rewriter, loc, in, i + j + 1);
         asFloatB = castToF32(elemB, loc, rewriter);
       }
-      thisResult = rewriter.create<amdgpu::PackedTrunc2xFp8Op>(
+      thisResult = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
           loc, truncResType, asFloatA, asFloatB, j / 2, thisResult);
     }
     if (elemsThisOp < 4)
-      thisResult = rewriter.create<vector::ExtractStridedSliceOp>(
+      thisResult = vector::ExtractStridedSliceOp::create(rewriter,
           loc, thisResult, 0, elemsThisOp, 1);
-    result = rewriter.create<vector::InsertStridedSliceOp>(loc, thisResult,
+    result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
                                                            result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
@@ -347,10 +347,10 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
 
   // Handle the case where input type is not a vector type
   if (!inVectorTy) {
-    auto sourceB = rewriter.create<LLVM::PoisonOp>(loc, rewriter.getF32Type());
+    auto sourceB = LLVM::PoisonOp::create(rewriter, loc, rewriter.getF32Type());
     Value asF16s =
-        rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, in, sourceB);
-    Value result = rewriter.create<vector::ExtractOp>(loc, asF16s, 0);
+        ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, in, sourceB);
+    Value result = vector::ExtractOp::create(rewriter, loc, asF16s, 0);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -362,7 +362,7 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
                                  inVectorTy.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVectorTy, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVectorTy, in);
   }
 
   // Handle the vector case. We also handle the (uncommon) case where the vector
@@ -370,25 +370,25 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   for (int64_t i = 0; i < numElements; i += 2) {
     int64_t elemsThisOp = std::min(numElements, i + 2) - i;
     Value thisResult = nullptr;
-    Value elemA = rewriter.create<vector::ExtractOp>(loc, in, i);
-    Value elemB = rewriter.create<LLVM::PoisonOp>(loc, rewriter.getF32Type());
+    Value elemA = vector::ExtractOp::create(rewriter, loc, in, i);
+    Value elemB = LLVM::PoisonOp::create(rewriter, loc, rewriter.getF32Type());
 
     if (elemsThisOp == 2) {
-      elemB = rewriter.create<vector::ExtractOp>(loc, in, i + 1);
+      elemB = vector::ExtractOp::create(rewriter, loc, in, i + 1);
     }
 
     thisResult =
-        rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, elemA, elemB);
+        ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, elemA, elemB);
     // Place back the truncated result into the possibly larger vector. If we
     // are operating on a size 2 vector, these operations should be folded away
-    thisResult = rewriter.create<vector::ExtractStridedSliceOp>(
+    thisResult = vector::ExtractStridedSliceOp::create(rewriter,
         loc, thisResult, 0, elemsThisOp, 1);
-    result = rewriter.create<vector::InsertStridedSliceOp>(loc, thisResult,
+    result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
                                                            result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
index cbe0b3fda3410..40bf8f1cffb04 100644
--- a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
+++ b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
@@ -74,14 +74,14 @@ struct ConstantOpToArmSMELowering : public OpRewritePattern<arith::ConstantOp> {
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
     auto denseAttr1D = DenseElementsAttr::get(
         tileSliceType, denseAttr.getSplatValue<Attribute>());
-    auto constantOp1D = rewriter.create<arith::ConstantOp>(loc, denseAttr1D);
+    auto constantOp1D = arith::ConstantOp::create(rewriter, loc, denseAttr1D);
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to write vector to tile
       // slice.
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, constantOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
index a5c08a6378021..c5b3fdc536f72 100644
--- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
+++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
@@ -110,7 +110,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     emitc::CmpPredicate predicate;
     switch (op.getPredicate()) {
     case arith::CmpFPredicate::AlwaysFalse: {
-      auto constant = rewriter.create<emitc::ConstantOp>(
+      auto constant = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getI1Type(),
           rewriter.getBoolAttr(/*value=*/false));
       rewriter.replaceOp(op, constant);
@@ -179,7 +179,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
       return success();
     }
     case arith::CmpFPredicate::AlwaysTrue: {
-      auto constant = rewriter.create<emitc::ConstantOp>(
+      auto constant = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getI1Type(),
           rewriter.getBoolAttr(/*value=*/true));
       rewriter.replaceOp(op, constant);
@@ -189,7 +189,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
 
     // Compare the values naively
     auto cmpResult =
-        rewriter.create<emitc::CmpOp>(op.getLoc(), op.getType(), predicate,
+        emitc::CmpOp::create(rewriter, op.getLoc(), op.getType(), predicate,
                                       adaptor.getLhs(), adaptor.getRhs());
 
     // Adjust the results for unordered/ordered semantics
@@ -213,7 +213,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNaN(ConversionPatternRewriter &rewriter, Location loc,
               Value operand) const {
     // A value is NaN exactly when it compares unequal to itself.
-    return rewriter.create<emitc::CmpOp>(
+    return emitc::CmpOp::create(rewriter,
         loc, rewriter.getI1Type(), emitc::CmpPredicate::ne, operand, operand);
   }
 
@@ -221,7 +221,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNotNaN(ConversionPatternRewriter &rewriter, Location loc,
                  Value operand) const {
     // A value is not NaN exactly when it compares equal to itself.
-    return rewriter.create<emitc::CmpOp>(
+    return emitc::CmpOp::create(rewriter,
         loc, rewriter.getI1Type(), emitc::CmpPredicate::eq, operand, operand);
   }
 
@@ -231,7 +231,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
                                Location loc, Value first, Value second) const {
     auto firstIsNaN = isNaN(rewriter, loc, first);
     auto secondIsNaN = isNaN(rewriter, loc, second);
-    return rewriter.create<emitc::LogicalOrOp>(loc, rewriter.getI1Type(),
+    return emitc::LogicalOrOp::create(rewriter, loc, rewriter.getI1Type(),
                                                firstIsNaN, secondIsNaN);
   }
 
@@ -241,7 +241,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
                              Value first, Value second) const {
     auto firstIsNotNaN = isNotNaN(rewriter, loc, first);
     auto secondIsNotNaN = isNotNaN(rewriter, loc, second);
-    return rewriter.create<emitc::LogicalAndOp>(loc, rewriter.getI1Type(),
+    return emitc::LogicalAndOp::create(rewriter, loc, rewriter.getI1Type(),
                                                 firstIsNotNaN, secondIsNotNaN);
   }
 };
@@ -378,9 +378,9 @@ class CastConversion : public OpConversionPattern<ArithOp> {
       Type attrType = (emitc::isPointerWideType(operandType))
                           ? rewriter.getIndexType()
                           : operandType;
-      auto constOne = rewriter.create<emitc::ConstantOp>(
+      auto constOne = emitc::ConstantOp::create(rewriter,
           op.getLoc(), operandType, rewriter.getOneAttr(attrType));
-      auto oneAndOperand = rewriter.create<emitc::BitwiseAndOp>(
+      auto oneAndOperand = emitc::BitwiseAndOp::create(rewriter,
           op.getLoc(), operandType, adaptor.getIn(), constOne);
       rewriter.replaceOpWithNewOp<emitc::CastOp>(op, opReturnType,
                                                  oneAndOperand);
@@ -467,7 +467,7 @@ class BinaryUIOpConversion final : public OpConversionPattern<ArithOp> {
     Value rhsAdapted = adaptValueType(uiBinOp.getRhs(), rewriter, unsignedType);
 
     auto newDivOp =
-        rewriter.create<EmitCOp>(uiBinOp.getLoc(), unsignedType,
+        EmitCOp::create(rewriter, uiBinOp.getLoc(), unsignedType,
                                  ArrayRef<Value>{lhsAdapted, rhsAdapted});
     Value resultAdapted = adaptValueType(newDivOp, rewriter, newRetTy);
     rewriter.replaceOp(uiBinOp, resultAdapted);
@@ -588,38 +588,38 @@ class ShiftOpConversion : public OpConversionPattern<ArithOp> {
     // Add a runtime check for overflow
     Value width;
     if (emitc::isPointerWideType(type)) {
-      Value eight = rewriter.create<emitc::ConstantOp>(
+      Value eight = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rhsType, rewriter.getIndexAttr(8));
-      emitc::CallOpaqueOp sizeOfCall = rewriter.create<emitc::CallOpaqueOp>(
+      emitc::CallOpaqueOp sizeOfCall = emitc::CallOpaqueOp::create(rewriter,
           op.getLoc(), rhsType, "sizeof", ArrayRef<Value>{eight});
-      width = rewriter.create<emitc::MulOp>(op.getLoc(), rhsType, eight,
+      width = emitc::MulOp::create(rewriter, op.getLoc(), rhsType, eight,
                                             sizeOfCall.getResult(0));
     } else {
-      width = rewriter.create<emitc::ConstantOp>(
+      width = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rhsType,
           rewriter.getIntegerAttr(rhsType, type.getIntOrFloatBitWidth()));
     }
 
-    Value excessCheck = rewriter.create<emitc::CmpOp>(
+    Value excessCheck = emitc::CmpOp::create(rewriter,
         op.getLoc(), rewriter.getI1Type(), emitc::CmpPredicate::lt, rhs, width);
 
     // Any concrete value is a valid refinement of poison.
-    Value poison = rewriter.create<emitc::ConstantOp>(
+    Value poison = emitc::ConstantOp::create(rewriter,
         op.getLoc(), arithmeticType,
         (isa<IntegerType>(arithmeticType)
              ? rewriter.getIntegerAttr(arithmeticType, 0)
              : rewriter.getIndexAttr(0)));
 
-    emitc::ExpressionOp ternary = rewriter.create<emitc::ExpressionOp>(
+    emitc::ExpressionOp ternary = emitc::ExpressionOp::create(rewriter,
         op.getLoc(), arithmeticType, /*do_not_inline=*/false);
     Block &bodyBlock = ternary.getBodyRegion().emplaceBlock();
     auto currentPoint = rewriter.getInsertionPoint();
     rewriter.setInsertionPointToStart(&bodyBlock);
     Value arithmeticResult =
-        rewriter.create<EmitCOp>(op.getLoc(), arithmeticType, lhs, rhs);
-    Value resultOrPoison = rewriter.create<emitc::ConditionalOp>(
+        EmitCOp::create(rewriter, op.getLoc(), arithmeticType, lhs, rhs);
+    Value resultOrPoison = emitc::ConditionalOp::create(rewriter,
         op.getLoc(), arithmeticType, excessCheck, arithmeticResult, poison);
-    rewriter.create<emitc::YieldOp>(op.getLoc(), resultOrPoison);
+    emitc::YieldOp::create(rewriter, op.getLoc(), resultOrPoison);
     rewriter.setInsertionPoint(op->getBlock(), currentPoint);
 
     Value result = adaptValueType(ternary, rewriter, type);
@@ -700,11 +700,11 @@ class FtoICastOpConversion : public OpConversionPattern<CastOp> {
                                   /*isSigned=*/false);
     }
 
-    Value result = rewriter.create<emitc::CastOp>(
+    Value result = emitc::CastOp::create(rewriter,
         castOp.getLoc(), actualResultType, adaptor.getOperands());
 
     if (isa<arith::FPToUIOp>(castOp)) {
-      result = rewriter.create<emitc::CastOp>(castOp.getLoc(), dstType, result);
+      result = emitc::CastOp::create(rewriter, castOp.getLoc(), dstType, result);
     }
     rewriter.replaceOp(castOp, result);
 
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index f7bf581adc9e3..f7d4963b5e293 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -293,10 +293,10 @@ LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
       [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
         typename OpTy::Adaptor adaptor(operands);
         if (targetBits < sourceBits) {
-          return rewriter.create<LLVM::TruncOp>(op.getLoc(), llvm1DVectorTy,
+          return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
                                                 adaptor.getIn());
         }
-        return rewriter.create<ExtCastTy>(op.getLoc(), llvm1DVectorTy,
+        return ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
                                           adaptor.getIn());
       },
       rewriter);
@@ -324,12 +324,12 @@ LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
     Type newOverflowType = typeConverter->convertType(overflowResultType);
     Type structType =
         LLVM::LLVMStructType::getLiteral(ctx, {sumResultType, newOverflowType});
-    Value addOverflow = rewriter.create<LLVM::UAddWithOverflowOp>(
+    Value addOverflow = LLVM::UAddWithOverflowOp::create(rewriter,
         loc, structType, adaptor.getLhs(), adaptor.getRhs());
     Value sumExtracted =
-        rewriter.create<LLVM::ExtractValueOp>(loc, addOverflow, 0);
+        LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 0);
     Value overflowExtracted =
-        rewriter.create<LLVM::ExtractValueOp>(loc, addOverflow, 1);
+        LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 1);
     rewriter.replaceOp(op, {sumExtracted, overflowExtracted});
     return success();
   }
@@ -381,15 +381,15 @@ LogicalResult MulIExtendedOpLowering<ArithMulOp, IsSigned>::matchAndRewrite(
            "LLVM dialect should support all signless integer types");
 
     using LLVMExtOp = std::conditional_t<IsSigned, LLVM::SExtOp, LLVM::ZExtOp>;
-    Value lhsExt = rewriter.create<LLVMExtOp>(loc, wideType, adaptor.getLhs());
-    Value rhsExt = rewriter.create<LLVMExtOp>(loc, wideType, adaptor.getRhs());
-    Value mulExt = rewriter.create<LLVM::MulOp>(loc, wideType, lhsExt, rhsExt);
+    Value lhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getLhs());
+    Value rhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getRhs());
+    Value mulExt = LLVM::MulOp::create(rewriter, loc, wideType, lhsExt, rhsExt);
 
     // Split the 2*N-bit wide result into two N-bit values.
-    Value low = rewriter.create<LLVM::TruncOp>(loc, resultType, mulExt);
-    Value shiftVal = rewriter.create<LLVM::ConstantOp>(loc, shiftValAttr);
-    Value highExt = rewriter.create<LLVM::LShrOp>(loc, mulExt, shiftVal);
-    Value high = rewriter.create<LLVM::TruncOp>(loc, resultType, highExt);
+    Value low = LLVM::TruncOp::create(rewriter, loc, resultType, mulExt);
+    Value shiftVal = LLVM::ConstantOp::create(rewriter, loc, shiftValAttr);
+    Value highExt = LLVM::LShrOp::create(rewriter, loc, mulExt, shiftVal);
+    Value high = LLVM::TruncOp::create(rewriter, loc, resultType, highExt);
 
     rewriter.replaceOp(op, {low, high});
     return success();
@@ -435,7 +435,7 @@ CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return rewriter.create<LLVM::ICmpOp>(
+        return LLVM::ICmpOp::create(rewriter,
             op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs());
@@ -471,7 +471,7 @@ CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return rewriter.create<LLVM::FCmpOp>(
+        return LLVM::FCmpOp::create(rewriter,
             op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs(), fmf);
diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index 434d7df853a5e..c93cf13f831ef 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -117,11 +117,11 @@ static Value getScalarOrVectorConstInt(Type type, uint64_t value,
   if (auto vectorType = dyn_cast<VectorType>(type)) {
     Attribute element = IntegerAttr::get(vectorType.getElementType(), value);
     auto attr = SplatElementsAttr::get(vectorType, element);
-    return builder.create<spirv::ConstantOp>(loc, vectorType, attr);
+    return spirv::ConstantOp::create(builder, loc, vectorType, attr);
   }
 
   if (auto intType = dyn_cast<IntegerType>(type))
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, value));
 
   return nullptr;
@@ -418,18 +418,18 @@ static Value emulateSignedRemainder(Location loc, Value lhs, Value rhs,
   Type type = lhs.getType();
 
   // Calculate the remainder with spirv.UMod.
-  Value lhsAbs = builder.create<SignedAbsOp>(loc, type, lhs);
-  Value rhsAbs = builder.create<SignedAbsOp>(loc, type, rhs);
-  Value abs = builder.create<spirv::UModOp>(loc, lhsAbs, rhsAbs);
+  Value lhsAbs = SignedAbsOp::create(builder, loc, type, lhs);
+  Value rhsAbs = SignedAbsOp::create(builder, loc, type, rhs);
+  Value abs = spirv::UModOp::create(builder, loc, lhsAbs, rhsAbs);
 
   // Fix the sign.
   Value isPositive;
   if (lhs == signOperand)
-    isPositive = builder.create<spirv::IEqualOp>(loc, lhs, lhsAbs);
+    isPositive = spirv::IEqualOp::create(builder, loc, lhs, lhsAbs);
   else
-    isPositive = builder.create<spirv::IEqualOp>(loc, rhs, rhsAbs);
-  Value absNegate = builder.create<spirv::SNegateOp>(loc, type, abs);
-  return builder.create<spirv::SelectOp>(loc, type, isPositive, abs, absNegate);
+    isPositive = spirv::IEqualOp::create(builder, loc, rhs, rhsAbs);
+  Value absNegate = spirv::SNegateOp::create(builder, loc, type, abs);
+  return spirv::SelectOp::create(builder, loc, type, isPositive, abs, absNegate);
 }
 
 /// Converts arith.remsi to GLSL SPIR-V ops.
@@ -601,12 +601,12 @@ struct ExtSII1Pattern final : public OpConversionPattern<arith::ExtSIOp> {
     Value allOnes;
     if (auto intTy = dyn_cast<IntegerType>(dstType)) {
       unsigned componentBitwidth = intTy.getWidth();
-      allOnes = rewriter.create<spirv::ConstantOp>(
+      allOnes = spirv::ConstantOp::create(rewriter,
           loc, intTy,
           rewriter.getIntegerAttr(intTy, APInt::getAllOnes(componentBitwidth)));
     } else if (auto vectorTy = dyn_cast<VectorType>(dstType)) {
       unsigned componentBitwidth = vectorTy.getElementTypeBitWidth();
-      allOnes = rewriter.create<spirv::ConstantOp>(
+      allOnes = spirv::ConstantOp::create(rewriter,
           loc, vectorTy,
           SplatElementsAttr::get(vectorTy,
                                  APInt::getAllOnes(componentBitwidth)));
@@ -653,7 +653,7 @@ struct ExtSIPattern final : public OpConversionPattern<arith::ExtSIOp> {
       // First shift left to sequeeze out all leading bits beyond the original
       // bitwidth. Here we need to use the original source and result type's
       // bitwidth.
-      auto shiftLOp = rewriter.create<spirv::ShiftLeftLogicalOp>(
+      auto shiftLOp = spirv::ShiftLeftLogicalOp::create(rewriter,
           op.getLoc(), dstType, adaptor.getIn(), shiftSize);
 
       // Then we perform arithmetic right shift to make sure we have the right
@@ -757,9 +757,9 @@ struct TruncII1Pattern final : public OpConversionPattern<arith::TruncIOp> {
     auto srcType = adaptor.getOperands().front().getType();
     // Check if (x & 1) == 1.
     Value mask = spirv::ConstantOp::getOne(srcType, loc, rewriter);
-    Value maskedSrc = rewriter.create<spirv::BitwiseAndOp>(
+    Value maskedSrc = spirv::BitwiseAndOp::create(rewriter,
         loc, srcType, adaptor.getOperands()[0], mask);
-    Value isOne = rewriter.create<spirv::IEqualOp>(loc, maskedSrc, mask);
+    Value isOne = spirv::IEqualOp::create(rewriter, loc, maskedSrc, mask);
 
     Value zero = spirv::ConstantOp::getZero(dstType, loc, rewriter);
     Value one = spirv::ConstantOp::getOne(dstType, loc, rewriter);
@@ -914,9 +914,9 @@ class CmpIOpBooleanPattern final : public OpConversionPattern<arith::CmpIOp> {
       if (auto vectorType = dyn_cast<VectorType>(dstType))
         type = VectorType::get(vectorType.getShape(), type);
       Value extLhs =
-          rewriter.create<arith::ExtUIOp>(op.getLoc(), type, adaptor.getLhs());
+          arith::ExtUIOp::create(rewriter, op.getLoc(), type, adaptor.getLhs());
       Value extRhs =
-          rewriter.create<arith::ExtUIOp>(op.getLoc(), type, adaptor.getRhs());
+          arith::ExtUIOp::create(rewriter, op.getLoc(), type, adaptor.getRhs());
 
       rewriter.replaceOpWithNewOp<arith::CmpIOp>(op, op.getPredicate(), extLhs,
                                                  extRhs);
@@ -1067,12 +1067,12 @@ class CmpFOpNanNonePattern final : public OpConversionPattern<arith::CmpFOp> {
         replace = spirv::ConstantOp::getZero(op.getType(), loc, rewriter);
       }
     } else {
-      Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-      Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+      Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+      Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-      replace = rewriter.create<spirv::LogicalOrOp>(loc, lhsIsNan, rhsIsNan);
+      replace = spirv::LogicalOrOp::create(rewriter, loc, lhsIsNan, rhsIsNan);
       if (op.getPredicate() == arith::CmpFPredicate::ORD)
-        replace = rewriter.create<spirv::LogicalNotOp>(loc, replace);
+        replace = spirv::LogicalNotOp::create(rewriter, loc, replace);
     }
 
     rewriter.replaceOp(op, replace);
@@ -1094,17 +1094,17 @@ class AddUIExtendedOpPattern final
                   ConversionPatternRewriter &rewriter) const override {
     Type dstElemTy = adaptor.getLhs().getType();
     Location loc = op->getLoc();
-    Value result = rewriter.create<spirv::IAddCarryOp>(loc, adaptor.getLhs(),
+    Value result = spirv::IAddCarryOp::create(rewriter, loc, adaptor.getLhs(),
                                                        adaptor.getRhs());
 
-    Value sumResult = rewriter.create<spirv::CompositeExtractOp>(
+    Value sumResult = spirv::CompositeExtractOp::create(rewriter,
         loc, result, llvm::ArrayRef(0));
-    Value carryValue = rewriter.create<spirv::CompositeExtractOp>(
+    Value carryValue = spirv::CompositeExtractOp::create(rewriter,
         loc, result, llvm::ArrayRef(1));
 
     // Convert the carry value to boolean.
     Value one = spirv::ConstantOp::getOne(dstElemTy, loc, rewriter);
-    Value carryResult = rewriter.create<spirv::IEqualOp>(loc, carryValue, one);
+    Value carryResult = spirv::IEqualOp::create(rewriter, loc, carryValue, one);
 
     rewriter.replaceOp(op, {sumResult, carryResult});
     return success();
@@ -1125,11 +1125,11 @@ class MulIExtendedOpPattern final : public OpConversionPattern<ArithMulOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op->getLoc();
     Value result =
-        rewriter.create<SPIRVMulOp>(loc, adaptor.getLhs(), adaptor.getRhs());
+        SPIRVMulOp::create(rewriter, loc, adaptor.getLhs(), adaptor.getRhs());
 
-    Value low = rewriter.create<spirv::CompositeExtractOp>(loc, result,
+    Value low = spirv::CompositeExtractOp::create(rewriter, loc, result,
                                                            llvm::ArrayRef(0));
-    Value high = rewriter.create<spirv::CompositeExtractOp>(loc, result,
+    Value high = spirv::CompositeExtractOp::create(rewriter, loc, result,
                                                             llvm::ArrayRef(1));
 
     rewriter.replaceOp(op, {low, high});
@@ -1183,19 +1183,19 @@ class MinimumMaximumFOpPattern final : public OpConversionPattern<Op> {
 
     Location loc = op.getLoc();
     Value spirvOp =
-        rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
+        SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
 
     if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
       rewriter.replaceOp(op, spirvOp);
       return success();
     }
 
-    Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-    Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+    Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+    Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-    Value select1 = rewriter.create<spirv::SelectOp>(loc, dstType, lhsIsNan,
+    Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
                                                      adaptor.getLhs(), spirvOp);
-    Value select2 = rewriter.create<spirv::SelectOp>(loc, dstType, rhsIsNan,
+    Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
                                                      adaptor.getRhs(), select1);
 
     rewriter.replaceOp(op, select2);
@@ -1237,7 +1237,7 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
 
     Location loc = op.getLoc();
     Value spirvOp =
-        rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
+        SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
 
     if (!shouldInsertNanGuards<SPIRVOp>() ||
         bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
@@ -1245,12 +1245,12 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
       return success();
     }
 
-    Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-    Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+    Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+    Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-    Value select1 = rewriter.create<spirv::SelectOp>(loc, dstType, lhsIsNan,
+    Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
                                                      adaptor.getRhs(), spirvOp);
-    Value select2 = rewriter.create<spirv::SelectOp>(loc, dstType, rhsIsNan,
+    Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
                                                      adaptor.getLhs(), select1);
 
     rewriter.replaceOp(op, select2);
diff --git a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
index 9c6de938a7108..2295d661d44a5 100644
--- a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
+++ b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
@@ -41,10 +41,10 @@ class Sdot2dLoweringPattern : public OpRewritePattern<Sdot2dOp> {
     Value c2d = op.getC();
     Location loc = op.getLoc();
     Value b1d =
-        rewriter.create<vector::ShapeCastOp>(loc, flattenedVectorType, b2d);
+        vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, b2d);
     Value c1d =
-        rewriter.create<vector::ShapeCastOp>(loc, flattenedVectorType, c2d);
-    Value newOp = rewriter.create<SdotOp>(loc, op.getRes().getType(), op.getA(),
+        vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, c2d);
+    Value newOp = SdotOp::create(rewriter, loc, op.getRes().getType(), op.getA(),
                                           b1d, c1d);
     rewriter.replaceOp(op, {newOp});
     return success();
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 21ea444e31821..cf46e0c10df09 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -45,37 +45,37 @@ static Operation *createLoadTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_ld1b_horiz>(
+      return arm_sme::aarch64_sme_ld1b_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_ld1h_horiz>(
+      return arm_sme::aarch64_sme_ld1h_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_ld1w_horiz>(
+      return arm_sme::aarch64_sme_ld1w_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_ld1d_horiz>(
+      return arm_sme::aarch64_sme_ld1d_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_ld1q_horiz>(
+      return arm_sme::aarch64_sme_ld1q_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_ld1b_vert>(
+      return arm_sme::aarch64_sme_ld1b_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_ld1h_vert>(
+      return arm_sme::aarch64_sme_ld1h_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_ld1w_vert>(
+      return arm_sme::aarch64_sme_ld1w_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_ld1d_vert>(
+      return arm_sme::aarch64_sme_ld1d_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_ld1q_vert>(
+      return arm_sme::aarch64_sme_ld1q_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
       break;
     }
@@ -91,37 +91,37 @@ static Operation *createStoreTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_st1b_horiz>(
+      return arm_sme::aarch64_sme_st1b_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_st1h_horiz>(
+      return arm_sme::aarch64_sme_st1h_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_st1w_horiz>(
+      return arm_sme::aarch64_sme_st1w_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_st1d_horiz>(
+      return arm_sme::aarch64_sme_st1d_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_st1q_horiz>(
+      return arm_sme::aarch64_sme_st1q_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_st1b_vert>(
+      return arm_sme::aarch64_sme_st1b_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_st1h_vert>(
+      return arm_sme::aarch64_sme_st1h_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_st1w_vert>(
+      return arm_sme::aarch64_sme_st1w_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_st1d_vert>(
+      return arm_sme::aarch64_sme_st1d_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_st1q_vert>(
+      return arm_sme::aarch64_sme_st1q_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   }
@@ -146,15 +146,15 @@ createAllocaForTile(RewriterBase &rewriter, Location loc,
   // Move to the first operation in the function.
   rewriter.setInsertionPointToStart(&func.getBlocks().front());
   // Create an alloca matching the tile size of the `tileOp`.
-  auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
+  auto vscale = vector::VectorScaleOp::create(rewriter, loc);
   auto tileElementType = tileOp.getTileType().getElementType();
   auto memrefType = MemRefType::get(
       {ShapedType::kDynamic, ShapedType::kDynamic}, tileElementType);
   unsigned minElements = arm_sme::getSMETileSliceMinNumElts(tileElementType);
   auto minElementsOp =
-      rewriter.create<arith::ConstantIndexOp>(loc, minElements);
-  auto vectorLen = rewriter.create<arith::MulIOp>(loc, vscale, minElementsOp);
-  auto alloca = rewriter.create<memref::AllocaOp>(
+      arith::ConstantIndexOp::create(rewriter, loc, minElements);
+  auto vectorLen = arith::MulIOp::create(rewriter, loc, vscale, minElementsOp);
+  auto alloca = memref::AllocaOp::create(rewriter,
       loc, memrefType, ValueRange{vectorLen, vectorLen});
   return alloca;
 }
@@ -293,9 +293,9 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                                 Value tileMemory, Value sliceIndex) const {
     auto llvmType = getTypeConverter()->convertType(tileMemory.getType());
     auto descriptor =
-        rewriter.create<UnrealizedConversionCastOp>(loc, llvmType, tileMemory);
-    auto zero = rewriter.create<arith::ConstantIntOp>(loc, 0, /*width=*/64);
-    auto sliceIndexI64 = rewriter.create<arith::IndexCastOp>(
+        UnrealizedConversionCastOp::create(rewriter, loc, llvmType, tileMemory);
+    auto zero = arith::ConstantIntOp::create(rewriter, loc, 0, /*width=*/64);
+    auto sliceIndexI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), sliceIndex);
     return getStridedElementPtr(
         static_cast<ConversionPatternRewriter &>(rewriter), loc,
@@ -309,27 +309,27 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                      arm_sme::ArmSMETileType tileType, VectorType sliceType,
                      IntegerAttr tileId, Value sliceIndex) const {
     // Cast the slice index to an i32.
-    auto sliceIndexI32 = rewriter.create<arith::IndexCastOp>(
+    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), sliceIndex);
     // Create an all-true predicate for the slice.
     auto predicateType = sliceType.clone(rewriter.getI1Type());
-    auto allTruePredicate = rewriter.create<arith::ConstantOp>(
+    auto allTruePredicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
     // Create padding vector (never used due to all-true predicate).
-    auto padVector = rewriter.create<LLVM::PoisonOp>(loc, sliceType);
+    auto padVector = LLVM::PoisonOp::create(rewriter, loc, sliceType);
     // Get a pointer to the current slice.
     auto slicePtr =
         getInMemoryTileSlicePtr(rewriter, loc, tileAlloca, sliceIndex);
     // Read the value of the current slice from ZA.
-    auto currentTileSlice = rewriter.create<arm_sme::aarch64_sme_read_horiz>(
+    auto currentTileSlice = arm_sme::aarch64_sme_read_horiz::create(rewriter,
         loc, sliceType, padVector, allTruePredicate, tileId, sliceIndexI32);
     // Load the new tile slice back from memory into ZA.
     createLoadTileSliceIntrinsic(
         rewriter, loc, tileType, arm_sme::TileSliceLayout::Horizontal,
         allTruePredicate, slicePtr, tileId, sliceIndexI32);
     // Store the current tile slice to memory.
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    rewriter.create<vector::StoreOp>(loc, currentTileSlice, tileAlloca,
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    vector::StoreOp::create(rewriter, loc, currentTileSlice, tileAlloca,
                                      ValueRange{sliceIndex, zero});
   }
 
@@ -341,12 +341,12 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     RewriterBase::InsertionGuard guard(rewriter);
     // Create an scf.for over all tile slices.
     auto minNumElts =
-        rewriter.create<arith::ConstantIndexOp>(loc, sliceType.getDimSize(0));
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto upperBound = rewriter.create<arith::MulIOp>(
-        loc, minNumElts, rewriter.create<vector::VectorScaleOp>(loc));
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        arith::ConstantIndexOp::create(rewriter, loc, sliceType.getDimSize(0));
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto upperBound = arith::MulIOp::create(rewriter,
+        loc, minNumElts, vector::VectorScaleOp::create(rewriter, loc));
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     // Emit a swap for each tile slice.
     rewriter.setInsertionPointToStart(forOp.getBody());
     auto sliceIndex = forOp.getInductionVar();
@@ -479,7 +479,7 @@ struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
     //
     // This holds for all tile sizes.
     int32_t zeroMask = baseMaskForSize << int32_t(tileId.getInt());
-    rewriter.create<arm_sme::aarch64_sme_zero>(
+    arm_sme::aarch64_sme_zero::create(rewriter,
         loc, rewriter.getI32IntegerAttr(zeroMask));
 
     // Create a placeholder op to preserve dataflow.
@@ -513,7 +513,7 @@ struct LoadTileSliceConversion
     auto tileSlice = loadTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
@@ -559,7 +559,7 @@ struct StoreTileSliceConversion
     auto tileSlice = storeTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     auto maskOp = storeTileSliceOp.getMask();
@@ -595,26 +595,26 @@ struct InsertTileSliceConversion
     auto tileSlice = insertTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
-    auto one = rewriter.create<arith::ConstantOp>(
+    auto one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI1Type(),
         rewriter.getIntegerAttr(rewriter.getI1Type(), 1));
     auto predTy = VectorType::get(tileType.getShape()[0], rewriter.getI1Type(),
                                   /*scalableDims=*/{true});
-    auto allActiveMask = rewriter.create<vector::SplatOp>(loc, predTy, one);
+    auto allActiveMask = vector::SplatOp::create(rewriter, loc, predTy, one);
 
     // Create 'arm_sme.intr.write.(horiz|vert)' to write vector to tile slice.
     switch (insertTileSliceOp.getLayout()) {
     case arm_sme::TileSliceLayout::Horizontal:
-      rewriter.create<arm_sme::aarch64_sme_write_horiz>(
+      arm_sme::aarch64_sme_write_horiz::create(rewriter,
           loc, tileId, tileSliceI32, allActiveMask,
           insertTileSliceOp.getVector());
       break;
     case arm_sme::TileSliceLayout::Vertical:
-      rewriter.create<arm_sme::aarch64_sme_write_vert>(
+      arm_sme::aarch64_sme_write_vert::create(rewriter,
           loc, tileId, tileSliceI32, allActiveMask,
           insertTileSliceOp.getVector());
       break;
@@ -646,15 +646,15 @@ struct ExtractTileSliceConversion
 
     // Create an 'all true' predicate for the tile slice.
     auto predicateType = sliceType.cloneWith({}, rewriter.getI1Type());
-    auto allTruePredicate = rewriter.create<arith::ConstantOp>(
+    auto allTruePredicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
 
     // Zero destination/fallback for tile slice extraction.
-    auto zeroVector = rewriter.create<arith::ConstantOp>(
+    auto zeroVector = arith::ConstantOp::create(rewriter,
         loc, sliceType, rewriter.getZeroAttr(sliceType));
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto sliceIndexI32 = rewriter.create<arith::IndexCastOp>(
+    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), sliceIndex);
 
     // Create 'arm_sme.intr.read.(horiz|vert)' to extract the tile slice.
@@ -743,7 +743,7 @@ struct OuterProductOpConversion
     Value acc = outerProductOp.getAcc();
     if (!acc) {
       // Initalize accumulator with zero.
-      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, resultVectorType);
+      auto zero = arm_sme::ZeroOp::create(rewriter, loc, resultVectorType);
       zero.setTileId(tileId);
       acc = zero;
     }
@@ -754,14 +754,14 @@ struct OuterProductOpConversion
     if (!lhsMask || !rhsMask) {
       auto predTy =
           outerProductOp.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = rewriter.create<arith::ConstantOp>(
+      Value allActiveMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
     // Create 'arm_sme.intr.mopa' outer product intrinsic.
-    rewriter.create<arm_sme::aarch64_sme_mopa>(loc, tileId, lhsMask, rhsMask,
+    arm_sme::aarch64_sme_mopa::create(rewriter, loc, tileId, lhsMask, rhsMask,
                                                outerProductOp.getLhs(),
                                                outerProductOp.getRhs());
 
@@ -792,7 +792,7 @@ struct OuterProductWideningOpConversion
     Value acc = op.getAcc();
     if (!acc) {
       // Initalize accumulator with zero.
-      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, op.getResultType());
+      auto zero = arm_sme::ZeroOp::create(rewriter, loc, op.getResultType());
       zero.setTileId(tileId);
       acc = zero;
     }
@@ -801,13 +801,13 @@ struct OuterProductWideningOpConversion
     Value rhsMask = op.getRhsMask();
     if (!lhsMask || !rhsMask) {
       auto predTy = op.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = rewriter.create<arith::ConstantOp>(
+      Value allActiveMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
-    rewriter.create<OuterProductWideningIntrOp>(
+    OuterProductWideningIntrOp::create(rewriter,
         loc, tileId, lhsMask, rhsMask, adaptor.getLhs(), adaptor.getRhs());
 
     // The outerproduct intrinsics have no result, replace
@@ -843,13 +843,13 @@ struct StreamingVLOpConversion
     auto *intrOp = [&]() -> Operation * {
       switch (streamingVlOp.getTypeSize()) {
       case arm_sme::TypeSize::Byte:
-        return rewriter.create<arm_sme::aarch64_sme_cntsb>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsb::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Half:
-        return rewriter.create<arm_sme::aarch64_sme_cntsh>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsh::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Word:
-        return rewriter.create<arm_sme::aarch64_sme_cntsw>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsw::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Double:
-        return rewriter.create<arm_sme::aarch64_sme_cntsd>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsd::create(rewriter, loc, i64Type);
       }
       llvm_unreachable("unknown type size in StreamingVLOpConversion");
     }();
@@ -872,7 +872,7 @@ static void mergeConsecutiveTileZerosInBlock(Block *block) {
     if (zeroOpsToMerge.size() <= 1)
       return;
     IRRewriter rewriter(zeroOpsToMerge.front());
-    rewriter.create<arm_sme::aarch64_sme_zero>(
+    arm_sme::aarch64_sme_zero::create(rewriter,
         zeroOpsToMerge.front().getLoc(),
         rewriter.getI32IntegerAttr(mergedZeroMask));
     for (auto zeroOp : zeroOpsToMerge)
diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 458628c29c6ac..3591ed65f8d84 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -39,7 +39,7 @@ SmallVector<Value, 2> getMemrefIndices(ValueRange indices, unsigned rank,
   auto tileSliceOffset = tileSliceIndex;
 
   auto baseIndexPlusTileSliceOffset =
-      rewriter.create<arith::AddIOp>(loc, indices[0], tileSliceOffset);
+      arith::AddIOp::create(rewriter, loc, indices[0], tileSliceOffset);
   outIndices.push_back(baseIndexPlusTileSliceOffset);
   outIndices.push_back(indices[1]);
 
@@ -59,10 +59,10 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   if (memrefIndices.size() != 2)
     return rewriter.notifyMatchFailure(loc, "invalid number of indices");
 
-  auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
       loc, arm_sme::getSMETileSliceMinNumElts(tileType.getElementType()));
   auto vscale =
-      rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
+      vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
   auto predicateType =
       VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
 
@@ -70,7 +70,7 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   // elements in a vector of SVL bits for a given element type (SVL_B,
   // SVL_H, ..., SVL_Q).
   auto numTileSlices =
-      rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
+      arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
 
   Value predicate;
   Value upperBound;
@@ -82,28 +82,28 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
     // The upper bound of the loop must be clamped at `numTileSlices` as
     // `vector.create_mask` allows operands to be greater than the size of a
     // dimension.
-    auto numRowI64 = rewriter.create<arith::IndexCastOp>(
+    auto numRowI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), maskDim0);
-    auto numTileSlicesI64 = rewriter.create<arith::IndexCastOp>(
+    auto numTileSlicesI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), numTileSlices);
     auto upperBoundI64 =
-        rewriter.create<arith::MinSIOp>(loc, numRowI64, numTileSlicesI64);
-    upperBound = rewriter.create<arith::IndexCastOp>(
+        arith::MinSIOp::create(rewriter, loc, numRowI64, numTileSlicesI64);
+    upperBound = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getIndexType(), upperBoundI64);
 
     predicate =
-        rewriter.create<vector::CreateMaskOp>(loc, predicateType, maskDim1);
+        vector::CreateMaskOp::create(rewriter, loc, predicateType, maskDim1);
   } else {
     upperBound = numTileSlices;
     // No mask. Create an 'all true' predicate for the tile slice.
-    predicate = rewriter.create<arith::ConstantOp>(
+    predicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
   }
 
   bool hasCarriedArgs = bool(initTile);
-  auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-  auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+  auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
                                            hasCarriedArgs ? ValueRange{initTile}
                                                           : ValueRange{});
 
@@ -118,7 +118,7 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
 
   assert(bool(nextTile) == hasCarriedArgs);
   if (nextTile)
-    rewriter.create<scf::YieldOp>(loc, nextTile);
+    scf::YieldOp::create(rewriter, loc, nextTile);
 
   return forOp;
 }
@@ -194,9 +194,9 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
       // Initialize tile with zero to satisfy padding. Inactive cols will be
       // zeroed anyway since the loads use zeroing predication. For inactive
       // rows however, no load will occur so these need to be zeroed.
-      initTile = rewriter.create<arm_sme::ZeroOp>(loc, tileType);
+      initTile = arm_sme::ZeroOp::create(rewriter, loc, tileType);
     } else {
-      initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+      initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
     }
 
     // Create a loop to load the active tile slices from memory.
@@ -207,7 +207,7 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
             Value currentTile) -> Value {
           // Create 'arm_sme.load_tile_slice' to load tile slice from memory
           // into tile.
-          return rewriter.create<arm_sme::LoadTileSliceOp>(
+          return arm_sme::LoadTileSliceOp::create(rewriter,
               loc, tileType, tileLoadOp.getBase(), predicate, currentTile,
               memrefIndices, tileSliceIndex, tileLoadOp.getLayout());
         });
@@ -283,21 +283,21 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto numRows = createMaskOp.getOperands()[0];
     auto numCols = createMaskOp.getOperands()[1];
 
-    auto numColsI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto numColsI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), numCols);
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     // Create a loop that loads each ZA tile slice from memory.
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
         loc, arm_sme::getSMETileSliceMinNumElts(tileElementType));
     auto vscale =
-        rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+        vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto numTileSlices =
-        rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, numTileSlices,
+        arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices,
                                              step, ValueRange{initTile});
 
     rewriter.setInsertionPointToStart(forOp.getBody());
@@ -306,16 +306,16 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto currentTile = forOp.getRegionIterArg(0);
 
     // Combine masks.
-    auto rowIsActive = rewriter.create<arith::CmpIOp>(
+    auto rowIsActive = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, tileSliceIndex, numRows);
-    auto rowIsActiveI32 = rewriter.create<arith::ExtSIOp>(
+    auto rowIsActiveI32 = arith::ExtSIOp::create(rewriter,
         loc, rewriter.getI32Type(), rowIsActive);
-    auto mask = rewriter.create<arith::AndIOp>(loc, rowIsActiveI32, numColsI32);
+    auto mask = arith::AndIOp::create(rewriter, loc, rowIsActiveI32, numColsI32);
     auto maskIndex =
-        rewriter.create<arith::IndexCastOp>(loc, rewriter.getIndexType(), mask);
+        arith::IndexCastOp::create(rewriter, loc, rewriter.getIndexType(), mask);
     auto predicateType =
         VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
-    auto maskOp1D = rewriter.create<vector::CreateMaskOp>(
+    auto maskOp1D = vector::CreateMaskOp::create(rewriter,
         loc, predicateType, maskIndex.getResult());
 
     auto memrefIndices = getMemrefIndices(
@@ -324,17 +324,17 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
 
     // Splat pad into 1-D vector matching type of tile slice.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    auto pad1DOp = rewriter.create<vector::SplatOp>(loc, tileSliceType, padOp);
+    auto pad1DOp = vector::SplatOp::create(rewriter, loc, tileSliceType, padOp);
 
-    auto loadSlice = rewriter.create<vector::MaskedLoadOp>(
+    auto loadSlice = vector::MaskedLoadOp::create(rewriter,
         loc, tileSliceType, tileLoadOp.getBase(), memrefIndices, maskOp1D,
         /*passthru=*/pad1DOp);
 
     // Create 'arm_sme.insert_tile_slice' to insert slice into tile.
-    auto insertSlice = rewriter.create<arm_sme::InsertTileSliceOp>(
+    auto insertSlice = arm_sme::InsertTileSliceOp::create(rewriter,
         loc, tileType, loadSlice->getResult(0), currentTile, tileSliceIndex,
         tileLoadOp.getLayout());
-    rewriter.create<scf::YieldOp>(loc, insertSlice.getResult());
+    scf::YieldOp::create(rewriter, loc, insertSlice.getResult());
 
     rewriter.setInsertionPointAfter(forOp);
 
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index 94f7caa315cf7..e1d28f19ee236 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -203,7 +203,7 @@ static void addAsyncRuntimeApiDeclarations(ModuleOp module) {
   auto addFuncDecl = [&](StringRef name, FunctionType type) {
     if (module.lookupSymbol(name))
       return;
-    builder.create<func::FuncOp>(name, type).setPrivate();
+    func::FuncOp::create(builder, name, type).setPrivate();
   };
 
   MLIRContext *ctx = module.getContext();
@@ -254,15 +254,15 @@ static void addResumeFunction(ModuleOp module) {
   auto voidTy = LLVM::LLVMVoidType::get(ctx);
   Type ptrType = AsyncAPI::opaquePointerType(ctx);
 
-  auto resumeOp = moduleBuilder.create<LLVM::LLVMFuncOp>(
+  auto resumeOp = LLVM::LLVMFuncOp::create(moduleBuilder,
       kResume, LLVM::LLVMFunctionType::get(voidTy, {ptrType}));
   resumeOp.setPrivate();
 
   auto *block = resumeOp.addEntryBlock(moduleBuilder);
   auto blockBuilder = ImplicitLocOpBuilder::atBlockEnd(loc, block);
 
-  blockBuilder.create<LLVM::CoroResumeOp>(resumeOp.getArgument(0));
-  blockBuilder.create<LLVM::ReturnOp>(ValueRange());
+  LLVM::CoroResumeOp::create(blockBuilder, resumeOp.getArgument(0));
+  LLVM::ReturnOp::create(blockBuilder, ValueRange());
 }
 
 //===----------------------------------------------------------------------===//
@@ -282,7 +282,7 @@ class AsyncRuntimeTypeConverter : public TypeConverter {
     // in patterns for other dialects.
     auto addUnrealizedCast = [](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+      auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
       return cast.getResult(0);
     };
 
@@ -343,8 +343,8 @@ class CoroIdOpConversion : public AsyncOpConversionPattern<CoroIdOp> {
 
     // Constants for initializing coroutine frame.
     auto constZero =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), 0);
-    auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, ptrType);
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), 0);
+    auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, ptrType);
 
     // Get coroutine id: @llvm.coro.id.
     rewriter.replaceOpWithNewOp<LLVM::CoroIdOp>(
@@ -372,32 +372,32 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
 
     // Get coroutine frame size: @llvm.coro.size.i64.
     Value coroSize =
-        rewriter.create<LLVM::CoroSizeOp>(loc, rewriter.getI64Type());
+        LLVM::CoroSizeOp::create(rewriter, loc, rewriter.getI64Type());
     // Get coroutine frame alignment: @llvm.coro.align.i64.
     Value coroAlign =
-        rewriter.create<LLVM::CoroAlignOp>(loc, rewriter.getI64Type());
+        LLVM::CoroAlignOp::create(rewriter, loc, rewriter.getI64Type());
 
     // Round up the size to be multiple of the alignment. Since aligned_alloc
     // requires the size parameter be an integral multiple of the alignment
     // parameter.
     auto makeConstant = [&](uint64_t c) {
-      return rewriter.create<LLVM::ConstantOp>(op->getLoc(),
+      return LLVM::ConstantOp::create(rewriter, op->getLoc(),
                                                rewriter.getI64Type(), c);
     };
-    coroSize = rewriter.create<LLVM::AddOp>(op->getLoc(), coroSize, coroAlign);
+    coroSize = LLVM::AddOp::create(rewriter, op->getLoc(), coroSize, coroAlign);
     coroSize =
-        rewriter.create<LLVM::SubOp>(op->getLoc(), coroSize, makeConstant(1));
+        LLVM::SubOp::create(rewriter, op->getLoc(), coroSize, makeConstant(1));
     Value negCoroAlign =
-        rewriter.create<LLVM::SubOp>(op->getLoc(), makeConstant(0), coroAlign);
+        LLVM::SubOp::create(rewriter, op->getLoc(), makeConstant(0), coroAlign);
     coroSize =
-        rewriter.create<LLVM::AndOp>(op->getLoc(), coroSize, negCoroAlign);
+        LLVM::AndOp::create(rewriter, op->getLoc(), coroSize, negCoroAlign);
 
     // Allocate memory for the coroutine frame.
     auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn(
         rewriter, op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
     if (failed(allocFuncOp))
       return failure();
-    auto coroAlloc = rewriter.create<LLVM::CallOp>(
+    auto coroAlloc = LLVM::CallOp::create(rewriter,
         loc, allocFuncOp.value(), ValueRange{coroAlign, coroSize});
 
     // Begin a coroutine: @llvm.coro.begin.
@@ -427,7 +427,7 @@ class CoroFreeOpConversion : public AsyncOpConversionPattern<CoroFreeOp> {
 
     // Get a pointer to the coroutine frame memory: @llvm.coro.free.
     auto coroMem =
-        rewriter.create<LLVM::CoroFreeOp>(loc, ptrType, adaptor.getOperands());
+        LLVM::CoroFreeOp::create(rewriter, loc, ptrType, adaptor.getOperands());
 
     // Free the memory.
     auto freeFuncOp =
@@ -455,13 +455,13 @@ class CoroEndOpConversion : public OpConversionPattern<CoroEndOp> {
   matchAndRewrite(CoroEndOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // We are not in the block that is part of the unwind sequence.
-    auto constFalse = rewriter.create<LLVM::ConstantOp>(
+    auto constFalse = LLVM::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getI1Type(), rewriter.getBoolAttr(false));
-    auto noneToken = rewriter.create<LLVM::NoneTokenOp>(op->getLoc());
+    auto noneToken = LLVM::NoneTokenOp::create(rewriter, op->getLoc());
 
     // Mark the end of a coroutine: @llvm.coro.end.
     auto coroHdl = adaptor.getHandle();
-    rewriter.create<LLVM::CoroEndOp>(
+    LLVM::CoroEndOp::create(rewriter,
         op->getLoc(), rewriter.getI1Type(),
         ValueRange({coroHdl, constFalse, noneToken}));
     rewriter.eraseOp(op);
@@ -534,12 +534,12 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     auto loc = op->getLoc();
 
     // This is not a final suspension point.
-    auto constFalse = rewriter.create<LLVM::ConstantOp>(
+    auto constFalse = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI1Type(), rewriter.getBoolAttr(false));
 
     // Suspend a coroutine: @llvm.coro.suspend
     auto coroState = adaptor.getState();
-    auto coroSuspend = rewriter.create<LLVM::CoroSuspendOp>(
+    auto coroSuspend = LLVM::CoroSuspendOp::create(rewriter,
         loc, i8, ValueRange({coroState, constFalse}));
 
     // Cast return code to i32.
@@ -551,7 +551,7 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     llvm::SmallVector<Block *, 2> caseDest = {op.getResumeDest(),
                                               op.getCleanupDest()};
     rewriter.replaceOpWithNewOp<LLVM::SwitchOp>(
-        op, rewriter.create<LLVM::SExtOp>(loc, i32, coroSuspend.getResult()),
+        op, LLVM::SExtOp::create(rewriter, loc, i32, coroSuspend.getResult()),
         /*defaultDestination=*/op.getSuspendDest(),
         /*defaultOperands=*/ValueRange(),
         /*caseValues=*/caseValues,
@@ -602,11 +602,11 @@ class RuntimeCreateOpLowering : public ConvertOpToLLVMPattern<RuntimeCreateOp> {
 
         // %Size = getelementptr %T* null, int 1
         // %SizeI = ptrtoint %T* %Size to i64
-        auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, storagePtrType);
+        auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, storagePtrType);
         auto gep =
-            rewriter.create<LLVM::GEPOp>(loc, storagePtrType, storedType,
+            LLVM::GEPOp::create(rewriter, loc, storagePtrType, storedType,
                                          nullPtr, ArrayRef<LLVM::GEPArg>{1});
-        return rewriter.create<LLVM::PtrToIntOp>(loc, i64, gep);
+        return LLVM::PtrToIntOp::create(rewriter, loc, i64, gep);
       };
 
       rewriter.replaceOpWithNewOp<func::CallOp>(op, kCreateValue, resultType,
@@ -739,7 +739,7 @@ class RuntimeAwaitOpLowering : public OpConversionPattern<RuntimeAwaitOp> {
             .Case<ValueType>([](Type) { return kAwaitValue; })
             .Case<GroupType>([](Type) { return kAwaitGroup; });
 
-    rewriter.create<func::CallOp>(op->getLoc(), apiFuncName, TypeRange(),
+    func::CallOp::create(rewriter, op->getLoc(), apiFuncName, TypeRange(),
                                   adaptor.getOperands());
     rewriter.eraseOp(op);
 
@@ -772,11 +772,11 @@ class RuntimeAwaitAndResumeOpLowering
 
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = rewriter.create<LLVM::AddressOfOp>(
+    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
         op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
         kResume);
 
-    rewriter.create<func::CallOp>(
+    func::CallOp::create(rewriter,
         op->getLoc(), apiFuncName, TypeRange(),
         ValueRange({operand, handle, resumePtr.getRes()}));
     rewriter.eraseOp(op);
@@ -801,7 +801,7 @@ class RuntimeResumeOpLowering
                   ConversionPatternRewriter &rewriter) const override {
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = rewriter.create<LLVM::AddressOfOp>(
+    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
         op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
         kResume);
 
@@ -832,7 +832,7 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = rewriter.create<func::CallOp>(
+    auto storagePtr = func::CallOp::create(rewriter,
         loc, kGetValueStorage, TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
@@ -845,7 +845,7 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     Value castedStoragePtr = storagePtr.getResult(0);
     // Store the yielded value into the async value storage.
     auto value = adaptor.getValue();
-    rewriter.create<LLVM::StoreOp>(loc, value, castedStoragePtr);
+    LLVM::StoreOp::create(rewriter, loc, value, castedStoragePtr);
 
     // Erase the original runtime store operation.
     rewriter.eraseOp(op);
@@ -872,7 +872,7 @@ class RuntimeLoadOpLowering : public ConvertOpToLLVMPattern<RuntimeLoadOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = rewriter.create<func::CallOp>(
+    auto storagePtr = func::CallOp::create(rewriter,
         loc, kGetValueStorage, TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
@@ -960,7 +960,7 @@ class RefCountingOpLowering : public OpConversionPattern<RefCountingOp> {
   LogicalResult
   matchAndRewrite(RefCountingOp op, typename RefCountingOp::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto count = rewriter.create<arith::ConstantOp>(
+    auto count = arith::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getI64Type(),
         rewriter.getI64IntegerAttr(op.getCount()));
 
diff --git a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
index b9991f36cdaaf..a63a1ceb00ebb 100644
--- a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
+++ b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
@@ -47,26 +47,26 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
 
     if (auto unrankedType = dyn_cast<UnrankedMemRefType>(type)) {
       // Constants
-      Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-      Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+      Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+      Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
       // Dynamically evaluate the size and shape of the unranked memref
-      Value rank = rewriter.create<memref::RankOp>(loc, op.getInput());
+      Value rank = memref::RankOp::create(rewriter, loc, op.getInput());
       MemRefType allocType =
           MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType());
-      Value shape = rewriter.create<memref::AllocaOp>(loc, allocType, rank);
+      Value shape = memref::AllocaOp::create(rewriter, loc, allocType, rank);
 
       // Create a loop to query dimension sizes, store them as a shape, and
       // compute the total size of the memref
       auto loopBody = [&](OpBuilder &builder, Location loc, Value i,
                           ValueRange args) {
         auto acc = args.front();
-        auto dim = rewriter.create<memref::DimOp>(loc, op.getInput(), i);
+        auto dim = memref::DimOp::create(rewriter, loc, op.getInput(), i);
 
-        rewriter.create<memref::StoreOp>(loc, dim, shape, i);
-        acc = rewriter.create<arith::MulIOp>(loc, acc, dim);
+        memref::StoreOp::create(rewriter, loc, dim, shape, i);
+        acc = arith::MulIOp::create(rewriter, loc, acc, dim);
 
-        rewriter.create<scf::YieldOp>(loc, acc);
+        scf::YieldOp::create(rewriter, loc, acc);
       };
       auto size = rewriter
                       .create<scf::ForOp>(loc, zero, rank, one, ValueRange(one),
@@ -78,9 +78,9 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
 
       // Allocate new memref with 1D dynamic shape, then reshape into the
       // shape of the original unranked memref
-      alloc = rewriter.create<memref::AllocOp>(loc, memrefType, size);
+      alloc = memref::AllocOp::create(rewriter, loc, memrefType, size);
       alloc =
-          rewriter.create<memref::ReshapeOp>(loc, unrankedType, alloc, shape);
+          memref::ReshapeOp::create(rewriter, loc, unrankedType, alloc, shape);
     } else {
       MemRefType memrefType = cast<MemRefType>(type);
       MemRefLayoutAttrInterface layout;
@@ -103,14 +103,14 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
       }
 
       // Allocate a memref with identity layout.
-      alloc = rewriter.create<memref::AllocOp>(loc, allocType, dynamicOperands);
+      alloc = memref::AllocOp::create(rewriter, loc, allocType, dynamicOperands);
       // Cast the allocation to the specified type if needed.
       if (memrefType != allocType)
         alloc =
-            rewriter.create<memref::CastOp>(op->getLoc(), memrefType, alloc);
+            memref::CastOp::create(rewriter, op->getLoc(), memrefType, alloc);
     }
 
-    rewriter.create<memref::CopyOp>(loc, op.getInput(), alloc);
+    memref::CopyOp::create(rewriter, loc, op.getInput(), alloc);
     rewriter.replaceOp(op, alloc);
     return success();
   }
diff --git a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
index 70b22386f1eea..1961e1004ba8e 100644
--- a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
+++ b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
@@ -23,41 +23,41 @@ void mlir::complex::convertDivToLLVMUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, LLVM::FastmathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, rhsRe, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, rhsIm, rhsIm, fmf), fmf);
+  Value rhsSqNorm = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsIm, fmf), fmf);
+  Value realNumerator = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
 
-  Value imagNumerator = rewriter.create<LLVM::FSubOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsIm, fmf), fmf);
+  Value imagNumerator = LLVM::FSubOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
-  *resultRe = rewriter.create<LLVM::FDivOp>(loc, realNumerator, rhsSqNorm, fmf);
-  *resultIm = rewriter.create<LLVM::FDivOp>(loc, imagNumerator, rhsSqNorm, fmf);
+  *resultRe = LLVM::FDivOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
+  *resultIm = LLVM::FDivOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 void mlir::complex::convertDivToStandardUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, arith::FastMathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, rhsRe, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, rhsIm, rhsIm, fmf), fmf);
+  Value rhsSqNorm = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsIm, fmf), fmf);
-  Value imagNumerator = rewriter.create<arith::SubFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsIm, fmf), fmf);
+  Value realNumerator = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
+  Value imagNumerator = arith::SubFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
   *resultRe =
-      rewriter.create<arith::DivFOp>(loc, realNumerator, rhsSqNorm, fmf);
+      arith::DivFOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
   *resultIm =
-      rewriter.create<arith::DivFOp>(loc, imagNumerator, rhsSqNorm, fmf);
+      arith::DivFOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 // Smith's algorithm to divide complex numbers. It is just a bit smarter
@@ -94,180 +94,180 @@ void mlir::complex::convertDivToLLVMUsingRangeReduction(
   auto elementType = cast<FloatType>(rhsRe.getType());
 
   Value rhsRealImagRatio =
-      rewriter.create<LLVM::FDivOp>(loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = rewriter.create<LLVM::FAddOp>(
+      LLVM::FDivOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
+  Value rhsRealImagDenom = LLVM::FAddOp::create(rewriter,
       loc, rhsIm,
-      rewriter.create<LLVM::FMulOp>(loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRealImagRatio, fmf),
+      LLVM::FMulOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
+  Value realNumerator1 = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
       lhsIm, fmf);
   Value resultReal1 =
-      rewriter.create<LLVM::FDivOp>(loc, realNumerator1, rhsRealImagDenom, fmf);
-  Value imagNumerator1 = rewriter.create<LLVM::FSubOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRealImagRatio, fmf),
+      LLVM::FDivOp::create(rewriter, loc, realNumerator1, rhsRealImagDenom, fmf);
+  Value imagNumerator1 = LLVM::FSubOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
       lhsRe, fmf);
   Value resultImag1 =
-      rewriter.create<LLVM::FDivOp>(loc, imagNumerator1, rhsRealImagDenom, fmf);
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator1, rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
-      rewriter.create<LLVM::FDivOp>(loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = rewriter.create<LLVM::FAddOp>(
+      LLVM::FDivOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
+  Value rhsImagRealDenom = LLVM::FAddOp::create(rewriter,
       loc, rhsRe,
-      rewriter.create<LLVM::FMulOp>(loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = rewriter.create<LLVM::FAddOp>(
+      LLVM::FMulOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
+  Value realNumerator2 = LLVM::FAddOp::create(rewriter,
       loc, lhsRe,
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsImagRealRatio, fmf), fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
   Value resultReal2 =
-      rewriter.create<LLVM::FDivOp>(loc, realNumerator2, rhsImagRealDenom, fmf);
-  Value imagNumerator2 = rewriter.create<LLVM::FSubOp>(
+      LLVM::FDivOp::create(rewriter, loc, realNumerator2, rhsImagRealDenom, fmf);
+  Value imagNumerator2 = LLVM::FSubOp::create(rewriter,
       loc, lhsIm,
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsImagRealRatio, fmf), fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
   Value resultImag2 =
-      rewriter.create<LLVM::FDivOp>(loc, imagNumerator2, rhsImagRealDenom, fmf);
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator2, rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = rewriter.create<LLVM::ConstantOp>(
+  Value zero = LLVM::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
-  Value rhsRealAbs = rewriter.create<LLVM::FAbsOp>(loc, rhsRe, fmf);
-  Value rhsRealIsZero = rewriter.create<LLVM::FCmpOp>(
+  Value rhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, rhsRe, fmf);
+  Value rhsRealIsZero = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, zero);
-  Value rhsImagAbs = rewriter.create<LLVM::FAbsOp>(loc, rhsIm, fmf);
-  Value rhsImagIsZero = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, rhsIm, fmf);
+  Value rhsImagIsZero = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, zero);
   Value lhsRealIsNotNaN =
-      rewriter.create<LLVM::FCmpOp>(loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
+      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
   Value lhsImagIsNotNaN =
-      rewriter.create<LLVM::FCmpOp>(loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
+      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
   Value lhsContainsNotNaNValue =
-      rewriter.create<LLVM::OrOp>(loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = rewriter.create<LLVM::AndOp>(
+      LLVM::OrOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
+  Value resultIsInfinity = LLVM::AndOp::create(rewriter,
       loc, lhsContainsNotNaNValue,
-      rewriter.create<LLVM::AndOp>(loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = rewriter.create<LLVM::ConstantOp>(
+      LLVM::AndOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
+  Value inf = LLVM::ConstantOp::create(rewriter,
       loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfrhsReal =
-      rewriter.create<LLVM::CopySignOp>(loc, inf, rhsRe);
+      LLVM::CopySignOp::create(rewriter, loc, inf, rhsRe);
   Value infinityResultReal =
-      rewriter.create<LLVM::FMulOp>(loc, infWithSignOfrhsReal, lhsRe, fmf);
+      LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsRe, fmf);
   Value infinityResultImag =
-      rewriter.create<LLVM::FMulOp>(loc, infWithSignOfrhsReal, lhsIm, fmf);
+      LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsRealFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, rhsRealAbs, inf);
-  Value rhsImagFinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, rhsImagAbs, inf);
   Value rhsFinite =
-      rewriter.create<LLVM::AndOp>(loc, rhsRealFinite, rhsImagFinite);
-  Value lhsRealAbs = rewriter.create<LLVM::FAbsOp>(loc, lhsRe, fmf);
-  Value lhsRealInfinite = rewriter.create<LLVM::FCmpOp>(
+      LLVM::AndOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
+  Value lhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, lhsRe, fmf);
+  Value lhsRealInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, lhsRealAbs, inf);
-  Value lhsImagAbs = rewriter.create<LLVM::FAbsOp>(loc, lhsIm, fmf);
-  Value lhsImagInfinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, lhsIm, fmf);
+  Value lhsImagInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, lhsImagAbs, inf);
   Value lhsInfinite =
-      rewriter.create<LLVM::OrOp>(loc, lhsRealInfinite, lhsImagInfinite);
+      LLVM::OrOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
-      rewriter.create<LLVM::AndOp>(loc, lhsInfinite, rhsFinite);
-  Value one = rewriter.create<LLVM::ConstantOp>(
+      LLVM::AndOp::create(rewriter, loc, lhsInfinite, rhsFinite);
+  Value one = LLVM::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, lhsRealInfinite, one, zero),
+  Value lhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesrhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRealIsInfWithSign, rhsRe, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesrhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
+  Value resultReal3 = LLVM::FMulOp::create(rewriter,
       loc, inf,
-      rewriter.create<LLVM::FAddOp>(loc, lhsRealIsInfWithSignTimesrhsReal,
+      LLVM::FAddOp::create(rewriter, loc, lhsRealIsInfWithSignTimesrhsReal,
                                     lhsImagIsInfWithSignTimesrhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesrhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRealIsInfWithSign, rhsIm, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesrhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
+  Value resultImag3 = LLVM::FMulOp::create(rewriter,
       loc, inf,
-      rewriter.create<LLVM::FSubOp>(loc, lhsImagIsInfWithSignTimesrhsReal,
+      LLVM::FSubOp::create(rewriter, loc, lhsImagIsInfWithSignTimesrhsReal,
                                     lhsRealIsInfWithSignTimesrhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsRealFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, lhsRealAbs, inf);
-  Value lhsImagFinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsImagFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, lhsImagAbs, inf);
   Value lhsFinite =
-      rewriter.create<LLVM::AndOp>(loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = rewriter.create<LLVM::FCmpOp>(
+      LLVM::AndOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
+  Value rhsRealInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, inf);
-  Value rhsImagInfinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, inf);
   Value rhsInfinite =
-      rewriter.create<LLVM::OrOp>(loc, rhsRealInfinite, rhsImagInfinite);
+      LLVM::OrOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
-      rewriter.create<LLVM::AndOp>(loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, rhsRealInfinite, one, zero),
+      LLVM::AndOp::create(rewriter, loc, lhsFinite, rhsInfinite);
+  Value rhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimeslhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRealIsInfWithSign, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
+  Value resultReal4 = LLVM::FMulOp::create(rewriter,
       loc, zero,
-      rewriter.create<LLVM::FAddOp>(loc, rhsRealIsInfWithSignTimeslhsReal,
+      LLVM::FAddOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsReal,
                                     rhsImagIsInfWithSignTimeslhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimeslhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRealIsInfWithSign, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
+  Value resultImag4 = LLVM::FMulOp::create(rewriter,
       loc, zero,
-      rewriter.create<LLVM::FSubOp>(loc, rhsRealIsInfWithSignTimeslhsImag,
+      LLVM::FSubOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsImag,
                                     rhsImagIsInfWithSignTimeslhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = rewriter.create<LLVM::FCmpOp>(
+  Value realAbsSmallerThanImagAbs = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::olt, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = rewriter.create<LLVM::SelectOp>(
+  Value resultReal5 = LLVM::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = rewriter.create<LLVM::SelectOp>(
+  Value resultImag5 = LLVM::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase3 = LLVM::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase3 = LLVM::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase2 = LLVM::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase2 = LLVM::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase1 = LLVM::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase1 = LLVM::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = rewriter.create<LLVM::FCmpOp>(
+  Value resultRealIsNaN = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::uno, resultReal5, zero);
-  Value resultImagIsNaN = rewriter.create<LLVM::FCmpOp>(
+  Value resultImagIsNaN = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::uno, resultImag5, zero);
   Value resultIsNaN =
-      rewriter.create<LLVM::AndOp>(loc, resultRealIsNaN, resultImagIsNaN);
+      LLVM::AndOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = rewriter.create<LLVM::SelectOp>(
+  *resultRe = LLVM::SelectOp::create(rewriter,
       loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = rewriter.create<LLVM::SelectOp>(
+  *resultIm = LLVM::SelectOp::create(rewriter,
       loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
 }
 
@@ -278,179 +278,179 @@ void mlir::complex::convertDivToStandardUsingRangeReduction(
   auto elementType = cast<FloatType>(rhsRe.getType());
 
   Value rhsRealImagRatio =
-      rewriter.create<arith::DivFOp>(loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = rewriter.create<arith::AddFOp>(
+      arith::DivFOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
+  Value rhsRealImagDenom = arith::AddFOp::create(rewriter,
       loc, rhsIm,
-      rewriter.create<arith::MulFOp>(loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRealImagRatio, fmf),
+      arith::MulFOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
+  Value realNumerator1 = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
       lhsIm, fmf);
-  Value resultReal1 = rewriter.create<arith::DivFOp>(loc, realNumerator1,
+  Value resultReal1 = arith::DivFOp::create(rewriter, loc, realNumerator1,
                                                      rhsRealImagDenom, fmf);
-  Value imagNumerator1 = rewriter.create<arith::SubFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRealImagRatio, fmf),
+  Value imagNumerator1 = arith::SubFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
       lhsRe, fmf);
-  Value resultImag1 = rewriter.create<arith::DivFOp>(loc, imagNumerator1,
+  Value resultImag1 = arith::DivFOp::create(rewriter, loc, imagNumerator1,
                                                      rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
-      rewriter.create<arith::DivFOp>(loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = rewriter.create<arith::AddFOp>(
+      arith::DivFOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
+  Value rhsImagRealDenom = arith::AddFOp::create(rewriter,
       loc, rhsRe,
-      rewriter.create<arith::MulFOp>(loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = rewriter.create<arith::AddFOp>(
+      arith::MulFOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
+  Value realNumerator2 = arith::AddFOp::create(rewriter,
       loc, lhsRe,
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsImagRealRatio, fmf), fmf);
-  Value resultReal2 = rewriter.create<arith::DivFOp>(loc, realNumerator2,
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
+  Value resultReal2 = arith::DivFOp::create(rewriter, loc, realNumerator2,
                                                      rhsImagRealDenom, fmf);
-  Value imagNumerator2 = rewriter.create<arith::SubFOp>(
+  Value imagNumerator2 = arith::SubFOp::create(rewriter,
       loc, lhsIm,
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsImagRealRatio, fmf), fmf);
-  Value resultImag2 = rewriter.create<arith::DivFOp>(loc, imagNumerator2,
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
+  Value resultImag2 = arith::DivFOp::create(rewriter, loc, imagNumerator2,
                                                      rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
-  Value rhsRealAbs = rewriter.create<math::AbsFOp>(loc, rhsRe, fmf);
-  Value rhsRealIsZero = rewriter.create<arith::CmpFOp>(
+  Value rhsRealAbs = math::AbsFOp::create(rewriter, loc, rhsRe, fmf);
+  Value rhsRealIsZero = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsRealAbs, zero);
-  Value rhsImagAbs = rewriter.create<math::AbsFOp>(loc, rhsIm, fmf);
-  Value rhsImagIsZero = rewriter.create<arith::CmpFOp>(
+  Value rhsImagAbs = math::AbsFOp::create(rewriter, loc, rhsIm, fmf);
+  Value rhsImagIsZero = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsImagAbs, zero);
-  Value lhsRealIsNotNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsRealIsNotNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ORD, lhsRe, zero);
-  Value lhsImagIsNotNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsImagIsNotNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ORD, lhsIm, zero);
   Value lhsContainsNotNaNValue =
-      rewriter.create<arith::OrIOp>(loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = rewriter.create<arith::AndIOp>(
+      arith::OrIOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
+  Value resultIsInfinity = arith::AndIOp::create(rewriter,
       loc, lhsContainsNotNaNValue,
-      rewriter.create<arith::AndIOp>(loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = rewriter.create<arith::ConstantOp>(
+      arith::AndIOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
+  Value inf = arith::ConstantOp::create(rewriter,
       loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfRhsReal =
-      rewriter.create<math::CopySignOp>(loc, inf, rhsRe);
+      math::CopySignOp::create(rewriter, loc, inf, rhsRe);
   Value infinityResultReal =
-      rewriter.create<arith::MulFOp>(loc, infWithSignOfRhsReal, lhsRe, fmf);
+      arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsRe, fmf);
   Value infinityResultImag =
-      rewriter.create<arith::MulFOp>(loc, infWithSignOfRhsReal, lhsIm, fmf);
+      arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = rewriter.create<arith::CmpFOp>(
+  Value rhsRealFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, rhsRealAbs, inf);
-  Value rhsImagFinite = rewriter.create<arith::CmpFOp>(
+  Value rhsImagFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, rhsImagAbs, inf);
   Value rhsFinite =
-      rewriter.create<arith::AndIOp>(loc, rhsRealFinite, rhsImagFinite);
-  Value lhsRealAbs = rewriter.create<math::AbsFOp>(loc, lhsRe, fmf);
-  Value lhsRealInfinite = rewriter.create<arith::CmpFOp>(
+      arith::AndIOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
+  Value lhsRealAbs = math::AbsFOp::create(rewriter, loc, lhsRe, fmf);
+  Value lhsRealInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, lhsRealAbs, inf);
-  Value lhsImagAbs = rewriter.create<math::AbsFOp>(loc, lhsIm, fmf);
-  Value lhsImagInfinite = rewriter.create<arith::CmpFOp>(
+  Value lhsImagAbs = math::AbsFOp::create(rewriter, loc, lhsIm, fmf);
+  Value lhsImagInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, lhsImagAbs, inf);
   Value lhsInfinite =
-      rewriter.create<arith::OrIOp>(loc, lhsRealInfinite, lhsImagInfinite);
+      arith::OrIOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
-      rewriter.create<arith::AndIOp>(loc, lhsInfinite, rhsFinite);
-  Value one = rewriter.create<arith::ConstantOp>(
+      arith::AndIOp::create(rewriter, loc, lhsInfinite, rhsFinite);
+  Value one = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, lhsRealInfinite, one, zero),
+  Value lhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesRhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRealIsInfWithSign, rhsRe, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesRhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
+  Value resultReal3 = arith::MulFOp::create(rewriter,
       loc, inf,
-      rewriter.create<arith::AddFOp>(loc, lhsRealIsInfWithSignTimesRhsReal,
+      arith::AddFOp::create(rewriter, loc, lhsRealIsInfWithSignTimesRhsReal,
                                      lhsImagIsInfWithSignTimesRhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesRhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsRealIsInfWithSign, rhsIm, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesRhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
+  Value resultImag3 = arith::MulFOp::create(rewriter,
       loc, inf,
-      rewriter.create<arith::SubFOp>(loc, lhsImagIsInfWithSignTimesRhsReal,
+      arith::SubFOp::create(rewriter, loc, lhsImagIsInfWithSignTimesRhsReal,
                                      lhsRealIsInfWithSignTimesRhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = rewriter.create<arith::CmpFOp>(
+  Value lhsRealFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, lhsRealAbs, inf);
-  Value lhsImagFinite = rewriter.create<arith::CmpFOp>(
+  Value lhsImagFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, lhsImagAbs, inf);
   Value lhsFinite =
-      rewriter.create<arith::AndIOp>(loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = rewriter.create<arith::CmpFOp>(
+      arith::AndIOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
+  Value rhsRealInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsRealAbs, inf);
-  Value rhsImagInfinite = rewriter.create<arith::CmpFOp>(
+  Value rhsImagInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsImagAbs, inf);
   Value rhsInfinite =
-      rewriter.create<arith::OrIOp>(loc, rhsRealInfinite, rhsImagInfinite);
+      arith::OrIOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
-      rewriter.create<arith::AndIOp>(loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, rhsRealInfinite, one, zero),
+      arith::AndIOp::create(rewriter, loc, lhsFinite, rhsInfinite);
+  Value rhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimesLhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRealIsInfWithSign, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
+  Value resultReal4 = arith::MulFOp::create(rewriter,
       loc, zero,
-      rewriter.create<arith::AddFOp>(loc, rhsRealIsInfWithSignTimesLhsReal,
+      arith::AddFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsReal,
                                      rhsImagIsInfWithSignTimesLhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimesLhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRealIsInfWithSign, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
+  Value resultImag4 = arith::MulFOp::create(rewriter,
       loc, zero,
-      rewriter.create<arith::SubFOp>(loc, rhsRealIsInfWithSignTimesLhsImag,
+      arith::SubFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsImag,
                                      rhsImagIsInfWithSignTimesLhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = rewriter.create<arith::CmpFOp>(
+  Value realAbsSmallerThanImagAbs = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OLT, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = rewriter.create<arith::SelectOp>(
+  Value resultReal5 = arith::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = rewriter.create<arith::SelectOp>(
+  Value resultImag5 = arith::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase3 = arith::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase3 = arith::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase2 = arith::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase2 = arith::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase1 = arith::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase1 = arith::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = rewriter.create<arith::CmpFOp>(
+  Value resultRealIsNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::UNO, resultReal5, zero);
-  Value resultImagIsNaN = rewriter.create<arith::CmpFOp>(
+  Value resultImagIsNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::UNO, resultImag5, zero);
   Value resultIsNaN =
-      rewriter.create<arith::AndIOp>(loc, resultRealIsNaN, resultImagIsNaN);
+      arith::AndIOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = rewriter.create<arith::SelectOp>(
+  *resultRe = arith::SelectOp::create(rewriter,
       loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = rewriter.create<arith::SelectOp>(
+  *resultIm = arith::SelectOp::create(rewriter,
       loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
 }
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index e5e862315941d..a06935c4387c6 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -35,7 +35,7 @@ static constexpr unsigned kImaginaryPosInComplexNumberStruct = 1;
 
 ComplexStructBuilder ComplexStructBuilder::poison(OpBuilder &builder,
                                                   Location loc, Type type) {
-  Value val = builder.create<LLVM::PoisonOp>(loc, type);
+  Value val = LLVM::PoisonOp::create(builder, loc, type);
   return ComplexStructBuilder(val);
 }
 
@@ -79,9 +79,9 @@ struct AbsOpConversion : public ConvertOpToLLVMPattern<complex::AbsOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value sqNorm = rewriter.create<LLVM::FAddOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, real, real, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, imag, imag, fmf), fmf);
+    Value sqNorm = LLVM::FAddOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, real, real, fmf),
+        LLVM::FMulOp::create(rewriter, loc, imag, imag, fmf), fmf);
 
     rewriter.replaceOpWithNewOp<LLVM::SqrtOp>(op, sqNorm);
     return success();
@@ -192,9 +192,9 @@ struct AddOpConversion : public ConvertOpToLLVMPattern<complex::AddOp> {
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
     Value real =
-        rewriter.create<LLVM::FAddOp>(loc, arg.lhs.real(), arg.rhs.real(), fmf);
+        LLVM::FAddOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
     Value imag =
-        rewriter.create<LLVM::FAddOp>(loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+        LLVM::FAddOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
@@ -278,13 +278,13 @@ struct MulOpConversion : public ConvertOpToLLVMPattern<complex::MulOp> {
     Value lhsRe = arg.lhs.real();
     Value lhsIm = arg.lhs.imag();
 
-    Value real = rewriter.create<LLVM::FSubOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, rhsRe, lhsRe, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, rhsIm, lhsIm, fmf), fmf);
+    Value real = LLVM::FSubOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, lhsRe, fmf),
+        LLVM::FMulOp::create(rewriter, loc, rhsIm, lhsIm, fmf), fmf);
 
-    Value imag = rewriter.create<LLVM::FAddOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRe, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsIm, fmf), fmf);
+    Value imag = LLVM::FAddOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+        LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
@@ -314,9 +314,9 @@ struct SubOpConversion : public ConvertOpToLLVMPattern<complex::SubOp> {
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
     Value real =
-        rewriter.create<LLVM::FSubOp>(loc, arg.lhs.real(), arg.rhs.real(), fmf);
+        LLVM::FSubOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
     Value imag =
-        rewriter.create<LLVM::FSubOp>(loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+        LLVM::FSubOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
diff --git a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
index 56269d189873a..4800f65c675e6 100644
--- a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
+++ b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
@@ -84,7 +84,7 @@ LogicalResult ScalarOpToLibmCall<Op, TypeResolver>::matchAndRewrite(
     rewriter.setInsertionPointToStart(&module->getRegion(0).front());
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-    opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name,
+    opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
                                            opFunctionTy);
     opFunc.setPrivate();
   }
diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 0c832c452718b..d14e3888b118c 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -31,44 +31,44 @@ enum class AbsFn { abs, sqrt, rsqrt };
 // Returns the absolute value, its square root or its reciprocal square root.
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
                  ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
-  Value one = b.create<arith::ConstantOp>(real.getType(),
+  Value one = arith::ConstantOp::create(b, real.getType(),
                                           b.getFloatAttr(real.getType(), 1.0));
 
-  Value absReal = b.create<math::AbsFOp>(real, fmf);
-  Value absImag = b.create<math::AbsFOp>(imag, fmf);
+  Value absReal = math::AbsFOp::create(b, real, fmf);
+  Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-  Value max = b.create<arith::MaximumFOp>(absReal, absImag, fmf);
-  Value min = b.create<arith::MinimumFOp>(absReal, absImag, fmf);
+  Value max = arith::MaximumFOp::create(b, absReal, absImag, fmf);
+  Value min = arith::MinimumFOp::create(b, absReal, absImag, fmf);
 
   // The lowering below requires NaNs and infinities to work correctly.
   arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear(
       fmf, arith::FastMathFlags::nnan | arith::FastMathFlags::ninf);
-  Value ratio = b.create<arith::DivFOp>(min, max, fmfWithNaNInf);
-  Value ratioSq = b.create<arith::MulFOp>(ratio, ratio, fmfWithNaNInf);
-  Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmfWithNaNInf);
+  Value ratio = arith::DivFOp::create(b, min, max, fmfWithNaNInf);
+  Value ratioSq = arith::MulFOp::create(b, ratio, ratio, fmfWithNaNInf);
+  Value ratioSqPlusOne = arith::AddFOp::create(b, ratioSq, one, fmfWithNaNInf);
   Value result;
 
   if (fn == AbsFn::rsqrt) {
-    ratioSqPlusOne = b.create<math::RsqrtOp>(ratioSqPlusOne, fmfWithNaNInf);
-    min = b.create<math::RsqrtOp>(min, fmfWithNaNInf);
-    max = b.create<math::RsqrtOp>(max, fmfWithNaNInf);
+    ratioSqPlusOne = math::RsqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
+    min = math::RsqrtOp::create(b, min, fmfWithNaNInf);
+    max = math::RsqrtOp::create(b, max, fmfWithNaNInf);
   }
 
   if (fn == AbsFn::sqrt) {
-    Value quarter = b.create<arith::ConstantOp>(
+    Value quarter = arith::ConstantOp::create(b,
         real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
-    Value sqrt = b.create<math::SqrtOp>(max, fmfWithNaNInf);
-    Value p025 = b.create<math::PowFOp>(ratioSqPlusOne, quarter, fmfWithNaNInf);
-    result = b.create<arith::MulFOp>(sqrt, p025, fmfWithNaNInf);
+    Value sqrt = math::SqrtOp::create(b, max, fmfWithNaNInf);
+    Value p025 = math::PowFOp::create(b, ratioSqPlusOne, quarter, fmfWithNaNInf);
+    result = arith::MulFOp::create(b, sqrt, p025, fmfWithNaNInf);
   } else {
-    Value sqrt = b.create<math::SqrtOp>(ratioSqPlusOne, fmfWithNaNInf);
-    result = b.create<arith::MulFOp>(max, sqrt, fmfWithNaNInf);
+    Value sqrt = math::SqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
+    result = arith::MulFOp::create(b, max, sqrt, fmfWithNaNInf);
   }
 
-  Value isNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, result,
+  Value isNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, result,
                                         result, fmfWithNaNInf);
-  return b.create<arith::SelectOp>(isNaN, min, result);
+  return arith::SelectOp::create(b, isNaN, min, result);
 }
 
 struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
@@ -81,8 +81,8 @@ struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
 
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
     rewriter.replaceOp(op, computeAbs(real, imag, fmf, b));
 
     return success();
@@ -105,28 +105,28 @@ struct Atan2OpConversion : public OpConversionPattern<complex::Atan2Op> {
     Value lhs = adaptor.getLhs();
     Value rhs = adaptor.getRhs();
 
-    Value rhsSquared = b.create<complex::MulOp>(type, rhs, rhs, fmf);
-    Value lhsSquared = b.create<complex::MulOp>(type, lhs, lhs, fmf);
+    Value rhsSquared = complex::MulOp::create(b, type, rhs, rhs, fmf);
+    Value lhsSquared = complex::MulOp::create(b, type, lhs, lhs, fmf);
     Value rhsSquaredPlusLhsSquared =
-        b.create<complex::AddOp>(type, rhsSquared, lhsSquared, fmf);
+        complex::AddOp::create(b, type, rhsSquared, lhsSquared, fmf);
     Value sqrtOfRhsSquaredPlusLhsSquared =
-        b.create<complex::SqrtOp>(type, rhsSquaredPlusLhsSquared, fmf);
+        complex::SqrtOp::create(b, type, rhsSquaredPlusLhsSquared, fmf);
 
     Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
-    Value one = b.create<arith::ConstantOp>(elementType,
+        arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
+    Value one = arith::ConstantOp::create(b, elementType,
                                             b.getFloatAttr(elementType, 1));
-    Value i = b.create<complex::CreateOp>(type, zero, one);
-    Value iTimesLhs = b.create<complex::MulOp>(i, lhs, fmf);
-    Value rhsPlusILhs = b.create<complex::AddOp>(rhs, iTimesLhs, fmf);
+    Value i = complex::CreateOp::create(b, type, zero, one);
+    Value iTimesLhs = complex::MulOp::create(b, i, lhs, fmf);
+    Value rhsPlusILhs = complex::AddOp::create(b, rhs, iTimesLhs, fmf);
 
-    Value divResult = b.create<complex::DivOp>(
+    Value divResult = complex::DivOp::create(b,
         rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf);
-    Value logResult = b.create<complex::LogOp>(divResult, fmf);
+    Value logResult = complex::LogOp::create(b, divResult, fmf);
 
-    Value negativeOne = b.create<arith::ConstantOp>(
+    Value negativeOne = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -1));
-    Value negativeI = b.create<complex::CreateOp>(type, zero, negativeOne);
+    Value negativeI = complex::CreateOp::create(b, type, zero, negativeOne);
 
     rewriter.replaceOpWithNewOp<complex::MulOp>(op, negativeI, logResult, fmf);
     return success();
@@ -146,14 +146,14 @@ struct ComparisonOpConversion : public OpConversionPattern<ComparisonOp> {
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getLhs().getType()).getElementType();
 
-    Value realLhs = rewriter.create<complex::ReOp>(loc, type, adaptor.getLhs());
-    Value imagLhs = rewriter.create<complex::ImOp>(loc, type, adaptor.getLhs());
-    Value realRhs = rewriter.create<complex::ReOp>(loc, type, adaptor.getRhs());
-    Value imagRhs = rewriter.create<complex::ImOp>(loc, type, adaptor.getRhs());
+    Value realLhs = complex::ReOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value imagLhs = complex::ImOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value realRhs = complex::ReOp::create(rewriter, loc, type, adaptor.getRhs());
+    Value imagRhs = complex::ImOp::create(rewriter, loc, type, adaptor.getRhs());
     Value realComparison =
-        rewriter.create<arith::CmpFOp>(loc, p, realLhs, realRhs);
+        arith::CmpFOp::create(rewriter, loc, p, realLhs, realRhs);
     Value imagComparison =
-        rewriter.create<arith::CmpFOp>(loc, p, imagLhs, imagRhs);
+        arith::CmpFOp::create(rewriter, loc, p, imagLhs, imagRhs);
 
     rewriter.replaceOpWithNewOp<ResultCombiner>(op, realComparison,
                                                 imagComparison);
@@ -176,13 +176,13 @@ struct BinaryComplexOpConversion : public OpConversionPattern<BinaryComplexOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
-    Value realLhs = b.create<complex::ReOp>(elementType, adaptor.getLhs());
-    Value realRhs = b.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value resultReal = b.create<BinaryStandardOp>(elementType, realLhs, realRhs,
+    Value realLhs = complex::ReOp::create(b, elementType, adaptor.getLhs());
+    Value realRhs = complex::ReOp::create(b, elementType, adaptor.getRhs());
+    Value resultReal = BinaryStandardOp::create(b, elementType, realLhs, realRhs,
                                                   fmf.getValue());
-    Value imagLhs = b.create<complex::ImOp>(elementType, adaptor.getLhs());
-    Value imagRhs = b.create<complex::ImOp>(elementType, adaptor.getRhs());
-    Value resultImag = b.create<BinaryStandardOp>(elementType, imagLhs, imagRhs,
+    Value imagLhs = complex::ImOp::create(b, elementType, adaptor.getLhs());
+    Value imagRhs = complex::ImOp::create(b, elementType, adaptor.getRhs());
+    Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs, imagRhs,
                                                   fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -205,20 +205,20 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
 
     // Trigonometric ops use a set of common building blocks to convert to real
     // ops. Here we create these building blocks and call into an op-specific
     // implementation in the subclass to combine them.
-    Value half = rewriter.create<arith::ConstantOp>(
+    Value half = arith::ConstantOp::create(rewriter,
         loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
-    Value exp = rewriter.create<math::ExpOp>(loc, imag, fmf);
-    Value scaledExp = rewriter.create<arith::MulFOp>(loc, half, exp, fmf);
-    Value reciprocalExp = rewriter.create<arith::DivFOp>(loc, half, exp, fmf);
-    Value sin = rewriter.create<math::SinOp>(loc, real, fmf);
-    Value cos = rewriter.create<math::CosOp>(loc, real, fmf);
+    Value exp = math::ExpOp::create(rewriter, loc, imag, fmf);
+    Value scaledExp = arith::MulFOp::create(rewriter, loc, half, exp, fmf);
+    Value reciprocalExp = arith::DivFOp::create(rewriter, loc, half, exp, fmf);
+    Value sin = math::SinOp::create(rewriter, loc, real, fmf);
+    Value cos = math::CosOp::create(rewriter, loc, real, fmf);
 
     auto resultPair =
         combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf);
@@ -251,11 +251,11 @@ struct CosOpConversion : public TrigonometricOpConversion<complex::CosOp> {
     //   Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x
     //   Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x
     Value sum =
-        rewriter.create<arith::AddFOp>(loc, reciprocalExp, scaledExp, fmf);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, cos, fmf);
+        arith::AddFOp::create(rewriter, loc, reciprocalExp, scaledExp, fmf);
+    Value resultReal = arith::MulFOp::create(rewriter, loc, sum, cos, fmf);
     Value diff =
-        rewriter.create<arith::SubFOp>(loc, reciprocalExp, scaledExp, fmf);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, sin, fmf);
+        arith::SubFOp::create(rewriter, loc, reciprocalExp, scaledExp, fmf);
+    Value resultImag = arith::MulFOp::create(rewriter, loc, diff, sin, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -275,13 +275,13 @@ struct DivOpConversion : public OpConversionPattern<complex::DivOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value lhsReal =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getLhs());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getLhs());
     Value lhsImag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getLhs());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getLhs());
     Value rhsReal =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getRhs());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getRhs());
     Value rhsImag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getRhs());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getRhs());
 
     Value resultReal, resultImag;
 
@@ -318,16 +318,16 @@ struct ExpOpConversion : public OpConversionPattern<complex::ExpOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value expReal = rewriter.create<math::ExpOp>(loc, real, fmf.getValue());
-    Value cosImag = rewriter.create<math::CosOp>(loc, imag, fmf.getValue());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value expReal = math::ExpOp::create(rewriter, loc, real, fmf.getValue());
+    Value cosImag = math::CosOp::create(rewriter, loc, imag, fmf.getValue());
     Value resultReal =
-        rewriter.create<arith::MulFOp>(loc, expReal, cosImag, fmf.getValue());
-    Value sinImag = rewriter.create<math::SinOp>(loc, imag, fmf.getValue());
+        arith::MulFOp::create(rewriter, loc, expReal, cosImag, fmf.getValue());
+    Value sinImag = math::SinOp::create(rewriter, loc, imag, fmf.getValue());
     Value resultImag =
-        rewriter.create<arith::MulFOp>(loc, expReal, sinImag, fmf.getValue());
+        arith::MulFOp::create(rewriter, loc, expReal, sinImag, fmf.getValue());
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -340,11 +340,11 @@ Value evaluatePolynomial(ImplicitLocOpBuilder &b, Value arg,
                          arith::FastMathFlagsAttr fmf) {
   auto argType = mlir::cast<FloatType>(arg.getType());
   Value poly =
-      b.create<arith::ConstantOp>(b.getFloatAttr(argType, coefficients[0]));
+      arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[0]));
   for (unsigned i = 1; i < coefficients.size(); ++i) {
-    poly = b.create<math::FmaOp>(
+    poly = math::FmaOp::create(b,
         poly, arg,
-        b.create<arith::ConstantOp>(b.getFloatAttr(argType, coefficients[i])),
+        arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[i])),
         fmf);
   }
   return poly;
@@ -365,26 +365,26 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
 
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     ImplicitLocOpBuilder b(op.getLoc(), rewriter);
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
-    Value zero = b.create<arith::ConstantOp>(b.getFloatAttr(elemType, 0.0));
-    Value one = b.create<arith::ConstantOp>(b.getFloatAttr(elemType, 1.0));
+    Value zero = arith::ConstantOp::create(b, b.getFloatAttr(elemType, 0.0));
+    Value one = arith::ConstantOp::create(b, b.getFloatAttr(elemType, 1.0));
 
-    Value expm1Real = b.create<math::ExpM1Op>(real, fmf);
-    Value expReal = b.create<arith::AddFOp>(expm1Real, one, fmf);
+    Value expm1Real = math::ExpM1Op::create(b, real, fmf);
+    Value expReal = arith::AddFOp::create(b, expm1Real, one, fmf);
 
-    Value sinImag = b.create<math::SinOp>(imag, fmf);
+    Value sinImag = math::SinOp::create(b, imag, fmf);
     Value cosm1Imag = emitCosm1(imag, fmf, b);
-    Value cosImag = b.create<arith::AddFOp>(cosm1Imag, one, fmf);
+    Value cosImag = arith::AddFOp::create(b, cosm1Imag, one, fmf);
 
-    Value realResult = b.create<arith::AddFOp>(
-        b.create<arith::MulFOp>(expm1Real, cosImag, fmf), cosm1Imag, fmf);
+    Value realResult = arith::AddFOp::create(b,
+        arith::MulFOp::create(b, expm1Real, cosImag, fmf), cosm1Imag, fmf);
 
-    Value imagIsZero = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag,
+    Value imagIsZero = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag,
                                                zero, fmf.getValue());
-    Value imagResult = b.create<arith::SelectOp>(
-        imagIsZero, zero, b.create<arith::MulFOp>(expReal, sinImag, fmf));
+    Value imagResult = arith::SelectOp::create(b,
+        imagIsZero, zero, arith::MulFOp::create(b, expReal, sinImag, fmf));
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, realResult,
                                                    imagResult);
@@ -395,8 +395,8 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
   Value emitCosm1(Value arg, arith::FastMathFlagsAttr fmf,
                   ImplicitLocOpBuilder &b) const {
     auto argType = mlir::cast<FloatType>(arg.getType());
-    auto negHalf = b.create<arith::ConstantOp>(b.getFloatAttr(argType, -0.5));
-    auto negOne = b.create<arith::ConstantOp>(b.getFloatAttr(argType, -1.0));
+    auto negHalf = arith::ConstantOp::create(b, b.getFloatAttr(argType, -0.5));
+    auto negOne = arith::ConstantOp::create(b, b.getFloatAttr(argType, -1.0));
 
     // Algorithm copied from cephes cosm1.
     SmallVector<double, 7> kCoeffs{
@@ -405,23 +405,23 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
         2.4801587301570552304991E-5,  -1.3888888888888872993737E-3,
         4.1666666666666666609054E-2,
     };
-    Value cos = b.create<math::CosOp>(arg, fmf);
-    Value forLargeArg = b.create<arith::AddFOp>(cos, negOne, fmf);
+    Value cos = math::CosOp::create(b, arg, fmf);
+    Value forLargeArg = arith::AddFOp::create(b, cos, negOne, fmf);
 
-    Value argPow2 = b.create<arith::MulFOp>(arg, arg, fmf);
-    Value argPow4 = b.create<arith::MulFOp>(argPow2, argPow2, fmf);
+    Value argPow2 = arith::MulFOp::create(b, arg, arg, fmf);
+    Value argPow4 = arith::MulFOp::create(b, argPow2, argPow2, fmf);
     Value poly = evaluatePolynomial(b, argPow2, kCoeffs, fmf);
 
     auto forSmallArg =
-        b.create<arith::AddFOp>(b.create<arith::MulFOp>(argPow4, poly, fmf),
-                                b.create<arith::MulFOp>(negHalf, argPow2, fmf));
+        arith::AddFOp::create(b, arith::MulFOp::create(b, argPow4, poly, fmf),
+                                arith::MulFOp::create(b, negHalf, argPow2, fmf));
 
     // (pi/4)^2 is approximately 0.61685
     Value piOver4Pow2 =
-        b.create<arith::ConstantOp>(b.getFloatAttr(argType, 0.61685));
-    Value cond = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, argPow2,
+        arith::ConstantOp::create(b, b.getFloatAttr(argType, 0.61685));
+    Value cond = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, argPow2,
                                          piOver4Pow2, fmf.getValue());
-    return b.create<arith::SelectOp>(cond, forLargeArg, forSmallArg);
+    return arith::SelectOp::create(b, cond, forLargeArg, forSmallArg);
   }
 };
 
@@ -436,13 +436,13 @@ struct LogOpConversion : public OpConversionPattern<complex::LogOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
-    Value abs = b.create<complex::AbsOp>(elementType, adaptor.getComplex(),
+    Value abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(),
                                          fmf.getValue());
-    Value resultReal = b.create<math::LogOp>(elementType, abs, fmf.getValue());
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value resultReal = math::LogOp::create(b, elementType, abs, fmf.getValue());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value resultImag =
-        b.create<math::Atan2Op>(elementType, imag, real, fmf.getValue());
+        math::Atan2Op::create(b, elementType, imag, real, fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -460,40 +460,40 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
-    Value half = b.create<arith::ConstantOp>(elementType,
+    Value half = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 0.5));
-    Value one = b.create<arith::ConstantOp>(elementType,
+    Value one = arith::ConstantOp::create(b, elementType,
                                             b.getFloatAttr(elementType, 1));
-    Value realPlusOne = b.create<arith::AddFOp>(real, one, fmf);
-    Value absRealPlusOne = b.create<math::AbsFOp>(realPlusOne, fmf);
-    Value absImag = b.create<math::AbsFOp>(imag, fmf);
+    Value realPlusOne = arith::AddFOp::create(b, real, one, fmf);
+    Value absRealPlusOne = math::AbsFOp::create(b, realPlusOne, fmf);
+    Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-    Value maxAbs = b.create<arith::MaximumFOp>(absRealPlusOne, absImag, fmf);
-    Value minAbs = b.create<arith::MinimumFOp>(absRealPlusOne, absImag, fmf);
+    Value maxAbs = arith::MaximumFOp::create(b, absRealPlusOne, absImag, fmf);
+    Value minAbs = arith::MinimumFOp::create(b, absRealPlusOne, absImag, fmf);
 
-    Value useReal = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGT,
+    Value useReal = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT,
                                             realPlusOne, absImag, fmf);
-    Value maxMinusOne = b.create<arith::SubFOp>(maxAbs, one, fmf);
+    Value maxMinusOne = arith::SubFOp::create(b, maxAbs, one, fmf);
     Value maxAbsOfRealPlusOneAndImagMinusOne =
-        b.create<arith::SelectOp>(useReal, real, maxMinusOne);
+        arith::SelectOp::create(b, useReal, real, maxMinusOne);
     arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear(
         fmf, arith::FastMathFlags::nnan | arith::FastMathFlags::ninf);
-    Value minMaxRatio = b.create<arith::DivFOp>(minAbs, maxAbs, fmfWithNaNInf);
+    Value minMaxRatio = arith::DivFOp::create(b, minAbs, maxAbs, fmfWithNaNInf);
     Value logOfMaxAbsOfRealPlusOneAndImag =
-        b.create<math::Log1pOp>(maxAbsOfRealPlusOneAndImagMinusOne, fmf);
-    Value logOfSqrtPart = b.create<math::Log1pOp>(
-        b.create<arith::MulFOp>(minMaxRatio, minMaxRatio, fmfWithNaNInf),
+        math::Log1pOp::create(b, maxAbsOfRealPlusOneAndImagMinusOne, fmf);
+    Value logOfSqrtPart = math::Log1pOp::create(b,
+        arith::MulFOp::create(b, minMaxRatio, minMaxRatio, fmfWithNaNInf),
         fmfWithNaNInf);
-    Value r = b.create<arith::AddFOp>(
-        b.create<arith::MulFOp>(half, logOfSqrtPart, fmfWithNaNInf),
+    Value r = arith::AddFOp::create(b,
+        arith::MulFOp::create(b, half, logOfSqrtPart, fmfWithNaNInf),
         logOfMaxAbsOfRealPlusOneAndImag, fmfWithNaNInf);
-    Value resultReal = b.create<arith::SelectOp>(
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
+    Value resultReal = arith::SelectOp::create(b,
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
         minAbs, r);
-    Value resultImag = b.create<math::Atan2Op>(imag, realPlusOne, fmf);
+    Value resultImag = math::Atan2Op::create(b, imag, realPlusOne, fmf);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -511,21 +511,21 @@ struct MulOpConversion : public OpConversionPattern<complex::MulOp> {
     auto elementType = cast<FloatType>(type.getElementType());
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     auto fmfValue = fmf.getValue();
-    Value lhsReal = b.create<complex::ReOp>(elementType, adaptor.getLhs());
-    Value lhsImag = b.create<complex::ImOp>(elementType, adaptor.getLhs());
-    Value rhsReal = b.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value rhsImag = b.create<complex::ImOp>(elementType, adaptor.getRhs());
+    Value lhsReal = complex::ReOp::create(b, elementType, adaptor.getLhs());
+    Value lhsImag = complex::ImOp::create(b, elementType, adaptor.getLhs());
+    Value rhsReal = complex::ReOp::create(b, elementType, adaptor.getRhs());
+    Value rhsImag = complex::ImOp::create(b, elementType, adaptor.getRhs());
     Value lhsRealTimesRhsReal =
-        b.create<arith::MulFOp>(lhsReal, rhsReal, fmfValue);
+        arith::MulFOp::create(b, lhsReal, rhsReal, fmfValue);
     Value lhsImagTimesRhsImag =
-        b.create<arith::MulFOp>(lhsImag, rhsImag, fmfValue);
-    Value real = b.create<arith::SubFOp>(lhsRealTimesRhsReal,
+        arith::MulFOp::create(b, lhsImag, rhsImag, fmfValue);
+    Value real = arith::SubFOp::create(b, lhsRealTimesRhsReal,
                                          lhsImagTimesRhsImag, fmfValue);
     Value lhsImagTimesRhsReal =
-        b.create<arith::MulFOp>(lhsImag, rhsReal, fmfValue);
+        arith::MulFOp::create(b, lhsImag, rhsReal, fmfValue);
     Value lhsRealTimesRhsImag =
-        b.create<arith::MulFOp>(lhsReal, rhsImag, fmfValue);
-    Value imag = b.create<arith::AddFOp>(lhsImagTimesRhsReal,
+        arith::MulFOp::create(b, lhsReal, rhsImag, fmfValue);
+    Value imag = arith::AddFOp::create(b, lhsImagTimesRhsReal,
                                          lhsRealTimesRhsImag, fmfValue);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, imag);
     return success();
@@ -543,11 +543,11 @@ struct NegOpConversion : public OpConversionPattern<complex::NegOp> {
     auto elementType = cast<FloatType>(type.getElementType());
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negReal = rewriter.create<arith::NegFOp>(loc, real);
-    Value negImag = rewriter.create<arith::NegFOp>(loc, imag);
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value negReal = arith::NegFOp::create(rewriter, loc, real);
+    Value negImag = arith::NegFOp::create(rewriter, loc, imag);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, negReal, negImag);
     return success();
   }
@@ -570,11 +570,11 @@ struct SinOpConversion : public TrigonometricOpConversion<complex::SinOp> {
     //   Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x
     //   Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x
     Value sum =
-        rewriter.create<arith::AddFOp>(loc, scaledExp, reciprocalExp, fmf);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, sin, fmf);
+        arith::AddFOp::create(rewriter, loc, scaledExp, reciprocalExp, fmf);
+    Value resultReal = arith::MulFOp::create(rewriter, loc, sum, sin, fmf);
     Value diff =
-        rewriter.create<arith::SubFOp>(loc, scaledExp, reciprocalExp, fmf);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, cos, fmf);
+        arith::SubFOp::create(rewriter, loc, scaledExp, reciprocalExp, fmf);
+    Value resultImag = arith::MulFOp::create(rewriter, loc, diff, cos, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -593,64 +593,64 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
-    Value half = b.create<arith::ConstantOp>(elementType,
+    Value half = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 0.5));
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value absSqrt = computeAbs(real, imag, fmf, b, AbsFn::sqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value sqrtArg = b.create<arith::MulFOp>(argArg, half, fmf);
-    Value cos = b.create<math::CosOp>(sqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(sqrtArg, fmf);
+    Value argArg = math::Atan2Op::create(b, imag, real, fmf);
+    Value sqrtArg = arith::MulFOp::create(b, argArg, half, fmf);
+    Value cos = math::CosOp::create(b, sqrtArg, fmf);
+    Value sin = math::SinOp::create(b, sqrtArg, fmf);
     // sin(atan2(0, inf)) = 0, sqrt(abs(inf)) = inf, but we can't multiply
     // 0 * inf.
     Value sinIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, sin, zero, fmf);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, sin, zero, fmf);
 
-    Value resultReal = b.create<arith::MulFOp>(absSqrt, cos, fmf);
-    Value resultImag = b.create<arith::SelectOp>(
-        sinIsZero, zero, b.create<arith::MulFOp>(absSqrt, sin, fmf));
+    Value resultReal = arith::MulFOp::create(b, absSqrt, cos, fmf);
+    Value resultImag = arith::SelectOp::create(b,
+        sinIsZero, zero, arith::MulFOp::create(b, absSqrt, sin, fmf));
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
       Value inf = cst(APFloat::getInf(floatSemantics));
       Value negInf = cst(APFloat::getInf(floatSemantics, true));
       Value nan = cst(APFloat::getNaN(floatSemantics));
-      Value absImag = b.create<math::AbsFOp>(elementType, imag, fmf);
+      Value absImag = math::AbsFOp::create(b, elementType, imag, fmf);
 
       Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
       Value absImagIsNotInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::ONE, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::ONE, absImag, inf, fmf);
       Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, inf, fmf);
       Value realIsNegInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, negInf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, negInf, fmf);
 
-      resultReal = b.create<arith::SelectOp>(
-          b.create<arith::AndIOp>(realIsNegInf, absImagIsNotInf), zero,
+      resultReal = arith::SelectOp::create(b,
+          arith::AndIOp::create(b, realIsNegInf, absImagIsNotInf), zero,
           resultReal);
-      resultReal = b.create<arith::SelectOp>(
-          b.create<arith::OrIOp>(absImagIsInf, realIsInf), inf, resultReal);
+      resultReal = arith::SelectOp::create(b,
+          arith::OrIOp::create(b, absImagIsInf, realIsInf), inf, resultReal);
 
-      Value imagSignInf = b.create<math::CopySignOp>(inf, imag, fmf);
-      resultImag = b.create<arith::SelectOp>(
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, absSqrt, absSqrt),
+      Value imagSignInf = math::CopySignOp::create(b, inf, imag, fmf);
+      resultImag = arith::SelectOp::create(b,
+          arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, absSqrt, absSqrt),
           nan, resultImag);
-      resultImag = b.create<arith::SelectOp>(
-          b.create<arith::OrIOp>(absImagIsInf, realIsNegInf), imagSignInf,
+      resultImag = arith::SelectOp::create(b,
+          arith::OrIOp::create(b, absImagIsInf, realIsNegInf), imagSignInf,
           resultImag);
     }
 
     Value resultIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
-    resultReal = b.create<arith::SelectOp>(resultIsZero, zero, resultReal);
-    resultImag = b.create<arith::SelectOp>(resultIsZero, zero, resultImag);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
+    resultReal = arith::SelectOp::create(b, resultIsZero, zero, resultReal);
+    resultImag = arith::SelectOp::create(b, resultIsZero, zero, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -669,19 +669,19 @@ struct SignOpConversion : public OpConversionPattern<complex::SignOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
+        arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
     Value realIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, zero);
     Value imagIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero);
-    Value isZero = b.create<arith::AndIOp>(realIsZero, imagIsZero);
-    auto abs = b.create<complex::AbsOp>(elementType, adaptor.getComplex(), fmf);
-    Value realSign = b.create<arith::DivFOp>(real, abs, fmf);
-    Value imagSign = b.create<arith::DivFOp>(imag, abs, fmf);
-    Value sign = b.create<complex::CreateOp>(type, realSign, imagSign);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero);
+    Value isZero = arith::AndIOp::create(b, realIsZero, imagIsZero);
+    auto abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(), fmf);
+    Value realSign = arith::DivFOp::create(b, real, abs, fmf);
+    Value imagSign = arith::DivFOp::create(b, imag, abs, fmf);
+    Value sign = complex::CreateOp::create(b, type, realSign, imagSign);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isZero,
                                                  adaptor.getComplex(), sign);
     return success();
@@ -703,84 +703,84 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
     const auto &floatSemantics = elementType.getFloatSemantics();
 
     Value real =
-        b.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(b, loc, elementType, adaptor.getComplex());
     Value imag =
-        b.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negOne = b.create<arith::ConstantOp>(
+        complex::ImOp::create(b, loc, elementType, adaptor.getComplex());
+    Value negOne = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -1.0));
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
       std::swap(real, imag);
-      real = b.create<arith::MulFOp>(real, negOne, fmf);
+      real = arith::MulFOp::create(b, real, negOne, fmf);
     }
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value four = b.create<arith::ConstantOp>(elementType,
+    Value four = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 4.0));
-    Value twoReal = b.create<arith::AddFOp>(real, real, fmf);
-    Value negTwoReal = b.create<arith::MulFOp>(negOne, twoReal, fmf);
+    Value twoReal = arith::AddFOp::create(b, real, real, fmf);
+    Value negTwoReal = arith::MulFOp::create(b, negOne, twoReal, fmf);
 
-    Value expTwoRealMinusOne = b.create<math::ExpM1Op>(twoReal, fmf);
-    Value expNegTwoRealMinusOne = b.create<math::ExpM1Op>(negTwoReal, fmf);
+    Value expTwoRealMinusOne = math::ExpM1Op::create(b, twoReal, fmf);
+    Value expNegTwoRealMinusOne = math::ExpM1Op::create(b, negTwoReal, fmf);
     Value realNum =
-        b.create<arith::SubFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+        arith::SubFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
 
-    Value cosImag = b.create<math::CosOp>(imag, fmf);
-    Value cosImagSq = b.create<arith::MulFOp>(cosImag, cosImag, fmf);
-    Value twoCosTwoImagPlusOne = b.create<arith::MulFOp>(cosImagSq, four, fmf);
-    Value sinImag = b.create<math::SinOp>(imag, fmf);
+    Value cosImag = math::CosOp::create(b, imag, fmf);
+    Value cosImagSq = arith::MulFOp::create(b, cosImag, cosImag, fmf);
+    Value twoCosTwoImagPlusOne = arith::MulFOp::create(b, cosImagSq, four, fmf);
+    Value sinImag = math::SinOp::create(b, imag, fmf);
 
-    Value imagNum = b.create<arith::MulFOp>(
-        four, b.create<arith::MulFOp>(cosImag, sinImag, fmf), fmf);
+    Value imagNum = arith::MulFOp::create(b,
+        four, arith::MulFOp::create(b, cosImag, sinImag, fmf), fmf);
 
     Value expSumMinusTwo =
-        b.create<arith::AddFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+        arith::AddFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
     Value denom =
-        b.create<arith::AddFOp>(expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
+        arith::AddFOp::create(b, expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
 
-    Value isInf = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+    Value isInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
                                           expSumMinusTwo, inf, fmf);
-    Value realLimit = b.create<math::CopySignOp>(negOne, real, fmf);
+    Value realLimit = math::CopySignOp::create(b, negOne, real, fmf);
 
-    Value resultReal = b.create<arith::SelectOp>(
-        isInf, realLimit, b.create<arith::DivFOp>(realNum, denom, fmf));
-    Value resultImag = b.create<arith::DivFOp>(imagNum, denom, fmf);
+    Value resultReal = arith::SelectOp::create(b,
+        isInf, realLimit, arith::DivFOp::create(b, realNum, denom, fmf));
+    Value resultImag = arith::DivFOp::create(b, imagNum, denom, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value zero = b.create<arith::ConstantOp>(
+      Value absReal = math::AbsFOp::create(b, real, fmf);
+      Value zero = arith::ConstantOp::create(b,
           elementType, b.getFloatAttr(elementType, 0.0));
       Value nan = cst(APFloat::getNaN(floatSemantics));
 
       Value absRealIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
       Value imagIsZero =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-      Value absRealIsNotInf = b.create<arith::XOrIOp>(
-          absRealIsInf, b.create<arith::ConstantIntOp>(true, /*width=*/1));
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
+      Value absRealIsNotInf = arith::XOrIOp::create(b,
+          absRealIsInf, arith::ConstantIntOp::create(b, true, /*width=*/1));
 
-      Value imagNumIsNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO,
+      Value imagNumIsNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO,
                                                    imagNum, imagNum, fmf);
       Value resultRealIsNaN =
-          b.create<arith::AndIOp>(imagNumIsNaN, absRealIsNotInf);
-      Value resultImagIsZero = b.create<arith::OrIOp>(
-          imagIsZero, b.create<arith::AndIOp>(absRealIsInf, imagNumIsNaN));
+          arith::AndIOp::create(b, imagNumIsNaN, absRealIsNotInf);
+      Value resultImagIsZero = arith::OrIOp::create(b,
+          imagIsZero, arith::AndIOp::create(b, absRealIsInf, imagNumIsNaN));
 
-      resultReal = b.create<arith::SelectOp>(resultRealIsNaN, nan, resultReal);
+      resultReal = arith::SelectOp::create(b, resultRealIsNaN, nan, resultReal);
       resultImag =
-          b.create<arith::SelectOp>(resultImagIsZero, zero, resultImag);
+          arith::SelectOp::create(b, resultImagIsZero, zero, resultImag);
     }
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
       std::swap(resultReal, resultImag);
-      resultImag = b.create<arith::MulFOp>(resultImag, negOne, fmf);
+      resultImag = arith::MulFOp::create(b, resultImag, negOne, fmf);
     }
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
@@ -799,10 +799,10 @@ struct ConjOpConversion : public OpConversionPattern<complex::ConjOp> {
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negImag = rewriter.create<arith::NegFOp>(loc, elementType, imag);
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value negImag = arith::NegFOp::create(rewriter, loc, elementType, imag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, negImag);
 
@@ -818,97 +818,97 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
                                  arith::FastMathFlags fmf) {
   auto elementType = cast<FloatType>(type.getElementType());
 
-  Value a = builder.create<complex::ReOp>(lhs);
-  Value b = builder.create<complex::ImOp>(lhs);
+  Value a = complex::ReOp::create(builder, lhs);
+  Value b = complex::ImOp::create(builder, lhs);
 
-  Value abs = builder.create<complex::AbsOp>(lhs, fmf);
-  Value absToC = builder.create<math::PowFOp>(abs, c, fmf);
+  Value abs = complex::AbsOp::create(builder, lhs, fmf);
+  Value absToC = math::PowFOp::create(builder, abs, c, fmf);
 
-  Value negD = builder.create<arith::NegFOp>(d, fmf);
-  Value argLhs = builder.create<math::Atan2Op>(b, a, fmf);
-  Value negDArgLhs = builder.create<arith::MulFOp>(negD, argLhs, fmf);
-  Value expNegDArgLhs = builder.create<math::ExpOp>(negDArgLhs, fmf);
+  Value negD = arith::NegFOp::create(builder, d, fmf);
+  Value argLhs = math::Atan2Op::create(builder, b, a, fmf);
+  Value negDArgLhs = arith::MulFOp::create(builder, negD, argLhs, fmf);
+  Value expNegDArgLhs = math::ExpOp::create(builder, negDArgLhs, fmf);
 
-  Value coeff = builder.create<arith::MulFOp>(absToC, expNegDArgLhs, fmf);
-  Value lnAbs = builder.create<math::LogOp>(abs, fmf);
-  Value cArgLhs = builder.create<arith::MulFOp>(c, argLhs, fmf);
-  Value dLnAbs = builder.create<arith::MulFOp>(d, lnAbs, fmf);
-  Value q = builder.create<arith::AddFOp>(cArgLhs, dLnAbs, fmf);
-  Value cosQ = builder.create<math::CosOp>(q, fmf);
-  Value sinQ = builder.create<math::SinOp>(q, fmf);
+  Value coeff = arith::MulFOp::create(builder, absToC, expNegDArgLhs, fmf);
+  Value lnAbs = math::LogOp::create(builder, abs, fmf);
+  Value cArgLhs = arith::MulFOp::create(builder, c, argLhs, fmf);
+  Value dLnAbs = arith::MulFOp::create(builder, d, lnAbs, fmf);
+  Value q = arith::AddFOp::create(builder, cArgLhs, dLnAbs, fmf);
+  Value cosQ = math::CosOp::create(builder, q, fmf);
+  Value sinQ = math::SinOp::create(builder, q, fmf);
 
-  Value inf = builder.create<arith::ConstantOp>(
+  Value inf = arith::ConstantOp::create(builder,
       elementType,
       builder.getFloatAttr(elementType,
                            APFloat::getInf(elementType.getFloatSemantics())));
-  Value zero = builder.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(builder,
       elementType, builder.getFloatAttr(elementType, 0.0));
-  Value one = builder.create<arith::ConstantOp>(
+  Value one = arith::ConstantOp::create(builder,
       elementType, builder.getFloatAttr(elementType, 1.0));
-  Value complexOne = builder.create<complex::CreateOp>(type, one, zero);
-  Value complexZero = builder.create<complex::CreateOp>(type, zero, zero);
-  Value complexInf = builder.create<complex::CreateOp>(type, inf, zero);
+  Value complexOne = complex::CreateOp::create(builder, type, one, zero);
+  Value complexZero = complex::CreateOp::create(builder, type, zero, zero);
+  Value complexInf = complex::CreateOp::create(builder, type, inf, zero);
 
   // Case 0:
   // d^c is 0 if d is 0 and c > 0. 0^0 is defined to be 1.0, see
   // Branch Cuts for Complex Elementary Functions or Much Ado About
   // Nothing's Sign Bit, W. Kahan, Section 10.
   Value absEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, abs, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, abs, zero, fmf);
   Value dEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, d, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, d, zero, fmf);
   Value cEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, c, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, c, zero, fmf);
   Value bEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, b, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, b, zero, fmf);
 
   Value zeroLeC =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLE, zero, c, fmf);
-  Value coeffCosQ = builder.create<arith::MulFOp>(coeff, cosQ, fmf);
-  Value coeffSinQ = builder.create<arith::MulFOp>(coeff, sinQ, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLE, zero, c, fmf);
+  Value coeffCosQ = arith::MulFOp::create(builder, coeff, cosQ, fmf);
+  Value coeffSinQ = arith::MulFOp::create(builder, coeff, sinQ, fmf);
   Value complexOneOrZero =
-      builder.create<arith::SelectOp>(cEqZero, complexOne, complexZero);
+      arith::SelectOp::create(builder, cEqZero, complexOne, complexZero);
   Value coeffCosSin =
-      builder.create<complex::CreateOp>(type, coeffCosQ, coeffSinQ);
-  Value cutoff0 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(
-          builder.create<arith::AndIOp>(absEqZero, dEqZero), zeroLeC),
+      complex::CreateOp::create(builder, type, coeffCosQ, coeffSinQ);
+  Value cutoff0 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder,
+          arith::AndIOp::create(builder, absEqZero, dEqZero), zeroLeC),
       complexOneOrZero, coeffCosSin);
 
   // Case 1:
   // x^0 is defined to be 1 for any x, see
   // Branch Cuts for Complex Elementary Functions or Much Ado About
   // Nothing's Sign Bit, W. Kahan, Section 10.
-  Value rhsEqZero = builder.create<arith::AndIOp>(cEqZero, dEqZero);
+  Value rhsEqZero = arith::AndIOp::create(builder, cEqZero, dEqZero);
   Value cutoff1 =
-      builder.create<arith::SelectOp>(rhsEqZero, complexOne, cutoff0);
+      arith::SelectOp::create(builder, rhsEqZero, complexOne, cutoff0);
 
   // Case 2:
   // 1^(c + d*i) = 1 + 0*i
-  Value lhsEqOne = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, one, fmf),
+  Value lhsEqOne = arith::AndIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, one, fmf),
       bEqZero);
   Value cutoff2 =
-      builder.create<arith::SelectOp>(lhsEqOne, complexOne, cutoff1);
+      arith::SelectOp::create(builder, lhsEqOne, complexOne, cutoff1);
 
   // Case 3:
   // inf^(c + 0*i) = inf + 0*i, c > 0
-  Value lhsEqInf = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, inf, fmf),
+  Value lhsEqInf = arith::AndIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, inf, fmf),
       bEqZero);
-  Value rhsGt0 = builder.create<arith::AndIOp>(
+  Value rhsGt0 = arith::AndIOp::create(builder,
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, c, zero, fmf));
-  Value cutoff3 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(lhsEqInf, rhsGt0), complexInf, cutoff2);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, c, zero, fmf));
+  Value cutoff3 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder, lhsEqInf, rhsGt0), complexInf, cutoff2);
 
   // Case 4:
   // inf^(c + 0*i) = 0 + 0*i, c < 0
-  Value rhsLt0 = builder.create<arith::AndIOp>(
+  Value rhsLt0 = arith::AndIOp::create(builder,
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, c, zero, fmf));
-  Value cutoff4 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(lhsEqInf, rhsLt0), complexZero, cutoff3);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, c, zero, fmf));
+  Value cutoff4 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder, lhsEqInf, rhsLt0), complexZero, cutoff3);
 
   return cutoff4;
 }
@@ -923,8 +923,8 @@ struct PowOpConversion : public OpConversionPattern<complex::PowOp> {
     auto type = cast<ComplexType>(adaptor.getLhs().getType());
     auto elementType = cast<FloatType>(type.getElementType());
 
-    Value c = builder.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value d = builder.create<complex::ImOp>(elementType, adaptor.getRhs());
+    Value c = complex::ReOp::create(builder, elementType, adaptor.getRhs());
+    Value d = complex::ImOp::create(builder, elementType, adaptor.getRhs());
 
     rewriter.replaceOp(op, {powOpConversionImpl(builder, type, adaptor.getLhs(),
                                                 c, d, op.getFastmath())});
@@ -945,64 +945,64 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value negHalf = b.create<arith::ConstantOp>(
+    Value negHalf = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -0.5));
     Value nan = cst(APFloat::getNaN(floatSemantics));
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value absRsqrt = computeAbs(real, imag, fmf, b, AbsFn::rsqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value rsqrtArg = b.create<arith::MulFOp>(argArg, negHalf, fmf);
-    Value cos = b.create<math::CosOp>(rsqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(rsqrtArg, fmf);
+    Value argArg = math::Atan2Op::create(b, imag, real, fmf);
+    Value rsqrtArg = arith::MulFOp::create(b, argArg, negHalf, fmf);
+    Value cos = math::CosOp::create(b, rsqrtArg, fmf);
+    Value sin = math::SinOp::create(b, rsqrtArg, fmf);
 
-    Value resultReal = b.create<arith::MulFOp>(absRsqrt, cos, fmf);
-    Value resultImag = b.create<arith::MulFOp>(absRsqrt, sin, fmf);
+    Value resultReal = arith::MulFOp::create(b, absRsqrt, cos, fmf);
+    Value resultImag = arith::MulFOp::create(b, absRsqrt, sin, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value negOne = b.create<arith::ConstantOp>(
+      Value negOne = arith::ConstantOp::create(b,
           elementType, b.getFloatAttr(elementType, -1));
 
-      Value realSignedZero = b.create<math::CopySignOp>(zero, real, fmf);
-      Value imagSignedZero = b.create<math::CopySignOp>(zero, imag, fmf);
+      Value realSignedZero = math::CopySignOp::create(b, zero, real, fmf);
+      Value imagSignedZero = math::CopySignOp::create(b, zero, imag, fmf);
       Value negImagSignedZero =
-          b.create<arith::MulFOp>(negOne, imagSignedZero, fmf);
+          arith::MulFOp::create(b, negOne, imagSignedZero, fmf);
 
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value absImag = b.create<math::AbsFOp>(imag, fmf);
+      Value absReal = math::AbsFOp::create(b, real, fmf);
+      Value absImag = math::AbsFOp::create(b, imag, fmf);
 
       Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
       Value realIsNan =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, real, real, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, real, real, fmf);
       Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
-      Value inIsNanInf = b.create<arith::AndIOp>(absImagIsInf, realIsNan);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value inIsNanInf = arith::AndIOp::create(b, absImagIsInf, realIsNan);
 
-      Value resultIsZero = b.create<arith::OrIOp>(inIsNanInf, realIsInf);
+      Value resultIsZero = arith::OrIOp::create(b, inIsNanInf, realIsInf);
 
       resultReal =
-          b.create<arith::SelectOp>(resultIsZero, realSignedZero, resultReal);
-      resultImag = b.create<arith::SelectOp>(resultIsZero, negImagSignedZero,
+          arith::SelectOp::create(b, resultIsZero, realSignedZero, resultReal);
+      resultImag = arith::SelectOp::create(b, resultIsZero, negImagSignedZero,
                                              resultImag);
     }
 
     Value isRealZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero, fmf);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, zero, fmf);
     Value isImagZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-    Value isZero = b.create<arith::AndIOp>(isRealZero, isImagZero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
+    Value isZero = arith::AndIOp::create(b, isRealZero, isImagZero);
 
-    resultReal = b.create<arith::SelectOp>(isZero, inf, resultReal);
-    resultImag = b.create<arith::SelectOp>(isZero, nan, resultImag);
+    resultReal = arith::SelectOp::create(b, isZero, inf, resultReal);
+    resultImag = arith::SelectOp::create(b, isZero, nan, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -1021,9 +1021,9 @@ struct AngleOpConversion : public OpConversionPattern<complex::AngleOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, type, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, type, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, type, adaptor.getComplex());
+        complex::ImOp::create(rewriter, loc, type, adaptor.getComplex());
 
     rewriter.replaceOpWithNewOp<math::Atan2Op>(op, imag, real, fmf);
 
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index 88a8b7fb185c5..6a1f7c105b2c6 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -73,13 +73,13 @@ struct AssertOpLowering : public ConvertOpToLLVMPattern<cf::AssertOp> {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
         auto abortFuncTy = LLVM::LLVMFunctionType::get(getVoidType(), {});
-        abortFunc = rewriter.create<LLVM::LLVMFuncOp>(rewriter.getUnknownLoc(),
+        abortFunc = LLVM::LLVMFuncOp::create(rewriter, rewriter.getUnknownLoc(),
                                                       "abort", abortFuncTy);
       }
-      rewriter.create<LLVM::CallOp>(loc, abortFunc, ValueRange());
-      rewriter.create<LLVM::UnreachableOp>(loc);
+      LLVM::CallOp::create(rewriter, loc, abortFunc, ValueRange());
+      LLVM::UnreachableOp::create(rewriter, loc);
     } else {
-      rewriter.create<LLVM::BrOp>(loc, ValueRange(), continuationBlock);
+      LLVM::BrOp::create(rewriter, loc, ValueRange(), continuationBlock);
     }
 
     // Generate assertion test.
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index 9831dcaaaccc8..aa468c29536c7 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -33,7 +33,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
     MutableArrayRef<Region> regions) {
   if (auto condBrOp = dyn_cast<cf::CondBranchOp>(controlFlowCondOp)) {
     assert(regions.size() == 2);
-    auto ifOp = builder.create<scf::IfOp>(controlFlowCondOp->getLoc(),
+    auto ifOp = scf::IfOp::create(builder, controlFlowCondOp->getLoc(),
                                           resultTypes, condBrOp.getCondition());
     ifOp.getThenRegion().takeBody(regions[0]);
     ifOp.getElseRegion().takeBody(regions[1]);
@@ -43,7 +43,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
   if (auto switchOp = dyn_cast<cf::SwitchOp>(controlFlowCondOp)) {
     // `getCFGSwitchValue` returns an i32 that we need to convert to index
     // fist.
-    auto cast = builder.create<arith::IndexCastUIOp>(
+    auto cast = arith::IndexCastUIOp::create(builder,
         controlFlowCondOp->getLoc(), builder.getIndexType(),
         switchOp.getFlag());
     SmallVector<int64_t> cases;
@@ -55,7 +55,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
 
     assert(regions.size() == cases.size() + 1);
 
-    auto indexSwitchOp = builder.create<scf::IndexSwitchOp>(
+    auto indexSwitchOp = scf::IndexSwitchOp::create(builder,
         controlFlowCondOp->getLoc(), resultTypes, cast, cases, cases.size());
 
     indexSwitchOp.getDefaultRegion().takeBody(regions[0]);
@@ -75,7 +75,7 @@ LogicalResult
 ControlFlowToSCFTransformation::createStructuredBranchRegionTerminatorOp(
     Location loc, OpBuilder &builder, Operation *branchRegionOp,
     Operation *replacedControlFlowOp, ValueRange results) {
-  builder.create<scf::YieldOp>(loc, results);
+  scf::YieldOp::create(builder, loc, results);
   return success();
 }
 
@@ -84,7 +84,7 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
     OpBuilder &builder, Operation *replacedOp, ValueRange loopVariablesInit,
     Value condition, ValueRange loopVariablesNextIter, Region &&loopBody) {
   Location loc = replacedOp->getLoc();
-  auto whileOp = builder.create<scf::WhileOp>(loc, loopVariablesInit.getTypes(),
+  auto whileOp = scf::WhileOp::create(builder, loc, loopVariablesInit.getTypes(),
                                               loopVariablesInit);
 
   whileOp.getBefore().takeBody(loopBody);
@@ -92,15 +92,15 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
   builder.setInsertionPointToEnd(&whileOp.getBefore().back());
   // `getCFGSwitchValue` returns a i32. We therefore need to truncate the
   // condition to i1 first. It is guaranteed to be either 0 or 1 already.
-  builder.create<scf::ConditionOp>(
-      loc, builder.create<arith::TruncIOp>(loc, builder.getI1Type(), condition),
+  scf::ConditionOp::create(builder,
+      loc, arith::TruncIOp::create(builder, loc, builder.getI1Type(), condition),
       loopVariablesNextIter);
 
   Block *afterBlock = builder.createBlock(&whileOp.getAfter());
   afterBlock->addArguments(
       loopVariablesInit.getTypes(),
       SmallVector<Location>(loopVariablesInit.size(), loc));
-  builder.create<scf::YieldOp>(loc, afterBlock->getArguments());
+  scf::YieldOp::create(builder, loc, afterBlock->getArguments());
 
   return whileOp.getOperation();
 }
@@ -108,7 +108,7 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
 Value ControlFlowToSCFTransformation::getCFGSwitchValue(Location loc,
                                                         OpBuilder &builder,
                                                         unsigned int value) {
-  return builder.create<arith::ConstantOp>(loc,
+  return arith::ConstantOp::create(builder, loc,
                                            builder.getI32IntegerAttr(value));
 }
 
@@ -117,7 +117,7 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
     ArrayRef<unsigned int> caseValues, BlockRange caseDestinations,
     ArrayRef<ValueRange> caseArguments, Block *defaultDest,
     ValueRange defaultArgs) {
-  builder.create<cf::SwitchOp>(loc, flag, defaultDest, defaultArgs,
+  cf::SwitchOp::create(builder, loc, flag, defaultDest, defaultArgs,
                                llvm::to_vector_of<int32_t>(caseValues),
                                caseDestinations, caseArguments);
 }
@@ -125,7 +125,7 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
 Value ControlFlowToSCFTransformation::getUndefValue(Location loc,
                                                     OpBuilder &builder,
                                                     Type type) {
-  return builder.create<ub::PoisonOp>(loc, type, nullptr);
+  return ub::PoisonOp::create(builder, loc, type, nullptr);
 }
 
 FailureOr<Operation *>
diff --git a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
index f8dc06f41ab87..20167f168185a 100644
--- a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
+++ b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
@@ -99,7 +99,7 @@ class FuncOpConversion final : public OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted `emitc.func` op.
-    emitc::FuncOp newFuncOp = rewriter.create<emitc::FuncOp>(
+    emitc::FuncOp newFuncOp = emitc::FuncOp::create(rewriter,
         funcOp.getLoc(), funcOp.getName(),
         FunctionType::get(rewriter.getContext(),
                           signatureConverter.getConvertedTypes(),
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 36235636d6ba2..51dbe6cbb7de3 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -115,7 +115,7 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   SmallVector<NamedAttribute> attributes;
   filterFuncAttributes(funcOp, attributes);
 
-  auto wrapperFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto wrapperFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperFuncType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
@@ -129,13 +129,13 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   for (auto [index, argType] : llvm::enumerate(type.getInputs())) {
     Value arg = wrapperFuncOp.getArgument(index + argOffset);
     if (auto memrefType = dyn_cast<MemRefType>(argType)) {
-      Value loaded = rewriter.create<LLVM::LoadOp>(
+      Value loaded = LLVM::LoadOp::create(rewriter,
           loc, typeConverter.convertType(memrefType), arg);
       MemRefDescriptor::unpack(rewriter, loc, loaded, memrefType, args);
       continue;
     }
     if (isa<UnrankedMemRefType>(argType)) {
-      Value loaded = rewriter.create<LLVM::LoadOp>(
+      Value loaded = LLVM::LoadOp::create(rewriter,
           loc, typeConverter.convertType(argType), arg);
       UnrankedMemRefDescriptor::unpack(rewriter, loc, loaded, args);
       continue;
@@ -144,14 +144,14 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
     args.push_back(arg);
   }
 
-  auto call = rewriter.create<LLVM::CallOp>(loc, newFuncOp, args);
+  auto call = LLVM::CallOp::create(rewriter, loc, newFuncOp, args);
 
   if (resultStructType) {
-    rewriter.create<LLVM::StoreOp>(loc, call.getResult(),
+    LLVM::StoreOp::create(rewriter, loc, call.getResult(),
                                    wrapperFuncOp.getArgument(0));
-    rewriter.create<LLVM::ReturnOp>(loc, ValueRange{});
+    LLVM::ReturnOp::create(rewriter, loc, ValueRange{});
   } else {
-    rewriter.create<LLVM::ReturnOp>(loc, call.getResults());
+    LLVM::ReturnOp::create(rewriter, loc, call.getResults());
   }
 }
 
@@ -182,7 +182,7 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   filterFuncAttributes(funcOp, attributes);
 
   // Create the auxiliary function.
-  auto wrapperFunc = builder.create<LLVM::LLVMFuncOp>(
+  auto wrapperFunc = LLVM::LLVMFuncOp::create(builder,
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
@@ -201,11 +201,11 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   if (resultStructType) {
     // Allocate the struct on the stack and pass the pointer.
     Type resultType = cast<LLVM::LLVMFunctionType>(wrapperType).getParamType(0);
-    Value one = builder.create<LLVM::ConstantOp>(
+    Value one = LLVM::ConstantOp::create(builder,
         loc, typeConverter.convertType(builder.getIndexType()),
         builder.getIntegerAttr(builder.getIndexType(), 1));
     Value result =
-        builder.create<LLVM::AllocaOp>(loc, resultType, resultStructType, one);
+        LLVM::AllocaOp::create(builder, loc, resultType, resultStructType, one);
     args.push_back(result);
   }
 
@@ -229,12 +229,12 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
                     wrapperArgsRange.take_front(numToDrop));
 
       auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext());
-      Value one = builder.create<LLVM::ConstantOp>(
+      Value one = LLVM::ConstantOp::create(builder,
           loc, typeConverter.convertType(builder.getIndexType()),
           builder.getIntegerAttr(builder.getIndexType(), 1));
-      Value allocated = builder.create<LLVM::AllocaOp>(
+      Value allocated = LLVM::AllocaOp::create(builder,
           loc, ptrTy, packed.getType(), one, /*alignment=*/0);
-      builder.create<LLVM::StoreOp>(loc, packed, allocated);
+      LLVM::StoreOp::create(builder, loc, packed, allocated);
       arg = allocated;
     } else {
       arg = wrapperArgsRange[0];
@@ -245,14 +245,14 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   }
   assert(wrapperArgsRange.empty() && "did not map some of the arguments");
 
-  auto call = builder.create<LLVM::CallOp>(loc, wrapperFunc, args);
+  auto call = LLVM::CallOp::create(builder, loc, wrapperFunc, args);
 
   if (resultStructType) {
     Value result =
-        builder.create<LLVM::LoadOp>(loc, resultStructType, args.front());
-    builder.create<LLVM::ReturnOp>(loc, result);
+        LLVM::LoadOp::create(builder, loc, resultStructType, args.front());
+    LLVM::ReturnOp::create(builder, loc, result);
   } else {
-    builder.create<LLVM::ReturnOp>(loc, call.getResults());
+    LLVM::ReturnOp::create(builder, loc, call.getResults());
   }
 }
 
@@ -283,7 +283,7 @@ static void restoreByValRefArgumentType(
     Type resTy = typeConverter.convertType(
         cast<TypeAttr>(byValRefAttr->getValue()).getValue());
 
-    Value valueArg = rewriter.create<LLVM::LoadOp>(arg.getLoc(), resTy, arg);
+    Value valueArg = LLVM::LoadOp::create(rewriter, arg.getLoc(), resTy, arg);
     rewriter.replaceUsesOfBlockArgument(arg, valueArg);
   }
 }
@@ -357,7 +357,7 @@ FailureOr<LLVM::LLVMFuncOp> mlir::convertFuncOpToLLVMFuncOp(
     symbolTable.remove(funcOp);
   }
 
-  auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
       /*dsoLocal=*/false, /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr,
       attributes);
@@ -509,7 +509,7 @@ struct ConstantOpLowering : public ConvertOpToLLVMPattern<func::ConstantOp> {
       return rewriter.notifyMatchFailure(op, "failed to convert result type");
 
     auto newOp =
-        rewriter.create<LLVM::AddressOfOp>(op.getLoc(), type, op.getValue());
+        LLVM::AddressOfOp::create(rewriter, op.getLoc(), type, op.getValue());
     for (const NamedAttribute &attr : op->getAttrs()) {
       if (attr.getName().strref() == "value")
         continue;
@@ -556,7 +556,7 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
     auto promoted = this->getTypeConverter()->promoteOperands(
         callOp.getLoc(), /*opOperands=*/callOp->getOperands(),
         adaptor.getOperands(), rewriter, useBarePtrCallConv);
-    auto newOp = rewriter.create<LLVM::CallOp>(
+    auto newOp = LLVM::CallOp::create(rewriter,
         callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
         promoted, callOp->getAttrs());
 
@@ -573,7 +573,7 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
       // Extract individual results from the structure and return them as list.
       results.reserve(numResults);
       for (unsigned i = 0; i < numResults; ++i) {
-        results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        results.push_back(LLVM::ExtractValueOp::create(rewriter,
             callOp.getLoc(), newOp->getResult(0), i));
       }
     }
@@ -726,9 +726,9 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<func::ReturnOp> {
       return rewriter.notifyMatchFailure(op, "could not convert result types");
     }
 
-    Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
+    Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
     for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
-      packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
+      packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
     }
     rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, TypeRange(), packed,
                                                 op->getAttrs());
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index 01ca5e99a9aff..f5d5460412da2 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -28,7 +28,7 @@ LLVM::LLVMFuncOp mlir::getOrDefineFunction(gpu::GPUModuleOp moduleOp,
   if (!(ret = moduleOp.template lookupSymbol<LLVM::LLVMFuncOp>(name))) {
     OpBuilder::InsertionGuard guard(b);
     b.setInsertionPointToStart(moduleOp.getBody());
-    ret = b.create<LLVM::LLVMFuncOp>(loc, name, type, LLVM::Linkage::External);
+    ret = LLVM::LLVMFuncOp::create(b, loc, name, type, LLVM::Linkage::External);
   }
   return ret;
 }
@@ -68,7 +68,7 @@ mlir::getOrCreateStringConstant(OpBuilder &b, Location loc,
   OpBuilder::InsertionGuard guard(b);
   b.setInsertionPointToStart(moduleOp.getBody());
   SmallString<16> name = getUniqueSymbolName(moduleOp, namePrefix);
-  return b.create<LLVM::GlobalOp>(loc, globalType,
+  return LLVM::GlobalOp::create(b, loc, globalType,
                                   /*isConstant=*/true, LLVM::Linkage::Internal,
                                   name, attr, alignment, addrSpace);
 }
@@ -151,7 +151,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
               gpuFuncOp.getWorkgroupAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      auto globalOp = rewriter.create<LLVM::GlobalOp>(
+      auto globalOp = LLVM::GlobalOp::create(rewriter,
           gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
           LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment,
           workgroupAddrSpace);
@@ -220,7 +220,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
   LLVM::CConv callingConvention = gpuFuncOp.isKernel()
                                       ? kernelCallingConvention
                                       : nonKernelCallingConvention;
-  auto llvmFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto llvmFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
       LLVM::Linkage::External, /*dsoLocal=*/false, callingConvention,
       /*comdat=*/nullptr, attributes);
@@ -266,10 +266,10 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       for (const auto [idx, global] : llvm::enumerate(workgroupBuffers)) {
         auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext(),
                                                   global.getAddrSpace());
-        Value address = rewriter.create<LLVM::AddressOfOp>(
+        Value address = LLVM::AddressOfOp::create(rewriter,
             loc, ptrType, global.getSymNameAttr());
         Value memory =
-            rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getType(),
+            LLVM::GEPOp::create(rewriter, loc, ptrType, global.getType(),
                                          address, ArrayRef<LLVM::GEPArg>{0, 0});
 
         // Build a memref descriptor pointing to the buffer to plug with the
@@ -298,14 +298,14 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       Type elementType = typeConverter->convertType(type.getElementType());
       auto ptrType =
           LLVM::LLVMPointerType::get(rewriter.getContext(), allocaAddrSpace);
-      Value numElements = rewriter.create<LLVM::ConstantOp>(
+      Value numElements = LLVM::ConstantOp::create(rewriter,
           gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
       uint64_t alignment = 0;
       if (auto alignAttr =
               dyn_cast_or_null<IntegerAttr>(gpuFuncOp.getPrivateAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      Value allocated = rewriter.create<LLVM::AllocaOp>(
+      Value allocated = LLVM::AllocaOp::create(rewriter,
           gpuFuncOp.getLoc(), ptrType, elementType, numElements, alignment);
       Value descr = MemRefDescriptor::fromStaticShape(
           rewriter, loc, *getTypeConverter(), type, allocated);
@@ -418,8 +418,8 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
           {llvmI64, ptrType, /*length (bytes)*/ llvmI64, /*isLast*/ llvmI32}));
 
   /// Start the printf hostcall
-  Value zeroI64 = rewriter.create<LLVM::ConstantOp>(loc, llvmI64, 0);
-  auto printfBeginCall = rewriter.create<LLVM::CallOp>(loc, ocklBegin, zeroI64);
+  Value zeroI64 = LLVM::ConstantOp::create(rewriter, loc, llvmI64, 0);
+  auto printfBeginCall = LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
   Value printfDesc = printfBeginCall.getResult();
 
   // Create the global op or find an existing one.
@@ -427,20 +427,20 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
       rewriter, loc, moduleOp, llvmI8, "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element and pass it to printf()
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
       loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
-  Value stringLen = rewriter.create<LLVM::ConstantOp>(
+  Value stringLen = LLVM::ConstantOp::create(rewriter,
       loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
 
-  Value oneI32 = rewriter.create<LLVM::ConstantOp>(loc, llvmI32, 1);
-  Value zeroI32 = rewriter.create<LLVM::ConstantOp>(loc, llvmI32, 0);
+  Value oneI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 1);
+  Value zeroI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 0);
 
-  auto appendFormatCall = rewriter.create<LLVM::CallOp>(
+  auto appendFormatCall = LLVM::CallOp::create(rewriter,
       loc, ocklAppendStringN,
       ValueRange{printfDesc, stringStart, stringLen,
                  adaptor.getArgs().empty() ? oneI32 : zeroI32});
@@ -456,17 +456,17 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
     SmallVector<mlir::Value, 2 + argsPerAppend + 1> arguments;
     arguments.push_back(printfDesc);
     arguments.push_back(
-        rewriter.create<LLVM::ConstantOp>(loc, llvmI32, numArgsThisCall));
+        LLVM::ConstantOp::create(rewriter, loc, llvmI32, numArgsThisCall));
     for (size_t i = group; i < bound; ++i) {
       Value arg = adaptor.getArgs()[i];
       if (auto floatType = dyn_cast<FloatType>(arg.getType())) {
         if (!floatType.isF64())
-          arg = rewriter.create<LLVM::FPExtOp>(
+          arg = LLVM::FPExtOp::create(rewriter,
               loc, typeConverter->convertType(rewriter.getF64Type()), arg);
-        arg = rewriter.create<LLVM::BitcastOp>(loc, llvmI64, arg);
+        arg = LLVM::BitcastOp::create(rewriter, loc, llvmI64, arg);
       }
       if (arg.getType().getIntOrFloatBitWidth() != 64)
-        arg = rewriter.create<LLVM::ZExtOp>(loc, llvmI64, arg);
+        arg = LLVM::ZExtOp::create(rewriter, loc, llvmI64, arg);
 
       arguments.push_back(arg);
     }
@@ -477,7 +477,7 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
 
     auto isLast = (bound == nArgs) ? oneI32 : zeroI32;
     arguments.push_back(isLast);
-    auto call = rewriter.create<LLVM::CallOp>(loc, ocklAppendArgs, arguments);
+    auto call = LLVM::CallOp::create(rewriter, loc, ocklAppendArgs, arguments);
     printfDesc = call.getResult();
   }
   rewriter.eraseOp(gpuPrintfOp);
@@ -510,12 +510,12 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
       /*alignment=*/0, addressSpace);
 
   // Get a pointer to the format string's first element
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
       loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
 
   // Construct arguments and function call
@@ -525,7 +525,7 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
   printfArgs.push_back(stringStart);
   printfArgs.append(argsRange.begin(), argsRange.end());
 
-  rewriter.create<LLVM::CallOp>(loc, printfDecl, printfArgs);
+  LLVM::CallOp::create(rewriter, loc, printfDecl, printfArgs);
   rewriter.eraseOp(gpuPrintfOp);
   return success();
 }
@@ -559,9 +559,9 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
                                 "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(loc, global);
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter, loc, global);
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
   SmallVector<Type> types;
   SmallVector<Value> args;
@@ -572,27 +572,27 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
     assert(type.isIntOrFloat());
     if (isa<FloatType>(type)) {
       type = rewriter.getF64Type();
-      promotedArg = rewriter.create<LLVM::FPExtOp>(loc, type, arg);
+      promotedArg = LLVM::FPExtOp::create(rewriter, loc, type, arg);
     }
     types.push_back(type);
     args.push_back(promotedArg);
   }
   Type structType =
       LLVM::LLVMStructType::getLiteral(gpuPrintfOp.getContext(), types);
-  Value one = rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
+  Value one = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
                                                 rewriter.getIndexAttr(1));
   Value tempAlloc =
-      rewriter.create<LLVM::AllocaOp>(loc, ptrType, structType, one,
+      LLVM::AllocaOp::create(rewriter, loc, ptrType, structType, one,
                                       /*alignment=*/0);
   for (auto [index, arg] : llvm::enumerate(args)) {
-    Value ptr = rewriter.create<LLVM::GEPOp>(
+    Value ptr = LLVM::GEPOp::create(rewriter,
         loc, ptrType, structType, tempAlloc,
         ArrayRef<LLVM::GEPArg>{0, static_cast<int32_t>(index)});
-    rewriter.create<LLVM::StoreOp>(loc, arg, ptr);
+    LLVM::StoreOp::create(rewriter, loc, arg, ptr);
   }
   std::array<Value, 2> printfArgs = {stringStart, tempAlloc};
 
-  rewriter.create<LLVM::CallOp>(loc, vprintfDecl, printfArgs);
+  LLVM::CallOp::create(rewriter, loc, vprintfDecl, printfArgs);
   rewriter.eraseOp(gpuPrintfOp);
   return success();
 }
@@ -607,22 +607,22 @@ static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands,
   TypeRange operandTypes(operands);
   VectorType vectorType = cast<VectorType>(llvm1DVectorTy);
   Location loc = op->getLoc();
-  Value result = rewriter.create<LLVM::PoisonOp>(loc, vectorType);
+  Value result = LLVM::PoisonOp::create(rewriter, loc, vectorType);
   Type indexType = converter.convertType(rewriter.getIndexType());
   StringAttr name = op->getName().getIdentifier();
   Type elementType = vectorType.getElementType();
 
   for (int64_t i = 0; i < vectorType.getNumElements(); ++i) {
-    Value index = rewriter.create<LLVM::ConstantOp>(loc, indexType, i);
+    Value index = LLVM::ConstantOp::create(rewriter, loc, indexType, i);
     auto extractElement = [&](Value operand) -> Value {
       if (!isa<VectorType>(operand.getType()))
         return operand;
-      return rewriter.create<LLVM::ExtractElementOp>(loc, operand, index);
+      return LLVM::ExtractElementOp::create(rewriter, loc, operand, index);
     };
     auto scalarOperands = llvm::map_to_vector(operands, extractElement);
     Operation *scalarOp =
         rewriter.create(loc, name, scalarOperands, elementType, op->getAttrs());
-    result = rewriter.create<LLVM::InsertElementOp>(
+    result = LLVM::InsertElementOp::create(rewriter,
         loc, result, scalarOp->getResult(0), index);
   }
   return result;
@@ -705,7 +705,7 @@ LLVM::GlobalOp getDynamicSharedMemorySymbol(
   auto zeroSizedArrayType = LLVM::LLVMArrayType::get(
       typeConverter->convertType(memrefType.getElementType()), 0);
 
-  return rewriter.create<LLVM::GlobalOp>(
+  return LLVM::GlobalOp::create(rewriter,
       op->getLoc(), zeroSizedArrayType, /*isConstant=*/false,
       LLVM::Linkage::Internal, symName, /*value=*/Attribute(), alignmentByte,
       addressSpace.value());
@@ -732,12 +732,12 @@ LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
   // Step 3. Get address of the global symbol
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(op);
-  auto basePtr = rewriter.create<LLVM::AddressOfOp>(loc, shmemOp);
+  auto basePtr = LLVM::AddressOfOp::create(rewriter, loc, shmemOp);
   Type baseType = basePtr->getResultTypes().front();
 
   // Step 4. Generate GEP using offsets
   SmallVector<LLVM::GEPArg> gepArgs = {0};
-  Value shmemPtr = rewriter.create<LLVM::GEPOp>(loc, baseType, elementType,
+  Value shmemPtr = LLVM::GEPOp::create(rewriter, loc, baseType, elementType,
                                                 basePtr, gepArgs);
   // Step 5. Create a memref descriptor
   SmallVector<Value> shape, strides;
@@ -799,9 +799,9 @@ LogicalResult GPUReturnOpLowering::matchAndRewrite(
     return rewriter.notifyMatchFailure(op, "could not convert result types");
   }
 
-  Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
+  Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
   for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
-    packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
+    packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
   }
   rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, TypeRange(), packed,
                                               op->getAttrs());
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 167cabbc57db9..5e8254a0b6035 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -79,7 +79,7 @@ class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern<OpTy> {
     uint64_t rank = type.getRank();
     Value numElements = desc.size(rewriter, loc, /*pos=*/0);
     for (unsigned i = 1; i < rank; i++)
-      numElements = rewriter.create<LLVM::MulOp>(
+      numElements = LLVM::MulOp::create(rewriter,
           loc, numElements, desc.size(rewriter, loc, /*pos=*/i));
     return numElements;
   }
@@ -582,7 +582,7 @@ LLVM::CallOp FunctionCallBuilder::create(Location loc, OpBuilder &builder,
     return OpBuilder::atBlockEnd(module.getBody())
         .create<LLVM::LLVMFuncOp>(loc, functionName, functionType);
   }();
-  return builder.create<LLVM::CallOp>(loc, function, arguments);
+  return LLVM::CallOp::create(builder, loc, function, arguments);
 }
 
 // Corresponding to cusparseIndexType_t defined in cusparse.h.
@@ -780,12 +780,12 @@ LogicalResult ConvertAllocOpToGpuRuntimeCallPattern::matchAndRewrite(
 
   // Allocate the underlying buffer and store a pointer to it in the MemRef
   // descriptor.
-  auto nullPtr = rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPointerType);
+  auto nullPtr = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPointerType);
   Value stream = adaptor.getAsyncDependencies().empty()
                      ? nullPtr
                      : adaptor.getAsyncDependencies().front();
 
-  auto isHostShared = rewriter.create<mlir::LLVM::ConstantOp>(
+  auto isHostShared = mlir::LLVM::ConstantOp::create(rewriter,
       loc, llvmInt8Type, rewriter.getI8IntegerAttr(isShared));
 
   Value allocatedPtr =
@@ -1012,7 +1012,7 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
       uint64_t staticSize = static_cast<uint64_t>(bitwidth / 8) *
                             static_cast<uint64_t>(memrefTy.getNumElements());
 
-      Value sizeArg = rewriter.create<LLVM::ConstantOp>(
+      Value sizeArg = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(staticSize));
       llvmArgumentsWithSizes.push_back(llvmArg); // Presumably a bare pointer.
       llvmArgumentsWithSizes.push_back(sizeArg);
@@ -1025,7 +1025,7 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
         gpu::KernelDim3{adaptor.getClusterSizeX(), adaptor.getClusterSizeY(),
                         adaptor.getClusterSizeZ()};
   }
-  rewriter.create<gpu::LaunchFuncOp>(
+  gpu::LaunchFuncOp::create(rewriter,
       launchOp.getLoc(), launchOp.getKernelAttr(),
       gpu::KernelDim3{adaptor.getGridSizeX(), adaptor.getGridSizeY(),
                       adaptor.getGridSizeZ()},
@@ -1048,7 +1048,7 @@ static Value bitAndAddrspaceCast(Location loc,
                                  const LLVMTypeConverter &typeConverter) {
   auto sourceTy = cast<LLVM::LLVMPointerType>(sourcePtr.getType());
   if (destinationType.getAddressSpace() != sourceTy.getAddressSpace())
-    sourcePtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+    sourcePtr = LLVM::AddrSpaceCastOp::create(rewriter,
         loc,
         LLVM::LLVMPointerType::get(rewriter.getContext(),
                                    destinationType.getAddressSpace()),
@@ -1072,13 +1072,13 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite(
   Value numElements = getNumElements(rewriter, loc, memRefType, srcDesc);
 
   Type elementPtrType = getElementPtrType(memRefType);
-  Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, elementPtrType);
-  Value gepPtr = rewriter.create<LLVM::GEPOp>(
+  Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
+  Value gepPtr = LLVM::GEPOp::create(rewriter,
       loc, elementPtrType,
       typeConverter->convertType(memRefType.getElementType()), nullPtr,
       numElements);
   auto sizeBytes =
-      rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);
+      LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
 
   auto src = bitAndAddrspaceCast(loc, rewriter, llvmPointerType,
                                  srcDesc.alignedPtr(rewriter, loc),
@@ -1123,7 +1123,7 @@ LogicalResult ConvertMemsetOpToGpuRuntimeCallPattern::matchAndRewrite(
   Value numElements = getNumElements(rewriter, loc, memRefType, dstDesc);
 
   auto value =
-      rewriter.create<LLVM::BitcastOp>(loc, bitCastType, adaptor.getValue());
+      LLVM::BitcastOp::create(rewriter, loc, bitCastType, adaptor.getValue());
   auto dst = bitAndAddrspaceCast(loc, rewriter, llvmPointerType,
                                  dstDesc.alignedPtr(rewriter, loc),
                                  *getTypeConverter());
@@ -1150,14 +1150,14 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite(
 template <typename T>
 static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmInt32Type = builder.getIntegerType(32);
-  return builder.create<LLVM::ConstantOp>(loc, llvmInt32Type,
+  return LLVM::ConstantOp::create(builder, loc, llvmInt32Type,
                                           static_cast<int32_t>(tValue));
 }
 
 template <typename T>
 static Value genConstFloat32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmFloat32Type = builder.getF32Type();
-  return builder.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(builder,
       loc, llvmFloat32Type,
       builder.getF32FloatAttr(static_cast<float>(tValue)));
 }
@@ -1189,11 +1189,11 @@ LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite(
   // the dnmat is used with spmat with 2:4 sparsity
   if (dims.size() == 2) {
     if (isSpMMCusparseLtOp(op.getDnTensor())) {
-      auto handleSz = rewriter.create<LLVM::ConstantOp>(
+      auto handleSz = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(11032));
-      handle = rewriter.create<LLVM::AllocaOp>(
+      handle = LLVM::AllocaOp::create(rewriter,
           loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
-      handle = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, handle);
+      handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
       createLtDnMatCallBuilder
           .create(loc, rewriter,
@@ -1351,11 +1351,11 @@ LogicalResult ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto dtp = genConstInt32From(rewriter, loc, getCuSparseDataTypeFrom(dType));
 
   // CUDA runner asserts the size is 44104 bytes.
-  auto handleSz = rewriter.create<LLVM::ConstantOp>(
+  auto handleSz = LLVM::ConstantOp::create(rewriter,
       loc, getIndexType(), rewriter.getIndexAttr(44104));
-  Value handle = rewriter.create<LLVM::AllocaOp>(
+  Value handle = LLVM::AllocaOp::create(rewriter,
       loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
-  handle = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, handle);
+  handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
   create2To4SpMatCallBuilder
       .create(loc, rewriter,
@@ -1441,9 +1441,9 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
         genConstInt32From(rewriter, loc, get2To4PruneFlag(op.getSpmatA()));
     auto computeType = genConstInt32From(
         rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType()));
-    auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+    auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(3));
-    auto bufferSize = rewriter.create<LLVM::AllocaOp>(
+    auto bufferSize = LLVM::AllocaOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, three, /*alignment=*/16);
     createCuSparseLtSpMMBufferSizeBuilder
         .create(loc, rewriter,
@@ -1452,20 +1452,20 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
                  pruneFlag, stream})
         .getResult();
 
-    auto bufferSizePtr1 = rewriter.create<LLVM::GEPOp>(
+    auto bufferSizePtr1 = LLVM::GEPOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{rewriter.create<LLVM::ConstantOp>(
+        ValueRange{LLVM::ConstantOp::create(rewriter,
             loc, getIndexType(), rewriter.getIndexAttr(1))});
-    auto bufferSizePtr2 = rewriter.create<LLVM::GEPOp>(
+    auto bufferSizePtr2 = LLVM::GEPOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{rewriter.create<LLVM::ConstantOp>(
+        ValueRange{LLVM::ConstantOp::create(rewriter,
             loc, getIndexType(), rewriter.getIndexAttr(2))});
     auto bufferSize0 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSize);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSize);
     auto bufferSize1 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr1);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSizePtr1);
     auto bufferSize2 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr2);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSizePtr2);
 
     rewriter.replaceOp(op, {bufferSize0, bufferSize1, bufferSize2, stream});
   } else {
@@ -1669,28 +1669,28 @@ LogicalResult ConvertSpMatGetSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
   Location loc = op.getLoc();
   auto stream = adaptor.getAsyncDependencies().front();
 
-  auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+  auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                  rewriter.getIndexAttr(3));
-  auto buffer = rewriter.create<LLVM::AllocaOp>(
+  auto buffer = LLVM::AllocaOp::create(rewriter,
       loc, llvmPointerType, llvmInt64Type, three, /*alignment=*/16);
 
-  auto rowsPtr = rewriter.create<LLVM::GEPOp>(
+  auto rowsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(0))});
-  auto colsPtr = rewriter.create<LLVM::GEPOp>(
+  auto colsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(1))});
-  auto nnzsPtr = rewriter.create<LLVM::GEPOp>(
+  auto nnzsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(2))});
   createSpMatGetSizeBuilder.create(
       loc, rewriter, {adaptor.getSpmat(), rowsPtr, colsPtr, nnzsPtr, stream});
-  auto rows = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, rowsPtr);
-  auto cols = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, colsPtr);
-  auto nnzs = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, nnzsPtr);
+  auto rows = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, rowsPtr);
+  auto cols = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, colsPtr);
+  auto nnzs = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, nnzsPtr);
 
   rewriter.replaceOp(op, {rows, cols, nnzs, stream});
   return success();
diff --git a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
index aab2409ed6328..70964727351a9 100644
--- a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
@@ -59,13 +59,13 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
     Operation *newOp;
     switch (op.getDimension()) {
     case gpu::Dimension::x:
-      newOp = rewriter.create<XOp>(loc, IntegerType::get(context, 32));
+      newOp = XOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     case gpu::Dimension::y:
-      newOp = rewriter.create<YOp>(loc, IntegerType::get(context, 32));
+      newOp = YOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     case gpu::Dimension::z:
-      newOp = rewriter.create<ZOp>(loc, IntegerType::get(context, 32));
+      newOp = ZOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     }
 
@@ -124,10 +124,10 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
                                   rewriter.getContext(), 32, min, max));
     }
     if (indexBitwidth > 32) {
-      newOp = rewriter.create<LLVM::SExtOp>(
+      newOp = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
     } else if (indexBitwidth < 32) {
-      newOp = rewriter.create<LLVM::TruncOp>(
+      newOp = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
     }
 
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index 64cf09e600b88..33b09b06649b9 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -103,7 +103,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
     LLVMFuncOp funcOp = appendOrGetFuncOp(funcName, funcType, op);
     auto callOp =
-        rewriter.create<LLVM::CallOp>(op->getLoc(), funcOp, castedOperands);
+        LLVM::CallOp::create(rewriter, op->getLoc(), funcOp, castedOperands);
 
     if (resultType == adaptor.getOperands().front().getType()) {
       rewriter.replaceOp(op, {callOp.getResult()});
@@ -115,10 +115,10 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // there is no guarantee of a specific value being used to indicate true,
     // compare for inequality with zero (rather than truncate or shift).
     if (isResultBool) {
-      Value zero = rewriter.create<LLVM::ConstantOp>(
+      Value zero = LLVM::ConstantOp::create(rewriter,
           op->getLoc(), rewriter.getIntegerType(32),
           rewriter.getI32IntegerAttr(0));
-      Value truncated = rewriter.create<LLVM::ICmpOp>(
+      Value truncated = LLVM::ICmpOp::create(rewriter,
           op->getLoc(), LLVM::ICmpPredicate::ne, callOp.getResult(), zero);
       rewriter.replaceOp(op, {truncated});
       return success();
@@ -126,7 +126,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
     assert(callOp.getResult().getType().isF32() &&
            "only f32 types are supposed to be truncated back");
-    Value truncated = rewriter.create<LLVM::FPTruncOp>(
+    Value truncated = LLVM::FPTruncOp::create(rewriter,
         op->getLoc(), adaptor.getOperands().front().getType(),
         callOp.getResult());
     rewriter.replaceOp(op, {truncated});
@@ -142,7 +142,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     if (!f16Func.empty() && isa<Float16Type>(type))
       return operand;
 
-    return rewriter.create<LLVM::FPExtOp>(
+    return LLVM::FPExtOp::create(rewriter,
         operand.getLoc(), Float32Type::get(rewriter.getContext()), operand);
   }
 
@@ -169,7 +169,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // location as debug info metadata inside of a function cannot be used
     // outside of that function.
     auto globalloc = op->getLoc()->findInstanceOfOrUnknown<FileLineColLoc>();
-    return b.create<LLVMFuncOp>(globalloc, funcName, funcType);
+    return LLVMFuncOp::create(b, globalloc, funcName, funcType);
   }
 
   StringRef getFunctionName(Type type, SourceOp op) const {
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index 8b6b553f6eed0..fc469102a54d2 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -54,7 +54,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
       SymbolTable::lookupSymbolIn(symbolTable, name));
   if (!func) {
     OpBuilder b(symbolTable->getRegion(0));
-    func = b.create<LLVM::LLVMFuncOp>(
+    func = LLVM::LLVMFuncOp::create(b,
         symbolTable->getLoc(), name,
         LLVM::LLVMFunctionType::get(resultType, paramTypes));
     func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
@@ -79,7 +79,7 @@ static LLVM::CallOp createSPIRVBuiltinCall(Location loc,
                                            ConversionPatternRewriter &rewriter,
                                            LLVM::LLVMFuncOp func,
                                            ValueRange args) {
-  auto call = rewriter.create<LLVM::CallOp>(loc, func, args);
+  auto call = LLVM::CallOp::create(rewriter, loc, func, args);
   call.setCConv(func.getCConv());
   call.setConvergentAttr(func.getConvergentAttr());
   call.setNoUnwindAttr(func.getNoUnwindAttr());
@@ -121,7 +121,7 @@ struct GPUBarrierConversion final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
     constexpr int64_t localMemFenceFlag = 1;
     Location loc = op->getLoc();
     Value flag =
-        rewriter.create<LLVM::ConstantOp>(loc, flagTy, localMemFenceFlag);
+        LLVM::ConstantOp::create(rewriter, loc, flagTy, localMemFenceFlag);
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, flag));
     return success();
   }
@@ -162,7 +162,7 @@ struct LaunchConfigConversion : ConvertToLLVMPattern {
 
     Location loc = op->getLoc();
     gpu::Dimension dim = getDimension(op);
-    Value dimVal = rewriter.create<LLVM::ConstantOp>(loc, dimTy,
+    Value dimVal = LLVM::ConstantOp::create(rewriter, loc, dimTy,
                                                      static_cast<int64_t>(dim));
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, dimVal));
     return success();
@@ -291,12 +291,12 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                                          ConversionPatternRewriter &rewriter) {
     return TypeSwitch<Type, Value>(oldVal.getType())
         .Case([&](BFloat16Type) {
-          return rewriter.create<LLVM::BitcastOp>(loc, rewriter.getI16Type(),
+          return LLVM::BitcastOp::create(rewriter, loc, rewriter.getI16Type(),
                                                   oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
-            return rewriter.create<LLVM::ZExtOp>(loc, rewriter.getI8Type(),
+            return LLVM::ZExtOp::create(rewriter, loc, rewriter.getI8Type(),
                                                  oldVal);
           return oldVal;
         })
@@ -308,11 +308,11 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                                           ConversionPatternRewriter &rewriter) {
     return TypeSwitch<Type, Value>(newTy)
         .Case([&](BFloat16Type) {
-          return rewriter.create<LLVM::BitcastOp>(loc, newTy, oldVal);
+          return LLVM::BitcastOp::create(rewriter, loc, newTy, oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
-            return rewriter.create<LLVM::TruncOp>(loc, newTy, oldVal);
+            return LLVM::TruncOp::create(rewriter, loc, newTy, oldVal);
           return oldVal;
         })
         .Default(oldVal);
@@ -349,7 +349,7 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
         bitcastOrTruncAfterShuffle(result, op.getType(0), loc, rewriter);
 
     Value trueVal =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI1Type(), true);
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getI1Type(), true);
     rewriter.replaceOp(op, {resultOrConversion, trueVal});
     return success();
   }
@@ -426,7 +426,7 @@ struct GPUSubgroupOpConversion final : ConvertOpToLLVMPattern<SubgroupOp> {
       if (indexTy.getIntOrFloatBitWidth() < resultTy.getIntOrFloatBitWidth()) {
         return failure();
       }
-      result = rewriter.create<LLVM::ZExtOp>(loc, indexTy, result);
+      result = LLVM::ZExtOp::create(rewriter, loc, indexTy, result);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 1ef6edea93c58..7a6601ea754e9 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -118,9 +118,9 @@ struct GPUSubgroupReduceOpLowering
 
     Location loc = op->getLoc();
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
-    Value offset = rewriter.create<LLVM::ConstantOp>(loc, int32Type, -1);
+    Value offset = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
 
-    auto reduxOp = rewriter.create<NVVM::ReduxOp>(loc, int32Type, op.getValue(),
+    auto reduxOp = NVVM::ReduxOp::create(rewriter, loc, int32Type, op.getValue(),
                                                   mode.value(), offset);
 
     rewriter.replaceOp(op, reduxOp->getResult(0));
@@ -158,13 +158,13 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     auto predTy = IntegerType::get(rewriter.getContext(), 1);
 
-    Value one = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 1);
-    Value minusOne = rewriter.create<LLVM::ConstantOp>(loc, int32Type, -1);
-    Value thirtyTwo = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 32);
-    Value numLeadInactiveLane = rewriter.create<LLVM::SubOp>(
+    Value one = LLVM::ConstantOp::create(rewriter, loc, int32Type, 1);
+    Value minusOne = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
+    Value thirtyTwo = LLVM::ConstantOp::create(rewriter, loc, int32Type, 32);
+    Value numLeadInactiveLane = LLVM::SubOp::create(rewriter,
         loc, int32Type, thirtyTwo, adaptor.getWidth());
     // Bit mask of active lanes: `(-1) >> (32 - activeWidth)`.
-    Value activeMask = rewriter.create<LLVM::LShrOp>(loc, int32Type, minusOne,
+    Value activeMask = LLVM::LShrOp::create(rewriter, loc, int32Type, minusOne,
                                                      numLeadInactiveLane);
     Value maskAndClamp;
     if (op.getMode() == gpu::ShuffleMode::UP) {
@@ -173,7 +173,7 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     } else {
       // Clamp lane: `activeWidth - 1`
       maskAndClamp =
-          rewriter.create<LLVM::SubOp>(loc, int32Type, adaptor.getWidth(), one);
+          LLVM::SubOp::create(rewriter, loc, int32Type, adaptor.getWidth(), one);
     }
 
     bool predIsUsed = !op->getResult(1).use_empty();
@@ -184,13 +184,13 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
       resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(),
                                                   {valueTy, predTy});
     }
-    Value shfl = rewriter.create<NVVM::ShflOp>(
+    Value shfl = NVVM::ShflOp::create(rewriter,
         loc, resultTy, activeMask, adaptor.getValue(), adaptor.getOffset(),
         maskAndClamp, convertShflKind(op.getMode()), returnValueAndIsValidAttr);
     if (predIsUsed) {
-      Value shflValue = rewriter.create<LLVM::ExtractValueOp>(loc, shfl, 0);
+      Value shflValue = LLVM::ExtractValueOp::create(rewriter, loc, shfl, 0);
       Value isActiveSrcLane =
-          rewriter.create<LLVM::ExtractValueOp>(loc, shfl, 1);
+          LLVM::ExtractValueOp::create(rewriter, loc, shfl, 1);
       rewriter.replaceOp(op, {shflValue, isActiveSrcLane});
     } else {
       rewriter.replaceOp(op, {shfl, nullptr});
@@ -215,15 +215,15 @@ struct GPULaneIdOpToNVVM : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
       bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
           /*bitWidth=*/32, /*lower=*/0, /*upper=*/kWarpSize);
     Value newOp =
-        rewriter.create<NVVM::LaneIdOp>(loc, rewriter.getI32Type(), bounds);
+        NVVM::LaneIdOp::create(rewriter, loc, rewriter.getI32Type(), bounds);
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      newOp = rewriter.create<LLVM::SExtOp>(
+      newOp = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp);
     } else if (indexBitwidth < 32) {
-      newOp = rewriter.create<LLVM::TruncOp>(
+      newOp = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp);
     }
     rewriter.replaceOp(op, {newOp});
@@ -271,10 +271,10 @@ struct AssertOpToAssertfailLowering
     Block *afterBlock =
         rewriter.splitBlock(assertBlock, ++assertOp->getIterator());
     rewriter.setInsertionPointToEnd(beforeBlock);
-    rewriter.create<cf::CondBranchOp>(loc, adaptor.getArg(), afterBlock,
+    cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(), afterBlock,
                                       assertBlock);
     rewriter.setInsertionPointToEnd(assertBlock);
-    rewriter.create<cf::BranchOp>(loc, afterBlock);
+    cf::BranchOp::create(rewriter, loc, afterBlock);
 
     // Continue cf.assert lowering.
     rewriter.setInsertionPoint(assertOp);
@@ -301,11 +301,11 @@ struct AssertOpToAssertfailLowering
     // Create constants.
     auto getGlobal = [&](LLVM::GlobalOp global) {
       // Get a pointer to the format string's first element.
-      Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+      Value globalPtr = LLVM::AddressOfOp::create(rewriter,
           loc, LLVM::LLVMPointerType::get(ctx, global.getAddrSpace()),
           global.getSymNameAttr());
       Value start =
-          rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+          LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                        globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
       return start;
     };
@@ -316,8 +316,8 @@ struct AssertOpToAssertfailLowering
     Value assertFunc = getGlobal(getOrCreateStringConstant(
         rewriter, loc, moduleOp, i8Type, "assert_func_", funcName));
     Value assertLine =
-        rewriter.create<LLVM::ConstantOp>(loc, i32Type, fileLine);
-    Value c1 = rewriter.create<LLVM::ConstantOp>(loc, i64Type, 1);
+        LLVM::ConstantOp::create(rewriter, loc, i32Type, fileLine);
+    Value c1 = LLVM::ConstantOp::create(rewriter, loc, i64Type, 1);
 
     // Insert function call to __assertfail.
     SmallVector<Value> arguments{assertMessage, assertFile, assertLine,
diff --git a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
index 45fd933d58857..7d5f65c470386 100644
--- a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
@@ -126,7 +126,7 @@ struct WmmaLoadOpToNVVMLowering
         cast<MemRefType>(subgroupMmaLoadMatrixOp.getSrcMemref().getType()),
         adaptor.getSrcMemref(), adaptor.getIndices());
 
-    Value leadingDim = rewriter.create<LLVM::ConstantOp>(
+    Value leadingDim = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI32Type(),
         subgroupMmaLoadMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMALoadOp>(
@@ -173,7 +173,7 @@ struct WmmaStoreOpToNVVMLowering
     auto matrixType = cast<LLVM::LLVMStructType>(adaptor.getSrc().getType());
     for (unsigned i = 0, e = matrixType.getBody().size(); i < e; ++i) {
       Value toUse =
-          rewriter.create<LLVM::ExtractValueOp>(loc, adaptor.getSrc(), i);
+          LLVM::ExtractValueOp::create(rewriter, loc, adaptor.getSrc(), i);
       storeOpOperands.push_back(toUse);
     }
 
@@ -181,7 +181,7 @@ struct WmmaStoreOpToNVVMLowering
         rewriter, loc,
         cast<MemRefType>(subgroupMmaStoreMatrixOp.getDstMemref().getType()),
         adaptor.getDstMemref(), adaptor.getIndices());
-    Value leadingDim = rewriter.create<LLVM::ConstantOp>(
+    Value leadingDim = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI32Type(),
         subgroupMmaStoreMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMAStoreOp>(
@@ -216,7 +216,7 @@ struct WmmaMmaOpToNVVMLowering
     auto unpackOp = [&](Value operand) {
       auto structType = cast<LLVM::LLVMStructType>(operand.getType());
       for (size_t i = 0, e = structType.getBody().size(); i < e; ++i) {
-        Value toUse = rewriter.create<LLVM::ExtractValueOp>(loc, operand, i);
+        Value toUse = LLVM::ExtractValueOp::create(rewriter, loc, operand, i);
         unpackedOps.push_back(toUse);
       }
     };
@@ -280,19 +280,19 @@ struct WmmaConstantOpToNVVMLowering
         cast<gpu::MMAMatrixType>(subgroupMmaConstantOp.getType()));
     // If the element type is a vector create a vector from the operand.
     if (auto vecType = dyn_cast<VectorType>(type.getBody()[0])) {
-      Value vecCst = rewriter.create<LLVM::PoisonOp>(loc, vecType);
+      Value vecCst = LLVM::PoisonOp::create(rewriter, loc, vecType);
       for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) {
-        Value idx = rewriter.create<LLVM::ConstantOp>(
+        Value idx = LLVM::ConstantOp::create(rewriter,
             loc, rewriter.getI32Type(), vecEl);
-        vecCst = rewriter.create<LLVM::InsertElementOp>(loc, vecType, vecCst,
+        vecCst = LLVM::InsertElementOp::create(rewriter, loc, vecType, vecCst,
                                                         cst, idx);
       }
       cst = vecCst;
     }
-    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, type);
+    Value matrixStruct = LLVM::PoisonOp::create(rewriter, loc, type);
     for (size_t i : llvm::seq(size_t(0), type.getBody().size())) {
       matrixStruct =
-          rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, cst, i);
+          LLVM::InsertValueOp::create(rewriter, loc, matrixStruct, cst, i);
     }
     rewriter.replaceOp(subgroupMmaConstantOp, matrixStruct);
     return success();
@@ -305,17 +305,17 @@ static Value createMinMaxF(OpBuilder &builder, Location loc, Value lhs,
   Type i1Type = builder.getI1Type();
   if (auto vecType = dyn_cast<VectorType>(lhs.getType()))
     i1Type = VectorType::get(vecType.getShape(), i1Type);
-  Value cmp = builder.create<LLVM::FCmpOp>(
+  Value cmp = LLVM::FCmpOp::create(builder,
       loc, i1Type, isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt,
       lhs, rhs);
-  Value sel = builder.create<LLVM::SelectOp>(loc, cmp, lhs, rhs);
-  Value isNan = builder.create<LLVM::FCmpOp>(
+  Value sel = LLVM::SelectOp::create(builder, loc, cmp, lhs, rhs);
+  Value isNan = LLVM::FCmpOp::create(builder,
       loc, i1Type, LLVM::FCmpPredicate::uno, lhs, rhs);
-  Value nan = builder.create<LLVM::ConstantOp>(
+  Value nan = LLVM::ConstantOp::create(builder,
       loc, lhs.getType(),
       builder.getFloatAttr(floatType,
                            APFloat::getQNaN(floatType.getFloatSemantics())));
-  return builder.create<LLVM::SelectOp>(loc, isNan, nan, sel);
+  return LLVM::SelectOp::create(builder, loc, isNan, nan, sel);
 }
 
 static Value createScalarOp(OpBuilder &builder, Location loc,
@@ -323,11 +323,11 @@ static Value createScalarOp(OpBuilder &builder, Location loc,
                             ArrayRef<Value> operands) {
   switch (op) {
   case gpu::MMAElementwiseOp::ADDF:
-    return builder.create<LLVM::FAddOp>(loc, operands[0].getType(), operands);
+    return LLVM::FAddOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::MULF:
-    return builder.create<LLVM::FMulOp>(loc, operands[0].getType(), operands);
+    return LLVM::FMulOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::DIVF:
-    return builder.create<LLVM::FDivOp>(loc, operands[0].getType(), operands);
+    return LLVM::FDivOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::MAXF:
     return createMinMaxF(builder, loc, operands[0], operands[1],
                          /*isMin=*/false);
@@ -356,18 +356,18 @@ struct WmmaElementwiseOpToNVVMLowering
     size_t numOperands = adaptor.getOperands().size();
     LLVM::LLVMStructType destType = convertMMAToLLVMType(
         cast<gpu::MMAMatrixType>(subgroupMmaElementwiseOp.getType()));
-    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, destType);
+    Value matrixStruct = LLVM::PoisonOp::create(rewriter, loc, destType);
     for (size_t i = 0, e = destType.getBody().size(); i < e; ++i) {
       SmallVector<Value> extractedOperands;
       for (size_t opIdx = 0; opIdx < numOperands; opIdx++) {
-        extractedOperands.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
             loc, adaptor.getOperands()[opIdx], i));
       }
       Value element =
           createScalarOp(rewriter, loc, subgroupMmaElementwiseOp.getOpType(),
                          extractedOperands);
       matrixStruct =
-          rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, element, i);
+          LLVM::InsertValueOp::create(rewriter, loc, matrixStruct, element, i);
     }
     rewriter.replaceOp(subgroupMmaElementwiseOp, matrixStruct);
     return success();
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 456bfaba980ca..9b43d3ef602d1 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -61,10 +61,10 @@ static Value truncOrExtToLLVMType(ConversionPatternRewriter &rewriter,
       IntegerType::get(rewriter.getContext(), converter.getIndexTypeBitwidth());
   // TODO: use <=> in C++20.
   if (indexBitwidth > intWidth) {
-    return rewriter.create<LLVM::SExtOp>(loc, indexBitwidthType, value);
+    return LLVM::SExtOp::create(rewriter, loc, indexBitwidthType, value);
   }
   if (indexBitwidth < intWidth) {
-    return rewriter.create<LLVM::TruncOp>(loc, indexBitwidthType, value);
+    return LLVM::TruncOp::create(rewriter, loc, indexBitwidthType, value);
   }
   return value;
 }
@@ -82,11 +82,11 @@ static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func) {
 static Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
                        const unsigned indexBitwidth) {
   auto int32Type = IntegerType::get(rewriter.getContext(), 32);
-  Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
-  Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
-  Value mbcntLo = rewriter.create<ROCDL::MbcntLoOp>(loc, int32Type,
+  Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
+  Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
+  Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, int32Type,
                                                     ValueRange{minus1, zero});
-  Value laneId = rewriter.create<ROCDL::MbcntHiOp>(loc, int32Type,
+  Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, int32Type,
                                                    ValueRange{minus1, mbcntLo});
   return laneId;
 }
@@ -110,20 +110,20 @@ struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
     // followed by: %lid = call @llvm.amdgcn.mbcnt.hi(-1, %mlo)
 
     Type intTy = IntegerType::get(context, 32);
-    Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
-    Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
+    Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
+    Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
     Value mbcntLo =
-        rewriter.create<ROCDL::MbcntLoOp>(loc, intTy, ValueRange{minus1, zero});
-    Value laneId = rewriter.create<ROCDL::MbcntHiOp>(
+        ROCDL::MbcntLoOp::create(rewriter, loc, intTy, ValueRange{minus1, zero});
+    Value laneId = ROCDL::MbcntHiOp::create(rewriter,
         loc, intTy, ValueRange{minus1, mbcntLo});
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      laneId = rewriter.create<LLVM::SExtOp>(
+      laneId = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), laneId);
     } else if (indexBitwidth < 32) {
-      laneId = rewriter.create<LLVM::TruncOp>(
+      laneId = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), laneId);
     }
     rewriter.replaceOp(op, {laneId});
@@ -149,7 +149,7 @@ struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
           /*bitWidth=*/32, /*lower=*/isBeforeGfx10 ? 64 : 32,
           /*upper=*/op.getUpperBoundAttr().getInt() + 1);
     }
-    Value wavefrontOp = rewriter.create<ROCDL::WavefrontSizeOp>(
+    Value wavefrontOp = ROCDL::WavefrontSizeOp::create(rewriter,
         op.getLoc(), rewriter.getI32Type(), bounds);
     wavefrontOp = truncOrExtToLLVMType(rewriter, op.getLoc(), wavefrontOp,
                                        *getTypeConverter());
@@ -190,43 +190,43 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
 
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     Value width = adaptor.getWidth();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 0);
-    Value negwidth = rewriter.create<LLVM::SubOp>(loc, int32Type, zero, width);
-    Value add = rewriter.create<LLVM::AddOp>(loc, int32Type, srcLaneId, width);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, int32Type, 0);
+    Value negwidth = LLVM::SubOp::create(rewriter, loc, int32Type, zero, width);
+    Value add = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId, width);
     Value widthOrZeroIfOutside =
-        rewriter.create<LLVM::AndOp>(loc, int32Type, add, negwidth);
+        LLVM::AndOp::create(rewriter, loc, int32Type, add, negwidth);
     Value dstLane;
 
     switch (op.getMode()) {
     case gpu::ShuffleMode::UP:
-      dstLane = rewriter.create<LLVM::SubOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::SubOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::DOWN:
-      dstLane = rewriter.create<LLVM::AddOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::XOR:
-      dstLane = rewriter.create<LLVM::XOrOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::XOrOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::IDX:
       dstLane = adaptor.getOffset();
       break;
     }
-    Value isActiveSrcLane = rewriter.create<LLVM::ICmpOp>(
+    Value isActiveSrcLane = LLVM::ICmpOp::create(rewriter,
         loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
-    Value selectDstLane = rewriter.create<LLVM::SelectOp>(loc, isActiveSrcLane,
+    Value selectDstLane = LLVM::SelectOp::create(rewriter, loc, isActiveSrcLane,
                                                           dstLane, srcLaneId);
-    Value two = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 2);
+    Value two = LLVM::ConstantOp::create(rewriter, loc, int32Type, 2);
     Value dwordAlignedDstLane =
-        rewriter.create<LLVM::ShlOp>(loc, int32Type, selectDstLane, two);
+        LLVM::ShlOp::create(rewriter, loc, int32Type, selectDstLane, two);
 
     SmallVector<Value> decomposed =
         LLVM::decomposeValue(rewriter, loc, initShflValue, int32Type);
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
-      Value res = rewriter.create<ROCDL::DsBpermuteOp>(loc, int32Type,
+      Value res = ROCDL::DsBpermuteOp::create(rewriter, loc, int32Type,
                                                        dwordAlignedDstLane, v);
       swizzled.emplace_back(res);
     }
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index b99ed261ecfa3..71618cbf8a2f2 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -169,11 +169,11 @@ LogicalResult LaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
 
   Value vector =
       spirv::getBuiltinVariableValue(op, builtin, builtinType, rewriter);
-  Value dim = rewriter.create<spirv::CompositeExtractOp>(
+  Value dim = spirv::CompositeExtractOp::create(rewriter,
       op.getLoc(), builtinType, vector,
       rewriter.getI32ArrayAttr({static_cast<int32_t>(op.getDimension())}));
   if (forShader && builtinType != indexType)
-    dim = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType, dim);
+    dim = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType, dim);
   rewriter.replaceOp(op, dim);
   return success();
 }
@@ -198,7 +198,7 @@ SingleDimLaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
   Value builtinValue =
       spirv::getBuiltinVariableValue(op, builtin, i32Type, rewriter);
   if (i32Type != indexType)
-    builtinValue = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType,
+    builtinValue = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType,
                                                       builtinValue);
   rewriter.replaceOp(op, builtinValue);
   return success();
@@ -257,7 +257,7 @@ lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter,
       signatureConverter.addInputs(argType.index(), convertedType);
     }
   }
-  auto newFuncOp = rewriter.create<spirv::FuncOp>(
+  auto newFuncOp = spirv::FuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(signatureConverter.getConvertedTypes(), {}));
   for (const auto &namedAttr : funcOp->getAttrs()) {
@@ -367,7 +367,7 @@ LogicalResult GPUModuleConversion::matchAndRewrite(
 
   // Add a keyword to the module name to avoid symbolic conflict.
   std::string spvModuleName = (kSPIRVModule + moduleOp.getName()).str();
-  auto spvModule = rewriter.create<spirv::ModuleOp>(
+  auto spvModule = spirv::ModuleOp::create(rewriter,
       moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
       StringRef(spvModuleName));
 
@@ -452,41 +452,41 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
 
   switch (shuffleOp.getMode()) {
   case gpu::ShuffleMode::XOR: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleXorOp>(
+    result = spirv::GroupNonUniformShuffleXorOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::IDX: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleOp>(
+    result = spirv::GroupNonUniformShuffleOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::DOWN: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleDownOp>(
+    result = spirv::GroupNonUniformShuffleDownOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
 
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
-        rewriter.create<arith::AddIOp>(loc, laneId, adaptor.getOffset());
-    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+        arith::AddIOp::create(rewriter, loc, laneId, adaptor.getOffset());
+    validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                               resultLaneId, adaptor.getWidth());
     break;
   }
   case gpu::ShuffleMode::UP: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleUpOp>(
+    result = spirv::GroupNonUniformShuffleUpOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
 
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
-        rewriter.create<arith::SubIOp>(loc, laneId, adaptor.getOffset());
+        arith::SubIOp::create(rewriter, loc, laneId, adaptor.getOffset());
     auto i32Type = rewriter.getIntegerType(32);
-    validVal = rewriter.create<arith::CmpIOp>(
+    validVal = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, resultLaneId,
-        rewriter.create<arith::ConstantOp>(
+        arith::ConstantOp::create(rewriter,
             loc, i32Type, rewriter.getIntegerAttr(i32Type, 0)));
     break;
   }
@@ -516,14 +516,14 @@ LogicalResult GPURotateConversion::matchAndRewrite(
 
   Location loc = rotateOp.getLoc();
   auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
-  Value rotateResult = rewriter.create<spirv::GroupNonUniformRotateKHROp>(
+  Value rotateResult = spirv::GroupNonUniformRotateKHROp::create(rewriter,
       loc, scope, adaptor.getValue(), adaptor.getOffset(), adaptor.getWidth());
   Value validVal;
   if (widthAttr.getValue().getZExtValue() == subgroupSize) {
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(), loc, rewriter);
   } else {
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
-    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
+    validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                               laneId, adaptor.getWidth());
   }
 
@@ -548,13 +548,13 @@ static Value createGroupReduceOpImpl(OpBuilder &builder, Location loc,
                                 ? spirv::GroupOperation::ClusteredReduce
                                 : spirv::GroupOperation::Reduce);
   if (isUniform) {
-    return builder.create<UniformOp>(loc, type, scope, groupOp, arg)
+    return UniformOp::create(builder, loc, type, scope, groupOp, arg)
         .getResult();
   }
 
   Value clusterSizeValue;
   if (clusterSize.has_value())
-    clusterSizeValue = builder.create<spirv::ConstantOp>(
+    clusterSizeValue = spirv::ConstantOp::create(builder,
         loc, builder.getI32Type(),
         builder.getIntegerAttr(builder.getI32Type(), *clusterSize));
 
@@ -740,7 +740,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstName =
         makeVarName(moduleOp, llvm::Twine(globalVarName) + "_sc");
 
-    return rewriter.create<spirv::SpecConstantOp>(
+    return spirv::SpecConstantOp::create(rewriter,
         loc, rewriter.getStringAttr(specCstName), attr);
   };
   {
@@ -774,7 +774,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstCompositeName =
         (llvm::Twine(globalVarName) + "_scc").str();
 
-    specCstComposite = rewriter.create<spirv::SpecConstantCompositeOp>(
+    specCstComposite = spirv::SpecConstantCompositeOp::create(rewriter,
         loc, TypeAttr::get(globalType),
         rewriter.getStringAttr(specCstCompositeName),
         rewriter.getArrayAttr(constituents));
@@ -785,15 +785,15 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     // Define a GlobalVarOp initialized using specialized constants
     // that is used to specify the printf format string
     // to be passed to the SPIRV CLPrintfOp.
-    globalVar = rewriter.create<spirv::GlobalVariableOp>(
+    globalVar = spirv::GlobalVariableOp::create(rewriter,
         loc, ptrType, globalVarName, FlatSymbolRefAttr::get(specCstComposite));
 
     globalVar->setAttr("Constant", rewriter.getUnitAttr());
   }
   // Get SSA value of Global variable and create pointer to i8 to point to
   // the format string.
-  Value globalPtr = rewriter.create<spirv::AddressOfOp>(loc, globalVar);
-  Value fmtStr = rewriter.create<spirv::BitcastOp>(
+  Value globalPtr = spirv::AddressOfOp::create(rewriter, loc, globalVar);
+  Value fmtStr = spirv::BitcastOp::create(rewriter,
       loc,
       spirv::PointerType::get(i8Type, spirv::StorageClass::UniformConstant),
       globalPtr);
@@ -801,7 +801,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
   // Get printf arguments.
   auto printfArgs = llvm::to_vector_of<Value, 4>(adaptor.getArgs());
 
-  rewriter.create<spirv::CLPrintfOp>(loc, i32Type, fmtStr, printfArgs);
+  spirv::CLPrintfOp::create(rewriter, loc, i32Type, fmtStr, printfArgs);
 
   // Need to erase the gpu.printf op as gpu.printf does not use result vs
   // spirv::CLPrintfOp has i32 resultType so cannot replace with new SPIR-V
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index 0b2c06a08db2d..da8ffe47a7d33 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -144,11 +144,11 @@ void GPUToSPIRVPass::runOnOperation() {
     if (targetEnvSupportsKernelCapability(moduleOp)) {
       moduleOp.walk([&](gpu::GPUFuncOp funcOp) {
         builder.setInsertionPoint(funcOp);
-        auto newFuncOp = builder.create<func::FuncOp>(
+        auto newFuncOp = func::FuncOp::create(builder,
             funcOp.getLoc(), funcOp.getName(), funcOp.getFunctionType());
         auto entryBlock = newFuncOp.addEntryBlock();
         builder.setInsertionPointToEnd(entryBlock);
-        builder.create<func::ReturnOp>(funcOp.getLoc());
+        func::ReturnOp::create(builder, funcOp.getLoc());
         newFuncOp->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                            builder.getUnitAttr());
         funcOp.erase();
diff --git a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
index 7bb86b5ce1ddd..70646aa5f2e74 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
@@ -283,7 +283,7 @@ struct WmmaLoadOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = rewriter.create<spirv::ConstantOp>(
+    auto strideValue = spirv::ConstantOp::create(rewriter,
         loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
@@ -315,7 +315,7 @@ struct WmmaStoreOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = rewriter.create<spirv::ConstantOp>(
+    auto strideValue = spirv::ConstantOp::create(rewriter,
         loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
diff --git a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
index 0473bb59fa6aa..0d5b318f739cc 100644
--- a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
+++ b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
@@ -36,34 +36,34 @@ struct ConvertIndexCeilDivS : mlir::ConvertOpToLLVMPattern<CeilDivSOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value posOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
-    Value negOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), -1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value posOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
+    Value negOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), -1);
 
     // Compute `x`.
     Value mPos =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::sgt, m, zero);
-    Value x = rewriter.create<LLVM::SelectOp>(loc, mPos, negOne, posOne);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, m, zero);
+    Value x = LLVM::SelectOp::create(rewriter, loc, mPos, negOne, posOne);
 
     // Compute the positive result.
-    Value nPlusX = rewriter.create<LLVM::AddOp>(loc, n, x);
-    Value nPlusXDivM = rewriter.create<LLVM::SDivOp>(loc, nPlusX, m);
-    Value posRes = rewriter.create<LLVM::AddOp>(loc, nPlusXDivM, posOne);
+    Value nPlusX = LLVM::AddOp::create(rewriter, loc, n, x);
+    Value nPlusXDivM = LLVM::SDivOp::create(rewriter, loc, nPlusX, m);
+    Value posRes = LLVM::AddOp::create(rewriter, loc, nPlusXDivM, posOne);
 
     // Compute the negative result.
-    Value negN = rewriter.create<LLVM::SubOp>(loc, zero, n);
-    Value negNDivM = rewriter.create<LLVM::SDivOp>(loc, negN, m);
-    Value negRes = rewriter.create<LLVM::SubOp>(loc, zero, negNDivM);
+    Value negN = LLVM::SubOp::create(rewriter, loc, zero, n);
+    Value negNDivM = LLVM::SDivOp::create(rewriter, loc, negN, m);
+    Value negRes = LLVM::SubOp::create(rewriter, loc, zero, negNDivM);
 
     // Pick the positive result if `n` and `m` have the same sign and `n` is
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
     Value nPos =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::sgt, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, n, zero);
     Value sameSign =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, nPos, mPos);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, nPos, mPos);
     Value nNonZero =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, n, zero);
-    Value cmp = rewriter.create<LLVM::AndOp>(loc, sameSign, nNonZero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
+    Value cmp = LLVM::AndOp::create(rewriter, loc, sameSign, nNonZero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
@@ -83,17 +83,17 @@ struct ConvertIndexCeilDivU : mlir::ConvertOpToLLVMPattern<CeilDivUOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value one = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value one = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
 
     // Compute the non-zero result.
-    Value minusOne = rewriter.create<LLVM::SubOp>(loc, n, one);
-    Value quotient = rewriter.create<LLVM::UDivOp>(loc, minusOne, m);
-    Value plusOne = rewriter.create<LLVM::AddOp>(loc, quotient, one);
+    Value minusOne = LLVM::SubOp::create(rewriter, loc, n, one);
+    Value quotient = LLVM::UDivOp::create(rewriter, loc, minusOne, m);
+    Value plusOne = LLVM::AddOp::create(rewriter, loc, quotient, one);
 
     // Pick the result.
     Value cmp =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, n, zero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, zero, plusOne);
     return success();
   }
@@ -114,32 +114,32 @@ struct ConvertIndexFloorDivS : mlir::ConvertOpToLLVMPattern<FloorDivSOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value posOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
-    Value negOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), -1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value posOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
+    Value negOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), -1);
 
     // Compute `x`.
     Value mNeg =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::slt, m, zero);
-    Value x = rewriter.create<LLVM::SelectOp>(loc, mNeg, posOne, negOne);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, m, zero);
+    Value x = LLVM::SelectOp::create(rewriter, loc, mNeg, posOne, negOne);
 
     // Compute the negative result.
-    Value xMinusN = rewriter.create<LLVM::SubOp>(loc, x, n);
-    Value xMinusNDivM = rewriter.create<LLVM::SDivOp>(loc, xMinusN, m);
-    Value negRes = rewriter.create<LLVM::SubOp>(loc, negOne, xMinusNDivM);
+    Value xMinusN = LLVM::SubOp::create(rewriter, loc, x, n);
+    Value xMinusNDivM = LLVM::SDivOp::create(rewriter, loc, xMinusN, m);
+    Value negRes = LLVM::SubOp::create(rewriter, loc, negOne, xMinusNDivM);
 
     // Compute the positive result.
-    Value posRes = rewriter.create<LLVM::SDivOp>(loc, n, m);
+    Value posRes = LLVM::SDivOp::create(rewriter, loc, n, m);
 
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
     Value nNeg =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::slt, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, n, zero);
     Value diffSign =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
     Value nNonZero =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, n, zero);
-    Value cmp = rewriter.create<LLVM::AndOp>(loc, diffSign, nNonZero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
+    Value cmp = LLVM::AndOp::create(rewriter, loc, diffSign, nNonZero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, negRes, posRes);
     return success();
   }
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index 4821962f989e6..6fc097059c5e5 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -111,33 +111,33 @@ struct ConvertIndexCeilDivSPattern final : OpConversionPattern<CeilDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = rewriter.create<spirv::ConstantOp>(
+    Value posOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = rewriter.create<spirv::ConstantOp>(
+    Value negOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
-    Value mPos = rewriter.create<spirv::SGreaterThanOp>(loc, m, zero);
-    Value x = rewriter.create<spirv::SelectOp>(loc, mPos, negOne, posOne);
+    Value mPos = spirv::SGreaterThanOp::create(rewriter, loc, m, zero);
+    Value x = spirv::SelectOp::create(rewriter, loc, mPos, negOne, posOne);
 
     // Compute the positive result.
-    Value nPlusX = rewriter.create<spirv::IAddOp>(loc, n, x);
-    Value nPlusXDivM = rewriter.create<spirv::SDivOp>(loc, nPlusX, m);
-    Value posRes = rewriter.create<spirv::IAddOp>(loc, nPlusXDivM, posOne);
+    Value nPlusX = spirv::IAddOp::create(rewriter, loc, n, x);
+    Value nPlusXDivM = spirv::SDivOp::create(rewriter, loc, nPlusX, m);
+    Value posRes = spirv::IAddOp::create(rewriter, loc, nPlusXDivM, posOne);
 
     // Compute the negative result.
-    Value negN = rewriter.create<spirv::ISubOp>(loc, zero, n);
-    Value negNDivM = rewriter.create<spirv::SDivOp>(loc, negN, m);
-    Value negRes = rewriter.create<spirv::ISubOp>(loc, zero, negNDivM);
+    Value negN = spirv::ISubOp::create(rewriter, loc, zero, n);
+    Value negNDivM = spirv::SDivOp::create(rewriter, loc, negN, m);
+    Value negRes = spirv::ISubOp::create(rewriter, loc, zero, negNDivM);
 
     // Pick the positive result if `n` and `m` have the same sign and `n` is
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
-    Value nPos = rewriter.create<spirv::SGreaterThanOp>(loc, n, zero);
-    Value sameSign = rewriter.create<spirv::LogicalEqualOp>(loc, nPos, mPos);
-    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
-    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, sameSign, nNonZero);
+    Value nPos = spirv::SGreaterThanOp::create(rewriter, loc, n, zero);
+    Value sameSign = spirv::LogicalEqualOp::create(rewriter, loc, nPos, mPos);
+    Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
+    Value cmp = spirv::LogicalAndOp::create(rewriter, loc, sameSign, nNonZero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
@@ -161,18 +161,18 @@ struct ConvertIndexCeilDivUPattern final : OpConversionPattern<CeilDivUOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value one = rewriter.create<spirv::ConstantOp>(loc, n_type,
+    Value one = spirv::ConstantOp::create(rewriter, loc, n_type,
                                                    IntegerAttr::get(n_type, 1));
 
     // Compute the non-zero result.
-    Value minusOne = rewriter.create<spirv::ISubOp>(loc, n, one);
-    Value quotient = rewriter.create<spirv::UDivOp>(loc, minusOne, m);
-    Value plusOne = rewriter.create<spirv::IAddOp>(loc, quotient, one);
+    Value minusOne = spirv::ISubOp::create(rewriter, loc, n, one);
+    Value quotient = spirv::UDivOp::create(rewriter, loc, minusOne, m);
+    Value plusOne = spirv::IAddOp::create(rewriter, loc, quotient, one);
 
     // Pick the result
-    Value cmp = rewriter.create<spirv::IEqualOp>(loc, n, zero);
+    Value cmp = spirv::IEqualOp::create(rewriter, loc, n, zero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, zero, plusOne);
     return success();
   }
@@ -197,32 +197,32 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = rewriter.create<spirv::ConstantOp>(
+    Value posOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = rewriter.create<spirv::ConstantOp>(
+    Value negOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
-    Value mNeg = rewriter.create<spirv::SLessThanOp>(loc, m, zero);
-    Value x = rewriter.create<spirv::SelectOp>(loc, mNeg, posOne, negOne);
+    Value mNeg = spirv::SLessThanOp::create(rewriter, loc, m, zero);
+    Value x = spirv::SelectOp::create(rewriter, loc, mNeg, posOne, negOne);
 
     // Compute the negative result
-    Value xMinusN = rewriter.create<spirv::ISubOp>(loc, x, n);
-    Value xMinusNDivM = rewriter.create<spirv::SDivOp>(loc, xMinusN, m);
-    Value negRes = rewriter.create<spirv::ISubOp>(loc, negOne, xMinusNDivM);
+    Value xMinusN = spirv::ISubOp::create(rewriter, loc, x, n);
+    Value xMinusNDivM = spirv::SDivOp::create(rewriter, loc, xMinusN, m);
+    Value negRes = spirv::ISubOp::create(rewriter, loc, negOne, xMinusNDivM);
 
     // Compute the positive result.
-    Value posRes = rewriter.create<spirv::SDivOp>(loc, n, m);
+    Value posRes = spirv::SDivOp::create(rewriter, loc, n, m);
 
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
-    Value nNeg = rewriter.create<spirv::SLessThanOp>(loc, n, zero);
-    Value diffSign = rewriter.create<spirv::LogicalNotEqualOp>(loc, nNeg, mNeg);
-    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
+    Value nNeg = spirv::SLessThanOp::create(rewriter, loc, n, zero);
+    Value diffSign = spirv::LogicalNotEqualOp::create(rewriter, loc, nNeg, mNeg);
+    Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
 
-    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, diffSign, nNonZero);
+    Value cmp = spirv::LogicalAndOp::create(rewriter, loc, diffSign, nNonZero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index e34d5f74d232f..e8ad0d4644389 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -32,7 +32,7 @@ MemRefDescriptor::MemRefDescriptor(Value descriptor)
 MemRefDescriptor MemRefDescriptor::poison(OpBuilder &builder, Location loc,
                                           Type descriptorType) {
 
-  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
+  Value descriptor = LLVM::PoisonOp::create(builder, loc, descriptorType);
   return MemRefDescriptor(descriptor);
 }
 
@@ -99,20 +99,20 @@ void MemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc,
 // integer attribute.
 static Value createIndexAttrConstant(OpBuilder &builder, Location loc,
                                      Type resultType, int64_t value) {
-  return builder.create<LLVM::ConstantOp>(loc, resultType,
+  return LLVM::ConstantOp::create(builder, loc, resultType,
                                           builder.getIndexAttr(value));
 }
 
 /// Builds IR extracting the offset from the descriptor.
 Value MemRefDescriptor::offset(OpBuilder &builder, Location loc) {
-  return builder.create<LLVM::ExtractValueOp>(loc, value,
+  return LLVM::ExtractValueOp::create(builder, loc, value,
                                               kOffsetPosInMemRefDescriptor);
 }
 
 /// Builds IR inserting the offset into the descriptor.
 void MemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                  Value offset) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, offset,
+  value = LLVM::InsertValueOp::create(builder, loc, value, offset,
                                               kOffsetPosInMemRefDescriptor);
 }
 
@@ -125,7 +125,7 @@ void MemRefDescriptor::setConstantOffset(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th size from the descriptor.
 Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) {
-  return builder.create<LLVM::ExtractValueOp>(
+  return LLVM::ExtractValueOp::create(builder,
       loc, value, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
@@ -137,22 +137,22 @@ Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
 
   // Copy size values to stack-allocated memory.
   auto one = createIndexAttrConstant(builder, loc, indexType, 1);
-  auto sizes = builder.create<LLVM::ExtractValueOp>(
+  auto sizes = LLVM::ExtractValueOp::create(builder,
       loc, value, llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
-  auto sizesPtr = builder.create<LLVM::AllocaOp>(loc, ptrTy, arrayTy, one,
+  auto sizesPtr = LLVM::AllocaOp::create(builder, loc, ptrTy, arrayTy, one,
                                                  /*alignment=*/0);
-  builder.create<LLVM::StoreOp>(loc, sizes, sizesPtr);
+  LLVM::StoreOp::create(builder, loc, sizes, sizesPtr);
 
   // Load an return size value of interest.
-  auto resultPtr = builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, sizesPtr,
+  auto resultPtr = LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, sizesPtr,
                                                ArrayRef<LLVM::GEPArg>{0, pos});
-  return builder.create<LLVM::LoadOp>(loc, indexType, resultPtr);
+  return LLVM::LoadOp::create(builder, loc, indexType, resultPtr);
 }
 
 /// Builds IR inserting the pos-th size into the descriptor
 void MemRefDescriptor::setSize(OpBuilder &builder, Location loc, unsigned pos,
                                Value size) {
-  value = builder.create<LLVM::InsertValueOp>(
+  value = LLVM::InsertValueOp::create(builder,
       loc, value, size, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
@@ -164,14 +164,14 @@ void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th stride from the descriptor.
 Value MemRefDescriptor::stride(OpBuilder &builder, Location loc, unsigned pos) {
-  return builder.create<LLVM::ExtractValueOp>(
+  return LLVM::ExtractValueOp::create(builder,
       loc, value, ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
 
 /// Builds IR inserting the pos-th stride into the descriptor
 void MemRefDescriptor::setStride(OpBuilder &builder, Location loc, unsigned pos,
                                  Value stride) {
-  value = builder.create<LLVM::InsertValueOp>(
+  value = LLVM::InsertValueOp::create(builder,
       loc, value, stride,
       ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
@@ -207,7 +207,7 @@ Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
           ? offset(builder, loc)
           : createIndexAttrConstant(builder, loc, indexType, offsetCst);
   Type elementType = converter.convertType(type.getElementType());
-  ptr = builder.create<LLVM::GEPOp>(loc, ptr.getType(), elementType, ptr,
+  ptr = LLVM::GEPOp::create(builder, loc, ptr.getType(), elementType, ptr,
                                     offsetVal);
   return ptr;
 }
@@ -303,7 +303,7 @@ UnrankedMemRefDescriptor::UnrankedMemRefDescriptor(Value descriptor)
 UnrankedMemRefDescriptor UnrankedMemRefDescriptor::poison(OpBuilder &builder,
                                                           Location loc,
                                                           Type descriptorType) {
-  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
+  Value descriptor = LLVM::PoisonOp::create(builder, loc, descriptorType);
   return UnrankedMemRefDescriptor(descriptor);
 }
 Value UnrankedMemRefDescriptor::rank(OpBuilder &builder, Location loc) const {
@@ -380,18 +380,18 @@ void UnrankedMemRefDescriptor::computeSizes(
         builder, loc, indexType,
         llvm::divideCeil(typeConverter.getPointerBitwidth(addressSpace), 8));
     Value doublePointerSize =
-        builder.create<LLVM::MulOp>(loc, indexType, two, pointerSize);
+        LLVM::MulOp::create(builder, loc, indexType, two, pointerSize);
 
     // (1 + 2 * rank) * sizeof(index)
     Value rank = desc.rank(builder, loc);
-    Value doubleRank = builder.create<LLVM::MulOp>(loc, indexType, two, rank);
+    Value doubleRank = LLVM::MulOp::create(builder, loc, indexType, two, rank);
     Value doubleRankIncremented =
-        builder.create<LLVM::AddOp>(loc, indexType, doubleRank, one);
-    Value rankIndexSize = builder.create<LLVM::MulOp>(
+        LLVM::AddOp::create(builder, loc, indexType, doubleRank, one);
+    Value rankIndexSize = LLVM::MulOp::create(builder,
         loc, indexType, doubleRankIncremented, indexSize);
 
     // Total allocation size.
-    Value allocationSize = builder.create<LLVM::AddOp>(
+    Value allocationSize = LLVM::AddOp::create(builder,
         loc, indexType, doublePointerSize, rankIndexSize);
     sizes.push_back(allocationSize);
   }
@@ -400,13 +400,13 @@ void UnrankedMemRefDescriptor::computeSizes(
 Value UnrankedMemRefDescriptor::allocatedPtr(
     OpBuilder &builder, Location loc, Value memRefDescPtr,
     LLVM::LLVMPointerType elemPtrType) {
-  return builder.create<LLVM::LoadOp>(loc, elemPtrType, memRefDescPtr);
+  return LLVM::LoadOp::create(builder, loc, elemPtrType, memRefDescPtr);
 }
 
 void UnrankedMemRefDescriptor::setAllocatedPtr(
     OpBuilder &builder, Location loc, Value memRefDescPtr,
     LLVM::LLVMPointerType elemPtrType, Value allocatedPtr) {
-  builder.create<LLVM::StoreOp>(loc, allocatedPtr, memRefDescPtr);
+  LLVM::StoreOp::create(builder, loc, allocatedPtr, memRefDescPtr);
 }
 
 static std::pair<Value, Type>
@@ -423,9 +423,9 @@ Value UnrankedMemRefDescriptor::alignedPtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   Value alignedGep =
-      builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+      LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                   elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
-  return builder.create<LLVM::LoadOp>(loc, elemPtrType, alignedGep);
+  return LLVM::LoadOp::create(builder, loc, elemPtrType, alignedGep);
 }
 
 void UnrankedMemRefDescriptor::setAlignedPtr(
@@ -435,9 +435,9 @@ void UnrankedMemRefDescriptor::setAlignedPtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   Value alignedGep =
-      builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+      LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                   elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
-  builder.create<LLVM::StoreOp>(loc, alignedPtr, alignedGep);
+  LLVM::StoreOp::create(builder, loc, alignedPtr, alignedGep);
 }
 
 Value UnrankedMemRefDescriptor::offsetBasePtr(
@@ -446,7 +446,7 @@ Value UnrankedMemRefDescriptor::offsetBasePtr(
   auto [elementPtrPtr, elemPtrPtrType] =
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
-  return builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+  return LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                      elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
 }
 
@@ -456,7 +456,7 @@ Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
                                        LLVM::LLVMPointerType elemPtrType) {
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
-  return builder.create<LLVM::LoadOp>(loc, typeConverter.getIndexType(),
+  return LLVM::LoadOp::create(builder, loc, typeConverter.getIndexType(),
                                       offsetPtr);
 }
 
@@ -467,7 +467,7 @@ void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                          Value offset) {
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
-  builder.create<LLVM::StoreOp>(loc, offset, offsetPtr);
+  LLVM::StoreOp::create(builder, loc, offset, offsetPtr);
 }
 
 Value UnrankedMemRefDescriptor::sizeBasePtr(
@@ -477,7 +477,7 @@ Value UnrankedMemRefDescriptor::sizeBasePtr(
   Type structTy = LLVM::LLVMStructType::getLiteral(
       indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy});
   auto resultType = LLVM::LLVMPointerType::get(builder.getContext());
-  return builder.create<LLVM::GEPOp>(loc, resultType, structTy, memRefDescPtr,
+  return LLVM::GEPOp::create(builder, loc, resultType, structTy, memRefDescPtr,
                                      ArrayRef<LLVM::GEPArg>{0, 3});
 }
 
@@ -489,8 +489,8 @@ Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value sizeStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, index);
-  return builder.create<LLVM::LoadOp>(loc, indexTy, sizeStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, index);
+  return LLVM::LoadOp::create(builder, loc, indexTy, sizeStoreGep);
 }
 
 void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
@@ -501,8 +501,8 @@ void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value sizeStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, index);
-  builder.create<LLVM::StoreOp>(loc, size, sizeStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, index);
+  LLVM::StoreOp::create(builder, loc, size, sizeStoreGep);
 }
 
 Value UnrankedMemRefDescriptor::strideBasePtr(
@@ -511,7 +511,7 @@ Value UnrankedMemRefDescriptor::strideBasePtr(
   Type indexTy = typeConverter.getIndexType();
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
-  return builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, rank);
+  return LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, rank);
 }
 
 Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
@@ -522,8 +522,8 @@ Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value strideStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, strideBasePtr, index);
-  return builder.create<LLVM::LoadOp>(loc, indexTy, strideStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, strideBasePtr, index);
+  return LLVM::LoadOp::create(builder, loc, indexTy, strideStoreGep);
 }
 
 void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
@@ -534,6 +534,6 @@ void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value strideStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, strideBasePtr, index);
-  builder.create<LLVM::StoreOp>(loc, stride, strideStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, strideBasePtr, index);
+  LLVM::StoreOp::create(builder, loc, stride, strideStoreGep);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index c5f72f7e10b8c..57aed79874ad0 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -57,7 +57,7 @@ Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder,
                                                     Location loc,
                                                     Type resultType,
                                                     int64_t value) {
-  return builder.create<LLVM::ConstantOp>(loc, resultType,
+  return LLVM::ConstantOp::create(builder, loc, resultType,
                                           builder.getIndexAttr(value));
 }
 
@@ -123,7 +123,7 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
       runningStride = sizes[i];
     else if (stride == ShapedType::kDynamic)
       runningStride =
-          rewriter.create<LLVM::MulOp>(loc, runningStride, sizes[i]);
+          LLVM::MulOp::create(rewriter, loc, runningStride, sizes[i]);
     else
       runningStride = createIndexAttrConstant(rewriter, loc, indexType, stride);
   }
@@ -131,10 +131,10 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
     // Buffer size in bytes.
     Type elementType = typeConverter->convertType(memRefType.getElementType());
     auto elementPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, elementPtrType);
-    Value gepPtr = rewriter.create<LLVM::GEPOp>(
+    Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
+    Value gepPtr = LLVM::GEPOp::create(rewriter,
         loc, elementPtrType, elementType, nullPtr, runningStride);
-    size = rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);
+    size = LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
   } else {
     size = runningStride;
   }
@@ -149,10 +149,10 @@ Value ConvertToLLVMPattern::getSizeInBytes(
   // which is a common pattern of getting the size of a type in bytes.
   Type llvmType = typeConverter->convertType(type);
   auto convertedPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-  auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, convertedPtrType);
-  auto gep = rewriter.create<LLVM::GEPOp>(loc, convertedPtrType, llvmType,
+  auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, convertedPtrType);
+  auto gep = LLVM::GEPOp::create(rewriter, loc, convertedPtrType, llvmType,
                                           nullPtr, ArrayRef<LLVM::GEPArg>{1});
-  return rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gep);
+  return LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gep);
 }
 
 Value ConvertToLLVMPattern::getNumElements(
@@ -175,7 +175,7 @@ Value ConvertToLLVMPattern::getNumElements(
           staticSize == ShapedType::kDynamic
               ? dynamicSizes[dynamicIndex++]
               : createIndexAttrConstant(rewriter, loc, indexType, staticSize);
-      numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
     } else {
       numElements =
           staticSize == ShapedType::kDynamic
@@ -276,14 +276,14 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
             ? builder
                   .create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
                   .getResult()
-            : builder.create<LLVM::AllocaOp>(loc, getPtrType(),
+            : LLVM::AllocaOp::create(builder, loc, getPtrType(),
                                              IntegerType::get(getContext(), 8),
                                              allocationSize,
                                              /*alignment=*/0);
     Value source = desc.memRefDescPtr(builder, loc);
-    builder.create<LLVM::MemcpyOp>(loc, memory, source, allocationSize, false);
+    LLVM::MemcpyOp::create(builder, loc, memory, source, allocationSize, false);
     if (!toDynamic)
-      builder.create<LLVM::CallOp>(loc, freeFunc.value(), source);
+      LLVM::CallOp::create(builder, loc, freeFunc.value(), source);
 
     // Create a new descriptor. The same descriptor can be returned multiple
     // times, attempting to modify its pointer can lead to memory leaks
@@ -349,7 +349,7 @@ LogicalResult LLVM::detail::oneToOneRewrite(
   SmallVector<Value, 4> results;
   results.reserve(numResults);
   for (unsigned i = 0; i < numResults; ++i) {
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+    results.push_back(LLVM::ExtractValueOp::create(rewriter,
         op->getLoc(), newOp->getResult(0), i));
   }
   rewriter.replaceOp(op, results);
@@ -371,7 +371,7 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   if (numResults != 0)
     resType = typeConverter.packOperationResults(op->getResultTypes());
 
-  auto callIntrOp = rewriter.create<LLVM::CallIntrinsicOp>(
+  auto callIntrOp = LLVM::CallIntrinsicOp::create(rewriter,
       loc, resType, rewriter.getStringAttr(intrinsic), operands);
   // Propagate attributes.
   callIntrOp->setAttrs(op->getAttrDictionary());
@@ -388,7 +388,7 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   results.reserve(numResults);
   Value intrRes = callIntrOp.getResults();
   for (unsigned i = 0; i < numResults; ++i)
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(loc, intrRes, i));
+    results.push_back(LLVM::ExtractValueOp::create(rewriter, loc, intrRes, i));
   rewriter.replaceOp(op, results);
 
   return success();
@@ -406,7 +406,7 @@ static unsigned getBitWidth(Type type) {
 static Value createI32Constant(OpBuilder &builder, Location loc,
                                int32_t value) {
   Type i32 = builder.getI32Type();
-  return builder.create<LLVM::ConstantOp>(loc, i32, value);
+  return LLVM::ConstantOp::create(builder, loc, i32, value);
 }
 
 SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
@@ -418,17 +418,17 @@ SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
   unsigned srcBitWidth = getBitWidth(srcType);
   unsigned dstBitWidth = getBitWidth(dstType);
   if (srcBitWidth == dstBitWidth) {
-    Value cast = builder.create<LLVM::BitcastOp>(loc, dstType, src);
+    Value cast = LLVM::BitcastOp::create(builder, loc, dstType, src);
     return {cast};
   }
 
   if (dstBitWidth > srcBitWidth) {
     auto smallerInt = builder.getIntegerType(srcBitWidth);
     if (srcType != smallerInt)
-      src = builder.create<LLVM::BitcastOp>(loc, smallerInt, src);
+      src = LLVM::BitcastOp::create(builder, loc, smallerInt, src);
 
     auto largerInt = builder.getIntegerType(dstBitWidth);
-    Value res = builder.create<LLVM::ZExtOp>(loc, largerInt, src);
+    Value res = LLVM::ZExtOp::create(builder, loc, largerInt, src);
     return {res};
   }
   assert(srcBitWidth % dstBitWidth == 0 &&
@@ -436,12 +436,12 @@ SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
   int64_t numElements = srcBitWidth / dstBitWidth;
   auto vecType = VectorType::get(numElements, dstType);
 
-  src = builder.create<LLVM::BitcastOp>(loc, vecType, src);
+  src = LLVM::BitcastOp::create(builder, loc, vecType, src);
 
   SmallVector<Value> res;
   for (auto i : llvm::seq(numElements)) {
     Value idx = createI32Constant(builder, loc, i);
-    Value elem = builder.create<LLVM::ExtractElementOp>(loc, src, idx);
+    Value elem = LLVM::ExtractElementOp::create(builder, loc, src, idx);
     res.emplace_back(elem);
   }
 
@@ -461,28 +461,28 @@ Value mlir::LLVM::composeValue(OpBuilder &builder, Location loc, ValueRange src,
     if (dstBitWidth < srcBitWidth) {
       auto largerInt = builder.getIntegerType(srcBitWidth);
       if (res.getType() != largerInt)
-        res = builder.create<LLVM::BitcastOp>(loc, largerInt, res);
+        res = LLVM::BitcastOp::create(builder, loc, largerInt, res);
 
       auto smallerInt = builder.getIntegerType(dstBitWidth);
-      res = builder.create<LLVM::TruncOp>(loc, smallerInt, res);
+      res = LLVM::TruncOp::create(builder, loc, smallerInt, res);
     }
 
     if (res.getType() != dstType)
-      res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+      res = LLVM::BitcastOp::create(builder, loc, dstType, res);
 
     return res;
   }
 
   int64_t numElements = src.size();
   auto srcType = VectorType::get(numElements, src.front().getType());
-  Value res = builder.create<LLVM::PoisonOp>(loc, srcType);
+  Value res = LLVM::PoisonOp::create(builder, loc, srcType);
   for (auto &&[i, elem] : llvm::enumerate(src)) {
     Value idx = createI32Constant(builder, loc, i);
-    res = builder.create<LLVM::InsertElementOp>(loc, srcType, res, elem, idx);
+    res = LLVM::InsertElementOp::create(builder, loc, srcType, res, elem, idx);
   }
 
   if (res.getType() != dstType)
-    res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+    res = LLVM::BitcastOp::create(builder, loc, dstType, res);
 
   return res;
 }
@@ -518,18 +518,18 @@ Value mlir::LLVM::getStridedElementPtr(OpBuilder &builder, Location loc,
       Value stride =
           ShapedType::isDynamic(strides[i])
               ? memRefDescriptor.stride(builder, loc, i)
-              : builder.create<LLVM::ConstantOp>(
+              : LLVM::ConstantOp::create(builder,
                     loc, indexType, builder.getIndexAttr(strides[i]));
       increment =
-          builder.create<LLVM::MulOp>(loc, increment, stride, intOverflowFlags);
+          LLVM::MulOp::create(builder, loc, increment, stride, intOverflowFlags);
     }
-    index = index ? builder.create<LLVM::AddOp>(loc, index, increment,
+    index = index ? LLVM::AddOp::create(builder, loc, index, increment,
                                                 intOverflowFlags)
                   : increment;
   }
 
   Type elementPtrType = memRefDescriptor.getElementPtrType();
-  return index ? builder.create<LLVM::GEPOp>(
+  return index ? LLVM::GEPOp::create(builder,
                      loc, elementPtrType,
                      converter.convertType(type.getElementType()), base, index,
                      noWrapFlags)
diff --git a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
index 49c73fbc9dd79..091fb7e4c2afb 100644
--- a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
@@ -66,7 +66,7 @@ LogicalResult mlir::LLVM::createPrintStrCall(
       DenseElementsAttr::get(dataAttrType, llvm::ArrayRef(elementVals));
   auto arrayTy =
       LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), elementVals.size());
-  auto globalOp = builder.create<LLVM::GlobalOp>(
+  auto globalOp = LLVM::GlobalOp::create(builder,
       loc, arrayTy, /*constant=*/true, LLVM::Linkage::Private,
       ensureSymbolNameIsUnique(moduleOp, symbolName, symbolTables), dataAttr);
 
@@ -74,15 +74,15 @@ LogicalResult mlir::LLVM::createPrintStrCall(
   // Emit call to `printStr` in runtime library.
   builder.restoreInsertionPoint(ip);
   auto msgAddr =
-      builder.create<LLVM::AddressOfOp>(loc, ptrTy, globalOp.getName());
+      LLVM::AddressOfOp::create(builder, loc, ptrTy, globalOp.getName());
   SmallVector<LLVM::GEPArg> indices(1, 0);
   Value gep =
-      builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, msgAddr, indices);
+      LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, msgAddr, indices);
   FailureOr<LLVM::LLVMFuncOp> printer =
       LLVM::lookupOrCreatePrintStringFn(builder, moduleOp, runtimeFunctionName);
   if (failed(printer))
     return failure();
-  builder.create<LLVM::CallOp>(loc, TypeRange(),
+  LLVM::CallOp::create(builder, loc, TypeRange(),
                                SymbolRefAttr::get(printer.value()), gep);
   return success();
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
index 1cd0bd85f9894..13ed4628c3c9e 100644
--- a/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
@@ -24,10 +24,10 @@ StructBuilder::StructBuilder(Value v) : value(v), structType(v.getType()) {
 
 Value StructBuilder::extractPtr(OpBuilder &builder, Location loc,
                                 unsigned pos) const {
-  return builder.create<LLVM::ExtractValueOp>(loc, value, pos);
+  return LLVM::ExtractValueOp::create(builder, loc, value, pos);
 }
 
 void StructBuilder::setPtr(OpBuilder &builder, Location loc, unsigned pos,
                            Value ptr) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, ptr, pos);
+  value = LLVM::InsertValueOp::create(builder, loc, value, ptr, pos);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 7312594c761f7..77135188dabcf 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -91,7 +91,7 @@ static Value unrankedMemRefMaterialization(OpBuilder &builder,
       packUnrankedMemRefDesc(builder, resultType, inputs, loc, converter);
   if (!packed)
     return Value();
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, packed)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, packed)
       .getResult(0);
 }
 
@@ -107,7 +107,7 @@ static Value rankedMemRefMaterialization(OpBuilder &builder,
       packRankedMemRefDesc(builder, resultType, inputs, loc, converter);
   if (!packed)
     return Value();
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, packed)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, packed)
       .getResult(0);
 }
 
@@ -224,12 +224,12 @@ LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
   // non-LLVM types persist after an LLVM conversion.
   addSourceMaterialization([&](OpBuilder &builder, Type resultType,
                                ValueRange inputs, Location loc) {
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
   addTargetMaterialization([&](OpBuilder &builder, Type resultType,
                                ValueRange inputs, Location loc) {
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
 
@@ -731,12 +731,12 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand,
   // Alloca with proper alignment. We do not expect optimizations of this
   // alloca op and so we omit allocating at the entry block.
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
-  Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
+  Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
                                                builder.getIndexAttr(1));
   Value allocated =
-      builder.create<LLVM::AllocaOp>(loc, ptrType, operand.getType(), one);
+      LLVM::AllocaOp::create(builder, loc, ptrType, operand.getType(), one);
   // Store into the alloca'ed descriptor.
-  builder.create<LLVM::StoreOp>(loc, operand, allocated);
+  LLVM::StoreOp::create(builder, loc, operand, allocated);
   return allocated;
 }
 
diff --git a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
index bf3f31729c3da..4fc208e05eb9d 100644
--- a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
@@ -87,17 +87,17 @@ LogicalResult LLVM::detail::handleMultidimensionalVectors(
   auto result1DVectorTy = resultTypeInfo.llvm1DVectorTy;
   auto resultNDVectoryTy = resultTypeInfo.llvmNDVectorTy;
   auto loc = op->getLoc();
-  Value desc = rewriter.create<LLVM::PoisonOp>(loc, resultNDVectoryTy);
+  Value desc = LLVM::PoisonOp::create(rewriter, loc, resultNDVectoryTy);
   nDVectorIterate(resultTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
     // For this unrolled `position` corresponding to the `linearIndex`^th
     // element, extract operand vectors
     SmallVector<Value, 4> extractedOperands;
     for (const auto &operand : llvm::enumerate(operands)) {
-      extractedOperands.push_back(rewriter.create<LLVM::ExtractValueOp>(
+      extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
           loc, operand.value(), position));
     }
     Value newVal = createOperand(result1DVectorTy, extractedOperands);
-    desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, newVal, position);
+    desc = LLVM::InsertValueOp::create(rewriter, loc, desc, newVal, position);
   });
   rewriter.replaceOp(op, desc);
   return success();
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index c3f213147b7a7..498a5cfc40204 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -78,7 +78,7 @@ getLibraryCallSymbolRef(Operation *op, PatternRewriter &rewriter) {
   // Insert before module terminator.
   rewriter.setInsertionPoint(module.getBody(),
                              std::prev(module.getBody()->end()));
-  func::FuncOp funcOp = rewriter.create<func::FuncOp>(
+  func::FuncOp funcOp = func::FuncOp::create(rewriter,
       op->getLoc(), fnNameAttr.getValue(), libFnType);
   // Insert a function attribute that will trigger the emission of the
   // corresponding `_mlir_ciface_xxx` interface so that external libraries see
@@ -101,7 +101,7 @@ createTypeCanonicalizedMemRefOperands(OpBuilder &b, Location loc,
       continue;
     }
     Value cast =
-        b.create<memref::CastOp>(loc, makeStridedLayoutDynamic(memrefType), op);
+        memref::CastOp::create(b, loc, makeStridedLayoutDynamic(memrefType), op);
     res.push_back(cast);
   }
   return res;
diff --git a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
index d4deff5b88070..30521d66a2e4c 100644
--- a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
+++ b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
@@ -54,18 +54,18 @@ std::pair<Value, Value> getRawPtrAndSize(const Location loc,
                                          Value memRef, Type elType) {
   Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
   Value dataPtr =
-      rewriter.create<LLVM::ExtractValueOp>(loc, ptrType, memRef, 1);
-  Value offset = rewriter.create<LLVM::ExtractValueOp>(
+      LLVM::ExtractValueOp::create(rewriter, loc, ptrType, memRef, 1);
+  Value offset = LLVM::ExtractValueOp::create(rewriter,
       loc, rewriter.getI64Type(), memRef, 2);
   Value resPtr =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, elType, dataPtr, offset);
+      LLVM::GEPOp::create(rewriter, loc, ptrType, elType, dataPtr, offset);
   Value size;
   if (cast<LLVM::LLVMStructType>(memRef.getType()).getBody().size() > 3) {
-    size = rewriter.create<LLVM::ExtractValueOp>(loc, memRef,
+    size = LLVM::ExtractValueOp::create(rewriter, loc, memRef,
                                                  ArrayRef<int64_t>{3, 0});
-    size = rewriter.create<LLVM::TruncOp>(loc, rewriter.getI32Type(), size);
+    size = LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), size);
   } else {
-    size = rewriter.create<arith::ConstantIntOp>(loc, 1, 32);
+    size = arith::ConstantIntOp::create(rewriter, loc, 1, 32);
   }
   return {resPtr, size};
 }
@@ -157,13 +157,13 @@ class MPICHImplTraits : public MPIImplTraits {
   Value getCommWorld(const Location loc,
                      ConversionPatternRewriter &rewriter) override {
     static constexpr int MPI_COMM_WORLD = 0x44000000;
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
                                              MPI_COMM_WORLD);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return rewriter.create<LLVM::TruncOp>(loc, rewriter.getI32Type(), comm);
+    return LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), comm);
   }
 
   intptr_t getStatusIgnore() override { return 1; }
@@ -195,7 +195,7 @@ class MPICHImplTraits : public MPIImplTraits {
       mtype = MPI_UINT8_T;
     else
       assert(false && "unsupported type");
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), mtype);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), mtype);
   }
 
   Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter,
@@ -245,7 +245,7 @@ class MPICHImplTraits : public MPIImplTraits {
       op = MPI_REPLACE;
       break;
     }
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), op);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), op);
   }
 };
 
@@ -281,15 +281,15 @@ class OMPIImplTraits : public MPIImplTraits {
     getOrDefineExternalStruct(loc, rewriter, name, commStructT);
 
     // get address of symbol
-    auto comm = rewriter.create<LLVM::AddressOfOp>(
+    auto comm = LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, name));
-    return rewriter.create<LLVM::PtrToIntOp>(loc, rewriter.getI64Type(), comm);
+    return LLVM::PtrToIntOp::create(rewriter, loc, rewriter.getI64Type(), comm);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return rewriter.create<LLVM::IntToPtrOp>(
+    return LLVM::IntToPtrOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(rewriter.getContext()), comm);
   }
 
@@ -330,7 +330,7 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, mtype, typeStructT);
     // get address of symbol
-    return rewriter.create<LLVM::AddressOfOp>(
+    return LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, mtype));
   }
@@ -389,7 +389,7 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, op, opStructT);
     // get address of symbol
-    return rewriter.create<LLVM::AddressOfOp>(
+    return LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, op));
   }
@@ -424,7 +424,7 @@ struct InitOpLowering : public ConvertOpToLLVMPattern<mpi::InitOp> {
     Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
 
     // instantiate nullptr `%nullptr = llvm.mlir.zero : !llvm.ptr`
-    auto nullPtrOp = rewriter.create<LLVM::ZeroOp>(loc, ptrType);
+    auto nullPtrOp = LLVM::ZeroOp::create(rewriter, loc, ptrType);
     Value llvmnull = nullPtrOp.getRes();
 
     // grab a reference to the global module op:
@@ -513,9 +513,9 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
 
     // get communicator
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, i32, 1);
+    auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
     auto outPtr =
-        rewriter.create<LLVM::AllocaOp>(loc, ptrType, comm.getType(), one);
+        LLVM::AllocaOp::create(rewriter, loc, ptrType, comm.getType(), one);
 
     // int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm * newcomm)
     auto funcType =
@@ -524,14 +524,14 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
     LLVM::LLVMFuncOp funcDecl = getOrDefineFunction(moduleOp, loc, rewriter,
                                                     "MPI_Comm_split", funcType);
 
-    auto callOp = rewriter.create<LLVM::CallOp>(
+    auto callOp = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{comm, adaptor.getColor(), adaptor.getKey(),
                    outPtr.getRes()});
 
     // load the communicator into a register
-    Value res = rewriter.create<LLVM::LoadOp>(loc, i32, outPtr.getResult());
-    res = rewriter.create<LLVM::SExtOp>(loc, rewriter.getI64Type(), res);
+    Value res = LLVM::LoadOp::create(rewriter, loc, i32, outPtr.getResult());
+    res = LLVM::SExtOp::create(rewriter, loc, rewriter.getI64Type(), res);
 
     // if retval is checked, replace uses of retval with the results from the
     // call op
@@ -580,14 +580,14 @@ struct CommRankOpLowering : public ConvertOpToLLVMPattern<mpi::CommRankOp> {
         moduleOp, loc, rewriter, "MPI_Comm_rank", rankFuncType);
 
     // replace with function call
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, i32, 1);
-    auto rankptr = rewriter.create<LLVM::AllocaOp>(loc, ptrType, i32, one);
-    auto callOp = rewriter.create<LLVM::CallOp>(
+    auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
+    auto rankptr = LLVM::AllocaOp::create(rewriter, loc, ptrType, i32, one);
+    auto callOp = LLVM::CallOp::create(rewriter,
         loc, initDecl, ValueRange{comm, rankptr.getRes()});
 
     // load the rank into a register
     auto loadedRank =
-        rewriter.create<LLVM::LoadOp>(loc, i32, rankptr.getResult());
+        LLVM::LoadOp::create(rewriter, loc, i32, rankptr.getResult());
 
     // if retval is checked, replace uses of retval with the results from the
     // call op
@@ -641,7 +641,7 @@ struct SendOpLowering : public ConvertOpToLLVMPattern<mpi::SendOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Send", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getDest(), adaptor.getTag(),
                    comm});
@@ -683,10 +683,10 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
     auto mpiTraits = MPIImplTraits::get(moduleOp);
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    Value statusIgnore = rewriter.create<LLVM::ConstantOp>(
+    Value statusIgnore = LLVM::ConstantOp::create(rewriter,
         loc, i64, mpiTraits->getStatusIgnore());
     statusIgnore =
-        rewriter.create<LLVM::IntToPtrOp>(loc, ptrType, statusIgnore);
+        LLVM::IntToPtrOp::create(rewriter, loc, ptrType, statusIgnore);
 
     // LLVM Function type representing `i32 MPI_Recv(data, count, datatype, dst,
     // tag, comm)`
@@ -698,7 +698,7 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Recv", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getSource(),
                    adaptor.getTag(), comm, statusIgnore});
@@ -738,9 +738,9 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
 
     // If input and output are the same, request in-place operation.
     if (adaptor.getSendbuf() == adaptor.getRecvbuf()) {
-      sendPtr = rewriter.create<LLVM::ConstantOp>(
+      sendPtr = LLVM::ConstantOp::create(rewriter,
           loc, i64, reinterpret_cast<int64_t>(mpiTraits->getInPlace()));
-      sendPtr = rewriter.create<LLVM::IntToPtrOp>(loc, ptrType, sendPtr);
+      sendPtr = LLVM::IntToPtrOp::create(rewriter, loc, ptrType, sendPtr);
     }
 
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
@@ -757,7 +757,7 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Allreduce", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{sendPtr, recvPtr, sendSize, dataType, mpiOp, commWorld});
 
diff --git a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
index 7f4655e53609e..bb3511440e041 100644
--- a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
+++ b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
@@ -121,7 +121,7 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     initValueAttr = FloatAttr::get(resultElementType, 0.0);
   else
     initValueAttr = IntegerAttr::get(resultElementType, 0);
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(vecType, initValueAttr));
   SmallVector<int64_t> strides = computeStrides(shape);
   for (int64_t linearIndex = 0; linearIndex < numElements; ++linearIndex) {
@@ -129,11 +129,11 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     SmallVector<Value> operands;
     for (Value input : op->getOperands())
       operands.push_back(
-          rewriter.create<vector::ExtractOp>(loc, input, positions));
+          vector::ExtractOp::create(rewriter, loc, input, positions));
     Value scalarOp =
-        rewriter.create<Op>(loc, vecType.getElementType(), operands);
+        Op::create(rewriter, loc, vecType.getElementType(), operands);
     result =
-        rewriter.create<vector::InsertOp>(loc, scalarOp, result, positions);
+        vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
   }
   rewriter.replaceOp(op, result);
   return success();
@@ -195,7 +195,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
 
   FunctionType funcType = FunctionType::get(
       builder.getContext(), {elementType, elementType}, elementType);
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
   LLVM::linkage::Linkage inlineLinkage = LLVM::linkage::Linkage::LinkonceODR;
   Attribute linkage =
       LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
@@ -208,11 +208,11 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value zeroValue = builder.create<arith::ConstantOp>(
+  Value zeroValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 0));
-  Value oneValue = builder.create<arith::ConstantOp>(
+  Value oneValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 1));
-  Value minusOneValue = builder.create<arith::ConstantOp>(
+  Value minusOneValue = arith::ConstantOp::create(builder,
       elementType,
       builder.getIntegerAttr(elementType,
                              APInt(elementType.getIntOrFloatBitWidth(), -1ULL,
@@ -221,81 +221,81 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   // if (p == T(0))
   //   return T(1);
   auto pIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroValue);
   Block *thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(pIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(pIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsZero, thenBlock, fallthroughBlock);
 
   // if (p < T(0)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto pIsNeg =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, pArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg, zeroValue);
   //   if (b == T(0))
   builder.createBlock(funcBody);
   auto bIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, zeroValue);
   //     return T(1) / T(0);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(
-      builder.create<arith::DivSIOp>(oneValue, zeroValue).getResult());
+  func::ReturnOp::create(builder,
+      arith::DivSIOp::create(builder, oneValue, zeroValue).getResult());
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(0)).
   builder.setInsertionPointToEnd(bIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(bIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, bIsZero, thenBlock, fallthroughBlock);
 
   //   if (b == T(1))
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto bIsOne =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bArg, oneValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, oneValue);
   //    return T(1);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(1)).
   builder.setInsertionPointToEnd(bIsOne->getBlock());
-  builder.create<cf::CondBranchOp>(bIsOne, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, bIsOne, thenBlock, fallthroughBlock);
 
   //   if (b == T(-1)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
-  auto bIsMinusOne = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto bIsMinusOne = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                    bArg, minusOneValue);
   //     if (p & T(1))
   builder.createBlock(funcBody);
-  auto pIsOdd = builder.create<arith::CmpIOp>(
-      arith::CmpIPredicate::ne, builder.create<arith::AndIOp>(pArg, oneValue),
+  auto pIsOdd = arith::CmpIOp::create(builder,
+      arith::CmpIPredicate::ne, arith::AndIOp::create(builder, pArg, oneValue),
       zeroValue);
   //       return T(-1);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(minusOneValue);
+  func::ReturnOp::create(builder, minusOneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(pIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(pIsOdd, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsOdd, thenBlock, fallthroughBlock);
 
   //     return T(1);
   //   } // b == T(-1)
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(-1)).
   builder.setInsertionPointToEnd(bIsMinusOne->getBlock());
-  builder.create<cf::CondBranchOp>(bIsMinusOne, pIsOdd->getBlock(),
+  cf::CondBranchOp::create(builder, bIsMinusOne, pIsOdd->getBlock(),
                                    fallthroughBlock);
 
   //   return T(0);
   // } // (p < T(0))
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<func::ReturnOp>(zeroValue);
+  func::ReturnOp::create(builder, zeroValue);
   Block *loopHeader = builder.createBlock(
       funcBody, funcBody->end(), {elementType, elementType, elementType},
       {builder.getLoc(), builder.getLoc(), builder.getLoc()});
   // Set up conditional branch for (p < T(0)).
   builder.setInsertionPointToEnd(pIsNeg->getBlock());
   // Set initial values of 'result', 'b' and 'p' for the loop.
-  builder.create<cf::CondBranchOp>(pIsNeg, bIsZero->getBlock(), loopHeader,
+  cf::CondBranchOp::create(builder, pIsNeg, bIsZero->getBlock(), loopHeader,
                                    ValueRange{oneValue, bArg, pArg});
 
   // T result = T(1);
@@ -313,44 +313,44 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & T(1))
-  auto powerTmpIsOdd = builder.create<arith::CmpIOp>(
+  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
       arith::CmpIPredicate::ne,
-      builder.create<arith::AndIOp>(powerTmp, oneValue), zeroValue);
+      arith::AndIOp::create(builder, powerTmp, oneValue), zeroValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
-  Value newResultTmp = builder.create<arith::MulIOp>(resultTmp, baseTmp);
+  Value newResultTmp = arith::MulIOp::create(builder, resultTmp, baseTmp);
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), elementType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(thenBlock);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(powerTmpIsOdd, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
                                    resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
   //   p >>= T(1);
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newPowerTmp = builder.create<arith::ShRUIOp>(powerTmp, oneValue);
+  Value newPowerTmp = arith::ShRUIOp::create(builder, powerTmp, oneValue);
 
   //   if (p == T(0))
-  auto newPowerIsZero = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                       newPowerTmp, zeroValue);
   //     return result;
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(newResultTmp);
+  func::ReturnOp::create(builder, newResultTmp);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(newPowerIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, newPowerIsZero, thenBlock, fallthroughBlock);
 
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newBaseTmp = builder.create<arith::MulIOp>(baseTmp, baseTmp);
+  Value newBaseTmp = arith::MulIOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  builder.create<cf::BranchOp>(
+  cf::BranchOp::create(builder,
       ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
   return funcOp;
 }
@@ -420,7 +420,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   llvm::raw_string_ostream nameOS(funcName);
   nameOS << '_' << baseType;
   nameOS << '_' << powType;
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
   LLVM::linkage::Linkage inlineLinkage = LLVM::linkage::Linkage::LinkonceODR;
   Attribute linkage =
       LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
@@ -433,46 +433,46 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value oneBValue = builder.create<arith::ConstantOp>(
+  Value oneBValue = arith::ConstantOp::create(builder,
       baseType, builder.getFloatAttr(baseType, 1.0));
-  Value zeroPValue = builder.create<arith::ConstantOp>(
+  Value zeroPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, 0));
-  Value onePValue = builder.create<arith::ConstantOp>(
+  Value onePValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, 1));
-  Value minPValue = builder.create<arith::ConstantOp>(
+  Value minPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMinValue(
                                                    powType.getWidth())));
-  Value maxPValue = builder.create<arith::ConstantOp>(
+  Value maxPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMaxValue(
                                                    powType.getWidth())));
 
   // if (p == Tp{0})
   //   return Tb{1};
   auto pIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, zeroPValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroPValue);
   Block *thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneBValue);
+  func::ReturnOp::create(builder, oneBValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == Tp{0}).
   builder.setInsertionPointToEnd(pIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(pIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsZero, thenBlock, fallthroughBlock);
 
   builder.setInsertionPointToEnd(fallthroughBlock);
   // bool isNegativePower{p < Tp{0}}
-  auto pIsNeg = builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, pArg,
+  auto pIsNeg = arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg,
                                               zeroPValue);
   // bool isMin{p == std::numeric_limits<Tp>::min()};
   auto pIsMin =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, minPValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, minPValue);
 
   // if (isMin) {
   //   p = std::numeric_limits<Tp>::max();
   // } else if (isNegativePower) {
   //   p = -p;
   // }
-  Value negP = builder.create<arith::SubIOp>(zeroPValue, pArg);
-  auto pInit = builder.create<arith::SelectOp>(pIsNeg, negP, pArg);
-  pInit = builder.create<arith::SelectOp>(pIsMin, maxPValue, pInit);
+  Value negP = arith::SubIOp::create(builder, zeroPValue, pArg);
+  auto pInit = arith::SelectOp::create(builder, pIsNeg, negP, pArg);
+  pInit = arith::SelectOp::create(builder, pIsMin, maxPValue, pInit);
 
   // Tb result = Tb{1};
   // Tb origBase = Tb{b};
@@ -489,7 +489,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
       {builder.getLoc(), builder.getLoc(), builder.getLoc()});
   // Set initial values of 'result', 'b' and 'p' for the loop.
   builder.setInsertionPointToEnd(pInit->getBlock());
-  builder.create<cf::BranchOp>(loopHeader, ValueRange{oneBValue, bArg, pInit});
+  cf::BranchOp::create(builder, loopHeader, ValueRange{oneBValue, bArg, pInit});
 
   // Create loop body.
   Value resultTmp = loopHeader->getArgument(0);
@@ -498,29 +498,29 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & Tp{1})
-  auto powerTmpIsOdd = builder.create<arith::CmpIOp>(
+  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
       arith::CmpIPredicate::ne,
-      builder.create<arith::AndIOp>(powerTmp, onePValue), zeroPValue);
+      arith::AndIOp::create(builder, powerTmp, onePValue), zeroPValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
-  Value newResultTmp = builder.create<arith::MulFOp>(resultTmp, baseTmp);
+  Value newResultTmp = arith::MulFOp::create(builder, resultTmp, baseTmp);
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(thenBlock);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
   // Set up conditional branch for (p & Tp{1}).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(powerTmpIsOdd, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
                                    resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
   //   p >>= Tp{1};
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newPowerTmp = builder.create<arith::ShRUIOp>(powerTmp, onePValue);
+  Value newPowerTmp = arith::ShRUIOp::create(builder, powerTmp, onePValue);
 
   //   if (p == Tp{0})
-  auto newPowerIsZero = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                       newPowerTmp, zeroPValue);
   //     break;
   //
@@ -531,9 +531,9 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newBaseTmp = builder.create<arith::MulFOp>(baseTmp, baseTmp);
+  Value newBaseTmp = arith::MulFOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  builder.create<cf::BranchOp>(
+  cf::BranchOp::create(builder,
       ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
 
   // Set up conditional branch for early loop exit:
@@ -542,7 +542,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Block *loopExit = builder.createBlock(funcBody, funcBody->end(), baseType,
                                         builder.getLoc());
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(newPowerIsZero, loopExit, newResultTmp,
+  cf::CondBranchOp::create(builder, newPowerIsZero, loopExit, newResultTmp,
                                    fallthroughBlock, ValueRange{});
 
   // if (isMin) {
@@ -553,11 +553,11 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(loopExit);
-  builder.create<cf::CondBranchOp>(pIsMin, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, pIsMin, thenBlock, fallthroughBlock,
                                    newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
-  newResultTmp = builder.create<arith::MulFOp>(newResultTmp, bArg);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  newResultTmp = arith::MulFOp::create(builder, newResultTmp, bArg);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
 
   /// if (isNegativePower) {
   ///   result = Tb{1} / result;
@@ -567,15 +567,15 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Block *returnBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                            builder.getLoc());
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<cf::CondBranchOp>(pIsNeg, thenBlock, returnBlock,
+  cf::CondBranchOp::create(builder, pIsNeg, thenBlock, returnBlock,
                                    newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
-  newResultTmp = builder.create<arith::DivFOp>(oneBValue, newResultTmp);
-  builder.create<cf::BranchOp>(newResultTmp, returnBlock);
+  newResultTmp = arith::DivFOp::create(builder, oneBValue, newResultTmp);
+  cf::BranchOp::create(builder, newResultTmp, returnBlock);
 
   // return result;
   builder.setInsertionPointToEnd(returnBlock);
-  builder.create<func::ReturnOp>(returnBlock->getArgument(0));
+  func::ReturnOp::create(builder, returnBlock->getArgument(0));
 
   return funcOp;
 }
@@ -667,7 +667,7 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
   nameOS << '_' << elementType;
   FunctionType funcType =
       FunctionType::get(builder.getContext(), {elementType}, elementType);
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
 
   // LinkonceODR ensures that there is only one implementation of this function
   // across all math.ctlz functions that are lowered in this way.
@@ -683,32 +683,32 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
 
   Value arg = funcOp.getArgument(0);
   Type indexType = builder.getIndexType();
-  Value bitWidthValue = builder.create<arith::ConstantOp>(
+  Value bitWidthValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, bitWidth));
-  Value zeroValue = builder.create<arith::ConstantOp>(
+  Value zeroValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 0));
 
   Value inputEqZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, arg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, arg, zeroValue);
 
   // if input == 0, return bit width, else enter loop.
-  scf::IfOp ifOp = builder.create<scf::IfOp>(
+  scf::IfOp ifOp = scf::IfOp::create(builder,
       elementType, inputEqZero, /*addThenBlock=*/true, /*addElseBlock=*/true);
   ifOp.getThenBodyBuilder().create<scf::YieldOp>(loc, bitWidthValue);
 
   auto elseBuilder =
       ImplicitLocOpBuilder::atBlockEnd(loc, &ifOp.getElseRegion().front());
 
-  Value oneIndex = elseBuilder.create<arith::ConstantOp>(
+  Value oneIndex = arith::ConstantOp::create(elseBuilder,
       indexType, elseBuilder.getIndexAttr(1));
-  Value oneValue = elseBuilder.create<arith::ConstantOp>(
+  Value oneValue = arith::ConstantOp::create(elseBuilder,
       elementType, elseBuilder.getIntegerAttr(elementType, 1));
-  Value bitWidthIndex = elseBuilder.create<arith::ConstantOp>(
+  Value bitWidthIndex = arith::ConstantOp::create(elseBuilder,
       indexType, elseBuilder.getIndexAttr(bitWidth));
-  Value nValue = elseBuilder.create<arith::ConstantOp>(
+  Value nValue = arith::ConstantOp::create(elseBuilder,
       elementType, elseBuilder.getIntegerAttr(elementType, 0));
 
-  auto loop = elseBuilder.create<scf::ForOp>(
+  auto loop = scf::ForOp::create(elseBuilder,
       oneIndex, bitWidthIndex, oneIndex,
       // Initial values for two loop induction variables, the arg which is being
       // shifted left in each iteration, and the n value which tracks the count
@@ -725,25 +725,25 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
         Value argIter = args[0];
         Value nIter = args[1];
 
-        Value argIsNonNegative = b.create<arith::CmpIOp>(
+        Value argIsNonNegative = arith::CmpIOp::create(b,
             loc, arith::CmpIPredicate::slt, argIter, zeroValue);
-        scf::IfOp ifOp = b.create<scf::IfOp>(
+        scf::IfOp ifOp = scf::IfOp::create(b,
             loc, argIsNonNegative,
             [&](OpBuilder &b, Location loc) {
               // If arg is negative, continue (effectively, break)
-              b.create<scf::YieldOp>(loc, ValueRange{argIter, nIter});
+              scf::YieldOp::create(b, loc, ValueRange{argIter, nIter});
             },
             [&](OpBuilder &b, Location loc) {
               // Otherwise, increment n and shift arg left.
-              Value nNext = b.create<arith::AddIOp>(loc, nIter, oneValue);
-              Value argNext = b.create<arith::ShLIOp>(loc, argIter, oneValue);
-              b.create<scf::YieldOp>(loc, ValueRange{argNext, nNext});
+              Value nNext = arith::AddIOp::create(b, loc, nIter, oneValue);
+              Value argNext = arith::ShLIOp::create(b, loc, argIter, oneValue);
+              scf::YieldOp::create(b, loc, ValueRange{argNext, nNext});
             });
-        b.create<scf::YieldOp>(loc, ifOp.getResults());
+        scf::YieldOp::create(b, loc, ifOp.getResults());
       });
-  elseBuilder.create<scf::YieldOp>(loop.getResult(1));
+  scf::YieldOp::create(elseBuilder, loop.getResult(1));
 
-  builder.create<func::ReturnOp>(ifOp.getResult(0));
+  func::ReturnOp::create(builder, ifOp.getResult(0));
   return funcOp;
 }
 
diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
index f4d69ce8235bb..26559f394c221 100644
--- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
+++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
@@ -107,7 +107,7 @@ struct IntOpWithFlagLowering : public ConvertOpToLLVMPattern<MathOp> {
     return LLVM::detail::handleMultidimensionalVectors(
         op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
-          return rewriter.create<LLVMOp>(loc, llvm1DVectorTy, operands[0],
+          return LLVMOp::create(rewriter, loc, llvm1DVectorTy, operands[0],
                                          false);
         },
         rewriter);
@@ -145,14 +145,14 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (LLVM::isCompatibleVectorType(llvmOperandType)) {
-        one = rewriter.create<LLVM::ConstantOp>(
+        one = LLVM::ConstantOp::create(rewriter,
             loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
+        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
-      auto exp = rewriter.create<LLVM::ExpOp>(loc, adaptor.getOperand(),
+      auto exp = LLVM::ExpOp::create(rewriter, loc, adaptor.getOperand(),
                                               expAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FSubOp>(
           op, llvmOperandType, ValueRange{exp, one}, subAttrs.getAttrs());
@@ -171,10 +171,10 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto exp = rewriter.create<LLVM::ExpOp>(
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto exp = LLVM::ExpOp::create(rewriter,
               loc, llvm1DVectorTy, operands[0], expAttrs.getAttrs());
-          return rewriter.create<LLVM::FSubOp>(
+          return LLVM::FSubOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{exp, one}, subAttrs.getAttrs());
         },
         rewriter);
@@ -205,14 +205,14 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one =
           isa<VectorType>(llvmOperandType)
-              ? rewriter.create<LLVM::ConstantOp>(
+              ? LLVM::ConstantOp::create(rewriter,
                     loc, llvmOperandType,
                     SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                            floatOne))
-              : rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType,
+              : LLVM::ConstantOp::create(rewriter, loc, llvmOperandType,
                                                   floatOne);
 
-      auto add = rewriter.create<LLVM::FAddOp>(
+      auto add = LLVM::FAddOp::create(rewriter,
           loc, llvmOperandType, ValueRange{one, adaptor.getOperand()},
           addAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::LogOp>(
@@ -232,11 +232,11 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto add = rewriter.create<LLVM::FAddOp>(loc, llvm1DVectorTy,
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto add = LLVM::FAddOp::create(rewriter, loc, llvm1DVectorTy,
                                                    ValueRange{one, operands[0]},
                                                    addAttrs.getAttrs());
-          return rewriter.create<LLVM::LogOp>(
+          return LLVM::LogOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{add}, logAttrs.getAttrs());
         },
         rewriter);
@@ -267,14 +267,14 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (isa<VectorType>(llvmOperandType)) {
-        one = rewriter.create<LLVM::ConstantOp>(
+        one = LLVM::ConstantOp::create(rewriter,
             loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
+        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
-      auto sqrt = rewriter.create<LLVM::SqrtOp>(loc, adaptor.getOperand(),
+      auto sqrt = LLVM::SqrtOp::create(rewriter, loc, adaptor.getOperand(),
                                                 sqrtAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FDivOp>(
           op, llvmOperandType, ValueRange{one, sqrt}, divAttrs.getAttrs());
@@ -293,10 +293,10 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto sqrt = rewriter.create<LLVM::SqrtOp>(
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto sqrt = LLVM::SqrtOp::create(rewriter,
               loc, llvm1DVectorTy, operands[0], sqrtAttrs.getAttrs());
-          return rewriter.create<LLVM::FDivOp>(
+          return LLVM::FDivOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{one, sqrt}, divAttrs.getAttrs());
         },
         rewriter);
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index a0ce7d3b75fc2..19303dc3e8b55 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -84,7 +84,7 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto shape = vecType.getShape();
   int64_t numElements = vecType.getNumElements();
 
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(
                vecType, FloatAttr::get(vecType.getElementType(), 0.0)));
   SmallVector<int64_t> strides = computeStrides(shape);
@@ -93,11 +93,11 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     SmallVector<Value> operands;
     for (auto input : op->getOperands())
       operands.push_back(
-          rewriter.create<vector::ExtractOp>(loc, input, positions));
+          vector::ExtractOp::create(rewriter, loc, input, positions));
     Value scalarOp =
-        rewriter.create<Op>(loc, vecType.getElementType(), operands);
+        Op::create(rewriter, loc, vecType.getElementType(), operands);
     result =
-        rewriter.create<vector::InsertOp>(loc, scalarOp, result, positions);
+        vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
   }
   rewriter.replaceOp(op, {result});
   return success();
@@ -114,9 +114,9 @@ PromoteOpToF32<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto f32 = rewriter.getF32Type();
   auto extendedOperands = llvm::to_vector(
       llvm::map_range(op->getOperands(), [&](Value operand) -> Value {
-        return rewriter.create<arith::ExtFOp>(loc, f32, operand);
+        return arith::ExtFOp::create(rewriter, loc, f32, operand);
       }));
-  auto newOp = rewriter.create<Op>(loc, f32, extendedOperands);
+  auto newOp = Op::create(rewriter, loc, f32, extendedOperands);
   rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, opType, newOp);
   return success();
 }
@@ -139,7 +139,7 @@ ScalarOpToLibmCall<Op>::matchAndRewrite(Op op,
     rewriter.setInsertionPointToStart(&module->getRegion(0).front());
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-    opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name,
+    opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
                                            opFunctionTy);
     opFunc.setPrivate();
 
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 59db14ed816be..26891aa7c2025 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -36,11 +36,11 @@ static Value getScalarOrVectorI32Constant(Type type, int value,
     if (!vectorType.getElementType().isInteger(32))
       return nullptr;
     SmallVector<int> values(vectorType.getNumElements(), value);
-    return builder.create<spirv::ConstantOp>(loc, type,
+    return spirv::ConstantOp::create(builder, loc, type,
                                              builder.getI32VectorAttr(values));
   }
   if (type.isInteger(32))
-    return builder.create<spirv::ConstantOp>(loc, type,
+    return spirv::ConstantOp::create(builder, loc, type,
                                              builder.getI32IntegerAttr(value));
 
   return nullptr;
@@ -144,9 +144,9 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
     Type intType = rewriter.getIntegerType(bitwidth);
     uint64_t intValue = uint64_t(1) << (bitwidth - 1);
 
-    Value signMask = rewriter.create<spirv::ConstantOp>(
+    Value signMask = spirv::ConstantOp::create(rewriter,
         loc, intType, rewriter.getIntegerAttr(intType, intValue));
-    Value valueMask = rewriter.create<spirv::ConstantOp>(
+    Value valueMask = spirv::ConstantOp::create(rewriter,
         loc, intType, rewriter.getIntegerAttr(intType, intValue - 1u));
 
     if (auto vectorType = dyn_cast<VectorType>(type)) {
@@ -156,24 +156,24 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
 
       SmallVector<Value> signSplat(count, signMask);
       signMask =
-          rewriter.create<spirv::CompositeConstructOp>(loc, intType, signSplat);
+          spirv::CompositeConstructOp::create(rewriter, loc, intType, signSplat);
 
       SmallVector<Value> valueSplat(count, valueMask);
-      valueMask = rewriter.create<spirv::CompositeConstructOp>(loc, intType,
+      valueMask = spirv::CompositeConstructOp::create(rewriter, loc, intType,
                                                                valueSplat);
     }
 
     Value lhsCast =
-        rewriter.create<spirv::BitcastOp>(loc, intType, adaptor.getLhs());
+        spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getLhs());
     Value rhsCast =
-        rewriter.create<spirv::BitcastOp>(loc, intType, adaptor.getRhs());
+        spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getRhs());
 
-    Value value = rewriter.create<spirv::BitwiseAndOp>(
+    Value value = spirv::BitwiseAndOp::create(rewriter,
         loc, intType, ValueRange{lhsCast, valueMask});
-    Value sign = rewriter.create<spirv::BitwiseAndOp>(
+    Value sign = spirv::BitwiseAndOp::create(rewriter,
         loc, intType, ValueRange{rhsCast, signMask});
 
-    Value result = rewriter.create<spirv::BitwiseOrOp>(loc, intType,
+    Value result = spirv::BitwiseOrOp::create(rewriter, loc, intType,
                                                        ValueRange{value, sign});
     rewriter.replaceOpWithNewOp<spirv::BitcastOp>(copySignOp, type, result);
     return success();
@@ -214,18 +214,18 @@ struct CountLeadingZerosPattern final
     Value val31 = getScalarOrVectorI32Constant(type, 31, rewriter, loc);
     Value val32 = getScalarOrVectorI32Constant(type, 32, rewriter, loc);
 
-    Value msb = rewriter.create<spirv::GLFindUMsbOp>(loc, input);
+    Value msb = spirv::GLFindUMsbOp::create(rewriter, loc, input);
     // We need to subtract from 31 given that the index returned by GLSL
     // FindUMsb is counted from the least significant bit. Theoretically this
     // also gives the correct result even if the integer has all zero bits, in
     // which case GL FindUMsb would return -1.
-    Value subMsb = rewriter.create<spirv::ISubOp>(loc, val31, msb);
+    Value subMsb = spirv::ISubOp::create(rewriter, loc, val31, msb);
     // However, certain Vulkan implementations have driver bugs for the corner
     // case where the input is zero. And.. it can be smart to optimize a select
     // only involving the corner case. So separately compute the result when the
     // input is either zero or one.
-    Value subInput = rewriter.create<spirv::ISubOp>(loc, val32, input);
-    Value cmp = rewriter.create<spirv::ULessThanEqualOp>(loc, input, val1);
+    Value subInput = spirv::ISubOp::create(rewriter, loc, val32, input);
+    Value cmp = spirv::ULessThanEqualOp::create(rewriter, loc, input, val1);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(countOp, cmp, subInput,
                                                  subMsb);
     return success();
@@ -253,7 +253,7 @@ struct ExpM1OpPattern final : public OpConversionPattern<math::ExpM1Op> {
     if (!type)
       return failure();
 
-    Value exp = rewriter.create<ExpOp>(loc, type, adaptor.getOperand());
+    Value exp = ExpOp::create(rewriter, loc, type, adaptor.getOperand());
     auto one = spirv::ConstantOp::getOne(type, loc, rewriter);
     rewriter.replaceOpWithNewOp<spirv::FSubOp>(operation, exp, one);
     return success();
@@ -283,7 +283,7 @@ struct Log1pOpPattern final : public OpConversionPattern<math::Log1pOp> {
 
     auto one = spirv::ConstantOp::getOne(type, operation.getLoc(), rewriter);
     Value onePlus =
-        rewriter.create<spirv::FAddOp>(loc, one, adaptor.getOperand());
+        spirv::FAddOp::create(rewriter, loc, one, adaptor.getOperand());
     rewriter.replaceOpWithNewOp<LogOp>(operation, type, onePlus);
     return success();
   }
@@ -321,14 +321,14 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
 
     auto getConstantValue = [&](double value) {
       if (auto floatType = dyn_cast<FloatType>(type)) {
-        return rewriter.create<spirv::ConstantOp>(
+        return spirv::ConstantOp::create(rewriter,
             loc, type, rewriter.getFloatAttr(floatType, value));
       }
       if (auto vectorType = dyn_cast<VectorType>(type)) {
         Type elemType = vectorType.getElementType();
 
         if (isa<FloatType>(elemType)) {
-          return rewriter.create<spirv::ConstantOp>(
+          return spirv::ConstantOp::create(rewriter,
               loc, type,
               DenseFPElementsAttr::get(
                   vectorType, FloatAttr::get(elemType, value).getValue()));
@@ -341,7 +341,7 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
     Value constantValue = getConstantValue(
         std::is_same<MathLogOp, math::Log2Op>() ? log2Reciprocal
                                                 : log10Reciprocal);
-    Value log = rewriter.create<SpirvLogOp>(loc, adaptor.getOperand());
+    Value log = SpirvLogOp::create(rewriter, loc, adaptor.getOperand());
     rewriter.replaceOpWithNewOp<spirv::FMulOp>(operation, type, log,
                                                constantValue);
     return success();
@@ -386,7 +386,7 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
     Location loc = powfOp.getLoc();
     Value zero = spirv::ConstantOp::getZero(operandType, loc, rewriter);
     Value lessThan =
-        rewriter.create<spirv::FOrdLessThanOp>(loc, adaptor.getLhs(), zero);
+        spirv::FOrdLessThanOp::create(rewriter, loc, adaptor.getLhs(), zero);
 
     // Per C/C++ spec:
     // > pow(base, exponent) returns NaN (and raises FE_INVALID) if base is
@@ -394,11 +394,11 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
     // Calculate the reminder from the exponent and check whether it is zero.
     Value floatOne = spirv::ConstantOp::getOne(operandType, loc, rewriter);
     Value expRem =
-        rewriter.create<spirv::FRemOp>(loc, adaptor.getRhs(), floatOne);
+        spirv::FRemOp::create(rewriter, loc, adaptor.getRhs(), floatOne);
     Value expRemNonZero =
-        rewriter.create<spirv::FOrdNotEqualOp>(loc, expRem, zero);
+        spirv::FOrdNotEqualOp::create(rewriter, loc, expRem, zero);
     Value cmpNegativeWithFractionalExp =
-        rewriter.create<spirv::LogicalAndOp>(loc, expRemNonZero, lessThan);
+        spirv::LogicalAndOp::create(rewriter, loc, expRemNonZero, lessThan);
     // Create NaN result and replace base value if conditions are met.
     const auto &floatSemantics = scalarFloatType.getFloatSemantics();
     const auto nan = APFloat::getNaN(floatSemantics);
@@ -407,10 +407,10 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
       nanAttr = DenseElementsAttr::get(vectorType, nan);
 
     Value NanValue =
-        rewriter.create<spirv::ConstantOp>(loc, operandType, nanAttr);
-    Value lhs = rewriter.create<spirv::SelectOp>(
+        spirv::ConstantOp::create(rewriter, loc, operandType, nanAttr);
+    Value lhs = spirv::SelectOp::create(rewriter,
         loc, cmpNegativeWithFractionalExp, NanValue, adaptor.getLhs());
-    Value abs = rewriter.create<spirv::GLFAbsOp>(loc, lhs);
+    Value abs = spirv::GLFAbsOp::create(rewriter, loc, lhs);
 
     // TODO: The following just forcefully casts y into an integer value in
     // order to properly propagate the sign, assuming integer y cases. It
@@ -418,18 +418,18 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
 
     // Cast exponent to integer and calculate exponent % 2 != 0.
     Value intRhs =
-        rewriter.create<spirv::ConvertFToSOp>(loc, intType, adaptor.getRhs());
+        spirv::ConvertFToSOp::create(rewriter, loc, intType, adaptor.getRhs());
     Value intOne = spirv::ConstantOp::getOne(intType, loc, rewriter);
     Value bitwiseAndOne =
-        rewriter.create<spirv::BitwiseAndOp>(loc, intRhs, intOne);
-    Value isOdd = rewriter.create<spirv::IEqualOp>(loc, bitwiseAndOne, intOne);
+        spirv::BitwiseAndOp::create(rewriter, loc, intRhs, intOne);
+    Value isOdd = spirv::IEqualOp::create(rewriter, loc, bitwiseAndOne, intOne);
 
     // calculate pow based on abs(lhs)^rhs.
-    Value pow = rewriter.create<spirv::GLPowOp>(loc, abs, adaptor.getRhs());
-    Value negate = rewriter.create<spirv::FNegateOp>(loc, pow);
+    Value pow = spirv::GLPowOp::create(rewriter, loc, abs, adaptor.getRhs());
+    Value negate = spirv::FNegateOp::create(rewriter, loc, pow);
     // if the exponent is odd and lhs < 0, negate the result.
     Value shouldNegate =
-        rewriter.create<spirv::LogicalAndOp>(loc, lessThan, isOdd);
+        spirv::LogicalAndOp::create(rewriter, loc, lessThan, isOdd);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(powfOp, shouldNegate, negate,
                                                  pow);
     return success();
@@ -455,22 +455,22 @@ struct RoundOpPattern final : public OpConversionPattern<math::RoundOp> {
     auto one = spirv::ConstantOp::getOne(ty, loc, rewriter);
     Value half;
     if (VectorType vty = dyn_cast<VectorType>(ty)) {
-      half = rewriter.create<spirv::ConstantOp>(
+      half = spirv::ConstantOp::create(rewriter,
           loc, vty,
           DenseElementsAttr::get(vty,
                                  rewriter.getFloatAttr(ety, 0.5).getValue()));
     } else {
-      half = rewriter.create<spirv::ConstantOp>(
+      half = spirv::ConstantOp::create(rewriter,
           loc, ty, rewriter.getFloatAttr(ety, 0.5));
     }
 
-    auto abs = rewriter.create<spirv::GLFAbsOp>(loc, operand);
-    auto floor = rewriter.create<spirv::GLFloorOp>(loc, abs);
-    auto sub = rewriter.create<spirv::FSubOp>(loc, abs, floor);
+    auto abs = spirv::GLFAbsOp::create(rewriter, loc, operand);
+    auto floor = spirv::GLFloorOp::create(rewriter, loc, abs);
+    auto sub = spirv::FSubOp::create(rewriter, loc, abs, floor);
     auto greater =
-        rewriter.create<spirv::FOrdGreaterThanEqualOp>(loc, sub, half);
-    auto select = rewriter.create<spirv::SelectOp>(loc, greater, one, zero);
-    auto add = rewriter.create<spirv::FAddOp>(loc, floor, select);
+        spirv::FOrdGreaterThanEqualOp::create(rewriter, loc, sub, half);
+    auto select = spirv::SelectOp::create(rewriter, loc, greater, one, zero);
+    auto add = spirv::FAddOp::create(rewriter, loc, floor, select);
     rewriter.replaceOpWithNewOp<math::CopySignOp>(roundOp, add, operand);
     return success();
   }
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
index db244d1d1cac8..db6ce600af76a 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
@@ -161,7 +161,7 @@ struct ConvertLoad final : public OpConversionPattern<memref::LoadOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = rewriter.create<emitc::SubscriptOp>(
+    auto subscript = emitc::SubscriptOp::create(rewriter,
         op.getLoc(), arrayValue, operands.getIndices());
 
     rewriter.replaceOpWithNewOp<emitc::LoadOp>(op, resultTy, subscript);
@@ -181,7 +181,7 @@ struct ConvertStore final : public OpConversionPattern<memref::StoreOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = rewriter.create<emitc::SubscriptOp>(
+    auto subscript = emitc::SubscriptOp::create(rewriter,
         op.getLoc(), arrayValue, operands.getIndices());
     rewriter.replaceOpWithNewOp<emitc::AssignOp>(op, subscript,
                                                  operands.getValue());
@@ -212,7 +212,7 @@ void mlir::populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   };
 
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 00bbdcb12e326..1603b1ba7eb9e 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -87,12 +87,12 @@ getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
 ///   aligned = bumped - bumped % alignment
 static Value createAligned(ConversionPatternRewriter &rewriter, Location loc,
                            Value input, Value alignment) {
-  Value one = rewriter.create<LLVM::ConstantOp>(loc, alignment.getType(),
+  Value one = LLVM::ConstantOp::create(rewriter, loc, alignment.getType(),
                                                 rewriter.getIndexAttr(1));
-  Value bump = rewriter.create<LLVM::SubOp>(loc, alignment, one);
-  Value bumped = rewriter.create<LLVM::AddOp>(loc, input, bump);
-  Value mod = rewriter.create<LLVM::URemOp>(loc, bumped, alignment);
-  return rewriter.create<LLVM::SubOp>(loc, bumped, mod);
+  Value bump = LLVM::SubOp::create(rewriter, loc, alignment, one);
+  Value bumped = LLVM::AddOp::create(rewriter, loc, input, bump);
+  Value mod = LLVM::URemOp::create(rewriter, loc, bumped, alignment);
+  return LLVM::SubOp::create(rewriter, loc, bumped, mod);
 }
 
 /// Computes the byte size for the MemRef element type.
@@ -123,7 +123,7 @@ static Value castAllocFuncResult(ConversionPatternRewriter &rewriter,
   assert(succeeded(maybeMemrefAddrSpace) && "unsupported address space");
   unsigned memrefAddrSpace = *maybeMemrefAddrSpace;
   if (allocatedPtrTy.getAddressSpace() != memrefAddrSpace)
-    allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+    allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
         allocatedPtr);
   return allocatedPtr;
@@ -168,14 +168,14 @@ class AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
     Value alignment = getAlignment(rewriter, loc, op);
     if (alignment) {
       // Adjust the allocation size to consider alignment.
-      sizeBytes = rewriter.create<LLVM::AddOp>(loc, sizeBytes, alignment);
+      sizeBytes = LLVM::AddOp::create(rewriter, loc, sizeBytes, alignment);
     }
 
     // Allocate the underlying buffer.
     Type elementPtrType = this->getElementPtrType(memRefType);
     assert(elementPtrType && "could not compute element ptr type");
     auto results =
-        rewriter.create<LLVM::CallOp>(loc, allocFuncOp.value(), sizeBytes);
+        LLVM::CallOp::create(rewriter, loc, allocFuncOp.value(), sizeBytes);
 
     Value allocatedPtr =
         castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
@@ -184,11 +184,11 @@ class AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
     if (alignment) {
       // Compute the aligned pointer.
       Value allocatedInt =
-          rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), allocatedPtr);
+          LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), allocatedPtr);
       Value alignmentInt =
           createAligned(rewriter, loc, allocatedInt, alignment);
       alignedPtr =
-          rewriter.create<LLVM::IntToPtrOp>(loc, elementPtrType, alignmentInt);
+          LLVM::IntToPtrOp::create(rewriter, loc, elementPtrType, alignmentInt);
     }
 
     // Create the MemRef descriptor.
@@ -268,7 +268,7 @@ class AlignedAllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
       sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
 
     Type elementPtrType = this->getElementPtrType(memRefType);
-    auto results = rewriter.create<LLVM::CallOp>(
+    auto results = LLVM::CallOp::create(rewriter,
         loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
 
     Value ptr =
@@ -360,7 +360,7 @@ struct AllocaOpLowering : public ConvertOpToLLVMPattern<memref::AllocaOp> {
     auto elementPtrType =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addrSpace);
 
-    auto allocatedElementPtr = rewriter.create<LLVM::AllocaOp>(
+    auto allocatedElementPtr = LLVM::AllocaOp::create(rewriter,
         loc, elementPtrType, elementType, size, op.getAlignment().value_or(0));
 
     // Create the MemRef descriptor.
@@ -397,7 +397,7 @@ struct AllocaScopeOpLowering
           remainingOpsBlock, allocaScopeOp.getResultTypes(),
           SmallVector<Location>(allocaScopeOp->getNumResults(),
                                 allocaScopeOp.getLoc()));
-      rewriter.create<LLVM::BrOp>(loc, ValueRange(), remainingOpsBlock);
+      LLVM::BrOp::create(rewriter, loc, ValueRange(), remainingOpsBlock);
     }
 
     // Inline body region.
@@ -407,8 +407,8 @@ struct AllocaScopeOpLowering
 
     // Save stack and then branch into the body of the region.
     rewriter.setInsertionPointToEnd(currentBlock);
-    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
-    rewriter.create<LLVM::BrOp>(loc, ValueRange(), beforeBody);
+    auto stackSaveOp = LLVM::StackSaveOp::create(rewriter, loc, getPtrType());
+    LLVM::BrOp::create(rewriter, loc, ValueRange(), beforeBody);
 
     // Replace the alloca_scope return with a branch that jumps out of the body.
     // Stack restore before leaving the body region.
@@ -420,7 +420,7 @@ struct AllocaScopeOpLowering
 
     // Insert stack restore before jumping out the body of the region.
     rewriter.setInsertionPoint(branchOp);
-    rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);
+    LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
 
     // Replace the op with values return from the body region.
     rewriter.replaceOp(allocaScopeOp, continueBlock->getArguments());
@@ -451,10 +451,10 @@ struct AssumeAlignmentOpLowering
     // This is more direct than ptrtoint-based checks, is explicitly supported,
     // and works with non-integral address spaces.
     Value trueCond =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getBoolAttr(true));
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getBoolAttr(true));
     Value alignmentConst =
         createIndexAttrConstant(rewriter, loc, getIndexType(), alignment);
-    rewriter.create<LLVM::AssumeOp>(loc, trueCond, LLVM::AssumeAlignTag(), ptr,
+    LLVM::AssumeOp::create(rewriter, loc, trueCond, LLVM::AssumeAlignTag(), ptr,
                                     alignmentConst);
     rewriter.replaceOp(op, memref);
     return success();
@@ -559,16 +559,16 @@ struct DimOpLowering : public ConvertOpToLLVMPattern<memref::DimOp> {
     // Get pointer to offset field of memref<element_type> descriptor.
     auto indexPtrTy =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addressSpace);
-    Value offsetPtr = rewriter.create<LLVM::GEPOp>(
+    Value offsetPtr = LLVM::GEPOp::create(rewriter,
         loc, indexPtrTy, elementType, underlyingRankedDesc,
         ArrayRef<LLVM::GEPArg>{0, 2});
 
     // The size value that we have to extract can be obtained using GEPop with
     // `dimOp.index() + 1` index argument.
-    Value idxPlusOne = rewriter.create<LLVM::AddOp>(
+    Value idxPlusOne = LLVM::AddOp::create(rewriter,
         loc, createIndexAttrConstant(rewriter, loc, getIndexType(), 1),
         adaptor.getIndex());
-    Value sizePtr = rewriter.create<LLVM::GEPOp>(
+    Value sizePtr = LLVM::GEPOp::create(rewriter,
         loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr,
         idxPlusOne);
     return rewriter
@@ -674,9 +674,9 @@ struct GenericAtomicRMWOpLowering
     auto memRefType = cast<MemRefType>(atomicOp.getMemref().getType());
     auto dataPtr = getStridedElementPtr(
         rewriter, loc, memRefType, adaptor.getMemref(), adaptor.getIndices());
-    Value init = rewriter.create<LLVM::LoadOp>(
+    Value init = LLVM::LoadOp::create(rewriter,
         loc, typeConverter->convertType(memRefType.getElementType()), dataPtr);
-    rewriter.create<LLVM::BrOp>(loc, init, loopBlock);
+    LLVM::BrOp::create(rewriter, loc, init, loopBlock);
 
     // Prepare the body of the loop block.
     rewriter.setInsertionPointToStart(loopBlock);
@@ -696,14 +696,14 @@ struct GenericAtomicRMWOpLowering
     // Append the cmpxchg op to the end of the loop block.
     auto successOrdering = LLVM::AtomicOrdering::acq_rel;
     auto failureOrdering = LLVM::AtomicOrdering::monotonic;
-    auto cmpxchg = rewriter.create<LLVM::AtomicCmpXchgOp>(
+    auto cmpxchg = LLVM::AtomicCmpXchgOp::create(rewriter,
         loc, dataPtr, loopArgument, result, successOrdering, failureOrdering);
     // Extract the %new_loaded and %ok values from the pair.
-    Value newLoaded = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 0);
-    Value ok = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 1);
+    Value newLoaded = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 0);
+    Value ok = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 1);
 
     // Conditionally branch to the end or back to the loop depending on %ok.
-    rewriter.create<LLVM::CondBrOp>(loc, ok, endBlock, ArrayRef<Value>(),
+    LLVM::CondBrOp::create(rewriter, loc, ok, endBlock, ArrayRef<Value>(),
                                     loopBlock, newLoaded);
 
     rewriter.setInsertionPointToEnd(endBlock);
@@ -800,8 +800,8 @@ class GlobalMemrefOpLowering : public ConvertOpToLLVMPattern<memref::GlobalOp> {
     if (!global.isExternal() && global.isUninitialized()) {
       rewriter.createBlock(&newGlobal.getInitializerRegion());
       Value undef[] = {
-          rewriter.create<LLVM::UndefOp>(global.getLoc(), arrayTy)};
-      rewriter.create<LLVM::ReturnOp>(global.getLoc(), undef);
+          LLVM::UndefOp::create(rewriter, global.getLoc(), arrayTy)};
+      LLVM::ReturnOp::create(rewriter, global.getLoc(), undef);
     }
     return success();
   }
@@ -846,11 +846,11 @@ struct GetGlobalMemrefOpLowering
     Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter());
     auto ptrTy = LLVM::LLVMPointerType::get(rewriter.getContext(), memSpace);
     auto addressOf =
-        rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, op.getName());
+        LLVM::AddressOfOp::create(rewriter, loc, ptrTy, op.getName());
 
     // Get the address of the first element in the array by creating a GEP with
     // the address of the GV as the base, and (rank + 1) number of 0 indices.
-    auto gep = rewriter.create<LLVM::GEPOp>(
+    auto gep = LLVM::GEPOp::create(rewriter,
         loc, ptrTy, arrayTy, addressOf,
         SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
 
@@ -861,7 +861,7 @@ struct GetGlobalMemrefOpLowering
     Value deadBeefConst =
         createIndexAttrConstant(rewriter, op->getLoc(), intPtrType, 0xdeadbeef);
     auto deadBeefPtr =
-        rewriter.create<LLVM::IntToPtrOp>(loc, ptrTy, deadBeefConst);
+        LLVM::IntToPtrOp::create(rewriter, loc, ptrTy, deadBeefConst);
 
     // Both allocated and aligned pointers are same. We could potentially stash
     // a nullptr for the allocated pointer since we do not expect any dealloc.
@@ -1013,7 +1013,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
           loc, adaptor.getSource(), rewriter);
 
       // rank = ConstantOp srcRank
-      auto rankVal = rewriter.create<LLVM::ConstantOp>(
+      auto rankVal = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(rank));
       // poison = PoisonOp
       UnrankedMemRefDescriptor memRefDesc =
@@ -1033,7 +1033,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
       auto ptr = memRefDesc.memRefDescPtr(rewriter, loc);
 
       // struct = LoadOp ptr
-      auto loadOp = rewriter.create<LLVM::LoadOp>(loc, targetStructType, ptr);
+      auto loadOp = LLVM::LoadOp::create(rewriter, loc, targetStructType, ptr);
       rewriter.replaceOp(memRefCastOp, loadOp.getResult());
     } else {
       llvm_unreachable("Unsupported unranked memref to unranked memref cast");
@@ -1067,31 +1067,31 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     MemRefDescriptor srcDesc(adaptor.getSource());
 
     // Compute number of elements.
-    Value numElements = rewriter.create<LLVM::ConstantOp>(
+    Value numElements = LLVM::ConstantOp::create(rewriter,
         loc, getIndexType(), rewriter.getIndexAttr(1));
     for (int pos = 0; pos < srcType.getRank(); ++pos) {
       auto size = srcDesc.size(rewriter, loc, pos);
-      numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
     }
 
     // Get element size.
     auto sizeInBytes = getSizeInBytes(loc, srcType.getElementType(), rewriter);
     // Compute total.
     Value totalSize =
-        rewriter.create<LLVM::MulOp>(loc, numElements, sizeInBytes);
+        LLVM::MulOp::create(rewriter, loc, numElements, sizeInBytes);
 
     Type elementType = typeConverter->convertType(srcType.getElementType());
 
     Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc);
     Value srcOffset = srcDesc.offset(rewriter, loc);
-    Value srcPtr = rewriter.create<LLVM::GEPOp>(
+    Value srcPtr = LLVM::GEPOp::create(rewriter,
         loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset);
     MemRefDescriptor targetDesc(adaptor.getTarget());
     Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc);
     Value targetOffset = targetDesc.offset(rewriter, loc);
-    Value targetPtr = rewriter.create<LLVM::GEPOp>(
+    Value targetPtr = LLVM::GEPOp::create(rewriter,
         loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset);
-    rewriter.create<LLVM::MemcpyOp>(loc, targetPtr, srcPtr, totalSize,
+    LLVM::MemcpyOp::create(rewriter, loc, targetPtr, srcPtr, totalSize,
                                     /*isVolatile=*/false);
     rewriter.eraseOp(op);
 
@@ -1107,7 +1107,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
 
     // First make sure we have an unranked memref descriptor representation.
     auto makeUnranked = [&, this](Value ranked, MemRefType type) {
-      auto rank = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      auto rank = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                     type.getRank());
       auto *typeConverter = getTypeConverter();
       auto ptr =
@@ -1120,7 +1120,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     };
 
     // Save stack position before promoting descriptors
-    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
+    auto stackSaveOp = LLVM::StackSaveOp::create(rewriter, loc, getPtrType());
 
     auto srcMemRefType = dyn_cast<MemRefType>(srcType);
     Value unrankedSource =
@@ -1132,13 +1132,13 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
                          : adaptor.getTarget();
 
     // Now promote the unranked descriptors to the stack.
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+    auto one = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                  rewriter.getIndexAttr(1));
     auto promote = [&](Value desc) {
       auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
       auto allocated =
-          rewriter.create<LLVM::AllocaOp>(loc, ptrType, desc.getType(), one);
-      rewriter.create<LLVM::StoreOp>(loc, desc, allocated);
+          LLVM::AllocaOp::create(rewriter, loc, ptrType, desc.getType(), one);
+      LLVM::StoreOp::create(rewriter, loc, desc, allocated);
       return allocated;
     };
 
@@ -1153,11 +1153,11 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
         sourcePtr.getType(), symbolTables);
     if (failed(copyFn))
       return failure();
-    rewriter.create<LLVM::CallOp>(loc, copyFn.value(),
+    LLVM::CallOp::create(rewriter, loc, copyFn.value(),
                                   ValueRange{elemSize, sourcePtr, targetPtr});
 
     // Restore stack used for descriptors
-    rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);
+    LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
 
     rewriter.eraseOp(op);
 
@@ -1208,9 +1208,9 @@ struct MemorySpaceCastOpLowering
       MemRefDescriptor::unpack(rewriter, loc, adaptor.getSource(), resultTypeR,
                                descVals);
       descVals[0] =
-          rewriter.create<LLVM::AddrSpaceCastOp>(loc, newPtrType, descVals[0]);
+          LLVM::AddrSpaceCastOp::create(rewriter, loc, newPtrType, descVals[0]);
       descVals[1] =
-          rewriter.create<LLVM::AddrSpaceCastOp>(loc, newPtrType, descVals[1]);
+          LLVM::AddrSpaceCastOp::create(rewriter, loc, newPtrType, descVals[1]);
       Value result = MemRefDescriptor::pack(rewriter, loc, *getTypeConverter(),
                                             resultTypeR, descVals);
       rewriter.replaceOp(op, result);
@@ -1245,7 +1245,7 @@ struct MemorySpaceCastOpLowering
       UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                              result, resultAddrSpace, sizes);
       Value resultUnderlyingSize = sizes.front();
-      Value resultUnderlyingDesc = rewriter.create<LLVM::AllocaOp>(
+      Value resultUnderlyingDesc = LLVM::AllocaOp::create(rewriter,
           loc, getPtrType(), rewriter.getI8Type(), resultUnderlyingSize);
       result.setMemRefDescPtr(rewriter, loc, resultUnderlyingDesc);
 
@@ -1260,9 +1260,9 @@ struct MemorySpaceCastOpLowering
       Value alignedPtr =
           sourceDesc.alignedPtr(rewriter, loc, *getTypeConverter(),
                                 sourceUnderlyingDesc, sourceElemPtrType);
-      allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+      allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
           loc, resultElemPtrType, allocatedPtr);
-      alignedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+      alignedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
           loc, resultElemPtrType, alignedPtr);
 
       result.setAllocatedPtr(rewriter, loc, resultUnderlyingDesc,
@@ -1281,11 +1281,11 @@ struct MemorySpaceCastOpLowering
       int64_t bytesToSkip =
           2 * llvm::divideCeil(
                   getTypeConverter()->getPointerBitwidth(resultAddrSpace), 8);
-      Value bytesToSkipConst = rewriter.create<LLVM::ConstantOp>(
+      Value bytesToSkipConst = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(bytesToSkip));
-      Value copySize = rewriter.create<LLVM::SubOp>(
+      Value copySize = LLVM::SubOp::create(rewriter,
           loc, getIndexType(), resultUnderlyingSize, bytesToSkipConst);
-      rewriter.create<LLVM::MemcpyOp>(loc, resultIndexVals, sourceIndexVals,
+      LLVM::MemcpyOp::create(rewriter, loc, resultIndexVals, sourceIndexVals,
                                       copySize, /*isVolatile=*/false);
 
       rewriter.replaceOp(op, ValueRange{result});
@@ -1489,7 +1489,7 @@ struct MemRefReshapeOpLowering
         } else {
           Value shapeOp = reshapeOp.getShape();
           Value index = createIndexAttrConstant(rewriter, loc, indexType, i);
-          dimSize = rewriter.create<memref::LoadOp>(loc, shapeOp, index);
+          dimSize = memref::LoadOp::create(rewriter, loc, shapeOp, index);
           Type indexType = getIndexType();
           if (dimSize.getType() != indexType)
             dimSize = typeConverter->materializeTargetConversion(
@@ -1501,7 +1501,7 @@ struct MemRefReshapeOpLowering
         desc.setStride(rewriter, loc, i, stride);
 
         // Prepare the stride value for the next dimension.
-        stride = rewriter.create<LLVM::MulOp>(loc, stride, dimSize);
+        stride = LLVM::MulOp::create(rewriter, loc, stride, dimSize);
       }
 
       *descriptor = desc;
@@ -1526,7 +1526,7 @@ struct MemRefReshapeOpLowering
     SmallVector<Value, 4> sizes;
     UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                            targetDesc, addressSpace, sizes);
-    Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>(
+    Value underlyingDescPtr = LLVM::AllocaOp::create(rewriter,
         loc, getPtrType(), IntegerType::get(getContext(), 8), sizes.front());
     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
 
@@ -1558,7 +1558,7 @@ struct MemRefReshapeOpLowering
     Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc);
     Value oneIndex = createIndexAttrConstant(rewriter, loc, getIndexType(), 1);
     Value resultRankMinusOne =
-        rewriter.create<LLVM::SubOp>(loc, resultRank, oneIndex);
+        LLVM::SubOp::create(rewriter, loc, resultRank, oneIndex);
 
     Block *initBlock = rewriter.getInsertionBlock();
     Type indexType = getTypeConverter()->getIndexType();
@@ -1572,14 +1572,14 @@ struct MemRefReshapeOpLowering
     rewriter.mergeBlocks(remainingBlock, condBlock, ValueRange());
 
     rewriter.setInsertionPointToEnd(initBlock);
-    rewriter.create<LLVM::BrOp>(loc, ValueRange({resultRankMinusOne, oneIndex}),
+    LLVM::BrOp::create(rewriter, loc, ValueRange({resultRankMinusOne, oneIndex}),
                                 condBlock);
     rewriter.setInsertionPointToStart(condBlock);
     Value indexArg = condBlock->getArgument(0);
     Value strideArg = condBlock->getArgument(1);
 
     Value zeroIndex = createIndexAttrConstant(rewriter, loc, indexType, 0);
-    Value pred = rewriter.create<LLVM::ICmpOp>(
+    Value pred = LLVM::ICmpOp::create(rewriter,
         loc, IntegerType::get(rewriter.getContext(), 1),
         LLVM::ICmpPredicate::sge, indexArg, zeroIndex);
 
@@ -1589,22 +1589,22 @@ struct MemRefReshapeOpLowering
 
     // Copy size from shape to descriptor.
     auto llvmIndexPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value sizeLoadGep = rewriter.create<LLVM::GEPOp>(
+    Value sizeLoadGep = LLVM::GEPOp::create(rewriter,
         loc, llvmIndexPtrType,
         typeConverter->convertType(shapeMemRefType.getElementType()),
         shapeOperandPtr, indexArg);
-    Value size = rewriter.create<LLVM::LoadOp>(loc, indexType, sizeLoadGep);
+    Value size = LLVM::LoadOp::create(rewriter, loc, indexType, sizeLoadGep);
     UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(),
                                       targetSizesBase, indexArg, size);
 
     // Write stride value and compute next one.
     UnrankedMemRefDescriptor::setStride(rewriter, loc, *getTypeConverter(),
                                         targetStridesBase, indexArg, strideArg);
-    Value nextStride = rewriter.create<LLVM::MulOp>(loc, strideArg, size);
+    Value nextStride = LLVM::MulOp::create(rewriter, loc, strideArg, size);
 
     // Decrement loop counter and branch back.
-    Value decrement = rewriter.create<LLVM::SubOp>(loc, indexArg, oneIndex);
-    rewriter.create<LLVM::BrOp>(loc, ValueRange({decrement, nextStride}),
+    Value decrement = LLVM::SubOp::create(rewriter, loc, indexArg, oneIndex);
+    LLVM::BrOp::create(rewriter, loc, ValueRange({decrement, nextStride}),
                                 condBlock);
 
     Block *remainder =
@@ -1612,7 +1612,7 @@ struct MemRefReshapeOpLowering
 
     // Hook up the cond exit to the remainder.
     rewriter.setInsertionPointToEnd(condBlock);
-    rewriter.create<LLVM::CondBrOp>(loc, pred, bodyBlock, ValueRange(),
+    LLVM::CondBrOp::create(rewriter, loc, pred, bodyBlock, ValueRange(),
                                     remainder, ValueRange());
 
     // Reset position to beginning of new remainder block.
@@ -1742,7 +1742,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
       return createIndexAttrConstant(rewriter, loc, indexType, strides[idx]);
     if (nextSize)
       return runningStride
-                 ? rewriter.create<LLVM::MulOp>(loc, runningStride, nextSize)
+                 ? LLVM::MulOp::create(rewriter, loc, runningStride, nextSize)
                  : nextSize;
     assert(!runningStride);
     return createIndexAttrConstant(rewriter, loc, indexType, 1);
@@ -1787,7 +1787,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
 
     // Field 2: Copy the actual aligned pointer to payload.
     Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc);
-    alignedPtr = rewriter.create<LLVM::GEPOp>(
+    alignedPtr = LLVM::GEPOp::create(rewriter,
         loc, alignedPtr.getType(),
         typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr,
         adaptor.getByteShift());
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index b866afbce98b0..7b7ad9b95b0ec 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -79,7 +79,7 @@ adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter,
   assert(indices.size() == 2);
   indices.back() = builder.createOrFold<spirv::SDivOp>(loc, lastDim, idx);
   Type t = typeConverter.convertType(op.getComponentPtr().getType());
-  return builder.create<spirv::AccessChainOp>(loc, t, op.getBasePtr(), indices);
+  return spirv::AccessChainOp::create(builder, loc, t, op.getBasePtr(), indices);
 }
 
 /// Casts the given `srcBool` into an integer of `dstType`.
@@ -107,7 +107,7 @@ static Value shiftValue(Location loc, Value value, Value offset, Value mask,
     value = castBoolToIntN(loc, value, dstType, builder);
   } else {
     if (valueBits < targetBits) {
-      value = builder.create<spirv::UConvertOp>(
+      value = spirv::UConvertOp::create(builder,
           loc, builder.getIntegerType(targetBits), value);
     }
 
@@ -372,7 +372,7 @@ AllocOpPattern::matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor,
     std::string varName =
         std::string("__workgroup_mem__") +
         std::to_string(std::distance(varOps.begin(), varOps.end()));
-    varOp = rewriter.create<spirv::GlobalVariableOp>(loc, spirvType, varName,
+    varOp = spirv::GlobalVariableOp::create(rewriter, loc, spirvType, varName,
                                                      /*initializer=*/nullptr);
   }
 
@@ -572,7 +572,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
           loadOp, "failed to determine memory requirements");
 
     auto [memoryAccess, alignment] = *memoryRequirements;
-    Value loadVal = rewriter.create<spirv::LoadOp>(loc, accessChain,
+    Value loadVal = spirv::LoadOp::create(rewriter, loc, accessChain,
                                                    memoryAccess, alignment);
     if (isBool)
       loadVal = castIntNToBool(loc, loadVal, rewriter);
@@ -601,7 +601,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
         loadOp, "failed to determine memory requirements");
 
   auto [memoryAccess, alignment] = *memoryRequirements;
-  Value spvLoadOp = rewriter.create<spirv::LoadOp>(loc, dstType, adjustedPtr,
+  Value spvLoadOp = spirv::LoadOp::create(rewriter, loc, dstType, adjustedPtr,
                                                    memoryAccess, alignment);
 
   // Shift the bits to the rightmost.
@@ -770,10 +770,10 @@ IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor,
   if (!scope)
     return rewriter.notifyMatchFailure(storeOp, "atomic scope not available");
 
-  Value result = rewriter.create<spirv::AtomicAndOp>(
+  Value result = spirv::AtomicAndOp::create(rewriter,
       loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
       clearBitsMask);
-  result = rewriter.create<spirv::AtomicOrOp>(
+  result = spirv::AtomicOrOp::create(rewriter,
       loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
       storeVal);
 
@@ -851,11 +851,11 @@ LogicalResult MemorySpaceCastOpPattern::matchAndRewrite(
   }
   if (sourceSc != spirv::StorageClass::Generic) {
     result =
-        rewriter.create<spirv::PtrCastToGenericOp>(loc, genericPtrType, result);
+        spirv::PtrCastToGenericOp::create(rewriter, loc, genericPtrType, result);
   }
   if (resultSc != spirv::StorageClass::Generic) {
     result =
-        rewriter.create<spirv::GenericCastToPtrOp>(loc, resultPtrType, result);
+        spirv::GenericCastToPtrOp::create(rewriter, loc, resultPtrType, result);
   }
   rewriter.replaceOp(addrCastOp, result);
   return success();
diff --git a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
index b93128441f2b5..f16b92a287d0a 100644
--- a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
+++ b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
@@ -65,7 +65,7 @@ static SmallVector<Value> getMixedAsValues(OpBuilder b, const Location &loc,
       values.emplace_back(*(dyn++));
     } else {
       TypedAttr val = type == i64 ? b.getI64IntegerAttr(s) : b.getIndexAttr(s);
-      values.emplace_back(b.create<arith::ConstantOp>(loc, type, val));
+      values.emplace_back(arith::ConstantOp::create(b, loc, type, val));
     }
   }
   return values;
@@ -79,9 +79,9 @@ static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
   SmallVector<Value> multiIndex(n);
 
   for (int i = n - 1; i >= 0; --i) {
-    multiIndex[i] = b.create<arith::RemSIOp>(loc, linearIndex, dimensions[i]);
+    multiIndex[i] = arith::RemSIOp::create(b, loc, linearIndex, dimensions[i]);
     if (i > 0)
-      linearIndex = b.create<arith::DivSIOp>(loc, linearIndex, dimensions[i]);
+      linearIndex = arith::DivSIOp::create(b, loc, linearIndex, dimensions[i]);
   }
 
   return multiIndex;
@@ -91,13 +91,13 @@ static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
 Value multiToLinearIndex(Location loc, OpBuilder b, ValueRange multiIndex,
                          ValueRange dimensions) {
 
-  Value linearIndex = b.create<arith::ConstantIndexOp>(loc, 0);
-  Value stride = b.create<arith::ConstantIndexOp>(loc, 1);
+  Value linearIndex = arith::ConstantIndexOp::create(b, loc, 0);
+  Value stride = arith::ConstantIndexOp::create(b, loc, 1);
 
   for (int i = multiIndex.size() - 1; i >= 0; --i) {
-    Value off = b.create<arith::MulIOp>(loc, multiIndex[i], stride);
-    linearIndex = b.create<arith::AddIOp>(loc, linearIndex, off);
-    stride = b.create<arith::MulIOp>(loc, stride, dimensions[i]);
+    Value off = arith::MulIOp::create(b, loc, multiIndex[i], stride);
+    linearIndex = arith::AddIOp::create(b, loc, linearIndex, off);
+    stride = arith::MulIOp::create(b, loc, stride, dimensions[i]);
   }
 
   return linearIndex;
@@ -144,10 +144,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     auto i64 = rewriter.getI64Type();
     std::array<int64_t, 2> shape = {static_cast<int64_t>(splitAxes.size()),
                                     maxNAxes};
-    Value resSplitAxes = rewriter.create<tensor::EmptyOp>(loc, shape, i16);
+    Value resSplitAxes = tensor::EmptyOp::create(rewriter, loc, shape, i16);
     auto attr = IntegerAttr::get(i16, -1);
-    Value fillValue = rewriter.create<arith::ConstantOp>(loc, i16, attr);
-    resSplitAxes = rewriter.create<linalg::FillOp>(loc, fillValue, resSplitAxes)
+    Value fillValue = arith::ConstantOp::create(rewriter, loc, i16, attr);
+    resSplitAxes = linalg::FillOp::create(rewriter, loc, fillValue, resSplitAxes)
                        .getResult(0);
 
     // explicitly write values into tensor row by row
@@ -162,8 +162,8 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       std::array<int64_t, 2> sizes = {1, size};
       auto tensorType = RankedTensorType::get({size}, i16);
       auto attrs = DenseIntElementsAttr::get(tensorType, axes.asArrayRef());
-      auto vals = rewriter.create<arith::ConstantOp>(loc, tensorType, attrs);
-      resSplitAxes = rewriter.create<tensor::InsertSliceOp>(
+      auto vals = arith::ConstantOp::create(rewriter, loc, tensorType, attrs);
+      resSplitAxes = tensor::InsertSliceOp::create(rewriter,
           loc, vals, resSplitAxes, empty, empty, empty, offs, sizes, strides);
     }
 
@@ -179,7 +179,7 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
                   .create<tensor::EmptyOp>(loc, std::array<int64_t, 2>{0, 0},
                                            i64)
                   .getResult()
-            : rewriter.create<tensor::FromElementsOp>(loc, type, haloSizes)
+            : tensor::FromElementsOp::create(rewriter, loc, type, haloSizes)
                   .getResult();
 
     // To hold sharded dims offsets, create Tensor with shape {nSplits,
@@ -189,7 +189,7 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     // MeshOp)
     Value resOffsets;
     if (adaptor.getStaticShardedDimsOffsets().empty()) {
-      resOffsets = rewriter.create<tensor::EmptyOp>(
+      resOffsets = tensor::EmptyOp::create(rewriter,
           loc, std::array<int64_t, 2>{0, 0}, i64);
     } else {
       SymbolTableCollection symbolTableCollection;
@@ -204,12 +204,12 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       assert(maxSplitSize);
       ++maxSplitSize; // add one for the total size
 
-      resOffsets = rewriter.create<tensor::EmptyOp>(
+      resOffsets = tensor::EmptyOp::create(rewriter,
           loc, std::array<int64_t, 2>{nSplits, maxSplitSize}, i64);
-      Value zero = rewriter.create<arith::ConstantOp>(
+      Value zero = arith::ConstantOp::create(rewriter,
           loc, i64, rewriter.getI64IntegerAttr(ShapedType::kDynamic));
       resOffsets =
-          rewriter.create<linalg::FillOp>(loc, zero, resOffsets).getResult(0);
+          linalg::FillOp::create(rewriter, loc, zero, resOffsets).getResult(0);
       SmallVector<Value> offsets =
           getMixedAsValues(rewriter, loc, adaptor.getStaticShardedDimsOffsets(),
                            adaptor.getDynamicShardedDimsOffsets());
@@ -220,10 +220,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
         assert(splitSize != ShapedType::kDynamic && splitSize < maxSplitSize);
         ++splitSize; // add one for the total size
         ArrayRef<Value> values(&offsets[curr], splitSize);
-        Value vals = rewriter.create<tensor::FromElementsOp>(loc, values);
+        Value vals = tensor::FromElementsOp::create(rewriter, loc, values);
         std::array<int64_t, 2> offs = {static_cast<int64_t>(i), 0};
         std::array<int64_t, 2> sizes = {1, splitSize};
-        resOffsets = rewriter.create<tensor::InsertSliceOp>(
+        resOffsets = tensor::InsertSliceOp::create(rewriter,
             loc, vals, resOffsets, empty, empty, empty, offs, sizes, strides);
         curr += splitSize;
       }
@@ -236,10 +236,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       return failure();
 
     resSplitAxes =
-        rewriter.create<tensor::CastOp>(loc, resTypes[0], resSplitAxes);
+        tensor::CastOp::create(rewriter, loc, resTypes[0], resSplitAxes);
     resHaloSizes =
-        rewriter.create<tensor::CastOp>(loc, resTypes[1], resHaloSizes);
-    resOffsets = rewriter.create<tensor::CastOp>(loc, resTypes[2], resOffsets);
+        tensor::CastOp::create(rewriter, loc, resTypes[1], resHaloSizes);
+    resOffsets = tensor::CastOp::create(rewriter, loc, resTypes[2], resOffsets);
 
     rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
         op, TupleType::get(op.getContext(), resTypes),
@@ -269,9 +269,9 @@ struct ConvertProcessMultiIndexOp
     SmallVector<Value> dims;
     llvm::transform(
         meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
-          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+          return arith::ConstantIndexOp::create(rewriter, loc, i).getResult();
         });
-    Value rank = rewriter.create<ProcessLinearIndexOp>(op.getLoc(), meshOp);
+    Value rank = ProcessLinearIndexOp::create(rewriter, op.getLoc(), meshOp);
     auto mIdx = linearToMultiIndex(loc, rewriter, rank, dims);
 
     // optionally extract subset of mesh axes
@@ -302,7 +302,7 @@ class ConvertProcessLinearIndexOp
     Location loc = op.getLoc();
     auto ctx = op.getContext();
     Value commWorld =
-        rewriter.create<mpi::CommWorldOp>(loc, mpi::CommType::get(ctx));
+        mpi::CommWorldOp::create(rewriter, loc, mpi::CommType::get(ctx));
     auto rank =
         rewriter
             .create<mpi::CommRankOp>(
@@ -341,40 +341,40 @@ struct ConvertNeighborsLinearIndicesOp
     SmallVector<Value> dims;
     llvm::transform(
         meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
-          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+          return arith::ConstantIndexOp::create(rewriter, loc, i).getResult();
         });
     Value dimSz = dims[axes[0]];
-    Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    Value minus1 = rewriter.create<arith::ConstantIndexOp>(loc, -1);
-    Value atBorder = rewriter.create<arith::CmpIOp>(
+    Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    Value minus1 = arith::ConstantIndexOp::create(rewriter, loc, -1);
+    Value atBorder = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sle, orgIdx,
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
-    auto down = rewriter.create<scf::IfOp>(
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
+    auto down = scf::IfOp::create(rewriter,
         loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
-          builder.create<scf::YieldOp>(loc, minus1);
+          scf::YieldOp::create(builder, loc, minus1);
         },
         [&](OpBuilder &builder, Location loc) {
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
-              rewriter.create<arith::SubIOp>(op.getLoc(), orgIdx, one)
+              arith::SubIOp::create(rewriter, op.getLoc(), orgIdx, one)
                   .getResult();
-          builder.create<scf::YieldOp>(
+          scf::YieldOp::create(builder,
               loc, multiToLinearIndex(loc, rewriter, tmp, dims));
         });
-    atBorder = rewriter.create<arith::CmpIOp>(
+    atBorder = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, orgIdx,
-        rewriter.create<arith::SubIOp>(loc, dimSz, one).getResult());
-    auto up = rewriter.create<scf::IfOp>(
+        arith::SubIOp::create(rewriter, loc, dimSz, one).getResult());
+    auto up = scf::IfOp::create(rewriter,
         loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
-          builder.create<scf::YieldOp>(loc, minus1);
+          scf::YieldOp::create(builder, loc, minus1);
         },
         [&](OpBuilder &builder, Location loc) {
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
-              rewriter.create<arith::AddIOp>(op.getLoc(), orgIdx, one);
-          builder.create<scf::YieldOp>(
+              arith::AddIOp::create(rewriter, op.getLoc(), orgIdx, one);
+          scf::YieldOp::create(builder,
               loc, multiToLinearIndex(loc, rewriter, tmp, dims));
         });
     rewriter.replaceOp(op, ValueRange{down.getResult(0), up.getResult(0)});
@@ -447,7 +447,7 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           rewriter, loc, sharding.getStaticShardedDimsOffsets(),
           sharding.getDynamicShardedDimsOffsets(), index);
       if (!tmp.empty())
-        shardedDimsOffs = rewriter.create<tensor::FromElementsOp>(
+        shardedDimsOffs = tensor::FromElementsOp::create(rewriter,
             loc, RankedTensorType::get({(int64_t)tmp.size()}, index), tmp);
     }
 
@@ -457,9 +457,9 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
     int64_t pos = 0;
     SmallVector<Value> shardShape;
     Value zero =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(index));
     Value one =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getOneAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getOneAttr(index));
 
     // Iterate over the dimensions of the tensor shape, get their split Axes,
     // and compute the sharded shape.
@@ -470,7 +470,7 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
         // The current dimension might not be sharded.
         // Create a value from the static position in shardDimsOffsets.
         Value posVal =
-            rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(pos));
+            arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(pos));
         // Get the index of the local shard in the mesh axis.
         Value idx = multiIdx[axes[0]];
         auto numShards =
@@ -482,29 +482,29 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
             return op->emitError() << "Only single axis sharding is "
                                    << "supported for each dimension.";
           }
-          idx = rewriter.create<arith::AddIOp>(loc, posVal, idx);
+          idx = arith::AddIOp::create(rewriter, loc, posVal, idx);
           // Compute size = shardedDimsOffs[idx+1] - shardedDimsOffs[idx].
           Value off =
-              rewriter.create<tensor::ExtractOp>(loc, shardedDimsOffs, idx);
-          idx = rewriter.create<arith::AddIOp>(loc, idx, one);
+              tensor::ExtractOp::create(rewriter, loc, shardedDimsOffs, idx);
+          idx = arith::AddIOp::create(rewriter, loc, idx, one);
           Value nextOff =
-              rewriter.create<tensor::ExtractOp>(loc, shardedDimsOffs, idx);
-          Value sz = rewriter.create<arith::SubIOp>(loc, nextOff, off);
+              tensor::ExtractOp::create(rewriter, loc, shardedDimsOffs, idx);
+          Value sz = arith::SubIOp::create(rewriter, loc, nextOff, off);
           shardShape.emplace_back(sz);
         } else {
-          Value numShardsVal = rewriter.create<arith::ConstantOp>(
+          Value numShardsVal = arith::ConstantOp::create(rewriter,
               loc, rewriter.getIndexAttr(numShards));
           // Compute shard dim size by distributing odd elements to trailing
           // shards:
           // sz = dim / numShards
           //      + (idx >= (numShards - (dim % numShards)) ? 1 : 0)
-          Value sz = rewriter.create<arith::DivSIOp>(loc, dim, numShardsVal);
-          Value sz1 = rewriter.create<arith::RemSIOp>(loc, dim, numShardsVal);
-          sz1 = rewriter.create<arith::SubIOp>(loc, numShardsVal, sz1);
-          auto cond = rewriter.create<arith::CmpIOp>(
+          Value sz = arith::DivSIOp::create(rewriter, loc, dim, numShardsVal);
+          Value sz1 = arith::RemSIOp::create(rewriter, loc, dim, numShardsVal);
+          sz1 = arith::SubIOp::create(rewriter, loc, numShardsVal, sz1);
+          auto cond = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::sge, idx, sz1);
-          Value odd = rewriter.create<arith::SelectOp>(loc, cond, one, zero);
-          sz = rewriter.create<arith::AddIOp>(loc, sz, odd);
+          Value odd = arith::SelectOp::create(rewriter, loc, cond, one, zero);
+          sz = arith::AddIOp::create(rewriter, loc, sz, odd);
           shardShape.emplace_back(sz);
         }
         pos += numShards + 1; // add one for the total size.
@@ -568,7 +568,7 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (isa<RankedTensorType>(input.getType())) {
       auto memrefType = MemRefType::get(
           inputShape, cast<ShapedType>(input.getType()).getElementType());
-      input = iBuilder.create<bufferization::ToBufferOp>(memrefType, input);
+      input = bufferization::ToBufferOp::create(iBuilder, memrefType, input);
     }
     MemRefType inType = cast<MemRefType>(input.getType());
 
@@ -577,15 +577,15 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     for (auto i = 0; i < inType.getRank(); ++i) {
       auto s = inputShape[i];
       if (ShapedType::isDynamic(s))
-        shape[i] = iBuilder.create<memref::DimOp>(input, s).getResult();
+        shape[i] = memref::DimOp::create(iBuilder, input, s).getResult();
       else
         shape[i] = iBuilder.getIndexAttr(s);
     }
 
     // Allocate buffer and copy input to buffer.
-    Value buffer = iBuilder.create<memref::AllocOp>(
+    Value buffer = memref::AllocOp::create(iBuilder,
         shape, cast<ShapedType>(op.getType()).getElementType());
-    iBuilder.create<linalg::CopyOp>(input, buffer);
+    linalg::CopyOp::create(iBuilder, input, buffer);
 
     // Get an MPI_Comm_split for the AllReduce operation.
     // The color is the linear index of the process in the mesh along the
@@ -594,9 +594,9 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
                                        iBuilder.getIndexType());
     SmallVector<Value> myMultiIndex =
-        iBuilder.create<ProcessMultiIndexOp>(indexResultTypes, mesh)
+        ProcessMultiIndexOp::create(iBuilder, indexResultTypes, mesh)
             .getResult();
-    Value zero = iBuilder.create<arith::ConstantIndexOp>(0);
+    Value zero = arith::ConstantIndexOp::create(iBuilder, 0);
     SmallVector<Value> multiKey(myMultiIndex.size(), zero);
 
     auto redAxes = adaptor.getMeshAxes();
@@ -607,15 +607,15 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
 
     Value color =
         createProcessLinearIndex(mesh, myMultiIndex, redAxes, iBuilder);
-    color = iBuilder.create<arith::IndexCastOp>(iBuilder.getI32Type(), color);
+    color = arith::IndexCastOp::create(iBuilder, iBuilder.getI32Type(), color);
     Value key = createProcessLinearIndex(mesh, multiKey, redAxes, iBuilder);
-    key = iBuilder.create<arith::IndexCastOp>(iBuilder.getI32Type(), key);
+    key = arith::IndexCastOp::create(iBuilder, iBuilder.getI32Type(), key);
 
     // Finally split the communicator
     auto commType = mpi::CommType::get(op->getContext());
-    Value commWorld = iBuilder.create<mpi::CommWorldOp>(commType);
+    Value commWorld = mpi::CommWorldOp::create(iBuilder, commType);
     auto comm =
-        iBuilder.create<mpi::CommSplitOp>(commType, commWorld, color, key)
+        mpi::CommSplitOp::create(iBuilder, commType, commWorld, color, key)
             .getNewcomm();
 
     Value buffer1d = buffer;
@@ -623,18 +623,18 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (inType.getRank() > 1) {
       ReassociationIndices reassociation(inType.getRank());
       std::iota(reassociation.begin(), reassociation.end(), 0);
-      buffer1d = iBuilder.create<memref::CollapseShapeOp>(
+      buffer1d = memref::CollapseShapeOp::create(iBuilder,
           buffer, ArrayRef<ReassociationIndices>(reassociation));
     }
 
     // Create the MPI AllReduce operation.
-    iBuilder.create<mpi::AllReduceOp>(
+    mpi::AllReduceOp::create(iBuilder,
         TypeRange(), buffer1d, buffer1d,
         getMPIReductionOp(adaptor.getReductionAttr()), comm);
 
     // If the destination is a memref, cast it to a tensor
     if (isa<RankedTensorType>(op.getType()))
-      buffer = iBuilder.create<bufferization::ToTensorOp>(op.getType(), buffer,
+      buffer = bufferization::ToTensorOp::create(iBuilder, op.getType(), buffer,
                                                           true);
 
     rewriter.replaceOp(op, buffer);
@@ -676,7 +676,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     auto toValue = [&rewriter, &loc](OpFoldResult &v) -> Value {
       if (auto value = dyn_cast<Value>(v))
         return value;
-      return rewriter.create<arith::ConstantOp>(
+      return arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(
                    cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
     };
@@ -689,7 +689,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       auto mmemrefType = MemRefType::get(
           dstShape, cast<ShapedType>(array.getType()).getElementType());
       array =
-          rewriter.create<bufferization::ToBufferOp>(loc, mmemrefType, array);
+          bufferization::ToBufferOp::create(rewriter, loc, mmemrefType, array);
     }
     auto rank = cast<ShapedType>(array.getType()).getRank();
     auto opSplitAxes = adaptor.getSplitAxes().getAxes();
@@ -713,7 +713,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     for (auto i = 0; i < rank; ++i) {
       auto s = dstShape[i];
       if (ShapedType::isDynamic(s))
-        shape[i] = rewriter.create<memref::DimOp>(loc, array, s).getResult();
+        shape[i] = memref::DimOp::create(rewriter, loc, array, s).getResult();
       else
         shape[i] = rewriter.getIndexAttr(s);
 
@@ -723,12 +723,12 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         offsets[i] = haloSizes[currHaloDim * 2];
 
         // prepare shape and offsets of highest dim's halo exchange
-        Value _haloSz = rewriter.create<arith::AddIOp>(
+        Value _haloSz = arith::AddIOp::create(rewriter,
             loc, toValue(haloSizes[currHaloDim * 2]),
             toValue(haloSizes[currHaloDim * 2 + 1]));
         // the halo shape of lower dims exlude the halos
         dimSizes[i] =
-            rewriter.create<arith::SubIOp>(loc, toValue(shape[i]), _haloSz)
+            arith::SubIOp::create(rewriter, loc, toValue(shape[i]), _haloSz)
                 .getResult();
       } else {
         dimSizes[i] = shape[i];
@@ -736,14 +736,14 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     }
 
     auto tagAttr = rewriter.getI32IntegerAttr(91); // we just pick something
-    auto tag = rewriter.create<arith::ConstantOp>(loc, tagAttr);
+    auto tag = arith::ConstantOp::create(rewriter, loc, tagAttr);
     auto zeroAttr = rewriter.getI32IntegerAttr(0); // for detecting v<0
-    auto zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
+    auto zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
 
     SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
                                        rewriter.getIndexType());
     auto myMultiIndex =
-        rewriter.create<ProcessMultiIndexOp>(loc, indexResultTypes, mesh)
+        ProcessMultiIndexOp::create(rewriter, loc, indexResultTypes, mesh)
             .getResult();
     // traverse all split axes from high to low dim
     for (ssize_t dim = opSplitAxes.size() - 1; dim >= 0; --dim) {
@@ -758,19 +758,19 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
                                                        splitAxes)
                      .getResults();
       // MPI operates on i32...
-      Value neighbourIDs[2] = {rewriter.create<arith::IndexCastOp>(
+      Value neighbourIDs[2] = {arith::IndexCastOp::create(rewriter,
                                    loc, rewriter.getI32Type(), tmp[0]),
-                               rewriter.create<arith::IndexCastOp>(
+                               arith::IndexCastOp::create(rewriter,
                                    loc, rewriter.getI32Type(), tmp[1])};
 
       auto lowerRecvOffset = rewriter.getIndexAttr(0);
       auto lowerSendOffset = toValue(haloSizes[currHaloDim * 2]);
-      auto upperRecvOffset = rewriter.create<arith::SubIOp>(
+      auto upperRecvOffset = arith::SubIOp::create(rewriter,
           loc, toValue(shape[dim]), toValue(haloSizes[currHaloDim * 2 + 1]));
-      auto upperSendOffset = rewriter.create<arith::SubIOp>(
+      auto upperSendOffset = arith::SubIOp::create(rewriter,
           loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
 
-      Value commWorld = rewriter.create<mpi::CommWorldOp>(
+      Value commWorld = mpi::CommWorldOp::create(rewriter,
           loc, mpi::CommType::get(op->getContext()));
 
       // Make sure we send/recv in a way that does not lead to a dead-lock.
@@ -787,37 +787,37 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         // Processes on the mesh borders have only one neighbor
         auto to = upperHalo ? neighbourIDs[0] : neighbourIDs[1];
         auto from = upperHalo ? neighbourIDs[1] : neighbourIDs[0];
-        auto hasFrom = rewriter.create<arith::CmpIOp>(
+        auto hasFrom = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sge, from, zero);
-        auto hasTo = rewriter.create<arith::CmpIOp>(
+        auto hasTo = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sge, to, zero);
-        auto buffer = rewriter.create<memref::AllocOp>(
+        auto buffer = memref::AllocOp::create(rewriter,
             loc, dimSizes, cast<ShapedType>(array.getType()).getElementType());
         // if has neighbor: copy halo data from array to buffer and send
-        rewriter.create<scf::IfOp>(
+        scf::IfOp::create(rewriter,
             loc, hasTo, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(lowerSendOffset)
                                        : OpFoldResult(upperSendOffset);
-              auto subview = builder.create<memref::SubViewOp>(
+              auto subview = memref::SubViewOp::create(builder,
                   loc, array, offsets, dimSizes, strides);
-              builder.create<memref::CopyOp>(loc, subview, buffer);
-              builder.create<mpi::SendOp>(loc, TypeRange{}, buffer, tag, to,
+              memref::CopyOp::create(builder, loc, subview, buffer);
+              mpi::SendOp::create(builder, loc, TypeRange{}, buffer, tag, to,
                                           commWorld);
-              builder.create<scf::YieldOp>(loc);
+              scf::YieldOp::create(builder, loc);
             });
         // if has neighbor: receive halo data into buffer and copy to array
-        rewriter.create<scf::IfOp>(
+        scf::IfOp::create(rewriter,
             loc, hasFrom, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(upperRecvOffset)
                                        : OpFoldResult(lowerRecvOffset);
-              builder.create<mpi::RecvOp>(loc, TypeRange{}, buffer, tag, from,
+              mpi::RecvOp::create(builder, loc, TypeRange{}, buffer, tag, from,
                                           commWorld);
-              auto subview = builder.create<memref::SubViewOp>(
+              auto subview = memref::SubViewOp::create(builder,
                   loc, array, offsets, dimSizes, strides);
-              builder.create<memref::CopyOp>(loc, buffer, subview);
-              builder.create<scf::YieldOp>(loc);
+              memref::CopyOp::create(builder, loc, buffer, subview);
+              scf::YieldOp::create(builder, loc);
             });
-        rewriter.create<memref::DeallocOp>(loc, buffer);
+        memref::DeallocOp::create(rewriter, loc, buffer);
         offsets[dim] = orgOffset;
       };
 
@@ -825,15 +825,15 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         OpFoldResult &v = haloSizes[currHaloDim * 2 + upOrDown];
         Value haloSz = dyn_cast<Value>(v);
         if (!haloSz)
-          haloSz = rewriter.create<arith::ConstantOp>(
+          haloSz = arith::ConstantOp::create(rewriter,
               loc, rewriter.getI32IntegerAttr(
                        cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
-        auto hasSize = rewriter.create<arith::CmpIOp>(
+        auto hasSize = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sgt, haloSz, zero);
-        rewriter.create<scf::IfOp>(loc, hasSize,
+        scf::IfOp::create(rewriter, loc, hasSize,
                                    [&](OpBuilder &builder, Location loc) {
                                      genSendRecv(upOrDown > 0);
-                                     builder.create<scf::YieldOp>(loc);
+                                     scf::YieldOp::create(builder, loc);
                                    });
       };
 
@@ -852,7 +852,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       rewriter.replaceOp(op, array);
     } else {
       assert(isa<RankedTensorType>(op.getResult().getType()));
-      rewriter.replaceOp(op, rewriter.create<bufferization::ToTensorOp>(
+      rewriter.replaceOp(op, bufferization::ToTensorOp::create(rewriter,
                                  loc, op.getResult().getType(), array,
                                  /*restrict=*/true, /*writable=*/true));
     }
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 80b3d85488495..3f82342001ea5 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -53,7 +53,7 @@ static Value truncToI32(ImplicitLocOpBuilder &b, Value value) {
   assert(llvm::isa<IntegerType>(type) && "expected an integer Value");
   if (type.getIntOrFloatBitWidth() <= 32)
     return value;
-  return b.create<LLVM::TruncOp>(b.getI32Type(), value);
+  return LLVM::TruncOp::create(b, b.getI32Type(), value);
 }
 
 /// Returns the type for the intrinsic given the vectorResultType of the
@@ -113,7 +113,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
   Type f32x1Ty = VectorType::get(1, f32Ty);
 
   auto makeConst = [&](int32_t index) -> Value {
-    return rewriter.create<LLVM::ConstantOp>(loc, IntegerType::get(ctx, 32),
+    return LLVM::ConstantOp::create(rewriter, loc, IntegerType::get(ctx, 32),
                                              rewriter.getI32IntegerAttr(index));
   };
 
@@ -126,7 +126,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
         arrayType.getElementType() == f32x1Ty) {
       for (unsigned i = 0; i < structType.getBody().size(); i++) {
         Value el =
-            rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i);
+            LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i);
         el = rewriter.createOrFold<LLVM::BitcastOp>(
             loc, arrayType.getElementType(), el);
         elements.push_back(el);
@@ -143,23 +143,23 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
       for (unsigned i = 0, e = structType.getBody().size() / 2; i < e; i++) {
         Value vec =
-            rewriter.create<LLVM::PoisonOp>(loc, arrayType.getElementType());
+            LLVM::PoisonOp::create(rewriter, loc, arrayType.getElementType());
         Value x1 =
-            rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i * 2);
-        Value x2 = rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult,
+            LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i * 2);
+        Value x2 = LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult,
                                                          i * 2 + 1);
-        vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec,
+        vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
                                                      x1, makeConst(0));
-        vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec,
+        vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
                                                      x2, makeConst(1));
         elements.push_back(vec);
       }
     }
 
     // Create the final vectorized result.
-    Value result = rewriter.create<LLVM::PoisonOp>(loc, arrayType);
+    Value result = LLVM::PoisonOp::create(rewriter, loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
-      result = rewriter.create<LLVM::InsertValueOp>(loc, result, el.value(),
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, el.value(),
                                                     el.index());
     }
     return result;
@@ -187,7 +187,7 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
   auto arrayTy = cast<LLVM::LLVMArrayType>(operand.getType());
 
   for (unsigned i = 0, e = arrayTy.getNumElements(); i < e; ++i) {
-    Value toUse = b.create<LLVM::ExtractValueOp>(operand, i);
+    Value toUse = LLVM::ExtractValueOp::create(b, operand, i);
 
     // For 4xi8 vectors, the intrinsic expects these to be provided as i32
     // scalar types.
@@ -195,7 +195,7 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
         arrayTy.getElementType() == i4x8Ty ||
         (arrayTy.getElementType() == f32x1Ty &&
          operandPtxType == NVVM::MMATypes::tf32)) {
-      result.push_back(b.create<LLVM::BitcastOp>(i32Ty, toUse));
+      result.push_back(LLVM::BitcastOp::create(b, i32Ty, toUse));
       continue;
     }
 
@@ -208,9 +208,9 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
                          innerArrayTy.getElementType() == f32Ty)) {
       for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
            idx < innerSize; idx++) {
-        result.push_back(b.create<LLVM::ExtractElementOp>(
+        result.push_back(LLVM::ExtractElementOp::create(b,
             toUse,
-            b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(idx))));
+            LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(idx))));
       }
       continue;
     }
@@ -285,7 +285,7 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
-    Value ldMatrixResult = b.create<NVVM::LdMatrixOp>(
+    Value ldMatrixResult = NVVM::LdMatrixOp::create(b,
         ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
@@ -296,13 +296,13 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     // actual vector type (still of width 32b) and repack them into a result
     // struct.
     Type finalResultType = typeConverter->convertType(vectorResultType);
-    Value result = b.create<LLVM::PoisonOp>(finalResultType);
+    Value result = LLVM::PoisonOp::create(b, finalResultType);
     for (int64_t i = 0, e = vectorResultType.getDimSize(0); i < e; i++) {
       Value i32Register =
-          num32BitRegs > 1 ? b.create<LLVM::ExtractValueOp>(ldMatrixResult, i)
+          num32BitRegs > 1 ? LLVM::ExtractValueOp::create(b, ldMatrixResult, i)
                            : ldMatrixResult;
-      Value casted = b.create<LLVM::BitcastOp>(innerVectorType, i32Register);
-      result = b.create<LLVM::InsertValueOp>(result, casted, i);
+      Value casted = LLVM::BitcastOp::create(b, innerVectorType, i32Register);
+      result = LLVM::InsertValueOp::create(b, result, casted, i);
     }
 
     rewriter.replaceOp(op, result);
@@ -375,7 +375,7 @@ struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
     Type desiredRetTy = typeConverter->convertType(op->getResultTypes()[0]);
     Type intrinsicResTy = inferIntrinsicResultType(
         typeConverter->convertType(op->getResultTypes()[0]));
-    Value intrinsicResult = b.create<NVVM::MmaOp>(
+    Value intrinsicResult = NVVM::MmaOp::create(b,
         intrinsicResTy, matA, matB, matC,
         /*shape=*/gemmShape,
         /*b1Op=*/std::nullopt,
@@ -565,7 +565,7 @@ static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
     llvm::append_range(asmVals, args);
   asmVals.push_back(indexData);
 
-  return b.create<LLVM::InlineAsmOp>(
+  return LLVM::InlineAsmOp::create(b,
       /*resultTypes=*/intrinsicResultType,
       /*operands=*/asmVals,
       /*asm_string=*/asmStr,
@@ -631,7 +631,7 @@ struct NVGPUMmaSparseSyncLowering
       return op->emitOpError() << "Expected metadata type to be LLVM "
                                   "VectorType of 2 i16 elements";
     sparseMetadata =
-        b.create<LLVM::BitcastOp>(rewriter.getI32Type(), sparseMetadata);
+        LLVM::BitcastOp::create(b, rewriter.getI32Type(), sparseMetadata);
 
     FailureOr<LLVM::InlineAsmOp> intrinsicResult = emitMmaSparseSyncOpAsm(
         b, *ptxTypeA, *ptxTypeB, *ptxTypeC, *ptxTypeC, overflow, matA, matB,
@@ -682,7 +682,7 @@ struct NVGPUAsyncCopyLowering
     // Intrinsics takes a global pointer so we need an address space cast.
     auto srcPointerGlobalType = LLVM::LLVMPointerType::get(
         op->getContext(), NVVM::NVVMMemorySpace::kGlobalMemorySpace);
-    scrPtr = b.create<LLVM::AddrSpaceCastOp>(srcPointerGlobalType, scrPtr);
+    scrPtr = LLVM::AddrSpaceCastOp::create(b, srcPointerGlobalType, scrPtr);
     int64_t dstElements = adaptor.getDstElements().getZExtValue();
     int64_t sizeInBytes =
         (dstMemrefType.getElementTypeBitWidth() * dstElements) / 8;
@@ -697,13 +697,13 @@ struct NVGPUAsyncCopyLowering
       // The rest of the DstElements in the destination (shared memory) are
       // filled with zeros.
       Value c3I32 =
-          b.create<LLVM::ConstantOp>(b.getI32Type(), b.getI32IntegerAttr(3));
-      Value bitwidth = b.create<LLVM::ConstantOp>(
+          LLVM::ConstantOp::create(b, b.getI32Type(), b.getI32IntegerAttr(3));
+      Value bitwidth = LLVM::ConstantOp::create(b,
           b.getI32Type(),
           b.getI32IntegerAttr(srcMemrefType.getElementTypeBitWidth()));
-      Value srcElementsI32 = b.create<LLVM::TruncOp>(b.getI32Type(), srcBytes);
-      srcBytes = b.create<LLVM::LShrOp>(
-          b.create<LLVM::MulOp>(bitwidth, srcElementsI32), c3I32);
+      Value srcElementsI32 = LLVM::TruncOp::create(b, b.getI32Type(), srcBytes);
+      srcBytes = LLVM::LShrOp::create(b,
+          LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
     }
     // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
     // 16 dst bytes.
@@ -712,13 +712,13 @@ struct NVGPUAsyncCopyLowering
             ? NVVM::LoadCacheModifierKind::CG
             : NVVM::LoadCacheModifierKind::CA;
 
-    b.create<NVVM::CpAsyncOp>(
+    NVVM::CpAsyncOp::create(b,
         dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
         NVVM::LoadCacheModifierKindAttr::get(op->getContext(), cacheModifier),
         srcBytes);
 
     // Drop the result token.
-    Value zero = b.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(b,
         IntegerType::get(op.getContext(), 32), rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
@@ -733,9 +733,9 @@ struct NVGPUAsyncCreateGroupLowering
   LogicalResult
   matchAndRewrite(nvgpu::DeviceAsyncCreateGroupOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    rewriter.create<NVVM::CpAsyncCommitGroupOp>(op.getLoc());
+    NVVM::CpAsyncCommitGroupOp::create(rewriter, op.getLoc());
     // Drop the result token.
-    Value zero = rewriter.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(rewriter,
         op->getLoc(), IntegerType::get(op.getContext(), 32),
         rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
@@ -753,7 +753,7 @@ struct NVGPUAsyncWaitLowering
                   ConversionPatternRewriter &rewriter) const override {
     // If numGroup is not present pick 0 as a conservative correct value.
     int32_t numGroups = adaptor.getNumGroups().value_or(0);
-    rewriter.create<NVVM::CpAsyncWaitGroupOp>(op.getLoc(), numGroups);
+    NVVM::CpAsyncWaitGroupOp::create(rewriter, op.getLoc(), numGroups);
     rewriter.eraseOp(op);
     return success();
   }
@@ -771,7 +771,7 @@ struct NVGPUMBarrierCreateLowering
     SymbolTable symbolTable(moduleOp);
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(&moduleOp.front());
-    auto global = rewriter.create<memref::GlobalOp>(
+    auto global = memref::GlobalOp::create(rewriter,
         funcOp->getLoc(), "__mbarrier",
         /*sym_visibility=*/rewriter.getStringAttr("private"),
         /*type=*/barrierType,
@@ -974,7 +974,7 @@ struct NVGPUMBarrierTryWaitParityLowering
                        adaptor.getMbarId(), rewriter);
     Value ticks = truncToI32(b, adaptor.getTicks());
     Value phase =
-        b.create<LLVM::ZExtOp>(b.getI32Type(), adaptor.getPhaseParity());
+        LLVM::ZExtOp::create(b, b.getI32Type(), adaptor.getPhaseParity());
 
     if (isMbarrierShared(op.getBarriers().getType())) {
       rewriter.replaceOpWithNewOp<NVVM::MBarrierTryWaitParitySharedOp>(
@@ -1063,16 +1063,16 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 
     auto ti64 = b.getIntegerType(64);
     auto makeConst = [&](uint64_t index) -> Value {
-      return b.create<LLVM::ConstantOp>(ti64, b.getI64IntegerAttr(index));
+      return LLVM::ConstantOp::create(b, ti64, b.getI64IntegerAttr(index));
     };
     auto shiftLeft = [&](Value value, unsigned shift) -> Value {
-      return b.create<LLVM::ShlOp>(ti64, value, makeConst(shift));
+      return LLVM::ShlOp::create(b, ti64, value, makeConst(shift));
     };
     auto shiftRight = [&](Value value, unsigned shift) -> Value {
-      return b.create<LLVM::LShrOp>(ti64, value, makeConst(shift));
+      return LLVM::LShrOp::create(b, ti64, value, makeConst(shift));
     };
     auto insertBit = [&](Value desc, Value val, int startBit) {
-      return b.create<LLVM::OrOp>(ti64, desc, shiftLeft(val, startBit));
+      return LLVM::OrOp::create(b, ti64, desc, shiftLeft(val, startBit));
     };
 
     int64_t sizeN = op.getTensorMap().getType().getTensor().getDimSize(0);
@@ -1086,7 +1086,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
     Value baseAddr = getStridedElementPtr(
         rewriter, op->getLoc(), cast<MemRefType>(op.getTensor().getType()),
         adaptor.getTensor(), {});
-    Value basePtr = b.create<LLVM::PtrToIntOp>(ti64, baseAddr);
+    Value basePtr = LLVM::PtrToIntOp::create(b, ti64, baseAddr);
     // Just use 14 bits for base address
     Value basePtr14bit = shiftRight(shiftLeft(basePtr, 46), 50);
 
@@ -1118,7 +1118,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 };
 
 static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
-  return b.create<LLVM::ConstantOp>(b.getIntegerType(64),
+  return LLVM::ConstantOp::create(b, b.getIntegerType(64),
                                     b.getI32IntegerAttr(index));
 }
 
@@ -1182,12 +1182,12 @@ struct NVGPUTmaCreateDescriptorOpLowering
     auto promotedOperands = getTypeConverter()->promoteOperands(
         b.getLoc(), op->getOperands(), adaptor.getOperands(), b);
 
-    Value boxArrayPtr = b.create<LLVM::AllocaOp>(llvmPointerType, llvmInt64Type,
+    Value boxArrayPtr = LLVM::AllocaOp::create(b, llvmPointerType, llvmInt64Type,
                                                  makeI64Const(b, 5));
     for (auto [index, value] : llvm::enumerate(adaptor.getBoxDimensions())) {
-      Value gep = b.create<LLVM::GEPOp>(llvmPointerType, llvmPointerType,
+      Value gep = LLVM::GEPOp::create(b, llvmPointerType, llvmPointerType,
                                         boxArrayPtr, makeI64Const(b, index));
-      b.create<LLVM::StoreOp>(value, gep);
+      LLVM::StoreOp::create(b, value, gep);
     }
 
     nvgpu::TensorMapDescriptorType desc = op.getTensorMap().getType();
@@ -1337,7 +1337,7 @@ struct NVGPUWarpgroupMmaOpLowering
 
     /// Basic function to generate Add
     Value makeAdd(Value lhs, Value rhs) {
-      return b.create<LLVM::AddOp>(lhs.getType(), lhs, rhs);
+      return LLVM::AddOp::create(b, lhs.getType(), lhs, rhs);
     };
 
     /// Moves the descriptor pointer of matrix-A for the next wgmma instruction.
@@ -1430,7 +1430,7 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      return b.create<NVVM::WgmmaMmaAsyncOp>(
+      return NVVM::WgmmaMmaAsyncOp::create(b,
           matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
           itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
           overflow);
@@ -1439,19 +1439,19 @@ struct NVGPUWarpgroupMmaOpLowering
     /// Generates multiple wgmma instructions to complete the given GEMM shape
     Value generateWgmmaGroup() {
       Value wgmmaResult =
-          b.create<LLVM::PoisonOp>(adaptor.getMatrixC().getType());
+          LLVM::PoisonOp::create(b, adaptor.getMatrixC().getType());
 
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = b.create<LLVM::ExtractValueOp>(adaptor.getMatrixC(), i);
+        Value matrixC = LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
             matrixC = generateWgmma(i, j, k, matrixC);
         wgmmaResults.push_back(matrixC);
       }
       for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
-        wgmmaResult = b.create<LLVM::InsertValueOp>(wgmmaResult.getType(),
+        wgmmaResult = LLVM::InsertValueOp::create(b, wgmmaResult.getType(),
                                                     wgmmaResult, matrix, idx);
       }
       return wgmmaResult;
@@ -1486,10 +1486,10 @@ struct NVGPUWarpgroupMmaOpLowering
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
     Value generateWarpgroupMma() {
-      b.create<NVVM::WgmmaFenceAlignedOp>();
+      NVVM::WgmmaFenceAlignedOp::create(b, );
       Value wgmmaResult = generateWgmmaGroup();
-      b.create<NVVM::WgmmaGroupSyncAlignedOp>();
-      b.create<NVVM::WgmmaWaitGroupSyncOp>(op.getWaitGroup());
+      NVVM::WgmmaGroupSyncAlignedOp::create(b, );
+      NVVM::WgmmaWaitGroupSyncOp::create(b, op.getWaitGroup());
       return wgmmaResult;
     }
   };
@@ -1557,7 +1557,7 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     Type i32 = b.getI32Type();
 
     auto makeConst = [&](int32_t index) -> Value {
-      return b.create<LLVM::ConstantOp>(i32, b.getI32IntegerAttr(index));
+      return LLVM::ConstantOp::create(b, i32, b.getI32IntegerAttr(index));
     };
     Value c1 = makeConst(1);
     Value c2 = makeConst(2);
@@ -1567,29 +1567,29 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     Value warpSize = makeConst(kWarpSize);
 
     auto makeMul = [&](Value lhs, Value rhs) -> Value {
-      return b.create<LLVM::MulOp>(lhs.getType(), lhs, rhs);
+      return LLVM::MulOp::create(b, lhs.getType(), lhs, rhs);
     };
     auto makeAdd = [&](Value lhs, Value rhs) -> Value {
-      return b.create<LLVM::AddOp>(lhs.getType(), lhs, rhs);
+      return LLVM::AddOp::create(b, lhs.getType(), lhs, rhs);
     };
 
     auto makeExtractAndStore = [&](int i, Value wgmmaResult, Value x, Value y,
                                    TypedValue<::mlir::MemRefType> memref) {
       Type it = b.getIndexType();
-      Value idx = b.create<arith::IndexCastOp>(it, x);
-      Value idy0 = b.create<arith::IndexCastOp>(it, y);
-      Value idy1 = b.create<arith::IndexCastOp>(it, makeAdd(y, c1));
-      Value d0 = b.create<LLVM::ExtractValueOp>(wgmmaResult, i);
-      Value d1 = b.create<LLVM::ExtractValueOp>(wgmmaResult, i + 1);
-      b.create<memref::StoreOp>(d0, memref, ValueRange{idx, idy0});
-      b.create<memref::StoreOp>(d1, memref, ValueRange{idx, idy1});
+      Value idx = arith::IndexCastOp::create(b, it, x);
+      Value idy0 = arith::IndexCastOp::create(b, it, y);
+      Value idy1 = arith::IndexCastOp::create(b, it, makeAdd(y, c1));
+      Value d0 = LLVM::ExtractValueOp::create(b, wgmmaResult, i);
+      Value d1 = LLVM::ExtractValueOp::create(b, wgmmaResult, i + 1);
+      memref::StoreOp::create(b, d0, memref, ValueRange{idx, idy0});
+      memref::StoreOp::create(b, d1, memref, ValueRange{idx, idy1});
     };
 
-    Value tidx = b.create<NVVM::ThreadIdXOp>(i32);
-    Value laneId = b.create<LLVM::URemOp>(i32, tidx, warpSize);
-    Value warpId = b.create<LLVM::UDivOp>(i32, tidx, warpSize);
-    Value lane4Id = b.create<LLVM::UDivOp>(i32, laneId, c4);
-    Value lane4modId = b.create<LLVM::URemOp>(i32, laneId, c4);
+    Value tidx = NVVM::ThreadIdXOp::create(b, i32);
+    Value laneId = LLVM::URemOp::create(b, i32, tidx, warpSize);
+    Value warpId = LLVM::UDivOp::create(b, i32, tidx, warpSize);
+    Value lane4Id = LLVM::UDivOp::create(b, i32, laneId, c4);
+    Value lane4modId = LLVM::URemOp::create(b, i32, laneId, c4);
 
     Value tj = makeMul(lane4modId, c2);
     Value ti = makeAdd(lane4Id, makeMul(warpId, c16));
@@ -1626,7 +1626,7 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     auto stype = cast<LLVM::LLVMStructType>(matriDValue.getType());
     for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(matrixD);
-      Value innerStructValue = b.create<LLVM::ExtractValueOp>(matriDValue, idx);
+      Value innerStructValue = LLVM::ExtractValueOp::create(b, matriDValue, idx);
       storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
@@ -1648,22 +1648,22 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
     Type elemType = cast<LLVM::LLVMStructType>(packStructType.getBody().front())
                         .getBody()
                         .front();
-    Value zero = b.create<LLVM::ConstantOp>(elemType, b.getZeroAttr(elemType));
-    Value packStruct = b.create<LLVM::PoisonOp>(packStructType);
+    Value zero = LLVM::ConstantOp::create(b, elemType, b.getZeroAttr(elemType));
+    Value packStruct = LLVM::PoisonOp::create(b, packStructType);
     SmallVector<Value> innerStructs;
     // Unpack the structs and set all values to zero
     for (auto [idx, s] : llvm::enumerate(packStructType.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(s);
-      Value structValue = b.create<LLVM::ExtractValueOp>(packStruct, idx);
+      Value structValue = LLVM::ExtractValueOp::create(b, packStruct, idx);
       for (unsigned i = 0; i < structType.getBody().size(); ++i) {
-        structValue = b.create<LLVM::InsertValueOp>(
+        structValue = LLVM::InsertValueOp::create(b,
             structType, structValue, zero, ArrayRef<int64_t>({i}));
       }
       innerStructs.push_back(structValue);
     }
     // Pack the inner structs into a single struct
     for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
-      packStruct = b.create<LLVM::InsertValueOp>(packStruct.getType(),
+      packStruct = LLVM::InsertValueOp::create(b, packStruct.getType(),
                                                  packStruct, matrix, idx);
     }
     rewriter.replaceOp(op, packStruct);
@@ -1681,7 +1681,7 @@ struct NVGPUTmaFenceOpLowering
     ImplicitLocOpBuilder b(op->getLoc(), rewriter);
     auto i32Ty = b.getI32Type();
     Value tensormapSize =
-        b.create<LLVM::ConstantOp>(i32Ty, rewriter.getI32IntegerAttr(128));
+        LLVM::ConstantOp::create(b, i32Ty, rewriter.getI32IntegerAttr(128));
 
     auto memscope =
         NVVM::MemScopeKindAttr::get(ctx, ::mlir::NVVM::MemScopeKind::SYS);
@@ -1716,13 +1716,13 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
     VectorType inTy = op.getIn().getType();
     // apply rcp.approx.ftz.f on each element in vector.
     auto convert1DVec = [&](Type llvm1DVectorTy, Value inVec) {
-      Value ret1DVec = b.create<LLVM::PoisonOp>(llvm1DVectorTy);
+      Value ret1DVec = LLVM::PoisonOp::create(b, llvm1DVectorTy);
       int numElems = llvm::cast<VectorType>(llvm1DVectorTy).getNumElements();
       for (int i = 0; i < numElems; i++) {
-        Value idx = b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(i));
-        Value elem = b.create<LLVM::ExtractElementOp>(inVec, idx);
-        Value dst = b.create<NVVM::RcpApproxFtzF32Op>(f32Ty, elem);
-        ret1DVec = b.create<LLVM::InsertElementOp>(ret1DVec, dst, idx);
+        Value idx = LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(i));
+        Value elem = LLVM::ExtractElementOp::create(b, inVec, idx);
+        Value dst = NVVM::RcpApproxFtzF32Op::create(b, f32Ty, elem);
+        ret1DVec = LLVM::InsertElementOp::create(b, ret1DVec, dst, idx);
       }
       return ret1DVec;
     };
diff --git a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
index 479725aae8afd..a6125fc07dcd5 100644
--- a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
+++ b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
@@ -39,7 +39,7 @@ class ExpandIfCondition : public OpRewritePattern<OpTy> {
 
     IntegerAttr constAttr;
     if (!matchPattern(op.getIfCond(), m_Constant(&constAttr))) {
-      auto ifOp = rewriter.create<scf::IfOp>(op.getLoc(), TypeRange(),
+      auto ifOp = scf::IfOp::create(rewriter, op.getLoc(), TypeRange(),
                                              op.getIfCond(), false);
       rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });
       auto thenBodyBuilder = ifOp.getThenBodyBuilder(rewriter.getListener());
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 77a2708653576..836a193b7fb4a 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -85,7 +85,7 @@ struct OpenMPOpConversion : public ConvertOpToLLVMPattern<T> {
     }
 
     // Create new operation.
-    auto newOp = rewriter.create<T>(op.getLoc(), resTypes, convertedOperands,
+    auto newOp = T::create(rewriter, op.getLoc(), resTypes, convertedOperands,
                                     convertedAttrs);
 
     // Translate regions.
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 7d20109b3db59..75cd35c0d7876 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -196,7 +196,7 @@ Block *PatternLowering::generateMatcher(MatcherNode &node, Region &region,
   // finalize.
   if (isa<ExitNode>(node)) {
     builder.setInsertionPointToEnd(block);
-    builder.create<pdl_interp::FinalizeOp>(matcherFunc.getLoc());
+    pdl_interp::FinalizeOp::create(builder, matcherFunc.getLoc());
     return block;
   }
 
@@ -272,7 +272,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     auto *operationPos = cast<OperationPosition>(pos);
     if (operationPos->isOperandDefiningOp())
       // Standard (downward) traversal which directly follows the defining op.
-      value = builder.create<pdl_interp::GetDefiningOpOp>(
+      value = pdl_interp::GetDefiningOpOp::create(builder,
           loc, builder.getType<pdl::OperationType>(), parentVal);
     else
       // A passthrough operation position.
@@ -287,23 +287,23 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     // requested to use a representative value (e.g., upward traversal).
     if (isa<pdl::RangeType>(parentVal.getType()) &&
         usersPos->useRepresentative())
-      value = builder.create<pdl_interp::ExtractOp>(loc, parentVal, 0);
+      value = pdl_interp::ExtractOp::create(builder, loc, parentVal, 0);
     else
       value = parentVal;
 
     // The second operation retrieves the users.
-    value = builder.create<pdl_interp::GetUsersOp>(loc, value);
+    value = pdl_interp::GetUsersOp::create(builder, loc, value);
     break;
   }
   case Predicates::ForEachPos: {
     assert(!failureBlockStack.empty() && "expected valid failure block");
-    auto foreach = builder.create<pdl_interp::ForEachOp>(
+    auto foreach = pdl_interp::ForEachOp::create(builder,
         loc, parentVal, failureBlockStack.back(), /*initLoop=*/true);
     value = foreach.getLoopVariable();
 
     // Create the continuation block.
     Block *continueBlock = builder.createBlock(&foreach.getRegion());
-    builder.create<pdl_interp::ContinueOp>(loc);
+    pdl_interp::ContinueOp::create(builder, loc);
     failureBlockStack.push_back(continueBlock);
 
     currentBlock = &foreach.getRegion().front();
@@ -311,7 +311,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::OperandPos: {
     auto *operandPos = cast<OperandPosition>(pos);
-    value = builder.create<pdl_interp::GetOperandOp>(
+    value = pdl_interp::GetOperandOp::create(builder,
         loc, builder.getType<pdl::ValueType>(), parentVal,
         operandPos->getOperandNumber());
     break;
@@ -319,28 +319,28 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::OperandGroupPos: {
     auto *operandPos = cast<OperandGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = builder.create<pdl_interp::GetOperandsOp>(
+    value = pdl_interp::GetOperandsOp::create(builder,
         loc, operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, operandPos->getOperandGroupNumber());
     break;
   }
   case Predicates::AttributePos: {
     auto *attrPos = cast<AttributePosition>(pos);
-    value = builder.create<pdl_interp::GetAttributeOp>(
+    value = pdl_interp::GetAttributeOp::create(builder,
         loc, builder.getType<pdl::AttributeType>(), parentVal,
         attrPos->getName().strref());
     break;
   }
   case Predicates::TypePos: {
     if (isa<pdl::AttributeType>(parentVal.getType()))
-      value = builder.create<pdl_interp::GetAttributeTypeOp>(loc, parentVal);
+      value = pdl_interp::GetAttributeTypeOp::create(builder, loc, parentVal);
     else
-      value = builder.create<pdl_interp::GetValueTypeOp>(loc, parentVal);
+      value = pdl_interp::GetValueTypeOp::create(builder, loc, parentVal);
     break;
   }
   case Predicates::ResultPos: {
     auto *resPos = cast<ResultPosition>(pos);
-    value = builder.create<pdl_interp::GetResultOp>(
+    value = pdl_interp::GetResultOp::create(builder,
         loc, builder.getType<pdl::ValueType>(), parentVal,
         resPos->getResultNumber());
     break;
@@ -348,7 +348,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::ResultGroupPos: {
     auto *resPos = cast<ResultGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = builder.create<pdl_interp::GetResultsOp>(
+    value = pdl_interp::GetResultsOp::create(builder,
         loc, resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, resPos->getResultGroupNumber());
     break;
@@ -356,16 +356,16 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::AttributeLiteralPos: {
     auto *attrPos = cast<AttributeLiteralPosition>(pos);
     value =
-        builder.create<pdl_interp::CreateAttributeOp>(loc, attrPos->getValue());
+        pdl_interp::CreateAttributeOp::create(builder, loc, attrPos->getValue());
     break;
   }
   case Predicates::TypeLiteralPos: {
     auto *typePos = cast<TypeLiteralPosition>(pos);
     Attribute rawTypeAttr = typePos->getValue();
     if (TypeAttr typeAttr = dyn_cast<TypeAttr>(rawTypeAttr))
-      value = builder.create<pdl_interp::CreateTypeOp>(loc, typeAttr);
+      value = pdl_interp::CreateTypeOp::create(builder, loc, typeAttr);
     else
-      value = builder.create<pdl_interp::CreateTypesOp>(
+      value = pdl_interp::CreateTypesOp::create(builder,
           loc, cast<ArrayAttr>(rawTypeAttr));
     break;
   }
@@ -413,54 +413,54 @@ void PatternLowering::generate(BoolNode *boolNode, Block *&currentBlock,
   Predicates::Kind kind = question->getKind();
   switch (kind) {
   case Predicates::IsNotNullQuestion:
-    builder.create<pdl_interp::IsNotNullOp>(loc, val, success, failure);
+    pdl_interp::IsNotNullOp::create(builder, loc, val, success, failure);
     break;
   case Predicates::OperationNameQuestion: {
     auto *opNameAnswer = cast<OperationNameAnswer>(answer);
-    builder.create<pdl_interp::CheckOperationNameOp>(
+    pdl_interp::CheckOperationNameOp::create(builder,
         loc, val, opNameAnswer->getValue().getStringRef(), success, failure);
     break;
   }
   case Predicates::TypeQuestion: {
     auto *ans = cast<TypeAnswer>(answer);
     if (isa<pdl::RangeType>(val.getType()))
-      builder.create<pdl_interp::CheckTypesOp>(
+      pdl_interp::CheckTypesOp::create(builder,
           loc, val, llvm::cast<ArrayAttr>(ans->getValue()), success, failure);
     else
-      builder.create<pdl_interp::CheckTypeOp>(
+      pdl_interp::CheckTypeOp::create(builder,
           loc, val, llvm::cast<TypeAttr>(ans->getValue()), success, failure);
     break;
   }
   case Predicates::AttributeQuestion: {
     auto *ans = cast<AttributeAnswer>(answer);
-    builder.create<pdl_interp::CheckAttributeOp>(loc, val, ans->getValue(),
+    pdl_interp::CheckAttributeOp::create(builder, loc, val, ans->getValue(),
                                                  success, failure);
     break;
   }
   case Predicates::OperandCountAtLeastQuestion:
   case Predicates::OperandCountQuestion:
-    builder.create<pdl_interp::CheckOperandCountOp>(
+    pdl_interp::CheckOperandCountOp::create(builder,
         loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::OperandCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::ResultCountAtLeastQuestion:
   case Predicates::ResultCountQuestion:
-    builder.create<pdl_interp::CheckResultCountOp>(
+    pdl_interp::CheckResultCountOp::create(builder,
         loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::ResultCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::EqualToQuestion: {
     bool trueAnswer = isa<TrueAnswer>(answer);
-    builder.create<pdl_interp::AreEqualOp>(loc, val, args.front(),
+    pdl_interp::AreEqualOp::create(builder, loc, val, args.front(),
                                            trueAnswer ? success : failure,
                                            trueAnswer ? failure : success);
     break;
   }
   case Predicates::ConstraintQuestion: {
     auto *cstQuestion = cast<ConstraintQuestion>(question);
-    auto applyConstraintOp = builder.create<pdl_interp::ApplyConstraintOp>(
+    auto applyConstraintOp = pdl_interp::ApplyConstraintOp::create(builder,
         loc, cstQuestion->getResultTypes(), cstQuestion->getName(), args,
         cstQuestion->getIsNegated(), success, failure);
 
@@ -487,7 +487,7 @@ static void createSwitchOp(Value val, Block *defaultDest, OpBuilder &builder,
     blocks.push_back(it.second);
     values.push_back(cast<PredT>(it.first)->getValue());
   }
-  builder.create<OpT>(val.getLoc(), val, values, defaultDest, blocks);
+  OpT::create(builder, val.getLoc(), val, values, defaultDest, blocks);
 }
 
 void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
@@ -536,11 +536,11 @@ void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
       unsigned ans = cast<UnsignedAnswer>(child.first)->getValue();
       switch (kind) {
       case Predicates::OperandCountAtLeastQuestion:
-        builder.create<pdl_interp::CheckOperandCountOp>(
+        pdl_interp::CheckOperandCountOp::create(builder,
             loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
         break;
       case Predicates::ResultCountAtLeastQuestion:
-        builder.create<pdl_interp::CheckResultCountOp>(
+        pdl_interp::CheckResultCountOp::create(builder,
             loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
         break;
       default:
@@ -619,7 +619,7 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
       rootKindAttr = builder.getStringAttr(*rootKind);
 
   builder.setInsertionPointToEnd(currentBlock);
-  auto matchOp = builder.create<pdl_interp::RecordMatchOp>(
+  auto matchOp = pdl_interp::RecordMatchOp::create(builder,
       pattern.getLoc(), mappedMatchValues, locOps.getArrayRef(),
       rewriterFuncRef, rootKindAttr, generatedOpsAttr, pattern.getBenefitAttr(),
       failureBlockStack.back());
@@ -632,7 +632,7 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
 SymbolRefAttr PatternLowering::generateRewriter(
     pdl::PatternOp pattern, SmallVectorImpl<Position *> &usedMatchValues) {
   builder.setInsertionPointToEnd(rewriterModule.getBody());
-  auto rewriterFunc = builder.create<pdl_interp::FuncOp>(
+  auto rewriterFunc = pdl_interp::FuncOp::create(builder,
       pattern.getLoc(), "pdl_generated_rewriter",
       builder.getFunctionType({}, {}));
   rewriterSymbolTable.insert(rewriterFunc);
@@ -651,17 +651,17 @@ SymbolRefAttr PatternLowering::generateRewriter(
     Operation *oldOp = oldValue.getDefiningOp();
     if (pdl::AttributeOp attrOp = dyn_cast<pdl::AttributeOp>(oldOp)) {
       if (Attribute value = attrOp.getValueAttr()) {
-        return newValue = builder.create<pdl_interp::CreateAttributeOp>(
+        return newValue = pdl_interp::CreateAttributeOp::create(builder,
                    attrOp.getLoc(), value);
       }
     } else if (pdl::TypeOp typeOp = dyn_cast<pdl::TypeOp>(oldOp)) {
       if (TypeAttr type = typeOp.getConstantTypeAttr()) {
-        return newValue = builder.create<pdl_interp::CreateTypeOp>(
+        return newValue = pdl_interp::CreateTypeOp::create(builder,
                    typeOp.getLoc(), type);
       }
     } else if (pdl::TypesOp typeOp = dyn_cast<pdl::TypesOp>(oldOp)) {
       if (ArrayAttr type = typeOp.getConstantTypesAttr()) {
-        return newValue = builder.create<pdl_interp::CreateTypesOp>(
+        return newValue = pdl_interp::CreateTypesOp::create(builder,
                    typeOp.getLoc(), typeOp.getType(), type);
       }
     }
@@ -684,7 +684,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
     auto mappedArgs =
         llvm::map_range(rewriter.getExternalArgs(), mapRewriteValue);
     args.append(mappedArgs.begin(), mappedArgs.end());
-    builder.create<pdl_interp::ApplyRewriteOp>(
+    pdl_interp::ApplyRewriteOp::create(builder,
         rewriter.getLoc(), /*resultTypes=*/TypeRange(), rewriteName, args);
   } else {
     // Otherwise this is a dag rewriter defined using PDL operations.
@@ -703,7 +703,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
       llvm::to_vector<8>(rewriterFunc.front().getArgumentTypes()),
       /*results=*/{}));
 
-  builder.create<pdl_interp::FinalizeOp>(rewriter.getLoc());
+  pdl_interp::FinalizeOp::create(builder, rewriter.getLoc());
   return SymbolRefAttr::get(
       builder.getContext(),
       pdl_interp::PDLInterpDialect::getRewriterModuleName(),
@@ -716,7 +716,7 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 2> arguments;
   for (Value argument : rewriteOp.getArgs())
     arguments.push_back(mapRewriteValue(argument));
-  auto interpOp = builder.create<pdl_interp::ApplyRewriteOp>(
+  auto interpOp = pdl_interp::ApplyRewriteOp::create(builder,
       rewriteOp.getLoc(), rewriteOp.getResultTypes(), rewriteOp.getNameAttr(),
       arguments);
   for (auto it : llvm::zip(rewriteOp.getResults(), interpOp.getResults()))
@@ -726,7 +726,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::AttributeOp attrOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  Value newAttr = builder.create<pdl_interp::CreateAttributeOp>(
+  Value newAttr = pdl_interp::CreateAttributeOp::create(builder,
       attrOp.getLoc(), attrOp.getValueAttr());
   rewriteValues[attrOp] = newAttr;
 }
@@ -734,7 +734,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::EraseOp eraseOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  builder.create<pdl_interp::EraseOp>(eraseOp.getLoc(),
+  pdl_interp::EraseOp::create(builder, eraseOp.getLoc(),
                                       mapRewriteValue(eraseOp.getOpValue()));
 }
 
@@ -756,7 +756,7 @@ void PatternLowering::generateRewriter(
 
   // Create the new operation.
   Location loc = operationOp.getLoc();
-  Value createdOp = builder.create<pdl_interp::CreateOperationOp>(
+  Value createdOp = pdl_interp::CreateOperationOp::create(builder,
       loc, *operationOp.getOpName(), types, hasInferredResultTypes, operands,
       attributes, operationOp.getAttributeValueNames());
   rewriteValues[operationOp.getOp()] = createdOp;
@@ -768,8 +768,8 @@ void PatternLowering::generateRewriter(
   if (resultTys.size() == 1 && isa<pdl::RangeType>(resultTys[0].getType())) {
     Value &type = rewriteValues[resultTys[0]];
     if (!type) {
-      auto results = builder.create<pdl_interp::GetResultsOp>(loc, createdOp);
-      type = builder.create<pdl_interp::GetValueTypeOp>(loc, results);
+      auto results = pdl_interp::GetResultsOp::create(builder, loc, createdOp);
+      type = pdl_interp::GetValueTypeOp::create(builder, loc, results);
     }
     return;
   }
@@ -789,12 +789,12 @@ void PatternLowering::generateRewriter(
     // groups because the exact index of the result is not statically known.
     Value resultVal;
     if (seenVariableLength)
-      resultVal = builder.create<pdl_interp::GetResultsOp>(
+      resultVal = pdl_interp::GetResultsOp::create(builder,
           loc, isVariadic ? valueRangeTy : valueTy, createdOp, it.index());
     else
-      resultVal = builder.create<pdl_interp::GetResultOp>(
+      resultVal = pdl_interp::GetResultOp::create(builder,
           loc, valueTy, createdOp, it.index());
-    type = builder.create<pdl_interp::GetValueTypeOp>(loc, resultVal);
+    type = pdl_interp::GetValueTypeOp::create(builder, loc, resultVal);
   }
 }
 
@@ -804,7 +804,7 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 4> replOperands;
   for (Value operand : rangeOp.getArguments())
     replOperands.push_back(mapRewriteValue(operand));
-  rewriteValues[rangeOp] = builder.create<pdl_interp::CreateRangeOp>(
+  rewriteValues[rangeOp] = pdl_interp::CreateRangeOp::create(builder,
       rangeOp.getLoc(), rangeOp.getType(), replOperands);
 }
 
@@ -820,7 +820,7 @@ void PatternLowering::generateRewriter(
     // Don't use replace if we know the replaced operation has no results.
     auto opOp = replaceOp.getOpValue().getDefiningOp<pdl::OperationOp>();
     if (!opOp || !opOp.getTypeValues().empty()) {
-      replOperands.push_back(builder.create<pdl_interp::GetResultsOp>(
+      replOperands.push_back(pdl_interp::GetResultsOp::create(builder,
           replOp.getLoc(), mapRewriteValue(replOp)));
     }
   } else {
@@ -830,12 +830,12 @@ void PatternLowering::generateRewriter(
 
   // If there are no replacement values, just create an erase instead.
   if (replOperands.empty()) {
-    builder.create<pdl_interp::EraseOp>(
+    pdl_interp::EraseOp::create(builder,
         replaceOp.getLoc(), mapRewriteValue(replaceOp.getOpValue()));
     return;
   }
 
-  builder.create<pdl_interp::ReplaceOp>(replaceOp.getLoc(),
+  pdl_interp::ReplaceOp::create(builder, replaceOp.getLoc(),
                                         mapRewriteValue(replaceOp.getOpValue()),
                                         replOperands);
 }
@@ -843,7 +843,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::ResultOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultOp>(
+  rewriteValues[resultOp] = pdl_interp::GetResultOp::create(builder,
       resultOp.getLoc(), builder.getType<pdl::ValueType>(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
@@ -851,7 +851,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::ResultsOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultsOp>(
+  rewriteValues[resultOp] = pdl_interp::GetResultsOp::create(builder,
       resultOp.getLoc(), resultOp.getType(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
@@ -863,7 +863,7 @@ void PatternLowering::generateRewriter(
   // type.
   if (TypeAttr typeAttr = typeOp.getConstantTypeAttr()) {
     rewriteValues[typeOp] =
-        builder.create<pdl_interp::CreateTypeOp>(typeOp.getLoc(), typeAttr);
+        pdl_interp::CreateTypeOp::create(builder, typeOp.getLoc(), typeAttr);
   }
 }
 
@@ -873,7 +873,7 @@ void PatternLowering::generateRewriter(
   // If the type isn't constant, the users (e.g. OperationOp) will resolve this
   // type.
   if (ArrayAttr typeAttr = typeOp.getConstantTypesAttr()) {
-    rewriteValues[typeOp] = builder.create<pdl_interp::CreateTypesOp>(
+    rewriteValues[typeOp] = pdl_interp::CreateTypesOp::create(builder,
         typeOp.getLoc(), typeOp.getType(), typeAttr);
   }
 }
@@ -939,9 +939,9 @@ void PatternLowering::generateOperationResultTypeRewriter(
         !replacedOp->isBeforeInBlock(op))
       continue;
 
-    Value replacedOpResults = builder.create<pdl_interp::GetResultsOp>(
+    Value replacedOpResults = pdl_interp::GetResultsOp::create(builder,
         replacedOp->getLoc(), mapRewriteValue(replOpVal));
-    types.push_back(builder.create<pdl_interp::GetValueTypeOp>(
+    types.push_back(pdl_interp::GetValueTypeOp::create(builder,
         replacedOp->getLoc(), replacedOpResults));
     return;
   }
@@ -985,7 +985,7 @@ void PDLToPDLInterpPass::runOnOperation() {
   // Create the main matcher function This function contains all of the match
   // related functionality from patterns in the module.
   OpBuilder builder = OpBuilder::atBlockBegin(module.getBody());
-  auto matcherFunc = builder.create<pdl_interp::FuncOp>(
+  auto matcherFunc = pdl_interp::FuncOp::create(builder,
       module.getLoc(), pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
       builder.getFunctionType(builder.getType<pdl::OperationType>(),
                               /*results=*/{}),
@@ -993,7 +993,7 @@ void PDLToPDLInterpPass::runOnOperation() {
 
   // Create a nested module to hold the functions invoked for rewriting the IR
   // after a successful match.
-  ModuleOp rewriterModule = builder.create<ModuleOp>(
+  ModuleOp rewriterModule = ModuleOp::create(builder,
       module.getLoc(), pdl_interp::PDLInterpDialect::getRewriterModuleName());
 
   // Generate the code for the patterns within the module.
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index ac6dcf2513e99..ffef20ce67818 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -340,14 +340,14 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   Operation *terminator = lastBodyBlock->getTerminator();
   rewriter.setInsertionPointToEnd(lastBodyBlock);
   auto step = forOp.getStep();
-  auto stepped = rewriter.create<arith::AddIOp>(loc, iv, step).getResult();
+  auto stepped = arith::AddIOp::create(rewriter, loc, iv, step).getResult();
   if (!stepped)
     return failure();
 
   SmallVector<Value, 8> loopCarried;
   loopCarried.push_back(stepped);
   loopCarried.append(terminator->operand_begin(), terminator->operand_end());
-  rewriter.create<cf::BranchOp>(loc, conditionBlock, loopCarried);
+  cf::BranchOp::create(rewriter, loc, conditionBlock, loopCarried);
   rewriter.eraseOp(terminator);
 
   // Compute loop bounds before branching to the condition.
@@ -362,14 +362,14 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   SmallVector<Value, 8> destOperands;
   destOperands.push_back(lowerBound);
   llvm::append_range(destOperands, forOp.getInitArgs());
-  rewriter.create<cf::BranchOp>(loc, conditionBlock, destOperands);
+  cf::BranchOp::create(rewriter, loc, conditionBlock, destOperands);
 
   // With the body block done, we can fill in the condition block.
   rewriter.setInsertionPointToEnd(conditionBlock);
-  auto comparison = rewriter.create<arith::CmpIOp>(
+  auto comparison = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::slt, iv, upperBound);
 
-  auto condBranchOp = rewriter.create<cf::CondBranchOp>(
+  auto condBranchOp = cf::CondBranchOp::create(rewriter,
       loc, comparison, firstBodyBlock, ArrayRef<Value>(), endBlock,
       ArrayRef<Value>());
 
@@ -404,7 +404,7 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
     continueBlock =
         rewriter.createBlock(remainingOpsBlock, ifOp.getResultTypes(),
                              SmallVector<Location>(ifOp.getNumResults(), loc));
-    rewriter.create<cf::BranchOp>(loc, remainingOpsBlock);
+    cf::BranchOp::create(rewriter, loc, remainingOpsBlock);
   }
 
   // Move blocks from the "then" region to the region containing 'scf.if',
@@ -414,7 +414,7 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
   Operation *thenTerminator = thenRegion.back().getTerminator();
   ValueRange thenTerminatorOperands = thenTerminator->getOperands();
   rewriter.setInsertionPointToEnd(&thenRegion.back());
-  rewriter.create<cf::BranchOp>(loc, continueBlock, thenTerminatorOperands);
+  cf::BranchOp::create(rewriter, loc, continueBlock, thenTerminatorOperands);
   rewriter.eraseOp(thenTerminator);
   rewriter.inlineRegionBefore(thenRegion, continueBlock);
 
@@ -428,13 +428,13 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
     Operation *elseTerminator = elseRegion.back().getTerminator();
     ValueRange elseTerminatorOperands = elseTerminator->getOperands();
     rewriter.setInsertionPointToEnd(&elseRegion.back());
-    rewriter.create<cf::BranchOp>(loc, continueBlock, elseTerminatorOperands);
+    cf::BranchOp::create(rewriter, loc, continueBlock, elseTerminatorOperands);
     rewriter.eraseOp(elseTerminator);
     rewriter.inlineRegionBefore(elseRegion, continueBlock);
   }
 
   rewriter.setInsertionPointToEnd(condBlock);
-  rewriter.create<cf::CondBranchOp>(loc, ifOp.getCondition(), thenBlock,
+  cf::CondBranchOp::create(rewriter, loc, ifOp.getCondition(), thenBlock,
                                     /*trueArgs=*/ArrayRef<Value>(), elseBlock,
                                     /*falseArgs=*/ArrayRef<Value>());
 
@@ -454,13 +454,13 @@ ExecuteRegionLowering::matchAndRewrite(ExecuteRegionOp op,
 
   auto &region = op.getRegion();
   rewriter.setInsertionPointToEnd(condBlock);
-  rewriter.create<cf::BranchOp>(loc, &region.front());
+  cf::BranchOp::create(rewriter, loc, &region.front());
 
   for (Block &block : region) {
     if (auto terminator = dyn_cast<scf::YieldOp>(block.getTerminator())) {
       ValueRange terminatorOperands = terminator->getOperands();
       rewriter.setInsertionPointToEnd(&block);
-      rewriter.create<cf::BranchOp>(loc, remainingOpsBlock, terminatorOperands);
+      cf::BranchOp::create(rewriter, loc, remainingOpsBlock, terminatorOperands);
       rewriter.eraseOp(terminator);
     }
   }
@@ -498,7 +498,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
   for (auto [iv, lower, upper, step] :
        llvm::zip(parallelOp.getInductionVars(), parallelOp.getLowerBound(),
                  parallelOp.getUpperBound(), parallelOp.getStep())) {
-    ForOp forOp = rewriter.create<ForOp>(loc, lower, upper, step, iterArgs);
+    ForOp forOp = ForOp::create(rewriter, loc, lower, upper, step, iterArgs);
     ivs.push_back(forOp.getInductionVar());
     auto iterRange = forOp.getRegionIterArgs();
     iterArgs.assign(iterRange.begin(), iterRange.end());
@@ -512,7 +512,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
       // A loop is constructed with an empty "yield" terminator if there are
       // no results.
       rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
-      rewriter.create<scf::YieldOp>(loc, forOp.getResults());
+      scf::YieldOp::create(rewriter, loc, forOp.getResults());
     }
 
     rewriter.setInsertionPointToStart(forOp.getBody());
@@ -544,7 +544,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
   // has been already created in loop construction).
   if (!yieldOperands.empty()) {
     rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
-    rewriter.create<scf::YieldOp>(loc, yieldOperands);
+    scf::YieldOp::create(rewriter, loc, yieldOperands);
   }
 
   rewriter.replaceOp(parallelOp, loopResults);
@@ -570,7 +570,7 @@ LogicalResult WhileLowering::matchAndRewrite(WhileOp whileOp,
 
   // Branch to the "before" region.
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(loc, before, whileOp.getInits());
+  cf::BranchOp::create(rewriter, loc, before, whileOp.getInits());
 
   // Replace terminators with branches. Assuming bodies are SESE, which holds
   // given only the patterns from this file, we only need to look at the last
@@ -620,7 +620,7 @@ DoWhileLowering::matchAndRewrite(WhileOp whileOp,
 
   // Branch to the "before" region.
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(whileOp.getLoc(), before, whileOp.getInits());
+  cf::BranchOp::create(rewriter, whileOp.getLoc(), before, whileOp.getInits());
 
   // Loop around the "before" region based on condition.
   rewriter.setInsertionPointToEnd(before);
@@ -688,10 +688,10 @@ IndexSwitchLowering::matchAndRewrite(IndexSwitchOp op,
   SmallVector<ValueRange> caseOperands(caseSuccessors.size(), {});
 
   // Cast switch index to integer case value.
-  Value caseValue = rewriter.create<arith::IndexCastOp>(
+  Value caseValue = arith::IndexCastOp::create(rewriter,
       op.getLoc(), rewriter.getI32Type(), op.getArg());
 
-  rewriter.create<cf::SwitchOp>(
+  cf::SwitchOp::create(rewriter,
       op.getLoc(), caseValue, *defaultBlock, ValueRange(),
       rewriter.getDenseI32ArrayAttr(caseValues), caseSuccessors, caseOperands);
   rewriter.replaceOp(op, continueBlock->getArguments());
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index dcb48529a74e6..37134fe9d992a 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -91,7 +91,7 @@ createVariablesForResults(T op, const TypeConverter *typeConverter,
     Type varType = emitc::LValueType::get(resultType);
     emitc::OpaqueAttr noInit = emitc::OpaqueAttr::get(context, "");
     emitc::VariableOp var =
-        rewriter.create<emitc::VariableOp>(loc, varType, noInit);
+        emitc::VariableOp::create(rewriter, loc, varType, noInit);
     resultVariables.push_back(var);
   }
 
@@ -103,14 +103,14 @@ createVariablesForResults(T op, const TypeConverter *typeConverter,
 static void assignValues(ValueRange values, ValueRange variables,
                          ConversionPatternRewriter &rewriter, Location loc) {
   for (auto [value, var] : llvm::zip(values, variables))
-    rewriter.create<emitc::AssignOp>(loc, var, value);
+    emitc::AssignOp::create(rewriter, loc, var, value);
 }
 
 SmallVector<Value> loadValues(const SmallVector<Value> &variables,
                               PatternRewriter &rewriter, Location loc) {
   return llvm::map_to_vector<>(variables, [&](Value var) {
     Type type = cast<emitc::LValueType>(var.getType()).getValueType();
-    return rewriter.create<emitc::LoadOp>(loc, type, var).getResult();
+    return emitc::LoadOp::create(rewriter, loc, type, var).getResult();
   });
 }
 
@@ -129,7 +129,7 @@ static LogicalResult lowerYield(Operation *op, ValueRange resultVariables,
 
   assignValues(yieldOperands, resultVariables, rewriter, loc);
 
-  rewriter.create<emitc::YieldOp>(loc);
+  emitc::YieldOp::create(rewriter, loc);
   rewriter.eraseOp(yield);
 
   return success();
@@ -164,7 +164,7 @@ ForLowering::matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
 
   assignValues(adaptor.getInitArgs(), resultVariables, rewriter, loc);
 
-  emitc::ForOp loweredFor = rewriter.create<emitc::ForOp>(
+  emitc::ForOp loweredFor = emitc::ForOp::create(rewriter,
       loc, adaptor.getLowerBound(), adaptor.getUpperBound(), adaptor.getStep());
 
   Block *loweredBody = loweredFor.getBody();
@@ -257,7 +257,7 @@ IfLowering::matchAndRewrite(IfOp ifOp, OpAdaptor adaptor,
   bool hasElseBlock = !elseRegion.empty();
 
   auto loweredIf =
-      rewriter.create<emitc::IfOp>(loc, adaptor.getCondition(), false, false);
+      emitc::IfOp::create(rewriter, loc, adaptor.getCondition(), false, false);
 
   Region &loweredThenRegion = loweredIf.getThenRegion();
   auto result = lowerRegion(thenRegion, loweredThenRegion);
@@ -304,7 +304,7 @@ LogicalResult IndexSwitchOpLowering::matchAndRewrite(
                                        "create variables for results failed");
   }
 
-  auto loweredSwitch = rewriter.create<emitc::SwitchOp>(
+  auto loweredSwitch = emitc::SwitchOp::create(rewriter,
       loc, adaptor.getArg(), adaptor.getCases(), indexSwitchOp.getNumCases());
 
   // Lowering all case regions.
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index 844e66e927c4d..bfdb5a688b286 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -84,7 +84,7 @@ static Operation::operand_range getUpperBoundOperands(AffineForOp forOp) {
 // Get a Value that corresponds to the loop step.  If the step is an attribute,
 // materialize a corresponding constant using builder.
 static Value getOrCreateStep(AffineForOp forOp, OpBuilder &builder) {
-  return builder.create<arith::ConstantIndexOp>(forOp.getLoc(),
+  return arith::ConstantIndexOp::create(builder, forOp.getLoc(),
                                                 forOp.getStepAsInt());
 }
 
@@ -190,12 +190,12 @@ AffineLoopToGpuConverter::collectBounds(AffineForOp forOp, unsigned numLoops) {
       return std::nullopt;
     }
 
-    Value range = builder.create<arith::SubIOp>(currentLoop.getLoc(),
+    Value range = arith::SubIOp::create(builder, currentLoop.getLoc(),
                                                 upperBound, lowerBound);
     Value step = getOrCreateStep(currentLoop, builder);
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
       range =
-          builder.create<arith::CeilDivSIOp>(currentLoop.getLoc(), range, step);
+          arith::CeilDivSIOp::create(builder, currentLoop.getLoc(), range, step);
     dims.push_back(range);
 
     lbs.push_back(lowerBound);
@@ -221,7 +221,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
   // no loop mapped to a specific dimension, use constant "1" as its size.
   Value constOne =
       (numBlockDims < 3 || numThreadDims < 3)
-          ? builder.create<arith::ConstantIndexOp>(rootForOp.getLoc(), 1)
+          ? arith::ConstantIndexOp::create(builder, rootForOp.getLoc(), 1)
           : nullptr;
   Value gridSizeX = numBlockDims > 0 ? dims[0] : constOne;
   Value gridSizeY = numBlockDims > 1 ? dims[1] : constOne;
@@ -232,7 +232,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
 
   // Create a launch op and move the body region of the innermost loop to the
   // launch op.
-  auto launchOp = builder.create<gpu::LaunchOp>(
+  auto launchOp = gpu::LaunchOp::create(builder,
       rootForOp.getLoc(), gridSizeX, gridSizeY, gridSizeZ, blockSizeX,
       blockSizeY, blockSizeZ);
 
@@ -244,7 +244,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
   Location terminatorLoc = terminator.getLoc();
   terminator.erase();
   builder.setInsertionPointToEnd(innermostForOp.getBody());
-  builder.create<gpu::TerminatorOp>(terminatorLoc, TypeRange());
+  gpu::TerminatorOp::create(builder, terminatorLoc, TypeRange());
   launchOp.getBody().front().getOperations().splice(
       launchOp.getBody().front().begin(),
       innermostForOp.getBody()->getOperations());
@@ -263,10 +263,10 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
             : getDim3Value(launchOp.getThreadIds(), en.index() - numBlockDims);
     Value step = steps[en.index()];
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
-      id = builder.create<arith::MulIOp>(rootForOp.getLoc(), step, id);
+      id = arith::MulIOp::create(builder, rootForOp.getLoc(), step, id);
 
     Value ivReplacement =
-        builder.create<arith::AddIOp>(rootForOp.getLoc(), *lbArgumentIt, id);
+        arith::AddIOp::create(builder, rootForOp.getLoc(), *lbArgumentIt, id);
     en.value().replaceAllUsesWith(ivReplacement);
     std::advance(lbArgumentIt, 1);
     std::advance(stepArgumentIt, 1);
@@ -319,7 +319,7 @@ static Value deriveStaticUpperBound(Value upperBound,
   if (auto minOp = upperBound.getDefiningOp<AffineMinOp>()) {
     for (const AffineExpr &result : minOp.getMap().getResults()) {
       if (auto constExpr = dyn_cast<AffineConstantExpr>(result)) {
-        return rewriter.create<arith::ConstantIndexOp>(minOp.getLoc(),
+        return arith::ConstantIndexOp::create(rewriter, minOp.getLoc(),
                                                        constExpr.getValue());
       }
     }
@@ -344,7 +344,7 @@ static Value deriveStaticUpperBound(Value upperBound,
         if ((lhs.value() < 0) != (rhs.value() < 0))
           return {};
 
-        return rewriter.create<arith::ConstantIndexOp>(
+        return arith::ConstantIndexOp::create(rewriter,
             multiplyOp.getLoc(), lhs.value() * rhs.value());
       }
   }
@@ -422,7 +422,7 @@ static LogicalResult processParallelLoop(
     if (launchIndependent(val))
       return val;
     if (auto constOp = val.getDefiningOp<arith::ConstantOp>())
-      return rewriter.create<arith::ConstantOp>(constOp.getLoc(),
+      return arith::ConstantOp::create(rewriter, constOp.getLoc(),
                                                 constOp.getValue());
     return {};
   };
@@ -453,7 +453,7 @@ static LogicalResult processParallelLoop(
           1, 2,
           rewriter.getAffineDimExpr(0) * rewriter.getAffineSymbolExpr(0) +
               rewriter.getAffineSymbolExpr(1));
-      newIndex = rewriter.create<AffineApplyOp>(
+      newIndex = AffineApplyOp::create(rewriter,
           loc, annotation.getMap().compose(lowerAndStep),
           ValueRange{operand, ensureLaunchIndependent(step),
                      ensureLaunchIndependent(lowerBound)});
@@ -498,7 +498,7 @@ static LogicalResult processParallelLoop(
               1, 2,
               ((rewriter.getAffineDimExpr(0) - rewriter.getAffineSymbolExpr(0))
                    .ceilDiv(rewriter.getAffineSymbolExpr(1))));
-          Value launchBound = rewriter.create<AffineApplyOp>(
+          Value launchBound = AffineApplyOp::create(rewriter,
               loc, annotation.getBound().compose(stepMap),
               ValueRange{
                   ensureLaunchIndependent(
@@ -517,10 +517,10 @@ static LogicalResult processParallelLoop(
         if (!boundIsPrecise) {
           // We are using an approximation, create a surrounding conditional.
           Value originalBound = std::get<3>(config);
-          arith::CmpIOp pred = rewriter.create<arith::CmpIOp>(
+          arith::CmpIOp pred = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::slt, newIndex,
               cloningMap.lookupOrDefault(originalBound));
-          scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, pred, false);
+          scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, pred, false);
           rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
           // Put a sentinel into the worklist so we know when to pop out of the
           // if body again. We use the launchOp here, as that cannot be part of
@@ -530,7 +530,7 @@ static LogicalResult processParallelLoop(
       }
     } else {
       // Create a sequential for loop.
-      auto loopOp = rewriter.create<scf::ForOp>(
+      auto loopOp = scf::ForOp::create(rewriter,
           loc, cloningMap.lookupOrDefault(lowerBound),
           cloningMap.lookupOrDefault(upperBound),
           cloningMap.lookupOrDefault(step));
@@ -608,12 +608,12 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
   // sizes. Those will be refined later as we discover them from mappings.
   Location loc = parallelOp.getLoc();
   Value constantOne =
-      rewriter.create<arith::ConstantIndexOp>(parallelOp.getLoc(), 1);
-  gpu::LaunchOp launchOp = rewriter.create<gpu::LaunchOp>(
+      arith::ConstantIndexOp::create(rewriter, parallelOp.getLoc(), 1);
+  gpu::LaunchOp launchOp = gpu::LaunchOp::create(rewriter,
       parallelOp.getLoc(), constantOne, constantOne, constantOne, constantOne,
       constantOne, constantOne);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
-  rewriter.create<gpu::TerminatorOp>(loc);
+  gpu::TerminatorOp::create(rewriter, loc);
   rewriter.setInsertionPointToStart(&launchOp.getBody().front());
 
   IRMapping cloningMap;
@@ -667,7 +667,7 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
       if (externalValues.size())
         return failure();
       // Replace by gpu.all_reduce.
-      auto gpuRedOp = rewriter.create<gpu::AllReduceOp>(loc, newValue);
+      auto gpuRedOp = gpu::AllReduceOp::create(rewriter, loc, newValue);
       cloningMap.map(parentLoop->getResult(0), gpuRedOp.getResult());
       // Copy region.
       rewriter.inlineRegionBefore(reduceOp.getRegion(0), gpuRedOp.getRegion(),
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 584ac2f11b670..7b4c37d44824d 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -187,7 +187,7 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
            scf::ReduceOp reduce, int64_t reductionIndex, Attribute initValue) {
   OpBuilder::InsertionGuard guard(builder);
   Type type = reduce.getOperands()[reductionIndex].getType();
-  auto decl = builder.create<omp::DeclareReductionOp>(reduce.getLoc(),
+  auto decl = omp::DeclareReductionOp::create(builder, reduce.getLoc(),
                                                       "__scf_reduction", type);
   symbolTable.insert(decl);
 
@@ -196,8 +196,8 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
                       {reduce.getOperands()[reductionIndex].getLoc()});
   builder.setInsertionPointToEnd(&decl.getInitializerRegion().back());
   Value init =
-      builder.create<LLVM::ConstantOp>(reduce.getLoc(), type, initValue);
-  builder.create<omp::YieldOp>(reduce.getLoc(), init);
+      LLVM::ConstantOp::create(builder, reduce.getLoc(), type, initValue);
+  omp::YieldOp::create(builder, reduce.getLoc(), init);
 
   Operation *terminator =
       &reduce.getReductions()[reductionIndex].front().back();
@@ -227,12 +227,12 @@ static omp::DeclareReductionOp addAtomicRMW(OpBuilder &builder,
                       {reduceOperandLoc, reduceOperandLoc});
   Block *atomicBlock = &decl.getAtomicReductionRegion().back();
   builder.setInsertionPointToEnd(atomicBlock);
-  Value loaded = builder.create<LLVM::LoadOp>(reduce.getLoc(), decl.getType(),
+  Value loaded = LLVM::LoadOp::create(builder, reduce.getLoc(), decl.getType(),
                                               atomicBlock->getArgument(1));
-  builder.create<LLVM::AtomicRMWOp>(reduce.getLoc(), atomicKind,
+  LLVM::AtomicRMWOp::create(builder, reduce.getLoc(), atomicKind,
                                     atomicBlock->getArgument(0), loaded,
                                     LLVM::AtomicOrdering::monotonic);
-  builder.create<omp::YieldOp>(reduce.getLoc(), ArrayRef<Value>());
+  omp::YieldOp::create(builder, reduce.getLoc(), ArrayRef<Value>());
   return decl;
 }
 
@@ -380,7 +380,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     // Allocate reduction variables. Make sure the we don't overflow the stack
     // with local `alloca`s by saving and restoring the stack pointer.
     Location loc = parallelOp.getLoc();
-    Value one = rewriter.create<LLVM::ConstantOp>(
+    Value one = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerType(64), rewriter.getI64IntegerAttr(1));
     SmallVector<Value> reductionVariables;
     reductionVariables.reserve(parallelOp.getNumReductions());
@@ -391,8 +391,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
              "cannot create a reduction variable if the type is not an LLVM "
              "pointer element");
       Value storage =
-          rewriter.create<LLVM::AllocaOp>(loc, ptrType, init.getType(), one, 0);
-      rewriter.create<LLVM::StoreOp>(loc, init, storage);
+          LLVM::AllocaOp::create(rewriter, loc, ptrType, init.getType(), one, 0);
+      LLVM::StoreOp::create(rewriter, loc, init, storage);
       reductionVariables.push_back(storage);
     }
 
@@ -411,7 +411,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
       assert(redRegion.hasOneBlock() &&
              "expect reduction region to have one block");
       Value pvtRedVar = parallelOp.getRegion().addArgument(x.getType(), loc);
-      Value pvtRedVal = rewriter.create<LLVM::LoadOp>(reduce.getLoc(),
+      Value pvtRedVal = LLVM::LoadOp::create(rewriter, reduce.getLoc(),
                                                       rD.getType(), pvtRedVar);
       // Make a copy of the reduction combiner region in the body
       mlir::OpBuilder builder(rewriter.getContext());
@@ -427,7 +427,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           assert(yieldOp && yieldOp.getResults().size() == 1 &&
                  "expect YieldOp in reduction region to return one result");
           Value redVal = yieldOp.getResults()[0];
-          rewriter.create<LLVM::StoreOp>(loc, redVal, pvtRedVar);
+          LLVM::StoreOp::create(rewriter, loc, redVal, pvtRedVar);
           rewriter.eraseOp(yieldOp);
           break;
         }
@@ -437,11 +437,11 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
 
     Value numThreadsVar;
     if (numThreads > 0) {
-      numThreadsVar = rewriter.create<LLVM::ConstantOp>(
+      numThreadsVar = LLVM::ConstantOp::create(rewriter,
           loc, rewriter.getI32IntegerAttr(numThreads));
     }
     // Create the parallel wrapper.
-    auto ompParallel = rewriter.create<omp::ParallelOp>(
+    auto ompParallel = omp::ParallelOp::create(rewriter,
         loc,
         /* allocate_vars = */ llvm::SmallVector<Value>{},
         /* allocator_vars = */ llvm::SmallVector<Value>{},
@@ -464,7 +464,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
       {
         OpBuilder::InsertionGuard allocaGuard(rewriter);
         // Create worksharing loop wrapper.
-        auto wsloopOp = rewriter.create<omp::WsloopOp>(parallelOp.getLoc());
+        auto wsloopOp = omp::WsloopOp::create(rewriter, parallelOp.getLoc());
         if (!reductionVariables.empty()) {
           wsloopOp.setReductionSymsAttr(
               ArrayAttr::get(rewriter.getContext(), reductionSyms));
@@ -476,7 +476,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           wsloopOp.setReductionByref(
               DenseBoolArrayAttr::get(rewriter.getContext(), reductionByRef));
         }
-        rewriter.create<omp::TerminatorOp>(loc); // omp.parallel terminator.
+        omp::TerminatorOp::create(rewriter, loc); // omp.parallel terminator.
 
         // The wrapper's entry block arguments will define the reduction
         // variables.
@@ -490,7 +490,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
                                               parallelOp.getLoc()));
 
         // Create loop nest and populate region with contents of scf.parallel.
-        auto loopOp = rewriter.create<omp::LoopNestOp>(
+        auto loopOp = omp::LoopNestOp::create(rewriter,
             parallelOp.getLoc(), parallelOp.getLowerBound(),
             parallelOp.getUpperBound(), parallelOp.getStep());
 
@@ -511,13 +511,13 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
             rewriter.splitBlock(&loopOpEntryBlock, loopOpEntryBlock.begin());
         rewriter.setInsertionPointToStart(&loopOpEntryBlock);
 
-        auto scope = rewriter.create<memref::AllocaScopeOp>(parallelOp.getLoc(),
+        auto scope = memref::AllocaScopeOp::create(rewriter, parallelOp.getLoc(),
                                                             TypeRange());
-        rewriter.create<omp::YieldOp>(loc, ValueRange());
+        omp::YieldOp::create(rewriter, loc, ValueRange());
         Block *scopeBlock = rewriter.createBlock(&scope.getBodyRegion());
         rewriter.mergeBlocks(ops, scopeBlock);
         rewriter.setInsertionPointToEnd(&*scope.getBodyRegion().begin());
-        rewriter.create<memref::AllocaScopeReturnOp>(loc, ValueRange());
+        memref::AllocaScopeReturnOp::create(rewriter, loc, ValueRange());
       }
     }
 
@@ -526,7 +526,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     results.reserve(reductionVariables.size());
     for (auto [variable, type] :
          llvm::zip(reductionVariables, parallelOp.getResultTypes())) {
-      Value res = rewriter.create<LLVM::LoadOp>(loc, type, variable);
+      Value res = LLVM::LoadOp::create(rewriter, loc, type, variable);
       results.push_back(res);
     }
     rewriter.replaceOp(parallelOp, results);
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index 78d13278fef53..b494fa98f1362 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -71,12 +71,12 @@ void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp,
     auto pointerType =
         spirv::PointerType::get(convertedType, spirv::StorageClass::Function);
     rewriter.setInsertionPoint(newOp);
-    auto alloc = rewriter.create<spirv::VariableOp>(
+    auto alloc = spirv::VariableOp::create(rewriter,
         loc, pointerType, spirv::StorageClass::Function,
         /*initializer=*/nullptr);
     allocas.push_back(alloc);
     rewriter.setInsertionPointAfter(newOp);
-    Value loadResult = rewriter.create<spirv::LoadOp>(loc, alloc);
+    Value loadResult = spirv::LoadOp::create(rewriter, loc, alloc);
     resultValue.push_back(loadResult);
   }
   rewriter.replaceOp(scfOp, resultValue);
@@ -135,7 +135,7 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // a single back edge from the continue to header block, and a single exit
     // from header to merge.
     auto loc = forOp.getLoc();
-    auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
+    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     OpBuilder::InsertionGuard guard(rewriter);
@@ -172,15 +172,15 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     args.append(adaptor.getInitArgs().begin(), adaptor.getInitArgs().end());
     // Branch into it from the entry.
     rewriter.setInsertionPointToEnd(&(loopOp.getBody().front()));
-    rewriter.create<spirv::BranchOp>(loc, header, args);
+    spirv::BranchOp::create(rewriter, loc, header, args);
 
     // Generate the rest of the loop header.
     rewriter.setInsertionPointToEnd(header);
     auto *mergeBlock = loopOp.getMergeBlock();
-    auto cmpOp = rewriter.create<spirv::SLessThanOp>(
+    auto cmpOp = spirv::SLessThanOp::create(rewriter,
         loc, rewriter.getI1Type(), newIndVar, adaptor.getUpperBound());
 
-    rewriter.create<spirv::BranchConditionalOp>(
+    spirv::BranchConditionalOp::create(rewriter,
         loc, cmpOp, body, ArrayRef<Value>(), mergeBlock, ArrayRef<Value>());
 
     // Generate instructions to increment the step of the induction variable and
@@ -189,9 +189,9 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     rewriter.setInsertionPointToEnd(continueBlock);
 
     // Add the step to the induction variable and branch to the header.
-    Value updatedIndVar = rewriter.create<spirv::IAddOp>(
+    Value updatedIndVar = spirv::IAddOp::create(rewriter,
         loc, newIndVar.getType(), newIndVar, adaptor.getStep());
-    rewriter.create<spirv::BranchOp>(loc, header, updatedIndVar);
+    spirv::BranchOp::create(rewriter, loc, header, updatedIndVar);
 
     // Infer the return types from the init operands. Vector type may get
     // converted to CooperativeMatrix or to Vector type, to avoid having complex
@@ -238,10 +238,10 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     // Create `spirv.selection` operation, selection header block and merge
     // block.
     auto selectionOp =
-        rewriter.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
+        spirv::SelectionOp::create(rewriter, loc, spirv::SelectionControl::None);
     auto *mergeBlock = rewriter.createBlock(&selectionOp.getBody(),
                                             selectionOp.getBody().end());
-    rewriter.create<spirv::MergeOp>(loc);
+    spirv::MergeOp::create(rewriter, loc);
 
     OpBuilder::InsertionGuard guard(rewriter);
     auto *selectionHeaderBlock =
@@ -251,7 +251,7 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     auto &thenRegion = ifOp.getThenRegion();
     auto *thenBlock = &thenRegion.front();
     rewriter.setInsertionPointToEnd(&thenRegion.back());
-    rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+    spirv::BranchOp::create(rewriter, loc, mergeBlock);
     rewriter.inlineRegionBefore(thenRegion, mergeBlock);
 
     auto *elseBlock = mergeBlock;
@@ -261,13 +261,13 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
       auto &elseRegion = ifOp.getElseRegion();
       elseBlock = &elseRegion.front();
       rewriter.setInsertionPointToEnd(&elseRegion.back());
-      rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+      spirv::BranchOp::create(rewriter, loc, mergeBlock);
       rewriter.inlineRegionBefore(elseRegion, mergeBlock);
     }
 
     // Create a `spirv.BranchConditional` operation for selection header block.
     rewriter.setInsertionPointToEnd(selectionHeaderBlock);
-    rewriter.create<spirv::BranchConditionalOp>(loc, adaptor.getCondition(),
+    spirv::BranchConditionalOp::create(rewriter, loc, adaptor.getCondition(),
                                                 thenBlock, ArrayRef<Value>(),
                                                 elseBlock, ArrayRef<Value>());
 
@@ -310,7 +310,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
 
       auto loc = terminatorOp.getLoc();
       for (unsigned i = 0, e = operands.size(); i < e; i++)
-        rewriter.create<spirv::StoreOp>(loc, allocas[i], operands[i]);
+        spirv::StoreOp::create(rewriter, loc, allocas[i], operands[i]);
       if (isa<spirv::LoopOp>(parent)) {
         // For loops we also need to update the branch jumping back to the
         // header.
@@ -319,7 +319,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
         SmallVector<Value, 8> args(br.getBlockArguments());
         args.append(operands.begin(), operands.end());
         rewriter.setInsertionPoint(br);
-        rewriter.create<spirv::BranchOp>(terminatorOp.getLoc(), br.getTarget(),
+        spirv::BranchOp::create(rewriter, terminatorOp.getLoc(), br.getTarget(),
                                          args);
         rewriter.eraseOp(br);
       }
@@ -340,7 +340,7 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
   matchAndRewrite(scf::WhileOp whileOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto loc = whileOp.getLoc();
-    auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
+    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     Region &beforeRegion = whileOp.getBefore();
@@ -382,7 +382,7 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
     // Jump from the loop entry block to the loop header block.
     rewriter.setInsertionPointToEnd(&entryBlock);
-    rewriter.create<spirv::BranchOp>(loc, &beforeBlock, adaptor.getInits());
+    spirv::BranchOp::create(rewriter, loc, &beforeBlock, adaptor.getInits());
 
     auto condLoc = cond.getLoc();
 
@@ -403,18 +403,18 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
       // Create local variables before the scf.while op.
       rewriter.setInsertionPoint(loopOp);
-      auto alloc = rewriter.create<spirv::VariableOp>(
+      auto alloc = spirv::VariableOp::create(rewriter,
           condLoc, pointerType, spirv::StorageClass::Function,
           /*initializer=*/nullptr);
 
       // Load the final result values after the scf.while op.
       rewriter.setInsertionPointAfter(loopOp);
-      auto loadResult = rewriter.create<spirv::LoadOp>(condLoc, alloc);
+      auto loadResult = spirv::LoadOp::create(rewriter, condLoc, alloc);
       resultValues[i] = loadResult;
 
       // Store the current iteration's result value.
       rewriter.setInsertionPointToEnd(&beforeBlock);
-      rewriter.create<spirv::StoreOp>(condLoc, alloc, res);
+      spirv::StoreOp::create(rewriter, condLoc, alloc, res);
     }
 
     rewriter.setInsertionPointToEnd(&beforeBlock);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index d7ae9f0e94fe8..934bb0d1c37fd 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -68,7 +68,7 @@ static unsigned calculateGlobalIndex(spirv::GlobalVariableOp op) {
 /// Copies the given number of bytes from src to dst pointers.
 static void copy(Location loc, Value dst, Value src, Value size,
                  OpBuilder &builder) {
-  builder.create<LLVM::MemcpyOp>(loc, dst, src, size, /*isVolatile=*/false);
+  LLVM::MemcpyOp::create(builder, loc, dst, src, size, /*isVolatile=*/false);
 }
 
 /// Encodes the binding and descriptor set numbers into a new symbolic name.
@@ -194,7 +194,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
     if (!kernelFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(module.getBody());
-      kernelFunc = rewriter.create<LLVM::LLVMFuncOp>(
+      kernelFunc = LLVM::LLVMFuncOp::create(rewriter,
           rewriter.getUnknownLoc(), newKernelFuncName,
           LLVM::LLVMFunctionType::get(LLVM::LLVMVoidType::get(context),
                                       ArrayRef<Type>()));
@@ -245,7 +245,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       if (!dstGlobal) {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
-        dstGlobal = rewriter.create<LLVM::GlobalOp>(
+        dstGlobal = LLVM::GlobalOp::create(rewriter,
             loc, dstGlobalType,
             /*isConstant=*/false, LLVM::Linkage::Linkonce, name, Attribute(),
             /*alignment=*/0);
@@ -255,7 +255,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       // Copy the data from src operand pointer to dst global variable. Save
       // src, dst and size so that we can copy data back after emulating the
       // kernel call.
-      Value dst = rewriter.create<LLVM::AddressOfOp>(
+      Value dst = LLVM::AddressOfOp::create(rewriter,
           loc, typeConverter->convertType(spirvGlobal.getType()),
           dstGlobal.getSymName());
       copy(loc, dst, src, sizeBytes, rewriter);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index cea9d1fdec809..901b1eb265e32 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -94,12 +94,12 @@ static IntegerAttr minusOneIntegerAttribute(Type type, Builder builder) {
 static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
                                       PatternRewriter &rewriter) {
   if (isa<VectorType>(srcType)) {
-    return rewriter.create<LLVM::ConstantOp>(
+    return LLVM::ConstantOp::create(rewriter,
         loc, dstType,
         SplatElementsAttr::get(cast<ShapedType>(srcType),
                                minusOneIntegerAttribute(srcType, rewriter)));
   }
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, dstType, minusOneIntegerAttribute(srcType, rewriter));
 }
 
@@ -108,13 +108,13 @@ static Value createFPConstant(Location loc, Type srcType, Type dstType,
                               PatternRewriter &rewriter, double value) {
   if (auto vecType = dyn_cast<VectorType>(srcType)) {
     auto floatType = cast<FloatType>(vecType.getElementType());
-    return rewriter.create<LLVM::ConstantOp>(
+    return LLVM::ConstantOp::create(rewriter,
         loc, dstType,
         SplatElementsAttr::get(vecType,
                                rewriter.getFloatAttr(floatType, value)));
   }
   auto floatType = cast<FloatType>(srcType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, dstType, rewriter.getFloatAttr(floatType, value));
 }
 
@@ -134,13 +134,13 @@ static Value optionallyTruncateOrExtend(Location loc, Value value,
                                : getBitWidth(srcType);
 
   if (valueBitWidth < targetBitWidth)
-    return rewriter.create<LLVM::ZExtOp>(loc, llvmType, value);
+    return LLVM::ZExtOp::create(rewriter, loc, llvmType, value);
   // If the bit widths of `Count` and `Offset` are greater than the bit width
   // of the target type, they are truncated. Truncation is safe since `Count`
   // and `Offset` must be no more than 64 for op behaviour to be defined. Hence,
   // both values can be expressed in 8 bits.
   if (valueBitWidth > targetBitWidth)
-    return rewriter.create<LLVM::TruncOp>(loc, llvmType, value);
+    return LLVM::TruncOp::create(rewriter, loc, llvmType, value);
   return value;
 }
 
@@ -151,11 +151,11 @@ static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
   auto vectorType = VectorType::get(numElements, toBroadcast.getType());
   auto llvmVectorType = typeConverter.convertType(vectorType);
   auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
-  Value broadcasted = rewriter.create<LLVM::PoisonOp>(loc, llvmVectorType);
+  Value broadcasted = LLVM::PoisonOp::create(rewriter, loc, llvmVectorType);
   for (unsigned i = 0; i < numElements; ++i) {
-    auto index = rewriter.create<LLVM::ConstantOp>(
+    auto index = LLVM::ConstantOp::create(rewriter,
         loc, llvmI32Type, rewriter.getI32IntegerAttr(i));
-    broadcasted = rewriter.create<LLVM::InsertElementOp>(
+    broadcasted = LLVM::InsertElementOp::create(rewriter,
         loc, llvmVectorType, broadcasted, toBroadcast, index);
   }
   return broadcasted;
@@ -217,7 +217,7 @@ static Type convertStructTypePacked(spirv::StructType type,
 /// Creates LLVM dialect constant with the given value.
 static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
                                  unsigned value) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, IntegerType::get(rewriter.getContext(), 32),
       rewriter.getIntegerAttr(rewriter.getI32Type(), value));
 }
@@ -322,7 +322,7 @@ class AccessChainPattern : public SPIRVToLLVMConversion<spirv::AccessChainOp> {
     auto llvmIndexType = getTypeConverter()->convertType(indexType);
     if (!llvmIndexType)
       return rewriter.notifyMatchFailure(op, "type conversion failed");
-    Value zero = rewriter.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(rewriter,
         op.getLoc(), llvmIndexType, rewriter.getIntegerAttr(indexType, 0));
     indices.insert(indices.begin(), zero);
 
@@ -375,20 +375,20 @@ class BitFieldInsertPattern
     // Create a mask with bits set outside [Offset, Offset + Count - 1].
     Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
     Value maskShiftedByCount =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
-    Value negated = rewriter.create<LLVM::XOrOp>(loc, dstType,
+        LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
+    Value negated = LLVM::XOrOp::create(rewriter, loc, dstType,
                                                  maskShiftedByCount, minusOne);
     Value maskShiftedByCountAndOffset =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, negated, offset);
-    Value mask = rewriter.create<LLVM::XOrOp>(
+        LLVM::ShlOp::create(rewriter, loc, dstType, negated, offset);
+    Value mask = LLVM::XOrOp::create(rewriter,
         loc, dstType, maskShiftedByCountAndOffset, minusOne);
 
     // Extract unchanged bits from the `Base`  that are outside of
     // [Offset, Offset + Count - 1]. Then `or` with shifted `Insert`.
     Value baseAndMask =
-        rewriter.create<LLVM::AndOp>(loc, dstType, op.getBase(), mask);
+        LLVM::AndOp::create(rewriter, loc, dstType, op.getBase(), mask);
     Value insertShiftedByOffset =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, op.getInsert(), offset);
+        LLVM::ShlOp::create(rewriter, loc, dstType, op.getInsert(), offset);
     rewriter.replaceOpWithNewOp<LLVM::OrOp>(op, dstType, baseAndMask,
                                             insertShiftedByOffset);
     return success();
@@ -470,23 +470,23 @@ class BitFieldSExtractPattern
     auto baseSize = rewriter.getIntegerAttr(integerType, getBitWidth(srcType));
     Value size =
         isa<VectorType>(srcType)
-            ? rewriter.create<LLVM::ConstantOp>(
+            ? LLVM::ConstantOp::create(rewriter,
                   loc, dstType,
                   SplatElementsAttr::get(cast<ShapedType>(srcType), baseSize))
-            : rewriter.create<LLVM::ConstantOp>(loc, dstType, baseSize);
+            : LLVM::ConstantOp::create(rewriter, loc, dstType, baseSize);
 
     // Shift `Base` left by [sizeof(Base) - (Count + Offset)], so that the bit
     // at Offset + Count - 1 is the most significant bit now.
     Value countPlusOffset =
-        rewriter.create<LLVM::AddOp>(loc, dstType, count, offset);
+        LLVM::AddOp::create(rewriter, loc, dstType, count, offset);
     Value amountToShiftLeft =
-        rewriter.create<LLVM::SubOp>(loc, dstType, size, countPlusOffset);
-    Value baseShiftedLeft = rewriter.create<LLVM::ShlOp>(
+        LLVM::SubOp::create(rewriter, loc, dstType, size, countPlusOffset);
+    Value baseShiftedLeft = LLVM::ShlOp::create(rewriter,
         loc, dstType, op.getBase(), amountToShiftLeft);
 
     // Shift the result right, filling the bits with the sign bit.
     Value amountToShiftRight =
-        rewriter.create<LLVM::AddOp>(loc, dstType, offset, amountToShiftLeft);
+        LLVM::AddOp::create(rewriter, loc, dstType, offset, amountToShiftLeft);
     rewriter.replaceOpWithNewOp<LLVM::AShrOp>(op, dstType, baseShiftedLeft,
                                               amountToShiftRight);
     return success();
@@ -516,13 +516,13 @@ class BitFieldUExtractPattern
     // Create a mask with bits set at [0, Count - 1].
     Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
     Value maskShiftedByCount =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
-    Value mask = rewriter.create<LLVM::XOrOp>(loc, dstType, maskShiftedByCount,
+        LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
+    Value mask = LLVM::XOrOp::create(rewriter, loc, dstType, maskShiftedByCount,
                                               minusOne);
 
     // Shift `Base` by `Offset` and apply the mask on it.
     Value shiftedBase =
-        rewriter.create<LLVM::LShrOp>(loc, dstType, op.getBase(), offset);
+        LLVM::LShrOp::create(rewriter, loc, dstType, op.getBase(), offset);
     rewriter.replaceOpWithNewOp<LLVM::AndOp>(op, dstType, shiftedBase, mask);
     return success();
   }
@@ -694,7 +694,7 @@ class ExecutionModePattern
     auto structType = LLVM::LLVMStructType::getLiteral(context, fields);
 
     // Create `llvm.mlir.global` with initializer region containing one block.
-    auto global = rewriter.create<LLVM::GlobalOp>(
+    auto global = LLVM::GlobalOp::create(rewriter,
         UnknownLoc::get(context), structType, /*isConstant=*/true,
         LLVM::Linkage::External, executionModeInfoName, Attribute(),
         /*alignment=*/0);
@@ -704,22 +704,22 @@ class ExecutionModePattern
 
     // Initialize the struct and set the execution mode value.
     rewriter.setInsertionPointToStart(block);
-    Value structValue = rewriter.create<LLVM::PoisonOp>(loc, structType);
-    Value executionMode = rewriter.create<LLVM::ConstantOp>(
+    Value structValue = LLVM::PoisonOp::create(rewriter, loc, structType);
+    Value executionMode = LLVM::ConstantOp::create(rewriter,
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
-    structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
+    structValue = LLVM::InsertValueOp::create(rewriter, loc, structValue,
                                                        executionMode, 0);
 
     // Insert extra operands if they exist into execution mode info struct.
     for (unsigned i = 0, e = values.size(); i < e; ++i) {
       auto attr = values.getValue()[i];
-      Value entry = rewriter.create<LLVM::ConstantOp>(loc, llvmI32Type, attr);
-      structValue = rewriter.create<LLVM::InsertValueOp>(
+      Value entry = LLVM::ConstantOp::create(rewriter, loc, llvmI32Type, attr);
+      structValue = LLVM::InsertValueOp::create(rewriter,
           loc, structValue, entry, ArrayRef<int64_t>({1, i}));
     }
-    rewriter.create<LLVM::ReturnOp>(loc, ArrayRef<Value>({structValue}));
+    LLVM::ReturnOp::create(rewriter, loc, ArrayRef<Value>({structValue}));
     rewriter.eraseOp(op);
     return success();
   }
@@ -911,7 +911,7 @@ class InverseSqrtPattern
 
     Location loc = op.getLoc();
     Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
-    Value sqrt = rewriter.create<LLVM::SqrtOp>(loc, dstType, op.getOperand());
+    Value sqrt = LLVM::SqrtOp::create(rewriter, loc, dstType, op.getOperand());
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(op, dstType, one, sqrt);
     return success();
   }
@@ -971,10 +971,10 @@ class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
     IntegerAttr minusOne = minusOneIntegerAttribute(srcType, rewriter);
     auto mask =
         isa<VectorType>(srcType)
-            ? rewriter.create<LLVM::ConstantOp>(
+            ? LLVM::ConstantOp::create(rewriter,
                   loc, dstType,
                   SplatElementsAttr::get(cast<VectorType>(srcType), minusOne))
-            : rewriter.create<LLVM::ConstantOp>(loc, dstType, minusOne);
+            : LLVM::ConstantOp::create(rewriter, loc, dstType, minusOne);
     rewriter.template replaceOpWithNewOp<LLVM::XOrOp>(notOp, dstType,
                                                       notOp.getOperand(), mask);
     return success();
@@ -1032,7 +1032,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
     return func;
 
   OpBuilder b(symbolTable->getRegion(0));
-  func = b.create<LLVM::LLVMFuncOp>(
+  func = LLVM::LLVMFuncOp::create(b,
       symbolTable->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes));
   func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
@@ -1045,7 +1045,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
 static LLVM::CallOp createSPIRVBuiltinCall(Location loc, OpBuilder &builder,
                                            LLVM::LLVMFuncOp func,
                                            ValueRange args) {
-  auto call = builder.create<LLVM::CallOp>(loc, func, args);
+  auto call = LLVM::CallOp::create(builder, loc, func, args);
   call.setCConv(func.getCConv());
   call.setConvergentAttr(func.getConvergentAttr());
   call.setNoUnwindAttr(func.getNoUnwindAttr());
@@ -1076,11 +1076,11 @@ class ControlBarrierPattern : public SPIRVToLLVMConversion<BarrierOpTy> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, {i32, i32, i32}, voidTy);
 
     Location loc = controlBarrierOp->getLoc();
-    Value execution = rewriter.create<LLVM::ConstantOp>(
+    Value execution = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value memory = rewriter.create<LLVM::ConstantOp>(
+    Value memory = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getMemoryScope()));
-    Value semantics = rewriter.create<LLVM::ConstantOp>(
+    Value semantics = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getMemorySemantics()));
 
     auto call = createSPIRVBuiltinCall(loc, rewriter, func,
@@ -1253,9 +1253,9 @@ class GroupReducePattern : public SPIRVToLLVMConversion<ReduceOp> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, paramTypes, retTy);
 
     Location loc = op.getLoc();
-    Value scope = rewriter.create<LLVM::ConstantOp>(
+    Value scope = LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value groupOp = rewriter.create<LLVM::ConstantOp>(
+    Value groupOp = LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, static_cast<int32_t>(adaptor.getGroupOperation()));
     SmallVector<Value> operands{scope, groupOp};
     operands.append(adaptor.getOperands().begin(), adaptor.getOperands().end());
@@ -1366,7 +1366,7 @@ class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
       return failure();
     Block *headerBlock = loopOp.getHeaderBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
-    rewriter.create<LLVM::BrOp>(loc, brOp.getBlockArguments(), headerBlock);
+    LLVM::BrOp::create(rewriter, loc, brOp.getBlockArguments(), headerBlock);
     rewriter.eraseBlock(entryBlock);
 
     // Branch from merge block to end block.
@@ -1374,7 +1374,7 @@ class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
     Operation *terminator = mergeBlock->getTerminator();
     ValueRange terminatorOperands = terminator->getOperands();
     rewriter.setInsertionPointToEnd(mergeBlock);
-    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, endBlock);
+    LLVM::BrOp::create(rewriter, loc, terminatorOperands, endBlock);
 
     rewriter.inlineRegionBefore(loopOp.getBody(), endBlock);
     rewriter.replaceOp(loopOp, endBlock->getArguments());
@@ -1433,13 +1433,13 @@ class SelectionPattern : public SPIRVToLLVMConversion<spirv::SelectionOp> {
     Operation *terminator = mergeBlock->getTerminator();
     ValueRange terminatorOperands = terminator->getOperands();
     rewriter.setInsertionPointToEnd(mergeBlock);
-    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, continueBlock);
+    LLVM::BrOp::create(rewriter, loc, terminatorOperands, continueBlock);
 
     // Link current block to `true` and `false` blocks within the selection.
     Block *trueBlock = condBrOp.getTrueBlock();
     Block *falseBlock = condBrOp.getFalseBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
-    rewriter.create<LLVM::CondBrOp>(loc, condBrOp.getCondition(), trueBlock,
+    LLVM::CondBrOp::create(rewriter, loc, condBrOp.getCondition(), trueBlock,
                                     condBrOp.getTrueTargetOperands(),
                                     falseBlock,
                                     condBrOp.getFalseTargetOperands());
@@ -1519,8 +1519,8 @@ class TanPattern : public SPIRVToLLVMConversion<spirv::GLTanOp> {
       return rewriter.notifyMatchFailure(tanOp, "type conversion failed");
 
     Location loc = tanOp.getLoc();
-    Value sin = rewriter.create<LLVM::SinOp>(loc, dstType, tanOp.getOperand());
-    Value cos = rewriter.create<LLVM::CosOp>(loc, dstType, tanOp.getOperand());
+    Value sin = LLVM::SinOp::create(rewriter, loc, dstType, tanOp.getOperand());
+    Value cos = LLVM::CosOp::create(rewriter, loc, dstType, tanOp.getOperand());
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanOp, dstType, sin, cos);
     return success();
   }
@@ -1547,13 +1547,13 @@ class TanhPattern : public SPIRVToLLVMConversion<spirv::GLTanhOp> {
     Location loc = tanhOp.getLoc();
     Value two = createFPConstant(loc, srcType, dstType, rewriter, 2.0);
     Value multiplied =
-        rewriter.create<LLVM::FMulOp>(loc, dstType, two, tanhOp.getOperand());
-    Value exponential = rewriter.create<LLVM::ExpOp>(loc, dstType, multiplied);
+        LLVM::FMulOp::create(rewriter, loc, dstType, two, tanhOp.getOperand());
+    Value exponential = LLVM::ExpOp::create(rewriter, loc, dstType, multiplied);
     Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
     Value numerator =
-        rewriter.create<LLVM::FSubOp>(loc, dstType, exponential, one);
+        LLVM::FSubOp::create(rewriter, loc, dstType, exponential, one);
     Value denominator =
-        rewriter.create<LLVM::FAddOp>(loc, dstType, exponential, one);
+        LLVM::FAddOp::create(rewriter, loc, dstType, exponential, one);
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanhOp, dstType, numerator,
                                               denominator);
     return success();
@@ -1592,8 +1592,8 @@ class VariablePattern : public SPIRVToLLVMConversion<spirv::VariableOp> {
     if (!elementType)
       return rewriter.notifyMatchFailure(varOp, "type conversion failed");
     Value allocated =
-        rewriter.create<LLVM::AllocaOp>(loc, dstType, elementType, size);
-    rewriter.create<LLVM::StoreOp>(loc, adaptor.getInitializer(), allocated);
+        LLVM::AllocaOp::create(rewriter, loc, dstType, elementType, size);
+    LLVM::StoreOp::create(rewriter, loc, adaptor.getInitializer(), allocated);
     rewriter.replaceOp(varOp, allocated);
     return success();
   }
@@ -1654,7 +1654,7 @@ class FuncConversionPattern : public SPIRVToLLVMConversion<spirv::FuncOp> {
     // Create a new `LLVMFuncOp`
     Location loc = funcOp.getLoc();
     StringRef name = funcOp.getName();
-    auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(loc, name, llvmType);
+    auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter, loc, name, llvmType);
 
     // Convert SPIR-V Function Control to equivalent LLVM function attribute
     MLIRContext *context = funcOp.getContext();
@@ -1708,7 +1708,7 @@ class ModuleConversionPattern : public SPIRVToLLVMConversion<spirv::ModuleOp> {
                   ConversionPatternRewriter &rewriter) const override {
 
     auto newModuleOp =
-        rewriter.create<ModuleOp>(spvModuleOp.getLoc(), spvModuleOp.getName());
+        ModuleOp::create(rewriter, spvModuleOp.getLoc(), spvModuleOp.getName());
     rewriter.inlineRegionBefore(spvModuleOp.getRegion(), newModuleOp.getBody());
 
     // Remove the terminator block that was automatically added by builder
@@ -1749,7 +1749,7 @@ class VectorShufflePattern
     auto componentsArray = components.getValue();
     auto *context = rewriter.getContext();
     auto llvmI32Type = IntegerType::get(context, 32);
-    Value targetOp = rewriter.create<LLVM::PoisonOp>(loc, dstType);
+    Value targetOp = LLVM::PoisonOp::create(rewriter, loc, dstType);
     for (unsigned i = 0; i < componentsArray.size(); i++) {
       if (!isa<IntegerAttr>(componentsArray[i]))
         return op.emitError("unable to support non-constant component");
@@ -1765,15 +1765,15 @@ class VectorShufflePattern
         baseVector = vector2;
       }
 
-      Value dstIndex = rewriter.create<LLVM::ConstantOp>(
+      Value dstIndex = LLVM::ConstantOp::create(rewriter,
           loc, llvmI32Type, rewriter.getIntegerAttr(rewriter.getI32Type(), i));
-      Value index = rewriter.create<LLVM::ConstantOp>(
+      Value index = LLVM::ConstantOp::create(rewriter,
           loc, llvmI32Type,
           rewriter.getIntegerAttr(rewriter.getI32Type(), indexVal - offsetVal));
 
-      auto extractOp = rewriter.create<LLVM::ExtractElementOp>(
+      auto extractOp = LLVM::ExtractElementOp::create(rewriter,
           loc, scalarType, baseVector, index);
-      targetOp = rewriter.create<LLVM::InsertElementOp>(loc, dstType, targetOp,
+      targetOp = LLVM::InsertElementOp::create(rewriter, loc, dstType, targetOp,
                                                         extractOp, dstIndex);
     }
     rewriter.replaceOp(op, targetOp);
diff --git a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
index da9ad3dd67328..245e60b04ec31 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
@@ -32,7 +32,7 @@ class ConvertCstrRequireOp : public OpRewritePattern<shape::CstrRequireOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(shape::CstrRequireOp op,
                                 PatternRewriter &rewriter) const override {
-    rewriter.create<cf::AssertOp>(op.getLoc(), op.getPred(), op.getMsgAttr());
+    cf::AssertOp::create(rewriter, op.getLoc(), op.getPred(), op.getMsgAttr());
     rewriter.replaceOpWithNewOp<shape::ConstWitnessOp>(op, true);
     return success();
   }
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index bbe1490137bf8..58cffa6119d93 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -82,19 +82,19 @@ struct BroadcastOpConverter : public OpConversionPattern<BroadcastOp> {
 // number of extent tensors and shifted offsets into them.
 Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
                         ValueRange rankDiffs, Value outputDimension) {
-  Value one = lb.create<arith::ConstantIndexOp>(1);
+  Value one = arith::ConstantIndexOp::create(lb, 1);
   Value broadcastedDim = one;
   for (auto tup : llvm::zip(extentTensors, rankDiffs)) {
     Value shape = std::get<0>(tup);
     Value rankDiff = std::get<1>(tup);
-    Value outOfBounds = lb.create<arith::CmpIOp>(arith::CmpIPredicate::ult,
+    Value outOfBounds = arith::CmpIOp::create(lb, arith::CmpIPredicate::ult,
                                                  outputDimension, rankDiff);
     Type indexTy = lb.getIndexType();
     broadcastedDim =
-        lb.create<IfOp>(
+        IfOp::create(lb,
               outOfBounds,
               [&](OpBuilder &b, Location loc) {
-                b.create<scf::YieldOp>(loc, broadcastedDim);
+                scf::YieldOp::create(b, loc, broadcastedDim);
               },
               [&](OpBuilder &b, Location loc) {
                 // The broadcasting logic is:
@@ -104,17 +104,17 @@ Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
                 // - otherwise, take the extent as-is.
                 // Note that this logic remains correct in the presence
                 // of dimensions of zero extent.
-                Value lesserRankOperandDimension = b.create<arith::SubIOp>(
+                Value lesserRankOperandDimension = arith::SubIOp::create(b,
                     loc, indexTy, outputDimension, rankDiff);
-                Value lesserRankOperandExtent = b.create<tensor::ExtractOp>(
+                Value lesserRankOperandExtent = tensor::ExtractOp::create(b,
                     loc, shape, ValueRange{lesserRankOperandDimension});
 
                 Value dimIsOne =
-                    b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+                    arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
                                             lesserRankOperandExtent, one);
-                Value dim = b.create<arith::SelectOp>(
+                Value dim = arith::SelectOp::create(b,
                     loc, dimIsOne, broadcastedDim, lesserRankOperandExtent);
-                b.create<scf::YieldOp>(loc, dim);
+                scf::YieldOp::create(b, loc, dim);
               })
             .getResult(0);
   }
@@ -133,7 +133,7 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
   auto loc = op.getLoc();
   ImplicitLocOpBuilder lb(loc, rewriter);
 
-  Value zero = lb.create<arith::ConstantIndexOp>(0);
+  Value zero = arith::ConstantIndexOp::create(lb, 0);
   Type indexTy = lb.getIndexType();
 
   // Save all the ranks for bounds checking. Because this is a tensor
@@ -141,31 +141,31 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
   // dimension in the tensor.
   SmallVector<Value> ranks, rankDiffs;
   llvm::append_range(ranks, llvm::map_range(adaptor.getShapes(), [&](Value v) {
-                       return lb.create<tensor::DimOp>(v, zero);
+                       return tensor::DimOp::create(lb, v, zero);
                      }));
 
   // Find the maximum rank
   Value maxRank = ranks.front();
   for (Value v : llvm::drop_begin(ranks, 1)) {
-    maxRank = lb.create<arith::MaxUIOp>(v, maxRank);
+    maxRank = arith::MaxUIOp::create(lb, v, maxRank);
   }
 
   // Calculate the difference of ranks and the maximum rank for later offsets.
   llvm::append_range(rankDiffs, llvm::map_range(ranks, [&](Value v) {
-                       return lb.create<arith::SubIOp>(indexTy, maxRank, v);
+                       return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
-  Value replacement = lb.create<tensor::GenerateOp>(
+  Value replacement = tensor::GenerateOp::create(lb,
       getExtentTensorType(lb.getContext()), ValueRange{maxRank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value broadcastedDim =
             getBroadcastedDim(ImplicitLocOpBuilder(loc, b), adaptor.getShapes(),
                               rankDiffs, args[0]);
 
-        b.create<tensor::YieldOp>(loc, broadcastedDim);
+        tensor::YieldOp::create(b, loc, broadcastedDim);
       });
   if (replacement.getType() != op.getType())
-    replacement = lb.create<tensor::CastOp>(op.getType(), replacement);
+    replacement = tensor::CastOp::create(lb, op.getType(), replacement);
   rewriter.replaceOp(op, replacement);
   return success();
 }
@@ -194,12 +194,12 @@ LogicalResult ConstShapeOpConverter::matchAndRewrite(
   SmallVector<Value, 4> extentOperands;
   for (auto extent : op.getShape()) {
     extentOperands.push_back(
-        rewriter.create<arith::ConstantIndexOp>(loc, extent.getLimitedValue()));
+        arith::ConstantIndexOp::create(rewriter, loc, extent.getLimitedValue()));
   }
   Type resultTy =
       RankedTensorType::get({op.getShape().size()}, rewriter.getIndexType());
   Value tensor =
-      rewriter.create<tensor::FromElementsOp>(loc, resultTy, extentOperands);
+      tensor::FromElementsOp::create(rewriter, loc, resultTy, extentOperands);
   rewriter.replaceOpWithNewOp<tensor::CastOp>(op, resultTy, tensor);
   return success();
 }
@@ -245,8 +245,8 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
 
   auto loc = op.getLoc();
   ImplicitLocOpBuilder lb(loc, rewriter);
-  Value zero = lb.create<arith::ConstantIndexOp>(0);
-  Value one = lb.create<arith::ConstantIndexOp>(1);
+  Value zero = arith::ConstantIndexOp::create(lb, 0);
+  Value one = arith::ConstantIndexOp::create(lb, 1);
   Type indexTy = lb.getIndexType();
 
   // Save all the ranks for bounds checking. Because this is a tensor
@@ -254,25 +254,25 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
   // dimension in the tensor.
   SmallVector<Value> ranks, rankDiffs;
   llvm::append_range(ranks, llvm::map_range(adaptor.getShapes(), [&](Value v) {
-                       return lb.create<tensor::DimOp>(v, zero);
+                       return tensor::DimOp::create(lb, v, zero);
                      }));
 
   // Find the maximum rank
   Value maxRank = ranks.front();
   for (Value v : llvm::drop_begin(ranks, 1)) {
-    maxRank = lb.create<arith::MaxUIOp>(v, maxRank);
+    maxRank = arith::MaxUIOp::create(lb, v, maxRank);
   }
 
   // Calculate the difference of ranks and the maximum rank for later offsets.
   llvm::append_range(rankDiffs, llvm::map_range(ranks, [&](Value v) {
-                       return lb.create<arith::SubIOp>(indexTy, maxRank, v);
+                       return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
   Type i1Ty = rewriter.getI1Type();
   Value trueVal =
-      rewriter.create<arith::ConstantOp>(loc, i1Ty, rewriter.getBoolAttr(true));
+      arith::ConstantOp::create(rewriter, loc, i1Ty, rewriter.getBoolAttr(true));
 
-  auto reduceResult = lb.create<ForOp>(
+  auto reduceResult = ForOp::create(lb,
       loc, zero, maxRank, one, ValueRange{trueVal},
       [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
         // Find a non-1 dim, if it exists. Note that the first part of this
@@ -285,38 +285,38 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
         for (auto tup : llvm::zip(adaptor.getShapes(), rankDiffs)) {
           Value shape, rankDiff;
           std::tie(shape, rankDiff) = tup;
-          Value outOfBounds = b.create<arith::CmpIOp>(
+          Value outOfBounds = arith::CmpIOp::create(b,
               loc, arith::CmpIPredicate::ult, iv, rankDiff);
           broadcastable =
-              b.create<IfOp>(
+              IfOp::create(b,
                    loc, outOfBounds,
                    [&](OpBuilder &b, Location loc) {
                      // Non existent dimensions are always broadcastable
-                     b.create<scf::YieldOp>(loc, broadcastable);
+                     scf::YieldOp::create(b, loc, broadcastable);
                    },
                    [&](OpBuilder &b, Location loc) {
                      // Every value needs to be either 1, or the same non-1
                      // value to be broadcastable in this dim.
                      Value operandDimension =
-                         b.create<arith::SubIOp>(loc, indexTy, iv, rankDiff);
-                     Value dimensionExtent = b.create<tensor::ExtractOp>(
+                         arith::SubIOp::create(b, loc, indexTy, iv, rankDiff);
+                     Value dimensionExtent = tensor::ExtractOp::create(b,
                          loc, shape, ValueRange{operandDimension});
 
-                     Value equalOne = b.create<arith::CmpIOp>(
+                     Value equalOne = arith::CmpIOp::create(b,
                          loc, arith::CmpIPredicate::eq, dimensionExtent, one);
-                     Value equalBroadcasted = b.create<arith::CmpIOp>(
+                     Value equalBroadcasted = arith::CmpIOp::create(b,
                          loc, arith::CmpIPredicate::eq, dimensionExtent,
                          broadcastedDim);
-                     Value result = b.create<arith::AndIOp>(
+                     Value result = arith::AndIOp::create(b,
                          loc, broadcastable,
-                         b.create<arith::OrIOp>(loc, equalOne,
+                         arith::OrIOp::create(b, loc, equalOne,
                                                 equalBroadcasted));
-                     b.create<scf::YieldOp>(loc, result);
+                     scf::YieldOp::create(b, loc, result);
                    })
                   .getResult(0);
         }
 
-        b.create<scf::YieldOp>(loc, broadcastable);
+        scf::YieldOp::create(b, loc, broadcastable);
       });
 
   rewriter.replaceOp(op, reduceResult.getResults().front());
@@ -339,7 +339,7 @@ DimOpConverter::matchAndRewrite(DimOp op, OpAdaptor adaptor,
   // Lower to dim(X, i) to get_extent(shape_of(X), i) and rely on further
   // lowerings. This can be further optimized if needed to avoid intermediate
   // steps.
-  auto shapeOf = rewriter.create<shape::ShapeOfOp>(op.getLoc(), op.getValue());
+  auto shapeOf = shape::ShapeOfOp::create(rewriter, op.getLoc(), op.getValue());
   rewriter.replaceOpWithNewOp<shape::GetExtentOp>(op, op.getType(), shapeOf,
                                                   op.getIndex());
   return success();
@@ -421,16 +421,16 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
 
   auto loc = op.getLoc();
 
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
   Type indexTy = rewriter.getIndexType();
   Value rank =
-      rewriter.create<tensor::DimOp>(loc, indexTy, adaptor.getShape(), zero);
+      tensor::DimOp::create(rewriter, loc, indexTy, adaptor.getShape(), zero);
 
-  auto loop = rewriter.create<scf::ForOp>(
+  auto loop = scf::ForOp::create(rewriter,
       loc, zero, rank, one, op.getInitVals(),
       [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
-        Value extent = b.create<tensor::ExtractOp>(loc, adaptor.getShape(), iv);
+        Value extent = tensor::ExtractOp::create(b, loc, adaptor.getShape(), iv);
 
         SmallVector<Value, 2> mappedValues{iv, extent};
         mappedValues.append(args.begin(), args.end());
@@ -444,7 +444,7 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
         SmallVector<Value, 2> mappedResults;
         for (auto result : reduceBody->getTerminator()->getOperands())
           mappedResults.push_back(mapping.lookup(result));
-        b.create<scf::YieldOp>(loc, mappedResults);
+        scf::YieldOp::create(b, loc, mappedResults);
       });
 
   rewriter.replaceOp(op, loop.getResults());
@@ -507,43 +507,43 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, OpAdaptor adaptor,
 
   auto loc = op.getLoc();
   Type indexTy = rewriter.getIndexType();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   Value firstShape = adaptor.getShapes().front();
   Value firstRank =
-      rewriter.create<tensor::DimOp>(loc, indexTy, firstShape, zero);
+      tensor::DimOp::create(rewriter, loc, indexTy, firstShape, zero);
   Value result = nullptr;
   // Generate a linear sequence of compares, all with firstShape as lhs.
   for (Value shape : adaptor.getShapes().drop_front(1)) {
-    Value rank = rewriter.create<tensor::DimOp>(loc, indexTy, shape, zero);
-    Value eqRank = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value rank = tensor::DimOp::create(rewriter, loc, indexTy, shape, zero);
+    Value eqRank = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                   firstRank, rank);
-    auto same = rewriter.create<IfOp>(
+    auto same = IfOp::create(rewriter,
         loc, eqRank,
         [&](OpBuilder &b, Location loc) {
-          Value one = b.create<arith::ConstantIndexOp>(loc, 1);
+          Value one = arith::ConstantIndexOp::create(b, loc, 1);
           Value init =
-              b.create<arith::ConstantOp>(loc, i1Ty, b.getBoolAttr(true));
-          auto loop = b.create<scf::ForOp>(
+              arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(true));
+          auto loop = scf::ForOp::create(b,
               loc, zero, firstRank, one, ValueRange{init},
               [&](OpBuilder &b, Location nestedLoc, Value iv, ValueRange args) {
                 Value conj = args[0];
                 Value lhsExtent =
-                    b.create<tensor::ExtractOp>(loc, firstShape, iv);
-                Value rhsExtent = b.create<tensor::ExtractOp>(loc, shape, iv);
-                Value eqExtent = b.create<arith::CmpIOp>(
+                    tensor::ExtractOp::create(b, loc, firstShape, iv);
+                Value rhsExtent = tensor::ExtractOp::create(b, loc, shape, iv);
+                Value eqExtent = arith::CmpIOp::create(b,
                     loc, arith::CmpIPredicate::eq, lhsExtent, rhsExtent);
-                Value conjNext = b.create<arith::AndIOp>(loc, conj, eqExtent);
-                b.create<scf::YieldOp>(loc, ValueRange({conjNext}));
+                Value conjNext = arith::AndIOp::create(b, loc, conj, eqExtent);
+                scf::YieldOp::create(b, loc, ValueRange({conjNext}));
               });
-          b.create<scf::YieldOp>(loc, loop.getResults());
+          scf::YieldOp::create(b, loc, loop.getResults());
         },
         [&](OpBuilder &b, Location loc) {
           Value result =
-              b.create<arith::ConstantOp>(loc, i1Ty, b.getBoolAttr(false));
-          b.create<scf::YieldOp>(loc, result);
+              arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(false));
+          scf::YieldOp::create(b, loc, result);
         });
     result = !result ? same.getResult(0)
-                     : rewriter.create<arith::AndIOp>(loc, result,
+                     : arith::AndIOp::create(rewriter, loc, result,
                                                       same.getResult(0));
   }
   rewriter.replaceOp(op, result);
@@ -581,17 +581,17 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
     int64_t rank = rankedTensorTy.getRank();
     for (int64_t i = 0; i < rank; i++) {
       if (rankedTensorTy.isDynamicDim(i)) {
-        Value extent = rewriter.create<tensor::DimOp>(loc, tensor, i);
+        Value extent = tensor::DimOp::create(rewriter, loc, tensor, i);
         extentValues.push_back(extent);
       } else {
-        Value extent = rewriter.create<arith::ConstantIndexOp>(
+        Value extent = arith::ConstantIndexOp::create(rewriter,
             loc, rankedTensorTy.getDimSize(i));
         extentValues.push_back(extent);
       }
     }
 
     // Materialize extent tensor.
-    Value staticExtentTensor = rewriter.create<tensor::FromElementsOp>(
+    Value staticExtentTensor = tensor::FromElementsOp::create(rewriter,
         loc, RankedTensorType::get({rank}, rewriter.getIndexType()),
         extentValues);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(),
@@ -601,13 +601,13 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
 
   // Lower to `tensor.generate` otherwise.
   auto *ctx = rewriter.getContext();
-  Value rank = rewriter.create<tensor::RankOp>(loc, tensor);
+  Value rank = tensor::RankOp::create(rewriter, loc, tensor);
   rewriter.replaceOpWithNewOp<tensor::GenerateOp>(
       op, getExtentTensorType(ctx), ValueRange{rank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value dim = args.front();
-        Value extent = b.create<tensor::DimOp>(loc, tensor, dim);
-        b.create<tensor::YieldOp>(loc, extent);
+        Value extent = tensor::DimOp::create(b, loc, tensor, dim);
+        tensor::YieldOp::create(b, loc, extent);
       });
 
   return success();
@@ -634,21 +634,21 @@ LogicalResult SplitAtOpConversion::matchAndRewrite(
     return failure();
 
   ImplicitLocOpBuilder b(op.getLoc(), rewriter);
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value rank = b.create<tensor::DimOp>(adaptor.getOperand(), zero);
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value rank = tensor::DimOp::create(b, adaptor.getOperand(), zero);
 
   // index < 0 ? index + rank : index
   Value originalIndex = adaptor.getIndex();
-  Value add = b.create<arith::AddIOp>(originalIndex, rank);
+  Value add = arith::AddIOp::create(b, originalIndex, rank);
   Value indexIsNegative =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::slt, originalIndex, zero);
-  Value index = b.create<arith::SelectOp>(indexIsNegative, add, originalIndex);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, originalIndex, zero);
+  Value index = arith::SelectOp::create(b, indexIsNegative, add, originalIndex);
 
-  Value one = b.create<arith::ConstantIndexOp>(1);
+  Value one = arith::ConstantIndexOp::create(b, 1);
   Value head =
-      b.create<tensor::ExtractSliceOp>(adaptor.getOperand(), zero, index, one);
-  Value tailSize = b.create<arith::SubIOp>(rank, index);
-  Value tail = b.create<tensor::ExtractSliceOp>(adaptor.getOperand(), index,
+      tensor::ExtractSliceOp::create(b, adaptor.getOperand(), zero, index, one);
+  Value tailSize = arith::SubIOp::create(b, rank, index);
+  Value tail = tensor::ExtractSliceOp::create(b, adaptor.getOperand(), index,
                                                 tailSize, one);
   rewriter.replaceOp(op, {head, tail});
   return success();
diff --git a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
index 2c4d27502a521..f06c3e1f7cb18 100644
--- a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
+++ b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
@@ -68,10 +68,10 @@ class TensorExtractPattern final
       // We could use the initializer directly; but certain driver compilers
       // have bugs dealing with that. So for now, use spirv.Store for
       // initialization.
-      varOp = rewriter.create<spirv::VariableOp>(loc, varType,
+      varOp = spirv::VariableOp::create(rewriter, loc, varType,
                                                  spirv::StorageClass::Function,
                                                  /*initializer=*/nullptr);
-      rewriter.create<spirv::StoreOp>(loc, varOp, adaptor.getTensor());
+      spirv::StoreOp::create(rewriter, loc, varOp, adaptor.getTensor());
     } else {
       // Need to store the value to the local variable. It's questionable
       // whether we want to support such case though.
@@ -83,7 +83,7 @@ class TensorExtractPattern final
 
     Value index = spirv::linearizeIndex(adaptor.getIndices(), strides,
                                         /*offset=*/0, indexType, loc, rewriter);
-    auto acOp = rewriter.create<spirv::AccessChainOp>(loc, varOp, index);
+    auto acOp = spirv::AccessChainOp::create(rewriter, loc, varOp, index);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(extractOp, acOp);
 
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 40ad63610e23f..44872c8d7540d 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -51,7 +51,7 @@ TypedAttr getConstantAttr(Type type, int64_t value, PatternRewriter &rewriter) {
 
 Value getConstantValue(Location loc, Type type, int64_t value,
                        PatternRewriter &rewriter) {
-  return rewriter.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(rewriter,
       loc, getConstantAttr(type, value, rewriter));
 }
 
@@ -82,41 +82,41 @@ class ApplyScaleGenericOpConverter
     Value one64 = getConstantValue(loc, i64Ty, 1, rewriter);
     Value thirtyOne32 = getConstantValue(loc, i32Ty, 31, rewriter);
 
-    Value shift32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, op.getShift());
+    Value shift32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, op.getShift());
 
     // Compute the multiplication in 64-bits then select the high / low parts.
     Value value64 = value;
     if (getElementTypeOrSelf(valueTy) != rewriter.getI64Type())
-      value64 = rewriter.create<arith::ExtSIOp>(loc, i64Ty, value);
+      value64 = arith::ExtSIOp::create(rewriter, loc, i64Ty, value);
     Value multiplier64 =
-        rewriter.create<arith::ExtSIOp>(loc, i64Ty, multiplier32);
+        arith::ExtSIOp::create(rewriter, loc, i64Ty, multiplier32);
     Value multiply64 =
-        rewriter.create<arith::MulIOp>(loc, value64, multiplier64);
+        arith::MulIOp::create(rewriter, loc, value64, multiplier64);
 
     // Apply normal rounding.
-    Value shift64 = rewriter.create<arith::ExtUIOp>(loc, i64Ty, shift32);
-    Value round = rewriter.create<arith::ShLIOp>(loc, one64, shift64);
-    round = rewriter.create<arith::ShRUIOp>(loc, round, one64);
-    multiply64 = rewriter.create<arith::AddIOp>(loc, multiply64, round);
+    Value shift64 = arith::ExtUIOp::create(rewriter, loc, i64Ty, shift32);
+    Value round = arith::ShLIOp::create(rewriter, loc, one64, shift64);
+    round = arith::ShRUIOp::create(rewriter, loc, round, one64);
+    multiply64 = arith::AddIOp::create(rewriter, loc, multiply64, round);
 
     // Apply double rounding if necessary.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       int64_t roundInt = 1 << 30;
       Value roundUp = getConstantValue(loc, i64Ty, roundInt, rewriter);
       Value roundDown = getConstantValue(loc, i64Ty, -roundInt, rewriter);
-      Value positive = rewriter.create<arith::CmpIOp>(
+      Value positive = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, value, zero);
       Value dir =
-          rewriter.create<arith::SelectOp>(loc, positive, roundUp, roundDown);
-      Value val = rewriter.create<arith::AddIOp>(loc, dir, multiply64);
-      Value valid = rewriter.create<arith::CmpIOp>(
+          arith::SelectOp::create(rewriter, loc, positive, roundUp, roundDown);
+      Value val = arith::AddIOp::create(rewriter, loc, dir, multiply64);
+      Value valid = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sgt, shift32, thirtyOne32);
       multiply64 =
-          rewriter.create<arith::SelectOp>(loc, valid, val, multiply64);
+          arith::SelectOp::create(rewriter, loc, valid, val, multiply64);
     }
 
-    Value result64 = rewriter.create<arith::ShRSIOp>(loc, multiply64, shift64);
-    Value result32 = rewriter.create<arith::TruncIOp>(loc, i32Ty, result64);
+    Value result64 = arith::ShRSIOp::create(rewriter, loc, multiply64, shift64);
+    Value result32 = arith::TruncIOp::create(rewriter, loc, i32Ty, result64);
 
     rewriter.replaceOp(op, result32);
     return success();
@@ -146,7 +146,7 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
 
     Value value32 = op.getValue();
     Value multiplier32 = op.getMultiplier();
-    Value shift32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, op.getShift());
+    Value shift32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, op.getShift());
 
     // Constants used during the scaling operation.
     Value zero32 = getConstantValue(loc, i32Ty, 0, rewriter);
@@ -158,86 +158,86 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
     // Compute the multiplication in 64-bits then select the high / low parts.
     // Grab out the high/low of the computation
     auto value64 =
-        rewriter.create<arith::MulSIExtendedOp>(loc, value32, multiplier32);
+        arith::MulSIExtendedOp::create(rewriter, loc, value32, multiplier32);
     Value low32 = value64.getLow();
     Value high32 = value64.getHigh();
 
     // Determine the direction and amount to shift the high bits.
-    Value shiftOver32 = rewriter.create<arith::CmpIOp>(
+    Value shiftOver32 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, shift32, thirtyTwo32);
-    Value roundHighBits = rewriter.create<arith::CmpIOp>(
+    Value roundHighBits = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, shift32, thirtyTwo32);
 
     Value shiftHighL =
-        rewriter.create<arith::SubIOp>(loc, thirtyTwo32, shift32);
+        arith::SubIOp::create(rewriter, loc, thirtyTwo32, shift32);
     Value shiftHighR =
-        rewriter.create<arith::SubIOp>(loc, shift32, thirtyTwo32);
+        arith::SubIOp::create(rewriter, loc, shift32, thirtyTwo32);
 
     shiftHighL =
-        rewriter.create<arith::SelectOp>(loc, shiftOver32, zero32, shiftHighL);
+        arith::SelectOp::create(rewriter, loc, shiftOver32, zero32, shiftHighL);
     shiftHighR =
-        rewriter.create<arith::SelectOp>(loc, shiftOver32, shiftHighR, zero32);
+        arith::SelectOp::create(rewriter, loc, shiftOver32, shiftHighR, zero32);
 
     // Conditionally perform our double round.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       Value negOne32 = getConstantValue(loc, i32Ty, -1, rewriter);
-      Value valuePositive = rewriter.create<arith::CmpIOp>(
+      Value valuePositive = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, value32, zero32);
 
       Value roundDir =
-          rewriter.create<arith::SelectOp>(loc, valuePositive, one32, negOne32);
+          arith::SelectOp::create(rewriter, loc, valuePositive, one32, negOne32);
       roundDir =
-          rewriter.create<arith::SelectOp>(loc, shiftOver32, roundDir, zero32);
+          arith::SelectOp::create(rewriter, loc, shiftOver32, roundDir, zero32);
 
-      Value shiftLow = rewriter.create<arith::ShRUIOp>(loc, low32, thirty32);
-      Value rounded = rewriter.create<arith::AddIOp>(loc, shiftLow, roundDir);
-      Value carry = rewriter.create<arith::ShRSIOp>(loc, rounded, two32);
+      Value shiftLow = arith::ShRUIOp::create(rewriter, loc, low32, thirty32);
+      Value rounded = arith::AddIOp::create(rewriter, loc, shiftLow, roundDir);
+      Value carry = arith::ShRSIOp::create(rewriter, loc, rounded, two32);
 
       Value shiftRound =
-          rewriter.create<arith::ShLIOp>(loc, roundDir, thirty32);
+          arith::ShLIOp::create(rewriter, loc, roundDir, thirty32);
 
-      low32 = rewriter.create<arith::AddIOp>(loc, low32, shiftRound);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, carry);
+      low32 = arith::AddIOp::create(rewriter, loc, low32, shiftRound);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, carry);
     }
 
     // Conditionally apply rounding in the low bits.
     {
-      Value shiftSubOne = rewriter.create<arith::SubIOp>(loc, shift32, one32);
-      Value roundBit = rewriter.create<arith::ShLIOp>(loc, one32, shiftSubOne);
-      roundBit = rewriter.create<arith::SelectOp>(loc, roundHighBits, zero32,
+      Value shiftSubOne = arith::SubIOp::create(rewriter, loc, shift32, one32);
+      Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
+      roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, zero32,
                                                   roundBit);
 
-      Value newLow32 = rewriter.create<arith::AddIOp>(loc, low32, roundBit);
-      Value wasRounded = rewriter.create<arith::CmpIOp>(
+      Value newLow32 = arith::AddIOp::create(rewriter, loc, low32, roundBit);
+      Value wasRounded = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ugt, low32, newLow32);
       low32 = newLow32;
 
-      Value rounded32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, wasRounded);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, rounded32);
+      Value rounded32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, wasRounded);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, rounded32);
     }
 
     // Conditionally apply rounding in the high bits.
     {
       Value shiftSubOne =
-          rewriter.create<arith::SubIOp>(loc, shiftHighR, one32);
-      Value roundBit = rewriter.create<arith::ShLIOp>(loc, one32, shiftSubOne);
-      roundBit = rewriter.create<arith::SelectOp>(loc, roundHighBits, roundBit,
+          arith::SubIOp::create(rewriter, loc, shiftHighR, one32);
+      Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
+      roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, roundBit,
                                                   zero32);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, roundBit);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, roundBit);
     }
 
     // Combine the correct high/low bits into the final rescale result.
-    high32 = rewriter.create<arith::ShLIOp>(loc, high32, shiftHighL);
-    high32 = rewriter.create<arith::ShRSIOp>(loc, high32, shiftHighR);
-    low32 = rewriter.create<arith::ShRUIOp>(loc, low32, shift32);
-    low32 = rewriter.create<arith::SelectOp>(loc, shiftOver32, zero32, low32);
+    high32 = arith::ShLIOp::create(rewriter, loc, high32, shiftHighL);
+    high32 = arith::ShRSIOp::create(rewriter, loc, high32, shiftHighR);
+    low32 = arith::ShRUIOp::create(rewriter, loc, low32, shift32);
+    low32 = arith::SelectOp::create(rewriter, loc, shiftOver32, zero32, low32);
 
     // Apply the rounding behavior and shift to the final alignment.
-    Value result = rewriter.create<arith::AddIOp>(loc, low32, high32);
+    Value result = arith::AddIOp::create(rewriter, loc, low32, high32);
 
     // Truncate if necessary.
     if (!getElementTypeOrSelf(resultTy).isInteger(32)) {
-      result = rewriter.create<arith::TruncIOp>(loc, resultTy, result);
+      result = arith::TruncIOp::create(rewriter, loc, resultTy, result);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 2f608bbd637b4..2aec35ed9d8ca 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -70,13 +70,13 @@ materializeBinaryNanCheckIfRequired(OpTy op, PatternRewriter &rewriter,
     return result;
 
   // Unordered comparison of NaN against itself will always return true.
-  Value lhsIsNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsIsNaN = arith::CmpFOp::create(rewriter,
       op.getLoc(), arith::CmpFPredicate::UNO, lhs, lhs);
-  Value rhsIsNaN = rewriter.create<arith::CmpFOp>(
+  Value rhsIsNaN = arith::CmpFOp::create(rewriter,
       op.getLoc(), arith::CmpFPredicate::UNO, rhs, rhs);
   Value rhsOrResult =
-      rewriter.create<arith::SelectOp>(op.getLoc(), lhsIsNaN, rhs, result);
-  return rewriter.create<arith::SelectOp>(op.getLoc(), rhsIsNaN, lhs,
+      arith::SelectOp::create(rewriter, op.getLoc(), lhsIsNaN, rhs, result);
+  return arith::SelectOp::create(rewriter, op.getLoc(), rhsIsNaN, lhs,
                                           rhsOrResult);
 }
 
@@ -89,38 +89,38 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
   // tosa::AbsOp
   if (isa<tosa::AbsOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::AbsFOp>(loc, resultTypes, args);
+    return math::AbsFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AbsOp>(op) && isa<IntegerType>(elementTy)) {
-    auto zero = rewriter.create<arith::ConstantOp>(
+    auto zero = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(elementTy));
-    auto neg = rewriter.create<arith::SubIOp>(loc, zero, args[0]);
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], neg);
+    auto neg = arith::SubIOp::create(rewriter, loc, zero, args[0]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], neg);
   }
 
   // tosa::AddOp
   if (isa<tosa::AddOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::AddFOp>(loc, resultTypes, args);
+    return arith::AddFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AddOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::AddIOp>(loc, resultTypes, args);
+    return arith::AddIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::SubOp
   if (isa<tosa::SubOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::SubFOp>(loc, resultTypes, args);
+    return arith::SubFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::SubOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::SubIOp>(loc, resultTypes, args);
+    return arith::SubIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::IntDivOp
   if (isa<tosa::IntDivOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::DivSIOp>(loc, resultTypes, args);
+    return arith::DivSIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ReciprocalOp
   if (isa<tosa::ReciprocalOp>(op) && isa<FloatType>(elementTy)) {
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, FloatAttr::get(elementTy, 1));
-    return rewriter.create<arith::DivFOp>(loc, resultTypes, one, args[0]);
+        arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
+    return arith::DivFOp::create(rewriter, loc, resultTypes, one, args[0]);
   }
 
   // tosa::MulOp
@@ -140,7 +140,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                                           "Cannot have shift value for float");
         return nullptr;
       }
-      return rewriter.create<arith::MulFOp>(loc, resultTypes, args[0], args[1]);
+      return arith::MulFOp::create(rewriter, loc, resultTypes, args[0], args[1]);
     }
 
     if (isa<IntegerType>(elementTy)) {
@@ -149,21 +149,21 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
       if (shift > 0) {
         auto shiftConst =
-            rewriter.create<arith::ConstantIntOp>(loc, shift, /*bitwidth=*/8);
+            arith::ConstantIntOp::create(rewriter, loc, shift, /*bitwidth=*/8);
         if (!a.getType().isInteger(32))
-          a = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), a);
+          a = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), a);
 
         if (!b.getType().isInteger(32))
-          b = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), b);
+          b = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), b);
 
-        auto result = rewriter.create<tosa::ApplyScaleOp>(
+        auto result = tosa::ApplyScaleOp::create(rewriter,
             loc, rewriter.getI32Type(), a, b, shiftConst,
             rewriter.getStringAttr("SINGLE_ROUND"));
 
         if (elementTy.isInteger(32))
           return result;
 
-        return rewriter.create<arith::TruncIOp>(loc, elementTy, result);
+        return arith::TruncIOp::create(rewriter, loc, elementTy, result);
       }
 
       int aWidth = a.getType().getIntOrFloatBitWidth();
@@ -171,11 +171,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       int cWidth = resultTypes[0].getIntOrFloatBitWidth();
 
       if (aWidth < cWidth)
-        a = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], a);
+        a = arith::ExtSIOp::create(rewriter, loc, resultTypes[0], a);
       if (bWidth < cWidth)
-        b = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], b);
+        b = arith::ExtSIOp::create(rewriter, loc, resultTypes[0], b);
 
-      return rewriter.create<arith::MulIOp>(loc, resultTypes, a, b);
+      return arith::MulIOp::create(rewriter, loc, resultTypes, a, b);
     }
   }
 
@@ -201,13 +201,13 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     int64_t outZp = *maybeOutZp;
 
     if (isa<FloatType>(elementTy))
-      return rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
+      return arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
 
     if (isa<IntegerType>(elementTy)) {
       if (!inZp && !outZp) {
-        auto constant = rewriter.create<arith::ConstantOp>(
+        auto constant = arith::ConstantOp::create(rewriter,
             loc, IntegerAttr::get(elementTy, 0));
-        return rewriter.create<arith::SubIOp>(loc, resultTypes, constant,
+        return arith::SubIOp::create(rewriter, loc, resultTypes, constant,
                                               args[0]);
       }
 
@@ -231,60 +231,60 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       }
 
       Type intermediateType = rewriter.getIntegerType(intermediateBitWidth);
-      Value zpAddValue = rewriter.create<arith::ConstantOp>(
+      Value zpAddValue = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
 
       // The negation can be applied by doing:
       //  outputValue = inZp + outZp - inputValue
       auto ext =
-          rewriter.create<arith::ExtSIOp>(loc, intermediateType, args[0]);
-      auto sub = rewriter.create<arith::SubIOp>(loc, zpAddValue, ext);
+          arith::ExtSIOp::create(rewriter, loc, intermediateType, args[0]);
+      auto sub = arith::SubIOp::create(rewriter, loc, zpAddValue, ext);
 
       // Clamp to the negation range.
-      Value min = rewriter.create<arith::ConstantIntOp>(
+      Value min = arith::ConstantIntOp::create(rewriter,
           loc, intermediateType,
           APInt::getSignedMinValue(inputBitWidth).getSExtValue());
-      Value max = rewriter.create<arith::ConstantIntOp>(
+      Value max = arith::ConstantIntOp::create(rewriter,
           loc, intermediateType,
           APInt::getSignedMaxValue(inputBitWidth).getSExtValue());
       auto clamp = clampIntHelper(loc, sub, min, max, rewriter, false);
 
       // Truncate to the final value.
-      return rewriter.create<arith::TruncIOp>(loc, elementTy, clamp);
+      return arith::TruncIOp::create(rewriter, loc, elementTy, clamp);
     }
   }
 
   // tosa::BitwiseAndOp
   if (isa<tosa::BitwiseAndOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::AndIOp>(loc, resultTypes, args);
+    return arith::AndIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::BitwiseOrOp
   if (isa<tosa::BitwiseOrOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::OrIOp>(loc, resultTypes, args);
+    return arith::OrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::BitwiseNotOp
   if (isa<tosa::BitwiseNotOp>(op) && isa<IntegerType>(elementTy)) {
     auto allOnesAttr = rewriter.getIntegerAttr(
         elementTy, APInt::getAllOnes(elementTy.getIntOrFloatBitWidth()));
-    auto allOnes = rewriter.create<arith::ConstantOp>(loc, allOnesAttr);
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args[0], allOnes);
+    auto allOnes = arith::ConstantOp::create(rewriter, loc, allOnesAttr);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], allOnes);
   }
 
   // tosa::BitwiseXOrOp
   if (isa<tosa::BitwiseXorOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalLeftShiftOp
   if (isa<tosa::LogicalLeftShiftOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::ShLIOp>(loc, resultTypes, args);
+    return arith::ShLIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalRightShiftOp
   if (isa<tosa::LogicalRightShiftOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::ShRUIOp>(loc, resultTypes, args);
+    return arith::ShRUIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ArithmeticRightShiftOp
   if (isa<tosa::ArithmeticRightShiftOp>(op) && isa<IntegerType>(elementTy)) {
-    auto result = rewriter.create<arith::ShRSIOp>(loc, resultTypes, args);
+    auto result = arith::ShRSIOp::create(rewriter, loc, resultTypes, args);
     auto round = cast<BoolAttr>(op->getAttr("round")).getValue();
     if (!round) {
       return result;
@@ -292,153 +292,153 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     Type i1Ty = IntegerType::get(rewriter.getContext(), /*width=*/1);
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(elementTy, 1));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 1));
     auto zero =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(elementTy, 0));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 0));
     auto i1one =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(i1Ty, 1));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(i1Ty, 1));
 
     // Checking that input2 != 0
-    auto shiftValueGreaterThanZero = rewriter.create<arith::CmpIOp>(
+    auto shiftValueGreaterThanZero = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, args[1], zero);
 
     // Checking for the last bit of input1 to be 1
     auto subtract =
-        rewriter.create<arith::SubIOp>(loc, resultTypes, args[1], one);
+        arith::SubIOp::create(rewriter, loc, resultTypes, args[1], one);
     auto shifted =
-        rewriter.create<arith::ShRSIOp>(loc, resultTypes, args[0], subtract)
+        arith::ShRSIOp::create(rewriter, loc, resultTypes, args[0], subtract)
             ->getResults();
-    auto truncated = rewriter.create<arith::TruncIOp>(
+    auto truncated = arith::TruncIOp::create(rewriter,
         loc, i1Ty, shifted, ArrayRef<NamedAttribute>());
     auto isInputOdd =
-        rewriter.create<arith::AndIOp>(loc, i1Ty, truncated, i1one);
+        arith::AndIOp::create(rewriter, loc, i1Ty, truncated, i1one);
 
-    auto shouldRound = rewriter.create<arith::AndIOp>(
+    auto shouldRound = arith::AndIOp::create(rewriter,
         loc, i1Ty, shiftValueGreaterThanZero, isInputOdd);
     auto extended =
-        rewriter.create<arith::ExtUIOp>(loc, resultTypes, shouldRound);
-    return rewriter.create<arith::AddIOp>(loc, resultTypes, result, extended);
+        arith::ExtUIOp::create(rewriter, loc, resultTypes, shouldRound);
+    return arith::AddIOp::create(rewriter, loc, resultTypes, result, extended);
   }
 
   // tosa::ClzOp
   if (isa<tosa::ClzOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<math::CountLeadingZerosOp>(loc, elementTy, args[0]);
+    return math::CountLeadingZerosOp::create(rewriter, loc, elementTy, args[0]);
   }
 
   // tosa::LogicalAnd
   if (isa<tosa::LogicalAndOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::AndIOp>(loc, resultTypes, args);
+    return arith::AndIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalNot
   if (isa<tosa::LogicalNotOp>(op) && elementTy.isInteger(1)) {
-    auto one = rewriter.create<arith::ConstantOp>(
+    auto one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(elementTy, 1));
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args[0], one);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], one);
   }
 
   // tosa::LogicalOr
   if (isa<tosa::LogicalOrOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::OrIOp>(loc, resultTypes, args);
+    return arith::OrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalXor
   if (isa<tosa::LogicalXorOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::PowOp
   if (isa<tosa::PowOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::PowFOp>(loc, resultTypes, args);
+    return mlir::math::PowFOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::RsqrtOp
   if (isa<tosa::RsqrtOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::RsqrtOp>(loc, resultTypes, args);
+    return mlir::math::RsqrtOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogOp
   if (isa<tosa::LogOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::LogOp>(loc, resultTypes, args);
+    return mlir::math::LogOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ExpOp
   if (isa<tosa::ExpOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::ExpOp>(loc, resultTypes, args);
+    return mlir::math::ExpOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::SinOp
   if (isa<tosa::SinOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::SinOp>(loc, resultTypes, args);
+    return mlir::math::SinOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::CosOp
   if (isa<tosa::CosOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::CosOp>(loc, resultTypes, args);
+    return mlir::math::CosOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::TanhOp
   if (isa<tosa::TanhOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::TanhOp>(loc, resultTypes, args);
+    return mlir::math::TanhOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ErfOp
   if (isa<tosa::ErfOp>(op) && llvm::isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::ErfOp>(loc, resultTypes, args);
+    return mlir::math::ErfOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::GreaterOp
   if (isa<tosa::GreaterOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGT,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGT,
                                           args[0], args[1]);
 
   if (isa<tosa::GreaterOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sgt,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sgt,
                                           args[0], args[1]);
 
   // tosa::GreaterEqualOp
   if (isa<tosa::GreaterEqualOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGE,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGE,
                                           args[0], args[1]);
 
   if (isa<tosa::GreaterEqualOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sge,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sge,
                                           args[0], args[1]);
 
   // tosa::EqualOp
   if (isa<tosa::EqualOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OEQ,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OEQ,
                                           args[0], args[1]);
 
   if (isa<tosa::EqualOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                           args[0], args[1]);
 
   // tosa::SelectOp
   if (isa<tosa::SelectOp>(op)) {
     elementTy = cast<ShapedType>(op->getOperand(1).getType()).getElementType();
     if (isa<FloatType>(elementTy) || isa<IntegerType>(elementTy))
-      return rewriter.create<arith::SelectOp>(loc, args[0], args[1], args[2]);
+      return arith::SelectOp::create(rewriter, loc, args[0], args[1], args[2]);
   }
 
   // tosa::MaximumOp
   if (isa<tosa::MaximumOp>(op) && isa<FloatType>(elementTy)) {
-    auto max = rewriter.create<arith::MaximumFOp>(loc, args[0], args[1]);
+    auto max = arith::MaximumFOp::create(rewriter, loc, args[0], args[1]);
     return materializeBinaryNanCheckIfRequired(llvm::cast<tosa::MaximumOp>(op),
                                                rewriter, args[0], args[1], max);
   }
 
   if (isa<tosa::MaximumOp>(op) && elementTy.isSignlessInteger()) {
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], args[1]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   // tosa::MinimumOp
   if (isa<tosa::MinimumOp>(op) && isa<FloatType>(elementTy)) {
-    auto min = rewriter.create<arith::MinimumFOp>(loc, args[0], args[1]);
+    auto min = arith::MinimumFOp::create(rewriter, loc, args[0], args[1]);
     return materializeBinaryNanCheckIfRequired(llvm::cast<tosa::MinimumOp>(op),
                                                rewriter, args[0], args[1], min);
   }
 
   if (isa<tosa::MinimumOp>(op) && elementTy.isSignlessInteger()) {
-    return rewriter.create<arith::MinSIOp>(loc, args[0], args[1]);
+    return arith::MinSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   // tosa::CeilOp
   if (isa<tosa::CeilOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::CeilOp>(loc, resultTypes, args);
+    return math::CeilOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::FloorOp
   if (isa<tosa::FloorOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::FloorOp>(loc, resultTypes, args);
+    return math::FloorOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ClampOp
   if (isa<tosa::ClampOp>(op) && isa<FloatType>(elementTy)) {
@@ -449,9 +449,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                    APFloat::rmNearestTiesToEven, &losesInfo);
     maxApf.convert(cast<FloatType>(elementTy).getFloatSemantics(),
                    APFloat::rmNearestTiesToEven, &losesInfo);
-    auto min = rewriter.create<arith::ConstantOp>(
+    auto min = arith::ConstantOp::create(rewriter,
         loc, elementTy, rewriter.getFloatAttr(elementTy, minApf));
-    auto max = rewriter.create<arith::ConstantOp>(
+    auto max = arith::ConstantOp::create(rewriter,
         loc, elementTy, rewriter.getFloatAttr(elementTy, maxApf));
     auto result = clampFloatHelper(loc, args[0], min, max, rewriter);
 
@@ -478,11 +478,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     //   return init if x == NaN else result
 
     // Unordered comparison of NaN against itself will always return true.
-    Value isNaN = rewriter.create<arith::CmpFOp>(
+    Value isNaN = arith::CmpFOp::create(rewriter,
         op->getLoc(), arith::CmpFPredicate::UNO, args[0], args[0]);
     // TOSA specifies that in "ignore" NaN mode the result is "min" if the input
     // is NaN.
-    return rewriter.create<arith::SelectOp>(op->getLoc(), isNaN, min, result);
+    return arith::SelectOp::create(rewriter, op->getLoc(), isNaN, min, result);
   }
 
   if (isa<tosa::ClampOp>(op) && isa<IntegerType>(elementTy)) {
@@ -515,9 +515,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     min = std::min(min, maxRepresentable);
     max = std::min(max, maxRepresentable);
 
-    auto minVal = rewriter.create<arith::ConstantIntOp>(
+    auto minVal = arith::ConstantIntOp::create(rewriter,
         loc, min, intTy.getIntOrFloatBitWidth());
-    auto maxVal = rewriter.create<arith::ConstantIntOp>(
+    auto maxVal = arith::ConstantIntOp::create(rewriter,
         loc, max, intTy.getIntOrFloatBitWidth());
     return clampIntHelper(loc, args[0], minVal, maxVal, rewriter,
                           intTy.isUnsignedInteger());
@@ -526,11 +526,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   // tosa::SigmoidOp
   if (isa<tosa::SigmoidOp>(op) && isa<FloatType>(elementTy)) {
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, FloatAttr::get(elementTy, 1));
-    auto negate = rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
-    auto exp = rewriter.create<mlir::math::ExpOp>(loc, resultTypes, negate);
-    auto added = rewriter.create<arith::AddFOp>(loc, resultTypes, exp, one);
-    return rewriter.create<arith::DivFOp>(loc, resultTypes, one, added);
+        arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
+    auto negate = arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
+    auto exp = mlir::math::ExpOp::create(rewriter, loc, resultTypes, negate);
+    auto added = arith::AddFOp::create(rewriter, loc, resultTypes, exp, one);
+    return arith::DivFOp::create(rewriter, loc, resultTypes, one, added);
   }
 
   // tosa::CastOp
@@ -549,20 +549,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       return args.front();
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtFOp>(loc, resultTypes, args,
+      return arith::ExtFOp::create(rewriter, loc, resultTypes, args,
                                             ArrayRef<NamedAttribute>());
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && !bitExtend)
-      return rewriter.create<arith::TruncFOp>(loc, resultTypes, args,
+      return arith::TruncFOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     // 1-bit integers need to be treated as signless.
     if (srcTy.isInteger(1) && arith::UIToFPOp::areCastCompatible(srcTy, dstTy))
-      return rewriter.create<arith::UIToFPOp>(loc, resultTypes, args,
+      return arith::UIToFPOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     if (srcTy.isInteger(1) && isa<IntegerType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtUIOp>(loc, resultTypes, args,
+      return arith::ExtUIOp::create(rewriter, loc, resultTypes, args,
                                              ArrayRef<NamedAttribute>());
 
     // Unsigned integers need an unrealized cast so that they can be passed
@@ -574,25 +574,25 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                   loc, rewriter.getIntegerType(srcTy.getIntOrFloatBitWidth()),
                   args[0])
               .getResult(0);
-      return rewriter.create<arith::UIToFPOp>(loc, resultTypes[0],
+      return arith::UIToFPOp::create(rewriter, loc, resultTypes[0],
                                               unrealizedCast);
     }
 
     // All other si-to-fp conversions should be handled by SIToFP.
     if (arith::SIToFPOp::areCastCompatible(srcTy, dstTy))
-      return rewriter.create<arith::SIToFPOp>(loc, resultTypes, args,
+      return arith::SIToFPOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     // Casting to boolean, floats need to only be checked as not-equal to zero.
     if (isa<FloatType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = rewriter.create<arith::ConstantOp>(
+      Value zero = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(srcTy, 0.0));
-      return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNE,
+      return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNE,
                                             args.front(), zero);
     }
 
     if (arith::FPToSIOp::areCastCompatible(srcTy, dstTy)) {
-      auto rounded = rewriter.create<math::RoundEvenOp>(loc, args[0]);
+      auto rounded = math::RoundEvenOp::create(rewriter, loc, args[0]);
 
       const auto &fltSemantics = cast<FloatType>(srcTy).getFloatSemantics();
       // Check whether neither int min nor int max can be represented in the
@@ -601,33 +601,33 @@ static Value createLinalgBodyCalculationForElementwiseOp(
           APFloat::semanticsMaxExponent(fltSemantics)) {
         // Use cmp + select to replace infinites by int min / int max. Other
         // integral values can be represented in the integer space.
-        auto conv = rewriter.create<arith::FPToSIOp>(loc, dstTy, rounded);
-        auto posInf = rewriter.create<arith::ConstantOp>(
+        auto conv = arith::FPToSIOp::create(rewriter, loc, dstTy, rounded);
+        auto posInf = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(getElementTypeOrSelf(srcTy),
                                        APFloat::getInf(fltSemantics)));
-        auto negInf = rewriter.create<arith::ConstantOp>(
+        auto negInf = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(
                      getElementTypeOrSelf(srcTy),
                      APFloat::getInf(fltSemantics, /*Negative=*/true)));
-        auto overflow = rewriter.create<arith::CmpFOp>(
+        auto overflow = arith::CmpFOp::create(rewriter,
             loc, arith::CmpFPredicate::UEQ, rounded, posInf);
-        auto underflow = rewriter.create<arith::CmpFOp>(
+        auto underflow = arith::CmpFOp::create(rewriter,
             loc, arith::CmpFPredicate::UEQ, rounded, negInf);
-        auto intMin = rewriter.create<arith::ConstantOp>(
+        auto intMin = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIntegerAttr(
                      getElementTypeOrSelf(dstTy),
                      APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())));
-        auto intMax = rewriter.create<arith::ConstantOp>(
+        auto intMax = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIntegerAttr(
                      getElementTypeOrSelf(dstTy),
                      APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
         auto maxClamped =
-            rewriter.create<arith::SelectOp>(loc, overflow, intMax, conv);
-        return rewriter.create<arith::SelectOp>(loc, underflow, intMin,
+            arith::SelectOp::create(rewriter, loc, overflow, intMax, conv);
+        return arith::SelectOp::create(rewriter, loc, underflow, intMin,
                                                 maxClamped);
       }
 
-      auto intMinFP = rewriter.create<arith::ConstantOp>(
+      auto intMinFP = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
                    APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())
@@ -640,7 +640,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         // consists of a single leading bit. Therefore we can clamp the input
         // in the floating-point domain.
 
-        auto intMaxFP = rewriter.create<arith::ConstantOp>(
+        auto intMaxFP = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(
                      getElementTypeOrSelf(srcTy),
                      APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
@@ -648,14 +648,14 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
         Value clamped =
             clampFloatHelper(loc, rounded, intMinFP, intMaxFP, rewriter);
-        return rewriter.create<arith::FPToSIOp>(loc, dstTy, clamped);
+        return arith::FPToSIOp::create(rewriter, loc, dstTy, clamped);
       }
 
       // Due to earlier check we know exponant range is big enough to represent
       // int min. We can therefore rely on int max + 1 being representable as
       // well because it's just int min with a positive sign. So clamp the min
       // value and compare against that to select the max int value if needed.
-      auto intMaxPlusOneFP = rewriter.create<arith::ConstantOp>(
+      auto intMaxPlusOneFP = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
                    static_cast<double>(
@@ -663,35 +663,35 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                            .getSExtValue()) +
                        1.0f));
 
-      auto intMax = rewriter.create<arith::ConstantOp>(
+      auto intMax = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIntegerAttr(
                    getElementTypeOrSelf(dstTy),
                    APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
       auto minClampedFP =
-          rewriter.create<arith::MaximumFOp>(loc, rounded, intMinFP);
+          arith::MaximumFOp::create(rewriter, loc, rounded, intMinFP);
       auto minClamped =
-          rewriter.create<arith::FPToSIOp>(loc, dstTy, minClampedFP);
-      auto overflow = rewriter.create<arith::CmpFOp>(
+          arith::FPToSIOp::create(rewriter, loc, dstTy, minClampedFP);
+      auto overflow = arith::CmpFOp::create(rewriter,
           loc, arith::CmpFPredicate::UGE, rounded, intMaxPlusOneFP);
-      return rewriter.create<arith::SelectOp>(loc, overflow, intMax,
+      return arith::SelectOp::create(rewriter, loc, overflow, intMax,
                                               minClamped);
     }
 
     // Casting to boolean, integers need to only be checked as not-equal to
     // zero.
     if (isa<IntegerType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = rewriter.create<arith::ConstantIntOp>(
+      Value zero = arith::ConstantIntOp::create(rewriter,
           loc, 0, srcTy.getIntOrFloatBitWidth());
-      return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+      return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
                                             args.front(), zero);
     }
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtSIOp>(loc, resultTypes, args,
+      return arith::ExtSIOp::create(rewriter, loc, resultTypes, args,
                                              ArrayRef<NamedAttribute>());
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && !bitExtend) {
-      return rewriter.create<arith::TruncIOp>(loc, dstTy, args[0]);
+      return arith::TruncIOp::create(rewriter, loc, dstTy, args[0]);
     }
   }
 
@@ -710,14 +710,14 @@ static Value createIndex(PatternRewriter &rewriter, Location loc,
   auto [it, inserted] = indexPool.try_emplace(index);
   if (inserted)
     it->second =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(index));
   return it->second;
 }
 
 static Value getTensorDim(PatternRewriter &rewriter, Location loc,
                           IndexPool &indexPool, Value tensor, int64_t index) {
   auto indexValue = createIndex(rewriter, loc, indexPool, index);
-  return rewriter.create<tensor::DimOp>(loc, tensor, indexValue).getResult();
+  return tensor::DimOp::create(rewriter, loc, tensor, indexValue).getResult();
 }
 
 static OpFoldResult getOrFoldTensorDim(PatternRewriter &rewriter, Location loc,
@@ -783,7 +783,7 @@ computeTargetSize(PatternRewriter &rewriter, Location loc, IndexPool &indexPool,
   for (size_t i = 1; i < operandsWithDynamicDim.size(); i++) {
     auto nextSize =
         getTensorDim(rewriter, loc, indexPool, operandsWithDynamicDim[i], dim);
-    targetSize = rewriter.create<arith::MaxUIOp>(loc, targetSize, nextSize);
+    targetSize = arith::MaxUIOp::create(rewriter, loc, targetSize, nextSize);
   }
   return {targetSize, nullptr};
 }
@@ -838,7 +838,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
   // Check if broadcast is necessary
   auto one = createIndex(rewriter, loc, indexPool, 1);
   auto runtimeSize = getTensorDim(rewriter, loc, indexPool, operand, dim);
-  auto broadcastNecessary = rewriter.create<arith::CmpIOp>(
+  auto broadcastNecessary = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::eq, runtimeSize, one);
 
   // Emit 'then' region of 'scf.if'
@@ -855,7 +855,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                                                     operand, index);
       outputTensorShape.push_back(size);
     }
-    Value outputTensor = opBuilder.create<tensor::EmptyOp>(
+    Value outputTensor = tensor::EmptyOp::create(opBuilder,
         loc, outputTensorShape, rankedTensorType.getElementType());
 
     // Emit 'linalg.generic' op
@@ -866,7 +866,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                 getNParallelLoopsAttrs(rank),
                 [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
                   // Emit 'linalg.yield' op
-                  opBuilder.create<linalg::YieldOp>(loc, blockArgs.front());
+                  linalg::YieldOp::create(opBuilder, loc, blockArgs.front());
                 })
             .getResult(0);
 
@@ -875,16 +875,16 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
         loc, operand.getType(), resultTensor);
 
     // Emit 'scf.yield' op
-    opBuilder.create<scf::YieldOp>(loc, castResultTensor);
+    scf::YieldOp::create(opBuilder, loc, castResultTensor);
   };
 
   // Emit 'else' region of 'scf.if'
   auto emitElseRegion = [&](OpBuilder &opBuilder, Location loc) {
-    opBuilder.create<scf::YieldOp>(loc, operand);
+    scf::YieldOp::create(opBuilder, loc, operand);
   };
 
   // Emit 'scf.if' op
-  auto ifOp = rewriter.create<scf::IfOp>(loc, broadcastNecessary,
+  auto ifOp = scf::IfOp::create(rewriter, loc, broadcastNecessary,
                                          emitThenRegion, emitElseRegion);
   return ifOp.getResult(0);
 }
@@ -930,7 +930,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
   if (!resultType) {
     return rewriter.notifyMatchFailure(operation, "failed to convert type");
   }
-  Value outputTensor = rewriter.create<tensor::EmptyOp>(
+  Value outputTensor = tensor::EmptyOp::create(rewriter,
       loc, targetShape, resultType.getElementType());
 
   // Create affine maps. Input affine maps broadcast static dimensions of size
@@ -957,7 +957,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
 
   // Emit 'linalg.generic' op
   bool encounteredError = false;
-  auto linalgOp = rewriter.create<linalg::GenericOp>(
+  auto linalgOp = linalg::GenericOp::create(rewriter,
       loc, outputTensor.getType(), operands, outputTensor, affineMaps,
       getNParallelLoopsAttrs(rank),
       [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
@@ -968,7 +968,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
           encounteredError = true;
           return;
         }
-        opBuilder.create<linalg::YieldOp>(loc, opResult);
+        linalg::YieldOp::create(opBuilder, loc, opResult);
       });
   if (encounteredError)
     return rewriter.notifyMatchFailure(
@@ -1078,42 +1078,42 @@ static Value createLinalgBodyCalculationForReduceOp(Operation *op,
                                                     PatternRewriter &rewriter) {
   Location loc = op->getLoc();
   if (isa<tosa::ReduceSumOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::AddFOp>(loc, args);
+    return arith::AddFOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceSumOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::AddIOp>(loc, args);
+    return arith::AddIOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceProductOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MulFOp>(loc, args);
+    return arith::MulFOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceProductOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MulIOp>(loc, args);
+    return arith::MulIOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceMinOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MinimumFOp>(loc, args[0], args[1]);
+    return arith::MinimumFOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMinOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MinSIOp>(loc, args[0], args[1]);
+    return arith::MinSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMaxOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MaximumFOp>(loc, args[0], args[1]);
+    return arith::MaximumFOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMaxOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], args[1]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceAllOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::AndIOp>(loc, args);
+    return arith::AndIOp::create(rewriter, loc, args);
 
   if (isa<tosa::ReduceAnyOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::OrIOp>(loc, args);
+    return arith::OrIOp::create(rewriter, loc, args);
 
   return {};
 }
@@ -1139,7 +1139,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     if (axis != i) {
       reduceShape.push_back(inputTy.getDimSize(i));
       if (inputTy.isDynamicDim(i))
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
     }
   }
 
@@ -1158,7 +1158,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     return rewriter.notifyMatchFailure(
         op, "No initial value found for reduction operation");
 
-  auto fillValue = rewriter.create<arith::ConstantOp>(loc, fillValueAttr);
+  auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
   auto filledTensor = rewriter
                           .create<linalg::FillOp>(loc, ValueRange{fillValue},
                                                   ValueRange{emptyTensor})
@@ -1176,7 +1176,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
       // Additionally we have to keep track of whether we've seen any non-NaN
       // values and then do a final select based on this predicate.
       auto trueAttr = rewriter.getBoolAttr(true);
-      auto trueValue = rewriter.create<arith::ConstantOp>(loc, trueAttr);
+      auto trueValue = arith::ConstantOp::create(rewriter, loc, trueAttr);
       auto emptyBoolTensor =
           rewriter
               .create<tensor::EmptyOp>(loc, reduceShape, trueValue.getType(),
@@ -1202,7 +1202,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
   }
 
   bool didEncounterError = false;
-  linalg::LinalgOp linalgOp = rewriter.create<linalg::ReduceOp>(
+  linalg::LinalgOp linalgOp = linalg::ReduceOp::create(rewriter,
       loc, inputs, outputs, axis,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange blockArgs) {
         std::array<Value, 2> binaryArgs{
@@ -1219,21 +1219,21 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
           auto oldAllResultsNanFlagValue = blockArgs[3];
 
           // Unordered comparison of NaN against itself will always return true.
-          Value isNaN = nestedBuilder.create<arith::CmpFOp>(
+          Value isNaN = arith::CmpFOp::create(nestedBuilder,
               op->getLoc(), arith::CmpFPredicate::UNO, inputValue, inputValue);
           // If we've encountered a NaN, take the non-NaN value.
-          auto selectOp = nestedBuilder.create<arith::SelectOp>(
+          auto selectOp = arith::SelectOp::create(nestedBuilder,
               op->getLoc(), isNaN, initialValue, result);
           // Update the flag which keeps track of whether we have seen a non-NaN
           // value.
-          auto newAllResultsNanFlagValue = nestedBuilder.create<arith::AndIOp>(
+          auto newAllResultsNanFlagValue = arith::AndIOp::create(nestedBuilder,
               op->getLoc(), oldAllResultsNanFlagValue, isNaN);
           resultsToYield.push_back(selectOp);
           resultsToYield.push_back(newAllResultsNanFlagValue);
         } else {
           resultsToYield.push_back(result);
         }
-        nestedBuilder.create<linalg::YieldOp>(loc, resultsToYield);
+        linalg::YieldOp::create(nestedBuilder, loc, resultsToYield);
       });
 
   if (!didEncounterError)
@@ -1250,7 +1250,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     auto nanValueAttr = rewriter.getFloatAttr(
         elementTy,
         APFloat::getNaN(cast<FloatType>(elementTy).getFloatSemantics(), false));
-    auto nanValue = rewriter.create<arith::ConstantOp>(loc, nanValueAttr);
+    auto nanValue = arith::ConstantOp::create(rewriter, loc, nanValueAttr);
     auto emptyNanTensor =
         rewriter
             .create<tensor::EmptyOp>(loc, reduceShape,
@@ -1278,7 +1278,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     ins.push_back(linalgOp->getResult(0));
     outs.push_back(finalEmptyTensor);
     auto linalgSelect =
-        rewriter.create<linalg::SelectOp>(op->getLoc(), ins, outs);
+        linalg::SelectOp::create(rewriter, op->getLoc(), ins, outs);
     linalgOp = linalgSelect;
   }
 
@@ -1350,7 +1350,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < outputTy.getRank(); i++) {
       if (outputTy.isDynamicDim(i)) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
@@ -1401,7 +1401,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value multiplierConstant;
     int64_t multiplierArg = 0;
     if (multiplierValues.size() == 1) {
-      multiplierConstant = rewriter.create<arith::ConstantOp>(
+      multiplierConstant = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32IntegerAttr(multiplierValues.front()));
     } else {
       SmallVector<AffineExpr, 2> multiplierExprs{
@@ -1409,7 +1409,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
       auto multiplierType =
           RankedTensorType::get({static_cast<int64_t>(multiplierValues.size())},
                                 rewriter.getI32Type());
-      genericInputs.push_back(rewriter.create<arith::ConstantOp>(
+      genericInputs.push_back(arith::ConstantOp::create(rewriter,
           loc, DenseIntElementsAttr::get(multiplierType, multiplierValues)));
 
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
@@ -1424,7 +1424,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value shiftConstant;
     int64_t shiftArg = 0;
     if (shiftValues.size() == 1) {
-      shiftConstant = rewriter.create<arith::ConstantOp>(
+      shiftConstant = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI8IntegerAttr(shiftValues.front()));
     } else {
       SmallVector<AffineExpr, 2> shiftExprs = {
@@ -1432,7 +1432,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
       auto shiftType =
           RankedTensorType::get({static_cast<int64_t>(shiftValues.size())},
                                 rewriter.getIntegerType(8));
-      genericInputs.push_back(rewriter.create<arith::ConstantOp>(
+      genericInputs.push_back(arith::ConstantOp::create(rewriter,
           loc, DenseIntElementsAttr::get(shiftType, shiftValues)));
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
                                             /*symbolCount=*/0, shiftExprs,
@@ -1444,11 +1444,11 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(rank));
 
     // Construct the indexing maps needed for linalg.generic ops.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, outputTy.getShape(), outputTy.getElementType(),
         ArrayRef<Value>({dynDims}));
 
-    auto linalgOp = rewriter.create<linalg::GenericOp>(
+    auto linalgOp = linalg::GenericOp::create(rewriter,
         loc, outputTy, genericInputs, ValueRange{emptyTensor}, indexingMaps,
         getNParallelLoopsAttrs(rank),
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
@@ -1466,7 +1466,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           const int32_t inBitwidth = valueTy.getIntOrFloatBitWidth();
           // Extend zeropoint for sub-32bits widths.
           const int32_t inAttrBitwidth = inBitwidth > 32 ? inBitwidth : 32;
-          auto inputZp = nestedBuilder.create<arith::ConstantOp>(
+          auto inputZp = arith::ConstantOp::create(nestedBuilder,
               loc, IntegerAttr::get(rewriter.getIntegerType(inAttrBitwidth),
                                     *maybeIZp));
 
@@ -1482,7 +1482,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           unsigned outBitWidth = outIntType.getWidth();
           const int32_t outAttrBitwidth = 32;
           assert(outBitWidth <= 32 && "Unexpected output zeropoint bitwidth");
-          auto outputZp = nestedBuilder.create<arith::ConstantOp>(
+          auto outputZp = arith::ConstantOp::create(nestedBuilder,
               loc, IntegerAttr::get(rewriter.getIntegerType(outAttrBitwidth),
                                     *maybeOZp));
 
@@ -1501,24 +1501,24 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           }
           if (valueTy.getIntOrFloatBitWidth() < 32) {
             if (op.getInputUnsigned()) {
-              value = nestedBuilder.create<arith::ExtUIOp>(
+              value = arith::ExtUIOp::create(nestedBuilder,
                   nestedLoc, nestedBuilder.getI32Type(), value);
             } else {
-              value = nestedBuilder.create<arith::ExtSIOp>(
+              value = arith::ExtSIOp::create(nestedBuilder,
                   nestedLoc, nestedBuilder.getI32Type(), value);
             }
           }
 
           value =
-              nestedBuilder.create<arith::SubIOp>(nestedLoc, value, inputZp);
+              arith::SubIOp::create(nestedBuilder, nestedLoc, value, inputZp);
 
-          value = nestedBuilder.create<tosa::ApplyScaleOp>(
+          value = tosa::ApplyScaleOp::create(nestedBuilder,
               loc, nestedBuilder.getI32Type(), value, multiplier, shift,
               roundingMode);
 
           // Move to the new zero-point.
           value =
-              nestedBuilder.create<arith::AddIOp>(nestedLoc, value, outputZp);
+              arith::AddIOp::create(nestedBuilder, nestedLoc, value, outputZp);
 
           // Saturate to the output size.
           int32_t intMin = APInt::getSignedMinValue(outBitWidth).getSExtValue();
@@ -1530,16 +1530,16 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
             intMax = APInt::getMaxValue(outBitWidth).getZExtValue();
           }
 
-          auto intMinVal = nestedBuilder.create<arith::ConstantOp>(
+          auto intMinVal = arith::ConstantOp::create(nestedBuilder,
               loc, nestedBuilder.getI32IntegerAttr(intMin));
-          auto intMaxVal = nestedBuilder.create<arith::ConstantOp>(
+          auto intMaxVal = arith::ConstantOp::create(nestedBuilder,
               loc, nestedBuilder.getI32IntegerAttr(intMax));
 
           value = clampIntHelper(nestedLoc, value, intMinVal, intMaxVal,
                                  nestedBuilder, /*isUnsigned=*/false);
 
           if (outIntType.getWidth() < 32) {
-            value = nestedBuilder.create<arith::TruncIOp>(
+            value = arith::TruncIOp::create(nestedBuilder,
                 nestedLoc, rewriter.getIntegerType(outIntType.getWidth()),
                 value);
           }
@@ -1550,7 +1550,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
                                                             outIntType, value)
                         .getResult(0);
           }
-          nestedBuilder.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(nestedBuilder, loc, value);
         });
 
     rewriter.replaceOp(op, linalgOp->getResults());
@@ -1608,25 +1608,25 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
     auto collapseTy =
         RankedTensorType::get({inputTy.getDimSize(0), inputTy.getDimSize(3)},
                               inputTy.getElementType());
-    Value collapse = builder.create<tensor::CollapseShapeOp>(collapseTy, input,
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, input,
                                                              reassociationMap);
 
     // Get any dynamic shapes that appear in the input format.
     llvm::SmallVector<Value> outputDynSize;
     if (inputTy.isDynamicDim(0))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 0));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 0));
     if (inputTy.isDynamicDim(3))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 3));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 3));
 
     // Generate the elementwise operation for casting scaling the input value.
     auto genericTy = collapseTy.clone(resultTy.getElementType());
-    Value empty = builder.create<tensor::EmptyOp>(
+    Value empty = tensor::EmptyOp::create(builder,
         genericTy.getShape(), resultTy.getElementType(), outputDynSize);
     auto genericMap = rewriter.getMultiDimIdentityMap(genericTy.getRank());
     SmallVector<utils::IteratorType> iterators(genericTy.getRank(),
                                                utils::IteratorType::parallel);
 
-    auto generic = builder.create<linalg::GenericOp>(
+    auto generic = linalg::GenericOp::create(builder,
         genericTy, ValueRange{collapse}, ValueRange{empty},
         ArrayRef<AffineMap>{genericMap, genericMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
@@ -1634,22 +1634,22 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
           // This is the quantized case.
           if (inputTy.getElementType() != resultTy.getElementType()) {
             value =
-                b.create<arith::ExtSIOp>(loc, resultTy.getElementType(), value);
+                arith::ExtSIOp::create(b, loc, resultTy.getElementType(), value);
 
             if (isBilinear && scale[0] != 0) {
-              Value scaleY = b.create<arith::ConstantOp>(
+              Value scaleY = arith::ConstantOp::create(b,
                   loc, b.getI32IntegerAttr(scale[0]));
-              value = b.create<arith::MulIOp>(loc, value, scaleY);
+              value = arith::MulIOp::create(b, loc, value, scaleY);
             }
 
             if (isBilinear && scale[2] != 0) {
-              Value scaleX = b.create<arith::ConstantOp>(
+              Value scaleX = arith::ConstantOp::create(b,
                   loc, b.getI32IntegerAttr(scale[2]));
-              value = b.create<arith::MulIOp>(loc, value, scaleX);
+              value = arith::MulIOp::create(b, loc, value, scaleX);
             }
           }
 
-          b.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(b, loc, value);
         });
 
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -1697,7 +1697,7 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     resizeShape.push_back(channels);
 
     auto resizeTy = resultTy.clone(resizeShape);
-    auto resize = builder.create<tosa::ResizeOp>(resizeTy, input, op.getScale(),
+    auto resize = tosa::ResizeOp::create(builder, resizeTy, input, op.getScale(),
                                                  op.getOffset(), op.getBorder(),
                                                  op.getMode());
 
@@ -1720,19 +1720,19 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     collapseShape.push_back(channels);
 
     auto collapseTy = resultTy.clone(collapseShape);
-    Value collapse = builder.create<tensor::CollapseShapeOp>(collapseTy, resize,
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, resize,
                                                              reassociationMap);
 
     // Broadcast the collapsed shape to the output result.
     llvm::SmallVector<Value> outputDynSize;
     if (inputTy.isDynamicDim(0))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 0));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 0));
     if (inputTy.isDynamicDim(3))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 3));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 3));
 
     SmallVector<utils::IteratorType> iterators(resultTy.getRank(),
                                                utils::IteratorType::parallel);
-    Value empty = builder.create<tensor::EmptyOp>(
+    Value empty = tensor::EmptyOp::create(builder,
         resultTy.getShape(), resultTy.getElementType(), outputDynSize);
 
     SmallVector<AffineExpr, 4> inputExprs{rewriter.getAffineDimExpr(0)};
@@ -1751,7 +1751,7 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
         ArrayRef<AffineMap>{inputMap, outputMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
           Value value = args[0];
-          b.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(b, loc, value);
         });
 
     return success();
@@ -1789,9 +1789,9 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
 
     SmallVector<AffineMap, 2> affineMaps = {
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
-    auto emptyTensor = b.create<tensor::EmptyOp>(resultTy.getShape(), resultETy,
+    auto emptyTensor = tensor::EmptyOp::create(b, resultTy.getShape(), resultETy,
                                                  *dynamicDimsOr);
-    auto genericOp = b.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(b,
         resultTy, ValueRange({}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     Value resize = genericOp.getResult(0);
@@ -1800,19 +1800,19 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       OpBuilder::InsertionGuard regionGuard(b);
       b.createBlock(&genericOp.getRegion(), genericOp.getRegion().end(),
                     TypeRange({resultETy}), loc);
-      Value batch = b.create<linalg::IndexOp>(0);
-      Value y = b.create<linalg::IndexOp>(1);
-      Value x = b.create<linalg::IndexOp>(2);
-      Value channel = b.create<linalg::IndexOp>(3);
+      Value batch = linalg::IndexOp::create(b, 0);
+      Value y = linalg::IndexOp::create(b, 1);
+      Value x = linalg::IndexOp::create(b, 2);
+      Value channel = linalg::IndexOp::create(b, 3);
 
       Value zeroI32 =
-          b.create<arith::ConstantOp>(b.getZeroAttr(b.getI32Type()));
-      Value zeroFp = b.create<arith::ConstantOp>(b.getZeroAttr(floatTy));
-      Value hMax = b.create<arith::ConstantOp>(b.getI32IntegerAttr(imageH - 1));
-      Value wMax = b.create<arith::ConstantOp>(b.getI32IntegerAttr(imageW - 1));
+          arith::ConstantOp::create(b, b.getZeroAttr(b.getI32Type()));
+      Value zeroFp = arith::ConstantOp::create(b, b.getZeroAttr(floatTy));
+      Value hMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageH - 1));
+      Value wMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageW - 1));
 
-      Value inY = b.create<arith::IndexCastOp>(b.getI32Type(), y);
-      Value inX = b.create<arith::IndexCastOp>(b.getI32Type(), x);
+      Value inY = arith::IndexCastOp::create(b, b.getI32Type(), y);
+      Value inX = arith::IndexCastOp::create(b, b.getI32Type(), x);
 
       SmallVector<int64_t> scale, offset, border;
       if (!tosa::getConstShapeValues(op.getScale().getDefiningOp(), scale) ||
@@ -1824,16 +1824,16 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       }
 
       Value yScaleN, yScaleD, xScaleN, xScaleD;
-      yScaleN = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[0]));
-      yScaleD = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[1]));
-      xScaleN = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[2]));
-      xScaleD = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[3]));
+      yScaleN = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[0]));
+      yScaleD = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[1]));
+      xScaleN = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[2]));
+      xScaleD = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[3]));
 
       Value yOffset, xOffset, yBorder, xBorder;
-      yOffset = b.create<arith::ConstantOp>(b.getI32IntegerAttr(offset[0]));
-      xOffset = b.create<arith::ConstantOp>(b.getI32IntegerAttr(offset[1]));
-      yBorder = b.create<arith::ConstantOp>(b.getI32IntegerAttr(border[0]));
-      xBorder = b.create<arith::ConstantOp>(b.getI32IntegerAttr(border[1]));
+      yOffset = arith::ConstantOp::create(b, b.getI32IntegerAttr(offset[0]));
+      xOffset = arith::ConstantOp::create(b, b.getI32IntegerAttr(offset[1]));
+      yBorder = arith::ConstantOp::create(b, b.getI32IntegerAttr(border[0]));
+      xBorder = arith::ConstantOp::create(b, b.getI32IntegerAttr(border[1]));
 
       // Compute the ix and dx values for both the X and Y dimensions.
       auto getIndexAndDeltaFp = [&](Value &index, Value &delta, Value in,
@@ -1846,16 +1846,16 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         }
         // x = x * scale_d + offset;
         // ix = floor(x / scale_n)
-        Value val = b.create<arith::MulIOp>(in, scaleD);
-        val = b.create<arith::AddIOp>(val, offset);
-        index = b.create<arith::FloorDivSIOp>(val, scaleN);
+        Value val = arith::MulIOp::create(b, in, scaleD);
+        val = arith::AddIOp::create(b, val, offset);
+        index = arith::FloorDivSIOp::create(b, val, scaleN);
 
         // rx = x % scale_n
         // dx = rx / scale_n
-        Value r = b.create<arith::RemSIOp>(val, scaleN);
-        Value rFp = b.create<arith::SIToFPOp>(floatTy, r);
-        Value scaleNfp = b.create<arith::UIToFPOp>(floatTy, scaleN);
-        delta = b.create<arith::DivFOp>(rFp, scaleNfp);
+        Value r = arith::RemSIOp::create(b, val, scaleN);
+        Value rFp = arith::SIToFPOp::create(b, floatTy, r);
+        Value scaleNfp = arith::UIToFPOp::create(b, floatTy, scaleN);
+        delta = arith::DivFOp::create(b, rFp, scaleNfp);
       };
 
       // Compute the ix and dx values for the X and Y dimensions - int case.
@@ -1870,11 +1870,11 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         // x = x * scale_d + offset;
         // ix = floor(x / scale_n)
         //  dx = x - ix * scale_n;
-        Value val = b.create<arith::MulIOp>(in, scaleD);
-        val = b.create<arith::AddIOp>(val, offset);
-        index = b.create<arith::DivSIOp>(val, scaleN);
-        delta = b.create<arith::MulIOp>(index, scaleN);
-        delta = b.create<arith::SubIOp>(val, delta);
+        Value val = arith::MulIOp::create(b, in, scaleD);
+        val = arith::AddIOp::create(b, val, offset);
+        index = arith::DivSIOp::create(b, val, scaleN);
+        delta = arith::MulIOp::create(b, index, scaleN);
+        delta = arith::SubIOp::create(b, val, delta);
       };
 
       Value ix, iy, dx, dy;
@@ -1887,54 +1887,54 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       }
 
       if (op.getMode() == "NEAREST_NEIGHBOR") {
-        auto one = b.create<arith::ConstantOp>(b.getI32IntegerAttr(1));
+        auto one = arith::ConstantOp::create(b, b.getI32IntegerAttr(1));
 
         auto getNearestIndexAndClamp = [&](Value val, Value dval, Value scale,
                                            Value max, int size,
                                            ImplicitLocOpBuilder &b) -> Value {
           if (size == 1) {
-            return b.create<arith::ConstantIndexOp>(0);
+            return arith::ConstantIndexOp::create(b, 0);
           }
 
           Value pred;
           if (floatingPointMode) {
-            auto h = b.create<arith::ConstantOp>(b.getFloatAttr(floatTy, 0.5f));
-            pred = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, dval, h);
+            auto h = arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 0.5f));
+            pred = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, dval, h);
           } else {
-            Value dvalDouble = b.create<arith::ShLIOp>(dval, one);
-            pred = b.create<arith::CmpIOp>(arith::CmpIPredicate::sge,
+            Value dvalDouble = arith::ShLIOp::create(b, dval, one);
+            pred = arith::CmpIOp::create(b, arith::CmpIPredicate::sge,
                                            dvalDouble, scale);
           }
 
-          auto offset = b.create<arith::SelectOp>(pred, one, zeroI32);
-          val = b.create<arith::AddIOp>(val, offset);
+          auto offset = arith::SelectOp::create(b, pred, one, zeroI32);
+          val = arith::AddIOp::create(b, val, offset);
           val = clampIntHelper(loc, val, zeroI32, max, b, /*isUnsigned=*/false);
-          return b.create<arith::IndexCastOp>(b.getIndexType(), val);
+          return arith::IndexCastOp::create(b, b.getIndexType(), val);
         };
 
         iy = getNearestIndexAndClamp(iy, dy, yScaleN, hMax, imageH, b);
         ix = getNearestIndexAndClamp(ix, dx, xScaleN, wMax, imageW, b);
 
-        Value result = b.create<tensor::ExtractOp>(
+        Value result = tensor::ExtractOp::create(b,
             input, ValueRange{batch, iy, ix, channel});
 
-        b.create<linalg::YieldOp>(result);
+        linalg::YieldOp::create(b, result);
       } else {
         // The mode here must be BILINEAR.
         assert(op.getMode() == "BILINEAR");
 
-        auto oneVal = b.create<arith::ConstantOp>(b.getI32IntegerAttr(1));
+        auto oneVal = arith::ConstantOp::create(b, b.getI32IntegerAttr(1));
 
         auto getClampedIdxs = [&](Value &val0, Value &val1, int size, Value in,
                                   Value max, ImplicitLocOpBuilder &b) {
           val0 = in;
-          val1 = b.create<arith::AddIOp>(val0, oneVal);
+          val1 = arith::AddIOp::create(b, val0, oneVal);
           val0 =
               clampIntHelper(loc, val0, zeroI32, max, b, /*isUnsigned=*/false);
           val1 =
               clampIntHelper(loc, val1, zeroI32, max, b, /*isUnsigned=*/false);
-          val0 = b.create<arith::IndexCastOp>(b.getIndexType(), val0);
-          val1 = b.create<arith::IndexCastOp>(b.getIndexType(), val1);
+          val0 = arith::IndexCastOp::create(b, b.getIndexType(), val0);
+          val1 = arith::IndexCastOp::create(b, b.getIndexType(), val1);
         };
 
         // Linalg equivalent to the section below:
@@ -1946,27 +1946,27 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         getClampedIdxs(y0, y1, imageH, iy, hMax, b);
         getClampedIdxs(x0, x1, imageW, ix, wMax, b);
 
-        Value y0x0 = b.create<tensor::ExtractOp>(
+        Value y0x0 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y0, x0, channel});
-        Value y0x1 = b.create<tensor::ExtractOp>(
+        Value y0x1 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y0, x1, channel});
-        Value y1x0 = b.create<tensor::ExtractOp>(
+        Value y1x0 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y1, x0, channel});
-        Value y1x1 = b.create<tensor::ExtractOp>(
+        Value y1x1 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y1, x1, channel});
 
         if (floatingPointMode) {
           auto oneVal =
-              b.create<arith::ConstantOp>(b.getFloatAttr(floatTy, 1.0f));
+              arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 1.0f));
           auto interpolate = [&](Value val0, Value val1, Value delta,
                                  int inputSize,
                                  ImplicitLocOpBuilder &b) -> Value {
             if (inputSize == 1)
               return val0;
-            Value oneMinusDelta = b.create<arith::SubFOp>(oneVal, delta);
-            Value mul0 = b.create<arith::MulFOp>(val0, oneMinusDelta);
-            Value mul1 = b.create<arith::MulFOp>(val1, delta);
-            return b.create<arith::AddFOp>(mul0, mul1);
+            Value oneMinusDelta = arith::SubFOp::create(b, oneVal, delta);
+            Value mul0 = arith::MulFOp::create(b, val0, oneMinusDelta);
+            Value mul1 = arith::MulFOp::create(b, val1, delta);
+            return arith::AddFOp::create(b, mul0, mul1);
           };
 
           // Linalg equivalent to the section below:
@@ -1982,18 +1982,18 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
           // Linalg equivalent to the section below:
           //   result = topAcc * (unit_y - dy) + bottomAcc * dy
           Value result = interpolate(topAcc, bottomAcc, dy, imageH, b);
-          b.create<linalg::YieldOp>(result);
+          linalg::YieldOp::create(b, result);
         } else {
           // Perform in quantized space.
-          y0x0 = b.create<arith::ExtSIOp>(resultETy, y0x0);
-          y0x1 = b.create<arith::ExtSIOp>(resultETy, y0x1);
-          y1x0 = b.create<arith::ExtSIOp>(resultETy, y1x0);
-          y1x1 = b.create<arith::ExtSIOp>(resultETy, y1x1);
+          y0x0 = arith::ExtSIOp::create(b, resultETy, y0x0);
+          y0x1 = arith::ExtSIOp::create(b, resultETy, y0x1);
+          y1x0 = arith::ExtSIOp::create(b, resultETy, y1x0);
+          y1x1 = arith::ExtSIOp::create(b, resultETy, y1x1);
 
           const int64_t deltaBitwidth = dx.getType().getIntOrFloatBitWidth();
           if (resultETy.getIntOrFloatBitWidth() > deltaBitwidth) {
-            dx = b.create<arith::ExtSIOp>(resultETy, dx);
-            dy = b.create<arith::ExtSIOp>(resultETy, dy);
+            dx = arith::ExtSIOp::create(b, resultETy, dx);
+            dy = arith::ExtSIOp::create(b, resultETy, dy);
           }
 
           Value yScaleNExt = yScaleN;
@@ -2002,26 +2002,26 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
           const int64_t scaleBitwidth =
               xScaleN.getType().getIntOrFloatBitWidth();
           if (resultETy.getIntOrFloatBitWidth() > scaleBitwidth) {
-            yScaleNExt = b.create<arith::ExtSIOp>(resultETy, yScaleN);
-            xScaleNExt = b.create<arith::ExtSIOp>(resultETy, xScaleN);
+            yScaleNExt = arith::ExtSIOp::create(b, resultETy, yScaleN);
+            xScaleNExt = arith::ExtSIOp::create(b, resultETy, xScaleN);
           }
 
           auto interpolate = [](Value val0, Value val1, Value weight1,
                                 Value scale, int inputSize,
                                 ImplicitLocOpBuilder &b) -> Value {
             if (inputSize == 1)
-              return b.create<arith::MulIOp>(val0, scale);
-            Value weight0 = b.create<arith::SubIOp>(scale, weight1);
-            Value mul0 = b.create<arith::MulIOp>(val0, weight0);
-            Value mul1 = b.create<arith::MulIOp>(val1, weight1);
-            return b.create<arith::AddIOp>(mul0, mul1);
+              return arith::MulIOp::create(b, val0, scale);
+            Value weight0 = arith::SubIOp::create(b, scale, weight1);
+            Value mul0 = arith::MulIOp::create(b, val0, weight0);
+            Value mul1 = arith::MulIOp::create(b, val1, weight1);
+            return arith::AddIOp::create(b, mul0, mul1);
           };
 
           Value topAcc = interpolate(y0x0, y0x1, dx, xScaleNExt, imageW, b);
           Value bottomAcc = interpolate(y1x0, y1x1, dx, xScaleNExt, imageW, b);
           Value result =
               interpolate(topAcc, bottomAcc, dy, yScaleNExt, imageH, b);
-          b.create<linalg::YieldOp>(result);
+          linalg::YieldOp::create(b, result);
         }
       }
     }
@@ -2072,11 +2072,11 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i)) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
-    Value axisDimSize = rewriter.create<tensor::DimOp>(loc, input, axis);
+    Value axisDimSize = tensor::DimOp::create(rewriter, loc, input, axis);
 
     // First fill the output buffer with the init value.
     auto emptyTensor = rewriter
@@ -2094,21 +2094,21 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
           llvm::SmallVector<Value> indices;
           for (unsigned int i = 0; i < inputTy.getRank(); i++) {
             Value index =
-                rewriter.create<linalg::IndexOp>(nestedLoc, i).getResult();
+                linalg::IndexOp::create(rewriter, nestedLoc, i).getResult();
             if (i == axis) {
-              auto one = rewriter.create<arith::ConstantIndexOp>(nestedLoc, 1);
+              auto one = arith::ConstantIndexOp::create(rewriter, nestedLoc, 1);
               auto sizeMinusOne =
-                  rewriter.create<arith::SubIOp>(nestedLoc, axisDimSize, one);
-              index = rewriter.create<arith::SubIOp>(nestedLoc, sizeMinusOne,
+                  arith::SubIOp::create(rewriter, nestedLoc, axisDimSize, one);
+              index = arith::SubIOp::create(rewriter, nestedLoc, sizeMinusOne,
                                                      index);
             }
 
             indices.push_back(index);
           }
 
-          auto extract = nestedBuilder.create<tensor::ExtractOp>(
+          auto extract = tensor::ExtractOp::create(nestedBuilder,
               nestedLoc, input, indices);
-          nestedBuilder.create<linalg::YieldOp>(op.getLoc(),
+          linalg::YieldOp::create(nestedBuilder, op.getLoc(),
                                                 extract.getResult());
         });
     return success();
@@ -2148,11 +2148,11 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i) || multiples[i] == -1) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         op.getLoc(), genericShape, elementTy, dynDims);
 
     // We needs to map the input shape to the non-broadcasted dimensions.
@@ -2168,12 +2168,12 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<AffineMap, 2> affineMaps = {
         readAffineMap, rewriter.getMultiDimIdentityMap(genericShape.size())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, RankedTensorType::get(genericShape, elementTy), input,
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(genericShape.size()),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-          nestedBuilder.create<linalg::YieldOp>(op.getLoc(), *args.begin());
+          linalg::YieldOp::create(nestedBuilder, op.getLoc(), *args.begin());
         });
 
     auto shapeValue = getTosaConstShape(
@@ -2220,7 +2220,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i) && i != axis) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
@@ -2229,7 +2229,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
                               .create<tensor::EmptyOp>(loc, resultTy.getShape(),
                                                        outElementTy, dynDims)
                               .getResult();
-    auto fillValueIdx = rewriter.create<arith::ConstantOp>(
+    auto fillValueIdx = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(outElementTy, 0));
     auto filledTensorIdx =
         rewriter
@@ -2250,7 +2250,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           argmaxOp, "unsupported tosa.argmax element type");
 
     auto fillValueMax =
-        rewriter.create<arith::ConstantOp>(loc, fillValueMaxAttr);
+        arith::ConstantOp::create(rewriter, loc, fillValueMaxAttr);
     auto filledTensorMax =
         rewriter
             .create<linalg::FillOp>(loc, ValueRange{fillValueMax},
@@ -2274,7 +2274,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     bool didEncounterError = false;
     auto maps = AffineMap::inferFromExprList({srcExprs, dstExprs, dstExprs},
                                              rewriter.getContext());
-    auto linalgOp = rewriter.create<linalg::GenericOp>(
+    auto linalgOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
         ValueRange({filledTensorIdx, filledTensorMax}), maps, iteratorTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
@@ -2283,41 +2283,41 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           auto oldIndex = blockArgs[1];
           auto oldValue = blockArgs[2];
 
-          Value newIndex = rewriter.create<arith::IndexCastOp>(
+          Value newIndex = arith::IndexCastOp::create(rewriter,
               nestedLoc, oldIndex.getType(),
-              rewriter.create<linalg::IndexOp>(loc, axis));
+              linalg::IndexOp::create(rewriter, loc, axis));
 
           Value predicate;
           if (isa<FloatType>(inElementTy)) {
             if (argmaxOp.getNanMode() == "IGNORE") {
               // Only update index & max value for non NaN values. If all
               // values are NaNs, the initial index will be return which is 0.
-              predicate = rewriter.create<arith::CmpFOp>(
+              predicate = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::OGT, newValue, oldValue);
             } else {
               // Update max value if either of the following is true:
               // - new value is bigger
               // - cur max is not NaN and new value is NaN
-              Value gt = rewriter.create<arith::CmpFOp>(
+              Value gt = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::UGT, newValue, oldValue);
-              Value oldNonNaN = rewriter.create<arith::CmpFOp>(
+              Value oldNonNaN = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::ORD, oldValue, oldValue);
-              predicate = rewriter.create<arith::AndIOp>(
+              predicate = arith::AndIOp::create(rewriter,
                   nestedLoc, rewriter.getI1Type(), gt, oldNonNaN);
             }
           } else if (isa<IntegerType>(inElementTy)) {
-            predicate = rewriter.create<arith::CmpIOp>(
+            predicate = arith::CmpIOp::create(rewriter,
                 nestedLoc, arith::CmpIPredicate::sgt, newValue, oldValue);
           } else {
             didEncounterError = true;
             return;
           }
 
-          auto resultMax = rewriter.create<arith::SelectOp>(
+          auto resultMax = arith::SelectOp::create(rewriter,
               nestedLoc, predicate, newValue, oldValue);
-          auto resultIndex = rewriter.create<arith::SelectOp>(
+          auto resultIndex = arith::SelectOp::create(rewriter,
               nestedLoc, predicate, newIndex, oldIndex);
-          nestedBuilder.create<linalg::YieldOp>(
+          linalg::YieldOp::create(nestedBuilder,
               nestedLoc, ValueRange({resultIndex, resultMax}));
         });
 
@@ -2363,19 +2363,19 @@ class GatherConverter : public OpConversionPattern<tosa::GatherOp> {
             rewriter.getContext()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy}), ValueRange{indices},
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
           auto indexValue = args[0];
-          auto index0 = rewriter.create<linalg::IndexOp>(loc, 0);
-          Value index1 = rewriter.create<arith::IndexCastOp>(
+          auto index0 = linalg::IndexOp::create(rewriter, loc, 0);
+          Value index1 = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getIndexType(), indexValue);
-          auto index2 = rewriter.create<linalg::IndexOp>(loc, 2);
-          Value extract = rewriter.create<tensor::ExtractOp>(
+          auto index2 = linalg::IndexOp::create(rewriter, loc, 2);
+          Value extract = tensor::ExtractOp::create(rewriter,
               loc, input, ValueRange{index0, index1, index2});
-          rewriter.create<linalg::YieldOp>(loc, extract);
+          linalg::YieldOp::create(rewriter, loc, extract);
         });
     rewriter.replaceOp(op, genericOp.getResult(0));
     return success();
@@ -2424,7 +2424,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
     for (int i = 0; i < resultTy.getRank(); ++i) {
       if (inputTy.isDynamicDim(i)) {
         dynDims.push_back(
-            rewriter.create<tensor::DimOp>(loc, op.getOperand(0), i));
+            tensor::DimOp::create(rewriter, loc, op.getOperand(0), i));
       }
     }
 
@@ -2437,7 +2437,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
         rewriter.getMultiDimIdentityMap(resultTy.getRank()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, resultTy, ValueRange({input}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -2452,69 +2452,69 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
       rewriter.setInsertionPointToStart(block);
       if (inputElementTy.isInteger(8) && tableElementTy.isInteger(8) &&
           resultElementTy.isInteger(8)) {
-        Value index = rewriter.create<arith::IndexCastOp>(
+        Value index = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), inputValue);
-        Value offset = rewriter.create<arith::ConstantIndexOp>(loc, 128);
-        index = rewriter.create<arith::AddIOp>(loc, rewriter.getIndexType(),
+        Value offset = arith::ConstantIndexOp::create(rewriter, loc, 128);
+        index = arith::AddIOp::create(rewriter, loc, rewriter.getIndexType(),
                                                index, offset);
         Value extract =
-            rewriter.create<tensor::ExtractOp>(loc, table, ValueRange{index});
-        rewriter.create<linalg::YieldOp>(loc, extract);
+            tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
+        linalg::YieldOp::create(rewriter, loc, extract);
         return success();
       }
 
       if (inputElementTy.isInteger(16) && tableElementTy.isInteger(16) &&
           resultElementTy.isInteger(32)) {
-        Value extend = rewriter.create<arith::ExtSIOp>(
+        Value extend = arith::ExtSIOp::create(rewriter,
             loc, rewriter.getI32Type(), inputValue);
 
-        auto offset = rewriter.create<arith::ConstantOp>(
+        auto offset = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(32768));
-        auto seven = rewriter.create<arith::ConstantOp>(
+        auto seven = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(7));
-        auto one = rewriter.create<arith::ConstantOp>(
+        auto one = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(1));
-        auto b1111111 = rewriter.create<arith::ConstantOp>(
+        auto b1111111 = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(127));
 
         // Compute the index and fractional part from the input value:
         // value = value + 32768
         // index = value >> 7;
         // fraction = 0x01111111 & value
-        auto extendAdd = rewriter.create<arith::AddIOp>(loc, extend, offset);
-        Value index = rewriter.create<arith::ShRUIOp>(loc, extendAdd, seven);
+        auto extendAdd = arith::AddIOp::create(rewriter, loc, extend, offset);
+        Value index = arith::ShRUIOp::create(rewriter, loc, extendAdd, seven);
         Value fraction =
-            rewriter.create<arith::AndIOp>(loc, extendAdd, b1111111);
+            arith::AndIOp::create(rewriter, loc, extendAdd, b1111111);
 
         // Extract the base and next values from the table.
         // base = (int32_t) table[index];
         // next = (int32_t) table[index + 1];
-        Value indexPlusOne = rewriter.create<arith::AddIOp>(loc, index, one);
+        Value indexPlusOne = arith::AddIOp::create(rewriter, loc, index, one);
 
-        index = rewriter.create<arith::IndexCastOp>(
+        index = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), index);
-        indexPlusOne = rewriter.create<arith::IndexCastOp>(
+        indexPlusOne = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), indexPlusOne);
 
         Value base =
-            rewriter.create<tensor::ExtractOp>(loc, table, ValueRange{index});
-        Value next = rewriter.create<tensor::ExtractOp>(
+            tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
+        Value next = tensor::ExtractOp::create(rewriter,
             loc, table, ValueRange{indexPlusOne});
 
         base =
-            rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), base);
+            arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), base);
         next =
-            rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), next);
+            arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), next);
 
         // Use the fractional part to interpolate between the input values:
         // result = (base << 7) + (next - base) * fraction
-        Value baseScaled = rewriter.create<arith::ShLIOp>(loc, base, seven);
-        Value diff = rewriter.create<arith::SubIOp>(loc, next, base);
-        Value diffScaled = rewriter.create<arith::MulIOp>(loc, diff, fraction);
+        Value baseScaled = arith::ShLIOp::create(rewriter, loc, base, seven);
+        Value diff = arith::SubIOp::create(rewriter, loc, next, base);
+        Value diffScaled = arith::MulIOp::create(rewriter, loc, diff, fraction);
         Value result =
-            rewriter.create<arith::AddIOp>(loc, baseScaled, diffScaled);
+            arith::AddIOp::create(rewriter, loc, baseScaled, diffScaled);
 
-        rewriter.create<linalg::YieldOp>(loc, result);
+        linalg::YieldOp::create(rewriter, loc, result);
 
         return success();
       }
@@ -2532,8 +2532,8 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static OpFoldResult halfPlusOne(OpBuilder &builder, Location loc,
                                   OpFoldResult ofr) {
-    auto one = builder.create<arith::ConstantIndexOp>(loc, 1);
-    auto two = builder.create<arith::ConstantIndexOp>(loc, 2);
+    auto one = arith::ConstantIndexOp::create(builder, loc, 1);
+    auto two = arith::ConstantIndexOp::create(builder, loc, 2);
 
     auto value = getValueOrCreateConstantIndexOp(builder, loc, ofr);
     auto divBy2 = builder.createOrFold<arith::DivUIOp>(loc, value, two);
@@ -2562,9 +2562,9 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
                                 RankedTensorType type,
                                 llvm::ArrayRef<Value> dynamicSizes) {
     auto emptyTensor =
-        rewriter.create<tensor::EmptyOp>(loc, type, dynamicSizes);
+        tensor::EmptyOp::create(rewriter, loc, type, dynamicSizes);
     auto fillValueAttr = rewriter.getZeroAttr(type.getElementType());
-    auto fillValue = rewriter.create<arith::ConstantOp>(loc, fillValueAttr);
+    auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
     auto filledTensor = rewriter
                             .create<linalg::FillOp>(loc, ValueRange{fillValue},
                                                     ValueRange{emptyTensor})
@@ -2574,18 +2574,18 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static Value castIndexToFloat(OpBuilder &builder, Location loc,
                                 FloatType type, Value value) {
-    auto integerVal = builder.create<arith::IndexCastUIOp>(
+    auto integerVal = arith::IndexCastUIOp::create(builder,
         loc,
         type.getIntOrFloatBitWidth() > 32 ? builder.getI64Type()
                                           : builder.getI32Type(),
         value);
 
-    return builder.create<arith::UIToFPOp>(loc, type, integerVal);
+    return arith::UIToFPOp::create(builder, loc, type, integerVal);
   }
 
   static Value createLinalgIndex(OpBuilder &builder, Location loc,
                                  FloatType type, int64_t index) {
-    auto indexVal = builder.create<linalg::IndexOp>(loc, index);
+    auto indexVal = linalg::IndexOp::create(builder, loc, index);
     return castIndexToFloat(builder, loc, type, indexVal);
   }
 
@@ -2640,7 +2640,7 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
     // Constants and dimension sizes
     auto twoPiAttr = rewriter.getFloatAttr(elementType, 6.283185307179586);
-    auto twoPi = rewriter.create<arith::ConstantOp>(loc, twoPiAttr);
+    auto twoPi = arith::ConstantOp::create(rewriter, loc, twoPiAttr);
     auto constH = castIndexToFloat(rewriter, loc, elementType, dimH);
     auto constW = castIndexToFloat(rewriter, loc, elementType, dimW);
 
@@ -2650,43 +2650,43 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
       Value sumImag = args[2];
 
       // Indices for angle computation
-      Value oy = builder.create<linalg::IndexOp>(loc, 1);
-      Value ox = builder.create<linalg::IndexOp>(loc, 2);
-      Value iy = builder.create<linalg::IndexOp>(loc, 3);
-      Value ix = builder.create<linalg::IndexOp>(loc, 4);
+      Value oy = linalg::IndexOp::create(builder, loc, 1);
+      Value ox = linalg::IndexOp::create(builder, loc, 2);
+      Value iy = linalg::IndexOp::create(builder, loc, 3);
+      Value ix = linalg::IndexOp::create(builder, loc, 4);
 
       // Calculating angle without integer parts of components as sin/cos are
       // periodic: angle = 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix * ox) % W )
       // / W);
-      auto iyXoy = builder.create<index::MulOp>(loc, iy, oy);
-      auto ixXox = builder.create<index::MulOp>(loc, ix, ox);
+      auto iyXoy = index::MulOp::create(builder, loc, iy, oy);
+      auto ixXox = index::MulOp::create(builder, loc, ix, ox);
 
-      auto iyRem = builder.create<index::RemUOp>(loc, iyXoy, dimH);
-      auto ixRem = builder.create<index::RemUOp>(loc, ixXox, dimW);
+      auto iyRem = index::RemUOp::create(builder, loc, iyXoy, dimH);
+      auto ixRem = index::RemUOp::create(builder, loc, ixXox, dimW);
 
       auto iyRemFloat = castIndexToFloat(builder, loc, elementType, iyRem);
       auto ixRemFloat = castIndexToFloat(builder, loc, elementType, ixRem);
 
-      auto yComponent = builder.create<arith::DivFOp>(loc, iyRemFloat, constH);
-      auto xComponent = builder.create<arith::DivFOp>(loc, ixRemFloat, constW);
-      auto sumXY = builder.create<arith::AddFOp>(loc, yComponent, xComponent);
-      auto angle = builder.create<arith::MulFOp>(loc, twoPi, sumXY);
+      auto yComponent = arith::DivFOp::create(builder, loc, iyRemFloat, constH);
+      auto xComponent = arith::DivFOp::create(builder, loc, ixRemFloat, constW);
+      auto sumXY = arith::AddFOp::create(builder, loc, yComponent, xComponent);
+      auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       // realComponent = valReal * cos(angle)
       // imagComponent = valReal * sin(angle)
-      auto cosAngle = builder.create<math::CosOp>(loc, angle);
-      auto sinAngle = builder.create<math::SinOp>(loc, angle);
+      auto cosAngle = math::CosOp::create(builder, loc, angle);
+      auto sinAngle = math::SinOp::create(builder, loc, angle);
       auto realComponent =
-          builder.create<arith::MulFOp>(loc, valReal, cosAngle);
+          arith::MulFOp::create(builder, loc, valReal, cosAngle);
       auto imagComponent =
-          builder.create<arith::MulFOp>(loc, valReal, sinAngle);
+          arith::MulFOp::create(builder, loc, valReal, sinAngle);
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = builder.create<arith::AddFOp>(loc, sumReal, realComponent);
-      auto outImag = builder.create<arith::SubFOp>(loc, sumImag, imagComponent);
+      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag = arith::SubFOp::create(builder, loc, sumImag, imagComponent);
 
-      builder.create<linalg::YieldOp>(loc, ValueRange{outReal, outImag});
+      linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
 
     rewriter.replaceOpWithNewOp<linalg::GenericOp>(
@@ -2760,7 +2760,7 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
 
     // Constants and dimension sizes
     auto twoPiAttr = rewriter.getFloatAttr(real_el_ty, 6.283185307179586);
-    auto twoPi = rewriter.create<arith::ConstantOp>(loc, twoPiAttr);
+    auto twoPi = arith::ConstantOp::create(rewriter, loc, twoPiAttr);
     Value constH =
         RFFT2dConverter::castIndexToFloat(rewriter, loc, real_el_ty, dimH);
     Value constW =
@@ -2773,57 +2773,57 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
       Value sumImag = args[3];
 
       // Indices for angle computation
-      Value oy = builder.create<linalg::IndexOp>(loc, 1);
-      Value ox = builder.create<linalg::IndexOp>(loc, 2);
-      Value iy = builder.create<linalg::IndexOp>(loc, 3);
-      Value ix = builder.create<linalg::IndexOp>(loc, 4);
+      Value oy = linalg::IndexOp::create(builder, loc, 1);
+      Value ox = linalg::IndexOp::create(builder, loc, 2);
+      Value iy = linalg::IndexOp::create(builder, loc, 3);
+      Value ix = linalg::IndexOp::create(builder, loc, 4);
 
       // float_t angle = sign_val * 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix *
       // ox) % W ) / W);
-      auto iyXoy = builder.create<index::MulOp>(loc, iy, oy);
-      auto ixXox = builder.create<index::MulOp>(loc, ix, ox);
+      auto iyXoy = index::MulOp::create(builder, loc, iy, oy);
+      auto ixXox = index::MulOp::create(builder, loc, ix, ox);
 
-      auto iyRem = builder.create<index::RemUOp>(loc, iyXoy, dimH);
-      auto ixRem = builder.create<index::RemUOp>(loc, ixXox, dimW);
+      auto iyRem = index::RemUOp::create(builder, loc, iyXoy, dimH);
+      auto ixRem = index::RemUOp::create(builder, loc, ixXox, dimW);
 
       auto iyRemFloat =
           RFFT2dConverter::castIndexToFloat(builder, loc, real_el_ty, iyRem);
       auto ixRemFloat =
           RFFT2dConverter::castIndexToFloat(builder, loc, real_el_ty, ixRem);
 
-      auto yComponent = builder.create<arith::DivFOp>(loc, iyRemFloat, constH);
-      auto xComponent = builder.create<arith::DivFOp>(loc, ixRemFloat, constW);
+      auto yComponent = arith::DivFOp::create(builder, loc, iyRemFloat, constH);
+      auto xComponent = arith::DivFOp::create(builder, loc, ixRemFloat, constW);
 
-      auto sumXY = builder.create<arith::AddFOp>(loc, yComponent, xComponent);
-      auto angle = builder.create<arith::MulFOp>(loc, twoPi, sumXY);
+      auto sumXY = arith::AddFOp::create(builder, loc, yComponent, xComponent);
+      auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       if (inverse.getValue()) {
-        angle = builder.create<arith::MulFOp>(
+        angle = arith::MulFOp::create(builder,
             loc, angle,
-            rewriter.create<arith::ConstantOp>(
+            arith::ConstantOp::create(rewriter,
                 loc, rewriter.getFloatAttr(real_el_ty, -1.0)));
       }
 
       // realComponent = val_real * cos(a) + val_imag * sin(a);
       // imagComponent = -val_real * sin(a) + val_imag * cos(a);
-      auto cosAngle = builder.create<math::CosOp>(loc, angle);
-      auto sinAngle = builder.create<math::SinOp>(loc, angle);
+      auto cosAngle = math::CosOp::create(builder, loc, angle);
+      auto sinAngle = math::SinOp::create(builder, loc, angle);
 
-      auto rcos = builder.create<arith::MulFOp>(loc, valReal, cosAngle);
-      auto rsin = builder.create<arith::MulFOp>(loc, valImag, sinAngle);
-      auto realComponent = builder.create<arith::AddFOp>(loc, rcos, rsin);
+      auto rcos = arith::MulFOp::create(builder, loc, valReal, cosAngle);
+      auto rsin = arith::MulFOp::create(builder, loc, valImag, sinAngle);
+      auto realComponent = arith::AddFOp::create(builder, loc, rcos, rsin);
 
-      auto icos = builder.create<arith::MulFOp>(loc, valImag, cosAngle);
-      auto isin = builder.create<arith::MulFOp>(loc, valReal, sinAngle);
+      auto icos = arith::MulFOp::create(builder, loc, valImag, cosAngle);
+      auto isin = arith::MulFOp::create(builder, loc, valReal, sinAngle);
 
-      auto imagComponent = builder.create<arith::SubFOp>(loc, icos, isin);
+      auto imagComponent = arith::SubFOp::create(builder, loc, icos, isin);
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = builder.create<arith::AddFOp>(loc, sumReal, realComponent);
-      auto outImag = builder.create<arith::AddFOp>(loc, sumImag, imagComponent);
+      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag = arith::AddFOp::create(builder, loc, sumImag, imagComponent);
 
-      builder.create<linalg::YieldOp>(loc, ValueRange{outReal, outImag});
+      linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
 
     rewriter.replaceOpWithNewOp<linalg::GenericOp>(
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index b89fde4fbc17e..d81b8645506c4 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -52,9 +52,9 @@ static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
     highIndices.push_back(rewriter.getIndexAttr(highPad));
   }
 
-  Value padValue = rewriter.create<arith::ConstantOp>(loc, padAttr);
+  Value padValue = arith::ConstantOp::create(rewriter, loc, padAttr);
 
-  return rewriter.create<tensor::PadOp>(
+  return tensor::PadOp::create(rewriter,
       loc, RankedTensorType::get(paddedShape, inputETy), input, lowIndices,
       highIndices, padValue);
 }
@@ -72,10 +72,10 @@ linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias,
             Value biasVal = args[0];
             Type resType = args[1].getType();
             if (resType != biasVal.getType()) {
-              biasVal = builder.create<arith::ExtSIOp>(loc, resType, biasVal);
+              biasVal = arith::ExtSIOp::create(builder, loc, resType, biasVal);
             }
-            Value added = builder.create<arith::AddIOp>(loc, biasVal, args[1]);
-            builder.create<linalg::YieldOp>(loc, added);
+            Value added = arith::AddIOp::create(builder, loc, biasVal, args[1]);
+            linalg::YieldOp::create(builder, loc, added);
           })
       .getResult(0);
 }
@@ -134,19 +134,19 @@ static mlir::Value linalgBroadcastAndMaybeExt(PatternRewriter &rewriter,
             if (resType != biasVal.getType()) {
               biasVal =
                   resultTy.getElementType().isFloat()
-                      ? builder.create<arith::ExtFOp>(loc, resType, biasVal)
+                      ? arith::ExtFOp::create(builder, loc, resType, biasVal)
                             .getResult()
-                      : builder.create<arith::ExtSIOp>(loc, resType, biasVal)
+                      : arith::ExtSIOp::create(builder, loc, resType, biasVal)
                             .getResult();
             }
-            builder.create<linalg::YieldOp>(loc, biasVal);
+            linalg::YieldOp::create(builder, loc, biasVal);
           })
       .getResult(0);
 }
 
 static mlir::Value reifyConstantDim(int64_t attr,
                                     ImplicitLocOpBuilder &builder) {
-  return builder.create<arith::ConstantIndexOp>(attr);
+  return arith::ConstantIndexOp::create(builder, attr);
 }
 
 // Calculating the output width/height using the formula:
@@ -160,22 +160,22 @@ static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim,
                                           int64_t dilationAttr,
                                           OpBuilder &rewriter) {
   ImplicitLocOpBuilder builder(loc, rewriter);
-  auto one = rewriter.create<arith::ConstantOp>(
+  auto one = arith::ConstantOp::create(rewriter,
       loc, IntegerAttr::get(inputDim.getType(), 1));
   Value padBefore = reifyConstantDim(padBeforeAttr, builder);
-  Value paddedBefore = builder.create<arith::AddIOp>(inputDim, padBefore);
+  Value paddedBefore = arith::AddIOp::create(builder, inputDim, padBefore);
   Value padAfter = reifyConstantDim(padAfterAttr, builder);
-  Value paddedAfter = builder.create<arith::AddIOp>(paddedBefore, padAfter);
+  Value paddedAfter = arith::AddIOp::create(builder, paddedBefore, padAfter);
 
-  Value subOne = builder.create<arith::SubIOp>(kernelDim, one);
+  Value subOne = arith::SubIOp::create(builder, kernelDim, one);
   Value dilation = reifyConstantDim(dilationAttr, builder);
-  Value dilated = builder.create<arith::MulIOp>(dilation, subOne);
-  Value addOne = builder.create<arith::AddIOp>(dilated, one);
+  Value dilated = arith::MulIOp::create(builder, dilation, subOne);
+  Value addOne = arith::AddIOp::create(builder, dilated, one);
 
-  Value subtract = builder.create<arith::SubIOp>(paddedAfter, addOne);
+  Value subtract = arith::SubIOp::create(builder, paddedAfter, addOne);
   Value stride = reifyConstantDim(strideAttr, builder);
-  Value divide = builder.create<arith::DivUIOp>(subtract, stride);
-  return builder.create<arith::AddIOp>(divide, one);
+  Value divide = arith::DivUIOp::create(builder, subtract, stride);
+  return arith::AddIOp::create(builder, divide, one);
 }
 
 // Creates a vector of the dynamic output dims for Conv2D and Depthwise_Conv2D
@@ -198,9 +198,9 @@ static SmallVector<Value> inferDynamicDimsForConv(
       auto padBottom = padAttr[i * 2 + 1];
       auto stride = strideAttr[i];
       auto dilation = dilationAttr[i];
-      Value initDynDim = rewriter.create<tensor::DimOp>(loc, input, inputDim);
+      Value initDynDim = tensor::DimOp::create(rewriter, loc, input, inputDim);
       Value kernelDynDim =
-          rewriter.create<tensor::DimOp>(loc, weight, kernelDim);
+          tensor::DimOp::create(rewriter, loc, weight, kernelDim);
       // H = F(IH, pad_top, pad_bottom, dilation_y, KH, stride_y)
       dynDims[inputDim] =
           getConvOrPoolOutputDim(loc, initDynDim, padTop, padBottom,
@@ -211,7 +211,7 @@ static SmallVector<Value> inferDynamicDimsForConv(
   // Get the batch/channels dimensions.
   for (int i = 0; i < inputRank; i++) {
     if (resultTy.isDynamicDim(i) && !dynDims[i])
-      dynDims[i] = rewriter.create<tensor::DimOp>(loc, input, i);
+      dynDims[i] = tensor::DimOp::create(rewriter, loc, input, i);
   }
 
   SmallVector<Value> filteredDims = condenseValues(dynDims);
@@ -350,7 +350,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
         auto weightPermAttr = rewriter.getDenseI32ArrayAttr(weightPerm);
         Type newWeightTy =
             RankedTensorType::get(newWeightShape, weightTy.getElementType());
-        weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
+        weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
                                                     weightPermAttr);
       }
     }
@@ -372,7 +372,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       auto weightPermAttr = rewriter.getDenseI32ArrayAttr(weightPerm);
       Type newWeightTy =
           RankedTensorType::get(newWeightShape, weightTy.getElementType());
-      weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
+      weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
                                                   weightPermAttr);
     }
 
@@ -384,7 +384,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     auto strideAttr = rewriter.getI64TensorAttr(stride);
     auto dilationAttr = rewriter.getI64TensorAttr(dilation);
 
-    Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), accETy, filteredDims);
 
     Value broadcastBias =
@@ -394,8 +394,8 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       auto iZp = rewriter.getI32IntegerAttr(inputZpVal);
       auto kZp = rewriter.getI32IntegerAttr(weightZpVal);
 
-      auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
+      auto iZpVal = arith::ConstantOp::create(rewriter, loc, iZp);
+      auto kZpVal = arith::ConstantOp::create(rewriter, loc, kZp);
 
       Value conv =
           rewriter
@@ -417,7 +417,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     // We may need to truncate back to the result type if the accumulator was
     // wider than the result.
     if (resultTy != accTy)
-      conv = rewriter.create<tosa::CastOp>(loc, resultTy, conv);
+      conv = tosa::CastOp::create(rewriter, loc, resultTy, conv);
 
     rewriter.replaceOp(op, conv);
     return success();
@@ -526,15 +526,15 @@ class DepthwiseConvConverter
                               accETy);
 
     auto resultZeroAttr = rewriter.getZeroAttr(accETy);
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, linalgConvTy.getShape(), accETy, filteredDims);
-    Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
+    Value zero = arith::ConstantOp::create(rewriter, loc, resultZeroAttr);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
                            .result();
 
-    Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultETy, filteredDims);
 
     // Broadcast the initial value to the output tensor before convolving.
@@ -553,7 +553,7 @@ class DepthwiseConvConverter
       // We may need to truncate back to the result type if the accumulator was
       // wider than the result.
       if (accETy != resultETy)
-        conv = rewriter.create<tosa::CastOp>(
+        conv = tosa::CastOp::create(rewriter,
             loc,
             RankedTensorType::get(cast<ShapedType>(conv.getType()).getShape(),
                                   resultETy),
@@ -561,7 +561,7 @@ class DepthwiseConvConverter
 
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
+      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
           loc, resultTy, conv, reassociationMap);
 
       Value result =
@@ -574,20 +574,20 @@ class DepthwiseConvConverter
                       ValueRange args) {
                     Value added;
                     if (llvm::isa<FloatType>(inputETy))
-                      added = nestedBuilder.create<arith::AddFOp>(loc, args[0],
+                      added = arith::AddFOp::create(nestedBuilder, loc, args[0],
                                                                   args[1]);
                     else
-                      added = nestedBuilder.create<arith::AddIOp>(loc, args[0],
+                      added = arith::AddIOp::create(nestedBuilder, loc, args[0],
                                                                   args[1]);
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
+                    linalg::YieldOp::create(nestedBuilder, nestedLoc, added);
                   })
               .getResult(0);
       rewriter.replaceOp(op, result);
     } else {
       IntegerAttr iZp = rewriter.getI32IntegerAttr(inputZpVal);
       IntegerAttr wZp = rewriter.getI32IntegerAttr(weightZpVal);
-      auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, wZp);
+      auto iZpVal = arith::ConstantOp::create(rewriter, loc, iZp);
+      auto kZpVal = arith::ConstantOp::create(rewriter, loc, wZp);
       Value conv =
           rewriter
               .create<linalg::DepthwiseConv2DNhwcHwcmQOp>(
@@ -596,7 +596,7 @@ class DepthwiseConvConverter
               .getResult(0);
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
+      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
           loc, resultTy, conv, reassociationMap);
       Value result = linalgIntBroadcastExtSIAdd(
           rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
@@ -621,22 +621,22 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
     dynDims.resize(cast<ShapedType>(op->getResult(0).getType()).getRank());
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(0)) {
-      dynDims[0] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 0);
+      dynDims[0] = tensor::DimOp::create(rewriter, loc, op->getOperand(0), 0);
     }
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(1)) {
-      dynDims[1] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 1);
+      dynDims[1] = tensor::DimOp::create(rewriter, loc, op->getOperand(0), 1);
     }
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(2)) {
-      dynDims[2] = rewriter.create<tensor::DimOp>(loc, op->getOperand(1), 2);
+      dynDims[2] = tensor::DimOp::create(rewriter, loc, op->getOperand(1), 2);
     }
 
     SmallVector<Value> filteredDims = condenseValues(dynDims);
 
     auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
-    Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
@@ -670,9 +670,9 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
       return success();
     }
 
-    auto aZp = rewriter.create<arith::ConstantOp>(
+    auto aZp = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI32IntegerAttr(aZpVal));
-    auto bZp = rewriter.create<arith::ConstantOp>(
+    auto bZp = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI32IntegerAttr(bZpVal));
     rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
         op, TypeRange{op.getType()},
@@ -702,7 +702,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     // Batch dimension
     if (resultTy.isDynamicDim(0))
-      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+      dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
     // Height/width dimensions
     for (int64_t dim : {1, 2}) {
@@ -713,10 +713,10 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       int64_t index = dim - 1;
 
       // Input height/width
-      Value ihw = rewriter.create<tensor::DimOp>(loc, input, dim);
+      Value ihw = tensor::DimOp::create(rewriter, loc, input, dim);
 
       // Kernel height/width
-      Value khw = rewriter.create<arith::ConstantIndexOp>(loc, kernel[index]);
+      Value khw = arith::ConstantIndexOp::create(rewriter, loc, kernel[index]);
 
       // Output height/width
       Value ohw = getConvOrPoolOutputDim(loc, ihw, pad[index * 2],
@@ -727,7 +727,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     // Channel dimension
     if (resultTy.isDynamicDim(3))
-      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 3));
+      dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 3));
 
     return dynamicDims;
   }
@@ -776,7 +776,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     Value paddedInput = applyPad(loc, input, pad, initialAttr, rewriter);
 
-    Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
+    Value initialValue = arith::ConstantOp::create(rewriter, loc, initialAttr);
 
     ArrayRef<int64_t> kernel = op.getKernel();
     ArrayRef<int64_t> stride = op.getStride();
@@ -785,15 +785,15 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
 
     Value filledEmptyTensor =
-        rewriter.create<linalg::FillOp>(loc, initialValue, emptyTensor)
+        linalg::FillOp::create(rewriter, loc, initialValue, emptyTensor)
             .result();
 
     Value fakeWindowDims =
-        rewriter.create<tensor::EmptyOp>(loc, kernel, resultETy);
+        tensor::EmptyOp::create(rewriter, loc, kernel, resultETy);
 
     if (isUnsigned) {
       rewriter.replaceOpWithNewOp<linalg::PoolingNhwcMaxUnsignedOp>(
@@ -802,7 +802,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       return llvm::success();
     }
 
-    auto resultOp = rewriter.create<linalg::PoolingNhwcMaxOp>(
+    auto resultOp = linalg::PoolingNhwcMaxOp::create(rewriter,
         op->getLoc(), ArrayRef<Type>{resultTy},
         ValueRange{paddedInput, fakeWindowDims}, filledEmptyTensor, strideAttr,
         dilationAttr);
@@ -821,7 +821,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     // it to include the appropriate checks. If the current value is NaN the
     // old value of pool will be taken otherwise we use the result.
     if (const auto nanMode = op.getNanMode(); nanMode == "IGNORE") {
-      auto genericOp = rewriter.create<linalg::GenericOp>(
+      auto genericOp = linalg::GenericOp::create(rewriter,
           op->getLoc(), resultOp.getType(0), resultOp.getInputs(),
           resultOp.getOutputs(), resultOp.getIndexingMapsArray(),
           resultOp.getIteratorTypesArray(),
@@ -832,12 +832,12 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
             auto &oldMaxOp = *resultOp.getBlock()->begin();
             map.map(oldArgs, blockArgs);
             auto *newOp = opBuilder.clone(oldMaxOp, map);
-            Value isNaN = opBuilder.create<arith::CmpFOp>(
+            Value isNaN = arith::CmpFOp::create(opBuilder,
                 op->getLoc(), arith::CmpFPredicate::UNO, blockArgs.front(),
                 blockArgs.front());
-            auto selectOp = opBuilder.create<arith::SelectOp>(
+            auto selectOp = arith::SelectOp::create(opBuilder,
                 op->getLoc(), isNaN, blockArgs.back(), newOp->getResult(0));
-            opBuilder.create<linalg::YieldOp>(loc, selectOp.getResult());
+            linalg::YieldOp::create(opBuilder, loc, selectOp.getResult());
           });
       rewriter.replaceOp(resultOp, genericOp);
     }
@@ -893,7 +893,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Value paddedInput = applyPad(loc, input, pad, padAttr, rewriter);
 
     auto initialAttr = rewriter.getZeroAttr(accETy);
-    Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
+    Value initialValue = arith::ConstantOp::create(rewriter, loc, initialAttr);
 
     ArrayRef<int64_t> kernel = op.getKernel();
     ArrayRef<int64_t> stride = op.getStride();
@@ -902,7 +902,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value poolEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value poolEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, accTy.getShape(), accETy, dynamicDims);
 
     Value filledEmptyTensor =
@@ -912,7 +912,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             .result();
 
     Value fakeWindowDims =
-        rewriter.create<tensor::EmptyOp>(loc, kernel, accETy);
+        tensor::EmptyOp::create(rewriter, loc, kernel, accETy);
 
     // Sum across the pooled region.
     Value poolingOp = rewriter
@@ -924,24 +924,24 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
 
     // Normalize the summed value by the number of elements grouped in each
     // pool.
-    Value iH = rewriter.create<tensor::DimOp>(loc, poolingOp, 1);
-    Value iW = rewriter.create<tensor::DimOp>(loc, poolingOp, 2);
+    Value iH = tensor::DimOp::create(rewriter, loc, poolingOp, 1);
+    Value iW = tensor::DimOp::create(rewriter, loc, poolingOp, 2);
 
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    iH = rewriter.create<arith::SubIOp>(loc, iH, one);
-    iW = rewriter.create<arith::SubIOp>(loc, iW, one);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    iH = arith::SubIOp::create(rewriter, loc, iH, one);
+    iW = arith::SubIOp::create(rewriter, loc, iW, one);
 
-    Value genericEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value genericEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultETy, dynamicDims);
 
     auto affineMap = rewriter.getMultiDimIdentityMap(resultTy.getRank());
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
         ValueRange{genericEmptyTensor},
         ArrayRef<AffineMap>({affineMap, affineMap}),
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
-          auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+          auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
           // Determines what the portion of valid input is covered by the
           // kernel.
@@ -949,30 +949,30 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             if (pad == 0)
               return valid;
 
-            auto padVal = rewriter.create<arith::ConstantIndexOp>(loc, pad);
-            Value dpos = rewriter.create<arith::SubIOp>(loc, pos, padVal);
+            auto padVal = arith::ConstantIndexOp::create(rewriter, loc, pad);
+            Value dpos = arith::SubIOp::create(rewriter, loc, pos, padVal);
 
-            Value offset = rewriter.create<arith::MinSIOp>(loc, dpos, zero);
-            return rewriter.create<arith::AddIOp>(loc, valid, offset)
+            Value offset = arith::MinSIOp::create(rewriter, loc, dpos, zero);
+            return arith::AddIOp::create(rewriter, loc, valid, offset)
                 ->getResult(0);
           };
 
           auto coverageFn = [&](int64_t i, Value isize) -> Value {
             Value strideVal =
-                rewriter.create<arith::ConstantIndexOp>(loc, stride[i - 1]);
+                arith::ConstantIndexOp::create(rewriter, loc, stride[i - 1]);
             Value val =
-                rewriter.create<arith::ConstantIndexOp>(loc, kernel[i - 1]);
+                arith::ConstantIndexOp::create(rewriter, loc, kernel[i - 1]);
 
             // Find the position relative to the input tensor's ends.
-            Value left = rewriter.create<linalg::IndexOp>(loc, i);
-            Value right = rewriter.create<arith::SubIOp>(loc, isize, left);
-            left = rewriter.create<arith::MulIOp>(loc, left, strideVal);
-            right = rewriter.create<arith::MulIOp>(loc, right, strideVal);
+            Value left = linalg::IndexOp::create(rewriter, loc, i);
+            Value right = arith::SubIOp::create(rewriter, loc, isize, left);
+            left = arith::MulIOp::create(rewriter, loc, left, strideVal);
+            right = arith::MulIOp::create(rewriter, loc, right, strideVal);
 
             // Determine how much padding was included.
             val = padFn(val, left, pad[i * 2]);
             val = padFn(val, right, pad[i * 2 + 1]);
-            return rewriter.create<arith::MaxSIOp>(loc, one, val);
+            return arith::MaxSIOp::create(rewriter, loc, one, val);
           };
 
           // Compute the indices from either end.
@@ -980,70 +980,70 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
           Value kW3 = coverageFn(2, iW);
 
           // Compute the total number of elements and normalize.
-          auto count = rewriter.create<arith::IndexCastOp>(
+          auto count = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getI32Type(),
-              rewriter.create<arith::MulIOp>(loc, kH3, kW3));
+              arith::MulIOp::create(rewriter, loc, kH3, kW3));
 
           // Divide by the number of summed values. For floats this is just
           // a div however for quantized values input normalization had
           // to be applied.
           Value poolVal = args[0];
           if (isa<FloatType>(accETy)) {
-            auto countF = rewriter.create<arith::SIToFPOp>(loc, accETy, count);
-            poolVal = rewriter.create<arith::DivFOp>(loc, poolVal, countF)
+            auto countF = arith::SIToFPOp::create(rewriter, loc, accETy, count);
+            poolVal = arith::DivFOp::create(rewriter, loc, poolVal, countF)
                           ->getResult(0);
             if (accETy.getIntOrFloatBitWidth() >
                 resultETy.getIntOrFloatBitWidth())
               poolVal =
-                  rewriter.create<arith::TruncFOp>(loc, resultETy, poolVal);
+                  arith::TruncFOp::create(rewriter, loc, resultETy, poolVal);
           } else {
 
             // If we have quantization information we need to apply an offset
             // for the input zp value.
             if (inputZpVal != 0) {
-              auto inputZp = rewriter.create<arith::ConstantOp>(
+              auto inputZp = arith::ConstantOp::create(rewriter,
                   loc, b.getIntegerAttr(accETy, inputZpVal));
               Value offset =
-                  rewriter.create<arith::MulIOp>(loc, accETy, count, inputZp);
+                  arith::MulIOp::create(rewriter, loc, accETy, count, inputZp);
               poolVal =
-                  rewriter.create<arith::SubIOp>(loc, accETy, poolVal, offset);
+                  arith::SubIOp::create(rewriter, loc, accETy, poolVal, offset);
             }
 
             // Compute: k = 32 - count_leading_zeros(value - 1)
-            Value one32 = rewriter.create<arith::ConstantOp>(
+            Value one32 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI32IntegerAttr(1));
-            Value thirtyTwo32 = rewriter.create<arith::ConstantOp>(
+            Value thirtyTwo32 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI32IntegerAttr(32));
 
             Value countSubOne =
-                rewriter.create<arith::SubIOp>(loc, count, one32);
+                arith::SubIOp::create(rewriter, loc, count, one32);
             Value leadingZeros =
-                rewriter.create<math::CountLeadingZerosOp>(loc, countSubOne);
+                math::CountLeadingZerosOp::create(rewriter, loc, countSubOne);
             Value k =
-                rewriter.create<arith::SubIOp>(loc, thirtyTwo32, leadingZeros);
+                arith::SubIOp::create(rewriter, loc, thirtyTwo32, leadingZeros);
 
             // Compute: numerator = ((1 << 30) + 1) << k
             Value k64 =
-                rewriter.create<arith::ExtUIOp>(loc, rewriter.getI64Type(), k);
-            Value thirtyShiftPlusOne = rewriter.create<arith::ConstantOp>(
+                arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), k);
+            Value thirtyShiftPlusOne = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
             Value numerator =
-                rewriter.create<arith::ShLIOp>(loc, thirtyShiftPlusOne, k64);
+                arith::ShLIOp::create(rewriter, loc, thirtyShiftPlusOne, k64);
 
             // Compute: scale.multiplier = numerator / value;
-            Value count64 = rewriter.create<arith::ExtUIOp>(
+            Value count64 = arith::ExtUIOp::create(rewriter,
                 loc, rewriter.getI64Type(), count);
             Value multiplier =
-                rewriter.create<arith::DivUIOp>(loc, numerator, count64);
-            multiplier = rewriter.create<arith::TruncIOp>(
+                arith::DivUIOp::create(rewriter, loc, numerator, count64);
+            multiplier = arith::TruncIOp::create(rewriter,
                 loc, rewriter.getI32Type(), multiplier);
 
             // Compute: scale.shift = 30 + k
             Value k8 =
-                rewriter.create<arith::TruncIOp>(loc, rewriter.getI8Type(), k);
-            Value thirty8 = rewriter.create<arith::ConstantOp>(
+                arith::TruncIOp::create(rewriter, loc, rewriter.getI8Type(), k);
+            Value thirty8 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI8IntegerAttr(30));
-            Value shift = rewriter.create<arith::AddIOp>(loc, k8, thirty8);
+            Value shift = arith::AddIOp::create(rewriter, loc, k8, thirty8);
 
             auto scaled =
                 rewriter
@@ -1055,19 +1055,19 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // If we have quantization information we need to apply output
             // zeropoint.
             if (outputZpVal != 0) {
-              auto outputZp = rewriter.create<arith::ConstantOp>(
+              auto outputZp = arith::ConstantOp::create(rewriter,
                   loc, b.getIntegerAttr(scaled.getType(), outputZpVal));
-              scaled = rewriter.create<arith::AddIOp>(loc, scaled, outputZp)
+              scaled = arith::AddIOp::create(rewriter, loc, scaled, outputZp)
                            .getResult();
             }
 
             // Apply Clip.
             int64_t outBitwidth = resultETy.getIntOrFloatBitWidth();
 
-            auto min = rewriter.create<arith::ConstantIntOp>(
+            auto min = arith::ConstantIntOp::create(rewriter,
                 loc, accETy,
                 APInt::getSignedMinValue(outBitwidth).getSExtValue());
-            auto max = rewriter.create<arith::ConstantIntOp>(
+            auto max = arith::ConstantIntOp::create(rewriter,
                 loc, accETy,
                 APInt::getSignedMaxValue(outBitwidth).getSExtValue());
             auto clamp = clampIntHelper(loc, scaled, min, max, rewriter,
@@ -1077,11 +1077,11 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // Convert type.
             if (resultETy != clamp.getType()) {
               poolVal =
-                  rewriter.create<arith::TruncIOp>(loc, resultETy, poolVal);
+                  arith::TruncIOp::create(rewriter, loc, resultETy, poolVal);
             }
           }
 
-          rewriter.create<linalg::YieldOp>(loc, poolVal);
+          linalg::YieldOp::create(rewriter, loc, poolVal);
         });
 
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -1106,7 +1106,7 @@ class TransposeConverter : public OpRewritePattern<tosa::TransposeOp> {
     auto permutedSizes =
         applyTOSAPermutation<OpFoldResult>(inputSizes, constantPerms);
 
-    auto permutedInit = rewriter.create<tensor::EmptyOp>(
+    auto permutedInit = tensor::EmptyOp::create(rewriter,
         loc, permutedSizes, op.getInput1().getType().getElementType());
     rewriter.replaceOpWithNewOp<linalg::TransposeOp>(
         op, op.getInput1(), permutedInit,
diff --git a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
index 7dbccd19a0518..cd36aadce410a 100644
--- a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
+++ b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
@@ -27,7 +27,7 @@ class VariableOpConverter : public OpRewritePattern<tosa::VariableOp> {
   LogicalResult matchAndRewrite(tosa::VariableOp op,
                                 PatternRewriter &rewriter) const final {
     auto variableType = tosa::getVariableType(op);
-    auto newVariable = rewriter.create<mlir::ml_program::GlobalOp>(
+    auto newVariable = mlir::ml_program::GlobalOp::create(rewriter,
         op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
         op.getInitialValueAttr(), /*sym_visibility=*/nullptr);
     newVariable.setPrivate();
@@ -45,7 +45,7 @@ class VariableWriteOpConverter
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableWrite = rewriter.create<ml_program::GlobalStoreOp>(
+    auto newVariableWrite = ml_program::GlobalStoreOp::create(rewriter,
         op.getLoc(), globalSymbolRef, op.getInput1());
     rewriter.replaceOp(op, newVariableWrite);
     return success();
@@ -60,7 +60,7 @@ class VariableReadOpConverter : public OpRewritePattern<tosa::VariableReadOp> {
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableRead = rewriter.create<ml_program::GlobalLoadOp>(
+    auto newVariableRead = ml_program::GlobalLoadOp::create(rewriter,
         op.getLoc(), op.getType(), globalSymbolRef);
     rewriter.replaceOp(op, newVariableRead);
 
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index 03f9d20ad69de..9188b8d1368da 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -30,7 +30,7 @@ static void inlineIfCase(Region &srcRegion, Region &dstRegion,
 
   auto yield = cast<YieldOp>(headBlock->getTerminator());
   rewriter.setInsertionPoint(yield);
-  rewriter.create<scf::YieldOp>(yield.getLoc(), yield.getInputs());
+  scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
   rewriter.eraseOp(yield);
 
   headBlock->eraseArguments(0, headBlock->getNumArguments());
@@ -47,12 +47,12 @@ static void inlineWhileCase(Region &srcRegion, Region &dstRegion,
   rewriter.setInsertionPoint(yield);
   if (isCond) {
     auto condition =
-        rewriter.create<tensor::ExtractOp>(yield.getLoc(), yield.getOperand(0));
-    rewriter.create<scf::ConditionOp>(yield.getLoc(), condition,
+        tensor::ExtractOp::create(rewriter, yield.getLoc(), yield.getOperand(0));
+    scf::ConditionOp::create(rewriter, yield.getLoc(), condition,
                                       headBlock->getArguments());
   } else {
     rewriter.setInsertionPoint(yield);
-    rewriter.create<scf::YieldOp>(yield.getLoc(), yield.getInputs());
+    scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
   }
   rewriter.eraseOp(yield);
 }
@@ -66,8 +66,8 @@ class IfOpConverter : public OpRewritePattern<tosa::IfOp> {
   LogicalResult matchAndRewrite(tosa::IfOp op,
                                 PatternRewriter &rewriter) const final {
     auto condition =
-        rewriter.create<tensor::ExtractOp>(op.getLoc(), op.getCondition());
-    auto newIf = rewriter.create<scf::IfOp>(op.getLoc(), op.getResultTypes(),
+        tensor::ExtractOp::create(rewriter, op.getLoc(), op.getCondition());
+    auto newIf = scf::IfOp::create(rewriter, op.getLoc(), op.getResultTypes(),
                                             condition, true);
 
     inlineIfCase(op.getThenGraph(), newIf.getThenRegion(), op.getInputList(),
@@ -88,7 +88,7 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
 
   static Value createIndexConst(OpBuilder &builder, Location loc,
                                 int64_t value) {
-    return builder.create<arith::ConstantIndexOp>(loc, value);
+    return arith::ConstantIndexOp::create(builder, loc, value);
   }
 
 public:
@@ -119,8 +119,8 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       auto n = ivs[0];
 
       // Read the index and cast it to index type
-      auto index = builder.create<tensor::ExtractOp>(loc, indices, ivs);
-      auto castIndex = builder.create<arith::IndexCastOp>(
+      auto index = tensor::ExtractOp::create(builder, loc, indices, ivs);
+      auto castIndex = arith::IndexCastOp::create(builder,
           loc, builder.getIndexType(), index);
 
       // Offset, sizes, and strides for the input tensor
@@ -130,12 +130,12 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       llvm::SmallVector<Value> sizes = {one, one, dimC};
       llvm::SmallVector<Value> strides = {one, one, one};
 
-      auto slice = builder.create<tensor::ExtractSliceOp>(
+      auto slice = tensor::ExtractSliceOp::create(builder,
           loc, input, inputOffset, sizes, strides);
 
       // Insert the slice into the output accumulator tensor.
       llvm::SmallVector<Value> outputOffset = {n, castIndex, zero};
-      auto updated = builder.create<tensor::InsertSliceOp>(
+      auto updated = tensor::InsertSliceOp::create(builder,
           loc, slice, args[0], outputOffset, sizes, strides);
 
       return {updated};
@@ -155,7 +155,7 @@ class WhileOpConverter : public OpRewritePattern<tosa::WhileOp> {
 
   LogicalResult matchAndRewrite(tosa::WhileOp op,
                                 PatternRewriter &rewriter) const final {
-    auto newWhile = rewriter.create<scf::WhileOp>(
+    auto newWhile = scf::WhileOp::create(rewriter,
         op.getLoc(), op.getResultTypes(), op.getInputList());
     rewriter.createBlock(&newWhile.getBefore());
     rewriter.createBlock(&newWhile.getAfter());
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index 9ceb5c0c7f2fe..e51cbfb702083 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -308,13 +308,13 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
       if (ShapedType::isStatic(sizes.back()))
         continue;
 
-      auto dim = rewriter.create<tensor::DimOp>(loc, input, index);
-      auto offset = rewriter.create<arith::ConstantOp>(
+      auto dim = tensor::DimOp::create(rewriter, loc, input, index);
+      auto offset = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(sliceStarts[index]));
-      dynSizes.push_back(rewriter.create<arith::SubIOp>(loc, dim, offset));
+      dynSizes.push_back(arith::SubIOp::create(rewriter, loc, dim, offset));
     }
 
-    auto newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}), dynSizes,
         ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
         rewriter.getDenseI64ArrayAttr(sizes),
@@ -362,7 +362,7 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
 
     Value padConstant = rewriter.createOrFold<tensor::ExtractOp>(
         loc, padOp.getPadConst(),
-        ValueRange({rewriter.create<arith::ConstantIndexOp>(loc, 0)}));
+        ValueRange({arith::ConstantIndexOp::create(rewriter, loc, 0)}));
 
     if (!padConstant) {
       return rewriter.notifyMatchFailure(
@@ -376,15 +376,15 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
     highValues.reserve(rank);
 
     for (int i = 0; i < rank; i++) {
-      Value lowVal = rewriter.create<arith::ConstantOp>(
+      Value lowVal = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(paddingVals[2 * i]));
-      Value highVal = rewriter.create<arith::ConstantOp>(
+      Value highVal = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(paddingVals[2 * i + 1]));
       lowValues.push_back(lowVal);
       highValues.push_back(highVal);
     }
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, padOp.getType(), input, lowValues, highValues, padConstant);
 
     rewriter.replaceOp(padOp, newPadOp.getResult());
@@ -403,7 +403,7 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
     Location loc = op.getLoc();
     int axis = op.getAxis();
     Value axisValue =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(axis));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(axis));
     int64_t rank = resultType.getRank();
 
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
@@ -440,7 +440,7 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
       }
     }
 
-    Value result = rewriter.create<tensor::EmptyOp>(
+    Value result = tensor::EmptyOp::create(rewriter,
         loc, resultType.getShape(), resultType.getElementType(), dynDims);
 
     for (auto [arg, offset] : llvm::zip(adaptor.getOperands(), axisOffsets)) {
diff --git a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
index d6f9495b2567c..9e4eb347b3689 100644
--- a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
+++ b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
@@ -226,7 +226,7 @@ struct BroadcastOpToArmSMELowering
         (srcVectorType && (srcVectorType.getRank() == 0))) {
       // Broadcast scalar or 0-d vector to 1-d vector.
       VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-      broadcastOp1D = rewriter.create<vector::BroadcastOp>(
+      broadcastOp1D = vector::BroadcastOp::create(rewriter,
           loc, tileSliceType, broadcastOp.getSource());
     } else if (srcVectorType && (srcVectorType.getRank() == 1))
       // Value to broadcast is already a 1-d vector, nothing to do.
@@ -234,13 +234,13 @@ struct BroadcastOpToArmSMELowering
     else
       return failure();
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to broadcast the value
       // to each tile slice.
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
@@ -292,14 +292,14 @@ struct SplatOpToArmSMELowering : public OpRewritePattern<vector::SplatOp> {
 
     // First, broadcast the scalar to a 1-d vector.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    Value broadcastOp1D = rewriter.create<vector::BroadcastOp>(
+    Value broadcastOp1D = vector::BroadcastOp::create(rewriter,
         loc, tileSliceType, splatOp.getInput());
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
@@ -370,21 +370,21 @@ struct TransposeOpToArmSMELowering
 
     // Allocate buffer to store input tile to.
     Value vscale =
-        rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-    Value minTileSlices = rewriter.create<arith::ConstantOp>(
+        vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+    Value minTileSlices = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexAttr(tileType.getDimSize(0)));
     Value c0 =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(0));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(0));
     Value numTileSlices =
-        rewriter.create<arith::MulIOp>(loc, vscale, minTileSlices);
+        arith::MulIOp::create(rewriter, loc, vscale, minTileSlices);
     auto bufferType =
         MemRefType::get({ShapedType::kDynamic, ShapedType::kDynamic},
                         tileType.getElementType());
-    auto buffer = rewriter.create<memref::AllocaOp>(
+    auto buffer = memref::AllocaOp::create(rewriter,
         loc, bufferType, ValueRange{numTileSlices, numTileSlices});
 
     // Store input tile.
-    auto tileStoreOp = rewriter.create<arm_sme::TileStoreOp>(
+    auto tileStoreOp = arm_sme::TileStoreOp::create(rewriter,
         loc, input, buffer, ValueRange{c0, c0});
 
     // Reload input tile vertically.
@@ -489,9 +489,9 @@ struct VectorOuterProductToArmSMELowering
 
     VectorType operandMaskType = VectorType::Builder(maskType).dropDim(0);
     Value lhsMask =
-        rewriter.create<vector::CreateMaskOp>(loc, operandMaskType, lhsMaskDim);
+        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, lhsMaskDim);
     Value rhsMask =
-        rewriter.create<vector::CreateMaskOp>(loc, operandMaskType, rhsMaskDim);
+        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, rhsMaskDim);
 
     return std::make_pair(lhsMask, rhsMask);
   }
@@ -531,7 +531,7 @@ struct VectorExtractToArmSMELowering
     }
 
     Value sliceIndex = vector::getAsValues(rewriter, loc, position[0]).front();
-    auto extractTileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+    auto extractTileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
         loc, sourceVector, sliceIndex);
 
     if (position.size() == 1) {
@@ -593,9 +593,9 @@ struct VectorInsertToArmSMELowering
     if (position.size() == 2) {
       // Two indices case: Insert single element into tile.
       // We need to first extract the existing slice and update the element.
-      tileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+      tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
           loc, insertOp.getDest(), sliceIndex);
-      tileSlice = rewriter.create<vector::InsertOp>(loc, source, tileSlice,
+      tileSlice = vector::InsertOp::create(rewriter, loc, source, tileSlice,
                                                     position[1]);
     }
 
@@ -642,22 +642,22 @@ struct VectorPrintToArmSMELowering : public OpRewritePattern<vector::PrintOp> {
     auto loc = printOp.getLoc();
 
     // Create a loop over the rows of the tile.
-    auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
+    auto vscale = vector::VectorScaleOp::create(rewriter, loc);
     auto minTileRows =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorType.getDimSize(0));
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto upperBound = rewriter.create<arith::MulIOp>(loc, minTileRows, vscale);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        arith::ConstantIndexOp::create(rewriter, loc, vectorType.getDimSize(0));
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto upperBound = arith::MulIOp::create(rewriter, loc, minTileRows, vscale);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     {
       // Loop body.
       rewriter.setInsertionPointToStart(forOp.getBody());
       // Extract the current row from the tile.
       Value rowIndex = forOp.getInductionVar();
-      auto tileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+      auto tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
           loc, printOp.getSource(), rowIndex);
       // Print the row with a 1D vector.print.
-      rewriter.create<vector::PrintOp>(loc, tileSlice,
+      vector::PrintOp::create(rewriter, loc, tileSlice,
                                        printOp.getPunctuation());
     }
 
@@ -707,7 +707,7 @@ struct FoldTransferWriteOfExtractTileSlice
     Value mask = writeOp.getMask();
     if (!mask) {
       auto maskType = writeOp.getVectorType().clone(rewriter.getI1Type());
-      mask = rewriter.create<arith::ConstantOp>(
+      mask = arith::ConstantOp::create(rewriter,
           writeOp.getLoc(), maskType, DenseElementsAttr::get(maskType, true));
     }
 
@@ -776,9 +776,9 @@ struct ExtractFromCreateMaskToPselLowering
     // Create the two 1-D masks at the location of the 2-D create_mask (which is
     // usually outside a loop). This prevents the need for later hoisting.
     rewriter.setInsertionPoint(createMaskOp);
-    auto rowMask = rewriter.create<vector::CreateMaskOp>(
+    auto rowMask = vector::CreateMaskOp::create(rewriter,
         loc, rowMaskType, createMaskOp.getOperand(0));
-    auto colMask = rewriter.create<vector::CreateMaskOp>(
+    auto colMask = vector::CreateMaskOp::create(rewriter,
         loc, colMaskType, createMaskOp.getOperand(1));
 
     rewriter.setInsertionPoint(extractOp);
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 18adaa793787c..5b3e67509672a 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -412,22 +412,22 @@ struct PrepareContractToGPUMMA
     if (maps == infer({{m, k}, {k, n}, {m, n}}))
       return rewriter.notifyMatchFailure(op, "contraction already prepared");
     if (maps == infer({{m, k}, {n, k}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
       std::swap(rhs, lhs);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
       std::swap(rhs, lhs);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
       std::swap(lhs, rhs);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
       std::swap(lhs, rhs);
     } else {
@@ -494,13 +494,13 @@ struct CombineTransferReadOpTranspose final
     // Fuse through the integer extend op.
     if (extOp) {
       if (isa<arith::ExtSIOp>(extOp))
-        result = rewriter.create<arith::ExtSIOp>(loc, op.getType(), result)
+        result = arith::ExtSIOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
       else if (isa<arith::ExtUIOp>(extOp))
-        result = rewriter.create<arith::ExtUIOp>(loc, op.getType(), result)
+        result = arith::ExtUIOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
       else
-        result = rewriter.create<arith::ExtFOp>(loc, op.getType(), result)
+        result = arith::ExtFOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
     }
 
@@ -579,7 +579,7 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op,
   }
   gpu::MMAMatrixType type =
       gpu::MMAMatrixType::get(op.getVectorType().getShape(), elType, fragType);
-  Value load = rewriter.create<gpu::SubgroupMmaLoadMatrixOp>(
+  Value load = gpu::SubgroupMmaLoadMatrixOp::create(rewriter,
       op.getLoc(), type, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride),
       isTranspose ? rewriter.getUnitAttr() : UnitAttr());
@@ -610,7 +610,7 @@ convertTransferWriteOp(RewriterBase &rewriter, vector::TransferWriteOp op,
   }
 
   Value matrix = it->second;
-  auto store = rewriter.create<gpu::SubgroupMmaStoreMatrixOp>(
+  auto store = gpu::SubgroupMmaStoreMatrixOp::create(rewriter,
       op.getLoc(), matrix, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride), /*transpose=*/UnitAttr());
   (void)store;
@@ -661,7 +661,7 @@ convertConstantOpMmaSync(RewriterBase &rewriter, arith::ConstantOp op,
     return rewriter.notifyMatchFailure(op, "not a splat");
   }
 
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       op.getLoc(), vectorType,
       DenseElementsAttr::get(vectorType, dense.getSplatValue<Attribute>()));
   valueMapping[op.getResult()] = result;
@@ -743,7 +743,7 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   }
 
   // Adjust the load offset.
-  auto laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  auto laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
   FailureOr<AffineMap> offsets =
       nvgpu::getLaneIdToLdMatrixMatrixCoord(rewriter, loc, *params);
   if (failed(offsets)) {
@@ -757,7 +757,7 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   getXferIndices<vector::TransferReadOp>(rewriter, op, *offsets, {laneId},
                                          indices);
 
-  nvgpu::LdMatrixOp newOp = rewriter.create<nvgpu::LdMatrixOp>(
+  nvgpu::LdMatrixOp newOp = nvgpu::LdMatrixOp::create(rewriter,
       loc, vectorType, op.getBase(), indices, *transpose, params->numTiles);
   valueMapping[op] = newOp->getResult(0);
   return success();
@@ -782,17 +782,17 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
             "conversion to distributed non-ldmatrix compatible load");
   }
 
-  Value laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   // This is the individual element type.
   Type loadedElType = regInfo->registerLLVMType;
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
 
-  Value fill = rewriter.create<arith::ConstantOp>(
+  Value fill = arith::ConstantOp::create(rewriter,
       op.getLoc(), vectorType.getElementType(),
       rewriter.getZeroAttr(vectorType.getElementType()));
   Value result =
-      rewriter.create<vector::SplatOp>(op.getLoc(), fill, vectorType);
+      vector::SplatOp::create(rewriter, op.getLoc(), fill, vectorType);
 
   bool isTransposeLoad = !op.getPermutationMap().isMinorIdentity();
 
@@ -809,16 +809,16 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       if (failed(coords))
         return rewriter.notifyMatchFailure(op, "no coords");
 
-      Value logicalValueId = rewriter.create<arith::ConstantOp>(
+      Value logicalValueId = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexType(),
           rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
       SmallVector<Value, 4> newIndices;
       getXferIndices<vector::TransferReadOp>(
           rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
 
-      Value el = rewriter.create<vector::LoadOp>(loc, loadedElType,
+      Value el = vector::LoadOp::create(rewriter, loc, loadedElType,
                                                  op.getBase(), newIndices);
-      result = rewriter.create<vector::InsertOp>(loc, el, result, i);
+      result = vector::InsertOp::create(rewriter, loc, el, result, i);
     }
   } else {
     if (auto vecType = dyn_cast<VectorType>(loadedElType)) {
@@ -828,7 +828,7 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       for (unsigned innerIdx = 0; innerIdx < vectorType.getShape()[1];
            innerIdx++) {
 
-        Value logicalValueId = rewriter.create<arith::ConstantOp>(
+        Value logicalValueId = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIndexType(),
             rewriter.getIndexAttr(i * regInfo->elementsPerRegister + innerIdx));
         FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
@@ -839,9 +839,9 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
         SmallVector<Value, 4> newIndices;
         getXferIndices<vector::TransferReadOp>(
             rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
-        Value el = rewriter.create<memref::LoadOp>(op.getLoc(), loadedElType,
+        Value el = memref::LoadOp::create(rewriter, op.getLoc(), loadedElType,
                                                    op.getBase(), newIndices);
-        result = rewriter.create<vector::InsertOp>(
+        result = vector::InsertOp::create(rewriter,
             op.getLoc(), el, result, ArrayRef<int64_t>{i, innerIdx});
       }
     }
@@ -916,10 +916,10 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
     return rewriter.notifyMatchFailure(op, "not mma sync reg info");
 
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
-  Value laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   for (unsigned i = 0; i < vectorType.getShape()[0]; i++) {
-    Value logicalValueId = rewriter.create<arith::ConstantOp>(
+    Value logicalValueId = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexType(),
         rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
     FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
@@ -928,11 +928,11 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
       return rewriter.notifyMatchFailure(op, "no coords");
 
     Value el =
-        rewriter.create<vector::ExtractOp>(loc, matrix, ArrayRef<int64_t>{i});
+        vector::ExtractOp::create(rewriter, loc, matrix, ArrayRef<int64_t>{i});
     SmallVector<Value, 4> newIndices;
     getXferIndices<vector::TransferWriteOp>(
         rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
-    rewriter.create<vector::StoreOp>(loc, el, op.getBase(), newIndices);
+    vector::StoreOp::create(rewriter, loc, el, op.getBase(), newIndices);
   }
 
   LLVM_DEBUG(DBGS() << "erase: " << op << "\n");
@@ -1015,7 +1015,7 @@ convertExtractStridedSlice(RewriterBase &rewriter,
   else if (offsets[1])
     sliceOffset[0] = (warpVectorShape[1] / offsets[1]);
 
-  Value newOp = rewriter.create<vector::ExtractStridedSliceOp>(
+  Value newOp = vector::ExtractStridedSliceOp::create(rewriter,
       loc, sourceVector, sliceOffset, sliceShape, strides);
 
   valueMapping[op] = newOp;
@@ -1035,7 +1035,7 @@ convertContractOp(RewriterBase &rewriter, vector::ContractionOp op,
       itC == valueMapping.end())
     return rewriter.notifyMatchFailure(op, "no mapping");
   Value opA = itA->second, opB = itB->second, opC = itC->second;
-  Value matmul = rewriter.create<gpu::SubgroupMmaComputeOp>(
+  Value matmul = gpu::SubgroupMmaComputeOp::create(rewriter,
       op.getLoc(), opC.getType(), opA, opB, opC, /*a_transpose=*/UnitAttr(),
       /*b_transpose=*/UnitAttr());
   valueMapping[op.getResult()] = matmul;
@@ -1058,7 +1058,7 @@ convertContractOpToMmaSync(RewriterBase &rewriter, vector::ContractionOp op,
   int64_t m = cast<VectorType>(op.getLhs().getType()).getShape()[0];
   int64_t n = cast<VectorType>(op.getRhs().getType()).getShape()[0];
   int64_t k = cast<VectorType>(op.getLhs().getType()).getShape()[1];
-  Value matmul = rewriter.create<nvgpu::MmaSyncOp>(
+  Value matmul = nvgpu::MmaSyncOp::create(rewriter,
       op.getLoc(), opA, opB, opC, rewriter.getI64ArrayAttr({m, n, k}));
   valueMapping[op.getResult()] = matmul;
   return success();
@@ -1076,12 +1076,12 @@ convertConstantOp(RewriterBase &rewriter, arith::ConstantOp op,
   auto splat =
       cast<SplatElementsAttr>(op.getValue()).getSplatValue<TypedAttr>();
   auto scalarConstant =
-      rewriter.create<arith::ConstantOp>(op.getLoc(), splat.getType(), splat);
+      arith::ConstantOp::create(rewriter, op.getLoc(), splat.getType(), splat);
   const char *fragType = inferFragType(op);
   auto vecType = cast<VectorType>(op.getType());
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = rewriter.create<gpu::SubgroupMmaConstantMatrixOp>(
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
       op.getLoc(), type, scalarConstant);
   valueMapping[op.getResult()] = matrix;
   return success();
@@ -1100,7 +1100,7 @@ convertBroadcastOp(RewriterBase &rewriter, vector::BroadcastOp op,
   auto vecType = op.getResultVectorType();
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = rewriter.create<gpu::SubgroupMmaConstantMatrixOp>(
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
       op.getLoc(), type, op.getSource());
   valueMapping[op.getResult()] = matrix;
   return success();
@@ -1118,7 +1118,7 @@ static scf::ForOp replaceForOpWithNewSignature(RewriterBase &rewriter,
   rewriter.setInsertionPoint(loop);
   auto operands = llvm::to_vector<4>(loop.getInitArgs());
   llvm::append_range(operands, newInitArgs);
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
       operands);
   rewriter.eraseBlock(newLoop.getBody());
@@ -1189,7 +1189,7 @@ convertYieldOp(RewriterBase &rewriter, scf::YieldOp op,
     yieldOperands[operand.index()] = loop.getInitArgs()[operand.index()];
     yieldOperands.push_back(it->second);
   }
-  rewriter.create<scf::YieldOp>(op.getLoc(), yieldOperands);
+  scf::YieldOp::create(rewriter, op.getLoc(), yieldOperands);
 
   LLVM_DEBUG(DBGS() << "erase: " << op << "\n");
   rewriter.eraseOp(op);
@@ -1220,7 +1220,7 @@ convertElementwiseOp(RewriterBase &rewriter, Operation *op,
                                          resultType.getOperand());
   }
 
-  Value newOp = rewriter.create<gpu::SubgroupMmaElementwiseOp>(
+  Value newOp = gpu::SubgroupMmaElementwiseOp::create(rewriter,
       op->getLoc(), resultType, matrixOperands, opType);
   valueMapping[op->getResult(0)] = newOp;
   return success();
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 501d98862672d..1b31d391cb0bc 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -43,13 +43,13 @@ static Value insertOne(ConversionPatternRewriter &rewriter,
   assert(rank > 0 && "0-D vector corner case should have been handled already");
   if (rank == 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = rewriter.create<LLVM::ConstantOp>(
+    auto constant = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
-    return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2,
+    return LLVM::InsertElementOp::create(rewriter, loc, llvmType, val1, val2,
                                                   constant);
   }
-  return rewriter.create<LLVM::InsertValueOp>(loc, val1, val2, pos);
+  return LLVM::InsertValueOp::create(rewriter, loc, val1, val2, pos);
 }
 
 // Helper that picks the proper sequence for extracting.
@@ -58,13 +58,13 @@ static Value extractOne(ConversionPatternRewriter &rewriter,
                         Value val, Type llvmType, int64_t rank, int64_t pos) {
   if (rank <= 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = rewriter.create<LLVM::ConstantOp>(
+    auto constant = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
-    return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val,
+    return LLVM::ExtractElementOp::create(rewriter, loc, llvmType, val,
                                                    constant);
   }
-  return rewriter.create<LLVM::ExtractValueOp>(loc, val, pos);
+  return LLVM::ExtractValueOp::create(rewriter, loc, val, pos);
 }
 
 // Helper that returns data layout alignment of a vector.
@@ -141,7 +141,7 @@ static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc,
   auto ptrsType =
       LLVM::getVectorType(pType, vectorType.getDimSize(0),
                           /*isScalable=*/vectorType.getScalableDims()[0]);
-  return rewriter.create<LLVM::GEPOp>(
+  return LLVM::GEPOp::create(rewriter,
       loc, ptrsType, typeConverter.convertType(memRefType.getElementType()),
       base, index);
 }
@@ -152,7 +152,7 @@ static Value getAsLLVMValue(OpBuilder &builder, Location loc,
                             OpFoldResult foldResult) {
   if (auto attr = dyn_cast<Attribute>(foldResult)) {
     auto intAttr = cast<IntegerAttr>(attr);
-    return builder.create<LLVM::ConstantOp>(loc, intAttr).getResult();
+    return LLVM::ConstantOp::create(builder, loc, intAttr).getResult();
   }
 
   return cast<Value>(foldResult);
@@ -475,7 +475,7 @@ class ReductionNeutralFPMax {};
 static Value createReductionNeutralValue(ReductionNeutralZero neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmType,
+  return LLVM::ConstantOp::create(rewriter, loc, llvmType,
                                            rewriter.getZeroAttr(llvmType));
 }
 
@@ -483,7 +483,7 @@ static Value createReductionNeutralValue(ReductionNeutralZero neutral,
 static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType, rewriter.getIntegerAttr(llvmType, 1));
 }
 
@@ -491,7 +491,7 @@ static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
 static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
 }
 
@@ -499,7 +499,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
 static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(
           llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth())));
@@ -509,7 +509,7 @@ static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -519,7 +519,7 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -529,7 +529,7 @@ static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -539,7 +539,7 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -550,7 +550,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
@@ -562,7 +562,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
@@ -591,7 +591,7 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
   auto vShape = vType.getShape();
   assert(vShape.size() == 1 && "Unexpected multi-dim vector type");
 
-  Value baseVecLength = rewriter.create<LLVM::ConstantOp>(
+  Value baseVecLength = LLVM::ConstantOp::create(rewriter,
       loc, rewriter.getI32Type(),
       rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0]));
 
@@ -599,11 +599,11 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
     return baseVecLength;
 
   // For a scalable vector type, create and return `vScale * baseVecLength`.
-  Value vScale = rewriter.create<vector::VectorScaleOp>(loc);
+  Value vScale = vector::VectorScaleOp::create(rewriter, loc);
   vScale =
-      rewriter.create<arith::IndexCastOp>(loc, rewriter.getI32Type(), vScale);
+      arith::IndexCastOp::create(rewriter, loc, rewriter.getI32Type(), vScale);
   Value scalableVecLength =
-      rewriter.create<arith::MulIOp>(loc, baseVecLength, vScale);
+      arith::MulIOp::create(rewriter, loc, baseVecLength, vScale);
   return scalableVecLength;
 }
 
@@ -616,10 +616,10 @@ static Value createIntegerReductionArithmeticOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator) {
 
-  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
 
   if (accumulator)
-    result = rewriter.create<ScalarOp>(loc, accumulator, result);
+    result = ScalarOp::create(rewriter, loc, accumulator, result);
   return result;
 }
 
@@ -631,11 +631,11 @@ template <class LLVMRedIntrinOp>
 static Value createIntegerReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) {
-  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
   if (accumulator) {
     Value cmp =
-        rewriter.create<LLVM::ICmpOp>(loc, predicate, accumulator, result);
-    result = rewriter.create<LLVM::SelectOp>(loc, cmp, accumulator, result);
+        LLVM::ICmpOp::create(rewriter, loc, predicate, accumulator, result);
+    result = LLVM::SelectOp::create(rewriter, loc, cmp, accumulator, result);
   }
   return result;
 }
@@ -666,11 +666,11 @@ static Value createFPReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::FastmathFlagsAttr fmf) {
   Value result =
-      rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand, fmf);
+      LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand, fmf);
 
   if (accumulator) {
     result =
-        rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
+        typename VectorToScalarMapper<LLVMRedIntrinOp>::Type::create(rewriter,
             loc, result, accumulator);
   }
 
@@ -702,7 +702,7 @@ static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
   const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
   auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
   auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
-  return rewriter.create<LLVM::ConstantOp>(loc, vectorType, denseValue);
+  return LLVM::ConstantOp::create(rewriter, loc, vectorType, denseValue);
 }
 
 /// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
@@ -717,7 +717,7 @@ lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
                                 Value mask, LLVM::FastmathFlagsAttr fmf) {
   const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
       rewriter, loc, llvmType, vectorOperand.getType());
-  const Value selectedVectorByMask = rewriter.create<LLVM::SelectOp>(
+  const Value selectedVectorByMask = LLVM::SelectOp::create(rewriter,
       loc, mask, vectorOperand, vectorMaskNeutral);
   return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
       rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
@@ -730,7 +730,7 @@ lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
                              Value accumulator, LLVM::FastmathFlagsAttr fmf) {
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
-  return rewriter.create<LLVMRedIntrinOp>(loc, llvmType,
+  return LLVMRedIntrinOp::create(rewriter, loc, llvmType,
                                           /*startValue=*/accumulator,
                                           vectorOperand, fmf);
 }
@@ -745,7 +745,7 @@ lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter,
                                        Value vectorOperand, Value accumulator) {
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
-  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
+  return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
                                             /*startValue=*/accumulator,
                                             vectorOperand);
 }
@@ -758,7 +758,7 @@ static Value lowerPredicatedReductionWithStartValue(
                                                          llvmType, accumulator);
   Value vectorLength =
       createVectorLengthValue(rewriter, loc, vectorOperand.getType());
-  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
+  return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
                                             /*startValue=*/accumulator,
                                             vectorOperand, mask, vectorLength);
 }
@@ -1071,7 +1071,7 @@ class VectorShuffleOpConversion
     // For rank 0 and 1, where both operands have *exactly* the same vector
     // type, there is direct shuffle support in LLVM. Use it!
     if (rank <= 1 && v1Type == v2Type) {
-      Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>(
+      Value llvmShuffleOp = LLVM::ShuffleVectorOp::create(rewriter,
           loc, adaptor.getV1(), adaptor.getV2(),
           llvm::to_vector_of<int32_t>(mask));
       rewriter.replaceOp(shuffleOp, llvmShuffleOp);
@@ -1085,7 +1085,7 @@ class VectorShuffleOpConversion
       eltType = arrayType.getElementType();
     else
       eltType = cast<VectorType>(llvmType).getElementType();
-    Value insert = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
+    Value insert = LLVM::PoisonOp::create(rewriter, loc, llvmType);
     int64_t insPos = 0;
     for (int64_t extPos : mask) {
       Value value = adaptor.getV1();
@@ -1122,7 +1122,7 @@ class VectorExtractElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = extractEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = rewriter.create<LLVM::ConstantOp>(
+      auto zero = LLVM::ConstantOp::create(rewriter,
           loc, typeConverter->convertType(idxType),
           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
@@ -1193,12 +1193,12 @@ class VectorExtractOpConversion
       if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) {
         return failure();
       }
-      extracted = rewriter.create<LLVM::ExtractValueOp>(
+      extracted = LLVM::ExtractValueOp::create(rewriter,
           loc, extracted, getAsIntegers(position));
     }
 
     if (extractsScalar) {
-      extracted = rewriter.create<LLVM::ExtractElementOp>(
+      extracted = LLVM::ExtractElementOp::create(rewriter,
           loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back()));
     }
 
@@ -1256,7 +1256,7 @@ class VectorInsertElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = insertEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = rewriter.create<LLVM::ConstantOp>(
+      auto zero = LLVM::ConstantOp::create(rewriter,
           loc, typeConverter->convertType(idxType),
           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
@@ -1342,7 +1342,7 @@ class VectorInsertOpConversion
           // llvm.extractvalue does not support dynamic dimensions.
           return failure();
         }
-        sourceAggregate = rewriter.create<LLVM::ExtractValueOp>(
+        sourceAggregate = LLVM::ExtractValueOp::create(rewriter,
             loc, adaptor.getDest(),
             getAsIntegers(positionOf1DVectorWithinAggregate));
       } else {
@@ -1351,7 +1351,7 @@ class VectorInsertOpConversion
         sourceAggregate = adaptor.getDest();
       }
       // Insert the scalar into the 1D vector.
-      sourceAggregate = rewriter.create<LLVM::InsertElementOp>(
+      sourceAggregate = LLVM::InsertElementOp::create(rewriter,
           loc, sourceAggregate.getType(), sourceAggregate,
           adaptor.getValueToStore(),
           getAsLLVMValue(rewriter, loc, positionOfScalarWithin1DVector));
@@ -1359,7 +1359,7 @@ class VectorInsertOpConversion
 
     Value result = sourceAggregate;
     if (isNestedAggregate) {
-      result = rewriter.create<LLVM::InsertValueOp>(
+      result = LLVM::InsertValueOp::create(rewriter,
           loc, adaptor.getDest(), sourceAggregate,
           getAsIntegers(positionOf1DVectorWithinAggregate));
     }
@@ -1439,15 +1439,15 @@ class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
 
     auto loc = op.getLoc();
     auto elemType = vType.getElementType();
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero);
+    Value desc = vector::SplatOp::create(rewriter, loc, vType, zero);
     for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
-      Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i);
-      Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i);
-      Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i);
-      Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC);
-      desc = rewriter.create<InsertOp>(loc, fma, desc, i);
+      Value extrLHS = ExtractOp::create(rewriter, loc, op.getLhs(), i);
+      Value extrRHS = ExtractOp::create(rewriter, loc, op.getRhs(), i);
+      Value extrACC = ExtractOp::create(rewriter, loc, op.getAcc(), i);
+      Value fma = FMAOp::create(rewriter, loc, extrLHS, extrRHS, extrACC);
+      desc = InsertOp::create(rewriter, loc, fma, desc, i);
     }
     rewriter.replaceOp(op, desc);
     return success();
@@ -1537,7 +1537,7 @@ class VectorTypeCastOpConversion
     desc.setAlignedPtr(rewriter, loc, ptr);
     // Fill offset 0.
     auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0);
-    auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr);
+    auto zero = LLVM::ConstantOp::create(rewriter, loc, int64Ty, attr);
     desc.setOffset(rewriter, loc, zero);
 
     // Fill size and stride descriptors in memref.
@@ -1546,11 +1546,11 @@ class VectorTypeCastOpConversion
       int64_t index = indexedSize.index();
       auto sizeAttr =
           rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value());
-      auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr);
+      auto size = LLVM::ConstantOp::create(rewriter, loc, int64Ty, sizeAttr);
       desc.setSize(rewriter, loc, index, size);
       auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(),
                                                 (*targetStrides)[index]);
-      auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr);
+      auto stride = LLVM::ConstantOp::create(rewriter, loc, int64Ty, strideAttr);
       desc.setStride(rewriter, loc, index, stride);
     }
 
@@ -1578,13 +1578,13 @@ class VectorCreateMaskOpConversion
     IntegerType idxType =
         force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
     auto loc = op->getLoc();
-    Value indices = rewriter.create<LLVM::StepVectorOp>(
+    Value indices = LLVM::StepVectorOp::create(rewriter,
         loc, LLVM::getVectorType(idxType, dstType.getShape()[0],
                                  /*isScalable=*/true));
     auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType,
                                                  adaptor.getOperands()[0]);
-    Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound);
-    Value comp = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+    Value bounds = SplatOp::create(rewriter, loc, indices.getType(), bound);
+    Value comp = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                 indices, bounds);
     rewriter.replaceOp(op, comp);
     return success();
@@ -1741,15 +1741,15 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 
     switch (conversion) {
     case PrintConversion::ZeroExt64:
-      value = rewriter.create<arith::ExtUIOp>(
+      value = arith::ExtUIOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::SignExt64:
-      value = rewriter.create<arith::ExtSIOp>(
+      value = arith::ExtSIOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::Bitcast16:
-      value = rewriter.create<LLVM::BitcastOp>(
+      value = LLVM::BitcastOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 16), value);
       break;
     case PrintConversion::None:
@@ -1762,7 +1762,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
   // Helper to emit a call.
   static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
                        Operation *ref, ValueRange params = ValueRange()) {
-    rewriter.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(ref),
+    LLVM::CallOp::create(rewriter, loc, TypeRange(), SymbolRefAttr::get(ref),
                                   params);
   }
 };
@@ -1782,8 +1782,8 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     // First insert it into a poison vector so we can shuffle it.
     auto vectorType = typeConverter->convertType(splatOp.getType());
     Value poison =
-        rewriter.create<LLVM::PoisonOp>(splatOp.getLoc(), vectorType);
-    auto zero = rewriter.create<LLVM::ConstantOp>(
+        LLVM::PoisonOp::create(rewriter, splatOp.getLoc(), vectorType);
+    auto zero = LLVM::ConstantOp::create(rewriter,
         splatOp.getLoc(),
         typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
@@ -1796,7 +1796,7 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     }
 
     // For 1-d vector, we additionally do a `vectorshuffle`.
-    auto v = rewriter.create<LLVM::InsertElementOp>(
+    auto v = LLVM::InsertElementOp::create(rewriter,
         splatOp.getLoc(), vectorType, poison, adaptor.getInput(), zero);
 
     int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
@@ -1832,26 +1832,26 @@ struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
       return failure();
 
     // Construct returned value.
-    Value desc = rewriter.create<LLVM::PoisonOp>(loc, llvmNDVectorTy);
+    Value desc = LLVM::PoisonOp::create(rewriter, loc, llvmNDVectorTy);
 
     // Construct a 1-D vector with the splatted value that we insert in all the
     // places within the returned descriptor.
-    Value vdesc = rewriter.create<LLVM::PoisonOp>(loc, llvm1DVectorTy);
-    auto zero = rewriter.create<LLVM::ConstantOp>(
+    Value vdesc = LLVM::PoisonOp::create(rewriter, loc, llvm1DVectorTy);
+    auto zero = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
-    Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc,
+    Value v = LLVM::InsertElementOp::create(rewriter, loc, llvm1DVectorTy, vdesc,
                                                      adaptor.getInput(), zero);
 
     // Shuffle the value across the desired number of elements.
     int64_t width = resultType.getDimSize(resultType.getRank() - 1);
     SmallVector<int32_t> zeroValues(width, 0);
-    v = rewriter.create<LLVM::ShuffleVectorOp>(loc, v, v, zeroValues);
+    v = LLVM::ShuffleVectorOp::create(rewriter, loc, v, v, zeroValues);
 
     // Iterate of linear index, convert to coords space and insert splatted 1-D
     // vector in each position.
     nDVectorIterate(vectorTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
-      desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, v, position);
+      desc = LLVM::InsertValueOp::create(rewriter, loc, desc, v, position);
     });
     rewriter.replaceOp(splatOp, desc);
     return success();
@@ -1921,12 +1921,12 @@ struct VectorDeinterleaveOpLowering
       auto deinterleaveResults = deinterleaveOp.getResultTypes();
       auto packedOpResults =
           llvmTypeConverter->packOperationResults(deinterleaveResults);
-      auto intrinsic = rewriter.create<LLVM::vector_deinterleave2>(
+      auto intrinsic = LLVM::vector_deinterleave2::create(rewriter,
           loc, packedOpResults, adaptor.getSource());
 
-      auto evenResult = rewriter.create<LLVM::ExtractValueOp>(
+      auto evenResult = LLVM::ExtractValueOp::create(rewriter,
           loc, intrinsic->getResult(0), 0);
-      auto oddResult = rewriter.create<LLVM::ExtractValueOp>(
+      auto oddResult = LLVM::ExtractValueOp::create(rewriter,
           loc, intrinsic->getResult(0), 1);
 
       rewriter.replaceOp(deinterleaveOp, ValueRange{evenResult, oddResult});
@@ -1950,10 +1950,10 @@ struct VectorDeinterleaveOpLowering
         oddShuffleMask.push_back(i);
     }
 
-    auto poison = rewriter.create<LLVM::PoisonOp>(loc, sourceType);
-    auto evenShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+    auto poison = LLVM::PoisonOp::create(rewriter, loc, sourceType);
+    auto evenShuffle = LLVM::ShuffleVectorOp::create(rewriter,
         loc, adaptor.getSource(), poison, evenShuffleMask);
-    auto oddShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+    auto oddShuffle = LLVM::ShuffleVectorOp::create(rewriter,
         loc, adaptor.getSource(), poison, oddShuffleMask);
 
     rewriter.replaceOp(deinterleaveOp, ValueRange{evenShuffle, oddShuffle});
@@ -1977,9 +1977,9 @@ struct VectorFromElementsLowering
       return rewriter.notifyMatchFailure(fromElementsOp,
                                          "rank > 1 vectors are not supported");
     Type llvmType = typeConverter->convertType(vectorType);
-    Value result = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
+    Value result = LLVM::PoisonOp::create(rewriter, loc, llvmType);
     for (auto [idx, val] : llvm::enumerate(adaptor.getElements()))
-      result = rewriter.create<vector::InsertOp>(loc, val, result, idx);
+      result = vector::InsertOp::create(rewriter, loc, val, result, idx);
     rewriter.replaceOp(fromElementsOp, result);
     return success();
   }
@@ -2003,11 +2003,11 @@ struct VectorToElementsLowering
       if (element.use_empty())
         continue;
 
-      auto constIdx = rewriter.create<LLVM::ConstantOp>(
+      auto constIdx = LLVM::ConstantOp::create(rewriter,
           loc, idxType, rewriter.getIntegerAttr(idxType, idx));
       auto llvmType = typeConverter->convertType(element.getType());
 
-      Value result = rewriter.create<LLVM::ExtractElementOp>(loc, llvmType,
+      Value result = LLVM::ExtractElementOp::create(rewriter, loc, llvmType,
                                                              source, constIdx);
       results[idx] = result;
     }
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index 9f5b8fcca6c26..9b79b913f2d4b 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -132,9 +132,9 @@ static void maybeYieldValue(OpBuilder &b, Location loc, bool hasRetVal,
                             Value value) {
   if (hasRetVal) {
     assert(value && "Expected non-empty value");
-    b.create<scf::YieldOp>(loc, value);
+    scf::YieldOp::create(b, loc, value);
   } else {
-    b.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(b, loc);
   }
 }
 
@@ -154,7 +154,7 @@ static Value generateMaskCheck(OpBuilder &b, OpTy xferOp, Value iv) {
     return Value();
 
   Location loc = xferOp.getLoc();
-  return b.create<vector::ExtractElementOp>(loc, xferOp.getMask(), iv);
+  return vector::ExtractElementOp::create(b, loc, xferOp.getMask(), iv);
 }
 
 /// Helper function TransferOpConversion and TransferOp1dConversion.
@@ -201,21 +201,21 @@ static Value generateInBoundsCheck(
     Value base = xferOp.getIndices()[*dim];
     Value memrefIdx =
         affine::makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
-    cond = lb.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, memrefDim,
+    cond = arith::CmpIOp::create(lb, arith::CmpIPredicate::sgt, memrefDim,
                                     memrefIdx);
   }
 
   // Condition check 2: Masked in?
   if (auto maskCond = generateMaskCheck(b, xferOp, iv)) {
     if (cond)
-      cond = lb.create<arith::AndIOp>(cond, maskCond);
+      cond = arith::AndIOp::create(lb, cond, maskCond);
     else
       cond = maskCond;
   }
 
   // If the condition is non-empty, generate an SCF::IfOp.
   if (cond) {
-    auto check = lb.create<scf::IfOp>(
+    auto check = scf::IfOp::create(lb,
         cond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
@@ -226,7 +226,7 @@ static Value generateInBoundsCheck(
           if (outOfBoundsCase) {
             maybeYieldValue(b, loc, hasRetVal, outOfBoundsCase(b, loc));
           } else {
-            b.create<scf::YieldOp>(loc);
+            scf::YieldOp::create(b, loc);
           }
         });
 
@@ -303,14 +303,14 @@ static BufferAllocs allocBuffers(OpBuilder &b, OpTy xferOp) {
 
   BufferAllocs result;
   auto bufferType = MemRefType::get({}, xferOp.getVectorType());
-  result.dataBuffer = b.create<memref::AllocaOp>(loc, bufferType);
+  result.dataBuffer = memref::AllocaOp::create(b, loc, bufferType);
 
   if (xferOp.getMask()) {
     auto maskType = MemRefType::get({}, xferOp.getMask().getType());
-    auto maskBuffer = b.create<memref::AllocaOp>(loc, maskType);
+    auto maskBuffer = memref::AllocaOp::create(b, loc, maskType);
     b.setInsertionPoint(xferOp);
-    b.create<memref::StoreOp>(loc, xferOp.getMask(), maskBuffer);
-    result.maskBuffer = b.create<memref::LoadOp>(loc, maskBuffer, ValueRange());
+    memref::StoreOp::create(b, loc, xferOp.getMask(), maskBuffer);
+    result.maskBuffer = memref::LoadOp::create(b, loc, maskBuffer, ValueRange());
   }
 
   return result;
@@ -421,14 +421,14 @@ struct Strategy<TransferReadOp> {
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
-    auto newXferOp = b.create<vector::TransferReadOp>(
+    auto newXferOp = vector::TransferReadOp::create(b,
         loc, vecType, xferOp.getBase(), xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
         xferOp.getPadding(), Value(), inBoundsAttr);
 
     maybeApplyPassLabel(b, newXferOp, options.targetRank);
 
-    b.create<memref::StoreOp>(loc, newXferOp.getVector(), buffer, storeIndices);
+    memref::StoreOp::create(b, loc, newXferOp.getVector(), buffer, storeIndices);
     return newXferOp;
   }
 
@@ -444,8 +444,8 @@ struct Strategy<TransferReadOp> {
     Location loc = xferOp.getLoc();
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
-    auto vec = b.create<vector::SplatOp>(loc, vecType, xferOp.getPadding());
-    b.create<memref::StoreOp>(loc, vec, buffer, storeIndices);
+    auto vec = vector::SplatOp::create(b, loc, vecType, xferOp.getPadding());
+    memref::StoreOp::create(b, loc, vec, buffer, storeIndices);
 
     return Value();
   }
@@ -506,11 +506,11 @@ struct Strategy<TransferWriteOp> {
     getXferIndices(b, xferOp, iv, xferIndices);
 
     Location loc = xferOp.getLoc();
-    auto vec = b.create<memref::LoadOp>(loc, buffer, loadIndices);
+    auto vec = memref::LoadOp::create(b, loc, buffer, loadIndices);
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
     auto source = loopState.empty() ? xferOp.getBase() : loopState[0];
     Type type = isTensorOp(xferOp) ? xferOp.getShapedType() : Type();
-    auto newXferOp = b.create<vector::TransferWriteOp>(
+    auto newXferOp = vector::TransferWriteOp::create(b,
         loc, type, vec, source, xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
         inBoundsAttr);
@@ -610,7 +610,7 @@ struct PrepareTransferReadConversion
     }
 
     Location loc = xferOp.getLoc();
-    rewriter.create<memref::StoreOp>(loc, newXfer->getResult(0),
+    memref::StoreOp::create(rewriter, loc, newXfer->getResult(0),
                                      buffers.dataBuffer);
     rewriter.replaceOpWithNewOp<memref::LoadOp>(xferOp, buffers.dataBuffer);
 
@@ -653,9 +653,9 @@ struct PrepareTransferWriteConversion
 
     Location loc = xferOp.getLoc();
     auto buffers = allocBuffers(rewriter, xferOp);
-    rewriter.create<memref::StoreOp>(loc, xferOp.getVector(),
+    memref::StoreOp::create(rewriter, loc, xferOp.getVector(),
                                      buffers.dataBuffer);
-    auto loadedVec = rewriter.create<memref::LoadOp>(loc, buffers.dataBuffer);
+    auto loadedVec = memref::LoadOp::create(rewriter, loc, buffers.dataBuffer);
     rewriter.modifyOpInPlace(xferOp, [&]() {
       xferOp.getValueToStoreMutable().assign(loadedVec);
       xferOp->setAttr(kPassLabel, rewriter.getUnitAttr());
@@ -735,17 +735,17 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
       auto signlessTargetVectorType =
           vectorType.cloneWith({}, getIntTypeWithSignlessSemantics(legalIntTy));
       auto targetVectorType = vectorType.cloneWith({}, legalIntTy);
-      value = rewriter.create<vector::BitCastOp>(loc, signlessSourceVectorType,
+      value = vector::BitCastOp::create(rewriter, loc, signlessSourceVectorType,
                                                  value);
       if (value.getType() != signlessTargetVectorType) {
         if (width == 1 || intTy.isUnsigned())
-          value = rewriter.create<arith::ExtUIOp>(loc, signlessTargetVectorType,
+          value = arith::ExtUIOp::create(rewriter, loc, signlessTargetVectorType,
                                                   value);
         else
-          value = rewriter.create<arith::ExtSIOp>(loc, signlessTargetVectorType,
+          value = arith::ExtSIOp::create(rewriter, loc, signlessTargetVectorType,
                                                   value);
       }
-      value = rewriter.create<vector::BitCastOp>(loc, targetVectorType, value);
+      value = vector::BitCastOp::create(rewriter, loc, targetVectorType, value);
       vectorType = targetVectorType;
     }
 
@@ -763,28 +763,28 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
                                         std::multiplies<int64_t>());
       auto flatVectorType =
           VectorType::get({flatLength}, vectorType.getElementType());
-      value = rewriter.create<vector::ShapeCastOp>(loc, flatVectorType, value);
+      value = vector::ShapeCastOp::create(rewriter, loc, flatVectorType, value);
     }
 
     vector::PrintOp firstClose;
     SmallVector<Value, 8> loopIndices;
     for (unsigned d = 0; d < shape.size(); d++) {
       // Setup loop bounds and step.
-      Value lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-      Value upperBound = rewriter.create<arith::ConstantIndexOp>(loc, shape[d]);
-      Value step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+      Value lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+      Value upperBound = arith::ConstantIndexOp::create(rewriter, loc, shape[d]);
+      Value step = arith::ConstantIndexOp::create(rewriter, loc, 1);
       if (!scalableDimensions.empty() && scalableDimensions[d]) {
-        auto vscale = rewriter.create<vector::VectorScaleOp>(
+        auto vscale = vector::VectorScaleOp::create(rewriter,
             loc, rewriter.getIndexType());
-        upperBound = rewriter.create<arith::MulIOp>(loc, upperBound, vscale);
+        upperBound = arith::MulIOp::create(rewriter, loc, upperBound, vscale);
       }
-      auto lastIndex = rewriter.create<arith::SubIOp>(loc, upperBound, step);
+      auto lastIndex = arith::SubIOp::create(rewriter, loc, upperBound, step);
 
       // Create a loop to print the elements surrounded by parentheses.
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+      vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
       auto loop =
-          rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
-      auto printClose = rewriter.create<vector::PrintOp>(
+          scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
+      auto printClose = vector::PrintOp::create(rewriter,
           loc, vector::PrintPunctuation::Close);
       if (!firstClose)
         firstClose = printClose;
@@ -794,13 +794,13 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
 
       // Print a comma after all but the last element.
       rewriter.setInsertionPointToStart(loop.getBody());
-      auto notLastIndex = rewriter.create<arith::CmpIOp>(
+      auto notLastIndex = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ult, loopIdx, lastIndex);
-      rewriter.create<scf::IfOp>(loc, notLastIndex,
+      scf::IfOp::create(rewriter, loc, notLastIndex,
                                  [&](OpBuilder &builder, Location loc) {
-                                   builder.create<vector::PrintOp>(
+                                   vector::PrintOp::create(builder,
                                        loc, vector::PrintPunctuation::Comma);
-                                   builder.create<scf::YieldOp>(loc);
+                                   scf::YieldOp::create(builder, loc);
                                  });
 
       rewriter.setInsertionPointToStart(loop.getBody());
@@ -811,10 +811,10 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     Value flatIndex;
     auto currentStride = 1;
     for (int d = shape.size() - 1; d >= 0; d--) {
-      auto stride = rewriter.create<arith::ConstantIndexOp>(loc, currentStride);
-      auto index = rewriter.create<arith::MulIOp>(loc, stride, loopIndices[d]);
+      auto stride = arith::ConstantIndexOp::create(rewriter, loc, currentStride);
+      auto index = arith::MulIOp::create(rewriter, loc, stride, loopIndices[d]);
       if (flatIndex)
-        flatIndex = rewriter.create<arith::AddIOp>(loc, flatIndex, index);
+        flatIndex = arith::AddIOp::create(rewriter, loc, flatIndex, index);
       else
         flatIndex = index;
       currentStride *= shape[d];
@@ -822,12 +822,12 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
 
     // Print the scalar elements in the inner most loop.
     auto element =
-        rewriter.create<vector::ExtractElementOp>(loc, value, flatIndex);
-    rewriter.create<vector::PrintOp>(loc, element,
+        vector::ExtractElementOp::create(rewriter, loc, value, flatIndex);
+    vector::PrintOp::create(rewriter, loc, element,
                                      vector::PrintPunctuation::NoPunctuation);
 
     rewriter.setInsertionPointAfter(firstClose);
-    rewriter.create<vector::PrintOp>(loc, printOp.getPunctuation());
+    vector::PrintOp::create(rewriter, loc, printOp.getPunctuation());
     rewriter.eraseOp(printOp);
     return success();
   }
@@ -918,7 +918,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                                          "Failed to unpack one vector dim.");
 
     auto castedDataBuffer =
-        locB.create<vector::TypeCastOp>(*castedDataType, dataBuffer);
+        vector::TypeCastOp::create(locB, *castedDataType, dataBuffer);
 
     // If the xferOp has a mask: Find and cast mask buffer.
     Value castedMaskBuffer;
@@ -937,21 +937,21 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
         auto maskBufferType = cast<MemRefType>(maskBuffer.getType());
         MemRefType castedMaskType = *unpackOneDim(maskBufferType);
         castedMaskBuffer =
-            locB.create<vector::TypeCastOp>(castedMaskType, maskBuffer);
+            vector::TypeCastOp::create(locB, castedMaskType, maskBuffer);
       }
     }
 
     // Loop bounds and step.
-    auto lb = locB.create<arith::ConstantIndexOp>(0);
-    auto ub = locB.create<arith::ConstantIndexOp>(
+    auto lb = arith::ConstantIndexOp::create(locB, 0);
+    auto ub = arith::ConstantIndexOp::create(locB,
         castedDataType->getDimSize(castedDataType->getRank() - 1));
-    auto step = locB.create<arith::ConstantIndexOp>(1);
+    auto step = arith::ConstantIndexOp::create(locB, 1);
     // TransferWriteOps that operate on tensors return the modified tensor and
     // require a loop state.
     auto loopState = Strategy<OpTy>::initialLoopState(xferOp);
 
     // Generate for loop.
-    auto result = locB.create<scf::ForOp>(
+    auto result = scf::ForOp::create(locB,
         lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(),
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopState) {
           Type stateType = loopState.empty() ? Type() : loopState[0].getType();
@@ -977,7 +977,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                   SmallVector<Value, 8> loadIndices;
                   getMaskBufferLoadIndices(xferOp, castedMaskBuffer,
                                            loadIndices, iv);
-                  auto mask = b.create<memref::LoadOp>(loc, castedMaskBuffer,
+                  auto mask = memref::LoadOp::create(b, loc, castedMaskBuffer,
                                                        loadIndices);
                   rewriter.modifyOpInPlace(newXfer, [&]() {
                     newXfer.getMaskMutable().assign(mask);
@@ -1121,29 +1121,29 @@ struct ScalableTransposeTransferWriteConversion
     auto transposeSource = transposeOp.getVector();
     SmallVector<Value> transposeSourceSlices =
         llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-          return rewriter.create<vector::ExtractOp>(loc, transposeSource, idx);
+          return vector::ExtractOp::create(rewriter, loc, transposeSource, idx);
         });
 
     // Loop bounds and step.
-    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lb = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto ub =
         maskDims->empty()
             ? Value(createVscaleMultiple(vectorType.getDimSize(0)))
             : vector::getAsValues(rewriter, loc, maskDims->front()).front();
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
     // Generate a new mask for the slice.
     VectorType sliceType = VectorType::Builder(vectorType).dropDim(0);
     Value sliceMask = nullptr;
     if (!maskDims->empty()) {
-      sliceMask = rewriter.create<vector::CreateMaskOp>(
+      sliceMask = vector::CreateMaskOp::create(rewriter,
           loc, sliceType.clone(rewriter.getI1Type()),
           ArrayRef<OpFoldResult>(*maskDims).drop_front());
     }
 
     Value initDest = isTensorOp(writeOp) ? writeOp.getBase() : Value{};
     ValueRange initLoopArgs = initDest ? initDest : ValueRange{};
-    auto result = rewriter.create<scf::ForOp>(
+    auto result = scf::ForOp::create(rewriter,
         loc, lb, ub, step, initLoopArgs,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopIterArgs) {
           // Indices for the new transfer op.
@@ -1153,23 +1153,23 @@ struct ScalableTransposeTransferWriteConversion
           // Extract a transposed slice from the source vector.
           SmallVector<Value> transposeElements =
               llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-                return b.create<vector::ExtractOp>(
+                return vector::ExtractOp::create(b,
                     loc, transposeSourceSlices[idx], iv);
               });
-          auto sliceVec = b.create<vector::FromElementsOp>(loc, sliceType,
+          auto sliceVec = vector::FromElementsOp::create(b, loc, sliceType,
                                                            transposeElements);
 
           // Create the transfer_write for the slice.
           Value dest =
               loopIterArgs.empty() ? writeOp.getBase() : loopIterArgs.front();
-          auto newWriteOp = b.create<vector::TransferWriteOp>(
+          auto newWriteOp = vector::TransferWriteOp::create(b,
               loc, sliceVec, dest, xferIndices,
               ArrayRef<bool>(writeOp.getInBoundsValues()).drop_front());
           if (sliceMask)
             newWriteOp.getMaskMutable().assign(sliceMask);
 
           // Yield from the loop.
-          b.create<scf::YieldOp>(loc, loopIterArgs.empty()
+          scf::YieldOp::create(b, loc, loopIterArgs.empty()
                                           ? ValueRange{}
                                           : newWriteOp.getResult());
         });
@@ -1209,7 +1209,7 @@ static void maybeAssignMask(OpBuilder &b, OpTy xferOp, OpTy newXferOp,
 
     llvm::SmallVector<int64_t, 1> indices({i});
     Location loc = xferOp.getLoc();
-    auto newMask = b.create<vector::ExtractOp>(loc, xferOp.getMask(), indices);
+    auto newMask = vector::ExtractOp::create(b, loc, xferOp.getMask(), indices);
     newXferOp.getMaskMutable().assign(newMask);
   }
 
@@ -1263,7 +1263,7 @@ struct UnrollTransferReadConversion
     if (auto insertOp = getInsertOp(xferOp))
       return insertOp.getDest();
     Location loc = xferOp.getLoc();
-    return rewriter.create<vector::SplatOp>(loc, xferOp.getVectorType(),
+    return vector::SplatOp::create(rewriter, loc, xferOp.getVectorType(),
                                             xferOp.getPadding());
   }
 
@@ -1319,7 +1319,7 @@ struct UnrollTransferReadConversion
     // Generate fully unrolled loop of transfer ops.
     Location loc = xferOp.getLoc();
     for (int64_t i = 0; i < dimSize; ++i) {
-      Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
+      Value iv = arith::ConstantIndexOp::create(rewriter, loc, i);
 
       // FIXME: Rename this lambda - it does much more than just
       // in-bounds-check generation.
@@ -1338,7 +1338,7 @@ struct UnrollTransferReadConversion
 
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
 
-            auto newXferOp = b.create<vector::TransferReadOp>(
+            auto newXferOp = vector::TransferReadOp::create(b,
                 loc, newXferVecType, xferOp.getBase(), xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
                 xferOp.getPadding(), Value(), inBoundsAttr);
@@ -1348,10 +1348,10 @@ struct UnrollTransferReadConversion
             if (newXferVecType.getRank() == 0) {
               // vector.insert does not accept rank-0 as the non-indexed
               // argument. Extract the scalar before inserting.
-              valToInser = b.create<vector::ExtractOp>(loc, valToInser,
+              valToInser = vector::ExtractOp::create(b, loc, valToInser,
                                                        SmallVector<int64_t>());
             }
-            return b.create<vector::InsertOp>(loc, valToInser, vec,
+            return vector::InsertOp::create(b, loc, valToInser, vec,
                                               insertionIndices);
           },
           /*outOfBoundsCase=*/
@@ -1462,7 +1462,7 @@ struct UnrollTransferWriteConversion
     // Generate fully unrolled loop of transfer ops.
     Location loc = xferOp.getLoc();
     for (int64_t i = 0; i < dimSize; ++i) {
-      Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
+      Value iv = arith::ConstantIndexOp::create(rewriter, loc, i);
 
       auto updatedSource = generateInBoundsCheck(
           rewriter, xferOp, iv, unpackedDim(xferOp),
@@ -1479,19 +1479,19 @@ struct UnrollTransferWriteConversion
             extractionIndices.push_back(b.getI64IntegerAttr(i));
 
             auto extracted =
-                b.create<vector::ExtractOp>(loc, vec, extractionIndices);
+                vector::ExtractOp::create(b, loc, vec, extractionIndices);
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
             Value xferVec;
             if (inputVectorTy.getRank() == 1) {
               // When target-rank=0, unrolling would causes the vector input
               // argument into `transfer_write` to become a scalar. We solve
               // this by broadcasting the scalar to a 0D vector.
-              xferVec = b.create<vector::BroadcastOp>(
+              xferVec = vector::BroadcastOp::create(b,
                   loc, VectorType::get({}, extracted.getType()), extracted);
             } else {
               xferVec = extracted;
             }
-            auto newXferOp = b.create<vector::TransferWriteOp>(
+            auto newXferOp = vector::TransferWriteOp::create(b,
                 loc, sourceType, xferVec, source, xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
                 inBoundsAttr);
@@ -1574,18 +1574,18 @@ struct Strategy1d<TransferReadOp> {
         b, xferOp, iv, dim, TypeRange(xferOp.getVectorType()),
         /*inBoundsCase=*/
         [&](OpBuilder &b, Location loc) {
-          Value val = b.create<memref::LoadOp>(loc, xferOp.getBase(), indices);
-          return b.create<vector::InsertElementOp>(loc, val, vec, iv);
+          Value val = memref::LoadOp::create(b, loc, xferOp.getBase(), indices);
+          return vector::InsertElementOp::create(b, loc, val, vec, iv);
         },
         /*outOfBoundsCase=*/
         [&](OpBuilder & /*b*/, Location loc) { return vec; });
-    b.create<scf::YieldOp>(loc, nextVec);
+    scf::YieldOp::create(b, loc, nextVec);
   }
 
   static Value initialLoopState(OpBuilder &b, TransferReadOp xferOp) {
     // Inititalize vector with padding value.
     Location loc = xferOp.getLoc();
-    return b.create<vector::SplatOp>(loc, xferOp.getVectorType(),
+    return vector::SplatOp::create(b, loc, xferOp.getVectorType(),
                                      xferOp.getPadding());
   }
 };
@@ -1604,10 +1604,10 @@ struct Strategy1d<TransferWriteOp> {
         b, xferOp, iv, dim,
         /*inBoundsCase=*/[&](OpBuilder &b, Location loc) {
           auto val =
-              b.create<vector::ExtractElementOp>(loc, xferOp.getVector(), iv);
-          b.create<memref::StoreOp>(loc, val, xferOp.getBase(), indices);
+              vector::ExtractElementOp::create(b, loc, xferOp.getVector(), iv);
+          memref::StoreOp::create(b, loc, val, xferOp.getBase(), indices);
         });
-    b.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(b, loc);
   }
 
   static Value initialLoopState(OpBuilder &b, TransferWriteOp xferOp) {
@@ -1668,15 +1668,15 @@ struct TransferOp1dConversion : public VectorToSCFPattern<OpTy> {
     // Loop bounds, step, state...
     Location loc = xferOp.getLoc();
     auto vecType = xferOp.getVectorType();
-    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lb = arith::ConstantIndexOp::create(rewriter, loc, 0);
     Value ub =
-        rewriter.create<arith::ConstantIndexOp>(loc, vecType.getDimSize(0));
+        arith::ConstantIndexOp::create(rewriter, loc, vecType.getDimSize(0));
     if (vecType.isScalable()) {
       Value vscale =
-          rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-      ub = rewriter.create<arith::MulIOp>(loc, ub, vscale);
+          vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+      ub = arith::MulIOp::create(rewriter, loc, ub, vscale);
     }
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto loopState = Strategy1d<OpTy>::initialLoopState(rewriter, xferOp);
 
     // Generate for loop.
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 21d8e1d9f1156..2466e72277441 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -147,18 +147,18 @@ static Value sanitizeDynamicIndex(ConversionPatternRewriter &rewriter,
                                   Location loc, Value dynamicIndex,
                                   int64_t kPoisonIndex, unsigned vectorSize) {
   if (llvm::isPowerOf2_32(vectorSize)) {
-    Value inBoundsMask = rewriter.create<spirv::ConstantOp>(
+    Value inBoundsMask = spirv::ConstantOp::create(rewriter,
         loc, dynamicIndex.getType(),
         rewriter.getIntegerAttr(dynamicIndex.getType(), vectorSize - 1));
-    return rewriter.create<spirv::BitwiseAndOp>(loc, dynamicIndex,
+    return spirv::BitwiseAndOp::create(rewriter, loc, dynamicIndex,
                                                 inBoundsMask);
   }
-  Value poisonIndex = rewriter.create<spirv::ConstantOp>(
+  Value poisonIndex = spirv::ConstantOp::create(rewriter,
       loc, dynamicIndex.getType(),
       rewriter.getIntegerAttr(dynamicIndex.getType(), kPoisonIndex));
   Value cmpResult =
-      rewriter.create<spirv::IEqualOp>(loc, dynamicIndex, poisonIndex);
-  return rewriter.create<spirv::SelectOp>(
+      spirv::IEqualOp::create(rewriter, loc, dynamicIndex, poisonIndex);
+  return spirv::SelectOp::create(rewriter,
       loc, cmpResult,
       spirv::ConstantOp::getZero(dynamicIndex.getType(), loc, rewriter),
       dynamicIndex);
@@ -427,7 +427,7 @@ static SmallVector<Value> extractAllElements(
   Location loc = reduceOp.getLoc();
 
   for (int i = 0; i < numElements; ++i) {
-    values.push_back(rewriter.create<spirv::CompositeExtractOp>(
+    values.push_back(spirv::CompositeExtractOp::create(rewriter,
         loc, srcVectorType.getElementType(), adaptor.getVector(),
         rewriter.getI32ArrayAttr({i})));
   }
@@ -481,16 +481,16 @@ struct VectorReductionPattern final : OpConversionPattern<vector::ReductionOp> {
 #define INT_AND_FLOAT_CASE(kind, iop, fop)                                     \
   case vector::CombiningKind::kind:                                            \
     if (llvm::isa<IntegerType>(resultType)) {                                  \
-      result = rewriter.create<spirv::iop>(loc, resultType, result, next);     \
+      result = spirv::iop::create(rewriter, loc, resultType, result, next);     \
     } else {                                                                   \
       assert(llvm::isa<FloatType>(resultType));                                \
-      result = rewriter.create<spirv::fop>(loc, resultType, result, next);     \
+      result = spirv::fop::create(rewriter, loc, resultType, result, next);     \
     }                                                                          \
     break
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = rewriter.create<fop>(loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);              \
     break
 
         INT_AND_FLOAT_CASE(ADD, IAddOp, FAddOp);
@@ -537,7 +537,7 @@ struct VectorReductionFloatMinMax final
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = rewriter.create<fop>(loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);              \
     break
 
         INT_OR_FLOAT_CASE(MAXIMUMF, SPIRVFMaxOp);
@@ -613,7 +613,7 @@ struct VectorShuffleOpConvert final
     auto getElementAtIdx = [&rewriter, loc = shuffleOp.getLoc()](
                                Value scalarOrVec, int32_t idx) -> Value {
       if (auto vecTy = dyn_cast<VectorType>(scalarOrVec.getType()))
-        return rewriter.create<spirv::CompositeExtractOp>(loc, scalarOrVec,
+        return spirv::CompositeExtractOp::create(rewriter, loc, scalarOrVec,
                                                           idx);
 
       assert(idx == 0 && "Invalid scalar element index");
@@ -712,10 +712,10 @@ struct VectorDeinterleaveOpConvert final
     // We cannot use `spirv::VectorShuffleOp` directly in this case, and need to
     // use `spirv::CompositeExtractOp`.
     if (n == 2) {
-      auto elem0 = rewriter.create<spirv::CompositeExtractOp>(
+      auto elem0 = spirv::CompositeExtractOp::create(rewriter,
           loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({0}));
 
-      auto elem1 = rewriter.create<spirv::CompositeExtractOp>(
+      auto elem1 = spirv::CompositeExtractOp::create(rewriter,
           loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({1}));
 
       rewriter.replaceOp(deinterleaveOp, {elem0, elem1});
@@ -733,11 +733,11 @@ struct VectorDeinterleaveOpConvert final
         llvm::map_to_vector(seqOdd, [](int i) { return i * 2 + 1; });
 
     // Create two SPIR-V shuffles.
-    auto shuffleEven = rewriter.create<spirv::VectorShuffleOp>(
+    auto shuffleEven = spirv::VectorShuffleOp::create(rewriter,
         loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesEven));
 
-    auto shuffleOdd = rewriter.create<spirv::VectorShuffleOp>(
+    auto shuffleOdd = spirv::VectorShuffleOp::create(rewriter,
         loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesOdd));
 
@@ -781,7 +781,7 @@ struct VectorLoadOpConverter final
     // to a scalar.
     Value castedAccessChain = (vectorType.getNumElements() == 1)
                                   ? accessChain
-                                  : rewriter.create<spirv::BitcastOp>(
+                                  : spirv::BitcastOp::create(rewriter,
                                         loc, vectorPtrType, accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, spirvVectorType,
@@ -823,7 +823,7 @@ struct VectorStoreOpConverter final
     // to a scalar.
     Value castedAccessChain = (vectorType.getNumElements() == 1)
                                   ? accessChain
-                                  : rewriter.create<spirv::BitcastOp>(
+                                  : spirv::BitcastOp::create(rewriter,
                                         loc, vectorPtrType, accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, castedAccessChain,
@@ -905,9 +905,9 @@ struct VectorReductionToIntDotProd final
       auto v4i8Type = VectorType::get({4}, i8Type);
       Location loc = op.getLoc();
       Value zero = spirv::ConstantOp::getZero(i8Type, loc, rewriter);
-      lhsIn = rewriter.create<spirv::CompositeConstructOp>(
+      lhsIn = spirv::CompositeConstructOp::create(rewriter,
           loc, v4i8Type, ValueRange{lhsIn, zero});
-      rhsIn = rewriter.create<spirv::CompositeConstructOp>(
+      rhsIn = spirv::CompositeConstructOp::create(rewriter,
           loc, v4i8Type, ValueRange{rhsIn, zero});
     }
 
@@ -971,14 +971,14 @@ struct VectorReductionToFPDotProd final
       Attribute oneAttr =
           rewriter.getFloatAttr(vectorType.getElementType(), 1.0);
       oneAttr = SplatElementsAttr::get(vectorType, oneAttr);
-      rhs = rewriter.create<spirv::ConstantOp>(loc, vectorType, oneAttr);
+      rhs = spirv::ConstantOp::create(rewriter, loc, vectorType, oneAttr);
     }
     assert(lhs);
     assert(rhs);
 
-    Value res = rewriter.create<spirv::DotOp>(loc, resultType, lhs, rhs);
+    Value res = spirv::DotOp::create(rewriter, loc, resultType, lhs, rhs);
     if (acc)
-      res = rewriter.create<spirv::FAddOp>(loc, acc, res);
+      res = spirv::FAddOp::create(rewriter, loc, acc, res);
 
     rewriter.replaceOp(op, res);
     return success();
@@ -1013,7 +1013,7 @@ struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
     source.reserve(numElements);
     for (int64_t i = 0; i < numElements; ++i) {
       Attribute intAttr = rewriter.getIntegerAttr(intType, i);
-      Value constOp = rewriter.create<spirv::ConstantOp>(loc, intType, intAttr);
+      Value constOp = spirv::ConstantOp::create(rewriter, loc, intType, intAttr);
       source.push_back(constOp);
     }
     rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(stepOp, dstType,
@@ -1056,7 +1056,7 @@ struct VectorToElementOpConvert final
       if (element.use_empty())
         continue;
 
-      Value result = rewriter.create<spirv::CompositeExtractOp>(
+      Value result = spirv::CompositeExtractOp::create(rewriter,
           loc, elementType, adaptor.getSource(),
           rewriter.getI32ArrayAttr({static_cast<int32_t>(idx)}));
       results[idx] = result;
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index 0ec7129a40a66..255013ee07c88 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -108,7 +108,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
 
   xegpu::CreateNdDescOp ndDesc;
   if (srcTy.hasStaticShape()) {
-    ndDesc = rewriter.create<xegpu::CreateNdDescOp>(loc, descType, src,
+    ndDesc = xegpu::CreateNdDescOp::create(rewriter, loc, descType, src,
                                                     getAsOpFoldResult(offsets));
   } else {
     // In case of any dynamic shapes, source's shape and strides have to be
@@ -116,7 +116,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
     SmallVector<Value> sourceDims;
     unsigned srcRank = srcTy.getRank();
     for (unsigned i = 0; i < srcRank; ++i)
-      sourceDims.push_back(rewriter.create<memref::DimOp>(loc, src, i));
+      sourceDims.push_back(memref::DimOp::create(rewriter, loc, src, i));
 
     SmallVector<int64_t> constOffsets;
     SmallVector<Value> dynOffsets;
@@ -135,17 +135,17 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
 
     // Compute strides in reverse order.
     SmallVector<Value> dynStrides;
-    Value accStride = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value accStride = arith::ConstantIndexOp::create(rewriter, loc, 1);
     // Last stride is guaranteed to be static and unit.
     for (int i = static_cast<int>(strides.size()) - 2; i >= 0; --i) {
       accStride =
-          rewriter.create<arith::MulIOp>(loc, accStride, sourceDims[i + 1]);
+          arith::MulIOp::create(rewriter, loc, accStride, sourceDims[i + 1]);
       if (strides[i] == ShapedType::kDynamic)
         dynStrides.push_back(accStride);
     }
     std::reverse(dynStrides.begin(), dynStrides.end());
 
-    ndDesc = rewriter.create<xegpu::CreateNdDescOp>(
+    ndDesc = xegpu::CreateNdDescOp::create(rewriter,
         loc, descType, src, dynOffsets, dynShapes, dynStrides,
         DenseI64ArrayAttr::get(rewriter.getContext(), constOffsets),
         DenseI64ArrayAttr::get(rewriter.getContext(), srcTy.getShape()),
@@ -200,7 +200,7 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
                                                   ArrayRef<int64_t>{1, 0});
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadOp = rewriter.create<xegpu::LoadNdOp>(
+    auto loadOp = xegpu::LoadNdOp::create(rewriter,
         loc, vecTy, ndDesc, /*packed=*/nullptr, transposeAttr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
@@ -238,7 +238,7 @@ struct TransferWriteLowering
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeOp =
-        rewriter.create<xegpu::StoreNdOp>(loc, writeOp.getVector(), ndDesc,
+        xegpu::StoreNdOp::create(rewriter, loc, writeOp.getVector(), ndDesc,
                                           /*l1_hint=*/hint,
                                           /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(writeOp, storeOp);
@@ -269,7 +269,7 @@ struct LoadLowering : public OpRewritePattern<vector::LoadOp> {
 
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadNdOp = rewriter.create<xegpu::LoadNdOp>(
+    auto loadNdOp = xegpu::LoadNdOp::create(rewriter,
         loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
@@ -303,7 +303,7 @@ struct StoreLowering : public OpRewritePattern<vector::StoreOp> {
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeNdOp =
-        rewriter.create<xegpu::StoreNdOp>(loc, vector, ndDesc,
+        xegpu::StoreNdOp::create(rewriter, loc, vector, ndDesc,
                                           /*l1_hint=*/hint,
                                           /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(storeOp, storeNdOp);
@@ -346,7 +346,7 @@ struct ContractionLowering : public OpRewritePattern<vector::ContractionOp> {
       return rewriter.notifyMatchFailure(contractOp,
                                          "Invalid operand dimensions");
 
-    auto dpasOp = rewriter.create<xegpu::DpasOp>(
+    auto dpasOp = xegpu::DpasOp::create(rewriter,
         loc, TypeRange{contractOp.getResultType()}, ValueRange{lhs, rhs, acc});
     rewriter.replaceOp(contractOp, dpasOp);
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
index fd2ba0683786e..4a1a2487806a7 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -100,8 +100,8 @@ static Value flattenVecToBits(ConversionPatternRewriter &rewriter, Location loc,
       vectorType.getElementTypeBitWidth() * vectorType.getNumElements();
   Type allBitsType = rewriter.getIntegerType(bitwidth);
   auto allBitsVecType = VectorType::get({1}, allBitsType);
-  Value bitcast = rewriter.create<vector::BitCastOp>(loc, allBitsVecType, val);
-  Value scalar = rewriter.create<vector::ExtractOp>(loc, bitcast, 0);
+  Value bitcast = vector::BitCastOp::create(rewriter, loc, allBitsVecType, val);
+  Value scalar = vector::ExtractOp::create(rewriter, loc, bitcast, 0);
   return scalar;
 }
 
@@ -120,25 +120,25 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   SmallVector<NamedAttribute> loadAttrs;
   patchOperandSegmentSizes(origAttrs, loadAttrs, DataArgAction::Drop);
   Value initialLoad =
-      rewriter.create<RawBufferLoadOp>(loc, dataType, invariantArgs, loadAttrs);
+      RawBufferLoadOp::create(rewriter, loc, dataType, invariantArgs, loadAttrs);
   Block *currentBlock = rewriter.getInsertionBlock();
   Block *afterAtomic =
       rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
   Block *loopBlock = rewriter.createBlock(afterAtomic, {dataType}, {loc});
 
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(loc, loopBlock, initialLoad);
+  cf::BranchOp::create(rewriter, loc, loopBlock, initialLoad);
 
   rewriter.setInsertionPointToEnd(loopBlock);
   Value prevLoad = loopBlock->getArgument(0);
-  Value operated = rewriter.create<ArithOp>(loc, data, prevLoad);
+  Value operated = ArithOp::create(rewriter, loc, data, prevLoad);
   dataType = operated.getType();
 
   SmallVector<NamedAttribute> cmpswapAttrs;
   patchOperandSegmentSizes(origAttrs, cmpswapAttrs, DataArgAction::Duplicate);
   SmallVector<Value> cmpswapArgs = {operated, prevLoad};
   cmpswapArgs.append(invariantArgs.begin(), invariantArgs.end());
-  Value atomicRes = rewriter.create<RawBufferAtomicCmpswapOp>(
+  Value atomicRes = RawBufferAtomicCmpswapOp::create(rewriter,
       loc, dataType, cmpswapArgs, cmpswapAttrs);
 
   // We care about exact bitwise equality here, so do some bitcasts.
@@ -151,13 +151,13 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   if (auto floatDataTy = dyn_cast<FloatType>(dataType)) {
     Type equivInt = rewriter.getIntegerType(floatDataTy.getWidth());
     prevLoadForCompare =
-        rewriter.create<arith::BitcastOp>(loc, equivInt, prevLoad);
+        arith::BitcastOp::create(rewriter, loc, equivInt, prevLoad);
     atomicResForCompare =
-        rewriter.create<arith::BitcastOp>(loc, equivInt, atomicRes);
+        arith::BitcastOp::create(rewriter, loc, equivInt, atomicRes);
   }
-  Value canLeave = rewriter.create<arith::CmpIOp>(
+  Value canLeave = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::eq, atomicResForCompare, prevLoadForCompare);
-  rewriter.create<cf::CondBranchOp>(loc, canLeave, afterAtomic, ValueRange{},
+  cf::CondBranchOp::create(rewriter, loc, canLeave, afterAtomic, ValueRange{},
                                     loopBlock, atomicRes);
   rewriter.eraseOp(atomicOp);
   return success();
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
index 9a368f372c296..cba0611f05ef8 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -54,9 +54,9 @@ static LogicalResult baseInBufferAddrSpace(PatternRewriter &rewriter,
 static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
                                            vector::MaskedLoadOp maskedOp) {
   VectorType vectorType = maskedOp.getVectorType();
-  Value load = builder.create<vector::LoadOp>(
+  Value load = vector::LoadOp::create(builder,
       loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
-  Value res = builder.create<arith::SelectOp>(
+  Value res = arith::SelectOp::create(builder,
       loc, vectorType, maskedOp.getMask(), load, maskedOp.getPassThru());
   return res;
 }
@@ -87,7 +87,7 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
     SmallVector<OpFoldResult> indices = maskedOp.getIndices();
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, src);
     SmallVector<OpFoldResult> strides =
         stridedMetadata.getConstifiedMixedStrides();
     SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
@@ -101,46 +101,46 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
 
     // delta = bufferSize - linearizedOffset
     Value vectorSizeOffset =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
+        arith::ConstantIndexOp::create(rewriter, loc, vectorSize);
     Value linearIndex =
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
     Value totalSize = getValueOrCreateConstantIndexOp(
         rewriter, loc, linearizedInfo.linearizedSize);
-    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
+    Value delta = arith::SubIOp::create(rewriter, loc, totalSize, linearIndex);
 
     // 1) check if delta < vectorSize
-    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
+    Value isOutofBounds = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
 
     // 2) check if (detla % elements_per_word != 0)
-    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
+    Value elementsPerWord = arith::ConstantIndexOp::create(rewriter,
         loc, llvm::divideCeil(32, elementBitWidth));
-    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
+    Value isNotWordAligned = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne,
-        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
+        arith::RemUIOp::create(rewriter, loc, delta, elementsPerWord),
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
 
     // We take the fallback of maskedload default lowering only it is both
     // out-of-bounds and not word aligned. The fallback ensures correct results
     // when loading at the boundary of the buffer since buffer load returns
     // inconsistent zeros for the whole word when boundary is crossed.
     Value ifCondition =
-        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
+        arith::AndIOp::create(rewriter, loc, isOutofBounds, isNotWordAligned);
 
     auto thenBuilder = [&](OpBuilder &builder, Location loc) {
       Operation *read = builder.clone(*maskedOp.getOperation());
       read->setAttr(kMaskedloadNeedsMask, builder.getUnitAttr());
       Value readResult = read->getResult(0);
-      builder.create<scf::YieldOp>(loc, readResult);
+      scf::YieldOp::create(builder, loc, readResult);
     };
 
     auto elseBuilder = [&](OpBuilder &builder, Location loc) {
       Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp);
-      rewriter.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(rewriter, loc, res);
     };
 
     auto ifOp =
-        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
+        scf::IfOp::create(rewriter, loc, ifCondition, thenBuilder, elseBuilder);
 
     rewriter.replaceOp(maskedOp, ifOp);
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
index 195f59d625554..4771d655e36c2 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
@@ -37,7 +37,7 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       return rewriter.notifyMatchFailure(metadataOp,
                                          "not a fat raw buffer cast");
     Location loc = castOp.getLoc();
-    auto sourceMetadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+    auto sourceMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
         loc, castOp.getSource());
     SmallVector<Value> results;
     if (metadataOp.getBaseBuffer().use_empty()) {
@@ -48,13 +48,13 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       if (baseBufferType == castOp.getResult().getType()) {
         results.push_back(castOp.getResult());
       } else {
-        results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+        results.push_back(memref::ReinterpretCastOp::create(rewriter,
             loc, baseBufferType, castOp.getResult(), /*offset=*/0,
             /*sizes=*/ArrayRef<int64_t>{}, /*strides=*/ArrayRef<int64_t>{}));
       }
     }
     if (castOp.getResetOffset())
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, 0));
     else
       results.push_back(sourceMetadata.getOffset());
     llvm::append_range(results, sourceMetadata.getSizes());
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 12b375b373fa9..748ff1edbfeb2 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -76,8 +76,8 @@ static SmallVector<Value> getTileSizes(Location loc, amx::TileType tType,
   auto mattr = rewriter.getI16IntegerAttr(tType.getDimSize(0));
   auto nattr = rewriter.getI16IntegerAttr(tType.getDimSize(1) * bytes);
   return SmallVector<Value>{
-      rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, mattr),
-      rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, nattr)};
+      LLVM::ConstantOp::create(rewriter, loc, llvmInt16Type, mattr),
+      LLVM::ConstantOp::create(rewriter, loc, llvmInt16Type, nattr)};
 }
 
 /// Maps the 2-dim memref shape to the 64-bit stride. Note that the buffer
@@ -95,7 +95,7 @@ static Value getStride(Location loc, MemRefType mType, Value base,
     // Dynamic stride needs code to compute the stride at runtime.
     MemRefDescriptor memrefDescriptor(base);
     auto attr = rewriter.getI64IntegerAttr(bytes);
-    Value scale = rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr);
+    Value scale = LLVM::ConstantOp::create(rewriter, loc, llvmInt64Type, attr);
     return rewriter
         .create<LLVM::MulOp>(loc, llvmInt64Type, scale,
                              memrefDescriptor.stride(rewriter, loc, preLast))
@@ -103,7 +103,7 @@ static Value getStride(Location loc, MemRefType mType, Value base,
   }
   // Use direct constant for static stride.
   auto attr = rewriter.getI64IntegerAttr(strides[preLast] * bytes);
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr)
+  return LLVM::ConstantOp::create(rewriter, loc, llvmInt64Type, attr)
       .getResult();
 }
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index bd4b2e56808b6..263663d041f3a 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -242,7 +242,7 @@ Operation *AffineDialect::materializeConstant(OpBuilder &builder,
                                               Attribute value, Type type,
                                               Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
 
@@ -1284,7 +1284,7 @@ mlir::affine::makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map,
   map = foldAttributesIntoMap(b, map, operands, valueOperands);
   composeAffineMapAndOperands(&map, &valueOperands, composeAffineMin);
   assert(map);
-  return b.create<AffineApplyOp>(loc, map, valueOperands);
+  return AffineApplyOp::create(b, loc, map, valueOperands);
 }
 
 AffineApplyOp
@@ -1391,7 +1391,7 @@ static OpTy makeComposedMinMax(OpBuilder &b, Location loc, AffineMap map,
   SmallVector<Value> valueOperands;
   map = foldAttributesIntoMap(b, map, operands, valueOperands);
   composeMultiResultAffineMap(map, valueOperands);
-  return b.create<OpTy>(loc, b.getIndexType(), map, valueOperands);
+  return OpTy::create(b, loc, b.getIndexType(), map, valueOperands);
 }
 
 AffineMinOp
@@ -2690,7 +2690,7 @@ FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInits());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  AffineForOp newLoop = rewriter.create<AffineForOp>(
+  AffineForOp newLoop = AffineForOp::create(rewriter,
       getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
       getUpperBoundOperands(), getUpperBoundMap(), getStepAsInt(), inits);
 
@@ -2833,7 +2833,7 @@ static void buildAffineLoopNestImpl(
         OpBuilder::InsertionGuard nestedGuard(nestedBuilder);
         bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
       }
-      nestedBuilder.create<AffineYieldOp>(nestedLoc);
+      AffineYieldOp::create(nestedBuilder, nestedLoc);
     };
 
     // Delegate actual loop creation to the callback in order to dispatch
@@ -2848,7 +2848,7 @@ static AffineForOp
 buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb,
                              int64_t ub, int64_t step,
                              AffineForOp::BodyBuilderFn bodyBuilderFn) {
-  return builder.create<AffineForOp>(loc, lb, ub, step,
+  return AffineForOp::create(builder, loc, lb, ub, step,
                                      /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
@@ -2862,7 +2862,7 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
   if (lbConst && ubConst)
     return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
                                         ubConst.value(), step, bodyBuilderFn);
-  return builder.create<AffineForOp>(loc, lb, builder.getDimIdentityMap(), ub,
+  return AffineForOp::create(builder, loc, lb, builder.getDimIdentityMap(), ub,
                                      builder.getDimIdentityMap(), step,
                                      /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
@@ -4885,7 +4885,7 @@ struct DropUnitExtentBasis
     Location loc = delinearizeOp->getLoc();
     auto getZero = [&]() -> Value {
       if (!zero)
-        zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+        zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
       return zero.value();
     };
 
@@ -4908,7 +4908,7 @@ struct DropUnitExtentBasis
 
     if (!newBasis.empty()) {
       // Will drop the leading nullptr from `basis` if there was no outer bound.
-      auto newDelinearizeOp = rewriter.create<affine::AffineDelinearizeIndexOp>(
+      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(rewriter,
           loc, delinearizeOp.getLinearIndex(), newBasis);
       int newIndex = 0;
       // Map back the new delinearized indices to the values they replace.
@@ -4973,11 +4973,11 @@ struct CancelDelinearizeOfLinearizeDisjointExactTail
       return success();
     }
 
-    Value newLinearize = rewriter.create<affine::AffineLinearizeIndexOp>(
+    Value newLinearize = affine::AffineLinearizeIndexOp::create(rewriter,
         linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
         ArrayRef<OpFoldResult>{linearizeBasis}.drop_back(numMatches),
         linearizeOp.getDisjoint());
-    auto newDelinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
         delinearizeOp.getLoc(), newLinearize,
         ArrayRef<OpFoldResult>{delinearizeBasis}.drop_back(numMatches),
         delinearizeOp.hasOuterBound());
@@ -5051,17 +5051,17 @@ struct SplitDelinearizeSpanningLastLinearizeArg final
           "need at least two elements to form the basis product");
 
     Value linearizeWithoutBack =
-        rewriter.create<affine::AffineLinearizeIndexOp>(
+        affine::AffineLinearizeIndexOp::create(rewriter,
             linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
             linearizeOp.getDynamicBasis(),
             linearizeOp.getStaticBasis().drop_back(),
             linearizeOp.getDisjoint());
     auto delinearizeWithoutSplitPart =
-        rewriter.create<affine::AffineDelinearizeIndexOp>(
+        affine::AffineDelinearizeIndexOp::create(rewriter,
             delinearizeOp.getLoc(), linearizeWithoutBack,
             delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
             delinearizeOp.hasOuterBound());
-    auto delinearizeBack = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(rewriter,
         delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
         basis.take_back(elemsToSplit), /*hasOuterBound=*/true);
     SmallVector<Value> results = llvm::to_vector(
@@ -5274,7 +5274,7 @@ OpFoldResult computeProduct(Location loc, OpBuilder &builder,
   }
   if (auto constant = dyn_cast<AffineConstantExpr>(result))
     return getAsIndexOpFoldResult(builder.getContext(), constant.getValue());
-  return builder.create<AffineApplyOp>(loc, result, dynamicPart).getResult();
+  return AffineApplyOp::create(builder, loc, result, dynamicPart).getResult();
 }
 
 /// If conseceutive outputs of a delinearize_index are linearized with the same
@@ -5439,7 +5439,7 @@ struct CancelLinearizeOfDelinearizePortion final
       newDelinBasis.erase(newDelinBasis.begin() + m.delinStart,
                           newDelinBasis.begin() + m.delinStart + m.length);
       newDelinBasis.insert(newDelinBasis.begin() + m.delinStart, newSize);
-      auto newDelinearize = rewriter.create<AffineDelinearizeIndexOp>(
+      auto newDelinearize = AffineDelinearizeIndexOp::create(rewriter,
           m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
           newDelinBasis);
 
@@ -5447,7 +5447,7 @@ struct CancelLinearizeOfDelinearizePortion final
       // create a residual affine.delinearize_index that delinearizes the
       // merged output into its component parts.
       Value combinedElem = newDelinearize.getResult(m.delinStart);
-      auto residualDelinearize = rewriter.create<AffineDelinearizeIndexOp>(
+      auto residualDelinearize = AffineDelinearizeIndexOp::create(rewriter,
           m.delinearize.getLoc(), combinedElem, basisToMerge);
 
       // Swap all the uses of the unaffected delinearize outputs to the new
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index c11f1bca5d49d..23facb83caebd 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -204,7 +204,7 @@ void AffineDataCopyGeneration::runOnBlock(Block *block,
 void AffineDataCopyGeneration::runOnOperation() {
   func::FuncOp f = getOperation();
   OpBuilder topBuilder(f.getBody());
-  zeroIndex = topBuilder.create<arith::ConstantIndexOp>(f.getLoc(), 0);
+  zeroIndex = arith::ConstantIndexOp::create(topBuilder, f.getLoc(), 0);
 
   // Nests that are copy-in's or copy-out's; the root AffineForOps of those
   // nests are stored herein.
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
index c0ef28c648ac5..5ad4893c18094 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
@@ -60,7 +60,7 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
       // Note: basis elements and their products are, definitionally,
       // non-negative, so `nuw` is justified.
       if (dynamicPart)
-        dynamicPart = rewriter.create<arith::MulIOp>(
+        dynamicPart = arith::MulIOp::create(rewriter,
             loc, dynamicPart, dynamicBasis[dynamicIndex - 1], ovflags);
       else
         dynamicPart = dynamicBasis[dynamicIndex - 1];
@@ -76,7 +76,7 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
           rewriter.createOrFold<arith::ConstantIndexOp>(loc, staticPart);
       if (dynamicPart)
         stride =
-            rewriter.create<arith::MulIOp>(loc, dynamicPart, stride, ovflags);
+            arith::MulIOp::create(rewriter, loc, dynamicPart, stride, ovflags);
       result.push_back(stride);
     }
   }
@@ -108,19 +108,19 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
   Value zero = rewriter.createOrFold<arith::ConstantIndexOp>(loc, 0);
 
   Value initialPart =
-      rewriter.create<arith::FloorDivSIOp>(loc, linearIdx, strides.front());
+      arith::FloorDivSIOp::create(rewriter, loc, linearIdx, strides.front());
   results.push_back(initialPart);
 
   auto emitModTerm = [&](Value stride) -> Value {
-    Value remainder = rewriter.create<arith::RemSIOp>(loc, linearIdx, stride);
-    Value remainderNegative = rewriter.create<arith::CmpIOp>(
+    Value remainder = arith::RemSIOp::create(rewriter, loc, linearIdx, stride);
+    Value remainderNegative = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, remainder, zero);
     // If the correction is relevant, this term is <= stride, which is known
     // to be positive in `index`. Otherwise, while 2 * stride might overflow,
     // this branch won't be taken, so the risk of `poison` is fine.
-    Value corrected = rewriter.create<arith::AddIOp>(
+    Value corrected = arith::AddIOp::create(rewriter,
         loc, remainder, stride, arith::IntegerOverflowFlags::nsw);
-    Value mod = rewriter.create<arith::SelectOp>(loc, remainderNegative,
+    Value mod = arith::SelectOp::create(rewriter, loc, remainderNegative,
                                                  corrected, remainder);
     return mod;
   };
@@ -133,7 +133,7 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
     // We know both inputs are positive, so floorDiv == div.
     // This could potentially be a divui, but it's not clear if that would
     // cause issues.
-    Value divided = rewriter.create<arith::DivSIOp>(loc, modulus, nextStride);
+    Value divided = arith::DivSIOp::create(rewriter, loc, modulus, nextStride);
     results.push_back(divided);
   }
 
@@ -169,7 +169,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   // our hands on an `OpOperand&` for the loop invariant counting function.
   for (auto [stride, idxOp] :
        llvm::zip_equal(strides, llvm::drop_end(op.getMultiIndexMutable()))) {
-    Value scaledIdx = rewriter.create<arith::MulIOp>(
+    Value scaledIdx = arith::MulIOp::create(rewriter,
         loc, idxOp.get(), stride, arith::IntegerOverflowFlags::nsw);
     int64_t numHoistableLoops = numEnclosingInvariantLoops(idxOp);
     scaledValues.emplace_back(scaledIdx, numHoistableLoops);
@@ -186,7 +186,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   Value result = scaledValues.front().first;
   for (auto [scaledValue, numHoistableLoops] : llvm::drop_begin(scaledValues)) {
     std::ignore = numHoistableLoops;
-    result = rewriter.create<arith::AddIOp>(loc, result, scaledValue,
+    result = arith::AddIOp::create(rewriter, loc, result, scaledValue,
                                             arith::IntegerOverflowFlags::nsw);
   }
   rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
index f28fb3acb7db7..bdaa2a968a0cb 100644
--- a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
@@ -89,7 +89,7 @@ static AffineApplyOp createSubApply(RewriterBase &rewriter,
   auto rhsMap = AffineMap::get(m.getNumDims(), m.getNumSymbols(), expr, ctx);
   SmallVector<Value> rhsOperands = originalOp->getOperands();
   canonicalizeMapAndOperands(&rhsMap, &rhsOperands);
-  return rewriter.create<AffineApplyOp>(originalOp.getLoc(), rhsMap,
+  return AffineApplyOp::create(rewriter, originalOp.getLoc(), rhsMap,
                                         rhsOperands);
 }
 
@@ -161,7 +161,7 @@ FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
   auto current = createSubApply(rewriter, op, subExpressions[0]);
   for (int64_t i = 1, e = subExpressions.size(); i < e; ++i) {
     Value tmp = createSubApply(rewriter, op, subExpressions[i]);
-    current = rewriter.create<AffineApplyOp>(op.getLoc(), binMap,
+    current = AffineApplyOp::create(rewriter, op.getLoc(), binMap,
                                              ValueRange{current, tmp});
     LLVM_DEBUG(DBGS() << "--reassociate into: " << current << "\n");
   }
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 8f42586e5d18a..bfc8a7296f554 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -426,7 +426,7 @@ static Value createPrivateMemRef(AffineForOp forOp,
   // consumer loop nests to reduce their live range. Currently they are added
   // at the beginning of the block, because loop nests can be reordered
   // during the fusion pass.
-  Value newMemRef = top.create<memref::AllocOp>(forOp.getLoc(), newMemRefType);
+  Value newMemRef = memref::AllocOp::create(top, forOp.getLoc(), newMemRefType);
 
   // Build an AffineMap to remap access functions based on lower bound offsets.
   SmallVector<AffineExpr, 4> remapExprs;
diff --git a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
index 92cb7075005a3..123d15f2f8d39 100644
--- a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
@@ -102,7 +102,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
 
   // Create and place the alloc right before the 'affine.for' operation.
-  Value newMemRef = bOuter.create<memref::AllocOp>(
+  Value newMemRef = memref::AllocOp::create(bOuter,
       forOp.getLoc(), newMemRefType, allocOperands);
 
   // Create 'iv mod 2' value to index the leading dimension.
@@ -110,7 +110,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   int64_t step = forOp.getStepAsInt();
   auto modTwoMap =
       AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, d0.floorDiv(step) % 2);
-  auto ivModTwoOp = bInner.create<AffineApplyOp>(forOp.getLoc(), modTwoMap,
+  auto ivModTwoOp = AffineApplyOp::create(bInner, forOp.getLoc(), modTwoMap,
                                                  forOp.getInductionVar());
 
   // replaceAllMemRefUsesWith will succeed unless the forOp body has
@@ -132,7 +132,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
   // Insert the dealloc op right after the for loop.
   bOuter.setInsertionPointAfter(forOp);
-  bOuter.create<memref::DeallocOp>(forOp.getLoc(), newMemRef);
+  memref::DeallocOp::create(bOuter, forOp.getLoc(), newMemRef);
 
   return true;
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
index 1a266b72d1f8d..9537d3e75c26a 100644
--- a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
@@ -51,10 +51,10 @@ OpFoldResult affine::materializeComputedBound(
            "expected dynamic dim");
     if (isa<RankedTensorType>(value.getType())) {
       // A tensor dimension is used: generate a tensor.dim.
-      operands.push_back(b.create<tensor::DimOp>(loc, value, *dim));
+      operands.push_back(tensor::DimOp::create(b, loc, value, *dim));
     } else if (isa<MemRefType>(value.getType())) {
       // A memref dimension is used: generate a memref.dim.
-      operands.push_back(b.create<memref::DimOp>(loc, value, *dim));
+      operands.push_back(memref::DimOp::create(b, loc, value, *dim));
     } else {
       llvm_unreachable("cannot generate DimOp for unsupported shaped type");
     }
@@ -76,7 +76,7 @@ OpFoldResult affine::materializeComputedBound(
         operands[expr.getPosition() + boundMap.getNumDims()]);
   // General case: build affine.apply op.
   return static_cast<OpFoldResult>(
-      b.create<affine::AffineApplyOp>(loc, boundMap, operands).getResult());
+      affine::AffineApplyOp::create(b, loc, boundMap, operands).getResult());
 }
 
 FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound(
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index 7fae260767e0a..b078514cbf978 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -905,7 +905,7 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   for (auto resultExpr : map.getResults()) {
     auto singleResMap =
         AffineMap::get(map.getNumDims(), map.getNumSymbols(), resultExpr);
-    auto afOp = state.builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+    auto afOp = state.AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                     mapOperands);
     results.push_back(afOp);
   }
@@ -961,7 +961,7 @@ static arith::ConstantOp vectorizeConstant(arith::ConstantOp constOp,
   auto vecForOp = cast<AffineForOp>(parentOp);
   state.builder.setInsertionPointToStart(vecForOp.getBody());
   auto newConstOp =
-      state.builder.create<arith::ConstantOp>(constOp.getLoc(), vecAttr);
+      state.arith::ConstantOp::create(builder, constOp.getLoc(), vecAttr);
 
   // Register vector replacement for future uses in the scope.
   state.registerOpVectorReplacement(constOp, newConstOp);
@@ -986,7 +986,7 @@ static Operation *vectorizeAffineApplyOp(AffineApplyOp applyOp,
     }
   }
 
-  auto newApplyOp = state.builder.create<AffineApplyOp>(
+  auto newApplyOp = state.AffineApplyOp::create(builder,
       applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
 
   // Register the new affine.apply result.
@@ -1010,7 +1010,7 @@ static arith::ConstantOp createInitialVector(arith::AtomicRMWKind reductionKind,
   auto vecTy = getVectorType(scalarTy, state.strategy);
   auto vecAttr = DenseElementsAttr::get(vecTy, valueAttr);
   auto newConstOp =
-      state.builder.create<arith::ConstantOp>(oldOperand.getLoc(), vecAttr);
+      state.arith::ConstantOp::create(builder, oldOperand.getLoc(), vecAttr);
 
   return newConstOp;
 }
@@ -1062,10 +1062,10 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   AffineMap ubMap = vecForOp.getUpperBoundMap();
   Value ub;
   if (ubMap.getNumResults() == 1)
-    ub = state.builder.create<AffineApplyOp>(loc, vecForOp.getUpperBoundMap(),
+    ub = state.AffineApplyOp::create(builder, loc, vecForOp.getUpperBoundMap(),
                                              vecForOp.getUpperBoundOperands());
   else
-    ub = state.builder.create<AffineMinOp>(loc, vecForOp.getUpperBoundMap(),
+    ub = state.AffineMinOp::create(builder, loc, vecForOp.getUpperBoundMap(),
                                            vecForOp.getUpperBoundOperands());
   // Then we compute the number of (original) iterations left in the loop.
   AffineExpr subExpr =
@@ -1080,7 +1080,7 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   Type maskTy = VectorType::get(state.strategy->vectorSizes,
                                 state.builder.getIntegerType(1));
   Value mask =
-      state.builder.create<vector::CreateMaskOp>(loc, maskTy, itersLeft);
+      state.vector::CreateMaskOp::create(builder, loc, maskTy, itersLeft);
 
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ creating a mask:\n"
                     << itersLeft << "\n"
@@ -1123,7 +1123,7 @@ static Operation *vectorizeUniform(Value uniformVal,
   state.builder.setInsertionPointAfterValue(uniformScalarRepl);
 
   auto vectorTy = getVectorType(uniformVal.getType(), state.strategy);
-  auto bcastOp = state.builder.create<BroadcastOp>(uniformVal.getLoc(),
+  auto bcastOp = state.BroadcastOp::create(builder, uniformVal.getLoc(),
                                                    vectorTy, uniformScalarRepl);
   state.registerValueVectorReplacement(uniformVal, bcastOp);
   return bcastOp;
@@ -1256,7 +1256,7 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.builder.create<vector::TransferReadOp>(
+  auto transfer = state.vector::TransferReadOp::create(builder,
       loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
       /*padding=*/std::nullopt, permutationMap);
 
@@ -1303,7 +1303,7 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.builder.create<vector::TransferWriteOp>(
+  auto transfer = state.vector::TransferWriteOp::create(builder,
       storeOp.getLoc(), vectorValue, storeOp.getMemRef(), indices,
       permutationMap);
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
@@ -1387,7 +1387,7 @@ static Operation *vectorizeAffineForOp(AffineForOp forOp,
     }
   }
 
-  auto vecForOp = state.builder.create<AffineForOp>(
+  auto vecForOp = state.AffineForOp::create(builder,
       forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(),
       forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep,
       vecIterOperands,
@@ -1512,7 +1512,7 @@ static Operation *vectorizeAffineYieldOp(AffineYieldOp yieldOp,
       // IterOperands are neutral element vectors.
       Value neutralVal = cast<AffineForOp>(newParentOp).getInits()[i];
       state.builder.setInsertionPoint(combinerOps.back());
-      Value maskedReducedVal = state.builder.create<arith::SelectOp>(
+      Value maskedReducedVal = state.arith::SelectOp::create(builder,
           reducedVal.getLoc(), mask, reducedVal, neutralVal);
       LLVM_DEBUG(
           dbgs() << "\n[early-vect]+++++ masking an input to a binary op that"
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 0501616ad912c..0f16e84263ec1 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -55,7 +55,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
 
   OpBuilder b(forOp);
   auto lbMap = forOp.getLowerBoundMap();
-  auto lb = b.create<AffineApplyOp>(forOp.getLoc(), lbMap,
+  auto lb = AffineApplyOp::create(b, forOp.getLoc(), lbMap,
                                     forOp.getLowerBoundOperands());
 
   // For each upper bound expr, get the range.
@@ -72,7 +72,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
     auto bumpMap = AffineMap::get(tripCountMap.getNumDims(),
                                   tripCountMap.getNumSymbols(), bumpExprs[i]);
     bumpValues[i] =
-        b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, tripCountOperands);
+        AffineApplyOp::create(b, forOp.getLoc(), bumpMap, tripCountOperands);
   }
 
   SmallVector<AffineExpr, 4> newUbExprs(tripCountMap.getNumResults());
@@ -135,7 +135,7 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         builder.setInsertionPointToStart(&func.getFunctionBody().front());
       else
         builder.setInsertionPoint(forOp);
-      auto constOp = builder.create<arith::ConstantIndexOp>(
+      auto constOp = arith::ConstantIndexOp::create(builder,
           forOp.getLoc(), forOp.getConstantLowerBound());
       iv.replaceAllUsesWith(constOp);
     } else {
@@ -147,7 +147,7 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         iv.replaceAllUsesWith(lbOperands[0]);
       } else {
         auto affineApplyOp =
-            builder.create<AffineApplyOp>(forOp.getLoc(), lbMap, lbOperands);
+            AffineApplyOp::create(builder, forOp.getLoc(), lbMap, lbOperands);
         iv.replaceAllUsesWith(affineApplyOp);
       }
     }
@@ -182,7 +182,7 @@ static AffineForOp generateShiftedLoop(
   assert(ubMap.getNumInputs() == ubOperands.size());
 
   auto loopChunk =
-      b.create<AffineForOp>(srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
+      AffineForOp::create(b, srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
                             ubMap, srcForOp.getStepAsInt());
   auto loopChunkIV = loopChunk.getInductionVar();
   auto srcIV = srcForOp.getInductionVar();
@@ -198,7 +198,7 @@ static AffineForOp generateShiftedLoop(
     // Generate the remapping if the shift is not zero: remappedIV = newIV -
     // shift.
     if (!srcIV.use_empty() && shift != 0) {
-      auto ivRemap = bodyBuilder.create<AffineApplyOp>(
+      auto ivRemap = AffineApplyOp::create(bodyBuilder,
           srcForOp.getLoc(),
           bodyBuilder.getSingleDimShiftAffineMap(
               -static_cast<int64_t>(srcForOp.getStepAsInt() * shift)),
@@ -434,7 +434,7 @@ static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
   for (unsigned i = 0; i < width; i++) {
     OpBuilder b(topLoop);
     // Loop bounds will be set later.
-    AffineForOp pointLoop = b.create<AffineForOp>(loc, 0, 0);
+    AffineForOp pointLoop = AffineForOp::create(b, loc, 0, 0);
     pointLoop.getBody()->getOperations().splice(
         pointLoop.getBody()->begin(), topLoop->getBlock()->getOperations(),
         topLoop);
@@ -448,7 +448,7 @@ static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
   for (unsigned i = width; i < 2 * width; i++) {
     OpBuilder b(topLoop);
     // Loop bounds will be set later.
-    AffineForOp tileSpaceLoop = b.create<AffineForOp>(loc, 0, 0);
+    AffineForOp tileSpaceLoop = AffineForOp::create(b, loc, 0, 0);
     tileSpaceLoop.getBody()->getOperations().splice(
         tileSpaceLoop.getBody()->begin(), topLoop->getBlock()->getOperations(),
         topLoop);
@@ -1063,7 +1063,7 @@ LogicalResult mlir::affine::loopUnrollByFactor(
         // iv' = iv + i * step
         auto d0 = b.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, d0 + i * step);
-        return b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, iv);
+        return AffineApplyOp::create(b, forOp.getLoc(), bumpMap, iv);
       },
       /*annotateFn=*/annotateFn,
       /*iterArgs=*/iterArgs, /*yieldedValues=*/yieldedValues);
@@ -1227,7 +1227,7 @@ LogicalResult mlir::affine::loopUnrollJamByFactor(AffineForOp forOp,
         auto d0 = builder.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, d0 + i * step);
         auto ivUnroll =
-            builder.create<AffineApplyOp>(forOp.getLoc(), bumpMap, forOpIV);
+            AffineApplyOp::create(builder, forOp.getLoc(), bumpMap, forOpIV);
         operandMaps[i - 1].map(forOpIV, ivUnroll);
       }
       // Clone the sub-block being unroll-jammed.
@@ -1556,7 +1556,7 @@ stripmineSink(AffineForOp forOp, uint64_t factor,
   for (auto t : targets) {
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator(t.getBody());
-    auto newForOp = b.create<AffineForOp>(t.getLoc(), lbOperands, lbMap,
+    auto newForOp = AffineForOp::create(b, t.getLoc(), lbOperands, lbMap,
                                           ubOperands, ubMap, originalStep);
     auto begin = t.getBody()->begin();
     // Skip terminator and `newForOp` which is just before the terminator.
@@ -1631,9 +1631,9 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
   // 1. Store the upper bound of the outermost loop in a variable.
   Value prev;
   if (!llvm::hasSingleElement(origUbMap.getResults()))
-    prev = builder.create<AffineMinOp>(loc, origUbMap, ubOperands);
+    prev = AffineMinOp::create(builder, loc, origUbMap, ubOperands);
   else
-    prev = builder.create<AffineApplyOp>(loc, origUbMap, ubOperands);
+    prev = AffineApplyOp::create(builder, loc, origUbMap, ubOperands);
   upperBoundSymbols.push_back(prev);
 
   // 2. Emit code computing the upper bound of the coalesced loop as product of
@@ -1645,15 +1645,15 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
     Value upperBound;
     // If upper bound map has more than one result, take their minimum.
     if (!llvm::hasSingleElement(origUbMap.getResults()))
-      upperBound = builder.create<AffineMinOp>(loc, origUbMap, ubOperands);
+      upperBound = AffineMinOp::create(builder, loc, origUbMap, ubOperands);
     else
-      upperBound = builder.create<AffineApplyOp>(loc, origUbMap, ubOperands);
+      upperBound = AffineApplyOp::create(builder, loc, origUbMap, ubOperands);
     upperBoundSymbols.push_back(upperBound);
     SmallVector<Value, 4> operands;
     operands.push_back(prev);
     operands.push_back(upperBound);
     // Maintain running product of loop upper bounds.
-    prev = builder.create<AffineApplyOp>(
+    prev = AffineApplyOp::create(builder,
         loc,
         AffineMap::get(/*dimCount=*/1,
                        /*symbolCount=*/1,
@@ -1683,7 +1683,7 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> operands;
       operands.push_back(previous);
       operands.push_back(upperBoundSymbols[idx]);
-      previous = builder.create<AffineApplyOp>(
+      previous = AffineApplyOp::create(builder,
           loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
@@ -1700,7 +1700,7 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> applyOperands;
       applyOperands.push_back(previous);
       applyOperands.push_back(upperBoundSymbols[idx - 1]);
-      inductionVariable = builder.create<AffineApplyOp>(
+      inductionVariable = AffineApplyOp::create(builder,
           loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
@@ -1738,21 +1738,21 @@ void mlir::affine::mapLoopToProcessorIds(scf::ForOp forOp,
 
   Value linearIndex = processorId.front();
   for (unsigned i = 1, e = processorId.size(); i < e; ++i) {
-    auto mulApplyOp = b.create<AffineApplyOp>(
+    auto mulApplyOp = AffineApplyOp::create(b,
         loc, mulMap, ValueRange{linearIndex, numProcessors[i]});
-    linearIndex = b.create<AffineApplyOp>(
+    linearIndex = AffineApplyOp::create(b,
         loc, addMap, ValueRange{mulApplyOp, processorId[i]});
   }
 
-  auto mulApplyOp = b.create<AffineApplyOp>(
+  auto mulApplyOp = AffineApplyOp::create(b,
       loc, mulMap, ValueRange{linearIndex, forOp.getStep()});
-  Value lb = b.create<AffineApplyOp>(
+  Value lb = AffineApplyOp::create(b,
       loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()});
   forOp.setLowerBound(lb);
 
   Value step = forOp.getStep();
   for (auto numProcs : numProcessors)
-    step = b.create<AffineApplyOp>(loc, mulMap, ValueRange{numProcs, step});
+    step = AffineApplyOp::create(b, loc, mulMap, ValueRange{numProcs, step});
   forOp.setStep(step);
 }
 
@@ -1889,7 +1889,7 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
 
     auto fastBufOffsetMap =
         AffineMap::get(lbOperands.size(), 0, fastBufOffsets[d]);
-    auto offset = b.create<AffineApplyOp>(loc, fastBufOffsetMap, lbOperands);
+    auto offset = AffineApplyOp::create(b, loc, fastBufOffsetMap, lbOperands);
 
     // Construct the subscript for the fast memref being copied into/from:
     // x - offset_x.
@@ -1916,16 +1916,16 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
 
   if (!isCopyOut) {
     // Copy in.
-    auto load = b.create<AffineLoadOp>(loc, memref, memIndices);
-    b.create<AffineStoreOp>(loc, load, fastMemRef, fastBufMap,
+    auto load = AffineLoadOp::create(b, loc, memref, memIndices);
+    AffineStoreOp::create(b, loc, load, fastMemRef, fastBufMap,
                             fastBufMapOperands);
     return copyNestRoot;
   }
 
   // Copy out.
   auto load =
-      b.create<AffineLoadOp>(loc, fastMemRef, fastBufMap, fastBufMapOperands);
-  b.create<AffineStoreOp>(loc, load, memref, memIndices);
+      AffineLoadOp::create(b, loc, fastMemRef, fastBufMap, fastBufMapOperands);
+  AffineStoreOp::create(b, loc, load, memref, memIndices);
   return copyNestRoot;
 }
 
@@ -1960,7 +1960,7 @@ static LogicalResult generateCopy(
 
   auto f = begin->getParentOfType<FunctionOpInterface>();
   OpBuilder topBuilder(f.getFunctionBody());
-  Value zeroIndex = topBuilder.create<arith::ConstantIndexOp>(f.getLoc(), 0);
+  Value zeroIndex = arith::ConstantIndexOp::create(topBuilder, f.getLoc(), 0);
 
   *sizeInBytes = 0;
 
@@ -2071,7 +2071,7 @@ static LogicalResult generateCopy(
         memIndices.push_back(zeroIndex);
       } else {
         memIndices.push_back(
-            top.create<arith::ConstantIndexOp>(loc, indexVal).getResult());
+            arith::ConstantIndexOp::create(top, loc, indexVal).getResult());
       }
     } else {
       // The coordinate for the start location is just the lower bound along the
@@ -2085,7 +2085,7 @@ static LogicalResult generateCopy(
       lbs[d] = lbs[d].replaceDimsAndSymbols(
           /*dimReplacements=*/{}, symReplacements, lbs[d].getNumSymbols(),
           /*numResultSyms=*/0);
-      memIndices.push_back(b.create<AffineApplyOp>(loc, lbs[d], regionSymbols));
+      memIndices.push_back(AffineApplyOp::create(b, loc, lbs[d], regionSymbols));
     }
     // The fast buffer is copied into at location zero; addressing is relative.
     bufIndices.push_back(zeroIndex);
@@ -2109,7 +2109,7 @@ static LogicalResult generateCopy(
     // Create the fast memory space buffer just before the 'affine.for'
     // operation.
     fastMemRef =
-        prologue.create<memref::AllocOp>(loc, fastMemRefType).getResult();
+        memref::AllocOp::create(prologue, loc, fastMemRefType).getResult();
     // Record it.
     fastBufferMap[memref] = fastMemRef;
     // fastMemRefType is a constant shaped memref.
@@ -2126,7 +2126,7 @@ static LogicalResult generateCopy(
     fastMemRef = fastBufferMap[memref];
   }
 
-  auto numElementsSSA = top.create<arith::ConstantIndexOp>(loc, *numElements);
+  auto numElementsSSA = arith::ConstantIndexOp::create(top, loc, *numElements);
 
   Value dmaStride;
   Value numEltPerDmaStride;
@@ -2143,8 +2143,8 @@ static LogicalResult generateCopy(
 
     if (!dmaStrideInfos.empty()) {
       dmaStride =
-          top.create<arith::ConstantIndexOp>(loc, dmaStrideInfos[0].stride);
-      numEltPerDmaStride = top.create<arith::ConstantIndexOp>(
+          arith::ConstantIndexOp::create(top, loc, dmaStrideInfos[0].stride);
+      numEltPerDmaStride = arith::ConstantIndexOp::create(top,
           loc, dmaStrideInfos[0].numEltPerStride);
     }
   }
@@ -2175,20 +2175,20 @@ static LogicalResult generateCopy(
     // Create a tag (single element 1-d memref) for the DMA.
     auto tagMemRefType = MemRefType::get({1}, top.getIntegerType(32), {},
                                          copyOptions.tagMemorySpace);
-    auto tagMemRef = prologue.create<memref::AllocOp>(loc, tagMemRefType);
+    auto tagMemRef = memref::AllocOp::create(prologue, loc, tagMemRefType);
 
     SmallVector<Value, 4> tagIndices({zeroIndex});
     auto tagAffineMap = b.getMultiDimIdentityMap(tagIndices.size());
     fullyComposeAffineMapAndOperands(&tagAffineMap, &tagIndices);
     if (!region.isWrite()) {
       // DMA non-blocking read from original buffer to fast buffer.
-      b.create<AffineDmaStartOp>(loc, memref, memAffineMap, memIndices,
+      AffineDmaStartOp::create(b, loc, memref, memAffineMap, memIndices,
                                  fastMemRef, bufAffineMap, bufIndices,
                                  tagMemRef, tagAffineMap, tagIndices,
                                  numElementsSSA, dmaStride, numEltPerDmaStride);
     } else {
       // DMA non-blocking write from fast buffer to the original memref.
-      auto op = b.create<AffineDmaStartOp>(
+      auto op = AffineDmaStartOp::create(b,
           loc, fastMemRef, bufAffineMap, bufIndices, memref, memAffineMap,
           memIndices, tagMemRef, tagAffineMap, tagIndices, numElementsSSA,
           dmaStride, numEltPerDmaStride);
@@ -2199,11 +2199,11 @@ static LogicalResult generateCopy(
     }
 
     // Matching DMA wait to block on completion; tag always has a 0 index.
-    b.create<AffineDmaWaitOp>(loc, tagMemRef, tagAffineMap, zeroIndex,
+    AffineDmaWaitOp::create(b, loc, tagMemRef, tagAffineMap, zeroIndex,
                               numElementsSSA);
 
     // Generate dealloc for the tag.
-    auto tagDeallocOp = epilogue.create<memref::DeallocOp>(loc, tagMemRef);
+    auto tagDeallocOp = memref::DeallocOp::create(epilogue, loc, tagMemRef);
     if (*nEnd == end && isCopyOutAtEndOfBlock)
       // Since new ops are being appended (for outgoing DMAs), adjust the end to
       // mark end of range of the original.
@@ -2212,7 +2212,7 @@ static LogicalResult generateCopy(
 
   // Generate dealloc for the buffer.
   if (!existingBuf) {
-    auto bufDeallocOp = epilogue.create<memref::DeallocOp>(loc, fastMemRef);
+    auto bufDeallocOp = memref::DeallocOp::create(epilogue, loc, fastMemRef);
     // When generating pointwise copies, `nEnd' has to be set to deallocOp on
     // the fast buffer (since it marks the new end insertion point).
     if (!copyOptions.generateDma && *nEnd == end && isCopyOutAtEndOfBlock)
@@ -2582,7 +2582,7 @@ AffineForOp mlir::affine::createCanonicalizedAffineForOp(
   canonicalizeMapAndOperands(&ubMap, &upperOperands);
   ubMap = removeDuplicateExprs(ubMap);
 
-  return b.create<AffineForOp>(loc, lowerOperands, lbMap, upperOperands, ubMap,
+  return AffineForOp::create(b, loc, lowerOperands, lbMap, upperOperands, ubMap,
                                step);
 }
 
@@ -2666,7 +2666,7 @@ static AffineIfOp createSeparationCondition(MutableArrayRef<AffineForOp> loops,
   SmallVector<Value, 4> setOperands;
   cst.getValues(0, cst.getNumDimAndSymbolVars(), &setOperands);
   canonicalizeSetAndOperands(&ifCondSet, &setOperands);
-  return b.create<AffineIfOp>(loops[0].getLoc(), ifCondSet, setOperands,
+  return AffineIfOp::create(b, loops[0].getLoc(), ifCondSet, setOperands,
                               /*withElseRegion=*/true);
 }
 
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 66b3f2a4f93a5..1666825aeac6a 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -58,7 +58,7 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     if (!lhs || !rhs)
       return nullptr;
-    auto op = builder.create<OpTy>(loc, lhs, rhs, overflowFlags);
+    auto op = OpTy::create(builder, loc, lhs, rhs, overflowFlags);
     return op.getResult();
   }
 
@@ -92,13 +92,13 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value remainder = builder.create<arith::RemSIOp>(loc, lhs, rhs);
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value isRemainderNegative = builder.create<arith::CmpIOp>(
+    Value remainder = arith::RemSIOp::create(builder, loc, lhs, rhs);
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value isRemainderNegative = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::slt, remainder, zeroCst);
     Value correctedRemainder =
-        builder.create<arith::AddIOp>(loc, remainder, rhs);
-    Value result = builder.create<arith::SelectOp>(
+        arith::AddIOp::create(builder, loc, remainder, rhs);
+    Value result = arith::SelectOp::create(builder,
         loc, isRemainderNegative, correctedRemainder, remainder);
     return result;
   }
@@ -131,17 +131,17 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value noneCst = builder.create<arith::ConstantIndexOp>(loc, -1);
-    Value negative = builder.create<arith::CmpIOp>(
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value noneCst = arith::ConstantIndexOp::create(builder, loc, -1);
+    Value negative = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::slt, lhs, zeroCst);
-    Value negatedDecremented = builder.create<arith::SubIOp>(loc, noneCst, lhs);
+    Value negatedDecremented = arith::SubIOp::create(builder, loc, noneCst, lhs);
     Value dividend =
-        builder.create<arith::SelectOp>(loc, negative, negatedDecremented, lhs);
-    Value quotient = builder.create<arith::DivSIOp>(loc, dividend, rhs);
+        arith::SelectOp::create(builder, loc, negative, negatedDecremented, lhs);
+    Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value correctedQuotient =
-        builder.create<arith::SubIOp>(loc, noneCst, quotient);
-    Value result = builder.create<arith::SelectOp>(loc, negative,
+        arith::SubIOp::create(builder, loc, noneCst, quotient);
+    Value result = arith::SelectOp::create(builder, loc, negative,
                                                    correctedQuotient, quotient);
     return result;
   }
@@ -170,26 +170,26 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value oneCst = builder.create<arith::ConstantIndexOp>(loc, 1);
-    Value nonPositive = builder.create<arith::CmpIOp>(
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value oneCst = arith::ConstantIndexOp::create(builder, loc, 1);
+    Value nonPositive = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::sle, lhs, zeroCst);
-    Value negated = builder.create<arith::SubIOp>(loc, zeroCst, lhs);
-    Value decremented = builder.create<arith::SubIOp>(loc, lhs, oneCst);
+    Value negated = arith::SubIOp::create(builder, loc, zeroCst, lhs);
+    Value decremented = arith::SubIOp::create(builder, loc, lhs, oneCst);
     Value dividend =
-        builder.create<arith::SelectOp>(loc, nonPositive, negated, decremented);
-    Value quotient = builder.create<arith::DivSIOp>(loc, dividend, rhs);
+        arith::SelectOp::create(builder, loc, nonPositive, negated, decremented);
+    Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value negatedQuotient =
-        builder.create<arith::SubIOp>(loc, zeroCst, quotient);
+        arith::SubIOp::create(builder, loc, zeroCst, quotient);
     Value incrementedQuotient =
-        builder.create<arith::AddIOp>(loc, quotient, oneCst);
-    Value result = builder.create<arith::SelectOp>(
+        arith::AddIOp::create(builder, loc, quotient, oneCst);
+    Value result = arith::SelectOp::create(builder,
         loc, nonPositive, negatedQuotient, incrementedQuotient);
     return result;
   }
 
   Value visitConstantExpr(AffineConstantExpr expr) {
-    auto op = builder.create<arith::ConstantIndexOp>(loc, expr.getValue());
+    auto op = arith::ConstantIndexOp::create(builder, loc, expr.getValue());
     return op.getResult();
   }
 
@@ -299,7 +299,7 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   // block.
   IRMapping operandMap;
   OpBuilder b(hoistOverOp);
-  auto hoistedIfOp = b.create<AffineIfOp>(ifOp.getLoc(), ifOp.getIntegerSet(),
+  auto hoistedIfOp = AffineIfOp::create(b, ifOp.getLoc(), ifOp.getIntegerSet(),
                                           ifOp.getOperands(),
                                           /*elseBlock=*/true);
 
@@ -370,7 +370,7 @@ mlir::affine::affineParallelize(AffineForOp forOp,
       parallelReductions, [](const LoopReduction &red) { return red.value; }));
   auto reductionKinds = llvm::to_vector<4>(llvm::map_range(
       parallelReductions, [](const LoopReduction &red) { return red.kind; }));
-  AffineParallelOp newPloop = outsideBuilder.create<AffineParallelOp>(
+  AffineParallelOp newPloop = AffineParallelOp::create(outsideBuilder,
       loc, ValueRange(reducedValues).getTypes(), reductionKinds,
       llvm::ArrayRef(lowerBoundMap), lowerBoundOperands,
       llvm::ArrayRef(upperBoundMap), upperBoundOperands,
@@ -542,7 +542,7 @@ void mlir::affine::normalizeAffineParallel(AffineParallelOp op) {
     SmallVector<Value, 8> applyOperands{dimOperands};
     applyOperands.push_back(iv);
     applyOperands.append(symbolOperands.begin(), symbolOperands.end());
-    auto apply = builder.create<AffineApplyOp>(op.getLoc(), map, applyOperands);
+    auto apply = AffineApplyOp::create(builder, op.getLoc(), map, applyOperands);
     iv.replaceAllUsesExcept(apply, apply);
   }
 
@@ -623,7 +623,7 @@ LogicalResult mlir::affine::normalizeAffineFor(AffineForOp op,
   AffineValueMap newIvToOldIvMap;
   AffineValueMap::difference(lbMap, scaleIvValueMap, &newIvToOldIvMap);
   (void)newIvToOldIvMap.canonicalize();
-  auto newIV = opBuilder.create<AffineApplyOp>(
+  auto newIV = AffineApplyOp::create(opBuilder,
       loc, newIvToOldIvMap.getAffineMap(), newIvToOldIvMap.getOperands());
   op.getInductionVar().replaceAllUsesExcept(newIV->getResult(0), newIV);
   return success();
@@ -1188,7 +1188,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     for (auto resultExpr : oldMap.getResults()) {
       auto singleResMap = AffineMap::get(oldMap.getNumDims(),
                                          oldMap.getNumSymbols(), resultExpr);
-      auto afOp = builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+      auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                 oldMapOperands);
       oldMemRefOperands.push_back(afOp);
       affineApplyOps.push_back(afOp);
@@ -1215,7 +1215,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     for (auto resultExpr : indexRemap.getResults()) {
       auto singleResMap = AffineMap::get(
           indexRemap.getNumDims(), indexRemap.getNumSymbols(), resultExpr);
-      auto afOp = builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+      auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                 remapOperands);
       remapOutputs.push_back(afOp);
       affineApplyOps.push_back(afOp);
@@ -1265,7 +1265,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     // AffineMapAccessInterface, we need to apply the values of `newMapOperands`
     // to the `newMap` to get the correct indices.
     for (unsigned i = 0; i < newMemRefRank; i++) {
-      state.operands.push_back(builder.create<AffineApplyOp>(
+      state.operands.push_back(AffineApplyOp::create(builder,
           op->getLoc(),
           AffineMap::get(newMap.getNumDims(), newMap.getNumSymbols(),
                          newMap.getResult(i)),
@@ -1451,7 +1451,7 @@ void mlir::affine::createAffineComputationSlice(
   for (auto resultExpr : composedMap.getResults()) {
     auto singleResMap = AffineMap::get(composedMap.getNumDims(),
                                        composedMap.getNumSymbols(), resultExpr);
-    sliceOps->push_back(builder.create<AffineApplyOp>(
+    sliceOps->push_back(AffineApplyOp::create(builder,
         opInst->getLoc(), singleResMap, composedOpOperands));
   }
 
@@ -1682,7 +1682,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
       // Create ConstantOp for static dimension.
       auto constantAttr = b.getIntegerAttr(b.getIndexType(), oldMemRefShape[d]);
       inAffineApply.emplace_back(
-          b.create<arith::ConstantOp>(allocOp.getLoc(), constantAttr));
+          arith::ConstantOp::create(b, allocOp.getLoc(), constantAttr));
     }
   }
 
@@ -1706,7 +1706,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
       AffineMap newMap =
           AffineMap::get(map.getNumInputs(), map.getNumSymbols(), newMapOutput);
       Value affineApp =
-          b.create<AffineApplyOp>(allocOp.getLoc(), newMap, inAffineApply);
+          AffineApplyOp::create(b, allocOp.getLoc(), newMap, inAffineApply);
       newDynamicSizes.emplace_back(affineApp);
     }
     newDimIdx++;
@@ -1742,10 +1742,10 @@ LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp allocOp) {
                           newDynamicSizes);
     // Add the new dynamic sizes in new AllocOp.
     newAlloc =
-        b.create<AllocLikeOp>(allocOp.getLoc(), newMemRefType, newDynamicSizes,
+        AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType, newDynamicSizes,
                               allocOp.getAlignmentAttr());
   } else {
-    newAlloc = b.create<AllocLikeOp>(allocOp.getLoc(), newMemRefType,
+    newAlloc = AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType,
                                      allocOp.getAlignmentAttr());
   }
   // Replace all uses of the old memref.
@@ -1804,10 +1804,10 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0, e = memrefType.getRank(); i < e; i++) {
     if (memrefType.isDynamicDim(i))
       mapOperands[i] =
-          b.create<arith::SubIOp>(loc, oldSizes[0].getType(), oldSizes[idx++],
-                                  b.create<arith::ConstantIndexOp>(loc, 1));
+          arith::SubIOp::create(b, loc, oldSizes[0].getType(), oldSizes[idx++],
+                                  arith::ConstantIndexOp::create(b, loc, 1));
     else
-      mapOperands[i] = b.create<arith::ConstantIndexOp>(loc, oldShape[i] - 1);
+      mapOperands[i] = arith::ConstantIndexOp::create(b, loc, oldShape[i] - 1);
   }
   for (unsigned i = 0, e = oldStrides.size(); i < e; i++)
     mapOperands[memrefType.getRank() + i] = oldStrides[i];
@@ -1817,7 +1817,7 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0; i < newRank; i++) {
     if (!newMemRefType.isDynamicDim(i))
       continue;
-    newSizes.push_back(b.create<AffineApplyOp>(
+    newSizes.push_back(AffineApplyOp::create(b,
         loc,
         AffineMap::get(oldLayoutMap.getNumDims(), oldLayoutMap.getNumSymbols(),
                        oldLayoutMap.getResult(i)),
@@ -1825,11 +1825,11 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   }
   for (unsigned i = 0, e = newSizes.size(); i < e; i++) {
     newSizes[i] =
-        b.create<arith::AddIOp>(loc, newSizes[i].getType(), newSizes[i],
-                                b.create<arith::ConstantIndexOp>(loc, 1));
+        arith::AddIOp::create(b, loc, newSizes[i].getType(), newSizes[i],
+                                arith::ConstantIndexOp::create(b, loc, 1));
   }
   // Create the new reinterpret_cast op.
-  auto newReinterpretCast = b.create<memref::ReinterpretCastOp>(
+  auto newReinterpretCast = memref::ReinterpretCastOp::create(b,
       loc, newMemRefType, reinterpretCastOp.getSource(),
       /*offsets=*/ValueRange(), newSizes,
       /*strides=*/ValueRange(),
diff --git a/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp b/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
index ebcb951cf3518..e7cbee6b06c45 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
@@ -64,7 +64,7 @@ Operation *arith::ArithDialect::materializeConstant(OpBuilder &builder,
                                                     Attribute value, Type type,
                                                     Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   return ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index ae2a00cedf6f5..c9b0c8414b775 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -243,7 +243,7 @@ bool arith::ConstantOp::isBuildableWith(Attribute value, Type type) {
 ConstantOp arith::ConstantOp::materialize(OpBuilder &builder, Attribute value,
                                           Type type, Location loc) {
   if (isBuildableWith(value, type))
-    return builder.create<arith::ConstantOp>(loc, cast<TypedAttr>(value));
+    return arith::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
   return nullptr;
 }
 
@@ -305,7 +305,7 @@ Value mlir::arith::getZeroConstant(OpBuilder &builder, Location loc,
          "type doesn't have a zero representation");
   TypedAttr zeroAttr = builder.getZeroAttr(type);
   assert(zeroAttr && "unsupported type for zero attribute");
-  return builder.create<arith::ConstantOp>(loc, zeroAttr);
+  return arith::ConstantOp::create(builder, loc, zeroAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2335,7 +2335,7 @@ class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
     // comparison.
     rewriter.replaceOpWithNewOp<CmpIOp>(
         op, pred, intVal,
-        rewriter.create<ConstantOp>(
+        ConstantOp::create(rewriter,
             op.getLoc(), intVal.getType(),
             rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
     return success();
@@ -2374,9 +2374,9 @@ struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
         matchPattern(op.getFalseValue(), m_One())) {
       rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
           op, op.getType(),
-          rewriter.create<arith::XOrIOp>(
+          arith::XOrIOp::create(rewriter,
               op.getLoc(), op.getCondition(),
-              rewriter.create<arith::ConstantIntOp>(
+              arith::ConstantIntOp::create(rewriter,
                   op.getLoc(), op.getCondition().getType(), 1)));
       return success();
     }
@@ -2693,7 +2693,7 @@ Value mlir::arith::getIdentityValue(AtomicRMWKind op, Type resultType,
                                     bool useOnlyFiniteValue) {
   auto attr =
       getIdentityValueAttr(op, resultType, builder, loc, useOnlyFiniteValue);
-  return builder.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(builder, loc, attr);
 }
 
 /// Return the value obtained by applying the reduction operation kind
@@ -2702,33 +2702,33 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
                                   Location loc, Value lhs, Value rhs) {
   switch (op) {
   case AtomicRMWKind::addf:
-    return builder.create<arith::AddFOp>(loc, lhs, rhs);
+    return arith::AddFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::addi:
-    return builder.create<arith::AddIOp>(loc, lhs, rhs);
+    return arith::AddIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::mulf:
-    return builder.create<arith::MulFOp>(loc, lhs, rhs);
+    return arith::MulFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::muli:
-    return builder.create<arith::MulIOp>(loc, lhs, rhs);
+    return arith::MulIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maximumf:
-    return builder.create<arith::MaximumFOp>(loc, lhs, rhs);
+    return arith::MaximumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minimumf:
-    return builder.create<arith::MinimumFOp>(loc, lhs, rhs);
+    return arith::MinimumFOp::create(builder, loc, lhs, rhs);
    case AtomicRMWKind::maxnumf:
-    return builder.create<arith::MaxNumFOp>(loc, lhs, rhs);
+    return arith::MaxNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minnumf:
-    return builder.create<arith::MinNumFOp>(loc, lhs, rhs);
+    return arith::MinNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maxs:
-    return builder.create<arith::MaxSIOp>(loc, lhs, rhs);
+    return arith::MaxSIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::mins:
-    return builder.create<arith::MinSIOp>(loc, lhs, rhs);
+    return arith::MinSIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maxu:
-    return builder.create<arith::MaxUIOp>(loc, lhs, rhs);
+    return arith::MaxUIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minu:
-    return builder.create<arith::MinUIOp>(loc, lhs, rhs);
+    return arith::MinUIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::ori:
-    return builder.create<arith::OrIOp>(loc, lhs, rhs);
+    return arith::OrIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::andi:
-    return builder.create<arith::AndIOp>(loc, lhs, rhs);
+    return arith::AndIOp::create(builder, loc, lhs, rhs);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index f2e7732e8ea4a..d21da23903544 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -67,7 +67,7 @@ struct SelectOpInterface
       return state.getMemrefWithUniqueOwnership(builder, value,
                                                 value.getParentBlock());
 
-    Value ownership = builder.create<arith::SelectOp>(
+    Value ownership = arith::SelectOp::create(builder,
         op->getLoc(), selectOp.getCondition(),
         state.getOwnership(selectOp.getTrueValue(), block).getIndicator(),
         state.getOwnership(selectOp.getFalseValue(), block).getIndicator());
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
index afee162053bea..b073a31850678 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -170,10 +170,10 @@ struct SelectOpInterface
         return failure();
       if (trueBuffer.getType() != *targetType)
         trueBuffer =
-            rewriter.create<memref::CastOp>(loc, *targetType, trueBuffer);
+            memref::CastOp::create(rewriter, loc, *targetType, trueBuffer);
       if (falseBuffer.getType() != *targetType)
         falseBuffer =
-            rewriter.create<memref::CastOp>(loc, *targetType, falseBuffer);
+            memref::CastOp::create(rewriter, loc, *targetType, falseBuffer);
     }
 
     replaceOpWithNewBufferizedOp<arith::SelectOp>(
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
index 62022bfb7df1e..674bd7b8ee201 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
@@ -75,7 +75,7 @@ LogicalResult EmulateFloatPattern::matchAndRewrite(
   for (auto [res, oldType, newType] : llvm::zip_equal(
            MutableArrayRef{newResults}, op->getResultTypes(), resultTypes)) {
     if (oldType != newType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, oldType, res);
+      auto truncFOp = arith::TruncFOp::create(rewriter, loc, oldType, res);
       truncFOp.setFastmath(arith::FastMathFlags::contract);
       res = truncFOp.getResult();
     }
@@ -98,7 +98,7 @@ void mlir::arith::populateEmulateUnsupportedFloatsConversions(
   });
   converter.addTargetMaterialization(
       [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        auto extFOp = arith::ExtFOp::create(b, loc, target, input);
         extFOp.setFastmath(arith::FastMathFlags::contract);
         return extFOp;
       });
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
index d5d1559c658ff..44e896a97cb4b 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
@@ -72,7 +72,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
 
   // Scalarize the result in case of 1D vectors.
   if (shape.size() == 1)
-    return rewriter.create<vector::ExtractOp>(loc, input, lastOffset);
+    return vector::ExtractOp::create(rewriter, loc, input, lastOffset);
 
   SmallVector<int64_t> offsets(shape.size(), 0);
   offsets.back() = lastOffset;
@@ -80,7 +80,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
   sizes.back() = 1;
   SmallVector<int64_t> strides(shape.size(), 1);
 
-  return rewriter.create<vector::ExtractStridedSliceOp>(loc, input, offsets,
+  return vector::ExtractStridedSliceOp::create(rewriter, loc, input, offsets,
                                                         sizes, strides);
 }
 
@@ -107,7 +107,7 @@ static Value dropTrailingX1Dim(ConversionPatternRewriter &rewriter,
   assert(shape.back() == 1 && "Expected the last vector dim to be x1");
 
   auto newVecTy = VectorType::get(shape.drop_back(), vecTy.getElementType());
-  return rewriter.create<vector::ShapeCastOp>(loc, newVecTy, input);
+  return vector::ShapeCastOp::create(rewriter, loc, newVecTy, input);
 }
 
 /// Performs a vector shape cast to append an x1 dimension. If the
@@ -122,7 +122,7 @@ static Value appendX1Dim(ConversionPatternRewriter &rewriter, Location loc,
   auto newShape = llvm::to_vector(vecTy.getShape());
   newShape.push_back(1);
   auto newTy = VectorType::get(newShape, vecTy.getElementType());
-  return rewriter.create<vector::ShapeCastOp>(loc, newTy, input);
+  return vector::ShapeCastOp::create(rewriter, loc, newTy, input);
 }
 
 /// Inserts the `source` vector slice into the `dest` vector at offset
@@ -136,12 +136,12 @@ static Value insertLastDimSlice(ConversionPatternRewriter &rewriter,
 
   // Handle scalar source.
   if (isa<IntegerType>(source.getType()))
-    return rewriter.create<vector::InsertOp>(loc, source, dest, lastOffset);
+    return vector::InsertOp::create(rewriter, loc, source, dest, lastOffset);
 
   SmallVector<int64_t> offsets(shape.size(), 0);
   offsets.back() = lastOffset;
   SmallVector<int64_t> strides(shape.size(), 1);
-  return rewriter.create<vector::InsertStridedSliceOp>(loc, source, dest,
+  return vector::InsertStridedSliceOp::create(rewriter, loc, source, dest,
                                                        offsets, strides);
 }
 
@@ -254,12 +254,12 @@ struct ConvertAddI final : OpConversionPattern<arith::AddIOp> {
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
     auto lowSum =
-        rewriter.create<arith::AddUIExtendedOp>(loc, lhsElem0, rhsElem0);
+        arith::AddUIExtendedOp::create(rewriter, loc, lhsElem0, rhsElem0);
     Value overflowVal =
-        rewriter.create<arith::ExtUIOp>(loc, newElemTy, lowSum.getOverflow());
+        arith::ExtUIOp::create(rewriter, loc, newElemTy, lowSum.getOverflow());
 
-    Value high0 = rewriter.create<arith::AddIOp>(loc, overflowVal, lhsElem1);
-    Value high = rewriter.create<arith::AddIOp>(loc, high0, rhsElem1);
+    Value high0 = arith::AddIOp::create(rewriter, loc, overflowVal, lhsElem1);
+    Value high = arith::AddIOp::create(rewriter, loc, high0, rhsElem1);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {lowSum.getSum(), high});
@@ -293,8 +293,8 @@ struct ConvertBitwiseBinary final : OpConversionPattern<BinaryOp> {
     auto [rhsElem0, rhsElem1] =
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
-    Value resElem0 = rewriter.create<BinaryOp>(loc, lhsElem0, rhsElem0);
-    Value resElem1 = rewriter.create<BinaryOp>(loc, lhsElem1, rhsElem1);
+    Value resElem0 = BinaryOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem1 = BinaryOp::create(rewriter, loc, lhsElem1, rhsElem1);
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
     rewriter.replaceOp(op, resultVec);
@@ -346,26 +346,26 @@ struct ConvertCmpI final : OpConversionPattern<arith::CmpIOp> {
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
     Value lowCmp =
-        rewriter.create<arith::CmpIOp>(loc, lowPred, lhsElem0, rhsElem0);
+        arith::CmpIOp::create(rewriter, loc, lowPred, lhsElem0, rhsElem0);
     Value highCmp =
-        rewriter.create<arith::CmpIOp>(loc, highPred, lhsElem1, rhsElem1);
+        arith::CmpIOp::create(rewriter, loc, highPred, lhsElem1, rhsElem1);
 
     Value cmpResult{};
     switch (highPred) {
     case arith::CmpIPredicate::eq: {
-      cmpResult = rewriter.create<arith::AndIOp>(loc, lowCmp, highCmp);
+      cmpResult = arith::AndIOp::create(rewriter, loc, lowCmp, highCmp);
       break;
     }
     case arith::CmpIPredicate::ne: {
-      cmpResult = rewriter.create<arith::OrIOp>(loc, lowCmp, highCmp);
+      cmpResult = arith::OrIOp::create(rewriter, loc, lowCmp, highCmp);
       break;
     }
     default: {
       // Handle inequality checks.
-      Value highEq = rewriter.create<arith::CmpIOp>(
+      Value highEq = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, lhsElem1, rhsElem1);
       cmpResult =
-          rewriter.create<arith::SelectOp>(loc, highEq, lowCmp, highCmp);
+          arith::SelectOp::create(rewriter, loc, highEq, lowCmp, highCmp);
       break;
     }
     }
@@ -401,14 +401,14 @@ struct ConvertMulI final : OpConversionPattern<arith::MulIOp> {
     // Multiplying two i2N integers produces (at most) an i4N result, but
     // because the calculation of top i2N is not necessary, we omit it.
     auto mulLowLow =
-        rewriter.create<arith::MulUIExtendedOp>(loc, lhsElem0, rhsElem0);
-    Value mulLowHi = rewriter.create<arith::MulIOp>(loc, lhsElem0, rhsElem1);
-    Value mulHiLow = rewriter.create<arith::MulIOp>(loc, lhsElem1, rhsElem0);
+        arith::MulUIExtendedOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value mulLowHi = arith::MulIOp::create(rewriter, loc, lhsElem0, rhsElem1);
+    Value mulHiLow = arith::MulIOp::create(rewriter, loc, lhsElem1, rhsElem0);
 
     Value resLow = mulLowLow.getLow();
     Value resHi =
-        rewriter.create<arith::AddIOp>(loc, mulLowLow.getHigh(), mulLowHi);
-    resHi = rewriter.create<arith::AddIOp>(loc, resHi, mulHiLow);
+        arith::AddIOp::create(rewriter, loc, mulLowLow.getHigh(), mulLowHi);
+    resHi = arith::AddIOp::create(rewriter, loc, resHi, mulHiLow);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resLow, resHi});
@@ -443,10 +443,10 @@ struct ConvertExtSI final : OpConversionPattern<arith::ExtSIOp> {
         loc, newResultComponentTy, newOperand);
     Value operandZeroCst =
         createScalarOrSplatConstant(rewriter, loc, newResultComponentTy, 0);
-    Value signBit = rewriter.create<arith::CmpIOp>(
+    Value signBit = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, extended, operandZeroCst);
     Value signValue =
-        rewriter.create<arith::ExtSIOp>(loc, newResultComponentTy, signBit);
+        arith::ExtSIOp::create(rewriter, loc, newResultComponentTy, signBit);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {extended, signValue});
@@ -508,7 +508,7 @@ struct ConvertMaxMin final : OpConversionPattern<SourceOp> {
     // Rewrite Max*I/Min*I as compare and select over original operands. Let
     // the CmpI and Select emulation patterns handle the final legalization.
     Value cmp =
-        rewriter.create<arith::CmpIOp>(loc, CmpPred, op.getLhs(), op.getRhs());
+        arith::CmpIOp::create(rewriter, loc, CmpPred, op.getLhs(), op.getRhs());
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, op.getLhs(),
                                                  op.getRhs());
     return success();
@@ -587,7 +587,7 @@ struct ConvertIndexCastIndexToInt final : OpConversionPattern<CastOp> {
     // Sign or zero-extend the result. Let the matching conversion pattern
     // legalize the extension op.
     Value underlyingVal =
-        rewriter.create<CastOp>(loc, narrowTy, adaptor.getIn());
+        CastOp::create(rewriter, loc, narrowTy, adaptor.getIn());
     rewriter.replaceOpWithNewOp<ExtensionOp>(op, resultType, underlyingVal);
     return success();
   }
@@ -616,9 +616,9 @@ struct ConvertSelect final : OpConversionPattern<arith::SelectOp> {
     Value cond = appendX1Dim(rewriter, loc, adaptor.getCondition());
 
     Value resElem0 =
-        rewriter.create<arith::SelectOp>(loc, cond, trueElem0, falseElem0);
+        arith::SelectOp::create(rewriter, loc, cond, trueElem0, falseElem0);
     Value resElem1 =
-        rewriter.create<arith::SelectOp>(loc, cond, trueElem1, falseElem1);
+        arith::SelectOp::create(rewriter, loc, cond, trueElem1, falseElem1);
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
     rewriter.replaceOp(op, resultVec);
@@ -680,33 +680,33 @@ struct ConvertShLI final : OpConversionPattern<arith::ShLIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = rewriter.create<arith::CmpIOp>(
+    Value illegalElemShift = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem0, rhsElem0);
-    Value resElem0 = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShLIOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem0 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                       zeroCst, shiftedElem0);
 
-    Value cappedShiftAmount = rewriter.create<arith::SelectOp>(
+    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value rightShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, elemBitWidth, cappedShiftAmount);
+        arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedRight =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem0, rightShiftAmount);
+        arith::ShRUIOp::create(rewriter, loc, lhsElem0, rightShiftAmount);
     Value overshotShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, rhsElem0, elemBitWidth);
+        arith::SubIOp::create(rewriter, loc, rhsElem0, elemBitWidth);
     Value shiftedLeft =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem0, overshotShiftAmount);
+        arith::ShLIOp::create(rewriter, loc, lhsElem0, overshotShiftAmount);
 
     Value shiftedElem1 =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem1, rhsElem0);
-    Value resElem1High = rewriter.create<arith::SelectOp>(
+        arith::ShLIOp::create(rewriter, loc, lhsElem1, rhsElem0);
+    Value resElem1High = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, zeroCst, shiftedElem1);
-    Value resElem1Low = rewriter.create<arith::SelectOp>(
+    Value resElem1Low = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, shiftedLeft, shiftedRight);
     Value resElem1 =
-        rewriter.create<arith::OrIOp>(loc, resElem1Low, resElem1High);
+        arith::OrIOp::create(rewriter, loc, resElem1Low, resElem1High);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
@@ -769,33 +769,33 @@ struct ConvertShRUI final : OpConversionPattern<arith::ShRUIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = rewriter.create<arith::CmpIOp>(
+    Value illegalElemShift = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem0, rhsElem0);
-    Value resElem0Low = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShRUIOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem0Low = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                          zeroCst, shiftedElem0);
     Value shiftedElem1 =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem1, rhsElem0);
-    Value resElem1 = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShRUIOp::create(rewriter, loc, lhsElem1, rhsElem0);
+    Value resElem1 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                       zeroCst, shiftedElem1);
 
-    Value cappedShiftAmount = rewriter.create<arith::SelectOp>(
+    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value leftShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, elemBitWidth, cappedShiftAmount);
+        arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedLeft =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem1, leftShiftAmount);
+        arith::ShLIOp::create(rewriter, loc, lhsElem1, leftShiftAmount);
     Value overshotShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, rhsElem0, elemBitWidth);
+        arith::SubIOp::create(rewriter, loc, rhsElem0, elemBitWidth);
     Value shiftedRight =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem1, overshotShiftAmount);
+        arith::ShRUIOp::create(rewriter, loc, lhsElem1, overshotShiftAmount);
 
-    Value resElem0High = rewriter.create<arith::SelectOp>(
+    Value resElem0High = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, shiftedRight, shiftedLeft);
     Value resElem0 =
-        rewriter.create<arith::OrIOp>(loc, resElem0Low, resElem0High);
+        arith::OrIOp::create(rewriter, loc, resElem0Low, resElem0High);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
@@ -832,32 +832,32 @@ struct ConvertShRSI final : OpConversionPattern<arith::ShRSIOp> {
     // Perform as many ops over the narrow integer type as possible and let the
     // other emulation patterns convert the rest.
     Value elemZero = createScalarOrSplatConstant(rewriter, loc, narrowTy, 0);
-    Value signBit = rewriter.create<arith::CmpIOp>(
+    Value signBit = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, lhsElem1, elemZero);
     signBit = dropTrailingX1Dim(rewriter, loc, signBit);
 
     // Create a bit pattern of either all ones or all zeros. Then shift it left
     // to calculate the sign extension bits created by shifting the original
     // sign bit right.
-    Value allSign = rewriter.create<arith::ExtSIOp>(loc, oldTy, signBit);
+    Value allSign = arith::ExtSIOp::create(rewriter, loc, oldTy, signBit);
     Value maxShift =
         createScalarOrSplatConstant(rewriter, loc, narrowTy, origBitwidth);
     Value numNonSignExtBits =
-        rewriter.create<arith::SubIOp>(loc, maxShift, rhsElem0);
+        arith::SubIOp::create(rewriter, loc, maxShift, rhsElem0);
     numNonSignExtBits = dropTrailingX1Dim(rewriter, loc, numNonSignExtBits);
     numNonSignExtBits =
-        rewriter.create<arith::ExtUIOp>(loc, oldTy, numNonSignExtBits);
+        arith::ExtUIOp::create(rewriter, loc, oldTy, numNonSignExtBits);
     Value signBits =
-        rewriter.create<arith::ShLIOp>(loc, allSign, numNonSignExtBits);
+        arith::ShLIOp::create(rewriter, loc, allSign, numNonSignExtBits);
 
     // Use original arguments to create the right shift.
     Value shrui =
-        rewriter.create<arith::ShRUIOp>(loc, op.getLhs(), op.getRhs());
-    Value shrsi = rewriter.create<arith::OrIOp>(loc, shrui, signBits);
+        arith::ShRUIOp::create(rewriter, loc, op.getLhs(), op.getRhs());
+    Value shrsi = arith::OrIOp::create(rewriter, loc, shrui, signBits);
 
     // Handle shifting by zero. This is necessary when the `signBits` shift is
     // invalid.
-    Value isNoop = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value isNoop = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                   rhsElem0, elemZero);
     isNoop = dropTrailingX1Dim(rewriter, loc, isNoop);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNoop, op.getLhs(),
@@ -892,14 +892,14 @@ struct ConvertSubI final : OpConversionPattern<arith::SubIOp> {
 
     // Emulates LHS - RHS by [LHS0 - RHS0, LHS1 - RHS1 - CARRY] where
     // CARRY is 1 or 0.
-    Value low = rewriter.create<arith::SubIOp>(loc, lhsElem0, rhsElem0);
+    Value low = arith::SubIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     // We have a carry if lhsElem0 < rhsElem0.
-    Value carry0 = rewriter.create<arith::CmpIOp>(
+    Value carry0 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, lhsElem0, rhsElem0);
-    Value carryVal = rewriter.create<arith::ExtUIOp>(loc, newElemTy, carry0);
+    Value carryVal = arith::ExtUIOp::create(rewriter, loc, newElemTy, carry0);
 
-    Value high0 = rewriter.create<arith::SubIOp>(loc, lhsElem1, carryVal);
-    Value high = rewriter.create<arith::SubIOp>(loc, high0, rhsElem1);
+    Value high0 = arith::SubIOp::create(rewriter, loc, lhsElem1, carryVal);
+    Value high = arith::SubIOp::create(rewriter, loc, high0, rhsElem1);
 
     Value resultVec = constructResultVector(rewriter, loc, newTy, {low, high});
     rewriter.replaceOp(op, resultVec);
@@ -933,13 +933,13 @@ struct ConvertSIToFP final : OpConversionPattern<arith::SIToFPOp> {
     // result or not based on that sign bit. We implement negation by
     // subtracting from zero. Note that this relies on the the other conversion
     // patterns to legalize created ops and narrow the bit widths.
-    Value isNeg = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+    Value isNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                  in, zeroCst);
-    Value neg = rewriter.create<arith::SubIOp>(loc, zeroCst, in);
-    Value abs = rewriter.create<arith::SelectOp>(loc, isNeg, neg, in);
+    Value neg = arith::SubIOp::create(rewriter, loc, zeroCst, in);
+    Value abs = arith::SelectOp::create(rewriter, loc, isNeg, neg, in);
 
-    Value absResult = rewriter.create<arith::UIToFPOp>(loc, op.getType(), abs);
-    Value negResult = rewriter.create<arith::NegFOp>(loc, absResult);
+    Value absResult = arith::UIToFPOp::create(rewriter, loc, op.getType(), abs);
+    Value negResult = arith::NegFOp::create(rewriter, loc, absResult);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNeg, negResult,
                                                  absResult);
     return success();
@@ -985,13 +985,13 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     //
     // Note 2: We do not strictly need the `hi == 0`, case, but it makes
     // constant folding easier.
-    Value hiEqZero = rewriter.create<arith::CmpIOp>(
+    Value hiEqZero = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, hiInt, zeroCst);
 
     Type resultTy = op.getType();
     Type resultElemTy = getElementTypeOrSelf(resultTy);
-    Value lowFp = rewriter.create<arith::UIToFPOp>(loc, resultTy, lowInt);
-    Value hiFp = rewriter.create<arith::UIToFPOp>(loc, resultTy, hiInt);
+    Value lowFp = arith::UIToFPOp::create(rewriter, loc, resultTy, lowInt);
+    Value hiFp = arith::UIToFPOp::create(rewriter, loc, resultTy, hiInt);
 
     int64_t pow2Int = int64_t(1) << newBitWidth;
     TypedAttr pow2Attr =
@@ -999,10 +999,10 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     if (auto vecTy = dyn_cast<VectorType>(resultTy))
       pow2Attr = SplatElementsAttr::get(vecTy, pow2Attr);
 
-    Value pow2Val = rewriter.create<arith::ConstantOp>(loc, resultTy, pow2Attr);
+    Value pow2Val = arith::ConstantOp::create(rewriter, loc, resultTy, pow2Attr);
 
-    Value hiVal = rewriter.create<arith::MulFOp>(loc, hiFp, pow2Val);
-    Value result = rewriter.create<arith::AddFOp>(loc, lowFp, hiVal);
+    Value hiVal = arith::MulFOp::create(rewriter, loc, hiFp, pow2Val);
+    Value result = arith::AddFOp::create(rewriter, loc, lowFp, hiVal);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, hiEqZero, lowFp, result);
     return success();
@@ -1037,22 +1037,22 @@ struct ConvertFPToSI final : OpConversionPattern<arith::FPToSIOp> {
     // result is UB.
 
     TypedAttr zeroAttr = rewriter.getZeroAttr(fpTy);
-    Value zeroCst = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
+    Value zeroCst = arith::ConstantOp::create(rewriter, loc, zeroAttr);
     Value zeroCstInt = createScalarOrSplatConstant(rewriter, loc, intTy, 0);
 
     // Get the absolute value. One could have used math.absf here, but that
     // introduces an extra dependency.
-    Value isNeg = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OLT,
+    Value isNeg = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OLT,
                                                  inFp, zeroCst);
-    Value negInFp = rewriter.create<arith::NegFOp>(loc, inFp);
+    Value negInFp = arith::NegFOp::create(rewriter, loc, inFp);
 
-    Value absVal = rewriter.create<arith::SelectOp>(loc, isNeg, negInFp, inFp);
+    Value absVal = arith::SelectOp::create(rewriter, loc, isNeg, negInFp, inFp);
 
     // Defer the absolute value to fptoui.
-    Value res = rewriter.create<arith::FPToUIOp>(loc, intTy, absVal);
+    Value res = arith::FPToUIOp::create(rewriter, loc, intTy, absVal);
 
     // Negate the value if < 0 .
-    Value neg = rewriter.create<arith::SubIOp>(loc, zeroCstInt, res);
+    Value neg = arith::SubIOp::create(rewriter, loc, zeroCstInt, res);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNeg, neg, res);
     return success();
@@ -1109,17 +1109,17 @@ struct ConvertFPToUI final : OpConversionPattern<arith::FPToUIOp> {
     if (auto vecType = dyn_cast<VectorType>(fpTy))
       powBitwidthAttr = SplatElementsAttr::get(vecType, powBitwidthAttr);
     Value powBitwidthFloatCst =
-        rewriter.create<arith::ConstantOp>(loc, powBitwidthAttr);
+        arith::ConstantOp::create(rewriter, loc, powBitwidthAttr);
 
     Value fpDivPowBitwidth =
-        rewriter.create<arith::DivFOp>(loc, inFp, powBitwidthFloatCst);
+        arith::DivFOp::create(rewriter, loc, inFp, powBitwidthFloatCst);
     Value resHigh =
-        rewriter.create<arith::FPToUIOp>(loc, newHalfType, fpDivPowBitwidth);
+        arith::FPToUIOp::create(rewriter, loc, newHalfType, fpDivPowBitwidth);
     // Calculate fp - resHigh * 2^N by getting the remainder of the division
     Value remainder =
-        rewriter.create<arith::RemFOp>(loc, inFp, powBitwidthFloatCst);
+        arith::RemFOp::create(rewriter, loc, inFp, powBitwidthFloatCst);
     Value resLow =
-        rewriter.create<arith::FPToUIOp>(loc, newHalfType, remainder);
+        arith::FPToUIOp::create(rewriter, loc, newHalfType, remainder);
 
     Value high = appendX1Dim(rewriter, loc, resHigh);
     Value low = appendX1Dim(rewriter, loc, resLow);
diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
index dfa01844737c6..7dd0541f3c4b5 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
@@ -31,10 +31,10 @@ static Value createConst(Location loc, Type type, int value,
                          PatternRewriter &rewriter) {
   auto attr = rewriter.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return rewriter.create<arith::ConstantOp>(
+    return arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
-  return rewriter.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(rewriter, loc, attr);
 }
 
 /// Create a float constant.
@@ -42,11 +42,11 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                               PatternRewriter &rewriter) {
   auto attr = rewriter.getFloatAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return rewriter.create<arith::ConstantOp>(
+    return arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return rewriter.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(rewriter, loc, attr);
 }
 
 /// Creates shapedType using shape from cloneFrom and base type from cloneTo
@@ -70,11 +70,11 @@ struct CeilDivUIOpConverter : public OpRewritePattern<arith::CeilDivUIOp> {
     Value b = op.getRhs();
     Value zero = createConst(loc, a.getType(), 0, rewriter);
     Value compare =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, a, zero);
     Value one = createConst(loc, a.getType(), 1, rewriter);
-    Value minusOne = rewriter.create<arith::SubIOp>(loc, a, one);
-    Value quotient = rewriter.create<arith::DivUIOp>(loc, minusOne, b);
-    Value plusOne = rewriter.create<arith::AddIOp>(loc, quotient, one);
+    Value minusOne = arith::SubIOp::create(rewriter, loc, a, one);
+    Value quotient = arith::DivUIOp::create(rewriter, loc, minusOne, b);
+    Value plusOne = arith::AddIOp::create(rewriter, loc, quotient, one);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, compare, zero, plusOne);
     return success();
   }
@@ -99,22 +99,22 @@ struct CeilDivSIOpConverter : public OpRewritePattern<arith::CeilDivSIOp> {
     Value zero = createConst(loc, type, 0, rewriter);
     Value one = createConst(loc, type, 1, rewriter);
 
-    Value quotient = rewriter.create<arith::DivSIOp>(loc, a, b);
-    Value product = rewriter.create<arith::MulIOp>(loc, quotient, b);
-    Value notEqualDivisor = rewriter.create<arith::CmpIOp>(
+    Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
+    Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
+    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, a, product);
 
     Value aNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
     Value bNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, b, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
 
-    Value signEqual = rewriter.create<arith::CmpIOp>(
+    Value signEqual = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, aNeg, bNeg);
     Value cond =
-        rewriter.create<arith::AndIOp>(loc, notEqualDivisor, signEqual);
+        arith::AndIOp::create(rewriter, loc, notEqualDivisor, signEqual);
 
-    Value quotientPlusOne = rewriter.create<arith::AddIOp>(loc, quotient, one);
+    Value quotientPlusOne = arith::AddIOp::create(rewriter, loc, quotient, one);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cond, quotientPlusOne,
                                                  quotient);
@@ -138,25 +138,25 @@ struct FloorDivSIOpConverter : public OpRewritePattern<arith::FloorDivSIOp> {
     Value a = op.getLhs();
     Value b = op.getRhs();
 
-    Value quotient = rewriter.create<arith::DivSIOp>(loc, a, b);
-    Value product = rewriter.create<arith::MulIOp>(loc, quotient, b);
-    Value notEqualDivisor = rewriter.create<arith::CmpIOp>(
+    Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
+    Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
+    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, a, product);
     Value zero = createConst(loc, type, 0, rewriter);
 
     Value aNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
     Value bNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, b, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
 
-    Value signOpposite = rewriter.create<arith::CmpIOp>(
+    Value signOpposite = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, aNeg, bNeg);
     Value cond =
-        rewriter.create<arith::AndIOp>(loc, notEqualDivisor, signOpposite);
+        arith::AndIOp::create(rewriter, loc, notEqualDivisor, signOpposite);
 
     Value minusOne = createConst(loc, type, -1, rewriter);
     Value quotientMinusOne =
-        rewriter.create<arith::AddIOp>(loc, quotient, minusOne);
+        arith::AddIOp::create(rewriter, loc, quotient, minusOne);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cond, quotientMinusOne,
                                                  quotient);
@@ -174,7 +174,7 @@ struct MaxMinIOpConverter : public OpRewritePattern<OpTy> {
     Value lhs = op.getLhs();
     Value rhs = op.getRhs();
 
-    Value cmp = rewriter.create<arith::CmpIOp>(op.getLoc(), pred, lhs, rhs);
+    Value cmp = arith::CmpIOp::create(rewriter, op.getLoc(), pred, lhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, lhs, rhs);
     return success();
   }
@@ -195,11 +195,11 @@ struct MaximumMinimumFOpConverter : public OpRewritePattern<OpTy> {
     static_assert(pred == arith::CmpFPredicate::UGT ||
                       pred == arith::CmpFPredicate::ULT,
                   "pred must be either UGT or ULT");
-    Value cmp = rewriter.create<arith::CmpFOp>(loc, pred, lhs, rhs);
-    Value select = rewriter.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+    Value cmp = arith::CmpFOp::create(rewriter, loc, pred, lhs, rhs);
+    Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'.
-    Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
+    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
                                                  rhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
@@ -221,11 +221,11 @@ struct MaxNumMinNumFOpConverter : public OpRewritePattern<OpTy> {
     static_assert(pred == arith::CmpFPredicate::UGT ||
                       pred == arith::CmpFPredicate::ULT,
                   "pred must be either UGT or ULT");
-    Value cmp = rewriter.create<arith::CmpFOp>(loc, pred, lhs, rhs);
-    Value select = rewriter.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+    Value cmp = arith::CmpFOp::create(rewriter, loc, pred, lhs, rhs);
+    Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where lhs is NaN: 'isNaN(lhs) ? rhs : select'.
-    Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
+    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
                                                  lhs, lhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
@@ -250,12 +250,12 @@ struct BFloat16ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type i16Ty = cloneToShapedType(operandTy, b.getI16Type());
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
 
-    Value bitcast = b.create<arith::BitcastOp>(i16Ty, operand);
-    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
+    Value bitcast = arith::BitcastOp::create(b, i16Ty, operand);
+    Value exti = arith::ExtUIOp::create(b, i32Ty, bitcast);
 
     Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
-    Value shl = b.create<arith::ShLIOp>(exti, c16);
-    Value result = b.create<arith::BitcastOp>(resultTy, shl);
+    Value shl = arith::ShLIOp::create(b, exti, c16);
+    Value result = arith::BitcastOp::create(b, resultTy, shl);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -299,7 +299,7 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     // exponent bits, that simple truncation is the desired outcome for
     // infinities.
     Value isNan =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNE, operand, operand);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNE, operand, operand);
     // Constant used to make the rounding bias.
     Value c7FFF = createConst(op.getLoc(), i32Ty, 0x7fff, rewriter);
     // Constant used to generate a quiet NaN.
@@ -308,30 +308,30 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
     Value c1 = createConst(op.getLoc(), i32Ty, 1, rewriter);
     // Reinterpret the input f32 value as bits.
-    Value bitcast = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value bitcast = arith::BitcastOp::create(b, i32Ty, operand);
     // Read bit 16 as a value in {0,1}.
     Value bit16 =
-        b.create<arith::AndIOp>(b.create<arith::ShRUIOp>(bitcast, c16), c1);
+        arith::AndIOp::create(b, arith::ShRUIOp::create(b, bitcast, c16), c1);
     // Determine the rounding bias to add as either 0x7fff or 0x8000 depending
     // on bit 16, implementing the tie-breaking "to nearest even".
-    Value roundingBias = b.create<arith::AddIOp>(bit16, c7FFF);
+    Value roundingBias = arith::AddIOp::create(b, bit16, c7FFF);
     // Add the rounding bias. Generally we want this to be added to the
     // mantissa, but nothing prevents this to from carrying into the exponent
     // bits, which would feel like a bug, but this is the magic trick here:
     // when that happens, the mantissa gets reset to zero and the exponent
     // gets incremented by the carry... which is actually exactly what we
     // want.
-    Value biased = b.create<arith::AddIOp>(bitcast, roundingBias);
+    Value biased = arith::AddIOp::create(b, bitcast, roundingBias);
     // Now that the rounding-bias has been added, truncating the low bits
     // yields the correctly rounded result.
-    Value biasedAndShifted = b.create<arith::ShRUIOp>(biased, c16);
+    Value biasedAndShifted = arith::ShRUIOp::create(b, biased, c16);
     Value normalCaseResultI16 =
-        b.create<arith::TruncIOp>(i16Ty, biasedAndShifted);
+        arith::TruncIOp::create(b, i16Ty, biasedAndShifted);
     // Select either the above-computed result, or a quiet NaN constant
     // if the input was NaN.
     Value select =
-        b.create<arith::SelectOp>(isNan, c7FC0I16, normalCaseResultI16);
-    Value result = b.create<arith::BitcastOp>(resultTy, select);
+        arith::SelectOp::create(b, isNan, c7FC0I16, normalCaseResultI16);
+    Value result = arith::BitcastOp::create(b, resultTy, select);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -384,7 +384,7 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
     Type i4Ty = cloneToShapedType(operandTy, b.getI4Type());
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
-    Value i4Bits = b.create<arith::BitcastOp>(i4Ty, operand);
+    Value i4Bits = arith::BitcastOp::create(b, i4Ty, operand);
 
     Value c0x0 = createConst(loc, i4Ty, 0x0, rewriter);
     Value c0x1 = createConst(loc, i4Ty, 0x1, rewriter);
@@ -393,38 +393,38 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
 
     // Set last Exponent bit and Mantissa.
     Value c0x00000014 = createConst(loc, i32Ty, 0x14, rewriter);
-    Value bits1To24 = b.create<arith::ShLIOp>(i4Bits, c0x2);
+    Value bits1To24 = arith::ShLIOp::create(b, i4Bits, c0x2);
     Value isHalf =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x1);
-    bits1To24 = b.create<arith::SelectOp>(isHalf, c0x0, bits1To24);
-    bits1To24 = b.create<arith::ExtUIOp>(i32Ty, bits1To24);
-    bits1To24 = b.create<arith::ShLIOp>(bits1To24, c0x00000014);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, i4Bits, c0x1);
+    bits1To24 = arith::SelectOp::create(b, isHalf, c0x0, bits1To24);
+    bits1To24 = arith::ExtUIOp::create(b, i32Ty, bits1To24);
+    bits1To24 = arith::ShLIOp::create(b, bits1To24, c0x00000014);
 
     // Set first 7 bits of Exponent.
     Value zeroExpBits = createConst(loc, i32Ty, 0x00000000, rewriter);
     Value highExpBits = createConst(loc, i32Ty, 0x40000000, rewriter);
     Value lowExpBits = createConst(loc, i32Ty, 0x3f000000, rewriter);
     Value useLargerExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x4);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x4);
     Value bits25To31 =
-        b.create<arith::SelectOp>(useLargerExp, highExpBits, lowExpBits);
+        arith::SelectOp::create(b, useLargerExp, highExpBits, lowExpBits);
     Value zeroExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x0);
-    bits25To31 = b.create<arith::SelectOp>(zeroExp, zeroExpBits, bits25To31);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, i4Bits, c0x0);
+    bits25To31 = arith::SelectOp::create(b, zeroExp, zeroExpBits, bits25To31);
 
     // Set sign.
     Value c0x80000000 = createConst(loc, i32Ty, 0x80000000, rewriter);
     Value c0x8 = createConst(loc, i4Ty, 0x8, rewriter);
     Value negative =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x8);
-    Value bit32 = b.create<arith::SelectOp>(negative, c0x80000000, zeroExpBits);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x8);
+    Value bit32 = arith::SelectOp::create(b, negative, c0x80000000, zeroExpBits);
 
     // Add segments together.
-    Value bits1To31 = b.create<arith::AddIOp>(bits1To24, bits25To31);
-    Value bits1To32 = b.create<arith::AddIOp>(bits1To31, bit32);
-    Value result = b.create<arith::BitcastOp>(f32Ty, bits1To32);
+    Value bits1To31 = arith::AddIOp::create(b, bits1To24, bits25To31);
+    Value bits1To32 = arith::AddIOp::create(b, bits1To31, bit32);
+    Value result = arith::BitcastOp::create(b, f32Ty, bits1To32);
     if (!isa<Float32Type>(resultETy))
-      result = b.create<arith::TruncFOp>(resultTy, result);
+      result = arith::TruncFOp::create(b, resultTy, result);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -450,25 +450,25 @@ struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
-    Value bitcast = b.create<arith::BitcastOp>(i8Ty, operand);
+    Value bitcast = arith::BitcastOp::create(b, i8Ty, operand);
     // create constants for NaNs
     Value cF8NaN = createConst(op.getLoc(), i8Ty, 0xff, rewriter);
     Value cF32NaN = createConst(op.getLoc(), i32Ty, 0xffffffff, rewriter);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
 
-    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
-    Value f32Bits = b.create<arith::ShLIOp>(exti, cF32MantissaWidth);
+    Value exti = arith::ExtUIOp::create(b, i32Ty, bitcast);
+    Value f32Bits = arith::ShLIOp::create(b, exti, cF32MantissaWidth);
 
     Value isNan =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bitcast, cF8NaN);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, bitcast, cF8NaN);
     // select for NaNs
-    f32Bits = b.create<arith::SelectOp>(isNan, cF32NaN, f32Bits);
-    Value result = b.create<arith::BitcastOp>(f32Ty, f32Bits);
+    f32Bits = arith::SelectOp::create(b, isNan, cF32NaN, f32Bits);
+    Value result = arith::BitcastOp::create(b, f32Ty, f32Bits);
     if (resultETy.getIntOrFloatBitWidth() < 32) {
-      result = b.create<arith::TruncFOp>(resultTy, result, nullptr,
+      result = arith::TruncFOp::create(b, resultTy, result, nullptr,
                                          op.getFastmathAttr());
     } else if (resultETy.getIntOrFloatBitWidth() > 32) {
-      result = b.create<arith::ExtFOp>(resultTy, result, op.getFastmathAttr());
+      result = arith::ExtFOp::create(b, resultTy, result, op.getFastmathAttr());
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -521,7 +521,7 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (!isa<Float32Type>(operandETy))
-      operand = b.create<arith::ExtFOp>(f32Ty, operand);
+      operand = arith::ExtFOp::create(b, f32Ty, operand);
     if (!isa<Float4E2M1FNType>(resultETy))
       return rewriter.notifyMatchFailure(op, "not a trunc of F4E2M1FN");
 
@@ -535,65 +535,65 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     // Step 0: Clamp to bounds.
     Value cHigherBound = createFloatConst(loc, f32Ty, APFloat(6.0f), rewriter);
     Value cLowerBound = createFloatConst(loc, f32Ty, APFloat(-6.0f), rewriter);
-    Value operandClamped = b.create<arith::MinNumFOp>(cHigherBound, operand);
-    operandClamped = b.create<arith::MaxNumFOp>(cLowerBound, operandClamped);
-    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operandClamped);
+    Value operandClamped = arith::MinNumFOp::create(b, cHigherBound, operand);
+    operandClamped = arith::MaxNumFOp::create(b, cLowerBound, operandClamped);
+    Value f32Bits = arith::BitcastOp::create(b, i32Ty, operandClamped);
 
     // Step 1: Set sign bit.
     Value cF32ExpManWidth = createConst(loc, i32Ty, 31, rewriter); // 23
-    Value f32Sign = b.create<arith::ShRUIOp>(f32Bits, cF32ExpManWidth);
-    Value f4Sign = b.create<arith::TruncIOp>(i4Ty, f32Sign);
-    Value f4Bits = b.create<arith::ShLIOp>(f4Sign, c0x3);
+    Value f32Sign = arith::ShRUIOp::create(b, f32Bits, cF32ExpManWidth);
+    Value f4Sign = arith::TruncIOp::create(b, i4Ty, f32Sign);
+    Value f4Bits = arith::ShLIOp::create(b, f4Sign, c0x3);
 
     // Step 2: Convert exponent by adjusting bias.
     Value biasAdjustment = createConst(loc, i32Ty, 0x7e, rewriter);
     Value cF4MantissaWidth = c0x1;                                   // 1
     Value cF32MantissaWidth = createConst(loc, i32Ty, 23, rewriter); // 23
-    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
+    Value f32SignExp = arith::ShRUIOp::create(b, f32Bits, cF32MantissaWidth);
     Value biasAdjustedSignExp =
-        b.create<arith::SubIOp>(f32SignExp, biasAdjustment);
-    Value f4Exp = b.create<arith::TruncIOp>(i4Ty, biasAdjustedSignExp);
-    f4Exp = b.create<arith::ShLIOp>(f4Exp, cF4MantissaWidth);
-    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Exp);
+        arith::SubIOp::create(b, f32SignExp, biasAdjustment);
+    Value f4Exp = arith::TruncIOp::create(b, i4Ty, biasAdjustedSignExp);
+    f4Exp = arith::ShLIOp::create(b, f4Exp, cF4MantissaWidth);
+    f4Bits = arith::AddIOp::create(b, f4Bits, f4Exp);
 
     // Step 3: Set mantissa to first bit.
     Value cF32FirstBitMask = createConst(loc, i32Ty, 0x400000, rewriter);
-    Value man1Bit = b.create<arith::AndIOp>(f32Bits, cF32FirstBitMask);
-    man1Bit = b.create<arith::ShRUIOp>(man1Bit, c0x00000016);
-    Value f4Man = b.create<arith::TruncIOp>(i4Ty, man1Bit);
-    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Man);
+    Value man1Bit = arith::AndIOp::create(b, f32Bits, cF32FirstBitMask);
+    man1Bit = arith::ShRUIOp::create(b, man1Bit, c0x00000016);
+    Value f4Man = arith::TruncIOp::create(b, i4Ty, man1Bit);
+    f4Bits = arith::AddIOp::create(b, f4Bits, f4Man);
 
     // Step 4: Special consideration for conversion to 0.5.
     Value cF32MantissaMask = createConst(loc, i32Ty, 0x7fffff, rewriter);
-    Value f8Exp = b.create<arith::TruncIOp>(i8Ty, biasAdjustedSignExp);
+    Value f8Exp = arith::TruncIOp::create(b, i8Ty, biasAdjustedSignExp);
     Value isSubnormal =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::sle, f8Exp, c0x00);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::sle, f8Exp, c0x00);
     Value isNegOneExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0xff);
-    Value man23Bits = b.create<arith::AndIOp>(f32Bits, cF32MantissaMask);
-    Value isNonZeroMan = b.create<arith::CmpIOp>(arith::CmpIPredicate::ugt,
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0xff);
+    Value man23Bits = arith::AndIOp::create(b, f32Bits, cF32MantissaMask);
+    Value isNonZeroMan = arith::CmpIOp::create(b, arith::CmpIPredicate::ugt,
                                                  man23Bits, zeroExpBits);
-    Value roundToHalf = b.create<arith::AndIOp>(isNegOneExp, isNonZeroMan);
+    Value roundToHalf = arith::AndIOp::create(b, isNegOneExp, isNonZeroMan);
     Value isZeroExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0x00);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0x00);
     Value subnormalF4Bits = createConst(loc, i4Ty, 0xf, rewriter);
     Value halfF4Bits = createConst(loc, i4Ty, 0x0, rewriter);
     Value subResult =
-        b.create<arith::SelectOp>(isSubnormal, subnormalF4Bits, f4Bits);
-    subResult = b.create<arith::SelectOp>(roundToHalf, halfF4Bits, subResult);
-    f4Bits = b.create<arith::SelectOp>(isZeroExp, f4Bits, subResult);
+        arith::SelectOp::create(b, isSubnormal, subnormalF4Bits, f4Bits);
+    subResult = arith::SelectOp::create(b, roundToHalf, halfF4Bits, subResult);
+    f4Bits = arith::SelectOp::create(b, isZeroExp, f4Bits, subResult);
 
     // Step 5: Round up if necessary.
     Value cF32Last22BitMask = createConst(loc, i32Ty, 0x3fffff, rewriter);
     Value cRound = createConst(loc, i32Ty, 0x200000, rewriter); // 010 0000...
-    Value man22Bits = b.create<arith::AndIOp>(f32Bits, cF32Last22BitMask);
+    Value man22Bits = arith::AndIOp::create(b, f32Bits, cF32Last22BitMask);
     Value shouldRound =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, man22Bits, cRound);
-    shouldRound = b.create<arith::OrIOp>(shouldRound, isSubnormal);
-    Value roundedF4Bits = b.create<arith::AddIOp>(f4Bits, c0x1);
-    f4Bits = b.create<arith::SelectOp>(shouldRound, roundedF4Bits, f4Bits);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, man22Bits, cRound);
+    shouldRound = arith::OrIOp::create(b, shouldRound, isSubnormal);
+    Value roundedF4Bits = arith::AddIOp::create(b, f4Bits, c0x1);
+    f4Bits = arith::SelectOp::create(b, shouldRound, roundedF4Bits, f4Bits);
 
-    Value result = b.create<arith::BitcastOp>(resultTy, f4Bits);
+    Value result = arith::BitcastOp::create(b, resultTy, f4Bits);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -628,16 +628,16 @@ struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (operandETy.getIntOrFloatBitWidth() < 32) {
-      operand = b.create<arith::ExtFOp>(f32Ty, operand, op.getFastmathAttr());
+      operand = arith::ExtFOp::create(b, f32Ty, operand, op.getFastmathAttr());
     } else if (operandETy.getIntOrFloatBitWidth() > 32) {
-      operand = b.create<arith::TruncFOp>(
+      operand = arith::TruncFOp::create(b,
           f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
     }
-    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value f32Bits = arith::BitcastOp::create(b, i32Ty, operand);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
-    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
-    Value exp8Bits = b.create<arith::TruncIOp>(i8Ty, f32SignExp);
-    Value result = b.create<arith::BitcastOp>(resultTy, exp8Bits);
+    Value f32SignExp = arith::ShRUIOp::create(b, f32Bits, cF32MantissaWidth);
+    Value exp8Bits = arith::TruncIOp::create(b, i8Ty, f32SignExp);
+    Value result = arith::BitcastOp::create(b, resultTy, exp8Bits);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -656,7 +656,7 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
     if (scaleETy.getIntOrFloatBitWidth() >= 16) {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
-      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+      scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
                                                op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
@@ -668,11 +668,11 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
     // extf on scale will essentially create floating point number
     // of type resulTy that is 2^scale and will also propagate NaNs
     Value scaleExt =
-        b.create<arith::ExtFOp>(resultTy, scaleOperand, op.getFastmathAttr());
+        arith::ExtFOp::create(b, resultTy, scaleOperand, op.getFastmathAttr());
     Value inputExt =
-        b.create<arith::ExtFOp>(resultTy, inputOperand, op.getFastmathAttr());
+        arith::ExtFOp::create(b, resultTy, inputOperand, op.getFastmathAttr());
     Value result =
-        b.create<arith::MulFOp>(inputExt, scaleExt, op.getFastmathAttr());
+        arith::MulFOp::create(b, inputExt, scaleExt, op.getFastmathAttr());
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -697,7 +697,7 @@ struct ScalingTruncFOpConverter
     if (scaleETy.getIntOrFloatBitWidth() >= 16) {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
-      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+      scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
                                                op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
@@ -710,10 +710,10 @@ struct ScalingTruncFOpConverter
     // this will create a floating point number of type
     // inputTy that is 2^scale and will also propagate NaNs
     scaleOperand =
-        b.create<arith::ExtFOp>(inputTy, scaleOperand, op.getFastmathAttr());
-    Value result = b.create<arith::DivFOp>(inputOperand, scaleOperand,
+        arith::ExtFOp::create(b, inputTy, scaleOperand, op.getFastmathAttr());
+    Value result = arith::DivFOp::create(b, inputOperand, scaleOperand,
                                            op.getFastmathAttr());
-    Value resultCast = b.create<arith::TruncFOp>(
+    Value resultCast = arith::TruncFOp::create(b,
         resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
     rewriter.replaceOp(op, resultCast);
     return success();
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index f2f93883eb2b7..777ff0ecaa314 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -305,18 +305,18 @@ static Value doCast(OpBuilder &builder, Location loc, Value src, Type dstType,
 
   if (isa<IndexType>(srcElemType) || isa<IndexType>(dstElemType)) {
     if (castKind == CastKind::Signed)
-      return builder.create<arith::IndexCastOp>(loc, dstType, src);
-    return builder.create<arith::IndexCastUIOp>(loc, dstType, src);
+      return arith::IndexCastOp::create(builder, loc, dstType, src);
+    return arith::IndexCastUIOp::create(builder, loc, dstType, src);
   }
 
   auto srcInt = cast<IntegerType>(srcElemType);
   auto dstInt = cast<IntegerType>(dstElemType);
   if (dstInt.getWidth() < srcInt.getWidth())
-    return builder.create<arith::TruncIOp>(loc, dstType, src);
+    return arith::TruncIOp::create(builder, loc, dstType, src);
 
   if (castKind == CastKind::Signed)
-    return builder.create<arith::ExtSIOp>(loc, dstType, src);
-  return builder.create<arith::ExtUIOp>(loc, dstType, src);
+    return arith::ExtSIOp::create(builder, loc, dstType, src);
+  return arith::ExtUIOp::create(builder, loc, dstType, src);
 }
 
 struct NarrowElementwise final : OpTraitRewritePattern<OpTrait::Elementwise> {
diff --git a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
index 5fb7953f93700..904aab33345f0 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
@@ -23,7 +23,7 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
   std::function<Value(AffineExpr)> buildExpr = [&](AffineExpr e) -> Value {
     switch (e.getKind()) {
     case AffineExprKind::Constant:
-      return b.create<ConstantIndexOp>(loc,
+      return ConstantIndexOp::create(b, loc,
                                        cast<AffineConstantExpr>(e).getValue());
     case AffineExprKind::DimId:
       return operands[cast<AffineDimExpr>(e).getPosition()];
@@ -32,27 +32,27 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
                       map.getNumDims()];
     case AffineExprKind::Add: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<AddIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return AddIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                               buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mul: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<MulIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return MulIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                               buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::FloorDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<DivSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return DivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::CeilDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<CeilDivSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return CeilDivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                    buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mod: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<RemSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return RemSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                buildExpr(binaryExpr.getRHS()));
     }
     }
@@ -89,10 +89,10 @@ FailureOr<OpFoldResult> mlir::arith::reifyValueBound(
            "expected dynamic dim");
     if (isa<RankedTensorType>(value.getType())) {
       // A tensor dimension is used: generate a tensor.dim.
-      operands.push_back(b.create<tensor::DimOp>(loc, value, *dim));
+      operands.push_back(tensor::DimOp::create(b, loc, value, *dim));
     } else if (isa<MemRefType>(value.getType())) {
       // A memref dimension is used: generate a memref.dim.
-      operands.push_back(b.create<memref::DimOp>(loc, value, *dim));
+      operands.push_back(memref::DimOp::create(b, loc, value, *dim));
     } else {
       llvm_unreachable("cannot generate DimOp for unsupported shaped type");
     }
diff --git a/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
index 62d137a4cfb0e..10c5b50cb771a 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
@@ -84,7 +84,7 @@ struct ConstantShardingInterface
           cOp.getType(), getMesh(op, sharding.getMeshAttr(), symbolTable),
           sharding));
       auto newValue = value.resizeSplat(newType);
-      auto newOp = builder.create<ConstantOp>(op->getLoc(), newType, newValue);
+      auto newOp = ConstantOp::create(builder, op->getLoc(), newType, newValue);
       spmdizationMap.map(op->getResult(0), newOp.getResult());
       spmdizationMap.map(op, newOp.getOperation());
     } else {
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index 3cd8684878a11..6e9cc14c6ada5 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -68,7 +68,7 @@ mlir::inferExpandShapeOutputShape(OpBuilder &b, Location loc,
     // dynamism.
     Value indexGroupSize = cast<Value>(inputShape[inputIndex]);
     Value indexGroupStaticSizesProduct =
-        b.create<arith::ConstantIndexOp>(loc, indexGroupStaticSizesProductInt);
+        arith::ConstantIndexOp::create(b, loc, indexGroupStaticSizesProductInt);
     Value dynamicDimSize = b.createOrFold<arith::DivSIOp>(
         loc, indexGroupSize, indexGroupStaticSizesProduct);
     outputShapeValues.push_back(dynamicDimSize);
@@ -105,7 +105,7 @@ Value mlir::getValueOrCreateConstantIntOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return b.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(b,
       loc, b.getIntegerAttr(attr.getType(), attr.getValue().getSExtValue()));
 }
 
@@ -114,7 +114,7 @@ Value mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return b.create<arith::ConstantIndexOp>(loc, attr.getValue().getSExtValue());
+  return arith::ConstantIndexOp::create(b, loc, attr.getValue().getSExtValue());
 }
 
 Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
@@ -125,7 +125,7 @@ Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
   bool targetIsIndex = targetType.isIndex();
   bool valueIsIndex = value.getType().isIndex();
   if (targetIsIndex ^ valueIsIndex)
-    return b.create<arith::IndexCastOp>(loc, targetType, value);
+    return arith::IndexCastOp::create(b, loc, targetType, value);
 
   auto targetIntegerType = dyn_cast<IntegerType>(targetType);
   auto valueIntegerType = dyn_cast<IntegerType>(value.getType());
@@ -134,8 +134,8 @@ Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
   assert(targetIntegerType.getSignedness() == valueIntegerType.getSignedness());
 
   if (targetIntegerType.getWidth() > valueIntegerType.getWidth())
-    return b.create<arith::ExtSIOp>(loc, targetIntegerType, value);
-  return b.create<arith::TruncIOp>(loc, targetIntegerType, value);
+    return arith::ExtSIOp::create(b, loc, targetIntegerType, value);
+  return arith::TruncIOp::create(b, loc, targetIntegerType, value);
 }
 
 static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
@@ -143,21 +143,21 @@ static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
   // If operand is floating point, cast directly to the int type.
   if (isa<FloatType>(operand.getType())) {
     if (isUnsigned)
-      return b.create<arith::FPToUIOp>(toType, operand);
-    return b.create<arith::FPToSIOp>(toType, operand);
+      return arith::FPToUIOp::create(b, toType, operand);
+    return arith::FPToSIOp::create(b, toType, operand);
   }
   // Cast index operands directly to the int type.
   if (operand.getType().isIndex())
-    return b.create<arith::IndexCastOp>(toType, operand);
+    return arith::IndexCastOp::create(b, toType, operand);
   if (auto fromIntType = dyn_cast<IntegerType>(operand.getType())) {
     // Either extend or truncate.
     if (toType.getWidth() > fromIntType.getWidth()) {
       if (isUnsigned)
-        return b.create<arith::ExtUIOp>(toType, operand);
-      return b.create<arith::ExtSIOp>(toType, operand);
+        return arith::ExtUIOp::create(b, toType, operand);
+      return arith::ExtSIOp::create(b, toType, operand);
     }
     if (toType.getWidth() < fromIntType.getWidth())
-      return b.create<arith::TruncIOp>(toType, operand);
+      return arith::TruncIOp::create(b, toType, operand);
     return operand;
   }
 
@@ -170,14 +170,14 @@ static Value convertScalarToFpDtype(ImplicitLocOpBuilder &b, Value operand,
   // Note that it is unclear how to cast from BF16<->FP16.
   if (isa<IntegerType>(operand.getType())) {
     if (isUnsigned)
-      return b.create<arith::UIToFPOp>(toType, operand);
-    return b.create<arith::SIToFPOp>(toType, operand);
+      return arith::UIToFPOp::create(b, toType, operand);
+    return arith::SIToFPOp::create(b, toType, operand);
   }
   if (auto fromFpTy = dyn_cast<FloatType>(operand.getType())) {
     if (toType.getWidth() > fromFpTy.getWidth())
-      return b.create<arith::ExtFOp>(toType, operand);
+      return arith::ExtFOp::create(b, toType, operand);
     if (toType.getWidth() < fromFpTy.getWidth())
-      return b.create<arith::TruncFOp>(toType, operand);
+      return arith::TruncFOp::create(b, toType, operand);
     return operand;
   }
 
@@ -190,18 +190,18 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
   if (auto fromComplexType = dyn_cast<ComplexType>(operand.getType())) {
     if (isa<FloatType>(targetType.getElementType()) &&
         isa<FloatType>(fromComplexType.getElementType())) {
-      Value real = b.create<complex::ReOp>(operand);
-      Value imag = b.create<complex::ImOp>(operand);
+      Value real = complex::ReOp::create(b, operand);
+      Value imag = complex::ImOp::create(b, operand);
       Type targetETy = targetType.getElementType();
       if (targetType.getElementType().getIntOrFloatBitWidth() <
           fromComplexType.getElementType().getIntOrFloatBitWidth()) {
-        real = b.create<arith::TruncFOp>(targetETy, real);
-        imag = b.create<arith::TruncFOp>(targetETy, imag);
+        real = arith::TruncFOp::create(b, targetETy, real);
+        imag = arith::TruncFOp::create(b, targetETy, imag);
       } else {
-        real = b.create<arith::ExtFOp>(targetETy, real);
-        imag = b.create<arith::ExtFOp>(targetETy, imag);
+        real = arith::ExtFOp::create(b, targetETy, real);
+        imag = arith::ExtFOp::create(b, targetETy, imag);
       }
-      return b.create<complex::CreateOp>(targetType, real, imag);
+      return complex::CreateOp::create(b, targetType, real, imag);
     }
   }
 
@@ -210,27 +210,27 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
     auto toBitwidth = toFpTy.getIntOrFloatBitWidth();
     Value from = operand;
     if (from.getType().getIntOrFloatBitWidth() < toBitwidth) {
-      from = b.create<arith::ExtFOp>(toFpTy, from);
+      from = arith::ExtFOp::create(b, toFpTy, from);
     }
     if (from.getType().getIntOrFloatBitWidth() > toBitwidth) {
-      from = b.create<arith::TruncFOp>(toFpTy, from);
+      from = arith::TruncFOp::create(b, toFpTy, from);
     }
-    Value zero = b.create<mlir::arith::ConstantFloatOp>(
+    Value zero = mlir::arith::ConstantFloatOp::create(b,
         toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
-    return b.create<complex::CreateOp>(targetType, from, zero);
+    return complex::CreateOp::create(b, targetType, from, zero);
   }
 
   if (isa<IntegerType>(operand.getType())) {
     FloatType toFpTy = cast<FloatType>(targetType.getElementType());
     Value from = operand;
     if (isUnsigned) {
-      from = b.create<arith::UIToFPOp>(toFpTy, from);
+      from = arith::UIToFPOp::create(b, toFpTy, from);
     } else {
-      from = b.create<arith::SIToFPOp>(toFpTy, from);
+      from = arith::SIToFPOp::create(b, toFpTy, from);
     }
-    Value zero = b.create<mlir::arith::ConstantFloatOp>(
+    Value zero = mlir::arith::ConstantFloatOp::create(b,
         toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
-    return b.create<complex::CreateOp>(targetType, from, zero);
+    return complex::CreateOp::create(b, targetType, from, zero);
   }
 
   return {};
@@ -278,7 +278,7 @@ Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
     attr = SplatElementsAttr::get(vecTy, value);
   }
 
-  return builder.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(builder, loc, attr);
 }
 
 Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
@@ -310,35 +310,35 @@ Type mlir::getType(OpFoldResult ofr) {
 }
 
 Value ArithBuilder::_and(Value lhs, Value rhs) {
-  return b.create<arith::AndIOp>(loc, lhs, rhs);
+  return arith::AndIOp::create(b, loc, lhs, rhs);
 }
 Value ArithBuilder::add(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::AddFOp>(loc, lhs, rhs);
-  return b.create<arith::AddIOp>(loc, lhs, rhs, ovf);
+    return arith::AddFOp::create(b, loc, lhs, rhs);
+  return arith::AddIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::sub(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::SubFOp>(loc, lhs, rhs);
-  return b.create<arith::SubIOp>(loc, lhs, rhs, ovf);
+    return arith::SubFOp::create(b, loc, lhs, rhs);
+  return arith::SubIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::mul(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::MulFOp>(loc, lhs, rhs);
-  return b.create<arith::MulIOp>(loc, lhs, rhs, ovf);
+    return arith::MulFOp::create(b, loc, lhs, rhs);
+  return arith::MulIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::sgt(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGT, lhs, rhs);
-  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sgt, lhs, rhs);
+    return arith::CmpFOp::create(b, loc, arith::CmpFPredicate::OGT, lhs, rhs);
+  return arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sgt, lhs, rhs);
 }
 Value ArithBuilder::slt(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OLT, lhs, rhs);
-  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, lhs, rhs);
+    return arith::CmpFOp::create(b, loc, arith::CmpFPredicate::OLT, lhs, rhs);
+  return arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, lhs, rhs);
 }
 Value ArithBuilder::select(Value cmp, Value lhs, Value rhs) {
-  return b.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+  return arith::SelectOp::create(b, loc, cmp, lhs, rhs);
 }
 
 namespace mlir::arith {
@@ -349,7 +349,7 @@ Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values) {
 
 Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
                     Type resultType) {
-  Value one = builder.create<ConstantOp>(loc, resultType,
+  Value one = ConstantOp::create(builder, loc, resultType,
                                          builder.getOneAttr(resultType));
   ArithBuilder arithBuilder(builder, loc);
   return std::accumulate(
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
index 7180884c77e98..0d5640b202fb3 100644
--- a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
@@ -231,7 +231,7 @@ class LowerContractionToNeonI8MMPattern
 
     // Initial accumulator for the final result. This is the un-tiled result if
     // tiling is done.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, op.getResultType(), rewriter.getZeroAttr(op.getResultType()));
 
     SmallVector<int64_t> unrolledSize = *op.getShapeForUnroll();
@@ -283,7 +283,7 @@ class LowerContractionToNeonI8MMPattern
       if (isVecmat) {
         auto expandForSMMLA = [&](Value tiledOperand,
                                   VectorType expandedTypeType) {
-          auto emptyOperand = rewriter.create<arith::ConstantOp>(
+          auto emptyOperand = arith::ConstantOp::create(rewriter,
               loc, expandedTypeType, rewriter.getZeroAttr(expandedTypeType));
           SmallVector<int64_t> offsets(
               cast<ShapedType>(emptyOperand.getType()).getRank(), 0);
@@ -300,7 +300,7 @@ class LowerContractionToNeonI8MMPattern
       // using the instruction for unsigned by signed multiplication with
       // reversed operands.
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledAcc = rewriter.create<vector::TransposeOp>(
+        tiledAcc = vector::TransposeOp::create(rewriter,
             loc, tiledAcc, ArrayRef<int64_t>({1, 0}));
 
       // Collapse tiled operands to 1D vectors required by smmla intrinsic
@@ -333,7 +333,7 @@ class LowerContractionToNeonI8MMPattern
       // Because of the reversed operands the result is obtained transposed.
       // Transpose it back,
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledRes = rewriter.create<vector::TransposeOp>(
+        tiledRes = vector::TransposeOp::create(rewriter,
             loc, tiledRes, ArrayRef<int64_t>({1, 0}));
 
       // With vecmat, only one row of tiled ACC can be inserted into the final
diff --git a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
index 1f7305a5f8141..a4aebb585c3de 100644
--- a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
@@ -76,21 +76,21 @@ scf::ForOp createLoopOverTileSlices(
     PatternRewriter &rewriter, Location loc, Value initTile,
     std::function<Value(OpBuilder &, Location, Value, Value)> makeLoopBody) {
   OpBuilder::InsertionGuard g(rewriter);
-  auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-  auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+  auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
       loc, llvm::cast<VectorType>(initTile.getType()).getDimSize(0));
   auto vscale =
-      rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-  auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+      vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+  auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto numTileSlices =
-      rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
-  auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, numTileSlices, step,
+      arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices, step,
                                            ValueRange{initTile});
   rewriter.setInsertionPointToStart(forOp.getBody());
   Value nextTile =
       makeLoopBody(rewriter, loc, /*tileSliceIndex=*/forOp.getInductionVar(),
                    /*currentTile=*/forOp.getRegionIterArg(0));
-  rewriter.create<scf::YieldOp>(loc, nextTile);
+  scf::YieldOp::create(rewriter, loc, nextTile);
   return forOp;
 }
 
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
index 23f2c2bf65e47..4d03bbdabdf09 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
@@ -136,7 +136,7 @@ class OuterProductFusion2Way
 
     auto loc = op.getLoc();
     auto packInputs = [&](Value lhs, Value rhs) {
-      return rewriter.create<vector::InterleaveOp>(loc, lhs, rhs);
+      return vector::InterleaveOp::create(rewriter, loc, lhs, rhs);
     };
 
     auto lhs = packInputs(op1.getLhs().getDefiningOp()->getOperand(0),
@@ -284,7 +284,7 @@ class OuterProductFusion4Way
 
     auto loc = op.getLoc();
     auto packInputs = [&](Value lhs, Value rhs) {
-      return rewriter.create<vector::InterleaveOp>(loc, lhs, rhs);
+      return vector::InterleaveOp::create(rewriter, loc, lhs, rhs);
     };
 
     auto lhs0 = packInputs(op1.getLhs().getDefiningOp()->getOperand(0),
@@ -456,7 +456,7 @@ struct SwapVectorExtractOfArithExtend
     Value extendSource = extendOp->getOperand(0);
 
     // Create new extract from source of extend.
-    Value newExtract = rewriter.create<vector::ExtractOp>(
+    Value newExtract = vector::ExtractOp::create(rewriter,
         loc, extendSource, extractOp.getMixedPosition());
 
     // Extend new extract to original result type.
@@ -503,7 +503,7 @@ struct SwapVectorScalableExtractOfArithExtend
     // Create new extract from source of extend.
     VectorType extractResultVectorType =
         resultType.clone(extendSourceVectorType.getElementType());
-    Value newExtract = rewriter.create<vector::ScalableExtractOp>(
+    Value newExtract = vector::ScalableExtractOp::create(rewriter,
         loc, extractResultVectorType, extendSource, extractOp.getPos());
 
     // Extend new extract to original result type.
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index e6c9adba62f34..2bea9de83143a 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -212,7 +212,7 @@ void splitCondBranches(IRRewriter &rewriter, FunctionOpInterface function) {
 
   auto insertJump = [&](Location loc, Block *source, Block *dest, auto args) {
     rewriter.setInsertionPointToEnd(source);
-    rewriter.create<cf::BranchOp>(loc, dest, args);
+    cf::BranchOp::create(rewriter, loc, dest, args);
   };
 
   for (auto condBranch : worklist) {
@@ -255,7 +255,7 @@ void insertCopiesAtBranches(IRRewriter &rewriter,
     for (OpOperand &operand : terminator->getOpOperands()) {
       if (isValidSMETileVectorType(operand.get().getType())) {
         auto copy =
-            rewriter.create<CopyTileOp>(terminator->getLoc(), operand.get());
+            CopyTileOp::create(rewriter, terminator->getLoc(), operand.get());
         rewriter.modifyOpInPlace(terminator, [&] { operand.assign(copy); });
       }
     }
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 1e8e1265affa0..3362d0ce13533 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -82,13 +82,13 @@ SmallVector<Value, 2> addConstantScalableOffset(OpBuilder &builder,
                                                 Location loc,
                                                 ValueRange indices,
                                                 ArrayRef<int> scalableOffsets) {
-  auto vscale = builder.create<vector::VectorScaleOp>(loc);
+  auto vscale = vector::VectorScaleOp::create(builder, loc);
   return llvm::map_to_vector(
       llvm::zip_equal(indices, scalableOffsets), [&](auto pair) -> Value {
         auto [index, base] = pair;
-        auto offset = builder.create<arith::MulIOp>(
-            loc, builder.create<arith::ConstantIndexOp>(loc, base), vscale);
-        return builder.create<arith::AddIOp>(loc, index, offset);
+        auto offset = arith::MulIOp::create(builder,
+            loc, arith::ConstantIndexOp::create(builder, loc, base), vscale);
+        return arith::AddIOp::create(builder, loc, index, offset);
       });
 }
 
@@ -132,7 +132,7 @@ Value extractSMEMask(OpBuilder &builder, Location loc, Value mask,
   // from the mask operands to get the parameters for this sub-tile.
   auto smeTileMaskDims = addConstantScalableOffset(
       builder, loc, createMask.getOperands(), {-smeTile.row, -smeTile.col});
-  auto smeTileCreateMask = builder.create<vector::CreateMaskOp>(
+  auto smeTileCreateMask = vector::CreateMaskOp::create(builder,
       loc, smeTile.type.clone(builder.getI1Type()), smeTileMaskDims);
   return smeTileCreateMask.getResult();
 }
@@ -190,7 +190,7 @@ struct LegalizeArithConstantOpsByDecomposition
 
     auto smeTileType = getSMETileTypeForElement(vectorType.getElementType());
     auto tileCount = getNumberOfSMETilesForVectorType(vectorType);
-    auto tileSplat = rewriter.create<arith::ConstantOp>(
+    auto tileSplat = arith::ConstantOp::create(rewriter,
         constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
     SmallVector<Value> repl(tileCount, tileSplat);
     rewriter.replaceOpWithMultiple(constantOp, {repl});
@@ -237,11 +237,11 @@ struct LegalizeVectorOuterProductOpsByDecomposition
              decomposeToSMETiles(rewriter, vectorType, smeTileType))) {
 
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto lhs = rewriter.create<vector::ScalableExtractOp>(
+      auto lhs = vector::ScalableExtractOp::create(rewriter,
           loc, sliceType, outerProductOp.getLhs(), smeTile.row);
-      auto rhs = rewriter.create<vector::ScalableExtractOp>(
+      auto rhs = vector::ScalableExtractOp::create(rewriter,
           loc, sliceType, outerProductOp.getRhs(), smeTile.col);
-      auto smeOuterProduct = rewriter.create<vector::OuterProductOp>(
+      auto smeOuterProduct = vector::OuterProductOp::create(rewriter,
           loc, smeTileType, lhs, rhs,
           !accSMETiles.empty() ? accSMETiles[index] : Value{},
           outerProductOp.getKind());
@@ -314,7 +314,7 @@ struct LegalizeTransferReadOpsByDecomposition
     for (SMESubTile smeTile :
          decomposeToSMETiles(rewriter, vectorType, smeTileType, transposed)) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeRead = rewriter.create<vector::TransferReadOp>(
+      auto smeRead = vector::TransferReadOp::create(rewriter,
           loc, smeTileType, readOp.getBase(),
           getSMESubTileIndices(rewriter, loc, readOp.getIndices(), smeTile),
           readOp.getPermutationMapAttr(), readOp.getPadding(), smeMask,
@@ -363,7 +363,7 @@ struct LegalizeTransferWriteOpsByDecomposition
     for (auto [index, smeTile] : llvm::enumerate(decomposeToSMETiles(
              rewriter, vectorType, smeTileType, transposed))) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeWrite = rewriter.create<vector::TransferWriteOp>(
+      auto smeWrite = vector::TransferWriteOp::create(rewriter,
           loc, inputSMETiles[index], destTensorOrMemref,
           getSMESubTileIndices(rewriter, loc, writeOp.getIndices(), smeTile),
           writeOp.getPermutationMapAttr(), smeMask, writeOp.getInBoundsAttr());
@@ -456,11 +456,11 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
         VectorType::get(minTileSlices, rewriter.getI1Type(), true);
 
     // Create loop over all tile slices.
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto upperBound = createVscaleMultiple(minTileSlices);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto storeLoop =
-        rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     rewriter.setInsertionPointToStart(storeLoop.getBody());
 
     // For each sub-tile of the multi-tile `vectorType`.
@@ -474,29 +474,29 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
 
       // The current slice of `vectorType` we are processing.
       auto sliceIndex =
-          rewriter.create<arith::AddIOp>(loc, tileRow, tileSliceIndex);
+          arith::AddIOp::create(rewriter, loc, tileRow, tileSliceIndex);
 
       // Where in the destination memref the current slice will be stored.
-      auto storeRow = rewriter.create<arith::AddIOp>(loc, sliceIndex,
+      auto storeRow = arith::AddIOp::create(rewriter, loc, sliceIndex,
                                                      writeOp.getIndices()[0]);
       auto storeCol =
-          rewriter.create<arith::AddIOp>(loc, tileCol, writeOp.getIndices()[1]);
+          arith::AddIOp::create(rewriter, loc, tileCol, writeOp.getIndices()[1]);
 
       // Extract the mask for the current slice.
       Value sliceMask = nullptr;
       if (mask) {
-        sliceMask = rewriter.create<vector::ExtractOp>(
+        sliceMask = vector::ExtractOp::create(rewriter,
             loc, mask, OpFoldResult(sliceIndex));
         if (sliceMaskType != sliceMask.getType())
-          sliceMask = rewriter.create<vector::ScalableExtractOp>(
+          sliceMask = vector::ScalableExtractOp::create(rewriter,
               loc, sliceMaskType, sliceMask, smeTile.col);
       }
 
       // Extract and store the current slice.
       Value tile = inputSMETiles[index];
       auto slice =
-          rewriter.create<vector::ExtractOp>(loc, tile, tileSliceIndex);
-      rewriter.create<vector::TransferWriteOp>(
+          vector::ExtractOp::create(rewriter, loc, tile, tileSliceIndex);
+      vector::TransferWriteOp::create(rewriter,
           loc, slice, writeOp.getBase(), ValueRange{storeRow, storeCol},
           AffineMapAttr::get(writeOp.getPermutationMap().dropResult(0)),
           sliceMask,
@@ -567,13 +567,13 @@ struct FoldExtractFromVectorOfSMELikeCreateMasks
           extractOp,
           "constant vector.create_masks dims should be folded elsewhere");
 
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto extractionIndex = getValueOrCreateConstantIndexOp(
         rewriter, loc, extractOp.getMixedPosition()[0]);
-    auto extractionInTrueRegion = rewriter.create<arith::CmpIOp>(
+    auto extractionInTrueRegion = arith::CmpIOp::create(rewriter,
         loc, rewriter.getI1Type(), arith::CmpIPredicate::slt, extractionIndex,
         frontMaskDim);
-    auto newMaskFrontDim = rewriter.create<arith::SelectOp>(
+    auto newMaskFrontDim = arith::SelectOp::create(rewriter,
         loc, extractionInTrueRegion, createMaskOp.getOperand(1), zero);
 
     rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(
@@ -660,8 +660,8 @@ struct LiftIllegalVectorTransposeToMemory
           illegalRead, "expected read to have identity permutation map");
 
     auto loc = transposeOp.getLoc();
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
     // Create a subview that matches the size of the illegal read vector type.
     auto readType = illegalRead.getVectorType();
@@ -669,14 +669,14 @@ struct LiftIllegalVectorTransposeToMemory
         llvm::zip_equal(readType.getShape(), readType.getScalableDims()),
         [&](auto dim) -> Value {
           auto [size, isScalable] = dim;
-          auto dimSize = rewriter.create<arith::ConstantIndexOp>(loc, size);
+          auto dimSize = arith::ConstantIndexOp::create(rewriter, loc, size);
           if (!isScalable)
             return dimSize;
-          auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
-          return rewriter.create<arith::MulIOp>(loc, vscale, dimSize);
+          auto vscale = vector::VectorScaleOp::create(rewriter, loc);
+          return arith::MulIOp::create(rewriter, loc, vscale, dimSize);
         });
     SmallVector<Value> strides(readType.getRank(), Value(one));
-    auto readSubview = rewriter.create<memref::SubViewOp>(
+    auto readSubview = memref::SubViewOp::create(rewriter,
         loc, illegalRead.getBase(), illegalRead.getIndices(), readSizes,
         strides);
 
@@ -686,13 +686,13 @@ struct LiftIllegalVectorTransposeToMemory
     if (mask) {
       // Note: The transpose for the mask should fold into the
       // vector.create_mask/constant_mask op, which will then become legal.
-      mask = rewriter.create<vector::TransposeOp>(loc, mask,
+      mask = vector::TransposeOp::create(rewriter, loc, mask,
                                                   transposeOp.getPermutation());
     }
     // - The source memref
     mlir::AffineMap transposeMap = AffineMap::getPermutationMap(
         transposeOp.getPermutation(), getContext());
-    auto transposedSubview = rewriter.create<memref::TransposeOp>(
+    auto transposedSubview = memref::TransposeOp::create(rewriter,
         loc, readSubview, AffineMapAttr::get(transposeMap));
     ArrayAttr inBoundsAttr = illegalRead.getInBoundsAttr();
     // - The `in_bounds` attribute
@@ -706,7 +706,7 @@ struct LiftIllegalVectorTransposeToMemory
     VectorType legalReadType = resultType.clone(readType.getElementType());
     // Note: The indices are all zero as the subview is already offset.
     SmallVector<Value> readIndices(illegalRead.getIndices().size(), zero);
-    auto legalRead = rewriter.create<vector::TransferReadOp>(
+    auto legalRead = vector::TransferReadOp::create(rewriter,
         loc, legalReadType, transposedSubview, readIndices,
         illegalRead.getPermutationMapAttr(), illegalRead.getPadding(), mask,
         inBoundsAttr);
@@ -797,12 +797,12 @@ struct LowerIllegalTransposeStoreViaZA
         AffineMap::getPermutationMap(ArrayRef<int64_t>{1, 0}, getContext()));
 
     // Note: We need to use `get_tile` as there's no vector-level `undef`.
-    Value undefTile = rewriter.create<arm_sme::GetTileOp>(loc, smeTileType);
+    Value undefTile = arm_sme::GetTileOp::create(rewriter, loc, smeTileType);
     Value destTensorOrMemref = writeOp.getBase();
     auto numSlicesPerTile =
         std::min(sourceType.getDimSize(0), smeTileType.getDimSize(0));
     auto numSlices =
-        rewriter.create<arith::ConstantIndexOp>(loc, numSlicesPerTile);
+        arith::ConstantIndexOp::create(rewriter, loc, numSlicesPerTile);
     for (auto [index, smeTile] : llvm::enumerate(
              decomposeToSMETiles(rewriter, sourceType, smeTileType))) {
       // 1. _Deliberately_ drop a scalable dimension and insert a fixed number
@@ -811,46 +811,46 @@ struct LowerIllegalTransposeStoreViaZA
       // rows of the tile after 1*vscale rows.
       Value tile = undefTile;
       for (int d = 0; d < numSlicesPerTile; ++d) {
-        Value vector = rewriter.create<vector::ExtractOp>(
+        Value vector = vector::ExtractOp::create(rewriter,
             loc, transposeOp.getVector(),
             rewriter.getIndexAttr(d + smeTile.row));
         if (vector.getType() != smeSliceType) {
-          vector = rewriter.create<vector::ScalableExtractOp>(
+          vector = vector::ScalableExtractOp::create(rewriter,
               loc, smeSliceType, vector, smeTile.col);
         }
-        tile = rewriter.create<vector::InsertOp>(loc, vector, tile, d);
+        tile = vector::InsertOp::create(rewriter, loc, vector, tile, d);
       }
 
       // 2. Transpose the tile position.
       auto transposedRow = createVscaleMultiple(smeTile.col);
       auto transposedCol =
-          rewriter.create<arith::ConstantIndexOp>(loc, smeTile.row);
+          arith::ConstantIndexOp::create(rewriter, loc, smeTile.row);
 
       // 3. Compute mask for tile store.
       Value maskRows;
       Value maskCols;
       if (auto mask = writeOp.getMask()) {
         auto createMask = mask.getDefiningOp<vector::CreateMaskOp>();
-        maskRows = rewriter.create<arith::SubIOp>(loc, createMask.getOperand(0),
+        maskRows = arith::SubIOp::create(rewriter, loc, createMask.getOperand(0),
                                                   transposedRow);
-        maskCols = rewriter.create<arith::SubIOp>(loc, createMask.getOperand(1),
+        maskCols = arith::SubIOp::create(rewriter, loc, createMask.getOperand(1),
                                                   transposedCol);
-        maskCols = rewriter.create<index::MinSOp>(loc, maskCols, numSlices);
+        maskCols = index::MinSOp::create(rewriter, loc, maskCols, numSlices);
       } else {
         maskRows = createVscaleMultiple(smeTileType.getDimSize(0));
         maskCols = numSlices;
       }
-      auto subMask = rewriter.create<vector::CreateMaskOp>(
+      auto subMask = vector::CreateMaskOp::create(rewriter,
           loc, smeTileType.clone(rewriter.getI1Type()),
           ValueRange{maskRows, maskCols});
 
       // 4. Emit a transposed tile write.
       auto writeIndices = writeOp.getIndices();
       Value destRow =
-          rewriter.create<arith::AddIOp>(loc, transposedRow, writeIndices[0]);
+          arith::AddIOp::create(rewriter, loc, transposedRow, writeIndices[0]);
       Value destCol =
-          rewriter.create<arith::AddIOp>(loc, transposedCol, writeIndices[1]);
-      auto smeWrite = rewriter.create<vector::TransferWriteOp>(
+          arith::AddIOp::create(rewriter, loc, transposedCol, writeIndices[1]);
+      auto smeWrite = vector::TransferWriteOp::create(rewriter,
           loc, tile, destTensorOrMemref, ValueRange{destRow, destCol},
           transposeMap, subMask, writeOp.getInBounds());
 
@@ -934,41 +934,41 @@ struct LowerColumnTransferReadToLoops
 
     // Create a loop over all rows and load one element at a time.
     auto loc = readOp.getLoc();
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto createVscaleMultiple =
         vector::makeVscaleConstantBuilder(rewriter, loc);
     auto upperBound = createVscaleMultiple(numRows);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    Value init = rewriter.create<arith::ConstantOp>(
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    Value init = arith::ConstantOp::create(rewriter,
         loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
 
     scf::ForOp loadLoop;
     {
       OpBuilder::InsertionGuard g(rewriter);
-      loadLoop = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+      loadLoop = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
                                              ValueRange{init});
       rewriter.setInsertionPointToStart(loadLoop.getBody());
 
       auto tileSliceIndex = loadLoop.getInductionVar();
 
-      auto idx0 = rewriter.create<arith::AddIOp>(loc, tileSliceIndex,
+      auto idx0 = arith::AddIOp::create(rewriter, loc, tileSliceIndex,
                                                  readOp.getIndices()[0]);
       auto idx1 = readOp.getIndices()[1];
 
-      Value scalar = rewriter.create<memref::LoadOp>(
+      Value scalar = memref::LoadOp::create(rewriter,
           loc, readOp.getBase(), SmallVector<Value>({idx0, idx1}));
 
-      Operation *updateInit = rewriter.create<vector::InsertOp>(
+      Operation *updateInit = vector::InsertOp::create(rewriter,
           loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
 
-      rewriter.create<scf::YieldOp>(loc, updateInit->getResult(0));
+      scf::YieldOp::create(rewriter, loc, updateInit->getResult(0));
     }
 
     // The read operation has been "legalized", but since the original result
     // type was a 2D vector, we need to cast before returning the result. This
     // ShapeCast should cancel-out with some other ShapeCast (i.e. it's a
     // no-op).
-    auto sc = rewriter.create<vector::ShapeCastOp>(
+    auto sc = vector::ShapeCastOp::create(rewriter,
         loc, readOp.getResult().getType(), loadLoop.getResult(0));
 
     rewriter.replaceOp(readOp, sc);
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index 006332b48325f..20f2ad1c3dcec 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -89,7 +89,7 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     VectorType sourceType = source.getType();
     VectorType resultType = convertOp.getResult().getType();
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
 
     // We want to iterate over the input vector in steps of the trailing
@@ -102,14 +102,14 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     for (SmallVector<int64_t> index :
          StaticTileOffsetRange(sourceType.getShape(), tileShape)) {
       auto extractOrInsertPosition = ArrayRef(index).drop_back();
-      auto sourceVector = rewriter.create<vector::ExtractOp>(
+      auto sourceVector = vector::ExtractOp::create(rewriter,
           loc, source, extractOrInsertPosition);
       VectorType convertedType =
           VectorType::Builder(llvm::cast<VectorType>(sourceVector.getType()))
               .setDim(0, resultType.getShape().back());
       auto convertedVector =
-          rewriter.create<IntrOp>(loc, TypeRange{convertedType}, sourceVector);
-      result = rewriter.create<vector::InsertOp>(loc, convertedVector, result,
+          IntrOp::create(rewriter, loc, TypeRange{convertedType}, sourceVector);
+      result = vector::InsertOp::create(rewriter, loc, convertedVector, result,
                                                  extractOrInsertPosition);
     }
 
@@ -137,11 +137,11 @@ struct PselOpLowering : public ConvertOpToLLVMPattern<PselOp> {
                   ConversionPatternRewriter &rewriter) const override {
     auto svboolType = VectorType::get(16, rewriter.getI1Type(), true);
     auto loc = pselOp.getLoc();
-    auto svboolP1 = rewriter.create<ConvertToSvboolIntrOp>(loc, svboolType,
+    auto svboolP1 = ConvertToSvboolIntrOp::create(rewriter, loc, svboolType,
                                                            adaptor.getP1());
-    auto indexI32 = rewriter.create<arith::IndexCastOp>(
+    auto indexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), pselOp.getIndex());
-    auto pselIntr = rewriter.create<PselIntrOp>(loc, svboolType, svboolP1,
+    auto pselIntr = PselIntrOp::create(rewriter, loc, svboolType, svboolP1,
                                                 pselOp.getP2(), indexI32);
     rewriter.replaceOpWithNewOp<ConvertFromSvboolIntrOp>(
         pselOp, adaptor.getP1().getType(), pselIntr);
@@ -176,7 +176,7 @@ struct CreateMaskOpLowering
                                          "not SVE predicate-sized");
 
     auto loc = createMaskOp.getLoc();
-    auto zero = rewriter.create<LLVM::ZeroOp>(loc, rewriter.getI64Type());
+    auto zero = LLVM::ZeroOp::create(rewriter, loc, rewriter.getI64Type());
     rewriter.replaceOpWithNewOp<WhileLTIntrOp>(createMaskOp, maskType, zero,
                                                adaptor.getOperands()[0]);
     return success();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
index 3dbb93b8a0669..33de12ca17cfc 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
@@ -71,7 +71,7 @@ void replaceOpWithUnrealizedConversion(PatternRewriter &rewriter, TOp op,
                                        TLegalizerCallback callback) {
   replaceOpWithLegalizedOp(rewriter, op, [&](TOp newOp) {
     // Mark our `unrealized_conversion_casts` with a pass label.
-    return rewriter.create<UnrealizedConversionCastOp>(
+    return UnrealizedConversionCastOp::create(rewriter,
         op.getLoc(), TypeRange{op.getResult().getType()},
         ValueRange{callback(newOp)},
         NamedAttribute(rewriter.getStringAttr(kSVELegalizerTag),
@@ -239,7 +239,7 @@ struct LegalizeSVEMaskStoreConversion
 
     auto legalMaskType = widenScalableMaskTypeToSvbool(
         llvm::cast<VectorType>(valueToStore.getType()));
-    auto convertToSvbool = rewriter.create<arm_sve::ConvertToSvboolOp>(
+    auto convertToSvbool = arm_sve::ConvertToSvboolOp::create(rewriter,
         loc, legalMaskType, valueToStore);
     // Replace this store with a conversion to a storable svbool mask [1],
     // followed by a wider store.
@@ -290,7 +290,7 @@ struct LegalizeSVEMaskLoadConversion : public OpRewritePattern<memref::LoadOp> {
     replaceOpWithLegalizedOp(rewriter, loadOp, [&](memref::LoadOp newLoadOp) {
       newLoadOp.setMemRef(*legalMemref);
       newLoadOp.getResult().setType(legalMaskType);
-      return rewriter.create<arm_sve::ConvertFromSvboolOp>(
+      return arm_sve::ConvertFromSvboolOp::create(rewriter,
           loc, loadedMask.getType(), newLoadOp);
     });
 
@@ -408,7 +408,7 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
       reassoc.back().push_back(i);
     if (!memref::CollapseShapeOp::isGuaranteedCollapsible(memTy, reassoc))
       return failure();
-    Value collapsedMem = rewriter.create<memref::CollapseShapeOp>(
+    Value collapsedMem = memref::CollapseShapeOp::create(rewriter,
         readOp.getLoc(), readOp.getBase(), reassoc);
 
     // Get a vector type with collapsed trailing dimensions.
@@ -424,13 +424,13 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
     auto indices = readOp.getIndices().drop_back(numCollapseDims - 1);
 
     // Create the new `transfer_read`.
-    auto newReadOp = rewriter.create<vector::TransferReadOp>(
+    auto newReadOp = vector::TransferReadOp::create(rewriter,
         readOp.getLoc(), collapsedVT, collapsedMem, indices,
         readOp.getPadding(),
         ArrayRef<bool>(origInBounds).drop_back(numCollapseDims - 1));
 
     // Cast back to the original vector type.
-    auto toOrigShape = rewriter.create<vector::ShapeCastOp>(readOp.getLoc(),
+    auto toOrigShape = vector::ShapeCastOp::create(rewriter, readOp.getLoc(),
                                                             origVT, newReadOp);
 
     rewriter.replaceOp(readOp, toOrigShape);
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
index b7703ff0393eb..959c5d7291c31 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
@@ -90,15 +90,15 @@ Value createMMLA(PatternRewriter &rewriter, MMLA op, Location loc,
                  mlir::VectorType accType, Value acc, Value lhs, Value rhs) {
   switch (op) {
   case MMLA::Signed:
-    return rewriter.create<arm_sve::SmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::SmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::Unsigned:
-    return rewriter.create<arm_sve::UmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::UmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::Mixed:
-    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::UsmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::MixedSwapped:
     // The accumulator comes transposed and the result will be transposed
     // later, so all we have to do here is swap the operands.
-    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, rhs, lhs);
+    return arm_sve::UsmmlaOp::create(rewriter, loc, accType, acc, rhs, lhs);
   }
 }
 
@@ -236,25 +236,25 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> lhsTile;
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the LHS tile.
-      auto r0 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+      auto r0 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
                                                    ArrayRef<int64_t>{i});
-      auto r1 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+      auto r1 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
                                                    ArrayRef<int64_t>{i + 1});
       // Concatenate to obtain a 16 x i8 flattened sub-tile.
-      auto t = rewriter.create<vector::ShuffleOp>(
+      auto t = vector::ShuffleOp::create(rewriter,
           loc, r0, r1,
           llvm::ArrayRef<int64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
                                   14, 15});
       // Turn it into a scalable vector.
-      auto s = rewriter.create<vector::ScalableInsertOp>(
-          loc, t, rewriter.create<ub::PoisonOp>(loc, nxv16i8), 0);
+      auto s = vector::ScalableInsertOp::create(rewriter,
+          loc, t, ub::PoisonOp::create(rewriter, loc, nxv16i8), 0);
       // Replicate the sub-tile VSCALE times to fill the entire vector.
-      auto r = rewriter.create<arm_sve::DupQLaneOp>(loc, s, 0);
+      auto r = arm_sve::DupQLaneOp::create(rewriter, loc, s, 0);
       lhsTile.push_back(r);
     }
 
     // "Flatten" the RHS tile from <[N]x8> to <[8*N]>.
-    auto rhs = rewriter.create<vector::ShapeCastOp>(
+    auto rhs = vector::ShapeCastOp::create(rewriter,
         maybeRhs->getLoc(),
         VectorType::get(/*shape=*/8 * N, rewriter.getI8Type(),
                         /*scalableDims=*/{true}),
@@ -264,7 +264,7 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> rhsTile;
     for (int64_t j = 0; j < N; j += 2)
       rhsTile.push_back(
-          rewriter.create<vector::ScalableExtractOp>(loc, nxv16i8, rhs, j * 8));
+          vector::ScalableExtractOp::create(rewriter, loc, nxv16i8, rhs, j * 8));
 
     // Handy types for packing/unpacking of the accumulator tile.
     auto accRowTy = VectorType::get(/*shape=*/N, rewriter.getI32Type(),
@@ -280,33 +280,33 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> accTile;
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the accumulator tile.
-      auto r0 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+      auto r0 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
                                                    ArrayRef<int64_t>{i});
-      auto r1 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+      auto r1 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
                                                    ArrayRef<int64_t>{i + 1});
       Value accTileVec;
       if (mmlaOp == MMLA::MixedSwapped) {
         // We need to swap the positions of the LHS and RHS (since we don't have
         // a signed * unsigned operation), but then each individual 2x2 tile of
         // the acumulator and (later) the result need to be transposed.
-        accTileVec = rewriter.create<vector::InterleaveOp>(loc, r0, r1);
+        accTileVec = vector::InterleaveOp::create(rewriter, loc, r0, r1);
       } else {
         // Bitcast them to 64-bit elements, so subsequent
         // interleave/deinterleave work on pairs of 32-bit numbers.
-        auto r0I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r0);
-        auto r1I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r1);
+        auto r0I64 = vector::BitCastOp::create(rewriter, loc, accRow64Ty, r0);
+        auto r1I64 = vector::BitCastOp::create(rewriter, loc, accRow64Ty, r1);
 
         // Interleave the rows, effectively flattening each 2x2 tile into 4
         // consecutive elements.
-        auto intrI64 = rewriter.create<vector::InterleaveOp>(loc, r0I64, r1I64);
+        auto intrI64 = vector::InterleaveOp::create(rewriter, loc, r0I64, r1I64);
 
         // Bitcast back to 32-bit elements.
         accTileVec =
-            rewriter.create<vector::BitCastOp>(loc, accRowX2Ty, intrI64);
+            vector::BitCastOp::create(rewriter, loc, accRowX2Ty, intrI64);
       }
       // Extract ACC sub-tiles.
       for (int64_t j = 0; j < N; j += 2)
-        accTile.push_back(rewriter.create<vector::ScalableExtractOp>(
+        accTile.push_back(vector::ScalableExtractOp::create(rewriter,
             loc, nxv4i32, accTileVec, j * 2));
     }
 
@@ -320,12 +320,12 @@ class LowerContractionToSVEI8MMPattern
       }
 
     // Unpack the OUT sub-tiles and insert into the result.
-    Value result = rewriter.create<ub::PoisonOp>(loc, op.getResultType());
+    Value result = ub::PoisonOp::create(rewriter, loc, op.getResultType());
     for (int64_t i = 0; i < M / 2; ++i) {
       // Collect a number of sub-tiles in a row.
-      Value row = rewriter.create<ub::PoisonOp>(loc, accRowX2Ty);
+      Value row = ub::PoisonOp::create(rewriter, loc, accRowX2Ty);
       for (int64_t j = 0; j < N / 2; ++j)
-        row = rewriter.create<vector::ScalableInsertOp>(
+        row = vector::ScalableInsertOp::create(rewriter,
             loc, outTile[i * N / 2 + j], row, j * 4);
 
       // Unpack the row to obtain two rows of the output. If we have the out
@@ -334,22 +334,22 @@ class LowerContractionToSVEI8MMPattern
       // Otherwise, the interleave is by pairs.
       Value out0, out1;
       if (mmlaOp == MMLA::MixedSwapped) {
-        auto tmp = rewriter.create<vector::DeinterleaveOp>(loc, row);
+        auto tmp = vector::DeinterleaveOp::create(rewriter, loc, row);
         out0 = tmp.getRes1();
         out1 = tmp.getRes2();
       } else {
         // Deinterleave by pairs.
-        auto row64 = rewriter.create<vector::BitCastOp>(loc, accRowX264Ty, row);
-        auto deintr64 = rewriter.create<vector::DeinterleaveOp>(loc, row64);
+        auto row64 = vector::BitCastOp::create(rewriter, loc, accRowX264Ty, row);
+        auto deintr64 = vector::DeinterleaveOp::create(rewriter, loc, row64);
 
         // Bitcast back into 32-bit elements and insert into the result.
-        out0 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+        out0 = vector::BitCastOp::create(rewriter, loc, accRowTy,
                                                   deintr64.getRes1());
-        out1 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+        out1 = vector::BitCastOp::create(rewriter, loc, accRowTy,
                                                   deintr64.getRes2());
       }
-      result = rewriter.create<vector::InsertOp>(loc, out0, result, i * 2);
-      result = rewriter.create<vector::InsertOp>(loc, out1, result, i * 2 + 1);
+      result = vector::InsertOp::create(rewriter, loc, out0, result, i * 2);
+      result = vector::InsertOp::create(rewriter, loc, out1, result, i * 2 + 1);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index b834afef7da79..71ee198fc0c23 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -99,7 +99,7 @@ void ExecuteOp::build(OpBuilder &builder, OperationState &result,
   // expected result is empty. Otherwise, leave this to the caller
   // because we don't know which values to return from the execute op.
   if (resultTypes.empty() && !bodyBuilder) {
-    builder.create<async::YieldOp>(result.location, ValueRange());
+    async::YieldOp::create(builder, result.location, ValueRange());
   } else if (bodyBuilder) {
     bodyBuilder(builder, result.location, bodyBlock->getArguments());
   }
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index 27fa92cee79c2..447411ad989ba 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -191,8 +191,8 @@ static SmallVector<Value> delinearize(ImplicitLocOpBuilder &b, Value index,
   assert(!tripCounts.empty() && "tripCounts must be not empty");
 
   for (ssize_t i = tripCounts.size() - 1; i >= 0; --i) {
-    coords[i] = b.create<arith::RemSIOp>(index, tripCounts[i]);
-    index = b.create<arith::DivSIOp>(index, tripCounts[i]);
+    coords[i] = arith::RemSIOp::create(b, index, tripCounts[i]);
+    index = arith::DivSIOp::create(b, index, tripCounts[i]);
   }
 
   return coords;
@@ -276,15 +276,15 @@ static ParallelComputeFunction createParallelComputeFunction(
   BlockArgument blockSize = args.blockSize();
 
   // Constants used below.
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Materialize known constants as constant operation in the function body.
   auto values = [&](ArrayRef<BlockArgument> args, ArrayRef<IntegerAttr> attrs) {
     return llvm::to_vector(
         llvm::map_range(llvm::zip(args, attrs), [&](auto tuple) -> Value {
           if (IntegerAttr attr = std::get<1>(tuple))
-            return b.create<arith::ConstantOp>(attr);
+            return arith::ConstantOp::create(b, attr);
           return std::get<0>(tuple);
         }));
   };
@@ -303,17 +303,17 @@ static ParallelComputeFunction createParallelComputeFunction(
   // one-dimensional iteration space.
   Value tripCount = tripCounts[0];
   for (unsigned i = 1; i < tripCounts.size(); ++i)
-    tripCount = b.create<arith::MulIOp>(tripCount, tripCounts[i]);
+    tripCount = arith::MulIOp::create(b, tripCount, tripCounts[i]);
 
   // Find one-dimensional iteration bounds: [blockFirstIndex, blockLastIndex]:
   //   blockFirstIndex = blockIndex * blockSize
-  Value blockFirstIndex = b.create<arith::MulIOp>(blockIndex, blockSize);
+  Value blockFirstIndex = arith::MulIOp::create(b, blockIndex, blockSize);
 
   // The last one-dimensional index in the block defined by the `blockIndex`:
   //   blockLastIndex = min(blockFirstIndex + blockSize, tripCount) - 1
-  Value blockEnd0 = b.create<arith::AddIOp>(blockFirstIndex, blockSize);
-  Value blockEnd1 = b.create<arith::MinSIOp>(blockEnd0, tripCount);
-  Value blockLastIndex = b.create<arith::SubIOp>(blockEnd1, c1);
+  Value blockEnd0 = arith::AddIOp::create(b, blockFirstIndex, blockSize);
+  Value blockEnd1 = arith::MinSIOp::create(b, blockEnd0, tripCount);
+  Value blockLastIndex = arith::SubIOp::create(b, blockEnd1, c1);
 
   // Convert one-dimensional indices to multi-dimensional coordinates.
   auto blockFirstCoord = delinearize(b, blockFirstIndex, tripCounts);
@@ -326,7 +326,7 @@ static ParallelComputeFunction createParallelComputeFunction(
   // dimension when inner compute dimension contains multiple blocks.
   SmallVector<Value> blockEndCoord(op.getNumLoops());
   for (size_t i = 0; i < blockLastCoord.size(); ++i)
-    blockEndCoord[i] = b.create<arith::AddIOp>(blockLastCoord[i], c1);
+    blockEndCoord[i] = arith::AddIOp::create(b, blockLastCoord[i], c1);
 
   // Construct a loop nest out of scf.for operations that will iterate over
   // all coordinates in [blockFirstCoord, blockLastCoord] range.
@@ -369,20 +369,20 @@ static ParallelComputeFunction createParallelComputeFunction(
       ImplicitLocOpBuilder b(loc, nestedBuilder);
 
       // Compute induction variable for `loopIdx`.
-      computeBlockInductionVars[loopIdx] = b.create<arith::AddIOp>(
-          lowerBounds[loopIdx], b.create<arith::MulIOp>(iv, steps[loopIdx]));
+      computeBlockInductionVars[loopIdx] = arith::AddIOp::create(b,
+          lowerBounds[loopIdx], arith::MulIOp::create(b, iv, steps[loopIdx]));
 
       // Check if we are inside first or last iteration of the loop.
-      isBlockFirstCoord[loopIdx] = b.create<arith::CmpIOp>(
+      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(b,
           arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
-      isBlockLastCoord[loopIdx] = b.create<arith::CmpIOp>(
+      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(b,
           arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
 
       // Check if the previous loop is in its first or last iteration.
       if (loopIdx > 0) {
-        isBlockFirstCoord[loopIdx] = b.create<arith::AndIOp>(
+        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(b,
             isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
-        isBlockLastCoord[loopIdx] = b.create<arith::AndIOp>(
+        isBlockLastCoord[loopIdx] = arith::AndIOp::create(b,
             isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
       }
 
@@ -391,24 +391,24 @@ static ParallelComputeFunction createParallelComputeFunction(
         if (loopIdx + 1 >= op.getNumLoops() - numBlockAlignedInnerLoops) {
           // For block aligned loops we always iterate starting from 0 up to
           // the loop trip counts.
-          b.create<scf::ForOp>(c0, tripCounts[loopIdx + 1], c1, ValueRange(),
+          scf::ForOp::create(b, c0, tripCounts[loopIdx + 1], c1, ValueRange(),
                                workLoopBuilder(loopIdx + 1));
 
         } else {
           // Select nested loop lower/upper bounds depending on our position in
           // the multi-dimensional iteration space.
-          auto lb = b.create<arith::SelectOp>(isBlockFirstCoord[loopIdx],
+          auto lb = arith::SelectOp::create(b, isBlockFirstCoord[loopIdx],
                                               blockFirstCoord[loopIdx + 1], c0);
 
-          auto ub = b.create<arith::SelectOp>(isBlockLastCoord[loopIdx],
+          auto ub = arith::SelectOp::create(b, isBlockLastCoord[loopIdx],
                                               blockEndCoord[loopIdx + 1],
                                               tripCounts[loopIdx + 1]);
 
-          b.create<scf::ForOp>(lb, ub, c1, ValueRange(),
+          scf::ForOp::create(b, lb, ub, c1, ValueRange(),
                                workLoopBuilder(loopIdx + 1));
         }
 
-        b.create<scf::YieldOp>(loc);
+        scf::YieldOp::create(b, loc);
         return;
       }
 
@@ -419,13 +419,13 @@ static ParallelComputeFunction createParallelComputeFunction(
 
       for (auto &bodyOp : op.getRegion().front().without_terminator())
         b.clone(bodyOp, mapping);
-      b.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(b, loc);
     };
   };
 
-  b.create<scf::ForOp>(blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
+  scf::ForOp::create(b, blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
                        workLoopBuilder(0));
-  b.create<func::ReturnOp>(ValueRange());
+  func::ReturnOp::create(b, ValueRange());
 
   return {op.getNumLoops(), func, std::move(computeFuncType.captures)};
 }
@@ -485,8 +485,8 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   b.setInsertionPointToEnd(block);
 
   Type indexTy = b.getIndexType();
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
-  Value c2 = b.create<arith::ConstantIndexOp>(2);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
+  Value c2 = arith::ConstantIndexOp::create(b, 2);
 
   // Get the async group that will track async dispatch completion.
   Value group = block->getArgument(0);
@@ -501,7 +501,7 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   SmallVector<Location> locations = {loc, loc};
 
   // Create a recursive dispatch loop.
-  scf::WhileOp whileOp = b.create<scf::WhileOp>(types, operands);
+  scf::WhileOp whileOp = scf::WhileOp::create(b, types, operands);
   Block *before = b.createBlock(&whileOp.getBefore(), {}, types, locations);
   Block *after = b.createBlock(&whileOp.getAfter(), {}, types, locations);
 
@@ -511,10 +511,10 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
     b.setInsertionPointToEnd(before);
     Value start = before->getArgument(0);
     Value end = before->getArgument(1);
-    Value distance = b.create<arith::SubIOp>(end, start);
+    Value distance = arith::SubIOp::create(b, end, start);
     Value dispatch =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, distance, c1);
-    b.create<scf::ConditionOp>(dispatch, before->getArguments());
+        arith::CmpIOp::create(b, arith::CmpIPredicate::sgt, distance, c1);
+    scf::ConditionOp::create(b, dispatch, before->getArguments());
   }
 
   // Setup the async dispatch loop body: recursively call dispatch function
@@ -523,9 +523,9 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
     b.setInsertionPointToEnd(after);
     Value start = after->getArgument(0);
     Value end = after->getArgument(1);
-    Value distance = b.create<arith::SubIOp>(end, start);
-    Value halfDistance = b.create<arith::DivSIOp>(distance, c2);
-    Value midIndex = b.create<arith::AddIOp>(start, halfDistance);
+    Value distance = arith::SubIOp::create(b, end, start);
+    Value halfDistance = arith::DivSIOp::create(b, distance, c2);
+    Value midIndex = arith::AddIOp::create(b, start, halfDistance);
 
     // Call parallel compute function inside the async.execute region.
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
@@ -536,16 +536,16 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
       operands[1] = midIndex;
       operands[2] = end;
 
-      executeBuilder.create<func::CallOp>(executeLoc, func.getSymName(),
+      func::CallOp::create(executeBuilder, executeLoc, func.getSymName(),
                                           func.getResultTypes(), operands);
-      executeBuilder.create<async::YieldOp>(executeLoc, ValueRange());
+      async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to dispatch half of the block range.
-    auto execute = b.create<ExecuteOp>(TypeRange(), ValueRange(), ValueRange(),
+    auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
                                        executeBodyBuilder);
-    b.create<AddToGroupOp>(indexTy, execute.getToken(), group);
-    b.create<scf::YieldOp>(ValueRange({start, midIndex}));
+    AddToGroupOp::create(b, indexTy, execute.getToken(), group);
+    scf::YieldOp::create(b, ValueRange({start, midIndex}));
   }
 
   // After dispatching async operations to process the tail of the block range
@@ -557,10 +557,10 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   SmallVector<Value> computeFuncOperands = {blockStart};
   computeFuncOperands.append(forwardedInputs.begin(), forwardedInputs.end());
 
-  b.create<func::CallOp>(computeFunc.func.getSymName(),
+  func::CallOp::create(b, computeFunc.func.getSymName(),
                          computeFunc.func.getResultTypes(),
                          computeFuncOperands);
-  b.create<func::ReturnOp>(ValueRange());
+  func::ReturnOp::create(b, ValueRange());
 
   return func;
 }
@@ -578,8 +578,8 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
   func::FuncOp asyncDispatchFunction =
       createAsyncDispatchFunction(parallelComputeFunction, rewriter);
 
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Appends operands shared by async dispatch and parallel compute functions to
   // the given operands vector.
@@ -595,7 +595,7 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
   // completely. If this will be known statically, then canonicalization will
   // erase async group operations.
   Value isSingleBlock =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, blockCount, c1);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, blockCount, c1);
 
   auto syncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
     ImplicitLocOpBuilder b(loc, nestedBuilder);
@@ -604,10 +604,10 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     SmallVector<Value> operands = {c0, blockSize};
     appendBlockComputeOperands(operands);
 
-    b.create<func::CallOp>(parallelComputeFunction.func.getSymName(),
+    func::CallOp::create(b, parallelComputeFunction.func.getSymName(),
                            parallelComputeFunction.func.getResultTypes(),
                            operands);
-    b.create<scf::YieldOp>();
+    scf::YieldOp::create(b, );
   };
 
   auto asyncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -616,24 +616,24 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     // Create an async.group to wait on all async tokens from the concurrent
     // execution of multiple parallel compute function. First block will be
     // executed synchronously in the caller thread.
-    Value groupSize = b.create<arith::SubIOp>(blockCount, c1);
-    Value group = b.create<CreateGroupOp>(GroupType::get(ctx), groupSize);
+    Value groupSize = arith::SubIOp::create(b, blockCount, c1);
+    Value group = CreateGroupOp::create(b, GroupType::get(ctx), groupSize);
 
     // Launch async dispatch function for [0, blockCount) range.
     SmallVector<Value> operands = {group, c0, blockCount, blockSize};
     appendBlockComputeOperands(operands);
 
-    b.create<func::CallOp>(asyncDispatchFunction.getSymName(),
+    func::CallOp::create(b, asyncDispatchFunction.getSymName(),
                            asyncDispatchFunction.getResultTypes(), operands);
 
     // Wait for the completion of all parallel compute operations.
-    b.create<AwaitAllOp>(group);
+    AwaitAllOp::create(b, group);
 
-    b.create<scf::YieldOp>();
+    scf::YieldOp::create(b, );
   };
 
   // Dispatch either single block compute function, or launch async dispatch.
-  b.create<scf::IfOp>(isSingleBlock, syncDispatch, asyncDispatch);
+  scf::IfOp::create(b, isSingleBlock, syncDispatch, asyncDispatch);
 }
 
 // Dispatch parallel compute functions by submitting all async compute tasks
@@ -647,14 +647,14 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
 
   func::FuncOp compute = parallelComputeFunction.func;
 
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Create an async.group to wait on all async tokens from the concurrent
   // execution of multiple parallel compute function. First block will be
   // executed synchronously in the caller thread.
-  Value groupSize = b.create<arith::SubIOp>(blockCount, c1);
-  Value group = b.create<CreateGroupOp>(GroupType::get(ctx), groupSize);
+  Value groupSize = arith::SubIOp::create(b, blockCount, c1);
+  Value group = CreateGroupOp::create(b, GroupType::get(ctx), groupSize);
 
   // Call parallel compute function for all blocks.
   using LoopBodyBuilder =
@@ -681,28 +681,28 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     // Call parallel compute function inside the async.execute region.
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
                                   Location executeLoc, ValueRange executeArgs) {
-      executeBuilder.create<func::CallOp>(executeLoc, compute.getSymName(),
+      func::CallOp::create(executeBuilder, executeLoc, compute.getSymName(),
                                           compute.getResultTypes(),
                                           computeFuncOperands(iv));
-      executeBuilder.create<async::YieldOp>(executeLoc, ValueRange());
+      async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to launch parallel computate function.
-    auto execute = b.create<ExecuteOp>(TypeRange(), ValueRange(), ValueRange(),
+    auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
                                        executeBodyBuilder);
-    b.create<AddToGroupOp>(rewriter.getIndexType(), execute.getToken(), group);
-    b.create<scf::YieldOp>();
+    AddToGroupOp::create(b, rewriter.getIndexType(), execute.getToken(), group);
+    scf::YieldOp::create(b, );
   };
 
   // Iterate over all compute blocks and launch parallel compute operations.
-  b.create<scf::ForOp>(c1, blockCount, c1, ValueRange(), loopBuilder);
+  scf::ForOp::create(b, c1, blockCount, c1, ValueRange(), loopBuilder);
 
   // Call parallel compute function for the first block in the caller thread.
-  b.create<func::CallOp>(compute.getSymName(), compute.getResultTypes(),
+  func::CallOp::create(b, compute.getSymName(), compute.getResultTypes(),
                          computeFuncOperands(c0));
 
   // Wait for the completion of all async compute operations.
-  b.create<AwaitAllOp>(group);
+  AwaitAllOp::create(b, group);
 }
 
 LogicalResult
@@ -738,17 +738,17 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
   // for the scf.parallel operation.
   Value tripCount = tripCounts[0];
   for (size_t i = 1; i < tripCounts.size(); ++i)
-    tripCount = b.create<arith::MulIOp>(tripCount, tripCounts[i]);
+    tripCount = arith::MulIOp::create(b, tripCount, tripCounts[i]);
 
   // Short circuit no-op parallel loops (zero iterations) that can arise from
   // the memrefs with dynamic dimension(s) equal to zero.
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
   Value isZeroIterations =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, tripCount, c0);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, tripCount, c0);
 
   // Do absolutely nothing if the trip count is zero.
   auto noOp = [&](OpBuilder &nestedBuilder, Location loc) {
-    nestedBuilder.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(nestedBuilder, loc);
   };
 
   // Compute the parallel block size and dispatch concurrent tasks computing
@@ -798,9 +798,9 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
     Value numWorkerThreadsVal;
     if (numWorkerThreads >= 0)
-      numWorkerThreadsVal = b.create<arith::ConstantIndexOp>(numWorkerThreads);
+      numWorkerThreadsVal = arith::ConstantIndexOp::create(b, numWorkerThreads);
     else
-      numWorkerThreadsVal = b.create<async::RuntimeNumWorkerThreadsOp>();
+      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b, );
 
     // With large number of threads the value of creating many compute blocks
     // is reduced because the problem typically becomes memory bound. For this
@@ -819,38 +819,38 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
         {4, 4.0f}, {8, 2.0f}, {16, 1.0f}, {32, 0.8f}, {64, 0.6f}};
     const float initialOvershardingFactor = 8.0f;
 
-    Value scalingFactor = b.create<arith::ConstantFloatOp>(
+    Value scalingFactor = arith::ConstantFloatOp::create(b,
         b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
     for (const std::pair<int, float> &p : overshardingBrackets) {
-      Value bracketBegin = b.create<arith::ConstantIndexOp>(p.first);
-      Value inBracket = b.create<arith::CmpIOp>(
+      Value bracketBegin = arith::ConstantIndexOp::create(b, p.first);
+      Value inBracket = arith::CmpIOp::create(b,
           arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
-      Value bracketScalingFactor = b.create<arith::ConstantFloatOp>(
+      Value bracketScalingFactor = arith::ConstantFloatOp::create(b,
           b.getF32Type(), llvm::APFloat(p.second));
-      scalingFactor = b.create<arith::SelectOp>(inBracket, bracketScalingFactor,
+      scalingFactor = arith::SelectOp::create(b, inBracket, bracketScalingFactor,
                                                 scalingFactor);
     }
     Value numWorkersIndex =
-        b.create<arith::IndexCastOp>(b.getI32Type(), numWorkerThreadsVal);
+        arith::IndexCastOp::create(b, b.getI32Type(), numWorkerThreadsVal);
     Value numWorkersFloat =
-        b.create<arith::SIToFPOp>(b.getF32Type(), numWorkersIndex);
+        arith::SIToFPOp::create(b, b.getF32Type(), numWorkersIndex);
     Value scaledNumWorkers =
-        b.create<arith::MulFOp>(scalingFactor, numWorkersFloat);
+        arith::MulFOp::create(b, scalingFactor, numWorkersFloat);
     Value scaledNumInt =
-        b.create<arith::FPToSIOp>(b.getI32Type(), scaledNumWorkers);
+        arith::FPToSIOp::create(b, b.getI32Type(), scaledNumWorkers);
     Value scaledWorkers =
-        b.create<arith::IndexCastOp>(b.getIndexType(), scaledNumInt);
+        arith::IndexCastOp::create(b, b.getIndexType(), scaledNumInt);
 
-    Value maxComputeBlocks = b.create<arith::MaxSIOp>(
-        b.create<arith::ConstantIndexOp>(1), scaledWorkers);
+    Value maxComputeBlocks = arith::MaxSIOp::create(b,
+        arith::ConstantIndexOp::create(b, 1), scaledWorkers);
 
     // Compute parallel block size from the parallel problem size:
     //   blockSize = min(tripCount,
     //                   max(ceil_div(tripCount, maxComputeBlocks),
     //                       minTaskSize))
-    Value bs0 = b.create<arith::CeilDivSIOp>(tripCount, maxComputeBlocks);
-    Value bs1 = b.create<arith::MaxSIOp>(bs0, minTaskSize);
-    Value blockSize = b.create<arith::MinSIOp>(tripCount, bs1);
+    Value bs0 = arith::CeilDivSIOp::create(b, tripCount, maxComputeBlocks);
+    Value bs1 = arith::MaxSIOp::create(b, bs0, minTaskSize);
+    Value blockSize = arith::MinSIOp::create(b, tripCount, bs1);
 
     // Dispatch parallel compute function using async recursive work splitting,
     // or by submitting compute task sequentially from a caller thread.
@@ -860,7 +860,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     // the parallel operation body for a subset of iteration space.
 
     // Compute the number of parallel compute blocks.
-    Value blockCount = b.create<arith::CeilDivSIOp>(tripCount, blockSize);
+    Value blockCount = arith::CeilDivSIOp::create(b, tripCount, blockSize);
 
     // Dispatch parallel compute function without hints to unroll inner loops.
     auto dispatchDefault = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -869,7 +869,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       doDispatch(b, rewriter, compute, op, blockSize, blockCount, tripCounts);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     };
 
     // Dispatch parallel compute function with hints for unrolling inner loops.
@@ -880,34 +880,34 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       // Align the block size to be a multiple of the statically known
       // number of iterations in the inner loops.
-      Value numIters = b.create<arith::ConstantIndexOp>(
+      Value numIters = arith::ConstantIndexOp::create(b,
           numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value alignedBlockSize = b.create<arith::MulIOp>(
-          b.create<arith::CeilDivSIOp>(blockSize, numIters), numIters);
+      Value alignedBlockSize = arith::MulIOp::create(b,
+          arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
       doDispatch(b, rewriter, compute, op, alignedBlockSize, blockCount,
                  tripCounts);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     };
 
     // Dispatch to block aligned compute function only if the computed block
     // size is larger than the number of iterations in the unrollable inner
     // loops, because otherwise it can reduce the available parallelism.
     if (numUnrollableLoops > 0) {
-      Value numIters = b.create<arith::ConstantIndexOp>(
+      Value numIters = arith::ConstantIndexOp::create(b,
           numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value useBlockAlignedComputeFn = b.create<arith::CmpIOp>(
+      Value useBlockAlignedComputeFn = arith::CmpIOp::create(b,
           arith::CmpIPredicate::sge, blockSize, numIters);
 
-      b.create<scf::IfOp>(useBlockAlignedComputeFn, dispatchBlockAligned,
+      scf::IfOp::create(b, useBlockAlignedComputeFn, dispatchBlockAligned,
                           dispatchDefault);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     } else {
       dispatchDefault(b, loc);
     }
   };
 
   // Replace the `scf.parallel` operation with the parallel compute function.
-  b.create<scf::IfOp>(isZeroIterations, noOp, dispatch);
+  scf::IfOp::create(b, isZeroIterations, noOp, dispatch);
 
   // Parallel operation was replaced with a block iteration loop.
   rewriter.eraseOp(op);
@@ -922,7 +922,7 @@ void AsyncParallelForPass::runOnOperation() {
   populateAsyncParallelForPatterns(
       patterns, asyncDispatch, numWorkerThreads,
       [&](ImplicitLocOpBuilder builder, scf::ParallelOp op) {
-        return builder.create<arith::ConstantIndexOp>(minTaskSize);
+        return arith::ConstantIndexOp::create(builder, minTaskSize);
       });
   if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
     signalPassFailure();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
index d11cd8444636a..6bcc5849eff1b 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
@@ -51,7 +51,7 @@ static LogicalResult dropRefIfNoUses(Value value, unsigned count = 1) {
   else
     b.setInsertionPointToStart(value.getParentBlock());
 
-  b.create<RuntimeDropRefOp>(value.getLoc(), value, b.getI64IntegerAttr(1));
+  RuntimeDropRefOp::create(b, value.getLoc(), value, b.getI64IntegerAttr(1));
   return success();
 }
 
@@ -312,7 +312,7 @@ LogicalResult AsyncRuntimeRefCountingPass::addDropRefAfterLastUse(Value value) {
 
     // Add a drop_ref immediately after the last user.
     builder.setInsertionPointAfter(lastUser);
-    builder.create<RuntimeDropRefOp>(loc, value, builder.getI64IntegerAttr(1));
+    RuntimeDropRefOp::create(builder, loc, value, builder.getI64IntegerAttr(1));
   }
 
   return success();
@@ -330,7 +330,7 @@ AsyncRuntimeRefCountingPass::addAddRefBeforeFunctionCall(Value value) {
     // Add a reference before the function call to pass the value at `+1`
     // reference to the function entry block.
     builder.setInsertionPoint(user);
-    builder.create<RuntimeAddRefOp>(loc, value, builder.getI64IntegerAttr(1));
+    RuntimeAddRefOp::create(builder, loc, value, builder.getI64IntegerAttr(1));
   }
 
   return success();
@@ -414,11 +414,11 @@ AsyncRuntimeRefCountingPass::addDropRefInDivergentLivenessSuccessor(
         refCountingBlock = &successor->getParent()->emplaceBlock();
         refCountingBlock->moveBefore(successor);
         OpBuilder builder = OpBuilder::atBlockEnd(refCountingBlock);
-        builder.create<cf::BranchOp>(value.getLoc(), successor);
+        cf::BranchOp::create(builder, value.getLoc(), successor);
       }
 
       OpBuilder builder = OpBuilder::atBlockBegin(refCountingBlock);
-      builder.create<RuntimeDropRefOp>(value.getLoc(), value,
+      RuntimeDropRefOp::create(builder, value.getLoc(), value,
                                        builder.getI64IntegerAttr(1));
 
       // No need to update the terminator operation.
@@ -510,13 +510,13 @@ AsyncRuntimePolicyBasedRefCountingPass::addRefCounting(Value value) {
       // Create `add_ref` operation before the operand owner.
       if (cnt > 0) {
         b.setInsertionPoint(operand.getOwner());
-        b.create<RuntimeAddRefOp>(loc, value, b.getI64IntegerAttr(cnt));
+        RuntimeAddRefOp::create(b, loc, value, b.getI64IntegerAttr(cnt));
       }
 
       // Create `drop_ref` operation after the operand owner.
       if (cnt < 0) {
         b.setInsertionPointAfter(operand.getOwner());
-        b.create<RuntimeDropRefOp>(loc, value, b.getI64IntegerAttr(-cnt));
+        RuntimeDropRefOp::create(b, loc, value, b.getI64IntegerAttr(-cnt));
       }
     }
   }
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index 8601bb5aaada9..8b9feb8214923 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -188,22 +188,22 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
 
   std::optional<Value> retToken;
   if (isStateful)
-    retToken.emplace(builder.create<RuntimeCreateOp>(TokenType::get(ctx)));
+    retToken.emplace(RuntimeCreateOp::create(builder, TokenType::get(ctx)));
 
   llvm::SmallVector<Value, 4> retValues;
   ArrayRef<Type> resValueTypes =
       isStateful ? func.getResultTypes().drop_front() : func.getResultTypes();
   for (auto resType : resValueTypes)
     retValues.emplace_back(
-        builder.create<RuntimeCreateOp>(resType).getResult());
+        RuntimeCreateOp::create(builder, resType).getResult());
 
   // ------------------------------------------------------------------------ //
   // Initialize coroutine: get coroutine id and coroutine handle.
   // ------------------------------------------------------------------------ //
-  auto coroIdOp = builder.create<CoroIdOp>(CoroIdType::get(ctx));
+  auto coroIdOp = CoroIdOp::create(builder, CoroIdType::get(ctx));
   auto coroHdlOp =
-      builder.create<CoroBeginOp>(CoroHandleType::get(ctx), coroIdOp.getId());
-  builder.create<cf::BranchOp>(originalEntryBlock);
+      CoroBeginOp::create(builder, CoroHandleType::get(ctx), coroIdOp.getId());
+  cf::BranchOp::create(builder, originalEntryBlock);
 
   Block *cleanupBlock = func.addBlock();
   Block *cleanupBlockForDestroy = func.addBlock();
@@ -214,10 +214,10 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   // ------------------------------------------------------------------------ //
   auto buildCleanupBlock = [&](Block *cb) {
     builder.setInsertionPointToStart(cb);
-    builder.create<CoroFreeOp>(coroIdOp.getId(), coroHdlOp.getHandle());
+    CoroFreeOp::create(builder, coroIdOp.getId(), coroHdlOp.getHandle());
 
     // Branch into the suspend block.
-    builder.create<cf::BranchOp>(suspendBlock);
+    cf::BranchOp::create(builder, suspendBlock);
   };
   buildCleanupBlock(cleanupBlock);
   buildCleanupBlock(cleanupBlockForDestroy);
@@ -229,7 +229,7 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   builder.setInsertionPointToStart(suspendBlock);
 
   // Mark the end of a coroutine: async.coro.end
-  builder.create<CoroEndOp>(coroHdlOp.getHandle());
+  CoroEndOp::create(builder, coroHdlOp.getHandle());
 
   // Return created optional `async.token` and `async.values` from the suspend
   // block. This will be the return value of a coroutine ramp function.
@@ -237,7 +237,7 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   if (retToken)
     ret.push_back(*retToken);
   llvm::append_range(ret, retValues);
-  builder.create<func::ReturnOp>(ret);
+  func::ReturnOp::create(builder, ret);
 
   // `async.await` op lowering will create resume blocks for async
   // continuations, and will conditionally branch to cleanup or suspend blocks.
@@ -274,13 +274,13 @@ static Block *setupSetErrorBlock(CoroMachinery &coro) {
 
   // Coroutine set_error block: set error on token and all returned values.
   if (coro.asyncToken)
-    builder.create<RuntimeSetErrorOp>(*coro.asyncToken);
+    RuntimeSetErrorOp::create(builder, *coro.asyncToken);
 
   for (Value retValue : coro.returnValues)
-    builder.create<RuntimeSetErrorOp>(retValue);
+    RuntimeSetErrorOp::create(builder, retValue);
 
   // Branch into the cleanup block.
-  builder.create<cf::BranchOp>(coro.cleanup);
+  cf::BranchOp::create(builder, coro.cleanup);
 
   return *coro.setError;
 }
@@ -335,13 +335,13 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Await on all dependencies before starting to execute the body region.
     for (size_t i = 0; i < numDependencies; ++i)
-      builder.create<AwaitOp>(func.getArgument(i));
+      AwaitOp::create(builder, func.getArgument(i));
 
     // Await on all async value operands and unwrap the payload.
     SmallVector<Value, 4> unwrappedOperands(numOperands);
     for (size_t i = 0; i < numOperands; ++i) {
       Value operand = func.getArgument(numDependencies + i);
-      unwrappedOperands[i] = builder.create<AwaitOp>(loc, operand).getResult();
+      unwrappedOperands[i] = AwaitOp::create(builder, loc, operand).getResult();
     }
 
     // Map from function inputs defined above the execute op to the function
@@ -368,14 +368,14 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Save the coroutine state: async.coro.save
     auto coroSaveOp =
-        builder.create<CoroSaveOp>(CoroStateType::get(ctx), coro.coroHandle);
+        CoroSaveOp::create(builder, CoroStateType::get(ctx), coro.coroHandle);
 
     // Pass coroutine to the runtime to be resumed on a runtime managed
     // thread.
-    builder.create<RuntimeResumeOp>(coro.coroHandle);
+    RuntimeResumeOp::create(builder, coro.coroHandle);
 
     // Add async.coro.suspend as a suspended block terminator.
-    builder.create<CoroSuspendOp>(coroSaveOp.getState(), coro.suspend,
+    CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend,
                                   branch.getDest(), coro.cleanupForDestroy);
 
     branch.erase();
@@ -384,7 +384,7 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
   // Replace the original `async.execute` with a call to outlined function.
   {
     ImplicitLocOpBuilder callBuilder(loc, execute);
-    auto callOutlinedFunc = callBuilder.create<func::CallOp>(
+    auto callOutlinedFunc = func::CallOp::create(callBuilder,
         func.getName(), execute.getResultTypes(), functionInputs.getArrayRef());
     execute.replaceAllUsesWith(callOutlinedFunc.getResults());
     execute.erase();
@@ -453,7 +453,7 @@ class AsyncFuncOpLowering : public OpConversionPattern<async::FuncOp> {
     Location loc = op->getLoc();
 
     auto newFuncOp =
-        rewriter.create<func::FuncOp>(loc, op.getName(), op.getFunctionType());
+        func::FuncOp::create(rewriter, loc, op.getName(), op.getFunctionType());
 
     SymbolTable::setSymbolVisibility(newFuncOp,
                                      SymbolTable::getSymbolVisibility(op));
@@ -523,16 +523,16 @@ class AsyncReturnOpLowering : public OpConversionPattern<async::ReturnOp> {
     for (auto tuple : llvm::zip(adaptor.getOperands(), coro.returnValues)) {
       Value returnValue = std::get<0>(tuple);
       Value asyncValue = std::get<1>(tuple);
-      rewriter.create<RuntimeStoreOp>(loc, returnValue, asyncValue);
-      rewriter.create<RuntimeSetAvailableOp>(loc, asyncValue);
+      RuntimeStoreOp::create(rewriter, loc, returnValue, asyncValue);
+      RuntimeSetAvailableOp::create(rewriter, loc, asyncValue);
     }
 
     if (coro.asyncToken)
       // Switch the coroutine completion token to available state.
-      rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
+      RuntimeSetAvailableOp::create(rewriter, loc, *coro.asyncToken);
 
     rewriter.eraseOp(op);
-    rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    cf::BranchOp::create(rewriter, loc, coro.cleanup);
     return success();
   }
 
@@ -583,15 +583,15 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
     // the async object (token, value or group) to become available.
     if (!isInCoroutine) {
       ImplicitLocOpBuilder builder(loc, rewriter);
-      builder.create<RuntimeAwaitOp>(loc, operand);
+      RuntimeAwaitOp::create(builder, loc, operand);
 
       // Assert that the awaited operands is not in the error state.
-      Value isError = builder.create<RuntimeIsErrorOp>(i1, operand);
-      Value notError = builder.create<arith::XOrIOp>(
-          isError, builder.create<arith::ConstantOp>(
+      Value isError = RuntimeIsErrorOp::create(builder, i1, operand);
+      Value notError = arith::XOrIOp::create(builder,
+          isError, arith::ConstantOp::create(builder,
                        loc, i1, builder.getIntegerAttr(i1, 1)));
 
-      builder.create<cf::AssertOp>(notError,
+      cf::AssertOp::create(builder, notError,
                                    "Awaited async operand is in error state");
     }
 
@@ -607,15 +607,15 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
       // Save the coroutine state and resume on a runtime managed thread when
       // the operand becomes available.
       auto coroSaveOp =
-          builder.create<CoroSaveOp>(CoroStateType::get(ctx), coro.coroHandle);
-      builder.create<RuntimeAwaitAndResumeOp>(operand, coro.coroHandle);
+          CoroSaveOp::create(builder, CoroStateType::get(ctx), coro.coroHandle);
+      RuntimeAwaitAndResumeOp::create(builder, operand, coro.coroHandle);
 
       // Split the entry block before the await operation.
       Block *resume = rewriter.splitBlock(suspended, Block::iterator(op));
 
       // Add async.coro.suspend as a suspended block terminator.
       builder.setInsertionPointToEnd(suspended);
-      builder.create<CoroSuspendOp>(coroSaveOp.getState(), coro.suspend, resume,
+      CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend, resume,
                                     coro.cleanupForDestroy);
 
       // Split the resume block into error checking and continuation.
@@ -623,8 +623,8 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
 
       // Check if the awaited value is in the error state.
       builder.setInsertionPointToStart(resume);
-      auto isError = builder.create<RuntimeIsErrorOp>(loc, i1, operand);
-      builder.create<cf::CondBranchOp>(isError,
+      auto isError = RuntimeIsErrorOp::create(builder, loc, i1, operand);
+      cf::CondBranchOp::create(builder, isError,
                                        /*trueDest=*/setupSetErrorBlock(coro),
                                        /*trueArgs=*/ArrayRef<Value>(),
                                        /*falseDest=*/continuation,
@@ -674,7 +674,7 @@ class AwaitValueOpLowering : public AwaitOpLoweringBase<AwaitOp, ValueType> {
                       ConversionPatternRewriter &rewriter) const override {
     // Load from the async value storage.
     auto valueType = cast<ValueType>(operand.getType()).getValueType();
-    return rewriter.create<RuntimeLoadOp>(op->getLoc(), valueType, operand);
+    return RuntimeLoadOp::create(rewriter, op->getLoc(), valueType, operand);
   }
 };
 
@@ -715,15 +715,15 @@ class YieldOpLowering : public OpConversionPattern<async::YieldOp> {
     for (auto tuple : llvm::zip(adaptor.getOperands(), coro.returnValues)) {
       Value yieldValue = std::get<0>(tuple);
       Value asyncValue = std::get<1>(tuple);
-      rewriter.create<RuntimeStoreOp>(loc, yieldValue, asyncValue);
-      rewriter.create<RuntimeSetAvailableOp>(loc, asyncValue);
+      RuntimeStoreOp::create(rewriter, loc, yieldValue, asyncValue);
+      RuntimeSetAvailableOp::create(rewriter, loc, asyncValue);
     }
 
     if (coro.asyncToken)
       // Switch the coroutine completion token to available state.
-      rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
+      RuntimeSetAvailableOp::create(rewriter, loc, *coro.asyncToken);
 
-    rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    cf::BranchOp::create(rewriter, loc, coro.cleanup);
     rewriter.eraseOp(op);
 
     return success();
@@ -757,7 +757,7 @@ class AssertOpLowering : public OpConversionPattern<cf::AssertOp> {
 
     Block *cont = rewriter.splitBlock(op->getBlock(), Block::iterator(op));
     rewriter.setInsertionPointToEnd(cont->getPrevNode());
-    rewriter.create<cf::CondBranchOp>(loc, adaptor.getArg(),
+    cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(),
                                       /*trueDest=*/cont,
                                       /*trueArgs=*/ArrayRef<Value>(),
                                       /*falseDest=*/setupSetErrorBlock(coro),
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index ca914df8b7890..59ebdbfdc727d 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -36,7 +36,7 @@ using namespace bufferization;
 //===----------------------------------------------------------------------===//
 
 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
-  return builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
 }
 
 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
@@ -151,7 +151,7 @@ DeallocationState::getMemrefWithUniqueOwnership(OpBuilder &builder,
   // ownerships more intelligently to not end up with an 'Unknown' ownership in
   // the first place.
   auto cloneOp =
-      builder.create<bufferization::CloneOp>(memref.getLoc(), memref);
+      bufferization::CloneOp::create(builder, memref.getLoc(), memref);
   Value condition = buildBoolValue(builder, memref.getLoc(), true);
   Value newMemref = cloneOp.getResult();
   updateOwnership(newMemref, condition);
@@ -197,7 +197,7 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // Simply cast unranked MemRefs to ranked memrefs with 0 dimensions such
     // that we can call extract_strided_metadata on it.
     if (auto unrankedMemRefTy = dyn_cast<UnrankedMemRefType>(memref.getType()))
-      memref = builder.create<memref::ReinterpretCastOp>(
+      memref = memref::ReinterpretCastOp::create(builder,
           loc, memref,
           /*offset=*/builder.getIndexAttr(0),
           /*sizes=*/ArrayRef<OpFoldResult>{},
@@ -208,7 +208,7 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // alloc operation has to be passed to the dealloc operation. Passing
     // subviews, etc. to a dealloc operation is not allowed.
     memrefs.push_back(
-        builder.create<memref::ExtractStridedMetadataOp>(loc, memref)
+        memref::ExtractStridedMetadataOp::create(builder, loc, memref)
             .getResult(0));
     conditions.push_back(ownership.getIndicator());
   }
@@ -297,7 +297,7 @@ FailureOr<Operation *> deallocation_impl::insertDeallocOpForReturnLike(
   if (memrefs.empty() && toRetain.empty())
     return op;
 
-  auto deallocOp = builder.create<bufferization::DeallocOp>(
+  auto deallocOp = bufferization::DeallocOp::create(builder,
       op->getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 8f17a82fabe03..05a1367a6b126 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -170,7 +170,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   if (llvm::isa<RankedTensorType>(shapedValue.getType())) {
     tensor = shapedValue;
   } else if (llvm::isa<MemRefType>(shapedValue.getType())) {
-    tensor = b.create<ToTensorOp>(
+    tensor = ToTensorOp::create(b,
         loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
         shapedValue);
   } else if (llvm::isa<UnrankedTensorType>(shapedValue.getType()) ||
@@ -209,7 +209,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   }
 
   // Create AllocTensorOp.
-  auto allocTensorOp = b.create<AllocTensorOp>(loc, tensorType, dynamicSizes,
+  auto allocTensorOp = AllocTensorOp::create(b, loc, tensorType, dynamicSizes,
                                                copy ? tensor : Value());
 
   // Add 'memory_space' attribute. Not needed if 'copy' operand is specified.
@@ -753,7 +753,7 @@ void bufferization::replaceOpWithBufferizedValues(RewriterBase &rewriter,
       // ToTensorOp. Throughout bufferization, this ToTensorOp will gradually
       // loose all of its users and eventually DCE away.
       rewriter.setInsertionPointAfter(op);
-      replacement = rewriter.create<bufferization::ToTensorOp>(
+      replacement = bufferization::ToTensorOp::create(rewriter,
           replacement.getLoc(), opResult.getType(), replacement);
     }
     replacements.push_back(replacement);
@@ -779,7 +779,7 @@ FailureOr<Value> BufferizationOptions::createAlloc(OpBuilder &b, Location loc,
         .create<memref::AllocOp>(loc, type, dynShape,
                                  b.getI64IntegerAttr(bufferAlignment))
         .getResult();
-  return b.create<memref::AllocOp>(loc, type, dynShape).getResult();
+  return memref::AllocOp::create(b, loc, type, dynShape).getResult();
 }
 
 /// Create a memory copy between two memref buffers.
@@ -788,7 +788,7 @@ LogicalResult BufferizationOptions::createMemCpy(OpBuilder &b, Location loc,
   if (memCpyFn)
     return (*memCpyFn)(b, loc, from, to);
 
-  b.create<memref::CopyOp>(loc, from, to);
+  memref::CopyOp::create(b, loc, from, to);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 5c1d42db18c47..4308f46bee6ff 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -59,7 +59,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
   // a fix extra conditions in `isGuaranteedCastCompatible`.
   if (memref::CastOp::areCastCompatible(srcType, destType) &&
       isGuaranteedCastCompatible(srcType, destType)) {
-    Value casted = b.create<memref::CastOp>(value.getLoc(), destType, value);
+    Value casted = memref::CastOp::create(b, value.getLoc(), destType, value);
     return casted;
   }
 
@@ -68,7 +68,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
   for (int i = 0; i < destType.getRank(); ++i) {
     if (destType.getShape()[i] != ShapedType::kDynamic)
       continue;
-    Value size = b.create<memref::DimOp>(loc, value, i);
+    Value size = memref::DimOp::create(b, loc, value, i);
     dynamicOperands.push_back(size);
   }
 
@@ -135,10 +135,10 @@ void mlir::bufferization::populateDynamicDimSizes(
   for (int64_t i = 0; i < shapedType.getRank(); ++i) {
     if (shapedType.isDynamicDim(i)) {
       if (llvm::isa<MemRefType>(shapedType)) {
-        dynamicDims.push_back(b.create<memref::DimOp>(loc, shapedValue, i));
+        dynamicDims.push_back(memref::DimOp::create(b, loc, shapedValue, i));
       } else {
         assert(llvm::isa<RankedTensorType>(shapedType) && "expected tensor");
-        dynamicDims.push_back(b.create<tensor::DimOp>(loc, shapedValue, i));
+        dynamicDims.push_back(tensor::DimOp::create(b, loc, shapedValue, i));
       }
     }
   }
@@ -322,7 +322,7 @@ struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
         newShape, op.getType().getElementType(), op.getType().getEncoding());
     if (newType == op.getType())
       return failure();
-    auto newOp = rewriter.create<AllocTensorOp>(
+    auto newOp = AllocTensorOp::create(rewriter,
         op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
@@ -428,7 +428,7 @@ void AllocTensorOp::print(OpAsmPrinter &p) {
 Value AllocTensorOp::getDynamicSize(OpBuilder &b, unsigned idx) {
   assert(isDynamicDim(idx) && "expected dynamic dim");
   if (getCopy())
-    return b.create<tensor::DimOp>(getLoc(), getCopy(), idx);
+    return tensor::DimOp::create(b, getLoc(), getCopy(), idx);
   return getOperand(getIndexOfDynamicSize(idx));
 }
 
@@ -514,7 +514,7 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
     }
 
     if (source.getType() != cloneOp.getType())
-      source = rewriter.create<memref::CastOp>(cloneOp.getLoc(),
+      source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
                                                cloneOp.getType(), source);
     rewriter.replaceOp(cloneOp, source);
     rewriter.eraseOp(redundantDealloc);
@@ -539,7 +539,7 @@ LogicalResult DeallocTensorOp::bufferize(RewriterBase &rewriter,
   FailureOr<Value> buffer = getBuffer(rewriter, getTensor(), options, state);
   if (failed(buffer))
     return failure();
-  rewriter.create<memref::DeallocOp>(getLoc(), *buffer);
+  memref::DeallocOp::create(rewriter, getLoc(), *buffer);
   rewriter.eraseOp(getOperation());
   return success();
 }
@@ -644,7 +644,7 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return builder.create<ToTensorOp>(
+  return ToTensorOp::create(builder,
       loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
       /*restrict=*/true, getWritable());
 }
@@ -807,7 +807,7 @@ struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
       return failure();
     auto memrefType = MemRefType::get(srcTensorType.getShape(),
                                       srcTensorType.getElementType());
-    Value memref = rewriter.create<ToBufferOp>(toBuffer.getLoc(), memrefType,
+    Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
                                                tensorCastOperand.getOperand());
     rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                 memref);
@@ -881,12 +881,12 @@ LogicalResult ToBufferOp::bufferize(RewriterBase &rewriter,
 
 std::optional<Operation *> CloneOp::buildDealloc(OpBuilder &builder,
                                                  Value alloc) {
-  return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
+  return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
       .getOperation();
 }
 
 std::optional<Value> CloneOp::buildClone(OpBuilder &builder, Value alloc) {
-  return builder.create<CloneOp>(alloc.getLoc(), alloc).getResult();
+  return CloneOp::create(builder, alloc.getLoc(), alloc).getResult();
 }
 
 //===----------------------------------------------------------------------===//
@@ -960,7 +960,7 @@ struct DeallocRemoveDuplicateDeallocMemrefs
         Value &newCond = newConditions[memrefToCondition[memref]];
         if (newCond != cond)
           newCond =
-              rewriter.create<arith::OrIOp>(deallocOp.getLoc(), newCond, cond);
+              arith::OrIOp::create(rewriter, deallocOp.getLoc(), newCond, cond);
       } else {
         memrefToCondition.insert({memref, newConditions.size()});
         newMemrefs.push_back(memref);
@@ -1015,7 +1015,7 @@ struct DeallocRemoveDuplicateRetainedMemrefs
     // We need to create a new op because the number of results is always the
     // same as the number of condition operands.
     auto newDeallocOp =
-        rewriter.create<DeallocOp>(deallocOp.getLoc(), deallocOp.getMemrefs(),
+        DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
                                    deallocOp.getConditions(), newRetained);
     SmallVector<Value> replacements(
         llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
@@ -1037,7 +1037,7 @@ struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     if (deallocOp.getMemrefs().empty()) {
-      Value constFalse = rewriter.create<arith::ConstantOp>(
+      Value constFalse = arith::ConstantOp::create(rewriter,
           deallocOp.getLoc(), rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
diff --git a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
index db1eb20512033..7f495b0ac164c 100644
--- a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
@@ -70,12 +70,12 @@ transform::OneShotBufferizeOp::apply(transform::TransformRewriter &rewriter,
         *getFunctionBoundaryTypeConversion());
   if (getMemcpyOp() == "memref.copy") {
     options.memCpyFn = [](OpBuilder &b, Location loc, Value from, Value to) {
-      b.create<memref::CopyOp>(loc, from, to);
+      memref::CopyOp::create(b, loc, from, to);
       return success();
     };
   } else if (getMemcpyOp() == "linalg.copy") {
     options.memCpyFn = [](OpBuilder &b, Location loc, Value from, Value to) {
-      b.create<linalg::CopyOp>(loc, from, to);
+      linalg::CopyOp::create(b, loc, from, to);
       return success();
     };
   } else {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index c5fab80ecaa08..835ad61f54da2 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -167,7 +167,7 @@ struct RemoveDeallocMemrefsContainedInRetained
       std::optional<bool> analysisResult =
           analysis.isSameAllocation(retained, memref);
       if (analysisResult == true) {
-        auto disjunction = rewriter.create<arith::OrIOp>(
+        auto disjunction = arith::OrIOp::create(rewriter,
             deallocOp.getLoc(), updatedCondition, cond);
         rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(),
                                       disjunction);
@@ -247,14 +247,14 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias
         continue;
       }
 
-      replacements.push_back(rewriter.create<arith::ConstantOp>(
+      replacements.push_back(arith::ConstantOp::create(rewriter,
           deallocOp.getLoc(), rewriter.getBoolAttr(false)));
     }
 
     if (newRetainedMemrefs.size() == deallocOp.getRetained().size())
       return failure();
 
-    auto newDeallocOp = rewriter.create<DeallocOp>(
+    auto newDeallocOp = DeallocOp::create(rewriter,
         deallocOp.getLoc(), deallocOp.getMemrefs(), deallocOp.getConditions(),
         newRetainedMemrefs);
     int i = 0;
@@ -326,7 +326,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       }
 
       // Create new bufferization.dealloc op for `memref`.
-      auto newDeallocOp = rewriter.create<DeallocOp>(loc, memref, cond,
+      auto newDeallocOp = DeallocOp::create(rewriter, loc, memref, cond,
                                                      deallocOp.getRetained());
       updatedConditions.push_back(
           llvm::to_vector(ValueRange(newDeallocOp.getUpdatedConditions())));
@@ -337,7 +337,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       return failure();
 
     // Create bufferization.dealloc op for all remaining memrefs.
-    auto newDeallocOp = rewriter.create<DeallocOp>(
+    auto newDeallocOp = DeallocOp::create(rewriter,
         loc, remainingMemrefs, remainingConditions, deallocOp.getRetained());
 
     // Bit-or all conditions.
@@ -347,7 +347,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       assert(replacements.size() == additionalConditions.size() &&
              "expected same number of updated conditions");
       for (int64_t i = 0, e = replacements.size(); i < e; ++i) {
-        replacements[i] = rewriter.create<arith::OrIOp>(
+        replacements[i] = arith::OrIOp::create(rewriter,
             loc, replacements[i], additionalConditions[i]);
       }
     }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index 15e03fbefe9c5..9f11f23a5dfeb 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -133,7 +133,7 @@ updateReturnOps(func::FuncOp func, ArrayRef<BlockArgument> appendedEntryArgs,
           return WalkResult::interrupt();
       }
     }
-    builder.create<func::ReturnOp>(op.getLoc(), keepAsReturnOperands);
+    func::ReturnOp::create(builder, op.getLoc(), keepAsReturnOperands);
     op.erase();
     return WalkResult::advance();
   });
@@ -191,7 +191,7 @@ updateCalls(ModuleOp module,
         assert(hasFullyDynamicLayoutMap(memrefType) &&
                "layout map not supported");
         outParam =
-            builder.create<memref::CastOp>(op.getLoc(), memrefType, outParam);
+            memref::CastOp::create(builder, op.getLoc(), memrefType, outParam);
       }
       memref.replaceAllUsesWith(outParam);
       outParams.push_back(outParam);
@@ -201,7 +201,7 @@ updateCalls(ModuleOp module,
     newOperands.append(outParams.begin(), outParams.end());
     auto newResultTypes = llvm::to_vector<6>(llvm::map_range(
         replaceWithNewCallResults, [](Value v) { return v.getType(); }));
-    auto newCall = builder.create<func::CallOp>(op.getLoc(), op.getCalleeAttr(),
+    auto newCall = func::CallOp::create(builder, op.getLoc(), op.getCalleeAttr(),
                                                 newResultTypes, newOperands);
     for (auto t : llvm::zip(replaceWithNewCallResults, newCall.getResults()))
       std::get<0>(t).replaceAllUsesWith(std::get<1>(t));
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
index ff2c83d228dbb..1e02d9bfa7ef6 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
@@ -146,7 +146,7 @@ bufferization::getGlobalFor(arith::ConstantOp constantOp,
       cast<MemRefType>(getMemRefTypeWithStaticIdentityLayout(type));
   if (memorySpace)
     memrefType = MemRefType::Builder(memrefType).setMemorySpace(memorySpace);
-  auto global = globalBuilder.create<memref::GlobalOp>(
+  auto global = memref::GlobalOp::create(globalBuilder,
       constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
       /*sym_visibility=*/globalBuilder.getStringAttr("private"),
       /*type=*/memrefType,
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 6472ef3eff2ac..c7343c62e2cc8 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -436,7 +436,7 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
     // Replace all uses of the original tensor bbArg.
     rewriter.setInsertionPointToStart(block);
     if (!bbArgUses.empty()) {
-      Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+      Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
           bbArg.getLoc(), tensorType, bbArg);
       for (OpOperand *use : bbArgUses)
         use->set(toTensorOp);
@@ -468,12 +468,12 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
       if (failed(operandBufferType))
         return failure();
       rewriter.setInsertionPointAfterValue(operand);
-      Value bufferizedOperand = rewriter.create<bufferization::ToBufferOp>(
+      Value bufferizedOperand = bufferization::ToBufferOp::create(rewriter,
           operand.getLoc(), *operandBufferType, operand);
       // A cast is needed if the operand and the block argument have different
       // bufferized types.
       if (type != *operandBufferType)
-        bufferizedOperand = rewriter.create<memref::CastOp>(
+        bufferizedOperand = memref::CastOp::create(rewriter,
             operand.getLoc(), type, bufferizedOperand);
       newOperands.push_back(bufferizedOperand);
     }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index ae011904cb972..9014b8b8ecdf0 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -120,7 +120,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
     // Update function calls.
     for (func::CallOp callOp : callerMap[funcOp]) {
       rewriter.setInsertionPoint(callOp);
-      auto newCallOp = rewriter.create<func::CallOp>(callOp.getLoc(), funcOp,
+      auto newCallOp = func::CallOp::create(rewriter, callOp.getLoc(), funcOp,
                                                      callOp.getOperands());
       SmallVector<Value> newResults;
       int64_t nextResult = 0;
@@ -136,7 +136,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
         Type expectedType = callOp.getResult(i).getType();
         if (replacement.getType() != expectedType) {
           // A cast must be inserted at the call site.
-          replacement = rewriter.create<memref::CastOp>(
+          replacement = memref::CastOp::create(rewriter,
               callOp.getLoc(), expectedType, replacement);
         }
         newResults.push_back(replacement);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
index 6f27563a45548..202b9eb0c69dc 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
@@ -169,7 +169,7 @@ LogicalResult mlir::bufferization::eliminateEmptyTensors(
             cast<ShapedType>(v.getType()).getElementType())
           continue;
         rewriter.setInsertionPointAfterValue(replacement);
-        replacement = rewriter.create<tensor::CastOp>(v.getLoc(), v.getType(),
+        replacement = tensor::CastOp::create(rewriter, v.getLoc(), v.getType(),
                                                       replacement);
       }
       // Replace the specific use of the tensor::EmptyOp.
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index bd2aebca68079..4d25ca87da888 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -320,7 +320,7 @@ struct CallOpInterface
     }
 
     // 3. Create the new CallOp.
-    Operation *newCallOp = rewriter.create<func::CallOp>(
+    Operation *newCallOp = func::CallOp::create(rewriter,
         callOp.getLoc(), funcOp.getSymName(), resultTypes, newOperands);
     newCallOp->setAttrs(callOp->getAttrs());
 
@@ -484,7 +484,7 @@ struct FuncOpInterface
 
         // Note: If `inferFunctionResultLayout = true`, casts are later folded
         // away.
-        Value toBufferOp = rewriter.create<bufferization::ToBufferOp>(
+        Value toBufferOp = bufferization::ToBufferOp::create(rewriter,
             returnOp.getLoc(), bufferizedType, returnVal);
         returnValues.push_back(toBufferOp);
       }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
index 2a17ae4f6a249..304c073a26c73 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
@@ -65,8 +65,8 @@ class DeallocOpConversion
 
     rewriter.replaceOpWithNewOp<scf::IfOp>(
         op, adaptor.getConditions()[0], [&](OpBuilder &builder, Location loc) {
-          builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[0]);
-          builder.create<scf::YieldOp>(loc);
+          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
+          scf::YieldOp::create(builder, loc);
         });
     return success();
   }
@@ -109,13 +109,13 @@ class DeallocOpConversion
     // Compute the base pointer indices, compare all retained indices to the
     // memref index to check if they alias.
     SmallVector<Value> doesNotAliasList;
-    Value memrefAsIdx = rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(
+    Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(rewriter,
         op->getLoc(), adaptor.getMemrefs()[0]);
     for (Value retained : adaptor.getRetained()) {
       Value retainedAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op->getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op->getLoc(),
                                                                   retained);
-      Value doesNotAlias = rewriter.create<arith::CmpIOp>(
+      Value doesNotAlias = arith::CmpIOp::create(rewriter,
           op->getLoc(), arith::CmpIPredicate::ne, memrefAsIdx, retainedAsIdx);
       doesNotAliasList.push_back(doesNotAlias);
     }
@@ -123,17 +123,17 @@ class DeallocOpConversion
     // AND-reduce the list of booleans from above.
     Value prev = doesNotAliasList.front();
     for (Value doesNotAlias : ArrayRef(doesNotAliasList).drop_front())
-      prev = rewriter.create<arith::AndIOp>(op->getLoc(), prev, doesNotAlias);
+      prev = arith::AndIOp::create(rewriter, op->getLoc(), prev, doesNotAlias);
 
     // Also consider the condition given by the dealloc operation and perform a
     // conditional deallocation guarded by that value.
-    Value shouldDealloc = rewriter.create<arith::AndIOp>(
+    Value shouldDealloc = arith::AndIOp::create(rewriter,
         op->getLoc(), prev, adaptor.getConditions()[0]);
 
-    rewriter.create<scf::IfOp>(
+    scf::IfOp::create(rewriter,
         op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-          builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[0]);
-          builder.create<scf::YieldOp>(loc);
+          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
+          scf::YieldOp::create(builder, loc);
         });
 
     // Compute the replacement values for the dealloc operation results. This
@@ -141,12 +141,12 @@ class DeallocOpConversion
     // `select(does_alias_with_memref(r), memref_cond, false)` for each retained
     // value r.
     SmallVector<Value> replacements;
-    Value trueVal = rewriter.create<arith::ConstantOp>(
+    Value trueVal = arith::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getBoolAttr(true));
     for (Value doesNotAlias : doesNotAliasList) {
       Value aliases =
-          rewriter.create<arith::XOrIOp>(op->getLoc(), doesNotAlias, trueVal);
-      Value result = rewriter.create<arith::AndIOp>(op->getLoc(), aliases,
+          arith::XOrIOp::create(rewriter, op->getLoc(), doesNotAlias, trueVal);
+      Value result = arith::AndIOp::create(rewriter, op->getLoc(), aliases,
                                                     adaptor.getConditions()[0]);
       replacements.push_back(result);
     }
@@ -231,18 +231,18 @@ class DeallocOpConversion
     // Without storing them to memrefs, we could not use for-loops but only a
     // completely unrolled version of it, potentially leading to code-size
     // blow-up.
-    Value toDeallocMemref = rewriter.create<memref::AllocOp>(
+    Value toDeallocMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
                                      rewriter.getIndexType()));
-    Value conditionMemref = rewriter.create<memref::AllocOp>(
+    Value conditionMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getConditions().size()},
                                      rewriter.getI1Type()));
-    Value toRetainMemref = rewriter.create<memref::AllocOp>(
+    Value toRetainMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
                                      rewriter.getIndexType()));
 
     auto getConstValue = [&](uint64_t value) -> Value {
-      return rewriter.create<arith::ConstantOp>(op.getLoc(),
+      return arith::ConstantOp::create(rewriter, op.getLoc(),
                                                 rewriter.getIndexAttr(value));
     };
 
@@ -250,58 +250,58 @@ class DeallocOpConversion
     // at runtime.
     for (auto [i, toDealloc] : llvm::enumerate(adaptor.getMemrefs())) {
       Value memrefAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op.getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
                                                                   toDealloc);
-      rewriter.create<memref::StoreOp>(op.getLoc(), memrefAsIdx,
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx,
                                        toDeallocMemref, getConstValue(i));
     }
 
     for (auto [i, cond] : llvm::enumerate(adaptor.getConditions()))
-      rewriter.create<memref::StoreOp>(op.getLoc(), cond, conditionMemref,
+      memref::StoreOp::create(rewriter, op.getLoc(), cond, conditionMemref,
                                        getConstValue(i));
 
     for (auto [i, toRetain] : llvm::enumerate(adaptor.getRetained())) {
       Value memrefAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op.getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
                                                                   toRetain);
-      rewriter.create<memref::StoreOp>(op.getLoc(), memrefAsIdx, toRetainMemref,
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx, toRetainMemref,
                                        getConstValue(i));
     }
 
     // Cast the allocated memrefs to dynamic shape because we want only one
     // helper function no matter how many operands the bufferization.dealloc
     // has.
-    Value castedDeallocMemref = rewriter.create<memref::CastOp>(
+    Value castedDeallocMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toDeallocMemref);
-    Value castedCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         conditionMemref);
-    Value castedRetainMemref = rewriter.create<memref::CastOp>(
+    Value castedRetainMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toRetainMemref);
 
-    Value deallocCondsMemref = rewriter.create<memref::AllocOp>(
+    Value deallocCondsMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
                                      rewriter.getI1Type()));
-    Value retainCondsMemref = rewriter.create<memref::AllocOp>(
+    Value retainCondsMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
                                      rewriter.getI1Type()));
 
-    Value castedDeallocCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedDeallocCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         deallocCondsMemref);
-    Value castedRetainCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedRetainCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         retainCondsMemref);
 
     Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>();
-    rewriter.create<func::CallOp>(
+    func::CallOp::create(rewriter,
         op.getLoc(), deallocHelperFuncMap.lookup(symtableOp),
         SmallVector<Value>{castedDeallocMemref, castedRetainMemref,
                            castedCondsMemref, castedDeallocCondsMemref,
@@ -309,30 +309,30 @@ class DeallocOpConversion
 
     for (unsigned i = 0, e = adaptor.getMemrefs().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value shouldDealloc = rewriter.create<memref::LoadOp>(
+      Value shouldDealloc = memref::LoadOp::create(rewriter,
           op.getLoc(), deallocCondsMemref, idxValue);
-      rewriter.create<scf::IfOp>(
+      scf::IfOp::create(rewriter,
           op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-            builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[i]);
-            builder.create<scf::YieldOp>(loc);
+            memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[i]);
+            scf::YieldOp::create(builder, loc);
           });
     }
 
     SmallVector<Value> replacements;
     for (unsigned i = 0, e = adaptor.getRetained().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value ownership = rewriter.create<memref::LoadOp>(
+      Value ownership = memref::LoadOp::create(rewriter,
           op.getLoc(), retainCondsMemref, idxValue);
       replacements.push_back(ownership);
     }
 
     // Deallocate above allocated memrefs again to avoid memory leaks.
     // Deallocation will not be run on code after this stage.
-    rewriter.create<memref::DeallocOp>(op.getLoc(), toDeallocMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), toRetainMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), conditionMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), deallocCondsMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), retainCondsMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), toDeallocMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), toRetainMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), conditionMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), deallocCondsMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), retainCondsMemref);
 
     rewriter.replaceOp(op, replacements);
     return success();
@@ -350,7 +350,7 @@ class DeallocOpConversion
                   ConversionPatternRewriter &rewriter) const override {
     // Lower the trivial case.
     if (adaptor.getMemrefs().empty()) {
-      Value falseVal = rewriter.create<arith::ConstantOp>(
+      Value falseVal = arith::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           op, SmallVector<Value>(adaptor.getRetained().size(), falseVal));
@@ -450,30 +450,30 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
   Value retainCondsMemref = helperFuncOp.getArguments()[4];
 
   // Insert some prerequisites.
-  Value c0 = builder.create<arith::ConstantOp>(loc, builder.getIndexAttr(0));
-  Value c1 = builder.create<arith::ConstantOp>(loc, builder.getIndexAttr(1));
+  Value c0 = arith::ConstantOp::create(builder, loc, builder.getIndexAttr(0));
+  Value c1 = arith::ConstantOp::create(builder, loc, builder.getIndexAttr(1));
   Value trueValue =
-      builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(true));
+      arith::ConstantOp::create(builder, loc, builder.getBoolAttr(true));
   Value falseValue =
-      builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(false));
-  Value toDeallocSize = builder.create<memref::DimOp>(loc, toDeallocMemref, c0);
-  Value toRetainSize = builder.create<memref::DimOp>(loc, toRetainMemref, c0);
+      arith::ConstantOp::create(builder, loc, builder.getBoolAttr(false));
+  Value toDeallocSize = memref::DimOp::create(builder, loc, toDeallocMemref, c0);
+  Value toRetainSize = memref::DimOp::create(builder, loc, toRetainMemref, c0);
 
-  builder.create<scf::ForOp>(
+  scf::ForOp::create(builder,
       loc, c0, toRetainSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value i, ValueRange iterArgs) {
-        builder.create<memref::StoreOp>(loc, falseValue, retainCondsMemref, i);
-        builder.create<scf::YieldOp>(loc);
+        memref::StoreOp::create(builder, loc, falseValue, retainCondsMemref, i);
+        scf::YieldOp::create(builder, loc);
       });
 
-  builder.create<scf::ForOp>(
+  scf::ForOp::create(builder,
       loc, c0, toDeallocSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value outerIter,
           ValueRange iterArgs) {
         Value toDealloc =
-            builder.create<memref::LoadOp>(loc, toDeallocMemref, outerIter);
+            memref::LoadOp::create(builder, loc, toDeallocMemref, outerIter);
         Value cond =
-            builder.create<memref::LoadOp>(loc, conditionMemref, outerIter);
+            memref::LoadOp::create(builder, loc, conditionMemref, outerIter);
 
         // Build the first for loop that computes aliasing with retained
         // memrefs.
@@ -483,31 +483,31 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, toRetainSize, c1, trueValue,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value retainValue = builder.create<memref::LoadOp>(
+                      Value retainValue = memref::LoadOp::create(builder,
                           loc, toRetainMemref, i);
-                      Value doesAlias = builder.create<arith::CmpIOp>(
+                      Value doesAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::eq, retainValue,
                           toDealloc);
-                      builder.create<scf::IfOp>(
+                      scf::IfOp::create(builder,
                           loc, doesAlias,
                           [&](OpBuilder &builder, Location loc) {
                             Value retainCondValue =
-                                builder.create<memref::LoadOp>(
+                                memref::LoadOp::create(builder,
                                     loc, retainCondsMemref, i);
                             Value aggregatedRetainCond =
-                                builder.create<arith::OrIOp>(
+                                arith::OrIOp::create(builder,
                                     loc, retainCondValue, cond);
-                            builder.create<memref::StoreOp>(
+                            memref::StoreOp::create(builder,
                                 loc, aggregatedRetainCond, retainCondsMemref,
                                 i);
-                            builder.create<scf::YieldOp>(loc);
+                            scf::YieldOp::create(builder, loc);
                           });
-                      Value doesntAlias = builder.create<arith::CmpIOp>(
+                      Value doesntAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::ne, retainValue,
                           toDealloc);
-                      Value yieldValue = builder.create<arith::AndIOp>(
+                      Value yieldValue = arith::AndIOp::create(builder,
                           loc, iterArgs[0], doesntAlias);
-                      builder.create<scf::YieldOp>(loc, yieldValue);
+                      scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
@@ -519,24 +519,24 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, outerIter, c1, noRetainAlias,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value prevDeallocValue = builder.create<memref::LoadOp>(
+                      Value prevDeallocValue = memref::LoadOp::create(builder,
                           loc, toDeallocMemref, i);
-                      Value doesntAlias = builder.create<arith::CmpIOp>(
+                      Value doesntAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::ne, prevDeallocValue,
                           toDealloc);
-                      Value yieldValue = builder.create<arith::AndIOp>(
+                      Value yieldValue = arith::AndIOp::create(builder,
                           loc, iterArgs[0], doesntAlias);
-                      builder.create<scf::YieldOp>(loc, yieldValue);
+                      scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
-        Value shouldDealoc = builder.create<arith::AndIOp>(loc, noAlias, cond);
-        builder.create<memref::StoreOp>(loc, shouldDealoc, deallocCondsMemref,
+        Value shouldDealoc = arith::AndIOp::create(builder, loc, noAlias, cond);
+        memref::StoreOp::create(builder, loc, shouldDealoc, deallocCondsMemref,
                                         outerIter);
-        builder.create<scf::YieldOp>(loc);
+        scf::YieldOp::create(builder, loc);
       });
 
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
   return helperFuncOp;
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index 1eeafc4df8cf1..71890bfea4e5d 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -43,7 +43,7 @@ using namespace mlir::bufferization;
 //===----------------------------------------------------------------------===//
 
 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
-  return builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
 }
 
 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
@@ -755,12 +755,12 @@ Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
           .create<scf::IfOp>(
               memref.getLoc(), condition,
               [&](OpBuilder &builder, Location loc) {
-                builder.create<scf::YieldOp>(loc, newMemref);
+                scf::YieldOp::create(builder, loc, newMemref);
               },
               [&](OpBuilder &builder, Location loc) {
                 Value clone =
-                    builder.create<bufferization::CloneOp>(loc, newMemref);
-                builder.create<scf::YieldOp>(loc, clone);
+                    bufferization::CloneOp::create(builder, loc, newMemref);
+                scf::YieldOp::create(builder, loc, clone);
               })
           .getResult(0);
   Value trueVal = buildBoolValue(builder, memref.getLoc(), true);
@@ -797,7 +797,7 @@ BufferDeallocation::handleInterface(BranchOpInterface op) {
   state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
                            toRetain);
 
-  auto deallocOp = builder.create<bufferization::DeallocOp>(
+  auto deallocOp = bufferization::DeallocOp::create(builder,
       op.getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
@@ -885,10 +885,10 @@ BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
       builder.setInsertionPoint(op);
       Ownership ownership = state.getOwnership(operand, block);
       if (ownership.isUnique()) {
-        Value ownershipInverted = builder.create<arith::XOrIOp>(
+        Value ownershipInverted = arith::XOrIOp::create(builder,
             op.getLoc(), ownership.getIndicator(),
             buildBoolValue(builder, op.getLoc(), true));
-        builder.create<cf::AssertOp>(
+        cf::AssertOp::create(builder,
             op.getLoc(), ownershipInverted,
             "expected that the block does not have ownership");
       }
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
index 0bdcf434e062f..b679bbe05cb2f 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
@@ -49,7 +49,7 @@ Operation *complex::ComplexDialect::materializeConstant(OpBuilder &builder,
                                                         Type type,
                                                         Location loc) {
   if (complex::ConstantOp::isBuildableWith(value, type)) {
-    return builder.create<complex::ConstantOp>(loc, type,
+    return complex::ConstantOp::create(builder, loc, type,
                                                llvm::cast<ArrayAttr>(value));
   }
   return arith::ConstantOp::materialize(builder, value, type, loc);
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 0c11c76cf1f71..38e47cc5e3714 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -312,7 +312,7 @@ struct SimplifyCondBranchIdenticalSuccessors
       if (std::get<0>(it) == std::get<1>(it))
         mergedOperands.push_back(std::get<0>(it));
       else
-        mergedOperands.push_back(rewriter.create<arith::SelectOp>(
+        mergedOperands.push_back(arith::SelectOp::create(rewriter,
             condbr.getLoc(), condition, std::get<0>(it), std::get<1>(it)));
     }
 
@@ -412,7 +412,7 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantTrue)
-            constantTrue = rewriter.create<arith::ConstantOp>(
+            constantTrue = arith::ConstantOp::create(rewriter,
                 condbr.getLoc(), ty, rewriter.getBoolAttr(true));
 
           rewriter.modifyOpInPlace(use.getOwner(),
@@ -427,7 +427,7 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantFalse)
-            constantFalse = rewriter.create<arith::ConstantOp>(
+            constantFalse = arith::ConstantOp::create(rewriter,
                 condbr.getLoc(), ty, rewriter.getBoolAttr(false));
 
           rewriter.modifyOpInPlace(use.getOwner(),
diff --git a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index a077f56f4f472..23f50930cb3b3 100644
--- a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -87,7 +87,7 @@ struct CondBranchOpInterface
                                destOperands.getAsOperandRange(), toRetain);
       SmallVector<Value> adaptedConditions(
           llvm::map_range(conditions, conditionModifier));
-      auto deallocOp = builder.create<bufferization::DeallocOp>(
+      auto deallocOp = bufferization::DeallocOp::create(builder,
           condBr.getLoc(), memrefs, adaptedConditions, toRetain);
       state.resetOwnerships(deallocOp.getRetained(), condBr->getBlock());
       for (auto [retained, ownership] : llvm::zip(
@@ -115,18 +115,18 @@ struct CondBranchOpInterface
     DeallocOp thenTakenDeallocOp = insertDeallocForBranch(
         condBr.getTrueDest(), condBr.getTrueDestOperandsMutable(),
         [&](Value cond) {
-          return builder.create<arith::AndIOp>(condBr.getLoc(), cond,
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond,
                                                condBr.getCondition());
         },
         thenMapping);
     DeallocOp elseTakenDeallocOp = insertDeallocForBranch(
         condBr.getFalseDest(), condBr.getFalseDestOperandsMutable(),
         [&](Value cond) {
-          Value trueVal = builder.create<arith::ConstantOp>(
+          Value trueVal = arith::ConstantOp::create(builder,
               condBr.getLoc(), builder.getBoolAttr(true));
-          Value negation = builder.create<arith::XOrIOp>(
+          Value negation = arith::XOrIOp::create(builder,
               condBr.getLoc(), trueVal, condBr.getCondition());
-          return builder.create<arith::AndIOp>(condBr.getLoc(), cond, negation);
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond, negation);
         },
         elseMapping);
 
@@ -143,7 +143,7 @@ struct CondBranchOpInterface
 
     for (Value retained : commonValues) {
       state.resetOwnerships(retained, condBr->getBlock());
-      Value combinedOwnership = builder.create<arith::SelectOp>(
+      Value combinedOwnership = arith::SelectOp::create(builder,
           condBr.getLoc(), condBr.getCondition(), thenMapping[retained],
           elseMapping[retained]);
       state.updateOwnership(retained, combinedOwnership, condBr->getBlock());
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index 27298e892e599..f9852257a4a7c 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -50,13 +50,13 @@ void EmitCDialect::initialize() {
 Operation *EmitCDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
-  return builder.create<emitc::ConstantOp>(loc, type, value);
+  return emitc::ConstantOp::create(builder, loc, type, value);
 }
 
 /// Default callback for builders of ops carrying a region. Inserts a yield
 /// without arguments.
 void mlir::emitc::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<emitc::YieldOp>(loc);
+  emitc::YieldOp::create(builder, loc);
 }
 
 bool mlir::emitc::isSupportedEmitCType(Type type) {
diff --git a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
index a578a86b499a6..7e04b4b196a6d 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
@@ -25,7 +25,7 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
   Location loc = op->getLoc();
 
   builder.setInsertionPointAfter(op);
-  auto expressionOp = builder.create<emitc::ExpressionOp>(loc, resultType);
+  auto expressionOp = emitc::ExpressionOp::create(builder, loc, resultType);
 
   // Replace all op's uses with the new expression's result.
   result.replaceAllUsesWith(expressionOp.getResult());
@@ -34,7 +34,7 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
   Region &region = expressionOp.getRegion();
   Block &block = region.emplaceBlock();
   builder.setInsertionPointToEnd(&block);
-  auto yieldOp = builder.create<emitc::YieldOp>(loc, result);
+  auto yieldOp = emitc::YieldOp::create(builder, loc, result);
 
   // Move op into the new expression.
   op->moveBefore(yieldOp);
diff --git a/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp b/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
index 72c8fd0f32485..ab7be8d6cedd9 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
@@ -21,7 +21,7 @@ Value materializeAsUnrealizedCast(OpBuilder &builder, Type resultType,
   if (inputs.size() != 1)
     return Value();
 
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
       .getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
index 17d436f6df028..72c331ad96607 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
@@ -50,7 +50,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
                                 PatternRewriter &rewriter) const override {
 
     auto className = funcOp.getSymNameAttr().str() + "Class";
-    ClassOp newClassOp = rewriter.create<ClassOp>(funcOp.getLoc(), className);
+    ClassOp newClassOp = ClassOp::create(rewriter, funcOp.getLoc(), className);
 
     SmallVector<std::pair<StringAttr, TypeAttr>> fields;
     rewriter.createBlock(&newClassOp.getBody());
@@ -67,7 +67,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
 
       TypeAttr typeAttr = TypeAttr::get(val.getType());
       fields.push_back({fieldName, typeAttr});
-      rewriter.create<emitc::FieldOp>(funcOp.getLoc(), fieldName, typeAttr,
+      emitc::FieldOp::create(rewriter, funcOp.getLoc(), fieldName, typeAttr,
                                       argAttr);
     }
 
@@ -75,7 +75,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
     FunctionType funcType = funcOp.getFunctionType();
     Location loc = funcOp.getLoc();
     FuncOp newFuncOp =
-        rewriter.create<emitc::FuncOp>(loc, ("execute"), funcType);
+        emitc::FuncOp::create(rewriter, loc, ("execute"), funcType);
 
     rewriter.createBlock(&newFuncOp.getBody());
     newFuncOp.getBody().takeBody(funcOp.getBody());
@@ -85,7 +85,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
     newArguments.reserve(fields.size());
     for (auto &[fieldName, attr] : fields) {
       GetFieldOp arg =
-          rewriter.create<emitc::GetFieldOp>(loc, attr.getValue(), fieldName);
+          emitc::GetFieldOp::create(rewriter, loc, attr.getValue(), fieldName);
       newArguments.push_back(arg);
     }
 
diff --git a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
index 3328d58551bff..7485ba40b4344 100644
--- a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
+++ b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
@@ -61,7 +61,7 @@ struct FuncInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    builder.create<cf::BranchOp>(op->getLoc(), newDest, returnOp.getOperands());
+    cf::BranchOp::create(builder, op->getLoc(), newDest, returnOp.getOperands());
     op->erase();
   }
 
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index 9d317f20521fb..1c0d3cd338006 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -54,7 +54,7 @@ void FuncDialect::initialize() {
 Operation *FuncDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                             Type type, Location loc) {
   if (ConstantOp::isBuildableWith(value, type))
-    return builder.create<ConstantOp>(loc, type,
+    return ConstantOp::create(builder, loc, type,
                                       llvm::cast<FlatSymbolRefAttr>(value));
   return nullptr;
 }
diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
index 3adbf092742be..2ece1bcffb1b0 100644
--- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
+++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
@@ -171,7 +171,7 @@ transform::CastAndCallOp::apply(transform::TransformRewriter &rewriter,
     }
   }
 
-  auto callOp = rewriter.create<func::CallOp>(insertionPoint->getLoc(),
+  auto callOp = func::CallOp::create(rewriter, insertionPoint->getLoc(),
                                               targetFunction, inputs);
 
   // Cast the call results back to the expected types. If any conversions fail
diff --git a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
index a3638c8766a5c..4b70cf172b407 100644
--- a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
@@ -46,7 +46,7 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 
     // Substitute with the new result types from the corresponding FuncType
     // conversion.
-    auto newCallOp = rewriter.create<CallOp>(
+    auto newCallOp = CallOp::create(rewriter,
         callOp.getLoc(), callOp.getCallee(), convertedResults,
         flattenValues(adaptor.getOperands()));
     SmallVector<ValueRange> replacements;
diff --git a/mlir/lib/Dialect/Func/Utils/Utils.cpp b/mlir/lib/Dialect/Func/Utils/Utils.cpp
index 0e9662689ef78..1eba777921828 100644
--- a/mlir/lib/Dialect/Func/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Func/Utils/Utils.cpp
@@ -44,7 +44,7 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
   for (unsigned int idx : newResultsOrder)
     newOutputTypes.push_back(origOutputTypes[idx]);
   rewriter.setInsertionPoint(funcOp);
-  auto newFuncOp = rewriter.create<func::FuncOp>(
+  auto newFuncOp = func::FuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(newInputTypes, newOutputTypes));
 
@@ -80,7 +80,7 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
     newReturnValues.push_back(returnOp.getOperand(idx));
   rewriter.setInsertionPoint(returnOp);
   auto newReturnOp =
-      rewriter.create<func::ReturnOp>(newFuncOp.getLoc(), newReturnValues);
+      func::ReturnOp::create(rewriter, newFuncOp.getLoc(), newReturnValues);
   newReturnOp->setDiscardableAttrs(returnOp->getDiscardableAttrDictionary());
   rewriter.eraseOp(returnOp);
 
@@ -109,7 +109,7 @@ func::replaceCallOpWithNewOrder(RewriterBase &rewriter, func::CallOp callOp,
   // Replace the kernel call operation with a new one that has the
   // reordered arguments.
   rewriter.setInsertionPoint(callOp);
-  auto newCallOp = rewriter.create<func::CallOp>(
+  auto newCallOp = func::CallOp::create(rewriter,
       callOp.getLoc(), callOp.getCallee(), newResultTypes, newArgsOrderValues);
   newCallOp.setNoInlineAttr(callOp.getNoInlineAttr());
   for (auto &&[newIndex, origIndex] : llvm::enumerate(newResultsOrder))
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 30b5ac9809139..f7eb41be287e4 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -1137,7 +1137,7 @@ struct FoldLaunchArguments : public OpRewritePattern<LaunchOp> {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(&op.getBody().front());
         zero =
-            rewriter.create<arith::ConstantIndexOp>(op.getLoc(), /*value=*/0);
+            arith::ConstantIndexOp::create(rewriter, op.getLoc(), /*value=*/0);
       }
       rewriter.replaceAllUsesWith(id, zero);
       simplified = true;
@@ -1381,9 +1381,9 @@ static void printLaunchFuncOperands(OpAsmPrinter &printer, Operation *,
 void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
                       int32_t offset, int32_t width, ShuffleMode mode) {
   build(builder, result, value,
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(offset)),
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(width)),
         mode);
 }
@@ -1395,9 +1395,9 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
 void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
                      int32_t offset, int32_t width) {
   build(builder, result, value,
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(offset)),
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(width)));
 }
 
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index c9e91535df946..2035f371694a7 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -560,7 +560,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   Value predicate;
   if (originalBasisWasProvided) {
     for (Value tmpPredicate : builderResult.predicateOps) {
-      predicate = predicate ? rewriter.create<arith::AndIOp>(loc, predicate,
+      predicate = predicate ? arith::AndIOp::create(rewriter, loc, predicate,
                                                              tmpPredicate)
                             : tmpPredicate;
     }
@@ -573,7 +573,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   Block::iterator insertionPoint;
   if (predicate) {
     // Step 6.a. If predicated, move at the beginning.
-    auto ifOp = rewriter.create<scf::IfOp>(loc, predicate,
+    auto ifOp = scf::IfOp::create(rewriter, loc, predicate,
                                            /*withElseRegion=*/false);
     targetBlock = ifOp.thenBlock();
     insertionPoint = ifOp.thenBlock()->begin();
@@ -632,7 +632,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapForallToBlocksImpl(
     // the insertion point.
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPointToStart(parentBlock);
-    zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   }
 
   ForallRewriteResult rewriteResult;
@@ -884,7 +884,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapOneForallToThreadsImpl(
     return diag;
   // Add a syncthreads if needed. TODO: warpsync
   if (syncAfterDistribute)
-    rewriter.create<BarrierOp>(loc);
+    BarrierOp::create(rewriter, loc);
 
   return DiagnosedSilenceableFailure::success();
 }
@@ -901,7 +901,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapNestedForallToThreadsImpl(
 
   // Create an early zero index value for replacements.
   Location loc = target->getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   DiagnosedSilenceableFailure diag = DiagnosedSilenceableFailure::success();
   WalkResult walkResult = target->walk([&](scf::ForallOp forallOp) {
     diag = mlir::transform::gpu::mapOneForallToThreadsImpl(
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index 12b7f39390967..fc4556df9d6f2 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -82,8 +82,8 @@ buildPredicates(RewriterBase &rewriter, Location loc, ArrayRef<Value> activeIds,
     }
     if (activeMappingSize == availableMappingSize)
       continue;
-    Value idx = rewriter.create<arith::ConstantIndexOp>(loc, activeMappingSize);
-    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value idx = arith::ConstantIndexOp::create(rewriter, loc, activeMappingSize);
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                                 activeId, idx);
     predicateOps.push_back(pred);
   }
@@ -104,11 +104,11 @@ static Value buildLinearId(RewriterBase &rewriter, Location loc,
   bindDims(rewriter.getContext(), tx, ty, tz);
   bindSymbols(rewriter.getContext(), bdx, bdy);
   SmallVector<OpFoldResult> vals{
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::x)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::x)
           .getResult(),
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::y)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::y)
           .getResult(),
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::z)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::z)
           .getResult(),
       originalBasisOfr[0], originalBasisOfr[1]};
   OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
@@ -215,9 +215,9 @@ static GpuIdBuilderFnType common3DIdBuilderFn(int64_t multiplicity = 1) {
                             ArrayRef<int64_t> originalBasis) {
     IndexType indexType = rewriter.getIndexType();
     SmallVector<Value> ids{
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::x),
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::y),
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::z)};
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::x),
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::y),
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::z)};
     // In the 3-D mapping case, scale the first dimension by the multiplicity.
     SmallVector<Value> scaledIds = ids;
     AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
@@ -417,7 +417,7 @@ DiagnosedSilenceableFailure createGpuLaunch(
     return diag;
 
   auto createConst = [&](int dim) {
-    return rewriter.create<arith::ConstantIndexOp>(loc, dim);
+    return arith::ConstantIndexOp::create(rewriter, loc, dim);
   };
   OpBuilder::InsertionGuard guard(rewriter);
   Value one = createConst(1);
@@ -427,10 +427,10 @@ DiagnosedSilenceableFailure createGpuLaunch(
   Value blkSizeX = blockDimX.has_value() ? createConst(blockDimX.value()) : one;
   Value blkSizeY = blockDimY.has_value() ? createConst(blockDimY.value()) : one;
   Value blkSizeZ = blockDimZ.has_value() ? createConst(blockDimZ.value()) : one;
-  launchOp = rewriter.create<LaunchOp>(loc, gridSizeX, gridSizeY, gridSizeZ,
+  launchOp = LaunchOp::create(rewriter, loc, gridSizeX, gridSizeY, gridSizeZ,
                                        blkSizeX, blkSizeY, blkSizeZ);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
-  rewriter.create<TerminatorOp>(loc);
+  TerminatorOp::create(rewriter, loc);
   return DiagnosedSilenceableFailure::success();
 }
 
@@ -451,7 +451,7 @@ DiagnosedSilenceableFailure alterGpuLaunch(
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPointAfterValue(currentBlockdim.x);
   auto createConstValue = [&](int dim) {
-    return rewriter.create<arith::ConstantIndexOp>(currentBlockdim.x.getLoc(),
+    return arith::ConstantIndexOp::create(rewriter, currentBlockdim.x.getLoc(),
                                                    dim);
   };
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
index d35f72e5a9e26..66e011a793485 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
@@ -147,7 +147,7 @@ struct GpuAllReduceRewriter {
   // Shortcut to create an op from rewriter using loc as the first argument.
   template <typename T, typename... Args>
   T create(Args... args) {
-    return rewriter.create<T>(loc, std::forward<Args>(args)...);
+    return T::create(rewriter, loc, std::forward<Args>(args)...);
   }
 
   // Creates dimension op of type T, with the result casted to int32.
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index 99a91ecd5642c..ab3dcc6a45d11 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -132,7 +132,7 @@ struct GpuAsyncRegionPass::ThreadTokenCallback {
   }
 
   Value createWaitOp(Location loc, Type resultType, ValueRange operands) {
-    return builder.create<gpu::WaitOp>(loc, resultType, operands)
+    return gpu::WaitOp::create(builder, loc, resultType, operands)
         .getAsyncToken();
   }
 
@@ -168,7 +168,7 @@ async::ExecuteOp addExecuteResults(async::ExecuteOp executeOp,
 
   // Clone executeOp with the extra results.
   OpBuilder builder(executeOp);
-  auto newOp = builder.create<async::ExecuteOp>(
+  auto newOp = async::ExecuteOp::create(builder,
       executeOp.getLoc(), TypeRange{resultTypes}.drop_front() /*drop token*/,
       executeOp.getDependencies(), executeOp.getBodyOperands());
   IRMapping mapper;
@@ -250,7 +250,7 @@ struct GpuAsyncRegionPass::DeferWaitCallback {
           builder.setInsertionPointAfter(op);
           for (auto asyncToken : asyncTokens)
             tokens.push_back(
-                builder.create<async::AwaitOp>(loc, asyncToken).getResult());
+                async::AwaitOp::create(builder, loc, asyncToken).getResult());
           // Set `it` after the inserted async.await ops.
           it = builder.getInsertionPoint();
         })
@@ -282,7 +282,7 @@ struct GpuAsyncRegionPass::DeferWaitCallback {
 
     // Otherwise, insert a gpu.wait before 'it'.
     builder.setInsertionPoint(it->getBlock(), it);
-    auto waitOp = builder.create<gpu::WaitOp>(loc, Type{}, tokens);
+    auto waitOp = gpu::WaitOp::create(builder, loc, Type{}, tokens);
 
     // If the new waitOp is at the end of an async.execute region, add it to the
     // worklist. 'operator()(executeOp)' would do the same, but this is faster.
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index f63af8da28087..1c7fdf357af20 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -64,7 +64,7 @@ getFlatOffsetAndStrides(OpBuilder &rewriter, Location loc, Value source,
     OpBuilder::InsertionGuard g(rewriter);
     setInsertionPointToStart(rewriter, source);
     newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, source);
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, source);
   }
 
   auto &&[sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
@@ -110,7 +110,7 @@ static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
   auto &&[base, offset, ignore] =
       getFlatOffsetAndStrides(rewriter, loc, source, offsetsTemp);
   MemRefType retType = inferCastResultType(base, offset);
-  return rewriter.create<memref::ReinterpretCastOp>(loc, retType, base, offset,
+  return memref::ReinterpretCastOp::create(rewriter, loc, retType, base, offset,
                                                     ArrayRef<OpFoldResult>(),
                                                     ArrayRef<OpFoldResult>());
 }
diff --git a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
index c40ddd9b15afc..8dfd52e5803f0 100644
--- a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
@@ -28,11 +28,11 @@ struct GpuGlobalIdRewriter : public OpRewritePattern<gpu::GlobalIdOp> {
                                 PatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     auto dim = op.getDimension();
-    auto blockId = rewriter.create<gpu::BlockIdOp>(loc, dim);
-    auto blockDim = rewriter.create<gpu::BlockDimOp>(loc, dim);
+    auto blockId = gpu::BlockIdOp::create(rewriter, loc, dim);
+    auto blockDim = gpu::BlockDimOp::create(rewriter, loc, dim);
     // Compute blockId.x * blockDim.x
-    auto tmp = rewriter.create<index::MulOp>(op.getLoc(), blockId, blockDim);
-    auto threadId = rewriter.create<gpu::ThreadIdOp>(loc, dim);
+    auto tmp = index::MulOp::create(rewriter, op.getLoc(), blockId, blockDim);
+    auto threadId = gpu::ThreadIdOp::create(rewriter, loc, dim);
     // Compute threadId.x + blockId.x * blockDim.x
     rewriter.replaceOpWithNewOp<index::AddOp>(op, threadId, tmp);
     return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index a64ec8d52daf0..af8b097ecf6a9 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -41,7 +41,7 @@ template <typename OpTy>
 static void createForAllDimensions(OpBuilder &builder, Location loc,
                                    SmallVectorImpl<Value> &values) {
   for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z})
-    values.push_back(builder.create<OpTy>(loc, builder.getIndexType(), dim));
+    values.push_back(OpTy::create(builder, loc, builder.getIndexType(), dim));
 }
 
 /// Adds operations generating block/thread ids and grid/block dimensions at the
@@ -196,7 +196,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   }
   FunctionType type =
       FunctionType::get(launchOp.getContext(), kernelOperandTypes, {});
-  auto outlinedFunc = builder.create<gpu::GPUFuncOp>(
+  auto outlinedFunc = gpu::GPUFuncOp::create(builder,
       loc, kernelFnName, type,
       TypeRange(ValueRange(launchOp.getWorkgroupAttributions())),
       TypeRange(ValueRange(launchOp.getPrivateAttributions())));
@@ -248,7 +248,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
     if (!terminator)
       continue;
     OpBuilder replacer(terminator);
-    replacer.create<gpu::ReturnOp>(terminator->getLoc());
+    gpu::ReturnOp::create(replacer, terminator->getLoc());
     terminator->erase();
   }
 
@@ -288,7 +288,7 @@ static void convertToLaunchFuncOp(gpu::LaunchOp launchOp,
   Value asyncToken = launchOp.getAsyncToken();
   std::optional<gpu::KernelDim3> clusterSize =
       launchOp.getClusterSizeOperandValues();
-  auto launchFunc = builder.create<gpu::LaunchFuncOp>(
+  auto launchFunc = gpu::LaunchFuncOp::create(builder,
       launchOp.getLoc(), kernelFunc, launchOp.getGridSizeOperandValues(),
       launchOp.getBlockSizeOperandValues(),
       launchOp.getDynamicSharedMemorySize(), operands,
@@ -416,7 +416,7 @@ class GpuKernelOutliningPass
     // Check if the module already exists in the symbol table
     if (!kernelModule) {
       // If not found, create a new GPU module
-      kernelModule = builder.create<gpu::GPUModuleOp>(kernelFunc.getLoc(),
+      kernelModule = gpu::GPUModuleOp::create(builder, kernelFunc.getLoc(),
                                                       kernelModuleName);
     }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 14c44f27a6249..2bee1a1619665 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -34,8 +34,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
   auto rank = memRefType.getRank();
 
   SmallVector<Value, 4> lbs, ubs, steps;
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value one = b.create<arith::ConstantIndexOp>(1);
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value one = arith::ConstantIndexOp::create(b, 1);
 
   // Make sure we have enough loops to use all thread dimensions, these trivial
   // loops should be outermost and therefore inserted first.
@@ -59,8 +59,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
   auto indexType = b.getIndexType();
   SmallVector<Value, 3> threadIds, blockDims;
   for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z}) {
-    threadIds.push_back(b.create<gpu::ThreadIdOp>(indexType, dim));
-    blockDims.push_back(b.create<gpu::BlockDimOp>(indexType, dim));
+    threadIds.push_back(gpu::ThreadIdOp::create(b, indexType, dim));
+    blockDims.push_back(gpu::BlockDimOp::create(b, indexType, dim));
   }
 
   // Produce the loop nest with copies.
@@ -70,8 +70,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
       [&](OpBuilder &b, Location loc, ValueRange loopIvs) {
         ivs.assign(loopIvs.begin(), loopIvs.end());
         auto activeIvs = llvm::ArrayRef(ivs).take_back(rank);
-        Value loaded = b.create<memref::LoadOp>(loc, from, activeIvs);
-        b.create<memref::StoreOp>(loc, loaded, to, activeIvs);
+        Value loaded = memref::LoadOp::create(b, loc, from, activeIvs);
+        memref::StoreOp::create(b, loc, loaded, to, activeIvs);
       });
 
   // Map the innermost loops to threads in reverse order.
@@ -131,10 +131,10 @@ static void insertCopies(Region &region, Location loc, Value from, Value to) {
 
   auto b = ImplicitLocOpBuilder::atBlockBegin(loc, &region.front());
   insertCopyLoops(b, from, to);
-  b.create<gpu::BarrierOp>();
+  gpu::BarrierOp::create(b, );
 
   b.setInsertionPoint(&region.front().back());
-  b.create<gpu::BarrierOp>();
+  gpu::BarrierOp::create(b, );
   insertCopyLoops(b, to, from);
 }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
index f8a548af6b3e8..5bc85f236575d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
@@ -115,7 +115,7 @@ LogicalResult moduleSerializer(GPUModuleOp op,
       !handler && moduleHandler)
     handler = moduleHandler;
   builder.setInsertionPointAfter(op);
-  builder.create<gpu::BinaryOp>(op.getLoc(), op.getName(), handler,
+  gpu::BinaryOp::create(builder, op.getLoc(), op.getName(), handler,
                                 builder.getArrayAttr(objects));
   op->erase();
   return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
index 171e64346f155..5a55a39a7748e 100644
--- a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
@@ -48,10 +48,10 @@ struct PromoteShuffleToSwizzlePattern
                                          "offset must be in the range [0, 31]");
 
     Location loc = op.getLoc();
-    Value res = rewriter.create<amdgpu::SwizzleBitModeOp>(
+    Value res = amdgpu::SwizzleBitModeOp::create(rewriter,
         loc, op.getResult(0).getType(), op.getValue(), /*andMask=*/31,
         /*orMask=*/0, /*xorMask=*/offsetValue);
-    Value valid = rewriter.create<arith::ConstantIntOp>(loc, 1, /*width*/ 1);
+    Value valid = arith::ConstantIntOp::create(rewriter, loc, 1, /*width*/ 1);
     rewriter.replaceOp(op, {res, valid});
     return success();
   }
diff --git a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
index 9f2900214e8b1..08dedf065f900 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
@@ -48,16 +48,16 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
 
     // Float types must be converted to i64 to extract the bits.
     if (isa<FloatType>(valueType))
-      value = rewriter.create<arith::BitcastOp>(valueLoc, i64, value);
+      value = arith::BitcastOp::create(rewriter, valueLoc, i64, value);
 
     // Get the low bits by trunc(value).
-    lo = rewriter.create<arith::TruncIOp>(valueLoc, i32, value);
+    lo = arith::TruncIOp::create(rewriter, valueLoc, i32, value);
 
     // Get the high bits by trunc(value >> 32).
-    auto c32 = rewriter.create<arith::ConstantOp>(
+    auto c32 = arith::ConstantOp::create(rewriter,
         valueLoc, rewriter.getIntegerAttr(i64, 32));
-    hi = rewriter.create<arith::ShRUIOp>(valueLoc, value, c32);
-    hi = rewriter.create<arith::TruncIOp>(valueLoc, i32, hi);
+    hi = arith::ShRUIOp::create(rewriter, valueLoc, value, c32);
+    hi = arith::TruncIOp::create(rewriter, valueLoc, i32, hi);
 
     // Shuffle the values.
     ValueRange loRes =
@@ -72,21 +72,21 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
             .getResults();
 
     // Convert lo back to i64.
-    lo = rewriter.create<arith::ExtUIOp>(valueLoc, i64, loRes[0]);
+    lo = arith::ExtUIOp::create(rewriter, valueLoc, i64, loRes[0]);
 
     // Convert hi back to i64.
-    hi = rewriter.create<arith::ExtUIOp>(valueLoc, i64, hiRes[0]);
-    hi = rewriter.create<arith::ShLIOp>(valueLoc, hi, c32);
+    hi = arith::ExtUIOp::create(rewriter, valueLoc, i64, hiRes[0]);
+    hi = arith::ShLIOp::create(rewriter, valueLoc, hi, c32);
 
     // Obtain the shuffled bits hi | lo.
-    value = rewriter.create<arith::OrIOp>(loc, hi, lo);
+    value = arith::OrIOp::create(rewriter, loc, hi, lo);
 
     // Convert the value back to float.
     if (isa<FloatType>(valueType))
-      value = rewriter.create<arith::BitcastOp>(valueLoc, valueType, value);
+      value = arith::BitcastOp::create(rewriter, valueLoc, valueType, value);
 
     // Obtain the shuffle validity by combining both validities.
-    auto validity = rewriter.create<arith::AndIOp>(loc, loRes[1], hiRes[1]);
+    auto validity = arith::AndIOp::create(rewriter, loc, loRes[1], hiRes[1]);
 
     // Replace the op.
     rewriter.replaceOp(op, {value, validity});
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
index d80578235f3c3..0dbd9713a2a19 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
@@ -55,22 +55,22 @@ struct GpuSubgroupIdRewriter final : OpRewritePattern<gpu::SubgroupIdOp> {
     Location loc = op->getLoc();
     Type indexType = rewriter.getIndexType();
 
-    Value dimX = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::x);
-    Value dimY = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::y);
-    Value tidX = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
-    Value tidY = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::y);
-    Value tidZ = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::z);
+    Value dimX = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::x);
+    Value dimY = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::y);
+    Value tidX = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
+    Value tidY = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::y);
+    Value tidZ = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::z);
 
-    Value dimYxIdZ = rewriter.create<arith::MulIOp>(loc, indexType, dimY, tidZ);
+    Value dimYxIdZ = arith::MulIOp::create(rewriter, loc, indexType, dimY, tidZ);
     Value dimYxIdZPlusIdY =
-        rewriter.create<arith::AddIOp>(loc, indexType, dimYxIdZ, tidY);
+        arith::AddIOp::create(rewriter, loc, indexType, dimYxIdZ, tidY);
     Value dimYxIdZPlusIdYTimesDimX =
-        rewriter.create<arith::MulIOp>(loc, indexType, dimX, dimYxIdZPlusIdY);
-    Value IdXPlusDimYxIdZPlusIdYTimesDimX = rewriter.create<arith::AddIOp>(
+        arith::MulIOp::create(rewriter, loc, indexType, dimX, dimYxIdZPlusIdY);
+    Value IdXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(rewriter,
         loc, indexType, tidX, dimYxIdZPlusIdYTimesDimX);
-    Value subgroupSize = rewriter.create<gpu::SubgroupSizeOp>(
+    Value subgroupSize = gpu::SubgroupSizeOp::create(rewriter,
         loc, rewriter.getIndexType(), /*upper_bound = */ nullptr);
-    Value subgroupIdOp = rewriter.create<arith::DivUIOp>(
+    Value subgroupIdOp = arith::DivUIOp::create(rewriter,
         loc, indexType, IdXPlusDimYxIdZPlusIdYTimesDimX, subgroupSize);
     rewriter.replaceOp(op, {subgroupIdOp});
     return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index af9be4cccecfc..d5470de5a4faa 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -80,7 +80,7 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
 
     Location loc = op.getLoc();
     Value res =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(vecTy));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(vecTy));
 
     for (unsigned i = 0; i != numNewReductions; ++i) {
       int64_t startIdx = i * elementsPerShuffle;
@@ -91,22 +91,22 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
       Value extracted;
       if (numElems == 1) {
         extracted =
-            rewriter.create<vector::ExtractOp>(loc, op.getValue(), startIdx);
+            vector::ExtractOp::create(rewriter, loc, op.getValue(), startIdx);
       } else {
-        extracted = rewriter.create<vector::ExtractStridedSliceOp>(
+        extracted = vector::ExtractStridedSliceOp::create(rewriter,
             loc, op.getValue(), /*offsets=*/startIdx, /*sizes=*/numElems,
             /*strides=*/1);
       }
 
-      Value reduce = rewriter.create<gpu::SubgroupReduceOp>(
+      Value reduce = gpu::SubgroupReduceOp::create(rewriter,
           loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
           op.getClusterStride());
       if (numElems == 1) {
-        res = rewriter.create<vector::InsertOp>(loc, reduce, res, startIdx);
+        res = vector::InsertOp::create(rewriter, loc, reduce, res, startIdx);
         continue;
       }
 
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, reduce, res, /*offsets=*/startIdx, /*strides=*/1);
     }
 
@@ -139,8 +139,8 @@ struct ScalarizeSingleElementReduce final
     assert(vecTy.getRank() == 1 && "Unexpected vector type");
     assert(!vecTy.isScalable() && "Unexpected vector type");
     Location loc = op.getLoc();
-    Value extracted = rewriter.create<vector::ExtractOp>(loc, op.getValue(), 0);
-    Value reduce = rewriter.create<gpu::SubgroupReduceOp>(
+    Value extracted = vector::ExtractOp::create(rewriter, loc, op.getValue(), 0);
+    Value reduce = gpu::SubgroupReduceOp::create(rewriter,
         loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
         op.getClusterStride());
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, vecTy, reduce);
@@ -255,14 +255,14 @@ struct ScalarSubgroupReduceToShuffles final
     auto packFn = [loc, &rewriter, equivIntType,
                    shuffleIntType](Value unpackedVal) -> Value {
       auto asInt =
-          rewriter.create<arith::BitcastOp>(loc, equivIntType, unpackedVal);
-      return rewriter.create<arith::ExtUIOp>(loc, shuffleIntType, asInt);
+          arith::BitcastOp::create(rewriter, loc, equivIntType, unpackedVal);
+      return arith::ExtUIOp::create(rewriter, loc, shuffleIntType, asInt);
     };
     auto unpackFn = [loc, &rewriter, equivIntType,
                      valueTy](Value packedVal) -> Value {
       auto asInt =
-          rewriter.create<arith::TruncIOp>(loc, equivIntType, packedVal);
-      return rewriter.create<arith::BitcastOp>(loc, valueTy, asInt);
+          arith::TruncIOp::create(rewriter, loc, equivIntType, packedVal);
+      return arith::BitcastOp::create(rewriter, loc, valueTy, asInt);
     };
 
     rewriter.replaceOp(
@@ -327,9 +327,9 @@ struct VectorSubgroupReduceToShuffles final
         static_cast<int64_t>(elementsPerShuffle), vecTy.getElementType());
     Value extendedInput = op.getValue();
     if (vecBitwidth < shuffleBitwidth) {
-      auto zero = rewriter.create<arith::ConstantOp>(
+      auto zero = arith::ConstantOp::create(rewriter,
           loc, rewriter.getZeroAttr(extendedVecTy));
-      extendedInput = rewriter.create<vector::InsertStridedSliceOp>(
+      extendedInput = vector::InsertStridedSliceOp::create(rewriter,
           loc, extendedInput, zero, /*offsets=*/0, /*strides=*/1);
     }
 
@@ -338,21 +338,21 @@ struct VectorSubgroupReduceToShuffles final
 
     auto packFn = [loc, &rewriter, shuffleVecType](Value unpackedVal) -> Value {
       auto asIntVec =
-          rewriter.create<vector::BitCastOp>(loc, shuffleVecType, unpackedVal);
-      return rewriter.create<vector::ExtractOp>(loc, asIntVec, 0);
+          vector::BitCastOp::create(rewriter, loc, shuffleVecType, unpackedVal);
+      return vector::ExtractOp::create(rewriter, loc, asIntVec, 0);
     };
     auto unpackFn = [loc, &rewriter, shuffleVecType,
                      extendedVecTy](Value packedVal) -> Value {
       auto asIntVec =
-          rewriter.create<vector::BroadcastOp>(loc, shuffleVecType, packedVal);
-      return rewriter.create<vector::BitCastOp>(loc, extendedVecTy, asIntVec);
+          vector::BroadcastOp::create(rewriter, loc, shuffleVecType, packedVal);
+      return vector::BitCastOp::create(rewriter, loc, extendedVecTy, asIntVec);
     };
 
     Value res = createSubgroupShuffleReduction(
         rewriter, loc, extendedInput, op.getOp(), *ci, packFn, unpackFn);
 
     if (vecBitwidth < shuffleBitwidth) {
-      res = rewriter.create<vector::ExtractStridedSliceOp>(
+      res = vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/0, /*sizes=*/vecTy.getNumElements(),
           /*strides=*/1);
     }
@@ -379,7 +379,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   const bool boundCtrl = true;
   if (ci.clusterSize >= 2) {
     // Perform reduction between all lanes N <-> N+1.
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({1, 0, 3, 2}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -388,7 +388,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
 
   if (ci.clusterSize >= 4) {
     // Perform reduction between all lanes N <-> N+2.
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({2, 3, 0, 1}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -397,7 +397,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 8) {
     // Perform reduction between all lanes N <-> 7-N,
     // e.g lane[0] <-> lane[7], lane[1] <-> lane[6]..., lane[3] <-> lane[4].
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::row_half_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -406,7 +406,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 16) {
     // Perform reduction between all lanes N <-> 15-N,
     // e.g lane[0] <-> lane[15], lane[1] <-> lane[14]..., lane[7] <-> lane[8].
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::row_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -416,7 +416,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
     if (chipset.majorVersion <= 9) {
       // Broadcast last value from each row to next row.
       // Use row mask to avoid polluting rows 1 and 3.
-      dpp = rewriter.create<amdgpu::DPPOp>(
+      dpp = amdgpu::DPPOp::create(rewriter,
           loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_15,
           rewriter.getUnitAttr(), 0xa, allBanks,
           /*bound_ctrl*/ false);
@@ -424,9 +424,9 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
           rewriter, loc, gpu::convertReductionKind(mode), res, dpp);
     } else if (chipset.majorVersion <= 12) {
       // Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2).
-      Value uint32Max = rewriter.create<arith::ConstantOp>(
+      Value uint32Max = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
-      dpp = rewriter.create<ROCDL::PermlaneX16Op>(loc, res.getType(), res, res,
+      dpp = ROCDL::PermlaneX16Op::create(rewriter, loc, res.getType(), res, res,
                                                   uint32Max, uint32Max,
                                                   /*fi=*/true,
                                                   /*bound_ctrl=*/false);
@@ -438,37 +438,37 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
               "this device.");
     }
     if (ci.subgroupSize == 32) {
-      Value lane31 = rewriter.create<arith::ConstantOp>(
+      Value lane31 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      res = rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane31);
+      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
     }
   }
   if (ci.clusterSize >= 64) {
     if (chipset.majorVersion <= 9) {
       // Broadcast 31st lane value to rows 2 and 3.
-      dpp = rewriter.create<amdgpu::DPPOp>(
+      dpp = amdgpu::DPPOp::create(rewriter,
           loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_31,
           rewriter.getUnitAttr(), 0xf, allBanks,
           /*bound_ctrl*/ true);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), dpp, res);
       // Obtain reduction from last rows, the previous rows are polluted.
-      Value lane63 = rewriter.create<arith::ConstantOp>(
+      Value lane63 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
-      res = rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane63);
+      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
 
     } else if (chipset.majorVersion <= 12) {
       // Assume reduction across 32 lanes has been done.
       // Perform final reduction manually by summing values in lane 0 and
       // lane 32.
-      Value lane31 = rewriter.create<arith::ConstantOp>(
+      Value lane31 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      Value lane63 = rewriter.create<arith::ConstantOp>(
+      Value lane63 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
       lane31 =
-          rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane31);
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
       lane63 =
-          rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane63);
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), lane31, lane63);
     } else {
diff --git a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
index 29f6f32892f72..473a2e515c12b 100644
--- a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
+++ b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
@@ -27,7 +27,7 @@ WarpDistributionPattern::moveRegionToNewWarpOpAndReplaceReturns(
   // Create a new op before the existing one, with the extra operands.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(warpOp);
-  auto newWarpOp = rewriter.create<WarpExecuteOnLane0Op>(
+  auto newWarpOp = WarpExecuteOnLane0Op::create(rewriter,
       warpOp.getLoc(), newReturnTypes, warpOp.getLaneid(), warpOp.getWarpSize(),
       warpOp.getArgs(), warpOp.getBody()->getArgumentTypes());
 
@@ -124,7 +124,7 @@ bool WarpDistributionPattern::delinearizeLaneId(
 
   int64_t usedThreads = 1;
 
-  Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
   delinearizedIds.assign(sizes.size(), zero);
 
   for (int i = sizes.size() - 1; i >= 0; --i) {
diff --git a/mlir/lib/Dialect/Index/IR/IndexOps.cpp b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
index 5c935c5f4b53e..2b32962f9eabb 100644
--- a/mlir/lib/Dialect/Index/IR/IndexOps.cpp
+++ b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
@@ -37,7 +37,7 @@ Operation *IndexDialect::materializeConstant(OpBuilder &b, Attribute value,
   if (auto boolValue = dyn_cast<BoolAttr>(value)) {
     if (!type.isSignlessInteger(1))
       return nullptr;
-    return b.create<BoolConstantOp>(loc, type, boolValue);
+    return BoolConstantOp::create(b, loc, type, boolValue);
   }
 
   // Materialize integer attributes as `index`.
@@ -47,7 +47,7 @@ Operation *IndexDialect::materializeConstant(OpBuilder &b, Attribute value,
       return nullptr;
     assert(indexValue.getValue().getBitWidth() ==
            IndexType::kInternalStorageBitWidth);
-    return b.create<ConstantOp>(loc, indexValue);
+    return ConstantOp::create(b, loc, indexValue);
   }
 
   return nullptr;
@@ -716,10 +716,10 @@ LogicalResult CmpOp::canonicalize(CmpOp op, PatternRewriter &rewriter) {
 
   index::CmpOp newCmp;
   if (rhsIsZero)
-    newCmp = rewriter.create<index::CmpOp>(op.getLoc(), op.getPred(),
+    newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
                                            subOp.getLhs(), subOp.getRhs());
   else
-    newCmp = rewriter.create<index::CmpOp>(op.getLoc(), op.getPred(),
+    newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
                                            subOp.getRhs(), subOp.getLhs());
   rewriter.replaceOp(op, newCmp);
   return success();
diff --git a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
index c17ef1029faf6..b1d376c7c03db 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
@@ -90,7 +90,7 @@ void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
     }
     for (auto [idx, t] : llvm::enumerate(stype.getBody())) {
       if (itype != PTXRegisterMod::Write) {
-        Value extractValue = rewriter.create<LLVM::ExtractValueOp>(
+        Value extractValue = LLVM::ExtractValueOp::create(rewriter,
             interfaceOp->getLoc(), v, idx);
         addValue(extractValue);
       }
@@ -132,7 +132,7 @@ LLVM::InlineAsmOp PtxBuilder::build() {
   // Replace all % with $
   llvm::replace(ptxInstruction, '%', '$');
 
-  return rewriter.create<LLVM::InlineAsmOp>(
+  return LLVM::InlineAsmOp::create(rewriter,
       interfaceOp->getLoc(),
       /*result types=*/resultTypes,
       /*operands=*/ptxOperands,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 89f765dacda35..7d3f59d128367 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -89,7 +89,7 @@ mlir::LLVM::lookupOrCreateFn(OpBuilder &b, Operation *moduleOp, StringRef name,
   OpBuilder::InsertionGuard g(b);
   assert(!moduleOp->getRegion(0).empty() && "expected non-empty region");
   b.setInsertionPointToStart(&moduleOp->getRegion(0).front());
-  auto funcOp = b.create<LLVM::LLVMFuncOp>(
+  auto funcOp = LLVM::LLVMFuncOp::create(b,
       moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 6dcd94e6eea17..546704e903f47 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3318,7 +3318,7 @@ bool LLVM::ConstantOp::isBuildableWith(Attribute value, Type type) {
 ConstantOp LLVM::ConstantOp::materialize(OpBuilder &builder, Attribute value,
                                          Type type, Location loc) {
   if (isBuildableWith(value, type))
-    return builder.create<LLVM::ConstantOp>(loc, cast<TypedAttr>(value));
+    return LLVM::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
   return nullptr;
 }
 
@@ -4284,13 +4284,13 @@ Operation *LLVMDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // a builtin zero attribute and thus will materialize as a llvm.mlir.constant.
   if (auto symbol = dyn_cast<FlatSymbolRefAttr>(value))
     if (isa<LLVM::LLVMPointerType>(type))
-      return builder.create<LLVM::AddressOfOp>(loc, type, symbol);
+      return LLVM::AddressOfOp::create(builder, loc, type, symbol);
   if (isa<LLVM::UndefAttr>(value))
-    return builder.create<LLVM::UndefOp>(loc, type);
+    return LLVM::UndefOp::create(builder, loc, type);
   if (isa<LLVM::PoisonAttr>(value))
-    return builder.create<LLVM::PoisonOp>(loc, type);
+    return LLVM::PoisonOp::create(builder, loc, type);
   if (isa<LLVM::ZeroAttr>(value))
-    return builder.create<LLVM::ZeroOp>(loc, type);
+    return LLVM::ZeroOp::create(builder, loc, type);
   // Otherwise try materializing it as a regular llvm.mlir.constant op.
   return LLVM::ConstantOp::materialize(builder, value, type, loc);
 }
@@ -4313,15 +4313,15 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
   MLIRContext *ctx = builder.getContext();
   auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
-  auto global = moduleBuilder.create<LLVM::GlobalOp>(
+  auto global = LLVM::GlobalOp::create(moduleBuilder,
       loc, type, /*isConstant=*/true, linkage, name,
       builder.getStringAttr(value), /*alignment=*/0);
 
   LLVMPointerType ptrType = LLVMPointerType::get(ctx);
   // Get the pointer to the first character in the global string.
   Value globalPtr =
-      builder.create<LLVM::AddressOfOp>(loc, ptrType, global.getSymNameAttr());
-  return builder.create<LLVM::GEPOp>(loc, ptrType, type, globalPtr,
+      LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
+  return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
                                      ArrayRef<GEPArg>{0, 0});
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index bc451f8b028fc..98fb9b09f0cd5 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -37,7 +37,7 @@ llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() {
 
 Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot,
                                       OpBuilder &builder) {
-  return builder.create<LLVM::UndefOp>(getLoc(), slot.elemType);
+  return LLVM::UndefOp::create(builder, getLoc(), slot.elemType);
 }
 
 void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
@@ -45,7 +45,7 @@ void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
                                          OpBuilder &builder) {
   for (Operation *user : getOperation()->getUsers())
     if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
-      builder.create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
+      LLVM::DbgValueOp::create(builder, declareOp.getLoc(), argument,
                                        declareOp.getVarInfo(),
                                        declareOp.getLocationExpr());
 }
@@ -89,7 +89,7 @@ DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
   for (Attribute index : usedIndices) {
     Type elemType = destructurableType.getTypeAtIndex(index);
     assert(elemType && "used index must exist");
-    auto subAlloca = builder.create<LLVM::AllocaOp>(
+    auto subAlloca = LLVM::AllocaOp::create(builder,
         getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
         getArraySize());
     newAllocators.push_back(subAlloca);
@@ -260,13 +260,13 @@ static Value createExtractAndCast(OpBuilder &builder, Location loc,
   // Truncate the integer if the size of the target is less than the value.
   if (isBigEndian(dataLayout)) {
     uint64_t shiftAmount = srcTypeSize - targetTypeSize;
-    auto shiftConstant = builder.create<LLVM::ConstantOp>(
+    auto shiftConstant = LLVM::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(srcType, shiftAmount));
     replacement =
         builder.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
   }
 
-  replacement = builder.create<LLVM::TruncOp>(
+  replacement = LLVM::TruncOp::create(builder,
       loc, builder.getIntegerType(targetTypeSize), replacement);
 
   // Now cast the integer to the actual target type if required.
@@ -304,7 +304,7 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
     // On big endian systems, a store to the base pointer overwrites the most
     // significant bits. To accomodate for this, the stored value needs to be
     // shifted into the according position.
-    Value bigEndianShift = builder.create<LLVM::ConstantOp>(
+    Value bigEndianShift = LLVM::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
     valueAsInt =
         builder.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
@@ -325,7 +325,7 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
   }
 
   // Mask out the affected bits ...
-  Value mask = builder.create<LLVM::ConstantOp>(
+  Value mask = LLVM::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
   Value masked = builder.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
 
@@ -644,7 +644,7 @@ DeletionKind LLVM::DbgValueOp::removeBlockingUses(
   // debug local variable info. This allows the debugger to inform the user that
   // the variable has been optimized out.
   auto undef =
-      builder.create<UndefOp>(getValue().getLoc(), getValue().getType());
+      UndefOp::create(builder, getValue().getLoc(), getValue().getType());
   getValueMutable().assign(undef);
   return DeletionKind::Keep;
 }
@@ -655,7 +655,7 @@ void LLVM::DbgDeclareOp::visitReplacedValues(
     ArrayRef<std::pair<Operation *, Value>> definitions, OpBuilder &builder) {
   for (auto [op, value] : definitions) {
     builder.setInsertionPointAfter(op);
-    builder.create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(),
+    LLVM::DbgValueOp::create(builder, getLoc(), value, getVarInfo(),
                                      getLocationExpr());
   }
 }
@@ -978,7 +978,7 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetOp toReplace,
               IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize))
           .getResult();
 
-  builder.create<LLVM::MemsetOp>(toReplace.getLoc(), subslots.at(index).ptr,
+  LLVM::MemsetOp::create(builder, toReplace.getLoc(), subslots.at(index).ptr,
                                  toReplace.getVal(), newMemsetSizeValue,
                                  toReplace.getIsVolatile());
 }
@@ -991,7 +991,7 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
   auto newMemsetSizeValue =
       IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize);
 
-  builder.create<LLVM::MemsetInlineOp>(
+  LLVM::MemsetInlineOp::create(builder,
       toReplace.getLoc(), subslots.at(index).ptr, toReplace.getVal(),
       newMemsetSizeValue, toReplace.getIsVolatile());
 }
@@ -1063,7 +1063,7 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       APInt memsetVal(/*numBits=*/width, /*val=*/0);
       for (unsigned loBit = 0; loBit < width; loBit += 8)
         memsetVal.insertBits(constantPattern.getValue(), loBit);
-      return builder.create<LLVM::ConstantOp>(
+      return LLVM::ConstantOp::create(builder,
           op.getLoc(), IntegerAttr::get(intType, memsetVal));
     }
 
@@ -1075,14 +1075,14 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
     // value and or-ing it with the previous value.
     uint64_t coveredBits = 8;
     Value currentValue =
-        builder.create<LLVM::ZExtOp>(op.getLoc(), intType, op.getVal());
+        LLVM::ZExtOp::create(builder, op.getLoc(), intType, op.getVal());
     while (coveredBits < width) {
       Value shiftBy =
-          builder.create<LLVM::ConstantOp>(op.getLoc(), intType, coveredBits);
+          LLVM::ConstantOp::create(builder, op.getLoc(), intType, coveredBits);
       Value shifted =
-          builder.create<LLVM::ShlOp>(op.getLoc(), currentValue, shiftBy);
+          LLVM::ShlOp::create(builder, op.getLoc(), currentValue, shiftBy);
       currentValue =
-          builder.create<LLVM::OrOp>(op.getLoc(), currentValue, shifted);
+          LLVM::OrOp::create(builder, op.getLoc(), currentValue, shifted);
       coveredBits *= 2;
     }
 
@@ -1094,7 +1094,7 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       })
       .Case([&](FloatType type) -> Value {
         Value intVal = buildMemsetValue(type.getWidth());
-        return builder.create<LLVM::BitcastOp>(op.getLoc(), type, intVal);
+        return LLVM::BitcastOp::create(builder, op.getLoc(), type, intVal);
       })
       .Default([](Type) -> Value {
         llvm_unreachable(
@@ -1282,7 +1282,7 @@ static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot) {
 template <class MemcpyLike>
 static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot,
                              OpBuilder &builder) {
-  return builder.create<LLVM::LoadOp>(op.getLoc(), slot.elemType, op.getSrc());
+  return LLVM::LoadOp::create(builder, op.getLoc(), slot.elemType, op.getSrc());
 }
 
 template <class MemcpyLike>
@@ -1309,7 +1309,7 @@ memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot,
                          const SmallPtrSetImpl<OpOperand *> &blockingUses,
                          OpBuilder &builder, Value reachingDefinition) {
   if (op.loadsFrom(slot))
-    builder.create<LLVM::StoreOp>(op.getLoc(), reachingDefinition, op.getDst());
+    LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition, op.getDst());
   return DeletionKind::Delete;
 }
 
@@ -1354,10 +1354,10 @@ template <class MemcpyLike>
 void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                MemcpyLike toReplace, Value dst, Value src,
                                Type toCpy, bool isVolatile) {
-  Value memcpySize = builder.create<LLVM::ConstantOp>(
+  Value memcpySize = LLVM::ConstantOp::create(builder,
       toReplace.getLoc(), IntegerAttr::get(toReplace.getLen().getType(),
                                            layout.getTypeSize(toCpy)));
-  builder.create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize,
+  MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize,
                              isVolatile);
 }
 
@@ -1367,7 +1367,7 @@ void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                Value src, Type toCpy, bool isVolatile) {
   Type lenType = IntegerType::get(toReplace->getContext(),
                                   toReplace.getLen().getBitWidth());
-  builder.create<LLVM::MemcpyInlineOp>(
+  LLVM::MemcpyInlineOp::create(builder,
       toReplace.getLoc(), dst, src,
       IntegerAttr::get(lenType, layout.getTypeSize(toCpy)), isVolatile);
 }
@@ -1409,7 +1409,7 @@ memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
     SmallVector<LLVM::GEPArg> gepIndices{
         0, static_cast<int32_t>(
                cast<IntegerAttr>(index).getValue().getZExtValue())};
-    Value subslotPtrInOther = builder.create<LLVM::GEPOp>(
+    Value subslotPtrInOther = LLVM::GEPOp::create(builder,
         op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()), slot.elemType,
         isDst ? op.getSrc() : op.getDst(), gepIndices);
 
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
index 6fbb0d24826d0..1b1268da15646 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
@@ -28,13 +28,13 @@ static void addComdat(LLVM::LLVMFuncOp &op, OpBuilder &builder,
     PatternRewriter::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(module.getBody());
     comdatOp =
-        builder.create<mlir::LLVM::ComdatOp>(module.getLoc(), comdatName);
+        mlir::LLVM::ComdatOp::create(builder, module.getLoc(), comdatName);
     symbolTable.insert(comdatOp);
   }
 
   PatternRewriter::InsertionGuard guard(builder);
   builder.setInsertionPointToStart(&comdatOp.getBody().back());
-  auto selectorOp = builder.create<mlir::LLVM::ComdatSelectorOp>(
+  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(builder,
       comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
   op.setComdatAttr(mlir::SymbolRefAttr::get(
       builder.getContext(), comdatName,
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index 7f3afffc9645e..2ecef4b4c8a51 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -113,18 +113,18 @@ handleInlinedAllocas(Operation *call,
     // scope if some are already present in the body of the caller. This is not
     // invalid IR, but LLVM cleans these up in InstCombineCalls.cpp, along with
     // other cases where the stacksave/stackrestore is redundant.
-    stackPtr = builder.create<LLVM::StackSaveOp>(
+    stackPtr = LLVM::StackSaveOp::create(builder,
         call->getLoc(), LLVM::LLVMPointerType::get(call->getContext()));
   }
   builder.setInsertionPointToStart(callerEntryBlock);
   for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
-    auto newConstant = builder.create<LLVM::ConstantOp>(
+    auto newConstant = LLVM::ConstantOp::create(builder,
         allocaOp->getLoc(), allocaOp.getArraySize().getType(), arraySize);
     // Insert a lifetime start intrinsic where the alloca was before moving it.
     if (shouldInsertLifetime) {
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(allocaOp);
-      builder.create<LLVM::LifetimeStartOp>(
+      LLVM::LifetimeStartOp::create(builder,
           allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
           allocaOp.getResult());
     }
@@ -139,10 +139,10 @@ handleInlinedAllocas(Operation *call,
       continue;
     builder.setInsertionPoint(block.getTerminator());
     if (hasDynamicAlloca)
-      builder.create<LLVM::StackRestoreOp>(call->getLoc(), stackPtr);
+      LLVM::StackRestoreOp::create(builder, call->getLoc(), stackPtr);
     for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
       if (shouldInsertLifetime)
-        builder.create<LLVM::LifetimeEndOp>(
+        LLVM::LifetimeEndOp::create(builder,
             allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
             allocaOp.getResult());
     }
@@ -603,15 +603,15 @@ static Value handleByValArgumentInit(OpBuilder &builder, Location loc,
     OpBuilder::InsertionGuard insertionGuard(builder);
     Block *entryBlock = &(*argument.getParentRegion()->begin());
     builder.setInsertionPointToStart(entryBlock);
-    Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
+    Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
                                                  builder.getI64IntegerAttr(1));
-    allocaOp = builder.create<LLVM::AllocaOp>(
+    allocaOp = LLVM::AllocaOp::create(builder,
         loc, argument.getType(), elementType, one, targetAlignment);
   }
   // Copy the pointee to the newly allocated value.
-  Value copySize = builder.create<LLVM::ConstantOp>(
+  Value copySize = LLVM::ConstantOp::create(builder,
       loc, builder.getI64Type(), builder.getI64IntegerAttr(elementTypeSize));
-  builder.create<LLVM::MemcpyOp>(loc, allocaOp, argument, copySize,
+  LLVM::MemcpyOp::create(builder, loc, allocaOp, argument, copySize,
                                  /*isVolatile=*/false);
   return allocaOp;
 }
@@ -747,7 +747,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    builder.create<LLVM::BrOp>(op->getLoc(), returnOp.getOperands(), newDest);
+    LLVM::BrOp::create(builder, op->getLoc(), returnOp.getOperands(), newDest);
     op->erase();
   }
 
@@ -801,7 +801,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
     // and is extremely unlikely to exist in the code prior to inlining, using
     // this to communicate between this method and `processInlinedCallBlocks`.
     // TODO: Fix this by refactoring the inliner interface.
-    auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
+    auto copyOp = LLVM::SSACopyOp::create(builder, call->getLoc(), argument);
     if (argumentAttrs.contains(LLVM::LLVMDialect::getNoAliasAttrName()))
       copyOp->setDiscardableAttr(
           builder.getStringAttr(LLVM::LLVMDialect::getNoAliasAttrName()),
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
index 8dd0c28d98522..32fe2edd6ea76 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
@@ -59,7 +59,7 @@ static void ensureDistinctSuccessors(Block &bb) {
       terminator->setSuccessor(dummyBlock, position);
       for (BlockArgument arg : successor.first->getArguments())
         dummyBlock->addArgument(arg.getType(), arg.getLoc());
-      builder.create<LLVM::BrOp>(terminator->getLoc(),
+      LLVM::BrOp::create(builder, terminator->getLoc(),
                                  dummyBlock->getArguments(), successor.first);
     }
   }
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
index 8db32ec1526c4..de6c3043bbcdd 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
@@ -59,32 +59,32 @@ LogicalResult ExpandDivF16::matchAndRewrite(LLVM::FDivOp op,
   Type i32Type = rewriter.getI32Type();
 
   // Extend lhs and rhs to fp32.
-  Value lhs = rewriter.create<LLVM::FPExtOp>(loc, f32Type, op.getLhs());
-  Value rhs = rewriter.create<LLVM::FPExtOp>(loc, f32Type, op.getRhs());
+  Value lhs = LLVM::FPExtOp::create(rewriter, loc, f32Type, op.getLhs());
+  Value rhs = LLVM::FPExtOp::create(rewriter, loc, f32Type, op.getRhs());
 
   // float rcp = rcp.approx.ftz.f32(rhs), approx = lhs * rcp.
-  Value rcp = rewriter.create<NVVM::RcpApproxFtzF32Op>(loc, f32Type, rhs);
-  Value approx = rewriter.create<LLVM::FMulOp>(loc, lhs, rcp);
+  Value rcp = NVVM::RcpApproxFtzF32Op::create(rewriter, loc, f32Type, rhs);
+  Value approx = LLVM::FMulOp::create(rewriter, loc, lhs, rcp);
 
   // Refine the approximation with one Newton iteration:
   // float refined = approx + (lhs - approx * rhs) * rcp;
-  Value err = rewriter.create<LLVM::FMAOp>(
-      loc, approx, rewriter.create<LLVM::FNegOp>(loc, rhs), lhs);
-  Value refined = rewriter.create<LLVM::FMAOp>(loc, err, rcp, approx);
+  Value err = LLVM::FMAOp::create(rewriter,
+      loc, approx, LLVM::FNegOp::create(rewriter, loc, rhs), lhs);
+  Value refined = LLVM::FMAOp::create(rewriter, loc, err, rcp, approx);
 
   // Use refined value if approx is normal (exponent neither all 0 or all 1).
-  Value mask = rewriter.create<LLVM::ConstantOp>(
+  Value mask = LLVM::ConstantOp::create(rewriter,
       loc, i32Type, rewriter.getUI32IntegerAttr(0x7f800000));
-  Value cast = rewriter.create<LLVM::BitcastOp>(loc, i32Type, approx);
-  Value exp = rewriter.create<LLVM::AndOp>(loc, i32Type, cast, mask);
-  Value zero = rewriter.create<LLVM::ConstantOp>(
+  Value cast = LLVM::BitcastOp::create(rewriter, loc, i32Type, approx);
+  Value exp = LLVM::AndOp::create(rewriter, loc, i32Type, cast, mask);
+  Value zero = LLVM::ConstantOp::create(rewriter,
       loc, i32Type, rewriter.getUI32IntegerAttr(0));
-  Value pred = rewriter.create<LLVM::OrOp>(
+  Value pred = LLVM::OrOp::create(rewriter,
       loc,
-      rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, exp, zero),
-      rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, exp, mask));
+      LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, zero),
+      LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, mask));
   Value result =
-      rewriter.create<LLVM::SelectOp>(loc, f32Type, pred, approx, refined);
+      LLVM::SelectOp::create(rewriter, loc, f32Type, pred, approx, refined);
 
   // Replace with trucation back to fp16.
   rewriter.replaceOpWithNewOp<LLVM::FPTruncOp>(op, op.getType(), result);
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 3aa6ac3ea0918..51cc1232df891 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -62,10 +62,10 @@ static OpFoldResult getDimValue(OpBuilder &builder, Location loc, Value v,
   return getAsOpFoldResult(
       TypeSwitch<Type, Value>(v.getType())
           .Case<RankedTensorType>([&](RankedTensorType t) -> Value {
-            return builder.create<tensor::DimOp>(loc, v, dim);
+            return tensor::DimOp::create(builder, loc, v, dim);
           })
           .Case<MemRefType>([&](MemRefType t) -> Value {
-            return builder.create<memref::DimOp>(loc, v, dim);
+            return memref::DimOp::create(builder, loc, v, dim);
           }));
 }
 
@@ -77,11 +77,11 @@ static Operation *getSlice(OpBuilder &b, Location loc, Value source,
                            ArrayRef<OpFoldResult> strides) {
   return TypeSwitch<Type, Operation *>(source.getType())
       .Case<RankedTensorType>([&](RankedTensorType t) -> Operation * {
-        return b.create<tensor::ExtractSliceOp>(loc, source, offsets, sizes,
+        return tensor::ExtractSliceOp::create(b, loc, source, offsets, sizes,
                                                 strides);
       })
       .Case<MemRefType>([&](MemRefType type) -> Operation * {
-        return b.create<memref::SubViewOp>(loc, source, offsets, sizes,
+        return memref::SubViewOp::create(b, loc, source, offsets, sizes,
                                            strides);
       })
       .Default([&](Type t) -> Operation * { return nullptr; });
@@ -453,35 +453,35 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     switch (unaryFn) {
     case UnaryFn::exp:
-      return builder.create<math::ExpOp>(arg.getLoc(), arg);
+      return math::ExpOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::log:
-      return builder.create<math::LogOp>(arg.getLoc(), arg);
+      return math::LogOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::abs:
-      return builder.create<math::AbsFOp>(arg.getLoc(), arg);
+      return math::AbsFOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::ceil:
-      return builder.create<math::CeilOp>(arg.getLoc(), arg);
+      return math::CeilOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::floor:
-      return builder.create<math::FloorOp>(arg.getLoc(), arg);
+      return math::FloorOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::negf:
-      return builder.create<arith::NegFOp>(arg.getLoc(), arg);
+      return arith::NegFOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::reciprocal: {
       Attribute oneAttr = builder.getOneAttr(arg.getType());
-      auto one = builder.create<arith::ConstantOp>(arg.getLoc(),
+      auto one = arith::ConstantOp::create(builder, arg.getLoc(),
                                                    ::cast<TypedAttr>(oneAttr));
-      return builder.create<arith::DivFOp>(arg.getLoc(), one, arg);
+      return arith::DivFOp::create(builder, arg.getLoc(), one, arg);
     }
     case UnaryFn::round:
-      return builder.create<math::RoundOp>(arg.getLoc(), arg);
+      return math::RoundOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::sqrt:
-      return builder.create<math::SqrtOp>(arg.getLoc(), arg);
+      return math::SqrtOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::rsqrt:
-      return builder.create<math::RsqrtOp>(arg.getLoc(), arg);
+      return math::RsqrtOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::square:
-      return builder.create<arith::MulFOp>(arg.getLoc(), arg, arg);
+      return arith::MulFOp::create(builder, arg.getLoc(), arg, arg);
     case UnaryFn::tanh:
-      return builder.create<math::TanhOp>(arg.getLoc(), arg);
+      return math::TanhOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::erf:
-      return builder.create<math::ErfOp>(arg.getLoc(), arg);
+      return math::ErfOp::create(builder, arg.getLoc(), arg);
     }
     if (emitError) {
       emitError() << "unsupported unary function";
@@ -516,17 +516,17 @@ class RegionBuilderHelper {
     switch (binaryFn) {
     case BinaryFn::add:
       if (allComplex)
-        return builder.create<complex::AddOp>(arg0.getLoc(), arg0, arg1);
+        return complex::AddOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::AddFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::AddFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool)
-        return builder.create<arith::OrIOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::AddIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::OrIOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::AddIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::sub:
       if (allComplex)
-        return builder.create<complex::SubOp>(arg0.getLoc(), arg0, arg1);
+        return complex::SubOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::SubFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::SubFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool) {
         if (emitError) {
           emitError() << "unsupported operation: sub with bools";
@@ -534,20 +534,20 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: sub with bools");
       }
-      return builder.create<arith::SubIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::SubIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::mul:
       if (allComplex)
-        return builder.create<complex::MulOp>(arg0.getLoc(), arg0, arg1);
+        return complex::MulOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::MulFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MulFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool)
-        return builder.create<arith::AndIOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MulIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::AndIOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MulIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::div:
       if (allComplex)
-        return builder.create<complex::DivOp>(arg0.getLoc(), arg0, arg1);
+        return complex::DivOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::DivFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::DivFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool) {
         if (emitError) {
           emitError() << "unsupported operation: div with bools";
@@ -555,7 +555,7 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: div with bools");
       }
-      return builder.create<arith::DivSIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::DivSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::div_unsigned:
       if (!allInteger || allBool) {
         if (emitError) {
@@ -564,30 +564,30 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: unsigned div not on uint");
       }
-      return builder.create<arith::DivUIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::DivUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_signed:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MaximumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MaxSIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MaximumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MaxSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::min_signed:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MinimumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MinSIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MinimumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MinSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_unsigned:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MaximumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MaxUIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MaximumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MaxUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::min_unsigned:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MinimumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MinUIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MinimumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MinUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::powf:
       assert(allFloatingPoint);
-      return builder.create<math::PowFOp>(arg0.getLoc(), arg0, arg1);
+      return math::PowFOp::create(builder, arg0.getLoc(), arg0, arg1);
     }
     if (emitError) {
       emitError() << "unsupported binary function";
@@ -610,7 +610,7 @@ class RegionBuilderHelper {
     case TernaryFn::select:
       if (!headBool && !(tailFloatingPoint || tailInteger))
         llvm_unreachable("unsupported non numeric type");
-      return builder.create<arith::SelectOp>(arg0.getLoc(), arg0, arg1, arg2);
+      return arith::SelectOp::create(builder, arg0.getLoc(), arg0, arg1, arg2);
     }
     if (emitError) {
       emitError() << "unsupported ternary function";
@@ -639,7 +639,7 @@ class RegionBuilderHelper {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
-    builder.create<YieldOp>(loc, values);
+    YieldOp::create(builder, loc, values);
   }
 
   Value constant(const std::string &value) {
@@ -647,13 +647,13 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
     Attribute valueAttr = parseAttribute(value, builder.getContext());
-    return builder.create<arith::ConstantOp>(loc, ::cast<TypedAttr>(valueAttr));
+    return arith::ConstantOp::create(builder, loc, ::cast<TypedAttr>(valueAttr));
   }
 
   Value index(int64_t dim) {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
-    return builder.create<IndexOp>(builder.getUnknownLoc(), dim);
+    return IndexOp::create(builder, builder.getUnknownLoc(), dim);
   }
 
   Type getIntegerType(unsigned width) {
@@ -749,12 +749,12 @@ struct FoldFillWithTensorReshape : OpRewritePattern<TensorReshapeOp> {
     TensorReshapeOp newInit;
     if constexpr (std::is_same<TensorReshapeOp, tensor::ExpandShapeOp>::value) {
 
-      newInit = rewriter.create<TensorReshapeOp>(
+      newInit = TensorReshapeOp::create(rewriter,
           loc, reshapeOp.getResultType(), oldFill.output(),
           reshapeOp.getReassociation(), reshapeOp.getOutputShape(),
           reshapeOp.getStaticOutputShape());
     } else {
-      newInit = rewriter.create<TensorReshapeOp>(loc, reshapeOp.getResultType(),
+      newInit = TensorReshapeOp::create(rewriter, loc, reshapeOp.getResultType(),
                                                  oldFill.output(),
                                                  reshapeOp.getReassociation());
     }
@@ -786,7 +786,7 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
       return rewriter.notifyMatchFailure(
           padOp, "failed to reify tensor.pad op result shape");
 
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         padOp.getLoc(), reifiedShape.front(),
         padOp.getResultType().getElementType());
     Value replacement =
@@ -795,7 +795,7 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
                             ValueRange{emptyTensor})
             .getResult(0);
     if (replacement.getType() != padOp.getResultType()) {
-      replacement = rewriter.create<tensor::CastOp>(
+      replacement = tensor::CastOp::create(rewriter,
           fillOp.getLoc(), padOp.getResultType(), replacement);
     }
     rewriter.replaceOp(padOp, replacement);
@@ -889,7 +889,7 @@ struct FoldInsertPadIntoFill : public OpRewritePattern<tensor::InsertSliceOp> {
     for (int i = 0, e = srcPadType.getRank(); i < e; ++i) {
       if (srcPadType.isDynamicDim(i)) {
         newSizes.push_back(
-            rewriter.create<tensor::DimOp>(loc, srcPadOp.getSource(), i)
+            tensor::DimOp::create(rewriter, loc, srcPadOp.getSource(), i)
                 .getResult());
       } else {
         newSizes.push_back(rewriter.getIndexAttr(srcPadType.getDimSize(i)));
@@ -942,7 +942,7 @@ static FailureOr<FillOp> foldFillPackIntoFillOp(RewriterBase &rewriter,
   if (!packOpDest.hasOneUse())
     return failure();
 
-  return rewriter.create<linalg::FillOp>(packOp.getLoc(), fillOp.getInputs(),
+  return linalg::FillOp::create(rewriter, packOp.getLoc(), fillOp.getInputs(),
                                          packOp.getDest());
 }
 
@@ -1042,7 +1042,7 @@ struct FoldConcatsOfFill : public OpRewritePattern<tensor::ConcatOp> {
           concatOp, "not all operands are defined by a compatible fill op");
     }
 
-    Value outsConcat = rewriter.create<tensor::ConcatOp>(
+    Value outsConcat = tensor::ConcatOp::create(rewriter,
         concatOp.getLoc(), concatOp.getDim(), allOuts);
     rewriter.replaceOpWithNewOp<linalg::FillOp>(
         concatOp, firstFillOp.getDpsInputOperand(0)->get(), outsConcat);
@@ -1407,13 +1407,13 @@ struct EraseIdentityLinalgOp : public OpRewritePattern<OpTy> {
         // TODO: unify the two ops?
         if (sparse_tensor::getSparseTensorEncoding(returnType) ||
             sparse_tensor::getSparseTensorEncoding(resultType))
-          returnedArg = rewriter.create<sparse_tensor::ConvertOp>(
+          returnedArg = sparse_tensor::ConvertOp::create(rewriter,
               linalgOp.getLoc(), resultType, returnedArg);
         else {
           if (!tensor::CastOp::areCastCompatible(returnedArg.getType(),
                                                  resultType))
             return failure();
-          returnedArg = rewriter.create<tensor::CastOp>(
+          returnedArg = tensor::CastOp::create(rewriter,
               linalgOp.getLoc(), resultType, returnedArg);
         }
       }
@@ -1528,7 +1528,7 @@ static void addBodyWithPayloadOp(OpAsmParser &parser, OperationState &result,
       TypeRange{llvm::cast<ShapedType>(result.operands.back().getType())
                     .getElementType()},
       payloadOpAttrs);
-  b.create<YieldOp>(result.location, payloadOp->getResults());
+  YieldOp::create(b, result.location, payloadOp->getResults());
 }
 
 ParseResult MapOp::parse(OpAsmParser &parser, OperationState &result) {
@@ -1945,7 +1945,7 @@ static void buildIdentityRegion(OpBuilder &builder, Location loc,
   buildGenericRegion(builder, loc, region, inputs, outputs,
                      [](OpBuilder &b, Location loc, ValueRange args) {
                        if (!args.empty())
-                         b.create<linalg::YieldOp>(loc, args[0]);
+                         linalg::YieldOp::create(b, loc, args[0]);
                      });
 }
 
@@ -2138,7 +2138,7 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     unsigned inputRank = broadcastInputTy.getRank();
     for (unsigned i = 0; i < inputRank; ++i) {
       if (broadcastInputTy.isDynamicDim(i)) {
-        dims.push_back(rewriter.create<tensor::DimOp>(loc, broadcastInput, i)
+        dims.push_back(tensor::DimOp::create(rewriter, loc, broadcastInput, i)
                            ->getResult(0));
       } else {
         dims.push_back(IntegerAttr::get(IndexType::get(ctx),
@@ -2147,7 +2147,7 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     }
     SmallVector<OpFoldResult> transposeResultShapes =
         applyPermutation(dims, resultPerms);
-    Value transposeInit = rewriter.create<tensor::EmptyOp>(
+    Value transposeInit = tensor::EmptyOp::create(rewriter,
         transposeOp.getLoc(), transposeResultShapes,
         broadcastInputTy.getElementType());
 
@@ -2547,7 +2547,7 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     // continue to propagate as far up the stack as it can go.
     OpOperand *outOperand = linalgOp.getDpsInitOperand(resultNumber);
     Value newOperand =
-        rewriter.create<tensor::CastOp>(loc, resultType, outOperand->get());
+        tensor::CastOp::create(rewriter, loc, resultType, outOperand->get());
     SmallVector<Value> newOperands = linalgOp.getDpsInputs();
     SmallVector<Value> outputOperands(linalgOp.getDpsInits().begin(),
                                       linalgOp.getDpsInits().end());
@@ -2560,7 +2560,7 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     Operation *newOp = clone(rewriter, linalgOp, resultTypes, newOperands);
 
     // Create a tensor.cast operation back to the original type.
-    Value castBack = rewriter.create<tensor::CastOp>(
+    Value castBack = tensor::CastOp::create(rewriter,
         loc, resultValue.getType(), newOp->getResult(resultNumber));
 
     SmallVector<Value> results(newOp->result_begin(), newOp->result_end());
@@ -2653,7 +2653,7 @@ static void createNewOperandWithStaticSizes(
     changeNeeded = true;
     // Get the new operand value given its size and element type by
     // casting it.
-    Value newOperand = rewriter.create<tensor::CastOp>(loc, resultType, src);
+    Value newOperand = tensor::CastOp::create(rewriter, loc, resultType, src);
     unsigned index = opOperand->getOperandNumber();
     newOperands[index] = newOperand;
   }
@@ -2718,7 +2718,7 @@ struct InferStaticShapeOfOperands : public OpInterfaceRewritePattern<LinalgOp> {
       Type oldType = oldResult.getType();
       replacements.push_back(
           (newType != oldType)
-              ? rewriter.create<tensor::CastOp>(loc, oldType, newResult)
+              ? tensor::CastOp::create(rewriter, loc, oldType, newResult)
               : newResult);
     }
     rewriter.replaceOp(linalgOp, replacements);
@@ -2756,8 +2756,8 @@ SmallVector<Range> SoftmaxOp::getIterationDomain(OpBuilder &builder) {
   int64_t operandRank = getInputOperandRank();
   SmallVector<Range> loopBounds(operandRank);
   Location loc = getLoc();
-  Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-  Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+  Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+  Value one = arith::ConstantIndexOp::create(builder, loc, 1);
   Value source = getInput();
   for (auto dim : llvm::seq<int64_t>(0, operandRank)) {
     loopBounds[dim].offset = zero;
@@ -2924,11 +2924,11 @@ static Value reduce(OpBuilder &builder, Location loc, Value input, Value output,
          "We should have two maps: 1 for the input, 1 for the output");
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
 
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, output.getType(), input, output, indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value result = b.create<T>(loc, args[0], args[1]);
-        b.create<linalg::YieldOp>(loc, result);
+        Value result = T::create(b, loc, args[0], args[1]);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -2947,12 +2947,12 @@ static Value buildSubAndExpOp(OpBuilder &builder, Location loc, Value input,
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
   // Add the affine map for the output argument.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, input.getType(), ValueRange{input, max}, output, indexingMaps,
       iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value diff = b.create<arith::SubFOp>(loc, args[0], args[1]);
-        Value result = b.create<math::ExpOp>(loc, diff);
-        b.create<linalg::YieldOp>(loc, result);
+        Value diff = arith::SubFOp::create(b, loc, args[0], args[1]);
+        Value result = math::ExpOp::create(b, loc, diff);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -2974,12 +2974,12 @@ static Value buildDivOp(OpBuilder &builder, Location loc, Value numerator,
   assert(indexingMaps[0].isIdentity() && "Numerator map should be identity");
   // Add the affine map for the output tensor.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, numerator.getType(), ValueRange{numerator, denominator}, output,
       indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value result = b.create<arith::DivFOp>(loc, args[0], args[1]);
-        b.create<linalg::YieldOp>(loc, result);
+        Value result = arith::DivFOp::create(b, loc, args[0], args[1]);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -3015,12 +3015,12 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
   Value output = getOutput();
   dims.erase(dims.begin() + reductionDim);
   // Step 1: Compute max along dim.
-  Value outputReduce = b.create<tensor::EmptyOp>(loc, dims, elementType);
+  Value outputReduce = tensor::EmptyOp::create(b, loc, dims, elementType);
   Value neutralForMaxF = arith::getIdentityValue(arith::AtomicRMWKind::maxnumf,
                                                  elementType, b, loc,
                                                  /*useOnlyFiniteValue=*/true);
   Value neutralForMaxFInit =
-      b.create<linalg::FillOp>(loc, Value{neutralForMaxF}, outputReduce)
+      linalg::FillOp::create(b, loc, Value{neutralForMaxF}, outputReduce)
           .result();
   Value max =
       reduce<arith::MaxNumFOp>(b, loc, input, neutralForMaxFInit, reductionDim);
@@ -3032,7 +3032,7 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
   Value zero = arith::getIdentityValue(arith::AtomicRMWKind::addf, elementType,
                                        b, loc, /*useOnlyFiniteValue=*/true);
   Value zeroInit =
-      b.create<linalg::FillOp>(loc, Value{zero}, outputReduce).result();
+      linalg::FillOp::create(b, loc, Value{zero}, outputReduce).result();
   Value denominator =
       reduce<arith::AddFOp>(b, loc, numerator, zeroInit, reductionDim);
 
@@ -3153,7 +3153,7 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
   int64_t filterRank = getFilterOperandRank();
   SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
   Location loc = getLoc();
-  auto filterSlice = builder.create<tensor::ExtractSliceOp>(
+  auto filterSlice = tensor::ExtractSliceOp::create(builder,
       loc, getFilter(), sliceOffsets, sliceSizes, filterStrides);
   tiledOperands.emplace_back(filterSlice);
 
@@ -3164,7 +3164,7 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -3333,7 +3333,7 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
       {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
   int64_t inputRank = getInputOperandRank();
   SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
-  auto inputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto inputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getInput(), sliceOffsets, sliceSizes, inputStrides);
   tiledOperands.emplace_back(inputSlice);
 
@@ -3344,7 +3344,7 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -3504,7 +3504,7 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
                      sizes[getValueFDim()]});
   int64_t valueRank = getValueOperandRank();
   SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
-  auto valueSlice = builder.create<tensor::ExtractSliceOp>(
+  auto valueSlice = tensor::ExtractSliceOp::create(builder,
       loc, getValue(), sliceOffsets, sliceSizes, sliceStrides);
   tiledOperands.emplace_back(valueSlice);
 
@@ -3515,7 +3515,7 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> strides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, strides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -4971,7 +4971,7 @@ Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
            llvm::cast<RankedTensorType>(source.getType()).getShape())) {
     if (ShapedType::isDynamic(value))
       mixedSizes.push_back(
-          b.create<tensor::DimOp>(loc, source, index).getResult());
+          tensor::DimOp::create(b, loc, source, index).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(value));
   }
@@ -4985,7 +4985,7 @@ Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
 
   mixedSizes.append(innerTileSizes.begin(), innerTileSizes.end());
   auto elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
-  return b.create<tensor::EmptyOp>(loc, mixedSizes, elemType);
+  return tensor::EmptyOp::create(b, loc, mixedSizes, elemType);
 }
 
 PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
@@ -4996,7 +4996,7 @@ PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
   Value transposedDest =
       createDestinationTensor(b, loc, getSource(), metadata.innerTiles,
                               metadata.innerDimsPos, metadata.outerDimsPerm);
-  return b.create<PackOp>(loc, getSource(), transposedDest,
+  return PackOp::create(b, loc, getSource(), transposedDest,
                           metadata.innerDimsPos, metadata.innerTiles,
                           getPaddingValue(), metadata.outerDimsPerm);
 }
@@ -5138,7 +5138,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (srcShape != packOp.getSourceType().getShape()) {
       auto newSrcType = packOp.getSourceType().clone(srcShape);
       source =
-          rewriter.create<tensor::CastOp>(loc, newSrcType, packOp.getSource());
+          tensor::CastOp::create(rewriter, loc, newSrcType, packOp.getSource());
     }
     Value dest = packOp.getDest();
     RankedTensorType originalResultType = packOp.getDestType();
@@ -5146,7 +5146,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (needUpdateDestType) {
       auto newDestType = packOp.getDestType().clone(destShape);
       dest =
-          rewriter.create<tensor::CastOp>(loc, newDestType, packOp.getDest());
+          tensor::CastOp::create(rewriter, loc, newDestType, packOp.getDest());
     }
     rewriter.modifyOpInPlace(packOp, [&] {
       packOp.getSourceMutable().assign(source);
@@ -5157,7 +5157,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (needUpdateDestType) {
       rewriter.setInsertionPointAfter(packOp);
       auto castOp =
-          rewriter.create<tensor::CastOp>(loc, originalResultType, packOp);
+          tensor::CastOp::create(rewriter, loc, originalResultType, packOp);
       rewriter.replaceAllUsesExcept(packOp, castOp, castOp);
     }
     return success();
@@ -5250,7 +5250,7 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    PackOp newOp = rewriter.create<PackOp>(
+    PackOp newOp = PackOp::create(rewriter,
         op.getLoc(), newOperands[0], newOperands[1], op.getInnerDimsPos(),
         newMixedTileSizes, op.getPaddingValue(), op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
@@ -5259,7 +5259,7 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
     Value replacement = (newResult.getType() != oldResult.getType())
-                            ? rewriter.create<tensor::CastOp>(
+                            ? tensor::CastOp::create(rewriter,
                                   op->getLoc(), oldResult.getType(), newResult)
                             : newResult;
 
@@ -5358,7 +5358,7 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
   for (auto i :
        llvm::seq<unsigned>(0, srcType.getRank() - innerTileSizes.size())) {
     if (srcType.isDynamicDim(i))
-      mixedSizes.push_back(b.create<tensor::DimOp>(loc, source, i).getResult());
+      mixedSizes.push_back(tensor::DimOp::create(b, loc, source, i).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(srcType.getDimSize(i)));
   }
@@ -5371,7 +5371,7 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
     mixedSizes[dimPos] = dimMul(mixedSizes[dimPos], tileSize);
 
   auto elemType = srcType.getElementType();
-  return b.create<tensor::EmptyOp>(loc, mixedSizes, elemType);
+  return tensor::EmptyOp::create(b, loc, mixedSizes, elemType);
 }
 
 UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
@@ -5380,7 +5380,7 @@ UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
                                          ArrayRef<int64_t> outerPermutation) {
   PackOrUnPackTransposeResult metadata = commonPermutationOfPackAndUnPackOp(
       *this, innerPermutation, outerPermutation);
-  return b.create<UnPackOp>(loc, transposedSource, getDest(),
+  return UnPackOp::create(b, loc, transposedSource, getDest(),
                             metadata.innerDimsPos, metadata.innerTiles,
                             metadata.outerDimsPerm);
 }
@@ -5454,7 +5454,7 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
             extractSliceUser.getResultType().getRank()) {
       OpBuilder::InsertionGuard g(rewriter);
       rewriter.setInsertionPoint(unPackOp);
-      auto newDest = rewriter.create<tensor::ExtractSliceOp>(
+      auto newDest = tensor::ExtractSliceOp::create(rewriter,
           unPackOp->getLoc(), unPackOp.getDest(),
           extractSliceUser.getMixedOffsets(), extractSliceUser.getMixedSizes(),
           extractSliceUser.getMixedStrides());
@@ -5474,16 +5474,16 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
     Value source = unPackOp.getSource();
     if (srcShape != unPackOp.getSourceType().getShape()) {
       auto newSrcType = unPackOp.getSourceType().clone(srcShape);
-      source = rewriter.create<tensor::CastOp>(loc, newSrcType,
+      source = tensor::CastOp::create(rewriter, loc, newSrcType,
                                                unPackOp.getSource());
     }
     Value dest = unPackOp.getDest();
     if (destShape != unPackOp.getDestType().getShape()) {
       auto newDestType = unPackOp.getDestType().clone(destShape);
       dest =
-          rewriter.create<tensor::CastOp>(loc, newDestType, unPackOp.getDest());
+          tensor::CastOp::create(rewriter, loc, newDestType, unPackOp.getDest());
     }
-    Value newOp = rewriter.create<UnPackOp>(
+    Value newOp = UnPackOp::create(rewriter,
         loc, source, dest, unPackOp.getInnerDimsPos(), unPackOp.getMixedTiles(),
         unPackOp.getOuterDimsPerm());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(
@@ -5542,7 +5542,7 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    UnPackOp newOp = rewriter.create<UnPackOp>(
+    UnPackOp newOp = UnPackOp::create(rewriter,
         op.getLoc(), sourceTensor, newOperands[1], op.getInnerDimsPos(),
         newMixedTileSizes, op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
@@ -5551,7 +5551,7 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
     Value replacement = (newResult.getType() != oldResult.getType())
-                            ? rewriter.create<tensor::CastOp>(
+                            ? tensor::CastOp::create(rewriter,
                                   op->getLoc(), oldResult.getType(), newResult)
                             : newResult;
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 5d5f9de465561..ba03d09e5072e 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -672,7 +672,7 @@ static Operation *replaceForAllWithNewSignature(
   newOuts.push_back(outputs[resultNumber]);
 
   // Create new scf.forall op
-  auto newforallOp = rewriter.create<scf::ForallOp>(
+  auto newforallOp = scf::ForallOp::create(rewriter,
       loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
       forallOp.getMixedStep(), newOuts, forallOp.getMapping());
   rewriter.eraseBlock(newforallOp.getBody());
@@ -699,7 +699,7 @@ static Operation *replaceForAllWithNewSignature(
   Value src = tileAndFuseResult.tiledValues[0];
   Value dst = newforallOp.getRegionIterArgs().back();
   SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-  rewriter.create<tensor::ParallelInsertSliceOp>(firstYieldOp->getLoc(), src,
+  tensor::ParallelInsertSliceOp::create(rewriter, firstYieldOp->getLoc(), src,
                                                  dst, offsets, sizes, strides);
 
   for (auto result : llvm::enumerate(forallOp.getResults())) {
@@ -3410,12 +3410,12 @@ transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
         for (auto [ofrIdx, ofr] : llvm::enumerate(getMixedSizes())) {
           if (auto attr = llvm::dyn_cast_if_present<Attribute>(ofr)) {
             if (scalableSizes[ofrIdx]) {
-              auto val = b.create<arith::ConstantIndexOp>(
+              auto val = arith::ConstantIndexOp::create(b,
                   getLoc(), cast<IntegerAttr>(attr).getInt());
               Value vscale =
-                  b.create<vector::VectorScaleOp>(getLoc(), b.getIndexType());
+                  vector::VectorScaleOp::create(b, getLoc(), b.getIndexType());
               sizes.push_back(
-                  b.create<arith::MulIOp>(getLoc(), val, vscale).getResult());
+                  arith::MulIOp::create(b, getLoc(), val, vscale).getResult());
             } else {
               sizes.push_back(attr);
             }
@@ -3626,7 +3626,7 @@ static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter,
   SmallVector<OpFoldResult> normalizedSteps(normalizedUbs.size(),
                                             rewriter.getIndexAttr(1));
 
-  auto normalizedForallOp = rewriter.create<scf::ForallOp>(
+  auto normalizedForallOp = scf::ForallOp::create(rewriter,
       loc, normalizedLbs, normalizedUbs, normalizedSteps, loop.getOutputs(),
       loop.getMapping(), [](OpBuilder &, Location, ValueRange) {});
 
@@ -4128,7 +4128,7 @@ DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target,
         target->template getParentOfType<scf::InParallelOp>());
   }
 
-  Value extracted = rewriter.create<tensor::ExtractSliceOp>(
+  Value extracted = tensor::ExtractSliceOp::create(rewriter,
       target.getLoc(), target.getDest(), target.getMixedOffsets(),
       target.getMixedSizes(), target.getMixedStrides());
   Value copied = rewriter
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
index 1f6d96ca0f81f..2cb8ad367f93c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -184,7 +184,7 @@ struct SoftmaxOpInterface
         getBuffer(rewriter, softmaxOp.getOutput(), options, state);
     if (failed(outputBuffer))
       return failure();
-    rewriter.create<linalg::SoftmaxOp>(softmaxOp.getLoc(),
+    linalg::SoftmaxOp::create(rewriter, softmaxOp.getLoc(),
                                        /*result=*/TypeRange(), *inputBuffer,
                                        *outputBuffer, softmaxOp.getDimension());
     replaceOpWithBufferizedValues(rewriter, op, *outputBuffer);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index a7732b939e70d..61c779850366f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -30,10 +30,10 @@ static bool hasAllOneValues(DenseIntElementsAttr attr) {
 
 static Value createAdd(Location loc, Value x, Value y, OpBuilder &builder) {
   if (isa<IntegerType>(x.getType()))
-    return builder.create<arith::AddIOp>(loc, x, y);
+    return arith::AddIOp::create(builder, loc, x, y);
   if (isa<ComplexType>(x.getType()))
-    return builder.create<complex::AddOp>(loc, x, y);
-  return builder.create<arith::AddFOp>(loc, x, y);
+    return complex::AddOp::create(builder, loc, x, y);
+  return arith::AddFOp::create(builder, loc, x, y);
 }
 
 static Value createMul(Location loc, Value x, Value y, Type accType,
@@ -44,10 +44,10 @@ static Value createMul(Location loc, Value x, Value y, Type accType,
   Value yConvert =
       convertScalarToDtype(builder, loc, y, accType, /*isUnsignedCast=*/false);
   if (isa<ComplexType>(accType))
-    return builder.create<complex::MulOp>(loc, xConvert, yConvert);
+    return complex::MulOp::create(builder, loc, xConvert, yConvert);
   if (isa<IntegerType>(accType))
-    return builder.create<arith::MulIOp>(loc, xConvert, yConvert);
-  return builder.create<arith::MulFOp>(loc, xConvert, yConvert);
+    return arith::MulIOp::create(builder, loc, xConvert, yConvert);
+  return arith::MulFOp::create(builder, loc, xConvert, yConvert);
 }
 
 // Delinearizes the given composite `index` by the basis specified in `factors`.
@@ -56,7 +56,7 @@ static SmallVector<Value> unrollIndex(OpBuilder &b, Location loc, Value index,
   assert(!factors.empty() && "empty factor list");
   SmallVector<Value> basis;
   for (int64_t f : factors)
-    basis.push_back(b.create<arith::ConstantOp>(loc, b.getIndexAttr(f)));
+    basis.push_back(arith::ConstantOp::create(b, loc, b.getIndexAttr(f)));
   FailureOr<SmallVector<Value>> multiIndex =
       affine::delinearizeIndex(b, loc, index, basis);
   assert(!failed(multiIndex) && "Failed to linearize img2col index");
@@ -115,17 +115,17 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0, 1, 2}, {3}};
   auto reshapedFilterType =
       RankedTensorType::get({fh * fw * ic, oc}, filterType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
@@ -138,15 +138,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value mIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> mIndices = unrollIndex(
@@ -170,9 +170,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because the filter does not share the same batch dimension,
@@ -187,7 +187,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -196,11 +196,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
@@ -244,7 +244,7 @@ rewriteInIm2Col(RewriterBase &rewriter,
     SmallVector<int64_t> targetShape = llvm::to_vector<4>(llvm::map_range(
         indices, [&](int64_t index) -> int64_t { return inputShape[index]; }));
 
-    Value outputTensor = rewriter.create<tensor::EmptyOp>(
+    Value outputTensor = tensor::EmptyOp::create(rewriter,
         loc, targetShape, operandTensorType.getElementType());
 
     SmallVector<utils::IteratorType> loopAttributeTypes(
@@ -255,12 +255,12 @@ rewriteInIm2Col(RewriterBase &rewriter,
             AffineMap::get(nloops, 0, exprs, rewriter.getContext())),
         AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-    auto transposedOp = rewriter.create<linalg::GenericOp>(
+    auto transposedOp = linalg::GenericOp::create(rewriter,
         loc, outputTensor.getType(),
         /*inputs=*/operand, /*outputs=*/outputTensor, indexingMaps,
         loopAttributeTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-          nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+          linalg::YieldOp::create(nestedBuilder, nestedLoc, args[0]);
         });
 
     return transposedOp.getResult(0);
@@ -307,15 +307,15 @@ rewriteInIm2Col(RewriterBase &rewriter,
       AffineMap::get(nloops, 0, inputExprs, rewriter.getContext()),
       AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/inputT, /*outputs=*/colTensor, indexingMaps,
       loopAttributeTypes,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, args[0]);
       });
 
   SmallVector<ReassociationIndices> img2ColTensorReassocIndices = {
@@ -331,16 +331,16 @@ rewriteInIm2Col(RewriterBase &rewriter,
   auto reshapedOutputTensorType =
       RankedTensorType::get({n * c, oh * ow}, outputType.getElementType());
 
-  Value reshapedImg2ColTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedImg2ColTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedImg2ColTensorType, img2ColTensor.getResult(0),
       img2ColTensorReassocIndices);
-  Value reshapedFilterTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilterTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterTensorType, filterT, filterReassociationIndice);
-  Value reshapedoutputTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedoutputTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputTensorType, transposedOutputTensor,
       outputReassociationIndice);
 
-  auto batchMatVecResult = rewriter.create<linalg::BatchMatvecOp>(
+  auto batchMatVecResult = linalg::BatchMatvecOp::create(rewriter,
       loc, TypeRange{reshapedoutputTensor.getType()},
       ValueRange{reshapedImg2ColTensor, reshapedFilterTensor},
       ValueRange{reshapedoutputTensor});
@@ -348,7 +348,7 @@ rewriteInIm2Col(RewriterBase &rewriter,
   SmallVector<ReassociationIndices> batchMatVecReassociationIndice = {{0, 1},
                                                                       {2, 3}};
 
-  auto batchMatVecResultReshaped = rewriter.create<tensor::ExpandShapeOp>(
+  auto batchMatVecResultReshaped = tensor::ExpandShapeOp::create(rewriter,
       loc, transposedOutputTensor.getType(), batchMatVecResult.getResult(0),
       batchMatVecReassociationIndice);
 
@@ -400,18 +400,18 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, ic * fh * fw}, inputType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1}, {2, 3}};
   auto reshapedOutputType =
       RankedTensorType::get({n, oc, oh * ow}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   // Convert the input to a (BKN) tensor.
   SmallVector<int64_t, 4> colTensorShape = {n, ic * fh * fw, oh * ow};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   auto nloops = colTensorShape.size();
@@ -423,15 +423,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<AffineMap, 4> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value nIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value nIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> kIndices = unrollIndex(
@@ -455,9 +455,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
 
         // im2col[n, ic*fh*fw, oh*ow] = input[n, ic, sh*oh + fh, sw*ow + fw]
         SmallVector<Value> extractionIndices{bIndex, icIndex, hIndex, wIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because the filter does not share the same batch dimension,
@@ -471,7 +471,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   auto resultMap = AffineMap::get(4, 0, {bDim, mDim, nDim}, context);
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{reshapedFilter, img2ColTensor.getResult(0)},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -480,11 +480,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
@@ -535,17 +535,17 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, fh * fw * ic}, filterType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
@@ -558,15 +558,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value mIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> mIndices = unrollIndex(
@@ -590,9 +590,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because we didn't transpose the filters we don't actually have a batched
@@ -606,7 +606,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -615,11 +615,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index 39e2aac27e213..f21563c323c1b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -37,7 +37,7 @@ static Value createInserts(RewriterBase &rewriter, Location loc, int dim,
   if (dim == static_cast<int>(shape.size()) - 1) {
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
-      destination = rewriter.create<tensor::InsertOp>(loc, *elementIt,
+      destination = tensor::InsertOp::create(rewriter, loc, *elementIt,
                                                       destination, indices);
       ++elementIt;
     }
@@ -65,7 +65,7 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
       MaterializeInDestination: {
     // Note: This is the preferred way of memcpy'ing because no layout map
     // and/or memory space must be specified for the source.
-    auto materializeOp = b.create<bufferization::MaterializeInDestinationOp>(
+    auto materializeOp = bufferization::MaterializeInDestinationOp::create(b,
         loc, tensorSource, memrefDest);
     materializeOp.setWritable(true);
   } break;
@@ -73,19 +73,19 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = b.create<bufferization::ToBufferOp>(
+    Value toBuffer = bufferization::ToBufferOp::create(b,
         loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
-    b.create<memref::CopyOp>(loc, toBuffer, memrefDest);
+    memref::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
   case linalg::BufferizeToAllocationOptions::MemcpyOp::LinalgCopy: {
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = b.create<bufferization::ToBufferOp>(
+    Value toBuffer = bufferization::ToBufferOp::create(b,
         loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
-    b.create<linalg::CopyOp>(loc, toBuffer, memrefDest);
+    linalg::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
   };
 }
@@ -120,14 +120,14 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
             ->materializeConstant(rewriter, constYieldedValue,
                                   yieldedValue.getType(), yieldedValue.getLoc())
             ->getResult(0);
-    auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(fillValue),
+    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(fillValue),
                                                   ValueRange(dest));
     return fillOp;
   }
 
   if (invariantYieldedValue) {
     // Padding with an invariant value.
-    auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(yieldedValue),
+    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(yieldedValue),
                                                   ValueRange(dest));
     return fillOp;
   }
@@ -137,7 +137,7 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(resultType.getRank()));
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, resultType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{dest}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
@@ -146,7 +146,7 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
   rewriter.setInsertionPointToStart(body);
   SmallVector<Value> bbArgReplacements;
   for (int64_t i = 0; i < resultType.getRank(); ++i)
-    bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
+    bbArgReplacements.push_back(linalg::IndexOp::create(rewriter, loc, i));
   rewriter.mergeBlocks(padOp.getBody(), body, bbArgReplacements);
 
   // Update terminator.
@@ -179,8 +179,8 @@ static SmallVector<Value> reifyOrComputeDynamicSizes(OpBuilder &b,
   for (int64_t i = 0; i < tensorType.getRank(); ++i) {
     if (tensorType.isDynamicDim(i))
       dynSizes.push_back(
-          b.create<DimOp>(value.getLoc(), value,
-                          b.create<arith::ConstantIndexOp>(value.getLoc(), i)));
+          DimOp::create(b, value.getLoc(), value,
+                          arith::ConstantIndexOp::create(b, value.getLoc(), i)));
   }
   return dynSizes;
 }
@@ -201,15 +201,15 @@ createAllocationForTensor(RewriterBase &rewriter, Location loc, Value value,
   Value alloc;
   if (options.allocOp ==
       linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloc) {
-    alloc = rewriter.create<memref::AllocOp>(loc, memrefType, dynamicSizes);
+    alloc = memref::AllocOp::create(rewriter, loc, memrefType, dynamicSizes);
     if (options.emitDealloc) {
       // Place deallocation at the end of the block.
       rewriter.setInsertionPoint(rewriter.getInsertionBlock()->getTerminator());
-      rewriter.create<memref::DeallocOp>(loc, alloc);
+      memref::DeallocOp::create(rewriter, loc, alloc);
     }
   } else if (options.allocOp ==
              linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloca) {
-    alloc = rewriter.create<memref::AllocaOp>(loc, memrefType, dynamicSizes);
+    alloc = memref::AllocaOp::create(rewriter, loc, memrefType, dynamicSizes);
     // No dealloc is needed.
   }
 
@@ -243,13 +243,13 @@ Value linalg::bufferizeToAllocation(
       getMixedSizes(rewriter, loc, padOp.getSource());
   SmallVector<OpFoldResult> strides(padOp.getResultType().getRank(),
                                     rewriter.getIndexAttr(1));
-  Value subview = rewriter.create<memref::SubViewOp>(
+  Value subview = memref::SubViewOp::create(rewriter,
       loc, alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
   createMemcpy(rewriter, loc, padOp.getSource(), subview, options);
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
       loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(padOp, toTensorOp);
@@ -338,7 +338,7 @@ Value linalg::bufferizeToAllocation(
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
       loc, allocTensorOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(allocTensorOp, toTensorOp);
@@ -354,7 +354,7 @@ FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
   auto shape = tensorType.getShape();
 
   // Create tensor.empty.
-  auto emptyOp = rewriter.create<EmptyOp>(loc, tensorType, ValueRange());
+  auto emptyOp = EmptyOp::create(rewriter, loc, tensorType, ValueRange());
 
   // Case: tensor<elem_type>.
   if (shape.empty()) {
@@ -369,7 +369,7 @@ FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
   SmallVector<Value, 2> constants;
   constants.reserve(maxDim);
   for (int i = 0; i < maxDim; ++i)
-    constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
+    constants.push_back(arith::ConstantIndexOp::create(rewriter, loc, i));
 
   // Traverse all elements and create tensor.insert ops.
   auto elementIt = fromElementsOp.getElements().begin();
@@ -395,14 +395,14 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
 
   // Create tensor.empty.
   auto emptyOp =
-      rewriter.create<EmptyOp>(loc, tensorType, generateOp.getDynamicExtents());
+      EmptyOp::create(rewriter, loc, tensorType, generateOp.getDynamicExtents());
 
   // Create linalg.generic.
   SmallVector<utils::IteratorType> iteratorTypes(tensorType.getRank(),
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(tensorType.getRank()));
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, tensorType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{emptyOp.getResult()}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
@@ -411,7 +411,7 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
   rewriter.setInsertionPointToStart(body);
   SmallVector<Value> bbArgReplacements;
   for (int64_t i = 0; i < tensorType.getRank(); ++i)
-    bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
+    bbArgReplacements.push_back(linalg::IndexOp::create(rewriter, loc, i));
   rewriter.mergeBlocks(&generateOp.getBody().front(), body, bbArgReplacements);
 
   // Update terminator.
@@ -450,13 +450,13 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
       llvm::all_of(padOp.getMixedHighPad(), isZeroInteger)) {
     using bufferization::AllocTensorOp;
     Value allocated =
-        rewriter.create<AllocTensorOp>(loc, resultType, dynamicSizes);
+        AllocTensorOp::create(rewriter, loc, resultType, dynamicSizes);
     auto copyOp = rewriter.replaceOpWithNewOp<linalg::CopyOp>(
         padOp, padOp.getSource(), allocated);
     return copyOp.getOperation();
   }
 
-  Value empty = rewriter.create<EmptyOp>(loc, resultType, dynamicSizes);
+  Value empty = EmptyOp::create(rewriter, loc, resultType, dynamicSizes);
   // Create linalg.fill or linalg.generic.
   Operation *fillOp = movePaddingToFillOrGenericOp(rewriter, loc, padOp, empty);
   rewriter.setInsertionPointAfter(fillOp);
@@ -567,7 +567,7 @@ Value linalg::bufferizeToAllocation(
       createMemcpy(rewriter, op->getLoc(), operand->get(), alloc, options);
     }
     rewriter.modifyOpInPlace(op, [&]() {
-      auto toTensorOp = rewriter.create<ToTensorOp>(
+      auto toTensorOp = ToTensorOp::create(rewriter,
           op->getLoc(), operand->get().getType(), alloc);
       operand->set(toTensorOp);
       if (options.bufferizeDestinationOnly) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index 70574903f7111..eb2c7756dc6ae 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -287,7 +287,7 @@ getOrCreatePackedViewOfOperand(OpBuilder &b, Location loc, PackInfo packInfo,
 
   auto empty = linalg::PackOp::createDestinationTensor(
       b, loc, opOperand->get(), innerTileSizes, innerDimsPos, outerDimsPerm);
-  auto packedOperand = b.create<linalg::PackOp>(
+  auto packedOperand = linalg::PackOp::create(b,
       loc, opOperand->get(), empty, innerDimsPos, innerTileSizes,
       /*padding=*/std::nullopt, outerDimsPerm);
   return std::make_tuple(packedOperand, indexingMap);
@@ -345,7 +345,7 @@ static GenericOp packGenericOp(RewriterBase &rewriter, GenericOp genericOp,
 
   indexingMaps.push_back(packedOutIndexingMap);
 
-  auto newGenericOp = rewriter.create<linalg::GenericOp>(
+  auto newGenericOp = linalg::GenericOp::create(rewriter,
       loc, dest.getType(), inputOperands, dest, indexingMaps, iterTypes,
       /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
   rewriter.cloneRegionBefore(genericOp.getRegion(), newGenericOp.getRegion(),
@@ -457,7 +457,7 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, linalg::PackOp packOp,
   if (!packOpDest.hasOneUse())
     return failure();
   if (auto emptyOp = packOpDest.getDefiningOp<tensor::EmptyOp>()) {
-    packOpDest = rewriter.create<tensor::EmptyOp>(
+    packOpDest = tensor::EmptyOp::create(rewriter,
         genericOp->getLoc(), emptyOp.getMixedSizes(),
         emptyOp.getType().getElementType());
   } else {
@@ -562,7 +562,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     auto empty = linalg::PackOp::createDestinationTensor(
         rewriter, loc, padOp.getSource(), mixedTiles, innerDimsPos,
         outerDimsPerm);
-    auto sourcePack = rewriter.create<linalg::PackOp>(
+    auto sourcePack = linalg::PackOp::create(rewriter,
         loc, padOp.getSource(), empty, innerDimsPos, mixedTiles,
         /*padding=*/std::nullopt, outerDimsPerm);
 
@@ -579,7 +579,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, /*result=*/Type(), sourcePack, lowPad, highPad, paddingVal,
         padOp.getNofold());
 
@@ -588,7 +588,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     if (!padOp->hasOneUse()) {
       auto unpackEmpty = linalg::UnPackOp::createDestinationTensor(
           rewriter, loc, newPadOp, mixedTiles, innerDimsPos, outerDimsPerm);
-      Value unpackedPad = rewriter.create<linalg::UnPackOp>(
+      Value unpackedPad = linalg::UnPackOp::create(rewriter,
           loc, newPadOp, unpackEmpty, innerDimsPos, mixedTiles, outerDimsPerm);
       rewriter.replaceAllUsesExcept(padOp, unpackedPad, sourcePack);
     }
@@ -719,7 +719,7 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
   auto emptyOp = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), collapseOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newPackOp = rewriter.create<linalg::PackOp>(
+  auto newPackOp = linalg::PackOp::create(rewriter,
       packOp.getLoc(), collapseOp.getSrc(), emptyOp, projectedInnerDimsPos,
       packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPerm);
 
@@ -735,7 +735,7 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
     nextPos += 1;
   }
 
-  auto newCollapseOp = rewriter.create<tensor::CollapseShapeOp>(
+  auto newCollapseOp = tensor::CollapseShapeOp::create(rewriter,
       collapseOp.getLoc(), packOp.getType(), newPackOp, newReassocIndices);
   rewriter.replaceOp(packOp, newCollapseOp);
 
@@ -853,12 +853,12 @@ bubbleUpPackOpThroughExpandShape(tensor::ExpandShapeOp expandOp,
   Value destTensor = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), expandOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, /*outerDimsPerm=*/SmallVector<int64_t>{});
-  Value packedVal = rewriter.create<linalg::PackOp>(
+  Value packedVal = linalg::PackOp::create(rewriter,
       packOp.getLoc(), expandOp.getSrc(), destTensor, projectedInnerDimsPos,
       packOp.getMixedTiles(), packOp.getPaddingValue(),
       /*outerDimsPerm=*/SmallVector<int64_t>{});
 
-  Value newExpandOp = rewriter.create<tensor::ExpandShapeOp>(
+  Value newExpandOp = tensor::ExpandShapeOp::create(rewriter,
       packOp.getLoc(), packOp.getDestType(), packedVal, *reassocExpand);
   rewriter.replaceOp(packOp, newExpandOp);
 
@@ -972,14 +972,14 @@ static LogicalResult pushDownUnPackOpThroughExpandShape(
 
   RankedTensorType newExpandType = linalg::PackOp::inferPackedType(
       expandTy, innerTileSizes, projectedInnerDimsPos, newOuterDimsPerm);
-  auto newExpandOp = rewriter.create<tensor::ExpandShapeOp>(
+  auto newExpandOp = tensor::ExpandShapeOp::create(rewriter,
       expandOp.getLoc(), newExpandType, unPackOp.getSource(),
       newReassocIndices);
 
   auto emptyOp = linalg::UnPackOp::createDestinationTensor(
       rewriter, unPackOp.getLoc(), newExpandOp, unPackOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newUnPackOp = rewriter.create<linalg::UnPackOp>(
+  auto newUnPackOp = linalg::UnPackOp::create(rewriter,
       unPackOp.getLoc(), newExpandOp.getResult(), emptyOp,
       projectedInnerDimsPos, unPackOp.getMixedTiles(), newOuterDimsPerm);
   rewriter.replaceOp(expandOp, newUnPackOp);
@@ -1212,16 +1212,16 @@ struct PushDownUnPackThroughPadOp : public OpRewritePattern<tensor::PadOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, /*result=*/Type(), unpackOp.getSource(), lowPad, highPad,
         paddingVal, padOp.getNofold());
 
     // Inject the linalg.unpack right after the packed padOp.
-    Value outputUnPack = rewriter.create<tensor::EmptyOp>(
+    Value outputUnPack = tensor::EmptyOp::create(rewriter,
         loc, padOp.getResultType().getShape(),
         padOp.getResultType().getElementType());
 
-    Value replacement = rewriter.create<linalg::UnPackOp>(
+    Value replacement = linalg::UnPackOp::create(rewriter,
         loc, newPadOp.getResult(), outputUnPack, innerDimsPos,
         unpackOp.getMixedTiles(), outerDimsPerm);
     rewriter.replaceOp(padOp, replacement);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
index 692bf595267d4..82d386b3b9099 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
@@ -198,9 +198,9 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
         transposedShape[i] = inputRTType.getShape()[permutation[i]];
 
       Value emptyTensor =
-          rewriter.create<tensor::EmptyOp>(loc, transposedShape, elType);
+          tensor::EmptyOp::create(rewriter, loc, transposedShape, elType);
 
-      auto transposeOp = rewriter.create<TransposeOp>(loc, newInitValues[i],
+      auto transposeOp = TransposeOp::create(rewriter, loc, newInitValues[i],
                                                       emptyTensor, permutation);
       newInitValues[i] = transposeOp->getResult(0);
       isChanged = true;
@@ -209,10 +209,10 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
     // Does it require broadcast?
     if (!broadcastedDims.empty()) {
       assert(broadcastedDims.size() && "should have non size broadcast");
-      Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+      Value emptyTensor = tensor::EmptyOp::create(rewriter,
           loc, outputShape, inputRTType.getElementType());
 
-      auto broadcastOp = rewriter.create<linalg::BroadcastOp>(
+      auto broadcastOp = linalg::BroadcastOp::create(rewriter,
           loc, newInitValues[i], emptyTensor, broadcastedDims);
 
       newInitValues[i] = broadcastOp->getResult(0);
@@ -227,7 +227,7 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
   SmallVector<Value> operands = op->getOperands();
   ValueRange operandsRef(operands);
 
-  auto newOp = rewriter.create<linalg::GenericOp>(
+  auto newOp = linalg::GenericOp::create(rewriter,
       /*location=*/op.getLoc(),
       /*resultTensorTypes=*/op->getResultTypes(),
       /*inputs=*/newInitValues,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
index 1419175304899..bd3fad01aa48b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
@@ -133,12 +133,12 @@ static Value getZero(OpBuilder &b, Location loc, Type elementType) {
   assert(elementType.isIntOrIndexOrFloat() &&
          "expected scalar type while computing zero value");
   if (isa<IntegerType>(elementType))
-    return b.create<arith::ConstantIntOp>(loc, elementType, 0);
+    return arith::ConstantIntOp::create(b, loc, elementType, 0);
   if (elementType.isIndex())
-    return b.create<arith::ConstantIndexOp>(loc, 0);
+    return arith::ConstantIndexOp::create(b, loc, 0);
   // Assume float.
   auto floatType = cast<FloatType>(elementType);
-  return b.create<arith::ConstantFloatOp>(
+  return arith::ConstantFloatOp::create(b,
       loc, floatType, APFloat::getZero(floatType.getFloatSemantics()));
 }
 
@@ -189,7 +189,7 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
     // Fall back path, use an `init_tensor` and identity indexing map.
     AffineMap indexingMap = rewriter.getMultiDimIdentityMap(domain.size());
     Value emptyTensor =
-        rewriter.create<tensor::EmptyOp>(loc, domain, scalarOpResult.getType());
+        tensor::EmptyOp::create(rewriter, loc, domain, scalarOpResult.getType());
     newInitValues.push_back(emptyTensor);
     newResultTypes.push_back(emptyTensor.getType());
     peeledGenericOpIndexingMaps.push_back(indexingMap);
@@ -202,7 +202,7 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
   resultTypes.append(newResultTypes.begin(), newResultTypes.end());
   auto indexingMapAttr =
       rewriter.getAffineMapArrayAttr(peeledGenericOpIndexingMaps);
-  return rewriter.create<GenericOp>(
+  return GenericOp::create(rewriter,
       loc, resultTypes, genericOp.getInputs(), outsOperands, indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
       [](OpBuilder, Location, ValueRange) {});
@@ -239,7 +239,7 @@ DecomposeLinalgOp::createResidualGenericOp(GenericOp genericOp,
     indexingMaps.push_back(genericOp.getMatchingIndexingMap(&outOperand));
 
   auto indexingMapAttr = rewriter.getAffineMapArrayAttr(indexingMaps);
-  return rewriter.create<GenericOp>(
+  return GenericOp::create(rewriter,
       genericOp->getLoc(), genericOp->getResultTypes(),
       residualGenericOpOperands, genericOp.getOutputs(), indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
@@ -324,7 +324,7 @@ DecomposeLinalgOp::matchAndRewrite(GenericOp genericOp,
     yieldedVals.append(llvm::to_vector(
         llvm::map_range(peeledScalarOperation->getResults(),
                         [](OpResult opr) -> Value { return opr; })));
-    rewriter.create<YieldOp>(genericOp.getLoc(), yieldedVals);
+    YieldOp::create(rewriter, genericOp.getLoc(), yieldedVals);
   }
 
   /// In the split operations, replace block arguments uses that refer to
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index ef24eb881d68b..b8e73bd5929dd 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -34,7 +34,7 @@ static Value sourceMaterializationCallback(OpBuilder &builder, Type type,
 
   // A detensored value is converted back by creating a new tensor from its
   // element(s).
-  return builder.create<tensor::FromElementsOp>(
+  return tensor::FromElementsOp::create(builder,
       loc, RankedTensorType::get({}, inputType), inputs[0]);
 }
 
@@ -147,7 +147,7 @@ class DetensorizeTypeConverter : public TypeConverter {
     // A tensor value is detensoried by extracting its element(s).
     addTargetMaterialization([](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      return builder.create<tensor::ExtractOp>(loc, inputs[0], ValueRange{});
+      return tensor::ExtractOp::create(builder, loc, inputs[0], ValueRange{});
     });
 
     addSourceMaterialization(sourceMaterializationCallback);
@@ -481,7 +481,7 @@ struct LinalgDetensorize
         rewriter.splitBlock(entryBlock, entryBlock->begin());
     rewriter.setInsertionPointToStart(entryBlock);
     auto branch =
-        rewriter.create<cf::BranchOp>(rewriter.getUnknownLoc(), postEntryBlock);
+        cf::BranchOp::create(rewriter, rewriter.getUnknownLoc(), postEntryBlock);
 
     if (aggressiveMode.getValue()) {
       AggressiveDetensoringModel costModel;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index e0062d15e61ca..fd8c177cc7247 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -118,14 +118,14 @@ struct MoveInitOperandsToInput : public OpRewritePattern<GenericOp> {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointAfterValue(op->get());
       auto elemType = cast<ShapedType>(op->get().getType()).getElementType();
-      auto empty = rewriter.create<tensor::EmptyOp>(
+      auto empty = tensor::EmptyOp::create(rewriter,
           loc, tensor::getMixedSizes(rewriter, loc, op->get()), elemType);
 
       unsigned start = genericOp.getDpsInits().getBeginOperandIndex();
       newOutputOperands[op->getOperandNumber() - start] = empty.getResult();
     }
 
-    auto newOp = rewriter.create<GenericOp>(
+    auto newOp = GenericOp::create(rewriter,
         loc, genericOp.getResultTypes(), newInputOperands, newOutputOperands,
         newIndexingMaps, genericOp.getIteratorTypesArray(),
         /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
@@ -295,7 +295,7 @@ static Value collapseValue(
     MemRefLayoutAttrInterface layout;
     auto targetType = MemRefType::get(targetShape, memrefType.getElementType(),
                                       layout, memrefType.getMemorySpace());
-    return rewriter.create<memref::CollapseShapeOp>(loc, targetType, operand,
+    return memref::CollapseShapeOp::create(rewriter, loc, targetType, operand,
                                                     reassociation);
   }
   if (auto tensorType = dyn_cast<RankedTensorType>(operand.getType())) {
@@ -314,7 +314,7 @@ static Value collapseValue(
         "unknown rank reduction strategy");
     auto targetType =
         RankedTensorType::get(targetShape, tensorType.getElementType());
-    return rewriter.create<tensor::CollapseShapeOp>(loc, targetType, operand,
+    return tensor::CollapseShapeOp::create(rewriter, loc, targetType, operand,
                                                     reassociation);
   }
   llvm_unreachable("unsupported operand type");
@@ -519,7 +519,7 @@ linalg::dropUnitDims(RewriterBase &rewriter, GenericOp genericOp,
   for (unsigned i : llvm::seq<unsigned>(0, genericOp.getNumResults()))
     resultTypes.push_back(newOutputs[i].getType());
   GenericOp replacementOp =
-      rewriter.create<GenericOp>(loc, resultTypes, newInputs, newOutputs,
+      GenericOp::create(rewriter, loc, resultTypes, newInputs, newOutputs,
                                  newIndexingMaps, newIteratorTypes);
   rewriter.inlineRegionBefore(genericOp.getRegion(), replacementOp.getRegion(),
                               replacementOp.getRegion().begin());
@@ -652,7 +652,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         collapseValue(rewriter, padOp.getLoc(), padOp.getSource(), newShape,
                       reassociationMap, options.rankReductionStrategy);
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         padOp.getLoc(), /*result=*/Type(), collapsedSource, newLowPad,
         newHighPad, paddingVal, padOp.getNofold());
 
@@ -670,7 +670,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         expandedSizes.push_back(tensor::getMixedSize(
             rewriter, padOp.getLoc(), newPadOp, dim - numUnitDims));
       }
-      dest = rewriter.create<tensor::EmptyOp>(
+      dest = tensor::EmptyOp::create(rewriter,
           padOp.getLoc(), expandedSizes,
           padOp.getResultType().getElementType());
     }
@@ -713,7 +713,7 @@ struct RankReducedExtractSliceOp
             strides));
 
     Location loc = sliceOp.getLoc();
-    Value newSlice = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSlice = tensor::ExtractSliceOp::create(rewriter,
         loc, rankReducedType, sliceOp.getSource(), offsets, sizes, strides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSlice, *reassociation);
@@ -747,7 +747,7 @@ struct RankReducedInsertSliceOp : public OpRewritePattern<InsertOpTy> {
       // parallel case.
       if (std::is_same<InsertOpTy, tensor::ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      reshapedSource = rewriter.create<tensor::CollapseShapeOp>(
+      reshapedSource = tensor::CollapseShapeOp::create(rewriter,
           loc, insertSliceOp.getSource(), *reassociation);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
@@ -898,7 +898,7 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
   /// Expand result tensor.
   Value expandResult(PatternRewriter &rewriter, Value result,
                      RankedTensorType expandedType, int64_t dim) const {
-    return rewriter.create<tensor::ExpandShapeOp>(
+    return tensor::ExpandShapeOp::create(rewriter,
         result.getLoc(), expandedType, result,
         getReassociationForReshapeAtDim(expandedType.getRank(), dim));
   }
@@ -934,7 +934,7 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
     SmallVector<Type, 1> collapsedResultTy;
     if (isa<RankedTensorType>(collapsedInit.getType()))
       collapsedResultTy.push_back(collapsedInit.getType());
-    auto collapsedOp = rewriter.create<ToOpTy>(
+    auto collapsedOp = ToOpTy::create(rewriter,
         loc, collapsedResultTy, ValueRange{collapsedLhs, collapsedRhs},
         ValueRange{collapsedInit});
     for (auto attr : contractionOp->getAttrs()) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index f97ed3d6d5111..1ae4375bd09eb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -236,11 +236,11 @@ static void generateFusedElementwiseOpRegion(
     fusedIndices.reserve(numFusedOpLoops);
     llvm::transform(llvm::seq<uint64_t>(0, numFusedOpLoops),
                     std::back_inserter(fusedIndices), [&](uint64_t dim) {
-                      return rewriter.create<IndexOp>(producer.getLoc(), dim);
+                      return IndexOp::create(rewriter, producer.getLoc(), dim);
                     });
     for (IndexOp indexOp :
          llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
-      Value newIndex = rewriter.create<affine::AffineApplyOp>(
+      Value newIndex = affine::AffineApplyOp::create(rewriter,
           producer.getLoc(),
           consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
       mapper.map(indexOp.getResult(), newIndex);
@@ -327,7 +327,7 @@ static void generateFusedElementwiseOpRegion(
   }
   for (auto consumerYieldVal : consumerYieldOp.getOperands())
     fusedYieldValues.push_back(mapper.lookupOrDefault(consumerYieldVal));
-  rewriter.create<YieldOp>(fusedOp.getLoc(), fusedYieldValues);
+  YieldOp::create(rewriter, fusedOp.getLoc(), fusedYieldValues);
 
   // Sanity checks.
   assert(fusedBlock->getNumArguments() == fusedOp.getNumOperands() &&
@@ -416,7 +416,7 @@ mlir::linalg::fuseElementwiseOps(RewriterBase &rewriter,
   }
 
   // Generate the fused op.
-  auto fusedOp = rewriter.create<GenericOp>(
+  auto fusedOp = GenericOp::create(rewriter,
       consumer.getLoc(), fusedResultTypes, fusedInputOperands,
       fusedOutputOperands, rewriter.getAffineMapArrayAttr(fusedIndexMaps),
       consumer.getIteratorTypes(),
@@ -750,9 +750,9 @@ static void updateExpandedGenericOpRegion(PatternRewriter &rewriter,
     expandedIndices.reserve(expandedDims.size() - 1);
     llvm::transform(
         expandedDims.drop_front(), std::back_inserter(expandedIndices),
-        [&](int64_t dim) { return rewriter.create<IndexOp>(loc, dim); });
+        [&](int64_t dim) { return IndexOp::create(rewriter, loc, dim); });
     OpFoldResult newIndex =
-        rewriter.create<IndexOp>(loc, expandedDims.front()).getResult();
+        IndexOp::create(rewriter, loc, expandedDims.front()).getResult();
     for (auto [expandedShape, expandedIndex] :
          llvm::zip(expandedDimsShape, expandedIndices)) {
       AffineExpr idx, acc, shape;
@@ -796,7 +796,7 @@ static Operation *createExpandedTransposeOp(PatternRewriter &rewriter,
       newPerm.push_back(dim);
     }
   }
-  return rewriter.create<TransposeOp>(transposeOp.getLoc(), expandedInput,
+  return TransposeOp::create(rewriter, transposeOp.getLoc(), expandedInput,
                                       output, invertPermutationVector(newPerm));
 }
 
@@ -813,7 +813,7 @@ static Operation *createExpandedGenericOp(
     for (auto j : expansionInfo.getExpandedDims(i))
       iteratorTypes[j] = type;
 
-  Operation *fused = rewriter.create<GenericOp>(
+  Operation *fused = GenericOp::create(rewriter,
       linalgOp.getLoc(), resultTypes, expandedOpOperands, outputs,
       expandedOpIndexingMaps, iteratorTypes);
 
@@ -933,7 +933,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
                 reassociation,
                 /*isExpandingReshape=*/true)))
           return std::nullopt;
-        expandedOpOperands.push_back(rewriter.create<tensor::ExpandShapeOp>(
+        expandedOpOperands.push_back(tensor::ExpandShapeOp::create(rewriter,
             loc, expandedOperandType, opOperand->get(), reassociation,
             expandedOperandShape));
         continue;
@@ -961,7 +961,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               reassociation,
               /*isExpandingReshape=*/true)))
         return std::nullopt;
-      outputs.push_back(rewriter.create<tensor::ExpandShapeOp>(
+      outputs.push_back(tensor::ExpandShapeOp::create(rewriter,
           loc, expandedOutputType, opOperand.get(), reassociation,
           expandedOutputShape));
     } else {
@@ -984,7 +984,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               linalgOp.getMatchingIndexingMap(
                   linalgOp.getDpsInitOperand(resultNumber)),
               expansionInfo);
-      resultVals.push_back(rewriter.create<tensor::CollapseShapeOp>(
+      resultVals.push_back(tensor::CollapseShapeOp::create(rewriter,
           linalgOp.getLoc(), opResult.getType(),
           fusedOp->getResult(resultNumber), reassociation));
     } else {
@@ -1086,7 +1086,7 @@ class FoldPadWithProducerReshapeOpByExpansion
 
     Location loc = padOp->getLoc();
     RankedTensorType expandedPaddedType = paddedType.clone(expandedPaddedShape);
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
@@ -1603,7 +1603,7 @@ static void generateCollapsedIndexingRegion(
        enumerate(collapsingInfo.getCollapsedOpToOrigOpMapping())) {
     ReassociationIndicesRef foldedDimsRef(foldedDims.value());
     Value newIndexVal =
-        rewriter.create<linalg::IndexOp>(loc, foldedDims.index());
+        linalg::IndexOp::create(rewriter, loc, foldedDims.index());
     for (auto dim : llvm::reverse(foldedDimsRef.drop_front())) {
       Value loopDim =
           getValueOrCreateConstantIndexOp(rewriter, loc, loopRange[dim]);
@@ -1687,7 +1687,7 @@ GenericOp cloneToCollapsedOp<GenericOp>(RewriterBase &rewriter,
   SmallVector<utils::IteratorType> iteratorTypes(getCollapsedOpIteratorTypes(
       origOp.getIteratorTypesArray(), collapsingInfo));
 
-  GenericOp collapsedOp = rewriter.create<linalg::GenericOp>(
+  GenericOp collapsedOp = linalg::GenericOp::create(rewriter,
       origOp.getLoc(), resultTypes, inputOperands, outputOperands, indexingMaps,
       iteratorTypes, [](OpBuilder &builder, Location loc, ValueRange args) {});
   Block *origOpBlock = &origOp->getRegion(0).front();
@@ -1790,11 +1790,11 @@ FailureOr<CollapseResult> mlir::linalg::collapseOpIterationDims(
       if (isa<MemRefType>(collapsedOpResult.getType())) {
         MemRefType expandShapeResultType = MemRefType::get(
             originalResultType.getShape(), originalResultType.getElementType());
-        result = rewriter.create<memref::ExpandShapeOp>(
+        result = memref::ExpandShapeOp::create(rewriter,
             loc, expandShapeResultType, collapsedOpResult, reassociation,
             resultShape);
       } else {
-        result = rewriter.create<tensor::ExpandShapeOp>(
+        result = tensor::ExpandShapeOp::create(rewriter,
             loc, originalResultType, collapsedOpResult, reassociation,
             resultShape);
       }
@@ -1978,7 +1978,7 @@ class FoldPadWithProducerReshapeOpByCollapsing
 
     RankedTensorType collapsedPaddedType =
         paddedType.clone(collapsedPaddedShape);
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
@@ -2113,10 +2113,10 @@ class FoldScalarOrSplatConstant : public OpRewritePattern<GenericOp> {
 
       // Create a constant scalar value from the splat constant.
       Value scalarConstant =
-          rewriter.create<arith::ConstantOp>(def->getLoc(), constantAttr);
+          arith::ConstantOp::create(rewriter, def->getLoc(), constantAttr);
 
       SmallVector<Value> outputOperands = genericOp.getOutputs();
-      auto fusedOp = rewriter.create<GenericOp>(
+      auto fusedOp = GenericOp::create(rewriter,
           rewriter.getFusedLoc(fusedLocs), genericOp->getResultTypes(),
           /*inputs=*/fusedOperands,
           /*outputs=*/outputOperands,
@@ -2179,7 +2179,7 @@ struct RemoveOutsDependency : public OpRewritePattern<GenericOp> {
         modifiedOutput = true;
         SmallVector<OpFoldResult> mixedSizes =
             tensor::getMixedSizes(rewriter, loc, operandVal);
-        Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+        Value emptyTensor = tensor::EmptyOp::create(rewriter,
             loc, mixedSizes, operandType.getElementType());
         op->setOperand(opOperand.getOperandNumber(), emptyTensor);
       }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index c4af09ca01421..d93f8c26a9f0a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -64,7 +64,7 @@ getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) {
       continue;
 
     // Extract static / dynamic shape mix from the first operand.
-    res.push_back(b.create<tensor::EmptyOp>(
+    res.push_back(tensor::EmptyOp::create(b,
         loc, tensor::getMixedSizes(b, loc, operands.front()),
         cast<RankedTensorType>(t).getElementType()));
   }
@@ -104,7 +104,7 @@ struct ConvertAnyElementwiseMappableOpOnRankedTensors : public RewritePattern {
               builder.create(loc, op->getName().getIdentifier(),
                              regionArgs.take_front(op->getNumOperands()),
                              resultTypes, op->getAttrs());
-          builder.create<linalg::YieldOp>(loc, scalarOp->getResults());
+          linalg::YieldOp::create(builder, loc, scalarOp->getResults());
         });
     return success();
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
index d375878fb2c91..4e95aae86e670 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
@@ -259,7 +259,7 @@ mlir::linalg::deduplicateOperandsAndRemoveDeadResults(
   for (Value v : newOutputOperands)
     if (isa<TensorType>(v.getType()))
       newResultTypes.push_back(v.getType());
-  auto newOp = rewriter.create<GenericOp>(
+  auto newOp = GenericOp::create(rewriter,
       loc, newResultTypes, newInputOperands, newOutputOperands,
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       genericOp.getIteratorTypes(), genericOp.getDocAttr(),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
index 44469bc404a7c..e19e224c25028 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
@@ -72,14 +72,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
     // Create the tensor of same size as output of the pad op.
     RankedTensorType padResultType = padOp.getResultType();
     auto resultSizes = resultShape[0];
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultSizes, padResultType.getElementType());
 
     // Fill the tensor with the pad value.
     // TODO: There is an option to fill only the boundaries. For now just
     // filling the whole tensor.
     auto fillTensor =
-        rewriter.create<linalg::FillOp>(loc, padValue, emptyTensor.getResult());
+        linalg::FillOp::create(rewriter, loc, padValue, emptyTensor.getResult());
 
     // Construct a slice of the fill result that is to be replaced with the
     // result of the generic op. The low pad values are the offsets, the size of
@@ -93,14 +93,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
          llvm::enumerate(cast<RankedTensorType>(source.getType()).getShape())) {
       if (ShapedType::isDynamic(shape.value())) {
         sizes.push_back(
-            rewriter.create<tensor::DimOp>(loc, source, shape.index())
+            tensor::DimOp::create(rewriter, loc, source, shape.index())
                 .getResult());
       } else {
         sizes.push_back(rewriter.getIndexAttr(shape.value()));
       }
     }
     SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-    auto slice = rewriter.create<tensor::ExtractSliceOp>(
+    auto slice = tensor::ExtractSliceOp::create(rewriter,
         loc, fillTensor.getResult(0), offsets, sizes, strides);
 
     // Clone the generic op.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 9bc7be2623849..41252c68ffda9 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -277,7 +277,7 @@ mlir::linalg::fuseProducerOfTensor(OpBuilder &b, OpResult producerOpResult,
   // mismatches. Insert a `tensor.cast` op to propagate the transformation
   // invariant that types are compatible.
   if (consumerType != def.getType())
-    def = b.create<tensor::CastOp>(fusedProducer.getLoc(), consumerType, def);
+    def = tensor::CastOp::create(b, fusedProducer.getLoc(), consumerType, def);
   consumerOpOperand.set(def);
   return FusionInfo{cast<LinalgOp>(producerOpResult.getOwner()), fusedProducer};
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 05f2157b77aeb..7994af8834990 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -61,7 +61,7 @@ FailureOr<GenericOp> mlir::linalg::generalizeNamedOp(RewriterBase &rewriter,
   // All named ops have a region attached that can be inlined.
   assert(linalgOp->getNumRegions() == 1 &&
          "expect named op to have one region attached");
-  GenericOp genericOp = rewriter.create<GenericOp>(
+  GenericOp genericOp = GenericOp::create(rewriter,
       linalgOp.getLoc(), resultTypes, inputs, outputs, indexingMaps, iterators);
   rewriter.inlineRegionBefore(linalgOp->getRegion(0), genericOp.getRegion(),
                               genericOp.getRegion().begin());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index 94ed46442180c..86f24578376c1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -591,7 +591,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
 
     // Create a packing loop that takes `hoistedPackedTensor` as iteration
     // argument.
-    auto clonedForOp = rewriter.create<scf::ForOp>(
+    auto clonedForOp = scf::ForOp::create(rewriter,
         loc, bvm.lookupOrDefault(forOp.getLowerBound()),
         bvm.lookupOrDefault(forOp.getUpperBound()),
         bvm.lookupOrDefault(forOp.getStep()), hoistedPackedTensor);
@@ -640,10 +640,10 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   TransposeOp maybeTransposeOp;
   Value paddedTensor = bvm.lookup(opToHoist.getResult());
   if (!transposeVector.empty()) {
-    Value outputTensor = rewriter.create<tensor::ExtractSliceOp>(
+    Value outputTensor = tensor::ExtractSliceOp::create(rewriter,
         loc, transposedTensorType, hoistedPackedTensor, offsets, sizes,
         strides);
-    maybeTransposeOp = rewriter.create<linalg::TransposeOp>(
+    maybeTransposeOp = linalg::TransposeOp::create(rewriter,
         loc, paddedTensor, outputTensor, transposeVector);
     paddedTensor = maybeTransposeOp.getResult()[0];
   }
@@ -652,7 +652,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   if (nPackedLoops > 0) {
     // Step 4. Create InsertSliceOp at the innermost loop level, inserting an
     // optionally transposed padded slice into the packed tensor.
-    Value inserted = rewriter.create<tensor::InsertSliceOp>(
+    Value inserted = tensor::InsertSliceOp::create(rewriter,
         loc, paddedTensor, hoistedPackedTensor, offsets, sizes, strides);
 
     // Step 5. Iteratively pop the stack and propagate the yield.
@@ -660,7 +660,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
     for (Value iv : llvm::reverse(clonedLoopIvs)) {
       auto forOp = scf::getForInductionVarOwner(iv);
       rewriter.setInsertionPointToEnd(&forOp.getRegion().front());
-      rewriter.create<scf::YieldOp>(loc, valueToYield);
+      scf::YieldOp::create(rewriter, loc, valueToYield);
       valueToYield = forOp.getResult(0);
     }
   }
@@ -712,7 +712,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   rewriter.setInsertionPoint(outerLoop);
   SmallVector<Value> dynamicTensorSizes =
       analysis.getHoistedPackedTensorSizes(rewriter, loc);
-  auto emptyOp = rewriter.create<tensor::EmptyOp>(
+  auto emptyOp = tensor::EmptyOp::create(rewriter,
       loc, hoistedPackedTensorType.getShape(),
       hoistedPackedTensorType.getElementType(), dynamicTensorSizes);
 
@@ -840,7 +840,7 @@ padThroughLoopIterArg(RewriterBase &rewriter, Value paddedValueBeforeHoisting,
   {
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(forOp);
-    extracted = rewriter.create<tensor::ExtractSliceOp>(
+    extracted = tensor::ExtractSliceOp::create(rewriter,
         hoistedPackedTensor.getLoc(), hoistedPackedTensor,
         outerSliceOp.getMixedOffsets(), outerSliceOp.getMixedSizes(),
         outerSliceOp.getMixedStrides());
@@ -934,7 +934,7 @@ static Value replaceByPackingResult(RewriterBase &rewriter,
   // offsets = [maybe_leading_ivs, 0 .. 0].
   // sizes = [1 .. 1, transposedShape] (defined above).
   // strides = [1 .. 1] (defined above)
-  return rewriter.create<tensor::ExtractSliceOp>(
+  return tensor::ExtractSliceOp::create(rewriter,
       loc, transposedTensorType, hoistedPackedTensor, offsets,
       packingResult.sizes, packingResult.strides);
 }
@@ -982,9 +982,9 @@ FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(newResult.getDefiningOp());
     // Transpose the packed tensor back to the original storage order.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, paddedTensorType.getShape(), paddedTensorType.getElementType());
-    TransposeOp unTransposeOp = rewriter.create<linalg::TransposeOp>(
+    TransposeOp unTransposeOp = linalg::TransposeOp::create(rewriter,
         loc, newResult, emptyTensor, transposeVector);
     newResult = unTransposeOp.getResult()[0];
     transposeOps.push_back(unTransposeOp);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index f2e51c29f3241..5397fc572e558 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -53,7 +53,7 @@ static scf::ForOp replaceWithDifferentYield(RewriterBase &rewriter,
   assert(index < inits.size());
   inits[index] = newInitOperand;
 
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
       inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
index 1f3336d2bfbb9..a900215ccf014 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
@@ -60,7 +60,7 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
 
     Location loc = genericOp->getLoc();
     SmallVector<Value> outputOperands = genericOp.getOutputs();
-    auto newOp = rewriter.create<GenericOp>(
+    auto newOp = GenericOp::create(rewriter,
         loc, genericOp->getResultTypes(), newOperands, outputOperands,
         newIndexingMaps, genericOp.getIteratorTypesArray());
     rewriter.cloneRegionBefore(genericOp.getRegion(), newOp.getRegion(),
@@ -77,11 +77,11 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
       SmallVector<Value> indicesValues;
       for (auto idx : indices)
         indicesValues.emplace_back(
-            rewriter.create<arith::ConstantIndexOp>(loc, idx));
+            arith::ConstantIndexOp::create(rewriter, loc, idx));
       Value scalarValue = opOperand->get();
       if (isa<RankedTensorType>(scalarValue.getType())) {
         scalarValue =
-            rewriter.create<tensor::ExtractOp>(loc, scalarValue, indicesValues);
+            tensor::ExtractOp::create(rewriter, loc, scalarValue, indicesValues);
       }
       body->getArgument(idx).replaceAllUsesWith(scalarValue);
       body->eraseArgument(idx);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index a92a0c83e0316..2372313e29c58 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -88,7 +88,7 @@ mlir::linalg::interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp,
       allIndices.reserve(genericOp.getNumLoops());
       llvm::transform(llvm::seq<uint64_t>(0, genericOp.getNumLoops()),
                       std::back_inserter(allIndices), [&](uint64_t dim) {
-                        return rewriter.create<IndexOp>(indexOp->getLoc(), dim);
+                        return IndexOp::create(rewriter, indexOp->getLoc(), dim);
                       });
       rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
           indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 488041a43a2ef..0c08c1d8b997d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -49,7 +49,7 @@ static SmallVector<Value> makeCanonicalAffineApplies(OpBuilder &b, Location loc,
     auto exprMap = AffineMap::get(dims, map.getNumSymbols(), e);
     SmallVector<Value> operands(vals);
     affine::canonicalizeMapAndOperands(&exprMap, &operands);
-    res.push_back(b.create<affine::AffineApplyOp>(loc, exprMap, operands));
+    res.push_back(affine::AffineApplyOp::create(b, loc, exprMap, operands));
   }
   return res;
 }
@@ -70,7 +70,7 @@ static void inlineRegionAndEmitStore(OpBuilder &b, Location loc, OpType op,
   Operation *terminator = block.getTerminator();
   for (OpOperand &operand : terminator->getOpOperands()) {
     Value toStore = map.lookupOrDefault(operand.get());
-    b.create<StoreOpTy>(loc, toStore, outputBuffers[operand.getOperandNumber()],
+    StoreOpTy::create(b, loc, toStore, outputBuffers[operand.getOperandNumber()],
                         indexing[operand.getOperandNumber()]);
   }
 }
@@ -145,7 +145,7 @@ static void emitScalarImplementation(OpBuilder &b, Location loc,
     auto indexing = makeCanonicalAffineApplies(
         b, loc, linalgOp.getMatchingIndexingMap(inputOperand), allIvsPlusDims);
     indexedValues.push_back(
-        b.create<LoadOpTy>(loc, inputOperand->get(), indexing));
+        LoadOpTy::create(b, loc, inputOperand->get(), indexing));
   }
   // 1.b. Emit load from output views.
   for (OpOperand &outputOperand : linalgOp.getDpsInitsMutable()) {
@@ -153,7 +153,7 @@ static void emitScalarImplementation(OpBuilder &b, Location loc,
         b, loc, linalgOp.getMatchingIndexingMap(&outputOperand),
         allIvsPlusDims);
     indexedValues.push_back(
-        b.create<LoadOpTy>(loc, outputOperand.get(), indexing));
+        LoadOpTy::create(b, loc, outputOperand.get(), indexing));
   }
 
   // TODO: When a region inliner exists, use it.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
index ee1957aaa6a53..28b246714df53 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
@@ -131,17 +131,17 @@ static Value createDestinationPassingStyleInitOperand(
     ImplicitLocOpBuilder &builder) {
   Value processLinearIndexInReductionGroup = mesh::createProcessLinearIndex(
       meshOp.getSymName(), reductionMeshAxes, builder);
-  Value zero = builder.create<arith::ConstantIndexOp>(0);
-  Value isLeadProcess = builder.create<arith::CmpIOp>(
+  Value zero = arith::ConstantIndexOp::create(builder, 0);
+  Value isLeadProcess = arith::CmpIOp::create(builder,
       builder.getI1Type(), arith::CmpIPredicate::eq,
       processLinearIndexInReductionGroup, zero);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(spmdizedOperand.getType(),
+  scf::IfOp ifOp = scf::IfOp::create(builder, spmdizedOperand.getType(),
                                              isLeadProcess, true, true);
   // Then block.
   {
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPointToEnd(&ifOp.getThenRegion().front());
-    builder.create<scf::YieldOp>(spmdizedOperand);
+    scf::YieldOp::create(builder, spmdizedOperand);
   }
 
   // Else block.
@@ -157,14 +157,14 @@ static Value createDestinationPassingStyleInitOperand(
     std::optional<TypedAttr> neutralEl =
         arith::getNeutralElement(combinerOps[0]);
 
-    Value init = builder.create<tensor::EmptyOp>(op.getLoc(), shape,
+    Value init = tensor::EmptyOp::create(builder, op.getLoc(), shape,
                                                  neutralEl.value().getType());
     Value constant =
-        builder.create<arith::ConstantOp>(op.getLoc(), neutralEl.value());
-    Value fill = builder.create<linalg::FillOp>(op.getLoc(), constant, init)
+        arith::ConstantOp::create(builder, op.getLoc(), neutralEl.value());
+    Value fill = linalg::FillOp::create(builder, op.getLoc(), constant, init)
                      .getResult(0);
 
-    builder.create<scf::YieldOp>(fill);
+    scf::YieldOp::create(builder, fill);
   }
   return ifOp.getResult(0);
 }
@@ -202,7 +202,7 @@ static void createAllReduceForResultWithoutPartialSharding(
   }
 
   Value spmdizedLinalgOpResult = spmdizationMap.lookup(unshardedLinalgOpResult);
-  Value reducedValue = builder.create<mesh::AllReduceOp>(
+  Value reducedValue = mesh::AllReduceOp::create(builder,
       spmdizedLinalgOpResult, resultSharding.getMesh(), allReduceMeshAxes,
       reductionKind);
   spmdizationMap.map(unshardedLinalgOpResult, reducedValue);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
index bb1e974391878..019c845b4e0e7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
@@ -59,7 +59,7 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
   auto newKernelTy = RankedTensorType::get(
       {kernelTy.getDimSize(0), kernelTy.getDimSize(1), kernelTy.getDimSize(2)},
       kernelTy.getElementType());
-  auto collapsedKernel = rewriter.create<tensor::CollapseShapeOp>(
+  auto collapsedKernel = tensor::CollapseShapeOp::create(rewriter,
       loc, newKernelTy, kernel, collapsedKernelDims);
 
   // Collapse init dims.
@@ -70,7 +70,7 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       RankedTensorType::get({initTy.getDimSize(0), initTy.getDimSize(1),
                              initTy.getDimSize(2), initTy.getDimSize(3)},
                             initTy.getElementType());
-  auto collapsedInit = rewriter.create<tensor::CollapseShapeOp>(
+  auto collapsedInit = tensor::CollapseShapeOp::create(rewriter,
       loc, newInitTy, init, collapsedInitDims);
 
   SmallVector<NamedAttribute> preservedAttrs;
@@ -78,13 +78,13 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       TypeSwitch<Operation *, Operation *>(operation)
           .Case<DepthwiseConv2DNhwcHwcmOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return rewriter.create<DepthwiseConv2DNhwcHwcOp>(
+            return DepthwiseConv2DNhwcHwcOp::create(rewriter,
                 loc, newInitTy, ValueRange{input, collapsedKernel},
                 ValueRange{collapsedInit}, stride, dilation);
           })
           .Case<DepthwiseConv2DNhwcHwcmQOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return rewriter.create<DepthwiseConv2DNhwcHwcQOp>(
+            return DepthwiseConv2DNhwcHwcQOp::create(rewriter,
                 loc, newInitTy, ValueRange{input, collapsedKernel, iZp, kZp},
                 ValueRange{collapsedInit}, stride, dilation);
           })
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
index 2afa2f9b71c2a..1368bb7e77db0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
@@ -143,7 +143,7 @@ struct SimplifyUnPackToCollapseShape : public OpRewritePattern<UnPackOp> {
                        Type newOperandType, ArrayAttr reassociation) const {
     if (operand.getType() == newOperandType)
       return operand;
-    return rewriter.create<tensor::CollapseShapeOp>(loc, newOperandType,
+    return tensor::CollapseShapeOp::create(rewriter, loc, newOperandType,
                                                     operand, reassociation);
   }
 
@@ -265,7 +265,7 @@ struct FoldUnpackWithExtractSliceOp
 
     // Create a new empty output tensor.
     Type elementType = unpackOp.getDestType().getElementType();
-    Value output = rewriter.create<tensor::EmptyOp>(
+    Value output = tensor::EmptyOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getMixedSizes(), elementType);
     rewriter.replaceOpWithNewOp<UnPackOp>(
         sliceOp, unpackOp.getSource(), output, unpackOp.getInnerDimsPos(),
@@ -529,7 +529,7 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
     auto elemType =
         cast<ShapedType>(unPackOp->getResultTypes()[0]).getElementType();
-    Value output = rewriter.create<tensor::EmptyOp>(
+    Value output = tensor::EmptyOp::create(rewriter,
         unPackOp->getLoc(), unpackOpResultDims[0], elemType);
 
     rewriter.replaceOpWithNewOp<UnPackOp>(
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index 5eb3761f7aca1..fb43c1f92e5b3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -192,10 +192,10 @@ static Value padOperand(RewriterBase &rewriter, TilingInterface opToPad,
   if (auto complexTy =
           dyn_cast<ComplexType>(getElementTypeOrSelf(v.getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingValueAttr);
-    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
                                                         complexTy, complexAttr);
   } else {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         opToPad.getLoc(), cast<TypedAttr>(paddingValueAttr));
   }
 
@@ -323,7 +323,7 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
         loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index dc9e11eccac4d..af614e88fb3d5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -219,10 +219,10 @@ static FailureOr<Value> padOperandToSmallestStaticBoundingBox(
   if (auto complexTy = dyn_cast<ComplexType>(
           getElementTypeOrSelf(opOperand->get().getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingAttr);
-    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
                                                         complexTy, complexAttr);
   } else {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         opToPad.getLoc(), cast<TypedAttr>(paddingAttr));
   }
 
@@ -313,7 +313,7 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
         loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index b274502e16903..4042a43b04091 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -63,10 +63,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
     staticBufferType =
         MemRefType::Builder(staticBufferType).setMemorySpace(memorySpaceAttr);
     if (options.useAlloca) {
-      return b.create<memref::AllocaOp>(staticBufferType, ValueRange{},
+      return memref::AllocaOp::create(b, staticBufferType, ValueRange{},
                                         alignmentAttr);
     }
-    return b.create<memref::AllocOp>(staticBufferType, ValueRange{},
+    return memref::AllocOp::create(b, staticBufferType, ValueRange{},
                                      alignmentAttr);
   }
 
@@ -76,10 +76,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
   dynamicBufferType =
       MemRefType::Builder(dynamicBufferType).setMemorySpace(memorySpaceAttr);
   Value mul = b.createOrFold<arith::MulIOp>(
-      b.create<arith::ConstantIndexOp>(width), allocSize);
+      arith::ConstantIndexOp::create(b, width), allocSize);
   if (options.useAlloca)
-    return b.create<memref::AllocaOp>(dynamicBufferType, mul, alignmentAttr);
-  return b.create<memref::AllocOp>(dynamicBufferType, mul, alignmentAttr);
+    return memref::AllocaOp::create(b, dynamicBufferType, mul, alignmentAttr);
+  return memref::AllocOp::create(b, dynamicBufferType, mul, alignmentAttr);
 }
 
 /// Default allocation callback function. This allocates a promoted buffer when
@@ -92,8 +92,8 @@ static std::optional<Value> defaultAllocBufferCallBack(
     std::optional<unsigned> alignment, DataLayout &layout) {
   ShapedType viewType = subView.getType();
   ImplicitLocOpBuilder b(subView.getLoc(), builder);
-  auto zero = b.create<arith::ConstantIndexOp>(0);
-  auto one = b.create<arith::ConstantIndexOp>(1);
+  auto zero = arith::ConstantIndexOp::create(b, 0);
+  auto one = arith::ConstantIndexOp::create(b, 1);
 
   Attribute memorySpaceAttr;
   if (options.memorySpace.has_value())
@@ -123,7 +123,7 @@ defaultDeallocBufferCallBack(const LinalgPromotionOptions &options,
                              OpBuilder &b, Value fullLocalView) {
   if (!options.useAlloca) {
     auto viewOp = cast<memref::ViewOp>(fullLocalView.getDefiningOp());
-    b.create<memref::DeallocOp>(viewOp.getSource().getLoc(),
+    memref::DeallocOp::create(b, viewOp.getSource().getLoc(),
                                 viewOp.getSource());
   }
   return success();
@@ -211,7 +211,7 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
   Location loc = linalgOp.getLoc();
   auto defaultCopyCallBack = [loc](OpBuilder &b, Value src,
                                    Value dst) -> LogicalResult {
-    b.create<linalg::CopyOp>(loc, src, dst);
+    linalg::CopyOp::create(b, loc, src, dst);
     return success();
   };
   copyInFn = (options.copyInFn ? *(options.copyInFn) : defaultCopyCallBack);
@@ -265,7 +265,7 @@ FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
               /*stopCondition=*/nullptr, /*closedUB=*/true);
       size = failed(upperBound)
                  ? getValueOrCreateConstantIndexOp(b, loc, rangeValue.size)
-                 : b.create<arith::ConstantIndexOp>(loc, *upperBound);
+                 : arith::ConstantIndexOp::create(b, loc, *upperBound);
     }
     LLVM_DEBUG(llvm::dbgs() << "Extracted tightest: " << size << "\n");
     fullSizes.push_back(size);
@@ -310,23 +310,23 @@ promoteSubViews(ImplicitLocOpBuilder &b,
     Value fillVal =
         llvm::TypeSwitch<Type, Value>(subviewEltType)
             .Case([&](FloatType t) {
-              return b.create<arith::ConstantOp>(FloatAttr::get(t, 0.0));
+              return arith::ConstantOp::create(b, FloatAttr::get(t, 0.0));
             })
             .Case([&](IntegerType t) {
-              return b.create<arith::ConstantOp>(IntegerAttr::get(t, 0));
+              return arith::ConstantOp::create(b, IntegerAttr::get(t, 0));
             })
             .Case([&](ComplexType t) {
               Value tmp;
               if (auto et = dyn_cast<FloatType>(t.getElementType()))
-                tmp = b.create<arith::ConstantOp>(FloatAttr::get(et, 0.0));
+                tmp = arith::ConstantOp::create(b, FloatAttr::get(et, 0.0));
               else if (auto et = cast<IntegerType>(t.getElementType()))
-                tmp = b.create<arith::ConstantOp>(IntegerAttr::get(et, 0));
-              return b.create<complex::CreateOp>(t, tmp, tmp);
+                tmp = arith::ConstantOp::create(b, IntegerAttr::get(et, 0));
+              return complex::CreateOp::create(b, t, tmp, tmp);
             })
             .Default([](auto) { return Value(); });
     if (!fillVal)
       return failure();
-    b.create<linalg::FillOp>(fillVal, promotionInfo->fullLocalView);
+    linalg::FillOp::create(b, fillVal, promotionInfo->fullLocalView);
   }
 
   // Copy data into the promoted buffers. Use callback if provided.
@@ -459,9 +459,9 @@ static std::optional<Value> allocateSubviewGPUMemoryInAddressSpace(
       gpu::AddressSpaceAttr::get(builder.getContext(), addressSpace));
   Value buffer;
   if (addressSpace == gpu::GPUDialect::getWorkgroupAddressSpace()) {
-    buffer = builder.create<memref::AllocOp>(funcOp.getLoc(), type);
+    buffer = memref::AllocOp::create(builder, funcOp.getLoc(), type);
   } else if (addressSpace == gpu::GPUDialect::getPrivateAddressSpace()) {
-    buffer = builder.create<memref::AllocaOp>(funcOp.getLoc(), type);
+    buffer = memref::AllocaOp::create(builder, funcOp.getLoc(), type);
   } else {
     return std::nullopt;
   }
@@ -487,9 +487,9 @@ LogicalResult mlir::linalg::deallocateWorkgroupMemory(OpBuilder &,
 /// the copy operation to ensure data integrity.
 LogicalResult mlir::linalg::copyToWorkgroupMemory(OpBuilder &b, Value src,
                                                   Value dst) {
-  b.create<gpu::BarrierOp>(src.getLoc());
-  Operation *copyOp = b.create<memref::CopyOp>(src.getLoc(), src, dst);
-  b.create<gpu::BarrierOp>(copyOp->getLoc());
+  gpu::BarrierOp::create(b, src.getLoc());
+  Operation *copyOp = memref::CopyOp::create(b, src.getLoc(), src, dst);
+  gpu::BarrierOp::create(b, copyOp->getLoc());
   return success();
 }
 
@@ -504,7 +504,7 @@ std::optional<Value> mlir::linalg::allocateGPUPrivateMemory(
 /// Normal copy to between src and dst.
 LogicalResult mlir::linalg::copyToGPUPrivateMemory(OpBuilder &b, Value src,
                                                    Value dst) {
-  b.create<memref::CopyOp>(src.getLoc(), src, dst);
+  memref::CopyOp::create(b, src.getLoc(), src, dst);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
index b30182dc84079..eac0e47b18a7d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
@@ -38,8 +38,8 @@ struct StructuredOpInterface
     SmallVector<Range> loopRanges = linalgOp.createLoopRanges(builder, loc);
     auto [starts, ends, _] = getOffsetsSizesAndStrides(loopRanges);
 
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    auto one = arith::ConstantIndexOp::create(builder, loc, 1);
 
     // Subtract one from the loop ends before composing with the indexing map
     transform(ends, ends.begin(), [&](OpFoldResult end) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
index 671dea8bb415f..1fac80142dddf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
@@ -52,7 +52,7 @@ createSplitPart(RewriterBase &b, Location loc, TilingInterface op,
       return nullptr;
     SmallVector<OpFoldResult> resultStrides(resultOffsets.size(),
                                             b.getIndexAttr(1));
-    Value inserted = b.create<tensor::InsertSliceOp>(
+    Value inserted = tensor::InsertSliceOp::create(b,
         loc, result, resultOperands[index], resultOffsets, resultSizes,
         resultStrides);
     results.push_back(inserted);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
index 5bfdbc6d0bb59..1b0bfbbec8624 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
@@ -115,7 +115,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
         newShape,
         cast<RankedTensorType>(operand->get().getType()).getElementType());
 
-    Value newInput = b.create<tensor::ExpandShapeOp>(
+    Value newInput = tensor::ExpandShapeOp::create(b,
         loc, newType, operand->get(), reassociation);
     newInputs.push_back(newInput);
   }
@@ -140,18 +140,18 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   Value emptyOrAllocTensor;
   if (useAlloc) {
-    emptyOrAllocTensor = b.create<bufferization::AllocTensorOp>(
+    emptyOrAllocTensor = bufferization::AllocTensorOp::create(b,
         loc,
         RankedTensorType::get(newOutputShape,
                               op.getRegionOutputArgs()[0].getType()),
         ValueRange{});
   } else {
-    emptyOrAllocTensor = b.create<tensor::EmptyOp>(
+    emptyOrAllocTensor = tensor::EmptyOp::create(b,
         loc, newOutputShape, op.getRegionOutputArgs()[0].getType());
   }
-  Value constantOp = b.create<arith::ConstantOp>(loc, *identity);
+  Value constantOp = arith::ConstantOp::create(b, loc, *identity);
   Value identityTensor =
-      b.create<linalg::FillOp>(op->getLoc(), constantOp, emptyOrAllocTensor)
+      linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor)
           .getResult(0);
 
   newMaps.push_back(AffineMap::get(oldOutputMap.getNumDims() + 1, 0, outputExpr,
@@ -168,7 +168,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   // Create the new op matching the original op with an extra parallel
   // dimension.
-  GenericOp genericOp = b.create<GenericOp>(
+  GenericOp genericOp = GenericOp::create(b,
       loc, TypeRange({emptyOrAllocTensor.getType()}), newInputs,
       ValueRange({identityTensor}), newMaps, newIteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
@@ -191,14 +191,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   AffineMap outputMap = AffineMap::get(intermRank, 0, exprs, op.getContext());
   SmallVector<AffineMap> reductionMaps = {inputMap, outputMap};
 
-  auto reduction = b.create<GenericOp>(
+  auto reduction = GenericOp::create(b,
       loc, op->getResultTypes(), ValueRange({genericOp.getResult(0)}),
       op.getDpsInits(), reductionMaps, reductionIteratorTypes,
       [reductionOp](OpBuilder &b, Location loc, ValueRange inputs) {
         Operation *clonedReductionOp = b.clone(*reductionOp);
         clonedReductionOp->setOperand(0, inputs[0]);
         clonedReductionOp->setOperand(1, inputs[1]);
-        b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+        linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
       });
   b.replaceOp(op, reduction.getResults());
 
@@ -318,14 +318,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
     Value emptyOrAllocTensor;
     if (useAlloc) {
       emptyOrAllocTensor =
-          b.create<bufferization::AllocTensorOp>(loc, newT, dims);
+          bufferization::AllocTensorOp::create(b, loc, newT, dims);
     } else {
-      emptyOrAllocTensor = b.create<tensor::EmptyOp>(loc, newT.getShape(),
+      emptyOrAllocTensor = tensor::EmptyOp::create(b, loc, newT.getShape(),
                                                      t.getElementType(), dims);
     }
-    Value constantOp = b.create<arith::ConstantOp>(loc, std::get<1>(it));
+    Value constantOp = arith::ConstantOp::create(b, loc, std::get<1>(it));
     fillOps.push_back(
-        b.create<linalg::FillOp>(op->getLoc(), constantOp, emptyOrAllocTensor));
+        linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor));
     newOutputs.push_back(fillOps.back().getResult(0));
     emptyOrAllocTensorOps.push_back(emptyOrAllocTensor.getDefiningOp());
   }
@@ -354,7 +354,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   SmallVector<Value> newInputs = op.getDpsInputs();
   // Add a single shape-only tensor to carry the dimensions without resorting to
   // more complex inversions.
-  newInputs.push_back(b.create<tensor::EmptyOp>(
+  newInputs.push_back(tensor::EmptyOp::create(b,
       loc, ArrayRef<int64_t>{reductionDimSize / splitFactor, splitFactor},
       b.getIntegerType(1)));
   // Output tensors are already good to go.
@@ -365,7 +365,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   iteratorTypes.insert(iteratorTypes.begin() + reductionDimPos,
                        utils::IteratorType::parallel);
   GenericOp genericOp =
-      b.create<GenericOp>(loc, ValueRange(newOutputs).getTypes(), newInputs,
+      GenericOp::create(b, loc, ValueRange(newOutputs).getTypes(), newInputs,
                           newOutputs, newMaps, iteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
                        genericOp.getRegion().begin());
@@ -396,7 +396,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
         utils::IteratorType::reduction;
 
     // clang-format off
-    auto reductionOp = b.create<GenericOp>(
+    auto reductionOp = GenericOp::create(b,
         loc,
         originalOutputType,
         reindexedOutput,
@@ -407,7 +407,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
           Operation *clonedReductionOp = b.clone(*combinerOp);
           clonedReductionOp->setOperand(0, bbArgs[0]);
           clonedReductionOp->setOperand(1, bbArgs[1]);
-          b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+          linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
         });
     // clang-format on
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
index d35aad514e884..854d3855d654d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
@@ -29,7 +29,7 @@ struct SwapExtractSliceOfFill final
     if (!fillOp || !fillOp->hasOneUse())
       return failure();
 
-    auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
         extractOp.getLoc(), extractOp.getType(), fillOp.getOutputs()[0],
         extractOp.getMixedOffsets(), extractOp.getMixedSizes(),
         extractOp.getMixedStrides());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 4741afe8a417d..c2803423fafd7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -94,10 +94,10 @@ static void emitIsPositiveIndexAssertion(ImplicitLocOpBuilder &b,
     return;
   }
 
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value condition = b.create<arith::CmpIOp>(arith::CmpIPredicate::sgt,
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value condition = arith::CmpIOp::create(b, arith::CmpIPredicate::sgt,
                                             cast<Value>(value), zero);
-  b.create<cf::AssertOp>(
+  cf::AssertOp::create(b,
       condition,
       b.getStringAttr("expected strictly positive tile size and divisor"));
 }
@@ -317,9 +317,9 @@ mlir::linalg::computeMultiTileSizes(OpBuilder &builder, LinalgOp op,
     Value coveredSize =
         apply(s0 * s1 + s2 * s3, {spec.lowTileSize, spec.lowTripCount,
                                   spec.highTileSize, spec.highTripCount});
-    Value equals = b.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+    Value equals = arith::CmpIOp::create(b, arith::CmpIPredicate::eq,
                                            coveredSize, tripCount);
-    b.create<cf::AssertOp>(
+    cf::AssertOp::create(b,
         equals, builder.getStringAttr(
                     "could not compute dynamic multi-size tile shapes"));
   }
@@ -656,7 +656,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       getValueOrCreateConstantIndexOp(b, loc, nonZeroNumThreads);
 
   // 2. Create the ForallOp with an empty region.
-  scf::ForallOp forallOp = b.create<scf::ForallOp>(
+  scf::ForallOp forallOp = scf::ForallOp::create(b,
       loc, getAsOpFoldResult(materializedNonZeroNumThreads), initTensors,
       mapping);
 
@@ -689,7 +689,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       sizes[reductionDim] = b.getIndexAttr(1);
       outOffsets[reductionDim] = forallOp.getInductionVars()[0];
       // TODO: use SubsetExtractOpInterface once it is available.
-      tiledDpsInitOperands.push_back(b.create<tensor::ExtractSliceOp>(
+      tiledDpsInitOperands.push_back(tensor::ExtractSliceOp::create(b,
           loc, cast<RankedTensorType>(initOperand.getType()),
           destBbArgs[destNum], outOffsets, sizes, strides));
     }
@@ -768,7 +768,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
     // 6.b. Parallel insertions are inserted at the end of the combining
     // terminator.
     b.setInsertionPointToEnd(forallOp.getTerminator().getBody());
-    b.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(b,
         loc, result, bbArg, resultOffsetsRank, resultSizesRank, strides);
   }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
index 513cecef29b61..94449fae260ee 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
@@ -44,7 +44,7 @@ static SmallVector<Value> getIndicesForAccess(OpBuilder &b, Location loc,
   for (auto result : indexingMap.getResults()) {
     AffineMap m = AffineMap::get(indexingMap.getNumDims(),
                                  indexingMap.getNumSymbols(), result);
-    Value v = b.create<affine::AffineApplyOp>(loc, m, ivs);
+    Value v = affine::AffineApplyOp::create(b, loc, m, ivs);
     indices.push_back(v);
   }
   return indices;
@@ -72,7 +72,7 @@ static LogicalResult inlinePayload(OpBuilder &b, LinalgOp linalgOp,
     OpOperand *storeInto = linalgOp.getDpsInitOperand(operand.index());
     auto indices = getIndicesForAccess(
         b, loc, linalgOp.getMatchingIndexingMap(storeInto), ivs);
-    b.create<memref::StoreOp>(
+    memref::StoreOp::create(b,
         loc, toStore, linalgOp.getDpsInitOperand(operand.index())->get(),
         indices);
   }
@@ -351,7 +351,7 @@ struct LinalgOpTilingInterface
       SmallVector<Value> indices = getIndicesForAccess(
           builder, linalgOpLoc, linalgOp.getMatchingIndexingMap(&operand), ivs);
       Value load =
-          builder.create<memref::LoadOp>(linalgOpLoc, operand.get(), indices);
+          memref::LoadOp::create(builder, linalgOpLoc, operand.get(), indices);
       indexedValues.push_back(load);
     }
 
@@ -519,10 +519,10 @@ struct LinalgOpPartialReductionInterface
 
       Type elType = getElementTypeOrSelf(result.getType());
       Value emptyTensor =
-          b.create<tensor::EmptyOp>(loc, partialResultShape, elType);
-      Value constantOp = b.create<arith::ConstantOp>(loc, *identity);
+          tensor::EmptyOp::create(b, loc, partialResultShape, elType);
+      Value constantOp = arith::ConstantOp::create(b, loc, *identity);
       auto identityTensor =
-          b.create<linalg::FillOp>(loc, constantOp, emptyTensor);
+          linalg::FillOp::create(b, loc, constantOp, emptyTensor);
       inits.push_back(identityTensor.getResult(0));
     }
 
@@ -574,7 +574,7 @@ struct LinalgOpPartialReductionInterface
       RankedTensorType sliceResultType = RankedTensorType::get(
           sliceInfo.resultShape, valueToTileType.getElementType(),
           valueToTileType.getEncoding());
-      auto sliceOp = b.create<tensor::ExtractSliceOp>(
+      auto sliceOp = tensor::ExtractSliceOp::create(b,
           loc, sliceResultType, valueToTile, sliceInfo.offsets, sliceInfo.sizes,
           sliceInfo.strides);
       tiledInits.push_back(sliceOp.getResult());
@@ -603,7 +603,7 @@ struct LinalgOpPartialReductionInterface
     auto resultTypes = ValueRange(tiledInits).getTypes();
     if (tilingStrategy ==
         ReductionTilingStrategy::PartialReductionOuterReduction) {
-      auto genericOp = b.create<GenericOp>(
+      auto genericOp = GenericOp::create(b,
           loc, resultTypes, tiledInputs, tiledInits, newMaps, newIteratorTypes);
       IRMapping mapping;
       op->getRegion(0).cloneInto(&genericOp.getRegion(),
@@ -648,7 +648,7 @@ struct LinalgOpPartialReductionInterface
         }
       }
 
-      auto reduction = b.create<linalg::ReduceOp>(
+      auto reduction = linalg::ReduceOp::create(b,
           loc, partialResult, init, partialReductionDims,
           [&linalgOp, &initIdx](OpBuilder &b, Location loc, ValueRange inputs) {
             // Get the combiner op.
@@ -659,7 +659,7 @@ struct LinalgOpPartialReductionInterface
             // Combine the input at idx and output at numInits + idx.
             clonedReductionOp->setOperand(0, inputs[0]);
             clonedReductionOp->setOperand(1, inputs[1]);
-            b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+            linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
           });
 
       mergeOperations.push_back(reduction);
@@ -790,7 +790,7 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto sourceSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getSource(), inputIndices, inputSizes, strides);
     tiledOperands.push_back(sourceSlice);
 
@@ -800,7 +800,7 @@ struct PackOpTiling
       return {};
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = b.create<tensor::ExtractSliceOp>(
+    auto outSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
@@ -809,7 +809,7 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = b.create<PackOp>(
+    Operation *tiledPackOp = PackOp::create(b,
         loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
@@ -968,7 +968,7 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto sourceSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getSource(), offsets, sizes, strides);
     tiledOperands.push_back(sourceSlice);
 
@@ -984,7 +984,7 @@ struct PackOpTiling
       return failure();
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = b.create<tensor::ExtractSliceOp>(
+    auto outSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
@@ -992,7 +992,7 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = b.create<PackOp>(
+    Operation *tiledPackOp = PackOp::create(b,
         loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
@@ -1172,7 +1172,7 @@ struct UnPackOpTiling
     sliceSrcSizes.append(unpackOp.getMixedTiles());
     sliceSrcStrides.append(numInnerTiles, oneAttr);
     SmallVector<Operation *> generatedSlices;
-    tensor::ExtractSliceOp sliceSource = b.create<tensor::ExtractSliceOp>(
+    tensor::ExtractSliceOp sliceSource = tensor::ExtractSliceOp::create(b,
         loc, unpackOp.getSource(), sliceSrcIndices, sliceSrcSizes,
         sliceSrcStrides);
     generatedSlices.push_back(sliceSource);
@@ -1180,12 +1180,12 @@ struct UnPackOpTiling
     SmallVector<OpFoldResult> destStrides(destRank, oneAttr);
     Value sliceDest;
     if (isPerfectTilingCase) {
-      auto destSliceOp = b.create<tensor::ExtractSliceOp>(
+      auto destSliceOp = tensor::ExtractSliceOp::create(b,
           loc, unpackOp.getDest(), offsets, sizes, destStrides);
       sliceDest = destSliceOp;
       generatedSlices.push_back(destSliceOp);
     } else {
-      sliceDest = b.create<tensor::EmptyOp>(
+      sliceDest = tensor::EmptyOp::create(b,
           loc, destExpandedSizes, unpackOp.getDestType().getElementType());
     }
 
@@ -1193,7 +1193,7 @@ struct UnPackOpTiling
     for (auto tile : unpackOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledUnpackOp = b.create<UnPackOp>(
+    Operation *tiledUnpackOp = UnPackOp::create(b,
         loc, TypeRange{sliceDest.getType()}, tiledOperands, op->getAttrs());
 
     if (isPerfectTilingCase)
@@ -1201,7 +1201,7 @@ struct UnPackOpTiling
                           SmallVector<Value>(tiledUnpackOp->getResults()),
                           generatedSlices};
 
-    auto extractSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractSlice = tensor::ExtractSliceOp::create(b,
         loc, tiledUnpackOp->getResult(0), resultOffsetsFromDest, sizes,
         destStrides);
     return TilingResult{
@@ -1337,13 +1337,13 @@ struct UnPackOpTiling
 
     SmallVector<Value> tiledOperands;
     // Create slice of the dest operand.
-    auto extractDestSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractDestSlice = tensor::ExtractSliceOp::create(b,
         loc, unPackOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(extractDestSlice);
 
     strides.append(unPackOp.getSourceRank() - outputRank, oneAttr);
     // Create slice of the source operand.
-    auto extractSourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractSourceSlice = tensor::ExtractSliceOp::create(b,
         loc, unPackOp.getSource(), offsets, sizes, strides);
     tiledOperands.insert(tiledOperands.begin(), extractSourceSlice);
     for (auto tile : unPackOp.getInnerTiles())
@@ -1351,7 +1351,7 @@ struct UnPackOpTiling
 
     // Create tiled unpack op.
     Operation *tiledUnPackOp =
-        b.create<UnPackOp>(loc, TypeRange{extractDestSlice.getType()},
+        UnPackOp::create(b, loc, TypeRange{extractDestSlice.getType()},
                            tiledOperands, op->getAttrs());
 
     return TilingResult{{tiledUnPackOp},
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index eab74dab4eb75..3526f62f770dc 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -269,11 +269,11 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       packingMetadata.reassociations);
   Value paddingValue = packOp.getPaddingValue();
   if (!paddingValue) {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(getElementTypeOrSelf(collapsed)));
   }
   auto padOp =
-      rewriter.create<tensor::PadOp>(loc, collapsed, packOp.getSource(), lows,
+      tensor::PadOp::create(rewriter, loc, collapsed, packOp.getSource(), lows,
                                      highs, paddingValue, /*nofold=*/false);
 
   LLVM_DEBUG(
@@ -313,7 +313,7 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       SmallVector<OpFoldResult> sizes =
           tensor::getMixedSizes(rewriter, loc, packOp.getDest());
 
-      auto insertSliceOp = rewriter.create<tensor::InsertSliceOp>(
+      auto insertSliceOp = tensor::InsertSliceOp::create(rewriter,
           loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
           /*offsets=*/zeros, sizes, /*strides=*/ones);
 
@@ -329,14 +329,14 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   // 5. Expand from the padded result to the stripMinedShape.
   auto expandShapeResultType =
       RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape);
-  auto reshapeOp = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapeOp = tensor::ExpandShapeOp::create(rewriter,
       loc, expandShapeResultType, padOp.getResult(),
       packingMetadata.reassociations);
 
   // 6. Transpose stripMinedShape to packedShape.
   SmallVector<int64_t> transpPerm =
       invertPermutationVector(packedToStripMinedShapePerm);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
 
   LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
@@ -371,7 +371,7 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
     SmallVector<OpFoldResult> sizes(packedRank - destShape.size(), one);
     sizes.append(tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()));
 
-    auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
         loc, destTensorType, unPackOp.getSource(),
         SmallVector<OpFoldResult>(packedRank, zero), sizes,
         SmallVector<OpFoldResult>(packedRank, one));
@@ -404,9 +404,9 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
   SmallVector<OpFoldResult, 4> dims =
       tensor::getMixedSizes(rewriter, loc, unPackOp.getSource());
   applyPermutationToVector(dims, packedToStripMinedShapePerm);
-  auto emptyOp = rewriter.create<tensor::EmptyOp>(
+  auto emptyOp = tensor::EmptyOp::create(rewriter,
       loc, dims, stripMinedTensorType.getElementType());
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm);
 
   LLVM_DEBUG(
@@ -426,20 +426,20 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
       DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
 
   // 4. Collapse from the stripMinedShape to the padded result.
-  auto reshapeOp = rewriter.create<tensor::CollapseShapeOp>(
+  auto reshapeOp = tensor::CollapseShapeOp::create(rewriter,
       loc, collapsedType, transposeOp->getResult(0),
       packingMetadata.reassociations);
 
   // 5. ExtractSlice.
   int64_t destRank = destTensorType.getRank();
-  auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+  auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
       loc, destTensorType, reshapeOp->getResult(0),
       SmallVector<OpFoldResult>(destRank, zero),
       tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()),
       SmallVector<OpFoldResult>(destRank, one));
 
   // 6. Inject a copy to preserve DPS.
-  auto copyOp = rewriter.create<linalg::CopyOp>(
+  auto copyOp = linalg::CopyOp::create(rewriter,
       loc, extractSliceOp->getResult(0), unPackOp.getDest());
 
   // 7. Replace unPackOp by copyOp.
@@ -554,15 +554,15 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
               operandType.getShape(), innerPos,
               cast<ShapedType>(dest.getType()).getShape(), {},
               innerPackSizes)) {
-        packOps.push_back(rewriter.create<linalg::PackOp>(
+        packOps.push_back(linalg::PackOp::create(rewriter,
             loc, operand, dest, innerPos, innerPackSizes));
       } else {
         // TODO: value of the padding attribute should be determined by
         // consumers.
         auto zeroAttr =
             rewriter.getZeroAttr(getElementTypeOrSelf(dest.getType()));
-        Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
-        packOps.push_back(rewriter.create<linalg::PackOp>(
+        Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
+        packOps.push_back(linalg::PackOp::create(rewriter,
             loc, operand, dest, innerPos, innerPackSizes, zero));
       }
       inputsAndInits.push_back(packOps.back());
@@ -574,7 +574,7 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       ValueRange{inputsAndInits}.take_front(linalgOp.getNumDpsInputs());
   ValueRange inits =
       ValueRange{inputsAndInits}.take_back(linalgOp.getNumDpsInits());
-  auto packedLinalgOp = rewriter.create<linalg::GenericOp>(
+  auto packedLinalgOp = linalg::GenericOp::create(rewriter,
       linalgOp.getLoc(), inits.getTypes(), inputs, inits, indexingMaps,
       iteratorTypes);
   packedLinalgOp.getRegion().takeBody(linalgOp->getRegion(0));
@@ -589,7 +589,7 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       continue;
     }
     // Build the symmetrical UnPackOp to the existing PackOp.
-    unPackOps.push_back(rewriter.create<linalg::UnPackOp>(
+    unPackOps.push_back(linalg::UnPackOp::create(rewriter,
         packedLinalgOp->getLoc(), result, maybePackedInit.getSource(),
         maybePackedInit.getInnerDimsPos(), maybePackedInit.getMixedTiles()));
     results.push_back(unPackOps.back());
@@ -655,7 +655,7 @@ static LinalgOp transposeOneLinalgOperandAndReplace(
   operands[opOperand.getOperandNumber()] = transposedValue;
 
   ValueRange operandsRef(operands);
-  auto transposedGenericOp = rewriter.create<linalg::GenericOp>(
+  auto transposedGenericOp = linalg::GenericOp::create(rewriter,
       /*location=*/linalgOp->getLoc(),
       /*resultTensorTypes=*/
       operandsRef.drop_front(linalgOp.getNumDpsInputs()).getTypes(),
@@ -904,7 +904,7 @@ mlir::linalg::LinalgTilingOptions::setTileSizes(ArrayRef<int64_t> ts) {
     b.setInsertionPointToStart(
         &op->getParentOfType<func::FuncOp>().getBody().front());
     return llvm::to_vector<4>(map_range(tileSizes, [&](int64_t s) {
-      Value v = b.create<arith::ConstantIndexOp>(op->getLoc(), s);
+      Value v = arith::ConstantIndexOp::create(b, op->getLoc(), s);
       return v;
     }));
   };
@@ -926,11 +926,11 @@ Value DecomposePadOpPattern::createFillOrGenerateOp(
     // Move the padding value defined inside the PadOp block to outside.
     if (padValue.getParentBlock() == &padOp.getRegion().front())
       rewriter.moveOpBefore(padValue.getDefiningOp(), padOp);
-    return rewriter.create<FillOp>(padOp.getLoc(), padValue, dest).result();
+    return FillOp::create(rewriter, padOp.getLoc(), padValue, dest).result();
   }
 
   // Fill could not be optimized: Lower to tensor::GenerateOp with region.
-  auto generateOp = rewriter.create<tensor::GenerateOp>(
+  auto generateOp = tensor::GenerateOp::create(rewriter,
       padOp.getLoc(), padOp.getResultType(), dynSizes);
   // Copy region to new op.
   IRMapping bvm;
@@ -970,7 +970,7 @@ DecomposePadOpPattern::matchAndRewrite(tensor::PadOp padOp,
   }
 
   // Init tensor and fill it with padding.
-  Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+  Value emptyTensor = tensor::EmptyOp::create(rewriter,
       padOp.getLoc(), staticSizes, resultType.getElementType(), dynSizes);
   Value fill = createFillOrGenerateOp(rewriter, padOp, emptyTensor, dynSizes);
 
@@ -1222,11 +1222,11 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
 
   applyPermutationToVector<OpFoldResult>(transShapeForEmptyOp,
                                          srcPermForTranspose);
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, transShapeForEmptyOp, packOp.getSourceType().getElementType());
 
   // 2.2 Create linalg.transpose
-  auto transposedOp = rewriter.create<linalg::TransposeOp>(loc, input, empty,
+  auto transposedOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
                                                            srcPermForTranspose);
 
   // 3. Insert the inner tile to the destination:
@@ -1246,7 +1246,7 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   }
 
   // 4. Replace tensor.packOp with tensor.insert_slice created above
-  auto insert = rewriter.create<tensor::InsertSliceOp>(
+  auto insert = tensor::InsertSliceOp::create(rewriter,
       loc, transposedOp.getResult()[0], packOp.getDest(), writeOffsets,
       writeSizes, writeStrides);
   rewriter.replaceOp(packOp, insert.getResult());
@@ -1313,7 +1313,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     // outer-untiled-dims
     if (ShapedType::isDynamic(srcShape[i])) {
       OpFoldResult dynamicDim =
-          rewriter.create<tensor::DimOp>(loc, source, i).getResult();
+          tensor::DimOp::create(rewriter, loc, source, i).getResult();
       extractSliceSizes.push_back(dynamicDim);
       shapeForEmptyOp.push_back(dynamicDim);
     } else {
@@ -1340,7 +1340,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   readShapeForExtractSlice.append(tileShape.begin(), tileShape.end());
   Type elemType = unpackOp.getSourceType().getElementType();
   auto readType = RankedTensorType::get(readShapeForExtractSlice, elemType);
-  Value innerTile = rewriter.create<tensor::ExtractSliceOp>(
+  Value innerTile = tensor::ExtractSliceOp::create(rewriter,
       loc, readType, unpackOp.getSource(), extractSliceOffsets,
       extractSliceSizes, extractSliceStrides);
 
@@ -1352,9 +1352,9 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   applyPermutationToVector<OpFoldResult>(shapeForEmptyOp, perm);
 
   Value empty =
-      rewriter.create<tensor::EmptyOp>(loc, shapeForEmptyOp, elemType);
+      tensor::EmptyOp::create(rewriter, loc, shapeForEmptyOp, elemType);
   auto transposedOp =
-      rewriter.create<linalg::TransposeOp>(loc, innerTile, empty, perm);
+      linalg::TransposeOp::create(rewriter, loc, innerTile, empty, perm);
 
   // 3. Handle in-complete tiles if needed. It truncates trailing data from the
   // transposed tile.
@@ -1369,7 +1369,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
           tensor::getMixedSize(rewriter, loc, unpackOp.getDest(), i));
   }
 
-  auto partialTile = rewriter.create<tensor::ExtractSliceOp>(
+  auto partialTile = tensor::ExtractSliceOp::create(rewriter,
       loc, transposedOp.getResult()[0], tileOffsets, tileSizes, tileStrides);
 
   // 4. Insert the result to the destination tensor.
@@ -1382,7 +1382,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     else
       writeSizes.push_back(oneIdxAttr);
   }
-  auto insert = rewriter.create<tensor::InsertSliceOp>(
+  auto insert = tensor::InsertSliceOp::create(rewriter,
       loc, partialTile, unpackOp.getDest(), writeOffsets, writeSizes,
       writeStrides);
   rewriter.replaceOp(unpackOp, insert.getResult());
@@ -1491,7 +1491,7 @@ FailureOr<Conv1DOp> DownscaleSizeOneWindowed2DConvolution<Conv2DOp, Conv1DOp>::
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = rewriter.create<Conv1DOp>(
+  auto conv1DOp = Conv1DOp::create(rewriter,
       loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
@@ -1578,7 +1578,7 @@ DownscaleDepthwiseConv2DNhwcHwcOp::returningMatchAndRewrite(
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = rewriter.create<DepthwiseConv1DNwcWcOp>(
+  auto conv1DOp = DepthwiseConv1DNwcWcOp::create(rewriter,
       loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
@@ -1635,7 +1635,7 @@ DownscaleConv2DOp::returningMatchAndRewrite(Conv2DOp convOp,
   Value newOutput = tensor::createCanonicalRankReducingExtractSliceOp(
       rewriter, loc, output, newOutputType);
 
-  auto conv1DOp = rewriter.create<Conv1DOp>(loc, newOutputType,
+  auto conv1DOp = Conv1DOp::create(rewriter, loc, newOutputType,
                                             ValueRange{newInput, newKernel},
                                             ValueRange{newOutput});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index 092aecceef6b3..b933a2d143925 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -67,7 +67,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
   Value input;
   if (isTensorOp) {
 
-    input = rewriter.create<tensor::EmptyOp>(loc, newFilterShape, elementTy)
+    input = tensor::EmptyOp::create(rewriter, loc, newFilterShape, elementTy)
                 .getResult();
   } else {
     input = rewriter
@@ -78,7 +78,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
 
   // We can then construct the transposition on our filter.
   auto transpose =
-      rewriter.create<linalg::TransposeOp>(loc, filter, input, filterPerm);
+      linalg::TransposeOp::create(rewriter, loc, filter, input, filterPerm);
 
   Value newFilter;
   if (isTensorOp) {
@@ -98,7 +98,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
     resultTy.push_back(op->getResult(0).getType());
   }
   auto newConv =
-      rewriter.create<HWCFConvOp>(loc, resultTy, newInputs, op.getOutputs(),
+      HWCFConvOp::create(rewriter, loc, resultTy, newInputs, op.getOutputs(),
                                   op.getStrides(), op.getDilations());
   rewriter.replaceOp(op, newConv);
   return newConv.getOperation();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index 934781d1cab75..6b1e89c1594d1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -47,24 +47,24 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
 
   SmallVector<Value> dynamicDims;
   if (type.isDynamicDim(1))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 1));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
   if (type.isDynamicDim(0))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, ArrayRef<int64_t>{shape[1], shape[0]}, type.getElementType(),
       dynamicDims);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, input, empty, ArrayRef<int64_t>{1, 0});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = rewriter.create<linalg::MatmulTransposeAOp>(
+    newMatmulOp = linalg::MatmulTransposeAOp::create(rewriter,
         loc, matmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), matmulOp.getInputs()[1]},
         matmulOp.getOutputs());
   } else {
-    newMatmulOp = rewriter.create<linalg::MatmulTransposeBOp>(
+    newMatmulOp = linalg::MatmulTransposeBOp::create(rewriter,
         loc, matmulOp.getResultTypes(),
         ValueRange{matmulOp.getInputs()[0], transposeOp->getResult(0)},
         matmulOp.getOutputs());
@@ -102,26 +102,26 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
 
   SmallVector<Value> dynamicDims;
   if (type.isDynamicDim(0))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
   if (type.isDynamicDim(2))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 2));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 2));
   if (type.isDynamicDim(1))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 1));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, ArrayRef<int64_t>{shape[0], shape[2], shape[1]},
       type.getElementType(), dynamicDims);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, input, empty, ArrayRef<int64_t>{0, 2, 1});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = rewriter.create<linalg::BatchMatmulTransposeAOp>(
+    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(rewriter,
         loc, batchMatmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), batchMatmulOp.getInputs()[1]},
         batchMatmulOp.getOutputs());
   } else {
-    newMatmulOp = rewriter.create<linalg::BatchMatmulTransposeBOp>(
+    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(rewriter,
         loc, batchMatmulOp.getResultTypes(),
         ValueRange{batchMatmulOp.getInputs()[0], transposeOp->getResult(0)},
         batchMatmulOp.getOutputs());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 5a8c5eab3f444..2fff28ffc7474 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -119,7 +119,7 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, input, /*offsets=*/ArrayRef<int64_t>{w + kw}, sizes, strides));
       }
     }
@@ -130,7 +130,7 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, input,
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             sizes, strides));
@@ -149,7 +149,7 @@ static SmallVector<Value> extractConvFilterSlices(RewriterBase &rewriter,
   // Extract rhs slice of size [{c, f} for channeled convolutions and {1} for
   // non-chanelled convolution] @ [kw].
   for (int64_t kw = 0; kw < kwSize; ++kw) {
-    result.push_back(rewriter.create<vector::ExtractOp>(
+    result.push_back(vector::ExtractOp::create(rewriter,
         loc, filter, /*offsets=*/ArrayRef<int64_t>{kw}));
   }
   return result;
@@ -167,7 +167,7 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {wSizeStep};
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/ArrayRef<int64_t>{w}, sizes, strides));
     }
   } else {
@@ -176,7 +176,7 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {nSize, wSizeStep, fSize};
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/ArrayRef<int64_t>{0, w, 0}, sizes, strides));
     }
   }
@@ -194,7 +194,7 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // This does not depend on kw.
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{w}, strides);
     }
   } else {
@@ -202,7 +202,7 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // convolution. This does not depend on kw.
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           strides);
     }
@@ -337,7 +337,7 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
   for (int vecDim = 0, end = canonicalVecShape.size(); vecDim < end; ++vecDim) {
     if (ShapedType::isStatic(iterSpaceStaticSizes[vecDim])) {
       // Create constant index op for static dimensions.
-      iterSpaceValueSizes.push_back(rewriter.create<arith::ConstantIndexOp>(
+      iterSpaceValueSizes.push_back(arith::ConstantIndexOp::create(rewriter,
           linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
       continue;
     }
@@ -351,9 +351,9 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
       return failure();
 
     Value dynamicDim = linalgOp.hasPureTensorSemantics()
-                           ? (Value)rewriter.create<tensor::DimOp>(
+                           ? (Value)tensor::DimOp::create(rewriter,
                                  linalgOp.getLoc(), operand, operandDimPos)
-                           : (Value)rewriter.create<memref::DimOp>(
+                           : (Value)memref::DimOp::create(rewriter,
                                  linalgOp.getLoc(), operand, operandDimPos);
     iterSpaceValueSizes.push_back(dynamicDim);
   }
@@ -474,7 +474,7 @@ Value VectorizationState::getOrCreateMaskFor(
          "Masked 0-d vectors are not supported yet");
 
   // Create the mask based on the dimension values.
-  Value mask = rewriter.create<vector::CreateMaskOp>(linalgOp.getLoc(),
+  Value mask = vector::CreateMaskOp::create(rewriter, linalgOp.getLoc(),
                                                      maskType, upperBounds);
   LDBG("Creating new mask: " << mask << "\n");
   activeMaskCache[maskingMap] = mask;
@@ -643,7 +643,7 @@ static Operation *buildMultiDimReduce(OpBuilder &b, Operation *reduceOp,
                                       ArrayRef<bool> dimsToMask) {
   auto maybeKind = getCombinerOpKind(reduceOp);
   assert(maybeKind && "Failed precondition: could not get reduction kind");
-  return b.create<vector::MultiDimReductionOp>(
+  return vector::MultiDimReductionOp::create(b,
       reduceOp->getLoc(), valueToReduce, acc, dimsToMask, *maybeKind);
 }
 
@@ -689,17 +689,17 @@ static Value buildVectorWrite(RewriterBase &rewriter, Value value,
   if (vectorType.getRank() > 0) {
     AffineMap writeMap = inversePermutation(reindexIndexingMap(opOperandMap));
     SmallVector<Value> indices(linalgOp.getRank(outputOperand),
-                               rewriter.create<arith::ConstantIndexOp>(loc, 0));
+                               arith::ConstantIndexOp::create(rewriter, loc, 0));
     value = broadcastIfNeeded(rewriter, value, vectorType);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = rewriter.create<vector::TransferWriteOp>(
+    write = vector::TransferWriteOp::create(rewriter,
         loc, value, outputOperand->get(), indices, writeMap);
   } else {
     // 0-d case is still special: do not invert the reindexing writeMap.
     if (!isa<VectorType>(value.getType()))
-      value = rewriter.create<vector::BroadcastOp>(loc, vectorType, value);
+      value = vector::BroadcastOp::create(rewriter, loc, vectorType, value);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = rewriter.create<vector::TransferWriteOp>(
+    write = vector::TransferWriteOp::create(rewriter,
         loc, value, outputOperand->get(), ValueRange{});
   }
 
@@ -778,7 +778,7 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto indexVectorType =
       VectorType::get({targetShape[dim]}, rewriter.getIndexType(),
                       state.getScalableVecDims()[dim]);
-  auto indexSteps = rewriter.create<vector::StepOp>(loc, indexVectorType);
+  auto indexSteps = vector::StepOp::create(rewriter, loc, indexVectorType);
   // Return the one-dimensional index vector if it lives in the trailing
   // dimension of the iteration space since the vectorization algorithm in this
   // case can handle the broadcast.
@@ -793,14 +793,14 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto permMap =
       AffineMap::getPermutationMap(permPattern, linalgOp.getContext());
 
-  auto broadCastOp = rewriter.create<vector::BroadcastOp>(
+  auto broadCastOp = vector::BroadcastOp::create(rewriter,
       loc, state.getCanonicalVecType(rewriter.getIndexType(), permMap),
       indexSteps);
   SmallVector<int64_t> transposition =
       llvm::to_vector<16>(llvm::seq<int64_t>(0, linalgOp.getNumLoops()));
   std::swap(transposition.back(), transposition[dim]);
   auto transposeOp =
-      rewriter.create<vector::TransposeOp>(loc, broadCastOp, transposition);
+      vector::TransposeOp::create(rewriter, loc, broadCastOp, transposition);
   return VectorizationHookResult{VectorizationHookStatus::NewOp, transposeOp};
 }
 
@@ -853,19 +853,19 @@ static Value calculateGatherOffset(RewriterBase &rewriter,
 
   const size_t numIndices = extractOp.getIndices().size();
   for (size_t i = 1; i < numIndices; i++) {
-    Value dimIdx = rewriter.create<arith::ConstantIndexOp>(loc, i);
+    Value dimIdx = arith::ConstantIndexOp::create(rewriter, loc, i);
 
     auto dimSize = broadcastIfNeeded(
         rewriter,
-        rewriter.create<tensor::DimOp>(loc, extractOp.getTensor(), dimIdx),
+        tensor::DimOp::create(rewriter, loc, extractOp.getTensor(), dimIdx),
         indexVecType);
 
-    offset = rewriter.create<arith::MulIOp>(loc, offset, dimSize);
+    offset = arith::MulIOp::create(rewriter, loc, offset, dimSize);
 
     auto extractOpIndex = broadcastIfNeeded(
         rewriter, bvm.lookup(extractOp.getIndices()[i]), indexVecType);
 
-    offset = rewriter.create<arith::AddIOp>(loc, extractOpIndex, offset);
+    offset = arith::AddIOp::create(rewriter, loc, extractOpIndex, offset);
   }
 
   return offset;
@@ -1110,18 +1110,18 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
 
   // Compute the static loop sizes of the extract op.
   auto resultType = state.getCanonicalVecType(extractOp.getResult().getType());
-  auto maskConstantOp = rewriter.create<arith::ConstantOp>(
+  auto maskConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseIntElementsAttr::get(state.getCanonicalVecType(rewriter.getI1Type()),
                                 /*value=*/true));
   auto passThruConstantOp =
-      rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(resultType));
+      arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(resultType));
 
   // Base indices are currently set to 0. We will need to re-visit if more
   // generic scenarios are to be supported.
   SmallVector<Value> baseIndices(
       extractOp.getIndices().size(),
-      rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      arith::ConstantIndexOp::create(rewriter, loc, 0));
 
   VectorMemoryAccessKind memAccessKind =
       getTensorExtractMemoryAccessPattern(extractOp, linalgOp, resultType);
@@ -1131,7 +1131,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     Value offset = calculateGatherOffset(rewriter, state, extractOp, bvm);
 
     // Generate the gather load
-    Operation *gatherOp = rewriter.create<vector::GatherOp>(
+    Operation *gatherOp = vector::GatherOp::create(rewriter,
         loc, resultType, extractOp.getTensor(), baseIndices, offset,
         maskConstantOp, passThruConstantOp);
     gatherOp = state.maskOperation(rewriter, gatherOp, linalgOp);
@@ -1166,13 +1166,13 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
       continue;
     }
 
-    auto indexAs1dVector = rewriter.create<vector::ShapeCastOp>(
+    auto indexAs1dVector = vector::ShapeCastOp::create(rewriter,
         loc,
         VectorType::get(resultType.getShape().back(), rewriter.getIndexType(),
                         resultType.getScalableDims().back()),
         idx);
     transferReadIdxs.push_back(
-        rewriter.create<vector::ExtractOp>(loc, indexAs1dVector, 0));
+        vector::ExtractOp::create(rewriter, loc, indexAs1dVector, 0));
   }
 
   // `tensor.extract_element` is always in-bounds, hence the following holds.
@@ -1186,7 +1186,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     SmallVector<AffineExpr> exprs(dstRank, getAffineConstantExpr(0, ctx));
     auto permutationMap = AffineMap::get(srcRank, 0, exprs, ctx);
 
-    auto transferReadOp = rewriter.create<vector::TransferReadOp>(
+    auto transferReadOp = vector::TransferReadOp::create(rewriter,
         loc, resultType, extractOp.getTensor(), transferReadIdxs,
         /*padding=*/std::nullopt, permutationMap, inBounds);
 
@@ -1195,7 +1195,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     // valid here).
     SmallVector<int64_t> readMaskShape = {1};
     auto readMaskType = VectorType::get(readMaskShape, rewriter.getI1Type());
-    auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+    auto allTrue = vector::ConstantMaskOp::create(rewriter,
         loc, readMaskType, vector::ConstantMaskKind::AllTrue);
     auto *maskedReadOp =
         mlir::vector::maskOperation(rewriter, transferReadOp, allTrue);
@@ -1223,7 +1223,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     rankDiff--;
   }
 
-  auto transferReadOp = rewriter.create<vector::TransferReadOp>(
+  auto transferReadOp = vector::TransferReadOp::create(rewriter,
       loc, resultType, extractOp.getTensor(), transferReadIdxs,
       /*padding=*/std::nullopt, permutationMap, inBounds);
 
@@ -1405,7 +1405,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
   // 3. Turn all BBArgs into vector.transfer_read / load.
   Location loc = linalgOp.getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   for (OpOperand *opOperand : linalgOp.getOpOperandsMatchingBBargs()) {
     BlockArgument bbarg = linalgOp.getMatchingBlockArgument(opOperand);
     if (linalgOp.isScalar(opOperand)) {
@@ -1435,7 +1435,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
-    Operation *read = rewriter.create<vector::TransferReadOp>(
+    Operation *read = vector::TransferReadOp::create(rewriter,
         loc, readType, opOperand->get(), indices,
         /*padding=*/std::nullopt, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
@@ -1452,7 +1452,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     // 3.c. Not all ops support 0-d vectors, extract the scalar for now.
     // TODO: remove this.
     if (readType.getRank() == 0)
-      readValue = rewriter.create<vector::ExtractOp>(loc, readValue,
+      readValue = vector::ExtractOp::create(rewriter, loc, readValue,
                                                      ArrayRef<int64_t>());
 
     LDBG("New vectorized bbarg(" << bbarg.getArgNumber() << "): " << readValue
@@ -1660,13 +1660,13 @@ createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
           writeIndices.size() == static_cast<size_t>(destRank)) &&
          "Invalid number of write indices!");
   if (writeIndices.empty()) {
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     writeIndices.assign(destRank, zero);
   }
 
   // Generate the xfer_write Op
   Operation *write =
-      builder.create<vector::TransferWriteOp>(loc,
+      vector::TransferWriteOp::create(builder, loc,
                                               /*vector=*/vecToStore,
                                               /*source=*/dest,
                                               /*indices=*/writeIndices,
@@ -1742,7 +1742,7 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   Location loc = packOp.getLoc();
   auto padValue = packOp.getPaddingValue();
   if (!padValue) {
-    padValue = rewriter.create<arith::ConstantOp>(
+    padValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(packOp.getSourceType().getElementType()));
   }
   ReifiedRankedShapedTypeDims reifiedReturnShapes;
@@ -1782,16 +1782,16 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   auto tiledPackType = VectorType::get(getTiledPackShape(packOp, destShape),
                                        packOp.getDestType().getElementType());
   auto shapeCastOp =
-      rewriter.create<vector::ShapeCastOp>(loc, tiledPackType, maskedRead);
+      vector::ShapeCastOp::create(rewriter, loc, tiledPackType, maskedRead);
 
   // Create TransposeOp.
   auto destPermutation =
       invertPermutationVector(getPackInverseDestPerm(packOp));
-  auto transposeOp = rewriter.create<vector::TransposeOp>(
+  auto transposeOp = vector::TransposeOp::create(rewriter,
       loc, shapeCastOp.getResult(), destPermutation);
 
   // Create TransferWriteOp.
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedReturnShapes[0],
       transposeOp.getResult().getType().getElementType());
   Operation *write =
@@ -1892,7 +1892,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   }
   Location loc = unpackOp->getLoc();
 
-  auto padValue = rewriter.create<arith::ConstantOp>(
+  auto padValue = arith::ConstantOp::create(rewriter,
       loc, rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
 
   // Read result, mask if necessary. If transferReadOp shape is not equal
@@ -1911,7 +1911,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   RankedTensorType stripMineTensorType =
       RankedTensorType::get(stripMineShape, stripMineElemType);
   // Transpose the appropriate rows to match output.
-  vector::TransposeOp transposeOp = rewriter.create<vector::TransposeOp>(
+  vector::TransposeOp transposeOp = vector::TransposeOp::create(rewriter,
       loc, readResult, lastDimToInsertPosPerm);
 
   // Collapse the vector to the size required by result.
@@ -1919,7 +1919,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       stripMineTensorType, packMetadata.reassociations);
   mlir::VectorType vecCollapsedType =
       VectorType::get(collapsedType.getShape(), collapsedType.getElementType());
-  vector::ShapeCastOp shapeCastOp = rewriter.create<vector::ShapeCastOp>(
+  vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(rewriter,
       loc, vecCollapsedType, transposeOp->getResult(0));
 
   // writeVectorSizes had to match the shapecast shape for dynamic sizes,
@@ -1928,7 +1928,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       unpackOp.getDestType().hasStaticShape()
           ? vectorSizes
           : shapeCastOp.getResultVectorType().getShape());
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedRetShapes[0],
       shapeCastOp.getResult().getType().getElementType());
   Operation *write = createWriteOrMaskedWrite(
@@ -1963,7 +1963,7 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
       /*useInBoundsInsteadOfMasking=*/false);
 
   // Create Xfer write Op
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedReturnShapes[0], padOp.getResultType().getElementType());
   Operation *write = createWriteOrMaskedWrite(rewriter, loc, maskedRead, dest);
   newResults.push_back(write->getResult(0));
@@ -2634,19 +2634,19 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
   auto writeType = VectorType::get(dstType.getShape(), dstElementType);
 
   Location loc = copyOp->getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   SmallVector<Value> indices(srcType.getRank(), zero);
 
-  Value readValue = rewriter.create<vector::TransferReadOp>(
+  Value readValue = vector::TransferReadOp::create(rewriter,
       loc, readType, copyOp.getSource(), indices,
       /*padding=*/std::nullopt,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   if (cast<VectorType>(readValue.getType()).getRank() == 0) {
     readValue =
-        rewriter.create<vector::ExtractOp>(loc, readValue, ArrayRef<int64_t>());
-    readValue = rewriter.create<vector::BroadcastOp>(loc, writeType, readValue);
+        vector::ExtractOp::create(rewriter, loc, readValue, ArrayRef<int64_t>());
+    readValue = vector::BroadcastOp::create(rewriter, loc, writeType, readValue);
   }
-  Operation *writeValue = rewriter.create<vector::TransferWriteOp>(
+  Operation *writeValue = vector::TransferWriteOp::create(rewriter,
       loc, readValue, copyOp.getTarget(), indices,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   rewriter.replaceOp(copyOp, writeValue->getResults());
@@ -2957,7 +2957,7 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   if (!padValue) {
     auto elemType = sourceType.getElementType();
-    padValue = rewriter.create<arith::ConstantOp>(
+    padValue = arith::ConstantOp::create(rewriter,
         sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
   }
 
@@ -2989,7 +2989,7 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   // Create read
   SmallVector<Value> readIndices(
-      vecType.getRank(), rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      vecType.getRank(), arith::ConstantIndexOp::create(rewriter, loc, 0));
   Value read = mlir::vector::createReadOrMaskedRead(
       rewriter, loc, source, vecType.getShape(), padValue,
       /*useInBoundsInsteadOfMasking=*/inputVectorSizes.empty());
@@ -3076,8 +3076,8 @@ struct PadOpVectorizationWithInsertSlicePattern
     // Generate TransferReadOp: Read entire source tensor and add high
     // padding.
     SmallVector<Value> readIndices(
-        vecRank, rewriter.create<arith::ConstantIndexOp>(padOp.getLoc(), 0));
-    auto read = rewriter.create<vector::TransferReadOp>(
+        vecRank, arith::ConstantIndexOp::create(rewriter, padOp.getLoc(), 0));
+    auto read = vector::TransferReadOp::create(rewriter,
         padOp.getLoc(), vecType, padOp.getSource(), readIndices, padValue);
 
     // Generate TransferWriteOp: Write to InsertSliceOp's dest tensor at
@@ -3212,7 +3212,7 @@ LogicalResult LinalgCopyVTRForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_read, the attribute must be reset
   // conservatively.
   auto vectorType = xferOp.getVectorType();
-  Value res = rewriter.create<vector::TransferReadOp>(
+  Value res = vector::TransferReadOp::create(rewriter,
       xferOp.getLoc(), vectorType, in, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getPadding(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(
@@ -3271,7 +3271,7 @@ LogicalResult LinalgCopyVTWForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_write, the attribute must be reset
   // conservatively.
   auto vector = xferOp.getVector();
-  rewriter.create<vector::TransferWriteOp>(
+  vector::TransferWriteOp::create(rewriter,
       xferOp.getLoc(), vector, out, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(SmallVector<bool>(
@@ -3467,7 +3467,7 @@ struct Conv1DGenerator
     }
 
     vector::TransferWriteOp write;
-    Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
     // w is unrolled (i.e. wSizeStep == 1) iff strideW > 1.
     // When strideW == 1, we can batch the contiguous loads and avoid
@@ -3486,16 +3486,16 @@ struct Conv1DGenerator
     SmallVector<Value> resPadding(resShape.size(), zero);
 
     // Read the whole lhs, rhs and res in one shot (with zero padding).
-    Value lhs = rewriter.create<vector::TransferReadOp>(
+    Value lhs = vector::TransferReadOp::create(rewriter,
         loc, lhsType, lhsShaped, lhsPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     // This is needed only for Conv.
     Value rhs = nullptr;
     if (oper == ConvOperationKind::Conv)
-      rhs = rewriter.create<vector::TransferReadOp>(
+      rhs = vector::TransferReadOp::create(rewriter,
           loc, rhsType, rhsShaped, rhsPadding,
           /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
-    Value res = rewriter.create<vector::TransferReadOp>(
+    Value res = vector::TransferReadOp::create(rewriter,
         loc, resType, resShaped, resPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
 
@@ -3511,16 +3511,16 @@ struct Conv1DGenerator
       // To match base vectorization case, we pre-transpose current case.
       // ncw -> nwc
       static constexpr std::array<int64_t, 3> permLhs = {0, 2, 1};
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, permLhs);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, permLhs);
       // fcw -> wcf
       static constexpr std::array<int64_t, 3> permRhs = {2, 1, 0};
 
       // This is needed only for Conv.
       if (oper == ConvOperationKind::Conv)
-        rhs = rewriter.create<vector::TransposeOp>(loc, rhs, permRhs);
+        rhs = vector::TransposeOp::create(rewriter, loc, rhs, permRhs);
       // nfw -> nwf
       static constexpr std::array<int64_t, 3> permRes = {0, 2, 1};
-      res = rewriter.create<vector::TransposeOp>(loc, res, permRes);
+      res = vector::TransposeOp::create(rewriter, loc, res, permRes);
       break;
     }
     }
@@ -3585,7 +3585,7 @@ struct Conv1DGenerator
     case Conv1DOpOrder::Ncw: {
       // nwf -> nfw
       static constexpr std::array<int64_t, 3> perm = {0, 2, 1};
-      res = rewriter.create<vector::TransposeOp>(loc, res, perm);
+      res = vector::TransposeOp::create(rewriter, loc, res, perm);
       break;
     }
     }
@@ -3609,16 +3609,16 @@ struct Conv1DGenerator
         cast<ShapedType>(val.getType()).cloneWith(std::nullopt, dstElementType);
 
     if (isa<IntegerType>(srcElementType) && isa<FloatType>(dstElementType)) {
-      return rewriter.create<arith::SIToFPOp>(loc, dstType, val);
+      return arith::SIToFPOp::create(rewriter, loc, dstType, val);
     }
 
     if (isa<FloatType>(srcElementType) && isa<FloatType>(dstElementType) &&
         srcWidth < dstWidth)
-      return rewriter.create<arith::ExtFOp>(loc, dstType, val);
+      return arith::ExtFOp::create(rewriter, loc, dstType, val);
 
     if (isa<IntegerType>(srcElementType) && isa<IntegerType>(dstElementType) &&
         srcWidth < dstWidth)
-      return rewriter.create<arith::ExtSIOp>(loc, dstType, val);
+      return arith::ExtSIOp::create(rewriter, loc, dstType, val);
 
     assert(false && "unhandled promotion case");
     return nullptr;
@@ -3633,7 +3633,7 @@ struct Conv1DGenerator
     bindDims(ctx, n, w, f, c);
     lhs = promote(rewriter, loc, lhs, res.getType());
     rhs = promote(rewriter, loc, rhs, res.getType());
-    auto contrationOp = rewriter.create<vector::ContractionOp>(
+    auto contrationOp = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, res,
         /*indexingMaps=*/MapList{{n, w, c}, {c, f}, {n, w, f}},
         /*iteratorTypes=*/ArrayRef<vector::IteratorType>{par, par, par, red});
@@ -3645,7 +3645,7 @@ struct Conv1DGenerator
   // convolution.
   Value conv1dSliceAsOuterProduct(RewriterBase &rewriter, Location loc,
                                   Value lhs, Value rhs, Value res) {
-    return rewriter.create<vector::OuterProductOp>(
+    return vector::OuterProductOp::create(rewriter,
         loc, res.getType(), lhs, rhs, res, vector::CombiningKind::ADD);
   }
 
@@ -3693,7 +3693,7 @@ struct Conv1DGenerator
     bindShapeDims(resShapedType, nSize, wSize);
 
     vector::TransferWriteOp write;
-    Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
     // w is unrolled (i.e. wSizeStep == 1) iff strideW > 1.
     // When strideW == 1, we can batch the contiguous loads and avoid
@@ -3736,28 +3736,28 @@ struct Conv1DGenerator
           cast<LinalgOp>(op).hasPureTensorSemantics(), opToMask, rewriter);
 
       Value maskOp =
-          rewriter.create<vector::CreateMaskOp>(loc, maskType, mixedDims);
+          vector::CreateMaskOp::create(rewriter, loc, maskType, mixedDims);
 
       return mlir::vector::maskOperation(rewriter, opToMask, maskOp);
     };
 
     // Read lhs slice of size {n, w * strideW + kw * dilationW, c} @ [0, 0,
     // 0].
-    Value lhs = rewriter.create<vector::TransferReadOp>(
+    Value lhs = vector::TransferReadOp::create(rewriter,
         loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     auto maybeMaskedLhs = maybeMaskXferOp(
         lhsType.getShape(), lhsType.getScalableDims(), lhs.getDefiningOp());
 
     // Read rhs slice of size {kw, c} @ [0, 0].
-    Value rhs = rewriter.create<vector::TransferReadOp>(
+    Value rhs = vector::TransferReadOp::create(rewriter,
         loc, rhsType, rhsShaped, ValueRange{zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
     auto maybeMaskedRhs = maybeMaskXferOp(
         rhsType.getShape(), rhsType.getScalableDims(), rhs.getDefiningOp());
 
     // Read res slice of size {n, w, c} @ [0, 0, 0].
-    Value res = rewriter.create<vector::TransferReadOp>(
+    Value res = vector::TransferReadOp::create(rewriter,
         loc, resType, resShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
     auto maybeMaskedRes = maybeMaskXferOp(
@@ -3775,7 +3775,7 @@ struct Conv1DGenerator
     //   @ [0, sw * w + dw * kw, 0].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        lhsVals.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        lhsVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, maybeMaskedLhs->getResult(0),
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             inOutSliceSizes, inOutStrides));
@@ -3783,13 +3783,13 @@ struct Conv1DGenerator
     }
     // Extract rhs slice of size {c} @ [kw].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
-      rhsVals.push_back(rewriter.create<vector::ExtractOp>(
+      rhsVals.push_back(vector::ExtractOp::create(rewriter,
           loc, maybeMaskedRhs->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{kw}));
     }
     // Extract res slice: {n, wSizeStep, c} @ [0, w, 0].
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      resVals.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      resVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0}, inOutSliceSizes,
           inOutStrides));
@@ -3815,16 +3815,16 @@ struct Conv1DGenerator
         if (flatten) {
           // Flatten the input and output vectors (collapse the channel
           // dimension)
-          lhsVal = rewriter.create<vector::ShapeCastOp>(
+          lhsVal = vector::ShapeCastOp::create(rewriter,
               loc, lhsTypeAfterFlattening, lhsVals[linearIndex(kw, w)]);
-          resVal = rewriter.create<vector::ShapeCastOp>(
+          resVal = vector::ShapeCastOp::create(rewriter,
               loc, resTypeAfterFlattening, resVals[w]);
         }
         resVals[w] = depthwiseConv1dSliceAsMulAcc(rewriter, loc, lhsVal,
                                                   rhsVals[kw], resVal, flatten);
         if (flatten) {
           // Un-flatten the output vector (restore the channel dimension)
-          resVals[w] = rewriter.create<vector::ShapeCastOp>(
+          resVals[w] = vector::ShapeCastOp::create(rewriter,
               loc, VectorType::get(inOutSliceSizes, resEltType), resVals[w]);
         }
       }
@@ -3843,7 +3843,7 @@ struct Conv1DGenerator
     // Write back res slice: {n, wSizeStep, c} @ [0, w, 0].
     // This does not depend on kw.
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      maybeMaskedRes = rewriter.create<vector::InsertStridedSliceOp>(
+      maybeMaskedRes = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           /*strides=*/ArrayRef<int64_t>{1, 1, 1});
@@ -3853,7 +3853,7 @@ struct Conv1DGenerator
     //===------------------------------------------------------------------===//
 
     // Write back res slice of size {n, w, c} @ [0, 0, 0].
-    Operation *resOut = rewriter.create<vector::TransferWriteOp>(
+    Operation *resOut = vector::TransferWriteOp::create(rewriter,
         loc, maybeMaskedRes->getResult(0), resShaped,
         ValueRange{zero, zero, zero});
     return maybeMaskXferOp(resType.getShape(), resType.getScalableDims(),
@@ -3891,10 +3891,10 @@ struct Conv1DGenerator
           indices.push_back(j);
       }
 
-      rhs = rewriter.create<vector::ShuffleOp>(loc, rhs, rhs, indices);
+      rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, indices);
     }
     // Broadcast the filter to match the output vector
-    rhs = rewriter.create<vector::BroadcastOp>(
+    rhs = vector::BroadcastOp::create(rewriter,
         loc, resTy.clone(rhsTy.getElementType()), rhs);
 
     rhs = promote(rewriter, loc, rhs, resTy);
@@ -3903,10 +3903,10 @@ struct Conv1DGenerator
       return nullptr;
 
     if (isa<FloatType>(resTy.getElementType()))
-      return rewriter.create<vector::FMAOp>(loc, lhs, rhs, res);
+      return vector::FMAOp::create(rewriter, loc, lhs, rhs, res);
 
-    auto mul = rewriter.create<arith::MulIOp>(loc, lhs, rhs);
-    return rewriter.create<arith::AddIOp>(loc, mul, res);
+    auto mul = arith::MulIOp::create(rewriter, loc, lhs, rhs);
+    return arith::AddIOp::create(rewriter, loc, mul, res);
   }
 
   /// Entry point for non-channeled convolution:
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index 9fd084487e3fd..f9c6aae9374a8 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -201,7 +201,7 @@ Value create2DTransformMatrix(OpBuilder &builder, Location loc,
                               TransformMatrix transform, Type type) {
   ArrayRef<float> constVec(transform.table, transform.rows * transform.cols);
 
-  return builder.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(builder,
       loc, DenseFPElementsAttr::get(
                RankedTensorType::get(
                    SmallVector<int64_t>{transform.rows, transform.cols}, type),
@@ -233,7 +233,7 @@ Value extract2DDataFrom4D(OpBuilder &builder, Location loc, Value source,
 
   auto extractFilterType =
       RankedTensorType::get({extractHeight, extractWidth}, elementType);
-  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
+  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
       loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
@@ -267,7 +267,7 @@ Value extract2DDataFrom6D(OpBuilder &builder, Location loc, Value source,
   SmallVector<OpFoldResult> strides(srcSize, oneIndex);
 
   auto extractFilterType = RankedTensorType::get({height, width}, elementType);
-  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
+  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
       loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
@@ -293,7 +293,7 @@ Value insert2DDataTo4D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
+  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
       loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
@@ -321,7 +321,7 @@ Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
+  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
       loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
@@ -372,7 +372,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
   if (filterW != r && filterW != 1)
     return Value();
 
-  Value zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
                        ValueRange args) -> scf::ValueVector {
     Value FIter = ivs[0];
@@ -386,7 +386,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
 
     int64_t retRows = 1;
     Value matmulRetValue = extractFilter;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix G.
@@ -401,11 +401,11 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value G = create2DTransformMatrix(builder, loc, GMatrix, elementType);
       // Multiply G x g.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{G, extractFilter}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -423,11 +423,11 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value GT = create2DTransformMatrix(builder, loc, GTMatrix, elementType);
       // Multiply u = (G x g) x GT.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, GT}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -445,9 +445,9 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
     return {insertSliceOp};
   };
 
-  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterF);
-  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterC);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterF);
+  auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterC);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx}, {fUpperBound, cUpperBound},
       {oneStep, oneStep}, {retValue}, buildBody);
@@ -516,9 +516,9 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = builder.create<affine::AffineApplyOp>(
+    Value heightOffset = affine::AffineApplyOp::create(builder,
         loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = builder.create<affine::AffineApplyOp>(
+    Value widthOffset = affine::AffineApplyOp::create(builder,
         loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     // Extract (H, W) from (N, H, W, C).
@@ -530,7 +530,7 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     int64_t retRows = 1;
     int64_t retCols = 1;
     Value matmulRetValue = extractInput;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix BT.
@@ -545,12 +545,12 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value BT =
           create2DTransformMatrix(builder, loc, BTMatrix, builder.getF32Type());
       // Multiply BT x d.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{BT, matmulRetValue}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -568,11 +568,11 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       Value B =
           create2DTransformMatrix(builder, loc, BMatrix, builder.getF32Type());
       // Multiply v = (BT x d) x B.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, B}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -586,12 +586,12 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     return {combinedVal};
   };
 
-  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tileH);
-  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
-  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputN);
-  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputC);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tileH);
+  auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
+  auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputN);
+  auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputC);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
       {tileHBound, tileWBound, nUpperBound, cUpperBound},
@@ -629,7 +629,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       {filterShape[0] * filterShape[1], filterShape[2], filterShape[3]},
       filterElementType);
   SmallVector<ReassociationIndices> filterReassoc = {{0, 1}, {2}, {3}};
-  Value collapseFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value collapseFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, filterReassocType, transformedFilter, filterReassoc);
 
   // Convert (alphaH, alphaW, tileH, tileW, N, C) to
@@ -643,7 +643,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
        inputShape[2] * inputShape[3] * inputShape[4], inputShape[5]},
       inputElementType);
   SmallVector<ReassociationIndices> inputReassoc = {{0, 1}, {2, 3, 4}, {5}};
-  Value collapseInput = rewriter.create<tensor::CollapseShapeOp>(
+  Value collapseInput = tensor::CollapseShapeOp::create(rewriter,
       loc, inputReassocType, transformedInput, inputReassoc);
 
   // Batched matrix multiply.
@@ -655,11 +655,11 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
                     .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                              outputElementType)
                     .getResult();
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, rewriter.getZeroAttr(outputElementType));
-  Value init = rewriter.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+  Value init = linalg::FillOp::create(rewriter, loc, zero, empty).getResult(0);
 
-  auto matmulOp = rewriter.create<linalg::BatchMatmulOp>(
+  auto matmulOp = linalg::BatchMatmulOp::create(rewriter,
       loc, matmulType, ValueRange({collapseInput, collapseFilter}),
       ValueRange{init});
 
@@ -670,7 +670,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       RankedTensorType::get({inputShape[0], inputShape[1], inputShape[2],
                              inputShape[3], inputShape[4], filterShape[3]},
                             outputElementType);
-  auto expandOutput = rewriter.create<tensor::ExpandShapeOp>(
+  auto expandOutput = tensor::ExpandShapeOp::create(rewriter,
       loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
   return expandOutput;
 }
@@ -750,15 +750,15 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
     int64_t retRows = leftTransform ? ATMatrix.rows : 1;
 
     Value matmulRetValue = extractValue;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
 
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = builder.create<affine::AffineApplyOp>(
+    Value heightOffset = affine::AffineApplyOp::create(builder,
         loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = builder.create<affine::AffineApplyOp>(
+    Value widthOffset = affine::AffineApplyOp::create(builder,
         loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     Value outInitVal =
@@ -775,12 +775,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                          .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                                   elementType)
                          .getResult();
-        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+        init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       }
 
       Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
       // Multiply AT x m.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{AT, matmulRetValue}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -794,19 +794,19 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                          .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                                   elementType)
                          .getResult();
-        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+        init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       }
 
       Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
       // Multiply y = (AT x m) x A.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, A}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
     if (scalarFactor != 1) {
       // Multiply by scalar factor and add outInitVal.
-      Value scalarFactorValue = builder.create<arith::ConstantOp>(
+      Value scalarFactorValue = arith::ConstantOp::create(builder,
           loc, FloatAttr::get(elementType, scalarFactor));
       auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
       auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
@@ -824,11 +824,11 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                       utils::IteratorType::parallel},
                   [&](OpBuilder &nestedBuilder, Location nestedLoc,
                       ValueRange args) {
-                    auto mulf = nestedBuilder.create<arith::MulFOp>(
+                    auto mulf = arith::MulFOp::create(nestedBuilder,
                         nestedLoc, args[0], args[1]);
-                    auto addf = nestedBuilder.create<arith::AddFOp>(
+                    auto addf = arith::AddFOp::create(nestedBuilder,
                         nestedLoc, mulf.getResult(), args[2]);
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc,
+                    linalg::YieldOp::create(nestedBuilder, nestedLoc,
                                                           addf.getResult());
                   })
               .getResult(0);
@@ -847,12 +847,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
   int64_t tilwH = valueShape[2];
   int64_t tileW = valueShape[3];
-  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tilwH);
-  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
-  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueN);
-  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueF);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tilwH);
+  auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
+  auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueN);
+  auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueF);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
       {tileHBound, tileWBound, nUpperBound, fUpperBound},
@@ -867,7 +867,7 @@ static Value padToAlignedTensor(RewriterBase &rewriter, Location loc,
   auto valueType = cast<ShapedType>(value.getType());
   Type elementType = valueType.getElementType();
   auto alignedType = RankedTensorType::get(alignedShape, elementType);
-  Value padValue = rewriter.create<arith::ConstantOp>(
+  Value padValue = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
 
   return linalg::makeComposedPadHighOp(rewriter, loc, alignedType, value,
@@ -887,7 +887,7 @@ static Value extractFromAlignedTensor(RewriterBase &rewriter, Location loc,
   SmallVector<OpFoldResult> sizes =
       getAsOpFoldResult(rewriter.getI64ArrayAttr(extractedShape));
 
-  return rewriter.create<tensor::ExtractSliceOp>(loc, extractedType, value,
+  return tensor::ExtractSliceOp::create(rewriter, loc, extractedType, value,
                                                  offsets, sizes, strides);
 }
 
@@ -979,9 +979,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   int64_t tileW = llvm::divideCeilSigned(outputW, widthM);
   auto retType = RankedTensorType::get({alphaH, alphaW, filterC, filterF},
                                        filterElementType);
-  Value retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
+  Value retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
                                                     filterElementType);
-  auto transformedFilter = rewriter.create<linalg::WinogradFilterTransformOp>(
+  auto transformedFilter = linalg::WinogradFilterTransformOp::create(rewriter,
       loc, retType, filter, retValue, fmr);
 
   // --- Create operation for input transform ---
@@ -998,9 +998,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
 
   retType = RankedTensorType::get(
       {alphaH, alphaW, tileH, tileW, inputN, inputC}, inputElementType);
-  retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
+  retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
                                               inputElementType);
-  auto transformedInput = rewriter.create<linalg::WinogradInputTransformOp>(
+  auto transformedInput = linalg::WinogradInputTransformOp::create(rewriter,
       loc, retType, input, retValue, fmr);
 
   Type outputElementType = outputType.getElementType();
@@ -1023,7 +1023,7 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
     outputType = alignedOutputType;
   }
 
-  Value transformedOutput = rewriter.create<linalg::WinogradOutputTransformOp>(
+  Value transformedOutput = linalg::WinogradOutputTransformOp::create(rewriter,
       loc, outputType, matmulRet, output, fmr);
 
   // When output size is not aligned with output tile size, extract the
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index e89e80b07d20d..d964aa5cb4e81 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -320,14 +320,14 @@ GenericOp makeMemRefCopyOp(OpBuilder &b, Location loc, Value from, Value to) {
       AffineMap::getMultiDimIdentityMap(memrefTypeTo.getRank(), b.getContext());
   SmallVector<utils::IteratorType> iteratorTypes(memrefTypeTo.getRank(),
                                                  utils::IteratorType::parallel);
-  return b.create<linalg::GenericOp>(
+  return linalg::GenericOp::create(b,
       loc,
       /*inputs=*/from,
       /*outputs=*/to,
       /*indexingMaps=*/llvm::ArrayRef({id, id}),
       /*iteratorTypes=*/iteratorTypes,
       [](OpBuilder &b, Location loc, ValueRange args) {
-        b.create<linalg::YieldOp>(loc, args.front());
+        linalg::YieldOp::create(b, loc, args.front());
       });
 }
 
@@ -483,7 +483,7 @@ static void generateParallelLoopNest(
   case DistributionMethod::None: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    b.create<scf::ParallelOp>(
+    scf::ParallelOp::create(b,
         loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
@@ -499,7 +499,7 @@ static void generateParallelLoopNest(
   case DistributionMethod::Cyclic: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    b.create<scf::ParallelOp>(
+    scf::ParallelOp::create(b,
         loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
@@ -519,13 +519,13 @@ static void generateParallelLoopNest(
     for (unsigned i = 1; i < numProcessed; ++i)
       cond = ab._and(cond, ab.slt(lbs[i], ubs[i]));
     ivStorage.append(lbs.begin(), std::next(lbs.begin(), numProcessed));
-    b.create<scf::IfOp>(loc, cond, [&](OpBuilder &b, Location loc) {
+    scf::IfOp::create(b, loc, cond, [&](OpBuilder &b, Location loc) {
       generateParallelLoopNest(b, loc, lbs.drop_front(numProcessed),
                                ubs.drop_front(numProcessed),
                                steps.drop_front(numProcessed),
                                iteratorTypes.drop_front(numProcessed),
                                remainderProcInfo, bodyBuilderFn, ivStorage);
-      b.create<scf::YieldOp>(loc, ValueRange{});
+      scf::YieldOp::create(b, loc, ValueRange{});
     });
     return;
   }
@@ -595,12 +595,12 @@ static Operation *materializeTiledShape(OpBuilder &builder, Location loc,
   auto shapedType = dyn_cast<ShapedType>(valueToTile.getType());
   auto *sliceOp = TypeSwitch<ShapedType, Operation *>(shapedType)
                       .Case([&](MemRefType) {
-                        return builder.create<memref::SubViewOp>(
+                        return memref::SubViewOp::create(builder,
                             loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
                       .Case([&](RankedTensorType) {
-                        return builder.create<tensor::ExtractSliceOp>(
+                        return tensor::ExtractSliceOp::create(builder,
                             loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
@@ -793,7 +793,7 @@ SmallVector<Value> insertSlicesBack(OpBuilder &builder, Location loc,
     // `tiledOperands`.
     Value outputTensor = operands[opOperand.getOperandNumber()];
     if (auto sliceOp = outputTensor.getDefiningOp<tensor::ExtractSliceOp>()) {
-      Value inserted = builder.create<tensor::InsertSliceOp>(
+      Value inserted = tensor::InsertSliceOp::create(builder,
           loc, sliceOp.getSource().getType(), results[resultIdx],
           sliceOp.getSource(), sliceOp.getOffsets(), sliceOp.getSizes(),
           sliceOp.getStrides(), sliceOp.getStaticOffsets(),
diff --git a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
index ff6af63eee531..9dadd9bea2f67 100644
--- a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -135,7 +135,7 @@ struct GlobalStoreOpInterface
     auto memrefType = getMemRefTypeWithStaticIdentityLayout(tensorType);
 
     auto loc = globalStoreOp.getLoc();
-    auto targetMemref = rewriter.create<memref::GetGlobalOp>(
+    auto targetMemref = memref::GetGlobalOp::create(rewriter,
         loc, memrefType, globalStoreOp.getGlobalAttr().getLeafReference());
 
     auto sourceMemref =
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index 7940ff60a48e7..749cca0575581 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -60,7 +60,7 @@ struct FoldRank final : public mlir::OpRewritePattern<mlir::mpi::CommRankOp> {
     if (!isa<IntegerAttr>(dltiAttr.value()))
       return op->emitError()
              << "Expected an integer attribute for MPI:comm_world_rank";
-    Value res = b.create<arith::ConstantIndexOp>(
+    Value res = arith::ConstantIndexOp::create(b,
         op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
     if (Value retVal = op.getRetval())
       b.replaceOp(op, {retVal, res});
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index 26441a9d78658..a21631cbf8510 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -746,7 +746,7 @@ Operation *math::MathDialect::materializeConstant(OpBuilder &builder,
                                                   Attribute value, Type type,
                                                   Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
index 13e2a4b5541b2..770f32841c0f6 100644
--- a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
@@ -65,7 +65,7 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
   // Maybe broadcasts scalar value into vector type compatible with `op`.
   auto bcast = [&](Value value) -> Value {
     if (auto vec = dyn_cast<VectorType>(op.getType()))
-      return rewriter.create<vector::BroadcastOp>(op.getLoc(), vec, value);
+      return vector::BroadcastOp::create(rewriter, op.getLoc(), vec, value);
     return value;
   };
 
@@ -84,14 +84,14 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
   // Replace `pow(x, 3.0)` with `x * x * x`.
   if (isExponentValue(3.0)) {
     Value square =
-        rewriter.create<arith::MulFOp>(op.getLoc(), ValueRange({x, x}));
+        arith::MulFOp::create(rewriter, op.getLoc(), ValueRange({x, x}));
     rewriter.replaceOpWithNewOp<arith::MulFOp>(op, ValueRange({x, square}));
     return success();
   }
 
   // Replace `pow(x, -1.0)` with `1.0 / x`.
   if (isExponentValue(-1.0)) {
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getFloatAttr(getElementTypeOrSelf(op.getType()), 1.0));
     rewriter.replaceOpWithNewOp<arith::DivFOp>(op, ValueRange({bcast(one), x}));
     return success();
@@ -111,8 +111,8 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
 
   // Replace `pow(x, 0.75)` with `sqrt(sqrt(x)) * sqrt(x)`.
   if (isExponentValue(0.75)) {
-    Value powHalf = rewriter.create<math::SqrtOp>(op.getLoc(), x);
-    Value powQuarter = rewriter.create<math::SqrtOp>(op.getLoc(), powHalf);
+    Value powHalf = math::SqrtOp::create(rewriter, op.getLoc(), x);
+    Value powQuarter = math::SqrtOp::create(rewriter, op.getLoc(), powHalf);
     rewriter.replaceOpWithNewOp<arith::MulFOp>(op,
                                                ValueRange{powHalf, powQuarter});
     return success();
@@ -168,17 +168,17 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   // Maybe broadcasts scalar value into vector type compatible with `op`.
   auto bcast = [&loc, &op, &rewriter](Value value) -> Value {
     if (auto vec = dyn_cast<VectorType>(op.getType()))
-      return rewriter.create<vector::BroadcastOp>(loc, vec, value);
+      return vector::BroadcastOp::create(rewriter, loc, vec, value);
     return value;
   };
 
   Value one;
   Type opType = getElementTypeOrSelf(op.getType());
   if constexpr (std::is_same_v<PowIOpTy, math::FPowIOp>)
-    one = rewriter.create<arith::ConstantOp>(
+    one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getFloatAttr(opType, 1.0));
   else
-    one = rewriter.create<arith::ConstantOp>(
+    one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(opType, 1));
 
   // Replace `[fi]powi(x, 0)` with `1`.
@@ -208,12 +208,12 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   //     with:
   //       (1 / x) * (1 / x) * (1 / x) * ...
   for (unsigned i = 1; i < exponentValue; ++i)
-    result = rewriter.create<MulOpTy>(loc, result, base);
+    result = MulOpTy::create(rewriter, loc, result, base);
 
   // Inverse the base for negative exponent, i.e. for
   // `[fi]powi(x, negative_exponent)` set `x` to `1 / x`.
   if (exponentIsNegative)
-    result = rewriter.create<DivOpTy>(loc, bcast(one), result);
+    result = DivOpTy::create(rewriter, loc, bcast(one), result);
 
   rewriter.replaceOp(op, result);
   return success();
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index 7b5350ca26b60..419ba0d2e500b 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -33,11 +33,11 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                 APFloat::rmNearestTiesToEven, &losesInfo);
   auto attr = b.getFloatAttr(eltType, value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return b.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(b, loc,
                                        DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return b.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(b, loc, attr);
 }
 
 static Value createFloatConst(Location loc, Type type, double value,
@@ -50,11 +50,11 @@ static Value createIntConst(Location loc, Type type, int64_t value,
                             OpBuilder &b) {
   auto attr = b.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return b.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(b, loc,
                                        DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return b.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(b, loc, attr);
 }
 
 static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
@@ -62,11 +62,11 @@ static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
   Type i64Ty = b.getI64Type();
   if (auto shapedTy = dyn_cast<ShapedType>(opType))
     i64Ty = shapedTy.clone(i64Ty);
-  Value fixedConvert = b.create<arith::FPToSIOp>(i64Ty, operand);
-  Value fpFixedConvert = b.create<arith::SIToFPOp>(opType, fixedConvert);
+  Value fixedConvert = arith::FPToSIOp::create(b, i64Ty, operand);
+  Value fpFixedConvert = arith::SIToFPOp::create(b, opType, fixedConvert);
   // The truncation does not preserve the sign when the truncated
   // value is -0. So here the sign is copied again.
-  return b.create<math::CopySignOp>(fpFixedConvert, operand);
+  return math::CopySignOp::create(b, fpFixedConvert, operand);
 }
 
 // sinhf(float x) -> (exp(x) - exp(-x)) / 2
@@ -75,12 +75,12 @@ static LogicalResult convertSinhOp(math::SinhOp op, PatternRewriter &rewriter) {
   Value operand = op.getOperand();
   Type opType = operand.getType();
 
-  Value exp = b.create<math::ExpOp>(operand);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value nexp = b.create<math::ExpOp>(neg);
-  Value sub = b.create<arith::SubFOp>(exp, nexp);
+  Value exp = math::ExpOp::create(b, operand);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value nexp = math::ExpOp::create(b, neg);
+  Value sub = arith::SubFOp::create(b, exp, nexp);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(sub, half);
+  Value res = arith::MulFOp::create(b, sub, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -91,12 +91,12 @@ static LogicalResult convertCoshOp(math::CoshOp op, PatternRewriter &rewriter) {
   Value operand = op.getOperand();
   Type opType = operand.getType();
 
-  Value exp = b.create<math::ExpOp>(operand);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value nexp = b.create<math::ExpOp>(neg);
-  Value add = b.create<arith::AddFOp>(exp, nexp);
+  Value exp = math::ExpOp::create(b, operand);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value nexp = math::ExpOp::create(b, neg);
+  Value add = arith::AddFOp::create(b, exp, nexp);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(add, half);
+  Value res = arith::MulFOp::create(b, add, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -117,23 +117,23 @@ static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) {
   Value negTwo = createFloatConst(loc, floatType, -2.0, rewriter);
 
   // Compute sign(x) = cast<float_type>(x < 0) * (-2) + 1
-  Value isNegative = rewriter.create<arith::CmpFOp>(
+  Value isNegative = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OLT, op.getOperand(), zero);
   Value isNegativeFloat =
-      rewriter.create<arith::UIToFPOp>(loc, floatType, isNegative);
+      arith::UIToFPOp::create(rewriter, loc, floatType, isNegative);
   Value isNegativeTimesNegTwo =
-      rewriter.create<arith::MulFOp>(loc, isNegativeFloat, negTwo);
-  Value sign = rewriter.create<arith::AddFOp>(loc, isNegativeTimesNegTwo, one);
+      arith::MulFOp::create(rewriter, loc, isNegativeFloat, negTwo);
+  Value sign = arith::AddFOp::create(rewriter, loc, isNegativeTimesNegTwo, one);
 
   // Normalize input to positive value: y = sign(x) * x
-  Value positiveX = rewriter.create<arith::MulFOp>(loc, sign, op.getOperand());
+  Value positiveX = arith::MulFOp::create(rewriter, loc, sign, op.getOperand());
 
   // Decompose on normalized input
-  Value negDoubledX = rewriter.create<arith::MulFOp>(loc, negTwo, positiveX);
-  Value exp2x = rewriter.create<math::ExpOp>(loc, negDoubledX);
-  Value dividend = rewriter.create<arith::SubFOp>(loc, one, exp2x);
-  Value divisor = rewriter.create<arith::AddFOp>(loc, one, exp2x);
-  Value positiveRes = rewriter.create<arith::DivFOp>(loc, dividend, divisor);
+  Value negDoubledX = arith::MulFOp::create(rewriter, loc, negTwo, positiveX);
+  Value exp2x = math::ExpOp::create(rewriter, loc, negDoubledX);
+  Value dividend = arith::SubFOp::create(rewriter, loc, one, exp2x);
+  Value divisor = arith::AddFOp::create(rewriter, loc, one, exp2x);
+  Value positiveRes = arith::DivFOp::create(rewriter, loc, dividend, divisor);
 
   // Multiply result by sign(x) to retain signs from negative inputs
   rewriter.replaceOpWithNewOp<arith::MulFOp>(op, sign, positiveRes);
@@ -146,9 +146,9 @@ static LogicalResult convertTanOp(math::TanOp op, PatternRewriter &rewriter) {
   ImplicitLocOpBuilder b(op->getLoc(), rewriter);
   Value operand = op.getOperand();
   Type type = operand.getType();
-  Value sin = b.create<math::SinOp>(type, operand);
-  Value cos = b.create<math::CosOp>(type, operand);
-  Value div = b.create<arith::DivFOp>(type, sin, cos);
+  Value sin = math::SinOp::create(b, type, operand);
+  Value cos = math::CosOp::create(b, type, operand);
+  Value div = arith::DivFOp::create(b, type, sin, cos);
   rewriter.replaceOp(op, div);
   return success();
 }
@@ -161,10 +161,10 @@ static LogicalResult convertAsinhOp(math::AsinhOp op,
   Type opType = operand.getType();
 
   Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value fma = b.create<math::FmaOp>(operand, operand, one);
-  Value sqrt = b.create<math::SqrtOp>(fma);
-  Value add = b.create<arith::AddFOp>(operand, sqrt);
-  Value res = b.create<math::LogOp>(add);
+  Value fma = math::FmaOp::create(b, operand, operand, one);
+  Value sqrt = math::SqrtOp::create(b, fma);
+  Value add = arith::AddFOp::create(b, operand, sqrt);
+  Value res = math::LogOp::create(b, add);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -177,10 +177,10 @@ static LogicalResult convertAcoshOp(math::AcoshOp op,
   Type opType = operand.getType();
 
   Value negOne = createFloatConst(op->getLoc(), opType, -1.0, rewriter);
-  Value fma = b.create<math::FmaOp>(operand, operand, negOne);
-  Value sqrt = b.create<math::SqrtOp>(fma);
-  Value add = b.create<arith::AddFOp>(operand, sqrt);
-  Value res = b.create<math::LogOp>(add);
+  Value fma = math::FmaOp::create(b, operand, operand, negOne);
+  Value sqrt = math::SqrtOp::create(b, fma);
+  Value add = arith::AddFOp::create(b, operand, sqrt);
+  Value res = math::LogOp::create(b, add);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -193,13 +193,13 @@ static LogicalResult convertAtanhOp(math::AtanhOp op,
   Type opType = operand.getType();
 
   Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value add = b.create<arith::AddFOp>(operand, one);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value sub = b.create<arith::AddFOp>(neg, one);
-  Value div = b.create<arith::DivFOp>(add, sub);
-  Value log = b.create<math::LogOp>(div);
+  Value add = arith::AddFOp::create(b, operand, one);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value sub = arith::AddFOp::create(b, neg, one);
+  Value div = arith::DivFOp::create(b, add, sub);
+  Value log = math::LogOp::create(b, div);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(log, half);
+  Value res = arith::MulFOp::create(b, log, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -210,8 +210,8 @@ static LogicalResult convertFmaFOp(math::FmaOp op, PatternRewriter &rewriter) {
   Value operandB = op.getOperand(1);
   Value operandC = op.getOperand(2);
   Type type = op.getType();
-  Value mult = b.create<arith::MulFOp>(type, operandA, operandB);
-  Value add = b.create<arith::AddFOp>(type, mult, operandC);
+  Value mult = arith::MulFOp::create(b, type, operandA, operandB);
+  Value add = arith::AddFOp::create(b, type, mult, operandC);
   rewriter.replaceOp(op, add);
   return success();
 }
@@ -236,11 +236,11 @@ static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) {
   Value zero = createFloatConst(op->getLoc(), opType, 0.00, rewriter);
   Value one = createFloatConst(op->getLoc(), opType, 1.00, rewriter);
 
-  Value gtCheck = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand,
+  Value gtCheck = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT, operand,
                                           fpFixedConvert);
-  Value incrValue = b.create<arith::SelectOp>(op->getLoc(), gtCheck, one, zero);
+  Value incrValue = arith::SelectOp::create(b, op->getLoc(), gtCheck, one, zero);
 
-  Value ret = b.create<arith::AddFOp>(opType, fpFixedConvert, incrValue);
+  Value ret = arith::AddFOp::create(b, opType, fpFixedConvert, incrValue);
   rewriter.replaceOp(op, ret);
   return success();
 }
@@ -258,8 +258,8 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
 
   auto convertFPowItoPowf = [&]() -> LogicalResult {
     Value castPowerToFp =
-        rewriter.create<arith::SIToFPOp>(op.getLoc(), baseType, power);
-    Value res = rewriter.create<math::PowFOp>(op.getLoc(), baseType, base,
+        arith::SIToFPOp::create(rewriter, op.getLoc(), baseType, power);
+    Value res = math::PowFOp::create(rewriter, op.getLoc(), baseType, base,
                                               castPowerToFp);
     rewriter.replaceOp(op, res);
     return success();
@@ -281,9 +281,9 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
 
   while (absPower > 0) {
     if (absPower & 1)
-      res = b.create<arith::MulFOp>(baseType, base, res);
+      res = arith::MulFOp::create(b, baseType, base, res);
     absPower >>= 1;
-    base = b.create<arith::MulFOp>(baseType, base, base);
+    base = arith::MulFOp::create(b, baseType, base, base);
   }
 
   // Make sure not to introduce UB in case of negative power.
@@ -303,13 +303,13 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
         createFloatConst(op->getLoc(), baseType,
                          APFloat::getInf(sem, /*Negative=*/true), rewriter);
     Value zeroEqCheck =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, res, zero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, res, zero);
     Value negZeroEqCheck =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, res, negZero);
-    res = b.create<arith::DivFOp>(baseType, one, res);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, res, negZero);
+    res = arith::DivFOp::create(b, baseType, one, res);
     res =
-        b.create<arith::SelectOp>(op->getLoc(), zeroEqCheck, posInfinity, res);
-    res = b.create<arith::SelectOp>(op->getLoc(), negZeroEqCheck, negInfinity,
+        arith::SelectOp::create(b, op->getLoc(), zeroEqCheck, posInfinity, res);
+    res = arith::SelectOp::create(b, op->getLoc(), negZeroEqCheck, negInfinity,
                                     res);
   }
 
@@ -331,7 +331,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       cast<mlir::FloatType>(getElementTypeOrSelf(typeB)).getFloatSemantics();
   APFloat valueB(sem);
   auto mulf = [&](Value x, Value y) -> Value {
-    return b.create<arith::MulFOp>(x, y);
+    return arith::MulFOp::create(b, x, y);
   };
   if (matchPattern(operandB, m_ConstantFloat(&valueB))) {
     if (valueB.isZero()) {
@@ -348,19 +348,19 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
     if (valueB.isExactlyValue(-1.0)) {
       // a^(-1) -> 1 / a
       Value one = createFloatConst(op->getLoc(), typeA, 1.0, rewriter);
-      Value div = b.create<arith::DivFOp>(one, operandA);
+      Value div = arith::DivFOp::create(b, one, operandA);
       rewriter.replaceOp(op, div);
       return success();
     }
     if (valueB.isExactlyValue(0.5)) {
       // a^(1/2) -> sqrt(a)
-      Value sqrt = b.create<math::SqrtOp>(operandA);
+      Value sqrt = math::SqrtOp::create(b, operandA);
       rewriter.replaceOp(op, sqrt);
       return success();
     }
     if (valueB.isExactlyValue(-0.5)) {
       // a^(-1/2) -> 1 / sqrt(a)
-      Value rsqrt = b.create<math::RsqrtOp>(operandA);
+      Value rsqrt = math::RsqrtOp::create(b, operandA);
       rewriter.replaceOp(op, rsqrt);
       return success();
     }
@@ -373,7 +373,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       // a^(-2) -> 1 / (a * a)
       Value one =
           createFloatConst(op->getLoc(), operandA.getType(), 1.0, rewriter);
-      Value div = b.create<arith::DivFOp>(one, mulf(operandA, operandA));
+      Value div = arith::DivFOp::create(b, one, mulf(operandA, operandA));
       rewriter.replaceOp(op, div);
       return success();
     }
@@ -383,9 +383,9 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
     }
   }
 
-  Value logA = b.create<math::LogOp>(operandA);
-  Value mult = b.create<arith::MulFOp>(operandB, logA);
-  Value expResult = b.create<math::ExpOp>(mult);
+  Value logA = math::LogOp::create(b, operandA);
+  Value mult = arith::MulFOp::create(b, operandB, logA);
+  Value expResult = math::ExpOp::create(b, mult);
   rewriter.replaceOp(op, expResult);
   return success();
 }
@@ -400,8 +400,8 @@ static LogicalResult convertExp2fOp(math::Exp2Op op,
   Value operand = op.getOperand();
   Type opType = operand.getType();
   Value ln2 = createFloatConst(op->getLoc(), opType, llvm::numbers::ln2, b);
-  Value mult = b.create<arith::MulFOp>(opType, operand, ln2);
-  Value exp = b.create<math::ExpOp>(op->getLoc(), mult);
+  Value mult = arith::MulFOp::create(b, opType, operand, ln2);
+  Value exp = math::ExpOp::create(b, op->getLoc(), mult);
   rewriter.replaceOp(op, exp);
   return success();
 }
@@ -427,8 +427,8 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   Value c127 = createIntConst(loc, i32Ty, 127, b);
   Value expMask = createIntConst(loc, i32Ty, (1 << 8) - 1, b);
 
-  Value incrValue = b.create<math::CopySignOp>(half, operand);
-  Value add = b.create<arith::AddFOp>(opType, operand, incrValue);
+  Value incrValue = math::CopySignOp::create(b, half, operand);
+  Value add = arith::AddFOp::create(b, opType, operand, incrValue);
   Value fpFixedConvert = createTruncatedFPValue(add, b);
 
   // There are three cases where adding 0.5 to the value and truncating by
@@ -451,14 +451,14 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   //     i64 leading to wrong outputs.
   //
   // All three cases satisfy the property `biasedExp >= 23`.
-  Value operandBitcast = b.create<arith::BitcastOp>(i32Ty, operand);
-  Value operandExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(operandBitcast, c23), expMask);
-  Value operandBiasedExp = b.create<arith::SubIOp>(operandExp, c127);
+  Value operandBitcast = arith::BitcastOp::create(b, i32Ty, operand);
+  Value operandExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
   Value isSpecialValOrLargeVal =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::sge, operandBiasedExp, c23);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::sge, operandBiasedExp, c23);
 
-  Value result = b.create<arith::SelectOp>(isSpecialValOrLargeVal, operand,
+  Value result = arith::SelectOp::create(b, isSpecialValOrLargeVal, operand,
                                            fpFixedConvert);
   rewriter.replaceOp(op, result);
   return success();
@@ -490,20 +490,20 @@ static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
     auto mask = createIntConst(loc, operandTy, allbits >> half, rewriter);
 
     Value pred =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule, x, mask);
-    Value add = rewriter.create<arith::AddIOp>(loc, count, bits);
-    Value shift = rewriter.create<arith::ShLIOp>(loc, x, bits);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ule, x, mask);
+    Value add = arith::AddIOp::create(rewriter, loc, count, bits);
+    Value shift = arith::ShLIOp::create(rewriter, loc, x, bits);
 
-    x = rewriter.create<arith::SelectOp>(loc, pred, shift, x);
-    count = rewriter.create<arith::SelectOp>(loc, pred, add, count);
+    x = arith::SelectOp::create(rewriter, loc, pred, shift, x);
+    count = arith::SelectOp::create(rewriter, loc, pred, add, count);
   }
 
   Value zero = createIntConst(loc, operandTy, 0, rewriter);
-  Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+  Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                               operand, zero);
 
   Value bwval = createIntConst(loc, operandTy, bitwidth, rewriter);
-  Value sel = rewriter.create<arith::SelectOp>(loc, pred, bwval, count);
+  Value sel = arith::SelectOp::create(rewriter, loc, pred, bwval, count);
   rewriter.replaceOp(op, sel);
   return success();
 }
@@ -550,29 +550,29 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   Value c23Mask = createIntConst(loc, iTy, (1ull << mantissaWidth) - 1, b);
   Value expMask = createIntConst(loc, iTy, (1ull << exponentWidth) - 1, b);
 
-  Value operandBitcast = b.create<arith::BitcastOp>(iTy, operand);
-  Value round = b.create<math::RoundOp>(operand);
-  Value roundBitcast = b.create<arith::BitcastOp>(iTy, round);
+  Value operandBitcast = arith::BitcastOp::create(b, iTy, operand);
+  Value round = math::RoundOp::create(b, operand);
+  Value roundBitcast = arith::BitcastOp::create(b, iTy, round);
 
   // Get biased exponents for operand and round(operand)
-  Value operandExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(operandBitcast, c23), expMask);
-  Value operandBiasedExp = b.create<arith::SubIOp>(operandExp, c127);
-  Value roundExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(roundBitcast, c23), expMask);
-  Value roundBiasedExp = b.create<arith::SubIOp>(roundExp, c127);
+  Value operandExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
+  Value roundExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, roundBitcast, c23), expMask);
+  Value roundBiasedExp = arith::SubIOp::create(b, roundExp, c127);
 
   auto safeShiftRight = [&](Value x, Value shift) -> Value {
     // Clamp shift to valid range [0, bitwidth - 1] to avoid undefined behavior
-    Value clampedShift = b.create<arith::MaxSIOp>(shift, c0);
-    clampedShift = b.create<arith::MinSIOp>(clampedShift, c31);
-    return b.create<arith::ShRUIOp>(x, clampedShift);
+    Value clampedShift = arith::MaxSIOp::create(b, shift, c0);
+    clampedShift = arith::MinSIOp::create(b, clampedShift, c31);
+    return arith::ShRUIOp::create(b, x, clampedShift);
   };
 
   auto maskMantissa = [&](Value mantissa,
                           Value mantissaMaskRightShift) -> Value {
     Value shiftedMantissaMask = safeShiftRight(c23Mask, mantissaMaskRightShift);
-    return b.create<arith::AndIOp>(mantissa, shiftedMantissaMask);
+    return arith::AndIOp::create(b, mantissa, shiftedMantissaMask);
   };
 
   // A whole number `x`, such that `|x| != 1`, is even if the mantissa, ignoring
@@ -590,13 +590,13 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // `biasedExp > 23`, so they get treated as large numbers with no room for
   // decimals, which are always even.
   Value roundBiasedExpEq0 =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, roundBiasedExp, c0);
-  Value roundBiasedExpMinus1 = b.create<arith::SubIOp>(roundBiasedExp, c1);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, roundBiasedExp, c0);
+  Value roundBiasedExpMinus1 = arith::SubIOp::create(b, roundBiasedExp, c1);
   Value roundMaskedMantissa = maskMantissa(roundBitcast, roundBiasedExpMinus1);
-  Value roundIsNotEvenOrSpecialVal = b.create<arith::CmpIOp>(
+  Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(b,
       arith::CmpIPredicate::ne, roundMaskedMantissa, c0);
   roundIsNotEvenOrSpecialVal =
-      b.create<arith::OrIOp>(roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
+      arith::OrIOp::create(b, roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
 
   // A value `x` with `0 <= biasedExp < 23`, is halfway between two consecutive
   // integers if the bit at index `biasedExp` starting from the left in the
@@ -605,37 +605,37 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // values +-0.5 are the only halfway values that have `biasedExp == -1 < 0`,
   // so these are handled separately. In particular, if `biasedExp == -1`, the
   // value is halfway if the entire mantissa is zero.
-  Value operandBiasedExpEqNeg1 = b.create<arith::CmpIOp>(
+  Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(b,
       arith::CmpIPredicate::eq, operandBiasedExp, cNeg1);
-  Value expectedOperandMaskedMantissa = b.create<arith::SelectOp>(
+  Value expectedOperandMaskedMantissa = arith::SelectOp::create(b,
       operandBiasedExpEqNeg1, c0, safeShiftRight(c2To22, operandBiasedExp));
   Value operandMaskedMantissa = maskMantissa(operandBitcast, operandBiasedExp);
   Value operandIsHalfway =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, operandMaskedMantissa,
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, operandMaskedMantissa,
                               expectedOperandMaskedMantissa);
   // Ensure `biasedExp` is in the valid range for half values.
-  Value operandBiasedExpGeNeg1 = b.create<arith::CmpIOp>(
+  Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(b,
       arith::CmpIPredicate::sge, operandBiasedExp, cNeg1);
   Value operandBiasedExpLt23 =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::slt, operandBiasedExp, c23);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, operandBiasedExp, c23);
   operandIsHalfway =
-      b.create<arith::AndIOp>(operandIsHalfway, operandBiasedExpLt23);
+      arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpLt23);
   operandIsHalfway =
-      b.create<arith::AndIOp>(operandIsHalfway, operandBiasedExpGeNeg1);
+      arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpGeNeg1);
 
   // Adjust rounded operand with `round(operand) - sign(operand)` to correct the
   // case where `round` rounded in the opposite direction of `roundeven`.
-  Value sign = b.create<math::CopySignOp>(c1Float, operand);
-  Value roundShifted = b.create<arith::SubFOp>(round, sign);
+  Value sign = math::CopySignOp::create(b, c1Float, operand);
+  Value roundShifted = arith::SubFOp::create(b, round, sign);
   // If the rounded value is even or a special value, we default to the behavior
   // of `math.round`.
   Value needsShift =
-      b.create<arith::AndIOp>(roundIsNotEvenOrSpecialVal, operandIsHalfway);
-  Value result = b.create<arith::SelectOp>(needsShift, roundShifted, round);
+      arith::AndIOp::create(b, roundIsNotEvenOrSpecialVal, operandIsHalfway);
+  Value result = arith::SelectOp::create(b, needsShift, roundShifted, round);
   // The `x - sign` adjustment does not preserve the sign when we are adjusting
   // the value -1 to -0. So here the sign is copied again to ensure that -0.5 is
   // rounded to -0.0.
-  result = b.create<math::CopySignOp>(result, operand);
+  result = math::CopySignOp::create(b, result, operand);
   rewriter.replaceOp(op, result);
   return success();
 }
@@ -657,7 +657,7 @@ static LogicalResult convertRsqrtOp(math::RsqrtOp op,
 
   Location loc = op->getLoc();
   auto constOneFloat = createFloatConst(loc, operandTy, 1.0, rewriter);
-  auto sqrtOp = rewriter.create<math::SqrtOp>(loc, operand);
+  auto sqrtOp = math::SqrtOp::create(rewriter, loc, operand);
   rewriter.replaceOpWithNewOp<arith::DivFOp>(op, constOneFloat, sqrtOp);
   return success();
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
index a570ed5118ef0..9d6ad613fc945 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
@@ -73,7 +73,7 @@ void mlir::math::populateExtendToSupportedTypesTypeConverter(
       });
   typeConverter.addTargetMaterialization(
       [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        auto extFOp = arith::ExtFOp::create(b, loc, target, input);
         extFOp.setFastmath(arith::FastMathFlags::contract);
         return extFOp;
       });
@@ -104,7 +104,7 @@ LogicalResult ExtendToSupportedTypesRewritePattern::matchAndRewrite(
   for (auto [result, newType, origType] : llvm::zip_equal(
            results, (*legalized)->getResultTypes(), op->getResultTypes())) {
     if (newType != origType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, origType, result);
+      auto truncFOp = arith::TruncFOp::create(rewriter, loc, origType, result);
       truncFOp.setFastmath(arith::FastMathFlags::contract);
       result = truncFOp.getResult();
     }
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index a26e380232a91..2e27aba30575b 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -73,7 +73,7 @@ static Value broadcast(ImplicitLocOpBuilder &builder, Value value,
                        std::optional<VectorShape> shape) {
   assert(!isa<VectorType>(value.getType()) && "must be scalar value");
   auto type = broadcast(value.getType(), shape);
-  return shape ? builder.create<BroadcastOp>(type, value) : value;
+  return shape ? BroadcastOp::create(builder, type, value) : value;
 }
 
 //----------------------------------------------------------------------------//
@@ -131,7 +131,7 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
       auto eltType = cast<VectorType>(operand.getType()).getElementType();
       auto expandedType = VectorType::get(expandedShape, eltType);
       expandedOperands[i] =
-          builder.create<vector::ShapeCastOp>(expandedType, operand);
+          vector::ShapeCastOp::create(builder, expandedType, operand);
     }
   }
 
@@ -149,7 +149,7 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
     SmallVector<Value> extracted(expandedOperands.size());
     for (const auto &tuple : llvm::enumerate(expandedOperands))
       extracted[tuple.index()] =
-          builder.create<vector::ExtractOp>(tuple.value(), offsets);
+          vector::ExtractOp::create(builder, tuple.value(), offsets);
 
     results[i] = compute(extracted);
   }
@@ -157,15 +157,15 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
   // Stitch results together into one large vector.
   Type resultEltType = cast<VectorType>(results[0].getType()).getElementType();
   Type resultExpandedType = VectorType::get(expandedShape, resultEltType);
-  Value result = builder.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(builder,
       resultExpandedType, builder.getZeroAttr(resultExpandedType));
 
   for (int64_t i = 0; i < maxIndex; ++i)
-    result = builder.create<vector::InsertOp>(results[i], result,
+    result = vector::InsertOp::create(builder, results[i], result,
                                               delinearize(i, strides));
 
   // Reshape back to the original vector shape.
-  return builder.create<vector::ShapeCastOp>(
+  return vector::ShapeCastOp::create(builder,
       VectorType::get(inputShape, resultEltType), result);
 }
 
@@ -174,28 +174,28 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
 //----------------------------------------------------------------------------//
 
 static Value boolCst(ImplicitLocOpBuilder &builder, bool value) {
-  return builder.create<arith::ConstantOp>(builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, builder.getBoolAttr(value));
 }
 
 static Value floatCst(ImplicitLocOpBuilder &builder, float value,
                       Type elementType) {
   assert((elementType.isF16() || elementType.isF32()) &&
          "x must be f16 or f32 type.");
-  return builder.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(builder,
       builder.getFloatAttr(elementType, value));
 }
 
 static Value f32Cst(ImplicitLocOpBuilder &builder, double value) {
-  return builder.create<arith::ConstantOp>(builder.getF32FloatAttr(value));
+  return arith::ConstantOp::create(builder, builder.getF32FloatAttr(value));
 }
 
 static Value i32Cst(ImplicitLocOpBuilder &builder, int32_t value) {
-  return builder.create<arith::ConstantOp>(builder.getI32IntegerAttr(value));
+  return arith::ConstantOp::create(builder, builder.getI32IntegerAttr(value));
 }
 
 static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
   Value i32Value = i32Cst(builder, static_cast<int32_t>(bits));
-  return builder.create<arith::BitcastOp>(builder.getF32Type(), i32Value);
+  return arith::BitcastOp::create(builder, builder.getF32Type(), i32Value);
 }
 
 //----------------------------------------------------------------------------//
@@ -204,15 +204,15 @@ static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
 
 // Return the minimum of the two values or NaN if value is NaN
 static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return builder.create<arith::SelectOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT, value, bound),
+  return arith::SelectOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT, value, bound),
       value, bound);
 }
 
 // Return the maximum of the two values or NaN if value is NaN
 static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return builder.create<arith::SelectOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::UGT, value, bound),
+  return arith::SelectOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::UGT, value, bound),
       value, bound);
 }
 
@@ -242,24 +242,24 @@ static std::pair<Value, Value> frexp(ImplicitLocOpBuilder &builder, Value arg,
   Value cstInvMantMask = f32FromBits(builder, ~0x7f800000u);
 
   // Bitcast to i32 for bitwise operations.
-  Value i32Half = builder.create<arith::BitcastOp>(i32, cstHalf);
-  Value i32InvMantMask = builder.create<arith::BitcastOp>(i32, cstInvMantMask);
-  Value i32Arg = builder.create<arith::BitcastOp>(i32Vec, arg);
+  Value i32Half = arith::BitcastOp::create(builder, i32, cstHalf);
+  Value i32InvMantMask = arith::BitcastOp::create(builder, i32, cstInvMantMask);
+  Value i32Arg = arith::BitcastOp::create(builder, i32Vec, arg);
 
   // Compute normalized fraction.
-  Value tmp0 = builder.create<arith::AndIOp>(i32Arg, bcast(i32InvMantMask));
-  Value tmp1 = builder.create<arith::OrIOp>(tmp0, bcast(i32Half));
-  Value normalizedFraction = builder.create<arith::BitcastOp>(f32Vec, tmp1);
+  Value tmp0 = arith::AndIOp::create(builder, i32Arg, bcast(i32InvMantMask));
+  Value tmp1 = arith::OrIOp::create(builder, tmp0, bcast(i32Half));
+  Value normalizedFraction = arith::BitcastOp::create(builder, f32Vec, tmp1);
 
   // Compute exponent.
-  Value arg0 = isPositive ? arg : builder.create<math::AbsFOp>(arg);
-  Value biasedExponentBits = builder.create<arith::ShRUIOp>(
-      builder.create<arith::BitcastOp>(i32Vec, arg0),
+  Value arg0 = isPositive ? arg : math::AbsFOp::create(builder, arg);
+  Value biasedExponentBits = arith::ShRUIOp::create(builder,
+      arith::BitcastOp::create(builder, i32Vec, arg0),
       bcast(i32Cst(builder, 23)));
   Value biasedExponent =
-      builder.create<arith::SIToFPOp>(f32Vec, biasedExponentBits);
+      arith::SIToFPOp::create(builder, f32Vec, biasedExponentBits);
   Value exponent =
-      builder.create<arith::SubFOp>(biasedExponent, bcast(cst126f));
+      arith::SubFOp::create(builder, biasedExponent, bcast(cst126f));
 
   return {normalizedFraction, exponent};
 }
@@ -279,10 +279,10 @@ static Value exp2I32(ImplicitLocOpBuilder &builder, Value arg) {
   // Set the exponent bias to zero.
   auto bias = bcast(i32Cst(builder, 127));
 
-  Value biasedArg = builder.create<arith::AddIOp>(arg, bias);
+  Value biasedArg = arith::AddIOp::create(builder, arg, bias);
   Value exp2ValueInt =
-      builder.create<arith::ShLIOp>(biasedArg, exponetBitLocation);
-  Value exp2ValueF32 = builder.create<arith::BitcastOp>(f32Vec, exp2ValueInt);
+      arith::ShLIOp::create(builder, biasedArg, exponetBitLocation);
+  Value exp2ValueF32 = arith::BitcastOp::create(builder, f32Vec, exp2ValueInt);
 
   return exp2ValueF32;
 }
@@ -301,10 +301,10 @@ Value makePolynomialCalculation(ImplicitLocOpBuilder &builder,
   if (coeffs.size() == 1)
     return coeffs[0];
 
-  Value res = builder.create<math::FmaOp>(x, coeffs[coeffs.size() - 1],
+  Value res = math::FmaOp::create(builder, x, coeffs[coeffs.size() - 1],
                                           coeffs[coeffs.size() - 2]);
   for (auto i = ptrdiff_t(coeffs.size()) - 3; i >= 0; --i) {
-    res = builder.create<math::FmaOp>(x, res, coeffs[i]);
+    res = math::FmaOp::create(builder, x, res, coeffs[i]);
   }
   return res;
 }
@@ -344,9 +344,9 @@ LogicalResult insertCasts(Operation *op, PatternRewriter &rewriter) {
   Location loc = op->getLoc();
   SmallVector<Value> operands;
   for (auto operand : op->getOperands())
-    operands.push_back(rewriter.create<arith::ExtFOp>(loc, newType, operand));
+    operands.push_back(arith::ExtFOp::create(rewriter, loc, newType, operand));
   auto result =
-      rewriter.create<T>(loc, TypeRange{newType}, operands, op->getAttrs());
+      T::create(rewriter, loc, TypeRange{newType}, operands, op->getAttrs());
   rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, origType, result);
   return success();
 }
@@ -394,18 +394,18 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
   std::optional<VectorShape> shape = vectorShape(op.getOperand());
 
   ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
-  Value abs = builder.create<math::AbsFOp>(operand);
+  Value abs = math::AbsFOp::create(builder, operand);
 
   auto one = broadcast(builder, f32Cst(builder, 1.0), shape);
 
   // When 0.66 < x <= 2.41 we do (x-1) / (x+1):
   auto twoThirds = broadcast(builder, f32Cst(builder, 0.66), shape);
   Value cmp2 =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, abs, twoThirds);
-  Value addone = builder.create<arith::AddFOp>(abs, one);
-  Value subone = builder.create<arith::SubFOp>(abs, one);
-  Value xnum = builder.create<arith::SelectOp>(cmp2, subone, abs);
-  Value xden = builder.create<arith::SelectOp>(cmp2, addone, one);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, abs, twoThirds);
+  Value addone = arith::AddFOp::create(builder, abs, one);
+  Value subone = arith::SubFOp::create(builder, abs, one);
+  Value xnum = arith::SelectOp::create(builder, cmp2, subone, abs);
+  Value xden = arith::SelectOp::create(builder, cmp2, addone, one);
 
   auto bcast = [&](Value value) -> Value {
     return broadcast(builder, value, shape);
@@ -414,12 +414,12 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
   // Break into the <= 0.66 or > 2.41 we do x or 1/x:
   auto tan3pio8 = bcast(f32Cst(builder, 2.41421356237309504880));
   Value cmp1 =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, abs, tan3pio8);
-  xnum = builder.create<arith::SelectOp>(cmp1, one, xnum);
-  xden = builder.create<arith::SelectOp>(cmp1, abs, xden);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, abs, tan3pio8);
+  xnum = arith::SelectOp::create(builder, cmp1, one, xnum);
+  xden = arith::SelectOp::create(builder, cmp1, abs, xden);
 
-  Value x = builder.create<arith::DivFOp>(xnum, xden);
-  Value xx = builder.create<arith::MulFOp>(x, x);
+  Value x = arith::DivFOp::create(builder, xnum, xden);
+  Value xx = arith::MulFOp::create(builder, x, x);
 
   // Perform the Taylor series approximation for atan over the range
   // [0.0, 0.66].
@@ -436,31 +436,31 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
 
   // Apply the polynomial approximation for the numerator:
   Value n = p0;
-  n = builder.create<math::FmaOp>(xx, n, p1);
-  n = builder.create<math::FmaOp>(xx, n, p2);
-  n = builder.create<math::FmaOp>(xx, n, p3);
-  n = builder.create<math::FmaOp>(xx, n, p4);
-  n = builder.create<arith::MulFOp>(n, xx);
+  n = math::FmaOp::create(builder, xx, n, p1);
+  n = math::FmaOp::create(builder, xx, n, p2);
+  n = math::FmaOp::create(builder, xx, n, p3);
+  n = math::FmaOp::create(builder, xx, n, p4);
+  n = arith::MulFOp::create(builder, n, xx);
 
   // Apply the polynomial approximation for the denominator:
   Value d = q0;
-  d = builder.create<math::FmaOp>(xx, d, q1);
-  d = builder.create<math::FmaOp>(xx, d, q2);
-  d = builder.create<math::FmaOp>(xx, d, q3);
-  d = builder.create<math::FmaOp>(xx, d, q4);
+  d = math::FmaOp::create(builder, xx, d, q1);
+  d = math::FmaOp::create(builder, xx, d, q2);
+  d = math::FmaOp::create(builder, xx, d, q3);
+  d = math::FmaOp::create(builder, xx, d, q4);
 
   // Compute approximation of theta:
-  Value ans0 = builder.create<arith::DivFOp>(n, d);
-  ans0 = builder.create<math::FmaOp>(ans0, x, x);
+  Value ans0 = arith::DivFOp::create(builder, n, d);
+  ans0 = math::FmaOp::create(builder, ans0, x, x);
 
   // Correct for the input mapping's angles:
   Value mpi4 = bcast(f32Cst(builder, llvm::numbers::pi / 4));
-  Value ans2 = builder.create<arith::AddFOp>(mpi4, ans0);
-  Value ans = builder.create<arith::SelectOp>(cmp2, ans2, ans0);
+  Value ans2 = arith::AddFOp::create(builder, mpi4, ans0);
+  Value ans = arith::SelectOp::create(builder, cmp2, ans2, ans0);
 
   Value mpi2 = bcast(f32Cst(builder, llvm::numbers::pi / 2));
-  Value ans1 = builder.create<arith::SubFOp>(mpi2, ans0);
-  ans = builder.create<arith::SelectOp>(cmp1, ans1, ans);
+  Value ans1 = arith::SubFOp::create(builder, mpi2, ans0);
+  ans = arith::SelectOp::create(builder, cmp1, ans1, ans);
 
   // Correct for signing of the input.
   rewriter.replaceOpWithNewOp<math::CopySignOp>(op, ans, operand);
@@ -493,44 +493,44 @@ Atan2Approximation::matchAndRewrite(math::Atan2Op op,
   std::optional<VectorShape> shape = vectorShape(op.getResult());
 
   // Compute atan in the valid range.
-  auto div = builder.create<arith::DivFOp>(y, x);
-  auto atan = builder.create<math::AtanOp>(div);
+  auto div = arith::DivFOp::create(builder, y, x);
+  auto atan = math::AtanOp::create(builder, div);
 
   // Determine what the atan would be for a 180 degree rotation.
   auto zero = broadcast(builder, f32Cst(builder, 0.0f), shape);
   auto pi = broadcast(builder, f32Cst(builder, 3.14159265359f), shape);
-  auto addPi = builder.create<arith::AddFOp>(atan, pi);
-  auto subPi = builder.create<arith::SubFOp>(atan, pi);
+  auto addPi = arith::AddFOp::create(builder, atan, pi);
+  auto subPi = arith::SubFOp::create(builder, atan, pi);
   auto atanGt =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, atan, zero);
-  auto flippedAtan = builder.create<arith::SelectOp>(atanGt, subPi, addPi);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, atan, zero);
+  auto flippedAtan = arith::SelectOp::create(builder, atanGt, subPi, addPi);
 
   // Determine whether to directly use atan or use the 180 degree flip
-  auto xGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, x, zero);
-  Value result = builder.create<arith::SelectOp>(xGt, atan, flippedAtan);
+  auto xGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, x, zero);
+  Value result = arith::SelectOp::create(builder, xGt, atan, flippedAtan);
 
   // Handle x = 0, y > 0
   Value xZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, x, zero);
-  Value yGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, y, zero);
-  Value isHalfPi = builder.create<arith::AndIOp>(xZero, yGt);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, x, zero);
+  Value yGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, y, zero);
+  Value isHalfPi = arith::AndIOp::create(builder, xZero, yGt);
   auto halfPi = broadcast(builder, f32Cst(builder, 1.57079632679f), shape);
-  result = builder.create<arith::SelectOp>(isHalfPi, halfPi, result);
+  result = arith::SelectOp::create(builder, isHalfPi, halfPi, result);
 
   // Handle x = 0, y < 0
-  Value yLt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, y, zero);
-  Value isNegativeHalfPiPi = builder.create<arith::AndIOp>(xZero, yLt);
+  Value yLt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, y, zero);
+  Value isNegativeHalfPiPi = arith::AndIOp::create(builder, xZero, yLt);
   auto negativeHalfPiPi =
       broadcast(builder, f32Cst(builder, -1.57079632679f), shape);
-  result = builder.create<arith::SelectOp>(isNegativeHalfPiPi, negativeHalfPiPi,
+  result = arith::SelectOp::create(builder, isNegativeHalfPiPi, negativeHalfPiPi,
                                            result);
 
   // Handle x = 0, y = 0;
   Value yZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, y, zero);
-  Value isNan = builder.create<arith::AndIOp>(xZero, yZero);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, y, zero);
+  Value isNan = arith::AndIOp::create(builder, xZero, yZero);
   Value cstNan = broadcast(builder, f32FromBits(builder, 0x7fc00000), shape);
-  result = builder.create<arith::SelectOp>(isNan, cstNan, result);
+  result = arith::SelectOp::create(builder, isNan, cstNan, result);
 
   rewriter.replaceOp(op, result);
   return success();
@@ -570,8 +570,8 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
 
   // Mask for tiny values that are approximated with `operand`.
   Value tiny = bcast(f32Cst(builder, 0.0004f));
-  Value tinyMask = builder.create<arith::CmpFOp>(
-      arith::CmpFPredicate::OLT, builder.create<math::AbsFOp>(op.getOperand()),
+  Value tinyMask = arith::CmpFOp::create(builder,
+      arith::CmpFPredicate::OLT, math::AbsFOp::create(builder, op.getOperand()),
       tiny);
 
   // The monomial coefficients of the numerator polynomial (odd).
@@ -590,25 +590,25 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
   Value beta6 = bcast(f32Cst(builder, 1.19825839466702e-06f));
 
   // Since the polynomials are odd/even, we need x^2.
-  Value x2 = builder.create<arith::MulFOp>(x, x);
+  Value x2 = arith::MulFOp::create(builder, x, x);
 
   // Evaluate the numerator polynomial p.
-  Value p = builder.create<math::FmaOp>(x2, alpha13, alpha11);
-  p = builder.create<math::FmaOp>(x2, p, alpha9);
-  p = builder.create<math::FmaOp>(x2, p, alpha7);
-  p = builder.create<math::FmaOp>(x2, p, alpha5);
-  p = builder.create<math::FmaOp>(x2, p, alpha3);
-  p = builder.create<math::FmaOp>(x2, p, alpha1);
-  p = builder.create<arith::MulFOp>(x, p);
+  Value p = math::FmaOp::create(builder, x2, alpha13, alpha11);
+  p = math::FmaOp::create(builder, x2, p, alpha9);
+  p = math::FmaOp::create(builder, x2, p, alpha7);
+  p = math::FmaOp::create(builder, x2, p, alpha5);
+  p = math::FmaOp::create(builder, x2, p, alpha3);
+  p = math::FmaOp::create(builder, x2, p, alpha1);
+  p = arith::MulFOp::create(builder, x, p);
 
   // Evaluate the denominator polynomial q.
-  Value q = builder.create<math::FmaOp>(x2, beta6, beta4);
-  q = builder.create<math::FmaOp>(x2, q, beta2);
-  q = builder.create<math::FmaOp>(x2, q, beta0);
+  Value q = math::FmaOp::create(builder, x2, beta6, beta4);
+  q = math::FmaOp::create(builder, x2, q, beta2);
+  q = math::FmaOp::create(builder, x2, q, beta0);
 
   // Divide the numerator by the denominator.
-  Value res = builder.create<arith::SelectOp>(
-      tinyMask, x, builder.create<arith::DivFOp>(p, q));
+  Value res = arith::SelectOp::create(builder,
+      tinyMask, x, arith::DivFOp::create(builder, p, q));
 
   rewriter.replaceOp(op, res);
 
@@ -691,57 +691,57 @@ LogApproximationBase<Op>::logMatchAndRewrite(Op op, PatternRewriter &rewriter,
   //     e -= 1;
   //     x = x + x - 1.0;
   //   } else { x = x - 1.0; }
-  Value mask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x,
+  Value mask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x,
                                              cstCephesSQRTHF);
-  Value tmp = builder.create<arith::SelectOp>(mask, x, cstZero);
+  Value tmp = arith::SelectOp::create(builder, mask, x, cstZero);
 
-  x = builder.create<arith::SubFOp>(x, cstOne);
-  e = builder.create<arith::SubFOp>(
-      e, builder.create<arith::SelectOp>(mask, cstOne, cstZero));
-  x = builder.create<arith::AddFOp>(x, tmp);
+  x = arith::SubFOp::create(builder, x, cstOne);
+  e = arith::SubFOp::create(builder,
+      e, arith::SelectOp::create(builder, mask, cstOne, cstZero));
+  x = arith::AddFOp::create(builder, x, tmp);
 
-  Value x2 = builder.create<arith::MulFOp>(x, x);
-  Value x3 = builder.create<arith::MulFOp>(x2, x);
+  Value x2 = arith::MulFOp::create(builder, x, x);
+  Value x3 = arith::MulFOp::create(builder, x2, x);
 
   // Evaluate the polynomial approximant of degree 8 in three parts.
   Value y0, y1, y2;
-  y0 = builder.create<math::FmaOp>(cstCephesLogP0, x, cstCephesLogP1);
-  y1 = builder.create<math::FmaOp>(cstCephesLogP3, x, cstCephesLogP4);
-  y2 = builder.create<math::FmaOp>(cstCephesLogP6, x, cstCephesLogP7);
-  y0 = builder.create<math::FmaOp>(y0, x, cstCephesLogP2);
-  y1 = builder.create<math::FmaOp>(y1, x, cstCephesLogP5);
-  y2 = builder.create<math::FmaOp>(y2, x, cstCephesLogP8);
-  y0 = builder.create<math::FmaOp>(y0, x3, y1);
-  y0 = builder.create<math::FmaOp>(y0, x3, y2);
-  y0 = builder.create<arith::MulFOp>(y0, x3);
-
-  y0 = builder.create<math::FmaOp>(cstNegHalf, x2, y0);
-  x = builder.create<arith::AddFOp>(x, y0);
+  y0 = math::FmaOp::create(builder, cstCephesLogP0, x, cstCephesLogP1);
+  y1 = math::FmaOp::create(builder, cstCephesLogP3, x, cstCephesLogP4);
+  y2 = math::FmaOp::create(builder, cstCephesLogP6, x, cstCephesLogP7);
+  y0 = math::FmaOp::create(builder, y0, x, cstCephesLogP2);
+  y1 = math::FmaOp::create(builder, y1, x, cstCephesLogP5);
+  y2 = math::FmaOp::create(builder, y2, x, cstCephesLogP8);
+  y0 = math::FmaOp::create(builder, y0, x3, y1);
+  y0 = math::FmaOp::create(builder, y0, x3, y2);
+  y0 = arith::MulFOp::create(builder, y0, x3);
+
+  y0 = math::FmaOp::create(builder, cstNegHalf, x2, y0);
+  x = arith::AddFOp::create(builder, x, y0);
 
   if (base2) {
     Value cstLog2e = bcast(f32Cst(builder, static_cast<float>(LOG2E_VALUE)));
-    x = builder.create<math::FmaOp>(x, cstLog2e, e);
+    x = math::FmaOp::create(builder, x, cstLog2e, e);
   } else {
     Value cstLn2 = bcast(f32Cst(builder, static_cast<float>(LN2_VALUE)));
-    x = builder.create<math::FmaOp>(e, cstLn2, x);
+    x = math::FmaOp::create(builder, e, cstLn2, x);
   }
 
-  Value invalidMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT,
+  Value invalidMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT,
                                                     op.getOperand(), cstZero);
-  Value zeroMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value zeroMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                  op.getOperand(), cstZero);
-  Value posInfMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value posInfMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                    op.getOperand(), cstPosInf);
 
   // Filter out invalid values:
   //  • x == 0     -> -INF
   //  • x < 0      ->  NAN
   //  • x == +INF  -> +INF
-  Value aproximation = builder.create<arith::SelectOp>(
+  Value aproximation = arith::SelectOp::create(builder,
       zeroMask, cstMinusInf,
-      builder.create<arith::SelectOp>(
+      arith::SelectOp::create(builder,
           invalidMask, cstNan,
-          builder.create<arith::SelectOp>(posInfMask, cstPosInf, x)));
+          arith::SelectOp::create(builder, posInfMask, cstPosInf, x)));
 
   rewriter.replaceOp(op, aproximation);
 
@@ -806,17 +806,17 @@ Log1pApproximation::matchAndRewrite(math::Log1pOp op,
   //             "logLarge" below.
   Value cstOne = bcast(f32Cst(builder, 1.0f));
   Value x = op.getOperand();
-  Value u = builder.create<arith::AddFOp>(x, cstOne);
+  Value u = arith::AddFOp::create(builder, x, cstOne);
   Value uSmall =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, cstOne);
-  Value logU = builder.create<math::LogOp>(u);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, cstOne);
+  Value logU = math::LogOp::create(builder, u);
   Value uInf =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, logU);
-  Value logLarge = builder.create<arith::MulFOp>(
-      x, builder.create<arith::DivFOp>(
-             logU, builder.create<arith::SubFOp>(u, cstOne)));
-  Value approximation = builder.create<arith::SelectOp>(
-      builder.create<arith::OrIOp>(uSmall, uInf), x, logLarge);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, logU);
+  Value logLarge = arith::MulFOp::create(builder,
+      x, arith::DivFOp::create(builder,
+             logU, arith::SubFOp::create(builder, u, cstOne)));
+  Value approximation = arith::SelectOp::create(builder,
+      arith::OrIOp::create(builder, uSmall, uInf), x, logLarge);
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -854,27 +854,27 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   };
 
   auto fma = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
   auto sub = [&](Value a, Value b) -> Value {
-    return builder.create<arith::SubFOp>(a, b);
+    return arith::SubFOp::create(builder, a, b);
   };
 
-  auto abs = [&](Value a) -> Value { return builder.create<math::AbsFOp>(a); };
+  auto abs = [&](Value a) -> Value { return math::AbsFOp::create(builder, a); };
 
-  auto sqrt = [&](Value a) -> Value { return builder.create<math::SqrtOp>(a); };
+  auto sqrt = [&](Value a) -> Value { return math::SqrtOp::create(builder, a); };
 
   auto scopy = [&](Value a, Value b) -> Value {
-    return builder.create<math::CopySignOp>(a, b);
+    return math::CopySignOp::create(builder, a, b);
   };
 
   auto sel = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<arith::SelectOp>(a, b, c);
+    return arith::SelectOp::create(builder, a, b, c);
   };
 
   Value abso = abs(operand);
@@ -882,7 +882,7 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   Value opp = sqrt(sub(bcast(floatCst(builder, 1.0, elementType)), aa));
 
   Value gt =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, aa,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, aa,
                                     bcast(floatCst(builder, 0.5, elementType)));
 
   Value x = sel(gt, opp, abso);
@@ -949,51 +949,51 @@ AcosPolynomialApproximation::matchAndRewrite(math::AcosOp op,
   };
 
   auto fma = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
-  Value negOperand = builder.create<arith::NegFOp>(operand);
+  Value negOperand = arith::NegFOp::create(builder, operand);
   Value zero = bcast(floatCst(builder, 0.0, elementType));
   Value half = bcast(floatCst(builder, 0.5, elementType));
   Value negOne = bcast(floatCst(builder, -1.0, elementType));
   Value selR =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand, zero);
-  Value r = builder.create<arith::SelectOp>(selR, negOperand, operand);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, operand, zero);
+  Value r = arith::SelectOp::create(builder, selR, negOperand, operand);
   Value chkConst = bcast(floatCst(builder, -0.5625, elementType));
   Value firstPred =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, r, chkConst);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, r, chkConst);
 
   Value trueVal =
       fma(bcast(floatCst(builder, 9.3282184640716537e-1, elementType)),
           bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
-          builder.create<math::AsinOp>(r));
+          math::AsinOp::create(builder, r));
 
-  Value falseVal = builder.create<math::SqrtOp>(fma(half, r, half));
-  falseVal = builder.create<math::AsinOp>(falseVal);
+  Value falseVal = math::SqrtOp::create(builder, fma(half, r, half));
+  falseVal = math::AsinOp::create(builder, falseVal);
   falseVal = mul(bcast(floatCst(builder, 2.0, elementType)), falseVal);
 
-  r = builder.create<arith::SelectOp>(firstPred, trueVal, falseVal);
+  r = arith::SelectOp::create(builder, firstPred, trueVal, falseVal);
 
   // Check whether the operand lies in between [-1.0, 0.0).
   Value greaterThanNegOne =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, operand, negOne);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGE, operand, negOne);
 
   Value lessThanZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
 
   Value betweenNegOneZero =
-      builder.create<arith::AndIOp>(greaterThanNegOne, lessThanZero);
+      arith::AndIOp::create(builder, greaterThanNegOne, lessThanZero);
 
   trueVal = fma(bcast(floatCst(builder, 1.8656436928143307e+0, elementType)),
                 bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
-                builder.create<arith::NegFOp>(r));
+                arith::NegFOp::create(builder, r));
 
   Value finalVal =
-      builder.create<arith::SelectOp>(betweenNegOneZero, trueVal, r);
+      arith::SelectOp::create(builder, betweenNegOneZero, trueVal, r);
 
   rewriter.replaceOp(op, finalVal);
   return success();
@@ -1076,9 +1076,9 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
   bounds[2] = bcast(floatCst(builder, 3.75f, elementType));
 
   Value isNegativeArg =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
-  Value negArg = builder.create<arith::NegFOp>(operand);
-  Value x = builder.create<arith::SelectOp>(isNegativeArg, negArg, operand);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
+  Value negArg = arith::NegFOp::create(builder, operand);
+  Value x = arith::SelectOp::create(builder, isNegativeArg, negArg, operand);
 
   Value offset = offsets[0];
   Value p[polyDegree + 1];
@@ -1092,30 +1092,30 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
   Value isLessThanBound[intervalsCount];
   for (int j = 0; j < intervalsCount - 1; ++j) {
     isLessThanBound[j] =
-        builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, bounds[j]);
+        arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, bounds[j]);
     for (int i = 0; i <= polyDegree; ++i) {
-      p[i] = builder.create<arith::SelectOp>(isLessThanBound[j], p[i],
+      p[i] = arith::SelectOp::create(builder, isLessThanBound[j], p[i],
                                              pp[j + 1][i]);
-      q[i] = builder.create<arith::SelectOp>(isLessThanBound[j], q[i],
+      q[i] = arith::SelectOp::create(builder, isLessThanBound[j], q[i],
                                              qq[j + 1][i]);
     }
-    offset = builder.create<arith::SelectOp>(isLessThanBound[j], offset,
+    offset = arith::SelectOp::create(builder, isLessThanBound[j], offset,
                                              offsets[j + 1]);
   }
-  isLessThanBound[intervalsCount - 1] = builder.create<arith::CmpFOp>(
+  isLessThanBound[intervalsCount - 1] = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::ULT, x, bounds[intervalsCount - 1]);
 
   Value pPoly = makePolynomialCalculation(builder, p, x);
   Value qPoly = makePolynomialCalculation(builder, q, x);
-  Value rationalPoly = builder.create<arith::DivFOp>(pPoly, qPoly);
-  Value formula = builder.create<arith::AddFOp>(offset, rationalPoly);
-  formula = builder.create<arith::SelectOp>(isLessThanBound[intervalsCount - 1],
+  Value rationalPoly = arith::DivFOp::create(builder, pPoly, qPoly);
+  Value formula = arith::AddFOp::create(builder, offset, rationalPoly);
+  formula = arith::SelectOp::create(builder, isLessThanBound[intervalsCount - 1],
                                             formula, one);
 
   // erf is odd function: erf(x) = -erf(-x).
-  Value negFormula = builder.create<arith::NegFOp>(formula);
+  Value negFormula = arith::NegFOp::create(builder, formula);
   Value res =
-      builder.create<arith::SelectOp>(isNegativeArg, negFormula, formula);
+      arith::SelectOp::create(builder, isNegativeArg, negFormula, formula);
 
   rewriter.replaceOp(op, res);
 
@@ -1156,65 +1156,65 @@ ErfcPolynomialApproximation::matchAndRewrite(math::ErfcOp op,
   Value posInf = bcast(floatCst(builder, INFINITY, et));
   Value clampVal = bcast(floatCst(builder, 10.0546875f, et));
 
-  Value a = builder.create<math::AbsFOp>(x);
-  Value p = builder.create<arith::AddFOp>(a, pos2);
-  Value r = builder.create<arith::DivFOp>(one, p);
-  Value q = builder.create<math::FmaOp>(neg4, r, one);
-  Value t = builder.create<math::FmaOp>(builder.create<arith::AddFOp>(q, one),
+  Value a = math::AbsFOp::create(builder, x);
+  Value p = arith::AddFOp::create(builder, a, pos2);
+  Value r = arith::DivFOp::create(builder, one, p);
+  Value q = math::FmaOp::create(builder, neg4, r, one);
+  Value t = math::FmaOp::create(builder, arith::AddFOp::create(builder, q, one),
                                         neg2, a);
-  Value e = builder.create<math::FmaOp>(builder.create<arith::NegFOp>(a), q, t);
-  q = builder.create<math::FmaOp>(r, e, q);
+  Value e = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), q, t);
+  q = math::FmaOp::create(builder, r, e, q);
 
   p = bcast(floatCst(builder, -0x1.a4a000p-12f, et));        // -4.01139259e-4
   Value c1 = bcast(floatCst(builder, -0x1.42a260p-10f, et)); // -1.23075210e-3
-  p = builder.create<math::FmaOp>(p, q, c1);
+  p = math::FmaOp::create(builder, p, q, c1);
   Value c2 = bcast(floatCst(builder, 0x1.585714p-10f, et)); //  1.31355342e-3
-  p = builder.create<math::FmaOp>(p, q, c2);
+  p = math::FmaOp::create(builder, p, q, c2);
   Value c3 = bcast(floatCst(builder, 0x1.1adcc4p-07f, et)); // 8.63227434e-3
-  p = builder.create<math::FmaOp>(p, q, c3);
+  p = math::FmaOp::create(builder, p, q, c3);
   Value c4 = bcast(floatCst(builder, -0x1.081b82p-07f, et)); // -8.05991981e-3
-  p = builder.create<math::FmaOp>(p, q, c4);
+  p = math::FmaOp::create(builder, p, q, c4);
   Value c5 = bcast(floatCst(builder, -0x1.bc0b6ap-05f, et)); // -5.42046614e-2
-  p = builder.create<math::FmaOp>(p, q, c5);
+  p = math::FmaOp::create(builder, p, q, c5);
   Value c6 = bcast(floatCst(builder, 0x1.4ffc46p-03f, et)); //  1.64055392e-1
-  p = builder.create<math::FmaOp>(p, q, c6);
+  p = math::FmaOp::create(builder, p, q, c6);
   Value c7 = bcast(floatCst(builder, -0x1.540840p-03f, et)); // -1.66031361e-1
-  p = builder.create<math::FmaOp>(p, q, c7);
+  p = math::FmaOp::create(builder, p, q, c7);
   Value c8 = bcast(floatCst(builder, -0x1.7bf616p-04f, et)); // -9.27639827e-2
-  p = builder.create<math::FmaOp>(p, q, c8);
+  p = math::FmaOp::create(builder, p, q, c8);
   Value c9 = bcast(floatCst(builder, 0x1.1ba03ap-02f, et)); // 2.76978403e-1
-  p = builder.create<math::FmaOp>(p, q, c9);
-
-  Value d = builder.create<math::FmaOp>(pos2, a, one);
-  r = builder.create<arith::DivFOp>(one, d);
-  q = builder.create<math::FmaOp>(p, r, r);
-  Value negfa = builder.create<arith::NegFOp>(a);
-  Value fmaqah = builder.create<math::FmaOp>(q, negfa, onehalf);
-  Value psubq = builder.create<arith::SubFOp>(p, q);
-  e = builder.create<math::FmaOp>(fmaqah, pos2, psubq);
-  r = builder.create<math::FmaOp>(e, r, q);
-
-  Value s = builder.create<arith::MulFOp>(a, a);
-  e = builder.create<math::ExpOp>(builder.create<arith::NegFOp>(s));
-
-  t = builder.create<math::FmaOp>(builder.create<arith::NegFOp>(a), a, s);
-  r = builder.create<math::FmaOp>(
+  p = math::FmaOp::create(builder, p, q, c9);
+
+  Value d = math::FmaOp::create(builder, pos2, a, one);
+  r = arith::DivFOp::create(builder, one, d);
+  q = math::FmaOp::create(builder, p, r, r);
+  Value negfa = arith::NegFOp::create(builder, a);
+  Value fmaqah = math::FmaOp::create(builder, q, negfa, onehalf);
+  Value psubq = arith::SubFOp::create(builder, p, q);
+  e = math::FmaOp::create(builder, fmaqah, pos2, psubq);
+  r = math::FmaOp::create(builder, e, r, q);
+
+  Value s = arith::MulFOp::create(builder, a, a);
+  e = math::ExpOp::create(builder, arith::NegFOp::create(builder, s));
+
+  t = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), a, s);
+  r = math::FmaOp::create(builder,
       r, e,
-      builder.create<arith::MulFOp>(builder.create<arith::MulFOp>(r, e), t));
+      arith::MulFOp::create(builder, arith::MulFOp::create(builder, r, e), t));
 
-  Value isNotLessThanInf = builder.create<arith::XOrIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, a, posInf),
+  Value isNotLessThanInf = arith::XOrIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, a, posInf),
       trueValue);
-  r = builder.create<arith::SelectOp>(isNotLessThanInf,
-                                      builder.create<arith::AddFOp>(x, x), r);
+  r = arith::SelectOp::create(builder, isNotLessThanInf,
+                                      arith::AddFOp::create(builder, x, x), r);
   Value isGreaterThanClamp =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, a, clampVal);
-  r = builder.create<arith::SelectOp>(isGreaterThanClamp, zero, r);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, a, clampVal);
+  r = arith::SelectOp::create(builder, isGreaterThanClamp, zero, r);
 
   Value isNegative =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, zero);
-  r = builder.create<arith::SelectOp>(
-      isNegative, builder.create<arith::SubFOp>(pos2, r), r);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, zero);
+  r = arith::SelectOp::create(builder,
+      isNegative, arith::SubFOp::create(builder, pos2, r), r);
 
   rewriter.replaceOp(op, r);
   return success();
@@ -1236,8 +1236,8 @@ Value clampWithNormals(ImplicitLocOpBuilder &builder,
   };
 
   auto selectCmp = [&builder](auto pred, Value value, Value bound) {
-    return builder.create<arith::SelectOp>(
-        builder.create<arith::CmpFOp>(pred, value, bound), value, bound);
+    return arith::SelectOp::create(builder,
+        arith::CmpFOp::create(builder, pred, value, bound), value, bound);
   };
 
   // Note: prefer UGE/ULE vs. UGT/ULT, since they generate vmaxps/vminps vs.
@@ -1269,17 +1269,17 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
   ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
 
   auto add = [&](Value a, Value b) -> Value {
-    return builder.create<arith::AddFOp>(a, b);
+    return arith::AddFOp::create(builder, a, b);
   };
   auto bcast = [&](Value value) -> Value {
     return broadcast(builder, value, shape);
   };
-  auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); };
+  auto floor = [&](Value a) { return math::FloorOp::create(builder, a); };
   auto fmla = [&](Value a, Value b, Value c) {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
   // Polynomial approximation from Cephes.
@@ -1383,7 +1383,7 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
 
   // Convert n' to an i32.  This is safe because we clamped it above.
   auto i32Vec = broadcast(builder.getI32Type(), shape);
-  Value nI32 = builder.create<arith::FPToSIOp>(i32Vec, n);
+  Value nI32 = arith::FPToSIOp::create(builder, i32Vec, n);
 
   // Creates the value 2^n' if -126 <= n' <= 127 and 0 if n' = -127.
   Value pow2 = exp2I32(builder, nI32);
@@ -1431,26 +1431,26 @@ ExpM1Approximation::matchAndRewrite(math::ExpM1Op op,
   Value cstOne = bcast(f32Cst(builder, 1.0f));
   Value cstNegOne = bcast(f32Cst(builder, -1.0f));
   Value x = op.getOperand();
-  Value u = builder.create<math::ExpOp>(x);
+  Value u = math::ExpOp::create(builder, x);
   Value uEqOneOrNaN =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::UEQ, u, cstOne);
-  Value uMinusOne = builder.create<arith::SubFOp>(u, cstOne);
-  Value uMinusOneEqNegOne = builder.create<arith::CmpFOp>(
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::UEQ, u, cstOne);
+  Value uMinusOne = arith::SubFOp::create(builder, u, cstOne);
+  Value uMinusOneEqNegOne = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::OEQ, uMinusOne, cstNegOne);
   // logU = log(u) ~= x
-  Value logU = builder.create<math::LogOp>(u);
+  Value logU = math::LogOp::create(builder, u);
 
   // Detect exp(x) = +inf; written this way to avoid having to form +inf.
   Value isInf =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, logU, u);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, logU, u);
 
   // (u - 1) * (x / ~x)
-  Value expm1 = builder.create<arith::MulFOp>(
-      uMinusOne, builder.create<arith::DivFOp>(x, logU));
-  expm1 = builder.create<arith::SelectOp>(isInf, u, expm1);
-  Value approximation = builder.create<arith::SelectOp>(
+  Value expm1 = arith::MulFOp::create(builder,
+      uMinusOne, arith::DivFOp::create(builder, x, logU));
+  expm1 = arith::SelectOp::create(builder, isInf, u, expm1);
+  Value approximation = arith::SelectOp::create(builder,
       uEqOneOrNaN, x,
-      builder.create<arith::SelectOp>(uMinusOneEqNegOne, cstNegOne, expm1));
+      arith::SelectOp::create(builder, uMinusOneEqNegOne, cstNegOne, expm1));
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -1495,40 +1495,40 @@ LogicalResult SinAndCosApproximation<isSine, OpTy>::matchAndRewrite(
     return broadcast(builder, value, shape);
   };
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
   auto sub = [&](Value a, Value b) -> Value {
-    return builder.create<arith::SubFOp>(a, b);
+    return arith::SubFOp::create(builder, a, b);
   };
-  auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); };
+  auto floor = [&](Value a) { return math::FloorOp::create(builder, a); };
 
   auto i32Vec = broadcast(builder.getI32Type(), shape);
   auto fPToSingedInteger = [&](Value a) -> Value {
-    return builder.create<arith::FPToSIOp>(i32Vec, a);
+    return arith::FPToSIOp::create(builder, i32Vec, a);
   };
 
   auto modulo4 = [&](Value a) -> Value {
-    return builder.create<arith::AndIOp>(a, bcast(i32Cst(builder, 3)));
+    return arith::AndIOp::create(builder, a, bcast(i32Cst(builder, 3)));
   };
 
   auto isEqualTo = [&](Value a, Value b) -> Value {
-    return builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, a, b);
+    return arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, a, b);
   };
 
   auto isGreaterThan = [&](Value a, Value b) -> Value {
-    return builder.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, a, b);
+    return arith::CmpIOp::create(builder, arith::CmpIPredicate::sgt, a, b);
   };
 
   auto select = [&](Value cond, Value t, Value f) -> Value {
-    return builder.create<arith::SelectOp>(cond, t, f);
+    return arith::SelectOp::create(builder, cond, t, f);
   };
 
   auto fmla = [&](Value a, Value b, Value c) {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto bitwiseOr = [&](Value a, Value b) {
-    return builder.create<arith::OrIOp>(a, b);
+    return arith::OrIOp::create(builder, a, b);
   };
 
   Value twoOverPi = bcast(f32Cst(builder, (float)TWO_OVER_PI));
@@ -1625,7 +1625,7 @@ CbrtApproximation::matchAndRewrite(math::CbrtOp op,
   intTy = broadcast(intTy, shape);
 
   auto bconst = [&](TypedAttr attr) -> Value {
-    Value value = b.create<arith::ConstantOp>(attr);
+    Value value = arith::ConstantOp::create(b, attr);
     return broadcast(b, value, shape);
   };
 
@@ -1642,44 +1642,44 @@ CbrtApproximation::matchAndRewrite(math::CbrtOp op,
   // union {int ix; float x;};
   // x = x0;
   // ix = ix/4 + ix/16;
-  Value absValue = b.create<math::AbsFOp>(operand);
-  Value intValue = b.create<arith::BitcastOp>(intTy, absValue);
-  Value divideBy4 = b.create<arith::ShRSIOp>(intValue, intTwo);
-  Value divideBy16 = b.create<arith::ShRSIOp>(intValue, intFour);
-  intValue = b.create<arith::AddIOp>(divideBy4, divideBy16);
+  Value absValue = math::AbsFOp::create(b, operand);
+  Value intValue = arith::BitcastOp::create(b, intTy, absValue);
+  Value divideBy4 = arith::ShRSIOp::create(b, intValue, intTwo);
+  Value divideBy16 = arith::ShRSIOp::create(b, intValue, intFour);
+  intValue = arith::AddIOp::create(b, divideBy4, divideBy16);
 
   // ix = ix + ix/16;
-  divideBy16 = b.create<arith::ShRSIOp>(intValue, intFour);
-  intValue = b.create<arith::AddIOp>(intValue, divideBy16);
+  divideBy16 = arith::ShRSIOp::create(b, intValue, intFour);
+  intValue = arith::AddIOp::create(b, intValue, divideBy16);
 
   // ix = ix + ix/256;
-  Value divideBy256 = b.create<arith::ShRSIOp>(intValue, intEight);
-  intValue = b.create<arith::AddIOp>(intValue, divideBy256);
+  Value divideBy256 = arith::ShRSIOp::create(b, intValue, intEight);
+  intValue = arith::AddIOp::create(b, intValue, divideBy256);
 
   // ix = 0x2a5137a0 + ix;
-  intValue = b.create<arith::AddIOp>(intValue, intMagic);
+  intValue = arith::AddIOp::create(b, intValue, intMagic);
 
   // Perform one newtons step:
   // x = 0.33333333f*(2.0f*x + x0/(x*x));
-  Value floatValue = b.create<arith::BitcastOp>(floatTy, intValue);
-  Value squared = b.create<arith::MulFOp>(floatValue, floatValue);
-  Value mulTwo = b.create<arith::MulFOp>(floatValue, fpTwo);
-  Value divSquared = b.create<arith::DivFOp>(absValue, squared);
-  floatValue = b.create<arith::AddFOp>(mulTwo, divSquared);
-  floatValue = b.create<arith::MulFOp>(floatValue, fpThird);
+  Value floatValue = arith::BitcastOp::create(b, floatTy, intValue);
+  Value squared = arith::MulFOp::create(b, floatValue, floatValue);
+  Value mulTwo = arith::MulFOp::create(b, floatValue, fpTwo);
+  Value divSquared = arith::DivFOp::create(b, absValue, squared);
+  floatValue = arith::AddFOp::create(b, mulTwo, divSquared);
+  floatValue = arith::MulFOp::create(b, floatValue, fpThird);
 
   // x = 0.33333333f*(2.0f*x + x0/(x*x));
-  squared = b.create<arith::MulFOp>(floatValue, floatValue);
-  mulTwo = b.create<arith::MulFOp>(floatValue, fpTwo);
-  divSquared = b.create<arith::DivFOp>(absValue, squared);
-  floatValue = b.create<arith::AddFOp>(mulTwo, divSquared);
-  floatValue = b.create<arith::MulFOp>(floatValue, fpThird);
+  squared = arith::MulFOp::create(b, floatValue, floatValue);
+  mulTwo = arith::MulFOp::create(b, floatValue, fpTwo);
+  divSquared = arith::DivFOp::create(b, absValue, squared);
+  floatValue = arith::AddFOp::create(b, mulTwo, divSquared);
+  floatValue = arith::MulFOp::create(b, floatValue, fpThird);
 
   // Check for zero and restore sign.
   Value isZero =
-      b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absValue, fpZero);
-  floatValue = b.create<arith::SelectOp>(isZero, fpZero, floatValue);
-  floatValue = b.create<math::CopySignOp>(floatValue, operand);
+      arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absValue, fpZero);
+  floatValue = arith::SelectOp::create(b, isZero, fpZero, floatValue);
+  floatValue = math::CopySignOp::create(b, floatValue, operand);
 
   rewriter.replaceOp(op, floatValue);
   return success();
@@ -1720,29 +1720,29 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
   Value cstNegHalf = bcast(f32Cst(builder, -0.5f));
   Value cstMinNormPos = bcast(f32FromBits(builder, 0x00800000u));
 
-  Value negHalf = builder.create<arith::MulFOp>(op.getOperand(), cstNegHalf);
+  Value negHalf = arith::MulFOp::create(builder, op.getOperand(), cstNegHalf);
 
   // Select only the inverse sqrt of positive normals (denormals are
   // flushed to zero).
-  Value ltMinMask = builder.create<arith::CmpFOp>(
+  Value ltMinMask = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::OLT, op.getOperand(), cstMinNormPos);
-  Value infMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value infMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                 op.getOperand(), cstPosInf);
-  Value notNormalFiniteMask = builder.create<arith::OrIOp>(ltMinMask, infMask);
+  Value notNormalFiniteMask = arith::OrIOp::create(builder, ltMinMask, infMask);
 
   // Compute an approximate result.
   Value yApprox = handleMultidimensionalVectors(
       builder, op->getOperands(), 8, [&builder](ValueRange operands) -> Value {
-        return builder.create<x86vector::RsqrtOp>(operands);
+        return x86vector::RsqrtOp::create(builder, operands);
       });
 
   // Do a single step of Newton-Raphson iteration to improve the approximation.
   // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n).
   // It is essential to evaluate the inner term like this because forming
   // y_n^2 may over- or underflow.
-  Value inner = builder.create<arith::MulFOp>(negHalf, yApprox);
-  Value fma = builder.create<math::FmaOp>(yApprox, inner, cstOnePointFive);
-  Value yNewton = builder.create<arith::MulFOp>(yApprox, fma);
+  Value inner = arith::MulFOp::create(builder, negHalf, yApprox);
+  Value fma = math::FmaOp::create(builder, yApprox, inner, cstOnePointFive);
+  Value yNewton = arith::MulFOp::create(builder, yApprox, fma);
 
   // Select the result of the Newton-Raphson step for positive normal arguments.
   // For other arguments, choose the output of the intrinsic. This will
@@ -1750,7 +1750,7 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
   // x is zero or a positive denormalized float (equivalent to flushing positive
   // denormalized inputs to zero).
   Value res =
-      builder.create<arith::SelectOp>(notNormalFiniteMask, yApprox, yNewton);
+      arith::SelectOp::create(builder, notNormalFiniteMask, yApprox, yNewton);
   rewriter.replaceOp(op, res);
 
   return success();
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index f630c48cdcaa1..ddb698e1bf501 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -87,10 +87,10 @@ Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
   // TODO: support more types.
   return TypeSwitch<Type, Value>(slot.elemType)
       .Case([&](MemRefType t) {
-        return builder.create<memref::AllocaOp>(getLoc(), t);
+        return memref::AllocaOp::create(builder, getLoc(), t);
       })
       .Default([&](Type t) {
-        return builder.create<arith::ConstantOp>(getLoc(), t,
+        return arith::ConstantOp::create(builder, getLoc(), t,
                                                  builder.getZeroAttr(t));
       });
 }
@@ -137,7 +137,7 @@ DenseMap<Attribute, MemorySlot> memref::AllocaOp::destructure(
   for (Attribute usedIndex : usedIndices) {
     Type elemType = memrefType.getTypeAtIndex(usedIndex);
     MemRefType elemPtr = MemRefType::get({}, elemType);
-    auto subAlloca = builder.create<memref::AllocaOp>(getLoc(), elemPtr);
+    auto subAlloca = memref::AllocaOp::create(builder, getLoc(), elemPtr);
     newAllocators.push_back(subAlloca);
     slotMap.try_emplace<MemorySlot>(usedIndex,
                                     {subAlloca.getResult(), elemType});
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index d1a9920aa66c5..706ac4b4867f4 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -213,7 +213,7 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
 
     // Create and insert the alloc op for the new memref.
-    auto newAlloc = rewriter.create<AllocLikeOp>(
+    auto newAlloc = AllocLikeOp::create(rewriter,
         alloc.getLoc(), newMemRefType, dynamicSizes, alloc.getSymbolOperands(),
         alloc.getAlignmentAttr());
     // Insert a cast so we have the same type as the old alloc.
@@ -836,7 +836,7 @@ void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
 void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
                   int64_t index) {
   auto loc = result.location;
-  Value indexValue = builder.create<arith::ConstantIndexOp>(loc, index);
+  Value indexValue = arith::ConstantIndexOp::create(builder, loc, index);
   build(builder, result, source, indexValue);
 }
 
@@ -1083,9 +1083,9 @@ struct DimOfMemRefReshape : public OpRewritePattern<DimOp> {
     rewriter.setInsertionPointAfter(reshape);
     Location loc = dim.getLoc();
     Value load =
-        rewriter.create<LoadOp>(loc, reshape.getShape(), dim.getIndex());
+        LoadOp::create(rewriter, loc, reshape.getShape(), dim.getIndex());
     if (load.getType() != dim.getType())
-      load = rewriter.create<arith::IndexCastOp>(loc, dim.getType(), load);
+      load = arith::IndexCastOp::create(rewriter, loc, dim.getType(), load);
     rewriter.replaceOp(dim, load);
     return success();
   }
@@ -1358,7 +1358,7 @@ static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
     assert(isa<Attribute>(maybeConstant) &&
            "The constified value should be either unchanged (i.e., == result) "
            "or a constant");
-    Value constantVal = rewriter.create<arith::ConstantIndexOp>(
+    Value constantVal = arith::ConstantIndexOp::create(rewriter,
         loc, llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
     for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
       // modifyOpInPlace: lambda cannot capture structured bindings in C++17
@@ -2587,7 +2587,7 @@ struct CollapseShapeOpMemRefCastFolder
       rewriter.modifyOpInPlace(
           op, [&]() { op.getSrcMutable().assign(cast.getSource()); });
     } else {
-      Value newOp = rewriter.create<CollapseShapeOp>(
+      Value newOp = CollapseShapeOp::create(rewriter,
           op->getLoc(), cast.getSource(), op.getReassociationIndices());
       rewriter.replaceOpWithNewOp<CastOp>(op, op.getType(), newOp);
     }
@@ -3045,15 +3045,15 @@ SmallVector<Range, 8> mlir::getOrCreateRanges(OffsetSizeAndStrideOpInterface op,
     Value offset =
         op.isDynamicOffset(idx)
             ? op.getDynamicOffset(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticOffset(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticOffset(idx));
     Value size =
         op.isDynamicSize(idx)
             ? op.getDynamicSize(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticSize(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticSize(idx));
     Value stride =
         op.isDynamicStride(idx)
             ? op.getDynamicStride(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticStride(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticStride(idx));
     res.emplace_back(Range{offset, size, stride});
   }
   return res;
@@ -3212,7 +3212,7 @@ class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
     if (!resultType)
       return failure();
 
-    Value newSubView = rewriter.create<SubViewOp>(
+    Value newSubView = SubViewOp::create(rewriter,
         subViewOp.getLoc(), resultType, castOp.getSource(),
         subViewOp.getOffsets(), subViewOp.getSizes(), subViewOp.getStrides(),
         subViewOp.getStaticOffsets(), subViewOp.getStaticSizes(),
@@ -3534,7 +3534,7 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
       return failure();
 
     // Create new ViewOp.
-    auto newViewOp = rewriter.create<ViewOp>(
+    auto newViewOp = ViewOp::create(rewriter,
         viewOp.getLoc(), newMemRefType, viewOp.getOperand(0),
         viewOp.getByteShift(), newOperands);
     // Insert a cast so we have the same type as the old memref type.
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 89640ac323b68..8176ed0f02dce 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -156,7 +156,7 @@ transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
       Type resultType = alloca.getResult().getType();
       OpBuilder builder(rewriter.getContext());
       // TODO: Add a better builder for this.
-      globalOp = builder.create<memref::GlobalOp>(
+      globalOp = memref::GlobalOp::create(builder,
           loc, StringAttr::get(ctx, "alloca"), StringAttr::get(ctx, "private"),
           TypeAttr::get(resultType), Attribute{}, UnitAttr{}, IntegerAttr{});
       symbolTable.insert(globalOp);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
index c433415944323..a2b689ed13cbf 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
@@ -22,11 +22,11 @@ struct DefaultAllocationInterface
           DefaultAllocationInterface, memref::AllocOp> {
   static std::optional<Operation *> buildDealloc(OpBuilder &builder,
                                                  Value alloc) {
-    return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
+    return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
         .getOperation();
   }
   static std::optional<Value> buildClone(OpBuilder &builder, Value alloc) {
-    return builder.create<bufferization::CloneOp>(alloc.getLoc(), alloc)
+    return bufferization::CloneOp::create(builder, alloc.getLoc(), alloc)
         .getResult();
   }
   static ::mlir::HoistingKind getHoistingKind() {
@@ -35,7 +35,7 @@ struct DefaultAllocationInterface
   static ::std::optional<::mlir::Operation *>
   buildPromotedAlloc(OpBuilder &builder, Value alloc) {
     Operation *definingOp = alloc.getDefiningOp();
-    return builder.create<memref::AllocaOp>(
+    return memref::AllocaOp::create(builder,
         definingOp->getLoc(), cast<MemRefType>(definingOp->getResultTypes()[0]),
         definingOp->getOperands(), definingOp->getAttrs());
   }
@@ -52,7 +52,7 @@ struct DefaultReallocationInterface
           DefaultAllocationInterface, memref::ReallocOp> {
   static std::optional<Operation *> buildDealloc(OpBuilder &builder,
                                                  Value realloc) {
-    return builder.create<memref::DeallocOp>(realloc.getLoc(), realloc)
+    return memref::DeallocOp::create(builder, realloc.getLoc(), realloc)
         .getOperation();
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
index d25ddb41aa4eb..4fcceeaa33deb 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
@@ -125,7 +125,7 @@ struct ComposeSubViewOpPattern : public OpRewritePattern<memref::SubViewOp> {
         }
 
         AffineMap map = AffineMap::get(0, affineApplyOperands.size(), expr);
-        Value result = rewriter.create<affine::AffineApplyOp>(
+        Value result = affine::AffineApplyOp::create(rewriter,
             op.getLoc(), map, affineApplyOperands);
         offsets.push_back(result);
       }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
index d2a032688fb6d..51970f019d27f 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
@@ -99,7 +99,7 @@ static Value getOffsetForBitwidth(Location loc, OpFoldResult srcIdx,
       affine::makeComposedFoldedAffineApply(builder, loc, offsetExpr, {srcIdx});
   Value bitOffset = getValueOrCreateConstantIndexOp(builder, loc, offsetVal);
   IntegerType dstType = builder.getIntegerType(targetBits);
-  return builder.create<arith::IndexCastOp>(loc, dstType, bitOffset);
+  return arith::IndexCastOp::create(builder, loc, dstType, bitOffset);
 }
 
 /// When writing a subbyte size, masked bitwise operations are used to only
@@ -112,14 +112,14 @@ static Value getSubByteWriteMask(Location loc, OpFoldResult linearizedIndices,
   auto dstIntegerType = builder.getIntegerType(dstBits);
   auto maskRightAlignedAttr =
       builder.getIntegerAttr(dstIntegerType, (1 << srcBits) - 1);
-  Value maskRightAligned = builder.create<arith::ConstantOp>(
+  Value maskRightAligned = arith::ConstantOp::create(builder,
       loc, dstIntegerType, maskRightAlignedAttr);
   Value writeMaskInverse =
-      builder.create<arith::ShLIOp>(loc, maskRightAligned, bitwidthOffset);
+      arith::ShLIOp::create(builder, loc, maskRightAligned, bitwidthOffset);
   auto flipValAttr = builder.getIntegerAttr(dstIntegerType, -1);
   Value flipVal =
-      builder.create<arith::ConstantOp>(loc, dstIntegerType, flipValAttr);
-  return builder.create<arith::XOrIOp>(loc, writeMaskInverse, flipVal);
+      arith::ConstantOp::create(builder, loc, dstIntegerType, flipValAttr);
+  return arith::XOrIOp::create(builder, loc, writeMaskInverse, flipVal);
 }
 
 /// Returns the scaled linearized index based on the `srcBits` and `dstBits`
@@ -141,7 +141,7 @@ getLinearizedSrcIndices(OpBuilder &builder, Location loc, int64_t srcBits,
                         const SmallVector<OpFoldResult> &indices,
                         Value memref) {
   auto stridedMetadata =
-      builder.create<memref::ExtractStridedMetadataOp>(loc, memref);
+      memref::ExtractStridedMetadataOp::create(builder, loc, memref);
   OpFoldResult linearizedIndices;
   std::tie(std::ignore, linearizedIndices) =
       memref::getLinearizedMemRefOffsetAndSize(
@@ -298,7 +298,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
     // Special case 0-rank memref loads.
     Value bitsLoad;
     if (convertedType.getRank() == 0) {
-      bitsLoad = rewriter.create<memref::LoadOp>(loc, adaptor.getMemref(),
+      bitsLoad = memref::LoadOp::create(rewriter, loc, adaptor.getMemref(),
                                                  ValueRange{});
     } else {
       // Linearize the indices of the original load instruction. Do not account
@@ -306,7 +306,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
       OpFoldResult linearizedIndices = getLinearizedSrcIndices(
           rewriter, loc, srcBits, adaptor.getIndices(), op.getMemRef());
 
-      Value newLoad = rewriter.create<memref::LoadOp>(
+      Value newLoad = memref::LoadOp::create(rewriter,
           loc, adaptor.getMemref(),
           getIndicesForLoadOrStore(rewriter, loc, linearizedIndices, srcBits,
                                    dstBits));
@@ -315,7 +315,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
       // Note, currently only the big-endian is supported.
       Value bitwidthOffset = getOffsetForBitwidth(loc, linearizedIndices,
                                                   srcBits, dstBits, rewriter);
-      bitsLoad = rewriter.create<arith::ShRSIOp>(loc, newLoad, bitwidthOffset);
+      bitsLoad = arith::ShRSIOp::create(rewriter, loc, newLoad, bitwidthOffset);
     }
 
     // Get the corresponding bits. If the arith computation bitwidth equals
@@ -326,13 +326,13 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
     Operation *result;
     auto resultTy = getTypeConverter()->convertType(oldElementType);
     if (resultTy == convertedElementType) {
-      auto mask = rewriter.create<arith::ConstantOp>(
+      auto mask = arith::ConstantOp::create(rewriter,
           loc, convertedElementType,
           rewriter.getIntegerAttr(convertedElementType, (1 << srcBits) - 1));
 
-      result = rewriter.create<arith::AndIOp>(loc, bitsLoad, mask);
+      result = arith::AndIOp::create(rewriter, loc, bitsLoad, mask);
     } else {
-      result = rewriter.create<arith::TruncIOp>(loc, resultTy, bitsLoad);
+      result = arith::TruncIOp::create(rewriter, loc, resultTy, bitsLoad);
     }
 
     rewriter.replaceOp(op, result->getResult(0));
@@ -415,12 +415,12 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
     }
 
     Location loc = op.getLoc();
-    Value extendedInput = rewriter.create<arith::ExtUIOp>(loc, dstIntegerType,
+    Value extendedInput = arith::ExtUIOp::create(rewriter, loc, dstIntegerType,
                                                           adaptor.getValue());
 
     // Special case 0-rank memref stores. No need for masking.
     if (convertedType.getRank() == 0) {
-      rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::assign,
+      memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::assign,
                                            extendedInput, adaptor.getMemref(),
                                            ValueRange{});
       rewriter.eraseOp(op);
@@ -437,14 +437,14 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
                                           dstBits, bitwidthOffset, rewriter);
     // Align the value to write with the destination bits
     Value alignedVal =
-        rewriter.create<arith::ShLIOp>(loc, extendedInput, bitwidthOffset);
+        arith::ShLIOp::create(rewriter, loc, extendedInput, bitwidthOffset);
 
     // Clear destination bits
-    rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::andi,
+    memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::andi,
                                          writeMask, adaptor.getMemref(),
                                          storeIndices);
     // Write srcs bits to destination
-    rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::ori,
+    memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::ori,
                                          alignedVal, adaptor.getMemref(),
                                          storeIndices);
     rewriter.eraseOp(op);
@@ -506,7 +506,7 @@ struct ConvertMemRefSubview final : OpConversionPattern<memref::SubViewOp> {
     }
 
     // Transform the offsets, sizes and strides according to the emulation.
-    auto stridedMetadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+    auto stridedMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
         loc, subViewOp.getViewSource());
 
     OpFoldResult linearizedIndices;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
index a617029ce470f..64e117fb77bcf 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
@@ -51,15 +51,15 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
       Value size;
       // Load dynamic sizes from the shape input, use constants for static dims.
       if (op.getType().isDynamicDim(i)) {
-        Value index = rewriter.create<arith::ConstantIndexOp>(loc, i);
-        size = rewriter.create<memref::LoadOp>(loc, op.getShape(), index);
+        Value index = arith::ConstantIndexOp::create(rewriter, loc, i);
+        size = memref::LoadOp::create(rewriter, loc, op.getShape(), index);
         if (!isa<IndexType>(size.getType()))
-          size = rewriter.create<arith::IndexCastOp>(
+          size = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getIndexType(), size);
         sizes[i] = size;
       } else {
         auto sizeAttr = rewriter.getIndexAttr(op.getType().getDimSize(i));
-        size = rewriter.create<arith::ConstantOp>(loc, sizeAttr);
+        size = arith::ConstantOp::create(rewriter, loc, sizeAttr);
         sizes[i] = sizeAttr;
       }
       if (stride)
@@ -69,10 +69,10 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
 
       if (i > 0) {
         if (stride) {
-          stride = rewriter.create<arith::MulIOp>(loc, stride, size);
+          stride = arith::MulIOp::create(rewriter, loc, stride, size);
         } else if (op.getType().isDynamicDim(i)) {
-          stride = rewriter.create<arith::MulIOp>(
-              loc, rewriter.create<arith::ConstantIndexOp>(loc, staticStride),
+          stride = arith::MulIOp::create(rewriter,
+              loc, arith::ConstantIndexOp::create(rewriter, loc, staticStride),
               size);
         } else {
           staticStride *= op.getType().getDimSize(i);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
index 7475d442b7b9a..b54fa61714323 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
@@ -73,7 +73,7 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     if (ShapedType::isDynamic(inputSize)) {
       Value dimZero = getValueOrCreateConstantIndexOp(rewriter, loc,
                                                       rewriter.getIndexAttr(0));
-      currSize = rewriter.create<memref::DimOp>(loc, op.getSource(), dimZero)
+      currSize = memref::DimOp::create(rewriter, loc, op.getSource(), dimZero)
                      .getResult();
     }
 
@@ -88,9 +88,9 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     // the old buffer is smaller than the requested size.
     Value lhs = getValueOrCreateConstantIndexOp(rewriter, loc, currSize);
     Value rhs = getValueOrCreateConstantIndexOp(rewriter, loc, targetSize);
-    Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                                 lhs, rhs);
-    auto ifOp = rewriter.create<scf::IfOp>(
+    auto ifOp = scf::IfOp::create(rewriter,
         loc, cond,
         [&](OpBuilder &builder, Location loc) {
           // Allocate the new buffer. If it is a dynamic memref we need to pass
@@ -100,25 +100,25 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           if (op.getDynamicResultSize())
             dynamicSizeOperands.push_back(op.getDynamicResultSize());
 
-          Value newAlloc = builder.create<memref::AllocOp>(
+          Value newAlloc = memref::AllocOp::create(builder,
               loc, op.getResult().getType(), dynamicSizeOperands,
               op.getAlignmentAttr());
 
           // Take a subview of the new (bigger) buffer such that we can copy the
           // old values over (the copy operation requires both operands to have
           // the same shape).
-          Value subview = builder.create<memref::SubViewOp>(
+          Value subview = memref::SubViewOp::create(builder,
               loc, newAlloc, ArrayRef<OpFoldResult>{rewriter.getIndexAttr(0)},
               ArrayRef<OpFoldResult>{currSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
-          builder.create<memref::CopyOp>(loc, op.getSource(), subview);
+          memref::CopyOp::create(builder, loc, op.getSource(), subview);
 
           // Insert the deallocation of the old buffer only if requested
           // (enabled by default).
           if (emitDeallocs)
-            builder.create<memref::DeallocOp>(loc, op.getSource());
+            memref::DeallocOp::create(builder, loc, op.getSource());
 
-          builder.create<scf::YieldOp>(loc, newAlloc);
+          scf::YieldOp::create(builder, loc, newAlloc);
         },
         [&](OpBuilder &builder, Location loc) {
           // We need to reinterpret-cast here because either the input or output
@@ -126,11 +126,11 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           // dynamic or vice-versa. If both are static and the original buffer
           // is already bigger than the requested size, the cast represents a
           // subview operation.
-          Value casted = builder.create<memref::ReinterpretCastOp>(
+          Value casted = memref::ReinterpretCastOp::create(builder,
               loc, cast<MemRefType>(op.getResult().getType()), op.getSource(),
               rewriter.getIndexAttr(0), ArrayRef<OpFoldResult>{targetSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
-          builder.create<scf::YieldOp>(loc, casted);
+          scf::YieldOp::create(builder, loc, casted);
         });
 
     rewriter.replaceOp(op, ifOp.getResult(0));
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index 2ba798f48ac7c..f581b6490fadb 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -66,7 +66,7 @@ resolveSubviewStridedMetadata(RewriterBase &rewriter,
   unsigned sourceRank = sourceType.getRank();
 
   auto newExtractStridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(origLoc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, origLoc, source);
 
   auto [sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
 #ifndef NDEBUG
@@ -577,7 +577,7 @@ static FailureOr<StridedMetadata> resolveReshapeStridedMetadata(
   unsigned sourceRank = sourceType.getRank();
 
   auto newExtractStridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(origLoc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, origLoc, source);
 
   // Collect statically known information.
   auto [strides, offset] = sourceType.getStridesAndOffset();
@@ -828,14 +828,14 @@ struct ExtractStridedMetadataOpAllocFolder
       if (allocLikeOp.getType() == baseBufferType)
         results.push_back(allocLikeOp);
       else
-        results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+        results.push_back(memref::ReinterpretCastOp::create(rewriter,
             loc, baseBufferType, allocLikeOp, offset,
             /*sizes=*/ArrayRef<int64_t>(),
             /*strides=*/ArrayRef<int64_t>()));
     }
 
     // Offset.
-    results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    results.push_back(arith::ConstantIndexOp::create(rewriter, loc, offset));
 
     for (OpFoldResult size : sizes)
       results.push_back(getValueOrCreateConstantIndexOp(rewriter, loc, size));
@@ -900,19 +900,19 @@ struct ExtractStridedMetadataOpGetGlobalFolder
     if (getGlobalOp.getType() == baseBufferType)
       results.push_back(getGlobalOp);
     else
-      results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+      results.push_back(memref::ReinterpretCastOp::create(rewriter,
           loc, baseBufferType, getGlobalOp, offset,
           /*sizes=*/ArrayRef<int64_t>(),
           /*strides=*/ArrayRef<int64_t>()));
 
     // Offset.
-    results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    results.push_back(arith::ConstantIndexOp::create(rewriter, loc, offset));
 
     for (auto size : sizes)
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, size));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, size));
 
     for (auto stride : strides)
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, stride));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, stride));
 
     rewriter.replaceOp(op, results);
     return success();
@@ -1009,7 +1009,7 @@ class ExtractStridedMetadataOpReinterpretCastFolder
     results.resize_for_overwrite(rank * 2 + 2);
 
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(
+        memref::ExtractStridedMetadataOp::create(rewriter,
             loc, reinterpretCastOp.getSource());
 
     // Register the base_buffer.
@@ -1083,7 +1083,7 @@ class ExtractStridedMetadataOpCastFolder
     results.resize_for_overwrite(rank * 2 + 2);
 
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc,
+        memref::ExtractStridedMetadataOp::create(rewriter, loc,
                                                           castOp.getSource());
 
     // Register the base_buffer.
@@ -1143,7 +1143,7 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
     if (!memSpaceCastOp)
       return failure();
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(
+        memref::ExtractStridedMetadataOp::create(rewriter,
             loc, memSpaceCastOp.getSource());
     SmallVector<Value> results(newExtractStridedMetadata.getResults());
     // As with most other strided metadata rewrite patterns, don't introduce
@@ -1158,7 +1158,7 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
       MemRefType::Builder newTypeBuilder(baseBufferType);
       newTypeBuilder.setMemorySpace(
           memSpaceCastOp.getResult().getType().getMemorySpace());
-      results[0] = rewriter.create<memref::MemorySpaceCastOp>(
+      results[0] = memref::MemorySpaceCastOp::create(rewriter,
           loc, Type{newTypeBuilder}, baseBuffer);
     } else {
       results[0] = nullptr;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
index 2f5c9436fb8c7..65f398f6dc5a8 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
@@ -42,7 +42,7 @@ static memref::LoadOp rebuildLoadOp(RewriterBase &rewriter,
                                     memref::LoadOp loadOp, Value srcMemRef,
                                     ArrayRef<Value> indices) {
   Location loc = loadOp.getLoc();
-  return rewriter.create<memref::LoadOp>(loc, srcMemRef, indices,
+  return memref::LoadOp::create(rewriter, loc, srcMemRef, indices,
                                          loadOp.getNontemporal());
 }
 
@@ -72,7 +72,7 @@ static memref::StoreOp rebuildStoreOp(RewriterBase &rewriter,
                                       memref::StoreOp storeOp, Value srcMemRef,
                                       ArrayRef<Value> indices) {
   Location loc = storeOp.getLoc();
-  return rewriter.create<memref::StoreOp>(loc, storeOp.getValueToStore(),
+  return memref::StoreOp::create(rewriter, loc, storeOp.getValueToStore(),
                                           srcMemRef, indices,
                                           storeOp.getNontemporal());
 }
@@ -104,7 +104,7 @@ static nvgpu::LdMatrixOp rebuildLdMatrixOp(RewriterBase &rewriter,
                                            Value srcMemRef,
                                            ArrayRef<Value> indices) {
   Location loc = ldMatrixOp.getLoc();
-  return rewriter.create<nvgpu::LdMatrixOp>(
+  return nvgpu::LdMatrixOp::create(rewriter,
       loc, ldMatrixOp.getResult().getType(), srcMemRef, indices,
       ldMatrixOp.getTranspose(), ldMatrixOp.getNumTiles());
 }
@@ -132,7 +132,7 @@ rebuildTransferReadOp(RewriterBase &rewriter,
                       vector::TransferReadOp transferReadOp, Value srcMemRef,
                       ArrayRef<Value> indices) {
   Location loc = transferReadOp.getLoc();
-  return rewriter.create<vector::TransferReadOp>(
+  return vector::TransferReadOp::create(rewriter,
       loc, transferReadOp.getResult().getType(), srcMemRef, indices,
       transferReadOp.getPermutationMap(), transferReadOp.getPadding(),
       transferReadOp.getMask(), transferReadOp.getInBoundsAttr());
@@ -150,7 +150,7 @@ rebuildTransferWriteOp(RewriterBase &rewriter,
                        vector::TransferWriteOp transferWriteOp, Value srcMemRef,
                        ArrayRef<Value> indices) {
   Location loc = transferWriteOp.getLoc();
-  return rewriter.create<vector::TransferWriteOp>(
+  return vector::TransferWriteOp::create(rewriter,
       loc, transferWriteOp.getValue(), srcMemRef, indices,
       transferWriteOp.getPermutationMapAttr(), transferWriteOp.getMask(),
       transferWriteOp.getInBoundsAttr());
@@ -183,7 +183,7 @@ getGenericOpViewSizeForEachDim(RewriterBase &rewriter,
                                LoadStoreLikeOp loadStoreLikeOp) {
   Location loc = loadStoreLikeOp.getLoc();
   auto extractStridedMetadataOp =
-      rewriter.create<memref::ExtractStridedMetadataOp>(
+      memref::ExtractStridedMetadataOp::create(rewriter,
           loc, getSrcMemRef(loadStoreLikeOp));
   SmallVector<OpFoldResult> srcSizes =
       extractStridedMetadataOp.getConstifiedMixedSizes();
@@ -267,12 +267,12 @@ struct LoadStoreLikeOpRewriter : public OpRewritePattern<LoadStoreLikeOp> {
     // apply them properly to the input indices.
     // Therefore the strides multipliers are simply ones.
     auto subview =
-        rewriter.create<memref::SubViewOp>(loc, /*source=*/srcMemRef,
+        memref::SubViewOp::create(rewriter, loc, /*source=*/srcMemRef,
                                            /*offsets=*/indices,
                                            /*sizes=*/sizes, /*strides=*/ones);
     // Rewrite the load/store with the subview as the base pointer.
     SmallVector<Value> zeros(loadStoreRank,
-                             rewriter.create<arith::ConstantIndexOp>(loc, 0));
+                             arith::ConstantIndexOp::create(rewriter, loc, 0));
     LoadStoreLikeOp newLoadStore = rebuildOpFromAddressAndIndices(
         rewriter, loadStoreLikeOp, subview.getResult(), zeros);
     rewriter.replaceOp(loadStoreLikeOp, newLoadStore->getResults());
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
index e9729a4766a0a..1a73cc6dd948a 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
@@ -45,7 +45,7 @@ using namespace mlir;
 static Value getValueFromOpFoldResult(OpBuilder &rewriter, Location loc,
                                       OpFoldResult in) {
   if (Attribute offsetAttr = dyn_cast<Attribute>(in)) {
-    return rewriter.create<arith::ConstantIndexOp>(
+    return arith::ConstantIndexOp::create(rewriter,
         loc, cast<IntegerAttr>(offsetAttr).getInt());
   }
   return cast<Value>(in);
@@ -65,7 +65,7 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
   }
 
   memref::ExtractStridedMetadataOp stridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(loc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, loc, source);
 
   auto typeBit = sourceType.getElementType().getIntOrFloatBitWidth();
   OpFoldResult linearizedIndices;
@@ -79,7 +79,7 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
           getAsOpFoldResult(indices));
 
   return std::make_pair(
-      rewriter.create<memref::ReinterpretCastOp>(
+      memref::ReinterpretCastOp::create(rewriter,
           loc, source,
           /* offset = */ linearizedInfo.linearizedOffset,
           /* shapes = */
@@ -116,7 +116,7 @@ template <typename T>
 static void castAllocResult(T oper, T newOper, Location loc,
                             PatternRewriter &rewriter) {
   memref::ExtractStridedMetadataOp stridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(loc, oper);
+      memref::ExtractStridedMetadataOp::create(rewriter, loc, oper);
   rewriter.replaceOpWithNewOp<memref::ReinterpretCastOp>(
       oper, cast<MemRefType>(oper.getType()), newOper,
       /*offset=*/rewriter.getIndexAttr(0),
@@ -130,62 +130,62 @@ static void replaceOp(T op, PatternRewriter &rewriter, Value flatMemref,
   Location loc = op->getLoc();
   llvm::TypeSwitch<Operation *>(op.getOperation())
       .template Case<memref::AllocOp>([&](auto oper) {
-        auto newAlloc = rewriter.create<memref::AllocOp>(
+        auto newAlloc = memref::AllocOp::create(rewriter,
             loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloc, loc, rewriter);
       })
       .template Case<memref::AllocaOp>([&](auto oper) {
-        auto newAlloca = rewriter.create<memref::AllocaOp>(
+        auto newAlloca = memref::AllocaOp::create(rewriter,
             loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloca, loc, rewriter);
       })
       .template Case<memref::LoadOp>([&](auto op) {
-        auto newLoad = rewriter.create<memref::LoadOp>(
+        auto newLoad = memref::LoadOp::create(rewriter,
             loc, op->getResultTypes(), flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<memref::StoreOp>([&](auto op) {
-        auto newStore = rewriter.create<memref::StoreOp>(
+        auto newStore = memref::StoreOp::create(rewriter,
             loc, op->getOperands().front(), flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::LoadOp>([&](auto op) {
-        auto newLoad = rewriter.create<vector::LoadOp>(
+        auto newLoad = vector::LoadOp::create(rewriter,
             loc, op->getResultTypes(), flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<vector::StoreOp>([&](auto op) {
-        auto newStore = rewriter.create<vector::StoreOp>(
+        auto newStore = vector::StoreOp::create(rewriter,
             loc, op->getOperands().front(), flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::MaskedLoadOp>([&](auto op) {
-        auto newMaskedLoad = rewriter.create<vector::MaskedLoadOp>(
+        auto newMaskedLoad = vector::MaskedLoadOp::create(rewriter,
             loc, op.getType(), flatMemref, ValueRange{offset}, op.getMask(),
             op.getPassThru());
         newMaskedLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedLoad.getResult());
       })
       .template Case<vector::MaskedStoreOp>([&](auto op) {
-        auto newMaskedStore = rewriter.create<vector::MaskedStoreOp>(
+        auto newMaskedStore = vector::MaskedStoreOp::create(rewriter,
             loc, flatMemref, ValueRange{offset}, op.getMask(),
             op.getValueToStore());
         newMaskedStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedStore);
       })
       .template Case<vector::TransferReadOp>([&](auto op) {
-        auto newTransferRead = rewriter.create<vector::TransferReadOp>(
+        auto newTransferRead = vector::TransferReadOp::create(rewriter,
             loc, op.getType(), flatMemref, ValueRange{offset}, op.getPadding());
         rewriter.replaceOp(op, newTransferRead.getResult());
       })
       .template Case<vector::TransferWriteOp>([&](auto op) {
-        auto newTransferWrite = rewriter.create<vector::TransferWriteOp>(
+        auto newTransferWrite = vector::TransferWriteOp::create(rewriter,
             loc, op.getVector(), flatMemref, ValueRange{offset});
         rewriter.replaceOp(op, newTransferWrite);
       })
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index 42c43ba8553a3..8052115610eaa 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -78,7 +78,7 @@ static LogicalResult resolveSourceIndicesExpandShape(
         llvm::map_to_vector(group, [&](int64_t d) { return destShape[d]; });
     SmallVector<Value> groupIndices =
         llvm::map_to_vector(group, [&](int64_t d) { return indices[d]; });
-    Value collapsedIndex = rewriter.create<affine::AffineLinearizeIndexOp>(
+    Value collapsedIndex = affine::AffineLinearizeIndexOp::create(rewriter,
         loc, groupIndices, groupBasis, /*disjoint=*/startsInbounds);
     sourceIndices.push_back(collapsedIndex);
   }
@@ -104,7 +104,7 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
                                   ValueRange indices,
                                   SmallVectorImpl<Value> &sourceIndices) {
   // Note: collapse_shape requires a strided memref, we can do this.
-  auto metadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+  auto metadata = memref::ExtractStridedMetadataOp::create(rewriter,
       loc, collapseShapeOp.getSrc());
   SmallVector<OpFoldResult> sourceSizes = metadata.getConstifiedMixedSizes();
   for (auto [index, group] :
@@ -119,7 +119,7 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
 
     SmallVector<OpFoldResult> basis =
         llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-    auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
         loc, index, basis, /*hasOuterBound=*/true);
     llvm::append_range(sourceIndices, delinearize.getResults());
   }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
index 21361d2e9a2d7..33be43da47679 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
@@ -53,7 +53,7 @@ FailureOr<Value> memref::buildIndependentOp(OpBuilder &b,
 
   // Create a new memref::AllocaOp.
   Value newAllocaOp =
-      b.create<AllocaOp>(loc, newSizes, allocaOp.getType().getElementType());
+      AllocaOp::create(b, loc, newSizes, allocaOp.getType().getElementType());
 
   // Create a memref::SubViewOp.
   SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
@@ -73,10 +73,10 @@ propagateSubViewOp(RewriterBase &rewriter,
   MemRefType newResultType = SubViewOp::inferRankReducedResultType(
       op.getType().getShape(), op.getSourceType(), op.getMixedOffsets(),
       op.getMixedSizes(), op.getMixedStrides());
-  Value newSubview = rewriter.create<SubViewOp>(
+  Value newSubview = SubViewOp::create(rewriter,
       op.getLoc(), newResultType, conversionOp.getOperand(0),
       op.getMixedOffsets(), op.getMixedSizes(), op.getMixedStrides());
-  auto newConversionOp = rewriter.create<UnrealizedConversionCastOp>(
+  auto newConversionOp = UnrealizedConversionCastOp::create(rewriter,
       op.getLoc(), op.getType(), newSubview);
   rewriter.replaceAllUsesWith(op.getResult(), newConversionOp->getResult(0));
   return newConversionOp;
@@ -108,7 +108,7 @@ static void replaceAndPropagateMemRefType(RewriterBase &rewriter,
   SmallVector<UnrealizedConversionCastOp> unrealizedConversions;
   for (const auto &it :
        llvm::enumerate(llvm::zip(from->getResults(), to->getResults()))) {
-    unrealizedConversions.push_back(rewriter.create<UnrealizedConversionCastOp>(
+    unrealizedConversions.push_back(UnrealizedConversionCastOp::create(rewriter,
         to->getLoc(), std::get<0>(it.value()).getType(),
         std::get<1>(it.value())));
     rewriter.replaceAllUsesWith(from->getResult(it.index()),
diff --git a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
index c475d92e0658e..a1b8821f38e7e 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
@@ -64,7 +64,7 @@ static void replaceUsesAndPropagateType(RewriterBase &rewriter,
         subviewUse.getType().getShape(), cast<MemRefType>(val.getType()),
         subviewUse.getStaticOffsets(), subviewUse.getStaticSizes(),
         subviewUse.getStaticStrides());
-    Value newSubview = rewriter.create<memref::SubViewOp>(
+    Value newSubview = memref::SubViewOp::create(rewriter,
         subviewUse->getLoc(), newType, val, subviewUse.getMixedOffsets(),
         subviewUse.getMixedSizes(), subviewUse.getMixedStrides());
 
@@ -178,7 +178,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   Location loc = allocOp->getLoc();
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(allocOp);
-  auto mbAlloc = rewriter.create<memref::AllocOp>(
+  auto mbAlloc = memref::AllocOp::create(rewriter,
       loc, mbMemRefType, ValueRange{}, allocOp->getAttrs());
   LLVM_DEBUG(DBGS() << "--multi-buffered alloc: " << mbAlloc << "\n");
 
@@ -212,7 +212,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   // Strides is [1, 1 ... 1 ].
   MemRefType dstMemref = memref::SubViewOp::inferRankReducedResultType(
       originalShape, mbMemRefType, offsets, sizes, strides);
-  Value subview = rewriter.create<memref::SubViewOp>(loc, dstMemref, mbAlloc,
+  Value subview = memref::SubViewOp::create(rewriter, loc, dstMemref, mbAlloc,
                                                      offsets, sizes, strides);
   LLVM_DEBUG(DBGS() << "--multi-buffered slice: " << subview << "\n");
 
@@ -225,7 +225,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPoint(deallocOp);
     auto newDeallocOp =
-        rewriter.create<memref::DeallocOp>(deallocOp->getLoc(), mbAlloc);
+        memref::DeallocOp::create(rewriter, deallocOp->getLoc(), mbAlloc);
     (void)newDeallocOp;
     LLVM_DEBUG(DBGS() << "----Created dealloc: " << newDeallocOp << "\n");
     rewriter.eraseOp(deallocOp);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index d6fcb8d9f0501..40dfad91b8959 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -278,7 +278,7 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
     if (!callOp)
       continue;
     Operation *newCallOp =
-        builder.create<func::CallOp>(userOp->getLoc(), callOp.getCalleeAttr(),
+        func::CallOp::create(builder, userOp->getLoc(), callOp.getCalleeAttr(),
                                      resultTypes, userOp->getOperands());
     bool replacingMemRefUsesFailed = false;
     bool returnTypeChanged = false;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
index e6b9e2f7e8213..ea7f394f48ba4 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
@@ -116,10 +116,10 @@ static LogicalResult reifyOpResultShapes(RewriterBase &rewriter,
     // Update the type.
     newRes.setType(reifiedTy);
     if (isa<RankedTensorType>(reifiedTy)) {
-      newResults.push_back(rewriter.create<tensor::CastOp>(loc, oldTy, newRes));
+      newResults.push_back(tensor::CastOp::create(rewriter, loc, oldTy, newRes));
     } else {
       assert(isa<MemRefType>(reifiedTy) && "expected a memref type");
-      newResults.push_back(rewriter.create<memref::CastOp>(loc, oldTy, newRes));
+      newResults.push_back(memref::CastOp::create(rewriter, loc, oldTy, newRes));
     }
   }
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
index 89a3895d06ba5..6a81a15f30e47 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
@@ -69,7 +69,7 @@ struct DimOfShapedTypeOpInterface : public OpRewritePattern<OpTy> {
     Location loc = dimOp->getLoc();
     rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
         dimOp, resultShape,
-        rewriter.create<arith::ConstantIndexOp>(loc, *dimIndex).getResult());
+        arith::ConstantIndexOp::create(rewriter, loc, *dimIndex).getResult());
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
index 7bf7c7b8e024c..e15bf379840b4 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
@@ -41,15 +41,15 @@ struct AssumeAlignmentOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto assumeOp = cast<AssumeAlignmentOp>(op);
-    Value ptr = builder.create<ExtractAlignedPointerAsIndexOp>(
+    Value ptr = ExtractAlignedPointerAsIndexOp::create(builder,
         loc, assumeOp.getMemref());
-    Value rest = builder.create<arith::RemUIOp>(
+    Value rest = arith::RemUIOp::create(builder,
         loc, ptr,
-        builder.create<arith::ConstantIndexOp>(loc, assumeOp.getAlignment()));
-    Value isAligned = builder.create<arith::CmpIOp>(
+        arith::ConstantIndexOp::create(builder, loc, assumeOp.getAlignment()));
+    Value isAligned = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::eq, rest,
-        builder.create<arith::ConstantIndexOp>(loc, 0));
-    builder.create<cf::AssertOp>(
+        arith::ConstantIndexOp::create(builder, loc, 0));
+    cf::AssertOp::create(builder,
         loc, isAligned,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "memref is not aligned to " +
@@ -72,12 +72,12 @@ struct CastOpInterface
 
     if (isa<UnrankedMemRefType>(srcType)) {
       // Check rank.
-      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
-          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
-      Value isSameRank = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
+      Value isSameRank = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameRank,
           RuntimeVerifiableOpInterface::generateErrorMessage(op,
                                                              "rank mismatch"));
@@ -96,8 +96,8 @@ struct CastOpInterface
         MemRefType::get(dynamicShape, resultType.getElementType(),
                         stridedLayout, resultType.getMemorySpace());
     Value helperCast =
-        builder.create<CastOp>(loc, dynStridesType, castOp.getSource());
-    auto metadataOp = builder.create<ExtractStridedMetadataOp>(loc, helperCast);
+        CastOp::create(builder, loc, dynStridesType, castOp.getSource());
+    auto metadataOp = ExtractStridedMetadataOp::create(builder, loc, helperCast);
 
     // Check dimension sizes.
     for (const auto &it : llvm::enumerate(resultType.getShape())) {
@@ -111,12 +111,12 @@ struct CastOpInterface
         continue;
 
       Value srcDimSz =
-          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+          DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameSz = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameSz = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
@@ -133,10 +133,10 @@ struct CastOpInterface
       // Static/dynamic offset -> dynamic offset does not need verification.
       Value srcOffset = metadataOp.getResult(1);
       Value resultOffsetVal =
-          builder.create<arith::ConstantIndexOp>(loc, resultOffset);
-      Value isSameOffset = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultOffset);
+      Value isSameOffset = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcOffset, resultOffsetVal);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameOffset,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset mismatch"));
@@ -151,10 +151,10 @@ struct CastOpInterface
       Value srcStride =
           metadataOp.getResult(2 + resultType.getRank() + it.index());
       Value resultStrideVal =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameStride = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameStride = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcStride, resultStrideVal);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameStride,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "stride mismatch of dim " + std::to_string(it.index())));
@@ -187,7 +187,7 @@ struct CopyOpInterface
       auto getDimSize = [&](Value memRef, MemRefType type,
                             int64_t dim) -> Value {
         return type.isDynamicDim(dim)
-                   ? builder.create<DimOp>(loc, memRef, dim).getResult()
+                   ? DimOp::create(builder, loc, memRef, dim).getResult()
                    : builder
                          .create<arith::ConstantIndexOp>(loc,
                                                          type.getDimSize(dim))
@@ -195,9 +195,9 @@ struct CopyOpInterface
       };
       Value sourceDim = getDimSize(copyOp.getSource(), rankedSourceType, i);
       Value targetDim = getDimSize(copyOp.getTarget(), rankedTargetType, i);
-      Value sameDimSize = builder.create<arith::CmpIOp>(
+      Value sameDimSize = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, sameDimSize,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size of " + std::to_string(i) +
@@ -212,9 +212,9 @@ struct DimOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto dimOp = cast<DimOp>(op);
-    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    builder.create<cf::AssertOp>(
+    Value rank = RankOp::create(builder, loc, dimOp.getSource());
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    cf::AssertOp::create(builder,
         loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
@@ -238,7 +238,7 @@ struct LoadStoreOpInterface
     }
     auto indices = loadStoreOp.getIndices();
 
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     Value assertCond;
     for (auto i : llvm::seq<int64_t>(0, rank)) {
       Value dimOp = builder.createOrFold<memref::DimOp>(loc, memref, i);
@@ -248,7 +248,7 @@ struct LoadStoreOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    builder.create<cf::AssertOp>(
+    cf::AssertOp::create(builder,
         loc, assertCond,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "out-of-bounds access"));
@@ -266,10 +266,10 @@ struct SubViewOpInterface
     // For each dimension, assert that:
     // 0 <= offset < dim_size
     // 0 <= offset + (size - 1) * stride < dim_size
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value one = arith::ConstantIndexOp::create(builder, loc, 1);
     auto metadataOp =
-        builder.create<ExtractStridedMetadataOp>(loc, subView.getSource());
+        ExtractStridedMetadataOp::create(builder, loc, subView.getSource());
     for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
       Value offset = getValueOrCreateConstantIndexOp(
           builder, loc, subView.getMixedOffsets()[i]);
@@ -282,20 +282,20 @@ struct SubViewOpInterface
       Value dimSize = metadataOp.getSizes()[i];
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
       // Verify that slice does not run out-of-bounds.
-      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOne = arith::SubIOp::create(builder, loc, size, one);
       Value sizeMinusOneTimesStride =
-          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+          arith::MulIOp::create(builder, loc, sizeMinusOne, stride);
       Value lastPos =
-          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+          arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "subview runs out-of-bounds along dimension " +
@@ -316,7 +316,7 @@ struct ExpandShapeOpInterface
     for (const auto &it :
          llvm::enumerate(expandShapeOp.getReassociationIndices())) {
       Value srcDimSz =
-          builder.create<DimOp>(loc, expandShapeOp.getSrc(), it.index());
+          DimOp::create(builder, loc, expandShapeOp.getSrc(), it.index());
       int64_t groupSz = 1;
       bool foundDynamicDim = false;
       for (int64_t resultDim : it.value()) {
@@ -331,14 +331,14 @@ struct ExpandShapeOpInterface
         groupSz *= expandShapeOp.getResultType().getDimSize(resultDim);
       }
       Value staticResultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, groupSz);
+          arith::ConstantIndexOp::create(builder, loc, groupSz);
       // staticResultDimSz must divide srcDimSz evenly.
       Value mod =
-          builder.create<arith::RemSIOp>(loc, srcDimSz, staticResultDimSz);
-      Value isModZero = builder.create<arith::CmpIOp>(
+          arith::RemSIOp::create(builder, loc, srcDimSz, staticResultDimSz);
+      Value isModZero = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, mod,
-          builder.create<arith::ConstantIndexOp>(loc, 0));
-      builder.create<cf::AssertOp>(
+          arith::ConstantIndexOp::create(builder, loc, 0));
+      cf::AssertOp::create(builder,
           loc, isModZero,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "static result dims in reassoc group do not "
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index 5a36984c9013c..5fd0d3c8a7f1b 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -92,7 +92,7 @@ SmallVector<Value> mlir::mesh::getMixedAsValues(OpBuilder b,
       values.emplace_back(*(dyn++));
     } else {
       TypedAttr val = type == i64 ? b.getI64IntegerAttr(s) : b.getIndexAttr(s);
-      values.emplace_back(b.create<arith::ConstantOp>(loc, type, val));
+      values.emplace_back(arith::ConstantOp::create(b, loc, type, val));
     }
   }
   return values;
@@ -317,9 +317,9 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
 
   if (!newShardOp) {
     auto shardingOp =
-        builder.create<ShardingOp>(operandValue.getLoc(), sharding);
+        ShardingOp::create(builder, operandValue.getLoc(), sharding);
     newShardOp =
-        builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+        ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                                 /*annotate_for_users*/ false);
   }
   operandValue.replaceUsesWithIf(
@@ -331,7 +331,7 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
     return;
   }
 
-  auto newShardOp2 = builder.create<ShardOp>(operandValue.getLoc(), newShardOp,
+  auto newShardOp2 = ShardOp::create(builder, operandValue.getLoc(), newShardOp,
                                              newShardOp.getSharding(),
                                              /*annotate_for_users*/ true);
   newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
@@ -379,9 +379,9 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
 
   builder.setInsertionPoint(operandOp);
   auto shardingOp =
-      builder.create<ShardingOp>(operand.get().getLoc(), sharding);
+      ShardingOp::create(builder, operand.get().getLoc(), sharding);
   auto newShardOp =
-      builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+      ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                               /*annotate_for_users*/ true);
   IRRewriter rewriter(builder);
   rewriter.replaceUsesWithIf(
@@ -396,7 +396,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
 
   builder.setInsertionPoint(newShardOp);
   auto newPreceedingShardOp =
-      builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+      ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                               /*annotate_for_users*/ false);
   rewriter.replaceUsesWithIf(
       newShardOp.getSrc(), newPreceedingShardOp, [&newShardOp](OpOperand &use) {
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
index 4c14b1c0ea4bb..b5b3a4d21f2af 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
@@ -92,7 +92,7 @@ struct MeshShapeFolder
         newShapeOpMeshAxes.push_back(opMeshAxes[i]);
       } else {
         // Fold static mesh axes.
-        newResults[i] = builder.create<arith::ConstantOp>(
+        newResults[i] = arith::ConstantOp::create(builder,
             builder.getIndexAttr(meshAxisSize));
       }
     }
@@ -100,7 +100,7 @@ struct MeshShapeFolder
     // Leave only the dynamic mesh axes to be queried.
     if (!newShapeOpMeshAxes.empty()) {
       MeshShapeOp newShapeOp =
-          builder.create<MeshShapeOp>(mesh.getSymName(), newShapeOpMeshAxes);
+          MeshShapeOp::create(builder, mesh.getSymName(), newShapeOpMeshAxes);
       for (size_t i = 0; i < newShapeOp->getResults().size(); ++i) {
         newResults[newToOldResultsIndexMap[i]] = newShapeOp->getResults()[i];
       }
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index a0284b093da94..6476e8b689baa 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -267,7 +267,7 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
       targetShardingInUnsplitLastAxis(ctx, sourceSharding, splitTensorAxis);
   ShapedType allGatherResultShape = allGatherResultShapeInUnsplitLastAxis(
       sourceShard.getType(), mesh.getShape()[splitMeshAxis], splitTensorAxis);
-  Value allGatherResult = builder.create<AllGatherOp>(
+  Value allGatherResult = AllGatherOp::create(builder,
       RankedTensorType::get(allGatherResultShape.getShape(),
                             allGatherResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({splitMeshAxis}), sourceShard,
@@ -275,7 +275,7 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      builder.create<tensor::CastOp>(targetShape, allGatherResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allGatherResult).getResult());
   return {targetShard, targetSharding};
 }
 
@@ -400,7 +400,7 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType allToAllResultShape = allToAllResultShapeInMoveLastAxis(
       sourceShard.getType(), mesh.getShape()[meshAxis], sourceTensorAxis,
       targetTensorAxis);
-  Value allToAllResult = builder.create<AllToAllOp>(
+  Value allToAllResult = AllToAllOp::create(builder,
       RankedTensorType::get(allToAllResultShape.getShape(),
                             allToAllResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({meshAxis}), sourceShard,
@@ -408,7 +408,7 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      builder.create<tensor::CastOp>(targetShape, allToAllResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allToAllResult).getResult());
   return {targetShard, targetSharding};
 }
 
@@ -479,13 +479,13 @@ tryUpdateHaloInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
 
   // Extract core from source and copy into destination core.
   auto noVals = ValueRange{};
-  auto initVal = builder.create<tensor::EmptyOp>(
+  auto initVal = tensor::EmptyOp::create(builder,
       sourceShard.getLoc(), outShape, sourceShard.getType().getElementType());
-  auto core = builder.create<tensor::ExtractSliceOp>(
+  auto core = tensor::ExtractSliceOp::create(builder,
       sourceShard.getLoc(),
       RankedTensorType::get(coreShape, sourceShard.getType().getElementType()),
       sourceShard, noVals, noVals, noVals, srcCoreOffs, coreShape, strides);
-  auto initOprnd = builder.create<tensor::InsertSliceOp>(
+  auto initOprnd = tensor::InsertSliceOp::create(builder,
       sourceShard.getLoc(), core, initVal, noVals, noVals, noVals, tgtCoreOffs,
       coreShape, strides);
 
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
index f08ef75d8a004..c85394924608d 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
@@ -49,10 +49,10 @@ struct ProcessMultiIndexOpLowering
 
     ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     builder.setInsertionPointAfter(op.getOperation());
-    Value linearIndex = builder.create<ProcessLinearIndexOp>(mesh);
-    ValueRange meshShape = builder.create<MeshShapeOp>(mesh).getResults();
+    Value linearIndex = ProcessLinearIndexOp::create(builder, mesh);
+    ValueRange meshShape = MeshShapeOp::create(builder, mesh).getResults();
     SmallVector<Value> completeMultiIndex =
-        builder.create<affine::AffineDelinearizeIndexOp>(linearIndex, meshShape)
+        affine::AffineDelinearizeIndexOp::create(builder, linearIndex, meshShape)
             .getMultiIndex();
     SmallVector<Value> multiIndex;
     ArrayRef<MeshAxis> opMeshAxes = op.getAxes();
@@ -101,32 +101,32 @@ struct AllSliceOpLowering
     ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     builder.setInsertionPointAfter(op.getOperation());
 
-    Value zero = builder.create<arith::ConstantOp>(builder.getIndexAttr(0));
+    Value zero = arith::ConstantOp::create(builder, builder.getIndexAttr(0));
 
     Operation::result_range processInGroupMultiIndex =
-        builder.create<ProcessMultiIndexOp>(mesh.getSymName(), op.getMeshAxes())
+        ProcessMultiIndexOp::create(builder, mesh.getSymName(), op.getMeshAxes())
             .getResults();
 
     Operation::result_range processGroupShape =
-        builder.create<MeshShapeOp>(mesh.getSymName(), op.getMeshAxes())
+        MeshShapeOp::create(builder, mesh.getSymName(), op.getMeshAxes())
             .getResult();
     Value processGroupSize =
         createCollectiveProcessGroupSize(mesh, op.getMeshAxes(), builder);
 
     int64_t sliceAxis = op.getSliceAxis().getSExtValue();
     Value operandSliceAxisSize =
-        builder.create<tensor::DimOp>(op.getOperand(), sliceAxis);
+        tensor::DimOp::create(builder, op.getOperand(), sliceAxis);
     Value operandSliceAxisSizeModProcessGroupSize =
-        builder.create<arith::RemUIOp>(operandSliceAxisSize, processGroupSize);
-    Value isTargetShapeExactlyDivisible = builder.create<arith::CmpIOp>(
+        arith::RemUIOp::create(builder, operandSliceAxisSize, processGroupSize);
+    Value isTargetShapeExactlyDivisible = arith::CmpIOp::create(builder,
         arith::CmpIPredicate::eq, operandSliceAxisSizeModProcessGroupSize,
         zero);
-    builder.create<cf::AssertOp>(isTargetShapeExactlyDivisible,
+    cf::AssertOp::create(builder, isTargetShapeExactlyDivisible,
                                  "Slicing a tensor with axis size that is "
                                  "not exactly divisible by the "
                                  "mesh process group size is not supported.");
     Value resultSliceAxisSize =
-        builder.create<arith::DivUIOp>(operandSliceAxisSize, processGroupSize);
+        arith::DivUIOp::create(builder, operandSliceAxisSize, processGroupSize);
     OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
         llvm::to_vector_of<OpFoldResult>(processInGroupMultiIndex),
         llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
@@ -139,7 +139,7 @@ struct AllSliceOpLowering
       if (i == sliceAxis) {
         sizes.emplace_back(resultSliceAxisSize);
       } else {
-        Value dimSize = builder.create<tensor::DimOp>(op.getOperand(), i);
+        Value dimSize = tensor::DimOp::create(builder, op.getOperand(), i);
         sizes.emplace_back(dimSize);
       }
     }
@@ -152,10 +152,10 @@ struct AllSliceOpLowering
                  resultSliceAxisSize);
     SmallVector<OpFoldResult> strides(
         operandType.getRank(), getAsIndexOpFoldResult(builder.getContext(), 1));
-    Value slice = builder.create<tensor::ExtractSliceOp>(
+    Value slice = tensor::ExtractSliceOp::create(builder,
         op.getOperand(), offsets, sizes, strides);
     Value newResult =
-        builder.create<tensor::CastOp>(op.getResult().getType(), slice);
+        tensor::CastOp::create(builder, op.getResult().getType(), slice);
     rewriter.replaceAllUsesWith(op.getResult(), newResult);
 
     return success();
@@ -201,7 +201,7 @@ TypedValue<IndexType>
 createCollectiveProcessGroupSize(MeshOp mesh, ArrayRef<MeshAxis> axes,
                                  ImplicitLocOpBuilder &builder) {
   Operation::result_range meshShape =
-      builder.create<mesh::MeshShapeOp>(mesh, axes).getResults();
+      mesh::MeshShapeOp::create(builder, mesh, axes).getResults();
   return cast<TypedValue<IndexType>>(arith::createProduct(
       builder, builder.getLoc(), llvm::to_vector_of<Value>(meshShape),
       builder.getIndexType()));
@@ -212,13 +212,13 @@ createProcessLinearIndex(StringRef mesh, ValueRange processInGroupMultiIndex,
                          ArrayRef<MeshAxis> meshAxes,
                          ImplicitLocOpBuilder &builder) {
   Operation::result_range processGroupShape =
-      builder.create<MeshShapeOp>(mesh, meshAxes).getResult();
+      MeshShapeOp::create(builder, mesh, meshAxes).getResult();
   OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
       llvm::to_vector_of<OpFoldResult>(processInGroupMultiIndex),
       llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
   auto res = dyn_cast<Value>(processInGroupLinearIndex);
   if (!res)
-    res = builder.create<arith::ConstantIndexOp>(
+    res = arith::ConstantIndexOp::create(builder,
         cast<IntegerAttr>(cast<Attribute>(processInGroupLinearIndex)).getInt());
   return cast<TypedValue<IndexType>>(res);
 }
@@ -227,7 +227,7 @@ TypedValue<IndexType> createProcessLinearIndex(StringRef mesh,
                                                ArrayRef<MeshAxis> meshAxes,
                                                ImplicitLocOpBuilder &builder) {
   return createProcessLinearIndex(
-      mesh, builder.create<ProcessMultiIndexOp>(mesh, meshAxes).getResults(),
+      mesh, ProcessMultiIndexOp::create(builder, mesh, meshAxes).getResults(),
       meshAxes, builder);
 }
 } // namespace mlir::mesh
diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
index d2c94b124cdfb..85317a68c9f00 100644
--- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
+++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
@@ -334,13 +334,13 @@ static Operation *replaceOpWithPredicatedOp(RewriterBase &rewriter,
   //
   Location loc = asyncCopyOp->getLoc();
   Value dstElements =
-      rewriter.create<arith::ConstantOp>(loc, asyncCopyOp.getDstElementsAttr());
+      arith::ConstantOp::create(rewriter, loc, asyncCopyOp.getDstElementsAttr());
   Value originalSrcElement =
       asyncCopyOp.getSrcElements() ? asyncCopyOp.getSrcElements() : dstElements;
-  Value c0Index = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto srcElements = rewriter.create<arith::SelectOp>(
+  Value c0Index = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto srcElements = arith::SelectOp::create(rewriter,
       loc, predicate, originalSrcElement, c0Index);
-  auto asyncCopyZeroFillOp = rewriter.create<nvgpu::DeviceAsyncCopyOp>(
+  auto asyncCopyZeroFillOp = nvgpu::DeviceAsyncCopyOp::create(rewriter,
       loc, nvgpu::DeviceAsyncTokenType::get(asyncCopyOp.getContext()),
       asyncCopyOp.getDst(), asyncCopyOp.getDstIndices(), asyncCopyOp.getSrc(),
       asyncCopyOp.getSrcIndices(), asyncCopyOp.getDstElements(), srcElements,
@@ -675,7 +675,7 @@ MmaSyncBuilder::buildMemRefLoads(OpBuilder &b, Location loc,
   for (auto indexing : indexings) {
     Value row = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.row()));
     Value col = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.col()));
-    auto load = b.create<memref::LoadOp>(loc, memref, ValueRange{row, col});
+    auto load = memref::LoadOp::create(b, loc, memref, ValueRange{row, col});
     res.push_back(load);
   }
   return res;
@@ -688,7 +688,7 @@ Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
 
   Type elementType = getElementTypeOrSelf(memref.getType());
   auto vt = VectorType::get(vectorShape, elementType);
-  Value res = b.create<vector::SplatOp>(loc, vt, loads[0]);
+  Value res = vector::SplatOp::create(b, loc, vt, loads[0]);
   foreachIndividualVectorElement(
       res,
       /*applyFn=*/
@@ -697,7 +697,7 @@ Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
       },
       /*reduceFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
-        res = b.create<vector::InsertOp>(loc, v, res, indices);
+        res = vector::InsertOp::create(b, loc, v, res, indices);
       });
 
   return res;
@@ -715,7 +715,7 @@ SmallVector<Operation *> MmaSyncBuilder::buildMemRefStores(
     Value row = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.row()));
     Value col = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.col()));
     Operation *store =
-        b.create<memref::StoreOp>(loc, val, memref, ValueRange{row, col});
+        memref::StoreOp::create(b, loc, val, memref, ValueRange{row, col});
     res.push_back(store);
   }
   return res;
@@ -730,7 +730,7 @@ SmallVector<Operation *> MmaSyncBuilder::buildMmaSyncMemRefStoreOperand(
       vectorToStore,
       /*applyFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
-        return b.create<vector::ExtractOp>(loc, vectorToStore, indices);
+        return vector::ExtractOp::create(b, loc, vectorToStore, indices);
       },
       /*reduceFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
@@ -810,7 +810,7 @@ FailureOr<Operation *> MmaSyncBuilder::buildMmaSync(LinalgOp linalgOp) {
                                             rhsIndexFn, rhsShape);
   Value res = buildMmaSyncMemRefLoadOperand(b, loc, laneId, resMemRef,
                                             resIndexFn, resShape);
-  res = b.create<nvgpu::MmaSyncOp>(loc, lhs, rhs, res, info.mmaShape,
+  res = nvgpu::MmaSyncOp::create(b, loc, lhs, rhs, res, info.mmaShape,
                                    info.tf32Enabled);
   buildMmaSyncMemRefStoreOperand(b, loc, res, laneId, resMemRef, resIndexFn,
                                  resShape);
@@ -832,7 +832,7 @@ DiagnosedSilenceableFailure transform::RewriteMatmulAsMmaSyncOp::applyToOne(
     }
     Location loc = linalgOp.getLoc();
     // TODO: more robust computation of laneId, for now assume a single warp.
-    Value laneId = rewriter.create<gpu::ThreadIdOp>(
+    Value laneId = gpu::ThreadIdOp::create(rewriter,
         loc, rewriter.getIndexType(), gpu::Dimension::x);
     if (succeeded(MmaSyncBuilder(rewriter, loc, laneId).buildMmaSync(linalgOp)))
       fail = false;
@@ -897,12 +897,12 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
     ArrayRef<TypedValue<MemRefType>> sharedMemBuffers,
     TypedValue<nvgpu::MBarrierGroupType> barrier) {
   SmallVector<Operation *> loadOps;
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Value tidx = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Value tidx = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
   Value cond =
-      rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, tidx, zero);
+      arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, tidx, zero);
   // clang-format off
-  rewriter.create<scf::IfOp>(
+  scf::IfOp::create(rewriter,
     /*location=*/loc,
     /*conditional=*/cond,
     /*thenBuilder=*/
@@ -917,14 +917,14 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
       // TODO: Note that cutlass predeclares the barrier arrive tx before the tma.async.load.
       // This may or may not have perf implications.
       buildBarrierArriveTx(barrier, sizes);
-      rewriter.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(rewriter, loc);
     },
     /*elseBuilder=*/
     [&](OpBuilder &lb, Location loc) {
       // TODO: is this for no-thread divergence?
       // Should we just yield the size and hoist?
       buildBarrierArriveTx(barrier, getAsIndexOpFoldResult(rewriter.getContext(), 0));
-      rewriter.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(rewriter, loc);
     });
   // clang-format on
   return loadOps;
@@ -939,14 +939,14 @@ static Attribute getSharedAddressSpaceAttribute(OpBuilder &b) {
 TypedValue<nvgpu::MBarrierGroupType>
 HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) {
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value barrier = rewriter.create<nvgpu::MBarrierCreateOp>(
+  Value barrier = nvgpu::MBarrierCreateOp::create(rewriter,
       loc,
       nvgpu::MBarrierGroupType::get(rewriter.getContext(), sharedMemorySpace));
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierInitOp>(
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierInitOp::create(rewriter,
       loc, barrier, getValueOrCreateConstantIndexOp(rewriter, loc, numThreads),
       zero, Value());
-  rewriter.create<gpu::BarrierOp>(loc);
+  gpu::BarrierOp::create(rewriter, loc);
   return cast<TypedValue<nvgpu::MBarrierGroupType>>(barrier);
 }
 
@@ -955,7 +955,7 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
                                            gpu::LaunchOp launchOp) {
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(launchOp);
-  Value unrankedMemRef = rewriter.create<memref::CastOp>(
+  Value unrankedMemRef = memref::CastOp::create(rewriter,
       loc,
       UnrankedMemRefType::get(memref.getType().getElementType(),
                               memref.getType().getMemorySpace()),
@@ -966,7 +966,7 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
       getValueOrCreateConstantIndexOp(rewriter, loc, mixedSizes);
 
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value desc = rewriter.create<nvgpu::TmaCreateDescriptorOp>(
+  Value desc = nvgpu::TmaCreateDescriptorOp::create(rewriter,
       loc,
       nvgpu::TensorMapDescriptorType::get(
           rewriter.getContext(),
@@ -985,8 +985,8 @@ OpFoldResult HopperBuilder::buildTmaAsyncLoad(
     TypedValue<nvgpu::MBarrierGroupType> barrier,
     SmallVectorImpl<Operation *> &loadOps) {
   MLIRContext *ctx = rewriter.getContext();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Operation *loadOp = rewriter.create<nvgpu::TmaAsyncLoadOp>(
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Operation *loadOp = nvgpu::TmaAsyncLoadOp::create(rewriter,
       loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero,
       Value(), Value());
   loadOps.push_back(loadOp);
@@ -1012,22 +1012,22 @@ void HopperBuilder::buildBarrierArriveTx(
   OpFoldResult size =
       affine::makeComposedFoldedAffineApply(rewriter, loc, sumExpr, mixedSizes);
   Value sizeVal = getValueOrCreateConstantIndexOp(rewriter, loc, size);
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierArriveExpectTxOp>(loc, barrier, sizeVal, zero,
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierArriveExpectTxOp::create(rewriter, loc, barrier, sizeVal, zero,
                                                    Value());
 }
 
 void HopperBuilder::buildTryWaitParity(
     TypedValue<nvgpu::MBarrierGroupType> barrier) {
   Type i1 = rewriter.getI1Type();
-  Value parity = rewriter.create<LLVM::ConstantOp>(loc, i1, 0);
+  Value parity = LLVM::ConstantOp::create(rewriter, loc, i1, 0);
   // 10M is an arbitrary, not too small or too big number to specify the number
   // of ticks before retry.
   // TODO: hoist this in a default dialect constant.
   Value ticksBeforeRetry =
-      rewriter.create<arith::ConstantIndexOp>(loc, 10000000);
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierTryWaitParityOp>(loc, barrier, parity,
+      arith::ConstantIndexOp::create(rewriter, loc, 10000000);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierTryWaitParityOp::create(rewriter, loc, barrier, parity,
                                                   ticksBeforeRetry, zero);
 }
 
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
index 10bc1993ffd96..6a167b1cb157b 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
@@ -110,17 +110,17 @@ static Value buildNumReadElements(OpBuilder &b, Location loc,
   for (auto [pos, sz] : llvm::zip(transferMask->extractPosition,
                                   transferMask->createMaskOp->getOperands())) {
     Value cmp =
-        b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
-                                b.create<arith::ConstantIndexOp>(loc, pos), sz);
+        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt,
+                                arith::ConstantIndexOp::create(b, loc, pos), sz);
     if (!cond) {
       cond = cmp;
       continue;
     }
-    cond = b.create<arith::AndIOp>(loc, cmp, cond);
+    cond = arith::AndIOp::create(b, loc, cmp, cond);
   }
-  return b.create<arith::SelectOp>(
+  return arith::SelectOp::create(b,
       loc, cond, transferMask->createMaskOp->getOperands().back(),
-      b.create<arith::ConstantIndexOp>(loc, 0));
+      arith::ConstantIndexOp::create(b, loc, 0));
 }
 
 /// Return "true" if the conversion to async copy is supported by "async copy".
@@ -252,7 +252,7 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
       int64_t sizeInBytes =
           (dstMemref.getElementTypeBitWidth() * numElements) / 8;
       // bypass_l1 only possible with 16 byte transfer.
-      Value token = rewriter.create<nvgpu::DeviceAsyncCopyOp>(
+      Value token = nvgpu::DeviceAsyncCopyOp::create(rewriter,
           writeOp->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
           /*dst=*/storeBase, /*dstIndices=*/nvgpu::getIndices(writeOp),
           /*src=*/loadBase,
@@ -265,10 +265,10 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
     }
 
     // Create the group and wait for it right after.
-    Value groupToken = rewriter.create<nvgpu::DeviceAsyncCreateGroupOp>(
+    Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(rewriter,
         op->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
         tokens);
-    rewriter.create<nvgpu::DeviceAsyncWaitOp>(op->getLoc(), groupToken,
+    nvgpu::DeviceAsyncWaitOp::create(rewriter, op->getLoc(), groupToken,
                                               nullptr);
     // Clean up old stores.
     for (Operation *writeOp : group)
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
index 72f7296a865f8..3c6b189606c6f 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
@@ -75,27 +75,27 @@ static Value permuteVectorOffset(OpBuilder &b, Location loc,
   int64_t mask = (1LL << (m - n)) - 1;
   if (permuteEveryN > 1)
     mask = mask << llvm::Log2_64(permuteEveryN);
-  Value srcBits = b.create<arith::ConstantIndexOp>(loc, mask);
-  srcBits = b.create<arith::AndIOp>(loc, src, srcBits);
+  Value srcBits = arith::ConstantIndexOp::create(b, loc, mask);
+  srcBits = arith::AndIOp::create(b, loc, src, srcBits);
 
   // Use the src bits to permute the target bits b[N:M] containing the
   // vector offset.
   if (permuteEveryN > 1) {
     int64_t shlBits = n - llvm::Log2_64(permuteEveryN);
     if (shlBits > 0) {
-      Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, shlBits);
+      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, shlBits);
       srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
     } else if (shlBits < 0) {
-      Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, -1 * shlBits);
+      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, -1 * shlBits);
       srcBits = b.createOrFold<arith::ShRUIOp>(loc, srcBits, finalShiftVal);
     }
   } else {
-    Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, n);
+    Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, n);
     srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
   }
 
   Value permutedVectorIdx =
-      b.create<arith::XOrIOp>(loc, indices[tgtDim], srcBits);
+      arith::XOrIOp::create(b, loc, indices[tgtDim], srcBits);
   return permutedVectorIdx;
 }
 
diff --git a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
index 793db73575b4f..a378b09c27c41 100644
--- a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
@@ -72,7 +72,7 @@ Value getScalarOrTensorConstant(OpBuilder &builder, Location loc, Value scalar,
 
   // Create tensor splat
   auto tensorConstant =
-      builder.create<tensor::SplatOp>(loc, scalar, referenceShape);
+      tensor::SplatOp::create(builder, loc, scalar, referenceShape);
   return tensorConstant;
 }
 
@@ -94,21 +94,21 @@ std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
   // Get unranked input shape and total size
   auto *context = builder.getContext();
   auto shapeType = shape::getExtentTensorType(context);
-  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
-  Value inputSize = builder.create<shape::NumElementsOp>(
+  auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
+  Value inputSize = shape::NumElementsOp::create(builder,
       loc, builder.getIndexType(), inputShape);
 
   // Turn input size into 1D tensor
   auto flatShapeType = shape::getExtentTensorType(context, 1);
   auto flatInputShape =
-      builder.create<tensor::FromElementsOp>(loc, flatShapeType, inputSize);
+      tensor::FromElementsOp::create(builder, loc, flatShapeType, inputSize);
 
   // Reshape input tensor into 1D
   auto inputType = cast<UnrankedTensorType>(input.getType());
   auto elementType = inputType.getElementType();
   auto flatInputType =
       RankedTensorType::get({ShapedType::kDynamic}, elementType);
-  auto flatInput = builder.create<tensor::ReshapeOp>(loc, flatInputType, input,
+  auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
                                                      flatInputShape);
   return std::make_pair(flatInput, inputShape);
 }
@@ -142,30 +142,30 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto *context = builder.getContext();
   auto indexType = builder.getIndexType();
   auto shapeType = shape::getExtentTensorType(context);
-  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
+  auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
 
   // Get shape and sizes on left and right of axis
-  auto axisValue = builder.create<arith::ConstantIndexOp>(loc, axis);
-  auto axisNextValue = builder.create<arith::ConstantIndexOp>(loc, axis + 1);
+  auto axisValue = arith::ConstantIndexOp::create(builder, loc, axis);
+  auto axisNextValue = arith::ConstantIndexOp::create(builder, loc, axis + 1);
   auto shapeLeft =
       builder
           .create<shape::SplitAtOp>(loc, TypeRange{shapeType, shapeType},
                                     inputShape, axisValue)
           .getResult(0);
   auto sizeLeft =
-      builder.create<shape::NumElementsOp>(loc, indexType, shapeLeft);
+      shape::NumElementsOp::create(builder, loc, indexType, shapeLeft);
   auto shapeRight =
       builder
           .create<shape::SplitAtOp>(loc, TypeRange{shapeType, shapeType},
                                     inputShape, axisNextValue)
           .getResult(1);
   auto sizeRight =
-      builder.create<shape::NumElementsOp>(loc, indexType, shapeRight);
+      shape::NumElementsOp::create(builder, loc, indexType, shapeRight);
 
   // Compute flat input shape as a 3-element 1D tensor
-  auto axisSizeValue = builder.create<arith::ConstantIndexOp>(loc, axisSize);
+  auto axisSizeValue = arith::ConstantIndexOp::create(builder, loc, axisSize);
   auto flatShapeType = shape::getExtentTensorType(context, 3);
-  auto flatInputShape = builder.create<tensor::FromElementsOp>(
+  auto flatInputShape = tensor::FromElementsOp::create(builder,
       loc, flatShapeType, ValueRange{sizeLeft, axisSizeValue, sizeRight});
 
   // Reshape input to 3D tensor
@@ -173,7 +173,7 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto elementType = inputType.getElementType();
   auto flatInputType = RankedTensorType::get(
       {ShapedType::kDynamic, axisSize, ShapedType::kDynamic}, elementType);
-  auto flatInput = builder.create<tensor::ReshapeOp>(loc, flatInputType, input,
+  auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
                                                      flatInputShape);
 
   return std::make_pair(flatInput, inputShape);
@@ -192,7 +192,7 @@ Value restoreUnrankedTensorShape(OpBuilder &builder, Location loc, Value input,
   auto inputType = cast<RankedTensorType>(input.getType());
   auto elementType = inputType.getElementType();
   auto unrankedType = UnrankedTensorType::get(elementType);
-  return builder.create<tensor::ReshapeOp>(loc, unrankedType, input,
+  return tensor::ReshapeOp::create(builder, loc, unrankedType, input,
                                            inputShape);
 }
 
@@ -215,7 +215,7 @@ Value materializePerChannelScales(OpBuilder &builder, Location loc,
   auto tensorType =
       RankedTensorType::get({(int64_t)scales.size()}, expressedType);
   auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, scalesAttr);
 }
 
 // Create a tensor constant containing all zero points in a per-channel
@@ -239,7 +239,7 @@ Value materializePerChannelZeroPoints(
   auto tensorType =
       RankedTensorType::get({(int64_t)zeroPoints.size()}, storageType);
   auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, zeroPointsAttr);
 }
 
 // Create a tensor constant containing all scales in a sub-channel quantized
@@ -263,7 +263,7 @@ Value materializeSubChannelScales(
   auto tensorType =
       RankedTensorType::get(scales.getType().getShape(), expressedType);
   auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, scalesAttr);
 }
 
 // Create a tensor constant containing all zero points in a sub-channel
@@ -287,7 +287,7 @@ Value materializeSubChannelZeroPoints(
   auto tensorType =
       RankedTensorType::get(zeroPoints.getType().getShape(), storageType);
   auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, zeroPointsAttr);
 }
 
 // Clamp the given scalar or tensor input using the storage bounds encoded in
@@ -314,9 +314,9 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
   // Materialize bounds
   auto inputType = input.getType();
   auto storageType = quantizedType.getStorageType();
-  auto storageMinScalar = builder.create<arith::ConstantIntOp>(
+  auto storageMinScalar = arith::ConstantIntOp::create(builder,
       loc, storageType, quantizedType.getStorageTypeMin());
-  auto storageMaxScalar = builder.create<arith::ConstantIntOp>(
+  auto storageMaxScalar = arith::ConstantIntOp::create(builder,
       loc, storageType, quantizedType.getStorageTypeMax());
   auto storageMin = getScalarOrTensorConstant(builder, loc, storageMinScalar,
                                               inputType, inputShape);
@@ -325,11 +325,11 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
 
   // Clamp
   if (quantizedType.isSigned()) {
-    input = builder.create<arith::MaxSIOp>(loc, input, storageMin);
-    input = builder.create<arith::MinSIOp>(loc, input, storageMax);
+    input = arith::MaxSIOp::create(builder, loc, input, storageMin);
+    input = arith::MinSIOp::create(builder, loc, input, storageMax);
   } else {
-    input = builder.create<arith::MaxUIOp>(loc, input, storageMin);
-    input = builder.create<arith::MinUIOp>(loc, input, storageMax);
+    input = arith::MaxUIOp::create(builder, loc, input, storageMin);
+    input = arith::MinUIOp::create(builder, loc, input, storageMax);
   }
   return input;
 }
@@ -338,16 +338,16 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
 Value convertFloatToInteger(OpBuilder &builder, Location loc, Value input,
                             Type resultType, bool isSigned) {
   if (isSigned)
-    return builder.create<arith::FPToSIOp>(loc, resultType, input);
-  return builder.create<arith::FPToUIOp>(loc, resultType, input);
+    return arith::FPToSIOp::create(builder, loc, resultType, input);
+  return arith::FPToUIOp::create(builder, loc, resultType, input);
 }
 
 // Emit op 'arith.sitofp' or 'arith.uitofp'.
 Value convertIntegerToFloat(OpBuilder &builder, Location loc, Value input,
                             Type resultType, bool isSigned) {
   if (isSigned)
-    return builder.create<arith::SIToFPOp>(loc, resultType, input);
-  return builder.create<arith::UIToFPOp>(loc, resultType, input);
+    return arith::SIToFPOp::create(builder, loc, resultType, input);
+  return arith::UIToFPOp::create(builder, loc, resultType, input);
 }
 
 // Quantize a scalar or ranked tensor value. The stored value is clamped using
@@ -362,7 +362,7 @@ Value quantizeValue(OpBuilder &builder, Location loc, Value input,
   scale = getScalarOrTensorConstant(builder, loc, scale, inputType, inputShape);
 
   // Scale input
-  auto scaledValue = builder.create<arith::DivFOp>(loc, input, scale);
+  auto scaledValue = arith::DivFOp::create(builder, loc, input, scale);
 
   // Skip unnecessary computations if no zero point is given
   Value storedValueFloat = scaledValue;
@@ -377,7 +377,7 @@ Value quantizeValue(OpBuilder &builder, Location loc, Value input,
 
     // Add zero point to stored value
     storedValueFloat =
-        builder.create<arith::AddFOp>(loc, scaledValue, zeroPoint);
+        arith::AddFOp::create(builder, loc, scaledValue, zeroPoint);
   }
 
   // Convert stored value to storage type
@@ -418,11 +418,11 @@ Value dequantizeValue(OpBuilder &builder, Location loc, Value input,
                                       quantizedType.isSigned());
 
     // Subtract zero point to stored value
-    result = builder.create<arith::SubFOp>(loc, result, zeroPoint);
+    result = arith::SubFOp::create(builder, loc, result, zeroPoint);
   }
 
   // Multiply by scale
-  result = builder.create<arith::MulFOp>(loc, result, scale);
+  result = arith::MulFOp::create(builder, loc, result, scale);
   return result;
 }
 
@@ -477,11 +477,11 @@ Value convertPerLayerRanked(OpBuilder &builder, Location loc, Operation *op,
   auto storageType = quantizedType.getStorageType();
   auto scaleAttr =
       builder.getFloatAttr(expressedType, quantizedType.getScale());
-  auto scale = builder.create<arith::ConstantOp>(loc, expressedType, scaleAttr);
+  auto scale = arith::ConstantOp::create(builder, loc, expressedType, scaleAttr);
   auto zeroPointAttr =
       builder.getIntegerAttr(storageType, quantizedType.getZeroPoint());
   auto zeroPoint =
-      builder.create<arith::ConstantOp>(loc, storageType, zeroPointAttr);
+      arith::ConstantOp::create(builder, loc, storageType, zeroPointAttr);
 
   auto inputShape = getScalarOrTensorShape(builder, loc, input);
   return convertRanked(builder, loc, op, input, inputShape, scale, zeroPoint,
@@ -546,7 +546,7 @@ Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
                          ? quantizedType.getStorageType()
                          : quantizedType.getExpressedType();
   auto initShape = tensor::getMixedSizes(builder, loc, input);
-  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+  Value init = tensor::EmptyOp::create(builder, loc, initShape, elementType);
 
   SmallVector<utils::IteratorType> iteratorTypes(inputRank,
                                                  utils::IteratorType::parallel);
@@ -572,7 +572,7 @@ Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
                               convertRanked(builder, loc, op, input, {}, scale,
                                             zeroPoint, quantizedType);
 
-                          builder.create<linalg::YieldOp>(loc, result);
+                          linalg::YieldOp::create(builder, loc, result);
                         })
                     .getResult(0);
 
@@ -642,7 +642,7 @@ Value convertSubChannel(OpBuilder &builder, Location loc, Operation *op,
                          ? quantizedType.getStorageType()
                          : quantizedType.getExpressedType();
   auto initShape = tensor::getMixedSizes(builder, loc, input);
-  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+  Value init = tensor::EmptyOp::create(builder, loc, initShape, elementType);
 
   SmallVector<utils::IteratorType> iteratorTypes(inputRank,
                                                  utils::IteratorType::parallel);
@@ -675,7 +675,7 @@ Value convertSubChannel(OpBuilder &builder, Location loc, Operation *op,
                               convertRanked(builder, loc, op, input, {}, scale,
                                             zeroPoint, quantizedType);
 
-                          builder.create<linalg::YieldOp>(loc, result);
+                          linalg::YieldOp::create(builder, loc, result);
                         })
                     .getResult(0);
 
@@ -729,7 +729,7 @@ struct DequantizeCastOpConversion
     // Convert quantized input to storage type
     auto storageScalarOrTensorType =
         getScalarOrTensorType(quantizedType.getStorageType(), input.getType());
-    input = rewriter.create<quant::StorageCastOp>(
+    input = quant::StorageCastOp::create(rewriter,
         loc, storageScalarOrTensorType, input);
 
     auto result = convertQuantized(rewriter, loc, op, input, quantizedType);
diff --git a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
index 4009faa21576d..f5e0aef365ff7 100644
--- a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
@@ -46,7 +46,7 @@ class QuantizedTypeConverter : public TypeConverter {
 
   static Value materializeConversion(OpBuilder &builder, Type type,
                                      ValueRange inputs, Location loc) {
-    return builder.create<quant::StorageCastOp>(loc, type,
+    return quant::StorageCastOp::create(builder, loc, type,
                                                 llvm::getSingleElement(inputs));
   }
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 5a3bd984530db..82fa50a6ac6e0 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -85,7 +85,7 @@ void SCFDialect::initialize() {
 
 /// Default callback for IfOp builders. Inserts a yield without arguments.
 void mlir::scf::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<scf::YieldOp>(loc);
+  scf::YieldOp::create(builder, loc);
 }
 
 /// Verifies that the first block of the given `region` is terminated by a
@@ -241,12 +241,12 @@ struct MultiBlockExecuteInliner : public OpRewritePattern<ExecuteRegionOp> {
     Block *postBlock = rewriter.splitBlock(prevBlock, op->getIterator());
     rewriter.setInsertionPointToEnd(prevBlock);
 
-    rewriter.create<cf::BranchOp>(op.getLoc(), &op.getRegion().front());
+    cf::BranchOp::create(rewriter, op.getLoc(), &op.getRegion().front());
 
     for (Block &blk : op.getRegion()) {
       if (YieldOp yieldOp = dyn_cast<YieldOp>(blk.getTerminator())) {
         rewriter.setInsertionPoint(yieldOp);
-        rewriter.create<cf::BranchOp>(yieldOp.getLoc(), postBlock,
+        cf::BranchOp::create(rewriter, yieldOp.getLoc(), postBlock,
                                       yieldOp.getResults());
         rewriter.eraseOp(yieldOp);
       }
@@ -557,7 +557,7 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
       [](OpBuilder &, Location, Value, ValueRange) {});
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
@@ -673,7 +673,7 @@ void mlir::scf::promote(RewriterBase &rewriter, scf::ForallOp forallOp) {
     Value dst = parallelInsertSliceOp.getDest();
     Value src = parallelInsertSliceOp.getSource();
     if (llvm::isa<TensorType>(src.getType())) {
-      results.push_back(rewriter.create<tensor::InsertSliceOp>(
+      results.push_back(tensor::InsertSliceOp::create(rewriter,
           forallOp.getLoc(), dst.getType(), src, dst,
           parallelInsertSliceOp.getOffsets(), parallelInsertSliceOp.getSizes(),
           parallelInsertSliceOp.getStrides(),
@@ -722,7 +722,7 @@ LoopNest mlir::scf::buildLoopNest(
   ValueRange currentIterArgs = iterArgs;
   Location currentLoc = loc;
   for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
-    auto loop = builder.create<scf::ForOp>(
+    auto loop = scf::ForOp::create(builder,
         currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
             ValueRange args) {
@@ -742,7 +742,7 @@ LoopNest mlir::scf::buildLoopNest(
   // For all loops but the innermost, yield the results of the nested loop.
   for (unsigned i = 0, e = loops.size() - 1; i < e; ++i) {
     builder.setInsertionPointToEnd(loops[i].getBody());
-    builder.create<scf::YieldOp>(loc, loops[i + 1].getResults());
+    scf::YieldOp::create(builder, loc, loops[i + 1].getResults());
   }
 
   // In the body of the innermost loop, call the body building function if any
@@ -756,7 +756,7 @@ LoopNest mlir::scf::buildLoopNest(
          "loop nest body must return as many values as loop has iteration "
          "arguments");
   builder.setInsertionPointToEnd(loops.back().getBody());
-  builder.create<scf::YieldOp>(loc, results);
+  scf::YieldOp::create(builder, loc, results);
 
   // Return the loops.
   ValueVector nestResults;
@@ -801,7 +801,7 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   }
 
   // 2. Create the new forOp shell.
-  scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+  scf::ForOp newForOp = scf::ForOp::create(rewriter,
       forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
       forOp.getStep(), newIterOperands);
   newForOp->setAttrs(forOp->getAttrs());
@@ -831,7 +831,7 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
                          clonedYieldOp.getOperand(yieldIdx));
   SmallVector<Value> newYieldOperands = clonedYieldOp.getOperands();
   newYieldOperands[yieldIdx] = castOut;
-  rewriter.create<scf::YieldOp>(newForOp.getLoc(), newYieldOperands);
+  scf::YieldOp::create(rewriter, newForOp.getLoc(), newYieldOperands);
   rewriter.eraseOp(clonedYieldOp);
 
   // 6. Inject an outgoing cast op after the forOp.
@@ -926,7 +926,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     if (!canonicalize)
       return failure();
 
-    scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+    scf::ForOp newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), newIterArgs);
     newForOp->setAttrs(forOp->getAttrs());
@@ -970,7 +970,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
       for (unsigned idx = 0, e = keepMask.size(); idx < e; ++idx)
         if (keepMask[idx])
           filteredOperands.push_back(mergedTerminator.getOperand(idx));
-      rewriter.create<scf::YieldOp>(mergedTerminator.getLoc(),
+      scf::YieldOp::create(rewriter, mergedTerminator.getLoc(),
                                     filteredOperands);
     };
 
@@ -1111,7 +1111,7 @@ struct ForOpTensorCastFolder : public OpRewritePattern<ForOp> {
           op, replaceAndCastForOpIterArg(
                   rewriter, op, iterOpOperand, incomingCast.getSource(),
                   [](OpBuilder &b, Location loc, Type type, Value source) {
-                    return b.create<tensor::CastOp>(loc, type, source);
+                    return tensor::CastOp::create(b, loc, type, source);
                   }));
       return success();
     }
@@ -1685,7 +1685,7 @@ struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
 
     // Step 3. Create a new scf.forall op with the new shared_outs' operands
     //         fetched earlier
-    auto newForallOp = rewriter.create<scf::ForallOp>(
+    auto newForallOp = scf::ForallOp::create(rewriter,
         forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
         forallOp.getMapping(),
@@ -1782,7 +1782,7 @@ struct ForallOpSingleOrZeroIterationDimsFolder
 
     // Replace the loop by a lower-dimensional loop.
     ForallOp newOp;
-    newOp = rewriter.create<ForallOp>(loc, newMixedLowerBounds,
+    newOp = ForallOp::create(rewriter, loc, newMixedLowerBounds,
                                       newMixedUpperBounds, newMixedSteps,
                                       op.getOutputs(), std::nullopt, nullptr);
     newOp.getBodyRegion().getBlocks().clear();
@@ -1867,7 +1867,7 @@ struct FoldTensorCastOfOutputIntoForallOp
 
     // Create new loop.
     Location loc = forallOp.getLoc();
-    auto newForallOp = rewriter.create<ForallOp>(
+    auto newForallOp = ForallOp::create(rewriter,
         loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
         forallOp.getMixedStep(), newOutputTensors, forallOp.getMapping(),
         [&](OpBuilder nestedBuilder, Location nestedLoc, ValueRange bbArgs) {
@@ -1875,7 +1875,7 @@ struct FoldTensorCastOfOutputIntoForallOp
               llvm::to_vector(bbArgs.take_back(forallOp->getNumResults()));
           for (auto [index, cast] : tensorCastProducers) {
             Value &oldTypeBBArg = castBlockArgs[index];
-            oldTypeBBArg = nestedBuilder.create<tensor::CastOp>(
+            oldTypeBBArg = tensor::CastOp::create(nestedBuilder,
                 nestedLoc, cast.dstType, oldTypeBBArg);
           }
 
@@ -1902,7 +1902,7 @@ struct FoldTensorCastOfOutputIntoForallOp
     SmallVector<Value> castResults = newForallOp.getResults();
     for (auto &item : tensorCastProducers) {
       Value &oldTypeResult = castResults[item.first];
-      oldTypeResult = rewriter.create<tensor::CastOp>(loc, item.second.dstType,
+      oldTypeResult = tensor::CastOp::create(rewriter, loc, item.second.dstType,
                                                       oldTypeResult);
     }
     rewriter.replaceOp(forallOp, castResults);
@@ -2309,7 +2309,7 @@ struct RemoveUnusedResults : public OpRewritePattern<IfOp> {
 
     // Create a replacement operation with empty then and else regions.
     auto newOp =
-        rewriter.create<IfOp>(op.getLoc(), newTypes, op.getCondition());
+        IfOp::create(rewriter, op.getLoc(), newTypes, op.getCondition());
     rewriter.createBlock(&newOp.getThenRegion());
     rewriter.createBlock(&newOp.getElseRegion());
 
@@ -2372,7 +2372,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
     if (nonHoistable.size() == op->getNumResults())
       return failure();
 
-    IfOp replacement = rewriter.create<IfOp>(op.getLoc(), nonHoistable, cond,
+    IfOp replacement = IfOp::create(rewriter, op.getLoc(), nonHoistable, cond,
                                              /*withElseRegion=*/false);
     if (replacement.thenBlock())
       rewriter.eraseBlock(replacement.thenBlock());
@@ -2398,7 +2398,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       } else if (trueVal == falseVal)
         results[it.index()] = trueVal;
       else
-        results[it.index()] = rewriter.create<arith::SelectOp>(
+        results[it.index()] = arith::SelectOp::create(rewriter,
             op.getLoc(), cond, trueVal, falseVal);
     }
 
@@ -2450,7 +2450,7 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantTrue)
-          constantTrue = rewriter.create<arith::ConstantOp>(
+          constantTrue = arith::ConstantOp::create(rewriter,
               op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
 
         rewriter.modifyOpInPlace(use.getOwner(),
@@ -2460,7 +2460,7 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantFalse)
-          constantFalse = rewriter.create<arith::ConstantOp>(
+          constantFalse = arith::ConstantOp::create(rewriter,
               op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
 
         rewriter.modifyOpInPlace(use.getOwner(),
@@ -2547,7 +2547,7 @@ struct ReplaceIfYieldWithConditionOrValue : public OpRewritePattern<IfOp> {
       if (!trueVal && falseVal) {
         if (!opResult.use_empty()) {
           Dialect *constDialect = trueResult.getDefiningOp()->getDialect();
-          Value notCond = rewriter.create<arith::XOrIOp>(
+          Value notCond = arith::XOrIOp::create(rewriter,
               op.getLoc(), op.getCondition(),
               constDialect
                   ->materializeConstant(rewriter,
@@ -2661,7 +2661,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
     SmallVector<Type> mergedTypes(prevIf.getResultTypes());
     llvm::append_range(mergedTypes, nextIf.getResultTypes());
 
-    IfOp combinedIf = rewriter.create<IfOp>(
+    IfOp combinedIf = IfOp::create(rewriter,
         nextIf.getLoc(), mergedTypes, prevIf.getCondition(), /*hasElse=*/false);
     rewriter.eraseBlock(&combinedIf.getThenRegion().back());
 
@@ -2677,7 +2677,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
 
       SmallVector<Value> mergedYields(thenYield.getOperands());
       llvm::append_range(mergedYields, thenYield2.getOperands());
-      rewriter.create<YieldOp>(thenYield2.getLoc(), mergedYields);
+      YieldOp::create(rewriter, thenYield2.getLoc(), mergedYields);
       rewriter.eraseOp(thenYield);
       rewriter.eraseOp(thenYield2);
     }
@@ -2701,7 +2701,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
         SmallVector<Value> mergedElseYields(elseYield.getOperands());
         llvm::append_range(mergedElseYields, elseYield2.getOperands());
 
-        rewriter.create<YieldOp>(elseYield2.getLoc(), mergedElseYields);
+        YieldOp::create(rewriter, elseYield2.getLoc(), mergedElseYields);
         rewriter.eraseOp(elseYield);
         rewriter.eraseOp(elseYield2);
       }
@@ -2823,9 +2823,9 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     }
 
     Location loc = op.getLoc();
-    Value newCondition = rewriter.create<arith::AndIOp>(
+    Value newCondition = arith::AndIOp::create(rewriter,
         loc, op.getCondition(), nestedIf.getCondition());
-    auto newIf = rewriter.create<IfOp>(loc, op.getResultTypes(), newCondition);
+    auto newIf = IfOp::create(rewriter, loc, op.getResultTypes(), newCondition);
     Block *newIfBlock = rewriter.createBlock(&newIf.getThenRegion());
 
     SmallVector<Value> results;
@@ -2833,7 +2833,7 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     rewriter.setInsertionPoint(newIf);
 
     for (auto idx : elseYieldsToUpgradeToSelect)
-      results[idx] = rewriter.create<arith::SelectOp>(
+      results[idx] = arith::SelectOp::create(rewriter,
           op.getLoc(), op.getCondition(), thenYield[idx], elseYield[idx]);
 
     rewriter.mergeBlocks(nestedIf.thenBlock(), newIfBlock);
@@ -2842,7 +2842,7 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     if (!elseYield.empty()) {
       rewriter.createBlock(&newIf.getElseRegion());
       rewriter.setInsertionPointToEnd(newIf.elseBlock());
-      rewriter.create<YieldOp>(loc, elseYield);
+      YieldOp::create(rewriter, loc, elseYield);
     }
     rewriter.replaceOp(op, results);
     return success();
@@ -3160,7 +3160,7 @@ struct ParallelOpSingleOrZeroIterationDimsFolder
     }
     // Replace the parallel loop by lower-dimensional parallel loop.
     auto newOp =
-        rewriter.create<ParallelOp>(op.getLoc(), newLowerBounds, newUpperBounds,
+        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds, newUpperBounds,
                                     newSteps, op.getInitVals(), nullptr);
     // Erase the empty block that was inserted by the builder.
     rewriter.eraseBlock(newOp.getBody());
@@ -3541,7 +3541,7 @@ struct WhileConditionTruth : public OpRewritePattern<WhileOp> {
       if (std::get<0>(yieldedAndBlockArgs) == term.getCondition()) {
         if (!std::get<1>(yieldedAndBlockArgs).use_empty()) {
           if (!constantTrue)
-            constantTrue = rewriter.create<arith::ConstantOp>(
+            constantTrue = arith::ConstantOp::create(rewriter,
                 op.getLoc(), term.getCondition().getType(),
                 rewriter.getBoolAttr(true));
 
@@ -3685,7 +3685,7 @@ struct RemoveLoopInvariantArgsFromBeforeBlock
     }
 
     auto newWhile =
-        rewriter.create<WhileOp>(op.getLoc(), op.getResultTypes(), newInitArgs);
+        WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(), newInitArgs);
 
     Block &newBeforeBlock = *rewriter.createBlock(
         &newWhile.getBefore(), /*insertPt*/ {},
@@ -3807,7 +3807,7 @@ struct RemoveLoopInvariantValueYielded : public OpRewritePattern<WhileOp> {
                                                newCondOpArgs);
     }
 
-    auto newWhile = rewriter.create<WhileOp>(op.getLoc(), newAfterBlockType,
+    auto newWhile = WhileOp::create(rewriter, op.getLoc(), newAfterBlockType,
                                              op.getOperands());
 
     Block &newAfterBlock =
@@ -3914,7 +3914,7 @@ struct WhileUnusedResult : public OpRewritePattern<WhileOp> {
     }
 
     auto newWhile =
-        rewriter.create<WhileOp>(op.getLoc(), newResultTypes, op.getInits());
+        WhileOp::create(rewriter, op.getLoc(), newResultTypes, op.getInits());
 
     Block &newAfterBlock = *rewriter.createBlock(
         &newWhile.getAfter(), /*insertPt*/ {}, newResultTypes, newArgLocs);
@@ -4043,7 +4043,7 @@ struct WhileRemoveUnusedArgs : public OpRewritePattern<WhileOp> {
 
     Location loc = op.getLoc();
     auto newWhileOp =
-        rewriter.create<WhileOp>(loc, op.getResultTypes(), newInits,
+        WhileOp::create(rewriter, loc, op.getResultTypes(), newInits,
                                  /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
@@ -4091,7 +4091,7 @@ struct WhileRemoveDuplicatedResults : public OpRewritePattern<WhileOp> {
     ValueRange argsRange(newArgs);
 
     Location loc = op.getLoc();
-    auto newWhileOp = rewriter.create<scf::WhileOp>(
+    auto newWhileOp = scf::WhileOp::create(rewriter,
         loc, argsRange.getTypes(), op.getInits(), /*beforeBody*/ nullptr,
         /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
@@ -4187,7 +4187,7 @@ struct WhileOpAlignBeforeArgs : public OpRewritePattern<WhileOp> {
     for (auto &&[i, j] : llvm::enumerate(*mapping))
       newResultTypes[j] = loop.getResult(i).getType();
 
-    auto newLoop = rewriter.create<WhileOp>(
+    auto newLoop = WhileOp::create(rewriter,
         loop.getLoc(), newResultTypes, loop.getInits(),
         /*beforeBuilder=*/nullptr, /*afterBuilder=*/nullptr);
     auto newBefore = newLoop.getBeforeBody();
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index 57c27231f2144..e9943a3489d46 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -163,7 +163,7 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
   OpBuilder::InsertionGuard g(b);
   b.setInsertionPoint(op);
   scf::ExecuteRegionOp executeRegionOp =
-      b.create<scf::ExecuteRegionOp>(op->getLoc(), op->getResultTypes());
+      scf::ExecuteRegionOp::create(b, op->getLoc(), op->getResultTypes());
   {
     OpBuilder::InsertionGuard g(b);
     b.setInsertionPointToStart(&executeRegionOp.getRegion().emplaceBlock());
@@ -172,7 +172,7 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
     assert(clonedRegion.empty() && "expected empty region");
     b.inlineRegionBefore(op->getRegions().front(), clonedRegion,
                          clonedRegion.end());
-    b.create<scf::YieldOp>(op->getLoc(), clonedOp->getResults());
+    scf::YieldOp::create(b, op->getLoc(), clonedOp->getResults());
   }
   b.replaceOp(op, executeRegionOp.getResults());
   return executeRegionOp;
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index d36d91249ed36..0214ce3090656 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -42,7 +42,7 @@ static Value castBuffer(OpBuilder &b, Value buffer, Type type) {
   // iter_arg's layout map must be changed (see uses of `castBuffer`).
   assert(memref::CastOp::areCastCompatible(buffer.getType(), type) &&
          "scf.while op bufferization: cast incompatible");
-  return b.create<memref::CastOp>(buffer.getLoc(), type, buffer).getResult();
+  return memref::CastOp::create(b, buffer.getLoc(), type, buffer).getResult();
 }
 
 /// Helper function for loop bufferization. Return "true" if the given value
@@ -191,7 +191,7 @@ struct ExecuteRegionOpInterface
 
     // Create new op and move over region.
     auto newOp =
-        rewriter.create<scf::ExecuteRegionOp>(op->getLoc(), newResultTypes);
+        scf::ExecuteRegionOp::create(rewriter, op->getLoc(), newResultTypes);
     newOp.getRegion().takeBody(executeRegionOp.getRegion());
 
     // Bufferize every block.
@@ -205,7 +205,7 @@ struct ExecuteRegionOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(executeRegionOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
+        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
             executeRegionOp.getLoc(), it.value(),
             newOp->getResult(it.index())));
       } else {
@@ -261,7 +261,7 @@ struct IfOpInterface
     // Create new op.
     rewriter.setInsertionPoint(ifOp);
     auto newIfOp =
-        rewriter.create<scf::IfOp>(ifOp.getLoc(), newTypes, ifOp.getCondition(),
+        scf::IfOp::create(rewriter, ifOp.getLoc(), newTypes, ifOp.getCondition(),
                                    /*withElseRegion=*/true);
 
     // Move over then/else blocks.
@@ -374,7 +374,7 @@ struct IndexSwitchOpInterface
 
     // Create new op.
     rewriter.setInsertionPoint(switchOp);
-    auto newSwitchOp = rewriter.create<scf::IndexSwitchOp>(
+    auto newSwitchOp = scf::IndexSwitchOp::create(rewriter,
         switchOp.getLoc(), newTypes, switchOp.getArg(), switchOp.getCases(),
         switchOp.getCases().size());
 
@@ -769,7 +769,7 @@ struct ForOpInterface
     }
 
     // Construct a new scf.for op with memref instead of tensor values.
-    auto newForOp = rewriter.create<scf::ForOp>(
+    auto newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), castedInitArgs);
     newForOp->setAttrs(forOp->getAttrs());
@@ -1005,7 +1005,7 @@ struct WhileOpInterface
     // Construct a new scf.while op with memref instead of tensor values.
     ValueRange argsRangeBefore(castedInitArgs);
     TypeRange argsTypesBefore(argsRangeBefore);
-    auto newWhileOp = rewriter.create<scf::WhileOp>(
+    auto newWhileOp = scf::WhileOp::create(rewriter,
         whileOp.getLoc(), argsTypesAfter, castedInitArgs);
 
     // Add before/after regions to the new op.
@@ -1265,7 +1265,7 @@ struct ForallOpInterface
              forallOp.getBody()->getArguments().drop_front(rank), buffers)) {
       BlockArgument bbArg = std::get<0>(it);
       Value buffer = std::get<1>(it);
-      Value bufferAsTensor = rewriter.create<ToTensorOp>(
+      Value bufferAsTensor = ToTensorOp::create(rewriter,
           forallOp.getLoc(), bbArg.getType(), buffer);
       bbArg.replaceAllUsesWith(bufferAsTensor);
     }
@@ -1274,7 +1274,7 @@ struct ForallOpInterface
     // introduced terminator.
     rewriter.setInsertionPoint(forallOp);
     ForallOp newForallOp;
-    newForallOp = rewriter.create<ForallOp>(
+    newForallOp = ForallOp::create(rewriter,
         forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
         /*outputs=*/ValueRange(), forallOp.getMapping());
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
index 3e93dc80b18ec..bf4eba3319a4d 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
@@ -50,7 +50,7 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     SmallVector<Value> initArgs;
     initArgs.push_back(forOp.getLowerBound());
     llvm::append_range(initArgs, forOp.getInitArgs());
-    auto whileOp = rewriter.create<WhileOp>(forOp.getLoc(), lcvTypes, initArgs,
+    auto whileOp = WhileOp::create(rewriter, forOp.getLoc(), lcvTypes, initArgs,
                                             forOp->getAttrs());
 
     // 'before' region contains the loop condition and forwarding of iteration
@@ -58,10 +58,10 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     auto *beforeBlock = rewriter.createBlock(
         &whileOp.getBefore(), whileOp.getBefore().begin(), lcvTypes, lcvLocs);
     rewriter.setInsertionPointToStart(whileOp.getBeforeBody());
-    auto cmpOp = rewriter.create<arith::CmpIOp>(
+    auto cmpOp = arith::CmpIOp::create(rewriter,
         whileOp.getLoc(), arith::CmpIPredicate::slt,
         beforeBlock->getArgument(0), forOp.getUpperBound());
-    rewriter.create<scf::ConditionOp>(whileOp.getLoc(), cmpOp.getResult(),
+    scf::ConditionOp::create(rewriter, whileOp.getLoc(), cmpOp.getResult(),
                                       beforeBlock->getArguments());
 
     // Inline for-loop body into an executeRegion operation in the "after"
@@ -72,7 +72,7 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
 
     // Add induction variable incrementation
     rewriter.setInsertionPointToEnd(afterBlock);
-    auto ivIncOp = rewriter.create<arith::AddIOp>(
+    auto ivIncOp = arith::AddIOp::create(rewriter,
         whileOp.getLoc(), afterBlock->getArgument(0), forOp.getStep());
 
     // Rewrite uses of the for-loop block arguments to the new while-loop
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
index 44e6840b03a3d..b95604fa44cb9 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
@@ -40,7 +40,7 @@ LogicalResult mlir::scf::forallToParallelLoop(RewriterBase &rewriter,
   SmallVector<Value> steps = forallOp.getStep(rewriter);
 
   // Create empty scf.parallel op.
-  auto parallelOp = rewriter.create<scf::ParallelOp>(loc, lbs, ubs, steps);
+  auto parallelOp = scf::ParallelOp::create(rewriter, loc, lbs, ubs, steps);
   rewriter.eraseBlock(&parallelOp.getRegion().front());
   rewriter.inlineRegionBefore(forallOp.getRegion(), parallelOp.getRegion(),
                               parallelOp.getRegion().begin());
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index bcecef5e6e0a9..f1ede64d01e0f 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -279,24 +279,24 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
     if (dynamicLoop) {
       Type t = ub.getType();
       // pred = ub > lb + (i * step)
-      Value iv = rewriter.create<arith::AddIOp>(
+      Value iv = arith::AddIOp::create(rewriter,
           loc, lb,
-          rewriter.create<arith::MulIOp>(
+          arith::MulIOp::create(rewriter,
               loc, step,
-              rewriter.create<arith::ConstantOp>(
+              arith::ConstantOp::create(rewriter,
                   loc, rewriter.getIntegerAttr(t, i))));
-      predicates[i] = rewriter.create<arith::CmpIOp>(
+      predicates[i] = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::slt, iv, ub);
     }
 
     // special handling for induction variable as the increment is implicit.
     // iv = lb + i * step
     Type t = lb.getType();
-    Value iv = rewriter.create<arith::AddIOp>(
+    Value iv = arith::AddIOp::create(rewriter,
         loc, lb,
-        rewriter.create<arith::MulIOp>(
+        arith::MulIOp::create(rewriter,
             loc, step,
-            rewriter.create<arith::ConstantOp>(loc,
+            arith::ConstantOp::create(rewriter, loc,
                                                rewriter.getIntegerAttr(t, i))));
     setValueMapping(forOp.getInductionVar(), iv, i);
     for (Operation *op : opOrder) {
@@ -332,7 +332,7 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
             Value prevValue = valueMapping
                 [forOp.getRegionIterArgs()[operand.getOperandNumber()]]
                 [i - stages[op]];
-            source = rewriter.create<arith::SelectOp>(
+            source = arith::SelectOp::create(rewriter,
                 loc, predicates[predicateIdx], source, prevValue);
           }
           setValueMapping(forOp.getRegionIterArgs()[operand.getOperandNumber()],
@@ -444,14 +444,14 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop(
     Type t = ub.getType();
     Location loc = forOp.getLoc();
     // newUb = ub - maxStage * step
-    Value maxStageValue = rewriter.create<arith::ConstantOp>(
+    Value maxStageValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(t, maxStage));
     Value maxStageByStep =
-        rewriter.create<arith::MulIOp>(loc, step, maxStageValue);
-    newUb = rewriter.create<arith::SubIOp>(loc, ub, maxStageByStep);
+        arith::MulIOp::create(rewriter, loc, step, maxStageValue);
+    newUb = arith::SubIOp::create(rewriter, loc, ub, maxStageByStep);
   }
   auto newForOp =
-      rewriter.create<scf::ForOp>(forOp.getLoc(), forOp.getLowerBound(), newUb,
+      scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(), newUb,
                                   forOp.getStep(), newLoopArg);
   // When there are no iter args, the loop body terminator will be created.
   // Since we always create it below, remove the terminator if it was created.
@@ -483,14 +483,14 @@ LogicalResult LoopPipelinerInternal::createKernel(
     Type t = ub.getType();
     for (unsigned i = 0; i < maxStage; i++) {
       // c = ub - (maxStage - i) * step
-      Value c = rewriter.create<arith::SubIOp>(
+      Value c = arith::SubIOp::create(rewriter,
           loc, ub,
-          rewriter.create<arith::MulIOp>(
+          arith::MulIOp::create(rewriter,
               loc, step,
-              rewriter.create<arith::ConstantOp>(
+              arith::ConstantOp::create(rewriter,
                   loc, rewriter.getIntegerAttr(t, int64_t(maxStage - i)))));
 
-      Value pred = rewriter.create<arith::CmpIOp>(
+      Value pred = arith::CmpIOp::create(rewriter,
           newForOp.getLoc(), arith::CmpIPredicate::slt,
           newForOp.getInductionVar(), c);
       predicates[i] = pred;
@@ -515,12 +515,12 @@ LogicalResult LoopPipelinerInternal::createKernel(
 
         // offset = (maxStage - stages[op]) * step
         Type t = step.getType();
-        Value offset = rewriter.create<arith::MulIOp>(
+        Value offset = arith::MulIOp::create(rewriter,
             forOp.getLoc(), step,
-            rewriter.create<arith::ConstantOp>(
+            arith::ConstantOp::create(rewriter,
                 forOp.getLoc(),
                 rewriter.getIntegerAttr(t, maxStage - stages[op])));
-        Value iv = rewriter.create<arith::AddIOp>(
+        Value iv = arith::AddIOp::create(rewriter,
             forOp.getLoc(), newForOp.getInductionVar(), offset);
         nestedNewOp->setOperand(operand->getOperandNumber(), iv);
         rewriter.setInsertionPointAfter(newOp);
@@ -594,7 +594,7 @@ LogicalResult LoopPipelinerInternal::createKernel(
         auto defStage = stages.find(def);
         if (defStage != stages.end() && defStage->second < maxStage) {
           Value pred = predicates[defStage->second];
-          source = rewriter.create<arith::SelectOp>(
+          source = arith::SelectOp::create(rewriter,
               pred.getLoc(), pred, source,
               newForOp.getBody()
                   ->getArguments()[yieldOperand.getOperandNumber() + 1]);
@@ -638,7 +638,7 @@ LogicalResult LoopPipelinerInternal::createKernel(
                       maxStage - defStage->second + 1);
     }
   }
-  rewriter.create<scf::YieldOp>(forOp.getLoc(), yieldOperands);
+  scf::YieldOp::create(rewriter, forOp.getLoc(), yieldOperands);
   return success();
 }
 
@@ -652,7 +652,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // removed by dead code if not used.
 
   auto createConst = [&](int v) {
-    return rewriter.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(rewriter, loc,
                                               rewriter.getIntegerAttr(t, v));
   };
 
@@ -661,41 +661,41 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // - total_iterations = (range_diff + step + (step < 0 ? 1 : -1)) / step
   Value zero = createConst(0);
   Value one = createConst(1);
-  Value stepLessZero = rewriter.create<arith::CmpIOp>(
+  Value stepLessZero = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::slt, step, zero);
   Value stepDecr =
-      rewriter.create<arith::SelectOp>(loc, stepLessZero, one, createConst(-1));
+      arith::SelectOp::create(rewriter, loc, stepLessZero, one, createConst(-1));
 
-  Value rangeDiff = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  Value rangeIncrStep = rewriter.create<arith::AddIOp>(loc, rangeDiff, step);
+  Value rangeDiff = arith::SubIOp::create(rewriter, loc, ub, lb);
+  Value rangeIncrStep = arith::AddIOp::create(rewriter, loc, rangeDiff, step);
   Value rangeDecr =
-      rewriter.create<arith::AddIOp>(loc, rangeIncrStep, stepDecr);
-  Value totalIterations = rewriter.create<arith::DivSIOp>(loc, rangeDecr, step);
+      arith::AddIOp::create(rewriter, loc, rangeIncrStep, stepDecr);
+  Value totalIterations = arith::DivSIOp::create(rewriter, loc, rangeDecr, step);
 
   // If total_iters < max_stage, start the epilogue at zero to match the
   // ramp-up in the prologue.
   // start_iter = max(0, total_iters - max_stage)
-  Value iterI = rewriter.create<arith::SubIOp>(loc, totalIterations,
+  Value iterI = arith::SubIOp::create(rewriter, loc, totalIterations,
                                                createConst(maxStage));
-  iterI = rewriter.create<arith::MaxSIOp>(loc, zero, iterI);
+  iterI = arith::MaxSIOp::create(rewriter, loc, zero, iterI);
 
   // Capture predicates for dynamic loops.
   SmallVector<Value> predicates(maxStage + 1);
 
   for (int64_t i = 1; i <= maxStage; i++) {
     // newLastIter = lb + step * iterI
-    Value newlastIter = rewriter.create<arith::AddIOp>(
-        loc, lb, rewriter.create<arith::MulIOp>(loc, step, iterI));
+    Value newlastIter = arith::AddIOp::create(rewriter,
+        loc, lb, arith::MulIOp::create(rewriter, loc, step, iterI));
 
     setValueMapping(forOp.getInductionVar(), newlastIter, i);
 
     // increment to next iterI
-    iterI = rewriter.create<arith::AddIOp>(loc, iterI, one);
+    iterI = arith::AddIOp::create(rewriter, loc, iterI, one);
 
     if (dynamicLoop) {
       // Disable stages when `i` is greater than total_iters.
       // pred = total_iters >= i
-      predicates[i] = rewriter.create<arith::CmpIOp>(
+      predicates[i] = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, totalIterations, createConst(i));
     }
   }
@@ -758,7 +758,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
           unsigned nextVersion = currentVersion + 1;
           Value pred = predicates[currentVersion];
           Value prevValue = valueMapping[mapVal][currentVersion];
-          auto selOp = rewriter.create<arith::SelectOp>(loc, pred, pair.value(),
+          auto selOp = arith::SelectOp::create(rewriter, loc, pred, pair.value(),
                                                         prevValue);
           returnValues[ri] = selOp;
           if (nextVersion <= maxStage)
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index b71ec985fa6a1..b24e276272c2c 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -64,13 +64,13 @@ static void specializeParallelLoopForUnrolling(ParallelOp op) {
   Value cond;
   for (auto bound : llvm::zip(op.getUpperBound(), constantIndices)) {
     Value constant =
-        b.create<arith::ConstantIndexOp>(op.getLoc(), std::get<1>(bound));
-    Value cmp = b.create<arith::CmpIOp>(op.getLoc(), arith::CmpIPredicate::eq,
+        arith::ConstantIndexOp::create(b, op.getLoc(), std::get<1>(bound));
+    Value cmp = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
                                         std::get<0>(bound), constant);
-    cond = cond ? b.create<arith::AndIOp>(op.getLoc(), cond, cmp) : cmp;
+    cond = cond ? arith::AndIOp::create(b, op.getLoc(), cond, cmp) : cmp;
     map.map(std::get<0>(bound), constant);
   }
-  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
   ifOp.getElseBodyBuilder().clone(*op.getOperation());
   op.erase();
@@ -95,11 +95,11 @@ static void specializeForLoopForUnrolling(ForOp op) {
 
   OpBuilder b(op);
   IRMapping map;
-  Value constant = b.create<arith::ConstantIndexOp>(op.getLoc(), minConstant);
-  Value cond = b.create<arith::CmpIOp>(op.getLoc(), arith::CmpIPredicate::eq,
+  Value constant = arith::ConstantIndexOp::create(b, op.getLoc(), minConstant);
+  Value cond = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
                                        bound, constant);
   map.map(bound, constant);
-  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
   ifOp.getElseBodyBuilder().clone(*op.getOperation());
   op.erase();
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index ad1267381c4f2..013c3b69620e2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -190,7 +190,7 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
 
   IRRewriter b(builder);
   b.setInsertionPoint(secondPloop);
-  auto newSecondPloop = b.create<ParallelOp>(
+  auto newSecondPloop = ParallelOp::create(b,
       secondPloop.getLoc(), secondPloop.getLowerBound(),
       secondPloop.getUpperBound(), secondPloop.getStep(), newInitVars);
 
@@ -212,7 +212,7 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
     SmallVector<Value> newReduceArgs(reduceArgs1.begin(), reduceArgs1.end());
     newReduceArgs.append(reduceArgs2.begin(), reduceArgs2.end());
 
-    auto newReduceOp = b.create<scf::ReduceOp>(term2.getLoc(), newReduceArgs);
+    auto newReduceOp = scf::ReduceOp::create(b, term2.getLoc(), newReduceArgs);
 
     for (auto &&[i, reg] : llvm::enumerate(llvm::concat<Region>(
              term1.getReductions(), term2.getReductions()))) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
index 66f7bc27f82ff..2ca3b1e366bdf 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
@@ -58,27 +58,27 @@ std::pair<ParallelOp, ParallelOp>
 mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
                             bool noMinMaxBounds) {
   OpBuilder b(op);
-  auto zero = b.create<arith::ConstantIndexOp>(op.getLoc(), 0);
+  auto zero = arith::ConstantIndexOp::create(b, op.getLoc(), 0);
   SmallVector<Value, 2> tileSizeConstants;
   tileSizeConstants.reserve(op.getUpperBound().size());
   for (size_t i = 0, end = op.getUpperBound().size(); i != end; ++i) {
     if (i < tileSizes.size())
       tileSizeConstants.push_back(
-          b.create<arith::ConstantIndexOp>(op.getLoc(), tileSizes[i]));
+          arith::ConstantIndexOp::create(b, op.getLoc(), tileSizes[i]));
     else
       // Just pick 1 for the remaining dimensions.
       tileSizeConstants.push_back(
-          b.create<arith::ConstantIndexOp>(op.getLoc(), 1));
+          arith::ConstantIndexOp::create(b, op.getLoc(), 1));
   }
 
   // Create the outer loop with adjusted steps.
   SmallVector<Value, 2> newSteps;
   newSteps.reserve(op.getStep().size());
   for (auto step : llvm::zip(op.getStep(), tileSizeConstants)) {
-    newSteps.push_back(b.create<arith::MulIOp>(op.getLoc(), std::get<0>(step),
+    newSteps.push_back(arith::MulIOp::create(b, op.getLoc(), std::get<0>(step),
                                                std::get<1>(step)));
   }
-  auto outerLoop = b.create<ParallelOp>(op.getLoc(), op.getLowerBound(),
+  auto outerLoop = ParallelOp::create(b, op.getLoc(), op.getLowerBound(),
                                         op.getUpperBound(), newSteps);
   b.setInsertionPointToStart(outerLoop.getBody());
 
@@ -130,10 +130,10 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // Otherwise, we dynamically compute the bound for
     // each iteration of the outer loop.
     newBounds.push_back(
-        b.create<affine::AffineMinOp>(op.getLoc(), b.getIndexType(), minMap,
+        affine::AffineMinOp::create(b, op.getLoc(), b.getIndexType(), minMap,
                                       ValueRange{newStep, upperBound, iv}));
   }
-  auto innerLoop = b.create<ParallelOp>(
+  auto innerLoop = ParallelOp::create(b,
       op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
       op.getStep());
 
@@ -141,20 +141,20 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     b.setInsertionPointToStart(innerLoop.getBody());
     // Insert in-bound check
     Value inbound =
-        b.create<arith::ConstantIntOp>(op.getLoc(), b.getIntegerType(1), 1);
+        arith::ConstantIntOp::create(b, op.getLoc(), b.getIntegerType(1), 1);
     for (auto [outerUpperBound, outerIV, innerIV, innerStep] :
          llvm::zip(outerLoop.getUpperBound(), outerLoop.getInductionVars(),
                    innerLoop.getInductionVars(), innerLoop.getStep())) {
       // %in_bound = %in_bound &&
       //             (%inner_iv * %inner_step + %outer_iv < %outer_upper_bound)
-      Value index = b.create<arith::AddIOp>(
-          op.getLoc(), b.create<arith::MulIOp>(op.getLoc(), innerIV, innerStep),
+      Value index = arith::AddIOp::create(b,
+          op.getLoc(), arith::MulIOp::create(b, op.getLoc(), innerIV, innerStep),
           outerIV);
-      Value dimInbound = b.create<arith::CmpIOp>(
+      Value dimInbound = arith::CmpIOp::create(b,
           op.getLoc(), arith::CmpIPredicate::ult, index, outerUpperBound);
-      inbound = b.create<arith::AndIOp>(op.getLoc(), inbound, dimInbound);
+      inbound = arith::AndIOp::create(b, op.getLoc(), inbound, dimInbound);
     }
-    auto ifInbound = b.create<IfOp>(op.getLoc(),
+    auto ifInbound = IfOp::create(b, op.getLoc(),
                                     /*resultTypes*/ ArrayRef<Type>{}, inbound,
                                     /*hasElseRegion*/ false);
     ifInbound.getThenRegion().takeBody(op.getRegion());
@@ -162,12 +162,12 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // Replace the scf.reduce terminator with an scf.yield terminator.
     Operation *reduceOp = thenBlock.getTerminator();
     b.setInsertionPointToEnd(&thenBlock);
-    b.create<scf::YieldOp>(reduceOp->getLoc());
+    scf::YieldOp::create(b, reduceOp->getLoc());
     reduceOp->erase();
     b.setInsertionPointToStart(innerLoop.getBody());
     for (const auto &ivs : llvm::enumerate(llvm::zip(
              innerLoop.getInductionVars(), outerLoop.getInductionVars()))) {
-      auto newIndex = b.create<arith::AddIOp>(
+      auto newIndex = arith::AddIOp::create(b,
           op.getLoc(), std::get<0>(ivs.value()), std::get<1>(ivs.value()));
       thenBlock.getArgument(ivs.index())
           .replaceAllUsesExcept(newIndex, newIndex);
@@ -179,7 +179,7 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     for (auto ivs : llvm::zip(innerLoop.getInductionVars(),
                               outerLoop.getInductionVars())) {
       Value innerIndex = std::get<0>(ivs);
-      auto newIndex = b.create<arith::AddIOp>(op.getLoc(), std::get<0>(ivs),
+      auto newIndex = arith::AddIOp::create(b, op.getLoc(), std::get<0>(ivs),
                                               std::get<1>(ivs));
       innerIndex.replaceAllUsesExcept(newIndex, newIndex);
     }
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 09326242eec2a..41895e46d953d 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -112,7 +112,7 @@ class ConvertForOpTypes
 
     // We can not do clone as the number of result types after conversion
     // might be different.
-    ForOp newOp = rewriter.create<ForOp>(
+    ForOp newOp = ForOp::create(rewriter,
         op.getLoc(), llvm::getSingleElement(adaptor.getLowerBound()),
         llvm::getSingleElement(adaptor.getUpperBound()),
         llvm::getSingleElement(adaptor.getStep()),
@@ -142,7 +142,7 @@ class ConvertIfOpTypes
                                       ConversionPatternRewriter &rewriter,
                                       TypeRange dstTypes) const {
 
-    IfOp newOp = rewriter.create<IfOp>(
+    IfOp newOp = IfOp::create(rewriter,
         op.getLoc(), dstTypes, llvm::getSingleElement(adaptor.getCondition()),
         true);
     newOp->setAttrs(op->getAttrs());
@@ -171,7 +171,7 @@ class ConvertWhileOpTypes
   std::optional<WhileOp> convertSourceOp(WhileOp op, OneToNOpAdaptor adaptor,
                                          ConversionPatternRewriter &rewriter,
                                          TypeRange dstTypes) const {
-    auto newOp = rewriter.create<WhileOp>(op.getLoc(), dstTypes,
+    auto newOp = WhileOp::create(rewriter, op.getLoc(), dstTypes,
                                           flattenValues(adaptor.getOperands()));
 
     for (auto i : {0u, 1u}) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index 995120ad8680e..f995f324a0dbd 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -450,7 +450,7 @@ static LogicalResult generateLoopNestUsingForOp(
   SmallVector<Value> ivs;
   for (auto [lb, ub, step] : llvm::zip_equal(lbVals, ubVals, stepVals)) {
     auto loop =
-        rewriter.create<scf::ForOp>(loc, lb, ub, step, destinationTensors,
+        scf::ForOp::create(rewriter, loc, lb, ub, step, destinationTensors,
                                     [](OpBuilder &bodyBuilder, Location bodyLoc,
                                        Value iv, ValueRange /*iterArgs*/) {});
     loops.push_back(loop);
@@ -479,12 +479,12 @@ static LogicalResult generateLoopNestUsingForOp(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    auto insertSlice = rewriter.create<tensor::InsertSliceOp>(
+    auto insertSlice = tensor::InsertSliceOp::create(rewriter,
         loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
     yieldedValues.push_back(insertSlice);
   }
-  rewriter.create<scf::YieldOp>(loc, yieldedValues);
+  scf::YieldOp::create(rewriter, loc, yieldedValues);
 
   // Add the scf.yield operations for all the outer loops.
   for (auto [outerLoop, innerLoop] :
@@ -492,7 +492,7 @@ static LogicalResult generateLoopNestUsingForOp(
                        MutableArrayRef(loops).drop_front())) {
     rewriter.setInsertionPointToEnd(
         cast<scf::ForOp>(outerLoop.getOperation()).getBody());
-    rewriter.create<scf::YieldOp>(outerLoop.getLoc(), innerLoop->getResults());
+    scf::YieldOp::create(rewriter, outerLoop.getLoc(), innerLoop->getResults());
   }
   return success();
 }
@@ -533,13 +533,13 @@ static LogicalResult generateLoopNestUsingForallOp(
         continue;
       nonZeroNumThreads.push_back(nt);
     }
-    forallOp = rewriter.create<scf::ForallOp>(loc, nonZeroNumThreads,
+    forallOp = scf::ForallOp::create(rewriter, loc, nonZeroNumThreads,
                                               destinationTensors, mappingAttr);
   } else {
     SmallVector<OpFoldResult> lbs, ubs, steps;
     std::tie(lbs, ubs, steps) =
         getLoopBounds(rewriter, loc, loopRanges, tileSizes);
-    forallOp = rewriter.create<scf::ForallOp>(loc, lbs, ubs, steps,
+    forallOp = scf::ForallOp::create(rewriter, loc, lbs, ubs, steps,
                                               destinationTensors, mappingAttr);
   }
   loops.push_back(forallOp);
@@ -561,7 +561,7 @@ static LogicalResult generateLoopNestUsingForallOp(
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
 
-    rewriter.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(rewriter,
         loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
   }
@@ -798,7 +798,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
 
   auto inits = llvm::to_vector(loopOp.getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = rewriter.create<scf::ForOp>(
+  auto newLoop = scf::ForOp::create(rewriter,
       loc, loopOp.getLowerBound(), loopOp.getUpperBound(), loopOp.getStep(),
       inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
@@ -829,7 +829,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    Value insert = rewriter.create<tensor::InsertSliceOp>(
+    Value insert = tensor::InsertSliceOp::create(rewriter,
         yieldOp->getLoc(), tiledValue, regionIterArg, resultOffset, resultSize,
         resultStride);
     newYieldValues.push_back(insert);
@@ -851,7 +851,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
   rewriter.setInsertionPoint(loopOp);
   auto inits = llvm::to_vector(loopOp.getOutputs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = rewriter.create<scf::ForallOp>(
+  auto newLoop = scf::ForallOp::create(rewriter,
       loc, loopOp.getMixedLowerBound(), loopOp.getMixedUpperBound(),
       loopOp.getMixedStep(), inits, loopOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
@@ -884,7 +884,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
            tiledValues, regionIterArgs, resultOffsets, resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    rewriter.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(rewriter,
         terminator.getLoc(), tiledValue, iterArg, resultOffset, resultSize,
         resultStride);
   }
@@ -935,7 +935,7 @@ static LogicalResult addInitOperandsToLoopNest(
     // Create a new loop with the new init values for this loop.
     SmallVector<Value> newInits = llvm::to_vector(forLoop.getInitArgs());
     newInits.append(newInitValues.begin(), newInitValues.end());
-    auto newLoop = rewriter.create<scf::ForOp>(
+    auto newLoop = scf::ForOp::create(rewriter,
         forLoop.getLoc(), forLoop.getLowerBound(), forLoop.getUpperBound(),
         forLoop.getStep(), newInits,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {});
@@ -1419,7 +1419,7 @@ FailureOr<SmallVector<Operation *>> mlir::scf::yieldReplacementForFusedProducer(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = rewriter.create<tensor::ExtractSliceOp>(
+        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
             loc, newRegionArg, offsetList[index], sizesList[index],
             SmallVector<OpFoldResult>(offsetList[index].size(),
                                       rewriter.getIndexAttr(1)));
@@ -2092,7 +2092,7 @@ cloneAsInsertSlice<tensor::InsertSliceOp>(RewriterBase &rewriter,
 template <>
 tensor::InsertSliceOp cloneAsInsertSlice<tensor::ParallelInsertSliceOp>(
     RewriterBase &rewriter, tensor::ParallelInsertSliceOp insertSliceOp) {
-  return rewriter.create<tensor::InsertSliceOp>(
+  return tensor::InsertSliceOp::create(rewriter,
       insertSliceOp->getLoc(), insertSliceOp.getSource(),
       insertSliceOp.getDest(), insertSliceOp.getMixedOffsets(),
       insertSliceOp.getMixedSizes(), insertSliceOp.getMixedStrides());
@@ -2314,7 +2314,7 @@ mlir::scf::tileAndFuseConsumerOfSlices(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = rewriter.create<tensor::ExtractSliceOp>(
+        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
             loc, newRegionArg, resultOffsets[index], resultSizes[index],
             SmallVector<OpFoldResult>(resultOffsets[index].size(),
                                       rewriter.getIndexAttr(1)));
@@ -2391,7 +2391,7 @@ mlir::scf::lowerToLoopsUsingSCFForOp(RewriterBase &rewriter,
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.size);
     Value strideVal =
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.stride);
-    auto loop = rewriter.create<scf::ForOp>(op.getLoc(), offsetVal, sizeVal,
+    auto loop = scf::ForOp::create(rewriter, op.getLoc(), offsetVal, sizeVal,
                                             strideVal, ValueRange{});
     loops.push_back(loop);
     ivs.push_back(loop.getInductionVar());
diff --git a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
index 29d6d2574a2be..14ded96516eab 100644
--- a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
@@ -191,7 +191,7 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
   // dummy builder instead.
   auto emptyBuilder = [](OpBuilder &, Location, Value, ValueRange) {};
   auto newLoop =
-      rewriter.create<scf::ForOp>(loc, lb, ub, step, newArgs, emptyBuilder);
+      scf::ForOp::create(rewriter, loc, lb, ub, step, newArgs, emptyBuilder);
 
   Block *newBody = newLoop.getBody();
 
@@ -238,18 +238,18 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
   rewriter.setInsertionPointAfter(newLoop);
   Value one;
   if (isa<IndexType>(step.getType())) {
-    one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    one = arith::ConstantIndexOp::create(rewriter, loc, 1);
   } else {
-    one = rewriter.create<arith::ConstantIntOp>(loc, step.getType(), 1);
+    one = arith::ConstantIntOp::create(rewriter, loc, step.getType(), 1);
   }
 
-  Value stepDec = rewriter.create<arith::SubIOp>(loc, step, one);
-  Value len = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  len = rewriter.create<arith::AddIOp>(loc, len, stepDec);
-  len = rewriter.create<arith::DivSIOp>(loc, len, step);
-  len = rewriter.create<arith::SubIOp>(loc, len, one);
-  Value res = rewriter.create<arith::MulIOp>(loc, len, step);
-  res = rewriter.create<arith::AddIOp>(loc, lb, res);
+  Value stepDec = arith::SubIOp::create(rewriter, loc, step, one);
+  Value len = arith::SubIOp::create(rewriter, loc, ub, lb);
+  len = arith::AddIOp::create(rewriter, loc, len, stepDec);
+  len = arith::DivSIOp::create(rewriter, loc, len, step);
+  len = arith::SubIOp::create(rewriter, loc, len, one);
+  Value res = arith::MulIOp::create(rewriter, loc, len, step);
+  res = arith::AddIOp::create(rewriter, loc, lb, res);
 
   // Reconstruct `scf.while` results, inserting final induction var value
   // into proper place.
diff --git a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
index f829208ce8798..707ed836cf63e 100644
--- a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
@@ -96,7 +96,7 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       condOp.getArgs(), [&](Value arg) { return mapper.lookupOrDefault(arg); });
 
   // Create rotated while loop.
-  auto newLoopOp = rewriter.create<scf::WhileOp>(
+  auto newLoopOp = scf::WhileOp::create(rewriter,
       whileOp.getLoc(), whileOp.getResultTypes(), clonedCondArgs,
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Rotate and move the loop body into before block.
@@ -109,21 +109,21 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       },
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Pass through values.
-        builder.create<scf::YieldOp>(loc, args);
+        scf::YieldOp::create(builder, loc, args);
       });
 
   // Create zero-trip-check and move the while loop in.
-  auto ifOp = rewriter.create<scf::IfOp>(
+  auto ifOp = scf::IfOp::create(rewriter,
       whileOp.getLoc(), clonedCondition,
       [&](OpBuilder &builder, Location loc) {
         // Then runs the while loop.
         rewriter.moveOpBefore(newLoopOp, builder.getInsertionBlock(),
                               builder.getInsertionPoint());
-        builder.create<scf::YieldOp>(loc, newLoopOp.getResults());
+        scf::YieldOp::create(builder, loc, newLoopOp.getResults());
       },
       [&](OpBuilder &builder, Location loc) {
         // Else returns the results from precondition.
-        builder.create<scf::YieldOp>(loc, clonedCondArgs);
+        scf::YieldOp::create(builder, loc, clonedCondArgs);
       });
 
   rewriter.replaceOp(whileOp, ifOp);
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index f4047be68ccf2..6a387ddf8908a 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -153,7 +153,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
       FunctionType::get(rewriter.getContext(), outlinedFuncArgTypes,
                         originalTerminator->getOperandTypes());
   auto outlinedFunc =
-      rewriter.create<func::FuncOp>(loc, funcName, outlinedFuncType);
+      func::FuncOp::create(rewriter, loc, funcName, outlinedFuncType);
   Block *outlinedFuncBody = outlinedFunc.addEntryBlock();
 
   // Merge blocks while replacing the original block operands.
@@ -168,7 +168,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
         outlinedFuncBlockArgs.take_front(numOriginalBlockArguments));
     // Explicitly set up a new ReturnOp terminator.
     rewriter.setInsertionPointToEnd(outlinedFuncBody);
-    rewriter.create<func::ReturnOp>(loc, originalTerminator->getResultTypes(),
+    func::ReturnOp::create(rewriter, loc, originalTerminator->getResultTypes(),
                                     originalTerminator->getOperands());
   }
 
@@ -185,7 +185,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
     SmallVector<Value> callValues;
     llvm::append_range(callValues, newBlock->getArguments());
     llvm::append_range(callValues, outlinedValues);
-    auto call = rewriter.create<func::CallOp>(loc, outlinedFunc, callValues);
+    auto call = func::CallOp::create(rewriter, loc, outlinedFunc, callValues);
     if (callOp)
       *callOp = call;
 
@@ -274,12 +274,12 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
 
-  Value divisorMinusOneCst = builder.create<arith::ConstantOp>(
+  Value divisorMinusOneCst = arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(dividend.getType(), divisor - 1));
-  Value divisorCst = builder.create<arith::ConstantOp>(
+  Value divisorCst = arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(dividend.getType(), divisor));
-  Value sum = builder.create<arith::AddIOp>(loc, dividend, divisorMinusOneCst);
-  return builder.create<arith::DivUIOp>(loc, sum, divisorCst);
+  Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOneCst);
+  return arith::DivUIOp::create(builder, loc, sum, divisorCst);
 }
 
 // Build the IR that performs ceil division of a positive value by another
@@ -290,11 +290,11 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
                              Value divisor) {
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
-  Value cstOne = builder.create<arith::ConstantOp>(
+  Value cstOne = arith::ConstantOp::create(builder,
       loc, builder.getOneAttr(dividend.getType()));
-  Value divisorMinusOne = builder.create<arith::SubIOp>(loc, divisor, cstOne);
-  Value sum = builder.create<arith::AddIOp>(loc, dividend, divisorMinusOne);
-  return builder.create<arith::DivUIOp>(loc, sum, divisor);
+  Value divisorMinusOne = arith::SubIOp::create(builder, loc, divisor, cstOne);
+  Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOne);
+  return arith::DivUIOp::create(builder, loc, sum, divisor);
 }
 
 /// Returns the trip count of `forOp` if its' low bound, high bound and step are
@@ -404,7 +404,7 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'upperBoundUnrolled' and set epilogue loop flag.
     generateEpilogueLoop = upperBoundUnrolledCst < ubCst;
     if (generateEpilogueLoop)
-      upperBoundUnrolled = boundsBuilder.create<arith::ConstantOp>(
+      upperBoundUnrolled = arith::ConstantOp::create(boundsBuilder,
           loc, boundsBuilder.getIntegerAttr(forOp.getUpperBound().getType(),
                                             upperBoundUnrolledCst));
     else
@@ -413,7 +413,7 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'stepUnrolled'.
     stepUnrolled = stepCst == stepUnrolledCst
                        ? step
-                       : boundsBuilder.create<arith::ConstantOp>(
+                       : arith::ConstantOp::create(boundsBuilder,
                              loc, boundsBuilder.getIntegerAttr(
                                       step.getType(), stepUnrolledCst));
   } else {
@@ -423,22 +423,22 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     auto lowerBound = forOp.getLowerBound();
     auto upperBound = forOp.getUpperBound();
     Value diff =
-        boundsBuilder.create<arith::SubIOp>(loc, upperBound, lowerBound);
+        arith::SubIOp::create(boundsBuilder, loc, upperBound, lowerBound);
     Value tripCount = ceilDivPositive(boundsBuilder, loc, diff, step);
-    Value unrollFactorCst = boundsBuilder.create<arith::ConstantOp>(
+    Value unrollFactorCst = arith::ConstantOp::create(boundsBuilder,
         loc, boundsBuilder.getIntegerAttr(tripCount.getType(), unrollFactor));
     Value tripCountRem =
-        boundsBuilder.create<arith::RemSIOp>(loc, tripCount, unrollFactorCst);
+        arith::RemSIOp::create(boundsBuilder, loc, tripCount, unrollFactorCst);
     // Compute tripCountEvenMultiple = tripCount - (tripCount % unrollFactor)
     Value tripCountEvenMultiple =
-        boundsBuilder.create<arith::SubIOp>(loc, tripCount, tripCountRem);
+        arith::SubIOp::create(boundsBuilder, loc, tripCount, tripCountRem);
     // Compute upperBoundUnrolled = lowerBound + tripCountEvenMultiple * step
-    upperBoundUnrolled = boundsBuilder.create<arith::AddIOp>(
+    upperBoundUnrolled = arith::AddIOp::create(boundsBuilder,
         loc, lowerBound,
-        boundsBuilder.create<arith::MulIOp>(loc, tripCountEvenMultiple, step));
+        arith::MulIOp::create(boundsBuilder, loc, tripCountEvenMultiple, step));
     // Scale 'step' by 'unrollFactor'.
     stepUnrolled =
-        boundsBuilder.create<arith::MulIOp>(loc, step, unrollFactorCst);
+        arith::MulIOp::create(boundsBuilder, loc, step, unrollFactorCst);
   }
 
   UnrolledLoopInfo resultLoops;
@@ -474,11 +474,11 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
       forOp.getBody(), forOp.getInductionVar(), unrollFactor,
       [&](unsigned i, Value iv, OpBuilder b) {
         // iv' = iv + step * i;
-        auto stride = b.create<arith::MulIOp>(
+        auto stride = arith::MulIOp::create(b,
             loc, step,
-            b.create<arith::ConstantOp>(loc,
+            arith::ConstantOp::create(b, loc,
                                         b.getIntegerAttr(iv.getType(), i)));
-        return b.create<arith::AddIOp>(loc, iv, stride);
+        return arith::AddIOp::create(b, loc, iv, stride);
       },
       annotateFn, iterArgs, yieldedValues);
   // Promote the loop body up if this has turned into a single iteration loop.
@@ -781,13 +781,13 @@ void mlir::denormalizeInductionVariable(RewriterBase &rewriter, Location loc,
   if (!isStepOne) {
     Value origStepValue =
         getValueOrCreateConstantIntOp(rewriter, loc, origStep);
-    scaled = rewriter.create<arith::MulIOp>(loc, normalizedIv, origStepValue);
+    scaled = arith::MulIOp::create(rewriter, loc, normalizedIv, origStepValue);
     preserve.insert(scaled.getDefiningOp());
   }
   denormalizedIv = scaled;
   if (!isZeroBased) {
     Value origLbValue = getValueOrCreateConstantIntOp(rewriter, loc, origLb);
-    denormalizedIv = rewriter.create<arith::AddIOp>(loc, scaled, origLbValue);
+    denormalizedIv = arith::AddIOp::create(rewriter, loc, scaled, origLbValue);
     preserve.insert(denormalizedIv.getDefiningOp());
   }
 
@@ -824,7 +824,7 @@ static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc,
       continue;
     if (productOf)
       productOf =
-          rewriter.create<arith::MulIOp>(loc, productOf.value(), v).getResult();
+          arith::MulIOp::create(rewriter, loc, productOf.value(), v).getResult();
     else
       productOf = v;
   }
@@ -851,7 +851,7 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
 
   if (linearizedIv.getType().isIndex()) {
     Operation *delinearizedOp =
-        rewriter.create<affine::AffineDelinearizeIndexOp>(loc, linearizedIv,
+        affine::AffineDelinearizeIndexOp::create(rewriter, loc, linearizedIv,
                                                           ubs);
     auto resultVals = llvm::map_to_vector(
         delinearizedOp->getResults(), [](OpResult r) -> Value { return r; });
@@ -874,7 +874,7 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
     if (!isUbOne.test(index)) {
       break;
     }
-    delinearizedIvs[index] = rewriter.create<arith::ConstantOp>(
+    delinearizedIvs[index] = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(ub.getType()));
     numLeadingOneUbs++;
   }
@@ -883,16 +883,16 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
   for (unsigned i = numLeadingOneUbs, e = ubs.size(); i < e; ++i) {
     unsigned idx = ubs.size() - (i - numLeadingOneUbs) - 1;
     if (i != numLeadingOneUbs && !isUbOne.test(idx + 1)) {
-      previous = rewriter.create<arith::DivSIOp>(loc, previous, ubs[idx + 1]);
+      previous = arith::DivSIOp::create(rewriter, loc, previous, ubs[idx + 1]);
       preservedUsers.insert(previous.getDefiningOp());
     }
     Value iv = previous;
     if (i != e - 1) {
       if (!isUbOne.test(idx)) {
-        iv = rewriter.create<arith::RemSIOp>(loc, previous, ubs[idx]);
+        iv = arith::RemSIOp::create(rewriter, loc, previous, ubs[idx]);
         preservedUsers.insert(iv.getDefiningOp());
       } else {
-        iv = rewriter.create<arith::ConstantOp>(
+        iv = arith::ConstantOp::create(rewriter,
             loc, rewriter.getZeroAttr(ubs[idx].getType()));
       }
     }
@@ -1093,12 +1093,12 @@ void mlir::collapseParallelLoops(
 
   // Combine iteration spaces.
   SmallVector<Value, 3> lowerBounds, upperBounds, steps;
-  auto cst0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto cst1 = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto cst0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto cst1 = arith::ConstantIndexOp::create(rewriter, loc, 1);
   for (auto &sortedDimension : sortedDimensions) {
-    Value newUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value newUpperBound = arith::ConstantIndexOp::create(rewriter, loc, 1);
     for (auto idx : sortedDimension) {
-      newUpperBound = rewriter.create<arith::MulIOp>(
+      newUpperBound = arith::MulIOp::create(rewriter,
           loc, newUpperBound, normalizedUpperBounds[idx]);
     }
     lowerBounds.push_back(cst0);
@@ -1112,7 +1112,7 @@ void mlir::collapseParallelLoops(
   // value. The remainders then determine based on that range, which iteration
   // of the original induction value this represents. This is a normalized value
   // that is un-normalized already by the previous logic.
-  auto newPloop = rewriter.create<scf::ParallelOp>(
+  auto newPloop = scf::ParallelOp::create(rewriter,
       loc, lowerBounds, upperBounds, steps,
       [&](OpBuilder &insideBuilder, Location, ValueRange ploopIVs) {
         for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) {
@@ -1123,14 +1123,14 @@ void mlir::collapseParallelLoops(
             unsigned idx = combinedDimensions[i][j];
 
             // Determine the current induction value's current loop iteration
-            Value iv = insideBuilder.create<arith::RemSIOp>(
+            Value iv = arith::RemSIOp::create(insideBuilder,
                 loc, previous, normalizedUpperBounds[idx]);
             replaceAllUsesInRegionWith(loops.getBody()->getArgument(idx), iv,
                                        loops.getRegion());
 
             // Remove the effect of the current induction value to prepare for
             // the next value.
-            previous = insideBuilder.create<arith::DivSIOp>(
+            previous = arith::DivSIOp::create(insideBuilder,
                 loc, previous, normalizedUpperBounds[idx]);
           }
 
@@ -1241,7 +1241,7 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
   auto iv = forOp.getInductionVar();
 
   OpBuilder b(forOp);
-  forOp.setStep(b.create<arith::MulIOp>(forOp.getLoc(), originalStep, factor));
+  forOp.setStep(arith::MulIOp::create(b, forOp.getLoc(), originalStep, factor));
 
   Loops innerLoops;
   for (auto t : targets) {
@@ -1251,12 +1251,12 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
 
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator((t.getBody()));
-    Value stepped = b.create<arith::AddIOp>(t.getLoc(), iv, forOp.getStep());
+    Value stepped = arith::AddIOp::create(b, t.getLoc(), iv, forOp.getStep());
     Value ub =
-        b.create<arith::MinSIOp>(t.getLoc(), forOp.getUpperBound(), stepped);
+        arith::MinSIOp::create(b, t.getLoc(), forOp.getUpperBound(), stepped);
 
     // Splice [begin, begin + nOps - 1) into `newForOp` and replace uses.
-    auto newForOp = b.create<scf::ForOp>(t.getLoc(), iv, ub, originalStep);
+    auto newForOp = scf::ForOp::create(b, t.getLoc(), iv, ub, originalStep);
     newForOp.getBody()->getOperations().splice(
         newForOp.getBody()->getOperations().begin(),
         t.getBody()->getOperations(), begin, std::next(begin, nOps - 1));
@@ -1343,7 +1343,7 @@ TileLoops mlir::extractFixedOuterLoops(scf::ForOp rootForOp,
     auto forOp = forOps[i];
     OpBuilder builder(forOp);
     auto loc = forOp.getLoc();
-    Value diff = builder.create<arith::SubIOp>(loc, forOp.getUpperBound(),
+    Value diff = arith::SubIOp::create(builder, loc, forOp.getUpperBound(),
                                                forOp.getLowerBound());
     Value numIterations = ceilDivPositive(builder, loc, diff, forOp.getStep());
     Value iterationsPerBlock =
@@ -1376,7 +1376,7 @@ scf::ForallOp mlir::fuseIndependentSiblingForallLoops(scf::ForallOp target,
 
   // Create a new scf.forall op after the source loop.
   rewriter.setInsertionPointAfter(source);
-  scf::ForallOp fusedLoop = rewriter.create<scf::ForallOp>(
+  scf::ForallOp fusedLoop = scf::ForallOp::create(rewriter,
       source.getLoc(), source.getMixedLowerBound(), source.getMixedUpperBound(),
       source.getMixedStep(), fusedOuts, source.getMapping());
 
@@ -1429,7 +1429,7 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   // Create a new scf.for op after the source loop (with scf.yield terminator
   // (without arguments) only in case its init_args is empty).
   rewriter.setInsertionPointAfter(source);
-  scf::ForOp fusedLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp fusedLoop = scf::ForOp::create(rewriter,
       source.getLoc(), source.getLowerBound(), source.getUpperBound(),
       source.getStep(), fusedInitArgs);
 
@@ -1456,7 +1456,7 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   for (Value operand : source.getBody()->getTerminator()->getOperands())
     yieldResults.push_back(mapping.lookupOrDefault(operand));
   if (!yieldResults.empty())
-    rewriter.create<scf::YieldOp>(source.getLoc(), yieldResults);
+    scf::YieldOp::create(rewriter, source.getLoc(), yieldResults);
 
   // Replace old loops by substituting their uses by results of the fused loop.
   rewriter.replaceOp(target, fusedLoop.getResults().take_front(numTargetOuts));
@@ -1487,7 +1487,7 @@ FailureOr<scf::ForallOp> mlir::normalizeForallOp(RewriterBase &rewriter,
 
   // Use the normalized builder since the lower bounds are always 0 and the
   // steps are always 1.
-  auto normalizedForallOp = rewriter.create<scf::ForallOp>(
+  auto normalizedForallOp = scf::ForallOp::create(rewriter,
       loc, newUbs, forallOp.getOutputs(), forallOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
 
diff --git a/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp b/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
index 66eed861b2bb7..48c0b1ed8e14f 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
@@ -30,14 +30,14 @@ Operation *SMTDialect::materializeConstant(OpBuilder &builder, Attribute value,
     if (auto attrValue = dyn_cast<BitVectorAttr>(value)) {
       assert(bvType == attrValue.getType() &&
              "attribute and desired result types have to match");
-      return builder.create<BVConstantOp>(loc, attrValue);
+      return BVConstantOp::create(builder, loc, attrValue);
     }
   }
 
   // BoolType constants can materialize into smt.constant
   if (auto boolType = dyn_cast<BoolType>(type)) {
     if (auto attrValue = dyn_cast<BoolAttr>(value))
-      return builder.create<BoolConstantOp>(loc, attrValue);
+      return BoolConstantOp::create(builder, loc, attrValue);
   }
 
   return nullptr;
diff --git a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
index 8977a3abc125d..c517ef2b18f6f 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
@@ -405,7 +405,7 @@ static void buildQuantifier(
         SmallVector<Location>(boundVarTypes.size(), odsState.location));
     Value returnVal =
         bodyBuilder(odsBuilder, odsState.location, block->getArguments());
-    odsBuilder.create<smt::YieldOp>(odsState.location, returnVal);
+    smt::YieldOp::create(odsBuilder, odsState.location, returnVal);
   }
   if (patternBuilder) {
     Region *region = odsState.addRegion();
@@ -416,7 +416,7 @@ static void buildQuantifier(
         SmallVector<Location>(boundVarTypes.size(), odsState.location));
     ValueRange returnVals =
         patternBuilder(odsBuilder, odsState.location, block->getArguments());
-    odsBuilder.create<smt::YieldOp>(odsState.location, returnVals);
+    smt::YieldOp::create(odsBuilder, odsState.location, returnVals);
   }
 }
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
index 371456552b5b5..cacc52d0feaf0 100644
--- a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
@@ -391,7 +391,7 @@ void LoopOp::addEntryAndMergeBlock(OpBuilder &builder) {
   builder.createBlock(&getBody());
 
   // Add a spirv.mlir.merge op into the merge block.
-  builder.create<spirv::MergeOp>(getLoc());
+  spirv::MergeOp::create(builder, getLoc());
 }
 
 //===----------------------------------------------------------------------===//
@@ -543,7 +543,7 @@ void SelectionOp::addMergeBlock(OpBuilder &builder) {
   builder.createBlock(&getBody());
 
   // Add a spirv.mlir.merge op into the merge block.
-  builder.create<spirv::MergeOp>(getLoc());
+  spirv::MergeOp::create(builder, getLoc());
 }
 
 SelectionOp
@@ -551,7 +551,7 @@ SelectionOp::createIfThen(Location loc, Value condition,
                           function_ref<void(OpBuilder &builder)> thenBody,
                           OpBuilder &builder) {
   auto selectionOp =
-      builder.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
+      spirv::SelectionOp::create(builder, loc, spirv::SelectionControl::None);
 
   selectionOp.addMergeBlock(builder);
   Block *mergeBlock = selectionOp.getMergeBlock();
@@ -562,14 +562,14 @@ SelectionOp::createIfThen(Location loc, Value condition,
     OpBuilder::InsertionGuard guard(builder);
     thenBlock = builder.createBlock(mergeBlock);
     thenBody(builder);
-    builder.create<spirv::BranchOp>(loc, mergeBlock);
+    spirv::BranchOp::create(builder, loc, mergeBlock);
   }
 
   // Build the header block.
   {
     OpBuilder::InsertionGuard guard(builder);
     builder.createBlock(thenBlock);
-    builder.create<spirv::BranchConditionalOp>(
+    spirv::BranchConditionalOp::create(builder,
         loc, condition, thenBlock,
         /*trueArguments=*/ArrayRef<Value>(), mergeBlock,
         /*falseArguments=*/ArrayRef<Value>());
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index 03af61c81ae6c..b1bf37092d2b6 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -179,16 +179,16 @@ struct IAddCarryFold final : OpRewritePattern<spirv::IAddCarryOp> {
       return failure();
 
     Value addsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, adds);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, adds);
 
     Value carrysVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, carrys);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, carrys);
 
     // Create empty struct
-    Value undef = rewriter.create<spirv::UndefOp>(loc, op.getType());
+    Value undef = spirv::UndefOp::create(rewriter, loc, op.getType());
     // Fill in adds at id 0
     Value intermediate =
-        rewriter.create<spirv::CompositeInsertOp>(loc, addsVal, undef, 0);
+        spirv::CompositeInsertOp::create(rewriter, loc, addsVal, undef, 0);
     // Fill in carrys at id 1
     rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(op, carrysVal,
                                                           intermediate, 1);
@@ -261,16 +261,16 @@ struct MulExtendedFold final : OpRewritePattern<MulOp> {
       return failure();
 
     Value lowBitsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, lowBits);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, lowBits);
 
     Value highBitsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, highBits);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, highBits);
 
     // Create empty struct
-    Value undef = rewriter.create<spirv::UndefOp>(loc, op.getType());
+    Value undef = spirv::UndefOp::create(rewriter, loc, op.getType());
     // Fill in lowBits at id 0
     Value intermediate =
-        rewriter.create<spirv::CompositeInsertOp>(loc, lowBitsVal, undef, 0);
+        spirv::CompositeInsertOp::create(rewriter, loc, lowBitsVal, undef, 0);
     // Fill in highBits at id 1
     rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(op, highBitsVal,
                                                           intermediate, 1);
@@ -1310,10 +1310,10 @@ struct ConvertSelectionOpToSelect final : OpRewritePattern<spirv::SelectionOp> {
     auto storeOpAttributes =
         cast<spirv::StoreOp>(trueBlock->front())->getAttrs();
 
-    auto selectOp = rewriter.create<spirv::SelectOp>(
+    auto selectOp = spirv::SelectOp::create(rewriter,
         selectionOp.getLoc(), trueValue.getType(),
         brConditionalOp.getCondition(), trueValue, falseValue);
-    rewriter.create<spirv::StoreOp>(selectOp.getLoc(), ptrValue,
+    spirv::StoreOp::create(rewriter, selectOp.getLoc(), ptrValue,
                                     selectOp.getResult(), storeOpAttributes);
 
     // `spirv.mlir.selection` is not needed anymore.
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
index 88c7adf3dfcb3..e85ea3455248f 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
@@ -944,12 +944,12 @@ Operation *SPIRVDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (!spirv::ConstantOp::isBuildableWith(type))
     return nullptr;
 
-  return builder.create<spirv::ConstantOp>(loc, type, value);
+  return spirv::ConstantOp::create(builder, loc, type, value);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index eb2974d62fdd1..f1299c8a926cd 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -653,25 +653,25 @@ spirv::ConstantOp spirv::ConstantOp::getZero(Type type, Location loc,
   if (auto intType = llvm::dyn_cast<IntegerType>(type)) {
     unsigned width = intType.getWidth();
     if (width == 1)
-      return builder.create<spirv::ConstantOp>(loc, type,
+      return spirv::ConstantOp::create(builder, loc, type,
                                                builder.getBoolAttr(false));
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getFloatAttr(floatType, 0.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 0).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 0.0).getValue()));
@@ -686,25 +686,25 @@ spirv::ConstantOp spirv::ConstantOp::getOne(Type type, Location loc,
   if (auto intType = llvm::dyn_cast<IntegerType>(type)) {
     unsigned width = intType.getWidth();
     if (width == 1)
-      return builder.create<spirv::ConstantOp>(loc, type,
+      return spirv::ConstantOp::create(builder, loc, type,
                                                builder.getBoolAttr(true));
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getFloatAttr(floatType, 1.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 1).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 1.0).getValue()));
@@ -1886,7 +1886,7 @@ ParseResult spirv::SpecConstantOperationOp::parse(OpAsmParser &parser,
 
   OpBuilder builder(parser.getContext());
   builder.setInsertionPointToEnd(&block);
-  builder.create<spirv::YieldOp>(wrappedOp->getLoc(), wrappedOp->getResult(0));
+  spirv::YieldOp::create(builder, wrappedOp->getLoc(), wrappedOp->getResult(0));
   result.location = wrappedOp->getLoc();
 
   result.addTypes(wrappedOp->getResult(0).getType());
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 71122f8e20512..4e53cef31d6ae 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -109,7 +109,7 @@ OwningOpRef<spirv::ModuleOp> combine(ArrayRef<spirv::ModuleOp> inputModules,
     }
   }
 
-  auto combinedModule = combinedModuleBuilder.create<spirv::ModuleOp>(
+  auto combinedModule = spirv::ModuleOp::create(combinedModuleBuilder,
       firstModule.getLoc(), addressingModel, memoryModel, vceTriple);
   combinedModuleBuilder.setInsertionPointToStart(combinedModule.getBody());
 
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 6fd20466e36e3..735af7ab80ef3 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -71,7 +71,7 @@ createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
   varType =
       spirv::PointerType::get(varPointeeType, varPtrType.getStorageClass());
 
-  return builder.create<spirv::GlobalVariableOp>(
+  return spirv::GlobalVariableOp::create(builder,
       funcOp.getLoc(), varType, varName, abiInfo.getDescriptorSet(),
       abiInfo.getBinding());
 }
@@ -147,7 +147,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     return funcOp.emitRemark("lower entry point failure: could not select "
                              "execution model based on 'spirv.target_env'");
 
-  builder.create<spirv::EntryPointOp>(funcOp.getLoc(), *executionModel, funcOp,
+  spirv::EntryPointOp::create(builder, funcOp.getLoc(), *executionModel, funcOp,
                                       interfaceVars);
 
   // Specifies the spirv.ExecutionModeOp.
@@ -155,7 +155,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::LocalSize);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
                                              spirv::ExecutionMode::LocalSize,
                                              workgroupSizeAttr.asArrayRef());
       // Erase workgroup size.
@@ -168,7 +168,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SubgroupSize);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
                                              spirv::ExecutionMode::SubgroupSize,
                                              *subgroupSize);
       // Erase subgroup size.
@@ -181,7 +181,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SignedZeroInfNanPreserve);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(
+      spirv::ExecutionModeOp::create(builder,
           funcOp.getLoc(), funcOp,
           spirv::ExecutionMode::SignedZeroInfNanPreserve, *targetWidth);
       // Erase target width.
@@ -260,7 +260,7 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
 
     // Insert spirv::AddressOf and spirv::AccessChain operations.
     Value replacement =
-        rewriter.create<spirv::AddressOfOp>(funcOp.getLoc(), var);
+        spirv::AddressOfOp::create(rewriter, funcOp.getLoc(), var);
     // Check if the arg is a scalar or vector type. In that case, the value
     // needs to be loaded into registers.
     // TODO: This is loading value of the scalar into registers
@@ -270,9 +270,9 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     if (cast<spirv::SPIRVType>(argType.value()).isScalarOrVector()) {
       auto zero =
           spirv::ConstantOp::getZero(indexType, funcOp.getLoc(), rewriter);
-      auto loadPtr = rewriter.create<spirv::AccessChainOp>(
+      auto loadPtr = spirv::AccessChainOp::create(rewriter,
           funcOp.getLoc(), replacement, zero.getConstant());
-      replacement = rewriter.create<spirv::LoadOp>(funcOp.getLoc(), loadPtr);
+      replacement = spirv::LoadOp::create(rewriter, funcOp.getLoc(), loadPtr);
     }
     signatureConverter.remapInput(argType.index(), replacement);
   }
@@ -309,7 +309,7 @@ void LowerABIAttributesPass::runOnOperation() {
                                             ValueRange inputs, Location loc) {
     if (inputs.size() != 1 || !isa<spirv::PointerType>(inputs[0].getType()))
       return Value();
-    return builder.create<spirv::BitcastOp>(loc, type, inputs[0]).getResult();
+    return spirv::BitcastOp::create(builder, loc, type, inputs[0]).getResult();
   });
 
   RewritePatternSet patterns(context);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
index 2e31172ab940b..637be91a82e33 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
@@ -66,7 +66,7 @@ void RewriteInsertsPass::runOnOperation() {
       operands.push_back(insertionOp.getObject());
 
     OpBuilder builder(lastCompositeInsertOp);
-    auto compositeConstructOp = builder.create<spirv::CompositeConstructOp>(
+    auto compositeConstructOp = spirv::CompositeConstructOp::create(builder,
         location, compositeType, operands);
 
     lastCompositeInsertOp.replaceAllUsesWith(
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index 1e7bb046d3752..4e398ca92be0e 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -672,21 +672,21 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
                               Location loc) {
   // We can only cast one value in SPIR-V.
   if (inputs.size() != 1) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   Value input = inputs.front();
 
   // Only support integer types for now. Floating point types to be implemented.
   if (!isa<IntegerType>(type)) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   auto inputType = cast<IntegerType>(input.getType());
 
   auto scalarType = dyn_cast<spirv::ScalarType>(type);
   if (!scalarType) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -694,14 +694,14 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // truncating to go back so we don't need to worry about the signedness.
   // For extension, we cannot have enough signal here to decide which op to use.
   if (inputType.getIntOrFloatBitWidth() < scalarType.getIntOrFloatBitWidth()) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
   // Boolean values would need to use different ops than normal integer values.
   if (type.isInteger(1)) {
     Value one = spirv::ConstantOp::getOne(inputType, loc, builder);
-    return builder.create<spirv::IEqualOp>(loc, input, one);
+    return spirv::IEqualOp::create(builder, loc, input, one);
   }
 
   // Check that the source integer type is supported by the environment.
@@ -711,7 +711,7 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   scalarType.getCapabilities(caps);
   if (failed(checkCapabilityRequirements(type, targetEnv, caps)) ||
       failed(checkExtensionRequirements(type, targetEnv, exts))) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -719,9 +719,9 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // care about signedness here. Still try to use a corresponding op for better
   // consistency though.
   if (type.isSignedInteger()) {
-    return builder.create<spirv::SConvertOp>(loc, type, input);
+    return spirv::SConvertOp::create(builder, loc, type, input);
   }
-  return builder.create<spirv::UConvertOp>(loc, type, input);
+  return spirv::UConvertOp::create(builder, loc, type, input);
 }
 
 //===----------------------------------------------------------------------===//
@@ -773,7 +773,7 @@ getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
                                            spirv::StorageClass::Input);
     std::string name = getBuiltinVarName(builtin, prefix, suffix);
     newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+        spirv::GlobalVariableOp::create(builder, loc, ptrType, name, builtin);
     break;
   }
   case spirv::BuiltIn::SubgroupId:
@@ -784,7 +784,7 @@ getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
         spirv::PointerType::get(integerType, spirv::StorageClass::Input);
     std::string name = getBuiltinVarName(builtin, prefix, suffix);
     newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+        spirv::GlobalVariableOp::create(builder, loc, ptrType, name, builtin);
     break;
   }
   default:
@@ -845,7 +845,7 @@ getOrInsertPushConstantVariable(Location loc, Block &block,
   auto builder = OpBuilder::atBlockBegin(&block, b.getListener());
   auto type = getPushConstantStorageType(elementCount, builder, indexType);
   const char *name = "__push_constant_var__";
-  return builder.create<spirv::GlobalVariableOp>(loc, type, name,
+  return spirv::GlobalVariableOp::create(builder, loc, type, name,
                                                  /*initializer=*/nullptr);
 }
 
@@ -882,7 +882,7 @@ struct FuncOpConversion final : OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted spirv.func op.
-    auto newFuncOp = rewriter.create<spirv::FuncOp>(
+    auto newFuncOp = spirv::FuncOp::create(rewriter,
         funcOp.getLoc(), funcOp.getName(),
         rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
                                  resultType ? TypeRange(resultType)
@@ -922,7 +922,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
     }
 
     // Create a new func op with the original type and copy the function body.
-    auto newFuncOp = rewriter.create<func::FuncOp>(funcOp.getLoc(),
+    auto newFuncOp = func::FuncOp::create(rewriter, funcOp.getLoc(),
                                                    funcOp.getName(), fnType);
     rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
                                 newFuncOp.end());
@@ -957,7 +957,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto origVecType = dyn_cast<VectorType>(origType);
       if (!origVecType) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = rewriter.create<arith::ConstantOp>(
+        Value result = arith::ConstantOp::create(rewriter,
             loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
@@ -970,7 +970,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto targetShape = getTargetShape(origVecType);
       if (!targetShape) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = rewriter.create<arith::ConstantOp>(
+        Value result = arith::ConstantOp::create(rewriter,
             loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
@@ -985,11 +985,11 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
           llvm::to_vector_of<int64_t, 4>(origVecType.getShape());
 
       // Prepare the result vector.
-      Value result = rewriter.create<arith::ConstantOp>(
+      Value result = arith::ConstantOp::create(rewriter,
           loc, origVecType, rewriter.getZeroAttr(origVecType));
       ++newOpCount;
       // Prepare the placeholder for the new arguments that will be added later.
-      Value dummy = rewriter.create<arith::ConstantOp>(
+      Value dummy = arith::ConstantOp::create(rewriter,
           loc, unrolledType, rewriter.getZeroAttr(unrolledType));
       ++newOpCount;
 
@@ -998,7 +998,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       SmallVector<Type> newTypes;
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        result = rewriter.create<vector::InsertStridedSliceOp>(
+        result = vector::InsertStridedSliceOp::create(rewriter,
             loc, dummy, result, offsets, strides);
         newTypes.push_back(unrolledType);
         unrolledInputNums.push_back(newInputNo);
@@ -1112,12 +1112,12 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
       Value returnValue = returnOp.getOperand(origResultNo);
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        Value result = rewriter.create<vector::ExtractStridedSliceOp>(
+        Value result = vector::ExtractStridedSliceOp::create(rewriter,
             loc, returnValue, offsets, extractShape, strides);
         if (originalShape.size() > 1) {
           SmallVector<int64_t> extractIndices(originalShape.size() - 1, 0);
           result =
-              rewriter.create<vector::ExtractOp>(loc, result, extractIndices);
+              vector::ExtractOp::create(rewriter, loc, result, extractIndices);
         }
         newOperands.push_back(result);
         newTypes.push_back(unrolledType);
@@ -1135,7 +1135,7 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
     // Replace the return op using the new operands. This will automatically
     // update the entry block as well.
     rewriter.replaceOp(returnOp,
-                       rewriter.create<func::ReturnOp>(loc, newOperands));
+                       func::ReturnOp::create(rewriter, loc, newOperands));
 
     return success();
   }
@@ -1160,8 +1160,8 @@ Value mlir::spirv::getBuiltinVariableValue(Operation *op,
   spirv::GlobalVariableOp varOp =
       getOrInsertBuiltinVariable(*parent->getRegion(0).begin(), op->getLoc(),
                                  builtin, integerType, builder, prefix, suffix);
-  Value ptr = builder.create<spirv::AddressOfOp>(op->getLoc(), varOp);
-  return builder.create<spirv::LoadOp>(op->getLoc(), ptr);
+  Value ptr = spirv::AddressOfOp::create(builder, op->getLoc(), varOp);
+  return spirv::LoadOp::create(builder, op->getLoc(), ptr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1182,12 +1182,12 @@ Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
       loc, parent->getRegion(0).front(), elementCount, builder, integerType);
 
   Value zeroOp = spirv::ConstantOp::getZero(integerType, loc, builder);
-  Value offsetOp = builder.create<spirv::ConstantOp>(
+  Value offsetOp = spirv::ConstantOp::create(builder,
       loc, integerType, builder.getI32IntegerAttr(offset));
-  auto addrOp = builder.create<spirv::AddressOfOp>(loc, varOp);
-  auto acOp = builder.create<spirv::AccessChainOp>(
+  auto addrOp = spirv::AddressOfOp::create(builder, loc, varOp);
+  auto acOp = spirv::AccessChainOp::create(builder,
       loc, addrOp, llvm::ArrayRef({zeroOp, offsetOp}));
-  return builder.create<spirv::LoadOp>(loc, acOp);
+  return spirv::LoadOp::create(builder, loc, acOp);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1247,7 +1247,7 @@ Value mlir::spirv::getVulkanElementPtr(const SPIRVTypeConverter &typeConverter,
     linearizedIndices.push_back(
         linearizeIndex(indices, strides, offset, indexType, loc, builder));
   }
-  return builder.create<spirv::AccessChainOp>(loc, basePtr, linearizedIndices);
+  return spirv::AccessChainOp::create(builder, loc, basePtr, linearizedIndices);
 }
 
 Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
@@ -1278,10 +1278,10 @@ Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
       cast<spirv::PointerType>(basePtr.getType()).getPointeeType();
   if (isa<spirv::ArrayType>(pointeeType)) {
     linearizedIndices.push_back(linearIndex);
-    return builder.create<spirv::AccessChainOp>(loc, basePtr,
+    return spirv::AccessChainOp::create(builder, loc, basePtr,
                                                 linearizedIndices);
   }
-  return builder.create<spirv::PtrAccessChainOp>(loc, basePtr, linearIndex,
+  return spirv::PtrAccessChainOp::create(builder, loc, basePtr, linearIndex,
                                                  linearizedIndices);
 }
 
@@ -1468,7 +1468,7 @@ SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr,
       });
   addTargetMaterialization([](OpBuilder &builder, Type type, ValueRange inputs,
                               Location loc) {
-    auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return cast.getResult(0);
   });
 }
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
index af1cf2a1373e3..17f79780afa20 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
@@ -64,16 +64,16 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   //     and 4 additions after constant folding.
   //   - With sign-extended arguments, we end up emitting 8 multiplications and
   //     and 12 additions after CSE.
-  Value cstLowMask = rewriter.create<ConstantOp>(
+  Value cstLowMask = ConstantOp::create(rewriter,
       loc, lhs.getType(), getScalarOrSplatAttr(argTy, (1 << 16) - 1));
   auto getLowDigit = [&rewriter, loc, cstLowMask](Value val) {
-    return rewriter.create<BitwiseAndOp>(loc, val, cstLowMask);
+    return BitwiseAndOp::create(rewriter, loc, val, cstLowMask);
   };
 
-  Value cst16 = rewriter.create<ConstantOp>(loc, lhs.getType(),
+  Value cst16 = ConstantOp::create(rewriter, loc, lhs.getType(),
                                             getScalarOrSplatAttr(argTy, 16));
   auto getHighDigit = [&rewriter, loc, cst16](Value val) {
-    return rewriter.create<ShiftRightLogicalOp>(loc, val, cst16);
+    return ShiftRightLogicalOp::create(rewriter, loc, val, cst16);
   };
 
   auto getSignDigit = [&rewriter, loc, cst16, &getHighDigit](Value val) {
@@ -82,10 +82,10 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
     // fine. We do not have to introduce an extra constant since any
     // value in [15, 32) would do.
     return getHighDigit(
-        rewriter.create<ShiftRightArithmeticOp>(loc, val, cst16));
+        ShiftRightArithmeticOp::create(rewriter, loc, val, cst16));
   };
 
-  Value cst0 = rewriter.create<ConstantOp>(loc, lhs.getType(),
+  Value cst0 = ConstantOp::create(rewriter, loc, lhs.getType(),
                                            getScalarOrSplatAttr(argTy, 0));
 
   Value lhsLow = getLowDigit(lhs);
@@ -108,7 +108,7 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
         continue;
 
       Value &thisResDigit = resultDigits[i + j];
-      Value mul = rewriter.create<IMulOp>(loc, lhsDigit, rhsDigit);
+      Value mul = IMulOp::create(rewriter, loc, lhsDigit, rhsDigit);
       Value current = rewriter.createOrFold<IAddOp>(loc, thisResDigit, mul);
       thisResDigit = getLowDigit(current);
 
@@ -122,13 +122,13 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   }
 
   auto combineDigits = [loc, cst16, &rewriter](Value low, Value high) {
-    Value highBits = rewriter.create<ShiftLeftLogicalOp>(loc, high, cst16);
-    return rewriter.create<BitwiseOrOp>(loc, low, highBits);
+    Value highBits = ShiftLeftLogicalOp::create(rewriter, loc, high, cst16);
+    return BitwiseOrOp::create(rewriter, loc, low, highBits);
   };
   Value low = combineDigits(resultDigits[0], resultDigits[1]);
   Value high = combineDigits(resultDigits[2], resultDigits[3]);
 
-  return rewriter.create<CompositeConstructOp>(
+  return CompositeConstructOp::create(rewriter,
       loc, mulOp->getResultTypes().front(), llvm::ArrayRef({low, high}));
 }
 
@@ -185,16 +185,16 @@ struct ExpandAddCarryPattern final : OpRewritePattern<IAddCarryOp> {
           llvm::formatv("Unexpected integer type for WebGPU: '{0}'", elemTy));
 
     Value one =
-        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 1));
+        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 1));
     Value zero =
-        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 0));
+        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 0));
 
     // Calculate the carry by checking if the addition resulted in an overflow.
-    Value out = rewriter.create<IAddOp>(loc, lhs, rhs);
-    Value cmp = rewriter.create<ULessThanOp>(loc, out, lhs);
-    Value carry = rewriter.create<SelectOp>(loc, cmp, one, zero);
+    Value out = IAddOp::create(rewriter, loc, lhs, rhs);
+    Value cmp = ULessThanOp::create(rewriter, loc, out, lhs);
+    Value carry = SelectOp::create(rewriter, loc, cmp, one, zero);
 
-    Value add = rewriter.create<CompositeConstructOp>(
+    Value add = CompositeConstructOp::create(rewriter,
         loc, op->getResultTypes().front(), llvm::ArrayRef({out, carry}));
 
     rewriter.replaceOp(op, add);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
index 07cf26926a1df..7fe2bd79403f9 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
@@ -384,13 +384,13 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = dstNumBytes / srcNumBytes;
-      auto ratioValue = rewriter.create<spirv::ConstantOp>(
+      auto ratioValue = spirv::ConstantOp::create(rewriter,
           loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
-          rewriter.create<spirv::SDivOp>(loc, indexType, oldIndex, ratioValue);
+          spirv::SDivOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
       indices.push_back(
-          rewriter.create<spirv::SModOp>(loc, indexType, oldIndex, ratioValue));
+          spirv::SModOp::create(rewriter, loc, indexType, oldIndex, ratioValue));
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -411,11 +411,11 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = srcNumBytes / dstNumBytes;
-      auto ratioValue = rewriter.create<spirv::ConstantOp>(
+      auto ratioValue = spirv::ConstantOp::create(rewriter,
           loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
-          rewriter.create<spirv::IMulOp>(loc, indexType, oldIndex, ratioValue);
+          spirv::IMulOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -439,14 +439,14 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
     auto dstElemType = cast<spirv::SPIRVType>(dstPtrType.getPointeeType());
 
     Location loc = loadOp.getLoc();
-    auto newLoadOp = rewriter.create<spirv::LoadOp>(loc, adaptor.getPtr());
+    auto newLoadOp = spirv::LoadOp::create(rewriter, loc, adaptor.getPtr());
     if (srcElemType == dstElemType) {
       rewriter.replaceOp(loadOp, newLoadOp->getResults());
       return success();
     }
 
     if (areSameBitwidthScalarType(srcElemType, dstElemType)) {
-      auto castOp = rewriter.create<spirv::BitcastOp>(loc, srcElemType,
+      auto castOp = spirv::BitcastOp::create(rewriter, loc, srcElemType,
                                                       newLoadOp.getValue());
       rewriter.replaceOp(loadOp, castOp->getResults());
 
@@ -479,14 +479,14 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
       auto indices = llvm::to_vector<4>(acOp.getIndices());
       for (int i = 1; i < ratio; ++i) {
         // Load all subsequent components belonging to this element.
-        indices.back() = rewriter.create<spirv::IAddOp>(
+        indices.back() = spirv::IAddOp::create(rewriter,
             loc, i32Type, indices.back(), oneValue);
-        auto componentAcOp = rewriter.create<spirv::AccessChainOp>(
+        auto componentAcOp = spirv::AccessChainOp::create(rewriter,
             loc, acOp.getBasePtr(), indices);
         // Assuming little endian, this reads lower-ordered bits of the number
         // to lower-numbered components of the vector.
         components.push_back(
-            rewriter.create<spirv::LoadOp>(loc, componentAcOp));
+            spirv::LoadOp::create(rewriter, loc, componentAcOp));
       }
 
       // Create a vector of the components and then cast back to the larger
@@ -514,15 +514,15 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
               castType = VectorType::get({count}, castType);
 
             for (Value &c : components)
-              c = rewriter.create<spirv::BitcastOp>(loc, castType, c);
+              c = spirv::BitcastOp::create(rewriter, loc, castType, c);
           }
         }
-      Value vectorValue = rewriter.create<spirv::CompositeConstructOp>(
+      Value vectorValue = spirv::CompositeConstructOp::create(rewriter,
           loc, vectorType, components);
 
       if (!isa<VectorType>(srcElemType))
         vectorValue =
-            rewriter.create<spirv::BitcastOp>(loc, srcElemType, vectorValue);
+            spirv::BitcastOp::create(rewriter, loc, srcElemType, vectorValue);
       rewriter.replaceOp(loadOp, vectorValue);
       return success();
     }
@@ -550,7 +550,7 @@ struct ConvertStore : public ConvertAliasResource<spirv::StoreOp> {
     Location loc = storeOp.getLoc();
     Value value = adaptor.getValue();
     if (srcElemType != dstElemType)
-      value = rewriter.create<spirv::BitcastOp>(loc, dstElemType, value);
+      value = spirv::BitcastOp::create(rewriter, loc, dstElemType, value);
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, adaptor.getPtr(),
                                                 value, storeOp->getAttrs());
     return success();
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 5f395eef9d601..7339ba5c2379c 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -151,16 +151,16 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (llvm::isa<ShapeType>(type) || isExtentTensorType(type))
-    return builder.create<ConstShapeOp>(
+    return ConstShapeOp::create(builder,
         loc, type, llvm::cast<DenseIntElementsAttr>(value));
   if (llvm::isa<SizeType>(type))
-    return builder.create<ConstSizeOp>(loc, type,
+    return ConstSizeOp::create(builder, loc, type,
                                        llvm::cast<IntegerAttr>(value));
   if (llvm::isa<WitnessType>(type))
-    return builder.create<ConstWitnessOp>(loc, type,
+    return ConstWitnessOp::create(builder, loc, type,
                                           llvm::cast<BoolAttr>(value));
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
@@ -316,7 +316,7 @@ struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
     auto newYieldOp =
         rewriter.replaceOpWithNewOp<AssumingYieldOp>(yieldOp, newYieldOperands);
     rewriter.setInsertionPoint(op);
-    auto newOp = rewriter.create<AssumingOp>(
+    auto newOp = AssumingOp::create(rewriter,
         op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
     newOp.getDoRegion().takeBody(op.getDoRegion());
 
@@ -385,7 +385,7 @@ void AssumingOp::build(
 
   // Build body.
   SmallVector<Value, 2> yieldValues = bodyBuilder(builder, result.location);
-  builder.create<AssumingYieldOp>(result.location, yieldValues);
+  AssumingYieldOp::create(builder, result.location, yieldValues);
 
   SmallVector<Type, 2> assumingTypes;
   for (Value v : yieldValues)
@@ -736,13 +736,13 @@ struct BroadcastForwardSingleOperandPattern
     if (replacement.getType() != op.getType()) {
       auto loc = op.getLoc();
       if (llvm::isa<ShapeType>(op.getType())) {
-        replacement = rewriter.create<FromExtentTensorOp>(loc, replacement);
+        replacement = FromExtentTensorOp::create(rewriter, loc, replacement);
       } else {
         assert(!llvm::isa<ShapeType>(op.getType()) &&
                !llvm::isa<ShapeType>(replacement.getType()) &&
                "expect extent tensor cast");
         replacement =
-            rewriter.create<tensor::CastOp>(loc, op.getType(), replacement);
+            tensor::CastOp::create(rewriter, loc, op.getType(), replacement);
       }
     }
 
@@ -780,7 +780,7 @@ struct BroadcastFoldConstantOperandsPattern
     auto foldedConstantOperandsTy = RankedTensorType::get(
         {static_cast<int64_t>(foldedConstantShape.size())},
         rewriter.getIndexType());
-    newShapeOperands.push_back(rewriter.create<ConstShapeOp>(
+    newShapeOperands.push_back(ConstShapeOp::create(rewriter,
         op.getLoc(), foldedConstantOperandsTy,
         rewriter.getIndexTensorAttr(foldedConstantShape)));
     rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(),
@@ -845,7 +845,7 @@ struct BroadcastConcretizeResultTypePattern
       }
     }
 
-    auto newOp = rewriter.create<BroadcastOp>(
+    auto newOp = BroadcastOp::create(rewriter,
         op.getLoc(), getExtentTensorType(getContext(), maxRank),
         op.getShapes());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
@@ -1354,11 +1354,11 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
   auto loc = result.location;
   auto dimAttr = builder.getIndexAttr(dim);
   if (llvm::isa<ShapeType>(shape.getType())) {
-    Value dim = builder.create<ConstSizeOp>(loc, dimAttr);
+    Value dim = ConstSizeOp::create(builder, loc, dimAttr);
     build(builder, result, builder.getType<SizeType>(), shape, dim);
   } else {
     Value dim =
-        builder.create<arith::ConstantOp>(loc, builder.getIndexType(), dimAttr);
+        arith::ConstantOp::create(builder, loc, builder.getIndexType(), dimAttr);
     build(builder, result, builder.getIndexType(), shape, dim);
   }
 }
@@ -1708,7 +1708,7 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
                                   rewriter.getIndexTensorAttr(type.getShape()))
             .getResult();
     if (constShape.getType() != op.getResult().getType())
-      constShape = rewriter.create<tensor::CastOp>(
+      constShape = tensor::CastOp::create(rewriter,
           loc, op.getResult().getType(), constShape);
     rewriter.replaceOp(op, constShape);
     return success();
@@ -1751,10 +1751,10 @@ struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
 
     if (opTensorTy != shapeTensorTy) {
       if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
-        shape = rewriter.create<tensor::CastOp>(op.getLoc(), opTensorTy, shape);
+        shape = tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
       else if (!isExtentTensorType(shapeTensorTy))
         shape =
-            rewriter.create<arith::IndexCastOp>(op.getLoc(), opTensorTy, shape);
+            arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
     }
 
     rewriter.replaceOp(op, shape);
diff --git a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
index 8a471c12d21e4..5533a7e914bf0 100644
--- a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -57,7 +57,7 @@ struct AssumingOpInterface
 
     // Create new op and move over region.
     TypeRange newResultTypes(yieldOp.getOperands());
-    auto newOp = rewriter.create<shape::AssumingOp>(
+    auto newOp = shape::AssumingOp::create(rewriter,
         op->getLoc(), newResultTypes, assumingOp.getWitness());
     newOp.getDoRegion().takeBody(assumingOp.getRegion());
 
@@ -66,7 +66,7 @@ struct AssumingOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(assumingOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
+        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
             assumingOp.getLoc(), it.value(), newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
diff --git a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
index ae06a34b65709..522347ff2b993 100644
--- a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
@@ -69,7 +69,7 @@ createFuncFromCluster(OpBuilder &b, const SmallVector<Operation *, 8> &cluster,
       cluster.empty()
           ? b.getFunctionType(shape.getType(), shape.getType())
           : b.getFunctionType(ValueRange(inputs).getTypes(), shape.getType());
-  shape::FuncOp fnOp = b.create<shape::FuncOp>(loc, fnName, fnType);
+  shape::FuncOp fnOp = shape::FuncOp::create(b, loc, fnName, fnType);
   Block *block = fnOp.addEntryBlock();
   b.setInsertionPointToEnd(block);
   IRMapping bvm;
@@ -85,7 +85,7 @@ createFuncFromCluster(OpBuilder &b, const SmallVector<Operation *, 8> &cluster,
   llvm::SmallVector<Value, 4> fnReturns;
   fnReturns.push_back(bvm.lookupOrDefault(shape));
 
-  b.create<shape::ReturnOp>(loc, fnReturns);
+  shape::ReturnOp::create(b, loc, fnReturns);
   fnOp.setPrivate();
   return std::make_pair(fnOp, inputs);
 }
@@ -187,7 +187,7 @@ class TensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
   LogicalResult matchAndRewrite(tensor::DimOp op,
                                 PatternRewriter &rewriter) const override {
     auto shapeOf =
-        rewriter.create<shape::ShapeOfOp>(op.getLoc(), op.getSource());
+        shape::ShapeOfOp::create(rewriter, op.getLoc(), op.getSource());
     rewriter.replaceOpWithNewOp<shape::GetExtentOp>(op, op.getType(), shapeOf,
                                                     op.getIndex());
     return success();
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index 121e0cc133e19..b5f6230bc2362 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -44,14 +44,14 @@ NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
                    ->materializeConstant(rewriter, rewriter.getIndexAttr(1),
                                          valueType, loc)
                    ->getResult(0);
-  ReduceOp reduce = rewriter.create<ReduceOp>(loc, op.getShape(), init);
+  ReduceOp reduce = ReduceOp::create(rewriter, loc, op.getShape(), init);
 
   // Generate reduce operator.
   Block *body = reduce.getBody();
   OpBuilder b = OpBuilder::atBlockEnd(body);
-  Value product = b.create<MulOp>(loc, valueType, body->getArgument(1),
+  Value product = MulOp::create(b, loc, valueType, body->getArgument(1),
                                   body->getArgument(2));
-  b.create<shape::YieldOp>(loc, product);
+  shape::YieldOp::create(b, loc, product);
 
   rewriter.replaceOp(op, reduce.getResult());
   return success();
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index ea7918fa14f23..d5e709bbed221 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -561,7 +561,7 @@ SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
   SmallVector<Type> retType(
       dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
       builder.getIndexType());
-  auto transOp = builder.create<CrdTranslateOp>(loc, retType, crds, dir, *this);
+  auto transOp = CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
   return transOp.getOutCrds();
 }
 
@@ -1483,7 +1483,7 @@ LogicalResult CrdTranslateOp::fold(FoldAdaptor adaptor,
 
 void LvlOp::build(OpBuilder &builder, OperationState &state, Value source,
                   int64_t index) {
-  Value val = builder.create<arith::ConstantIndexOp>(state.location, index);
+  Value val = arith::ConstantIndexOp::create(builder, state.location, index);
   return build(builder, state, source, val);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
index 9c84f4c25866f..74842cc1f310b 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
@@ -41,7 +41,7 @@ LogicalResult sparse_tensor::detail::stageWithSortImpl(
 
   // -> sort
   Type dstCOOTp = dstStt.getCOOType(/*ordered=*/true);
-  Value dstCOO = rewriter.create<ReorderCOOOp>(
+  Value dstCOO = ReorderCOOOp::create(rewriter,
       loc, dstCOOTp, srcCOO, SparseTensorSortKind::HybridQuickSort);
 
   // -> dest.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
index 5461987fb49d9..dce6e438d55d0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
@@ -90,13 +90,13 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
         } else if (directOut) {
           Value mem;
           if (kind == SparseTensorFieldKind::PosMemRef)
-            mem = builder.create<sparse_tensor::ToPositionsOp>(loc, inputs[0],
+            mem = sparse_tensor::ToPositionsOp::create(builder, loc, inputs[0],
                                                                lv);
           else if (kind == SparseTensorFieldKind::CrdMemRef)
-            mem = builder.create<sparse_tensor::ToCoordinatesOp>(loc, inputs[0],
+            mem = sparse_tensor::ToCoordinatesOp::create(builder, loc, inputs[0],
                                                                  lv);
           else
-            mem = builder.create<sparse_tensor::ToValuesOp>(loc, inputs[0]);
+            mem = sparse_tensor::ToValuesOp::create(builder, loc, inputs[0]);
           toVals.push_back(mem);
         } else {
           ShapedType rtp = cast<ShapedType>(t);
@@ -111,7 +111,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
 
     if (isIn) {
       // Assemble multiple inputs into a single sparse tensor.
-      auto a = builder.create<sparse_tensor::AssembleOp>(loc, rtp, inputs);
+      auto a = sparse_tensor::AssembleOp::create(builder, loc, rtp, inputs);
       toVals.push_back(a.getResult());
     } else if (!directOut) {
       // Disassemble a single sparse input into multiple outputs.
@@ -119,7 +119,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
       unsigned len = retTypes.size();
       retTypes.append(cntTypes);
       auto d =
-          builder.create<sparse_tensor::DisassembleOp>(loc, retTypes, inputs);
+          sparse_tensor::DisassembleOp::create(builder, loc, retTypes, inputs);
       for (unsigned i = 0; i < len; i++)
         toVals.push_back(d.getResult(i));
     }
@@ -201,7 +201,7 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     OpBuilder moduleBuilder(modOp.getBodyRegion());
     unsigned extra = inputTypes.size();
     inputTypes.append(extraTypes);
-    auto func = moduleBuilder.create<func::FuncOp>(
+    auto func = func::FuncOp::create(moduleBuilder,
         loc, orgName, FunctionType::get(context, inputTypes, outputTypes));
     func.setPublic();
 
@@ -218,14 +218,14 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     // Call the original, now private method. A subsequent inlining pass can
     // determine whether cloning the method body in place is worthwhile.
     auto org = SymbolRefAttr::get(context, wrapper);
-    auto call = rewriter.create<func::CallOp>(loc, funcOp.getResultTypes(), org,
+    auto call = func::CallOp::create(rewriter, loc, funcOp.getResultTypes(), org,
                                               inputs);
 
     // Convert outputs and return.
     SmallVector<Value> outputs;
     convVals(rewriter, loc, funcOp.getResultTypes(), call.getResults(),
              body->getArguments(), outputs, extra, /*isIn=*/false, directOut);
-    rewriter.create<func::ReturnOp>(loc, outputs);
+    func::ReturnOp::create(rewriter, loc, outputs);
 
     // Finally, migrate a potential c-interface property.
     if (funcOp->getAttrOfType<UnitAttr>(
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
index 0c5912bb73772..12507ff1c5251 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
@@ -94,7 +94,7 @@ static FlatSymbolRefAttr getMangledSortHelperFunc(
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPoint(insertPoint);
     Location loc = insertPoint.getLoc();
-    func = builder.create<func::FuncOp>(
+    func = func::FuncOp::create(builder,
         loc, nameOstream.str(),
         FunctionType::get(context, operands.getTypes(), resultTypes));
     func.setPrivate();
@@ -111,13 +111,13 @@ static void forEachIJPairInXs(
     uint64_t ny,
     function_ref<void(uint64_t, Value, Value, Value)> bodyBuilder) {
   Value cstep = constantIndex(builder, loc, xPerm.getNumResults() + ny);
-  Value iOffset = builder.create<arith::MulIOp>(loc, args[0], cstep);
-  Value jOffset = builder.create<arith::MulIOp>(loc, args[1], cstep);
+  Value iOffset = arith::MulIOp::create(builder, loc, args[0], cstep);
+  Value jOffset = arith::MulIOp::create(builder, loc, args[1], cstep);
   for (unsigned k = 0, e = xPerm.getNumResults(); k < e; k++) {
     unsigned actualK = cast<AffineDimExpr>(xPerm.getResult(k)).getPosition();
     Value ak = constantIndex(builder, loc, actualK);
-    Value i = builder.create<arith::AddIOp>(loc, ak, iOffset);
-    Value j = builder.create<arith::AddIOp>(loc, ak, jOffset);
+    Value i = arith::AddIOp::create(builder, loc, ak, iOffset);
+    Value j = arith::AddIOp::create(builder, loc, ak, jOffset);
     Value buffer = args[xStartIdx];
 
     bodyBuilder(k, i, j, buffer);
@@ -165,10 +165,10 @@ static void forEachIJPairInAllBuffers(
 static void createSwap(OpBuilder &builder, Location loc, ValueRange args,
                        AffineMap xPerm, uint64_t ny) {
   auto swapOnePair = [&](uint64_t unused, Value i, Value j, Value buffer) {
-    Value vi = builder.create<memref::LoadOp>(loc, buffer, i);
-    Value vj = builder.create<memref::LoadOp>(loc, buffer, j);
-    builder.create<memref::StoreOp>(loc, vj, buffer, i);
-    builder.create<memref::StoreOp>(loc, vi, buffer, j);
+    Value vi = memref::LoadOp::create(builder, loc, buffer, i);
+    Value vj = memref::LoadOp::create(builder, loc, buffer, j);
+    memref::StoreOp::create(builder, loc, vj, buffer, i);
+    memref::StoreOp::create(builder, loc, vi, buffer, j);
   };
 
   forEachIJPairInAllBuffers(builder, loc, args, xPerm, ny, swapOnePair);
@@ -193,7 +193,7 @@ static Value createInlinedCompareImplementation(
       OpBuilder::InsertionGuard insertionGuard(builder);
       auto ifOp = cast<scf::IfOp>(val.getDefiningOp());
       builder.setInsertionPointAfter(ifOp);
-      builder.create<scf::YieldOp>(loc, ifOp.getResult(0));
+      scf::YieldOp::create(builder, loc, ifOp.getResult(0));
     }
   };
 
@@ -207,25 +207,25 @@ static Value createInlinedCompareImplementation(
 /// result of the comparison.
 static Value createEqCompare(OpBuilder &builder, Location loc, Value i, Value j,
                              Value x, bool isFirstDim, bool isLastDim) {
-  Value vi = builder.create<memref::LoadOp>(loc, x, i);
-  Value vj = builder.create<memref::LoadOp>(loc, x, j);
+  Value vi = memref::LoadOp::create(builder, loc, x, i);
+  Value vj = memref::LoadOp::create(builder, loc, x, j);
 
   Value res;
   if (isLastDim) {
-    res = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, vi, vj);
+    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
-      builder.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(builder, loc, res);
   } else {
     Value ne =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, vi, vj);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIntegerType(1),
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
                                                ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value f = constantI1(builder, loc, false);
-    builder.create<scf::YieldOp>(loc, f);
+    scf::YieldOp::create(builder, loc, f);
 
     // If (x[i] == x[j]). Set up the insertion point for the nested if-stmt that
     // checks the remaining dimensions.
@@ -261,26 +261,26 @@ static Value createInlinedEqCompare(OpBuilder &builder, Location loc,
 static Value createLessThanCompare(OpBuilder &builder, Location loc, Value i,
                                    Value j, Value x, bool isFirstDim,
                                    bool isLastDim) {
-  Value vi = builder.create<memref::LoadOp>(loc, x, i);
-  Value vj = builder.create<memref::LoadOp>(loc, x, j);
+  Value vi = memref::LoadOp::create(builder, loc, x, i);
+  Value vj = memref::LoadOp::create(builder, loc, x, j);
 
   Value res;
   if (isLastDim) {
-    res = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, vi, vj);
+    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
-      builder.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(builder, loc, res);
   } else {
     Value ne =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, vi, vj);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIntegerType(1),
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
                                                ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value lt =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, vi, vj);
-    builder.create<scf::YieldOp>(loc, lt);
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
+    scf::YieldOp::create(builder, loc, lt);
 
     // If (x[i] == x[j]). Set up the insertion point for the nested if-stmt that
     // checks the remaining dimensions.
@@ -337,17 +337,17 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value p = args[hiIdx];
   SmallVector<Type, 2> types(2, p.getType()); // Only two types.
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(
+  scf::WhileOp whileOp = scf::WhileOp::create(builder,
       loc, types, SmallVector<Value, 2>{args[loIdx], args[hiIdx]});
 
   // The before-region of the WhileOp.
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, types, {loc, loc});
   builder.setInsertionPointToEnd(before);
-  Value cond1 = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                               before->getArgument(0),
                                               before->getArgument(1));
-  builder.create<scf::ConditionOp>(loc, cond1, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond1, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -357,9 +357,9 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   Value hi = after->getArgument(1);
   // Compute mid = (lo + hi) >> 1.
   Value c1 = constantIndex(builder, loc, 1);
-  Value mid = builder.create<arith::ShRUIOp>(
-      loc, builder.create<arith::AddIOp>(loc, lo, hi), c1);
-  Value midp1 = builder.create<arith::AddIOp>(loc, mid, c1);
+  Value mid = arith::ShRUIOp::create(builder,
+      loc, arith::AddIOp::create(builder, loc, lo, hi), c1);
+  Value midp1 = arith::AddIOp::create(builder, loc, mid, c1);
 
   // Compare xs[p] < xs[mid].
   SmallVector<Value> compareOperands{p, mid};
@@ -372,12 +372,12 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   //     hi = mid;
   //   else
   //     lo = mid + 1;
-  Value newLo = builder.create<arith::SelectOp>(loc, cond2, lo, midp1);
-  Value newHi = builder.create<arith::SelectOp>(loc, cond2, mid, hi);
-  builder.create<scf::YieldOp>(loc, ValueRange{newLo, newHi});
+  Value newLo = arith::SelectOp::create(builder, loc, cond2, lo, midp1);
+  Value newHi = arith::SelectOp::create(builder, loc, cond2, mid, hi);
+  scf::YieldOp::create(builder, loc, ValueRange{newLo, newHi});
 
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc, whileOp.getResult(0));
+  func::ReturnOp::create(builder, loc, whileOp.getResult(0));
 }
 
 /// Creates code to advance i in a loop based on xs[p] as follows:
@@ -393,7 +393,7 @@ static std::pair<Value, Value> createScanLoop(OpBuilder &builder,
                                               uint64_t ny, int step) {
   Location loc = func.getLoc();
   scf::WhileOp whileOp =
-      builder.create<scf::WhileOp>(loc, TypeRange{i.getType()}, ValueRange{i});
+      scf::WhileOp::create(builder, loc, TypeRange{i.getType()}, ValueRange{i});
 
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, {i.getType()}, {loc});
@@ -409,14 +409,14 @@ static std::pair<Value, Value> createScanLoop(OpBuilder &builder,
   }
   compareOperands.append(xs.begin(), xs.end());
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  builder.create<scf::ConditionOp>(loc, cond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond, before->getArguments());
 
   Block *after =
       builder.createBlock(&whileOp.getAfter(), {}, {i.getType()}, {loc});
   builder.setInsertionPointToEnd(after);
   Value cs = constantIndex(builder, loc, step);
-  i = builder.create<arith::AddIOp>(loc, after->getArgument(0), cs);
-  builder.create<scf::YieldOp>(loc, ValueRange{i});
+  i = arith::AddIOp::create(builder, loc, after->getArgument(0), cs);
+  scf::YieldOp::create(builder, loc, ValueRange{i});
   i = whileOp.getResult(0);
 
   builder.setInsertionPointAfter(whileOp);
@@ -440,7 +440,7 @@ static scf::IfOp createCompareThenSwap(OpBuilder &builder, Location loc,
   compareOperands[0] = b;
   compareOperands[1] = a;
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else=*/false);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, cond, /*else=*/false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   swapOperands[0] = b;
   swapOperands[1] = a;
@@ -517,12 +517,12 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
   swapOperands.append(args.begin() + xStartIdx, args.end());
   Location loc = func.getLoc();
   Value c1 = constantIndex(builder, loc, 1);
-  Value hiP1 = builder.create<arith::AddIOp>(loc, hi, c1);
-  Value len = builder.create<arith::SubIOp>(loc, hiP1, lo);
+  Value hiP1 = arith::AddIOp::create(builder, loc, hi, c1);
+  Value len = arith::SubIOp::create(builder, loc, hiP1, lo);
   Value lenThreshold = constantIndex(builder, loc, 1000);
-  Value lenCond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value lenCond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                                 len, lenThreshold);
-  scf::IfOp lenIf = builder.create<scf::IfOp>(loc, lenCond, /*else=*/true);
+  scf::IfOp lenIf = scf::IfOp::create(builder, loc, lenCond, /*else=*/true);
 
   // When len < 1000, choose pivot from median of 3 values.
   builder.setInsertionPointToStart(&lenIf.getThenRegion().front());
@@ -531,13 +531,13 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
 
   // When len >= 1000, choose pivot from median of 5 values.
   builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
-  Value miP1 = builder.create<arith::AddIOp>(loc, hi, c1);
-  Value a = builder.create<arith::AddIOp>(loc, lo, miP1);
+  Value miP1 = arith::AddIOp::create(builder, loc, hi, c1);
+  Value a = arith::AddIOp::create(builder, loc, lo, miP1);
   // Value a is the middle between [loc, mi].
-  a = builder.create<arith::ShRUIOp>(loc, a, c1);
-  Value b = builder.create<arith::AddIOp>(loc, mi, hiP1);
+  a = arith::ShRUIOp::create(builder, loc, a, c1);
+  Value b = arith::AddIOp::create(builder, loc, mi, hiP1);
   // Value b is the middle between [mi, hi].
-  b = builder.create<arith::ShRUIOp>(loc, b, c1);
+  b = arith::ShRUIOp::create(builder, loc, b, c1);
   createSort5(builder, loc, xPerm, ny, swapOperands, compareOperands, lo, a, mi,
               b, hi);
 
@@ -589,24 +589,24 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value sum = builder.create<arith::AddIOp>(loc, lo, hi);
+  Value sum = arith::AddIOp::create(builder, loc, lo, hi);
   Value c1 = constantIndex(builder, loc, 1);
-  Value p = builder.create<arith::ShRUIOp>(loc, sum, c1);
+  Value p = arith::ShRUIOp::create(builder, loc, sum, c1);
 
   Value i = lo;
-  Value j = builder.create<arith::SubIOp>(loc, hi, c1);
+  Value j = arith::SubIOp::create(builder, loc, hi, c1);
   createChoosePivot(builder, module, func, xPerm, ny, i, j, p, args);
   Value trueVal = constantI1(builder, loc, true); // The value for while (true)
   SmallVector<Value, 4> operands{i, j, p, trueVal}; // Exactly four values.
   SmallVector<Type, 4> types{i.getType(), j.getType(), p.getType(),
                              trueVal.getType()};
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(loc, types, operands);
+  scf::WhileOp whileOp = scf::WhileOp::create(builder, loc, types, operands);
 
   // The before-region of the WhileOp.
   Block *before = builder.createBlock(&whileOp.getBefore(), {}, types,
                                       {loc, loc, loc, loc});
   builder.setInsertionPointToEnd(before);
-  builder.create<scf::ConditionOp>(loc, before->getArgument(3),
+  scf::ConditionOp::create(builder, loc, before->getArgument(3),
                                    before->getArguments());
 
   // The after-region of the WhileOp.
@@ -629,70 +629,70 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
 
   // If i < j:
   Value cond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, i, j);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, i, j);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   SmallVector<Value> swapOperands{i, j};
   swapOperands.append(args.begin() + xStartIdx, args.end());
   createSwap(builder, loc, swapOperands, xPerm, ny);
   // If the pivot is moved, update p with the new pivot.
   Value icond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, i, p);
-  scf::IfOp ifOpI = builder.create<scf::IfOp>(loc, TypeRange{p.getType()},
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, i, p);
+  scf::IfOp ifOpI = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
                                               icond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpI.getThenRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{j});
+  scf::YieldOp::create(builder, loc, ValueRange{j});
   builder.setInsertionPointToStart(&ifOpI.getElseRegion().front());
   Value jcond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, j, p);
-  scf::IfOp ifOpJ = builder.create<scf::IfOp>(loc, TypeRange{p.getType()},
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, j, p);
+  scf::IfOp ifOpJ = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
                                               jcond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpJ.getThenRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{i});
+  scf::YieldOp::create(builder, loc, ValueRange{i});
   builder.setInsertionPointToStart(&ifOpJ.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{p});
+  scf::YieldOp::create(builder, loc, ValueRange{p});
   builder.setInsertionPointAfter(ifOpJ);
-  builder.create<scf::YieldOp>(loc, ifOpJ.getResults());
+  scf::YieldOp::create(builder, loc, ifOpJ.getResults());
   builder.setInsertionPointAfter(ifOpI);
   Value compareEqIJ =
-      builder.create<arith::AndIOp>(loc, iCompareEq, jCompareEq);
-  scf::IfOp ifOp2 = builder.create<scf::IfOp>(
+      arith::AndIOp::create(builder, loc, iCompareEq, jCompareEq);
+  scf::IfOp ifOp2 = scf::IfOp::create(builder,
       loc, TypeRange{i.getType(), j.getType()}, compareEqIJ, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
-  Value i2 = builder.create<arith::AddIOp>(loc, i, c1);
-  Value j2 = builder.create<arith::SubIOp>(loc, j, c1);
-  builder.create<scf::YieldOp>(loc, ValueRange{i2, j2});
+  Value i2 = arith::AddIOp::create(builder, loc, i, c1);
+  Value j2 = arith::SubIOp::create(builder, loc, j, c1);
+  scf::YieldOp::create(builder, loc, ValueRange{i2, j2});
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{i, j});
+  scf::YieldOp::create(builder, loc, ValueRange{i, j});
   builder.setInsertionPointAfter(ifOp2);
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc,
       ValueRange{ifOp2.getResult(0), ifOp2.getResult(1), ifOpI.getResult(0),
                  /*cont=*/constantI1(builder, loc, true)});
 
   // False branch for if i < j (i.e., i >= j):
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  p = builder.create<arith::AddIOp>(loc, j,
+  p = arith::AddIOp::create(builder, loc, j,
                                     constantOne(builder, loc, j.getType()));
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc, ValueRange{i, j, p, /*cont=*/constantI1(builder, loc, false)});
 
   // Return for the whileOp.
   builder.setInsertionPointAfter(ifOp);
-  builder.create<scf::YieldOp>(loc, ifOp.getResults());
+  scf::YieldOp::create(builder, loc, ifOp.getResults());
 
   // Return for the function.
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc, whileOp.getResult(2));
+  func::ReturnOp::create(builder, loc, whileOp.getResult(2));
 }
 
 /// Computes (n-2)/n, assuming n has index type.
 static Value createSubTwoDividedByTwo(OpBuilder &builder, Location loc,
                                       Value n) {
   Value i2 = constantIndex(builder, loc, 2);
-  Value res = builder.create<arith::SubIOp>(loc, n, i2);
+  Value res = arith::SubIOp::create(builder, loc, n, i2);
   Value i1 = constantIndex(builder, loc, 1);
-  return builder.create<arith::ShRUIOp>(loc, res, i1);
+  return arith::ShRUIOp::create(builder, loc, res, i1);
 }
 
 /// Creates a function to heapify the subtree with root `start` within the full
@@ -743,16 +743,16 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   // If (n >= 2).
   Value c2 = constantIndex(builder, loc, 2);
   Value condN =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, n, c2);
-  scf::IfOp ifN = builder.create<scf::IfOp>(loc, condN, /*else=*/false);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, n, c2);
+  scf::IfOp ifN = scf::IfOp::create(builder, loc, condN, /*else=*/false);
   builder.setInsertionPointToStart(&ifN.getThenRegion().front());
-  Value child = builder.create<arith::SubIOp>(loc, start, first);
+  Value child = arith::SubIOp::create(builder, loc, start, first);
 
   // If ((n-2)/2 >= child).
   Value t = createSubTwoDividedByTwo(builder, loc, n);
   Value condNc =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp ifNc = builder.create<scf::IfOp>(loc, condNc, /*else=*/false);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
+  scf::IfOp ifNc = scf::IfOp::create(builder, loc, condNc, /*else=*/false);
 
   builder.setInsertionPointToStart(&ifNc.getThenRegion().front());
   Value c1 = constantIndex(builder, loc, 1);
@@ -768,32 +768,32 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   //   if (child+1 < n && data[childIndex] < data[childIndex+1])
   //     childIndex ++; child ++ // Right child is bigger.
   auto getLargerChild = [&](Value r) -> std::pair<Value, Value> {
-    Value lChild = builder.create<arith::ShLIOp>(loc, r, c1);
-    lChild = builder.create<arith::AddIOp>(loc, lChild, c1);
-    Value lChildIdx = builder.create<arith::AddIOp>(loc, lChild, first);
-    Value rChild = builder.create<arith::AddIOp>(loc, lChild, c1);
-    Value cond1 = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value lChild = arith::ShLIOp::create(builder, loc, r, c1);
+    lChild = arith::AddIOp::create(builder, loc, lChild, c1);
+    Value lChildIdx = arith::AddIOp::create(builder, loc, lChild, first);
+    Value rChild = arith::AddIOp::create(builder, loc, lChild, c1);
+    Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                                 rChild, n);
     SmallVector<Type, 2> ifTypes(2, r.getType());
     scf::IfOp if1 =
-        builder.create<scf::IfOp>(loc, ifTypes, cond1, /*else=*/true);
+        scf::IfOp::create(builder, loc, ifTypes, cond1, /*else=*/true);
     builder.setInsertionPointToStart(&if1.getThenRegion().front());
-    Value rChildIdx = builder.create<arith::AddIOp>(loc, rChild, first);
+    Value rChildIdx = arith::AddIOp::create(builder, loc, rChild, first);
     // Compare data[left] < data[right].
     compareOperands[0] = lChildIdx;
     compareOperands[1] = rChildIdx;
     Value cond2 =
         createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
     scf::IfOp if2 =
-        builder.create<scf::IfOp>(loc, ifTypes, cond2, /*else=*/true);
+        scf::IfOp::create(builder, loc, ifTypes, cond2, /*else=*/true);
     builder.setInsertionPointToStart(&if2.getThenRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{rChild, rChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{rChild, rChildIdx});
     builder.setInsertionPointToStart(&if2.getElseRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{lChild, lChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{lChild, lChildIdx});
     builder.setInsertionPointAfter(if2);
-    builder.create<scf::YieldOp>(loc, if2.getResults());
+    scf::YieldOp::create(builder, loc, if2.getResults());
     builder.setInsertionPointToStart(&if1.getElseRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{lChild, lChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{lChild, lChildIdx});
     builder.setInsertionPointAfter(if1);
     return std::make_pair(if1.getResult(0), if1.getResult(1));
   };
@@ -803,7 +803,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
 
   // While (data[start] < data[childIndex]).
   SmallVector<Type, 3> types(3, child.getType());
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(
+  scf::WhileOp whileOp = scf::WhileOp::create(builder,
       loc, types, SmallVector<Value, 2>{start, child, childIdx});
 
   // The before-region of the WhileOp.
@@ -815,7 +815,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   compareOperands[0] = start;
   compareOperands[1] = childIdx;
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  builder.create<scf::ConditionOp>(loc, cond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after = builder.createBlock(&whileOp.getAfter(), {}, types, locs);
@@ -827,20 +827,20 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   createSwap(builder, loc, swapOperands, xPerm, ny);
   start = childIdx;
   Value cond2 =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp if2 = builder.create<scf::IfOp>(
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
+  scf::IfOp if2 = scf::IfOp::create(builder,
       loc, TypeRange{child.getType(), child.getType()}, cond2, /*else=*/true);
   builder.setInsertionPointToStart(&if2.getThenRegion().front());
   auto [newChild, newChildIdx] = getLargerChild(child);
-  builder.create<scf::YieldOp>(loc, ValueRange{newChild, newChildIdx});
+  scf::YieldOp::create(builder, loc, ValueRange{newChild, newChildIdx});
   builder.setInsertionPointToStart(&if2.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{child, childIdx});
+  scf::YieldOp::create(builder, loc, ValueRange{child, childIdx});
   builder.setInsertionPointAfter(if2);
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc, ValueRange{start, if2.getResult(0), if2.getResult(1)});
 
   builder.setInsertionPointAfter(ifN);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Creates a function to perform heap sort on the values in the range of index
@@ -870,45 +870,45 @@ static void createHeapSortFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value n = builder.create<arith::SubIOp>(loc, hi, lo);
+  Value n = arith::SubIOp::create(builder, loc, hi, lo);
 
   // For i = (n-2)/2 downto 0.
   Value c0 = constantIndex(builder, loc, 0);
   Value c1 = constantIndex(builder, loc, 1);
   Value s = createSubTwoDividedByTwo(builder, loc, n);
-  Value up = builder.create<arith::AddIOp>(loc, s, c1);
-  scf::ForOp forI = builder.create<scf::ForOp>(loc, c0, up, c1);
+  Value up = arith::AddIOp::create(builder, loc, s, c1);
+  scf::ForOp forI = scf::ForOp::create(builder, loc, c0, up, c1);
   builder.setInsertionPointToStart(forI.getBody());
-  Value i = builder.create<arith::SubIOp>(loc, s, forI.getInductionVar());
-  Value lopi = builder.create<arith::AddIOp>(loc, lo, i);
+  Value i = arith::SubIOp::create(builder, loc, s, forI.getInductionVar());
+  Value lopi = arith::AddIOp::create(builder, loc, lo, i);
   SmallVector<Value> shiftDownOperands = {lo, lopi};
   shiftDownOperands.append(args.begin() + xStartIdx, args.end());
   shiftDownOperands.push_back(n);
   FlatSymbolRefAttr shiftDownFunc = getMangledSortHelperFunc(
       builder, func, TypeRange(), kShiftDownFuncNamePrefix, xPerm, ny,
       shiftDownOperands, createShiftDownFunc, /*nTrailingP=*/1);
-  builder.create<func::CallOp>(loc, shiftDownFunc, TypeRange(),
+  func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
                                shiftDownOperands);
 
   builder.setInsertionPointAfter(forI);
   // For l = n downto 2.
-  up = builder.create<arith::SubIOp>(loc, n, c1);
-  scf::ForOp forL = builder.create<scf::ForOp>(loc, c0, up, c1);
+  up = arith::SubIOp::create(builder, loc, n, c1);
+  scf::ForOp forL = scf::ForOp::create(builder, loc, c0, up, c1);
   builder.setInsertionPointToStart(forL.getBody());
-  Value l = builder.create<arith::SubIOp>(loc, n, forL.getInductionVar());
-  Value loplm1 = builder.create<arith::AddIOp>(loc, lo, l);
-  loplm1 = builder.create<arith::SubIOp>(loc, loplm1, c1);
+  Value l = arith::SubIOp::create(builder, loc, n, forL.getInductionVar());
+  Value loplm1 = arith::AddIOp::create(builder, loc, lo, l);
+  loplm1 = arith::SubIOp::create(builder, loc, loplm1, c1);
   SmallVector<Value> swapOperands{lo, loplm1};
   swapOperands.append(args.begin() + xStartIdx, args.end());
   createSwap(builder, loc, swapOperands, xPerm, ny);
   shiftDownOperands[1] = lo;
   shiftDownOperands[shiftDownOperands.size() - 1] =
-      builder.create<arith::SubIOp>(loc, l, c1);
-  builder.create<func::CallOp>(loc, shiftDownFunc, TypeRange(),
+      arith::SubIOp::create(builder, loc, l, c1);
+  func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
                                shiftDownOperands);
 
   builder.setInsertionPointAfter(forL);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// A helper for generating code to perform quick sort. It partitions [lo, hi),
@@ -933,35 +933,35 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
                                       args.drop_back(nTrailingP))
                 .getResult(0);
 
-  Value lenLow = builder.create<arith::SubIOp>(loc, p, lo);
-  Value lenHigh = builder.create<arith::SubIOp>(loc, hi, p);
+  Value lenLow = arith::SubIOp::create(builder, loc, p, lo);
+  Value lenHigh = arith::SubIOp::create(builder, loc, hi, p);
   // Partition already sorts array with len <= 2
   Value c2 = constantIndex(builder, loc, 2);
-  Value len = builder.create<arith::SubIOp>(loc, hi, lo);
+  Value len = arith::SubIOp::create(builder, loc, hi, lo);
   Value lenGtTwo =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ugt, len, c2);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ugt, len, c2);
   scf::IfOp ifLenGtTwo =
-      builder.create<scf::IfOp>(loc, types, lenGtTwo, /*else=*/true);
+      scf::IfOp::create(builder, loc, types, lenGtTwo, /*else=*/true);
   builder.setInsertionPointToStart(&ifLenGtTwo.getElseRegion().front());
   // Returns an empty range to mark the entire region is fully sorted.
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
   // Else len > 2, need recursion.
   builder.setInsertionPointToStart(&ifLenGtTwo.getThenRegion().front());
-  Value cond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule,
+  Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
                                              lenLow, lenHigh);
 
   Value c0 = constantIndex(builder, loc, 0);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
 
   auto mayRecursion = [&](Value low, Value high, Value len) {
     Value cond =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, len, c0);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else=*/false);
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, len, c0);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, cond, /*else=*/false);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     SmallVector<Value> operands{low, high};
     operands.append(args.begin() + xStartIdx, args.end());
-    builder.create<func::CallOp>(loc, func, operands);
+    func::CallOp::create(builder, loc, func, operands);
     builder.setInsertionPointAfter(ifOp);
   };
 
@@ -969,14 +969,14 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
   // the bigger partition to be processed by the enclosed while-loop.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   mayRecursion(lo, p, lenLow);
-  builder.create<scf::YieldOp>(loc, ValueRange{p, hi});
+  scf::YieldOp::create(builder, loc, ValueRange{p, hi});
 
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
   mayRecursion(p, hi, lenHigh);
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, p});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, p});
 
   builder.setInsertionPointAfter(ifOp);
-  builder.create<scf::YieldOp>(loc, ifOp.getResults());
+  scf::YieldOp::create(builder, loc, ifOp.getResults());
 
   builder.setInsertionPointAfter(ifLenGtTwo);
   return std::make_pair(ifLenGtTwo.getResult(0), ifLenGtTwo.getResult(1));
@@ -1011,10 +1011,10 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   Value c1 = constantIndex(builder, loc, 1);
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value lop1 = builder.create<arith::AddIOp>(loc, lo, c1);
+  Value lop1 = arith::AddIOp::create(builder, loc, lo, c1);
 
   // Start the outer for-stmt with induction variable i.
-  scf::ForOp forOpI = builder.create<scf::ForOp>(loc, lop1, hi, c1);
+  scf::ForOp forOpI = scf::ForOp::create(builder, loc, lop1, hi, c1);
   builder.setInsertionPointToStart(forOpI.getBody());
   Value i = forOpI.getInductionVar();
 
@@ -1035,24 +1035,24 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t unused, Value i, Value unused2, Value buffer) {
-        d.push_back(builder.create<memref::LoadOp>(loc, buffer, i));
+        d.push_back(memref::LoadOp::create(builder, loc, buffer, i));
       });
 
   // Start the inner for-stmt with induction variable j, for moving data[p..i)
   // to data[p+1..i+1).
-  Value imp = builder.create<arith::SubIOp>(loc, i, p);
+  Value imp = arith::SubIOp::create(builder, loc, i, p);
   Value c0 = constantIndex(builder, loc, 0);
-  scf::ForOp forOpJ = builder.create<scf::ForOp>(loc, c0, imp, c1);
+  scf::ForOp forOpJ = scf::ForOp::create(builder, loc, c0, imp, c1);
   builder.setInsertionPointToStart(forOpJ.getBody());
   Value j = forOpJ.getInductionVar();
-  Value imj = builder.create<arith::SubIOp>(loc, i, j);
+  Value imj = arith::SubIOp::create(builder, loc, i, j);
   operands[1] = imj;
-  operands[0] = builder.create<arith::SubIOp>(loc, imj, c1);
+  operands[0] = arith::SubIOp::create(builder, loc, imj, c1);
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t unused, Value imjm1, Value imj, Value buffer) {
-        Value t = builder.create<memref::LoadOp>(loc, buffer, imjm1);
-        builder.create<memref::StoreOp>(loc, t, buffer, imj);
+        Value t = memref::LoadOp::create(builder, loc, buffer, imjm1);
+        memref::StoreOp::create(builder, loc, t, buffer, imj);
       });
 
   // Store the value at data[i] to data[p].
@@ -1061,11 +1061,11 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t k, Value p, Value usused, Value buffer) {
-        builder.create<memref::StoreOp>(loc, d[k], buffer, p);
+        memref::StoreOp::create(builder, loc, d[k], buffer, p);
       });
 
   builder.setInsertionPointAfter(forOpI);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Creates a function to perform quick sort or a hybrid quick sort on the
@@ -1127,7 +1127,7 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   Value hi = args[hiIdx];
   SmallVector<Type, 2> types(2, lo.getType()); // Only two types.
   scf::WhileOp whileOp =
-      builder.create<scf::WhileOp>(loc, types, SmallVector<Value, 2>{lo, hi});
+      scf::WhileOp::create(builder, loc, types, SmallVector<Value, 2>{lo, hi});
 
   // The before-region of the WhileOp.
   Block *before =
@@ -1136,10 +1136,10 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   lo = before->getArgument(0);
   hi = before->getArgument(1);
   Value loP1 =
-      builder.create<arith::AddIOp>(loc, lo, constantIndex(builder, loc, 1));
+      arith::AddIOp::create(builder, loc, lo, constantIndex(builder, loc, 1));
   Value needSort =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, loP1, hi);
-  builder.create<scf::ConditionOp>(loc, needSort, before->getArguments());
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, loP1, hi);
+  scf::ConditionOp::create(builder, loc, needSort, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -1151,53 +1151,53 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   args[1] = hi;
 
   if (isHybrid) {
-    Value len = builder.create<arith::SubIOp>(loc, hi, lo);
+    Value len = arith::SubIOp::create(builder, loc, hi, lo);
     Value lenLimit = constantIndex(builder, loc, 30);
-    Value lenCond = builder.create<arith::CmpIOp>(
+    Value lenCond = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::ule, len, lenLimit);
     scf::IfOp lenIf =
-        builder.create<scf::IfOp>(loc, types, lenCond, /*else=*/true);
+        scf::IfOp::create(builder, loc, types, lenCond, /*else=*/true);
 
     // When len <= limit.
     builder.setInsertionPointToStart(&lenIf.getThenRegion().front());
     FlatSymbolRefAttr insertionSortFunc = getMangledSortHelperFunc(
         builder, func, TypeRange(), kSortStableFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createSortStableFunc);
-    builder.create<func::CallOp>(loc, insertionSortFunc, TypeRange(),
+    func::CallOp::create(builder, loc, insertionSortFunc, TypeRange(),
                                  ValueRange(args).drop_back(nTrailingP));
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When len > limit.
     builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
     Value depthLimit = args.back();
-    depthLimit = builder.create<arith::SubIOp>(loc, depthLimit,
+    depthLimit = arith::SubIOp::create(builder, loc, depthLimit,
                                                constantI64(builder, loc, 1));
     Value depthCond =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule,
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
                                       depthLimit, constantI64(builder, loc, 0));
     scf::IfOp depthIf =
-        builder.create<scf::IfOp>(loc, types, depthCond, /*else=*/true);
+        scf::IfOp::create(builder, loc, types, depthCond, /*else=*/true);
 
     // When depth exceeds limit.
     builder.setInsertionPointToStart(&depthIf.getThenRegion().front());
     FlatSymbolRefAttr heapSortFunc = getMangledSortHelperFunc(
         builder, func, TypeRange(), kHeapSortFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createHeapSortFunc);
-    builder.create<func::CallOp>(loc, heapSortFunc, TypeRange(),
+    func::CallOp::create(builder, loc, heapSortFunc, TypeRange(),
                                  ValueRange(args).drop_back(nTrailingP));
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When depth doesn't exceed limit.
     builder.setInsertionPointToStart(&depthIf.getElseRegion().front());
     args.back() = depthLimit;
     std::tie(lo, hi) =
         createQuickSort(builder, module, func, args, xPerm, ny, nTrailingP);
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
     builder.setInsertionPointAfter(depthIf);
     lo = depthIf.getResult(0);
     hi = depthIf.getResult(1);
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
     builder.setInsertionPointAfter(lenIf);
     lo = lenIf.getResult(0);
@@ -1208,11 +1208,11 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   }
 
   // New [lo, hi) for the next while-loop iteration.
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
   // After the while-loop.
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Implements the rewriting for operator sort and sort_coo.
@@ -1228,7 +1228,7 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     if (!mtp.isDynamicDim(0)) {
       auto newMtp =
           MemRefType::get({ShapedType::kDynamic}, mtp.getElementType());
-      v = rewriter.create<memref::CastOp>(loc, newMtp, v);
+      v = memref::CastOp::create(rewriter, loc, newMtp, v);
     }
     operands.push_back(v);
   }
@@ -1248,12 +1248,12 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     // As a heuristics, set depthLimit = 2 * log2(n).
     Value lo = operands[loIdx];
     Value hi = operands[hiIdx];
-    Value len = rewriter.create<arith::IndexCastOp>(
+    Value len = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(),
-        rewriter.create<arith::SubIOp>(loc, hi, lo));
-    Value depthLimit = rewriter.create<arith::SubIOp>(
+        arith::SubIOp::create(rewriter, loc, hi, lo));
+    Value depthLimit = arith::SubIOp::create(rewriter,
         loc, constantI64(rewriter, loc, 64),
-        rewriter.create<math::CountLeadingZerosOp>(loc, len));
+        math::CountLeadingZerosOp::create(rewriter, loc, len));
     operands.push_back(depthLimit);
     break;
   }
@@ -1307,33 +1307,33 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
     Location loc = op->getLoc();
     Value c0 = constantIndex(rewriter, loc, 0);
     Value buffer = op.getInBuffer();
-    Value capacity = rewriter.create<memref::DimOp>(loc, buffer, c0);
+    Value capacity = memref::DimOp::create(rewriter, loc, buffer, c0);
     Value size = op.getCurSize();
     Value value = op.getValue();
 
     Value n = op.getN() ? op.getN() : constantIndex(rewriter, loc, 1);
-    Value newSize = rewriter.create<arith::AddIOp>(loc, size, n);
+    Value newSize = arith::AddIOp::create(rewriter, loc, size, n);
     auto nValue = dyn_cast_or_null<arith::ConstantIndexOp>(n.getDefiningOp());
     bool nIsOne = (nValue && nValue.value() == 1);
 
     if (!op.getInbounds()) {
-      Value cond = rewriter.create<arith::CmpIOp>(
+      Value cond = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ugt, newSize, capacity);
 
       Value c2 = constantIndex(rewriter, loc, 2);
       auto bufferType =
           MemRefType::get({ShapedType::kDynamic}, value.getType());
-      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, bufferType, cond,
+      scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, bufferType, cond,
                                                   /*else=*/true);
       // True branch.
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       if (nIsOne) {
-        capacity = rewriter.create<arith::MulIOp>(loc, capacity, c2);
+        capacity = arith::MulIOp::create(rewriter, loc, capacity, c2);
       } else {
         // Use a do-while loop to calculate the new capacity as follows:
         //   do { new_capacity *= 2 } while (size > new_capacity)
         scf::WhileOp whileOp =
-            rewriter.create<scf::WhileOp>(loc, capacity.getType(), capacity);
+            scf::WhileOp::create(rewriter, loc, capacity.getType(), capacity);
 
         // The before-region of the WhileOp.
         Block *before = rewriter.createBlock(&whileOp.getBefore(), {},
@@ -1341,36 +1341,36 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
         rewriter.setInsertionPointToEnd(before);
 
         capacity =
-            rewriter.create<arith::MulIOp>(loc, before->getArgument(0), c2);
-        cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ugt,
+            arith::MulIOp::create(rewriter, loc, before->getArgument(0), c2);
+        cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ugt,
                                               newSize, capacity);
-        rewriter.create<scf::ConditionOp>(loc, cond, ValueRange{capacity});
+        scf::ConditionOp::create(rewriter, loc, cond, ValueRange{capacity});
         // The after-region of the WhileOp.
         Block *after = rewriter.createBlock(&whileOp.getAfter(), {},
                                             {capacity.getType()}, {loc});
         rewriter.setInsertionPointToEnd(after);
-        rewriter.create<scf::YieldOp>(loc, after->getArguments());
+        scf::YieldOp::create(rewriter, loc, after->getArguments());
 
         rewriter.setInsertionPointAfter(whileOp);
         capacity = whileOp.getResult(0);
       }
 
       Value newBuffer =
-          rewriter.create<memref::ReallocOp>(loc, bufferType, buffer, capacity);
+          memref::ReallocOp::create(rewriter, loc, bufferType, buffer, capacity);
       if (enableBufferInitialization) {
-        Value fillSize = rewriter.create<arith::SubIOp>(loc, capacity, newSize);
+        Value fillSize = arith::SubIOp::create(rewriter, loc, capacity, newSize);
         Value fillValue = constantZero(rewriter, loc, value.getType());
-        Value subBuffer = rewriter.create<memref::SubViewOp>(
+        Value subBuffer = memref::SubViewOp::create(rewriter,
             loc, newBuffer, /*offset=*/ValueRange{newSize},
             /*size=*/ValueRange{fillSize},
             /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
-        rewriter.create<linalg::FillOp>(loc, fillValue, subBuffer);
+        linalg::FillOp::create(rewriter, loc, fillValue, subBuffer);
       }
-      rewriter.create<scf::YieldOp>(loc, newBuffer);
+      scf::YieldOp::create(rewriter, loc, newBuffer);
 
       // False branch.
       rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      rewriter.create<scf::YieldOp>(loc, buffer);
+      scf::YieldOp::create(rewriter, loc, buffer);
 
       // Prepare for adding the value to the end of the buffer.
       rewriter.setInsertionPointAfter(ifOp);
@@ -1379,12 +1379,12 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
 
     // Add the value to the end of the buffer.
     if (nIsOne) {
-      rewriter.create<memref::StoreOp>(loc, value, buffer, size);
+      memref::StoreOp::create(rewriter, loc, value, buffer, size);
     } else {
-      Value subBuffer = rewriter.create<memref::SubViewOp>(
+      Value subBuffer = memref::SubViewOp::create(rewriter,
           loc, buffer, /*offset=*/ValueRange{size}, /*size=*/ValueRange{n},
           /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
-      rewriter.create<linalg::FillOp>(loc, value, subBuffer);
+      linalg::FillOp::create(rewriter, loc, value, subBuffer);
     }
 
     // Update the buffer size.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index e89b34d457ff8..001754b5531be 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -59,7 +59,7 @@ static gpu::GPUModuleOp genGPUModule(OpBuilder &builder, ModuleOp topModule) {
     return op; // existing
   markAsGPUContainer(topModule);
   builder.setInsertionPointToStart(topModule.getBody());
-  return builder.create<gpu::GPUModuleOp>(topModule->getLoc(),
+  return gpu::GPUModuleOp::create(builder, topModule->getLoc(),
                                           "sparse_kernels");
 }
 
@@ -81,7 +81,7 @@ static gpu::GPUFuncOp genGPUFunc(OpBuilder &builder, gpu::GPUModuleOp gpuModule,
     argsTp.push_back(arg.getType());
   FunctionType type = FunctionType::get(gpuModule->getContext(), argsTp, {});
   auto gpuFunc =
-      builder.create<gpu::GPUFuncOp>(gpuModule->getLoc(), kernelName, type);
+      gpu::GPUFuncOp::create(builder, gpuModule->getLoc(), kernelName, type);
   gpuFunc->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                    builder.getUnitAttr());
   return gpuFunc;
@@ -115,28 +115,28 @@ static Value genHostRegisterMemref(OpBuilder &builder, Location loc,
   MemRefType memTp = cast<MemRefType>(mem.getType());
   UnrankedMemRefType resTp =
       UnrankedMemRefType::get(memTp.getElementType(), /*memorySpace=*/0);
-  Value cast = builder.create<memref::CastOp>(loc, resTp, mem);
-  builder.create<gpu::HostRegisterOp>(loc, cast);
+  Value cast = memref::CastOp::create(builder, loc, resTp, mem);
+  gpu::HostRegisterOp::create(builder, loc, cast);
   return cast;
 }
 
 /// Unmaps the provided buffer, expecting the casted buffer.
 static void genHostUnregisterMemref(OpBuilder &builder, Location loc,
                                     Value cast) {
-  builder.create<gpu::HostUnregisterOp>(loc, cast);
+  gpu::HostUnregisterOp::create(builder, loc, cast);
 }
 
 /// Generates first wait in an asynchronous chain.
 static Value genFirstWait(OpBuilder &builder, Location loc) {
   Type tokenType = builder.getType<gpu::AsyncTokenType>();
-  return builder.create<gpu::WaitOp>(loc, tokenType, ValueRange())
+  return gpu::WaitOp::create(builder, loc, tokenType, ValueRange())
       .getAsyncToken();
 }
 
 /// Generates last, blocking wait in an asynchronous chain.
 static void genBlockingWait(OpBuilder &builder, Location loc,
                             ValueRange operands) {
-  builder.create<gpu::WaitOp>(loc, Type(), operands);
+  gpu::WaitOp::create(builder, loc, Type(), operands);
 }
 
 /// Allocates memory on the device.
@@ -156,7 +156,7 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
       dynamicSizes.push_back(dimOp);
     }
   }
-  return builder.create<gpu::AllocOp>(loc, TypeRange({memTp, token.getType()}),
+  return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
                                       token, dynamicSizes, ValueRange());
 }
 
@@ -164,14 +164,14 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
 static Value genHostBuffer(OpBuilder &builder, Location loc, Type type,
                            Value size) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
-  return builder.create<memref::AllocOp>(loc, memTp, size).getResult();
+  return memref::AllocOp::create(builder, loc, memTp, size).getResult();
 }
 
 // Allocates a typed buffer on the device with given size.
 static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Type type,
                                    Value size, Value token) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
-  return builder.create<gpu::AllocOp>(loc, TypeRange({memTp, token.getType()}),
+  return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
                                       token, size, ValueRange());
 }
 
@@ -184,14 +184,14 @@ static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Value size,
 /// Deallocates memory from the device.
 static Value genDeallocMemRef(OpBuilder &builder, Location loc, Value mem,
                               Value token) {
-  return builder.create<gpu::DeallocOp>(loc, token.getType(), token, mem)
+  return gpu::DeallocOp::create(builder, loc, token.getType(), token, mem)
       .getAsyncToken();
 }
 
 /// Copies memory between host and device (direction is implicit).
 static Value genCopyMemRef(OpBuilder &builder, Location loc, Value dst,
                            Value src, Value token) {
-  return builder.create<gpu::MemcpyOp>(loc, token.getType(), token, dst, src)
+  return gpu::MemcpyOp::create(builder, loc, token.getType(), token, dst, src)
       .getAsyncToken();
 }
 
@@ -212,7 +212,7 @@ static Value genTensorToMemref(PatternRewriter &rewriter, Location loc,
   auto tensorType = llvm::cast<ShapedType>(tensor.getType());
   auto memrefType =
       MemRefType::get(tensorType.getShape(), tensorType.getElementType());
-  return rewriter.create<bufferization::ToBufferOp>(loc, memrefType, tensor);
+  return bufferization::ToBufferOp::create(rewriter, loc, memrefType, tensor);
 }
 
 /// Prepares the outlined arguments, passing scalars and buffers in. Here we
@@ -293,13 +293,13 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
   // so that:
   //   row = blockIdx.x * blockDim.x + threadIdx.x
   //   inc = blockDim.x * gridDim.x
-  Value bid = rewriter.create<gpu::BlockIdOp>(loc, gpu::Dimension::x);
-  Value bsz = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::x);
-  Value tid = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
-  Value gsz = rewriter.create<gpu::GridDimOp>(loc, gpu::Dimension::x);
-  Value mul = rewriter.create<arith::MulIOp>(loc, bid, bsz);
-  Value row = rewriter.create<arith::AddIOp>(loc, mul, tid);
-  Value inc = rewriter.create<arith::MulIOp>(loc, bsz, gsz);
+  Value bid = gpu::BlockIdOp::create(rewriter, loc, gpu::Dimension::x);
+  Value bsz = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::x);
+  Value tid = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
+  Value gsz = gpu::GridDimOp::create(rewriter, loc, gpu::Dimension::x);
+  Value mul = arith::MulIOp::create(rewriter, loc, bid, bsz);
+  Value row = arith::AddIOp::create(rewriter, loc, mul, tid);
+  Value inc = arith::MulIOp::create(rewriter, loc, bsz, gsz);
 
   // Construct the iteration over the computational space that
   // accounts for the fact that the total number of threads and
@@ -308,7 +308,7 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
   //     <loop-body>
   //   }
   Value upper = irMap.lookup(forallOp.getUpperBound()[0]);
-  scf::ForOp forOp = rewriter.create<scf::ForOp>(loc, row, upper, inc);
+  scf::ForOp forOp = scf::ForOp::create(rewriter, loc, row, upper, inc);
   // The scf.for builder creates an empty block. scf.for does not allow multiple
   // blocks in its region, so delete the block before `cloneRegionBefore` adds
   // an additional block.
@@ -321,7 +321,7 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
 
   // Done.
   rewriter.setInsertionPointAfter(forOp);
-  rewriter.create<gpu::ReturnOp>(gpuFunc->getLoc());
+  gpu::ReturnOp::create(rewriter, gpuFunc->getLoc());
 }
 
 //===----------------------------------------------------------------------===//
@@ -496,11 +496,11 @@ static Value genFirstPosOrCrds(OpBuilder &builder, Location loc, Value a,
   if (format == CuSparseFormat::kCOO) {
     // Library uses SoA COO, direct IR uses AoS COO.
     if (enableRT)
-      return builder.create<ToCoordinatesOp>(loc, a, 0);
-    return builder.create<ToCoordinatesBufferOp>(loc, a);
+      return ToCoordinatesOp::create(builder, loc, a, 0);
+    return ToCoordinatesBufferOp::create(builder, loc, a);
   }
   // Formats CSR/CSC and BSR use positions at 1.
-  return builder.create<ToPositionsOp>(loc, a, 1);
+  return ToPositionsOp::create(builder, loc, a, 1);
 }
 
 /// Generates the second coordinates of a sparse matrix.
@@ -510,7 +510,7 @@ static Value genSecondCrds(OpBuilder &builder, Location loc, Value a,
   if (isCOO && !enableRT)
     return Value(); // nothing needed
   // Formats CSR/CSC and BSR use coordinates at 1.
-  return builder.create<ToCoordinatesOp>(loc, a, 1);
+  return ToCoordinatesOp::create(builder, loc, a, 1);
 }
 
 /// Generates the sparse matrix handle.
@@ -523,12 +523,12 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
     // Library uses SoA COO, direct IR uses AoS COO.
     if (enableRT) {
       assert(colA);
-      return builder.create<gpu::CreateCooOp>(loc, handleTp, tokenTp, token,
+      return gpu::CreateCooOp::create(builder, loc, handleTp, tokenTp, token,
                                               sz1, sz2, nseA, rowA, colA, valA);
     }
 #ifdef CUSPARSE_COO_AOS
     assert(!colA);
-    return builder.create<gpu::CreateCooAoSOp>(loc, handleTp, tokenTp, token,
+    return gpu::CreateCooAoSOp::create(builder, loc, handleTp, tokenTp, token,
                                                sz1, sz2, nseA, rowA, valA);
 #else
     llvm_unreachable("gpu::CreateCooAoSOp is deprecated");
@@ -536,10 +536,10 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   }
   assert(colA);
   if (format == CuSparseFormat::kCSR)
-    return builder.create<gpu::CreateCsrOp>(loc, handleTp, tokenTp, token, sz1,
+    return gpu::CreateCsrOp::create(builder, loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
   if (format == CuSparseFormat::kCSC)
-    return builder.create<gpu::CreateCscOp>(loc, handleTp, tokenTp, token, sz1,
+    return gpu::CreateCscOp::create(builder, loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
   // BSR requires a bit more work since we need to pass in the block size
   // and all others sizes in terms of blocks (#block-rows, #block-cols,
@@ -549,11 +549,11 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   assert(dims.size() == 2 && dims[0] == dims[1]);
   uint64_t b = dims[0];
   Value bSz = constantIndex(builder, loc, b);
-  Value bRows = builder.create<arith::DivUIOp>(loc, sz1, bSz);
-  Value bCols = builder.create<arith::DivUIOp>(loc, sz2, bSz);
-  Value bNum = builder.create<arith::DivUIOp>(
+  Value bRows = arith::DivUIOp::create(builder, loc, sz1, bSz);
+  Value bCols = arith::DivUIOp::create(builder, loc, sz2, bSz);
+  Value bNum = arith::DivUIOp::create(builder,
       loc, nseA, constantIndex(builder, loc, b * b));
-  return builder.create<gpu::CreateBsrOp>(loc, handleTp, tokenTp, token, bRows,
+  return gpu::CreateBsrOp::create(builder, loc, handleTp, tokenTp, token, bRows,
                                           bCols, bNum, bSz, bSz, rowA, colA,
                                           valA);
 }
@@ -579,12 +579,12 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   //   a : memR/memC/memV -> rowA,colA,valA
   //   x : memX           -> vecX
   //   y : memY           -> vecY
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
   Value szY = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szX = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value memR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, a, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, a);
+  Value memV = ToValuesOp::create(rewriter, loc, a);
   Value rowA = genAllocCopy(rewriter, loc, memR, tokens);
   Value colA = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valA = genAllocCopy(rewriter, loc, memV, tokens);
@@ -606,18 +606,18 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dvecX = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dvecX = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, vecX, szX);
   Value dnX = dvecX.getResult(0);
   token = dvecX.getAsyncToken();
-  auto dvecY = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dvecY = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, vecY, szY);
   Value dnY = dvecY.getResult(0);
   token = dvecY.getAsyncToken();
   auto dnYType = llvm::cast<ShapedType>(y.getType()).getElementType();
 
   // Precompute buffersize for SpMV.
-  auto bufferComp = rewriter.create<gpu::SpMVBufferSizeOp>(
+  auto bufferComp = gpu::SpMVBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, spMatA, dnX, dnY,
       /*computeType=*/dnYType);
   Value bufferSz = bufferComp.getResult(0);
@@ -627,16 +627,16 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SpMV.
-  auto spmvComp = rewriter.create<gpu::SpMVOp>(
+  auto spmvComp = gpu::SpMVOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnX, dnY, /*computeType=*/dnYType, buffer);
   token = spmvComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnX)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnX)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnY)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnY)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, rowA, token);
   if (colA)
@@ -676,13 +676,13 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   //   a : memR/memC/memV -> rowA,colA,valA
   //   b : bufB           -> matB
   //   c : bufC           -> matC
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
   Value memR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, a, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, a);
+  Value memV = ToValuesOp::create(rewriter, loc, a);
   Value rowA = genAllocCopy(rewriter, loc, memR, tokens);
   Value colA = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valA = genAllocCopy(rewriter, loc, memV, tokens);
@@ -704,12 +704,12 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matB,
       SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matC,
       SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
@@ -717,7 +717,7 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dmatCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SpMM.
-  auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
   Value bufferSz = bufferComp.getResult(0);
@@ -728,16 +728,16 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Perform the SpMM.
-  auto spmmComp = rewriter.create<gpu::SpMMOp>(
+  auto spmmComp = gpu::SpMMOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType, buffer);
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnC)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, rowA, token);
   if (colA)
@@ -778,17 +778,17 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   //   b : bmemR/bmemC/bmemV -> rowB,colB,valB
   //   c : materializes
   auto dnCType = cTp.getElementType();
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
-  Value nseB = rewriter.create<NumberOfEntriesOp>(loc, b);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
+  Value nseB = NumberOfEntriesOp::create(rewriter, loc, b);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
   Value amemR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value amemC = genSecondCrds(rewriter, loc, a, format, enableRT); // not empty
-  Value amemV = rewriter.create<ToValuesOp>(loc, a);
+  Value amemV = ToValuesOp::create(rewriter, loc, a);
   Value bmemR = genFirstPosOrCrds(rewriter, loc, b, format, enableRT);
   Value bmemC = genSecondCrds(rewriter, loc, b, format, enableRT); // not empty
-  Value bmemV = rewriter.create<ToValuesOp>(loc, b);
+  Value bmemV = ToValuesOp::create(rewriter, loc, b);
   Value rowA = genAllocCopy(rewriter, loc, amemR, tokens);
   Value colA = genAllocCopy(rewriter, loc, amemC, tokens);
   Value valA = genAllocCopy(rewriter, loc, amemV, tokens);
@@ -818,7 +818,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   // Sparse matrix C materializes (also assumes beta == 0).
   Value zero = constantIndex(rewriter, loc, 0);
   Value one = constantIndex(rewriter, loc, 1);
-  Value mplus1 = rewriter.create<arith::AddIOp>(loc, szm, one);
+  Value mplus1 = arith::AddIOp::create(rewriter, loc, szm, one);
   auto e1 = genAllocBuffer(rewriter, loc, cTp.getPosType(), mplus1, token);
   Value rowC = e1.getResult(0);
   token = e1.getAsyncToken();
@@ -836,10 +836,10 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
 
   // Precompute buffersizes for SpGEMM.
   Operation *descOp =
-      rewriter.create<gpu::SpGEMMCreateDescrOp>(loc, descTp, tokenTp, token);
+      gpu::SpGEMMCreateDescrOp::create(rewriter, loc, descTp, tokenTp, token);
   Value desc = descOp->getResult(0);
   token = descOp->getResult(1);
-  Operation *work1 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
       valC, gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
@@ -848,7 +848,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   auto buf1 = genAllocBuffer(rewriter, loc, bufferSz1, token);
   Value buffer1 = buf1.getResult(0);
   token = buf1.getAsyncToken();
-  Operation *work2 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
       bufferSz1, buffer1,
@@ -856,7 +856,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = work2->getResult(1);
 
   // Compute step.
-  Operation *compute1 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
       valC, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
@@ -865,14 +865,14 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   auto buf2 = genAllocBuffer(rewriter, loc, bufferSz2, token);
   Value buffer2 = buf2.getResult(0);
   token = buf2.getAsyncToken();
-  Operation *compute2 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
       bufferSz2, buffer2, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
   token = compute2->getResult(1);
 
   // Get sizes.
-  Operation *sizes = rewriter.create<gpu::SpMatGetSizeOp>(
+  Operation *sizes = gpu::SpMatGetSizeOp::create(rewriter,
       loc, indexTp, indexTp, indexTp, tokenTp, token, spMatC);
   Value nnz = sizes->getResult(2);
   token = sizes->getResult(3);
@@ -884,10 +884,10 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = a3.getAsyncToken();
 
   // Update C with new pointers and copy final product back into C.
-  Operation *update = rewriter.create<gpu::SetCsrPointersOp>(
+  Operation *update = gpu::SetCsrPointersOp::create(rewriter,
       loc, tokenTp, token, spMatC, rowC, colC, valC);
   token = update->getResult(0);
-  Operation *copy = rewriter.create<gpu::SpGEMMCopyOp>(
+  Operation *copy = gpu::SpGEMMCopyOp::create(rewriter,
       loc, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType);
   token = copy->getResult(0);
@@ -898,13 +898,13 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   Value valH = genHostBuffer(rewriter, loc, dnCType, nnz);
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::SpGEMMDestroyDescrOp>(loc, tokenTp, token, desc)
+  token = gpu::SpGEMMDestroyDescrOp::create(rewriter, loc, tokenTp, token, desc)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatB)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatC)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatC)
               .getAsyncToken();
   token = genCopyMemRef(rewriter, loc, rowH, rowC, token);
   token = genCopyMemRef(rewriter, loc, colH, colC, token);
@@ -925,11 +925,11 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  Value vt = rewriter.create<bufferization::ToTensorOp>(
+  Value vt = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
-  Value rt = rewriter.create<bufferization::ToTensorOp>(
+  Value rt = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
-  Value ct = rewriter.create<bufferization::ToTensorOp>(
+  Value ct = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
   rewriter.replaceOpWithNewOp<AssembleOp>(op, c.getType(), ValueRange{rt, ct},
                                           vt);
@@ -980,17 +980,17 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  Operation *spGenA = rewriter.create<gpu::Create2To4SpMatOp>(
+  Operation *spGenA = gpu::Create2To4SpMatOp::create(rewriter,
       loc, spMatHandleTp, tokenTp, token, szm, szk,
       gpu::Prune2To4SpMatFlag::PRUNE_AND_CHECK, matA);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matB,
       SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matC,
       SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
@@ -1000,7 +1000,7 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   // Precompute buffersize for SpMM.
   SmallVector<Type> bufferTypes_{indexTp, indexTp, indexTp};
   TypeRange bufferTypes(bufferTypes_);
-  auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
       loc, bufferTypes, tokenTp, token, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
@@ -1022,17 +1022,17 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
 
   // Perform the SpMM.
   auto dnCType = llvm::cast<ShapedType>(matC.getType()).getElementType();
-  auto spmmComp = rewriter.create<gpu::SpMMOp>(
+  auto spmmComp = gpu::SpMMOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType,
       SmallVector<Value>{buffer1, buffer2, buffer3});
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resources.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnC)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, buffer1, token);
   token = genDeallocMemRef(rewriter, loc, buffer2, token);
@@ -1073,7 +1073,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   //   a : bufA           -> matA
   //   b : bufB           -> matB
   //   c : memR/memC/memV -> rowC,colC,valC
-  Value nseC = rewriter.create<NumberOfEntriesOp>(loc, c);
+  Value nseC = NumberOfEntriesOp::create(rewriter, loc, c);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
@@ -1083,7 +1083,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Value matB = genAllocCopy(rewriter, loc, bufB, tokens);
   Value memR = genFirstPosOrCrds(rewriter, loc, c, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, c, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, c);
+  Value memV = ToValuesOp::create(rewriter, loc, c);
   Value rowC = genAllocCopy(rewriter, loc, memR, tokens);
   Value colC = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valC = genAllocCopy(rewriter, loc, memV, tokens);
@@ -1096,11 +1096,11 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  auto dmatA = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatA = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnMatHandleTp, tokenTp, token, matA, SmallVector<Value>{szm, szk});
   Value dnA = dmatA.getResult(0);
   token = dmatA.getAsyncToken();
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnMatHandleTp, tokenTp, token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
@@ -1112,7 +1112,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SDDMM.
-  auto bufferComp = rewriter.create<gpu::SDDMMBufferSizeOp>(
+  auto bufferComp = gpu::SDDMMBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, dnA, dnB, spMatC, dnCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
@@ -1121,16 +1121,16 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SDDMM.
-  auto sddmmComp = rewriter.create<gpu::SDDMMOp>(loc, tokenTp, token, dnA, dnB,
+  auto sddmmComp = gpu::SDDMMOp::create(rewriter, loc, tokenTp, token, dnA, dnB,
                                                  spMatC, dnCType, buffer);
   token = sddmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnA)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatC)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, buffer, token);
   token = genDeallocMemRef(rewriter, loc, matA, token);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
index 2f68008e68b5f..0986a0f2ea0f0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
@@ -67,11 +67,11 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
       op.getRegionDefinedSpace(newBlock->getParent()->getRegionNumber());
   for (unsigned i : caseBits.bits()) {
     SparseIterator *it = iters[i].get();
-    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                 it->getCrd(), loopCrd);
-    casePred = rewriter.create<arith::AndIOp>(loc, casePred, pred);
+    casePred = arith::AndIOp::create(rewriter, loc, casePred, pred);
   }
-  scf::IfOp ifOp = rewriter.create<scf::IfOp>(
+  scf::IfOp ifOp = scf::IfOp::create(rewriter,
       loc, ValueRange(userReduc).getTypes(), casePred, /*else=*/true);
   rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
@@ -103,7 +103,7 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
   ValueRange yields = spY.getResults();
   rewriter.eraseOp(spY);
   rewriter.setInsertionPointToEnd(&ifOp.getThenRegion().front());
-  rewriter.create<scf::YieldOp>(loc, yields);
+  scf::YieldOp::create(rewriter, loc, yields);
 
   // Generates remaining case recursively.
   rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -111,7 +111,7 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
                                           newBlocks.drop_front(),
                                           oldBlocks.drop_front(), userReduc);
   if (!res.empty())
-    rewriter.create<scf::YieldOp>(loc, res);
+    scf::YieldOp::create(rewriter, loc, res);
 
   rewriter.setInsertionPointAfter(ifOp);
   return ifOp.getResults();
@@ -127,7 +127,7 @@ static ValueRange genLoopWithIterator(
   if (it->iteratableByFor()) {
     auto [lo, hi] = it->genForCond(rewriter, loc);
     Value step = constantIndex(rewriter, loc, 1);
-    scf::ForOp forOp = rewriter.create<scf::ForOp>(
+    scf::ForOp forOp = scf::ForOp::create(rewriter,
         loc, lo, hi, step, reduc,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
           // Empty builder function to ensure that no terminator is created.
@@ -140,7 +140,7 @@ static ValueRange genLoopWithIterator(
                                            it, forOp.getRegionIterArgs());
 
       rewriter.setInsertionPointToEnd(forOp.getBody());
-      rewriter.create<scf::YieldOp>(loc, ret);
+      scf::YieldOp::create(rewriter, loc, ret);
     }
     return forOp.getResults();
   }
@@ -149,7 +149,7 @@ static ValueRange genLoopWithIterator(
   llvm::append_range(ivs, it->getCursor());
 
   TypeRange types = ValueRange(ivs).getTypes();
-  auto whileOp = rewriter.create<scf::WhileOp>(loc, types, ivs);
+  auto whileOp = scf::WhileOp::create(rewriter, loc, types, ivs);
   {
     OpBuilder::InsertionGuard guard(rewriter);
     // Generates loop conditions.
@@ -158,7 +158,7 @@ static ValueRange genLoopWithIterator(
     rewriter.setInsertionPointToStart(before);
     ValueRange bArgs = before->getArguments();
     auto [whileCond, remArgs] = it->genWhileCond(rewriter, loc, bArgs);
-    rewriter.create<scf::ConditionOp>(loc, whileCond, before->getArguments());
+    scf::ConditionOp::create(rewriter, loc, whileCond, before->getArguments());
 
     // Delegates loop body generation.
     Region &dstRegion = whileOp.getAfter();
@@ -175,7 +175,7 @@ static ValueRange genLoopWithIterator(
     SmallVector<Value> yields;
     llvm::append_range(yields, ret);
     llvm::append_range(yields, it->forward(rewriter, loc));
-    rewriter.create<scf::YieldOp>(loc, yields);
+    scf::YieldOp::create(rewriter, loc, yields);
   }
   return whileOp.getResults().drop_front(it->getCursor().size());
 }
@@ -212,7 +212,7 @@ class ExtractValOpConverter : public OpConversionPattern<ExtractValOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value pos = adaptor.getIterator().back();
-    Value valBuf = rewriter.create<ToValuesOp>(
+    Value valBuf = ToValuesOp::create(rewriter,
         loc, llvm::getSingleElement(adaptor.getTensor()));
     rewriter.replaceOpWithNewOp<memref::LoadOp>(op, valBuf, pos);
     return success();
@@ -385,12 +385,12 @@ class SparseCoIterateOpConverter : public OpConversionPattern<CoIterateOp> {
         SmallVector<Value> nextIterYields(res);
         // 2nd. foward the loop.
         for (SparseIterator *it : validIters) {
-          Value cmp = rewriter.create<arith::CmpIOp>(
+          Value cmp = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::eq, it->getCrd(), loopCrd);
           it->forwardIf(rewriter, loc, cmp);
           llvm::append_range(nextIterYields, it->getCursor());
         }
-        rewriter.create<scf::YieldOp>(loc, nextIterYields);
+        scf::YieldOp::create(rewriter, loc, nextIterYields);
 
         // Exit the loop, relink the iterator SSA value.
         rewriter.setInsertionPointAfter(loop);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
index 0473e058646b9..fd1bbb300b6a6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
@@ -44,7 +44,7 @@ struct DemapInsRewriter : public OpRewritePattern<SourceOp> {
     SmallVector<Value> deMappedIns(op->getOperands());
     for (Value &in : deMappedIns) {
       if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) {
-        in = rewriter.create<ReinterpretMapOp>(loc, stt->getDemappedType(), in);
+        in = ReinterpretMapOp::create(rewriter, loc, stt->getDemappedType(), in);
         changed = true;
       }
     }
@@ -338,14 +338,14 @@ translateMap(linalg::GenericOp op, PatternRewriter &rewriter) {
 // Generates a "de"mapping reinterpretation of the map.
 static Value genDemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
-  return builder.create<ReinterpretMapOp>(val.getLoc(), enc.withoutDimToLvl(),
+  return ReinterpretMapOp::create(builder, val.getLoc(), enc.withoutDimToLvl(),
                                           val);
 }
 
 // Generates a "re"mapping reinterpretation of the map.
 static Value genRemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
-  return builder.create<ReinterpretMapOp>(val.getLoc(), enc, val);
+  return ReinterpretMapOp::create(builder, val.getLoc(), enc, val);
 }
 
 static SmallVector<Value> remapValueRange(OpBuilder &rewriter, TypeRange types,
@@ -354,7 +354,7 @@ static SmallVector<Value> remapValueRange(OpBuilder &rewriter, TypeRange types,
   assert(outs.size() == types.size());
   for (auto [r, t] : llvm::zip(ret, types))
     if (r.getType() != t)
-      r = rewriter.create<ReinterpretMapOp>(r.getLoc(), t, r);
+      r = ReinterpretMapOp::create(rewriter, r.getLoc(), t, r);
   return ret;
 }
 
@@ -567,7 +567,7 @@ struct GenericOpScheduler : public OpRewritePattern<linalg::GenericOp> {
       // Inserting the transpose
       rewriter.setInsertionPoint(linalgOp);
       RankedTensorType dstTp = stt.withDimToLvl(dimToLvl).getRankedTensorType();
-      Value dst = rewriter.create<ConvertOp>(tval.getLoc(), dstTp, tval);
+      Value dst = ConvertOp::create(rewriter, tval.getLoc(), dstTp, tval);
       rewriter.modifyOpInPlace(linalgOp, [&]() {
         linalgOp->setOperand(t->getOperandNumber(), dst);
       });
@@ -575,7 +575,7 @@ struct GenericOpScheduler : public OpRewritePattern<linalg::GenericOp> {
       // Release the transposed form afterwards.
       // TODO: CSE when used in more than one following op?
       rewriter.setInsertionPointAfter(linalgOp);
-      rewriter.create<bufferization::DeallocTensorOp>(dst.getLoc(), dst);
+      bufferization::DeallocTensorOp::create(rewriter, dst.getLoc(), dst);
 
       return success();
     }
@@ -605,7 +605,7 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     ValueRange dynSz = op.getDynamicSizes();
     for (int64_t dimSz : stt.getDimShape()) {
       if (ShapedType::isDynamic(dimSz)) {
-        Value maxCrd = rewriter.create<arith::SubIOp>(
+        Value maxCrd = arith::SubIOp::create(rewriter,
             loc, dynSz.front(), constantIndex(rewriter, loc, 1));
         maxDimCrds.push_back(maxCrd);
         dynSz = dynSz.drop_front();
@@ -620,7 +620,7 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     SmallVector<Value> dynLvlSzs;
     for (unsigned i = 0, e = lvlShape.size(); i < e; i++) {
       if (ShapedType::isDynamic(lvlShape[i])) {
-        Value sz = rewriter.create<arith::AddIOp>(
+        Value sz = arith::AddIOp::create(rewriter,
             loc, maxLvlCrds[i], constantIndex(rewriter, loc, 1));
         dynLvlSzs.push_back(sz);
       }
@@ -651,7 +651,7 @@ struct TensorInsertDemapper
     auto stt = getSparseTensorType(op.getResult());
     ValueRange lvlCrd = stt.translateCrds(rewriter, loc, op.getIndices(),
                                           CrdTransDirectionKind::dim2lvl);
-    auto insertOp = rewriter.create<tensor::InsertOp>(
+    auto insertOp = tensor::InsertOp::create(rewriter,
         loc, op.getScalar(), adaptor.getDest(), lvlCrd);
 
     Value out = genRemap(rewriter, stt.getEncoding(), insertOp.getResult());
@@ -766,7 +766,7 @@ struct ForeachOpDemapper
           stt && !stt->isIdentity()) {
         Value y =
             genDemap(rewriter, stt->getEncoding(), yield.getSingleResult());
-        rewriter.create<YieldOp>(loc, y);
+        YieldOp::create(rewriter, loc, y);
         rewriter.eraseOp(yield);
       }
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
index f85c4761a8d52..7b662c2fb0db0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
@@ -126,7 +126,7 @@ void collapseSparseSpace(MutableArrayRef<CollapseSpaceInfo> toCollapse) {
   OpBuilder builder(root);
 
   // Construct the collapsed iteration space.
-  auto collapsedSpace = builder.create<ExtractIterSpaceOp>(
+  auto collapsedSpace = ExtractIterSpaceOp::create(builder,
       loc, root.getTensor(), root.getParentIter(), root.getLoLvl(),
       leaf.getHiLvl());
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
index 01028f71c20bb..a1b9035f3b6e6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
@@ -69,13 +69,13 @@ class SpecifierStructBuilder : public StructBuilder {
   Value extractField(OpBuilder &builder, Location loc,
                      ArrayRef<int64_t> indices) const {
     return genCast(builder, loc,
-                   builder.create<LLVM::ExtractValueOp>(loc, value, indices),
+                   LLVM::ExtractValueOp::create(builder, loc, value, indices),
                    builder.getIndexType());
   }
 
   void insertField(OpBuilder &builder, Location loc, ArrayRef<int64_t> indices,
                    Value v) {
-    value = builder.create<LLVM::InsertValueOp>(
+    value = LLVM::InsertValueOp::create(builder,
         loc, value, genCast(builder, loc, v, builder.getIntegerType(64)),
         indices);
   }
@@ -110,7 +110,7 @@ class SpecifierStructBuilder : public StructBuilder {
 
 Value SpecifierStructBuilder::getInitValue(OpBuilder &builder, Location loc,
                                            Type structType, Value source) {
-  Value metaData = builder.create<LLVM::PoisonOp>(loc, structType);
+  Value metaData = LLVM::PoisonOp::create(builder, loc, structType);
   SpecifierStructBuilder md(metaData);
   if (!source) {
     auto memSizeArrayType =
@@ -204,14 +204,14 @@ void SpecifierStructBuilder::setMemSize(OpBuilder &builder, Location loc,
 /// Builds IR extracting the memory size array from the descriptor.
 Value SpecifierStructBuilder::memSizeArray(OpBuilder &builder,
                                            Location loc) const {
-  return builder.create<LLVM::ExtractValueOp>(loc, value,
+  return LLVM::ExtractValueOp::create(builder, loc, value,
                                               kMemSizePosInSpecifier);
 }
 
 /// Builds IR inserting the memory size array into the descriptor.
 void SpecifierStructBuilder::setMemSizeArray(OpBuilder &builder, Location loc,
                                              Value array) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, array,
+  value = LLVM::InsertValueOp::create(builder, loc, value, array,
                                               kMemSizePosInSpecifier);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 001ea62b07360..0450e1d6e8775 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -50,7 +50,7 @@ static SmallVector<Value> flattenValues(ArrayRef<ValueRange> values) {
 /// Generates a load with proper `index` typing.
 static Value genLoad(OpBuilder &builder, Location loc, Value mem, Value idx) {
   idx = genCast(builder, loc, idx, builder.getIndexType());
-  return builder.create<memref::LoadOp>(loc, mem, idx);
+  return memref::LoadOp::create(builder, loc, mem, idx);
 }
 
 /// Generates a store with proper `index` typing and proper value.
@@ -59,7 +59,7 @@ static void genStore(OpBuilder &builder, Location loc, Value val, Value mem,
   idx = genCast(builder, loc, idx, builder.getIndexType());
   val = genCast(builder, loc, val,
                 cast<ShapedType>(mem.getType()).getElementType());
-  builder.create<memref::StoreOp>(loc, val, mem, idx);
+  memref::StoreOp::create(builder, loc, val, mem, idx);
 }
 
 /// Creates a straightforward counting for-loop.
@@ -70,7 +70,7 @@ static scf::ForOp createFor(OpBuilder &builder, Location loc, Value upper,
   if (!lower)
     lower = constantZero(builder, loc, indexType);
   Value one = constantOne(builder, loc, indexType);
-  scf::ForOp forOp = builder.create<scf::ForOp>(loc, lower, upper, one, fields);
+  scf::ForOp forOp = scf::ForOp::create(builder, loc, lower, upper, one, fields);
   for (unsigned i = 0, e = fields.size(); i < e; i++)
     fields[i] = forOp.getRegionIterArg(i);
   builder.setInsertionPointToStart(forOp.getBody());
@@ -86,7 +86,7 @@ static void createPushback(OpBuilder &builder, Location loc,
   Value field = desc.getMemRefField(kind, lvl);
   StorageSpecifierKind specFieldKind = toSpecifierKind(kind);
 
-  auto pushBackOp = builder.create<PushBackOp>(
+  auto pushBackOp = PushBackOp::create(builder,
       loc, desc.getSpecifierField(builder, loc, specFieldKind, lvl), field,
       genCast(builder, loc, value, etp), repeat);
 
@@ -112,7 +112,7 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
       Value posZero = constantZero(builder, loc, stt.getPosType());
       if (isLooseCompressedLT(lt)) {
         Value two = constantIndex(builder, loc, 2);
-        linear = builder.create<arith::MulIOp>(loc, linear, two);
+        linear = arith::MulIOp::create(builder, loc, linear, two);
       }
       createPushback(builder, loc, desc, SparseTensorFieldKind::PosMemRef, lvl,
                      /*value=*/posZero, /*repeat=*/linear);
@@ -125,7 +125,7 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
     // otherwise the values array for the from-here "all-dense" case.
     assert(isDenseLT(lt));
     Value size = desc.getLvlSize(builder, loc, lvl);
-    linear = builder.create<arith::MulIOp>(loc, linear, size);
+    linear = arith::MulIOp::create(builder, loc, linear, size);
   }
   // Reached values array so prepare for an insertion.
   Value valZero = constantZero(builder, loc, stt.getElementType());
@@ -137,11 +137,11 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
 static Value createAllocation(OpBuilder &builder, Location loc,
                               MemRefType memRefType, Value sz,
                               bool enableInit) {
-  Value buffer = builder.create<memref::AllocOp>(loc, memRefType, sz);
+  Value buffer = memref::AllocOp::create(builder, loc, memRefType, sz);
   Type elemType = memRefType.getElementType();
   if (enableInit) {
     Value fillValue = constantZero(builder, loc, elemType);
-    builder.create<linalg::FillOp>(loc, fillValue, buffer);
+    linalg::FillOp::create(builder, loc, fillValue, buffer);
   }
   return buffer;
 }
@@ -179,14 +179,14 @@ static void createAllocFields(OpBuilder &builder, Location loc,
     valHeuristic = lvlSizesValues[0];
     for (Level lvl = 1; lvl < lvlRank; lvl++)
       valHeuristic =
-          builder.create<arith::MulIOp>(loc, valHeuristic, lvlSizesValues[lvl]);
+          arith::MulIOp::create(builder, loc, valHeuristic, lvlSizesValues[lvl]);
   } else if (sizeHint) {
     if (stt.getAoSCOOStart() == 0) {
       posHeuristic = constantIndex(builder, loc, 2);
-      crdHeuristic = builder.create<arith::MulIOp>(
+      crdHeuristic = arith::MulIOp::create(builder,
           loc, constantIndex(builder, loc, lvlRank), sizeHint); // AOS
     } else if (lvlRank == 2 && stt.isDenseLvl(0) && stt.isCompressedLvl(1)) {
-      posHeuristic = builder.create<arith::AddIOp>(
+      posHeuristic = arith::AddIOp::create(builder,
           loc, sizeHint, constantIndex(builder, loc, 1));
       crdHeuristic = sizeHint;
     } else {
@@ -280,7 +280,7 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   unsigned crdStride;
   std::tie(crdFidx, crdStride) = desc.getCrdMemRefIndexAndStride(lvl);
   const Value one = constantIndex(builder, loc, 1);
-  const Value pp1 = builder.create<arith::AddIOp>(loc, parentPos, one);
+  const Value pp1 = arith::AddIOp::create(builder, loc, parentPos, one);
   const Value positionsAtLvl = desc.getPosMemRef(lvl);
   const Value pstart = genLoad(builder, loc, positionsAtLvl, parentPos);
   const Value pstop = genLoad(builder, loc, positionsAtLvl, pp1);
@@ -288,29 +288,29 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   const Value crdStrideC =
       crdStride > 1 ? constantIndex(builder, loc, crdStride) : Value();
   const Value msz =
-      crdStrideC ? builder.create<arith::DivUIOp>(loc, crdMsz, crdStrideC)
+      crdStrideC ? arith::DivUIOp::create(builder, loc, crdMsz, crdStrideC)
                  : crdMsz;
-  const Value plast = builder.create<arith::SubIOp>(
+  const Value plast = arith::SubIOp::create(builder,
       loc, genCast(builder, loc, pstop, indexType), one);
   // Conditional expression.
-  Value lt = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value lt = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                            pstart, pstop);
   types.push_back(boolType);
-  scf::IfOp ifOp1 = builder.create<scf::IfOp>(loc, types, lt, /*else*/ true);
+  scf::IfOp ifOp1 = scf::IfOp::create(builder, loc, types, lt, /*else*/ true);
   types.pop_back();
   builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
   Value crd =
       genLoad(builder, loc, desc.getMemRefField(crdFidx),
-              crdStrideC ? builder.create<arith::MulIOp>(loc, plast, crdStrideC)
+              crdStrideC ? arith::MulIOp::create(builder, loc, plast, crdStrideC)
                          : plast);
-  Value eq = builder.create<arith::CmpIOp>(
+  Value eq = arith::CmpIOp::create(builder,
       loc, arith::CmpIPredicate::eq, genCast(builder, loc, crd, indexType),
       lvlCoords[lvl]);
-  builder.create<scf::YieldOp>(loc, eq);
+  scf::YieldOp::create(builder, loc, eq);
   builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
   if (lvl > 0)
     genStore(builder, loc, msz, positionsAtLvl, parentPos);
-  builder.create<scf::YieldOp>(loc, constantI1(builder, loc, false));
+  scf::YieldOp::create(builder, loc, constantI1(builder, loc, false));
   builder.setInsertionPointAfter(ifOp1);
   // If present construct. Note that for a non-unique dimension level, we
   // simply set the condition to false and rely on CSE/DCE to clean up the IR.
@@ -322,19 +322,19 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   types.push_back(indexType);
   const Value p = stt.isUniqueLvl(lvl) ? ifOp1.getResult(0)
                                        : constantI1(builder, loc, false);
-  scf::IfOp ifOp2 = builder.create<scf::IfOp>(loc, types, p, /*else*/ true);
+  scf::IfOp ifOp2 = scf::IfOp::create(builder, loc, types, p, /*else*/ true);
   // If present (fields unaffected, update pnext to plast).
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
 
   // FIXME: This does not looks like a clean way, but probably the most
   // efficient way.
   desc.getFields().push_back(plast);
-  builder.create<scf::YieldOp>(loc, desc.getFields());
+  scf::YieldOp::create(builder, loc, desc.getFields());
   desc.getFields().pop_back();
 
   // If !present (changes fields, update pnext).
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
-  Value mszp1 = builder.create<arith::AddIOp>(loc, msz, one);
+  Value mszp1 = arith::AddIOp::create(builder, loc, msz, one);
   genStore(builder, loc, mszp1, positionsAtLvl, pp1);
   createPushback(builder, loc, desc, SparseTensorFieldKind::CrdMemRef, lvl,
                  /*value=*/lvlCoords[lvl]);
@@ -343,7 +343,7 @@ static Value genCompressed(OpBuilder &builder, Location loc,
     allocSchemeForRank(builder, loc, desc, lvl + 1);
 
   desc.getFields().push_back(msz);
-  builder.create<scf::YieldOp>(loc, desc.getFields());
+  scf::YieldOp::create(builder, loc, desc.getFields());
   desc.getFields().pop_back();
 
   // Update fields and return next pos.
@@ -381,17 +381,17 @@ static void genEndInsert(OpBuilder &builder, Location loc,
         Value oldv = loop.getRegionIterArg(0);
         Value newv = genLoad(builder, loc, posMemRef, i);
         Value posZero = constantZero(builder, loc, posType);
-        Value cond = builder.create<arith::CmpIOp>(
+        Value cond = arith::CmpIOp::create(builder,
             loc, arith::CmpIPredicate::eq, newv, posZero);
-        scf::IfOp ifOp = builder.create<scf::IfOp>(loc, TypeRange(posType),
+        scf::IfOp ifOp = scf::IfOp::create(builder, loc, TypeRange(posType),
                                                    cond, /*else*/ true);
         builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         genStore(builder, loc, oldv, posMemRef, i);
-        builder.create<scf::YieldOp>(loc, oldv);
+        scf::YieldOp::create(builder, loc, oldv);
         builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-        builder.create<scf::YieldOp>(loc, newv);
+        scf::YieldOp::create(builder, loc, newv);
         builder.setInsertionPointAfter(ifOp);
-        builder.create<scf::YieldOp>(loc, ifOp.getResult(0));
+        scf::YieldOp::create(builder, loc, ifOp.getResult(0));
         builder.setInsertionPointAfter(loop);
       }
     } else {
@@ -484,7 +484,7 @@ class SparseInsertGenerator
         //   <insert @ positions[lvl] at next level lvl + 1>
         if (isLooseCompressedLT(lt)) {
           Value two = constantIndex(builder, loc, 2);
-          parentPos = builder.create<arith::MulIOp>(loc, parentPos, two);
+          parentPos = arith::MulIOp::create(builder, loc, parentPos, two);
         }
         parentPos =
             genCompressed(builder, loc, desc, coords, value, parentPos, lvl);
@@ -501,8 +501,8 @@ class SparseInsertGenerator
         //   positions[lvl] = size * positions[lvl-1] + coords[lvl]
         //   <insert @ positions[lvl] at next level lvl + 1>
         Value size = desc.getLvlSize(builder, loc, lvl);
-        Value mult = builder.create<arith::MulIOp>(loc, size, parentPos);
-        parentPos = builder.create<arith::AddIOp>(loc, mult, coords[lvl]);
+        Value mult = arith::MulIOp::create(builder, loc, size, parentPos);
+        parentPos = arith::AddIOp::create(builder, loc, mult, coords[lvl]);
       }
     }
     // Reached the actual value append/insert.
@@ -582,7 +582,7 @@ class SparseCallConverter : public OpConversionPattern<func::CallOp> {
       return failure();
 
     // (1) Generates new call with flattened return value.
-    auto newCall = rewriter.create<func::CallOp>(
+    auto newCall = func::CallOp::create(rewriter,
         loc, op.getCallee(), finalRetTy, flattenValues(adaptor.getOperands()));
     // (2) Gather sparse tensor returns.
     SmallVector<SmallVector<Value>> packedResultVals;
@@ -671,7 +671,7 @@ struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
 
     auto id = AffineMap::getMultiDimIdentityMap(srcStt.getLvlRank(), ctx);
 
-    rewriter.create<SortOp>(loc, nnz, crd, ValueRange{val}, id,
+    SortOp::create(rewriter, loc, nnz, crd, ValueRange{val}, id,
                             rewriter.getIndexAttr(0), op.getAlgorithm());
 
     // Since we do in-place sorting, the destinate tensor will have the same set
@@ -757,10 +757,10 @@ class SparseTensorAllocConverter
       // Memcpy on memref fields.
       for (auto field : desc.getMemRefFields()) {
         auto memrefTp = cast<MemRefType>(field.getType());
-        auto size = rewriter.create<memref::DimOp>(loc, field, 0);
+        auto size = memref::DimOp::create(rewriter, loc, field, 0);
         auto copied =
-            rewriter.create<memref::AllocOp>(loc, memrefTp, ValueRange{size});
-        rewriter.create<memref::CopyOp>(loc, field, copied);
+            memref::AllocOp::create(rewriter, loc, memrefTp, ValueRange{size});
+        memref::CopyOp::create(rewriter, loc, field, copied);
         fields.push_back(copied);
       }
       // Reuses specifier.
@@ -863,7 +863,7 @@ class SparseTensorDeallocConverter
           cast<RankedTensorType>(op.getTensor().getType()));
       for (auto input : desc.getMemRefFields())
         // Deallocate every buffer used to store the sparse tensor handler.
-        rewriter.create<memref::DeallocOp>(loc, input);
+        memref::DeallocOp::create(rewriter, loc, input);
     }
     rewriter.eraseOp(op);
     return success();
@@ -917,7 +917,7 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // Generate a memref for `sz` elements of type `t`.
     const auto genAlloc = [&](Type t) {
       const auto memTp = MemRefType::get({ShapedType::kDynamic}, t);
-      return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
+      return memref::AllocOp::create(rewriter, loc, memTp, ValueRange{sz});
     };
     // Allocate temporary buffers for values/filled-switch and added.
     // We do not use stack buffers for this, since the expanded size may
@@ -931,10 +931,10 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, eltType)},
         ValueRange{values});
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, boolType)},
         ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
@@ -965,7 +965,7 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     // If the innermost level is ordered, we need to sort the coordinates
     // in the "added" array prior to applying the compression.
     if (dstType.isOrderedLvl(dstType.getLvlRank() - 1))
-      rewriter.create<SortOp>(
+      SortOp::create(rewriter,
           loc, count, added, ValueRange{}, rewriter.getMultiDimIdentityMap(1),
           rewriter.getIndexAttr(0), SparseTensorSortKind::HybridQuickSort);
     // While performing the insertions, we also need to reset the elements
@@ -1000,15 +1000,15 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     SmallVector<Value> insertRet = insertGen.genCallOrInline(rewriter, loc);
     genStore(rewriter, loc, constantZero(rewriter, loc, eltType), values, crd);
     genStore(rewriter, loc, constantI1(rewriter, loc, false), filled, crd);
-    rewriter.create<scf::YieldOp>(loc, insertRet);
+    scf::YieldOp::create(rewriter, loc, insertRet);
 
     rewriter.setInsertionPointAfter(loop);
     // Deallocate the buffers on exit of the full loop nest.
     Operation *parent = getTop(op);
     rewriter.setInsertionPointAfter(parent);
-    rewriter.create<memref::DeallocOp>(loc, values);
-    rewriter.create<memref::DeallocOp>(loc, filled);
-    rewriter.create<memref::DeallocOp>(loc, added);
+    memref::DeallocOp::create(rewriter, loc, values);
+    memref::DeallocOp::create(rewriter, loc, filled);
+    memref::DeallocOp::create(rewriter, loc, added);
     // Replace operation with resulting memrefs.
     rewriter.replaceOpWithMultiple(op, {loop->getResults()});
     return success();
@@ -1192,7 +1192,7 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
             // would require a subViewOp to avoid overflow when copying
             // values.
             Value sz = linalg::createOrFoldDimOp(rewriter, loc, srcMem, 0);
-            auto dstMem = rewriter.create<memref::AllocOp>(
+            auto dstMem = memref::AllocOp::create(rewriter,
                 loc, cast<MemRefType>(fTp), sz);
             if (fTp != srcMem.getType()) {
               // Converts elements type.
@@ -1201,16 +1201,16 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
                   constantIndex(rewriter, loc, 1),
                   [srcMem, &dstMem](OpBuilder &builder, Location loc,
                                     ValueRange ivs) {
-                    Value v = builder.create<memref::LoadOp>(loc, srcMem, ivs);
+                    Value v = memref::LoadOp::create(builder, loc, srcMem, ivs);
                     Value casted = genCast(builder, loc, v,
                                            dstMem.getType().getElementType());
-                    builder.create<memref::StoreOp>(loc, casted, dstMem, ivs);
+                    memref::StoreOp::create(builder, loc, casted, dstMem, ivs);
                   });
             } else {
               // TODO: We can even reuse the same memref for the new tensor,
               // but that requires a `ref-counting` based memory management
               // for shared memrefs between multiple sparse tensors.
-              rewriter.create<memref::CopyOp>(loc, srcMem, dstMem);
+              memref::CopyOp::create(rewriter, loc, srcMem, dstMem);
             }
             fields.push_back(dstMem);
           }
@@ -1242,7 +1242,7 @@ class SparseExtractSliceConverter
     auto desc = getMutDescriptorFromTensorTuple(adaptor.getSource(), fields,
                                                 op.getSource().getType());
 
-    auto newSpec = rewriter.create<StorageSpecifierInitOp>(
+    auto newSpec = StorageSpecifierInitOp::create(rewriter,
         loc, StorageSpecifierType::get(ctx, dstEnc), desc.getSpecifier());
     desc.setSpecifier(newSpec);
 
@@ -1326,11 +1326,11 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
               // Flattens the buffer to batchLvlRank.
               auto reassoc = getReassociationForFlattening(
                   mem.getType(), stt.getBatchLvlRank());
-              mem = rewriter.create<memref::CastOp>(
+              mem = memref::CastOp::create(rewriter,
                   loc, fType,
-                  rewriter.create<memref::CollapseShapeOp>(loc, mem, reassoc));
+                  memref::CollapseShapeOp::create(rewriter, loc, mem, reassoc));
             } else {
-              mem = rewriter.create<memref::CastOp>(loc, fType, mem);
+              mem = memref::CastOp::create(rewriter, loc, fType, mem);
             }
             fields.push_back(mem);
           }
@@ -1362,8 +1362,8 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       LevelType lt = stt.getLvlType(lvl);
       // Simply forwards the position index when this is a dense level.
       if (lt.isa<LevelFormat::Dense>()) {
-        memSize = rewriter.create<arith::MulIOp>(loc, lvlSize, memSize);
-        posBack = rewriter.create<arith::SubIOp>(loc, memSize, c1);
+        memSize = arith::MulIOp::create(rewriter, loc, lvlSize, memSize);
+        posBack = arith::SubIOp::create(rewriter, loc, memSize, c1);
         continue;
       }
       if (lt.isa<LevelFormat::Batch>()) {
@@ -1376,12 +1376,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       if (isWithPosLT(lt)) {
         assert(isCompressedLT(lt) || isLooseCompressedLT(lt));
         if (isLooseCompressedLT(lt)) {
-          memSize = rewriter.create<arith::MulIOp>(loc, memSize, c2);
-          posBack = rewriter.create<arith::SubIOp>(loc, memSize, c1);
+          memSize = arith::MulIOp::create(rewriter, loc, memSize, c2);
+          posBack = arith::SubIOp::create(rewriter, loc, memSize, c1);
         } else {
           assert(isCompressedLT(lt));
           posBack = memSize;
-          memSize = rewriter.create<arith::AddIOp>(loc, memSize, c1);
+          memSize = arith::AddIOp::create(rewriter, loc, memSize, c1);
         }
         desc.setPosMemSize(rewriter, loc, lvl, memSize);
         // The last value in position array is the memory size for next level.
@@ -1391,12 +1391,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
                                    constantIndex(rewriter, loc, 0));
         batched.push_back(posBack);
         memSize = genIndexLoad(rewriter, loc, desc.getPosMemRef(lvl), batched);
-        posBack = rewriter.create<arith::SubIOp>(loc, posBack, c1);
+        posBack = arith::SubIOp::create(rewriter, loc, posBack, c1);
       }
       assert(isWithCrdLT(lt) && lvl <= trailCOOStart);
       // FIXME: This seems to be unnecessarily complex, can we simplify it?
       if (lvl == trailCOOStart) {
-        Value cooSz = rewriter.create<arith::MulIOp>(
+        Value cooSz = arith::MulIOp::create(rewriter,
             loc, memSize, constantIndex(rewriter, loc, trailCOORank));
         desc.setCrdMemSize(rewriter, loc, lvl, cooSz);
       } else {
@@ -1460,18 +1460,18 @@ struct SparseDisassembleOpConverter
       if (dst.getType().getRank() > stt.getBatchLvlRank() + 1) {
         auto reassoc =
             getReassociationForFlattening(dst.getType(), stt.getBatchLvlRank());
-        flatOut = rewriter.create<memref::CollapseShapeOp>(loc, dst, reassoc);
+        flatOut = memref::CollapseShapeOp::create(rewriter, loc, dst, reassoc);
       }
       Value dstMem = genSliceToSize(rewriter, loc, flatOut, sz);
       Value srcMem = genSliceToSize(rewriter, loc, src, sz);
-      rewriter.create<memref::CopyOp>(loc, srcMem, dstMem);
+      memref::CopyOp::create(rewriter, loc, srcMem, dstMem);
       return true;
     });
 
     // Converts MemRefs back to Tensors.
     SmallVector<Value> retValues = llvm::to_vector(
         llvm::map_range(retMem, [&rewriter, loc](Value v) -> Value {
-          return rewriter.create<bufferization::ToTensorOp>(
+          return bufferization::ToTensorOp::create(rewriter,
               loc, memref::getTensorTypeFromMemRefType(v.getType()), v);
         }));
     // Appends the actual memory length used in each buffer returned.
@@ -1549,13 +1549,13 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Level lvlRank = dstTp.getLvlRank();
     if (dstTp.isOrderedLvl(lvlRank - 1)) {
       Value kFalse = constantI1(rewriter, loc, false);
-      Value notSorted = rewriter.create<arith::CmpIOp>(
+      Value notSorted = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, isSorted, kFalse);
       scf::IfOp ifOp =
-          rewriter.create<scf::IfOp>(loc, notSorted, /*else*/ false);
+          scf::IfOp::create(rewriter, loc, notSorted, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       auto xPerm = rewriter.getMultiDimIdentityMap(lvlRank);
-      rewriter.create<SortOp>(loc, nse, xs, ValueRange{ys}, xPerm,
+      SortOp::create(rewriter, loc, nse, xs, ValueRange{ys}, xPerm,
                               rewriter.getIndexAttr(0),
                               SparseTensorSortKind::HybridQuickSort);
       rewriter.setInsertionPointAfter(ifOp);
@@ -1566,10 +1566,10 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Value posMemref0 = desc.getPosMemRef(0);
     const Type posTp = dstTp.getPosType();
     const Value posNse = genCast(rewriter, loc, nse, posTp);
-    rewriter.create<memref::StoreOp>(loc, posNse, posMemref0, c1);
+    memref::StoreOp::create(rewriter, loc, posNse, posMemref0, c1);
 
     // Update storage specifier.
-    Value coordinatesSize = rewriter.create<arith::MulIOp>(
+    Value coordinatesSize = arith::MulIOp::create(rewriter,
         loc, nse, constantIndex(rewriter, loc, lvlRank));
     desc.setSpecifierField(rewriter, loc, StorageSpecifierKind::CrdMemSize, 0,
                            coordinatesSize);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 50ccb43d432b6..54ea1843b457f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -137,7 +137,7 @@ static SmallVector<Value> getDimSizes(OpBuilder &builder, Location loc,
 /// this buffer must be explicitly deallocated by client.
 static Value genAlloc(RewriterBase &rewriter, Location loc, Value sz, Type tp) {
   auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
-  return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
+  return memref::AllocOp::create(rewriter, loc, memTp, ValueRange{sz});
 }
 
 /// Generates a temporary buffer for the level-types of the given encoding.
@@ -154,7 +154,7 @@ static Value genLvlTypesBuffer(OpBuilder &builder, Location loc,
 static Value extractBarePtrFromTensor(OpBuilder &builder, Location loc,
                                       Value tensor) {
   auto buf = genToMemref(builder, loc, tensor);
-  return builder.create<memref::ExtractAlignedPointerAsIndexOp>(loc, buf);
+  return memref::ExtractAlignedPointerAsIndexOp::create(builder, loc, buf);
 }
 
 /// Generates a temporary buffer for the level-types of the given encoding.
@@ -168,11 +168,11 @@ static Value genLvlPtrsBuffers(OpBuilder &builder, Location loc,
 
   // Passing in value buffer pointers.
   lvlBarePtrs.push_back(extractBarePtrFromTensor(builder, loc, valTensor));
-  Value idxPtr = builder.create<memref::ExtractAlignedPointerAsIndexOp>(
+  Value idxPtr = memref::ExtractAlignedPointerAsIndexOp::create(builder,
       loc, allocaBuffer(builder, loc, lvlBarePtrs));
   Value idxCast =
-      builder.create<arith::IndexCastOp>(loc, builder.getI64Type(), idxPtr);
-  return builder.create<LLVM::IntToPtrOp>(loc, getOpaquePointerType(builder),
+      arith::IndexCastOp::create(builder, loc, builder.getI64Type(), idxPtr);
+  return LLVM::IntToPtrOp::create(builder, loc, getOpaquePointerType(builder),
                                           idxCast);
 }
 
@@ -227,7 +227,7 @@ class NewCallParams final {
     assert(isInitialized() && "Must initialize before genNewCall");
     StringRef name = "newSparseTensor";
     params[kParamAction] = constantAction(builder, loc, action);
-    params[kParamPtr] = ptr ? ptr : builder.create<LLVM::ZeroOp>(loc, pTp);
+    params[kParamPtr] = ptr ? ptr : LLVM::ZeroOp::create(builder, loc, pTp);
     return createFuncCall(builder, loc, name, pTp, params, EmitCInterface::On)
         .getResult(0);
   }
@@ -539,7 +539,7 @@ class SparseTensorToCoordinatesConverter
     // Cast the MemRef type to the type expected by the users, though these
     // two types should be compatible at runtime.
     if (op.getType() != crds.getType())
-      crds = rewriter.create<memref::CastOp>(loc, op.getType(), crds);
+      crds = memref::CastOp::create(rewriter, loc, op.getType(), crds);
     rewriter.replaceOp(op, crds);
     return success();
   }
@@ -560,7 +560,7 @@ class SparseToCoordinatesBufferConverter
     // Cast the MemRef type to the type expected by the users, though these
     // two types should be compatible at runtime.
     if (op.getType() != crds.getType())
-      crds = rewriter.create<memref::CastOp>(loc, op.getType(), crds);
+      crds = memref::CastOp::create(rewriter, loc, op.getType(), crds);
     rewriter.replaceOp(op, crds);
     return success();
   }
@@ -652,7 +652,7 @@ class SparseTensorInsertConverter
       vref = genAllocaScalar(rewriter, loc, elemTp);
     }
     storeAll(rewriter, loc, lvlCoords, adaptor.getIndices());
-    rewriter.create<memref::StoreOp>(loc, adaptor.getScalar(), vref);
+    memref::StoreOp::create(rewriter, loc, adaptor.getScalar(), vref);
     SmallString<12> name{"lexInsert", primaryTypeFunctionSuffix(elemTp)};
     createFuncCall(rewriter, loc, name, {},
                    {adaptor.getDest(), lvlCoords, vref}, EmitCInterface::On);
@@ -690,10 +690,10 @@ class SparseTensorExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, eltType)},
         ValueRange{values});
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, boolType)},
         ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
@@ -733,9 +733,9 @@ class SparseTensorCompressConverter : public OpConversionPattern<CompressOp> {
     rewriter.replaceOp(op, adaptor.getTensor());
     // Deallocate the buffers on exit of the loop nest.
     rewriter.setInsertionPointAfter(parent);
-    rewriter.create<memref::DeallocOp>(loc, values);
-    rewriter.create<memref::DeallocOp>(loc, filled);
-    rewriter.create<memref::DeallocOp>(loc, added);
+    memref::DeallocOp::create(rewriter, loc, values);
+    memref::DeallocOp::create(rewriter, loc, filled);
+    memref::DeallocOp::create(rewriter, loc, added);
     return success();
   }
 };
@@ -837,21 +837,21 @@ class SparseTensorDisassembleConverter
                                       cooStartLvl + 1);
       auto crdLen = linalg::createOrFoldDimOp(rewriter, loc, crds0, 0);
       auto two = constantIndex(rewriter, loc, 2);
-      auto bufLen = rewriter.create<arith::MulIOp>(loc, crdLen, two);
+      auto bufLen = arith::MulIOp::create(rewriter, loc, crdLen, two);
       Type indexType = rewriter.getIndexType();
       auto zero = constantZero(rewriter, loc, indexType);
       auto one = constantOne(rewriter, loc, indexType);
-      scf::ForOp forOp = rewriter.create<scf::ForOp>(loc, zero, crdLen, one);
+      scf::ForOp forOp = scf::ForOp::create(rewriter, loc, zero, crdLen, one);
       auto idx = forOp.getInductionVar();
       rewriter.setInsertionPointToStart(forOp.getBody());
-      auto c0 = rewriter.create<memref::LoadOp>(loc, crds0, idx);
-      auto c1 = rewriter.create<memref::LoadOp>(loc, crds1, idx);
+      auto c0 = memref::LoadOp::create(rewriter, loc, crds0, idx);
+      auto c1 = memref::LoadOp::create(rewriter, loc, crds1, idx);
       SmallVector<Value> args;
       args.push_back(idx);
       args.push_back(zero);
-      rewriter.create<memref::StoreOp>(loc, c0, buf, args);
+      memref::StoreOp::create(rewriter, loc, c0, buf, args);
       args[1] = one;
-      rewriter.create<memref::StoreOp>(loc, c1, buf, args);
+      memref::StoreOp::create(rewriter, loc, c1, buf, args);
       rewriter.setInsertionPointAfter(forOp);
       auto bufLenTp = op.getLvlLens().getTypes()[retLen.size()];
       retVal.push_back(buf);
@@ -867,11 +867,11 @@ class SparseTensorDisassembleConverter
     // Converts MemRefs back to Tensors.
     assert(retVal.size() + retLen.size() == op.getNumResults());
     for (unsigned i = 0, sz = retVal.size(); i < sz; i++) {
-      auto tensor = rewriter.create<bufferization::ToTensorOp>(
+      auto tensor = bufferization::ToTensorOp::create(rewriter,
           loc, memref::getTensorTypeFromMemRefType(retVal[i].getType()),
           retVal[i]);
       retVal[i] =
-          rewriter.create<tensor::CastOp>(loc, op.getResultTypes()[i], tensor);
+          tensor::CastOp::create(rewriter, loc, op.getResultTypes()[i], tensor);
     }
 
     // Appends the actual memory length used in each buffer returned.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index d4a02bf7a70b6..ee963ab5d1ffb 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -127,7 +127,7 @@ static void sizesForTensor(OpBuilder &builder, SmallVectorImpl<Value> &sizes,
   for (const auto &d : enumerate(stp.getShape())) {
     Value dim;
     if (d.value() == ShapedType::kDynamic)
-      dim = builder.create<tensor::DimOp>(loc, tensor, d.index());
+      dim = tensor::DimOp::create(builder, loc, tensor, d.index());
     else
       dim = constantIndex(builder, loc, d.value());
     sizes.push_back(dim);
@@ -198,7 +198,7 @@ static void concatSizesFromInputs(OpBuilder &builder,
     for (const auto &src : srcs.drop_front()) {
       Value srcSz = linalg::createOrFoldDimOp(builder, loc, src, dim);
       // Sum up all the sizes.
-      sizes[dim] = builder.create<arith::AddIOp>(loc, sizes[dim], srcSz);
+      sizes[dim] = arith::AddIOp::create(builder, loc, sizes[dim], srcSz);
     }
   }
 }
@@ -405,7 +405,7 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     inputOps.push_back(op.getDpsInputOperand(1 - other)->get());
     fusedIndexMaps.push_back(fusedIndexMaps.back()); // mimic other
     // Fuse producer and consumer into a new generic op.
-    auto fusedOp = rewriter.create<GenericOp>(
+    auto fusedOp = GenericOp::create(rewriter,
         loc, op.getResult(0).getType(), inputOps, outputOps,
         rewriter.getAffineMapArrayAttr(fusedIndexMaps), prod.getIteratorTypes(),
         /*doc=*/nullptr, /*library_call=*/nullptr);
@@ -430,7 +430,7 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     mapper.map(consBlock.getArgument(other), fusedBlock->back().getResult(0));
     mapper.map(last, rewriter.clone(*sampler, mapper)->getResult(0));
     last = rewriter.clone(*acc, mapper)->getResult(0);
-    rewriter.create<linalg::YieldOp>(loc, last);
+    linalg::YieldOp::create(rewriter, loc, last);
     // Force initial value on merged allocation for dense outputs.
     // TODO: deal with non alloc tensor here one day
     if (!getSparseTensorEncoding(op.getResult(0).getType())) {
@@ -534,7 +534,7 @@ struct GenSemiRingSelect : public OpRewritePattern<GenericOp> {
       assert(t.getType() == f.getType());
       auto selTp = t.getType();
       auto c0 = constantZero(rewriter, loc, selTp);
-      auto binOp = rewriter.create<sparse_tensor::BinaryOp>(loc, selTp, t, f);
+      auto binOp = sparse_tensor::BinaryOp::create(rewriter, loc, selTp, t, f);
       // Initializes all the blocks.
       rewriter.createBlock(&binOp.getOverlapRegion(), {}, {selTp, selTp},
                            {t.getLoc(), f.getLoc()});
@@ -564,7 +564,7 @@ struct GenSemiRingSelect : public OpRewritePattern<GenericOp> {
           irMap.map(f, b->getArgument(1));
         }
         auto y = rewriter.clone(inst, irMap)->getResult(0);
-        rewriter.create<sparse_tensor::YieldOp>(loc, y);
+        sparse_tensor::YieldOp::create(rewriter, loc, y);
       }
 
       // We successfully rewrited a operation. We can not do replacement here
@@ -674,28 +674,28 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     // Identity.
     Location loc = op.getLoc();
     Value identity =
-        rewriter.create<tensor::ExtractOp>(loc, init->get(), ValueRange());
+        tensor::ExtractOp::create(rewriter, loc, init->get(), ValueRange());
     // Unary {
     //    present -> value
     //    absent  -> zero.
     // }
     Type rtp = s0.getType();
     rewriter.setInsertionPointToStart(&op.getRegion().front());
-    auto semiring = rewriter.create<sparse_tensor::UnaryOp>(loc, rtp, s0);
+    auto semiring = sparse_tensor::UnaryOp::create(rewriter, loc, rtp, s0);
     Block *present =
         rewriter.createBlock(&semiring.getPresentRegion(), {}, rtp, loc);
     rewriter.setInsertionPointToStart(&semiring.getPresentRegion().front());
-    rewriter.create<sparse_tensor::YieldOp>(loc, present->getArgument(0));
+    sparse_tensor::YieldOp::create(rewriter, loc, present->getArgument(0));
     rewriter.createBlock(&semiring.getAbsentRegion(), {}, {}, {});
     rewriter.setInsertionPointToStart(&semiring.getAbsentRegion().front());
     auto zero =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(rtp));
-    rewriter.create<sparse_tensor::YieldOp>(loc, zero);
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(rtp));
+    sparse_tensor::YieldOp::create(rewriter, loc, zero);
     rewriter.setInsertionPointAfter(semiring);
     // CustomReduce {
     //    x = x REDUC y, identity
     // }
-    auto custom = rewriter.create<sparse_tensor::ReduceOp>(
+    auto custom = sparse_tensor::ReduceOp::create(rewriter,
         loc, rtp, semiring.getResult(), s1, identity);
     Block *region =
         rewriter.createBlock(&custom.getRegion(), {}, {rtp, rtp}, {loc, loc});
@@ -704,7 +704,7 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     irMap.map(red->getOperand(0), region->getArgument(0));
     irMap.map(red->getOperand(1), region->getArgument(1));
     auto *cloned = rewriter.clone(*red, irMap);
-    rewriter.create<sparse_tensor::YieldOp>(loc, cloned->getResult(0));
+    sparse_tensor::YieldOp::create(rewriter, loc, cloned->getResult(0));
     rewriter.setInsertionPointAfter(custom);
     rewriter.replaceOp(red, custom.getResult());
     return success();
@@ -723,14 +723,14 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto tensor = op.getTensor();
     auto stt = getSparseTensorType(tensor);
     // Header with NSE.
-    auto nse = rewriter.create<NumberOfEntriesOp>(loc, tensor);
-    rewriter.create<vector::PrintOp>(
+    auto nse = NumberOfEntriesOp::create(rewriter, loc, tensor);
+    vector::PrintOp::create(rewriter,
         loc, rewriter.getStringAttr("---- Sparse Tensor ----\nnse = "));
-    rewriter.create<vector::PrintOp>(loc, nse);
+    vector::PrintOp::create(rewriter, loc, nse);
     // Print run-time contents for dim/lvl sizes.
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("dim = "));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("dim = "));
     printSizes(rewriter, loc, tensor, stt.getDimRank(), /*isDim=*/true);
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("lvl = "));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("lvl = "));
     printSizes(rewriter, loc, tensor, stt.getLvlRank(), /*isDim=*/false);
     // Use the "codegen" foreach loop construct to iterate over
     // all typical sparse tensor components for printing.
@@ -744,42 +744,42 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       }
       case SparseTensorFieldKind::PosMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("pos["));
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("pos["));
+        vector::PrintOp::create(rewriter,
             loc, lvl, vector::PrintPunctuation::NoPunctuation);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("] : "));
-        auto pos = rewriter.create<ToPositionsOp>(loc, tensor, l);
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
+        auto pos = ToPositionsOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, pos);
         break;
       }
       case SparseTensorFieldKind::CrdMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("crd["));
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("crd["));
+        vector::PrintOp::create(rewriter,
             loc, lvl, vector::PrintPunctuation::NoPunctuation);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("] : "));
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
         Value crd = nullptr;
         // For COO AoS storage, we want to print a single, linear view of
         // the full coordinate storage at this level. For any other storage,
         // we show the coordinate storage for every indivual level.
         if (stt.getAoSCOOStart() == l)
-          crd = rewriter.create<ToCoordinatesBufferOp>(loc, tensor);
+          crd = ToCoordinatesBufferOp::create(rewriter, loc, tensor);
         else
-          crd = rewriter.create<ToCoordinatesOp>(loc, tensor, l);
+          crd = ToCoordinatesOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, crd);
         break;
       }
       case SparseTensorFieldKind::ValMemRef: {
-        rewriter.create<vector::PrintOp>(loc,
+        vector::PrintOp::create(rewriter, loc,
                                          rewriter.getStringAttr("values : "));
-        auto val = rewriter.create<ToValuesOp>(loc, tensor);
+        auto val = ToValuesOp::create(rewriter, loc, tensor);
         printContents(rewriter, loc, val);
         break;
       }
       }
       return true;
     });
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("----\n"));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("----\n"));
     rewriter.eraseOp(op);
     return success();
   }
@@ -797,7 +797,7 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto shape = cast<ShapedType>(vec.getType()).getShape();
     SmallVector<Value> idxs;
     printContentsLevel(rewriter, loc, vec, 0, shape, idxs);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::NewLine);
   }
 
   // Helper to the helper.
@@ -805,13 +805,13 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
                                  Value vec, unsigned i, ArrayRef<int64_t> shape,
                                  SmallVectorImpl<Value> &idxs) {
     // Open bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
     // Generate for loop.
     auto zero = constantIndex(rewriter, loc, 0);
     auto index = constantIndex(rewriter, loc, i);
-    auto size = rewriter.create<memref::DimOp>(loc, vec, index);
+    auto size = memref::DimOp::create(rewriter, loc, vec, index);
     auto step = constantIndex(rewriter, loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, zero, size, step);
+    auto forOp = scf::ForOp::create(rewriter, loc, zero, size, step);
     idxs.push_back(forOp.getInductionVar());
     rewriter.setInsertionPointToStart(forOp.getBody());
     if (i < shape.size() - 1) {
@@ -819,56 +819,56 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       printContentsLevel(rewriter, loc, vec, i + 1, shape, idxs);
     } else {
       // Actual contents printing.
-      auto val = rewriter.create<memref::LoadOp>(loc, vec, idxs);
+      auto val = memref::LoadOp::create(rewriter, loc, vec, idxs);
       if (llvm::isa<ComplexType>(val.getType())) {
         // Since the vector dialect does not support complex types in any op,
         // we split those into (real, imag) pairs here.
-        Value real = rewriter.create<complex::ReOp>(loc, val);
-        Value imag = rewriter.create<complex::ImOp>(loc, val);
-        rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
-        rewriter.create<vector::PrintOp>(loc, real,
+        Value real = complex::ReOp::create(rewriter, loc, val);
+        Value imag = complex::ImOp::create(rewriter, loc, val);
+        vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
+        vector::PrintOp::create(rewriter, loc, real,
                                          vector::PrintPunctuation::Comma);
-        rewriter.create<vector::PrintOp>(loc, imag,
+        vector::PrintOp::create(rewriter, loc, imag,
                                          vector::PrintPunctuation::Close);
       } else {
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter,
             loc, val, vector::PrintPunctuation::NoPunctuation);
       }
       // Terminating comma (except at end).
-      auto bound = rewriter.create<arith::AddIOp>(loc, idxs.back(), step);
-      Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+      auto bound = arith::AddIOp::create(rewriter, loc, idxs.back(), step);
+      Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
                                                   bound, size);
-      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, cond, /*else*/ false);
+      scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, cond, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Comma);
+      vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Comma);
     }
     idxs.pop_back();
     rewriter.setInsertionPointAfter(forOp);
     // Close bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
   }
 
   // Helper method to print run-time lvl/dim sizes.
   static void printSizes(PatternRewriter &rewriter, Location loc, Value tensor,
                          unsigned size, bool isDim) {
     // Open bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
     // Print unrolled contents (dimop requires constant value).
     for (unsigned i = 0; i < size; i++) {
       auto idx = constantIndex(rewriter, loc, i);
       Value val;
       if (isDim)
-        val = rewriter.create<tensor::DimOp>(loc, tensor, idx);
+        val = tensor::DimOp::create(rewriter, loc, tensor, idx);
       else
-        val = rewriter.create<LvlOp>(loc, tensor, idx);
-      rewriter.create<vector::PrintOp>(
+        val = LvlOp::create(rewriter, loc, tensor, idx);
+      vector::PrintOp::create(rewriter,
           loc, val,
           i != size - 1 ? vector::PrintPunctuation::Comma
                         : vector::PrintPunctuation::NoPunctuation);
     }
     // Close bracket and end of line.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::NewLine);
   }
 };
 
@@ -896,7 +896,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     for (Dimension d : dstTp->getDimShape())
       dstSizes.push_back(constantIndex(rewriter, loc, d));
 
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, srcTensor);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, srcTensor);
     // Only need an unordered COO buffer if input and output are not sorted
     // in the same way.
     Type bufferTp = getBufferType(
@@ -920,7 +920,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp->getEncoding();
-    ForeachOp foreachOp = rewriter.create<ForeachOp>(
+    ForeachOp foreachOp = ForeachOp::create(rewriter,
         loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
@@ -935,7 +935,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
           Value collapseSize = constantIndex(builder, loc, 1);
           for (Dimension d = 0; d < srcRank; d++)
             collapseSize =
-                builder.create<arith::MulIOp>(loc, collapseSize, srcSizes[d]);
+                arith::MulIOp::create(builder, loc, collapseSize, srcSizes[d]);
           SmallVector<Value, 1> collapsedSizes = {collapseSize};
 
           ReassociationIndices collapseIdx;
@@ -955,15 +955,15 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
                      dstSizes, dstDcvs);
 
           auto t =
-              builder.create<tensor::InsertOp>(loc, v, reduc.front(), dstDcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+              tensor::InsertOp::create(builder, loc, v, reduc.front(), dstDcvs);
+          sparse_tensor::YieldOp::create(builder, loc, t);
         });
 
-    Value t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
+    Value t = LoadOp::create(rewriter, loc, foreachOp.getResult(0), true);
     if (bufferTp != *dstTp) {
       auto dstRTT = dstTp->getRankedTensorType();
-      Value converted = rewriter.create<ConvertOp>(loc, dstRTT, t).getResult();
-      rewriter.create<DeallocTensorOp>(loc, t);
+      Value converted = ConvertOp::create(rewriter, loc, dstRTT, t).getResult();
+      DeallocTensorOp::create(rewriter, loc, t);
       t = converted;
     }
     rewriter.replaceOp(op, t);
@@ -1004,7 +1004,7 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           dstDynSizes.push_back(dstSizes[idx]);
       }
     }
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, srcTensor);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, srcTensor);
     // Only need a unordered COO buffer if input and output are not sorted
     // in the same way.
     Type bufferTp = getBufferType(
@@ -1025,7 +1025,7 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp.getEncoding();
-    ForeachOp foreachOp = rewriter.create<ForeachOp>(
+    ForeachOp foreachOp = ForeachOp::create(rewriter,
         loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
@@ -1040,15 +1040,15 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           reshapeCvs(builder, loc, op.getReassociationIndices(), srcSizes,
                      srcDcvs, dstSizes, dstDcvs);
           auto t =
-              builder.create<tensor::InsertOp>(loc, v, reduc.front(), dstDcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+              tensor::InsertOp::create(builder, loc, v, reduc.front(), dstDcvs);
+          sparse_tensor::YieldOp::create(builder, loc, t);
         });
 
-    Value t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
+    Value t = LoadOp::create(rewriter, loc, foreachOp.getResult(0), true);
     if (bufferTp != dstTp) {
       auto dstRTT = dstTp.getRankedTensorType();
-      Value converted = rewriter.create<ConvertOp>(loc, dstRTT, t).getResult();
-      rewriter.create<DeallocTensorOp>(loc, t);
+      Value converted = ConvertOp::create(rewriter, loc, dstRTT, t).getResult();
+      DeallocTensorOp::create(rewriter, loc, t);
       t = converted;
     }
     rewriter.replaceOp(op, t);
@@ -1079,7 +1079,7 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
       auto rtp = getRankedTensorType(op.getSrc());
       auto denseTp =
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
-      auto convert = rewriter.create<ConvertOp>(loc, denseTp, op.getSrc());
+      auto convert = ConvertOp::create(rewriter, loc, denseTp, op.getSrc());
       rewriter.modifyOpInPlace(op, [&]() { op->setOperand(0, convert); });
       return success();
     }
@@ -1089,14 +1089,14 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
       ReshapeOp reshape;
       if constexpr (std::is_same<ReshapeOp, tensor::ExpandShapeOp>::value) {
-        reshape = rewriter.create<ReshapeOp>(
+        reshape = ReshapeOp::create(rewriter,
             loc, denseTp, op.getSrc(), op.getReassociation(),
             op.getOutputShape(), op.getStaticOutputShape());
       } else {
-        reshape = rewriter.create<ReshapeOp>(loc, denseTp, op.getSrc(),
+        reshape = ReshapeOp::create(rewriter, loc, denseTp, op.getSrc(),
                                              op.getReassociation());
       }
-      Value convert = rewriter.create<ConvertOp>(loc, rtp, reshape);
+      Value convert = ConvertOp::create(rewriter, loc, rtp, reshape);
       rewriter.replaceOp(op, convert);
       return success();
     }
@@ -1112,20 +1112,20 @@ struct TensorLike {
     SmallVector<Value> dynSzs;
     getDynamicSizes(rtt, sizes, dynSzs);
 
-    val = builder.create<AllocTensorOp>(loc, rtt, dynSzs);
+    val = AllocTensorOp::create(builder, loc, rtt, dynSzs);
     if (!isSparse()) {
       Value c0 = constantZero(builder, loc, rtt.getElementType());
-      val = builder.create<linalg::FillOp>(loc, c0, val).getResult(0);
+      val = linalg::FillOp::create(builder, loc, c0, val).getResult(0);
     }
   }
 
   void insert(OpBuilder &builder, Location loc, Value v, ValueRange crds) {
-    val = builder.create<tensor::InsertOp>(loc, v, val, crds);
+    val = tensor::InsertOp::create(builder, loc, v, val, crds);
   }
 
   Value finalize(OpBuilder &builder, Location loc, RankedTensorType rtp) const {
     if (isSparse())
-      return builder.create<LoadOp>(loc, val, true);
+      return LoadOp::create(builder, loc, val, true);
     return val;
   }
 
@@ -1160,18 +1160,18 @@ struct SparseTensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
     Location loc = op.getLoc();
     SmallVector<Value> maxLvlCrds;
     for (Level l = 0; l < stt->getLvlRank(); l++) {
-      Value lvlSz = rewriter.create<LvlOp>(loc, op.getSource(), l);
-      Value maxLvlCrd = rewriter.create<arith::SubIOp>(
+      Value lvlSz = LvlOp::create(rewriter, loc, op.getSource(), l);
+      Value maxLvlCrd = arith::SubIOp::create(rewriter,
           loc, lvlSz, constantOne(rewriter, loc, rewriter.getIndexType()));
       maxLvlCrds.push_back(maxLvlCrd);
     }
 
     AffineExpr lvl2DimExp = stt->getLvlToDim().getResult(*dim);
-    Value maxDimCrd = rewriter.create<affine::AffineApplyOp>(
+    Value maxDimCrd = affine::AffineApplyOp::create(rewriter,
         op.getLoc(), AffineMap::get(stt->getLvlRank(), 0, lvl2DimExp),
         maxLvlCrds);
 
-    Value dimSz = rewriter.create<arith::AddIOp>(
+    Value dimSz = arith::AddIOp::create(rewriter,
         loc, maxDimCrd, constantOne(rewriter, loc, rewriter.getIndexType()));
     rewriter.replaceOp(op, dimSz);
     return success();
@@ -1212,26 +1212,26 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
     for (Value input : op.getInputs()) {
       // Builds a for op for each input tensor to append new values into the
       // output tensor.
-      foreachOp = rewriter.create<ForeachOp>(
+      foreachOp = ForeachOp::create(rewriter,
           loc, input, iterArg,
           [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
               ValueRange reduc) {
             SmallVector<Value> offDimCrd(dcvs);
             offDimCrd[conDim] =
-                builder.create<arith::AddIOp>(loc, offDimCrd[conDim], offset);
+                arith::AddIOp::create(builder, loc, offDimCrd[conDim], offset);
 
             // Enters foreach, updates the SSA chain.
             dstBuf.val = reduc.front();
             if (!dstTp.isAllDense()) {
               Value cond = genIsNonzero(builder, loc, v);
-              auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+              auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
                                                     /*else*/ true);
               builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-              builder.create<scf::YieldOp>(loc, dstBuf.val);
+              scf::YieldOp::create(builder, loc, dstBuf.val);
 
               builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
               dstBuf.insert(builder, loc, v, offDimCrd);
-              builder.create<scf::YieldOp>(loc, dstBuf.val);
+              scf::YieldOp::create(builder, loc, dstBuf.val);
 
               // Exits the ifOp, update the sparse tensor SSA value.
               builder.setInsertionPointAfter(ifOp);
@@ -1239,14 +1239,14 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
             } else {
               dstBuf.insert(builder, loc, v, offDimCrd);
             }
-            builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
+            sparse_tensor::YieldOp::create(builder, loc, dstBuf.val);
           });
       // Accumulates the offset. Note that only static-shaped inputs are allowed
       // by concatenate op verifier, which saves us from computing the offset
       // dynamically.
       const Size sz = getSparseTensorType(input).getDynamicDimSize(conDim);
       assert(ShapedType::isStatic(sz));
-      offset = rewriter.create<arith::AddIOp>(loc, offset,
+      offset = arith::AddIOp::create(rewriter, loc, offset,
                                               constantIndex(rewriter, loc, sz));
       iterArg = foreachOp.getResult(0);
       dstBuf.val = iterArg;
@@ -1299,7 +1299,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
     ValueRange vs;
     TensorLike dstBuf(rewriter, loc, dstStt.getRankedTensorType(), sizes);
 
-    auto foreachOp = rewriter.create<ForeachOp>(
+    auto foreachOp = ForeachOp::create(rewriter,
         loc, src, dstBuf.val, foreachOrder,
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
@@ -1307,14 +1307,14 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           dstBuf.val = reduc.front();
           if (!skipZeroCheck) {
             Value cond = genIsNonzero(builder, loc, v);
-            auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+            auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
                                                   /*else*/ true);
             builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-            builder.create<scf::YieldOp>(loc, dstBuf.val);
+            scf::YieldOp::create(builder, loc, dstBuf.val);
 
             builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
             dstBuf.insert(builder, loc, v, dcvs);
-            builder.create<scf::YieldOp>(loc, dstBuf.val);
+            scf::YieldOp::create(builder, loc, dstBuf.val);
 
             // Exits the ifOp, update the sparse tensor SSA value.
             builder.setInsertionPointAfter(ifOp);
@@ -1322,7 +1322,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           } else {
             dstBuf.insert(builder, loc, v, dcvs);
           }
-          builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
+          sparse_tensor::YieldOp::create(builder, loc, dstBuf.val);
         });
 
     rewriter.setInsertionPointAfter(foreachOp);
@@ -1349,7 +1349,7 @@ struct CrdTranslateRewriter : public OpRewritePattern<CrdTranslateOp> {
       // TODO: we should probably expand the affine map to IR using our own
       // rules, since affine.apply assume signed value, while the cooridinates
       // we provided must always be signless.
-      Value trans = rewriter.create<affine::AffineApplyOp>(
+      Value trans = affine::AffineApplyOp::create(rewriter,
           op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
           op.getInCrds());
       outCrds.push_back(trans);
@@ -1412,8 +1412,8 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
     SmallVector<Value> pos = loopEmitter.getValPosits(0);
     // Loads the value from sparse tensor using position-index;
     // loads the value from dense tensor using coords.
-    Value val = enc ? rewriter.create<memref::LoadOp>(loc, vals, pos)
-                    : rewriter.create<memref::LoadOp>(loc, vals, lcvs);
+    Value val = enc ? memref::LoadOp::create(rewriter, loc, vals, pos)
+                    : memref::LoadOp::create(rewriter, loc, vals, lcvs);
 
     // 2. Inline the block in the foreach operator.
     Block *srcBlock = op.getBody();
@@ -1472,22 +1472,22 @@ struct NewRewriter : public OpRewritePattern<NewOp> {
     // with enveloping reinterpreted_map ops for non-permutations.
     RankedTensorType dstTp = stt.getRankedTensorType();
     RankedTensorType cooTp = stt.getCOOType(/*ordered=*/true);
-    Value cooTensor = rewriter.create<NewOp>(loc, cooTp, op.getSource());
+    Value cooTensor = NewOp::create(rewriter, loc, cooTp, op.getSource());
     Value convert = cooTensor;
     auto enc = stt.getEncoding();
     if (!stt.isPermutation()) { // demap coo, demap dstTp
       auto coo = getSparseTensorType(cooTensor).getEncoding().withoutDimToLvl();
-      convert = rewriter.create<ReinterpretMapOp>(loc, coo, convert);
+      convert = ReinterpretMapOp::create(rewriter, loc, coo, convert);
       dstTp = getSparseTensorType(convert).withEncoding(enc.withoutDimToLvl());
     }
-    convert = rewriter.create<ConvertOp>(loc, dstTp, convert);
+    convert = ConvertOp::create(rewriter, loc, dstTp, convert);
     if (!stt.isPermutation()) // remap to original enc
-      convert = rewriter.create<ReinterpretMapOp>(loc, enc, convert);
+      convert = ReinterpretMapOp::create(rewriter, loc, enc, convert);
     rewriter.replaceOp(op, convert);
 
     // Release the temporary ordered COO tensor.
     rewriter.setInsertionPointAfterValue(convert);
-    rewriter.create<DeallocTensorOp>(loc, cooTensor);
+    DeallocTensorOp::create(rewriter, loc, cooTensor);
 
     return success();
   }
@@ -1501,7 +1501,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     Location loc = op.getLoc();
     // Calculate NNZ.
     Value src = op.getTensor();
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, src);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, src);
 
     // Allocate a temporary buffer for storing dimension-sizes/coordinates.
     const auto srcTp = getSparseTensorType(src);
@@ -1514,7 +1514,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     SmallVector<Value> dims;
     sizesForTensor(rewriter, dims, loc, srcTp, src);
     for (Dimension d = 0; d < dimRank; d++) {
-      rewriter.create<memref::StoreOp>(loc, dims[d], dimSizes,
+      memref::StoreOp::create(rewriter, loc, dims[d], dimSizes,
                                        constantIndex(rewriter, loc, d));
     }
 
@@ -1536,20 +1536,20 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     ModuleOp module = op->getParentOfType<ModuleOp>();
 
     // For each element in the source tensor, output the element.
-    rewriter.create<ForeachOp>(
+    ForeachOp::create(rewriter,
         loc, src, ValueRange(),
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
           for (Dimension d = 0; d < dimRank; d++) {
-            rewriter.create<memref::StoreOp>(loc, dcvs[d], dimCoords,
+            memref::StoreOp::create(rewriter, loc, dcvs[d], dimCoords,
                                              constantIndex(builder, loc, d));
           }
-          rewriter.create<memref::StoreOp>(loc, v, value);
+          memref::StoreOp::create(rewriter, loc, v, value);
           SmallVector<Value> operands{writer, rankValue, dimCoords, value};
           FlatSymbolRefAttr fn = getFunc(module, outNextFuncName, {}, operands,
                                          EmitCInterface::On);
-          builder.create<func::CallOp>(loc, TypeRange(), fn, operands);
-          builder.create<sparse_tensor::YieldOp>(loc);
+          func::CallOp::create(builder, loc, TypeRange(), fn, operands);
+          sparse_tensor::YieldOp::create(builder, loc);
         });
 
     // Release the writer.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 359590f2434dc..5e1f872d5b185 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -79,7 +79,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
       matchPattern(step, m_Constant(&stepInt))) {
     if (((hiInt.getInt() - loInt.getInt()) % stepInt.getInt()) == 0) {
       Value trueVal = constantI1(rewriter, loc, true);
-      return rewriter.create<vector::BroadcastOp>(loc, mtp, trueVal);
+      return vector::BroadcastOp::create(rewriter, loc, mtp, trueVal);
     }
   }
   // Otherwise, generate a vector mask that avoids overrunning the upperbound
@@ -93,7 +93,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
       rewriter.getContext());
   Value end = rewriter.createOrFold<affine::AffineMinOp>(
       loc, min, ValueRange{hi, iv, step});
-  return rewriter.create<vector::CreateMaskOp>(loc, mtp, end);
+  return vector::CreateMaskOp::create(rewriter, loc, mtp, end);
 }
 
 /// Generates a vectorized invariant. Here we rely on subsequent loop
@@ -101,7 +101,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
 static Value genVectorInvariantValue(PatternRewriter &rewriter, VL vl,
                                      Value val) {
   VectorType vtp = vectorType(vl, val.getType());
-  return rewriter.create<vector::BroadcastOp>(val.getLoc(), vtp, val);
+  return vector::BroadcastOp::create(rewriter, val.getLoc(), vtp, val);
 }
 
 /// Generates a vectorized load lhs = a[ind[lo:hi]] or lhs = a[lo:hi],
@@ -116,10 +116,10 @@ static Value genVectorLoad(PatternRewriter &rewriter, Location loc, VL vl,
     SmallVector<Value> scalarArgs(idxs);
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
-    return rewriter.create<vector::GatherOp>(loc, vtp, mem, scalarArgs,
+    return vector::GatherOp::create(rewriter, loc, vtp, mem, scalarArgs,
                                              indexVec, vmask, pass);
   }
-  return rewriter.create<vector::MaskedLoadOp>(loc, vtp, mem, idxs, vmask,
+  return vector::MaskedLoadOp::create(rewriter, loc, vtp, mem, idxs, vmask,
                                                pass);
 }
 
@@ -133,11 +133,11 @@ static void genVectorStore(PatternRewriter &rewriter, Location loc, Value mem,
     SmallVector<Value> scalarArgs(idxs);
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
-    rewriter.create<vector::ScatterOp>(loc, mem, scalarArgs, indexVec, vmask,
+    vector::ScatterOp::create(rewriter, loc, mem, scalarArgs, indexVec, vmask,
                                        rhs);
     return;
   }
-  rewriter.create<vector::MaskedStoreOp>(loc, mem, idxs, vmask, rhs);
+  vector::MaskedStoreOp::create(rewriter, loc, mem, idxs, vmask, rhs);
 }
 
 /// Detects a vectorizable reduction operations and returns the
@@ -198,18 +198,18 @@ static Value genVectorReducInit(PatternRewriter &rewriter, Location loc,
   case vector::CombiningKind::ADD:
   case vector::CombiningKind::XOR:
     // Initialize reduction vector to: | 0 | .. | 0 | r |
-    return rewriter.create<vector::InsertOp>(loc, r,
+    return vector::InsertOp::create(rewriter, loc, r,
                                              constantZero(rewriter, loc, vtp),
                                              constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::MUL:
     // Initialize reduction vector to: | 1 | .. | 1 | r |
-    return rewriter.create<vector::InsertOp>(loc, r,
+    return vector::InsertOp::create(rewriter, loc, r,
                                              constantOne(rewriter, loc, vtp),
                                              constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::AND:
   case vector::CombiningKind::OR:
     // Initialize reduction vector to: | r | .. | r | r |
-    return rewriter.create<vector::BroadcastOp>(loc, vtp, r);
+    return vector::BroadcastOp::create(rewriter, loc, vtp, r);
   default:
     break;
   }
@@ -301,10 +301,10 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
         Type etp = llvm::cast<VectorType>(vload.getType()).getElementType();
         if (!llvm::isa<IndexType>(etp)) {
           if (etp.getIntOrFloatBitWidth() < 32)
-            vload = rewriter.create<arith::ExtUIOp>(
+            vload = arith::ExtUIOp::create(rewriter,
                 loc, vectorType(vl, rewriter.getI32Type()), vload);
           else if (etp.getIntOrFloatBitWidth() < 64 && !vl.enableSIMDIndex32)
-            vload = rewriter.create<arith::ExtUIOp>(
+            vload = arith::ExtUIOp::create(rewriter,
                 loc, vectorType(vl, rewriter.getI64Type()), vload);
         }
         idxs.push_back(vload);
@@ -329,7 +329,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
             return false;
           if (codegen)
             idxs.push_back(
-                rewriter.create<arith::AddIOp>(forOp.getLoc(), inv, idx));
+                arith::AddIOp::create(rewriter, forOp.getLoc(), inv, idx));
           continue; // success so far
         }
       }
@@ -342,7 +342,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define UNAOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = rewriter.create<xxx>(loc, vx);                                    \
+      vexp = xxx::create(rewriter, loc, vx);                                    \
     return true;                                                               \
   }
 
@@ -350,7 +350,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
   if (auto x = dyn_cast<xxx>(def)) {                                           \
     if (codegen) {                                                             \
       VectorType vtp = vectorType(vl, x.getType());                            \
-      vexp = rewriter.create<xxx>(loc, vtp, vx);                               \
+      vexp = xxx::create(rewriter, loc, vtp, vx);                               \
     }                                                                          \
     return true;                                                               \
   }
@@ -358,7 +358,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define BINOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = rewriter.create<xxx>(loc, vx, vy);                                \
+      vexp = xxx::create(rewriter, loc, vx, vy);                                \
     return true;                                                               \
   }
 
@@ -381,9 +381,9 @@ static bool vectorizeExpr(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       // such as a[i] = i, which must convert to [i, i+1, ...].
       if (codegen) {
         VectorType vtp = vectorType(vl, arg.getType());
-        Value veci = rewriter.create<vector::BroadcastOp>(loc, vtp, arg);
-        Value incr = rewriter.create<vector::StepOp>(loc, vtp);
-        vexp = rewriter.create<arith::AddIOp>(loc, veci, incr);
+        Value veci = vector::BroadcastOp::create(rewriter, loc, vtp, arg);
+        Value incr = vector::StepOp::create(rewriter, loc, vtp);
+        vexp = arith::AddIOp::create(rewriter, loc, veci, incr);
       }
       return true;
     }
@@ -526,15 +526,15 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
     Value step = constantIndex(rewriter, loc, vl.vectorLength);
     if (vl.enableVLAVectorization) {
       Value vscale =
-          rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-      step = rewriter.create<arith::MulIOp>(loc, vscale, step);
+          vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+      step = arith::MulIOp::create(rewriter, loc, vscale, step);
     }
     if (!yield.getResults().empty()) {
       Value init = forOp.getInitArgs()[0];
       VectorType vtp = vectorType(vl, init.getType());
       Value vinit = genVectorReducInit(rewriter, loc, yield->getOperand(0),
                                        forOp.getRegionIterArg(0), init, vtp);
-      forOpNew = rewriter.create<scf::ForOp>(
+      forOpNew = scf::ForOp::create(rewriter,
           loc, forOp.getLowerBound(), forOp.getUpperBound(), step, vinit);
       forOpNew->setAttr(
           LoopEmitter::getLoopEmitterLoopAttrName(),
@@ -563,10 +563,10 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       if (codegen) {
         Value partial = forOpNew.getResult(0);
         Value vpass = genVectorInvariantValue(rewriter, vl, iter);
-        Value vred = rewriter.create<arith::SelectOp>(loc, vmask, vrhs, vpass);
-        rewriter.create<scf::YieldOp>(loc, vred);
+        Value vred = arith::SelectOp::create(rewriter, loc, vmask, vrhs, vpass);
+        scf::YieldOp::create(rewriter, loc, vred);
         rewriter.setInsertionPointAfter(forOpNew);
-        Value vres = rewriter.create<vector::ReductionOp>(loc, kind, partial);
+        Value vres = vector::ReductionOp::create(rewriter, loc, kind, partial);
         // Now do some relinking (last one is not completely type safe
         // but all bad ones are removed right away). This also folds away
         // nop broadcast operations.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
index 2d604ed7a8ffc..0cabe5498d7f4 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
@@ -322,7 +322,7 @@ static void genBuffers(CodegenEnv &env, OpBuilder &builder) {
         if (!isInit) {
           Value zero = constantZero(builder, loc,
                                     getElementTypeOrSelf(tensor.getType()));
-          builder.create<linalg::FillOp>(loc, ValueRange{zero},
+          linalg::FillOp::create(builder, loc, ValueRange{zero},
                                          ValueRange{init});
         }
         return init;
@@ -385,7 +385,7 @@ static Value genInsertionLoad(CodegenEnv &env, OpBuilder &builder,
   }
   // Load from expanded access pattern.
   Value index = genIndex(env, t);
-  return builder.create<memref::LoadOp>(loc, env.getExpandValues(), index);
+  return memref::LoadOp::create(builder, loc, env.getExpandValues(), index);
 }
 
 /// Generates insertion code to implement dynamic tensor load for reduction.
@@ -401,22 +401,22 @@ static Value genInsertionLoadReduce(CodegenEnv &env, OpBuilder &builder,
   Value values = env.getExpandValues();
   Value filled = env.getExpandFilled();
   Value index = genIndex(env, t);
-  Value isFilled = builder.create<memref::LoadOp>(loc, filled, index);
-  Value valAtIndex = builder.create<memref::LoadOp>(loc, values, index);
-  return builder.create<arith::SelectOp>(loc, isFilled, valAtIndex, identity);
+  Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
+  Value valAtIndex = memref::LoadOp::create(builder, loc, values, index);
+  return arith::SelectOp::create(builder, loc, isFilled, valAtIndex, identity);
 }
 
 static Value genConditionalInsert(Location loc, OpBuilder &builder, Value cond,
                                   Value sparseOut, ValueRange ivs, Value v) {
   scf::IfOp condInsert =
-      builder.create<scf::IfOp>(loc, sparseOut.getType(), cond, true);
+      scf::IfOp::create(builder, loc, sparseOut.getType(), cond, true);
   // True branch.
   builder.setInsertionPointToStart(condInsert.thenBlock());
-  Value res = builder.create<tensor::InsertOp>(loc, v, sparseOut, ivs);
-  builder.create<scf::YieldOp>(loc, res);
+  Value res = tensor::InsertOp::create(builder, loc, v, sparseOut, ivs);
+  scf::YieldOp::create(builder, loc, res);
   // False branch.
   builder.setInsertionPointToStart(condInsert.elseBlock());
-  builder.create<scf::YieldOp>(loc, sparseOut);
+  scf::YieldOp::create(builder, loc, sparseOut);
   // Value assignment.
   builder.setInsertionPointAfter(condInsert);
   return condInsert.getResult(0);
@@ -453,7 +453,7 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
         Value nz = genIsNonzero(builder, loc, rhs);
         sparseOut = genConditionalInsert(loc, builder, nz, chain, ivs, rhs);
       } else {
-        sparseOut = builder.create<tensor::InsertOp>(loc, rhs, chain, ivs);
+        sparseOut = tensor::InsertOp::create(builder, loc, rhs, chain, ivs);
       }
       // Generates regular insertion chain.
       env.updateInsertionChain(sparseOut);
@@ -474,25 +474,25 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
   Value fval = constantI1(builder, loc, false);
   Value tval = constantI1(builder, loc, true);
   // If statement.
-  Value isFilled = builder.create<memref::LoadOp>(loc, filled, index);
-  Value cond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+  Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
+  Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
                                              isFilled, fval);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIndexType(), cond,
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIndexType(), cond,
                                              /*else=*/true);
   // True branch.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  builder.create<memref::StoreOp>(loc, tval, filled, index);
-  builder.create<memref::StoreOp>(loc, index, added, count);
+  memref::StoreOp::create(builder, loc, tval, filled, index);
+  memref::StoreOp::create(builder, loc, index, added, count);
   Value one = constantIndex(builder, loc, 1);
-  Value add = builder.create<arith::AddIOp>(loc, count, one);
-  builder.create<scf::YieldOp>(loc, add);
+  Value add = arith::AddIOp::create(builder, loc, count, one);
+  scf::YieldOp::create(builder, loc, add);
   // False branch.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, count);
+  scf::YieldOp::create(builder, loc, count);
   builder.setInsertionPointAfter(ifOp);
   // Value assignment.
   env.updateExpandCount(ifOp.getResult(0));
-  builder.create<memref::StoreOp>(loc, rhs, values, index);
+  memref::StoreOp::create(builder, loc, rhs, values, index);
 }
 
 /// Generates a load on a dense or sparse tensor.
@@ -522,9 +522,9 @@ static Value genTensorLoad(CodegenEnv &env, OpBuilder &builder, ExprId exp) {
   if (llvm::isa<TensorType>(ptr.getType())) {
     assert(env.options().sparseEmitStrategy ==
            SparseEmitStrategy::kSparseIterator);
-    return builder.create<ExtractValOp>(loc, ptr, llvm::getSingleElement(args));
+    return ExtractValOp::create(builder, loc, ptr, llvm::getSingleElement(args));
   }
-  return builder.create<memref::LoadOp>(loc, ptr, args);
+  return memref::LoadOp::create(builder, loc, ptr, args);
 }
 
 /// Generates a store on a dense or sparse tensor.
@@ -551,7 +551,7 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   if (!env.isSparseOutput(t)) {
     SmallVector<Value> args;
     Value ptr = genSubscript(env, builder, t, args);
-    builder.create<memref::StoreOp>(loc, rhs, ptr, args);
+    memref::StoreOp::create(builder, loc, rhs, ptr, args);
     return;
   }
   // Store during sparse insertion.
@@ -562,7 +562,7 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   // Select operation insertion.
   Value chain = env.getInsertionChain();
   scf::IfOp ifOp =
-      builder.create<scf::IfOp>(loc, chain.getType(), rhs, /*else=*/true);
+      scf::IfOp::create(builder, loc, chain.getType(), rhs, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   // Existing value was preserved to be used here.
   assert(env.exp(exp).val);
@@ -571,10 +571,10 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   env.merger().clearExprValue(exp);
   // Yield modified insertion chain along true branch.
   Value mchain = env.getInsertionChain();
-  builder.create<scf::YieldOp>(op.getLoc(), mchain);
+  scf::YieldOp::create(builder, op.getLoc(), mchain);
   // Yield original insertion chain along false branch.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, chain);
+  scf::YieldOp::create(builder, loc, chain);
   // Done with if statement.
   env.updateInsertionChain(ifOp->getResult(0));
   builder.setInsertionPointAfter(ifOp);
@@ -603,7 +603,7 @@ static Value relinkBranch(CodegenEnv &env, RewriterBase &rewriter, Block *block,
       assert(!getSparseTensorType(t->get()).hasEncoding()); // dense!
       SmallVector<Value> args;
       Value ptr = genSubscript(env, rewriter, t, args);
-      return rewriter.create<memref::LoadOp>(op.getLoc(), ptr, args);
+      return memref::LoadOp::create(rewriter, op.getLoc(), ptr, args);
     }
   } else if (Operation *def = e.getDefiningOp()) {
     // Handle index computation.
@@ -774,7 +774,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Type t2 = MemRefType::get(dynShape, builder.getI1Type());
     Type t3 = MemRefType::get(dynShape, builder.getIndexType());
     Type t4 = builder.getIndexType();
-    auto r = builder.create<ExpandOp>(loc, TypeRange({t1, t2, t3, t4}), tensor);
+    auto r = ExpandOp::create(builder, loc, TypeRange({t1, t2, t3, t4}), tensor);
     assert(r.getNumResults() == 4);
     env.startExpand(r.getResult(0), r.getResult(1), r.getResult(2),
                     r.getResult(3));
@@ -787,7 +787,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Value added = env.getExpandAdded();
     Value count = env.getExpandCount();
     Value chain = env.getInsertionChain();
-    Value compress = builder.create<CompressOp>(loc, values, filled, added,
+    Value compress = CompressOp::create(builder, loc, values, filled, added,
                                                 count, chain, indices);
     env.updateInsertionChain(compress);
     env.endExpand();
@@ -895,7 +895,7 @@ static void finalizeWhileOp(CodegenEnv &env, OpBuilder &builder,
         env.updateInsertionChain(ifOp->getResult(y++));
       }
       assert(y == yields.size());
-      builder.create<scf::YieldOp>(loc, yields);
+      scf::YieldOp::create(builder, loc, yields);
       builder.setInsertionPointAfter(ifOp);
     }
   }
@@ -948,13 +948,13 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
           assert(lvl.has_value());
           const Value crd = env.emitter().getCoord(tid, *lvl);
           const Value lvar = env.getLoopVar(curr);
-          clause = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+          clause = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
                                                  crd, lvar);
         } else {
           assert(lt.hasDenseSemantic() || isUndefLT(lt));
           clause = constantI1(builder, loc, true);
         }
-        cond = cond ? builder.create<arith::AndIOp>(loc, cond, clause) : clause;
+        cond = cond ? arith::AndIOp::create(builder, loc, cond, clause) : clause;
       });
   if (env.isReduc()) {
     types.push_back(env.getReduc().getType());
@@ -965,7 +965,7 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     types.push_back(builder.getIndexType());
   if (env.getInsertionChain())
     types.push_back(env.getInsertionChain().getType());
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   return ifOp;
 }
@@ -993,7 +993,7 @@ static void endIf(CodegenEnv &env, OpBuilder &builder, scf::IfOp ifOp,
     env.updateInsertionChain(insInput);
   }
   if (!operands.empty())
-    builder.create<scf::YieldOp>(env.op().getLoc(), operands);
+    scf::YieldOp::create(builder, env.op().getLoc(), operands);
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
 }
 
@@ -1307,7 +1307,7 @@ static void genStmt(CodegenEnv &env, RewriterBase &rewriter, ExprId exp,
           genStmt(env, rewriter, ej, curr + 1);
           // TODO: handle yield values.
           assert(reduc.empty() && "Not Implemented");
-          rewriter.create<sparse_tensor::YieldOp>(env.op().getLoc());
+          sparse_tensor::YieldOp::create(rewriter, env.op().getLoc());
           return std::nullopt;
         });
         // endIf(env, rewriter, ifOp, redInput, cntInput, insInput, validIns);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
index c370d104e0985..79c8d6c171249 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
@@ -42,7 +42,7 @@ struct GuardSparseAlloc
     // operation that leaves the underlying storage in a proper state
     // before the tensor escapes across the method boundary.
     rewriter.setInsertionPointAfter(op);
-    auto load = rewriter.create<LoadOp>(op.getLoc(), op.getResult(), true);
+    auto load = LoadOp::create(rewriter, op.getLoc(), op.getResult(), true);
     rewriter.replaceAllUsesExcept(op, load, load);
     return success();
   }
@@ -61,7 +61,7 @@ struct StageUnorderedSparseOps : public OpRewritePattern<StageWithSortOp> {
     // Deallocate tmpBuf.
     // TODO: Delegate to buffer deallocation pass in the future.
     if (succeeded(stageResult) && tmpBuf)
-      rewriter.create<bufferization::DeallocTensorOp>(loc, tmpBuf);
+      bufferization::DeallocTensorOp::create(rewriter, loc, tmpBuf);
 
     return stageResult;
   }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
index 1bd9563b3db07..7a735474b6d57 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
@@ -156,7 +156,7 @@ Value sparse_tensor::genCast(OpBuilder &builder, Location loc, Value value,
 
   // int <=> index
   if (isa<IndexType>(srcTp) || isa<IndexType>(dstTp))
-    return builder.create<arith::IndexCastOp>(loc, dstTp, value);
+    return arith::IndexCastOp::create(builder, loc, dstTp, value);
 
   const auto srcIntTp = dyn_cast_or_null<IntegerType>(srcTp);
   const bool isUnsignedCast = srcIntTp ? srcIntTp.isUnsigned() : false;
@@ -169,19 +169,19 @@ Value sparse_tensor::genScalarToTensor(OpBuilder &builder, Location loc,
     // Scalars can only be converted to 0-ranked tensors.
     assert(rtp.getRank() == 0);
     elem = sparse_tensor::genCast(builder, loc, elem, rtp.getElementType());
-    return builder.create<tensor::FromElementsOp>(loc, rtp, elem);
+    return tensor::FromElementsOp::create(builder, loc, rtp, elem);
   }
   return sparse_tensor::genCast(builder, loc, elem, dstTp);
 }
 
 Value sparse_tensor::genIndexLoad(OpBuilder &builder, Location loc, Value mem,
                                   ValueRange s) {
-  Value load = builder.create<memref::LoadOp>(loc, mem, s);
+  Value load = memref::LoadOp::create(builder, loc, mem, s);
   if (!isa<IndexType>(load.getType())) {
     if (load.getType().getIntOrFloatBitWidth() < 64)
-      load = builder.create<arith::ExtUIOp>(loc, builder.getI64Type(), load);
+      load = arith::ExtUIOp::create(builder, loc, builder.getI64Type(), load);
     load =
-        builder.create<arith::IndexCastOp>(loc, builder.getIndexType(), load);
+        arith::IndexCastOp::create(builder, loc, builder.getIndexType(), load);
   }
   return load;
 }
@@ -206,13 +206,13 @@ Value mlir::sparse_tensor::genIsNonzero(OpBuilder &builder, mlir::Location loc,
   Type tp = v.getType();
   Value zero = constantZero(builder, loc, tp);
   if (isa<FloatType>(tp))
-    return builder.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNE, v,
+    return arith::CmpFOp::create(builder, loc, arith::CmpFPredicate::UNE, v,
                                          zero);
   if (tp.isIntOrIndex())
-    return builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, v,
+    return arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, v,
                                          zero);
   if (isa<ComplexType>(tp))
-    return builder.create<complex::NotEqualOp>(loc, v, zero);
+    return complex::NotEqualOp::create(builder, loc, v, zero);
   llvm_unreachable("Non-numeric type");
 }
 
@@ -226,7 +226,7 @@ void mlir::sparse_tensor::genReshapeDstShape(
     for (const auto &map : llvm::enumerate(reassociation)) {
       auto dstDim = constantIndex(builder, loc, 1);
       for (unsigned i = start; i < start + map.value().size(); i++) {
-        dstDim = builder.create<arith::MulIOp>(loc, dstDim, srcShape[i]);
+        dstDim = arith::MulIOp::create(builder, loc, dstDim, srcShape[i]);
       }
       dstShape.push_back(dstDim);
       start = start + map.value().size();
@@ -260,7 +260,7 @@ void mlir::sparse_tensor::genReshapeDstShape(
         // Compute the dynamic dimension size.
         Value productVal = constantIndex(builder, loc, product);
         Value dynamicSize =
-            builder.create<arith::DivUIOp>(loc, srcDim, productVal);
+            arith::DivUIOp::create(builder, loc, srcDim, productVal);
         dstShape.push_back(dynamicSize);
       } else {
         // The expanded dimension is statically known.
@@ -289,7 +289,7 @@ void mlir::sparse_tensor::reshapeCvs(
     // Prepare strides information in dimension slice.
     Value linear = constantIndex(builder, loc, 1);
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
-      linear = builder.create<arith::MulIOp>(loc, linear, sizes[j]);
+      linear = arith::MulIOp::create(builder, loc, linear, sizes[j]);
     }
     // Start expansion.
     Value val;
@@ -297,16 +297,16 @@ void mlir::sparse_tensor::reshapeCvs(
       val = srcCvs[i];
     // Iterate over dimension slice.
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
-      linear = builder.create<arith::DivUIOp>(loc, linear, sizes[j]);
+      linear = arith::DivUIOp::create(builder, loc, linear, sizes[j]);
       if (isCollapse) {
-        const Value mul = builder.create<arith::MulIOp>(loc, srcCvs[j], linear);
-        val = val ? builder.create<arith::AddIOp>(loc, val, mul) : mul;
+        const Value mul = arith::MulIOp::create(builder, loc, srcCvs[j], linear);
+        val = val ? arith::AddIOp::create(builder, loc, val, mul) : mul;
       } else {
         const Value old = val;
-        val = builder.create<arith::DivUIOp>(loc, val, linear);
+        val = arith::DivUIOp::create(builder, loc, val, linear);
         assert(dstCvs.size() == j);
         dstCvs.push_back(val);
-        val = builder.create<arith::RemUIOp>(loc, old, linear);
+        val = arith::RemUIOp::create(builder, loc, old, linear);
       }
     }
     // Finalize collapse.
@@ -329,7 +329,7 @@ FlatSymbolRefAttr mlir::sparse_tensor::getFunc(ModuleOp module, StringRef name,
   auto func = module.lookupSymbol<func::FuncOp>(result.getAttr());
   if (!func) {
     OpBuilder moduleBuilder(module.getBodyRegion());
-    func = moduleBuilder.create<func::FuncOp>(
+    func = func::FuncOp::create(moduleBuilder,
         module.getLoc(), name,
         FunctionType::get(context, operands.getTypes(), resultType));
     func.setPrivate();
@@ -346,7 +346,7 @@ func::CallOp mlir::sparse_tensor::createFuncCall(
   auto module = builder.getBlock()->getParentOp()->getParentOfType<ModuleOp>();
   FlatSymbolRefAttr fn =
       getFunc(module, name, resultType, operands, emitCInterface);
-  return builder.create<func::CallOp>(loc, resultType, fn, operands);
+  return func::CallOp::create(builder, loc, resultType, fn, operands);
 }
 
 Type mlir::sparse_tensor::getOpaquePointerType(MLIRContext *ctx) {
@@ -361,7 +361,7 @@ Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
                                      unsigned sz, Type tp, bool staticShape) {
   if (staticShape) {
     auto memTp = MemRefType::get({sz}, tp);
-    return builder.create<memref::AllocaOp>(loc, memTp);
+    return memref::AllocaOp::create(builder, loc, memTp);
   }
   return genAlloca(builder, loc, constantIndex(builder, loc, sz), tp);
 }
@@ -369,12 +369,12 @@ Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
 Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc, Value sz,
                                      Type tp) {
   auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
-  return builder.create<memref::AllocaOp>(loc, memTp, ValueRange{sz});
+  return memref::AllocaOp::create(builder, loc, memTp, ValueRange{sz});
 }
 
 Value mlir::sparse_tensor::genAllocaScalar(OpBuilder &builder, Location loc,
                                            Type tp) {
-  return builder.create<memref::AllocaOp>(loc, MemRefType::get({}, tp));
+  return memref::AllocaOp::create(builder, loc, MemRefType::get({}, tp));
 }
 
 Value mlir::sparse_tensor::allocaBuffer(OpBuilder &builder, Location loc,
@@ -384,7 +384,7 @@ Value mlir::sparse_tensor::allocaBuffer(OpBuilder &builder, Location loc,
   Value buffer = genAlloca(builder, loc, sz, values[0].getType());
   for (unsigned i = 0; i < sz; i++) {
     Value idx = constantIndex(builder, loc, i);
-    builder.create<memref::StoreOp>(loc, values[i], buffer, idx);
+    memref::StoreOp::create(builder, loc, values[i], buffer, idx);
   }
   return buffer;
 }
@@ -400,15 +400,15 @@ Value mlir::sparse_tensor::allocDenseTensor(OpBuilder &builder, Location loc,
     if (shape[i] == ShapedType::kDynamic)
       dynamicSizes.push_back(sizes[i]);
   }
-  Value mem = builder.create<memref::AllocOp>(loc, memTp, dynamicSizes);
+  Value mem = memref::AllocOp::create(builder, loc, memTp, dynamicSizes);
   Value zero = constantZero(builder, loc, elemTp);
-  builder.create<linalg::FillOp>(loc, ValueRange{zero}, ValueRange{mem});
+  linalg::FillOp::create(builder, loc, ValueRange{zero}, ValueRange{mem});
   return mem;
 }
 
 void mlir::sparse_tensor::deallocDenseTensor(OpBuilder &builder, Location loc,
                                              Value buffer) {
-  builder.create<memref::DeallocOp>(loc, buffer);
+  memref::DeallocOp::create(builder, loc, buffer);
 }
 
 void mlir::sparse_tensor::sizesFromSrc(OpBuilder &builder,
@@ -486,17 +486,17 @@ void sparse_tensor::foreachInSparseConstant(
     cvs.clear();
     for (Dimension d = 0; d < dimRank; d++) {
       auto crd = elems[i].first[d].getInt();
-      cvs.push_back(builder.create<arith::ConstantIndexOp>(loc, crd));
+      cvs.push_back(arith::ConstantIndexOp::create(builder, loc, crd));
     }
     // Remap value.
     Value val;
     if (isa<ComplexType>(attr.getElementType())) {
       auto valAttr = cast<ArrayAttr>(elems[i].second);
-      val = builder.create<complex::ConstantOp>(loc, attr.getElementType(),
+      val = complex::ConstantOp::create(builder, loc, attr.getElementType(),
                                                 valAttr);
     } else {
       auto valAttr = cast<TypedAttr>(elems[i].second);
-      val = builder.create<arith::ConstantOp>(loc, valAttr);
+      val = arith::ConstantOp::create(builder, loc, valAttr);
     }
     assert(val);
     callback(cvs, val);
@@ -516,10 +516,10 @@ SmallVector<Value> sparse_tensor::loadAll(OpBuilder &builder, Location loc,
   SmallVector<Value> vs;
   vs.reserve(size);
   for (unsigned i = 0; i < size; i++) {
-    Value v = builder.create<memref::LoadOp>(loc, mem,
+    Value v = memref::LoadOp::create(builder, loc, mem,
                                              constantIndex(builder, loc, i));
     if (i == offsetIdx && offsetVal)
-      v = builder.create<arith::AddIOp>(loc, v, offsetVal);
+      v = arith::AddIOp::create(builder, loc, v, offsetVal);
     vs.push_back(v);
   }
   return vs;
@@ -538,9 +538,9 @@ void sparse_tensor::storeAll(OpBuilder &builder, Location loc, Value mem,
   for (const auto &v : llvm::enumerate(vs)) {
     const Value w =
         (offsetIdx == v.index() && offsetVal)
-            ? builder.create<arith::AddIOp>(loc, v.value(), offsetVal)
+            ? arith::AddIOp::create(builder, loc, v.value(), offsetVal)
             : v.value();
-    builder.create<memref::StoreOp>(loc, w, mem,
+    memref::StoreOp::create(builder, loc, w, mem,
                                     constantIndex(builder, loc, v.index()));
   }
 }
@@ -550,7 +550,7 @@ sparse_tensor::genToMemref(OpBuilder &builder, Location loc, Value tensor) {
   auto tTp = llvm::cast<TensorType>(tensor.getType());
   auto mTp = MemRefType::get(tTp.getShape(), tTp.getElementType());
   return cast<TypedValue<BaseMemRefType>>(
-      builder.create<bufferization::ToBufferOp>(loc, mTp, tensor).getResult());
+      bufferization::ToBufferOp::create(builder, loc, mTp, tensor).getResult());
 }
 
 Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
@@ -560,7 +560,7 @@ Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
   std::optional<unsigned> offset = enc.getStaticDimSliceOffset(dim);
   if (offset.has_value())
     return constantIndex(builder, loc, *offset);
-  return builder.create<ToSliceOffsetOp>(loc, tensor, APInt(64, dim));
+  return ToSliceOffsetOp::create(builder, loc, tensor, APInt(64, dim));
 }
 
 Value sparse_tensor::createOrFoldSliceStrideOp(OpBuilder &builder, Location loc,
@@ -570,7 +570,7 @@ Value sparse_tensor::createOrFoldSliceStrideOp(OpBuilder &builder, Location loc,
   std::optional<unsigned> stride = enc.getStaticDimSliceStride(dim);
   if (stride.has_value())
     return constantIndex(builder, loc, *stride);
-  return builder.create<ToSliceStrideOp>(loc, tensor, APInt(64, dim));
+  return ToSliceStrideOp::create(builder, loc, tensor, APInt(64, dim));
 }
 
 Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
@@ -612,7 +612,7 @@ Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
     // subsequent clients need the values (DCE will remove unused).
     for (Dimension d = 0; d < dimRank; d++) {
       if (stt.isDynamicDim(d))
-        dimSizesValues[d] = builder.create<memref::LoadOp>(
+        dimSizesValues[d] = memref::LoadOp::create(builder,
             loc, dimSizesBuffer, constantIndex(builder, loc, d));
     }
   }
@@ -689,7 +689,7 @@ Value sparse_tensor::genMapBuffers(
     if (cm == 0) {
       lvlSz = dimSizesValues[d];
       if (cf != 0)
-        lvlSz = builder.create<arith::DivUIOp>(loc, lvlSz,
+        lvlSz = arith::DivUIOp::create(builder, loc, lvlSz,
                                                constantIndex(builder, loc, cf));
     } else {
       lvlSz = constantIndex(builder, loc, cm);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
index dc017e6baa6dc..400be54aeb86f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
@@ -97,7 +97,7 @@ class FuncCallOrInlineGenerator {
       // Create the function if not already exist.
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(getParentOpOf<func::FuncOp>(builder));
-      func = builder.create<func::FuncOp>(
+      func = func::FuncOp::create(builder,
           loc, funcName,
           FunctionType::get(context, params.getTypes(), retTypes));
       func.setPrivate();
@@ -108,10 +108,10 @@ class FuncCallOrInlineGenerator {
       // Delegates to user to generate the actually implementation.
       SmallVector<Value> result =
           genImplementation(retTypes, args, builder, loc);
-      builder.create<func::ReturnOp>(loc, result);
+      func::ReturnOp::create(builder, loc, result);
     }
     // Returns the CallOp result.
-    func::CallOp call = builder.create<func::CallOp>(loc, func, params);
+    func::CallOp call = func::CallOp::create(builder, loc, func, params);
     return call.getResults();
   }
 
@@ -310,9 +310,9 @@ inline Value constantZero(OpBuilder &builder, Location loc, Type tp) {
   if (auto ctp = dyn_cast<ComplexType>(tp)) {
     auto zeroe = builder.getZeroAttr(ctp.getElementType());
     auto zeroa = builder.getArrayAttr({zeroe, zeroe});
-    return builder.create<complex::ConstantOp>(loc, tp, zeroa);
+    return complex::ConstantOp::create(builder, loc, tp, zeroa);
   }
-  return builder.create<arith::ConstantOp>(loc, tp, builder.getZeroAttr(tp));
+  return arith::ConstantOp::create(builder, loc, tp, builder.getZeroAttr(tp));
 }
 
 /// Generates a 1-valued constant of the given type.  This supports all
@@ -322,39 +322,39 @@ inline Value constantOne(OpBuilder &builder, Location loc, Type tp) {
     auto zeroe = builder.getZeroAttr(ctp.getElementType());
     auto onee = getOneAttr(builder, ctp.getElementType());
     auto zeroa = builder.getArrayAttr({onee, zeroe});
-    return builder.create<complex::ConstantOp>(loc, tp, zeroa);
+    return complex::ConstantOp::create(builder, loc, tp, zeroa);
   }
-  return builder.create<arith::ConstantOp>(loc, tp, getOneAttr(builder, tp));
+  return arith::ConstantOp::create(builder, loc, tp, getOneAttr(builder, tp));
 }
 
 /// Generates a constant of `index` type.
 inline Value constantIndex(OpBuilder &builder, Location loc, int64_t i) {
-  return builder.create<arith::ConstantIndexOp>(loc, i);
+  return arith::ConstantIndexOp::create(builder, loc, i);
 }
 
 /// Generates a constant of `i64` type.
 inline Value constantI64(OpBuilder &builder, Location loc, int64_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 64);
+  return arith::ConstantIntOp::create(builder, loc, i, 64);
 }
 
 /// Generates a constant of `i32` type.
 inline Value constantI32(OpBuilder &builder, Location loc, int32_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 32);
+  return arith::ConstantIntOp::create(builder, loc, i, 32);
 }
 
 /// Generates a constant of `i16` type.
 inline Value constantI16(OpBuilder &builder, Location loc, int16_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 16);
+  return arith::ConstantIntOp::create(builder, loc, i, 16);
 }
 
 /// Generates a constant of `i8` type.
 inline Value constantI8(OpBuilder &builder, Location loc, int8_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 8);
+  return arith::ConstantIntOp::create(builder, loc, i, 8);
 }
 
 /// Generates a constant of `i1` type.
 inline Value constantI1(OpBuilder &builder, Location loc, bool b) {
-  return builder.create<arith::ConstantIntOp>(loc, b, 1);
+  return arith::ConstantIntOp::create(builder, loc, b, 1);
 }
 
 /// Generates a constant of the given `Action`.
@@ -400,12 +400,12 @@ inline Value constantLevelTypeEncoding(OpBuilder &builder, Location loc,
 inline Value genValFromAttr(OpBuilder &builder, Location loc, Attribute attr) {
   if (auto complexAttr = dyn_cast<complex::NumberAttr>(attr)) {
     Type tp = cast<ComplexType>(complexAttr.getType()).getElementType();
-    return builder.create<complex::ConstantOp>(
+    return complex::ConstantOp::create(builder,
         loc, complexAttr.getType(),
         builder.getArrayAttr({FloatAttr::get(tp, complexAttr.getReal()),
                               FloatAttr::get(tp, complexAttr.getImag())}));
   }
-  return builder.create<arith::ConstantOp>(loc, cast<TypedAttr>(attr));
+  return arith::ConstantOp::create(builder, loc, cast<TypedAttr>(attr));
 }
 
 // TODO: is this at the right place?
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
index 9e41c8ec19bcd..6d795f91b1ab2 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
@@ -27,18 +27,18 @@ using namespace mlir::sparse_tensor;
 //===----------------------------------------------------------------------===//
 
 #define CMPI(p, l, r)                                                          \
-  (builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::p, (l), (r))       \
+  (arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::p, (l), (r))       \
        .getResult())
 
 #define C_IDX(v) (constantIndex(builder, loc, (v)))
-#define YIELD(vs) (builder.create<scf::YieldOp>(loc, (vs)))
-#define ADDI(lhs, rhs) (builder.create<arith::AddIOp>(loc, (lhs), (rhs)))
-#define ANDI(lhs, rhs) (builder.create<arith::AndIOp>(loc, (lhs), (rhs)))
-#define SUBI(lhs, rhs) (builder.create<arith::SubIOp>(loc, (lhs), (rhs)))
-#define MULI(lhs, rhs) (builder.create<arith::MulIOp>(loc, (lhs), (rhs)))
-#define REMUI(lhs, rhs) (builder.create<arith::RemUIOp>(loc, (lhs), (rhs)))
-#define DIVUI(lhs, rhs) (builder.create<arith::DivUIOp>(loc, (lhs), (rhs)))
-#define SELECT(c, l, r) (builder.create<arith::SelectOp>(loc, (c), (l), (r)))
+#define YIELD(vs) (scf::YieldOp::create(builder, loc, (vs)))
+#define ADDI(lhs, rhs) (arith::AddIOp::create(builder, loc, (lhs), (rhs)))
+#define ANDI(lhs, rhs) (arith::AndIOp::create(builder, loc, (lhs), (rhs)))
+#define SUBI(lhs, rhs) (arith::SubIOp::create(builder, loc, (lhs), (rhs)))
+#define MULI(lhs, rhs) (arith::MulIOp::create(builder, loc, (lhs), (rhs)))
+#define REMUI(lhs, rhs) (arith::RemUIOp::create(builder, loc, (lhs), (rhs)))
+#define DIVUI(lhs, rhs) (arith::DivUIOp::create(builder, loc, (lhs), (rhs)))
+#define SELECT(c, l, r) (arith::SelectOp::create(builder, loc, (c), (l), (r)))
 
 //===----------------------------------------------------------------------===//
 // Debugging utils
@@ -47,7 +47,7 @@ using namespace mlir::sparse_tensor;
 #ifndef NDEBUG
 LLVM_ATTRIBUTE_UNUSED static void dumpIndexMemRef(OpBuilder &builder,
                                                   Location loc, Value memref) {
-  memref = builder.create<memref::CastOp>(
+  memref = memref::CastOp::create(builder,
       loc, UnrankedMemRefType::get(builder.getIndexType(), 0), memref);
   createFuncCall(builder, loc, "printMemrefInd", TypeRange{},
                  ValueRange{memref}, EmitCInterface::On);
@@ -263,7 +263,7 @@ void LoopEmitter::initializeLoopEmit(
         denseTp = bufferization::getMemRefTypeWithFullyDynamicLayout(rtp);
 
       Value denseVal =
-          builder.create<bufferization::ToBufferOp>(loc, denseTp, tensor);
+          bufferization::ToBufferOp::create(builder, loc, denseTp, tensor);
       // Dense outputs need special handling.
       if (isOutput && updater)
         denseVal = updater(builder, loc, denseVal, tensor);
@@ -273,7 +273,7 @@ void LoopEmitter::initializeLoopEmit(
       // Annotated sparse tensors.
       // We also need the value buffer for all-dense annotated "sparse"
       // tensors.
-      valBuffer[t] = builder.create<ToValuesOp>(loc, tensor);
+      valBuffer[t] = ToValuesOp::create(builder, loc, tensor);
     }
   }
 
@@ -481,7 +481,7 @@ std::pair<Operation *, Value> LoopEmitter::emitForLoopOverTensorAtLvl(
   Value iv;
   if (isParallel) {
     scf::ParallelOp parOp =
-        builder.create<scf::ParallelOp>(loc, lo, hi, step, reduc);
+        scf::ParallelOp::create(builder, loc, lo, hi, step, reduc);
     builder.setInsertionPointToStart(parOp.getBody());
     assert(parOp.getNumReductions() == reduc.size());
     iv = parOp.getInductionVars()[0];
@@ -497,7 +497,7 @@ std::pair<Operation *, Value> LoopEmitter::emitForLoopOverTensorAtLvl(
       reduc[i] = parOp.getInitVals()[i];
     loop = parOp;
   } else {
-    scf::ForOp forOp = builder.create<scf::ForOp>(loc, lo, hi, step, reduc);
+    scf::ForOp forOp = scf::ForOp::create(builder, loc, lo, hi, step, reduc);
     builder.setInsertionPointToStart(forOp.getBody());
     iv = forOp.getInductionVar();
 
@@ -605,11 +605,11 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
 
       // Extract and iterate over the iteration space.
       ExtractIterSpaceOp extractSpaceOp =
-          lvl == 0 ? builder.create<ExtractIterSpaceOp>(loc, t)
-                   : builder.create<ExtractIterSpaceOp>(
+          lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
+                   : ExtractIterSpaceOp::create(builder,
                          loc, t, spIterVals[tid][lvl - 1], lvl);
 
-      IterateOp iterOp = builder.create<IterateOp>(
+      IterateOp iterOp = IterateOp::create(builder,
           loc, extractSpaceOp.getExtractedSpace(), reduc);
       spIterVals[tid][lvl] = iterOp.getIterator();
 
@@ -627,12 +627,12 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
     for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
       Value t = tensors[tid];
       ExtractIterSpaceOp extractSpaceOp =
-          lvl == 0 ? builder.create<ExtractIterSpaceOp>(loc, t)
-                   : builder.create<ExtractIterSpaceOp>(
+          lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
+                   : ExtractIterSpaceOp::create(builder,
                          loc, t, spIterVals[tid][lvl - 1], lvl);
       spaces.push_back(extractSpaceOp.getExtractedSpace());
     }
-    auto coIterOp = builder.create<CoIterateOp>(loc, spaces, reduc, numCases);
+    auto coIterOp = CoIterateOp::create(builder, loc, spaces, reduc, numCases);
     // The CoIterationOp does not have insertion block nor induction variable.
     // TODO: the `struct LoopInfo` should be simplied after full migration.
     loopStack.emplace_back(tidLvls, coIterOp, /*insertion block*/ nullptr,
@@ -730,7 +730,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   if (emitStrategy == SparseEmitStrategy::kSparseIterator) {
     auto iterateOp = llvm::cast<IterateOp>(loopInfo.loop);
     assert(reduc.size() == iterateOp.getNumResults());
-    rewriter.create<sparse_tensor::YieldOp>(loc, reduc);
+    sparse_tensor::YieldOp::create(rewriter, loc, reduc);
     // Exit the loop.
     rewriter.setInsertionPointAfter(iterateOp);
     // In-place update reduction variables.
@@ -740,7 +740,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   if (auto forOp = llvm::dyn_cast<scf::ForOp>(loopInfo.loop)) {
     if (!reduc.empty()) {
       assert(reduc.size() == forOp.getNumResults());
-      rewriter.create<scf::YieldOp>(loc, reduc);
+      scf::YieldOp::create(rewriter, loc, reduc);
     }
     // Exit the loop.
     rewriter.setInsertionPointAfter(forOp);
@@ -779,7 +779,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
 #endif // NDEBUG
 
       rewriter.setInsertionPointAfter(redExp);
-      auto redOp = rewriter.create<scf::ReduceOp>(loc, curVal);
+      auto redOp = scf::ReduceOp::create(rewriter, loc, curVal);
       // Attach to the reduction op.
       Block *redBlock = &redOp.getReductions().front().front();
       rewriter.setInsertionPointToEnd(redBlock);
@@ -791,7 +791,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
       // Erases the out-dated reduction expression.
       rewriter.eraseOp(redExp);
       rewriter.setInsertionPointToEnd(redBlock);
-      rewriter.create<scf::ReduceReturnOp>(loc, newRed->getResult(0));
+      scf::ReduceReturnOp::create(rewriter, loc, newRed->getResult(0));
     }
     rewriter.setInsertionPointAfter(parOp);
     // In-place update reduction variables.
@@ -865,7 +865,7 @@ void LoopEmitter::exitCurrentLoop(RewriterBase &rewriter, Location loc,
   if (emitStrategy == SparseEmitStrategy::kSparseIterator) {
     Operation *p = loopInfo.loop;
     if (isa<IterateOp>(p))
-      rewriter.create<sparse_tensor::YieldOp>(loc, reduc);
+      sparse_tensor::YieldOp::create(rewriter, loc, reduc);
 
     // Exit the loop.
     rewriter.setInsertionPointAfter(p);
@@ -931,7 +931,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
   // Ensures all operands are valid.
   assert(!llvm::is_contained(ivs, nullptr));
   TypeRange types = ValueRange(ivs).getTypes();
-  auto whileOp = builder.create<scf::WhileOp>(loc, types, ivs);
+  auto whileOp = scf::WhileOp::create(builder, loc, types, ivs);
 
   SmallVector<Location> locs(types.size(), loc);
   Block *before = builder.createBlock(&whileOp.getBefore(), {}, types, locs);
@@ -950,7 +950,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
   // The remaining block arguments are user-provided reduction values and an
   // optional universal index. Make sure their sizes match.
   assert(bArgs.size() == reduc.size() + (uniIdx ? 1 : 0));
-  builder.create<scf::ConditionOp>(loc, whileCond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, whileCond, before->getArguments());
 
   // Generates loop body.
   builder.setInsertionPointToStart(after);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
index cf99117065c5f..3b6976f8b7341 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
@@ -79,14 +79,14 @@ SparseTensorTypeToBufferConverter::SparseTensorTypeToBufferConverter() {
 
 Value SparseTensorSpecifier::getInitValue(OpBuilder &builder, Location loc,
                                           SparseTensorType stt) {
-  return builder.create<StorageSpecifierInitOp>(
+  return StorageSpecifierInitOp::create(builder,
       loc, StorageSpecifierType::get(stt.getEncoding()));
 }
 
 Value SparseTensorSpecifier::getSpecifierField(OpBuilder &builder, Location loc,
                                                StorageSpecifierKind kind,
                                                std::optional<Level> lvl) {
-  return builder.create<GetStorageSpecifierOp>(
+  return GetStorageSpecifierOp::create(builder,
       loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl));
 }
 
@@ -96,7 +96,7 @@ void SparseTensorSpecifier::setSpecifierField(OpBuilder &builder, Location loc,
                                               std::optional<Level> lvl) {
   // TODO: make `v` have type `TypedValue<IndexType>` instead.
   assert(v.getType().isIndex());
-  specifier = builder.create<SetStorageSpecifierOp>(
+  specifier = SetStorageSpecifierOp::create(builder,
       loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl), v);
 }
 
@@ -112,8 +112,8 @@ Value sparse_tensor::SparseTensorDescriptor::getCrdMemRefOrView(
 
   Value stride = constantIndex(builder, loc, rType.getLvlRank() - cooStart);
   Value size = getCrdMemSize(builder, loc, cooStart);
-  size = builder.create<arith::DivUIOp>(loc, size, stride);
-  return builder.create<memref::SubViewOp>(
+  size = arith::DivUIOp::create(builder, loc, size, stride);
+  return memref::SubViewOp::create(builder,
       loc, getMemRefField(SparseTensorFieldKind::CrdMemRef, cooStart),
       /*offset=*/ValueRange{constantIndex(builder, loc, lvl - cooStart)},
       /*size=*/ValueRange{size},
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
index 869c7864d7535..45d142a807c36 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
@@ -231,7 +231,7 @@ class MutSparseTensorDescriptor
 /// Packs the given values as a "tuple" value.
 inline Value genTuple(OpBuilder &builder, Location loc, Type tp,
                       ValueRange values) {
-  return builder.create<UnrealizedConversionCastOp>(loc, TypeRange(tp), values)
+  return UnrealizedConversionCastOp::create(builder, loc, TypeRange(tp), values)
       .getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
index aad5e97ed14ab..212ee99647167 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
@@ -22,23 +22,23 @@ using ValueTuple = std::tuple<Value, Value, Value>;
 // File local helper functions/macros.
 //===----------------------------------------------------------------------===//
 #define CMPI(p, lhs, rhs)                                                      \
-  (b.create<arith::CmpIOp>(l, arith::CmpIPredicate::p, (lhs), (rhs))           \
+  (arith::CmpIOp::create(b, l, arith::CmpIPredicate::p, (lhs), (rhs))           \
        .getResult())
 
 #define C_FALSE (constantI1(b, l, false))
 #define C_TRUE (constantI1(b, l, true))
 #define C_IDX(v) (constantIndex(b, l, (v)))
-#define YIELD(vs) (b.create<scf::YieldOp>(l, (vs)))
-#define ADDI(lhs, rhs) (b.create<arith::AddIOp>(l, (lhs), (rhs)).getResult())
-#define ORI(lhs, rhs) (b.create<arith::OrIOp>(l, (lhs), (rhs)).getResult())
-#define ANDI(lhs, rhs) (b.create<arith::AndIOp>(l, (lhs), (rhs)).getResult())
-#define SUBI(lhs, rhs) (b.create<arith::SubIOp>(l, (lhs), (rhs)).getResult())
-#define MULI(lhs, rhs) (b.create<arith::MulIOp>(l, (lhs), (rhs)).getResult())
-#define MINUI(lhs, rhs) (b.create<arith::MinUIOp>(l, (lhs), (rhs)).getResult())
-#define REMUI(lhs, rhs) (b.create<arith::RemUIOp>(l, (lhs), (rhs)).getResult())
-#define DIVUI(lhs, rhs) (b.create<arith::DivUIOp>(l, (lhs), (rhs)).getResult())
+#define YIELD(vs) (scf::YieldOp::create(b, l, (vs)))
+#define ADDI(lhs, rhs) (arith::AddIOp::create(b, l, (lhs), (rhs)).getResult())
+#define ORI(lhs, rhs) (arith::OrIOp::create(b, l, (lhs), (rhs)).getResult())
+#define ANDI(lhs, rhs) (arith::AndIOp::create(b, l, (lhs), (rhs)).getResult())
+#define SUBI(lhs, rhs) (arith::SubIOp::create(b, l, (lhs), (rhs)).getResult())
+#define MULI(lhs, rhs) (arith::MulIOp::create(b, l, (lhs), (rhs)).getResult())
+#define MINUI(lhs, rhs) (arith::MinUIOp::create(b, l, (lhs), (rhs)).getResult())
+#define REMUI(lhs, rhs) (arith::RemUIOp::create(b, l, (lhs), (rhs)).getResult())
+#define DIVUI(lhs, rhs) (arith::DivUIOp::create(b, l, (lhs), (rhs)).getResult())
 #define SELECT(c, lhs, rhs)                                                    \
-  (b.create<arith::SelectOp>(l, (c), (lhs), (rhs)).getResult())
+  (arith::SelectOp::create(b, l, (c), (lhs), (rhs)).getResult())
 
 //===----------------------------------------------------------------------===//
 // SparseTensorLevel derived classes.
@@ -150,19 +150,19 @@ class CompressedLevel : public SparseLevel</*hasPosBuf=*/true> {
       return loadRange();
 
     SmallVector<Type, 2> types{b.getIndexType(), b.getIndexType()};
-    scf::IfOp posRangeIf = b.create<scf::IfOp>(l, types, inPadZone, true);
+    scf::IfOp posRangeIf = scf::IfOp::create(b, l, types, inPadZone, true);
     // True branch, returns a "fake" empty range [0, 0) if parent
     // iterator is in pad zone.
     b.setInsertionPointToStart(posRangeIf.thenBlock());
 
     SmallVector<Value, 2> emptyRange{C_IDX(0), C_IDX(0)};
-    b.create<scf::YieldOp>(l, emptyRange);
+    scf::YieldOp::create(b, l, emptyRange);
 
     // False branch, returns the actual range.
     b.setInsertionPointToStart(posRangeIf.elseBlock());
     auto [pLo, pHi] = loadRange();
     SmallVector<Value, 2> loadedRange{pLo, pHi};
-    b.create<scf::YieldOp>(l, loadedRange);
+    scf::YieldOp::create(b, l, loadedRange);
 
     b.setInsertionPointAfter(posRangeIf);
     ValueRange posRange = posRangeIf.getResults();
@@ -248,7 +248,7 @@ static scf::ValueVector genWhenInBound(
     llvm::function_ref<scf::ValueVector(OpBuilder &, Location, Value)>
         builder) {
   TypeRange ifRetTypes = elseRet.getTypes();
-  auto ifOp = b.create<scf::IfOp>(l, ifRetTypes, it.genNotEnd(b, l), true);
+  auto ifOp = scf::IfOp::create(b, l, ifRetTypes, it.genNotEnd(b, l), true);
 
   b.setInsertionPointToStart(ifOp.thenBlock());
   Value crd = it.deref(b, l);
@@ -732,7 +732,7 @@ class NonEmptySubSectIterator : public SparseIterator {
   //      [itVal0, itVal1, ..., pNx0],
   //      ...]
   Value allocSubSectPosBuf(OpBuilder &b, Location l) {
-    return b.create<memref::AllocaOp>(
+    return memref::AllocaOp::create(b,
         l,
         MemRefType::get({ShapedType::kDynamic, tupleSz + 1}, b.getIndexType()),
         maxTupleCnt);
@@ -740,12 +740,12 @@ class NonEmptySubSectIterator : public SparseIterator {
 
   void storeNxLvlStart(OpBuilder &b, Location l, Value tupleId,
                        Value start) const {
-    b.create<memref::StoreOp>(l, start, subSectPosBuf,
+    memref::StoreOp::create(b, l, start, subSectPosBuf,
                               ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
   Value loadNxLvlStart(OpBuilder &b, Location l, Value tupleId) const {
-    return b.create<memref::LoadOp>(l, subSectPosBuf,
+    return memref::LoadOp::create(b, l, subSectPosBuf,
                                     ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
@@ -753,7 +753,7 @@ class NonEmptySubSectIterator : public SparseIterator {
                        ValueRange itVals) const {
     assert(itVals.size() == tupleSz);
     for (unsigned i = 0; i < tupleSz; i++) {
-      b.create<memref::StoreOp>(l, itVals[i], subSectPosBuf,
+      memref::StoreOp::create(b, l, itVals[i], subSectPosBuf,
                                 ValueRange{tupleId, C_IDX(i)});
     }
   }
@@ -762,7 +762,7 @@ class NonEmptySubSectIterator : public SparseIterator {
                                     Value tupleId) const {
     SmallVector<Value> ret;
     for (unsigned i = 0; i < tupleSz; i++) {
-      Value v = b.create<memref::LoadOp>(l, subSectPosBuf,
+      Value v = memref::LoadOp::create(b, l, subSectPosBuf,
                                          ValueRange{tupleId, C_IDX(i)});
       ret.push_back(v);
     }
@@ -1043,7 +1043,7 @@ ValueRange SparseIterator::forward(OpBuilder &b, Location l) {
 }
 
 ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
-  auto ifOp = b.create<scf::IfOp>(l, getCursor().getTypes(), cond, true);
+  auto ifOp = scf::IfOp::create(b, l, getCursor().getTypes(), cond, true);
   // Generate else branch first, otherwise iterator values will be updated by
   // `forward()`.
   b.setInsertionPointToStart(ifOp.elseBlock());
@@ -1058,12 +1058,12 @@ ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
 }
 
 Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
-  auto whileOp = b.create<scf::WhileOp>(
+  auto whileOp = scf::WhileOp::create(b,
       l, pos.getType(), pos,
       /*beforeBuilder=*/
       [this, pos](OpBuilder &b, Location l, ValueRange ivs) {
         Value inBound = CMPI(ult, ivs.front(), posHi);
-        auto ifInBound = b.create<scf::IfOp>(l, b.getI1Type(), inBound, true);
+        auto ifInBound = scf::IfOp::create(b, l, b.getI1Type(), inBound, true);
         {
           OpBuilder::InsertionGuard guard(b);
           // If in bound, load the next coordinates and check duplication.
@@ -1076,7 +1076,7 @@ Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
           b.setInsertionPointToStart(ifInBound.elseBlock());
           YIELD(constantI1(b, l, false));
         }
-        b.create<scf::ConditionOp>(l, ifInBound.getResults()[0], ivs);
+        scf::ConditionOp::create(b, l, ifInBound.getResults()[0], ivs);
       },
       /*afterBuilder=*/
       [](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1137,7 +1137,7 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
 
   SmallVector<Value> whileArgs(getCursor().begin(), getCursor().end());
   whileArgs.push_back(isFirst);
-  auto whileOp = b.create<scf::WhileOp>(
+  auto whileOp = scf::WhileOp::create(b,
       l, ValueRange(whileArgs).getTypes(), whileArgs,
       /*beforeBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1154,7 +1154,7 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
                              ret = ORI(ret, llvm::getSingleElement(isFirst));
                              return {ret};
                            });
-        b.create<scf::ConditionOp>(l, cont.front(), ivs);
+        scf::ConditionOp::create(b, l, cont.front(), ivs);
       },
       /*afterBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1219,7 +1219,7 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
     SmallVector<Value> iterArgs;
     iterArgs.push_back(C_IDX(0));
     iterArgs.append(reduc.begin(), reduc.end());
-    auto forEachLeaf = b.create<scf::ForOp>(
+    auto forEachLeaf = scf::ForOp::create(b,
         l, /*lb=*/C_IDX(0), /*ub=*/tupleCnt, /*step=*/C_IDX(1), iterArgs,
         [&helper, &builder](OpBuilder &b, Location l, Value tupleId,
                             ValueRange iterArgs) {
@@ -1235,12 +1235,12 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
           SmallVector<Value> whileArgs(helper.wrap.getCursor());
           whileArgs.append(iterArgs.begin(), iterArgs.end());
 
-          auto whileOp = b.create<scf::WhileOp>(
+          auto whileOp = scf::WhileOp::create(b,
               l, ValueRange(whileArgs).getTypes(), whileArgs,
               /*beforeBuilder=*/
               [&helper](OpBuilder &b, Location l, ValueRange ivs) {
                 helper.wrap.linkNewScope(ivs);
-                b.create<scf::ConditionOp>(l, helper.genNotEnd(b, l), ivs);
+                scf::ConditionOp::create(b, l, helper.genNotEnd(b, l), ivs);
               },
               /*afterBuilder=*/
               [&helper, &builder](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1267,7 +1267,7 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
                                      ValueRange reduc) {
     assert(!parent || parent->lvl + 1 == lvl);
     delegate->genInit(b, l, parent);
-    auto forOp = b.create<scf::ForOp>(
+    auto forOp = scf::ForOp::create(b,
         l, /*lb=*/C_IDX(0), /*ub=*/subSectSz, /*step=*/C_IDX(1), reduc,
         [&](OpBuilder &b, Location l, Value crd, ValueRange iterArgs) {
           helper.locate(b, l, crd);
@@ -1411,7 +1411,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
   // if (offset + size > parents.size)
   //   isNonEmpty = false;
   Value fastPathP = CMPI(ugt, getMinCrd(), getAbsOff());
-  auto ifOp = b.create<scf::IfOp>(l, getCursor().getTypes(), fastPathP, true);
+  auto ifOp = scf::IfOp::create(b, l, getCursor().getTypes(), fastPathP, true);
   {
     OpBuilder::InsertionGuard guard(b);
     // Take the fast path
@@ -1448,7 +1448,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 Value isMin = CMPI(eq, crd, getMinCrd());
                 delegate->forwardIf(b, l, isMin);
                 // Update the forwarded iterator values if needed.
-                auto ifIsMin = b.create<scf::IfOp>(l, isMin, false);
+                auto ifIsMin = scf::IfOp::create(b, l, isMin, false);
                 b.setInsertionPointToStart(&ifIsMin.getThenRegion().front());
                 storeCursorVals(b, l, tupleId, delegate->serialize());
                 b.setInsertionPointAfter(ifIsMin);
@@ -1458,7 +1458,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 return genWhenInBound(b, l, *delegate, /*elseRet=*/iterArgs,
                                       [nxMin](OpBuilder &b, Location l,
                                               Value crd) -> scf::ValueVector {
-                                        Value nx = b.create<arith::MinUIOp>(
+                                        Value nx = arith::MinUIOp::create(b,
                                             l, crd, nxMin);
                                         return {nx, C_TRUE};
                                       });
@@ -1480,7 +1480,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
 
   // We should at least forward the offset by one.
   Value minAbsOff = ADDI(getAbsOff(), c1);
-  nxAbsOff = b.create<arith::MaxUIOp>(l, minAbsOff, nxAbsOff);
+  nxAbsOff = arith::MaxUIOp::create(b, l, minAbsOff, nxAbsOff);
 
   seek(ValueRange{nxMinCrd, nxAbsOff, nxNotEnd});
   // The coordinate should not exceeds the space upper bound.
@@ -1581,16 +1581,16 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &b, Location l, Value t,
   auto stt = getSparseTensorType(t);
 
   LevelType lt = stt.getLvlType(lvl);
-  Value sz = stt.hasEncoding() ? b.create<LvlOp>(l, t, lvl).getResult()
-                               : b.create<tensor::DimOp>(l, t, lvl).getResult();
+  Value sz = stt.hasEncoding() ? LvlOp::create(b, l, t, lvl).getResult()
+                               : tensor::DimOp::create(b, l, t, lvl).getResult();
 
   SmallVector<Value, 2> buffers;
   if (lt.isWithPosLT()) {
-    Value pos = b.create<ToPositionsOp>(l, t, lvl);
+    Value pos = ToPositionsOp::create(b, l, t, lvl);
     buffers.push_back(pos);
   }
   if (lt.isWithCrdLT()) {
-    Value pos = b.create<ToCoordinatesOp>(l, t, lvl);
+    Value pos = ToCoordinatesOp::create(b, l, t, lvl);
     buffers.push_back(pos);
   }
   return makeSparseTensorLevel(lt, sz, buffers, tid, lvl);
diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
index 0258f797143cb..fb68620c378e5 100644
--- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
@@ -1563,7 +1563,7 @@ static Value insertYieldOp(RewriterBase &rewriter, Location loc, Region &region,
   Block &clonedBlock = tmpRegion.front();
   YieldOp clonedYield = cast<YieldOp>(clonedBlock.getTerminator());
   // Merge cloned block and return yield value.
-  Operation *placeholder = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Operation *placeholder = arith::ConstantIndexOp::create(rewriter, loc, 0);
   rewriter.inlineBlockBefore(&tmpRegion.front(), placeholder, vals);
   Value val = clonedYield.getSingleResult();
   rewriter.eraseOp(clonedYield);
@@ -1603,16 +1603,16 @@ static Value buildRelu(RewriterBase &rewriter, Location loc, Value v0,
                        Attribute attr) {
   Type tp = v0.getType();
   auto zero =
-      rewriter.create<arith::ConstantOp>(loc, tp, rewriter.getZeroAttr(tp));
+      arith::ConstantOp::create(rewriter, loc, tp, rewriter.getZeroAttr(tp));
   Value cmp;
   if (isa<FloatType>(tp)) {
     auto pred = llvm::cast<arith::CmpFPredicateAttr>(attr);
-    cmp = rewriter.create<arith::CmpFOp>(loc, pred, v0, zero);
+    cmp = arith::CmpFOp::create(rewriter, loc, pred, v0, zero);
   } else {
     auto pred = llvm::cast<arith::CmpIPredicateAttr>(attr);
-    cmp = rewriter.create<arith::CmpIOp>(loc, pred, v0, zero);
+    cmp = arith::CmpIOp::create(rewriter, loc, pred, v0, zero);
   }
-  return rewriter.create<arith::SelectOp>(loc, cmp, v0, zero);
+  return arith::SelectOp::create(rewriter, loc, cmp, v0, zero);
 }
 
 Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
@@ -1627,128 +1627,128 @@ Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
     llvm_unreachable("unexpected non-op");
   // Unary operations.
   case TensorExp::Kind::kAbsF:
-    return rewriter.create<math::AbsFOp>(loc, v0);
+    return math::AbsFOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kAbsC: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::AbsOp>(loc, eltType, v0);
+    return complex::AbsOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kAbsI:
-    return rewriter.create<math::AbsIOp>(loc, v0);
+    return math::AbsIOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kCeilF:
-    return rewriter.create<math::CeilOp>(loc, v0);
+    return math::CeilOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kFloorF:
-    return rewriter.create<math::FloorOp>(loc, v0);
+    return math::FloorOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSqrtF:
-    return rewriter.create<math::SqrtOp>(loc, v0);
+    return math::SqrtOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSqrtC:
-    return rewriter.create<complex::SqrtOp>(loc, v0);
+    return complex::SqrtOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kExpm1F:
-    return rewriter.create<math::ExpM1Op>(loc, v0);
+    return math::ExpM1Op::create(rewriter, loc, v0);
   case TensorExp::Kind::kExpm1C:
-    return rewriter.create<complex::Expm1Op>(loc, v0);
+    return complex::Expm1Op::create(rewriter, loc, v0);
   case TensorExp::Kind::kLog1pF:
-    return rewriter.create<math::Log1pOp>(loc, v0);
+    return math::Log1pOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kLog1pC:
-    return rewriter.create<complex::Log1pOp>(loc, v0);
+    return complex::Log1pOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kRelu:
     return buildRelu(rewriter, loc, v0, expr.attr);
   case TensorExp::Kind::kSinF:
-    return rewriter.create<math::SinOp>(loc, v0);
+    return math::SinOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSinC:
-    return rewriter.create<complex::SinOp>(loc, v0);
+    return complex::SinOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kTanhF:
-    return rewriter.create<math::TanhOp>(loc, v0);
+    return math::TanhOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kTanhC:
-    return rewriter.create<complex::TanhOp>(loc, v0);
+    return complex::TanhOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegF:
-    return rewriter.create<arith::NegFOp>(loc, v0);
+    return arith::NegFOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegC:
-    return rewriter.create<complex::NegOp>(loc, v0);
+    return complex::NegOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegI: // no negi in std
-    return rewriter.create<arith::SubIOp>(
+    return arith::SubIOp::create(rewriter,
         loc,
-        rewriter.create<arith::ConstantOp>(loc, v0.getType(),
+        arith::ConstantOp::create(rewriter, loc, v0.getType(),
                                            rewriter.getZeroAttr(v0.getType())),
         v0);
   case TensorExp::Kind::kTruncF:
-    return rewriter.create<arith::TruncFOp>(loc, inferType(e, v0), v0);
+    return arith::TruncFOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kExtF:
-    return rewriter.create<arith::ExtFOp>(loc, inferType(e, v0), v0);
+    return arith::ExtFOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastFS:
-    return rewriter.create<arith::FPToSIOp>(loc, inferType(e, v0), v0);
+    return arith::FPToSIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastFU:
-    return rewriter.create<arith::FPToUIOp>(loc, inferType(e, v0), v0);
+    return arith::FPToUIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastSF:
-    return rewriter.create<arith::SIToFPOp>(loc, inferType(e, v0), v0);
+    return arith::SIToFPOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastUF:
-    return rewriter.create<arith::UIToFPOp>(loc, inferType(e, v0), v0);
+    return arith::UIToFPOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastS:
-    return rewriter.create<arith::ExtSIOp>(loc, inferType(e, v0), v0);
+    return arith::ExtSIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastU:
-    return rewriter.create<arith::ExtUIOp>(loc, inferType(e, v0), v0);
+    return arith::ExtUIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastIdx:
-    return rewriter.create<arith::IndexCastOp>(loc, inferType(e, v0), v0);
+    return arith::IndexCastOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kTruncI:
-    return rewriter.create<arith::TruncIOp>(loc, inferType(e, v0), v0);
+    return arith::TruncIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCIm: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::ImOp>(loc, eltType, v0);
+    return complex::ImOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kCRe: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::ReOp>(loc, eltType, v0);
+    return complex::ReOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kBitCast:
-    return rewriter.create<arith::BitcastOp>(loc, inferType(e, v0), v0);
+    return arith::BitcastOp::create(rewriter, loc, inferType(e, v0), v0);
   // Binary operations.
   case TensorExp::Kind::kMulF:
-    return rewriter.create<arith::MulFOp>(loc, v0, v1);
+    return arith::MulFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kMulC:
-    return rewriter.create<complex::MulOp>(loc, v0, v1);
+    return complex::MulOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kMulI:
-    return rewriter.create<arith::MulIOp>(loc, v0, v1);
+    return arith::MulIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivF:
-    return rewriter.create<arith::DivFOp>(loc, v0, v1);
+    return arith::DivFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivC:
-    return rewriter.create<complex::DivOp>(loc, v0, v1);
+    return complex::DivOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivS:
-    return rewriter.create<arith::DivSIOp>(loc, v0, v1);
+    return arith::DivSIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivU:
-    return rewriter.create<arith::DivUIOp>(loc, v0, v1);
+    return arith::DivUIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddF:
-    return rewriter.create<arith::AddFOp>(loc, v0, v1);
+    return arith::AddFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddC:
-    return rewriter.create<complex::AddOp>(loc, v0, v1);
+    return complex::AddOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddI:
-    return rewriter.create<arith::AddIOp>(loc, v0, v1);
+    return arith::AddIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubF:
-    return rewriter.create<arith::SubFOp>(loc, v0, v1);
+    return arith::SubFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubC:
-    return rewriter.create<complex::SubOp>(loc, v0, v1);
+    return complex::SubOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubI:
-    return rewriter.create<arith::SubIOp>(loc, v0, v1);
+    return arith::SubIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAndI:
-    return rewriter.create<arith::AndIOp>(loc, v0, v1);
+    return arith::AndIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kOrI:
-    return rewriter.create<arith::OrIOp>(loc, v0, v1);
+    return arith::OrIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kXorI:
-    return rewriter.create<arith::XOrIOp>(loc, v0, v1);
+    return arith::XOrIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShrS:
-    return rewriter.create<arith::ShRSIOp>(loc, v0, v1);
+    return arith::ShRSIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShrU:
-    return rewriter.create<arith::ShRUIOp>(loc, v0, v1);
+    return arith::ShRUIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShlI:
-    return rewriter.create<arith::ShLIOp>(loc, v0, v1);
+    return arith::ShLIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kCmpI: {
     auto predicate = llvm::cast<arith::CmpIPredicateAttr>(expr.attr);
-    return rewriter.create<arith::CmpIOp>(loc, predicate, v0, v1);
+    return arith::CmpIOp::create(rewriter, loc, predicate, v0, v1);
   }
   case TensorExp::Kind::kCmpF: {
     auto predicate = llvm::cast<arith::CmpFPredicateAttr>(expr.attr);
-    return rewriter.create<arith::CmpFOp>(loc, predicate, v0, v1);
+    return arith::CmpFOp::create(rewriter, loc, predicate, v0, v1);
   }
   case TensorExp::Kind::kBinaryBranch: // semi-ring ops with custom logic.
     return insertYieldOp(rewriter, loc, *expr.op->getBlock()->getParent(),
diff --git a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
index fc93f1c1c9220..5e6e70987f6bd 100644
--- a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
+++ b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
@@ -78,11 +78,11 @@ struct CreatorOpShardingInterface
         if (!oldType.isDynamicDim(i) && shardType.isDynamicDim(i)) {
           if (!newSharding) {
             newSharding =
-                builder.create<ShardingOp>(op->getLoc(), resultShardings[0]);
+                ShardingOp::create(builder, op->getLoc(), resultShardings[0]);
             device =
-                builder.create<mesh::ProcessMultiIndexOp>(op->getLoc(), mesh)
+                mesh::ProcessMultiIndexOp::create(builder, op->getLoc(), mesh)
                     .getResults();
-            shapeForDevice = builder.create<mesh::ShardShapeOp>(
+            shapeForDevice = mesh::ShardShapeOp::create(builder,
                 op->getLoc(), oldType.getShape(), spmdizedOperands,
                 newSharding->getResult(0), device);
           }
@@ -92,7 +92,7 @@ struct CreatorOpShardingInterface
           newOperands.emplace_back(spmdizedOperands[++currOldOprndNum]);
         }
       }
-      newOp = builder.create<OpTy>(op->getLoc(), shardType, newOperands);
+      newOp = OpTy::create(builder, op->getLoc(), shardType, newOperands);
       spmdizationMap.map(op->getResult(0), newOp->getResult(0));
     } else {
       // `clone` will populate the mapping of old to new results.
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index a3e863254405c..79bddee1e7b84 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -56,7 +56,7 @@ Operation *TensorDialect::materializeConstant(OpBuilder &builder,
   if (auto op = arith::ConstantOp::materialize(builder, value, type, loc))
     return op;
   if (complex::ConstantOp::isBuildableWith(value, type))
-    return builder.create<complex::ConstantOp>(loc, type,
+    return complex::ConstantOp::create(builder, loc, type,
                                                llvm::cast<ArrayAttr>(value));
   return nullptr;
 }
@@ -110,7 +110,7 @@ FailureOr<Value> tensor::getOrCreateDestination(OpBuilder &b, Location loc,
 
   // Create empty tensor.
   Value emptyTensor =
-      b.create<tensor::EmptyOp>(loc, mixedSizes, tensorType.getElementType());
+      tensor::EmptyOp::create(b, loc, mixedSizes, tensorType.getElementType());
   return emptyTensor;
 }
 
@@ -681,7 +681,7 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
     inputShapes.emplace_back(std::move(inputShape));
   }
 
-  Value replacement = builder.create<tensor::EmptyOp>(
+  Value replacement = tensor::EmptyOp::create(builder,
       loc, outputShape, getType().getElementType());
 
   int64_t rank = getType().getRank();
@@ -690,12 +690,12 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
   SmallVector<OpFoldResult> offsets(rank, zero);
   for (auto [index, input] : llvm::enumerate(getInputs())) {
     offsets[concatDim] = concatOffsets[index];
-    auto insertSlice = builder.create<tensor::InsertSliceOp>(
+    auto insertSlice = tensor::InsertSliceOp::create(builder,
         loc, input, replacement, offsets, inputShapes[index], strides);
     replacement = insertSlice.getResult();
   }
   if (replacement.getType() != getType()) {
-    replacement = builder.create<tensor::CastOp>(loc, getType(), replacement);
+    replacement = tensor::CastOp::create(builder, loc, getType(), replacement);
   }
   return SmallVector<Value>{replacement};
 }
@@ -726,7 +726,7 @@ ConcatOp::reifyResultShapes(OpBuilder &builder,
           builder.getIndexAttr(inferredResultType.getDimSize(i)));
     } else {
       reifiedReturnShapes[0][i] =
-          builder.create<tensor::DimOp>(init.getLoc(), init, i).getResult();
+          tensor::DimOp::create(builder, init.getLoc(), init, i).getResult();
     }
   }
 
@@ -826,7 +826,7 @@ struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
 
         // Use refined operand type and create cast from original operand.
         auto castOp =
-            rewriter.create<CastOp>(concatOp->getLoc(), inferredOperandType,
+            CastOp::create(rewriter, concatOp->getLoc(), inferredOperandType,
                                     concatOp.getOperand(operandIdx));
         rewriter.modifyOpInPlace(concatOp, [=, operandIdx = operandIdx] {
           concatOp->setOperand(operandIdx, castOp->getResult(0));
@@ -867,7 +867,7 @@ struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
       return failure();
     }
 
-    auto newConcatOp = rewriter.create<ConcatOp>(
+    auto newConcatOp = ConcatOp::create(rewriter,
         concatOp->getLoc(), inferredResultType, dim, concatOp->getOperands());
     rewriter.replaceOpWithNewOp<CastOp>(concatOp, concatOp.getResultType(),
                                         newConcatOp);
@@ -895,7 +895,7 @@ void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
 void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
                   int64_t index) {
   auto loc = result.location;
-  Value indexValue = builder.create<arith::ConstantIndexOp>(loc, index);
+  Value indexValue = arith::ConstantIndexOp::create(builder, loc, index);
   build(builder, result, source, indexValue);
 }
 
@@ -1039,10 +1039,10 @@ struct DimOfReshapeOp : public OpRewritePattern<DimOp> {
     rewriter.setInsertionPointAfter(dim);
     Location loc = dim.getLoc();
     Value extract =
-        rewriter.create<ExtractOp>(loc, reshape.getShape(), dim.getIndex());
+        ExtractOp::create(rewriter, loc, reshape.getShape(), dim.getIndex());
     if (extract.getType() != dim.getType())
       extract =
-          rewriter.create<arith::IndexCastOp>(loc, dim.getType(), extract);
+          arith::IndexCastOp::create(rewriter, loc, dim.getType(), extract);
     rewriter.replaceOp(dim, extract);
     return success();
   }
@@ -1153,7 +1153,7 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
     if (foldedTensorType == op.getType())
       return failure();
 
-    auto newOp = rewriter.create<EmptyOp>(op.getLoc(), foldedTensorType,
+    auto newOp = EmptyOp::create(rewriter, op.getLoc(), foldedTensorType,
                                           foldedDynamicSizes);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
@@ -1329,7 +1329,7 @@ struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
 
       SmallVector<int64_t> basis =
           llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-      auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+      auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
           extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
       llvm::append_range(sourceIndices, delinearize.getResults());
     }
@@ -1501,7 +1501,7 @@ struct ExtractElementFromIndexCast
 
     Type elementTy = getElementTypeOrSelf(indexCast.getIn());
 
-    auto newExtract = rewriter.create<tensor::ExtractOp>(
+    auto newExtract = tensor::ExtractOp::create(rewriter,
         loc, elementTy, indexCast.getIn(), extract.getIndices());
 
     rewriter.replaceOpWithNewOp<arith::IndexCastOp>(extract, extract.getType(),
@@ -1739,7 +1739,7 @@ struct StaticTensorGenerate : public OpRewritePattern<GenerateOp> {
 
     auto loc = generateOp.getLoc();
     auto newOp =
-        rewriter.create<GenerateOp>(loc, foldedTensorType, foldedDynamicSizes);
+        GenerateOp::create(rewriter, loc, foldedTensorType, foldedDynamicSizes);
     rewriter.inlineRegionBefore(generateOp.getBody(), newOp.getBody(),
                                 newOp.getBody().begin());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(generateOp,
@@ -2164,7 +2164,7 @@ struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
         collapseShapeOp.getSrcMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = rewriter.create<CollapseShapeOp>(
+      auto newOp = CollapseShapeOp::create(rewriter,
           collapseShapeOp.getLoc(), newResultType, castOp.getSource(),
           collapseShapeOp.getReassociation());
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
@@ -2243,9 +2243,9 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
         newInputShape, expandOp.getSrcType().getElementType());
     auto outputType = RankedTensorType::get(
         newOutputShape, expandOp.getSrcType().getElementType());
-    auto inputCast = rewriter.create<CastOp>(expandOp.getLoc(), inputType,
+    auto inputCast = CastOp::create(rewriter, expandOp.getLoc(), inputType,
                                              expandOp.getSrc());
-    auto newExpand = rewriter.create<ExpandShapeOp>(
+    auto newExpand = ExpandShapeOp::create(rewriter,
         expandOp.getLoc(), outputType, inputCast.getResult(),
         expandOp.getReassociationIndices(), outputOfr);
     rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
@@ -2558,7 +2558,7 @@ class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
 
     // Create folded extract.
     Location loc = sliceOp.getLoc();
-    Value newResult = rewriter.create<ExtractSliceOp>(
+    Value newResult = ExtractSliceOp::create(rewriter,
         loc, sliceOp.getType(), castOp.getSource(), sliceOp.getOffsets(),
         sliceOp.getSizes(), sliceOp.getStrides(), sliceOp.getStaticOffsets(),
         sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
@@ -2712,7 +2712,7 @@ struct SliceCanonicalizer {
                   ExtractSliceOp newOp) {
     Value replacement = newOp.getResult();
     if (replacement.getType() != op.getType())
-      replacement = rewriter.create<tensor::CastOp>(op.getLoc(), op.getType(),
+      replacement = tensor::CastOp::create(rewriter, op.getLoc(), op.getType(),
                                                     replacement);
     rewriter.replaceOp(op, replacement);
   }
@@ -2981,7 +2981,7 @@ class InsertSliceOpConstantArgumentFolder final
       // the parallel case.
       if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      toInsert = rewriter.create<tensor::CastOp>(insertSliceOp.getLoc(),
+      toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
                                                  sourceType, toInsert);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
@@ -3078,7 +3078,7 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     if (!sliceResult.isValid)
       return failure();
 
-    Operation *replacement = rewriter.create<InsertOpTy>(
+    Operation *replacement = InsertOpTy::create(rewriter,
         insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
         mixedSizes, insertSliceOp.getMixedStrides());
 
@@ -3086,7 +3086,7 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     bool isParallelInsert =
         std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
     if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
-      replacement = rewriter.create<tensor::CastOp>(insertSliceOp.getLoc(),
+      replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
                                                     insertSliceOp.getDestType(),
                                                     replacement->getResult(0));
     }
@@ -3157,7 +3157,7 @@ struct InsertSliceOpSourceCastInserter final
     // parallel case.
     if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
       rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-    Value cast = rewriter.create<tensor::CastOp>(
+    Value cast = tensor::CastOp::create(rewriter,
         insertSliceOp.getLoc(), newSrcType, insertSliceOp.getSource());
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, cast, insertSliceOp.getDest(),
@@ -3356,7 +3356,7 @@ void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
   // a guard to reset the insertion point of the builder after it is destroyed.
   OpBuilder::InsertionGuard guard(b);
   b.createBlock(region, region->end(), blockArgTypes, blockArgLocs);
-  b.create<tensor::YieldOp>(result.location, constantPadValue);
+  tensor::YieldOp::create(b, result.location, constantPadValue);
 }
 
 llvm::SmallBitVector PadOp::getPaddedDims() {
@@ -3410,7 +3410,7 @@ struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
         padTensorOp.getSourceMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = rewriter.create<PadOp>(
+      auto newOp = PadOp::create(rewriter,
           padTensorOp->getLoc(), newResultType, padTensorOp.getSource(),
           padTensorOp.getStaticLow(), padTensorOp.getStaticHigh(),
           padTensorOp.getLow(), padTensorOp.getHigh(), padTensorOp.getNofold(),
@@ -3442,7 +3442,7 @@ struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
                                             tensorCastOp.getDest().getType()))
       return failure();
 
-    auto replacementOp = rewriter.create<PadOp>(
+    auto replacementOp = PadOp::create(rewriter,
         padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
         padTensorOp.getSource(), padTensorOp.getStaticLow(),
         padTensorOp.getStaticHigh(), padTensorOp.getLow(),
@@ -3600,10 +3600,10 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
 
     // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
     // the two paddings in one step.
-    auto newSliceOp = rewriter.create<ExtractSliceOp>(
+    auto newSliceOp = ExtractSliceOp::create(rewriter,
         padOp.getLoc(), outerSliceOp.getSource(), newOffsets, newSizes,
         innerSliceOp.getMixedStrides());
-    auto newPadOp = rewriter.create<PadOp>(
+    auto newPadOp = PadOp::create(rewriter,
         padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
         padOp.getMixedLowPad(), newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
@@ -3700,7 +3700,7 @@ struct FoldStaticPadding : public OpRewritePattern<PadOp> {
     // Rewrite the op using the new static type.
     auto newResultType = RankedTensorType::get(
         newOutDims, padTensorOp.getType().getElementType());
-    auto newOp = rewriter.create<PadOp>(
+    auto newOp = PadOp::create(rewriter,
         padTensorOp->getLoc(), newResultType, input, staticLow, staticHigh,
         newLows, newHighs, padTensorOp.getNofold(),
         getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
@@ -3780,7 +3780,7 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
     SmallVector<OpFoldResult> newLowPad =
         addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         padOp.getLoc(), padOp.getResultType(), producerPad.getSource(),
         newLowPad, newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, tensor::PadOp::getAttributeNames()));
@@ -3806,7 +3806,7 @@ PadOp::reifyResultShapes(OpBuilder &b,
     }
     Location loc = getLoc();
     Value dim = b.createOrFold<tensor::DimOp>(
-        loc, getSource(), b.create<arith::ConstantIndexOp>(loc, i));
+        loc, getSource(), arith::ConstantIndexOp::create(b, loc, i));
 
     AffineExpr d0, d1, d2;
     bindDims(b.getContext(), d0, d1, d2);
@@ -4111,7 +4111,7 @@ struct FoldTensorCastProducerOp
     for (auto [oldResult, newResult] :
          llvm::zip(op->getResults(), newOp->getResults())) {
       if (newResult.getType() != oldResult.getType()) {
-        replacements.push_back(rewriter.create<tensor::CastOp>(
+        replacements.push_back(tensor::CastOp::create(rewriter,
             op->getLoc(), oldResult.getType(), newResult));
       } else {
         replacements.push_back(newResult);
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 92540bd56ecbc..e6cb8bdbc472a 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -211,13 +211,13 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     if (isZeroInteger(newLength)) {
       hasZeroLen = true;
     } else if (!hasZeroLen) {
-      Value check = b.create<arith::CmpIOp>(
+      Value check = arith::CmpIOp::create(b,
           loc, arith::CmpIPredicate::eq,
           getValueOrCreateConstantIndexOp(b, loc, newLength),
           getValueOrCreateConstantIndexOp(b, loc, zero));
       dynHasZeroLenCond =
           dynHasZeroLenCond
-              ? b.create<arith::OrIOp>(loc, check, dynHasZeroLenCond)
+              ? arith::OrIOp::create(b, loc, check, dynHasZeroLenCond)
               : check;
     }
 
@@ -241,7 +241,7 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   auto castResult = [&](Value val) -> Value {
     if (resultType == val.getType())
       return val;
-    return b.create<tensor::CastOp>(loc, resultType, val);
+    return tensor::CastOp::create(b, loc, resultType, val);
   };
 
   // In cases where the original data source is unused: Emit a GenerateOp and
@@ -249,10 +249,10 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // have a dimension of size 0, the semantics of which is unclear.)
   auto createGenerateOp = [&]() {
     // Create GenerateOp.
-    auto generateOp = b.create<tensor::GenerateOp>(
+    auto generateOp = tensor::GenerateOp::create(b,
         loc, resultType, dynDims,
         [&](OpBuilder &builder, Location gLoc, ValueRange indices) {
-          builder.create<tensor::YieldOp>(gLoc, padValue);
+          tensor::YieldOp::create(builder, gLoc, padValue);
         });
     return generateOp;
   };
@@ -261,9 +261,9 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // the result shape of the new SliceOp has a zero dimension.
   auto createPadOfExtractSlice = [&]() {
     // Create pad(extract_slice(x)).
-    auto newSliceOp = b.create<tensor::ExtractSliceOp>(
+    auto newSliceOp = tensor::ExtractSliceOp::create(b,
         loc, padOp.getSource(), newOffsets, newLengths, newStrides);
-    auto newPadOp = b.create<PadOp>(
+    auto newPadOp = PadOp::create(b,
         loc, Type(), newSliceOp, newLows, newHighs,
         /*nofold=*/padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
@@ -291,17 +291,17 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     Operation *thenOp;
     Operation *elseOp;
     Operation *sliceOp;
-    auto result = b.create<scf::IfOp>(
+    auto result = scf::IfOp::create(b,
         loc, dynHasZeroLenCond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
           thenOp = createGenerateOp();
-          b.create<scf::YieldOp>(loc, castResult(thenOp->getResult(0)));
+          scf::YieldOp::create(b, loc, castResult(thenOp->getResult(0)));
         },
         /*elseBuilder=*/
         [&](OpBuilder &b, Location loc) {
           std::tie(elseOp, sliceOp) = createPadOfExtractSlice();
-          b.create<scf::YieldOp>(loc, castResult(elseOp->getResult(0)));
+          scf::YieldOp::create(b, loc, castResult(elseOp->getResult(0)));
         });
     return TilingResult{
         {elseOp}, SmallVector<Value>(result->getResults()), {sliceOp}};
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index 723731b8bed61..ba258402b61b3 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -165,7 +165,7 @@ void transform::TypeConversionCastShapeDynamicDimsOp::
     if (!tensor::CastOp::areCastCompatible(input.getType(), resultType)) {
       return Value();
     }
-    return builder.create<tensor::CastOp>(loc, resultType, input).getResult();
+    return tensor::CastOp::create(builder, loc, resultType, input).getResult();
   });
   converter.addTargetMaterialization([](OpBuilder &builder, Type resultType,
                                         ValueRange inputs,
@@ -177,7 +177,7 @@ void transform::TypeConversionCastShapeDynamicDimsOp::
     if (!tensor::CastOp::areCastCompatible(input.getType(), resultType)) {
       return Value();
     }
-    return builder.create<tensor::CastOp>(loc, resultType, input).getResult();
+    return tensor::CastOp::create(builder, loc, resultType, input).getResult();
   });
 }
 
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index 829b2ab92ac24..381cea7b93e41 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -223,7 +223,7 @@ struct CollapseShapeOpInterface
           MemRefType::get(collapseShapeOp.getSrcType().getShape(),
                           collapseShapeOp.getSrcType().getElementType(),
                           AffineMap(), bufferType.getMemorySpace());
-      buffer = rewriter.create<bufferization::ToBufferOp>(
+      buffer = bufferization::ToBufferOp::create(rewriter,
           op->getLoc(), memrefType, *tensorAlloc);
     }
 
@@ -350,7 +350,7 @@ struct ExpandShapeOpInterface
     if (failed(buffer))
       return failure();
 
-    auto memrefExpandShape = rewriter.create<memref::ExpandShapeOp>(
+    auto memrefExpandShape = memref::ExpandShapeOp::create(rewriter,
         op->getLoc(), tensorResultType.getShape(), *buffer,
         expandShapeOp.getReassociationIndices(),
         expandShapeOp.getMixedOutputShape());
@@ -399,7 +399,7 @@ struct ExtractSliceOpInterface
         extractSliceOp.getResult(), options, state);
     if (failed(resultMemrefType))
       return failure();
-    Value subView = rewriter.create<memref::SubViewOp>(
+    Value subView = memref::SubViewOp::create(rewriter,
         loc, llvm::cast<MemRefType>(*resultMemrefType), *srcMemref,
         mixedOffsets, mixedSizes, mixedStrides);
 
@@ -470,7 +470,7 @@ static void createStores(RewriterBase &rewriter, Location loc, int dim,
   if (dim == static_cast<int>(shape.size()) - 1) {
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
-      rewriter.create<memref::StoreOp>(loc, *elementIt, buffer, indices);
+      memref::StoreOp::create(rewriter, loc, *elementIt, buffer, indices);
       ++elementIt;
     }
     return;
@@ -508,7 +508,7 @@ struct FromElementsOpInterface
         bufferization::getBufferType(*tensorAlloc, options, state);
     if (failed(memrefType))
       return failure();
-    Value buffer = rewriter.create<bufferization::ToBufferOp>(
+    Value buffer = bufferization::ToBufferOp::create(rewriter,
         op->getLoc(), *memrefType, *tensorAlloc);
 
     // Case: tensor<0xelem_type>.
@@ -519,7 +519,7 @@ struct FromElementsOpInterface
 
     // Case: tensor<elem_type>.
     if (shape.empty()) {
-      rewriter.create<memref::StoreOp>(
+      memref::StoreOp::create(rewriter,
           loc, fromElementsOp.getElements().front(), buffer);
       replaceOpWithBufferizedValues(rewriter, op, buffer);
       return success();
@@ -530,7 +530,7 @@ struct FromElementsOpInterface
     SmallVector<Value, 2> constants;
     constants.reserve(maxDim);
     for (int i = 0; i < maxDim; ++i)
-      constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
+      constants.push_back(arith::ConstantIndexOp::create(rewriter, loc, i));
 
     // Traverse all `elements` and create `memref.store` ops.
     auto elementIt = fromElementsOp.getElements().begin();
@@ -577,7 +577,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   // Create linalg::MapOp.
   OpBuilder::InsertionGuard g(rewriter);
   auto linalgOp =
-      rewriter.create<linalg::MapOp>(loc, tensorType, /*inputs=*/ValueRange(),
+      linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
                                      /*init=*/tensorDestination);
   Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
@@ -585,7 +585,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   rewriter.setInsertionPointToStart(&linalgBody);
   SmallVector<Value> indices;
   for (int64_t dim = 0; dim < tensorType.getRank(); ++dim)
-    indices.push_back(rewriter.create<linalg::IndexOp>(loc, dim));
+    indices.push_back(linalg::IndexOp::create(rewriter, loc, dim));
 
   // Move over body.
   rewriter.mergeBlocks(&generateBody.front(), &linalgBody, indices);
@@ -645,7 +645,7 @@ struct InsertOpInterface
         getBuffer(rewriter, insertOp.getDest(), options, state);
     if (failed(destMemref))
       return failure();
-    rewriter.create<memref::StoreOp>(insertOp.getLoc(), insertOp.getScalar(),
+    memref::StoreOp::create(rewriter, insertOp.getLoc(), insertOp.getScalar(),
                                      *destMemref, insertOp.getIndices());
     replaceOpWithBufferizedValues(rewriter, op, *destMemref);
     return success();
@@ -714,7 +714,7 @@ struct InsertSliceOpInterface
         memref::SubViewOp::inferRankReducedResultType(
             insertSliceOp.getSourceType().getShape(), dstMemrefType,
             mixedOffsets, mixedSizes, mixedStrides);
-    Value subView = rewriter.create<memref::SubViewOp>(
+    Value subView = memref::SubViewOp::create(rewriter,
         loc, subviewMemRefType, *dstMemref, mixedOffsets, mixedSizes,
         mixedStrides);
 
@@ -797,13 +797,13 @@ struct PadOpInterface
     for (int64_t i = 0; i < resultType.getRank(); ++i) {
       if (!resultType.isDynamicDim(i))
         continue;
-      Value srcDim = rewriter.create<tensor::DimOp>(loc, padOp.getSource(), i);
+      Value srcDim = tensor::DimOp::create(rewriter, loc, padOp.getSource(), i);
       Value lowPad = toValue(mixedLowPad[i]);
       Value highPad = toValue(mixedHighPad[i]);
       AffineExpr s0, s1, s2;
       bindSymbols(op->getContext(), s0, s1, s2);
       AffineExpr sumExpr = s0 + s1 + s2;
-      Value sum = rewriter.create<affine::AffineApplyOp>(
+      Value sum = affine::AffineApplyOp::create(rewriter,
           loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
       dynamicSizes.push_back(sum);
     }
@@ -996,7 +996,7 @@ struct ParallelInsertSliceOpInterface
             parallelInsertSliceOp.getMixedOffsets(),
             parallelInsertSliceOp.getMixedSizes(),
             parallelInsertSliceOp.getMixedStrides());
-    Value subview = rewriter.create<memref::SubViewOp>(
+    Value subview = memref::SubViewOp::create(rewriter,
         parallelInsertSliceOp.getLoc(), subviewMemRefType, *destBuffer,
         parallelInsertSliceOp.getMixedOffsets(),
         parallelInsertSliceOp.getMixedSizes(),
@@ -1067,13 +1067,13 @@ struct SplatOpInterface
       return op->emitError("memory space not implemented yet");
 
     auto linalgOp =
-        rewriter.create<linalg::MapOp>(loc, tensorType, /*inputs=*/ValueRange(),
+        linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
                                        /*init=*/*tensorAlloc);
     Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
     // Create linalg::IndexOps.
     rewriter.setInsertionPointToStart(&linalgBody);
-    rewriter.create<linalg::YieldOp>(loc, splatOp.getInput());
+    linalg::YieldOp::create(rewriter, loc, splatOp.getInput());
     rewriter.replaceOp(splatOp, linalgOp.getResult()[0]);
 
     return success();
@@ -1127,7 +1127,7 @@ struct ConcatOpInterface
     MemRefType memrefType =
         MemRefType::get(concatOp.getResultType().getShape(),
                         concatOp.getResultType().getElementType(), layout);
-    Value dstBuffer = rewriter.create<bufferization::ToBufferOp>(
+    Value dstBuffer = bufferization::ToBufferOp::create(rewriter,
         op->getLoc(), memrefType, *tensorAlloc);
 
     // Extract the dimension for the concat op
@@ -1143,7 +1143,7 @@ struct ConcatOpInterface
     for (const auto &[dimIdx, dimSize] :
          llvm::enumerate(tensorType.getShape())) {
       if (dimSize == ShapedType::kDynamic) {
-        auto dimOp = rewriter.create<memref::DimOp>(loc, dstBuffer, dimIdx);
+        auto dimOp = memref::DimOp::create(rewriter, loc, dstBuffer, dimIdx);
         sizes.push_back(dimOp.getResult());
         if (dimIdx == concatDim)
           dynamicConcatDim = true;
@@ -1158,7 +1158,7 @@ struct ConcatOpInterface
     if (dynamicConcatDim) {
       // One or more operands have dynamic size, so we must accumulate the
       // offset with arith ops.
-      dynamicOffset = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+      dynamicOffset = arith::ConstantIndexOp::create(rewriter, loc, 0);
     }
 
     for (auto operand : concatOp.getInputs()) {
@@ -1175,7 +1175,7 @@ struct ConcatOpInterface
 
       if (dynamicConcatDim) {
         offsets[concatDim] = dynamicOffset.value();
-        dynamicSize = rewriter.create<memref::DimOp>(loc, *srcBuffer, concatDim)
+        dynamicSize = memref::DimOp::create(rewriter, loc, *srcBuffer, concatDim)
                           .getResult();
         sizes[concatDim] = dynamicSize.value();
       } else {
@@ -1189,7 +1189,7 @@ struct ConcatOpInterface
           memref::SubViewOp::inferRankReducedResultType(
               operandTensorType.getShape(), dstMemrefType, offsets, sizes,
               strides);
-      Value subview = rewriter.create<memref::SubViewOp>(
+      Value subview = memref::SubViewOp::create(rewriter,
           loc, subviewMemRefType, dstBuffer, offsets, sizes, strides);
 
       // Copy the source buffer into the destination subview.
@@ -1197,7 +1197,7 @@ struct ConcatOpInterface
         return failure();
 
       if (dynamicConcatDim) {
-        dynamicOffset = rewriter.create<arith::AddIOp>(
+        dynamicOffset = arith::AddIOp::create(rewriter,
             loc, dynamicOffset.value(), dynamicSize.value());
       } else {
         concatDimOffset += operandConcatDimSize;
diff --git a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
index fa748cf01977f..5b28791126bd7 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
@@ -43,7 +43,7 @@ struct FoldEmptyTensorWithReshapeOp : public OpRewritePattern<ReshapeOp> {
 
     // Create new tensor.empty op.
     // TODO: Do not drop tensor type encoding.
-    Value emptyTensor = rewriter.create<EmptyOp>(
+    Value emptyTensor = EmptyOp::create(rewriter,
         loc, resultShapes[0], reshapeOp.getResultType().getElementType());
     if (emptyTensor.getType() != reshapeOp.getResultType()) {
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
index e0acaee9f6626..50c72c329ed95 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
@@ -55,7 +55,7 @@ static ValueRange invertCollapseShapeIndexing(
   for (int64_t i : reassociation[dim])
     basis.push_back(reshapeSourceShape[i]);
   auto delinearized =
-      b.create<AffineDelinearizeIndexOp>(loc, indexValue, basis);
+      AffineDelinearizeIndexOp::create(b, loc, indexValue, basis);
   return delinearized->getResults();
 }
 
@@ -144,14 +144,14 @@ tensor::ExtractSliceFromCollapseHelper::emitLoopNestBody(
   SmallVector<Range> extractParams =
       helper.getExtractSliceParams(builder.getContext(), multiIndices);
 
-  Value subTileResult = builder.create<tensor::ExtractSliceOp>(
+  Value subTileResult = tensor::ExtractSliceOp::create(builder,
       loc, collapseShapeOp.getSrc(), extractParams);
 
   SmallVector<Range> insertParams =
       helper.getInsertSliceParams(builder.getContext(), tileInductionVars);
 
   // Collapse the dimensions of the source slice back down.
-  Value collapsedResult = builder.create<tensor::CollapseShapeOp>(
+  Value collapsedResult = tensor::CollapseShapeOp::create(builder,
       loc, subTileResult, reassociationIndices);
   return std::make_pair(collapsedResult, insertParams);
 }
@@ -175,7 +175,7 @@ tensor::simplifyCollapseShapeWithRankReducingExtractSlice(
   SmallVector<OpFoldResult> sizes =
       tensor::getMixedSizes(rewriter, op.getLoc(), op.getSrc());
   SmallVector<OpFoldResult> strides(sourceType.getRank(), one);
-  auto sliceOp = rewriter.create<tensor::ExtractSliceOp>(
+  auto sliceOp = tensor::ExtractSliceOp::create(rewriter,
       op.getLoc(), info->sliceResultType, op.getSrc(), offsets, sizes, strides);
 
   if (!info->newReassociationIndices.has_value()) {
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
index a787b485f7162..fd40ece63276e 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
@@ -115,7 +115,7 @@ TransferReadOfExtractSliceOpFolder::matchAndRewriteMaskableOp(
       extractSliceOp.getMixedStrides(), extractSliceOp.getDroppedDims(),
       indices, sourceIndices);
 
-  Operation *newOp = rewriter.create<vector::TransferReadOp>(
+  Operation *newOp = vector::TransferReadOp::create(rewriter,
       readOp.getLoc(), readOp.getVectorType(), extractSliceOp.getSource(),
       sourceIndices,
       AffineMapAttr::get(expandDimsToRank(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
index 4655fa3cf0d23..e3a2854038bdd 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
@@ -65,7 +65,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     return padOp.getResult();
 
   // Create a new tensor::PadOp.
-  auto newPadOp = b.create<PadOp>(
+  auto newPadOp = PadOp::create(b,
       loc, padOp.getResultType(), padOp.getSource(), newMixedLow, newMixedHigh,
       constantPadding, padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
 
@@ -84,7 +84,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
       offsets.push_back(b.getIndexAttr(0));
     } else {
       offsets.push_back(
-          b.create<affine::AffineApplyOp>(
+          affine::AffineApplyOp::create(b,
                loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
                std::initializer_list<Value>{cast<Value>(newMixedLow[i]),
                                             cast<Value>(prevLow)})
@@ -100,7 +100,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     strides.push_back(b.getIndexAttr(1));
   }
 
-  return b.create<ExtractSliceOp>(loc, newPadOp, offsets, sizes, strides)
+  return ExtractSliceOp::create(b, loc, newPadOp, offsets, sizes, strides)
       .getResult();
 }
 
@@ -125,7 +125,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b,
 
   // Create a new tensor::EmptyOp.
   Value newEmptyOp =
-      b.create<EmptyOp>(loc, newSizes, emptyOp.getType().getElementType());
+      EmptyOp::create(b, loc, newSizes, emptyOp.getType().getElementType());
 
   // Create a tensor::ExtractSliceOp.
   SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index 657624b817af2..247ea15640b39 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -77,7 +77,7 @@ struct FoldUnPaddingCollapseIntoExtract
       return rewriter.notifyMatchFailure(collapseShapeOp,
                                          "expected unpadding collapse");
 
-    Value unPaddedExtractSlice = rewriter.create<tensor::ExtractSliceOp>(
+    Value unPaddedExtractSlice = tensor::ExtractSliceOp::create(rewriter,
         extractSliceOp.getLoc(), collapseShapeOp.getResultType(),
         extractSliceOp.getSource(), extractSliceOp.getMixedOffsets(),
         extractSliceOp.getMixedSizes(), extractSliceOp.getMixedStrides());
@@ -271,7 +271,7 @@ struct BubbleUpExpandThroughParallelCollapse
     // matches the number of dimensions of the result, then the expand_shape
     // is a no-op.
     if (newExpandReInds.size() != newExpandSizes.size()) {
-      newCollapseSrc = rewriter.create<tensor::ExpandShapeOp>(
+      newCollapseSrc = tensor::ExpandShapeOp::create(rewriter,
           loc, expandResultType, newCollapseSrc, newExpandReInds,
           newExpandSizes);
     }
@@ -281,7 +281,7 @@ struct BubbleUpExpandThroughParallelCollapse
     // is a no-op.
     Value replacement = newCollapseSrc;
     if (newCollapseReInds.size() != newExpandSizes.size()) {
-      replacement = rewriter.create<tensor::CollapseShapeOp>(
+      replacement = tensor::CollapseShapeOp::create(rewriter,
           loc, newCollapseSrc, newCollapseReInds);
     }
     rewriter.replaceOp(expandOp, replacement);
@@ -406,7 +406,7 @@ struct BubbleUpExpandShapeThroughExtractSlice
         shape, expandShapeOp.getResultType().getElementType());
 
     // Create a new ExtractSliceOp and ExpandShapeOp.
-    Value newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
         collapsedStrides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -736,7 +736,7 @@ struct BubbleUpCollapseShapeThroughExtractSlice
                              groupExpandedOffsets.rend());
     }
 
-    Value newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         collapseShapeOp->getLoc(), collapseShapeOp.getSrc(), expandedOffsets,
         expandedSizes, expandedStrides);
     rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
index 7c9fced540adb..69e649d2eebe8 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
@@ -196,7 +196,7 @@ struct PadOpToConstant final : public OpRewritePattern<PadOp> {
                                          "tensor type not supported");
 
     if (newOp.getType() != resultType)
-      newOp = rewriter.create<tensor::CastOp>(loc, resultType, newOp);
+      newOp = tensor::CastOp::create(rewriter, loc, resultType, newOp);
 
     rewriter.replaceOp(padTensorOp, newOp);
     return success();
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
index 6138821ee8c61..8522c3ed28745 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
@@ -48,12 +48,12 @@ struct CastOpInterface
 
     if (isa<UnrankedTensorType>(srcType)) {
       // Check rank.
-      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
-          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
-      Value isSameRank = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
+      Value isSameRank = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameRank,
           RuntimeVerifiableOpInterface::generateErrorMessage(op,
                                                              "rank mismatch"));
@@ -71,12 +71,12 @@ struct CastOpInterface
         continue;
 
       Value srcDimSz =
-          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+          DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameSz = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameSz = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
@@ -90,9 +90,9 @@ struct DimOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto dimOp = cast<DimOp>(op);
-    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    builder.create<cf::AssertOp>(
+    Value rank = RankOp::create(builder, loc, dimOp.getSource());
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    cf::AssertOp::create(builder,
         loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
@@ -125,7 +125,7 @@ struct ExtractInsertOpInterface
     }
 
     auto indices = extractInsertOp.getIndices();
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     Value assertCond;
     for (auto i : llvm::seq<int64_t>(0, rank)) {
       Value dimOp = builder.createOrFold<tensor::DimOp>(loc, tensor, i);
@@ -135,7 +135,7 @@ struct ExtractInsertOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    builder.create<cf::AssertOp>(
+    cf::AssertOp::create(builder,
         loc, assertCond,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "out-of-bounds access"));
@@ -153,8 +153,8 @@ struct ExtractSliceOpInterface
     // For each dimension, assert that:
     // 0 <= offset < dim_size
     // 0 <= offset + (size - 1) * stride < dim_size
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value one = arith::ConstantIndexOp::create(builder, loc, 1);
     for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
       Value offset = getValueOrCreateConstantIndexOp(
           builder, loc, extractSliceOp.getMixedOffsets()[i]);
@@ -168,20 +168,20 @@ struct ExtractSliceOpInterface
           loc, extractSliceOp.getSource(), i);
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
       // Verify that slice does not run out-of-bounds.
-      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOne = arith::SubIOp::create(builder, loc, size, one);
       Value sizeMinusOneTimesStride =
-          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+          arith::MulIOp::create(builder, loc, sizeMinusOne, stride);
       Value lastPos =
-          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+          arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "extract_slice runs out-of-bounds along dimension " +
diff --git a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
index d50d7c62b789c..21f70e52b8706 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
@@ -59,7 +59,7 @@ struct InsertSliceLikeOpSubsetInsertionOpInterface
   Value buildSubsetExtraction(Operation *op, OpBuilder &builder,
                               Location loc) const {
     auto insertSliceOp = cast<OpTy>(op);
-    auto extractOp = builder.create<tensor::ExtractSliceOp>(
+    auto extractOp = tensor::ExtractSliceOp::create(builder,
         loc, insertSliceOp.getSourceType(), insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
         insertSliceOp.getMixedStrides());
diff --git a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
index 289296a07d9d3..728feab6a6d13 100644
--- a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
@@ -58,7 +58,7 @@ PadOp mlir::tensor::createPadHighOp(RankedTensorType resType, Value source,
     high[idx] = affine::makeComposedFoldedAffineApply(b, loc, d0 - d1,
                                                       {outDim, sourceDim});
   }
-  return b.create<PadOp>(loc, resType, source, low, high, pad, nofold);
+  return PadOp::create(b, loc, resType, source, low, high, pad, nofold);
 }
 
 SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
@@ -69,7 +69,7 @@ SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
   for (const auto &en : llvm::enumerate(tensorTy.getShape())) {
     if (en.value() == ShapedType::kDynamic)
       dynamicDims.push_back(
-          b.create<tensor::DimOp>(loc, rankedTensor, en.index()));
+          tensor::DimOp::create(b, loc, rankedTensor, en.index()));
   }
   return dynamicDims;
 }
@@ -121,7 +121,7 @@ mlir::tensor::dropGivenUnitDims(OpBuilder &b, Location loc, Value src,
     reassocMaps.emplace_back(llvm::make_range(seq.begin(), seq.end()));
     nextDimToGroup = setBit + 1;
   }
-  return b.create<tensor::CollapseShapeOp>(loc, src, reassocMaps);
+  return tensor::CollapseShapeOp::create(b, loc, src, reassocMaps);
 }
 
 bool mlir::tensor::isCastLikeInsertSliceOp(InsertSliceOp op) {
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 1d21096e8920b..477d35fc8d9c5 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -851,7 +851,7 @@ struct PadSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
         getTosaConstShape(rewriter, sliceOp.getLoc(), newPadPaddings);
     auto newPadTy =
         RankedTensorType::get(newPadShape, inputTy.getElementType());
-    auto newPadOp = rewriter.create<tosa::PadOp>(
+    auto newPadOp = tosa::PadOp::create(rewriter,
         padOp.getLoc(), newPadTy, padOp.getInput1(), newPaddingsOp,
         padOp.getPadConst());
 
@@ -903,7 +903,7 @@ struct SliceDynamicSizeCanonicalization
     }
 
     auto size_op = getTosaConstShape(rewriter, sliceOp.getLoc(), sliceSizes);
-    auto newSliceOp = rewriter.create<tosa::SliceOp>(
+    auto newSliceOp = tosa::SliceOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getType(), sliceOp.getInput1(),
         sliceOp.getStart(), size_op);
 
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 5170a11523845..48ae70c51c279 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -182,11 +182,11 @@ Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // Tosa dialect constants only support ElementsAttr unlike standard dialect
   // constant which supports all attributes.
   if (llvm::isa<shapeType>(type) && llvm::isa<DenseIntElementsAttr>(value)) {
-    return builder.create<tosa::ConstShapeOp>(
+    return tosa::ConstShapeOp::create(builder,
         loc, type, llvm::cast<DenseIntElementsAttr>(value));
   }
   if (llvm::isa<ElementsAttr>(value))
-    return builder.create<tosa::ConstOp>(loc, type,
+    return tosa::ConstOp::create(builder, loc, type,
                                          llvm::cast<ElementsAttr>(value));
   return nullptr;
 }
@@ -325,7 +325,7 @@ Value mlir::tosa::createPadConstTensor(OpBuilder &builder, Location loc,
                                    builder.getFloatAttr(srcElemType, val))
           : DenseElementsAttr::get(padConstEType,
                                    builder.getIntegerAttr(srcElemType, val))};
-  return builder.create<tosa::ConstOp>(loc, padConstType, padConstAttr);
+  return tosa::ConstOp::create(builder, loc, padConstType, padConstAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2417,7 +2417,7 @@ LogicalResult TransposeOp::reifyResultShapes(
     int32_t dimInInput = transposePerms[dim];
     if (inputType.isDynamicDim(dimInInput))
       returnedDims[dim] =
-          builder.create<tensor::DimOp>(getLoc(), input, dimInInput)
+          tensor::DimOp::create(builder, getLoc(), input, dimInInput)
               .getResult();
     else
       returnedDims[dim] =
@@ -3948,12 +3948,12 @@ std::optional<Value> mlir::tosa::createZeroPointTensor(OpBuilder &builder,
   if (llvm::isa<FloatType>(srcElemType)) {
     auto zpAttr = DenseElementsAttr::get(
         zpType, builder.getFloatAttr(srcElemType, static_cast<double>(zp)));
-    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+    return tosa::ConstOp::create(builder, loc, zpType, zpAttr);
   }
   if (llvm::isa<IntegerType>(srcElemType)) {
     auto zpAttr =
         DenseElementsAttr::get(zpType, builder.getIntegerAttr(srcElemType, zp));
-    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+    return tosa::ConstOp::create(builder, loc, zpType, zpAttr);
   }
   llvm::errs() << "zero point is not allowed for unsupported data types\n";
   return std::nullopt;
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
index 9b4cf85c480d3..72e3c0590183c 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
@@ -91,12 +91,12 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
 
     if (inputETy != resultETy) {
       inputType = inputType.clone(resultETy);
-      input = rewriter.create<tosa::CastOp>(op.getLoc(), inputType, input);
+      input = tosa::CastOp::create(rewriter, op.getLoc(), inputType, input);
     }
 
     if (weightETy != resultETy) {
       weightType = weightType.clone(resultETy);
-      weight = rewriter.create<tosa::CastOp>(op.getLoc(), weightType, weight);
+      weight = tosa::CastOp::create(rewriter, op.getLoc(), weightType, weight);
     }
 
     if (iZp != 0 || wZp != 0) {
@@ -110,8 +110,8 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         auto zpTy = RankedTensorType::get(shape, ety);
         auto zpAttr =
             DenseElementsAttr::get(zpTy, rewriter.getIntegerAttr(ety, zp));
-        auto zpVal = rewriter.create<tosa::ConstOp>(op.getLoc(), zpTy, zpAttr);
-        return rewriter.create<tosa::SubOp>(op.getLoc(), val.getType(), val,
+        auto zpVal = tosa::ConstOp::create(rewriter, op.getLoc(), zpTy, zpAttr);
+        return tosa::SubOp::create(rewriter, op.getLoc(), val.getType(), val,
                                             zpVal);
       };
 
@@ -139,9 +139,9 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
       auto padTy = RankedTensorType::get({1}, inputETy);
       auto padAttr = DenseElementsAttr::get(padTy, zeroAttr);
       Value padVal =
-          rewriter.create<tosa::ConstOp>(op->getLoc(), padTy, padAttr);
+          tosa::ConstOp::create(rewriter, op->getLoc(), padTy, padAttr);
       inputType = RankedTensorType::get(newShape, inputETy);
-      input = rewriter.create<tosa::PadOp>(op->getLoc(), inputType, input,
+      input = tosa::PadOp::create(rewriter, op->getLoc(), inputType, input,
                                            padSizeVal, padVal);
     }
 
@@ -162,7 +162,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
     auto shiftZeroAttr = DenseElementsAttr::get(
         shiftType, rewriter.getIntegerAttr(shiftElementType, 0));
     Value constZero =
-        rewriter.create<tosa::ConstOp>(op.getLoc(), shiftType, shiftZeroAttr);
+        tosa::ConstOp::create(rewriter, op.getLoc(), shiftType, shiftZeroAttr);
     Value mulValue = rewriter
                          .create<tosa::MulOp>(op.getLoc(), mulShapeType, input,
                                               weight, constZero)
@@ -175,7 +175,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         dyn_cast<RankedTensorType>(input.getType()).getElementType());
     auto outputShapeValue =
         getTosaConstShape(rewriter, op->getLoc(), outputShape);
-    Value outputValue = rewriter.create<tosa::ReshapeOp>(
+    Value outputValue = tosa::ReshapeOp::create(rewriter,
         op.getLoc(), outputShapeType, mulValue, outputShapeValue);
 
     Value bias = op.getBias();
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
index ea6ac981b53cc..12af0ce56a8bc 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
@@ -64,12 +64,12 @@ class TransposeConvNonStridedConverter
     convPad[2] = kernelWidth - 1 + pad[2];
     convPad[3] = kernelWidth - 1 + pad[3];
 
-    auto reverse1 = rewriter.create<tosa::ReverseOp>(
+    auto reverse1 = tosa::ReverseOp::create(rewriter,
         loc, weightTy, weight, /* axis = */ rewriter.getI32IntegerAttr(1));
-    auto reverse2 = rewriter.create<tosa::ReverseOp>(
+    auto reverse2 = tosa::ReverseOp::create(rewriter,
         loc, weightTy, reverse1, /* axis = */ rewriter.getI32IntegerAttr(2));
 
-    Value conv2d = rewriter.create<tosa::Conv2DOp>(
+    Value conv2d = tosa::Conv2DOp::create(rewriter,
         loc, resultTy, input, reverse2, bias, op.getInputZp(), op.getWeightZp(),
         rewriter.getDenseI64ArrayAttr(convPad),
         rewriter.getDenseI64ArrayAttr(stride),
@@ -218,7 +218,7 @@ class TransposeConvStridedConverter
         inputPaddingVal, inputPadConst);
 
     // We use a zero bias as we need to broadcast the bias.
-    auto zeroBias = rewriter.create<tosa::ConstOp>(
+    auto zeroBias = tosa::ConstOp::create(rewriter,
         loc,
         RankedTensorType::get({outputChannels * stride[0] * stride[1]},
                               biasETy),
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index 29ec9f8db2615..9a648015d23df 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -114,7 +114,7 @@ class TypeModificationState {
           OpBuilder builder{value.getContext()};
           builder.setInsertionPointAfter(value.getDefiningOp());
           castValue =
-              builder.create<tensor::CastOp>(value.getLoc(), oldType, value);
+              tensor::CastOp::create(builder, value.getLoc(), oldType, value);
         }
 
         use->set(castValue);
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index 7f85cd52f6bde..51a093bd0cc38 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -405,7 +405,7 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
     return std::nullopt;
   }
   ImplicitLocOpBuilder builder(reshapeOp.getLoc(), rewriter);
-  auto foldedReshape = rewriter.create<ReshapeOp>(
+  auto foldedReshape = ReshapeOp::create(rewriter,
       reshapeOp.getLoc(),
       RankedTensorType::get(applyTOSAPermutation(shape, hoistedPerms),
                             reshapeOutputType.getElementType()),
@@ -425,7 +425,7 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
   if (!maybeNewDenseAttr.has_value())
     return std::nullopt;
   auto newDenseAttr = maybeNewDenseAttr.value();
-  auto newConstOp = rewriter.create<ConstOp>(
+  auto newConstOp = ConstOp::create(rewriter,
       constOp.getLoc(), newDenseAttr.getType(), newDenseAttr);
   return newConstOp->getResult(0);
 }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
index 3b697a2ee3e47..677d8e9904a67 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
@@ -37,7 +37,7 @@ void mlir::tosa::populateTosaTypeConversion(TypeConverter &converter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
   converter.addTargetMaterialization([&](OpBuilder &builder, Type resultType,
@@ -46,7 +46,7 @@ void mlir::tosa::populateTosaTypeConversion(TypeConverter &converter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
 }
diff --git a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
index 9844abcc34cb1..69ef0ba40d72e 100644
--- a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
@@ -33,18 +33,18 @@ mlir::tosa::condenseValues(const SmallVector<Value> &values) {
 
 Value mlir::tosa::clampFloatHelper(Location loc, Value arg, Value min,
                                    Value max, OpBuilder &rewriter) {
-  Value minValue = rewriter.create<arith::MinimumFOp>(loc, arg, max);
-  return rewriter.create<arith::MaximumFOp>(loc, minValue, min);
+  Value minValue = arith::MinimumFOp::create(rewriter, loc, arg, max);
+  return arith::MaximumFOp::create(rewriter, loc, minValue, min);
 }
 
 Value mlir::tosa::clampIntHelper(Location loc, Value arg, Value min, Value max,
                                  OpBuilder &rewriter, bool isUnsigned) {
   if (isUnsigned) {
-    auto minOrArg = rewriter.create<arith::MaxUIOp>(loc, min, arg);
-    return rewriter.create<arith::MinUIOp>(loc, max, minOrArg);
+    auto minOrArg = arith::MaxUIOp::create(rewriter, loc, min, arg);
+    return arith::MinUIOp::create(rewriter, loc, max, minOrArg);
   }
-  auto minOrArg = rewriter.create<arith::MaxSIOp>(loc, min, arg);
-  return rewriter.create<arith::MinSIOp>(loc, max, minOrArg);
+  auto minOrArg = arith::MaxSIOp::create(rewriter, loc, min, arg);
+  return arith::MinSIOp::create(rewriter, loc, max, minOrArg);
 }
 
 bool mlir::tosa::validIntegerRange(IntegerType ty, int64_t value) {
@@ -144,7 +144,7 @@ LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder,
       ArrayRef<int64_t>(reshapeOutputShape), reshapeInputType.getElementType());
   auto reshapeOutputShapeValue = getTosaConstShape(builder, reshapeOutputShape);
 
-  auto reshapeLower = builder.create<tosa::ReshapeOp>(
+  auto reshapeLower = tosa::ReshapeOp::create(builder,
       reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
 
   if (input1Rank > input2Rank) {
@@ -162,7 +162,7 @@ Value mlir::tosa::getTosaConstShape(ImplicitLocOpBuilder &builder,
                                     llvm::ArrayRef<int64_t> shape) {
   auto attr = builder.getIndexTensorAttr(convertFromMlirShape(shape));
   auto type = mlir::tosa::shapeType::get(builder.getContext(), shape.size());
-  mlir::Operation *mlir_op = builder.create<tosa::ConstShapeOp>(type, attr);
+  mlir::Operation *mlir_op = tosa::ConstShapeOp::create(builder, type, attr);
   return mlir_op->getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/UB/IR/UBOps.cpp b/mlir/lib/Dialect/UB/IR/UBOps.cpp
index 5b2cfe7bf4264..ee523f9522953 100644
--- a/mlir/lib/Dialect/UB/IR/UBOps.cpp
+++ b/mlir/lib/Dialect/UB/IR/UBOps.cpp
@@ -52,7 +52,7 @@ void UBDialect::initialize() {
 Operation *UBDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                           Type type, Location loc) {
   if (auto attr = dyn_cast<PoisonAttr>(value))
-    return builder.create<PoisonOp>(loc, type, attr);
+    return PoisonOp::create(builder, loc, type, attr);
 
   return nullptr;
 }
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 14e626a6b23e3..e32b2ca8d76b4 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -125,7 +125,7 @@ static MaskFormat getMaskFormat(Value mask) {
 /// Default callback to build a region with a 'vector.yield' terminator with no
 /// arguments.
 void mlir::vector::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<vector::YieldOp>(loc);
+  vector::YieldOp::create(builder, loc);
 }
 
 // Helper for verifying combining kinds in contractions and reductions.
@@ -597,16 +597,16 @@ struct ElideUnitDimsInMultiDimReduction
         VectorType newMaskType =
             VectorType::get(dstVecType.getShape(), rewriter.getI1Type(),
                             dstVecType.getScalableDims());
-        mask = rewriter.create<vector::ShapeCastOp>(loc, newMaskType, mask);
+        mask = vector::ShapeCastOp::create(rewriter, loc, newMaskType, mask);
       }
-      cast = rewriter.create<vector::ShapeCastOp>(
+      cast = vector::ShapeCastOp::create(rewriter,
           loc, reductionOp.getDestType(), reductionOp.getSource());
     } else {
       // This means we are reducing all the dimensions, and all reduction
       // dimensions are of size 1. So a simple extraction would do.
       if (mask)
-        mask = rewriter.create<vector::ExtractOp>(loc, mask);
-      cast = rewriter.create<vector::ExtractOp>(loc, reductionOp.getSource());
+        mask = vector::ExtractOp::create(rewriter, loc, mask);
+      cast = vector::ExtractOp::create(rewriter, loc, reductionOp.getSource());
     }
 
     Value result =
@@ -673,35 +673,35 @@ Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
   switch (op) {
   case arith::AtomicRMWKind::addf:
   case arith::AtomicRMWKind::addi:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::ADD, vector);
   case arith::AtomicRMWKind::mulf:
   case arith::AtomicRMWKind::muli:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MUL, vector);
   case arith::AtomicRMWKind::minimumf:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINIMUMF, vector);
   case arith::AtomicRMWKind::mins:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINSI, vector);
   case arith::AtomicRMWKind::minu:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINUI, vector);
   case arith::AtomicRMWKind::maximumf:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXIMUMF, vector);
   case arith::AtomicRMWKind::maxs:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXSI, vector);
   case arith::AtomicRMWKind::maxu:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXUI, vector);
   case arith::AtomicRMWKind::andi:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::AND, vector);
   case arith::AtomicRMWKind::ori:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::OR, vector);
   // TODO: Add remaining reduction operations.
   default:
@@ -741,8 +741,8 @@ struct ElideSingleElementReduction : public OpRewritePattern<ReductionOp> {
 
     Location loc = reductionOp.getLoc();
     if (mask)
-      mask = rewriter.create<ExtractOp>(loc, mask);
-    Value result = rewriter.create<ExtractOp>(loc, reductionOp.getVector());
+      mask = ExtractOp::create(rewriter, loc, mask);
+    Value result = ExtractOp::create(rewriter, loc, reductionOp.getVector());
 
     if (Value acc = reductionOp.getAcc())
       result = vector::makeArithReduction(rewriter, loc, reductionOp.getKind(),
@@ -4171,7 +4171,7 @@ class StridedSliceBroadcast final
     // just a single scalar.
     bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
     if (!lowerDimMatch && !isScalarSrc) {
-      source = rewriter.create<ExtractStridedSliceOp>(
+      source = ExtractStridedSliceOp::create(rewriter,
           op->getLoc(), source,
           getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
@@ -4266,7 +4266,7 @@ class ContiguousExtractStridedSliceToExtract final
 
     SmallVector<int64_t> offsets = getI64SubArray(op.getOffsets());
     auto extractOffsets = ArrayRef(offsets).take_front(numOffsets);
-    Value extract = rewriter.create<vector::ExtractOp>(op->getLoc(), source,
+    Value extract = vector::ExtractOp::create(rewriter, op->getLoc(), source,
                                                        extractOffsets);
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(), extract);
     return success();
@@ -4297,7 +4297,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
 
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
         *padding, /*mask=*/Value(), inBoundsAttr);
 }
@@ -4315,7 +4315,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                                 SmallVector<bool>(vectorType.getRank(), false));
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, *padding,
         permutationMapAttr, inBoundsAttr);
 }
@@ -4334,7 +4334,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                                 SmallVector<bool>(vectorType.getRank(), false));
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
         *padding,
         /*mask=*/Value(), inBoundsAttr);
@@ -4859,7 +4859,7 @@ struct TransferReadAfterWriteToBroadcast
     VectorType broadcastedType = VectorType::get(
         broadcastShape, defWrite.getVectorType().getElementType(),
         broadcastScalableFlags);
-    vec = rewriter.create<vector::BroadcastOp>(loc, broadcastedType, vec);
+    vec = vector::BroadcastOp::create(rewriter, loc, broadcastedType, vec);
     SmallVector<int64_t> transposePerm(permutation.begin(), permutation.end());
     rewriter.replaceOpWithNewOp<vector::TransposeOp>(readOp, vec,
                                                      transposePerm);
@@ -5337,11 +5337,11 @@ struct SwapExtractSliceOfTransferWrite
     // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
     // Set all in_bounds to false and let the folder infer them.
     SmallVector<bool> newInBounds(vectorShape.size(), false);
-    auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
         extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
         insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
         insertOp.getMixedStrides());
-    auto newTransferWriteOp = rewriter.create<TransferWriteOp>(
+    auto newTransferWriteOp = TransferWriteOp::create(rewriter,
         transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
         transferOp.getIndices(), transferOp.getPermutationMapAttr(),
         rewriter.getBoolArrayAttr(newInBounds));
@@ -6867,7 +6867,7 @@ void MaskOp::ensureTerminator(Region &region, Builder &builder, Location loc) {
   OpBuilder opBuilder(builder.getContext());
   Operation *maskedOp = &block.front();
   opBuilder.setInsertionPointToEnd(&block);
-  opBuilder.create<vector::YieldOp>(loc, maskedOp->getResults());
+  vector::YieldOp::create(opBuilder, loc, maskedOp->getResults());
 }
 
 LogicalResult MaskOp::verify() {
@@ -7202,7 +7202,7 @@ void mlir::vector::createMaskOpRegion(OpBuilder &builder,
   // Create a block and move the op to that block.
   insBlock->getOperations().splice(
       insBlock->begin(), maskableOp->getBlock()->getOperations(), maskableOp);
-  builder.create<YieldOp>(maskableOp->getLoc(), maskableOp->getResults());
+  YieldOp::create(builder, maskableOp->getLoc(), maskableOp->getResults());
 }
 
 /// Creates a vector.mask operation around a maskable operation. Returns the
@@ -7214,10 +7214,10 @@ Operation *mlir::vector::maskOperation(OpBuilder &builder,
   if (!mask)
     return maskableOp;
   if (passthru)
-    return builder.create<MaskOp>(maskableOp->getLoc(),
+    return MaskOp::create(builder, maskableOp->getLoc(),
                                   maskableOp->getResultTypes(), mask, passthru,
                                   maskableOp, createMaskOpRegion);
-  return builder.create<MaskOp>(maskableOp->getLoc(),
+  return MaskOp::create(builder, maskableOp->getLoc(),
                                 maskableOp->getResultTypes(), mask, maskableOp,
                                 createMaskOpRegion);
 }
@@ -7234,7 +7234,7 @@ Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
   if (!mask)
     return newValue;
 
-  return builder.create<arith::SelectOp>(newValue.getLoc(), newValue.getType(),
+  return arith::SelectOp::create(builder, newValue.getLoc(), newValue.getType(),
                                          mask, newValue, passthru);
 }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
index 9da051150e409..c750017a6b504 100644
--- a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -116,7 +116,7 @@ struct TransferWriteOpInterface
         getBuffer(rewriter, writeOp.getBase(), options, state);
     if (failed(resultBuffer))
       return failure();
-    rewriter.create<vector::TransferWriteOp>(
+    vector::TransferWriteOp::create(rewriter,
         writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
         writeOp.getIndices(), writeOp.getPermutationMapAttr(),
         writeOp.getMask(), writeOp.getInBoundsAttr());
@@ -241,7 +241,7 @@ struct MaskOpInterface
     // Create a new vector.mask op.
     ValueRange newYieldedValuesRange(newYieldedValues);
     TypeRange newResultTypes(newYieldedValuesRange);
-    auto newOp = rewriter.create<vector::MaskOp>(
+    auto newOp = vector::MaskOp::create(rewriter,
         op->getLoc(), newResultTypes, maskOp.getMask(), maskOp.getPassthru(),
         /*maskableOp=*/nullptr,
         /*maskRegionBuilder=*/[](OpBuilder &b, Operation *) {});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
index 89930a6bd35fa..4c3a04cfb5bfa 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
@@ -64,14 +64,14 @@ class UnrollBitCastOp final : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(shape, resultType.getElementType(), scalableDims);
 
     Location loc = op.getLoc();
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
     for (auto position : *unrollIterator) {
       Value extract =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+          vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       Value bitcast =
-          rewriter.create<vector::BitCastOp>(loc, bitcastResType, extract);
+          vector::BitCastOp::create(rewriter, loc, bitcastResType, extract);
       result =
-          rewriter.create<vector::InsertOp>(loc, bitcast, result, position);
+          vector::InsertOp::create(rewriter, loc, bitcast, result, position);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
index 11dcfe421e0c4..cb8e566869cfd 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
@@ -52,7 +52,7 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
 
     // Stretching scalar inside vector (e.g. vector<1xf32>) can use splat.
     if (srcRank <= 1 && dstRank == 1) {
-      Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource());
+      Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource());
       rewriter.replaceOpWithNewOp<vector::SplatOp>(op, dstType, ext);
       return success();
     }
@@ -70,10 +70,10 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
       // Duplication.
       VectorType resType = VectorType::Builder(dstType).dropDim(0);
       Value bcst =
-          rewriter.create<vector::BroadcastOp>(loc, resType, op.getSource());
-      Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
+          vector::BroadcastOp::create(rewriter, loc, resType, op.getSource());
+      Value result = ub::PoisonOp::create(rewriter, loc, dstType);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
       rewriter.replaceOp(op, result);
       return success();
     }
@@ -111,13 +111,13 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType =
         VectorType::get(dstType.getShape().drop_front(), eltType,
                         dstType.getScalableDims().drop_front());
-    Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
+    Value result = ub::PoisonOp::create(rewriter, loc, dstType);
     if (m == 0) {
       // Stetch at start.
-      Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), 0);
-      Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
+      Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource(), 0);
+      Value bcst = vector::BroadcastOp::create(rewriter, loc, resType, ext);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
     } else {
       // Stetch not at start.
       if (dstType.getScalableDims()[0]) {
@@ -125,9 +125,9 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
         return failure();
       }
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d) {
-        Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), d);
-        Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource(), d);
+        Value bcst = vector::BroadcastOp::create(rewriter, loc, resType, ext);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
       }
     }
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
index c6627b5ec0d77..b2486c5cbbcf8 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
@@ -90,17 +90,17 @@ static Value reshapeLoad(Location loc, Value val, VectorType type,
 
   // At extraction dimension?
   if (index == 0)
-    return rewriter.create<vector::ExtractOp>(loc, val, pos);
+    return vector::ExtractOp::create(rewriter, loc, val, pos);
 
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   VectorType resType = VectorType::Builder(type).dropDim(index);
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, resType, rewriter.getZeroAttr(resType));
   for (int64_t d = 0, e = resType.getDimSize(0); d < e; d++) {
-    Value ext = rewriter.create<vector::ExtractOp>(loc, val, d);
+    Value ext = vector::ExtractOp::create(rewriter, loc, val, d);
     Value load = reshapeLoad(loc, ext, vType, index - 1, pos, rewriter);
-    result = rewriter.create<vector::InsertOp>(loc, load, result, d);
+    result = vector::InsertOp::create(rewriter, loc, load, result, d);
   }
   return result;
 }
@@ -115,15 +115,15 @@ static Value reshapeStore(Location loc, Value val, Value result,
     return val;
   // At insertion dimension?
   if (index == 0)
-    return rewriter.create<vector::InsertOp>(loc, val, result, pos);
+    return vector::InsertOp::create(rewriter, loc, val, result, pos);
 
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   for (int64_t d = 0, e = type.getDimSize(0); d < e; d++) {
-    Value ext = rewriter.create<vector::ExtractOp>(loc, result, d);
-    Value ins = rewriter.create<vector::ExtractOp>(loc, val, d);
+    Value ext = vector::ExtractOp::create(rewriter, loc, result, d);
+    Value ins = vector::ExtractOp::create(rewriter, loc, val, d);
     Value sto = reshapeStore(loc, ins, ext, vType, index - 1, pos, rewriter);
-    result = rewriter.create<vector::InsertOp>(loc, sto, result, d);
+    result = vector::InsertOp::create(rewriter, loc, sto, result, d);
   }
   return result;
 }
@@ -141,7 +141,7 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
         kind == CombiningKind::MINIMUMF || kind == CombiningKind::MAXIMUMF)
       // Only valid for floating point types.
       return std::nullopt;
-    mul = rewriter.create<arith::MulIOp>(loc, x, y);
+    mul = arith::MulIOp::create(rewriter, loc, x, y);
   } else {
     // Float case.
     if (kind == CombiningKind::AND || kind == CombiningKind::MINUI ||
@@ -152,14 +152,14 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
       return std::nullopt;
     // Special case for fused multiply-add.
     if (acc && isa<VectorType>(acc.getType()) && kind == CombiningKind::ADD) {
-      Value fma = rewriter.create<vector::FMAOp>(loc, x, y, acc);
+      Value fma = vector::FMAOp::create(rewriter, loc, x, y, acc);
       if (mask)
         // The fma op doesn't need explicit masking. However, fma ops used in
         // reductions must preserve previous 'acc' values for masked-out lanes.
         fma = selectPassthru(rewriter, mask, fma, acc);
       return fma;
     }
-    mul = rewriter.create<arith::MulFOp>(loc, x, y);
+    mul = arith::MulFOp::create(rewriter, loc, x, y);
   }
 
   if (!acc)
@@ -195,8 +195,8 @@ static std::optional<unsigned> getDimPosition(AffineMap map, unsigned dim) {
 static Value createAdd(Location loc, Value x, Value y, bool isInt,
                        PatternRewriter &rewriter) {
   if (isInt)
-    return rewriter.create<arith::AddIOp>(loc, x, y);
-  return rewriter.create<arith::AddFOp>(loc, x, y);
+    return arith::AddIOp::create(rewriter, loc, x, y);
+  return arith::AddFOp::create(rewriter, loc, x, y);
 }
 
 /// Creates a MulIOp if `isInt` is true otherwise create an MulFOp using
@@ -204,8 +204,8 @@ static Value createAdd(Location loc, Value x, Value y, bool isInt,
 static Value createMul(Location loc, Value x, Value y, bool isInt,
                        PatternRewriter &rewriter) {
   if (isInt)
-    return rewriter.create<arith::MulIOp>(loc, x, y);
-  return rewriter.create<arith::MulFOp>(loc, x, y);
+    return arith::MulIOp::create(rewriter, loc, x, y);
+  return arith::MulFOp::create(rewriter, loc, x, y);
 }
 
 namespace {
@@ -411,7 +411,7 @@ struct UnrolledOuterProductGenerator
   Value t(Value v, ArrayRef<int64_t> perm = {1, 0}) {
     if (!v)
       return v;
-    return rewriter.create<vector::TransposeOp>(loc, v, perm);
+    return vector::TransposeOp::create(rewriter, loc, v, perm);
   }
 
   Value promote(Value v, Type dstElementType) {
@@ -425,8 +425,8 @@ struct UnrolledOuterProductGenerator
     if (vecType)
       promotedType = vecType.clone(promotedType);
     if (isa<FloatType>(dstElementType))
-      return rewriter.create<arith::ExtFOp>(loc, promotedType, v);
-    return rewriter.create<arith::ExtSIOp>(loc, promotedType, v);
+      return arith::ExtFOp::create(rewriter, loc, promotedType, v);
+    return arith::ExtSIOp::create(rewriter, loc, promotedType, v);
   }
 
   FailureOr<Value> outerProd(Value lhs, Value rhs, Value res,
@@ -438,16 +438,16 @@ struct UnrolledOuterProductGenerator
 
     Type resElementType = cast<VectorType>(res.getType()).getElementType();
     for (int64_t k = 0; k < reductionSize; ++k) {
-      Value extractA = rewriter.create<vector::ExtractOp>(loc, lhs, k);
-      Value extractB = rewriter.create<vector::ExtractOp>(loc, rhs, k);
+      Value extractA = vector::ExtractOp::create(rewriter, loc, lhs, k);
+      Value extractB = vector::ExtractOp::create(rewriter, loc, rhs, k);
       extractA = promote(extractA, resElementType);
       extractB = promote(extractB, resElementType);
       Value extractMask;
       if (maybeMask.has_value() && maybeMask.value())
         extractMask =
-            rewriter.create<vector::ExtractOp>(loc, maybeMask.value(), k);
+            vector::ExtractOp::create(rewriter, loc, maybeMask.value(), k);
 
-      Operation *outerProdOp = rewriter.create<vector::OuterProductOp>(
+      Operation *outerProdOp = vector::OuterProductOp::create(rewriter,
           loc, res.getType(), extractA, extractB, res, kind);
       res = maskOperation(rewriter, outerProdOp, extractMask)->getResult(0);
     }
@@ -698,28 +698,28 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
     // Two outer parallel, one inner reduction (matmat flavor).
     //
     if (maps == infer({{m, k}, {k, n}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{m, k}, {n, k}, {m, n}})) {
       // No need to permute anything.
     } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
       // This is the classical row-major matmul. Just permute the lhs.
       Value tmp = lhs;
-      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
       rhs = tmp;
     } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
       std::swap(lhs, rhs);
     } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
       Value tmp = lhs;
-      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      rhs = rewriter.create<vector::TransposeOp>(loc, tmp, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, tmp, perm);
     } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
       Value tmp = rhs;
-      rhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
       lhs = tmp;
     } else {
       return failure();
@@ -732,12 +732,12 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
     if (maps == infer({{m, n}, {n}, {m}})) {
       // No need to permute anything.
     } else if (maps == infer({{n, m}, {n}, {m}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{n}, {m, n}, {m}})) {
       std::swap(lhs, rhs);
     } else if (maps == infer({{n}, {n, m}, {m}})) {
       std::swap(lhs, rhs);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else {
       return failure();
     }
@@ -754,31 +754,31 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
   unsigned dstColumns = rank == 1 ? 1 : dstType.getShape()[1];
 
   // ExtractOp does not allow dynamic indexing, we must unroll explicitly.
-  Value res = rewriter.create<arith::ConstantOp>(loc, dstType,
+  Value res = arith::ConstantOp::create(rewriter, loc, dstType,
                                                  rewriter.getZeroAttr(dstType));
   bool isInt = isa<IntegerType>(dstType.getElementType());
   llvm::SmallVector<Value> extractedCols;
   extractedCols.reserve(dstColumns);
   for (unsigned r = 0; r < dstRows; ++r) {
-    Value rowLhs = rewriter.create<vector::ExtractOp>(op.getLoc(), lhs, r);
+    Value rowLhs = vector::ExtractOp::create(rewriter, op.getLoc(), lhs, r);
     for (unsigned c = 0; c < dstColumns; ++c) {
       // Extract each respective row and column of the LHS and RHS once to
       // avoid having duplicate SSA values pointing to the same rows/columns.
       if (r == 0) {
         Value colRhs =
             rank == 1 ? rhs
-                      : rewriter.create<vector::ExtractOp>(op.getLoc(), rhs, c);
+                      : vector::ExtractOp::create(rewriter, op.getLoc(), rhs, c);
         extractedCols.push_back(colRhs);
       }
       Value extractedColRhs = extractedCols[c];
       Value product =
           createMul(op.getLoc(), rowLhs, extractedColRhs, isInt, rewriter);
-      Value sum = rewriter.create<vector::ReductionOp>(
+      Value sum = vector::ReductionOp::create(rewriter,
           op.getLoc(), vector::CombiningKind::ADD, product);
 
       SmallVector<int64_t, 2> pos = rank == 1 ? SmallVector<int64_t, 2>{r}
                                               : SmallVector<int64_t, 2>{r, c};
-      res = rewriter.create<vector::InsertOp>(op.getLoc(), sum, res, pos);
+      res = vector::InsertOp::create(rewriter, op.getLoc(), sum, res, pos);
     }
   }
   if (auto acc = op.getAcc())
@@ -879,21 +879,21 @@ struct ContractOpToElementwise
       lhsDims.append(lhsShape.begin(), lhsShape.end());
       auto expandedType =
           VectorType::get(lhsDims, contractOp.getLhsType().getElementType());
-      newLhs = rewriter.create<vector::BroadcastOp>(loc, expandedType, newLhs);
+      newLhs = vector::BroadcastOp::create(rewriter, loc, expandedType, newLhs);
     }
     if (!rhsDims.empty()) {
       rhsDims.append(rhsShape.begin(), rhsShape.end());
       auto expandedType =
           VectorType::get(rhsDims, contractOp.getRhsType().getElementType());
-      newRhs = rewriter.create<vector::BroadcastOp>(loc, expandedType, newRhs);
+      newRhs = vector::BroadcastOp::create(rewriter, loc, expandedType, newRhs);
     }
     bool isInt = contractOp.getLhsType().getElementType().isIntOrIndex();
-    newLhs = rewriter.create<vector::TransposeOp>(loc, newLhs, lhsTranspose);
-    newRhs = rewriter.create<vector::TransposeOp>(loc, newRhs, rhsTranspose);
+    newLhs = vector::TransposeOp::create(rewriter, loc, newLhs, lhsTranspose);
+    newRhs = vector::TransposeOp::create(rewriter, loc, newRhs, rhsTranspose);
     SmallVector<int64_t> lhsOffsets(lhsReductionDims.size(), 0);
     SmallVector<int64_t> rhsOffsets(rhsReductionDims.size(), 0);
-    newLhs = rewriter.create<vector::ExtractOp>(loc, newLhs, lhsOffsets);
-    newRhs = rewriter.create<vector::ExtractOp>(loc, newRhs, rhsOffsets);
+    newLhs = vector::ExtractOp::create(rewriter, loc, newLhs, lhsOffsets);
+    newRhs = vector::ExtractOp::create(rewriter, loc, newRhs, rhsOffsets);
     std::optional<Value> result =
         createContractArithOp(loc, newLhs, newRhs, contractOp.getAcc(),
                               contractOp.getKind(), rewriter, isInt);
@@ -1097,7 +1097,7 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       rewriter.getArrayAttr(adjustIter(op.getIteratorTypes(), iterIndex));
   // Unroll into a series of lower dimensional vector.contract ops.
   Location loc = op.getLoc();
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, resType, rewriter.getZeroAttr(resType));
 
   for (int64_t d = 0; d < dimSize; ++d) {
@@ -1110,7 +1110,7 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       lowMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *lowContract = rewriter.create<vector::ContractionOp>(
+    Operation *lowContract = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, acc, lowAffine, lowIter);
     lowContract = maskOperation(rewriter, lowContract, lowMask);
     result = reshapeStore(loc, lowContract->getResult(0), result, resType,
@@ -1161,8 +1161,8 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
 
     Value acc = op.getAcc();
     Operation *reductionOp =
-        acc ? rewriter.create<vector::ReductionOp>(loc, kind, m, acc)
-            : rewriter.create<vector::ReductionOp>(loc, kind, m);
+        acc ? vector::ReductionOp::create(rewriter, loc, kind, m, acc)
+            : vector::ReductionOp::create(rewriter, loc, kind, m);
     return maskOperation(rewriter, reductionOp, mask)->getResult(0);
   }
   // Construct new iterator types and affine map array attribute.
@@ -1186,7 +1186,7 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
       newMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *newContract = rewriter.create<vector::ContractionOp>(
+    Operation *newContract = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, result, lowAffine, lowIter);
     result = maskOperation(rewriter, newContract, newMask)->getResult(0);
   }
@@ -1240,7 +1240,7 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
 
     if (!rhsType) {
       // Special case: AXPY operation.
-      Value b = rewriter.create<vector::BroadcastOp>(loc, lhsType, op.getRhs());
+      Value b = vector::BroadcastOp::create(rewriter, loc, lhsType, op.getRhs());
       std::optional<Value> mult = createContractArithOp(
           loc, op.getLhs(), b, acc, kind, rewriter, isInt, mask);
       if (!mult.has_value())
@@ -1249,23 +1249,23 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
       return success();
     }
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
     for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
-      Value x = rewriter.create<vector::ExtractOp>(loc, op.getLhs(), d);
-      Value a = rewriter.create<vector::BroadcastOp>(loc, rhsType, x);
+      Value x = vector::ExtractOp::create(rewriter, loc, op.getLhs(), d);
+      Value a = vector::BroadcastOp::create(rewriter, loc, rhsType, x);
       Value r = nullptr;
       if (acc)
-        r = rewriter.create<vector::ExtractOp>(loc, acc, d);
+        r = vector::ExtractOp::create(rewriter, loc, acc, d);
       Value extrMask;
       if (mask)
-        extrMask = rewriter.create<vector::ExtractOp>(loc, mask, d);
+        extrMask = vector::ExtractOp::create(rewriter, loc, mask, d);
 
       std::optional<Value> m = createContractArithOp(
           loc, a, op.getRhs(), r, kind, rewriter, isInt, extrMask);
       if (!m.has_value())
         return failure();
-      result = rewriter.create<vector::InsertOp>(loc, *m, result, d);
+      result = vector::InsertOp::create(rewriter, loc, *m, result, d);
     }
 
     rewriter.replaceOp(rootOp, result);
@@ -1335,7 +1335,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   Value lhs = op.getLhs();
   auto lhsMap = op.getIndexingMapsArray()[0];
   if (lhsMap == AffineMap::get(3, 0, {k, m}, ctx))
-    lhs = rew.create<vector::TransposeOp>(loc, lhs, ArrayRef<int64_t>{1, 0});
+    lhs = vector::TransposeOp::create(rew, loc, lhs, ArrayRef<int64_t>{1, 0});
   else if (lhsMap != AffineMap::get(3, 0, {m, k}, ctx))
     return failure();
 
@@ -1343,7 +1343,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   Value rhs = op.getRhs();
   auto rhsMap = op.getIndexingMapsArray()[1];
   if (rhsMap == AffineMap::get(3, 0, {n, k}, ctx))
-    rhs = rew.create<vector::TransposeOp>(loc, rhs, ArrayRef<int64_t>{1, 0});
+    rhs = vector::TransposeOp::create(rew, loc, rhs, ArrayRef<int64_t>{1, 0});
   else if (rhsMap != AffineMap::get(3, 0, {k, n}, ctx))
     return failure();
 
@@ -1356,15 +1356,15 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
 
   Type flattenedLHSType =
       VectorType::get(lhsType.getNumElements(), lhsType.getElementType());
-  lhs = rew.create<vector::ShapeCastOp>(loc, flattenedLHSType, lhs);
+  lhs = vector::ShapeCastOp::create(rew, loc, flattenedLHSType, lhs);
 
   Type flattenedRHSType =
       VectorType::get(rhsType.getNumElements(), rhsType.getElementType());
-  rhs = rew.create<vector::ShapeCastOp>(loc, flattenedRHSType, rhs);
+  rhs = vector::ShapeCastOp::create(rew, loc, flattenedRHSType, rhs);
 
-  Value mul = rew.create<vector::MatmulOp>(loc, lhs, rhs, lhsRows, lhsColumns,
+  Value mul = vector::MatmulOp::create(rew, loc, lhs, rhs, lhsRows, lhsColumns,
                                            rhsColumns);
-  mul = rew.create<vector::ShapeCastOp>(
+  mul = vector::ShapeCastOp::create(rew,
       loc,
       VectorType::get({lhsRows, rhsColumns},
                       getElementTypeOrSelf(op.getAcc().getType())),
@@ -1373,15 +1373,15 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   // ACC must be C(m, n) or C(n, m).
   auto accMap = op.getIndexingMapsArray()[2];
   if (accMap == AffineMap::get(3, 0, {n, m}, ctx))
-    mul = rew.create<vector::TransposeOp>(loc, mul, ArrayRef<int64_t>{1, 0});
+    mul = vector::TransposeOp::create(rew, loc, mul, ArrayRef<int64_t>{1, 0});
   else if (accMap != AffineMap::get(3, 0, {m, n}, ctx))
     llvm_unreachable("invalid contraction semantics");
 
   Value res =
       isa<IntegerType>(elementType)
-          ? static_cast<Value>(rew.create<arith::AddIOp>(loc, op.getAcc(), mul))
+          ? static_cast<Value>(arith::AddIOp::create(rew, loc, op.getAcc(), mul))
           : static_cast<Value>(
-                rew.create<arith::AddFOp>(loc, op.getAcc(), mul));
+                arith::AddFOp::create(rew, loc, op.getAcc(), mul));
 
   return res;
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index 3000204c8ce17..c586059ca55bc 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -75,7 +75,7 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
     Value maskVec = op.getMask();
     Value passThruVec = op.getPassThru();
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultTy, rewriter.getZeroAttr(resultTy));
 
     VectorType subTy = VectorType::Builder(resultTy).dropDim(0);
@@ -84,16 +84,16 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
       int64_t thisIdx[1] = {i};
 
       Value indexSubVec =
-          rewriter.create<vector::ExtractOp>(loc, indexVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
       Value maskSubVec =
-          rewriter.create<vector::ExtractOp>(loc, maskVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, maskVec, thisIdx);
       Value passThruSubVec =
-          rewriter.create<vector::ExtractOp>(loc, passThruVec, thisIdx);
-      Value subGather = rewriter.create<vector::GatherOp>(
+          vector::ExtractOp::create(rewriter, loc, passThruVec, thisIdx);
+      Value subGather = vector::GatherOp::create(rewriter,
           loc, subTy, op.getBase(), op.getIndices(), indexSubVec, maskSubVec,
           passThruSubVec);
       result =
-          rewriter.create<vector::InsertOp>(loc, subGather, result, thisIdx);
+          vector::InsertOp::create(rewriter, loc, subGather, result, thisIdx);
     }
 
     rewriter.replaceOp(op, result);
@@ -159,22 +159,22 @@ struct RemoveStrideFromGatherSource : OpRewritePattern<vector::GatherOp> {
 
     // 1. Collapse the input memref so that it's "flat".
     SmallVector<ReassociationIndices> reassoc = {{0, 1}};
-    Value collapsed = rewriter.create<memref::CollapseShapeOp>(
+    Value collapsed = memref::CollapseShapeOp::create(rewriter,
         op.getLoc(), subview.getSource(), reassoc);
 
     // 2. Generate new gather indices that will model the
     // strided access.
     IntegerAttr stride = rewriter.getIndexAttr(srcTrailingDim);
     VectorType vType = op.getIndexVec().getType();
-    Value mulCst = rewriter.create<arith::ConstantOp>(
+    Value mulCst = arith::ConstantOp::create(rewriter,
         op.getLoc(), vType, DenseElementsAttr::get(vType, stride));
 
     Value newIdxs =
-        rewriter.create<arith::MulIOp>(op.getLoc(), op.getIndexVec(), mulCst);
+        arith::MulIOp::create(rewriter, op.getLoc(), op.getIndexVec(), mulCst);
 
     // 3. Create an updated gather op with the collapsed input memref and the
     // updated indices.
-    Value newGather = rewriter.create<vector::GatherOp>(
+    Value newGather = vector::GatherOp::create(rewriter,
         op.getLoc(), op.getResult().getType(), collapsed, op.getIndices(),
         newIdxs, op.getMask(), op.getPassThru());
     rewriter.replaceOp(op, newGather);
@@ -229,8 +229,8 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
     for (int64_t i = 0, e = resultTy.getNumElements(); i < e; ++i) {
       int64_t thisIdx[1] = {i};
       Value condition =
-          rewriter.create<vector::ExtractOp>(loc, condMask, thisIdx);
-      Value index = rewriter.create<vector::ExtractOp>(loc, indexVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, condMask, thisIdx);
+      Value index = vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
       baseOffsets.back() =
           rewriter.createOrFold<arith::AddIOp>(loc, lastBaseOffset, index);
 
@@ -240,19 +240,19 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
           // `vector.load` does not support scalar result; emit a vector load
           // and extract the single result instead.
           Value load =
-              b.create<vector::LoadOp>(loc, elemVecTy, base, baseOffsets);
+              vector::LoadOp::create(b, loc, elemVecTy, base, baseOffsets);
           int64_t zeroIdx[1] = {0};
-          extracted = b.create<vector::ExtractOp>(loc, load, zeroIdx);
+          extracted = vector::ExtractOp::create(b, loc, load, zeroIdx);
         } else {
-          extracted = b.create<tensor::ExtractOp>(loc, base, baseOffsets);
+          extracted = tensor::ExtractOp::create(b, loc, base, baseOffsets);
         }
 
         Value newResult =
-            b.create<vector::InsertOp>(loc, extracted, result, thisIdx);
-        b.create<scf::YieldOp>(loc, newResult);
+            vector::InsertOp::create(b, loc, extracted, result, thisIdx);
+        scf::YieldOp::create(b, loc, newResult);
       };
       auto passThruBuilder = [result](OpBuilder &b, Location loc) {
-        b.create<scf::YieldOp>(loc, result);
+        scf::YieldOp::create(b, loc, result);
       };
 
       result =
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index cab0f213b14a9..b0affbb699d9e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -60,14 +60,14 @@ class UnrollInterleaveOp final : public OpRewritePattern<vector::InterleaveOp> {
       return failure();
 
     auto loc = op.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
     for (auto position : *unrollIterator) {
-      Value extractLhs = rewriter.create<ExtractOp>(loc, op.getLhs(), position);
-      Value extractRhs = rewriter.create<ExtractOp>(loc, op.getRhs(), position);
+      Value extractLhs = ExtractOp::create(rewriter, loc, op.getLhs(), position);
+      Value extractRhs = ExtractOp::create(rewriter, loc, op.getRhs(), position);
       Value interleave =
-          rewriter.create<InterleaveOp>(loc, extractLhs, extractRhs);
-      result = rewriter.create<InsertOp>(loc, interleave, result, position);
+          InterleaveOp::create(rewriter, loc, extractLhs, extractRhs);
+      result = InsertOp::create(rewriter, loc, interleave, result, position);
     }
 
     rewriter.replaceOp(op, result);
@@ -123,19 +123,19 @@ class UnrollDeinterleaveOp final
       return failure();
 
     auto loc = op.getLoc();
-    Value emptyResult = rewriter.create<arith::ConstantOp>(
+    Value emptyResult = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
     Value evenResult = emptyResult;
     Value oddResult = emptyResult;
 
     for (auto position : *unrollIterator) {
       auto extractSrc =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+          vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       auto deinterleave =
-          rewriter.create<vector::DeinterleaveOp>(loc, extractSrc);
-      evenResult = rewriter.create<vector::InsertOp>(
+          vector::DeinterleaveOp::create(rewriter, loc, extractSrc);
+      evenResult = vector::InsertOp::create(rewriter,
           loc, deinterleave.getRes1(), evenResult, position);
-      oddResult = rewriter.create<vector::InsertOp>(loc, deinterleave.getRes2(),
+      oddResult = vector::InsertOp::create(rewriter, loc, deinterleave.getRes2(),
                                                     oddResult, position);
     }
     rewriter.replaceOp(op, ValueRange{evenResult, oddResult});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
index ba21092d2af3c..edd840d8fbc7b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
@@ -67,19 +67,19 @@ class CreateMaskOpLowering : public OpRewritePattern<vector::CreateMaskOp> {
     Value idx = op.getOperand(0);
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = rewriter.create<vector::CreateMaskOp>(
+    Value trueVal = vector::CreateMaskOp::create(rewriter,
         loc, lowType, op.getOperands().drop_front());
-    Value falseVal = rewriter.create<arith::ConstantOp>(
+    Value falseVal = arith::ConstantOp::create(rewriter,
         loc, lowType, rewriter.getZeroAttr(lowType));
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstType, rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < dim; d++) {
       Value bnd =
-          rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(d));
-      Value val = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+          arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(d));
+      Value val = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                  bnd, idx);
-      Value sel = rewriter.create<arith::SelectOp>(loc, val, trueVal, falseVal);
-      result = rewriter.create<vector::InsertOp>(loc, sel, result, d);
+      Value sel = arith::SelectOp::create(rewriter, loc, val, trueVal, falseVal);
+      result = vector::InsertOp::create(rewriter, loc, sel, result, d);
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -146,12 +146,12 @@ class ConstantMaskOpLowering : public OpRewritePattern<vector::ConstantMaskOp> {
           op, "Cannot unroll leading scalable dim in dstType");
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = rewriter.create<vector::ConstantMaskOp>(
+    Value trueVal = vector::ConstantMaskOp::create(rewriter,
         loc, lowType, dimSizes.drop_front());
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstType, rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < trueDimSize; d++)
-      result = rewriter.create<vector::InsertOp>(loc, trueVal, result, d);
+      result = vector::InsertOp::create(rewriter, loc, trueVal, result, d);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -261,7 +261,7 @@ struct MaskedGatherOpPattern : public MaskOpRewritePattern<GatherOp> {
                             PatternRewriter &rewriter) const override {
     Value passthru = maskingOp.hasPassthru()
                          ? maskingOp.getPassthru()
-                         : rewriter.create<arith::ConstantOp>(
+                         : arith::ConstantOp::create(rewriter,
                                gatherOp.getLoc(),
                                rewriter.getZeroAttr(gatherOp.getVectorType()));
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
index ce524b259d8d4..c5631ba00bbc6 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
@@ -103,12 +103,12 @@ class InnerOuterDimReductionConversion
     // If masked, transpose the original mask.
     Value transposedMask;
     if (maskableOp.isMasked()) {
-      transposedMask = rewriter.create<vector::TransposeOp>(
+      transposedMask = vector::TransposeOp::create(rewriter,
           loc, maskableOp.getMaskingOp().getMask(), indices);
     }
 
     // Transpose reduction source.
-    auto transposeOp = rewriter.create<vector::TransposeOp>(loc, src, indices);
+    auto transposeOp = vector::TransposeOp::create(rewriter, loc, src, indices);
     SmallVector<bool> reductionMask(srcRank, false);
     for (int i = 0; i < reductionSize; ++i) {
       if (useInnerDimsForReduction)
@@ -117,7 +117,7 @@ class InnerOuterDimReductionConversion
         reductionMask[i] = true;
     }
 
-    Operation *newMultiRedOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newMultiRedOp = vector::MultiDimReductionOp::create(rewriter,
         multiReductionOp.getLoc(), transposeOp.getResult(),
         multiReductionOp.getAcc(), reductionMask, multiReductionOp.getKind());
     newMultiRedOp =
@@ -256,13 +256,13 @@ class ReduceMultiDimReductionRank
           vectorShape,
           llvm::cast<VectorType>(vectorMask.getType()).getElementType());
       newVectorMask =
-          rewriter.create<vector::ShapeCastOp>(loc, maskCastedType, vectorMask);
+          vector::ShapeCastOp::create(rewriter, loc, maskCastedType, vectorMask);
     }
 
     auto castedType = VectorType::get(
         vectorShape, multiReductionOp.getSourceVectorType().getElementType(),
         scalableDims);
-    Value cast = rewriter.create<vector::ShapeCastOp>(
+    Value cast = vector::ShapeCastOp::create(rewriter,
         loc, castedType, multiReductionOp.getSource());
 
     Value acc = multiReductionOp.getAcc();
@@ -271,11 +271,11 @@ class ReduceMultiDimReductionRank
           {flattenedParallelDim},
           multiReductionOp.getSourceVectorType().getElementType(),
           /*scalableDims=*/{isParallelDimScalable});
-      acc = rewriter.create<vector::ShapeCastOp>(loc, accType, acc);
+      acc = vector::ShapeCastOp::create(rewriter, loc, accType, acc);
     }
     // 6. Creates the flattened form of vector.multi_reduction with inner/outer
     // most dim as reduction.
-    Operation *newMultiDimRedOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(rewriter,
         loc, cast, acc, mask, multiReductionOp.getKind());
     newMultiDimRedOp =
         mlir::vector::maskOperation(rewriter, newMultiDimRedOp, newVectorMask);
@@ -339,11 +339,11 @@ struct TwoDimMultiReductionToElementWise
 
     Value result = multiReductionOp.getAcc();
     for (int64_t i = 0; i < srcShape[0]; i++) {
-      auto operand = rewriter.create<vector::ExtractOp>(
+      auto operand = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getSource(), i);
       Value extractMask = nullptr;
       if (mask) {
-        extractMask = rewriter.create<vector::ExtractOp>(loc, mask, i);
+        extractMask = vector::ExtractOp::create(rewriter, loc, mask, i);
       }
       result =
           makeArithReduction(rewriter, loc, multiReductionOp.getKind(), operand,
@@ -383,27 +383,27 @@ struct TwoDimMultiReductionToReduction
     }
 
     auto loc = multiReductionOp.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, multiReductionOp.getDestType(),
         rewriter.getZeroAttr(multiReductionOp.getDestType()));
     int outerDim = multiReductionOp.getSourceVectorType().getShape()[0];
 
     for (int i = 0; i < outerDim; ++i) {
-      auto v = rewriter.create<vector::ExtractOp>(
+      auto v = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
-      auto acc = rewriter.create<vector::ExtractOp>(
+      auto acc = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getAcc(), ArrayRef<int64_t>{i});
-      Operation *reductionOp = rewriter.create<vector::ReductionOp>(
+      Operation *reductionOp = vector::ReductionOp::create(rewriter,
           loc, multiReductionOp.getKind(), v, acc);
 
       // If masked, slice the mask and mask the new reduction operation.
       if (maskableOp.isMasked()) {
-        Value mask = rewriter.create<vector::ExtractOp>(
+        Value mask = vector::ExtractOp::create(rewriter,
             loc, maskableOp.getMaskingOp().getMask(), ArrayRef<int64_t>{i});
         reductionOp = mlir::vector::maskOperation(rewriter, reductionOp, mask);
       }
 
-      result = rewriter.create<vector::InsertOp>(loc, reductionOp->getResult(0),
+      result = vector::InsertOp::create(rewriter, loc, reductionOp->getResult(0),
                                                  result, i);
     }
 
@@ -459,9 +459,9 @@ struct OneDimMultiReductionToTwoDim
     SmallVector<bool, 2> reductionMask{false, true};
 
     /// vector.extract(vector.multi_reduce(vector.shape_cast(v, 1xk)), 0)
-    Value cast = rewriter.create<vector::ShapeCastOp>(
+    Value cast = vector::ShapeCastOp::create(rewriter,
         loc, castedType, multiReductionOp.getSource());
-    Value castAcc = rewriter.create<vector::BroadcastOp>(
+    Value castAcc = vector::BroadcastOp::create(rewriter,
         loc, accType, multiReductionOp.getAcc());
     Value castMask;
     if (maskableOp.isMasked()) {
@@ -470,10 +470,10 @@ struct OneDimMultiReductionToTwoDim
           ArrayRef<int64_t>{1, maskType.getShape().back()},
           maskType.getElementType(),
           ArrayRef<bool>{false, maskType.getScalableDims().back()});
-      castMask = rewriter.create<vector::BroadcastOp>(loc, castMaskType, mask);
+      castMask = vector::BroadcastOp::create(rewriter, loc, castMaskType, mask);
     }
 
-    Operation *newOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newOp = vector::MultiDimReductionOp::create(rewriter,
         loc, cast, castAcc, reductionMask, multiReductionOp.getKind());
     newOp = vector::maskOperation(rewriter, newOp, castMask);
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
index a1f67bd0e9ed3..a6985b96b59df 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
@@ -122,7 +122,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
       return failure();
 
     VectorType resType = VectorType::get(destShape, elType);
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
     int64_t reductionDim = scanOp.getReductionDim();
     bool inclusive = scanOp.getInclusive();
@@ -144,7 +144,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
     for (int i = 0; i < destShape[reductionDim]; i++) {
       offsets[reductionDim] = i;
       ArrayAttr scanOffsets = rewriter.getI64ArrayAttr(offsets);
-      Value input = rewriter.create<vector::ExtractStridedSliceOp>(
+      Value input = vector::ExtractStridedSliceOp::create(rewriter,
           loc, reductionType, scanOp.getSource(), scanOffsets, scanSizes,
           scanStrides);
       Value output;
@@ -154,10 +154,10 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         } else {
           if (initialValueRank == 0) {
             // ShapeCastOp cannot handle 0-D vectors
-            output = rewriter.create<vector::BroadcastOp>(
+            output = vector::BroadcastOp::create(rewriter,
                 loc, input.getType(), scanOp.getInitialValue());
           } else {
-            output = rewriter.create<vector::ShapeCastOp>(
+            output = vector::ShapeCastOp::create(rewriter,
                 loc, input.getType(), scanOp.getInitialValue());
           }
         }
@@ -166,7 +166,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         output = vector::makeArithReduction(rewriter, loc, scanOp.getKind(),
                                             lastOutput, y);
       }
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = vector::InsertStridedSliceOp::create(rewriter,
           loc, output, result, offsets, strides);
       lastOutput = output;
       lastInput = input;
@@ -174,11 +174,11 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
 
     Value reduction;
     if (initialValueRank == 0) {
-      Value v = rewriter.create<vector::ExtractOp>(loc, lastOutput, 0);
+      Value v = vector::ExtractOp::create(rewriter, loc, lastOutput, 0);
       reduction =
-          rewriter.create<vector::BroadcastOp>(loc, initialValueType, v);
+          vector::BroadcastOp::create(rewriter, loc, initialValueType, v);
     } else {
-      reduction = rewriter.create<vector::ShapeCastOp>(loc, initialValueType,
+      reduction = vector::ShapeCastOp::create(rewriter, loc, initialValueType,
                                                        lastOutput);
     }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
index 39c16fab21c4e..69d3666a13879 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
@@ -137,10 +137,10 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     const int64_t resultLeading = delta > 0 ? 0 : -delta;
 
     const Value source = shapeCast.getSource();
-    const Value poison = rewriter.create<ub::PoisonOp>(loc, resultType);
-    const Value extracted = rewriter.create<vector::ExtractOp>(
+    const Value poison = ub::PoisonOp::create(rewriter, loc, resultType);
+    const Value extracted = vector::ExtractOp::create(rewriter,
         loc, source, SmallVector<int64_t>(sourceLeading, 0));
-    const Value result = rewriter.create<vector::InsertOp>(
+    const Value result = vector::InsertOp::create(rewriter,
         loc, extracted, poison, SmallVector<int64_t>(resultLeading, 0));
 
     rewriter.replaceOp(shapeCast, result);
@@ -171,13 +171,13 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 
     SmallVector<int64_t> extractIndex(sourceDim, 0);
     SmallVector<int64_t> insertIndex(resultDim, 0);
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
 
     for (int i = 0; i < nSlices; ++i) {
       Value extracted =
-          rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+          vector::ExtractOp::create(rewriter, loc, source, extractIndex);
 
-      result = rewriter.create<vector::InsertOp>(loc, extracted, result,
+      result = vector::InsertOp::create(rewriter, loc, extracted, result,
                                                  insertIndex);
 
       inplaceAdd(1, sourceShape.take_front(sourceDim), extractIndex);
@@ -276,9 +276,9 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     Value extracted = {};
     Value extractedStrided = {};
     Value insertedSlice = {};
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
     const Value partResult =
-        rewriter.create<ub::PoisonOp>(loc, insertStridedType);
+        ub::PoisonOp::create(rewriter, loc, insertStridedType);
 
     for (size_t i = 0; i < nAtomicSlices; ++i) {
 
@@ -288,14 +288,14 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
       // vector.extract
       if (extractStridedPhase == 0) {
         extracted =
-            rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+            vector::ExtractOp::create(rewriter, loc, source, extractIndex);
         inplaceAdd(1, sourceShape.take_front(sourceSuffixStartDim),
                    extractIndex);
       }
 
       // vector.extract_strided_slice
       extractOffsets[0] = extractStridedPhase * greatestCommonDivisor;
-      extractedStrided = rewriter.create<vector::ExtractStridedSliceOp>(
+      extractedStrided = vector::ExtractStridedSliceOp::create(rewriter,
           loc, extracted, extractOffsets, atomicShape, sizes);
 
       // vector.insert_strided_slice
@@ -303,12 +303,12 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
         insertedSlice = partResult;
       }
       insertOffsets[0] = insertStridedPhase * greatestCommonDivisor;
-      insertedSlice = rewriter.create<vector::InsertStridedSliceOp>(
+      insertedSlice = vector::InsertStridedSliceOp::create(rewriter,
           loc, extractedStrided, insertedSlice, insertOffsets, sizes);
 
       // vector.insert
       if (insertStridedPhase + 1 == insertPeriod) {
-        result = rewriter.create<vector::InsertOp>(loc, insertedSlice, result,
+        result = vector::InsertOp::create(rewriter, loc, insertedSlice, result,
                                                    insertIndex);
         inplaceAdd(1, resultType.getShape().take_front(resultSuffixStartDim),
                    insertIndex);
@@ -394,7 +394,7 @@ class ScalableShapeCastOpRewritePattern
     auto extractionVectorType = VectorType::get(
         {minExtractionSize}, sourceVectorType.getElementType(), {true});
 
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultVectorType);
     SmallVector<int64_t> srcIdx(srcRank, 0);
     SmallVector<int64_t> resIdx(resRank, 0);
 
@@ -406,7 +406,7 @@ class ScalableShapeCastOpRewritePattern
       // 1. Extract a scalable subvector from the source vector.
       if (!currentSourceScalableVector) {
         if (srcRank != 1) {
-          currentSourceScalableVector = rewriter.create<vector::ExtractOp>(
+          currentSourceScalableVector = vector::ExtractOp::create(rewriter,
               loc, op.getSource(), llvm::ArrayRef(srcIdx).drop_back());
         } else {
           currentSourceScalableVector = op.getSource();
@@ -414,7 +414,7 @@ class ScalableShapeCastOpRewritePattern
       }
       Value sourceSubVector = currentSourceScalableVector;
       if (minExtractionSize < minSourceTrailingSize) {
-        sourceSubVector = rewriter.create<vector::ScalableExtractOp>(
+        sourceSubVector = vector::ScalableExtractOp::create(rewriter,
             loc, extractionVectorType, sourceSubVector, srcIdx.back());
       }
 
@@ -423,14 +423,14 @@ class ScalableShapeCastOpRewritePattern
         if (minExtractionSize == minResultTrailingSize) {
           currentResultScalableVector = sourceSubVector;
         } else if (resRank != 1) {
-          currentResultScalableVector = rewriter.create<vector::ExtractOp>(
+          currentResultScalableVector = vector::ExtractOp::create(rewriter,
               loc, result, llvm::ArrayRef(resIdx).drop_back());
         } else {
           currentResultScalableVector = result;
         }
       }
       if (minExtractionSize < minResultTrailingSize) {
-        currentResultScalableVector = rewriter.create<vector::ScalableInsertOp>(
+        currentResultScalableVector = vector::ScalableInsertOp::create(rewriter,
             loc, sourceSubVector, currentResultScalableVector, resIdx.back());
       }
 
@@ -439,7 +439,7 @@ class ScalableShapeCastOpRewritePattern
           currentResultScalableVector != result) {
         // Finished row of result. Insert complete scalable vector into result
         // (n-D) vector.
-        result = rewriter.create<vector::InsertOp>(
+        result = vector::InsertOp::create(rewriter,
             loc, currentResultScalableVector, result,
             llvm::ArrayRef(resIdx).drop_back());
         currentResultScalableVector = {};
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
index 5b81d0d33d484..fe02cd5656571 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
@@ -47,7 +47,7 @@ static Value extendVectorRank(OpBuilder &builder, Location loc, Value vec,
                          originalVecType.getScalableDims().end());
   VectorType newVecType = VectorType::get(
       newShape, originalVecType.getElementType(), newScalableDims);
-  return builder.create<vector::BroadcastOp>(loc, newVecType, vec);
+  return vector::BroadcastOp::create(builder, loc, newVecType, vec);
 }
 
 /// Extend the rank of a vector Value by `addedRanks` by adding inner unit
@@ -62,7 +62,7 @@ static Value extendMaskRank(OpBuilder &builder, Location loc, Value vec,
     permutation.push_back(i);
   for (int64_t i = 0; i < addedRank; ++i)
     permutation.push_back(i);
-  return builder.create<vector::TransposeOp>(loc, broadcasted, permutation);
+  return vector::TransposeOp::create(builder, loc, broadcasted, permutation);
 }
 
 //===----------------------------------------------------------------------===//
@@ -138,7 +138,7 @@ struct TransferReadPermutationLowering
     // Generate new transfer_read operation.
     VectorType newReadType = VectorType::get(
         newVectorShape, op.getVectorType().getElementType(), newScalableDims);
-    Value newRead = rewriter.create<vector::TransferReadOp>(
+    Value newRead = vector::TransferReadOp::create(rewriter,
         op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
@@ -209,11 +209,11 @@ struct TransferWritePermutationLowering
         inverseTransposeInBoundsAttr(rewriter, op.getInBounds(), permutation);
 
     // Generate new transfer_write operation.
-    Value newVec = rewriter.create<vector::TransposeOp>(
+    Value newVec = vector::TransposeOp::create(rewriter,
         op.getLoc(), op.getVector(), indices);
     auto newMap = AffineMap::getMinorIdentityMap(
         map.getNumDims(), map.getNumResults(), rewriter.getContext());
-    auto newWrite = rewriter.create<vector::TransferWriteOp>(
+    auto newWrite = vector::TransferWriteOp::create(rewriter,
         op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getMask(), newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
@@ -299,7 +299,7 @@ struct TransferWriteNonPermutationLowering
       newInBoundsValues.push_back(op.isDimInBounds(i));
     }
     ArrayAttr newInBoundsAttr = rewriter.getBoolArrayAttr(newInBoundsValues);
-    auto newWrite = rewriter.create<vector::TransferWriteOp>(
+    auto newWrite = vector::TransferWriteOp::create(rewriter,
         op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), newMask, newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
@@ -370,7 +370,7 @@ struct TransferOpReduceRank
             ? rewriter.getArrayAttr(
                   op.getInBoundsAttr().getValue().take_back(reducedShapeRank))
             : ArrayAttr();
-    Value newRead = rewriter.create<vector::TransferReadOp>(
+    Value newRead = vector::TransferReadOp::create(rewriter,
         op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
@@ -471,20 +471,20 @@ struct TransferReadToVectorLoadLowering
             read, "vector type is not rank 1, can't create masked load, needs "
                   "VectorToSCF");
 
-      Value fill = rewriter.create<vector::SplatOp>(
+      Value fill = vector::SplatOp::create(rewriter,
           read.getLoc(), unbroadcastedVectorType, read.getPadding());
-      res = rewriter.create<vector::MaskedLoadOp>(
+      res = vector::MaskedLoadOp::create(rewriter,
           read.getLoc(), unbroadcastedVectorType, read.getBase(),
           read.getIndices(), read.getMask(), fill);
     } else {
-      res = rewriter.create<vector::LoadOp>(read.getLoc(),
+      res = vector::LoadOp::create(rewriter, read.getLoc(),
                                             unbroadcastedVectorType,
                                             read.getBase(), read.getIndices());
     }
 
     // Insert a broadcasting op if required.
     if (!broadcastedDims.empty())
-      res = rewriter.create<vector::BroadcastOp>(
+      res = vector::BroadcastOp::create(rewriter,
           read.getLoc(), read.getVectorType(), res->getResult(0));
     return res->getResult(0);
   }
@@ -569,11 +569,11 @@ struct TransferWriteToVectorStoreLowering
                    << write;
             });
 
-      rewriter.create<vector::MaskedStoreOp>(
+      vector::MaskedStoreOp::create(rewriter,
           write.getLoc(), write.getBase(), write.getIndices(), write.getMask(),
           write.getVector());
     } else {
-      rewriter.create<vector::StoreOp>(write.getLoc(), write.getVector(),
+      vector::StoreOp::create(rewriter, write.getLoc(), write.getVector(),
                                        write.getBase(), write.getIndices());
     }
     // There's no return value for StoreOps. Use Value() to signal success to
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index 732e316c93381..e9591bfcf6feb 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -80,7 +80,7 @@ getUnpackShufflePermFor128Lane(ArrayRef<int64_t> vals, int numBits) {
 static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({0, 1, numElem, numElem + 1}, numBits));
 }
@@ -94,7 +94,7 @@ static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({2, 3, numElem + 2, numElem + 3},
                                      numBits));
@@ -109,7 +109,7 @@ static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  auto shuffle = b.create<vector::ShuffleOp>(
+  auto shuffle = vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({0, numElem, 1, numElem + 1}, numBits));
   return shuffle;
@@ -124,7 +124,7 @@ static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({2, numElem + 2, 3, numElem + 3},
                                      numBits));
@@ -181,7 +181,7 @@ static Value create4x128BitSuffle(ImplicitLocOpBuilder &b, Value v1, Value v2,
   appendToMask(0, b23);
   appendToMask(16, b45);
   appendToMask(16, b67);
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// Lowers the value to a vector.shuffle op. The `source` is expected to be a
@@ -192,7 +192,7 @@ static Value transposeToShuffle1D(OpBuilder &b, Value source, int m, int n) {
   for (int64_t j = 0; j < n; ++j)
     for (int64_t i = 0; i < m; ++i)
       mask.push_back(i * n + j);
-  return b.create<vector::ShuffleOp>(source.getLoc(), source, source, mask);
+  return vector::ShuffleOp::create(b, source.getLoc(), source, source, mask);
 }
 
 /// Lowers the value to a sequence of vector.shuffle ops. The `source` is
@@ -284,9 +284,9 @@ static Value transposeToShuffle16x16(OpBuilder &builder, Value source, int m,
 
   auto reshInputType = VectorType::get(
       {m, n}, cast<VectorType>(source.getType()).getElementType());
-  Value res = b.create<ub::PoisonOp>(reshInputType);
+  Value res = ub::PoisonOp::create(b, reshInputType);
   for (int64_t i = 0; i < m; ++i)
-    res = b.create<vector::InsertOp>(vs[i], res, i);
+    res = vector::InsertOp::create(b, vs[i], res, i);
   return res;
 }
 
@@ -335,10 +335,10 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
       Type flattenedType =
           VectorType::get(resType.getNumElements(), resType.getElementType());
       auto matrix =
-          rewriter.create<vector::ShapeCastOp>(loc, flattenedType, input);
+          vector::ShapeCastOp::create(rewriter, loc, flattenedType, input);
       auto rows = rewriter.getI32IntegerAttr(resType.getShape()[0]);
       auto columns = rewriter.getI32IntegerAttr(resType.getShape()[1]);
-      Value trans = rewriter.create<vector::FlatTransposeOp>(
+      Value trans = vector::FlatTransposeOp::create(rewriter,
           loc, flattenedType, matrix, rows, columns);
       rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, trans);
       return success();
@@ -359,7 +359,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
     // of the leftmost transposed dimensions. We traverse every transpose
     // element using a linearized index that we delinearize to generate the
     // appropriate indices for the extract/insert operations.
-    Value result = rewriter.create<ub::PoisonOp>(loc, resType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resType);
     int64_t numTransposedElements = ShapedType::getNumElements(prunedInShape);
 
     for (int64_t linearIdx = 0; linearIdx < numTransposedElements;
@@ -482,14 +482,14 @@ class TransposeOp2DToShuffleLowering
     Location loc = op.getLoc();
     auto flattenedType = VectorType::get({n * m}, srcType.getElementType());
     auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
-    auto reshInput = rewriter.create<vector::ShapeCastOp>(loc, flattenedType,
+    auto reshInput = vector::ShapeCastOp::create(rewriter, loc, flattenedType,
                                                           op.getVector());
 
     Value res;
     if (vectorTransposeLowering == VectorTransposeLowering::Shuffle16x16 &&
         m == 16 && n == 16) {
       reshInput =
-          rewriter.create<vector::ShapeCastOp>(loc, reshInputType, reshInput);
+          vector::ShapeCastOp::create(rewriter, loc, reshInputType, reshInput);
       res = transposeToShuffle16x16(rewriter, reshInput, m, n);
     } else {
       // Fallback to shuffle on 1D approach.
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index fb99e22c77ea0..b555c3f6d9f85 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -114,7 +114,7 @@ struct DistributedLoadStoreHelper {
            "preregistered sequential value.");
     // Scalar case can directly use memref.store.
     if (!isa<VectorType>(val.getType()))
-      return b.create<memref::StoreOp>(loc, val, buffer, zero);
+      return memref::StoreOp::create(b, loc, val, buffer, zero);
 
     // Vector case must use vector::TransferWriteOp which will later lower to
     //   vector.store of memref.store depending on further lowerings.
@@ -127,7 +127,7 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return b.create<vector::TransferWriteOp>(
+    return vector::TransferWriteOp::create(b,
         loc, val, buffer, indices,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
@@ -156,7 +156,7 @@ struct DistributedLoadStoreHelper {
 
     // Scalar case can directly use memref.store.
     if (!isa<VectorType>(type))
-      return b.create<memref::LoadOp>(loc, buffer, zero);
+      return memref::LoadOp::create(b, loc, buffer, zero);
 
     // Other cases must be vector atm.
     // Vector case must use vector::TransferReadOp which will later lower to
@@ -172,7 +172,7 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return b.create<vector::TransferReadOp>(
+    return vector::TransferReadOp::create(b,
         loc, cast<VectorType>(type), buffer, indices, /*padding=*/std::nullopt,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
@@ -243,10 +243,10 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
     rewriter.setInsertionPoint(warpOp);
 
     // Step 1: Create scf.if op.
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    Value isLane0 = rewriter.create<arith::CmpIOp>(
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    Value isLane0 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, warpOp.getLaneid(), c0);
-    auto ifOp = rewriter.create<scf::IfOp>(loc, isLane0,
+    auto ifOp = scf::IfOp::create(rewriter, loc, isLane0,
                                            /*withElseRegion=*/false);
     rewriter.eraseOp(ifOp.thenBlock()->getTerminator());
 
@@ -325,7 +325,7 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
     // Step 7. Delete terminator and add empty scf.yield.
     rewriter.eraseOp(yieldOp);
     rewriter.setInsertionPointToEnd(ifOp.thenBlock());
-    rewriter.create<scf::YieldOp>(yieldLoc);
+    scf::YieldOp::create(rewriter, yieldLoc);
 
     // Compute replacements for WarpOp results.
     rewriter.replaceOp(warpOp, replacements);
@@ -512,7 +512,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
 
     // Create a second warp op that contains only writeOp.
-    auto secondWarpOp = rewriter.create<WarpExecuteOnLane0Op>(
+    auto secondWarpOp = WarpExecuteOnLane0Op::create(rewriter,
         loc, TypeRange(), newWarpOp.getLaneid(), newWarpOp.getWarpSize());
     Block &body = secondWarpOp.getBodyRegion().front();
     rewriter.setInsertionPointToStart(&body);
@@ -521,7 +521,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     newWriteOp.getValueToStoreMutable().assign(
         newWarpOp.getResult(newRetIndices[0]));
     rewriter.eraseOp(writeOp);
-    rewriter.create<gpu::YieldOp>(newWarpOp.getLoc());
+    gpu::YieldOp::create(rewriter, newWarpOp.getLoc());
     return success();
   }
 
@@ -698,7 +698,7 @@ struct WarpOpConstant : public WarpDistributionPattern {
         cast<ShapedType>(warpOp.getResult(operandIndex).getType()), scalarAttr);
     Location loc = warpOp.getLoc();
     rewriter.setInsertionPointAfter(warpOp);
-    Value distConstant = rewriter.create<arith::ConstantOp>(loc, newAttr);
+    Value distConstant = arith::ConstantOp::create(rewriter, loc, newAttr);
     rewriter.replaceAllUsesWith(warpOp.getResult(operandIndex), distConstant);
     rewriter.finalizeOpModification(warpOp);
     return success();
@@ -823,7 +823,7 @@ struct WarpOpTransferRead : public WarpDistributionPattern {
     Value newMask =
         hasMask ? newWarpOp.getResult(newRetIndices[newRetIndices.size() - 1])
                 : Value();
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         read.getLoc(), distributedVal.getType(), read.getBase(), newIndices,
         read.getPermutationMapAttr(), newPadding, newMask,
         read.getInBoundsAttr());
@@ -965,7 +965,7 @@ struct WarpOpBroadcast : public WarpDistributionPattern {
     WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
         rewriter, warpOp, {broadcastSrc}, {broadcastSrcType}, newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value broadcasted = rewriter.create<vector::BroadcastOp>(
+    Value broadcasted = vector::BroadcastOp::create(rewriter,
         loc, destVecType, newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 broadcasted);
@@ -1008,7 +1008,7 @@ struct WarpOpShapeCast : public WarpDistributionPattern {
         rewriter, warpOp, {oldCastOp.getSource()}, {castDistributedType},
         newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value newCast = rewriter.create<vector::ShapeCastOp>(
+    Value newCast = vector::ShapeCastOp::create(rewriter,
         oldCastOp.getLoc(), castResultType,
         newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newCast);
@@ -1091,7 +1091,7 @@ struct WarpOpCreateMask : public WarpDistributionPattern {
     }
 
     auto newMask =
-        rewriter.create<vector::CreateMaskOp>(loc, distType, newOperands);
+        vector::CreateMaskOp::create(rewriter, loc, distType, newOperands);
     rewriter.replaceAllUsesWith(warpOp.getResult(operandIndex), newMask);
     rewriter.finalizeOpModification(warpOp);
     return success();
@@ -1182,7 +1182,7 @@ struct WarpOpInsertStridedSlice : public WarpDistributionPattern {
     Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
     // Create a new insert strided slice op that inserts distributed source into
     // distributed dest.
-    Value newInsert = rewriter.create<vector::InsertStridedSliceOp>(
+    Value newInsert = vector::InsertStridedSliceOp::create(rewriter,
         insertOp.getLoc(), distributedDest.getType(), distributedSource,
         distributedDest, insertOp.getOffsets(), insertOp.getStrides());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newInsert);
@@ -1277,7 +1277,7 @@ struct WarpOpExtractStridedSlice : public WarpDistributionPattern {
     // Create a new extract strided slice op that extracts from the
     // distributed vector.
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
-    Value newExtract = rewriter.create<vector::ExtractStridedSliceOp>(
+    Value newExtract = vector::ExtractStridedSliceOp::create(rewriter,
         extractOp.getLoc(), distributedType, distributedVec,
         extractOp.getOffsets(),
         ArrayAttr::get(rewriter.getContext(), distributedSizes),
@@ -1323,7 +1323,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
       // Extract from distributed vector.
-      Value newExtract = rewriter.create<vector::ExtractOp>(
+      Value newExtract = vector::ExtractOp::create(rewriter,
           loc, distributedVec, extractOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
@@ -1352,7 +1352,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
     // Extract from distributed vector.
-    Value newExtract = rewriter.create<vector::ExtractOp>(
+    Value newExtract = vector::ExtractOp::create(rewriter,
         loc, distributedVec, extractOp.getMixedPosition());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 newExtract);
@@ -1422,7 +1422,7 @@ struct WarpOpExtractScalar : public WarpDistributionPattern {
       Value newExtract;
       SmallVector<int64_t> indices(extractSrcType.getRank(), 0);
       newExtract =
-          rewriter.create<vector::ExtractOp>(loc, distributedVec, indices);
+          vector::ExtractOp::create(rewriter, loc, distributedVec, indices);
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
       return success();
@@ -1442,11 +1442,11 @@ struct WarpOpExtractScalar : public WarpDistributionPattern {
     // Extract at position: pos % elementsPerLane
     Value newPos =
         elementsPerLane == 1
-            ? rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult()
+            ? arith::ConstantIndexOp::create(rewriter, loc, 0).getResult()
             : affine::makeComposedAffineApply(rewriter, loc,
                                               sym0 % elementsPerLane, pos);
     Value extracted =
-        rewriter.create<vector::ExtractOp>(loc, distributedVec, newPos);
+        vector::ExtractOp::create(rewriter, loc, distributedVec, newPos);
 
     // Shuffle the extracted value to all lanes.
     Value shuffled = warpShuffleFromIdxFn(
@@ -1535,7 +1535,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
       if (pos) {
         indices.push_back(pos);
       }
-      newInsert = rewriter.create<vector::InsertOp>(loc, newSource,
+      newInsert = vector::InsertOp::create(rewriter, loc, newSource,
                                                     distributedVec, indices);
       // Broadcast: Simply move the vector.insert op out.
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
@@ -1552,7 +1552,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
     // Insert position: pos % elementsPerLane
     OpFoldResult newPos = affine::makeComposedFoldedAffineApply(
         rewriter, loc, sym0 % elementsPerLane, pos);
-    Value isInsertingLane = rewriter.create<arith::CmpIOp>(
+    Value isInsertingLane = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
     Value newResult =
         rewriter
@@ -1560,13 +1560,13 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
                 loc, isInsertingLane,
                 /*thenBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  Value newInsert = builder.create<vector::InsertOp>(
+                  Value newInsert = vector::InsertOp::create(builder,
                       loc, newSource, distributedVec, newPos);
-                  builder.create<scf::YieldOp>(loc, newInsert);
+                  scf::YieldOp::create(builder, loc, newInsert);
                 },
                 /*elseBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  builder.create<scf::YieldOp>(loc, distributedVec);
+                  scf::YieldOp::create(builder, loc, distributedVec);
                 })
             .getResult(0);
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newResult);
@@ -1603,7 +1603,7 @@ struct WarpOpInsert : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedSrc = newWarpOp->getResult(newRetIndices[0]);
       Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
-      Value newResult = rewriter.create<vector::InsertOp>(
+      Value newResult = vector::InsertOp::create(rewriter,
           loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newResult);
@@ -1653,7 +1653,7 @@ struct WarpOpInsert : public WarpDistributionPattern {
     Value newResult;
     if (distrSrcDim >= 0) {
       // Every lane inserts a small piece.
-      newResult = rewriter.create<vector::InsertOp>(
+      newResult = vector::InsertOp::create(rewriter,
           loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
     } else {
       // One lane inserts the entire source vector.
@@ -1661,19 +1661,19 @@ struct WarpOpInsert : public WarpDistributionPattern {
       SmallVector<OpFoldResult> pos = insertOp.getMixedPosition();
       SmallVector<int64_t> newPos = getAsIntegers(pos);
       // tid of inserting lane: pos / elementsPerLane
-      Value insertingLane = rewriter.create<arith::ConstantIndexOp>(
+      Value insertingLane = arith::ConstantIndexOp::create(rewriter,
           loc, newPos[distrDestDim] / elementsPerLane);
-      Value isInsertingLane = rewriter.create<arith::CmpIOp>(
+      Value isInsertingLane = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
       // Insert position: pos % elementsPerLane
       newPos[distrDestDim] %= elementsPerLane;
       auto insertingBuilder = [&](OpBuilder &builder, Location loc) {
-        Value newInsert = builder.create<vector::InsertOp>(
+        Value newInsert = vector::InsertOp::create(builder,
             loc, distributedSrc, distributedDest, newPos);
-        builder.create<scf::YieldOp>(loc, newInsert);
+        scf::YieldOp::create(builder, loc, newInsert);
       };
       auto nonInsertingBuilder = [&](OpBuilder &builder, Location loc) {
-        builder.create<scf::YieldOp>(loc, distributedDest);
+        scf::YieldOp::create(builder, loc, distributedDest);
       };
       newResult = rewriter
                       .create<scf::IfOp>(loc, isInsertingLane,
@@ -1802,7 +1802,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
 
     // Create a new for op outside the region with a WarpExecuteOnLane0Op
     // region inside.
-    auto newForOp = rewriter.create<scf::ForOp>(
+    auto newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), newOperands);
     rewriter.setInsertionPointToStart(newForOp.getBody());
@@ -1817,7 +1817,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
       argIndexMapping[escapingValues[i]] = warpInputType.size();
       warpInputType.push_back(inputTypes[i]);
     }
-    auto innerWarp = rewriter.create<WarpExecuteOnLane0Op>(
+    auto innerWarp = WarpExecuteOnLane0Op::create(rewriter,
         newWarpOp.getLoc(), newForOp.getResultTypes(), newWarpOp.getLaneid(),
         newWarpOp.getWarpSize(), warpInput, warpInputType);
 
@@ -1833,10 +1833,10 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
     rewriter.eraseOp(forOp.getBody()->getTerminator());
     rewriter.mergeBlocks(forOp.getBody(), innerWarp.getBody(), argMapping);
     rewriter.setInsertionPointToEnd(innerWarp.getBody());
-    rewriter.create<gpu::YieldOp>(innerWarp.getLoc(), yieldOperands);
+    gpu::YieldOp::create(rewriter, innerWarp.getLoc(), yieldOperands);
     rewriter.setInsertionPointAfter(innerWarp);
     if (!innerWarp.getResults().empty())
-      rewriter.create<scf::YieldOp>(forOp.getLoc(), innerWarp.getResults());
+      scf::YieldOp::create(rewriter, forOp.getLoc(), innerWarp.getResults());
     rewriter.eraseOp(forOp);
     // Replace the warpOp result coming from the original ForOp.
     for (const auto &res : llvm::enumerate(resultIdx)) {
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
index 067d4e3491391..e9718a7795fc5 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
@@ -77,7 +77,7 @@ struct CastAwayExtractStridedSliceLeadingOneDim
 
     Location loc = extractOp.getLoc();
 
-    Value newSrcVector = rewriter.create<vector::ExtractOp>(
+    Value newSrcVector = vector::ExtractOp::create(rewriter,
         loc, extractOp.getVector(), splatZero(dropCount));
 
     // The offsets/sizes/strides attribute can have a less number of elements
@@ -89,7 +89,7 @@ struct CastAwayExtractStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         extractOp.getStrides().getValue().drop_front(dropCount));
 
-    auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
         loc, newDstType, newSrcVector, newOffsets, newSizes, newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(extractOp, oldDstType,
@@ -120,9 +120,9 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     // Trim leading one dimensions from both operands.
     Location loc = insertOp.getLoc();
 
-    Value newSrcVector = rewriter.create<vector::ExtractOp>(
+    Value newSrcVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getValueToStore(), splatZero(srcDropCount));
-    Value newDstVector = rewriter.create<vector::ExtractOp>(
+    Value newDstVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getDest(), splatZero(dstDropCount));
 
     auto newOffsets = rewriter.getArrayAttr(
@@ -130,7 +130,7 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         insertOp.getStrides().getValue().take_back(newSrcType.getRank()));
 
-    auto newInsertOp = rewriter.create<vector::InsertStridedSliceOp>(
+    auto newInsertOp = vector::InsertStridedSliceOp::create(rewriter,
         loc, newDstType, newSrcVector, newDstVector, newOffsets, newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
@@ -169,10 +169,10 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
 
     Value newSrcVector = insertOp.getValueToStore();
     if (oldSrcRank != 0) {
-      newSrcVector = rewriter.create<vector::ExtractOp>(
+      newSrcVector = vector::ExtractOp::create(rewriter,
           loc, insertOp.getValueToStore(), splatZero(srcDropCount));
     }
-    Value newDstVector = rewriter.create<vector::ExtractOp>(
+    Value newDstVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getDest(), splatZero(dstDropCount));
 
     // New position rank needs to be computed in two steps: (1) if destination
@@ -187,7 +187,7 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
     newPosition.resize(newDstType.getRank() - newSrcRank,
                        rewriter.getI64IntegerAttr(0));
 
-    auto newInsertOp = rewriter.create<vector::InsertOp>(
+    auto newInsertOp = vector::InsertOp::create(rewriter,
         loc, newSrcVector, newDstVector, newPosition);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
@@ -209,9 +209,9 @@ static Value dropUnitDimsFromMask(OpBuilder &b, Location loc, Value mask,
   if (vector::isBroadcastableTo(newMaskType, oldMaskType) ==
       BroadcastableToResult::Success) {
     int64_t dropDim = oldMaskType.getRank() - newMaskType.getRank();
-    return b.create<vector::ExtractOp>(loc, mask, splatZero(dropDim));
+    return vector::ExtractOp::create(b, loc, mask, splatZero(dropDim));
   }
-  return b.create<vector::ShapeCastOp>(loc, newMaskType, mask);
+  return vector::ShapeCastOp::create(b, loc, newMaskType, mask);
 }
 
 // Turns vector.transfer_read on vector with leading 1 dimensions into
@@ -259,7 +259,7 @@ struct CastAwayTransferReadLeadingOneDim
                                   newType, newMap, maskType);
     }
 
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         read.getLoc(), newType, read.getBase(), read.getIndices(),
         AffineMapAttr::get(newMap), read.getPadding(), mask, inBoundsAttr);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(read, oldType, newRead);
@@ -306,7 +306,7 @@ struct CastAwayTransferWriteLeadingOneDim
       inBoundsAttr = rewriter.getArrayAttr(
           write.getInBoundsAttr().getValue().take_back(newType.getRank()));
 
-    auto newVector = rewriter.create<vector::ExtractOp>(
+    auto newVector = vector::ExtractOp::create(rewriter,
         write.getLoc(), write.getVector(), splatZero(dropDim));
 
     if (write.getMask()) {
@@ -444,20 +444,20 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
     // Extract if its a valid extraction, otherwise use the operand
     // without extraction.
     newOperands.push_back(
-        validExtract ? rewriter.create<vector::ExtractOp>(
+        validExtract ? vector::ExtractOp::create(rewriter,
                            loc, operands[it.index()], splatZero(dropDim))
                      : operands[it.index()]);
   }
 
   // Depending on whether this vector.contract is masked, the replacing Op
   // should either be a new vector.contract Op or vector.mask Op.
-  Operation *newOp = rewriter.create<vector::ContractionOp>(
+  Operation *newOp = vector::ContractionOp::create(rewriter,
       loc, newOperands[0], newOperands[1], newOperands[2],
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind());
 
   if (maskingOp) {
-    auto newMask = rewriter.create<vector::ExtractOp>(loc, maskingOp.getMask(),
+    auto newMask = vector::ExtractOp::create(rewriter, loc, maskingOp.getMask(),
                                                       splatZero(dropDim));
 
     newOp = mlir::vector::maskOperation(rewriter, newOp, newMask);
@@ -519,7 +519,7 @@ class CastAwayElementwiseLeadingOneDim : public RewritePattern {
     SmallVector<Value, 4> newOperands;
     for (Value operand : op->getOperands()) {
       if (auto opVecType = dyn_cast<VectorType>(operand.getType())) {
-        newOperands.push_back(rewriter.create<vector::ExtractOp>(
+        newOperands.push_back(vector::ExtractOp::create(rewriter,
             op->getLoc(), operand, splatZero(dropDim)));
       } else {
         newOperands.push_back(operand);
@@ -559,7 +559,7 @@ struct CastAwayConstantMaskLeadingOneDim
     SmallVector<int64_t> newDimSizes = {flatLeadingSize};
     newDimSizes.append(dimSizes.begin() + dropDim + 1, dimSizes.end());
 
-    auto newMask = rewriter.create<vector::ConstantMaskOp>(
+    auto newMask = vector::ConstantMaskOp::create(rewriter,
         mask.getLoc(), newType, newDimSizes);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(mask, oldType, newMask);
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
index 8cc7008d80b3e..3411e010f3499 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
@@ -65,26 +65,26 @@ struct VectorMaskedLoadOpConverter final
     Value base = maskedLoadOp.getBase();
     Value iValue = maskedLoadOp.getPassThru();
     auto indices = llvm::to_vector_of<Value>(maskedLoadOp.getIndices());
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, indexType, IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
-      auto maskBit = rewriter.create<vector::ExtractOp>(loc, mask, i);
+      auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = rewriter.create<scf::IfOp>(
+      auto ifOp = scf::IfOp::create(rewriter,
           loc, maskBit,
           [&](OpBuilder &builder, Location loc) {
             auto loadedValue =
-                builder.create<memref::LoadOp>(loc, base, indices);
+                memref::LoadOp::create(builder, loc, base, indices);
             auto combinedValue =
-                builder.create<vector::InsertOp>(loc, loadedValue, iValue, i);
-            builder.create<scf::YieldOp>(loc, combinedValue.getResult());
+                vector::InsertOp::create(builder, loc, loadedValue, iValue, i);
+            scf::YieldOp::create(builder, loc, combinedValue.getResult());
           },
           [&](OpBuilder &builder, Location loc) {
-            builder.create<scf::YieldOp>(loc, iValue);
+            scf::YieldOp::create(builder, loc, iValue);
           });
       iValue = ifOp.getResult(0);
 
-      indices.back() = rewriter.create<arith::AddIOp>(loc, indices.back(), one);
+      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.replaceOp(maskedLoadOp, iValue);
@@ -132,18 +132,18 @@ struct VectorMaskedStoreOpConverter final
     Value base = maskedStoreOp.getBase();
     Value value = maskedStoreOp.getValueToStore();
     auto indices = llvm::to_vector_of<Value>(maskedStoreOp.getIndices());
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, indexType, IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
-      auto maskBit = rewriter.create<vector::ExtractOp>(loc, mask, i);
+      auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = rewriter.create<scf::IfOp>(loc, maskBit, /*else=*/false);
+      auto ifOp = scf::IfOp::create(rewriter, loc, maskBit, /*else=*/false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      auto extractedValue = rewriter.create<vector::ExtractOp>(loc, value, i);
-      rewriter.create<memref::StoreOp>(loc, extractedValue, base, indices);
+      auto extractedValue = vector::ExtractOp::create(rewriter, loc, value, i);
+      memref::StoreOp::create(rewriter, loc, extractedValue, base, indices);
 
       rewriter.setInsertionPointAfter(ifOp);
-      indices.back() = rewriter.create<arith::AddIOp>(loc, indices.back(), one);
+      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.eraseOp(maskedStoreOp);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 004beadc9ec7d..12b548bcee646 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -132,7 +132,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 SmallVector<Value> newMaskOperands(maskOperands.drop_back());
                 newMaskOperands.push_back(
                     getValueOrCreateConstantIndexOp(rewriter, loc, maskIndex));
-                return rewriter.create<vector::CreateMaskOp>(loc, newMaskType,
+                return vector::CreateMaskOp::create(rewriter, loc, newMaskType,
                                                              newMaskOperands);
               })
           .Case<vector::ConstantMaskOp>(
@@ -143,7 +143,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 int64_t &maskIndex = maskDimSizes.back();
                 maskIndex = llvm::divideCeil(numFrontPadElems + maskIndex,
                                              numSrcElemsPerDest);
-                return rewriter.create<vector::ConstantMaskOp>(loc, newMaskType,
+                return vector::ConstantMaskOp::create(rewriter, loc, newMaskType,
                                                                maskDimSizes);
               })
           .Case<arith::ConstantOp>([&](auto constantOp)
@@ -182,7 +182,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
               }
               compressedMaskValues.push_back(combinedValue);
             }
-            return rewriter.create<arith::ConstantOp>(
+            return arith::ConstantOp::create(rewriter,
                 loc, DenseElementsAttr::get(newMaskType, compressedMaskValues));
           });
 
@@ -190,7 +190,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
     return failure();
 
   while (!extractOps.empty()) {
-    newMask = rewriter.create<vector::ExtractOp>(
+    newMask = vector::ExtractOp::create(rewriter,
         loc, (*newMask)->getResults()[0], extractOps.back().getMixedPosition());
     extractOps.pop_back();
   }
@@ -258,7 +258,7 @@ static Value staticallyInsertSubvector(OpBuilder &rewriter, Location loc,
 
   auto offsets = rewriter.getI64ArrayAttr({offset});
   auto strides = rewriter.getI64ArrayAttr({1});
-  return rewriter.create<vector::InsertStridedSliceOp>(loc, destVecTy, src,
+  return vector::InsertStridedSliceOp::create(rewriter, loc, destVecTy, src,
                                                        dest, offsets, strides);
 }
 
@@ -301,11 +301,11 @@ static Value dynamicallyExtractSubVector(OpBuilder &rewriter, Location loc,
   for (int i = 0; i < numElemsToExtract; ++i) {
     Value extractLoc =
         (i == 0) ? dyn_cast<Value>(offset)
-                 : rewriter.create<arith::AddIOp>(
+                 : arith::AddIOp::create(rewriter,
                        loc, rewriter.getIndexType(), dyn_cast<Value>(offset),
-                       rewriter.create<arith::ConstantIndexOp>(loc, i));
-    auto extractOp = rewriter.create<vector::ExtractOp>(loc, src, extractLoc);
-    dest = rewriter.create<vector::InsertOp>(loc, extractOp, dest, i);
+                       arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto extractOp = vector::ExtractOp::create(rewriter, loc, src, extractLoc);
+    dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, i);
   }
   return dest;
 }
@@ -346,11 +346,11 @@ static Value dynamicallyInsertSubVector(RewriterBase &rewriter, Location loc,
   for (int64_t i = 0; i < numElemsToInsert; ++i) {
     auto insertLoc = i == 0
                          ? destOffsetVal
-                         : rewriter.create<arith::AddIOp>(
+                         : arith::AddIOp::create(rewriter,
                                loc, rewriter.getIndexType(), destOffsetVal,
-                               rewriter.create<arith::ConstantIndexOp>(loc, i));
-    auto extractOp = rewriter.create<vector::ExtractOp>(loc, src, i);
-    dest = rewriter.create<vector::InsertOp>(loc, extractOp, dest, insertLoc);
+                               arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto extractOp = vector::ExtractOp::create(rewriter, loc, src, i);
+    dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, insertLoc);
   }
   return dest;
 }
@@ -369,10 +369,10 @@ static VectorValue emulatedVectorLoad(OpBuilder &rewriter, Location loc,
                                       Type containerElemTy) {
   auto emulatedPerContainerElem = containerElemTy.getIntOrFloatBitWidth() /
                                   emulatedElemTy.getIntOrFloatBitWidth();
-  auto newLoad = rewriter.create<vector::LoadOp>(
+  auto newLoad = vector::LoadOp::create(rewriter,
       loc, VectorType::get(numContainerElemsToLoad, containerElemTy), base,
       getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
-  return rewriter.create<vector::BitCastOp>(
+  return vector::BitCastOp::create(rewriter,
       loc,
       VectorType::get(numContainerElemsToLoad * emulatedPerContainerElem,
                       emulatedElemTy),
@@ -390,16 +390,16 @@ static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
           upcastType.getNumElements() * upcastType.getElementTypeBitWidth() &&
       "expected input and output number of bits to match");
   if (trueValue.getType() != downcastType) {
-    trueValue = builder.create<vector::BitCastOp>(loc, downcastType, trueValue);
+    trueValue = vector::BitCastOp::create(builder, loc, downcastType, trueValue);
   }
   if (falseValue.getType() != downcastType) {
     falseValue =
-        builder.create<vector::BitCastOp>(loc, downcastType, falseValue);
+        vector::BitCastOp::create(builder, loc, downcastType, falseValue);
   }
   Value selectedType =
-      builder.create<arith::SelectOp>(loc, mask, trueValue, falseValue);
+      arith::SelectOp::create(builder, loc, mask, trueValue, falseValue);
   // Upcast the selected value to the new type.
-  return builder.create<vector::BitCastOp>(loc, upcastType, selectedType);
+  return vector::BitCastOp::create(builder, loc, upcastType, selectedType);
 }
 
 /// Emits `memref.generic_atomic_rmw` op to store a subbyte-sized value to a
@@ -422,7 +422,7 @@ static void atomicRMW(OpBuilder &builder, Location loc,
 
   // Create an atomic load-modify-write region using
   // `memref.generic_atomic_rmw`.
-  auto atomicOp = builder.create<memref::GenericAtomicRMWOp>(
+  auto atomicOp = memref::GenericAtomicRMWOp::create(builder,
       loc, linearizedMemref, ValueRange{storeIdx});
   Value origValue = atomicOp.getCurrentValue();
 
@@ -432,7 +432,7 @@ static void atomicRMW(OpBuilder &builder, Location loc,
   // Load the original value from memory, and cast it to the original element
   // type.
   auto oneElemVecType = VectorType::get({1}, origValue.getType());
-  Value origVecValue = builder.create<vector::FromElementsOp>(
+  Value origVecValue = vector::FromElementsOp::create(builder,
       loc, oneElemVecType, ValueRange{origValue});
 
   // Construct the final masked value and yield it.
@@ -440,8 +440,8 @@ static void atomicRMW(OpBuilder &builder, Location loc,
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
   auto scalarMaskedValue =
-      builder.create<vector::ExtractOp>(loc, maskedValue, 0);
-  builder.create<memref::AtomicYieldOp>(loc, scalarMaskedValue);
+      vector::ExtractOp::create(builder, loc, maskedValue, 0);
+  memref::AtomicYieldOp::create(builder, loc, scalarMaskedValue);
 }
 
 /// Generate a non-atomic read-modify-write sequence for storing to the emulated
@@ -453,15 +453,15 @@ static void nonAtomicRMW(OpBuilder &builder, Location loc,
 
   auto oneElemVecType =
       VectorType::get({1}, linearizedMemref.getType().getElementType());
-  Value origVecValue = builder.create<vector::LoadOp>(
+  Value origVecValue = vector::LoadOp::create(builder,
       loc, oneElemVecType, linearizedMemref, ValueRange{linearizedIndex});
-  origVecValue = builder.create<vector::BitCastOp>(loc, valueToStore.getType(),
+  origVecValue = vector::BitCastOp::create(builder, loc, valueToStore.getType(),
                                                    origVecValue);
 
   Value maskedValue =
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
-  builder.create<vector::StoreOp>(loc, maskedValue, linearizedMemref,
+  vector::StoreOp::create(builder, loc, maskedValue, linearizedMemref,
                                   linearizedIndex);
 }
 
@@ -489,7 +489,7 @@ static Value extractSliceIntoByte(ConversionPatternRewriter &rewriter,
   assert(8 % vectorElementType.getIntOrFloatBitWidth() == 0 &&
          "vector element must be a valid sub-byte type");
   auto emulatedPerContainerElem = 8 / vectorElementType.getIntOrFloatBitWidth();
-  auto emptyByteVector = rewriter.create<arith::ConstantOp>(
+  auto emptyByteVector = arith::ConstantOp::create(rewriter,
       loc, VectorType::get({emulatedPerContainerElem}, vectorElementType),
       rewriter.getZeroAttr(
           VectorType::get({emulatedPerContainerElem}, vectorElementType)));
@@ -602,7 +602,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         ShapedType::isDynamic(trailingDim) || trailingDim == origElements;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     // FIXME: ATM, we do not test cases where offsets, sizes, or strides are
     // non-zero. As such, this is not needed.
@@ -664,7 +664,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     if (!emulationRequiresPartialStores) {
       // Basic case: storing full bytes.
       auto numElements = origElements / emulatedPerContainerElem;
-      auto bitCast = rewriter.create<vector::BitCastOp>(
+      auto bitCast = vector::BitCastOp::create(rewriter,
           loc, VectorType::get(numElements, containerElemTy),
           op.getValueToStore());
       rewriter.replaceOpWithNewOp<vector::StoreOp>(
@@ -732,7 +732,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         std::fill_n(frontMaskValues.end() - frontSubWidthStoreElem,
                     *foldedNumFrontPadElems, true);
       }
-      auto frontMask = rewriter.create<arith::ConstantOp>(
+      auto frontMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
 
       currentSourceIndex = emulatedPerContainerElem - (*foldedNumFrontPadElems);
@@ -751,8 +751,8 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
 
     // Increment the destination index by 1 to align to the emulated width
     // boundary.
-    auto constantOne = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    currentDestIndex = rewriter.create<arith::AddIOp>(
+    auto constantOne = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    currentDestIndex = arith::AddIOp::create(rewriter,
         loc, rewriter.getIndexType(), currentDestIndex, constantOne);
 
     // 2. Full width store for the inner output bytes.
@@ -772,15 +772,15 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       auto storeType = VectorType::get(
           {originType.getNumElements() / emulatedPerContainerElem},
           memrefElemType);
-      auto bitCast = rewriter.create<vector::BitCastOp>(loc, storeType,
+      auto bitCast = vector::BitCastOp::create(rewriter, loc, storeType,
                                                         fullWidthStorePart);
-      rewriter.create<vector::StoreOp>(loc, bitCast.getResult(), memrefBase,
+      vector::StoreOp::create(rewriter, loc, bitCast.getResult(), memrefBase,
                                        currentDestIndex);
 
       currentSourceIndex += numNonFullWidthElements;
-      currentDestIndex = rewriter.create<arith::AddIOp>(
+      currentDestIndex = arith::AddIOp::create(rewriter,
           loc, rewriter.getIndexType(), currentDestIndex,
-          rewriter.create<arith::ConstantIndexOp>(loc, fullWidthStoreSize));
+          arith::ConstantIndexOp::create(rewriter, loc, fullWidthStoreSize));
     }
 
     // 3. Partial width store for the trailing output byte.
@@ -795,7 +795,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       // Generate back mask.
       auto maskValues = SmallVector<bool>(emulatedPerContainerElem, 0);
       std::fill_n(maskValues.begin(), remainingElements, 1);
-      auto backMask = rewriter.create<arith::ConstantOp>(
+      auto backMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
 
       storeFunc(rewriter, loc, memrefBase, currentDestIndex,
@@ -848,7 +848,7 @@ struct ConvertVectorMaskedStore final
       return failure();
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
     OpFoldResult linearizedIndicesOfr;
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndicesOfr) =
@@ -901,21 +901,21 @@ struct ConvertVectorMaskedStore final
     auto numElements = (origElements + emulatedPerContainerElem - 1) /
                        emulatedPerContainerElem;
     auto newType = VectorType::get(numElements, containerElemTy);
-    auto passThru = rewriter.create<arith::ConstantOp>(
+    auto passThru = arith::ConstantOp::create(rewriter,
         loc, newType, rewriter.getZeroAttr(newType));
 
-    auto newLoad = rewriter.create<vector::MaskedLoadOp>(
+    auto newLoad = vector::MaskedLoadOp::create(rewriter,
         loc, newType, adaptor.getBase(), linearizedIndices,
         newMask.value()->getResult(0), passThru);
 
     auto newBitCastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
     Value valueToStore =
-        rewriter.create<vector::BitCastOp>(loc, newBitCastType, newLoad);
-    valueToStore = rewriter.create<arith::SelectOp>(
+        vector::BitCastOp::create(rewriter, loc, newBitCastType, newLoad);
+    valueToStore = arith::SelectOp::create(rewriter,
         loc, op.getMask(), op.getValueToStore(), valueToStore);
     valueToStore =
-        rewriter.create<vector::BitCastOp>(loc, newType, valueToStore);
+        vector::BitCastOp::create(rewriter, loc, newType, valueToStore);
 
     rewriter.replaceOpWithNewOp<vector::MaskedStoreOp>(
         op, adaptor.getBase(), linearizedIndices, newMask.value()->getResult(0),
@@ -990,7 +990,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
     bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
@@ -1016,7 +1016,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
                            numElements, emulatedElemTy, containerElemTy);
 
     if (!foldedIntraVectorOffset) {
-      auto resultVector = rewriter.create<arith::ConstantOp>(
+      auto resultVector = arith::ConstantOp::create(rewriter,
           loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(
           rewriter, loc, dyn_cast<TypedValue<VectorType>>(result), resultVector,
@@ -1111,7 +1111,7 @@ struct ConvertVectorMaskedLoad final
     bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndices) =
@@ -1142,7 +1142,7 @@ struct ConvertVectorMaskedLoad final
     auto newBitcastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
 
-    auto emptyVector = rewriter.create<arith::ConstantOp>(
+    auto emptyVector = arith::ConstantOp::create(rewriter,
         loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
     if (!foldedIntraVectorOffset) {
       passthru = dynamicallyInsertSubVector(
@@ -1153,10 +1153,10 @@ struct ConvertVectorMaskedLoad final
                                            *foldedIntraVectorOffset);
     }
     auto newPassThru =
-        rewriter.create<vector::BitCastOp>(loc, loadType, passthru);
+        vector::BitCastOp::create(rewriter, loc, loadType, passthru);
 
     // Generating the new masked load.
-    auto newLoad = rewriter.create<vector::MaskedLoadOp>(
+    auto newLoad = vector::MaskedLoadOp::create(rewriter,
         loc, loadType, adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newMask.value()->getResult(0), newPassThru);
@@ -1164,13 +1164,13 @@ struct ConvertVectorMaskedLoad final
     // Setting the part that originally was not effectively loaded from memory
     // to pass through.
     auto bitCast =
-        rewriter.create<vector::BitCastOp>(loc, newBitcastType, newLoad);
+        vector::BitCastOp::create(rewriter, loc, newBitcastType, newLoad);
 
     Value mask = op.getMask();
     auto newSelectMaskType = VectorType::get(
         numElements * emulatedPerContainerElem, rewriter.getI1Type());
     // TODO: try to fold if op's mask is constant
-    auto emptyMask = rewriter.create<arith::ConstantOp>(
+    auto emptyMask = arith::ConstantOp::create(rewriter,
         loc, newSelectMaskType, rewriter.getZeroAttr(newSelectMaskType));
     if (!foldedIntraVectorOffset) {
       mask = dynamicallyInsertSubVector(rewriter, loc, mask, emptyMask,
@@ -1182,7 +1182,7 @@ struct ConvertVectorMaskedLoad final
     }
 
     Value result =
-        rewriter.create<arith::SelectOp>(loc, mask, bitCast, passthru);
+        arith::SelectOp::create(rewriter, loc, mask, bitCast, passthru);
     if (!foldedIntraVectorOffset) {
       result = dynamicallyExtractSubVector(
           rewriter, loc, result, op.getPassThru(),
@@ -1268,11 +1268,11 @@ struct ConvertVectorTransferRead final
     bool isDivisibleInSize =
         fitsInMultiByteContainerTy(op.getVectorType(), containerElemTy);
 
-    auto newPadding = rewriter.create<arith::ExtUIOp>(loc, containerElemTy,
+    auto newPadding = arith::ExtUIOp::create(rewriter, loc, containerElemTy,
                                                       adaptor.getPadding());
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
@@ -1293,19 +1293,19 @@ struct ConvertVectorTransferRead final
     auto numElements = llvm::divideCeil(maxIntraDataOffset + origElements,
                                         emulatedPerContainerElem);
 
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         loc, VectorType::get(numElements, containerElemTy), adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newPadding);
 
-    auto bitCast = rewriter.create<vector::BitCastOp>(
+    auto bitCast = vector::BitCastOp::create(rewriter,
         loc,
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy),
         newRead);
 
     Value result = bitCast->getResult(0);
     if (!foldedIntraVectorOffset) {
-      auto zeros = rewriter.create<arith::ConstantOp>(
+      auto zeros = arith::ConstantOp::create(rewriter,
           loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(rewriter, loc, bitCast, zeros,
                                            linearizedInfo.intraDataOffset,
@@ -1679,32 +1679,32 @@ Value BitCastRewriter::genericRewriteStep(
     PatternRewriter &rewriter, Location loc, Value initialValue,
     Value runningResult, const BitCastRewriter::Metadata &metadata) {
   // Create vector.shuffle from the metadata.
-  auto shuffleOp = rewriter.create<vector::ShuffleOp>(
+  auto shuffleOp = vector::ShuffleOp::create(rewriter,
       loc, initialValue, initialValue, metadata.shuffles);
 
   // Intersect with the mask.
   VectorType shuffledVectorType = shuffleOp.getResultVectorType();
-  auto constOp = rewriter.create<arith::ConstantOp>(
+  auto constOp = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(shuffledVectorType, metadata.masks));
-  Value andValue = rewriter.create<arith::AndIOp>(loc, shuffleOp, constOp);
+  Value andValue = arith::AndIOp::create(rewriter, loc, shuffleOp, constOp);
 
   // Align right on 0.
-  auto shiftRightConstantOp = rewriter.create<arith::ConstantOp>(
+  auto shiftRightConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftRightAmounts));
   Value shiftedRight =
-      rewriter.create<arith::ShRUIOp>(loc, andValue, shiftRightConstantOp);
+      arith::ShRUIOp::create(rewriter, loc, andValue, shiftRightConstantOp);
 
   // Shift bits left into their final position.
-  auto shiftLeftConstantOp = rewriter.create<arith::ConstantOp>(
+  auto shiftLeftConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftLeftAmounts));
   Value shiftedLeft =
-      rewriter.create<arith::ShLIOp>(loc, shiftedRight, shiftLeftConstantOp);
+      arith::ShLIOp::create(rewriter, loc, shiftedRight, shiftLeftConstantOp);
 
   runningResult =
       runningResult
-          ? rewriter.create<arith::OrIOp>(loc, runningResult, shiftedLeft)
+          ? arith::OrIOp::create(rewriter, loc, runningResult, shiftedLeft)
           : shiftedLeft;
 
   return runningResult;
@@ -1727,7 +1727,7 @@ static Value bitcastSubByteVectorToI8(PatternRewriter &rewriter, Location loc,
   // Adjust last dimension of the vector, so the total size remains the same.
   vecShape.back() = vecShape.back() / numSrcElemsPerByte;
   auto i8VecType = VectorType::get(vecShape, rewriter.getI8Type());
-  return rewriter.create<vector::BitCastOp>(loc, i8VecType, subByteVec);
+  return vector::BitCastOp::create(rewriter, loc, i8VecType, subByteVec);
 }
 
 /// Extracts a signed N-bit sequence from each element of a vector of bytes,
@@ -1755,15 +1755,15 @@ static Value extractNBitsPerByteAndSignExtendToI8(PatternRewriter &rewriter,
   assert(bitIdx >= 0 && bitsToShiftLeft >= 0 && numBits > 0 && numBits <= 8 &&
          "Invalid bitIdx range");
   if (bitsToShiftLeft != 0) {
-    Value shiftLeftValues = rewriter.create<arith::ConstantOp>(
+    Value shiftLeftValues = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(srcType, bitsToShiftLeft));
-    shl = rewriter.create<arith::ShLIOp>(loc, src, shiftLeftValues);
+    shl = arith::ShLIOp::create(rewriter, loc, src, shiftLeftValues);
   }
 
   int8_t bitsToShiftRight = 8 - numBits;
-  Value shiftRightValues = rewriter.create<arith::ConstantOp>(
+  Value shiftRightValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
-  Value shr = rewriter.create<arith::ShRSIOp>(loc, shl, shiftRightValues);
+  Value shr = arith::ShRSIOp::create(rewriter, loc, shl, shiftRightValues);
   return shr;
 }
 
@@ -1797,17 +1797,17 @@ static Value extractNBitsPerByteAndExtendToI8(PatternRewriter &rewriter,
   int8_t bitsToShiftRight = bitIdx;
   Value shr = src;
   if (bitsToShiftRight != 0) {
-    Value shiftRightValues = rewriter.create<arith::ConstantOp>(
+    Value shiftRightValues = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
-    shr = rewriter.create<arith::ShRUIOp>(loc, src, shiftRightValues);
+    shr = arith::ShRUIOp::create(rewriter, loc, src, shiftRightValues);
   }
   if (bitIdx + numBits == 8) {
     return shr;
   }
   uint8_t lowBitsMask = (1 << numBits) - 1;
-  Value lowBitsMaskValues = rewriter.create<arith::ConstantOp>(
+  Value lowBitsMaskValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(srcType, lowBitsMask));
-  return rewriter.create<arith::AndIOp>(loc, shr, lowBitsMaskValues);
+  return arith::AndIOp::create(rewriter, loc, shr, lowBitsMaskValues);
 }
 
 using ExtractNBitsFn =
@@ -1830,7 +1830,7 @@ static Value rewriteI4ToI8Ext(PatternRewriter &rewriter, Location loc,
   Value high = extFn(rewriter, loc, i8Vector, 4, 4);
 
   // 3. Interleave low and high i8 elements.
-  return rewriter.create<vector::InterleaveOp>(loc, low, high);
+  return vector::InterleaveOp::create(rewriter, loc, low, high);
 }
 
 /// Rewrite the i2 -> i8  extension into a sequence of shuffles and
@@ -1863,9 +1863,9 @@ static Value rewriteI2ToI8Ext(PatternRewriter &rewriter, Location loc,
   // 02    = [0,2,0,2,0,2,0,2],...
   // 13    = [1,3,1,3,1,3,1,3],...
   // 0213  = [0,1,2,3,...],...
-  Value interleave02 = rewriter.create<vector::InterleaveOp>(loc, vec0, vec2);
-  Value interleave13 = rewriter.create<vector::InterleaveOp>(loc, vec1, vec3);
-  return rewriter.create<vector::InterleaveOp>(loc, interleave02, interleave13);
+  Value interleave02 = vector::InterleaveOp::create(rewriter, loc, vec0, vec2);
+  Value interleave13 = vector::InterleaveOp::create(rewriter, loc, vec1, vec3);
+  return vector::InterleaveOp::create(rewriter, loc, interleave02, interleave13);
 }
 
 /// Rewrite the i8 -> i4 truncation into a deinterleave and series of bitwise
@@ -1877,29 +1877,29 @@ static Value rewriteI8ToI4Trunc(PatternRewriter &rewriter, Location loc,
          "Expected i8 type");
 
   // 1. De-interleave low and high i8 elements.
-  auto deinterleaveOp = rewriter.create<vector::DeinterleaveOp>(loc, srcValue);
+  auto deinterleaveOp = vector::DeinterleaveOp::create(rewriter, loc, srcValue);
 
   // 2. Zero out the upper side of each low i8 element.
   constexpr int8_t i8LowBitMask = 0x0F;
   VectorType deinterI8VecType = deinterleaveOp.getResultVectorType();
-  Value zeroOutMask = rewriter.create<arith::ConstantOp>(
+  Value zeroOutMask = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(deinterI8VecType, i8LowBitMask));
-  Value zeroOutLow = rewriter.create<arith::AndIOp>(
+  Value zeroOutLow = arith::AndIOp::create(rewriter,
       loc, deinterleaveOp.getRes1(), zeroOutMask);
 
   // 3. Move high i4 values to upper side of the byte.
   constexpr int8_t bitsToShift = 4;
-  auto shiftValues = rewriter.create<arith::ConstantOp>(
+  auto shiftValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(deinterI8VecType, bitsToShift));
-  Value shlHigh = rewriter.create<arith::ShLIOp>(loc, deinterleaveOp.getRes2(),
+  Value shlHigh = arith::ShLIOp::create(rewriter, loc, deinterleaveOp.getRes2(),
                                                  shiftValues);
 
   // 4. Merge high and low i4 values.
-  auto mergedHiLowOp = rewriter.create<arith::OrIOp>(loc, zeroOutLow, shlHigh);
+  auto mergedHiLowOp = arith::OrIOp::create(rewriter, loc, zeroOutLow, shlHigh);
 
   // 5. Generate a bitcast vector<Xxi8> -> vector<2Xxi4>.
   auto i4VecType = srcVecType.cloneWith(std::nullopt, rewriter.getI4Type());
-  return rewriter.create<vector::BitCastOp>(loc, i4VecType, mergedHiLowOp);
+  return vector::BitCastOp::create(rewriter, loc, i4VecType, mergedHiLowOp);
 }
 
 namespace {
@@ -2141,7 +2141,7 @@ struct RewriteAlignedSubByteIntTrunc : OpRewritePattern<arith::TruncIOp> {
     Location loc = truncOp.getLoc();
     auto i8VecType = srcVecType.cloneWith(std::nullopt, rewriter.getI8Type());
     Value i8TruncVal =
-        rewriter.create<arith::TruncIOp>(loc, i8VecType, srcValue);
+        arith::TruncIOp::create(rewriter, loc, i8VecType, srcValue);
 
     // Rewrite the i8 -> i4 truncation part.
     Value subByteTrunc = rewriteI8ToI4Trunc(rewriter, loc, i8TruncVal);
@@ -2189,9 +2189,9 @@ struct RewriteVectorTranspose : OpRewritePattern<vector::TransposeOp> {
     // support is available.
     auto srcNativeVecType = srcSubByteVecType.cloneWith(
         std::nullopt, rewriter.getIntegerType(minNativeBitwidth));
-    Value extOp = rewriter.create<arith::ExtSIOp>(loc, srcNativeVecType,
+    Value extOp = arith::ExtSIOp::create(rewriter, loc, srcNativeVecType,
                                                   transposeOp.getVector());
-    Value newTranspose = rewriter.create<vector::TransposeOp>(
+    Value newTranspose = vector::TransposeOp::create(rewriter,
         loc, extOp, transposeOp.getPermutation());
     VectorType dstSubByteVecType = transposeOp.getResultVectorType();
     rewriter.replaceOpWithNewOp<arith::TruncIOp>(transposeOp, dstSubByteVecType,
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
index d834a99076834..baeca29298334 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
@@ -53,14 +53,14 @@ class DecomposeDifferentRankInsertStridedSlice
     int64_t rankRest = dstType.getRank() - rankDiff;
     // Extract / insert the subvector of matching rank and InsertStridedSlice
     // on it.
-    Value extracted = rewriter.create<ExtractOp>(
+    Value extracted = ExtractOp::create(rewriter,
         loc, op.getDest(),
         getI64SubArray(op.getOffsets(), /*dropFront=*/0,
                        /*dropBack=*/rankRest));
 
     // A different pattern will kick in for InsertStridedSlice with matching
     // ranks.
-    auto stridedSliceInnerOp = rewriter.create<InsertStridedSliceOp>(
+    auto stridedSliceInnerOp = InsertStridedSliceOp::create(rewriter,
         loc, op.getValueToStore(), extracted,
         getI64SubArray(op.getOffsets(), /*dropFront=*/rankDiff),
         getI64SubArray(op.getStrides(), /*dropFront=*/0));
@@ -131,7 +131,7 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
       SmallVector<int64_t> offsets(nDest, 0);
       for (int64_t i = 0; i < nSrc; ++i)
         offsets[i] = i;
-      Value scaledSource = rewriter.create<ShuffleOp>(
+      Value scaledSource = ShuffleOp::create(rewriter,
           loc, op.getValueToStore(), op.getValueToStore(), offsets);
 
       // 2. Create a mask where we take the value from scaledSource of dest
@@ -156,21 +156,21 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
          off += stride, ++idx) {
       // 1. extract the proper subvector (or element) from source
       Value extractedSource =
-          rewriter.create<ExtractOp>(loc, op.getValueToStore(), idx);
+          ExtractOp::create(rewriter, loc, op.getValueToStore(), idx);
       if (isa<VectorType>(extractedSource.getType())) {
         // 2. If we have a vector, extract the proper subvector from destination
         // Otherwise we are at the element level and no need to recurse.
         Value extractedDest =
-            rewriter.create<ExtractOp>(loc, op.getDest(), off);
+            ExtractOp::create(rewriter, loc, op.getDest(), off);
         // 3. Reduce the problem to lowering a new InsertStridedSlice op with
         // smaller rank.
-        extractedSource = rewriter.create<InsertStridedSliceOp>(
+        extractedSource = InsertStridedSliceOp::create(rewriter,
             loc, extractedSource, extractedDest,
             getI64SubArray(op.getOffsets(), /* dropFront=*/1),
             getI64SubArray(op.getStrides(), /* dropFront=*/1));
       }
       // 4. Insert the extractedSource into the res vector.
-      res = rewriter.create<InsertOp>(loc, extractedSource, res, off);
+      res = InsertOp::create(rewriter, loc, extractedSource, res, off);
     }
 
     rewriter.replaceOp(op, res);
@@ -250,12 +250,12 @@ class Convert1DExtractStridedSliceIntoExtractInsertChain final
     SmallVector<Value> elements;
     elements.reserve(size);
     for (int64_t i = offset, e = offset + size * stride; i < e; i += stride)
-      elements.push_back(rewriter.create<ExtractOp>(loc, op.getVector(), i));
+      elements.push_back(ExtractOp::create(rewriter, loc, op.getVector(), i));
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(op.getType()));
     for (int64_t i = 0; i < size; ++i)
-      result = rewriter.create<InsertOp>(loc, elements[i], result, i);
+      result = InsertOp::create(rewriter, loc, elements[i], result, i);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -301,17 +301,17 @@ class DecomposeNDExtractStridedSlice
       return failure();
 
     // Extract/insert on a lower ranked extract strided slice op.
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value res = rewriter.create<SplatOp>(loc, dstType, zero);
+    Value res = SplatOp::create(rewriter, loc, dstType, zero);
     for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
          off += stride, ++idx) {
-      Value one = rewriter.create<ExtractOp>(loc, op.getVector(), off);
-      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+      Value one = ExtractOp::create(rewriter, loc, op.getVector(), off);
+      Value extracted = ExtractStridedSliceOp::create(rewriter,
           loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
           getI64SubArray(op.getSizes(), /* dropFront=*/1),
           getI64SubArray(op.getStrides(), /* dropFront=*/1));
-      res = rewriter.create<InsertOp>(loc, extracted, res, idx);
+      res = InsertOp::create(rewriter, loc, extracted, res, idx);
     }
     rewriter.replaceOp(op, res);
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
index 7cac1cbafdd64..880506142bfdd 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
@@ -634,7 +634,7 @@ struct LinearizeVectorCreateMask final
     // The result of the comparison is then multiplied with
     // the second operand of create_mask to get the 1D mask.
     auto firstOperand = adaptor.getOperands().front();
-    auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto isNonZero = rewriter.createOrFold<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::sgt, firstOperand, zero);
     auto isNonZeroIndex = rewriter.createOrFold<mlir::arith::IndexCastOp>(
@@ -644,7 +644,7 @@ struct LinearizeVectorCreateMask final
         loc, rewriter.getIndexType(), isNonZeroIndex, secondOperand);
 
     auto newMask =
-        rewriter.create<mlir::vector::CreateMaskOp>(loc, dstTy, maskSize);
+        mlir::vector::CreateMaskOp::create(rewriter, loc, dstTy, maskSize);
     rewriter.replaceOp(createMaskOp, newMask);
     return success();
   }
@@ -721,7 +721,7 @@ void mlir::vector::populateForVectorLinearize(TypeConverter &typeConverter,
     if (!isa<VectorType>(type) || !isa<VectorType>(value.getType()))
       return nullptr;
 
-    return builder.create<vector::ShapeCastOp>(loc, type, value);
+    return vector::ShapeCastOp::create(builder, loc, type, value);
   };
   typeConverter.addSourceMaterialization(materializeCast);
   typeConverter.addTargetMaterialization(materializeCast);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
index 363108238e596..70d786a516859 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
@@ -84,7 +84,7 @@ LogicalResult resolveAllTrueCreateMaskOp(IRRewriter &rewriter,
   // Replace createMaskOp with an all-true constant. This should result in the
   // mask being removed in most cases (as xfer ops + vector.mask have folds to
   // remove all-true masks).
-  auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+  auto allTrue = vector::ConstantMaskOp::create(rewriter,
       createMaskOp.getLoc(), maskType, ConstantMaskKind::AllTrue);
   rewriter.replaceAllUsesWith(createMaskOp, allTrue);
   return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index efdae93e730bd..aa38fc20dfb70 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -286,7 +286,7 @@ static Value rankReducingSubviewDroppingUnitDims(PatternRewriter &rewriter,
   if (resultType.canonicalizeStridedLayout() ==
       inputType.canonicalizeStridedLayout())
     return input;
-  return rewriter.create<memref::SubViewOp>(loc, resultType, input, offsets,
+  return memref::SubViewOp::create(rewriter, loc, resultType, input, offsets,
                                             sizes, strides);
 }
 
@@ -395,11 +395,11 @@ class TransferReadDropUnitDimsPattern
 
     Value reducedShapeSource =
         rankReducingSubviewDroppingUnitDims(rewriter, loc, source);
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
-    Operation *newTransferReadOp = rewriter.create<vector::TransferReadOp>(
+    Operation *newTransferReadOp = vector::TransferReadOp::create(rewriter,
         loc, reducedVectorType, reducedShapeSource, zeros, identityMap,
         transferReadOp.getPadding(), maskOp,
         rewriter.getBoolArrayAttr(inBounds));
@@ -477,13 +477,13 @@ class TransferWriteDropUnitDimsPattern
     }
     Value reducedShapeSource =
         rankReducingSubviewDroppingUnitDims(rewriter, loc, source);
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
     auto shapeCastSrc = rewriter.createOrFold<vector::ShapeCastOp>(
         loc, reducedVectorType, vector);
-    Operation *newXferWrite = rewriter.create<vector::TransferWriteOp>(
+    Operation *newXferWrite = vector::TransferWriteOp::create(rewriter,
         loc, Type(), shapeCastSrc, reducedShapeSource, zeros, identityMap,
         maskOp, rewriter.getBoolArrayAttr(inBounds));
 
@@ -520,7 +520,7 @@ static Value collapseInnerDims(PatternRewriter &rewriter, mlir::Location loc,
   for (int64_t i = firstDimToCollapse; i < inputType.getRank(); ++i)
     collapsedIndices.push_back(i);
   reassociation.push_back(collapsedIndices);
-  return rewriter.create<memref::CollapseShapeOp>(loc, input, reassociation);
+  return memref::CollapseShapeOp::create(rewriter, loc, input, reassociation);
 }
 
 /// Returns the new indices that collapses the inner dimensions starting from
@@ -559,7 +559,7 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   // one would get the following offset:
   //    %offset = %arg0 * 43
   OpFoldResult collapsedOffset =
-      rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult();
+      arith::ConstantIndexOp::create(rewriter, loc, 0).getResult();
 
   auto collapsedStrides = computeSuffixProduct(
       ArrayRef<int64_t>(shape.begin() + firstDimToCollapse, shape.end()));
@@ -573,7 +573,7 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   if (auto value = dyn_cast<Value>(collapsedOffset)) {
     indicesAfterCollapsing.push_back(value);
   } else {
-    indicesAfterCollapsing.push_back(rewriter.create<arith::ConstantIndexOp>(
+    indicesAfterCollapsing.push_back(arith::ConstantIndexOp::create(rewriter,
         loc, *getConstantIntValue(collapsedOffset)));
   }
 
@@ -659,7 +659,7 @@ class FlattenContiguousRowMajorTransferReadPattern
     // 3. Create new vector.transfer_read that reads from the collapsed memref
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
-    vector::TransferReadOp flatRead = rewriter.create<vector::TransferReadOp>(
+    vector::TransferReadOp flatRead = vector::TransferReadOp::create(rewriter,
         loc, flatVectorType, collapsedSource, collapsedIndices,
         transferReadOp.getPadding(), collapsedMap);
     flatRead.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
@@ -757,9 +757,9 @@ class FlattenContiguousRowMajorTransferWritePattern
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
     Value flatVector =
-        rewriter.create<vector::ShapeCastOp>(loc, flatVectorType, vector);
+        vector::ShapeCastOp::create(rewriter, loc, flatVectorType, vector);
     vector::TransferWriteOp flatWrite =
-        rewriter.create<vector::TransferWriteOp>(
+        vector::TransferWriteOp::create(rewriter,
             loc, flatVector, collapsedSource, collapsedIndices, collapsedMap);
     flatWrite.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
@@ -855,7 +855,7 @@ class RewriteScalarExtractElementOfTransferRead
         newIndices[newIndices.size() - 1] = value;
       } else {
         newIndices[newIndices.size() - 1] =
-            rewriter.create<arith::ConstantIndexOp>(loc,
+            arith::ConstantIndexOp::create(rewriter, loc,
                                                     *getConstantIntValue(ofr));
       }
     }
@@ -917,7 +917,7 @@ class RewriteScalarExtractOfTransferRead
       if (auto value = dyn_cast<Value>(composedIdx)) {
         newIndices[idx] = value;
       } else {
-        newIndices[idx] = rewriter.create<arith::ConstantIndexOp>(
+        newIndices[idx] = arith::ConstantIndexOp::create(rewriter,
             extractOp.getLoc(), *getConstantIntValue(composedIdx));
       }
     }
@@ -952,7 +952,7 @@ class RewriteScalarWrite : public OpRewritePattern<vector::TransferWriteOp> {
       return failure();
     // Only float and integer element types are supported.
     Value scalar =
-        rewriter.create<vector::ExtractOp>(xferOp.getLoc(), xferOp.getVector());
+        vector::ExtractOp::create(rewriter, xferOp.getLoc(), xferOp.getVector());
     // Construct a scalar store.
     if (isa<MemRefType>(xferOp.getBase().getType())) {
       rewriter.replaceOpWithNewOp<memref::StoreOp>(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
index 256c8cb69b1ba..a1f36a9744932 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
@@ -64,12 +64,12 @@ static Value createInBoundsCond(RewriterBase &b,
     if (maybeCstSum && maybeCstDimSz && *maybeCstSum <= *maybeCstDimSz)
       return;
     Value cond =
-        b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sle,
+        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sle,
                                 getValueOrCreateConstantIndexOp(b, loc, sum),
                                 getValueOrCreateConstantIndexOp(b, loc, dimSz));
     // Conjunction over all dims for which we are in-bounds.
     if (inBoundsCond)
-      inBoundsCond = b.create<arith::AndIOp>(loc, inBoundsCond, cond);
+      inBoundsCond = arith::AndIOp::create(b, loc, inBoundsCond, cond);
     else
       inBoundsCond = cond;
   });
@@ -177,11 +177,11 @@ static Value castToCompatibleMemRefType(OpBuilder &b, Value memref,
     sourceType = MemRefType::get(
         sourceType.getShape(), sourceType.getElementType(),
         sourceType.getLayout(), compatibleMemRefType.getMemorySpace());
-    res = b.create<memref::MemorySpaceCastOp>(memref.getLoc(), sourceType, res);
+    res = memref::MemorySpaceCastOp::create(b, memref.getLoc(), sourceType, res);
   }
   if (sourceType == compatibleMemRefType)
     return res;
-  return b.create<memref::CastOp>(memref.getLoc(), compatibleMemRefType, res);
+  return memref::CastOp::create(b, memref.getLoc(), compatibleMemRefType, res);
 }
 
 /// Operates under a scoped context to build the intersection between the
@@ -203,15 +203,15 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
   xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
     using MapList = ArrayRef<ArrayRef<AffineExpr>>;
     Value dimMemRef =
-        b.create<memref::DimOp>(xferOp.getLoc(), xferOp.getBase(), indicesIdx);
-    Value dimAlloc = b.create<memref::DimOp>(loc, alloc, resultIdx);
+        memref::DimOp::create(b, xferOp.getLoc(), xferOp.getBase(), indicesIdx);
+    Value dimAlloc = memref::DimOp::create(b, loc, alloc, resultIdx);
     Value index = xferOp.getIndices()[indicesIdx];
     AffineExpr i, j, k;
     bindDims(xferOp.getContext(), i, j, k);
     SmallVector<AffineMap, 4> maps =
         AffineMap::inferFromExprList(MapList{{i - j, k}}, b.getContext());
     // affine_min(%dimMemRef - %index, %dimAlloc)
-    Value affineMin = b.create<affine::AffineMinOp>(
+    Value affineMin = affine::AffineMinOp::create(b,
         loc, index.getType(), maps[0], ValueRange{dimMemRef, index, dimAlloc});
     sizes.push_back(affineMin);
   });
@@ -220,9 +220,9 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
       xferOp.getIndices(), [](Value idx) -> OpFoldResult { return idx; }));
   SmallVector<OpFoldResult> destIndices(memrefRank, b.getIndexAttr(0));
   SmallVector<OpFoldResult> strides(memrefRank, b.getIndexAttr(1));
-  auto copySrc = b.create<memref::SubViewOp>(
+  auto copySrc = memref::SubViewOp::create(b,
       loc, isaWrite ? alloc : xferOp.getBase(), srcIndices, sizes, strides);
-  auto copyDest = b.create<memref::SubViewOp>(
+  auto copyDest = memref::SubViewOp::create(b,
       loc, isaWrite ? xferOp.getBase() : alloc, destIndices, sizes, strides);
   return std::make_pair(copySrc, copyDest);
 }
@@ -251,18 +251,18 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
                             TypeRange returnTypes, Value inBoundsCond,
                             MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return b.create<scf::IfOp>(
+  return scf::IfOp::create(b,
       loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
         llvm::append_range(viewAndIndices, xferOp.getIndices());
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       },
       [&](OpBuilder &b, Location loc) {
-        b.create<linalg::FillOp>(loc, ValueRange{xferOp.getPadding()},
+        linalg::FillOp::create(b, loc, ValueRange{xferOp.getPadding()},
                                  ValueRange{alloc});
         // Take partial subview of memref which guarantees no dimension
         // overflows.
@@ -270,13 +270,13 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
         std::pair<Value, Value> copyArgs = createSubViewIntersection(
             rewriter, cast<VectorTransferOpInterface>(xferOp.getOperation()),
             alloc);
-        b.create<memref::CopyOp>(loc, copyArgs.first, copyArgs.second);
+        memref::CopyOp::create(b, loc, copyArgs.first, copyArgs.second);
         Value casted =
             castToCompatibleMemRefType(b, alloc, compatibleMemRefType);
         scf::ValueVector viewAndIndices{casted};
         viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
                               zero);
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       });
 }
 
@@ -304,22 +304,22 @@ static scf::IfOp createFullPartialVectorTransferRead(
     Value inBoundsCond, MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
   scf::IfOp fullPartialIfOp;
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return b.create<scf::IfOp>(
+  return scf::IfOp::create(b,
       loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
         llvm::append_range(viewAndIndices, xferOp.getIndices());
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       },
       [&](OpBuilder &b, Location loc) {
         Operation *newXfer = b.clone(*xferOp.getOperation());
         Value vector = cast<VectorTransferOpInterface>(newXfer).getVector();
-        b.create<memref::StoreOp>(
+        memref::StoreOp::create(b,
             loc, vector,
-            b.create<vector::TypeCastOp>(
+            vector::TypeCastOp::create(b,
                 loc, MemRefType::get({}, vector.getType()), alloc));
 
         Value casted =
@@ -327,7 +327,7 @@ static scf::IfOp createFullPartialVectorTransferRead(
         scf::ValueVector viewAndIndices{casted};
         viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
                               zero);
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       });
 }
 
@@ -351,7 +351,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
                           TypeRange returnTypes, Value inBoundsCond,
                           MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
   return b
       .create<scf::IfOp>(
@@ -361,7 +361,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
                 castToCompatibleMemRefType(b, memref, compatibleMemRefType);
             scf::ValueVector viewAndIndices{res};
             llvm::append_range(viewAndIndices, xferOp.getIndices());
-            b.create<scf::YieldOp>(loc, viewAndIndices);
+            scf::YieldOp::create(b, loc, viewAndIndices);
           },
           [&](OpBuilder &b, Location loc) {
             Value casted =
@@ -369,7 +369,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
             scf::ValueVector viewAndIndices{casted};
             viewAndIndices.insert(viewAndIndices.end(),
                                   xferOp.getTransferRank(), zero);
-            b.create<scf::YieldOp>(loc, viewAndIndices);
+            scf::YieldOp::create(b, loc, viewAndIndices);
           })
       ->getResults();
 }
@@ -391,15 +391,15 @@ static void createFullPartialLinalgCopy(RewriterBase &b,
                                         vector::TransferWriteOp xferOp,
                                         Value inBoundsCond, Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = b.create<arith::XOrIOp>(
-      loc, inBoundsCond, b.create<arith::ConstantIntOp>(loc, true, 1));
-  b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
+  auto notInBounds = arith::XOrIOp::create(b,
+      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRRewriter rewriter(b);
     std::pair<Value, Value> copyArgs = createSubViewIntersection(
         rewriter, cast<VectorTransferOpInterface>(xferOp.getOperation()),
         alloc);
-    b.create<memref::CopyOp>(loc, copyArgs.first, copyArgs.second);
-    b.create<scf::YieldOp>(loc, ValueRange{});
+    memref::CopyOp::create(b, loc, copyArgs.first, copyArgs.second);
+    scf::YieldOp::create(b, loc, ValueRange{});
   });
 }
 
@@ -420,18 +420,18 @@ static void createFullPartialVectorTransferWrite(RewriterBase &b,
                                                  Value inBoundsCond,
                                                  Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = b.create<arith::XOrIOp>(
-      loc, inBoundsCond, b.create<arith::ConstantIntOp>(loc, true, 1));
-  b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
+  auto notInBounds = arith::XOrIOp::create(b,
+      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRMapping mapping;
-    Value load = b.create<memref::LoadOp>(
+    Value load = memref::LoadOp::create(b,
         loc,
-        b.create<vector::TypeCastOp>(
+        vector::TypeCastOp::create(b,
             loc, MemRefType::get({}, xferOp.getVector().getType()), alloc),
         ValueRange());
     mapping.map(xferOp.getVector(), load);
     b.clone(*xferOp.getOperation(), mapping);
-    b.create<scf::YieldOp>(loc, ValueRange{});
+    scf::YieldOp::create(b, loc, ValueRange{});
   });
 }
 
@@ -561,7 +561,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
     b.setInsertionPointToStart(&scope->getRegion(0).front());
     auto shape = xferOp.getVectorType().getShape();
     Type elementType = xferOp.getVectorType().getElementType();
-    alloc = b.create<memref::AllocaOp>(scope->getLoc(),
+    alloc = memref::AllocaOp::create(b, scope->getLoc(),
                                        MemRefType::get(shape, elementType),
                                        ValueRange{}, b.getI64IntegerAttr(32));
   }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index bcaea1c79471f..2e54c89df7800 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -381,7 +381,7 @@ FailureOr<Value> combineContractAndBroadcast(vector::ContractionOp contractOp,
   if (getUnusedDimsBitVector({maps[0], maps[1]}).any())
     return failure();
 
-  Operation *newOp = rewriter.create<vector::ContractionOp>(
+  Operation *newOp = vector::ContractionOp::create(rewriter,
       contractOp.getLoc(), lhs, rhs, contractOp.getAcc(),
       rewriter.getAffineMapArrayAttr(maps), rewriter.getArrayAttr(iterators));
 
@@ -534,7 +534,7 @@ struct ReorderElementwiseOpsOnTranspose final
         // This is a constant. Create a reverse transpose op for it.
         auto vectorType =
             srcType.clone(cast<VectorType>(operand.getType()).getElementType());
-        srcValues.push_back(rewriter.create<vector::TransposeOp>(
+        srcValues.push_back(vector::TransposeOp::create(rewriter,
             operand.getLoc(), vectorType, operand, invOrder));
       }
     }
@@ -608,12 +608,12 @@ struct BubbleDownVectorBitCastForExtract
     // Get the single scalar (as a vector) in the source value that packs the
     // desired scalar. E.g. extract vector<1xf32> from vector<4xf32>
     Location loc = extractOp.getLoc();
-    Value packedValue = rewriter.create<vector::ExtractOp>(
+    Value packedValue = vector::ExtractOp::create(rewriter,
         loc, castOp.getSource(), index / expandRatio);
     Type packedVecType = VectorType::get(/*shape=*/{1}, packedValue.getType());
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, packedVecType, rewriter.getZeroAttr(packedVecType));
-    packedValue = rewriter.create<vector::InsertOp>(loc, packedValue, zero,
+    packedValue = vector::InsertOp::create(rewriter, loc, packedValue, zero,
                                                     /*position=*/0);
 
     // Cast it to a vector with the desired scalar's type.
@@ -621,7 +621,7 @@ struct BubbleDownVectorBitCastForExtract
     VectorType packedType =
         VectorType::get({expandRatio}, castDstType.getElementType());
     Value castedValue =
-        rewriter.create<vector::BitCastOp>(loc, packedType, packedValue);
+        vector::BitCastOp::create(rewriter, loc, packedType, packedValue);
 
     // Finally extract the desired scalar.
     rewriter.replaceOpWithNewOp<vector::ExtractOp>(extractOp, castedValue,
@@ -700,7 +700,7 @@ struct BubbleDownBitCastForStridedSliceExtract
     VectorType newExtractType =
         VectorType::get(dims, castSrcType.getElementType());
 
-    auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
         extractOp.getLoc(), newExtractType, castOp.getSource(), newOffsets,
         newSizes, extractOp.getStrides());
 
@@ -761,7 +761,7 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         isNumElemsShrink ? srcDims.back() / ratio : srcDims.back() * ratio;
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
-    auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
+    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims(insertOp.getDestVectorType().getShape());
@@ -771,7 +771,7 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(dstDims, castDstType.getElementType());
 
     // Bitcast the destination.
-    auto newCastDstOp = rewriter.create<vector::BitCastOp>(
+    auto newCastDstOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     // Generate new insert.
@@ -852,7 +852,7 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
 
-    auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
+    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims =
@@ -861,7 +861,7 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastDstType =
         VectorType::get(dstDims, castDstType.getElementType());
 
-    auto newCastDstOp = rewriter.create<vector::BitCastOp>(
+    auto newCastDstOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     rewriter.replaceOpWithNewOp<vector::InsertStridedSliceOp>(
@@ -936,9 +936,9 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
     Type elemType = castDstType.getElementType();
     assert(elemType.isSignlessIntOrIndexOrFloat());
 
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value res = rewriter.create<SplatOp>(loc, castDstType, zero);
+    Value res = SplatOp::create(rewriter, loc, castDstType, zero);
 
     SmallVector<int64_t> sliceShape = {castDstLastDim};
     SmallVector<int64_t> strides = {1};
@@ -947,12 +947,12 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
                         castDstType.getElementType());
 
     for (int i = 0, e = shrinkRatio; i < e; ++i) {
-      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+      Value extracted = ExtractStridedSliceOp::create(rewriter,
           loc, bitcastOp.getSource(), ArrayRef<int64_t>{i * castDstLastDim},
           sliceShape, strides);
       Value bitcast =
-          rewriter.create<BitCastOp>(loc, newCastDstType, extracted);
-      res = rewriter.create<InsertStridedSliceOp>(
+          BitCastOp::create(rewriter, loc, newCastDstType, extracted);
+      res = InsertStridedSliceOp::create(rewriter,
           loc, bitcast, res,
           ArrayRef<int64_t>{i * castDstLastDim / shrinkRatio}, strides);
     }
@@ -1097,7 +1097,7 @@ class ExtractOpFromElementwise final
     Location loc = eltwise->getLoc();
     SmallVector<OpFoldResult> pos = op.getMixedPosition();
     for (Value arg : eltwise->getOperands()) {
-      Value newArg = rewriter.create<vector::ExtractOp>(loc, arg, pos);
+      Value newArg = vector::ExtractOp::create(rewriter, loc, arg, pos);
       mapping.map(arg, newArg);
     }
 
@@ -1286,18 +1286,18 @@ static Value buildVectorComparison(PatternRewriter &rewriter, Operation *op,
     indicesAttr = rewriter.getI64VectorAttr(
         llvm::to_vector<4>(llvm::seq<int64_t>(0, dim)));
   }
-  Value indices = rewriter.create<arith::ConstantOp>(loc, indicesAttr);
+  Value indices = arith::ConstantOp::create(rewriter, loc, indicesAttr);
   // Add in an offset if requested.
   if (off) {
     Value o = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, *off);
-    Value ov = rewriter.create<vector::SplatOp>(loc, indices.getType(), o);
-    indices = rewriter.create<arith::AddIOp>(loc, ov, indices);
+    Value ov = vector::SplatOp::create(rewriter, loc, indices.getType(), o);
+    indices = arith::AddIOp::create(rewriter, loc, ov, indices);
   }
   // Construct the vector comparison.
   Value bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, b);
   Value bounds =
-      rewriter.create<vector::SplatOp>(loc, indices.getType(), bound);
-  return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, indices,
+      vector::SplatOp::create(rewriter, loc, indices.getType(), bound);
+  return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, indices,
                                         bounds);
 }
 
@@ -1329,15 +1329,15 @@ struct MaterializeTransferMask : public OpRewritePattern<ConcreteOp> {
     Value off = xferOp.getIndices()[lastIndex];
     Value dim =
         vector::createOrFoldDimOp(rewriter, loc, xferOp.getBase(), lastIndex);
-    Value b = rewriter.create<arith::SubIOp>(loc, dim.getType(), dim, off);
-    Value mask = rewriter.create<vector::CreateMaskOp>(
+    Value b = arith::SubIOp::create(rewriter, loc, dim.getType(), dim, off);
+    Value mask = vector::CreateMaskOp::create(rewriter,
         loc,
         VectorType::get(vtp.getShape(), rewriter.getI1Type(),
                         vtp.getScalableDims()),
         b);
     if (xferOp.getMask()) {
       // Intersect the in-bounds with the mask specified as an op parameter.
-      mask = rewriter.create<arith::AndIOp>(loc, mask, xferOp.getMask());
+      mask = arith::AndIOp::create(rewriter, loc, mask, xferOp.getMask());
     }
 
     rewriter.modifyOpInPlace(xferOp, [&]() {
@@ -1542,11 +1542,11 @@ class DropInnerMostUnitDimsTransferRead
         strides);
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         readOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
-    Value rankedReducedView = rewriter.create<memref::SubViewOp>(
+    Value rankedReducedView = memref::SubViewOp::create(rewriter,
         loc, resultMemrefType, readOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
-    Value result = rewriter.create<vector::TransferReadOp>(
+    Value result = vector::TransferReadOp::create(rewriter,
         loc, resultTargetVecType, rankedReducedView,
         readOp.getIndices().drop_back(dimsToDrop), AffineMapAttr::get(permMap),
         readOp.getPadding(),
@@ -1633,7 +1633,7 @@ class DropInnerMostUnitDimsTransferWrite
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         writeOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
 
-    Value rankedReducedView = rewriter.create<memref::SubViewOp>(
+    Value rankedReducedView = memref::SubViewOp::create(rewriter,
         loc, resultMemrefType, writeOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
@@ -1702,21 +1702,21 @@ struct CanonicalizeContractMatmulToMMT final
     auto createTranspose = [&rewriter, loc](Value mat) -> Value {
       if (auto sext = mat.getDefiningOp<arith::ExtSIOp>()) {
         Value trans =
-            rewriter.create<vector::TransposeOp>(loc, sext.getIn(), perm);
+            vector::TransposeOp::create(rewriter, loc, sext.getIn(), perm);
         VectorType newType =
             cast<VectorType>(trans.getType())
                 .clone(cast<VectorType>(mat.getType()).getElementType());
-        return rewriter.create<arith::ExtSIOp>(loc, newType, trans);
+        return arith::ExtSIOp::create(rewriter, loc, newType, trans);
       }
       if (auto zext = mat.getDefiningOp<arith::ExtUIOp>()) {
         Value trans =
-            rewriter.create<vector::TransposeOp>(loc, zext.getIn(), perm);
+            vector::TransposeOp::create(rewriter, loc, zext.getIn(), perm);
         VectorType newType =
             VectorType::get(cast<VectorType>(trans.getType()).getShape(),
                             cast<VectorType>(mat.getType()).getElementType());
-        return rewriter.create<arith::ExtUIOp>(loc, newType, trans);
+        return arith::ExtUIOp::create(rewriter, loc, newType, trans);
       }
-      return rewriter.create<vector::TransposeOp>(loc, mat, perm);
+      return vector::TransposeOp::create(rewriter, loc, mat, perm);
     };
 
     if (maps == infer({{m, k}, {k, n}, {m, n}})) {
@@ -1830,7 +1830,7 @@ struct ChainedReduction final : OpRewritePattern<vector::ReductionOp> {
       vAdd = rewriter.createOrFold<arith::AddIOp>(
           loc, parentReduction.getVector(), op.getVector());
     } else {
-      vAdd = rewriter.create<arith::AddFOp>(loc, parentReduction.getVector(),
+      vAdd = arith::AddFOp::create(rewriter, loc, parentReduction.getVector(),
                                             op.getVector());
     }
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), vAdd,
@@ -1919,7 +1919,7 @@ struct DropUnitDimFromElementwiseOps final
       if (newVType == opVectorType)
         return rewriter.notifyMatchFailure(op, "No unit dimension to remove.");
 
-      auto opSC = rewriter.create<vector::ShapeCastOp>(loc, newVType, operand);
+      auto opSC = vector::ShapeCastOp::create(rewriter, loc, newVType, operand);
       newOperands.push_back(opSC);
     }
 
@@ -1998,11 +1998,11 @@ struct DropUnitDimsFromTransposeOp final
 
     Location loc = op.getLoc();
     // Drop the unit dims via shape_cast.
-    auto dropDimsShapeCast = rewriter.create<vector::ShapeCastOp>(
+    auto dropDimsShapeCast = vector::ShapeCastOp::create(rewriter,
         loc, sourceTypeWithoutUnitDims, op.getVector());
     // Create the new transpose.
     auto transposeWithoutUnitDims =
-        rewriter.create<vector::TransposeOp>(loc, dropDimsShapeCast, newPerm);
+        vector::TransposeOp::create(rewriter, loc, dropDimsShapeCast, newPerm);
     // Restore the unit dims via shape cast.
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
         op, op.getResultVectorType(), transposeWithoutUnitDims);
@@ -2053,7 +2053,7 @@ struct DropUnitDimsFromScfForOp final : OpRewritePattern<scf::ForOp> {
 
       // Create a new ForOp with that iter operand replaced.
       auto castFn = [](OpBuilder &b, Location loc, Type type, Value source) {
-        return b.create<vector::ShapeCastOp>(loc, type, source);
+        return vector::ShapeCastOp::create(b, loc, type, source);
       };
 
       Value replacement =
@@ -2105,7 +2105,7 @@ struct ReduceRedundantZero final : OpRewritePattern<vector::ReductionOp> {
     if (!matchPattern(addLhs.getRhs(), m_AnyZeroFloat()))
       return failure();
 
-    auto newAdd = rewriter.create<arith::AddFOp>(vAdd.getLoc(), addLhs.getLhs(),
+    auto newAdd = arith::AddFOp::create(rewriter, vAdd.getLoc(), addLhs.getLhs(),
                                                  vAdd.getRhs());
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), newAdd,
                                                      op.getAcc());
@@ -2148,7 +2148,7 @@ struct BreakDownVectorReduction final : OpRewritePattern<vector::ReductionOp> {
     Location loc = op.getLoc();
     SmallVector<Value> extracted(numElems, nullptr);
     for (auto [idx, extractedElem] : llvm::enumerate(extracted))
-      extractedElem = rewriter.create<vector::ExtractOp>(
+      extractedElem = vector::ExtractOp::create(rewriter,
           loc, op.getVector(), static_cast<int64_t>(idx));
 
     Value res = extracted.front();
@@ -2228,7 +2228,7 @@ struct FoldArithToVectorOuterProduct : public OpRewritePattern<MulOpType> {
       if (!broadcastedRhs || !isValidBroadcastSource(broadcastedRhs))
         return failure();
 
-      return rewriter.create<vector::OuterProductOp>(
+      return vector::OuterProductOp::create(rewriter,
           mulOp->getLoc(), resType, broadcastedLhs.getSource(),
           broadcastedRhs.getSource(), Value(), vector::CombiningKind::ADD);
     };
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 693f4f955994d..9fa01061c1bed 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -49,7 +49,7 @@ static SmallVector<Value> sliceTransferIndices(ArrayRef<int64_t> elementOffsets,
                 getAffineConstantExpr(elementOffsets[dim.index()], ctx);
     auto map = AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, expr);
     slicedIndices[pos] =
-        builder.create<affine::AffineApplyOp>(loc, map, indices[pos]);
+        affine::AffineApplyOp::create(builder, loc, map, indices[pos]);
   }
   return slicedIndices;
 }
@@ -68,9 +68,9 @@ static SmallVector<Value> sliceLoadStoreIndices(PatternRewriter &rewriter,
   auto start = indices.size() - offsets.size();
   for (auto [i, offset] : llvm::enumerate(offsets)) {
     if (offset != 0) {
-      indices[start + i] = rewriter.create<arith::AddIOp>(
+      indices[start + i] = arith::AddIOp::create(rewriter,
           loc, originalIndices[start + i],
-          rewriter.create<arith::ConstantIndexOp>(loc, offset));
+          arith::ConstantIndexOp::create(rewriter, loc, offset));
     }
   }
   return indices;
@@ -172,7 +172,7 @@ struct UnrollTransferReadPattern
     ArrayRef<int64_t> originalSize = readOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
@@ -185,7 +185,7 @@ struct UnrollTransferReadPattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                readOp.getPermutationMap(), loc, rewriter);
-      auto slicedRead = rewriter.create<vector::TransferReadOp>(
+      auto slicedRead = vector::TransferReadOp::create(rewriter,
           loc, targetType, readOp.getBase(), indices,
           readOp.getPermutationMapAttr(), readOp.getPadding(), readOp.getMask(),
           readOp.getInBoundsAttr());
@@ -236,7 +236,7 @@ struct UnrollTransferWritePattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                writeOp.getPermutationMap(), loc, rewriter);
-      Operation *slicedWrite = rewriter.create<vector::TransferWriteOp>(
+      Operation *slicedWrite = vector::TransferWriteOp::create(rewriter,
           loc, slicedVector, resultTensor ? resultTensor : writeOp.getBase(),
           indices, writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
       // For the tensor case update the destination for the next transfer write.
@@ -348,7 +348,7 @@ struct UnrollContractionPattern
       accCache[dstOffets] = newOp->getResult(0);
     }
     // Assemble back the accumulator into a single vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
@@ -427,7 +427,7 @@ struct UnrollMultiReductionPattern
       accCache[destOffset] = result;
     }
     // Assemble back the accumulator into a single vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, reductionOp.getDestType(),
         rewriter.getZeroAttr(reductionOp.getDestType()));
     for (const auto &it : accCache) {
@@ -468,7 +468,7 @@ struct UnrollElementwisePattern : public RewritePattern {
           op, "expected input vector rank to match target shape rank");
     Location loc = op->getLoc();
     // Prepare the result vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     SmallVector<int64_t> strides(targetShape->size(), 1);
     VectorType newVecType =
@@ -567,7 +567,7 @@ struct UnrollTransposePattern : public OpRewritePattern<vector::TransposeOp> {
     ArrayRef<int64_t> originalSize = originalVectorType.getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, originalVectorType, rewriter.getZeroAttr(originalVectorType));
     ArrayRef<int64_t> permutation = transposeOp.getPermutation();
 
@@ -618,7 +618,7 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
     ArrayRef<int64_t> originalSize = gatherOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
@@ -638,7 +638,7 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
           rewriter.createOrFold<vector::ExtractStridedSliceOp>(
               loc, gatherOp.getPassThru(), elementOffsets, *targetShape,
               strides);
-      auto slicedGather = rewriter.create<vector::GatherOp>(
+      auto slicedGather = vector::GatherOp::create(rewriter,
           loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
           indexSubVec, maskSubVec, passThruSubVec);
 
@@ -671,7 +671,7 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
     ArrayRef<int64_t> originalShape = vecType.getShape();
     SmallVector<int64_t> strides(targetShape->size(), 1);
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, vecType, rewriter.getZeroAttr(vecType));
 
     SmallVector<int64_t> loopOrder =
@@ -684,7 +684,7 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
          StaticTileOffsetRange(originalShape, *targetShape, loopOrder)) {
       SmallVector<Value> indices =
           sliceLoadStoreIndices(rewriter, loc, loadOp.getIndices(), offsets);
-      Value slicedLoad = rewriter.create<vector::LoadOp>(
+      Value slicedLoad = vector::LoadOp::create(rewriter,
           loc, targetVecType, loadOp.getBase(), indices);
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, slicedLoad, result, offsets, strides);
@@ -727,7 +727,7 @@ struct UnrollStorePattern : public OpRewritePattern<vector::StoreOp> {
           sliceLoadStoreIndices(rewriter, loc, storeOp.getIndices(), offsets);
       Value slice = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
           loc, vector, offsets, *targetShape, strides);
-      rewriter.create<vector::StoreOp>(loc, slice, base, indices);
+      vector::StoreOp::create(rewriter, loc, slice, base, indices);
     }
     rewriter.eraseOp(storeOp);
     return success();
@@ -755,7 +755,7 @@ struct UnrollBroadcastPattern : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType = broadcastOp.getResultVectorType();
     VectorType targetType =
         resType.cloneWith(*targetShape, resType.getElementType());
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
 
     SmallVector<int64_t> originalShape = *broadcastOp.getShapeForUnroll();
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 7e4984582b373..7c31ada1644c5 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -331,7 +331,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
   assert(padValue.getType() == sourceShapedType.getElementType() &&
          "expected same pad element type to match source element type");
   int64_t readRank = inputVectorSizes.size();
-  auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+  auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
   SmallVector<bool> inBoundsVal(readRank, true);
 
   if (useInBoundsInsteadOfMasking) {
@@ -341,7 +341,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
       inBoundsVal[i] = (sourceShape[i] == inputVectorSizes[i]) &&
                        ShapedType::isStatic(sourceShape[i]);
   }
-  auto transferReadOp = builder.create<vector::TransferReadOp>(
+  auto transferReadOp = vector::TransferReadOp::create(builder,
       loc,
       /*vectorType=*/vectorType,
       /*source=*/source,
@@ -356,7 +356,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
 
   auto maskType = VectorType::get(inputVectorSizes, builder.getI1Type());
   Value mask =
-      builder.create<vector::CreateMaskOp>(loc, maskType, mixedSourceDims);
+      vector::CreateMaskOp::create(builder, loc, maskType, mixedSourceDims);
   return mlir::vector::maskOperation(builder, transferReadOp, mask)
       ->getResult(0);
 }
diff --git a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
index cc7ab7f3f3895..eafe3a97758c1 100644
--- a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
+++ b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
@@ -63,11 +63,11 @@ SmallVector<Value> x86vector::MaskCompressOp::getIntrinsicOperands(
   if (adaptor.getSrc()) {
     src = adaptor.getSrc();
   } else if (adaptor.getConstantSrc()) {
-    src = rewriter.create<LLVM::ConstantOp>(loc, opType,
+    src = LLVM::ConstantOp::create(rewriter, loc, opType,
                                             adaptor.getConstantSrcAttr());
   } else {
     auto zeroAttr = rewriter.getZeroAttr(opType);
-    src = rewriter.create<LLVM::ConstantOp>(loc, opType, zeroAttr);
+    src = LLVM::ConstantOp::create(rewriter, loc, opType, zeroAttr);
   }
 
   return SmallVector<Value>{adaptor.getA(), src, adaptor.getK()};
@@ -80,7 +80,7 @@ x86vector::DotOp::getIntrinsicOperands(ArrayRef<Value> operands,
   SmallVector<Value> intrinsicOperands(operands);
   // Dot product of all elements, broadcasted to all elements.
   Value scale =
-      rewriter.create<LLVM::ConstantOp>(getLoc(), rewriter.getI8Type(), 0xff);
+      LLVM::ConstantOp::create(rewriter, getLoc(), rewriter.getI8Type(), 0xff);
   intrinsicOperands.push_back(scale);
 
   return intrinsicOperands;
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
index 63f725084cabf..8f87661a68d96 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
+++ b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
@@ -38,7 +38,7 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
       "=x,x,x"; // Careful: constraint parser is very brittle: no ws!
   SmallVector<Value> asmVals{v1, v2};
   auto asmStr = llvm::formatv(asmTp, llvm::format_hex(mask, /*width=*/2)).str();
-  auto asmOp = b.create<LLVM::InlineAsmOp>(
+  auto asmOp = LLVM::InlineAsmOp::create(b,
       v1.getType(), /*operands=*/asmVals, /*asm_string=*/asmStr,
       /*constraints=*/asmCstr, /*has_side_effects=*/false,
       /*is_align_stack=*/false, LLVM::TailCallKind::None,
@@ -49,13 +49,13 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackLoPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2, ArrayRef<int64_t>{0, 8, 1, 9, 4, 12, 5, 13});
 }
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackHiPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2, ArrayRef<int64_t>{2, 10, 3, 11, 6, 14, 7, 15});
 }
 ///                            a  a   b   b  a  a   b   b
@@ -69,7 +69,7 @@ Value mlir::x86vector::avx2::intrin::mm256ShufflePs(ImplicitLocOpBuilder &b,
   MaskHelper::extractShuffle(mask, b01, b23, b45, b67);
   SmallVector<int64_t> shuffleMask = {
       b01, b23, b45 + 8, b67 + 8, b01 + 4, b23 + 4, b45 + 8 + 4, b67 + 8 + 4};
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 // imm[0:1] out of imm[0:3] is:
@@ -97,7 +97,7 @@ Value mlir::x86vector::avx2::intrin::mm256Permute2f128Ps(
   MaskHelper::extractPermute(mask, b03, b47);
   appendToMask(b03);
   appendToMask(b47);
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// If bit i of `mask` is zero, take f32 at i from v1 else take it from v2.
@@ -109,7 +109,7 @@ Value mlir::x86vector::avx2::intrin::mm256BlendPs(ImplicitLocOpBuilder &b,
     bool isSet = mask & (1 << i);
     shuffleMask.push_back(!isSet ? i : i + 8);
   }
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// AVX2 4x8xf32-specific transpose lowering using a "C intrinsics" model.
@@ -245,13 +245,13 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
           VectorType::get({n * m}, op.getSourceVectorType().getElementType());
       auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
       auto reshInput =
-          ib.create<vector::ShapeCastOp>(flattenedType, op.getVector());
-      reshInput = ib.create<vector::ShapeCastOp>(reshInputType, reshInput);
+          vector::ShapeCastOp::create(ib, flattenedType, op.getVector());
+      reshInput = vector::ShapeCastOp::create(ib, reshInputType, reshInput);
 
       // Extract 1-D vectors from the higher-order dimension of the input
       // vector.
       for (int64_t i = 0; i < m; ++i)
-        vs.push_back(ib.create<vector::ExtractOp>(reshInput, i));
+        vs.push_back(vector::ExtractOp::create(ib, reshInput, i));
 
       // Transpose set of 1-D vectors.
       if (m == 4)
@@ -261,16 +261,16 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
 
       // Insert transposed 1-D vectors into the higher-order dimension of the
       // output vector.
-      Value res = ib.create<arith::ConstantOp>(reshInputType,
+      Value res = arith::ConstantOp::create(ib, reshInputType,
                                                ib.getZeroAttr(reshInputType));
       for (int64_t i = 0; i < m; ++i)
-        res = ib.create<vector::InsertOp>(vs[i], res, i);
+        res = vector::InsertOp::create(ib, vs[i], res, i);
 
       // The output vector still has the shape of the input vector (e.g., 4x8).
       // We have to transpose their dimensions and retrieve its original rank
       // (e.g., 1x8x1x4x1).
-      res = ib.create<vector::ShapeCastOp>(flattenedType, res);
-      res = ib.create<vector::ShapeCastOp>(op.getResultVectorType(), res);
+      res = vector::ShapeCastOp::create(ib, flattenedType, res);
+      res = vector::ShapeCastOp::create(ib, op.getResultVectorType(), res);
       rewriter.replaceOp(op, res);
       return success();
     };
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 3bea6cb50ff7b..453d53ee41d54 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -416,7 +416,7 @@ void CreateDescOp::build(OpBuilder &builder, OperationState &state,
   int64_t size = static_cast<int64_t>(offsets.size());
   auto type = VectorType::get(size, builder.getIndexType());
   auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
-  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  auto offset = vector::FromElementsOp::create(builder, loc, type, values);
   build(builder, state, TensorDesc, source, offset);
 }
 
@@ -553,7 +553,7 @@ void UpdateOffsetOp::build(OpBuilder &builder, OperationState &state,
   int64_t size = static_cast<int64_t>(offsets.size());
   auto type = VectorType::get({size}, builder.getIndexType());
   auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
-  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  auto offset = vector::FromElementsOp::create(builder, loc, type, values);
   build(builder, state, tdescTy, tensorDesc, offset);
 }
 
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index c072557c2bd22..613a33fe63d87 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -120,13 +120,13 @@ static Value resolveDistributedTy(Value orig, T expected,
   // If orig is a vector type, create a shape cast op to reconcile the types.
   if (isa<VectorType>(orig.getType())) {
     auto castOp =
-        rewriter.create<vector::ShapeCastOp>(orig.getLoc(), expected, orig);
+        vector::ShapeCastOp::create(rewriter, orig.getLoc(), expected, orig);
     return castOp.getResult();
   }
   // If orig is a tensor descriptor type, create an unrealized conversion cast
   // op to reconcile the types.
   if (isa<xegpu::TensorDescType>(orig.getType())) {
-    auto castOp = rewriter.create<UnrealizedConversionCastOp>(orig.getLoc(),
+    auto castOp = UnrealizedConversionCastOp::create(rewriter, orig.getLoc(),
                                                               expected, orig);
     castOp->setAttr(resolveSIMTTypeMismatch, rewriter.getUnitAttr());
     return castOp.getResult(0);
@@ -198,16 +198,16 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         }))
       return failure();
     // Create a new function with the same signature.
-    auto newGpuFunc = rewriter.create<gpu::GPUFuncOp>(
+    auto newGpuFunc = gpu::GPUFuncOp::create(rewriter,
         gpuFuncOp.getLoc(), gpuFuncOp.getName(), gpuFuncOp.getFunctionType());
     // Create a WarpExecuteOnLane0Op with same arguments and results as the
     // original gpuFuncOp.
     rewriter.setInsertionPointToEnd(&newGpuFunc.getFunctionBody().front());
-    auto laneId = rewriter.create<gpu::LaneIdOp>(
+    auto laneId = gpu::LaneIdOp::create(rewriter,
         newGpuFunc.getLoc(), rewriter.getIndexType(),
         /** upperBound = **/ mlir::IntegerAttr());
     ArrayRef<Type> gpuFuncResultType = gpuFuncOp.getFunctionType().getResults();
-    auto warpOp = rewriter.create<gpu::WarpExecuteOnLane0Op>(
+    auto warpOp = gpu::WarpExecuteOnLane0Op::create(rewriter,
         laneId.getLoc(), gpuFuncResultType, laneId,
         xegpu::targetinfo::subgroupSize, newGpuFunc.getArguments(),
         newGpuFunc.getArgumentTypes());
@@ -216,7 +216,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
     auto origRetunOp =
         cast<gpu::ReturnOp>(gpuFuncOp.getBlocks().back().getTerminator());
     rewriter.setInsertionPointAfter(origRetunOp);
-    rewriter.create<gpu::YieldOp>(origRetunOp.getLoc(),
+    gpu::YieldOp::create(rewriter, origRetunOp.getLoc(),
                                   origRetunOp.getOperands());
     rewriter.eraseOp(origRetunOp);
     // Move the original function body to the WarpExecuteOnLane0Op body.
@@ -225,7 +225,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
     rewriter.eraseBlock(&warpBodyBlock);
     // Insert a new ReturnOp after the WarpExecuteOnLane0Op.
     rewriter.setInsertionPointAfter(warpOp);
-    rewriter.create<gpu::ReturnOp>(newGpuFunc.getLoc(), warpOp.getResults());
+    gpu::ReturnOp::create(rewriter, newGpuFunc.getLoc(), warpOp.getResults());
     rewriter.replaceOp(gpuFuncOp, newGpuFunc);
     return success();
   }
@@ -301,7 +301,7 @@ struct CreateNdDescDistribution final : public gpu::WarpDistributionPattern {
     xegpu::TensorDescType distributedTensorDescTy =
         descOp.getType().dropLayouts(); // Distributed tensor descriptor type
                                         // does not contain layout info.
-    Value newDescOp = rewriter.create<xegpu::CreateNdDescOp>(
+    Value newDescOp = xegpu::CreateNdDescOp::create(rewriter,
         newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
         descOp->getAttrs());
 
@@ -403,7 +403,7 @@ struct StoreNdDistribution final : public gpu::WarpDistributionPattern {
         resolveDistributedTy(newWarpOp.getResult(newRetIndices[1]),
                              distributedTensorDescTy, rewriter));
 
-    rewriter.create<xegpu::StoreNdOp>(
+    xegpu::StoreNdOp::create(rewriter,
         newWarpOp.getLoc(), TypeRange{}, newStoreOperands,
         removeTemporaryLayoutAttributes(storeOp->getAttrs()));
     rewriter.eraseOp(storeOp);
@@ -494,7 +494,7 @@ struct LoadNdDistribution final : public gpu::WarpDistributionPattern {
         loadOp.getTensorDescType().dropLayouts(); // Distributed tensor
                                                   // descriptor type does not
                                                   // contain layout info.
-    auto newLoadOp = rewriter.create<xegpu::LoadNdOp>(
+    auto newLoadOp = xegpu::LoadNdOp::create(rewriter,
         newWarpOp.getLoc(), loadNdDistValueTyOrFailure.value(),
         resolveDistributedTy(newWarpOp->getResult(newRetIndices[0]),
                              distributedTensorDescTy, rewriter),
@@ -630,7 +630,7 @@ struct DpasDistribution final : public gpu::WarpDistributionPattern {
           resolveDistributedTy(newWarpOp.getResult(newRetIndices[i]),
                                newDpasOperandExpectedTypes[i], rewriter));
     }
-    Value newDpasOp = rewriter.create<xegpu::DpasOp>(
+    Value newDpasOp = xegpu::DpasOp::create(rewriter,
         newWarpOp->getLoc(), distributedResultTy, newDpasOperands,
         removeTemporaryLayoutAttributes(dpasOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -717,7 +717,7 @@ struct UpdateNdOffsetDistribution final : public gpu::WarpDistributionPattern {
       }
     }
     // Create a new update op outside the warp op.
-    Value newUpdateOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+    Value newUpdateOp = xegpu::UpdateNdOffsetOp::create(rewriter,
         newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
         removeTemporaryLayoutAttributes(updateOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -783,7 +783,7 @@ struct PrefetchNdDistribution final : public gpu::WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     SmallVector<Value> newPrefetchOperands = {resolveDistributedTy(
         newWarpOp.getResult(newRetIndices[0]), newTensorDescTy, rewriter)};
-    rewriter.create<xegpu::PrefetchNdOp>(
+    xegpu::PrefetchNdOp::create(rewriter,
         newWarpOp.getLoc(), TypeRange{}, newPrefetchOperands,
         removeTemporaryLayoutAttributes(prefetchOp->getAttrs()));
     rewriter.eraseOp(prefetchOp);
@@ -806,7 +806,7 @@ struct GpuBarrierDistribution final : public gpu::WarpDistributionPattern {
       return failure();
     // Move the barrier op outside of the warp op.
     rewriter.setInsertionPointAfter(subgroupOp);
-    rewriter.create<gpu::BarrierOp>(
+    gpu::BarrierOp::create(rewriter,
         barrierOp.getLoc(), barrierOp->getResultTypes(),
         barrierOp->getOperands(), barrierOp->getAttrs());
     rewriter.eraseOp(barrierOp);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
index 2c48a735bf956..1d9f4dcc6b9e6 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
@@ -83,7 +83,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = rewriter.create<UnrealizedConversionCastOp>(
+      auto castOp = UnrealizedConversionCastOp::create(rewriter,
           loc, destTy, srcs, ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResult(0);
     }
@@ -109,7 +109,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = rewriter.create<UnrealizedConversionCastOp>(
+      auto castOp = UnrealizedConversionCastOp::create(rewriter,
           loc, destTypes, src, ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResults();
     }
@@ -144,10 +144,10 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
     auto addi = [&](OpFoldResult a, int64_t b) -> Value {
       std::optional<int64_t> maybeInt = getConstantIntValue(a);
       if (maybeInt) {
-        return rewriter.create<arith::ConstantIndexOp>(loc, *maybeInt + b);
+        return arith::ConstantIndexOp::create(rewriter, loc, *maybeInt + b);
       } else {
         auto aV = llvm::cast<Value>(a);
-        auto bV = rewriter.create<arith::ConstantIndexOp>(loc, b);
+        auto bV = arith::ConstantIndexOp::create(rewriter, loc, b);
         return rewriter.createOrFold<arith::AddIOp>(loc, aV, bV);
       }
     };
@@ -169,7 +169,7 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
            llvm::zip(validIdxes, oldOffsets, offsets))
         mixedOffsets[idx] = addi(oldOff, offset);
 
-      auto newOp = rewriter.create<xegpu::CreateNdDescOp>(
+      auto newOp = xegpu::CreateNdDescOp::create(rewriter,
           loc, newTdescTy, op.getSource(), mixedOffsets, op.getMixedSizes(),
           op.getMixedStrides());
       newOps.push_back(newOp);
@@ -199,7 +199,7 @@ struct UnrollUpdateNdOffsetOp : public UnrollPattern<xegpu::UpdateNdOffsetOp> {
 
     SmallVector<Value> newOps;
     for (auto t : convertedTdesc) {
-      auto newOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+      auto newOp = xegpu::UpdateNdOffsetOp::create(rewriter,
           loc, t.getType(), t, op.getOffsets(), op.getConstOffsets());
       newOps.push_back(newOp);
     }
@@ -226,7 +226,7 @@ struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      rewriter.create<xegpu::PrefetchNdOp>(loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchNdOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -257,7 +257,7 @@ struct UnrollLoadNdOp : public UnrollPattern<xegpu::LoadNdOp> {
     SmallVector<Value> newOps;
     for (auto t : convertedTdescs) {
       auto newOp =
-          rewriter.create<xegpu::LoadNdOp>(loc, newValueTy, t, op->getAttrs());
+          xegpu::LoadNdOp::create(rewriter, loc, newValueTy, t, op->getAttrs());
       newOps.push_back(newOp);
     }
 
@@ -291,7 +291,7 @@ struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto [v, t] : llvm::zip(convertedValues, convertedTdescs))
-      rewriter.create<xegpu::StoreNdOp>(loc, v, t, op.getL1HintAttr(),
+      xegpu::StoreNdOp::create(rewriter, loc, v, t, op.getL1HintAttr(),
                                         op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
@@ -384,7 +384,7 @@ struct UnrollDpasOp : public UnrollPattern<xegpu::DpasOp> {
           if (tmpC)
             operands.push_back(tmpC);
 
-          tmpC = rewriter.create<xegpu::DpasOp>(loc, vecTy, operands,
+          tmpC = xegpu::DpasOp::create(rewriter, loc, vecTy, operands,
                                                 op->getAttrs());
         }
         newOps.push_back(tmpC);
@@ -436,13 +436,13 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
            llvm::zip(convertedIndiceVec, convertedIndiceTypes)) {
         for (int64_t i = 0; i < numNewChunks; ++i) {
           // Compute the offset
-          Value inc = rewriter.create<arith::ConstantIndexOp>(
+          Value inc = arith::ConstantIndexOp::create(rewriter,
               loc, i * blockedChunkSize);
-          Value incVec = rewriter.create<vector::SplatOp>(loc, indiceType, inc);
+          Value incVec = vector::SplatOp::create(rewriter, loc, indiceType, inc);
           Value offsetIndice =
-              rewriter.create<arith::AddIOp>(loc, indice, incVec);
+              arith::AddIOp::create(rewriter, loc, indice, incVec);
 
-          auto newOp = rewriter.create<xegpu::CreateDescOp>(
+          auto newOp = xegpu::CreateDescOp::create(rewriter,
               loc, newTdescTy, op.getSource(), offsetIndice);
 
           newOps.push_back(newOp);
@@ -450,7 +450,7 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
       }
     } else {
       for (auto indice : convertedIndiceVec) {
-        auto newOp = rewriter.create<xegpu::CreateDescOp>(
+        auto newOp = xegpu::CreateDescOp::create(rewriter,
             loc, newTdescTy, op.getSource(), indice);
         newOps.push_back(newOp);
       }
@@ -515,7 +515,7 @@ struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
 
     SmallVector<Value> newOps;
     for (auto [t, m] : llvm::zip(convertedTdescs, convertedMasks)) {
-      auto newOp = rewriter.create<xegpu::LoadGatherOp>(
+      auto newOp = xegpu::LoadGatherOp::create(rewriter,
           loc, newValueTy, t, m, op.getL1HintAttr(), op.getL2HintAttr(),
           op.getL3HintAttr());
       newOps.push_back(newOp);
@@ -547,7 +547,7 @@ struct UnrollPrefetchOp : public UnrollPattern<xegpu::PrefetchOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      rewriter.create<xegpu::PrefetchOp>(loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -608,7 +608,7 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
       Value v = convertedValues[i];
       Value t = convertedTdescs[i];
       Value m = op.getMask() ? convertedMasks[i] : nullptr;
-      rewriter.create<xegpu::StoreScatterOp>(loc, v, t, m, op.getL1HintAttr(),
+      xegpu::StoreScatterOp::create(rewriter, loc, v, t, m, op.getL1HintAttr(),
                                              op.getL2HintAttr(),
                                              op.getL3HintAttr());
     }
@@ -665,7 +665,7 @@ struct UnrollUpdateOffsetOp : public UnrollPattern<xegpu::UpdateOffsetOp> {
 
     for (auto [t, o] : llvm::zip(convertedTdesc, convertedOffsetVec)) {
       auto newOp =
-          rewriter.create<xegpu::UpdateOffsetOp>(loc, t.getType(), t, o);
+          xegpu::UpdateOffsetOp::create(rewriter, loc, t.getType(), t, o);
       newOps.push_back(newOp);
     }
     Value castOp = unpack(newOps, op.getType(), *targetShape, loc, rewriter);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index be7b860dd1729..aa2fc78c77f39 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -121,11 +121,11 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
     for (size_t i = 0; i < rank; ++i) {
       size_t dimIdx = originalOffsets.size() - rank + i;
       Value constOffset =
-          rewriter.create<arith::ConstantIndexOp>(loc, distUnitBaseAddr[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, distUnitBaseAddr[i]);
       Value offset =
           rewriter.createOrFold<index::AddOp>(loc, localOffset[i], constOffset);
       Value modValue =
-          rewriter.create<arith::ConstantIndexOp>(loc, distUnitShape[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, distUnitShape[i]);
       Value offsetMod =
           rewriter.createOrFold<index::RemUOp>(loc, offset, modValue);
       Value origOffset = getValueOrCreateConstantIndexOp(
@@ -162,7 +162,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
     // TODO : Handle order attribute
     // Get the subgroup ID
     auto linearSgId =
-        rewriter.create<gpu::SubgroupIdOp>(loc, /*upper_bound=*/nullptr);
+        gpu::SubgroupIdOp::create(rewriter, loc, /*upper_bound=*/nullptr);
 
     // Create constants for layout dimensions
     SmallVector<Value> sgLayoutDim(sgLayout.size());
@@ -170,8 +170,8 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
 
     for (size_t i = 0; i < sgLayout.size(); i++) {
       sgLayoutDim[i] =
-          rewriter.create<arith::ConstantIndexOp>(loc, sgLayout[i]);
-      sgDataDim[i] = rewriter.create<arith::ConstantIndexOp>(loc, sgShape[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, sgLayout[i]);
+      sgDataDim[i] = arith::ConstantIndexOp::create(rewriter, loc, sgShape[i]);
     }
 
     auto deLinearizeSgId =
@@ -201,7 +201,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
           calculateGlobalOffsets(rewriter, loc, originalOffsets, localOffset,
                                  distUnitBaseAddr, distUnitShape);
 
-      auto newCreateNdOp = rewriter.create<xegpu::CreateNdDescOp>(
+      auto newCreateNdOp = xegpu::CreateNdDescOp::create(rewriter,
           loc, newTdescTy, op.getSource(), globalOffsets, op.getMixedSizes(),
           op.getMixedStrides());
       newCreateNdOps.push_back(newCreateNdOp);
@@ -224,7 +224,7 @@ struct WgToSgLoadNdOp : public OpConversionPattern<xegpu::LoadNdOp> {
           dyn_cast<xegpu::TensorDescType>(src.getType());
       ArrayRef<int64_t> srcShape = tdescTy.getShape();
       VectorType newResTy = VectorType::get(srcShape, tdescTy.getElementType());
-      auto newLoadOp = rewriter.create<xegpu::LoadNdOp>(op.getLoc(), newResTy,
+      auto newLoadOp = xegpu::LoadNdOp::create(rewriter, op.getLoc(), newResTy,
                                                         src, op->getAttrs());
       newLoadOps.push_back(newLoadOp);
     }
@@ -242,7 +242,7 @@ struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
   matchAndRewrite(xegpu::StoreNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     for (auto [v, t] : llvm::zip(adaptor.getValue(), adaptor.getTensorDesc()))
-      rewriter.create<xegpu::StoreNdOp>(op.getLoc(), v, t, op.getL1HintAttr(),
+      xegpu::StoreNdOp::create(rewriter, op.getLoc(), v, t, op.getL1HintAttr(),
                                         op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
@@ -261,7 +261,7 @@ struct WgToSgUpdateNdOffsetOp
                   ConversionPatternRewriter &rewriter) const override {
     llvm::SmallVector<Value> newUpdateTileOffsetOps;
     for (auto tDesc : adaptor.getTensorDesc()) {
-      auto newUpdateTileOffsetOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+      auto newUpdateTileOffsetOp = xegpu::UpdateNdOffsetOp::create(rewriter,
           op.getLoc(), tDesc.getType(), tDesc, op.getOffsets(),
           op.getConstOffsets());
       newUpdateTileOffsetOps.push_back(newUpdateTileOffsetOp);
@@ -305,7 +305,7 @@ struct WgToSgDpasOp : public OpConversionPattern<xegpu::DpasOp> {
             llvm::cast<VectorType>(bVec.getType()).getShape();
         VectorType resTy = VectorType::get({aVecShape[0], bVecShape[1]},
                                            resultTy.getElementType());
-        tmpC = rewriter.create<xegpu::DpasOp>(loc, resTy, operands);
+        tmpC = xegpu::DpasOp::create(rewriter, loc, resTy, operands);
         xegpu::setLayoutAttr(cast<OpResult>(tmpC),
                              originalLayout.dropSgLayoutAndData());
 
@@ -324,7 +324,7 @@ struct WgToSgPrefetchNdOp : public OpConversionPattern<xegpu::PrefetchNdOp> {
   matchAndRewrite(xegpu::PrefetchNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     for (auto src : adaptor.getTensorDesc())
-      rewriter.create<xegpu::PrefetchNdOp>(op.getLoc(), TypeRange(), src,
+      xegpu::PrefetchNdOp::create(rewriter, op.getLoc(), TypeRange(), src,
                                            op->getAttrs());
     rewriter.eraseOp(op);
     return success();
diff --git a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index 6b85a66a8bd36..9acb209d031bd 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -198,7 +198,7 @@ xegpu::extractVectorsWithShapeFromValue(OpBuilder &builder, Location loc,
   SmallVector<Value> result;
   for (SmallVector<int64_t> offsets : StaticTileOffsetRange(srcShape, shape)) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result.push_back(builder.create<vector::ExtractStridedSliceOp>(
+    result.push_back(vector::ExtractStridedSliceOp::create(builder,
         loc, value, offsets, shape, staticStrides));
   }
 
@@ -218,13 +218,13 @@ Value xegpu::createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
 
   VectorType resultTy = VectorType::get(shape, elemTy);
   auto zeroAttr = builder.getZeroAttr(elemTy);
-  Value result = builder.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(builder,
       loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
 
   for (auto [src, offsets] :
        llvm::zip_equal(values, StaticTileOffsetRange(shape, tileShape))) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result = builder.create<vector::InsertStridedSliceOp>(
+    result = vector::InsertStridedSliceOp::create(builder,
         loc, src, result, offsets, staticStrides);
   }
   return result;
@@ -236,7 +236,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
   auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
                             Location loc) -> Value {
-    return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
         .getResult(0);
   };
 
@@ -343,7 +343,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
         if (isa<RankedTensorType>(inputTy) && isa<VectorType>(outputTy)) {
           SmallVector<Value> values = xegpu::flattenValues(adaptor.getInputs());
-          auto newOp = rewriter.create<UnrealizedConversionCastOp>(
+          auto newOp = UnrealizedConversionCastOp::create(rewriter,
               op.getLoc(), outputTy, values);
           rewriter.replaceOp(op, newOp);
           return success();
@@ -355,7 +355,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
     converter.addSourceMaterialization(materializeCast);
     converter.addTargetMaterialization([&](OpBuilder &builder, TypeRange type,
                                            ValueRange inputs, Location loc) {
-      return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+      return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
           .getResults();
     });
 
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 6d7e2aa0ece7d..c88b328282275 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -132,7 +132,7 @@ void ModuleOp::build(OpBuilder &builder, OperationState &state,
 /// Construct a module from the given context.
 ModuleOp ModuleOp::create(Location loc, std::optional<StringRef> name) {
   OpBuilder builder(loc->getContext());
-  return builder.create<ModuleOp>(loc, name);
+  return ModuleOp::create(builder, loc, name);
 }
 
 DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {
diff --git a/mlir/lib/Query/Query.cpp b/mlir/lib/Query/Query.cpp
index 637e1f3cdef87..8ddb35d75734b 100644
--- a/mlir/lib/Query/Query.cpp
+++ b/mlir/lib/Query/Query.cpp
@@ -78,7 +78,7 @@ static Operation *extractFunction(std::vector<Operation *> &ops,
                       clonedOp->result_end());
   }
   // Add return operation
-  builder.create<func::ReturnOp>(loc, clonedVals);
+  func::ReturnOp::create(builder, loc, clonedVals);
 
   // Remove unused function arguments
   size_t currentIndex = 0;
diff --git a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
index 4e7f1d3185129..c0f19f6dce600 100644
--- a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
+++ b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
@@ -37,7 +37,7 @@ LogicalResult mlir::LLVMImportInterface::convertUnregisteredIntrinsic(
     return failure();
 
   Type resultType = moduleImport.convertType(inst->getType());
-  auto op = builder.create<::mlir::LLVM::CallIntrinsicOp>(
+  auto op = ::mlir::LLVM::CallIntrinsicOp::create(builder,
       moduleImport.translateLoc(inst->getDebugLoc()),
       isa<LLVMVoidType>(resultType) ? TypeRange{} : TypeRange{resultType},
       StringAttr::get(builder.getContext(), intrinName),
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index bfda223fe0f5f..6e47db6d1dc6a 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -185,7 +185,7 @@ ComdatOp ModuleImport::getGlobalComdatOp() {
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(mlirModule.getBody());
   globalComdatOp =
-      builder.create<ComdatOp>(mlirModule.getLoc(), getGlobalComdatOpName());
+      ComdatOp::create(builder, mlirModule.getLoc(), getGlobalComdatOpName());
   globalInsertionOp = globalComdatOp;
   return globalComdatOp;
 }
@@ -864,7 +864,7 @@ LogicalResult ModuleImport::convertModuleFlagsMetadata() {
   }
 
   if (!moduleFlags.empty())
-    builder.create<LLVM::ModuleFlagsOp>(mlirModule.getLoc(),
+    LLVM::ModuleFlagsOp::create(builder, mlirModule.getLoc(),
                                         builder.getArrayAttr(moduleFlags));
 
   return success();
@@ -880,7 +880,7 @@ LogicalResult ModuleImport::convertLinkerOptionsMetadata() {
       options.reserve(node->getNumOperands());
       for (const llvm::MDOperand &option : node->operands())
         options.push_back(cast<llvm::MDString>(option)->getString());
-      builder.create<LLVM::LinkerOptionsOp>(mlirModule.getLoc(),
+      LLVM::LinkerOptionsOp::create(builder, mlirModule.getLoc(),
                                             builder.getStrArrayAttr(options));
     }
   }
@@ -984,7 +984,7 @@ void ModuleImport::processComdat(const llvm::Comdat *comdat) {
   ComdatOp comdatOp = getGlobalComdatOp();
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(&comdatOp.getBody().back());
-  auto selectorOp = builder.create<ComdatSelectorOp>(
+  auto selectorOp = ComdatSelectorOp::create(builder,
       mlirModule.getLoc(), comdat->getName(),
       convertComdatFromLLVM(comdat->getSelectionKind()));
   auto symbolRef =
@@ -1346,7 +1346,7 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   Type type = convertType(alias->getValueType());
-  AliasOp aliasOp = builder.create<AliasOp>(
+  AliasOp aliasOp = AliasOp::create(builder,
       mlirModule.getLoc(), type, convertLinkageFromLLVM(alias->getLinkage()),
       alias->getName(),
       /*dso_local=*/alias->isDSOLocal(),
@@ -1360,7 +1360,7 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   FailureOr<Value> initializer = convertConstantExpr(alias->getAliasee());
   if (failed(initializer))
     return failure();
-  builder.create<ReturnOp>(aliasOp.getLoc(), *initializer);
+  ReturnOp::create(builder, aliasOp.getLoc(), *initializer);
 
   if (alias->hasAtLeastLocalUnnamedAddr())
     aliasOp.setUnnamedAddr(convertUnnamedAddrFromLLVM(alias->getUnnamedAddr()));
@@ -1403,7 +1403,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
   if (globalName.empty())
     globalName = getOrCreateNamelessSymbolName(globalVar).getValue();
 
-  GlobalOp globalOp = builder.create<GlobalOp>(
+  GlobalOp globalOp = GlobalOp::create(builder,
       mlirModule.getLoc(), type, globalVar->isConstant(),
       convertLinkageFromLLVM(globalVar->getLinkage()), StringRef(globalName),
       valueAttr, alignment, /*addr_space=*/globalVar->getAddressSpace(),
@@ -1420,7 +1420,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
         convertConstantExpr(globalVar->getInitializer());
     if (failed(initializer))
       return failure();
-    builder.create<ReturnOp>(globalOp.getLoc(), *initializer);
+    ReturnOp::create(builder, globalOp.getLoc(), *initializer);
   }
   if (globalVar->hasAtLeastLocalUnnamedAddr()) {
     globalOp.setUnnamedAddr(
@@ -1488,12 +1488,12 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   if (globalVar->getName() == getGlobalCtorsVarName()) {
-    globalInsertionOp = builder.create<LLVM::GlobalCtorsOp>(
+    globalInsertionOp = LLVM::GlobalCtorsOp::create(builder,
         mlirModule.getLoc(), builder.getArrayAttr(funcs),
         builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
     return success();
   }
-  globalInsertionOp = builder.create<LLVM::GlobalDtorsOp>(
+  globalInsertionOp = LLVM::GlobalDtorsOp::create(builder,
       mlirModule.getLoc(), builder.getArrayAttr(funcs),
       builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
   return success();
@@ -1569,33 +1569,33 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
   if (Attribute attr = getConstantAsAttr(constant)) {
     Type type = convertType(constant->getType());
     if (auto symbolRef = dyn_cast<FlatSymbolRefAttr>(attr)) {
-      return builder.create<AddressOfOp>(loc, type, symbolRef.getValue())
+      return AddressOfOp::create(builder, loc, type, symbolRef.getValue())
           .getResult();
     }
-    return builder.create<ConstantOp>(loc, type, attr).getResult();
+    return ConstantOp::create(builder, loc, type, attr).getResult();
   }
 
   // Convert null pointer constants.
   if (auto *nullPtr = dyn_cast<llvm::ConstantPointerNull>(constant)) {
     Type type = convertType(nullPtr->getType());
-    return builder.create<ZeroOp>(loc, type).getResult();
+    return ZeroOp::create(builder, loc, type).getResult();
   }
 
   // Convert none token constants.
   if (isa<llvm::ConstantTokenNone>(constant)) {
-    return builder.create<NoneTokenOp>(loc).getResult();
+    return NoneTokenOp::create(builder, loc).getResult();
   }
 
   // Convert poison.
   if (auto *poisonVal = dyn_cast<llvm::PoisonValue>(constant)) {
     Type type = convertType(poisonVal->getType());
-    return builder.create<PoisonOp>(loc, type).getResult();
+    return PoisonOp::create(builder, loc, type).getResult();
   }
 
   // Convert undef.
   if (auto *undefVal = dyn_cast<llvm::UndefValue>(constant)) {
     Type type = convertType(undefVal->getType());
-    return builder.create<UndefOp>(loc, type).getResult();
+    return UndefOp::create(builder, loc, type).getResult();
   }
 
   // Convert dso_local_equivalent.
@@ -1621,7 +1621,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
           getOrCreateNamelessSymbolName(cast<llvm::GlobalVariable>(globalObj));
     else
       symbolRef = FlatSymbolRefAttr::get(context, globalName);
-    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+    return AddressOfOp::create(builder, loc, type, symbolRef).getResult();
   }
 
   // Convert global alias accesses.
@@ -1629,7 +1629,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     Type type = convertType(globalAliasObj->getType());
     StringRef aliaseeName = globalAliasObj->getName();
     FlatSymbolRefAttr symbolRef = FlatSymbolRefAttr::get(context, aliaseeName);
-    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+    return AddressOfOp::create(builder, loc, type, symbolRef).getResult();
   }
 
   // Convert constant expressions.
@@ -1680,15 +1680,15 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     bool isArrayOrStruct = isa<LLVMArrayType, LLVMStructType>(rootType);
     assert((isArrayOrStruct || LLVM::isCompatibleVectorType(rootType)) &&
            "unrecognized aggregate type");
-    Value root = builder.create<UndefOp>(loc, rootType);
+    Value root = UndefOp::create(builder, loc, rootType);
     for (const auto &it : llvm::enumerate(elementValues)) {
       if (isArrayOrStruct) {
-        root = builder.create<InsertValueOp>(loc, root, it.value(), it.index());
+        root = InsertValueOp::create(builder, loc, root, it.value(), it.index());
       } else {
         Attribute indexAttr = builder.getI32IntegerAttr(it.index());
         Value indexValue =
-            builder.create<ConstantOp>(loc, builder.getI32Type(), indexAttr);
-        root = builder.create<InsertElementOp>(loc, rootType, root, it.value(),
+            ConstantOp::create(builder, loc, builder.getI32Type(), indexAttr);
+        root = InsertElementOp::create(builder, loc, rootType, root, it.value(),
                                                indexValue);
       }
     }
@@ -1702,7 +1702,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
            "target extension type does not support zero-initialization");
     // Create llvm.mlir.zero operation to represent zero-initialization of
     // target extension type.
-    return builder.create<LLVM::ZeroOp>(loc, targetExtType).getRes();
+    return LLVM::ZeroOp::create(builder, loc, targetExtType).getRes();
   }
 
   if (auto *blockAddr = dyn_cast<llvm::BlockAddress>(constant)) {
@@ -2124,7 +2124,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     if (!brInst->isConditional()) {
-      auto brOp = builder.create<LLVM::BrOp>(loc, succBlockArgs.front(),
+      auto brOp = LLVM::BrOp::create(builder, loc, succBlockArgs.front(),
                                              succBlocks.front());
       mapNoResultOp(inst, brOp);
       return success();
@@ -2132,7 +2132,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     FailureOr<Value> condition = convertValue(brInst->getCondition());
     if (failed(condition))
       return failure();
-    auto condBrOp = builder.create<LLVM::CondBrOp>(
+    auto condBrOp = LLVM::CondBrOp::create(builder,
         loc, *condition, succBlocks.front(), succBlockArgs.front(),
         succBlocks.back(), succBlockArgs.back());
     mapNoResultOp(inst, condBrOp);
@@ -2166,7 +2166,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       caseBlocks[it.index()] = lookupBlock(succBB);
     }
 
-    auto switchOp = builder.create<SwitchOp>(
+    auto switchOp = SwitchOp::create(builder,
         loc, *condition, lookupBlock(defaultBB), defaultBlockArgs, caseValues,
         caseBlocks, caseOperandRefs);
     mapNoResultOp(inst, switchOp);
@@ -2218,14 +2218,14 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
         // IR). Build the indirect call by passing an empty `callee` operand and
         // insert into `operands` to include the indirect call target.
         FlatSymbolRefAttr calleeSym = convertCalleeName(callInst);
-        Value indirectCallVal = builder.create<LLVM::AddressOfOp>(
+        Value indirectCallVal = LLVM::AddressOfOp::create(builder,
             loc, LLVM::LLVMPointerType::get(context), calleeSym);
         operands->insert(operands->begin(), indirectCallVal);
       } else {
         // Regular direct call using callee name.
         callee = convertCalleeName(callInst);
       }
-      CallOp callOp = builder.create<CallOp>(loc, *funcTy, callee, *operands);
+      CallOp callOp = CallOp::create(builder, loc, *funcTy, callee, *operands);
 
       if (failed(convertCallAttributes(callInst, callOp)))
         return failure();
@@ -2260,7 +2260,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
 
     Type type = convertType(lpInst->getType());
     auto lpOp =
-        builder.create<LandingpadOp>(loc, type, lpInst->isCleanup(), operands);
+        LandingpadOp::create(builder, loc, type, lpInst->isCleanup(), operands);
     mapValue(inst, lpOp);
     return success();
   }
@@ -2310,7 +2310,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // IR). Build the indirect invoke by passing an empty `callee` operand and
       // insert into `operands` to include the indirect invoke target.
       FlatSymbolRefAttr calleeSym = convertCalleeName(invokeInst);
-      Value indirectInvokeVal = builder.create<LLVM::AddressOfOp>(
+      Value indirectInvokeVal = LLVM::AddressOfOp::create(builder,
           loc, LLVM::LLVMPointerType::get(context), calleeSym);
       operands->insert(operands->begin(), indirectInvokeVal);
     } else {
@@ -2320,7 +2320,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     // Create the invoke operation. Normal destination block arguments will be
     // added later on to handle the case in which the operation result is
     // included in this list.
-    auto invokeOp = builder.create<InvokeOp>(
+    auto invokeOp = InvokeOp::create(builder,
         loc, *funcTy, calleeName, *operands, directNormalDest, ValueRange(),
         lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
 
@@ -2348,7 +2348,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // arguments (including the invoke operation's result).
       OpBuilder::InsertionGuard g(builder);
       builder.setInsertionPointToStart(directNormalDest);
-      builder.create<LLVM::BrOp>(loc, normalArgs, normalDest);
+      LLVM::BrOp::create(builder, loc, normalArgs, normalDest);
     } else {
       // If the invoke operation's result is not a block argument to the normal
       // destination block, just add the block arguments as usual.
@@ -2382,7 +2382,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     Type type = convertType(inst->getType());
-    auto gepOp = builder.create<GEPOp>(
+    auto gepOp = GEPOp::create(builder,
         loc, type, sourceElementType, *basePtr, indices,
         static_cast<GEPNoWrapFlags>(gepInst->getNoWrapFlags().getRaw()));
     mapValue(inst, gepOp);
@@ -2409,7 +2409,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     SmallVector<ValueRange> succBlockArgsRange =
         llvm::to_vector_of<ValueRange>(succBlockArgs);
     Location loc = translateLoc(inst->getDebugLoc());
-    auto indBrOp = builder.create<LLVM::IndirectBrOp>(
+    auto indBrOp = LLVM::IndirectBrOp::create(builder,
         loc, *basePtr, succBlockArgsRange, succBlocks);
 
     mapNoResultOp(inst, indBrOp);
@@ -2854,7 +2854,7 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) {
   builder.setInsertionPointToEnd(mlirModule.getBody());
 
   Location loc = debugImporter->translateFuncLocation(func);
-  LLVMFuncOp funcOp = builder.create<LLVMFuncOp>(
+  LLVMFuncOp funcOp = LLVMFuncOp::create(builder,
       loc, func->getName(), functionType,
       convertLinkageFromLLVM(func->getLinkage()), dsoLocal, cconv);
 
@@ -3032,11 +3032,11 @@ ModuleImport::processDebugIntrinsic(llvm::DbgVariableIntrinsic *dbgIntr,
   Operation *op =
       llvm::TypeSwitch<llvm::DbgVariableIntrinsic *, Operation *>(dbgIntr)
           .Case([&](llvm::DbgDeclareInst *) {
-            return builder.create<LLVM::DbgDeclareOp>(
+            return LLVM::DbgDeclareOp::create(builder,
                 loc, *argOperand, localVariableAttr, locationExprAttr);
           })
           .Case([&](llvm::DbgValueInst *) {
-            return builder.create<LLVM::DbgValueOp>(
+            return LLVM::DbgValueOp::create(builder,
                 loc, *argOperand, localVariableAttr, locationExprAttr);
           });
   mapNoResultOp(dbgIntr, op);
@@ -3082,7 +3082,7 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb,
   if (bb->hasAddressTaken()) {
     OpBuilder::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(block);
-    builder.create<BlockTagOp>(block->getParentOp()->getLoc(),
+    BlockTagOp::create(builder, block->getParentOp()->getLoc(),
                                BlockTagAttr::get(context, bb->getNumber()));
   }
   return success();
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index 55d6a380d0bff..6e14b94e87407 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -42,22 +42,22 @@ static inline spirv::Opcode extractOpcode(uint32_t word) {
 Value spirv::Deserializer::getValue(uint32_t id) {
   if (auto constInfo = getConstant(id)) {
     // Materialize a `spirv.Constant` op at every use site.
-    return opBuilder.create<spirv::ConstantOp>(unknownLoc, constInfo->second,
+    return spirv::ConstantOp::create(opBuilder, unknownLoc, constInfo->second,
                                                constInfo->first);
   }
   if (auto varOp = getGlobalVariable(id)) {
-    auto addressOfOp = opBuilder.create<spirv::AddressOfOp>(
+    auto addressOfOp = spirv::AddressOfOp::create(opBuilder,
         unknownLoc, varOp.getType(), SymbolRefAttr::get(varOp.getOperation()));
     return addressOfOp.getPointer();
   }
   if (auto constOp = getSpecConstant(id)) {
-    auto referenceOfOp = opBuilder.create<spirv::ReferenceOfOp>(
+    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
         unknownLoc, constOp.getDefaultValue().getType(),
         SymbolRefAttr::get(constOp.getOperation()));
     return referenceOfOp.getReference();
   }
   if (auto constCompositeOp = getSpecConstantComposite(id)) {
-    auto referenceOfOp = opBuilder.create<spirv::ReferenceOfOp>(
+    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
         unknownLoc, constCompositeOp.getType(),
         SymbolRefAttr::get(constCompositeOp.getOperation()));
     return referenceOfOp.getReference();
@@ -69,7 +69,7 @@ Value spirv::Deserializer::getValue(uint32_t id) {
         specConstOperationInfo->enclosedOpOperands);
   }
   if (auto undef = getUndefType(id)) {
-    return opBuilder.create<spirv::UndefOp>(unknownLoc, undef);
+    return spirv::UndefOp::create(opBuilder, unknownLoc, undef);
   }
   return valueMap.lookup(id);
 }
@@ -369,7 +369,7 @@ Deserializer::processOp<spirv::EntryPointOp>(ArrayRef<uint32_t> words) {
     interface.push_back(SymbolRefAttr::get(arg.getOperation()));
     wordIndex++;
   }
-  opBuilder.create<spirv::EntryPointOp>(
+  spirv::EntryPointOp::create(opBuilder,
       unknownLoc, execModel, SymbolRefAttr::get(opBuilder.getContext(), fnName),
       opBuilder.getArrayAttr(interface));
   return success();
@@ -402,7 +402,7 @@ Deserializer::processOp<spirv::ExecutionModeOp>(ArrayRef<uint32_t> words) {
     attrListElems.push_back(opBuilder.getI32IntegerAttr(words[wordIndex++]));
   }
   auto values = opBuilder.getArrayAttr(attrListElems);
-  opBuilder.create<spirv::ExecutionModeOp>(
+  spirv::ExecutionModeOp::create(opBuilder,
       unknownLoc, SymbolRefAttr::get(opBuilder.getContext(), fn.getName()),
       execMode, values);
   return success();
@@ -441,7 +441,7 @@ Deserializer::processOp<spirv::FunctionCallOp>(ArrayRef<uint32_t> operands) {
     arguments.push_back(value);
   }
 
-  auto opFunctionCall = opBuilder.create<spirv::FunctionCallOp>(
+  auto opFunctionCall = spirv::FunctionCallOp::create(opBuilder,
       unknownLoc, resultType,
       SymbolRefAttr::get(opBuilder.getContext(), functionName), arguments);
 
@@ -518,7 +518,7 @@ Deserializer::processOp<spirv::CopyMemoryOp>(ArrayRef<uint32_t> words) {
   }
 
   Location loc = createFileLineColLoc(opBuilder);
-  opBuilder.create<spirv::CopyMemoryOp>(loc, resultTypes, operands, attributes);
+  spirv::CopyMemoryOp::create(opBuilder, loc, resultTypes, operands, attributes);
 
   return success();
 }
@@ -549,7 +549,7 @@ LogicalResult Deserializer::processOp<spirv::GenericCastToPtrExplicitOp>(
   operands.push_back(arg);
 
   Location loc = createFileLineColLoc(opBuilder);
-  Operation *op = opBuilder.create<spirv::GenericCastToPtrExplicitOp>(
+  Operation *op = spirv::GenericCastToPtrExplicitOp::create(opBuilder,
       loc, resultTypes, operands);
   valueMap[valueID] = op->getResult(0);
   return success();
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index b1abd8b3dffe9..7b1f8055fadce 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -518,7 +518,7 @@ spirv::Deserializer::processFunction(ArrayRef<uint32_t> operands) {
   }
 
   std::string fnName = getFunctionSymbol(fnID);
-  auto funcOp = opBuilder.create<spirv::FuncOp>(
+  auto funcOp = spirv::FuncOp::create(opBuilder,
       unknownLoc, fnName, functionType, fnControl.value());
   // Processing other function attributes.
   if (decorations.count(fnID)) {
@@ -706,7 +706,7 @@ spirv::SpecConstantOp
 spirv::Deserializer::createSpecConstant(Location loc, uint32_t resultID,
                                         TypedAttr defaultValue) {
   auto symName = opBuilder.getStringAttr(getSpecConstantSymbol(resultID));
-  auto op = opBuilder.create<spirv::SpecConstantOp>(unknownLoc, symName,
+  auto op = spirv::SpecConstantOp::create(opBuilder, unknownLoc, symName,
                                                     defaultValue);
   if (decorations.count(resultID)) {
     for (auto attr : decorations[resultID].getAttrs())
@@ -782,7 +782,7 @@ spirv::Deserializer::processGlobalVariable(ArrayRef<uint32_t> operands) {
            << wordIndex << " of " << operands.size() << " processed";
   }
   auto loc = createFileLineColLoc(opBuilder);
-  auto varOp = opBuilder.create<spirv::GlobalVariableOp>(
+  auto varOp = spirv::GlobalVariableOp::create(opBuilder,
       loc, TypeAttr::get(type), opBuilder.getStringAttr(variableName),
       initializer);
 
@@ -1581,7 +1581,7 @@ spirv::Deserializer::processSpecConstantComposite(ArrayRef<uint32_t> operands) {
     elements.push_back(SymbolRefAttr::get(elementInfo));
   }
 
-  auto op = opBuilder.create<spirv::SpecConstantCompositeOp>(
+  auto op = spirv::SpecConstantCompositeOp::create(opBuilder,
       unknownLoc, TypeAttr::get(resultType), symName,
       opBuilder.getArrayAttr(elements));
   specConstCompositeMap[resultID] = op;
@@ -1656,7 +1656,7 @@ Value spirv::Deserializer::materializeSpecConstantOperation(
 
   auto loc = createFileLineColLoc(opBuilder);
   auto specConstOperationOp =
-      opBuilder.create<spirv::SpecConstantOperationOp>(loc, resultType);
+      spirv::SpecConstantOperationOp::create(opBuilder, loc, resultType);
 
   Region &body = specConstOperationOp.getBody();
   // Move the new block into SpecConstantOperation's body.
@@ -1669,7 +1669,7 @@ Value spirv::Deserializer::materializeSpecConstantOperation(
   OpBuilder::InsertionGuard moduleInsertionGuard(opBuilder);
   opBuilder.setInsertionPointToEnd(&block);
 
-  opBuilder.create<spirv::YieldOp>(loc, block.front().getResult(0));
+  spirv::YieldOp::create(opBuilder, loc, block.front().getResult(0));
   return specConstOperationOp.getResult();
 }
 
@@ -1733,7 +1733,7 @@ LogicalResult spirv::Deserializer::processBranch(ArrayRef<uint32_t> operands) {
   // The preceding instruction for the OpBranch instruction could be an
   // OpLoopMerge or an OpSelectionMerge instruction, in this case they will have
   // the same OpLine information.
-  opBuilder.create<spirv::BranchOp>(loc, target);
+  spirv::BranchOp::create(opBuilder, loc, target);
 
   clearDebugLine();
   return success();
@@ -1764,7 +1764,7 @@ spirv::Deserializer::processBranchConditional(ArrayRef<uint32_t> operands) {
   // an OpSelectionMerge instruction, in this case they will have the same
   // OpLine information.
   auto loc = createFileLineColLoc(opBuilder);
-  opBuilder.create<spirv::BranchConditionalOp>(
+  spirv::BranchConditionalOp::create(opBuilder,
       loc, condition, trueBlock,
       /*trueArguments=*/ArrayRef<Value>(), falseBlock,
       /*falseArguments=*/ArrayRef<Value>(), weights);
@@ -1947,7 +1947,7 @@ ControlFlowStructurizer::createSelectionOp(uint32_t selectionControl) {
   OpBuilder builder(&mergeBlock->front());
 
   auto control = static_cast<spirv::SelectionControl>(selectionControl);
-  auto selectionOp = builder.create<spirv::SelectionOp>(location, control);
+  auto selectionOp = spirv::SelectionOp::create(builder, location, control);
   selectionOp.addMergeBlock(builder);
 
   return selectionOp;
@@ -1959,7 +1959,7 @@ spirv::LoopOp ControlFlowStructurizer::createLoopOp(uint32_t loopControl) {
   OpBuilder builder(&mergeBlock->front());
 
   auto control = static_cast<spirv::LoopControl>(loopControl);
-  auto loopOp = builder.create<spirv::LoopOp>(location, control);
+  auto loopOp = spirv::LoopOp::create(builder, location, control);
   loopOp.addEntryAndMergeBlock(builder);
 
   return loopOp;
@@ -2092,7 +2092,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
     // The loop entry block should have a unconditional branch jumping to the
     // loop header block.
     builder.setInsertionPointToEnd(&body.front());
-    builder.create<spirv::BranchOp>(location, mapper.lookupOrNull(headerBlock),
+    spirv::BranchOp::create(builder, location, mapper.lookupOrNull(headerBlock),
                                     ArrayRef<Value>(blockArgs));
   }
 
@@ -2177,11 +2177,11 @@ LogicalResult ControlFlowStructurizer::structurize() {
     Operation *newOp = nullptr;
 
     if (isLoop)
-      newOp = builder.create<spirv::LoopOp>(
+      newOp = spirv::LoopOp::create(builder,
           location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::LoopControl>(control));
     else
-      newOp = builder.create<spirv::SelectionOp>(
+      newOp = spirv::SelectionOp::create(builder,
           location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::SelectionControl>(control));
 
@@ -2308,7 +2308,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
       // but replace all ops inside with a branch to the merge block.
       block->clear();
       builder.setInsertionPointToEnd(block);
-      builder.create<spirv::BranchOp>(location, mergeBlock);
+      spirv::BranchOp::create(builder, location, mergeBlock);
     } else {
       LLVM_DEBUG(logger.startLine() << "[cf] erasing block " << block << "\n");
       block->erase();
@@ -2362,7 +2362,7 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
 
     if (auto branchOp = dyn_cast<spirv::BranchOp>(op)) {
       // Replace the previous branch op with a new one with block arguments.
-      opBuilder.create<spirv::BranchOp>(branchOp.getLoc(), branchOp.getTarget(),
+      spirv::BranchOp::create(opBuilder, branchOp.getLoc(), branchOp.getTarget(),
                                         blockArgs);
       branchOp.erase();
     } else if (auto branchCondOp = dyn_cast<spirv::BranchConditionalOp>(op)) {
@@ -2370,13 +2370,13 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
               branchCondOp.getFalseBlock() == target) &&
              "expected target to be either the true or false target");
       if (target == branchCondOp.getTrueTarget())
-        opBuilder.create<spirv::BranchConditionalOp>(
+        spirv::BranchConditionalOp::create(opBuilder,
             branchCondOp.getLoc(), branchCondOp.getCondition(), blockArgs,
             branchCondOp.getFalseBlockArguments(),
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueTarget(),
             branchCondOp.getFalseTarget());
       else
-        opBuilder.create<spirv::BranchConditionalOp>(
+        spirv::BranchConditionalOp::create(opBuilder,
             branchCondOp.getLoc(), branchCondOp.getCondition(),
             branchCondOp.getTrueBlockArguments(), blockArgs,
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueBlock(),
@@ -2437,7 +2437,7 @@ LogicalResult spirv::Deserializer::splitConditionalBlocks() {
     if (!llvm::hasSingleElement(*block) || splitHeaderMergeBlock) {
       Block *newBlock = block->splitBlock(terminator);
       OpBuilder builder(block, block->end());
-      builder.create<spirv::BranchOp>(block->getParent()->getLoc(), newBlock);
+      spirv::BranchOp::create(builder, block->getParent()->getLoc(), newBlock);
 
       // After splitting we need to update the map to use the new block as a
       // header.
diff --git a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
index 824201d17b5ab..3f4e7762fed0e 100644
--- a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
+++ b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
@@ -134,7 +134,7 @@ class CodeGen {
 
 OwningOpRef<ModuleOp> CodeGen::generate(const ast::Module &module) {
   OwningOpRef<ModuleOp> mlirModule =
-      builder.create<ModuleOp>(genLoc(module.getLoc()));
+      ModuleOp::create(builder, genLoc(module.getLoc()));
   builder.setInsertionPointToStart(mlirModule->getBody());
 
   // Generate code for each of the decls within the module.
@@ -205,7 +205,7 @@ static void checkAndNestUnderRewriteOp(OpBuilder &builder, Value rootExpr,
                                        Location loc) {
   if (isa<pdl::PatternOp>(builder.getInsertionBlock()->getParentOp())) {
     pdl::RewriteOp rewrite =
-        builder.create<pdl::RewriteOp>(loc, rootExpr, /*name=*/StringAttr(),
+        pdl::RewriteOp::create(builder, loc, rootExpr, /*name=*/StringAttr(),
                                        /*externalArgs=*/ValueRange());
     builder.createBlock(&rewrite.getBodyRegion());
   }
@@ -219,7 +219,7 @@ void CodeGen::genImpl(const ast::EraseStmt *stmt) {
   // Make sure we are nested in a RewriteOp.
   OpBuilder::InsertionGuard guard(builder);
   checkAndNestUnderRewriteOp(builder, rootExpr, loc);
-  builder.create<pdl::EraseOp>(loc, rootExpr);
+  pdl::EraseOp::create(builder, loc, rootExpr);
 }
 
 void CodeGen::genImpl(const ast::LetStmt *stmt) { genVar(stmt->getVarDecl()); }
@@ -242,7 +242,7 @@ void CodeGen::genImpl(const ast::ReplaceStmt *stmt) {
   bool usesReplOperation =
       replValues.size() == 1 &&
       isa<pdl::OperationType>(replValues.front().getType());
-  builder.create<pdl::ReplaceOp>(
+  pdl::ReplaceOp::create(builder,
       loc, rootExpr, usesReplOperation ? replValues[0] : Value(),
       usesReplOperation ? ValueRange() : ValueRange(replValues));
 }
@@ -283,7 +283,7 @@ void CodeGen::genImpl(const ast::PatternDecl *decl) {
 
   // FIXME: Properly model HasBoundedRecursion in PDL so that we don't drop it
   // here.
-  pdl::PatternOp pattern = builder.create<pdl::PatternOp>(
+  pdl::PatternOp pattern = pdl::PatternOp::create(builder,
       genLoc(decl->getLoc()), decl->getBenefit(),
       name ? std::optional<StringRef>(name->getName())
            : std::optional<StringRef>());
@@ -338,19 +338,19 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   ast::Type type = varDecl->getType();
   Type mlirType = genType(type);
   if (isa<ast::ValueType>(type))
-    return builder.create<pdl::OperandOp>(loc, mlirType, getTypeConstraint());
+    return pdl::OperandOp::create(builder, loc, mlirType, getTypeConstraint());
   if (isa<ast::TypeType>(type))
-    return builder.create<pdl::TypeOp>(loc, mlirType, /*type=*/TypeAttr());
+    return pdl::TypeOp::create(builder, loc, mlirType, /*type=*/TypeAttr());
   if (isa<ast::AttributeType>(type))
-    return builder.create<pdl::AttributeOp>(loc, getTypeConstraint());
+    return pdl::AttributeOp::create(builder, loc, getTypeConstraint());
   if (ast::OperationType opType = dyn_cast<ast::OperationType>(type)) {
-    Value operands = builder.create<pdl::OperandsOp>(
+    Value operands = pdl::OperandsOp::create(builder,
         loc, pdl::RangeType::get(builder.getType<pdl::ValueType>()),
         /*type=*/Value());
-    Value results = builder.create<pdl::TypesOp>(
+    Value results = pdl::TypesOp::create(builder,
         loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
         /*types=*/ArrayAttr());
-    return builder.create<pdl::OperationOp>(loc, opType.getName(), operands,
+    return pdl::OperationOp::create(builder, loc, opType.getName(), operands,
                                             ArrayRef<StringRef>(), ValueRange(),
                                             results);
   }
@@ -358,10 +358,10 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   if (ast::RangeType rangeTy = dyn_cast<ast::RangeType>(type)) {
     ast::Type eleTy = rangeTy.getElementType();
     if (isa<ast::ValueType>(eleTy))
-      return builder.create<pdl::OperandsOp>(loc, mlirType,
+      return pdl::OperandsOp::create(builder, loc, mlirType,
                                              getTypeConstraint());
     if (isa<ast::TypeType>(eleTy))
-      return builder.create<pdl::TypesOp>(loc, mlirType, /*types=*/ArrayAttr());
+      return pdl::TypesOp::create(builder, loc, mlirType, /*types=*/ArrayAttr());
   }
 
   llvm_unreachable("invalid non-initialized variable type");
@@ -404,7 +404,7 @@ SmallVector<Value> CodeGen::genExpr(const ast::Expr *expr) {
 Value CodeGen::genExprImpl(const ast::AttributeExpr *expr) {
   Attribute attr = parseAttribute(expr->getValue(), builder.getContext());
   assert(attr && "invalid MLIR attribute data");
-  return builder.create<pdl::AttributeOp>(genLoc(expr->getLoc()), attr);
+  return pdl::AttributeOp::create(builder, genLoc(expr->getLoc()), attr);
 }
 
 SmallVector<Value> CodeGen::genExprImpl(const ast::CallExpr *expr) {
@@ -443,9 +443,9 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
     if (isa<ast::AllResultsMemberAccessExpr>(expr)) {
       Type mlirType = genType(expr->getType());
       if (isa<pdl::ValueType>(mlirType))
-        return builder.create<pdl::ResultOp>(loc, mlirType, parentExprs[0],
+        return pdl::ResultOp::create(builder, loc, mlirType, parentExprs[0],
                                              builder.getI32IntegerAttr(0));
-      return builder.create<pdl::ResultsOp>(loc, mlirType, parentExprs[0]);
+      return pdl::ResultsOp::create(builder, loc, mlirType, parentExprs[0]);
     }
 
     const ods::Operation *odsOp = opType.getODSOperation();
@@ -455,7 +455,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
       unsigned resultIndex;
       name.getAsInteger(/*Radix=*/10, resultIndex);
       IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
-      return builder.create<pdl::ResultOp>(loc, genType(expr->getType()),
+      return pdl::ResultOp::create(builder, loc, genType(expr->getType()),
                                            parentExprs[0], index);
     }
 
@@ -474,7 +474,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
 
     // Generate the result access.
     IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
-    return builder.create<pdl::ResultsOp>(loc, genType(expr->getType()),
+    return pdl::ResultsOp::create(builder, loc, genType(expr->getType()),
                                           parentExprs[0], index);
   }
 
@@ -518,7 +518,7 @@ Value CodeGen::genExprImpl(const ast::OperationExpr *expr) {
   for (const ast::Expr *result : expr->getResultTypes())
     results.push_back(genSingleExpr(result));
 
-  return builder.create<pdl::OperationOp>(loc, opName, operands, attrNames,
+  return pdl::OperationOp::create(builder, loc, opName, operands, attrNames,
                                           attrValues, results);
 }
 
@@ -527,7 +527,7 @@ Value CodeGen::genExprImpl(const ast::RangeExpr *expr) {
   for (const ast::Expr *element : expr->getElements())
     llvm::append_range(elements, genExpr(element));
 
-  return builder.create<pdl::RangeOp>(genLoc(expr->getLoc()),
+  return pdl::RangeOp::create(builder, genLoc(expr->getLoc()),
                                       genType(expr->getType()), elements);
 }
 
@@ -541,7 +541,7 @@ SmallVector<Value> CodeGen::genExprImpl(const ast::TupleExpr *expr) {
 Value CodeGen::genExprImpl(const ast::TypeExpr *expr) {
   Type type = parseType(expr->getValue(), builder.getContext());
   assert(type && "invalid MLIR type data");
-  return builder.create<pdl::TypeOp>(genLoc(expr->getLoc()),
+  return pdl::TypeOp::create(builder, genLoc(expr->getLoc()),
                                      builder.getType<pdl::TypeType>(),
                                      TypeAttr::get(type));
 }
@@ -586,7 +586,7 @@ CodeGen::genConstraintOrRewriteCall(const T *decl, Location loc,
     } else {
       resultTypes.push_back(genType(declResultType));
     }
-    PDLOpT pdlOp = builder.create<PDLOpT>(loc, resultTypes,
+    PDLOpT pdlOp = PDLOpT::create(builder, loc, resultTypes,
                                           decl->getName().getName(), inputs);
     if (isNegated && std::is_same_v<PDLOpT, pdl::ApplyNativeConstraintOp>)
       cast<pdl::ApplyNativeConstraintOp>(pdlOp).setIsNegated(true);
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index d118fe422f2f2..68b8335ed9262 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -1505,7 +1505,7 @@ ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
   OpBuilder builder(outputTypes.front().getContext());
   builder.setInsertionPoint(ip.getBlock(), ip.getPoint());
   auto convertOp =
-      builder.create<UnrealizedConversionCastOp>(loc, outputTypes, inputs);
+      UnrealizedConversionCastOp::create(builder, loc, outputTypes, inputs);
   if (!valuesToMap.empty())
     mapping.map(valuesToMap, convertOp.getResults());
   if (castOp)

>From e8d226d756df2460ca4343ad034fd585dc2b619c Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 12:56:56 -0400
Subject: [PATCH 3/5] fix cpp custom builders

---
 mlir/include/mlir/Dialect/Arith/IR/Arith.h | 10 +++
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp     | 79 ++++++++++++++++++----
 mlir/lib/IR/BuiltinDialect.cpp             |  2 +-
 3 files changed, 76 insertions(+), 15 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 7c50c2036ffdc..79b09b5fcaccf 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -59,15 +59,21 @@ class ConstantIntOp : public arith::ConstantOp {
   /// Build a constant int op that produces an integer of the specified width.
   static void build(OpBuilder &builder, OperationState &result, int64_t value,
                     unsigned width);
+  static ConstantIntOp create(OpBuilder &builder, Location location,
+                              int64_t value, unsigned width);
 
   /// Build a constant int op that produces an integer of the specified type,
   /// which must be an integer type.
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     int64_t value);
+  static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
+                              int64_t value);
 
   /// Build a constant int op that produces an integer from an APInt
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     const APInt &value);
+  static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
+                              const APInt &value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
@@ -85,6 +91,8 @@ class ConstantFloatOp : public arith::ConstantOp {
   /// Build a constant float op that produces a float of the specified type.
   static void build(OpBuilder &builder, OperationState &result, FloatType type,
                     const APFloat &value);
+  static ConstantFloatOp create(OpBuilder &builder, Location location,
+                                FloatType type, const APFloat &value);
 
   inline APFloat value() {
     return cast<FloatAttr>(arith::ConstantOp::getValue()).getValue();
@@ -100,6 +108,8 @@ class ConstantIndexOp : public arith::ConstantOp {
   static ::mlir::TypeID resolveTypeID() { return TypeID::get<ConstantOp>(); }
   /// Build a constant int op that produces an index.
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
+  static ConstantIndexOp create(OpBuilder &builder, Location location,
+                                int64_t value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index c9b0c8414b775..aa623c19a9218 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -256,18 +256,49 @@ void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location,
+                                                  int64_t value,
+                                                  unsigned width) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, value, width);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, int64_t value) {
   arith::ConstantOp::build(builder, result, type,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location, Type type,
+                                                  int64_t value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, const APInt &value) {
   arith::ConstantOp::build(builder, result, type,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location, Type type,
+                                                  const APInt &value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -280,6 +311,17 @@ void arith::ConstantFloatOp::build(OpBuilder &builder, OperationState &result,
                            builder.getFloatAttr(type, value));
 }
 
+arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
+                                                      Location location,
+                                                      FloatType type,
+                                                      const APFloat &value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantFloatOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantFloatOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return llvm::isa<FloatType>(constOp.getType());
@@ -292,6 +334,16 @@ void arith::ConstantIndexOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIndexAttr(value));
 }
 
+arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
+                                                      Location location,
+                                                      int64_t value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, value);
+  auto result = llvm::dyn_cast<ConstantIndexOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantIndexOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isIndex();
@@ -2335,9 +2387,8 @@ class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
     // comparison.
     rewriter.replaceOpWithNewOp<CmpIOp>(
         op, pred, intVal,
-        ConstantOp::create(rewriter,
-            op.getLoc(), intVal.getType(),
-            rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
+        ConstantOp::create(rewriter, op.getLoc(), intVal.getType(),
+                           rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
     return success();
   }
 };
@@ -2374,10 +2425,10 @@ struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
         matchPattern(op.getFalseValue(), m_One())) {
       rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
           op, op.getType(),
-          arith::XOrIOp::create(rewriter,
-              op.getLoc(), op.getCondition(),
-              arith::ConstantIntOp::create(rewriter,
-                  op.getLoc(), op.getCondition().getType(), 1)));
+          arith::XOrIOp::create(
+              rewriter, op.getLoc(), op.getCondition(),
+              arith::ConstantIntOp::create(rewriter, op.getLoc(),
+                                           op.getCondition().getType(), 1)));
       return success();
     }
 
@@ -2440,12 +2491,12 @@ OpFoldResult arith::SelectOp::fold(FoldAdaptor adaptor) {
 
   // Constant-fold constant operands over non-splat constant condition.
   // select %cst_vec, %cst0, %cst1 => %cst2
-  if (auto cond =
-          llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getCondition())) {
-    if (auto lhs =
-            llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getTrueValue())) {
-      if (auto rhs =
-              llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getFalseValue())) {
+  if (auto cond = llvm::dyn_cast_if_present<DenseElementsAttr>(
+          adaptor.getCondition())) {
+    if (auto lhs = llvm::dyn_cast_if_present<DenseElementsAttr>(
+            adaptor.getTrueValue())) {
+      if (auto rhs = llvm::dyn_cast_if_present<DenseElementsAttr>(
+              adaptor.getFalseValue())) {
         SmallVector<Attribute> results;
         results.reserve(static_cast<size_t>(cond.getNumElements()));
         auto condVals = llvm::make_range(cond.value_begin<BoolAttr>(),
@@ -2713,7 +2764,7 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
     return arith::MaximumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minimumf:
     return arith::MinimumFOp::create(builder, loc, lhs, rhs);
-   case AtomicRMWKind::maxnumf:
+  case AtomicRMWKind::maxnumf:
     return arith::MaxNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minnumf:
     return arith::MinNumFOp::create(builder, loc, lhs, rhs);
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index c88b328282275..6d7e2aa0ece7d 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -132,7 +132,7 @@ void ModuleOp::build(OpBuilder &builder, OperationState &state,
 /// Construct a module from the given context.
 ModuleOp ModuleOp::create(Location loc, std::optional<StringRef> name) {
   OpBuilder builder(loc->getContext());
-  return ModuleOp::create(builder, loc, name);
+  return builder.create<ModuleOp>(loc, name);
 }
 
 DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {

>From 949f3a13eb07f55418bebe6697105179d74988d3 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 17:06:46 -0400
Subject: [PATCH 4/5] fix arith+smt+create impmlicit

---
 .../mlir/Dialect/Affine/IR/AffineOps.h        |  21 ++++
 mlir/include/mlir/Dialect/Arith/IR/Arith.h    |   9 ++
 .../Linalg/TransformOps/LinalgTransformOps.td |  80 ++++++------
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   6 +-
 mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td    |   6 +-
 mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td |   6 +-
 .../mlir/Dialect/Vector/IR/VectorOps.td       |  12 +-
 mlir/include/mlir/IR/OpDefinition.h           |   1 +
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    | 116 +++++++++---------
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    |   2 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      |   2 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  98 ++++++++++-----
 .../Affine/Transforms/SuperVectorize.cpp      |  49 ++++----
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |  28 +++++
 .../Async/Transforms/AsyncParallelFor.cpp     | 105 ++++++++--------
 .../GPU/Transforms/MemoryPromotion.cpp        |   6 +-
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp   |  38 ++++--
 17 files changed, 352 insertions(+), 233 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
index 2091faa6b0b02..333de6bbd8a05 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
@@ -114,6 +114,21 @@ class AffineDmaStartOp
                     AffineMap tagMap, ValueRange tagIndices, Value numElements,
                     Value stride = nullptr, Value elementsPerStride = nullptr);
 
+  static AffineDmaStartOp
+  create(OpBuilder &builder, Location location, Value srcMemRef,
+         AffineMap srcMap, ValueRange srcIndices, Value destMemRef,
+         AffineMap dstMap, ValueRange destIndices, Value tagMemRef,
+         AffineMap tagMap, ValueRange tagIndices, Value numElements,
+         Value stride = nullptr, Value elementsPerStride = nullptr);
+
+  static AffineDmaStartOp create(ImplicitLocOpBuilder &builder, Value srcMemRef,
+                                 AffineMap srcMap, ValueRange srcIndices,
+                                 Value destMemRef, AffineMap dstMap,
+                                 ValueRange destIndices, Value tagMemRef,
+                                 AffineMap tagMap, ValueRange tagIndices,
+                                 Value numElements, Value stride = nullptr,
+                                 Value elementsPerStride = nullptr);
+
   /// Returns the operand index of the source memref.
   unsigned getSrcMemRefOperandIndex() { return 0; }
 
@@ -319,6 +334,12 @@ class AffineDmaWaitOp
 
   static void build(OpBuilder &builder, OperationState &result, Value tagMemRef,
                     AffineMap tagMap, ValueRange tagIndices, Value numElements);
+  static AffineDmaWaitOp create(OpBuilder &builder, Location location,
+                                Value tagMemRef, AffineMap tagMap,
+                                ValueRange tagIndices, Value numElements);
+  static AffineDmaWaitOp create(ImplicitLocOpBuilder &builder, Value tagMemRef,
+                                AffineMap tagMap, ValueRange tagIndices,
+                                Value numElements);
 
   static StringRef getOperationName() { return "affine.dma_wait"; }
 
diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 79b09b5fcaccf..0fc3db8e993d8 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -61,6 +61,8 @@ class ConstantIntOp : public arith::ConstantOp {
                     unsigned width);
   static ConstantIntOp create(OpBuilder &builder, Location location,
                               int64_t value, unsigned width);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, int64_t value,
+                              unsigned width);
 
   /// Build a constant int op that produces an integer of the specified type,
   /// which must be an integer type.
@@ -68,12 +70,16 @@ class ConstantIntOp : public arith::ConstantOp {
                     int64_t value);
   static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
                               int64_t value);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, Type type,
+                              int64_t value);
 
   /// Build a constant int op that produces an integer from an APInt
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     const APInt &value);
   static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
                               const APInt &value);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, Type type,
+                              const APInt &value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
@@ -93,6 +99,8 @@ class ConstantFloatOp : public arith::ConstantOp {
                     const APFloat &value);
   static ConstantFloatOp create(OpBuilder &builder, Location location,
                                 FloatType type, const APFloat &value);
+  static ConstantFloatOp create(ImplicitLocOpBuilder &builder, FloatType type,
+                                const APFloat &value);
 
   inline APFloat value() {
     return cast<FloatAttr>(arith::ConstantOp::getValue()).getValue();
@@ -110,6 +118,7 @@ class ConstantIndexOp : public arith::ConstantOp {
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
   static ConstantIndexOp create(OpBuilder &builder, Location location,
                                 int64_t value);
+  static ConstantIndexOp create(ImplicitLocOpBuilder &builder, int64_t value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 4360055e78691..c5c984e09bf67 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -582,9 +582,9 @@ def LinalgCopyToMemrefOp :
   let assemblyFormat = "$target attr-dict `:` "
                        "functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -997,12 +997,12 @@ def PackGreedilyOp : Op<Transform_Dialect, "structured.pack_greedily", [
                                  [DenseArrayCount<3>]>:$matmul_inner_dims_order);
   let results = (outs TransformHandleTypeInterface:$packed_op);
 
-  let builders = [
-    OpBuilder<(ins "Value":$target,
-                   "ArrayRef<OpFoldResult>":$mixedMatmulPackedSizes,
-                   "ArrayRef<int64_t>":$matmulPaddededSizesNextMultipleOf,
-                   CArg<"ArrayRef<int64_t>", "{}">:$matmulDimsInnerDimsOrder)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target,
+  //                  "ArrayRef<OpFoldResult>":$mixedMatmulPackedSizes,
+  //                  "ArrayRef<int64_t>":$matmulPaddededSizesNextMultipleOf,
+  //                  CArg<"ArrayRef<int64_t>", "{}">:$matmulDimsInnerDimsOrder)>
+  // ];
 
   let assemblyFormat = [{
     $target
@@ -2509,10 +2509,10 @@ def HoistRedundantVectorTransfersOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target,
-               CArg<"bool", "false">:$verify_non_zero_trip)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target,
+  //              CArg<"bool", "false">:$verify_non_zero_trip)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
          ::mlir::transform::TransformRewriter &rewriter,
@@ -2546,9 +2546,9 @@ def HoistRedundantVectorBroadcastsOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
          ::mlir::transform::TransformRewriter &rewriter,
@@ -2623,9 +2623,9 @@ def ConvertConv2DToImg2ColOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2666,9 +2666,9 @@ def FlattenElementwiseLinalgOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2715,9 +2715,9 @@ def TransposeConv2DOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2761,9 +2761,9 @@ def TransposeMatmulOp : Op<Transform_Dialect,
     attr-dict `:` functional-type($target, results)
   }];
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2801,9 +2801,9 @@ def InsertSliceToCopyOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2859,9 +2859,9 @@ def MapCopyToThreadsOp :
     `:` functional-type(operands, results)
   }];
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2910,9 +2910,9 @@ def WinogradConv2DOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2947,9 +2947,9 @@ def DecomposeWinogradOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..118877ea145fd 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1751,9 +1751,9 @@ def DeclareMapperInfoOp : OpenMP_Op<"declare_mapper.info", [
     parent DeclareMapperOp.
   }] # clausesDescription;
 
-  let builders = [
-    OpBuilder<(ins CArg<"const DeclareMapperInfoOperands &">:$clauses)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins CArg<"const DeclareMapperInfoOperands &">:$clauses)>
+  // ];
 
   let extraClassDeclaration = [{
     // Override BlockArgOpenMPOpInterface method because `map` clauses have no
diff --git a/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td b/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
index 1523762efc18f..7fecf89605f06 100644
--- a/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
+++ b/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
@@ -171,9 +171,9 @@ def Ptr_TypeOffsetOp : Pointer_Op<"type_offset", [Pure]> {
 
   let arguments = (ins TypeAttr:$elementType);
   let results = (outs AnySignlessIntegerOrIndex:$result);
-  let builders = [
-    OpBuilder<(ins "Type":$elementType)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Type":$elementType)>
+  // ];
   let assemblyFormat = [{
     $elementType attr-dict `:` type($result)
   }];
diff --git a/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td b/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
index ea94dfd8fbd2a..5a919395c0954 100644
--- a/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
+++ b/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
@@ -54,9 +54,9 @@ class VariadicIntOp<string mnemonic> : SMTIntOp<mnemonic, [Pure, Commutative]> {
   let assemblyFormat = "$inputs attr-dict";
 
   let builders = [
-    OpBuilder<(ins "mlir::ValueRange":$inputs), [{
-      build($_builder, $_state, $_builder.getType<smt::IntType>(), inputs);
-    }]>,
+    // OpBuilder<(ins "mlir::ValueRange":$inputs), [{
+    //   build($_builder, $_state, $_builder.getType<smt::IntType>(), inputs);
+    // }]>,
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index ec2c87ca1cf44..5d0c34ff8da34 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -3011,12 +3011,12 @@ def Vector_ScanOp :
       vector<4x8x16x32xf32>, vector<4x16x32xf32>
     ```
   }];
-  let builders = [
-    OpBuilder<(ins "Value":$source, "Value":$initial_value,
-                   "CombiningKind":$kind,
-                   CArg<"int64_t", "0">:$reduction_dim,
-                   CArg<"bool", "true">:$inclusive)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$source, "Value":$initial_value,
+  //                  "CombiningKind":$kind,
+  //                  CArg<"int64_t", "0">:$reduction_dim,
+  //                  CArg<"bool", "true">:$inclusive)>
+  // ];
   let extraClassDeclaration = [{
     VectorType getSourceType() {
       return ::llvm::cast<VectorType>(getSource().getType());
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index 663c256c848df..75c3aea0792ac 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -30,6 +30,7 @@
 namespace mlir {
 class Builder;
 class OpBuilder;
+class ImplicitLocOpBuilder;
 
 /// This class implements `Optional` functionality for ParseResult. We don't
 /// directly use Optional here, because it provides an implicit conversion
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 3f82342001ea5..69eec9d9bbf3e 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -20,8 +20,8 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
+#include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
@@ -114,7 +114,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
   auto makeConst = [&](int32_t index) -> Value {
     return LLVM::ConstantOp::create(rewriter, loc, IntegerType::get(ctx, 32),
-                                             rewriter.getI32IntegerAttr(index));
+                                    rewriter.getI32IntegerAttr(index));
   };
 
   if (arrayType) {
@@ -147,11 +147,11 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
         Value x1 =
             LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i * 2);
         Value x2 = LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult,
-                                                         i * 2 + 1);
+                                                i * 2 + 1);
         vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
-                                                     x1, makeConst(0));
+                                            x1, makeConst(0));
         vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
-                                                     x2, makeConst(1));
+                                            x2, makeConst(1));
         elements.push_back(vec);
       }
     }
@@ -160,7 +160,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
     Value result = LLVM::PoisonOp::create(rewriter, loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
       result = LLVM::InsertValueOp::create(rewriter, loc, result, el.value(),
-                                                    el.index());
+                                           el.index());
     }
     return result;
   }
@@ -208,8 +208,8 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
                          innerArrayTy.getElementType() == f32Ty)) {
       for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
            idx < innerSize; idx++) {
-        result.push_back(LLVM::ExtractElementOp::create(b,
-            toUse,
+        result.push_back(LLVM::ExtractElementOp::create(
+            b, toUse,
             LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(idx))));
       }
       continue;
@@ -285,8 +285,8 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
-    Value ldMatrixResult = NVVM::LdMatrixOp::create(b,
-        ldMatrixResultType, srcPtr,
+    Value ldMatrixResult = NVVM::LdMatrixOp::create(
+        b, ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
                                      : NVVM::MMALayout::row);
@@ -375,16 +375,16 @@ struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
     Type desiredRetTy = typeConverter->convertType(op->getResultTypes()[0]);
     Type intrinsicResTy = inferIntrinsicResultType(
         typeConverter->convertType(op->getResultTypes()[0]));
-    Value intrinsicResult = NVVM::MmaOp::create(b,
-        intrinsicResTy, matA, matB, matC,
-        /*shape=*/gemmShape,
-        /*b1Op=*/std::nullopt,
-        /*intOverflow=*/overflow,
-        /*multiplicandPtxTypes=*/
-        std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
-        /*multiplicandLayouts=*/
-        std::array<NVVM::MMALayout, 2>{NVVM::MMALayout::row,
-                                       NVVM::MMALayout::col});
+    Value intrinsicResult =
+        NVVM::MmaOp::create(b, intrinsicResTy, matA, matB, matC,
+                            /*shape=*/gemmShape,
+                            /*b1Op=*/std::nullopt,
+                            /*intOverflow=*/overflow,
+                            /*multiplicandPtxTypes=*/
+                            std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
+                            /*multiplicandLayouts=*/
+                            std::array<NVVM::MMALayout, 2>{
+                                NVVM::MMALayout::row, NVVM::MMALayout::col});
     rewriter.replaceOp(op, convertIntrinsicResult(op.getLoc(), intrinsicResTy,
                                                   desiredRetTy, intrinsicResult,
                                                   rewriter));
@@ -566,14 +566,15 @@ static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
   asmVals.push_back(indexData);
 
   return LLVM::InlineAsmOp::create(b,
-      /*resultTypes=*/intrinsicResultType,
-      /*operands=*/asmVals,
-      /*asm_string=*/asmStr,
-      /*constraints=*/constraintStr,
-      /*has_side_effects=*/true,
-      /*is_align_stack=*/false, LLVM::TailCallKind::None,
-      /*asm_dialect=*/asmDialectAttr,
-      /*operand_attrs=*/ArrayAttr());
+                                   /*resultTypes=*/intrinsicResultType,
+                                   /*operands=*/asmVals,
+                                   /*asm_string=*/asmStr,
+                                   /*constraints=*/constraintStr,
+                                   /*has_side_effects=*/true,
+                                   /*is_align_stack=*/false,
+                                   LLVM::TailCallKind::None,
+                                   /*asm_dialect=*/asmDialectAttr,
+                                   /*operand_attrs=*/ArrayAttr());
 }
 
 /// Lowers `nvgpu.mma.sp.sync` to inline assembly.
@@ -698,12 +699,12 @@ struct NVGPUAsyncCopyLowering
       // filled with zeros.
       Value c3I32 =
           LLVM::ConstantOp::create(b, b.getI32Type(), b.getI32IntegerAttr(3));
-      Value bitwidth = LLVM::ConstantOp::create(b,
-          b.getI32Type(),
+      Value bitwidth = LLVM::ConstantOp::create(
+          b, b.getI32Type(),
           b.getI32IntegerAttr(srcMemrefType.getElementTypeBitWidth()));
       Value srcElementsI32 = LLVM::TruncOp::create(b, b.getI32Type(), srcBytes);
-      srcBytes = LLVM::LShrOp::create(b,
-          LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
+      srcBytes = LLVM::LShrOp::create(
+          b, LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
     }
     // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
     // 16 dst bytes.
@@ -712,14 +713,15 @@ struct NVGPUAsyncCopyLowering
             ? NVVM::LoadCacheModifierKind::CG
             : NVVM::LoadCacheModifierKind::CA;
 
-    NVVM::CpAsyncOp::create(b,
-        dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
+    NVVM::CpAsyncOp::create(
+        b, dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
         NVVM::LoadCacheModifierKindAttr::get(op->getContext(), cacheModifier),
         srcBytes);
 
     // Drop the result token.
-    Value zero = LLVM::ConstantOp::create(b,
-        IntegerType::get(op.getContext(), 32), rewriter.getI32IntegerAttr(0));
+    Value zero =
+        LLVM::ConstantOp::create(b, IntegerType::get(op.getContext(), 32),
+                                 rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
   }
@@ -735,9 +737,9 @@ struct NVGPUAsyncCreateGroupLowering
                   ConversionPatternRewriter &rewriter) const override {
     NVVM::CpAsyncCommitGroupOp::create(rewriter, op.getLoc());
     // Drop the result token.
-    Value zero = LLVM::ConstantOp::create(rewriter,
-        op->getLoc(), IntegerType::get(op.getContext(), 32),
-        rewriter.getI32IntegerAttr(0));
+    Value zero = LLVM::ConstantOp::create(rewriter, op->getLoc(),
+                                          IntegerType::get(op.getContext(), 32),
+                                          rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
   }
@@ -771,8 +773,8 @@ struct NVGPUMBarrierCreateLowering
     SymbolTable symbolTable(moduleOp);
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(&moduleOp.front());
-    auto global = memref::GlobalOp::create(rewriter,
-        funcOp->getLoc(), "__mbarrier",
+    auto global = memref::GlobalOp::create(
+        rewriter, funcOp->getLoc(), "__mbarrier",
         /*sym_visibility=*/rewriter.getStringAttr("private"),
         /*type=*/barrierType,
         /*initial_value=*/ElementsAttr(),
@@ -1119,7 +1121,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 
 static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
   return LLVM::ConstantOp::create(b, b.getIntegerType(64),
-                                    b.getI32IntegerAttr(index));
+                                  b.getI32IntegerAttr(index));
 }
 
 /// Returns a Value that holds data type enum that is expected by CUDA driver.
@@ -1182,11 +1184,11 @@ struct NVGPUTmaCreateDescriptorOpLowering
     auto promotedOperands = getTypeConverter()->promoteOperands(
         b.getLoc(), op->getOperands(), adaptor.getOperands(), b);
 
-    Value boxArrayPtr = LLVM::AllocaOp::create(b, llvmPointerType, llvmInt64Type,
-                                                 makeI64Const(b, 5));
+    Value boxArrayPtr = LLVM::AllocaOp::create(
+        b, llvmPointerType, llvmInt64Type, makeI64Const(b, 5));
     for (auto [index, value] : llvm::enumerate(adaptor.getBoxDimensions())) {
       Value gep = LLVM::GEPOp::create(b, llvmPointerType, llvmPointerType,
-                                        boxArrayPtr, makeI64Const(b, index));
+                                      boxArrayPtr, makeI64Const(b, index));
       LLVM::StoreOp::create(b, value, gep);
     }
 
@@ -1430,9 +1432,9 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      return NVVM::WgmmaMmaAsyncOp::create(b,
-          matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
-          itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
+      return NVVM::WgmmaMmaAsyncOp::create(
+          b, matrixC.getType(), matrixC, descriptorA, descriptorB, shape,
+          itypeA, itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
           overflow);
     }
 
@@ -1444,7 +1446,8 @@ struct NVGPUWarpgroupMmaOpLowering
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
+        Value matrixC =
+            LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
             matrixC = generateWgmma(i, j, k, matrixC);
@@ -1452,7 +1455,7 @@ struct NVGPUWarpgroupMmaOpLowering
       }
       for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
         wgmmaResult = LLVM::InsertValueOp::create(b, wgmmaResult.getType(),
-                                                    wgmmaResult, matrix, idx);
+                                                  wgmmaResult, matrix, idx);
       }
       return wgmmaResult;
     }
@@ -1486,9 +1489,9 @@ struct NVGPUWarpgroupMmaOpLowering
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
     Value generateWarpgroupMma() {
-      NVVM::WgmmaFenceAlignedOp::create(b, );
+      NVVM::WgmmaFenceAlignedOp::create(b);
       Value wgmmaResult = generateWgmmaGroup();
-      NVVM::WgmmaGroupSyncAlignedOp::create(b, );
+      NVVM::WgmmaGroupSyncAlignedOp::create(b);
       NVVM::WgmmaWaitGroupSyncOp::create(b, op.getWaitGroup());
       return wgmmaResult;
     }
@@ -1626,7 +1629,8 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     auto stype = cast<LLVM::LLVMStructType>(matriDValue.getType());
     for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(matrixD);
-      Value innerStructValue = LLVM::ExtractValueOp::create(b, matriDValue, idx);
+      Value innerStructValue =
+          LLVM::ExtractValueOp::create(b, matriDValue, idx);
       storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
@@ -1656,15 +1660,15 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
       auto structType = cast<LLVM::LLVMStructType>(s);
       Value structValue = LLVM::ExtractValueOp::create(b, packStruct, idx);
       for (unsigned i = 0; i < structType.getBody().size(); ++i) {
-        structValue = LLVM::InsertValueOp::create(b,
-            structType, structValue, zero, ArrayRef<int64_t>({i}));
+        structValue = LLVM::InsertValueOp::create(b, structType, structValue,
+                                                  zero, ArrayRef<int64_t>({i}));
       }
       innerStructs.push_back(structValue);
     }
     // Pack the inner structs into a single struct
     for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
       packStruct = LLVM::InsertValueOp::create(b, packStruct.getType(),
-                                                 packStruct, matrix, idx);
+                                               packStruct, matrix, idx);
     }
     rewriter.replaceOp(op, packStruct);
     return success();
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 901b1eb265e32..4e8468078b9ad 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -709,7 +709,7 @@ class ExecutionModePattern
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
-    structValue = LLVM::InsertValueOp::create(rewriter, loc, structValue,
+    structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
                                                        executionMode, 0);
 
     // Insert extra operands if they exist into execution mode info struct.
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 1b31d391cb0bc..d21833736a718 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -670,7 +670,7 @@ static Value createFPReductionComparisonOpLowering(
 
   if (accumulator) {
     result =
-        typename VectorToScalarMapper<LLVMRedIntrinOp>::Type::create(rewriter,
+        rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
             loc, result, accumulator);
   }
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 263663d041f3a..95cca386d2360 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -1749,6 +1749,32 @@ void AffineDmaStartOp::build(OpBuilder &builder, OperationState &result,
   }
 }
 
+AffineDmaStartOp AffineDmaStartOp::create(
+    OpBuilder &builder, Location location, Value srcMemRef, AffineMap srcMap,
+    ValueRange srcIndices, Value destMemRef, AffineMap dstMap,
+    ValueRange destIndices, Value tagMemRef, AffineMap tagMap,
+    ValueRange tagIndices, Value numElements, Value stride,
+    Value elementsPerStride) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, srcMemRef, srcMap, srcIndices, destMemRef, dstMap,
+        destIndices, tagMemRef, tagMap, tagIndices, numElements, stride,
+        elementsPerStride);
+  auto result = llvm::dyn_cast<AffineDmaStartOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
+AffineDmaStartOp AffineDmaStartOp::create(
+    ImplicitLocOpBuilder &builder, Value srcMemRef, AffineMap srcMap,
+    ValueRange srcIndices, Value destMemRef, AffineMap dstMap,
+    ValueRange destIndices, Value tagMemRef, AffineMap tagMap,
+    ValueRange tagIndices, Value numElements, Value stride,
+    Value elementsPerStride) {
+  return create(builder, builder.getLoc(), srcMemRef, srcMap, srcIndices,
+                destMemRef, dstMap, destIndices, tagMemRef, tagMap, tagIndices,
+                numElements, stride, elementsPerStride);
+}
+
 void AffineDmaStartOp::print(OpAsmPrinter &p) {
   p << " " << getSrcMemRef() << '[';
   p.printAffineMapOfSSAIds(getSrcMapAttr(), getSrcIndices());
@@ -1919,6 +1945,25 @@ void AffineDmaWaitOp::build(OpBuilder &builder, OperationState &result,
   result.addOperands(numElements);
 }
 
+AffineDmaWaitOp AffineDmaWaitOp::create(OpBuilder &builder, Location location,
+                                        Value tagMemRef, AffineMap tagMap,
+                                        ValueRange tagIndices,
+                                        Value numElements) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, tagMemRef, tagMap, tagIndices, numElements);
+  auto result = llvm::dyn_cast<AffineDmaWaitOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
+AffineDmaWaitOp AffineDmaWaitOp::create(ImplicitLocOpBuilder &builder,
+                                        Value tagMemRef, AffineMap tagMap,
+                                        ValueRange tagIndices,
+                                        Value numElements) {
+  return create(builder, builder.getLoc(), tagMemRef, tagMap, tagIndices,
+                numElements);
+}
+
 void AffineDmaWaitOp::print(OpAsmPrinter &p) {
   p << " " << getTagMemRef() << '[';
   SmallVector<Value, 2> operands(getTagIndices());
@@ -2690,8 +2735,8 @@ FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInits());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  AffineForOp newLoop = AffineForOp::create(rewriter,
-      getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
+  AffineForOp newLoop = AffineForOp::create(
+      rewriter, getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
       getUpperBoundOperands(), getUpperBoundMap(), getStepAsInt(), inits);
 
   // Generate the new yield values and append them to the scf.yield operation.
@@ -2849,7 +2894,7 @@ buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb,
                              int64_t ub, int64_t step,
                              AffineForOp::BodyBuilderFn bodyBuilderFn) {
   return AffineForOp::create(builder, loc, lb, ub, step,
-                                     /*iterArgs=*/ValueRange(), bodyBuilderFn);
+                             /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
 /// Creates an affine loop from the bounds that may or may not be constants.
@@ -2863,8 +2908,8 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
     return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
                                         ubConst.value(), step, bodyBuilderFn);
   return AffineForOp::create(builder, loc, lb, builder.getDimIdentityMap(), ub,
-                                     builder.getDimIdentityMap(), step,
-                                     /*iterArgs=*/ValueRange(), bodyBuilderFn);
+                             builder.getDimIdentityMap(), step,
+                             /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
 void mlir::affine::buildAffineLoopNest(
@@ -4908,8 +4953,8 @@ struct DropUnitExtentBasis
 
     if (!newBasis.empty()) {
       // Will drop the leading nullptr from `basis` if there was no outer bound.
-      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(rewriter,
-          loc, delinearizeOp.getLinearIndex(), newBasis);
+      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(
+          rewriter, loc, delinearizeOp.getLinearIndex(), newBasis);
       int newIndex = 0;
       // Map back the new delinearized indices to the values they replace.
       for (auto &replacement : replacements) {
@@ -4973,12 +5018,12 @@ struct CancelDelinearizeOfLinearizeDisjointExactTail
       return success();
     }
 
-    Value newLinearize = affine::AffineLinearizeIndexOp::create(rewriter,
-        linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
+    Value newLinearize = affine::AffineLinearizeIndexOp::create(
+        rewriter, linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
         ArrayRef<OpFoldResult>{linearizeBasis}.drop_back(numMatches),
         linearizeOp.getDisjoint());
-    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-        delinearizeOp.getLoc(), newLinearize,
+    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), newLinearize,
         ArrayRef<OpFoldResult>{delinearizeBasis}.drop_back(numMatches),
         delinearizeOp.hasOuterBound());
     SmallVector<Value> mergedResults(newDelinearize.getResults());
@@ -5050,19 +5095,16 @@ struct SplitDelinearizeSpanningLastLinearizeArg final
           delinearizeOp,
           "need at least two elements to form the basis product");
 
-    Value linearizeWithoutBack =
-        affine::AffineLinearizeIndexOp::create(rewriter,
-            linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
-            linearizeOp.getDynamicBasis(),
-            linearizeOp.getStaticBasis().drop_back(),
-            linearizeOp.getDisjoint());
-    auto delinearizeWithoutSplitPart =
-        affine::AffineDelinearizeIndexOp::create(rewriter,
-            delinearizeOp.getLoc(), linearizeWithoutBack,
-            delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
-            delinearizeOp.hasOuterBound());
-    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(rewriter,
-        delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
+    Value linearizeWithoutBack = affine::AffineLinearizeIndexOp::create(
+        rewriter, linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
+        linearizeOp.getDynamicBasis(), linearizeOp.getStaticBasis().drop_back(),
+        linearizeOp.getDisjoint());
+    auto delinearizeWithoutSplitPart = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), linearizeWithoutBack,
+        delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
+        delinearizeOp.hasOuterBound());
+    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
         basis.take_back(elemsToSplit), /*hasOuterBound=*/true);
     SmallVector<Value> results = llvm::to_vector(
         llvm::concat<Value>(delinearizeWithoutSplitPart.getResults(),
@@ -5439,16 +5481,16 @@ struct CancelLinearizeOfDelinearizePortion final
       newDelinBasis.erase(newDelinBasis.begin() + m.delinStart,
                           newDelinBasis.begin() + m.delinStart + m.length);
       newDelinBasis.insert(newDelinBasis.begin() + m.delinStart, newSize);
-      auto newDelinearize = AffineDelinearizeIndexOp::create(rewriter,
-          m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
+      auto newDelinearize = AffineDelinearizeIndexOp::create(
+          rewriter, m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
           newDelinBasis);
 
       // Since there may be other uses of the indices we just merged together,
       // create a residual affine.delinearize_index that delinearizes the
       // merged output into its component parts.
       Value combinedElem = newDelinearize.getResult(m.delinStart);
-      auto residualDelinearize = AffineDelinearizeIndexOp::create(rewriter,
-          m.delinearize.getLoc(), combinedElem, basisToMerge);
+      auto residualDelinearize = AffineDelinearizeIndexOp::create(
+          rewriter, m.delinearize.getLoc(), combinedElem, basisToMerge);
 
       // Swap all the uses of the unaffected delinearize outputs to the new
       // delinearization so that the old code can be removed if this
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index b078514cbf978..827e1dff9f319 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -905,8 +905,8 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   for (auto resultExpr : map.getResults()) {
     auto singleResMap =
         AffineMap::get(map.getNumDims(), map.getNumSymbols(), resultExpr);
-    auto afOp = state.AffineApplyOp::create(builder, op->getLoc(), singleResMap,
-                                                    mapOperands);
+    auto afOp = AffineApplyOp::create(state.builder, op->getLoc(), singleResMap,
+                                      mapOperands);
     results.push_back(afOp);
   }
 }
@@ -961,7 +961,7 @@ static arith::ConstantOp vectorizeConstant(arith::ConstantOp constOp,
   auto vecForOp = cast<AffineForOp>(parentOp);
   state.builder.setInsertionPointToStart(vecForOp.getBody());
   auto newConstOp =
-      state.arith::ConstantOp::create(builder, constOp.getLoc(), vecAttr);
+      arith::ConstantOp::create(state.builder, constOp.getLoc(), vecAttr);
 
   // Register vector replacement for future uses in the scope.
   state.registerOpVectorReplacement(constOp, newConstOp);
@@ -986,8 +986,8 @@ static Operation *vectorizeAffineApplyOp(AffineApplyOp applyOp,
     }
   }
 
-  auto newApplyOp = state.AffineApplyOp::create(builder,
-      applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
+  auto newApplyOp = AffineApplyOp::create(
+      state.builder, applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
 
   // Register the new affine.apply result.
   state.registerValueScalarReplacement(applyOp.getResult(),
@@ -1010,7 +1010,7 @@ static arith::ConstantOp createInitialVector(arith::AtomicRMWKind reductionKind,
   auto vecTy = getVectorType(scalarTy, state.strategy);
   auto vecAttr = DenseElementsAttr::get(vecTy, valueAttr);
   auto newConstOp =
-      state.arith::ConstantOp::create(builder, oldOperand.getLoc(), vecAttr);
+      arith::ConstantOp::create(state.builder, oldOperand.getLoc(), vecAttr);
 
   return newConstOp;
 }
@@ -1062,11 +1062,11 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   AffineMap ubMap = vecForOp.getUpperBoundMap();
   Value ub;
   if (ubMap.getNumResults() == 1)
-    ub = state.AffineApplyOp::create(builder, loc, vecForOp.getUpperBoundMap(),
-                                             vecForOp.getUpperBoundOperands());
+    ub = AffineApplyOp::create(state.builder, loc, vecForOp.getUpperBoundMap(),
+                               vecForOp.getUpperBoundOperands());
   else
-    ub = state.AffineMinOp::create(builder, loc, vecForOp.getUpperBoundMap(),
-                                           vecForOp.getUpperBoundOperands());
+    ub = AffineMinOp::create(state.builder, loc, vecForOp.getUpperBoundMap(),
+                             vecForOp.getUpperBoundOperands());
   // Then we compute the number of (original) iterations left in the loop.
   AffineExpr subExpr =
       state.builder.getAffineDimExpr(0) - state.builder.getAffineDimExpr(1);
@@ -1080,7 +1080,7 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   Type maskTy = VectorType::get(state.strategy->vectorSizes,
                                 state.builder.getIntegerType(1));
   Value mask =
-      state.vector::CreateMaskOp::create(builder, loc, maskTy, itersLeft);
+      vector::CreateMaskOp::create(state.builder, loc, maskTy, itersLeft);
 
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ creating a mask:\n"
                     << itersLeft << "\n"
@@ -1123,8 +1123,8 @@ static Operation *vectorizeUniform(Value uniformVal,
   state.builder.setInsertionPointAfterValue(uniformScalarRepl);
 
   auto vectorTy = getVectorType(uniformVal.getType(), state.strategy);
-  auto bcastOp = state.BroadcastOp::create(builder, uniformVal.getLoc(),
-                                                   vectorTy, uniformScalarRepl);
+  auto bcastOp = BroadcastOp::create(state.builder, uniformVal.getLoc(),
+                                     vectorTy, uniformScalarRepl);
   state.registerValueVectorReplacement(uniformVal, bcastOp);
   return bcastOp;
 }
@@ -1256,8 +1256,8 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.vector::TransferReadOp::create(builder,
-      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
+  auto transfer = vector::TransferReadOp::create(
+      state.builder, loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
       /*padding=*/std::nullopt, permutationMap);
 
   // Register replacement for future uses in the scope.
@@ -1303,9 +1303,9 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.vector::TransferWriteOp::create(builder,
-      storeOp.getLoc(), vectorValue, storeOp.getMemRef(), indices,
-      permutationMap);
+  auto transfer = vector::TransferWriteOp::create(
+      state.builder, storeOp.getLoc(), vectorValue, storeOp.getMemRef(),
+      indices, permutationMap);
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
 
   // Register replacement for future uses in the scope.
@@ -1387,10 +1387,10 @@ static Operation *vectorizeAffineForOp(AffineForOp forOp,
     }
   }
 
-  auto vecForOp = state.AffineForOp::create(builder,
-      forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(),
-      forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep,
-      vecIterOperands,
+  auto vecForOp = AffineForOp::create(
+      state.builder, forOp.getLoc(), forOp.getLowerBoundOperands(),
+      forOp.getLowerBoundMap(), forOp.getUpperBoundOperands(),
+      forOp.getUpperBoundMap(), newStep, vecIterOperands,
       /*bodyBuilder=*/[](OpBuilder &, Location, Value, ValueRange) {
         // Make sure we don't create a default terminator in the loop body as
         // the proper terminator will be added during vectorization.
@@ -1512,8 +1512,8 @@ static Operation *vectorizeAffineYieldOp(AffineYieldOp yieldOp,
       // IterOperands are neutral element vectors.
       Value neutralVal = cast<AffineForOp>(newParentOp).getInits()[i];
       state.builder.setInsertionPoint(combinerOps.back());
-      Value maskedReducedVal = state.arith::SelectOp::create(builder,
-          reducedVal.getLoc(), mask, reducedVal, neutralVal);
+      Value maskedReducedVal = arith::SelectOp::create(
+          state.builder, reducedVal.getLoc(), mask, reducedVal, neutralVal);
       LLVM_DEBUG(
           dbgs() << "\n[early-vect]+++++ masking an input to a binary op that"
                     "produces value for a yield Op: "
@@ -1865,7 +1865,6 @@ verifyLoopNesting(const std::vector<SmallVector<AffineForOp, 2>> &loops) {
   return success();
 }
 
-
 /// External utility to vectorize affine loops in 'loops' using the n-D
 /// vectorization factors in 'vectorSizes'. By default, each vectorization
 /// factor is applied inner-to-outer to the loops of each loop nest.
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index aa623c19a9218..164489422a3e4 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -267,6 +267,12 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  int64_t value,
+                                                  unsigned width) {
+  return create(builder, builder.getLoc(), value, width);
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, int64_t value) {
   arith::ConstantOp::build(builder, result, type,
@@ -283,6 +289,11 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  Type type, int64_t value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, const APInt &value) {
   arith::ConstantOp::build(builder, result, type,
@@ -299,6 +310,12 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  Type type,
+                                                  const APInt &value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -322,6 +339,12 @@ arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantFloatOp
+arith::ConstantFloatOp::create(ImplicitLocOpBuilder &builder, FloatType type,
+                               const APFloat &value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 bool arith::ConstantFloatOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return llvm::isa<FloatType>(constOp.getType());
@@ -344,6 +367,11 @@ arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIndexOp
+arith::ConstantIndexOp::create(ImplicitLocOpBuilder &builder, int64_t value) {
+  return create(builder, builder.getLoc(), value);
+}
+
 bool arith::ConstantIndexOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isIndex();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index 447411ad989ba..09367ab544ff8 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -369,21 +369,22 @@ static ParallelComputeFunction createParallelComputeFunction(
       ImplicitLocOpBuilder b(loc, nestedBuilder);
 
       // Compute induction variable for `loopIdx`.
-      computeBlockInductionVars[loopIdx] = arith::AddIOp::create(b,
-          lowerBounds[loopIdx], arith::MulIOp::create(b, iv, steps[loopIdx]));
+      computeBlockInductionVars[loopIdx] =
+          arith::AddIOp::create(b, lowerBounds[loopIdx],
+                                arith::MulIOp::create(b, iv, steps[loopIdx]));
 
       // Check if we are inside first or last iteration of the loop.
-      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
-      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
+      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
+      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
 
       // Check if the previous loop is in its first or last iteration.
       if (loopIdx > 0) {
-        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(b,
-            isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
-        isBlockLastCoord[loopIdx] = arith::AndIOp::create(b,
-            isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
+        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(
+            b, isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
+        isBlockLastCoord[loopIdx] = arith::AndIOp::create(
+            b, isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
       }
 
       // Keep building loop nest.
@@ -392,20 +393,20 @@ static ParallelComputeFunction createParallelComputeFunction(
           // For block aligned loops we always iterate starting from 0 up to
           // the loop trip counts.
           scf::ForOp::create(b, c0, tripCounts[loopIdx + 1], c1, ValueRange(),
-                               workLoopBuilder(loopIdx + 1));
+                             workLoopBuilder(loopIdx + 1));
 
         } else {
           // Select nested loop lower/upper bounds depending on our position in
           // the multi-dimensional iteration space.
           auto lb = arith::SelectOp::create(b, isBlockFirstCoord[loopIdx],
-                                              blockFirstCoord[loopIdx + 1], c0);
+                                            blockFirstCoord[loopIdx + 1], c0);
 
           auto ub = arith::SelectOp::create(b, isBlockLastCoord[loopIdx],
-                                              blockEndCoord[loopIdx + 1],
-                                              tripCounts[loopIdx + 1]);
+                                            blockEndCoord[loopIdx + 1],
+                                            tripCounts[loopIdx + 1]);
 
           scf::ForOp::create(b, lb, ub, c1, ValueRange(),
-                               workLoopBuilder(loopIdx + 1));
+                             workLoopBuilder(loopIdx + 1));
         }
 
         scf::YieldOp::create(b, loc);
@@ -424,7 +425,7 @@ static ParallelComputeFunction createParallelComputeFunction(
   };
 
   scf::ForOp::create(b, blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
-                       workLoopBuilder(0));
+                     workLoopBuilder(0));
   func::ReturnOp::create(b, ValueRange());
 
   return {op.getNumLoops(), func, std::move(computeFuncType.captures)};
@@ -537,13 +538,13 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
       operands[2] = end;
 
       func::CallOp::create(executeBuilder, executeLoc, func.getSymName(),
-                                          func.getResultTypes(), operands);
+                           func.getResultTypes(), operands);
       async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to dispatch half of the block range.
     auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
-                                       executeBodyBuilder);
+                                     executeBodyBuilder);
     AddToGroupOp::create(b, indexTy, execute.getToken(), group);
     scf::YieldOp::create(b, ValueRange({start, midIndex}));
   }
@@ -558,8 +559,7 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   computeFuncOperands.append(forwardedInputs.begin(), forwardedInputs.end());
 
   func::CallOp::create(b, computeFunc.func.getSymName(),
-                         computeFunc.func.getResultTypes(),
-                         computeFuncOperands);
+                       computeFunc.func.getResultTypes(), computeFuncOperands);
   func::ReturnOp::create(b, ValueRange());
 
   return func;
@@ -605,9 +605,9 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     appendBlockComputeOperands(operands);
 
     func::CallOp::create(b, parallelComputeFunction.func.getSymName(),
-                           parallelComputeFunction.func.getResultTypes(),
-                           operands);
-    scf::YieldOp::create(b, );
+                         parallelComputeFunction.func.getResultTypes(),
+                         operands);
+    scf::YieldOp::create(b);
   };
 
   auto asyncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -624,12 +624,12 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     appendBlockComputeOperands(operands);
 
     func::CallOp::create(b, asyncDispatchFunction.getSymName(),
-                           asyncDispatchFunction.getResultTypes(), operands);
+                         asyncDispatchFunction.getResultTypes(), operands);
 
     // Wait for the completion of all parallel compute operations.
     AwaitAllOp::create(b, group);
 
-    scf::YieldOp::create(b, );
+    scf::YieldOp::create(b);
   };
 
   // Dispatch either single block compute function, or launch async dispatch.
@@ -682,16 +682,15 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
                                   Location executeLoc, ValueRange executeArgs) {
       func::CallOp::create(executeBuilder, executeLoc, compute.getSymName(),
-                                          compute.getResultTypes(),
-                                          computeFuncOperands(iv));
+                           compute.getResultTypes(), computeFuncOperands(iv));
       async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to launch parallel computate function.
     auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
-                                       executeBodyBuilder);
+                                     executeBodyBuilder);
     AddToGroupOp::create(b, rewriter.getIndexType(), execute.getToken(), group);
-    scf::YieldOp::create(b, );
+    scf::YieldOp::create(b);
   };
 
   // Iterate over all compute blocks and launch parallel compute operations.
@@ -699,7 +698,7 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
 
   // Call parallel compute function for the first block in the caller thread.
   func::CallOp::create(b, compute.getSymName(), compute.getResultTypes(),
-                         computeFuncOperands(c0));
+                       computeFuncOperands(c0));
 
   // Wait for the completion of all async compute operations.
   AwaitAllOp::create(b, group);
@@ -800,7 +799,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     if (numWorkerThreads >= 0)
       numWorkerThreadsVal = arith::ConstantIndexOp::create(b, numWorkerThreads);
     else
-      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b, );
+      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b);
 
     // With large number of threads the value of creating many compute blocks
     // is reduced because the problem typically becomes memory bound. For this
@@ -819,16 +818,16 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
         {4, 4.0f}, {8, 2.0f}, {16, 1.0f}, {32, 0.8f}, {64, 0.6f}};
     const float initialOvershardingFactor = 8.0f;
 
-    Value scalingFactor = arith::ConstantFloatOp::create(b,
-        b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
+    Value scalingFactor = arith::ConstantFloatOp::create(
+        b, b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
     for (const std::pair<int, float> &p : overshardingBrackets) {
       Value bracketBegin = arith::ConstantIndexOp::create(b, p.first);
-      Value inBracket = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
-      Value bracketScalingFactor = arith::ConstantFloatOp::create(b,
-          b.getF32Type(), llvm::APFloat(p.second));
-      scalingFactor = arith::SelectOp::create(b, inBracket, bracketScalingFactor,
-                                                scalingFactor);
+      Value inBracket = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
+      Value bracketScalingFactor = arith::ConstantFloatOp::create(
+          b, b.getF32Type(), llvm::APFloat(p.second));
+      scalingFactor = arith::SelectOp::create(
+          b, inBracket, bracketScalingFactor, scalingFactor);
     }
     Value numWorkersIndex =
         arith::IndexCastOp::create(b, b.getI32Type(), numWorkerThreadsVal);
@@ -841,8 +840,8 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     Value scaledWorkers =
         arith::IndexCastOp::create(b, b.getIndexType(), scaledNumInt);
 
-    Value maxComputeBlocks = arith::MaxSIOp::create(b,
-        arith::ConstantIndexOp::create(b, 1), scaledWorkers);
+    Value maxComputeBlocks = arith::MaxSIOp::create(
+        b, arith::ConstantIndexOp::create(b, 1), scaledWorkers);
 
     // Compute parallel block size from the parallel problem size:
     //   blockSize = min(tripCount,
@@ -869,7 +868,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       doDispatch(b, rewriter, compute, op, blockSize, blockCount, tripCounts);
-      scf::YieldOp::create(b, );
+      scf::YieldOp::create(b);
     };
 
     // Dispatch parallel compute function with hints for unrolling inner loops.
@@ -880,27 +879,27 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       // Align the block size to be a multiple of the statically known
       // number of iterations in the inner loops.
-      Value numIters = arith::ConstantIndexOp::create(b,
-          numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value alignedBlockSize = arith::MulIOp::create(b,
-          arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
+      Value numIters = arith::ConstantIndexOp::create(
+          b, numIterations[op.getNumLoops() - numUnrollableLoops]);
+      Value alignedBlockSize = arith::MulIOp::create(
+          b, arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
       doDispatch(b, rewriter, compute, op, alignedBlockSize, blockCount,
                  tripCounts);
-      scf::YieldOp::create(b, );
+      scf::YieldOp::create(b);
     };
 
     // Dispatch to block aligned compute function only if the computed block
     // size is larger than the number of iterations in the unrollable inner
     // loops, because otherwise it can reduce the available parallelism.
     if (numUnrollableLoops > 0) {
-      Value numIters = arith::ConstantIndexOp::create(b,
-          numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value useBlockAlignedComputeFn = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::sge, blockSize, numIters);
+      Value numIters = arith::ConstantIndexOp::create(
+          b, numIterations[op.getNumLoops() - numUnrollableLoops]);
+      Value useBlockAlignedComputeFn = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::sge, blockSize, numIters);
 
       scf::IfOp::create(b, useBlockAlignedComputeFn, dispatchBlockAligned,
-                          dispatchDefault);
-      scf::YieldOp::create(b, );
+                        dispatchDefault);
+      scf::YieldOp::create(b);
     } else {
       dispatchDefault(b, loc);
     }
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 2bee1a1619665..4af08dad08462 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -18,7 +18,7 @@
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
+#include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
@@ -131,10 +131,10 @@ static void insertCopies(Region &region, Location loc, Value from, Value to) {
 
   auto b = ImplicitLocOpBuilder::atBlockBegin(loc, &region.front());
   insertCopyLoops(b, from, to);
-  gpu::BarrierOp::create(b, );
+  gpu::BarrierOp::create(b);
 
   b.setInsertionPoint(&region.front().back());
-  gpu::BarrierOp::create(b, );
+  gpu::BarrierOp::create(b);
   insertCopyLoops(b, to, from);
 }
 
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 20c483e52da04..f35cfa6826388 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -238,6 +238,10 @@ static const char *const inlineCreateBody = R"(
   return __res__;
 )";
 
+static const char *const inlineCreateBodyImplicitLoc = R"(
+  return create(builder, builder.getLoc(){0});
+)";
+
 //===----------------------------------------------------------------------===//
 // Utility structs and functions
 //===----------------------------------------------------------------------===//
@@ -2579,35 +2583,47 @@ static bool canInferType(const Operator &op) {
 
 void OpEmitter::genInlineCreateBody(
     const SmallVector<MethodParameter> &paramList) {
-  SmallVector<MethodParameter> createParamList;
+  SmallVector<MethodParameter> createParamListOpBuilder;
+  SmallVector<MethodParameter> createParamListImplicitLocOpBuilder;
   SmallVector<llvm::StringRef, 4> nonBuilderStateArgsList;
-  createParamList.emplace_back("::mlir::OpBuilder &", "builder");
+  createParamListOpBuilder.emplace_back("::mlir::OpBuilder &", "builder");
+  createParamListImplicitLocOpBuilder.emplace_back(
+      "::mlir::ImplicitLocOpBuilder &", "builder");
   std::string locParamName = "location";
   while (llvm::find_if(paramList, [&locParamName](const MethodParameter &p) {
            return p.getName() == locParamName;
          }) != paramList.end()) {
     locParamName += "_";
   }
-  createParamList.emplace_back("::mlir::Location", locParamName);
+  createParamListOpBuilder.emplace_back("::mlir::Location", locParamName);
 
   for (auto &param : paramList) {
     if (param.getType() == "::mlir::OpBuilder &" ||
         param.getType() == "::mlir::OperationState &")
       continue;
-    createParamList.emplace_back(param.getType(), param.getName(),
-                                 param.getDefaultValue(), param.isOptional());
+    createParamListOpBuilder.emplace_back(param.getType(), param.getName(),
+                                          param.getDefaultValue(),
+                                          param.isOptional());
+    createParamListImplicitLocOpBuilder.emplace_back(
+        param.getType(), param.getName(), param.getDefaultValue(),
+        param.isOptional());
     nonBuilderStateArgsList.push_back(param.getName());
   }
-  auto *c = opClass.addStaticMethod(opClass.getClassName(), "create",
-                                    createParamList);
+  auto *cWithLoc = opClass.addStaticMethod(opClass.getClassName(), "create",
+                                           createParamListOpBuilder);
+  auto *cImplicitLoc = opClass.addStaticMethod(
+      opClass.getClassName(), "create", createParamListImplicitLocOpBuilder);
   std::string nonBuilderStateArgs = "";
   if (!nonBuilderStateArgsList.empty()) {
     llvm::raw_string_ostream nonBuilderStateArgsOS(nonBuilderStateArgs);
     interleaveComma(nonBuilderStateArgsList, nonBuilderStateArgsOS);
     nonBuilderStateArgs = ", " + nonBuilderStateArgs;
   }
-  c->body() << llvm::formatv(inlineCreateBody, locParamName,
-                             nonBuilderStateArgs, opClass.getClassName());
+  cWithLoc->body() << llvm::formatv(inlineCreateBody, locParamName,
+                                    nonBuilderStateArgs,
+                                    opClass.getClassName());
+  cImplicitLoc->body() << llvm::formatv(inlineCreateBodyImplicitLoc,
+                                        nonBuilderStateArgs);
 }
 
 void OpEmitter::genSeparateArgParamBuilder() {
@@ -3087,8 +3103,7 @@ void OpEmitter::genBuilder() {
 
     std::optional<StringRef> body = builder.getBody();
     auto properties = body ? Method::Static : Method::StaticDeclaration;
-    auto *method =
-        opClass.addMethod("void", "build", properties, std::move(arguments));
+    auto *method = opClass.addMethod("void", "build", properties, arguments);
     if (body)
       ERROR_IF_PRUNED(method, "build", op);
 
@@ -3100,6 +3115,7 @@ void OpEmitter::genBuilder() {
     fctx.addSubst("_state", builderOpState);
     if (body)
       method->body() << tgfmt(*body, &fctx);
+    genInlineCreateBody(arguments);
   }
 
   // Generate default builders that requires all result type, operands, and

>From 4eaf6e6ac28b654d756c67ef171abd80569b281a Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 18:26:17 -0400
Subject: [PATCH 5/5] implicitbuilder

---
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    |   1 +
 flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp   |   1 +
 flang/lib/Optimizer/Dialect/FIROps.cpp        |   1 +
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp     |   1 +
 mlir/examples/toy/Ch2/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch3/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch4/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch5/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch6/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch7/mlir/Dialect.cpp        |   4 +-
 .../transform/Ch2/lib/MyExtension.cpp         |   1 +
 .../transform/Ch3/lib/MyExtension.cpp         |   1 +
 .../transform/Ch4/lib/MyExtension.cpp         |   1 +
 mlir/lib/CAPI/Dialect/PDL.cpp                 |   1 +
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    |   1 +
 mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp  |   1 +
 mlir/lib/Dialect/AMX/IR/AMXDialect.cpp        |   1 +
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |   1 +
 .../TransformOps/AffineTransformOps.cpp       |   1 +
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |   1 +
 .../lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp |   1 +
 .../ArmNeonVectorTransformOps.cpp             |   1 +
 mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp         |   1 +
 mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp  |   1 +
 .../TransformOps/ArmSVEVectorTransformOps.cpp |   1 +
 mlir/lib/Dialect/Async/IR/Async.cpp           |   1 +
 .../Bufferization/IR/BufferizationOps.cpp     |  20 ++--
 mlir/lib/Dialect/Complex/IR/ComplexOps.cpp    |   5 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.cpp |  16 +--
 .../DLTI/TransformOps/DLTITransformOps.cpp    |   1 +
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp           |   1 +
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |   7 +-
 .../Func/TransformOps/FuncTransformOps.cpp    |   3 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |   9 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |   5 +-
 .../GPU/Transforms/MemoryPromotion.cpp        |   1 +
 mlir/lib/Dialect/IRDL/IR/IRDL.cpp             |   1 +
 mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp          |   1 +
 mlir/lib/Dialect/Index/IR/IndexOps.cpp        |  16 ++-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |   7 +-
 mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp    |   1 +
 mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp   |   1 +
 mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp    |   1 +
 mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp    |   1 +
 .../lib/Dialect/MLProgram/IR/MLProgramOps.cpp |   1 +
 mlir/lib/Dialect/MPI/IR/MPIOps.cpp            |   5 +-
 mlir/lib/Dialect/Math/IR/MathOps.cpp          |   1 +
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |  27 +++--
 .../TransformOps/MemRefTransformOps.cpp       |   8 +-
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |  15 +--
 mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp    |   1 +
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       |   1 +
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |   1 +
 mlir/lib/Dialect/PDL/IR/PDL.cpp               |   1 +
 mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp   |   1 +
 mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp        |   1 +
 mlir/lib/Dialect/Quant/IR/QuantOps.cpp        |   1 +
 mlir/lib/Dialect/SCF/IR/SCF.cpp               | 106 ++++++++---------
 .../SCF/TransformOps/SCFTransformOps.cpp      |   1 +
 mlir/lib/Dialect/SMT/IR/SMTOps.cpp            |   1 +
 .../Dialect/SPIRV/IR/SPIRVOpDefinition.cpp    |   1 +
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  64 ++++++-----
 .../SparseTensor/IR/SparseTensorDialect.cpp   |   4 +-
 .../TransformOps/SparseTensorTransformOps.cpp |   1 +
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      | 107 +++++++++---------
 .../TransformOps/TensorTransformOps.cpp       |   1 +
 .../DebugExtension/DebugExtension.cpp         |   1 +
 .../DebugExtension/DebugExtensionOps.cpp      |   1 +
 .../lib/Dialect/Transform/IR/TransformOps.cpp |   1 +
 .../IRDLExtension/IRDLExtensionOps.cpp        |   1 +
 .../LoopExtension/LoopExtensionOps.cpp        |   1 +
 .../PDLExtension/PDLExtensionOps.cpp          |   1 +
 mlir/lib/Dialect/UB/IR/UBOps.cpp              |   1 +
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  56 ++++-----
 .../TransformOps/VectorTransformOps.cpp       |   1 +
 .../Dialect/X86Vector/IR/X86VectorDialect.cpp |   3 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   1 +
 mlir/lib/IR/BuiltinDialect.cpp                |   1 +
 mlir/test/lib/Dialect/Test/TestOps.td         |   2 +-
 .../TestTransformDialectExtension.cpp         |   1 +
 .../TestTilingInterfaceTransformOps.cpp       |   1 +
 .../test/lib/Transforms/TestTransformsOps.cpp |   1 +
 mlir/test/python/lib/PythonTestDialect.cpp    |   1 +
 83 files changed, 346 insertions(+), 235 deletions(-)

diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index 687007d957225..cef95a13c5a50 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
index 19ad6bed512c7..d7d294ab621d8 100644
--- a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Optimizer/Dialect/FIRCG/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..2381d44e7bb77 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -26,6 +26,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index df6ce12215d26..7f2ec418af40a 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/APInt.h"
diff --git a/mlir/examples/toy/Ch2/mlir/Dialect.cpp b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
index 489f348c8be52..fc4c3c75dcfda 100644
--- a/mlir/examples/toy/Ch2/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -144,7 +145,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -280,7 +282,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch3/mlir/Dialect.cpp b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
index 708855f18cf45..af3cbb385d332 100644
--- a/mlir/examples/toy/Ch3/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -144,7 +145,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -280,7 +282,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch4/mlir/Dialect.cpp b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
index 076a75a26619b..e3ab656c193a3 100644
--- a/mlir/examples/toy/Ch4/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch5/mlir/Dialect.cpp b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
index fb7c742a01802..81c9e8f9a143f 100644
--- a/mlir/examples/toy/Ch5/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch6/mlir/Dialect.cpp b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
index fb7c742a01802..81c9e8f9a143f 100644
--- a/mlir/examples/toy/Ch6/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch7/mlir/Dialect.cpp b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
index 52881db87d86b..64aa4d8e8995d 100644
--- a/mlir/examples/toy/Ch7/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpImplementation.h"
@@ -429,7 +430,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/transform/Ch2/lib/MyExtension.cpp b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
index b4b27e97d266e..e24714b408e29 100644
--- a/mlir/examples/transform/Ch2/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/examples/transform/Ch3/lib/MyExtension.cpp b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
index 4b2123fa71d31..5af5cc8ac5052 100644
--- a/mlir/examples/transform/Ch3/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
@@ -17,6 +17,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/examples/transform/Ch4/lib/MyExtension.cpp b/mlir/examples/transform/Ch4/lib/MyExtension.cpp
index fa0ffc9dc2e8a..86884d3bee7a0 100644
--- a/mlir/examples/transform/Ch4/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch4/lib/MyExtension.cpp
@@ -13,6 +13,7 @@
 
 #include "MyExtension.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE_MATCHER "transform-matcher"
diff --git a/mlir/lib/CAPI/Dialect/PDL.cpp b/mlir/lib/CAPI/Dialect/PDL.cpp
index bd8b13c6516e2..d2a83da643a89 100644
--- a/mlir/lib/CAPI/Dialect/PDL.cpp
+++ b/mlir/lib/CAPI/Dialect/PDL.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 69eec9d9bbf3e..cd92e1dbba090 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -22,6 +22,7 @@
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index 4613d14461969..79859f0513e84 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 748ff1edbfeb2..111d93e39c15e 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 95cca386d2360..91f85f2f366e8 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
diff --git a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
index b1e40d9b289ec..267179f26c52e 100644
--- a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
+++ b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 164489422a3e4..c4c2a043a8024 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
index 1ded35af78052..5f9cd148f5e0a 100644
--- a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
+++ b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
index d07e6a52d8b5f..f0b7a8a4b0954 100644
--- a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/ArmNeon/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
index cb3a665844872..2d8903df00055 100644
--- a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp b/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
index 594c9b4c270f2..bd7130b667b6b 100644
--- a/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
+++ b/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp b/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
index b2ca4fc1eaa8c..b50679239a840 100644
--- a/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index 71ee198fc0c23..e7d95575c6edf 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -10,6 +10,7 @@
 
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 4308f46bee6ff..b4910d45e1ae6 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include <optional>
 
@@ -322,8 +323,8 @@ struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
         newShape, op.getType().getElementType(), op.getType().getEncoding());
     if (newType == op.getType())
       return failure();
-    auto newOp = AllocTensorOp::create(rewriter,
-        op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
+    auto newOp = AllocTensorOp::create(rewriter, op.getLoc(), newType,
+                                       newDynamicSizes, /*copy=*/Value());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -515,7 +516,7 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
 
     if (source.getType() != cloneOp.getType())
       source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
-                                               cloneOp.getType(), source);
+                                      cloneOp.getType(), source);
     rewriter.replaceOp(cloneOp, source);
     rewriter.eraseOp(redundantDealloc);
     return success();
@@ -644,8 +645,9 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return ToTensorOp::create(builder,
-      loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
+  return ToTensorOp::create(
+      builder, loc, memref::getTensorTypeFromMemRefType(getDest().getType()),
+      getDest(),
       /*restrict=*/true, getWritable());
 }
 
@@ -808,7 +810,7 @@ struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
     auto memrefType = MemRefType::get(srcTensorType.getShape(),
                                       srcTensorType.getElementType());
     Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
-                                               tensorCastOperand.getOperand());
+                                      tensorCastOperand.getOperand());
     rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                 memref);
     return success();
@@ -1016,7 +1018,7 @@ struct DeallocRemoveDuplicateRetainedMemrefs
     // same as the number of condition operands.
     auto newDeallocOp =
         DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
-                                   deallocOp.getConditions(), newRetained);
+                          deallocOp.getConditions(), newRetained);
     SmallVector<Value> replacements(
         llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
           return newDeallocOp.getUpdatedConditions()[idx];
@@ -1037,8 +1039,8 @@ struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     if (deallocOp.getMemrefs().empty()) {
-      Value constFalse = arith::ConstantOp::create(rewriter,
-          deallocOp.getLoc(), rewriter.getBoolAttr(false));
+      Value constFalse = arith::ConstantOp::create(rewriter, deallocOp.getLoc(),
+                                                   rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
                                         constFalse));
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
index fb97045687d65..34008aa5c4d6a 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 
@@ -20,9 +21,7 @@ using namespace mlir::complex;
 // ConstantOp
 //===----------------------------------------------------------------------===//
 
-OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) {
-  return getValue();
-}
+OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValue(); }
 
 void ConstantOp::getAsmResultNames(
     function_ref<void(Value, StringRef)> setNameFn) {
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 38e47cc5e3714..66593783a14f1 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -89,7 +90,7 @@ LogicalResult AssertOp::canonicalize(AssertOp op, PatternRewriter &rewriter) {
   return failure();
 }
 
-// This side effect models "program termination". 
+// This side effect models "program termination".
 void AssertOp::getEffects(
     SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
         &effects) {
@@ -312,8 +313,9 @@ struct SimplifyCondBranchIdenticalSuccessors
       if (std::get<0>(it) == std::get<1>(it))
         mergedOperands.push_back(std::get<0>(it));
       else
-        mergedOperands.push_back(arith::SelectOp::create(rewriter,
-            condbr.getLoc(), condition, std::get<0>(it), std::get<1>(it)));
+        mergedOperands.push_back(
+            arith::SelectOp::create(rewriter, condbr.getLoc(), condition,
+                                    std::get<0>(it), std::get<1>(it)));
     }
 
     rewriter.replaceOpWithNewOp<BranchOp>(condbr, trueDest, mergedOperands);
@@ -412,8 +414,8 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantTrue)
-            constantTrue = arith::ConstantOp::create(rewriter,
-                condbr.getLoc(), ty, rewriter.getBoolAttr(true));
+            constantTrue = arith::ConstantOp::create(
+                rewriter, condbr.getLoc(), ty, rewriter.getBoolAttr(true));
 
           rewriter.modifyOpInPlace(use.getOwner(),
                                    [&] { use.set(constantTrue); });
@@ -427,8 +429,8 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantFalse)
-            constantFalse = arith::ConstantOp::create(rewriter,
-                condbr.getLoc(), ty, rewriter.getBoolAttr(false));
+            constantFalse = arith::ConstantOp::create(
+                rewriter, condbr.getLoc(), ty, rewriter.getBoolAttr(false));
 
           rewriter.modifyOpInPlace(use.getOwner(),
                                    [&] { use.set(constantFalse); });
diff --git a/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp b/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
index 02c41b4fe8113..4dbd1659e24dc 100644
--- a/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
+++ b/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/Utils/Utils.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index f9852257a4a7c..432c90836c66d 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Types.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index 1c0d3cd338006..1ac11f6a774f3 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -55,7 +56,7 @@ Operation *FuncDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                             Type type, Location loc) {
   if (ConstantOp::isBuildableWith(value, type))
     return ConstantOp::create(builder, loc, type,
-                                      llvm::cast<FlatSymbolRefAttr>(value));
+                              llvm::cast<FlatSymbolRefAttr>(value));
   return nullptr;
 }
 
@@ -143,9 +144,7 @@ LogicalResult ConstantOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   return success();
 }
 
-OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) {
-  return getValueAttr();
-}
+OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValueAttr(); }
 
 void ConstantOp::getAsmResultNames(
     function_ref<void(Value, StringRef)> setNameFn) {
diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
index 2ece1bcffb1b0..822c21e90ca21 100644
--- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
+++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
@@ -172,7 +173,7 @@ transform::CastAndCallOp::apply(transform::TransformRewriter &rewriter,
   }
 
   auto callOp = func::CallOp::create(rewriter, insertionPoint->getLoc(),
-                                              targetFunction, inputs);
+                                     targetFunction, inputs);
 
   // Cast the call results back to the expected types. If any conversions fail
   // this is a definite failure as the call has been constructed at this point.
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index f7eb41be287e4..8d6962d2d2be8 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -23,6 +23,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -1382,9 +1383,9 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
                       int32_t offset, int32_t width, ShuffleMode mode) {
   build(builder, result, value,
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(offset)),
+                                  builder.getI32IntegerAttr(offset)),
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(width)),
+                                  builder.getI32IntegerAttr(width)),
         mode);
 }
 
@@ -1396,9 +1397,9 @@ void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
                      int32_t offset, int32_t width) {
   build(builder, result, value,
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(offset)),
+                                  builder.getI32IntegerAttr(offset)),
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(width)));
+                                  builder.getI32IntegerAttr(width)));
 }
 
 LogicalResult RotateOp::verify() {
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index 2035f371694a7..26adfade879d8 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -31,6 +31,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/Visitors.h"
@@ -561,7 +562,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   if (originalBasisWasProvided) {
     for (Value tmpPredicate : builderResult.predicateOps) {
       predicate = predicate ? arith::AndIOp::create(rewriter, loc, predicate,
-                                                             tmpPredicate)
+                                                    tmpPredicate)
                             : tmpPredicate;
     }
   }
@@ -574,7 +575,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   if (predicate) {
     // Step 6.a. If predicated, move at the beginning.
     auto ifOp = scf::IfOp::create(rewriter, loc, predicate,
-                                           /*withElseRegion=*/false);
+                                  /*withElseRegion=*/false);
     targetBlock = ifOp.thenBlock();
     insertionPoint = ifOp.thenBlock()->begin();
   } else {
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 4af08dad08462..4a670373272a7 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
index bcc9f0b109ac2..d7bc15b866686 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
index 7ec3aa2741023..c62e5c9c50183 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/IRDL/IR/IRDL.h"
 #include "mlir/Dialect/IRDL/IRDLSymbols.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/ValueRange.h"
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Index/IR/IndexOps.cpp b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
index 2b32962f9eabb..61089d7665e6c 100644
--- a/mlir/lib/Dialect/Index/IR/IndexOps.cpp
+++ b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Index/IR/IndexAttrs.h"
 #include "mlir/Dialect/Index/IR/IndexDialect.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -130,15 +131,18 @@ canonicalizeAssociativeCommutativeBinaryOp(BinaryOp op,
 
   auto lhsOp = op.getLhs().template getDefiningOp<BinaryOp>();
   if (!lhsOp)
-    return rewriter.notifyMatchFailure(op.getLoc(), "LHS is not the same BinaryOp");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "LHS is not the same BinaryOp");
 
   if (!mlir::matchPattern(lhsOp.getRhs(), mlir::m_Constant()))
-    return rewriter.notifyMatchFailure(op.getLoc(), "RHS of LHS op is not a constant");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "RHS of LHS op is not a constant");
 
   Value c = rewriter.createOrFold<BinaryOp>(op->getLoc(), op.getRhs(),
-                                           lhsOp.getRhs());
+                                            lhsOp.getRhs());
   if (c.getDefiningOp<BinaryOp>())
-    return rewriter.notifyMatchFailure(op.getLoc(), "new BinaryOp was not folded");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "new BinaryOp was not folded");
 
   rewriter.replaceOpWithNewOp<BinaryOp>(op, lhsOp.getLhs(), c);
   return success();
@@ -717,10 +721,10 @@ LogicalResult CmpOp::canonicalize(CmpOp op, PatternRewriter &rewriter) {
   index::CmpOp newCmp;
   if (rhsIsZero)
     newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
-                                           subOp.getLhs(), subOp.getRhs());
+                                  subOp.getLhs(), subOp.getRhs());
   else
     newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
-                                           subOp.getRhs(), subOp.getLhs());
+                                  subOp.getRhs(), subOp.getLhs());
   rewriter.replaceOp(op, newCmp);
   return success();
 }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 546704e903f47..e25a90b1d276e 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
@@ -4313,8 +4314,8 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
   MLIRContext *ctx = builder.getContext();
   auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
-  auto global = LLVM::GlobalOp::create(moduleBuilder,
-      loc, type, /*isConstant=*/true, linkage, name,
+  auto global = LLVM::GlobalOp::create(
+      moduleBuilder, loc, type, /*isConstant=*/true, linkage, name,
       builder.getStringAttr(value), /*alignment=*/0);
 
   LLVMPointerType ptrType = LLVMPointerType::get(ctx);
@@ -4322,7 +4323,7 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   Value globalPtr =
       LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
   return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
-                                     ArrayRef<GEPArg>{0, 0});
+                             ArrayRef<GEPArg>{0, 0});
 }
 
 bool mlir::LLVM::satisfiesLLVMModule(Operation *op) {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index aaf6b0593c2e6..d95b146d27e5c 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -25,6 +25,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
index 9671afd52fa77..0bb8ecf98c533 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
index bd9d3528ceb74..b1676b4e58c39 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
index 9e497829ba723..fbedc714c200b 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp b/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
index 37ddca101f64b..877f8e72caa31 100644
--- a/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
+++ b/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/MLProgram/IR/MLProgram.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index 749cca0575581..da2317cd4eb97 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 
 using namespace mlir;
@@ -60,8 +61,8 @@ struct FoldRank final : public mlir::OpRewritePattern<mlir::mpi::CommRankOp> {
     if (!isa<IntegerAttr>(dltiAttr.value()))
       return op->emitError()
              << "Expected an integer attribute for MPI:comm_world_rank";
-    Value res = arith::ConstantIndexOp::create(b,
-        op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
+    Value res = arith::ConstantIndexOp::create(
+        b, op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
     if (Value retVal = op.getRetval())
       b.replaceOp(op, {retVal, res});
     else
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index a21631cbf8510..b09a67818fff6 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 706ac4b4867f4..cb5fe132494b3 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -213,9 +214,9 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
 
     // Create and insert the alloc op for the new memref.
-    auto newAlloc = AllocLikeOp::create(rewriter,
-        alloc.getLoc(), newMemRefType, dynamicSizes, alloc.getSymbolOperands(),
-        alloc.getAlignmentAttr());
+    auto newAlloc = AllocLikeOp::create(rewriter, alloc.getLoc(), newMemRefType,
+                                        dynamicSizes, alloc.getSymbolOperands(),
+                                        alloc.getAlignmentAttr());
     // Insert a cast so we have the same type as the old alloc.
     rewriter.replaceOpWithNewOp<CastOp>(alloc, alloc.getType(), newAlloc);
     return success();
@@ -1358,8 +1359,9 @@ static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
     assert(isa<Attribute>(maybeConstant) &&
            "The constified value should be either unchanged (i.e., == result) "
            "or a constant");
-    Value constantVal = arith::ConstantIndexOp::create(rewriter,
-        loc, llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
+    Value constantVal = arith::ConstantIndexOp::create(
+        rewriter, loc,
+        llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
     for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
       // modifyOpInPlace: lambda cannot capture structured bindings in C++17
       // yet.
@@ -2587,8 +2589,9 @@ struct CollapseShapeOpMemRefCastFolder
       rewriter.modifyOpInPlace(
           op, [&]() { op.getSrcMutable().assign(cast.getSource()); });
     } else {
-      Value newOp = CollapseShapeOp::create(rewriter,
-          op->getLoc(), cast.getSource(), op.getReassociationIndices());
+      Value newOp =
+          CollapseShapeOp::create(rewriter, op->getLoc(), cast.getSource(),
+                                  op.getReassociationIndices());
       rewriter.replaceOpWithNewOp<CastOp>(op, op.getType(), newOp);
     }
     return success();
@@ -3212,8 +3215,8 @@ class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
     if (!resultType)
       return failure();
 
-    Value newSubView = SubViewOp::create(rewriter,
-        subViewOp.getLoc(), resultType, castOp.getSource(),
+    Value newSubView = SubViewOp::create(
+        rewriter, subViewOp.getLoc(), resultType, castOp.getSource(),
         subViewOp.getOffsets(), subViewOp.getSizes(), subViewOp.getStrides(),
         subViewOp.getStaticOffsets(), subViewOp.getStaticSizes(),
         subViewOp.getStaticStrides());
@@ -3534,9 +3537,9 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
       return failure();
 
     // Create new ViewOp.
-    auto newViewOp = ViewOp::create(rewriter,
-        viewOp.getLoc(), newMemRefType, viewOp.getOperand(0),
-        viewOp.getByteShift(), newOperands);
+    auto newViewOp = ViewOp::create(rewriter, viewOp.getLoc(), newMemRefType,
+                                    viewOp.getOperand(0), viewOp.getByteShift(),
+                                    newOperands);
     // Insert a cast so we have the same type as the old memref type.
     rewriter.replaceOpWithNewOp<CastOp>(viewOp, viewOp.getType(), newViewOp);
     return success();
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 8176ed0f02dce..525ae1e15bca6 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -23,6 +23,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/Support/Debug.h"
@@ -156,9 +157,10 @@ transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
       Type resultType = alloca.getResult().getType();
       OpBuilder builder(rewriter.getContext());
       // TODO: Add a better builder for this.
-      globalOp = memref::GlobalOp::create(builder,
-          loc, StringAttr::get(ctx, "alloca"), StringAttr::get(ctx, "private"),
-          TypeAttr::get(resultType), Attribute{}, UnitAttr{}, IntegerAttr{});
+      globalOp = memref::GlobalOp::create(
+          builder, loc, StringAttr::get(ctx, "alloca"),
+          StringAttr::get(ctx, "private"), TypeAttr::get(resultType),
+          Attribute{}, UnitAttr{}, IntegerAttr{});
       symbolTable.insert(globalOp);
     }
 
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index 5fd0d3c8a7f1b..7167289ec0e14 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -318,9 +319,9 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
   if (!newShardOp) {
     auto shardingOp =
         ShardingOp::create(builder, operandValue.getLoc(), sharding);
-    newShardOp =
-        ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                                /*annotate_for_users*/ false);
+    newShardOp = ShardOp::create(builder, operandValue.getLoc(), operandValue,
+                                 shardingOp,
+                                 /*annotate_for_users*/ false);
   }
   operandValue.replaceUsesWithIf(
       newShardOp, [operandOp, operandValue](OpOperand &use) {
@@ -332,8 +333,8 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
   }
 
   auto newShardOp2 = ShardOp::create(builder, operandValue.getLoc(), newShardOp,
-                                             newShardOp.getSharding(),
-                                             /*annotate_for_users*/ true);
+                                     newShardOp.getSharding(),
+                                     /*annotate_for_users*/ true);
   newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
 }
 
@@ -382,7 +383,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
       ShardingOp::create(builder, operand.get().getLoc(), sharding);
   auto newShardOp =
       ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                              /*annotate_for_users*/ true);
+                      /*annotate_for_users*/ true);
   IRRewriter rewriter(builder);
   rewriter.replaceUsesWithIf(
       operandValue, newShardOp, [operandOp, operandValue](OpOperand &use) {
@@ -397,7 +398,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
   builder.setInsertionPoint(newShardOp);
   auto newPreceedingShardOp =
       ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                              /*annotate_for_users*/ false);
+                      /*annotate_for_users*/ false);
   rewriter.replaceUsesWithIf(
       newShardOp.getSrc(), newPreceedingShardOp, [&newShardOp](OpOperand &use) {
         return use.getOwner() == newShardOp.getOperation();
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index abbdb6a0f53ec..4a89699b80023 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 80c807e774a7e..d4707619fa273 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index ffc84781f77ff..8fa59488678ee 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/SymbolTable.h"
diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp
index d3f7c9798b9b8..ffe53bbe80607 100644
--- a/mlir/lib/Dialect/PDL/IR/PDL.cpp
+++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
index 9b1f11d835282..5cc84ab29e246 100644
--- a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
+++ b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
index c488144508128..3695262155387 100644
--- a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
+++ b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/Dialect/Ptr/IR/PtrOps.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
diff --git a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
index e23a0d6aba825..cd4198b431669 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Quant/IR/Quant.h"
 #include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 82fa50a6ac6e0..8ead566703ab4 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
@@ -247,7 +248,7 @@ struct MultiBlockExecuteInliner : public OpRewritePattern<ExecuteRegionOp> {
       if (YieldOp yieldOp = dyn_cast<YieldOp>(blk.getTerminator())) {
         rewriter.setInsertionPoint(yieldOp);
         cf::BranchOp::create(rewriter, yieldOp.getLoc(), postBlock,
-                                      yieldOp.getResults());
+                             yieldOp.getResults());
         rewriter.eraseOp(yieldOp);
       }
     }
@@ -557,8 +558,8 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  scf::ForOp newLoop = scf::ForOp::create(rewriter,
-      getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
+  scf::ForOp newLoop = scf::ForOp::create(
+      rewriter, getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
       [](OpBuilder &, Location, Value, ValueRange) {});
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
 
@@ -673,8 +674,8 @@ void mlir::scf::promote(RewriterBase &rewriter, scf::ForallOp forallOp) {
     Value dst = parallelInsertSliceOp.getDest();
     Value src = parallelInsertSliceOp.getSource();
     if (llvm::isa<TensorType>(src.getType())) {
-      results.push_back(tensor::InsertSliceOp::create(rewriter,
-          forallOp.getLoc(), dst.getType(), src, dst,
+      results.push_back(tensor::InsertSliceOp::create(
+          rewriter, forallOp.getLoc(), dst.getType(), src, dst,
           parallelInsertSliceOp.getOffsets(), parallelInsertSliceOp.getSizes(),
           parallelInsertSliceOp.getStrides(),
           parallelInsertSliceOp.getStaticOffsets(),
@@ -722,8 +723,8 @@ LoopNest mlir::scf::buildLoopNest(
   ValueRange currentIterArgs = iterArgs;
   Location currentLoc = loc;
   for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
-    auto loop = scf::ForOp::create(builder,
-        currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
+    auto loop = scf::ForOp::create(
+        builder, currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
             ValueRange args) {
           ivs.push_back(iv);
@@ -801,8 +802,8 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   }
 
   // 2. Create the new forOp shell.
-  scf::ForOp newForOp = scf::ForOp::create(rewriter,
-      forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
+  scf::ForOp newForOp = scf::ForOp::create(
+      rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
       forOp.getStep(), newIterOperands);
   newForOp->setAttrs(forOp->getAttrs());
   Block &newBlock = newForOp.getRegion().front();
@@ -926,9 +927,9 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     if (!canonicalize)
       return failure();
 
-    scf::ForOp newForOp = scf::ForOp::create(rewriter,
-        forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-        forOp.getStep(), newIterArgs);
+    scf::ForOp newForOp =
+        scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(),
+                           forOp.getUpperBound(), forOp.getStep(), newIterArgs);
     newForOp->setAttrs(forOp->getAttrs());
     Block &newBlock = newForOp.getRegion().front();
 
@@ -971,7 +972,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
         if (keepMask[idx])
           filteredOperands.push_back(mergedTerminator.getOperand(idx));
       scf::YieldOp::create(rewriter, mergedTerminator.getLoc(),
-                                    filteredOperands);
+                           filteredOperands);
     };
 
     rewriter.mergeBlocks(&oldBlock, &newBlock, newBlockTransferArgs);
@@ -1685,8 +1686,8 @@ struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
 
     // Step 3. Create a new scf.forall op with the new shared_outs' operands
     //         fetched earlier
-    auto newForallOp = scf::ForallOp::create(rewriter,
-        forallOp.getLoc(), forallOp.getMixedLowerBound(),
+    auto newForallOp = scf::ForallOp::create(
+        rewriter, forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
         forallOp.getMapping(),
         /*bodyBuilderFn =*/[](OpBuilder &, Location, ValueRange) {});
@@ -1783,8 +1784,8 @@ struct ForallOpSingleOrZeroIterationDimsFolder
     // Replace the loop by a lower-dimensional loop.
     ForallOp newOp;
     newOp = ForallOp::create(rewriter, loc, newMixedLowerBounds,
-                                      newMixedUpperBounds, newMixedSteps,
-                                      op.getOutputs(), std::nullopt, nullptr);
+                             newMixedUpperBounds, newMixedSteps,
+                             op.getOutputs(), std::nullopt, nullptr);
     newOp.getBodyRegion().getBlocks().clear();
     // The new loop needs to keep all attributes from the old one, except for
     // "operandSegmentSizes" and static loop bound attributes which capture
@@ -1867,16 +1868,17 @@ struct FoldTensorCastOfOutputIntoForallOp
 
     // Create new loop.
     Location loc = forallOp.getLoc();
-    auto newForallOp = ForallOp::create(rewriter,
-        loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
-        forallOp.getMixedStep(), newOutputTensors, forallOp.getMapping(),
+    auto newForallOp = ForallOp::create(
+        rewriter, loc, forallOp.getMixedLowerBound(),
+        forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
+        newOutputTensors, forallOp.getMapping(),
         [&](OpBuilder nestedBuilder, Location nestedLoc, ValueRange bbArgs) {
           auto castBlockArgs =
               llvm::to_vector(bbArgs.take_back(forallOp->getNumResults()));
           for (auto [index, cast] : tensorCastProducers) {
             Value &oldTypeBBArg = castBlockArgs[index];
-            oldTypeBBArg = tensor::CastOp::create(nestedBuilder,
-                nestedLoc, cast.dstType, oldTypeBBArg);
+            oldTypeBBArg = tensor::CastOp::create(nestedBuilder, nestedLoc,
+                                                  cast.dstType, oldTypeBBArg);
           }
 
           // Move old body into new parallel loop.
@@ -1903,7 +1905,7 @@ struct FoldTensorCastOfOutputIntoForallOp
     for (auto &item : tensorCastProducers) {
       Value &oldTypeResult = castResults[item.first];
       oldTypeResult = tensor::CastOp::create(rewriter, loc, item.second.dstType,
-                                                      oldTypeResult);
+                                             oldTypeResult);
     }
     rewriter.replaceOp(forallOp, castResults);
     return success();
@@ -2373,7 +2375,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       return failure();
 
     IfOp replacement = IfOp::create(rewriter, op.getLoc(), nonHoistable, cond,
-                                             /*withElseRegion=*/false);
+                                    /*withElseRegion=*/false);
     if (replacement.thenBlock())
       rewriter.eraseBlock(replacement.thenBlock());
     replacement.getThenRegion().takeBody(op.getThenRegion());
@@ -2398,8 +2400,8 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       } else if (trueVal == falseVal)
         results[it.index()] = trueVal;
       else
-        results[it.index()] = arith::SelectOp::create(rewriter,
-            op.getLoc(), cond, trueVal, falseVal);
+        results[it.index()] = arith::SelectOp::create(rewriter, op.getLoc(),
+                                                      cond, trueVal, falseVal);
     }
 
     rewriter.setInsertionPointToEnd(replacement.thenBlock());
@@ -2450,8 +2452,8 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantTrue)
-          constantTrue = arith::ConstantOp::create(rewriter,
-              op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
+          constantTrue = arith::ConstantOp::create(
+              rewriter, op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
 
         rewriter.modifyOpInPlace(use.getOwner(),
                                  [&]() { use.set(constantTrue); });
@@ -2460,8 +2462,8 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantFalse)
-          constantFalse = arith::ConstantOp::create(rewriter,
-              op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
+          constantFalse = arith::ConstantOp::create(
+              rewriter, op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
 
         rewriter.modifyOpInPlace(use.getOwner(),
                                  [&]() { use.set(constantFalse); });
@@ -2547,8 +2549,8 @@ struct ReplaceIfYieldWithConditionOrValue : public OpRewritePattern<IfOp> {
       if (!trueVal && falseVal) {
         if (!opResult.use_empty()) {
           Dialect *constDialect = trueResult.getDefiningOp()->getDialect();
-          Value notCond = arith::XOrIOp::create(rewriter,
-              op.getLoc(), op.getCondition(),
+          Value notCond = arith::XOrIOp::create(
+              rewriter, op.getLoc(), op.getCondition(),
               constDialect
                   ->materializeConstant(rewriter,
                                         rewriter.getIntegerAttr(i1Ty, 1), i1Ty,
@@ -2661,8 +2663,8 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
     SmallVector<Type> mergedTypes(prevIf.getResultTypes());
     llvm::append_range(mergedTypes, nextIf.getResultTypes());
 
-    IfOp combinedIf = IfOp::create(rewriter,
-        nextIf.getLoc(), mergedTypes, prevIf.getCondition(), /*hasElse=*/false);
+    IfOp combinedIf = IfOp::create(rewriter, nextIf.getLoc(), mergedTypes,
+                                   prevIf.getCondition(), /*hasElse=*/false);
     rewriter.eraseBlock(&combinedIf.getThenRegion().back());
 
     rewriter.inlineRegionBefore(prevIf.getThenRegion(),
@@ -2823,8 +2825,8 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     }
 
     Location loc = op.getLoc();
-    Value newCondition = arith::AndIOp::create(rewriter,
-        loc, op.getCondition(), nestedIf.getCondition());
+    Value newCondition = arith::AndIOp::create(rewriter, loc, op.getCondition(),
+                                               nestedIf.getCondition());
     auto newIf = IfOp::create(rewriter, loc, op.getResultTypes(), newCondition);
     Block *newIfBlock = rewriter.createBlock(&newIf.getThenRegion());
 
@@ -2833,8 +2835,9 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     rewriter.setInsertionPoint(newIf);
 
     for (auto idx : elseYieldsToUpgradeToSelect)
-      results[idx] = arith::SelectOp::create(rewriter,
-          op.getLoc(), op.getCondition(), thenYield[idx], elseYield[idx]);
+      results[idx] =
+          arith::SelectOp::create(rewriter, op.getLoc(), op.getCondition(),
+                                  thenYield[idx], elseYield[idx]);
 
     rewriter.mergeBlocks(nestedIf.thenBlock(), newIfBlock);
     rewriter.setInsertionPointToEnd(newIf.thenBlock());
@@ -3160,8 +3163,8 @@ struct ParallelOpSingleOrZeroIterationDimsFolder
     }
     // Replace the parallel loop by lower-dimensional parallel loop.
     auto newOp =
-        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds, newUpperBounds,
-                                    newSteps, op.getInitVals(), nullptr);
+        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds,
+                           newUpperBounds, newSteps, op.getInitVals(), nullptr);
     // Erase the empty block that was inserted by the builder.
     rewriter.eraseBlock(newOp.getBody());
     // Clone the loop body and remap the block arguments of the collapsed loops
@@ -3541,8 +3544,8 @@ struct WhileConditionTruth : public OpRewritePattern<WhileOp> {
       if (std::get<0>(yieldedAndBlockArgs) == term.getCondition()) {
         if (!std::get<1>(yieldedAndBlockArgs).use_empty()) {
           if (!constantTrue)
-            constantTrue = arith::ConstantOp::create(rewriter,
-                op.getLoc(), term.getCondition().getType(),
+            constantTrue = arith::ConstantOp::create(
+                rewriter, op.getLoc(), term.getCondition().getType(),
                 rewriter.getBoolAttr(true));
 
           rewriter.replaceAllUsesWith(std::get<1>(yieldedAndBlockArgs),
@@ -3684,8 +3687,8 @@ struct RemoveLoopInvariantArgsFromBeforeBlock
       rewriter.replaceOpWithNewOp<YieldOp>(yieldOp, newYieldOpArgs);
     }
 
-    auto newWhile =
-        WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(), newInitArgs);
+    auto newWhile = WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(),
+                                    newInitArgs);
 
     Block &newBeforeBlock = *rewriter.createBlock(
         &newWhile.getBefore(), /*insertPt*/ {},
@@ -3808,7 +3811,7 @@ struct RemoveLoopInvariantValueYielded : public OpRewritePattern<WhileOp> {
     }
 
     auto newWhile = WhileOp::create(rewriter, op.getLoc(), newAfterBlockType,
-                                             op.getOperands());
+                                    op.getOperands());
 
     Block &newAfterBlock =
         *rewriter.createBlock(&newWhile.getAfter(), /*insertPt*/ {},
@@ -4044,7 +4047,7 @@ struct WhileRemoveUnusedArgs : public OpRewritePattern<WhileOp> {
     Location loc = op.getLoc();
     auto newWhileOp =
         WhileOp::create(rewriter, loc, op.getResultTypes(), newInits,
-                                 /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
+                        /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
 
@@ -4091,9 +4094,10 @@ struct WhileRemoveDuplicatedResults : public OpRewritePattern<WhileOp> {
     ValueRange argsRange(newArgs);
 
     Location loc = op.getLoc();
-    auto newWhileOp = scf::WhileOp::create(rewriter,
-        loc, argsRange.getTypes(), op.getInits(), /*beforeBody*/ nullptr,
-        /*afterBody*/ nullptr);
+    auto newWhileOp =
+        scf::WhileOp::create(rewriter, loc, argsRange.getTypes(), op.getInits(),
+                             /*beforeBody*/ nullptr,
+                             /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
 
@@ -4187,8 +4191,8 @@ struct WhileOpAlignBeforeArgs : public OpRewritePattern<WhileOp> {
     for (auto &&[i, j] : llvm::enumerate(*mapping))
       newResultTypes[j] = loop.getResult(i).getType();
 
-    auto newLoop = WhileOp::create(rewriter,
-        loop.getLoc(), newResultTypes, loop.getInits(),
+    auto newLoop = WhileOp::create(
+        rewriter, loop.getLoc(), newResultTypes, loop.getInits(),
         /*beforeBuilder=*/nullptr, /*afterBuilder=*/nullptr);
     auto newBefore = newLoop.getBeforeBody();
     auto newAfter = newLoop.getAfterBody();
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index e9943a3489d46..145afac6df15e 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -25,6 +25,7 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
index c517ef2b18f6f..6af87afd8b83e 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/SMT/IR/SMTOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/APSInt.h"
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
index d8dfe164458e2..8ba2af581e00d 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 #include "SPIRVParsingUtils.h"
 
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 7339ba5c2379c..19dba20f43fa9 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -154,14 +155,14 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
     return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (llvm::isa<ShapeType>(type) || isExtentTensorType(type))
-    return ConstShapeOp::create(builder,
-        loc, type, llvm::cast<DenseIntElementsAttr>(value));
+    return ConstShapeOp::create(builder, loc, type,
+                                llvm::cast<DenseIntElementsAttr>(value));
   if (llvm::isa<SizeType>(type))
     return ConstSizeOp::create(builder, loc, type,
-                                       llvm::cast<IntegerAttr>(value));
+                               llvm::cast<IntegerAttr>(value));
   if (llvm::isa<WitnessType>(type))
     return ConstWitnessOp::create(builder, loc, type,
-                                          llvm::cast<BoolAttr>(value));
+                                  llvm::cast<BoolAttr>(value));
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
@@ -316,8 +317,8 @@ struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
     auto newYieldOp =
         rewriter.replaceOpWithNewOp<AssumingYieldOp>(yieldOp, newYieldOperands);
     rewriter.setInsertionPoint(op);
-    auto newOp = AssumingOp::create(rewriter,
-        op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
+    auto newOp = AssumingOp::create(
+        rewriter, op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
     newOp.getDoRegion().takeBody(op.getDoRegion());
 
     // Use the new results to replace the previously used ones.
@@ -780,9 +781,9 @@ struct BroadcastFoldConstantOperandsPattern
     auto foldedConstantOperandsTy = RankedTensorType::get(
         {static_cast<int64_t>(foldedConstantShape.size())},
         rewriter.getIndexType());
-    newShapeOperands.push_back(ConstShapeOp::create(rewriter,
-        op.getLoc(), foldedConstantOperandsTy,
-        rewriter.getIndexTensorAttr(foldedConstantShape)));
+    newShapeOperands.push_back(
+        ConstShapeOp::create(rewriter, op.getLoc(), foldedConstantOperandsTy,
+                             rewriter.getIndexTensorAttr(foldedConstantShape)));
     rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(),
                                              newShapeOperands);
     return success();
@@ -845,9 +846,9 @@ struct BroadcastConcretizeResultTypePattern
       }
     }
 
-    auto newOp = BroadcastOp::create(rewriter,
-        op.getLoc(), getExtentTensorType(getContext(), maxRank),
-        op.getShapes());
+    auto newOp = BroadcastOp::create(rewriter, op.getLoc(),
+                                     getExtentTensorType(getContext(), maxRank),
+                                     op.getShapes());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -1338,7 +1339,8 @@ std::optional<int64_t> GetExtentOp::getConstantDim() {
 }
 
 OpFoldResult GetExtentOp::fold(FoldAdaptor adaptor) {
-  auto elements = llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
+  auto elements =
+      llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
   if (!elements)
     return nullptr;
   std::optional<int64_t> dim = getConstantDim();
@@ -1357,8 +1359,8 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
     Value dim = ConstSizeOp::create(builder, loc, dimAttr);
     build(builder, result, builder.getType<SizeType>(), shape, dim);
   } else {
-    Value dim =
-        arith::ConstantOp::create(builder, loc, builder.getIndexType(), dimAttr);
+    Value dim = arith::ConstantOp::create(builder, loc, builder.getIndexType(),
+                                          dimAttr);
     build(builder, result, builder.getIndexType(), shape, dim);
   }
 }
@@ -1480,7 +1482,8 @@ bool mlir::shape::MeetOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
 //===----------------------------------------------------------------------===//
 
 OpFoldResult shape::RankOp::fold(FoldAdaptor adaptor) {
-  auto shape = llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
+  auto shape =
+      llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
   if (!shape)
     return {};
   int64_t rank = shape.getNumElements();
@@ -1708,8 +1711,8 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
                                   rewriter.getIndexTensorAttr(type.getShape()))
             .getResult();
     if (constShape.getType() != op.getResult().getType())
-      constShape = tensor::CastOp::create(rewriter,
-          loc, op.getResult().getType(), constShape);
+      constShape = tensor::CastOp::create(rewriter, loc,
+                                          op.getResult().getType(), constShape);
     rewriter.replaceOp(op, constShape);
     return success();
   }
@@ -1717,13 +1720,13 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
 
 // Canonicalize
 //
-// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
-// %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
+// tensor<*xf32> %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
 //
 // to
 //
-// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
-// %1 = %shape
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
+// tensor<*xf32> %1 = %shape
 //
 struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
   using OpRewritePattern<shape::ShapeOfOp>::OpRewritePattern;
@@ -1751,10 +1754,11 @@ struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
 
     if (opTensorTy != shapeTensorTy) {
       if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
-        shape = tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
-      else if (!isExtentTensorType(shapeTensorTy))
         shape =
-            arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
+            tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
+      else if (!isExtentTensorType(shapeTensorTy))
+        shape = arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy,
+                                           shape);
     }
 
     rewriter.replaceOp(op, shape);
@@ -1895,8 +1899,9 @@ LogicalResult SplitAtOp::fold(FoldAdaptor adaptor,
                               SmallVectorImpl<OpFoldResult> &results) {
   if (!adaptor.getOperand() || !adaptor.getIndex())
     return failure();
-  auto shapeVec = llvm::to_vector<6>(
-      llvm::cast<DenseIntElementsAttr>(adaptor.getOperand()).getValues<int64_t>());
+  auto shapeVec =
+      llvm::to_vector<6>(llvm::cast<DenseIntElementsAttr>(adaptor.getOperand())
+                             .getValues<int64_t>());
   auto shape = llvm::ArrayRef(shapeVec);
   auto splitPoint = llvm::cast<IntegerAttr>(adaptor.getIndex()).getInt();
   // Verify that the split point is in the correct range.
@@ -1920,8 +1925,9 @@ OpFoldResult ToExtentTensorOp::fold(FoldAdaptor adaptor) {
   if (!adaptor.getInput())
     return OpFoldResult();
   Builder builder(getContext());
-  auto shape = llvm::to_vector<6>(
-      llvm::cast<DenseIntElementsAttr>(adaptor.getInput()).getValues<int64_t>());
+  auto shape =
+      llvm::to_vector<6>(llvm::cast<DenseIntElementsAttr>(adaptor.getInput())
+                             .getValues<int64_t>());
   auto type = RankedTensorType::get({static_cast<int64_t>(shape.size())},
                                     builder.getIndexType());
   return DenseIntElementsAttr::get(type, shape);
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index d5e709bbed221..eb42335fa8c94 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -561,7 +562,8 @@ SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
   SmallVector<Type> retType(
       dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
       builder.getIndexType());
-  auto transOp = CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
+  auto transOp =
+      CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
   return transOp.getOutCrds();
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp b/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
index bdec43825ddc2..33249368f7ed3 100644
--- a/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
+++ b/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h"
 #include "mlir/Dialect/Linalg/TransformOps/Syntax.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 using namespace mlir::sparse_tensor;
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 79bddee1e7b84..b19a413d6d9fd 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinTypeInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -57,7 +58,7 @@ Operation *TensorDialect::materializeConstant(OpBuilder &builder,
     return op;
   if (complex::ConstantOp::isBuildableWith(value, type))
     return complex::ConstantOp::create(builder, loc, type,
-                                               llvm::cast<ArrayAttr>(value));
+                                       llvm::cast<ArrayAttr>(value));
   return nullptr;
 }
 
@@ -681,8 +682,8 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
     inputShapes.emplace_back(std::move(inputShape));
   }
 
-  Value replacement = tensor::EmptyOp::create(builder,
-      loc, outputShape, getType().getElementType());
+  Value replacement = tensor::EmptyOp::create(builder, loc, outputShape,
+                                              getType().getElementType());
 
   int64_t rank = getType().getRank();
   OpFoldResult one = builder.getIndexAttr(1);
@@ -690,8 +691,8 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
   SmallVector<OpFoldResult> offsets(rank, zero);
   for (auto [index, input] : llvm::enumerate(getInputs())) {
     offsets[concatDim] = concatOffsets[index];
-    auto insertSlice = tensor::InsertSliceOp::create(builder,
-        loc, input, replacement, offsets, inputShapes[index], strides);
+    auto insertSlice = tensor::InsertSliceOp::create(
+        builder, loc, input, replacement, offsets, inputShapes[index], strides);
     replacement = insertSlice.getResult();
   }
   if (replacement.getType() != getType()) {
@@ -827,7 +828,7 @@ struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
         // Use refined operand type and create cast from original operand.
         auto castOp =
             CastOp::create(rewriter, concatOp->getLoc(), inferredOperandType,
-                                    concatOp.getOperand(operandIdx));
+                           concatOp.getOperand(operandIdx));
         rewriter.modifyOpInPlace(concatOp, [=, operandIdx = operandIdx] {
           concatOp->setOperand(operandIdx, castOp->getResult(0));
         });
@@ -867,8 +868,9 @@ struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
       return failure();
     }
 
-    auto newConcatOp = ConcatOp::create(rewriter,
-        concatOp->getLoc(), inferredResultType, dim, concatOp->getOperands());
+    auto newConcatOp =
+        ConcatOp::create(rewriter, concatOp->getLoc(), inferredResultType, dim,
+                         concatOp->getOperands());
     rewriter.replaceOpWithNewOp<CastOp>(concatOp, concatOp.getResultType(),
                                         newConcatOp);
 
@@ -1154,7 +1156,7 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
       return failure();
 
     auto newOp = EmptyOp::create(rewriter, op.getLoc(), foldedTensorType,
-                                          foldedDynamicSizes);
+                                 foldedDynamicSizes);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -1329,8 +1331,8 @@ struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
 
       SmallVector<int64_t> basis =
           llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-      auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-          extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
+      auto delinearize = affine::AffineDelinearizeIndexOp::create(
+          rewriter, extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
       llvm::append_range(sourceIndices, delinearize.getResults());
     }
     if (collapseOp.getReassociationIndices().empty()) {
@@ -1501,8 +1503,8 @@ struct ExtractElementFromIndexCast
 
     Type elementTy = getElementTypeOrSelf(indexCast.getIn());
 
-    auto newExtract = tensor::ExtractOp::create(rewriter,
-        loc, elementTy, indexCast.getIn(), extract.getIndices());
+    auto newExtract = tensor::ExtractOp::create(
+        rewriter, loc, elementTy, indexCast.getIn(), extract.getIndices());
 
     rewriter.replaceOpWithNewOp<arith::IndexCastOp>(extract, extract.getType(),
                                                     newExtract);
@@ -2164,9 +2166,9 @@ struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
         collapseShapeOp.getSrcMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = CollapseShapeOp::create(rewriter,
-          collapseShapeOp.getLoc(), newResultType, castOp.getSource(),
-          collapseShapeOp.getReassociation());
+      auto newOp = CollapseShapeOp::create(rewriter, collapseShapeOp.getLoc(),
+                                           newResultType, castOp.getSource(),
+                                           collapseShapeOp.getReassociation());
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
           collapseShapeOp, collapseShapeOp.getResultType(), newOp);
     }
@@ -2244,9 +2246,9 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
     auto outputType = RankedTensorType::get(
         newOutputShape, expandOp.getSrcType().getElementType());
     auto inputCast = CastOp::create(rewriter, expandOp.getLoc(), inputType,
-                                             expandOp.getSrc());
-    auto newExpand = ExpandShapeOp::create(rewriter,
-        expandOp.getLoc(), outputType, inputCast.getResult(),
+                                    expandOp.getSrc());
+    auto newExpand = ExpandShapeOp::create(
+        rewriter, expandOp.getLoc(), outputType, inputCast.getResult(),
         expandOp.getReassociationIndices(), outputOfr);
     rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
                                         newExpand.getResult());
@@ -2558,10 +2560,11 @@ class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
 
     // Create folded extract.
     Location loc = sliceOp.getLoc();
-    Value newResult = ExtractSliceOp::create(rewriter,
-        loc, sliceOp.getType(), castOp.getSource(), sliceOp.getOffsets(),
-        sliceOp.getSizes(), sliceOp.getStrides(), sliceOp.getStaticOffsets(),
-        sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
+    Value newResult = ExtractSliceOp::create(
+        rewriter, loc, sliceOp.getType(), castOp.getSource(),
+        sliceOp.getOffsets(), sliceOp.getSizes(), sliceOp.getStrides(),
+        sliceOp.getStaticOffsets(), sliceOp.getStaticSizes(),
+        sliceOp.getStaticStrides());
     rewriter.replaceOp(sliceOp, newResult);
     return success();
   }
@@ -2713,7 +2716,7 @@ struct SliceCanonicalizer {
     Value replacement = newOp.getResult();
     if (replacement.getType() != op.getType())
       replacement = tensor::CastOp::create(rewriter, op.getLoc(), op.getType(),
-                                                    replacement);
+                                           replacement);
     rewriter.replaceOp(op, replacement);
   }
 };
@@ -2982,7 +2985,7 @@ class InsertSliceOpConstantArgumentFolder final
       if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
       toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
-                                                 sourceType, toInsert);
+                                        sourceType, toInsert);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, toInsert, insertSliceOp.getDest(), mixedOffsets,
@@ -3078,17 +3081,18 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     if (!sliceResult.isValid)
       return failure();
 
-    Operation *replacement = InsertOpTy::create(rewriter,
-        insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
-        mixedSizes, insertSliceOp.getMixedStrides());
+    Operation *replacement =
+        InsertOpTy::create(rewriter, insertSliceOp.getLoc(), src, dst,
+                           insertSliceOp.getMixedOffsets(), mixedSizes,
+                           insertSliceOp.getMixedStrides());
 
     // In the parallel case there is no result and so nothing to cast.
     bool isParallelInsert =
         std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
     if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
       replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
-                                                    insertSliceOp.getDestType(),
-                                                    replacement->getResult(0));
+                                           insertSliceOp.getDestType(),
+                                           replacement->getResult(0));
     }
     rewriter.replaceOp(insertSliceOp, replacement->getResults());
     return success();
@@ -3157,8 +3161,8 @@ struct InsertSliceOpSourceCastInserter final
     // parallel case.
     if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
       rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-    Value cast = tensor::CastOp::create(rewriter,
-        insertSliceOp.getLoc(), newSrcType, insertSliceOp.getSource());
+    Value cast = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
+                                        newSrcType, insertSliceOp.getSource());
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, cast, insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
@@ -3410,10 +3414,11 @@ struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
         padTensorOp.getSourceMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = PadOp::create(rewriter,
-          padTensorOp->getLoc(), newResultType, padTensorOp.getSource(),
-          padTensorOp.getStaticLow(), padTensorOp.getStaticHigh(),
-          padTensorOp.getLow(), padTensorOp.getHigh(), padTensorOp.getNofold(),
+      auto newOp = PadOp::create(
+          rewriter, padTensorOp->getLoc(), newResultType,
+          padTensorOp.getSource(), padTensorOp.getStaticLow(),
+          padTensorOp.getStaticHigh(), padTensorOp.getLow(),
+          padTensorOp.getHigh(), padTensorOp.getNofold(),
           getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
       IRMapping mapper;
       padTensorOp.getRegion().cloneInto(&newOp.getRegion(), mapper);
@@ -3442,8 +3447,8 @@ struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
                                             tensorCastOp.getDest().getType()))
       return failure();
 
-    auto replacementOp = PadOp::create(rewriter,
-        padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
+    auto replacementOp = PadOp::create(
+        rewriter, padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
         padTensorOp.getSource(), padTensorOp.getStaticLow(),
         padTensorOp.getStaticHigh(), padTensorOp.getLow(),
         padTensorOp.getHigh(), padTensorOp.getNofold(),
@@ -3600,11 +3605,11 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
 
     // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
     // the two paddings in one step.
-    auto newSliceOp = ExtractSliceOp::create(rewriter,
-        padOp.getLoc(), outerSliceOp.getSource(), newOffsets, newSizes,
-        innerSliceOp.getMixedStrides());
-    auto newPadOp = PadOp::create(rewriter,
-        padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
+    auto newSliceOp = ExtractSliceOp::create(
+        rewriter, padOp.getLoc(), outerSliceOp.getSource(), newOffsets,
+        newSizes, innerSliceOp.getMixedStrides());
+    auto newPadOp = PadOp::create(
+        rewriter, padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
         padOp.getMixedLowPad(), newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
     rewriter.inlineRegionBefore(padOp.getRegion(), newPadOp.getRegion(),
@@ -3700,9 +3705,9 @@ struct FoldStaticPadding : public OpRewritePattern<PadOp> {
     // Rewrite the op using the new static type.
     auto newResultType = RankedTensorType::get(
         newOutDims, padTensorOp.getType().getElementType());
-    auto newOp = PadOp::create(rewriter,
-        padTensorOp->getLoc(), newResultType, input, staticLow, staticHigh,
-        newLows, newHighs, padTensorOp.getNofold(),
+    auto newOp = PadOp::create(
+        rewriter, padTensorOp->getLoc(), newResultType, input, staticLow,
+        staticHigh, newLows, newHighs, padTensorOp.getNofold(),
         getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
 
     IRMapping mapper;
@@ -3780,9 +3785,9 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
     SmallVector<OpFoldResult> newLowPad =
         addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        padOp.getLoc(), padOp.getResultType(), producerPad.getSource(),
-        newLowPad, newHighPad, padOp.getNofold(),
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, padOp.getLoc(), padOp.getResultType(),
+        producerPad.getSource(), newLowPad, newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, tensor::PadOp::getAttributeNames()));
     rewriter.inlineRegionBefore(padOp.getRegion(), newPadOp.getRegion(),
                                 newPadOp.getRegion().begin());
@@ -4111,8 +4116,8 @@ struct FoldTensorCastProducerOp
     for (auto [oldResult, newResult] :
          llvm::zip(op->getResults(), newOp->getResults())) {
       if (newResult.getType() != oldResult.getType()) {
-        replacements.push_back(tensor::CastOp::create(rewriter,
-            op->getLoc(), oldResult.getType(), newResult));
+        replacements.push_back(tensor::CastOp::create(
+            rewriter, op->getLoc(), oldResult.getType(), newResult));
       } else {
         replacements.push_back(newResult);
       }
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index ba258402b61b3..b8b1a592fdca2 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
index d69535169f956..003d81fbc7771 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
index 12257da878a40..2d6d4e17aff66 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
@@ -10,6 +10,7 @@
 
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/Support/InterleavedRange.h"
 
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 0db0317461c03..d4a9d4ef1d656 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp b/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
index 93b0bc591ca02..15c2ae22f56ea 100644
--- a/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/ADT/STLExtras.h"
 
diff --git a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
index 34d6221d15fb0..fc6ea93b49218 100644
--- a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h"
 
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
diff --git a/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp b/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
index 85f61245eb734..9d5c749f02dfd 100644
--- a/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h"
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Rewrite/FrozenRewritePatternSet.h"
 #include "mlir/Rewrite/PatternApplicator.h"
diff --git a/mlir/lib/Dialect/UB/IR/UBOps.cpp b/mlir/lib/Dialect/UB/IR/UBOps.cpp
index ee523f9522953..442a53ca8162a 100644
--- a/mlir/lib/Dialect/UB/IR/UBOps.cpp
+++ b/mlir/lib/Dialect/UB/IR/UBOps.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "mlir/Dialect/UB/IR/UBOpsDialect.cpp.inc"
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index e32b2ca8d76b4..a8277a2a805e9 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -30,6 +30,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -599,8 +600,8 @@ struct ElideUnitDimsInMultiDimReduction
                             dstVecType.getScalableDims());
         mask = vector::ShapeCastOp::create(rewriter, loc, newMaskType, mask);
       }
-      cast = vector::ShapeCastOp::create(rewriter,
-          loc, reductionOp.getDestType(), reductionOp.getSource());
+      cast = vector::ShapeCastOp::create(
+          rewriter, loc, reductionOp.getDestType(), reductionOp.getSource());
     } else {
       // This means we are reducing all the dimensions, and all reduction
       // dimensions are of size 1. So a simple extraction would do.
@@ -674,35 +675,35 @@ Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
   case arith::AtomicRMWKind::addf:
   case arith::AtomicRMWKind::addi:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::ADD, vector);
+                                       CombiningKind::ADD, vector);
   case arith::AtomicRMWKind::mulf:
   case arith::AtomicRMWKind::muli:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MUL, vector);
+                                       CombiningKind::MUL, vector);
   case arith::AtomicRMWKind::minimumf:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINIMUMF, vector);
+                                       CombiningKind::MINIMUMF, vector);
   case arith::AtomicRMWKind::mins:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINSI, vector);
+                                       CombiningKind::MINSI, vector);
   case arith::AtomicRMWKind::minu:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINUI, vector);
+                                       CombiningKind::MINUI, vector);
   case arith::AtomicRMWKind::maximumf:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXIMUMF, vector);
+                                       CombiningKind::MAXIMUMF, vector);
   case arith::AtomicRMWKind::maxs:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXSI, vector);
+                                       CombiningKind::MAXSI, vector);
   case arith::AtomicRMWKind::maxu:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXUI, vector);
+                                       CombiningKind::MAXUI, vector);
   case arith::AtomicRMWKind::andi:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::AND, vector);
+                                       CombiningKind::AND, vector);
   case arith::AtomicRMWKind::ori:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::OR, vector);
+                                       CombiningKind::OR, vector);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
@@ -4171,8 +4172,8 @@ class StridedSliceBroadcast final
     // just a single scalar.
     bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
     if (!lowerDimMatch && !isScalarSrc) {
-      source = ExtractStridedSliceOp::create(rewriter,
-          op->getLoc(), source,
+      source = ExtractStridedSliceOp::create(
+          rewriter, op->getLoc(), source,
           getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getStrides(), /* dropFront=*/rankDiff));
@@ -4267,7 +4268,7 @@ class ContiguousExtractStridedSliceToExtract final
     SmallVector<int64_t> offsets = getI64SubArray(op.getOffsets());
     auto extractOffsets = ArrayRef(offsets).take_front(numOffsets);
     Value extract = vector::ExtractOp::create(rewriter, op->getLoc(), source,
-                                                       extractOffsets);
+                                              extractOffsets);
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(), extract);
     return success();
   }
@@ -5337,13 +5338,14 @@ struct SwapExtractSliceOfTransferWrite
     // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
     // Set all in_bounds to false and let the folder infer them.
     SmallVector<bool> newInBounds(vectorShape.size(), false);
-    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
-        extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
-        insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
-        insertOp.getMixedStrides());
-    auto newTransferWriteOp = TransferWriteOp::create(rewriter,
-        transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
-        transferOp.getIndices(), transferOp.getPermutationMapAttr(),
+    auto newExtractOp = tensor::ExtractSliceOp::create(
+        rewriter, extractOp.getLoc(), insertOp.getSourceType(),
+        insertOp.getDest(), insertOp.getMixedOffsets(),
+        insertOp.getMixedSizes(), insertOp.getMixedStrides());
+    auto newTransferWriteOp = TransferWriteOp::create(
+        rewriter, transferOp.getLoc(), transferOp.getVector(),
+        newExtractOp.getResult(), transferOp.getIndices(),
+        transferOp.getPermutationMapAttr(),
         rewriter.getBoolArrayAttr(newInBounds));
     rewriter.modifyOpInPlace(insertOp, [&]() {
       insertOp.getSourceMutable().assign(newTransferWriteOp.getResult());
@@ -7215,11 +7217,11 @@ Operation *mlir::vector::maskOperation(OpBuilder &builder,
     return maskableOp;
   if (passthru)
     return MaskOp::create(builder, maskableOp->getLoc(),
-                                  maskableOp->getResultTypes(), mask, passthru,
-                                  maskableOp, createMaskOpRegion);
+                          maskableOp->getResultTypes(), mask, passthru,
+                          maskableOp, createMaskOpRegion);
   return MaskOp::create(builder, maskableOp->getLoc(),
-                                maskableOp->getResultTypes(), mask, maskableOp,
-                                createMaskOpRegion);
+                        maskableOp->getResultTypes(), mask, maskableOp,
+                        createMaskOpRegion);
 }
 
 /// Creates a vector select operation that picks values from `newValue` or
@@ -7235,7 +7237,7 @@ Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
     return newValue;
 
   return arith::SelectOp::create(builder, newValue.getLoc(), newValue.getType(),
-                                         mask, newValue, passthru);
+                                 mask, newValue, passthru);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 125c3d918284c..bb051650897b4 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
 #include "mlir/Dialect/X86Vector/Transforms.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
index eafe3a97758c1..f7b1c0966fe27 100644
--- a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
+++ b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
@@ -64,7 +65,7 @@ SmallVector<Value> x86vector::MaskCompressOp::getIntrinsicOperands(
     src = adaptor.getSrc();
   } else if (adaptor.getConstantSrc()) {
     src = LLVM::ConstantOp::create(rewriter, loc, opType,
-                                            adaptor.getConstantSrcAttr());
+                                   adaptor.getConstantSrcAttr());
   } else {
     auto zeroAttr = rewriter.getZeroAttr(opType);
     src = LLVM::ConstantOp::create(rewriter, loc, opType, zeroAttr);
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 453d53ee41d54..81687a6fc37f2 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
 
 #include "llvm/Support/Debug.h"
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 6d7e2aa0ece7d..23988a00c7a2c 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectResourceBlobManager.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeRange.h"
 
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 535f5e9b4a15d..73366110b104e 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -1192,7 +1192,7 @@ def OpFuncRef : TEST_Op<"op_funcref"> {
   let description = [{
     The "test.op_funcref" is a test op with a reference to a function symbol.
   }];
-  let builders = [OpBuilder<(ins "::mlir::func::FuncOp":$function)>];
+  // let builders = [OpBuilder<(ins "::mlir::func::FuncOp":$function)>];
 }
 
 // Pattern add the argument plus a increasing static number hidden in
diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
index cdf44c2959d50..9753ac00af88a 100644
--- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
+++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
index ee3eb9522db7e..44f117e93da6b 100644
--- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
+++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "llvm/Support/Debug.h"
diff --git a/mlir/test/lib/Transforms/TestTransformsOps.cpp b/mlir/test/lib/Transforms/TestTransformsOps.cpp
index 9a5632bb99c06..3d7b6be8a78e9 100644
--- a/mlir/test/lib/Transforms/TestTransformsOps.cpp
+++ b/mlir/test/lib/Transforms/TestTransformsOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Transforms/RegionUtils.h"
 
diff --git a/mlir/test/python/lib/PythonTestDialect.cpp b/mlir/test/python/lib/PythonTestDialect.cpp
index a0ff31504c691..e0a76f1b75d38 100644
--- a/mlir/test/python/lib/PythonTestDialect.cpp
+++ b/mlir/test/python/lib/PythonTestDialect.cpp
@@ -9,6 +9,7 @@
 #include "PythonTestDialect.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "PythonTestDialect.cpp.inc"


From flang-commits at lists.llvm.org  Mon Jul  7 17:48:50 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 07 Jul 2025 17:48:50 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][core] NFC update builder
	create API usage (PR #147311)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147311@github.com>
Message-ID: <686c6af2.050a0220.2b21ac.bcb1@mx.google.com>


https://github.com/makslevental updated https://github.com/llvm/llvm-project/pull/147311

>From 1e57cb84299e022230dc8443254d92083ea4816b Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental at gmail.com>
Date: Sat, 5 Jul 2025 20:26:55 -0400
Subject: [PATCH 1/5] [mlir][tblgen] add concrete create methods

---
 mlir/include/mlir/TableGen/Class.h          |  2 +
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp | 61 +++++++++++++++++++--
 2 files changed, 57 insertions(+), 6 deletions(-)

diff --git a/mlir/include/mlir/TableGen/Class.h b/mlir/include/mlir/TableGen/Class.h
index f750a34a3b2ba..69cefbbc43e0a 100644
--- a/mlir/include/mlir/TableGen/Class.h
+++ b/mlir/include/mlir/TableGen/Class.h
@@ -71,6 +71,8 @@ class MethodParameter {
   StringRef getName() const { return name; }
   /// Returns true if the parameter has a default value.
   bool hasDefaultValue() const { return !defaultValue.empty(); }
+  StringRef getDefaultValue() const { return defaultValue; }
+  bool isOptional() const { return optional; }
 
 private:
   /// The C++ type.
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index cbb4030f3adb4..20c483e52da04 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -230,6 +230,14 @@ static const char *const opCommentHeader = R"(
 
 )";
 
+static const char *const inlineCreateBody = R"(
+  ::mlir::OperationState __state__({0}, getOperationName());
+  build(builder, __state__{1});
+  auto __res__ = ::llvm::dyn_cast<{2}>(builder.create(__state__));
+  assert(__res__ && "builder didn't return the right type");
+  return __res__;
+)";
+
 //===----------------------------------------------------------------------===//
 // Utility structs and functions
 //===----------------------------------------------------------------------===//
@@ -665,6 +673,7 @@ class OpEmitter {
   // Generates the build() method that takes each operand/attribute
   // as a stand-alone parameter.
   void genSeparateArgParamBuilder();
+  void genInlineCreateBody(const SmallVector<MethodParameter> &paramList);
 
   // Generates the build() method that takes each operand/attribute as a
   // stand-alone parameter. The generated build() method uses first operand's
@@ -2568,6 +2577,39 @@ static bool canInferType(const Operator &op) {
   return op.getTrait("::mlir::InferTypeOpInterface::Trait");
 }
 
+void OpEmitter::genInlineCreateBody(
+    const SmallVector<MethodParameter> &paramList) {
+  SmallVector<MethodParameter> createParamList;
+  SmallVector<llvm::StringRef, 4> nonBuilderStateArgsList;
+  createParamList.emplace_back("::mlir::OpBuilder &", "builder");
+  std::string locParamName = "location";
+  while (llvm::find_if(paramList, [&locParamName](const MethodParameter &p) {
+           return p.getName() == locParamName;
+         }) != paramList.end()) {
+    locParamName += "_";
+  }
+  createParamList.emplace_back("::mlir::Location", locParamName);
+
+  for (auto &param : paramList) {
+    if (param.getType() == "::mlir::OpBuilder &" ||
+        param.getType() == "::mlir::OperationState &")
+      continue;
+    createParamList.emplace_back(param.getType(), param.getName(),
+                                 param.getDefaultValue(), param.isOptional());
+    nonBuilderStateArgsList.push_back(param.getName());
+  }
+  auto *c = opClass.addStaticMethod(opClass.getClassName(), "create",
+                                    createParamList);
+  std::string nonBuilderStateArgs = "";
+  if (!nonBuilderStateArgsList.empty()) {
+    llvm::raw_string_ostream nonBuilderStateArgsOS(nonBuilderStateArgs);
+    interleaveComma(nonBuilderStateArgsList, nonBuilderStateArgsOS);
+    nonBuilderStateArgs = ", " + nonBuilderStateArgs;
+  }
+  c->body() << llvm::formatv(inlineCreateBody, locParamName,
+                             nonBuilderStateArgs, opClass.getClassName());
+}
+
 void OpEmitter::genSeparateArgParamBuilder() {
   SmallVector<AttrParamKind, 2> attrBuilderType;
   attrBuilderType.push_back(AttrParamKind::WrappedAttr);
@@ -2584,10 +2626,12 @@ void OpEmitter::genSeparateArgParamBuilder() {
     buildParamList(paramList, inferredAttributes, resultNames, paramKind,
                    attrType);
 
-    auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+    auto *m = opClass.addStaticMethod("void", "build", paramList);
     // If the builder is redundant, skip generating the method.
     if (!m)
       return;
+    genInlineCreateBody(paramList);
+
     auto &body = m->body();
     genCodeForAddingArgAndRegionForBuilder(body, inferredAttributes,
                                            /*isRawValueAttr=*/attrType ==
@@ -2712,10 +2756,11 @@ void OpEmitter::genUseOperandAsResultTypeCollectiveParamBuilder(
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
-  auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+  auto *m = opClass.addStaticMethod("void", "build", paramList);
   // If the builder is redundant, skip generating the method
   if (!m)
     return;
+  genInlineCreateBody(paramList);
   auto &body = m->body();
 
   // Operands
@@ -2826,10 +2871,11 @@ void OpEmitter::genInferredTypeCollectiveParamBuilder(
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
-  auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+  auto *m = opClass.addStaticMethod("void", "build", paramList);
   // If the builder is redundant, skip generating the method
   if (!m)
     return;
+  genInlineCreateBody(paramList);
   auto &body = m->body();
 
   int numResults = op.getNumResults();
@@ -2906,10 +2952,11 @@ void OpEmitter::genUseOperandAsResultTypeSeparateParamBuilder() {
     buildParamList(paramList, inferredAttributes, resultNames,
                    TypeParamKind::None, attrType);
 
-    auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+    auto *m = opClass.addStaticMethod("void", "build", paramList);
     // If the builder is redundant, skip generating the method
     if (!m)
       return;
+    genInlineCreateBody(paramList);
     auto &body = m->body();
     genCodeForAddingArgAndRegionForBuilder(body, inferredAttributes,
                                            /*isRawValueAttr=*/attrType ==
@@ -2948,10 +2995,11 @@ void OpEmitter::genUseAttrAsResultTypeCollectiveParamBuilder(
                                  : "attributes";
   paramList.emplace_back("::llvm::ArrayRef<::mlir::NamedAttribute>",
                          attributesName, "{}");
-  auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+  auto *m = opClass.addStaticMethod("void", "build", paramList);
   // If the builder is redundant, skip generating the method
   if (!m)
     return;
+  genInlineCreateBody(paramList);
 
   auto &body = m->body();
 
@@ -3114,10 +3162,11 @@ void OpEmitter::genCollectiveParamBuilder(CollectiveBuilderKind kind) {
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
-  auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
+  auto *m = opClass.addStaticMethod("void", "build", paramList);
   // If the builder is redundant, skip generating the method
   if (!m)
     return;
+  genInlineCreateBody(paramList);
   auto &body = m->body();
 
   // Operands

>From 25c06f35a4779f6f323debe4f77f1a90e804d998 Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 10:35:48 -0400
Subject: [PATCH 2/5] [mlir][core] update builder create API

---
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           | 176 ++---
 .../AffineToStandard/AffineToStandard.cpp     |  26 +-
 .../ArithToAMDGPU/ArithToAMDGPU.cpp           |  90 +--
 .../ArithToArmSME/ArithToArmSME.cpp           |   6 +-
 .../Conversion/ArithToEmitC/ArithToEmitC.cpp  |  44 +-
 .../Conversion/ArithToLLVM/ArithToLLVM.cpp    |  28 +-
 .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp  |  74 +-
 .../ArmNeon2dToIntr/ArmNeon2dToIntr.cpp       |   6 +-
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  | 122 +--
 .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp    |  68 +-
 .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp    |  66 +-
 .../BufferizationToMemRef.cpp                 |  26 +-
 .../ComplexCommon/DivisionConverter.cpp       | 392 ++++-----
 .../ComplexToLLVM/ComplexToLLVM.cpp           |  28 +-
 .../ComplexToLibm/ComplexToLibm.cpp           |   2 +-
 .../ComplexToStandard/ComplexToStandard.cpp   | 598 +++++++-------
 .../ControlFlowToLLVM/ControlFlowToLLVM.cpp   |   8 +-
 .../ControlFlowToSCF/ControlFlowToSCF.cpp     |  22 +-
 .../Conversion/FuncToEmitC/FuncToEmitC.cpp    |   2 +-
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp |  48 +-
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |  82 +-
 .../GPUCommon/GPUToLLVMConversion.cpp         |  78 +-
 .../GPUCommon/IndexIntrinsicsOpLowering.h     |  10 +-
 .../GPUCommon/OpToFuncCallLowering.h          |  12 +-
 .../Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp  |  20 +-
 .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp        |  40 +-
 .../Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp    |  40 +-
 .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp      |  50 +-
 mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp |  54 +-
 .../Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp  |   4 +-
 .../Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp  |   4 +-
 .../Conversion/IndexToLLVM/IndexToLLVM.cpp    |  68 +-
 .../Conversion/IndexToSPIRV/IndexToSPIRV.cpp  |  68 +-
 .../Conversion/LLVMCommon/MemRefBuilder.cpp   |  78 +-
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    |  62 +-
 .../Conversion/LLVMCommon/PrintCallHelper.cpp |   8 +-
 .../Conversion/LLVMCommon/StructBuilder.cpp   |   4 +-
 .../Conversion/LLVMCommon/TypeConverter.cpp   |  14 +-
 .../Conversion/LLVMCommon/VectorPattern.cpp   |   6 +-
 .../LinalgToStandard/LinalgToStandard.cpp     |   4 +-
 mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp   |  64 +-
 .../Conversion/MathToFuncs/MathToFuncs.cpp    | 180 ++---
 mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp |  38 +-
 mlir/lib/Conversion/MathToLibm/MathToLibm.cpp |  14 +-
 .../Conversion/MathToSPIRV/MathToSPIRV.cpp    |  82 +-
 .../MemRefToEmitC/MemRefToEmitC.cpp           |   6 +-
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  | 138 ++--
 .../MemRefToSPIRV/MemRefToSPIRV.cpp           |  18 +-
 mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp   | 204 ++---
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    | 158 ++--
 .../Conversion/OpenACCToSCF/OpenACCToSCF.cpp  |   2 +-
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |   2 +-
 .../PDLToPDLInterp/PDLToPDLInterp.cpp         | 112 +--
 .../SCFToControlFlow/SCFToControlFlow.cpp     |  36 +-
 mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp |  14 +-
 mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp     |  40 +-
 .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp    |  40 +-
 mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp |  40 +-
 .../ConvertLaunchFuncToLLVMCalls.cpp          |   8 +-
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    | 124 +--
 .../ConvertShapeConstraints.cpp               |   2 +-
 .../ShapeToStandard/ShapeToStandard.cpp       | 150 ++--
 .../TensorToSPIRV/TensorToSPIRV.cpp           |   6 +-
 .../Conversion/TosaToArith/TosaToArith.cpp    | 100 +--
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  | 742 +++++++++---------
 .../TosaToLinalg/TosaToLinalgNamed.cpp        | 218 ++---
 .../TosaToMLProgram/TosaToMLProgram.cpp       |   6 +-
 mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp   |  24 +-
 .../Conversion/TosaToTensor/TosaToTensor.cpp  |  20 +-
 .../VectorToArmSME/VectorToArmSME.cpp         |  56 +-
 .../Conversion/VectorToGPU/VectorToGPU.cpp    |  76 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      | 158 ++--
 .../Conversion/VectorToSCF/VectorToSCF.cpp    | 164 ++--
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  48 +-
 .../VectorToXeGPU/VectorToXeGPU.cpp           |  20 +-
 .../AMDGPU/Transforms/EmulateAtomics.cpp      |  20 +-
 .../AMDGPU/Transforms/MaskedloadToLoad.cpp    |  28 +-
 .../Transforms/ResolveStridedMetadata.cpp     |   6 +-
 mlir/lib/Dialect/AMX/IR/AMXDialect.cpp        |   8 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  34 +-
 .../Transforms/AffineDataCopyGeneration.cpp   |   2 +-
 .../Transforms/AffineExpandIndexOps.cpp       |  20 +-
 .../Affine/Transforms/DecomposeAffineOps.cpp  |   4 +-
 .../Dialect/Affine/Transforms/LoopFusion.cpp  |   2 +-
 .../Transforms/PipelineDataTransfer.cpp       |   6 +-
 .../Affine/Transforms/ReifyValueBounds.cpp    |   6 +-
 .../Affine/Transforms/SuperVectorize.cpp      |  24 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |  86 +-
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |  88 +--
 mlir/lib/Dialect/Arith/IR/ArithDialect.cpp    |   2 +-
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |  40 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |   2 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/EmulateUnsupportedFloats.cpp   |   4 +-
 .../Arith/Transforms/EmulateWideInt.cpp       | 176 ++---
 .../Dialect/Arith/Transforms/ExpandOps.cpp    | 228 +++---
 .../Transforms/IntRangeOptimizations.cpp      |  10 +-
 .../Arith/Transforms/ReifyValueBounds.cpp     |  16 +-
 .../Transforms/ShardingInterfaceImpl.cpp      |   2 +-
 mlir/lib/Dialect/Arith/Utils/Utils.cpp        |  90 +--
 .../LowerContractionToNeonI8MMPattern.cpp     |   8 +-
 mlir/lib/Dialect/ArmSME/IR/Utils.cpp          |  14 +-
 .../ArmSME/Transforms/OuterProductFusion.cpp  |   8 +-
 .../ArmSME/Transforms/TileAllocation.cpp      |   4 +-
 .../ArmSME/Transforms/VectorLegalization.cpp  | 110 +--
 .../Transforms/LegalizeForLLVMExport.cpp      |  16 +-
 .../Transforms/LegalizeVectorStorage.cpp      |  12 +-
 .../LowerContractionToSVEI8MMPattern.cpp      |  60 +-
 mlir/lib/Dialect/Async/IR/Async.cpp           |   2 +-
 .../Async/Transforms/AsyncParallelFor.cpp     | 194 ++---
 .../Transforms/AsyncRuntimeRefCounting.cpp    |  14 +-
 .../Async/Transforms/AsyncToAsyncRuntime.cpp  |  78 +-
 .../IR/BufferDeallocationOpInterface.cpp      |  10 +-
 .../IR/BufferizableOpInterface.cpp            |  10 +-
 .../Bufferization/IR/BufferizationOps.cpp     |  30 +-
 .../BufferizationTransformOps.cpp             |   4 +-
 .../BufferDeallocationSimplification.cpp      |  12 +-
 .../Transforms/BufferResultsToOutParams.cpp   |   6 +-
 .../Bufferization/Transforms/BufferUtils.cpp  |   2 +-
 .../Bufferization/Transforms/Bufferize.cpp    |   6 +-
 .../DropEquivalentBufferResults.cpp           |   4 +-
 .../Transforms/EmptyTensorElimination.cpp     |   2 +-
 .../FuncBufferizableOpInterfaceImpl.cpp       |   4 +-
 .../Transforms/LowerDeallocations.cpp         | 142 ++--
 .../OwnershipBasedBufferDeallocation.cpp      |  14 +-
 .../lib/Dialect/Complex/IR/ComplexDialect.cpp |   2 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.cpp |   6 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |  12 +-
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp           |   4 +-
 .../Dialect/EmitC/Transforms/Transforms.cpp   |   4 +-
 .../EmitC/Transforms/TypeConversions.cpp      |   2 +-
 .../EmitC/Transforms/WrapFuncInClass.cpp      |   8 +-
 .../Func/Extensions/InlinerExtension.cpp      |   2 +-
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |   2 +-
 .../Func/TransformOps/FuncTransformOps.cpp    |   2 +-
 .../Func/Transforms/FuncConversions.cpp       |   2 +-
 mlir/lib/Dialect/Func/Utils/Utils.cpp         |   6 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |  10 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |  10 +-
 mlir/lib/Dialect/GPU/TransformOps/Utils.cpp   |  24 +-
 .../GPU/Transforms/AllReduceLowering.cpp      |   2 +-
 .../GPU/Transforms/AsyncRegionRewriter.cpp    |   8 +-
 .../GPU/Transforms/DecomposeMemRefs.cpp       |   4 +-
 .../GPU/Transforms/GlobalIdRewriter.cpp       |   8 +-
 .../GPU/Transforms/KernelOutlining.cpp        |  10 +-
 .../GPU/Transforms/MemoryPromotion.cpp        |  16 +-
 .../Dialect/GPU/Transforms/ModuleToBinary.cpp |   2 +-
 .../GPU/Transforms/PromoteShuffleToAMDGPU.cpp |   4 +-
 .../GPU/Transforms/ShuffleRewriter.cpp        |  22 +-
 .../GPU/Transforms/SubgroupIdRewriter.cpp     |  22 +-
 .../GPU/Transforms/SubgroupReduceLowering.cpp |  70 +-
 .../Dialect/GPU/Utils/DistributionUtils.cpp   |   4 +-
 mlir/lib/Dialect/Index/IR/IndexOps.cpp        |   8 +-
 .../LLVMIR/IR/BasicPtxBuilderInterface.cpp    |   4 +-
 .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp   |   2 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |  16 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp |  46 +-
 .../Dialect/LLVMIR/Transforms/AddComdats.cpp  |   4 +-
 .../Transforms/InlinerInterfaceImpl.cpp       |  22 +-
 .../LLVMIR/Transforms/LegalizeForExport.cpp   |   2 +-
 .../LLVMIR/Transforms/OptimizeForNVVM.cpp     |  30 +-
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 202 ++---
 .../TransformOps/LinalgTransformOps.cpp       |  14 +-
 .../BufferizableOpInterfaceImpl.cpp           |   2 +-
 .../Transforms/ConvertConv2DToImg2Col.cpp     | 108 +--
 .../Transforms/ConvertToDestinationStyle.cpp  |  52 +-
 .../Transforms/DataLayoutPropagation.cpp      |  30 +-
 ...DecomposeGenericByUnfoldingPermutation.cpp |  10 +-
 .../Linalg/Transforms/DecomposeLinalgOps.cpp  |  14 +-
 .../Dialect/Linalg/Transforms/Detensorize.cpp |   6 +-
 .../Linalg/Transforms/DropUnitDims.cpp        |  22 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  40 +-
 .../Linalg/Transforms/ElementwiseToLinalg.cpp |   4 +-
 .../EraseUnusedOperandsAndResults.cpp         |   2 +-
 .../FusePadOpWithLinalgProducer.cpp           |   8 +-
 mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp |   2 +-
 .../Linalg/Transforms/Generalization.cpp      |   2 +-
 .../Linalg/Transforms/HoistPadding.cpp        |  20 +-
 .../Dialect/Linalg/Transforms/Hoisting.cpp    |   2 +-
 .../Transforms/InlineScalarOperands.cpp       |   6 +-
 .../Dialect/Linalg/Transforms/Interchange.cpp |   2 +-
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |   8 +-
 .../Transforms/MeshShardingInterfaceImpl.cpp  |  18 +-
 .../Linalg/Transforms/NamedOpConversions.cpp  |   8 +-
 .../Transforms/PackAndUnpackPatterns.cpp      |   6 +-
 .../Linalg/Transforms/PadTilingInterface.cpp  |   6 +-
 .../lib/Dialect/Linalg/Transforms/Padding.cpp |   6 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |  44 +-
 .../Transforms/RuntimeOpVerification.cpp      |   4 +-
 mlir/lib/Dialect/Linalg/Transforms/Split.cpp  |   2 +-
 .../Linalg/Transforms/SplitReduction.cpp      |  32 +-
 .../SwapExtractSliceWithFillPatterns.cpp      |   2 +-
 mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp |  16 +-
 .../Linalg/Transforms/TilingInterfaceImpl.cpp |  48 +-
 .../Dialect/Linalg/Transforms/Transforms.cpp  |  66 +-
 .../Linalg/Transforms/TransposeConv2D.cpp     |   6 +-
 .../Linalg/Transforms/TransposeMatmul.cpp     |  26 +-
 .../Linalg/Transforms/Vectorization.cpp       | 176 ++---
 .../Linalg/Transforms/WinogradConv2D.cpp      | 114 +--
 mlir/lib/Dialect/Linalg/Utils/Utils.cpp       |  18 +-
 .../BufferizableOpInterfaceImpl.cpp           |   2 +-
 mlir/lib/Dialect/MPI/IR/MPIOps.cpp            |   2 +-
 mlir/lib/Dialect/Math/IR/MathOps.cpp          |   2 +-
 .../Transforms/AlgebraicSimplification.cpp    |  20 +-
 .../Math/Transforms/ExpandPatterns.cpp        | 226 +++---
 .../Transforms/ExtendToSupportedTypes.cpp     |   4 +-
 .../Transforms/PolynomialApproximation.cpp    | 536 ++++++-------
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    |   6 +-
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |  22 +-
 .../TransformOps/MemRefTransformOps.cpp       |   2 +-
 .../Transforms/AllocationOpInterfaceImpl.cpp  |   8 +-
 .../MemRef/Transforms/ComposeSubView.cpp      |   2 +-
 .../MemRef/Transforms/EmulateNarrowType.cpp   |  36 +-
 .../Dialect/MemRef/Transforms/ExpandOps.cpp   |  14 +-
 .../MemRef/Transforms/ExpandRealloc.cpp       |  20 +-
 .../Transforms/ExpandStridedMetadata.cpp      |  24 +-
 .../Transforms/ExtractAddressComputations.cpp |  16 +-
 .../MemRef/Transforms/FlattenMemRefs.cpp      |  28 +-
 .../MemRef/Transforms/FoldMemRefAliasOps.cpp  |   6 +-
 .../Transforms/IndependenceTransforms.cpp     |   8 +-
 .../Dialect/MemRef/Transforms/MultiBuffer.cpp |   8 +-
 .../MemRef/Transforms/NormalizeMemRefs.cpp    |   2 +-
 .../MemRef/Transforms/ReifyResultShapes.cpp   |   4 +-
 .../ResolveShapedTypeResultDims.cpp           |   2 +-
 .../Transforms/RuntimeOpVerification.cpp      |  88 +--
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |  14 +-
 .../Mesh/Transforms/Simplifications.cpp       |   4 +-
 .../Dialect/Mesh/Transforms/Spmdization.cpp   |  14 +-
 .../Dialect/Mesh/Transforms/Transforms.cpp    |  36 +-
 .../NVGPU/TransformOps/NVGPUTransformOps.cpp  |  62 +-
 .../NVGPU/Transforms/CreateAsyncGroups.cpp    |  16 +-
 .../NVGPU/Transforms/OptimizeSharedMemory.cpp |  12 +-
 .../Quant/Transforms/LowerQuantOps.cpp        |  78 +-
 .../Quant/Transforms/StripFuncQuantTypes.cpp  |   2 +-
 mlir/lib/Dialect/SCF/IR/SCF.cpp               |  78 +-
 .../SCF/TransformOps/SCFTransformOps.cpp      |   4 +-
 .../BufferizableOpInterfaceImpl.cpp           |  18 +-
 .../lib/Dialect/SCF/Transforms/ForToWhile.cpp |   8 +-
 .../SCF/Transforms/ForallToParallel.cpp       |   2 +-
 .../Dialect/SCF/Transforms/LoopPipelining.cpp |  70 +-
 .../SCF/Transforms/LoopSpecialization.cpp     |  14 +-
 .../SCF/Transforms/ParallelLoopFusion.cpp     |   4 +-
 .../SCF/Transforms/ParallelLoopTiling.cpp     |  32 +-
 .../Transforms/StructuralTypeConversions.cpp  |   6 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  32 +-
 .../SCF/Transforms/UpliftWhileToFor.cpp       |  20 +-
 .../SCF/Transforms/WrapInZeroTripCheck.cpp    |  10 +-
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          |  94 +--
 mlir/lib/Dialect/SMT/IR/SMTDialect.cpp        |   4 +-
 mlir/lib/Dialect/SMT/IR/SMTOps.cpp            |   4 +-
 mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp  |  10 +-
 .../SPIRV/IR/SPIRVCanonicalization.cpp        |  20 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp    |   4 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp        |  22 +-
 .../Linking/ModuleCombiner/ModuleCombiner.cpp |   2 +-
 .../Transforms/LowerABIAttributesPass.cpp     |  18 +-
 .../SPIRV/Transforms/RewriteInsertsPass.cpp   |   2 +-
 .../SPIRV/Transforms/SPIRVConversion.cpp      |  62 +-
 .../Transforms/SPIRVWebGPUTransforms.cpp      |  32 +-
 .../Transforms/UnifyAliasedResourcePass.cpp   |  28 +-
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  30 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/OutlineShapeComputation.cpp    |   6 +-
 .../Shape/Transforms/ShapeToShapeLowering.cpp |   6 +-
 .../SparseTensor/IR/SparseTensorDialect.cpp   |   4 +-
 .../IR/SparseTensorInterfaces.cpp             |   2 +-
 .../Transforms/SparseAssembler.cpp            |  16 +-
 .../Transforms/SparseBufferRewriting.cpp      | 366 ++++-----
 .../Transforms/SparseGPUCodegen.cpp           | 178 ++---
 .../Transforms/SparseIterationToScf.cpp       |  26 +-
 .../Transforms/SparseReinterpretMap.cpp       |  20 +-
 .../Transforms/SparseSpaceCollapse.cpp        |   2 +-
 .../SparseStorageSpecifierToLLVM.cpp          |  10 +-
 .../Transforms/SparseTensorCodegen.cpp        | 138 ++--
 .../Transforms/SparseTensorConversion.cpp     |  44 +-
 .../Transforms/SparseTensorRewriting.cpp      | 212 ++---
 .../Transforms/SparseVectorization.cpp        |  50 +-
 .../Transforms/Sparsification.cpp             |  68 +-
 .../Transforms/StageSparseOperations.cpp      |   4 +-
 .../Transforms/Utils/CodegenUtils.cpp         |  74 +-
 .../Transforms/Utils/CodegenUtils.h           |  30 +-
 .../Transforms/Utils/LoopEmitter.cpp          |  54 +-
 .../Utils/SparseTensorDescriptor.cpp          |  10 +-
 .../Transforms/Utils/SparseTensorDescriptor.h |   2 +-
 .../Transforms/Utils/SparseTensorIterator.cpp |  76 +-
 .../lib/Dialect/SparseTensor/Utils/Merger.cpp | 116 +--
 .../Extensions/MeshShardingExtensions.cpp     |   8 +-
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      |  66 +-
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp   |  20 +-
 .../TransformOps/TensorTransformOps.cpp       |   4 +-
 .../BufferizableOpInterfaceImpl.cpp           |  44 +-
 .../Tensor/Transforms/EmptyOpPatterns.cpp     |   2 +-
 .../ExtractSliceFromReshapeUtils.cpp          |   8 +-
 .../Tensor/Transforms/FoldTensorSubsetOps.cpp |   2 +-
 .../Transforms/IndependenceTransforms.cpp     |   8 +-
 .../Tensor/Transforms/ReshapePatterns.cpp     |  10 +-
 .../Tensor/Transforms/RewriteAsConstant.cpp   |   2 +-
 .../Transforms/RuntimeOpVerification.cpp      |  40 +-
 .../SubsetInsertionOpInterfaceImpl.cpp        |   2 +-
 mlir/lib/Dialect/Tensor/Utils/Utils.cpp       |   6 +-
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |   4 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |  12 +-
 .../Transforms/TosaDecomposeDepthwise.cpp     |  16 +-
 .../Transforms/TosaDecomposeTransposeConv.cpp |   8 +-
 .../Tosa/Transforms/TosaInferShapes.cpp       |   2 +-
 .../Tosa/Transforms/TosaReduceTransposes.cpp  |   4 +-
 .../Tosa/Transforms/TosaTypeConverters.cpp    |   4 +-
 .../Dialect/Tosa/Utils/ConversionUtils.cpp    |  16 +-
 mlir/lib/Dialect/UB/IR/UBOps.cpp              |   2 +-
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  60 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Vector/Transforms/LowerVectorBitCast.cpp  |   8 +-
 .../Transforms/LowerVectorBroadcast.cpp       |  22 +-
 .../Vector/Transforms/LowerVectorContract.cpp | 128 +--
 .../Vector/Transforms/LowerVectorGather.cpp   |  36 +-
 .../Transforms/LowerVectorInterleave.cpp      |  20 +-
 .../Vector/Transforms/LowerVectorMask.cpp     |  22 +-
 .../Transforms/LowerVectorMultiReduction.cpp  |  38 +-
 .../Vector/Transforms/LowerVectorScan.cpp     |  16 +-
 .../Transforms/LowerVectorShapeCast.cpp       |  36 +-
 .../Vector/Transforms/LowerVectorTransfer.cpp |  26 +-
 .../Transforms/LowerVectorTranspose.cpp       |  26 +-
 .../Vector/Transforms/VectorDistribute.cpp    |  76 +-
 .../Transforms/VectorDropLeadUnitDim.cpp      |  34 +-
 .../VectorEmulateMaskedLoadStore.cpp          |  28 +-
 .../Transforms/VectorEmulateNarrowType.cpp    | 176 ++---
 ...sertExtractStridedSliceRewritePatterns.cpp |  30 +-
 .../Vector/Transforms/VectorLinearize.cpp     |   6 +-
 .../Transforms/VectorMaskElimination.cpp      |   2 +-
 .../Transforms/VectorTransferOpTransforms.cpp |  28 +-
 .../VectorTransferSplitRewritePatterns.cpp    |  72 +-
 .../Vector/Transforms/VectorTransforms.cpp    |  82 +-
 .../Vector/Transforms/VectorUnroll.cpp        |  32 +-
 mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp |   6 +-
 .../Dialect/X86Vector/IR/X86VectorDialect.cpp |   6 +-
 .../X86Vector/Transforms/AVXTranspose.cpp     |  26 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   4 +-
 .../Transforms/XeGPUSubgroupDistribute.cpp    |  28 +-
 .../Dialect/XeGPU/Transforms/XeGPUUnroll.cpp  |  38 +-
 .../Transforms/XeGPUWgToSgDistribute.cpp      |  22 +-
 mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp   |  12 +-
 mlir/lib/IR/BuiltinDialect.cpp                |   2 +-
 mlir/lib/Query/Query.cpp                      |   2 +-
 .../lib/Target/LLVMIR/LLVMImportInterface.cpp |   2 +-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |  76 +-
 .../SPIRV/Deserialization/DeserializeOps.cpp  |  20 +-
 .../SPIRV/Deserialization/Deserializer.cpp    |  36 +-
 mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp       |  44 +-
 .../Transforms/Utils/DialectConversion.cpp    |   2 +-
 349 files changed, 6988 insertions(+), 6988 deletions(-)

diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 910fe1b1d93c1..34e859bc9c86e 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -50,20 +50,20 @@ static Value convertUnsignedToI32(ConversionPatternRewriter &rewriter,
   if (i32 == valTy)
     return val;
   return valTy.getWidth() > 32
-             ? Value(rewriter.create<LLVM::TruncOp>(loc, i32, val))
-             : Value(rewriter.create<LLVM::ZExtOp>(loc, i32, val));
+             ? Value(LLVM::TruncOp::create(rewriter, loc, i32, val))
+             : Value(LLVM::ZExtOp::create(rewriter, loc, i32, val));
 }
 
 static Value createI32Constant(ConversionPatternRewriter &rewriter,
                                Location loc, int32_t value) {
   Type i32 = rewriter.getI32Type();
-  return rewriter.create<LLVM::ConstantOp>(loc, i32, value);
+  return LLVM::ConstantOp::create(rewriter, loc, i32, value);
 }
 
 static Value createI1Constant(ConversionPatternRewriter &rewriter, Location loc,
                               bool value) {
   Type llvmI1 = rewriter.getI1Type();
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmI1, value);
+  return LLVM::ConstantOp::create(rewriter, loc, llvmI1, value);
 }
 
 /// Returns the linear index used to access an element in the memref.
@@ -78,11 +78,11 @@ static Value getLinearIndexI32(ConversionPatternRewriter &rewriter,
           ShapedType::isDynamic(stride)
               ? convertUnsignedToI32(rewriter, loc,
                                      memRefDescriptor.stride(rewriter, loc, i))
-              : rewriter.create<LLVM::ConstantOp>(loc, i32, stride);
-      increment = rewriter.create<LLVM::MulOp>(loc, increment, strideValue);
+              : LLVM::ConstantOp::create(rewriter, loc, i32, stride);
+      increment = LLVM::MulOp::create(rewriter, loc, increment, strideValue);
     }
     index =
-        index ? rewriter.create<LLVM::AddOp>(loc, index, increment) : increment;
+        index ? LLVM::AddOp::create(rewriter, loc, index, increment) : increment;
   }
   return index ? index : createI32Constant(rewriter, loc, 0);
 }
@@ -110,14 +110,14 @@ static Value getNumRecords(ConversionPatternRewriter &rewriter, Location loc,
   for (uint32_t i = 0, e = memrefType.getRank(); i < e; ++i) {
     Value size = memrefDescriptor.size(rewriter, loc, i);
     Value stride = memrefDescriptor.stride(rewriter, loc, i);
-    Value maxThisDim = rewriter.create<LLVM::MulOp>(loc, size, stride);
+    Value maxThisDim = LLVM::MulOp::create(rewriter, loc, size, stride);
     maxIndex = maxIndex
-                   ? rewriter.create<LLVM::UMaxOp>(loc, maxIndex, maxThisDim)
+                   ? LLVM::UMaxOp::create(rewriter, loc, maxIndex, maxThisDim)
                    : maxThisDim;
   }
   Value maxIndexI32 = convertUnsignedToI32(rewriter, loc, maxIndex);
   Value byteWidthConst = createI32Constant(rewriter, loc, elementByteWidth);
-  return rewriter.create<LLVM::MulOp>(loc, maxIndexI32, byteWidthConst);
+  return LLVM::MulOp::create(rewriter, loc, maxIndexI32, byteWidthConst);
 }
 
 static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
@@ -132,13 +132,13 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
   Value stride;
   if (chipset.majorVersion == 9 && chipset >= kGfx942 && cacheSwizzleStride) {
     Value cacheStrideZext =
-        rewriter.create<LLVM::ZExtOp>(loc, i16, cacheSwizzleStride);
-    Value swizzleBit = rewriter.create<LLVM::ConstantOp>(
+        LLVM::ZExtOp::create(rewriter, loc, i16, cacheSwizzleStride);
+    Value swizzleBit = LLVM::ConstantOp::create(rewriter,
         loc, i16, rewriter.getI16IntegerAttr(1 << 14));
-    stride = rewriter.create<LLVM::OrOp>(loc, cacheStrideZext, swizzleBit,
+    stride = LLVM::OrOp::create(rewriter, loc, cacheStrideZext, swizzleBit,
                                          /*isDisjoint=*/true);
   } else {
-    stride = rewriter.create<LLVM::ConstantOp>(loc, i16,
+    stride = LLVM::ConstantOp::create(rewriter, loc, i16,
                                                rewriter.getI16IntegerAttr(0));
   }
   // Get the number of elements.
@@ -209,18 +209,18 @@ struct FatRawBufferCastLowering
             : descriptor.alignedPtr(rewriter, loc);
 
     Value offset = adaptor.getResetOffset()
-                       ? rewriter.create<LLVM::ConstantOp>(
+                       ? LLVM::ConstantOp::create(rewriter,
                              loc, getIndexType(), rewriter.getIndexAttr(0))
                        : descriptor.offset(rewriter, loc);
 
     bool hasSizes = memrefType.getRank() > 0;
     // No need to unpack() and pack() all the individual sizes and strides,
     // so we'll just extract the arrays.
-    Value sizes = hasSizes ? rewriter.create<LLVM::ExtractValueOp>(
+    Value sizes = hasSizes ? LLVM::ExtractValueOp::create(rewriter,
                                  loc, descriptor, kSizePosInMemRefDescriptor)
                            : Value{};
     Value strides = hasSizes
-                        ? rewriter.create<LLVM::ExtractValueOp>(
+                        ? LLVM::ExtractValueOp::create(rewriter,
                               loc, descriptor, kStridePosInMemRefDescriptor)
                         : Value{};
 
@@ -231,16 +231,16 @@ struct FatRawBufferCastLowering
     Value result = MemRefDescriptor::poison(
         rewriter, loc,
         getTypeConverter()->convertType(op.getResult().getType()));
-    result = rewriter.create<LLVM::InsertValueOp>(
+    result = LLVM::InsertValueOp::create(rewriter,
         loc, result, fatPtr, kAllocatedPtrPosInMemRefDescriptor);
-    result = rewriter.create<LLVM::InsertValueOp>(
+    result = LLVM::InsertValueOp::create(rewriter,
         loc, result, fatPtr, kAlignedPtrPosInMemRefDescriptor);
-    result = rewriter.create<LLVM::InsertValueOp>(loc, result, offset,
+    result = LLVM::InsertValueOp::create(rewriter, loc, result, offset,
                                                   kOffsetPosInMemRefDescriptor);
     if (hasSizes) {
-      result = rewriter.create<LLVM::InsertValueOp>(loc, result, sizes,
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, sizes,
                                                     kSizePosInMemRefDescriptor);
-      result = rewriter.create<LLVM::InsertValueOp>(
+      result = LLVM::InsertValueOp::create(rewriter,
           loc, result, strides, kStridePosInMemRefDescriptor);
     }
     rewriter.replaceOp(op, result);
@@ -343,7 +343,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     if (storeData) {
       if (llvmBufferValType != llvmWantedDataType) {
         Value castForStore =
-            rewriter.create<LLVM::BitcastOp>(loc, llvmBufferValType, storeData);
+            LLVM::BitcastOp::create(rewriter, loc, llvmBufferValType, storeData);
         args.push_back(castForStore);
       } else {
         args.push_back(storeData);
@@ -352,7 +352,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     if (atomicCmpData) {
       if (llvmBufferValType != llvmWantedDataType) {
-        Value castForCmp = rewriter.create<LLVM::BitcastOp>(
+        Value castForCmp = LLVM::BitcastOp::create(rewriter,
             loc, llvmBufferValType, atomicCmpData);
         args.push_back(castForCmp);
       } else {
@@ -383,17 +383,17 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
         indexOffset && *indexOffset > 0) {
       Value extraOffsetConst = createI32Constant(rewriter, loc, *indexOffset);
       voffset =
-          voffset ? rewriter.create<LLVM::AddOp>(loc, voffset, extraOffsetConst)
+          voffset ? LLVM::AddOp::create(rewriter, loc, voffset, extraOffsetConst)
                   : extraOffsetConst;
     }
-    voffset = rewriter.create<LLVM::MulOp>(loc, voffset, byteWidthConst);
+    voffset = LLVM::MulOp::create(rewriter, loc, voffset, byteWidthConst);
     args.push_back(voffset);
 
     // SGPR offset.
     Value sgprOffset = adaptor.getSgprOffset();
     if (!sgprOffset)
       sgprOffset = createI32Constant(rewriter, loc, 0);
-    sgprOffset = rewriter.create<LLVM::MulOp>(loc, sgprOffset, byteWidthConst);
+    sgprOffset = LLVM::MulOp::create(rewriter, loc, sgprOffset, byteWidthConst);
     args.push_back(sgprOffset);
 
     // bit 0: GLC = 0 (atomics drop value, less coherency)
@@ -403,12 +403,12 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
                                            llvmBufferValType);
-    Operation *lowered = rewriter.create<Intrinsic>(loc, resultTypes, args,
+    Operation *lowered = Intrinsic::create(rewriter, loc, resultTypes, args,
                                                     ArrayRef<NamedAttribute>());
     if (lowered->getNumResults() == 1) {
       Value replacement = lowered->getResult(0);
       if (llvmBufferValType != llvmWantedDataType) {
-        replacement = rewriter.create<LLVM::BitcastOp>(loc, llvmWantedDataType,
+        replacement = LLVM::BitcastOp::create(rewriter, loc, llvmWantedDataType,
                                                        replacement);
       }
       rewriter.replaceOp(gpuOp, replacement);
@@ -465,12 +465,12 @@ struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
                << chipset.majorVersion;
 
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::SWaitcntOp>(loc, ldsOnlyBits);
+      ROCDL::SWaitcntOp::create(rewriter, loc, ldsOnlyBits);
       rewriter.replaceOpWithNewOp<ROCDL::SBarrierOp>(op);
     } else {
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::WaitDscntOp>(loc, 0);
-      rewriter.create<ROCDL::BarrierSignalOp>(loc, -1);
+      ROCDL::WaitDscntOp::create(rewriter, loc, 0);
+      ROCDL::BarrierSignalOp::create(rewriter, loc, -1);
       rewriter.replaceOpWithNewOp<ROCDL::BarrierWaitOp>(op, -1);
     }
 
@@ -516,18 +516,18 @@ static Value convertMFMAVectorOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   if (auto vectorType = dyn_cast<VectorType>(inputType)) {
     if (vectorType.getElementType().isBF16() && !allowBf16)
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, vectorType.clone(rewriter.getI16Type()), input);
     if (vectorType.getElementType().isInteger(8) &&
         vectorType.getNumElements() <= 8)
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, rewriter.getIntegerType(vectorType.getNumElements() * 8), input);
     if (isa<IntegerType>(vectorType.getElementType()) &&
         vectorType.getElementTypeBitWidth() <= 8) {
       int64_t numWords = llvm::divideCeil(
           vectorType.getNumElements() * vectorType.getElementTypeBitWidth(),
           32);
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, VectorType::get(numWords, rewriter.getI32Type()), input);
     }
   }
@@ -549,8 +549,8 @@ static Value castMFMAScaleOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   Type outputType = rewriter.getI32Type();
   if (auto intType = dyn_cast<IntegerType>(inputType))
-    return rewriter.create<LLVM::ZExtOp>(loc, outputType, input);
-  return rewriter.create<LLVM::BitcastOp>(loc, outputType, input);
+    return LLVM::ZExtOp::create(rewriter, loc, outputType, input);
+  return LLVM::BitcastOp::create(rewriter, loc, outputType, input);
 }
 
 /// Push an input operand. If it is a float type, nothing to do. If it is
@@ -576,7 +576,7 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   Type elemType = vectorType.getElementType();
 
   if (elemType.isBF16())
-    llvmInput = rewriter.create<LLVM::BitcastOp>(
+    llvmInput = LLVM::BitcastOp::create(rewriter,
         loc, vectorType.clone(rewriter.getI16Type()), llvmInput);
   if (elemType.getIntOrFloatBitWidth() > 8) {
     operands.push_back(llvmInput);
@@ -613,7 +613,7 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   // (256 / 64) * 4 = 16 bits of input (on gfx12+) but take i32 arguments.
   // Add in the zeros here.
   if (numBits < 32)
-    castInput = rewriter.create<LLVM::ZExtOp>(loc, i32, castInput);
+    castInput = LLVM::ZExtOp::create(rewriter, loc, i32, castInput);
   operands.push_back(castInput);
 }
 
@@ -633,7 +633,7 @@ static void wmmaPushOutputOperand(ConversionPatternRewriter &rewriter,
   auto vectorType = dyn_cast<VectorType>(inputType);
   Type elemType = vectorType.getElementType();
   if (elemType.isBF16())
-    output = rewriter.create<LLVM::BitcastOp>(
+    output = LLVM::BitcastOp::create(rewriter,
         loc, vectorType.clone(rewriter.getI16Type()), output);
   operands.push_back(output);
   if (elemType.isF16() || elemType.isBF16() || elemType.isInteger(16)) {
@@ -992,7 +992,7 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
     };
     Value lowered = rewriter.create(loweredOp)->getResult(0);
     if (outType != intrinsicOutType)
-      lowered = rewriter.create<LLVM::BitcastOp>(loc, outType, lowered);
+      lowered = LLVM::BitcastOp::create(rewriter, loc, outType, lowered);
     rewriter.replaceOp(op, lowered);
     return success();
   }
@@ -1093,7 +1093,7 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
     Operation *maybeCastBack = lowered;
     if (rawOutType != outType)
       maybeCastBack =
-          rewriter.create<LLVM::BitcastOp>(loc, outType, lowered->getResult(0));
+          LLVM::BitcastOp::create(rewriter, loc, outType, lowered->getResult(0));
     rewriter.replaceOp(op, maybeCastBack->getResults());
 
     return success();
@@ -1144,21 +1144,21 @@ struct TransposeLoadOpLowering
     case 4: {
       assert(numElements == 16);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr4_b64>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr4_b64::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 6: {
       assert(numElements == 16);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr6_b96>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr6_b96::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 8: {
       assert(numElements == 8);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr8_b64>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr8_b64::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
@@ -1311,21 +1311,21 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
   Type sourceElemType = getElementTypeOrSelf(op.getSource());
   // Extend to a v4i8
   if (!sourceVecType || sourceVecType.getNumElements() < 4) {
-    Value longVec = rewriter.create<LLVM::UndefOp>(loc, v4i8);
+    Value longVec = LLVM::UndefOp::create(rewriter, loc, v4i8);
     if (!sourceVecType) {
-      longVec = rewriter.create<LLVM::InsertElementOp>(
+      longVec = LLVM::InsertElementOp::create(rewriter,
           loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
-        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        Value elem = LLVM::ExtractElementOp::create(rewriter, loc, source, idx);
         longVec =
-            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+            LLVM::InsertElementOp::create(rewriter, loc, longVec, elem, idx);
       }
     }
     source = longVec;
   }
-  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+  Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
   if (resultVecType) {
     if (typeIsExpectedBf8ForChipset(chipset, sourceElemType)) {
       rewriter.replaceOpWithNewOp<ROCDL::CvtPkF32Bf8Op>(op, f32, i32Source,
@@ -1377,21 +1377,21 @@ LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
 
   // Extend to a packedVectorType
   if (sourceVecType.getNumElements() < packedVecType.getNumElements()) {
-    Value longVec = rewriter.create<LLVM::ZeroOp>(loc, packedVecType);
+    Value longVec = LLVM::ZeroOp::create(rewriter, loc, packedVecType);
     if (!sourceVecType) {
-      longVec = rewriter.create<LLVM::InsertElementOp>(
+      longVec = LLVM::InsertElementOp::create(rewriter,
           loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
-        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        Value elem = LLVM::ExtractElementOp::create(rewriter, loc, source, idx);
         longVec =
-            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+            LLVM::InsertElementOp::create(rewriter, loc, longVec, elem, idx);
       }
     }
     source = longVec;
   }
-  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+  Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
 
   if (isa<Float8E5M2Type>(sourceElemType) && destElemType.isF32())
     rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Bf8Op>(
@@ -1449,53 +1449,53 @@ LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
   Value scale = adaptor.getScale();
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, intResultType, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, intResultType, existing);
   else
-    existing = rewriter.create<LLVM::ZeroOp>(loc, intResultType);
+    existing = LLVM::ZeroOp::create(rewriter, loc, intResultType);
 
   if (sourceVecType.getNumElements() < 2) {
     Value c0 = createI32Constant(rewriter, loc, 0);
-    Value elem0 = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
+    Value elem0 = LLVM::ExtractElementOp::create(rewriter, loc, source, c0);
     VectorType v2 = VectorType::get(2, sourceElemType);
-    source = rewriter.create<LLVM::ZeroOp>(loc, v2);
-    source = rewriter.create<LLVM::InsertElementOp>(loc, source, elem0, c0);
+    source = LLVM::ZeroOp::create(rewriter, loc, v2);
+    source = LLVM::InsertElementOp::create(rewriter, loc, source, elem0, c0);
   }
 
   Value sourceA, sourceB;
   if (sourceElemType.isF32()) {
     Value c0 = createI32Constant(rewriter, loc, 0);
     Value c1 = createI32Constant(rewriter, loc, 1);
-    sourceA = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
-    sourceB = rewriter.create<LLVM::ExtractElementOp>(loc, source, c1);
+    sourceA = LLVM::ExtractElementOp::create(rewriter, loc, source, c0);
+    sourceB = LLVM::ExtractElementOp::create(rewriter, loc, source, c1);
   }
 
   Value result;
   if (sourceElemType.isF32() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F32Op>(
+    result = ROCDL::CvtScaleF32PkBf8F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F16Op>(
+    result = ROCDL::CvtScaleF32PkBf8F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8Bf16Op>(
+    result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F32Op>(
+    result = ROCDL::CvtScaleF32PkFp8F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F16Op>(
+    result = ROCDL::CvtScaleF32PkFp8F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8Bf16Op>(
+    result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F32Op>(
+    result = ROCDL::CvtScaleF32PkFp4F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F16Op>(
+    result = ROCDL::CvtScaleF32PkFp4F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4Bf16Op>(
+    result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else
     return failure();
@@ -1521,19 +1521,19 @@ LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
   Value sourceA = adaptor.getSourceA();
   Value sourceB = adaptor.getSourceB();
   if (!sourceB)
-    sourceB = rewriter.create<LLVM::UndefOp>(loc, sourceA.getType());
+    sourceB = LLVM::UndefOp::create(rewriter, loc, sourceA.getType());
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, i32, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, i32, existing);
   else
-    existing = rewriter.create<LLVM::UndefOp>(loc, i32);
+    existing = LLVM::UndefOp::create(rewriter, loc, i32);
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtPkBf8F32Op>(loc, i32, sourceA, sourceB,
+    result = ROCDL::CvtPkBf8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
                                                    existing, op.getWordIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtPkFp8F32Op>(loc, i32, sourceA, sourceB,
+    result = ROCDL::CvtPkFp8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
                                                    existing, op.getWordIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
@@ -1558,16 +1558,16 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
   Value stoch = adaptor.getStochiasticParam();
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, i32, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, i32, existing);
   else
-    existing = rewriter.create<LLVM::UndefOp>(loc, i32);
+    existing = LLVM::UndefOp::create(rewriter, loc, i32);
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtSrBf8F32Op>(
+    result = ROCDL::CvtSrBf8F32Op::create(rewriter,
         loc, i32, source, stoch, existing, op.getStoreIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtSrFp8F32Op>(
+    result = ROCDL::CvtSrFp8F32Op::create(rewriter,
         loc, i32, source, stoch, existing, op.getStoreIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
@@ -1612,14 +1612,14 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
       if (operandType.getIntOrFloatBitWidth() <= 16) {
         if (llvm::isa<FloatType>(operandType)) {
           operand =
-              rewriter.create<LLVM::BitcastOp>(loc, llvmSrcIntType, operand);
+              LLVM::BitcastOp::create(rewriter, loc, llvmSrcIntType, operand);
         }
         auto llvmVecType = typeConverter->convertType(mlir::VectorType::get(
             32 / operandType.getIntOrFloatBitWidth(), llvmSrcIntType));
-        Value undefVec = rewriter.create<LLVM::UndefOp>(loc, llvmVecType);
-        operand = rewriter.create<LLVM::InsertElementOp>(
+        Value undefVec = LLVM::UndefOp::create(rewriter, loc, llvmVecType);
+        operand = LLVM::InsertElementOp::create(rewriter,
             loc, undefVec, operand, createI32Constant(rewriter, loc, 0));
-        operand = rewriter.create<LLVM::BitcastOp>(loc, llvmType, operand);
+        operand = LLVM::BitcastOp::create(rewriter, loc, llvmType, operand);
       }
       return operand;
     };
@@ -1706,14 +1706,14 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
     bool boundCtrl = DppOp->getAttrOfType<BoolAttr>("bound_ctrl").getValue();
 
     // create a ROCDL_DPPMovOp instruction with the appropriate attributes
-    auto dppMovOp = rewriter.create<ROCDL::DPPUpdateOp>(
+    auto dppMovOp = ROCDL::DPPUpdateOp::create(rewriter,
         loc, llvmType, old, src, DppCtrl, rowMask, bankMask, boundCtrl);
 
     Value result = dppMovOp.getRes();
     if (srcType.getIntOrFloatBitWidth() < 32) {
-      result = rewriter.create<LLVM::TruncOp>(loc, llvmSrcIntType, result);
+      result = LLVM::TruncOp::create(rewriter, loc, llvmSrcIntType, result);
       if (!llvm::isa<IntegerType>(srcType)) {
-        result = rewriter.create<LLVM::BitcastOp>(loc, srcType, result);
+        result = LLVM::BitcastOp::create(rewriter, loc, srcType, result);
       }
     }
 
@@ -1747,7 +1747,7 @@ struct AMDGPUSwizzleBitModeLowering
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
       Value res =
-          rewriter.create<ROCDL::DsSwizzleOp>(loc, v.getType(), v, maskValue);
+          ROCDL::DsSwizzleOp::create(rewriter, loc, v.getType(), v, maskValue);
       swizzled.emplace_back(res);
     }
 
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 3b143ca1ef9ce..5a9d946c2fe4e 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -50,9 +50,9 @@ static Value buildMinMaxReductionSeq(Location loc,
   Value value = *valueIt++;
   for (; valueIt != values.end(); ++valueIt) {
     if (predicate == arith::CmpIPredicate::sgt)
-      value = builder.create<arith::MaxSIOp>(loc, value, *valueIt);
+      value = arith::MaxSIOp::create(builder, loc, value, *valueIt);
     else
-      value = builder.create<arith::MinSIOp>(loc, value, *valueIt);
+      value = arith::MinSIOp::create(builder, loc, value, *valueIt);
   }
 
   return value;
@@ -154,8 +154,8 @@ class AffineForLowering : public OpRewritePattern<AffineForOp> {
     Value lowerBound = lowerAffineLowerBound(op, rewriter);
     Value upperBound = lowerAffineUpperBound(op, rewriter);
     Value step =
-        rewriter.create<arith::ConstantIndexOp>(loc, op.getStepAsInt());
-    auto scfForOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound,
+        arith::ConstantIndexOp::create(rewriter, loc, op.getStepAsInt());
+    auto scfForOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound,
                                                 step, op.getInits());
     rewriter.eraseBlock(scfForOp.getBody());
     rewriter.inlineRegionBefore(op.getRegion(), scfForOp.getRegion(),
@@ -197,7 +197,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     }
     steps.reserve(op.getSteps().size());
     for (int64_t step : op.getSteps())
-      steps.push_back(rewriter.create<arith::ConstantIndexOp>(loc, step));
+      steps.push_back(arith::ConstantIndexOp::create(rewriter, loc, step));
 
     // Get the terminator op.
     auto affineParOpTerminator =
@@ -205,7 +205,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     scf::ParallelOp parOp;
     if (op.getResults().empty()) {
       // Case with no reduction operations/return values.
-      parOp = rewriter.create<scf::ParallelOp>(loc, lowerBoundTuple,
+      parOp = scf::ParallelOp::create(rewriter, loc, lowerBoundTuple,
                                                upperBoundTuple, steps,
                                                /*bodyBuilderFn=*/nullptr);
       rewriter.eraseBlock(parOp.getBody());
@@ -233,7 +233,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       identityVals.push_back(
           arith::getIdentityValue(reductionOpValue, resultType, rewriter, loc));
     }
-    parOp = rewriter.create<scf::ParallelOp>(
+    parOp = scf::ParallelOp::create(rewriter,
         loc, lowerBoundTuple, upperBoundTuple, steps, identityVals,
         /*bodyBuilderFn=*/nullptr);
 
@@ -261,7 +261,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       Value reductionResult = arith::getReductionOp(
           reductionOpValue, rewriter, loc, reductionBody.getArgument(0),
           reductionBody.getArgument(1));
-      rewriter.create<scf::ReduceReturnOp>(loc, reductionResult);
+      scf::ReduceReturnOp::create(rewriter, loc, reductionResult);
     }
     rewriter.replaceOp(op, parOp.getResults());
     return success();
@@ -278,7 +278,7 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
 
     // Now we just have to handle the condition logic.
     auto integerSet = op.getIntegerSet();
-    Value zeroConstant = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zeroConstant = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value, 8> operands(op.getOperands());
     auto operandsRef = llvm::ArrayRef(operands);
 
@@ -298,17 +298,17 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
       auto pred =
           isEquality ? arith::CmpIPredicate::eq : arith::CmpIPredicate::sge;
       Value cmpVal =
-          rewriter.create<arith::CmpIOp>(loc, pred, affResult, zeroConstant);
+          arith::CmpIOp::create(rewriter, loc, pred, affResult, zeroConstant);
       cond = cond
-                 ? rewriter.create<arith::AndIOp>(loc, cond, cmpVal).getResult()
+                 ? arith::AndIOp::create(rewriter, loc, cond, cmpVal).getResult()
                  : cmpVal;
     }
     cond = cond ? cond
-                : rewriter.create<arith::ConstantIntOp>(loc, /*value=*/1,
+                : arith::ConstantIntOp::create(rewriter, loc, /*value=*/1,
                                                         /*width=*/1);
 
     bool hasElseRegion = !op.getElseRegion().empty();
-    auto ifOp = rewriter.create<scf::IfOp>(loc, op.getResultTypes(), cond,
+    auto ifOp = scf::IfOp::create(rewriter, loc, op.getResultTypes(), cond,
                                            hasElseRegion);
     rewriter.inlineRegionBefore(op.getThenRegion(),
                                 &ifOp.getThenRegion().back());
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index 3596b3235a631..d0155b12a6ec2 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -89,9 +89,9 @@ static Value castF32To(Type desType, Value f32, Location loc,
   if (elementType.isF32())
     return f32;
   if (elementType.getIntOrFloatBitWidth() < 32)
-    return rewriter.create<arith::TruncFOp>(loc, desType, f32);
+    return arith::TruncFOp::create(rewriter, loc, desType, f32);
   if (elementType.getIntOrFloatBitWidth() > 32)
-    return rewriter.create<arith::ExtFOp>(loc, desType, f32);
+    return arith::ExtFOp::create(rewriter, loc, desType, f32);
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
@@ -113,7 +113,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
   VectorType extResType = VectorType::get(2, rewriter.getF32Type());
   if (!inVecType) {
-    Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
+    Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
         loc, rewriter.getF32Type(), in, 0);
     Value result = castF32To(outElemType, asFloat, loc, rewriter);
     rewriter.replaceOp(op, result);
@@ -121,7 +121,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   }
   int64_t numElements = inVecType.getNumElements();
 
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   VectorType outType = cast<VectorType>(op.getOut().getType());
 
@@ -129,10 +129,10 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
     Value zerodSplat =
         rewriter.createOrFold<vector::SplatOp>(loc, outType, zero);
     Value scalarIn =
-        rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
+        vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     Value scalarExt =
-        rewriter.create<arith::ExtFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarExt, zerodSplat,
+        arith::ExtFOp::create(rewriter, loc, outElemType, scalarIn);
+    Value result = vector::InsertOp::create(rewriter, loc, scalarExt, zerodSplat,
                                                      ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
@@ -145,32 +145,32 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   if (inVecType.getRank() > 1) {
     inVecType = VectorType::get(SmallVector<int64_t>{numElements},
                                 inVecType.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVecType, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVecType, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
-    Value inSlice = rewriter.create<vector::ExtractStridedSliceOp>(
+    Value inSlice = vector::ExtractStridedSliceOp::create(rewriter,
         loc, in, i, elemsThisOp, 1);
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
       if (i + j + 1 < numElements) { // Convert two 8-bit elements
-        Value asFloats = rewriter.create<amdgpu::ExtPackedFp8Op>(
+        Value asFloats = amdgpu::ExtPackedFp8Op::create(rewriter,
             loc, extResType, inSlice, j / 2);
         Type desType = VectorType::get(2, outElemType);
         Value asType = castF32To(desType, asFloats, loc, rewriter);
-        result = rewriter.create<vector::InsertStridedSliceOp>(
+        result = vector::InsertStridedSliceOp::create(rewriter,
             loc, asType, result, i + j, 1);
       } else { // Convert a 8-bit element
-        Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
+        Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
             loc, rewriter.getF32Type(), inSlice, j / 2 * 2);
         Value asType = castF32To(outElemType, asFloat, loc, rewriter);
-        result = rewriter.create<vector::InsertOp>(loc, asType, result, i + j);
+        result = vector::InsertOp::create(rewriter, loc, asType, result, i + j);
       }
     }
   }
 
   if (inVecType.getRank() != outType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outType, result);
   }
 
   rewriter.replaceOp(op, result);
@@ -182,9 +182,9 @@ static Value castToF32(Value value, Location loc, PatternRewriter &rewriter) {
   if (type.isF32())
     return value;
   if (type.getIntOrFloatBitWidth() < 32)
-    return rewriter.create<arith::ExtFOp>(loc, rewriter.getF32Type(), value);
+    return arith::ExtFOp::create(rewriter, loc, rewriter.getF32Type(), value);
   if (type.getIntOrFloatBitWidth() > 32)
-    return rewriter.create<arith::TruncFOp>(loc, rewriter.getF32Type(), value);
+    return arith::TruncFOp::create(rewriter, loc, rewriter.getF32Type(), value);
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
@@ -224,13 +224,13 @@ static Value clampInput(PatternRewriter &rewriter, Location loc,
       loc, arith::CmpFPredicate::OEQ, source, negInf);
   Value isNan = rewriter.createOrFold<arith::CmpFOp>(
       loc, arith::CmpFPredicate::UNO, source, source);
-  Value isNonFinite = rewriter.create<arith::OrIOp>(
-      loc, rewriter.create<arith::OrIOp>(loc, isInf, isNegInf), isNan);
+  Value isNonFinite = arith::OrIOp::create(rewriter,
+      loc, arith::OrIOp::create(rewriter, loc, isInf, isNegInf), isNan);
 
-  Value clampedBelow = rewriter.create<arith::MaximumFOp>(loc, source, minCst);
-  Value clamped = rewriter.create<arith::MinimumFOp>(loc, clampedBelow, maxCst);
+  Value clampedBelow = arith::MaximumFOp::create(rewriter, loc, source, minCst);
+  Value clamped = arith::MinimumFOp::create(rewriter, loc, clampedBelow, maxCst);
   Value res =
-      rewriter.create<arith::SelectOp>(loc, isNonFinite, source, clamped);
+      arith::SelectOp::create(rewriter, loc, isNonFinite, source, clamped);
   return res;
 }
 
@@ -264,24 +264,24 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   VectorType truncResType = VectorType::get(4, outElemType);
   if (!inVectorTy) {
     Value asFloat = castToF32(in, loc, rewriter);
-    Value asF8s = rewriter.create<amdgpu::PackedTrunc2xFp8Op>(
+    Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
         loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
         /*existing=*/nullptr);
-    Value result = rewriter.create<vector::ExtractOp>(loc, asF8s, 0);
+    Value result = vector::ExtractOp::create(rewriter, loc, asF8s, 0);
     rewriter.replaceOp(op, result);
     return success();
   }
 
   int64_t numElements = outVecType.getNumElements();
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   if (outVecType.getShape().empty()) {
     Value scalarIn =
-        rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
+        vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     // Recurse to send the 0-D vector case to the 1-D vector case
     Value scalarTrunc =
-        rewriter.create<arith::TruncFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarTrunc, zero,
+        arith::TruncFOp::create(rewriter, loc, outElemType, scalarIn);
+    Value result = vector::InsertOp::create(rewriter, loc, scalarTrunc, zero,
                                                      ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
@@ -294,32 +294,32 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
                                  inVectorTy.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVectorTy, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVectorTy, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
     Value thisResult = nullptr;
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
-      Value elemA = rewriter.create<vector::ExtractOp>(loc, in, i + j);
+      Value elemA = vector::ExtractOp::create(rewriter, loc, in, i + j);
       Value asFloatA = castToF32(elemA, loc, rewriter);
       Value asFloatB = nullptr;
       if (j + 1 < elemsThisOp) {
-        Value elemB = rewriter.create<vector::ExtractOp>(loc, in, i + j + 1);
+        Value elemB = vector::ExtractOp::create(rewriter, loc, in, i + j + 1);
         asFloatB = castToF32(elemB, loc, rewriter);
       }
-      thisResult = rewriter.create<amdgpu::PackedTrunc2xFp8Op>(
+      thisResult = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
           loc, truncResType, asFloatA, asFloatB, j / 2, thisResult);
     }
     if (elemsThisOp < 4)
-      thisResult = rewriter.create<vector::ExtractStridedSliceOp>(
+      thisResult = vector::ExtractStridedSliceOp::create(rewriter,
           loc, thisResult, 0, elemsThisOp, 1);
-    result = rewriter.create<vector::InsertStridedSliceOp>(loc, thisResult,
+    result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
                                                            result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
@@ -347,10 +347,10 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
 
   // Handle the case where input type is not a vector type
   if (!inVectorTy) {
-    auto sourceB = rewriter.create<LLVM::PoisonOp>(loc, rewriter.getF32Type());
+    auto sourceB = LLVM::PoisonOp::create(rewriter, loc, rewriter.getF32Type());
     Value asF16s =
-        rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, in, sourceB);
-    Value result = rewriter.create<vector::ExtractOp>(loc, asF16s, 0);
+        ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, in, sourceB);
+    Value result = vector::ExtractOp::create(rewriter, loc, asF16s, 0);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -362,7 +362,7 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
                                  inVectorTy.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVectorTy, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVectorTy, in);
   }
 
   // Handle the vector case. We also handle the (uncommon) case where the vector
@@ -370,25 +370,25 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   for (int64_t i = 0; i < numElements; i += 2) {
     int64_t elemsThisOp = std::min(numElements, i + 2) - i;
     Value thisResult = nullptr;
-    Value elemA = rewriter.create<vector::ExtractOp>(loc, in, i);
-    Value elemB = rewriter.create<LLVM::PoisonOp>(loc, rewriter.getF32Type());
+    Value elemA = vector::ExtractOp::create(rewriter, loc, in, i);
+    Value elemB = LLVM::PoisonOp::create(rewriter, loc, rewriter.getF32Type());
 
     if (elemsThisOp == 2) {
-      elemB = rewriter.create<vector::ExtractOp>(loc, in, i + 1);
+      elemB = vector::ExtractOp::create(rewriter, loc, in, i + 1);
     }
 
     thisResult =
-        rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, elemA, elemB);
+        ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, elemA, elemB);
     // Place back the truncated result into the possibly larger vector. If we
     // are operating on a size 2 vector, these operations should be folded away
-    thisResult = rewriter.create<vector::ExtractStridedSliceOp>(
+    thisResult = vector::ExtractStridedSliceOp::create(rewriter,
         loc, thisResult, 0, elemsThisOp, 1);
-    result = rewriter.create<vector::InsertStridedSliceOp>(loc, thisResult,
+    result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
                                                            result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
index cbe0b3fda3410..40bf8f1cffb04 100644
--- a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
+++ b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
@@ -74,14 +74,14 @@ struct ConstantOpToArmSMELowering : public OpRewritePattern<arith::ConstantOp> {
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
     auto denseAttr1D = DenseElementsAttr::get(
         tileSliceType, denseAttr.getSplatValue<Attribute>());
-    auto constantOp1D = rewriter.create<arith::ConstantOp>(loc, denseAttr1D);
+    auto constantOp1D = arith::ConstantOp::create(rewriter, loc, denseAttr1D);
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to write vector to tile
       // slice.
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, constantOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
index a5c08a6378021..c5b3fdc536f72 100644
--- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
+++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
@@ -110,7 +110,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     emitc::CmpPredicate predicate;
     switch (op.getPredicate()) {
     case arith::CmpFPredicate::AlwaysFalse: {
-      auto constant = rewriter.create<emitc::ConstantOp>(
+      auto constant = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getI1Type(),
           rewriter.getBoolAttr(/*value=*/false));
       rewriter.replaceOp(op, constant);
@@ -179,7 +179,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
       return success();
     }
     case arith::CmpFPredicate::AlwaysTrue: {
-      auto constant = rewriter.create<emitc::ConstantOp>(
+      auto constant = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getI1Type(),
           rewriter.getBoolAttr(/*value=*/true));
       rewriter.replaceOp(op, constant);
@@ -189,7 +189,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
 
     // Compare the values naively
     auto cmpResult =
-        rewriter.create<emitc::CmpOp>(op.getLoc(), op.getType(), predicate,
+        emitc::CmpOp::create(rewriter, op.getLoc(), op.getType(), predicate,
                                       adaptor.getLhs(), adaptor.getRhs());
 
     // Adjust the results for unordered/ordered semantics
@@ -213,7 +213,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNaN(ConversionPatternRewriter &rewriter, Location loc,
               Value operand) const {
     // A value is NaN exactly when it compares unequal to itself.
-    return rewriter.create<emitc::CmpOp>(
+    return emitc::CmpOp::create(rewriter,
         loc, rewriter.getI1Type(), emitc::CmpPredicate::ne, operand, operand);
   }
 
@@ -221,7 +221,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNotNaN(ConversionPatternRewriter &rewriter, Location loc,
                  Value operand) const {
     // A value is not NaN exactly when it compares equal to itself.
-    return rewriter.create<emitc::CmpOp>(
+    return emitc::CmpOp::create(rewriter,
         loc, rewriter.getI1Type(), emitc::CmpPredicate::eq, operand, operand);
   }
 
@@ -231,7 +231,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
                                Location loc, Value first, Value second) const {
     auto firstIsNaN = isNaN(rewriter, loc, first);
     auto secondIsNaN = isNaN(rewriter, loc, second);
-    return rewriter.create<emitc::LogicalOrOp>(loc, rewriter.getI1Type(),
+    return emitc::LogicalOrOp::create(rewriter, loc, rewriter.getI1Type(),
                                                firstIsNaN, secondIsNaN);
   }
 
@@ -241,7 +241,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
                              Value first, Value second) const {
     auto firstIsNotNaN = isNotNaN(rewriter, loc, first);
     auto secondIsNotNaN = isNotNaN(rewriter, loc, second);
-    return rewriter.create<emitc::LogicalAndOp>(loc, rewriter.getI1Type(),
+    return emitc::LogicalAndOp::create(rewriter, loc, rewriter.getI1Type(),
                                                 firstIsNotNaN, secondIsNotNaN);
   }
 };
@@ -378,9 +378,9 @@ class CastConversion : public OpConversionPattern<ArithOp> {
       Type attrType = (emitc::isPointerWideType(operandType))
                           ? rewriter.getIndexType()
                           : operandType;
-      auto constOne = rewriter.create<emitc::ConstantOp>(
+      auto constOne = emitc::ConstantOp::create(rewriter,
           op.getLoc(), operandType, rewriter.getOneAttr(attrType));
-      auto oneAndOperand = rewriter.create<emitc::BitwiseAndOp>(
+      auto oneAndOperand = emitc::BitwiseAndOp::create(rewriter,
           op.getLoc(), operandType, adaptor.getIn(), constOne);
       rewriter.replaceOpWithNewOp<emitc::CastOp>(op, opReturnType,
                                                  oneAndOperand);
@@ -467,7 +467,7 @@ class BinaryUIOpConversion final : public OpConversionPattern<ArithOp> {
     Value rhsAdapted = adaptValueType(uiBinOp.getRhs(), rewriter, unsignedType);
 
     auto newDivOp =
-        rewriter.create<EmitCOp>(uiBinOp.getLoc(), unsignedType,
+        EmitCOp::create(rewriter, uiBinOp.getLoc(), unsignedType,
                                  ArrayRef<Value>{lhsAdapted, rhsAdapted});
     Value resultAdapted = adaptValueType(newDivOp, rewriter, newRetTy);
     rewriter.replaceOp(uiBinOp, resultAdapted);
@@ -588,38 +588,38 @@ class ShiftOpConversion : public OpConversionPattern<ArithOp> {
     // Add a runtime check for overflow
     Value width;
     if (emitc::isPointerWideType(type)) {
-      Value eight = rewriter.create<emitc::ConstantOp>(
+      Value eight = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rhsType, rewriter.getIndexAttr(8));
-      emitc::CallOpaqueOp sizeOfCall = rewriter.create<emitc::CallOpaqueOp>(
+      emitc::CallOpaqueOp sizeOfCall = emitc::CallOpaqueOp::create(rewriter,
           op.getLoc(), rhsType, "sizeof", ArrayRef<Value>{eight});
-      width = rewriter.create<emitc::MulOp>(op.getLoc(), rhsType, eight,
+      width = emitc::MulOp::create(rewriter, op.getLoc(), rhsType, eight,
                                             sizeOfCall.getResult(0));
     } else {
-      width = rewriter.create<emitc::ConstantOp>(
+      width = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rhsType,
           rewriter.getIntegerAttr(rhsType, type.getIntOrFloatBitWidth()));
     }
 
-    Value excessCheck = rewriter.create<emitc::CmpOp>(
+    Value excessCheck = emitc::CmpOp::create(rewriter,
         op.getLoc(), rewriter.getI1Type(), emitc::CmpPredicate::lt, rhs, width);
 
     // Any concrete value is a valid refinement of poison.
-    Value poison = rewriter.create<emitc::ConstantOp>(
+    Value poison = emitc::ConstantOp::create(rewriter,
         op.getLoc(), arithmeticType,
         (isa<IntegerType>(arithmeticType)
              ? rewriter.getIntegerAttr(arithmeticType, 0)
              : rewriter.getIndexAttr(0)));
 
-    emitc::ExpressionOp ternary = rewriter.create<emitc::ExpressionOp>(
+    emitc::ExpressionOp ternary = emitc::ExpressionOp::create(rewriter,
         op.getLoc(), arithmeticType, /*do_not_inline=*/false);
     Block &bodyBlock = ternary.getBodyRegion().emplaceBlock();
     auto currentPoint = rewriter.getInsertionPoint();
     rewriter.setInsertionPointToStart(&bodyBlock);
     Value arithmeticResult =
-        rewriter.create<EmitCOp>(op.getLoc(), arithmeticType, lhs, rhs);
-    Value resultOrPoison = rewriter.create<emitc::ConditionalOp>(
+        EmitCOp::create(rewriter, op.getLoc(), arithmeticType, lhs, rhs);
+    Value resultOrPoison = emitc::ConditionalOp::create(rewriter,
         op.getLoc(), arithmeticType, excessCheck, arithmeticResult, poison);
-    rewriter.create<emitc::YieldOp>(op.getLoc(), resultOrPoison);
+    emitc::YieldOp::create(rewriter, op.getLoc(), resultOrPoison);
     rewriter.setInsertionPoint(op->getBlock(), currentPoint);
 
     Value result = adaptValueType(ternary, rewriter, type);
@@ -700,11 +700,11 @@ class FtoICastOpConversion : public OpConversionPattern<CastOp> {
                                   /*isSigned=*/false);
     }
 
-    Value result = rewriter.create<emitc::CastOp>(
+    Value result = emitc::CastOp::create(rewriter,
         castOp.getLoc(), actualResultType, adaptor.getOperands());
 
     if (isa<arith::FPToUIOp>(castOp)) {
-      result = rewriter.create<emitc::CastOp>(castOp.getLoc(), dstType, result);
+      result = emitc::CastOp::create(rewriter, castOp.getLoc(), dstType, result);
     }
     rewriter.replaceOp(castOp, result);
 
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index f7bf581adc9e3..f7d4963b5e293 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -293,10 +293,10 @@ LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
       [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
         typename OpTy::Adaptor adaptor(operands);
         if (targetBits < sourceBits) {
-          return rewriter.create<LLVM::TruncOp>(op.getLoc(), llvm1DVectorTy,
+          return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
                                                 adaptor.getIn());
         }
-        return rewriter.create<ExtCastTy>(op.getLoc(), llvm1DVectorTy,
+        return ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
                                           adaptor.getIn());
       },
       rewriter);
@@ -324,12 +324,12 @@ LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
     Type newOverflowType = typeConverter->convertType(overflowResultType);
     Type structType =
         LLVM::LLVMStructType::getLiteral(ctx, {sumResultType, newOverflowType});
-    Value addOverflow = rewriter.create<LLVM::UAddWithOverflowOp>(
+    Value addOverflow = LLVM::UAddWithOverflowOp::create(rewriter,
         loc, structType, adaptor.getLhs(), adaptor.getRhs());
     Value sumExtracted =
-        rewriter.create<LLVM::ExtractValueOp>(loc, addOverflow, 0);
+        LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 0);
     Value overflowExtracted =
-        rewriter.create<LLVM::ExtractValueOp>(loc, addOverflow, 1);
+        LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 1);
     rewriter.replaceOp(op, {sumExtracted, overflowExtracted});
     return success();
   }
@@ -381,15 +381,15 @@ LogicalResult MulIExtendedOpLowering<ArithMulOp, IsSigned>::matchAndRewrite(
            "LLVM dialect should support all signless integer types");
 
     using LLVMExtOp = std::conditional_t<IsSigned, LLVM::SExtOp, LLVM::ZExtOp>;
-    Value lhsExt = rewriter.create<LLVMExtOp>(loc, wideType, adaptor.getLhs());
-    Value rhsExt = rewriter.create<LLVMExtOp>(loc, wideType, adaptor.getRhs());
-    Value mulExt = rewriter.create<LLVM::MulOp>(loc, wideType, lhsExt, rhsExt);
+    Value lhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getLhs());
+    Value rhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getRhs());
+    Value mulExt = LLVM::MulOp::create(rewriter, loc, wideType, lhsExt, rhsExt);
 
     // Split the 2*N-bit wide result into two N-bit values.
-    Value low = rewriter.create<LLVM::TruncOp>(loc, resultType, mulExt);
-    Value shiftVal = rewriter.create<LLVM::ConstantOp>(loc, shiftValAttr);
-    Value highExt = rewriter.create<LLVM::LShrOp>(loc, mulExt, shiftVal);
-    Value high = rewriter.create<LLVM::TruncOp>(loc, resultType, highExt);
+    Value low = LLVM::TruncOp::create(rewriter, loc, resultType, mulExt);
+    Value shiftVal = LLVM::ConstantOp::create(rewriter, loc, shiftValAttr);
+    Value highExt = LLVM::LShrOp::create(rewriter, loc, mulExt, shiftVal);
+    Value high = LLVM::TruncOp::create(rewriter, loc, resultType, highExt);
 
     rewriter.replaceOp(op, {low, high});
     return success();
@@ -435,7 +435,7 @@ CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return rewriter.create<LLVM::ICmpOp>(
+        return LLVM::ICmpOp::create(rewriter,
             op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs());
@@ -471,7 +471,7 @@ CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return rewriter.create<LLVM::FCmpOp>(
+        return LLVM::FCmpOp::create(rewriter,
             op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs(), fmf);
diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index 434d7df853a5e..c93cf13f831ef 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -117,11 +117,11 @@ static Value getScalarOrVectorConstInt(Type type, uint64_t value,
   if (auto vectorType = dyn_cast<VectorType>(type)) {
     Attribute element = IntegerAttr::get(vectorType.getElementType(), value);
     auto attr = SplatElementsAttr::get(vectorType, element);
-    return builder.create<spirv::ConstantOp>(loc, vectorType, attr);
+    return spirv::ConstantOp::create(builder, loc, vectorType, attr);
   }
 
   if (auto intType = dyn_cast<IntegerType>(type))
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, value));
 
   return nullptr;
@@ -418,18 +418,18 @@ static Value emulateSignedRemainder(Location loc, Value lhs, Value rhs,
   Type type = lhs.getType();
 
   // Calculate the remainder with spirv.UMod.
-  Value lhsAbs = builder.create<SignedAbsOp>(loc, type, lhs);
-  Value rhsAbs = builder.create<SignedAbsOp>(loc, type, rhs);
-  Value abs = builder.create<spirv::UModOp>(loc, lhsAbs, rhsAbs);
+  Value lhsAbs = SignedAbsOp::create(builder, loc, type, lhs);
+  Value rhsAbs = SignedAbsOp::create(builder, loc, type, rhs);
+  Value abs = spirv::UModOp::create(builder, loc, lhsAbs, rhsAbs);
 
   // Fix the sign.
   Value isPositive;
   if (lhs == signOperand)
-    isPositive = builder.create<spirv::IEqualOp>(loc, lhs, lhsAbs);
+    isPositive = spirv::IEqualOp::create(builder, loc, lhs, lhsAbs);
   else
-    isPositive = builder.create<spirv::IEqualOp>(loc, rhs, rhsAbs);
-  Value absNegate = builder.create<spirv::SNegateOp>(loc, type, abs);
-  return builder.create<spirv::SelectOp>(loc, type, isPositive, abs, absNegate);
+    isPositive = spirv::IEqualOp::create(builder, loc, rhs, rhsAbs);
+  Value absNegate = spirv::SNegateOp::create(builder, loc, type, abs);
+  return spirv::SelectOp::create(builder, loc, type, isPositive, abs, absNegate);
 }
 
 /// Converts arith.remsi to GLSL SPIR-V ops.
@@ -601,12 +601,12 @@ struct ExtSII1Pattern final : public OpConversionPattern<arith::ExtSIOp> {
     Value allOnes;
     if (auto intTy = dyn_cast<IntegerType>(dstType)) {
       unsigned componentBitwidth = intTy.getWidth();
-      allOnes = rewriter.create<spirv::ConstantOp>(
+      allOnes = spirv::ConstantOp::create(rewriter,
           loc, intTy,
           rewriter.getIntegerAttr(intTy, APInt::getAllOnes(componentBitwidth)));
     } else if (auto vectorTy = dyn_cast<VectorType>(dstType)) {
       unsigned componentBitwidth = vectorTy.getElementTypeBitWidth();
-      allOnes = rewriter.create<spirv::ConstantOp>(
+      allOnes = spirv::ConstantOp::create(rewriter,
           loc, vectorTy,
           SplatElementsAttr::get(vectorTy,
                                  APInt::getAllOnes(componentBitwidth)));
@@ -653,7 +653,7 @@ struct ExtSIPattern final : public OpConversionPattern<arith::ExtSIOp> {
       // First shift left to sequeeze out all leading bits beyond the original
       // bitwidth. Here we need to use the original source and result type's
       // bitwidth.
-      auto shiftLOp = rewriter.create<spirv::ShiftLeftLogicalOp>(
+      auto shiftLOp = spirv::ShiftLeftLogicalOp::create(rewriter,
           op.getLoc(), dstType, adaptor.getIn(), shiftSize);
 
       // Then we perform arithmetic right shift to make sure we have the right
@@ -757,9 +757,9 @@ struct TruncII1Pattern final : public OpConversionPattern<arith::TruncIOp> {
     auto srcType = adaptor.getOperands().front().getType();
     // Check if (x & 1) == 1.
     Value mask = spirv::ConstantOp::getOne(srcType, loc, rewriter);
-    Value maskedSrc = rewriter.create<spirv::BitwiseAndOp>(
+    Value maskedSrc = spirv::BitwiseAndOp::create(rewriter,
         loc, srcType, adaptor.getOperands()[0], mask);
-    Value isOne = rewriter.create<spirv::IEqualOp>(loc, maskedSrc, mask);
+    Value isOne = spirv::IEqualOp::create(rewriter, loc, maskedSrc, mask);
 
     Value zero = spirv::ConstantOp::getZero(dstType, loc, rewriter);
     Value one = spirv::ConstantOp::getOne(dstType, loc, rewriter);
@@ -914,9 +914,9 @@ class CmpIOpBooleanPattern final : public OpConversionPattern<arith::CmpIOp> {
       if (auto vectorType = dyn_cast<VectorType>(dstType))
         type = VectorType::get(vectorType.getShape(), type);
       Value extLhs =
-          rewriter.create<arith::ExtUIOp>(op.getLoc(), type, adaptor.getLhs());
+          arith::ExtUIOp::create(rewriter, op.getLoc(), type, adaptor.getLhs());
       Value extRhs =
-          rewriter.create<arith::ExtUIOp>(op.getLoc(), type, adaptor.getRhs());
+          arith::ExtUIOp::create(rewriter, op.getLoc(), type, adaptor.getRhs());
 
       rewriter.replaceOpWithNewOp<arith::CmpIOp>(op, op.getPredicate(), extLhs,
                                                  extRhs);
@@ -1067,12 +1067,12 @@ class CmpFOpNanNonePattern final : public OpConversionPattern<arith::CmpFOp> {
         replace = spirv::ConstantOp::getZero(op.getType(), loc, rewriter);
       }
     } else {
-      Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-      Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+      Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+      Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-      replace = rewriter.create<spirv::LogicalOrOp>(loc, lhsIsNan, rhsIsNan);
+      replace = spirv::LogicalOrOp::create(rewriter, loc, lhsIsNan, rhsIsNan);
       if (op.getPredicate() == arith::CmpFPredicate::ORD)
-        replace = rewriter.create<spirv::LogicalNotOp>(loc, replace);
+        replace = spirv::LogicalNotOp::create(rewriter, loc, replace);
     }
 
     rewriter.replaceOp(op, replace);
@@ -1094,17 +1094,17 @@ class AddUIExtendedOpPattern final
                   ConversionPatternRewriter &rewriter) const override {
     Type dstElemTy = adaptor.getLhs().getType();
     Location loc = op->getLoc();
-    Value result = rewriter.create<spirv::IAddCarryOp>(loc, adaptor.getLhs(),
+    Value result = spirv::IAddCarryOp::create(rewriter, loc, adaptor.getLhs(),
                                                        adaptor.getRhs());
 
-    Value sumResult = rewriter.create<spirv::CompositeExtractOp>(
+    Value sumResult = spirv::CompositeExtractOp::create(rewriter,
         loc, result, llvm::ArrayRef(0));
-    Value carryValue = rewriter.create<spirv::CompositeExtractOp>(
+    Value carryValue = spirv::CompositeExtractOp::create(rewriter,
         loc, result, llvm::ArrayRef(1));
 
     // Convert the carry value to boolean.
     Value one = spirv::ConstantOp::getOne(dstElemTy, loc, rewriter);
-    Value carryResult = rewriter.create<spirv::IEqualOp>(loc, carryValue, one);
+    Value carryResult = spirv::IEqualOp::create(rewriter, loc, carryValue, one);
 
     rewriter.replaceOp(op, {sumResult, carryResult});
     return success();
@@ -1125,11 +1125,11 @@ class MulIExtendedOpPattern final : public OpConversionPattern<ArithMulOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op->getLoc();
     Value result =
-        rewriter.create<SPIRVMulOp>(loc, adaptor.getLhs(), adaptor.getRhs());
+        SPIRVMulOp::create(rewriter, loc, adaptor.getLhs(), adaptor.getRhs());
 
-    Value low = rewriter.create<spirv::CompositeExtractOp>(loc, result,
+    Value low = spirv::CompositeExtractOp::create(rewriter, loc, result,
                                                            llvm::ArrayRef(0));
-    Value high = rewriter.create<spirv::CompositeExtractOp>(loc, result,
+    Value high = spirv::CompositeExtractOp::create(rewriter, loc, result,
                                                             llvm::ArrayRef(1));
 
     rewriter.replaceOp(op, {low, high});
@@ -1183,19 +1183,19 @@ class MinimumMaximumFOpPattern final : public OpConversionPattern<Op> {
 
     Location loc = op.getLoc();
     Value spirvOp =
-        rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
+        SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
 
     if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
       rewriter.replaceOp(op, spirvOp);
       return success();
     }
 
-    Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-    Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+    Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+    Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-    Value select1 = rewriter.create<spirv::SelectOp>(loc, dstType, lhsIsNan,
+    Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
                                                      adaptor.getLhs(), spirvOp);
-    Value select2 = rewriter.create<spirv::SelectOp>(loc, dstType, rhsIsNan,
+    Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
                                                      adaptor.getRhs(), select1);
 
     rewriter.replaceOp(op, select2);
@@ -1237,7 +1237,7 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
 
     Location loc = op.getLoc();
     Value spirvOp =
-        rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
+        SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
 
     if (!shouldInsertNanGuards<SPIRVOp>() ||
         bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
@@ -1245,12 +1245,12 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
       return success();
     }
 
-    Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-    Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+    Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+    Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-    Value select1 = rewriter.create<spirv::SelectOp>(loc, dstType, lhsIsNan,
+    Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
                                                      adaptor.getRhs(), spirvOp);
-    Value select2 = rewriter.create<spirv::SelectOp>(loc, dstType, rhsIsNan,
+    Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
                                                      adaptor.getLhs(), select1);
 
     rewriter.replaceOp(op, select2);
diff --git a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
index 9c6de938a7108..2295d661d44a5 100644
--- a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
+++ b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
@@ -41,10 +41,10 @@ class Sdot2dLoweringPattern : public OpRewritePattern<Sdot2dOp> {
     Value c2d = op.getC();
     Location loc = op.getLoc();
     Value b1d =
-        rewriter.create<vector::ShapeCastOp>(loc, flattenedVectorType, b2d);
+        vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, b2d);
     Value c1d =
-        rewriter.create<vector::ShapeCastOp>(loc, flattenedVectorType, c2d);
-    Value newOp = rewriter.create<SdotOp>(loc, op.getRes().getType(), op.getA(),
+        vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, c2d);
+    Value newOp = SdotOp::create(rewriter, loc, op.getRes().getType(), op.getA(),
                                           b1d, c1d);
     rewriter.replaceOp(op, {newOp});
     return success();
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 21ea444e31821..cf46e0c10df09 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -45,37 +45,37 @@ static Operation *createLoadTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_ld1b_horiz>(
+      return arm_sme::aarch64_sme_ld1b_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_ld1h_horiz>(
+      return arm_sme::aarch64_sme_ld1h_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_ld1w_horiz>(
+      return arm_sme::aarch64_sme_ld1w_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_ld1d_horiz>(
+      return arm_sme::aarch64_sme_ld1d_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_ld1q_horiz>(
+      return arm_sme::aarch64_sme_ld1q_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_ld1b_vert>(
+      return arm_sme::aarch64_sme_ld1b_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_ld1h_vert>(
+      return arm_sme::aarch64_sme_ld1h_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_ld1w_vert>(
+      return arm_sme::aarch64_sme_ld1w_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_ld1d_vert>(
+      return arm_sme::aarch64_sme_ld1d_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_ld1q_vert>(
+      return arm_sme::aarch64_sme_ld1q_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
       break;
     }
@@ -91,37 +91,37 @@ static Operation *createStoreTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_st1b_horiz>(
+      return arm_sme::aarch64_sme_st1b_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_st1h_horiz>(
+      return arm_sme::aarch64_sme_st1h_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_st1w_horiz>(
+      return arm_sme::aarch64_sme_st1w_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_st1d_horiz>(
+      return arm_sme::aarch64_sme_st1d_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_st1q_horiz>(
+      return arm_sme::aarch64_sme_st1q_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_st1b_vert>(
+      return arm_sme::aarch64_sme_st1b_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_st1h_vert>(
+      return arm_sme::aarch64_sme_st1h_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_st1w_vert>(
+      return arm_sme::aarch64_sme_st1w_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_st1d_vert>(
+      return arm_sme::aarch64_sme_st1d_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_st1q_vert>(
+      return arm_sme::aarch64_sme_st1q_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   }
@@ -146,15 +146,15 @@ createAllocaForTile(RewriterBase &rewriter, Location loc,
   // Move to the first operation in the function.
   rewriter.setInsertionPointToStart(&func.getBlocks().front());
   // Create an alloca matching the tile size of the `tileOp`.
-  auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
+  auto vscale = vector::VectorScaleOp::create(rewriter, loc);
   auto tileElementType = tileOp.getTileType().getElementType();
   auto memrefType = MemRefType::get(
       {ShapedType::kDynamic, ShapedType::kDynamic}, tileElementType);
   unsigned minElements = arm_sme::getSMETileSliceMinNumElts(tileElementType);
   auto minElementsOp =
-      rewriter.create<arith::ConstantIndexOp>(loc, minElements);
-  auto vectorLen = rewriter.create<arith::MulIOp>(loc, vscale, minElementsOp);
-  auto alloca = rewriter.create<memref::AllocaOp>(
+      arith::ConstantIndexOp::create(rewriter, loc, minElements);
+  auto vectorLen = arith::MulIOp::create(rewriter, loc, vscale, minElementsOp);
+  auto alloca = memref::AllocaOp::create(rewriter,
       loc, memrefType, ValueRange{vectorLen, vectorLen});
   return alloca;
 }
@@ -293,9 +293,9 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                                 Value tileMemory, Value sliceIndex) const {
     auto llvmType = getTypeConverter()->convertType(tileMemory.getType());
     auto descriptor =
-        rewriter.create<UnrealizedConversionCastOp>(loc, llvmType, tileMemory);
-    auto zero = rewriter.create<arith::ConstantIntOp>(loc, 0, /*width=*/64);
-    auto sliceIndexI64 = rewriter.create<arith::IndexCastOp>(
+        UnrealizedConversionCastOp::create(rewriter, loc, llvmType, tileMemory);
+    auto zero = arith::ConstantIntOp::create(rewriter, loc, 0, /*width=*/64);
+    auto sliceIndexI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), sliceIndex);
     return getStridedElementPtr(
         static_cast<ConversionPatternRewriter &>(rewriter), loc,
@@ -309,27 +309,27 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                      arm_sme::ArmSMETileType tileType, VectorType sliceType,
                      IntegerAttr tileId, Value sliceIndex) const {
     // Cast the slice index to an i32.
-    auto sliceIndexI32 = rewriter.create<arith::IndexCastOp>(
+    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), sliceIndex);
     // Create an all-true predicate for the slice.
     auto predicateType = sliceType.clone(rewriter.getI1Type());
-    auto allTruePredicate = rewriter.create<arith::ConstantOp>(
+    auto allTruePredicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
     // Create padding vector (never used due to all-true predicate).
-    auto padVector = rewriter.create<LLVM::PoisonOp>(loc, sliceType);
+    auto padVector = LLVM::PoisonOp::create(rewriter, loc, sliceType);
     // Get a pointer to the current slice.
     auto slicePtr =
         getInMemoryTileSlicePtr(rewriter, loc, tileAlloca, sliceIndex);
     // Read the value of the current slice from ZA.
-    auto currentTileSlice = rewriter.create<arm_sme::aarch64_sme_read_horiz>(
+    auto currentTileSlice = arm_sme::aarch64_sme_read_horiz::create(rewriter,
         loc, sliceType, padVector, allTruePredicate, tileId, sliceIndexI32);
     // Load the new tile slice back from memory into ZA.
     createLoadTileSliceIntrinsic(
         rewriter, loc, tileType, arm_sme::TileSliceLayout::Horizontal,
         allTruePredicate, slicePtr, tileId, sliceIndexI32);
     // Store the current tile slice to memory.
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    rewriter.create<vector::StoreOp>(loc, currentTileSlice, tileAlloca,
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    vector::StoreOp::create(rewriter, loc, currentTileSlice, tileAlloca,
                                      ValueRange{sliceIndex, zero});
   }
 
@@ -341,12 +341,12 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     RewriterBase::InsertionGuard guard(rewriter);
     // Create an scf.for over all tile slices.
     auto minNumElts =
-        rewriter.create<arith::ConstantIndexOp>(loc, sliceType.getDimSize(0));
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto upperBound = rewriter.create<arith::MulIOp>(
-        loc, minNumElts, rewriter.create<vector::VectorScaleOp>(loc));
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        arith::ConstantIndexOp::create(rewriter, loc, sliceType.getDimSize(0));
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto upperBound = arith::MulIOp::create(rewriter,
+        loc, minNumElts, vector::VectorScaleOp::create(rewriter, loc));
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     // Emit a swap for each tile slice.
     rewriter.setInsertionPointToStart(forOp.getBody());
     auto sliceIndex = forOp.getInductionVar();
@@ -479,7 +479,7 @@ struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
     //
     // This holds for all tile sizes.
     int32_t zeroMask = baseMaskForSize << int32_t(tileId.getInt());
-    rewriter.create<arm_sme::aarch64_sme_zero>(
+    arm_sme::aarch64_sme_zero::create(rewriter,
         loc, rewriter.getI32IntegerAttr(zeroMask));
 
     // Create a placeholder op to preserve dataflow.
@@ -513,7 +513,7 @@ struct LoadTileSliceConversion
     auto tileSlice = loadTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
@@ -559,7 +559,7 @@ struct StoreTileSliceConversion
     auto tileSlice = storeTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     auto maskOp = storeTileSliceOp.getMask();
@@ -595,26 +595,26 @@ struct InsertTileSliceConversion
     auto tileSlice = insertTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
-    auto one = rewriter.create<arith::ConstantOp>(
+    auto one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI1Type(),
         rewriter.getIntegerAttr(rewriter.getI1Type(), 1));
     auto predTy = VectorType::get(tileType.getShape()[0], rewriter.getI1Type(),
                                   /*scalableDims=*/{true});
-    auto allActiveMask = rewriter.create<vector::SplatOp>(loc, predTy, one);
+    auto allActiveMask = vector::SplatOp::create(rewriter, loc, predTy, one);
 
     // Create 'arm_sme.intr.write.(horiz|vert)' to write vector to tile slice.
     switch (insertTileSliceOp.getLayout()) {
     case arm_sme::TileSliceLayout::Horizontal:
-      rewriter.create<arm_sme::aarch64_sme_write_horiz>(
+      arm_sme::aarch64_sme_write_horiz::create(rewriter,
           loc, tileId, tileSliceI32, allActiveMask,
           insertTileSliceOp.getVector());
       break;
     case arm_sme::TileSliceLayout::Vertical:
-      rewriter.create<arm_sme::aarch64_sme_write_vert>(
+      arm_sme::aarch64_sme_write_vert::create(rewriter,
           loc, tileId, tileSliceI32, allActiveMask,
           insertTileSliceOp.getVector());
       break;
@@ -646,15 +646,15 @@ struct ExtractTileSliceConversion
 
     // Create an 'all true' predicate for the tile slice.
     auto predicateType = sliceType.cloneWith({}, rewriter.getI1Type());
-    auto allTruePredicate = rewriter.create<arith::ConstantOp>(
+    auto allTruePredicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
 
     // Zero destination/fallback for tile slice extraction.
-    auto zeroVector = rewriter.create<arith::ConstantOp>(
+    auto zeroVector = arith::ConstantOp::create(rewriter,
         loc, sliceType, rewriter.getZeroAttr(sliceType));
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto sliceIndexI32 = rewriter.create<arith::IndexCastOp>(
+    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), sliceIndex);
 
     // Create 'arm_sme.intr.read.(horiz|vert)' to extract the tile slice.
@@ -743,7 +743,7 @@ struct OuterProductOpConversion
     Value acc = outerProductOp.getAcc();
     if (!acc) {
       // Initalize accumulator with zero.
-      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, resultVectorType);
+      auto zero = arm_sme::ZeroOp::create(rewriter, loc, resultVectorType);
       zero.setTileId(tileId);
       acc = zero;
     }
@@ -754,14 +754,14 @@ struct OuterProductOpConversion
     if (!lhsMask || !rhsMask) {
       auto predTy =
           outerProductOp.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = rewriter.create<arith::ConstantOp>(
+      Value allActiveMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
     // Create 'arm_sme.intr.mopa' outer product intrinsic.
-    rewriter.create<arm_sme::aarch64_sme_mopa>(loc, tileId, lhsMask, rhsMask,
+    arm_sme::aarch64_sme_mopa::create(rewriter, loc, tileId, lhsMask, rhsMask,
                                                outerProductOp.getLhs(),
                                                outerProductOp.getRhs());
 
@@ -792,7 +792,7 @@ struct OuterProductWideningOpConversion
     Value acc = op.getAcc();
     if (!acc) {
       // Initalize accumulator with zero.
-      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, op.getResultType());
+      auto zero = arm_sme::ZeroOp::create(rewriter, loc, op.getResultType());
       zero.setTileId(tileId);
       acc = zero;
     }
@@ -801,13 +801,13 @@ struct OuterProductWideningOpConversion
     Value rhsMask = op.getRhsMask();
     if (!lhsMask || !rhsMask) {
       auto predTy = op.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = rewriter.create<arith::ConstantOp>(
+      Value allActiveMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
-    rewriter.create<OuterProductWideningIntrOp>(
+    OuterProductWideningIntrOp::create(rewriter,
         loc, tileId, lhsMask, rhsMask, adaptor.getLhs(), adaptor.getRhs());
 
     // The outerproduct intrinsics have no result, replace
@@ -843,13 +843,13 @@ struct StreamingVLOpConversion
     auto *intrOp = [&]() -> Operation * {
       switch (streamingVlOp.getTypeSize()) {
       case arm_sme::TypeSize::Byte:
-        return rewriter.create<arm_sme::aarch64_sme_cntsb>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsb::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Half:
-        return rewriter.create<arm_sme::aarch64_sme_cntsh>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsh::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Word:
-        return rewriter.create<arm_sme::aarch64_sme_cntsw>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsw::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Double:
-        return rewriter.create<arm_sme::aarch64_sme_cntsd>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsd::create(rewriter, loc, i64Type);
       }
       llvm_unreachable("unknown type size in StreamingVLOpConversion");
     }();
@@ -872,7 +872,7 @@ static void mergeConsecutiveTileZerosInBlock(Block *block) {
     if (zeroOpsToMerge.size() <= 1)
       return;
     IRRewriter rewriter(zeroOpsToMerge.front());
-    rewriter.create<arm_sme::aarch64_sme_zero>(
+    arm_sme::aarch64_sme_zero::create(rewriter,
         zeroOpsToMerge.front().getLoc(),
         rewriter.getI32IntegerAttr(mergedZeroMask));
     for (auto zeroOp : zeroOpsToMerge)
diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 458628c29c6ac..3591ed65f8d84 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -39,7 +39,7 @@ SmallVector<Value, 2> getMemrefIndices(ValueRange indices, unsigned rank,
   auto tileSliceOffset = tileSliceIndex;
 
   auto baseIndexPlusTileSliceOffset =
-      rewriter.create<arith::AddIOp>(loc, indices[0], tileSliceOffset);
+      arith::AddIOp::create(rewriter, loc, indices[0], tileSliceOffset);
   outIndices.push_back(baseIndexPlusTileSliceOffset);
   outIndices.push_back(indices[1]);
 
@@ -59,10 +59,10 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   if (memrefIndices.size() != 2)
     return rewriter.notifyMatchFailure(loc, "invalid number of indices");
 
-  auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
       loc, arm_sme::getSMETileSliceMinNumElts(tileType.getElementType()));
   auto vscale =
-      rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
+      vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
   auto predicateType =
       VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
 
@@ -70,7 +70,7 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   // elements in a vector of SVL bits for a given element type (SVL_B,
   // SVL_H, ..., SVL_Q).
   auto numTileSlices =
-      rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
+      arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
 
   Value predicate;
   Value upperBound;
@@ -82,28 +82,28 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
     // The upper bound of the loop must be clamped at `numTileSlices` as
     // `vector.create_mask` allows operands to be greater than the size of a
     // dimension.
-    auto numRowI64 = rewriter.create<arith::IndexCastOp>(
+    auto numRowI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), maskDim0);
-    auto numTileSlicesI64 = rewriter.create<arith::IndexCastOp>(
+    auto numTileSlicesI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), numTileSlices);
     auto upperBoundI64 =
-        rewriter.create<arith::MinSIOp>(loc, numRowI64, numTileSlicesI64);
-    upperBound = rewriter.create<arith::IndexCastOp>(
+        arith::MinSIOp::create(rewriter, loc, numRowI64, numTileSlicesI64);
+    upperBound = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getIndexType(), upperBoundI64);
 
     predicate =
-        rewriter.create<vector::CreateMaskOp>(loc, predicateType, maskDim1);
+        vector::CreateMaskOp::create(rewriter, loc, predicateType, maskDim1);
   } else {
     upperBound = numTileSlices;
     // No mask. Create an 'all true' predicate for the tile slice.
-    predicate = rewriter.create<arith::ConstantOp>(
+    predicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
   }
 
   bool hasCarriedArgs = bool(initTile);
-  auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-  auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+  auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
                                            hasCarriedArgs ? ValueRange{initTile}
                                                           : ValueRange{});
 
@@ -118,7 +118,7 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
 
   assert(bool(nextTile) == hasCarriedArgs);
   if (nextTile)
-    rewriter.create<scf::YieldOp>(loc, nextTile);
+    scf::YieldOp::create(rewriter, loc, nextTile);
 
   return forOp;
 }
@@ -194,9 +194,9 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
       // Initialize tile with zero to satisfy padding. Inactive cols will be
       // zeroed anyway since the loads use zeroing predication. For inactive
       // rows however, no load will occur so these need to be zeroed.
-      initTile = rewriter.create<arm_sme::ZeroOp>(loc, tileType);
+      initTile = arm_sme::ZeroOp::create(rewriter, loc, tileType);
     } else {
-      initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+      initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
     }
 
     // Create a loop to load the active tile slices from memory.
@@ -207,7 +207,7 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
             Value currentTile) -> Value {
           // Create 'arm_sme.load_tile_slice' to load tile slice from memory
           // into tile.
-          return rewriter.create<arm_sme::LoadTileSliceOp>(
+          return arm_sme::LoadTileSliceOp::create(rewriter,
               loc, tileType, tileLoadOp.getBase(), predicate, currentTile,
               memrefIndices, tileSliceIndex, tileLoadOp.getLayout());
         });
@@ -283,21 +283,21 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto numRows = createMaskOp.getOperands()[0];
     auto numCols = createMaskOp.getOperands()[1];
 
-    auto numColsI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto numColsI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), numCols);
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     // Create a loop that loads each ZA tile slice from memory.
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
         loc, arm_sme::getSMETileSliceMinNumElts(tileElementType));
     auto vscale =
-        rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+        vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto numTileSlices =
-        rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, numTileSlices,
+        arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices,
                                              step, ValueRange{initTile});
 
     rewriter.setInsertionPointToStart(forOp.getBody());
@@ -306,16 +306,16 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto currentTile = forOp.getRegionIterArg(0);
 
     // Combine masks.
-    auto rowIsActive = rewriter.create<arith::CmpIOp>(
+    auto rowIsActive = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, tileSliceIndex, numRows);
-    auto rowIsActiveI32 = rewriter.create<arith::ExtSIOp>(
+    auto rowIsActiveI32 = arith::ExtSIOp::create(rewriter,
         loc, rewriter.getI32Type(), rowIsActive);
-    auto mask = rewriter.create<arith::AndIOp>(loc, rowIsActiveI32, numColsI32);
+    auto mask = arith::AndIOp::create(rewriter, loc, rowIsActiveI32, numColsI32);
     auto maskIndex =
-        rewriter.create<arith::IndexCastOp>(loc, rewriter.getIndexType(), mask);
+        arith::IndexCastOp::create(rewriter, loc, rewriter.getIndexType(), mask);
     auto predicateType =
         VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
-    auto maskOp1D = rewriter.create<vector::CreateMaskOp>(
+    auto maskOp1D = vector::CreateMaskOp::create(rewriter,
         loc, predicateType, maskIndex.getResult());
 
     auto memrefIndices = getMemrefIndices(
@@ -324,17 +324,17 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
 
     // Splat pad into 1-D vector matching type of tile slice.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    auto pad1DOp = rewriter.create<vector::SplatOp>(loc, tileSliceType, padOp);
+    auto pad1DOp = vector::SplatOp::create(rewriter, loc, tileSliceType, padOp);
 
-    auto loadSlice = rewriter.create<vector::MaskedLoadOp>(
+    auto loadSlice = vector::MaskedLoadOp::create(rewriter,
         loc, tileSliceType, tileLoadOp.getBase(), memrefIndices, maskOp1D,
         /*passthru=*/pad1DOp);
 
     // Create 'arm_sme.insert_tile_slice' to insert slice into tile.
-    auto insertSlice = rewriter.create<arm_sme::InsertTileSliceOp>(
+    auto insertSlice = arm_sme::InsertTileSliceOp::create(rewriter,
         loc, tileType, loadSlice->getResult(0), currentTile, tileSliceIndex,
         tileLoadOp.getLayout());
-    rewriter.create<scf::YieldOp>(loc, insertSlice.getResult());
+    scf::YieldOp::create(rewriter, loc, insertSlice.getResult());
 
     rewriter.setInsertionPointAfter(forOp);
 
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index 94f7caa315cf7..e1d28f19ee236 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -203,7 +203,7 @@ static void addAsyncRuntimeApiDeclarations(ModuleOp module) {
   auto addFuncDecl = [&](StringRef name, FunctionType type) {
     if (module.lookupSymbol(name))
       return;
-    builder.create<func::FuncOp>(name, type).setPrivate();
+    func::FuncOp::create(builder, name, type).setPrivate();
   };
 
   MLIRContext *ctx = module.getContext();
@@ -254,15 +254,15 @@ static void addResumeFunction(ModuleOp module) {
   auto voidTy = LLVM::LLVMVoidType::get(ctx);
   Type ptrType = AsyncAPI::opaquePointerType(ctx);
 
-  auto resumeOp = moduleBuilder.create<LLVM::LLVMFuncOp>(
+  auto resumeOp = LLVM::LLVMFuncOp::create(moduleBuilder,
       kResume, LLVM::LLVMFunctionType::get(voidTy, {ptrType}));
   resumeOp.setPrivate();
 
   auto *block = resumeOp.addEntryBlock(moduleBuilder);
   auto blockBuilder = ImplicitLocOpBuilder::atBlockEnd(loc, block);
 
-  blockBuilder.create<LLVM::CoroResumeOp>(resumeOp.getArgument(0));
-  blockBuilder.create<LLVM::ReturnOp>(ValueRange());
+  LLVM::CoroResumeOp::create(blockBuilder, resumeOp.getArgument(0));
+  LLVM::ReturnOp::create(blockBuilder, ValueRange());
 }
 
 //===----------------------------------------------------------------------===//
@@ -282,7 +282,7 @@ class AsyncRuntimeTypeConverter : public TypeConverter {
     // in patterns for other dialects.
     auto addUnrealizedCast = [](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+      auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
       return cast.getResult(0);
     };
 
@@ -343,8 +343,8 @@ class CoroIdOpConversion : public AsyncOpConversionPattern<CoroIdOp> {
 
     // Constants for initializing coroutine frame.
     auto constZero =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), 0);
-    auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, ptrType);
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), 0);
+    auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, ptrType);
 
     // Get coroutine id: @llvm.coro.id.
     rewriter.replaceOpWithNewOp<LLVM::CoroIdOp>(
@@ -372,32 +372,32 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
 
     // Get coroutine frame size: @llvm.coro.size.i64.
     Value coroSize =
-        rewriter.create<LLVM::CoroSizeOp>(loc, rewriter.getI64Type());
+        LLVM::CoroSizeOp::create(rewriter, loc, rewriter.getI64Type());
     // Get coroutine frame alignment: @llvm.coro.align.i64.
     Value coroAlign =
-        rewriter.create<LLVM::CoroAlignOp>(loc, rewriter.getI64Type());
+        LLVM::CoroAlignOp::create(rewriter, loc, rewriter.getI64Type());
 
     // Round up the size to be multiple of the alignment. Since aligned_alloc
     // requires the size parameter be an integral multiple of the alignment
     // parameter.
     auto makeConstant = [&](uint64_t c) {
-      return rewriter.create<LLVM::ConstantOp>(op->getLoc(),
+      return LLVM::ConstantOp::create(rewriter, op->getLoc(),
                                                rewriter.getI64Type(), c);
     };
-    coroSize = rewriter.create<LLVM::AddOp>(op->getLoc(), coroSize, coroAlign);
+    coroSize = LLVM::AddOp::create(rewriter, op->getLoc(), coroSize, coroAlign);
     coroSize =
-        rewriter.create<LLVM::SubOp>(op->getLoc(), coroSize, makeConstant(1));
+        LLVM::SubOp::create(rewriter, op->getLoc(), coroSize, makeConstant(1));
     Value negCoroAlign =
-        rewriter.create<LLVM::SubOp>(op->getLoc(), makeConstant(0), coroAlign);
+        LLVM::SubOp::create(rewriter, op->getLoc(), makeConstant(0), coroAlign);
     coroSize =
-        rewriter.create<LLVM::AndOp>(op->getLoc(), coroSize, negCoroAlign);
+        LLVM::AndOp::create(rewriter, op->getLoc(), coroSize, negCoroAlign);
 
     // Allocate memory for the coroutine frame.
     auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn(
         rewriter, op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
     if (failed(allocFuncOp))
       return failure();
-    auto coroAlloc = rewriter.create<LLVM::CallOp>(
+    auto coroAlloc = LLVM::CallOp::create(rewriter,
         loc, allocFuncOp.value(), ValueRange{coroAlign, coroSize});
 
     // Begin a coroutine: @llvm.coro.begin.
@@ -427,7 +427,7 @@ class CoroFreeOpConversion : public AsyncOpConversionPattern<CoroFreeOp> {
 
     // Get a pointer to the coroutine frame memory: @llvm.coro.free.
     auto coroMem =
-        rewriter.create<LLVM::CoroFreeOp>(loc, ptrType, adaptor.getOperands());
+        LLVM::CoroFreeOp::create(rewriter, loc, ptrType, adaptor.getOperands());
 
     // Free the memory.
     auto freeFuncOp =
@@ -455,13 +455,13 @@ class CoroEndOpConversion : public OpConversionPattern<CoroEndOp> {
   matchAndRewrite(CoroEndOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // We are not in the block that is part of the unwind sequence.
-    auto constFalse = rewriter.create<LLVM::ConstantOp>(
+    auto constFalse = LLVM::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getI1Type(), rewriter.getBoolAttr(false));
-    auto noneToken = rewriter.create<LLVM::NoneTokenOp>(op->getLoc());
+    auto noneToken = LLVM::NoneTokenOp::create(rewriter, op->getLoc());
 
     // Mark the end of a coroutine: @llvm.coro.end.
     auto coroHdl = adaptor.getHandle();
-    rewriter.create<LLVM::CoroEndOp>(
+    LLVM::CoroEndOp::create(rewriter,
         op->getLoc(), rewriter.getI1Type(),
         ValueRange({coroHdl, constFalse, noneToken}));
     rewriter.eraseOp(op);
@@ -534,12 +534,12 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     auto loc = op->getLoc();
 
     // This is not a final suspension point.
-    auto constFalse = rewriter.create<LLVM::ConstantOp>(
+    auto constFalse = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI1Type(), rewriter.getBoolAttr(false));
 
     // Suspend a coroutine: @llvm.coro.suspend
     auto coroState = adaptor.getState();
-    auto coroSuspend = rewriter.create<LLVM::CoroSuspendOp>(
+    auto coroSuspend = LLVM::CoroSuspendOp::create(rewriter,
         loc, i8, ValueRange({coroState, constFalse}));
 
     // Cast return code to i32.
@@ -551,7 +551,7 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     llvm::SmallVector<Block *, 2> caseDest = {op.getResumeDest(),
                                               op.getCleanupDest()};
     rewriter.replaceOpWithNewOp<LLVM::SwitchOp>(
-        op, rewriter.create<LLVM::SExtOp>(loc, i32, coroSuspend.getResult()),
+        op, LLVM::SExtOp::create(rewriter, loc, i32, coroSuspend.getResult()),
         /*defaultDestination=*/op.getSuspendDest(),
         /*defaultOperands=*/ValueRange(),
         /*caseValues=*/caseValues,
@@ -602,11 +602,11 @@ class RuntimeCreateOpLowering : public ConvertOpToLLVMPattern<RuntimeCreateOp> {
 
         // %Size = getelementptr %T* null, int 1
         // %SizeI = ptrtoint %T* %Size to i64
-        auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, storagePtrType);
+        auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, storagePtrType);
         auto gep =
-            rewriter.create<LLVM::GEPOp>(loc, storagePtrType, storedType,
+            LLVM::GEPOp::create(rewriter, loc, storagePtrType, storedType,
                                          nullPtr, ArrayRef<LLVM::GEPArg>{1});
-        return rewriter.create<LLVM::PtrToIntOp>(loc, i64, gep);
+        return LLVM::PtrToIntOp::create(rewriter, loc, i64, gep);
       };
 
       rewriter.replaceOpWithNewOp<func::CallOp>(op, kCreateValue, resultType,
@@ -739,7 +739,7 @@ class RuntimeAwaitOpLowering : public OpConversionPattern<RuntimeAwaitOp> {
             .Case<ValueType>([](Type) { return kAwaitValue; })
             .Case<GroupType>([](Type) { return kAwaitGroup; });
 
-    rewriter.create<func::CallOp>(op->getLoc(), apiFuncName, TypeRange(),
+    func::CallOp::create(rewriter, op->getLoc(), apiFuncName, TypeRange(),
                                   adaptor.getOperands());
     rewriter.eraseOp(op);
 
@@ -772,11 +772,11 @@ class RuntimeAwaitAndResumeOpLowering
 
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = rewriter.create<LLVM::AddressOfOp>(
+    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
         op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
         kResume);
 
-    rewriter.create<func::CallOp>(
+    func::CallOp::create(rewriter,
         op->getLoc(), apiFuncName, TypeRange(),
         ValueRange({operand, handle, resumePtr.getRes()}));
     rewriter.eraseOp(op);
@@ -801,7 +801,7 @@ class RuntimeResumeOpLowering
                   ConversionPatternRewriter &rewriter) const override {
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = rewriter.create<LLVM::AddressOfOp>(
+    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
         op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
         kResume);
 
@@ -832,7 +832,7 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = rewriter.create<func::CallOp>(
+    auto storagePtr = func::CallOp::create(rewriter,
         loc, kGetValueStorage, TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
@@ -845,7 +845,7 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     Value castedStoragePtr = storagePtr.getResult(0);
     // Store the yielded value into the async value storage.
     auto value = adaptor.getValue();
-    rewriter.create<LLVM::StoreOp>(loc, value, castedStoragePtr);
+    LLVM::StoreOp::create(rewriter, loc, value, castedStoragePtr);
 
     // Erase the original runtime store operation.
     rewriter.eraseOp(op);
@@ -872,7 +872,7 @@ class RuntimeLoadOpLowering : public ConvertOpToLLVMPattern<RuntimeLoadOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = rewriter.create<func::CallOp>(
+    auto storagePtr = func::CallOp::create(rewriter,
         loc, kGetValueStorage, TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
@@ -960,7 +960,7 @@ class RefCountingOpLowering : public OpConversionPattern<RefCountingOp> {
   LogicalResult
   matchAndRewrite(RefCountingOp op, typename RefCountingOp::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto count = rewriter.create<arith::ConstantOp>(
+    auto count = arith::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getI64Type(),
         rewriter.getI64IntegerAttr(op.getCount()));
 
diff --git a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
index b9991f36cdaaf..a63a1ceb00ebb 100644
--- a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
+++ b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
@@ -47,26 +47,26 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
 
     if (auto unrankedType = dyn_cast<UnrankedMemRefType>(type)) {
       // Constants
-      Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-      Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+      Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+      Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
       // Dynamically evaluate the size and shape of the unranked memref
-      Value rank = rewriter.create<memref::RankOp>(loc, op.getInput());
+      Value rank = memref::RankOp::create(rewriter, loc, op.getInput());
       MemRefType allocType =
           MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType());
-      Value shape = rewriter.create<memref::AllocaOp>(loc, allocType, rank);
+      Value shape = memref::AllocaOp::create(rewriter, loc, allocType, rank);
 
       // Create a loop to query dimension sizes, store them as a shape, and
       // compute the total size of the memref
       auto loopBody = [&](OpBuilder &builder, Location loc, Value i,
                           ValueRange args) {
         auto acc = args.front();
-        auto dim = rewriter.create<memref::DimOp>(loc, op.getInput(), i);
+        auto dim = memref::DimOp::create(rewriter, loc, op.getInput(), i);
 
-        rewriter.create<memref::StoreOp>(loc, dim, shape, i);
-        acc = rewriter.create<arith::MulIOp>(loc, acc, dim);
+        memref::StoreOp::create(rewriter, loc, dim, shape, i);
+        acc = arith::MulIOp::create(rewriter, loc, acc, dim);
 
-        rewriter.create<scf::YieldOp>(loc, acc);
+        scf::YieldOp::create(rewriter, loc, acc);
       };
       auto size = rewriter
                       .create<scf::ForOp>(loc, zero, rank, one, ValueRange(one),
@@ -78,9 +78,9 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
 
       // Allocate new memref with 1D dynamic shape, then reshape into the
       // shape of the original unranked memref
-      alloc = rewriter.create<memref::AllocOp>(loc, memrefType, size);
+      alloc = memref::AllocOp::create(rewriter, loc, memrefType, size);
       alloc =
-          rewriter.create<memref::ReshapeOp>(loc, unrankedType, alloc, shape);
+          memref::ReshapeOp::create(rewriter, loc, unrankedType, alloc, shape);
     } else {
       MemRefType memrefType = cast<MemRefType>(type);
       MemRefLayoutAttrInterface layout;
@@ -103,14 +103,14 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
       }
 
       // Allocate a memref with identity layout.
-      alloc = rewriter.create<memref::AllocOp>(loc, allocType, dynamicOperands);
+      alloc = memref::AllocOp::create(rewriter, loc, allocType, dynamicOperands);
       // Cast the allocation to the specified type if needed.
       if (memrefType != allocType)
         alloc =
-            rewriter.create<memref::CastOp>(op->getLoc(), memrefType, alloc);
+            memref::CastOp::create(rewriter, op->getLoc(), memrefType, alloc);
     }
 
-    rewriter.create<memref::CopyOp>(loc, op.getInput(), alloc);
+    memref::CopyOp::create(rewriter, loc, op.getInput(), alloc);
     rewriter.replaceOp(op, alloc);
     return success();
   }
diff --git a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
index 70b22386f1eea..1961e1004ba8e 100644
--- a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
+++ b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
@@ -23,41 +23,41 @@ void mlir::complex::convertDivToLLVMUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, LLVM::FastmathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, rhsRe, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, rhsIm, rhsIm, fmf), fmf);
+  Value rhsSqNorm = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsIm, fmf), fmf);
+  Value realNumerator = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
 
-  Value imagNumerator = rewriter.create<LLVM::FSubOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsIm, fmf), fmf);
+  Value imagNumerator = LLVM::FSubOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
-  *resultRe = rewriter.create<LLVM::FDivOp>(loc, realNumerator, rhsSqNorm, fmf);
-  *resultIm = rewriter.create<LLVM::FDivOp>(loc, imagNumerator, rhsSqNorm, fmf);
+  *resultRe = LLVM::FDivOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
+  *resultIm = LLVM::FDivOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 void mlir::complex::convertDivToStandardUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, arith::FastMathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, rhsRe, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, rhsIm, rhsIm, fmf), fmf);
+  Value rhsSqNorm = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsIm, fmf), fmf);
-  Value imagNumerator = rewriter.create<arith::SubFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsIm, fmf), fmf);
+  Value realNumerator = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
+  Value imagNumerator = arith::SubFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
   *resultRe =
-      rewriter.create<arith::DivFOp>(loc, realNumerator, rhsSqNorm, fmf);
+      arith::DivFOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
   *resultIm =
-      rewriter.create<arith::DivFOp>(loc, imagNumerator, rhsSqNorm, fmf);
+      arith::DivFOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 // Smith's algorithm to divide complex numbers. It is just a bit smarter
@@ -94,180 +94,180 @@ void mlir::complex::convertDivToLLVMUsingRangeReduction(
   auto elementType = cast<FloatType>(rhsRe.getType());
 
   Value rhsRealImagRatio =
-      rewriter.create<LLVM::FDivOp>(loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = rewriter.create<LLVM::FAddOp>(
+      LLVM::FDivOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
+  Value rhsRealImagDenom = LLVM::FAddOp::create(rewriter,
       loc, rhsIm,
-      rewriter.create<LLVM::FMulOp>(loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRealImagRatio, fmf),
+      LLVM::FMulOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
+  Value realNumerator1 = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
       lhsIm, fmf);
   Value resultReal1 =
-      rewriter.create<LLVM::FDivOp>(loc, realNumerator1, rhsRealImagDenom, fmf);
-  Value imagNumerator1 = rewriter.create<LLVM::FSubOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRealImagRatio, fmf),
+      LLVM::FDivOp::create(rewriter, loc, realNumerator1, rhsRealImagDenom, fmf);
+  Value imagNumerator1 = LLVM::FSubOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
       lhsRe, fmf);
   Value resultImag1 =
-      rewriter.create<LLVM::FDivOp>(loc, imagNumerator1, rhsRealImagDenom, fmf);
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator1, rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
-      rewriter.create<LLVM::FDivOp>(loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = rewriter.create<LLVM::FAddOp>(
+      LLVM::FDivOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
+  Value rhsImagRealDenom = LLVM::FAddOp::create(rewriter,
       loc, rhsRe,
-      rewriter.create<LLVM::FMulOp>(loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = rewriter.create<LLVM::FAddOp>(
+      LLVM::FMulOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
+  Value realNumerator2 = LLVM::FAddOp::create(rewriter,
       loc, lhsRe,
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsImagRealRatio, fmf), fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
   Value resultReal2 =
-      rewriter.create<LLVM::FDivOp>(loc, realNumerator2, rhsImagRealDenom, fmf);
-  Value imagNumerator2 = rewriter.create<LLVM::FSubOp>(
+      LLVM::FDivOp::create(rewriter, loc, realNumerator2, rhsImagRealDenom, fmf);
+  Value imagNumerator2 = LLVM::FSubOp::create(rewriter,
       loc, lhsIm,
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsImagRealRatio, fmf), fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
   Value resultImag2 =
-      rewriter.create<LLVM::FDivOp>(loc, imagNumerator2, rhsImagRealDenom, fmf);
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator2, rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = rewriter.create<LLVM::ConstantOp>(
+  Value zero = LLVM::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
-  Value rhsRealAbs = rewriter.create<LLVM::FAbsOp>(loc, rhsRe, fmf);
-  Value rhsRealIsZero = rewriter.create<LLVM::FCmpOp>(
+  Value rhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, rhsRe, fmf);
+  Value rhsRealIsZero = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, zero);
-  Value rhsImagAbs = rewriter.create<LLVM::FAbsOp>(loc, rhsIm, fmf);
-  Value rhsImagIsZero = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, rhsIm, fmf);
+  Value rhsImagIsZero = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, zero);
   Value lhsRealIsNotNaN =
-      rewriter.create<LLVM::FCmpOp>(loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
+      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
   Value lhsImagIsNotNaN =
-      rewriter.create<LLVM::FCmpOp>(loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
+      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
   Value lhsContainsNotNaNValue =
-      rewriter.create<LLVM::OrOp>(loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = rewriter.create<LLVM::AndOp>(
+      LLVM::OrOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
+  Value resultIsInfinity = LLVM::AndOp::create(rewriter,
       loc, lhsContainsNotNaNValue,
-      rewriter.create<LLVM::AndOp>(loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = rewriter.create<LLVM::ConstantOp>(
+      LLVM::AndOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
+  Value inf = LLVM::ConstantOp::create(rewriter,
       loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfrhsReal =
-      rewriter.create<LLVM::CopySignOp>(loc, inf, rhsRe);
+      LLVM::CopySignOp::create(rewriter, loc, inf, rhsRe);
   Value infinityResultReal =
-      rewriter.create<LLVM::FMulOp>(loc, infWithSignOfrhsReal, lhsRe, fmf);
+      LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsRe, fmf);
   Value infinityResultImag =
-      rewriter.create<LLVM::FMulOp>(loc, infWithSignOfrhsReal, lhsIm, fmf);
+      LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsRealFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, rhsRealAbs, inf);
-  Value rhsImagFinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, rhsImagAbs, inf);
   Value rhsFinite =
-      rewriter.create<LLVM::AndOp>(loc, rhsRealFinite, rhsImagFinite);
-  Value lhsRealAbs = rewriter.create<LLVM::FAbsOp>(loc, lhsRe, fmf);
-  Value lhsRealInfinite = rewriter.create<LLVM::FCmpOp>(
+      LLVM::AndOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
+  Value lhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, lhsRe, fmf);
+  Value lhsRealInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, lhsRealAbs, inf);
-  Value lhsImagAbs = rewriter.create<LLVM::FAbsOp>(loc, lhsIm, fmf);
-  Value lhsImagInfinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, lhsIm, fmf);
+  Value lhsImagInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, lhsImagAbs, inf);
   Value lhsInfinite =
-      rewriter.create<LLVM::OrOp>(loc, lhsRealInfinite, lhsImagInfinite);
+      LLVM::OrOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
-      rewriter.create<LLVM::AndOp>(loc, lhsInfinite, rhsFinite);
-  Value one = rewriter.create<LLVM::ConstantOp>(
+      LLVM::AndOp::create(rewriter, loc, lhsInfinite, rhsFinite);
+  Value one = LLVM::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, lhsRealInfinite, one, zero),
+  Value lhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesrhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRealIsInfWithSign, rhsRe, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesrhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
+  Value resultReal3 = LLVM::FMulOp::create(rewriter,
       loc, inf,
-      rewriter.create<LLVM::FAddOp>(loc, lhsRealIsInfWithSignTimesrhsReal,
+      LLVM::FAddOp::create(rewriter, loc, lhsRealIsInfWithSignTimesrhsReal,
                                     lhsImagIsInfWithSignTimesrhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesrhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRealIsInfWithSign, rhsIm, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesrhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
+  Value resultImag3 = LLVM::FMulOp::create(rewriter,
       loc, inf,
-      rewriter.create<LLVM::FSubOp>(loc, lhsImagIsInfWithSignTimesrhsReal,
+      LLVM::FSubOp::create(rewriter, loc, lhsImagIsInfWithSignTimesrhsReal,
                                     lhsRealIsInfWithSignTimesrhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsRealFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, lhsRealAbs, inf);
-  Value lhsImagFinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsImagFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, lhsImagAbs, inf);
   Value lhsFinite =
-      rewriter.create<LLVM::AndOp>(loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = rewriter.create<LLVM::FCmpOp>(
+      LLVM::AndOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
+  Value rhsRealInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, inf);
-  Value rhsImagInfinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, inf);
   Value rhsInfinite =
-      rewriter.create<LLVM::OrOp>(loc, rhsRealInfinite, rhsImagInfinite);
+      LLVM::OrOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
-      rewriter.create<LLVM::AndOp>(loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, rhsRealInfinite, one, zero),
+      LLVM::AndOp::create(rewriter, loc, lhsFinite, rhsInfinite);
+  Value rhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimeslhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRealIsInfWithSign, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
+  Value resultReal4 = LLVM::FMulOp::create(rewriter,
       loc, zero,
-      rewriter.create<LLVM::FAddOp>(loc, rhsRealIsInfWithSignTimeslhsReal,
+      LLVM::FAddOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsReal,
                                     rhsImagIsInfWithSignTimeslhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimeslhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRealIsInfWithSign, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
+  Value resultImag4 = LLVM::FMulOp::create(rewriter,
       loc, zero,
-      rewriter.create<LLVM::FSubOp>(loc, rhsRealIsInfWithSignTimeslhsImag,
+      LLVM::FSubOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsImag,
                                     rhsImagIsInfWithSignTimeslhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = rewriter.create<LLVM::FCmpOp>(
+  Value realAbsSmallerThanImagAbs = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::olt, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = rewriter.create<LLVM::SelectOp>(
+  Value resultReal5 = LLVM::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = rewriter.create<LLVM::SelectOp>(
+  Value resultImag5 = LLVM::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase3 = LLVM::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase3 = LLVM::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase2 = LLVM::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase2 = LLVM::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase1 = LLVM::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase1 = LLVM::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = rewriter.create<LLVM::FCmpOp>(
+  Value resultRealIsNaN = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::uno, resultReal5, zero);
-  Value resultImagIsNaN = rewriter.create<LLVM::FCmpOp>(
+  Value resultImagIsNaN = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::uno, resultImag5, zero);
   Value resultIsNaN =
-      rewriter.create<LLVM::AndOp>(loc, resultRealIsNaN, resultImagIsNaN);
+      LLVM::AndOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = rewriter.create<LLVM::SelectOp>(
+  *resultRe = LLVM::SelectOp::create(rewriter,
       loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = rewriter.create<LLVM::SelectOp>(
+  *resultIm = LLVM::SelectOp::create(rewriter,
       loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
 }
 
@@ -278,179 +278,179 @@ void mlir::complex::convertDivToStandardUsingRangeReduction(
   auto elementType = cast<FloatType>(rhsRe.getType());
 
   Value rhsRealImagRatio =
-      rewriter.create<arith::DivFOp>(loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = rewriter.create<arith::AddFOp>(
+      arith::DivFOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
+  Value rhsRealImagDenom = arith::AddFOp::create(rewriter,
       loc, rhsIm,
-      rewriter.create<arith::MulFOp>(loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRealImagRatio, fmf),
+      arith::MulFOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
+  Value realNumerator1 = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
       lhsIm, fmf);
-  Value resultReal1 = rewriter.create<arith::DivFOp>(loc, realNumerator1,
+  Value resultReal1 = arith::DivFOp::create(rewriter, loc, realNumerator1,
                                                      rhsRealImagDenom, fmf);
-  Value imagNumerator1 = rewriter.create<arith::SubFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRealImagRatio, fmf),
+  Value imagNumerator1 = arith::SubFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
       lhsRe, fmf);
-  Value resultImag1 = rewriter.create<arith::DivFOp>(loc, imagNumerator1,
+  Value resultImag1 = arith::DivFOp::create(rewriter, loc, imagNumerator1,
                                                      rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
-      rewriter.create<arith::DivFOp>(loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = rewriter.create<arith::AddFOp>(
+      arith::DivFOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
+  Value rhsImagRealDenom = arith::AddFOp::create(rewriter,
       loc, rhsRe,
-      rewriter.create<arith::MulFOp>(loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = rewriter.create<arith::AddFOp>(
+      arith::MulFOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
+  Value realNumerator2 = arith::AddFOp::create(rewriter,
       loc, lhsRe,
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsImagRealRatio, fmf), fmf);
-  Value resultReal2 = rewriter.create<arith::DivFOp>(loc, realNumerator2,
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
+  Value resultReal2 = arith::DivFOp::create(rewriter, loc, realNumerator2,
                                                      rhsImagRealDenom, fmf);
-  Value imagNumerator2 = rewriter.create<arith::SubFOp>(
+  Value imagNumerator2 = arith::SubFOp::create(rewriter,
       loc, lhsIm,
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsImagRealRatio, fmf), fmf);
-  Value resultImag2 = rewriter.create<arith::DivFOp>(loc, imagNumerator2,
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
+  Value resultImag2 = arith::DivFOp::create(rewriter, loc, imagNumerator2,
                                                      rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
-  Value rhsRealAbs = rewriter.create<math::AbsFOp>(loc, rhsRe, fmf);
-  Value rhsRealIsZero = rewriter.create<arith::CmpFOp>(
+  Value rhsRealAbs = math::AbsFOp::create(rewriter, loc, rhsRe, fmf);
+  Value rhsRealIsZero = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsRealAbs, zero);
-  Value rhsImagAbs = rewriter.create<math::AbsFOp>(loc, rhsIm, fmf);
-  Value rhsImagIsZero = rewriter.create<arith::CmpFOp>(
+  Value rhsImagAbs = math::AbsFOp::create(rewriter, loc, rhsIm, fmf);
+  Value rhsImagIsZero = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsImagAbs, zero);
-  Value lhsRealIsNotNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsRealIsNotNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ORD, lhsRe, zero);
-  Value lhsImagIsNotNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsImagIsNotNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ORD, lhsIm, zero);
   Value lhsContainsNotNaNValue =
-      rewriter.create<arith::OrIOp>(loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = rewriter.create<arith::AndIOp>(
+      arith::OrIOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
+  Value resultIsInfinity = arith::AndIOp::create(rewriter,
       loc, lhsContainsNotNaNValue,
-      rewriter.create<arith::AndIOp>(loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = rewriter.create<arith::ConstantOp>(
+      arith::AndIOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
+  Value inf = arith::ConstantOp::create(rewriter,
       loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfRhsReal =
-      rewriter.create<math::CopySignOp>(loc, inf, rhsRe);
+      math::CopySignOp::create(rewriter, loc, inf, rhsRe);
   Value infinityResultReal =
-      rewriter.create<arith::MulFOp>(loc, infWithSignOfRhsReal, lhsRe, fmf);
+      arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsRe, fmf);
   Value infinityResultImag =
-      rewriter.create<arith::MulFOp>(loc, infWithSignOfRhsReal, lhsIm, fmf);
+      arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = rewriter.create<arith::CmpFOp>(
+  Value rhsRealFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, rhsRealAbs, inf);
-  Value rhsImagFinite = rewriter.create<arith::CmpFOp>(
+  Value rhsImagFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, rhsImagAbs, inf);
   Value rhsFinite =
-      rewriter.create<arith::AndIOp>(loc, rhsRealFinite, rhsImagFinite);
-  Value lhsRealAbs = rewriter.create<math::AbsFOp>(loc, lhsRe, fmf);
-  Value lhsRealInfinite = rewriter.create<arith::CmpFOp>(
+      arith::AndIOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
+  Value lhsRealAbs = math::AbsFOp::create(rewriter, loc, lhsRe, fmf);
+  Value lhsRealInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, lhsRealAbs, inf);
-  Value lhsImagAbs = rewriter.create<math::AbsFOp>(loc, lhsIm, fmf);
-  Value lhsImagInfinite = rewriter.create<arith::CmpFOp>(
+  Value lhsImagAbs = math::AbsFOp::create(rewriter, loc, lhsIm, fmf);
+  Value lhsImagInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, lhsImagAbs, inf);
   Value lhsInfinite =
-      rewriter.create<arith::OrIOp>(loc, lhsRealInfinite, lhsImagInfinite);
+      arith::OrIOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
-      rewriter.create<arith::AndIOp>(loc, lhsInfinite, rhsFinite);
-  Value one = rewriter.create<arith::ConstantOp>(
+      arith::AndIOp::create(rewriter, loc, lhsInfinite, rhsFinite);
+  Value one = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, lhsRealInfinite, one, zero),
+  Value lhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesRhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRealIsInfWithSign, rhsRe, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesRhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
+  Value resultReal3 = arith::MulFOp::create(rewriter,
       loc, inf,
-      rewriter.create<arith::AddFOp>(loc, lhsRealIsInfWithSignTimesRhsReal,
+      arith::AddFOp::create(rewriter, loc, lhsRealIsInfWithSignTimesRhsReal,
                                      lhsImagIsInfWithSignTimesRhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesRhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsRealIsInfWithSign, rhsIm, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesRhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
+  Value resultImag3 = arith::MulFOp::create(rewriter,
       loc, inf,
-      rewriter.create<arith::SubFOp>(loc, lhsImagIsInfWithSignTimesRhsReal,
+      arith::SubFOp::create(rewriter, loc, lhsImagIsInfWithSignTimesRhsReal,
                                      lhsRealIsInfWithSignTimesRhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = rewriter.create<arith::CmpFOp>(
+  Value lhsRealFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, lhsRealAbs, inf);
-  Value lhsImagFinite = rewriter.create<arith::CmpFOp>(
+  Value lhsImagFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, lhsImagAbs, inf);
   Value lhsFinite =
-      rewriter.create<arith::AndIOp>(loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = rewriter.create<arith::CmpFOp>(
+      arith::AndIOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
+  Value rhsRealInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsRealAbs, inf);
-  Value rhsImagInfinite = rewriter.create<arith::CmpFOp>(
+  Value rhsImagInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsImagAbs, inf);
   Value rhsInfinite =
-      rewriter.create<arith::OrIOp>(loc, rhsRealInfinite, rhsImagInfinite);
+      arith::OrIOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
-      rewriter.create<arith::AndIOp>(loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, rhsRealInfinite, one, zero),
+      arith::AndIOp::create(rewriter, loc, lhsFinite, rhsInfinite);
+  Value rhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimesLhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRealIsInfWithSign, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
+  Value resultReal4 = arith::MulFOp::create(rewriter,
       loc, zero,
-      rewriter.create<arith::AddFOp>(loc, rhsRealIsInfWithSignTimesLhsReal,
+      arith::AddFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsReal,
                                      rhsImagIsInfWithSignTimesLhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimesLhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRealIsInfWithSign, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
+  Value resultImag4 = arith::MulFOp::create(rewriter,
       loc, zero,
-      rewriter.create<arith::SubFOp>(loc, rhsRealIsInfWithSignTimesLhsImag,
+      arith::SubFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsImag,
                                      rhsImagIsInfWithSignTimesLhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = rewriter.create<arith::CmpFOp>(
+  Value realAbsSmallerThanImagAbs = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OLT, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = rewriter.create<arith::SelectOp>(
+  Value resultReal5 = arith::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = rewriter.create<arith::SelectOp>(
+  Value resultImag5 = arith::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase3 = arith::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase3 = arith::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase2 = arith::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase2 = arith::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase1 = arith::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase1 = arith::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = rewriter.create<arith::CmpFOp>(
+  Value resultRealIsNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::UNO, resultReal5, zero);
-  Value resultImagIsNaN = rewriter.create<arith::CmpFOp>(
+  Value resultImagIsNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::UNO, resultImag5, zero);
   Value resultIsNaN =
-      rewriter.create<arith::AndIOp>(loc, resultRealIsNaN, resultImagIsNaN);
+      arith::AndIOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = rewriter.create<arith::SelectOp>(
+  *resultRe = arith::SelectOp::create(rewriter,
       loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = rewriter.create<arith::SelectOp>(
+  *resultIm = arith::SelectOp::create(rewriter,
       loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
 }
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index e5e862315941d..a06935c4387c6 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -35,7 +35,7 @@ static constexpr unsigned kImaginaryPosInComplexNumberStruct = 1;
 
 ComplexStructBuilder ComplexStructBuilder::poison(OpBuilder &builder,
                                                   Location loc, Type type) {
-  Value val = builder.create<LLVM::PoisonOp>(loc, type);
+  Value val = LLVM::PoisonOp::create(builder, loc, type);
   return ComplexStructBuilder(val);
 }
 
@@ -79,9 +79,9 @@ struct AbsOpConversion : public ConvertOpToLLVMPattern<complex::AbsOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value sqNorm = rewriter.create<LLVM::FAddOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, real, real, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, imag, imag, fmf), fmf);
+    Value sqNorm = LLVM::FAddOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, real, real, fmf),
+        LLVM::FMulOp::create(rewriter, loc, imag, imag, fmf), fmf);
 
     rewriter.replaceOpWithNewOp<LLVM::SqrtOp>(op, sqNorm);
     return success();
@@ -192,9 +192,9 @@ struct AddOpConversion : public ConvertOpToLLVMPattern<complex::AddOp> {
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
     Value real =
-        rewriter.create<LLVM::FAddOp>(loc, arg.lhs.real(), arg.rhs.real(), fmf);
+        LLVM::FAddOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
     Value imag =
-        rewriter.create<LLVM::FAddOp>(loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+        LLVM::FAddOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
@@ -278,13 +278,13 @@ struct MulOpConversion : public ConvertOpToLLVMPattern<complex::MulOp> {
     Value lhsRe = arg.lhs.real();
     Value lhsIm = arg.lhs.imag();
 
-    Value real = rewriter.create<LLVM::FSubOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, rhsRe, lhsRe, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, rhsIm, lhsIm, fmf), fmf);
+    Value real = LLVM::FSubOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, lhsRe, fmf),
+        LLVM::FMulOp::create(rewriter, loc, rhsIm, lhsIm, fmf), fmf);
 
-    Value imag = rewriter.create<LLVM::FAddOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRe, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsIm, fmf), fmf);
+    Value imag = LLVM::FAddOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+        LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
@@ -314,9 +314,9 @@ struct SubOpConversion : public ConvertOpToLLVMPattern<complex::SubOp> {
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
     Value real =
-        rewriter.create<LLVM::FSubOp>(loc, arg.lhs.real(), arg.rhs.real(), fmf);
+        LLVM::FSubOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
     Value imag =
-        rewriter.create<LLVM::FSubOp>(loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+        LLVM::FSubOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
diff --git a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
index 56269d189873a..4800f65c675e6 100644
--- a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
+++ b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
@@ -84,7 +84,7 @@ LogicalResult ScalarOpToLibmCall<Op, TypeResolver>::matchAndRewrite(
     rewriter.setInsertionPointToStart(&module->getRegion(0).front());
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-    opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name,
+    opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
                                            opFunctionTy);
     opFunc.setPrivate();
   }
diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 0c832c452718b..d14e3888b118c 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -31,44 +31,44 @@ enum class AbsFn { abs, sqrt, rsqrt };
 // Returns the absolute value, its square root or its reciprocal square root.
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
                  ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
-  Value one = b.create<arith::ConstantOp>(real.getType(),
+  Value one = arith::ConstantOp::create(b, real.getType(),
                                           b.getFloatAttr(real.getType(), 1.0));
 
-  Value absReal = b.create<math::AbsFOp>(real, fmf);
-  Value absImag = b.create<math::AbsFOp>(imag, fmf);
+  Value absReal = math::AbsFOp::create(b, real, fmf);
+  Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-  Value max = b.create<arith::MaximumFOp>(absReal, absImag, fmf);
-  Value min = b.create<arith::MinimumFOp>(absReal, absImag, fmf);
+  Value max = arith::MaximumFOp::create(b, absReal, absImag, fmf);
+  Value min = arith::MinimumFOp::create(b, absReal, absImag, fmf);
 
   // The lowering below requires NaNs and infinities to work correctly.
   arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear(
       fmf, arith::FastMathFlags::nnan | arith::FastMathFlags::ninf);
-  Value ratio = b.create<arith::DivFOp>(min, max, fmfWithNaNInf);
-  Value ratioSq = b.create<arith::MulFOp>(ratio, ratio, fmfWithNaNInf);
-  Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmfWithNaNInf);
+  Value ratio = arith::DivFOp::create(b, min, max, fmfWithNaNInf);
+  Value ratioSq = arith::MulFOp::create(b, ratio, ratio, fmfWithNaNInf);
+  Value ratioSqPlusOne = arith::AddFOp::create(b, ratioSq, one, fmfWithNaNInf);
   Value result;
 
   if (fn == AbsFn::rsqrt) {
-    ratioSqPlusOne = b.create<math::RsqrtOp>(ratioSqPlusOne, fmfWithNaNInf);
-    min = b.create<math::RsqrtOp>(min, fmfWithNaNInf);
-    max = b.create<math::RsqrtOp>(max, fmfWithNaNInf);
+    ratioSqPlusOne = math::RsqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
+    min = math::RsqrtOp::create(b, min, fmfWithNaNInf);
+    max = math::RsqrtOp::create(b, max, fmfWithNaNInf);
   }
 
   if (fn == AbsFn::sqrt) {
-    Value quarter = b.create<arith::ConstantOp>(
+    Value quarter = arith::ConstantOp::create(b,
         real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
-    Value sqrt = b.create<math::SqrtOp>(max, fmfWithNaNInf);
-    Value p025 = b.create<math::PowFOp>(ratioSqPlusOne, quarter, fmfWithNaNInf);
-    result = b.create<arith::MulFOp>(sqrt, p025, fmfWithNaNInf);
+    Value sqrt = math::SqrtOp::create(b, max, fmfWithNaNInf);
+    Value p025 = math::PowFOp::create(b, ratioSqPlusOne, quarter, fmfWithNaNInf);
+    result = arith::MulFOp::create(b, sqrt, p025, fmfWithNaNInf);
   } else {
-    Value sqrt = b.create<math::SqrtOp>(ratioSqPlusOne, fmfWithNaNInf);
-    result = b.create<arith::MulFOp>(max, sqrt, fmfWithNaNInf);
+    Value sqrt = math::SqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
+    result = arith::MulFOp::create(b, max, sqrt, fmfWithNaNInf);
   }
 
-  Value isNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, result,
+  Value isNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, result,
                                         result, fmfWithNaNInf);
-  return b.create<arith::SelectOp>(isNaN, min, result);
+  return arith::SelectOp::create(b, isNaN, min, result);
 }
 
 struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
@@ -81,8 +81,8 @@ struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
 
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
     rewriter.replaceOp(op, computeAbs(real, imag, fmf, b));
 
     return success();
@@ -105,28 +105,28 @@ struct Atan2OpConversion : public OpConversionPattern<complex::Atan2Op> {
     Value lhs = adaptor.getLhs();
     Value rhs = adaptor.getRhs();
 
-    Value rhsSquared = b.create<complex::MulOp>(type, rhs, rhs, fmf);
-    Value lhsSquared = b.create<complex::MulOp>(type, lhs, lhs, fmf);
+    Value rhsSquared = complex::MulOp::create(b, type, rhs, rhs, fmf);
+    Value lhsSquared = complex::MulOp::create(b, type, lhs, lhs, fmf);
     Value rhsSquaredPlusLhsSquared =
-        b.create<complex::AddOp>(type, rhsSquared, lhsSquared, fmf);
+        complex::AddOp::create(b, type, rhsSquared, lhsSquared, fmf);
     Value sqrtOfRhsSquaredPlusLhsSquared =
-        b.create<complex::SqrtOp>(type, rhsSquaredPlusLhsSquared, fmf);
+        complex::SqrtOp::create(b, type, rhsSquaredPlusLhsSquared, fmf);
 
     Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
-    Value one = b.create<arith::ConstantOp>(elementType,
+        arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
+    Value one = arith::ConstantOp::create(b, elementType,
                                             b.getFloatAttr(elementType, 1));
-    Value i = b.create<complex::CreateOp>(type, zero, one);
-    Value iTimesLhs = b.create<complex::MulOp>(i, lhs, fmf);
-    Value rhsPlusILhs = b.create<complex::AddOp>(rhs, iTimesLhs, fmf);
+    Value i = complex::CreateOp::create(b, type, zero, one);
+    Value iTimesLhs = complex::MulOp::create(b, i, lhs, fmf);
+    Value rhsPlusILhs = complex::AddOp::create(b, rhs, iTimesLhs, fmf);
 
-    Value divResult = b.create<complex::DivOp>(
+    Value divResult = complex::DivOp::create(b,
         rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf);
-    Value logResult = b.create<complex::LogOp>(divResult, fmf);
+    Value logResult = complex::LogOp::create(b, divResult, fmf);
 
-    Value negativeOne = b.create<arith::ConstantOp>(
+    Value negativeOne = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -1));
-    Value negativeI = b.create<complex::CreateOp>(type, zero, negativeOne);
+    Value negativeI = complex::CreateOp::create(b, type, zero, negativeOne);
 
     rewriter.replaceOpWithNewOp<complex::MulOp>(op, negativeI, logResult, fmf);
     return success();
@@ -146,14 +146,14 @@ struct ComparisonOpConversion : public OpConversionPattern<ComparisonOp> {
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getLhs().getType()).getElementType();
 
-    Value realLhs = rewriter.create<complex::ReOp>(loc, type, adaptor.getLhs());
-    Value imagLhs = rewriter.create<complex::ImOp>(loc, type, adaptor.getLhs());
-    Value realRhs = rewriter.create<complex::ReOp>(loc, type, adaptor.getRhs());
-    Value imagRhs = rewriter.create<complex::ImOp>(loc, type, adaptor.getRhs());
+    Value realLhs = complex::ReOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value imagLhs = complex::ImOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value realRhs = complex::ReOp::create(rewriter, loc, type, adaptor.getRhs());
+    Value imagRhs = complex::ImOp::create(rewriter, loc, type, adaptor.getRhs());
     Value realComparison =
-        rewriter.create<arith::CmpFOp>(loc, p, realLhs, realRhs);
+        arith::CmpFOp::create(rewriter, loc, p, realLhs, realRhs);
     Value imagComparison =
-        rewriter.create<arith::CmpFOp>(loc, p, imagLhs, imagRhs);
+        arith::CmpFOp::create(rewriter, loc, p, imagLhs, imagRhs);
 
     rewriter.replaceOpWithNewOp<ResultCombiner>(op, realComparison,
                                                 imagComparison);
@@ -176,13 +176,13 @@ struct BinaryComplexOpConversion : public OpConversionPattern<BinaryComplexOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
-    Value realLhs = b.create<complex::ReOp>(elementType, adaptor.getLhs());
-    Value realRhs = b.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value resultReal = b.create<BinaryStandardOp>(elementType, realLhs, realRhs,
+    Value realLhs = complex::ReOp::create(b, elementType, adaptor.getLhs());
+    Value realRhs = complex::ReOp::create(b, elementType, adaptor.getRhs());
+    Value resultReal = BinaryStandardOp::create(b, elementType, realLhs, realRhs,
                                                   fmf.getValue());
-    Value imagLhs = b.create<complex::ImOp>(elementType, adaptor.getLhs());
-    Value imagRhs = b.create<complex::ImOp>(elementType, adaptor.getRhs());
-    Value resultImag = b.create<BinaryStandardOp>(elementType, imagLhs, imagRhs,
+    Value imagLhs = complex::ImOp::create(b, elementType, adaptor.getLhs());
+    Value imagRhs = complex::ImOp::create(b, elementType, adaptor.getRhs());
+    Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs, imagRhs,
                                                   fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -205,20 +205,20 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
 
     // Trigonometric ops use a set of common building blocks to convert to real
     // ops. Here we create these building blocks and call into an op-specific
     // implementation in the subclass to combine them.
-    Value half = rewriter.create<arith::ConstantOp>(
+    Value half = arith::ConstantOp::create(rewriter,
         loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
-    Value exp = rewriter.create<math::ExpOp>(loc, imag, fmf);
-    Value scaledExp = rewriter.create<arith::MulFOp>(loc, half, exp, fmf);
-    Value reciprocalExp = rewriter.create<arith::DivFOp>(loc, half, exp, fmf);
-    Value sin = rewriter.create<math::SinOp>(loc, real, fmf);
-    Value cos = rewriter.create<math::CosOp>(loc, real, fmf);
+    Value exp = math::ExpOp::create(rewriter, loc, imag, fmf);
+    Value scaledExp = arith::MulFOp::create(rewriter, loc, half, exp, fmf);
+    Value reciprocalExp = arith::DivFOp::create(rewriter, loc, half, exp, fmf);
+    Value sin = math::SinOp::create(rewriter, loc, real, fmf);
+    Value cos = math::CosOp::create(rewriter, loc, real, fmf);
 
     auto resultPair =
         combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf);
@@ -251,11 +251,11 @@ struct CosOpConversion : public TrigonometricOpConversion<complex::CosOp> {
     //   Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x
     //   Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x
     Value sum =
-        rewriter.create<arith::AddFOp>(loc, reciprocalExp, scaledExp, fmf);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, cos, fmf);
+        arith::AddFOp::create(rewriter, loc, reciprocalExp, scaledExp, fmf);
+    Value resultReal = arith::MulFOp::create(rewriter, loc, sum, cos, fmf);
     Value diff =
-        rewriter.create<arith::SubFOp>(loc, reciprocalExp, scaledExp, fmf);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, sin, fmf);
+        arith::SubFOp::create(rewriter, loc, reciprocalExp, scaledExp, fmf);
+    Value resultImag = arith::MulFOp::create(rewriter, loc, diff, sin, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -275,13 +275,13 @@ struct DivOpConversion : public OpConversionPattern<complex::DivOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value lhsReal =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getLhs());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getLhs());
     Value lhsImag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getLhs());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getLhs());
     Value rhsReal =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getRhs());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getRhs());
     Value rhsImag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getRhs());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getRhs());
 
     Value resultReal, resultImag;
 
@@ -318,16 +318,16 @@ struct ExpOpConversion : public OpConversionPattern<complex::ExpOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value expReal = rewriter.create<math::ExpOp>(loc, real, fmf.getValue());
-    Value cosImag = rewriter.create<math::CosOp>(loc, imag, fmf.getValue());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value expReal = math::ExpOp::create(rewriter, loc, real, fmf.getValue());
+    Value cosImag = math::CosOp::create(rewriter, loc, imag, fmf.getValue());
     Value resultReal =
-        rewriter.create<arith::MulFOp>(loc, expReal, cosImag, fmf.getValue());
-    Value sinImag = rewriter.create<math::SinOp>(loc, imag, fmf.getValue());
+        arith::MulFOp::create(rewriter, loc, expReal, cosImag, fmf.getValue());
+    Value sinImag = math::SinOp::create(rewriter, loc, imag, fmf.getValue());
     Value resultImag =
-        rewriter.create<arith::MulFOp>(loc, expReal, sinImag, fmf.getValue());
+        arith::MulFOp::create(rewriter, loc, expReal, sinImag, fmf.getValue());
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -340,11 +340,11 @@ Value evaluatePolynomial(ImplicitLocOpBuilder &b, Value arg,
                          arith::FastMathFlagsAttr fmf) {
   auto argType = mlir::cast<FloatType>(arg.getType());
   Value poly =
-      b.create<arith::ConstantOp>(b.getFloatAttr(argType, coefficients[0]));
+      arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[0]));
   for (unsigned i = 1; i < coefficients.size(); ++i) {
-    poly = b.create<math::FmaOp>(
+    poly = math::FmaOp::create(b,
         poly, arg,
-        b.create<arith::ConstantOp>(b.getFloatAttr(argType, coefficients[i])),
+        arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[i])),
         fmf);
   }
   return poly;
@@ -365,26 +365,26 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
 
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     ImplicitLocOpBuilder b(op.getLoc(), rewriter);
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
-    Value zero = b.create<arith::ConstantOp>(b.getFloatAttr(elemType, 0.0));
-    Value one = b.create<arith::ConstantOp>(b.getFloatAttr(elemType, 1.0));
+    Value zero = arith::ConstantOp::create(b, b.getFloatAttr(elemType, 0.0));
+    Value one = arith::ConstantOp::create(b, b.getFloatAttr(elemType, 1.0));
 
-    Value expm1Real = b.create<math::ExpM1Op>(real, fmf);
-    Value expReal = b.create<arith::AddFOp>(expm1Real, one, fmf);
+    Value expm1Real = math::ExpM1Op::create(b, real, fmf);
+    Value expReal = arith::AddFOp::create(b, expm1Real, one, fmf);
 
-    Value sinImag = b.create<math::SinOp>(imag, fmf);
+    Value sinImag = math::SinOp::create(b, imag, fmf);
     Value cosm1Imag = emitCosm1(imag, fmf, b);
-    Value cosImag = b.create<arith::AddFOp>(cosm1Imag, one, fmf);
+    Value cosImag = arith::AddFOp::create(b, cosm1Imag, one, fmf);
 
-    Value realResult = b.create<arith::AddFOp>(
-        b.create<arith::MulFOp>(expm1Real, cosImag, fmf), cosm1Imag, fmf);
+    Value realResult = arith::AddFOp::create(b,
+        arith::MulFOp::create(b, expm1Real, cosImag, fmf), cosm1Imag, fmf);
 
-    Value imagIsZero = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag,
+    Value imagIsZero = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag,
                                                zero, fmf.getValue());
-    Value imagResult = b.create<arith::SelectOp>(
-        imagIsZero, zero, b.create<arith::MulFOp>(expReal, sinImag, fmf));
+    Value imagResult = arith::SelectOp::create(b,
+        imagIsZero, zero, arith::MulFOp::create(b, expReal, sinImag, fmf));
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, realResult,
                                                    imagResult);
@@ -395,8 +395,8 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
   Value emitCosm1(Value arg, arith::FastMathFlagsAttr fmf,
                   ImplicitLocOpBuilder &b) const {
     auto argType = mlir::cast<FloatType>(arg.getType());
-    auto negHalf = b.create<arith::ConstantOp>(b.getFloatAttr(argType, -0.5));
-    auto negOne = b.create<arith::ConstantOp>(b.getFloatAttr(argType, -1.0));
+    auto negHalf = arith::ConstantOp::create(b, b.getFloatAttr(argType, -0.5));
+    auto negOne = arith::ConstantOp::create(b, b.getFloatAttr(argType, -1.0));
 
     // Algorithm copied from cephes cosm1.
     SmallVector<double, 7> kCoeffs{
@@ -405,23 +405,23 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
         2.4801587301570552304991E-5,  -1.3888888888888872993737E-3,
         4.1666666666666666609054E-2,
     };
-    Value cos = b.create<math::CosOp>(arg, fmf);
-    Value forLargeArg = b.create<arith::AddFOp>(cos, negOne, fmf);
+    Value cos = math::CosOp::create(b, arg, fmf);
+    Value forLargeArg = arith::AddFOp::create(b, cos, negOne, fmf);
 
-    Value argPow2 = b.create<arith::MulFOp>(arg, arg, fmf);
-    Value argPow4 = b.create<arith::MulFOp>(argPow2, argPow2, fmf);
+    Value argPow2 = arith::MulFOp::create(b, arg, arg, fmf);
+    Value argPow4 = arith::MulFOp::create(b, argPow2, argPow2, fmf);
     Value poly = evaluatePolynomial(b, argPow2, kCoeffs, fmf);
 
     auto forSmallArg =
-        b.create<arith::AddFOp>(b.create<arith::MulFOp>(argPow4, poly, fmf),
-                                b.create<arith::MulFOp>(negHalf, argPow2, fmf));
+        arith::AddFOp::create(b, arith::MulFOp::create(b, argPow4, poly, fmf),
+                                arith::MulFOp::create(b, negHalf, argPow2, fmf));
 
     // (pi/4)^2 is approximately 0.61685
     Value piOver4Pow2 =
-        b.create<arith::ConstantOp>(b.getFloatAttr(argType, 0.61685));
-    Value cond = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, argPow2,
+        arith::ConstantOp::create(b, b.getFloatAttr(argType, 0.61685));
+    Value cond = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, argPow2,
                                          piOver4Pow2, fmf.getValue());
-    return b.create<arith::SelectOp>(cond, forLargeArg, forSmallArg);
+    return arith::SelectOp::create(b, cond, forLargeArg, forSmallArg);
   }
 };
 
@@ -436,13 +436,13 @@ struct LogOpConversion : public OpConversionPattern<complex::LogOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
-    Value abs = b.create<complex::AbsOp>(elementType, adaptor.getComplex(),
+    Value abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(),
                                          fmf.getValue());
-    Value resultReal = b.create<math::LogOp>(elementType, abs, fmf.getValue());
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value resultReal = math::LogOp::create(b, elementType, abs, fmf.getValue());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value resultImag =
-        b.create<math::Atan2Op>(elementType, imag, real, fmf.getValue());
+        math::Atan2Op::create(b, elementType, imag, real, fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -460,40 +460,40 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
-    Value half = b.create<arith::ConstantOp>(elementType,
+    Value half = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 0.5));
-    Value one = b.create<arith::ConstantOp>(elementType,
+    Value one = arith::ConstantOp::create(b, elementType,
                                             b.getFloatAttr(elementType, 1));
-    Value realPlusOne = b.create<arith::AddFOp>(real, one, fmf);
-    Value absRealPlusOne = b.create<math::AbsFOp>(realPlusOne, fmf);
-    Value absImag = b.create<math::AbsFOp>(imag, fmf);
+    Value realPlusOne = arith::AddFOp::create(b, real, one, fmf);
+    Value absRealPlusOne = math::AbsFOp::create(b, realPlusOne, fmf);
+    Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-    Value maxAbs = b.create<arith::MaximumFOp>(absRealPlusOne, absImag, fmf);
-    Value minAbs = b.create<arith::MinimumFOp>(absRealPlusOne, absImag, fmf);
+    Value maxAbs = arith::MaximumFOp::create(b, absRealPlusOne, absImag, fmf);
+    Value minAbs = arith::MinimumFOp::create(b, absRealPlusOne, absImag, fmf);
 
-    Value useReal = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGT,
+    Value useReal = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT,
                                             realPlusOne, absImag, fmf);
-    Value maxMinusOne = b.create<arith::SubFOp>(maxAbs, one, fmf);
+    Value maxMinusOne = arith::SubFOp::create(b, maxAbs, one, fmf);
     Value maxAbsOfRealPlusOneAndImagMinusOne =
-        b.create<arith::SelectOp>(useReal, real, maxMinusOne);
+        arith::SelectOp::create(b, useReal, real, maxMinusOne);
     arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear(
         fmf, arith::FastMathFlags::nnan | arith::FastMathFlags::ninf);
-    Value minMaxRatio = b.create<arith::DivFOp>(minAbs, maxAbs, fmfWithNaNInf);
+    Value minMaxRatio = arith::DivFOp::create(b, minAbs, maxAbs, fmfWithNaNInf);
     Value logOfMaxAbsOfRealPlusOneAndImag =
-        b.create<math::Log1pOp>(maxAbsOfRealPlusOneAndImagMinusOne, fmf);
-    Value logOfSqrtPart = b.create<math::Log1pOp>(
-        b.create<arith::MulFOp>(minMaxRatio, minMaxRatio, fmfWithNaNInf),
+        math::Log1pOp::create(b, maxAbsOfRealPlusOneAndImagMinusOne, fmf);
+    Value logOfSqrtPart = math::Log1pOp::create(b,
+        arith::MulFOp::create(b, minMaxRatio, minMaxRatio, fmfWithNaNInf),
         fmfWithNaNInf);
-    Value r = b.create<arith::AddFOp>(
-        b.create<arith::MulFOp>(half, logOfSqrtPart, fmfWithNaNInf),
+    Value r = arith::AddFOp::create(b,
+        arith::MulFOp::create(b, half, logOfSqrtPart, fmfWithNaNInf),
         logOfMaxAbsOfRealPlusOneAndImag, fmfWithNaNInf);
-    Value resultReal = b.create<arith::SelectOp>(
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
+    Value resultReal = arith::SelectOp::create(b,
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
         minAbs, r);
-    Value resultImag = b.create<math::Atan2Op>(imag, realPlusOne, fmf);
+    Value resultImag = math::Atan2Op::create(b, imag, realPlusOne, fmf);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -511,21 +511,21 @@ struct MulOpConversion : public OpConversionPattern<complex::MulOp> {
     auto elementType = cast<FloatType>(type.getElementType());
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     auto fmfValue = fmf.getValue();
-    Value lhsReal = b.create<complex::ReOp>(elementType, adaptor.getLhs());
-    Value lhsImag = b.create<complex::ImOp>(elementType, adaptor.getLhs());
-    Value rhsReal = b.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value rhsImag = b.create<complex::ImOp>(elementType, adaptor.getRhs());
+    Value lhsReal = complex::ReOp::create(b, elementType, adaptor.getLhs());
+    Value lhsImag = complex::ImOp::create(b, elementType, adaptor.getLhs());
+    Value rhsReal = complex::ReOp::create(b, elementType, adaptor.getRhs());
+    Value rhsImag = complex::ImOp::create(b, elementType, adaptor.getRhs());
     Value lhsRealTimesRhsReal =
-        b.create<arith::MulFOp>(lhsReal, rhsReal, fmfValue);
+        arith::MulFOp::create(b, lhsReal, rhsReal, fmfValue);
     Value lhsImagTimesRhsImag =
-        b.create<arith::MulFOp>(lhsImag, rhsImag, fmfValue);
-    Value real = b.create<arith::SubFOp>(lhsRealTimesRhsReal,
+        arith::MulFOp::create(b, lhsImag, rhsImag, fmfValue);
+    Value real = arith::SubFOp::create(b, lhsRealTimesRhsReal,
                                          lhsImagTimesRhsImag, fmfValue);
     Value lhsImagTimesRhsReal =
-        b.create<arith::MulFOp>(lhsImag, rhsReal, fmfValue);
+        arith::MulFOp::create(b, lhsImag, rhsReal, fmfValue);
     Value lhsRealTimesRhsImag =
-        b.create<arith::MulFOp>(lhsReal, rhsImag, fmfValue);
-    Value imag = b.create<arith::AddFOp>(lhsImagTimesRhsReal,
+        arith::MulFOp::create(b, lhsReal, rhsImag, fmfValue);
+    Value imag = arith::AddFOp::create(b, lhsImagTimesRhsReal,
                                          lhsRealTimesRhsImag, fmfValue);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, imag);
     return success();
@@ -543,11 +543,11 @@ struct NegOpConversion : public OpConversionPattern<complex::NegOp> {
     auto elementType = cast<FloatType>(type.getElementType());
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negReal = rewriter.create<arith::NegFOp>(loc, real);
-    Value negImag = rewriter.create<arith::NegFOp>(loc, imag);
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value negReal = arith::NegFOp::create(rewriter, loc, real);
+    Value negImag = arith::NegFOp::create(rewriter, loc, imag);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, negReal, negImag);
     return success();
   }
@@ -570,11 +570,11 @@ struct SinOpConversion : public TrigonometricOpConversion<complex::SinOp> {
     //   Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x
     //   Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x
     Value sum =
-        rewriter.create<arith::AddFOp>(loc, scaledExp, reciprocalExp, fmf);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, sin, fmf);
+        arith::AddFOp::create(rewriter, loc, scaledExp, reciprocalExp, fmf);
+    Value resultReal = arith::MulFOp::create(rewriter, loc, sum, sin, fmf);
     Value diff =
-        rewriter.create<arith::SubFOp>(loc, scaledExp, reciprocalExp, fmf);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, cos, fmf);
+        arith::SubFOp::create(rewriter, loc, scaledExp, reciprocalExp, fmf);
+    Value resultImag = arith::MulFOp::create(rewriter, loc, diff, cos, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -593,64 +593,64 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
-    Value half = b.create<arith::ConstantOp>(elementType,
+    Value half = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 0.5));
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value absSqrt = computeAbs(real, imag, fmf, b, AbsFn::sqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value sqrtArg = b.create<arith::MulFOp>(argArg, half, fmf);
-    Value cos = b.create<math::CosOp>(sqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(sqrtArg, fmf);
+    Value argArg = math::Atan2Op::create(b, imag, real, fmf);
+    Value sqrtArg = arith::MulFOp::create(b, argArg, half, fmf);
+    Value cos = math::CosOp::create(b, sqrtArg, fmf);
+    Value sin = math::SinOp::create(b, sqrtArg, fmf);
     // sin(atan2(0, inf)) = 0, sqrt(abs(inf)) = inf, but we can't multiply
     // 0 * inf.
     Value sinIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, sin, zero, fmf);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, sin, zero, fmf);
 
-    Value resultReal = b.create<arith::MulFOp>(absSqrt, cos, fmf);
-    Value resultImag = b.create<arith::SelectOp>(
-        sinIsZero, zero, b.create<arith::MulFOp>(absSqrt, sin, fmf));
+    Value resultReal = arith::MulFOp::create(b, absSqrt, cos, fmf);
+    Value resultImag = arith::SelectOp::create(b,
+        sinIsZero, zero, arith::MulFOp::create(b, absSqrt, sin, fmf));
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
       Value inf = cst(APFloat::getInf(floatSemantics));
       Value negInf = cst(APFloat::getInf(floatSemantics, true));
       Value nan = cst(APFloat::getNaN(floatSemantics));
-      Value absImag = b.create<math::AbsFOp>(elementType, imag, fmf);
+      Value absImag = math::AbsFOp::create(b, elementType, imag, fmf);
 
       Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
       Value absImagIsNotInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::ONE, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::ONE, absImag, inf, fmf);
       Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, inf, fmf);
       Value realIsNegInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, negInf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, negInf, fmf);
 
-      resultReal = b.create<arith::SelectOp>(
-          b.create<arith::AndIOp>(realIsNegInf, absImagIsNotInf), zero,
+      resultReal = arith::SelectOp::create(b,
+          arith::AndIOp::create(b, realIsNegInf, absImagIsNotInf), zero,
           resultReal);
-      resultReal = b.create<arith::SelectOp>(
-          b.create<arith::OrIOp>(absImagIsInf, realIsInf), inf, resultReal);
+      resultReal = arith::SelectOp::create(b,
+          arith::OrIOp::create(b, absImagIsInf, realIsInf), inf, resultReal);
 
-      Value imagSignInf = b.create<math::CopySignOp>(inf, imag, fmf);
-      resultImag = b.create<arith::SelectOp>(
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, absSqrt, absSqrt),
+      Value imagSignInf = math::CopySignOp::create(b, inf, imag, fmf);
+      resultImag = arith::SelectOp::create(b,
+          arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, absSqrt, absSqrt),
           nan, resultImag);
-      resultImag = b.create<arith::SelectOp>(
-          b.create<arith::OrIOp>(absImagIsInf, realIsNegInf), imagSignInf,
+      resultImag = arith::SelectOp::create(b,
+          arith::OrIOp::create(b, absImagIsInf, realIsNegInf), imagSignInf,
           resultImag);
     }
 
     Value resultIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
-    resultReal = b.create<arith::SelectOp>(resultIsZero, zero, resultReal);
-    resultImag = b.create<arith::SelectOp>(resultIsZero, zero, resultImag);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
+    resultReal = arith::SelectOp::create(b, resultIsZero, zero, resultReal);
+    resultImag = arith::SelectOp::create(b, resultIsZero, zero, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -669,19 +669,19 @@ struct SignOpConversion : public OpConversionPattern<complex::SignOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
+        arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
     Value realIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, zero);
     Value imagIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero);
-    Value isZero = b.create<arith::AndIOp>(realIsZero, imagIsZero);
-    auto abs = b.create<complex::AbsOp>(elementType, adaptor.getComplex(), fmf);
-    Value realSign = b.create<arith::DivFOp>(real, abs, fmf);
-    Value imagSign = b.create<arith::DivFOp>(imag, abs, fmf);
-    Value sign = b.create<complex::CreateOp>(type, realSign, imagSign);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero);
+    Value isZero = arith::AndIOp::create(b, realIsZero, imagIsZero);
+    auto abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(), fmf);
+    Value realSign = arith::DivFOp::create(b, real, abs, fmf);
+    Value imagSign = arith::DivFOp::create(b, imag, abs, fmf);
+    Value sign = complex::CreateOp::create(b, type, realSign, imagSign);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isZero,
                                                  adaptor.getComplex(), sign);
     return success();
@@ -703,84 +703,84 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
     const auto &floatSemantics = elementType.getFloatSemantics();
 
     Value real =
-        b.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(b, loc, elementType, adaptor.getComplex());
     Value imag =
-        b.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negOne = b.create<arith::ConstantOp>(
+        complex::ImOp::create(b, loc, elementType, adaptor.getComplex());
+    Value negOne = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -1.0));
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
       std::swap(real, imag);
-      real = b.create<arith::MulFOp>(real, negOne, fmf);
+      real = arith::MulFOp::create(b, real, negOne, fmf);
     }
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value four = b.create<arith::ConstantOp>(elementType,
+    Value four = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 4.0));
-    Value twoReal = b.create<arith::AddFOp>(real, real, fmf);
-    Value negTwoReal = b.create<arith::MulFOp>(negOne, twoReal, fmf);
+    Value twoReal = arith::AddFOp::create(b, real, real, fmf);
+    Value negTwoReal = arith::MulFOp::create(b, negOne, twoReal, fmf);
 
-    Value expTwoRealMinusOne = b.create<math::ExpM1Op>(twoReal, fmf);
-    Value expNegTwoRealMinusOne = b.create<math::ExpM1Op>(negTwoReal, fmf);
+    Value expTwoRealMinusOne = math::ExpM1Op::create(b, twoReal, fmf);
+    Value expNegTwoRealMinusOne = math::ExpM1Op::create(b, negTwoReal, fmf);
     Value realNum =
-        b.create<arith::SubFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+        arith::SubFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
 
-    Value cosImag = b.create<math::CosOp>(imag, fmf);
-    Value cosImagSq = b.create<arith::MulFOp>(cosImag, cosImag, fmf);
-    Value twoCosTwoImagPlusOne = b.create<arith::MulFOp>(cosImagSq, four, fmf);
-    Value sinImag = b.create<math::SinOp>(imag, fmf);
+    Value cosImag = math::CosOp::create(b, imag, fmf);
+    Value cosImagSq = arith::MulFOp::create(b, cosImag, cosImag, fmf);
+    Value twoCosTwoImagPlusOne = arith::MulFOp::create(b, cosImagSq, four, fmf);
+    Value sinImag = math::SinOp::create(b, imag, fmf);
 
-    Value imagNum = b.create<arith::MulFOp>(
-        four, b.create<arith::MulFOp>(cosImag, sinImag, fmf), fmf);
+    Value imagNum = arith::MulFOp::create(b,
+        four, arith::MulFOp::create(b, cosImag, sinImag, fmf), fmf);
 
     Value expSumMinusTwo =
-        b.create<arith::AddFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+        arith::AddFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
     Value denom =
-        b.create<arith::AddFOp>(expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
+        arith::AddFOp::create(b, expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
 
-    Value isInf = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+    Value isInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
                                           expSumMinusTwo, inf, fmf);
-    Value realLimit = b.create<math::CopySignOp>(negOne, real, fmf);
+    Value realLimit = math::CopySignOp::create(b, negOne, real, fmf);
 
-    Value resultReal = b.create<arith::SelectOp>(
-        isInf, realLimit, b.create<arith::DivFOp>(realNum, denom, fmf));
-    Value resultImag = b.create<arith::DivFOp>(imagNum, denom, fmf);
+    Value resultReal = arith::SelectOp::create(b,
+        isInf, realLimit, arith::DivFOp::create(b, realNum, denom, fmf));
+    Value resultImag = arith::DivFOp::create(b, imagNum, denom, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value zero = b.create<arith::ConstantOp>(
+      Value absReal = math::AbsFOp::create(b, real, fmf);
+      Value zero = arith::ConstantOp::create(b,
           elementType, b.getFloatAttr(elementType, 0.0));
       Value nan = cst(APFloat::getNaN(floatSemantics));
 
       Value absRealIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
       Value imagIsZero =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-      Value absRealIsNotInf = b.create<arith::XOrIOp>(
-          absRealIsInf, b.create<arith::ConstantIntOp>(true, /*width=*/1));
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
+      Value absRealIsNotInf = arith::XOrIOp::create(b,
+          absRealIsInf, arith::ConstantIntOp::create(b, true, /*width=*/1));
 
-      Value imagNumIsNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO,
+      Value imagNumIsNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO,
                                                    imagNum, imagNum, fmf);
       Value resultRealIsNaN =
-          b.create<arith::AndIOp>(imagNumIsNaN, absRealIsNotInf);
-      Value resultImagIsZero = b.create<arith::OrIOp>(
-          imagIsZero, b.create<arith::AndIOp>(absRealIsInf, imagNumIsNaN));
+          arith::AndIOp::create(b, imagNumIsNaN, absRealIsNotInf);
+      Value resultImagIsZero = arith::OrIOp::create(b,
+          imagIsZero, arith::AndIOp::create(b, absRealIsInf, imagNumIsNaN));
 
-      resultReal = b.create<arith::SelectOp>(resultRealIsNaN, nan, resultReal);
+      resultReal = arith::SelectOp::create(b, resultRealIsNaN, nan, resultReal);
       resultImag =
-          b.create<arith::SelectOp>(resultImagIsZero, zero, resultImag);
+          arith::SelectOp::create(b, resultImagIsZero, zero, resultImag);
     }
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
       std::swap(resultReal, resultImag);
-      resultImag = b.create<arith::MulFOp>(resultImag, negOne, fmf);
+      resultImag = arith::MulFOp::create(b, resultImag, negOne, fmf);
     }
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
@@ -799,10 +799,10 @@ struct ConjOpConversion : public OpConversionPattern<complex::ConjOp> {
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negImag = rewriter.create<arith::NegFOp>(loc, elementType, imag);
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value negImag = arith::NegFOp::create(rewriter, loc, elementType, imag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, negImag);
 
@@ -818,97 +818,97 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
                                  arith::FastMathFlags fmf) {
   auto elementType = cast<FloatType>(type.getElementType());
 
-  Value a = builder.create<complex::ReOp>(lhs);
-  Value b = builder.create<complex::ImOp>(lhs);
+  Value a = complex::ReOp::create(builder, lhs);
+  Value b = complex::ImOp::create(builder, lhs);
 
-  Value abs = builder.create<complex::AbsOp>(lhs, fmf);
-  Value absToC = builder.create<math::PowFOp>(abs, c, fmf);
+  Value abs = complex::AbsOp::create(builder, lhs, fmf);
+  Value absToC = math::PowFOp::create(builder, abs, c, fmf);
 
-  Value negD = builder.create<arith::NegFOp>(d, fmf);
-  Value argLhs = builder.create<math::Atan2Op>(b, a, fmf);
-  Value negDArgLhs = builder.create<arith::MulFOp>(negD, argLhs, fmf);
-  Value expNegDArgLhs = builder.create<math::ExpOp>(negDArgLhs, fmf);
+  Value negD = arith::NegFOp::create(builder, d, fmf);
+  Value argLhs = math::Atan2Op::create(builder, b, a, fmf);
+  Value negDArgLhs = arith::MulFOp::create(builder, negD, argLhs, fmf);
+  Value expNegDArgLhs = math::ExpOp::create(builder, negDArgLhs, fmf);
 
-  Value coeff = builder.create<arith::MulFOp>(absToC, expNegDArgLhs, fmf);
-  Value lnAbs = builder.create<math::LogOp>(abs, fmf);
-  Value cArgLhs = builder.create<arith::MulFOp>(c, argLhs, fmf);
-  Value dLnAbs = builder.create<arith::MulFOp>(d, lnAbs, fmf);
-  Value q = builder.create<arith::AddFOp>(cArgLhs, dLnAbs, fmf);
-  Value cosQ = builder.create<math::CosOp>(q, fmf);
-  Value sinQ = builder.create<math::SinOp>(q, fmf);
+  Value coeff = arith::MulFOp::create(builder, absToC, expNegDArgLhs, fmf);
+  Value lnAbs = math::LogOp::create(builder, abs, fmf);
+  Value cArgLhs = arith::MulFOp::create(builder, c, argLhs, fmf);
+  Value dLnAbs = arith::MulFOp::create(builder, d, lnAbs, fmf);
+  Value q = arith::AddFOp::create(builder, cArgLhs, dLnAbs, fmf);
+  Value cosQ = math::CosOp::create(builder, q, fmf);
+  Value sinQ = math::SinOp::create(builder, q, fmf);
 
-  Value inf = builder.create<arith::ConstantOp>(
+  Value inf = arith::ConstantOp::create(builder,
       elementType,
       builder.getFloatAttr(elementType,
                            APFloat::getInf(elementType.getFloatSemantics())));
-  Value zero = builder.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(builder,
       elementType, builder.getFloatAttr(elementType, 0.0));
-  Value one = builder.create<arith::ConstantOp>(
+  Value one = arith::ConstantOp::create(builder,
       elementType, builder.getFloatAttr(elementType, 1.0));
-  Value complexOne = builder.create<complex::CreateOp>(type, one, zero);
-  Value complexZero = builder.create<complex::CreateOp>(type, zero, zero);
-  Value complexInf = builder.create<complex::CreateOp>(type, inf, zero);
+  Value complexOne = complex::CreateOp::create(builder, type, one, zero);
+  Value complexZero = complex::CreateOp::create(builder, type, zero, zero);
+  Value complexInf = complex::CreateOp::create(builder, type, inf, zero);
 
   // Case 0:
   // d^c is 0 if d is 0 and c > 0. 0^0 is defined to be 1.0, see
   // Branch Cuts for Complex Elementary Functions or Much Ado About
   // Nothing's Sign Bit, W. Kahan, Section 10.
   Value absEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, abs, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, abs, zero, fmf);
   Value dEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, d, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, d, zero, fmf);
   Value cEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, c, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, c, zero, fmf);
   Value bEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, b, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, b, zero, fmf);
 
   Value zeroLeC =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLE, zero, c, fmf);
-  Value coeffCosQ = builder.create<arith::MulFOp>(coeff, cosQ, fmf);
-  Value coeffSinQ = builder.create<arith::MulFOp>(coeff, sinQ, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLE, zero, c, fmf);
+  Value coeffCosQ = arith::MulFOp::create(builder, coeff, cosQ, fmf);
+  Value coeffSinQ = arith::MulFOp::create(builder, coeff, sinQ, fmf);
   Value complexOneOrZero =
-      builder.create<arith::SelectOp>(cEqZero, complexOne, complexZero);
+      arith::SelectOp::create(builder, cEqZero, complexOne, complexZero);
   Value coeffCosSin =
-      builder.create<complex::CreateOp>(type, coeffCosQ, coeffSinQ);
-  Value cutoff0 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(
-          builder.create<arith::AndIOp>(absEqZero, dEqZero), zeroLeC),
+      complex::CreateOp::create(builder, type, coeffCosQ, coeffSinQ);
+  Value cutoff0 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder,
+          arith::AndIOp::create(builder, absEqZero, dEqZero), zeroLeC),
       complexOneOrZero, coeffCosSin);
 
   // Case 1:
   // x^0 is defined to be 1 for any x, see
   // Branch Cuts for Complex Elementary Functions or Much Ado About
   // Nothing's Sign Bit, W. Kahan, Section 10.
-  Value rhsEqZero = builder.create<arith::AndIOp>(cEqZero, dEqZero);
+  Value rhsEqZero = arith::AndIOp::create(builder, cEqZero, dEqZero);
   Value cutoff1 =
-      builder.create<arith::SelectOp>(rhsEqZero, complexOne, cutoff0);
+      arith::SelectOp::create(builder, rhsEqZero, complexOne, cutoff0);
 
   // Case 2:
   // 1^(c + d*i) = 1 + 0*i
-  Value lhsEqOne = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, one, fmf),
+  Value lhsEqOne = arith::AndIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, one, fmf),
       bEqZero);
   Value cutoff2 =
-      builder.create<arith::SelectOp>(lhsEqOne, complexOne, cutoff1);
+      arith::SelectOp::create(builder, lhsEqOne, complexOne, cutoff1);
 
   // Case 3:
   // inf^(c + 0*i) = inf + 0*i, c > 0
-  Value lhsEqInf = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, inf, fmf),
+  Value lhsEqInf = arith::AndIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, inf, fmf),
       bEqZero);
-  Value rhsGt0 = builder.create<arith::AndIOp>(
+  Value rhsGt0 = arith::AndIOp::create(builder,
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, c, zero, fmf));
-  Value cutoff3 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(lhsEqInf, rhsGt0), complexInf, cutoff2);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, c, zero, fmf));
+  Value cutoff3 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder, lhsEqInf, rhsGt0), complexInf, cutoff2);
 
   // Case 4:
   // inf^(c + 0*i) = 0 + 0*i, c < 0
-  Value rhsLt0 = builder.create<arith::AndIOp>(
+  Value rhsLt0 = arith::AndIOp::create(builder,
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, c, zero, fmf));
-  Value cutoff4 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(lhsEqInf, rhsLt0), complexZero, cutoff3);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, c, zero, fmf));
+  Value cutoff4 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder, lhsEqInf, rhsLt0), complexZero, cutoff3);
 
   return cutoff4;
 }
@@ -923,8 +923,8 @@ struct PowOpConversion : public OpConversionPattern<complex::PowOp> {
     auto type = cast<ComplexType>(adaptor.getLhs().getType());
     auto elementType = cast<FloatType>(type.getElementType());
 
-    Value c = builder.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value d = builder.create<complex::ImOp>(elementType, adaptor.getRhs());
+    Value c = complex::ReOp::create(builder, elementType, adaptor.getRhs());
+    Value d = complex::ImOp::create(builder, elementType, adaptor.getRhs());
 
     rewriter.replaceOp(op, {powOpConversionImpl(builder, type, adaptor.getLhs(),
                                                 c, d, op.getFastmath())});
@@ -945,64 +945,64 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value negHalf = b.create<arith::ConstantOp>(
+    Value negHalf = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -0.5));
     Value nan = cst(APFloat::getNaN(floatSemantics));
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value absRsqrt = computeAbs(real, imag, fmf, b, AbsFn::rsqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value rsqrtArg = b.create<arith::MulFOp>(argArg, negHalf, fmf);
-    Value cos = b.create<math::CosOp>(rsqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(rsqrtArg, fmf);
+    Value argArg = math::Atan2Op::create(b, imag, real, fmf);
+    Value rsqrtArg = arith::MulFOp::create(b, argArg, negHalf, fmf);
+    Value cos = math::CosOp::create(b, rsqrtArg, fmf);
+    Value sin = math::SinOp::create(b, rsqrtArg, fmf);
 
-    Value resultReal = b.create<arith::MulFOp>(absRsqrt, cos, fmf);
-    Value resultImag = b.create<arith::MulFOp>(absRsqrt, sin, fmf);
+    Value resultReal = arith::MulFOp::create(b, absRsqrt, cos, fmf);
+    Value resultImag = arith::MulFOp::create(b, absRsqrt, sin, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value negOne = b.create<arith::ConstantOp>(
+      Value negOne = arith::ConstantOp::create(b,
           elementType, b.getFloatAttr(elementType, -1));
 
-      Value realSignedZero = b.create<math::CopySignOp>(zero, real, fmf);
-      Value imagSignedZero = b.create<math::CopySignOp>(zero, imag, fmf);
+      Value realSignedZero = math::CopySignOp::create(b, zero, real, fmf);
+      Value imagSignedZero = math::CopySignOp::create(b, zero, imag, fmf);
       Value negImagSignedZero =
-          b.create<arith::MulFOp>(negOne, imagSignedZero, fmf);
+          arith::MulFOp::create(b, negOne, imagSignedZero, fmf);
 
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value absImag = b.create<math::AbsFOp>(imag, fmf);
+      Value absReal = math::AbsFOp::create(b, real, fmf);
+      Value absImag = math::AbsFOp::create(b, imag, fmf);
 
       Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
       Value realIsNan =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, real, real, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, real, real, fmf);
       Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
-      Value inIsNanInf = b.create<arith::AndIOp>(absImagIsInf, realIsNan);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value inIsNanInf = arith::AndIOp::create(b, absImagIsInf, realIsNan);
 
-      Value resultIsZero = b.create<arith::OrIOp>(inIsNanInf, realIsInf);
+      Value resultIsZero = arith::OrIOp::create(b, inIsNanInf, realIsInf);
 
       resultReal =
-          b.create<arith::SelectOp>(resultIsZero, realSignedZero, resultReal);
-      resultImag = b.create<arith::SelectOp>(resultIsZero, negImagSignedZero,
+          arith::SelectOp::create(b, resultIsZero, realSignedZero, resultReal);
+      resultImag = arith::SelectOp::create(b, resultIsZero, negImagSignedZero,
                                              resultImag);
     }
 
     Value isRealZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero, fmf);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, zero, fmf);
     Value isImagZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-    Value isZero = b.create<arith::AndIOp>(isRealZero, isImagZero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
+    Value isZero = arith::AndIOp::create(b, isRealZero, isImagZero);
 
-    resultReal = b.create<arith::SelectOp>(isZero, inf, resultReal);
-    resultImag = b.create<arith::SelectOp>(isZero, nan, resultImag);
+    resultReal = arith::SelectOp::create(b, isZero, inf, resultReal);
+    resultImag = arith::SelectOp::create(b, isZero, nan, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -1021,9 +1021,9 @@ struct AngleOpConversion : public OpConversionPattern<complex::AngleOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, type, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, type, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, type, adaptor.getComplex());
+        complex::ImOp::create(rewriter, loc, type, adaptor.getComplex());
 
     rewriter.replaceOpWithNewOp<math::Atan2Op>(op, imag, real, fmf);
 
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index 88a8b7fb185c5..6a1f7c105b2c6 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -73,13 +73,13 @@ struct AssertOpLowering : public ConvertOpToLLVMPattern<cf::AssertOp> {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
         auto abortFuncTy = LLVM::LLVMFunctionType::get(getVoidType(), {});
-        abortFunc = rewriter.create<LLVM::LLVMFuncOp>(rewriter.getUnknownLoc(),
+        abortFunc = LLVM::LLVMFuncOp::create(rewriter, rewriter.getUnknownLoc(),
                                                       "abort", abortFuncTy);
       }
-      rewriter.create<LLVM::CallOp>(loc, abortFunc, ValueRange());
-      rewriter.create<LLVM::UnreachableOp>(loc);
+      LLVM::CallOp::create(rewriter, loc, abortFunc, ValueRange());
+      LLVM::UnreachableOp::create(rewriter, loc);
     } else {
-      rewriter.create<LLVM::BrOp>(loc, ValueRange(), continuationBlock);
+      LLVM::BrOp::create(rewriter, loc, ValueRange(), continuationBlock);
     }
 
     // Generate assertion test.
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index 9831dcaaaccc8..aa468c29536c7 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -33,7 +33,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
     MutableArrayRef<Region> regions) {
   if (auto condBrOp = dyn_cast<cf::CondBranchOp>(controlFlowCondOp)) {
     assert(regions.size() == 2);
-    auto ifOp = builder.create<scf::IfOp>(controlFlowCondOp->getLoc(),
+    auto ifOp = scf::IfOp::create(builder, controlFlowCondOp->getLoc(),
                                           resultTypes, condBrOp.getCondition());
     ifOp.getThenRegion().takeBody(regions[0]);
     ifOp.getElseRegion().takeBody(regions[1]);
@@ -43,7 +43,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
   if (auto switchOp = dyn_cast<cf::SwitchOp>(controlFlowCondOp)) {
     // `getCFGSwitchValue` returns an i32 that we need to convert to index
     // fist.
-    auto cast = builder.create<arith::IndexCastUIOp>(
+    auto cast = arith::IndexCastUIOp::create(builder,
         controlFlowCondOp->getLoc(), builder.getIndexType(),
         switchOp.getFlag());
     SmallVector<int64_t> cases;
@@ -55,7 +55,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
 
     assert(regions.size() == cases.size() + 1);
 
-    auto indexSwitchOp = builder.create<scf::IndexSwitchOp>(
+    auto indexSwitchOp = scf::IndexSwitchOp::create(builder,
         controlFlowCondOp->getLoc(), resultTypes, cast, cases, cases.size());
 
     indexSwitchOp.getDefaultRegion().takeBody(regions[0]);
@@ -75,7 +75,7 @@ LogicalResult
 ControlFlowToSCFTransformation::createStructuredBranchRegionTerminatorOp(
     Location loc, OpBuilder &builder, Operation *branchRegionOp,
     Operation *replacedControlFlowOp, ValueRange results) {
-  builder.create<scf::YieldOp>(loc, results);
+  scf::YieldOp::create(builder, loc, results);
   return success();
 }
 
@@ -84,7 +84,7 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
     OpBuilder &builder, Operation *replacedOp, ValueRange loopVariablesInit,
     Value condition, ValueRange loopVariablesNextIter, Region &&loopBody) {
   Location loc = replacedOp->getLoc();
-  auto whileOp = builder.create<scf::WhileOp>(loc, loopVariablesInit.getTypes(),
+  auto whileOp = scf::WhileOp::create(builder, loc, loopVariablesInit.getTypes(),
                                               loopVariablesInit);
 
   whileOp.getBefore().takeBody(loopBody);
@@ -92,15 +92,15 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
   builder.setInsertionPointToEnd(&whileOp.getBefore().back());
   // `getCFGSwitchValue` returns a i32. We therefore need to truncate the
   // condition to i1 first. It is guaranteed to be either 0 or 1 already.
-  builder.create<scf::ConditionOp>(
-      loc, builder.create<arith::TruncIOp>(loc, builder.getI1Type(), condition),
+  scf::ConditionOp::create(builder,
+      loc, arith::TruncIOp::create(builder, loc, builder.getI1Type(), condition),
       loopVariablesNextIter);
 
   Block *afterBlock = builder.createBlock(&whileOp.getAfter());
   afterBlock->addArguments(
       loopVariablesInit.getTypes(),
       SmallVector<Location>(loopVariablesInit.size(), loc));
-  builder.create<scf::YieldOp>(loc, afterBlock->getArguments());
+  scf::YieldOp::create(builder, loc, afterBlock->getArguments());
 
   return whileOp.getOperation();
 }
@@ -108,7 +108,7 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
 Value ControlFlowToSCFTransformation::getCFGSwitchValue(Location loc,
                                                         OpBuilder &builder,
                                                         unsigned int value) {
-  return builder.create<arith::ConstantOp>(loc,
+  return arith::ConstantOp::create(builder, loc,
                                            builder.getI32IntegerAttr(value));
 }
 
@@ -117,7 +117,7 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
     ArrayRef<unsigned int> caseValues, BlockRange caseDestinations,
     ArrayRef<ValueRange> caseArguments, Block *defaultDest,
     ValueRange defaultArgs) {
-  builder.create<cf::SwitchOp>(loc, flag, defaultDest, defaultArgs,
+  cf::SwitchOp::create(builder, loc, flag, defaultDest, defaultArgs,
                                llvm::to_vector_of<int32_t>(caseValues),
                                caseDestinations, caseArguments);
 }
@@ -125,7 +125,7 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
 Value ControlFlowToSCFTransformation::getUndefValue(Location loc,
                                                     OpBuilder &builder,
                                                     Type type) {
-  return builder.create<ub::PoisonOp>(loc, type, nullptr);
+  return ub::PoisonOp::create(builder, loc, type, nullptr);
 }
 
 FailureOr<Operation *>
diff --git a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
index f8dc06f41ab87..20167f168185a 100644
--- a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
+++ b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
@@ -99,7 +99,7 @@ class FuncOpConversion final : public OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted `emitc.func` op.
-    emitc::FuncOp newFuncOp = rewriter.create<emitc::FuncOp>(
+    emitc::FuncOp newFuncOp = emitc::FuncOp::create(rewriter,
         funcOp.getLoc(), funcOp.getName(),
         FunctionType::get(rewriter.getContext(),
                           signatureConverter.getConvertedTypes(),
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 36235636d6ba2..51dbe6cbb7de3 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -115,7 +115,7 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   SmallVector<NamedAttribute> attributes;
   filterFuncAttributes(funcOp, attributes);
 
-  auto wrapperFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto wrapperFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperFuncType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
@@ -129,13 +129,13 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   for (auto [index, argType] : llvm::enumerate(type.getInputs())) {
     Value arg = wrapperFuncOp.getArgument(index + argOffset);
     if (auto memrefType = dyn_cast<MemRefType>(argType)) {
-      Value loaded = rewriter.create<LLVM::LoadOp>(
+      Value loaded = LLVM::LoadOp::create(rewriter,
           loc, typeConverter.convertType(memrefType), arg);
       MemRefDescriptor::unpack(rewriter, loc, loaded, memrefType, args);
       continue;
     }
     if (isa<UnrankedMemRefType>(argType)) {
-      Value loaded = rewriter.create<LLVM::LoadOp>(
+      Value loaded = LLVM::LoadOp::create(rewriter,
           loc, typeConverter.convertType(argType), arg);
       UnrankedMemRefDescriptor::unpack(rewriter, loc, loaded, args);
       continue;
@@ -144,14 +144,14 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
     args.push_back(arg);
   }
 
-  auto call = rewriter.create<LLVM::CallOp>(loc, newFuncOp, args);
+  auto call = LLVM::CallOp::create(rewriter, loc, newFuncOp, args);
 
   if (resultStructType) {
-    rewriter.create<LLVM::StoreOp>(loc, call.getResult(),
+    LLVM::StoreOp::create(rewriter, loc, call.getResult(),
                                    wrapperFuncOp.getArgument(0));
-    rewriter.create<LLVM::ReturnOp>(loc, ValueRange{});
+    LLVM::ReturnOp::create(rewriter, loc, ValueRange{});
   } else {
-    rewriter.create<LLVM::ReturnOp>(loc, call.getResults());
+    LLVM::ReturnOp::create(rewriter, loc, call.getResults());
   }
 }
 
@@ -182,7 +182,7 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   filterFuncAttributes(funcOp, attributes);
 
   // Create the auxiliary function.
-  auto wrapperFunc = builder.create<LLVM::LLVMFuncOp>(
+  auto wrapperFunc = LLVM::LLVMFuncOp::create(builder,
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
@@ -201,11 +201,11 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   if (resultStructType) {
     // Allocate the struct on the stack and pass the pointer.
     Type resultType = cast<LLVM::LLVMFunctionType>(wrapperType).getParamType(0);
-    Value one = builder.create<LLVM::ConstantOp>(
+    Value one = LLVM::ConstantOp::create(builder,
         loc, typeConverter.convertType(builder.getIndexType()),
         builder.getIntegerAttr(builder.getIndexType(), 1));
     Value result =
-        builder.create<LLVM::AllocaOp>(loc, resultType, resultStructType, one);
+        LLVM::AllocaOp::create(builder, loc, resultType, resultStructType, one);
     args.push_back(result);
   }
 
@@ -229,12 +229,12 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
                     wrapperArgsRange.take_front(numToDrop));
 
       auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext());
-      Value one = builder.create<LLVM::ConstantOp>(
+      Value one = LLVM::ConstantOp::create(builder,
           loc, typeConverter.convertType(builder.getIndexType()),
           builder.getIntegerAttr(builder.getIndexType(), 1));
-      Value allocated = builder.create<LLVM::AllocaOp>(
+      Value allocated = LLVM::AllocaOp::create(builder,
           loc, ptrTy, packed.getType(), one, /*alignment=*/0);
-      builder.create<LLVM::StoreOp>(loc, packed, allocated);
+      LLVM::StoreOp::create(builder, loc, packed, allocated);
       arg = allocated;
     } else {
       arg = wrapperArgsRange[0];
@@ -245,14 +245,14 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   }
   assert(wrapperArgsRange.empty() && "did not map some of the arguments");
 
-  auto call = builder.create<LLVM::CallOp>(loc, wrapperFunc, args);
+  auto call = LLVM::CallOp::create(builder, loc, wrapperFunc, args);
 
   if (resultStructType) {
     Value result =
-        builder.create<LLVM::LoadOp>(loc, resultStructType, args.front());
-    builder.create<LLVM::ReturnOp>(loc, result);
+        LLVM::LoadOp::create(builder, loc, resultStructType, args.front());
+    LLVM::ReturnOp::create(builder, loc, result);
   } else {
-    builder.create<LLVM::ReturnOp>(loc, call.getResults());
+    LLVM::ReturnOp::create(builder, loc, call.getResults());
   }
 }
 
@@ -283,7 +283,7 @@ static void restoreByValRefArgumentType(
     Type resTy = typeConverter.convertType(
         cast<TypeAttr>(byValRefAttr->getValue()).getValue());
 
-    Value valueArg = rewriter.create<LLVM::LoadOp>(arg.getLoc(), resTy, arg);
+    Value valueArg = LLVM::LoadOp::create(rewriter, arg.getLoc(), resTy, arg);
     rewriter.replaceUsesOfBlockArgument(arg, valueArg);
   }
 }
@@ -357,7 +357,7 @@ FailureOr<LLVM::LLVMFuncOp> mlir::convertFuncOpToLLVMFuncOp(
     symbolTable.remove(funcOp);
   }
 
-  auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
       /*dsoLocal=*/false, /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr,
       attributes);
@@ -509,7 +509,7 @@ struct ConstantOpLowering : public ConvertOpToLLVMPattern<func::ConstantOp> {
       return rewriter.notifyMatchFailure(op, "failed to convert result type");
 
     auto newOp =
-        rewriter.create<LLVM::AddressOfOp>(op.getLoc(), type, op.getValue());
+        LLVM::AddressOfOp::create(rewriter, op.getLoc(), type, op.getValue());
     for (const NamedAttribute &attr : op->getAttrs()) {
       if (attr.getName().strref() == "value")
         continue;
@@ -556,7 +556,7 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
     auto promoted = this->getTypeConverter()->promoteOperands(
         callOp.getLoc(), /*opOperands=*/callOp->getOperands(),
         adaptor.getOperands(), rewriter, useBarePtrCallConv);
-    auto newOp = rewriter.create<LLVM::CallOp>(
+    auto newOp = LLVM::CallOp::create(rewriter,
         callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
         promoted, callOp->getAttrs());
 
@@ -573,7 +573,7 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
       // Extract individual results from the structure and return them as list.
       results.reserve(numResults);
       for (unsigned i = 0; i < numResults; ++i) {
-        results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        results.push_back(LLVM::ExtractValueOp::create(rewriter,
             callOp.getLoc(), newOp->getResult(0), i));
       }
     }
@@ -726,9 +726,9 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<func::ReturnOp> {
       return rewriter.notifyMatchFailure(op, "could not convert result types");
     }
 
-    Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
+    Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
     for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
-      packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
+      packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
     }
     rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, TypeRange(), packed,
                                                 op->getAttrs());
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index 01ca5e99a9aff..f5d5460412da2 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -28,7 +28,7 @@ LLVM::LLVMFuncOp mlir::getOrDefineFunction(gpu::GPUModuleOp moduleOp,
   if (!(ret = moduleOp.template lookupSymbol<LLVM::LLVMFuncOp>(name))) {
     OpBuilder::InsertionGuard guard(b);
     b.setInsertionPointToStart(moduleOp.getBody());
-    ret = b.create<LLVM::LLVMFuncOp>(loc, name, type, LLVM::Linkage::External);
+    ret = LLVM::LLVMFuncOp::create(b, loc, name, type, LLVM::Linkage::External);
   }
   return ret;
 }
@@ -68,7 +68,7 @@ mlir::getOrCreateStringConstant(OpBuilder &b, Location loc,
   OpBuilder::InsertionGuard guard(b);
   b.setInsertionPointToStart(moduleOp.getBody());
   SmallString<16> name = getUniqueSymbolName(moduleOp, namePrefix);
-  return b.create<LLVM::GlobalOp>(loc, globalType,
+  return LLVM::GlobalOp::create(b, loc, globalType,
                                   /*isConstant=*/true, LLVM::Linkage::Internal,
                                   name, attr, alignment, addrSpace);
 }
@@ -151,7 +151,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
               gpuFuncOp.getWorkgroupAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      auto globalOp = rewriter.create<LLVM::GlobalOp>(
+      auto globalOp = LLVM::GlobalOp::create(rewriter,
           gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
           LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment,
           workgroupAddrSpace);
@@ -220,7 +220,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
   LLVM::CConv callingConvention = gpuFuncOp.isKernel()
                                       ? kernelCallingConvention
                                       : nonKernelCallingConvention;
-  auto llvmFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto llvmFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
       LLVM::Linkage::External, /*dsoLocal=*/false, callingConvention,
       /*comdat=*/nullptr, attributes);
@@ -266,10 +266,10 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       for (const auto [idx, global] : llvm::enumerate(workgroupBuffers)) {
         auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext(),
                                                   global.getAddrSpace());
-        Value address = rewriter.create<LLVM::AddressOfOp>(
+        Value address = LLVM::AddressOfOp::create(rewriter,
             loc, ptrType, global.getSymNameAttr());
         Value memory =
-            rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getType(),
+            LLVM::GEPOp::create(rewriter, loc, ptrType, global.getType(),
                                          address, ArrayRef<LLVM::GEPArg>{0, 0});
 
         // Build a memref descriptor pointing to the buffer to plug with the
@@ -298,14 +298,14 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       Type elementType = typeConverter->convertType(type.getElementType());
       auto ptrType =
           LLVM::LLVMPointerType::get(rewriter.getContext(), allocaAddrSpace);
-      Value numElements = rewriter.create<LLVM::ConstantOp>(
+      Value numElements = LLVM::ConstantOp::create(rewriter,
           gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
       uint64_t alignment = 0;
       if (auto alignAttr =
               dyn_cast_or_null<IntegerAttr>(gpuFuncOp.getPrivateAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      Value allocated = rewriter.create<LLVM::AllocaOp>(
+      Value allocated = LLVM::AllocaOp::create(rewriter,
           gpuFuncOp.getLoc(), ptrType, elementType, numElements, alignment);
       Value descr = MemRefDescriptor::fromStaticShape(
           rewriter, loc, *getTypeConverter(), type, allocated);
@@ -418,8 +418,8 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
           {llvmI64, ptrType, /*length (bytes)*/ llvmI64, /*isLast*/ llvmI32}));
 
   /// Start the printf hostcall
-  Value zeroI64 = rewriter.create<LLVM::ConstantOp>(loc, llvmI64, 0);
-  auto printfBeginCall = rewriter.create<LLVM::CallOp>(loc, ocklBegin, zeroI64);
+  Value zeroI64 = LLVM::ConstantOp::create(rewriter, loc, llvmI64, 0);
+  auto printfBeginCall = LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
   Value printfDesc = printfBeginCall.getResult();
 
   // Create the global op or find an existing one.
@@ -427,20 +427,20 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
       rewriter, loc, moduleOp, llvmI8, "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element and pass it to printf()
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
       loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
-  Value stringLen = rewriter.create<LLVM::ConstantOp>(
+  Value stringLen = LLVM::ConstantOp::create(rewriter,
       loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
 
-  Value oneI32 = rewriter.create<LLVM::ConstantOp>(loc, llvmI32, 1);
-  Value zeroI32 = rewriter.create<LLVM::ConstantOp>(loc, llvmI32, 0);
+  Value oneI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 1);
+  Value zeroI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 0);
 
-  auto appendFormatCall = rewriter.create<LLVM::CallOp>(
+  auto appendFormatCall = LLVM::CallOp::create(rewriter,
       loc, ocklAppendStringN,
       ValueRange{printfDesc, stringStart, stringLen,
                  adaptor.getArgs().empty() ? oneI32 : zeroI32});
@@ -456,17 +456,17 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
     SmallVector<mlir::Value, 2 + argsPerAppend + 1> arguments;
     arguments.push_back(printfDesc);
     arguments.push_back(
-        rewriter.create<LLVM::ConstantOp>(loc, llvmI32, numArgsThisCall));
+        LLVM::ConstantOp::create(rewriter, loc, llvmI32, numArgsThisCall));
     for (size_t i = group; i < bound; ++i) {
       Value arg = adaptor.getArgs()[i];
       if (auto floatType = dyn_cast<FloatType>(arg.getType())) {
         if (!floatType.isF64())
-          arg = rewriter.create<LLVM::FPExtOp>(
+          arg = LLVM::FPExtOp::create(rewriter,
               loc, typeConverter->convertType(rewriter.getF64Type()), arg);
-        arg = rewriter.create<LLVM::BitcastOp>(loc, llvmI64, arg);
+        arg = LLVM::BitcastOp::create(rewriter, loc, llvmI64, arg);
       }
       if (arg.getType().getIntOrFloatBitWidth() != 64)
-        arg = rewriter.create<LLVM::ZExtOp>(loc, llvmI64, arg);
+        arg = LLVM::ZExtOp::create(rewriter, loc, llvmI64, arg);
 
       arguments.push_back(arg);
     }
@@ -477,7 +477,7 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
 
     auto isLast = (bound == nArgs) ? oneI32 : zeroI32;
     arguments.push_back(isLast);
-    auto call = rewriter.create<LLVM::CallOp>(loc, ocklAppendArgs, arguments);
+    auto call = LLVM::CallOp::create(rewriter, loc, ocklAppendArgs, arguments);
     printfDesc = call.getResult();
   }
   rewriter.eraseOp(gpuPrintfOp);
@@ -510,12 +510,12 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
       /*alignment=*/0, addressSpace);
 
   // Get a pointer to the format string's first element
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
       loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
 
   // Construct arguments and function call
@@ -525,7 +525,7 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
   printfArgs.push_back(stringStart);
   printfArgs.append(argsRange.begin(), argsRange.end());
 
-  rewriter.create<LLVM::CallOp>(loc, printfDecl, printfArgs);
+  LLVM::CallOp::create(rewriter, loc, printfDecl, printfArgs);
   rewriter.eraseOp(gpuPrintfOp);
   return success();
 }
@@ -559,9 +559,9 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
                                 "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(loc, global);
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter, loc, global);
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
   SmallVector<Type> types;
   SmallVector<Value> args;
@@ -572,27 +572,27 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
     assert(type.isIntOrFloat());
     if (isa<FloatType>(type)) {
       type = rewriter.getF64Type();
-      promotedArg = rewriter.create<LLVM::FPExtOp>(loc, type, arg);
+      promotedArg = LLVM::FPExtOp::create(rewriter, loc, type, arg);
     }
     types.push_back(type);
     args.push_back(promotedArg);
   }
   Type structType =
       LLVM::LLVMStructType::getLiteral(gpuPrintfOp.getContext(), types);
-  Value one = rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
+  Value one = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
                                                 rewriter.getIndexAttr(1));
   Value tempAlloc =
-      rewriter.create<LLVM::AllocaOp>(loc, ptrType, structType, one,
+      LLVM::AllocaOp::create(rewriter, loc, ptrType, structType, one,
                                       /*alignment=*/0);
   for (auto [index, arg] : llvm::enumerate(args)) {
-    Value ptr = rewriter.create<LLVM::GEPOp>(
+    Value ptr = LLVM::GEPOp::create(rewriter,
         loc, ptrType, structType, tempAlloc,
         ArrayRef<LLVM::GEPArg>{0, static_cast<int32_t>(index)});
-    rewriter.create<LLVM::StoreOp>(loc, arg, ptr);
+    LLVM::StoreOp::create(rewriter, loc, arg, ptr);
   }
   std::array<Value, 2> printfArgs = {stringStart, tempAlloc};
 
-  rewriter.create<LLVM::CallOp>(loc, vprintfDecl, printfArgs);
+  LLVM::CallOp::create(rewriter, loc, vprintfDecl, printfArgs);
   rewriter.eraseOp(gpuPrintfOp);
   return success();
 }
@@ -607,22 +607,22 @@ static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands,
   TypeRange operandTypes(operands);
   VectorType vectorType = cast<VectorType>(llvm1DVectorTy);
   Location loc = op->getLoc();
-  Value result = rewriter.create<LLVM::PoisonOp>(loc, vectorType);
+  Value result = LLVM::PoisonOp::create(rewriter, loc, vectorType);
   Type indexType = converter.convertType(rewriter.getIndexType());
   StringAttr name = op->getName().getIdentifier();
   Type elementType = vectorType.getElementType();
 
   for (int64_t i = 0; i < vectorType.getNumElements(); ++i) {
-    Value index = rewriter.create<LLVM::ConstantOp>(loc, indexType, i);
+    Value index = LLVM::ConstantOp::create(rewriter, loc, indexType, i);
     auto extractElement = [&](Value operand) -> Value {
       if (!isa<VectorType>(operand.getType()))
         return operand;
-      return rewriter.create<LLVM::ExtractElementOp>(loc, operand, index);
+      return LLVM::ExtractElementOp::create(rewriter, loc, operand, index);
     };
     auto scalarOperands = llvm::map_to_vector(operands, extractElement);
     Operation *scalarOp =
         rewriter.create(loc, name, scalarOperands, elementType, op->getAttrs());
-    result = rewriter.create<LLVM::InsertElementOp>(
+    result = LLVM::InsertElementOp::create(rewriter,
         loc, result, scalarOp->getResult(0), index);
   }
   return result;
@@ -705,7 +705,7 @@ LLVM::GlobalOp getDynamicSharedMemorySymbol(
   auto zeroSizedArrayType = LLVM::LLVMArrayType::get(
       typeConverter->convertType(memrefType.getElementType()), 0);
 
-  return rewriter.create<LLVM::GlobalOp>(
+  return LLVM::GlobalOp::create(rewriter,
       op->getLoc(), zeroSizedArrayType, /*isConstant=*/false,
       LLVM::Linkage::Internal, symName, /*value=*/Attribute(), alignmentByte,
       addressSpace.value());
@@ -732,12 +732,12 @@ LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
   // Step 3. Get address of the global symbol
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(op);
-  auto basePtr = rewriter.create<LLVM::AddressOfOp>(loc, shmemOp);
+  auto basePtr = LLVM::AddressOfOp::create(rewriter, loc, shmemOp);
   Type baseType = basePtr->getResultTypes().front();
 
   // Step 4. Generate GEP using offsets
   SmallVector<LLVM::GEPArg> gepArgs = {0};
-  Value shmemPtr = rewriter.create<LLVM::GEPOp>(loc, baseType, elementType,
+  Value shmemPtr = LLVM::GEPOp::create(rewriter, loc, baseType, elementType,
                                                 basePtr, gepArgs);
   // Step 5. Create a memref descriptor
   SmallVector<Value> shape, strides;
@@ -799,9 +799,9 @@ LogicalResult GPUReturnOpLowering::matchAndRewrite(
     return rewriter.notifyMatchFailure(op, "could not convert result types");
   }
 
-  Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
+  Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
   for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
-    packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
+    packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
   }
   rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, TypeRange(), packed,
                                               op->getAttrs());
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 167cabbc57db9..5e8254a0b6035 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -79,7 +79,7 @@ class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern<OpTy> {
     uint64_t rank = type.getRank();
     Value numElements = desc.size(rewriter, loc, /*pos=*/0);
     for (unsigned i = 1; i < rank; i++)
-      numElements = rewriter.create<LLVM::MulOp>(
+      numElements = LLVM::MulOp::create(rewriter,
           loc, numElements, desc.size(rewriter, loc, /*pos=*/i));
     return numElements;
   }
@@ -582,7 +582,7 @@ LLVM::CallOp FunctionCallBuilder::create(Location loc, OpBuilder &builder,
     return OpBuilder::atBlockEnd(module.getBody())
         .create<LLVM::LLVMFuncOp>(loc, functionName, functionType);
   }();
-  return builder.create<LLVM::CallOp>(loc, function, arguments);
+  return LLVM::CallOp::create(builder, loc, function, arguments);
 }
 
 // Corresponding to cusparseIndexType_t defined in cusparse.h.
@@ -780,12 +780,12 @@ LogicalResult ConvertAllocOpToGpuRuntimeCallPattern::matchAndRewrite(
 
   // Allocate the underlying buffer and store a pointer to it in the MemRef
   // descriptor.
-  auto nullPtr = rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPointerType);
+  auto nullPtr = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPointerType);
   Value stream = adaptor.getAsyncDependencies().empty()
                      ? nullPtr
                      : adaptor.getAsyncDependencies().front();
 
-  auto isHostShared = rewriter.create<mlir::LLVM::ConstantOp>(
+  auto isHostShared = mlir::LLVM::ConstantOp::create(rewriter,
       loc, llvmInt8Type, rewriter.getI8IntegerAttr(isShared));
 
   Value allocatedPtr =
@@ -1012,7 +1012,7 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
       uint64_t staticSize = static_cast<uint64_t>(bitwidth / 8) *
                             static_cast<uint64_t>(memrefTy.getNumElements());
 
-      Value sizeArg = rewriter.create<LLVM::ConstantOp>(
+      Value sizeArg = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(staticSize));
       llvmArgumentsWithSizes.push_back(llvmArg); // Presumably a bare pointer.
       llvmArgumentsWithSizes.push_back(sizeArg);
@@ -1025,7 +1025,7 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
         gpu::KernelDim3{adaptor.getClusterSizeX(), adaptor.getClusterSizeY(),
                         adaptor.getClusterSizeZ()};
   }
-  rewriter.create<gpu::LaunchFuncOp>(
+  gpu::LaunchFuncOp::create(rewriter,
       launchOp.getLoc(), launchOp.getKernelAttr(),
       gpu::KernelDim3{adaptor.getGridSizeX(), adaptor.getGridSizeY(),
                       adaptor.getGridSizeZ()},
@@ -1048,7 +1048,7 @@ static Value bitAndAddrspaceCast(Location loc,
                                  const LLVMTypeConverter &typeConverter) {
   auto sourceTy = cast<LLVM::LLVMPointerType>(sourcePtr.getType());
   if (destinationType.getAddressSpace() != sourceTy.getAddressSpace())
-    sourcePtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+    sourcePtr = LLVM::AddrSpaceCastOp::create(rewriter,
         loc,
         LLVM::LLVMPointerType::get(rewriter.getContext(),
                                    destinationType.getAddressSpace()),
@@ -1072,13 +1072,13 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite(
   Value numElements = getNumElements(rewriter, loc, memRefType, srcDesc);
 
   Type elementPtrType = getElementPtrType(memRefType);
-  Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, elementPtrType);
-  Value gepPtr = rewriter.create<LLVM::GEPOp>(
+  Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
+  Value gepPtr = LLVM::GEPOp::create(rewriter,
       loc, elementPtrType,
       typeConverter->convertType(memRefType.getElementType()), nullPtr,
       numElements);
   auto sizeBytes =
-      rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);
+      LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
 
   auto src = bitAndAddrspaceCast(loc, rewriter, llvmPointerType,
                                  srcDesc.alignedPtr(rewriter, loc),
@@ -1123,7 +1123,7 @@ LogicalResult ConvertMemsetOpToGpuRuntimeCallPattern::matchAndRewrite(
   Value numElements = getNumElements(rewriter, loc, memRefType, dstDesc);
 
   auto value =
-      rewriter.create<LLVM::BitcastOp>(loc, bitCastType, adaptor.getValue());
+      LLVM::BitcastOp::create(rewriter, loc, bitCastType, adaptor.getValue());
   auto dst = bitAndAddrspaceCast(loc, rewriter, llvmPointerType,
                                  dstDesc.alignedPtr(rewriter, loc),
                                  *getTypeConverter());
@@ -1150,14 +1150,14 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite(
 template <typename T>
 static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmInt32Type = builder.getIntegerType(32);
-  return builder.create<LLVM::ConstantOp>(loc, llvmInt32Type,
+  return LLVM::ConstantOp::create(builder, loc, llvmInt32Type,
                                           static_cast<int32_t>(tValue));
 }
 
 template <typename T>
 static Value genConstFloat32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmFloat32Type = builder.getF32Type();
-  return builder.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(builder,
       loc, llvmFloat32Type,
       builder.getF32FloatAttr(static_cast<float>(tValue)));
 }
@@ -1189,11 +1189,11 @@ LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite(
   // the dnmat is used with spmat with 2:4 sparsity
   if (dims.size() == 2) {
     if (isSpMMCusparseLtOp(op.getDnTensor())) {
-      auto handleSz = rewriter.create<LLVM::ConstantOp>(
+      auto handleSz = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(11032));
-      handle = rewriter.create<LLVM::AllocaOp>(
+      handle = LLVM::AllocaOp::create(rewriter,
           loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
-      handle = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, handle);
+      handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
       createLtDnMatCallBuilder
           .create(loc, rewriter,
@@ -1351,11 +1351,11 @@ LogicalResult ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto dtp = genConstInt32From(rewriter, loc, getCuSparseDataTypeFrom(dType));
 
   // CUDA runner asserts the size is 44104 bytes.
-  auto handleSz = rewriter.create<LLVM::ConstantOp>(
+  auto handleSz = LLVM::ConstantOp::create(rewriter,
       loc, getIndexType(), rewriter.getIndexAttr(44104));
-  Value handle = rewriter.create<LLVM::AllocaOp>(
+  Value handle = LLVM::AllocaOp::create(rewriter,
       loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
-  handle = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, handle);
+  handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
   create2To4SpMatCallBuilder
       .create(loc, rewriter,
@@ -1441,9 +1441,9 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
         genConstInt32From(rewriter, loc, get2To4PruneFlag(op.getSpmatA()));
     auto computeType = genConstInt32From(
         rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType()));
-    auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+    auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(3));
-    auto bufferSize = rewriter.create<LLVM::AllocaOp>(
+    auto bufferSize = LLVM::AllocaOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, three, /*alignment=*/16);
     createCuSparseLtSpMMBufferSizeBuilder
         .create(loc, rewriter,
@@ -1452,20 +1452,20 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
                  pruneFlag, stream})
         .getResult();
 
-    auto bufferSizePtr1 = rewriter.create<LLVM::GEPOp>(
+    auto bufferSizePtr1 = LLVM::GEPOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{rewriter.create<LLVM::ConstantOp>(
+        ValueRange{LLVM::ConstantOp::create(rewriter,
             loc, getIndexType(), rewriter.getIndexAttr(1))});
-    auto bufferSizePtr2 = rewriter.create<LLVM::GEPOp>(
+    auto bufferSizePtr2 = LLVM::GEPOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{rewriter.create<LLVM::ConstantOp>(
+        ValueRange{LLVM::ConstantOp::create(rewriter,
             loc, getIndexType(), rewriter.getIndexAttr(2))});
     auto bufferSize0 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSize);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSize);
     auto bufferSize1 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr1);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSizePtr1);
     auto bufferSize2 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr2);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSizePtr2);
 
     rewriter.replaceOp(op, {bufferSize0, bufferSize1, bufferSize2, stream});
   } else {
@@ -1669,28 +1669,28 @@ LogicalResult ConvertSpMatGetSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
   Location loc = op.getLoc();
   auto stream = adaptor.getAsyncDependencies().front();
 
-  auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+  auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                  rewriter.getIndexAttr(3));
-  auto buffer = rewriter.create<LLVM::AllocaOp>(
+  auto buffer = LLVM::AllocaOp::create(rewriter,
       loc, llvmPointerType, llvmInt64Type, three, /*alignment=*/16);
 
-  auto rowsPtr = rewriter.create<LLVM::GEPOp>(
+  auto rowsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(0))});
-  auto colsPtr = rewriter.create<LLVM::GEPOp>(
+  auto colsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(1))});
-  auto nnzsPtr = rewriter.create<LLVM::GEPOp>(
+  auto nnzsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(2))});
   createSpMatGetSizeBuilder.create(
       loc, rewriter, {adaptor.getSpmat(), rowsPtr, colsPtr, nnzsPtr, stream});
-  auto rows = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, rowsPtr);
-  auto cols = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, colsPtr);
-  auto nnzs = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, nnzsPtr);
+  auto rows = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, rowsPtr);
+  auto cols = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, colsPtr);
+  auto nnzs = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, nnzsPtr);
 
   rewriter.replaceOp(op, {rows, cols, nnzs, stream});
   return success();
diff --git a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
index aab2409ed6328..70964727351a9 100644
--- a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
@@ -59,13 +59,13 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
     Operation *newOp;
     switch (op.getDimension()) {
     case gpu::Dimension::x:
-      newOp = rewriter.create<XOp>(loc, IntegerType::get(context, 32));
+      newOp = XOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     case gpu::Dimension::y:
-      newOp = rewriter.create<YOp>(loc, IntegerType::get(context, 32));
+      newOp = YOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     case gpu::Dimension::z:
-      newOp = rewriter.create<ZOp>(loc, IntegerType::get(context, 32));
+      newOp = ZOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     }
 
@@ -124,10 +124,10 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
                                   rewriter.getContext(), 32, min, max));
     }
     if (indexBitwidth > 32) {
-      newOp = rewriter.create<LLVM::SExtOp>(
+      newOp = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
     } else if (indexBitwidth < 32) {
-      newOp = rewriter.create<LLVM::TruncOp>(
+      newOp = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
     }
 
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index 64cf09e600b88..33b09b06649b9 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -103,7 +103,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
     LLVMFuncOp funcOp = appendOrGetFuncOp(funcName, funcType, op);
     auto callOp =
-        rewriter.create<LLVM::CallOp>(op->getLoc(), funcOp, castedOperands);
+        LLVM::CallOp::create(rewriter, op->getLoc(), funcOp, castedOperands);
 
     if (resultType == adaptor.getOperands().front().getType()) {
       rewriter.replaceOp(op, {callOp.getResult()});
@@ -115,10 +115,10 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // there is no guarantee of a specific value being used to indicate true,
     // compare for inequality with zero (rather than truncate or shift).
     if (isResultBool) {
-      Value zero = rewriter.create<LLVM::ConstantOp>(
+      Value zero = LLVM::ConstantOp::create(rewriter,
           op->getLoc(), rewriter.getIntegerType(32),
           rewriter.getI32IntegerAttr(0));
-      Value truncated = rewriter.create<LLVM::ICmpOp>(
+      Value truncated = LLVM::ICmpOp::create(rewriter,
           op->getLoc(), LLVM::ICmpPredicate::ne, callOp.getResult(), zero);
       rewriter.replaceOp(op, {truncated});
       return success();
@@ -126,7 +126,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
     assert(callOp.getResult().getType().isF32() &&
            "only f32 types are supposed to be truncated back");
-    Value truncated = rewriter.create<LLVM::FPTruncOp>(
+    Value truncated = LLVM::FPTruncOp::create(rewriter,
         op->getLoc(), adaptor.getOperands().front().getType(),
         callOp.getResult());
     rewriter.replaceOp(op, {truncated});
@@ -142,7 +142,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     if (!f16Func.empty() && isa<Float16Type>(type))
       return operand;
 
-    return rewriter.create<LLVM::FPExtOp>(
+    return LLVM::FPExtOp::create(rewriter,
         operand.getLoc(), Float32Type::get(rewriter.getContext()), operand);
   }
 
@@ -169,7 +169,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // location as debug info metadata inside of a function cannot be used
     // outside of that function.
     auto globalloc = op->getLoc()->findInstanceOfOrUnknown<FileLineColLoc>();
-    return b.create<LLVMFuncOp>(globalloc, funcName, funcType);
+    return LLVMFuncOp::create(b, globalloc, funcName, funcType);
   }
 
   StringRef getFunctionName(Type type, SourceOp op) const {
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index 8b6b553f6eed0..fc469102a54d2 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -54,7 +54,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
       SymbolTable::lookupSymbolIn(symbolTable, name));
   if (!func) {
     OpBuilder b(symbolTable->getRegion(0));
-    func = b.create<LLVM::LLVMFuncOp>(
+    func = LLVM::LLVMFuncOp::create(b,
         symbolTable->getLoc(), name,
         LLVM::LLVMFunctionType::get(resultType, paramTypes));
     func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
@@ -79,7 +79,7 @@ static LLVM::CallOp createSPIRVBuiltinCall(Location loc,
                                            ConversionPatternRewriter &rewriter,
                                            LLVM::LLVMFuncOp func,
                                            ValueRange args) {
-  auto call = rewriter.create<LLVM::CallOp>(loc, func, args);
+  auto call = LLVM::CallOp::create(rewriter, loc, func, args);
   call.setCConv(func.getCConv());
   call.setConvergentAttr(func.getConvergentAttr());
   call.setNoUnwindAttr(func.getNoUnwindAttr());
@@ -121,7 +121,7 @@ struct GPUBarrierConversion final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
     constexpr int64_t localMemFenceFlag = 1;
     Location loc = op->getLoc();
     Value flag =
-        rewriter.create<LLVM::ConstantOp>(loc, flagTy, localMemFenceFlag);
+        LLVM::ConstantOp::create(rewriter, loc, flagTy, localMemFenceFlag);
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, flag));
     return success();
   }
@@ -162,7 +162,7 @@ struct LaunchConfigConversion : ConvertToLLVMPattern {
 
     Location loc = op->getLoc();
     gpu::Dimension dim = getDimension(op);
-    Value dimVal = rewriter.create<LLVM::ConstantOp>(loc, dimTy,
+    Value dimVal = LLVM::ConstantOp::create(rewriter, loc, dimTy,
                                                      static_cast<int64_t>(dim));
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, dimVal));
     return success();
@@ -291,12 +291,12 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                                          ConversionPatternRewriter &rewriter) {
     return TypeSwitch<Type, Value>(oldVal.getType())
         .Case([&](BFloat16Type) {
-          return rewriter.create<LLVM::BitcastOp>(loc, rewriter.getI16Type(),
+          return LLVM::BitcastOp::create(rewriter, loc, rewriter.getI16Type(),
                                                   oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
-            return rewriter.create<LLVM::ZExtOp>(loc, rewriter.getI8Type(),
+            return LLVM::ZExtOp::create(rewriter, loc, rewriter.getI8Type(),
                                                  oldVal);
           return oldVal;
         })
@@ -308,11 +308,11 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                                           ConversionPatternRewriter &rewriter) {
     return TypeSwitch<Type, Value>(newTy)
         .Case([&](BFloat16Type) {
-          return rewriter.create<LLVM::BitcastOp>(loc, newTy, oldVal);
+          return LLVM::BitcastOp::create(rewriter, loc, newTy, oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
-            return rewriter.create<LLVM::TruncOp>(loc, newTy, oldVal);
+            return LLVM::TruncOp::create(rewriter, loc, newTy, oldVal);
           return oldVal;
         })
         .Default(oldVal);
@@ -349,7 +349,7 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
         bitcastOrTruncAfterShuffle(result, op.getType(0), loc, rewriter);
 
     Value trueVal =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI1Type(), true);
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getI1Type(), true);
     rewriter.replaceOp(op, {resultOrConversion, trueVal});
     return success();
   }
@@ -426,7 +426,7 @@ struct GPUSubgroupOpConversion final : ConvertOpToLLVMPattern<SubgroupOp> {
       if (indexTy.getIntOrFloatBitWidth() < resultTy.getIntOrFloatBitWidth()) {
         return failure();
       }
-      result = rewriter.create<LLVM::ZExtOp>(loc, indexTy, result);
+      result = LLVM::ZExtOp::create(rewriter, loc, indexTy, result);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 1ef6edea93c58..7a6601ea754e9 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -118,9 +118,9 @@ struct GPUSubgroupReduceOpLowering
 
     Location loc = op->getLoc();
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
-    Value offset = rewriter.create<LLVM::ConstantOp>(loc, int32Type, -1);
+    Value offset = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
 
-    auto reduxOp = rewriter.create<NVVM::ReduxOp>(loc, int32Type, op.getValue(),
+    auto reduxOp = NVVM::ReduxOp::create(rewriter, loc, int32Type, op.getValue(),
                                                   mode.value(), offset);
 
     rewriter.replaceOp(op, reduxOp->getResult(0));
@@ -158,13 +158,13 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     auto predTy = IntegerType::get(rewriter.getContext(), 1);
 
-    Value one = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 1);
-    Value minusOne = rewriter.create<LLVM::ConstantOp>(loc, int32Type, -1);
-    Value thirtyTwo = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 32);
-    Value numLeadInactiveLane = rewriter.create<LLVM::SubOp>(
+    Value one = LLVM::ConstantOp::create(rewriter, loc, int32Type, 1);
+    Value minusOne = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
+    Value thirtyTwo = LLVM::ConstantOp::create(rewriter, loc, int32Type, 32);
+    Value numLeadInactiveLane = LLVM::SubOp::create(rewriter,
         loc, int32Type, thirtyTwo, adaptor.getWidth());
     // Bit mask of active lanes: `(-1) >> (32 - activeWidth)`.
-    Value activeMask = rewriter.create<LLVM::LShrOp>(loc, int32Type, minusOne,
+    Value activeMask = LLVM::LShrOp::create(rewriter, loc, int32Type, minusOne,
                                                      numLeadInactiveLane);
     Value maskAndClamp;
     if (op.getMode() == gpu::ShuffleMode::UP) {
@@ -173,7 +173,7 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     } else {
       // Clamp lane: `activeWidth - 1`
       maskAndClamp =
-          rewriter.create<LLVM::SubOp>(loc, int32Type, adaptor.getWidth(), one);
+          LLVM::SubOp::create(rewriter, loc, int32Type, adaptor.getWidth(), one);
     }
 
     bool predIsUsed = !op->getResult(1).use_empty();
@@ -184,13 +184,13 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
       resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(),
                                                   {valueTy, predTy});
     }
-    Value shfl = rewriter.create<NVVM::ShflOp>(
+    Value shfl = NVVM::ShflOp::create(rewriter,
         loc, resultTy, activeMask, adaptor.getValue(), adaptor.getOffset(),
         maskAndClamp, convertShflKind(op.getMode()), returnValueAndIsValidAttr);
     if (predIsUsed) {
-      Value shflValue = rewriter.create<LLVM::ExtractValueOp>(loc, shfl, 0);
+      Value shflValue = LLVM::ExtractValueOp::create(rewriter, loc, shfl, 0);
       Value isActiveSrcLane =
-          rewriter.create<LLVM::ExtractValueOp>(loc, shfl, 1);
+          LLVM::ExtractValueOp::create(rewriter, loc, shfl, 1);
       rewriter.replaceOp(op, {shflValue, isActiveSrcLane});
     } else {
       rewriter.replaceOp(op, {shfl, nullptr});
@@ -215,15 +215,15 @@ struct GPULaneIdOpToNVVM : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
       bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
           /*bitWidth=*/32, /*lower=*/0, /*upper=*/kWarpSize);
     Value newOp =
-        rewriter.create<NVVM::LaneIdOp>(loc, rewriter.getI32Type(), bounds);
+        NVVM::LaneIdOp::create(rewriter, loc, rewriter.getI32Type(), bounds);
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      newOp = rewriter.create<LLVM::SExtOp>(
+      newOp = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp);
     } else if (indexBitwidth < 32) {
-      newOp = rewriter.create<LLVM::TruncOp>(
+      newOp = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp);
     }
     rewriter.replaceOp(op, {newOp});
@@ -271,10 +271,10 @@ struct AssertOpToAssertfailLowering
     Block *afterBlock =
         rewriter.splitBlock(assertBlock, ++assertOp->getIterator());
     rewriter.setInsertionPointToEnd(beforeBlock);
-    rewriter.create<cf::CondBranchOp>(loc, adaptor.getArg(), afterBlock,
+    cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(), afterBlock,
                                       assertBlock);
     rewriter.setInsertionPointToEnd(assertBlock);
-    rewriter.create<cf::BranchOp>(loc, afterBlock);
+    cf::BranchOp::create(rewriter, loc, afterBlock);
 
     // Continue cf.assert lowering.
     rewriter.setInsertionPoint(assertOp);
@@ -301,11 +301,11 @@ struct AssertOpToAssertfailLowering
     // Create constants.
     auto getGlobal = [&](LLVM::GlobalOp global) {
       // Get a pointer to the format string's first element.
-      Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+      Value globalPtr = LLVM::AddressOfOp::create(rewriter,
           loc, LLVM::LLVMPointerType::get(ctx, global.getAddrSpace()),
           global.getSymNameAttr());
       Value start =
-          rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+          LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                        globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
       return start;
     };
@@ -316,8 +316,8 @@ struct AssertOpToAssertfailLowering
     Value assertFunc = getGlobal(getOrCreateStringConstant(
         rewriter, loc, moduleOp, i8Type, "assert_func_", funcName));
     Value assertLine =
-        rewriter.create<LLVM::ConstantOp>(loc, i32Type, fileLine);
-    Value c1 = rewriter.create<LLVM::ConstantOp>(loc, i64Type, 1);
+        LLVM::ConstantOp::create(rewriter, loc, i32Type, fileLine);
+    Value c1 = LLVM::ConstantOp::create(rewriter, loc, i64Type, 1);
 
     // Insert function call to __assertfail.
     SmallVector<Value> arguments{assertMessage, assertFile, assertLine,
diff --git a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
index 45fd933d58857..7d5f65c470386 100644
--- a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
@@ -126,7 +126,7 @@ struct WmmaLoadOpToNVVMLowering
         cast<MemRefType>(subgroupMmaLoadMatrixOp.getSrcMemref().getType()),
         adaptor.getSrcMemref(), adaptor.getIndices());
 
-    Value leadingDim = rewriter.create<LLVM::ConstantOp>(
+    Value leadingDim = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI32Type(),
         subgroupMmaLoadMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMALoadOp>(
@@ -173,7 +173,7 @@ struct WmmaStoreOpToNVVMLowering
     auto matrixType = cast<LLVM::LLVMStructType>(adaptor.getSrc().getType());
     for (unsigned i = 0, e = matrixType.getBody().size(); i < e; ++i) {
       Value toUse =
-          rewriter.create<LLVM::ExtractValueOp>(loc, adaptor.getSrc(), i);
+          LLVM::ExtractValueOp::create(rewriter, loc, adaptor.getSrc(), i);
       storeOpOperands.push_back(toUse);
     }
 
@@ -181,7 +181,7 @@ struct WmmaStoreOpToNVVMLowering
         rewriter, loc,
         cast<MemRefType>(subgroupMmaStoreMatrixOp.getDstMemref().getType()),
         adaptor.getDstMemref(), adaptor.getIndices());
-    Value leadingDim = rewriter.create<LLVM::ConstantOp>(
+    Value leadingDim = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI32Type(),
         subgroupMmaStoreMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMAStoreOp>(
@@ -216,7 +216,7 @@ struct WmmaMmaOpToNVVMLowering
     auto unpackOp = [&](Value operand) {
       auto structType = cast<LLVM::LLVMStructType>(operand.getType());
       for (size_t i = 0, e = structType.getBody().size(); i < e; ++i) {
-        Value toUse = rewriter.create<LLVM::ExtractValueOp>(loc, operand, i);
+        Value toUse = LLVM::ExtractValueOp::create(rewriter, loc, operand, i);
         unpackedOps.push_back(toUse);
       }
     };
@@ -280,19 +280,19 @@ struct WmmaConstantOpToNVVMLowering
         cast<gpu::MMAMatrixType>(subgroupMmaConstantOp.getType()));
     // If the element type is a vector create a vector from the operand.
     if (auto vecType = dyn_cast<VectorType>(type.getBody()[0])) {
-      Value vecCst = rewriter.create<LLVM::PoisonOp>(loc, vecType);
+      Value vecCst = LLVM::PoisonOp::create(rewriter, loc, vecType);
       for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) {
-        Value idx = rewriter.create<LLVM::ConstantOp>(
+        Value idx = LLVM::ConstantOp::create(rewriter,
             loc, rewriter.getI32Type(), vecEl);
-        vecCst = rewriter.create<LLVM::InsertElementOp>(loc, vecType, vecCst,
+        vecCst = LLVM::InsertElementOp::create(rewriter, loc, vecType, vecCst,
                                                         cst, idx);
       }
       cst = vecCst;
     }
-    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, type);
+    Value matrixStruct = LLVM::PoisonOp::create(rewriter, loc, type);
     for (size_t i : llvm::seq(size_t(0), type.getBody().size())) {
       matrixStruct =
-          rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, cst, i);
+          LLVM::InsertValueOp::create(rewriter, loc, matrixStruct, cst, i);
     }
     rewriter.replaceOp(subgroupMmaConstantOp, matrixStruct);
     return success();
@@ -305,17 +305,17 @@ static Value createMinMaxF(OpBuilder &builder, Location loc, Value lhs,
   Type i1Type = builder.getI1Type();
   if (auto vecType = dyn_cast<VectorType>(lhs.getType()))
     i1Type = VectorType::get(vecType.getShape(), i1Type);
-  Value cmp = builder.create<LLVM::FCmpOp>(
+  Value cmp = LLVM::FCmpOp::create(builder,
       loc, i1Type, isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt,
       lhs, rhs);
-  Value sel = builder.create<LLVM::SelectOp>(loc, cmp, lhs, rhs);
-  Value isNan = builder.create<LLVM::FCmpOp>(
+  Value sel = LLVM::SelectOp::create(builder, loc, cmp, lhs, rhs);
+  Value isNan = LLVM::FCmpOp::create(builder,
       loc, i1Type, LLVM::FCmpPredicate::uno, lhs, rhs);
-  Value nan = builder.create<LLVM::ConstantOp>(
+  Value nan = LLVM::ConstantOp::create(builder,
       loc, lhs.getType(),
       builder.getFloatAttr(floatType,
                            APFloat::getQNaN(floatType.getFloatSemantics())));
-  return builder.create<LLVM::SelectOp>(loc, isNan, nan, sel);
+  return LLVM::SelectOp::create(builder, loc, isNan, nan, sel);
 }
 
 static Value createScalarOp(OpBuilder &builder, Location loc,
@@ -323,11 +323,11 @@ static Value createScalarOp(OpBuilder &builder, Location loc,
                             ArrayRef<Value> operands) {
   switch (op) {
   case gpu::MMAElementwiseOp::ADDF:
-    return builder.create<LLVM::FAddOp>(loc, operands[0].getType(), operands);
+    return LLVM::FAddOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::MULF:
-    return builder.create<LLVM::FMulOp>(loc, operands[0].getType(), operands);
+    return LLVM::FMulOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::DIVF:
-    return builder.create<LLVM::FDivOp>(loc, operands[0].getType(), operands);
+    return LLVM::FDivOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::MAXF:
     return createMinMaxF(builder, loc, operands[0], operands[1],
                          /*isMin=*/false);
@@ -356,18 +356,18 @@ struct WmmaElementwiseOpToNVVMLowering
     size_t numOperands = adaptor.getOperands().size();
     LLVM::LLVMStructType destType = convertMMAToLLVMType(
         cast<gpu::MMAMatrixType>(subgroupMmaElementwiseOp.getType()));
-    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, destType);
+    Value matrixStruct = LLVM::PoisonOp::create(rewriter, loc, destType);
     for (size_t i = 0, e = destType.getBody().size(); i < e; ++i) {
       SmallVector<Value> extractedOperands;
       for (size_t opIdx = 0; opIdx < numOperands; opIdx++) {
-        extractedOperands.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
             loc, adaptor.getOperands()[opIdx], i));
       }
       Value element =
           createScalarOp(rewriter, loc, subgroupMmaElementwiseOp.getOpType(),
                          extractedOperands);
       matrixStruct =
-          rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, element, i);
+          LLVM::InsertValueOp::create(rewriter, loc, matrixStruct, element, i);
     }
     rewriter.replaceOp(subgroupMmaElementwiseOp, matrixStruct);
     return success();
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 456bfaba980ca..9b43d3ef602d1 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -61,10 +61,10 @@ static Value truncOrExtToLLVMType(ConversionPatternRewriter &rewriter,
       IntegerType::get(rewriter.getContext(), converter.getIndexTypeBitwidth());
   // TODO: use <=> in C++20.
   if (indexBitwidth > intWidth) {
-    return rewriter.create<LLVM::SExtOp>(loc, indexBitwidthType, value);
+    return LLVM::SExtOp::create(rewriter, loc, indexBitwidthType, value);
   }
   if (indexBitwidth < intWidth) {
-    return rewriter.create<LLVM::TruncOp>(loc, indexBitwidthType, value);
+    return LLVM::TruncOp::create(rewriter, loc, indexBitwidthType, value);
   }
   return value;
 }
@@ -82,11 +82,11 @@ static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func) {
 static Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
                        const unsigned indexBitwidth) {
   auto int32Type = IntegerType::get(rewriter.getContext(), 32);
-  Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
-  Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
-  Value mbcntLo = rewriter.create<ROCDL::MbcntLoOp>(loc, int32Type,
+  Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
+  Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
+  Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, int32Type,
                                                     ValueRange{minus1, zero});
-  Value laneId = rewriter.create<ROCDL::MbcntHiOp>(loc, int32Type,
+  Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, int32Type,
                                                    ValueRange{minus1, mbcntLo});
   return laneId;
 }
@@ -110,20 +110,20 @@ struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
     // followed by: %lid = call @llvm.amdgcn.mbcnt.hi(-1, %mlo)
 
     Type intTy = IntegerType::get(context, 32);
-    Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
-    Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
+    Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
+    Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
     Value mbcntLo =
-        rewriter.create<ROCDL::MbcntLoOp>(loc, intTy, ValueRange{minus1, zero});
-    Value laneId = rewriter.create<ROCDL::MbcntHiOp>(
+        ROCDL::MbcntLoOp::create(rewriter, loc, intTy, ValueRange{minus1, zero});
+    Value laneId = ROCDL::MbcntHiOp::create(rewriter,
         loc, intTy, ValueRange{minus1, mbcntLo});
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      laneId = rewriter.create<LLVM::SExtOp>(
+      laneId = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), laneId);
     } else if (indexBitwidth < 32) {
-      laneId = rewriter.create<LLVM::TruncOp>(
+      laneId = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), laneId);
     }
     rewriter.replaceOp(op, {laneId});
@@ -149,7 +149,7 @@ struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
           /*bitWidth=*/32, /*lower=*/isBeforeGfx10 ? 64 : 32,
           /*upper=*/op.getUpperBoundAttr().getInt() + 1);
     }
-    Value wavefrontOp = rewriter.create<ROCDL::WavefrontSizeOp>(
+    Value wavefrontOp = ROCDL::WavefrontSizeOp::create(rewriter,
         op.getLoc(), rewriter.getI32Type(), bounds);
     wavefrontOp = truncOrExtToLLVMType(rewriter, op.getLoc(), wavefrontOp,
                                        *getTypeConverter());
@@ -190,43 +190,43 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
 
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     Value width = adaptor.getWidth();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 0);
-    Value negwidth = rewriter.create<LLVM::SubOp>(loc, int32Type, zero, width);
-    Value add = rewriter.create<LLVM::AddOp>(loc, int32Type, srcLaneId, width);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, int32Type, 0);
+    Value negwidth = LLVM::SubOp::create(rewriter, loc, int32Type, zero, width);
+    Value add = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId, width);
     Value widthOrZeroIfOutside =
-        rewriter.create<LLVM::AndOp>(loc, int32Type, add, negwidth);
+        LLVM::AndOp::create(rewriter, loc, int32Type, add, negwidth);
     Value dstLane;
 
     switch (op.getMode()) {
     case gpu::ShuffleMode::UP:
-      dstLane = rewriter.create<LLVM::SubOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::SubOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::DOWN:
-      dstLane = rewriter.create<LLVM::AddOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::XOR:
-      dstLane = rewriter.create<LLVM::XOrOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::XOrOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::IDX:
       dstLane = adaptor.getOffset();
       break;
     }
-    Value isActiveSrcLane = rewriter.create<LLVM::ICmpOp>(
+    Value isActiveSrcLane = LLVM::ICmpOp::create(rewriter,
         loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
-    Value selectDstLane = rewriter.create<LLVM::SelectOp>(loc, isActiveSrcLane,
+    Value selectDstLane = LLVM::SelectOp::create(rewriter, loc, isActiveSrcLane,
                                                           dstLane, srcLaneId);
-    Value two = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 2);
+    Value two = LLVM::ConstantOp::create(rewriter, loc, int32Type, 2);
     Value dwordAlignedDstLane =
-        rewriter.create<LLVM::ShlOp>(loc, int32Type, selectDstLane, two);
+        LLVM::ShlOp::create(rewriter, loc, int32Type, selectDstLane, two);
 
     SmallVector<Value> decomposed =
         LLVM::decomposeValue(rewriter, loc, initShflValue, int32Type);
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
-      Value res = rewriter.create<ROCDL::DsBpermuteOp>(loc, int32Type,
+      Value res = ROCDL::DsBpermuteOp::create(rewriter, loc, int32Type,
                                                        dwordAlignedDstLane, v);
       swizzled.emplace_back(res);
     }
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index b99ed261ecfa3..71618cbf8a2f2 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -169,11 +169,11 @@ LogicalResult LaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
 
   Value vector =
       spirv::getBuiltinVariableValue(op, builtin, builtinType, rewriter);
-  Value dim = rewriter.create<spirv::CompositeExtractOp>(
+  Value dim = spirv::CompositeExtractOp::create(rewriter,
       op.getLoc(), builtinType, vector,
       rewriter.getI32ArrayAttr({static_cast<int32_t>(op.getDimension())}));
   if (forShader && builtinType != indexType)
-    dim = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType, dim);
+    dim = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType, dim);
   rewriter.replaceOp(op, dim);
   return success();
 }
@@ -198,7 +198,7 @@ SingleDimLaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
   Value builtinValue =
       spirv::getBuiltinVariableValue(op, builtin, i32Type, rewriter);
   if (i32Type != indexType)
-    builtinValue = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType,
+    builtinValue = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType,
                                                       builtinValue);
   rewriter.replaceOp(op, builtinValue);
   return success();
@@ -257,7 +257,7 @@ lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter,
       signatureConverter.addInputs(argType.index(), convertedType);
     }
   }
-  auto newFuncOp = rewriter.create<spirv::FuncOp>(
+  auto newFuncOp = spirv::FuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(signatureConverter.getConvertedTypes(), {}));
   for (const auto &namedAttr : funcOp->getAttrs()) {
@@ -367,7 +367,7 @@ LogicalResult GPUModuleConversion::matchAndRewrite(
 
   // Add a keyword to the module name to avoid symbolic conflict.
   std::string spvModuleName = (kSPIRVModule + moduleOp.getName()).str();
-  auto spvModule = rewriter.create<spirv::ModuleOp>(
+  auto spvModule = spirv::ModuleOp::create(rewriter,
       moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
       StringRef(spvModuleName));
 
@@ -452,41 +452,41 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
 
   switch (shuffleOp.getMode()) {
   case gpu::ShuffleMode::XOR: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleXorOp>(
+    result = spirv::GroupNonUniformShuffleXorOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::IDX: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleOp>(
+    result = spirv::GroupNonUniformShuffleOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::DOWN: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleDownOp>(
+    result = spirv::GroupNonUniformShuffleDownOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
 
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
-        rewriter.create<arith::AddIOp>(loc, laneId, adaptor.getOffset());
-    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+        arith::AddIOp::create(rewriter, loc, laneId, adaptor.getOffset());
+    validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                               resultLaneId, adaptor.getWidth());
     break;
   }
   case gpu::ShuffleMode::UP: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleUpOp>(
+    result = spirv::GroupNonUniformShuffleUpOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
 
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
-        rewriter.create<arith::SubIOp>(loc, laneId, adaptor.getOffset());
+        arith::SubIOp::create(rewriter, loc, laneId, adaptor.getOffset());
     auto i32Type = rewriter.getIntegerType(32);
-    validVal = rewriter.create<arith::CmpIOp>(
+    validVal = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, resultLaneId,
-        rewriter.create<arith::ConstantOp>(
+        arith::ConstantOp::create(rewriter,
             loc, i32Type, rewriter.getIntegerAttr(i32Type, 0)));
     break;
   }
@@ -516,14 +516,14 @@ LogicalResult GPURotateConversion::matchAndRewrite(
 
   Location loc = rotateOp.getLoc();
   auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
-  Value rotateResult = rewriter.create<spirv::GroupNonUniformRotateKHROp>(
+  Value rotateResult = spirv::GroupNonUniformRotateKHROp::create(rewriter,
       loc, scope, adaptor.getValue(), adaptor.getOffset(), adaptor.getWidth());
   Value validVal;
   if (widthAttr.getValue().getZExtValue() == subgroupSize) {
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(), loc, rewriter);
   } else {
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
-    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
+    validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                               laneId, adaptor.getWidth());
   }
 
@@ -548,13 +548,13 @@ static Value createGroupReduceOpImpl(OpBuilder &builder, Location loc,
                                 ? spirv::GroupOperation::ClusteredReduce
                                 : spirv::GroupOperation::Reduce);
   if (isUniform) {
-    return builder.create<UniformOp>(loc, type, scope, groupOp, arg)
+    return UniformOp::create(builder, loc, type, scope, groupOp, arg)
         .getResult();
   }
 
   Value clusterSizeValue;
   if (clusterSize.has_value())
-    clusterSizeValue = builder.create<spirv::ConstantOp>(
+    clusterSizeValue = spirv::ConstantOp::create(builder,
         loc, builder.getI32Type(),
         builder.getIntegerAttr(builder.getI32Type(), *clusterSize));
 
@@ -740,7 +740,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstName =
         makeVarName(moduleOp, llvm::Twine(globalVarName) + "_sc");
 
-    return rewriter.create<spirv::SpecConstantOp>(
+    return spirv::SpecConstantOp::create(rewriter,
         loc, rewriter.getStringAttr(specCstName), attr);
   };
   {
@@ -774,7 +774,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstCompositeName =
         (llvm::Twine(globalVarName) + "_scc").str();
 
-    specCstComposite = rewriter.create<spirv::SpecConstantCompositeOp>(
+    specCstComposite = spirv::SpecConstantCompositeOp::create(rewriter,
         loc, TypeAttr::get(globalType),
         rewriter.getStringAttr(specCstCompositeName),
         rewriter.getArrayAttr(constituents));
@@ -785,15 +785,15 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     // Define a GlobalVarOp initialized using specialized constants
     // that is used to specify the printf format string
     // to be passed to the SPIRV CLPrintfOp.
-    globalVar = rewriter.create<spirv::GlobalVariableOp>(
+    globalVar = spirv::GlobalVariableOp::create(rewriter,
         loc, ptrType, globalVarName, FlatSymbolRefAttr::get(specCstComposite));
 
     globalVar->setAttr("Constant", rewriter.getUnitAttr());
   }
   // Get SSA value of Global variable and create pointer to i8 to point to
   // the format string.
-  Value globalPtr = rewriter.create<spirv::AddressOfOp>(loc, globalVar);
-  Value fmtStr = rewriter.create<spirv::BitcastOp>(
+  Value globalPtr = spirv::AddressOfOp::create(rewriter, loc, globalVar);
+  Value fmtStr = spirv::BitcastOp::create(rewriter,
       loc,
       spirv::PointerType::get(i8Type, spirv::StorageClass::UniformConstant),
       globalPtr);
@@ -801,7 +801,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
   // Get printf arguments.
   auto printfArgs = llvm::to_vector_of<Value, 4>(adaptor.getArgs());
 
-  rewriter.create<spirv::CLPrintfOp>(loc, i32Type, fmtStr, printfArgs);
+  spirv::CLPrintfOp::create(rewriter, loc, i32Type, fmtStr, printfArgs);
 
   // Need to erase the gpu.printf op as gpu.printf does not use result vs
   // spirv::CLPrintfOp has i32 resultType so cannot replace with new SPIR-V
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index 0b2c06a08db2d..da8ffe47a7d33 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -144,11 +144,11 @@ void GPUToSPIRVPass::runOnOperation() {
     if (targetEnvSupportsKernelCapability(moduleOp)) {
       moduleOp.walk([&](gpu::GPUFuncOp funcOp) {
         builder.setInsertionPoint(funcOp);
-        auto newFuncOp = builder.create<func::FuncOp>(
+        auto newFuncOp = func::FuncOp::create(builder,
             funcOp.getLoc(), funcOp.getName(), funcOp.getFunctionType());
         auto entryBlock = newFuncOp.addEntryBlock();
         builder.setInsertionPointToEnd(entryBlock);
-        builder.create<func::ReturnOp>(funcOp.getLoc());
+        func::ReturnOp::create(builder, funcOp.getLoc());
         newFuncOp->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                            builder.getUnitAttr());
         funcOp.erase();
diff --git a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
index 7bb86b5ce1ddd..70646aa5f2e74 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
@@ -283,7 +283,7 @@ struct WmmaLoadOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = rewriter.create<spirv::ConstantOp>(
+    auto strideValue = spirv::ConstantOp::create(rewriter,
         loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
@@ -315,7 +315,7 @@ struct WmmaStoreOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = rewriter.create<spirv::ConstantOp>(
+    auto strideValue = spirv::ConstantOp::create(rewriter,
         loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
diff --git a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
index 0473bb59fa6aa..0d5b318f739cc 100644
--- a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
+++ b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
@@ -36,34 +36,34 @@ struct ConvertIndexCeilDivS : mlir::ConvertOpToLLVMPattern<CeilDivSOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value posOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
-    Value negOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), -1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value posOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
+    Value negOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), -1);
 
     // Compute `x`.
     Value mPos =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::sgt, m, zero);
-    Value x = rewriter.create<LLVM::SelectOp>(loc, mPos, negOne, posOne);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, m, zero);
+    Value x = LLVM::SelectOp::create(rewriter, loc, mPos, negOne, posOne);
 
     // Compute the positive result.
-    Value nPlusX = rewriter.create<LLVM::AddOp>(loc, n, x);
-    Value nPlusXDivM = rewriter.create<LLVM::SDivOp>(loc, nPlusX, m);
-    Value posRes = rewriter.create<LLVM::AddOp>(loc, nPlusXDivM, posOne);
+    Value nPlusX = LLVM::AddOp::create(rewriter, loc, n, x);
+    Value nPlusXDivM = LLVM::SDivOp::create(rewriter, loc, nPlusX, m);
+    Value posRes = LLVM::AddOp::create(rewriter, loc, nPlusXDivM, posOne);
 
     // Compute the negative result.
-    Value negN = rewriter.create<LLVM::SubOp>(loc, zero, n);
-    Value negNDivM = rewriter.create<LLVM::SDivOp>(loc, negN, m);
-    Value negRes = rewriter.create<LLVM::SubOp>(loc, zero, negNDivM);
+    Value negN = LLVM::SubOp::create(rewriter, loc, zero, n);
+    Value negNDivM = LLVM::SDivOp::create(rewriter, loc, negN, m);
+    Value negRes = LLVM::SubOp::create(rewriter, loc, zero, negNDivM);
 
     // Pick the positive result if `n` and `m` have the same sign and `n` is
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
     Value nPos =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::sgt, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, n, zero);
     Value sameSign =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, nPos, mPos);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, nPos, mPos);
     Value nNonZero =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, n, zero);
-    Value cmp = rewriter.create<LLVM::AndOp>(loc, sameSign, nNonZero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
+    Value cmp = LLVM::AndOp::create(rewriter, loc, sameSign, nNonZero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
@@ -83,17 +83,17 @@ struct ConvertIndexCeilDivU : mlir::ConvertOpToLLVMPattern<CeilDivUOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value one = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value one = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
 
     // Compute the non-zero result.
-    Value minusOne = rewriter.create<LLVM::SubOp>(loc, n, one);
-    Value quotient = rewriter.create<LLVM::UDivOp>(loc, minusOne, m);
-    Value plusOne = rewriter.create<LLVM::AddOp>(loc, quotient, one);
+    Value minusOne = LLVM::SubOp::create(rewriter, loc, n, one);
+    Value quotient = LLVM::UDivOp::create(rewriter, loc, minusOne, m);
+    Value plusOne = LLVM::AddOp::create(rewriter, loc, quotient, one);
 
     // Pick the result.
     Value cmp =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, n, zero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, zero, plusOne);
     return success();
   }
@@ -114,32 +114,32 @@ struct ConvertIndexFloorDivS : mlir::ConvertOpToLLVMPattern<FloorDivSOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value posOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
-    Value negOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), -1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value posOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
+    Value negOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), -1);
 
     // Compute `x`.
     Value mNeg =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::slt, m, zero);
-    Value x = rewriter.create<LLVM::SelectOp>(loc, mNeg, posOne, negOne);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, m, zero);
+    Value x = LLVM::SelectOp::create(rewriter, loc, mNeg, posOne, negOne);
 
     // Compute the negative result.
-    Value xMinusN = rewriter.create<LLVM::SubOp>(loc, x, n);
-    Value xMinusNDivM = rewriter.create<LLVM::SDivOp>(loc, xMinusN, m);
-    Value negRes = rewriter.create<LLVM::SubOp>(loc, negOne, xMinusNDivM);
+    Value xMinusN = LLVM::SubOp::create(rewriter, loc, x, n);
+    Value xMinusNDivM = LLVM::SDivOp::create(rewriter, loc, xMinusN, m);
+    Value negRes = LLVM::SubOp::create(rewriter, loc, negOne, xMinusNDivM);
 
     // Compute the positive result.
-    Value posRes = rewriter.create<LLVM::SDivOp>(loc, n, m);
+    Value posRes = LLVM::SDivOp::create(rewriter, loc, n, m);
 
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
     Value nNeg =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::slt, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, n, zero);
     Value diffSign =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
     Value nNonZero =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, n, zero);
-    Value cmp = rewriter.create<LLVM::AndOp>(loc, diffSign, nNonZero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
+    Value cmp = LLVM::AndOp::create(rewriter, loc, diffSign, nNonZero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, negRes, posRes);
     return success();
   }
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index 4821962f989e6..6fc097059c5e5 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -111,33 +111,33 @@ struct ConvertIndexCeilDivSPattern final : OpConversionPattern<CeilDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = rewriter.create<spirv::ConstantOp>(
+    Value posOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = rewriter.create<spirv::ConstantOp>(
+    Value negOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
-    Value mPos = rewriter.create<spirv::SGreaterThanOp>(loc, m, zero);
-    Value x = rewriter.create<spirv::SelectOp>(loc, mPos, negOne, posOne);
+    Value mPos = spirv::SGreaterThanOp::create(rewriter, loc, m, zero);
+    Value x = spirv::SelectOp::create(rewriter, loc, mPos, negOne, posOne);
 
     // Compute the positive result.
-    Value nPlusX = rewriter.create<spirv::IAddOp>(loc, n, x);
-    Value nPlusXDivM = rewriter.create<spirv::SDivOp>(loc, nPlusX, m);
-    Value posRes = rewriter.create<spirv::IAddOp>(loc, nPlusXDivM, posOne);
+    Value nPlusX = spirv::IAddOp::create(rewriter, loc, n, x);
+    Value nPlusXDivM = spirv::SDivOp::create(rewriter, loc, nPlusX, m);
+    Value posRes = spirv::IAddOp::create(rewriter, loc, nPlusXDivM, posOne);
 
     // Compute the negative result.
-    Value negN = rewriter.create<spirv::ISubOp>(loc, zero, n);
-    Value negNDivM = rewriter.create<spirv::SDivOp>(loc, negN, m);
-    Value negRes = rewriter.create<spirv::ISubOp>(loc, zero, negNDivM);
+    Value negN = spirv::ISubOp::create(rewriter, loc, zero, n);
+    Value negNDivM = spirv::SDivOp::create(rewriter, loc, negN, m);
+    Value negRes = spirv::ISubOp::create(rewriter, loc, zero, negNDivM);
 
     // Pick the positive result if `n` and `m` have the same sign and `n` is
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
-    Value nPos = rewriter.create<spirv::SGreaterThanOp>(loc, n, zero);
-    Value sameSign = rewriter.create<spirv::LogicalEqualOp>(loc, nPos, mPos);
-    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
-    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, sameSign, nNonZero);
+    Value nPos = spirv::SGreaterThanOp::create(rewriter, loc, n, zero);
+    Value sameSign = spirv::LogicalEqualOp::create(rewriter, loc, nPos, mPos);
+    Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
+    Value cmp = spirv::LogicalAndOp::create(rewriter, loc, sameSign, nNonZero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
@@ -161,18 +161,18 @@ struct ConvertIndexCeilDivUPattern final : OpConversionPattern<CeilDivUOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value one = rewriter.create<spirv::ConstantOp>(loc, n_type,
+    Value one = spirv::ConstantOp::create(rewriter, loc, n_type,
                                                    IntegerAttr::get(n_type, 1));
 
     // Compute the non-zero result.
-    Value minusOne = rewriter.create<spirv::ISubOp>(loc, n, one);
-    Value quotient = rewriter.create<spirv::UDivOp>(loc, minusOne, m);
-    Value plusOne = rewriter.create<spirv::IAddOp>(loc, quotient, one);
+    Value minusOne = spirv::ISubOp::create(rewriter, loc, n, one);
+    Value quotient = spirv::UDivOp::create(rewriter, loc, minusOne, m);
+    Value plusOne = spirv::IAddOp::create(rewriter, loc, quotient, one);
 
     // Pick the result
-    Value cmp = rewriter.create<spirv::IEqualOp>(loc, n, zero);
+    Value cmp = spirv::IEqualOp::create(rewriter, loc, n, zero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, zero, plusOne);
     return success();
   }
@@ -197,32 +197,32 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = rewriter.create<spirv::ConstantOp>(
+    Value posOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = rewriter.create<spirv::ConstantOp>(
+    Value negOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
-    Value mNeg = rewriter.create<spirv::SLessThanOp>(loc, m, zero);
-    Value x = rewriter.create<spirv::SelectOp>(loc, mNeg, posOne, negOne);
+    Value mNeg = spirv::SLessThanOp::create(rewriter, loc, m, zero);
+    Value x = spirv::SelectOp::create(rewriter, loc, mNeg, posOne, negOne);
 
     // Compute the negative result
-    Value xMinusN = rewriter.create<spirv::ISubOp>(loc, x, n);
-    Value xMinusNDivM = rewriter.create<spirv::SDivOp>(loc, xMinusN, m);
-    Value negRes = rewriter.create<spirv::ISubOp>(loc, negOne, xMinusNDivM);
+    Value xMinusN = spirv::ISubOp::create(rewriter, loc, x, n);
+    Value xMinusNDivM = spirv::SDivOp::create(rewriter, loc, xMinusN, m);
+    Value negRes = spirv::ISubOp::create(rewriter, loc, negOne, xMinusNDivM);
 
     // Compute the positive result.
-    Value posRes = rewriter.create<spirv::SDivOp>(loc, n, m);
+    Value posRes = spirv::SDivOp::create(rewriter, loc, n, m);
 
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
-    Value nNeg = rewriter.create<spirv::SLessThanOp>(loc, n, zero);
-    Value diffSign = rewriter.create<spirv::LogicalNotEqualOp>(loc, nNeg, mNeg);
-    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
+    Value nNeg = spirv::SLessThanOp::create(rewriter, loc, n, zero);
+    Value diffSign = spirv::LogicalNotEqualOp::create(rewriter, loc, nNeg, mNeg);
+    Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
 
-    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, diffSign, nNonZero);
+    Value cmp = spirv::LogicalAndOp::create(rewriter, loc, diffSign, nNonZero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index e34d5f74d232f..e8ad0d4644389 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -32,7 +32,7 @@ MemRefDescriptor::MemRefDescriptor(Value descriptor)
 MemRefDescriptor MemRefDescriptor::poison(OpBuilder &builder, Location loc,
                                           Type descriptorType) {
 
-  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
+  Value descriptor = LLVM::PoisonOp::create(builder, loc, descriptorType);
   return MemRefDescriptor(descriptor);
 }
 
@@ -99,20 +99,20 @@ void MemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc,
 // integer attribute.
 static Value createIndexAttrConstant(OpBuilder &builder, Location loc,
                                      Type resultType, int64_t value) {
-  return builder.create<LLVM::ConstantOp>(loc, resultType,
+  return LLVM::ConstantOp::create(builder, loc, resultType,
                                           builder.getIndexAttr(value));
 }
 
 /// Builds IR extracting the offset from the descriptor.
 Value MemRefDescriptor::offset(OpBuilder &builder, Location loc) {
-  return builder.create<LLVM::ExtractValueOp>(loc, value,
+  return LLVM::ExtractValueOp::create(builder, loc, value,
                                               kOffsetPosInMemRefDescriptor);
 }
 
 /// Builds IR inserting the offset into the descriptor.
 void MemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                  Value offset) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, offset,
+  value = LLVM::InsertValueOp::create(builder, loc, value, offset,
                                               kOffsetPosInMemRefDescriptor);
 }
 
@@ -125,7 +125,7 @@ void MemRefDescriptor::setConstantOffset(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th size from the descriptor.
 Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) {
-  return builder.create<LLVM::ExtractValueOp>(
+  return LLVM::ExtractValueOp::create(builder,
       loc, value, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
@@ -137,22 +137,22 @@ Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
 
   // Copy size values to stack-allocated memory.
   auto one = createIndexAttrConstant(builder, loc, indexType, 1);
-  auto sizes = builder.create<LLVM::ExtractValueOp>(
+  auto sizes = LLVM::ExtractValueOp::create(builder,
       loc, value, llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
-  auto sizesPtr = builder.create<LLVM::AllocaOp>(loc, ptrTy, arrayTy, one,
+  auto sizesPtr = LLVM::AllocaOp::create(builder, loc, ptrTy, arrayTy, one,
                                                  /*alignment=*/0);
-  builder.create<LLVM::StoreOp>(loc, sizes, sizesPtr);
+  LLVM::StoreOp::create(builder, loc, sizes, sizesPtr);
 
   // Load an return size value of interest.
-  auto resultPtr = builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, sizesPtr,
+  auto resultPtr = LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, sizesPtr,
                                                ArrayRef<LLVM::GEPArg>{0, pos});
-  return builder.create<LLVM::LoadOp>(loc, indexType, resultPtr);
+  return LLVM::LoadOp::create(builder, loc, indexType, resultPtr);
 }
 
 /// Builds IR inserting the pos-th size into the descriptor
 void MemRefDescriptor::setSize(OpBuilder &builder, Location loc, unsigned pos,
                                Value size) {
-  value = builder.create<LLVM::InsertValueOp>(
+  value = LLVM::InsertValueOp::create(builder,
       loc, value, size, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
@@ -164,14 +164,14 @@ void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th stride from the descriptor.
 Value MemRefDescriptor::stride(OpBuilder &builder, Location loc, unsigned pos) {
-  return builder.create<LLVM::ExtractValueOp>(
+  return LLVM::ExtractValueOp::create(builder,
       loc, value, ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
 
 /// Builds IR inserting the pos-th stride into the descriptor
 void MemRefDescriptor::setStride(OpBuilder &builder, Location loc, unsigned pos,
                                  Value stride) {
-  value = builder.create<LLVM::InsertValueOp>(
+  value = LLVM::InsertValueOp::create(builder,
       loc, value, stride,
       ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
@@ -207,7 +207,7 @@ Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
           ? offset(builder, loc)
           : createIndexAttrConstant(builder, loc, indexType, offsetCst);
   Type elementType = converter.convertType(type.getElementType());
-  ptr = builder.create<LLVM::GEPOp>(loc, ptr.getType(), elementType, ptr,
+  ptr = LLVM::GEPOp::create(builder, loc, ptr.getType(), elementType, ptr,
                                     offsetVal);
   return ptr;
 }
@@ -303,7 +303,7 @@ UnrankedMemRefDescriptor::UnrankedMemRefDescriptor(Value descriptor)
 UnrankedMemRefDescriptor UnrankedMemRefDescriptor::poison(OpBuilder &builder,
                                                           Location loc,
                                                           Type descriptorType) {
-  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
+  Value descriptor = LLVM::PoisonOp::create(builder, loc, descriptorType);
   return UnrankedMemRefDescriptor(descriptor);
 }
 Value UnrankedMemRefDescriptor::rank(OpBuilder &builder, Location loc) const {
@@ -380,18 +380,18 @@ void UnrankedMemRefDescriptor::computeSizes(
         builder, loc, indexType,
         llvm::divideCeil(typeConverter.getPointerBitwidth(addressSpace), 8));
     Value doublePointerSize =
-        builder.create<LLVM::MulOp>(loc, indexType, two, pointerSize);
+        LLVM::MulOp::create(builder, loc, indexType, two, pointerSize);
 
     // (1 + 2 * rank) * sizeof(index)
     Value rank = desc.rank(builder, loc);
-    Value doubleRank = builder.create<LLVM::MulOp>(loc, indexType, two, rank);
+    Value doubleRank = LLVM::MulOp::create(builder, loc, indexType, two, rank);
     Value doubleRankIncremented =
-        builder.create<LLVM::AddOp>(loc, indexType, doubleRank, one);
-    Value rankIndexSize = builder.create<LLVM::MulOp>(
+        LLVM::AddOp::create(builder, loc, indexType, doubleRank, one);
+    Value rankIndexSize = LLVM::MulOp::create(builder,
         loc, indexType, doubleRankIncremented, indexSize);
 
     // Total allocation size.
-    Value allocationSize = builder.create<LLVM::AddOp>(
+    Value allocationSize = LLVM::AddOp::create(builder,
         loc, indexType, doublePointerSize, rankIndexSize);
     sizes.push_back(allocationSize);
   }
@@ -400,13 +400,13 @@ void UnrankedMemRefDescriptor::computeSizes(
 Value UnrankedMemRefDescriptor::allocatedPtr(
     OpBuilder &builder, Location loc, Value memRefDescPtr,
     LLVM::LLVMPointerType elemPtrType) {
-  return builder.create<LLVM::LoadOp>(loc, elemPtrType, memRefDescPtr);
+  return LLVM::LoadOp::create(builder, loc, elemPtrType, memRefDescPtr);
 }
 
 void UnrankedMemRefDescriptor::setAllocatedPtr(
     OpBuilder &builder, Location loc, Value memRefDescPtr,
     LLVM::LLVMPointerType elemPtrType, Value allocatedPtr) {
-  builder.create<LLVM::StoreOp>(loc, allocatedPtr, memRefDescPtr);
+  LLVM::StoreOp::create(builder, loc, allocatedPtr, memRefDescPtr);
 }
 
 static std::pair<Value, Type>
@@ -423,9 +423,9 @@ Value UnrankedMemRefDescriptor::alignedPtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   Value alignedGep =
-      builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+      LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                   elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
-  return builder.create<LLVM::LoadOp>(loc, elemPtrType, alignedGep);
+  return LLVM::LoadOp::create(builder, loc, elemPtrType, alignedGep);
 }
 
 void UnrankedMemRefDescriptor::setAlignedPtr(
@@ -435,9 +435,9 @@ void UnrankedMemRefDescriptor::setAlignedPtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   Value alignedGep =
-      builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+      LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                   elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
-  builder.create<LLVM::StoreOp>(loc, alignedPtr, alignedGep);
+  LLVM::StoreOp::create(builder, loc, alignedPtr, alignedGep);
 }
 
 Value UnrankedMemRefDescriptor::offsetBasePtr(
@@ -446,7 +446,7 @@ Value UnrankedMemRefDescriptor::offsetBasePtr(
   auto [elementPtrPtr, elemPtrPtrType] =
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
-  return builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+  return LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                      elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
 }
 
@@ -456,7 +456,7 @@ Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
                                        LLVM::LLVMPointerType elemPtrType) {
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
-  return builder.create<LLVM::LoadOp>(loc, typeConverter.getIndexType(),
+  return LLVM::LoadOp::create(builder, loc, typeConverter.getIndexType(),
                                       offsetPtr);
 }
 
@@ -467,7 +467,7 @@ void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                          Value offset) {
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
-  builder.create<LLVM::StoreOp>(loc, offset, offsetPtr);
+  LLVM::StoreOp::create(builder, loc, offset, offsetPtr);
 }
 
 Value UnrankedMemRefDescriptor::sizeBasePtr(
@@ -477,7 +477,7 @@ Value UnrankedMemRefDescriptor::sizeBasePtr(
   Type structTy = LLVM::LLVMStructType::getLiteral(
       indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy});
   auto resultType = LLVM::LLVMPointerType::get(builder.getContext());
-  return builder.create<LLVM::GEPOp>(loc, resultType, structTy, memRefDescPtr,
+  return LLVM::GEPOp::create(builder, loc, resultType, structTy, memRefDescPtr,
                                      ArrayRef<LLVM::GEPArg>{0, 3});
 }
 
@@ -489,8 +489,8 @@ Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value sizeStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, index);
-  return builder.create<LLVM::LoadOp>(loc, indexTy, sizeStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, index);
+  return LLVM::LoadOp::create(builder, loc, indexTy, sizeStoreGep);
 }
 
 void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
@@ -501,8 +501,8 @@ void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value sizeStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, index);
-  builder.create<LLVM::StoreOp>(loc, size, sizeStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, index);
+  LLVM::StoreOp::create(builder, loc, size, sizeStoreGep);
 }
 
 Value UnrankedMemRefDescriptor::strideBasePtr(
@@ -511,7 +511,7 @@ Value UnrankedMemRefDescriptor::strideBasePtr(
   Type indexTy = typeConverter.getIndexType();
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
-  return builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, rank);
+  return LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, rank);
 }
 
 Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
@@ -522,8 +522,8 @@ Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value strideStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, strideBasePtr, index);
-  return builder.create<LLVM::LoadOp>(loc, indexTy, strideStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, strideBasePtr, index);
+  return LLVM::LoadOp::create(builder, loc, indexTy, strideStoreGep);
 }
 
 void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
@@ -534,6 +534,6 @@ void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value strideStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, strideBasePtr, index);
-  builder.create<LLVM::StoreOp>(loc, stride, strideStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, strideBasePtr, index);
+  LLVM::StoreOp::create(builder, loc, stride, strideStoreGep);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index c5f72f7e10b8c..57aed79874ad0 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -57,7 +57,7 @@ Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder,
                                                     Location loc,
                                                     Type resultType,
                                                     int64_t value) {
-  return builder.create<LLVM::ConstantOp>(loc, resultType,
+  return LLVM::ConstantOp::create(builder, loc, resultType,
                                           builder.getIndexAttr(value));
 }
 
@@ -123,7 +123,7 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
       runningStride = sizes[i];
     else if (stride == ShapedType::kDynamic)
       runningStride =
-          rewriter.create<LLVM::MulOp>(loc, runningStride, sizes[i]);
+          LLVM::MulOp::create(rewriter, loc, runningStride, sizes[i]);
     else
       runningStride = createIndexAttrConstant(rewriter, loc, indexType, stride);
   }
@@ -131,10 +131,10 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
     // Buffer size in bytes.
     Type elementType = typeConverter->convertType(memRefType.getElementType());
     auto elementPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, elementPtrType);
-    Value gepPtr = rewriter.create<LLVM::GEPOp>(
+    Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
+    Value gepPtr = LLVM::GEPOp::create(rewriter,
         loc, elementPtrType, elementType, nullPtr, runningStride);
-    size = rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);
+    size = LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
   } else {
     size = runningStride;
   }
@@ -149,10 +149,10 @@ Value ConvertToLLVMPattern::getSizeInBytes(
   // which is a common pattern of getting the size of a type in bytes.
   Type llvmType = typeConverter->convertType(type);
   auto convertedPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-  auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, convertedPtrType);
-  auto gep = rewriter.create<LLVM::GEPOp>(loc, convertedPtrType, llvmType,
+  auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, convertedPtrType);
+  auto gep = LLVM::GEPOp::create(rewriter, loc, convertedPtrType, llvmType,
                                           nullPtr, ArrayRef<LLVM::GEPArg>{1});
-  return rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gep);
+  return LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gep);
 }
 
 Value ConvertToLLVMPattern::getNumElements(
@@ -175,7 +175,7 @@ Value ConvertToLLVMPattern::getNumElements(
           staticSize == ShapedType::kDynamic
               ? dynamicSizes[dynamicIndex++]
               : createIndexAttrConstant(rewriter, loc, indexType, staticSize);
-      numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
     } else {
       numElements =
           staticSize == ShapedType::kDynamic
@@ -276,14 +276,14 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
             ? builder
                   .create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
                   .getResult()
-            : builder.create<LLVM::AllocaOp>(loc, getPtrType(),
+            : LLVM::AllocaOp::create(builder, loc, getPtrType(),
                                              IntegerType::get(getContext(), 8),
                                              allocationSize,
                                              /*alignment=*/0);
     Value source = desc.memRefDescPtr(builder, loc);
-    builder.create<LLVM::MemcpyOp>(loc, memory, source, allocationSize, false);
+    LLVM::MemcpyOp::create(builder, loc, memory, source, allocationSize, false);
     if (!toDynamic)
-      builder.create<LLVM::CallOp>(loc, freeFunc.value(), source);
+      LLVM::CallOp::create(builder, loc, freeFunc.value(), source);
 
     // Create a new descriptor. The same descriptor can be returned multiple
     // times, attempting to modify its pointer can lead to memory leaks
@@ -349,7 +349,7 @@ LogicalResult LLVM::detail::oneToOneRewrite(
   SmallVector<Value, 4> results;
   results.reserve(numResults);
   for (unsigned i = 0; i < numResults; ++i) {
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+    results.push_back(LLVM::ExtractValueOp::create(rewriter,
         op->getLoc(), newOp->getResult(0), i));
   }
   rewriter.replaceOp(op, results);
@@ -371,7 +371,7 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   if (numResults != 0)
     resType = typeConverter.packOperationResults(op->getResultTypes());
 
-  auto callIntrOp = rewriter.create<LLVM::CallIntrinsicOp>(
+  auto callIntrOp = LLVM::CallIntrinsicOp::create(rewriter,
       loc, resType, rewriter.getStringAttr(intrinsic), operands);
   // Propagate attributes.
   callIntrOp->setAttrs(op->getAttrDictionary());
@@ -388,7 +388,7 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   results.reserve(numResults);
   Value intrRes = callIntrOp.getResults();
   for (unsigned i = 0; i < numResults; ++i)
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(loc, intrRes, i));
+    results.push_back(LLVM::ExtractValueOp::create(rewriter, loc, intrRes, i));
   rewriter.replaceOp(op, results);
 
   return success();
@@ -406,7 +406,7 @@ static unsigned getBitWidth(Type type) {
 static Value createI32Constant(OpBuilder &builder, Location loc,
                                int32_t value) {
   Type i32 = builder.getI32Type();
-  return builder.create<LLVM::ConstantOp>(loc, i32, value);
+  return LLVM::ConstantOp::create(builder, loc, i32, value);
 }
 
 SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
@@ -418,17 +418,17 @@ SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
   unsigned srcBitWidth = getBitWidth(srcType);
   unsigned dstBitWidth = getBitWidth(dstType);
   if (srcBitWidth == dstBitWidth) {
-    Value cast = builder.create<LLVM::BitcastOp>(loc, dstType, src);
+    Value cast = LLVM::BitcastOp::create(builder, loc, dstType, src);
     return {cast};
   }
 
   if (dstBitWidth > srcBitWidth) {
     auto smallerInt = builder.getIntegerType(srcBitWidth);
     if (srcType != smallerInt)
-      src = builder.create<LLVM::BitcastOp>(loc, smallerInt, src);
+      src = LLVM::BitcastOp::create(builder, loc, smallerInt, src);
 
     auto largerInt = builder.getIntegerType(dstBitWidth);
-    Value res = builder.create<LLVM::ZExtOp>(loc, largerInt, src);
+    Value res = LLVM::ZExtOp::create(builder, loc, largerInt, src);
     return {res};
   }
   assert(srcBitWidth % dstBitWidth == 0 &&
@@ -436,12 +436,12 @@ SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
   int64_t numElements = srcBitWidth / dstBitWidth;
   auto vecType = VectorType::get(numElements, dstType);
 
-  src = builder.create<LLVM::BitcastOp>(loc, vecType, src);
+  src = LLVM::BitcastOp::create(builder, loc, vecType, src);
 
   SmallVector<Value> res;
   for (auto i : llvm::seq(numElements)) {
     Value idx = createI32Constant(builder, loc, i);
-    Value elem = builder.create<LLVM::ExtractElementOp>(loc, src, idx);
+    Value elem = LLVM::ExtractElementOp::create(builder, loc, src, idx);
     res.emplace_back(elem);
   }
 
@@ -461,28 +461,28 @@ Value mlir::LLVM::composeValue(OpBuilder &builder, Location loc, ValueRange src,
     if (dstBitWidth < srcBitWidth) {
       auto largerInt = builder.getIntegerType(srcBitWidth);
       if (res.getType() != largerInt)
-        res = builder.create<LLVM::BitcastOp>(loc, largerInt, res);
+        res = LLVM::BitcastOp::create(builder, loc, largerInt, res);
 
       auto smallerInt = builder.getIntegerType(dstBitWidth);
-      res = builder.create<LLVM::TruncOp>(loc, smallerInt, res);
+      res = LLVM::TruncOp::create(builder, loc, smallerInt, res);
     }
 
     if (res.getType() != dstType)
-      res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+      res = LLVM::BitcastOp::create(builder, loc, dstType, res);
 
     return res;
   }
 
   int64_t numElements = src.size();
   auto srcType = VectorType::get(numElements, src.front().getType());
-  Value res = builder.create<LLVM::PoisonOp>(loc, srcType);
+  Value res = LLVM::PoisonOp::create(builder, loc, srcType);
   for (auto &&[i, elem] : llvm::enumerate(src)) {
     Value idx = createI32Constant(builder, loc, i);
-    res = builder.create<LLVM::InsertElementOp>(loc, srcType, res, elem, idx);
+    res = LLVM::InsertElementOp::create(builder, loc, srcType, res, elem, idx);
   }
 
   if (res.getType() != dstType)
-    res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+    res = LLVM::BitcastOp::create(builder, loc, dstType, res);
 
   return res;
 }
@@ -518,18 +518,18 @@ Value mlir::LLVM::getStridedElementPtr(OpBuilder &builder, Location loc,
       Value stride =
           ShapedType::isDynamic(strides[i])
               ? memRefDescriptor.stride(builder, loc, i)
-              : builder.create<LLVM::ConstantOp>(
+              : LLVM::ConstantOp::create(builder,
                     loc, indexType, builder.getIndexAttr(strides[i]));
       increment =
-          builder.create<LLVM::MulOp>(loc, increment, stride, intOverflowFlags);
+          LLVM::MulOp::create(builder, loc, increment, stride, intOverflowFlags);
     }
-    index = index ? builder.create<LLVM::AddOp>(loc, index, increment,
+    index = index ? LLVM::AddOp::create(builder, loc, index, increment,
                                                 intOverflowFlags)
                   : increment;
   }
 
   Type elementPtrType = memRefDescriptor.getElementPtrType();
-  return index ? builder.create<LLVM::GEPOp>(
+  return index ? LLVM::GEPOp::create(builder,
                      loc, elementPtrType,
                      converter.convertType(type.getElementType()), base, index,
                      noWrapFlags)
diff --git a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
index 49c73fbc9dd79..091fb7e4c2afb 100644
--- a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
@@ -66,7 +66,7 @@ LogicalResult mlir::LLVM::createPrintStrCall(
       DenseElementsAttr::get(dataAttrType, llvm::ArrayRef(elementVals));
   auto arrayTy =
       LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), elementVals.size());
-  auto globalOp = builder.create<LLVM::GlobalOp>(
+  auto globalOp = LLVM::GlobalOp::create(builder,
       loc, arrayTy, /*constant=*/true, LLVM::Linkage::Private,
       ensureSymbolNameIsUnique(moduleOp, symbolName, symbolTables), dataAttr);
 
@@ -74,15 +74,15 @@ LogicalResult mlir::LLVM::createPrintStrCall(
   // Emit call to `printStr` in runtime library.
   builder.restoreInsertionPoint(ip);
   auto msgAddr =
-      builder.create<LLVM::AddressOfOp>(loc, ptrTy, globalOp.getName());
+      LLVM::AddressOfOp::create(builder, loc, ptrTy, globalOp.getName());
   SmallVector<LLVM::GEPArg> indices(1, 0);
   Value gep =
-      builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, msgAddr, indices);
+      LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, msgAddr, indices);
   FailureOr<LLVM::LLVMFuncOp> printer =
       LLVM::lookupOrCreatePrintStringFn(builder, moduleOp, runtimeFunctionName);
   if (failed(printer))
     return failure();
-  builder.create<LLVM::CallOp>(loc, TypeRange(),
+  LLVM::CallOp::create(builder, loc, TypeRange(),
                                SymbolRefAttr::get(printer.value()), gep);
   return success();
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
index 1cd0bd85f9894..13ed4628c3c9e 100644
--- a/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
@@ -24,10 +24,10 @@ StructBuilder::StructBuilder(Value v) : value(v), structType(v.getType()) {
 
 Value StructBuilder::extractPtr(OpBuilder &builder, Location loc,
                                 unsigned pos) const {
-  return builder.create<LLVM::ExtractValueOp>(loc, value, pos);
+  return LLVM::ExtractValueOp::create(builder, loc, value, pos);
 }
 
 void StructBuilder::setPtr(OpBuilder &builder, Location loc, unsigned pos,
                            Value ptr) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, ptr, pos);
+  value = LLVM::InsertValueOp::create(builder, loc, value, ptr, pos);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 7312594c761f7..77135188dabcf 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -91,7 +91,7 @@ static Value unrankedMemRefMaterialization(OpBuilder &builder,
       packUnrankedMemRefDesc(builder, resultType, inputs, loc, converter);
   if (!packed)
     return Value();
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, packed)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, packed)
       .getResult(0);
 }
 
@@ -107,7 +107,7 @@ static Value rankedMemRefMaterialization(OpBuilder &builder,
       packRankedMemRefDesc(builder, resultType, inputs, loc, converter);
   if (!packed)
     return Value();
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, packed)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, packed)
       .getResult(0);
 }
 
@@ -224,12 +224,12 @@ LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
   // non-LLVM types persist after an LLVM conversion.
   addSourceMaterialization([&](OpBuilder &builder, Type resultType,
                                ValueRange inputs, Location loc) {
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
   addTargetMaterialization([&](OpBuilder &builder, Type resultType,
                                ValueRange inputs, Location loc) {
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
 
@@ -731,12 +731,12 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand,
   // Alloca with proper alignment. We do not expect optimizations of this
   // alloca op and so we omit allocating at the entry block.
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
-  Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
+  Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
                                                builder.getIndexAttr(1));
   Value allocated =
-      builder.create<LLVM::AllocaOp>(loc, ptrType, operand.getType(), one);
+      LLVM::AllocaOp::create(builder, loc, ptrType, operand.getType(), one);
   // Store into the alloca'ed descriptor.
-  builder.create<LLVM::StoreOp>(loc, operand, allocated);
+  LLVM::StoreOp::create(builder, loc, operand, allocated);
   return allocated;
 }
 
diff --git a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
index bf3f31729c3da..4fc208e05eb9d 100644
--- a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
@@ -87,17 +87,17 @@ LogicalResult LLVM::detail::handleMultidimensionalVectors(
   auto result1DVectorTy = resultTypeInfo.llvm1DVectorTy;
   auto resultNDVectoryTy = resultTypeInfo.llvmNDVectorTy;
   auto loc = op->getLoc();
-  Value desc = rewriter.create<LLVM::PoisonOp>(loc, resultNDVectoryTy);
+  Value desc = LLVM::PoisonOp::create(rewriter, loc, resultNDVectoryTy);
   nDVectorIterate(resultTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
     // For this unrolled `position` corresponding to the `linearIndex`^th
     // element, extract operand vectors
     SmallVector<Value, 4> extractedOperands;
     for (const auto &operand : llvm::enumerate(operands)) {
-      extractedOperands.push_back(rewriter.create<LLVM::ExtractValueOp>(
+      extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
           loc, operand.value(), position));
     }
     Value newVal = createOperand(result1DVectorTy, extractedOperands);
-    desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, newVal, position);
+    desc = LLVM::InsertValueOp::create(rewriter, loc, desc, newVal, position);
   });
   rewriter.replaceOp(op, desc);
   return success();
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index c3f213147b7a7..498a5cfc40204 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -78,7 +78,7 @@ getLibraryCallSymbolRef(Operation *op, PatternRewriter &rewriter) {
   // Insert before module terminator.
   rewriter.setInsertionPoint(module.getBody(),
                              std::prev(module.getBody()->end()));
-  func::FuncOp funcOp = rewriter.create<func::FuncOp>(
+  func::FuncOp funcOp = func::FuncOp::create(rewriter,
       op->getLoc(), fnNameAttr.getValue(), libFnType);
   // Insert a function attribute that will trigger the emission of the
   // corresponding `_mlir_ciface_xxx` interface so that external libraries see
@@ -101,7 +101,7 @@ createTypeCanonicalizedMemRefOperands(OpBuilder &b, Location loc,
       continue;
     }
     Value cast =
-        b.create<memref::CastOp>(loc, makeStridedLayoutDynamic(memrefType), op);
+        memref::CastOp::create(b, loc, makeStridedLayoutDynamic(memrefType), op);
     res.push_back(cast);
   }
   return res;
diff --git a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
index d4deff5b88070..30521d66a2e4c 100644
--- a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
+++ b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
@@ -54,18 +54,18 @@ std::pair<Value, Value> getRawPtrAndSize(const Location loc,
                                          Value memRef, Type elType) {
   Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
   Value dataPtr =
-      rewriter.create<LLVM::ExtractValueOp>(loc, ptrType, memRef, 1);
-  Value offset = rewriter.create<LLVM::ExtractValueOp>(
+      LLVM::ExtractValueOp::create(rewriter, loc, ptrType, memRef, 1);
+  Value offset = LLVM::ExtractValueOp::create(rewriter,
       loc, rewriter.getI64Type(), memRef, 2);
   Value resPtr =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, elType, dataPtr, offset);
+      LLVM::GEPOp::create(rewriter, loc, ptrType, elType, dataPtr, offset);
   Value size;
   if (cast<LLVM::LLVMStructType>(memRef.getType()).getBody().size() > 3) {
-    size = rewriter.create<LLVM::ExtractValueOp>(loc, memRef,
+    size = LLVM::ExtractValueOp::create(rewriter, loc, memRef,
                                                  ArrayRef<int64_t>{3, 0});
-    size = rewriter.create<LLVM::TruncOp>(loc, rewriter.getI32Type(), size);
+    size = LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), size);
   } else {
-    size = rewriter.create<arith::ConstantIntOp>(loc, 1, 32);
+    size = arith::ConstantIntOp::create(rewriter, loc, 1, 32);
   }
   return {resPtr, size};
 }
@@ -157,13 +157,13 @@ class MPICHImplTraits : public MPIImplTraits {
   Value getCommWorld(const Location loc,
                      ConversionPatternRewriter &rewriter) override {
     static constexpr int MPI_COMM_WORLD = 0x44000000;
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
                                              MPI_COMM_WORLD);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return rewriter.create<LLVM::TruncOp>(loc, rewriter.getI32Type(), comm);
+    return LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), comm);
   }
 
   intptr_t getStatusIgnore() override { return 1; }
@@ -195,7 +195,7 @@ class MPICHImplTraits : public MPIImplTraits {
       mtype = MPI_UINT8_T;
     else
       assert(false && "unsupported type");
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), mtype);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), mtype);
   }
 
   Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter,
@@ -245,7 +245,7 @@ class MPICHImplTraits : public MPIImplTraits {
       op = MPI_REPLACE;
       break;
     }
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), op);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), op);
   }
 };
 
@@ -281,15 +281,15 @@ class OMPIImplTraits : public MPIImplTraits {
     getOrDefineExternalStruct(loc, rewriter, name, commStructT);
 
     // get address of symbol
-    auto comm = rewriter.create<LLVM::AddressOfOp>(
+    auto comm = LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, name));
-    return rewriter.create<LLVM::PtrToIntOp>(loc, rewriter.getI64Type(), comm);
+    return LLVM::PtrToIntOp::create(rewriter, loc, rewriter.getI64Type(), comm);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return rewriter.create<LLVM::IntToPtrOp>(
+    return LLVM::IntToPtrOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(rewriter.getContext()), comm);
   }
 
@@ -330,7 +330,7 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, mtype, typeStructT);
     // get address of symbol
-    return rewriter.create<LLVM::AddressOfOp>(
+    return LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, mtype));
   }
@@ -389,7 +389,7 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, op, opStructT);
     // get address of symbol
-    return rewriter.create<LLVM::AddressOfOp>(
+    return LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, op));
   }
@@ -424,7 +424,7 @@ struct InitOpLowering : public ConvertOpToLLVMPattern<mpi::InitOp> {
     Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
 
     // instantiate nullptr `%nullptr = llvm.mlir.zero : !llvm.ptr`
-    auto nullPtrOp = rewriter.create<LLVM::ZeroOp>(loc, ptrType);
+    auto nullPtrOp = LLVM::ZeroOp::create(rewriter, loc, ptrType);
     Value llvmnull = nullPtrOp.getRes();
 
     // grab a reference to the global module op:
@@ -513,9 +513,9 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
 
     // get communicator
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, i32, 1);
+    auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
     auto outPtr =
-        rewriter.create<LLVM::AllocaOp>(loc, ptrType, comm.getType(), one);
+        LLVM::AllocaOp::create(rewriter, loc, ptrType, comm.getType(), one);
 
     // int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm * newcomm)
     auto funcType =
@@ -524,14 +524,14 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
     LLVM::LLVMFuncOp funcDecl = getOrDefineFunction(moduleOp, loc, rewriter,
                                                     "MPI_Comm_split", funcType);
 
-    auto callOp = rewriter.create<LLVM::CallOp>(
+    auto callOp = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{comm, adaptor.getColor(), adaptor.getKey(),
                    outPtr.getRes()});
 
     // load the communicator into a register
-    Value res = rewriter.create<LLVM::LoadOp>(loc, i32, outPtr.getResult());
-    res = rewriter.create<LLVM::SExtOp>(loc, rewriter.getI64Type(), res);
+    Value res = LLVM::LoadOp::create(rewriter, loc, i32, outPtr.getResult());
+    res = LLVM::SExtOp::create(rewriter, loc, rewriter.getI64Type(), res);
 
     // if retval is checked, replace uses of retval with the results from the
     // call op
@@ -580,14 +580,14 @@ struct CommRankOpLowering : public ConvertOpToLLVMPattern<mpi::CommRankOp> {
         moduleOp, loc, rewriter, "MPI_Comm_rank", rankFuncType);
 
     // replace with function call
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, i32, 1);
-    auto rankptr = rewriter.create<LLVM::AllocaOp>(loc, ptrType, i32, one);
-    auto callOp = rewriter.create<LLVM::CallOp>(
+    auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
+    auto rankptr = LLVM::AllocaOp::create(rewriter, loc, ptrType, i32, one);
+    auto callOp = LLVM::CallOp::create(rewriter,
         loc, initDecl, ValueRange{comm, rankptr.getRes()});
 
     // load the rank into a register
     auto loadedRank =
-        rewriter.create<LLVM::LoadOp>(loc, i32, rankptr.getResult());
+        LLVM::LoadOp::create(rewriter, loc, i32, rankptr.getResult());
 
     // if retval is checked, replace uses of retval with the results from the
     // call op
@@ -641,7 +641,7 @@ struct SendOpLowering : public ConvertOpToLLVMPattern<mpi::SendOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Send", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getDest(), adaptor.getTag(),
                    comm});
@@ -683,10 +683,10 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
     auto mpiTraits = MPIImplTraits::get(moduleOp);
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    Value statusIgnore = rewriter.create<LLVM::ConstantOp>(
+    Value statusIgnore = LLVM::ConstantOp::create(rewriter,
         loc, i64, mpiTraits->getStatusIgnore());
     statusIgnore =
-        rewriter.create<LLVM::IntToPtrOp>(loc, ptrType, statusIgnore);
+        LLVM::IntToPtrOp::create(rewriter, loc, ptrType, statusIgnore);
 
     // LLVM Function type representing `i32 MPI_Recv(data, count, datatype, dst,
     // tag, comm)`
@@ -698,7 +698,7 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Recv", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getSource(),
                    adaptor.getTag(), comm, statusIgnore});
@@ -738,9 +738,9 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
 
     // If input and output are the same, request in-place operation.
     if (adaptor.getSendbuf() == adaptor.getRecvbuf()) {
-      sendPtr = rewriter.create<LLVM::ConstantOp>(
+      sendPtr = LLVM::ConstantOp::create(rewriter,
           loc, i64, reinterpret_cast<int64_t>(mpiTraits->getInPlace()));
-      sendPtr = rewriter.create<LLVM::IntToPtrOp>(loc, ptrType, sendPtr);
+      sendPtr = LLVM::IntToPtrOp::create(rewriter, loc, ptrType, sendPtr);
     }
 
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
@@ -757,7 +757,7 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Allreduce", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{sendPtr, recvPtr, sendSize, dataType, mpiOp, commWorld});
 
diff --git a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
index 7f4655e53609e..bb3511440e041 100644
--- a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
+++ b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
@@ -121,7 +121,7 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     initValueAttr = FloatAttr::get(resultElementType, 0.0);
   else
     initValueAttr = IntegerAttr::get(resultElementType, 0);
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(vecType, initValueAttr));
   SmallVector<int64_t> strides = computeStrides(shape);
   for (int64_t linearIndex = 0; linearIndex < numElements; ++linearIndex) {
@@ -129,11 +129,11 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     SmallVector<Value> operands;
     for (Value input : op->getOperands())
       operands.push_back(
-          rewriter.create<vector::ExtractOp>(loc, input, positions));
+          vector::ExtractOp::create(rewriter, loc, input, positions));
     Value scalarOp =
-        rewriter.create<Op>(loc, vecType.getElementType(), operands);
+        Op::create(rewriter, loc, vecType.getElementType(), operands);
     result =
-        rewriter.create<vector::InsertOp>(loc, scalarOp, result, positions);
+        vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
   }
   rewriter.replaceOp(op, result);
   return success();
@@ -195,7 +195,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
 
   FunctionType funcType = FunctionType::get(
       builder.getContext(), {elementType, elementType}, elementType);
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
   LLVM::linkage::Linkage inlineLinkage = LLVM::linkage::Linkage::LinkonceODR;
   Attribute linkage =
       LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
@@ -208,11 +208,11 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value zeroValue = builder.create<arith::ConstantOp>(
+  Value zeroValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 0));
-  Value oneValue = builder.create<arith::ConstantOp>(
+  Value oneValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 1));
-  Value minusOneValue = builder.create<arith::ConstantOp>(
+  Value minusOneValue = arith::ConstantOp::create(builder,
       elementType,
       builder.getIntegerAttr(elementType,
                              APInt(elementType.getIntOrFloatBitWidth(), -1ULL,
@@ -221,81 +221,81 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   // if (p == T(0))
   //   return T(1);
   auto pIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroValue);
   Block *thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(pIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(pIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsZero, thenBlock, fallthroughBlock);
 
   // if (p < T(0)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto pIsNeg =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, pArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg, zeroValue);
   //   if (b == T(0))
   builder.createBlock(funcBody);
   auto bIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, zeroValue);
   //     return T(1) / T(0);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(
-      builder.create<arith::DivSIOp>(oneValue, zeroValue).getResult());
+  func::ReturnOp::create(builder,
+      arith::DivSIOp::create(builder, oneValue, zeroValue).getResult());
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(0)).
   builder.setInsertionPointToEnd(bIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(bIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, bIsZero, thenBlock, fallthroughBlock);
 
   //   if (b == T(1))
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto bIsOne =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bArg, oneValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, oneValue);
   //    return T(1);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(1)).
   builder.setInsertionPointToEnd(bIsOne->getBlock());
-  builder.create<cf::CondBranchOp>(bIsOne, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, bIsOne, thenBlock, fallthroughBlock);
 
   //   if (b == T(-1)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
-  auto bIsMinusOne = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto bIsMinusOne = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                    bArg, minusOneValue);
   //     if (p & T(1))
   builder.createBlock(funcBody);
-  auto pIsOdd = builder.create<arith::CmpIOp>(
-      arith::CmpIPredicate::ne, builder.create<arith::AndIOp>(pArg, oneValue),
+  auto pIsOdd = arith::CmpIOp::create(builder,
+      arith::CmpIPredicate::ne, arith::AndIOp::create(builder, pArg, oneValue),
       zeroValue);
   //       return T(-1);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(minusOneValue);
+  func::ReturnOp::create(builder, minusOneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(pIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(pIsOdd, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsOdd, thenBlock, fallthroughBlock);
 
   //     return T(1);
   //   } // b == T(-1)
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(-1)).
   builder.setInsertionPointToEnd(bIsMinusOne->getBlock());
-  builder.create<cf::CondBranchOp>(bIsMinusOne, pIsOdd->getBlock(),
+  cf::CondBranchOp::create(builder, bIsMinusOne, pIsOdd->getBlock(),
                                    fallthroughBlock);
 
   //   return T(0);
   // } // (p < T(0))
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<func::ReturnOp>(zeroValue);
+  func::ReturnOp::create(builder, zeroValue);
   Block *loopHeader = builder.createBlock(
       funcBody, funcBody->end(), {elementType, elementType, elementType},
       {builder.getLoc(), builder.getLoc(), builder.getLoc()});
   // Set up conditional branch for (p < T(0)).
   builder.setInsertionPointToEnd(pIsNeg->getBlock());
   // Set initial values of 'result', 'b' and 'p' for the loop.
-  builder.create<cf::CondBranchOp>(pIsNeg, bIsZero->getBlock(), loopHeader,
+  cf::CondBranchOp::create(builder, pIsNeg, bIsZero->getBlock(), loopHeader,
                                    ValueRange{oneValue, bArg, pArg});
 
   // T result = T(1);
@@ -313,44 +313,44 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & T(1))
-  auto powerTmpIsOdd = builder.create<arith::CmpIOp>(
+  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
       arith::CmpIPredicate::ne,
-      builder.create<arith::AndIOp>(powerTmp, oneValue), zeroValue);
+      arith::AndIOp::create(builder, powerTmp, oneValue), zeroValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
-  Value newResultTmp = builder.create<arith::MulIOp>(resultTmp, baseTmp);
+  Value newResultTmp = arith::MulIOp::create(builder, resultTmp, baseTmp);
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), elementType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(thenBlock);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(powerTmpIsOdd, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
                                    resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
   //   p >>= T(1);
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newPowerTmp = builder.create<arith::ShRUIOp>(powerTmp, oneValue);
+  Value newPowerTmp = arith::ShRUIOp::create(builder, powerTmp, oneValue);
 
   //   if (p == T(0))
-  auto newPowerIsZero = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                       newPowerTmp, zeroValue);
   //     return result;
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(newResultTmp);
+  func::ReturnOp::create(builder, newResultTmp);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(newPowerIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, newPowerIsZero, thenBlock, fallthroughBlock);
 
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newBaseTmp = builder.create<arith::MulIOp>(baseTmp, baseTmp);
+  Value newBaseTmp = arith::MulIOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  builder.create<cf::BranchOp>(
+  cf::BranchOp::create(builder,
       ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
   return funcOp;
 }
@@ -420,7 +420,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   llvm::raw_string_ostream nameOS(funcName);
   nameOS << '_' << baseType;
   nameOS << '_' << powType;
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
   LLVM::linkage::Linkage inlineLinkage = LLVM::linkage::Linkage::LinkonceODR;
   Attribute linkage =
       LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
@@ -433,46 +433,46 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value oneBValue = builder.create<arith::ConstantOp>(
+  Value oneBValue = arith::ConstantOp::create(builder,
       baseType, builder.getFloatAttr(baseType, 1.0));
-  Value zeroPValue = builder.create<arith::ConstantOp>(
+  Value zeroPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, 0));
-  Value onePValue = builder.create<arith::ConstantOp>(
+  Value onePValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, 1));
-  Value minPValue = builder.create<arith::ConstantOp>(
+  Value minPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMinValue(
                                                    powType.getWidth())));
-  Value maxPValue = builder.create<arith::ConstantOp>(
+  Value maxPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMaxValue(
                                                    powType.getWidth())));
 
   // if (p == Tp{0})
   //   return Tb{1};
   auto pIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, zeroPValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroPValue);
   Block *thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneBValue);
+  func::ReturnOp::create(builder, oneBValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == Tp{0}).
   builder.setInsertionPointToEnd(pIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(pIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsZero, thenBlock, fallthroughBlock);
 
   builder.setInsertionPointToEnd(fallthroughBlock);
   // bool isNegativePower{p < Tp{0}}
-  auto pIsNeg = builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, pArg,
+  auto pIsNeg = arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg,
                                               zeroPValue);
   // bool isMin{p == std::numeric_limits<Tp>::min()};
   auto pIsMin =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, minPValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, minPValue);
 
   // if (isMin) {
   //   p = std::numeric_limits<Tp>::max();
   // } else if (isNegativePower) {
   //   p = -p;
   // }
-  Value negP = builder.create<arith::SubIOp>(zeroPValue, pArg);
-  auto pInit = builder.create<arith::SelectOp>(pIsNeg, negP, pArg);
-  pInit = builder.create<arith::SelectOp>(pIsMin, maxPValue, pInit);
+  Value negP = arith::SubIOp::create(builder, zeroPValue, pArg);
+  auto pInit = arith::SelectOp::create(builder, pIsNeg, negP, pArg);
+  pInit = arith::SelectOp::create(builder, pIsMin, maxPValue, pInit);
 
   // Tb result = Tb{1};
   // Tb origBase = Tb{b};
@@ -489,7 +489,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
       {builder.getLoc(), builder.getLoc(), builder.getLoc()});
   // Set initial values of 'result', 'b' and 'p' for the loop.
   builder.setInsertionPointToEnd(pInit->getBlock());
-  builder.create<cf::BranchOp>(loopHeader, ValueRange{oneBValue, bArg, pInit});
+  cf::BranchOp::create(builder, loopHeader, ValueRange{oneBValue, bArg, pInit});
 
   // Create loop body.
   Value resultTmp = loopHeader->getArgument(0);
@@ -498,29 +498,29 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & Tp{1})
-  auto powerTmpIsOdd = builder.create<arith::CmpIOp>(
+  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
       arith::CmpIPredicate::ne,
-      builder.create<arith::AndIOp>(powerTmp, onePValue), zeroPValue);
+      arith::AndIOp::create(builder, powerTmp, onePValue), zeroPValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
-  Value newResultTmp = builder.create<arith::MulFOp>(resultTmp, baseTmp);
+  Value newResultTmp = arith::MulFOp::create(builder, resultTmp, baseTmp);
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(thenBlock);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
   // Set up conditional branch for (p & Tp{1}).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(powerTmpIsOdd, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
                                    resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
   //   p >>= Tp{1};
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newPowerTmp = builder.create<arith::ShRUIOp>(powerTmp, onePValue);
+  Value newPowerTmp = arith::ShRUIOp::create(builder, powerTmp, onePValue);
 
   //   if (p == Tp{0})
-  auto newPowerIsZero = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                       newPowerTmp, zeroPValue);
   //     break;
   //
@@ -531,9 +531,9 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newBaseTmp = builder.create<arith::MulFOp>(baseTmp, baseTmp);
+  Value newBaseTmp = arith::MulFOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  builder.create<cf::BranchOp>(
+  cf::BranchOp::create(builder,
       ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
 
   // Set up conditional branch for early loop exit:
@@ -542,7 +542,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Block *loopExit = builder.createBlock(funcBody, funcBody->end(), baseType,
                                         builder.getLoc());
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(newPowerIsZero, loopExit, newResultTmp,
+  cf::CondBranchOp::create(builder, newPowerIsZero, loopExit, newResultTmp,
                                    fallthroughBlock, ValueRange{});
 
   // if (isMin) {
@@ -553,11 +553,11 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(loopExit);
-  builder.create<cf::CondBranchOp>(pIsMin, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, pIsMin, thenBlock, fallthroughBlock,
                                    newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
-  newResultTmp = builder.create<arith::MulFOp>(newResultTmp, bArg);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  newResultTmp = arith::MulFOp::create(builder, newResultTmp, bArg);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
 
   /// if (isNegativePower) {
   ///   result = Tb{1} / result;
@@ -567,15 +567,15 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Block *returnBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                            builder.getLoc());
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<cf::CondBranchOp>(pIsNeg, thenBlock, returnBlock,
+  cf::CondBranchOp::create(builder, pIsNeg, thenBlock, returnBlock,
                                    newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
-  newResultTmp = builder.create<arith::DivFOp>(oneBValue, newResultTmp);
-  builder.create<cf::BranchOp>(newResultTmp, returnBlock);
+  newResultTmp = arith::DivFOp::create(builder, oneBValue, newResultTmp);
+  cf::BranchOp::create(builder, newResultTmp, returnBlock);
 
   // return result;
   builder.setInsertionPointToEnd(returnBlock);
-  builder.create<func::ReturnOp>(returnBlock->getArgument(0));
+  func::ReturnOp::create(builder, returnBlock->getArgument(0));
 
   return funcOp;
 }
@@ -667,7 +667,7 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
   nameOS << '_' << elementType;
   FunctionType funcType =
       FunctionType::get(builder.getContext(), {elementType}, elementType);
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
 
   // LinkonceODR ensures that there is only one implementation of this function
   // across all math.ctlz functions that are lowered in this way.
@@ -683,32 +683,32 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
 
   Value arg = funcOp.getArgument(0);
   Type indexType = builder.getIndexType();
-  Value bitWidthValue = builder.create<arith::ConstantOp>(
+  Value bitWidthValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, bitWidth));
-  Value zeroValue = builder.create<arith::ConstantOp>(
+  Value zeroValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 0));
 
   Value inputEqZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, arg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, arg, zeroValue);
 
   // if input == 0, return bit width, else enter loop.
-  scf::IfOp ifOp = builder.create<scf::IfOp>(
+  scf::IfOp ifOp = scf::IfOp::create(builder,
       elementType, inputEqZero, /*addThenBlock=*/true, /*addElseBlock=*/true);
   ifOp.getThenBodyBuilder().create<scf::YieldOp>(loc, bitWidthValue);
 
   auto elseBuilder =
       ImplicitLocOpBuilder::atBlockEnd(loc, &ifOp.getElseRegion().front());
 
-  Value oneIndex = elseBuilder.create<arith::ConstantOp>(
+  Value oneIndex = arith::ConstantOp::create(elseBuilder,
       indexType, elseBuilder.getIndexAttr(1));
-  Value oneValue = elseBuilder.create<arith::ConstantOp>(
+  Value oneValue = arith::ConstantOp::create(elseBuilder,
       elementType, elseBuilder.getIntegerAttr(elementType, 1));
-  Value bitWidthIndex = elseBuilder.create<arith::ConstantOp>(
+  Value bitWidthIndex = arith::ConstantOp::create(elseBuilder,
       indexType, elseBuilder.getIndexAttr(bitWidth));
-  Value nValue = elseBuilder.create<arith::ConstantOp>(
+  Value nValue = arith::ConstantOp::create(elseBuilder,
       elementType, elseBuilder.getIntegerAttr(elementType, 0));
 
-  auto loop = elseBuilder.create<scf::ForOp>(
+  auto loop = scf::ForOp::create(elseBuilder,
       oneIndex, bitWidthIndex, oneIndex,
       // Initial values for two loop induction variables, the arg which is being
       // shifted left in each iteration, and the n value which tracks the count
@@ -725,25 +725,25 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
         Value argIter = args[0];
         Value nIter = args[1];
 
-        Value argIsNonNegative = b.create<arith::CmpIOp>(
+        Value argIsNonNegative = arith::CmpIOp::create(b,
             loc, arith::CmpIPredicate::slt, argIter, zeroValue);
-        scf::IfOp ifOp = b.create<scf::IfOp>(
+        scf::IfOp ifOp = scf::IfOp::create(b,
             loc, argIsNonNegative,
             [&](OpBuilder &b, Location loc) {
               // If arg is negative, continue (effectively, break)
-              b.create<scf::YieldOp>(loc, ValueRange{argIter, nIter});
+              scf::YieldOp::create(b, loc, ValueRange{argIter, nIter});
             },
             [&](OpBuilder &b, Location loc) {
               // Otherwise, increment n and shift arg left.
-              Value nNext = b.create<arith::AddIOp>(loc, nIter, oneValue);
-              Value argNext = b.create<arith::ShLIOp>(loc, argIter, oneValue);
-              b.create<scf::YieldOp>(loc, ValueRange{argNext, nNext});
+              Value nNext = arith::AddIOp::create(b, loc, nIter, oneValue);
+              Value argNext = arith::ShLIOp::create(b, loc, argIter, oneValue);
+              scf::YieldOp::create(b, loc, ValueRange{argNext, nNext});
             });
-        b.create<scf::YieldOp>(loc, ifOp.getResults());
+        scf::YieldOp::create(b, loc, ifOp.getResults());
       });
-  elseBuilder.create<scf::YieldOp>(loop.getResult(1));
+  scf::YieldOp::create(elseBuilder, loop.getResult(1));
 
-  builder.create<func::ReturnOp>(ifOp.getResult(0));
+  func::ReturnOp::create(builder, ifOp.getResult(0));
   return funcOp;
 }
 
diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
index f4d69ce8235bb..26559f394c221 100644
--- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
+++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
@@ -107,7 +107,7 @@ struct IntOpWithFlagLowering : public ConvertOpToLLVMPattern<MathOp> {
     return LLVM::detail::handleMultidimensionalVectors(
         op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
-          return rewriter.create<LLVMOp>(loc, llvm1DVectorTy, operands[0],
+          return LLVMOp::create(rewriter, loc, llvm1DVectorTy, operands[0],
                                          false);
         },
         rewriter);
@@ -145,14 +145,14 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (LLVM::isCompatibleVectorType(llvmOperandType)) {
-        one = rewriter.create<LLVM::ConstantOp>(
+        one = LLVM::ConstantOp::create(rewriter,
             loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
+        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
-      auto exp = rewriter.create<LLVM::ExpOp>(loc, adaptor.getOperand(),
+      auto exp = LLVM::ExpOp::create(rewriter, loc, adaptor.getOperand(),
                                               expAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FSubOp>(
           op, llvmOperandType, ValueRange{exp, one}, subAttrs.getAttrs());
@@ -171,10 +171,10 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto exp = rewriter.create<LLVM::ExpOp>(
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto exp = LLVM::ExpOp::create(rewriter,
               loc, llvm1DVectorTy, operands[0], expAttrs.getAttrs());
-          return rewriter.create<LLVM::FSubOp>(
+          return LLVM::FSubOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{exp, one}, subAttrs.getAttrs());
         },
         rewriter);
@@ -205,14 +205,14 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one =
           isa<VectorType>(llvmOperandType)
-              ? rewriter.create<LLVM::ConstantOp>(
+              ? LLVM::ConstantOp::create(rewriter,
                     loc, llvmOperandType,
                     SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                            floatOne))
-              : rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType,
+              : LLVM::ConstantOp::create(rewriter, loc, llvmOperandType,
                                                   floatOne);
 
-      auto add = rewriter.create<LLVM::FAddOp>(
+      auto add = LLVM::FAddOp::create(rewriter,
           loc, llvmOperandType, ValueRange{one, adaptor.getOperand()},
           addAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::LogOp>(
@@ -232,11 +232,11 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto add = rewriter.create<LLVM::FAddOp>(loc, llvm1DVectorTy,
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto add = LLVM::FAddOp::create(rewriter, loc, llvm1DVectorTy,
                                                    ValueRange{one, operands[0]},
                                                    addAttrs.getAttrs());
-          return rewriter.create<LLVM::LogOp>(
+          return LLVM::LogOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{add}, logAttrs.getAttrs());
         },
         rewriter);
@@ -267,14 +267,14 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (isa<VectorType>(llvmOperandType)) {
-        one = rewriter.create<LLVM::ConstantOp>(
+        one = LLVM::ConstantOp::create(rewriter,
             loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
+        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
-      auto sqrt = rewriter.create<LLVM::SqrtOp>(loc, adaptor.getOperand(),
+      auto sqrt = LLVM::SqrtOp::create(rewriter, loc, adaptor.getOperand(),
                                                 sqrtAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FDivOp>(
           op, llvmOperandType, ValueRange{one, sqrt}, divAttrs.getAttrs());
@@ -293,10 +293,10 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto sqrt = rewriter.create<LLVM::SqrtOp>(
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto sqrt = LLVM::SqrtOp::create(rewriter,
               loc, llvm1DVectorTy, operands[0], sqrtAttrs.getAttrs());
-          return rewriter.create<LLVM::FDivOp>(
+          return LLVM::FDivOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{one, sqrt}, divAttrs.getAttrs());
         },
         rewriter);
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index a0ce7d3b75fc2..19303dc3e8b55 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -84,7 +84,7 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto shape = vecType.getShape();
   int64_t numElements = vecType.getNumElements();
 
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(
                vecType, FloatAttr::get(vecType.getElementType(), 0.0)));
   SmallVector<int64_t> strides = computeStrides(shape);
@@ -93,11 +93,11 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     SmallVector<Value> operands;
     for (auto input : op->getOperands())
       operands.push_back(
-          rewriter.create<vector::ExtractOp>(loc, input, positions));
+          vector::ExtractOp::create(rewriter, loc, input, positions));
     Value scalarOp =
-        rewriter.create<Op>(loc, vecType.getElementType(), operands);
+        Op::create(rewriter, loc, vecType.getElementType(), operands);
     result =
-        rewriter.create<vector::InsertOp>(loc, scalarOp, result, positions);
+        vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
   }
   rewriter.replaceOp(op, {result});
   return success();
@@ -114,9 +114,9 @@ PromoteOpToF32<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto f32 = rewriter.getF32Type();
   auto extendedOperands = llvm::to_vector(
       llvm::map_range(op->getOperands(), [&](Value operand) -> Value {
-        return rewriter.create<arith::ExtFOp>(loc, f32, operand);
+        return arith::ExtFOp::create(rewriter, loc, f32, operand);
       }));
-  auto newOp = rewriter.create<Op>(loc, f32, extendedOperands);
+  auto newOp = Op::create(rewriter, loc, f32, extendedOperands);
   rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, opType, newOp);
   return success();
 }
@@ -139,7 +139,7 @@ ScalarOpToLibmCall<Op>::matchAndRewrite(Op op,
     rewriter.setInsertionPointToStart(&module->getRegion(0).front());
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-    opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name,
+    opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
                                            opFunctionTy);
     opFunc.setPrivate();
 
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 59db14ed816be..26891aa7c2025 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -36,11 +36,11 @@ static Value getScalarOrVectorI32Constant(Type type, int value,
     if (!vectorType.getElementType().isInteger(32))
       return nullptr;
     SmallVector<int> values(vectorType.getNumElements(), value);
-    return builder.create<spirv::ConstantOp>(loc, type,
+    return spirv::ConstantOp::create(builder, loc, type,
                                              builder.getI32VectorAttr(values));
   }
   if (type.isInteger(32))
-    return builder.create<spirv::ConstantOp>(loc, type,
+    return spirv::ConstantOp::create(builder, loc, type,
                                              builder.getI32IntegerAttr(value));
 
   return nullptr;
@@ -144,9 +144,9 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
     Type intType = rewriter.getIntegerType(bitwidth);
     uint64_t intValue = uint64_t(1) << (bitwidth - 1);
 
-    Value signMask = rewriter.create<spirv::ConstantOp>(
+    Value signMask = spirv::ConstantOp::create(rewriter,
         loc, intType, rewriter.getIntegerAttr(intType, intValue));
-    Value valueMask = rewriter.create<spirv::ConstantOp>(
+    Value valueMask = spirv::ConstantOp::create(rewriter,
         loc, intType, rewriter.getIntegerAttr(intType, intValue - 1u));
 
     if (auto vectorType = dyn_cast<VectorType>(type)) {
@@ -156,24 +156,24 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
 
       SmallVector<Value> signSplat(count, signMask);
       signMask =
-          rewriter.create<spirv::CompositeConstructOp>(loc, intType, signSplat);
+          spirv::CompositeConstructOp::create(rewriter, loc, intType, signSplat);
 
       SmallVector<Value> valueSplat(count, valueMask);
-      valueMask = rewriter.create<spirv::CompositeConstructOp>(loc, intType,
+      valueMask = spirv::CompositeConstructOp::create(rewriter, loc, intType,
                                                                valueSplat);
     }
 
     Value lhsCast =
-        rewriter.create<spirv::BitcastOp>(loc, intType, adaptor.getLhs());
+        spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getLhs());
     Value rhsCast =
-        rewriter.create<spirv::BitcastOp>(loc, intType, adaptor.getRhs());
+        spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getRhs());
 
-    Value value = rewriter.create<spirv::BitwiseAndOp>(
+    Value value = spirv::BitwiseAndOp::create(rewriter,
         loc, intType, ValueRange{lhsCast, valueMask});
-    Value sign = rewriter.create<spirv::BitwiseAndOp>(
+    Value sign = spirv::BitwiseAndOp::create(rewriter,
         loc, intType, ValueRange{rhsCast, signMask});
 
-    Value result = rewriter.create<spirv::BitwiseOrOp>(loc, intType,
+    Value result = spirv::BitwiseOrOp::create(rewriter, loc, intType,
                                                        ValueRange{value, sign});
     rewriter.replaceOpWithNewOp<spirv::BitcastOp>(copySignOp, type, result);
     return success();
@@ -214,18 +214,18 @@ struct CountLeadingZerosPattern final
     Value val31 = getScalarOrVectorI32Constant(type, 31, rewriter, loc);
     Value val32 = getScalarOrVectorI32Constant(type, 32, rewriter, loc);
 
-    Value msb = rewriter.create<spirv::GLFindUMsbOp>(loc, input);
+    Value msb = spirv::GLFindUMsbOp::create(rewriter, loc, input);
     // We need to subtract from 31 given that the index returned by GLSL
     // FindUMsb is counted from the least significant bit. Theoretically this
     // also gives the correct result even if the integer has all zero bits, in
     // which case GL FindUMsb would return -1.
-    Value subMsb = rewriter.create<spirv::ISubOp>(loc, val31, msb);
+    Value subMsb = spirv::ISubOp::create(rewriter, loc, val31, msb);
     // However, certain Vulkan implementations have driver bugs for the corner
     // case where the input is zero. And.. it can be smart to optimize a select
     // only involving the corner case. So separately compute the result when the
     // input is either zero or one.
-    Value subInput = rewriter.create<spirv::ISubOp>(loc, val32, input);
-    Value cmp = rewriter.create<spirv::ULessThanEqualOp>(loc, input, val1);
+    Value subInput = spirv::ISubOp::create(rewriter, loc, val32, input);
+    Value cmp = spirv::ULessThanEqualOp::create(rewriter, loc, input, val1);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(countOp, cmp, subInput,
                                                  subMsb);
     return success();
@@ -253,7 +253,7 @@ struct ExpM1OpPattern final : public OpConversionPattern<math::ExpM1Op> {
     if (!type)
       return failure();
 
-    Value exp = rewriter.create<ExpOp>(loc, type, adaptor.getOperand());
+    Value exp = ExpOp::create(rewriter, loc, type, adaptor.getOperand());
     auto one = spirv::ConstantOp::getOne(type, loc, rewriter);
     rewriter.replaceOpWithNewOp<spirv::FSubOp>(operation, exp, one);
     return success();
@@ -283,7 +283,7 @@ struct Log1pOpPattern final : public OpConversionPattern<math::Log1pOp> {
 
     auto one = spirv::ConstantOp::getOne(type, operation.getLoc(), rewriter);
     Value onePlus =
-        rewriter.create<spirv::FAddOp>(loc, one, adaptor.getOperand());
+        spirv::FAddOp::create(rewriter, loc, one, adaptor.getOperand());
     rewriter.replaceOpWithNewOp<LogOp>(operation, type, onePlus);
     return success();
   }
@@ -321,14 +321,14 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
 
     auto getConstantValue = [&](double value) {
       if (auto floatType = dyn_cast<FloatType>(type)) {
-        return rewriter.create<spirv::ConstantOp>(
+        return spirv::ConstantOp::create(rewriter,
             loc, type, rewriter.getFloatAttr(floatType, value));
       }
       if (auto vectorType = dyn_cast<VectorType>(type)) {
         Type elemType = vectorType.getElementType();
 
         if (isa<FloatType>(elemType)) {
-          return rewriter.create<spirv::ConstantOp>(
+          return spirv::ConstantOp::create(rewriter,
               loc, type,
               DenseFPElementsAttr::get(
                   vectorType, FloatAttr::get(elemType, value).getValue()));
@@ -341,7 +341,7 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
     Value constantValue = getConstantValue(
         std::is_same<MathLogOp, math::Log2Op>() ? log2Reciprocal
                                                 : log10Reciprocal);
-    Value log = rewriter.create<SpirvLogOp>(loc, adaptor.getOperand());
+    Value log = SpirvLogOp::create(rewriter, loc, adaptor.getOperand());
     rewriter.replaceOpWithNewOp<spirv::FMulOp>(operation, type, log,
                                                constantValue);
     return success();
@@ -386,7 +386,7 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
     Location loc = powfOp.getLoc();
     Value zero = spirv::ConstantOp::getZero(operandType, loc, rewriter);
     Value lessThan =
-        rewriter.create<spirv::FOrdLessThanOp>(loc, adaptor.getLhs(), zero);
+        spirv::FOrdLessThanOp::create(rewriter, loc, adaptor.getLhs(), zero);
 
     // Per C/C++ spec:
     // > pow(base, exponent) returns NaN (and raises FE_INVALID) if base is
@@ -394,11 +394,11 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
     // Calculate the reminder from the exponent and check whether it is zero.
     Value floatOne = spirv::ConstantOp::getOne(operandType, loc, rewriter);
     Value expRem =
-        rewriter.create<spirv::FRemOp>(loc, adaptor.getRhs(), floatOne);
+        spirv::FRemOp::create(rewriter, loc, adaptor.getRhs(), floatOne);
     Value expRemNonZero =
-        rewriter.create<spirv::FOrdNotEqualOp>(loc, expRem, zero);
+        spirv::FOrdNotEqualOp::create(rewriter, loc, expRem, zero);
     Value cmpNegativeWithFractionalExp =
-        rewriter.create<spirv::LogicalAndOp>(loc, expRemNonZero, lessThan);
+        spirv::LogicalAndOp::create(rewriter, loc, expRemNonZero, lessThan);
     // Create NaN result and replace base value if conditions are met.
     const auto &floatSemantics = scalarFloatType.getFloatSemantics();
     const auto nan = APFloat::getNaN(floatSemantics);
@@ -407,10 +407,10 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
       nanAttr = DenseElementsAttr::get(vectorType, nan);
 
     Value NanValue =
-        rewriter.create<spirv::ConstantOp>(loc, operandType, nanAttr);
-    Value lhs = rewriter.create<spirv::SelectOp>(
+        spirv::ConstantOp::create(rewriter, loc, operandType, nanAttr);
+    Value lhs = spirv::SelectOp::create(rewriter,
         loc, cmpNegativeWithFractionalExp, NanValue, adaptor.getLhs());
-    Value abs = rewriter.create<spirv::GLFAbsOp>(loc, lhs);
+    Value abs = spirv::GLFAbsOp::create(rewriter, loc, lhs);
 
     // TODO: The following just forcefully casts y into an integer value in
     // order to properly propagate the sign, assuming integer y cases. It
@@ -418,18 +418,18 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
 
     // Cast exponent to integer and calculate exponent % 2 != 0.
     Value intRhs =
-        rewriter.create<spirv::ConvertFToSOp>(loc, intType, adaptor.getRhs());
+        spirv::ConvertFToSOp::create(rewriter, loc, intType, adaptor.getRhs());
     Value intOne = spirv::ConstantOp::getOne(intType, loc, rewriter);
     Value bitwiseAndOne =
-        rewriter.create<spirv::BitwiseAndOp>(loc, intRhs, intOne);
-    Value isOdd = rewriter.create<spirv::IEqualOp>(loc, bitwiseAndOne, intOne);
+        spirv::BitwiseAndOp::create(rewriter, loc, intRhs, intOne);
+    Value isOdd = spirv::IEqualOp::create(rewriter, loc, bitwiseAndOne, intOne);
 
     // calculate pow based on abs(lhs)^rhs.
-    Value pow = rewriter.create<spirv::GLPowOp>(loc, abs, adaptor.getRhs());
-    Value negate = rewriter.create<spirv::FNegateOp>(loc, pow);
+    Value pow = spirv::GLPowOp::create(rewriter, loc, abs, adaptor.getRhs());
+    Value negate = spirv::FNegateOp::create(rewriter, loc, pow);
     // if the exponent is odd and lhs < 0, negate the result.
     Value shouldNegate =
-        rewriter.create<spirv::LogicalAndOp>(loc, lessThan, isOdd);
+        spirv::LogicalAndOp::create(rewriter, loc, lessThan, isOdd);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(powfOp, shouldNegate, negate,
                                                  pow);
     return success();
@@ -455,22 +455,22 @@ struct RoundOpPattern final : public OpConversionPattern<math::RoundOp> {
     auto one = spirv::ConstantOp::getOne(ty, loc, rewriter);
     Value half;
     if (VectorType vty = dyn_cast<VectorType>(ty)) {
-      half = rewriter.create<spirv::ConstantOp>(
+      half = spirv::ConstantOp::create(rewriter,
           loc, vty,
           DenseElementsAttr::get(vty,
                                  rewriter.getFloatAttr(ety, 0.5).getValue()));
     } else {
-      half = rewriter.create<spirv::ConstantOp>(
+      half = spirv::ConstantOp::create(rewriter,
           loc, ty, rewriter.getFloatAttr(ety, 0.5));
     }
 
-    auto abs = rewriter.create<spirv::GLFAbsOp>(loc, operand);
-    auto floor = rewriter.create<spirv::GLFloorOp>(loc, abs);
-    auto sub = rewriter.create<spirv::FSubOp>(loc, abs, floor);
+    auto abs = spirv::GLFAbsOp::create(rewriter, loc, operand);
+    auto floor = spirv::GLFloorOp::create(rewriter, loc, abs);
+    auto sub = spirv::FSubOp::create(rewriter, loc, abs, floor);
     auto greater =
-        rewriter.create<spirv::FOrdGreaterThanEqualOp>(loc, sub, half);
-    auto select = rewriter.create<spirv::SelectOp>(loc, greater, one, zero);
-    auto add = rewriter.create<spirv::FAddOp>(loc, floor, select);
+        spirv::FOrdGreaterThanEqualOp::create(rewriter, loc, sub, half);
+    auto select = spirv::SelectOp::create(rewriter, loc, greater, one, zero);
+    auto add = spirv::FAddOp::create(rewriter, loc, floor, select);
     rewriter.replaceOpWithNewOp<math::CopySignOp>(roundOp, add, operand);
     return success();
   }
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
index db244d1d1cac8..db6ce600af76a 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
@@ -161,7 +161,7 @@ struct ConvertLoad final : public OpConversionPattern<memref::LoadOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = rewriter.create<emitc::SubscriptOp>(
+    auto subscript = emitc::SubscriptOp::create(rewriter,
         op.getLoc(), arrayValue, operands.getIndices());
 
     rewriter.replaceOpWithNewOp<emitc::LoadOp>(op, resultTy, subscript);
@@ -181,7 +181,7 @@ struct ConvertStore final : public OpConversionPattern<memref::StoreOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = rewriter.create<emitc::SubscriptOp>(
+    auto subscript = emitc::SubscriptOp::create(rewriter,
         op.getLoc(), arrayValue, operands.getIndices());
     rewriter.replaceOpWithNewOp<emitc::AssignOp>(op, subscript,
                                                  operands.getValue());
@@ -212,7 +212,7 @@ void mlir::populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   };
 
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 00bbdcb12e326..1603b1ba7eb9e 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -87,12 +87,12 @@ getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
 ///   aligned = bumped - bumped % alignment
 static Value createAligned(ConversionPatternRewriter &rewriter, Location loc,
                            Value input, Value alignment) {
-  Value one = rewriter.create<LLVM::ConstantOp>(loc, alignment.getType(),
+  Value one = LLVM::ConstantOp::create(rewriter, loc, alignment.getType(),
                                                 rewriter.getIndexAttr(1));
-  Value bump = rewriter.create<LLVM::SubOp>(loc, alignment, one);
-  Value bumped = rewriter.create<LLVM::AddOp>(loc, input, bump);
-  Value mod = rewriter.create<LLVM::URemOp>(loc, bumped, alignment);
-  return rewriter.create<LLVM::SubOp>(loc, bumped, mod);
+  Value bump = LLVM::SubOp::create(rewriter, loc, alignment, one);
+  Value bumped = LLVM::AddOp::create(rewriter, loc, input, bump);
+  Value mod = LLVM::URemOp::create(rewriter, loc, bumped, alignment);
+  return LLVM::SubOp::create(rewriter, loc, bumped, mod);
 }
 
 /// Computes the byte size for the MemRef element type.
@@ -123,7 +123,7 @@ static Value castAllocFuncResult(ConversionPatternRewriter &rewriter,
   assert(succeeded(maybeMemrefAddrSpace) && "unsupported address space");
   unsigned memrefAddrSpace = *maybeMemrefAddrSpace;
   if (allocatedPtrTy.getAddressSpace() != memrefAddrSpace)
-    allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+    allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
         allocatedPtr);
   return allocatedPtr;
@@ -168,14 +168,14 @@ class AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
     Value alignment = getAlignment(rewriter, loc, op);
     if (alignment) {
       // Adjust the allocation size to consider alignment.
-      sizeBytes = rewriter.create<LLVM::AddOp>(loc, sizeBytes, alignment);
+      sizeBytes = LLVM::AddOp::create(rewriter, loc, sizeBytes, alignment);
     }
 
     // Allocate the underlying buffer.
     Type elementPtrType = this->getElementPtrType(memRefType);
     assert(elementPtrType && "could not compute element ptr type");
     auto results =
-        rewriter.create<LLVM::CallOp>(loc, allocFuncOp.value(), sizeBytes);
+        LLVM::CallOp::create(rewriter, loc, allocFuncOp.value(), sizeBytes);
 
     Value allocatedPtr =
         castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
@@ -184,11 +184,11 @@ class AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
     if (alignment) {
       // Compute the aligned pointer.
       Value allocatedInt =
-          rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), allocatedPtr);
+          LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), allocatedPtr);
       Value alignmentInt =
           createAligned(rewriter, loc, allocatedInt, alignment);
       alignedPtr =
-          rewriter.create<LLVM::IntToPtrOp>(loc, elementPtrType, alignmentInt);
+          LLVM::IntToPtrOp::create(rewriter, loc, elementPtrType, alignmentInt);
     }
 
     // Create the MemRef descriptor.
@@ -268,7 +268,7 @@ class AlignedAllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
       sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
 
     Type elementPtrType = this->getElementPtrType(memRefType);
-    auto results = rewriter.create<LLVM::CallOp>(
+    auto results = LLVM::CallOp::create(rewriter,
         loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
 
     Value ptr =
@@ -360,7 +360,7 @@ struct AllocaOpLowering : public ConvertOpToLLVMPattern<memref::AllocaOp> {
     auto elementPtrType =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addrSpace);
 
-    auto allocatedElementPtr = rewriter.create<LLVM::AllocaOp>(
+    auto allocatedElementPtr = LLVM::AllocaOp::create(rewriter,
         loc, elementPtrType, elementType, size, op.getAlignment().value_or(0));
 
     // Create the MemRef descriptor.
@@ -397,7 +397,7 @@ struct AllocaScopeOpLowering
           remainingOpsBlock, allocaScopeOp.getResultTypes(),
           SmallVector<Location>(allocaScopeOp->getNumResults(),
                                 allocaScopeOp.getLoc()));
-      rewriter.create<LLVM::BrOp>(loc, ValueRange(), remainingOpsBlock);
+      LLVM::BrOp::create(rewriter, loc, ValueRange(), remainingOpsBlock);
     }
 
     // Inline body region.
@@ -407,8 +407,8 @@ struct AllocaScopeOpLowering
 
     // Save stack and then branch into the body of the region.
     rewriter.setInsertionPointToEnd(currentBlock);
-    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
-    rewriter.create<LLVM::BrOp>(loc, ValueRange(), beforeBody);
+    auto stackSaveOp = LLVM::StackSaveOp::create(rewriter, loc, getPtrType());
+    LLVM::BrOp::create(rewriter, loc, ValueRange(), beforeBody);
 
     // Replace the alloca_scope return with a branch that jumps out of the body.
     // Stack restore before leaving the body region.
@@ -420,7 +420,7 @@ struct AllocaScopeOpLowering
 
     // Insert stack restore before jumping out the body of the region.
     rewriter.setInsertionPoint(branchOp);
-    rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);
+    LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
 
     // Replace the op with values return from the body region.
     rewriter.replaceOp(allocaScopeOp, continueBlock->getArguments());
@@ -451,10 +451,10 @@ struct AssumeAlignmentOpLowering
     // This is more direct than ptrtoint-based checks, is explicitly supported,
     // and works with non-integral address spaces.
     Value trueCond =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getBoolAttr(true));
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getBoolAttr(true));
     Value alignmentConst =
         createIndexAttrConstant(rewriter, loc, getIndexType(), alignment);
-    rewriter.create<LLVM::AssumeOp>(loc, trueCond, LLVM::AssumeAlignTag(), ptr,
+    LLVM::AssumeOp::create(rewriter, loc, trueCond, LLVM::AssumeAlignTag(), ptr,
                                     alignmentConst);
     rewriter.replaceOp(op, memref);
     return success();
@@ -559,16 +559,16 @@ struct DimOpLowering : public ConvertOpToLLVMPattern<memref::DimOp> {
     // Get pointer to offset field of memref<element_type> descriptor.
     auto indexPtrTy =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addressSpace);
-    Value offsetPtr = rewriter.create<LLVM::GEPOp>(
+    Value offsetPtr = LLVM::GEPOp::create(rewriter,
         loc, indexPtrTy, elementType, underlyingRankedDesc,
         ArrayRef<LLVM::GEPArg>{0, 2});
 
     // The size value that we have to extract can be obtained using GEPop with
     // `dimOp.index() + 1` index argument.
-    Value idxPlusOne = rewriter.create<LLVM::AddOp>(
+    Value idxPlusOne = LLVM::AddOp::create(rewriter,
         loc, createIndexAttrConstant(rewriter, loc, getIndexType(), 1),
         adaptor.getIndex());
-    Value sizePtr = rewriter.create<LLVM::GEPOp>(
+    Value sizePtr = LLVM::GEPOp::create(rewriter,
         loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr,
         idxPlusOne);
     return rewriter
@@ -674,9 +674,9 @@ struct GenericAtomicRMWOpLowering
     auto memRefType = cast<MemRefType>(atomicOp.getMemref().getType());
     auto dataPtr = getStridedElementPtr(
         rewriter, loc, memRefType, adaptor.getMemref(), adaptor.getIndices());
-    Value init = rewriter.create<LLVM::LoadOp>(
+    Value init = LLVM::LoadOp::create(rewriter,
         loc, typeConverter->convertType(memRefType.getElementType()), dataPtr);
-    rewriter.create<LLVM::BrOp>(loc, init, loopBlock);
+    LLVM::BrOp::create(rewriter, loc, init, loopBlock);
 
     // Prepare the body of the loop block.
     rewriter.setInsertionPointToStart(loopBlock);
@@ -696,14 +696,14 @@ struct GenericAtomicRMWOpLowering
     // Append the cmpxchg op to the end of the loop block.
     auto successOrdering = LLVM::AtomicOrdering::acq_rel;
     auto failureOrdering = LLVM::AtomicOrdering::monotonic;
-    auto cmpxchg = rewriter.create<LLVM::AtomicCmpXchgOp>(
+    auto cmpxchg = LLVM::AtomicCmpXchgOp::create(rewriter,
         loc, dataPtr, loopArgument, result, successOrdering, failureOrdering);
     // Extract the %new_loaded and %ok values from the pair.
-    Value newLoaded = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 0);
-    Value ok = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 1);
+    Value newLoaded = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 0);
+    Value ok = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 1);
 
     // Conditionally branch to the end or back to the loop depending on %ok.
-    rewriter.create<LLVM::CondBrOp>(loc, ok, endBlock, ArrayRef<Value>(),
+    LLVM::CondBrOp::create(rewriter, loc, ok, endBlock, ArrayRef<Value>(),
                                     loopBlock, newLoaded);
 
     rewriter.setInsertionPointToEnd(endBlock);
@@ -800,8 +800,8 @@ class GlobalMemrefOpLowering : public ConvertOpToLLVMPattern<memref::GlobalOp> {
     if (!global.isExternal() && global.isUninitialized()) {
       rewriter.createBlock(&newGlobal.getInitializerRegion());
       Value undef[] = {
-          rewriter.create<LLVM::UndefOp>(global.getLoc(), arrayTy)};
-      rewriter.create<LLVM::ReturnOp>(global.getLoc(), undef);
+          LLVM::UndefOp::create(rewriter, global.getLoc(), arrayTy)};
+      LLVM::ReturnOp::create(rewriter, global.getLoc(), undef);
     }
     return success();
   }
@@ -846,11 +846,11 @@ struct GetGlobalMemrefOpLowering
     Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter());
     auto ptrTy = LLVM::LLVMPointerType::get(rewriter.getContext(), memSpace);
     auto addressOf =
-        rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, op.getName());
+        LLVM::AddressOfOp::create(rewriter, loc, ptrTy, op.getName());
 
     // Get the address of the first element in the array by creating a GEP with
     // the address of the GV as the base, and (rank + 1) number of 0 indices.
-    auto gep = rewriter.create<LLVM::GEPOp>(
+    auto gep = LLVM::GEPOp::create(rewriter,
         loc, ptrTy, arrayTy, addressOf,
         SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
 
@@ -861,7 +861,7 @@ struct GetGlobalMemrefOpLowering
     Value deadBeefConst =
         createIndexAttrConstant(rewriter, op->getLoc(), intPtrType, 0xdeadbeef);
     auto deadBeefPtr =
-        rewriter.create<LLVM::IntToPtrOp>(loc, ptrTy, deadBeefConst);
+        LLVM::IntToPtrOp::create(rewriter, loc, ptrTy, deadBeefConst);
 
     // Both allocated and aligned pointers are same. We could potentially stash
     // a nullptr for the allocated pointer since we do not expect any dealloc.
@@ -1013,7 +1013,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
           loc, adaptor.getSource(), rewriter);
 
       // rank = ConstantOp srcRank
-      auto rankVal = rewriter.create<LLVM::ConstantOp>(
+      auto rankVal = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(rank));
       // poison = PoisonOp
       UnrankedMemRefDescriptor memRefDesc =
@@ -1033,7 +1033,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
       auto ptr = memRefDesc.memRefDescPtr(rewriter, loc);
 
       // struct = LoadOp ptr
-      auto loadOp = rewriter.create<LLVM::LoadOp>(loc, targetStructType, ptr);
+      auto loadOp = LLVM::LoadOp::create(rewriter, loc, targetStructType, ptr);
       rewriter.replaceOp(memRefCastOp, loadOp.getResult());
     } else {
       llvm_unreachable("Unsupported unranked memref to unranked memref cast");
@@ -1067,31 +1067,31 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     MemRefDescriptor srcDesc(adaptor.getSource());
 
     // Compute number of elements.
-    Value numElements = rewriter.create<LLVM::ConstantOp>(
+    Value numElements = LLVM::ConstantOp::create(rewriter,
         loc, getIndexType(), rewriter.getIndexAttr(1));
     for (int pos = 0; pos < srcType.getRank(); ++pos) {
       auto size = srcDesc.size(rewriter, loc, pos);
-      numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
     }
 
     // Get element size.
     auto sizeInBytes = getSizeInBytes(loc, srcType.getElementType(), rewriter);
     // Compute total.
     Value totalSize =
-        rewriter.create<LLVM::MulOp>(loc, numElements, sizeInBytes);
+        LLVM::MulOp::create(rewriter, loc, numElements, sizeInBytes);
 
     Type elementType = typeConverter->convertType(srcType.getElementType());
 
     Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc);
     Value srcOffset = srcDesc.offset(rewriter, loc);
-    Value srcPtr = rewriter.create<LLVM::GEPOp>(
+    Value srcPtr = LLVM::GEPOp::create(rewriter,
         loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset);
     MemRefDescriptor targetDesc(adaptor.getTarget());
     Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc);
     Value targetOffset = targetDesc.offset(rewriter, loc);
-    Value targetPtr = rewriter.create<LLVM::GEPOp>(
+    Value targetPtr = LLVM::GEPOp::create(rewriter,
         loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset);
-    rewriter.create<LLVM::MemcpyOp>(loc, targetPtr, srcPtr, totalSize,
+    LLVM::MemcpyOp::create(rewriter, loc, targetPtr, srcPtr, totalSize,
                                     /*isVolatile=*/false);
     rewriter.eraseOp(op);
 
@@ -1107,7 +1107,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
 
     // First make sure we have an unranked memref descriptor representation.
     auto makeUnranked = [&, this](Value ranked, MemRefType type) {
-      auto rank = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      auto rank = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                     type.getRank());
       auto *typeConverter = getTypeConverter();
       auto ptr =
@@ -1120,7 +1120,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     };
 
     // Save stack position before promoting descriptors
-    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
+    auto stackSaveOp = LLVM::StackSaveOp::create(rewriter, loc, getPtrType());
 
     auto srcMemRefType = dyn_cast<MemRefType>(srcType);
     Value unrankedSource =
@@ -1132,13 +1132,13 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
                          : adaptor.getTarget();
 
     // Now promote the unranked descriptors to the stack.
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+    auto one = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                  rewriter.getIndexAttr(1));
     auto promote = [&](Value desc) {
       auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
       auto allocated =
-          rewriter.create<LLVM::AllocaOp>(loc, ptrType, desc.getType(), one);
-      rewriter.create<LLVM::StoreOp>(loc, desc, allocated);
+          LLVM::AllocaOp::create(rewriter, loc, ptrType, desc.getType(), one);
+      LLVM::StoreOp::create(rewriter, loc, desc, allocated);
       return allocated;
     };
 
@@ -1153,11 +1153,11 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
         sourcePtr.getType(), symbolTables);
     if (failed(copyFn))
       return failure();
-    rewriter.create<LLVM::CallOp>(loc, copyFn.value(),
+    LLVM::CallOp::create(rewriter, loc, copyFn.value(),
                                   ValueRange{elemSize, sourcePtr, targetPtr});
 
     // Restore stack used for descriptors
-    rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);
+    LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
 
     rewriter.eraseOp(op);
 
@@ -1208,9 +1208,9 @@ struct MemorySpaceCastOpLowering
       MemRefDescriptor::unpack(rewriter, loc, adaptor.getSource(), resultTypeR,
                                descVals);
       descVals[0] =
-          rewriter.create<LLVM::AddrSpaceCastOp>(loc, newPtrType, descVals[0]);
+          LLVM::AddrSpaceCastOp::create(rewriter, loc, newPtrType, descVals[0]);
       descVals[1] =
-          rewriter.create<LLVM::AddrSpaceCastOp>(loc, newPtrType, descVals[1]);
+          LLVM::AddrSpaceCastOp::create(rewriter, loc, newPtrType, descVals[1]);
       Value result = MemRefDescriptor::pack(rewriter, loc, *getTypeConverter(),
                                             resultTypeR, descVals);
       rewriter.replaceOp(op, result);
@@ -1245,7 +1245,7 @@ struct MemorySpaceCastOpLowering
       UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                              result, resultAddrSpace, sizes);
       Value resultUnderlyingSize = sizes.front();
-      Value resultUnderlyingDesc = rewriter.create<LLVM::AllocaOp>(
+      Value resultUnderlyingDesc = LLVM::AllocaOp::create(rewriter,
           loc, getPtrType(), rewriter.getI8Type(), resultUnderlyingSize);
       result.setMemRefDescPtr(rewriter, loc, resultUnderlyingDesc);
 
@@ -1260,9 +1260,9 @@ struct MemorySpaceCastOpLowering
       Value alignedPtr =
           sourceDesc.alignedPtr(rewriter, loc, *getTypeConverter(),
                                 sourceUnderlyingDesc, sourceElemPtrType);
-      allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+      allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
           loc, resultElemPtrType, allocatedPtr);
-      alignedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+      alignedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
           loc, resultElemPtrType, alignedPtr);
 
       result.setAllocatedPtr(rewriter, loc, resultUnderlyingDesc,
@@ -1281,11 +1281,11 @@ struct MemorySpaceCastOpLowering
       int64_t bytesToSkip =
           2 * llvm::divideCeil(
                   getTypeConverter()->getPointerBitwidth(resultAddrSpace), 8);
-      Value bytesToSkipConst = rewriter.create<LLVM::ConstantOp>(
+      Value bytesToSkipConst = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(bytesToSkip));
-      Value copySize = rewriter.create<LLVM::SubOp>(
+      Value copySize = LLVM::SubOp::create(rewriter,
           loc, getIndexType(), resultUnderlyingSize, bytesToSkipConst);
-      rewriter.create<LLVM::MemcpyOp>(loc, resultIndexVals, sourceIndexVals,
+      LLVM::MemcpyOp::create(rewriter, loc, resultIndexVals, sourceIndexVals,
                                       copySize, /*isVolatile=*/false);
 
       rewriter.replaceOp(op, ValueRange{result});
@@ -1489,7 +1489,7 @@ struct MemRefReshapeOpLowering
         } else {
           Value shapeOp = reshapeOp.getShape();
           Value index = createIndexAttrConstant(rewriter, loc, indexType, i);
-          dimSize = rewriter.create<memref::LoadOp>(loc, shapeOp, index);
+          dimSize = memref::LoadOp::create(rewriter, loc, shapeOp, index);
           Type indexType = getIndexType();
           if (dimSize.getType() != indexType)
             dimSize = typeConverter->materializeTargetConversion(
@@ -1501,7 +1501,7 @@ struct MemRefReshapeOpLowering
         desc.setStride(rewriter, loc, i, stride);
 
         // Prepare the stride value for the next dimension.
-        stride = rewriter.create<LLVM::MulOp>(loc, stride, dimSize);
+        stride = LLVM::MulOp::create(rewriter, loc, stride, dimSize);
       }
 
       *descriptor = desc;
@@ -1526,7 +1526,7 @@ struct MemRefReshapeOpLowering
     SmallVector<Value, 4> sizes;
     UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                            targetDesc, addressSpace, sizes);
-    Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>(
+    Value underlyingDescPtr = LLVM::AllocaOp::create(rewriter,
         loc, getPtrType(), IntegerType::get(getContext(), 8), sizes.front());
     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
 
@@ -1558,7 +1558,7 @@ struct MemRefReshapeOpLowering
     Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc);
     Value oneIndex = createIndexAttrConstant(rewriter, loc, getIndexType(), 1);
     Value resultRankMinusOne =
-        rewriter.create<LLVM::SubOp>(loc, resultRank, oneIndex);
+        LLVM::SubOp::create(rewriter, loc, resultRank, oneIndex);
 
     Block *initBlock = rewriter.getInsertionBlock();
     Type indexType = getTypeConverter()->getIndexType();
@@ -1572,14 +1572,14 @@ struct MemRefReshapeOpLowering
     rewriter.mergeBlocks(remainingBlock, condBlock, ValueRange());
 
     rewriter.setInsertionPointToEnd(initBlock);
-    rewriter.create<LLVM::BrOp>(loc, ValueRange({resultRankMinusOne, oneIndex}),
+    LLVM::BrOp::create(rewriter, loc, ValueRange({resultRankMinusOne, oneIndex}),
                                 condBlock);
     rewriter.setInsertionPointToStart(condBlock);
     Value indexArg = condBlock->getArgument(0);
     Value strideArg = condBlock->getArgument(1);
 
     Value zeroIndex = createIndexAttrConstant(rewriter, loc, indexType, 0);
-    Value pred = rewriter.create<LLVM::ICmpOp>(
+    Value pred = LLVM::ICmpOp::create(rewriter,
         loc, IntegerType::get(rewriter.getContext(), 1),
         LLVM::ICmpPredicate::sge, indexArg, zeroIndex);
 
@@ -1589,22 +1589,22 @@ struct MemRefReshapeOpLowering
 
     // Copy size from shape to descriptor.
     auto llvmIndexPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value sizeLoadGep = rewriter.create<LLVM::GEPOp>(
+    Value sizeLoadGep = LLVM::GEPOp::create(rewriter,
         loc, llvmIndexPtrType,
         typeConverter->convertType(shapeMemRefType.getElementType()),
         shapeOperandPtr, indexArg);
-    Value size = rewriter.create<LLVM::LoadOp>(loc, indexType, sizeLoadGep);
+    Value size = LLVM::LoadOp::create(rewriter, loc, indexType, sizeLoadGep);
     UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(),
                                       targetSizesBase, indexArg, size);
 
     // Write stride value and compute next one.
     UnrankedMemRefDescriptor::setStride(rewriter, loc, *getTypeConverter(),
                                         targetStridesBase, indexArg, strideArg);
-    Value nextStride = rewriter.create<LLVM::MulOp>(loc, strideArg, size);
+    Value nextStride = LLVM::MulOp::create(rewriter, loc, strideArg, size);
 
     // Decrement loop counter and branch back.
-    Value decrement = rewriter.create<LLVM::SubOp>(loc, indexArg, oneIndex);
-    rewriter.create<LLVM::BrOp>(loc, ValueRange({decrement, nextStride}),
+    Value decrement = LLVM::SubOp::create(rewriter, loc, indexArg, oneIndex);
+    LLVM::BrOp::create(rewriter, loc, ValueRange({decrement, nextStride}),
                                 condBlock);
 
     Block *remainder =
@@ -1612,7 +1612,7 @@ struct MemRefReshapeOpLowering
 
     // Hook up the cond exit to the remainder.
     rewriter.setInsertionPointToEnd(condBlock);
-    rewriter.create<LLVM::CondBrOp>(loc, pred, bodyBlock, ValueRange(),
+    LLVM::CondBrOp::create(rewriter, loc, pred, bodyBlock, ValueRange(),
                                     remainder, ValueRange());
 
     // Reset position to beginning of new remainder block.
@@ -1742,7 +1742,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
       return createIndexAttrConstant(rewriter, loc, indexType, strides[idx]);
     if (nextSize)
       return runningStride
-                 ? rewriter.create<LLVM::MulOp>(loc, runningStride, nextSize)
+                 ? LLVM::MulOp::create(rewriter, loc, runningStride, nextSize)
                  : nextSize;
     assert(!runningStride);
     return createIndexAttrConstant(rewriter, loc, indexType, 1);
@@ -1787,7 +1787,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
 
     // Field 2: Copy the actual aligned pointer to payload.
     Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc);
-    alignedPtr = rewriter.create<LLVM::GEPOp>(
+    alignedPtr = LLVM::GEPOp::create(rewriter,
         loc, alignedPtr.getType(),
         typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr,
         adaptor.getByteShift());
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index b866afbce98b0..7b7ad9b95b0ec 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -79,7 +79,7 @@ adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter,
   assert(indices.size() == 2);
   indices.back() = builder.createOrFold<spirv::SDivOp>(loc, lastDim, idx);
   Type t = typeConverter.convertType(op.getComponentPtr().getType());
-  return builder.create<spirv::AccessChainOp>(loc, t, op.getBasePtr(), indices);
+  return spirv::AccessChainOp::create(builder, loc, t, op.getBasePtr(), indices);
 }
 
 /// Casts the given `srcBool` into an integer of `dstType`.
@@ -107,7 +107,7 @@ static Value shiftValue(Location loc, Value value, Value offset, Value mask,
     value = castBoolToIntN(loc, value, dstType, builder);
   } else {
     if (valueBits < targetBits) {
-      value = builder.create<spirv::UConvertOp>(
+      value = spirv::UConvertOp::create(builder,
           loc, builder.getIntegerType(targetBits), value);
     }
 
@@ -372,7 +372,7 @@ AllocOpPattern::matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor,
     std::string varName =
         std::string("__workgroup_mem__") +
         std::to_string(std::distance(varOps.begin(), varOps.end()));
-    varOp = rewriter.create<spirv::GlobalVariableOp>(loc, spirvType, varName,
+    varOp = spirv::GlobalVariableOp::create(rewriter, loc, spirvType, varName,
                                                      /*initializer=*/nullptr);
   }
 
@@ -572,7 +572,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
           loadOp, "failed to determine memory requirements");
 
     auto [memoryAccess, alignment] = *memoryRequirements;
-    Value loadVal = rewriter.create<spirv::LoadOp>(loc, accessChain,
+    Value loadVal = spirv::LoadOp::create(rewriter, loc, accessChain,
                                                    memoryAccess, alignment);
     if (isBool)
       loadVal = castIntNToBool(loc, loadVal, rewriter);
@@ -601,7 +601,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
         loadOp, "failed to determine memory requirements");
 
   auto [memoryAccess, alignment] = *memoryRequirements;
-  Value spvLoadOp = rewriter.create<spirv::LoadOp>(loc, dstType, adjustedPtr,
+  Value spvLoadOp = spirv::LoadOp::create(rewriter, loc, dstType, adjustedPtr,
                                                    memoryAccess, alignment);
 
   // Shift the bits to the rightmost.
@@ -770,10 +770,10 @@ IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor,
   if (!scope)
     return rewriter.notifyMatchFailure(storeOp, "atomic scope not available");
 
-  Value result = rewriter.create<spirv::AtomicAndOp>(
+  Value result = spirv::AtomicAndOp::create(rewriter,
       loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
       clearBitsMask);
-  result = rewriter.create<spirv::AtomicOrOp>(
+  result = spirv::AtomicOrOp::create(rewriter,
       loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
       storeVal);
 
@@ -851,11 +851,11 @@ LogicalResult MemorySpaceCastOpPattern::matchAndRewrite(
   }
   if (sourceSc != spirv::StorageClass::Generic) {
     result =
-        rewriter.create<spirv::PtrCastToGenericOp>(loc, genericPtrType, result);
+        spirv::PtrCastToGenericOp::create(rewriter, loc, genericPtrType, result);
   }
   if (resultSc != spirv::StorageClass::Generic) {
     result =
-        rewriter.create<spirv::GenericCastToPtrOp>(loc, resultPtrType, result);
+        spirv::GenericCastToPtrOp::create(rewriter, loc, resultPtrType, result);
   }
   rewriter.replaceOp(addrCastOp, result);
   return success();
diff --git a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
index b93128441f2b5..f16b92a287d0a 100644
--- a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
+++ b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
@@ -65,7 +65,7 @@ static SmallVector<Value> getMixedAsValues(OpBuilder b, const Location &loc,
       values.emplace_back(*(dyn++));
     } else {
       TypedAttr val = type == i64 ? b.getI64IntegerAttr(s) : b.getIndexAttr(s);
-      values.emplace_back(b.create<arith::ConstantOp>(loc, type, val));
+      values.emplace_back(arith::ConstantOp::create(b, loc, type, val));
     }
   }
   return values;
@@ -79,9 +79,9 @@ static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
   SmallVector<Value> multiIndex(n);
 
   for (int i = n - 1; i >= 0; --i) {
-    multiIndex[i] = b.create<arith::RemSIOp>(loc, linearIndex, dimensions[i]);
+    multiIndex[i] = arith::RemSIOp::create(b, loc, linearIndex, dimensions[i]);
     if (i > 0)
-      linearIndex = b.create<arith::DivSIOp>(loc, linearIndex, dimensions[i]);
+      linearIndex = arith::DivSIOp::create(b, loc, linearIndex, dimensions[i]);
   }
 
   return multiIndex;
@@ -91,13 +91,13 @@ static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
 Value multiToLinearIndex(Location loc, OpBuilder b, ValueRange multiIndex,
                          ValueRange dimensions) {
 
-  Value linearIndex = b.create<arith::ConstantIndexOp>(loc, 0);
-  Value stride = b.create<arith::ConstantIndexOp>(loc, 1);
+  Value linearIndex = arith::ConstantIndexOp::create(b, loc, 0);
+  Value stride = arith::ConstantIndexOp::create(b, loc, 1);
 
   for (int i = multiIndex.size() - 1; i >= 0; --i) {
-    Value off = b.create<arith::MulIOp>(loc, multiIndex[i], stride);
-    linearIndex = b.create<arith::AddIOp>(loc, linearIndex, off);
-    stride = b.create<arith::MulIOp>(loc, stride, dimensions[i]);
+    Value off = arith::MulIOp::create(b, loc, multiIndex[i], stride);
+    linearIndex = arith::AddIOp::create(b, loc, linearIndex, off);
+    stride = arith::MulIOp::create(b, loc, stride, dimensions[i]);
   }
 
   return linearIndex;
@@ -144,10 +144,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     auto i64 = rewriter.getI64Type();
     std::array<int64_t, 2> shape = {static_cast<int64_t>(splitAxes.size()),
                                     maxNAxes};
-    Value resSplitAxes = rewriter.create<tensor::EmptyOp>(loc, shape, i16);
+    Value resSplitAxes = tensor::EmptyOp::create(rewriter, loc, shape, i16);
     auto attr = IntegerAttr::get(i16, -1);
-    Value fillValue = rewriter.create<arith::ConstantOp>(loc, i16, attr);
-    resSplitAxes = rewriter.create<linalg::FillOp>(loc, fillValue, resSplitAxes)
+    Value fillValue = arith::ConstantOp::create(rewriter, loc, i16, attr);
+    resSplitAxes = linalg::FillOp::create(rewriter, loc, fillValue, resSplitAxes)
                        .getResult(0);
 
     // explicitly write values into tensor row by row
@@ -162,8 +162,8 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       std::array<int64_t, 2> sizes = {1, size};
       auto tensorType = RankedTensorType::get({size}, i16);
       auto attrs = DenseIntElementsAttr::get(tensorType, axes.asArrayRef());
-      auto vals = rewriter.create<arith::ConstantOp>(loc, tensorType, attrs);
-      resSplitAxes = rewriter.create<tensor::InsertSliceOp>(
+      auto vals = arith::ConstantOp::create(rewriter, loc, tensorType, attrs);
+      resSplitAxes = tensor::InsertSliceOp::create(rewriter,
           loc, vals, resSplitAxes, empty, empty, empty, offs, sizes, strides);
     }
 
@@ -179,7 +179,7 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
                   .create<tensor::EmptyOp>(loc, std::array<int64_t, 2>{0, 0},
                                            i64)
                   .getResult()
-            : rewriter.create<tensor::FromElementsOp>(loc, type, haloSizes)
+            : tensor::FromElementsOp::create(rewriter, loc, type, haloSizes)
                   .getResult();
 
     // To hold sharded dims offsets, create Tensor with shape {nSplits,
@@ -189,7 +189,7 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     // MeshOp)
     Value resOffsets;
     if (adaptor.getStaticShardedDimsOffsets().empty()) {
-      resOffsets = rewriter.create<tensor::EmptyOp>(
+      resOffsets = tensor::EmptyOp::create(rewriter,
           loc, std::array<int64_t, 2>{0, 0}, i64);
     } else {
       SymbolTableCollection symbolTableCollection;
@@ -204,12 +204,12 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       assert(maxSplitSize);
       ++maxSplitSize; // add one for the total size
 
-      resOffsets = rewriter.create<tensor::EmptyOp>(
+      resOffsets = tensor::EmptyOp::create(rewriter,
           loc, std::array<int64_t, 2>{nSplits, maxSplitSize}, i64);
-      Value zero = rewriter.create<arith::ConstantOp>(
+      Value zero = arith::ConstantOp::create(rewriter,
           loc, i64, rewriter.getI64IntegerAttr(ShapedType::kDynamic));
       resOffsets =
-          rewriter.create<linalg::FillOp>(loc, zero, resOffsets).getResult(0);
+          linalg::FillOp::create(rewriter, loc, zero, resOffsets).getResult(0);
       SmallVector<Value> offsets =
           getMixedAsValues(rewriter, loc, adaptor.getStaticShardedDimsOffsets(),
                            adaptor.getDynamicShardedDimsOffsets());
@@ -220,10 +220,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
         assert(splitSize != ShapedType::kDynamic && splitSize < maxSplitSize);
         ++splitSize; // add one for the total size
         ArrayRef<Value> values(&offsets[curr], splitSize);
-        Value vals = rewriter.create<tensor::FromElementsOp>(loc, values);
+        Value vals = tensor::FromElementsOp::create(rewriter, loc, values);
         std::array<int64_t, 2> offs = {static_cast<int64_t>(i), 0};
         std::array<int64_t, 2> sizes = {1, splitSize};
-        resOffsets = rewriter.create<tensor::InsertSliceOp>(
+        resOffsets = tensor::InsertSliceOp::create(rewriter,
             loc, vals, resOffsets, empty, empty, empty, offs, sizes, strides);
         curr += splitSize;
       }
@@ -236,10 +236,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       return failure();
 
     resSplitAxes =
-        rewriter.create<tensor::CastOp>(loc, resTypes[0], resSplitAxes);
+        tensor::CastOp::create(rewriter, loc, resTypes[0], resSplitAxes);
     resHaloSizes =
-        rewriter.create<tensor::CastOp>(loc, resTypes[1], resHaloSizes);
-    resOffsets = rewriter.create<tensor::CastOp>(loc, resTypes[2], resOffsets);
+        tensor::CastOp::create(rewriter, loc, resTypes[1], resHaloSizes);
+    resOffsets = tensor::CastOp::create(rewriter, loc, resTypes[2], resOffsets);
 
     rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
         op, TupleType::get(op.getContext(), resTypes),
@@ -269,9 +269,9 @@ struct ConvertProcessMultiIndexOp
     SmallVector<Value> dims;
     llvm::transform(
         meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
-          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+          return arith::ConstantIndexOp::create(rewriter, loc, i).getResult();
         });
-    Value rank = rewriter.create<ProcessLinearIndexOp>(op.getLoc(), meshOp);
+    Value rank = ProcessLinearIndexOp::create(rewriter, op.getLoc(), meshOp);
     auto mIdx = linearToMultiIndex(loc, rewriter, rank, dims);
 
     // optionally extract subset of mesh axes
@@ -302,7 +302,7 @@ class ConvertProcessLinearIndexOp
     Location loc = op.getLoc();
     auto ctx = op.getContext();
     Value commWorld =
-        rewriter.create<mpi::CommWorldOp>(loc, mpi::CommType::get(ctx));
+        mpi::CommWorldOp::create(rewriter, loc, mpi::CommType::get(ctx));
     auto rank =
         rewriter
             .create<mpi::CommRankOp>(
@@ -341,40 +341,40 @@ struct ConvertNeighborsLinearIndicesOp
     SmallVector<Value> dims;
     llvm::transform(
         meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
-          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+          return arith::ConstantIndexOp::create(rewriter, loc, i).getResult();
         });
     Value dimSz = dims[axes[0]];
-    Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    Value minus1 = rewriter.create<arith::ConstantIndexOp>(loc, -1);
-    Value atBorder = rewriter.create<arith::CmpIOp>(
+    Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    Value minus1 = arith::ConstantIndexOp::create(rewriter, loc, -1);
+    Value atBorder = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sle, orgIdx,
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
-    auto down = rewriter.create<scf::IfOp>(
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
+    auto down = scf::IfOp::create(rewriter,
         loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
-          builder.create<scf::YieldOp>(loc, minus1);
+          scf::YieldOp::create(builder, loc, minus1);
         },
         [&](OpBuilder &builder, Location loc) {
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
-              rewriter.create<arith::SubIOp>(op.getLoc(), orgIdx, one)
+              arith::SubIOp::create(rewriter, op.getLoc(), orgIdx, one)
                   .getResult();
-          builder.create<scf::YieldOp>(
+          scf::YieldOp::create(builder,
               loc, multiToLinearIndex(loc, rewriter, tmp, dims));
         });
-    atBorder = rewriter.create<arith::CmpIOp>(
+    atBorder = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, orgIdx,
-        rewriter.create<arith::SubIOp>(loc, dimSz, one).getResult());
-    auto up = rewriter.create<scf::IfOp>(
+        arith::SubIOp::create(rewriter, loc, dimSz, one).getResult());
+    auto up = scf::IfOp::create(rewriter,
         loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
-          builder.create<scf::YieldOp>(loc, minus1);
+          scf::YieldOp::create(builder, loc, minus1);
         },
         [&](OpBuilder &builder, Location loc) {
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
-              rewriter.create<arith::AddIOp>(op.getLoc(), orgIdx, one);
-          builder.create<scf::YieldOp>(
+              arith::AddIOp::create(rewriter, op.getLoc(), orgIdx, one);
+          scf::YieldOp::create(builder,
               loc, multiToLinearIndex(loc, rewriter, tmp, dims));
         });
     rewriter.replaceOp(op, ValueRange{down.getResult(0), up.getResult(0)});
@@ -447,7 +447,7 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           rewriter, loc, sharding.getStaticShardedDimsOffsets(),
           sharding.getDynamicShardedDimsOffsets(), index);
       if (!tmp.empty())
-        shardedDimsOffs = rewriter.create<tensor::FromElementsOp>(
+        shardedDimsOffs = tensor::FromElementsOp::create(rewriter,
             loc, RankedTensorType::get({(int64_t)tmp.size()}, index), tmp);
     }
 
@@ -457,9 +457,9 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
     int64_t pos = 0;
     SmallVector<Value> shardShape;
     Value zero =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(index));
     Value one =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getOneAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getOneAttr(index));
 
     // Iterate over the dimensions of the tensor shape, get their split Axes,
     // and compute the sharded shape.
@@ -470,7 +470,7 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
         // The current dimension might not be sharded.
         // Create a value from the static position in shardDimsOffsets.
         Value posVal =
-            rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(pos));
+            arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(pos));
         // Get the index of the local shard in the mesh axis.
         Value idx = multiIdx[axes[0]];
         auto numShards =
@@ -482,29 +482,29 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
             return op->emitError() << "Only single axis sharding is "
                                    << "supported for each dimension.";
           }
-          idx = rewriter.create<arith::AddIOp>(loc, posVal, idx);
+          idx = arith::AddIOp::create(rewriter, loc, posVal, idx);
           // Compute size = shardedDimsOffs[idx+1] - shardedDimsOffs[idx].
           Value off =
-              rewriter.create<tensor::ExtractOp>(loc, shardedDimsOffs, idx);
-          idx = rewriter.create<arith::AddIOp>(loc, idx, one);
+              tensor::ExtractOp::create(rewriter, loc, shardedDimsOffs, idx);
+          idx = arith::AddIOp::create(rewriter, loc, idx, one);
           Value nextOff =
-              rewriter.create<tensor::ExtractOp>(loc, shardedDimsOffs, idx);
-          Value sz = rewriter.create<arith::SubIOp>(loc, nextOff, off);
+              tensor::ExtractOp::create(rewriter, loc, shardedDimsOffs, idx);
+          Value sz = arith::SubIOp::create(rewriter, loc, nextOff, off);
           shardShape.emplace_back(sz);
         } else {
-          Value numShardsVal = rewriter.create<arith::ConstantOp>(
+          Value numShardsVal = arith::ConstantOp::create(rewriter,
               loc, rewriter.getIndexAttr(numShards));
           // Compute shard dim size by distributing odd elements to trailing
           // shards:
           // sz = dim / numShards
           //      + (idx >= (numShards - (dim % numShards)) ? 1 : 0)
-          Value sz = rewriter.create<arith::DivSIOp>(loc, dim, numShardsVal);
-          Value sz1 = rewriter.create<arith::RemSIOp>(loc, dim, numShardsVal);
-          sz1 = rewriter.create<arith::SubIOp>(loc, numShardsVal, sz1);
-          auto cond = rewriter.create<arith::CmpIOp>(
+          Value sz = arith::DivSIOp::create(rewriter, loc, dim, numShardsVal);
+          Value sz1 = arith::RemSIOp::create(rewriter, loc, dim, numShardsVal);
+          sz1 = arith::SubIOp::create(rewriter, loc, numShardsVal, sz1);
+          auto cond = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::sge, idx, sz1);
-          Value odd = rewriter.create<arith::SelectOp>(loc, cond, one, zero);
-          sz = rewriter.create<arith::AddIOp>(loc, sz, odd);
+          Value odd = arith::SelectOp::create(rewriter, loc, cond, one, zero);
+          sz = arith::AddIOp::create(rewriter, loc, sz, odd);
           shardShape.emplace_back(sz);
         }
         pos += numShards + 1; // add one for the total size.
@@ -568,7 +568,7 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (isa<RankedTensorType>(input.getType())) {
       auto memrefType = MemRefType::get(
           inputShape, cast<ShapedType>(input.getType()).getElementType());
-      input = iBuilder.create<bufferization::ToBufferOp>(memrefType, input);
+      input = bufferization::ToBufferOp::create(iBuilder, memrefType, input);
     }
     MemRefType inType = cast<MemRefType>(input.getType());
 
@@ -577,15 +577,15 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     for (auto i = 0; i < inType.getRank(); ++i) {
       auto s = inputShape[i];
       if (ShapedType::isDynamic(s))
-        shape[i] = iBuilder.create<memref::DimOp>(input, s).getResult();
+        shape[i] = memref::DimOp::create(iBuilder, input, s).getResult();
       else
         shape[i] = iBuilder.getIndexAttr(s);
     }
 
     // Allocate buffer and copy input to buffer.
-    Value buffer = iBuilder.create<memref::AllocOp>(
+    Value buffer = memref::AllocOp::create(iBuilder,
         shape, cast<ShapedType>(op.getType()).getElementType());
-    iBuilder.create<linalg::CopyOp>(input, buffer);
+    linalg::CopyOp::create(iBuilder, input, buffer);
 
     // Get an MPI_Comm_split for the AllReduce operation.
     // The color is the linear index of the process in the mesh along the
@@ -594,9 +594,9 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
                                        iBuilder.getIndexType());
     SmallVector<Value> myMultiIndex =
-        iBuilder.create<ProcessMultiIndexOp>(indexResultTypes, mesh)
+        ProcessMultiIndexOp::create(iBuilder, indexResultTypes, mesh)
             .getResult();
-    Value zero = iBuilder.create<arith::ConstantIndexOp>(0);
+    Value zero = arith::ConstantIndexOp::create(iBuilder, 0);
     SmallVector<Value> multiKey(myMultiIndex.size(), zero);
 
     auto redAxes = adaptor.getMeshAxes();
@@ -607,15 +607,15 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
 
     Value color =
         createProcessLinearIndex(mesh, myMultiIndex, redAxes, iBuilder);
-    color = iBuilder.create<arith::IndexCastOp>(iBuilder.getI32Type(), color);
+    color = arith::IndexCastOp::create(iBuilder, iBuilder.getI32Type(), color);
     Value key = createProcessLinearIndex(mesh, multiKey, redAxes, iBuilder);
-    key = iBuilder.create<arith::IndexCastOp>(iBuilder.getI32Type(), key);
+    key = arith::IndexCastOp::create(iBuilder, iBuilder.getI32Type(), key);
 
     // Finally split the communicator
     auto commType = mpi::CommType::get(op->getContext());
-    Value commWorld = iBuilder.create<mpi::CommWorldOp>(commType);
+    Value commWorld = mpi::CommWorldOp::create(iBuilder, commType);
     auto comm =
-        iBuilder.create<mpi::CommSplitOp>(commType, commWorld, color, key)
+        mpi::CommSplitOp::create(iBuilder, commType, commWorld, color, key)
             .getNewcomm();
 
     Value buffer1d = buffer;
@@ -623,18 +623,18 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (inType.getRank() > 1) {
       ReassociationIndices reassociation(inType.getRank());
       std::iota(reassociation.begin(), reassociation.end(), 0);
-      buffer1d = iBuilder.create<memref::CollapseShapeOp>(
+      buffer1d = memref::CollapseShapeOp::create(iBuilder,
           buffer, ArrayRef<ReassociationIndices>(reassociation));
     }
 
     // Create the MPI AllReduce operation.
-    iBuilder.create<mpi::AllReduceOp>(
+    mpi::AllReduceOp::create(iBuilder,
         TypeRange(), buffer1d, buffer1d,
         getMPIReductionOp(adaptor.getReductionAttr()), comm);
 
     // If the destination is a memref, cast it to a tensor
     if (isa<RankedTensorType>(op.getType()))
-      buffer = iBuilder.create<bufferization::ToTensorOp>(op.getType(), buffer,
+      buffer = bufferization::ToTensorOp::create(iBuilder, op.getType(), buffer,
                                                           true);
 
     rewriter.replaceOp(op, buffer);
@@ -676,7 +676,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     auto toValue = [&rewriter, &loc](OpFoldResult &v) -> Value {
       if (auto value = dyn_cast<Value>(v))
         return value;
-      return rewriter.create<arith::ConstantOp>(
+      return arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(
                    cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
     };
@@ -689,7 +689,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       auto mmemrefType = MemRefType::get(
           dstShape, cast<ShapedType>(array.getType()).getElementType());
       array =
-          rewriter.create<bufferization::ToBufferOp>(loc, mmemrefType, array);
+          bufferization::ToBufferOp::create(rewriter, loc, mmemrefType, array);
     }
     auto rank = cast<ShapedType>(array.getType()).getRank();
     auto opSplitAxes = adaptor.getSplitAxes().getAxes();
@@ -713,7 +713,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     for (auto i = 0; i < rank; ++i) {
       auto s = dstShape[i];
       if (ShapedType::isDynamic(s))
-        shape[i] = rewriter.create<memref::DimOp>(loc, array, s).getResult();
+        shape[i] = memref::DimOp::create(rewriter, loc, array, s).getResult();
       else
         shape[i] = rewriter.getIndexAttr(s);
 
@@ -723,12 +723,12 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         offsets[i] = haloSizes[currHaloDim * 2];
 
         // prepare shape and offsets of highest dim's halo exchange
-        Value _haloSz = rewriter.create<arith::AddIOp>(
+        Value _haloSz = arith::AddIOp::create(rewriter,
             loc, toValue(haloSizes[currHaloDim * 2]),
             toValue(haloSizes[currHaloDim * 2 + 1]));
         // the halo shape of lower dims exlude the halos
         dimSizes[i] =
-            rewriter.create<arith::SubIOp>(loc, toValue(shape[i]), _haloSz)
+            arith::SubIOp::create(rewriter, loc, toValue(shape[i]), _haloSz)
                 .getResult();
       } else {
         dimSizes[i] = shape[i];
@@ -736,14 +736,14 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     }
 
     auto tagAttr = rewriter.getI32IntegerAttr(91); // we just pick something
-    auto tag = rewriter.create<arith::ConstantOp>(loc, tagAttr);
+    auto tag = arith::ConstantOp::create(rewriter, loc, tagAttr);
     auto zeroAttr = rewriter.getI32IntegerAttr(0); // for detecting v<0
-    auto zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
+    auto zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
 
     SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
                                        rewriter.getIndexType());
     auto myMultiIndex =
-        rewriter.create<ProcessMultiIndexOp>(loc, indexResultTypes, mesh)
+        ProcessMultiIndexOp::create(rewriter, loc, indexResultTypes, mesh)
             .getResult();
     // traverse all split axes from high to low dim
     for (ssize_t dim = opSplitAxes.size() - 1; dim >= 0; --dim) {
@@ -758,19 +758,19 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
                                                        splitAxes)
                      .getResults();
       // MPI operates on i32...
-      Value neighbourIDs[2] = {rewriter.create<arith::IndexCastOp>(
+      Value neighbourIDs[2] = {arith::IndexCastOp::create(rewriter,
                                    loc, rewriter.getI32Type(), tmp[0]),
-                               rewriter.create<arith::IndexCastOp>(
+                               arith::IndexCastOp::create(rewriter,
                                    loc, rewriter.getI32Type(), tmp[1])};
 
       auto lowerRecvOffset = rewriter.getIndexAttr(0);
       auto lowerSendOffset = toValue(haloSizes[currHaloDim * 2]);
-      auto upperRecvOffset = rewriter.create<arith::SubIOp>(
+      auto upperRecvOffset = arith::SubIOp::create(rewriter,
           loc, toValue(shape[dim]), toValue(haloSizes[currHaloDim * 2 + 1]));
-      auto upperSendOffset = rewriter.create<arith::SubIOp>(
+      auto upperSendOffset = arith::SubIOp::create(rewriter,
           loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
 
-      Value commWorld = rewriter.create<mpi::CommWorldOp>(
+      Value commWorld = mpi::CommWorldOp::create(rewriter,
           loc, mpi::CommType::get(op->getContext()));
 
       // Make sure we send/recv in a way that does not lead to a dead-lock.
@@ -787,37 +787,37 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         // Processes on the mesh borders have only one neighbor
         auto to = upperHalo ? neighbourIDs[0] : neighbourIDs[1];
         auto from = upperHalo ? neighbourIDs[1] : neighbourIDs[0];
-        auto hasFrom = rewriter.create<arith::CmpIOp>(
+        auto hasFrom = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sge, from, zero);
-        auto hasTo = rewriter.create<arith::CmpIOp>(
+        auto hasTo = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sge, to, zero);
-        auto buffer = rewriter.create<memref::AllocOp>(
+        auto buffer = memref::AllocOp::create(rewriter,
             loc, dimSizes, cast<ShapedType>(array.getType()).getElementType());
         // if has neighbor: copy halo data from array to buffer and send
-        rewriter.create<scf::IfOp>(
+        scf::IfOp::create(rewriter,
             loc, hasTo, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(lowerSendOffset)
                                        : OpFoldResult(upperSendOffset);
-              auto subview = builder.create<memref::SubViewOp>(
+              auto subview = memref::SubViewOp::create(builder,
                   loc, array, offsets, dimSizes, strides);
-              builder.create<memref::CopyOp>(loc, subview, buffer);
-              builder.create<mpi::SendOp>(loc, TypeRange{}, buffer, tag, to,
+              memref::CopyOp::create(builder, loc, subview, buffer);
+              mpi::SendOp::create(builder, loc, TypeRange{}, buffer, tag, to,
                                           commWorld);
-              builder.create<scf::YieldOp>(loc);
+              scf::YieldOp::create(builder, loc);
             });
         // if has neighbor: receive halo data into buffer and copy to array
-        rewriter.create<scf::IfOp>(
+        scf::IfOp::create(rewriter,
             loc, hasFrom, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(upperRecvOffset)
                                        : OpFoldResult(lowerRecvOffset);
-              builder.create<mpi::RecvOp>(loc, TypeRange{}, buffer, tag, from,
+              mpi::RecvOp::create(builder, loc, TypeRange{}, buffer, tag, from,
                                           commWorld);
-              auto subview = builder.create<memref::SubViewOp>(
+              auto subview = memref::SubViewOp::create(builder,
                   loc, array, offsets, dimSizes, strides);
-              builder.create<memref::CopyOp>(loc, buffer, subview);
-              builder.create<scf::YieldOp>(loc);
+              memref::CopyOp::create(builder, loc, buffer, subview);
+              scf::YieldOp::create(builder, loc);
             });
-        rewriter.create<memref::DeallocOp>(loc, buffer);
+        memref::DeallocOp::create(rewriter, loc, buffer);
         offsets[dim] = orgOffset;
       };
 
@@ -825,15 +825,15 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         OpFoldResult &v = haloSizes[currHaloDim * 2 + upOrDown];
         Value haloSz = dyn_cast<Value>(v);
         if (!haloSz)
-          haloSz = rewriter.create<arith::ConstantOp>(
+          haloSz = arith::ConstantOp::create(rewriter,
               loc, rewriter.getI32IntegerAttr(
                        cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
-        auto hasSize = rewriter.create<arith::CmpIOp>(
+        auto hasSize = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sgt, haloSz, zero);
-        rewriter.create<scf::IfOp>(loc, hasSize,
+        scf::IfOp::create(rewriter, loc, hasSize,
                                    [&](OpBuilder &builder, Location loc) {
                                      genSendRecv(upOrDown > 0);
-                                     builder.create<scf::YieldOp>(loc);
+                                     scf::YieldOp::create(builder, loc);
                                    });
       };
 
@@ -852,7 +852,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       rewriter.replaceOp(op, array);
     } else {
       assert(isa<RankedTensorType>(op.getResult().getType()));
-      rewriter.replaceOp(op, rewriter.create<bufferization::ToTensorOp>(
+      rewriter.replaceOp(op, bufferization::ToTensorOp::create(rewriter,
                                  loc, op.getResult().getType(), array,
                                  /*restrict=*/true, /*writable=*/true));
     }
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 80b3d85488495..3f82342001ea5 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -53,7 +53,7 @@ static Value truncToI32(ImplicitLocOpBuilder &b, Value value) {
   assert(llvm::isa<IntegerType>(type) && "expected an integer Value");
   if (type.getIntOrFloatBitWidth() <= 32)
     return value;
-  return b.create<LLVM::TruncOp>(b.getI32Type(), value);
+  return LLVM::TruncOp::create(b, b.getI32Type(), value);
 }
 
 /// Returns the type for the intrinsic given the vectorResultType of the
@@ -113,7 +113,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
   Type f32x1Ty = VectorType::get(1, f32Ty);
 
   auto makeConst = [&](int32_t index) -> Value {
-    return rewriter.create<LLVM::ConstantOp>(loc, IntegerType::get(ctx, 32),
+    return LLVM::ConstantOp::create(rewriter, loc, IntegerType::get(ctx, 32),
                                              rewriter.getI32IntegerAttr(index));
   };
 
@@ -126,7 +126,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
         arrayType.getElementType() == f32x1Ty) {
       for (unsigned i = 0; i < structType.getBody().size(); i++) {
         Value el =
-            rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i);
+            LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i);
         el = rewriter.createOrFold<LLVM::BitcastOp>(
             loc, arrayType.getElementType(), el);
         elements.push_back(el);
@@ -143,23 +143,23 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
       for (unsigned i = 0, e = structType.getBody().size() / 2; i < e; i++) {
         Value vec =
-            rewriter.create<LLVM::PoisonOp>(loc, arrayType.getElementType());
+            LLVM::PoisonOp::create(rewriter, loc, arrayType.getElementType());
         Value x1 =
-            rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i * 2);
-        Value x2 = rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult,
+            LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i * 2);
+        Value x2 = LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult,
                                                          i * 2 + 1);
-        vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec,
+        vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
                                                      x1, makeConst(0));
-        vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec,
+        vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
                                                      x2, makeConst(1));
         elements.push_back(vec);
       }
     }
 
     // Create the final vectorized result.
-    Value result = rewriter.create<LLVM::PoisonOp>(loc, arrayType);
+    Value result = LLVM::PoisonOp::create(rewriter, loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
-      result = rewriter.create<LLVM::InsertValueOp>(loc, result, el.value(),
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, el.value(),
                                                     el.index());
     }
     return result;
@@ -187,7 +187,7 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
   auto arrayTy = cast<LLVM::LLVMArrayType>(operand.getType());
 
   for (unsigned i = 0, e = arrayTy.getNumElements(); i < e; ++i) {
-    Value toUse = b.create<LLVM::ExtractValueOp>(operand, i);
+    Value toUse = LLVM::ExtractValueOp::create(b, operand, i);
 
     // For 4xi8 vectors, the intrinsic expects these to be provided as i32
     // scalar types.
@@ -195,7 +195,7 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
         arrayTy.getElementType() == i4x8Ty ||
         (arrayTy.getElementType() == f32x1Ty &&
          operandPtxType == NVVM::MMATypes::tf32)) {
-      result.push_back(b.create<LLVM::BitcastOp>(i32Ty, toUse));
+      result.push_back(LLVM::BitcastOp::create(b, i32Ty, toUse));
       continue;
     }
 
@@ -208,9 +208,9 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
                          innerArrayTy.getElementType() == f32Ty)) {
       for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
            idx < innerSize; idx++) {
-        result.push_back(b.create<LLVM::ExtractElementOp>(
+        result.push_back(LLVM::ExtractElementOp::create(b,
             toUse,
-            b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(idx))));
+            LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(idx))));
       }
       continue;
     }
@@ -285,7 +285,7 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
-    Value ldMatrixResult = b.create<NVVM::LdMatrixOp>(
+    Value ldMatrixResult = NVVM::LdMatrixOp::create(b,
         ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
@@ -296,13 +296,13 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     // actual vector type (still of width 32b) and repack them into a result
     // struct.
     Type finalResultType = typeConverter->convertType(vectorResultType);
-    Value result = b.create<LLVM::PoisonOp>(finalResultType);
+    Value result = LLVM::PoisonOp::create(b, finalResultType);
     for (int64_t i = 0, e = vectorResultType.getDimSize(0); i < e; i++) {
       Value i32Register =
-          num32BitRegs > 1 ? b.create<LLVM::ExtractValueOp>(ldMatrixResult, i)
+          num32BitRegs > 1 ? LLVM::ExtractValueOp::create(b, ldMatrixResult, i)
                            : ldMatrixResult;
-      Value casted = b.create<LLVM::BitcastOp>(innerVectorType, i32Register);
-      result = b.create<LLVM::InsertValueOp>(result, casted, i);
+      Value casted = LLVM::BitcastOp::create(b, innerVectorType, i32Register);
+      result = LLVM::InsertValueOp::create(b, result, casted, i);
     }
 
     rewriter.replaceOp(op, result);
@@ -375,7 +375,7 @@ struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
     Type desiredRetTy = typeConverter->convertType(op->getResultTypes()[0]);
     Type intrinsicResTy = inferIntrinsicResultType(
         typeConverter->convertType(op->getResultTypes()[0]));
-    Value intrinsicResult = b.create<NVVM::MmaOp>(
+    Value intrinsicResult = NVVM::MmaOp::create(b,
         intrinsicResTy, matA, matB, matC,
         /*shape=*/gemmShape,
         /*b1Op=*/std::nullopt,
@@ -565,7 +565,7 @@ static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
     llvm::append_range(asmVals, args);
   asmVals.push_back(indexData);
 
-  return b.create<LLVM::InlineAsmOp>(
+  return LLVM::InlineAsmOp::create(b,
       /*resultTypes=*/intrinsicResultType,
       /*operands=*/asmVals,
       /*asm_string=*/asmStr,
@@ -631,7 +631,7 @@ struct NVGPUMmaSparseSyncLowering
       return op->emitOpError() << "Expected metadata type to be LLVM "
                                   "VectorType of 2 i16 elements";
     sparseMetadata =
-        b.create<LLVM::BitcastOp>(rewriter.getI32Type(), sparseMetadata);
+        LLVM::BitcastOp::create(b, rewriter.getI32Type(), sparseMetadata);
 
     FailureOr<LLVM::InlineAsmOp> intrinsicResult = emitMmaSparseSyncOpAsm(
         b, *ptxTypeA, *ptxTypeB, *ptxTypeC, *ptxTypeC, overflow, matA, matB,
@@ -682,7 +682,7 @@ struct NVGPUAsyncCopyLowering
     // Intrinsics takes a global pointer so we need an address space cast.
     auto srcPointerGlobalType = LLVM::LLVMPointerType::get(
         op->getContext(), NVVM::NVVMMemorySpace::kGlobalMemorySpace);
-    scrPtr = b.create<LLVM::AddrSpaceCastOp>(srcPointerGlobalType, scrPtr);
+    scrPtr = LLVM::AddrSpaceCastOp::create(b, srcPointerGlobalType, scrPtr);
     int64_t dstElements = adaptor.getDstElements().getZExtValue();
     int64_t sizeInBytes =
         (dstMemrefType.getElementTypeBitWidth() * dstElements) / 8;
@@ -697,13 +697,13 @@ struct NVGPUAsyncCopyLowering
       // The rest of the DstElements in the destination (shared memory) are
       // filled with zeros.
       Value c3I32 =
-          b.create<LLVM::ConstantOp>(b.getI32Type(), b.getI32IntegerAttr(3));
-      Value bitwidth = b.create<LLVM::ConstantOp>(
+          LLVM::ConstantOp::create(b, b.getI32Type(), b.getI32IntegerAttr(3));
+      Value bitwidth = LLVM::ConstantOp::create(b,
           b.getI32Type(),
           b.getI32IntegerAttr(srcMemrefType.getElementTypeBitWidth()));
-      Value srcElementsI32 = b.create<LLVM::TruncOp>(b.getI32Type(), srcBytes);
-      srcBytes = b.create<LLVM::LShrOp>(
-          b.create<LLVM::MulOp>(bitwidth, srcElementsI32), c3I32);
+      Value srcElementsI32 = LLVM::TruncOp::create(b, b.getI32Type(), srcBytes);
+      srcBytes = LLVM::LShrOp::create(b,
+          LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
     }
     // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
     // 16 dst bytes.
@@ -712,13 +712,13 @@ struct NVGPUAsyncCopyLowering
             ? NVVM::LoadCacheModifierKind::CG
             : NVVM::LoadCacheModifierKind::CA;
 
-    b.create<NVVM::CpAsyncOp>(
+    NVVM::CpAsyncOp::create(b,
         dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
         NVVM::LoadCacheModifierKindAttr::get(op->getContext(), cacheModifier),
         srcBytes);
 
     // Drop the result token.
-    Value zero = b.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(b,
         IntegerType::get(op.getContext(), 32), rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
@@ -733,9 +733,9 @@ struct NVGPUAsyncCreateGroupLowering
   LogicalResult
   matchAndRewrite(nvgpu::DeviceAsyncCreateGroupOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    rewriter.create<NVVM::CpAsyncCommitGroupOp>(op.getLoc());
+    NVVM::CpAsyncCommitGroupOp::create(rewriter, op.getLoc());
     // Drop the result token.
-    Value zero = rewriter.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(rewriter,
         op->getLoc(), IntegerType::get(op.getContext(), 32),
         rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
@@ -753,7 +753,7 @@ struct NVGPUAsyncWaitLowering
                   ConversionPatternRewriter &rewriter) const override {
     // If numGroup is not present pick 0 as a conservative correct value.
     int32_t numGroups = adaptor.getNumGroups().value_or(0);
-    rewriter.create<NVVM::CpAsyncWaitGroupOp>(op.getLoc(), numGroups);
+    NVVM::CpAsyncWaitGroupOp::create(rewriter, op.getLoc(), numGroups);
     rewriter.eraseOp(op);
     return success();
   }
@@ -771,7 +771,7 @@ struct NVGPUMBarrierCreateLowering
     SymbolTable symbolTable(moduleOp);
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(&moduleOp.front());
-    auto global = rewriter.create<memref::GlobalOp>(
+    auto global = memref::GlobalOp::create(rewriter,
         funcOp->getLoc(), "__mbarrier",
         /*sym_visibility=*/rewriter.getStringAttr("private"),
         /*type=*/barrierType,
@@ -974,7 +974,7 @@ struct NVGPUMBarrierTryWaitParityLowering
                        adaptor.getMbarId(), rewriter);
     Value ticks = truncToI32(b, adaptor.getTicks());
     Value phase =
-        b.create<LLVM::ZExtOp>(b.getI32Type(), adaptor.getPhaseParity());
+        LLVM::ZExtOp::create(b, b.getI32Type(), adaptor.getPhaseParity());
 
     if (isMbarrierShared(op.getBarriers().getType())) {
       rewriter.replaceOpWithNewOp<NVVM::MBarrierTryWaitParitySharedOp>(
@@ -1063,16 +1063,16 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 
     auto ti64 = b.getIntegerType(64);
     auto makeConst = [&](uint64_t index) -> Value {
-      return b.create<LLVM::ConstantOp>(ti64, b.getI64IntegerAttr(index));
+      return LLVM::ConstantOp::create(b, ti64, b.getI64IntegerAttr(index));
     };
     auto shiftLeft = [&](Value value, unsigned shift) -> Value {
-      return b.create<LLVM::ShlOp>(ti64, value, makeConst(shift));
+      return LLVM::ShlOp::create(b, ti64, value, makeConst(shift));
     };
     auto shiftRight = [&](Value value, unsigned shift) -> Value {
-      return b.create<LLVM::LShrOp>(ti64, value, makeConst(shift));
+      return LLVM::LShrOp::create(b, ti64, value, makeConst(shift));
     };
     auto insertBit = [&](Value desc, Value val, int startBit) {
-      return b.create<LLVM::OrOp>(ti64, desc, shiftLeft(val, startBit));
+      return LLVM::OrOp::create(b, ti64, desc, shiftLeft(val, startBit));
     };
 
     int64_t sizeN = op.getTensorMap().getType().getTensor().getDimSize(0);
@@ -1086,7 +1086,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
     Value baseAddr = getStridedElementPtr(
         rewriter, op->getLoc(), cast<MemRefType>(op.getTensor().getType()),
         adaptor.getTensor(), {});
-    Value basePtr = b.create<LLVM::PtrToIntOp>(ti64, baseAddr);
+    Value basePtr = LLVM::PtrToIntOp::create(b, ti64, baseAddr);
     // Just use 14 bits for base address
     Value basePtr14bit = shiftRight(shiftLeft(basePtr, 46), 50);
 
@@ -1118,7 +1118,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 };
 
 static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
-  return b.create<LLVM::ConstantOp>(b.getIntegerType(64),
+  return LLVM::ConstantOp::create(b, b.getIntegerType(64),
                                     b.getI32IntegerAttr(index));
 }
 
@@ -1182,12 +1182,12 @@ struct NVGPUTmaCreateDescriptorOpLowering
     auto promotedOperands = getTypeConverter()->promoteOperands(
         b.getLoc(), op->getOperands(), adaptor.getOperands(), b);
 
-    Value boxArrayPtr = b.create<LLVM::AllocaOp>(llvmPointerType, llvmInt64Type,
+    Value boxArrayPtr = LLVM::AllocaOp::create(b, llvmPointerType, llvmInt64Type,
                                                  makeI64Const(b, 5));
     for (auto [index, value] : llvm::enumerate(adaptor.getBoxDimensions())) {
-      Value gep = b.create<LLVM::GEPOp>(llvmPointerType, llvmPointerType,
+      Value gep = LLVM::GEPOp::create(b, llvmPointerType, llvmPointerType,
                                         boxArrayPtr, makeI64Const(b, index));
-      b.create<LLVM::StoreOp>(value, gep);
+      LLVM::StoreOp::create(b, value, gep);
     }
 
     nvgpu::TensorMapDescriptorType desc = op.getTensorMap().getType();
@@ -1337,7 +1337,7 @@ struct NVGPUWarpgroupMmaOpLowering
 
     /// Basic function to generate Add
     Value makeAdd(Value lhs, Value rhs) {
-      return b.create<LLVM::AddOp>(lhs.getType(), lhs, rhs);
+      return LLVM::AddOp::create(b, lhs.getType(), lhs, rhs);
     };
 
     /// Moves the descriptor pointer of matrix-A for the next wgmma instruction.
@@ -1430,7 +1430,7 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      return b.create<NVVM::WgmmaMmaAsyncOp>(
+      return NVVM::WgmmaMmaAsyncOp::create(b,
           matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
           itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
           overflow);
@@ -1439,19 +1439,19 @@ struct NVGPUWarpgroupMmaOpLowering
     /// Generates multiple wgmma instructions to complete the given GEMM shape
     Value generateWgmmaGroup() {
       Value wgmmaResult =
-          b.create<LLVM::PoisonOp>(adaptor.getMatrixC().getType());
+          LLVM::PoisonOp::create(b, adaptor.getMatrixC().getType());
 
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = b.create<LLVM::ExtractValueOp>(adaptor.getMatrixC(), i);
+        Value matrixC = LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
             matrixC = generateWgmma(i, j, k, matrixC);
         wgmmaResults.push_back(matrixC);
       }
       for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
-        wgmmaResult = b.create<LLVM::InsertValueOp>(wgmmaResult.getType(),
+        wgmmaResult = LLVM::InsertValueOp::create(b, wgmmaResult.getType(),
                                                     wgmmaResult, matrix, idx);
       }
       return wgmmaResult;
@@ -1486,10 +1486,10 @@ struct NVGPUWarpgroupMmaOpLowering
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
     Value generateWarpgroupMma() {
-      b.create<NVVM::WgmmaFenceAlignedOp>();
+      NVVM::WgmmaFenceAlignedOp::create(b, );
       Value wgmmaResult = generateWgmmaGroup();
-      b.create<NVVM::WgmmaGroupSyncAlignedOp>();
-      b.create<NVVM::WgmmaWaitGroupSyncOp>(op.getWaitGroup());
+      NVVM::WgmmaGroupSyncAlignedOp::create(b, );
+      NVVM::WgmmaWaitGroupSyncOp::create(b, op.getWaitGroup());
       return wgmmaResult;
     }
   };
@@ -1557,7 +1557,7 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     Type i32 = b.getI32Type();
 
     auto makeConst = [&](int32_t index) -> Value {
-      return b.create<LLVM::ConstantOp>(i32, b.getI32IntegerAttr(index));
+      return LLVM::ConstantOp::create(b, i32, b.getI32IntegerAttr(index));
     };
     Value c1 = makeConst(1);
     Value c2 = makeConst(2);
@@ -1567,29 +1567,29 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     Value warpSize = makeConst(kWarpSize);
 
     auto makeMul = [&](Value lhs, Value rhs) -> Value {
-      return b.create<LLVM::MulOp>(lhs.getType(), lhs, rhs);
+      return LLVM::MulOp::create(b, lhs.getType(), lhs, rhs);
     };
     auto makeAdd = [&](Value lhs, Value rhs) -> Value {
-      return b.create<LLVM::AddOp>(lhs.getType(), lhs, rhs);
+      return LLVM::AddOp::create(b, lhs.getType(), lhs, rhs);
     };
 
     auto makeExtractAndStore = [&](int i, Value wgmmaResult, Value x, Value y,
                                    TypedValue<::mlir::MemRefType> memref) {
       Type it = b.getIndexType();
-      Value idx = b.create<arith::IndexCastOp>(it, x);
-      Value idy0 = b.create<arith::IndexCastOp>(it, y);
-      Value idy1 = b.create<arith::IndexCastOp>(it, makeAdd(y, c1));
-      Value d0 = b.create<LLVM::ExtractValueOp>(wgmmaResult, i);
-      Value d1 = b.create<LLVM::ExtractValueOp>(wgmmaResult, i + 1);
-      b.create<memref::StoreOp>(d0, memref, ValueRange{idx, idy0});
-      b.create<memref::StoreOp>(d1, memref, ValueRange{idx, idy1});
+      Value idx = arith::IndexCastOp::create(b, it, x);
+      Value idy0 = arith::IndexCastOp::create(b, it, y);
+      Value idy1 = arith::IndexCastOp::create(b, it, makeAdd(y, c1));
+      Value d0 = LLVM::ExtractValueOp::create(b, wgmmaResult, i);
+      Value d1 = LLVM::ExtractValueOp::create(b, wgmmaResult, i + 1);
+      memref::StoreOp::create(b, d0, memref, ValueRange{idx, idy0});
+      memref::StoreOp::create(b, d1, memref, ValueRange{idx, idy1});
     };
 
-    Value tidx = b.create<NVVM::ThreadIdXOp>(i32);
-    Value laneId = b.create<LLVM::URemOp>(i32, tidx, warpSize);
-    Value warpId = b.create<LLVM::UDivOp>(i32, tidx, warpSize);
-    Value lane4Id = b.create<LLVM::UDivOp>(i32, laneId, c4);
-    Value lane4modId = b.create<LLVM::URemOp>(i32, laneId, c4);
+    Value tidx = NVVM::ThreadIdXOp::create(b, i32);
+    Value laneId = LLVM::URemOp::create(b, i32, tidx, warpSize);
+    Value warpId = LLVM::UDivOp::create(b, i32, tidx, warpSize);
+    Value lane4Id = LLVM::UDivOp::create(b, i32, laneId, c4);
+    Value lane4modId = LLVM::URemOp::create(b, i32, laneId, c4);
 
     Value tj = makeMul(lane4modId, c2);
     Value ti = makeAdd(lane4Id, makeMul(warpId, c16));
@@ -1626,7 +1626,7 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     auto stype = cast<LLVM::LLVMStructType>(matriDValue.getType());
     for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(matrixD);
-      Value innerStructValue = b.create<LLVM::ExtractValueOp>(matriDValue, idx);
+      Value innerStructValue = LLVM::ExtractValueOp::create(b, matriDValue, idx);
       storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
@@ -1648,22 +1648,22 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
     Type elemType = cast<LLVM::LLVMStructType>(packStructType.getBody().front())
                         .getBody()
                         .front();
-    Value zero = b.create<LLVM::ConstantOp>(elemType, b.getZeroAttr(elemType));
-    Value packStruct = b.create<LLVM::PoisonOp>(packStructType);
+    Value zero = LLVM::ConstantOp::create(b, elemType, b.getZeroAttr(elemType));
+    Value packStruct = LLVM::PoisonOp::create(b, packStructType);
     SmallVector<Value> innerStructs;
     // Unpack the structs and set all values to zero
     for (auto [idx, s] : llvm::enumerate(packStructType.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(s);
-      Value structValue = b.create<LLVM::ExtractValueOp>(packStruct, idx);
+      Value structValue = LLVM::ExtractValueOp::create(b, packStruct, idx);
       for (unsigned i = 0; i < structType.getBody().size(); ++i) {
-        structValue = b.create<LLVM::InsertValueOp>(
+        structValue = LLVM::InsertValueOp::create(b,
             structType, structValue, zero, ArrayRef<int64_t>({i}));
       }
       innerStructs.push_back(structValue);
     }
     // Pack the inner structs into a single struct
     for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
-      packStruct = b.create<LLVM::InsertValueOp>(packStruct.getType(),
+      packStruct = LLVM::InsertValueOp::create(b, packStruct.getType(),
                                                  packStruct, matrix, idx);
     }
     rewriter.replaceOp(op, packStruct);
@@ -1681,7 +1681,7 @@ struct NVGPUTmaFenceOpLowering
     ImplicitLocOpBuilder b(op->getLoc(), rewriter);
     auto i32Ty = b.getI32Type();
     Value tensormapSize =
-        b.create<LLVM::ConstantOp>(i32Ty, rewriter.getI32IntegerAttr(128));
+        LLVM::ConstantOp::create(b, i32Ty, rewriter.getI32IntegerAttr(128));
 
     auto memscope =
         NVVM::MemScopeKindAttr::get(ctx, ::mlir::NVVM::MemScopeKind::SYS);
@@ -1716,13 +1716,13 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
     VectorType inTy = op.getIn().getType();
     // apply rcp.approx.ftz.f on each element in vector.
     auto convert1DVec = [&](Type llvm1DVectorTy, Value inVec) {
-      Value ret1DVec = b.create<LLVM::PoisonOp>(llvm1DVectorTy);
+      Value ret1DVec = LLVM::PoisonOp::create(b, llvm1DVectorTy);
       int numElems = llvm::cast<VectorType>(llvm1DVectorTy).getNumElements();
       for (int i = 0; i < numElems; i++) {
-        Value idx = b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(i));
-        Value elem = b.create<LLVM::ExtractElementOp>(inVec, idx);
-        Value dst = b.create<NVVM::RcpApproxFtzF32Op>(f32Ty, elem);
-        ret1DVec = b.create<LLVM::InsertElementOp>(ret1DVec, dst, idx);
+        Value idx = LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(i));
+        Value elem = LLVM::ExtractElementOp::create(b, inVec, idx);
+        Value dst = NVVM::RcpApproxFtzF32Op::create(b, f32Ty, elem);
+        ret1DVec = LLVM::InsertElementOp::create(b, ret1DVec, dst, idx);
       }
       return ret1DVec;
     };
diff --git a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
index 479725aae8afd..a6125fc07dcd5 100644
--- a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
+++ b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
@@ -39,7 +39,7 @@ class ExpandIfCondition : public OpRewritePattern<OpTy> {
 
     IntegerAttr constAttr;
     if (!matchPattern(op.getIfCond(), m_Constant(&constAttr))) {
-      auto ifOp = rewriter.create<scf::IfOp>(op.getLoc(), TypeRange(),
+      auto ifOp = scf::IfOp::create(rewriter, op.getLoc(), TypeRange(),
                                              op.getIfCond(), false);
       rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });
       auto thenBodyBuilder = ifOp.getThenBodyBuilder(rewriter.getListener());
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 77a2708653576..836a193b7fb4a 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -85,7 +85,7 @@ struct OpenMPOpConversion : public ConvertOpToLLVMPattern<T> {
     }
 
     // Create new operation.
-    auto newOp = rewriter.create<T>(op.getLoc(), resTypes, convertedOperands,
+    auto newOp = T::create(rewriter, op.getLoc(), resTypes, convertedOperands,
                                     convertedAttrs);
 
     // Translate regions.
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 7d20109b3db59..75cd35c0d7876 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -196,7 +196,7 @@ Block *PatternLowering::generateMatcher(MatcherNode &node, Region &region,
   // finalize.
   if (isa<ExitNode>(node)) {
     builder.setInsertionPointToEnd(block);
-    builder.create<pdl_interp::FinalizeOp>(matcherFunc.getLoc());
+    pdl_interp::FinalizeOp::create(builder, matcherFunc.getLoc());
     return block;
   }
 
@@ -272,7 +272,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     auto *operationPos = cast<OperationPosition>(pos);
     if (operationPos->isOperandDefiningOp())
       // Standard (downward) traversal which directly follows the defining op.
-      value = builder.create<pdl_interp::GetDefiningOpOp>(
+      value = pdl_interp::GetDefiningOpOp::create(builder,
           loc, builder.getType<pdl::OperationType>(), parentVal);
     else
       // A passthrough operation position.
@@ -287,23 +287,23 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     // requested to use a representative value (e.g., upward traversal).
     if (isa<pdl::RangeType>(parentVal.getType()) &&
         usersPos->useRepresentative())
-      value = builder.create<pdl_interp::ExtractOp>(loc, parentVal, 0);
+      value = pdl_interp::ExtractOp::create(builder, loc, parentVal, 0);
     else
       value = parentVal;
 
     // The second operation retrieves the users.
-    value = builder.create<pdl_interp::GetUsersOp>(loc, value);
+    value = pdl_interp::GetUsersOp::create(builder, loc, value);
     break;
   }
   case Predicates::ForEachPos: {
     assert(!failureBlockStack.empty() && "expected valid failure block");
-    auto foreach = builder.create<pdl_interp::ForEachOp>(
+    auto foreach = pdl_interp::ForEachOp::create(builder,
         loc, parentVal, failureBlockStack.back(), /*initLoop=*/true);
     value = foreach.getLoopVariable();
 
     // Create the continuation block.
     Block *continueBlock = builder.createBlock(&foreach.getRegion());
-    builder.create<pdl_interp::ContinueOp>(loc);
+    pdl_interp::ContinueOp::create(builder, loc);
     failureBlockStack.push_back(continueBlock);
 
     currentBlock = &foreach.getRegion().front();
@@ -311,7 +311,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::OperandPos: {
     auto *operandPos = cast<OperandPosition>(pos);
-    value = builder.create<pdl_interp::GetOperandOp>(
+    value = pdl_interp::GetOperandOp::create(builder,
         loc, builder.getType<pdl::ValueType>(), parentVal,
         operandPos->getOperandNumber());
     break;
@@ -319,28 +319,28 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::OperandGroupPos: {
     auto *operandPos = cast<OperandGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = builder.create<pdl_interp::GetOperandsOp>(
+    value = pdl_interp::GetOperandsOp::create(builder,
         loc, operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, operandPos->getOperandGroupNumber());
     break;
   }
   case Predicates::AttributePos: {
     auto *attrPos = cast<AttributePosition>(pos);
-    value = builder.create<pdl_interp::GetAttributeOp>(
+    value = pdl_interp::GetAttributeOp::create(builder,
         loc, builder.getType<pdl::AttributeType>(), parentVal,
         attrPos->getName().strref());
     break;
   }
   case Predicates::TypePos: {
     if (isa<pdl::AttributeType>(parentVal.getType()))
-      value = builder.create<pdl_interp::GetAttributeTypeOp>(loc, parentVal);
+      value = pdl_interp::GetAttributeTypeOp::create(builder, loc, parentVal);
     else
-      value = builder.create<pdl_interp::GetValueTypeOp>(loc, parentVal);
+      value = pdl_interp::GetValueTypeOp::create(builder, loc, parentVal);
     break;
   }
   case Predicates::ResultPos: {
     auto *resPos = cast<ResultPosition>(pos);
-    value = builder.create<pdl_interp::GetResultOp>(
+    value = pdl_interp::GetResultOp::create(builder,
         loc, builder.getType<pdl::ValueType>(), parentVal,
         resPos->getResultNumber());
     break;
@@ -348,7 +348,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::ResultGroupPos: {
     auto *resPos = cast<ResultGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = builder.create<pdl_interp::GetResultsOp>(
+    value = pdl_interp::GetResultsOp::create(builder,
         loc, resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, resPos->getResultGroupNumber());
     break;
@@ -356,16 +356,16 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::AttributeLiteralPos: {
     auto *attrPos = cast<AttributeLiteralPosition>(pos);
     value =
-        builder.create<pdl_interp::CreateAttributeOp>(loc, attrPos->getValue());
+        pdl_interp::CreateAttributeOp::create(builder, loc, attrPos->getValue());
     break;
   }
   case Predicates::TypeLiteralPos: {
     auto *typePos = cast<TypeLiteralPosition>(pos);
     Attribute rawTypeAttr = typePos->getValue();
     if (TypeAttr typeAttr = dyn_cast<TypeAttr>(rawTypeAttr))
-      value = builder.create<pdl_interp::CreateTypeOp>(loc, typeAttr);
+      value = pdl_interp::CreateTypeOp::create(builder, loc, typeAttr);
     else
-      value = builder.create<pdl_interp::CreateTypesOp>(
+      value = pdl_interp::CreateTypesOp::create(builder,
           loc, cast<ArrayAttr>(rawTypeAttr));
     break;
   }
@@ -413,54 +413,54 @@ void PatternLowering::generate(BoolNode *boolNode, Block *&currentBlock,
   Predicates::Kind kind = question->getKind();
   switch (kind) {
   case Predicates::IsNotNullQuestion:
-    builder.create<pdl_interp::IsNotNullOp>(loc, val, success, failure);
+    pdl_interp::IsNotNullOp::create(builder, loc, val, success, failure);
     break;
   case Predicates::OperationNameQuestion: {
     auto *opNameAnswer = cast<OperationNameAnswer>(answer);
-    builder.create<pdl_interp::CheckOperationNameOp>(
+    pdl_interp::CheckOperationNameOp::create(builder,
         loc, val, opNameAnswer->getValue().getStringRef(), success, failure);
     break;
   }
   case Predicates::TypeQuestion: {
     auto *ans = cast<TypeAnswer>(answer);
     if (isa<pdl::RangeType>(val.getType()))
-      builder.create<pdl_interp::CheckTypesOp>(
+      pdl_interp::CheckTypesOp::create(builder,
           loc, val, llvm::cast<ArrayAttr>(ans->getValue()), success, failure);
     else
-      builder.create<pdl_interp::CheckTypeOp>(
+      pdl_interp::CheckTypeOp::create(builder,
           loc, val, llvm::cast<TypeAttr>(ans->getValue()), success, failure);
     break;
   }
   case Predicates::AttributeQuestion: {
     auto *ans = cast<AttributeAnswer>(answer);
-    builder.create<pdl_interp::CheckAttributeOp>(loc, val, ans->getValue(),
+    pdl_interp::CheckAttributeOp::create(builder, loc, val, ans->getValue(),
                                                  success, failure);
     break;
   }
   case Predicates::OperandCountAtLeastQuestion:
   case Predicates::OperandCountQuestion:
-    builder.create<pdl_interp::CheckOperandCountOp>(
+    pdl_interp::CheckOperandCountOp::create(builder,
         loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::OperandCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::ResultCountAtLeastQuestion:
   case Predicates::ResultCountQuestion:
-    builder.create<pdl_interp::CheckResultCountOp>(
+    pdl_interp::CheckResultCountOp::create(builder,
         loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::ResultCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::EqualToQuestion: {
     bool trueAnswer = isa<TrueAnswer>(answer);
-    builder.create<pdl_interp::AreEqualOp>(loc, val, args.front(),
+    pdl_interp::AreEqualOp::create(builder, loc, val, args.front(),
                                            trueAnswer ? success : failure,
                                            trueAnswer ? failure : success);
     break;
   }
   case Predicates::ConstraintQuestion: {
     auto *cstQuestion = cast<ConstraintQuestion>(question);
-    auto applyConstraintOp = builder.create<pdl_interp::ApplyConstraintOp>(
+    auto applyConstraintOp = pdl_interp::ApplyConstraintOp::create(builder,
         loc, cstQuestion->getResultTypes(), cstQuestion->getName(), args,
         cstQuestion->getIsNegated(), success, failure);
 
@@ -487,7 +487,7 @@ static void createSwitchOp(Value val, Block *defaultDest, OpBuilder &builder,
     blocks.push_back(it.second);
     values.push_back(cast<PredT>(it.first)->getValue());
   }
-  builder.create<OpT>(val.getLoc(), val, values, defaultDest, blocks);
+  OpT::create(builder, val.getLoc(), val, values, defaultDest, blocks);
 }
 
 void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
@@ -536,11 +536,11 @@ void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
       unsigned ans = cast<UnsignedAnswer>(child.first)->getValue();
       switch (kind) {
       case Predicates::OperandCountAtLeastQuestion:
-        builder.create<pdl_interp::CheckOperandCountOp>(
+        pdl_interp::CheckOperandCountOp::create(builder,
             loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
         break;
       case Predicates::ResultCountAtLeastQuestion:
-        builder.create<pdl_interp::CheckResultCountOp>(
+        pdl_interp::CheckResultCountOp::create(builder,
             loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
         break;
       default:
@@ -619,7 +619,7 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
       rootKindAttr = builder.getStringAttr(*rootKind);
 
   builder.setInsertionPointToEnd(currentBlock);
-  auto matchOp = builder.create<pdl_interp::RecordMatchOp>(
+  auto matchOp = pdl_interp::RecordMatchOp::create(builder,
       pattern.getLoc(), mappedMatchValues, locOps.getArrayRef(),
       rewriterFuncRef, rootKindAttr, generatedOpsAttr, pattern.getBenefitAttr(),
       failureBlockStack.back());
@@ -632,7 +632,7 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
 SymbolRefAttr PatternLowering::generateRewriter(
     pdl::PatternOp pattern, SmallVectorImpl<Position *> &usedMatchValues) {
   builder.setInsertionPointToEnd(rewriterModule.getBody());
-  auto rewriterFunc = builder.create<pdl_interp::FuncOp>(
+  auto rewriterFunc = pdl_interp::FuncOp::create(builder,
       pattern.getLoc(), "pdl_generated_rewriter",
       builder.getFunctionType({}, {}));
   rewriterSymbolTable.insert(rewriterFunc);
@@ -651,17 +651,17 @@ SymbolRefAttr PatternLowering::generateRewriter(
     Operation *oldOp = oldValue.getDefiningOp();
     if (pdl::AttributeOp attrOp = dyn_cast<pdl::AttributeOp>(oldOp)) {
       if (Attribute value = attrOp.getValueAttr()) {
-        return newValue = builder.create<pdl_interp::CreateAttributeOp>(
+        return newValue = pdl_interp::CreateAttributeOp::create(builder,
                    attrOp.getLoc(), value);
       }
     } else if (pdl::TypeOp typeOp = dyn_cast<pdl::TypeOp>(oldOp)) {
       if (TypeAttr type = typeOp.getConstantTypeAttr()) {
-        return newValue = builder.create<pdl_interp::CreateTypeOp>(
+        return newValue = pdl_interp::CreateTypeOp::create(builder,
                    typeOp.getLoc(), type);
       }
     } else if (pdl::TypesOp typeOp = dyn_cast<pdl::TypesOp>(oldOp)) {
       if (ArrayAttr type = typeOp.getConstantTypesAttr()) {
-        return newValue = builder.create<pdl_interp::CreateTypesOp>(
+        return newValue = pdl_interp::CreateTypesOp::create(builder,
                    typeOp.getLoc(), typeOp.getType(), type);
       }
     }
@@ -684,7 +684,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
     auto mappedArgs =
         llvm::map_range(rewriter.getExternalArgs(), mapRewriteValue);
     args.append(mappedArgs.begin(), mappedArgs.end());
-    builder.create<pdl_interp::ApplyRewriteOp>(
+    pdl_interp::ApplyRewriteOp::create(builder,
         rewriter.getLoc(), /*resultTypes=*/TypeRange(), rewriteName, args);
   } else {
     // Otherwise this is a dag rewriter defined using PDL operations.
@@ -703,7 +703,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
       llvm::to_vector<8>(rewriterFunc.front().getArgumentTypes()),
       /*results=*/{}));
 
-  builder.create<pdl_interp::FinalizeOp>(rewriter.getLoc());
+  pdl_interp::FinalizeOp::create(builder, rewriter.getLoc());
   return SymbolRefAttr::get(
       builder.getContext(),
       pdl_interp::PDLInterpDialect::getRewriterModuleName(),
@@ -716,7 +716,7 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 2> arguments;
   for (Value argument : rewriteOp.getArgs())
     arguments.push_back(mapRewriteValue(argument));
-  auto interpOp = builder.create<pdl_interp::ApplyRewriteOp>(
+  auto interpOp = pdl_interp::ApplyRewriteOp::create(builder,
       rewriteOp.getLoc(), rewriteOp.getResultTypes(), rewriteOp.getNameAttr(),
       arguments);
   for (auto it : llvm::zip(rewriteOp.getResults(), interpOp.getResults()))
@@ -726,7 +726,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::AttributeOp attrOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  Value newAttr = builder.create<pdl_interp::CreateAttributeOp>(
+  Value newAttr = pdl_interp::CreateAttributeOp::create(builder,
       attrOp.getLoc(), attrOp.getValueAttr());
   rewriteValues[attrOp] = newAttr;
 }
@@ -734,7 +734,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::EraseOp eraseOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  builder.create<pdl_interp::EraseOp>(eraseOp.getLoc(),
+  pdl_interp::EraseOp::create(builder, eraseOp.getLoc(),
                                       mapRewriteValue(eraseOp.getOpValue()));
 }
 
@@ -756,7 +756,7 @@ void PatternLowering::generateRewriter(
 
   // Create the new operation.
   Location loc = operationOp.getLoc();
-  Value createdOp = builder.create<pdl_interp::CreateOperationOp>(
+  Value createdOp = pdl_interp::CreateOperationOp::create(builder,
       loc, *operationOp.getOpName(), types, hasInferredResultTypes, operands,
       attributes, operationOp.getAttributeValueNames());
   rewriteValues[operationOp.getOp()] = createdOp;
@@ -768,8 +768,8 @@ void PatternLowering::generateRewriter(
   if (resultTys.size() == 1 && isa<pdl::RangeType>(resultTys[0].getType())) {
     Value &type = rewriteValues[resultTys[0]];
     if (!type) {
-      auto results = builder.create<pdl_interp::GetResultsOp>(loc, createdOp);
-      type = builder.create<pdl_interp::GetValueTypeOp>(loc, results);
+      auto results = pdl_interp::GetResultsOp::create(builder, loc, createdOp);
+      type = pdl_interp::GetValueTypeOp::create(builder, loc, results);
     }
     return;
   }
@@ -789,12 +789,12 @@ void PatternLowering::generateRewriter(
     // groups because the exact index of the result is not statically known.
     Value resultVal;
     if (seenVariableLength)
-      resultVal = builder.create<pdl_interp::GetResultsOp>(
+      resultVal = pdl_interp::GetResultsOp::create(builder,
           loc, isVariadic ? valueRangeTy : valueTy, createdOp, it.index());
     else
-      resultVal = builder.create<pdl_interp::GetResultOp>(
+      resultVal = pdl_interp::GetResultOp::create(builder,
           loc, valueTy, createdOp, it.index());
-    type = builder.create<pdl_interp::GetValueTypeOp>(loc, resultVal);
+    type = pdl_interp::GetValueTypeOp::create(builder, loc, resultVal);
   }
 }
 
@@ -804,7 +804,7 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 4> replOperands;
   for (Value operand : rangeOp.getArguments())
     replOperands.push_back(mapRewriteValue(operand));
-  rewriteValues[rangeOp] = builder.create<pdl_interp::CreateRangeOp>(
+  rewriteValues[rangeOp] = pdl_interp::CreateRangeOp::create(builder,
       rangeOp.getLoc(), rangeOp.getType(), replOperands);
 }
 
@@ -820,7 +820,7 @@ void PatternLowering::generateRewriter(
     // Don't use replace if we know the replaced operation has no results.
     auto opOp = replaceOp.getOpValue().getDefiningOp<pdl::OperationOp>();
     if (!opOp || !opOp.getTypeValues().empty()) {
-      replOperands.push_back(builder.create<pdl_interp::GetResultsOp>(
+      replOperands.push_back(pdl_interp::GetResultsOp::create(builder,
           replOp.getLoc(), mapRewriteValue(replOp)));
     }
   } else {
@@ -830,12 +830,12 @@ void PatternLowering::generateRewriter(
 
   // If there are no replacement values, just create an erase instead.
   if (replOperands.empty()) {
-    builder.create<pdl_interp::EraseOp>(
+    pdl_interp::EraseOp::create(builder,
         replaceOp.getLoc(), mapRewriteValue(replaceOp.getOpValue()));
     return;
   }
 
-  builder.create<pdl_interp::ReplaceOp>(replaceOp.getLoc(),
+  pdl_interp::ReplaceOp::create(builder, replaceOp.getLoc(),
                                         mapRewriteValue(replaceOp.getOpValue()),
                                         replOperands);
 }
@@ -843,7 +843,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::ResultOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultOp>(
+  rewriteValues[resultOp] = pdl_interp::GetResultOp::create(builder,
       resultOp.getLoc(), builder.getType<pdl::ValueType>(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
@@ -851,7 +851,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::ResultsOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultsOp>(
+  rewriteValues[resultOp] = pdl_interp::GetResultsOp::create(builder,
       resultOp.getLoc(), resultOp.getType(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
@@ -863,7 +863,7 @@ void PatternLowering::generateRewriter(
   // type.
   if (TypeAttr typeAttr = typeOp.getConstantTypeAttr()) {
     rewriteValues[typeOp] =
-        builder.create<pdl_interp::CreateTypeOp>(typeOp.getLoc(), typeAttr);
+        pdl_interp::CreateTypeOp::create(builder, typeOp.getLoc(), typeAttr);
   }
 }
 
@@ -873,7 +873,7 @@ void PatternLowering::generateRewriter(
   // If the type isn't constant, the users (e.g. OperationOp) will resolve this
   // type.
   if (ArrayAttr typeAttr = typeOp.getConstantTypesAttr()) {
-    rewriteValues[typeOp] = builder.create<pdl_interp::CreateTypesOp>(
+    rewriteValues[typeOp] = pdl_interp::CreateTypesOp::create(builder,
         typeOp.getLoc(), typeOp.getType(), typeAttr);
   }
 }
@@ -939,9 +939,9 @@ void PatternLowering::generateOperationResultTypeRewriter(
         !replacedOp->isBeforeInBlock(op))
       continue;
 
-    Value replacedOpResults = builder.create<pdl_interp::GetResultsOp>(
+    Value replacedOpResults = pdl_interp::GetResultsOp::create(builder,
         replacedOp->getLoc(), mapRewriteValue(replOpVal));
-    types.push_back(builder.create<pdl_interp::GetValueTypeOp>(
+    types.push_back(pdl_interp::GetValueTypeOp::create(builder,
         replacedOp->getLoc(), replacedOpResults));
     return;
   }
@@ -985,7 +985,7 @@ void PDLToPDLInterpPass::runOnOperation() {
   // Create the main matcher function This function contains all of the match
   // related functionality from patterns in the module.
   OpBuilder builder = OpBuilder::atBlockBegin(module.getBody());
-  auto matcherFunc = builder.create<pdl_interp::FuncOp>(
+  auto matcherFunc = pdl_interp::FuncOp::create(builder,
       module.getLoc(), pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
       builder.getFunctionType(builder.getType<pdl::OperationType>(),
                               /*results=*/{}),
@@ -993,7 +993,7 @@ void PDLToPDLInterpPass::runOnOperation() {
 
   // Create a nested module to hold the functions invoked for rewriting the IR
   // after a successful match.
-  ModuleOp rewriterModule = builder.create<ModuleOp>(
+  ModuleOp rewriterModule = ModuleOp::create(builder,
       module.getLoc(), pdl_interp::PDLInterpDialect::getRewriterModuleName());
 
   // Generate the code for the patterns within the module.
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index ac6dcf2513e99..ffef20ce67818 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -340,14 +340,14 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   Operation *terminator = lastBodyBlock->getTerminator();
   rewriter.setInsertionPointToEnd(lastBodyBlock);
   auto step = forOp.getStep();
-  auto stepped = rewriter.create<arith::AddIOp>(loc, iv, step).getResult();
+  auto stepped = arith::AddIOp::create(rewriter, loc, iv, step).getResult();
   if (!stepped)
     return failure();
 
   SmallVector<Value, 8> loopCarried;
   loopCarried.push_back(stepped);
   loopCarried.append(terminator->operand_begin(), terminator->operand_end());
-  rewriter.create<cf::BranchOp>(loc, conditionBlock, loopCarried);
+  cf::BranchOp::create(rewriter, loc, conditionBlock, loopCarried);
   rewriter.eraseOp(terminator);
 
   // Compute loop bounds before branching to the condition.
@@ -362,14 +362,14 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   SmallVector<Value, 8> destOperands;
   destOperands.push_back(lowerBound);
   llvm::append_range(destOperands, forOp.getInitArgs());
-  rewriter.create<cf::BranchOp>(loc, conditionBlock, destOperands);
+  cf::BranchOp::create(rewriter, loc, conditionBlock, destOperands);
 
   // With the body block done, we can fill in the condition block.
   rewriter.setInsertionPointToEnd(conditionBlock);
-  auto comparison = rewriter.create<arith::CmpIOp>(
+  auto comparison = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::slt, iv, upperBound);
 
-  auto condBranchOp = rewriter.create<cf::CondBranchOp>(
+  auto condBranchOp = cf::CondBranchOp::create(rewriter,
       loc, comparison, firstBodyBlock, ArrayRef<Value>(), endBlock,
       ArrayRef<Value>());
 
@@ -404,7 +404,7 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
     continueBlock =
         rewriter.createBlock(remainingOpsBlock, ifOp.getResultTypes(),
                              SmallVector<Location>(ifOp.getNumResults(), loc));
-    rewriter.create<cf::BranchOp>(loc, remainingOpsBlock);
+    cf::BranchOp::create(rewriter, loc, remainingOpsBlock);
   }
 
   // Move blocks from the "then" region to the region containing 'scf.if',
@@ -414,7 +414,7 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
   Operation *thenTerminator = thenRegion.back().getTerminator();
   ValueRange thenTerminatorOperands = thenTerminator->getOperands();
   rewriter.setInsertionPointToEnd(&thenRegion.back());
-  rewriter.create<cf::BranchOp>(loc, continueBlock, thenTerminatorOperands);
+  cf::BranchOp::create(rewriter, loc, continueBlock, thenTerminatorOperands);
   rewriter.eraseOp(thenTerminator);
   rewriter.inlineRegionBefore(thenRegion, continueBlock);
 
@@ -428,13 +428,13 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
     Operation *elseTerminator = elseRegion.back().getTerminator();
     ValueRange elseTerminatorOperands = elseTerminator->getOperands();
     rewriter.setInsertionPointToEnd(&elseRegion.back());
-    rewriter.create<cf::BranchOp>(loc, continueBlock, elseTerminatorOperands);
+    cf::BranchOp::create(rewriter, loc, continueBlock, elseTerminatorOperands);
     rewriter.eraseOp(elseTerminator);
     rewriter.inlineRegionBefore(elseRegion, continueBlock);
   }
 
   rewriter.setInsertionPointToEnd(condBlock);
-  rewriter.create<cf::CondBranchOp>(loc, ifOp.getCondition(), thenBlock,
+  cf::CondBranchOp::create(rewriter, loc, ifOp.getCondition(), thenBlock,
                                     /*trueArgs=*/ArrayRef<Value>(), elseBlock,
                                     /*falseArgs=*/ArrayRef<Value>());
 
@@ -454,13 +454,13 @@ ExecuteRegionLowering::matchAndRewrite(ExecuteRegionOp op,
 
   auto &region = op.getRegion();
   rewriter.setInsertionPointToEnd(condBlock);
-  rewriter.create<cf::BranchOp>(loc, &region.front());
+  cf::BranchOp::create(rewriter, loc, &region.front());
 
   for (Block &block : region) {
     if (auto terminator = dyn_cast<scf::YieldOp>(block.getTerminator())) {
       ValueRange terminatorOperands = terminator->getOperands();
       rewriter.setInsertionPointToEnd(&block);
-      rewriter.create<cf::BranchOp>(loc, remainingOpsBlock, terminatorOperands);
+      cf::BranchOp::create(rewriter, loc, remainingOpsBlock, terminatorOperands);
       rewriter.eraseOp(terminator);
     }
   }
@@ -498,7 +498,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
   for (auto [iv, lower, upper, step] :
        llvm::zip(parallelOp.getInductionVars(), parallelOp.getLowerBound(),
                  parallelOp.getUpperBound(), parallelOp.getStep())) {
-    ForOp forOp = rewriter.create<ForOp>(loc, lower, upper, step, iterArgs);
+    ForOp forOp = ForOp::create(rewriter, loc, lower, upper, step, iterArgs);
     ivs.push_back(forOp.getInductionVar());
     auto iterRange = forOp.getRegionIterArgs();
     iterArgs.assign(iterRange.begin(), iterRange.end());
@@ -512,7 +512,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
       // A loop is constructed with an empty "yield" terminator if there are
       // no results.
       rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
-      rewriter.create<scf::YieldOp>(loc, forOp.getResults());
+      scf::YieldOp::create(rewriter, loc, forOp.getResults());
     }
 
     rewriter.setInsertionPointToStart(forOp.getBody());
@@ -544,7 +544,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
   // has been already created in loop construction).
   if (!yieldOperands.empty()) {
     rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
-    rewriter.create<scf::YieldOp>(loc, yieldOperands);
+    scf::YieldOp::create(rewriter, loc, yieldOperands);
   }
 
   rewriter.replaceOp(parallelOp, loopResults);
@@ -570,7 +570,7 @@ LogicalResult WhileLowering::matchAndRewrite(WhileOp whileOp,
 
   // Branch to the "before" region.
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(loc, before, whileOp.getInits());
+  cf::BranchOp::create(rewriter, loc, before, whileOp.getInits());
 
   // Replace terminators with branches. Assuming bodies are SESE, which holds
   // given only the patterns from this file, we only need to look at the last
@@ -620,7 +620,7 @@ DoWhileLowering::matchAndRewrite(WhileOp whileOp,
 
   // Branch to the "before" region.
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(whileOp.getLoc(), before, whileOp.getInits());
+  cf::BranchOp::create(rewriter, whileOp.getLoc(), before, whileOp.getInits());
 
   // Loop around the "before" region based on condition.
   rewriter.setInsertionPointToEnd(before);
@@ -688,10 +688,10 @@ IndexSwitchLowering::matchAndRewrite(IndexSwitchOp op,
   SmallVector<ValueRange> caseOperands(caseSuccessors.size(), {});
 
   // Cast switch index to integer case value.
-  Value caseValue = rewriter.create<arith::IndexCastOp>(
+  Value caseValue = arith::IndexCastOp::create(rewriter,
       op.getLoc(), rewriter.getI32Type(), op.getArg());
 
-  rewriter.create<cf::SwitchOp>(
+  cf::SwitchOp::create(rewriter,
       op.getLoc(), caseValue, *defaultBlock, ValueRange(),
       rewriter.getDenseI32ArrayAttr(caseValues), caseSuccessors, caseOperands);
   rewriter.replaceOp(op, continueBlock->getArguments());
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index dcb48529a74e6..37134fe9d992a 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -91,7 +91,7 @@ createVariablesForResults(T op, const TypeConverter *typeConverter,
     Type varType = emitc::LValueType::get(resultType);
     emitc::OpaqueAttr noInit = emitc::OpaqueAttr::get(context, "");
     emitc::VariableOp var =
-        rewriter.create<emitc::VariableOp>(loc, varType, noInit);
+        emitc::VariableOp::create(rewriter, loc, varType, noInit);
     resultVariables.push_back(var);
   }
 
@@ -103,14 +103,14 @@ createVariablesForResults(T op, const TypeConverter *typeConverter,
 static void assignValues(ValueRange values, ValueRange variables,
                          ConversionPatternRewriter &rewriter, Location loc) {
   for (auto [value, var] : llvm::zip(values, variables))
-    rewriter.create<emitc::AssignOp>(loc, var, value);
+    emitc::AssignOp::create(rewriter, loc, var, value);
 }
 
 SmallVector<Value> loadValues(const SmallVector<Value> &variables,
                               PatternRewriter &rewriter, Location loc) {
   return llvm::map_to_vector<>(variables, [&](Value var) {
     Type type = cast<emitc::LValueType>(var.getType()).getValueType();
-    return rewriter.create<emitc::LoadOp>(loc, type, var).getResult();
+    return emitc::LoadOp::create(rewriter, loc, type, var).getResult();
   });
 }
 
@@ -129,7 +129,7 @@ static LogicalResult lowerYield(Operation *op, ValueRange resultVariables,
 
   assignValues(yieldOperands, resultVariables, rewriter, loc);
 
-  rewriter.create<emitc::YieldOp>(loc);
+  emitc::YieldOp::create(rewriter, loc);
   rewriter.eraseOp(yield);
 
   return success();
@@ -164,7 +164,7 @@ ForLowering::matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
 
   assignValues(adaptor.getInitArgs(), resultVariables, rewriter, loc);
 
-  emitc::ForOp loweredFor = rewriter.create<emitc::ForOp>(
+  emitc::ForOp loweredFor = emitc::ForOp::create(rewriter,
       loc, adaptor.getLowerBound(), adaptor.getUpperBound(), adaptor.getStep());
 
   Block *loweredBody = loweredFor.getBody();
@@ -257,7 +257,7 @@ IfLowering::matchAndRewrite(IfOp ifOp, OpAdaptor adaptor,
   bool hasElseBlock = !elseRegion.empty();
 
   auto loweredIf =
-      rewriter.create<emitc::IfOp>(loc, adaptor.getCondition(), false, false);
+      emitc::IfOp::create(rewriter, loc, adaptor.getCondition(), false, false);
 
   Region &loweredThenRegion = loweredIf.getThenRegion();
   auto result = lowerRegion(thenRegion, loweredThenRegion);
@@ -304,7 +304,7 @@ LogicalResult IndexSwitchOpLowering::matchAndRewrite(
                                        "create variables for results failed");
   }
 
-  auto loweredSwitch = rewriter.create<emitc::SwitchOp>(
+  auto loweredSwitch = emitc::SwitchOp::create(rewriter,
       loc, adaptor.getArg(), adaptor.getCases(), indexSwitchOp.getNumCases());
 
   // Lowering all case regions.
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index 844e66e927c4d..bfdb5a688b286 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -84,7 +84,7 @@ static Operation::operand_range getUpperBoundOperands(AffineForOp forOp) {
 // Get a Value that corresponds to the loop step.  If the step is an attribute,
 // materialize a corresponding constant using builder.
 static Value getOrCreateStep(AffineForOp forOp, OpBuilder &builder) {
-  return builder.create<arith::ConstantIndexOp>(forOp.getLoc(),
+  return arith::ConstantIndexOp::create(builder, forOp.getLoc(),
                                                 forOp.getStepAsInt());
 }
 
@@ -190,12 +190,12 @@ AffineLoopToGpuConverter::collectBounds(AffineForOp forOp, unsigned numLoops) {
       return std::nullopt;
     }
 
-    Value range = builder.create<arith::SubIOp>(currentLoop.getLoc(),
+    Value range = arith::SubIOp::create(builder, currentLoop.getLoc(),
                                                 upperBound, lowerBound);
     Value step = getOrCreateStep(currentLoop, builder);
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
       range =
-          builder.create<arith::CeilDivSIOp>(currentLoop.getLoc(), range, step);
+          arith::CeilDivSIOp::create(builder, currentLoop.getLoc(), range, step);
     dims.push_back(range);
 
     lbs.push_back(lowerBound);
@@ -221,7 +221,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
   // no loop mapped to a specific dimension, use constant "1" as its size.
   Value constOne =
       (numBlockDims < 3 || numThreadDims < 3)
-          ? builder.create<arith::ConstantIndexOp>(rootForOp.getLoc(), 1)
+          ? arith::ConstantIndexOp::create(builder, rootForOp.getLoc(), 1)
           : nullptr;
   Value gridSizeX = numBlockDims > 0 ? dims[0] : constOne;
   Value gridSizeY = numBlockDims > 1 ? dims[1] : constOne;
@@ -232,7 +232,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
 
   // Create a launch op and move the body region of the innermost loop to the
   // launch op.
-  auto launchOp = builder.create<gpu::LaunchOp>(
+  auto launchOp = gpu::LaunchOp::create(builder,
       rootForOp.getLoc(), gridSizeX, gridSizeY, gridSizeZ, blockSizeX,
       blockSizeY, blockSizeZ);
 
@@ -244,7 +244,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
   Location terminatorLoc = terminator.getLoc();
   terminator.erase();
   builder.setInsertionPointToEnd(innermostForOp.getBody());
-  builder.create<gpu::TerminatorOp>(terminatorLoc, TypeRange());
+  gpu::TerminatorOp::create(builder, terminatorLoc, TypeRange());
   launchOp.getBody().front().getOperations().splice(
       launchOp.getBody().front().begin(),
       innermostForOp.getBody()->getOperations());
@@ -263,10 +263,10 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
             : getDim3Value(launchOp.getThreadIds(), en.index() - numBlockDims);
     Value step = steps[en.index()];
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
-      id = builder.create<arith::MulIOp>(rootForOp.getLoc(), step, id);
+      id = arith::MulIOp::create(builder, rootForOp.getLoc(), step, id);
 
     Value ivReplacement =
-        builder.create<arith::AddIOp>(rootForOp.getLoc(), *lbArgumentIt, id);
+        arith::AddIOp::create(builder, rootForOp.getLoc(), *lbArgumentIt, id);
     en.value().replaceAllUsesWith(ivReplacement);
     std::advance(lbArgumentIt, 1);
     std::advance(stepArgumentIt, 1);
@@ -319,7 +319,7 @@ static Value deriveStaticUpperBound(Value upperBound,
   if (auto minOp = upperBound.getDefiningOp<AffineMinOp>()) {
     for (const AffineExpr &result : minOp.getMap().getResults()) {
       if (auto constExpr = dyn_cast<AffineConstantExpr>(result)) {
-        return rewriter.create<arith::ConstantIndexOp>(minOp.getLoc(),
+        return arith::ConstantIndexOp::create(rewriter, minOp.getLoc(),
                                                        constExpr.getValue());
       }
     }
@@ -344,7 +344,7 @@ static Value deriveStaticUpperBound(Value upperBound,
         if ((lhs.value() < 0) != (rhs.value() < 0))
           return {};
 
-        return rewriter.create<arith::ConstantIndexOp>(
+        return arith::ConstantIndexOp::create(rewriter,
             multiplyOp.getLoc(), lhs.value() * rhs.value());
       }
   }
@@ -422,7 +422,7 @@ static LogicalResult processParallelLoop(
     if (launchIndependent(val))
       return val;
     if (auto constOp = val.getDefiningOp<arith::ConstantOp>())
-      return rewriter.create<arith::ConstantOp>(constOp.getLoc(),
+      return arith::ConstantOp::create(rewriter, constOp.getLoc(),
                                                 constOp.getValue());
     return {};
   };
@@ -453,7 +453,7 @@ static LogicalResult processParallelLoop(
           1, 2,
           rewriter.getAffineDimExpr(0) * rewriter.getAffineSymbolExpr(0) +
               rewriter.getAffineSymbolExpr(1));
-      newIndex = rewriter.create<AffineApplyOp>(
+      newIndex = AffineApplyOp::create(rewriter,
           loc, annotation.getMap().compose(lowerAndStep),
           ValueRange{operand, ensureLaunchIndependent(step),
                      ensureLaunchIndependent(lowerBound)});
@@ -498,7 +498,7 @@ static LogicalResult processParallelLoop(
               1, 2,
               ((rewriter.getAffineDimExpr(0) - rewriter.getAffineSymbolExpr(0))
                    .ceilDiv(rewriter.getAffineSymbolExpr(1))));
-          Value launchBound = rewriter.create<AffineApplyOp>(
+          Value launchBound = AffineApplyOp::create(rewriter,
               loc, annotation.getBound().compose(stepMap),
               ValueRange{
                   ensureLaunchIndependent(
@@ -517,10 +517,10 @@ static LogicalResult processParallelLoop(
         if (!boundIsPrecise) {
           // We are using an approximation, create a surrounding conditional.
           Value originalBound = std::get<3>(config);
-          arith::CmpIOp pred = rewriter.create<arith::CmpIOp>(
+          arith::CmpIOp pred = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::slt, newIndex,
               cloningMap.lookupOrDefault(originalBound));
-          scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, pred, false);
+          scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, pred, false);
           rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
           // Put a sentinel into the worklist so we know when to pop out of the
           // if body again. We use the launchOp here, as that cannot be part of
@@ -530,7 +530,7 @@ static LogicalResult processParallelLoop(
       }
     } else {
       // Create a sequential for loop.
-      auto loopOp = rewriter.create<scf::ForOp>(
+      auto loopOp = scf::ForOp::create(rewriter,
           loc, cloningMap.lookupOrDefault(lowerBound),
           cloningMap.lookupOrDefault(upperBound),
           cloningMap.lookupOrDefault(step));
@@ -608,12 +608,12 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
   // sizes. Those will be refined later as we discover them from mappings.
   Location loc = parallelOp.getLoc();
   Value constantOne =
-      rewriter.create<arith::ConstantIndexOp>(parallelOp.getLoc(), 1);
-  gpu::LaunchOp launchOp = rewriter.create<gpu::LaunchOp>(
+      arith::ConstantIndexOp::create(rewriter, parallelOp.getLoc(), 1);
+  gpu::LaunchOp launchOp = gpu::LaunchOp::create(rewriter,
       parallelOp.getLoc(), constantOne, constantOne, constantOne, constantOne,
       constantOne, constantOne);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
-  rewriter.create<gpu::TerminatorOp>(loc);
+  gpu::TerminatorOp::create(rewriter, loc);
   rewriter.setInsertionPointToStart(&launchOp.getBody().front());
 
   IRMapping cloningMap;
@@ -667,7 +667,7 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
       if (externalValues.size())
         return failure();
       // Replace by gpu.all_reduce.
-      auto gpuRedOp = rewriter.create<gpu::AllReduceOp>(loc, newValue);
+      auto gpuRedOp = gpu::AllReduceOp::create(rewriter, loc, newValue);
       cloningMap.map(parentLoop->getResult(0), gpuRedOp.getResult());
       // Copy region.
       rewriter.inlineRegionBefore(reduceOp.getRegion(0), gpuRedOp.getRegion(),
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 584ac2f11b670..7b4c37d44824d 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -187,7 +187,7 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
            scf::ReduceOp reduce, int64_t reductionIndex, Attribute initValue) {
   OpBuilder::InsertionGuard guard(builder);
   Type type = reduce.getOperands()[reductionIndex].getType();
-  auto decl = builder.create<omp::DeclareReductionOp>(reduce.getLoc(),
+  auto decl = omp::DeclareReductionOp::create(builder, reduce.getLoc(),
                                                       "__scf_reduction", type);
   symbolTable.insert(decl);
 
@@ -196,8 +196,8 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
                       {reduce.getOperands()[reductionIndex].getLoc()});
   builder.setInsertionPointToEnd(&decl.getInitializerRegion().back());
   Value init =
-      builder.create<LLVM::ConstantOp>(reduce.getLoc(), type, initValue);
-  builder.create<omp::YieldOp>(reduce.getLoc(), init);
+      LLVM::ConstantOp::create(builder, reduce.getLoc(), type, initValue);
+  omp::YieldOp::create(builder, reduce.getLoc(), init);
 
   Operation *terminator =
       &reduce.getReductions()[reductionIndex].front().back();
@@ -227,12 +227,12 @@ static omp::DeclareReductionOp addAtomicRMW(OpBuilder &builder,
                       {reduceOperandLoc, reduceOperandLoc});
   Block *atomicBlock = &decl.getAtomicReductionRegion().back();
   builder.setInsertionPointToEnd(atomicBlock);
-  Value loaded = builder.create<LLVM::LoadOp>(reduce.getLoc(), decl.getType(),
+  Value loaded = LLVM::LoadOp::create(builder, reduce.getLoc(), decl.getType(),
                                               atomicBlock->getArgument(1));
-  builder.create<LLVM::AtomicRMWOp>(reduce.getLoc(), atomicKind,
+  LLVM::AtomicRMWOp::create(builder, reduce.getLoc(), atomicKind,
                                     atomicBlock->getArgument(0), loaded,
                                     LLVM::AtomicOrdering::monotonic);
-  builder.create<omp::YieldOp>(reduce.getLoc(), ArrayRef<Value>());
+  omp::YieldOp::create(builder, reduce.getLoc(), ArrayRef<Value>());
   return decl;
 }
 
@@ -380,7 +380,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     // Allocate reduction variables. Make sure the we don't overflow the stack
     // with local `alloca`s by saving and restoring the stack pointer.
     Location loc = parallelOp.getLoc();
-    Value one = rewriter.create<LLVM::ConstantOp>(
+    Value one = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerType(64), rewriter.getI64IntegerAttr(1));
     SmallVector<Value> reductionVariables;
     reductionVariables.reserve(parallelOp.getNumReductions());
@@ -391,8 +391,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
              "cannot create a reduction variable if the type is not an LLVM "
              "pointer element");
       Value storage =
-          rewriter.create<LLVM::AllocaOp>(loc, ptrType, init.getType(), one, 0);
-      rewriter.create<LLVM::StoreOp>(loc, init, storage);
+          LLVM::AllocaOp::create(rewriter, loc, ptrType, init.getType(), one, 0);
+      LLVM::StoreOp::create(rewriter, loc, init, storage);
       reductionVariables.push_back(storage);
     }
 
@@ -411,7 +411,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
       assert(redRegion.hasOneBlock() &&
              "expect reduction region to have one block");
       Value pvtRedVar = parallelOp.getRegion().addArgument(x.getType(), loc);
-      Value pvtRedVal = rewriter.create<LLVM::LoadOp>(reduce.getLoc(),
+      Value pvtRedVal = LLVM::LoadOp::create(rewriter, reduce.getLoc(),
                                                       rD.getType(), pvtRedVar);
       // Make a copy of the reduction combiner region in the body
       mlir::OpBuilder builder(rewriter.getContext());
@@ -427,7 +427,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           assert(yieldOp && yieldOp.getResults().size() == 1 &&
                  "expect YieldOp in reduction region to return one result");
           Value redVal = yieldOp.getResults()[0];
-          rewriter.create<LLVM::StoreOp>(loc, redVal, pvtRedVar);
+          LLVM::StoreOp::create(rewriter, loc, redVal, pvtRedVar);
           rewriter.eraseOp(yieldOp);
           break;
         }
@@ -437,11 +437,11 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
 
     Value numThreadsVar;
     if (numThreads > 0) {
-      numThreadsVar = rewriter.create<LLVM::ConstantOp>(
+      numThreadsVar = LLVM::ConstantOp::create(rewriter,
           loc, rewriter.getI32IntegerAttr(numThreads));
     }
     // Create the parallel wrapper.
-    auto ompParallel = rewriter.create<omp::ParallelOp>(
+    auto ompParallel = omp::ParallelOp::create(rewriter,
         loc,
         /* allocate_vars = */ llvm::SmallVector<Value>{},
         /* allocator_vars = */ llvm::SmallVector<Value>{},
@@ -464,7 +464,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
       {
         OpBuilder::InsertionGuard allocaGuard(rewriter);
         // Create worksharing loop wrapper.
-        auto wsloopOp = rewriter.create<omp::WsloopOp>(parallelOp.getLoc());
+        auto wsloopOp = omp::WsloopOp::create(rewriter, parallelOp.getLoc());
         if (!reductionVariables.empty()) {
           wsloopOp.setReductionSymsAttr(
               ArrayAttr::get(rewriter.getContext(), reductionSyms));
@@ -476,7 +476,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           wsloopOp.setReductionByref(
               DenseBoolArrayAttr::get(rewriter.getContext(), reductionByRef));
         }
-        rewriter.create<omp::TerminatorOp>(loc); // omp.parallel terminator.
+        omp::TerminatorOp::create(rewriter, loc); // omp.parallel terminator.
 
         // The wrapper's entry block arguments will define the reduction
         // variables.
@@ -490,7 +490,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
                                               parallelOp.getLoc()));
 
         // Create loop nest and populate region with contents of scf.parallel.
-        auto loopOp = rewriter.create<omp::LoopNestOp>(
+        auto loopOp = omp::LoopNestOp::create(rewriter,
             parallelOp.getLoc(), parallelOp.getLowerBound(),
             parallelOp.getUpperBound(), parallelOp.getStep());
 
@@ -511,13 +511,13 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
             rewriter.splitBlock(&loopOpEntryBlock, loopOpEntryBlock.begin());
         rewriter.setInsertionPointToStart(&loopOpEntryBlock);
 
-        auto scope = rewriter.create<memref::AllocaScopeOp>(parallelOp.getLoc(),
+        auto scope = memref::AllocaScopeOp::create(rewriter, parallelOp.getLoc(),
                                                             TypeRange());
-        rewriter.create<omp::YieldOp>(loc, ValueRange());
+        omp::YieldOp::create(rewriter, loc, ValueRange());
         Block *scopeBlock = rewriter.createBlock(&scope.getBodyRegion());
         rewriter.mergeBlocks(ops, scopeBlock);
         rewriter.setInsertionPointToEnd(&*scope.getBodyRegion().begin());
-        rewriter.create<memref::AllocaScopeReturnOp>(loc, ValueRange());
+        memref::AllocaScopeReturnOp::create(rewriter, loc, ValueRange());
       }
     }
 
@@ -526,7 +526,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     results.reserve(reductionVariables.size());
     for (auto [variable, type] :
          llvm::zip(reductionVariables, parallelOp.getResultTypes())) {
-      Value res = rewriter.create<LLVM::LoadOp>(loc, type, variable);
+      Value res = LLVM::LoadOp::create(rewriter, loc, type, variable);
       results.push_back(res);
     }
     rewriter.replaceOp(parallelOp, results);
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index 78d13278fef53..b494fa98f1362 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -71,12 +71,12 @@ void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp,
     auto pointerType =
         spirv::PointerType::get(convertedType, spirv::StorageClass::Function);
     rewriter.setInsertionPoint(newOp);
-    auto alloc = rewriter.create<spirv::VariableOp>(
+    auto alloc = spirv::VariableOp::create(rewriter,
         loc, pointerType, spirv::StorageClass::Function,
         /*initializer=*/nullptr);
     allocas.push_back(alloc);
     rewriter.setInsertionPointAfter(newOp);
-    Value loadResult = rewriter.create<spirv::LoadOp>(loc, alloc);
+    Value loadResult = spirv::LoadOp::create(rewriter, loc, alloc);
     resultValue.push_back(loadResult);
   }
   rewriter.replaceOp(scfOp, resultValue);
@@ -135,7 +135,7 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // a single back edge from the continue to header block, and a single exit
     // from header to merge.
     auto loc = forOp.getLoc();
-    auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
+    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     OpBuilder::InsertionGuard guard(rewriter);
@@ -172,15 +172,15 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     args.append(adaptor.getInitArgs().begin(), adaptor.getInitArgs().end());
     // Branch into it from the entry.
     rewriter.setInsertionPointToEnd(&(loopOp.getBody().front()));
-    rewriter.create<spirv::BranchOp>(loc, header, args);
+    spirv::BranchOp::create(rewriter, loc, header, args);
 
     // Generate the rest of the loop header.
     rewriter.setInsertionPointToEnd(header);
     auto *mergeBlock = loopOp.getMergeBlock();
-    auto cmpOp = rewriter.create<spirv::SLessThanOp>(
+    auto cmpOp = spirv::SLessThanOp::create(rewriter,
         loc, rewriter.getI1Type(), newIndVar, adaptor.getUpperBound());
 
-    rewriter.create<spirv::BranchConditionalOp>(
+    spirv::BranchConditionalOp::create(rewriter,
         loc, cmpOp, body, ArrayRef<Value>(), mergeBlock, ArrayRef<Value>());
 
     // Generate instructions to increment the step of the induction variable and
@@ -189,9 +189,9 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     rewriter.setInsertionPointToEnd(continueBlock);
 
     // Add the step to the induction variable and branch to the header.
-    Value updatedIndVar = rewriter.create<spirv::IAddOp>(
+    Value updatedIndVar = spirv::IAddOp::create(rewriter,
         loc, newIndVar.getType(), newIndVar, adaptor.getStep());
-    rewriter.create<spirv::BranchOp>(loc, header, updatedIndVar);
+    spirv::BranchOp::create(rewriter, loc, header, updatedIndVar);
 
     // Infer the return types from the init operands. Vector type may get
     // converted to CooperativeMatrix or to Vector type, to avoid having complex
@@ -238,10 +238,10 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     // Create `spirv.selection` operation, selection header block and merge
     // block.
     auto selectionOp =
-        rewriter.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
+        spirv::SelectionOp::create(rewriter, loc, spirv::SelectionControl::None);
     auto *mergeBlock = rewriter.createBlock(&selectionOp.getBody(),
                                             selectionOp.getBody().end());
-    rewriter.create<spirv::MergeOp>(loc);
+    spirv::MergeOp::create(rewriter, loc);
 
     OpBuilder::InsertionGuard guard(rewriter);
     auto *selectionHeaderBlock =
@@ -251,7 +251,7 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     auto &thenRegion = ifOp.getThenRegion();
     auto *thenBlock = &thenRegion.front();
     rewriter.setInsertionPointToEnd(&thenRegion.back());
-    rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+    spirv::BranchOp::create(rewriter, loc, mergeBlock);
     rewriter.inlineRegionBefore(thenRegion, mergeBlock);
 
     auto *elseBlock = mergeBlock;
@@ -261,13 +261,13 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
       auto &elseRegion = ifOp.getElseRegion();
       elseBlock = &elseRegion.front();
       rewriter.setInsertionPointToEnd(&elseRegion.back());
-      rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+      spirv::BranchOp::create(rewriter, loc, mergeBlock);
       rewriter.inlineRegionBefore(elseRegion, mergeBlock);
     }
 
     // Create a `spirv.BranchConditional` operation for selection header block.
     rewriter.setInsertionPointToEnd(selectionHeaderBlock);
-    rewriter.create<spirv::BranchConditionalOp>(loc, adaptor.getCondition(),
+    spirv::BranchConditionalOp::create(rewriter, loc, adaptor.getCondition(),
                                                 thenBlock, ArrayRef<Value>(),
                                                 elseBlock, ArrayRef<Value>());
 
@@ -310,7 +310,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
 
       auto loc = terminatorOp.getLoc();
       for (unsigned i = 0, e = operands.size(); i < e; i++)
-        rewriter.create<spirv::StoreOp>(loc, allocas[i], operands[i]);
+        spirv::StoreOp::create(rewriter, loc, allocas[i], operands[i]);
       if (isa<spirv::LoopOp>(parent)) {
         // For loops we also need to update the branch jumping back to the
         // header.
@@ -319,7 +319,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
         SmallVector<Value, 8> args(br.getBlockArguments());
         args.append(operands.begin(), operands.end());
         rewriter.setInsertionPoint(br);
-        rewriter.create<spirv::BranchOp>(terminatorOp.getLoc(), br.getTarget(),
+        spirv::BranchOp::create(rewriter, terminatorOp.getLoc(), br.getTarget(),
                                          args);
         rewriter.eraseOp(br);
       }
@@ -340,7 +340,7 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
   matchAndRewrite(scf::WhileOp whileOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto loc = whileOp.getLoc();
-    auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
+    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     Region &beforeRegion = whileOp.getBefore();
@@ -382,7 +382,7 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
     // Jump from the loop entry block to the loop header block.
     rewriter.setInsertionPointToEnd(&entryBlock);
-    rewriter.create<spirv::BranchOp>(loc, &beforeBlock, adaptor.getInits());
+    spirv::BranchOp::create(rewriter, loc, &beforeBlock, adaptor.getInits());
 
     auto condLoc = cond.getLoc();
 
@@ -403,18 +403,18 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
       // Create local variables before the scf.while op.
       rewriter.setInsertionPoint(loopOp);
-      auto alloc = rewriter.create<spirv::VariableOp>(
+      auto alloc = spirv::VariableOp::create(rewriter,
           condLoc, pointerType, spirv::StorageClass::Function,
           /*initializer=*/nullptr);
 
       // Load the final result values after the scf.while op.
       rewriter.setInsertionPointAfter(loopOp);
-      auto loadResult = rewriter.create<spirv::LoadOp>(condLoc, alloc);
+      auto loadResult = spirv::LoadOp::create(rewriter, condLoc, alloc);
       resultValues[i] = loadResult;
 
       // Store the current iteration's result value.
       rewriter.setInsertionPointToEnd(&beforeBlock);
-      rewriter.create<spirv::StoreOp>(condLoc, alloc, res);
+      spirv::StoreOp::create(rewriter, condLoc, alloc, res);
     }
 
     rewriter.setInsertionPointToEnd(&beforeBlock);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index d7ae9f0e94fe8..934bb0d1c37fd 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -68,7 +68,7 @@ static unsigned calculateGlobalIndex(spirv::GlobalVariableOp op) {
 /// Copies the given number of bytes from src to dst pointers.
 static void copy(Location loc, Value dst, Value src, Value size,
                  OpBuilder &builder) {
-  builder.create<LLVM::MemcpyOp>(loc, dst, src, size, /*isVolatile=*/false);
+  LLVM::MemcpyOp::create(builder, loc, dst, src, size, /*isVolatile=*/false);
 }
 
 /// Encodes the binding and descriptor set numbers into a new symbolic name.
@@ -194,7 +194,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
     if (!kernelFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(module.getBody());
-      kernelFunc = rewriter.create<LLVM::LLVMFuncOp>(
+      kernelFunc = LLVM::LLVMFuncOp::create(rewriter,
           rewriter.getUnknownLoc(), newKernelFuncName,
           LLVM::LLVMFunctionType::get(LLVM::LLVMVoidType::get(context),
                                       ArrayRef<Type>()));
@@ -245,7 +245,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       if (!dstGlobal) {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
-        dstGlobal = rewriter.create<LLVM::GlobalOp>(
+        dstGlobal = LLVM::GlobalOp::create(rewriter,
             loc, dstGlobalType,
             /*isConstant=*/false, LLVM::Linkage::Linkonce, name, Attribute(),
             /*alignment=*/0);
@@ -255,7 +255,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       // Copy the data from src operand pointer to dst global variable. Save
       // src, dst and size so that we can copy data back after emulating the
       // kernel call.
-      Value dst = rewriter.create<LLVM::AddressOfOp>(
+      Value dst = LLVM::AddressOfOp::create(rewriter,
           loc, typeConverter->convertType(spirvGlobal.getType()),
           dstGlobal.getSymName());
       copy(loc, dst, src, sizeBytes, rewriter);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index cea9d1fdec809..901b1eb265e32 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -94,12 +94,12 @@ static IntegerAttr minusOneIntegerAttribute(Type type, Builder builder) {
 static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
                                       PatternRewriter &rewriter) {
   if (isa<VectorType>(srcType)) {
-    return rewriter.create<LLVM::ConstantOp>(
+    return LLVM::ConstantOp::create(rewriter,
         loc, dstType,
         SplatElementsAttr::get(cast<ShapedType>(srcType),
                                minusOneIntegerAttribute(srcType, rewriter)));
   }
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, dstType, minusOneIntegerAttribute(srcType, rewriter));
 }
 
@@ -108,13 +108,13 @@ static Value createFPConstant(Location loc, Type srcType, Type dstType,
                               PatternRewriter &rewriter, double value) {
   if (auto vecType = dyn_cast<VectorType>(srcType)) {
     auto floatType = cast<FloatType>(vecType.getElementType());
-    return rewriter.create<LLVM::ConstantOp>(
+    return LLVM::ConstantOp::create(rewriter,
         loc, dstType,
         SplatElementsAttr::get(vecType,
                                rewriter.getFloatAttr(floatType, value)));
   }
   auto floatType = cast<FloatType>(srcType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, dstType, rewriter.getFloatAttr(floatType, value));
 }
 
@@ -134,13 +134,13 @@ static Value optionallyTruncateOrExtend(Location loc, Value value,
                                : getBitWidth(srcType);
 
   if (valueBitWidth < targetBitWidth)
-    return rewriter.create<LLVM::ZExtOp>(loc, llvmType, value);
+    return LLVM::ZExtOp::create(rewriter, loc, llvmType, value);
   // If the bit widths of `Count` and `Offset` are greater than the bit width
   // of the target type, they are truncated. Truncation is safe since `Count`
   // and `Offset` must be no more than 64 for op behaviour to be defined. Hence,
   // both values can be expressed in 8 bits.
   if (valueBitWidth > targetBitWidth)
-    return rewriter.create<LLVM::TruncOp>(loc, llvmType, value);
+    return LLVM::TruncOp::create(rewriter, loc, llvmType, value);
   return value;
 }
 
@@ -151,11 +151,11 @@ static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
   auto vectorType = VectorType::get(numElements, toBroadcast.getType());
   auto llvmVectorType = typeConverter.convertType(vectorType);
   auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
-  Value broadcasted = rewriter.create<LLVM::PoisonOp>(loc, llvmVectorType);
+  Value broadcasted = LLVM::PoisonOp::create(rewriter, loc, llvmVectorType);
   for (unsigned i = 0; i < numElements; ++i) {
-    auto index = rewriter.create<LLVM::ConstantOp>(
+    auto index = LLVM::ConstantOp::create(rewriter,
         loc, llvmI32Type, rewriter.getI32IntegerAttr(i));
-    broadcasted = rewriter.create<LLVM::InsertElementOp>(
+    broadcasted = LLVM::InsertElementOp::create(rewriter,
         loc, llvmVectorType, broadcasted, toBroadcast, index);
   }
   return broadcasted;
@@ -217,7 +217,7 @@ static Type convertStructTypePacked(spirv::StructType type,
 /// Creates LLVM dialect constant with the given value.
 static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
                                  unsigned value) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, IntegerType::get(rewriter.getContext(), 32),
       rewriter.getIntegerAttr(rewriter.getI32Type(), value));
 }
@@ -322,7 +322,7 @@ class AccessChainPattern : public SPIRVToLLVMConversion<spirv::AccessChainOp> {
     auto llvmIndexType = getTypeConverter()->convertType(indexType);
     if (!llvmIndexType)
       return rewriter.notifyMatchFailure(op, "type conversion failed");
-    Value zero = rewriter.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(rewriter,
         op.getLoc(), llvmIndexType, rewriter.getIntegerAttr(indexType, 0));
     indices.insert(indices.begin(), zero);
 
@@ -375,20 +375,20 @@ class BitFieldInsertPattern
     // Create a mask with bits set outside [Offset, Offset + Count - 1].
     Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
     Value maskShiftedByCount =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
-    Value negated = rewriter.create<LLVM::XOrOp>(loc, dstType,
+        LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
+    Value negated = LLVM::XOrOp::create(rewriter, loc, dstType,
                                                  maskShiftedByCount, minusOne);
     Value maskShiftedByCountAndOffset =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, negated, offset);
-    Value mask = rewriter.create<LLVM::XOrOp>(
+        LLVM::ShlOp::create(rewriter, loc, dstType, negated, offset);
+    Value mask = LLVM::XOrOp::create(rewriter,
         loc, dstType, maskShiftedByCountAndOffset, minusOne);
 
     // Extract unchanged bits from the `Base`  that are outside of
     // [Offset, Offset + Count - 1]. Then `or` with shifted `Insert`.
     Value baseAndMask =
-        rewriter.create<LLVM::AndOp>(loc, dstType, op.getBase(), mask);
+        LLVM::AndOp::create(rewriter, loc, dstType, op.getBase(), mask);
     Value insertShiftedByOffset =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, op.getInsert(), offset);
+        LLVM::ShlOp::create(rewriter, loc, dstType, op.getInsert(), offset);
     rewriter.replaceOpWithNewOp<LLVM::OrOp>(op, dstType, baseAndMask,
                                             insertShiftedByOffset);
     return success();
@@ -470,23 +470,23 @@ class BitFieldSExtractPattern
     auto baseSize = rewriter.getIntegerAttr(integerType, getBitWidth(srcType));
     Value size =
         isa<VectorType>(srcType)
-            ? rewriter.create<LLVM::ConstantOp>(
+            ? LLVM::ConstantOp::create(rewriter,
                   loc, dstType,
                   SplatElementsAttr::get(cast<ShapedType>(srcType), baseSize))
-            : rewriter.create<LLVM::ConstantOp>(loc, dstType, baseSize);
+            : LLVM::ConstantOp::create(rewriter, loc, dstType, baseSize);
 
     // Shift `Base` left by [sizeof(Base) - (Count + Offset)], so that the bit
     // at Offset + Count - 1 is the most significant bit now.
     Value countPlusOffset =
-        rewriter.create<LLVM::AddOp>(loc, dstType, count, offset);
+        LLVM::AddOp::create(rewriter, loc, dstType, count, offset);
     Value amountToShiftLeft =
-        rewriter.create<LLVM::SubOp>(loc, dstType, size, countPlusOffset);
-    Value baseShiftedLeft = rewriter.create<LLVM::ShlOp>(
+        LLVM::SubOp::create(rewriter, loc, dstType, size, countPlusOffset);
+    Value baseShiftedLeft = LLVM::ShlOp::create(rewriter,
         loc, dstType, op.getBase(), amountToShiftLeft);
 
     // Shift the result right, filling the bits with the sign bit.
     Value amountToShiftRight =
-        rewriter.create<LLVM::AddOp>(loc, dstType, offset, amountToShiftLeft);
+        LLVM::AddOp::create(rewriter, loc, dstType, offset, amountToShiftLeft);
     rewriter.replaceOpWithNewOp<LLVM::AShrOp>(op, dstType, baseShiftedLeft,
                                               amountToShiftRight);
     return success();
@@ -516,13 +516,13 @@ class BitFieldUExtractPattern
     // Create a mask with bits set at [0, Count - 1].
     Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
     Value maskShiftedByCount =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
-    Value mask = rewriter.create<LLVM::XOrOp>(loc, dstType, maskShiftedByCount,
+        LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
+    Value mask = LLVM::XOrOp::create(rewriter, loc, dstType, maskShiftedByCount,
                                               minusOne);
 
     // Shift `Base` by `Offset` and apply the mask on it.
     Value shiftedBase =
-        rewriter.create<LLVM::LShrOp>(loc, dstType, op.getBase(), offset);
+        LLVM::LShrOp::create(rewriter, loc, dstType, op.getBase(), offset);
     rewriter.replaceOpWithNewOp<LLVM::AndOp>(op, dstType, shiftedBase, mask);
     return success();
   }
@@ -694,7 +694,7 @@ class ExecutionModePattern
     auto structType = LLVM::LLVMStructType::getLiteral(context, fields);
 
     // Create `llvm.mlir.global` with initializer region containing one block.
-    auto global = rewriter.create<LLVM::GlobalOp>(
+    auto global = LLVM::GlobalOp::create(rewriter,
         UnknownLoc::get(context), structType, /*isConstant=*/true,
         LLVM::Linkage::External, executionModeInfoName, Attribute(),
         /*alignment=*/0);
@@ -704,22 +704,22 @@ class ExecutionModePattern
 
     // Initialize the struct and set the execution mode value.
     rewriter.setInsertionPointToStart(block);
-    Value structValue = rewriter.create<LLVM::PoisonOp>(loc, structType);
-    Value executionMode = rewriter.create<LLVM::ConstantOp>(
+    Value structValue = LLVM::PoisonOp::create(rewriter, loc, structType);
+    Value executionMode = LLVM::ConstantOp::create(rewriter,
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
-    structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
+    structValue = LLVM::InsertValueOp::create(rewriter, loc, structValue,
                                                        executionMode, 0);
 
     // Insert extra operands if they exist into execution mode info struct.
     for (unsigned i = 0, e = values.size(); i < e; ++i) {
       auto attr = values.getValue()[i];
-      Value entry = rewriter.create<LLVM::ConstantOp>(loc, llvmI32Type, attr);
-      structValue = rewriter.create<LLVM::InsertValueOp>(
+      Value entry = LLVM::ConstantOp::create(rewriter, loc, llvmI32Type, attr);
+      structValue = LLVM::InsertValueOp::create(rewriter,
           loc, structValue, entry, ArrayRef<int64_t>({1, i}));
     }
-    rewriter.create<LLVM::ReturnOp>(loc, ArrayRef<Value>({structValue}));
+    LLVM::ReturnOp::create(rewriter, loc, ArrayRef<Value>({structValue}));
     rewriter.eraseOp(op);
     return success();
   }
@@ -911,7 +911,7 @@ class InverseSqrtPattern
 
     Location loc = op.getLoc();
     Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
-    Value sqrt = rewriter.create<LLVM::SqrtOp>(loc, dstType, op.getOperand());
+    Value sqrt = LLVM::SqrtOp::create(rewriter, loc, dstType, op.getOperand());
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(op, dstType, one, sqrt);
     return success();
   }
@@ -971,10 +971,10 @@ class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
     IntegerAttr minusOne = minusOneIntegerAttribute(srcType, rewriter);
     auto mask =
         isa<VectorType>(srcType)
-            ? rewriter.create<LLVM::ConstantOp>(
+            ? LLVM::ConstantOp::create(rewriter,
                   loc, dstType,
                   SplatElementsAttr::get(cast<VectorType>(srcType), minusOne))
-            : rewriter.create<LLVM::ConstantOp>(loc, dstType, minusOne);
+            : LLVM::ConstantOp::create(rewriter, loc, dstType, minusOne);
     rewriter.template replaceOpWithNewOp<LLVM::XOrOp>(notOp, dstType,
                                                       notOp.getOperand(), mask);
     return success();
@@ -1032,7 +1032,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
     return func;
 
   OpBuilder b(symbolTable->getRegion(0));
-  func = b.create<LLVM::LLVMFuncOp>(
+  func = LLVM::LLVMFuncOp::create(b,
       symbolTable->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes));
   func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
@@ -1045,7 +1045,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
 static LLVM::CallOp createSPIRVBuiltinCall(Location loc, OpBuilder &builder,
                                            LLVM::LLVMFuncOp func,
                                            ValueRange args) {
-  auto call = builder.create<LLVM::CallOp>(loc, func, args);
+  auto call = LLVM::CallOp::create(builder, loc, func, args);
   call.setCConv(func.getCConv());
   call.setConvergentAttr(func.getConvergentAttr());
   call.setNoUnwindAttr(func.getNoUnwindAttr());
@@ -1076,11 +1076,11 @@ class ControlBarrierPattern : public SPIRVToLLVMConversion<BarrierOpTy> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, {i32, i32, i32}, voidTy);
 
     Location loc = controlBarrierOp->getLoc();
-    Value execution = rewriter.create<LLVM::ConstantOp>(
+    Value execution = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value memory = rewriter.create<LLVM::ConstantOp>(
+    Value memory = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getMemoryScope()));
-    Value semantics = rewriter.create<LLVM::ConstantOp>(
+    Value semantics = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getMemorySemantics()));
 
     auto call = createSPIRVBuiltinCall(loc, rewriter, func,
@@ -1253,9 +1253,9 @@ class GroupReducePattern : public SPIRVToLLVMConversion<ReduceOp> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, paramTypes, retTy);
 
     Location loc = op.getLoc();
-    Value scope = rewriter.create<LLVM::ConstantOp>(
+    Value scope = LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value groupOp = rewriter.create<LLVM::ConstantOp>(
+    Value groupOp = LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, static_cast<int32_t>(adaptor.getGroupOperation()));
     SmallVector<Value> operands{scope, groupOp};
     operands.append(adaptor.getOperands().begin(), adaptor.getOperands().end());
@@ -1366,7 +1366,7 @@ class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
       return failure();
     Block *headerBlock = loopOp.getHeaderBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
-    rewriter.create<LLVM::BrOp>(loc, brOp.getBlockArguments(), headerBlock);
+    LLVM::BrOp::create(rewriter, loc, brOp.getBlockArguments(), headerBlock);
     rewriter.eraseBlock(entryBlock);
 
     // Branch from merge block to end block.
@@ -1374,7 +1374,7 @@ class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
     Operation *terminator = mergeBlock->getTerminator();
     ValueRange terminatorOperands = terminator->getOperands();
     rewriter.setInsertionPointToEnd(mergeBlock);
-    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, endBlock);
+    LLVM::BrOp::create(rewriter, loc, terminatorOperands, endBlock);
 
     rewriter.inlineRegionBefore(loopOp.getBody(), endBlock);
     rewriter.replaceOp(loopOp, endBlock->getArguments());
@@ -1433,13 +1433,13 @@ class SelectionPattern : public SPIRVToLLVMConversion<spirv::SelectionOp> {
     Operation *terminator = mergeBlock->getTerminator();
     ValueRange terminatorOperands = terminator->getOperands();
     rewriter.setInsertionPointToEnd(mergeBlock);
-    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, continueBlock);
+    LLVM::BrOp::create(rewriter, loc, terminatorOperands, continueBlock);
 
     // Link current block to `true` and `false` blocks within the selection.
     Block *trueBlock = condBrOp.getTrueBlock();
     Block *falseBlock = condBrOp.getFalseBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
-    rewriter.create<LLVM::CondBrOp>(loc, condBrOp.getCondition(), trueBlock,
+    LLVM::CondBrOp::create(rewriter, loc, condBrOp.getCondition(), trueBlock,
                                     condBrOp.getTrueTargetOperands(),
                                     falseBlock,
                                     condBrOp.getFalseTargetOperands());
@@ -1519,8 +1519,8 @@ class TanPattern : public SPIRVToLLVMConversion<spirv::GLTanOp> {
       return rewriter.notifyMatchFailure(tanOp, "type conversion failed");
 
     Location loc = tanOp.getLoc();
-    Value sin = rewriter.create<LLVM::SinOp>(loc, dstType, tanOp.getOperand());
-    Value cos = rewriter.create<LLVM::CosOp>(loc, dstType, tanOp.getOperand());
+    Value sin = LLVM::SinOp::create(rewriter, loc, dstType, tanOp.getOperand());
+    Value cos = LLVM::CosOp::create(rewriter, loc, dstType, tanOp.getOperand());
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanOp, dstType, sin, cos);
     return success();
   }
@@ -1547,13 +1547,13 @@ class TanhPattern : public SPIRVToLLVMConversion<spirv::GLTanhOp> {
     Location loc = tanhOp.getLoc();
     Value two = createFPConstant(loc, srcType, dstType, rewriter, 2.0);
     Value multiplied =
-        rewriter.create<LLVM::FMulOp>(loc, dstType, two, tanhOp.getOperand());
-    Value exponential = rewriter.create<LLVM::ExpOp>(loc, dstType, multiplied);
+        LLVM::FMulOp::create(rewriter, loc, dstType, two, tanhOp.getOperand());
+    Value exponential = LLVM::ExpOp::create(rewriter, loc, dstType, multiplied);
     Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
     Value numerator =
-        rewriter.create<LLVM::FSubOp>(loc, dstType, exponential, one);
+        LLVM::FSubOp::create(rewriter, loc, dstType, exponential, one);
     Value denominator =
-        rewriter.create<LLVM::FAddOp>(loc, dstType, exponential, one);
+        LLVM::FAddOp::create(rewriter, loc, dstType, exponential, one);
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanhOp, dstType, numerator,
                                               denominator);
     return success();
@@ -1592,8 +1592,8 @@ class VariablePattern : public SPIRVToLLVMConversion<spirv::VariableOp> {
     if (!elementType)
       return rewriter.notifyMatchFailure(varOp, "type conversion failed");
     Value allocated =
-        rewriter.create<LLVM::AllocaOp>(loc, dstType, elementType, size);
-    rewriter.create<LLVM::StoreOp>(loc, adaptor.getInitializer(), allocated);
+        LLVM::AllocaOp::create(rewriter, loc, dstType, elementType, size);
+    LLVM::StoreOp::create(rewriter, loc, adaptor.getInitializer(), allocated);
     rewriter.replaceOp(varOp, allocated);
     return success();
   }
@@ -1654,7 +1654,7 @@ class FuncConversionPattern : public SPIRVToLLVMConversion<spirv::FuncOp> {
     // Create a new `LLVMFuncOp`
     Location loc = funcOp.getLoc();
     StringRef name = funcOp.getName();
-    auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(loc, name, llvmType);
+    auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter, loc, name, llvmType);
 
     // Convert SPIR-V Function Control to equivalent LLVM function attribute
     MLIRContext *context = funcOp.getContext();
@@ -1708,7 +1708,7 @@ class ModuleConversionPattern : public SPIRVToLLVMConversion<spirv::ModuleOp> {
                   ConversionPatternRewriter &rewriter) const override {
 
     auto newModuleOp =
-        rewriter.create<ModuleOp>(spvModuleOp.getLoc(), spvModuleOp.getName());
+        ModuleOp::create(rewriter, spvModuleOp.getLoc(), spvModuleOp.getName());
     rewriter.inlineRegionBefore(spvModuleOp.getRegion(), newModuleOp.getBody());
 
     // Remove the terminator block that was automatically added by builder
@@ -1749,7 +1749,7 @@ class VectorShufflePattern
     auto componentsArray = components.getValue();
     auto *context = rewriter.getContext();
     auto llvmI32Type = IntegerType::get(context, 32);
-    Value targetOp = rewriter.create<LLVM::PoisonOp>(loc, dstType);
+    Value targetOp = LLVM::PoisonOp::create(rewriter, loc, dstType);
     for (unsigned i = 0; i < componentsArray.size(); i++) {
       if (!isa<IntegerAttr>(componentsArray[i]))
         return op.emitError("unable to support non-constant component");
@@ -1765,15 +1765,15 @@ class VectorShufflePattern
         baseVector = vector2;
       }
 
-      Value dstIndex = rewriter.create<LLVM::ConstantOp>(
+      Value dstIndex = LLVM::ConstantOp::create(rewriter,
           loc, llvmI32Type, rewriter.getIntegerAttr(rewriter.getI32Type(), i));
-      Value index = rewriter.create<LLVM::ConstantOp>(
+      Value index = LLVM::ConstantOp::create(rewriter,
           loc, llvmI32Type,
           rewriter.getIntegerAttr(rewriter.getI32Type(), indexVal - offsetVal));
 
-      auto extractOp = rewriter.create<LLVM::ExtractElementOp>(
+      auto extractOp = LLVM::ExtractElementOp::create(rewriter,
           loc, scalarType, baseVector, index);
-      targetOp = rewriter.create<LLVM::InsertElementOp>(loc, dstType, targetOp,
+      targetOp = LLVM::InsertElementOp::create(rewriter, loc, dstType, targetOp,
                                                         extractOp, dstIndex);
     }
     rewriter.replaceOp(op, targetOp);
diff --git a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
index da9ad3dd67328..245e60b04ec31 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
@@ -32,7 +32,7 @@ class ConvertCstrRequireOp : public OpRewritePattern<shape::CstrRequireOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(shape::CstrRequireOp op,
                                 PatternRewriter &rewriter) const override {
-    rewriter.create<cf::AssertOp>(op.getLoc(), op.getPred(), op.getMsgAttr());
+    cf::AssertOp::create(rewriter, op.getLoc(), op.getPred(), op.getMsgAttr());
     rewriter.replaceOpWithNewOp<shape::ConstWitnessOp>(op, true);
     return success();
   }
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index bbe1490137bf8..58cffa6119d93 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -82,19 +82,19 @@ struct BroadcastOpConverter : public OpConversionPattern<BroadcastOp> {
 // number of extent tensors and shifted offsets into them.
 Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
                         ValueRange rankDiffs, Value outputDimension) {
-  Value one = lb.create<arith::ConstantIndexOp>(1);
+  Value one = arith::ConstantIndexOp::create(lb, 1);
   Value broadcastedDim = one;
   for (auto tup : llvm::zip(extentTensors, rankDiffs)) {
     Value shape = std::get<0>(tup);
     Value rankDiff = std::get<1>(tup);
-    Value outOfBounds = lb.create<arith::CmpIOp>(arith::CmpIPredicate::ult,
+    Value outOfBounds = arith::CmpIOp::create(lb, arith::CmpIPredicate::ult,
                                                  outputDimension, rankDiff);
     Type indexTy = lb.getIndexType();
     broadcastedDim =
-        lb.create<IfOp>(
+        IfOp::create(lb,
               outOfBounds,
               [&](OpBuilder &b, Location loc) {
-                b.create<scf::YieldOp>(loc, broadcastedDim);
+                scf::YieldOp::create(b, loc, broadcastedDim);
               },
               [&](OpBuilder &b, Location loc) {
                 // The broadcasting logic is:
@@ -104,17 +104,17 @@ Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
                 // - otherwise, take the extent as-is.
                 // Note that this logic remains correct in the presence
                 // of dimensions of zero extent.
-                Value lesserRankOperandDimension = b.create<arith::SubIOp>(
+                Value lesserRankOperandDimension = arith::SubIOp::create(b,
                     loc, indexTy, outputDimension, rankDiff);
-                Value lesserRankOperandExtent = b.create<tensor::ExtractOp>(
+                Value lesserRankOperandExtent = tensor::ExtractOp::create(b,
                     loc, shape, ValueRange{lesserRankOperandDimension});
 
                 Value dimIsOne =
-                    b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+                    arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
                                             lesserRankOperandExtent, one);
-                Value dim = b.create<arith::SelectOp>(
+                Value dim = arith::SelectOp::create(b,
                     loc, dimIsOne, broadcastedDim, lesserRankOperandExtent);
-                b.create<scf::YieldOp>(loc, dim);
+                scf::YieldOp::create(b, loc, dim);
               })
             .getResult(0);
   }
@@ -133,7 +133,7 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
   auto loc = op.getLoc();
   ImplicitLocOpBuilder lb(loc, rewriter);
 
-  Value zero = lb.create<arith::ConstantIndexOp>(0);
+  Value zero = arith::ConstantIndexOp::create(lb, 0);
   Type indexTy = lb.getIndexType();
 
   // Save all the ranks for bounds checking. Because this is a tensor
@@ -141,31 +141,31 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
   // dimension in the tensor.
   SmallVector<Value> ranks, rankDiffs;
   llvm::append_range(ranks, llvm::map_range(adaptor.getShapes(), [&](Value v) {
-                       return lb.create<tensor::DimOp>(v, zero);
+                       return tensor::DimOp::create(lb, v, zero);
                      }));
 
   // Find the maximum rank
   Value maxRank = ranks.front();
   for (Value v : llvm::drop_begin(ranks, 1)) {
-    maxRank = lb.create<arith::MaxUIOp>(v, maxRank);
+    maxRank = arith::MaxUIOp::create(lb, v, maxRank);
   }
 
   // Calculate the difference of ranks and the maximum rank for later offsets.
   llvm::append_range(rankDiffs, llvm::map_range(ranks, [&](Value v) {
-                       return lb.create<arith::SubIOp>(indexTy, maxRank, v);
+                       return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
-  Value replacement = lb.create<tensor::GenerateOp>(
+  Value replacement = tensor::GenerateOp::create(lb,
       getExtentTensorType(lb.getContext()), ValueRange{maxRank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value broadcastedDim =
             getBroadcastedDim(ImplicitLocOpBuilder(loc, b), adaptor.getShapes(),
                               rankDiffs, args[0]);
 
-        b.create<tensor::YieldOp>(loc, broadcastedDim);
+        tensor::YieldOp::create(b, loc, broadcastedDim);
       });
   if (replacement.getType() != op.getType())
-    replacement = lb.create<tensor::CastOp>(op.getType(), replacement);
+    replacement = tensor::CastOp::create(lb, op.getType(), replacement);
   rewriter.replaceOp(op, replacement);
   return success();
 }
@@ -194,12 +194,12 @@ LogicalResult ConstShapeOpConverter::matchAndRewrite(
   SmallVector<Value, 4> extentOperands;
   for (auto extent : op.getShape()) {
     extentOperands.push_back(
-        rewriter.create<arith::ConstantIndexOp>(loc, extent.getLimitedValue()));
+        arith::ConstantIndexOp::create(rewriter, loc, extent.getLimitedValue()));
   }
   Type resultTy =
       RankedTensorType::get({op.getShape().size()}, rewriter.getIndexType());
   Value tensor =
-      rewriter.create<tensor::FromElementsOp>(loc, resultTy, extentOperands);
+      tensor::FromElementsOp::create(rewriter, loc, resultTy, extentOperands);
   rewriter.replaceOpWithNewOp<tensor::CastOp>(op, resultTy, tensor);
   return success();
 }
@@ -245,8 +245,8 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
 
   auto loc = op.getLoc();
   ImplicitLocOpBuilder lb(loc, rewriter);
-  Value zero = lb.create<arith::ConstantIndexOp>(0);
-  Value one = lb.create<arith::ConstantIndexOp>(1);
+  Value zero = arith::ConstantIndexOp::create(lb, 0);
+  Value one = arith::ConstantIndexOp::create(lb, 1);
   Type indexTy = lb.getIndexType();
 
   // Save all the ranks for bounds checking. Because this is a tensor
@@ -254,25 +254,25 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
   // dimension in the tensor.
   SmallVector<Value> ranks, rankDiffs;
   llvm::append_range(ranks, llvm::map_range(adaptor.getShapes(), [&](Value v) {
-                       return lb.create<tensor::DimOp>(v, zero);
+                       return tensor::DimOp::create(lb, v, zero);
                      }));
 
   // Find the maximum rank
   Value maxRank = ranks.front();
   for (Value v : llvm::drop_begin(ranks, 1)) {
-    maxRank = lb.create<arith::MaxUIOp>(v, maxRank);
+    maxRank = arith::MaxUIOp::create(lb, v, maxRank);
   }
 
   // Calculate the difference of ranks and the maximum rank for later offsets.
   llvm::append_range(rankDiffs, llvm::map_range(ranks, [&](Value v) {
-                       return lb.create<arith::SubIOp>(indexTy, maxRank, v);
+                       return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
   Type i1Ty = rewriter.getI1Type();
   Value trueVal =
-      rewriter.create<arith::ConstantOp>(loc, i1Ty, rewriter.getBoolAttr(true));
+      arith::ConstantOp::create(rewriter, loc, i1Ty, rewriter.getBoolAttr(true));
 
-  auto reduceResult = lb.create<ForOp>(
+  auto reduceResult = ForOp::create(lb,
       loc, zero, maxRank, one, ValueRange{trueVal},
       [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
         // Find a non-1 dim, if it exists. Note that the first part of this
@@ -285,38 +285,38 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
         for (auto tup : llvm::zip(adaptor.getShapes(), rankDiffs)) {
           Value shape, rankDiff;
           std::tie(shape, rankDiff) = tup;
-          Value outOfBounds = b.create<arith::CmpIOp>(
+          Value outOfBounds = arith::CmpIOp::create(b,
               loc, arith::CmpIPredicate::ult, iv, rankDiff);
           broadcastable =
-              b.create<IfOp>(
+              IfOp::create(b,
                    loc, outOfBounds,
                    [&](OpBuilder &b, Location loc) {
                      // Non existent dimensions are always broadcastable
-                     b.create<scf::YieldOp>(loc, broadcastable);
+                     scf::YieldOp::create(b, loc, broadcastable);
                    },
                    [&](OpBuilder &b, Location loc) {
                      // Every value needs to be either 1, or the same non-1
                      // value to be broadcastable in this dim.
                      Value operandDimension =
-                         b.create<arith::SubIOp>(loc, indexTy, iv, rankDiff);
-                     Value dimensionExtent = b.create<tensor::ExtractOp>(
+                         arith::SubIOp::create(b, loc, indexTy, iv, rankDiff);
+                     Value dimensionExtent = tensor::ExtractOp::create(b,
                          loc, shape, ValueRange{operandDimension});
 
-                     Value equalOne = b.create<arith::CmpIOp>(
+                     Value equalOne = arith::CmpIOp::create(b,
                          loc, arith::CmpIPredicate::eq, dimensionExtent, one);
-                     Value equalBroadcasted = b.create<arith::CmpIOp>(
+                     Value equalBroadcasted = arith::CmpIOp::create(b,
                          loc, arith::CmpIPredicate::eq, dimensionExtent,
                          broadcastedDim);
-                     Value result = b.create<arith::AndIOp>(
+                     Value result = arith::AndIOp::create(b,
                          loc, broadcastable,
-                         b.create<arith::OrIOp>(loc, equalOne,
+                         arith::OrIOp::create(b, loc, equalOne,
                                                 equalBroadcasted));
-                     b.create<scf::YieldOp>(loc, result);
+                     scf::YieldOp::create(b, loc, result);
                    })
                   .getResult(0);
         }
 
-        b.create<scf::YieldOp>(loc, broadcastable);
+        scf::YieldOp::create(b, loc, broadcastable);
       });
 
   rewriter.replaceOp(op, reduceResult.getResults().front());
@@ -339,7 +339,7 @@ DimOpConverter::matchAndRewrite(DimOp op, OpAdaptor adaptor,
   // Lower to dim(X, i) to get_extent(shape_of(X), i) and rely on further
   // lowerings. This can be further optimized if needed to avoid intermediate
   // steps.
-  auto shapeOf = rewriter.create<shape::ShapeOfOp>(op.getLoc(), op.getValue());
+  auto shapeOf = shape::ShapeOfOp::create(rewriter, op.getLoc(), op.getValue());
   rewriter.replaceOpWithNewOp<shape::GetExtentOp>(op, op.getType(), shapeOf,
                                                   op.getIndex());
   return success();
@@ -421,16 +421,16 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
 
   auto loc = op.getLoc();
 
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
   Type indexTy = rewriter.getIndexType();
   Value rank =
-      rewriter.create<tensor::DimOp>(loc, indexTy, adaptor.getShape(), zero);
+      tensor::DimOp::create(rewriter, loc, indexTy, adaptor.getShape(), zero);
 
-  auto loop = rewriter.create<scf::ForOp>(
+  auto loop = scf::ForOp::create(rewriter,
       loc, zero, rank, one, op.getInitVals(),
       [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
-        Value extent = b.create<tensor::ExtractOp>(loc, adaptor.getShape(), iv);
+        Value extent = tensor::ExtractOp::create(b, loc, adaptor.getShape(), iv);
 
         SmallVector<Value, 2> mappedValues{iv, extent};
         mappedValues.append(args.begin(), args.end());
@@ -444,7 +444,7 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
         SmallVector<Value, 2> mappedResults;
         for (auto result : reduceBody->getTerminator()->getOperands())
           mappedResults.push_back(mapping.lookup(result));
-        b.create<scf::YieldOp>(loc, mappedResults);
+        scf::YieldOp::create(b, loc, mappedResults);
       });
 
   rewriter.replaceOp(op, loop.getResults());
@@ -507,43 +507,43 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, OpAdaptor adaptor,
 
   auto loc = op.getLoc();
   Type indexTy = rewriter.getIndexType();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   Value firstShape = adaptor.getShapes().front();
   Value firstRank =
-      rewriter.create<tensor::DimOp>(loc, indexTy, firstShape, zero);
+      tensor::DimOp::create(rewriter, loc, indexTy, firstShape, zero);
   Value result = nullptr;
   // Generate a linear sequence of compares, all with firstShape as lhs.
   for (Value shape : adaptor.getShapes().drop_front(1)) {
-    Value rank = rewriter.create<tensor::DimOp>(loc, indexTy, shape, zero);
-    Value eqRank = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value rank = tensor::DimOp::create(rewriter, loc, indexTy, shape, zero);
+    Value eqRank = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                   firstRank, rank);
-    auto same = rewriter.create<IfOp>(
+    auto same = IfOp::create(rewriter,
         loc, eqRank,
         [&](OpBuilder &b, Location loc) {
-          Value one = b.create<arith::ConstantIndexOp>(loc, 1);
+          Value one = arith::ConstantIndexOp::create(b, loc, 1);
           Value init =
-              b.create<arith::ConstantOp>(loc, i1Ty, b.getBoolAttr(true));
-          auto loop = b.create<scf::ForOp>(
+              arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(true));
+          auto loop = scf::ForOp::create(b,
               loc, zero, firstRank, one, ValueRange{init},
               [&](OpBuilder &b, Location nestedLoc, Value iv, ValueRange args) {
                 Value conj = args[0];
                 Value lhsExtent =
-                    b.create<tensor::ExtractOp>(loc, firstShape, iv);
-                Value rhsExtent = b.create<tensor::ExtractOp>(loc, shape, iv);
-                Value eqExtent = b.create<arith::CmpIOp>(
+                    tensor::ExtractOp::create(b, loc, firstShape, iv);
+                Value rhsExtent = tensor::ExtractOp::create(b, loc, shape, iv);
+                Value eqExtent = arith::CmpIOp::create(b,
                     loc, arith::CmpIPredicate::eq, lhsExtent, rhsExtent);
-                Value conjNext = b.create<arith::AndIOp>(loc, conj, eqExtent);
-                b.create<scf::YieldOp>(loc, ValueRange({conjNext}));
+                Value conjNext = arith::AndIOp::create(b, loc, conj, eqExtent);
+                scf::YieldOp::create(b, loc, ValueRange({conjNext}));
               });
-          b.create<scf::YieldOp>(loc, loop.getResults());
+          scf::YieldOp::create(b, loc, loop.getResults());
         },
         [&](OpBuilder &b, Location loc) {
           Value result =
-              b.create<arith::ConstantOp>(loc, i1Ty, b.getBoolAttr(false));
-          b.create<scf::YieldOp>(loc, result);
+              arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(false));
+          scf::YieldOp::create(b, loc, result);
         });
     result = !result ? same.getResult(0)
-                     : rewriter.create<arith::AndIOp>(loc, result,
+                     : arith::AndIOp::create(rewriter, loc, result,
                                                       same.getResult(0));
   }
   rewriter.replaceOp(op, result);
@@ -581,17 +581,17 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
     int64_t rank = rankedTensorTy.getRank();
     for (int64_t i = 0; i < rank; i++) {
       if (rankedTensorTy.isDynamicDim(i)) {
-        Value extent = rewriter.create<tensor::DimOp>(loc, tensor, i);
+        Value extent = tensor::DimOp::create(rewriter, loc, tensor, i);
         extentValues.push_back(extent);
       } else {
-        Value extent = rewriter.create<arith::ConstantIndexOp>(
+        Value extent = arith::ConstantIndexOp::create(rewriter,
             loc, rankedTensorTy.getDimSize(i));
         extentValues.push_back(extent);
       }
     }
 
     // Materialize extent tensor.
-    Value staticExtentTensor = rewriter.create<tensor::FromElementsOp>(
+    Value staticExtentTensor = tensor::FromElementsOp::create(rewriter,
         loc, RankedTensorType::get({rank}, rewriter.getIndexType()),
         extentValues);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(),
@@ -601,13 +601,13 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
 
   // Lower to `tensor.generate` otherwise.
   auto *ctx = rewriter.getContext();
-  Value rank = rewriter.create<tensor::RankOp>(loc, tensor);
+  Value rank = tensor::RankOp::create(rewriter, loc, tensor);
   rewriter.replaceOpWithNewOp<tensor::GenerateOp>(
       op, getExtentTensorType(ctx), ValueRange{rank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value dim = args.front();
-        Value extent = b.create<tensor::DimOp>(loc, tensor, dim);
-        b.create<tensor::YieldOp>(loc, extent);
+        Value extent = tensor::DimOp::create(b, loc, tensor, dim);
+        tensor::YieldOp::create(b, loc, extent);
       });
 
   return success();
@@ -634,21 +634,21 @@ LogicalResult SplitAtOpConversion::matchAndRewrite(
     return failure();
 
   ImplicitLocOpBuilder b(op.getLoc(), rewriter);
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value rank = b.create<tensor::DimOp>(adaptor.getOperand(), zero);
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value rank = tensor::DimOp::create(b, adaptor.getOperand(), zero);
 
   // index < 0 ? index + rank : index
   Value originalIndex = adaptor.getIndex();
-  Value add = b.create<arith::AddIOp>(originalIndex, rank);
+  Value add = arith::AddIOp::create(b, originalIndex, rank);
   Value indexIsNegative =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::slt, originalIndex, zero);
-  Value index = b.create<arith::SelectOp>(indexIsNegative, add, originalIndex);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, originalIndex, zero);
+  Value index = arith::SelectOp::create(b, indexIsNegative, add, originalIndex);
 
-  Value one = b.create<arith::ConstantIndexOp>(1);
+  Value one = arith::ConstantIndexOp::create(b, 1);
   Value head =
-      b.create<tensor::ExtractSliceOp>(adaptor.getOperand(), zero, index, one);
-  Value tailSize = b.create<arith::SubIOp>(rank, index);
-  Value tail = b.create<tensor::ExtractSliceOp>(adaptor.getOperand(), index,
+      tensor::ExtractSliceOp::create(b, adaptor.getOperand(), zero, index, one);
+  Value tailSize = arith::SubIOp::create(b, rank, index);
+  Value tail = tensor::ExtractSliceOp::create(b, adaptor.getOperand(), index,
                                                 tailSize, one);
   rewriter.replaceOp(op, {head, tail});
   return success();
diff --git a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
index 2c4d27502a521..f06c3e1f7cb18 100644
--- a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
+++ b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
@@ -68,10 +68,10 @@ class TensorExtractPattern final
       // We could use the initializer directly; but certain driver compilers
       // have bugs dealing with that. So for now, use spirv.Store for
       // initialization.
-      varOp = rewriter.create<spirv::VariableOp>(loc, varType,
+      varOp = spirv::VariableOp::create(rewriter, loc, varType,
                                                  spirv::StorageClass::Function,
                                                  /*initializer=*/nullptr);
-      rewriter.create<spirv::StoreOp>(loc, varOp, adaptor.getTensor());
+      spirv::StoreOp::create(rewriter, loc, varOp, adaptor.getTensor());
     } else {
       // Need to store the value to the local variable. It's questionable
       // whether we want to support such case though.
@@ -83,7 +83,7 @@ class TensorExtractPattern final
 
     Value index = spirv::linearizeIndex(adaptor.getIndices(), strides,
                                         /*offset=*/0, indexType, loc, rewriter);
-    auto acOp = rewriter.create<spirv::AccessChainOp>(loc, varOp, index);
+    auto acOp = spirv::AccessChainOp::create(rewriter, loc, varOp, index);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(extractOp, acOp);
 
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 40ad63610e23f..44872c8d7540d 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -51,7 +51,7 @@ TypedAttr getConstantAttr(Type type, int64_t value, PatternRewriter &rewriter) {
 
 Value getConstantValue(Location loc, Type type, int64_t value,
                        PatternRewriter &rewriter) {
-  return rewriter.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(rewriter,
       loc, getConstantAttr(type, value, rewriter));
 }
 
@@ -82,41 +82,41 @@ class ApplyScaleGenericOpConverter
     Value one64 = getConstantValue(loc, i64Ty, 1, rewriter);
     Value thirtyOne32 = getConstantValue(loc, i32Ty, 31, rewriter);
 
-    Value shift32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, op.getShift());
+    Value shift32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, op.getShift());
 
     // Compute the multiplication in 64-bits then select the high / low parts.
     Value value64 = value;
     if (getElementTypeOrSelf(valueTy) != rewriter.getI64Type())
-      value64 = rewriter.create<arith::ExtSIOp>(loc, i64Ty, value);
+      value64 = arith::ExtSIOp::create(rewriter, loc, i64Ty, value);
     Value multiplier64 =
-        rewriter.create<arith::ExtSIOp>(loc, i64Ty, multiplier32);
+        arith::ExtSIOp::create(rewriter, loc, i64Ty, multiplier32);
     Value multiply64 =
-        rewriter.create<arith::MulIOp>(loc, value64, multiplier64);
+        arith::MulIOp::create(rewriter, loc, value64, multiplier64);
 
     // Apply normal rounding.
-    Value shift64 = rewriter.create<arith::ExtUIOp>(loc, i64Ty, shift32);
-    Value round = rewriter.create<arith::ShLIOp>(loc, one64, shift64);
-    round = rewriter.create<arith::ShRUIOp>(loc, round, one64);
-    multiply64 = rewriter.create<arith::AddIOp>(loc, multiply64, round);
+    Value shift64 = arith::ExtUIOp::create(rewriter, loc, i64Ty, shift32);
+    Value round = arith::ShLIOp::create(rewriter, loc, one64, shift64);
+    round = arith::ShRUIOp::create(rewriter, loc, round, one64);
+    multiply64 = arith::AddIOp::create(rewriter, loc, multiply64, round);
 
     // Apply double rounding if necessary.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       int64_t roundInt = 1 << 30;
       Value roundUp = getConstantValue(loc, i64Ty, roundInt, rewriter);
       Value roundDown = getConstantValue(loc, i64Ty, -roundInt, rewriter);
-      Value positive = rewriter.create<arith::CmpIOp>(
+      Value positive = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, value, zero);
       Value dir =
-          rewriter.create<arith::SelectOp>(loc, positive, roundUp, roundDown);
-      Value val = rewriter.create<arith::AddIOp>(loc, dir, multiply64);
-      Value valid = rewriter.create<arith::CmpIOp>(
+          arith::SelectOp::create(rewriter, loc, positive, roundUp, roundDown);
+      Value val = arith::AddIOp::create(rewriter, loc, dir, multiply64);
+      Value valid = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sgt, shift32, thirtyOne32);
       multiply64 =
-          rewriter.create<arith::SelectOp>(loc, valid, val, multiply64);
+          arith::SelectOp::create(rewriter, loc, valid, val, multiply64);
     }
 
-    Value result64 = rewriter.create<arith::ShRSIOp>(loc, multiply64, shift64);
-    Value result32 = rewriter.create<arith::TruncIOp>(loc, i32Ty, result64);
+    Value result64 = arith::ShRSIOp::create(rewriter, loc, multiply64, shift64);
+    Value result32 = arith::TruncIOp::create(rewriter, loc, i32Ty, result64);
 
     rewriter.replaceOp(op, result32);
     return success();
@@ -146,7 +146,7 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
 
     Value value32 = op.getValue();
     Value multiplier32 = op.getMultiplier();
-    Value shift32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, op.getShift());
+    Value shift32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, op.getShift());
 
     // Constants used during the scaling operation.
     Value zero32 = getConstantValue(loc, i32Ty, 0, rewriter);
@@ -158,86 +158,86 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
     // Compute the multiplication in 64-bits then select the high / low parts.
     // Grab out the high/low of the computation
     auto value64 =
-        rewriter.create<arith::MulSIExtendedOp>(loc, value32, multiplier32);
+        arith::MulSIExtendedOp::create(rewriter, loc, value32, multiplier32);
     Value low32 = value64.getLow();
     Value high32 = value64.getHigh();
 
     // Determine the direction and amount to shift the high bits.
-    Value shiftOver32 = rewriter.create<arith::CmpIOp>(
+    Value shiftOver32 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, shift32, thirtyTwo32);
-    Value roundHighBits = rewriter.create<arith::CmpIOp>(
+    Value roundHighBits = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, shift32, thirtyTwo32);
 
     Value shiftHighL =
-        rewriter.create<arith::SubIOp>(loc, thirtyTwo32, shift32);
+        arith::SubIOp::create(rewriter, loc, thirtyTwo32, shift32);
     Value shiftHighR =
-        rewriter.create<arith::SubIOp>(loc, shift32, thirtyTwo32);
+        arith::SubIOp::create(rewriter, loc, shift32, thirtyTwo32);
 
     shiftHighL =
-        rewriter.create<arith::SelectOp>(loc, shiftOver32, zero32, shiftHighL);
+        arith::SelectOp::create(rewriter, loc, shiftOver32, zero32, shiftHighL);
     shiftHighR =
-        rewriter.create<arith::SelectOp>(loc, shiftOver32, shiftHighR, zero32);
+        arith::SelectOp::create(rewriter, loc, shiftOver32, shiftHighR, zero32);
 
     // Conditionally perform our double round.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       Value negOne32 = getConstantValue(loc, i32Ty, -1, rewriter);
-      Value valuePositive = rewriter.create<arith::CmpIOp>(
+      Value valuePositive = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, value32, zero32);
 
       Value roundDir =
-          rewriter.create<arith::SelectOp>(loc, valuePositive, one32, negOne32);
+          arith::SelectOp::create(rewriter, loc, valuePositive, one32, negOne32);
       roundDir =
-          rewriter.create<arith::SelectOp>(loc, shiftOver32, roundDir, zero32);
+          arith::SelectOp::create(rewriter, loc, shiftOver32, roundDir, zero32);
 
-      Value shiftLow = rewriter.create<arith::ShRUIOp>(loc, low32, thirty32);
-      Value rounded = rewriter.create<arith::AddIOp>(loc, shiftLow, roundDir);
-      Value carry = rewriter.create<arith::ShRSIOp>(loc, rounded, two32);
+      Value shiftLow = arith::ShRUIOp::create(rewriter, loc, low32, thirty32);
+      Value rounded = arith::AddIOp::create(rewriter, loc, shiftLow, roundDir);
+      Value carry = arith::ShRSIOp::create(rewriter, loc, rounded, two32);
 
       Value shiftRound =
-          rewriter.create<arith::ShLIOp>(loc, roundDir, thirty32);
+          arith::ShLIOp::create(rewriter, loc, roundDir, thirty32);
 
-      low32 = rewriter.create<arith::AddIOp>(loc, low32, shiftRound);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, carry);
+      low32 = arith::AddIOp::create(rewriter, loc, low32, shiftRound);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, carry);
     }
 
     // Conditionally apply rounding in the low bits.
     {
-      Value shiftSubOne = rewriter.create<arith::SubIOp>(loc, shift32, one32);
-      Value roundBit = rewriter.create<arith::ShLIOp>(loc, one32, shiftSubOne);
-      roundBit = rewriter.create<arith::SelectOp>(loc, roundHighBits, zero32,
+      Value shiftSubOne = arith::SubIOp::create(rewriter, loc, shift32, one32);
+      Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
+      roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, zero32,
                                                   roundBit);
 
-      Value newLow32 = rewriter.create<arith::AddIOp>(loc, low32, roundBit);
-      Value wasRounded = rewriter.create<arith::CmpIOp>(
+      Value newLow32 = arith::AddIOp::create(rewriter, loc, low32, roundBit);
+      Value wasRounded = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ugt, low32, newLow32);
       low32 = newLow32;
 
-      Value rounded32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, wasRounded);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, rounded32);
+      Value rounded32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, wasRounded);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, rounded32);
     }
 
     // Conditionally apply rounding in the high bits.
     {
       Value shiftSubOne =
-          rewriter.create<arith::SubIOp>(loc, shiftHighR, one32);
-      Value roundBit = rewriter.create<arith::ShLIOp>(loc, one32, shiftSubOne);
-      roundBit = rewriter.create<arith::SelectOp>(loc, roundHighBits, roundBit,
+          arith::SubIOp::create(rewriter, loc, shiftHighR, one32);
+      Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
+      roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, roundBit,
                                                   zero32);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, roundBit);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, roundBit);
     }
 
     // Combine the correct high/low bits into the final rescale result.
-    high32 = rewriter.create<arith::ShLIOp>(loc, high32, shiftHighL);
-    high32 = rewriter.create<arith::ShRSIOp>(loc, high32, shiftHighR);
-    low32 = rewriter.create<arith::ShRUIOp>(loc, low32, shift32);
-    low32 = rewriter.create<arith::SelectOp>(loc, shiftOver32, zero32, low32);
+    high32 = arith::ShLIOp::create(rewriter, loc, high32, shiftHighL);
+    high32 = arith::ShRSIOp::create(rewriter, loc, high32, shiftHighR);
+    low32 = arith::ShRUIOp::create(rewriter, loc, low32, shift32);
+    low32 = arith::SelectOp::create(rewriter, loc, shiftOver32, zero32, low32);
 
     // Apply the rounding behavior and shift to the final alignment.
-    Value result = rewriter.create<arith::AddIOp>(loc, low32, high32);
+    Value result = arith::AddIOp::create(rewriter, loc, low32, high32);
 
     // Truncate if necessary.
     if (!getElementTypeOrSelf(resultTy).isInteger(32)) {
-      result = rewriter.create<arith::TruncIOp>(loc, resultTy, result);
+      result = arith::TruncIOp::create(rewriter, loc, resultTy, result);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 2f608bbd637b4..2aec35ed9d8ca 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -70,13 +70,13 @@ materializeBinaryNanCheckIfRequired(OpTy op, PatternRewriter &rewriter,
     return result;
 
   // Unordered comparison of NaN against itself will always return true.
-  Value lhsIsNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsIsNaN = arith::CmpFOp::create(rewriter,
       op.getLoc(), arith::CmpFPredicate::UNO, lhs, lhs);
-  Value rhsIsNaN = rewriter.create<arith::CmpFOp>(
+  Value rhsIsNaN = arith::CmpFOp::create(rewriter,
       op.getLoc(), arith::CmpFPredicate::UNO, rhs, rhs);
   Value rhsOrResult =
-      rewriter.create<arith::SelectOp>(op.getLoc(), lhsIsNaN, rhs, result);
-  return rewriter.create<arith::SelectOp>(op.getLoc(), rhsIsNaN, lhs,
+      arith::SelectOp::create(rewriter, op.getLoc(), lhsIsNaN, rhs, result);
+  return arith::SelectOp::create(rewriter, op.getLoc(), rhsIsNaN, lhs,
                                           rhsOrResult);
 }
 
@@ -89,38 +89,38 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
   // tosa::AbsOp
   if (isa<tosa::AbsOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::AbsFOp>(loc, resultTypes, args);
+    return math::AbsFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AbsOp>(op) && isa<IntegerType>(elementTy)) {
-    auto zero = rewriter.create<arith::ConstantOp>(
+    auto zero = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(elementTy));
-    auto neg = rewriter.create<arith::SubIOp>(loc, zero, args[0]);
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], neg);
+    auto neg = arith::SubIOp::create(rewriter, loc, zero, args[0]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], neg);
   }
 
   // tosa::AddOp
   if (isa<tosa::AddOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::AddFOp>(loc, resultTypes, args);
+    return arith::AddFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AddOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::AddIOp>(loc, resultTypes, args);
+    return arith::AddIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::SubOp
   if (isa<tosa::SubOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::SubFOp>(loc, resultTypes, args);
+    return arith::SubFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::SubOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::SubIOp>(loc, resultTypes, args);
+    return arith::SubIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::IntDivOp
   if (isa<tosa::IntDivOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::DivSIOp>(loc, resultTypes, args);
+    return arith::DivSIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ReciprocalOp
   if (isa<tosa::ReciprocalOp>(op) && isa<FloatType>(elementTy)) {
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, FloatAttr::get(elementTy, 1));
-    return rewriter.create<arith::DivFOp>(loc, resultTypes, one, args[0]);
+        arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
+    return arith::DivFOp::create(rewriter, loc, resultTypes, one, args[0]);
   }
 
   // tosa::MulOp
@@ -140,7 +140,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                                           "Cannot have shift value for float");
         return nullptr;
       }
-      return rewriter.create<arith::MulFOp>(loc, resultTypes, args[0], args[1]);
+      return arith::MulFOp::create(rewriter, loc, resultTypes, args[0], args[1]);
     }
 
     if (isa<IntegerType>(elementTy)) {
@@ -149,21 +149,21 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
       if (shift > 0) {
         auto shiftConst =
-            rewriter.create<arith::ConstantIntOp>(loc, shift, /*bitwidth=*/8);
+            arith::ConstantIntOp::create(rewriter, loc, shift, /*bitwidth=*/8);
         if (!a.getType().isInteger(32))
-          a = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), a);
+          a = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), a);
 
         if (!b.getType().isInteger(32))
-          b = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), b);
+          b = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), b);
 
-        auto result = rewriter.create<tosa::ApplyScaleOp>(
+        auto result = tosa::ApplyScaleOp::create(rewriter,
             loc, rewriter.getI32Type(), a, b, shiftConst,
             rewriter.getStringAttr("SINGLE_ROUND"));
 
         if (elementTy.isInteger(32))
           return result;
 
-        return rewriter.create<arith::TruncIOp>(loc, elementTy, result);
+        return arith::TruncIOp::create(rewriter, loc, elementTy, result);
       }
 
       int aWidth = a.getType().getIntOrFloatBitWidth();
@@ -171,11 +171,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       int cWidth = resultTypes[0].getIntOrFloatBitWidth();
 
       if (aWidth < cWidth)
-        a = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], a);
+        a = arith::ExtSIOp::create(rewriter, loc, resultTypes[0], a);
       if (bWidth < cWidth)
-        b = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], b);
+        b = arith::ExtSIOp::create(rewriter, loc, resultTypes[0], b);
 
-      return rewriter.create<arith::MulIOp>(loc, resultTypes, a, b);
+      return arith::MulIOp::create(rewriter, loc, resultTypes, a, b);
     }
   }
 
@@ -201,13 +201,13 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     int64_t outZp = *maybeOutZp;
 
     if (isa<FloatType>(elementTy))
-      return rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
+      return arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
 
     if (isa<IntegerType>(elementTy)) {
       if (!inZp && !outZp) {
-        auto constant = rewriter.create<arith::ConstantOp>(
+        auto constant = arith::ConstantOp::create(rewriter,
             loc, IntegerAttr::get(elementTy, 0));
-        return rewriter.create<arith::SubIOp>(loc, resultTypes, constant,
+        return arith::SubIOp::create(rewriter, loc, resultTypes, constant,
                                               args[0]);
       }
 
@@ -231,60 +231,60 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       }
 
       Type intermediateType = rewriter.getIntegerType(intermediateBitWidth);
-      Value zpAddValue = rewriter.create<arith::ConstantOp>(
+      Value zpAddValue = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
 
       // The negation can be applied by doing:
       //  outputValue = inZp + outZp - inputValue
       auto ext =
-          rewriter.create<arith::ExtSIOp>(loc, intermediateType, args[0]);
-      auto sub = rewriter.create<arith::SubIOp>(loc, zpAddValue, ext);
+          arith::ExtSIOp::create(rewriter, loc, intermediateType, args[0]);
+      auto sub = arith::SubIOp::create(rewriter, loc, zpAddValue, ext);
 
       // Clamp to the negation range.
-      Value min = rewriter.create<arith::ConstantIntOp>(
+      Value min = arith::ConstantIntOp::create(rewriter,
           loc, intermediateType,
           APInt::getSignedMinValue(inputBitWidth).getSExtValue());
-      Value max = rewriter.create<arith::ConstantIntOp>(
+      Value max = arith::ConstantIntOp::create(rewriter,
           loc, intermediateType,
           APInt::getSignedMaxValue(inputBitWidth).getSExtValue());
       auto clamp = clampIntHelper(loc, sub, min, max, rewriter, false);
 
       // Truncate to the final value.
-      return rewriter.create<arith::TruncIOp>(loc, elementTy, clamp);
+      return arith::TruncIOp::create(rewriter, loc, elementTy, clamp);
     }
   }
 
   // tosa::BitwiseAndOp
   if (isa<tosa::BitwiseAndOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::AndIOp>(loc, resultTypes, args);
+    return arith::AndIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::BitwiseOrOp
   if (isa<tosa::BitwiseOrOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::OrIOp>(loc, resultTypes, args);
+    return arith::OrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::BitwiseNotOp
   if (isa<tosa::BitwiseNotOp>(op) && isa<IntegerType>(elementTy)) {
     auto allOnesAttr = rewriter.getIntegerAttr(
         elementTy, APInt::getAllOnes(elementTy.getIntOrFloatBitWidth()));
-    auto allOnes = rewriter.create<arith::ConstantOp>(loc, allOnesAttr);
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args[0], allOnes);
+    auto allOnes = arith::ConstantOp::create(rewriter, loc, allOnesAttr);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], allOnes);
   }
 
   // tosa::BitwiseXOrOp
   if (isa<tosa::BitwiseXorOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalLeftShiftOp
   if (isa<tosa::LogicalLeftShiftOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::ShLIOp>(loc, resultTypes, args);
+    return arith::ShLIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalRightShiftOp
   if (isa<tosa::LogicalRightShiftOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::ShRUIOp>(loc, resultTypes, args);
+    return arith::ShRUIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ArithmeticRightShiftOp
   if (isa<tosa::ArithmeticRightShiftOp>(op) && isa<IntegerType>(elementTy)) {
-    auto result = rewriter.create<arith::ShRSIOp>(loc, resultTypes, args);
+    auto result = arith::ShRSIOp::create(rewriter, loc, resultTypes, args);
     auto round = cast<BoolAttr>(op->getAttr("round")).getValue();
     if (!round) {
       return result;
@@ -292,153 +292,153 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     Type i1Ty = IntegerType::get(rewriter.getContext(), /*width=*/1);
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(elementTy, 1));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 1));
     auto zero =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(elementTy, 0));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 0));
     auto i1one =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(i1Ty, 1));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(i1Ty, 1));
 
     // Checking that input2 != 0
-    auto shiftValueGreaterThanZero = rewriter.create<arith::CmpIOp>(
+    auto shiftValueGreaterThanZero = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, args[1], zero);
 
     // Checking for the last bit of input1 to be 1
     auto subtract =
-        rewriter.create<arith::SubIOp>(loc, resultTypes, args[1], one);
+        arith::SubIOp::create(rewriter, loc, resultTypes, args[1], one);
     auto shifted =
-        rewriter.create<arith::ShRSIOp>(loc, resultTypes, args[0], subtract)
+        arith::ShRSIOp::create(rewriter, loc, resultTypes, args[0], subtract)
             ->getResults();
-    auto truncated = rewriter.create<arith::TruncIOp>(
+    auto truncated = arith::TruncIOp::create(rewriter,
         loc, i1Ty, shifted, ArrayRef<NamedAttribute>());
     auto isInputOdd =
-        rewriter.create<arith::AndIOp>(loc, i1Ty, truncated, i1one);
+        arith::AndIOp::create(rewriter, loc, i1Ty, truncated, i1one);
 
-    auto shouldRound = rewriter.create<arith::AndIOp>(
+    auto shouldRound = arith::AndIOp::create(rewriter,
         loc, i1Ty, shiftValueGreaterThanZero, isInputOdd);
     auto extended =
-        rewriter.create<arith::ExtUIOp>(loc, resultTypes, shouldRound);
-    return rewriter.create<arith::AddIOp>(loc, resultTypes, result, extended);
+        arith::ExtUIOp::create(rewriter, loc, resultTypes, shouldRound);
+    return arith::AddIOp::create(rewriter, loc, resultTypes, result, extended);
   }
 
   // tosa::ClzOp
   if (isa<tosa::ClzOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<math::CountLeadingZerosOp>(loc, elementTy, args[0]);
+    return math::CountLeadingZerosOp::create(rewriter, loc, elementTy, args[0]);
   }
 
   // tosa::LogicalAnd
   if (isa<tosa::LogicalAndOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::AndIOp>(loc, resultTypes, args);
+    return arith::AndIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalNot
   if (isa<tosa::LogicalNotOp>(op) && elementTy.isInteger(1)) {
-    auto one = rewriter.create<arith::ConstantOp>(
+    auto one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(elementTy, 1));
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args[0], one);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], one);
   }
 
   // tosa::LogicalOr
   if (isa<tosa::LogicalOrOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::OrIOp>(loc, resultTypes, args);
+    return arith::OrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalXor
   if (isa<tosa::LogicalXorOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::PowOp
   if (isa<tosa::PowOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::PowFOp>(loc, resultTypes, args);
+    return mlir::math::PowFOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::RsqrtOp
   if (isa<tosa::RsqrtOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::RsqrtOp>(loc, resultTypes, args);
+    return mlir::math::RsqrtOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogOp
   if (isa<tosa::LogOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::LogOp>(loc, resultTypes, args);
+    return mlir::math::LogOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ExpOp
   if (isa<tosa::ExpOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::ExpOp>(loc, resultTypes, args);
+    return mlir::math::ExpOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::SinOp
   if (isa<tosa::SinOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::SinOp>(loc, resultTypes, args);
+    return mlir::math::SinOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::CosOp
   if (isa<tosa::CosOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::CosOp>(loc, resultTypes, args);
+    return mlir::math::CosOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::TanhOp
   if (isa<tosa::TanhOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::TanhOp>(loc, resultTypes, args);
+    return mlir::math::TanhOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ErfOp
   if (isa<tosa::ErfOp>(op) && llvm::isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::ErfOp>(loc, resultTypes, args);
+    return mlir::math::ErfOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::GreaterOp
   if (isa<tosa::GreaterOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGT,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGT,
                                           args[0], args[1]);
 
   if (isa<tosa::GreaterOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sgt,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sgt,
                                           args[0], args[1]);
 
   // tosa::GreaterEqualOp
   if (isa<tosa::GreaterEqualOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGE,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGE,
                                           args[0], args[1]);
 
   if (isa<tosa::GreaterEqualOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sge,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sge,
                                           args[0], args[1]);
 
   // tosa::EqualOp
   if (isa<tosa::EqualOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OEQ,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OEQ,
                                           args[0], args[1]);
 
   if (isa<tosa::EqualOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                           args[0], args[1]);
 
   // tosa::SelectOp
   if (isa<tosa::SelectOp>(op)) {
     elementTy = cast<ShapedType>(op->getOperand(1).getType()).getElementType();
     if (isa<FloatType>(elementTy) || isa<IntegerType>(elementTy))
-      return rewriter.create<arith::SelectOp>(loc, args[0], args[1], args[2]);
+      return arith::SelectOp::create(rewriter, loc, args[0], args[1], args[2]);
   }
 
   // tosa::MaximumOp
   if (isa<tosa::MaximumOp>(op) && isa<FloatType>(elementTy)) {
-    auto max = rewriter.create<arith::MaximumFOp>(loc, args[0], args[1]);
+    auto max = arith::MaximumFOp::create(rewriter, loc, args[0], args[1]);
     return materializeBinaryNanCheckIfRequired(llvm::cast<tosa::MaximumOp>(op),
                                                rewriter, args[0], args[1], max);
   }
 
   if (isa<tosa::MaximumOp>(op) && elementTy.isSignlessInteger()) {
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], args[1]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   // tosa::MinimumOp
   if (isa<tosa::MinimumOp>(op) && isa<FloatType>(elementTy)) {
-    auto min = rewriter.create<arith::MinimumFOp>(loc, args[0], args[1]);
+    auto min = arith::MinimumFOp::create(rewriter, loc, args[0], args[1]);
     return materializeBinaryNanCheckIfRequired(llvm::cast<tosa::MinimumOp>(op),
                                                rewriter, args[0], args[1], min);
   }
 
   if (isa<tosa::MinimumOp>(op) && elementTy.isSignlessInteger()) {
-    return rewriter.create<arith::MinSIOp>(loc, args[0], args[1]);
+    return arith::MinSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   // tosa::CeilOp
   if (isa<tosa::CeilOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::CeilOp>(loc, resultTypes, args);
+    return math::CeilOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::FloorOp
   if (isa<tosa::FloorOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::FloorOp>(loc, resultTypes, args);
+    return math::FloorOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ClampOp
   if (isa<tosa::ClampOp>(op) && isa<FloatType>(elementTy)) {
@@ -449,9 +449,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                    APFloat::rmNearestTiesToEven, &losesInfo);
     maxApf.convert(cast<FloatType>(elementTy).getFloatSemantics(),
                    APFloat::rmNearestTiesToEven, &losesInfo);
-    auto min = rewriter.create<arith::ConstantOp>(
+    auto min = arith::ConstantOp::create(rewriter,
         loc, elementTy, rewriter.getFloatAttr(elementTy, minApf));
-    auto max = rewriter.create<arith::ConstantOp>(
+    auto max = arith::ConstantOp::create(rewriter,
         loc, elementTy, rewriter.getFloatAttr(elementTy, maxApf));
     auto result = clampFloatHelper(loc, args[0], min, max, rewriter);
 
@@ -478,11 +478,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     //   return init if x == NaN else result
 
     // Unordered comparison of NaN against itself will always return true.
-    Value isNaN = rewriter.create<arith::CmpFOp>(
+    Value isNaN = arith::CmpFOp::create(rewriter,
         op->getLoc(), arith::CmpFPredicate::UNO, args[0], args[0]);
     // TOSA specifies that in "ignore" NaN mode the result is "min" if the input
     // is NaN.
-    return rewriter.create<arith::SelectOp>(op->getLoc(), isNaN, min, result);
+    return arith::SelectOp::create(rewriter, op->getLoc(), isNaN, min, result);
   }
 
   if (isa<tosa::ClampOp>(op) && isa<IntegerType>(elementTy)) {
@@ -515,9 +515,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     min = std::min(min, maxRepresentable);
     max = std::min(max, maxRepresentable);
 
-    auto minVal = rewriter.create<arith::ConstantIntOp>(
+    auto minVal = arith::ConstantIntOp::create(rewriter,
         loc, min, intTy.getIntOrFloatBitWidth());
-    auto maxVal = rewriter.create<arith::ConstantIntOp>(
+    auto maxVal = arith::ConstantIntOp::create(rewriter,
         loc, max, intTy.getIntOrFloatBitWidth());
     return clampIntHelper(loc, args[0], minVal, maxVal, rewriter,
                           intTy.isUnsignedInteger());
@@ -526,11 +526,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   // tosa::SigmoidOp
   if (isa<tosa::SigmoidOp>(op) && isa<FloatType>(elementTy)) {
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, FloatAttr::get(elementTy, 1));
-    auto negate = rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
-    auto exp = rewriter.create<mlir::math::ExpOp>(loc, resultTypes, negate);
-    auto added = rewriter.create<arith::AddFOp>(loc, resultTypes, exp, one);
-    return rewriter.create<arith::DivFOp>(loc, resultTypes, one, added);
+        arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
+    auto negate = arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
+    auto exp = mlir::math::ExpOp::create(rewriter, loc, resultTypes, negate);
+    auto added = arith::AddFOp::create(rewriter, loc, resultTypes, exp, one);
+    return arith::DivFOp::create(rewriter, loc, resultTypes, one, added);
   }
 
   // tosa::CastOp
@@ -549,20 +549,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       return args.front();
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtFOp>(loc, resultTypes, args,
+      return arith::ExtFOp::create(rewriter, loc, resultTypes, args,
                                             ArrayRef<NamedAttribute>());
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && !bitExtend)
-      return rewriter.create<arith::TruncFOp>(loc, resultTypes, args,
+      return arith::TruncFOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     // 1-bit integers need to be treated as signless.
     if (srcTy.isInteger(1) && arith::UIToFPOp::areCastCompatible(srcTy, dstTy))
-      return rewriter.create<arith::UIToFPOp>(loc, resultTypes, args,
+      return arith::UIToFPOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     if (srcTy.isInteger(1) && isa<IntegerType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtUIOp>(loc, resultTypes, args,
+      return arith::ExtUIOp::create(rewriter, loc, resultTypes, args,
                                              ArrayRef<NamedAttribute>());
 
     // Unsigned integers need an unrealized cast so that they can be passed
@@ -574,25 +574,25 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                   loc, rewriter.getIntegerType(srcTy.getIntOrFloatBitWidth()),
                   args[0])
               .getResult(0);
-      return rewriter.create<arith::UIToFPOp>(loc, resultTypes[0],
+      return arith::UIToFPOp::create(rewriter, loc, resultTypes[0],
                                               unrealizedCast);
     }
 
     // All other si-to-fp conversions should be handled by SIToFP.
     if (arith::SIToFPOp::areCastCompatible(srcTy, dstTy))
-      return rewriter.create<arith::SIToFPOp>(loc, resultTypes, args,
+      return arith::SIToFPOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     // Casting to boolean, floats need to only be checked as not-equal to zero.
     if (isa<FloatType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = rewriter.create<arith::ConstantOp>(
+      Value zero = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(srcTy, 0.0));
-      return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNE,
+      return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNE,
                                             args.front(), zero);
     }
 
     if (arith::FPToSIOp::areCastCompatible(srcTy, dstTy)) {
-      auto rounded = rewriter.create<math::RoundEvenOp>(loc, args[0]);
+      auto rounded = math::RoundEvenOp::create(rewriter, loc, args[0]);
 
       const auto &fltSemantics = cast<FloatType>(srcTy).getFloatSemantics();
       // Check whether neither int min nor int max can be represented in the
@@ -601,33 +601,33 @@ static Value createLinalgBodyCalculationForElementwiseOp(
           APFloat::semanticsMaxExponent(fltSemantics)) {
         // Use cmp + select to replace infinites by int min / int max. Other
         // integral values can be represented in the integer space.
-        auto conv = rewriter.create<arith::FPToSIOp>(loc, dstTy, rounded);
-        auto posInf = rewriter.create<arith::ConstantOp>(
+        auto conv = arith::FPToSIOp::create(rewriter, loc, dstTy, rounded);
+        auto posInf = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(getElementTypeOrSelf(srcTy),
                                        APFloat::getInf(fltSemantics)));
-        auto negInf = rewriter.create<arith::ConstantOp>(
+        auto negInf = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(
                      getElementTypeOrSelf(srcTy),
                      APFloat::getInf(fltSemantics, /*Negative=*/true)));
-        auto overflow = rewriter.create<arith::CmpFOp>(
+        auto overflow = arith::CmpFOp::create(rewriter,
             loc, arith::CmpFPredicate::UEQ, rounded, posInf);
-        auto underflow = rewriter.create<arith::CmpFOp>(
+        auto underflow = arith::CmpFOp::create(rewriter,
             loc, arith::CmpFPredicate::UEQ, rounded, negInf);
-        auto intMin = rewriter.create<arith::ConstantOp>(
+        auto intMin = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIntegerAttr(
                      getElementTypeOrSelf(dstTy),
                      APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())));
-        auto intMax = rewriter.create<arith::ConstantOp>(
+        auto intMax = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIntegerAttr(
                      getElementTypeOrSelf(dstTy),
                      APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
         auto maxClamped =
-            rewriter.create<arith::SelectOp>(loc, overflow, intMax, conv);
-        return rewriter.create<arith::SelectOp>(loc, underflow, intMin,
+            arith::SelectOp::create(rewriter, loc, overflow, intMax, conv);
+        return arith::SelectOp::create(rewriter, loc, underflow, intMin,
                                                 maxClamped);
       }
 
-      auto intMinFP = rewriter.create<arith::ConstantOp>(
+      auto intMinFP = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
                    APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())
@@ -640,7 +640,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         // consists of a single leading bit. Therefore we can clamp the input
         // in the floating-point domain.
 
-        auto intMaxFP = rewriter.create<arith::ConstantOp>(
+        auto intMaxFP = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(
                      getElementTypeOrSelf(srcTy),
                      APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
@@ -648,14 +648,14 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
         Value clamped =
             clampFloatHelper(loc, rounded, intMinFP, intMaxFP, rewriter);
-        return rewriter.create<arith::FPToSIOp>(loc, dstTy, clamped);
+        return arith::FPToSIOp::create(rewriter, loc, dstTy, clamped);
       }
 
       // Due to earlier check we know exponant range is big enough to represent
       // int min. We can therefore rely on int max + 1 being representable as
       // well because it's just int min with a positive sign. So clamp the min
       // value and compare against that to select the max int value if needed.
-      auto intMaxPlusOneFP = rewriter.create<arith::ConstantOp>(
+      auto intMaxPlusOneFP = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
                    static_cast<double>(
@@ -663,35 +663,35 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                            .getSExtValue()) +
                        1.0f));
 
-      auto intMax = rewriter.create<arith::ConstantOp>(
+      auto intMax = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIntegerAttr(
                    getElementTypeOrSelf(dstTy),
                    APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
       auto minClampedFP =
-          rewriter.create<arith::MaximumFOp>(loc, rounded, intMinFP);
+          arith::MaximumFOp::create(rewriter, loc, rounded, intMinFP);
       auto minClamped =
-          rewriter.create<arith::FPToSIOp>(loc, dstTy, minClampedFP);
-      auto overflow = rewriter.create<arith::CmpFOp>(
+          arith::FPToSIOp::create(rewriter, loc, dstTy, minClampedFP);
+      auto overflow = arith::CmpFOp::create(rewriter,
           loc, arith::CmpFPredicate::UGE, rounded, intMaxPlusOneFP);
-      return rewriter.create<arith::SelectOp>(loc, overflow, intMax,
+      return arith::SelectOp::create(rewriter, loc, overflow, intMax,
                                               minClamped);
     }
 
     // Casting to boolean, integers need to only be checked as not-equal to
     // zero.
     if (isa<IntegerType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = rewriter.create<arith::ConstantIntOp>(
+      Value zero = arith::ConstantIntOp::create(rewriter,
           loc, 0, srcTy.getIntOrFloatBitWidth());
-      return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+      return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
                                             args.front(), zero);
     }
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtSIOp>(loc, resultTypes, args,
+      return arith::ExtSIOp::create(rewriter, loc, resultTypes, args,
                                              ArrayRef<NamedAttribute>());
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && !bitExtend) {
-      return rewriter.create<arith::TruncIOp>(loc, dstTy, args[0]);
+      return arith::TruncIOp::create(rewriter, loc, dstTy, args[0]);
     }
   }
 
@@ -710,14 +710,14 @@ static Value createIndex(PatternRewriter &rewriter, Location loc,
   auto [it, inserted] = indexPool.try_emplace(index);
   if (inserted)
     it->second =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(index));
   return it->second;
 }
 
 static Value getTensorDim(PatternRewriter &rewriter, Location loc,
                           IndexPool &indexPool, Value tensor, int64_t index) {
   auto indexValue = createIndex(rewriter, loc, indexPool, index);
-  return rewriter.create<tensor::DimOp>(loc, tensor, indexValue).getResult();
+  return tensor::DimOp::create(rewriter, loc, tensor, indexValue).getResult();
 }
 
 static OpFoldResult getOrFoldTensorDim(PatternRewriter &rewriter, Location loc,
@@ -783,7 +783,7 @@ computeTargetSize(PatternRewriter &rewriter, Location loc, IndexPool &indexPool,
   for (size_t i = 1; i < operandsWithDynamicDim.size(); i++) {
     auto nextSize =
         getTensorDim(rewriter, loc, indexPool, operandsWithDynamicDim[i], dim);
-    targetSize = rewriter.create<arith::MaxUIOp>(loc, targetSize, nextSize);
+    targetSize = arith::MaxUIOp::create(rewriter, loc, targetSize, nextSize);
   }
   return {targetSize, nullptr};
 }
@@ -838,7 +838,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
   // Check if broadcast is necessary
   auto one = createIndex(rewriter, loc, indexPool, 1);
   auto runtimeSize = getTensorDim(rewriter, loc, indexPool, operand, dim);
-  auto broadcastNecessary = rewriter.create<arith::CmpIOp>(
+  auto broadcastNecessary = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::eq, runtimeSize, one);
 
   // Emit 'then' region of 'scf.if'
@@ -855,7 +855,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                                                     operand, index);
       outputTensorShape.push_back(size);
     }
-    Value outputTensor = opBuilder.create<tensor::EmptyOp>(
+    Value outputTensor = tensor::EmptyOp::create(opBuilder,
         loc, outputTensorShape, rankedTensorType.getElementType());
 
     // Emit 'linalg.generic' op
@@ -866,7 +866,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                 getNParallelLoopsAttrs(rank),
                 [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
                   // Emit 'linalg.yield' op
-                  opBuilder.create<linalg::YieldOp>(loc, blockArgs.front());
+                  linalg::YieldOp::create(opBuilder, loc, blockArgs.front());
                 })
             .getResult(0);
 
@@ -875,16 +875,16 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
         loc, operand.getType(), resultTensor);
 
     // Emit 'scf.yield' op
-    opBuilder.create<scf::YieldOp>(loc, castResultTensor);
+    scf::YieldOp::create(opBuilder, loc, castResultTensor);
   };
 
   // Emit 'else' region of 'scf.if'
   auto emitElseRegion = [&](OpBuilder &opBuilder, Location loc) {
-    opBuilder.create<scf::YieldOp>(loc, operand);
+    scf::YieldOp::create(opBuilder, loc, operand);
   };
 
   // Emit 'scf.if' op
-  auto ifOp = rewriter.create<scf::IfOp>(loc, broadcastNecessary,
+  auto ifOp = scf::IfOp::create(rewriter, loc, broadcastNecessary,
                                          emitThenRegion, emitElseRegion);
   return ifOp.getResult(0);
 }
@@ -930,7 +930,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
   if (!resultType) {
     return rewriter.notifyMatchFailure(operation, "failed to convert type");
   }
-  Value outputTensor = rewriter.create<tensor::EmptyOp>(
+  Value outputTensor = tensor::EmptyOp::create(rewriter,
       loc, targetShape, resultType.getElementType());
 
   // Create affine maps. Input affine maps broadcast static dimensions of size
@@ -957,7 +957,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
 
   // Emit 'linalg.generic' op
   bool encounteredError = false;
-  auto linalgOp = rewriter.create<linalg::GenericOp>(
+  auto linalgOp = linalg::GenericOp::create(rewriter,
       loc, outputTensor.getType(), operands, outputTensor, affineMaps,
       getNParallelLoopsAttrs(rank),
       [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
@@ -968,7 +968,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
           encounteredError = true;
           return;
         }
-        opBuilder.create<linalg::YieldOp>(loc, opResult);
+        linalg::YieldOp::create(opBuilder, loc, opResult);
       });
   if (encounteredError)
     return rewriter.notifyMatchFailure(
@@ -1078,42 +1078,42 @@ static Value createLinalgBodyCalculationForReduceOp(Operation *op,
                                                     PatternRewriter &rewriter) {
   Location loc = op->getLoc();
   if (isa<tosa::ReduceSumOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::AddFOp>(loc, args);
+    return arith::AddFOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceSumOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::AddIOp>(loc, args);
+    return arith::AddIOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceProductOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MulFOp>(loc, args);
+    return arith::MulFOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceProductOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MulIOp>(loc, args);
+    return arith::MulIOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceMinOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MinimumFOp>(loc, args[0], args[1]);
+    return arith::MinimumFOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMinOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MinSIOp>(loc, args[0], args[1]);
+    return arith::MinSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMaxOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MaximumFOp>(loc, args[0], args[1]);
+    return arith::MaximumFOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMaxOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], args[1]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceAllOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::AndIOp>(loc, args);
+    return arith::AndIOp::create(rewriter, loc, args);
 
   if (isa<tosa::ReduceAnyOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::OrIOp>(loc, args);
+    return arith::OrIOp::create(rewriter, loc, args);
 
   return {};
 }
@@ -1139,7 +1139,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     if (axis != i) {
       reduceShape.push_back(inputTy.getDimSize(i));
       if (inputTy.isDynamicDim(i))
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
     }
   }
 
@@ -1158,7 +1158,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     return rewriter.notifyMatchFailure(
         op, "No initial value found for reduction operation");
 
-  auto fillValue = rewriter.create<arith::ConstantOp>(loc, fillValueAttr);
+  auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
   auto filledTensor = rewriter
                           .create<linalg::FillOp>(loc, ValueRange{fillValue},
                                                   ValueRange{emptyTensor})
@@ -1176,7 +1176,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
       // Additionally we have to keep track of whether we've seen any non-NaN
       // values and then do a final select based on this predicate.
       auto trueAttr = rewriter.getBoolAttr(true);
-      auto trueValue = rewriter.create<arith::ConstantOp>(loc, trueAttr);
+      auto trueValue = arith::ConstantOp::create(rewriter, loc, trueAttr);
       auto emptyBoolTensor =
           rewriter
               .create<tensor::EmptyOp>(loc, reduceShape, trueValue.getType(),
@@ -1202,7 +1202,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
   }
 
   bool didEncounterError = false;
-  linalg::LinalgOp linalgOp = rewriter.create<linalg::ReduceOp>(
+  linalg::LinalgOp linalgOp = linalg::ReduceOp::create(rewriter,
       loc, inputs, outputs, axis,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange blockArgs) {
         std::array<Value, 2> binaryArgs{
@@ -1219,21 +1219,21 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
           auto oldAllResultsNanFlagValue = blockArgs[3];
 
           // Unordered comparison of NaN against itself will always return true.
-          Value isNaN = nestedBuilder.create<arith::CmpFOp>(
+          Value isNaN = arith::CmpFOp::create(nestedBuilder,
               op->getLoc(), arith::CmpFPredicate::UNO, inputValue, inputValue);
           // If we've encountered a NaN, take the non-NaN value.
-          auto selectOp = nestedBuilder.create<arith::SelectOp>(
+          auto selectOp = arith::SelectOp::create(nestedBuilder,
               op->getLoc(), isNaN, initialValue, result);
           // Update the flag which keeps track of whether we have seen a non-NaN
           // value.
-          auto newAllResultsNanFlagValue = nestedBuilder.create<arith::AndIOp>(
+          auto newAllResultsNanFlagValue = arith::AndIOp::create(nestedBuilder,
               op->getLoc(), oldAllResultsNanFlagValue, isNaN);
           resultsToYield.push_back(selectOp);
           resultsToYield.push_back(newAllResultsNanFlagValue);
         } else {
           resultsToYield.push_back(result);
         }
-        nestedBuilder.create<linalg::YieldOp>(loc, resultsToYield);
+        linalg::YieldOp::create(nestedBuilder, loc, resultsToYield);
       });
 
   if (!didEncounterError)
@@ -1250,7 +1250,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     auto nanValueAttr = rewriter.getFloatAttr(
         elementTy,
         APFloat::getNaN(cast<FloatType>(elementTy).getFloatSemantics(), false));
-    auto nanValue = rewriter.create<arith::ConstantOp>(loc, nanValueAttr);
+    auto nanValue = arith::ConstantOp::create(rewriter, loc, nanValueAttr);
     auto emptyNanTensor =
         rewriter
             .create<tensor::EmptyOp>(loc, reduceShape,
@@ -1278,7 +1278,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     ins.push_back(linalgOp->getResult(0));
     outs.push_back(finalEmptyTensor);
     auto linalgSelect =
-        rewriter.create<linalg::SelectOp>(op->getLoc(), ins, outs);
+        linalg::SelectOp::create(rewriter, op->getLoc(), ins, outs);
     linalgOp = linalgSelect;
   }
 
@@ -1350,7 +1350,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < outputTy.getRank(); i++) {
       if (outputTy.isDynamicDim(i)) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
@@ -1401,7 +1401,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value multiplierConstant;
     int64_t multiplierArg = 0;
     if (multiplierValues.size() == 1) {
-      multiplierConstant = rewriter.create<arith::ConstantOp>(
+      multiplierConstant = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32IntegerAttr(multiplierValues.front()));
     } else {
       SmallVector<AffineExpr, 2> multiplierExprs{
@@ -1409,7 +1409,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
       auto multiplierType =
           RankedTensorType::get({static_cast<int64_t>(multiplierValues.size())},
                                 rewriter.getI32Type());
-      genericInputs.push_back(rewriter.create<arith::ConstantOp>(
+      genericInputs.push_back(arith::ConstantOp::create(rewriter,
           loc, DenseIntElementsAttr::get(multiplierType, multiplierValues)));
 
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
@@ -1424,7 +1424,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value shiftConstant;
     int64_t shiftArg = 0;
     if (shiftValues.size() == 1) {
-      shiftConstant = rewriter.create<arith::ConstantOp>(
+      shiftConstant = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI8IntegerAttr(shiftValues.front()));
     } else {
       SmallVector<AffineExpr, 2> shiftExprs = {
@@ -1432,7 +1432,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
       auto shiftType =
           RankedTensorType::get({static_cast<int64_t>(shiftValues.size())},
                                 rewriter.getIntegerType(8));
-      genericInputs.push_back(rewriter.create<arith::ConstantOp>(
+      genericInputs.push_back(arith::ConstantOp::create(rewriter,
           loc, DenseIntElementsAttr::get(shiftType, shiftValues)));
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
                                             /*symbolCount=*/0, shiftExprs,
@@ -1444,11 +1444,11 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(rank));
 
     // Construct the indexing maps needed for linalg.generic ops.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, outputTy.getShape(), outputTy.getElementType(),
         ArrayRef<Value>({dynDims}));
 
-    auto linalgOp = rewriter.create<linalg::GenericOp>(
+    auto linalgOp = linalg::GenericOp::create(rewriter,
         loc, outputTy, genericInputs, ValueRange{emptyTensor}, indexingMaps,
         getNParallelLoopsAttrs(rank),
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
@@ -1466,7 +1466,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           const int32_t inBitwidth = valueTy.getIntOrFloatBitWidth();
           // Extend zeropoint for sub-32bits widths.
           const int32_t inAttrBitwidth = inBitwidth > 32 ? inBitwidth : 32;
-          auto inputZp = nestedBuilder.create<arith::ConstantOp>(
+          auto inputZp = arith::ConstantOp::create(nestedBuilder,
               loc, IntegerAttr::get(rewriter.getIntegerType(inAttrBitwidth),
                                     *maybeIZp));
 
@@ -1482,7 +1482,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           unsigned outBitWidth = outIntType.getWidth();
           const int32_t outAttrBitwidth = 32;
           assert(outBitWidth <= 32 && "Unexpected output zeropoint bitwidth");
-          auto outputZp = nestedBuilder.create<arith::ConstantOp>(
+          auto outputZp = arith::ConstantOp::create(nestedBuilder,
               loc, IntegerAttr::get(rewriter.getIntegerType(outAttrBitwidth),
                                     *maybeOZp));
 
@@ -1501,24 +1501,24 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           }
           if (valueTy.getIntOrFloatBitWidth() < 32) {
             if (op.getInputUnsigned()) {
-              value = nestedBuilder.create<arith::ExtUIOp>(
+              value = arith::ExtUIOp::create(nestedBuilder,
                   nestedLoc, nestedBuilder.getI32Type(), value);
             } else {
-              value = nestedBuilder.create<arith::ExtSIOp>(
+              value = arith::ExtSIOp::create(nestedBuilder,
                   nestedLoc, nestedBuilder.getI32Type(), value);
             }
           }
 
           value =
-              nestedBuilder.create<arith::SubIOp>(nestedLoc, value, inputZp);
+              arith::SubIOp::create(nestedBuilder, nestedLoc, value, inputZp);
 
-          value = nestedBuilder.create<tosa::ApplyScaleOp>(
+          value = tosa::ApplyScaleOp::create(nestedBuilder,
               loc, nestedBuilder.getI32Type(), value, multiplier, shift,
               roundingMode);
 
           // Move to the new zero-point.
           value =
-              nestedBuilder.create<arith::AddIOp>(nestedLoc, value, outputZp);
+              arith::AddIOp::create(nestedBuilder, nestedLoc, value, outputZp);
 
           // Saturate to the output size.
           int32_t intMin = APInt::getSignedMinValue(outBitWidth).getSExtValue();
@@ -1530,16 +1530,16 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
             intMax = APInt::getMaxValue(outBitWidth).getZExtValue();
           }
 
-          auto intMinVal = nestedBuilder.create<arith::ConstantOp>(
+          auto intMinVal = arith::ConstantOp::create(nestedBuilder,
               loc, nestedBuilder.getI32IntegerAttr(intMin));
-          auto intMaxVal = nestedBuilder.create<arith::ConstantOp>(
+          auto intMaxVal = arith::ConstantOp::create(nestedBuilder,
               loc, nestedBuilder.getI32IntegerAttr(intMax));
 
           value = clampIntHelper(nestedLoc, value, intMinVal, intMaxVal,
                                  nestedBuilder, /*isUnsigned=*/false);
 
           if (outIntType.getWidth() < 32) {
-            value = nestedBuilder.create<arith::TruncIOp>(
+            value = arith::TruncIOp::create(nestedBuilder,
                 nestedLoc, rewriter.getIntegerType(outIntType.getWidth()),
                 value);
           }
@@ -1550,7 +1550,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
                                                             outIntType, value)
                         .getResult(0);
           }
-          nestedBuilder.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(nestedBuilder, loc, value);
         });
 
     rewriter.replaceOp(op, linalgOp->getResults());
@@ -1608,25 +1608,25 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
     auto collapseTy =
         RankedTensorType::get({inputTy.getDimSize(0), inputTy.getDimSize(3)},
                               inputTy.getElementType());
-    Value collapse = builder.create<tensor::CollapseShapeOp>(collapseTy, input,
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, input,
                                                              reassociationMap);
 
     // Get any dynamic shapes that appear in the input format.
     llvm::SmallVector<Value> outputDynSize;
     if (inputTy.isDynamicDim(0))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 0));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 0));
     if (inputTy.isDynamicDim(3))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 3));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 3));
 
     // Generate the elementwise operation for casting scaling the input value.
     auto genericTy = collapseTy.clone(resultTy.getElementType());
-    Value empty = builder.create<tensor::EmptyOp>(
+    Value empty = tensor::EmptyOp::create(builder,
         genericTy.getShape(), resultTy.getElementType(), outputDynSize);
     auto genericMap = rewriter.getMultiDimIdentityMap(genericTy.getRank());
     SmallVector<utils::IteratorType> iterators(genericTy.getRank(),
                                                utils::IteratorType::parallel);
 
-    auto generic = builder.create<linalg::GenericOp>(
+    auto generic = linalg::GenericOp::create(builder,
         genericTy, ValueRange{collapse}, ValueRange{empty},
         ArrayRef<AffineMap>{genericMap, genericMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
@@ -1634,22 +1634,22 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
           // This is the quantized case.
           if (inputTy.getElementType() != resultTy.getElementType()) {
             value =
-                b.create<arith::ExtSIOp>(loc, resultTy.getElementType(), value);
+                arith::ExtSIOp::create(b, loc, resultTy.getElementType(), value);
 
             if (isBilinear && scale[0] != 0) {
-              Value scaleY = b.create<arith::ConstantOp>(
+              Value scaleY = arith::ConstantOp::create(b,
                   loc, b.getI32IntegerAttr(scale[0]));
-              value = b.create<arith::MulIOp>(loc, value, scaleY);
+              value = arith::MulIOp::create(b, loc, value, scaleY);
             }
 
             if (isBilinear && scale[2] != 0) {
-              Value scaleX = b.create<arith::ConstantOp>(
+              Value scaleX = arith::ConstantOp::create(b,
                   loc, b.getI32IntegerAttr(scale[2]));
-              value = b.create<arith::MulIOp>(loc, value, scaleX);
+              value = arith::MulIOp::create(b, loc, value, scaleX);
             }
           }
 
-          b.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(b, loc, value);
         });
 
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -1697,7 +1697,7 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     resizeShape.push_back(channels);
 
     auto resizeTy = resultTy.clone(resizeShape);
-    auto resize = builder.create<tosa::ResizeOp>(resizeTy, input, op.getScale(),
+    auto resize = tosa::ResizeOp::create(builder, resizeTy, input, op.getScale(),
                                                  op.getOffset(), op.getBorder(),
                                                  op.getMode());
 
@@ -1720,19 +1720,19 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     collapseShape.push_back(channels);
 
     auto collapseTy = resultTy.clone(collapseShape);
-    Value collapse = builder.create<tensor::CollapseShapeOp>(collapseTy, resize,
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, resize,
                                                              reassociationMap);
 
     // Broadcast the collapsed shape to the output result.
     llvm::SmallVector<Value> outputDynSize;
     if (inputTy.isDynamicDim(0))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 0));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 0));
     if (inputTy.isDynamicDim(3))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 3));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 3));
 
     SmallVector<utils::IteratorType> iterators(resultTy.getRank(),
                                                utils::IteratorType::parallel);
-    Value empty = builder.create<tensor::EmptyOp>(
+    Value empty = tensor::EmptyOp::create(builder,
         resultTy.getShape(), resultTy.getElementType(), outputDynSize);
 
     SmallVector<AffineExpr, 4> inputExprs{rewriter.getAffineDimExpr(0)};
@@ -1751,7 +1751,7 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
         ArrayRef<AffineMap>{inputMap, outputMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
           Value value = args[0];
-          b.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(b, loc, value);
         });
 
     return success();
@@ -1789,9 +1789,9 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
 
     SmallVector<AffineMap, 2> affineMaps = {
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
-    auto emptyTensor = b.create<tensor::EmptyOp>(resultTy.getShape(), resultETy,
+    auto emptyTensor = tensor::EmptyOp::create(b, resultTy.getShape(), resultETy,
                                                  *dynamicDimsOr);
-    auto genericOp = b.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(b,
         resultTy, ValueRange({}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     Value resize = genericOp.getResult(0);
@@ -1800,19 +1800,19 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       OpBuilder::InsertionGuard regionGuard(b);
       b.createBlock(&genericOp.getRegion(), genericOp.getRegion().end(),
                     TypeRange({resultETy}), loc);
-      Value batch = b.create<linalg::IndexOp>(0);
-      Value y = b.create<linalg::IndexOp>(1);
-      Value x = b.create<linalg::IndexOp>(2);
-      Value channel = b.create<linalg::IndexOp>(3);
+      Value batch = linalg::IndexOp::create(b, 0);
+      Value y = linalg::IndexOp::create(b, 1);
+      Value x = linalg::IndexOp::create(b, 2);
+      Value channel = linalg::IndexOp::create(b, 3);
 
       Value zeroI32 =
-          b.create<arith::ConstantOp>(b.getZeroAttr(b.getI32Type()));
-      Value zeroFp = b.create<arith::ConstantOp>(b.getZeroAttr(floatTy));
-      Value hMax = b.create<arith::ConstantOp>(b.getI32IntegerAttr(imageH - 1));
-      Value wMax = b.create<arith::ConstantOp>(b.getI32IntegerAttr(imageW - 1));
+          arith::ConstantOp::create(b, b.getZeroAttr(b.getI32Type()));
+      Value zeroFp = arith::ConstantOp::create(b, b.getZeroAttr(floatTy));
+      Value hMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageH - 1));
+      Value wMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageW - 1));
 
-      Value inY = b.create<arith::IndexCastOp>(b.getI32Type(), y);
-      Value inX = b.create<arith::IndexCastOp>(b.getI32Type(), x);
+      Value inY = arith::IndexCastOp::create(b, b.getI32Type(), y);
+      Value inX = arith::IndexCastOp::create(b, b.getI32Type(), x);
 
       SmallVector<int64_t> scale, offset, border;
       if (!tosa::getConstShapeValues(op.getScale().getDefiningOp(), scale) ||
@@ -1824,16 +1824,16 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       }
 
       Value yScaleN, yScaleD, xScaleN, xScaleD;
-      yScaleN = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[0]));
-      yScaleD = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[1]));
-      xScaleN = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[2]));
-      xScaleD = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[3]));
+      yScaleN = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[0]));
+      yScaleD = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[1]));
+      xScaleN = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[2]));
+      xScaleD = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[3]));
 
       Value yOffset, xOffset, yBorder, xBorder;
-      yOffset = b.create<arith::ConstantOp>(b.getI32IntegerAttr(offset[0]));
-      xOffset = b.create<arith::ConstantOp>(b.getI32IntegerAttr(offset[1]));
-      yBorder = b.create<arith::ConstantOp>(b.getI32IntegerAttr(border[0]));
-      xBorder = b.create<arith::ConstantOp>(b.getI32IntegerAttr(border[1]));
+      yOffset = arith::ConstantOp::create(b, b.getI32IntegerAttr(offset[0]));
+      xOffset = arith::ConstantOp::create(b, b.getI32IntegerAttr(offset[1]));
+      yBorder = arith::ConstantOp::create(b, b.getI32IntegerAttr(border[0]));
+      xBorder = arith::ConstantOp::create(b, b.getI32IntegerAttr(border[1]));
 
       // Compute the ix and dx values for both the X and Y dimensions.
       auto getIndexAndDeltaFp = [&](Value &index, Value &delta, Value in,
@@ -1846,16 +1846,16 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         }
         // x = x * scale_d + offset;
         // ix = floor(x / scale_n)
-        Value val = b.create<arith::MulIOp>(in, scaleD);
-        val = b.create<arith::AddIOp>(val, offset);
-        index = b.create<arith::FloorDivSIOp>(val, scaleN);
+        Value val = arith::MulIOp::create(b, in, scaleD);
+        val = arith::AddIOp::create(b, val, offset);
+        index = arith::FloorDivSIOp::create(b, val, scaleN);
 
         // rx = x % scale_n
         // dx = rx / scale_n
-        Value r = b.create<arith::RemSIOp>(val, scaleN);
-        Value rFp = b.create<arith::SIToFPOp>(floatTy, r);
-        Value scaleNfp = b.create<arith::UIToFPOp>(floatTy, scaleN);
-        delta = b.create<arith::DivFOp>(rFp, scaleNfp);
+        Value r = arith::RemSIOp::create(b, val, scaleN);
+        Value rFp = arith::SIToFPOp::create(b, floatTy, r);
+        Value scaleNfp = arith::UIToFPOp::create(b, floatTy, scaleN);
+        delta = arith::DivFOp::create(b, rFp, scaleNfp);
       };
 
       // Compute the ix and dx values for the X and Y dimensions - int case.
@@ -1870,11 +1870,11 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         // x = x * scale_d + offset;
         // ix = floor(x / scale_n)
         //  dx = x - ix * scale_n;
-        Value val = b.create<arith::MulIOp>(in, scaleD);
-        val = b.create<arith::AddIOp>(val, offset);
-        index = b.create<arith::DivSIOp>(val, scaleN);
-        delta = b.create<arith::MulIOp>(index, scaleN);
-        delta = b.create<arith::SubIOp>(val, delta);
+        Value val = arith::MulIOp::create(b, in, scaleD);
+        val = arith::AddIOp::create(b, val, offset);
+        index = arith::DivSIOp::create(b, val, scaleN);
+        delta = arith::MulIOp::create(b, index, scaleN);
+        delta = arith::SubIOp::create(b, val, delta);
       };
 
       Value ix, iy, dx, dy;
@@ -1887,54 +1887,54 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       }
 
       if (op.getMode() == "NEAREST_NEIGHBOR") {
-        auto one = b.create<arith::ConstantOp>(b.getI32IntegerAttr(1));
+        auto one = arith::ConstantOp::create(b, b.getI32IntegerAttr(1));
 
         auto getNearestIndexAndClamp = [&](Value val, Value dval, Value scale,
                                            Value max, int size,
                                            ImplicitLocOpBuilder &b) -> Value {
           if (size == 1) {
-            return b.create<arith::ConstantIndexOp>(0);
+            return arith::ConstantIndexOp::create(b, 0);
           }
 
           Value pred;
           if (floatingPointMode) {
-            auto h = b.create<arith::ConstantOp>(b.getFloatAttr(floatTy, 0.5f));
-            pred = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, dval, h);
+            auto h = arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 0.5f));
+            pred = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, dval, h);
           } else {
-            Value dvalDouble = b.create<arith::ShLIOp>(dval, one);
-            pred = b.create<arith::CmpIOp>(arith::CmpIPredicate::sge,
+            Value dvalDouble = arith::ShLIOp::create(b, dval, one);
+            pred = arith::CmpIOp::create(b, arith::CmpIPredicate::sge,
                                            dvalDouble, scale);
           }
 
-          auto offset = b.create<arith::SelectOp>(pred, one, zeroI32);
-          val = b.create<arith::AddIOp>(val, offset);
+          auto offset = arith::SelectOp::create(b, pred, one, zeroI32);
+          val = arith::AddIOp::create(b, val, offset);
           val = clampIntHelper(loc, val, zeroI32, max, b, /*isUnsigned=*/false);
-          return b.create<arith::IndexCastOp>(b.getIndexType(), val);
+          return arith::IndexCastOp::create(b, b.getIndexType(), val);
         };
 
         iy = getNearestIndexAndClamp(iy, dy, yScaleN, hMax, imageH, b);
         ix = getNearestIndexAndClamp(ix, dx, xScaleN, wMax, imageW, b);
 
-        Value result = b.create<tensor::ExtractOp>(
+        Value result = tensor::ExtractOp::create(b,
             input, ValueRange{batch, iy, ix, channel});
 
-        b.create<linalg::YieldOp>(result);
+        linalg::YieldOp::create(b, result);
       } else {
         // The mode here must be BILINEAR.
         assert(op.getMode() == "BILINEAR");
 
-        auto oneVal = b.create<arith::ConstantOp>(b.getI32IntegerAttr(1));
+        auto oneVal = arith::ConstantOp::create(b, b.getI32IntegerAttr(1));
 
         auto getClampedIdxs = [&](Value &val0, Value &val1, int size, Value in,
                                   Value max, ImplicitLocOpBuilder &b) {
           val0 = in;
-          val1 = b.create<arith::AddIOp>(val0, oneVal);
+          val1 = arith::AddIOp::create(b, val0, oneVal);
           val0 =
               clampIntHelper(loc, val0, zeroI32, max, b, /*isUnsigned=*/false);
           val1 =
               clampIntHelper(loc, val1, zeroI32, max, b, /*isUnsigned=*/false);
-          val0 = b.create<arith::IndexCastOp>(b.getIndexType(), val0);
-          val1 = b.create<arith::IndexCastOp>(b.getIndexType(), val1);
+          val0 = arith::IndexCastOp::create(b, b.getIndexType(), val0);
+          val1 = arith::IndexCastOp::create(b, b.getIndexType(), val1);
         };
 
         // Linalg equivalent to the section below:
@@ -1946,27 +1946,27 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         getClampedIdxs(y0, y1, imageH, iy, hMax, b);
         getClampedIdxs(x0, x1, imageW, ix, wMax, b);
 
-        Value y0x0 = b.create<tensor::ExtractOp>(
+        Value y0x0 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y0, x0, channel});
-        Value y0x1 = b.create<tensor::ExtractOp>(
+        Value y0x1 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y0, x1, channel});
-        Value y1x0 = b.create<tensor::ExtractOp>(
+        Value y1x0 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y1, x0, channel});
-        Value y1x1 = b.create<tensor::ExtractOp>(
+        Value y1x1 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y1, x1, channel});
 
         if (floatingPointMode) {
           auto oneVal =
-              b.create<arith::ConstantOp>(b.getFloatAttr(floatTy, 1.0f));
+              arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 1.0f));
           auto interpolate = [&](Value val0, Value val1, Value delta,
                                  int inputSize,
                                  ImplicitLocOpBuilder &b) -> Value {
             if (inputSize == 1)
               return val0;
-            Value oneMinusDelta = b.create<arith::SubFOp>(oneVal, delta);
-            Value mul0 = b.create<arith::MulFOp>(val0, oneMinusDelta);
-            Value mul1 = b.create<arith::MulFOp>(val1, delta);
-            return b.create<arith::AddFOp>(mul0, mul1);
+            Value oneMinusDelta = arith::SubFOp::create(b, oneVal, delta);
+            Value mul0 = arith::MulFOp::create(b, val0, oneMinusDelta);
+            Value mul1 = arith::MulFOp::create(b, val1, delta);
+            return arith::AddFOp::create(b, mul0, mul1);
           };
 
           // Linalg equivalent to the section below:
@@ -1982,18 +1982,18 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
           // Linalg equivalent to the section below:
           //   result = topAcc * (unit_y - dy) + bottomAcc * dy
           Value result = interpolate(topAcc, bottomAcc, dy, imageH, b);
-          b.create<linalg::YieldOp>(result);
+          linalg::YieldOp::create(b, result);
         } else {
           // Perform in quantized space.
-          y0x0 = b.create<arith::ExtSIOp>(resultETy, y0x0);
-          y0x1 = b.create<arith::ExtSIOp>(resultETy, y0x1);
-          y1x0 = b.create<arith::ExtSIOp>(resultETy, y1x0);
-          y1x1 = b.create<arith::ExtSIOp>(resultETy, y1x1);
+          y0x0 = arith::ExtSIOp::create(b, resultETy, y0x0);
+          y0x1 = arith::ExtSIOp::create(b, resultETy, y0x1);
+          y1x0 = arith::ExtSIOp::create(b, resultETy, y1x0);
+          y1x1 = arith::ExtSIOp::create(b, resultETy, y1x1);
 
           const int64_t deltaBitwidth = dx.getType().getIntOrFloatBitWidth();
           if (resultETy.getIntOrFloatBitWidth() > deltaBitwidth) {
-            dx = b.create<arith::ExtSIOp>(resultETy, dx);
-            dy = b.create<arith::ExtSIOp>(resultETy, dy);
+            dx = arith::ExtSIOp::create(b, resultETy, dx);
+            dy = arith::ExtSIOp::create(b, resultETy, dy);
           }
 
           Value yScaleNExt = yScaleN;
@@ -2002,26 +2002,26 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
           const int64_t scaleBitwidth =
               xScaleN.getType().getIntOrFloatBitWidth();
           if (resultETy.getIntOrFloatBitWidth() > scaleBitwidth) {
-            yScaleNExt = b.create<arith::ExtSIOp>(resultETy, yScaleN);
-            xScaleNExt = b.create<arith::ExtSIOp>(resultETy, xScaleN);
+            yScaleNExt = arith::ExtSIOp::create(b, resultETy, yScaleN);
+            xScaleNExt = arith::ExtSIOp::create(b, resultETy, xScaleN);
           }
 
           auto interpolate = [](Value val0, Value val1, Value weight1,
                                 Value scale, int inputSize,
                                 ImplicitLocOpBuilder &b) -> Value {
             if (inputSize == 1)
-              return b.create<arith::MulIOp>(val0, scale);
-            Value weight0 = b.create<arith::SubIOp>(scale, weight1);
-            Value mul0 = b.create<arith::MulIOp>(val0, weight0);
-            Value mul1 = b.create<arith::MulIOp>(val1, weight1);
-            return b.create<arith::AddIOp>(mul0, mul1);
+              return arith::MulIOp::create(b, val0, scale);
+            Value weight0 = arith::SubIOp::create(b, scale, weight1);
+            Value mul0 = arith::MulIOp::create(b, val0, weight0);
+            Value mul1 = arith::MulIOp::create(b, val1, weight1);
+            return arith::AddIOp::create(b, mul0, mul1);
           };
 
           Value topAcc = interpolate(y0x0, y0x1, dx, xScaleNExt, imageW, b);
           Value bottomAcc = interpolate(y1x0, y1x1, dx, xScaleNExt, imageW, b);
           Value result =
               interpolate(topAcc, bottomAcc, dy, yScaleNExt, imageH, b);
-          b.create<linalg::YieldOp>(result);
+          linalg::YieldOp::create(b, result);
         }
       }
     }
@@ -2072,11 +2072,11 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i)) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
-    Value axisDimSize = rewriter.create<tensor::DimOp>(loc, input, axis);
+    Value axisDimSize = tensor::DimOp::create(rewriter, loc, input, axis);
 
     // First fill the output buffer with the init value.
     auto emptyTensor = rewriter
@@ -2094,21 +2094,21 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
           llvm::SmallVector<Value> indices;
           for (unsigned int i = 0; i < inputTy.getRank(); i++) {
             Value index =
-                rewriter.create<linalg::IndexOp>(nestedLoc, i).getResult();
+                linalg::IndexOp::create(rewriter, nestedLoc, i).getResult();
             if (i == axis) {
-              auto one = rewriter.create<arith::ConstantIndexOp>(nestedLoc, 1);
+              auto one = arith::ConstantIndexOp::create(rewriter, nestedLoc, 1);
               auto sizeMinusOne =
-                  rewriter.create<arith::SubIOp>(nestedLoc, axisDimSize, one);
-              index = rewriter.create<arith::SubIOp>(nestedLoc, sizeMinusOne,
+                  arith::SubIOp::create(rewriter, nestedLoc, axisDimSize, one);
+              index = arith::SubIOp::create(rewriter, nestedLoc, sizeMinusOne,
                                                      index);
             }
 
             indices.push_back(index);
           }
 
-          auto extract = nestedBuilder.create<tensor::ExtractOp>(
+          auto extract = tensor::ExtractOp::create(nestedBuilder,
               nestedLoc, input, indices);
-          nestedBuilder.create<linalg::YieldOp>(op.getLoc(),
+          linalg::YieldOp::create(nestedBuilder, op.getLoc(),
                                                 extract.getResult());
         });
     return success();
@@ -2148,11 +2148,11 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i) || multiples[i] == -1) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         op.getLoc(), genericShape, elementTy, dynDims);
 
     // We needs to map the input shape to the non-broadcasted dimensions.
@@ -2168,12 +2168,12 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<AffineMap, 2> affineMaps = {
         readAffineMap, rewriter.getMultiDimIdentityMap(genericShape.size())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, RankedTensorType::get(genericShape, elementTy), input,
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(genericShape.size()),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-          nestedBuilder.create<linalg::YieldOp>(op.getLoc(), *args.begin());
+          linalg::YieldOp::create(nestedBuilder, op.getLoc(), *args.begin());
         });
 
     auto shapeValue = getTosaConstShape(
@@ -2220,7 +2220,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i) && i != axis) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
@@ -2229,7 +2229,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
                               .create<tensor::EmptyOp>(loc, resultTy.getShape(),
                                                        outElementTy, dynDims)
                               .getResult();
-    auto fillValueIdx = rewriter.create<arith::ConstantOp>(
+    auto fillValueIdx = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(outElementTy, 0));
     auto filledTensorIdx =
         rewriter
@@ -2250,7 +2250,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           argmaxOp, "unsupported tosa.argmax element type");
 
     auto fillValueMax =
-        rewriter.create<arith::ConstantOp>(loc, fillValueMaxAttr);
+        arith::ConstantOp::create(rewriter, loc, fillValueMaxAttr);
     auto filledTensorMax =
         rewriter
             .create<linalg::FillOp>(loc, ValueRange{fillValueMax},
@@ -2274,7 +2274,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     bool didEncounterError = false;
     auto maps = AffineMap::inferFromExprList({srcExprs, dstExprs, dstExprs},
                                              rewriter.getContext());
-    auto linalgOp = rewriter.create<linalg::GenericOp>(
+    auto linalgOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
         ValueRange({filledTensorIdx, filledTensorMax}), maps, iteratorTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
@@ -2283,41 +2283,41 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           auto oldIndex = blockArgs[1];
           auto oldValue = blockArgs[2];
 
-          Value newIndex = rewriter.create<arith::IndexCastOp>(
+          Value newIndex = arith::IndexCastOp::create(rewriter,
               nestedLoc, oldIndex.getType(),
-              rewriter.create<linalg::IndexOp>(loc, axis));
+              linalg::IndexOp::create(rewriter, loc, axis));
 
           Value predicate;
           if (isa<FloatType>(inElementTy)) {
             if (argmaxOp.getNanMode() == "IGNORE") {
               // Only update index & max value for non NaN values. If all
               // values are NaNs, the initial index will be return which is 0.
-              predicate = rewriter.create<arith::CmpFOp>(
+              predicate = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::OGT, newValue, oldValue);
             } else {
               // Update max value if either of the following is true:
               // - new value is bigger
               // - cur max is not NaN and new value is NaN
-              Value gt = rewriter.create<arith::CmpFOp>(
+              Value gt = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::UGT, newValue, oldValue);
-              Value oldNonNaN = rewriter.create<arith::CmpFOp>(
+              Value oldNonNaN = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::ORD, oldValue, oldValue);
-              predicate = rewriter.create<arith::AndIOp>(
+              predicate = arith::AndIOp::create(rewriter,
                   nestedLoc, rewriter.getI1Type(), gt, oldNonNaN);
             }
           } else if (isa<IntegerType>(inElementTy)) {
-            predicate = rewriter.create<arith::CmpIOp>(
+            predicate = arith::CmpIOp::create(rewriter,
                 nestedLoc, arith::CmpIPredicate::sgt, newValue, oldValue);
           } else {
             didEncounterError = true;
             return;
           }
 
-          auto resultMax = rewriter.create<arith::SelectOp>(
+          auto resultMax = arith::SelectOp::create(rewriter,
               nestedLoc, predicate, newValue, oldValue);
-          auto resultIndex = rewriter.create<arith::SelectOp>(
+          auto resultIndex = arith::SelectOp::create(rewriter,
               nestedLoc, predicate, newIndex, oldIndex);
-          nestedBuilder.create<linalg::YieldOp>(
+          linalg::YieldOp::create(nestedBuilder,
               nestedLoc, ValueRange({resultIndex, resultMax}));
         });
 
@@ -2363,19 +2363,19 @@ class GatherConverter : public OpConversionPattern<tosa::GatherOp> {
             rewriter.getContext()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy}), ValueRange{indices},
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
           auto indexValue = args[0];
-          auto index0 = rewriter.create<linalg::IndexOp>(loc, 0);
-          Value index1 = rewriter.create<arith::IndexCastOp>(
+          auto index0 = linalg::IndexOp::create(rewriter, loc, 0);
+          Value index1 = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getIndexType(), indexValue);
-          auto index2 = rewriter.create<linalg::IndexOp>(loc, 2);
-          Value extract = rewriter.create<tensor::ExtractOp>(
+          auto index2 = linalg::IndexOp::create(rewriter, loc, 2);
+          Value extract = tensor::ExtractOp::create(rewriter,
               loc, input, ValueRange{index0, index1, index2});
-          rewriter.create<linalg::YieldOp>(loc, extract);
+          linalg::YieldOp::create(rewriter, loc, extract);
         });
     rewriter.replaceOp(op, genericOp.getResult(0));
     return success();
@@ -2424,7 +2424,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
     for (int i = 0; i < resultTy.getRank(); ++i) {
       if (inputTy.isDynamicDim(i)) {
         dynDims.push_back(
-            rewriter.create<tensor::DimOp>(loc, op.getOperand(0), i));
+            tensor::DimOp::create(rewriter, loc, op.getOperand(0), i));
       }
     }
 
@@ -2437,7 +2437,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
         rewriter.getMultiDimIdentityMap(resultTy.getRank()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, resultTy, ValueRange({input}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -2452,69 +2452,69 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
       rewriter.setInsertionPointToStart(block);
       if (inputElementTy.isInteger(8) && tableElementTy.isInteger(8) &&
           resultElementTy.isInteger(8)) {
-        Value index = rewriter.create<arith::IndexCastOp>(
+        Value index = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), inputValue);
-        Value offset = rewriter.create<arith::ConstantIndexOp>(loc, 128);
-        index = rewriter.create<arith::AddIOp>(loc, rewriter.getIndexType(),
+        Value offset = arith::ConstantIndexOp::create(rewriter, loc, 128);
+        index = arith::AddIOp::create(rewriter, loc, rewriter.getIndexType(),
                                                index, offset);
         Value extract =
-            rewriter.create<tensor::ExtractOp>(loc, table, ValueRange{index});
-        rewriter.create<linalg::YieldOp>(loc, extract);
+            tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
+        linalg::YieldOp::create(rewriter, loc, extract);
         return success();
       }
 
       if (inputElementTy.isInteger(16) && tableElementTy.isInteger(16) &&
           resultElementTy.isInteger(32)) {
-        Value extend = rewriter.create<arith::ExtSIOp>(
+        Value extend = arith::ExtSIOp::create(rewriter,
             loc, rewriter.getI32Type(), inputValue);
 
-        auto offset = rewriter.create<arith::ConstantOp>(
+        auto offset = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(32768));
-        auto seven = rewriter.create<arith::ConstantOp>(
+        auto seven = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(7));
-        auto one = rewriter.create<arith::ConstantOp>(
+        auto one = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(1));
-        auto b1111111 = rewriter.create<arith::ConstantOp>(
+        auto b1111111 = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(127));
 
         // Compute the index and fractional part from the input value:
         // value = value + 32768
         // index = value >> 7;
         // fraction = 0x01111111 & value
-        auto extendAdd = rewriter.create<arith::AddIOp>(loc, extend, offset);
-        Value index = rewriter.create<arith::ShRUIOp>(loc, extendAdd, seven);
+        auto extendAdd = arith::AddIOp::create(rewriter, loc, extend, offset);
+        Value index = arith::ShRUIOp::create(rewriter, loc, extendAdd, seven);
         Value fraction =
-            rewriter.create<arith::AndIOp>(loc, extendAdd, b1111111);
+            arith::AndIOp::create(rewriter, loc, extendAdd, b1111111);
 
         // Extract the base and next values from the table.
         // base = (int32_t) table[index];
         // next = (int32_t) table[index + 1];
-        Value indexPlusOne = rewriter.create<arith::AddIOp>(loc, index, one);
+        Value indexPlusOne = arith::AddIOp::create(rewriter, loc, index, one);
 
-        index = rewriter.create<arith::IndexCastOp>(
+        index = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), index);
-        indexPlusOne = rewriter.create<arith::IndexCastOp>(
+        indexPlusOne = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), indexPlusOne);
 
         Value base =
-            rewriter.create<tensor::ExtractOp>(loc, table, ValueRange{index});
-        Value next = rewriter.create<tensor::ExtractOp>(
+            tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
+        Value next = tensor::ExtractOp::create(rewriter,
             loc, table, ValueRange{indexPlusOne});
 
         base =
-            rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), base);
+            arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), base);
         next =
-            rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), next);
+            arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), next);
 
         // Use the fractional part to interpolate between the input values:
         // result = (base << 7) + (next - base) * fraction
-        Value baseScaled = rewriter.create<arith::ShLIOp>(loc, base, seven);
-        Value diff = rewriter.create<arith::SubIOp>(loc, next, base);
-        Value diffScaled = rewriter.create<arith::MulIOp>(loc, diff, fraction);
+        Value baseScaled = arith::ShLIOp::create(rewriter, loc, base, seven);
+        Value diff = arith::SubIOp::create(rewriter, loc, next, base);
+        Value diffScaled = arith::MulIOp::create(rewriter, loc, diff, fraction);
         Value result =
-            rewriter.create<arith::AddIOp>(loc, baseScaled, diffScaled);
+            arith::AddIOp::create(rewriter, loc, baseScaled, diffScaled);
 
-        rewriter.create<linalg::YieldOp>(loc, result);
+        linalg::YieldOp::create(rewriter, loc, result);
 
         return success();
       }
@@ -2532,8 +2532,8 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static OpFoldResult halfPlusOne(OpBuilder &builder, Location loc,
                                   OpFoldResult ofr) {
-    auto one = builder.create<arith::ConstantIndexOp>(loc, 1);
-    auto two = builder.create<arith::ConstantIndexOp>(loc, 2);
+    auto one = arith::ConstantIndexOp::create(builder, loc, 1);
+    auto two = arith::ConstantIndexOp::create(builder, loc, 2);
 
     auto value = getValueOrCreateConstantIndexOp(builder, loc, ofr);
     auto divBy2 = builder.createOrFold<arith::DivUIOp>(loc, value, two);
@@ -2562,9 +2562,9 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
                                 RankedTensorType type,
                                 llvm::ArrayRef<Value> dynamicSizes) {
     auto emptyTensor =
-        rewriter.create<tensor::EmptyOp>(loc, type, dynamicSizes);
+        tensor::EmptyOp::create(rewriter, loc, type, dynamicSizes);
     auto fillValueAttr = rewriter.getZeroAttr(type.getElementType());
-    auto fillValue = rewriter.create<arith::ConstantOp>(loc, fillValueAttr);
+    auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
     auto filledTensor = rewriter
                             .create<linalg::FillOp>(loc, ValueRange{fillValue},
                                                     ValueRange{emptyTensor})
@@ -2574,18 +2574,18 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static Value castIndexToFloat(OpBuilder &builder, Location loc,
                                 FloatType type, Value value) {
-    auto integerVal = builder.create<arith::IndexCastUIOp>(
+    auto integerVal = arith::IndexCastUIOp::create(builder,
         loc,
         type.getIntOrFloatBitWidth() > 32 ? builder.getI64Type()
                                           : builder.getI32Type(),
         value);
 
-    return builder.create<arith::UIToFPOp>(loc, type, integerVal);
+    return arith::UIToFPOp::create(builder, loc, type, integerVal);
   }
 
   static Value createLinalgIndex(OpBuilder &builder, Location loc,
                                  FloatType type, int64_t index) {
-    auto indexVal = builder.create<linalg::IndexOp>(loc, index);
+    auto indexVal = linalg::IndexOp::create(builder, loc, index);
     return castIndexToFloat(builder, loc, type, indexVal);
   }
 
@@ -2640,7 +2640,7 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
     // Constants and dimension sizes
     auto twoPiAttr = rewriter.getFloatAttr(elementType, 6.283185307179586);
-    auto twoPi = rewriter.create<arith::ConstantOp>(loc, twoPiAttr);
+    auto twoPi = arith::ConstantOp::create(rewriter, loc, twoPiAttr);
     auto constH = castIndexToFloat(rewriter, loc, elementType, dimH);
     auto constW = castIndexToFloat(rewriter, loc, elementType, dimW);
 
@@ -2650,43 +2650,43 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
       Value sumImag = args[2];
 
       // Indices for angle computation
-      Value oy = builder.create<linalg::IndexOp>(loc, 1);
-      Value ox = builder.create<linalg::IndexOp>(loc, 2);
-      Value iy = builder.create<linalg::IndexOp>(loc, 3);
-      Value ix = builder.create<linalg::IndexOp>(loc, 4);
+      Value oy = linalg::IndexOp::create(builder, loc, 1);
+      Value ox = linalg::IndexOp::create(builder, loc, 2);
+      Value iy = linalg::IndexOp::create(builder, loc, 3);
+      Value ix = linalg::IndexOp::create(builder, loc, 4);
 
       // Calculating angle without integer parts of components as sin/cos are
       // periodic: angle = 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix * ox) % W )
       // / W);
-      auto iyXoy = builder.create<index::MulOp>(loc, iy, oy);
-      auto ixXox = builder.create<index::MulOp>(loc, ix, ox);
+      auto iyXoy = index::MulOp::create(builder, loc, iy, oy);
+      auto ixXox = index::MulOp::create(builder, loc, ix, ox);
 
-      auto iyRem = builder.create<index::RemUOp>(loc, iyXoy, dimH);
-      auto ixRem = builder.create<index::RemUOp>(loc, ixXox, dimW);
+      auto iyRem = index::RemUOp::create(builder, loc, iyXoy, dimH);
+      auto ixRem = index::RemUOp::create(builder, loc, ixXox, dimW);
 
       auto iyRemFloat = castIndexToFloat(builder, loc, elementType, iyRem);
       auto ixRemFloat = castIndexToFloat(builder, loc, elementType, ixRem);
 
-      auto yComponent = builder.create<arith::DivFOp>(loc, iyRemFloat, constH);
-      auto xComponent = builder.create<arith::DivFOp>(loc, ixRemFloat, constW);
-      auto sumXY = builder.create<arith::AddFOp>(loc, yComponent, xComponent);
-      auto angle = builder.create<arith::MulFOp>(loc, twoPi, sumXY);
+      auto yComponent = arith::DivFOp::create(builder, loc, iyRemFloat, constH);
+      auto xComponent = arith::DivFOp::create(builder, loc, ixRemFloat, constW);
+      auto sumXY = arith::AddFOp::create(builder, loc, yComponent, xComponent);
+      auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       // realComponent = valReal * cos(angle)
       // imagComponent = valReal * sin(angle)
-      auto cosAngle = builder.create<math::CosOp>(loc, angle);
-      auto sinAngle = builder.create<math::SinOp>(loc, angle);
+      auto cosAngle = math::CosOp::create(builder, loc, angle);
+      auto sinAngle = math::SinOp::create(builder, loc, angle);
       auto realComponent =
-          builder.create<arith::MulFOp>(loc, valReal, cosAngle);
+          arith::MulFOp::create(builder, loc, valReal, cosAngle);
       auto imagComponent =
-          builder.create<arith::MulFOp>(loc, valReal, sinAngle);
+          arith::MulFOp::create(builder, loc, valReal, sinAngle);
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = builder.create<arith::AddFOp>(loc, sumReal, realComponent);
-      auto outImag = builder.create<arith::SubFOp>(loc, sumImag, imagComponent);
+      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag = arith::SubFOp::create(builder, loc, sumImag, imagComponent);
 
-      builder.create<linalg::YieldOp>(loc, ValueRange{outReal, outImag});
+      linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
 
     rewriter.replaceOpWithNewOp<linalg::GenericOp>(
@@ -2760,7 +2760,7 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
 
     // Constants and dimension sizes
     auto twoPiAttr = rewriter.getFloatAttr(real_el_ty, 6.283185307179586);
-    auto twoPi = rewriter.create<arith::ConstantOp>(loc, twoPiAttr);
+    auto twoPi = arith::ConstantOp::create(rewriter, loc, twoPiAttr);
     Value constH =
         RFFT2dConverter::castIndexToFloat(rewriter, loc, real_el_ty, dimH);
     Value constW =
@@ -2773,57 +2773,57 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
       Value sumImag = args[3];
 
       // Indices for angle computation
-      Value oy = builder.create<linalg::IndexOp>(loc, 1);
-      Value ox = builder.create<linalg::IndexOp>(loc, 2);
-      Value iy = builder.create<linalg::IndexOp>(loc, 3);
-      Value ix = builder.create<linalg::IndexOp>(loc, 4);
+      Value oy = linalg::IndexOp::create(builder, loc, 1);
+      Value ox = linalg::IndexOp::create(builder, loc, 2);
+      Value iy = linalg::IndexOp::create(builder, loc, 3);
+      Value ix = linalg::IndexOp::create(builder, loc, 4);
 
       // float_t angle = sign_val * 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix *
       // ox) % W ) / W);
-      auto iyXoy = builder.create<index::MulOp>(loc, iy, oy);
-      auto ixXox = builder.create<index::MulOp>(loc, ix, ox);
+      auto iyXoy = index::MulOp::create(builder, loc, iy, oy);
+      auto ixXox = index::MulOp::create(builder, loc, ix, ox);
 
-      auto iyRem = builder.create<index::RemUOp>(loc, iyXoy, dimH);
-      auto ixRem = builder.create<index::RemUOp>(loc, ixXox, dimW);
+      auto iyRem = index::RemUOp::create(builder, loc, iyXoy, dimH);
+      auto ixRem = index::RemUOp::create(builder, loc, ixXox, dimW);
 
       auto iyRemFloat =
           RFFT2dConverter::castIndexToFloat(builder, loc, real_el_ty, iyRem);
       auto ixRemFloat =
           RFFT2dConverter::castIndexToFloat(builder, loc, real_el_ty, ixRem);
 
-      auto yComponent = builder.create<arith::DivFOp>(loc, iyRemFloat, constH);
-      auto xComponent = builder.create<arith::DivFOp>(loc, ixRemFloat, constW);
+      auto yComponent = arith::DivFOp::create(builder, loc, iyRemFloat, constH);
+      auto xComponent = arith::DivFOp::create(builder, loc, ixRemFloat, constW);
 
-      auto sumXY = builder.create<arith::AddFOp>(loc, yComponent, xComponent);
-      auto angle = builder.create<arith::MulFOp>(loc, twoPi, sumXY);
+      auto sumXY = arith::AddFOp::create(builder, loc, yComponent, xComponent);
+      auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       if (inverse.getValue()) {
-        angle = builder.create<arith::MulFOp>(
+        angle = arith::MulFOp::create(builder,
             loc, angle,
-            rewriter.create<arith::ConstantOp>(
+            arith::ConstantOp::create(rewriter,
                 loc, rewriter.getFloatAttr(real_el_ty, -1.0)));
       }
 
       // realComponent = val_real * cos(a) + val_imag * sin(a);
       // imagComponent = -val_real * sin(a) + val_imag * cos(a);
-      auto cosAngle = builder.create<math::CosOp>(loc, angle);
-      auto sinAngle = builder.create<math::SinOp>(loc, angle);
+      auto cosAngle = math::CosOp::create(builder, loc, angle);
+      auto sinAngle = math::SinOp::create(builder, loc, angle);
 
-      auto rcos = builder.create<arith::MulFOp>(loc, valReal, cosAngle);
-      auto rsin = builder.create<arith::MulFOp>(loc, valImag, sinAngle);
-      auto realComponent = builder.create<arith::AddFOp>(loc, rcos, rsin);
+      auto rcos = arith::MulFOp::create(builder, loc, valReal, cosAngle);
+      auto rsin = arith::MulFOp::create(builder, loc, valImag, sinAngle);
+      auto realComponent = arith::AddFOp::create(builder, loc, rcos, rsin);
 
-      auto icos = builder.create<arith::MulFOp>(loc, valImag, cosAngle);
-      auto isin = builder.create<arith::MulFOp>(loc, valReal, sinAngle);
+      auto icos = arith::MulFOp::create(builder, loc, valImag, cosAngle);
+      auto isin = arith::MulFOp::create(builder, loc, valReal, sinAngle);
 
-      auto imagComponent = builder.create<arith::SubFOp>(loc, icos, isin);
+      auto imagComponent = arith::SubFOp::create(builder, loc, icos, isin);
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = builder.create<arith::AddFOp>(loc, sumReal, realComponent);
-      auto outImag = builder.create<arith::AddFOp>(loc, sumImag, imagComponent);
+      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag = arith::AddFOp::create(builder, loc, sumImag, imagComponent);
 
-      builder.create<linalg::YieldOp>(loc, ValueRange{outReal, outImag});
+      linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
 
     rewriter.replaceOpWithNewOp<linalg::GenericOp>(
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index b89fde4fbc17e..d81b8645506c4 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -52,9 +52,9 @@ static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
     highIndices.push_back(rewriter.getIndexAttr(highPad));
   }
 
-  Value padValue = rewriter.create<arith::ConstantOp>(loc, padAttr);
+  Value padValue = arith::ConstantOp::create(rewriter, loc, padAttr);
 
-  return rewriter.create<tensor::PadOp>(
+  return tensor::PadOp::create(rewriter,
       loc, RankedTensorType::get(paddedShape, inputETy), input, lowIndices,
       highIndices, padValue);
 }
@@ -72,10 +72,10 @@ linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias,
             Value biasVal = args[0];
             Type resType = args[1].getType();
             if (resType != biasVal.getType()) {
-              biasVal = builder.create<arith::ExtSIOp>(loc, resType, biasVal);
+              biasVal = arith::ExtSIOp::create(builder, loc, resType, biasVal);
             }
-            Value added = builder.create<arith::AddIOp>(loc, biasVal, args[1]);
-            builder.create<linalg::YieldOp>(loc, added);
+            Value added = arith::AddIOp::create(builder, loc, biasVal, args[1]);
+            linalg::YieldOp::create(builder, loc, added);
           })
       .getResult(0);
 }
@@ -134,19 +134,19 @@ static mlir::Value linalgBroadcastAndMaybeExt(PatternRewriter &rewriter,
             if (resType != biasVal.getType()) {
               biasVal =
                   resultTy.getElementType().isFloat()
-                      ? builder.create<arith::ExtFOp>(loc, resType, biasVal)
+                      ? arith::ExtFOp::create(builder, loc, resType, biasVal)
                             .getResult()
-                      : builder.create<arith::ExtSIOp>(loc, resType, biasVal)
+                      : arith::ExtSIOp::create(builder, loc, resType, biasVal)
                             .getResult();
             }
-            builder.create<linalg::YieldOp>(loc, biasVal);
+            linalg::YieldOp::create(builder, loc, biasVal);
           })
       .getResult(0);
 }
 
 static mlir::Value reifyConstantDim(int64_t attr,
                                     ImplicitLocOpBuilder &builder) {
-  return builder.create<arith::ConstantIndexOp>(attr);
+  return arith::ConstantIndexOp::create(builder, attr);
 }
 
 // Calculating the output width/height using the formula:
@@ -160,22 +160,22 @@ static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim,
                                           int64_t dilationAttr,
                                           OpBuilder &rewriter) {
   ImplicitLocOpBuilder builder(loc, rewriter);
-  auto one = rewriter.create<arith::ConstantOp>(
+  auto one = arith::ConstantOp::create(rewriter,
       loc, IntegerAttr::get(inputDim.getType(), 1));
   Value padBefore = reifyConstantDim(padBeforeAttr, builder);
-  Value paddedBefore = builder.create<arith::AddIOp>(inputDim, padBefore);
+  Value paddedBefore = arith::AddIOp::create(builder, inputDim, padBefore);
   Value padAfter = reifyConstantDim(padAfterAttr, builder);
-  Value paddedAfter = builder.create<arith::AddIOp>(paddedBefore, padAfter);
+  Value paddedAfter = arith::AddIOp::create(builder, paddedBefore, padAfter);
 
-  Value subOne = builder.create<arith::SubIOp>(kernelDim, one);
+  Value subOne = arith::SubIOp::create(builder, kernelDim, one);
   Value dilation = reifyConstantDim(dilationAttr, builder);
-  Value dilated = builder.create<arith::MulIOp>(dilation, subOne);
-  Value addOne = builder.create<arith::AddIOp>(dilated, one);
+  Value dilated = arith::MulIOp::create(builder, dilation, subOne);
+  Value addOne = arith::AddIOp::create(builder, dilated, one);
 
-  Value subtract = builder.create<arith::SubIOp>(paddedAfter, addOne);
+  Value subtract = arith::SubIOp::create(builder, paddedAfter, addOne);
   Value stride = reifyConstantDim(strideAttr, builder);
-  Value divide = builder.create<arith::DivUIOp>(subtract, stride);
-  return builder.create<arith::AddIOp>(divide, one);
+  Value divide = arith::DivUIOp::create(builder, subtract, stride);
+  return arith::AddIOp::create(builder, divide, one);
 }
 
 // Creates a vector of the dynamic output dims for Conv2D and Depthwise_Conv2D
@@ -198,9 +198,9 @@ static SmallVector<Value> inferDynamicDimsForConv(
       auto padBottom = padAttr[i * 2 + 1];
       auto stride = strideAttr[i];
       auto dilation = dilationAttr[i];
-      Value initDynDim = rewriter.create<tensor::DimOp>(loc, input, inputDim);
+      Value initDynDim = tensor::DimOp::create(rewriter, loc, input, inputDim);
       Value kernelDynDim =
-          rewriter.create<tensor::DimOp>(loc, weight, kernelDim);
+          tensor::DimOp::create(rewriter, loc, weight, kernelDim);
       // H = F(IH, pad_top, pad_bottom, dilation_y, KH, stride_y)
       dynDims[inputDim] =
           getConvOrPoolOutputDim(loc, initDynDim, padTop, padBottom,
@@ -211,7 +211,7 @@ static SmallVector<Value> inferDynamicDimsForConv(
   // Get the batch/channels dimensions.
   for (int i = 0; i < inputRank; i++) {
     if (resultTy.isDynamicDim(i) && !dynDims[i])
-      dynDims[i] = rewriter.create<tensor::DimOp>(loc, input, i);
+      dynDims[i] = tensor::DimOp::create(rewriter, loc, input, i);
   }
 
   SmallVector<Value> filteredDims = condenseValues(dynDims);
@@ -350,7 +350,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
         auto weightPermAttr = rewriter.getDenseI32ArrayAttr(weightPerm);
         Type newWeightTy =
             RankedTensorType::get(newWeightShape, weightTy.getElementType());
-        weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
+        weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
                                                     weightPermAttr);
       }
     }
@@ -372,7 +372,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       auto weightPermAttr = rewriter.getDenseI32ArrayAttr(weightPerm);
       Type newWeightTy =
           RankedTensorType::get(newWeightShape, weightTy.getElementType());
-      weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
+      weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
                                                   weightPermAttr);
     }
 
@@ -384,7 +384,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     auto strideAttr = rewriter.getI64TensorAttr(stride);
     auto dilationAttr = rewriter.getI64TensorAttr(dilation);
 
-    Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), accETy, filteredDims);
 
     Value broadcastBias =
@@ -394,8 +394,8 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       auto iZp = rewriter.getI32IntegerAttr(inputZpVal);
       auto kZp = rewriter.getI32IntegerAttr(weightZpVal);
 
-      auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
+      auto iZpVal = arith::ConstantOp::create(rewriter, loc, iZp);
+      auto kZpVal = arith::ConstantOp::create(rewriter, loc, kZp);
 
       Value conv =
           rewriter
@@ -417,7 +417,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     // We may need to truncate back to the result type if the accumulator was
     // wider than the result.
     if (resultTy != accTy)
-      conv = rewriter.create<tosa::CastOp>(loc, resultTy, conv);
+      conv = tosa::CastOp::create(rewriter, loc, resultTy, conv);
 
     rewriter.replaceOp(op, conv);
     return success();
@@ -526,15 +526,15 @@ class DepthwiseConvConverter
                               accETy);
 
     auto resultZeroAttr = rewriter.getZeroAttr(accETy);
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, linalgConvTy.getShape(), accETy, filteredDims);
-    Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
+    Value zero = arith::ConstantOp::create(rewriter, loc, resultZeroAttr);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
                            .result();
 
-    Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultETy, filteredDims);
 
     // Broadcast the initial value to the output tensor before convolving.
@@ -553,7 +553,7 @@ class DepthwiseConvConverter
       // We may need to truncate back to the result type if the accumulator was
       // wider than the result.
       if (accETy != resultETy)
-        conv = rewriter.create<tosa::CastOp>(
+        conv = tosa::CastOp::create(rewriter,
             loc,
             RankedTensorType::get(cast<ShapedType>(conv.getType()).getShape(),
                                   resultETy),
@@ -561,7 +561,7 @@ class DepthwiseConvConverter
 
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
+      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
           loc, resultTy, conv, reassociationMap);
 
       Value result =
@@ -574,20 +574,20 @@ class DepthwiseConvConverter
                       ValueRange args) {
                     Value added;
                     if (llvm::isa<FloatType>(inputETy))
-                      added = nestedBuilder.create<arith::AddFOp>(loc, args[0],
+                      added = arith::AddFOp::create(nestedBuilder, loc, args[0],
                                                                   args[1]);
                     else
-                      added = nestedBuilder.create<arith::AddIOp>(loc, args[0],
+                      added = arith::AddIOp::create(nestedBuilder, loc, args[0],
                                                                   args[1]);
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
+                    linalg::YieldOp::create(nestedBuilder, nestedLoc, added);
                   })
               .getResult(0);
       rewriter.replaceOp(op, result);
     } else {
       IntegerAttr iZp = rewriter.getI32IntegerAttr(inputZpVal);
       IntegerAttr wZp = rewriter.getI32IntegerAttr(weightZpVal);
-      auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, wZp);
+      auto iZpVal = arith::ConstantOp::create(rewriter, loc, iZp);
+      auto kZpVal = arith::ConstantOp::create(rewriter, loc, wZp);
       Value conv =
           rewriter
               .create<linalg::DepthwiseConv2DNhwcHwcmQOp>(
@@ -596,7 +596,7 @@ class DepthwiseConvConverter
               .getResult(0);
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
+      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
           loc, resultTy, conv, reassociationMap);
       Value result = linalgIntBroadcastExtSIAdd(
           rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
@@ -621,22 +621,22 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
     dynDims.resize(cast<ShapedType>(op->getResult(0).getType()).getRank());
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(0)) {
-      dynDims[0] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 0);
+      dynDims[0] = tensor::DimOp::create(rewriter, loc, op->getOperand(0), 0);
     }
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(1)) {
-      dynDims[1] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 1);
+      dynDims[1] = tensor::DimOp::create(rewriter, loc, op->getOperand(0), 1);
     }
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(2)) {
-      dynDims[2] = rewriter.create<tensor::DimOp>(loc, op->getOperand(1), 2);
+      dynDims[2] = tensor::DimOp::create(rewriter, loc, op->getOperand(1), 2);
     }
 
     SmallVector<Value> filteredDims = condenseValues(dynDims);
 
     auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
-    Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
@@ -670,9 +670,9 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
       return success();
     }
 
-    auto aZp = rewriter.create<arith::ConstantOp>(
+    auto aZp = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI32IntegerAttr(aZpVal));
-    auto bZp = rewriter.create<arith::ConstantOp>(
+    auto bZp = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI32IntegerAttr(bZpVal));
     rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
         op, TypeRange{op.getType()},
@@ -702,7 +702,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     // Batch dimension
     if (resultTy.isDynamicDim(0))
-      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+      dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
     // Height/width dimensions
     for (int64_t dim : {1, 2}) {
@@ -713,10 +713,10 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       int64_t index = dim - 1;
 
       // Input height/width
-      Value ihw = rewriter.create<tensor::DimOp>(loc, input, dim);
+      Value ihw = tensor::DimOp::create(rewriter, loc, input, dim);
 
       // Kernel height/width
-      Value khw = rewriter.create<arith::ConstantIndexOp>(loc, kernel[index]);
+      Value khw = arith::ConstantIndexOp::create(rewriter, loc, kernel[index]);
 
       // Output height/width
       Value ohw = getConvOrPoolOutputDim(loc, ihw, pad[index * 2],
@@ -727,7 +727,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     // Channel dimension
     if (resultTy.isDynamicDim(3))
-      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 3));
+      dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 3));
 
     return dynamicDims;
   }
@@ -776,7 +776,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     Value paddedInput = applyPad(loc, input, pad, initialAttr, rewriter);
 
-    Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
+    Value initialValue = arith::ConstantOp::create(rewriter, loc, initialAttr);
 
     ArrayRef<int64_t> kernel = op.getKernel();
     ArrayRef<int64_t> stride = op.getStride();
@@ -785,15 +785,15 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
 
     Value filledEmptyTensor =
-        rewriter.create<linalg::FillOp>(loc, initialValue, emptyTensor)
+        linalg::FillOp::create(rewriter, loc, initialValue, emptyTensor)
             .result();
 
     Value fakeWindowDims =
-        rewriter.create<tensor::EmptyOp>(loc, kernel, resultETy);
+        tensor::EmptyOp::create(rewriter, loc, kernel, resultETy);
 
     if (isUnsigned) {
       rewriter.replaceOpWithNewOp<linalg::PoolingNhwcMaxUnsignedOp>(
@@ -802,7 +802,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       return llvm::success();
     }
 
-    auto resultOp = rewriter.create<linalg::PoolingNhwcMaxOp>(
+    auto resultOp = linalg::PoolingNhwcMaxOp::create(rewriter,
         op->getLoc(), ArrayRef<Type>{resultTy},
         ValueRange{paddedInput, fakeWindowDims}, filledEmptyTensor, strideAttr,
         dilationAttr);
@@ -821,7 +821,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     // it to include the appropriate checks. If the current value is NaN the
     // old value of pool will be taken otherwise we use the result.
     if (const auto nanMode = op.getNanMode(); nanMode == "IGNORE") {
-      auto genericOp = rewriter.create<linalg::GenericOp>(
+      auto genericOp = linalg::GenericOp::create(rewriter,
           op->getLoc(), resultOp.getType(0), resultOp.getInputs(),
           resultOp.getOutputs(), resultOp.getIndexingMapsArray(),
           resultOp.getIteratorTypesArray(),
@@ -832,12 +832,12 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
             auto &oldMaxOp = *resultOp.getBlock()->begin();
             map.map(oldArgs, blockArgs);
             auto *newOp = opBuilder.clone(oldMaxOp, map);
-            Value isNaN = opBuilder.create<arith::CmpFOp>(
+            Value isNaN = arith::CmpFOp::create(opBuilder,
                 op->getLoc(), arith::CmpFPredicate::UNO, blockArgs.front(),
                 blockArgs.front());
-            auto selectOp = opBuilder.create<arith::SelectOp>(
+            auto selectOp = arith::SelectOp::create(opBuilder,
                 op->getLoc(), isNaN, blockArgs.back(), newOp->getResult(0));
-            opBuilder.create<linalg::YieldOp>(loc, selectOp.getResult());
+            linalg::YieldOp::create(opBuilder, loc, selectOp.getResult());
           });
       rewriter.replaceOp(resultOp, genericOp);
     }
@@ -893,7 +893,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Value paddedInput = applyPad(loc, input, pad, padAttr, rewriter);
 
     auto initialAttr = rewriter.getZeroAttr(accETy);
-    Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
+    Value initialValue = arith::ConstantOp::create(rewriter, loc, initialAttr);
 
     ArrayRef<int64_t> kernel = op.getKernel();
     ArrayRef<int64_t> stride = op.getStride();
@@ -902,7 +902,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value poolEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value poolEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, accTy.getShape(), accETy, dynamicDims);
 
     Value filledEmptyTensor =
@@ -912,7 +912,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             .result();
 
     Value fakeWindowDims =
-        rewriter.create<tensor::EmptyOp>(loc, kernel, accETy);
+        tensor::EmptyOp::create(rewriter, loc, kernel, accETy);
 
     // Sum across the pooled region.
     Value poolingOp = rewriter
@@ -924,24 +924,24 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
 
     // Normalize the summed value by the number of elements grouped in each
     // pool.
-    Value iH = rewriter.create<tensor::DimOp>(loc, poolingOp, 1);
-    Value iW = rewriter.create<tensor::DimOp>(loc, poolingOp, 2);
+    Value iH = tensor::DimOp::create(rewriter, loc, poolingOp, 1);
+    Value iW = tensor::DimOp::create(rewriter, loc, poolingOp, 2);
 
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    iH = rewriter.create<arith::SubIOp>(loc, iH, one);
-    iW = rewriter.create<arith::SubIOp>(loc, iW, one);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    iH = arith::SubIOp::create(rewriter, loc, iH, one);
+    iW = arith::SubIOp::create(rewriter, loc, iW, one);
 
-    Value genericEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value genericEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultETy, dynamicDims);
 
     auto affineMap = rewriter.getMultiDimIdentityMap(resultTy.getRank());
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
         ValueRange{genericEmptyTensor},
         ArrayRef<AffineMap>({affineMap, affineMap}),
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
-          auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+          auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
           // Determines what the portion of valid input is covered by the
           // kernel.
@@ -949,30 +949,30 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             if (pad == 0)
               return valid;
 
-            auto padVal = rewriter.create<arith::ConstantIndexOp>(loc, pad);
-            Value dpos = rewriter.create<arith::SubIOp>(loc, pos, padVal);
+            auto padVal = arith::ConstantIndexOp::create(rewriter, loc, pad);
+            Value dpos = arith::SubIOp::create(rewriter, loc, pos, padVal);
 
-            Value offset = rewriter.create<arith::MinSIOp>(loc, dpos, zero);
-            return rewriter.create<arith::AddIOp>(loc, valid, offset)
+            Value offset = arith::MinSIOp::create(rewriter, loc, dpos, zero);
+            return arith::AddIOp::create(rewriter, loc, valid, offset)
                 ->getResult(0);
           };
 
           auto coverageFn = [&](int64_t i, Value isize) -> Value {
             Value strideVal =
-                rewriter.create<arith::ConstantIndexOp>(loc, stride[i - 1]);
+                arith::ConstantIndexOp::create(rewriter, loc, stride[i - 1]);
             Value val =
-                rewriter.create<arith::ConstantIndexOp>(loc, kernel[i - 1]);
+                arith::ConstantIndexOp::create(rewriter, loc, kernel[i - 1]);
 
             // Find the position relative to the input tensor's ends.
-            Value left = rewriter.create<linalg::IndexOp>(loc, i);
-            Value right = rewriter.create<arith::SubIOp>(loc, isize, left);
-            left = rewriter.create<arith::MulIOp>(loc, left, strideVal);
-            right = rewriter.create<arith::MulIOp>(loc, right, strideVal);
+            Value left = linalg::IndexOp::create(rewriter, loc, i);
+            Value right = arith::SubIOp::create(rewriter, loc, isize, left);
+            left = arith::MulIOp::create(rewriter, loc, left, strideVal);
+            right = arith::MulIOp::create(rewriter, loc, right, strideVal);
 
             // Determine how much padding was included.
             val = padFn(val, left, pad[i * 2]);
             val = padFn(val, right, pad[i * 2 + 1]);
-            return rewriter.create<arith::MaxSIOp>(loc, one, val);
+            return arith::MaxSIOp::create(rewriter, loc, one, val);
           };
 
           // Compute the indices from either end.
@@ -980,70 +980,70 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
           Value kW3 = coverageFn(2, iW);
 
           // Compute the total number of elements and normalize.
-          auto count = rewriter.create<arith::IndexCastOp>(
+          auto count = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getI32Type(),
-              rewriter.create<arith::MulIOp>(loc, kH3, kW3));
+              arith::MulIOp::create(rewriter, loc, kH3, kW3));
 
           // Divide by the number of summed values. For floats this is just
           // a div however for quantized values input normalization had
           // to be applied.
           Value poolVal = args[0];
           if (isa<FloatType>(accETy)) {
-            auto countF = rewriter.create<arith::SIToFPOp>(loc, accETy, count);
-            poolVal = rewriter.create<arith::DivFOp>(loc, poolVal, countF)
+            auto countF = arith::SIToFPOp::create(rewriter, loc, accETy, count);
+            poolVal = arith::DivFOp::create(rewriter, loc, poolVal, countF)
                           ->getResult(0);
             if (accETy.getIntOrFloatBitWidth() >
                 resultETy.getIntOrFloatBitWidth())
               poolVal =
-                  rewriter.create<arith::TruncFOp>(loc, resultETy, poolVal);
+                  arith::TruncFOp::create(rewriter, loc, resultETy, poolVal);
           } else {
 
             // If we have quantization information we need to apply an offset
             // for the input zp value.
             if (inputZpVal != 0) {
-              auto inputZp = rewriter.create<arith::ConstantOp>(
+              auto inputZp = arith::ConstantOp::create(rewriter,
                   loc, b.getIntegerAttr(accETy, inputZpVal));
               Value offset =
-                  rewriter.create<arith::MulIOp>(loc, accETy, count, inputZp);
+                  arith::MulIOp::create(rewriter, loc, accETy, count, inputZp);
               poolVal =
-                  rewriter.create<arith::SubIOp>(loc, accETy, poolVal, offset);
+                  arith::SubIOp::create(rewriter, loc, accETy, poolVal, offset);
             }
 
             // Compute: k = 32 - count_leading_zeros(value - 1)
-            Value one32 = rewriter.create<arith::ConstantOp>(
+            Value one32 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI32IntegerAttr(1));
-            Value thirtyTwo32 = rewriter.create<arith::ConstantOp>(
+            Value thirtyTwo32 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI32IntegerAttr(32));
 
             Value countSubOne =
-                rewriter.create<arith::SubIOp>(loc, count, one32);
+                arith::SubIOp::create(rewriter, loc, count, one32);
             Value leadingZeros =
-                rewriter.create<math::CountLeadingZerosOp>(loc, countSubOne);
+                math::CountLeadingZerosOp::create(rewriter, loc, countSubOne);
             Value k =
-                rewriter.create<arith::SubIOp>(loc, thirtyTwo32, leadingZeros);
+                arith::SubIOp::create(rewriter, loc, thirtyTwo32, leadingZeros);
 
             // Compute: numerator = ((1 << 30) + 1) << k
             Value k64 =
-                rewriter.create<arith::ExtUIOp>(loc, rewriter.getI64Type(), k);
-            Value thirtyShiftPlusOne = rewriter.create<arith::ConstantOp>(
+                arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), k);
+            Value thirtyShiftPlusOne = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
             Value numerator =
-                rewriter.create<arith::ShLIOp>(loc, thirtyShiftPlusOne, k64);
+                arith::ShLIOp::create(rewriter, loc, thirtyShiftPlusOne, k64);
 
             // Compute: scale.multiplier = numerator / value;
-            Value count64 = rewriter.create<arith::ExtUIOp>(
+            Value count64 = arith::ExtUIOp::create(rewriter,
                 loc, rewriter.getI64Type(), count);
             Value multiplier =
-                rewriter.create<arith::DivUIOp>(loc, numerator, count64);
-            multiplier = rewriter.create<arith::TruncIOp>(
+                arith::DivUIOp::create(rewriter, loc, numerator, count64);
+            multiplier = arith::TruncIOp::create(rewriter,
                 loc, rewriter.getI32Type(), multiplier);
 
             // Compute: scale.shift = 30 + k
             Value k8 =
-                rewriter.create<arith::TruncIOp>(loc, rewriter.getI8Type(), k);
-            Value thirty8 = rewriter.create<arith::ConstantOp>(
+                arith::TruncIOp::create(rewriter, loc, rewriter.getI8Type(), k);
+            Value thirty8 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI8IntegerAttr(30));
-            Value shift = rewriter.create<arith::AddIOp>(loc, k8, thirty8);
+            Value shift = arith::AddIOp::create(rewriter, loc, k8, thirty8);
 
             auto scaled =
                 rewriter
@@ -1055,19 +1055,19 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // If we have quantization information we need to apply output
             // zeropoint.
             if (outputZpVal != 0) {
-              auto outputZp = rewriter.create<arith::ConstantOp>(
+              auto outputZp = arith::ConstantOp::create(rewriter,
                   loc, b.getIntegerAttr(scaled.getType(), outputZpVal));
-              scaled = rewriter.create<arith::AddIOp>(loc, scaled, outputZp)
+              scaled = arith::AddIOp::create(rewriter, loc, scaled, outputZp)
                            .getResult();
             }
 
             // Apply Clip.
             int64_t outBitwidth = resultETy.getIntOrFloatBitWidth();
 
-            auto min = rewriter.create<arith::ConstantIntOp>(
+            auto min = arith::ConstantIntOp::create(rewriter,
                 loc, accETy,
                 APInt::getSignedMinValue(outBitwidth).getSExtValue());
-            auto max = rewriter.create<arith::ConstantIntOp>(
+            auto max = arith::ConstantIntOp::create(rewriter,
                 loc, accETy,
                 APInt::getSignedMaxValue(outBitwidth).getSExtValue());
             auto clamp = clampIntHelper(loc, scaled, min, max, rewriter,
@@ -1077,11 +1077,11 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // Convert type.
             if (resultETy != clamp.getType()) {
               poolVal =
-                  rewriter.create<arith::TruncIOp>(loc, resultETy, poolVal);
+                  arith::TruncIOp::create(rewriter, loc, resultETy, poolVal);
             }
           }
 
-          rewriter.create<linalg::YieldOp>(loc, poolVal);
+          linalg::YieldOp::create(rewriter, loc, poolVal);
         });
 
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -1106,7 +1106,7 @@ class TransposeConverter : public OpRewritePattern<tosa::TransposeOp> {
     auto permutedSizes =
         applyTOSAPermutation<OpFoldResult>(inputSizes, constantPerms);
 
-    auto permutedInit = rewriter.create<tensor::EmptyOp>(
+    auto permutedInit = tensor::EmptyOp::create(rewriter,
         loc, permutedSizes, op.getInput1().getType().getElementType());
     rewriter.replaceOpWithNewOp<linalg::TransposeOp>(
         op, op.getInput1(), permutedInit,
diff --git a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
index 7dbccd19a0518..cd36aadce410a 100644
--- a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
+++ b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
@@ -27,7 +27,7 @@ class VariableOpConverter : public OpRewritePattern<tosa::VariableOp> {
   LogicalResult matchAndRewrite(tosa::VariableOp op,
                                 PatternRewriter &rewriter) const final {
     auto variableType = tosa::getVariableType(op);
-    auto newVariable = rewriter.create<mlir::ml_program::GlobalOp>(
+    auto newVariable = mlir::ml_program::GlobalOp::create(rewriter,
         op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
         op.getInitialValueAttr(), /*sym_visibility=*/nullptr);
     newVariable.setPrivate();
@@ -45,7 +45,7 @@ class VariableWriteOpConverter
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableWrite = rewriter.create<ml_program::GlobalStoreOp>(
+    auto newVariableWrite = ml_program::GlobalStoreOp::create(rewriter,
         op.getLoc(), globalSymbolRef, op.getInput1());
     rewriter.replaceOp(op, newVariableWrite);
     return success();
@@ -60,7 +60,7 @@ class VariableReadOpConverter : public OpRewritePattern<tosa::VariableReadOp> {
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableRead = rewriter.create<ml_program::GlobalLoadOp>(
+    auto newVariableRead = ml_program::GlobalLoadOp::create(rewriter,
         op.getLoc(), op.getType(), globalSymbolRef);
     rewriter.replaceOp(op, newVariableRead);
 
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index 03f9d20ad69de..9188b8d1368da 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -30,7 +30,7 @@ static void inlineIfCase(Region &srcRegion, Region &dstRegion,
 
   auto yield = cast<YieldOp>(headBlock->getTerminator());
   rewriter.setInsertionPoint(yield);
-  rewriter.create<scf::YieldOp>(yield.getLoc(), yield.getInputs());
+  scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
   rewriter.eraseOp(yield);
 
   headBlock->eraseArguments(0, headBlock->getNumArguments());
@@ -47,12 +47,12 @@ static void inlineWhileCase(Region &srcRegion, Region &dstRegion,
   rewriter.setInsertionPoint(yield);
   if (isCond) {
     auto condition =
-        rewriter.create<tensor::ExtractOp>(yield.getLoc(), yield.getOperand(0));
-    rewriter.create<scf::ConditionOp>(yield.getLoc(), condition,
+        tensor::ExtractOp::create(rewriter, yield.getLoc(), yield.getOperand(0));
+    scf::ConditionOp::create(rewriter, yield.getLoc(), condition,
                                       headBlock->getArguments());
   } else {
     rewriter.setInsertionPoint(yield);
-    rewriter.create<scf::YieldOp>(yield.getLoc(), yield.getInputs());
+    scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
   }
   rewriter.eraseOp(yield);
 }
@@ -66,8 +66,8 @@ class IfOpConverter : public OpRewritePattern<tosa::IfOp> {
   LogicalResult matchAndRewrite(tosa::IfOp op,
                                 PatternRewriter &rewriter) const final {
     auto condition =
-        rewriter.create<tensor::ExtractOp>(op.getLoc(), op.getCondition());
-    auto newIf = rewriter.create<scf::IfOp>(op.getLoc(), op.getResultTypes(),
+        tensor::ExtractOp::create(rewriter, op.getLoc(), op.getCondition());
+    auto newIf = scf::IfOp::create(rewriter, op.getLoc(), op.getResultTypes(),
                                             condition, true);
 
     inlineIfCase(op.getThenGraph(), newIf.getThenRegion(), op.getInputList(),
@@ -88,7 +88,7 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
 
   static Value createIndexConst(OpBuilder &builder, Location loc,
                                 int64_t value) {
-    return builder.create<arith::ConstantIndexOp>(loc, value);
+    return arith::ConstantIndexOp::create(builder, loc, value);
   }
 
 public:
@@ -119,8 +119,8 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       auto n = ivs[0];
 
       // Read the index and cast it to index type
-      auto index = builder.create<tensor::ExtractOp>(loc, indices, ivs);
-      auto castIndex = builder.create<arith::IndexCastOp>(
+      auto index = tensor::ExtractOp::create(builder, loc, indices, ivs);
+      auto castIndex = arith::IndexCastOp::create(builder,
           loc, builder.getIndexType(), index);
 
       // Offset, sizes, and strides for the input tensor
@@ -130,12 +130,12 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       llvm::SmallVector<Value> sizes = {one, one, dimC};
       llvm::SmallVector<Value> strides = {one, one, one};
 
-      auto slice = builder.create<tensor::ExtractSliceOp>(
+      auto slice = tensor::ExtractSliceOp::create(builder,
           loc, input, inputOffset, sizes, strides);
 
       // Insert the slice into the output accumulator tensor.
       llvm::SmallVector<Value> outputOffset = {n, castIndex, zero};
-      auto updated = builder.create<tensor::InsertSliceOp>(
+      auto updated = tensor::InsertSliceOp::create(builder,
           loc, slice, args[0], outputOffset, sizes, strides);
 
       return {updated};
@@ -155,7 +155,7 @@ class WhileOpConverter : public OpRewritePattern<tosa::WhileOp> {
 
   LogicalResult matchAndRewrite(tosa::WhileOp op,
                                 PatternRewriter &rewriter) const final {
-    auto newWhile = rewriter.create<scf::WhileOp>(
+    auto newWhile = scf::WhileOp::create(rewriter,
         op.getLoc(), op.getResultTypes(), op.getInputList());
     rewriter.createBlock(&newWhile.getBefore());
     rewriter.createBlock(&newWhile.getAfter());
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index 9ceb5c0c7f2fe..e51cbfb702083 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -308,13 +308,13 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
       if (ShapedType::isStatic(sizes.back()))
         continue;
 
-      auto dim = rewriter.create<tensor::DimOp>(loc, input, index);
-      auto offset = rewriter.create<arith::ConstantOp>(
+      auto dim = tensor::DimOp::create(rewriter, loc, input, index);
+      auto offset = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(sliceStarts[index]));
-      dynSizes.push_back(rewriter.create<arith::SubIOp>(loc, dim, offset));
+      dynSizes.push_back(arith::SubIOp::create(rewriter, loc, dim, offset));
     }
 
-    auto newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}), dynSizes,
         ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
         rewriter.getDenseI64ArrayAttr(sizes),
@@ -362,7 +362,7 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
 
     Value padConstant = rewriter.createOrFold<tensor::ExtractOp>(
         loc, padOp.getPadConst(),
-        ValueRange({rewriter.create<arith::ConstantIndexOp>(loc, 0)}));
+        ValueRange({arith::ConstantIndexOp::create(rewriter, loc, 0)}));
 
     if (!padConstant) {
       return rewriter.notifyMatchFailure(
@@ -376,15 +376,15 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
     highValues.reserve(rank);
 
     for (int i = 0; i < rank; i++) {
-      Value lowVal = rewriter.create<arith::ConstantOp>(
+      Value lowVal = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(paddingVals[2 * i]));
-      Value highVal = rewriter.create<arith::ConstantOp>(
+      Value highVal = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(paddingVals[2 * i + 1]));
       lowValues.push_back(lowVal);
       highValues.push_back(highVal);
     }
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, padOp.getType(), input, lowValues, highValues, padConstant);
 
     rewriter.replaceOp(padOp, newPadOp.getResult());
@@ -403,7 +403,7 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
     Location loc = op.getLoc();
     int axis = op.getAxis();
     Value axisValue =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(axis));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(axis));
     int64_t rank = resultType.getRank();
 
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
@@ -440,7 +440,7 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
       }
     }
 
-    Value result = rewriter.create<tensor::EmptyOp>(
+    Value result = tensor::EmptyOp::create(rewriter,
         loc, resultType.getShape(), resultType.getElementType(), dynDims);
 
     for (auto [arg, offset] : llvm::zip(adaptor.getOperands(), axisOffsets)) {
diff --git a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
index d6f9495b2567c..9e4eb347b3689 100644
--- a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
+++ b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
@@ -226,7 +226,7 @@ struct BroadcastOpToArmSMELowering
         (srcVectorType && (srcVectorType.getRank() == 0))) {
       // Broadcast scalar or 0-d vector to 1-d vector.
       VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-      broadcastOp1D = rewriter.create<vector::BroadcastOp>(
+      broadcastOp1D = vector::BroadcastOp::create(rewriter,
           loc, tileSliceType, broadcastOp.getSource());
     } else if (srcVectorType && (srcVectorType.getRank() == 1))
       // Value to broadcast is already a 1-d vector, nothing to do.
@@ -234,13 +234,13 @@ struct BroadcastOpToArmSMELowering
     else
       return failure();
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to broadcast the value
       // to each tile slice.
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
@@ -292,14 +292,14 @@ struct SplatOpToArmSMELowering : public OpRewritePattern<vector::SplatOp> {
 
     // First, broadcast the scalar to a 1-d vector.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    Value broadcastOp1D = rewriter.create<vector::BroadcastOp>(
+    Value broadcastOp1D = vector::BroadcastOp::create(rewriter,
         loc, tileSliceType, splatOp.getInput());
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
@@ -370,21 +370,21 @@ struct TransposeOpToArmSMELowering
 
     // Allocate buffer to store input tile to.
     Value vscale =
-        rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-    Value minTileSlices = rewriter.create<arith::ConstantOp>(
+        vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+    Value minTileSlices = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexAttr(tileType.getDimSize(0)));
     Value c0 =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(0));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(0));
     Value numTileSlices =
-        rewriter.create<arith::MulIOp>(loc, vscale, minTileSlices);
+        arith::MulIOp::create(rewriter, loc, vscale, minTileSlices);
     auto bufferType =
         MemRefType::get({ShapedType::kDynamic, ShapedType::kDynamic},
                         tileType.getElementType());
-    auto buffer = rewriter.create<memref::AllocaOp>(
+    auto buffer = memref::AllocaOp::create(rewriter,
         loc, bufferType, ValueRange{numTileSlices, numTileSlices});
 
     // Store input tile.
-    auto tileStoreOp = rewriter.create<arm_sme::TileStoreOp>(
+    auto tileStoreOp = arm_sme::TileStoreOp::create(rewriter,
         loc, input, buffer, ValueRange{c0, c0});
 
     // Reload input tile vertically.
@@ -489,9 +489,9 @@ struct VectorOuterProductToArmSMELowering
 
     VectorType operandMaskType = VectorType::Builder(maskType).dropDim(0);
     Value lhsMask =
-        rewriter.create<vector::CreateMaskOp>(loc, operandMaskType, lhsMaskDim);
+        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, lhsMaskDim);
     Value rhsMask =
-        rewriter.create<vector::CreateMaskOp>(loc, operandMaskType, rhsMaskDim);
+        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, rhsMaskDim);
 
     return std::make_pair(lhsMask, rhsMask);
   }
@@ -531,7 +531,7 @@ struct VectorExtractToArmSMELowering
     }
 
     Value sliceIndex = vector::getAsValues(rewriter, loc, position[0]).front();
-    auto extractTileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+    auto extractTileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
         loc, sourceVector, sliceIndex);
 
     if (position.size() == 1) {
@@ -593,9 +593,9 @@ struct VectorInsertToArmSMELowering
     if (position.size() == 2) {
       // Two indices case: Insert single element into tile.
       // We need to first extract the existing slice and update the element.
-      tileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+      tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
           loc, insertOp.getDest(), sliceIndex);
-      tileSlice = rewriter.create<vector::InsertOp>(loc, source, tileSlice,
+      tileSlice = vector::InsertOp::create(rewriter, loc, source, tileSlice,
                                                     position[1]);
     }
 
@@ -642,22 +642,22 @@ struct VectorPrintToArmSMELowering : public OpRewritePattern<vector::PrintOp> {
     auto loc = printOp.getLoc();
 
     // Create a loop over the rows of the tile.
-    auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
+    auto vscale = vector::VectorScaleOp::create(rewriter, loc);
     auto minTileRows =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorType.getDimSize(0));
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto upperBound = rewriter.create<arith::MulIOp>(loc, minTileRows, vscale);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        arith::ConstantIndexOp::create(rewriter, loc, vectorType.getDimSize(0));
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto upperBound = arith::MulIOp::create(rewriter, loc, minTileRows, vscale);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     {
       // Loop body.
       rewriter.setInsertionPointToStart(forOp.getBody());
       // Extract the current row from the tile.
       Value rowIndex = forOp.getInductionVar();
-      auto tileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+      auto tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
           loc, printOp.getSource(), rowIndex);
       // Print the row with a 1D vector.print.
-      rewriter.create<vector::PrintOp>(loc, tileSlice,
+      vector::PrintOp::create(rewriter, loc, tileSlice,
                                        printOp.getPunctuation());
     }
 
@@ -707,7 +707,7 @@ struct FoldTransferWriteOfExtractTileSlice
     Value mask = writeOp.getMask();
     if (!mask) {
       auto maskType = writeOp.getVectorType().clone(rewriter.getI1Type());
-      mask = rewriter.create<arith::ConstantOp>(
+      mask = arith::ConstantOp::create(rewriter,
           writeOp.getLoc(), maskType, DenseElementsAttr::get(maskType, true));
     }
 
@@ -776,9 +776,9 @@ struct ExtractFromCreateMaskToPselLowering
     // Create the two 1-D masks at the location of the 2-D create_mask (which is
     // usually outside a loop). This prevents the need for later hoisting.
     rewriter.setInsertionPoint(createMaskOp);
-    auto rowMask = rewriter.create<vector::CreateMaskOp>(
+    auto rowMask = vector::CreateMaskOp::create(rewriter,
         loc, rowMaskType, createMaskOp.getOperand(0));
-    auto colMask = rewriter.create<vector::CreateMaskOp>(
+    auto colMask = vector::CreateMaskOp::create(rewriter,
         loc, colMaskType, createMaskOp.getOperand(1));
 
     rewriter.setInsertionPoint(extractOp);
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 18adaa793787c..5b3e67509672a 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -412,22 +412,22 @@ struct PrepareContractToGPUMMA
     if (maps == infer({{m, k}, {k, n}, {m, n}}))
       return rewriter.notifyMatchFailure(op, "contraction already prepared");
     if (maps == infer({{m, k}, {n, k}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
       std::swap(rhs, lhs);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
       std::swap(rhs, lhs);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
       std::swap(lhs, rhs);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
       std::swap(lhs, rhs);
     } else {
@@ -494,13 +494,13 @@ struct CombineTransferReadOpTranspose final
     // Fuse through the integer extend op.
     if (extOp) {
       if (isa<arith::ExtSIOp>(extOp))
-        result = rewriter.create<arith::ExtSIOp>(loc, op.getType(), result)
+        result = arith::ExtSIOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
       else if (isa<arith::ExtUIOp>(extOp))
-        result = rewriter.create<arith::ExtUIOp>(loc, op.getType(), result)
+        result = arith::ExtUIOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
       else
-        result = rewriter.create<arith::ExtFOp>(loc, op.getType(), result)
+        result = arith::ExtFOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
     }
 
@@ -579,7 +579,7 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op,
   }
   gpu::MMAMatrixType type =
       gpu::MMAMatrixType::get(op.getVectorType().getShape(), elType, fragType);
-  Value load = rewriter.create<gpu::SubgroupMmaLoadMatrixOp>(
+  Value load = gpu::SubgroupMmaLoadMatrixOp::create(rewriter,
       op.getLoc(), type, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride),
       isTranspose ? rewriter.getUnitAttr() : UnitAttr());
@@ -610,7 +610,7 @@ convertTransferWriteOp(RewriterBase &rewriter, vector::TransferWriteOp op,
   }
 
   Value matrix = it->second;
-  auto store = rewriter.create<gpu::SubgroupMmaStoreMatrixOp>(
+  auto store = gpu::SubgroupMmaStoreMatrixOp::create(rewriter,
       op.getLoc(), matrix, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride), /*transpose=*/UnitAttr());
   (void)store;
@@ -661,7 +661,7 @@ convertConstantOpMmaSync(RewriterBase &rewriter, arith::ConstantOp op,
     return rewriter.notifyMatchFailure(op, "not a splat");
   }
 
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       op.getLoc(), vectorType,
       DenseElementsAttr::get(vectorType, dense.getSplatValue<Attribute>()));
   valueMapping[op.getResult()] = result;
@@ -743,7 +743,7 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   }
 
   // Adjust the load offset.
-  auto laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  auto laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
   FailureOr<AffineMap> offsets =
       nvgpu::getLaneIdToLdMatrixMatrixCoord(rewriter, loc, *params);
   if (failed(offsets)) {
@@ -757,7 +757,7 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   getXferIndices<vector::TransferReadOp>(rewriter, op, *offsets, {laneId},
                                          indices);
 
-  nvgpu::LdMatrixOp newOp = rewriter.create<nvgpu::LdMatrixOp>(
+  nvgpu::LdMatrixOp newOp = nvgpu::LdMatrixOp::create(rewriter,
       loc, vectorType, op.getBase(), indices, *transpose, params->numTiles);
   valueMapping[op] = newOp->getResult(0);
   return success();
@@ -782,17 +782,17 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
             "conversion to distributed non-ldmatrix compatible load");
   }
 
-  Value laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   // This is the individual element type.
   Type loadedElType = regInfo->registerLLVMType;
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
 
-  Value fill = rewriter.create<arith::ConstantOp>(
+  Value fill = arith::ConstantOp::create(rewriter,
       op.getLoc(), vectorType.getElementType(),
       rewriter.getZeroAttr(vectorType.getElementType()));
   Value result =
-      rewriter.create<vector::SplatOp>(op.getLoc(), fill, vectorType);
+      vector::SplatOp::create(rewriter, op.getLoc(), fill, vectorType);
 
   bool isTransposeLoad = !op.getPermutationMap().isMinorIdentity();
 
@@ -809,16 +809,16 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       if (failed(coords))
         return rewriter.notifyMatchFailure(op, "no coords");
 
-      Value logicalValueId = rewriter.create<arith::ConstantOp>(
+      Value logicalValueId = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexType(),
           rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
       SmallVector<Value, 4> newIndices;
       getXferIndices<vector::TransferReadOp>(
           rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
 
-      Value el = rewriter.create<vector::LoadOp>(loc, loadedElType,
+      Value el = vector::LoadOp::create(rewriter, loc, loadedElType,
                                                  op.getBase(), newIndices);
-      result = rewriter.create<vector::InsertOp>(loc, el, result, i);
+      result = vector::InsertOp::create(rewriter, loc, el, result, i);
     }
   } else {
     if (auto vecType = dyn_cast<VectorType>(loadedElType)) {
@@ -828,7 +828,7 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       for (unsigned innerIdx = 0; innerIdx < vectorType.getShape()[1];
            innerIdx++) {
 
-        Value logicalValueId = rewriter.create<arith::ConstantOp>(
+        Value logicalValueId = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIndexType(),
             rewriter.getIndexAttr(i * regInfo->elementsPerRegister + innerIdx));
         FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
@@ -839,9 +839,9 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
         SmallVector<Value, 4> newIndices;
         getXferIndices<vector::TransferReadOp>(
             rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
-        Value el = rewriter.create<memref::LoadOp>(op.getLoc(), loadedElType,
+        Value el = memref::LoadOp::create(rewriter, op.getLoc(), loadedElType,
                                                    op.getBase(), newIndices);
-        result = rewriter.create<vector::InsertOp>(
+        result = vector::InsertOp::create(rewriter,
             op.getLoc(), el, result, ArrayRef<int64_t>{i, innerIdx});
       }
     }
@@ -916,10 +916,10 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
     return rewriter.notifyMatchFailure(op, "not mma sync reg info");
 
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
-  Value laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   for (unsigned i = 0; i < vectorType.getShape()[0]; i++) {
-    Value logicalValueId = rewriter.create<arith::ConstantOp>(
+    Value logicalValueId = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexType(),
         rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
     FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
@@ -928,11 +928,11 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
       return rewriter.notifyMatchFailure(op, "no coords");
 
     Value el =
-        rewriter.create<vector::ExtractOp>(loc, matrix, ArrayRef<int64_t>{i});
+        vector::ExtractOp::create(rewriter, loc, matrix, ArrayRef<int64_t>{i});
     SmallVector<Value, 4> newIndices;
     getXferIndices<vector::TransferWriteOp>(
         rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
-    rewriter.create<vector::StoreOp>(loc, el, op.getBase(), newIndices);
+    vector::StoreOp::create(rewriter, loc, el, op.getBase(), newIndices);
   }
 
   LLVM_DEBUG(DBGS() << "erase: " << op << "\n");
@@ -1015,7 +1015,7 @@ convertExtractStridedSlice(RewriterBase &rewriter,
   else if (offsets[1])
     sliceOffset[0] = (warpVectorShape[1] / offsets[1]);
 
-  Value newOp = rewriter.create<vector::ExtractStridedSliceOp>(
+  Value newOp = vector::ExtractStridedSliceOp::create(rewriter,
       loc, sourceVector, sliceOffset, sliceShape, strides);
 
   valueMapping[op] = newOp;
@@ -1035,7 +1035,7 @@ convertContractOp(RewriterBase &rewriter, vector::ContractionOp op,
       itC == valueMapping.end())
     return rewriter.notifyMatchFailure(op, "no mapping");
   Value opA = itA->second, opB = itB->second, opC = itC->second;
-  Value matmul = rewriter.create<gpu::SubgroupMmaComputeOp>(
+  Value matmul = gpu::SubgroupMmaComputeOp::create(rewriter,
       op.getLoc(), opC.getType(), opA, opB, opC, /*a_transpose=*/UnitAttr(),
       /*b_transpose=*/UnitAttr());
   valueMapping[op.getResult()] = matmul;
@@ -1058,7 +1058,7 @@ convertContractOpToMmaSync(RewriterBase &rewriter, vector::ContractionOp op,
   int64_t m = cast<VectorType>(op.getLhs().getType()).getShape()[0];
   int64_t n = cast<VectorType>(op.getRhs().getType()).getShape()[0];
   int64_t k = cast<VectorType>(op.getLhs().getType()).getShape()[1];
-  Value matmul = rewriter.create<nvgpu::MmaSyncOp>(
+  Value matmul = nvgpu::MmaSyncOp::create(rewriter,
       op.getLoc(), opA, opB, opC, rewriter.getI64ArrayAttr({m, n, k}));
   valueMapping[op.getResult()] = matmul;
   return success();
@@ -1076,12 +1076,12 @@ convertConstantOp(RewriterBase &rewriter, arith::ConstantOp op,
   auto splat =
       cast<SplatElementsAttr>(op.getValue()).getSplatValue<TypedAttr>();
   auto scalarConstant =
-      rewriter.create<arith::ConstantOp>(op.getLoc(), splat.getType(), splat);
+      arith::ConstantOp::create(rewriter, op.getLoc(), splat.getType(), splat);
   const char *fragType = inferFragType(op);
   auto vecType = cast<VectorType>(op.getType());
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = rewriter.create<gpu::SubgroupMmaConstantMatrixOp>(
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
       op.getLoc(), type, scalarConstant);
   valueMapping[op.getResult()] = matrix;
   return success();
@@ -1100,7 +1100,7 @@ convertBroadcastOp(RewriterBase &rewriter, vector::BroadcastOp op,
   auto vecType = op.getResultVectorType();
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = rewriter.create<gpu::SubgroupMmaConstantMatrixOp>(
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
       op.getLoc(), type, op.getSource());
   valueMapping[op.getResult()] = matrix;
   return success();
@@ -1118,7 +1118,7 @@ static scf::ForOp replaceForOpWithNewSignature(RewriterBase &rewriter,
   rewriter.setInsertionPoint(loop);
   auto operands = llvm::to_vector<4>(loop.getInitArgs());
   llvm::append_range(operands, newInitArgs);
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
       operands);
   rewriter.eraseBlock(newLoop.getBody());
@@ -1189,7 +1189,7 @@ convertYieldOp(RewriterBase &rewriter, scf::YieldOp op,
     yieldOperands[operand.index()] = loop.getInitArgs()[operand.index()];
     yieldOperands.push_back(it->second);
   }
-  rewriter.create<scf::YieldOp>(op.getLoc(), yieldOperands);
+  scf::YieldOp::create(rewriter, op.getLoc(), yieldOperands);
 
   LLVM_DEBUG(DBGS() << "erase: " << op << "\n");
   rewriter.eraseOp(op);
@@ -1220,7 +1220,7 @@ convertElementwiseOp(RewriterBase &rewriter, Operation *op,
                                          resultType.getOperand());
   }
 
-  Value newOp = rewriter.create<gpu::SubgroupMmaElementwiseOp>(
+  Value newOp = gpu::SubgroupMmaElementwiseOp::create(rewriter,
       op->getLoc(), resultType, matrixOperands, opType);
   valueMapping[op->getResult(0)] = newOp;
   return success();
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 501d98862672d..1b31d391cb0bc 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -43,13 +43,13 @@ static Value insertOne(ConversionPatternRewriter &rewriter,
   assert(rank > 0 && "0-D vector corner case should have been handled already");
   if (rank == 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = rewriter.create<LLVM::ConstantOp>(
+    auto constant = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
-    return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2,
+    return LLVM::InsertElementOp::create(rewriter, loc, llvmType, val1, val2,
                                                   constant);
   }
-  return rewriter.create<LLVM::InsertValueOp>(loc, val1, val2, pos);
+  return LLVM::InsertValueOp::create(rewriter, loc, val1, val2, pos);
 }
 
 // Helper that picks the proper sequence for extracting.
@@ -58,13 +58,13 @@ static Value extractOne(ConversionPatternRewriter &rewriter,
                         Value val, Type llvmType, int64_t rank, int64_t pos) {
   if (rank <= 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = rewriter.create<LLVM::ConstantOp>(
+    auto constant = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
-    return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val,
+    return LLVM::ExtractElementOp::create(rewriter, loc, llvmType, val,
                                                    constant);
   }
-  return rewriter.create<LLVM::ExtractValueOp>(loc, val, pos);
+  return LLVM::ExtractValueOp::create(rewriter, loc, val, pos);
 }
 
 // Helper that returns data layout alignment of a vector.
@@ -141,7 +141,7 @@ static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc,
   auto ptrsType =
       LLVM::getVectorType(pType, vectorType.getDimSize(0),
                           /*isScalable=*/vectorType.getScalableDims()[0]);
-  return rewriter.create<LLVM::GEPOp>(
+  return LLVM::GEPOp::create(rewriter,
       loc, ptrsType, typeConverter.convertType(memRefType.getElementType()),
       base, index);
 }
@@ -152,7 +152,7 @@ static Value getAsLLVMValue(OpBuilder &builder, Location loc,
                             OpFoldResult foldResult) {
   if (auto attr = dyn_cast<Attribute>(foldResult)) {
     auto intAttr = cast<IntegerAttr>(attr);
-    return builder.create<LLVM::ConstantOp>(loc, intAttr).getResult();
+    return LLVM::ConstantOp::create(builder, loc, intAttr).getResult();
   }
 
   return cast<Value>(foldResult);
@@ -475,7 +475,7 @@ class ReductionNeutralFPMax {};
 static Value createReductionNeutralValue(ReductionNeutralZero neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmType,
+  return LLVM::ConstantOp::create(rewriter, loc, llvmType,
                                            rewriter.getZeroAttr(llvmType));
 }
 
@@ -483,7 +483,7 @@ static Value createReductionNeutralValue(ReductionNeutralZero neutral,
 static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType, rewriter.getIntegerAttr(llvmType, 1));
 }
 
@@ -491,7 +491,7 @@ static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
 static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
 }
 
@@ -499,7 +499,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
 static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(
           llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth())));
@@ -509,7 +509,7 @@ static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -519,7 +519,7 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -529,7 +529,7 @@ static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -539,7 +539,7 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -550,7 +550,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
@@ -562,7 +562,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
@@ -591,7 +591,7 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
   auto vShape = vType.getShape();
   assert(vShape.size() == 1 && "Unexpected multi-dim vector type");
 
-  Value baseVecLength = rewriter.create<LLVM::ConstantOp>(
+  Value baseVecLength = LLVM::ConstantOp::create(rewriter,
       loc, rewriter.getI32Type(),
       rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0]));
 
@@ -599,11 +599,11 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
     return baseVecLength;
 
   // For a scalable vector type, create and return `vScale * baseVecLength`.
-  Value vScale = rewriter.create<vector::VectorScaleOp>(loc);
+  Value vScale = vector::VectorScaleOp::create(rewriter, loc);
   vScale =
-      rewriter.create<arith::IndexCastOp>(loc, rewriter.getI32Type(), vScale);
+      arith::IndexCastOp::create(rewriter, loc, rewriter.getI32Type(), vScale);
   Value scalableVecLength =
-      rewriter.create<arith::MulIOp>(loc, baseVecLength, vScale);
+      arith::MulIOp::create(rewriter, loc, baseVecLength, vScale);
   return scalableVecLength;
 }
 
@@ -616,10 +616,10 @@ static Value createIntegerReductionArithmeticOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator) {
 
-  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
 
   if (accumulator)
-    result = rewriter.create<ScalarOp>(loc, accumulator, result);
+    result = ScalarOp::create(rewriter, loc, accumulator, result);
   return result;
 }
 
@@ -631,11 +631,11 @@ template <class LLVMRedIntrinOp>
 static Value createIntegerReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) {
-  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
   if (accumulator) {
     Value cmp =
-        rewriter.create<LLVM::ICmpOp>(loc, predicate, accumulator, result);
-    result = rewriter.create<LLVM::SelectOp>(loc, cmp, accumulator, result);
+        LLVM::ICmpOp::create(rewriter, loc, predicate, accumulator, result);
+    result = LLVM::SelectOp::create(rewriter, loc, cmp, accumulator, result);
   }
   return result;
 }
@@ -666,11 +666,11 @@ static Value createFPReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::FastmathFlagsAttr fmf) {
   Value result =
-      rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand, fmf);
+      LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand, fmf);
 
   if (accumulator) {
     result =
-        rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
+        typename VectorToScalarMapper<LLVMRedIntrinOp>::Type::create(rewriter,
             loc, result, accumulator);
   }
 
@@ -702,7 +702,7 @@ static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
   const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
   auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
   auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
-  return rewriter.create<LLVM::ConstantOp>(loc, vectorType, denseValue);
+  return LLVM::ConstantOp::create(rewriter, loc, vectorType, denseValue);
 }
 
 /// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
@@ -717,7 +717,7 @@ lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
                                 Value mask, LLVM::FastmathFlagsAttr fmf) {
   const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
       rewriter, loc, llvmType, vectorOperand.getType());
-  const Value selectedVectorByMask = rewriter.create<LLVM::SelectOp>(
+  const Value selectedVectorByMask = LLVM::SelectOp::create(rewriter,
       loc, mask, vectorOperand, vectorMaskNeutral);
   return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
       rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
@@ -730,7 +730,7 @@ lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
                              Value accumulator, LLVM::FastmathFlagsAttr fmf) {
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
-  return rewriter.create<LLVMRedIntrinOp>(loc, llvmType,
+  return LLVMRedIntrinOp::create(rewriter, loc, llvmType,
                                           /*startValue=*/accumulator,
                                           vectorOperand, fmf);
 }
@@ -745,7 +745,7 @@ lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter,
                                        Value vectorOperand, Value accumulator) {
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
-  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
+  return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
                                             /*startValue=*/accumulator,
                                             vectorOperand);
 }
@@ -758,7 +758,7 @@ static Value lowerPredicatedReductionWithStartValue(
                                                          llvmType, accumulator);
   Value vectorLength =
       createVectorLengthValue(rewriter, loc, vectorOperand.getType());
-  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
+  return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
                                             /*startValue=*/accumulator,
                                             vectorOperand, mask, vectorLength);
 }
@@ -1071,7 +1071,7 @@ class VectorShuffleOpConversion
     // For rank 0 and 1, where both operands have *exactly* the same vector
     // type, there is direct shuffle support in LLVM. Use it!
     if (rank <= 1 && v1Type == v2Type) {
-      Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>(
+      Value llvmShuffleOp = LLVM::ShuffleVectorOp::create(rewriter,
           loc, adaptor.getV1(), adaptor.getV2(),
           llvm::to_vector_of<int32_t>(mask));
       rewriter.replaceOp(shuffleOp, llvmShuffleOp);
@@ -1085,7 +1085,7 @@ class VectorShuffleOpConversion
       eltType = arrayType.getElementType();
     else
       eltType = cast<VectorType>(llvmType).getElementType();
-    Value insert = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
+    Value insert = LLVM::PoisonOp::create(rewriter, loc, llvmType);
     int64_t insPos = 0;
     for (int64_t extPos : mask) {
       Value value = adaptor.getV1();
@@ -1122,7 +1122,7 @@ class VectorExtractElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = extractEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = rewriter.create<LLVM::ConstantOp>(
+      auto zero = LLVM::ConstantOp::create(rewriter,
           loc, typeConverter->convertType(idxType),
           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
@@ -1193,12 +1193,12 @@ class VectorExtractOpConversion
       if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) {
         return failure();
       }
-      extracted = rewriter.create<LLVM::ExtractValueOp>(
+      extracted = LLVM::ExtractValueOp::create(rewriter,
           loc, extracted, getAsIntegers(position));
     }
 
     if (extractsScalar) {
-      extracted = rewriter.create<LLVM::ExtractElementOp>(
+      extracted = LLVM::ExtractElementOp::create(rewriter,
           loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back()));
     }
 
@@ -1256,7 +1256,7 @@ class VectorInsertElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = insertEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = rewriter.create<LLVM::ConstantOp>(
+      auto zero = LLVM::ConstantOp::create(rewriter,
           loc, typeConverter->convertType(idxType),
           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
@@ -1342,7 +1342,7 @@ class VectorInsertOpConversion
           // llvm.extractvalue does not support dynamic dimensions.
           return failure();
         }
-        sourceAggregate = rewriter.create<LLVM::ExtractValueOp>(
+        sourceAggregate = LLVM::ExtractValueOp::create(rewriter,
             loc, adaptor.getDest(),
             getAsIntegers(positionOf1DVectorWithinAggregate));
       } else {
@@ -1351,7 +1351,7 @@ class VectorInsertOpConversion
         sourceAggregate = adaptor.getDest();
       }
       // Insert the scalar into the 1D vector.
-      sourceAggregate = rewriter.create<LLVM::InsertElementOp>(
+      sourceAggregate = LLVM::InsertElementOp::create(rewriter,
           loc, sourceAggregate.getType(), sourceAggregate,
           adaptor.getValueToStore(),
           getAsLLVMValue(rewriter, loc, positionOfScalarWithin1DVector));
@@ -1359,7 +1359,7 @@ class VectorInsertOpConversion
 
     Value result = sourceAggregate;
     if (isNestedAggregate) {
-      result = rewriter.create<LLVM::InsertValueOp>(
+      result = LLVM::InsertValueOp::create(rewriter,
           loc, adaptor.getDest(), sourceAggregate,
           getAsIntegers(positionOf1DVectorWithinAggregate));
     }
@@ -1439,15 +1439,15 @@ class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
 
     auto loc = op.getLoc();
     auto elemType = vType.getElementType();
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero);
+    Value desc = vector::SplatOp::create(rewriter, loc, vType, zero);
     for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
-      Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i);
-      Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i);
-      Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i);
-      Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC);
-      desc = rewriter.create<InsertOp>(loc, fma, desc, i);
+      Value extrLHS = ExtractOp::create(rewriter, loc, op.getLhs(), i);
+      Value extrRHS = ExtractOp::create(rewriter, loc, op.getRhs(), i);
+      Value extrACC = ExtractOp::create(rewriter, loc, op.getAcc(), i);
+      Value fma = FMAOp::create(rewriter, loc, extrLHS, extrRHS, extrACC);
+      desc = InsertOp::create(rewriter, loc, fma, desc, i);
     }
     rewriter.replaceOp(op, desc);
     return success();
@@ -1537,7 +1537,7 @@ class VectorTypeCastOpConversion
     desc.setAlignedPtr(rewriter, loc, ptr);
     // Fill offset 0.
     auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0);
-    auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr);
+    auto zero = LLVM::ConstantOp::create(rewriter, loc, int64Ty, attr);
     desc.setOffset(rewriter, loc, zero);
 
     // Fill size and stride descriptors in memref.
@@ -1546,11 +1546,11 @@ class VectorTypeCastOpConversion
       int64_t index = indexedSize.index();
       auto sizeAttr =
           rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value());
-      auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr);
+      auto size = LLVM::ConstantOp::create(rewriter, loc, int64Ty, sizeAttr);
       desc.setSize(rewriter, loc, index, size);
       auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(),
                                                 (*targetStrides)[index]);
-      auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr);
+      auto stride = LLVM::ConstantOp::create(rewriter, loc, int64Ty, strideAttr);
       desc.setStride(rewriter, loc, index, stride);
     }
 
@@ -1578,13 +1578,13 @@ class VectorCreateMaskOpConversion
     IntegerType idxType =
         force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
     auto loc = op->getLoc();
-    Value indices = rewriter.create<LLVM::StepVectorOp>(
+    Value indices = LLVM::StepVectorOp::create(rewriter,
         loc, LLVM::getVectorType(idxType, dstType.getShape()[0],
                                  /*isScalable=*/true));
     auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType,
                                                  adaptor.getOperands()[0]);
-    Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound);
-    Value comp = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+    Value bounds = SplatOp::create(rewriter, loc, indices.getType(), bound);
+    Value comp = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                 indices, bounds);
     rewriter.replaceOp(op, comp);
     return success();
@@ -1741,15 +1741,15 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 
     switch (conversion) {
     case PrintConversion::ZeroExt64:
-      value = rewriter.create<arith::ExtUIOp>(
+      value = arith::ExtUIOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::SignExt64:
-      value = rewriter.create<arith::ExtSIOp>(
+      value = arith::ExtSIOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::Bitcast16:
-      value = rewriter.create<LLVM::BitcastOp>(
+      value = LLVM::BitcastOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 16), value);
       break;
     case PrintConversion::None:
@@ -1762,7 +1762,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
   // Helper to emit a call.
   static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
                        Operation *ref, ValueRange params = ValueRange()) {
-    rewriter.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(ref),
+    LLVM::CallOp::create(rewriter, loc, TypeRange(), SymbolRefAttr::get(ref),
                                   params);
   }
 };
@@ -1782,8 +1782,8 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     // First insert it into a poison vector so we can shuffle it.
     auto vectorType = typeConverter->convertType(splatOp.getType());
     Value poison =
-        rewriter.create<LLVM::PoisonOp>(splatOp.getLoc(), vectorType);
-    auto zero = rewriter.create<LLVM::ConstantOp>(
+        LLVM::PoisonOp::create(rewriter, splatOp.getLoc(), vectorType);
+    auto zero = LLVM::ConstantOp::create(rewriter,
         splatOp.getLoc(),
         typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
@@ -1796,7 +1796,7 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     }
 
     // For 1-d vector, we additionally do a `vectorshuffle`.
-    auto v = rewriter.create<LLVM::InsertElementOp>(
+    auto v = LLVM::InsertElementOp::create(rewriter,
         splatOp.getLoc(), vectorType, poison, adaptor.getInput(), zero);
 
     int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
@@ -1832,26 +1832,26 @@ struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
       return failure();
 
     // Construct returned value.
-    Value desc = rewriter.create<LLVM::PoisonOp>(loc, llvmNDVectorTy);
+    Value desc = LLVM::PoisonOp::create(rewriter, loc, llvmNDVectorTy);
 
     // Construct a 1-D vector with the splatted value that we insert in all the
     // places within the returned descriptor.
-    Value vdesc = rewriter.create<LLVM::PoisonOp>(loc, llvm1DVectorTy);
-    auto zero = rewriter.create<LLVM::ConstantOp>(
+    Value vdesc = LLVM::PoisonOp::create(rewriter, loc, llvm1DVectorTy);
+    auto zero = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
-    Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc,
+    Value v = LLVM::InsertElementOp::create(rewriter, loc, llvm1DVectorTy, vdesc,
                                                      adaptor.getInput(), zero);
 
     // Shuffle the value across the desired number of elements.
     int64_t width = resultType.getDimSize(resultType.getRank() - 1);
     SmallVector<int32_t> zeroValues(width, 0);
-    v = rewriter.create<LLVM::ShuffleVectorOp>(loc, v, v, zeroValues);
+    v = LLVM::ShuffleVectorOp::create(rewriter, loc, v, v, zeroValues);
 
     // Iterate of linear index, convert to coords space and insert splatted 1-D
     // vector in each position.
     nDVectorIterate(vectorTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
-      desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, v, position);
+      desc = LLVM::InsertValueOp::create(rewriter, loc, desc, v, position);
     });
     rewriter.replaceOp(splatOp, desc);
     return success();
@@ -1921,12 +1921,12 @@ struct VectorDeinterleaveOpLowering
       auto deinterleaveResults = deinterleaveOp.getResultTypes();
       auto packedOpResults =
           llvmTypeConverter->packOperationResults(deinterleaveResults);
-      auto intrinsic = rewriter.create<LLVM::vector_deinterleave2>(
+      auto intrinsic = LLVM::vector_deinterleave2::create(rewriter,
           loc, packedOpResults, adaptor.getSource());
 
-      auto evenResult = rewriter.create<LLVM::ExtractValueOp>(
+      auto evenResult = LLVM::ExtractValueOp::create(rewriter,
           loc, intrinsic->getResult(0), 0);
-      auto oddResult = rewriter.create<LLVM::ExtractValueOp>(
+      auto oddResult = LLVM::ExtractValueOp::create(rewriter,
           loc, intrinsic->getResult(0), 1);
 
       rewriter.replaceOp(deinterleaveOp, ValueRange{evenResult, oddResult});
@@ -1950,10 +1950,10 @@ struct VectorDeinterleaveOpLowering
         oddShuffleMask.push_back(i);
     }
 
-    auto poison = rewriter.create<LLVM::PoisonOp>(loc, sourceType);
-    auto evenShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+    auto poison = LLVM::PoisonOp::create(rewriter, loc, sourceType);
+    auto evenShuffle = LLVM::ShuffleVectorOp::create(rewriter,
         loc, adaptor.getSource(), poison, evenShuffleMask);
-    auto oddShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+    auto oddShuffle = LLVM::ShuffleVectorOp::create(rewriter,
         loc, adaptor.getSource(), poison, oddShuffleMask);
 
     rewriter.replaceOp(deinterleaveOp, ValueRange{evenShuffle, oddShuffle});
@@ -1977,9 +1977,9 @@ struct VectorFromElementsLowering
       return rewriter.notifyMatchFailure(fromElementsOp,
                                          "rank > 1 vectors are not supported");
     Type llvmType = typeConverter->convertType(vectorType);
-    Value result = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
+    Value result = LLVM::PoisonOp::create(rewriter, loc, llvmType);
     for (auto [idx, val] : llvm::enumerate(adaptor.getElements()))
-      result = rewriter.create<vector::InsertOp>(loc, val, result, idx);
+      result = vector::InsertOp::create(rewriter, loc, val, result, idx);
     rewriter.replaceOp(fromElementsOp, result);
     return success();
   }
@@ -2003,11 +2003,11 @@ struct VectorToElementsLowering
       if (element.use_empty())
         continue;
 
-      auto constIdx = rewriter.create<LLVM::ConstantOp>(
+      auto constIdx = LLVM::ConstantOp::create(rewriter,
           loc, idxType, rewriter.getIntegerAttr(idxType, idx));
       auto llvmType = typeConverter->convertType(element.getType());
 
-      Value result = rewriter.create<LLVM::ExtractElementOp>(loc, llvmType,
+      Value result = LLVM::ExtractElementOp::create(rewriter, loc, llvmType,
                                                              source, constIdx);
       results[idx] = result;
     }
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index 9f5b8fcca6c26..9b79b913f2d4b 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -132,9 +132,9 @@ static void maybeYieldValue(OpBuilder &b, Location loc, bool hasRetVal,
                             Value value) {
   if (hasRetVal) {
     assert(value && "Expected non-empty value");
-    b.create<scf::YieldOp>(loc, value);
+    scf::YieldOp::create(b, loc, value);
   } else {
-    b.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(b, loc);
   }
 }
 
@@ -154,7 +154,7 @@ static Value generateMaskCheck(OpBuilder &b, OpTy xferOp, Value iv) {
     return Value();
 
   Location loc = xferOp.getLoc();
-  return b.create<vector::ExtractElementOp>(loc, xferOp.getMask(), iv);
+  return vector::ExtractElementOp::create(b, loc, xferOp.getMask(), iv);
 }
 
 /// Helper function TransferOpConversion and TransferOp1dConversion.
@@ -201,21 +201,21 @@ static Value generateInBoundsCheck(
     Value base = xferOp.getIndices()[*dim];
     Value memrefIdx =
         affine::makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
-    cond = lb.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, memrefDim,
+    cond = arith::CmpIOp::create(lb, arith::CmpIPredicate::sgt, memrefDim,
                                     memrefIdx);
   }
 
   // Condition check 2: Masked in?
   if (auto maskCond = generateMaskCheck(b, xferOp, iv)) {
     if (cond)
-      cond = lb.create<arith::AndIOp>(cond, maskCond);
+      cond = arith::AndIOp::create(lb, cond, maskCond);
     else
       cond = maskCond;
   }
 
   // If the condition is non-empty, generate an SCF::IfOp.
   if (cond) {
-    auto check = lb.create<scf::IfOp>(
+    auto check = scf::IfOp::create(lb,
         cond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
@@ -226,7 +226,7 @@ static Value generateInBoundsCheck(
           if (outOfBoundsCase) {
             maybeYieldValue(b, loc, hasRetVal, outOfBoundsCase(b, loc));
           } else {
-            b.create<scf::YieldOp>(loc);
+            scf::YieldOp::create(b, loc);
           }
         });
 
@@ -303,14 +303,14 @@ static BufferAllocs allocBuffers(OpBuilder &b, OpTy xferOp) {
 
   BufferAllocs result;
   auto bufferType = MemRefType::get({}, xferOp.getVectorType());
-  result.dataBuffer = b.create<memref::AllocaOp>(loc, bufferType);
+  result.dataBuffer = memref::AllocaOp::create(b, loc, bufferType);
 
   if (xferOp.getMask()) {
     auto maskType = MemRefType::get({}, xferOp.getMask().getType());
-    auto maskBuffer = b.create<memref::AllocaOp>(loc, maskType);
+    auto maskBuffer = memref::AllocaOp::create(b, loc, maskType);
     b.setInsertionPoint(xferOp);
-    b.create<memref::StoreOp>(loc, xferOp.getMask(), maskBuffer);
-    result.maskBuffer = b.create<memref::LoadOp>(loc, maskBuffer, ValueRange());
+    memref::StoreOp::create(b, loc, xferOp.getMask(), maskBuffer);
+    result.maskBuffer = memref::LoadOp::create(b, loc, maskBuffer, ValueRange());
   }
 
   return result;
@@ -421,14 +421,14 @@ struct Strategy<TransferReadOp> {
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
-    auto newXferOp = b.create<vector::TransferReadOp>(
+    auto newXferOp = vector::TransferReadOp::create(b,
         loc, vecType, xferOp.getBase(), xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
         xferOp.getPadding(), Value(), inBoundsAttr);
 
     maybeApplyPassLabel(b, newXferOp, options.targetRank);
 
-    b.create<memref::StoreOp>(loc, newXferOp.getVector(), buffer, storeIndices);
+    memref::StoreOp::create(b, loc, newXferOp.getVector(), buffer, storeIndices);
     return newXferOp;
   }
 
@@ -444,8 +444,8 @@ struct Strategy<TransferReadOp> {
     Location loc = xferOp.getLoc();
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
-    auto vec = b.create<vector::SplatOp>(loc, vecType, xferOp.getPadding());
-    b.create<memref::StoreOp>(loc, vec, buffer, storeIndices);
+    auto vec = vector::SplatOp::create(b, loc, vecType, xferOp.getPadding());
+    memref::StoreOp::create(b, loc, vec, buffer, storeIndices);
 
     return Value();
   }
@@ -506,11 +506,11 @@ struct Strategy<TransferWriteOp> {
     getXferIndices(b, xferOp, iv, xferIndices);
 
     Location loc = xferOp.getLoc();
-    auto vec = b.create<memref::LoadOp>(loc, buffer, loadIndices);
+    auto vec = memref::LoadOp::create(b, loc, buffer, loadIndices);
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
     auto source = loopState.empty() ? xferOp.getBase() : loopState[0];
     Type type = isTensorOp(xferOp) ? xferOp.getShapedType() : Type();
-    auto newXferOp = b.create<vector::TransferWriteOp>(
+    auto newXferOp = vector::TransferWriteOp::create(b,
         loc, type, vec, source, xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
         inBoundsAttr);
@@ -610,7 +610,7 @@ struct PrepareTransferReadConversion
     }
 
     Location loc = xferOp.getLoc();
-    rewriter.create<memref::StoreOp>(loc, newXfer->getResult(0),
+    memref::StoreOp::create(rewriter, loc, newXfer->getResult(0),
                                      buffers.dataBuffer);
     rewriter.replaceOpWithNewOp<memref::LoadOp>(xferOp, buffers.dataBuffer);
 
@@ -653,9 +653,9 @@ struct PrepareTransferWriteConversion
 
     Location loc = xferOp.getLoc();
     auto buffers = allocBuffers(rewriter, xferOp);
-    rewriter.create<memref::StoreOp>(loc, xferOp.getVector(),
+    memref::StoreOp::create(rewriter, loc, xferOp.getVector(),
                                      buffers.dataBuffer);
-    auto loadedVec = rewriter.create<memref::LoadOp>(loc, buffers.dataBuffer);
+    auto loadedVec = memref::LoadOp::create(rewriter, loc, buffers.dataBuffer);
     rewriter.modifyOpInPlace(xferOp, [&]() {
       xferOp.getValueToStoreMutable().assign(loadedVec);
       xferOp->setAttr(kPassLabel, rewriter.getUnitAttr());
@@ -735,17 +735,17 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
       auto signlessTargetVectorType =
           vectorType.cloneWith({}, getIntTypeWithSignlessSemantics(legalIntTy));
       auto targetVectorType = vectorType.cloneWith({}, legalIntTy);
-      value = rewriter.create<vector::BitCastOp>(loc, signlessSourceVectorType,
+      value = vector::BitCastOp::create(rewriter, loc, signlessSourceVectorType,
                                                  value);
       if (value.getType() != signlessTargetVectorType) {
         if (width == 1 || intTy.isUnsigned())
-          value = rewriter.create<arith::ExtUIOp>(loc, signlessTargetVectorType,
+          value = arith::ExtUIOp::create(rewriter, loc, signlessTargetVectorType,
                                                   value);
         else
-          value = rewriter.create<arith::ExtSIOp>(loc, signlessTargetVectorType,
+          value = arith::ExtSIOp::create(rewriter, loc, signlessTargetVectorType,
                                                   value);
       }
-      value = rewriter.create<vector::BitCastOp>(loc, targetVectorType, value);
+      value = vector::BitCastOp::create(rewriter, loc, targetVectorType, value);
       vectorType = targetVectorType;
     }
 
@@ -763,28 +763,28 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
                                         std::multiplies<int64_t>());
       auto flatVectorType =
           VectorType::get({flatLength}, vectorType.getElementType());
-      value = rewriter.create<vector::ShapeCastOp>(loc, flatVectorType, value);
+      value = vector::ShapeCastOp::create(rewriter, loc, flatVectorType, value);
     }
 
     vector::PrintOp firstClose;
     SmallVector<Value, 8> loopIndices;
     for (unsigned d = 0; d < shape.size(); d++) {
       // Setup loop bounds and step.
-      Value lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-      Value upperBound = rewriter.create<arith::ConstantIndexOp>(loc, shape[d]);
-      Value step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+      Value lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+      Value upperBound = arith::ConstantIndexOp::create(rewriter, loc, shape[d]);
+      Value step = arith::ConstantIndexOp::create(rewriter, loc, 1);
       if (!scalableDimensions.empty() && scalableDimensions[d]) {
-        auto vscale = rewriter.create<vector::VectorScaleOp>(
+        auto vscale = vector::VectorScaleOp::create(rewriter,
             loc, rewriter.getIndexType());
-        upperBound = rewriter.create<arith::MulIOp>(loc, upperBound, vscale);
+        upperBound = arith::MulIOp::create(rewriter, loc, upperBound, vscale);
       }
-      auto lastIndex = rewriter.create<arith::SubIOp>(loc, upperBound, step);
+      auto lastIndex = arith::SubIOp::create(rewriter, loc, upperBound, step);
 
       // Create a loop to print the elements surrounded by parentheses.
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+      vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
       auto loop =
-          rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
-      auto printClose = rewriter.create<vector::PrintOp>(
+          scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
+      auto printClose = vector::PrintOp::create(rewriter,
           loc, vector::PrintPunctuation::Close);
       if (!firstClose)
         firstClose = printClose;
@@ -794,13 +794,13 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
 
       // Print a comma after all but the last element.
       rewriter.setInsertionPointToStart(loop.getBody());
-      auto notLastIndex = rewriter.create<arith::CmpIOp>(
+      auto notLastIndex = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ult, loopIdx, lastIndex);
-      rewriter.create<scf::IfOp>(loc, notLastIndex,
+      scf::IfOp::create(rewriter, loc, notLastIndex,
                                  [&](OpBuilder &builder, Location loc) {
-                                   builder.create<vector::PrintOp>(
+                                   vector::PrintOp::create(builder,
                                        loc, vector::PrintPunctuation::Comma);
-                                   builder.create<scf::YieldOp>(loc);
+                                   scf::YieldOp::create(builder, loc);
                                  });
 
       rewriter.setInsertionPointToStart(loop.getBody());
@@ -811,10 +811,10 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     Value flatIndex;
     auto currentStride = 1;
     for (int d = shape.size() - 1; d >= 0; d--) {
-      auto stride = rewriter.create<arith::ConstantIndexOp>(loc, currentStride);
-      auto index = rewriter.create<arith::MulIOp>(loc, stride, loopIndices[d]);
+      auto stride = arith::ConstantIndexOp::create(rewriter, loc, currentStride);
+      auto index = arith::MulIOp::create(rewriter, loc, stride, loopIndices[d]);
       if (flatIndex)
-        flatIndex = rewriter.create<arith::AddIOp>(loc, flatIndex, index);
+        flatIndex = arith::AddIOp::create(rewriter, loc, flatIndex, index);
       else
         flatIndex = index;
       currentStride *= shape[d];
@@ -822,12 +822,12 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
 
     // Print the scalar elements in the inner most loop.
     auto element =
-        rewriter.create<vector::ExtractElementOp>(loc, value, flatIndex);
-    rewriter.create<vector::PrintOp>(loc, element,
+        vector::ExtractElementOp::create(rewriter, loc, value, flatIndex);
+    vector::PrintOp::create(rewriter, loc, element,
                                      vector::PrintPunctuation::NoPunctuation);
 
     rewriter.setInsertionPointAfter(firstClose);
-    rewriter.create<vector::PrintOp>(loc, printOp.getPunctuation());
+    vector::PrintOp::create(rewriter, loc, printOp.getPunctuation());
     rewriter.eraseOp(printOp);
     return success();
   }
@@ -918,7 +918,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                                          "Failed to unpack one vector dim.");
 
     auto castedDataBuffer =
-        locB.create<vector::TypeCastOp>(*castedDataType, dataBuffer);
+        vector::TypeCastOp::create(locB, *castedDataType, dataBuffer);
 
     // If the xferOp has a mask: Find and cast mask buffer.
     Value castedMaskBuffer;
@@ -937,21 +937,21 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
         auto maskBufferType = cast<MemRefType>(maskBuffer.getType());
         MemRefType castedMaskType = *unpackOneDim(maskBufferType);
         castedMaskBuffer =
-            locB.create<vector::TypeCastOp>(castedMaskType, maskBuffer);
+            vector::TypeCastOp::create(locB, castedMaskType, maskBuffer);
       }
     }
 
     // Loop bounds and step.
-    auto lb = locB.create<arith::ConstantIndexOp>(0);
-    auto ub = locB.create<arith::ConstantIndexOp>(
+    auto lb = arith::ConstantIndexOp::create(locB, 0);
+    auto ub = arith::ConstantIndexOp::create(locB,
         castedDataType->getDimSize(castedDataType->getRank() - 1));
-    auto step = locB.create<arith::ConstantIndexOp>(1);
+    auto step = arith::ConstantIndexOp::create(locB, 1);
     // TransferWriteOps that operate on tensors return the modified tensor and
     // require a loop state.
     auto loopState = Strategy<OpTy>::initialLoopState(xferOp);
 
     // Generate for loop.
-    auto result = locB.create<scf::ForOp>(
+    auto result = scf::ForOp::create(locB,
         lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(),
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopState) {
           Type stateType = loopState.empty() ? Type() : loopState[0].getType();
@@ -977,7 +977,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                   SmallVector<Value, 8> loadIndices;
                   getMaskBufferLoadIndices(xferOp, castedMaskBuffer,
                                            loadIndices, iv);
-                  auto mask = b.create<memref::LoadOp>(loc, castedMaskBuffer,
+                  auto mask = memref::LoadOp::create(b, loc, castedMaskBuffer,
                                                        loadIndices);
                   rewriter.modifyOpInPlace(newXfer, [&]() {
                     newXfer.getMaskMutable().assign(mask);
@@ -1121,29 +1121,29 @@ struct ScalableTransposeTransferWriteConversion
     auto transposeSource = transposeOp.getVector();
     SmallVector<Value> transposeSourceSlices =
         llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-          return rewriter.create<vector::ExtractOp>(loc, transposeSource, idx);
+          return vector::ExtractOp::create(rewriter, loc, transposeSource, idx);
         });
 
     // Loop bounds and step.
-    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lb = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto ub =
         maskDims->empty()
             ? Value(createVscaleMultiple(vectorType.getDimSize(0)))
             : vector::getAsValues(rewriter, loc, maskDims->front()).front();
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
     // Generate a new mask for the slice.
     VectorType sliceType = VectorType::Builder(vectorType).dropDim(0);
     Value sliceMask = nullptr;
     if (!maskDims->empty()) {
-      sliceMask = rewriter.create<vector::CreateMaskOp>(
+      sliceMask = vector::CreateMaskOp::create(rewriter,
           loc, sliceType.clone(rewriter.getI1Type()),
           ArrayRef<OpFoldResult>(*maskDims).drop_front());
     }
 
     Value initDest = isTensorOp(writeOp) ? writeOp.getBase() : Value{};
     ValueRange initLoopArgs = initDest ? initDest : ValueRange{};
-    auto result = rewriter.create<scf::ForOp>(
+    auto result = scf::ForOp::create(rewriter,
         loc, lb, ub, step, initLoopArgs,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopIterArgs) {
           // Indices for the new transfer op.
@@ -1153,23 +1153,23 @@ struct ScalableTransposeTransferWriteConversion
           // Extract a transposed slice from the source vector.
           SmallVector<Value> transposeElements =
               llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-                return b.create<vector::ExtractOp>(
+                return vector::ExtractOp::create(b,
                     loc, transposeSourceSlices[idx], iv);
               });
-          auto sliceVec = b.create<vector::FromElementsOp>(loc, sliceType,
+          auto sliceVec = vector::FromElementsOp::create(b, loc, sliceType,
                                                            transposeElements);
 
           // Create the transfer_write for the slice.
           Value dest =
               loopIterArgs.empty() ? writeOp.getBase() : loopIterArgs.front();
-          auto newWriteOp = b.create<vector::TransferWriteOp>(
+          auto newWriteOp = vector::TransferWriteOp::create(b,
               loc, sliceVec, dest, xferIndices,
               ArrayRef<bool>(writeOp.getInBoundsValues()).drop_front());
           if (sliceMask)
             newWriteOp.getMaskMutable().assign(sliceMask);
 
           // Yield from the loop.
-          b.create<scf::YieldOp>(loc, loopIterArgs.empty()
+          scf::YieldOp::create(b, loc, loopIterArgs.empty()
                                           ? ValueRange{}
                                           : newWriteOp.getResult());
         });
@@ -1209,7 +1209,7 @@ static void maybeAssignMask(OpBuilder &b, OpTy xferOp, OpTy newXferOp,
 
     llvm::SmallVector<int64_t, 1> indices({i});
     Location loc = xferOp.getLoc();
-    auto newMask = b.create<vector::ExtractOp>(loc, xferOp.getMask(), indices);
+    auto newMask = vector::ExtractOp::create(b, loc, xferOp.getMask(), indices);
     newXferOp.getMaskMutable().assign(newMask);
   }
 
@@ -1263,7 +1263,7 @@ struct UnrollTransferReadConversion
     if (auto insertOp = getInsertOp(xferOp))
       return insertOp.getDest();
     Location loc = xferOp.getLoc();
-    return rewriter.create<vector::SplatOp>(loc, xferOp.getVectorType(),
+    return vector::SplatOp::create(rewriter, loc, xferOp.getVectorType(),
                                             xferOp.getPadding());
   }
 
@@ -1319,7 +1319,7 @@ struct UnrollTransferReadConversion
     // Generate fully unrolled loop of transfer ops.
     Location loc = xferOp.getLoc();
     for (int64_t i = 0; i < dimSize; ++i) {
-      Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
+      Value iv = arith::ConstantIndexOp::create(rewriter, loc, i);
 
       // FIXME: Rename this lambda - it does much more than just
       // in-bounds-check generation.
@@ -1338,7 +1338,7 @@ struct UnrollTransferReadConversion
 
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
 
-            auto newXferOp = b.create<vector::TransferReadOp>(
+            auto newXferOp = vector::TransferReadOp::create(b,
                 loc, newXferVecType, xferOp.getBase(), xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
                 xferOp.getPadding(), Value(), inBoundsAttr);
@@ -1348,10 +1348,10 @@ struct UnrollTransferReadConversion
             if (newXferVecType.getRank() == 0) {
               // vector.insert does not accept rank-0 as the non-indexed
               // argument. Extract the scalar before inserting.
-              valToInser = b.create<vector::ExtractOp>(loc, valToInser,
+              valToInser = vector::ExtractOp::create(b, loc, valToInser,
                                                        SmallVector<int64_t>());
             }
-            return b.create<vector::InsertOp>(loc, valToInser, vec,
+            return vector::InsertOp::create(b, loc, valToInser, vec,
                                               insertionIndices);
           },
           /*outOfBoundsCase=*/
@@ -1462,7 +1462,7 @@ struct UnrollTransferWriteConversion
     // Generate fully unrolled loop of transfer ops.
     Location loc = xferOp.getLoc();
     for (int64_t i = 0; i < dimSize; ++i) {
-      Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
+      Value iv = arith::ConstantIndexOp::create(rewriter, loc, i);
 
       auto updatedSource = generateInBoundsCheck(
           rewriter, xferOp, iv, unpackedDim(xferOp),
@@ -1479,19 +1479,19 @@ struct UnrollTransferWriteConversion
             extractionIndices.push_back(b.getI64IntegerAttr(i));
 
             auto extracted =
-                b.create<vector::ExtractOp>(loc, vec, extractionIndices);
+                vector::ExtractOp::create(b, loc, vec, extractionIndices);
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
             Value xferVec;
             if (inputVectorTy.getRank() == 1) {
               // When target-rank=0, unrolling would causes the vector input
               // argument into `transfer_write` to become a scalar. We solve
               // this by broadcasting the scalar to a 0D vector.
-              xferVec = b.create<vector::BroadcastOp>(
+              xferVec = vector::BroadcastOp::create(b,
                   loc, VectorType::get({}, extracted.getType()), extracted);
             } else {
               xferVec = extracted;
             }
-            auto newXferOp = b.create<vector::TransferWriteOp>(
+            auto newXferOp = vector::TransferWriteOp::create(b,
                 loc, sourceType, xferVec, source, xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
                 inBoundsAttr);
@@ -1574,18 +1574,18 @@ struct Strategy1d<TransferReadOp> {
         b, xferOp, iv, dim, TypeRange(xferOp.getVectorType()),
         /*inBoundsCase=*/
         [&](OpBuilder &b, Location loc) {
-          Value val = b.create<memref::LoadOp>(loc, xferOp.getBase(), indices);
-          return b.create<vector::InsertElementOp>(loc, val, vec, iv);
+          Value val = memref::LoadOp::create(b, loc, xferOp.getBase(), indices);
+          return vector::InsertElementOp::create(b, loc, val, vec, iv);
         },
         /*outOfBoundsCase=*/
         [&](OpBuilder & /*b*/, Location loc) { return vec; });
-    b.create<scf::YieldOp>(loc, nextVec);
+    scf::YieldOp::create(b, loc, nextVec);
   }
 
   static Value initialLoopState(OpBuilder &b, TransferReadOp xferOp) {
     // Inititalize vector with padding value.
     Location loc = xferOp.getLoc();
-    return b.create<vector::SplatOp>(loc, xferOp.getVectorType(),
+    return vector::SplatOp::create(b, loc, xferOp.getVectorType(),
                                      xferOp.getPadding());
   }
 };
@@ -1604,10 +1604,10 @@ struct Strategy1d<TransferWriteOp> {
         b, xferOp, iv, dim,
         /*inBoundsCase=*/[&](OpBuilder &b, Location loc) {
           auto val =
-              b.create<vector::ExtractElementOp>(loc, xferOp.getVector(), iv);
-          b.create<memref::StoreOp>(loc, val, xferOp.getBase(), indices);
+              vector::ExtractElementOp::create(b, loc, xferOp.getVector(), iv);
+          memref::StoreOp::create(b, loc, val, xferOp.getBase(), indices);
         });
-    b.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(b, loc);
   }
 
   static Value initialLoopState(OpBuilder &b, TransferWriteOp xferOp) {
@@ -1668,15 +1668,15 @@ struct TransferOp1dConversion : public VectorToSCFPattern<OpTy> {
     // Loop bounds, step, state...
     Location loc = xferOp.getLoc();
     auto vecType = xferOp.getVectorType();
-    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lb = arith::ConstantIndexOp::create(rewriter, loc, 0);
     Value ub =
-        rewriter.create<arith::ConstantIndexOp>(loc, vecType.getDimSize(0));
+        arith::ConstantIndexOp::create(rewriter, loc, vecType.getDimSize(0));
     if (vecType.isScalable()) {
       Value vscale =
-          rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-      ub = rewriter.create<arith::MulIOp>(loc, ub, vscale);
+          vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+      ub = arith::MulIOp::create(rewriter, loc, ub, vscale);
     }
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto loopState = Strategy1d<OpTy>::initialLoopState(rewriter, xferOp);
 
     // Generate for loop.
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 21d8e1d9f1156..2466e72277441 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -147,18 +147,18 @@ static Value sanitizeDynamicIndex(ConversionPatternRewriter &rewriter,
                                   Location loc, Value dynamicIndex,
                                   int64_t kPoisonIndex, unsigned vectorSize) {
   if (llvm::isPowerOf2_32(vectorSize)) {
-    Value inBoundsMask = rewriter.create<spirv::ConstantOp>(
+    Value inBoundsMask = spirv::ConstantOp::create(rewriter,
         loc, dynamicIndex.getType(),
         rewriter.getIntegerAttr(dynamicIndex.getType(), vectorSize - 1));
-    return rewriter.create<spirv::BitwiseAndOp>(loc, dynamicIndex,
+    return spirv::BitwiseAndOp::create(rewriter, loc, dynamicIndex,
                                                 inBoundsMask);
   }
-  Value poisonIndex = rewriter.create<spirv::ConstantOp>(
+  Value poisonIndex = spirv::ConstantOp::create(rewriter,
       loc, dynamicIndex.getType(),
       rewriter.getIntegerAttr(dynamicIndex.getType(), kPoisonIndex));
   Value cmpResult =
-      rewriter.create<spirv::IEqualOp>(loc, dynamicIndex, poisonIndex);
-  return rewriter.create<spirv::SelectOp>(
+      spirv::IEqualOp::create(rewriter, loc, dynamicIndex, poisonIndex);
+  return spirv::SelectOp::create(rewriter,
       loc, cmpResult,
       spirv::ConstantOp::getZero(dynamicIndex.getType(), loc, rewriter),
       dynamicIndex);
@@ -427,7 +427,7 @@ static SmallVector<Value> extractAllElements(
   Location loc = reduceOp.getLoc();
 
   for (int i = 0; i < numElements; ++i) {
-    values.push_back(rewriter.create<spirv::CompositeExtractOp>(
+    values.push_back(spirv::CompositeExtractOp::create(rewriter,
         loc, srcVectorType.getElementType(), adaptor.getVector(),
         rewriter.getI32ArrayAttr({i})));
   }
@@ -481,16 +481,16 @@ struct VectorReductionPattern final : OpConversionPattern<vector::ReductionOp> {
 #define INT_AND_FLOAT_CASE(kind, iop, fop)                                     \
   case vector::CombiningKind::kind:                                            \
     if (llvm::isa<IntegerType>(resultType)) {                                  \
-      result = rewriter.create<spirv::iop>(loc, resultType, result, next);     \
+      result = spirv::iop::create(rewriter, loc, resultType, result, next);     \
     } else {                                                                   \
       assert(llvm::isa<FloatType>(resultType));                                \
-      result = rewriter.create<spirv::fop>(loc, resultType, result, next);     \
+      result = spirv::fop::create(rewriter, loc, resultType, result, next);     \
     }                                                                          \
     break
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = rewriter.create<fop>(loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);              \
     break
 
         INT_AND_FLOAT_CASE(ADD, IAddOp, FAddOp);
@@ -537,7 +537,7 @@ struct VectorReductionFloatMinMax final
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = rewriter.create<fop>(loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);              \
     break
 
         INT_OR_FLOAT_CASE(MAXIMUMF, SPIRVFMaxOp);
@@ -613,7 +613,7 @@ struct VectorShuffleOpConvert final
     auto getElementAtIdx = [&rewriter, loc = shuffleOp.getLoc()](
                                Value scalarOrVec, int32_t idx) -> Value {
       if (auto vecTy = dyn_cast<VectorType>(scalarOrVec.getType()))
-        return rewriter.create<spirv::CompositeExtractOp>(loc, scalarOrVec,
+        return spirv::CompositeExtractOp::create(rewriter, loc, scalarOrVec,
                                                           idx);
 
       assert(idx == 0 && "Invalid scalar element index");
@@ -712,10 +712,10 @@ struct VectorDeinterleaveOpConvert final
     // We cannot use `spirv::VectorShuffleOp` directly in this case, and need to
     // use `spirv::CompositeExtractOp`.
     if (n == 2) {
-      auto elem0 = rewriter.create<spirv::CompositeExtractOp>(
+      auto elem0 = spirv::CompositeExtractOp::create(rewriter,
           loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({0}));
 
-      auto elem1 = rewriter.create<spirv::CompositeExtractOp>(
+      auto elem1 = spirv::CompositeExtractOp::create(rewriter,
           loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({1}));
 
       rewriter.replaceOp(deinterleaveOp, {elem0, elem1});
@@ -733,11 +733,11 @@ struct VectorDeinterleaveOpConvert final
         llvm::map_to_vector(seqOdd, [](int i) { return i * 2 + 1; });
 
     // Create two SPIR-V shuffles.
-    auto shuffleEven = rewriter.create<spirv::VectorShuffleOp>(
+    auto shuffleEven = spirv::VectorShuffleOp::create(rewriter,
         loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesEven));
 
-    auto shuffleOdd = rewriter.create<spirv::VectorShuffleOp>(
+    auto shuffleOdd = spirv::VectorShuffleOp::create(rewriter,
         loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesOdd));
 
@@ -781,7 +781,7 @@ struct VectorLoadOpConverter final
     // to a scalar.
     Value castedAccessChain = (vectorType.getNumElements() == 1)
                                   ? accessChain
-                                  : rewriter.create<spirv::BitcastOp>(
+                                  : spirv::BitcastOp::create(rewriter,
                                         loc, vectorPtrType, accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, spirvVectorType,
@@ -823,7 +823,7 @@ struct VectorStoreOpConverter final
     // to a scalar.
     Value castedAccessChain = (vectorType.getNumElements() == 1)
                                   ? accessChain
-                                  : rewriter.create<spirv::BitcastOp>(
+                                  : spirv::BitcastOp::create(rewriter,
                                         loc, vectorPtrType, accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, castedAccessChain,
@@ -905,9 +905,9 @@ struct VectorReductionToIntDotProd final
       auto v4i8Type = VectorType::get({4}, i8Type);
       Location loc = op.getLoc();
       Value zero = spirv::ConstantOp::getZero(i8Type, loc, rewriter);
-      lhsIn = rewriter.create<spirv::CompositeConstructOp>(
+      lhsIn = spirv::CompositeConstructOp::create(rewriter,
           loc, v4i8Type, ValueRange{lhsIn, zero});
-      rhsIn = rewriter.create<spirv::CompositeConstructOp>(
+      rhsIn = spirv::CompositeConstructOp::create(rewriter,
           loc, v4i8Type, ValueRange{rhsIn, zero});
     }
 
@@ -971,14 +971,14 @@ struct VectorReductionToFPDotProd final
       Attribute oneAttr =
           rewriter.getFloatAttr(vectorType.getElementType(), 1.0);
       oneAttr = SplatElementsAttr::get(vectorType, oneAttr);
-      rhs = rewriter.create<spirv::ConstantOp>(loc, vectorType, oneAttr);
+      rhs = spirv::ConstantOp::create(rewriter, loc, vectorType, oneAttr);
     }
     assert(lhs);
     assert(rhs);
 
-    Value res = rewriter.create<spirv::DotOp>(loc, resultType, lhs, rhs);
+    Value res = spirv::DotOp::create(rewriter, loc, resultType, lhs, rhs);
     if (acc)
-      res = rewriter.create<spirv::FAddOp>(loc, acc, res);
+      res = spirv::FAddOp::create(rewriter, loc, acc, res);
 
     rewriter.replaceOp(op, res);
     return success();
@@ -1013,7 +1013,7 @@ struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
     source.reserve(numElements);
     for (int64_t i = 0; i < numElements; ++i) {
       Attribute intAttr = rewriter.getIntegerAttr(intType, i);
-      Value constOp = rewriter.create<spirv::ConstantOp>(loc, intType, intAttr);
+      Value constOp = spirv::ConstantOp::create(rewriter, loc, intType, intAttr);
       source.push_back(constOp);
     }
     rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(stepOp, dstType,
@@ -1056,7 +1056,7 @@ struct VectorToElementOpConvert final
       if (element.use_empty())
         continue;
 
-      Value result = rewriter.create<spirv::CompositeExtractOp>(
+      Value result = spirv::CompositeExtractOp::create(rewriter,
           loc, elementType, adaptor.getSource(),
           rewriter.getI32ArrayAttr({static_cast<int32_t>(idx)}));
       results[idx] = result;
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index 0ec7129a40a66..255013ee07c88 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -108,7 +108,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
 
   xegpu::CreateNdDescOp ndDesc;
   if (srcTy.hasStaticShape()) {
-    ndDesc = rewriter.create<xegpu::CreateNdDescOp>(loc, descType, src,
+    ndDesc = xegpu::CreateNdDescOp::create(rewriter, loc, descType, src,
                                                     getAsOpFoldResult(offsets));
   } else {
     // In case of any dynamic shapes, source's shape and strides have to be
@@ -116,7 +116,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
     SmallVector<Value> sourceDims;
     unsigned srcRank = srcTy.getRank();
     for (unsigned i = 0; i < srcRank; ++i)
-      sourceDims.push_back(rewriter.create<memref::DimOp>(loc, src, i));
+      sourceDims.push_back(memref::DimOp::create(rewriter, loc, src, i));
 
     SmallVector<int64_t> constOffsets;
     SmallVector<Value> dynOffsets;
@@ -135,17 +135,17 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
 
     // Compute strides in reverse order.
     SmallVector<Value> dynStrides;
-    Value accStride = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value accStride = arith::ConstantIndexOp::create(rewriter, loc, 1);
     // Last stride is guaranteed to be static and unit.
     for (int i = static_cast<int>(strides.size()) - 2; i >= 0; --i) {
       accStride =
-          rewriter.create<arith::MulIOp>(loc, accStride, sourceDims[i + 1]);
+          arith::MulIOp::create(rewriter, loc, accStride, sourceDims[i + 1]);
       if (strides[i] == ShapedType::kDynamic)
         dynStrides.push_back(accStride);
     }
     std::reverse(dynStrides.begin(), dynStrides.end());
 
-    ndDesc = rewriter.create<xegpu::CreateNdDescOp>(
+    ndDesc = xegpu::CreateNdDescOp::create(rewriter,
         loc, descType, src, dynOffsets, dynShapes, dynStrides,
         DenseI64ArrayAttr::get(rewriter.getContext(), constOffsets),
         DenseI64ArrayAttr::get(rewriter.getContext(), srcTy.getShape()),
@@ -200,7 +200,7 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
                                                   ArrayRef<int64_t>{1, 0});
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadOp = rewriter.create<xegpu::LoadNdOp>(
+    auto loadOp = xegpu::LoadNdOp::create(rewriter,
         loc, vecTy, ndDesc, /*packed=*/nullptr, transposeAttr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
@@ -238,7 +238,7 @@ struct TransferWriteLowering
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeOp =
-        rewriter.create<xegpu::StoreNdOp>(loc, writeOp.getVector(), ndDesc,
+        xegpu::StoreNdOp::create(rewriter, loc, writeOp.getVector(), ndDesc,
                                           /*l1_hint=*/hint,
                                           /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(writeOp, storeOp);
@@ -269,7 +269,7 @@ struct LoadLowering : public OpRewritePattern<vector::LoadOp> {
 
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadNdOp = rewriter.create<xegpu::LoadNdOp>(
+    auto loadNdOp = xegpu::LoadNdOp::create(rewriter,
         loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
@@ -303,7 +303,7 @@ struct StoreLowering : public OpRewritePattern<vector::StoreOp> {
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeNdOp =
-        rewriter.create<xegpu::StoreNdOp>(loc, vector, ndDesc,
+        xegpu::StoreNdOp::create(rewriter, loc, vector, ndDesc,
                                           /*l1_hint=*/hint,
                                           /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(storeOp, storeNdOp);
@@ -346,7 +346,7 @@ struct ContractionLowering : public OpRewritePattern<vector::ContractionOp> {
       return rewriter.notifyMatchFailure(contractOp,
                                          "Invalid operand dimensions");
 
-    auto dpasOp = rewriter.create<xegpu::DpasOp>(
+    auto dpasOp = xegpu::DpasOp::create(rewriter,
         loc, TypeRange{contractOp.getResultType()}, ValueRange{lhs, rhs, acc});
     rewriter.replaceOp(contractOp, dpasOp);
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
index fd2ba0683786e..4a1a2487806a7 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -100,8 +100,8 @@ static Value flattenVecToBits(ConversionPatternRewriter &rewriter, Location loc,
       vectorType.getElementTypeBitWidth() * vectorType.getNumElements();
   Type allBitsType = rewriter.getIntegerType(bitwidth);
   auto allBitsVecType = VectorType::get({1}, allBitsType);
-  Value bitcast = rewriter.create<vector::BitCastOp>(loc, allBitsVecType, val);
-  Value scalar = rewriter.create<vector::ExtractOp>(loc, bitcast, 0);
+  Value bitcast = vector::BitCastOp::create(rewriter, loc, allBitsVecType, val);
+  Value scalar = vector::ExtractOp::create(rewriter, loc, bitcast, 0);
   return scalar;
 }
 
@@ -120,25 +120,25 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   SmallVector<NamedAttribute> loadAttrs;
   patchOperandSegmentSizes(origAttrs, loadAttrs, DataArgAction::Drop);
   Value initialLoad =
-      rewriter.create<RawBufferLoadOp>(loc, dataType, invariantArgs, loadAttrs);
+      RawBufferLoadOp::create(rewriter, loc, dataType, invariantArgs, loadAttrs);
   Block *currentBlock = rewriter.getInsertionBlock();
   Block *afterAtomic =
       rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
   Block *loopBlock = rewriter.createBlock(afterAtomic, {dataType}, {loc});
 
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(loc, loopBlock, initialLoad);
+  cf::BranchOp::create(rewriter, loc, loopBlock, initialLoad);
 
   rewriter.setInsertionPointToEnd(loopBlock);
   Value prevLoad = loopBlock->getArgument(0);
-  Value operated = rewriter.create<ArithOp>(loc, data, prevLoad);
+  Value operated = ArithOp::create(rewriter, loc, data, prevLoad);
   dataType = operated.getType();
 
   SmallVector<NamedAttribute> cmpswapAttrs;
   patchOperandSegmentSizes(origAttrs, cmpswapAttrs, DataArgAction::Duplicate);
   SmallVector<Value> cmpswapArgs = {operated, prevLoad};
   cmpswapArgs.append(invariantArgs.begin(), invariantArgs.end());
-  Value atomicRes = rewriter.create<RawBufferAtomicCmpswapOp>(
+  Value atomicRes = RawBufferAtomicCmpswapOp::create(rewriter,
       loc, dataType, cmpswapArgs, cmpswapAttrs);
 
   // We care about exact bitwise equality here, so do some bitcasts.
@@ -151,13 +151,13 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   if (auto floatDataTy = dyn_cast<FloatType>(dataType)) {
     Type equivInt = rewriter.getIntegerType(floatDataTy.getWidth());
     prevLoadForCompare =
-        rewriter.create<arith::BitcastOp>(loc, equivInt, prevLoad);
+        arith::BitcastOp::create(rewriter, loc, equivInt, prevLoad);
     atomicResForCompare =
-        rewriter.create<arith::BitcastOp>(loc, equivInt, atomicRes);
+        arith::BitcastOp::create(rewriter, loc, equivInt, atomicRes);
   }
-  Value canLeave = rewriter.create<arith::CmpIOp>(
+  Value canLeave = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::eq, atomicResForCompare, prevLoadForCompare);
-  rewriter.create<cf::CondBranchOp>(loc, canLeave, afterAtomic, ValueRange{},
+  cf::CondBranchOp::create(rewriter, loc, canLeave, afterAtomic, ValueRange{},
                                     loopBlock, atomicRes);
   rewriter.eraseOp(atomicOp);
   return success();
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
index 9a368f372c296..cba0611f05ef8 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -54,9 +54,9 @@ static LogicalResult baseInBufferAddrSpace(PatternRewriter &rewriter,
 static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
                                            vector::MaskedLoadOp maskedOp) {
   VectorType vectorType = maskedOp.getVectorType();
-  Value load = builder.create<vector::LoadOp>(
+  Value load = vector::LoadOp::create(builder,
       loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
-  Value res = builder.create<arith::SelectOp>(
+  Value res = arith::SelectOp::create(builder,
       loc, vectorType, maskedOp.getMask(), load, maskedOp.getPassThru());
   return res;
 }
@@ -87,7 +87,7 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
     SmallVector<OpFoldResult> indices = maskedOp.getIndices();
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, src);
     SmallVector<OpFoldResult> strides =
         stridedMetadata.getConstifiedMixedStrides();
     SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
@@ -101,46 +101,46 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
 
     // delta = bufferSize - linearizedOffset
     Value vectorSizeOffset =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
+        arith::ConstantIndexOp::create(rewriter, loc, vectorSize);
     Value linearIndex =
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
     Value totalSize = getValueOrCreateConstantIndexOp(
         rewriter, loc, linearizedInfo.linearizedSize);
-    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
+    Value delta = arith::SubIOp::create(rewriter, loc, totalSize, linearIndex);
 
     // 1) check if delta < vectorSize
-    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
+    Value isOutofBounds = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
 
     // 2) check if (detla % elements_per_word != 0)
-    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
+    Value elementsPerWord = arith::ConstantIndexOp::create(rewriter,
         loc, llvm::divideCeil(32, elementBitWidth));
-    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
+    Value isNotWordAligned = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne,
-        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
+        arith::RemUIOp::create(rewriter, loc, delta, elementsPerWord),
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
 
     // We take the fallback of maskedload default lowering only it is both
     // out-of-bounds and not word aligned. The fallback ensures correct results
     // when loading at the boundary of the buffer since buffer load returns
     // inconsistent zeros for the whole word when boundary is crossed.
     Value ifCondition =
-        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
+        arith::AndIOp::create(rewriter, loc, isOutofBounds, isNotWordAligned);
 
     auto thenBuilder = [&](OpBuilder &builder, Location loc) {
       Operation *read = builder.clone(*maskedOp.getOperation());
       read->setAttr(kMaskedloadNeedsMask, builder.getUnitAttr());
       Value readResult = read->getResult(0);
-      builder.create<scf::YieldOp>(loc, readResult);
+      scf::YieldOp::create(builder, loc, readResult);
     };
 
     auto elseBuilder = [&](OpBuilder &builder, Location loc) {
       Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp);
-      rewriter.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(rewriter, loc, res);
     };
 
     auto ifOp =
-        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
+        scf::IfOp::create(rewriter, loc, ifCondition, thenBuilder, elseBuilder);
 
     rewriter.replaceOp(maskedOp, ifOp);
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
index 195f59d625554..4771d655e36c2 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
@@ -37,7 +37,7 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       return rewriter.notifyMatchFailure(metadataOp,
                                          "not a fat raw buffer cast");
     Location loc = castOp.getLoc();
-    auto sourceMetadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+    auto sourceMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
         loc, castOp.getSource());
     SmallVector<Value> results;
     if (metadataOp.getBaseBuffer().use_empty()) {
@@ -48,13 +48,13 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       if (baseBufferType == castOp.getResult().getType()) {
         results.push_back(castOp.getResult());
       } else {
-        results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+        results.push_back(memref::ReinterpretCastOp::create(rewriter,
             loc, baseBufferType, castOp.getResult(), /*offset=*/0,
             /*sizes=*/ArrayRef<int64_t>{}, /*strides=*/ArrayRef<int64_t>{}));
       }
     }
     if (castOp.getResetOffset())
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, 0));
     else
       results.push_back(sourceMetadata.getOffset());
     llvm::append_range(results, sourceMetadata.getSizes());
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 12b375b373fa9..748ff1edbfeb2 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -76,8 +76,8 @@ static SmallVector<Value> getTileSizes(Location loc, amx::TileType tType,
   auto mattr = rewriter.getI16IntegerAttr(tType.getDimSize(0));
   auto nattr = rewriter.getI16IntegerAttr(tType.getDimSize(1) * bytes);
   return SmallVector<Value>{
-      rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, mattr),
-      rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, nattr)};
+      LLVM::ConstantOp::create(rewriter, loc, llvmInt16Type, mattr),
+      LLVM::ConstantOp::create(rewriter, loc, llvmInt16Type, nattr)};
 }
 
 /// Maps the 2-dim memref shape to the 64-bit stride. Note that the buffer
@@ -95,7 +95,7 @@ static Value getStride(Location loc, MemRefType mType, Value base,
     // Dynamic stride needs code to compute the stride at runtime.
     MemRefDescriptor memrefDescriptor(base);
     auto attr = rewriter.getI64IntegerAttr(bytes);
-    Value scale = rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr);
+    Value scale = LLVM::ConstantOp::create(rewriter, loc, llvmInt64Type, attr);
     return rewriter
         .create<LLVM::MulOp>(loc, llvmInt64Type, scale,
                              memrefDescriptor.stride(rewriter, loc, preLast))
@@ -103,7 +103,7 @@ static Value getStride(Location loc, MemRefType mType, Value base,
   }
   // Use direct constant for static stride.
   auto attr = rewriter.getI64IntegerAttr(strides[preLast] * bytes);
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr)
+  return LLVM::ConstantOp::create(rewriter, loc, llvmInt64Type, attr)
       .getResult();
 }
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index bd4b2e56808b6..263663d041f3a 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -242,7 +242,7 @@ Operation *AffineDialect::materializeConstant(OpBuilder &builder,
                                               Attribute value, Type type,
                                               Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
 
@@ -1284,7 +1284,7 @@ mlir::affine::makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map,
   map = foldAttributesIntoMap(b, map, operands, valueOperands);
   composeAffineMapAndOperands(&map, &valueOperands, composeAffineMin);
   assert(map);
-  return b.create<AffineApplyOp>(loc, map, valueOperands);
+  return AffineApplyOp::create(b, loc, map, valueOperands);
 }
 
 AffineApplyOp
@@ -1391,7 +1391,7 @@ static OpTy makeComposedMinMax(OpBuilder &b, Location loc, AffineMap map,
   SmallVector<Value> valueOperands;
   map = foldAttributesIntoMap(b, map, operands, valueOperands);
   composeMultiResultAffineMap(map, valueOperands);
-  return b.create<OpTy>(loc, b.getIndexType(), map, valueOperands);
+  return OpTy::create(b, loc, b.getIndexType(), map, valueOperands);
 }
 
 AffineMinOp
@@ -2690,7 +2690,7 @@ FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInits());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  AffineForOp newLoop = rewriter.create<AffineForOp>(
+  AffineForOp newLoop = AffineForOp::create(rewriter,
       getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
       getUpperBoundOperands(), getUpperBoundMap(), getStepAsInt(), inits);
 
@@ -2833,7 +2833,7 @@ static void buildAffineLoopNestImpl(
         OpBuilder::InsertionGuard nestedGuard(nestedBuilder);
         bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
       }
-      nestedBuilder.create<AffineYieldOp>(nestedLoc);
+      AffineYieldOp::create(nestedBuilder, nestedLoc);
     };
 
     // Delegate actual loop creation to the callback in order to dispatch
@@ -2848,7 +2848,7 @@ static AffineForOp
 buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb,
                              int64_t ub, int64_t step,
                              AffineForOp::BodyBuilderFn bodyBuilderFn) {
-  return builder.create<AffineForOp>(loc, lb, ub, step,
+  return AffineForOp::create(builder, loc, lb, ub, step,
                                      /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
@@ -2862,7 +2862,7 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
   if (lbConst && ubConst)
     return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
                                         ubConst.value(), step, bodyBuilderFn);
-  return builder.create<AffineForOp>(loc, lb, builder.getDimIdentityMap(), ub,
+  return AffineForOp::create(builder, loc, lb, builder.getDimIdentityMap(), ub,
                                      builder.getDimIdentityMap(), step,
                                      /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
@@ -4885,7 +4885,7 @@ struct DropUnitExtentBasis
     Location loc = delinearizeOp->getLoc();
     auto getZero = [&]() -> Value {
       if (!zero)
-        zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+        zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
       return zero.value();
     };
 
@@ -4908,7 +4908,7 @@ struct DropUnitExtentBasis
 
     if (!newBasis.empty()) {
       // Will drop the leading nullptr from `basis` if there was no outer bound.
-      auto newDelinearizeOp = rewriter.create<affine::AffineDelinearizeIndexOp>(
+      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(rewriter,
           loc, delinearizeOp.getLinearIndex(), newBasis);
       int newIndex = 0;
       // Map back the new delinearized indices to the values they replace.
@@ -4973,11 +4973,11 @@ struct CancelDelinearizeOfLinearizeDisjointExactTail
       return success();
     }
 
-    Value newLinearize = rewriter.create<affine::AffineLinearizeIndexOp>(
+    Value newLinearize = affine::AffineLinearizeIndexOp::create(rewriter,
         linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
         ArrayRef<OpFoldResult>{linearizeBasis}.drop_back(numMatches),
         linearizeOp.getDisjoint());
-    auto newDelinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
         delinearizeOp.getLoc(), newLinearize,
         ArrayRef<OpFoldResult>{delinearizeBasis}.drop_back(numMatches),
         delinearizeOp.hasOuterBound());
@@ -5051,17 +5051,17 @@ struct SplitDelinearizeSpanningLastLinearizeArg final
           "need at least two elements to form the basis product");
 
     Value linearizeWithoutBack =
-        rewriter.create<affine::AffineLinearizeIndexOp>(
+        affine::AffineLinearizeIndexOp::create(rewriter,
             linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
             linearizeOp.getDynamicBasis(),
             linearizeOp.getStaticBasis().drop_back(),
             linearizeOp.getDisjoint());
     auto delinearizeWithoutSplitPart =
-        rewriter.create<affine::AffineDelinearizeIndexOp>(
+        affine::AffineDelinearizeIndexOp::create(rewriter,
             delinearizeOp.getLoc(), linearizeWithoutBack,
             delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
             delinearizeOp.hasOuterBound());
-    auto delinearizeBack = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(rewriter,
         delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
         basis.take_back(elemsToSplit), /*hasOuterBound=*/true);
     SmallVector<Value> results = llvm::to_vector(
@@ -5274,7 +5274,7 @@ OpFoldResult computeProduct(Location loc, OpBuilder &builder,
   }
   if (auto constant = dyn_cast<AffineConstantExpr>(result))
     return getAsIndexOpFoldResult(builder.getContext(), constant.getValue());
-  return builder.create<AffineApplyOp>(loc, result, dynamicPart).getResult();
+  return AffineApplyOp::create(builder, loc, result, dynamicPart).getResult();
 }
 
 /// If conseceutive outputs of a delinearize_index are linearized with the same
@@ -5439,7 +5439,7 @@ struct CancelLinearizeOfDelinearizePortion final
       newDelinBasis.erase(newDelinBasis.begin() + m.delinStart,
                           newDelinBasis.begin() + m.delinStart + m.length);
       newDelinBasis.insert(newDelinBasis.begin() + m.delinStart, newSize);
-      auto newDelinearize = rewriter.create<AffineDelinearizeIndexOp>(
+      auto newDelinearize = AffineDelinearizeIndexOp::create(rewriter,
           m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
           newDelinBasis);
 
@@ -5447,7 +5447,7 @@ struct CancelLinearizeOfDelinearizePortion final
       // create a residual affine.delinearize_index that delinearizes the
       // merged output into its component parts.
       Value combinedElem = newDelinearize.getResult(m.delinStart);
-      auto residualDelinearize = rewriter.create<AffineDelinearizeIndexOp>(
+      auto residualDelinearize = AffineDelinearizeIndexOp::create(rewriter,
           m.delinearize.getLoc(), combinedElem, basisToMerge);
 
       // Swap all the uses of the unaffected delinearize outputs to the new
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index c11f1bca5d49d..23facb83caebd 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -204,7 +204,7 @@ void AffineDataCopyGeneration::runOnBlock(Block *block,
 void AffineDataCopyGeneration::runOnOperation() {
   func::FuncOp f = getOperation();
   OpBuilder topBuilder(f.getBody());
-  zeroIndex = topBuilder.create<arith::ConstantIndexOp>(f.getLoc(), 0);
+  zeroIndex = arith::ConstantIndexOp::create(topBuilder, f.getLoc(), 0);
 
   // Nests that are copy-in's or copy-out's; the root AffineForOps of those
   // nests are stored herein.
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
index c0ef28c648ac5..5ad4893c18094 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
@@ -60,7 +60,7 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
       // Note: basis elements and their products are, definitionally,
       // non-negative, so `nuw` is justified.
       if (dynamicPart)
-        dynamicPart = rewriter.create<arith::MulIOp>(
+        dynamicPart = arith::MulIOp::create(rewriter,
             loc, dynamicPart, dynamicBasis[dynamicIndex - 1], ovflags);
       else
         dynamicPart = dynamicBasis[dynamicIndex - 1];
@@ -76,7 +76,7 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
           rewriter.createOrFold<arith::ConstantIndexOp>(loc, staticPart);
       if (dynamicPart)
         stride =
-            rewriter.create<arith::MulIOp>(loc, dynamicPart, stride, ovflags);
+            arith::MulIOp::create(rewriter, loc, dynamicPart, stride, ovflags);
       result.push_back(stride);
     }
   }
@@ -108,19 +108,19 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
   Value zero = rewriter.createOrFold<arith::ConstantIndexOp>(loc, 0);
 
   Value initialPart =
-      rewriter.create<arith::FloorDivSIOp>(loc, linearIdx, strides.front());
+      arith::FloorDivSIOp::create(rewriter, loc, linearIdx, strides.front());
   results.push_back(initialPart);
 
   auto emitModTerm = [&](Value stride) -> Value {
-    Value remainder = rewriter.create<arith::RemSIOp>(loc, linearIdx, stride);
-    Value remainderNegative = rewriter.create<arith::CmpIOp>(
+    Value remainder = arith::RemSIOp::create(rewriter, loc, linearIdx, stride);
+    Value remainderNegative = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, remainder, zero);
     // If the correction is relevant, this term is <= stride, which is known
     // to be positive in `index`. Otherwise, while 2 * stride might overflow,
     // this branch won't be taken, so the risk of `poison` is fine.
-    Value corrected = rewriter.create<arith::AddIOp>(
+    Value corrected = arith::AddIOp::create(rewriter,
         loc, remainder, stride, arith::IntegerOverflowFlags::nsw);
-    Value mod = rewriter.create<arith::SelectOp>(loc, remainderNegative,
+    Value mod = arith::SelectOp::create(rewriter, loc, remainderNegative,
                                                  corrected, remainder);
     return mod;
   };
@@ -133,7 +133,7 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
     // We know both inputs are positive, so floorDiv == div.
     // This could potentially be a divui, but it's not clear if that would
     // cause issues.
-    Value divided = rewriter.create<arith::DivSIOp>(loc, modulus, nextStride);
+    Value divided = arith::DivSIOp::create(rewriter, loc, modulus, nextStride);
     results.push_back(divided);
   }
 
@@ -169,7 +169,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   // our hands on an `OpOperand&` for the loop invariant counting function.
   for (auto [stride, idxOp] :
        llvm::zip_equal(strides, llvm::drop_end(op.getMultiIndexMutable()))) {
-    Value scaledIdx = rewriter.create<arith::MulIOp>(
+    Value scaledIdx = arith::MulIOp::create(rewriter,
         loc, idxOp.get(), stride, arith::IntegerOverflowFlags::nsw);
     int64_t numHoistableLoops = numEnclosingInvariantLoops(idxOp);
     scaledValues.emplace_back(scaledIdx, numHoistableLoops);
@@ -186,7 +186,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   Value result = scaledValues.front().first;
   for (auto [scaledValue, numHoistableLoops] : llvm::drop_begin(scaledValues)) {
     std::ignore = numHoistableLoops;
-    result = rewriter.create<arith::AddIOp>(loc, result, scaledValue,
+    result = arith::AddIOp::create(rewriter, loc, result, scaledValue,
                                             arith::IntegerOverflowFlags::nsw);
   }
   rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
index f28fb3acb7db7..bdaa2a968a0cb 100644
--- a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
@@ -89,7 +89,7 @@ static AffineApplyOp createSubApply(RewriterBase &rewriter,
   auto rhsMap = AffineMap::get(m.getNumDims(), m.getNumSymbols(), expr, ctx);
   SmallVector<Value> rhsOperands = originalOp->getOperands();
   canonicalizeMapAndOperands(&rhsMap, &rhsOperands);
-  return rewriter.create<AffineApplyOp>(originalOp.getLoc(), rhsMap,
+  return AffineApplyOp::create(rewriter, originalOp.getLoc(), rhsMap,
                                         rhsOperands);
 }
 
@@ -161,7 +161,7 @@ FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
   auto current = createSubApply(rewriter, op, subExpressions[0]);
   for (int64_t i = 1, e = subExpressions.size(); i < e; ++i) {
     Value tmp = createSubApply(rewriter, op, subExpressions[i]);
-    current = rewriter.create<AffineApplyOp>(op.getLoc(), binMap,
+    current = AffineApplyOp::create(rewriter, op.getLoc(), binMap,
                                              ValueRange{current, tmp});
     LLVM_DEBUG(DBGS() << "--reassociate into: " << current << "\n");
   }
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 8f42586e5d18a..bfc8a7296f554 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -426,7 +426,7 @@ static Value createPrivateMemRef(AffineForOp forOp,
   // consumer loop nests to reduce their live range. Currently they are added
   // at the beginning of the block, because loop nests can be reordered
   // during the fusion pass.
-  Value newMemRef = top.create<memref::AllocOp>(forOp.getLoc(), newMemRefType);
+  Value newMemRef = memref::AllocOp::create(top, forOp.getLoc(), newMemRefType);
 
   // Build an AffineMap to remap access functions based on lower bound offsets.
   SmallVector<AffineExpr, 4> remapExprs;
diff --git a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
index 92cb7075005a3..123d15f2f8d39 100644
--- a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
@@ -102,7 +102,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
 
   // Create and place the alloc right before the 'affine.for' operation.
-  Value newMemRef = bOuter.create<memref::AllocOp>(
+  Value newMemRef = memref::AllocOp::create(bOuter,
       forOp.getLoc(), newMemRefType, allocOperands);
 
   // Create 'iv mod 2' value to index the leading dimension.
@@ -110,7 +110,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   int64_t step = forOp.getStepAsInt();
   auto modTwoMap =
       AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, d0.floorDiv(step) % 2);
-  auto ivModTwoOp = bInner.create<AffineApplyOp>(forOp.getLoc(), modTwoMap,
+  auto ivModTwoOp = AffineApplyOp::create(bInner, forOp.getLoc(), modTwoMap,
                                                  forOp.getInductionVar());
 
   // replaceAllMemRefUsesWith will succeed unless the forOp body has
@@ -132,7 +132,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
   // Insert the dealloc op right after the for loop.
   bOuter.setInsertionPointAfter(forOp);
-  bOuter.create<memref::DeallocOp>(forOp.getLoc(), newMemRef);
+  memref::DeallocOp::create(bOuter, forOp.getLoc(), newMemRef);
 
   return true;
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
index 1a266b72d1f8d..9537d3e75c26a 100644
--- a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
@@ -51,10 +51,10 @@ OpFoldResult affine::materializeComputedBound(
            "expected dynamic dim");
     if (isa<RankedTensorType>(value.getType())) {
       // A tensor dimension is used: generate a tensor.dim.
-      operands.push_back(b.create<tensor::DimOp>(loc, value, *dim));
+      operands.push_back(tensor::DimOp::create(b, loc, value, *dim));
     } else if (isa<MemRefType>(value.getType())) {
       // A memref dimension is used: generate a memref.dim.
-      operands.push_back(b.create<memref::DimOp>(loc, value, *dim));
+      operands.push_back(memref::DimOp::create(b, loc, value, *dim));
     } else {
       llvm_unreachable("cannot generate DimOp for unsupported shaped type");
     }
@@ -76,7 +76,7 @@ OpFoldResult affine::materializeComputedBound(
         operands[expr.getPosition() + boundMap.getNumDims()]);
   // General case: build affine.apply op.
   return static_cast<OpFoldResult>(
-      b.create<affine::AffineApplyOp>(loc, boundMap, operands).getResult());
+      affine::AffineApplyOp::create(b, loc, boundMap, operands).getResult());
 }
 
 FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound(
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index 7fae260767e0a..b078514cbf978 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -905,7 +905,7 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   for (auto resultExpr : map.getResults()) {
     auto singleResMap =
         AffineMap::get(map.getNumDims(), map.getNumSymbols(), resultExpr);
-    auto afOp = state.builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+    auto afOp = state.AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                     mapOperands);
     results.push_back(afOp);
   }
@@ -961,7 +961,7 @@ static arith::ConstantOp vectorizeConstant(arith::ConstantOp constOp,
   auto vecForOp = cast<AffineForOp>(parentOp);
   state.builder.setInsertionPointToStart(vecForOp.getBody());
   auto newConstOp =
-      state.builder.create<arith::ConstantOp>(constOp.getLoc(), vecAttr);
+      state.arith::ConstantOp::create(builder, constOp.getLoc(), vecAttr);
 
   // Register vector replacement for future uses in the scope.
   state.registerOpVectorReplacement(constOp, newConstOp);
@@ -986,7 +986,7 @@ static Operation *vectorizeAffineApplyOp(AffineApplyOp applyOp,
     }
   }
 
-  auto newApplyOp = state.builder.create<AffineApplyOp>(
+  auto newApplyOp = state.AffineApplyOp::create(builder,
       applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
 
   // Register the new affine.apply result.
@@ -1010,7 +1010,7 @@ static arith::ConstantOp createInitialVector(arith::AtomicRMWKind reductionKind,
   auto vecTy = getVectorType(scalarTy, state.strategy);
   auto vecAttr = DenseElementsAttr::get(vecTy, valueAttr);
   auto newConstOp =
-      state.builder.create<arith::ConstantOp>(oldOperand.getLoc(), vecAttr);
+      state.arith::ConstantOp::create(builder, oldOperand.getLoc(), vecAttr);
 
   return newConstOp;
 }
@@ -1062,10 +1062,10 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   AffineMap ubMap = vecForOp.getUpperBoundMap();
   Value ub;
   if (ubMap.getNumResults() == 1)
-    ub = state.builder.create<AffineApplyOp>(loc, vecForOp.getUpperBoundMap(),
+    ub = state.AffineApplyOp::create(builder, loc, vecForOp.getUpperBoundMap(),
                                              vecForOp.getUpperBoundOperands());
   else
-    ub = state.builder.create<AffineMinOp>(loc, vecForOp.getUpperBoundMap(),
+    ub = state.AffineMinOp::create(builder, loc, vecForOp.getUpperBoundMap(),
                                            vecForOp.getUpperBoundOperands());
   // Then we compute the number of (original) iterations left in the loop.
   AffineExpr subExpr =
@@ -1080,7 +1080,7 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   Type maskTy = VectorType::get(state.strategy->vectorSizes,
                                 state.builder.getIntegerType(1));
   Value mask =
-      state.builder.create<vector::CreateMaskOp>(loc, maskTy, itersLeft);
+      state.vector::CreateMaskOp::create(builder, loc, maskTy, itersLeft);
 
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ creating a mask:\n"
                     << itersLeft << "\n"
@@ -1123,7 +1123,7 @@ static Operation *vectorizeUniform(Value uniformVal,
   state.builder.setInsertionPointAfterValue(uniformScalarRepl);
 
   auto vectorTy = getVectorType(uniformVal.getType(), state.strategy);
-  auto bcastOp = state.builder.create<BroadcastOp>(uniformVal.getLoc(),
+  auto bcastOp = state.BroadcastOp::create(builder, uniformVal.getLoc(),
                                                    vectorTy, uniformScalarRepl);
   state.registerValueVectorReplacement(uniformVal, bcastOp);
   return bcastOp;
@@ -1256,7 +1256,7 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.builder.create<vector::TransferReadOp>(
+  auto transfer = state.vector::TransferReadOp::create(builder,
       loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
       /*padding=*/std::nullopt, permutationMap);
 
@@ -1303,7 +1303,7 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.builder.create<vector::TransferWriteOp>(
+  auto transfer = state.vector::TransferWriteOp::create(builder,
       storeOp.getLoc(), vectorValue, storeOp.getMemRef(), indices,
       permutationMap);
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
@@ -1387,7 +1387,7 @@ static Operation *vectorizeAffineForOp(AffineForOp forOp,
     }
   }
 
-  auto vecForOp = state.builder.create<AffineForOp>(
+  auto vecForOp = state.AffineForOp::create(builder,
       forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(),
       forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep,
       vecIterOperands,
@@ -1512,7 +1512,7 @@ static Operation *vectorizeAffineYieldOp(AffineYieldOp yieldOp,
       // IterOperands are neutral element vectors.
       Value neutralVal = cast<AffineForOp>(newParentOp).getInits()[i];
       state.builder.setInsertionPoint(combinerOps.back());
-      Value maskedReducedVal = state.builder.create<arith::SelectOp>(
+      Value maskedReducedVal = state.arith::SelectOp::create(builder,
           reducedVal.getLoc(), mask, reducedVal, neutralVal);
       LLVM_DEBUG(
           dbgs() << "\n[early-vect]+++++ masking an input to a binary op that"
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 0501616ad912c..0f16e84263ec1 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -55,7 +55,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
 
   OpBuilder b(forOp);
   auto lbMap = forOp.getLowerBoundMap();
-  auto lb = b.create<AffineApplyOp>(forOp.getLoc(), lbMap,
+  auto lb = AffineApplyOp::create(b, forOp.getLoc(), lbMap,
                                     forOp.getLowerBoundOperands());
 
   // For each upper bound expr, get the range.
@@ -72,7 +72,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
     auto bumpMap = AffineMap::get(tripCountMap.getNumDims(),
                                   tripCountMap.getNumSymbols(), bumpExprs[i]);
     bumpValues[i] =
-        b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, tripCountOperands);
+        AffineApplyOp::create(b, forOp.getLoc(), bumpMap, tripCountOperands);
   }
 
   SmallVector<AffineExpr, 4> newUbExprs(tripCountMap.getNumResults());
@@ -135,7 +135,7 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         builder.setInsertionPointToStart(&func.getFunctionBody().front());
       else
         builder.setInsertionPoint(forOp);
-      auto constOp = builder.create<arith::ConstantIndexOp>(
+      auto constOp = arith::ConstantIndexOp::create(builder,
           forOp.getLoc(), forOp.getConstantLowerBound());
       iv.replaceAllUsesWith(constOp);
     } else {
@@ -147,7 +147,7 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         iv.replaceAllUsesWith(lbOperands[0]);
       } else {
         auto affineApplyOp =
-            builder.create<AffineApplyOp>(forOp.getLoc(), lbMap, lbOperands);
+            AffineApplyOp::create(builder, forOp.getLoc(), lbMap, lbOperands);
         iv.replaceAllUsesWith(affineApplyOp);
       }
     }
@@ -182,7 +182,7 @@ static AffineForOp generateShiftedLoop(
   assert(ubMap.getNumInputs() == ubOperands.size());
 
   auto loopChunk =
-      b.create<AffineForOp>(srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
+      AffineForOp::create(b, srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
                             ubMap, srcForOp.getStepAsInt());
   auto loopChunkIV = loopChunk.getInductionVar();
   auto srcIV = srcForOp.getInductionVar();
@@ -198,7 +198,7 @@ static AffineForOp generateShiftedLoop(
     // Generate the remapping if the shift is not zero: remappedIV = newIV -
     // shift.
     if (!srcIV.use_empty() && shift != 0) {
-      auto ivRemap = bodyBuilder.create<AffineApplyOp>(
+      auto ivRemap = AffineApplyOp::create(bodyBuilder,
           srcForOp.getLoc(),
           bodyBuilder.getSingleDimShiftAffineMap(
               -static_cast<int64_t>(srcForOp.getStepAsInt() * shift)),
@@ -434,7 +434,7 @@ static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
   for (unsigned i = 0; i < width; i++) {
     OpBuilder b(topLoop);
     // Loop bounds will be set later.
-    AffineForOp pointLoop = b.create<AffineForOp>(loc, 0, 0);
+    AffineForOp pointLoop = AffineForOp::create(b, loc, 0, 0);
     pointLoop.getBody()->getOperations().splice(
         pointLoop.getBody()->begin(), topLoop->getBlock()->getOperations(),
         topLoop);
@@ -448,7 +448,7 @@ static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
   for (unsigned i = width; i < 2 * width; i++) {
     OpBuilder b(topLoop);
     // Loop bounds will be set later.
-    AffineForOp tileSpaceLoop = b.create<AffineForOp>(loc, 0, 0);
+    AffineForOp tileSpaceLoop = AffineForOp::create(b, loc, 0, 0);
     tileSpaceLoop.getBody()->getOperations().splice(
         tileSpaceLoop.getBody()->begin(), topLoop->getBlock()->getOperations(),
         topLoop);
@@ -1063,7 +1063,7 @@ LogicalResult mlir::affine::loopUnrollByFactor(
         // iv' = iv + i * step
         auto d0 = b.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, d0 + i * step);
-        return b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, iv);
+        return AffineApplyOp::create(b, forOp.getLoc(), bumpMap, iv);
       },
       /*annotateFn=*/annotateFn,
       /*iterArgs=*/iterArgs, /*yieldedValues=*/yieldedValues);
@@ -1227,7 +1227,7 @@ LogicalResult mlir::affine::loopUnrollJamByFactor(AffineForOp forOp,
         auto d0 = builder.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, d0 + i * step);
         auto ivUnroll =
-            builder.create<AffineApplyOp>(forOp.getLoc(), bumpMap, forOpIV);
+            AffineApplyOp::create(builder, forOp.getLoc(), bumpMap, forOpIV);
         operandMaps[i - 1].map(forOpIV, ivUnroll);
       }
       // Clone the sub-block being unroll-jammed.
@@ -1556,7 +1556,7 @@ stripmineSink(AffineForOp forOp, uint64_t factor,
   for (auto t : targets) {
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator(t.getBody());
-    auto newForOp = b.create<AffineForOp>(t.getLoc(), lbOperands, lbMap,
+    auto newForOp = AffineForOp::create(b, t.getLoc(), lbOperands, lbMap,
                                           ubOperands, ubMap, originalStep);
     auto begin = t.getBody()->begin();
     // Skip terminator and `newForOp` which is just before the terminator.
@@ -1631,9 +1631,9 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
   // 1. Store the upper bound of the outermost loop in a variable.
   Value prev;
   if (!llvm::hasSingleElement(origUbMap.getResults()))
-    prev = builder.create<AffineMinOp>(loc, origUbMap, ubOperands);
+    prev = AffineMinOp::create(builder, loc, origUbMap, ubOperands);
   else
-    prev = builder.create<AffineApplyOp>(loc, origUbMap, ubOperands);
+    prev = AffineApplyOp::create(builder, loc, origUbMap, ubOperands);
   upperBoundSymbols.push_back(prev);
 
   // 2. Emit code computing the upper bound of the coalesced loop as product of
@@ -1645,15 +1645,15 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
     Value upperBound;
     // If upper bound map has more than one result, take their minimum.
     if (!llvm::hasSingleElement(origUbMap.getResults()))
-      upperBound = builder.create<AffineMinOp>(loc, origUbMap, ubOperands);
+      upperBound = AffineMinOp::create(builder, loc, origUbMap, ubOperands);
     else
-      upperBound = builder.create<AffineApplyOp>(loc, origUbMap, ubOperands);
+      upperBound = AffineApplyOp::create(builder, loc, origUbMap, ubOperands);
     upperBoundSymbols.push_back(upperBound);
     SmallVector<Value, 4> operands;
     operands.push_back(prev);
     operands.push_back(upperBound);
     // Maintain running product of loop upper bounds.
-    prev = builder.create<AffineApplyOp>(
+    prev = AffineApplyOp::create(builder,
         loc,
         AffineMap::get(/*dimCount=*/1,
                        /*symbolCount=*/1,
@@ -1683,7 +1683,7 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> operands;
       operands.push_back(previous);
       operands.push_back(upperBoundSymbols[idx]);
-      previous = builder.create<AffineApplyOp>(
+      previous = AffineApplyOp::create(builder,
           loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
@@ -1700,7 +1700,7 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> applyOperands;
       applyOperands.push_back(previous);
       applyOperands.push_back(upperBoundSymbols[idx - 1]);
-      inductionVariable = builder.create<AffineApplyOp>(
+      inductionVariable = AffineApplyOp::create(builder,
           loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
@@ -1738,21 +1738,21 @@ void mlir::affine::mapLoopToProcessorIds(scf::ForOp forOp,
 
   Value linearIndex = processorId.front();
   for (unsigned i = 1, e = processorId.size(); i < e; ++i) {
-    auto mulApplyOp = b.create<AffineApplyOp>(
+    auto mulApplyOp = AffineApplyOp::create(b,
         loc, mulMap, ValueRange{linearIndex, numProcessors[i]});
-    linearIndex = b.create<AffineApplyOp>(
+    linearIndex = AffineApplyOp::create(b,
         loc, addMap, ValueRange{mulApplyOp, processorId[i]});
   }
 
-  auto mulApplyOp = b.create<AffineApplyOp>(
+  auto mulApplyOp = AffineApplyOp::create(b,
       loc, mulMap, ValueRange{linearIndex, forOp.getStep()});
-  Value lb = b.create<AffineApplyOp>(
+  Value lb = AffineApplyOp::create(b,
       loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()});
   forOp.setLowerBound(lb);
 
   Value step = forOp.getStep();
   for (auto numProcs : numProcessors)
-    step = b.create<AffineApplyOp>(loc, mulMap, ValueRange{numProcs, step});
+    step = AffineApplyOp::create(b, loc, mulMap, ValueRange{numProcs, step});
   forOp.setStep(step);
 }
 
@@ -1889,7 +1889,7 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
 
     auto fastBufOffsetMap =
         AffineMap::get(lbOperands.size(), 0, fastBufOffsets[d]);
-    auto offset = b.create<AffineApplyOp>(loc, fastBufOffsetMap, lbOperands);
+    auto offset = AffineApplyOp::create(b, loc, fastBufOffsetMap, lbOperands);
 
     // Construct the subscript for the fast memref being copied into/from:
     // x - offset_x.
@@ -1916,16 +1916,16 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
 
   if (!isCopyOut) {
     // Copy in.
-    auto load = b.create<AffineLoadOp>(loc, memref, memIndices);
-    b.create<AffineStoreOp>(loc, load, fastMemRef, fastBufMap,
+    auto load = AffineLoadOp::create(b, loc, memref, memIndices);
+    AffineStoreOp::create(b, loc, load, fastMemRef, fastBufMap,
                             fastBufMapOperands);
     return copyNestRoot;
   }
 
   // Copy out.
   auto load =
-      b.create<AffineLoadOp>(loc, fastMemRef, fastBufMap, fastBufMapOperands);
-  b.create<AffineStoreOp>(loc, load, memref, memIndices);
+      AffineLoadOp::create(b, loc, fastMemRef, fastBufMap, fastBufMapOperands);
+  AffineStoreOp::create(b, loc, load, memref, memIndices);
   return copyNestRoot;
 }
 
@@ -1960,7 +1960,7 @@ static LogicalResult generateCopy(
 
   auto f = begin->getParentOfType<FunctionOpInterface>();
   OpBuilder topBuilder(f.getFunctionBody());
-  Value zeroIndex = topBuilder.create<arith::ConstantIndexOp>(f.getLoc(), 0);
+  Value zeroIndex = arith::ConstantIndexOp::create(topBuilder, f.getLoc(), 0);
 
   *sizeInBytes = 0;
 
@@ -2071,7 +2071,7 @@ static LogicalResult generateCopy(
         memIndices.push_back(zeroIndex);
       } else {
         memIndices.push_back(
-            top.create<arith::ConstantIndexOp>(loc, indexVal).getResult());
+            arith::ConstantIndexOp::create(top, loc, indexVal).getResult());
       }
     } else {
       // The coordinate for the start location is just the lower bound along the
@@ -2085,7 +2085,7 @@ static LogicalResult generateCopy(
       lbs[d] = lbs[d].replaceDimsAndSymbols(
           /*dimReplacements=*/{}, symReplacements, lbs[d].getNumSymbols(),
           /*numResultSyms=*/0);
-      memIndices.push_back(b.create<AffineApplyOp>(loc, lbs[d], regionSymbols));
+      memIndices.push_back(AffineApplyOp::create(b, loc, lbs[d], regionSymbols));
     }
     // The fast buffer is copied into at location zero; addressing is relative.
     bufIndices.push_back(zeroIndex);
@@ -2109,7 +2109,7 @@ static LogicalResult generateCopy(
     // Create the fast memory space buffer just before the 'affine.for'
     // operation.
     fastMemRef =
-        prologue.create<memref::AllocOp>(loc, fastMemRefType).getResult();
+        memref::AllocOp::create(prologue, loc, fastMemRefType).getResult();
     // Record it.
     fastBufferMap[memref] = fastMemRef;
     // fastMemRefType is a constant shaped memref.
@@ -2126,7 +2126,7 @@ static LogicalResult generateCopy(
     fastMemRef = fastBufferMap[memref];
   }
 
-  auto numElementsSSA = top.create<arith::ConstantIndexOp>(loc, *numElements);
+  auto numElementsSSA = arith::ConstantIndexOp::create(top, loc, *numElements);
 
   Value dmaStride;
   Value numEltPerDmaStride;
@@ -2143,8 +2143,8 @@ static LogicalResult generateCopy(
 
     if (!dmaStrideInfos.empty()) {
       dmaStride =
-          top.create<arith::ConstantIndexOp>(loc, dmaStrideInfos[0].stride);
-      numEltPerDmaStride = top.create<arith::ConstantIndexOp>(
+          arith::ConstantIndexOp::create(top, loc, dmaStrideInfos[0].stride);
+      numEltPerDmaStride = arith::ConstantIndexOp::create(top,
           loc, dmaStrideInfos[0].numEltPerStride);
     }
   }
@@ -2175,20 +2175,20 @@ static LogicalResult generateCopy(
     // Create a tag (single element 1-d memref) for the DMA.
     auto tagMemRefType = MemRefType::get({1}, top.getIntegerType(32), {},
                                          copyOptions.tagMemorySpace);
-    auto tagMemRef = prologue.create<memref::AllocOp>(loc, tagMemRefType);
+    auto tagMemRef = memref::AllocOp::create(prologue, loc, tagMemRefType);
 
     SmallVector<Value, 4> tagIndices({zeroIndex});
     auto tagAffineMap = b.getMultiDimIdentityMap(tagIndices.size());
     fullyComposeAffineMapAndOperands(&tagAffineMap, &tagIndices);
     if (!region.isWrite()) {
       // DMA non-blocking read from original buffer to fast buffer.
-      b.create<AffineDmaStartOp>(loc, memref, memAffineMap, memIndices,
+      AffineDmaStartOp::create(b, loc, memref, memAffineMap, memIndices,
                                  fastMemRef, bufAffineMap, bufIndices,
                                  tagMemRef, tagAffineMap, tagIndices,
                                  numElementsSSA, dmaStride, numEltPerDmaStride);
     } else {
       // DMA non-blocking write from fast buffer to the original memref.
-      auto op = b.create<AffineDmaStartOp>(
+      auto op = AffineDmaStartOp::create(b,
           loc, fastMemRef, bufAffineMap, bufIndices, memref, memAffineMap,
           memIndices, tagMemRef, tagAffineMap, tagIndices, numElementsSSA,
           dmaStride, numEltPerDmaStride);
@@ -2199,11 +2199,11 @@ static LogicalResult generateCopy(
     }
 
     // Matching DMA wait to block on completion; tag always has a 0 index.
-    b.create<AffineDmaWaitOp>(loc, tagMemRef, tagAffineMap, zeroIndex,
+    AffineDmaWaitOp::create(b, loc, tagMemRef, tagAffineMap, zeroIndex,
                               numElementsSSA);
 
     // Generate dealloc for the tag.
-    auto tagDeallocOp = epilogue.create<memref::DeallocOp>(loc, tagMemRef);
+    auto tagDeallocOp = memref::DeallocOp::create(epilogue, loc, tagMemRef);
     if (*nEnd == end && isCopyOutAtEndOfBlock)
       // Since new ops are being appended (for outgoing DMAs), adjust the end to
       // mark end of range of the original.
@@ -2212,7 +2212,7 @@ static LogicalResult generateCopy(
 
   // Generate dealloc for the buffer.
   if (!existingBuf) {
-    auto bufDeallocOp = epilogue.create<memref::DeallocOp>(loc, fastMemRef);
+    auto bufDeallocOp = memref::DeallocOp::create(epilogue, loc, fastMemRef);
     // When generating pointwise copies, `nEnd' has to be set to deallocOp on
     // the fast buffer (since it marks the new end insertion point).
     if (!copyOptions.generateDma && *nEnd == end && isCopyOutAtEndOfBlock)
@@ -2582,7 +2582,7 @@ AffineForOp mlir::affine::createCanonicalizedAffineForOp(
   canonicalizeMapAndOperands(&ubMap, &upperOperands);
   ubMap = removeDuplicateExprs(ubMap);
 
-  return b.create<AffineForOp>(loc, lowerOperands, lbMap, upperOperands, ubMap,
+  return AffineForOp::create(b, loc, lowerOperands, lbMap, upperOperands, ubMap,
                                step);
 }
 
@@ -2666,7 +2666,7 @@ static AffineIfOp createSeparationCondition(MutableArrayRef<AffineForOp> loops,
   SmallVector<Value, 4> setOperands;
   cst.getValues(0, cst.getNumDimAndSymbolVars(), &setOperands);
   canonicalizeSetAndOperands(&ifCondSet, &setOperands);
-  return b.create<AffineIfOp>(loops[0].getLoc(), ifCondSet, setOperands,
+  return AffineIfOp::create(b, loops[0].getLoc(), ifCondSet, setOperands,
                               /*withElseRegion=*/true);
 }
 
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 66b3f2a4f93a5..1666825aeac6a 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -58,7 +58,7 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     if (!lhs || !rhs)
       return nullptr;
-    auto op = builder.create<OpTy>(loc, lhs, rhs, overflowFlags);
+    auto op = OpTy::create(builder, loc, lhs, rhs, overflowFlags);
     return op.getResult();
   }
 
@@ -92,13 +92,13 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value remainder = builder.create<arith::RemSIOp>(loc, lhs, rhs);
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value isRemainderNegative = builder.create<arith::CmpIOp>(
+    Value remainder = arith::RemSIOp::create(builder, loc, lhs, rhs);
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value isRemainderNegative = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::slt, remainder, zeroCst);
     Value correctedRemainder =
-        builder.create<arith::AddIOp>(loc, remainder, rhs);
-    Value result = builder.create<arith::SelectOp>(
+        arith::AddIOp::create(builder, loc, remainder, rhs);
+    Value result = arith::SelectOp::create(builder,
         loc, isRemainderNegative, correctedRemainder, remainder);
     return result;
   }
@@ -131,17 +131,17 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value noneCst = builder.create<arith::ConstantIndexOp>(loc, -1);
-    Value negative = builder.create<arith::CmpIOp>(
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value noneCst = arith::ConstantIndexOp::create(builder, loc, -1);
+    Value negative = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::slt, lhs, zeroCst);
-    Value negatedDecremented = builder.create<arith::SubIOp>(loc, noneCst, lhs);
+    Value negatedDecremented = arith::SubIOp::create(builder, loc, noneCst, lhs);
     Value dividend =
-        builder.create<arith::SelectOp>(loc, negative, negatedDecremented, lhs);
-    Value quotient = builder.create<arith::DivSIOp>(loc, dividend, rhs);
+        arith::SelectOp::create(builder, loc, negative, negatedDecremented, lhs);
+    Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value correctedQuotient =
-        builder.create<arith::SubIOp>(loc, noneCst, quotient);
-    Value result = builder.create<arith::SelectOp>(loc, negative,
+        arith::SubIOp::create(builder, loc, noneCst, quotient);
+    Value result = arith::SelectOp::create(builder, loc, negative,
                                                    correctedQuotient, quotient);
     return result;
   }
@@ -170,26 +170,26 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value oneCst = builder.create<arith::ConstantIndexOp>(loc, 1);
-    Value nonPositive = builder.create<arith::CmpIOp>(
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value oneCst = arith::ConstantIndexOp::create(builder, loc, 1);
+    Value nonPositive = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::sle, lhs, zeroCst);
-    Value negated = builder.create<arith::SubIOp>(loc, zeroCst, lhs);
-    Value decremented = builder.create<arith::SubIOp>(loc, lhs, oneCst);
+    Value negated = arith::SubIOp::create(builder, loc, zeroCst, lhs);
+    Value decremented = arith::SubIOp::create(builder, loc, lhs, oneCst);
     Value dividend =
-        builder.create<arith::SelectOp>(loc, nonPositive, negated, decremented);
-    Value quotient = builder.create<arith::DivSIOp>(loc, dividend, rhs);
+        arith::SelectOp::create(builder, loc, nonPositive, negated, decremented);
+    Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value negatedQuotient =
-        builder.create<arith::SubIOp>(loc, zeroCst, quotient);
+        arith::SubIOp::create(builder, loc, zeroCst, quotient);
     Value incrementedQuotient =
-        builder.create<arith::AddIOp>(loc, quotient, oneCst);
-    Value result = builder.create<arith::SelectOp>(
+        arith::AddIOp::create(builder, loc, quotient, oneCst);
+    Value result = arith::SelectOp::create(builder,
         loc, nonPositive, negatedQuotient, incrementedQuotient);
     return result;
   }
 
   Value visitConstantExpr(AffineConstantExpr expr) {
-    auto op = builder.create<arith::ConstantIndexOp>(loc, expr.getValue());
+    auto op = arith::ConstantIndexOp::create(builder, loc, expr.getValue());
     return op.getResult();
   }
 
@@ -299,7 +299,7 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   // block.
   IRMapping operandMap;
   OpBuilder b(hoistOverOp);
-  auto hoistedIfOp = b.create<AffineIfOp>(ifOp.getLoc(), ifOp.getIntegerSet(),
+  auto hoistedIfOp = AffineIfOp::create(b, ifOp.getLoc(), ifOp.getIntegerSet(),
                                           ifOp.getOperands(),
                                           /*elseBlock=*/true);
 
@@ -370,7 +370,7 @@ mlir::affine::affineParallelize(AffineForOp forOp,
       parallelReductions, [](const LoopReduction &red) { return red.value; }));
   auto reductionKinds = llvm::to_vector<4>(llvm::map_range(
       parallelReductions, [](const LoopReduction &red) { return red.kind; }));
-  AffineParallelOp newPloop = outsideBuilder.create<AffineParallelOp>(
+  AffineParallelOp newPloop = AffineParallelOp::create(outsideBuilder,
       loc, ValueRange(reducedValues).getTypes(), reductionKinds,
       llvm::ArrayRef(lowerBoundMap), lowerBoundOperands,
       llvm::ArrayRef(upperBoundMap), upperBoundOperands,
@@ -542,7 +542,7 @@ void mlir::affine::normalizeAffineParallel(AffineParallelOp op) {
     SmallVector<Value, 8> applyOperands{dimOperands};
     applyOperands.push_back(iv);
     applyOperands.append(symbolOperands.begin(), symbolOperands.end());
-    auto apply = builder.create<AffineApplyOp>(op.getLoc(), map, applyOperands);
+    auto apply = AffineApplyOp::create(builder, op.getLoc(), map, applyOperands);
     iv.replaceAllUsesExcept(apply, apply);
   }
 
@@ -623,7 +623,7 @@ LogicalResult mlir::affine::normalizeAffineFor(AffineForOp op,
   AffineValueMap newIvToOldIvMap;
   AffineValueMap::difference(lbMap, scaleIvValueMap, &newIvToOldIvMap);
   (void)newIvToOldIvMap.canonicalize();
-  auto newIV = opBuilder.create<AffineApplyOp>(
+  auto newIV = AffineApplyOp::create(opBuilder,
       loc, newIvToOldIvMap.getAffineMap(), newIvToOldIvMap.getOperands());
   op.getInductionVar().replaceAllUsesExcept(newIV->getResult(0), newIV);
   return success();
@@ -1188,7 +1188,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     for (auto resultExpr : oldMap.getResults()) {
       auto singleResMap = AffineMap::get(oldMap.getNumDims(),
                                          oldMap.getNumSymbols(), resultExpr);
-      auto afOp = builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+      auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                 oldMapOperands);
       oldMemRefOperands.push_back(afOp);
       affineApplyOps.push_back(afOp);
@@ -1215,7 +1215,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     for (auto resultExpr : indexRemap.getResults()) {
       auto singleResMap = AffineMap::get(
           indexRemap.getNumDims(), indexRemap.getNumSymbols(), resultExpr);
-      auto afOp = builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+      auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                 remapOperands);
       remapOutputs.push_back(afOp);
       affineApplyOps.push_back(afOp);
@@ -1265,7 +1265,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     // AffineMapAccessInterface, we need to apply the values of `newMapOperands`
     // to the `newMap` to get the correct indices.
     for (unsigned i = 0; i < newMemRefRank; i++) {
-      state.operands.push_back(builder.create<AffineApplyOp>(
+      state.operands.push_back(AffineApplyOp::create(builder,
           op->getLoc(),
           AffineMap::get(newMap.getNumDims(), newMap.getNumSymbols(),
                          newMap.getResult(i)),
@@ -1451,7 +1451,7 @@ void mlir::affine::createAffineComputationSlice(
   for (auto resultExpr : composedMap.getResults()) {
     auto singleResMap = AffineMap::get(composedMap.getNumDims(),
                                        composedMap.getNumSymbols(), resultExpr);
-    sliceOps->push_back(builder.create<AffineApplyOp>(
+    sliceOps->push_back(AffineApplyOp::create(builder,
         opInst->getLoc(), singleResMap, composedOpOperands));
   }
 
@@ -1682,7 +1682,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
       // Create ConstantOp for static dimension.
       auto constantAttr = b.getIntegerAttr(b.getIndexType(), oldMemRefShape[d]);
       inAffineApply.emplace_back(
-          b.create<arith::ConstantOp>(allocOp.getLoc(), constantAttr));
+          arith::ConstantOp::create(b, allocOp.getLoc(), constantAttr));
     }
   }
 
@@ -1706,7 +1706,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
       AffineMap newMap =
           AffineMap::get(map.getNumInputs(), map.getNumSymbols(), newMapOutput);
       Value affineApp =
-          b.create<AffineApplyOp>(allocOp.getLoc(), newMap, inAffineApply);
+          AffineApplyOp::create(b, allocOp.getLoc(), newMap, inAffineApply);
       newDynamicSizes.emplace_back(affineApp);
     }
     newDimIdx++;
@@ -1742,10 +1742,10 @@ LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp allocOp) {
                           newDynamicSizes);
     // Add the new dynamic sizes in new AllocOp.
     newAlloc =
-        b.create<AllocLikeOp>(allocOp.getLoc(), newMemRefType, newDynamicSizes,
+        AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType, newDynamicSizes,
                               allocOp.getAlignmentAttr());
   } else {
-    newAlloc = b.create<AllocLikeOp>(allocOp.getLoc(), newMemRefType,
+    newAlloc = AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType,
                                      allocOp.getAlignmentAttr());
   }
   // Replace all uses of the old memref.
@@ -1804,10 +1804,10 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0, e = memrefType.getRank(); i < e; i++) {
     if (memrefType.isDynamicDim(i))
       mapOperands[i] =
-          b.create<arith::SubIOp>(loc, oldSizes[0].getType(), oldSizes[idx++],
-                                  b.create<arith::ConstantIndexOp>(loc, 1));
+          arith::SubIOp::create(b, loc, oldSizes[0].getType(), oldSizes[idx++],
+                                  arith::ConstantIndexOp::create(b, loc, 1));
     else
-      mapOperands[i] = b.create<arith::ConstantIndexOp>(loc, oldShape[i] - 1);
+      mapOperands[i] = arith::ConstantIndexOp::create(b, loc, oldShape[i] - 1);
   }
   for (unsigned i = 0, e = oldStrides.size(); i < e; i++)
     mapOperands[memrefType.getRank() + i] = oldStrides[i];
@@ -1817,7 +1817,7 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0; i < newRank; i++) {
     if (!newMemRefType.isDynamicDim(i))
       continue;
-    newSizes.push_back(b.create<AffineApplyOp>(
+    newSizes.push_back(AffineApplyOp::create(b,
         loc,
         AffineMap::get(oldLayoutMap.getNumDims(), oldLayoutMap.getNumSymbols(),
                        oldLayoutMap.getResult(i)),
@@ -1825,11 +1825,11 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   }
   for (unsigned i = 0, e = newSizes.size(); i < e; i++) {
     newSizes[i] =
-        b.create<arith::AddIOp>(loc, newSizes[i].getType(), newSizes[i],
-                                b.create<arith::ConstantIndexOp>(loc, 1));
+        arith::AddIOp::create(b, loc, newSizes[i].getType(), newSizes[i],
+                                arith::ConstantIndexOp::create(b, loc, 1));
   }
   // Create the new reinterpret_cast op.
-  auto newReinterpretCast = b.create<memref::ReinterpretCastOp>(
+  auto newReinterpretCast = memref::ReinterpretCastOp::create(b,
       loc, newMemRefType, reinterpretCastOp.getSource(),
       /*offsets=*/ValueRange(), newSizes,
       /*strides=*/ValueRange(),
diff --git a/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp b/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
index ebcb951cf3518..e7cbee6b06c45 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
@@ -64,7 +64,7 @@ Operation *arith::ArithDialect::materializeConstant(OpBuilder &builder,
                                                     Attribute value, Type type,
                                                     Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   return ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index ae2a00cedf6f5..c9b0c8414b775 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -243,7 +243,7 @@ bool arith::ConstantOp::isBuildableWith(Attribute value, Type type) {
 ConstantOp arith::ConstantOp::materialize(OpBuilder &builder, Attribute value,
                                           Type type, Location loc) {
   if (isBuildableWith(value, type))
-    return builder.create<arith::ConstantOp>(loc, cast<TypedAttr>(value));
+    return arith::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
   return nullptr;
 }
 
@@ -305,7 +305,7 @@ Value mlir::arith::getZeroConstant(OpBuilder &builder, Location loc,
          "type doesn't have a zero representation");
   TypedAttr zeroAttr = builder.getZeroAttr(type);
   assert(zeroAttr && "unsupported type for zero attribute");
-  return builder.create<arith::ConstantOp>(loc, zeroAttr);
+  return arith::ConstantOp::create(builder, loc, zeroAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2335,7 +2335,7 @@ class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
     // comparison.
     rewriter.replaceOpWithNewOp<CmpIOp>(
         op, pred, intVal,
-        rewriter.create<ConstantOp>(
+        ConstantOp::create(rewriter,
             op.getLoc(), intVal.getType(),
             rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
     return success();
@@ -2374,9 +2374,9 @@ struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
         matchPattern(op.getFalseValue(), m_One())) {
       rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
           op, op.getType(),
-          rewriter.create<arith::XOrIOp>(
+          arith::XOrIOp::create(rewriter,
               op.getLoc(), op.getCondition(),
-              rewriter.create<arith::ConstantIntOp>(
+              arith::ConstantIntOp::create(rewriter,
                   op.getLoc(), op.getCondition().getType(), 1)));
       return success();
     }
@@ -2693,7 +2693,7 @@ Value mlir::arith::getIdentityValue(AtomicRMWKind op, Type resultType,
                                     bool useOnlyFiniteValue) {
   auto attr =
       getIdentityValueAttr(op, resultType, builder, loc, useOnlyFiniteValue);
-  return builder.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(builder, loc, attr);
 }
 
 /// Return the value obtained by applying the reduction operation kind
@@ -2702,33 +2702,33 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
                                   Location loc, Value lhs, Value rhs) {
   switch (op) {
   case AtomicRMWKind::addf:
-    return builder.create<arith::AddFOp>(loc, lhs, rhs);
+    return arith::AddFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::addi:
-    return builder.create<arith::AddIOp>(loc, lhs, rhs);
+    return arith::AddIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::mulf:
-    return builder.create<arith::MulFOp>(loc, lhs, rhs);
+    return arith::MulFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::muli:
-    return builder.create<arith::MulIOp>(loc, lhs, rhs);
+    return arith::MulIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maximumf:
-    return builder.create<arith::MaximumFOp>(loc, lhs, rhs);
+    return arith::MaximumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minimumf:
-    return builder.create<arith::MinimumFOp>(loc, lhs, rhs);
+    return arith::MinimumFOp::create(builder, loc, lhs, rhs);
    case AtomicRMWKind::maxnumf:
-    return builder.create<arith::MaxNumFOp>(loc, lhs, rhs);
+    return arith::MaxNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minnumf:
-    return builder.create<arith::MinNumFOp>(loc, lhs, rhs);
+    return arith::MinNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maxs:
-    return builder.create<arith::MaxSIOp>(loc, lhs, rhs);
+    return arith::MaxSIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::mins:
-    return builder.create<arith::MinSIOp>(loc, lhs, rhs);
+    return arith::MinSIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maxu:
-    return builder.create<arith::MaxUIOp>(loc, lhs, rhs);
+    return arith::MaxUIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minu:
-    return builder.create<arith::MinUIOp>(loc, lhs, rhs);
+    return arith::MinUIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::ori:
-    return builder.create<arith::OrIOp>(loc, lhs, rhs);
+    return arith::OrIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::andi:
-    return builder.create<arith::AndIOp>(loc, lhs, rhs);
+    return arith::AndIOp::create(builder, loc, lhs, rhs);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index f2e7732e8ea4a..d21da23903544 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -67,7 +67,7 @@ struct SelectOpInterface
       return state.getMemrefWithUniqueOwnership(builder, value,
                                                 value.getParentBlock());
 
-    Value ownership = builder.create<arith::SelectOp>(
+    Value ownership = arith::SelectOp::create(builder,
         op->getLoc(), selectOp.getCondition(),
         state.getOwnership(selectOp.getTrueValue(), block).getIndicator(),
         state.getOwnership(selectOp.getFalseValue(), block).getIndicator());
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
index afee162053bea..b073a31850678 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -170,10 +170,10 @@ struct SelectOpInterface
         return failure();
       if (trueBuffer.getType() != *targetType)
         trueBuffer =
-            rewriter.create<memref::CastOp>(loc, *targetType, trueBuffer);
+            memref::CastOp::create(rewriter, loc, *targetType, trueBuffer);
       if (falseBuffer.getType() != *targetType)
         falseBuffer =
-            rewriter.create<memref::CastOp>(loc, *targetType, falseBuffer);
+            memref::CastOp::create(rewriter, loc, *targetType, falseBuffer);
     }
 
     replaceOpWithNewBufferizedOp<arith::SelectOp>(
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
index 62022bfb7df1e..674bd7b8ee201 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
@@ -75,7 +75,7 @@ LogicalResult EmulateFloatPattern::matchAndRewrite(
   for (auto [res, oldType, newType] : llvm::zip_equal(
            MutableArrayRef{newResults}, op->getResultTypes(), resultTypes)) {
     if (oldType != newType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, oldType, res);
+      auto truncFOp = arith::TruncFOp::create(rewriter, loc, oldType, res);
       truncFOp.setFastmath(arith::FastMathFlags::contract);
       res = truncFOp.getResult();
     }
@@ -98,7 +98,7 @@ void mlir::arith::populateEmulateUnsupportedFloatsConversions(
   });
   converter.addTargetMaterialization(
       [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        auto extFOp = arith::ExtFOp::create(b, loc, target, input);
         extFOp.setFastmath(arith::FastMathFlags::contract);
         return extFOp;
       });
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
index d5d1559c658ff..44e896a97cb4b 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
@@ -72,7 +72,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
 
   // Scalarize the result in case of 1D vectors.
   if (shape.size() == 1)
-    return rewriter.create<vector::ExtractOp>(loc, input, lastOffset);
+    return vector::ExtractOp::create(rewriter, loc, input, lastOffset);
 
   SmallVector<int64_t> offsets(shape.size(), 0);
   offsets.back() = lastOffset;
@@ -80,7 +80,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
   sizes.back() = 1;
   SmallVector<int64_t> strides(shape.size(), 1);
 
-  return rewriter.create<vector::ExtractStridedSliceOp>(loc, input, offsets,
+  return vector::ExtractStridedSliceOp::create(rewriter, loc, input, offsets,
                                                         sizes, strides);
 }
 
@@ -107,7 +107,7 @@ static Value dropTrailingX1Dim(ConversionPatternRewriter &rewriter,
   assert(shape.back() == 1 && "Expected the last vector dim to be x1");
 
   auto newVecTy = VectorType::get(shape.drop_back(), vecTy.getElementType());
-  return rewriter.create<vector::ShapeCastOp>(loc, newVecTy, input);
+  return vector::ShapeCastOp::create(rewriter, loc, newVecTy, input);
 }
 
 /// Performs a vector shape cast to append an x1 dimension. If the
@@ -122,7 +122,7 @@ static Value appendX1Dim(ConversionPatternRewriter &rewriter, Location loc,
   auto newShape = llvm::to_vector(vecTy.getShape());
   newShape.push_back(1);
   auto newTy = VectorType::get(newShape, vecTy.getElementType());
-  return rewriter.create<vector::ShapeCastOp>(loc, newTy, input);
+  return vector::ShapeCastOp::create(rewriter, loc, newTy, input);
 }
 
 /// Inserts the `source` vector slice into the `dest` vector at offset
@@ -136,12 +136,12 @@ static Value insertLastDimSlice(ConversionPatternRewriter &rewriter,
 
   // Handle scalar source.
   if (isa<IntegerType>(source.getType()))
-    return rewriter.create<vector::InsertOp>(loc, source, dest, lastOffset);
+    return vector::InsertOp::create(rewriter, loc, source, dest, lastOffset);
 
   SmallVector<int64_t> offsets(shape.size(), 0);
   offsets.back() = lastOffset;
   SmallVector<int64_t> strides(shape.size(), 1);
-  return rewriter.create<vector::InsertStridedSliceOp>(loc, source, dest,
+  return vector::InsertStridedSliceOp::create(rewriter, loc, source, dest,
                                                        offsets, strides);
 }
 
@@ -254,12 +254,12 @@ struct ConvertAddI final : OpConversionPattern<arith::AddIOp> {
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
     auto lowSum =
-        rewriter.create<arith::AddUIExtendedOp>(loc, lhsElem0, rhsElem0);
+        arith::AddUIExtendedOp::create(rewriter, loc, lhsElem0, rhsElem0);
     Value overflowVal =
-        rewriter.create<arith::ExtUIOp>(loc, newElemTy, lowSum.getOverflow());
+        arith::ExtUIOp::create(rewriter, loc, newElemTy, lowSum.getOverflow());
 
-    Value high0 = rewriter.create<arith::AddIOp>(loc, overflowVal, lhsElem1);
-    Value high = rewriter.create<arith::AddIOp>(loc, high0, rhsElem1);
+    Value high0 = arith::AddIOp::create(rewriter, loc, overflowVal, lhsElem1);
+    Value high = arith::AddIOp::create(rewriter, loc, high0, rhsElem1);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {lowSum.getSum(), high});
@@ -293,8 +293,8 @@ struct ConvertBitwiseBinary final : OpConversionPattern<BinaryOp> {
     auto [rhsElem0, rhsElem1] =
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
-    Value resElem0 = rewriter.create<BinaryOp>(loc, lhsElem0, rhsElem0);
-    Value resElem1 = rewriter.create<BinaryOp>(loc, lhsElem1, rhsElem1);
+    Value resElem0 = BinaryOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem1 = BinaryOp::create(rewriter, loc, lhsElem1, rhsElem1);
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
     rewriter.replaceOp(op, resultVec);
@@ -346,26 +346,26 @@ struct ConvertCmpI final : OpConversionPattern<arith::CmpIOp> {
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
     Value lowCmp =
-        rewriter.create<arith::CmpIOp>(loc, lowPred, lhsElem0, rhsElem0);
+        arith::CmpIOp::create(rewriter, loc, lowPred, lhsElem0, rhsElem0);
     Value highCmp =
-        rewriter.create<arith::CmpIOp>(loc, highPred, lhsElem1, rhsElem1);
+        arith::CmpIOp::create(rewriter, loc, highPred, lhsElem1, rhsElem1);
 
     Value cmpResult{};
     switch (highPred) {
     case arith::CmpIPredicate::eq: {
-      cmpResult = rewriter.create<arith::AndIOp>(loc, lowCmp, highCmp);
+      cmpResult = arith::AndIOp::create(rewriter, loc, lowCmp, highCmp);
       break;
     }
     case arith::CmpIPredicate::ne: {
-      cmpResult = rewriter.create<arith::OrIOp>(loc, lowCmp, highCmp);
+      cmpResult = arith::OrIOp::create(rewriter, loc, lowCmp, highCmp);
       break;
     }
     default: {
       // Handle inequality checks.
-      Value highEq = rewriter.create<arith::CmpIOp>(
+      Value highEq = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, lhsElem1, rhsElem1);
       cmpResult =
-          rewriter.create<arith::SelectOp>(loc, highEq, lowCmp, highCmp);
+          arith::SelectOp::create(rewriter, loc, highEq, lowCmp, highCmp);
       break;
     }
     }
@@ -401,14 +401,14 @@ struct ConvertMulI final : OpConversionPattern<arith::MulIOp> {
     // Multiplying two i2N integers produces (at most) an i4N result, but
     // because the calculation of top i2N is not necessary, we omit it.
     auto mulLowLow =
-        rewriter.create<arith::MulUIExtendedOp>(loc, lhsElem0, rhsElem0);
-    Value mulLowHi = rewriter.create<arith::MulIOp>(loc, lhsElem0, rhsElem1);
-    Value mulHiLow = rewriter.create<arith::MulIOp>(loc, lhsElem1, rhsElem0);
+        arith::MulUIExtendedOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value mulLowHi = arith::MulIOp::create(rewriter, loc, lhsElem0, rhsElem1);
+    Value mulHiLow = arith::MulIOp::create(rewriter, loc, lhsElem1, rhsElem0);
 
     Value resLow = mulLowLow.getLow();
     Value resHi =
-        rewriter.create<arith::AddIOp>(loc, mulLowLow.getHigh(), mulLowHi);
-    resHi = rewriter.create<arith::AddIOp>(loc, resHi, mulHiLow);
+        arith::AddIOp::create(rewriter, loc, mulLowLow.getHigh(), mulLowHi);
+    resHi = arith::AddIOp::create(rewriter, loc, resHi, mulHiLow);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resLow, resHi});
@@ -443,10 +443,10 @@ struct ConvertExtSI final : OpConversionPattern<arith::ExtSIOp> {
         loc, newResultComponentTy, newOperand);
     Value operandZeroCst =
         createScalarOrSplatConstant(rewriter, loc, newResultComponentTy, 0);
-    Value signBit = rewriter.create<arith::CmpIOp>(
+    Value signBit = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, extended, operandZeroCst);
     Value signValue =
-        rewriter.create<arith::ExtSIOp>(loc, newResultComponentTy, signBit);
+        arith::ExtSIOp::create(rewriter, loc, newResultComponentTy, signBit);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {extended, signValue});
@@ -508,7 +508,7 @@ struct ConvertMaxMin final : OpConversionPattern<SourceOp> {
     // Rewrite Max*I/Min*I as compare and select over original operands. Let
     // the CmpI and Select emulation patterns handle the final legalization.
     Value cmp =
-        rewriter.create<arith::CmpIOp>(loc, CmpPred, op.getLhs(), op.getRhs());
+        arith::CmpIOp::create(rewriter, loc, CmpPred, op.getLhs(), op.getRhs());
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, op.getLhs(),
                                                  op.getRhs());
     return success();
@@ -587,7 +587,7 @@ struct ConvertIndexCastIndexToInt final : OpConversionPattern<CastOp> {
     // Sign or zero-extend the result. Let the matching conversion pattern
     // legalize the extension op.
     Value underlyingVal =
-        rewriter.create<CastOp>(loc, narrowTy, adaptor.getIn());
+        CastOp::create(rewriter, loc, narrowTy, adaptor.getIn());
     rewriter.replaceOpWithNewOp<ExtensionOp>(op, resultType, underlyingVal);
     return success();
   }
@@ -616,9 +616,9 @@ struct ConvertSelect final : OpConversionPattern<arith::SelectOp> {
     Value cond = appendX1Dim(rewriter, loc, adaptor.getCondition());
 
     Value resElem0 =
-        rewriter.create<arith::SelectOp>(loc, cond, trueElem0, falseElem0);
+        arith::SelectOp::create(rewriter, loc, cond, trueElem0, falseElem0);
     Value resElem1 =
-        rewriter.create<arith::SelectOp>(loc, cond, trueElem1, falseElem1);
+        arith::SelectOp::create(rewriter, loc, cond, trueElem1, falseElem1);
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
     rewriter.replaceOp(op, resultVec);
@@ -680,33 +680,33 @@ struct ConvertShLI final : OpConversionPattern<arith::ShLIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = rewriter.create<arith::CmpIOp>(
+    Value illegalElemShift = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem0, rhsElem0);
-    Value resElem0 = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShLIOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem0 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                       zeroCst, shiftedElem0);
 
-    Value cappedShiftAmount = rewriter.create<arith::SelectOp>(
+    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value rightShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, elemBitWidth, cappedShiftAmount);
+        arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedRight =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem0, rightShiftAmount);
+        arith::ShRUIOp::create(rewriter, loc, lhsElem0, rightShiftAmount);
     Value overshotShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, rhsElem0, elemBitWidth);
+        arith::SubIOp::create(rewriter, loc, rhsElem0, elemBitWidth);
     Value shiftedLeft =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem0, overshotShiftAmount);
+        arith::ShLIOp::create(rewriter, loc, lhsElem0, overshotShiftAmount);
 
     Value shiftedElem1 =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem1, rhsElem0);
-    Value resElem1High = rewriter.create<arith::SelectOp>(
+        arith::ShLIOp::create(rewriter, loc, lhsElem1, rhsElem0);
+    Value resElem1High = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, zeroCst, shiftedElem1);
-    Value resElem1Low = rewriter.create<arith::SelectOp>(
+    Value resElem1Low = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, shiftedLeft, shiftedRight);
     Value resElem1 =
-        rewriter.create<arith::OrIOp>(loc, resElem1Low, resElem1High);
+        arith::OrIOp::create(rewriter, loc, resElem1Low, resElem1High);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
@@ -769,33 +769,33 @@ struct ConvertShRUI final : OpConversionPattern<arith::ShRUIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = rewriter.create<arith::CmpIOp>(
+    Value illegalElemShift = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem0, rhsElem0);
-    Value resElem0Low = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShRUIOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem0Low = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                          zeroCst, shiftedElem0);
     Value shiftedElem1 =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem1, rhsElem0);
-    Value resElem1 = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShRUIOp::create(rewriter, loc, lhsElem1, rhsElem0);
+    Value resElem1 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                       zeroCst, shiftedElem1);
 
-    Value cappedShiftAmount = rewriter.create<arith::SelectOp>(
+    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value leftShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, elemBitWidth, cappedShiftAmount);
+        arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedLeft =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem1, leftShiftAmount);
+        arith::ShLIOp::create(rewriter, loc, lhsElem1, leftShiftAmount);
     Value overshotShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, rhsElem0, elemBitWidth);
+        arith::SubIOp::create(rewriter, loc, rhsElem0, elemBitWidth);
     Value shiftedRight =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem1, overshotShiftAmount);
+        arith::ShRUIOp::create(rewriter, loc, lhsElem1, overshotShiftAmount);
 
-    Value resElem0High = rewriter.create<arith::SelectOp>(
+    Value resElem0High = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, shiftedRight, shiftedLeft);
     Value resElem0 =
-        rewriter.create<arith::OrIOp>(loc, resElem0Low, resElem0High);
+        arith::OrIOp::create(rewriter, loc, resElem0Low, resElem0High);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
@@ -832,32 +832,32 @@ struct ConvertShRSI final : OpConversionPattern<arith::ShRSIOp> {
     // Perform as many ops over the narrow integer type as possible and let the
     // other emulation patterns convert the rest.
     Value elemZero = createScalarOrSplatConstant(rewriter, loc, narrowTy, 0);
-    Value signBit = rewriter.create<arith::CmpIOp>(
+    Value signBit = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, lhsElem1, elemZero);
     signBit = dropTrailingX1Dim(rewriter, loc, signBit);
 
     // Create a bit pattern of either all ones or all zeros. Then shift it left
     // to calculate the sign extension bits created by shifting the original
     // sign bit right.
-    Value allSign = rewriter.create<arith::ExtSIOp>(loc, oldTy, signBit);
+    Value allSign = arith::ExtSIOp::create(rewriter, loc, oldTy, signBit);
     Value maxShift =
         createScalarOrSplatConstant(rewriter, loc, narrowTy, origBitwidth);
     Value numNonSignExtBits =
-        rewriter.create<arith::SubIOp>(loc, maxShift, rhsElem0);
+        arith::SubIOp::create(rewriter, loc, maxShift, rhsElem0);
     numNonSignExtBits = dropTrailingX1Dim(rewriter, loc, numNonSignExtBits);
     numNonSignExtBits =
-        rewriter.create<arith::ExtUIOp>(loc, oldTy, numNonSignExtBits);
+        arith::ExtUIOp::create(rewriter, loc, oldTy, numNonSignExtBits);
     Value signBits =
-        rewriter.create<arith::ShLIOp>(loc, allSign, numNonSignExtBits);
+        arith::ShLIOp::create(rewriter, loc, allSign, numNonSignExtBits);
 
     // Use original arguments to create the right shift.
     Value shrui =
-        rewriter.create<arith::ShRUIOp>(loc, op.getLhs(), op.getRhs());
-    Value shrsi = rewriter.create<arith::OrIOp>(loc, shrui, signBits);
+        arith::ShRUIOp::create(rewriter, loc, op.getLhs(), op.getRhs());
+    Value shrsi = arith::OrIOp::create(rewriter, loc, shrui, signBits);
 
     // Handle shifting by zero. This is necessary when the `signBits` shift is
     // invalid.
-    Value isNoop = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value isNoop = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                   rhsElem0, elemZero);
     isNoop = dropTrailingX1Dim(rewriter, loc, isNoop);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNoop, op.getLhs(),
@@ -892,14 +892,14 @@ struct ConvertSubI final : OpConversionPattern<arith::SubIOp> {
 
     // Emulates LHS - RHS by [LHS0 - RHS0, LHS1 - RHS1 - CARRY] where
     // CARRY is 1 or 0.
-    Value low = rewriter.create<arith::SubIOp>(loc, lhsElem0, rhsElem0);
+    Value low = arith::SubIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     // We have a carry if lhsElem0 < rhsElem0.
-    Value carry0 = rewriter.create<arith::CmpIOp>(
+    Value carry0 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, lhsElem0, rhsElem0);
-    Value carryVal = rewriter.create<arith::ExtUIOp>(loc, newElemTy, carry0);
+    Value carryVal = arith::ExtUIOp::create(rewriter, loc, newElemTy, carry0);
 
-    Value high0 = rewriter.create<arith::SubIOp>(loc, lhsElem1, carryVal);
-    Value high = rewriter.create<arith::SubIOp>(loc, high0, rhsElem1);
+    Value high0 = arith::SubIOp::create(rewriter, loc, lhsElem1, carryVal);
+    Value high = arith::SubIOp::create(rewriter, loc, high0, rhsElem1);
 
     Value resultVec = constructResultVector(rewriter, loc, newTy, {low, high});
     rewriter.replaceOp(op, resultVec);
@@ -933,13 +933,13 @@ struct ConvertSIToFP final : OpConversionPattern<arith::SIToFPOp> {
     // result or not based on that sign bit. We implement negation by
     // subtracting from zero. Note that this relies on the the other conversion
     // patterns to legalize created ops and narrow the bit widths.
-    Value isNeg = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+    Value isNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                  in, zeroCst);
-    Value neg = rewriter.create<arith::SubIOp>(loc, zeroCst, in);
-    Value abs = rewriter.create<arith::SelectOp>(loc, isNeg, neg, in);
+    Value neg = arith::SubIOp::create(rewriter, loc, zeroCst, in);
+    Value abs = arith::SelectOp::create(rewriter, loc, isNeg, neg, in);
 
-    Value absResult = rewriter.create<arith::UIToFPOp>(loc, op.getType(), abs);
-    Value negResult = rewriter.create<arith::NegFOp>(loc, absResult);
+    Value absResult = arith::UIToFPOp::create(rewriter, loc, op.getType(), abs);
+    Value negResult = arith::NegFOp::create(rewriter, loc, absResult);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNeg, negResult,
                                                  absResult);
     return success();
@@ -985,13 +985,13 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     //
     // Note 2: We do not strictly need the `hi == 0`, case, but it makes
     // constant folding easier.
-    Value hiEqZero = rewriter.create<arith::CmpIOp>(
+    Value hiEqZero = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, hiInt, zeroCst);
 
     Type resultTy = op.getType();
     Type resultElemTy = getElementTypeOrSelf(resultTy);
-    Value lowFp = rewriter.create<arith::UIToFPOp>(loc, resultTy, lowInt);
-    Value hiFp = rewriter.create<arith::UIToFPOp>(loc, resultTy, hiInt);
+    Value lowFp = arith::UIToFPOp::create(rewriter, loc, resultTy, lowInt);
+    Value hiFp = arith::UIToFPOp::create(rewriter, loc, resultTy, hiInt);
 
     int64_t pow2Int = int64_t(1) << newBitWidth;
     TypedAttr pow2Attr =
@@ -999,10 +999,10 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     if (auto vecTy = dyn_cast<VectorType>(resultTy))
       pow2Attr = SplatElementsAttr::get(vecTy, pow2Attr);
 
-    Value pow2Val = rewriter.create<arith::ConstantOp>(loc, resultTy, pow2Attr);
+    Value pow2Val = arith::ConstantOp::create(rewriter, loc, resultTy, pow2Attr);
 
-    Value hiVal = rewriter.create<arith::MulFOp>(loc, hiFp, pow2Val);
-    Value result = rewriter.create<arith::AddFOp>(loc, lowFp, hiVal);
+    Value hiVal = arith::MulFOp::create(rewriter, loc, hiFp, pow2Val);
+    Value result = arith::AddFOp::create(rewriter, loc, lowFp, hiVal);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, hiEqZero, lowFp, result);
     return success();
@@ -1037,22 +1037,22 @@ struct ConvertFPToSI final : OpConversionPattern<arith::FPToSIOp> {
     // result is UB.
 
     TypedAttr zeroAttr = rewriter.getZeroAttr(fpTy);
-    Value zeroCst = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
+    Value zeroCst = arith::ConstantOp::create(rewriter, loc, zeroAttr);
     Value zeroCstInt = createScalarOrSplatConstant(rewriter, loc, intTy, 0);
 
     // Get the absolute value. One could have used math.absf here, but that
     // introduces an extra dependency.
-    Value isNeg = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OLT,
+    Value isNeg = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OLT,
                                                  inFp, zeroCst);
-    Value negInFp = rewriter.create<arith::NegFOp>(loc, inFp);
+    Value negInFp = arith::NegFOp::create(rewriter, loc, inFp);
 
-    Value absVal = rewriter.create<arith::SelectOp>(loc, isNeg, negInFp, inFp);
+    Value absVal = arith::SelectOp::create(rewriter, loc, isNeg, negInFp, inFp);
 
     // Defer the absolute value to fptoui.
-    Value res = rewriter.create<arith::FPToUIOp>(loc, intTy, absVal);
+    Value res = arith::FPToUIOp::create(rewriter, loc, intTy, absVal);
 
     // Negate the value if < 0 .
-    Value neg = rewriter.create<arith::SubIOp>(loc, zeroCstInt, res);
+    Value neg = arith::SubIOp::create(rewriter, loc, zeroCstInt, res);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNeg, neg, res);
     return success();
@@ -1109,17 +1109,17 @@ struct ConvertFPToUI final : OpConversionPattern<arith::FPToUIOp> {
     if (auto vecType = dyn_cast<VectorType>(fpTy))
       powBitwidthAttr = SplatElementsAttr::get(vecType, powBitwidthAttr);
     Value powBitwidthFloatCst =
-        rewriter.create<arith::ConstantOp>(loc, powBitwidthAttr);
+        arith::ConstantOp::create(rewriter, loc, powBitwidthAttr);
 
     Value fpDivPowBitwidth =
-        rewriter.create<arith::DivFOp>(loc, inFp, powBitwidthFloatCst);
+        arith::DivFOp::create(rewriter, loc, inFp, powBitwidthFloatCst);
     Value resHigh =
-        rewriter.create<arith::FPToUIOp>(loc, newHalfType, fpDivPowBitwidth);
+        arith::FPToUIOp::create(rewriter, loc, newHalfType, fpDivPowBitwidth);
     // Calculate fp - resHigh * 2^N by getting the remainder of the division
     Value remainder =
-        rewriter.create<arith::RemFOp>(loc, inFp, powBitwidthFloatCst);
+        arith::RemFOp::create(rewriter, loc, inFp, powBitwidthFloatCst);
     Value resLow =
-        rewriter.create<arith::FPToUIOp>(loc, newHalfType, remainder);
+        arith::FPToUIOp::create(rewriter, loc, newHalfType, remainder);
 
     Value high = appendX1Dim(rewriter, loc, resHigh);
     Value low = appendX1Dim(rewriter, loc, resLow);
diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
index dfa01844737c6..7dd0541f3c4b5 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
@@ -31,10 +31,10 @@ static Value createConst(Location loc, Type type, int value,
                          PatternRewriter &rewriter) {
   auto attr = rewriter.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return rewriter.create<arith::ConstantOp>(
+    return arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
-  return rewriter.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(rewriter, loc, attr);
 }
 
 /// Create a float constant.
@@ -42,11 +42,11 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                               PatternRewriter &rewriter) {
   auto attr = rewriter.getFloatAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return rewriter.create<arith::ConstantOp>(
+    return arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return rewriter.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(rewriter, loc, attr);
 }
 
 /// Creates shapedType using shape from cloneFrom and base type from cloneTo
@@ -70,11 +70,11 @@ struct CeilDivUIOpConverter : public OpRewritePattern<arith::CeilDivUIOp> {
     Value b = op.getRhs();
     Value zero = createConst(loc, a.getType(), 0, rewriter);
     Value compare =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, a, zero);
     Value one = createConst(loc, a.getType(), 1, rewriter);
-    Value minusOne = rewriter.create<arith::SubIOp>(loc, a, one);
-    Value quotient = rewriter.create<arith::DivUIOp>(loc, minusOne, b);
-    Value plusOne = rewriter.create<arith::AddIOp>(loc, quotient, one);
+    Value minusOne = arith::SubIOp::create(rewriter, loc, a, one);
+    Value quotient = arith::DivUIOp::create(rewriter, loc, minusOne, b);
+    Value plusOne = arith::AddIOp::create(rewriter, loc, quotient, one);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, compare, zero, plusOne);
     return success();
   }
@@ -99,22 +99,22 @@ struct CeilDivSIOpConverter : public OpRewritePattern<arith::CeilDivSIOp> {
     Value zero = createConst(loc, type, 0, rewriter);
     Value one = createConst(loc, type, 1, rewriter);
 
-    Value quotient = rewriter.create<arith::DivSIOp>(loc, a, b);
-    Value product = rewriter.create<arith::MulIOp>(loc, quotient, b);
-    Value notEqualDivisor = rewriter.create<arith::CmpIOp>(
+    Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
+    Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
+    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, a, product);
 
     Value aNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
     Value bNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, b, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
 
-    Value signEqual = rewriter.create<arith::CmpIOp>(
+    Value signEqual = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, aNeg, bNeg);
     Value cond =
-        rewriter.create<arith::AndIOp>(loc, notEqualDivisor, signEqual);
+        arith::AndIOp::create(rewriter, loc, notEqualDivisor, signEqual);
 
-    Value quotientPlusOne = rewriter.create<arith::AddIOp>(loc, quotient, one);
+    Value quotientPlusOne = arith::AddIOp::create(rewriter, loc, quotient, one);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cond, quotientPlusOne,
                                                  quotient);
@@ -138,25 +138,25 @@ struct FloorDivSIOpConverter : public OpRewritePattern<arith::FloorDivSIOp> {
     Value a = op.getLhs();
     Value b = op.getRhs();
 
-    Value quotient = rewriter.create<arith::DivSIOp>(loc, a, b);
-    Value product = rewriter.create<arith::MulIOp>(loc, quotient, b);
-    Value notEqualDivisor = rewriter.create<arith::CmpIOp>(
+    Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
+    Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
+    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, a, product);
     Value zero = createConst(loc, type, 0, rewriter);
 
     Value aNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
     Value bNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, b, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
 
-    Value signOpposite = rewriter.create<arith::CmpIOp>(
+    Value signOpposite = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, aNeg, bNeg);
     Value cond =
-        rewriter.create<arith::AndIOp>(loc, notEqualDivisor, signOpposite);
+        arith::AndIOp::create(rewriter, loc, notEqualDivisor, signOpposite);
 
     Value minusOne = createConst(loc, type, -1, rewriter);
     Value quotientMinusOne =
-        rewriter.create<arith::AddIOp>(loc, quotient, minusOne);
+        arith::AddIOp::create(rewriter, loc, quotient, minusOne);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cond, quotientMinusOne,
                                                  quotient);
@@ -174,7 +174,7 @@ struct MaxMinIOpConverter : public OpRewritePattern<OpTy> {
     Value lhs = op.getLhs();
     Value rhs = op.getRhs();
 
-    Value cmp = rewriter.create<arith::CmpIOp>(op.getLoc(), pred, lhs, rhs);
+    Value cmp = arith::CmpIOp::create(rewriter, op.getLoc(), pred, lhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, lhs, rhs);
     return success();
   }
@@ -195,11 +195,11 @@ struct MaximumMinimumFOpConverter : public OpRewritePattern<OpTy> {
     static_assert(pred == arith::CmpFPredicate::UGT ||
                       pred == arith::CmpFPredicate::ULT,
                   "pred must be either UGT or ULT");
-    Value cmp = rewriter.create<arith::CmpFOp>(loc, pred, lhs, rhs);
-    Value select = rewriter.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+    Value cmp = arith::CmpFOp::create(rewriter, loc, pred, lhs, rhs);
+    Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'.
-    Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
+    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
                                                  rhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
@@ -221,11 +221,11 @@ struct MaxNumMinNumFOpConverter : public OpRewritePattern<OpTy> {
     static_assert(pred == arith::CmpFPredicate::UGT ||
                       pred == arith::CmpFPredicate::ULT,
                   "pred must be either UGT or ULT");
-    Value cmp = rewriter.create<arith::CmpFOp>(loc, pred, lhs, rhs);
-    Value select = rewriter.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+    Value cmp = arith::CmpFOp::create(rewriter, loc, pred, lhs, rhs);
+    Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where lhs is NaN: 'isNaN(lhs) ? rhs : select'.
-    Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
+    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
                                                  lhs, lhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
@@ -250,12 +250,12 @@ struct BFloat16ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type i16Ty = cloneToShapedType(operandTy, b.getI16Type());
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
 
-    Value bitcast = b.create<arith::BitcastOp>(i16Ty, operand);
-    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
+    Value bitcast = arith::BitcastOp::create(b, i16Ty, operand);
+    Value exti = arith::ExtUIOp::create(b, i32Ty, bitcast);
 
     Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
-    Value shl = b.create<arith::ShLIOp>(exti, c16);
-    Value result = b.create<arith::BitcastOp>(resultTy, shl);
+    Value shl = arith::ShLIOp::create(b, exti, c16);
+    Value result = arith::BitcastOp::create(b, resultTy, shl);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -299,7 +299,7 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     // exponent bits, that simple truncation is the desired outcome for
     // infinities.
     Value isNan =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNE, operand, operand);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNE, operand, operand);
     // Constant used to make the rounding bias.
     Value c7FFF = createConst(op.getLoc(), i32Ty, 0x7fff, rewriter);
     // Constant used to generate a quiet NaN.
@@ -308,30 +308,30 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
     Value c1 = createConst(op.getLoc(), i32Ty, 1, rewriter);
     // Reinterpret the input f32 value as bits.
-    Value bitcast = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value bitcast = arith::BitcastOp::create(b, i32Ty, operand);
     // Read bit 16 as a value in {0,1}.
     Value bit16 =
-        b.create<arith::AndIOp>(b.create<arith::ShRUIOp>(bitcast, c16), c1);
+        arith::AndIOp::create(b, arith::ShRUIOp::create(b, bitcast, c16), c1);
     // Determine the rounding bias to add as either 0x7fff or 0x8000 depending
     // on bit 16, implementing the tie-breaking "to nearest even".
-    Value roundingBias = b.create<arith::AddIOp>(bit16, c7FFF);
+    Value roundingBias = arith::AddIOp::create(b, bit16, c7FFF);
     // Add the rounding bias. Generally we want this to be added to the
     // mantissa, but nothing prevents this to from carrying into the exponent
     // bits, which would feel like a bug, but this is the magic trick here:
     // when that happens, the mantissa gets reset to zero and the exponent
     // gets incremented by the carry... which is actually exactly what we
     // want.
-    Value biased = b.create<arith::AddIOp>(bitcast, roundingBias);
+    Value biased = arith::AddIOp::create(b, bitcast, roundingBias);
     // Now that the rounding-bias has been added, truncating the low bits
     // yields the correctly rounded result.
-    Value biasedAndShifted = b.create<arith::ShRUIOp>(biased, c16);
+    Value biasedAndShifted = arith::ShRUIOp::create(b, biased, c16);
     Value normalCaseResultI16 =
-        b.create<arith::TruncIOp>(i16Ty, biasedAndShifted);
+        arith::TruncIOp::create(b, i16Ty, biasedAndShifted);
     // Select either the above-computed result, or a quiet NaN constant
     // if the input was NaN.
     Value select =
-        b.create<arith::SelectOp>(isNan, c7FC0I16, normalCaseResultI16);
-    Value result = b.create<arith::BitcastOp>(resultTy, select);
+        arith::SelectOp::create(b, isNan, c7FC0I16, normalCaseResultI16);
+    Value result = arith::BitcastOp::create(b, resultTy, select);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -384,7 +384,7 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
     Type i4Ty = cloneToShapedType(operandTy, b.getI4Type());
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
-    Value i4Bits = b.create<arith::BitcastOp>(i4Ty, operand);
+    Value i4Bits = arith::BitcastOp::create(b, i4Ty, operand);
 
     Value c0x0 = createConst(loc, i4Ty, 0x0, rewriter);
     Value c0x1 = createConst(loc, i4Ty, 0x1, rewriter);
@@ -393,38 +393,38 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
 
     // Set last Exponent bit and Mantissa.
     Value c0x00000014 = createConst(loc, i32Ty, 0x14, rewriter);
-    Value bits1To24 = b.create<arith::ShLIOp>(i4Bits, c0x2);
+    Value bits1To24 = arith::ShLIOp::create(b, i4Bits, c0x2);
     Value isHalf =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x1);
-    bits1To24 = b.create<arith::SelectOp>(isHalf, c0x0, bits1To24);
-    bits1To24 = b.create<arith::ExtUIOp>(i32Ty, bits1To24);
-    bits1To24 = b.create<arith::ShLIOp>(bits1To24, c0x00000014);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, i4Bits, c0x1);
+    bits1To24 = arith::SelectOp::create(b, isHalf, c0x0, bits1To24);
+    bits1To24 = arith::ExtUIOp::create(b, i32Ty, bits1To24);
+    bits1To24 = arith::ShLIOp::create(b, bits1To24, c0x00000014);
 
     // Set first 7 bits of Exponent.
     Value zeroExpBits = createConst(loc, i32Ty, 0x00000000, rewriter);
     Value highExpBits = createConst(loc, i32Ty, 0x40000000, rewriter);
     Value lowExpBits = createConst(loc, i32Ty, 0x3f000000, rewriter);
     Value useLargerExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x4);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x4);
     Value bits25To31 =
-        b.create<arith::SelectOp>(useLargerExp, highExpBits, lowExpBits);
+        arith::SelectOp::create(b, useLargerExp, highExpBits, lowExpBits);
     Value zeroExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x0);
-    bits25To31 = b.create<arith::SelectOp>(zeroExp, zeroExpBits, bits25To31);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, i4Bits, c0x0);
+    bits25To31 = arith::SelectOp::create(b, zeroExp, zeroExpBits, bits25To31);
 
     // Set sign.
     Value c0x80000000 = createConst(loc, i32Ty, 0x80000000, rewriter);
     Value c0x8 = createConst(loc, i4Ty, 0x8, rewriter);
     Value negative =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x8);
-    Value bit32 = b.create<arith::SelectOp>(negative, c0x80000000, zeroExpBits);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x8);
+    Value bit32 = arith::SelectOp::create(b, negative, c0x80000000, zeroExpBits);
 
     // Add segments together.
-    Value bits1To31 = b.create<arith::AddIOp>(bits1To24, bits25To31);
-    Value bits1To32 = b.create<arith::AddIOp>(bits1To31, bit32);
-    Value result = b.create<arith::BitcastOp>(f32Ty, bits1To32);
+    Value bits1To31 = arith::AddIOp::create(b, bits1To24, bits25To31);
+    Value bits1To32 = arith::AddIOp::create(b, bits1To31, bit32);
+    Value result = arith::BitcastOp::create(b, f32Ty, bits1To32);
     if (!isa<Float32Type>(resultETy))
-      result = b.create<arith::TruncFOp>(resultTy, result);
+      result = arith::TruncFOp::create(b, resultTy, result);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -450,25 +450,25 @@ struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
-    Value bitcast = b.create<arith::BitcastOp>(i8Ty, operand);
+    Value bitcast = arith::BitcastOp::create(b, i8Ty, operand);
     // create constants for NaNs
     Value cF8NaN = createConst(op.getLoc(), i8Ty, 0xff, rewriter);
     Value cF32NaN = createConst(op.getLoc(), i32Ty, 0xffffffff, rewriter);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
 
-    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
-    Value f32Bits = b.create<arith::ShLIOp>(exti, cF32MantissaWidth);
+    Value exti = arith::ExtUIOp::create(b, i32Ty, bitcast);
+    Value f32Bits = arith::ShLIOp::create(b, exti, cF32MantissaWidth);
 
     Value isNan =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bitcast, cF8NaN);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, bitcast, cF8NaN);
     // select for NaNs
-    f32Bits = b.create<arith::SelectOp>(isNan, cF32NaN, f32Bits);
-    Value result = b.create<arith::BitcastOp>(f32Ty, f32Bits);
+    f32Bits = arith::SelectOp::create(b, isNan, cF32NaN, f32Bits);
+    Value result = arith::BitcastOp::create(b, f32Ty, f32Bits);
     if (resultETy.getIntOrFloatBitWidth() < 32) {
-      result = b.create<arith::TruncFOp>(resultTy, result, nullptr,
+      result = arith::TruncFOp::create(b, resultTy, result, nullptr,
                                          op.getFastmathAttr());
     } else if (resultETy.getIntOrFloatBitWidth() > 32) {
-      result = b.create<arith::ExtFOp>(resultTy, result, op.getFastmathAttr());
+      result = arith::ExtFOp::create(b, resultTy, result, op.getFastmathAttr());
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -521,7 +521,7 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (!isa<Float32Type>(operandETy))
-      operand = b.create<arith::ExtFOp>(f32Ty, operand);
+      operand = arith::ExtFOp::create(b, f32Ty, operand);
     if (!isa<Float4E2M1FNType>(resultETy))
       return rewriter.notifyMatchFailure(op, "not a trunc of F4E2M1FN");
 
@@ -535,65 +535,65 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     // Step 0: Clamp to bounds.
     Value cHigherBound = createFloatConst(loc, f32Ty, APFloat(6.0f), rewriter);
     Value cLowerBound = createFloatConst(loc, f32Ty, APFloat(-6.0f), rewriter);
-    Value operandClamped = b.create<arith::MinNumFOp>(cHigherBound, operand);
-    operandClamped = b.create<arith::MaxNumFOp>(cLowerBound, operandClamped);
-    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operandClamped);
+    Value operandClamped = arith::MinNumFOp::create(b, cHigherBound, operand);
+    operandClamped = arith::MaxNumFOp::create(b, cLowerBound, operandClamped);
+    Value f32Bits = arith::BitcastOp::create(b, i32Ty, operandClamped);
 
     // Step 1: Set sign bit.
     Value cF32ExpManWidth = createConst(loc, i32Ty, 31, rewriter); // 23
-    Value f32Sign = b.create<arith::ShRUIOp>(f32Bits, cF32ExpManWidth);
-    Value f4Sign = b.create<arith::TruncIOp>(i4Ty, f32Sign);
-    Value f4Bits = b.create<arith::ShLIOp>(f4Sign, c0x3);
+    Value f32Sign = arith::ShRUIOp::create(b, f32Bits, cF32ExpManWidth);
+    Value f4Sign = arith::TruncIOp::create(b, i4Ty, f32Sign);
+    Value f4Bits = arith::ShLIOp::create(b, f4Sign, c0x3);
 
     // Step 2: Convert exponent by adjusting bias.
     Value biasAdjustment = createConst(loc, i32Ty, 0x7e, rewriter);
     Value cF4MantissaWidth = c0x1;                                   // 1
     Value cF32MantissaWidth = createConst(loc, i32Ty, 23, rewriter); // 23
-    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
+    Value f32SignExp = arith::ShRUIOp::create(b, f32Bits, cF32MantissaWidth);
     Value biasAdjustedSignExp =
-        b.create<arith::SubIOp>(f32SignExp, biasAdjustment);
-    Value f4Exp = b.create<arith::TruncIOp>(i4Ty, biasAdjustedSignExp);
-    f4Exp = b.create<arith::ShLIOp>(f4Exp, cF4MantissaWidth);
-    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Exp);
+        arith::SubIOp::create(b, f32SignExp, biasAdjustment);
+    Value f4Exp = arith::TruncIOp::create(b, i4Ty, biasAdjustedSignExp);
+    f4Exp = arith::ShLIOp::create(b, f4Exp, cF4MantissaWidth);
+    f4Bits = arith::AddIOp::create(b, f4Bits, f4Exp);
 
     // Step 3: Set mantissa to first bit.
     Value cF32FirstBitMask = createConst(loc, i32Ty, 0x400000, rewriter);
-    Value man1Bit = b.create<arith::AndIOp>(f32Bits, cF32FirstBitMask);
-    man1Bit = b.create<arith::ShRUIOp>(man1Bit, c0x00000016);
-    Value f4Man = b.create<arith::TruncIOp>(i4Ty, man1Bit);
-    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Man);
+    Value man1Bit = arith::AndIOp::create(b, f32Bits, cF32FirstBitMask);
+    man1Bit = arith::ShRUIOp::create(b, man1Bit, c0x00000016);
+    Value f4Man = arith::TruncIOp::create(b, i4Ty, man1Bit);
+    f4Bits = arith::AddIOp::create(b, f4Bits, f4Man);
 
     // Step 4: Special consideration for conversion to 0.5.
     Value cF32MantissaMask = createConst(loc, i32Ty, 0x7fffff, rewriter);
-    Value f8Exp = b.create<arith::TruncIOp>(i8Ty, biasAdjustedSignExp);
+    Value f8Exp = arith::TruncIOp::create(b, i8Ty, biasAdjustedSignExp);
     Value isSubnormal =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::sle, f8Exp, c0x00);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::sle, f8Exp, c0x00);
     Value isNegOneExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0xff);
-    Value man23Bits = b.create<arith::AndIOp>(f32Bits, cF32MantissaMask);
-    Value isNonZeroMan = b.create<arith::CmpIOp>(arith::CmpIPredicate::ugt,
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0xff);
+    Value man23Bits = arith::AndIOp::create(b, f32Bits, cF32MantissaMask);
+    Value isNonZeroMan = arith::CmpIOp::create(b, arith::CmpIPredicate::ugt,
                                                  man23Bits, zeroExpBits);
-    Value roundToHalf = b.create<arith::AndIOp>(isNegOneExp, isNonZeroMan);
+    Value roundToHalf = arith::AndIOp::create(b, isNegOneExp, isNonZeroMan);
     Value isZeroExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0x00);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0x00);
     Value subnormalF4Bits = createConst(loc, i4Ty, 0xf, rewriter);
     Value halfF4Bits = createConst(loc, i4Ty, 0x0, rewriter);
     Value subResult =
-        b.create<arith::SelectOp>(isSubnormal, subnormalF4Bits, f4Bits);
-    subResult = b.create<arith::SelectOp>(roundToHalf, halfF4Bits, subResult);
-    f4Bits = b.create<arith::SelectOp>(isZeroExp, f4Bits, subResult);
+        arith::SelectOp::create(b, isSubnormal, subnormalF4Bits, f4Bits);
+    subResult = arith::SelectOp::create(b, roundToHalf, halfF4Bits, subResult);
+    f4Bits = arith::SelectOp::create(b, isZeroExp, f4Bits, subResult);
 
     // Step 5: Round up if necessary.
     Value cF32Last22BitMask = createConst(loc, i32Ty, 0x3fffff, rewriter);
     Value cRound = createConst(loc, i32Ty, 0x200000, rewriter); // 010 0000...
-    Value man22Bits = b.create<arith::AndIOp>(f32Bits, cF32Last22BitMask);
+    Value man22Bits = arith::AndIOp::create(b, f32Bits, cF32Last22BitMask);
     Value shouldRound =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, man22Bits, cRound);
-    shouldRound = b.create<arith::OrIOp>(shouldRound, isSubnormal);
-    Value roundedF4Bits = b.create<arith::AddIOp>(f4Bits, c0x1);
-    f4Bits = b.create<arith::SelectOp>(shouldRound, roundedF4Bits, f4Bits);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, man22Bits, cRound);
+    shouldRound = arith::OrIOp::create(b, shouldRound, isSubnormal);
+    Value roundedF4Bits = arith::AddIOp::create(b, f4Bits, c0x1);
+    f4Bits = arith::SelectOp::create(b, shouldRound, roundedF4Bits, f4Bits);
 
-    Value result = b.create<arith::BitcastOp>(resultTy, f4Bits);
+    Value result = arith::BitcastOp::create(b, resultTy, f4Bits);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -628,16 +628,16 @@ struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (operandETy.getIntOrFloatBitWidth() < 32) {
-      operand = b.create<arith::ExtFOp>(f32Ty, operand, op.getFastmathAttr());
+      operand = arith::ExtFOp::create(b, f32Ty, operand, op.getFastmathAttr());
     } else if (operandETy.getIntOrFloatBitWidth() > 32) {
-      operand = b.create<arith::TruncFOp>(
+      operand = arith::TruncFOp::create(b,
           f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
     }
-    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value f32Bits = arith::BitcastOp::create(b, i32Ty, operand);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
-    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
-    Value exp8Bits = b.create<arith::TruncIOp>(i8Ty, f32SignExp);
-    Value result = b.create<arith::BitcastOp>(resultTy, exp8Bits);
+    Value f32SignExp = arith::ShRUIOp::create(b, f32Bits, cF32MantissaWidth);
+    Value exp8Bits = arith::TruncIOp::create(b, i8Ty, f32SignExp);
+    Value result = arith::BitcastOp::create(b, resultTy, exp8Bits);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -656,7 +656,7 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
     if (scaleETy.getIntOrFloatBitWidth() >= 16) {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
-      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+      scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
                                                op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
@@ -668,11 +668,11 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
     // extf on scale will essentially create floating point number
     // of type resulTy that is 2^scale and will also propagate NaNs
     Value scaleExt =
-        b.create<arith::ExtFOp>(resultTy, scaleOperand, op.getFastmathAttr());
+        arith::ExtFOp::create(b, resultTy, scaleOperand, op.getFastmathAttr());
     Value inputExt =
-        b.create<arith::ExtFOp>(resultTy, inputOperand, op.getFastmathAttr());
+        arith::ExtFOp::create(b, resultTy, inputOperand, op.getFastmathAttr());
     Value result =
-        b.create<arith::MulFOp>(inputExt, scaleExt, op.getFastmathAttr());
+        arith::MulFOp::create(b, inputExt, scaleExt, op.getFastmathAttr());
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -697,7 +697,7 @@ struct ScalingTruncFOpConverter
     if (scaleETy.getIntOrFloatBitWidth() >= 16) {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
-      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+      scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
                                                op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
@@ -710,10 +710,10 @@ struct ScalingTruncFOpConverter
     // this will create a floating point number of type
     // inputTy that is 2^scale and will also propagate NaNs
     scaleOperand =
-        b.create<arith::ExtFOp>(inputTy, scaleOperand, op.getFastmathAttr());
-    Value result = b.create<arith::DivFOp>(inputOperand, scaleOperand,
+        arith::ExtFOp::create(b, inputTy, scaleOperand, op.getFastmathAttr());
+    Value result = arith::DivFOp::create(b, inputOperand, scaleOperand,
                                            op.getFastmathAttr());
-    Value resultCast = b.create<arith::TruncFOp>(
+    Value resultCast = arith::TruncFOp::create(b,
         resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
     rewriter.replaceOp(op, resultCast);
     return success();
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index f2f93883eb2b7..777ff0ecaa314 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -305,18 +305,18 @@ static Value doCast(OpBuilder &builder, Location loc, Value src, Type dstType,
 
   if (isa<IndexType>(srcElemType) || isa<IndexType>(dstElemType)) {
     if (castKind == CastKind::Signed)
-      return builder.create<arith::IndexCastOp>(loc, dstType, src);
-    return builder.create<arith::IndexCastUIOp>(loc, dstType, src);
+      return arith::IndexCastOp::create(builder, loc, dstType, src);
+    return arith::IndexCastUIOp::create(builder, loc, dstType, src);
   }
 
   auto srcInt = cast<IntegerType>(srcElemType);
   auto dstInt = cast<IntegerType>(dstElemType);
   if (dstInt.getWidth() < srcInt.getWidth())
-    return builder.create<arith::TruncIOp>(loc, dstType, src);
+    return arith::TruncIOp::create(builder, loc, dstType, src);
 
   if (castKind == CastKind::Signed)
-    return builder.create<arith::ExtSIOp>(loc, dstType, src);
-  return builder.create<arith::ExtUIOp>(loc, dstType, src);
+    return arith::ExtSIOp::create(builder, loc, dstType, src);
+  return arith::ExtUIOp::create(builder, loc, dstType, src);
 }
 
 struct NarrowElementwise final : OpTraitRewritePattern<OpTrait::Elementwise> {
diff --git a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
index 5fb7953f93700..904aab33345f0 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
@@ -23,7 +23,7 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
   std::function<Value(AffineExpr)> buildExpr = [&](AffineExpr e) -> Value {
     switch (e.getKind()) {
     case AffineExprKind::Constant:
-      return b.create<ConstantIndexOp>(loc,
+      return ConstantIndexOp::create(b, loc,
                                        cast<AffineConstantExpr>(e).getValue());
     case AffineExprKind::DimId:
       return operands[cast<AffineDimExpr>(e).getPosition()];
@@ -32,27 +32,27 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
                       map.getNumDims()];
     case AffineExprKind::Add: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<AddIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return AddIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                               buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mul: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<MulIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return MulIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                               buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::FloorDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<DivSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return DivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::CeilDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<CeilDivSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return CeilDivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                    buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mod: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<RemSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return RemSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                buildExpr(binaryExpr.getRHS()));
     }
     }
@@ -89,10 +89,10 @@ FailureOr<OpFoldResult> mlir::arith::reifyValueBound(
            "expected dynamic dim");
     if (isa<RankedTensorType>(value.getType())) {
       // A tensor dimension is used: generate a tensor.dim.
-      operands.push_back(b.create<tensor::DimOp>(loc, value, *dim));
+      operands.push_back(tensor::DimOp::create(b, loc, value, *dim));
     } else if (isa<MemRefType>(value.getType())) {
       // A memref dimension is used: generate a memref.dim.
-      operands.push_back(b.create<memref::DimOp>(loc, value, *dim));
+      operands.push_back(memref::DimOp::create(b, loc, value, *dim));
     } else {
       llvm_unreachable("cannot generate DimOp for unsupported shaped type");
     }
diff --git a/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
index 62d137a4cfb0e..10c5b50cb771a 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
@@ -84,7 +84,7 @@ struct ConstantShardingInterface
           cOp.getType(), getMesh(op, sharding.getMeshAttr(), symbolTable),
           sharding));
       auto newValue = value.resizeSplat(newType);
-      auto newOp = builder.create<ConstantOp>(op->getLoc(), newType, newValue);
+      auto newOp = ConstantOp::create(builder, op->getLoc(), newType, newValue);
       spmdizationMap.map(op->getResult(0), newOp.getResult());
       spmdizationMap.map(op, newOp.getOperation());
     } else {
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index 3cd8684878a11..6e9cc14c6ada5 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -68,7 +68,7 @@ mlir::inferExpandShapeOutputShape(OpBuilder &b, Location loc,
     // dynamism.
     Value indexGroupSize = cast<Value>(inputShape[inputIndex]);
     Value indexGroupStaticSizesProduct =
-        b.create<arith::ConstantIndexOp>(loc, indexGroupStaticSizesProductInt);
+        arith::ConstantIndexOp::create(b, loc, indexGroupStaticSizesProductInt);
     Value dynamicDimSize = b.createOrFold<arith::DivSIOp>(
         loc, indexGroupSize, indexGroupStaticSizesProduct);
     outputShapeValues.push_back(dynamicDimSize);
@@ -105,7 +105,7 @@ Value mlir::getValueOrCreateConstantIntOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return b.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(b,
       loc, b.getIntegerAttr(attr.getType(), attr.getValue().getSExtValue()));
 }
 
@@ -114,7 +114,7 @@ Value mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return b.create<arith::ConstantIndexOp>(loc, attr.getValue().getSExtValue());
+  return arith::ConstantIndexOp::create(b, loc, attr.getValue().getSExtValue());
 }
 
 Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
@@ -125,7 +125,7 @@ Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
   bool targetIsIndex = targetType.isIndex();
   bool valueIsIndex = value.getType().isIndex();
   if (targetIsIndex ^ valueIsIndex)
-    return b.create<arith::IndexCastOp>(loc, targetType, value);
+    return arith::IndexCastOp::create(b, loc, targetType, value);
 
   auto targetIntegerType = dyn_cast<IntegerType>(targetType);
   auto valueIntegerType = dyn_cast<IntegerType>(value.getType());
@@ -134,8 +134,8 @@ Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
   assert(targetIntegerType.getSignedness() == valueIntegerType.getSignedness());
 
   if (targetIntegerType.getWidth() > valueIntegerType.getWidth())
-    return b.create<arith::ExtSIOp>(loc, targetIntegerType, value);
-  return b.create<arith::TruncIOp>(loc, targetIntegerType, value);
+    return arith::ExtSIOp::create(b, loc, targetIntegerType, value);
+  return arith::TruncIOp::create(b, loc, targetIntegerType, value);
 }
 
 static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
@@ -143,21 +143,21 @@ static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
   // If operand is floating point, cast directly to the int type.
   if (isa<FloatType>(operand.getType())) {
     if (isUnsigned)
-      return b.create<arith::FPToUIOp>(toType, operand);
-    return b.create<arith::FPToSIOp>(toType, operand);
+      return arith::FPToUIOp::create(b, toType, operand);
+    return arith::FPToSIOp::create(b, toType, operand);
   }
   // Cast index operands directly to the int type.
   if (operand.getType().isIndex())
-    return b.create<arith::IndexCastOp>(toType, operand);
+    return arith::IndexCastOp::create(b, toType, operand);
   if (auto fromIntType = dyn_cast<IntegerType>(operand.getType())) {
     // Either extend or truncate.
     if (toType.getWidth() > fromIntType.getWidth()) {
       if (isUnsigned)
-        return b.create<arith::ExtUIOp>(toType, operand);
-      return b.create<arith::ExtSIOp>(toType, operand);
+        return arith::ExtUIOp::create(b, toType, operand);
+      return arith::ExtSIOp::create(b, toType, operand);
     }
     if (toType.getWidth() < fromIntType.getWidth())
-      return b.create<arith::TruncIOp>(toType, operand);
+      return arith::TruncIOp::create(b, toType, operand);
     return operand;
   }
 
@@ -170,14 +170,14 @@ static Value convertScalarToFpDtype(ImplicitLocOpBuilder &b, Value operand,
   // Note that it is unclear how to cast from BF16<->FP16.
   if (isa<IntegerType>(operand.getType())) {
     if (isUnsigned)
-      return b.create<arith::UIToFPOp>(toType, operand);
-    return b.create<arith::SIToFPOp>(toType, operand);
+      return arith::UIToFPOp::create(b, toType, operand);
+    return arith::SIToFPOp::create(b, toType, operand);
   }
   if (auto fromFpTy = dyn_cast<FloatType>(operand.getType())) {
     if (toType.getWidth() > fromFpTy.getWidth())
-      return b.create<arith::ExtFOp>(toType, operand);
+      return arith::ExtFOp::create(b, toType, operand);
     if (toType.getWidth() < fromFpTy.getWidth())
-      return b.create<arith::TruncFOp>(toType, operand);
+      return arith::TruncFOp::create(b, toType, operand);
     return operand;
   }
 
@@ -190,18 +190,18 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
   if (auto fromComplexType = dyn_cast<ComplexType>(operand.getType())) {
     if (isa<FloatType>(targetType.getElementType()) &&
         isa<FloatType>(fromComplexType.getElementType())) {
-      Value real = b.create<complex::ReOp>(operand);
-      Value imag = b.create<complex::ImOp>(operand);
+      Value real = complex::ReOp::create(b, operand);
+      Value imag = complex::ImOp::create(b, operand);
       Type targetETy = targetType.getElementType();
       if (targetType.getElementType().getIntOrFloatBitWidth() <
           fromComplexType.getElementType().getIntOrFloatBitWidth()) {
-        real = b.create<arith::TruncFOp>(targetETy, real);
-        imag = b.create<arith::TruncFOp>(targetETy, imag);
+        real = arith::TruncFOp::create(b, targetETy, real);
+        imag = arith::TruncFOp::create(b, targetETy, imag);
       } else {
-        real = b.create<arith::ExtFOp>(targetETy, real);
-        imag = b.create<arith::ExtFOp>(targetETy, imag);
+        real = arith::ExtFOp::create(b, targetETy, real);
+        imag = arith::ExtFOp::create(b, targetETy, imag);
       }
-      return b.create<complex::CreateOp>(targetType, real, imag);
+      return complex::CreateOp::create(b, targetType, real, imag);
     }
   }
 
@@ -210,27 +210,27 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
     auto toBitwidth = toFpTy.getIntOrFloatBitWidth();
     Value from = operand;
     if (from.getType().getIntOrFloatBitWidth() < toBitwidth) {
-      from = b.create<arith::ExtFOp>(toFpTy, from);
+      from = arith::ExtFOp::create(b, toFpTy, from);
     }
     if (from.getType().getIntOrFloatBitWidth() > toBitwidth) {
-      from = b.create<arith::TruncFOp>(toFpTy, from);
+      from = arith::TruncFOp::create(b, toFpTy, from);
     }
-    Value zero = b.create<mlir::arith::ConstantFloatOp>(
+    Value zero = mlir::arith::ConstantFloatOp::create(b,
         toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
-    return b.create<complex::CreateOp>(targetType, from, zero);
+    return complex::CreateOp::create(b, targetType, from, zero);
   }
 
   if (isa<IntegerType>(operand.getType())) {
     FloatType toFpTy = cast<FloatType>(targetType.getElementType());
     Value from = operand;
     if (isUnsigned) {
-      from = b.create<arith::UIToFPOp>(toFpTy, from);
+      from = arith::UIToFPOp::create(b, toFpTy, from);
     } else {
-      from = b.create<arith::SIToFPOp>(toFpTy, from);
+      from = arith::SIToFPOp::create(b, toFpTy, from);
     }
-    Value zero = b.create<mlir::arith::ConstantFloatOp>(
+    Value zero = mlir::arith::ConstantFloatOp::create(b,
         toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
-    return b.create<complex::CreateOp>(targetType, from, zero);
+    return complex::CreateOp::create(b, targetType, from, zero);
   }
 
   return {};
@@ -278,7 +278,7 @@ Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
     attr = SplatElementsAttr::get(vecTy, value);
   }
 
-  return builder.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(builder, loc, attr);
 }
 
 Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
@@ -310,35 +310,35 @@ Type mlir::getType(OpFoldResult ofr) {
 }
 
 Value ArithBuilder::_and(Value lhs, Value rhs) {
-  return b.create<arith::AndIOp>(loc, lhs, rhs);
+  return arith::AndIOp::create(b, loc, lhs, rhs);
 }
 Value ArithBuilder::add(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::AddFOp>(loc, lhs, rhs);
-  return b.create<arith::AddIOp>(loc, lhs, rhs, ovf);
+    return arith::AddFOp::create(b, loc, lhs, rhs);
+  return arith::AddIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::sub(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::SubFOp>(loc, lhs, rhs);
-  return b.create<arith::SubIOp>(loc, lhs, rhs, ovf);
+    return arith::SubFOp::create(b, loc, lhs, rhs);
+  return arith::SubIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::mul(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::MulFOp>(loc, lhs, rhs);
-  return b.create<arith::MulIOp>(loc, lhs, rhs, ovf);
+    return arith::MulFOp::create(b, loc, lhs, rhs);
+  return arith::MulIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::sgt(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGT, lhs, rhs);
-  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sgt, lhs, rhs);
+    return arith::CmpFOp::create(b, loc, arith::CmpFPredicate::OGT, lhs, rhs);
+  return arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sgt, lhs, rhs);
 }
 Value ArithBuilder::slt(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OLT, lhs, rhs);
-  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, lhs, rhs);
+    return arith::CmpFOp::create(b, loc, arith::CmpFPredicate::OLT, lhs, rhs);
+  return arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, lhs, rhs);
 }
 Value ArithBuilder::select(Value cmp, Value lhs, Value rhs) {
-  return b.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+  return arith::SelectOp::create(b, loc, cmp, lhs, rhs);
 }
 
 namespace mlir::arith {
@@ -349,7 +349,7 @@ Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values) {
 
 Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
                     Type resultType) {
-  Value one = builder.create<ConstantOp>(loc, resultType,
+  Value one = ConstantOp::create(builder, loc, resultType,
                                          builder.getOneAttr(resultType));
   ArithBuilder arithBuilder(builder, loc);
   return std::accumulate(
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
index 7180884c77e98..0d5640b202fb3 100644
--- a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
@@ -231,7 +231,7 @@ class LowerContractionToNeonI8MMPattern
 
     // Initial accumulator for the final result. This is the un-tiled result if
     // tiling is done.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, op.getResultType(), rewriter.getZeroAttr(op.getResultType()));
 
     SmallVector<int64_t> unrolledSize = *op.getShapeForUnroll();
@@ -283,7 +283,7 @@ class LowerContractionToNeonI8MMPattern
       if (isVecmat) {
         auto expandForSMMLA = [&](Value tiledOperand,
                                   VectorType expandedTypeType) {
-          auto emptyOperand = rewriter.create<arith::ConstantOp>(
+          auto emptyOperand = arith::ConstantOp::create(rewriter,
               loc, expandedTypeType, rewriter.getZeroAttr(expandedTypeType));
           SmallVector<int64_t> offsets(
               cast<ShapedType>(emptyOperand.getType()).getRank(), 0);
@@ -300,7 +300,7 @@ class LowerContractionToNeonI8MMPattern
       // using the instruction for unsigned by signed multiplication with
       // reversed operands.
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledAcc = rewriter.create<vector::TransposeOp>(
+        tiledAcc = vector::TransposeOp::create(rewriter,
             loc, tiledAcc, ArrayRef<int64_t>({1, 0}));
 
       // Collapse tiled operands to 1D vectors required by smmla intrinsic
@@ -333,7 +333,7 @@ class LowerContractionToNeonI8MMPattern
       // Because of the reversed operands the result is obtained transposed.
       // Transpose it back,
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledRes = rewriter.create<vector::TransposeOp>(
+        tiledRes = vector::TransposeOp::create(rewriter,
             loc, tiledRes, ArrayRef<int64_t>({1, 0}));
 
       // With vecmat, only one row of tiled ACC can be inserted into the final
diff --git a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
index 1f7305a5f8141..a4aebb585c3de 100644
--- a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
@@ -76,21 +76,21 @@ scf::ForOp createLoopOverTileSlices(
     PatternRewriter &rewriter, Location loc, Value initTile,
     std::function<Value(OpBuilder &, Location, Value, Value)> makeLoopBody) {
   OpBuilder::InsertionGuard g(rewriter);
-  auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-  auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+  auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
       loc, llvm::cast<VectorType>(initTile.getType()).getDimSize(0));
   auto vscale =
-      rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-  auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+      vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+  auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto numTileSlices =
-      rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
-  auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, numTileSlices, step,
+      arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices, step,
                                            ValueRange{initTile});
   rewriter.setInsertionPointToStart(forOp.getBody());
   Value nextTile =
       makeLoopBody(rewriter, loc, /*tileSliceIndex=*/forOp.getInductionVar(),
                    /*currentTile=*/forOp.getRegionIterArg(0));
-  rewriter.create<scf::YieldOp>(loc, nextTile);
+  scf::YieldOp::create(rewriter, loc, nextTile);
   return forOp;
 }
 
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
index 23f2c2bf65e47..4d03bbdabdf09 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
@@ -136,7 +136,7 @@ class OuterProductFusion2Way
 
     auto loc = op.getLoc();
     auto packInputs = [&](Value lhs, Value rhs) {
-      return rewriter.create<vector::InterleaveOp>(loc, lhs, rhs);
+      return vector::InterleaveOp::create(rewriter, loc, lhs, rhs);
     };
 
     auto lhs = packInputs(op1.getLhs().getDefiningOp()->getOperand(0),
@@ -284,7 +284,7 @@ class OuterProductFusion4Way
 
     auto loc = op.getLoc();
     auto packInputs = [&](Value lhs, Value rhs) {
-      return rewriter.create<vector::InterleaveOp>(loc, lhs, rhs);
+      return vector::InterleaveOp::create(rewriter, loc, lhs, rhs);
     };
 
     auto lhs0 = packInputs(op1.getLhs().getDefiningOp()->getOperand(0),
@@ -456,7 +456,7 @@ struct SwapVectorExtractOfArithExtend
     Value extendSource = extendOp->getOperand(0);
 
     // Create new extract from source of extend.
-    Value newExtract = rewriter.create<vector::ExtractOp>(
+    Value newExtract = vector::ExtractOp::create(rewriter,
         loc, extendSource, extractOp.getMixedPosition());
 
     // Extend new extract to original result type.
@@ -503,7 +503,7 @@ struct SwapVectorScalableExtractOfArithExtend
     // Create new extract from source of extend.
     VectorType extractResultVectorType =
         resultType.clone(extendSourceVectorType.getElementType());
-    Value newExtract = rewriter.create<vector::ScalableExtractOp>(
+    Value newExtract = vector::ScalableExtractOp::create(rewriter,
         loc, extractResultVectorType, extendSource, extractOp.getPos());
 
     // Extend new extract to original result type.
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index e6c9adba62f34..2bea9de83143a 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -212,7 +212,7 @@ void splitCondBranches(IRRewriter &rewriter, FunctionOpInterface function) {
 
   auto insertJump = [&](Location loc, Block *source, Block *dest, auto args) {
     rewriter.setInsertionPointToEnd(source);
-    rewriter.create<cf::BranchOp>(loc, dest, args);
+    cf::BranchOp::create(rewriter, loc, dest, args);
   };
 
   for (auto condBranch : worklist) {
@@ -255,7 +255,7 @@ void insertCopiesAtBranches(IRRewriter &rewriter,
     for (OpOperand &operand : terminator->getOpOperands()) {
       if (isValidSMETileVectorType(operand.get().getType())) {
         auto copy =
-            rewriter.create<CopyTileOp>(terminator->getLoc(), operand.get());
+            CopyTileOp::create(rewriter, terminator->getLoc(), operand.get());
         rewriter.modifyOpInPlace(terminator, [&] { operand.assign(copy); });
       }
     }
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 1e8e1265affa0..3362d0ce13533 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -82,13 +82,13 @@ SmallVector<Value, 2> addConstantScalableOffset(OpBuilder &builder,
                                                 Location loc,
                                                 ValueRange indices,
                                                 ArrayRef<int> scalableOffsets) {
-  auto vscale = builder.create<vector::VectorScaleOp>(loc);
+  auto vscale = vector::VectorScaleOp::create(builder, loc);
   return llvm::map_to_vector(
       llvm::zip_equal(indices, scalableOffsets), [&](auto pair) -> Value {
         auto [index, base] = pair;
-        auto offset = builder.create<arith::MulIOp>(
-            loc, builder.create<arith::ConstantIndexOp>(loc, base), vscale);
-        return builder.create<arith::AddIOp>(loc, index, offset);
+        auto offset = arith::MulIOp::create(builder,
+            loc, arith::ConstantIndexOp::create(builder, loc, base), vscale);
+        return arith::AddIOp::create(builder, loc, index, offset);
       });
 }
 
@@ -132,7 +132,7 @@ Value extractSMEMask(OpBuilder &builder, Location loc, Value mask,
   // from the mask operands to get the parameters for this sub-tile.
   auto smeTileMaskDims = addConstantScalableOffset(
       builder, loc, createMask.getOperands(), {-smeTile.row, -smeTile.col});
-  auto smeTileCreateMask = builder.create<vector::CreateMaskOp>(
+  auto smeTileCreateMask = vector::CreateMaskOp::create(builder,
       loc, smeTile.type.clone(builder.getI1Type()), smeTileMaskDims);
   return smeTileCreateMask.getResult();
 }
@@ -190,7 +190,7 @@ struct LegalizeArithConstantOpsByDecomposition
 
     auto smeTileType = getSMETileTypeForElement(vectorType.getElementType());
     auto tileCount = getNumberOfSMETilesForVectorType(vectorType);
-    auto tileSplat = rewriter.create<arith::ConstantOp>(
+    auto tileSplat = arith::ConstantOp::create(rewriter,
         constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
     SmallVector<Value> repl(tileCount, tileSplat);
     rewriter.replaceOpWithMultiple(constantOp, {repl});
@@ -237,11 +237,11 @@ struct LegalizeVectorOuterProductOpsByDecomposition
              decomposeToSMETiles(rewriter, vectorType, smeTileType))) {
 
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto lhs = rewriter.create<vector::ScalableExtractOp>(
+      auto lhs = vector::ScalableExtractOp::create(rewriter,
           loc, sliceType, outerProductOp.getLhs(), smeTile.row);
-      auto rhs = rewriter.create<vector::ScalableExtractOp>(
+      auto rhs = vector::ScalableExtractOp::create(rewriter,
           loc, sliceType, outerProductOp.getRhs(), smeTile.col);
-      auto smeOuterProduct = rewriter.create<vector::OuterProductOp>(
+      auto smeOuterProduct = vector::OuterProductOp::create(rewriter,
           loc, smeTileType, lhs, rhs,
           !accSMETiles.empty() ? accSMETiles[index] : Value{},
           outerProductOp.getKind());
@@ -314,7 +314,7 @@ struct LegalizeTransferReadOpsByDecomposition
     for (SMESubTile smeTile :
          decomposeToSMETiles(rewriter, vectorType, smeTileType, transposed)) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeRead = rewriter.create<vector::TransferReadOp>(
+      auto smeRead = vector::TransferReadOp::create(rewriter,
           loc, smeTileType, readOp.getBase(),
           getSMESubTileIndices(rewriter, loc, readOp.getIndices(), smeTile),
           readOp.getPermutationMapAttr(), readOp.getPadding(), smeMask,
@@ -363,7 +363,7 @@ struct LegalizeTransferWriteOpsByDecomposition
     for (auto [index, smeTile] : llvm::enumerate(decomposeToSMETiles(
              rewriter, vectorType, smeTileType, transposed))) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeWrite = rewriter.create<vector::TransferWriteOp>(
+      auto smeWrite = vector::TransferWriteOp::create(rewriter,
           loc, inputSMETiles[index], destTensorOrMemref,
           getSMESubTileIndices(rewriter, loc, writeOp.getIndices(), smeTile),
           writeOp.getPermutationMapAttr(), smeMask, writeOp.getInBoundsAttr());
@@ -456,11 +456,11 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
         VectorType::get(minTileSlices, rewriter.getI1Type(), true);
 
     // Create loop over all tile slices.
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto upperBound = createVscaleMultiple(minTileSlices);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto storeLoop =
-        rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     rewriter.setInsertionPointToStart(storeLoop.getBody());
 
     // For each sub-tile of the multi-tile `vectorType`.
@@ -474,29 +474,29 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
 
       // The current slice of `vectorType` we are processing.
       auto sliceIndex =
-          rewriter.create<arith::AddIOp>(loc, tileRow, tileSliceIndex);
+          arith::AddIOp::create(rewriter, loc, tileRow, tileSliceIndex);
 
       // Where in the destination memref the current slice will be stored.
-      auto storeRow = rewriter.create<arith::AddIOp>(loc, sliceIndex,
+      auto storeRow = arith::AddIOp::create(rewriter, loc, sliceIndex,
                                                      writeOp.getIndices()[0]);
       auto storeCol =
-          rewriter.create<arith::AddIOp>(loc, tileCol, writeOp.getIndices()[1]);
+          arith::AddIOp::create(rewriter, loc, tileCol, writeOp.getIndices()[1]);
 
       // Extract the mask for the current slice.
       Value sliceMask = nullptr;
       if (mask) {
-        sliceMask = rewriter.create<vector::ExtractOp>(
+        sliceMask = vector::ExtractOp::create(rewriter,
             loc, mask, OpFoldResult(sliceIndex));
         if (sliceMaskType != sliceMask.getType())
-          sliceMask = rewriter.create<vector::ScalableExtractOp>(
+          sliceMask = vector::ScalableExtractOp::create(rewriter,
               loc, sliceMaskType, sliceMask, smeTile.col);
       }
 
       // Extract and store the current slice.
       Value tile = inputSMETiles[index];
       auto slice =
-          rewriter.create<vector::ExtractOp>(loc, tile, tileSliceIndex);
-      rewriter.create<vector::TransferWriteOp>(
+          vector::ExtractOp::create(rewriter, loc, tile, tileSliceIndex);
+      vector::TransferWriteOp::create(rewriter,
           loc, slice, writeOp.getBase(), ValueRange{storeRow, storeCol},
           AffineMapAttr::get(writeOp.getPermutationMap().dropResult(0)),
           sliceMask,
@@ -567,13 +567,13 @@ struct FoldExtractFromVectorOfSMELikeCreateMasks
           extractOp,
           "constant vector.create_masks dims should be folded elsewhere");
 
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto extractionIndex = getValueOrCreateConstantIndexOp(
         rewriter, loc, extractOp.getMixedPosition()[0]);
-    auto extractionInTrueRegion = rewriter.create<arith::CmpIOp>(
+    auto extractionInTrueRegion = arith::CmpIOp::create(rewriter,
         loc, rewriter.getI1Type(), arith::CmpIPredicate::slt, extractionIndex,
         frontMaskDim);
-    auto newMaskFrontDim = rewriter.create<arith::SelectOp>(
+    auto newMaskFrontDim = arith::SelectOp::create(rewriter,
         loc, extractionInTrueRegion, createMaskOp.getOperand(1), zero);
 
     rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(
@@ -660,8 +660,8 @@ struct LiftIllegalVectorTransposeToMemory
           illegalRead, "expected read to have identity permutation map");
 
     auto loc = transposeOp.getLoc();
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
     // Create a subview that matches the size of the illegal read vector type.
     auto readType = illegalRead.getVectorType();
@@ -669,14 +669,14 @@ struct LiftIllegalVectorTransposeToMemory
         llvm::zip_equal(readType.getShape(), readType.getScalableDims()),
         [&](auto dim) -> Value {
           auto [size, isScalable] = dim;
-          auto dimSize = rewriter.create<arith::ConstantIndexOp>(loc, size);
+          auto dimSize = arith::ConstantIndexOp::create(rewriter, loc, size);
           if (!isScalable)
             return dimSize;
-          auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
-          return rewriter.create<arith::MulIOp>(loc, vscale, dimSize);
+          auto vscale = vector::VectorScaleOp::create(rewriter, loc);
+          return arith::MulIOp::create(rewriter, loc, vscale, dimSize);
         });
     SmallVector<Value> strides(readType.getRank(), Value(one));
-    auto readSubview = rewriter.create<memref::SubViewOp>(
+    auto readSubview = memref::SubViewOp::create(rewriter,
         loc, illegalRead.getBase(), illegalRead.getIndices(), readSizes,
         strides);
 
@@ -686,13 +686,13 @@ struct LiftIllegalVectorTransposeToMemory
     if (mask) {
       // Note: The transpose for the mask should fold into the
       // vector.create_mask/constant_mask op, which will then become legal.
-      mask = rewriter.create<vector::TransposeOp>(loc, mask,
+      mask = vector::TransposeOp::create(rewriter, loc, mask,
                                                   transposeOp.getPermutation());
     }
     // - The source memref
     mlir::AffineMap transposeMap = AffineMap::getPermutationMap(
         transposeOp.getPermutation(), getContext());
-    auto transposedSubview = rewriter.create<memref::TransposeOp>(
+    auto transposedSubview = memref::TransposeOp::create(rewriter,
         loc, readSubview, AffineMapAttr::get(transposeMap));
     ArrayAttr inBoundsAttr = illegalRead.getInBoundsAttr();
     // - The `in_bounds` attribute
@@ -706,7 +706,7 @@ struct LiftIllegalVectorTransposeToMemory
     VectorType legalReadType = resultType.clone(readType.getElementType());
     // Note: The indices are all zero as the subview is already offset.
     SmallVector<Value> readIndices(illegalRead.getIndices().size(), zero);
-    auto legalRead = rewriter.create<vector::TransferReadOp>(
+    auto legalRead = vector::TransferReadOp::create(rewriter,
         loc, legalReadType, transposedSubview, readIndices,
         illegalRead.getPermutationMapAttr(), illegalRead.getPadding(), mask,
         inBoundsAttr);
@@ -797,12 +797,12 @@ struct LowerIllegalTransposeStoreViaZA
         AffineMap::getPermutationMap(ArrayRef<int64_t>{1, 0}, getContext()));
 
     // Note: We need to use `get_tile` as there's no vector-level `undef`.
-    Value undefTile = rewriter.create<arm_sme::GetTileOp>(loc, smeTileType);
+    Value undefTile = arm_sme::GetTileOp::create(rewriter, loc, smeTileType);
     Value destTensorOrMemref = writeOp.getBase();
     auto numSlicesPerTile =
         std::min(sourceType.getDimSize(0), smeTileType.getDimSize(0));
     auto numSlices =
-        rewriter.create<arith::ConstantIndexOp>(loc, numSlicesPerTile);
+        arith::ConstantIndexOp::create(rewriter, loc, numSlicesPerTile);
     for (auto [index, smeTile] : llvm::enumerate(
              decomposeToSMETiles(rewriter, sourceType, smeTileType))) {
       // 1. _Deliberately_ drop a scalable dimension and insert a fixed number
@@ -811,46 +811,46 @@ struct LowerIllegalTransposeStoreViaZA
       // rows of the tile after 1*vscale rows.
       Value tile = undefTile;
       for (int d = 0; d < numSlicesPerTile; ++d) {
-        Value vector = rewriter.create<vector::ExtractOp>(
+        Value vector = vector::ExtractOp::create(rewriter,
             loc, transposeOp.getVector(),
             rewriter.getIndexAttr(d + smeTile.row));
         if (vector.getType() != smeSliceType) {
-          vector = rewriter.create<vector::ScalableExtractOp>(
+          vector = vector::ScalableExtractOp::create(rewriter,
               loc, smeSliceType, vector, smeTile.col);
         }
-        tile = rewriter.create<vector::InsertOp>(loc, vector, tile, d);
+        tile = vector::InsertOp::create(rewriter, loc, vector, tile, d);
       }
 
       // 2. Transpose the tile position.
       auto transposedRow = createVscaleMultiple(smeTile.col);
       auto transposedCol =
-          rewriter.create<arith::ConstantIndexOp>(loc, smeTile.row);
+          arith::ConstantIndexOp::create(rewriter, loc, smeTile.row);
 
       // 3. Compute mask for tile store.
       Value maskRows;
       Value maskCols;
       if (auto mask = writeOp.getMask()) {
         auto createMask = mask.getDefiningOp<vector::CreateMaskOp>();
-        maskRows = rewriter.create<arith::SubIOp>(loc, createMask.getOperand(0),
+        maskRows = arith::SubIOp::create(rewriter, loc, createMask.getOperand(0),
                                                   transposedRow);
-        maskCols = rewriter.create<arith::SubIOp>(loc, createMask.getOperand(1),
+        maskCols = arith::SubIOp::create(rewriter, loc, createMask.getOperand(1),
                                                   transposedCol);
-        maskCols = rewriter.create<index::MinSOp>(loc, maskCols, numSlices);
+        maskCols = index::MinSOp::create(rewriter, loc, maskCols, numSlices);
       } else {
         maskRows = createVscaleMultiple(smeTileType.getDimSize(0));
         maskCols = numSlices;
       }
-      auto subMask = rewriter.create<vector::CreateMaskOp>(
+      auto subMask = vector::CreateMaskOp::create(rewriter,
           loc, smeTileType.clone(rewriter.getI1Type()),
           ValueRange{maskRows, maskCols});
 
       // 4. Emit a transposed tile write.
       auto writeIndices = writeOp.getIndices();
       Value destRow =
-          rewriter.create<arith::AddIOp>(loc, transposedRow, writeIndices[0]);
+          arith::AddIOp::create(rewriter, loc, transposedRow, writeIndices[0]);
       Value destCol =
-          rewriter.create<arith::AddIOp>(loc, transposedCol, writeIndices[1]);
-      auto smeWrite = rewriter.create<vector::TransferWriteOp>(
+          arith::AddIOp::create(rewriter, loc, transposedCol, writeIndices[1]);
+      auto smeWrite = vector::TransferWriteOp::create(rewriter,
           loc, tile, destTensorOrMemref, ValueRange{destRow, destCol},
           transposeMap, subMask, writeOp.getInBounds());
 
@@ -934,41 +934,41 @@ struct LowerColumnTransferReadToLoops
 
     // Create a loop over all rows and load one element at a time.
     auto loc = readOp.getLoc();
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto createVscaleMultiple =
         vector::makeVscaleConstantBuilder(rewriter, loc);
     auto upperBound = createVscaleMultiple(numRows);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    Value init = rewriter.create<arith::ConstantOp>(
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    Value init = arith::ConstantOp::create(rewriter,
         loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
 
     scf::ForOp loadLoop;
     {
       OpBuilder::InsertionGuard g(rewriter);
-      loadLoop = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+      loadLoop = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
                                              ValueRange{init});
       rewriter.setInsertionPointToStart(loadLoop.getBody());
 
       auto tileSliceIndex = loadLoop.getInductionVar();
 
-      auto idx0 = rewriter.create<arith::AddIOp>(loc, tileSliceIndex,
+      auto idx0 = arith::AddIOp::create(rewriter, loc, tileSliceIndex,
                                                  readOp.getIndices()[0]);
       auto idx1 = readOp.getIndices()[1];
 
-      Value scalar = rewriter.create<memref::LoadOp>(
+      Value scalar = memref::LoadOp::create(rewriter,
           loc, readOp.getBase(), SmallVector<Value>({idx0, idx1}));
 
-      Operation *updateInit = rewriter.create<vector::InsertOp>(
+      Operation *updateInit = vector::InsertOp::create(rewriter,
           loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
 
-      rewriter.create<scf::YieldOp>(loc, updateInit->getResult(0));
+      scf::YieldOp::create(rewriter, loc, updateInit->getResult(0));
     }
 
     // The read operation has been "legalized", but since the original result
     // type was a 2D vector, we need to cast before returning the result. This
     // ShapeCast should cancel-out with some other ShapeCast (i.e. it's a
     // no-op).
-    auto sc = rewriter.create<vector::ShapeCastOp>(
+    auto sc = vector::ShapeCastOp::create(rewriter,
         loc, readOp.getResult().getType(), loadLoop.getResult(0));
 
     rewriter.replaceOp(readOp, sc);
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index 006332b48325f..20f2ad1c3dcec 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -89,7 +89,7 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     VectorType sourceType = source.getType();
     VectorType resultType = convertOp.getResult().getType();
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
 
     // We want to iterate over the input vector in steps of the trailing
@@ -102,14 +102,14 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     for (SmallVector<int64_t> index :
          StaticTileOffsetRange(sourceType.getShape(), tileShape)) {
       auto extractOrInsertPosition = ArrayRef(index).drop_back();
-      auto sourceVector = rewriter.create<vector::ExtractOp>(
+      auto sourceVector = vector::ExtractOp::create(rewriter,
           loc, source, extractOrInsertPosition);
       VectorType convertedType =
           VectorType::Builder(llvm::cast<VectorType>(sourceVector.getType()))
               .setDim(0, resultType.getShape().back());
       auto convertedVector =
-          rewriter.create<IntrOp>(loc, TypeRange{convertedType}, sourceVector);
-      result = rewriter.create<vector::InsertOp>(loc, convertedVector, result,
+          IntrOp::create(rewriter, loc, TypeRange{convertedType}, sourceVector);
+      result = vector::InsertOp::create(rewriter, loc, convertedVector, result,
                                                  extractOrInsertPosition);
     }
 
@@ -137,11 +137,11 @@ struct PselOpLowering : public ConvertOpToLLVMPattern<PselOp> {
                   ConversionPatternRewriter &rewriter) const override {
     auto svboolType = VectorType::get(16, rewriter.getI1Type(), true);
     auto loc = pselOp.getLoc();
-    auto svboolP1 = rewriter.create<ConvertToSvboolIntrOp>(loc, svboolType,
+    auto svboolP1 = ConvertToSvboolIntrOp::create(rewriter, loc, svboolType,
                                                            adaptor.getP1());
-    auto indexI32 = rewriter.create<arith::IndexCastOp>(
+    auto indexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), pselOp.getIndex());
-    auto pselIntr = rewriter.create<PselIntrOp>(loc, svboolType, svboolP1,
+    auto pselIntr = PselIntrOp::create(rewriter, loc, svboolType, svboolP1,
                                                 pselOp.getP2(), indexI32);
     rewriter.replaceOpWithNewOp<ConvertFromSvboolIntrOp>(
         pselOp, adaptor.getP1().getType(), pselIntr);
@@ -176,7 +176,7 @@ struct CreateMaskOpLowering
                                          "not SVE predicate-sized");
 
     auto loc = createMaskOp.getLoc();
-    auto zero = rewriter.create<LLVM::ZeroOp>(loc, rewriter.getI64Type());
+    auto zero = LLVM::ZeroOp::create(rewriter, loc, rewriter.getI64Type());
     rewriter.replaceOpWithNewOp<WhileLTIntrOp>(createMaskOp, maskType, zero,
                                                adaptor.getOperands()[0]);
     return success();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
index 3dbb93b8a0669..33de12ca17cfc 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
@@ -71,7 +71,7 @@ void replaceOpWithUnrealizedConversion(PatternRewriter &rewriter, TOp op,
                                        TLegalizerCallback callback) {
   replaceOpWithLegalizedOp(rewriter, op, [&](TOp newOp) {
     // Mark our `unrealized_conversion_casts` with a pass label.
-    return rewriter.create<UnrealizedConversionCastOp>(
+    return UnrealizedConversionCastOp::create(rewriter,
         op.getLoc(), TypeRange{op.getResult().getType()},
         ValueRange{callback(newOp)},
         NamedAttribute(rewriter.getStringAttr(kSVELegalizerTag),
@@ -239,7 +239,7 @@ struct LegalizeSVEMaskStoreConversion
 
     auto legalMaskType = widenScalableMaskTypeToSvbool(
         llvm::cast<VectorType>(valueToStore.getType()));
-    auto convertToSvbool = rewriter.create<arm_sve::ConvertToSvboolOp>(
+    auto convertToSvbool = arm_sve::ConvertToSvboolOp::create(rewriter,
         loc, legalMaskType, valueToStore);
     // Replace this store with a conversion to a storable svbool mask [1],
     // followed by a wider store.
@@ -290,7 +290,7 @@ struct LegalizeSVEMaskLoadConversion : public OpRewritePattern<memref::LoadOp> {
     replaceOpWithLegalizedOp(rewriter, loadOp, [&](memref::LoadOp newLoadOp) {
       newLoadOp.setMemRef(*legalMemref);
       newLoadOp.getResult().setType(legalMaskType);
-      return rewriter.create<arm_sve::ConvertFromSvboolOp>(
+      return arm_sve::ConvertFromSvboolOp::create(rewriter,
           loc, loadedMask.getType(), newLoadOp);
     });
 
@@ -408,7 +408,7 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
       reassoc.back().push_back(i);
     if (!memref::CollapseShapeOp::isGuaranteedCollapsible(memTy, reassoc))
       return failure();
-    Value collapsedMem = rewriter.create<memref::CollapseShapeOp>(
+    Value collapsedMem = memref::CollapseShapeOp::create(rewriter,
         readOp.getLoc(), readOp.getBase(), reassoc);
 
     // Get a vector type with collapsed trailing dimensions.
@@ -424,13 +424,13 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
     auto indices = readOp.getIndices().drop_back(numCollapseDims - 1);
 
     // Create the new `transfer_read`.
-    auto newReadOp = rewriter.create<vector::TransferReadOp>(
+    auto newReadOp = vector::TransferReadOp::create(rewriter,
         readOp.getLoc(), collapsedVT, collapsedMem, indices,
         readOp.getPadding(),
         ArrayRef<bool>(origInBounds).drop_back(numCollapseDims - 1));
 
     // Cast back to the original vector type.
-    auto toOrigShape = rewriter.create<vector::ShapeCastOp>(readOp.getLoc(),
+    auto toOrigShape = vector::ShapeCastOp::create(rewriter, readOp.getLoc(),
                                                             origVT, newReadOp);
 
     rewriter.replaceOp(readOp, toOrigShape);
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
index b7703ff0393eb..959c5d7291c31 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
@@ -90,15 +90,15 @@ Value createMMLA(PatternRewriter &rewriter, MMLA op, Location loc,
                  mlir::VectorType accType, Value acc, Value lhs, Value rhs) {
   switch (op) {
   case MMLA::Signed:
-    return rewriter.create<arm_sve::SmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::SmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::Unsigned:
-    return rewriter.create<arm_sve::UmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::UmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::Mixed:
-    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::UsmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::MixedSwapped:
     // The accumulator comes transposed and the result will be transposed
     // later, so all we have to do here is swap the operands.
-    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, rhs, lhs);
+    return arm_sve::UsmmlaOp::create(rewriter, loc, accType, acc, rhs, lhs);
   }
 }
 
@@ -236,25 +236,25 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> lhsTile;
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the LHS tile.
-      auto r0 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+      auto r0 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
                                                    ArrayRef<int64_t>{i});
-      auto r1 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+      auto r1 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
                                                    ArrayRef<int64_t>{i + 1});
       // Concatenate to obtain a 16 x i8 flattened sub-tile.
-      auto t = rewriter.create<vector::ShuffleOp>(
+      auto t = vector::ShuffleOp::create(rewriter,
           loc, r0, r1,
           llvm::ArrayRef<int64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
                                   14, 15});
       // Turn it into a scalable vector.
-      auto s = rewriter.create<vector::ScalableInsertOp>(
-          loc, t, rewriter.create<ub::PoisonOp>(loc, nxv16i8), 0);
+      auto s = vector::ScalableInsertOp::create(rewriter,
+          loc, t, ub::PoisonOp::create(rewriter, loc, nxv16i8), 0);
       // Replicate the sub-tile VSCALE times to fill the entire vector.
-      auto r = rewriter.create<arm_sve::DupQLaneOp>(loc, s, 0);
+      auto r = arm_sve::DupQLaneOp::create(rewriter, loc, s, 0);
       lhsTile.push_back(r);
     }
 
     // "Flatten" the RHS tile from <[N]x8> to <[8*N]>.
-    auto rhs = rewriter.create<vector::ShapeCastOp>(
+    auto rhs = vector::ShapeCastOp::create(rewriter,
         maybeRhs->getLoc(),
         VectorType::get(/*shape=*/8 * N, rewriter.getI8Type(),
                         /*scalableDims=*/{true}),
@@ -264,7 +264,7 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> rhsTile;
     for (int64_t j = 0; j < N; j += 2)
       rhsTile.push_back(
-          rewriter.create<vector::ScalableExtractOp>(loc, nxv16i8, rhs, j * 8));
+          vector::ScalableExtractOp::create(rewriter, loc, nxv16i8, rhs, j * 8));
 
     // Handy types for packing/unpacking of the accumulator tile.
     auto accRowTy = VectorType::get(/*shape=*/N, rewriter.getI32Type(),
@@ -280,33 +280,33 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> accTile;
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the accumulator tile.
-      auto r0 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+      auto r0 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
                                                    ArrayRef<int64_t>{i});
-      auto r1 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+      auto r1 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
                                                    ArrayRef<int64_t>{i + 1});
       Value accTileVec;
       if (mmlaOp == MMLA::MixedSwapped) {
         // We need to swap the positions of the LHS and RHS (since we don't have
         // a signed * unsigned operation), but then each individual 2x2 tile of
         // the acumulator and (later) the result need to be transposed.
-        accTileVec = rewriter.create<vector::InterleaveOp>(loc, r0, r1);
+        accTileVec = vector::InterleaveOp::create(rewriter, loc, r0, r1);
       } else {
         // Bitcast them to 64-bit elements, so subsequent
         // interleave/deinterleave work on pairs of 32-bit numbers.
-        auto r0I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r0);
-        auto r1I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r1);
+        auto r0I64 = vector::BitCastOp::create(rewriter, loc, accRow64Ty, r0);
+        auto r1I64 = vector::BitCastOp::create(rewriter, loc, accRow64Ty, r1);
 
         // Interleave the rows, effectively flattening each 2x2 tile into 4
         // consecutive elements.
-        auto intrI64 = rewriter.create<vector::InterleaveOp>(loc, r0I64, r1I64);
+        auto intrI64 = vector::InterleaveOp::create(rewriter, loc, r0I64, r1I64);
 
         // Bitcast back to 32-bit elements.
         accTileVec =
-            rewriter.create<vector::BitCastOp>(loc, accRowX2Ty, intrI64);
+            vector::BitCastOp::create(rewriter, loc, accRowX2Ty, intrI64);
       }
       // Extract ACC sub-tiles.
       for (int64_t j = 0; j < N; j += 2)
-        accTile.push_back(rewriter.create<vector::ScalableExtractOp>(
+        accTile.push_back(vector::ScalableExtractOp::create(rewriter,
             loc, nxv4i32, accTileVec, j * 2));
     }
 
@@ -320,12 +320,12 @@ class LowerContractionToSVEI8MMPattern
       }
 
     // Unpack the OUT sub-tiles and insert into the result.
-    Value result = rewriter.create<ub::PoisonOp>(loc, op.getResultType());
+    Value result = ub::PoisonOp::create(rewriter, loc, op.getResultType());
     for (int64_t i = 0; i < M / 2; ++i) {
       // Collect a number of sub-tiles in a row.
-      Value row = rewriter.create<ub::PoisonOp>(loc, accRowX2Ty);
+      Value row = ub::PoisonOp::create(rewriter, loc, accRowX2Ty);
       for (int64_t j = 0; j < N / 2; ++j)
-        row = rewriter.create<vector::ScalableInsertOp>(
+        row = vector::ScalableInsertOp::create(rewriter,
             loc, outTile[i * N / 2 + j], row, j * 4);
 
       // Unpack the row to obtain two rows of the output. If we have the out
@@ -334,22 +334,22 @@ class LowerContractionToSVEI8MMPattern
       // Otherwise, the interleave is by pairs.
       Value out0, out1;
       if (mmlaOp == MMLA::MixedSwapped) {
-        auto tmp = rewriter.create<vector::DeinterleaveOp>(loc, row);
+        auto tmp = vector::DeinterleaveOp::create(rewriter, loc, row);
         out0 = tmp.getRes1();
         out1 = tmp.getRes2();
       } else {
         // Deinterleave by pairs.
-        auto row64 = rewriter.create<vector::BitCastOp>(loc, accRowX264Ty, row);
-        auto deintr64 = rewriter.create<vector::DeinterleaveOp>(loc, row64);
+        auto row64 = vector::BitCastOp::create(rewriter, loc, accRowX264Ty, row);
+        auto deintr64 = vector::DeinterleaveOp::create(rewriter, loc, row64);
 
         // Bitcast back into 32-bit elements and insert into the result.
-        out0 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+        out0 = vector::BitCastOp::create(rewriter, loc, accRowTy,
                                                   deintr64.getRes1());
-        out1 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+        out1 = vector::BitCastOp::create(rewriter, loc, accRowTy,
                                                   deintr64.getRes2());
       }
-      result = rewriter.create<vector::InsertOp>(loc, out0, result, i * 2);
-      result = rewriter.create<vector::InsertOp>(loc, out1, result, i * 2 + 1);
+      result = vector::InsertOp::create(rewriter, loc, out0, result, i * 2);
+      result = vector::InsertOp::create(rewriter, loc, out1, result, i * 2 + 1);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index b834afef7da79..71ee198fc0c23 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -99,7 +99,7 @@ void ExecuteOp::build(OpBuilder &builder, OperationState &result,
   // expected result is empty. Otherwise, leave this to the caller
   // because we don't know which values to return from the execute op.
   if (resultTypes.empty() && !bodyBuilder) {
-    builder.create<async::YieldOp>(result.location, ValueRange());
+    async::YieldOp::create(builder, result.location, ValueRange());
   } else if (bodyBuilder) {
     bodyBuilder(builder, result.location, bodyBlock->getArguments());
   }
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index 27fa92cee79c2..447411ad989ba 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -191,8 +191,8 @@ static SmallVector<Value> delinearize(ImplicitLocOpBuilder &b, Value index,
   assert(!tripCounts.empty() && "tripCounts must be not empty");
 
   for (ssize_t i = tripCounts.size() - 1; i >= 0; --i) {
-    coords[i] = b.create<arith::RemSIOp>(index, tripCounts[i]);
-    index = b.create<arith::DivSIOp>(index, tripCounts[i]);
+    coords[i] = arith::RemSIOp::create(b, index, tripCounts[i]);
+    index = arith::DivSIOp::create(b, index, tripCounts[i]);
   }
 
   return coords;
@@ -276,15 +276,15 @@ static ParallelComputeFunction createParallelComputeFunction(
   BlockArgument blockSize = args.blockSize();
 
   // Constants used below.
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Materialize known constants as constant operation in the function body.
   auto values = [&](ArrayRef<BlockArgument> args, ArrayRef<IntegerAttr> attrs) {
     return llvm::to_vector(
         llvm::map_range(llvm::zip(args, attrs), [&](auto tuple) -> Value {
           if (IntegerAttr attr = std::get<1>(tuple))
-            return b.create<arith::ConstantOp>(attr);
+            return arith::ConstantOp::create(b, attr);
           return std::get<0>(tuple);
         }));
   };
@@ -303,17 +303,17 @@ static ParallelComputeFunction createParallelComputeFunction(
   // one-dimensional iteration space.
   Value tripCount = tripCounts[0];
   for (unsigned i = 1; i < tripCounts.size(); ++i)
-    tripCount = b.create<arith::MulIOp>(tripCount, tripCounts[i]);
+    tripCount = arith::MulIOp::create(b, tripCount, tripCounts[i]);
 
   // Find one-dimensional iteration bounds: [blockFirstIndex, blockLastIndex]:
   //   blockFirstIndex = blockIndex * blockSize
-  Value blockFirstIndex = b.create<arith::MulIOp>(blockIndex, blockSize);
+  Value blockFirstIndex = arith::MulIOp::create(b, blockIndex, blockSize);
 
   // The last one-dimensional index in the block defined by the `blockIndex`:
   //   blockLastIndex = min(blockFirstIndex + blockSize, tripCount) - 1
-  Value blockEnd0 = b.create<arith::AddIOp>(blockFirstIndex, blockSize);
-  Value blockEnd1 = b.create<arith::MinSIOp>(blockEnd0, tripCount);
-  Value blockLastIndex = b.create<arith::SubIOp>(blockEnd1, c1);
+  Value blockEnd0 = arith::AddIOp::create(b, blockFirstIndex, blockSize);
+  Value blockEnd1 = arith::MinSIOp::create(b, blockEnd0, tripCount);
+  Value blockLastIndex = arith::SubIOp::create(b, blockEnd1, c1);
 
   // Convert one-dimensional indices to multi-dimensional coordinates.
   auto blockFirstCoord = delinearize(b, blockFirstIndex, tripCounts);
@@ -326,7 +326,7 @@ static ParallelComputeFunction createParallelComputeFunction(
   // dimension when inner compute dimension contains multiple blocks.
   SmallVector<Value> blockEndCoord(op.getNumLoops());
   for (size_t i = 0; i < blockLastCoord.size(); ++i)
-    blockEndCoord[i] = b.create<arith::AddIOp>(blockLastCoord[i], c1);
+    blockEndCoord[i] = arith::AddIOp::create(b, blockLastCoord[i], c1);
 
   // Construct a loop nest out of scf.for operations that will iterate over
   // all coordinates in [blockFirstCoord, blockLastCoord] range.
@@ -369,20 +369,20 @@ static ParallelComputeFunction createParallelComputeFunction(
       ImplicitLocOpBuilder b(loc, nestedBuilder);
 
       // Compute induction variable for `loopIdx`.
-      computeBlockInductionVars[loopIdx] = b.create<arith::AddIOp>(
-          lowerBounds[loopIdx], b.create<arith::MulIOp>(iv, steps[loopIdx]));
+      computeBlockInductionVars[loopIdx] = arith::AddIOp::create(b,
+          lowerBounds[loopIdx], arith::MulIOp::create(b, iv, steps[loopIdx]));
 
       // Check if we are inside first or last iteration of the loop.
-      isBlockFirstCoord[loopIdx] = b.create<arith::CmpIOp>(
+      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(b,
           arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
-      isBlockLastCoord[loopIdx] = b.create<arith::CmpIOp>(
+      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(b,
           arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
 
       // Check if the previous loop is in its first or last iteration.
       if (loopIdx > 0) {
-        isBlockFirstCoord[loopIdx] = b.create<arith::AndIOp>(
+        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(b,
             isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
-        isBlockLastCoord[loopIdx] = b.create<arith::AndIOp>(
+        isBlockLastCoord[loopIdx] = arith::AndIOp::create(b,
             isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
       }
 
@@ -391,24 +391,24 @@ static ParallelComputeFunction createParallelComputeFunction(
         if (loopIdx + 1 >= op.getNumLoops() - numBlockAlignedInnerLoops) {
           // For block aligned loops we always iterate starting from 0 up to
           // the loop trip counts.
-          b.create<scf::ForOp>(c0, tripCounts[loopIdx + 1], c1, ValueRange(),
+          scf::ForOp::create(b, c0, tripCounts[loopIdx + 1], c1, ValueRange(),
                                workLoopBuilder(loopIdx + 1));
 
         } else {
           // Select nested loop lower/upper bounds depending on our position in
           // the multi-dimensional iteration space.
-          auto lb = b.create<arith::SelectOp>(isBlockFirstCoord[loopIdx],
+          auto lb = arith::SelectOp::create(b, isBlockFirstCoord[loopIdx],
                                               blockFirstCoord[loopIdx + 1], c0);
 
-          auto ub = b.create<arith::SelectOp>(isBlockLastCoord[loopIdx],
+          auto ub = arith::SelectOp::create(b, isBlockLastCoord[loopIdx],
                                               blockEndCoord[loopIdx + 1],
                                               tripCounts[loopIdx + 1]);
 
-          b.create<scf::ForOp>(lb, ub, c1, ValueRange(),
+          scf::ForOp::create(b, lb, ub, c1, ValueRange(),
                                workLoopBuilder(loopIdx + 1));
         }
 
-        b.create<scf::YieldOp>(loc);
+        scf::YieldOp::create(b, loc);
         return;
       }
 
@@ -419,13 +419,13 @@ static ParallelComputeFunction createParallelComputeFunction(
 
       for (auto &bodyOp : op.getRegion().front().without_terminator())
         b.clone(bodyOp, mapping);
-      b.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(b, loc);
     };
   };
 
-  b.create<scf::ForOp>(blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
+  scf::ForOp::create(b, blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
                        workLoopBuilder(0));
-  b.create<func::ReturnOp>(ValueRange());
+  func::ReturnOp::create(b, ValueRange());
 
   return {op.getNumLoops(), func, std::move(computeFuncType.captures)};
 }
@@ -485,8 +485,8 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   b.setInsertionPointToEnd(block);
 
   Type indexTy = b.getIndexType();
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
-  Value c2 = b.create<arith::ConstantIndexOp>(2);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
+  Value c2 = arith::ConstantIndexOp::create(b, 2);
 
   // Get the async group that will track async dispatch completion.
   Value group = block->getArgument(0);
@@ -501,7 +501,7 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   SmallVector<Location> locations = {loc, loc};
 
   // Create a recursive dispatch loop.
-  scf::WhileOp whileOp = b.create<scf::WhileOp>(types, operands);
+  scf::WhileOp whileOp = scf::WhileOp::create(b, types, operands);
   Block *before = b.createBlock(&whileOp.getBefore(), {}, types, locations);
   Block *after = b.createBlock(&whileOp.getAfter(), {}, types, locations);
 
@@ -511,10 +511,10 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
     b.setInsertionPointToEnd(before);
     Value start = before->getArgument(0);
     Value end = before->getArgument(1);
-    Value distance = b.create<arith::SubIOp>(end, start);
+    Value distance = arith::SubIOp::create(b, end, start);
     Value dispatch =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, distance, c1);
-    b.create<scf::ConditionOp>(dispatch, before->getArguments());
+        arith::CmpIOp::create(b, arith::CmpIPredicate::sgt, distance, c1);
+    scf::ConditionOp::create(b, dispatch, before->getArguments());
   }
 
   // Setup the async dispatch loop body: recursively call dispatch function
@@ -523,9 +523,9 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
     b.setInsertionPointToEnd(after);
     Value start = after->getArgument(0);
     Value end = after->getArgument(1);
-    Value distance = b.create<arith::SubIOp>(end, start);
-    Value halfDistance = b.create<arith::DivSIOp>(distance, c2);
-    Value midIndex = b.create<arith::AddIOp>(start, halfDistance);
+    Value distance = arith::SubIOp::create(b, end, start);
+    Value halfDistance = arith::DivSIOp::create(b, distance, c2);
+    Value midIndex = arith::AddIOp::create(b, start, halfDistance);
 
     // Call parallel compute function inside the async.execute region.
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
@@ -536,16 +536,16 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
       operands[1] = midIndex;
       operands[2] = end;
 
-      executeBuilder.create<func::CallOp>(executeLoc, func.getSymName(),
+      func::CallOp::create(executeBuilder, executeLoc, func.getSymName(),
                                           func.getResultTypes(), operands);
-      executeBuilder.create<async::YieldOp>(executeLoc, ValueRange());
+      async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to dispatch half of the block range.
-    auto execute = b.create<ExecuteOp>(TypeRange(), ValueRange(), ValueRange(),
+    auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
                                        executeBodyBuilder);
-    b.create<AddToGroupOp>(indexTy, execute.getToken(), group);
-    b.create<scf::YieldOp>(ValueRange({start, midIndex}));
+    AddToGroupOp::create(b, indexTy, execute.getToken(), group);
+    scf::YieldOp::create(b, ValueRange({start, midIndex}));
   }
 
   // After dispatching async operations to process the tail of the block range
@@ -557,10 +557,10 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   SmallVector<Value> computeFuncOperands = {blockStart};
   computeFuncOperands.append(forwardedInputs.begin(), forwardedInputs.end());
 
-  b.create<func::CallOp>(computeFunc.func.getSymName(),
+  func::CallOp::create(b, computeFunc.func.getSymName(),
                          computeFunc.func.getResultTypes(),
                          computeFuncOperands);
-  b.create<func::ReturnOp>(ValueRange());
+  func::ReturnOp::create(b, ValueRange());
 
   return func;
 }
@@ -578,8 +578,8 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
   func::FuncOp asyncDispatchFunction =
       createAsyncDispatchFunction(parallelComputeFunction, rewriter);
 
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Appends operands shared by async dispatch and parallel compute functions to
   // the given operands vector.
@@ -595,7 +595,7 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
   // completely. If this will be known statically, then canonicalization will
   // erase async group operations.
   Value isSingleBlock =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, blockCount, c1);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, blockCount, c1);
 
   auto syncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
     ImplicitLocOpBuilder b(loc, nestedBuilder);
@@ -604,10 +604,10 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     SmallVector<Value> operands = {c0, blockSize};
     appendBlockComputeOperands(operands);
 
-    b.create<func::CallOp>(parallelComputeFunction.func.getSymName(),
+    func::CallOp::create(b, parallelComputeFunction.func.getSymName(),
                            parallelComputeFunction.func.getResultTypes(),
                            operands);
-    b.create<scf::YieldOp>();
+    scf::YieldOp::create(b, );
   };
 
   auto asyncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -616,24 +616,24 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     // Create an async.group to wait on all async tokens from the concurrent
     // execution of multiple parallel compute function. First block will be
     // executed synchronously in the caller thread.
-    Value groupSize = b.create<arith::SubIOp>(blockCount, c1);
-    Value group = b.create<CreateGroupOp>(GroupType::get(ctx), groupSize);
+    Value groupSize = arith::SubIOp::create(b, blockCount, c1);
+    Value group = CreateGroupOp::create(b, GroupType::get(ctx), groupSize);
 
     // Launch async dispatch function for [0, blockCount) range.
     SmallVector<Value> operands = {group, c0, blockCount, blockSize};
     appendBlockComputeOperands(operands);
 
-    b.create<func::CallOp>(asyncDispatchFunction.getSymName(),
+    func::CallOp::create(b, asyncDispatchFunction.getSymName(),
                            asyncDispatchFunction.getResultTypes(), operands);
 
     // Wait for the completion of all parallel compute operations.
-    b.create<AwaitAllOp>(group);
+    AwaitAllOp::create(b, group);
 
-    b.create<scf::YieldOp>();
+    scf::YieldOp::create(b, );
   };
 
   // Dispatch either single block compute function, or launch async dispatch.
-  b.create<scf::IfOp>(isSingleBlock, syncDispatch, asyncDispatch);
+  scf::IfOp::create(b, isSingleBlock, syncDispatch, asyncDispatch);
 }
 
 // Dispatch parallel compute functions by submitting all async compute tasks
@@ -647,14 +647,14 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
 
   func::FuncOp compute = parallelComputeFunction.func;
 
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Create an async.group to wait on all async tokens from the concurrent
   // execution of multiple parallel compute function. First block will be
   // executed synchronously in the caller thread.
-  Value groupSize = b.create<arith::SubIOp>(blockCount, c1);
-  Value group = b.create<CreateGroupOp>(GroupType::get(ctx), groupSize);
+  Value groupSize = arith::SubIOp::create(b, blockCount, c1);
+  Value group = CreateGroupOp::create(b, GroupType::get(ctx), groupSize);
 
   // Call parallel compute function for all blocks.
   using LoopBodyBuilder =
@@ -681,28 +681,28 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     // Call parallel compute function inside the async.execute region.
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
                                   Location executeLoc, ValueRange executeArgs) {
-      executeBuilder.create<func::CallOp>(executeLoc, compute.getSymName(),
+      func::CallOp::create(executeBuilder, executeLoc, compute.getSymName(),
                                           compute.getResultTypes(),
                                           computeFuncOperands(iv));
-      executeBuilder.create<async::YieldOp>(executeLoc, ValueRange());
+      async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to launch parallel computate function.
-    auto execute = b.create<ExecuteOp>(TypeRange(), ValueRange(), ValueRange(),
+    auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
                                        executeBodyBuilder);
-    b.create<AddToGroupOp>(rewriter.getIndexType(), execute.getToken(), group);
-    b.create<scf::YieldOp>();
+    AddToGroupOp::create(b, rewriter.getIndexType(), execute.getToken(), group);
+    scf::YieldOp::create(b, );
   };
 
   // Iterate over all compute blocks and launch parallel compute operations.
-  b.create<scf::ForOp>(c1, blockCount, c1, ValueRange(), loopBuilder);
+  scf::ForOp::create(b, c1, blockCount, c1, ValueRange(), loopBuilder);
 
   // Call parallel compute function for the first block in the caller thread.
-  b.create<func::CallOp>(compute.getSymName(), compute.getResultTypes(),
+  func::CallOp::create(b, compute.getSymName(), compute.getResultTypes(),
                          computeFuncOperands(c0));
 
   // Wait for the completion of all async compute operations.
-  b.create<AwaitAllOp>(group);
+  AwaitAllOp::create(b, group);
 }
 
 LogicalResult
@@ -738,17 +738,17 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
   // for the scf.parallel operation.
   Value tripCount = tripCounts[0];
   for (size_t i = 1; i < tripCounts.size(); ++i)
-    tripCount = b.create<arith::MulIOp>(tripCount, tripCounts[i]);
+    tripCount = arith::MulIOp::create(b, tripCount, tripCounts[i]);
 
   // Short circuit no-op parallel loops (zero iterations) that can arise from
   // the memrefs with dynamic dimension(s) equal to zero.
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
   Value isZeroIterations =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, tripCount, c0);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, tripCount, c0);
 
   // Do absolutely nothing if the trip count is zero.
   auto noOp = [&](OpBuilder &nestedBuilder, Location loc) {
-    nestedBuilder.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(nestedBuilder, loc);
   };
 
   // Compute the parallel block size and dispatch concurrent tasks computing
@@ -798,9 +798,9 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
     Value numWorkerThreadsVal;
     if (numWorkerThreads >= 0)
-      numWorkerThreadsVal = b.create<arith::ConstantIndexOp>(numWorkerThreads);
+      numWorkerThreadsVal = arith::ConstantIndexOp::create(b, numWorkerThreads);
     else
-      numWorkerThreadsVal = b.create<async::RuntimeNumWorkerThreadsOp>();
+      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b, );
 
     // With large number of threads the value of creating many compute blocks
     // is reduced because the problem typically becomes memory bound. For this
@@ -819,38 +819,38 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
         {4, 4.0f}, {8, 2.0f}, {16, 1.0f}, {32, 0.8f}, {64, 0.6f}};
     const float initialOvershardingFactor = 8.0f;
 
-    Value scalingFactor = b.create<arith::ConstantFloatOp>(
+    Value scalingFactor = arith::ConstantFloatOp::create(b,
         b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
     for (const std::pair<int, float> &p : overshardingBrackets) {
-      Value bracketBegin = b.create<arith::ConstantIndexOp>(p.first);
-      Value inBracket = b.create<arith::CmpIOp>(
+      Value bracketBegin = arith::ConstantIndexOp::create(b, p.first);
+      Value inBracket = arith::CmpIOp::create(b,
           arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
-      Value bracketScalingFactor = b.create<arith::ConstantFloatOp>(
+      Value bracketScalingFactor = arith::ConstantFloatOp::create(b,
           b.getF32Type(), llvm::APFloat(p.second));
-      scalingFactor = b.create<arith::SelectOp>(inBracket, bracketScalingFactor,
+      scalingFactor = arith::SelectOp::create(b, inBracket, bracketScalingFactor,
                                                 scalingFactor);
     }
     Value numWorkersIndex =
-        b.create<arith::IndexCastOp>(b.getI32Type(), numWorkerThreadsVal);
+        arith::IndexCastOp::create(b, b.getI32Type(), numWorkerThreadsVal);
     Value numWorkersFloat =
-        b.create<arith::SIToFPOp>(b.getF32Type(), numWorkersIndex);
+        arith::SIToFPOp::create(b, b.getF32Type(), numWorkersIndex);
     Value scaledNumWorkers =
-        b.create<arith::MulFOp>(scalingFactor, numWorkersFloat);
+        arith::MulFOp::create(b, scalingFactor, numWorkersFloat);
     Value scaledNumInt =
-        b.create<arith::FPToSIOp>(b.getI32Type(), scaledNumWorkers);
+        arith::FPToSIOp::create(b, b.getI32Type(), scaledNumWorkers);
     Value scaledWorkers =
-        b.create<arith::IndexCastOp>(b.getIndexType(), scaledNumInt);
+        arith::IndexCastOp::create(b, b.getIndexType(), scaledNumInt);
 
-    Value maxComputeBlocks = b.create<arith::MaxSIOp>(
-        b.create<arith::ConstantIndexOp>(1), scaledWorkers);
+    Value maxComputeBlocks = arith::MaxSIOp::create(b,
+        arith::ConstantIndexOp::create(b, 1), scaledWorkers);
 
     // Compute parallel block size from the parallel problem size:
     //   blockSize = min(tripCount,
     //                   max(ceil_div(tripCount, maxComputeBlocks),
     //                       minTaskSize))
-    Value bs0 = b.create<arith::CeilDivSIOp>(tripCount, maxComputeBlocks);
-    Value bs1 = b.create<arith::MaxSIOp>(bs0, minTaskSize);
-    Value blockSize = b.create<arith::MinSIOp>(tripCount, bs1);
+    Value bs0 = arith::CeilDivSIOp::create(b, tripCount, maxComputeBlocks);
+    Value bs1 = arith::MaxSIOp::create(b, bs0, minTaskSize);
+    Value blockSize = arith::MinSIOp::create(b, tripCount, bs1);
 
     // Dispatch parallel compute function using async recursive work splitting,
     // or by submitting compute task sequentially from a caller thread.
@@ -860,7 +860,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     // the parallel operation body for a subset of iteration space.
 
     // Compute the number of parallel compute blocks.
-    Value blockCount = b.create<arith::CeilDivSIOp>(tripCount, blockSize);
+    Value blockCount = arith::CeilDivSIOp::create(b, tripCount, blockSize);
 
     // Dispatch parallel compute function without hints to unroll inner loops.
     auto dispatchDefault = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -869,7 +869,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       doDispatch(b, rewriter, compute, op, blockSize, blockCount, tripCounts);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     };
 
     // Dispatch parallel compute function with hints for unrolling inner loops.
@@ -880,34 +880,34 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       // Align the block size to be a multiple of the statically known
       // number of iterations in the inner loops.
-      Value numIters = b.create<arith::ConstantIndexOp>(
+      Value numIters = arith::ConstantIndexOp::create(b,
           numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value alignedBlockSize = b.create<arith::MulIOp>(
-          b.create<arith::CeilDivSIOp>(blockSize, numIters), numIters);
+      Value alignedBlockSize = arith::MulIOp::create(b,
+          arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
       doDispatch(b, rewriter, compute, op, alignedBlockSize, blockCount,
                  tripCounts);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     };
 
     // Dispatch to block aligned compute function only if the computed block
     // size is larger than the number of iterations in the unrollable inner
     // loops, because otherwise it can reduce the available parallelism.
     if (numUnrollableLoops > 0) {
-      Value numIters = b.create<arith::ConstantIndexOp>(
+      Value numIters = arith::ConstantIndexOp::create(b,
           numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value useBlockAlignedComputeFn = b.create<arith::CmpIOp>(
+      Value useBlockAlignedComputeFn = arith::CmpIOp::create(b,
           arith::CmpIPredicate::sge, blockSize, numIters);
 
-      b.create<scf::IfOp>(useBlockAlignedComputeFn, dispatchBlockAligned,
+      scf::IfOp::create(b, useBlockAlignedComputeFn, dispatchBlockAligned,
                           dispatchDefault);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     } else {
       dispatchDefault(b, loc);
     }
   };
 
   // Replace the `scf.parallel` operation with the parallel compute function.
-  b.create<scf::IfOp>(isZeroIterations, noOp, dispatch);
+  scf::IfOp::create(b, isZeroIterations, noOp, dispatch);
 
   // Parallel operation was replaced with a block iteration loop.
   rewriter.eraseOp(op);
@@ -922,7 +922,7 @@ void AsyncParallelForPass::runOnOperation() {
   populateAsyncParallelForPatterns(
       patterns, asyncDispatch, numWorkerThreads,
       [&](ImplicitLocOpBuilder builder, scf::ParallelOp op) {
-        return builder.create<arith::ConstantIndexOp>(minTaskSize);
+        return arith::ConstantIndexOp::create(builder, minTaskSize);
       });
   if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
     signalPassFailure();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
index d11cd8444636a..6bcc5849eff1b 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
@@ -51,7 +51,7 @@ static LogicalResult dropRefIfNoUses(Value value, unsigned count = 1) {
   else
     b.setInsertionPointToStart(value.getParentBlock());
 
-  b.create<RuntimeDropRefOp>(value.getLoc(), value, b.getI64IntegerAttr(1));
+  RuntimeDropRefOp::create(b, value.getLoc(), value, b.getI64IntegerAttr(1));
   return success();
 }
 
@@ -312,7 +312,7 @@ LogicalResult AsyncRuntimeRefCountingPass::addDropRefAfterLastUse(Value value) {
 
     // Add a drop_ref immediately after the last user.
     builder.setInsertionPointAfter(lastUser);
-    builder.create<RuntimeDropRefOp>(loc, value, builder.getI64IntegerAttr(1));
+    RuntimeDropRefOp::create(builder, loc, value, builder.getI64IntegerAttr(1));
   }
 
   return success();
@@ -330,7 +330,7 @@ AsyncRuntimeRefCountingPass::addAddRefBeforeFunctionCall(Value value) {
     // Add a reference before the function call to pass the value at `+1`
     // reference to the function entry block.
     builder.setInsertionPoint(user);
-    builder.create<RuntimeAddRefOp>(loc, value, builder.getI64IntegerAttr(1));
+    RuntimeAddRefOp::create(builder, loc, value, builder.getI64IntegerAttr(1));
   }
 
   return success();
@@ -414,11 +414,11 @@ AsyncRuntimeRefCountingPass::addDropRefInDivergentLivenessSuccessor(
         refCountingBlock = &successor->getParent()->emplaceBlock();
         refCountingBlock->moveBefore(successor);
         OpBuilder builder = OpBuilder::atBlockEnd(refCountingBlock);
-        builder.create<cf::BranchOp>(value.getLoc(), successor);
+        cf::BranchOp::create(builder, value.getLoc(), successor);
       }
 
       OpBuilder builder = OpBuilder::atBlockBegin(refCountingBlock);
-      builder.create<RuntimeDropRefOp>(value.getLoc(), value,
+      RuntimeDropRefOp::create(builder, value.getLoc(), value,
                                        builder.getI64IntegerAttr(1));
 
       // No need to update the terminator operation.
@@ -510,13 +510,13 @@ AsyncRuntimePolicyBasedRefCountingPass::addRefCounting(Value value) {
       // Create `add_ref` operation before the operand owner.
       if (cnt > 0) {
         b.setInsertionPoint(operand.getOwner());
-        b.create<RuntimeAddRefOp>(loc, value, b.getI64IntegerAttr(cnt));
+        RuntimeAddRefOp::create(b, loc, value, b.getI64IntegerAttr(cnt));
       }
 
       // Create `drop_ref` operation after the operand owner.
       if (cnt < 0) {
         b.setInsertionPointAfter(operand.getOwner());
-        b.create<RuntimeDropRefOp>(loc, value, b.getI64IntegerAttr(-cnt));
+        RuntimeDropRefOp::create(b, loc, value, b.getI64IntegerAttr(-cnt));
       }
     }
   }
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index 8601bb5aaada9..8b9feb8214923 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -188,22 +188,22 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
 
   std::optional<Value> retToken;
   if (isStateful)
-    retToken.emplace(builder.create<RuntimeCreateOp>(TokenType::get(ctx)));
+    retToken.emplace(RuntimeCreateOp::create(builder, TokenType::get(ctx)));
 
   llvm::SmallVector<Value, 4> retValues;
   ArrayRef<Type> resValueTypes =
       isStateful ? func.getResultTypes().drop_front() : func.getResultTypes();
   for (auto resType : resValueTypes)
     retValues.emplace_back(
-        builder.create<RuntimeCreateOp>(resType).getResult());
+        RuntimeCreateOp::create(builder, resType).getResult());
 
   // ------------------------------------------------------------------------ //
   // Initialize coroutine: get coroutine id and coroutine handle.
   // ------------------------------------------------------------------------ //
-  auto coroIdOp = builder.create<CoroIdOp>(CoroIdType::get(ctx));
+  auto coroIdOp = CoroIdOp::create(builder, CoroIdType::get(ctx));
   auto coroHdlOp =
-      builder.create<CoroBeginOp>(CoroHandleType::get(ctx), coroIdOp.getId());
-  builder.create<cf::BranchOp>(originalEntryBlock);
+      CoroBeginOp::create(builder, CoroHandleType::get(ctx), coroIdOp.getId());
+  cf::BranchOp::create(builder, originalEntryBlock);
 
   Block *cleanupBlock = func.addBlock();
   Block *cleanupBlockForDestroy = func.addBlock();
@@ -214,10 +214,10 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   // ------------------------------------------------------------------------ //
   auto buildCleanupBlock = [&](Block *cb) {
     builder.setInsertionPointToStart(cb);
-    builder.create<CoroFreeOp>(coroIdOp.getId(), coroHdlOp.getHandle());
+    CoroFreeOp::create(builder, coroIdOp.getId(), coroHdlOp.getHandle());
 
     // Branch into the suspend block.
-    builder.create<cf::BranchOp>(suspendBlock);
+    cf::BranchOp::create(builder, suspendBlock);
   };
   buildCleanupBlock(cleanupBlock);
   buildCleanupBlock(cleanupBlockForDestroy);
@@ -229,7 +229,7 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   builder.setInsertionPointToStart(suspendBlock);
 
   // Mark the end of a coroutine: async.coro.end
-  builder.create<CoroEndOp>(coroHdlOp.getHandle());
+  CoroEndOp::create(builder, coroHdlOp.getHandle());
 
   // Return created optional `async.token` and `async.values` from the suspend
   // block. This will be the return value of a coroutine ramp function.
@@ -237,7 +237,7 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   if (retToken)
     ret.push_back(*retToken);
   llvm::append_range(ret, retValues);
-  builder.create<func::ReturnOp>(ret);
+  func::ReturnOp::create(builder, ret);
 
   // `async.await` op lowering will create resume blocks for async
   // continuations, and will conditionally branch to cleanup or suspend blocks.
@@ -274,13 +274,13 @@ static Block *setupSetErrorBlock(CoroMachinery &coro) {
 
   // Coroutine set_error block: set error on token and all returned values.
   if (coro.asyncToken)
-    builder.create<RuntimeSetErrorOp>(*coro.asyncToken);
+    RuntimeSetErrorOp::create(builder, *coro.asyncToken);
 
   for (Value retValue : coro.returnValues)
-    builder.create<RuntimeSetErrorOp>(retValue);
+    RuntimeSetErrorOp::create(builder, retValue);
 
   // Branch into the cleanup block.
-  builder.create<cf::BranchOp>(coro.cleanup);
+  cf::BranchOp::create(builder, coro.cleanup);
 
   return *coro.setError;
 }
@@ -335,13 +335,13 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Await on all dependencies before starting to execute the body region.
     for (size_t i = 0; i < numDependencies; ++i)
-      builder.create<AwaitOp>(func.getArgument(i));
+      AwaitOp::create(builder, func.getArgument(i));
 
     // Await on all async value operands and unwrap the payload.
     SmallVector<Value, 4> unwrappedOperands(numOperands);
     for (size_t i = 0; i < numOperands; ++i) {
       Value operand = func.getArgument(numDependencies + i);
-      unwrappedOperands[i] = builder.create<AwaitOp>(loc, operand).getResult();
+      unwrappedOperands[i] = AwaitOp::create(builder, loc, operand).getResult();
     }
 
     // Map from function inputs defined above the execute op to the function
@@ -368,14 +368,14 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Save the coroutine state: async.coro.save
     auto coroSaveOp =
-        builder.create<CoroSaveOp>(CoroStateType::get(ctx), coro.coroHandle);
+        CoroSaveOp::create(builder, CoroStateType::get(ctx), coro.coroHandle);
 
     // Pass coroutine to the runtime to be resumed on a runtime managed
     // thread.
-    builder.create<RuntimeResumeOp>(coro.coroHandle);
+    RuntimeResumeOp::create(builder, coro.coroHandle);
 
     // Add async.coro.suspend as a suspended block terminator.
-    builder.create<CoroSuspendOp>(coroSaveOp.getState(), coro.suspend,
+    CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend,
                                   branch.getDest(), coro.cleanupForDestroy);
 
     branch.erase();
@@ -384,7 +384,7 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
   // Replace the original `async.execute` with a call to outlined function.
   {
     ImplicitLocOpBuilder callBuilder(loc, execute);
-    auto callOutlinedFunc = callBuilder.create<func::CallOp>(
+    auto callOutlinedFunc = func::CallOp::create(callBuilder,
         func.getName(), execute.getResultTypes(), functionInputs.getArrayRef());
     execute.replaceAllUsesWith(callOutlinedFunc.getResults());
     execute.erase();
@@ -453,7 +453,7 @@ class AsyncFuncOpLowering : public OpConversionPattern<async::FuncOp> {
     Location loc = op->getLoc();
 
     auto newFuncOp =
-        rewriter.create<func::FuncOp>(loc, op.getName(), op.getFunctionType());
+        func::FuncOp::create(rewriter, loc, op.getName(), op.getFunctionType());
 
     SymbolTable::setSymbolVisibility(newFuncOp,
                                      SymbolTable::getSymbolVisibility(op));
@@ -523,16 +523,16 @@ class AsyncReturnOpLowering : public OpConversionPattern<async::ReturnOp> {
     for (auto tuple : llvm::zip(adaptor.getOperands(), coro.returnValues)) {
       Value returnValue = std::get<0>(tuple);
       Value asyncValue = std::get<1>(tuple);
-      rewriter.create<RuntimeStoreOp>(loc, returnValue, asyncValue);
-      rewriter.create<RuntimeSetAvailableOp>(loc, asyncValue);
+      RuntimeStoreOp::create(rewriter, loc, returnValue, asyncValue);
+      RuntimeSetAvailableOp::create(rewriter, loc, asyncValue);
     }
 
     if (coro.asyncToken)
       // Switch the coroutine completion token to available state.
-      rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
+      RuntimeSetAvailableOp::create(rewriter, loc, *coro.asyncToken);
 
     rewriter.eraseOp(op);
-    rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    cf::BranchOp::create(rewriter, loc, coro.cleanup);
     return success();
   }
 
@@ -583,15 +583,15 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
     // the async object (token, value or group) to become available.
     if (!isInCoroutine) {
       ImplicitLocOpBuilder builder(loc, rewriter);
-      builder.create<RuntimeAwaitOp>(loc, operand);
+      RuntimeAwaitOp::create(builder, loc, operand);
 
       // Assert that the awaited operands is not in the error state.
-      Value isError = builder.create<RuntimeIsErrorOp>(i1, operand);
-      Value notError = builder.create<arith::XOrIOp>(
-          isError, builder.create<arith::ConstantOp>(
+      Value isError = RuntimeIsErrorOp::create(builder, i1, operand);
+      Value notError = arith::XOrIOp::create(builder,
+          isError, arith::ConstantOp::create(builder,
                        loc, i1, builder.getIntegerAttr(i1, 1)));
 
-      builder.create<cf::AssertOp>(notError,
+      cf::AssertOp::create(builder, notError,
                                    "Awaited async operand is in error state");
     }
 
@@ -607,15 +607,15 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
       // Save the coroutine state and resume on a runtime managed thread when
       // the operand becomes available.
       auto coroSaveOp =
-          builder.create<CoroSaveOp>(CoroStateType::get(ctx), coro.coroHandle);
-      builder.create<RuntimeAwaitAndResumeOp>(operand, coro.coroHandle);
+          CoroSaveOp::create(builder, CoroStateType::get(ctx), coro.coroHandle);
+      RuntimeAwaitAndResumeOp::create(builder, operand, coro.coroHandle);
 
       // Split the entry block before the await operation.
       Block *resume = rewriter.splitBlock(suspended, Block::iterator(op));
 
       // Add async.coro.suspend as a suspended block terminator.
       builder.setInsertionPointToEnd(suspended);
-      builder.create<CoroSuspendOp>(coroSaveOp.getState(), coro.suspend, resume,
+      CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend, resume,
                                     coro.cleanupForDestroy);
 
       // Split the resume block into error checking and continuation.
@@ -623,8 +623,8 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
 
       // Check if the awaited value is in the error state.
       builder.setInsertionPointToStart(resume);
-      auto isError = builder.create<RuntimeIsErrorOp>(loc, i1, operand);
-      builder.create<cf::CondBranchOp>(isError,
+      auto isError = RuntimeIsErrorOp::create(builder, loc, i1, operand);
+      cf::CondBranchOp::create(builder, isError,
                                        /*trueDest=*/setupSetErrorBlock(coro),
                                        /*trueArgs=*/ArrayRef<Value>(),
                                        /*falseDest=*/continuation,
@@ -674,7 +674,7 @@ class AwaitValueOpLowering : public AwaitOpLoweringBase<AwaitOp, ValueType> {
                       ConversionPatternRewriter &rewriter) const override {
     // Load from the async value storage.
     auto valueType = cast<ValueType>(operand.getType()).getValueType();
-    return rewriter.create<RuntimeLoadOp>(op->getLoc(), valueType, operand);
+    return RuntimeLoadOp::create(rewriter, op->getLoc(), valueType, operand);
   }
 };
 
@@ -715,15 +715,15 @@ class YieldOpLowering : public OpConversionPattern<async::YieldOp> {
     for (auto tuple : llvm::zip(adaptor.getOperands(), coro.returnValues)) {
       Value yieldValue = std::get<0>(tuple);
       Value asyncValue = std::get<1>(tuple);
-      rewriter.create<RuntimeStoreOp>(loc, yieldValue, asyncValue);
-      rewriter.create<RuntimeSetAvailableOp>(loc, asyncValue);
+      RuntimeStoreOp::create(rewriter, loc, yieldValue, asyncValue);
+      RuntimeSetAvailableOp::create(rewriter, loc, asyncValue);
     }
 
     if (coro.asyncToken)
       // Switch the coroutine completion token to available state.
-      rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
+      RuntimeSetAvailableOp::create(rewriter, loc, *coro.asyncToken);
 
-    rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    cf::BranchOp::create(rewriter, loc, coro.cleanup);
     rewriter.eraseOp(op);
 
     return success();
@@ -757,7 +757,7 @@ class AssertOpLowering : public OpConversionPattern<cf::AssertOp> {
 
     Block *cont = rewriter.splitBlock(op->getBlock(), Block::iterator(op));
     rewriter.setInsertionPointToEnd(cont->getPrevNode());
-    rewriter.create<cf::CondBranchOp>(loc, adaptor.getArg(),
+    cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(),
                                       /*trueDest=*/cont,
                                       /*trueArgs=*/ArrayRef<Value>(),
                                       /*falseDest=*/setupSetErrorBlock(coro),
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index ca914df8b7890..59ebdbfdc727d 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -36,7 +36,7 @@ using namespace bufferization;
 //===----------------------------------------------------------------------===//
 
 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
-  return builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
 }
 
 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
@@ -151,7 +151,7 @@ DeallocationState::getMemrefWithUniqueOwnership(OpBuilder &builder,
   // ownerships more intelligently to not end up with an 'Unknown' ownership in
   // the first place.
   auto cloneOp =
-      builder.create<bufferization::CloneOp>(memref.getLoc(), memref);
+      bufferization::CloneOp::create(builder, memref.getLoc(), memref);
   Value condition = buildBoolValue(builder, memref.getLoc(), true);
   Value newMemref = cloneOp.getResult();
   updateOwnership(newMemref, condition);
@@ -197,7 +197,7 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // Simply cast unranked MemRefs to ranked memrefs with 0 dimensions such
     // that we can call extract_strided_metadata on it.
     if (auto unrankedMemRefTy = dyn_cast<UnrankedMemRefType>(memref.getType()))
-      memref = builder.create<memref::ReinterpretCastOp>(
+      memref = memref::ReinterpretCastOp::create(builder,
           loc, memref,
           /*offset=*/builder.getIndexAttr(0),
           /*sizes=*/ArrayRef<OpFoldResult>{},
@@ -208,7 +208,7 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // alloc operation has to be passed to the dealloc operation. Passing
     // subviews, etc. to a dealloc operation is not allowed.
     memrefs.push_back(
-        builder.create<memref::ExtractStridedMetadataOp>(loc, memref)
+        memref::ExtractStridedMetadataOp::create(builder, loc, memref)
             .getResult(0));
     conditions.push_back(ownership.getIndicator());
   }
@@ -297,7 +297,7 @@ FailureOr<Operation *> deallocation_impl::insertDeallocOpForReturnLike(
   if (memrefs.empty() && toRetain.empty())
     return op;
 
-  auto deallocOp = builder.create<bufferization::DeallocOp>(
+  auto deallocOp = bufferization::DeallocOp::create(builder,
       op->getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 8f17a82fabe03..05a1367a6b126 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -170,7 +170,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   if (llvm::isa<RankedTensorType>(shapedValue.getType())) {
     tensor = shapedValue;
   } else if (llvm::isa<MemRefType>(shapedValue.getType())) {
-    tensor = b.create<ToTensorOp>(
+    tensor = ToTensorOp::create(b,
         loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
         shapedValue);
   } else if (llvm::isa<UnrankedTensorType>(shapedValue.getType()) ||
@@ -209,7 +209,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   }
 
   // Create AllocTensorOp.
-  auto allocTensorOp = b.create<AllocTensorOp>(loc, tensorType, dynamicSizes,
+  auto allocTensorOp = AllocTensorOp::create(b, loc, tensorType, dynamicSizes,
                                                copy ? tensor : Value());
 
   // Add 'memory_space' attribute. Not needed if 'copy' operand is specified.
@@ -753,7 +753,7 @@ void bufferization::replaceOpWithBufferizedValues(RewriterBase &rewriter,
       // ToTensorOp. Throughout bufferization, this ToTensorOp will gradually
       // loose all of its users and eventually DCE away.
       rewriter.setInsertionPointAfter(op);
-      replacement = rewriter.create<bufferization::ToTensorOp>(
+      replacement = bufferization::ToTensorOp::create(rewriter,
           replacement.getLoc(), opResult.getType(), replacement);
     }
     replacements.push_back(replacement);
@@ -779,7 +779,7 @@ FailureOr<Value> BufferizationOptions::createAlloc(OpBuilder &b, Location loc,
         .create<memref::AllocOp>(loc, type, dynShape,
                                  b.getI64IntegerAttr(bufferAlignment))
         .getResult();
-  return b.create<memref::AllocOp>(loc, type, dynShape).getResult();
+  return memref::AllocOp::create(b, loc, type, dynShape).getResult();
 }
 
 /// Create a memory copy between two memref buffers.
@@ -788,7 +788,7 @@ LogicalResult BufferizationOptions::createMemCpy(OpBuilder &b, Location loc,
   if (memCpyFn)
     return (*memCpyFn)(b, loc, from, to);
 
-  b.create<memref::CopyOp>(loc, from, to);
+  memref::CopyOp::create(b, loc, from, to);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 5c1d42db18c47..4308f46bee6ff 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -59,7 +59,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
   // a fix extra conditions in `isGuaranteedCastCompatible`.
   if (memref::CastOp::areCastCompatible(srcType, destType) &&
       isGuaranteedCastCompatible(srcType, destType)) {
-    Value casted = b.create<memref::CastOp>(value.getLoc(), destType, value);
+    Value casted = memref::CastOp::create(b, value.getLoc(), destType, value);
     return casted;
   }
 
@@ -68,7 +68,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
   for (int i = 0; i < destType.getRank(); ++i) {
     if (destType.getShape()[i] != ShapedType::kDynamic)
       continue;
-    Value size = b.create<memref::DimOp>(loc, value, i);
+    Value size = memref::DimOp::create(b, loc, value, i);
     dynamicOperands.push_back(size);
   }
 
@@ -135,10 +135,10 @@ void mlir::bufferization::populateDynamicDimSizes(
   for (int64_t i = 0; i < shapedType.getRank(); ++i) {
     if (shapedType.isDynamicDim(i)) {
       if (llvm::isa<MemRefType>(shapedType)) {
-        dynamicDims.push_back(b.create<memref::DimOp>(loc, shapedValue, i));
+        dynamicDims.push_back(memref::DimOp::create(b, loc, shapedValue, i));
       } else {
         assert(llvm::isa<RankedTensorType>(shapedType) && "expected tensor");
-        dynamicDims.push_back(b.create<tensor::DimOp>(loc, shapedValue, i));
+        dynamicDims.push_back(tensor::DimOp::create(b, loc, shapedValue, i));
       }
     }
   }
@@ -322,7 +322,7 @@ struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
         newShape, op.getType().getElementType(), op.getType().getEncoding());
     if (newType == op.getType())
       return failure();
-    auto newOp = rewriter.create<AllocTensorOp>(
+    auto newOp = AllocTensorOp::create(rewriter,
         op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
@@ -428,7 +428,7 @@ void AllocTensorOp::print(OpAsmPrinter &p) {
 Value AllocTensorOp::getDynamicSize(OpBuilder &b, unsigned idx) {
   assert(isDynamicDim(idx) && "expected dynamic dim");
   if (getCopy())
-    return b.create<tensor::DimOp>(getLoc(), getCopy(), idx);
+    return tensor::DimOp::create(b, getLoc(), getCopy(), idx);
   return getOperand(getIndexOfDynamicSize(idx));
 }
 
@@ -514,7 +514,7 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
     }
 
     if (source.getType() != cloneOp.getType())
-      source = rewriter.create<memref::CastOp>(cloneOp.getLoc(),
+      source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
                                                cloneOp.getType(), source);
     rewriter.replaceOp(cloneOp, source);
     rewriter.eraseOp(redundantDealloc);
@@ -539,7 +539,7 @@ LogicalResult DeallocTensorOp::bufferize(RewriterBase &rewriter,
   FailureOr<Value> buffer = getBuffer(rewriter, getTensor(), options, state);
   if (failed(buffer))
     return failure();
-  rewriter.create<memref::DeallocOp>(getLoc(), *buffer);
+  memref::DeallocOp::create(rewriter, getLoc(), *buffer);
   rewriter.eraseOp(getOperation());
   return success();
 }
@@ -644,7 +644,7 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return builder.create<ToTensorOp>(
+  return ToTensorOp::create(builder,
       loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
       /*restrict=*/true, getWritable());
 }
@@ -807,7 +807,7 @@ struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
       return failure();
     auto memrefType = MemRefType::get(srcTensorType.getShape(),
                                       srcTensorType.getElementType());
-    Value memref = rewriter.create<ToBufferOp>(toBuffer.getLoc(), memrefType,
+    Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
                                                tensorCastOperand.getOperand());
     rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                 memref);
@@ -881,12 +881,12 @@ LogicalResult ToBufferOp::bufferize(RewriterBase &rewriter,
 
 std::optional<Operation *> CloneOp::buildDealloc(OpBuilder &builder,
                                                  Value alloc) {
-  return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
+  return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
       .getOperation();
 }
 
 std::optional<Value> CloneOp::buildClone(OpBuilder &builder, Value alloc) {
-  return builder.create<CloneOp>(alloc.getLoc(), alloc).getResult();
+  return CloneOp::create(builder, alloc.getLoc(), alloc).getResult();
 }
 
 //===----------------------------------------------------------------------===//
@@ -960,7 +960,7 @@ struct DeallocRemoveDuplicateDeallocMemrefs
         Value &newCond = newConditions[memrefToCondition[memref]];
         if (newCond != cond)
           newCond =
-              rewriter.create<arith::OrIOp>(deallocOp.getLoc(), newCond, cond);
+              arith::OrIOp::create(rewriter, deallocOp.getLoc(), newCond, cond);
       } else {
         memrefToCondition.insert({memref, newConditions.size()});
         newMemrefs.push_back(memref);
@@ -1015,7 +1015,7 @@ struct DeallocRemoveDuplicateRetainedMemrefs
     // We need to create a new op because the number of results is always the
     // same as the number of condition operands.
     auto newDeallocOp =
-        rewriter.create<DeallocOp>(deallocOp.getLoc(), deallocOp.getMemrefs(),
+        DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
                                    deallocOp.getConditions(), newRetained);
     SmallVector<Value> replacements(
         llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
@@ -1037,7 +1037,7 @@ struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     if (deallocOp.getMemrefs().empty()) {
-      Value constFalse = rewriter.create<arith::ConstantOp>(
+      Value constFalse = arith::ConstantOp::create(rewriter,
           deallocOp.getLoc(), rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
diff --git a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
index db1eb20512033..7f495b0ac164c 100644
--- a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
@@ -70,12 +70,12 @@ transform::OneShotBufferizeOp::apply(transform::TransformRewriter &rewriter,
         *getFunctionBoundaryTypeConversion());
   if (getMemcpyOp() == "memref.copy") {
     options.memCpyFn = [](OpBuilder &b, Location loc, Value from, Value to) {
-      b.create<memref::CopyOp>(loc, from, to);
+      memref::CopyOp::create(b, loc, from, to);
       return success();
     };
   } else if (getMemcpyOp() == "linalg.copy") {
     options.memCpyFn = [](OpBuilder &b, Location loc, Value from, Value to) {
-      b.create<linalg::CopyOp>(loc, from, to);
+      linalg::CopyOp::create(b, loc, from, to);
       return success();
     };
   } else {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index c5fab80ecaa08..835ad61f54da2 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -167,7 +167,7 @@ struct RemoveDeallocMemrefsContainedInRetained
       std::optional<bool> analysisResult =
           analysis.isSameAllocation(retained, memref);
       if (analysisResult == true) {
-        auto disjunction = rewriter.create<arith::OrIOp>(
+        auto disjunction = arith::OrIOp::create(rewriter,
             deallocOp.getLoc(), updatedCondition, cond);
         rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(),
                                       disjunction);
@@ -247,14 +247,14 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias
         continue;
       }
 
-      replacements.push_back(rewriter.create<arith::ConstantOp>(
+      replacements.push_back(arith::ConstantOp::create(rewriter,
           deallocOp.getLoc(), rewriter.getBoolAttr(false)));
     }
 
     if (newRetainedMemrefs.size() == deallocOp.getRetained().size())
       return failure();
 
-    auto newDeallocOp = rewriter.create<DeallocOp>(
+    auto newDeallocOp = DeallocOp::create(rewriter,
         deallocOp.getLoc(), deallocOp.getMemrefs(), deallocOp.getConditions(),
         newRetainedMemrefs);
     int i = 0;
@@ -326,7 +326,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       }
 
       // Create new bufferization.dealloc op for `memref`.
-      auto newDeallocOp = rewriter.create<DeallocOp>(loc, memref, cond,
+      auto newDeallocOp = DeallocOp::create(rewriter, loc, memref, cond,
                                                      deallocOp.getRetained());
       updatedConditions.push_back(
           llvm::to_vector(ValueRange(newDeallocOp.getUpdatedConditions())));
@@ -337,7 +337,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       return failure();
 
     // Create bufferization.dealloc op for all remaining memrefs.
-    auto newDeallocOp = rewriter.create<DeallocOp>(
+    auto newDeallocOp = DeallocOp::create(rewriter,
         loc, remainingMemrefs, remainingConditions, deallocOp.getRetained());
 
     // Bit-or all conditions.
@@ -347,7 +347,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       assert(replacements.size() == additionalConditions.size() &&
              "expected same number of updated conditions");
       for (int64_t i = 0, e = replacements.size(); i < e; ++i) {
-        replacements[i] = rewriter.create<arith::OrIOp>(
+        replacements[i] = arith::OrIOp::create(rewriter,
             loc, replacements[i], additionalConditions[i]);
       }
     }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index 15e03fbefe9c5..9f11f23a5dfeb 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -133,7 +133,7 @@ updateReturnOps(func::FuncOp func, ArrayRef<BlockArgument> appendedEntryArgs,
           return WalkResult::interrupt();
       }
     }
-    builder.create<func::ReturnOp>(op.getLoc(), keepAsReturnOperands);
+    func::ReturnOp::create(builder, op.getLoc(), keepAsReturnOperands);
     op.erase();
     return WalkResult::advance();
   });
@@ -191,7 +191,7 @@ updateCalls(ModuleOp module,
         assert(hasFullyDynamicLayoutMap(memrefType) &&
                "layout map not supported");
         outParam =
-            builder.create<memref::CastOp>(op.getLoc(), memrefType, outParam);
+            memref::CastOp::create(builder, op.getLoc(), memrefType, outParam);
       }
       memref.replaceAllUsesWith(outParam);
       outParams.push_back(outParam);
@@ -201,7 +201,7 @@ updateCalls(ModuleOp module,
     newOperands.append(outParams.begin(), outParams.end());
     auto newResultTypes = llvm::to_vector<6>(llvm::map_range(
         replaceWithNewCallResults, [](Value v) { return v.getType(); }));
-    auto newCall = builder.create<func::CallOp>(op.getLoc(), op.getCalleeAttr(),
+    auto newCall = func::CallOp::create(builder, op.getLoc(), op.getCalleeAttr(),
                                                 newResultTypes, newOperands);
     for (auto t : llvm::zip(replaceWithNewCallResults, newCall.getResults()))
       std::get<0>(t).replaceAllUsesWith(std::get<1>(t));
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
index ff2c83d228dbb..1e02d9bfa7ef6 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
@@ -146,7 +146,7 @@ bufferization::getGlobalFor(arith::ConstantOp constantOp,
       cast<MemRefType>(getMemRefTypeWithStaticIdentityLayout(type));
   if (memorySpace)
     memrefType = MemRefType::Builder(memrefType).setMemorySpace(memorySpace);
-  auto global = globalBuilder.create<memref::GlobalOp>(
+  auto global = memref::GlobalOp::create(globalBuilder,
       constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
       /*sym_visibility=*/globalBuilder.getStringAttr("private"),
       /*type=*/memrefType,
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 6472ef3eff2ac..c7343c62e2cc8 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -436,7 +436,7 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
     // Replace all uses of the original tensor bbArg.
     rewriter.setInsertionPointToStart(block);
     if (!bbArgUses.empty()) {
-      Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+      Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
           bbArg.getLoc(), tensorType, bbArg);
       for (OpOperand *use : bbArgUses)
         use->set(toTensorOp);
@@ -468,12 +468,12 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
       if (failed(operandBufferType))
         return failure();
       rewriter.setInsertionPointAfterValue(operand);
-      Value bufferizedOperand = rewriter.create<bufferization::ToBufferOp>(
+      Value bufferizedOperand = bufferization::ToBufferOp::create(rewriter,
           operand.getLoc(), *operandBufferType, operand);
       // A cast is needed if the operand and the block argument have different
       // bufferized types.
       if (type != *operandBufferType)
-        bufferizedOperand = rewriter.create<memref::CastOp>(
+        bufferizedOperand = memref::CastOp::create(rewriter,
             operand.getLoc(), type, bufferizedOperand);
       newOperands.push_back(bufferizedOperand);
     }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index ae011904cb972..9014b8b8ecdf0 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -120,7 +120,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
     // Update function calls.
     for (func::CallOp callOp : callerMap[funcOp]) {
       rewriter.setInsertionPoint(callOp);
-      auto newCallOp = rewriter.create<func::CallOp>(callOp.getLoc(), funcOp,
+      auto newCallOp = func::CallOp::create(rewriter, callOp.getLoc(), funcOp,
                                                      callOp.getOperands());
       SmallVector<Value> newResults;
       int64_t nextResult = 0;
@@ -136,7 +136,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
         Type expectedType = callOp.getResult(i).getType();
         if (replacement.getType() != expectedType) {
           // A cast must be inserted at the call site.
-          replacement = rewriter.create<memref::CastOp>(
+          replacement = memref::CastOp::create(rewriter,
               callOp.getLoc(), expectedType, replacement);
         }
         newResults.push_back(replacement);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
index 6f27563a45548..202b9eb0c69dc 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
@@ -169,7 +169,7 @@ LogicalResult mlir::bufferization::eliminateEmptyTensors(
             cast<ShapedType>(v.getType()).getElementType())
           continue;
         rewriter.setInsertionPointAfterValue(replacement);
-        replacement = rewriter.create<tensor::CastOp>(v.getLoc(), v.getType(),
+        replacement = tensor::CastOp::create(rewriter, v.getLoc(), v.getType(),
                                                       replacement);
       }
       // Replace the specific use of the tensor::EmptyOp.
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index bd2aebca68079..4d25ca87da888 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -320,7 +320,7 @@ struct CallOpInterface
     }
 
     // 3. Create the new CallOp.
-    Operation *newCallOp = rewriter.create<func::CallOp>(
+    Operation *newCallOp = func::CallOp::create(rewriter,
         callOp.getLoc(), funcOp.getSymName(), resultTypes, newOperands);
     newCallOp->setAttrs(callOp->getAttrs());
 
@@ -484,7 +484,7 @@ struct FuncOpInterface
 
         // Note: If `inferFunctionResultLayout = true`, casts are later folded
         // away.
-        Value toBufferOp = rewriter.create<bufferization::ToBufferOp>(
+        Value toBufferOp = bufferization::ToBufferOp::create(rewriter,
             returnOp.getLoc(), bufferizedType, returnVal);
         returnValues.push_back(toBufferOp);
       }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
index 2a17ae4f6a249..304c073a26c73 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
@@ -65,8 +65,8 @@ class DeallocOpConversion
 
     rewriter.replaceOpWithNewOp<scf::IfOp>(
         op, adaptor.getConditions()[0], [&](OpBuilder &builder, Location loc) {
-          builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[0]);
-          builder.create<scf::YieldOp>(loc);
+          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
+          scf::YieldOp::create(builder, loc);
         });
     return success();
   }
@@ -109,13 +109,13 @@ class DeallocOpConversion
     // Compute the base pointer indices, compare all retained indices to the
     // memref index to check if they alias.
     SmallVector<Value> doesNotAliasList;
-    Value memrefAsIdx = rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(
+    Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(rewriter,
         op->getLoc(), adaptor.getMemrefs()[0]);
     for (Value retained : adaptor.getRetained()) {
       Value retainedAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op->getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op->getLoc(),
                                                                   retained);
-      Value doesNotAlias = rewriter.create<arith::CmpIOp>(
+      Value doesNotAlias = arith::CmpIOp::create(rewriter,
           op->getLoc(), arith::CmpIPredicate::ne, memrefAsIdx, retainedAsIdx);
       doesNotAliasList.push_back(doesNotAlias);
     }
@@ -123,17 +123,17 @@ class DeallocOpConversion
     // AND-reduce the list of booleans from above.
     Value prev = doesNotAliasList.front();
     for (Value doesNotAlias : ArrayRef(doesNotAliasList).drop_front())
-      prev = rewriter.create<arith::AndIOp>(op->getLoc(), prev, doesNotAlias);
+      prev = arith::AndIOp::create(rewriter, op->getLoc(), prev, doesNotAlias);
 
     // Also consider the condition given by the dealloc operation and perform a
     // conditional deallocation guarded by that value.
-    Value shouldDealloc = rewriter.create<arith::AndIOp>(
+    Value shouldDealloc = arith::AndIOp::create(rewriter,
         op->getLoc(), prev, adaptor.getConditions()[0]);
 
-    rewriter.create<scf::IfOp>(
+    scf::IfOp::create(rewriter,
         op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-          builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[0]);
-          builder.create<scf::YieldOp>(loc);
+          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
+          scf::YieldOp::create(builder, loc);
         });
 
     // Compute the replacement values for the dealloc operation results. This
@@ -141,12 +141,12 @@ class DeallocOpConversion
     // `select(does_alias_with_memref(r), memref_cond, false)` for each retained
     // value r.
     SmallVector<Value> replacements;
-    Value trueVal = rewriter.create<arith::ConstantOp>(
+    Value trueVal = arith::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getBoolAttr(true));
     for (Value doesNotAlias : doesNotAliasList) {
       Value aliases =
-          rewriter.create<arith::XOrIOp>(op->getLoc(), doesNotAlias, trueVal);
-      Value result = rewriter.create<arith::AndIOp>(op->getLoc(), aliases,
+          arith::XOrIOp::create(rewriter, op->getLoc(), doesNotAlias, trueVal);
+      Value result = arith::AndIOp::create(rewriter, op->getLoc(), aliases,
                                                     adaptor.getConditions()[0]);
       replacements.push_back(result);
     }
@@ -231,18 +231,18 @@ class DeallocOpConversion
     // Without storing them to memrefs, we could not use for-loops but only a
     // completely unrolled version of it, potentially leading to code-size
     // blow-up.
-    Value toDeallocMemref = rewriter.create<memref::AllocOp>(
+    Value toDeallocMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
                                      rewriter.getIndexType()));
-    Value conditionMemref = rewriter.create<memref::AllocOp>(
+    Value conditionMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getConditions().size()},
                                      rewriter.getI1Type()));
-    Value toRetainMemref = rewriter.create<memref::AllocOp>(
+    Value toRetainMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
                                      rewriter.getIndexType()));
 
     auto getConstValue = [&](uint64_t value) -> Value {
-      return rewriter.create<arith::ConstantOp>(op.getLoc(),
+      return arith::ConstantOp::create(rewriter, op.getLoc(),
                                                 rewriter.getIndexAttr(value));
     };
 
@@ -250,58 +250,58 @@ class DeallocOpConversion
     // at runtime.
     for (auto [i, toDealloc] : llvm::enumerate(adaptor.getMemrefs())) {
       Value memrefAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op.getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
                                                                   toDealloc);
-      rewriter.create<memref::StoreOp>(op.getLoc(), memrefAsIdx,
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx,
                                        toDeallocMemref, getConstValue(i));
     }
 
     for (auto [i, cond] : llvm::enumerate(adaptor.getConditions()))
-      rewriter.create<memref::StoreOp>(op.getLoc(), cond, conditionMemref,
+      memref::StoreOp::create(rewriter, op.getLoc(), cond, conditionMemref,
                                        getConstValue(i));
 
     for (auto [i, toRetain] : llvm::enumerate(adaptor.getRetained())) {
       Value memrefAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op.getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
                                                                   toRetain);
-      rewriter.create<memref::StoreOp>(op.getLoc(), memrefAsIdx, toRetainMemref,
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx, toRetainMemref,
                                        getConstValue(i));
     }
 
     // Cast the allocated memrefs to dynamic shape because we want only one
     // helper function no matter how many operands the bufferization.dealloc
     // has.
-    Value castedDeallocMemref = rewriter.create<memref::CastOp>(
+    Value castedDeallocMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toDeallocMemref);
-    Value castedCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         conditionMemref);
-    Value castedRetainMemref = rewriter.create<memref::CastOp>(
+    Value castedRetainMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toRetainMemref);
 
-    Value deallocCondsMemref = rewriter.create<memref::AllocOp>(
+    Value deallocCondsMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
                                      rewriter.getI1Type()));
-    Value retainCondsMemref = rewriter.create<memref::AllocOp>(
+    Value retainCondsMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
                                      rewriter.getI1Type()));
 
-    Value castedDeallocCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedDeallocCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         deallocCondsMemref);
-    Value castedRetainCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedRetainCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         retainCondsMemref);
 
     Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>();
-    rewriter.create<func::CallOp>(
+    func::CallOp::create(rewriter,
         op.getLoc(), deallocHelperFuncMap.lookup(symtableOp),
         SmallVector<Value>{castedDeallocMemref, castedRetainMemref,
                            castedCondsMemref, castedDeallocCondsMemref,
@@ -309,30 +309,30 @@ class DeallocOpConversion
 
     for (unsigned i = 0, e = adaptor.getMemrefs().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value shouldDealloc = rewriter.create<memref::LoadOp>(
+      Value shouldDealloc = memref::LoadOp::create(rewriter,
           op.getLoc(), deallocCondsMemref, idxValue);
-      rewriter.create<scf::IfOp>(
+      scf::IfOp::create(rewriter,
           op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-            builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[i]);
-            builder.create<scf::YieldOp>(loc);
+            memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[i]);
+            scf::YieldOp::create(builder, loc);
           });
     }
 
     SmallVector<Value> replacements;
     for (unsigned i = 0, e = adaptor.getRetained().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value ownership = rewriter.create<memref::LoadOp>(
+      Value ownership = memref::LoadOp::create(rewriter,
           op.getLoc(), retainCondsMemref, idxValue);
       replacements.push_back(ownership);
     }
 
     // Deallocate above allocated memrefs again to avoid memory leaks.
     // Deallocation will not be run on code after this stage.
-    rewriter.create<memref::DeallocOp>(op.getLoc(), toDeallocMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), toRetainMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), conditionMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), deallocCondsMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), retainCondsMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), toDeallocMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), toRetainMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), conditionMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), deallocCondsMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), retainCondsMemref);
 
     rewriter.replaceOp(op, replacements);
     return success();
@@ -350,7 +350,7 @@ class DeallocOpConversion
                   ConversionPatternRewriter &rewriter) const override {
     // Lower the trivial case.
     if (adaptor.getMemrefs().empty()) {
-      Value falseVal = rewriter.create<arith::ConstantOp>(
+      Value falseVal = arith::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           op, SmallVector<Value>(adaptor.getRetained().size(), falseVal));
@@ -450,30 +450,30 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
   Value retainCondsMemref = helperFuncOp.getArguments()[4];
 
   // Insert some prerequisites.
-  Value c0 = builder.create<arith::ConstantOp>(loc, builder.getIndexAttr(0));
-  Value c1 = builder.create<arith::ConstantOp>(loc, builder.getIndexAttr(1));
+  Value c0 = arith::ConstantOp::create(builder, loc, builder.getIndexAttr(0));
+  Value c1 = arith::ConstantOp::create(builder, loc, builder.getIndexAttr(1));
   Value trueValue =
-      builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(true));
+      arith::ConstantOp::create(builder, loc, builder.getBoolAttr(true));
   Value falseValue =
-      builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(false));
-  Value toDeallocSize = builder.create<memref::DimOp>(loc, toDeallocMemref, c0);
-  Value toRetainSize = builder.create<memref::DimOp>(loc, toRetainMemref, c0);
+      arith::ConstantOp::create(builder, loc, builder.getBoolAttr(false));
+  Value toDeallocSize = memref::DimOp::create(builder, loc, toDeallocMemref, c0);
+  Value toRetainSize = memref::DimOp::create(builder, loc, toRetainMemref, c0);
 
-  builder.create<scf::ForOp>(
+  scf::ForOp::create(builder,
       loc, c0, toRetainSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value i, ValueRange iterArgs) {
-        builder.create<memref::StoreOp>(loc, falseValue, retainCondsMemref, i);
-        builder.create<scf::YieldOp>(loc);
+        memref::StoreOp::create(builder, loc, falseValue, retainCondsMemref, i);
+        scf::YieldOp::create(builder, loc);
       });
 
-  builder.create<scf::ForOp>(
+  scf::ForOp::create(builder,
       loc, c0, toDeallocSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value outerIter,
           ValueRange iterArgs) {
         Value toDealloc =
-            builder.create<memref::LoadOp>(loc, toDeallocMemref, outerIter);
+            memref::LoadOp::create(builder, loc, toDeallocMemref, outerIter);
         Value cond =
-            builder.create<memref::LoadOp>(loc, conditionMemref, outerIter);
+            memref::LoadOp::create(builder, loc, conditionMemref, outerIter);
 
         // Build the first for loop that computes aliasing with retained
         // memrefs.
@@ -483,31 +483,31 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, toRetainSize, c1, trueValue,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value retainValue = builder.create<memref::LoadOp>(
+                      Value retainValue = memref::LoadOp::create(builder,
                           loc, toRetainMemref, i);
-                      Value doesAlias = builder.create<arith::CmpIOp>(
+                      Value doesAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::eq, retainValue,
                           toDealloc);
-                      builder.create<scf::IfOp>(
+                      scf::IfOp::create(builder,
                           loc, doesAlias,
                           [&](OpBuilder &builder, Location loc) {
                             Value retainCondValue =
-                                builder.create<memref::LoadOp>(
+                                memref::LoadOp::create(builder,
                                     loc, retainCondsMemref, i);
                             Value aggregatedRetainCond =
-                                builder.create<arith::OrIOp>(
+                                arith::OrIOp::create(builder,
                                     loc, retainCondValue, cond);
-                            builder.create<memref::StoreOp>(
+                            memref::StoreOp::create(builder,
                                 loc, aggregatedRetainCond, retainCondsMemref,
                                 i);
-                            builder.create<scf::YieldOp>(loc);
+                            scf::YieldOp::create(builder, loc);
                           });
-                      Value doesntAlias = builder.create<arith::CmpIOp>(
+                      Value doesntAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::ne, retainValue,
                           toDealloc);
-                      Value yieldValue = builder.create<arith::AndIOp>(
+                      Value yieldValue = arith::AndIOp::create(builder,
                           loc, iterArgs[0], doesntAlias);
-                      builder.create<scf::YieldOp>(loc, yieldValue);
+                      scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
@@ -519,24 +519,24 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, outerIter, c1, noRetainAlias,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value prevDeallocValue = builder.create<memref::LoadOp>(
+                      Value prevDeallocValue = memref::LoadOp::create(builder,
                           loc, toDeallocMemref, i);
-                      Value doesntAlias = builder.create<arith::CmpIOp>(
+                      Value doesntAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::ne, prevDeallocValue,
                           toDealloc);
-                      Value yieldValue = builder.create<arith::AndIOp>(
+                      Value yieldValue = arith::AndIOp::create(builder,
                           loc, iterArgs[0], doesntAlias);
-                      builder.create<scf::YieldOp>(loc, yieldValue);
+                      scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
-        Value shouldDealoc = builder.create<arith::AndIOp>(loc, noAlias, cond);
-        builder.create<memref::StoreOp>(loc, shouldDealoc, deallocCondsMemref,
+        Value shouldDealoc = arith::AndIOp::create(builder, loc, noAlias, cond);
+        memref::StoreOp::create(builder, loc, shouldDealoc, deallocCondsMemref,
                                         outerIter);
-        builder.create<scf::YieldOp>(loc);
+        scf::YieldOp::create(builder, loc);
       });
 
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
   return helperFuncOp;
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index 1eeafc4df8cf1..71890bfea4e5d 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -43,7 +43,7 @@ using namespace mlir::bufferization;
 //===----------------------------------------------------------------------===//
 
 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
-  return builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
 }
 
 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
@@ -755,12 +755,12 @@ Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
           .create<scf::IfOp>(
               memref.getLoc(), condition,
               [&](OpBuilder &builder, Location loc) {
-                builder.create<scf::YieldOp>(loc, newMemref);
+                scf::YieldOp::create(builder, loc, newMemref);
               },
               [&](OpBuilder &builder, Location loc) {
                 Value clone =
-                    builder.create<bufferization::CloneOp>(loc, newMemref);
-                builder.create<scf::YieldOp>(loc, clone);
+                    bufferization::CloneOp::create(builder, loc, newMemref);
+                scf::YieldOp::create(builder, loc, clone);
               })
           .getResult(0);
   Value trueVal = buildBoolValue(builder, memref.getLoc(), true);
@@ -797,7 +797,7 @@ BufferDeallocation::handleInterface(BranchOpInterface op) {
   state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
                            toRetain);
 
-  auto deallocOp = builder.create<bufferization::DeallocOp>(
+  auto deallocOp = bufferization::DeallocOp::create(builder,
       op.getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
@@ -885,10 +885,10 @@ BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
       builder.setInsertionPoint(op);
       Ownership ownership = state.getOwnership(operand, block);
       if (ownership.isUnique()) {
-        Value ownershipInverted = builder.create<arith::XOrIOp>(
+        Value ownershipInverted = arith::XOrIOp::create(builder,
             op.getLoc(), ownership.getIndicator(),
             buildBoolValue(builder, op.getLoc(), true));
-        builder.create<cf::AssertOp>(
+        cf::AssertOp::create(builder,
             op.getLoc(), ownershipInverted,
             "expected that the block does not have ownership");
       }
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
index 0bdcf434e062f..b679bbe05cb2f 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
@@ -49,7 +49,7 @@ Operation *complex::ComplexDialect::materializeConstant(OpBuilder &builder,
                                                         Type type,
                                                         Location loc) {
   if (complex::ConstantOp::isBuildableWith(value, type)) {
-    return builder.create<complex::ConstantOp>(loc, type,
+    return complex::ConstantOp::create(builder, loc, type,
                                                llvm::cast<ArrayAttr>(value));
   }
   return arith::ConstantOp::materialize(builder, value, type, loc);
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 0c11c76cf1f71..38e47cc5e3714 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -312,7 +312,7 @@ struct SimplifyCondBranchIdenticalSuccessors
       if (std::get<0>(it) == std::get<1>(it))
         mergedOperands.push_back(std::get<0>(it));
       else
-        mergedOperands.push_back(rewriter.create<arith::SelectOp>(
+        mergedOperands.push_back(arith::SelectOp::create(rewriter,
             condbr.getLoc(), condition, std::get<0>(it), std::get<1>(it)));
     }
 
@@ -412,7 +412,7 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantTrue)
-            constantTrue = rewriter.create<arith::ConstantOp>(
+            constantTrue = arith::ConstantOp::create(rewriter,
                 condbr.getLoc(), ty, rewriter.getBoolAttr(true));
 
           rewriter.modifyOpInPlace(use.getOwner(),
@@ -427,7 +427,7 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantFalse)
-            constantFalse = rewriter.create<arith::ConstantOp>(
+            constantFalse = arith::ConstantOp::create(rewriter,
                 condbr.getLoc(), ty, rewriter.getBoolAttr(false));
 
           rewriter.modifyOpInPlace(use.getOwner(),
diff --git a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index a077f56f4f472..23f50930cb3b3 100644
--- a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -87,7 +87,7 @@ struct CondBranchOpInterface
                                destOperands.getAsOperandRange(), toRetain);
       SmallVector<Value> adaptedConditions(
           llvm::map_range(conditions, conditionModifier));
-      auto deallocOp = builder.create<bufferization::DeallocOp>(
+      auto deallocOp = bufferization::DeallocOp::create(builder,
           condBr.getLoc(), memrefs, adaptedConditions, toRetain);
       state.resetOwnerships(deallocOp.getRetained(), condBr->getBlock());
       for (auto [retained, ownership] : llvm::zip(
@@ -115,18 +115,18 @@ struct CondBranchOpInterface
     DeallocOp thenTakenDeallocOp = insertDeallocForBranch(
         condBr.getTrueDest(), condBr.getTrueDestOperandsMutable(),
         [&](Value cond) {
-          return builder.create<arith::AndIOp>(condBr.getLoc(), cond,
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond,
                                                condBr.getCondition());
         },
         thenMapping);
     DeallocOp elseTakenDeallocOp = insertDeallocForBranch(
         condBr.getFalseDest(), condBr.getFalseDestOperandsMutable(),
         [&](Value cond) {
-          Value trueVal = builder.create<arith::ConstantOp>(
+          Value trueVal = arith::ConstantOp::create(builder,
               condBr.getLoc(), builder.getBoolAttr(true));
-          Value negation = builder.create<arith::XOrIOp>(
+          Value negation = arith::XOrIOp::create(builder,
               condBr.getLoc(), trueVal, condBr.getCondition());
-          return builder.create<arith::AndIOp>(condBr.getLoc(), cond, negation);
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond, negation);
         },
         elseMapping);
 
@@ -143,7 +143,7 @@ struct CondBranchOpInterface
 
     for (Value retained : commonValues) {
       state.resetOwnerships(retained, condBr->getBlock());
-      Value combinedOwnership = builder.create<arith::SelectOp>(
+      Value combinedOwnership = arith::SelectOp::create(builder,
           condBr.getLoc(), condBr.getCondition(), thenMapping[retained],
           elseMapping[retained]);
       state.updateOwnership(retained, combinedOwnership, condBr->getBlock());
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index 27298e892e599..f9852257a4a7c 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -50,13 +50,13 @@ void EmitCDialect::initialize() {
 Operation *EmitCDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
-  return builder.create<emitc::ConstantOp>(loc, type, value);
+  return emitc::ConstantOp::create(builder, loc, type, value);
 }
 
 /// Default callback for builders of ops carrying a region. Inserts a yield
 /// without arguments.
 void mlir::emitc::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<emitc::YieldOp>(loc);
+  emitc::YieldOp::create(builder, loc);
 }
 
 bool mlir::emitc::isSupportedEmitCType(Type type) {
diff --git a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
index a578a86b499a6..7e04b4b196a6d 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
@@ -25,7 +25,7 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
   Location loc = op->getLoc();
 
   builder.setInsertionPointAfter(op);
-  auto expressionOp = builder.create<emitc::ExpressionOp>(loc, resultType);
+  auto expressionOp = emitc::ExpressionOp::create(builder, loc, resultType);
 
   // Replace all op's uses with the new expression's result.
   result.replaceAllUsesWith(expressionOp.getResult());
@@ -34,7 +34,7 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
   Region &region = expressionOp.getRegion();
   Block &block = region.emplaceBlock();
   builder.setInsertionPointToEnd(&block);
-  auto yieldOp = builder.create<emitc::YieldOp>(loc, result);
+  auto yieldOp = emitc::YieldOp::create(builder, loc, result);
 
   // Move op into the new expression.
   op->moveBefore(yieldOp);
diff --git a/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp b/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
index 72c8fd0f32485..ab7be8d6cedd9 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
@@ -21,7 +21,7 @@ Value materializeAsUnrealizedCast(OpBuilder &builder, Type resultType,
   if (inputs.size() != 1)
     return Value();
 
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
       .getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
index 17d436f6df028..72c331ad96607 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
@@ -50,7 +50,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
                                 PatternRewriter &rewriter) const override {
 
     auto className = funcOp.getSymNameAttr().str() + "Class";
-    ClassOp newClassOp = rewriter.create<ClassOp>(funcOp.getLoc(), className);
+    ClassOp newClassOp = ClassOp::create(rewriter, funcOp.getLoc(), className);
 
     SmallVector<std::pair<StringAttr, TypeAttr>> fields;
     rewriter.createBlock(&newClassOp.getBody());
@@ -67,7 +67,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
 
       TypeAttr typeAttr = TypeAttr::get(val.getType());
       fields.push_back({fieldName, typeAttr});
-      rewriter.create<emitc::FieldOp>(funcOp.getLoc(), fieldName, typeAttr,
+      emitc::FieldOp::create(rewriter, funcOp.getLoc(), fieldName, typeAttr,
                                       argAttr);
     }
 
@@ -75,7 +75,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
     FunctionType funcType = funcOp.getFunctionType();
     Location loc = funcOp.getLoc();
     FuncOp newFuncOp =
-        rewriter.create<emitc::FuncOp>(loc, ("execute"), funcType);
+        emitc::FuncOp::create(rewriter, loc, ("execute"), funcType);
 
     rewriter.createBlock(&newFuncOp.getBody());
     newFuncOp.getBody().takeBody(funcOp.getBody());
@@ -85,7 +85,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
     newArguments.reserve(fields.size());
     for (auto &[fieldName, attr] : fields) {
       GetFieldOp arg =
-          rewriter.create<emitc::GetFieldOp>(loc, attr.getValue(), fieldName);
+          emitc::GetFieldOp::create(rewriter, loc, attr.getValue(), fieldName);
       newArguments.push_back(arg);
     }
 
diff --git a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
index 3328d58551bff..7485ba40b4344 100644
--- a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
+++ b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
@@ -61,7 +61,7 @@ struct FuncInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    builder.create<cf::BranchOp>(op->getLoc(), newDest, returnOp.getOperands());
+    cf::BranchOp::create(builder, op->getLoc(), newDest, returnOp.getOperands());
     op->erase();
   }
 
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index 9d317f20521fb..1c0d3cd338006 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -54,7 +54,7 @@ void FuncDialect::initialize() {
 Operation *FuncDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                             Type type, Location loc) {
   if (ConstantOp::isBuildableWith(value, type))
-    return builder.create<ConstantOp>(loc, type,
+    return ConstantOp::create(builder, loc, type,
                                       llvm::cast<FlatSymbolRefAttr>(value));
   return nullptr;
 }
diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
index 3adbf092742be..2ece1bcffb1b0 100644
--- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
+++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
@@ -171,7 +171,7 @@ transform::CastAndCallOp::apply(transform::TransformRewriter &rewriter,
     }
   }
 
-  auto callOp = rewriter.create<func::CallOp>(insertionPoint->getLoc(),
+  auto callOp = func::CallOp::create(rewriter, insertionPoint->getLoc(),
                                               targetFunction, inputs);
 
   // Cast the call results back to the expected types. If any conversions fail
diff --git a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
index a3638c8766a5c..4b70cf172b407 100644
--- a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
@@ -46,7 +46,7 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 
     // Substitute with the new result types from the corresponding FuncType
     // conversion.
-    auto newCallOp = rewriter.create<CallOp>(
+    auto newCallOp = CallOp::create(rewriter,
         callOp.getLoc(), callOp.getCallee(), convertedResults,
         flattenValues(adaptor.getOperands()));
     SmallVector<ValueRange> replacements;
diff --git a/mlir/lib/Dialect/Func/Utils/Utils.cpp b/mlir/lib/Dialect/Func/Utils/Utils.cpp
index 0e9662689ef78..1eba777921828 100644
--- a/mlir/lib/Dialect/Func/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Func/Utils/Utils.cpp
@@ -44,7 +44,7 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
   for (unsigned int idx : newResultsOrder)
     newOutputTypes.push_back(origOutputTypes[idx]);
   rewriter.setInsertionPoint(funcOp);
-  auto newFuncOp = rewriter.create<func::FuncOp>(
+  auto newFuncOp = func::FuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(newInputTypes, newOutputTypes));
 
@@ -80,7 +80,7 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
     newReturnValues.push_back(returnOp.getOperand(idx));
   rewriter.setInsertionPoint(returnOp);
   auto newReturnOp =
-      rewriter.create<func::ReturnOp>(newFuncOp.getLoc(), newReturnValues);
+      func::ReturnOp::create(rewriter, newFuncOp.getLoc(), newReturnValues);
   newReturnOp->setDiscardableAttrs(returnOp->getDiscardableAttrDictionary());
   rewriter.eraseOp(returnOp);
 
@@ -109,7 +109,7 @@ func::replaceCallOpWithNewOrder(RewriterBase &rewriter, func::CallOp callOp,
   // Replace the kernel call operation with a new one that has the
   // reordered arguments.
   rewriter.setInsertionPoint(callOp);
-  auto newCallOp = rewriter.create<func::CallOp>(
+  auto newCallOp = func::CallOp::create(rewriter,
       callOp.getLoc(), callOp.getCallee(), newResultTypes, newArgsOrderValues);
   newCallOp.setNoInlineAttr(callOp.getNoInlineAttr());
   for (auto &&[newIndex, origIndex] : llvm::enumerate(newResultsOrder))
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 30b5ac9809139..f7eb41be287e4 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -1137,7 +1137,7 @@ struct FoldLaunchArguments : public OpRewritePattern<LaunchOp> {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(&op.getBody().front());
         zero =
-            rewriter.create<arith::ConstantIndexOp>(op.getLoc(), /*value=*/0);
+            arith::ConstantIndexOp::create(rewriter, op.getLoc(), /*value=*/0);
       }
       rewriter.replaceAllUsesWith(id, zero);
       simplified = true;
@@ -1381,9 +1381,9 @@ static void printLaunchFuncOperands(OpAsmPrinter &printer, Operation *,
 void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
                       int32_t offset, int32_t width, ShuffleMode mode) {
   build(builder, result, value,
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(offset)),
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(width)),
         mode);
 }
@@ -1395,9 +1395,9 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
 void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
                      int32_t offset, int32_t width) {
   build(builder, result, value,
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(offset)),
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(width)));
 }
 
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index c9e91535df946..2035f371694a7 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -560,7 +560,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   Value predicate;
   if (originalBasisWasProvided) {
     for (Value tmpPredicate : builderResult.predicateOps) {
-      predicate = predicate ? rewriter.create<arith::AndIOp>(loc, predicate,
+      predicate = predicate ? arith::AndIOp::create(rewriter, loc, predicate,
                                                              tmpPredicate)
                             : tmpPredicate;
     }
@@ -573,7 +573,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   Block::iterator insertionPoint;
   if (predicate) {
     // Step 6.a. If predicated, move at the beginning.
-    auto ifOp = rewriter.create<scf::IfOp>(loc, predicate,
+    auto ifOp = scf::IfOp::create(rewriter, loc, predicate,
                                            /*withElseRegion=*/false);
     targetBlock = ifOp.thenBlock();
     insertionPoint = ifOp.thenBlock()->begin();
@@ -632,7 +632,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapForallToBlocksImpl(
     // the insertion point.
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPointToStart(parentBlock);
-    zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   }
 
   ForallRewriteResult rewriteResult;
@@ -884,7 +884,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapOneForallToThreadsImpl(
     return diag;
   // Add a syncthreads if needed. TODO: warpsync
   if (syncAfterDistribute)
-    rewriter.create<BarrierOp>(loc);
+    BarrierOp::create(rewriter, loc);
 
   return DiagnosedSilenceableFailure::success();
 }
@@ -901,7 +901,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapNestedForallToThreadsImpl(
 
   // Create an early zero index value for replacements.
   Location loc = target->getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   DiagnosedSilenceableFailure diag = DiagnosedSilenceableFailure::success();
   WalkResult walkResult = target->walk([&](scf::ForallOp forallOp) {
     diag = mlir::transform::gpu::mapOneForallToThreadsImpl(
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index 12b7f39390967..fc4556df9d6f2 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -82,8 +82,8 @@ buildPredicates(RewriterBase &rewriter, Location loc, ArrayRef<Value> activeIds,
     }
     if (activeMappingSize == availableMappingSize)
       continue;
-    Value idx = rewriter.create<arith::ConstantIndexOp>(loc, activeMappingSize);
-    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value idx = arith::ConstantIndexOp::create(rewriter, loc, activeMappingSize);
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                                 activeId, idx);
     predicateOps.push_back(pred);
   }
@@ -104,11 +104,11 @@ static Value buildLinearId(RewriterBase &rewriter, Location loc,
   bindDims(rewriter.getContext(), tx, ty, tz);
   bindSymbols(rewriter.getContext(), bdx, bdy);
   SmallVector<OpFoldResult> vals{
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::x)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::x)
           .getResult(),
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::y)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::y)
           .getResult(),
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::z)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::z)
           .getResult(),
       originalBasisOfr[0], originalBasisOfr[1]};
   OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
@@ -215,9 +215,9 @@ static GpuIdBuilderFnType common3DIdBuilderFn(int64_t multiplicity = 1) {
                             ArrayRef<int64_t> originalBasis) {
     IndexType indexType = rewriter.getIndexType();
     SmallVector<Value> ids{
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::x),
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::y),
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::z)};
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::x),
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::y),
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::z)};
     // In the 3-D mapping case, scale the first dimension by the multiplicity.
     SmallVector<Value> scaledIds = ids;
     AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
@@ -417,7 +417,7 @@ DiagnosedSilenceableFailure createGpuLaunch(
     return diag;
 
   auto createConst = [&](int dim) {
-    return rewriter.create<arith::ConstantIndexOp>(loc, dim);
+    return arith::ConstantIndexOp::create(rewriter, loc, dim);
   };
   OpBuilder::InsertionGuard guard(rewriter);
   Value one = createConst(1);
@@ -427,10 +427,10 @@ DiagnosedSilenceableFailure createGpuLaunch(
   Value blkSizeX = blockDimX.has_value() ? createConst(blockDimX.value()) : one;
   Value blkSizeY = blockDimY.has_value() ? createConst(blockDimY.value()) : one;
   Value blkSizeZ = blockDimZ.has_value() ? createConst(blockDimZ.value()) : one;
-  launchOp = rewriter.create<LaunchOp>(loc, gridSizeX, gridSizeY, gridSizeZ,
+  launchOp = LaunchOp::create(rewriter, loc, gridSizeX, gridSizeY, gridSizeZ,
                                        blkSizeX, blkSizeY, blkSizeZ);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
-  rewriter.create<TerminatorOp>(loc);
+  TerminatorOp::create(rewriter, loc);
   return DiagnosedSilenceableFailure::success();
 }
 
@@ -451,7 +451,7 @@ DiagnosedSilenceableFailure alterGpuLaunch(
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPointAfterValue(currentBlockdim.x);
   auto createConstValue = [&](int dim) {
-    return rewriter.create<arith::ConstantIndexOp>(currentBlockdim.x.getLoc(),
+    return arith::ConstantIndexOp::create(rewriter, currentBlockdim.x.getLoc(),
                                                    dim);
   };
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
index d35f72e5a9e26..66e011a793485 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
@@ -147,7 +147,7 @@ struct GpuAllReduceRewriter {
   // Shortcut to create an op from rewriter using loc as the first argument.
   template <typename T, typename... Args>
   T create(Args... args) {
-    return rewriter.create<T>(loc, std::forward<Args>(args)...);
+    return T::create(rewriter, loc, std::forward<Args>(args)...);
   }
 
   // Creates dimension op of type T, with the result casted to int32.
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index 99a91ecd5642c..ab3dcc6a45d11 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -132,7 +132,7 @@ struct GpuAsyncRegionPass::ThreadTokenCallback {
   }
 
   Value createWaitOp(Location loc, Type resultType, ValueRange operands) {
-    return builder.create<gpu::WaitOp>(loc, resultType, operands)
+    return gpu::WaitOp::create(builder, loc, resultType, operands)
         .getAsyncToken();
   }
 
@@ -168,7 +168,7 @@ async::ExecuteOp addExecuteResults(async::ExecuteOp executeOp,
 
   // Clone executeOp with the extra results.
   OpBuilder builder(executeOp);
-  auto newOp = builder.create<async::ExecuteOp>(
+  auto newOp = async::ExecuteOp::create(builder,
       executeOp.getLoc(), TypeRange{resultTypes}.drop_front() /*drop token*/,
       executeOp.getDependencies(), executeOp.getBodyOperands());
   IRMapping mapper;
@@ -250,7 +250,7 @@ struct GpuAsyncRegionPass::DeferWaitCallback {
           builder.setInsertionPointAfter(op);
           for (auto asyncToken : asyncTokens)
             tokens.push_back(
-                builder.create<async::AwaitOp>(loc, asyncToken).getResult());
+                async::AwaitOp::create(builder, loc, asyncToken).getResult());
           // Set `it` after the inserted async.await ops.
           it = builder.getInsertionPoint();
         })
@@ -282,7 +282,7 @@ struct GpuAsyncRegionPass::DeferWaitCallback {
 
     // Otherwise, insert a gpu.wait before 'it'.
     builder.setInsertionPoint(it->getBlock(), it);
-    auto waitOp = builder.create<gpu::WaitOp>(loc, Type{}, tokens);
+    auto waitOp = gpu::WaitOp::create(builder, loc, Type{}, tokens);
 
     // If the new waitOp is at the end of an async.execute region, add it to the
     // worklist. 'operator()(executeOp)' would do the same, but this is faster.
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index f63af8da28087..1c7fdf357af20 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -64,7 +64,7 @@ getFlatOffsetAndStrides(OpBuilder &rewriter, Location loc, Value source,
     OpBuilder::InsertionGuard g(rewriter);
     setInsertionPointToStart(rewriter, source);
     newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, source);
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, source);
   }
 
   auto &&[sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
@@ -110,7 +110,7 @@ static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
   auto &&[base, offset, ignore] =
       getFlatOffsetAndStrides(rewriter, loc, source, offsetsTemp);
   MemRefType retType = inferCastResultType(base, offset);
-  return rewriter.create<memref::ReinterpretCastOp>(loc, retType, base, offset,
+  return memref::ReinterpretCastOp::create(rewriter, loc, retType, base, offset,
                                                     ArrayRef<OpFoldResult>(),
                                                     ArrayRef<OpFoldResult>());
 }
diff --git a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
index c40ddd9b15afc..8dfd52e5803f0 100644
--- a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
@@ -28,11 +28,11 @@ struct GpuGlobalIdRewriter : public OpRewritePattern<gpu::GlobalIdOp> {
                                 PatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     auto dim = op.getDimension();
-    auto blockId = rewriter.create<gpu::BlockIdOp>(loc, dim);
-    auto blockDim = rewriter.create<gpu::BlockDimOp>(loc, dim);
+    auto blockId = gpu::BlockIdOp::create(rewriter, loc, dim);
+    auto blockDim = gpu::BlockDimOp::create(rewriter, loc, dim);
     // Compute blockId.x * blockDim.x
-    auto tmp = rewriter.create<index::MulOp>(op.getLoc(), blockId, blockDim);
-    auto threadId = rewriter.create<gpu::ThreadIdOp>(loc, dim);
+    auto tmp = index::MulOp::create(rewriter, op.getLoc(), blockId, blockDim);
+    auto threadId = gpu::ThreadIdOp::create(rewriter, loc, dim);
     // Compute threadId.x + blockId.x * blockDim.x
     rewriter.replaceOpWithNewOp<index::AddOp>(op, threadId, tmp);
     return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index a64ec8d52daf0..af8b097ecf6a9 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -41,7 +41,7 @@ template <typename OpTy>
 static void createForAllDimensions(OpBuilder &builder, Location loc,
                                    SmallVectorImpl<Value> &values) {
   for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z})
-    values.push_back(builder.create<OpTy>(loc, builder.getIndexType(), dim));
+    values.push_back(OpTy::create(builder, loc, builder.getIndexType(), dim));
 }
 
 /// Adds operations generating block/thread ids and grid/block dimensions at the
@@ -196,7 +196,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   }
   FunctionType type =
       FunctionType::get(launchOp.getContext(), kernelOperandTypes, {});
-  auto outlinedFunc = builder.create<gpu::GPUFuncOp>(
+  auto outlinedFunc = gpu::GPUFuncOp::create(builder,
       loc, kernelFnName, type,
       TypeRange(ValueRange(launchOp.getWorkgroupAttributions())),
       TypeRange(ValueRange(launchOp.getPrivateAttributions())));
@@ -248,7 +248,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
     if (!terminator)
       continue;
     OpBuilder replacer(terminator);
-    replacer.create<gpu::ReturnOp>(terminator->getLoc());
+    gpu::ReturnOp::create(replacer, terminator->getLoc());
     terminator->erase();
   }
 
@@ -288,7 +288,7 @@ static void convertToLaunchFuncOp(gpu::LaunchOp launchOp,
   Value asyncToken = launchOp.getAsyncToken();
   std::optional<gpu::KernelDim3> clusterSize =
       launchOp.getClusterSizeOperandValues();
-  auto launchFunc = builder.create<gpu::LaunchFuncOp>(
+  auto launchFunc = gpu::LaunchFuncOp::create(builder,
       launchOp.getLoc(), kernelFunc, launchOp.getGridSizeOperandValues(),
       launchOp.getBlockSizeOperandValues(),
       launchOp.getDynamicSharedMemorySize(), operands,
@@ -416,7 +416,7 @@ class GpuKernelOutliningPass
     // Check if the module already exists in the symbol table
     if (!kernelModule) {
       // If not found, create a new GPU module
-      kernelModule = builder.create<gpu::GPUModuleOp>(kernelFunc.getLoc(),
+      kernelModule = gpu::GPUModuleOp::create(builder, kernelFunc.getLoc(),
                                                       kernelModuleName);
     }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 14c44f27a6249..2bee1a1619665 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -34,8 +34,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
   auto rank = memRefType.getRank();
 
   SmallVector<Value, 4> lbs, ubs, steps;
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value one = b.create<arith::ConstantIndexOp>(1);
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value one = arith::ConstantIndexOp::create(b, 1);
 
   // Make sure we have enough loops to use all thread dimensions, these trivial
   // loops should be outermost and therefore inserted first.
@@ -59,8 +59,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
   auto indexType = b.getIndexType();
   SmallVector<Value, 3> threadIds, blockDims;
   for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z}) {
-    threadIds.push_back(b.create<gpu::ThreadIdOp>(indexType, dim));
-    blockDims.push_back(b.create<gpu::BlockDimOp>(indexType, dim));
+    threadIds.push_back(gpu::ThreadIdOp::create(b, indexType, dim));
+    blockDims.push_back(gpu::BlockDimOp::create(b, indexType, dim));
   }
 
   // Produce the loop nest with copies.
@@ -70,8 +70,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
       [&](OpBuilder &b, Location loc, ValueRange loopIvs) {
         ivs.assign(loopIvs.begin(), loopIvs.end());
         auto activeIvs = llvm::ArrayRef(ivs).take_back(rank);
-        Value loaded = b.create<memref::LoadOp>(loc, from, activeIvs);
-        b.create<memref::StoreOp>(loc, loaded, to, activeIvs);
+        Value loaded = memref::LoadOp::create(b, loc, from, activeIvs);
+        memref::StoreOp::create(b, loc, loaded, to, activeIvs);
       });
 
   // Map the innermost loops to threads in reverse order.
@@ -131,10 +131,10 @@ static void insertCopies(Region &region, Location loc, Value from, Value to) {
 
   auto b = ImplicitLocOpBuilder::atBlockBegin(loc, &region.front());
   insertCopyLoops(b, from, to);
-  b.create<gpu::BarrierOp>();
+  gpu::BarrierOp::create(b, );
 
   b.setInsertionPoint(&region.front().back());
-  b.create<gpu::BarrierOp>();
+  gpu::BarrierOp::create(b, );
   insertCopyLoops(b, to, from);
 }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
index f8a548af6b3e8..5bc85f236575d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
@@ -115,7 +115,7 @@ LogicalResult moduleSerializer(GPUModuleOp op,
       !handler && moduleHandler)
     handler = moduleHandler;
   builder.setInsertionPointAfter(op);
-  builder.create<gpu::BinaryOp>(op.getLoc(), op.getName(), handler,
+  gpu::BinaryOp::create(builder, op.getLoc(), op.getName(), handler,
                                 builder.getArrayAttr(objects));
   op->erase();
   return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
index 171e64346f155..5a55a39a7748e 100644
--- a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
@@ -48,10 +48,10 @@ struct PromoteShuffleToSwizzlePattern
                                          "offset must be in the range [0, 31]");
 
     Location loc = op.getLoc();
-    Value res = rewriter.create<amdgpu::SwizzleBitModeOp>(
+    Value res = amdgpu::SwizzleBitModeOp::create(rewriter,
         loc, op.getResult(0).getType(), op.getValue(), /*andMask=*/31,
         /*orMask=*/0, /*xorMask=*/offsetValue);
-    Value valid = rewriter.create<arith::ConstantIntOp>(loc, 1, /*width*/ 1);
+    Value valid = arith::ConstantIntOp::create(rewriter, loc, 1, /*width*/ 1);
     rewriter.replaceOp(op, {res, valid});
     return success();
   }
diff --git a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
index 9f2900214e8b1..08dedf065f900 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
@@ -48,16 +48,16 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
 
     // Float types must be converted to i64 to extract the bits.
     if (isa<FloatType>(valueType))
-      value = rewriter.create<arith::BitcastOp>(valueLoc, i64, value);
+      value = arith::BitcastOp::create(rewriter, valueLoc, i64, value);
 
     // Get the low bits by trunc(value).
-    lo = rewriter.create<arith::TruncIOp>(valueLoc, i32, value);
+    lo = arith::TruncIOp::create(rewriter, valueLoc, i32, value);
 
     // Get the high bits by trunc(value >> 32).
-    auto c32 = rewriter.create<arith::ConstantOp>(
+    auto c32 = arith::ConstantOp::create(rewriter,
         valueLoc, rewriter.getIntegerAttr(i64, 32));
-    hi = rewriter.create<arith::ShRUIOp>(valueLoc, value, c32);
-    hi = rewriter.create<arith::TruncIOp>(valueLoc, i32, hi);
+    hi = arith::ShRUIOp::create(rewriter, valueLoc, value, c32);
+    hi = arith::TruncIOp::create(rewriter, valueLoc, i32, hi);
 
     // Shuffle the values.
     ValueRange loRes =
@@ -72,21 +72,21 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
             .getResults();
 
     // Convert lo back to i64.
-    lo = rewriter.create<arith::ExtUIOp>(valueLoc, i64, loRes[0]);
+    lo = arith::ExtUIOp::create(rewriter, valueLoc, i64, loRes[0]);
 
     // Convert hi back to i64.
-    hi = rewriter.create<arith::ExtUIOp>(valueLoc, i64, hiRes[0]);
-    hi = rewriter.create<arith::ShLIOp>(valueLoc, hi, c32);
+    hi = arith::ExtUIOp::create(rewriter, valueLoc, i64, hiRes[0]);
+    hi = arith::ShLIOp::create(rewriter, valueLoc, hi, c32);
 
     // Obtain the shuffled bits hi | lo.
-    value = rewriter.create<arith::OrIOp>(loc, hi, lo);
+    value = arith::OrIOp::create(rewriter, loc, hi, lo);
 
     // Convert the value back to float.
     if (isa<FloatType>(valueType))
-      value = rewriter.create<arith::BitcastOp>(valueLoc, valueType, value);
+      value = arith::BitcastOp::create(rewriter, valueLoc, valueType, value);
 
     // Obtain the shuffle validity by combining both validities.
-    auto validity = rewriter.create<arith::AndIOp>(loc, loRes[1], hiRes[1]);
+    auto validity = arith::AndIOp::create(rewriter, loc, loRes[1], hiRes[1]);
 
     // Replace the op.
     rewriter.replaceOp(op, {value, validity});
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
index d80578235f3c3..0dbd9713a2a19 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
@@ -55,22 +55,22 @@ struct GpuSubgroupIdRewriter final : OpRewritePattern<gpu::SubgroupIdOp> {
     Location loc = op->getLoc();
     Type indexType = rewriter.getIndexType();
 
-    Value dimX = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::x);
-    Value dimY = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::y);
-    Value tidX = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
-    Value tidY = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::y);
-    Value tidZ = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::z);
+    Value dimX = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::x);
+    Value dimY = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::y);
+    Value tidX = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
+    Value tidY = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::y);
+    Value tidZ = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::z);
 
-    Value dimYxIdZ = rewriter.create<arith::MulIOp>(loc, indexType, dimY, tidZ);
+    Value dimYxIdZ = arith::MulIOp::create(rewriter, loc, indexType, dimY, tidZ);
     Value dimYxIdZPlusIdY =
-        rewriter.create<arith::AddIOp>(loc, indexType, dimYxIdZ, tidY);
+        arith::AddIOp::create(rewriter, loc, indexType, dimYxIdZ, tidY);
     Value dimYxIdZPlusIdYTimesDimX =
-        rewriter.create<arith::MulIOp>(loc, indexType, dimX, dimYxIdZPlusIdY);
-    Value IdXPlusDimYxIdZPlusIdYTimesDimX = rewriter.create<arith::AddIOp>(
+        arith::MulIOp::create(rewriter, loc, indexType, dimX, dimYxIdZPlusIdY);
+    Value IdXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(rewriter,
         loc, indexType, tidX, dimYxIdZPlusIdYTimesDimX);
-    Value subgroupSize = rewriter.create<gpu::SubgroupSizeOp>(
+    Value subgroupSize = gpu::SubgroupSizeOp::create(rewriter,
         loc, rewriter.getIndexType(), /*upper_bound = */ nullptr);
-    Value subgroupIdOp = rewriter.create<arith::DivUIOp>(
+    Value subgroupIdOp = arith::DivUIOp::create(rewriter,
         loc, indexType, IdXPlusDimYxIdZPlusIdYTimesDimX, subgroupSize);
     rewriter.replaceOp(op, {subgroupIdOp});
     return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index af9be4cccecfc..d5470de5a4faa 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -80,7 +80,7 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
 
     Location loc = op.getLoc();
     Value res =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(vecTy));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(vecTy));
 
     for (unsigned i = 0; i != numNewReductions; ++i) {
       int64_t startIdx = i * elementsPerShuffle;
@@ -91,22 +91,22 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
       Value extracted;
       if (numElems == 1) {
         extracted =
-            rewriter.create<vector::ExtractOp>(loc, op.getValue(), startIdx);
+            vector::ExtractOp::create(rewriter, loc, op.getValue(), startIdx);
       } else {
-        extracted = rewriter.create<vector::ExtractStridedSliceOp>(
+        extracted = vector::ExtractStridedSliceOp::create(rewriter,
             loc, op.getValue(), /*offsets=*/startIdx, /*sizes=*/numElems,
             /*strides=*/1);
       }
 
-      Value reduce = rewriter.create<gpu::SubgroupReduceOp>(
+      Value reduce = gpu::SubgroupReduceOp::create(rewriter,
           loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
           op.getClusterStride());
       if (numElems == 1) {
-        res = rewriter.create<vector::InsertOp>(loc, reduce, res, startIdx);
+        res = vector::InsertOp::create(rewriter, loc, reduce, res, startIdx);
         continue;
       }
 
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, reduce, res, /*offsets=*/startIdx, /*strides=*/1);
     }
 
@@ -139,8 +139,8 @@ struct ScalarizeSingleElementReduce final
     assert(vecTy.getRank() == 1 && "Unexpected vector type");
     assert(!vecTy.isScalable() && "Unexpected vector type");
     Location loc = op.getLoc();
-    Value extracted = rewriter.create<vector::ExtractOp>(loc, op.getValue(), 0);
-    Value reduce = rewriter.create<gpu::SubgroupReduceOp>(
+    Value extracted = vector::ExtractOp::create(rewriter, loc, op.getValue(), 0);
+    Value reduce = gpu::SubgroupReduceOp::create(rewriter,
         loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
         op.getClusterStride());
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, vecTy, reduce);
@@ -255,14 +255,14 @@ struct ScalarSubgroupReduceToShuffles final
     auto packFn = [loc, &rewriter, equivIntType,
                    shuffleIntType](Value unpackedVal) -> Value {
       auto asInt =
-          rewriter.create<arith::BitcastOp>(loc, equivIntType, unpackedVal);
-      return rewriter.create<arith::ExtUIOp>(loc, shuffleIntType, asInt);
+          arith::BitcastOp::create(rewriter, loc, equivIntType, unpackedVal);
+      return arith::ExtUIOp::create(rewriter, loc, shuffleIntType, asInt);
     };
     auto unpackFn = [loc, &rewriter, equivIntType,
                      valueTy](Value packedVal) -> Value {
       auto asInt =
-          rewriter.create<arith::TruncIOp>(loc, equivIntType, packedVal);
-      return rewriter.create<arith::BitcastOp>(loc, valueTy, asInt);
+          arith::TruncIOp::create(rewriter, loc, equivIntType, packedVal);
+      return arith::BitcastOp::create(rewriter, loc, valueTy, asInt);
     };
 
     rewriter.replaceOp(
@@ -327,9 +327,9 @@ struct VectorSubgroupReduceToShuffles final
         static_cast<int64_t>(elementsPerShuffle), vecTy.getElementType());
     Value extendedInput = op.getValue();
     if (vecBitwidth < shuffleBitwidth) {
-      auto zero = rewriter.create<arith::ConstantOp>(
+      auto zero = arith::ConstantOp::create(rewriter,
           loc, rewriter.getZeroAttr(extendedVecTy));
-      extendedInput = rewriter.create<vector::InsertStridedSliceOp>(
+      extendedInput = vector::InsertStridedSliceOp::create(rewriter,
           loc, extendedInput, zero, /*offsets=*/0, /*strides=*/1);
     }
 
@@ -338,21 +338,21 @@ struct VectorSubgroupReduceToShuffles final
 
     auto packFn = [loc, &rewriter, shuffleVecType](Value unpackedVal) -> Value {
       auto asIntVec =
-          rewriter.create<vector::BitCastOp>(loc, shuffleVecType, unpackedVal);
-      return rewriter.create<vector::ExtractOp>(loc, asIntVec, 0);
+          vector::BitCastOp::create(rewriter, loc, shuffleVecType, unpackedVal);
+      return vector::ExtractOp::create(rewriter, loc, asIntVec, 0);
     };
     auto unpackFn = [loc, &rewriter, shuffleVecType,
                      extendedVecTy](Value packedVal) -> Value {
       auto asIntVec =
-          rewriter.create<vector::BroadcastOp>(loc, shuffleVecType, packedVal);
-      return rewriter.create<vector::BitCastOp>(loc, extendedVecTy, asIntVec);
+          vector::BroadcastOp::create(rewriter, loc, shuffleVecType, packedVal);
+      return vector::BitCastOp::create(rewriter, loc, extendedVecTy, asIntVec);
     };
 
     Value res = createSubgroupShuffleReduction(
         rewriter, loc, extendedInput, op.getOp(), *ci, packFn, unpackFn);
 
     if (vecBitwidth < shuffleBitwidth) {
-      res = rewriter.create<vector::ExtractStridedSliceOp>(
+      res = vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/0, /*sizes=*/vecTy.getNumElements(),
           /*strides=*/1);
     }
@@ -379,7 +379,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   const bool boundCtrl = true;
   if (ci.clusterSize >= 2) {
     // Perform reduction between all lanes N <-> N+1.
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({1, 0, 3, 2}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -388,7 +388,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
 
   if (ci.clusterSize >= 4) {
     // Perform reduction between all lanes N <-> N+2.
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({2, 3, 0, 1}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -397,7 +397,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 8) {
     // Perform reduction between all lanes N <-> 7-N,
     // e.g lane[0] <-> lane[7], lane[1] <-> lane[6]..., lane[3] <-> lane[4].
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::row_half_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -406,7 +406,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 16) {
     // Perform reduction between all lanes N <-> 15-N,
     // e.g lane[0] <-> lane[15], lane[1] <-> lane[14]..., lane[7] <-> lane[8].
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::row_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -416,7 +416,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
     if (chipset.majorVersion <= 9) {
       // Broadcast last value from each row to next row.
       // Use row mask to avoid polluting rows 1 and 3.
-      dpp = rewriter.create<amdgpu::DPPOp>(
+      dpp = amdgpu::DPPOp::create(rewriter,
           loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_15,
           rewriter.getUnitAttr(), 0xa, allBanks,
           /*bound_ctrl*/ false);
@@ -424,9 +424,9 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
           rewriter, loc, gpu::convertReductionKind(mode), res, dpp);
     } else if (chipset.majorVersion <= 12) {
       // Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2).
-      Value uint32Max = rewriter.create<arith::ConstantOp>(
+      Value uint32Max = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
-      dpp = rewriter.create<ROCDL::PermlaneX16Op>(loc, res.getType(), res, res,
+      dpp = ROCDL::PermlaneX16Op::create(rewriter, loc, res.getType(), res, res,
                                                   uint32Max, uint32Max,
                                                   /*fi=*/true,
                                                   /*bound_ctrl=*/false);
@@ -438,37 +438,37 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
               "this device.");
     }
     if (ci.subgroupSize == 32) {
-      Value lane31 = rewriter.create<arith::ConstantOp>(
+      Value lane31 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      res = rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane31);
+      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
     }
   }
   if (ci.clusterSize >= 64) {
     if (chipset.majorVersion <= 9) {
       // Broadcast 31st lane value to rows 2 and 3.
-      dpp = rewriter.create<amdgpu::DPPOp>(
+      dpp = amdgpu::DPPOp::create(rewriter,
           loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_31,
           rewriter.getUnitAttr(), 0xf, allBanks,
           /*bound_ctrl*/ true);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), dpp, res);
       // Obtain reduction from last rows, the previous rows are polluted.
-      Value lane63 = rewriter.create<arith::ConstantOp>(
+      Value lane63 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
-      res = rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane63);
+      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
 
     } else if (chipset.majorVersion <= 12) {
       // Assume reduction across 32 lanes has been done.
       // Perform final reduction manually by summing values in lane 0 and
       // lane 32.
-      Value lane31 = rewriter.create<arith::ConstantOp>(
+      Value lane31 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      Value lane63 = rewriter.create<arith::ConstantOp>(
+      Value lane63 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
       lane31 =
-          rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane31);
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
       lane63 =
-          rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane63);
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), lane31, lane63);
     } else {
diff --git a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
index 29f6f32892f72..473a2e515c12b 100644
--- a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
+++ b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
@@ -27,7 +27,7 @@ WarpDistributionPattern::moveRegionToNewWarpOpAndReplaceReturns(
   // Create a new op before the existing one, with the extra operands.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(warpOp);
-  auto newWarpOp = rewriter.create<WarpExecuteOnLane0Op>(
+  auto newWarpOp = WarpExecuteOnLane0Op::create(rewriter,
       warpOp.getLoc(), newReturnTypes, warpOp.getLaneid(), warpOp.getWarpSize(),
       warpOp.getArgs(), warpOp.getBody()->getArgumentTypes());
 
@@ -124,7 +124,7 @@ bool WarpDistributionPattern::delinearizeLaneId(
 
   int64_t usedThreads = 1;
 
-  Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
   delinearizedIds.assign(sizes.size(), zero);
 
   for (int i = sizes.size() - 1; i >= 0; --i) {
diff --git a/mlir/lib/Dialect/Index/IR/IndexOps.cpp b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
index 5c935c5f4b53e..2b32962f9eabb 100644
--- a/mlir/lib/Dialect/Index/IR/IndexOps.cpp
+++ b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
@@ -37,7 +37,7 @@ Operation *IndexDialect::materializeConstant(OpBuilder &b, Attribute value,
   if (auto boolValue = dyn_cast<BoolAttr>(value)) {
     if (!type.isSignlessInteger(1))
       return nullptr;
-    return b.create<BoolConstantOp>(loc, type, boolValue);
+    return BoolConstantOp::create(b, loc, type, boolValue);
   }
 
   // Materialize integer attributes as `index`.
@@ -47,7 +47,7 @@ Operation *IndexDialect::materializeConstant(OpBuilder &b, Attribute value,
       return nullptr;
     assert(indexValue.getValue().getBitWidth() ==
            IndexType::kInternalStorageBitWidth);
-    return b.create<ConstantOp>(loc, indexValue);
+    return ConstantOp::create(b, loc, indexValue);
   }
 
   return nullptr;
@@ -716,10 +716,10 @@ LogicalResult CmpOp::canonicalize(CmpOp op, PatternRewriter &rewriter) {
 
   index::CmpOp newCmp;
   if (rhsIsZero)
-    newCmp = rewriter.create<index::CmpOp>(op.getLoc(), op.getPred(),
+    newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
                                            subOp.getLhs(), subOp.getRhs());
   else
-    newCmp = rewriter.create<index::CmpOp>(op.getLoc(), op.getPred(),
+    newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
                                            subOp.getRhs(), subOp.getLhs());
   rewriter.replaceOp(op, newCmp);
   return success();
diff --git a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
index c17ef1029faf6..b1d376c7c03db 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
@@ -90,7 +90,7 @@ void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
     }
     for (auto [idx, t] : llvm::enumerate(stype.getBody())) {
       if (itype != PTXRegisterMod::Write) {
-        Value extractValue = rewriter.create<LLVM::ExtractValueOp>(
+        Value extractValue = LLVM::ExtractValueOp::create(rewriter,
             interfaceOp->getLoc(), v, idx);
         addValue(extractValue);
       }
@@ -132,7 +132,7 @@ LLVM::InlineAsmOp PtxBuilder::build() {
   // Replace all % with $
   llvm::replace(ptxInstruction, '%', '$');
 
-  return rewriter.create<LLVM::InlineAsmOp>(
+  return LLVM::InlineAsmOp::create(rewriter,
       interfaceOp->getLoc(),
       /*result types=*/resultTypes,
       /*operands=*/ptxOperands,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 89f765dacda35..7d3f59d128367 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -89,7 +89,7 @@ mlir::LLVM::lookupOrCreateFn(OpBuilder &b, Operation *moduleOp, StringRef name,
   OpBuilder::InsertionGuard g(b);
   assert(!moduleOp->getRegion(0).empty() && "expected non-empty region");
   b.setInsertionPointToStart(&moduleOp->getRegion(0).front());
-  auto funcOp = b.create<LLVM::LLVMFuncOp>(
+  auto funcOp = LLVM::LLVMFuncOp::create(b,
       moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 6dcd94e6eea17..546704e903f47 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3318,7 +3318,7 @@ bool LLVM::ConstantOp::isBuildableWith(Attribute value, Type type) {
 ConstantOp LLVM::ConstantOp::materialize(OpBuilder &builder, Attribute value,
                                          Type type, Location loc) {
   if (isBuildableWith(value, type))
-    return builder.create<LLVM::ConstantOp>(loc, cast<TypedAttr>(value));
+    return LLVM::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
   return nullptr;
 }
 
@@ -4284,13 +4284,13 @@ Operation *LLVMDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // a builtin zero attribute and thus will materialize as a llvm.mlir.constant.
   if (auto symbol = dyn_cast<FlatSymbolRefAttr>(value))
     if (isa<LLVM::LLVMPointerType>(type))
-      return builder.create<LLVM::AddressOfOp>(loc, type, symbol);
+      return LLVM::AddressOfOp::create(builder, loc, type, symbol);
   if (isa<LLVM::UndefAttr>(value))
-    return builder.create<LLVM::UndefOp>(loc, type);
+    return LLVM::UndefOp::create(builder, loc, type);
   if (isa<LLVM::PoisonAttr>(value))
-    return builder.create<LLVM::PoisonOp>(loc, type);
+    return LLVM::PoisonOp::create(builder, loc, type);
   if (isa<LLVM::ZeroAttr>(value))
-    return builder.create<LLVM::ZeroOp>(loc, type);
+    return LLVM::ZeroOp::create(builder, loc, type);
   // Otherwise try materializing it as a regular llvm.mlir.constant op.
   return LLVM::ConstantOp::materialize(builder, value, type, loc);
 }
@@ -4313,15 +4313,15 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
   MLIRContext *ctx = builder.getContext();
   auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
-  auto global = moduleBuilder.create<LLVM::GlobalOp>(
+  auto global = LLVM::GlobalOp::create(moduleBuilder,
       loc, type, /*isConstant=*/true, linkage, name,
       builder.getStringAttr(value), /*alignment=*/0);
 
   LLVMPointerType ptrType = LLVMPointerType::get(ctx);
   // Get the pointer to the first character in the global string.
   Value globalPtr =
-      builder.create<LLVM::AddressOfOp>(loc, ptrType, global.getSymNameAttr());
-  return builder.create<LLVM::GEPOp>(loc, ptrType, type, globalPtr,
+      LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
+  return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
                                      ArrayRef<GEPArg>{0, 0});
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index bc451f8b028fc..98fb9b09f0cd5 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -37,7 +37,7 @@ llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() {
 
 Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot,
                                       OpBuilder &builder) {
-  return builder.create<LLVM::UndefOp>(getLoc(), slot.elemType);
+  return LLVM::UndefOp::create(builder, getLoc(), slot.elemType);
 }
 
 void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
@@ -45,7 +45,7 @@ void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
                                          OpBuilder &builder) {
   for (Operation *user : getOperation()->getUsers())
     if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
-      builder.create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
+      LLVM::DbgValueOp::create(builder, declareOp.getLoc(), argument,
                                        declareOp.getVarInfo(),
                                        declareOp.getLocationExpr());
 }
@@ -89,7 +89,7 @@ DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
   for (Attribute index : usedIndices) {
     Type elemType = destructurableType.getTypeAtIndex(index);
     assert(elemType && "used index must exist");
-    auto subAlloca = builder.create<LLVM::AllocaOp>(
+    auto subAlloca = LLVM::AllocaOp::create(builder,
         getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
         getArraySize());
     newAllocators.push_back(subAlloca);
@@ -260,13 +260,13 @@ static Value createExtractAndCast(OpBuilder &builder, Location loc,
   // Truncate the integer if the size of the target is less than the value.
   if (isBigEndian(dataLayout)) {
     uint64_t shiftAmount = srcTypeSize - targetTypeSize;
-    auto shiftConstant = builder.create<LLVM::ConstantOp>(
+    auto shiftConstant = LLVM::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(srcType, shiftAmount));
     replacement =
         builder.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
   }
 
-  replacement = builder.create<LLVM::TruncOp>(
+  replacement = LLVM::TruncOp::create(builder,
       loc, builder.getIntegerType(targetTypeSize), replacement);
 
   // Now cast the integer to the actual target type if required.
@@ -304,7 +304,7 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
     // On big endian systems, a store to the base pointer overwrites the most
     // significant bits. To accomodate for this, the stored value needs to be
     // shifted into the according position.
-    Value bigEndianShift = builder.create<LLVM::ConstantOp>(
+    Value bigEndianShift = LLVM::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
     valueAsInt =
         builder.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
@@ -325,7 +325,7 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
   }
 
   // Mask out the affected bits ...
-  Value mask = builder.create<LLVM::ConstantOp>(
+  Value mask = LLVM::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
   Value masked = builder.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
 
@@ -644,7 +644,7 @@ DeletionKind LLVM::DbgValueOp::removeBlockingUses(
   // debug local variable info. This allows the debugger to inform the user that
   // the variable has been optimized out.
   auto undef =
-      builder.create<UndefOp>(getValue().getLoc(), getValue().getType());
+      UndefOp::create(builder, getValue().getLoc(), getValue().getType());
   getValueMutable().assign(undef);
   return DeletionKind::Keep;
 }
@@ -655,7 +655,7 @@ void LLVM::DbgDeclareOp::visitReplacedValues(
     ArrayRef<std::pair<Operation *, Value>> definitions, OpBuilder &builder) {
   for (auto [op, value] : definitions) {
     builder.setInsertionPointAfter(op);
-    builder.create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(),
+    LLVM::DbgValueOp::create(builder, getLoc(), value, getVarInfo(),
                                      getLocationExpr());
   }
 }
@@ -978,7 +978,7 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetOp toReplace,
               IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize))
           .getResult();
 
-  builder.create<LLVM::MemsetOp>(toReplace.getLoc(), subslots.at(index).ptr,
+  LLVM::MemsetOp::create(builder, toReplace.getLoc(), subslots.at(index).ptr,
                                  toReplace.getVal(), newMemsetSizeValue,
                                  toReplace.getIsVolatile());
 }
@@ -991,7 +991,7 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
   auto newMemsetSizeValue =
       IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize);
 
-  builder.create<LLVM::MemsetInlineOp>(
+  LLVM::MemsetInlineOp::create(builder,
       toReplace.getLoc(), subslots.at(index).ptr, toReplace.getVal(),
       newMemsetSizeValue, toReplace.getIsVolatile());
 }
@@ -1063,7 +1063,7 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       APInt memsetVal(/*numBits=*/width, /*val=*/0);
       for (unsigned loBit = 0; loBit < width; loBit += 8)
         memsetVal.insertBits(constantPattern.getValue(), loBit);
-      return builder.create<LLVM::ConstantOp>(
+      return LLVM::ConstantOp::create(builder,
           op.getLoc(), IntegerAttr::get(intType, memsetVal));
     }
 
@@ -1075,14 +1075,14 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
     // value and or-ing it with the previous value.
     uint64_t coveredBits = 8;
     Value currentValue =
-        builder.create<LLVM::ZExtOp>(op.getLoc(), intType, op.getVal());
+        LLVM::ZExtOp::create(builder, op.getLoc(), intType, op.getVal());
     while (coveredBits < width) {
       Value shiftBy =
-          builder.create<LLVM::ConstantOp>(op.getLoc(), intType, coveredBits);
+          LLVM::ConstantOp::create(builder, op.getLoc(), intType, coveredBits);
       Value shifted =
-          builder.create<LLVM::ShlOp>(op.getLoc(), currentValue, shiftBy);
+          LLVM::ShlOp::create(builder, op.getLoc(), currentValue, shiftBy);
       currentValue =
-          builder.create<LLVM::OrOp>(op.getLoc(), currentValue, shifted);
+          LLVM::OrOp::create(builder, op.getLoc(), currentValue, shifted);
       coveredBits *= 2;
     }
 
@@ -1094,7 +1094,7 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       })
       .Case([&](FloatType type) -> Value {
         Value intVal = buildMemsetValue(type.getWidth());
-        return builder.create<LLVM::BitcastOp>(op.getLoc(), type, intVal);
+        return LLVM::BitcastOp::create(builder, op.getLoc(), type, intVal);
       })
       .Default([](Type) -> Value {
         llvm_unreachable(
@@ -1282,7 +1282,7 @@ static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot) {
 template <class MemcpyLike>
 static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot,
                              OpBuilder &builder) {
-  return builder.create<LLVM::LoadOp>(op.getLoc(), slot.elemType, op.getSrc());
+  return LLVM::LoadOp::create(builder, op.getLoc(), slot.elemType, op.getSrc());
 }
 
 template <class MemcpyLike>
@@ -1309,7 +1309,7 @@ memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot,
                          const SmallPtrSetImpl<OpOperand *> &blockingUses,
                          OpBuilder &builder, Value reachingDefinition) {
   if (op.loadsFrom(slot))
-    builder.create<LLVM::StoreOp>(op.getLoc(), reachingDefinition, op.getDst());
+    LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition, op.getDst());
   return DeletionKind::Delete;
 }
 
@@ -1354,10 +1354,10 @@ template <class MemcpyLike>
 void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                MemcpyLike toReplace, Value dst, Value src,
                                Type toCpy, bool isVolatile) {
-  Value memcpySize = builder.create<LLVM::ConstantOp>(
+  Value memcpySize = LLVM::ConstantOp::create(builder,
       toReplace.getLoc(), IntegerAttr::get(toReplace.getLen().getType(),
                                            layout.getTypeSize(toCpy)));
-  builder.create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize,
+  MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize,
                              isVolatile);
 }
 
@@ -1367,7 +1367,7 @@ void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                Value src, Type toCpy, bool isVolatile) {
   Type lenType = IntegerType::get(toReplace->getContext(),
                                   toReplace.getLen().getBitWidth());
-  builder.create<LLVM::MemcpyInlineOp>(
+  LLVM::MemcpyInlineOp::create(builder,
       toReplace.getLoc(), dst, src,
       IntegerAttr::get(lenType, layout.getTypeSize(toCpy)), isVolatile);
 }
@@ -1409,7 +1409,7 @@ memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
     SmallVector<LLVM::GEPArg> gepIndices{
         0, static_cast<int32_t>(
                cast<IntegerAttr>(index).getValue().getZExtValue())};
-    Value subslotPtrInOther = builder.create<LLVM::GEPOp>(
+    Value subslotPtrInOther = LLVM::GEPOp::create(builder,
         op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()), slot.elemType,
         isDst ? op.getSrc() : op.getDst(), gepIndices);
 
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
index 6fbb0d24826d0..1b1268da15646 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
@@ -28,13 +28,13 @@ static void addComdat(LLVM::LLVMFuncOp &op, OpBuilder &builder,
     PatternRewriter::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(module.getBody());
     comdatOp =
-        builder.create<mlir::LLVM::ComdatOp>(module.getLoc(), comdatName);
+        mlir::LLVM::ComdatOp::create(builder, module.getLoc(), comdatName);
     symbolTable.insert(comdatOp);
   }
 
   PatternRewriter::InsertionGuard guard(builder);
   builder.setInsertionPointToStart(&comdatOp.getBody().back());
-  auto selectorOp = builder.create<mlir::LLVM::ComdatSelectorOp>(
+  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(builder,
       comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
   op.setComdatAttr(mlir::SymbolRefAttr::get(
       builder.getContext(), comdatName,
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index 7f3afffc9645e..2ecef4b4c8a51 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -113,18 +113,18 @@ handleInlinedAllocas(Operation *call,
     // scope if some are already present in the body of the caller. This is not
     // invalid IR, but LLVM cleans these up in InstCombineCalls.cpp, along with
     // other cases where the stacksave/stackrestore is redundant.
-    stackPtr = builder.create<LLVM::StackSaveOp>(
+    stackPtr = LLVM::StackSaveOp::create(builder,
         call->getLoc(), LLVM::LLVMPointerType::get(call->getContext()));
   }
   builder.setInsertionPointToStart(callerEntryBlock);
   for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
-    auto newConstant = builder.create<LLVM::ConstantOp>(
+    auto newConstant = LLVM::ConstantOp::create(builder,
         allocaOp->getLoc(), allocaOp.getArraySize().getType(), arraySize);
     // Insert a lifetime start intrinsic where the alloca was before moving it.
     if (shouldInsertLifetime) {
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(allocaOp);
-      builder.create<LLVM::LifetimeStartOp>(
+      LLVM::LifetimeStartOp::create(builder,
           allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
           allocaOp.getResult());
     }
@@ -139,10 +139,10 @@ handleInlinedAllocas(Operation *call,
       continue;
     builder.setInsertionPoint(block.getTerminator());
     if (hasDynamicAlloca)
-      builder.create<LLVM::StackRestoreOp>(call->getLoc(), stackPtr);
+      LLVM::StackRestoreOp::create(builder, call->getLoc(), stackPtr);
     for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
       if (shouldInsertLifetime)
-        builder.create<LLVM::LifetimeEndOp>(
+        LLVM::LifetimeEndOp::create(builder,
             allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
             allocaOp.getResult());
     }
@@ -603,15 +603,15 @@ static Value handleByValArgumentInit(OpBuilder &builder, Location loc,
     OpBuilder::InsertionGuard insertionGuard(builder);
     Block *entryBlock = &(*argument.getParentRegion()->begin());
     builder.setInsertionPointToStart(entryBlock);
-    Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
+    Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
                                                  builder.getI64IntegerAttr(1));
-    allocaOp = builder.create<LLVM::AllocaOp>(
+    allocaOp = LLVM::AllocaOp::create(builder,
         loc, argument.getType(), elementType, one, targetAlignment);
   }
   // Copy the pointee to the newly allocated value.
-  Value copySize = builder.create<LLVM::ConstantOp>(
+  Value copySize = LLVM::ConstantOp::create(builder,
       loc, builder.getI64Type(), builder.getI64IntegerAttr(elementTypeSize));
-  builder.create<LLVM::MemcpyOp>(loc, allocaOp, argument, copySize,
+  LLVM::MemcpyOp::create(builder, loc, allocaOp, argument, copySize,
                                  /*isVolatile=*/false);
   return allocaOp;
 }
@@ -747,7 +747,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    builder.create<LLVM::BrOp>(op->getLoc(), returnOp.getOperands(), newDest);
+    LLVM::BrOp::create(builder, op->getLoc(), returnOp.getOperands(), newDest);
     op->erase();
   }
 
@@ -801,7 +801,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
     // and is extremely unlikely to exist in the code prior to inlining, using
     // this to communicate between this method and `processInlinedCallBlocks`.
     // TODO: Fix this by refactoring the inliner interface.
-    auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
+    auto copyOp = LLVM::SSACopyOp::create(builder, call->getLoc(), argument);
     if (argumentAttrs.contains(LLVM::LLVMDialect::getNoAliasAttrName()))
       copyOp->setDiscardableAttr(
           builder.getStringAttr(LLVM::LLVMDialect::getNoAliasAttrName()),
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
index 8dd0c28d98522..32fe2edd6ea76 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
@@ -59,7 +59,7 @@ static void ensureDistinctSuccessors(Block &bb) {
       terminator->setSuccessor(dummyBlock, position);
       for (BlockArgument arg : successor.first->getArguments())
         dummyBlock->addArgument(arg.getType(), arg.getLoc());
-      builder.create<LLVM::BrOp>(terminator->getLoc(),
+      LLVM::BrOp::create(builder, terminator->getLoc(),
                                  dummyBlock->getArguments(), successor.first);
     }
   }
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
index 8db32ec1526c4..de6c3043bbcdd 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
@@ -59,32 +59,32 @@ LogicalResult ExpandDivF16::matchAndRewrite(LLVM::FDivOp op,
   Type i32Type = rewriter.getI32Type();
 
   // Extend lhs and rhs to fp32.
-  Value lhs = rewriter.create<LLVM::FPExtOp>(loc, f32Type, op.getLhs());
-  Value rhs = rewriter.create<LLVM::FPExtOp>(loc, f32Type, op.getRhs());
+  Value lhs = LLVM::FPExtOp::create(rewriter, loc, f32Type, op.getLhs());
+  Value rhs = LLVM::FPExtOp::create(rewriter, loc, f32Type, op.getRhs());
 
   // float rcp = rcp.approx.ftz.f32(rhs), approx = lhs * rcp.
-  Value rcp = rewriter.create<NVVM::RcpApproxFtzF32Op>(loc, f32Type, rhs);
-  Value approx = rewriter.create<LLVM::FMulOp>(loc, lhs, rcp);
+  Value rcp = NVVM::RcpApproxFtzF32Op::create(rewriter, loc, f32Type, rhs);
+  Value approx = LLVM::FMulOp::create(rewriter, loc, lhs, rcp);
 
   // Refine the approximation with one Newton iteration:
   // float refined = approx + (lhs - approx * rhs) * rcp;
-  Value err = rewriter.create<LLVM::FMAOp>(
-      loc, approx, rewriter.create<LLVM::FNegOp>(loc, rhs), lhs);
-  Value refined = rewriter.create<LLVM::FMAOp>(loc, err, rcp, approx);
+  Value err = LLVM::FMAOp::create(rewriter,
+      loc, approx, LLVM::FNegOp::create(rewriter, loc, rhs), lhs);
+  Value refined = LLVM::FMAOp::create(rewriter, loc, err, rcp, approx);
 
   // Use refined value if approx is normal (exponent neither all 0 or all 1).
-  Value mask = rewriter.create<LLVM::ConstantOp>(
+  Value mask = LLVM::ConstantOp::create(rewriter,
       loc, i32Type, rewriter.getUI32IntegerAttr(0x7f800000));
-  Value cast = rewriter.create<LLVM::BitcastOp>(loc, i32Type, approx);
-  Value exp = rewriter.create<LLVM::AndOp>(loc, i32Type, cast, mask);
-  Value zero = rewriter.create<LLVM::ConstantOp>(
+  Value cast = LLVM::BitcastOp::create(rewriter, loc, i32Type, approx);
+  Value exp = LLVM::AndOp::create(rewriter, loc, i32Type, cast, mask);
+  Value zero = LLVM::ConstantOp::create(rewriter,
       loc, i32Type, rewriter.getUI32IntegerAttr(0));
-  Value pred = rewriter.create<LLVM::OrOp>(
+  Value pred = LLVM::OrOp::create(rewriter,
       loc,
-      rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, exp, zero),
-      rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, exp, mask));
+      LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, zero),
+      LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, mask));
   Value result =
-      rewriter.create<LLVM::SelectOp>(loc, f32Type, pred, approx, refined);
+      LLVM::SelectOp::create(rewriter, loc, f32Type, pred, approx, refined);
 
   // Replace with trucation back to fp16.
   rewriter.replaceOpWithNewOp<LLVM::FPTruncOp>(op, op.getType(), result);
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 3aa6ac3ea0918..51cc1232df891 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -62,10 +62,10 @@ static OpFoldResult getDimValue(OpBuilder &builder, Location loc, Value v,
   return getAsOpFoldResult(
       TypeSwitch<Type, Value>(v.getType())
           .Case<RankedTensorType>([&](RankedTensorType t) -> Value {
-            return builder.create<tensor::DimOp>(loc, v, dim);
+            return tensor::DimOp::create(builder, loc, v, dim);
           })
           .Case<MemRefType>([&](MemRefType t) -> Value {
-            return builder.create<memref::DimOp>(loc, v, dim);
+            return memref::DimOp::create(builder, loc, v, dim);
           }));
 }
 
@@ -77,11 +77,11 @@ static Operation *getSlice(OpBuilder &b, Location loc, Value source,
                            ArrayRef<OpFoldResult> strides) {
   return TypeSwitch<Type, Operation *>(source.getType())
       .Case<RankedTensorType>([&](RankedTensorType t) -> Operation * {
-        return b.create<tensor::ExtractSliceOp>(loc, source, offsets, sizes,
+        return tensor::ExtractSliceOp::create(b, loc, source, offsets, sizes,
                                                 strides);
       })
       .Case<MemRefType>([&](MemRefType type) -> Operation * {
-        return b.create<memref::SubViewOp>(loc, source, offsets, sizes,
+        return memref::SubViewOp::create(b, loc, source, offsets, sizes,
                                            strides);
       })
       .Default([&](Type t) -> Operation * { return nullptr; });
@@ -453,35 +453,35 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     switch (unaryFn) {
     case UnaryFn::exp:
-      return builder.create<math::ExpOp>(arg.getLoc(), arg);
+      return math::ExpOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::log:
-      return builder.create<math::LogOp>(arg.getLoc(), arg);
+      return math::LogOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::abs:
-      return builder.create<math::AbsFOp>(arg.getLoc(), arg);
+      return math::AbsFOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::ceil:
-      return builder.create<math::CeilOp>(arg.getLoc(), arg);
+      return math::CeilOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::floor:
-      return builder.create<math::FloorOp>(arg.getLoc(), arg);
+      return math::FloorOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::negf:
-      return builder.create<arith::NegFOp>(arg.getLoc(), arg);
+      return arith::NegFOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::reciprocal: {
       Attribute oneAttr = builder.getOneAttr(arg.getType());
-      auto one = builder.create<arith::ConstantOp>(arg.getLoc(),
+      auto one = arith::ConstantOp::create(builder, arg.getLoc(),
                                                    ::cast<TypedAttr>(oneAttr));
-      return builder.create<arith::DivFOp>(arg.getLoc(), one, arg);
+      return arith::DivFOp::create(builder, arg.getLoc(), one, arg);
     }
     case UnaryFn::round:
-      return builder.create<math::RoundOp>(arg.getLoc(), arg);
+      return math::RoundOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::sqrt:
-      return builder.create<math::SqrtOp>(arg.getLoc(), arg);
+      return math::SqrtOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::rsqrt:
-      return builder.create<math::RsqrtOp>(arg.getLoc(), arg);
+      return math::RsqrtOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::square:
-      return builder.create<arith::MulFOp>(arg.getLoc(), arg, arg);
+      return arith::MulFOp::create(builder, arg.getLoc(), arg, arg);
     case UnaryFn::tanh:
-      return builder.create<math::TanhOp>(arg.getLoc(), arg);
+      return math::TanhOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::erf:
-      return builder.create<math::ErfOp>(arg.getLoc(), arg);
+      return math::ErfOp::create(builder, arg.getLoc(), arg);
     }
     if (emitError) {
       emitError() << "unsupported unary function";
@@ -516,17 +516,17 @@ class RegionBuilderHelper {
     switch (binaryFn) {
     case BinaryFn::add:
       if (allComplex)
-        return builder.create<complex::AddOp>(arg0.getLoc(), arg0, arg1);
+        return complex::AddOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::AddFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::AddFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool)
-        return builder.create<arith::OrIOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::AddIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::OrIOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::AddIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::sub:
       if (allComplex)
-        return builder.create<complex::SubOp>(arg0.getLoc(), arg0, arg1);
+        return complex::SubOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::SubFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::SubFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool) {
         if (emitError) {
           emitError() << "unsupported operation: sub with bools";
@@ -534,20 +534,20 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: sub with bools");
       }
-      return builder.create<arith::SubIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::SubIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::mul:
       if (allComplex)
-        return builder.create<complex::MulOp>(arg0.getLoc(), arg0, arg1);
+        return complex::MulOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::MulFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MulFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool)
-        return builder.create<arith::AndIOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MulIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::AndIOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MulIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::div:
       if (allComplex)
-        return builder.create<complex::DivOp>(arg0.getLoc(), arg0, arg1);
+        return complex::DivOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::DivFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::DivFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool) {
         if (emitError) {
           emitError() << "unsupported operation: div with bools";
@@ -555,7 +555,7 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: div with bools");
       }
-      return builder.create<arith::DivSIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::DivSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::div_unsigned:
       if (!allInteger || allBool) {
         if (emitError) {
@@ -564,30 +564,30 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: unsigned div not on uint");
       }
-      return builder.create<arith::DivUIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::DivUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_signed:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MaximumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MaxSIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MaximumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MaxSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::min_signed:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MinimumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MinSIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MinimumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MinSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_unsigned:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MaximumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MaxUIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MaximumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MaxUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::min_unsigned:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MinimumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MinUIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MinimumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MinUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::powf:
       assert(allFloatingPoint);
-      return builder.create<math::PowFOp>(arg0.getLoc(), arg0, arg1);
+      return math::PowFOp::create(builder, arg0.getLoc(), arg0, arg1);
     }
     if (emitError) {
       emitError() << "unsupported binary function";
@@ -610,7 +610,7 @@ class RegionBuilderHelper {
     case TernaryFn::select:
       if (!headBool && !(tailFloatingPoint || tailInteger))
         llvm_unreachable("unsupported non numeric type");
-      return builder.create<arith::SelectOp>(arg0.getLoc(), arg0, arg1, arg2);
+      return arith::SelectOp::create(builder, arg0.getLoc(), arg0, arg1, arg2);
     }
     if (emitError) {
       emitError() << "unsupported ternary function";
@@ -639,7 +639,7 @@ class RegionBuilderHelper {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
-    builder.create<YieldOp>(loc, values);
+    YieldOp::create(builder, loc, values);
   }
 
   Value constant(const std::string &value) {
@@ -647,13 +647,13 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
     Attribute valueAttr = parseAttribute(value, builder.getContext());
-    return builder.create<arith::ConstantOp>(loc, ::cast<TypedAttr>(valueAttr));
+    return arith::ConstantOp::create(builder, loc, ::cast<TypedAttr>(valueAttr));
   }
 
   Value index(int64_t dim) {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
-    return builder.create<IndexOp>(builder.getUnknownLoc(), dim);
+    return IndexOp::create(builder, builder.getUnknownLoc(), dim);
   }
 
   Type getIntegerType(unsigned width) {
@@ -749,12 +749,12 @@ struct FoldFillWithTensorReshape : OpRewritePattern<TensorReshapeOp> {
     TensorReshapeOp newInit;
     if constexpr (std::is_same<TensorReshapeOp, tensor::ExpandShapeOp>::value) {
 
-      newInit = rewriter.create<TensorReshapeOp>(
+      newInit = TensorReshapeOp::create(rewriter,
           loc, reshapeOp.getResultType(), oldFill.output(),
           reshapeOp.getReassociation(), reshapeOp.getOutputShape(),
           reshapeOp.getStaticOutputShape());
     } else {
-      newInit = rewriter.create<TensorReshapeOp>(loc, reshapeOp.getResultType(),
+      newInit = TensorReshapeOp::create(rewriter, loc, reshapeOp.getResultType(),
                                                  oldFill.output(),
                                                  reshapeOp.getReassociation());
     }
@@ -786,7 +786,7 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
       return rewriter.notifyMatchFailure(
           padOp, "failed to reify tensor.pad op result shape");
 
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         padOp.getLoc(), reifiedShape.front(),
         padOp.getResultType().getElementType());
     Value replacement =
@@ -795,7 +795,7 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
                             ValueRange{emptyTensor})
             .getResult(0);
     if (replacement.getType() != padOp.getResultType()) {
-      replacement = rewriter.create<tensor::CastOp>(
+      replacement = tensor::CastOp::create(rewriter,
           fillOp.getLoc(), padOp.getResultType(), replacement);
     }
     rewriter.replaceOp(padOp, replacement);
@@ -889,7 +889,7 @@ struct FoldInsertPadIntoFill : public OpRewritePattern<tensor::InsertSliceOp> {
     for (int i = 0, e = srcPadType.getRank(); i < e; ++i) {
       if (srcPadType.isDynamicDim(i)) {
         newSizes.push_back(
-            rewriter.create<tensor::DimOp>(loc, srcPadOp.getSource(), i)
+            tensor::DimOp::create(rewriter, loc, srcPadOp.getSource(), i)
                 .getResult());
       } else {
         newSizes.push_back(rewriter.getIndexAttr(srcPadType.getDimSize(i)));
@@ -942,7 +942,7 @@ static FailureOr<FillOp> foldFillPackIntoFillOp(RewriterBase &rewriter,
   if (!packOpDest.hasOneUse())
     return failure();
 
-  return rewriter.create<linalg::FillOp>(packOp.getLoc(), fillOp.getInputs(),
+  return linalg::FillOp::create(rewriter, packOp.getLoc(), fillOp.getInputs(),
                                          packOp.getDest());
 }
 
@@ -1042,7 +1042,7 @@ struct FoldConcatsOfFill : public OpRewritePattern<tensor::ConcatOp> {
           concatOp, "not all operands are defined by a compatible fill op");
     }
 
-    Value outsConcat = rewriter.create<tensor::ConcatOp>(
+    Value outsConcat = tensor::ConcatOp::create(rewriter,
         concatOp.getLoc(), concatOp.getDim(), allOuts);
     rewriter.replaceOpWithNewOp<linalg::FillOp>(
         concatOp, firstFillOp.getDpsInputOperand(0)->get(), outsConcat);
@@ -1407,13 +1407,13 @@ struct EraseIdentityLinalgOp : public OpRewritePattern<OpTy> {
         // TODO: unify the two ops?
         if (sparse_tensor::getSparseTensorEncoding(returnType) ||
             sparse_tensor::getSparseTensorEncoding(resultType))
-          returnedArg = rewriter.create<sparse_tensor::ConvertOp>(
+          returnedArg = sparse_tensor::ConvertOp::create(rewriter,
               linalgOp.getLoc(), resultType, returnedArg);
         else {
           if (!tensor::CastOp::areCastCompatible(returnedArg.getType(),
                                                  resultType))
             return failure();
-          returnedArg = rewriter.create<tensor::CastOp>(
+          returnedArg = tensor::CastOp::create(rewriter,
               linalgOp.getLoc(), resultType, returnedArg);
         }
       }
@@ -1528,7 +1528,7 @@ static void addBodyWithPayloadOp(OpAsmParser &parser, OperationState &result,
       TypeRange{llvm::cast<ShapedType>(result.operands.back().getType())
                     .getElementType()},
       payloadOpAttrs);
-  b.create<YieldOp>(result.location, payloadOp->getResults());
+  YieldOp::create(b, result.location, payloadOp->getResults());
 }
 
 ParseResult MapOp::parse(OpAsmParser &parser, OperationState &result) {
@@ -1945,7 +1945,7 @@ static void buildIdentityRegion(OpBuilder &builder, Location loc,
   buildGenericRegion(builder, loc, region, inputs, outputs,
                      [](OpBuilder &b, Location loc, ValueRange args) {
                        if (!args.empty())
-                         b.create<linalg::YieldOp>(loc, args[0]);
+                         linalg::YieldOp::create(b, loc, args[0]);
                      });
 }
 
@@ -2138,7 +2138,7 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     unsigned inputRank = broadcastInputTy.getRank();
     for (unsigned i = 0; i < inputRank; ++i) {
       if (broadcastInputTy.isDynamicDim(i)) {
-        dims.push_back(rewriter.create<tensor::DimOp>(loc, broadcastInput, i)
+        dims.push_back(tensor::DimOp::create(rewriter, loc, broadcastInput, i)
                            ->getResult(0));
       } else {
         dims.push_back(IntegerAttr::get(IndexType::get(ctx),
@@ -2147,7 +2147,7 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     }
     SmallVector<OpFoldResult> transposeResultShapes =
         applyPermutation(dims, resultPerms);
-    Value transposeInit = rewriter.create<tensor::EmptyOp>(
+    Value transposeInit = tensor::EmptyOp::create(rewriter,
         transposeOp.getLoc(), transposeResultShapes,
         broadcastInputTy.getElementType());
 
@@ -2547,7 +2547,7 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     // continue to propagate as far up the stack as it can go.
     OpOperand *outOperand = linalgOp.getDpsInitOperand(resultNumber);
     Value newOperand =
-        rewriter.create<tensor::CastOp>(loc, resultType, outOperand->get());
+        tensor::CastOp::create(rewriter, loc, resultType, outOperand->get());
     SmallVector<Value> newOperands = linalgOp.getDpsInputs();
     SmallVector<Value> outputOperands(linalgOp.getDpsInits().begin(),
                                       linalgOp.getDpsInits().end());
@@ -2560,7 +2560,7 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     Operation *newOp = clone(rewriter, linalgOp, resultTypes, newOperands);
 
     // Create a tensor.cast operation back to the original type.
-    Value castBack = rewriter.create<tensor::CastOp>(
+    Value castBack = tensor::CastOp::create(rewriter,
         loc, resultValue.getType(), newOp->getResult(resultNumber));
 
     SmallVector<Value> results(newOp->result_begin(), newOp->result_end());
@@ -2653,7 +2653,7 @@ static void createNewOperandWithStaticSizes(
     changeNeeded = true;
     // Get the new operand value given its size and element type by
     // casting it.
-    Value newOperand = rewriter.create<tensor::CastOp>(loc, resultType, src);
+    Value newOperand = tensor::CastOp::create(rewriter, loc, resultType, src);
     unsigned index = opOperand->getOperandNumber();
     newOperands[index] = newOperand;
   }
@@ -2718,7 +2718,7 @@ struct InferStaticShapeOfOperands : public OpInterfaceRewritePattern<LinalgOp> {
       Type oldType = oldResult.getType();
       replacements.push_back(
           (newType != oldType)
-              ? rewriter.create<tensor::CastOp>(loc, oldType, newResult)
+              ? tensor::CastOp::create(rewriter, loc, oldType, newResult)
               : newResult);
     }
     rewriter.replaceOp(linalgOp, replacements);
@@ -2756,8 +2756,8 @@ SmallVector<Range> SoftmaxOp::getIterationDomain(OpBuilder &builder) {
   int64_t operandRank = getInputOperandRank();
   SmallVector<Range> loopBounds(operandRank);
   Location loc = getLoc();
-  Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-  Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+  Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+  Value one = arith::ConstantIndexOp::create(builder, loc, 1);
   Value source = getInput();
   for (auto dim : llvm::seq<int64_t>(0, operandRank)) {
     loopBounds[dim].offset = zero;
@@ -2924,11 +2924,11 @@ static Value reduce(OpBuilder &builder, Location loc, Value input, Value output,
          "We should have two maps: 1 for the input, 1 for the output");
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
 
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, output.getType(), input, output, indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value result = b.create<T>(loc, args[0], args[1]);
-        b.create<linalg::YieldOp>(loc, result);
+        Value result = T::create(b, loc, args[0], args[1]);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -2947,12 +2947,12 @@ static Value buildSubAndExpOp(OpBuilder &builder, Location loc, Value input,
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
   // Add the affine map for the output argument.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, input.getType(), ValueRange{input, max}, output, indexingMaps,
       iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value diff = b.create<arith::SubFOp>(loc, args[0], args[1]);
-        Value result = b.create<math::ExpOp>(loc, diff);
-        b.create<linalg::YieldOp>(loc, result);
+        Value diff = arith::SubFOp::create(b, loc, args[0], args[1]);
+        Value result = math::ExpOp::create(b, loc, diff);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -2974,12 +2974,12 @@ static Value buildDivOp(OpBuilder &builder, Location loc, Value numerator,
   assert(indexingMaps[0].isIdentity() && "Numerator map should be identity");
   // Add the affine map for the output tensor.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, numerator.getType(), ValueRange{numerator, denominator}, output,
       indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value result = b.create<arith::DivFOp>(loc, args[0], args[1]);
-        b.create<linalg::YieldOp>(loc, result);
+        Value result = arith::DivFOp::create(b, loc, args[0], args[1]);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -3015,12 +3015,12 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
   Value output = getOutput();
   dims.erase(dims.begin() + reductionDim);
   // Step 1: Compute max along dim.
-  Value outputReduce = b.create<tensor::EmptyOp>(loc, dims, elementType);
+  Value outputReduce = tensor::EmptyOp::create(b, loc, dims, elementType);
   Value neutralForMaxF = arith::getIdentityValue(arith::AtomicRMWKind::maxnumf,
                                                  elementType, b, loc,
                                                  /*useOnlyFiniteValue=*/true);
   Value neutralForMaxFInit =
-      b.create<linalg::FillOp>(loc, Value{neutralForMaxF}, outputReduce)
+      linalg::FillOp::create(b, loc, Value{neutralForMaxF}, outputReduce)
           .result();
   Value max =
       reduce<arith::MaxNumFOp>(b, loc, input, neutralForMaxFInit, reductionDim);
@@ -3032,7 +3032,7 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
   Value zero = arith::getIdentityValue(arith::AtomicRMWKind::addf, elementType,
                                        b, loc, /*useOnlyFiniteValue=*/true);
   Value zeroInit =
-      b.create<linalg::FillOp>(loc, Value{zero}, outputReduce).result();
+      linalg::FillOp::create(b, loc, Value{zero}, outputReduce).result();
   Value denominator =
       reduce<arith::AddFOp>(b, loc, numerator, zeroInit, reductionDim);
 
@@ -3153,7 +3153,7 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
   int64_t filterRank = getFilterOperandRank();
   SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
   Location loc = getLoc();
-  auto filterSlice = builder.create<tensor::ExtractSliceOp>(
+  auto filterSlice = tensor::ExtractSliceOp::create(builder,
       loc, getFilter(), sliceOffsets, sliceSizes, filterStrides);
   tiledOperands.emplace_back(filterSlice);
 
@@ -3164,7 +3164,7 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -3333,7 +3333,7 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
       {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
   int64_t inputRank = getInputOperandRank();
   SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
-  auto inputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto inputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getInput(), sliceOffsets, sliceSizes, inputStrides);
   tiledOperands.emplace_back(inputSlice);
 
@@ -3344,7 +3344,7 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -3504,7 +3504,7 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
                      sizes[getValueFDim()]});
   int64_t valueRank = getValueOperandRank();
   SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
-  auto valueSlice = builder.create<tensor::ExtractSliceOp>(
+  auto valueSlice = tensor::ExtractSliceOp::create(builder,
       loc, getValue(), sliceOffsets, sliceSizes, sliceStrides);
   tiledOperands.emplace_back(valueSlice);
 
@@ -3515,7 +3515,7 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> strides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, strides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -4971,7 +4971,7 @@ Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
            llvm::cast<RankedTensorType>(source.getType()).getShape())) {
     if (ShapedType::isDynamic(value))
       mixedSizes.push_back(
-          b.create<tensor::DimOp>(loc, source, index).getResult());
+          tensor::DimOp::create(b, loc, source, index).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(value));
   }
@@ -4985,7 +4985,7 @@ Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
 
   mixedSizes.append(innerTileSizes.begin(), innerTileSizes.end());
   auto elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
-  return b.create<tensor::EmptyOp>(loc, mixedSizes, elemType);
+  return tensor::EmptyOp::create(b, loc, mixedSizes, elemType);
 }
 
 PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
@@ -4996,7 +4996,7 @@ PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
   Value transposedDest =
       createDestinationTensor(b, loc, getSource(), metadata.innerTiles,
                               metadata.innerDimsPos, metadata.outerDimsPerm);
-  return b.create<PackOp>(loc, getSource(), transposedDest,
+  return PackOp::create(b, loc, getSource(), transposedDest,
                           metadata.innerDimsPos, metadata.innerTiles,
                           getPaddingValue(), metadata.outerDimsPerm);
 }
@@ -5138,7 +5138,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (srcShape != packOp.getSourceType().getShape()) {
       auto newSrcType = packOp.getSourceType().clone(srcShape);
       source =
-          rewriter.create<tensor::CastOp>(loc, newSrcType, packOp.getSource());
+          tensor::CastOp::create(rewriter, loc, newSrcType, packOp.getSource());
     }
     Value dest = packOp.getDest();
     RankedTensorType originalResultType = packOp.getDestType();
@@ -5146,7 +5146,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (needUpdateDestType) {
       auto newDestType = packOp.getDestType().clone(destShape);
       dest =
-          rewriter.create<tensor::CastOp>(loc, newDestType, packOp.getDest());
+          tensor::CastOp::create(rewriter, loc, newDestType, packOp.getDest());
     }
     rewriter.modifyOpInPlace(packOp, [&] {
       packOp.getSourceMutable().assign(source);
@@ -5157,7 +5157,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (needUpdateDestType) {
       rewriter.setInsertionPointAfter(packOp);
       auto castOp =
-          rewriter.create<tensor::CastOp>(loc, originalResultType, packOp);
+          tensor::CastOp::create(rewriter, loc, originalResultType, packOp);
       rewriter.replaceAllUsesExcept(packOp, castOp, castOp);
     }
     return success();
@@ -5250,7 +5250,7 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    PackOp newOp = rewriter.create<PackOp>(
+    PackOp newOp = PackOp::create(rewriter,
         op.getLoc(), newOperands[0], newOperands[1], op.getInnerDimsPos(),
         newMixedTileSizes, op.getPaddingValue(), op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
@@ -5259,7 +5259,7 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
     Value replacement = (newResult.getType() != oldResult.getType())
-                            ? rewriter.create<tensor::CastOp>(
+                            ? tensor::CastOp::create(rewriter,
                                   op->getLoc(), oldResult.getType(), newResult)
                             : newResult;
 
@@ -5358,7 +5358,7 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
   for (auto i :
        llvm::seq<unsigned>(0, srcType.getRank() - innerTileSizes.size())) {
     if (srcType.isDynamicDim(i))
-      mixedSizes.push_back(b.create<tensor::DimOp>(loc, source, i).getResult());
+      mixedSizes.push_back(tensor::DimOp::create(b, loc, source, i).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(srcType.getDimSize(i)));
   }
@@ -5371,7 +5371,7 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
     mixedSizes[dimPos] = dimMul(mixedSizes[dimPos], tileSize);
 
   auto elemType = srcType.getElementType();
-  return b.create<tensor::EmptyOp>(loc, mixedSizes, elemType);
+  return tensor::EmptyOp::create(b, loc, mixedSizes, elemType);
 }
 
 UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
@@ -5380,7 +5380,7 @@ UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
                                          ArrayRef<int64_t> outerPermutation) {
   PackOrUnPackTransposeResult metadata = commonPermutationOfPackAndUnPackOp(
       *this, innerPermutation, outerPermutation);
-  return b.create<UnPackOp>(loc, transposedSource, getDest(),
+  return UnPackOp::create(b, loc, transposedSource, getDest(),
                             metadata.innerDimsPos, metadata.innerTiles,
                             metadata.outerDimsPerm);
 }
@@ -5454,7 +5454,7 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
             extractSliceUser.getResultType().getRank()) {
       OpBuilder::InsertionGuard g(rewriter);
       rewriter.setInsertionPoint(unPackOp);
-      auto newDest = rewriter.create<tensor::ExtractSliceOp>(
+      auto newDest = tensor::ExtractSliceOp::create(rewriter,
           unPackOp->getLoc(), unPackOp.getDest(),
           extractSliceUser.getMixedOffsets(), extractSliceUser.getMixedSizes(),
           extractSliceUser.getMixedStrides());
@@ -5474,16 +5474,16 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
     Value source = unPackOp.getSource();
     if (srcShape != unPackOp.getSourceType().getShape()) {
       auto newSrcType = unPackOp.getSourceType().clone(srcShape);
-      source = rewriter.create<tensor::CastOp>(loc, newSrcType,
+      source = tensor::CastOp::create(rewriter, loc, newSrcType,
                                                unPackOp.getSource());
     }
     Value dest = unPackOp.getDest();
     if (destShape != unPackOp.getDestType().getShape()) {
       auto newDestType = unPackOp.getDestType().clone(destShape);
       dest =
-          rewriter.create<tensor::CastOp>(loc, newDestType, unPackOp.getDest());
+          tensor::CastOp::create(rewriter, loc, newDestType, unPackOp.getDest());
     }
-    Value newOp = rewriter.create<UnPackOp>(
+    Value newOp = UnPackOp::create(rewriter,
         loc, source, dest, unPackOp.getInnerDimsPos(), unPackOp.getMixedTiles(),
         unPackOp.getOuterDimsPerm());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(
@@ -5542,7 +5542,7 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    UnPackOp newOp = rewriter.create<UnPackOp>(
+    UnPackOp newOp = UnPackOp::create(rewriter,
         op.getLoc(), sourceTensor, newOperands[1], op.getInnerDimsPos(),
         newMixedTileSizes, op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
@@ -5551,7 +5551,7 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
     Value replacement = (newResult.getType() != oldResult.getType())
-                            ? rewriter.create<tensor::CastOp>(
+                            ? tensor::CastOp::create(rewriter,
                                   op->getLoc(), oldResult.getType(), newResult)
                             : newResult;
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 5d5f9de465561..ba03d09e5072e 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -672,7 +672,7 @@ static Operation *replaceForAllWithNewSignature(
   newOuts.push_back(outputs[resultNumber]);
 
   // Create new scf.forall op
-  auto newforallOp = rewriter.create<scf::ForallOp>(
+  auto newforallOp = scf::ForallOp::create(rewriter,
       loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
       forallOp.getMixedStep(), newOuts, forallOp.getMapping());
   rewriter.eraseBlock(newforallOp.getBody());
@@ -699,7 +699,7 @@ static Operation *replaceForAllWithNewSignature(
   Value src = tileAndFuseResult.tiledValues[0];
   Value dst = newforallOp.getRegionIterArgs().back();
   SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-  rewriter.create<tensor::ParallelInsertSliceOp>(firstYieldOp->getLoc(), src,
+  tensor::ParallelInsertSliceOp::create(rewriter, firstYieldOp->getLoc(), src,
                                                  dst, offsets, sizes, strides);
 
   for (auto result : llvm::enumerate(forallOp.getResults())) {
@@ -3410,12 +3410,12 @@ transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
         for (auto [ofrIdx, ofr] : llvm::enumerate(getMixedSizes())) {
           if (auto attr = llvm::dyn_cast_if_present<Attribute>(ofr)) {
             if (scalableSizes[ofrIdx]) {
-              auto val = b.create<arith::ConstantIndexOp>(
+              auto val = arith::ConstantIndexOp::create(b,
                   getLoc(), cast<IntegerAttr>(attr).getInt());
               Value vscale =
-                  b.create<vector::VectorScaleOp>(getLoc(), b.getIndexType());
+                  vector::VectorScaleOp::create(b, getLoc(), b.getIndexType());
               sizes.push_back(
-                  b.create<arith::MulIOp>(getLoc(), val, vscale).getResult());
+                  arith::MulIOp::create(b, getLoc(), val, vscale).getResult());
             } else {
               sizes.push_back(attr);
             }
@@ -3626,7 +3626,7 @@ static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter,
   SmallVector<OpFoldResult> normalizedSteps(normalizedUbs.size(),
                                             rewriter.getIndexAttr(1));
 
-  auto normalizedForallOp = rewriter.create<scf::ForallOp>(
+  auto normalizedForallOp = scf::ForallOp::create(rewriter,
       loc, normalizedLbs, normalizedUbs, normalizedSteps, loop.getOutputs(),
       loop.getMapping(), [](OpBuilder &, Location, ValueRange) {});
 
@@ -4128,7 +4128,7 @@ DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target,
         target->template getParentOfType<scf::InParallelOp>());
   }
 
-  Value extracted = rewriter.create<tensor::ExtractSliceOp>(
+  Value extracted = tensor::ExtractSliceOp::create(rewriter,
       target.getLoc(), target.getDest(), target.getMixedOffsets(),
       target.getMixedSizes(), target.getMixedStrides());
   Value copied = rewriter
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
index 1f6d96ca0f81f..2cb8ad367f93c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -184,7 +184,7 @@ struct SoftmaxOpInterface
         getBuffer(rewriter, softmaxOp.getOutput(), options, state);
     if (failed(outputBuffer))
       return failure();
-    rewriter.create<linalg::SoftmaxOp>(softmaxOp.getLoc(),
+    linalg::SoftmaxOp::create(rewriter, softmaxOp.getLoc(),
                                        /*result=*/TypeRange(), *inputBuffer,
                                        *outputBuffer, softmaxOp.getDimension());
     replaceOpWithBufferizedValues(rewriter, op, *outputBuffer);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index a7732b939e70d..61c779850366f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -30,10 +30,10 @@ static bool hasAllOneValues(DenseIntElementsAttr attr) {
 
 static Value createAdd(Location loc, Value x, Value y, OpBuilder &builder) {
   if (isa<IntegerType>(x.getType()))
-    return builder.create<arith::AddIOp>(loc, x, y);
+    return arith::AddIOp::create(builder, loc, x, y);
   if (isa<ComplexType>(x.getType()))
-    return builder.create<complex::AddOp>(loc, x, y);
-  return builder.create<arith::AddFOp>(loc, x, y);
+    return complex::AddOp::create(builder, loc, x, y);
+  return arith::AddFOp::create(builder, loc, x, y);
 }
 
 static Value createMul(Location loc, Value x, Value y, Type accType,
@@ -44,10 +44,10 @@ static Value createMul(Location loc, Value x, Value y, Type accType,
   Value yConvert =
       convertScalarToDtype(builder, loc, y, accType, /*isUnsignedCast=*/false);
   if (isa<ComplexType>(accType))
-    return builder.create<complex::MulOp>(loc, xConvert, yConvert);
+    return complex::MulOp::create(builder, loc, xConvert, yConvert);
   if (isa<IntegerType>(accType))
-    return builder.create<arith::MulIOp>(loc, xConvert, yConvert);
-  return builder.create<arith::MulFOp>(loc, xConvert, yConvert);
+    return arith::MulIOp::create(builder, loc, xConvert, yConvert);
+  return arith::MulFOp::create(builder, loc, xConvert, yConvert);
 }
 
 // Delinearizes the given composite `index` by the basis specified in `factors`.
@@ -56,7 +56,7 @@ static SmallVector<Value> unrollIndex(OpBuilder &b, Location loc, Value index,
   assert(!factors.empty() && "empty factor list");
   SmallVector<Value> basis;
   for (int64_t f : factors)
-    basis.push_back(b.create<arith::ConstantOp>(loc, b.getIndexAttr(f)));
+    basis.push_back(arith::ConstantOp::create(b, loc, b.getIndexAttr(f)));
   FailureOr<SmallVector<Value>> multiIndex =
       affine::delinearizeIndex(b, loc, index, basis);
   assert(!failed(multiIndex) && "Failed to linearize img2col index");
@@ -115,17 +115,17 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0, 1, 2}, {3}};
   auto reshapedFilterType =
       RankedTensorType::get({fh * fw * ic, oc}, filterType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
@@ -138,15 +138,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value mIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> mIndices = unrollIndex(
@@ -170,9 +170,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because the filter does not share the same batch dimension,
@@ -187,7 +187,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -196,11 +196,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
@@ -244,7 +244,7 @@ rewriteInIm2Col(RewriterBase &rewriter,
     SmallVector<int64_t> targetShape = llvm::to_vector<4>(llvm::map_range(
         indices, [&](int64_t index) -> int64_t { return inputShape[index]; }));
 
-    Value outputTensor = rewriter.create<tensor::EmptyOp>(
+    Value outputTensor = tensor::EmptyOp::create(rewriter,
         loc, targetShape, operandTensorType.getElementType());
 
     SmallVector<utils::IteratorType> loopAttributeTypes(
@@ -255,12 +255,12 @@ rewriteInIm2Col(RewriterBase &rewriter,
             AffineMap::get(nloops, 0, exprs, rewriter.getContext())),
         AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-    auto transposedOp = rewriter.create<linalg::GenericOp>(
+    auto transposedOp = linalg::GenericOp::create(rewriter,
         loc, outputTensor.getType(),
         /*inputs=*/operand, /*outputs=*/outputTensor, indexingMaps,
         loopAttributeTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-          nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+          linalg::YieldOp::create(nestedBuilder, nestedLoc, args[0]);
         });
 
     return transposedOp.getResult(0);
@@ -307,15 +307,15 @@ rewriteInIm2Col(RewriterBase &rewriter,
       AffineMap::get(nloops, 0, inputExprs, rewriter.getContext()),
       AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/inputT, /*outputs=*/colTensor, indexingMaps,
       loopAttributeTypes,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, args[0]);
       });
 
   SmallVector<ReassociationIndices> img2ColTensorReassocIndices = {
@@ -331,16 +331,16 @@ rewriteInIm2Col(RewriterBase &rewriter,
   auto reshapedOutputTensorType =
       RankedTensorType::get({n * c, oh * ow}, outputType.getElementType());
 
-  Value reshapedImg2ColTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedImg2ColTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedImg2ColTensorType, img2ColTensor.getResult(0),
       img2ColTensorReassocIndices);
-  Value reshapedFilterTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilterTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterTensorType, filterT, filterReassociationIndice);
-  Value reshapedoutputTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedoutputTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputTensorType, transposedOutputTensor,
       outputReassociationIndice);
 
-  auto batchMatVecResult = rewriter.create<linalg::BatchMatvecOp>(
+  auto batchMatVecResult = linalg::BatchMatvecOp::create(rewriter,
       loc, TypeRange{reshapedoutputTensor.getType()},
       ValueRange{reshapedImg2ColTensor, reshapedFilterTensor},
       ValueRange{reshapedoutputTensor});
@@ -348,7 +348,7 @@ rewriteInIm2Col(RewriterBase &rewriter,
   SmallVector<ReassociationIndices> batchMatVecReassociationIndice = {{0, 1},
                                                                       {2, 3}};
 
-  auto batchMatVecResultReshaped = rewriter.create<tensor::ExpandShapeOp>(
+  auto batchMatVecResultReshaped = tensor::ExpandShapeOp::create(rewriter,
       loc, transposedOutputTensor.getType(), batchMatVecResult.getResult(0),
       batchMatVecReassociationIndice);
 
@@ -400,18 +400,18 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, ic * fh * fw}, inputType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1}, {2, 3}};
   auto reshapedOutputType =
       RankedTensorType::get({n, oc, oh * ow}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   // Convert the input to a (BKN) tensor.
   SmallVector<int64_t, 4> colTensorShape = {n, ic * fh * fw, oh * ow};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   auto nloops = colTensorShape.size();
@@ -423,15 +423,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<AffineMap, 4> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value nIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value nIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> kIndices = unrollIndex(
@@ -455,9 +455,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
 
         // im2col[n, ic*fh*fw, oh*ow] = input[n, ic, sh*oh + fh, sw*ow + fw]
         SmallVector<Value> extractionIndices{bIndex, icIndex, hIndex, wIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because the filter does not share the same batch dimension,
@@ -471,7 +471,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   auto resultMap = AffineMap::get(4, 0, {bDim, mDim, nDim}, context);
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{reshapedFilter, img2ColTensor.getResult(0)},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -480,11 +480,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
@@ -535,17 +535,17 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, fh * fw * ic}, filterType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
@@ -558,15 +558,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value mIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> mIndices = unrollIndex(
@@ -590,9 +590,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because we didn't transpose the filters we don't actually have a batched
@@ -606,7 +606,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -615,11 +615,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index 39e2aac27e213..f21563c323c1b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -37,7 +37,7 @@ static Value createInserts(RewriterBase &rewriter, Location loc, int dim,
   if (dim == static_cast<int>(shape.size()) - 1) {
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
-      destination = rewriter.create<tensor::InsertOp>(loc, *elementIt,
+      destination = tensor::InsertOp::create(rewriter, loc, *elementIt,
                                                       destination, indices);
       ++elementIt;
     }
@@ -65,7 +65,7 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
       MaterializeInDestination: {
     // Note: This is the preferred way of memcpy'ing because no layout map
     // and/or memory space must be specified for the source.
-    auto materializeOp = b.create<bufferization::MaterializeInDestinationOp>(
+    auto materializeOp = bufferization::MaterializeInDestinationOp::create(b,
         loc, tensorSource, memrefDest);
     materializeOp.setWritable(true);
   } break;
@@ -73,19 +73,19 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = b.create<bufferization::ToBufferOp>(
+    Value toBuffer = bufferization::ToBufferOp::create(b,
         loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
-    b.create<memref::CopyOp>(loc, toBuffer, memrefDest);
+    memref::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
   case linalg::BufferizeToAllocationOptions::MemcpyOp::LinalgCopy: {
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = b.create<bufferization::ToBufferOp>(
+    Value toBuffer = bufferization::ToBufferOp::create(b,
         loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
-    b.create<linalg::CopyOp>(loc, toBuffer, memrefDest);
+    linalg::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
   };
 }
@@ -120,14 +120,14 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
             ->materializeConstant(rewriter, constYieldedValue,
                                   yieldedValue.getType(), yieldedValue.getLoc())
             ->getResult(0);
-    auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(fillValue),
+    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(fillValue),
                                                   ValueRange(dest));
     return fillOp;
   }
 
   if (invariantYieldedValue) {
     // Padding with an invariant value.
-    auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(yieldedValue),
+    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(yieldedValue),
                                                   ValueRange(dest));
     return fillOp;
   }
@@ -137,7 +137,7 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(resultType.getRank()));
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, resultType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{dest}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
@@ -146,7 +146,7 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
   rewriter.setInsertionPointToStart(body);
   SmallVector<Value> bbArgReplacements;
   for (int64_t i = 0; i < resultType.getRank(); ++i)
-    bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
+    bbArgReplacements.push_back(linalg::IndexOp::create(rewriter, loc, i));
   rewriter.mergeBlocks(padOp.getBody(), body, bbArgReplacements);
 
   // Update terminator.
@@ -179,8 +179,8 @@ static SmallVector<Value> reifyOrComputeDynamicSizes(OpBuilder &b,
   for (int64_t i = 0; i < tensorType.getRank(); ++i) {
     if (tensorType.isDynamicDim(i))
       dynSizes.push_back(
-          b.create<DimOp>(value.getLoc(), value,
-                          b.create<arith::ConstantIndexOp>(value.getLoc(), i)));
+          DimOp::create(b, value.getLoc(), value,
+                          arith::ConstantIndexOp::create(b, value.getLoc(), i)));
   }
   return dynSizes;
 }
@@ -201,15 +201,15 @@ createAllocationForTensor(RewriterBase &rewriter, Location loc, Value value,
   Value alloc;
   if (options.allocOp ==
       linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloc) {
-    alloc = rewriter.create<memref::AllocOp>(loc, memrefType, dynamicSizes);
+    alloc = memref::AllocOp::create(rewriter, loc, memrefType, dynamicSizes);
     if (options.emitDealloc) {
       // Place deallocation at the end of the block.
       rewriter.setInsertionPoint(rewriter.getInsertionBlock()->getTerminator());
-      rewriter.create<memref::DeallocOp>(loc, alloc);
+      memref::DeallocOp::create(rewriter, loc, alloc);
     }
   } else if (options.allocOp ==
              linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloca) {
-    alloc = rewriter.create<memref::AllocaOp>(loc, memrefType, dynamicSizes);
+    alloc = memref::AllocaOp::create(rewriter, loc, memrefType, dynamicSizes);
     // No dealloc is needed.
   }
 
@@ -243,13 +243,13 @@ Value linalg::bufferizeToAllocation(
       getMixedSizes(rewriter, loc, padOp.getSource());
   SmallVector<OpFoldResult> strides(padOp.getResultType().getRank(),
                                     rewriter.getIndexAttr(1));
-  Value subview = rewriter.create<memref::SubViewOp>(
+  Value subview = memref::SubViewOp::create(rewriter,
       loc, alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
   createMemcpy(rewriter, loc, padOp.getSource(), subview, options);
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
       loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(padOp, toTensorOp);
@@ -338,7 +338,7 @@ Value linalg::bufferizeToAllocation(
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
       loc, allocTensorOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(allocTensorOp, toTensorOp);
@@ -354,7 +354,7 @@ FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
   auto shape = tensorType.getShape();
 
   // Create tensor.empty.
-  auto emptyOp = rewriter.create<EmptyOp>(loc, tensorType, ValueRange());
+  auto emptyOp = EmptyOp::create(rewriter, loc, tensorType, ValueRange());
 
   // Case: tensor<elem_type>.
   if (shape.empty()) {
@@ -369,7 +369,7 @@ FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
   SmallVector<Value, 2> constants;
   constants.reserve(maxDim);
   for (int i = 0; i < maxDim; ++i)
-    constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
+    constants.push_back(arith::ConstantIndexOp::create(rewriter, loc, i));
 
   // Traverse all elements and create tensor.insert ops.
   auto elementIt = fromElementsOp.getElements().begin();
@@ -395,14 +395,14 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
 
   // Create tensor.empty.
   auto emptyOp =
-      rewriter.create<EmptyOp>(loc, tensorType, generateOp.getDynamicExtents());
+      EmptyOp::create(rewriter, loc, tensorType, generateOp.getDynamicExtents());
 
   // Create linalg.generic.
   SmallVector<utils::IteratorType> iteratorTypes(tensorType.getRank(),
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(tensorType.getRank()));
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, tensorType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{emptyOp.getResult()}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
@@ -411,7 +411,7 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
   rewriter.setInsertionPointToStart(body);
   SmallVector<Value> bbArgReplacements;
   for (int64_t i = 0; i < tensorType.getRank(); ++i)
-    bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
+    bbArgReplacements.push_back(linalg::IndexOp::create(rewriter, loc, i));
   rewriter.mergeBlocks(&generateOp.getBody().front(), body, bbArgReplacements);
 
   // Update terminator.
@@ -450,13 +450,13 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
       llvm::all_of(padOp.getMixedHighPad(), isZeroInteger)) {
     using bufferization::AllocTensorOp;
     Value allocated =
-        rewriter.create<AllocTensorOp>(loc, resultType, dynamicSizes);
+        AllocTensorOp::create(rewriter, loc, resultType, dynamicSizes);
     auto copyOp = rewriter.replaceOpWithNewOp<linalg::CopyOp>(
         padOp, padOp.getSource(), allocated);
     return copyOp.getOperation();
   }
 
-  Value empty = rewriter.create<EmptyOp>(loc, resultType, dynamicSizes);
+  Value empty = EmptyOp::create(rewriter, loc, resultType, dynamicSizes);
   // Create linalg.fill or linalg.generic.
   Operation *fillOp = movePaddingToFillOrGenericOp(rewriter, loc, padOp, empty);
   rewriter.setInsertionPointAfter(fillOp);
@@ -567,7 +567,7 @@ Value linalg::bufferizeToAllocation(
       createMemcpy(rewriter, op->getLoc(), operand->get(), alloc, options);
     }
     rewriter.modifyOpInPlace(op, [&]() {
-      auto toTensorOp = rewriter.create<ToTensorOp>(
+      auto toTensorOp = ToTensorOp::create(rewriter,
           op->getLoc(), operand->get().getType(), alloc);
       operand->set(toTensorOp);
       if (options.bufferizeDestinationOnly) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index 70574903f7111..eb2c7756dc6ae 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -287,7 +287,7 @@ getOrCreatePackedViewOfOperand(OpBuilder &b, Location loc, PackInfo packInfo,
 
   auto empty = linalg::PackOp::createDestinationTensor(
       b, loc, opOperand->get(), innerTileSizes, innerDimsPos, outerDimsPerm);
-  auto packedOperand = b.create<linalg::PackOp>(
+  auto packedOperand = linalg::PackOp::create(b,
       loc, opOperand->get(), empty, innerDimsPos, innerTileSizes,
       /*padding=*/std::nullopt, outerDimsPerm);
   return std::make_tuple(packedOperand, indexingMap);
@@ -345,7 +345,7 @@ static GenericOp packGenericOp(RewriterBase &rewriter, GenericOp genericOp,
 
   indexingMaps.push_back(packedOutIndexingMap);
 
-  auto newGenericOp = rewriter.create<linalg::GenericOp>(
+  auto newGenericOp = linalg::GenericOp::create(rewriter,
       loc, dest.getType(), inputOperands, dest, indexingMaps, iterTypes,
       /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
   rewriter.cloneRegionBefore(genericOp.getRegion(), newGenericOp.getRegion(),
@@ -457,7 +457,7 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, linalg::PackOp packOp,
   if (!packOpDest.hasOneUse())
     return failure();
   if (auto emptyOp = packOpDest.getDefiningOp<tensor::EmptyOp>()) {
-    packOpDest = rewriter.create<tensor::EmptyOp>(
+    packOpDest = tensor::EmptyOp::create(rewriter,
         genericOp->getLoc(), emptyOp.getMixedSizes(),
         emptyOp.getType().getElementType());
   } else {
@@ -562,7 +562,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     auto empty = linalg::PackOp::createDestinationTensor(
         rewriter, loc, padOp.getSource(), mixedTiles, innerDimsPos,
         outerDimsPerm);
-    auto sourcePack = rewriter.create<linalg::PackOp>(
+    auto sourcePack = linalg::PackOp::create(rewriter,
         loc, padOp.getSource(), empty, innerDimsPos, mixedTiles,
         /*padding=*/std::nullopt, outerDimsPerm);
 
@@ -579,7 +579,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, /*result=*/Type(), sourcePack, lowPad, highPad, paddingVal,
         padOp.getNofold());
 
@@ -588,7 +588,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     if (!padOp->hasOneUse()) {
       auto unpackEmpty = linalg::UnPackOp::createDestinationTensor(
           rewriter, loc, newPadOp, mixedTiles, innerDimsPos, outerDimsPerm);
-      Value unpackedPad = rewriter.create<linalg::UnPackOp>(
+      Value unpackedPad = linalg::UnPackOp::create(rewriter,
           loc, newPadOp, unpackEmpty, innerDimsPos, mixedTiles, outerDimsPerm);
       rewriter.replaceAllUsesExcept(padOp, unpackedPad, sourcePack);
     }
@@ -719,7 +719,7 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
   auto emptyOp = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), collapseOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newPackOp = rewriter.create<linalg::PackOp>(
+  auto newPackOp = linalg::PackOp::create(rewriter,
       packOp.getLoc(), collapseOp.getSrc(), emptyOp, projectedInnerDimsPos,
       packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPerm);
 
@@ -735,7 +735,7 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
     nextPos += 1;
   }
 
-  auto newCollapseOp = rewriter.create<tensor::CollapseShapeOp>(
+  auto newCollapseOp = tensor::CollapseShapeOp::create(rewriter,
       collapseOp.getLoc(), packOp.getType(), newPackOp, newReassocIndices);
   rewriter.replaceOp(packOp, newCollapseOp);
 
@@ -853,12 +853,12 @@ bubbleUpPackOpThroughExpandShape(tensor::ExpandShapeOp expandOp,
   Value destTensor = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), expandOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, /*outerDimsPerm=*/SmallVector<int64_t>{});
-  Value packedVal = rewriter.create<linalg::PackOp>(
+  Value packedVal = linalg::PackOp::create(rewriter,
       packOp.getLoc(), expandOp.getSrc(), destTensor, projectedInnerDimsPos,
       packOp.getMixedTiles(), packOp.getPaddingValue(),
       /*outerDimsPerm=*/SmallVector<int64_t>{});
 
-  Value newExpandOp = rewriter.create<tensor::ExpandShapeOp>(
+  Value newExpandOp = tensor::ExpandShapeOp::create(rewriter,
       packOp.getLoc(), packOp.getDestType(), packedVal, *reassocExpand);
   rewriter.replaceOp(packOp, newExpandOp);
 
@@ -972,14 +972,14 @@ static LogicalResult pushDownUnPackOpThroughExpandShape(
 
   RankedTensorType newExpandType = linalg::PackOp::inferPackedType(
       expandTy, innerTileSizes, projectedInnerDimsPos, newOuterDimsPerm);
-  auto newExpandOp = rewriter.create<tensor::ExpandShapeOp>(
+  auto newExpandOp = tensor::ExpandShapeOp::create(rewriter,
       expandOp.getLoc(), newExpandType, unPackOp.getSource(),
       newReassocIndices);
 
   auto emptyOp = linalg::UnPackOp::createDestinationTensor(
       rewriter, unPackOp.getLoc(), newExpandOp, unPackOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newUnPackOp = rewriter.create<linalg::UnPackOp>(
+  auto newUnPackOp = linalg::UnPackOp::create(rewriter,
       unPackOp.getLoc(), newExpandOp.getResult(), emptyOp,
       projectedInnerDimsPos, unPackOp.getMixedTiles(), newOuterDimsPerm);
   rewriter.replaceOp(expandOp, newUnPackOp);
@@ -1212,16 +1212,16 @@ struct PushDownUnPackThroughPadOp : public OpRewritePattern<tensor::PadOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, /*result=*/Type(), unpackOp.getSource(), lowPad, highPad,
         paddingVal, padOp.getNofold());
 
     // Inject the linalg.unpack right after the packed padOp.
-    Value outputUnPack = rewriter.create<tensor::EmptyOp>(
+    Value outputUnPack = tensor::EmptyOp::create(rewriter,
         loc, padOp.getResultType().getShape(),
         padOp.getResultType().getElementType());
 
-    Value replacement = rewriter.create<linalg::UnPackOp>(
+    Value replacement = linalg::UnPackOp::create(rewriter,
         loc, newPadOp.getResult(), outputUnPack, innerDimsPos,
         unpackOp.getMixedTiles(), outerDimsPerm);
     rewriter.replaceOp(padOp, replacement);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
index 692bf595267d4..82d386b3b9099 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
@@ -198,9 +198,9 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
         transposedShape[i] = inputRTType.getShape()[permutation[i]];
 
       Value emptyTensor =
-          rewriter.create<tensor::EmptyOp>(loc, transposedShape, elType);
+          tensor::EmptyOp::create(rewriter, loc, transposedShape, elType);
 
-      auto transposeOp = rewriter.create<TransposeOp>(loc, newInitValues[i],
+      auto transposeOp = TransposeOp::create(rewriter, loc, newInitValues[i],
                                                       emptyTensor, permutation);
       newInitValues[i] = transposeOp->getResult(0);
       isChanged = true;
@@ -209,10 +209,10 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
     // Does it require broadcast?
     if (!broadcastedDims.empty()) {
       assert(broadcastedDims.size() && "should have non size broadcast");
-      Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+      Value emptyTensor = tensor::EmptyOp::create(rewriter,
           loc, outputShape, inputRTType.getElementType());
 
-      auto broadcastOp = rewriter.create<linalg::BroadcastOp>(
+      auto broadcastOp = linalg::BroadcastOp::create(rewriter,
           loc, newInitValues[i], emptyTensor, broadcastedDims);
 
       newInitValues[i] = broadcastOp->getResult(0);
@@ -227,7 +227,7 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
   SmallVector<Value> operands = op->getOperands();
   ValueRange operandsRef(operands);
 
-  auto newOp = rewriter.create<linalg::GenericOp>(
+  auto newOp = linalg::GenericOp::create(rewriter,
       /*location=*/op.getLoc(),
       /*resultTensorTypes=*/op->getResultTypes(),
       /*inputs=*/newInitValues,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
index 1419175304899..bd3fad01aa48b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
@@ -133,12 +133,12 @@ static Value getZero(OpBuilder &b, Location loc, Type elementType) {
   assert(elementType.isIntOrIndexOrFloat() &&
          "expected scalar type while computing zero value");
   if (isa<IntegerType>(elementType))
-    return b.create<arith::ConstantIntOp>(loc, elementType, 0);
+    return arith::ConstantIntOp::create(b, loc, elementType, 0);
   if (elementType.isIndex())
-    return b.create<arith::ConstantIndexOp>(loc, 0);
+    return arith::ConstantIndexOp::create(b, loc, 0);
   // Assume float.
   auto floatType = cast<FloatType>(elementType);
-  return b.create<arith::ConstantFloatOp>(
+  return arith::ConstantFloatOp::create(b,
       loc, floatType, APFloat::getZero(floatType.getFloatSemantics()));
 }
 
@@ -189,7 +189,7 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
     // Fall back path, use an `init_tensor` and identity indexing map.
     AffineMap indexingMap = rewriter.getMultiDimIdentityMap(domain.size());
     Value emptyTensor =
-        rewriter.create<tensor::EmptyOp>(loc, domain, scalarOpResult.getType());
+        tensor::EmptyOp::create(rewriter, loc, domain, scalarOpResult.getType());
     newInitValues.push_back(emptyTensor);
     newResultTypes.push_back(emptyTensor.getType());
     peeledGenericOpIndexingMaps.push_back(indexingMap);
@@ -202,7 +202,7 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
   resultTypes.append(newResultTypes.begin(), newResultTypes.end());
   auto indexingMapAttr =
       rewriter.getAffineMapArrayAttr(peeledGenericOpIndexingMaps);
-  return rewriter.create<GenericOp>(
+  return GenericOp::create(rewriter,
       loc, resultTypes, genericOp.getInputs(), outsOperands, indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
       [](OpBuilder, Location, ValueRange) {});
@@ -239,7 +239,7 @@ DecomposeLinalgOp::createResidualGenericOp(GenericOp genericOp,
     indexingMaps.push_back(genericOp.getMatchingIndexingMap(&outOperand));
 
   auto indexingMapAttr = rewriter.getAffineMapArrayAttr(indexingMaps);
-  return rewriter.create<GenericOp>(
+  return GenericOp::create(rewriter,
       genericOp->getLoc(), genericOp->getResultTypes(),
       residualGenericOpOperands, genericOp.getOutputs(), indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
@@ -324,7 +324,7 @@ DecomposeLinalgOp::matchAndRewrite(GenericOp genericOp,
     yieldedVals.append(llvm::to_vector(
         llvm::map_range(peeledScalarOperation->getResults(),
                         [](OpResult opr) -> Value { return opr; })));
-    rewriter.create<YieldOp>(genericOp.getLoc(), yieldedVals);
+    YieldOp::create(rewriter, genericOp.getLoc(), yieldedVals);
   }
 
   /// In the split operations, replace block arguments uses that refer to
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index ef24eb881d68b..b8e73bd5929dd 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -34,7 +34,7 @@ static Value sourceMaterializationCallback(OpBuilder &builder, Type type,
 
   // A detensored value is converted back by creating a new tensor from its
   // element(s).
-  return builder.create<tensor::FromElementsOp>(
+  return tensor::FromElementsOp::create(builder,
       loc, RankedTensorType::get({}, inputType), inputs[0]);
 }
 
@@ -147,7 +147,7 @@ class DetensorizeTypeConverter : public TypeConverter {
     // A tensor value is detensoried by extracting its element(s).
     addTargetMaterialization([](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      return builder.create<tensor::ExtractOp>(loc, inputs[0], ValueRange{});
+      return tensor::ExtractOp::create(builder, loc, inputs[0], ValueRange{});
     });
 
     addSourceMaterialization(sourceMaterializationCallback);
@@ -481,7 +481,7 @@ struct LinalgDetensorize
         rewriter.splitBlock(entryBlock, entryBlock->begin());
     rewriter.setInsertionPointToStart(entryBlock);
     auto branch =
-        rewriter.create<cf::BranchOp>(rewriter.getUnknownLoc(), postEntryBlock);
+        cf::BranchOp::create(rewriter, rewriter.getUnknownLoc(), postEntryBlock);
 
     if (aggressiveMode.getValue()) {
       AggressiveDetensoringModel costModel;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index e0062d15e61ca..fd8c177cc7247 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -118,14 +118,14 @@ struct MoveInitOperandsToInput : public OpRewritePattern<GenericOp> {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointAfterValue(op->get());
       auto elemType = cast<ShapedType>(op->get().getType()).getElementType();
-      auto empty = rewriter.create<tensor::EmptyOp>(
+      auto empty = tensor::EmptyOp::create(rewriter,
           loc, tensor::getMixedSizes(rewriter, loc, op->get()), elemType);
 
       unsigned start = genericOp.getDpsInits().getBeginOperandIndex();
       newOutputOperands[op->getOperandNumber() - start] = empty.getResult();
     }
 
-    auto newOp = rewriter.create<GenericOp>(
+    auto newOp = GenericOp::create(rewriter,
         loc, genericOp.getResultTypes(), newInputOperands, newOutputOperands,
         newIndexingMaps, genericOp.getIteratorTypesArray(),
         /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
@@ -295,7 +295,7 @@ static Value collapseValue(
     MemRefLayoutAttrInterface layout;
     auto targetType = MemRefType::get(targetShape, memrefType.getElementType(),
                                       layout, memrefType.getMemorySpace());
-    return rewriter.create<memref::CollapseShapeOp>(loc, targetType, operand,
+    return memref::CollapseShapeOp::create(rewriter, loc, targetType, operand,
                                                     reassociation);
   }
   if (auto tensorType = dyn_cast<RankedTensorType>(operand.getType())) {
@@ -314,7 +314,7 @@ static Value collapseValue(
         "unknown rank reduction strategy");
     auto targetType =
         RankedTensorType::get(targetShape, tensorType.getElementType());
-    return rewriter.create<tensor::CollapseShapeOp>(loc, targetType, operand,
+    return tensor::CollapseShapeOp::create(rewriter, loc, targetType, operand,
                                                     reassociation);
   }
   llvm_unreachable("unsupported operand type");
@@ -519,7 +519,7 @@ linalg::dropUnitDims(RewriterBase &rewriter, GenericOp genericOp,
   for (unsigned i : llvm::seq<unsigned>(0, genericOp.getNumResults()))
     resultTypes.push_back(newOutputs[i].getType());
   GenericOp replacementOp =
-      rewriter.create<GenericOp>(loc, resultTypes, newInputs, newOutputs,
+      GenericOp::create(rewriter, loc, resultTypes, newInputs, newOutputs,
                                  newIndexingMaps, newIteratorTypes);
   rewriter.inlineRegionBefore(genericOp.getRegion(), replacementOp.getRegion(),
                               replacementOp.getRegion().begin());
@@ -652,7 +652,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         collapseValue(rewriter, padOp.getLoc(), padOp.getSource(), newShape,
                       reassociationMap, options.rankReductionStrategy);
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         padOp.getLoc(), /*result=*/Type(), collapsedSource, newLowPad,
         newHighPad, paddingVal, padOp.getNofold());
 
@@ -670,7 +670,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         expandedSizes.push_back(tensor::getMixedSize(
             rewriter, padOp.getLoc(), newPadOp, dim - numUnitDims));
       }
-      dest = rewriter.create<tensor::EmptyOp>(
+      dest = tensor::EmptyOp::create(rewriter,
           padOp.getLoc(), expandedSizes,
           padOp.getResultType().getElementType());
     }
@@ -713,7 +713,7 @@ struct RankReducedExtractSliceOp
             strides));
 
     Location loc = sliceOp.getLoc();
-    Value newSlice = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSlice = tensor::ExtractSliceOp::create(rewriter,
         loc, rankReducedType, sliceOp.getSource(), offsets, sizes, strides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSlice, *reassociation);
@@ -747,7 +747,7 @@ struct RankReducedInsertSliceOp : public OpRewritePattern<InsertOpTy> {
       // parallel case.
       if (std::is_same<InsertOpTy, tensor::ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      reshapedSource = rewriter.create<tensor::CollapseShapeOp>(
+      reshapedSource = tensor::CollapseShapeOp::create(rewriter,
           loc, insertSliceOp.getSource(), *reassociation);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
@@ -898,7 +898,7 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
   /// Expand result tensor.
   Value expandResult(PatternRewriter &rewriter, Value result,
                      RankedTensorType expandedType, int64_t dim) const {
-    return rewriter.create<tensor::ExpandShapeOp>(
+    return tensor::ExpandShapeOp::create(rewriter,
         result.getLoc(), expandedType, result,
         getReassociationForReshapeAtDim(expandedType.getRank(), dim));
   }
@@ -934,7 +934,7 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
     SmallVector<Type, 1> collapsedResultTy;
     if (isa<RankedTensorType>(collapsedInit.getType()))
       collapsedResultTy.push_back(collapsedInit.getType());
-    auto collapsedOp = rewriter.create<ToOpTy>(
+    auto collapsedOp = ToOpTy::create(rewriter,
         loc, collapsedResultTy, ValueRange{collapsedLhs, collapsedRhs},
         ValueRange{collapsedInit});
     for (auto attr : contractionOp->getAttrs()) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index f97ed3d6d5111..1ae4375bd09eb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -236,11 +236,11 @@ static void generateFusedElementwiseOpRegion(
     fusedIndices.reserve(numFusedOpLoops);
     llvm::transform(llvm::seq<uint64_t>(0, numFusedOpLoops),
                     std::back_inserter(fusedIndices), [&](uint64_t dim) {
-                      return rewriter.create<IndexOp>(producer.getLoc(), dim);
+                      return IndexOp::create(rewriter, producer.getLoc(), dim);
                     });
     for (IndexOp indexOp :
          llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
-      Value newIndex = rewriter.create<affine::AffineApplyOp>(
+      Value newIndex = affine::AffineApplyOp::create(rewriter,
           producer.getLoc(),
           consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
       mapper.map(indexOp.getResult(), newIndex);
@@ -327,7 +327,7 @@ static void generateFusedElementwiseOpRegion(
   }
   for (auto consumerYieldVal : consumerYieldOp.getOperands())
     fusedYieldValues.push_back(mapper.lookupOrDefault(consumerYieldVal));
-  rewriter.create<YieldOp>(fusedOp.getLoc(), fusedYieldValues);
+  YieldOp::create(rewriter, fusedOp.getLoc(), fusedYieldValues);
 
   // Sanity checks.
   assert(fusedBlock->getNumArguments() == fusedOp.getNumOperands() &&
@@ -416,7 +416,7 @@ mlir::linalg::fuseElementwiseOps(RewriterBase &rewriter,
   }
 
   // Generate the fused op.
-  auto fusedOp = rewriter.create<GenericOp>(
+  auto fusedOp = GenericOp::create(rewriter,
       consumer.getLoc(), fusedResultTypes, fusedInputOperands,
       fusedOutputOperands, rewriter.getAffineMapArrayAttr(fusedIndexMaps),
       consumer.getIteratorTypes(),
@@ -750,9 +750,9 @@ static void updateExpandedGenericOpRegion(PatternRewriter &rewriter,
     expandedIndices.reserve(expandedDims.size() - 1);
     llvm::transform(
         expandedDims.drop_front(), std::back_inserter(expandedIndices),
-        [&](int64_t dim) { return rewriter.create<IndexOp>(loc, dim); });
+        [&](int64_t dim) { return IndexOp::create(rewriter, loc, dim); });
     OpFoldResult newIndex =
-        rewriter.create<IndexOp>(loc, expandedDims.front()).getResult();
+        IndexOp::create(rewriter, loc, expandedDims.front()).getResult();
     for (auto [expandedShape, expandedIndex] :
          llvm::zip(expandedDimsShape, expandedIndices)) {
       AffineExpr idx, acc, shape;
@@ -796,7 +796,7 @@ static Operation *createExpandedTransposeOp(PatternRewriter &rewriter,
       newPerm.push_back(dim);
     }
   }
-  return rewriter.create<TransposeOp>(transposeOp.getLoc(), expandedInput,
+  return TransposeOp::create(rewriter, transposeOp.getLoc(), expandedInput,
                                       output, invertPermutationVector(newPerm));
 }
 
@@ -813,7 +813,7 @@ static Operation *createExpandedGenericOp(
     for (auto j : expansionInfo.getExpandedDims(i))
       iteratorTypes[j] = type;
 
-  Operation *fused = rewriter.create<GenericOp>(
+  Operation *fused = GenericOp::create(rewriter,
       linalgOp.getLoc(), resultTypes, expandedOpOperands, outputs,
       expandedOpIndexingMaps, iteratorTypes);
 
@@ -933,7 +933,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
                 reassociation,
                 /*isExpandingReshape=*/true)))
           return std::nullopt;
-        expandedOpOperands.push_back(rewriter.create<tensor::ExpandShapeOp>(
+        expandedOpOperands.push_back(tensor::ExpandShapeOp::create(rewriter,
             loc, expandedOperandType, opOperand->get(), reassociation,
             expandedOperandShape));
         continue;
@@ -961,7 +961,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               reassociation,
               /*isExpandingReshape=*/true)))
         return std::nullopt;
-      outputs.push_back(rewriter.create<tensor::ExpandShapeOp>(
+      outputs.push_back(tensor::ExpandShapeOp::create(rewriter,
           loc, expandedOutputType, opOperand.get(), reassociation,
           expandedOutputShape));
     } else {
@@ -984,7 +984,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               linalgOp.getMatchingIndexingMap(
                   linalgOp.getDpsInitOperand(resultNumber)),
               expansionInfo);
-      resultVals.push_back(rewriter.create<tensor::CollapseShapeOp>(
+      resultVals.push_back(tensor::CollapseShapeOp::create(rewriter,
           linalgOp.getLoc(), opResult.getType(),
           fusedOp->getResult(resultNumber), reassociation));
     } else {
@@ -1086,7 +1086,7 @@ class FoldPadWithProducerReshapeOpByExpansion
 
     Location loc = padOp->getLoc();
     RankedTensorType expandedPaddedType = paddedType.clone(expandedPaddedShape);
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
@@ -1603,7 +1603,7 @@ static void generateCollapsedIndexingRegion(
        enumerate(collapsingInfo.getCollapsedOpToOrigOpMapping())) {
     ReassociationIndicesRef foldedDimsRef(foldedDims.value());
     Value newIndexVal =
-        rewriter.create<linalg::IndexOp>(loc, foldedDims.index());
+        linalg::IndexOp::create(rewriter, loc, foldedDims.index());
     for (auto dim : llvm::reverse(foldedDimsRef.drop_front())) {
       Value loopDim =
           getValueOrCreateConstantIndexOp(rewriter, loc, loopRange[dim]);
@@ -1687,7 +1687,7 @@ GenericOp cloneToCollapsedOp<GenericOp>(RewriterBase &rewriter,
   SmallVector<utils::IteratorType> iteratorTypes(getCollapsedOpIteratorTypes(
       origOp.getIteratorTypesArray(), collapsingInfo));
 
-  GenericOp collapsedOp = rewriter.create<linalg::GenericOp>(
+  GenericOp collapsedOp = linalg::GenericOp::create(rewriter,
       origOp.getLoc(), resultTypes, inputOperands, outputOperands, indexingMaps,
       iteratorTypes, [](OpBuilder &builder, Location loc, ValueRange args) {});
   Block *origOpBlock = &origOp->getRegion(0).front();
@@ -1790,11 +1790,11 @@ FailureOr<CollapseResult> mlir::linalg::collapseOpIterationDims(
       if (isa<MemRefType>(collapsedOpResult.getType())) {
         MemRefType expandShapeResultType = MemRefType::get(
             originalResultType.getShape(), originalResultType.getElementType());
-        result = rewriter.create<memref::ExpandShapeOp>(
+        result = memref::ExpandShapeOp::create(rewriter,
             loc, expandShapeResultType, collapsedOpResult, reassociation,
             resultShape);
       } else {
-        result = rewriter.create<tensor::ExpandShapeOp>(
+        result = tensor::ExpandShapeOp::create(rewriter,
             loc, originalResultType, collapsedOpResult, reassociation,
             resultShape);
       }
@@ -1978,7 +1978,7 @@ class FoldPadWithProducerReshapeOpByCollapsing
 
     RankedTensorType collapsedPaddedType =
         paddedType.clone(collapsedPaddedShape);
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
@@ -2113,10 +2113,10 @@ class FoldScalarOrSplatConstant : public OpRewritePattern<GenericOp> {
 
       // Create a constant scalar value from the splat constant.
       Value scalarConstant =
-          rewriter.create<arith::ConstantOp>(def->getLoc(), constantAttr);
+          arith::ConstantOp::create(rewriter, def->getLoc(), constantAttr);
 
       SmallVector<Value> outputOperands = genericOp.getOutputs();
-      auto fusedOp = rewriter.create<GenericOp>(
+      auto fusedOp = GenericOp::create(rewriter,
           rewriter.getFusedLoc(fusedLocs), genericOp->getResultTypes(),
           /*inputs=*/fusedOperands,
           /*outputs=*/outputOperands,
@@ -2179,7 +2179,7 @@ struct RemoveOutsDependency : public OpRewritePattern<GenericOp> {
         modifiedOutput = true;
         SmallVector<OpFoldResult> mixedSizes =
             tensor::getMixedSizes(rewriter, loc, operandVal);
-        Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+        Value emptyTensor = tensor::EmptyOp::create(rewriter,
             loc, mixedSizes, operandType.getElementType());
         op->setOperand(opOperand.getOperandNumber(), emptyTensor);
       }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index c4af09ca01421..d93f8c26a9f0a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -64,7 +64,7 @@ getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) {
       continue;
 
     // Extract static / dynamic shape mix from the first operand.
-    res.push_back(b.create<tensor::EmptyOp>(
+    res.push_back(tensor::EmptyOp::create(b,
         loc, tensor::getMixedSizes(b, loc, operands.front()),
         cast<RankedTensorType>(t).getElementType()));
   }
@@ -104,7 +104,7 @@ struct ConvertAnyElementwiseMappableOpOnRankedTensors : public RewritePattern {
               builder.create(loc, op->getName().getIdentifier(),
                              regionArgs.take_front(op->getNumOperands()),
                              resultTypes, op->getAttrs());
-          builder.create<linalg::YieldOp>(loc, scalarOp->getResults());
+          linalg::YieldOp::create(builder, loc, scalarOp->getResults());
         });
     return success();
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
index d375878fb2c91..4e95aae86e670 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
@@ -259,7 +259,7 @@ mlir::linalg::deduplicateOperandsAndRemoveDeadResults(
   for (Value v : newOutputOperands)
     if (isa<TensorType>(v.getType()))
       newResultTypes.push_back(v.getType());
-  auto newOp = rewriter.create<GenericOp>(
+  auto newOp = GenericOp::create(rewriter,
       loc, newResultTypes, newInputOperands, newOutputOperands,
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       genericOp.getIteratorTypes(), genericOp.getDocAttr(),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
index 44469bc404a7c..e19e224c25028 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
@@ -72,14 +72,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
     // Create the tensor of same size as output of the pad op.
     RankedTensorType padResultType = padOp.getResultType();
     auto resultSizes = resultShape[0];
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultSizes, padResultType.getElementType());
 
     // Fill the tensor with the pad value.
     // TODO: There is an option to fill only the boundaries. For now just
     // filling the whole tensor.
     auto fillTensor =
-        rewriter.create<linalg::FillOp>(loc, padValue, emptyTensor.getResult());
+        linalg::FillOp::create(rewriter, loc, padValue, emptyTensor.getResult());
 
     // Construct a slice of the fill result that is to be replaced with the
     // result of the generic op. The low pad values are the offsets, the size of
@@ -93,14 +93,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
          llvm::enumerate(cast<RankedTensorType>(source.getType()).getShape())) {
       if (ShapedType::isDynamic(shape.value())) {
         sizes.push_back(
-            rewriter.create<tensor::DimOp>(loc, source, shape.index())
+            tensor::DimOp::create(rewriter, loc, source, shape.index())
                 .getResult());
       } else {
         sizes.push_back(rewriter.getIndexAttr(shape.value()));
       }
     }
     SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-    auto slice = rewriter.create<tensor::ExtractSliceOp>(
+    auto slice = tensor::ExtractSliceOp::create(rewriter,
         loc, fillTensor.getResult(0), offsets, sizes, strides);
 
     // Clone the generic op.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 9bc7be2623849..41252c68ffda9 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -277,7 +277,7 @@ mlir::linalg::fuseProducerOfTensor(OpBuilder &b, OpResult producerOpResult,
   // mismatches. Insert a `tensor.cast` op to propagate the transformation
   // invariant that types are compatible.
   if (consumerType != def.getType())
-    def = b.create<tensor::CastOp>(fusedProducer.getLoc(), consumerType, def);
+    def = tensor::CastOp::create(b, fusedProducer.getLoc(), consumerType, def);
   consumerOpOperand.set(def);
   return FusionInfo{cast<LinalgOp>(producerOpResult.getOwner()), fusedProducer};
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 05f2157b77aeb..7994af8834990 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -61,7 +61,7 @@ FailureOr<GenericOp> mlir::linalg::generalizeNamedOp(RewriterBase &rewriter,
   // All named ops have a region attached that can be inlined.
   assert(linalgOp->getNumRegions() == 1 &&
          "expect named op to have one region attached");
-  GenericOp genericOp = rewriter.create<GenericOp>(
+  GenericOp genericOp = GenericOp::create(rewriter,
       linalgOp.getLoc(), resultTypes, inputs, outputs, indexingMaps, iterators);
   rewriter.inlineRegionBefore(linalgOp->getRegion(0), genericOp.getRegion(),
                               genericOp.getRegion().begin());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index 94ed46442180c..86f24578376c1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -591,7 +591,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
 
     // Create a packing loop that takes `hoistedPackedTensor` as iteration
     // argument.
-    auto clonedForOp = rewriter.create<scf::ForOp>(
+    auto clonedForOp = scf::ForOp::create(rewriter,
         loc, bvm.lookupOrDefault(forOp.getLowerBound()),
         bvm.lookupOrDefault(forOp.getUpperBound()),
         bvm.lookupOrDefault(forOp.getStep()), hoistedPackedTensor);
@@ -640,10 +640,10 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   TransposeOp maybeTransposeOp;
   Value paddedTensor = bvm.lookup(opToHoist.getResult());
   if (!transposeVector.empty()) {
-    Value outputTensor = rewriter.create<tensor::ExtractSliceOp>(
+    Value outputTensor = tensor::ExtractSliceOp::create(rewriter,
         loc, transposedTensorType, hoistedPackedTensor, offsets, sizes,
         strides);
-    maybeTransposeOp = rewriter.create<linalg::TransposeOp>(
+    maybeTransposeOp = linalg::TransposeOp::create(rewriter,
         loc, paddedTensor, outputTensor, transposeVector);
     paddedTensor = maybeTransposeOp.getResult()[0];
   }
@@ -652,7 +652,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   if (nPackedLoops > 0) {
     // Step 4. Create InsertSliceOp at the innermost loop level, inserting an
     // optionally transposed padded slice into the packed tensor.
-    Value inserted = rewriter.create<tensor::InsertSliceOp>(
+    Value inserted = tensor::InsertSliceOp::create(rewriter,
         loc, paddedTensor, hoistedPackedTensor, offsets, sizes, strides);
 
     // Step 5. Iteratively pop the stack and propagate the yield.
@@ -660,7 +660,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
     for (Value iv : llvm::reverse(clonedLoopIvs)) {
       auto forOp = scf::getForInductionVarOwner(iv);
       rewriter.setInsertionPointToEnd(&forOp.getRegion().front());
-      rewriter.create<scf::YieldOp>(loc, valueToYield);
+      scf::YieldOp::create(rewriter, loc, valueToYield);
       valueToYield = forOp.getResult(0);
     }
   }
@@ -712,7 +712,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   rewriter.setInsertionPoint(outerLoop);
   SmallVector<Value> dynamicTensorSizes =
       analysis.getHoistedPackedTensorSizes(rewriter, loc);
-  auto emptyOp = rewriter.create<tensor::EmptyOp>(
+  auto emptyOp = tensor::EmptyOp::create(rewriter,
       loc, hoistedPackedTensorType.getShape(),
       hoistedPackedTensorType.getElementType(), dynamicTensorSizes);
 
@@ -840,7 +840,7 @@ padThroughLoopIterArg(RewriterBase &rewriter, Value paddedValueBeforeHoisting,
   {
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(forOp);
-    extracted = rewriter.create<tensor::ExtractSliceOp>(
+    extracted = tensor::ExtractSliceOp::create(rewriter,
         hoistedPackedTensor.getLoc(), hoistedPackedTensor,
         outerSliceOp.getMixedOffsets(), outerSliceOp.getMixedSizes(),
         outerSliceOp.getMixedStrides());
@@ -934,7 +934,7 @@ static Value replaceByPackingResult(RewriterBase &rewriter,
   // offsets = [maybe_leading_ivs, 0 .. 0].
   // sizes = [1 .. 1, transposedShape] (defined above).
   // strides = [1 .. 1] (defined above)
-  return rewriter.create<tensor::ExtractSliceOp>(
+  return tensor::ExtractSliceOp::create(rewriter,
       loc, transposedTensorType, hoistedPackedTensor, offsets,
       packingResult.sizes, packingResult.strides);
 }
@@ -982,9 +982,9 @@ FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(newResult.getDefiningOp());
     // Transpose the packed tensor back to the original storage order.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, paddedTensorType.getShape(), paddedTensorType.getElementType());
-    TransposeOp unTransposeOp = rewriter.create<linalg::TransposeOp>(
+    TransposeOp unTransposeOp = linalg::TransposeOp::create(rewriter,
         loc, newResult, emptyTensor, transposeVector);
     newResult = unTransposeOp.getResult()[0];
     transposeOps.push_back(unTransposeOp);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index f2e51c29f3241..5397fc572e558 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -53,7 +53,7 @@ static scf::ForOp replaceWithDifferentYield(RewriterBase &rewriter,
   assert(index < inits.size());
   inits[index] = newInitOperand;
 
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
       inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
index 1f3336d2bfbb9..a900215ccf014 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
@@ -60,7 +60,7 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
 
     Location loc = genericOp->getLoc();
     SmallVector<Value> outputOperands = genericOp.getOutputs();
-    auto newOp = rewriter.create<GenericOp>(
+    auto newOp = GenericOp::create(rewriter,
         loc, genericOp->getResultTypes(), newOperands, outputOperands,
         newIndexingMaps, genericOp.getIteratorTypesArray());
     rewriter.cloneRegionBefore(genericOp.getRegion(), newOp.getRegion(),
@@ -77,11 +77,11 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
       SmallVector<Value> indicesValues;
       for (auto idx : indices)
         indicesValues.emplace_back(
-            rewriter.create<arith::ConstantIndexOp>(loc, idx));
+            arith::ConstantIndexOp::create(rewriter, loc, idx));
       Value scalarValue = opOperand->get();
       if (isa<RankedTensorType>(scalarValue.getType())) {
         scalarValue =
-            rewriter.create<tensor::ExtractOp>(loc, scalarValue, indicesValues);
+            tensor::ExtractOp::create(rewriter, loc, scalarValue, indicesValues);
       }
       body->getArgument(idx).replaceAllUsesWith(scalarValue);
       body->eraseArgument(idx);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index a92a0c83e0316..2372313e29c58 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -88,7 +88,7 @@ mlir::linalg::interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp,
       allIndices.reserve(genericOp.getNumLoops());
       llvm::transform(llvm::seq<uint64_t>(0, genericOp.getNumLoops()),
                       std::back_inserter(allIndices), [&](uint64_t dim) {
-                        return rewriter.create<IndexOp>(indexOp->getLoc(), dim);
+                        return IndexOp::create(rewriter, indexOp->getLoc(), dim);
                       });
       rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
           indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 488041a43a2ef..0c08c1d8b997d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -49,7 +49,7 @@ static SmallVector<Value> makeCanonicalAffineApplies(OpBuilder &b, Location loc,
     auto exprMap = AffineMap::get(dims, map.getNumSymbols(), e);
     SmallVector<Value> operands(vals);
     affine::canonicalizeMapAndOperands(&exprMap, &operands);
-    res.push_back(b.create<affine::AffineApplyOp>(loc, exprMap, operands));
+    res.push_back(affine::AffineApplyOp::create(b, loc, exprMap, operands));
   }
   return res;
 }
@@ -70,7 +70,7 @@ static void inlineRegionAndEmitStore(OpBuilder &b, Location loc, OpType op,
   Operation *terminator = block.getTerminator();
   for (OpOperand &operand : terminator->getOpOperands()) {
     Value toStore = map.lookupOrDefault(operand.get());
-    b.create<StoreOpTy>(loc, toStore, outputBuffers[operand.getOperandNumber()],
+    StoreOpTy::create(b, loc, toStore, outputBuffers[operand.getOperandNumber()],
                         indexing[operand.getOperandNumber()]);
   }
 }
@@ -145,7 +145,7 @@ static void emitScalarImplementation(OpBuilder &b, Location loc,
     auto indexing = makeCanonicalAffineApplies(
         b, loc, linalgOp.getMatchingIndexingMap(inputOperand), allIvsPlusDims);
     indexedValues.push_back(
-        b.create<LoadOpTy>(loc, inputOperand->get(), indexing));
+        LoadOpTy::create(b, loc, inputOperand->get(), indexing));
   }
   // 1.b. Emit load from output views.
   for (OpOperand &outputOperand : linalgOp.getDpsInitsMutable()) {
@@ -153,7 +153,7 @@ static void emitScalarImplementation(OpBuilder &b, Location loc,
         b, loc, linalgOp.getMatchingIndexingMap(&outputOperand),
         allIvsPlusDims);
     indexedValues.push_back(
-        b.create<LoadOpTy>(loc, outputOperand.get(), indexing));
+        LoadOpTy::create(b, loc, outputOperand.get(), indexing));
   }
 
   // TODO: When a region inliner exists, use it.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
index ee1957aaa6a53..28b246714df53 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
@@ -131,17 +131,17 @@ static Value createDestinationPassingStyleInitOperand(
     ImplicitLocOpBuilder &builder) {
   Value processLinearIndexInReductionGroup = mesh::createProcessLinearIndex(
       meshOp.getSymName(), reductionMeshAxes, builder);
-  Value zero = builder.create<arith::ConstantIndexOp>(0);
-  Value isLeadProcess = builder.create<arith::CmpIOp>(
+  Value zero = arith::ConstantIndexOp::create(builder, 0);
+  Value isLeadProcess = arith::CmpIOp::create(builder,
       builder.getI1Type(), arith::CmpIPredicate::eq,
       processLinearIndexInReductionGroup, zero);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(spmdizedOperand.getType(),
+  scf::IfOp ifOp = scf::IfOp::create(builder, spmdizedOperand.getType(),
                                              isLeadProcess, true, true);
   // Then block.
   {
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPointToEnd(&ifOp.getThenRegion().front());
-    builder.create<scf::YieldOp>(spmdizedOperand);
+    scf::YieldOp::create(builder, spmdizedOperand);
   }
 
   // Else block.
@@ -157,14 +157,14 @@ static Value createDestinationPassingStyleInitOperand(
     std::optional<TypedAttr> neutralEl =
         arith::getNeutralElement(combinerOps[0]);
 
-    Value init = builder.create<tensor::EmptyOp>(op.getLoc(), shape,
+    Value init = tensor::EmptyOp::create(builder, op.getLoc(), shape,
                                                  neutralEl.value().getType());
     Value constant =
-        builder.create<arith::ConstantOp>(op.getLoc(), neutralEl.value());
-    Value fill = builder.create<linalg::FillOp>(op.getLoc(), constant, init)
+        arith::ConstantOp::create(builder, op.getLoc(), neutralEl.value());
+    Value fill = linalg::FillOp::create(builder, op.getLoc(), constant, init)
                      .getResult(0);
 
-    builder.create<scf::YieldOp>(fill);
+    scf::YieldOp::create(builder, fill);
   }
   return ifOp.getResult(0);
 }
@@ -202,7 +202,7 @@ static void createAllReduceForResultWithoutPartialSharding(
   }
 
   Value spmdizedLinalgOpResult = spmdizationMap.lookup(unshardedLinalgOpResult);
-  Value reducedValue = builder.create<mesh::AllReduceOp>(
+  Value reducedValue = mesh::AllReduceOp::create(builder,
       spmdizedLinalgOpResult, resultSharding.getMesh(), allReduceMeshAxes,
       reductionKind);
   spmdizationMap.map(unshardedLinalgOpResult, reducedValue);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
index bb1e974391878..019c845b4e0e7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
@@ -59,7 +59,7 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
   auto newKernelTy = RankedTensorType::get(
       {kernelTy.getDimSize(0), kernelTy.getDimSize(1), kernelTy.getDimSize(2)},
       kernelTy.getElementType());
-  auto collapsedKernel = rewriter.create<tensor::CollapseShapeOp>(
+  auto collapsedKernel = tensor::CollapseShapeOp::create(rewriter,
       loc, newKernelTy, kernel, collapsedKernelDims);
 
   // Collapse init dims.
@@ -70,7 +70,7 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       RankedTensorType::get({initTy.getDimSize(0), initTy.getDimSize(1),
                              initTy.getDimSize(2), initTy.getDimSize(3)},
                             initTy.getElementType());
-  auto collapsedInit = rewriter.create<tensor::CollapseShapeOp>(
+  auto collapsedInit = tensor::CollapseShapeOp::create(rewriter,
       loc, newInitTy, init, collapsedInitDims);
 
   SmallVector<NamedAttribute> preservedAttrs;
@@ -78,13 +78,13 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       TypeSwitch<Operation *, Operation *>(operation)
           .Case<DepthwiseConv2DNhwcHwcmOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return rewriter.create<DepthwiseConv2DNhwcHwcOp>(
+            return DepthwiseConv2DNhwcHwcOp::create(rewriter,
                 loc, newInitTy, ValueRange{input, collapsedKernel},
                 ValueRange{collapsedInit}, stride, dilation);
           })
           .Case<DepthwiseConv2DNhwcHwcmQOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return rewriter.create<DepthwiseConv2DNhwcHwcQOp>(
+            return DepthwiseConv2DNhwcHwcQOp::create(rewriter,
                 loc, newInitTy, ValueRange{input, collapsedKernel, iZp, kZp},
                 ValueRange{collapsedInit}, stride, dilation);
           })
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
index 2afa2f9b71c2a..1368bb7e77db0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
@@ -143,7 +143,7 @@ struct SimplifyUnPackToCollapseShape : public OpRewritePattern<UnPackOp> {
                        Type newOperandType, ArrayAttr reassociation) const {
     if (operand.getType() == newOperandType)
       return operand;
-    return rewriter.create<tensor::CollapseShapeOp>(loc, newOperandType,
+    return tensor::CollapseShapeOp::create(rewriter, loc, newOperandType,
                                                     operand, reassociation);
   }
 
@@ -265,7 +265,7 @@ struct FoldUnpackWithExtractSliceOp
 
     // Create a new empty output tensor.
     Type elementType = unpackOp.getDestType().getElementType();
-    Value output = rewriter.create<tensor::EmptyOp>(
+    Value output = tensor::EmptyOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getMixedSizes(), elementType);
     rewriter.replaceOpWithNewOp<UnPackOp>(
         sliceOp, unpackOp.getSource(), output, unpackOp.getInnerDimsPos(),
@@ -529,7 +529,7 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
     auto elemType =
         cast<ShapedType>(unPackOp->getResultTypes()[0]).getElementType();
-    Value output = rewriter.create<tensor::EmptyOp>(
+    Value output = tensor::EmptyOp::create(rewriter,
         unPackOp->getLoc(), unpackOpResultDims[0], elemType);
 
     rewriter.replaceOpWithNewOp<UnPackOp>(
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index 5eb3761f7aca1..fb43c1f92e5b3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -192,10 +192,10 @@ static Value padOperand(RewriterBase &rewriter, TilingInterface opToPad,
   if (auto complexTy =
           dyn_cast<ComplexType>(getElementTypeOrSelf(v.getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingValueAttr);
-    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
                                                         complexTy, complexAttr);
   } else {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         opToPad.getLoc(), cast<TypedAttr>(paddingValueAttr));
   }
 
@@ -323,7 +323,7 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
         loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index dc9e11eccac4d..af614e88fb3d5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -219,10 +219,10 @@ static FailureOr<Value> padOperandToSmallestStaticBoundingBox(
   if (auto complexTy = dyn_cast<ComplexType>(
           getElementTypeOrSelf(opOperand->get().getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingAttr);
-    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
                                                         complexTy, complexAttr);
   } else {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         opToPad.getLoc(), cast<TypedAttr>(paddingAttr));
   }
 
@@ -313,7 +313,7 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
         loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index b274502e16903..4042a43b04091 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -63,10 +63,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
     staticBufferType =
         MemRefType::Builder(staticBufferType).setMemorySpace(memorySpaceAttr);
     if (options.useAlloca) {
-      return b.create<memref::AllocaOp>(staticBufferType, ValueRange{},
+      return memref::AllocaOp::create(b, staticBufferType, ValueRange{},
                                         alignmentAttr);
     }
-    return b.create<memref::AllocOp>(staticBufferType, ValueRange{},
+    return memref::AllocOp::create(b, staticBufferType, ValueRange{},
                                      alignmentAttr);
   }
 
@@ -76,10 +76,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
   dynamicBufferType =
       MemRefType::Builder(dynamicBufferType).setMemorySpace(memorySpaceAttr);
   Value mul = b.createOrFold<arith::MulIOp>(
-      b.create<arith::ConstantIndexOp>(width), allocSize);
+      arith::ConstantIndexOp::create(b, width), allocSize);
   if (options.useAlloca)
-    return b.create<memref::AllocaOp>(dynamicBufferType, mul, alignmentAttr);
-  return b.create<memref::AllocOp>(dynamicBufferType, mul, alignmentAttr);
+    return memref::AllocaOp::create(b, dynamicBufferType, mul, alignmentAttr);
+  return memref::AllocOp::create(b, dynamicBufferType, mul, alignmentAttr);
 }
 
 /// Default allocation callback function. This allocates a promoted buffer when
@@ -92,8 +92,8 @@ static std::optional<Value> defaultAllocBufferCallBack(
     std::optional<unsigned> alignment, DataLayout &layout) {
   ShapedType viewType = subView.getType();
   ImplicitLocOpBuilder b(subView.getLoc(), builder);
-  auto zero = b.create<arith::ConstantIndexOp>(0);
-  auto one = b.create<arith::ConstantIndexOp>(1);
+  auto zero = arith::ConstantIndexOp::create(b, 0);
+  auto one = arith::ConstantIndexOp::create(b, 1);
 
   Attribute memorySpaceAttr;
   if (options.memorySpace.has_value())
@@ -123,7 +123,7 @@ defaultDeallocBufferCallBack(const LinalgPromotionOptions &options,
                              OpBuilder &b, Value fullLocalView) {
   if (!options.useAlloca) {
     auto viewOp = cast<memref::ViewOp>(fullLocalView.getDefiningOp());
-    b.create<memref::DeallocOp>(viewOp.getSource().getLoc(),
+    memref::DeallocOp::create(b, viewOp.getSource().getLoc(),
                                 viewOp.getSource());
   }
   return success();
@@ -211,7 +211,7 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
   Location loc = linalgOp.getLoc();
   auto defaultCopyCallBack = [loc](OpBuilder &b, Value src,
                                    Value dst) -> LogicalResult {
-    b.create<linalg::CopyOp>(loc, src, dst);
+    linalg::CopyOp::create(b, loc, src, dst);
     return success();
   };
   copyInFn = (options.copyInFn ? *(options.copyInFn) : defaultCopyCallBack);
@@ -265,7 +265,7 @@ FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
               /*stopCondition=*/nullptr, /*closedUB=*/true);
       size = failed(upperBound)
                  ? getValueOrCreateConstantIndexOp(b, loc, rangeValue.size)
-                 : b.create<arith::ConstantIndexOp>(loc, *upperBound);
+                 : arith::ConstantIndexOp::create(b, loc, *upperBound);
     }
     LLVM_DEBUG(llvm::dbgs() << "Extracted tightest: " << size << "\n");
     fullSizes.push_back(size);
@@ -310,23 +310,23 @@ promoteSubViews(ImplicitLocOpBuilder &b,
     Value fillVal =
         llvm::TypeSwitch<Type, Value>(subviewEltType)
             .Case([&](FloatType t) {
-              return b.create<arith::ConstantOp>(FloatAttr::get(t, 0.0));
+              return arith::ConstantOp::create(b, FloatAttr::get(t, 0.0));
             })
             .Case([&](IntegerType t) {
-              return b.create<arith::ConstantOp>(IntegerAttr::get(t, 0));
+              return arith::ConstantOp::create(b, IntegerAttr::get(t, 0));
             })
             .Case([&](ComplexType t) {
               Value tmp;
               if (auto et = dyn_cast<FloatType>(t.getElementType()))
-                tmp = b.create<arith::ConstantOp>(FloatAttr::get(et, 0.0));
+                tmp = arith::ConstantOp::create(b, FloatAttr::get(et, 0.0));
               else if (auto et = cast<IntegerType>(t.getElementType()))
-                tmp = b.create<arith::ConstantOp>(IntegerAttr::get(et, 0));
-              return b.create<complex::CreateOp>(t, tmp, tmp);
+                tmp = arith::ConstantOp::create(b, IntegerAttr::get(et, 0));
+              return complex::CreateOp::create(b, t, tmp, tmp);
             })
             .Default([](auto) { return Value(); });
     if (!fillVal)
       return failure();
-    b.create<linalg::FillOp>(fillVal, promotionInfo->fullLocalView);
+    linalg::FillOp::create(b, fillVal, promotionInfo->fullLocalView);
   }
 
   // Copy data into the promoted buffers. Use callback if provided.
@@ -459,9 +459,9 @@ static std::optional<Value> allocateSubviewGPUMemoryInAddressSpace(
       gpu::AddressSpaceAttr::get(builder.getContext(), addressSpace));
   Value buffer;
   if (addressSpace == gpu::GPUDialect::getWorkgroupAddressSpace()) {
-    buffer = builder.create<memref::AllocOp>(funcOp.getLoc(), type);
+    buffer = memref::AllocOp::create(builder, funcOp.getLoc(), type);
   } else if (addressSpace == gpu::GPUDialect::getPrivateAddressSpace()) {
-    buffer = builder.create<memref::AllocaOp>(funcOp.getLoc(), type);
+    buffer = memref::AllocaOp::create(builder, funcOp.getLoc(), type);
   } else {
     return std::nullopt;
   }
@@ -487,9 +487,9 @@ LogicalResult mlir::linalg::deallocateWorkgroupMemory(OpBuilder &,
 /// the copy operation to ensure data integrity.
 LogicalResult mlir::linalg::copyToWorkgroupMemory(OpBuilder &b, Value src,
                                                   Value dst) {
-  b.create<gpu::BarrierOp>(src.getLoc());
-  Operation *copyOp = b.create<memref::CopyOp>(src.getLoc(), src, dst);
-  b.create<gpu::BarrierOp>(copyOp->getLoc());
+  gpu::BarrierOp::create(b, src.getLoc());
+  Operation *copyOp = memref::CopyOp::create(b, src.getLoc(), src, dst);
+  gpu::BarrierOp::create(b, copyOp->getLoc());
   return success();
 }
 
@@ -504,7 +504,7 @@ std::optional<Value> mlir::linalg::allocateGPUPrivateMemory(
 /// Normal copy to between src and dst.
 LogicalResult mlir::linalg::copyToGPUPrivateMemory(OpBuilder &b, Value src,
                                                    Value dst) {
-  b.create<memref::CopyOp>(src.getLoc(), src, dst);
+  memref::CopyOp::create(b, src.getLoc(), src, dst);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
index b30182dc84079..eac0e47b18a7d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
@@ -38,8 +38,8 @@ struct StructuredOpInterface
     SmallVector<Range> loopRanges = linalgOp.createLoopRanges(builder, loc);
     auto [starts, ends, _] = getOffsetsSizesAndStrides(loopRanges);
 
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    auto one = arith::ConstantIndexOp::create(builder, loc, 1);
 
     // Subtract one from the loop ends before composing with the indexing map
     transform(ends, ends.begin(), [&](OpFoldResult end) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
index 671dea8bb415f..1fac80142dddf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
@@ -52,7 +52,7 @@ createSplitPart(RewriterBase &b, Location loc, TilingInterface op,
       return nullptr;
     SmallVector<OpFoldResult> resultStrides(resultOffsets.size(),
                                             b.getIndexAttr(1));
-    Value inserted = b.create<tensor::InsertSliceOp>(
+    Value inserted = tensor::InsertSliceOp::create(b,
         loc, result, resultOperands[index], resultOffsets, resultSizes,
         resultStrides);
     results.push_back(inserted);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
index 5bfdbc6d0bb59..1b0bfbbec8624 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
@@ -115,7 +115,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
         newShape,
         cast<RankedTensorType>(operand->get().getType()).getElementType());
 
-    Value newInput = b.create<tensor::ExpandShapeOp>(
+    Value newInput = tensor::ExpandShapeOp::create(b,
         loc, newType, operand->get(), reassociation);
     newInputs.push_back(newInput);
   }
@@ -140,18 +140,18 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   Value emptyOrAllocTensor;
   if (useAlloc) {
-    emptyOrAllocTensor = b.create<bufferization::AllocTensorOp>(
+    emptyOrAllocTensor = bufferization::AllocTensorOp::create(b,
         loc,
         RankedTensorType::get(newOutputShape,
                               op.getRegionOutputArgs()[0].getType()),
         ValueRange{});
   } else {
-    emptyOrAllocTensor = b.create<tensor::EmptyOp>(
+    emptyOrAllocTensor = tensor::EmptyOp::create(b,
         loc, newOutputShape, op.getRegionOutputArgs()[0].getType());
   }
-  Value constantOp = b.create<arith::ConstantOp>(loc, *identity);
+  Value constantOp = arith::ConstantOp::create(b, loc, *identity);
   Value identityTensor =
-      b.create<linalg::FillOp>(op->getLoc(), constantOp, emptyOrAllocTensor)
+      linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor)
           .getResult(0);
 
   newMaps.push_back(AffineMap::get(oldOutputMap.getNumDims() + 1, 0, outputExpr,
@@ -168,7 +168,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   // Create the new op matching the original op with an extra parallel
   // dimension.
-  GenericOp genericOp = b.create<GenericOp>(
+  GenericOp genericOp = GenericOp::create(b,
       loc, TypeRange({emptyOrAllocTensor.getType()}), newInputs,
       ValueRange({identityTensor}), newMaps, newIteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
@@ -191,14 +191,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   AffineMap outputMap = AffineMap::get(intermRank, 0, exprs, op.getContext());
   SmallVector<AffineMap> reductionMaps = {inputMap, outputMap};
 
-  auto reduction = b.create<GenericOp>(
+  auto reduction = GenericOp::create(b,
       loc, op->getResultTypes(), ValueRange({genericOp.getResult(0)}),
       op.getDpsInits(), reductionMaps, reductionIteratorTypes,
       [reductionOp](OpBuilder &b, Location loc, ValueRange inputs) {
         Operation *clonedReductionOp = b.clone(*reductionOp);
         clonedReductionOp->setOperand(0, inputs[0]);
         clonedReductionOp->setOperand(1, inputs[1]);
-        b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+        linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
       });
   b.replaceOp(op, reduction.getResults());
 
@@ -318,14 +318,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
     Value emptyOrAllocTensor;
     if (useAlloc) {
       emptyOrAllocTensor =
-          b.create<bufferization::AllocTensorOp>(loc, newT, dims);
+          bufferization::AllocTensorOp::create(b, loc, newT, dims);
     } else {
-      emptyOrAllocTensor = b.create<tensor::EmptyOp>(loc, newT.getShape(),
+      emptyOrAllocTensor = tensor::EmptyOp::create(b, loc, newT.getShape(),
                                                      t.getElementType(), dims);
     }
-    Value constantOp = b.create<arith::ConstantOp>(loc, std::get<1>(it));
+    Value constantOp = arith::ConstantOp::create(b, loc, std::get<1>(it));
     fillOps.push_back(
-        b.create<linalg::FillOp>(op->getLoc(), constantOp, emptyOrAllocTensor));
+        linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor));
     newOutputs.push_back(fillOps.back().getResult(0));
     emptyOrAllocTensorOps.push_back(emptyOrAllocTensor.getDefiningOp());
   }
@@ -354,7 +354,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   SmallVector<Value> newInputs = op.getDpsInputs();
   // Add a single shape-only tensor to carry the dimensions without resorting to
   // more complex inversions.
-  newInputs.push_back(b.create<tensor::EmptyOp>(
+  newInputs.push_back(tensor::EmptyOp::create(b,
       loc, ArrayRef<int64_t>{reductionDimSize / splitFactor, splitFactor},
       b.getIntegerType(1)));
   // Output tensors are already good to go.
@@ -365,7 +365,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   iteratorTypes.insert(iteratorTypes.begin() + reductionDimPos,
                        utils::IteratorType::parallel);
   GenericOp genericOp =
-      b.create<GenericOp>(loc, ValueRange(newOutputs).getTypes(), newInputs,
+      GenericOp::create(b, loc, ValueRange(newOutputs).getTypes(), newInputs,
                           newOutputs, newMaps, iteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
                        genericOp.getRegion().begin());
@@ -396,7 +396,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
         utils::IteratorType::reduction;
 
     // clang-format off
-    auto reductionOp = b.create<GenericOp>(
+    auto reductionOp = GenericOp::create(b,
         loc,
         originalOutputType,
         reindexedOutput,
@@ -407,7 +407,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
           Operation *clonedReductionOp = b.clone(*combinerOp);
           clonedReductionOp->setOperand(0, bbArgs[0]);
           clonedReductionOp->setOperand(1, bbArgs[1]);
-          b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+          linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
         });
     // clang-format on
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
index d35aad514e884..854d3855d654d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
@@ -29,7 +29,7 @@ struct SwapExtractSliceOfFill final
     if (!fillOp || !fillOp->hasOneUse())
       return failure();
 
-    auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
         extractOp.getLoc(), extractOp.getType(), fillOp.getOutputs()[0],
         extractOp.getMixedOffsets(), extractOp.getMixedSizes(),
         extractOp.getMixedStrides());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 4741afe8a417d..c2803423fafd7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -94,10 +94,10 @@ static void emitIsPositiveIndexAssertion(ImplicitLocOpBuilder &b,
     return;
   }
 
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value condition = b.create<arith::CmpIOp>(arith::CmpIPredicate::sgt,
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value condition = arith::CmpIOp::create(b, arith::CmpIPredicate::sgt,
                                             cast<Value>(value), zero);
-  b.create<cf::AssertOp>(
+  cf::AssertOp::create(b,
       condition,
       b.getStringAttr("expected strictly positive tile size and divisor"));
 }
@@ -317,9 +317,9 @@ mlir::linalg::computeMultiTileSizes(OpBuilder &builder, LinalgOp op,
     Value coveredSize =
         apply(s0 * s1 + s2 * s3, {spec.lowTileSize, spec.lowTripCount,
                                   spec.highTileSize, spec.highTripCount});
-    Value equals = b.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+    Value equals = arith::CmpIOp::create(b, arith::CmpIPredicate::eq,
                                            coveredSize, tripCount);
-    b.create<cf::AssertOp>(
+    cf::AssertOp::create(b,
         equals, builder.getStringAttr(
                     "could not compute dynamic multi-size tile shapes"));
   }
@@ -656,7 +656,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       getValueOrCreateConstantIndexOp(b, loc, nonZeroNumThreads);
 
   // 2. Create the ForallOp with an empty region.
-  scf::ForallOp forallOp = b.create<scf::ForallOp>(
+  scf::ForallOp forallOp = scf::ForallOp::create(b,
       loc, getAsOpFoldResult(materializedNonZeroNumThreads), initTensors,
       mapping);
 
@@ -689,7 +689,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       sizes[reductionDim] = b.getIndexAttr(1);
       outOffsets[reductionDim] = forallOp.getInductionVars()[0];
       // TODO: use SubsetExtractOpInterface once it is available.
-      tiledDpsInitOperands.push_back(b.create<tensor::ExtractSliceOp>(
+      tiledDpsInitOperands.push_back(tensor::ExtractSliceOp::create(b,
           loc, cast<RankedTensorType>(initOperand.getType()),
           destBbArgs[destNum], outOffsets, sizes, strides));
     }
@@ -768,7 +768,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
     // 6.b. Parallel insertions are inserted at the end of the combining
     // terminator.
     b.setInsertionPointToEnd(forallOp.getTerminator().getBody());
-    b.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(b,
         loc, result, bbArg, resultOffsetsRank, resultSizesRank, strides);
   }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
index 513cecef29b61..94449fae260ee 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
@@ -44,7 +44,7 @@ static SmallVector<Value> getIndicesForAccess(OpBuilder &b, Location loc,
   for (auto result : indexingMap.getResults()) {
     AffineMap m = AffineMap::get(indexingMap.getNumDims(),
                                  indexingMap.getNumSymbols(), result);
-    Value v = b.create<affine::AffineApplyOp>(loc, m, ivs);
+    Value v = affine::AffineApplyOp::create(b, loc, m, ivs);
     indices.push_back(v);
   }
   return indices;
@@ -72,7 +72,7 @@ static LogicalResult inlinePayload(OpBuilder &b, LinalgOp linalgOp,
     OpOperand *storeInto = linalgOp.getDpsInitOperand(operand.index());
     auto indices = getIndicesForAccess(
         b, loc, linalgOp.getMatchingIndexingMap(storeInto), ivs);
-    b.create<memref::StoreOp>(
+    memref::StoreOp::create(b,
         loc, toStore, linalgOp.getDpsInitOperand(operand.index())->get(),
         indices);
   }
@@ -351,7 +351,7 @@ struct LinalgOpTilingInterface
       SmallVector<Value> indices = getIndicesForAccess(
           builder, linalgOpLoc, linalgOp.getMatchingIndexingMap(&operand), ivs);
       Value load =
-          builder.create<memref::LoadOp>(linalgOpLoc, operand.get(), indices);
+          memref::LoadOp::create(builder, linalgOpLoc, operand.get(), indices);
       indexedValues.push_back(load);
     }
 
@@ -519,10 +519,10 @@ struct LinalgOpPartialReductionInterface
 
       Type elType = getElementTypeOrSelf(result.getType());
       Value emptyTensor =
-          b.create<tensor::EmptyOp>(loc, partialResultShape, elType);
-      Value constantOp = b.create<arith::ConstantOp>(loc, *identity);
+          tensor::EmptyOp::create(b, loc, partialResultShape, elType);
+      Value constantOp = arith::ConstantOp::create(b, loc, *identity);
       auto identityTensor =
-          b.create<linalg::FillOp>(loc, constantOp, emptyTensor);
+          linalg::FillOp::create(b, loc, constantOp, emptyTensor);
       inits.push_back(identityTensor.getResult(0));
     }
 
@@ -574,7 +574,7 @@ struct LinalgOpPartialReductionInterface
       RankedTensorType sliceResultType = RankedTensorType::get(
           sliceInfo.resultShape, valueToTileType.getElementType(),
           valueToTileType.getEncoding());
-      auto sliceOp = b.create<tensor::ExtractSliceOp>(
+      auto sliceOp = tensor::ExtractSliceOp::create(b,
           loc, sliceResultType, valueToTile, sliceInfo.offsets, sliceInfo.sizes,
           sliceInfo.strides);
       tiledInits.push_back(sliceOp.getResult());
@@ -603,7 +603,7 @@ struct LinalgOpPartialReductionInterface
     auto resultTypes = ValueRange(tiledInits).getTypes();
     if (tilingStrategy ==
         ReductionTilingStrategy::PartialReductionOuterReduction) {
-      auto genericOp = b.create<GenericOp>(
+      auto genericOp = GenericOp::create(b,
           loc, resultTypes, tiledInputs, tiledInits, newMaps, newIteratorTypes);
       IRMapping mapping;
       op->getRegion(0).cloneInto(&genericOp.getRegion(),
@@ -648,7 +648,7 @@ struct LinalgOpPartialReductionInterface
         }
       }
 
-      auto reduction = b.create<linalg::ReduceOp>(
+      auto reduction = linalg::ReduceOp::create(b,
           loc, partialResult, init, partialReductionDims,
           [&linalgOp, &initIdx](OpBuilder &b, Location loc, ValueRange inputs) {
             // Get the combiner op.
@@ -659,7 +659,7 @@ struct LinalgOpPartialReductionInterface
             // Combine the input at idx and output at numInits + idx.
             clonedReductionOp->setOperand(0, inputs[0]);
             clonedReductionOp->setOperand(1, inputs[1]);
-            b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+            linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
           });
 
       mergeOperations.push_back(reduction);
@@ -790,7 +790,7 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto sourceSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getSource(), inputIndices, inputSizes, strides);
     tiledOperands.push_back(sourceSlice);
 
@@ -800,7 +800,7 @@ struct PackOpTiling
       return {};
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = b.create<tensor::ExtractSliceOp>(
+    auto outSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
@@ -809,7 +809,7 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = b.create<PackOp>(
+    Operation *tiledPackOp = PackOp::create(b,
         loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
@@ -968,7 +968,7 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto sourceSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getSource(), offsets, sizes, strides);
     tiledOperands.push_back(sourceSlice);
 
@@ -984,7 +984,7 @@ struct PackOpTiling
       return failure();
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = b.create<tensor::ExtractSliceOp>(
+    auto outSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
@@ -992,7 +992,7 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = b.create<PackOp>(
+    Operation *tiledPackOp = PackOp::create(b,
         loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
@@ -1172,7 +1172,7 @@ struct UnPackOpTiling
     sliceSrcSizes.append(unpackOp.getMixedTiles());
     sliceSrcStrides.append(numInnerTiles, oneAttr);
     SmallVector<Operation *> generatedSlices;
-    tensor::ExtractSliceOp sliceSource = b.create<tensor::ExtractSliceOp>(
+    tensor::ExtractSliceOp sliceSource = tensor::ExtractSliceOp::create(b,
         loc, unpackOp.getSource(), sliceSrcIndices, sliceSrcSizes,
         sliceSrcStrides);
     generatedSlices.push_back(sliceSource);
@@ -1180,12 +1180,12 @@ struct UnPackOpTiling
     SmallVector<OpFoldResult> destStrides(destRank, oneAttr);
     Value sliceDest;
     if (isPerfectTilingCase) {
-      auto destSliceOp = b.create<tensor::ExtractSliceOp>(
+      auto destSliceOp = tensor::ExtractSliceOp::create(b,
           loc, unpackOp.getDest(), offsets, sizes, destStrides);
       sliceDest = destSliceOp;
       generatedSlices.push_back(destSliceOp);
     } else {
-      sliceDest = b.create<tensor::EmptyOp>(
+      sliceDest = tensor::EmptyOp::create(b,
           loc, destExpandedSizes, unpackOp.getDestType().getElementType());
     }
 
@@ -1193,7 +1193,7 @@ struct UnPackOpTiling
     for (auto tile : unpackOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledUnpackOp = b.create<UnPackOp>(
+    Operation *tiledUnpackOp = UnPackOp::create(b,
         loc, TypeRange{sliceDest.getType()}, tiledOperands, op->getAttrs());
 
     if (isPerfectTilingCase)
@@ -1201,7 +1201,7 @@ struct UnPackOpTiling
                           SmallVector<Value>(tiledUnpackOp->getResults()),
                           generatedSlices};
 
-    auto extractSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractSlice = tensor::ExtractSliceOp::create(b,
         loc, tiledUnpackOp->getResult(0), resultOffsetsFromDest, sizes,
         destStrides);
     return TilingResult{
@@ -1337,13 +1337,13 @@ struct UnPackOpTiling
 
     SmallVector<Value> tiledOperands;
     // Create slice of the dest operand.
-    auto extractDestSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractDestSlice = tensor::ExtractSliceOp::create(b,
         loc, unPackOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(extractDestSlice);
 
     strides.append(unPackOp.getSourceRank() - outputRank, oneAttr);
     // Create slice of the source operand.
-    auto extractSourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractSourceSlice = tensor::ExtractSliceOp::create(b,
         loc, unPackOp.getSource(), offsets, sizes, strides);
     tiledOperands.insert(tiledOperands.begin(), extractSourceSlice);
     for (auto tile : unPackOp.getInnerTiles())
@@ -1351,7 +1351,7 @@ struct UnPackOpTiling
 
     // Create tiled unpack op.
     Operation *tiledUnPackOp =
-        b.create<UnPackOp>(loc, TypeRange{extractDestSlice.getType()},
+        UnPackOp::create(b, loc, TypeRange{extractDestSlice.getType()},
                            tiledOperands, op->getAttrs());
 
     return TilingResult{{tiledUnPackOp},
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index eab74dab4eb75..3526f62f770dc 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -269,11 +269,11 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       packingMetadata.reassociations);
   Value paddingValue = packOp.getPaddingValue();
   if (!paddingValue) {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(getElementTypeOrSelf(collapsed)));
   }
   auto padOp =
-      rewriter.create<tensor::PadOp>(loc, collapsed, packOp.getSource(), lows,
+      tensor::PadOp::create(rewriter, loc, collapsed, packOp.getSource(), lows,
                                      highs, paddingValue, /*nofold=*/false);
 
   LLVM_DEBUG(
@@ -313,7 +313,7 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       SmallVector<OpFoldResult> sizes =
           tensor::getMixedSizes(rewriter, loc, packOp.getDest());
 
-      auto insertSliceOp = rewriter.create<tensor::InsertSliceOp>(
+      auto insertSliceOp = tensor::InsertSliceOp::create(rewriter,
           loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
           /*offsets=*/zeros, sizes, /*strides=*/ones);
 
@@ -329,14 +329,14 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   // 5. Expand from the padded result to the stripMinedShape.
   auto expandShapeResultType =
       RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape);
-  auto reshapeOp = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapeOp = tensor::ExpandShapeOp::create(rewriter,
       loc, expandShapeResultType, padOp.getResult(),
       packingMetadata.reassociations);
 
   // 6. Transpose stripMinedShape to packedShape.
   SmallVector<int64_t> transpPerm =
       invertPermutationVector(packedToStripMinedShapePerm);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
 
   LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
@@ -371,7 +371,7 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
     SmallVector<OpFoldResult> sizes(packedRank - destShape.size(), one);
     sizes.append(tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()));
 
-    auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
         loc, destTensorType, unPackOp.getSource(),
         SmallVector<OpFoldResult>(packedRank, zero), sizes,
         SmallVector<OpFoldResult>(packedRank, one));
@@ -404,9 +404,9 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
   SmallVector<OpFoldResult, 4> dims =
       tensor::getMixedSizes(rewriter, loc, unPackOp.getSource());
   applyPermutationToVector(dims, packedToStripMinedShapePerm);
-  auto emptyOp = rewriter.create<tensor::EmptyOp>(
+  auto emptyOp = tensor::EmptyOp::create(rewriter,
       loc, dims, stripMinedTensorType.getElementType());
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm);
 
   LLVM_DEBUG(
@@ -426,20 +426,20 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
       DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
 
   // 4. Collapse from the stripMinedShape to the padded result.
-  auto reshapeOp = rewriter.create<tensor::CollapseShapeOp>(
+  auto reshapeOp = tensor::CollapseShapeOp::create(rewriter,
       loc, collapsedType, transposeOp->getResult(0),
       packingMetadata.reassociations);
 
   // 5. ExtractSlice.
   int64_t destRank = destTensorType.getRank();
-  auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+  auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
       loc, destTensorType, reshapeOp->getResult(0),
       SmallVector<OpFoldResult>(destRank, zero),
       tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()),
       SmallVector<OpFoldResult>(destRank, one));
 
   // 6. Inject a copy to preserve DPS.
-  auto copyOp = rewriter.create<linalg::CopyOp>(
+  auto copyOp = linalg::CopyOp::create(rewriter,
       loc, extractSliceOp->getResult(0), unPackOp.getDest());
 
   // 7. Replace unPackOp by copyOp.
@@ -554,15 +554,15 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
               operandType.getShape(), innerPos,
               cast<ShapedType>(dest.getType()).getShape(), {},
               innerPackSizes)) {
-        packOps.push_back(rewriter.create<linalg::PackOp>(
+        packOps.push_back(linalg::PackOp::create(rewriter,
             loc, operand, dest, innerPos, innerPackSizes));
       } else {
         // TODO: value of the padding attribute should be determined by
         // consumers.
         auto zeroAttr =
             rewriter.getZeroAttr(getElementTypeOrSelf(dest.getType()));
-        Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
-        packOps.push_back(rewriter.create<linalg::PackOp>(
+        Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
+        packOps.push_back(linalg::PackOp::create(rewriter,
             loc, operand, dest, innerPos, innerPackSizes, zero));
       }
       inputsAndInits.push_back(packOps.back());
@@ -574,7 +574,7 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       ValueRange{inputsAndInits}.take_front(linalgOp.getNumDpsInputs());
   ValueRange inits =
       ValueRange{inputsAndInits}.take_back(linalgOp.getNumDpsInits());
-  auto packedLinalgOp = rewriter.create<linalg::GenericOp>(
+  auto packedLinalgOp = linalg::GenericOp::create(rewriter,
       linalgOp.getLoc(), inits.getTypes(), inputs, inits, indexingMaps,
       iteratorTypes);
   packedLinalgOp.getRegion().takeBody(linalgOp->getRegion(0));
@@ -589,7 +589,7 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       continue;
     }
     // Build the symmetrical UnPackOp to the existing PackOp.
-    unPackOps.push_back(rewriter.create<linalg::UnPackOp>(
+    unPackOps.push_back(linalg::UnPackOp::create(rewriter,
         packedLinalgOp->getLoc(), result, maybePackedInit.getSource(),
         maybePackedInit.getInnerDimsPos(), maybePackedInit.getMixedTiles()));
     results.push_back(unPackOps.back());
@@ -655,7 +655,7 @@ static LinalgOp transposeOneLinalgOperandAndReplace(
   operands[opOperand.getOperandNumber()] = transposedValue;
 
   ValueRange operandsRef(operands);
-  auto transposedGenericOp = rewriter.create<linalg::GenericOp>(
+  auto transposedGenericOp = linalg::GenericOp::create(rewriter,
       /*location=*/linalgOp->getLoc(),
       /*resultTensorTypes=*/
       operandsRef.drop_front(linalgOp.getNumDpsInputs()).getTypes(),
@@ -904,7 +904,7 @@ mlir::linalg::LinalgTilingOptions::setTileSizes(ArrayRef<int64_t> ts) {
     b.setInsertionPointToStart(
         &op->getParentOfType<func::FuncOp>().getBody().front());
     return llvm::to_vector<4>(map_range(tileSizes, [&](int64_t s) {
-      Value v = b.create<arith::ConstantIndexOp>(op->getLoc(), s);
+      Value v = arith::ConstantIndexOp::create(b, op->getLoc(), s);
       return v;
     }));
   };
@@ -926,11 +926,11 @@ Value DecomposePadOpPattern::createFillOrGenerateOp(
     // Move the padding value defined inside the PadOp block to outside.
     if (padValue.getParentBlock() == &padOp.getRegion().front())
       rewriter.moveOpBefore(padValue.getDefiningOp(), padOp);
-    return rewriter.create<FillOp>(padOp.getLoc(), padValue, dest).result();
+    return FillOp::create(rewriter, padOp.getLoc(), padValue, dest).result();
   }
 
   // Fill could not be optimized: Lower to tensor::GenerateOp with region.
-  auto generateOp = rewriter.create<tensor::GenerateOp>(
+  auto generateOp = tensor::GenerateOp::create(rewriter,
       padOp.getLoc(), padOp.getResultType(), dynSizes);
   // Copy region to new op.
   IRMapping bvm;
@@ -970,7 +970,7 @@ DecomposePadOpPattern::matchAndRewrite(tensor::PadOp padOp,
   }
 
   // Init tensor and fill it with padding.
-  Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+  Value emptyTensor = tensor::EmptyOp::create(rewriter,
       padOp.getLoc(), staticSizes, resultType.getElementType(), dynSizes);
   Value fill = createFillOrGenerateOp(rewriter, padOp, emptyTensor, dynSizes);
 
@@ -1222,11 +1222,11 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
 
   applyPermutationToVector<OpFoldResult>(transShapeForEmptyOp,
                                          srcPermForTranspose);
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, transShapeForEmptyOp, packOp.getSourceType().getElementType());
 
   // 2.2 Create linalg.transpose
-  auto transposedOp = rewriter.create<linalg::TransposeOp>(loc, input, empty,
+  auto transposedOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
                                                            srcPermForTranspose);
 
   // 3. Insert the inner tile to the destination:
@@ -1246,7 +1246,7 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   }
 
   // 4. Replace tensor.packOp with tensor.insert_slice created above
-  auto insert = rewriter.create<tensor::InsertSliceOp>(
+  auto insert = tensor::InsertSliceOp::create(rewriter,
       loc, transposedOp.getResult()[0], packOp.getDest(), writeOffsets,
       writeSizes, writeStrides);
   rewriter.replaceOp(packOp, insert.getResult());
@@ -1313,7 +1313,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     // outer-untiled-dims
     if (ShapedType::isDynamic(srcShape[i])) {
       OpFoldResult dynamicDim =
-          rewriter.create<tensor::DimOp>(loc, source, i).getResult();
+          tensor::DimOp::create(rewriter, loc, source, i).getResult();
       extractSliceSizes.push_back(dynamicDim);
       shapeForEmptyOp.push_back(dynamicDim);
     } else {
@@ -1340,7 +1340,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   readShapeForExtractSlice.append(tileShape.begin(), tileShape.end());
   Type elemType = unpackOp.getSourceType().getElementType();
   auto readType = RankedTensorType::get(readShapeForExtractSlice, elemType);
-  Value innerTile = rewriter.create<tensor::ExtractSliceOp>(
+  Value innerTile = tensor::ExtractSliceOp::create(rewriter,
       loc, readType, unpackOp.getSource(), extractSliceOffsets,
       extractSliceSizes, extractSliceStrides);
 
@@ -1352,9 +1352,9 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   applyPermutationToVector<OpFoldResult>(shapeForEmptyOp, perm);
 
   Value empty =
-      rewriter.create<tensor::EmptyOp>(loc, shapeForEmptyOp, elemType);
+      tensor::EmptyOp::create(rewriter, loc, shapeForEmptyOp, elemType);
   auto transposedOp =
-      rewriter.create<linalg::TransposeOp>(loc, innerTile, empty, perm);
+      linalg::TransposeOp::create(rewriter, loc, innerTile, empty, perm);
 
   // 3. Handle in-complete tiles if needed. It truncates trailing data from the
   // transposed tile.
@@ -1369,7 +1369,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
           tensor::getMixedSize(rewriter, loc, unpackOp.getDest(), i));
   }
 
-  auto partialTile = rewriter.create<tensor::ExtractSliceOp>(
+  auto partialTile = tensor::ExtractSliceOp::create(rewriter,
       loc, transposedOp.getResult()[0], tileOffsets, tileSizes, tileStrides);
 
   // 4. Insert the result to the destination tensor.
@@ -1382,7 +1382,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     else
       writeSizes.push_back(oneIdxAttr);
   }
-  auto insert = rewriter.create<tensor::InsertSliceOp>(
+  auto insert = tensor::InsertSliceOp::create(rewriter,
       loc, partialTile, unpackOp.getDest(), writeOffsets, writeSizes,
       writeStrides);
   rewriter.replaceOp(unpackOp, insert.getResult());
@@ -1491,7 +1491,7 @@ FailureOr<Conv1DOp> DownscaleSizeOneWindowed2DConvolution<Conv2DOp, Conv1DOp>::
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = rewriter.create<Conv1DOp>(
+  auto conv1DOp = Conv1DOp::create(rewriter,
       loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
@@ -1578,7 +1578,7 @@ DownscaleDepthwiseConv2DNhwcHwcOp::returningMatchAndRewrite(
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = rewriter.create<DepthwiseConv1DNwcWcOp>(
+  auto conv1DOp = DepthwiseConv1DNwcWcOp::create(rewriter,
       loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
@@ -1635,7 +1635,7 @@ DownscaleConv2DOp::returningMatchAndRewrite(Conv2DOp convOp,
   Value newOutput = tensor::createCanonicalRankReducingExtractSliceOp(
       rewriter, loc, output, newOutputType);
 
-  auto conv1DOp = rewriter.create<Conv1DOp>(loc, newOutputType,
+  auto conv1DOp = Conv1DOp::create(rewriter, loc, newOutputType,
                                             ValueRange{newInput, newKernel},
                                             ValueRange{newOutput});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index 092aecceef6b3..b933a2d143925 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -67,7 +67,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
   Value input;
   if (isTensorOp) {
 
-    input = rewriter.create<tensor::EmptyOp>(loc, newFilterShape, elementTy)
+    input = tensor::EmptyOp::create(rewriter, loc, newFilterShape, elementTy)
                 .getResult();
   } else {
     input = rewriter
@@ -78,7 +78,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
 
   // We can then construct the transposition on our filter.
   auto transpose =
-      rewriter.create<linalg::TransposeOp>(loc, filter, input, filterPerm);
+      linalg::TransposeOp::create(rewriter, loc, filter, input, filterPerm);
 
   Value newFilter;
   if (isTensorOp) {
@@ -98,7 +98,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
     resultTy.push_back(op->getResult(0).getType());
   }
   auto newConv =
-      rewriter.create<HWCFConvOp>(loc, resultTy, newInputs, op.getOutputs(),
+      HWCFConvOp::create(rewriter, loc, resultTy, newInputs, op.getOutputs(),
                                   op.getStrides(), op.getDilations());
   rewriter.replaceOp(op, newConv);
   return newConv.getOperation();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index 934781d1cab75..6b1e89c1594d1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -47,24 +47,24 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
 
   SmallVector<Value> dynamicDims;
   if (type.isDynamicDim(1))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 1));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
   if (type.isDynamicDim(0))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, ArrayRef<int64_t>{shape[1], shape[0]}, type.getElementType(),
       dynamicDims);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, input, empty, ArrayRef<int64_t>{1, 0});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = rewriter.create<linalg::MatmulTransposeAOp>(
+    newMatmulOp = linalg::MatmulTransposeAOp::create(rewriter,
         loc, matmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), matmulOp.getInputs()[1]},
         matmulOp.getOutputs());
   } else {
-    newMatmulOp = rewriter.create<linalg::MatmulTransposeBOp>(
+    newMatmulOp = linalg::MatmulTransposeBOp::create(rewriter,
         loc, matmulOp.getResultTypes(),
         ValueRange{matmulOp.getInputs()[0], transposeOp->getResult(0)},
         matmulOp.getOutputs());
@@ -102,26 +102,26 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
 
   SmallVector<Value> dynamicDims;
   if (type.isDynamicDim(0))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
   if (type.isDynamicDim(2))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 2));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 2));
   if (type.isDynamicDim(1))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 1));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, ArrayRef<int64_t>{shape[0], shape[2], shape[1]},
       type.getElementType(), dynamicDims);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, input, empty, ArrayRef<int64_t>{0, 2, 1});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = rewriter.create<linalg::BatchMatmulTransposeAOp>(
+    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(rewriter,
         loc, batchMatmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), batchMatmulOp.getInputs()[1]},
         batchMatmulOp.getOutputs());
   } else {
-    newMatmulOp = rewriter.create<linalg::BatchMatmulTransposeBOp>(
+    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(rewriter,
         loc, batchMatmulOp.getResultTypes(),
         ValueRange{batchMatmulOp.getInputs()[0], transposeOp->getResult(0)},
         batchMatmulOp.getOutputs());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 5a8c5eab3f444..2fff28ffc7474 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -119,7 +119,7 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, input, /*offsets=*/ArrayRef<int64_t>{w + kw}, sizes, strides));
       }
     }
@@ -130,7 +130,7 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, input,
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             sizes, strides));
@@ -149,7 +149,7 @@ static SmallVector<Value> extractConvFilterSlices(RewriterBase &rewriter,
   // Extract rhs slice of size [{c, f} for channeled convolutions and {1} for
   // non-chanelled convolution] @ [kw].
   for (int64_t kw = 0; kw < kwSize; ++kw) {
-    result.push_back(rewriter.create<vector::ExtractOp>(
+    result.push_back(vector::ExtractOp::create(rewriter,
         loc, filter, /*offsets=*/ArrayRef<int64_t>{kw}));
   }
   return result;
@@ -167,7 +167,7 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {wSizeStep};
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/ArrayRef<int64_t>{w}, sizes, strides));
     }
   } else {
@@ -176,7 +176,7 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {nSize, wSizeStep, fSize};
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/ArrayRef<int64_t>{0, w, 0}, sizes, strides));
     }
   }
@@ -194,7 +194,7 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // This does not depend on kw.
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{w}, strides);
     }
   } else {
@@ -202,7 +202,7 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // convolution. This does not depend on kw.
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           strides);
     }
@@ -337,7 +337,7 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
   for (int vecDim = 0, end = canonicalVecShape.size(); vecDim < end; ++vecDim) {
     if (ShapedType::isStatic(iterSpaceStaticSizes[vecDim])) {
       // Create constant index op for static dimensions.
-      iterSpaceValueSizes.push_back(rewriter.create<arith::ConstantIndexOp>(
+      iterSpaceValueSizes.push_back(arith::ConstantIndexOp::create(rewriter,
           linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
       continue;
     }
@@ -351,9 +351,9 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
       return failure();
 
     Value dynamicDim = linalgOp.hasPureTensorSemantics()
-                           ? (Value)rewriter.create<tensor::DimOp>(
+                           ? (Value)tensor::DimOp::create(rewriter,
                                  linalgOp.getLoc(), operand, operandDimPos)
-                           : (Value)rewriter.create<memref::DimOp>(
+                           : (Value)memref::DimOp::create(rewriter,
                                  linalgOp.getLoc(), operand, operandDimPos);
     iterSpaceValueSizes.push_back(dynamicDim);
   }
@@ -474,7 +474,7 @@ Value VectorizationState::getOrCreateMaskFor(
          "Masked 0-d vectors are not supported yet");
 
   // Create the mask based on the dimension values.
-  Value mask = rewriter.create<vector::CreateMaskOp>(linalgOp.getLoc(),
+  Value mask = vector::CreateMaskOp::create(rewriter, linalgOp.getLoc(),
                                                      maskType, upperBounds);
   LDBG("Creating new mask: " << mask << "\n");
   activeMaskCache[maskingMap] = mask;
@@ -643,7 +643,7 @@ static Operation *buildMultiDimReduce(OpBuilder &b, Operation *reduceOp,
                                       ArrayRef<bool> dimsToMask) {
   auto maybeKind = getCombinerOpKind(reduceOp);
   assert(maybeKind && "Failed precondition: could not get reduction kind");
-  return b.create<vector::MultiDimReductionOp>(
+  return vector::MultiDimReductionOp::create(b,
       reduceOp->getLoc(), valueToReduce, acc, dimsToMask, *maybeKind);
 }
 
@@ -689,17 +689,17 @@ static Value buildVectorWrite(RewriterBase &rewriter, Value value,
   if (vectorType.getRank() > 0) {
     AffineMap writeMap = inversePermutation(reindexIndexingMap(opOperandMap));
     SmallVector<Value> indices(linalgOp.getRank(outputOperand),
-                               rewriter.create<arith::ConstantIndexOp>(loc, 0));
+                               arith::ConstantIndexOp::create(rewriter, loc, 0));
     value = broadcastIfNeeded(rewriter, value, vectorType);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = rewriter.create<vector::TransferWriteOp>(
+    write = vector::TransferWriteOp::create(rewriter,
         loc, value, outputOperand->get(), indices, writeMap);
   } else {
     // 0-d case is still special: do not invert the reindexing writeMap.
     if (!isa<VectorType>(value.getType()))
-      value = rewriter.create<vector::BroadcastOp>(loc, vectorType, value);
+      value = vector::BroadcastOp::create(rewriter, loc, vectorType, value);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = rewriter.create<vector::TransferWriteOp>(
+    write = vector::TransferWriteOp::create(rewriter,
         loc, value, outputOperand->get(), ValueRange{});
   }
 
@@ -778,7 +778,7 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto indexVectorType =
       VectorType::get({targetShape[dim]}, rewriter.getIndexType(),
                       state.getScalableVecDims()[dim]);
-  auto indexSteps = rewriter.create<vector::StepOp>(loc, indexVectorType);
+  auto indexSteps = vector::StepOp::create(rewriter, loc, indexVectorType);
   // Return the one-dimensional index vector if it lives in the trailing
   // dimension of the iteration space since the vectorization algorithm in this
   // case can handle the broadcast.
@@ -793,14 +793,14 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto permMap =
       AffineMap::getPermutationMap(permPattern, linalgOp.getContext());
 
-  auto broadCastOp = rewriter.create<vector::BroadcastOp>(
+  auto broadCastOp = vector::BroadcastOp::create(rewriter,
       loc, state.getCanonicalVecType(rewriter.getIndexType(), permMap),
       indexSteps);
   SmallVector<int64_t> transposition =
       llvm::to_vector<16>(llvm::seq<int64_t>(0, linalgOp.getNumLoops()));
   std::swap(transposition.back(), transposition[dim]);
   auto transposeOp =
-      rewriter.create<vector::TransposeOp>(loc, broadCastOp, transposition);
+      vector::TransposeOp::create(rewriter, loc, broadCastOp, transposition);
   return VectorizationHookResult{VectorizationHookStatus::NewOp, transposeOp};
 }
 
@@ -853,19 +853,19 @@ static Value calculateGatherOffset(RewriterBase &rewriter,
 
   const size_t numIndices = extractOp.getIndices().size();
   for (size_t i = 1; i < numIndices; i++) {
-    Value dimIdx = rewriter.create<arith::ConstantIndexOp>(loc, i);
+    Value dimIdx = arith::ConstantIndexOp::create(rewriter, loc, i);
 
     auto dimSize = broadcastIfNeeded(
         rewriter,
-        rewriter.create<tensor::DimOp>(loc, extractOp.getTensor(), dimIdx),
+        tensor::DimOp::create(rewriter, loc, extractOp.getTensor(), dimIdx),
         indexVecType);
 
-    offset = rewriter.create<arith::MulIOp>(loc, offset, dimSize);
+    offset = arith::MulIOp::create(rewriter, loc, offset, dimSize);
 
     auto extractOpIndex = broadcastIfNeeded(
         rewriter, bvm.lookup(extractOp.getIndices()[i]), indexVecType);
 
-    offset = rewriter.create<arith::AddIOp>(loc, extractOpIndex, offset);
+    offset = arith::AddIOp::create(rewriter, loc, extractOpIndex, offset);
   }
 
   return offset;
@@ -1110,18 +1110,18 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
 
   // Compute the static loop sizes of the extract op.
   auto resultType = state.getCanonicalVecType(extractOp.getResult().getType());
-  auto maskConstantOp = rewriter.create<arith::ConstantOp>(
+  auto maskConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseIntElementsAttr::get(state.getCanonicalVecType(rewriter.getI1Type()),
                                 /*value=*/true));
   auto passThruConstantOp =
-      rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(resultType));
+      arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(resultType));
 
   // Base indices are currently set to 0. We will need to re-visit if more
   // generic scenarios are to be supported.
   SmallVector<Value> baseIndices(
       extractOp.getIndices().size(),
-      rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      arith::ConstantIndexOp::create(rewriter, loc, 0));
 
   VectorMemoryAccessKind memAccessKind =
       getTensorExtractMemoryAccessPattern(extractOp, linalgOp, resultType);
@@ -1131,7 +1131,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     Value offset = calculateGatherOffset(rewriter, state, extractOp, bvm);
 
     // Generate the gather load
-    Operation *gatherOp = rewriter.create<vector::GatherOp>(
+    Operation *gatherOp = vector::GatherOp::create(rewriter,
         loc, resultType, extractOp.getTensor(), baseIndices, offset,
         maskConstantOp, passThruConstantOp);
     gatherOp = state.maskOperation(rewriter, gatherOp, linalgOp);
@@ -1166,13 +1166,13 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
       continue;
     }
 
-    auto indexAs1dVector = rewriter.create<vector::ShapeCastOp>(
+    auto indexAs1dVector = vector::ShapeCastOp::create(rewriter,
         loc,
         VectorType::get(resultType.getShape().back(), rewriter.getIndexType(),
                         resultType.getScalableDims().back()),
         idx);
     transferReadIdxs.push_back(
-        rewriter.create<vector::ExtractOp>(loc, indexAs1dVector, 0));
+        vector::ExtractOp::create(rewriter, loc, indexAs1dVector, 0));
   }
 
   // `tensor.extract_element` is always in-bounds, hence the following holds.
@@ -1186,7 +1186,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     SmallVector<AffineExpr> exprs(dstRank, getAffineConstantExpr(0, ctx));
     auto permutationMap = AffineMap::get(srcRank, 0, exprs, ctx);
 
-    auto transferReadOp = rewriter.create<vector::TransferReadOp>(
+    auto transferReadOp = vector::TransferReadOp::create(rewriter,
         loc, resultType, extractOp.getTensor(), transferReadIdxs,
         /*padding=*/std::nullopt, permutationMap, inBounds);
 
@@ -1195,7 +1195,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     // valid here).
     SmallVector<int64_t> readMaskShape = {1};
     auto readMaskType = VectorType::get(readMaskShape, rewriter.getI1Type());
-    auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+    auto allTrue = vector::ConstantMaskOp::create(rewriter,
         loc, readMaskType, vector::ConstantMaskKind::AllTrue);
     auto *maskedReadOp =
         mlir::vector::maskOperation(rewriter, transferReadOp, allTrue);
@@ -1223,7 +1223,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     rankDiff--;
   }
 
-  auto transferReadOp = rewriter.create<vector::TransferReadOp>(
+  auto transferReadOp = vector::TransferReadOp::create(rewriter,
       loc, resultType, extractOp.getTensor(), transferReadIdxs,
       /*padding=*/std::nullopt, permutationMap, inBounds);
 
@@ -1405,7 +1405,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
   // 3. Turn all BBArgs into vector.transfer_read / load.
   Location loc = linalgOp.getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   for (OpOperand *opOperand : linalgOp.getOpOperandsMatchingBBargs()) {
     BlockArgument bbarg = linalgOp.getMatchingBlockArgument(opOperand);
     if (linalgOp.isScalar(opOperand)) {
@@ -1435,7 +1435,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
-    Operation *read = rewriter.create<vector::TransferReadOp>(
+    Operation *read = vector::TransferReadOp::create(rewriter,
         loc, readType, opOperand->get(), indices,
         /*padding=*/std::nullopt, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
@@ -1452,7 +1452,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     // 3.c. Not all ops support 0-d vectors, extract the scalar for now.
     // TODO: remove this.
     if (readType.getRank() == 0)
-      readValue = rewriter.create<vector::ExtractOp>(loc, readValue,
+      readValue = vector::ExtractOp::create(rewriter, loc, readValue,
                                                      ArrayRef<int64_t>());
 
     LDBG("New vectorized bbarg(" << bbarg.getArgNumber() << "): " << readValue
@@ -1660,13 +1660,13 @@ createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
           writeIndices.size() == static_cast<size_t>(destRank)) &&
          "Invalid number of write indices!");
   if (writeIndices.empty()) {
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     writeIndices.assign(destRank, zero);
   }
 
   // Generate the xfer_write Op
   Operation *write =
-      builder.create<vector::TransferWriteOp>(loc,
+      vector::TransferWriteOp::create(builder, loc,
                                               /*vector=*/vecToStore,
                                               /*source=*/dest,
                                               /*indices=*/writeIndices,
@@ -1742,7 +1742,7 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   Location loc = packOp.getLoc();
   auto padValue = packOp.getPaddingValue();
   if (!padValue) {
-    padValue = rewriter.create<arith::ConstantOp>(
+    padValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(packOp.getSourceType().getElementType()));
   }
   ReifiedRankedShapedTypeDims reifiedReturnShapes;
@@ -1782,16 +1782,16 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   auto tiledPackType = VectorType::get(getTiledPackShape(packOp, destShape),
                                        packOp.getDestType().getElementType());
   auto shapeCastOp =
-      rewriter.create<vector::ShapeCastOp>(loc, tiledPackType, maskedRead);
+      vector::ShapeCastOp::create(rewriter, loc, tiledPackType, maskedRead);
 
   // Create TransposeOp.
   auto destPermutation =
       invertPermutationVector(getPackInverseDestPerm(packOp));
-  auto transposeOp = rewriter.create<vector::TransposeOp>(
+  auto transposeOp = vector::TransposeOp::create(rewriter,
       loc, shapeCastOp.getResult(), destPermutation);
 
   // Create TransferWriteOp.
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedReturnShapes[0],
       transposeOp.getResult().getType().getElementType());
   Operation *write =
@@ -1892,7 +1892,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   }
   Location loc = unpackOp->getLoc();
 
-  auto padValue = rewriter.create<arith::ConstantOp>(
+  auto padValue = arith::ConstantOp::create(rewriter,
       loc, rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
 
   // Read result, mask if necessary. If transferReadOp shape is not equal
@@ -1911,7 +1911,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   RankedTensorType stripMineTensorType =
       RankedTensorType::get(stripMineShape, stripMineElemType);
   // Transpose the appropriate rows to match output.
-  vector::TransposeOp transposeOp = rewriter.create<vector::TransposeOp>(
+  vector::TransposeOp transposeOp = vector::TransposeOp::create(rewriter,
       loc, readResult, lastDimToInsertPosPerm);
 
   // Collapse the vector to the size required by result.
@@ -1919,7 +1919,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       stripMineTensorType, packMetadata.reassociations);
   mlir::VectorType vecCollapsedType =
       VectorType::get(collapsedType.getShape(), collapsedType.getElementType());
-  vector::ShapeCastOp shapeCastOp = rewriter.create<vector::ShapeCastOp>(
+  vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(rewriter,
       loc, vecCollapsedType, transposeOp->getResult(0));
 
   // writeVectorSizes had to match the shapecast shape for dynamic sizes,
@@ -1928,7 +1928,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       unpackOp.getDestType().hasStaticShape()
           ? vectorSizes
           : shapeCastOp.getResultVectorType().getShape());
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedRetShapes[0],
       shapeCastOp.getResult().getType().getElementType());
   Operation *write = createWriteOrMaskedWrite(
@@ -1963,7 +1963,7 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
       /*useInBoundsInsteadOfMasking=*/false);
 
   // Create Xfer write Op
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedReturnShapes[0], padOp.getResultType().getElementType());
   Operation *write = createWriteOrMaskedWrite(rewriter, loc, maskedRead, dest);
   newResults.push_back(write->getResult(0));
@@ -2634,19 +2634,19 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
   auto writeType = VectorType::get(dstType.getShape(), dstElementType);
 
   Location loc = copyOp->getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   SmallVector<Value> indices(srcType.getRank(), zero);
 
-  Value readValue = rewriter.create<vector::TransferReadOp>(
+  Value readValue = vector::TransferReadOp::create(rewriter,
       loc, readType, copyOp.getSource(), indices,
       /*padding=*/std::nullopt,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   if (cast<VectorType>(readValue.getType()).getRank() == 0) {
     readValue =
-        rewriter.create<vector::ExtractOp>(loc, readValue, ArrayRef<int64_t>());
-    readValue = rewriter.create<vector::BroadcastOp>(loc, writeType, readValue);
+        vector::ExtractOp::create(rewriter, loc, readValue, ArrayRef<int64_t>());
+    readValue = vector::BroadcastOp::create(rewriter, loc, writeType, readValue);
   }
-  Operation *writeValue = rewriter.create<vector::TransferWriteOp>(
+  Operation *writeValue = vector::TransferWriteOp::create(rewriter,
       loc, readValue, copyOp.getTarget(), indices,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   rewriter.replaceOp(copyOp, writeValue->getResults());
@@ -2957,7 +2957,7 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   if (!padValue) {
     auto elemType = sourceType.getElementType();
-    padValue = rewriter.create<arith::ConstantOp>(
+    padValue = arith::ConstantOp::create(rewriter,
         sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
   }
 
@@ -2989,7 +2989,7 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   // Create read
   SmallVector<Value> readIndices(
-      vecType.getRank(), rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      vecType.getRank(), arith::ConstantIndexOp::create(rewriter, loc, 0));
   Value read = mlir::vector::createReadOrMaskedRead(
       rewriter, loc, source, vecType.getShape(), padValue,
       /*useInBoundsInsteadOfMasking=*/inputVectorSizes.empty());
@@ -3076,8 +3076,8 @@ struct PadOpVectorizationWithInsertSlicePattern
     // Generate TransferReadOp: Read entire source tensor and add high
     // padding.
     SmallVector<Value> readIndices(
-        vecRank, rewriter.create<arith::ConstantIndexOp>(padOp.getLoc(), 0));
-    auto read = rewriter.create<vector::TransferReadOp>(
+        vecRank, arith::ConstantIndexOp::create(rewriter, padOp.getLoc(), 0));
+    auto read = vector::TransferReadOp::create(rewriter,
         padOp.getLoc(), vecType, padOp.getSource(), readIndices, padValue);
 
     // Generate TransferWriteOp: Write to InsertSliceOp's dest tensor at
@@ -3212,7 +3212,7 @@ LogicalResult LinalgCopyVTRForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_read, the attribute must be reset
   // conservatively.
   auto vectorType = xferOp.getVectorType();
-  Value res = rewriter.create<vector::TransferReadOp>(
+  Value res = vector::TransferReadOp::create(rewriter,
       xferOp.getLoc(), vectorType, in, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getPadding(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(
@@ -3271,7 +3271,7 @@ LogicalResult LinalgCopyVTWForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_write, the attribute must be reset
   // conservatively.
   auto vector = xferOp.getVector();
-  rewriter.create<vector::TransferWriteOp>(
+  vector::TransferWriteOp::create(rewriter,
       xferOp.getLoc(), vector, out, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(SmallVector<bool>(
@@ -3467,7 +3467,7 @@ struct Conv1DGenerator
     }
 
     vector::TransferWriteOp write;
-    Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
     // w is unrolled (i.e. wSizeStep == 1) iff strideW > 1.
     // When strideW == 1, we can batch the contiguous loads and avoid
@@ -3486,16 +3486,16 @@ struct Conv1DGenerator
     SmallVector<Value> resPadding(resShape.size(), zero);
 
     // Read the whole lhs, rhs and res in one shot (with zero padding).
-    Value lhs = rewriter.create<vector::TransferReadOp>(
+    Value lhs = vector::TransferReadOp::create(rewriter,
         loc, lhsType, lhsShaped, lhsPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     // This is needed only for Conv.
     Value rhs = nullptr;
     if (oper == ConvOperationKind::Conv)
-      rhs = rewriter.create<vector::TransferReadOp>(
+      rhs = vector::TransferReadOp::create(rewriter,
           loc, rhsType, rhsShaped, rhsPadding,
           /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
-    Value res = rewriter.create<vector::TransferReadOp>(
+    Value res = vector::TransferReadOp::create(rewriter,
         loc, resType, resShaped, resPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
 
@@ -3511,16 +3511,16 @@ struct Conv1DGenerator
       // To match base vectorization case, we pre-transpose current case.
       // ncw -> nwc
       static constexpr std::array<int64_t, 3> permLhs = {0, 2, 1};
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, permLhs);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, permLhs);
       // fcw -> wcf
       static constexpr std::array<int64_t, 3> permRhs = {2, 1, 0};
 
       // This is needed only for Conv.
       if (oper == ConvOperationKind::Conv)
-        rhs = rewriter.create<vector::TransposeOp>(loc, rhs, permRhs);
+        rhs = vector::TransposeOp::create(rewriter, loc, rhs, permRhs);
       // nfw -> nwf
       static constexpr std::array<int64_t, 3> permRes = {0, 2, 1};
-      res = rewriter.create<vector::TransposeOp>(loc, res, permRes);
+      res = vector::TransposeOp::create(rewriter, loc, res, permRes);
       break;
     }
     }
@@ -3585,7 +3585,7 @@ struct Conv1DGenerator
     case Conv1DOpOrder::Ncw: {
       // nwf -> nfw
       static constexpr std::array<int64_t, 3> perm = {0, 2, 1};
-      res = rewriter.create<vector::TransposeOp>(loc, res, perm);
+      res = vector::TransposeOp::create(rewriter, loc, res, perm);
       break;
     }
     }
@@ -3609,16 +3609,16 @@ struct Conv1DGenerator
         cast<ShapedType>(val.getType()).cloneWith(std::nullopt, dstElementType);
 
     if (isa<IntegerType>(srcElementType) && isa<FloatType>(dstElementType)) {
-      return rewriter.create<arith::SIToFPOp>(loc, dstType, val);
+      return arith::SIToFPOp::create(rewriter, loc, dstType, val);
     }
 
     if (isa<FloatType>(srcElementType) && isa<FloatType>(dstElementType) &&
         srcWidth < dstWidth)
-      return rewriter.create<arith::ExtFOp>(loc, dstType, val);
+      return arith::ExtFOp::create(rewriter, loc, dstType, val);
 
     if (isa<IntegerType>(srcElementType) && isa<IntegerType>(dstElementType) &&
         srcWidth < dstWidth)
-      return rewriter.create<arith::ExtSIOp>(loc, dstType, val);
+      return arith::ExtSIOp::create(rewriter, loc, dstType, val);
 
     assert(false && "unhandled promotion case");
     return nullptr;
@@ -3633,7 +3633,7 @@ struct Conv1DGenerator
     bindDims(ctx, n, w, f, c);
     lhs = promote(rewriter, loc, lhs, res.getType());
     rhs = promote(rewriter, loc, rhs, res.getType());
-    auto contrationOp = rewriter.create<vector::ContractionOp>(
+    auto contrationOp = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, res,
         /*indexingMaps=*/MapList{{n, w, c}, {c, f}, {n, w, f}},
         /*iteratorTypes=*/ArrayRef<vector::IteratorType>{par, par, par, red});
@@ -3645,7 +3645,7 @@ struct Conv1DGenerator
   // convolution.
   Value conv1dSliceAsOuterProduct(RewriterBase &rewriter, Location loc,
                                   Value lhs, Value rhs, Value res) {
-    return rewriter.create<vector::OuterProductOp>(
+    return vector::OuterProductOp::create(rewriter,
         loc, res.getType(), lhs, rhs, res, vector::CombiningKind::ADD);
   }
 
@@ -3693,7 +3693,7 @@ struct Conv1DGenerator
     bindShapeDims(resShapedType, nSize, wSize);
 
     vector::TransferWriteOp write;
-    Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
     // w is unrolled (i.e. wSizeStep == 1) iff strideW > 1.
     // When strideW == 1, we can batch the contiguous loads and avoid
@@ -3736,28 +3736,28 @@ struct Conv1DGenerator
           cast<LinalgOp>(op).hasPureTensorSemantics(), opToMask, rewriter);
 
       Value maskOp =
-          rewriter.create<vector::CreateMaskOp>(loc, maskType, mixedDims);
+          vector::CreateMaskOp::create(rewriter, loc, maskType, mixedDims);
 
       return mlir::vector::maskOperation(rewriter, opToMask, maskOp);
     };
 
     // Read lhs slice of size {n, w * strideW + kw * dilationW, c} @ [0, 0,
     // 0].
-    Value lhs = rewriter.create<vector::TransferReadOp>(
+    Value lhs = vector::TransferReadOp::create(rewriter,
         loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     auto maybeMaskedLhs = maybeMaskXferOp(
         lhsType.getShape(), lhsType.getScalableDims(), lhs.getDefiningOp());
 
     // Read rhs slice of size {kw, c} @ [0, 0].
-    Value rhs = rewriter.create<vector::TransferReadOp>(
+    Value rhs = vector::TransferReadOp::create(rewriter,
         loc, rhsType, rhsShaped, ValueRange{zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
     auto maybeMaskedRhs = maybeMaskXferOp(
         rhsType.getShape(), rhsType.getScalableDims(), rhs.getDefiningOp());
 
     // Read res slice of size {n, w, c} @ [0, 0, 0].
-    Value res = rewriter.create<vector::TransferReadOp>(
+    Value res = vector::TransferReadOp::create(rewriter,
         loc, resType, resShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
     auto maybeMaskedRes = maybeMaskXferOp(
@@ -3775,7 +3775,7 @@ struct Conv1DGenerator
     //   @ [0, sw * w + dw * kw, 0].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        lhsVals.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        lhsVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, maybeMaskedLhs->getResult(0),
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             inOutSliceSizes, inOutStrides));
@@ -3783,13 +3783,13 @@ struct Conv1DGenerator
     }
     // Extract rhs slice of size {c} @ [kw].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
-      rhsVals.push_back(rewriter.create<vector::ExtractOp>(
+      rhsVals.push_back(vector::ExtractOp::create(rewriter,
           loc, maybeMaskedRhs->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{kw}));
     }
     // Extract res slice: {n, wSizeStep, c} @ [0, w, 0].
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      resVals.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      resVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0}, inOutSliceSizes,
           inOutStrides));
@@ -3815,16 +3815,16 @@ struct Conv1DGenerator
         if (flatten) {
           // Flatten the input and output vectors (collapse the channel
           // dimension)
-          lhsVal = rewriter.create<vector::ShapeCastOp>(
+          lhsVal = vector::ShapeCastOp::create(rewriter,
               loc, lhsTypeAfterFlattening, lhsVals[linearIndex(kw, w)]);
-          resVal = rewriter.create<vector::ShapeCastOp>(
+          resVal = vector::ShapeCastOp::create(rewriter,
               loc, resTypeAfterFlattening, resVals[w]);
         }
         resVals[w] = depthwiseConv1dSliceAsMulAcc(rewriter, loc, lhsVal,
                                                   rhsVals[kw], resVal, flatten);
         if (flatten) {
           // Un-flatten the output vector (restore the channel dimension)
-          resVals[w] = rewriter.create<vector::ShapeCastOp>(
+          resVals[w] = vector::ShapeCastOp::create(rewriter,
               loc, VectorType::get(inOutSliceSizes, resEltType), resVals[w]);
         }
       }
@@ -3843,7 +3843,7 @@ struct Conv1DGenerator
     // Write back res slice: {n, wSizeStep, c} @ [0, w, 0].
     // This does not depend on kw.
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      maybeMaskedRes = rewriter.create<vector::InsertStridedSliceOp>(
+      maybeMaskedRes = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           /*strides=*/ArrayRef<int64_t>{1, 1, 1});
@@ -3853,7 +3853,7 @@ struct Conv1DGenerator
     //===------------------------------------------------------------------===//
 
     // Write back res slice of size {n, w, c} @ [0, 0, 0].
-    Operation *resOut = rewriter.create<vector::TransferWriteOp>(
+    Operation *resOut = vector::TransferWriteOp::create(rewriter,
         loc, maybeMaskedRes->getResult(0), resShaped,
         ValueRange{zero, zero, zero});
     return maybeMaskXferOp(resType.getShape(), resType.getScalableDims(),
@@ -3891,10 +3891,10 @@ struct Conv1DGenerator
           indices.push_back(j);
       }
 
-      rhs = rewriter.create<vector::ShuffleOp>(loc, rhs, rhs, indices);
+      rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, indices);
     }
     // Broadcast the filter to match the output vector
-    rhs = rewriter.create<vector::BroadcastOp>(
+    rhs = vector::BroadcastOp::create(rewriter,
         loc, resTy.clone(rhsTy.getElementType()), rhs);
 
     rhs = promote(rewriter, loc, rhs, resTy);
@@ -3903,10 +3903,10 @@ struct Conv1DGenerator
       return nullptr;
 
     if (isa<FloatType>(resTy.getElementType()))
-      return rewriter.create<vector::FMAOp>(loc, lhs, rhs, res);
+      return vector::FMAOp::create(rewriter, loc, lhs, rhs, res);
 
-    auto mul = rewriter.create<arith::MulIOp>(loc, lhs, rhs);
-    return rewriter.create<arith::AddIOp>(loc, mul, res);
+    auto mul = arith::MulIOp::create(rewriter, loc, lhs, rhs);
+    return arith::AddIOp::create(rewriter, loc, mul, res);
   }
 
   /// Entry point for non-channeled convolution:
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index 9fd084487e3fd..f9c6aae9374a8 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -201,7 +201,7 @@ Value create2DTransformMatrix(OpBuilder &builder, Location loc,
                               TransformMatrix transform, Type type) {
   ArrayRef<float> constVec(transform.table, transform.rows * transform.cols);
 
-  return builder.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(builder,
       loc, DenseFPElementsAttr::get(
                RankedTensorType::get(
                    SmallVector<int64_t>{transform.rows, transform.cols}, type),
@@ -233,7 +233,7 @@ Value extract2DDataFrom4D(OpBuilder &builder, Location loc, Value source,
 
   auto extractFilterType =
       RankedTensorType::get({extractHeight, extractWidth}, elementType);
-  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
+  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
       loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
@@ -267,7 +267,7 @@ Value extract2DDataFrom6D(OpBuilder &builder, Location loc, Value source,
   SmallVector<OpFoldResult> strides(srcSize, oneIndex);
 
   auto extractFilterType = RankedTensorType::get({height, width}, elementType);
-  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
+  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
       loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
@@ -293,7 +293,7 @@ Value insert2DDataTo4D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
+  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
       loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
@@ -321,7 +321,7 @@ Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
+  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
       loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
@@ -372,7 +372,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
   if (filterW != r && filterW != 1)
     return Value();
 
-  Value zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
                        ValueRange args) -> scf::ValueVector {
     Value FIter = ivs[0];
@@ -386,7 +386,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
 
     int64_t retRows = 1;
     Value matmulRetValue = extractFilter;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix G.
@@ -401,11 +401,11 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value G = create2DTransformMatrix(builder, loc, GMatrix, elementType);
       // Multiply G x g.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{G, extractFilter}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -423,11 +423,11 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value GT = create2DTransformMatrix(builder, loc, GTMatrix, elementType);
       // Multiply u = (G x g) x GT.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, GT}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -445,9 +445,9 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
     return {insertSliceOp};
   };
 
-  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterF);
-  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterC);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterF);
+  auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterC);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx}, {fUpperBound, cUpperBound},
       {oneStep, oneStep}, {retValue}, buildBody);
@@ -516,9 +516,9 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = builder.create<affine::AffineApplyOp>(
+    Value heightOffset = affine::AffineApplyOp::create(builder,
         loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = builder.create<affine::AffineApplyOp>(
+    Value widthOffset = affine::AffineApplyOp::create(builder,
         loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     // Extract (H, W) from (N, H, W, C).
@@ -530,7 +530,7 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     int64_t retRows = 1;
     int64_t retCols = 1;
     Value matmulRetValue = extractInput;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix BT.
@@ -545,12 +545,12 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value BT =
           create2DTransformMatrix(builder, loc, BTMatrix, builder.getF32Type());
       // Multiply BT x d.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{BT, matmulRetValue}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -568,11 +568,11 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       Value B =
           create2DTransformMatrix(builder, loc, BMatrix, builder.getF32Type());
       // Multiply v = (BT x d) x B.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, B}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -586,12 +586,12 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     return {combinedVal};
   };
 
-  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tileH);
-  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
-  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputN);
-  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputC);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tileH);
+  auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
+  auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputN);
+  auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputC);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
       {tileHBound, tileWBound, nUpperBound, cUpperBound},
@@ -629,7 +629,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       {filterShape[0] * filterShape[1], filterShape[2], filterShape[3]},
       filterElementType);
   SmallVector<ReassociationIndices> filterReassoc = {{0, 1}, {2}, {3}};
-  Value collapseFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value collapseFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, filterReassocType, transformedFilter, filterReassoc);
 
   // Convert (alphaH, alphaW, tileH, tileW, N, C) to
@@ -643,7 +643,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
        inputShape[2] * inputShape[3] * inputShape[4], inputShape[5]},
       inputElementType);
   SmallVector<ReassociationIndices> inputReassoc = {{0, 1}, {2, 3, 4}, {5}};
-  Value collapseInput = rewriter.create<tensor::CollapseShapeOp>(
+  Value collapseInput = tensor::CollapseShapeOp::create(rewriter,
       loc, inputReassocType, transformedInput, inputReassoc);
 
   // Batched matrix multiply.
@@ -655,11 +655,11 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
                     .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                              outputElementType)
                     .getResult();
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, rewriter.getZeroAttr(outputElementType));
-  Value init = rewriter.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+  Value init = linalg::FillOp::create(rewriter, loc, zero, empty).getResult(0);
 
-  auto matmulOp = rewriter.create<linalg::BatchMatmulOp>(
+  auto matmulOp = linalg::BatchMatmulOp::create(rewriter,
       loc, matmulType, ValueRange({collapseInput, collapseFilter}),
       ValueRange{init});
 
@@ -670,7 +670,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       RankedTensorType::get({inputShape[0], inputShape[1], inputShape[2],
                              inputShape[3], inputShape[4], filterShape[3]},
                             outputElementType);
-  auto expandOutput = rewriter.create<tensor::ExpandShapeOp>(
+  auto expandOutput = tensor::ExpandShapeOp::create(rewriter,
       loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
   return expandOutput;
 }
@@ -750,15 +750,15 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
     int64_t retRows = leftTransform ? ATMatrix.rows : 1;
 
     Value matmulRetValue = extractValue;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
 
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = builder.create<affine::AffineApplyOp>(
+    Value heightOffset = affine::AffineApplyOp::create(builder,
         loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = builder.create<affine::AffineApplyOp>(
+    Value widthOffset = affine::AffineApplyOp::create(builder,
         loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     Value outInitVal =
@@ -775,12 +775,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                          .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                                   elementType)
                          .getResult();
-        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+        init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       }
 
       Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
       // Multiply AT x m.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{AT, matmulRetValue}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -794,19 +794,19 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                          .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                                   elementType)
                          .getResult();
-        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+        init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       }
 
       Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
       // Multiply y = (AT x m) x A.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, A}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
     if (scalarFactor != 1) {
       // Multiply by scalar factor and add outInitVal.
-      Value scalarFactorValue = builder.create<arith::ConstantOp>(
+      Value scalarFactorValue = arith::ConstantOp::create(builder,
           loc, FloatAttr::get(elementType, scalarFactor));
       auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
       auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
@@ -824,11 +824,11 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                       utils::IteratorType::parallel},
                   [&](OpBuilder &nestedBuilder, Location nestedLoc,
                       ValueRange args) {
-                    auto mulf = nestedBuilder.create<arith::MulFOp>(
+                    auto mulf = arith::MulFOp::create(nestedBuilder,
                         nestedLoc, args[0], args[1]);
-                    auto addf = nestedBuilder.create<arith::AddFOp>(
+                    auto addf = arith::AddFOp::create(nestedBuilder,
                         nestedLoc, mulf.getResult(), args[2]);
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc,
+                    linalg::YieldOp::create(nestedBuilder, nestedLoc,
                                                           addf.getResult());
                   })
               .getResult(0);
@@ -847,12 +847,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
   int64_t tilwH = valueShape[2];
   int64_t tileW = valueShape[3];
-  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tilwH);
-  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
-  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueN);
-  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueF);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tilwH);
+  auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
+  auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueN);
+  auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueF);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
       {tileHBound, tileWBound, nUpperBound, fUpperBound},
@@ -867,7 +867,7 @@ static Value padToAlignedTensor(RewriterBase &rewriter, Location loc,
   auto valueType = cast<ShapedType>(value.getType());
   Type elementType = valueType.getElementType();
   auto alignedType = RankedTensorType::get(alignedShape, elementType);
-  Value padValue = rewriter.create<arith::ConstantOp>(
+  Value padValue = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
 
   return linalg::makeComposedPadHighOp(rewriter, loc, alignedType, value,
@@ -887,7 +887,7 @@ static Value extractFromAlignedTensor(RewriterBase &rewriter, Location loc,
   SmallVector<OpFoldResult> sizes =
       getAsOpFoldResult(rewriter.getI64ArrayAttr(extractedShape));
 
-  return rewriter.create<tensor::ExtractSliceOp>(loc, extractedType, value,
+  return tensor::ExtractSliceOp::create(rewriter, loc, extractedType, value,
                                                  offsets, sizes, strides);
 }
 
@@ -979,9 +979,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   int64_t tileW = llvm::divideCeilSigned(outputW, widthM);
   auto retType = RankedTensorType::get({alphaH, alphaW, filterC, filterF},
                                        filterElementType);
-  Value retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
+  Value retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
                                                     filterElementType);
-  auto transformedFilter = rewriter.create<linalg::WinogradFilterTransformOp>(
+  auto transformedFilter = linalg::WinogradFilterTransformOp::create(rewriter,
       loc, retType, filter, retValue, fmr);
 
   // --- Create operation for input transform ---
@@ -998,9 +998,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
 
   retType = RankedTensorType::get(
       {alphaH, alphaW, tileH, tileW, inputN, inputC}, inputElementType);
-  retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
+  retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
                                               inputElementType);
-  auto transformedInput = rewriter.create<linalg::WinogradInputTransformOp>(
+  auto transformedInput = linalg::WinogradInputTransformOp::create(rewriter,
       loc, retType, input, retValue, fmr);
 
   Type outputElementType = outputType.getElementType();
@@ -1023,7 +1023,7 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
     outputType = alignedOutputType;
   }
 
-  Value transformedOutput = rewriter.create<linalg::WinogradOutputTransformOp>(
+  Value transformedOutput = linalg::WinogradOutputTransformOp::create(rewriter,
       loc, outputType, matmulRet, output, fmr);
 
   // When output size is not aligned with output tile size, extract the
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index e89e80b07d20d..d964aa5cb4e81 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -320,14 +320,14 @@ GenericOp makeMemRefCopyOp(OpBuilder &b, Location loc, Value from, Value to) {
       AffineMap::getMultiDimIdentityMap(memrefTypeTo.getRank(), b.getContext());
   SmallVector<utils::IteratorType> iteratorTypes(memrefTypeTo.getRank(),
                                                  utils::IteratorType::parallel);
-  return b.create<linalg::GenericOp>(
+  return linalg::GenericOp::create(b,
       loc,
       /*inputs=*/from,
       /*outputs=*/to,
       /*indexingMaps=*/llvm::ArrayRef({id, id}),
       /*iteratorTypes=*/iteratorTypes,
       [](OpBuilder &b, Location loc, ValueRange args) {
-        b.create<linalg::YieldOp>(loc, args.front());
+        linalg::YieldOp::create(b, loc, args.front());
       });
 }
 
@@ -483,7 +483,7 @@ static void generateParallelLoopNest(
   case DistributionMethod::None: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    b.create<scf::ParallelOp>(
+    scf::ParallelOp::create(b,
         loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
@@ -499,7 +499,7 @@ static void generateParallelLoopNest(
   case DistributionMethod::Cyclic: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    b.create<scf::ParallelOp>(
+    scf::ParallelOp::create(b,
         loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
@@ -519,13 +519,13 @@ static void generateParallelLoopNest(
     for (unsigned i = 1; i < numProcessed; ++i)
       cond = ab._and(cond, ab.slt(lbs[i], ubs[i]));
     ivStorage.append(lbs.begin(), std::next(lbs.begin(), numProcessed));
-    b.create<scf::IfOp>(loc, cond, [&](OpBuilder &b, Location loc) {
+    scf::IfOp::create(b, loc, cond, [&](OpBuilder &b, Location loc) {
       generateParallelLoopNest(b, loc, lbs.drop_front(numProcessed),
                                ubs.drop_front(numProcessed),
                                steps.drop_front(numProcessed),
                                iteratorTypes.drop_front(numProcessed),
                                remainderProcInfo, bodyBuilderFn, ivStorage);
-      b.create<scf::YieldOp>(loc, ValueRange{});
+      scf::YieldOp::create(b, loc, ValueRange{});
     });
     return;
   }
@@ -595,12 +595,12 @@ static Operation *materializeTiledShape(OpBuilder &builder, Location loc,
   auto shapedType = dyn_cast<ShapedType>(valueToTile.getType());
   auto *sliceOp = TypeSwitch<ShapedType, Operation *>(shapedType)
                       .Case([&](MemRefType) {
-                        return builder.create<memref::SubViewOp>(
+                        return memref::SubViewOp::create(builder,
                             loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
                       .Case([&](RankedTensorType) {
-                        return builder.create<tensor::ExtractSliceOp>(
+                        return tensor::ExtractSliceOp::create(builder,
                             loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
@@ -793,7 +793,7 @@ SmallVector<Value> insertSlicesBack(OpBuilder &builder, Location loc,
     // `tiledOperands`.
     Value outputTensor = operands[opOperand.getOperandNumber()];
     if (auto sliceOp = outputTensor.getDefiningOp<tensor::ExtractSliceOp>()) {
-      Value inserted = builder.create<tensor::InsertSliceOp>(
+      Value inserted = tensor::InsertSliceOp::create(builder,
           loc, sliceOp.getSource().getType(), results[resultIdx],
           sliceOp.getSource(), sliceOp.getOffsets(), sliceOp.getSizes(),
           sliceOp.getStrides(), sliceOp.getStaticOffsets(),
diff --git a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
index ff6af63eee531..9dadd9bea2f67 100644
--- a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -135,7 +135,7 @@ struct GlobalStoreOpInterface
     auto memrefType = getMemRefTypeWithStaticIdentityLayout(tensorType);
 
     auto loc = globalStoreOp.getLoc();
-    auto targetMemref = rewriter.create<memref::GetGlobalOp>(
+    auto targetMemref = memref::GetGlobalOp::create(rewriter,
         loc, memrefType, globalStoreOp.getGlobalAttr().getLeafReference());
 
     auto sourceMemref =
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index 7940ff60a48e7..749cca0575581 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -60,7 +60,7 @@ struct FoldRank final : public mlir::OpRewritePattern<mlir::mpi::CommRankOp> {
     if (!isa<IntegerAttr>(dltiAttr.value()))
       return op->emitError()
              << "Expected an integer attribute for MPI:comm_world_rank";
-    Value res = b.create<arith::ConstantIndexOp>(
+    Value res = arith::ConstantIndexOp::create(b,
         op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
     if (Value retVal = op.getRetval())
       b.replaceOp(op, {retVal, res});
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index 26441a9d78658..a21631cbf8510 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -746,7 +746,7 @@ Operation *math::MathDialect::materializeConstant(OpBuilder &builder,
                                                   Attribute value, Type type,
                                                   Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
index 13e2a4b5541b2..770f32841c0f6 100644
--- a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
@@ -65,7 +65,7 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
   // Maybe broadcasts scalar value into vector type compatible with `op`.
   auto bcast = [&](Value value) -> Value {
     if (auto vec = dyn_cast<VectorType>(op.getType()))
-      return rewriter.create<vector::BroadcastOp>(op.getLoc(), vec, value);
+      return vector::BroadcastOp::create(rewriter, op.getLoc(), vec, value);
     return value;
   };
 
@@ -84,14 +84,14 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
   // Replace `pow(x, 3.0)` with `x * x * x`.
   if (isExponentValue(3.0)) {
     Value square =
-        rewriter.create<arith::MulFOp>(op.getLoc(), ValueRange({x, x}));
+        arith::MulFOp::create(rewriter, op.getLoc(), ValueRange({x, x}));
     rewriter.replaceOpWithNewOp<arith::MulFOp>(op, ValueRange({x, square}));
     return success();
   }
 
   // Replace `pow(x, -1.0)` with `1.0 / x`.
   if (isExponentValue(-1.0)) {
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getFloatAttr(getElementTypeOrSelf(op.getType()), 1.0));
     rewriter.replaceOpWithNewOp<arith::DivFOp>(op, ValueRange({bcast(one), x}));
     return success();
@@ -111,8 +111,8 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
 
   // Replace `pow(x, 0.75)` with `sqrt(sqrt(x)) * sqrt(x)`.
   if (isExponentValue(0.75)) {
-    Value powHalf = rewriter.create<math::SqrtOp>(op.getLoc(), x);
-    Value powQuarter = rewriter.create<math::SqrtOp>(op.getLoc(), powHalf);
+    Value powHalf = math::SqrtOp::create(rewriter, op.getLoc(), x);
+    Value powQuarter = math::SqrtOp::create(rewriter, op.getLoc(), powHalf);
     rewriter.replaceOpWithNewOp<arith::MulFOp>(op,
                                                ValueRange{powHalf, powQuarter});
     return success();
@@ -168,17 +168,17 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   // Maybe broadcasts scalar value into vector type compatible with `op`.
   auto bcast = [&loc, &op, &rewriter](Value value) -> Value {
     if (auto vec = dyn_cast<VectorType>(op.getType()))
-      return rewriter.create<vector::BroadcastOp>(loc, vec, value);
+      return vector::BroadcastOp::create(rewriter, loc, vec, value);
     return value;
   };
 
   Value one;
   Type opType = getElementTypeOrSelf(op.getType());
   if constexpr (std::is_same_v<PowIOpTy, math::FPowIOp>)
-    one = rewriter.create<arith::ConstantOp>(
+    one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getFloatAttr(opType, 1.0));
   else
-    one = rewriter.create<arith::ConstantOp>(
+    one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(opType, 1));
 
   // Replace `[fi]powi(x, 0)` with `1`.
@@ -208,12 +208,12 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   //     with:
   //       (1 / x) * (1 / x) * (1 / x) * ...
   for (unsigned i = 1; i < exponentValue; ++i)
-    result = rewriter.create<MulOpTy>(loc, result, base);
+    result = MulOpTy::create(rewriter, loc, result, base);
 
   // Inverse the base for negative exponent, i.e. for
   // `[fi]powi(x, negative_exponent)` set `x` to `1 / x`.
   if (exponentIsNegative)
-    result = rewriter.create<DivOpTy>(loc, bcast(one), result);
+    result = DivOpTy::create(rewriter, loc, bcast(one), result);
 
   rewriter.replaceOp(op, result);
   return success();
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index 7b5350ca26b60..419ba0d2e500b 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -33,11 +33,11 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                 APFloat::rmNearestTiesToEven, &losesInfo);
   auto attr = b.getFloatAttr(eltType, value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return b.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(b, loc,
                                        DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return b.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(b, loc, attr);
 }
 
 static Value createFloatConst(Location loc, Type type, double value,
@@ -50,11 +50,11 @@ static Value createIntConst(Location loc, Type type, int64_t value,
                             OpBuilder &b) {
   auto attr = b.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return b.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(b, loc,
                                        DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return b.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(b, loc, attr);
 }
 
 static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
@@ -62,11 +62,11 @@ static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
   Type i64Ty = b.getI64Type();
   if (auto shapedTy = dyn_cast<ShapedType>(opType))
     i64Ty = shapedTy.clone(i64Ty);
-  Value fixedConvert = b.create<arith::FPToSIOp>(i64Ty, operand);
-  Value fpFixedConvert = b.create<arith::SIToFPOp>(opType, fixedConvert);
+  Value fixedConvert = arith::FPToSIOp::create(b, i64Ty, operand);
+  Value fpFixedConvert = arith::SIToFPOp::create(b, opType, fixedConvert);
   // The truncation does not preserve the sign when the truncated
   // value is -0. So here the sign is copied again.
-  return b.create<math::CopySignOp>(fpFixedConvert, operand);
+  return math::CopySignOp::create(b, fpFixedConvert, operand);
 }
 
 // sinhf(float x) -> (exp(x) - exp(-x)) / 2
@@ -75,12 +75,12 @@ static LogicalResult convertSinhOp(math::SinhOp op, PatternRewriter &rewriter) {
   Value operand = op.getOperand();
   Type opType = operand.getType();
 
-  Value exp = b.create<math::ExpOp>(operand);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value nexp = b.create<math::ExpOp>(neg);
-  Value sub = b.create<arith::SubFOp>(exp, nexp);
+  Value exp = math::ExpOp::create(b, operand);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value nexp = math::ExpOp::create(b, neg);
+  Value sub = arith::SubFOp::create(b, exp, nexp);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(sub, half);
+  Value res = arith::MulFOp::create(b, sub, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -91,12 +91,12 @@ static LogicalResult convertCoshOp(math::CoshOp op, PatternRewriter &rewriter) {
   Value operand = op.getOperand();
   Type opType = operand.getType();
 
-  Value exp = b.create<math::ExpOp>(operand);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value nexp = b.create<math::ExpOp>(neg);
-  Value add = b.create<arith::AddFOp>(exp, nexp);
+  Value exp = math::ExpOp::create(b, operand);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value nexp = math::ExpOp::create(b, neg);
+  Value add = arith::AddFOp::create(b, exp, nexp);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(add, half);
+  Value res = arith::MulFOp::create(b, add, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -117,23 +117,23 @@ static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) {
   Value negTwo = createFloatConst(loc, floatType, -2.0, rewriter);
 
   // Compute sign(x) = cast<float_type>(x < 0) * (-2) + 1
-  Value isNegative = rewriter.create<arith::CmpFOp>(
+  Value isNegative = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OLT, op.getOperand(), zero);
   Value isNegativeFloat =
-      rewriter.create<arith::UIToFPOp>(loc, floatType, isNegative);
+      arith::UIToFPOp::create(rewriter, loc, floatType, isNegative);
   Value isNegativeTimesNegTwo =
-      rewriter.create<arith::MulFOp>(loc, isNegativeFloat, negTwo);
-  Value sign = rewriter.create<arith::AddFOp>(loc, isNegativeTimesNegTwo, one);
+      arith::MulFOp::create(rewriter, loc, isNegativeFloat, negTwo);
+  Value sign = arith::AddFOp::create(rewriter, loc, isNegativeTimesNegTwo, one);
 
   // Normalize input to positive value: y = sign(x) * x
-  Value positiveX = rewriter.create<arith::MulFOp>(loc, sign, op.getOperand());
+  Value positiveX = arith::MulFOp::create(rewriter, loc, sign, op.getOperand());
 
   // Decompose on normalized input
-  Value negDoubledX = rewriter.create<arith::MulFOp>(loc, negTwo, positiveX);
-  Value exp2x = rewriter.create<math::ExpOp>(loc, negDoubledX);
-  Value dividend = rewriter.create<arith::SubFOp>(loc, one, exp2x);
-  Value divisor = rewriter.create<arith::AddFOp>(loc, one, exp2x);
-  Value positiveRes = rewriter.create<arith::DivFOp>(loc, dividend, divisor);
+  Value negDoubledX = arith::MulFOp::create(rewriter, loc, negTwo, positiveX);
+  Value exp2x = math::ExpOp::create(rewriter, loc, negDoubledX);
+  Value dividend = arith::SubFOp::create(rewriter, loc, one, exp2x);
+  Value divisor = arith::AddFOp::create(rewriter, loc, one, exp2x);
+  Value positiveRes = arith::DivFOp::create(rewriter, loc, dividend, divisor);
 
   // Multiply result by sign(x) to retain signs from negative inputs
   rewriter.replaceOpWithNewOp<arith::MulFOp>(op, sign, positiveRes);
@@ -146,9 +146,9 @@ static LogicalResult convertTanOp(math::TanOp op, PatternRewriter &rewriter) {
   ImplicitLocOpBuilder b(op->getLoc(), rewriter);
   Value operand = op.getOperand();
   Type type = operand.getType();
-  Value sin = b.create<math::SinOp>(type, operand);
-  Value cos = b.create<math::CosOp>(type, operand);
-  Value div = b.create<arith::DivFOp>(type, sin, cos);
+  Value sin = math::SinOp::create(b, type, operand);
+  Value cos = math::CosOp::create(b, type, operand);
+  Value div = arith::DivFOp::create(b, type, sin, cos);
   rewriter.replaceOp(op, div);
   return success();
 }
@@ -161,10 +161,10 @@ static LogicalResult convertAsinhOp(math::AsinhOp op,
   Type opType = operand.getType();
 
   Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value fma = b.create<math::FmaOp>(operand, operand, one);
-  Value sqrt = b.create<math::SqrtOp>(fma);
-  Value add = b.create<arith::AddFOp>(operand, sqrt);
-  Value res = b.create<math::LogOp>(add);
+  Value fma = math::FmaOp::create(b, operand, operand, one);
+  Value sqrt = math::SqrtOp::create(b, fma);
+  Value add = arith::AddFOp::create(b, operand, sqrt);
+  Value res = math::LogOp::create(b, add);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -177,10 +177,10 @@ static LogicalResult convertAcoshOp(math::AcoshOp op,
   Type opType = operand.getType();
 
   Value negOne = createFloatConst(op->getLoc(), opType, -1.0, rewriter);
-  Value fma = b.create<math::FmaOp>(operand, operand, negOne);
-  Value sqrt = b.create<math::SqrtOp>(fma);
-  Value add = b.create<arith::AddFOp>(operand, sqrt);
-  Value res = b.create<math::LogOp>(add);
+  Value fma = math::FmaOp::create(b, operand, operand, negOne);
+  Value sqrt = math::SqrtOp::create(b, fma);
+  Value add = arith::AddFOp::create(b, operand, sqrt);
+  Value res = math::LogOp::create(b, add);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -193,13 +193,13 @@ static LogicalResult convertAtanhOp(math::AtanhOp op,
   Type opType = operand.getType();
 
   Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value add = b.create<arith::AddFOp>(operand, one);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value sub = b.create<arith::AddFOp>(neg, one);
-  Value div = b.create<arith::DivFOp>(add, sub);
-  Value log = b.create<math::LogOp>(div);
+  Value add = arith::AddFOp::create(b, operand, one);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value sub = arith::AddFOp::create(b, neg, one);
+  Value div = arith::DivFOp::create(b, add, sub);
+  Value log = math::LogOp::create(b, div);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(log, half);
+  Value res = arith::MulFOp::create(b, log, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -210,8 +210,8 @@ static LogicalResult convertFmaFOp(math::FmaOp op, PatternRewriter &rewriter) {
   Value operandB = op.getOperand(1);
   Value operandC = op.getOperand(2);
   Type type = op.getType();
-  Value mult = b.create<arith::MulFOp>(type, operandA, operandB);
-  Value add = b.create<arith::AddFOp>(type, mult, operandC);
+  Value mult = arith::MulFOp::create(b, type, operandA, operandB);
+  Value add = arith::AddFOp::create(b, type, mult, operandC);
   rewriter.replaceOp(op, add);
   return success();
 }
@@ -236,11 +236,11 @@ static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) {
   Value zero = createFloatConst(op->getLoc(), opType, 0.00, rewriter);
   Value one = createFloatConst(op->getLoc(), opType, 1.00, rewriter);
 
-  Value gtCheck = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand,
+  Value gtCheck = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT, operand,
                                           fpFixedConvert);
-  Value incrValue = b.create<arith::SelectOp>(op->getLoc(), gtCheck, one, zero);
+  Value incrValue = arith::SelectOp::create(b, op->getLoc(), gtCheck, one, zero);
 
-  Value ret = b.create<arith::AddFOp>(opType, fpFixedConvert, incrValue);
+  Value ret = arith::AddFOp::create(b, opType, fpFixedConvert, incrValue);
   rewriter.replaceOp(op, ret);
   return success();
 }
@@ -258,8 +258,8 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
 
   auto convertFPowItoPowf = [&]() -> LogicalResult {
     Value castPowerToFp =
-        rewriter.create<arith::SIToFPOp>(op.getLoc(), baseType, power);
-    Value res = rewriter.create<math::PowFOp>(op.getLoc(), baseType, base,
+        arith::SIToFPOp::create(rewriter, op.getLoc(), baseType, power);
+    Value res = math::PowFOp::create(rewriter, op.getLoc(), baseType, base,
                                               castPowerToFp);
     rewriter.replaceOp(op, res);
     return success();
@@ -281,9 +281,9 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
 
   while (absPower > 0) {
     if (absPower & 1)
-      res = b.create<arith::MulFOp>(baseType, base, res);
+      res = arith::MulFOp::create(b, baseType, base, res);
     absPower >>= 1;
-    base = b.create<arith::MulFOp>(baseType, base, base);
+    base = arith::MulFOp::create(b, baseType, base, base);
   }
 
   // Make sure not to introduce UB in case of negative power.
@@ -303,13 +303,13 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
         createFloatConst(op->getLoc(), baseType,
                          APFloat::getInf(sem, /*Negative=*/true), rewriter);
     Value zeroEqCheck =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, res, zero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, res, zero);
     Value negZeroEqCheck =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, res, negZero);
-    res = b.create<arith::DivFOp>(baseType, one, res);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, res, negZero);
+    res = arith::DivFOp::create(b, baseType, one, res);
     res =
-        b.create<arith::SelectOp>(op->getLoc(), zeroEqCheck, posInfinity, res);
-    res = b.create<arith::SelectOp>(op->getLoc(), negZeroEqCheck, negInfinity,
+        arith::SelectOp::create(b, op->getLoc(), zeroEqCheck, posInfinity, res);
+    res = arith::SelectOp::create(b, op->getLoc(), negZeroEqCheck, negInfinity,
                                     res);
   }
 
@@ -331,7 +331,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       cast<mlir::FloatType>(getElementTypeOrSelf(typeB)).getFloatSemantics();
   APFloat valueB(sem);
   auto mulf = [&](Value x, Value y) -> Value {
-    return b.create<arith::MulFOp>(x, y);
+    return arith::MulFOp::create(b, x, y);
   };
   if (matchPattern(operandB, m_ConstantFloat(&valueB))) {
     if (valueB.isZero()) {
@@ -348,19 +348,19 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
     if (valueB.isExactlyValue(-1.0)) {
       // a^(-1) -> 1 / a
       Value one = createFloatConst(op->getLoc(), typeA, 1.0, rewriter);
-      Value div = b.create<arith::DivFOp>(one, operandA);
+      Value div = arith::DivFOp::create(b, one, operandA);
       rewriter.replaceOp(op, div);
       return success();
     }
     if (valueB.isExactlyValue(0.5)) {
       // a^(1/2) -> sqrt(a)
-      Value sqrt = b.create<math::SqrtOp>(operandA);
+      Value sqrt = math::SqrtOp::create(b, operandA);
       rewriter.replaceOp(op, sqrt);
       return success();
     }
     if (valueB.isExactlyValue(-0.5)) {
       // a^(-1/2) -> 1 / sqrt(a)
-      Value rsqrt = b.create<math::RsqrtOp>(operandA);
+      Value rsqrt = math::RsqrtOp::create(b, operandA);
       rewriter.replaceOp(op, rsqrt);
       return success();
     }
@@ -373,7 +373,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       // a^(-2) -> 1 / (a * a)
       Value one =
           createFloatConst(op->getLoc(), operandA.getType(), 1.0, rewriter);
-      Value div = b.create<arith::DivFOp>(one, mulf(operandA, operandA));
+      Value div = arith::DivFOp::create(b, one, mulf(operandA, operandA));
       rewriter.replaceOp(op, div);
       return success();
     }
@@ -383,9 +383,9 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
     }
   }
 
-  Value logA = b.create<math::LogOp>(operandA);
-  Value mult = b.create<arith::MulFOp>(operandB, logA);
-  Value expResult = b.create<math::ExpOp>(mult);
+  Value logA = math::LogOp::create(b, operandA);
+  Value mult = arith::MulFOp::create(b, operandB, logA);
+  Value expResult = math::ExpOp::create(b, mult);
   rewriter.replaceOp(op, expResult);
   return success();
 }
@@ -400,8 +400,8 @@ static LogicalResult convertExp2fOp(math::Exp2Op op,
   Value operand = op.getOperand();
   Type opType = operand.getType();
   Value ln2 = createFloatConst(op->getLoc(), opType, llvm::numbers::ln2, b);
-  Value mult = b.create<arith::MulFOp>(opType, operand, ln2);
-  Value exp = b.create<math::ExpOp>(op->getLoc(), mult);
+  Value mult = arith::MulFOp::create(b, opType, operand, ln2);
+  Value exp = math::ExpOp::create(b, op->getLoc(), mult);
   rewriter.replaceOp(op, exp);
   return success();
 }
@@ -427,8 +427,8 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   Value c127 = createIntConst(loc, i32Ty, 127, b);
   Value expMask = createIntConst(loc, i32Ty, (1 << 8) - 1, b);
 
-  Value incrValue = b.create<math::CopySignOp>(half, operand);
-  Value add = b.create<arith::AddFOp>(opType, operand, incrValue);
+  Value incrValue = math::CopySignOp::create(b, half, operand);
+  Value add = arith::AddFOp::create(b, opType, operand, incrValue);
   Value fpFixedConvert = createTruncatedFPValue(add, b);
 
   // There are three cases where adding 0.5 to the value and truncating by
@@ -451,14 +451,14 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   //     i64 leading to wrong outputs.
   //
   // All three cases satisfy the property `biasedExp >= 23`.
-  Value operandBitcast = b.create<arith::BitcastOp>(i32Ty, operand);
-  Value operandExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(operandBitcast, c23), expMask);
-  Value operandBiasedExp = b.create<arith::SubIOp>(operandExp, c127);
+  Value operandBitcast = arith::BitcastOp::create(b, i32Ty, operand);
+  Value operandExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
   Value isSpecialValOrLargeVal =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::sge, operandBiasedExp, c23);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::sge, operandBiasedExp, c23);
 
-  Value result = b.create<arith::SelectOp>(isSpecialValOrLargeVal, operand,
+  Value result = arith::SelectOp::create(b, isSpecialValOrLargeVal, operand,
                                            fpFixedConvert);
   rewriter.replaceOp(op, result);
   return success();
@@ -490,20 +490,20 @@ static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
     auto mask = createIntConst(loc, operandTy, allbits >> half, rewriter);
 
     Value pred =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule, x, mask);
-    Value add = rewriter.create<arith::AddIOp>(loc, count, bits);
-    Value shift = rewriter.create<arith::ShLIOp>(loc, x, bits);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ule, x, mask);
+    Value add = arith::AddIOp::create(rewriter, loc, count, bits);
+    Value shift = arith::ShLIOp::create(rewriter, loc, x, bits);
 
-    x = rewriter.create<arith::SelectOp>(loc, pred, shift, x);
-    count = rewriter.create<arith::SelectOp>(loc, pred, add, count);
+    x = arith::SelectOp::create(rewriter, loc, pred, shift, x);
+    count = arith::SelectOp::create(rewriter, loc, pred, add, count);
   }
 
   Value zero = createIntConst(loc, operandTy, 0, rewriter);
-  Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+  Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                               operand, zero);
 
   Value bwval = createIntConst(loc, operandTy, bitwidth, rewriter);
-  Value sel = rewriter.create<arith::SelectOp>(loc, pred, bwval, count);
+  Value sel = arith::SelectOp::create(rewriter, loc, pred, bwval, count);
   rewriter.replaceOp(op, sel);
   return success();
 }
@@ -550,29 +550,29 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   Value c23Mask = createIntConst(loc, iTy, (1ull << mantissaWidth) - 1, b);
   Value expMask = createIntConst(loc, iTy, (1ull << exponentWidth) - 1, b);
 
-  Value operandBitcast = b.create<arith::BitcastOp>(iTy, operand);
-  Value round = b.create<math::RoundOp>(operand);
-  Value roundBitcast = b.create<arith::BitcastOp>(iTy, round);
+  Value operandBitcast = arith::BitcastOp::create(b, iTy, operand);
+  Value round = math::RoundOp::create(b, operand);
+  Value roundBitcast = arith::BitcastOp::create(b, iTy, round);
 
   // Get biased exponents for operand and round(operand)
-  Value operandExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(operandBitcast, c23), expMask);
-  Value operandBiasedExp = b.create<arith::SubIOp>(operandExp, c127);
-  Value roundExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(roundBitcast, c23), expMask);
-  Value roundBiasedExp = b.create<arith::SubIOp>(roundExp, c127);
+  Value operandExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
+  Value roundExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, roundBitcast, c23), expMask);
+  Value roundBiasedExp = arith::SubIOp::create(b, roundExp, c127);
 
   auto safeShiftRight = [&](Value x, Value shift) -> Value {
     // Clamp shift to valid range [0, bitwidth - 1] to avoid undefined behavior
-    Value clampedShift = b.create<arith::MaxSIOp>(shift, c0);
-    clampedShift = b.create<arith::MinSIOp>(clampedShift, c31);
-    return b.create<arith::ShRUIOp>(x, clampedShift);
+    Value clampedShift = arith::MaxSIOp::create(b, shift, c0);
+    clampedShift = arith::MinSIOp::create(b, clampedShift, c31);
+    return arith::ShRUIOp::create(b, x, clampedShift);
   };
 
   auto maskMantissa = [&](Value mantissa,
                           Value mantissaMaskRightShift) -> Value {
     Value shiftedMantissaMask = safeShiftRight(c23Mask, mantissaMaskRightShift);
-    return b.create<arith::AndIOp>(mantissa, shiftedMantissaMask);
+    return arith::AndIOp::create(b, mantissa, shiftedMantissaMask);
   };
 
   // A whole number `x`, such that `|x| != 1`, is even if the mantissa, ignoring
@@ -590,13 +590,13 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // `biasedExp > 23`, so they get treated as large numbers with no room for
   // decimals, which are always even.
   Value roundBiasedExpEq0 =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, roundBiasedExp, c0);
-  Value roundBiasedExpMinus1 = b.create<arith::SubIOp>(roundBiasedExp, c1);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, roundBiasedExp, c0);
+  Value roundBiasedExpMinus1 = arith::SubIOp::create(b, roundBiasedExp, c1);
   Value roundMaskedMantissa = maskMantissa(roundBitcast, roundBiasedExpMinus1);
-  Value roundIsNotEvenOrSpecialVal = b.create<arith::CmpIOp>(
+  Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(b,
       arith::CmpIPredicate::ne, roundMaskedMantissa, c0);
   roundIsNotEvenOrSpecialVal =
-      b.create<arith::OrIOp>(roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
+      arith::OrIOp::create(b, roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
 
   // A value `x` with `0 <= biasedExp < 23`, is halfway between two consecutive
   // integers if the bit at index `biasedExp` starting from the left in the
@@ -605,37 +605,37 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // values +-0.5 are the only halfway values that have `biasedExp == -1 < 0`,
   // so these are handled separately. In particular, if `biasedExp == -1`, the
   // value is halfway if the entire mantissa is zero.
-  Value operandBiasedExpEqNeg1 = b.create<arith::CmpIOp>(
+  Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(b,
       arith::CmpIPredicate::eq, operandBiasedExp, cNeg1);
-  Value expectedOperandMaskedMantissa = b.create<arith::SelectOp>(
+  Value expectedOperandMaskedMantissa = arith::SelectOp::create(b,
       operandBiasedExpEqNeg1, c0, safeShiftRight(c2To22, operandBiasedExp));
   Value operandMaskedMantissa = maskMantissa(operandBitcast, operandBiasedExp);
   Value operandIsHalfway =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, operandMaskedMantissa,
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, operandMaskedMantissa,
                               expectedOperandMaskedMantissa);
   // Ensure `biasedExp` is in the valid range for half values.
-  Value operandBiasedExpGeNeg1 = b.create<arith::CmpIOp>(
+  Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(b,
       arith::CmpIPredicate::sge, operandBiasedExp, cNeg1);
   Value operandBiasedExpLt23 =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::slt, operandBiasedExp, c23);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, operandBiasedExp, c23);
   operandIsHalfway =
-      b.create<arith::AndIOp>(operandIsHalfway, operandBiasedExpLt23);
+      arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpLt23);
   operandIsHalfway =
-      b.create<arith::AndIOp>(operandIsHalfway, operandBiasedExpGeNeg1);
+      arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpGeNeg1);
 
   // Adjust rounded operand with `round(operand) - sign(operand)` to correct the
   // case where `round` rounded in the opposite direction of `roundeven`.
-  Value sign = b.create<math::CopySignOp>(c1Float, operand);
-  Value roundShifted = b.create<arith::SubFOp>(round, sign);
+  Value sign = math::CopySignOp::create(b, c1Float, operand);
+  Value roundShifted = arith::SubFOp::create(b, round, sign);
   // If the rounded value is even or a special value, we default to the behavior
   // of `math.round`.
   Value needsShift =
-      b.create<arith::AndIOp>(roundIsNotEvenOrSpecialVal, operandIsHalfway);
-  Value result = b.create<arith::SelectOp>(needsShift, roundShifted, round);
+      arith::AndIOp::create(b, roundIsNotEvenOrSpecialVal, operandIsHalfway);
+  Value result = arith::SelectOp::create(b, needsShift, roundShifted, round);
   // The `x - sign` adjustment does not preserve the sign when we are adjusting
   // the value -1 to -0. So here the sign is copied again to ensure that -0.5 is
   // rounded to -0.0.
-  result = b.create<math::CopySignOp>(result, operand);
+  result = math::CopySignOp::create(b, result, operand);
   rewriter.replaceOp(op, result);
   return success();
 }
@@ -657,7 +657,7 @@ static LogicalResult convertRsqrtOp(math::RsqrtOp op,
 
   Location loc = op->getLoc();
   auto constOneFloat = createFloatConst(loc, operandTy, 1.0, rewriter);
-  auto sqrtOp = rewriter.create<math::SqrtOp>(loc, operand);
+  auto sqrtOp = math::SqrtOp::create(rewriter, loc, operand);
   rewriter.replaceOpWithNewOp<arith::DivFOp>(op, constOneFloat, sqrtOp);
   return success();
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
index a570ed5118ef0..9d6ad613fc945 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
@@ -73,7 +73,7 @@ void mlir::math::populateExtendToSupportedTypesTypeConverter(
       });
   typeConverter.addTargetMaterialization(
       [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        auto extFOp = arith::ExtFOp::create(b, loc, target, input);
         extFOp.setFastmath(arith::FastMathFlags::contract);
         return extFOp;
       });
@@ -104,7 +104,7 @@ LogicalResult ExtendToSupportedTypesRewritePattern::matchAndRewrite(
   for (auto [result, newType, origType] : llvm::zip_equal(
            results, (*legalized)->getResultTypes(), op->getResultTypes())) {
     if (newType != origType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, origType, result);
+      auto truncFOp = arith::TruncFOp::create(rewriter, loc, origType, result);
       truncFOp.setFastmath(arith::FastMathFlags::contract);
       result = truncFOp.getResult();
     }
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index a26e380232a91..2e27aba30575b 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -73,7 +73,7 @@ static Value broadcast(ImplicitLocOpBuilder &builder, Value value,
                        std::optional<VectorShape> shape) {
   assert(!isa<VectorType>(value.getType()) && "must be scalar value");
   auto type = broadcast(value.getType(), shape);
-  return shape ? builder.create<BroadcastOp>(type, value) : value;
+  return shape ? BroadcastOp::create(builder, type, value) : value;
 }
 
 //----------------------------------------------------------------------------//
@@ -131,7 +131,7 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
       auto eltType = cast<VectorType>(operand.getType()).getElementType();
       auto expandedType = VectorType::get(expandedShape, eltType);
       expandedOperands[i] =
-          builder.create<vector::ShapeCastOp>(expandedType, operand);
+          vector::ShapeCastOp::create(builder, expandedType, operand);
     }
   }
 
@@ -149,7 +149,7 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
     SmallVector<Value> extracted(expandedOperands.size());
     for (const auto &tuple : llvm::enumerate(expandedOperands))
       extracted[tuple.index()] =
-          builder.create<vector::ExtractOp>(tuple.value(), offsets);
+          vector::ExtractOp::create(builder, tuple.value(), offsets);
 
     results[i] = compute(extracted);
   }
@@ -157,15 +157,15 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
   // Stitch results together into one large vector.
   Type resultEltType = cast<VectorType>(results[0].getType()).getElementType();
   Type resultExpandedType = VectorType::get(expandedShape, resultEltType);
-  Value result = builder.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(builder,
       resultExpandedType, builder.getZeroAttr(resultExpandedType));
 
   for (int64_t i = 0; i < maxIndex; ++i)
-    result = builder.create<vector::InsertOp>(results[i], result,
+    result = vector::InsertOp::create(builder, results[i], result,
                                               delinearize(i, strides));
 
   // Reshape back to the original vector shape.
-  return builder.create<vector::ShapeCastOp>(
+  return vector::ShapeCastOp::create(builder,
       VectorType::get(inputShape, resultEltType), result);
 }
 
@@ -174,28 +174,28 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
 //----------------------------------------------------------------------------//
 
 static Value boolCst(ImplicitLocOpBuilder &builder, bool value) {
-  return builder.create<arith::ConstantOp>(builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, builder.getBoolAttr(value));
 }
 
 static Value floatCst(ImplicitLocOpBuilder &builder, float value,
                       Type elementType) {
   assert((elementType.isF16() || elementType.isF32()) &&
          "x must be f16 or f32 type.");
-  return builder.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(builder,
       builder.getFloatAttr(elementType, value));
 }
 
 static Value f32Cst(ImplicitLocOpBuilder &builder, double value) {
-  return builder.create<arith::ConstantOp>(builder.getF32FloatAttr(value));
+  return arith::ConstantOp::create(builder, builder.getF32FloatAttr(value));
 }
 
 static Value i32Cst(ImplicitLocOpBuilder &builder, int32_t value) {
-  return builder.create<arith::ConstantOp>(builder.getI32IntegerAttr(value));
+  return arith::ConstantOp::create(builder, builder.getI32IntegerAttr(value));
 }
 
 static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
   Value i32Value = i32Cst(builder, static_cast<int32_t>(bits));
-  return builder.create<arith::BitcastOp>(builder.getF32Type(), i32Value);
+  return arith::BitcastOp::create(builder, builder.getF32Type(), i32Value);
 }
 
 //----------------------------------------------------------------------------//
@@ -204,15 +204,15 @@ static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
 
 // Return the minimum of the two values or NaN if value is NaN
 static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return builder.create<arith::SelectOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT, value, bound),
+  return arith::SelectOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT, value, bound),
       value, bound);
 }
 
 // Return the maximum of the two values or NaN if value is NaN
 static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return builder.create<arith::SelectOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::UGT, value, bound),
+  return arith::SelectOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::UGT, value, bound),
       value, bound);
 }
 
@@ -242,24 +242,24 @@ static std::pair<Value, Value> frexp(ImplicitLocOpBuilder &builder, Value arg,
   Value cstInvMantMask = f32FromBits(builder, ~0x7f800000u);
 
   // Bitcast to i32 for bitwise operations.
-  Value i32Half = builder.create<arith::BitcastOp>(i32, cstHalf);
-  Value i32InvMantMask = builder.create<arith::BitcastOp>(i32, cstInvMantMask);
-  Value i32Arg = builder.create<arith::BitcastOp>(i32Vec, arg);
+  Value i32Half = arith::BitcastOp::create(builder, i32, cstHalf);
+  Value i32InvMantMask = arith::BitcastOp::create(builder, i32, cstInvMantMask);
+  Value i32Arg = arith::BitcastOp::create(builder, i32Vec, arg);
 
   // Compute normalized fraction.
-  Value tmp0 = builder.create<arith::AndIOp>(i32Arg, bcast(i32InvMantMask));
-  Value tmp1 = builder.create<arith::OrIOp>(tmp0, bcast(i32Half));
-  Value normalizedFraction = builder.create<arith::BitcastOp>(f32Vec, tmp1);
+  Value tmp0 = arith::AndIOp::create(builder, i32Arg, bcast(i32InvMantMask));
+  Value tmp1 = arith::OrIOp::create(builder, tmp0, bcast(i32Half));
+  Value normalizedFraction = arith::BitcastOp::create(builder, f32Vec, tmp1);
 
   // Compute exponent.
-  Value arg0 = isPositive ? arg : builder.create<math::AbsFOp>(arg);
-  Value biasedExponentBits = builder.create<arith::ShRUIOp>(
-      builder.create<arith::BitcastOp>(i32Vec, arg0),
+  Value arg0 = isPositive ? arg : math::AbsFOp::create(builder, arg);
+  Value biasedExponentBits = arith::ShRUIOp::create(builder,
+      arith::BitcastOp::create(builder, i32Vec, arg0),
       bcast(i32Cst(builder, 23)));
   Value biasedExponent =
-      builder.create<arith::SIToFPOp>(f32Vec, biasedExponentBits);
+      arith::SIToFPOp::create(builder, f32Vec, biasedExponentBits);
   Value exponent =
-      builder.create<arith::SubFOp>(biasedExponent, bcast(cst126f));
+      arith::SubFOp::create(builder, biasedExponent, bcast(cst126f));
 
   return {normalizedFraction, exponent};
 }
@@ -279,10 +279,10 @@ static Value exp2I32(ImplicitLocOpBuilder &builder, Value arg) {
   // Set the exponent bias to zero.
   auto bias = bcast(i32Cst(builder, 127));
 
-  Value biasedArg = builder.create<arith::AddIOp>(arg, bias);
+  Value biasedArg = arith::AddIOp::create(builder, arg, bias);
   Value exp2ValueInt =
-      builder.create<arith::ShLIOp>(biasedArg, exponetBitLocation);
-  Value exp2ValueF32 = builder.create<arith::BitcastOp>(f32Vec, exp2ValueInt);
+      arith::ShLIOp::create(builder, biasedArg, exponetBitLocation);
+  Value exp2ValueF32 = arith::BitcastOp::create(builder, f32Vec, exp2ValueInt);
 
   return exp2ValueF32;
 }
@@ -301,10 +301,10 @@ Value makePolynomialCalculation(ImplicitLocOpBuilder &builder,
   if (coeffs.size() == 1)
     return coeffs[0];
 
-  Value res = builder.create<math::FmaOp>(x, coeffs[coeffs.size() - 1],
+  Value res = math::FmaOp::create(builder, x, coeffs[coeffs.size() - 1],
                                           coeffs[coeffs.size() - 2]);
   for (auto i = ptrdiff_t(coeffs.size()) - 3; i >= 0; --i) {
-    res = builder.create<math::FmaOp>(x, res, coeffs[i]);
+    res = math::FmaOp::create(builder, x, res, coeffs[i]);
   }
   return res;
 }
@@ -344,9 +344,9 @@ LogicalResult insertCasts(Operation *op, PatternRewriter &rewriter) {
   Location loc = op->getLoc();
   SmallVector<Value> operands;
   for (auto operand : op->getOperands())
-    operands.push_back(rewriter.create<arith::ExtFOp>(loc, newType, operand));
+    operands.push_back(arith::ExtFOp::create(rewriter, loc, newType, operand));
   auto result =
-      rewriter.create<T>(loc, TypeRange{newType}, operands, op->getAttrs());
+      T::create(rewriter, loc, TypeRange{newType}, operands, op->getAttrs());
   rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, origType, result);
   return success();
 }
@@ -394,18 +394,18 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
   std::optional<VectorShape> shape = vectorShape(op.getOperand());
 
   ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
-  Value abs = builder.create<math::AbsFOp>(operand);
+  Value abs = math::AbsFOp::create(builder, operand);
 
   auto one = broadcast(builder, f32Cst(builder, 1.0), shape);
 
   // When 0.66 < x <= 2.41 we do (x-1) / (x+1):
   auto twoThirds = broadcast(builder, f32Cst(builder, 0.66), shape);
   Value cmp2 =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, abs, twoThirds);
-  Value addone = builder.create<arith::AddFOp>(abs, one);
-  Value subone = builder.create<arith::SubFOp>(abs, one);
-  Value xnum = builder.create<arith::SelectOp>(cmp2, subone, abs);
-  Value xden = builder.create<arith::SelectOp>(cmp2, addone, one);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, abs, twoThirds);
+  Value addone = arith::AddFOp::create(builder, abs, one);
+  Value subone = arith::SubFOp::create(builder, abs, one);
+  Value xnum = arith::SelectOp::create(builder, cmp2, subone, abs);
+  Value xden = arith::SelectOp::create(builder, cmp2, addone, one);
 
   auto bcast = [&](Value value) -> Value {
     return broadcast(builder, value, shape);
@@ -414,12 +414,12 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
   // Break into the <= 0.66 or > 2.41 we do x or 1/x:
   auto tan3pio8 = bcast(f32Cst(builder, 2.41421356237309504880));
   Value cmp1 =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, abs, tan3pio8);
-  xnum = builder.create<arith::SelectOp>(cmp1, one, xnum);
-  xden = builder.create<arith::SelectOp>(cmp1, abs, xden);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, abs, tan3pio8);
+  xnum = arith::SelectOp::create(builder, cmp1, one, xnum);
+  xden = arith::SelectOp::create(builder, cmp1, abs, xden);
 
-  Value x = builder.create<arith::DivFOp>(xnum, xden);
-  Value xx = builder.create<arith::MulFOp>(x, x);
+  Value x = arith::DivFOp::create(builder, xnum, xden);
+  Value xx = arith::MulFOp::create(builder, x, x);
 
   // Perform the Taylor series approximation for atan over the range
   // [0.0, 0.66].
@@ -436,31 +436,31 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
 
   // Apply the polynomial approximation for the numerator:
   Value n = p0;
-  n = builder.create<math::FmaOp>(xx, n, p1);
-  n = builder.create<math::FmaOp>(xx, n, p2);
-  n = builder.create<math::FmaOp>(xx, n, p3);
-  n = builder.create<math::FmaOp>(xx, n, p4);
-  n = builder.create<arith::MulFOp>(n, xx);
+  n = math::FmaOp::create(builder, xx, n, p1);
+  n = math::FmaOp::create(builder, xx, n, p2);
+  n = math::FmaOp::create(builder, xx, n, p3);
+  n = math::FmaOp::create(builder, xx, n, p4);
+  n = arith::MulFOp::create(builder, n, xx);
 
   // Apply the polynomial approximation for the denominator:
   Value d = q0;
-  d = builder.create<math::FmaOp>(xx, d, q1);
-  d = builder.create<math::FmaOp>(xx, d, q2);
-  d = builder.create<math::FmaOp>(xx, d, q3);
-  d = builder.create<math::FmaOp>(xx, d, q4);
+  d = math::FmaOp::create(builder, xx, d, q1);
+  d = math::FmaOp::create(builder, xx, d, q2);
+  d = math::FmaOp::create(builder, xx, d, q3);
+  d = math::FmaOp::create(builder, xx, d, q4);
 
   // Compute approximation of theta:
-  Value ans0 = builder.create<arith::DivFOp>(n, d);
-  ans0 = builder.create<math::FmaOp>(ans0, x, x);
+  Value ans0 = arith::DivFOp::create(builder, n, d);
+  ans0 = math::FmaOp::create(builder, ans0, x, x);
 
   // Correct for the input mapping's angles:
   Value mpi4 = bcast(f32Cst(builder, llvm::numbers::pi / 4));
-  Value ans2 = builder.create<arith::AddFOp>(mpi4, ans0);
-  Value ans = builder.create<arith::SelectOp>(cmp2, ans2, ans0);
+  Value ans2 = arith::AddFOp::create(builder, mpi4, ans0);
+  Value ans = arith::SelectOp::create(builder, cmp2, ans2, ans0);
 
   Value mpi2 = bcast(f32Cst(builder, llvm::numbers::pi / 2));
-  Value ans1 = builder.create<arith::SubFOp>(mpi2, ans0);
-  ans = builder.create<arith::SelectOp>(cmp1, ans1, ans);
+  Value ans1 = arith::SubFOp::create(builder, mpi2, ans0);
+  ans = arith::SelectOp::create(builder, cmp1, ans1, ans);
 
   // Correct for signing of the input.
   rewriter.replaceOpWithNewOp<math::CopySignOp>(op, ans, operand);
@@ -493,44 +493,44 @@ Atan2Approximation::matchAndRewrite(math::Atan2Op op,
   std::optional<VectorShape> shape = vectorShape(op.getResult());
 
   // Compute atan in the valid range.
-  auto div = builder.create<arith::DivFOp>(y, x);
-  auto atan = builder.create<math::AtanOp>(div);
+  auto div = arith::DivFOp::create(builder, y, x);
+  auto atan = math::AtanOp::create(builder, div);
 
   // Determine what the atan would be for a 180 degree rotation.
   auto zero = broadcast(builder, f32Cst(builder, 0.0f), shape);
   auto pi = broadcast(builder, f32Cst(builder, 3.14159265359f), shape);
-  auto addPi = builder.create<arith::AddFOp>(atan, pi);
-  auto subPi = builder.create<arith::SubFOp>(atan, pi);
+  auto addPi = arith::AddFOp::create(builder, atan, pi);
+  auto subPi = arith::SubFOp::create(builder, atan, pi);
   auto atanGt =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, atan, zero);
-  auto flippedAtan = builder.create<arith::SelectOp>(atanGt, subPi, addPi);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, atan, zero);
+  auto flippedAtan = arith::SelectOp::create(builder, atanGt, subPi, addPi);
 
   // Determine whether to directly use atan or use the 180 degree flip
-  auto xGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, x, zero);
-  Value result = builder.create<arith::SelectOp>(xGt, atan, flippedAtan);
+  auto xGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, x, zero);
+  Value result = arith::SelectOp::create(builder, xGt, atan, flippedAtan);
 
   // Handle x = 0, y > 0
   Value xZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, x, zero);
-  Value yGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, y, zero);
-  Value isHalfPi = builder.create<arith::AndIOp>(xZero, yGt);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, x, zero);
+  Value yGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, y, zero);
+  Value isHalfPi = arith::AndIOp::create(builder, xZero, yGt);
   auto halfPi = broadcast(builder, f32Cst(builder, 1.57079632679f), shape);
-  result = builder.create<arith::SelectOp>(isHalfPi, halfPi, result);
+  result = arith::SelectOp::create(builder, isHalfPi, halfPi, result);
 
   // Handle x = 0, y < 0
-  Value yLt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, y, zero);
-  Value isNegativeHalfPiPi = builder.create<arith::AndIOp>(xZero, yLt);
+  Value yLt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, y, zero);
+  Value isNegativeHalfPiPi = arith::AndIOp::create(builder, xZero, yLt);
   auto negativeHalfPiPi =
       broadcast(builder, f32Cst(builder, -1.57079632679f), shape);
-  result = builder.create<arith::SelectOp>(isNegativeHalfPiPi, negativeHalfPiPi,
+  result = arith::SelectOp::create(builder, isNegativeHalfPiPi, negativeHalfPiPi,
                                            result);
 
   // Handle x = 0, y = 0;
   Value yZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, y, zero);
-  Value isNan = builder.create<arith::AndIOp>(xZero, yZero);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, y, zero);
+  Value isNan = arith::AndIOp::create(builder, xZero, yZero);
   Value cstNan = broadcast(builder, f32FromBits(builder, 0x7fc00000), shape);
-  result = builder.create<arith::SelectOp>(isNan, cstNan, result);
+  result = arith::SelectOp::create(builder, isNan, cstNan, result);
 
   rewriter.replaceOp(op, result);
   return success();
@@ -570,8 +570,8 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
 
   // Mask for tiny values that are approximated with `operand`.
   Value tiny = bcast(f32Cst(builder, 0.0004f));
-  Value tinyMask = builder.create<arith::CmpFOp>(
-      arith::CmpFPredicate::OLT, builder.create<math::AbsFOp>(op.getOperand()),
+  Value tinyMask = arith::CmpFOp::create(builder,
+      arith::CmpFPredicate::OLT, math::AbsFOp::create(builder, op.getOperand()),
       tiny);
 
   // The monomial coefficients of the numerator polynomial (odd).
@@ -590,25 +590,25 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
   Value beta6 = bcast(f32Cst(builder, 1.19825839466702e-06f));
 
   // Since the polynomials are odd/even, we need x^2.
-  Value x2 = builder.create<arith::MulFOp>(x, x);
+  Value x2 = arith::MulFOp::create(builder, x, x);
 
   // Evaluate the numerator polynomial p.
-  Value p = builder.create<math::FmaOp>(x2, alpha13, alpha11);
-  p = builder.create<math::FmaOp>(x2, p, alpha9);
-  p = builder.create<math::FmaOp>(x2, p, alpha7);
-  p = builder.create<math::FmaOp>(x2, p, alpha5);
-  p = builder.create<math::FmaOp>(x2, p, alpha3);
-  p = builder.create<math::FmaOp>(x2, p, alpha1);
-  p = builder.create<arith::MulFOp>(x, p);
+  Value p = math::FmaOp::create(builder, x2, alpha13, alpha11);
+  p = math::FmaOp::create(builder, x2, p, alpha9);
+  p = math::FmaOp::create(builder, x2, p, alpha7);
+  p = math::FmaOp::create(builder, x2, p, alpha5);
+  p = math::FmaOp::create(builder, x2, p, alpha3);
+  p = math::FmaOp::create(builder, x2, p, alpha1);
+  p = arith::MulFOp::create(builder, x, p);
 
   // Evaluate the denominator polynomial q.
-  Value q = builder.create<math::FmaOp>(x2, beta6, beta4);
-  q = builder.create<math::FmaOp>(x2, q, beta2);
-  q = builder.create<math::FmaOp>(x2, q, beta0);
+  Value q = math::FmaOp::create(builder, x2, beta6, beta4);
+  q = math::FmaOp::create(builder, x2, q, beta2);
+  q = math::FmaOp::create(builder, x2, q, beta0);
 
   // Divide the numerator by the denominator.
-  Value res = builder.create<arith::SelectOp>(
-      tinyMask, x, builder.create<arith::DivFOp>(p, q));
+  Value res = arith::SelectOp::create(builder,
+      tinyMask, x, arith::DivFOp::create(builder, p, q));
 
   rewriter.replaceOp(op, res);
 
@@ -691,57 +691,57 @@ LogApproximationBase<Op>::logMatchAndRewrite(Op op, PatternRewriter &rewriter,
   //     e -= 1;
   //     x = x + x - 1.0;
   //   } else { x = x - 1.0; }
-  Value mask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x,
+  Value mask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x,
                                              cstCephesSQRTHF);
-  Value tmp = builder.create<arith::SelectOp>(mask, x, cstZero);
+  Value tmp = arith::SelectOp::create(builder, mask, x, cstZero);
 
-  x = builder.create<arith::SubFOp>(x, cstOne);
-  e = builder.create<arith::SubFOp>(
-      e, builder.create<arith::SelectOp>(mask, cstOne, cstZero));
-  x = builder.create<arith::AddFOp>(x, tmp);
+  x = arith::SubFOp::create(builder, x, cstOne);
+  e = arith::SubFOp::create(builder,
+      e, arith::SelectOp::create(builder, mask, cstOne, cstZero));
+  x = arith::AddFOp::create(builder, x, tmp);
 
-  Value x2 = builder.create<arith::MulFOp>(x, x);
-  Value x3 = builder.create<arith::MulFOp>(x2, x);
+  Value x2 = arith::MulFOp::create(builder, x, x);
+  Value x3 = arith::MulFOp::create(builder, x2, x);
 
   // Evaluate the polynomial approximant of degree 8 in three parts.
   Value y0, y1, y2;
-  y0 = builder.create<math::FmaOp>(cstCephesLogP0, x, cstCephesLogP1);
-  y1 = builder.create<math::FmaOp>(cstCephesLogP3, x, cstCephesLogP4);
-  y2 = builder.create<math::FmaOp>(cstCephesLogP6, x, cstCephesLogP7);
-  y0 = builder.create<math::FmaOp>(y0, x, cstCephesLogP2);
-  y1 = builder.create<math::FmaOp>(y1, x, cstCephesLogP5);
-  y2 = builder.create<math::FmaOp>(y2, x, cstCephesLogP8);
-  y0 = builder.create<math::FmaOp>(y0, x3, y1);
-  y0 = builder.create<math::FmaOp>(y0, x3, y2);
-  y0 = builder.create<arith::MulFOp>(y0, x3);
-
-  y0 = builder.create<math::FmaOp>(cstNegHalf, x2, y0);
-  x = builder.create<arith::AddFOp>(x, y0);
+  y0 = math::FmaOp::create(builder, cstCephesLogP0, x, cstCephesLogP1);
+  y1 = math::FmaOp::create(builder, cstCephesLogP3, x, cstCephesLogP4);
+  y2 = math::FmaOp::create(builder, cstCephesLogP6, x, cstCephesLogP7);
+  y0 = math::FmaOp::create(builder, y0, x, cstCephesLogP2);
+  y1 = math::FmaOp::create(builder, y1, x, cstCephesLogP5);
+  y2 = math::FmaOp::create(builder, y2, x, cstCephesLogP8);
+  y0 = math::FmaOp::create(builder, y0, x3, y1);
+  y0 = math::FmaOp::create(builder, y0, x3, y2);
+  y0 = arith::MulFOp::create(builder, y0, x3);
+
+  y0 = math::FmaOp::create(builder, cstNegHalf, x2, y0);
+  x = arith::AddFOp::create(builder, x, y0);
 
   if (base2) {
     Value cstLog2e = bcast(f32Cst(builder, static_cast<float>(LOG2E_VALUE)));
-    x = builder.create<math::FmaOp>(x, cstLog2e, e);
+    x = math::FmaOp::create(builder, x, cstLog2e, e);
   } else {
     Value cstLn2 = bcast(f32Cst(builder, static_cast<float>(LN2_VALUE)));
-    x = builder.create<math::FmaOp>(e, cstLn2, x);
+    x = math::FmaOp::create(builder, e, cstLn2, x);
   }
 
-  Value invalidMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT,
+  Value invalidMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT,
                                                     op.getOperand(), cstZero);
-  Value zeroMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value zeroMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                  op.getOperand(), cstZero);
-  Value posInfMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value posInfMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                    op.getOperand(), cstPosInf);
 
   // Filter out invalid values:
   //  • x == 0     -> -INF
   //  • x < 0      ->  NAN
   //  • x == +INF  -> +INF
-  Value aproximation = builder.create<arith::SelectOp>(
+  Value aproximation = arith::SelectOp::create(builder,
       zeroMask, cstMinusInf,
-      builder.create<arith::SelectOp>(
+      arith::SelectOp::create(builder,
           invalidMask, cstNan,
-          builder.create<arith::SelectOp>(posInfMask, cstPosInf, x)));
+          arith::SelectOp::create(builder, posInfMask, cstPosInf, x)));
 
   rewriter.replaceOp(op, aproximation);
 
@@ -806,17 +806,17 @@ Log1pApproximation::matchAndRewrite(math::Log1pOp op,
   //             "logLarge" below.
   Value cstOne = bcast(f32Cst(builder, 1.0f));
   Value x = op.getOperand();
-  Value u = builder.create<arith::AddFOp>(x, cstOne);
+  Value u = arith::AddFOp::create(builder, x, cstOne);
   Value uSmall =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, cstOne);
-  Value logU = builder.create<math::LogOp>(u);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, cstOne);
+  Value logU = math::LogOp::create(builder, u);
   Value uInf =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, logU);
-  Value logLarge = builder.create<arith::MulFOp>(
-      x, builder.create<arith::DivFOp>(
-             logU, builder.create<arith::SubFOp>(u, cstOne)));
-  Value approximation = builder.create<arith::SelectOp>(
-      builder.create<arith::OrIOp>(uSmall, uInf), x, logLarge);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, logU);
+  Value logLarge = arith::MulFOp::create(builder,
+      x, arith::DivFOp::create(builder,
+             logU, arith::SubFOp::create(builder, u, cstOne)));
+  Value approximation = arith::SelectOp::create(builder,
+      arith::OrIOp::create(builder, uSmall, uInf), x, logLarge);
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -854,27 +854,27 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   };
 
   auto fma = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
   auto sub = [&](Value a, Value b) -> Value {
-    return builder.create<arith::SubFOp>(a, b);
+    return arith::SubFOp::create(builder, a, b);
   };
 
-  auto abs = [&](Value a) -> Value { return builder.create<math::AbsFOp>(a); };
+  auto abs = [&](Value a) -> Value { return math::AbsFOp::create(builder, a); };
 
-  auto sqrt = [&](Value a) -> Value { return builder.create<math::SqrtOp>(a); };
+  auto sqrt = [&](Value a) -> Value { return math::SqrtOp::create(builder, a); };
 
   auto scopy = [&](Value a, Value b) -> Value {
-    return builder.create<math::CopySignOp>(a, b);
+    return math::CopySignOp::create(builder, a, b);
   };
 
   auto sel = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<arith::SelectOp>(a, b, c);
+    return arith::SelectOp::create(builder, a, b, c);
   };
 
   Value abso = abs(operand);
@@ -882,7 +882,7 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   Value opp = sqrt(sub(bcast(floatCst(builder, 1.0, elementType)), aa));
 
   Value gt =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, aa,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, aa,
                                     bcast(floatCst(builder, 0.5, elementType)));
 
   Value x = sel(gt, opp, abso);
@@ -949,51 +949,51 @@ AcosPolynomialApproximation::matchAndRewrite(math::AcosOp op,
   };
 
   auto fma = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
-  Value negOperand = builder.create<arith::NegFOp>(operand);
+  Value negOperand = arith::NegFOp::create(builder, operand);
   Value zero = bcast(floatCst(builder, 0.0, elementType));
   Value half = bcast(floatCst(builder, 0.5, elementType));
   Value negOne = bcast(floatCst(builder, -1.0, elementType));
   Value selR =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand, zero);
-  Value r = builder.create<arith::SelectOp>(selR, negOperand, operand);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, operand, zero);
+  Value r = arith::SelectOp::create(builder, selR, negOperand, operand);
   Value chkConst = bcast(floatCst(builder, -0.5625, elementType));
   Value firstPred =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, r, chkConst);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, r, chkConst);
 
   Value trueVal =
       fma(bcast(floatCst(builder, 9.3282184640716537e-1, elementType)),
           bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
-          builder.create<math::AsinOp>(r));
+          math::AsinOp::create(builder, r));
 
-  Value falseVal = builder.create<math::SqrtOp>(fma(half, r, half));
-  falseVal = builder.create<math::AsinOp>(falseVal);
+  Value falseVal = math::SqrtOp::create(builder, fma(half, r, half));
+  falseVal = math::AsinOp::create(builder, falseVal);
   falseVal = mul(bcast(floatCst(builder, 2.0, elementType)), falseVal);
 
-  r = builder.create<arith::SelectOp>(firstPred, trueVal, falseVal);
+  r = arith::SelectOp::create(builder, firstPred, trueVal, falseVal);
 
   // Check whether the operand lies in between [-1.0, 0.0).
   Value greaterThanNegOne =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, operand, negOne);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGE, operand, negOne);
 
   Value lessThanZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
 
   Value betweenNegOneZero =
-      builder.create<arith::AndIOp>(greaterThanNegOne, lessThanZero);
+      arith::AndIOp::create(builder, greaterThanNegOne, lessThanZero);
 
   trueVal = fma(bcast(floatCst(builder, 1.8656436928143307e+0, elementType)),
                 bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
-                builder.create<arith::NegFOp>(r));
+                arith::NegFOp::create(builder, r));
 
   Value finalVal =
-      builder.create<arith::SelectOp>(betweenNegOneZero, trueVal, r);
+      arith::SelectOp::create(builder, betweenNegOneZero, trueVal, r);
 
   rewriter.replaceOp(op, finalVal);
   return success();
@@ -1076,9 +1076,9 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
   bounds[2] = bcast(floatCst(builder, 3.75f, elementType));
 
   Value isNegativeArg =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
-  Value negArg = builder.create<arith::NegFOp>(operand);
-  Value x = builder.create<arith::SelectOp>(isNegativeArg, negArg, operand);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
+  Value negArg = arith::NegFOp::create(builder, operand);
+  Value x = arith::SelectOp::create(builder, isNegativeArg, negArg, operand);
 
   Value offset = offsets[0];
   Value p[polyDegree + 1];
@@ -1092,30 +1092,30 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
   Value isLessThanBound[intervalsCount];
   for (int j = 0; j < intervalsCount - 1; ++j) {
     isLessThanBound[j] =
-        builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, bounds[j]);
+        arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, bounds[j]);
     for (int i = 0; i <= polyDegree; ++i) {
-      p[i] = builder.create<arith::SelectOp>(isLessThanBound[j], p[i],
+      p[i] = arith::SelectOp::create(builder, isLessThanBound[j], p[i],
                                              pp[j + 1][i]);
-      q[i] = builder.create<arith::SelectOp>(isLessThanBound[j], q[i],
+      q[i] = arith::SelectOp::create(builder, isLessThanBound[j], q[i],
                                              qq[j + 1][i]);
     }
-    offset = builder.create<arith::SelectOp>(isLessThanBound[j], offset,
+    offset = arith::SelectOp::create(builder, isLessThanBound[j], offset,
                                              offsets[j + 1]);
   }
-  isLessThanBound[intervalsCount - 1] = builder.create<arith::CmpFOp>(
+  isLessThanBound[intervalsCount - 1] = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::ULT, x, bounds[intervalsCount - 1]);
 
   Value pPoly = makePolynomialCalculation(builder, p, x);
   Value qPoly = makePolynomialCalculation(builder, q, x);
-  Value rationalPoly = builder.create<arith::DivFOp>(pPoly, qPoly);
-  Value formula = builder.create<arith::AddFOp>(offset, rationalPoly);
-  formula = builder.create<arith::SelectOp>(isLessThanBound[intervalsCount - 1],
+  Value rationalPoly = arith::DivFOp::create(builder, pPoly, qPoly);
+  Value formula = arith::AddFOp::create(builder, offset, rationalPoly);
+  formula = arith::SelectOp::create(builder, isLessThanBound[intervalsCount - 1],
                                             formula, one);
 
   // erf is odd function: erf(x) = -erf(-x).
-  Value negFormula = builder.create<arith::NegFOp>(formula);
+  Value negFormula = arith::NegFOp::create(builder, formula);
   Value res =
-      builder.create<arith::SelectOp>(isNegativeArg, negFormula, formula);
+      arith::SelectOp::create(builder, isNegativeArg, negFormula, formula);
 
   rewriter.replaceOp(op, res);
 
@@ -1156,65 +1156,65 @@ ErfcPolynomialApproximation::matchAndRewrite(math::ErfcOp op,
   Value posInf = bcast(floatCst(builder, INFINITY, et));
   Value clampVal = bcast(floatCst(builder, 10.0546875f, et));
 
-  Value a = builder.create<math::AbsFOp>(x);
-  Value p = builder.create<arith::AddFOp>(a, pos2);
-  Value r = builder.create<arith::DivFOp>(one, p);
-  Value q = builder.create<math::FmaOp>(neg4, r, one);
-  Value t = builder.create<math::FmaOp>(builder.create<arith::AddFOp>(q, one),
+  Value a = math::AbsFOp::create(builder, x);
+  Value p = arith::AddFOp::create(builder, a, pos2);
+  Value r = arith::DivFOp::create(builder, one, p);
+  Value q = math::FmaOp::create(builder, neg4, r, one);
+  Value t = math::FmaOp::create(builder, arith::AddFOp::create(builder, q, one),
                                         neg2, a);
-  Value e = builder.create<math::FmaOp>(builder.create<arith::NegFOp>(a), q, t);
-  q = builder.create<math::FmaOp>(r, e, q);
+  Value e = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), q, t);
+  q = math::FmaOp::create(builder, r, e, q);
 
   p = bcast(floatCst(builder, -0x1.a4a000p-12f, et));        // -4.01139259e-4
   Value c1 = bcast(floatCst(builder, -0x1.42a260p-10f, et)); // -1.23075210e-3
-  p = builder.create<math::FmaOp>(p, q, c1);
+  p = math::FmaOp::create(builder, p, q, c1);
   Value c2 = bcast(floatCst(builder, 0x1.585714p-10f, et)); //  1.31355342e-3
-  p = builder.create<math::FmaOp>(p, q, c2);
+  p = math::FmaOp::create(builder, p, q, c2);
   Value c3 = bcast(floatCst(builder, 0x1.1adcc4p-07f, et)); // 8.63227434e-3
-  p = builder.create<math::FmaOp>(p, q, c3);
+  p = math::FmaOp::create(builder, p, q, c3);
   Value c4 = bcast(floatCst(builder, -0x1.081b82p-07f, et)); // -8.05991981e-3
-  p = builder.create<math::FmaOp>(p, q, c4);
+  p = math::FmaOp::create(builder, p, q, c4);
   Value c5 = bcast(floatCst(builder, -0x1.bc0b6ap-05f, et)); // -5.42046614e-2
-  p = builder.create<math::FmaOp>(p, q, c5);
+  p = math::FmaOp::create(builder, p, q, c5);
   Value c6 = bcast(floatCst(builder, 0x1.4ffc46p-03f, et)); //  1.64055392e-1
-  p = builder.create<math::FmaOp>(p, q, c6);
+  p = math::FmaOp::create(builder, p, q, c6);
   Value c7 = bcast(floatCst(builder, -0x1.540840p-03f, et)); // -1.66031361e-1
-  p = builder.create<math::FmaOp>(p, q, c7);
+  p = math::FmaOp::create(builder, p, q, c7);
   Value c8 = bcast(floatCst(builder, -0x1.7bf616p-04f, et)); // -9.27639827e-2
-  p = builder.create<math::FmaOp>(p, q, c8);
+  p = math::FmaOp::create(builder, p, q, c8);
   Value c9 = bcast(floatCst(builder, 0x1.1ba03ap-02f, et)); // 2.76978403e-1
-  p = builder.create<math::FmaOp>(p, q, c9);
-
-  Value d = builder.create<math::FmaOp>(pos2, a, one);
-  r = builder.create<arith::DivFOp>(one, d);
-  q = builder.create<math::FmaOp>(p, r, r);
-  Value negfa = builder.create<arith::NegFOp>(a);
-  Value fmaqah = builder.create<math::FmaOp>(q, negfa, onehalf);
-  Value psubq = builder.create<arith::SubFOp>(p, q);
-  e = builder.create<math::FmaOp>(fmaqah, pos2, psubq);
-  r = builder.create<math::FmaOp>(e, r, q);
-
-  Value s = builder.create<arith::MulFOp>(a, a);
-  e = builder.create<math::ExpOp>(builder.create<arith::NegFOp>(s));
-
-  t = builder.create<math::FmaOp>(builder.create<arith::NegFOp>(a), a, s);
-  r = builder.create<math::FmaOp>(
+  p = math::FmaOp::create(builder, p, q, c9);
+
+  Value d = math::FmaOp::create(builder, pos2, a, one);
+  r = arith::DivFOp::create(builder, one, d);
+  q = math::FmaOp::create(builder, p, r, r);
+  Value negfa = arith::NegFOp::create(builder, a);
+  Value fmaqah = math::FmaOp::create(builder, q, negfa, onehalf);
+  Value psubq = arith::SubFOp::create(builder, p, q);
+  e = math::FmaOp::create(builder, fmaqah, pos2, psubq);
+  r = math::FmaOp::create(builder, e, r, q);
+
+  Value s = arith::MulFOp::create(builder, a, a);
+  e = math::ExpOp::create(builder, arith::NegFOp::create(builder, s));
+
+  t = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), a, s);
+  r = math::FmaOp::create(builder,
       r, e,
-      builder.create<arith::MulFOp>(builder.create<arith::MulFOp>(r, e), t));
+      arith::MulFOp::create(builder, arith::MulFOp::create(builder, r, e), t));
 
-  Value isNotLessThanInf = builder.create<arith::XOrIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, a, posInf),
+  Value isNotLessThanInf = arith::XOrIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, a, posInf),
       trueValue);
-  r = builder.create<arith::SelectOp>(isNotLessThanInf,
-                                      builder.create<arith::AddFOp>(x, x), r);
+  r = arith::SelectOp::create(builder, isNotLessThanInf,
+                                      arith::AddFOp::create(builder, x, x), r);
   Value isGreaterThanClamp =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, a, clampVal);
-  r = builder.create<arith::SelectOp>(isGreaterThanClamp, zero, r);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, a, clampVal);
+  r = arith::SelectOp::create(builder, isGreaterThanClamp, zero, r);
 
   Value isNegative =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, zero);
-  r = builder.create<arith::SelectOp>(
-      isNegative, builder.create<arith::SubFOp>(pos2, r), r);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, zero);
+  r = arith::SelectOp::create(builder,
+      isNegative, arith::SubFOp::create(builder, pos2, r), r);
 
   rewriter.replaceOp(op, r);
   return success();
@@ -1236,8 +1236,8 @@ Value clampWithNormals(ImplicitLocOpBuilder &builder,
   };
 
   auto selectCmp = [&builder](auto pred, Value value, Value bound) {
-    return builder.create<arith::SelectOp>(
-        builder.create<arith::CmpFOp>(pred, value, bound), value, bound);
+    return arith::SelectOp::create(builder,
+        arith::CmpFOp::create(builder, pred, value, bound), value, bound);
   };
 
   // Note: prefer UGE/ULE vs. UGT/ULT, since they generate vmaxps/vminps vs.
@@ -1269,17 +1269,17 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
   ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
 
   auto add = [&](Value a, Value b) -> Value {
-    return builder.create<arith::AddFOp>(a, b);
+    return arith::AddFOp::create(builder, a, b);
   };
   auto bcast = [&](Value value) -> Value {
     return broadcast(builder, value, shape);
   };
-  auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); };
+  auto floor = [&](Value a) { return math::FloorOp::create(builder, a); };
   auto fmla = [&](Value a, Value b, Value c) {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
   // Polynomial approximation from Cephes.
@@ -1383,7 +1383,7 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
 
   // Convert n' to an i32.  This is safe because we clamped it above.
   auto i32Vec = broadcast(builder.getI32Type(), shape);
-  Value nI32 = builder.create<arith::FPToSIOp>(i32Vec, n);
+  Value nI32 = arith::FPToSIOp::create(builder, i32Vec, n);
 
   // Creates the value 2^n' if -126 <= n' <= 127 and 0 if n' = -127.
   Value pow2 = exp2I32(builder, nI32);
@@ -1431,26 +1431,26 @@ ExpM1Approximation::matchAndRewrite(math::ExpM1Op op,
   Value cstOne = bcast(f32Cst(builder, 1.0f));
   Value cstNegOne = bcast(f32Cst(builder, -1.0f));
   Value x = op.getOperand();
-  Value u = builder.create<math::ExpOp>(x);
+  Value u = math::ExpOp::create(builder, x);
   Value uEqOneOrNaN =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::UEQ, u, cstOne);
-  Value uMinusOne = builder.create<arith::SubFOp>(u, cstOne);
-  Value uMinusOneEqNegOne = builder.create<arith::CmpFOp>(
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::UEQ, u, cstOne);
+  Value uMinusOne = arith::SubFOp::create(builder, u, cstOne);
+  Value uMinusOneEqNegOne = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::OEQ, uMinusOne, cstNegOne);
   // logU = log(u) ~= x
-  Value logU = builder.create<math::LogOp>(u);
+  Value logU = math::LogOp::create(builder, u);
 
   // Detect exp(x) = +inf; written this way to avoid having to form +inf.
   Value isInf =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, logU, u);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, logU, u);
 
   // (u - 1) * (x / ~x)
-  Value expm1 = builder.create<arith::MulFOp>(
-      uMinusOne, builder.create<arith::DivFOp>(x, logU));
-  expm1 = builder.create<arith::SelectOp>(isInf, u, expm1);
-  Value approximation = builder.create<arith::SelectOp>(
+  Value expm1 = arith::MulFOp::create(builder,
+      uMinusOne, arith::DivFOp::create(builder, x, logU));
+  expm1 = arith::SelectOp::create(builder, isInf, u, expm1);
+  Value approximation = arith::SelectOp::create(builder,
       uEqOneOrNaN, x,
-      builder.create<arith::SelectOp>(uMinusOneEqNegOne, cstNegOne, expm1));
+      arith::SelectOp::create(builder, uMinusOneEqNegOne, cstNegOne, expm1));
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -1495,40 +1495,40 @@ LogicalResult SinAndCosApproximation<isSine, OpTy>::matchAndRewrite(
     return broadcast(builder, value, shape);
   };
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
   auto sub = [&](Value a, Value b) -> Value {
-    return builder.create<arith::SubFOp>(a, b);
+    return arith::SubFOp::create(builder, a, b);
   };
-  auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); };
+  auto floor = [&](Value a) { return math::FloorOp::create(builder, a); };
 
   auto i32Vec = broadcast(builder.getI32Type(), shape);
   auto fPToSingedInteger = [&](Value a) -> Value {
-    return builder.create<arith::FPToSIOp>(i32Vec, a);
+    return arith::FPToSIOp::create(builder, i32Vec, a);
   };
 
   auto modulo4 = [&](Value a) -> Value {
-    return builder.create<arith::AndIOp>(a, bcast(i32Cst(builder, 3)));
+    return arith::AndIOp::create(builder, a, bcast(i32Cst(builder, 3)));
   };
 
   auto isEqualTo = [&](Value a, Value b) -> Value {
-    return builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, a, b);
+    return arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, a, b);
   };
 
   auto isGreaterThan = [&](Value a, Value b) -> Value {
-    return builder.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, a, b);
+    return arith::CmpIOp::create(builder, arith::CmpIPredicate::sgt, a, b);
   };
 
   auto select = [&](Value cond, Value t, Value f) -> Value {
-    return builder.create<arith::SelectOp>(cond, t, f);
+    return arith::SelectOp::create(builder, cond, t, f);
   };
 
   auto fmla = [&](Value a, Value b, Value c) {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto bitwiseOr = [&](Value a, Value b) {
-    return builder.create<arith::OrIOp>(a, b);
+    return arith::OrIOp::create(builder, a, b);
   };
 
   Value twoOverPi = bcast(f32Cst(builder, (float)TWO_OVER_PI));
@@ -1625,7 +1625,7 @@ CbrtApproximation::matchAndRewrite(math::CbrtOp op,
   intTy = broadcast(intTy, shape);
 
   auto bconst = [&](TypedAttr attr) -> Value {
-    Value value = b.create<arith::ConstantOp>(attr);
+    Value value = arith::ConstantOp::create(b, attr);
     return broadcast(b, value, shape);
   };
 
@@ -1642,44 +1642,44 @@ CbrtApproximation::matchAndRewrite(math::CbrtOp op,
   // union {int ix; float x;};
   // x = x0;
   // ix = ix/4 + ix/16;
-  Value absValue = b.create<math::AbsFOp>(operand);
-  Value intValue = b.create<arith::BitcastOp>(intTy, absValue);
-  Value divideBy4 = b.create<arith::ShRSIOp>(intValue, intTwo);
-  Value divideBy16 = b.create<arith::ShRSIOp>(intValue, intFour);
-  intValue = b.create<arith::AddIOp>(divideBy4, divideBy16);
+  Value absValue = math::AbsFOp::create(b, operand);
+  Value intValue = arith::BitcastOp::create(b, intTy, absValue);
+  Value divideBy4 = arith::ShRSIOp::create(b, intValue, intTwo);
+  Value divideBy16 = arith::ShRSIOp::create(b, intValue, intFour);
+  intValue = arith::AddIOp::create(b, divideBy4, divideBy16);
 
   // ix = ix + ix/16;
-  divideBy16 = b.create<arith::ShRSIOp>(intValue, intFour);
-  intValue = b.create<arith::AddIOp>(intValue, divideBy16);
+  divideBy16 = arith::ShRSIOp::create(b, intValue, intFour);
+  intValue = arith::AddIOp::create(b, intValue, divideBy16);
 
   // ix = ix + ix/256;
-  Value divideBy256 = b.create<arith::ShRSIOp>(intValue, intEight);
-  intValue = b.create<arith::AddIOp>(intValue, divideBy256);
+  Value divideBy256 = arith::ShRSIOp::create(b, intValue, intEight);
+  intValue = arith::AddIOp::create(b, intValue, divideBy256);
 
   // ix = 0x2a5137a0 + ix;
-  intValue = b.create<arith::AddIOp>(intValue, intMagic);
+  intValue = arith::AddIOp::create(b, intValue, intMagic);
 
   // Perform one newtons step:
   // x = 0.33333333f*(2.0f*x + x0/(x*x));
-  Value floatValue = b.create<arith::BitcastOp>(floatTy, intValue);
-  Value squared = b.create<arith::MulFOp>(floatValue, floatValue);
-  Value mulTwo = b.create<arith::MulFOp>(floatValue, fpTwo);
-  Value divSquared = b.create<arith::DivFOp>(absValue, squared);
-  floatValue = b.create<arith::AddFOp>(mulTwo, divSquared);
-  floatValue = b.create<arith::MulFOp>(floatValue, fpThird);
+  Value floatValue = arith::BitcastOp::create(b, floatTy, intValue);
+  Value squared = arith::MulFOp::create(b, floatValue, floatValue);
+  Value mulTwo = arith::MulFOp::create(b, floatValue, fpTwo);
+  Value divSquared = arith::DivFOp::create(b, absValue, squared);
+  floatValue = arith::AddFOp::create(b, mulTwo, divSquared);
+  floatValue = arith::MulFOp::create(b, floatValue, fpThird);
 
   // x = 0.33333333f*(2.0f*x + x0/(x*x));
-  squared = b.create<arith::MulFOp>(floatValue, floatValue);
-  mulTwo = b.create<arith::MulFOp>(floatValue, fpTwo);
-  divSquared = b.create<arith::DivFOp>(absValue, squared);
-  floatValue = b.create<arith::AddFOp>(mulTwo, divSquared);
-  floatValue = b.create<arith::MulFOp>(floatValue, fpThird);
+  squared = arith::MulFOp::create(b, floatValue, floatValue);
+  mulTwo = arith::MulFOp::create(b, floatValue, fpTwo);
+  divSquared = arith::DivFOp::create(b, absValue, squared);
+  floatValue = arith::AddFOp::create(b, mulTwo, divSquared);
+  floatValue = arith::MulFOp::create(b, floatValue, fpThird);
 
   // Check for zero and restore sign.
   Value isZero =
-      b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absValue, fpZero);
-  floatValue = b.create<arith::SelectOp>(isZero, fpZero, floatValue);
-  floatValue = b.create<math::CopySignOp>(floatValue, operand);
+      arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absValue, fpZero);
+  floatValue = arith::SelectOp::create(b, isZero, fpZero, floatValue);
+  floatValue = math::CopySignOp::create(b, floatValue, operand);
 
   rewriter.replaceOp(op, floatValue);
   return success();
@@ -1720,29 +1720,29 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
   Value cstNegHalf = bcast(f32Cst(builder, -0.5f));
   Value cstMinNormPos = bcast(f32FromBits(builder, 0x00800000u));
 
-  Value negHalf = builder.create<arith::MulFOp>(op.getOperand(), cstNegHalf);
+  Value negHalf = arith::MulFOp::create(builder, op.getOperand(), cstNegHalf);
 
   // Select only the inverse sqrt of positive normals (denormals are
   // flushed to zero).
-  Value ltMinMask = builder.create<arith::CmpFOp>(
+  Value ltMinMask = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::OLT, op.getOperand(), cstMinNormPos);
-  Value infMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value infMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                 op.getOperand(), cstPosInf);
-  Value notNormalFiniteMask = builder.create<arith::OrIOp>(ltMinMask, infMask);
+  Value notNormalFiniteMask = arith::OrIOp::create(builder, ltMinMask, infMask);
 
   // Compute an approximate result.
   Value yApprox = handleMultidimensionalVectors(
       builder, op->getOperands(), 8, [&builder](ValueRange operands) -> Value {
-        return builder.create<x86vector::RsqrtOp>(operands);
+        return x86vector::RsqrtOp::create(builder, operands);
       });
 
   // Do a single step of Newton-Raphson iteration to improve the approximation.
   // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n).
   // It is essential to evaluate the inner term like this because forming
   // y_n^2 may over- or underflow.
-  Value inner = builder.create<arith::MulFOp>(negHalf, yApprox);
-  Value fma = builder.create<math::FmaOp>(yApprox, inner, cstOnePointFive);
-  Value yNewton = builder.create<arith::MulFOp>(yApprox, fma);
+  Value inner = arith::MulFOp::create(builder, negHalf, yApprox);
+  Value fma = math::FmaOp::create(builder, yApprox, inner, cstOnePointFive);
+  Value yNewton = arith::MulFOp::create(builder, yApprox, fma);
 
   // Select the result of the Newton-Raphson step for positive normal arguments.
   // For other arguments, choose the output of the intrinsic. This will
@@ -1750,7 +1750,7 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
   // x is zero or a positive denormalized float (equivalent to flushing positive
   // denormalized inputs to zero).
   Value res =
-      builder.create<arith::SelectOp>(notNormalFiniteMask, yApprox, yNewton);
+      arith::SelectOp::create(builder, notNormalFiniteMask, yApprox, yNewton);
   rewriter.replaceOp(op, res);
 
   return success();
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index f630c48cdcaa1..ddb698e1bf501 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -87,10 +87,10 @@ Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
   // TODO: support more types.
   return TypeSwitch<Type, Value>(slot.elemType)
       .Case([&](MemRefType t) {
-        return builder.create<memref::AllocaOp>(getLoc(), t);
+        return memref::AllocaOp::create(builder, getLoc(), t);
       })
       .Default([&](Type t) {
-        return builder.create<arith::ConstantOp>(getLoc(), t,
+        return arith::ConstantOp::create(builder, getLoc(), t,
                                                  builder.getZeroAttr(t));
       });
 }
@@ -137,7 +137,7 @@ DenseMap<Attribute, MemorySlot> memref::AllocaOp::destructure(
   for (Attribute usedIndex : usedIndices) {
     Type elemType = memrefType.getTypeAtIndex(usedIndex);
     MemRefType elemPtr = MemRefType::get({}, elemType);
-    auto subAlloca = builder.create<memref::AllocaOp>(getLoc(), elemPtr);
+    auto subAlloca = memref::AllocaOp::create(builder, getLoc(), elemPtr);
     newAllocators.push_back(subAlloca);
     slotMap.try_emplace<MemorySlot>(usedIndex,
                                     {subAlloca.getResult(), elemType});
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index d1a9920aa66c5..706ac4b4867f4 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -213,7 +213,7 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
 
     // Create and insert the alloc op for the new memref.
-    auto newAlloc = rewriter.create<AllocLikeOp>(
+    auto newAlloc = AllocLikeOp::create(rewriter,
         alloc.getLoc(), newMemRefType, dynamicSizes, alloc.getSymbolOperands(),
         alloc.getAlignmentAttr());
     // Insert a cast so we have the same type as the old alloc.
@@ -836,7 +836,7 @@ void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
 void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
                   int64_t index) {
   auto loc = result.location;
-  Value indexValue = builder.create<arith::ConstantIndexOp>(loc, index);
+  Value indexValue = arith::ConstantIndexOp::create(builder, loc, index);
   build(builder, result, source, indexValue);
 }
 
@@ -1083,9 +1083,9 @@ struct DimOfMemRefReshape : public OpRewritePattern<DimOp> {
     rewriter.setInsertionPointAfter(reshape);
     Location loc = dim.getLoc();
     Value load =
-        rewriter.create<LoadOp>(loc, reshape.getShape(), dim.getIndex());
+        LoadOp::create(rewriter, loc, reshape.getShape(), dim.getIndex());
     if (load.getType() != dim.getType())
-      load = rewriter.create<arith::IndexCastOp>(loc, dim.getType(), load);
+      load = arith::IndexCastOp::create(rewriter, loc, dim.getType(), load);
     rewriter.replaceOp(dim, load);
     return success();
   }
@@ -1358,7 +1358,7 @@ static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
     assert(isa<Attribute>(maybeConstant) &&
            "The constified value should be either unchanged (i.e., == result) "
            "or a constant");
-    Value constantVal = rewriter.create<arith::ConstantIndexOp>(
+    Value constantVal = arith::ConstantIndexOp::create(rewriter,
         loc, llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
     for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
       // modifyOpInPlace: lambda cannot capture structured bindings in C++17
@@ -2587,7 +2587,7 @@ struct CollapseShapeOpMemRefCastFolder
       rewriter.modifyOpInPlace(
           op, [&]() { op.getSrcMutable().assign(cast.getSource()); });
     } else {
-      Value newOp = rewriter.create<CollapseShapeOp>(
+      Value newOp = CollapseShapeOp::create(rewriter,
           op->getLoc(), cast.getSource(), op.getReassociationIndices());
       rewriter.replaceOpWithNewOp<CastOp>(op, op.getType(), newOp);
     }
@@ -3045,15 +3045,15 @@ SmallVector<Range, 8> mlir::getOrCreateRanges(OffsetSizeAndStrideOpInterface op,
     Value offset =
         op.isDynamicOffset(idx)
             ? op.getDynamicOffset(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticOffset(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticOffset(idx));
     Value size =
         op.isDynamicSize(idx)
             ? op.getDynamicSize(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticSize(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticSize(idx));
     Value stride =
         op.isDynamicStride(idx)
             ? op.getDynamicStride(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticStride(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticStride(idx));
     res.emplace_back(Range{offset, size, stride});
   }
   return res;
@@ -3212,7 +3212,7 @@ class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
     if (!resultType)
       return failure();
 
-    Value newSubView = rewriter.create<SubViewOp>(
+    Value newSubView = SubViewOp::create(rewriter,
         subViewOp.getLoc(), resultType, castOp.getSource(),
         subViewOp.getOffsets(), subViewOp.getSizes(), subViewOp.getStrides(),
         subViewOp.getStaticOffsets(), subViewOp.getStaticSizes(),
@@ -3534,7 +3534,7 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
       return failure();
 
     // Create new ViewOp.
-    auto newViewOp = rewriter.create<ViewOp>(
+    auto newViewOp = ViewOp::create(rewriter,
         viewOp.getLoc(), newMemRefType, viewOp.getOperand(0),
         viewOp.getByteShift(), newOperands);
     // Insert a cast so we have the same type as the old memref type.
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 89640ac323b68..8176ed0f02dce 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -156,7 +156,7 @@ transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
       Type resultType = alloca.getResult().getType();
       OpBuilder builder(rewriter.getContext());
       // TODO: Add a better builder for this.
-      globalOp = builder.create<memref::GlobalOp>(
+      globalOp = memref::GlobalOp::create(builder,
           loc, StringAttr::get(ctx, "alloca"), StringAttr::get(ctx, "private"),
           TypeAttr::get(resultType), Attribute{}, UnitAttr{}, IntegerAttr{});
       symbolTable.insert(globalOp);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
index c433415944323..a2b689ed13cbf 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
@@ -22,11 +22,11 @@ struct DefaultAllocationInterface
           DefaultAllocationInterface, memref::AllocOp> {
   static std::optional<Operation *> buildDealloc(OpBuilder &builder,
                                                  Value alloc) {
-    return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
+    return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
         .getOperation();
   }
   static std::optional<Value> buildClone(OpBuilder &builder, Value alloc) {
-    return builder.create<bufferization::CloneOp>(alloc.getLoc(), alloc)
+    return bufferization::CloneOp::create(builder, alloc.getLoc(), alloc)
         .getResult();
   }
   static ::mlir::HoistingKind getHoistingKind() {
@@ -35,7 +35,7 @@ struct DefaultAllocationInterface
   static ::std::optional<::mlir::Operation *>
   buildPromotedAlloc(OpBuilder &builder, Value alloc) {
     Operation *definingOp = alloc.getDefiningOp();
-    return builder.create<memref::AllocaOp>(
+    return memref::AllocaOp::create(builder,
         definingOp->getLoc(), cast<MemRefType>(definingOp->getResultTypes()[0]),
         definingOp->getOperands(), definingOp->getAttrs());
   }
@@ -52,7 +52,7 @@ struct DefaultReallocationInterface
           DefaultAllocationInterface, memref::ReallocOp> {
   static std::optional<Operation *> buildDealloc(OpBuilder &builder,
                                                  Value realloc) {
-    return builder.create<memref::DeallocOp>(realloc.getLoc(), realloc)
+    return memref::DeallocOp::create(builder, realloc.getLoc(), realloc)
         .getOperation();
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
index d25ddb41aa4eb..4fcceeaa33deb 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
@@ -125,7 +125,7 @@ struct ComposeSubViewOpPattern : public OpRewritePattern<memref::SubViewOp> {
         }
 
         AffineMap map = AffineMap::get(0, affineApplyOperands.size(), expr);
-        Value result = rewriter.create<affine::AffineApplyOp>(
+        Value result = affine::AffineApplyOp::create(rewriter,
             op.getLoc(), map, affineApplyOperands);
         offsets.push_back(result);
       }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
index d2a032688fb6d..51970f019d27f 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
@@ -99,7 +99,7 @@ static Value getOffsetForBitwidth(Location loc, OpFoldResult srcIdx,
       affine::makeComposedFoldedAffineApply(builder, loc, offsetExpr, {srcIdx});
   Value bitOffset = getValueOrCreateConstantIndexOp(builder, loc, offsetVal);
   IntegerType dstType = builder.getIntegerType(targetBits);
-  return builder.create<arith::IndexCastOp>(loc, dstType, bitOffset);
+  return arith::IndexCastOp::create(builder, loc, dstType, bitOffset);
 }
 
 /// When writing a subbyte size, masked bitwise operations are used to only
@@ -112,14 +112,14 @@ static Value getSubByteWriteMask(Location loc, OpFoldResult linearizedIndices,
   auto dstIntegerType = builder.getIntegerType(dstBits);
   auto maskRightAlignedAttr =
       builder.getIntegerAttr(dstIntegerType, (1 << srcBits) - 1);
-  Value maskRightAligned = builder.create<arith::ConstantOp>(
+  Value maskRightAligned = arith::ConstantOp::create(builder,
       loc, dstIntegerType, maskRightAlignedAttr);
   Value writeMaskInverse =
-      builder.create<arith::ShLIOp>(loc, maskRightAligned, bitwidthOffset);
+      arith::ShLIOp::create(builder, loc, maskRightAligned, bitwidthOffset);
   auto flipValAttr = builder.getIntegerAttr(dstIntegerType, -1);
   Value flipVal =
-      builder.create<arith::ConstantOp>(loc, dstIntegerType, flipValAttr);
-  return builder.create<arith::XOrIOp>(loc, writeMaskInverse, flipVal);
+      arith::ConstantOp::create(builder, loc, dstIntegerType, flipValAttr);
+  return arith::XOrIOp::create(builder, loc, writeMaskInverse, flipVal);
 }
 
 /// Returns the scaled linearized index based on the `srcBits` and `dstBits`
@@ -141,7 +141,7 @@ getLinearizedSrcIndices(OpBuilder &builder, Location loc, int64_t srcBits,
                         const SmallVector<OpFoldResult> &indices,
                         Value memref) {
   auto stridedMetadata =
-      builder.create<memref::ExtractStridedMetadataOp>(loc, memref);
+      memref::ExtractStridedMetadataOp::create(builder, loc, memref);
   OpFoldResult linearizedIndices;
   std::tie(std::ignore, linearizedIndices) =
       memref::getLinearizedMemRefOffsetAndSize(
@@ -298,7 +298,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
     // Special case 0-rank memref loads.
     Value bitsLoad;
     if (convertedType.getRank() == 0) {
-      bitsLoad = rewriter.create<memref::LoadOp>(loc, adaptor.getMemref(),
+      bitsLoad = memref::LoadOp::create(rewriter, loc, adaptor.getMemref(),
                                                  ValueRange{});
     } else {
       // Linearize the indices of the original load instruction. Do not account
@@ -306,7 +306,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
       OpFoldResult linearizedIndices = getLinearizedSrcIndices(
           rewriter, loc, srcBits, adaptor.getIndices(), op.getMemRef());
 
-      Value newLoad = rewriter.create<memref::LoadOp>(
+      Value newLoad = memref::LoadOp::create(rewriter,
           loc, adaptor.getMemref(),
           getIndicesForLoadOrStore(rewriter, loc, linearizedIndices, srcBits,
                                    dstBits));
@@ -315,7 +315,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
       // Note, currently only the big-endian is supported.
       Value bitwidthOffset = getOffsetForBitwidth(loc, linearizedIndices,
                                                   srcBits, dstBits, rewriter);
-      bitsLoad = rewriter.create<arith::ShRSIOp>(loc, newLoad, bitwidthOffset);
+      bitsLoad = arith::ShRSIOp::create(rewriter, loc, newLoad, bitwidthOffset);
     }
 
     // Get the corresponding bits. If the arith computation bitwidth equals
@@ -326,13 +326,13 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
     Operation *result;
     auto resultTy = getTypeConverter()->convertType(oldElementType);
     if (resultTy == convertedElementType) {
-      auto mask = rewriter.create<arith::ConstantOp>(
+      auto mask = arith::ConstantOp::create(rewriter,
           loc, convertedElementType,
           rewriter.getIntegerAttr(convertedElementType, (1 << srcBits) - 1));
 
-      result = rewriter.create<arith::AndIOp>(loc, bitsLoad, mask);
+      result = arith::AndIOp::create(rewriter, loc, bitsLoad, mask);
     } else {
-      result = rewriter.create<arith::TruncIOp>(loc, resultTy, bitsLoad);
+      result = arith::TruncIOp::create(rewriter, loc, resultTy, bitsLoad);
     }
 
     rewriter.replaceOp(op, result->getResult(0));
@@ -415,12 +415,12 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
     }
 
     Location loc = op.getLoc();
-    Value extendedInput = rewriter.create<arith::ExtUIOp>(loc, dstIntegerType,
+    Value extendedInput = arith::ExtUIOp::create(rewriter, loc, dstIntegerType,
                                                           adaptor.getValue());
 
     // Special case 0-rank memref stores. No need for masking.
     if (convertedType.getRank() == 0) {
-      rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::assign,
+      memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::assign,
                                            extendedInput, adaptor.getMemref(),
                                            ValueRange{});
       rewriter.eraseOp(op);
@@ -437,14 +437,14 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
                                           dstBits, bitwidthOffset, rewriter);
     // Align the value to write with the destination bits
     Value alignedVal =
-        rewriter.create<arith::ShLIOp>(loc, extendedInput, bitwidthOffset);
+        arith::ShLIOp::create(rewriter, loc, extendedInput, bitwidthOffset);
 
     // Clear destination bits
-    rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::andi,
+    memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::andi,
                                          writeMask, adaptor.getMemref(),
                                          storeIndices);
     // Write srcs bits to destination
-    rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::ori,
+    memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::ori,
                                          alignedVal, adaptor.getMemref(),
                                          storeIndices);
     rewriter.eraseOp(op);
@@ -506,7 +506,7 @@ struct ConvertMemRefSubview final : OpConversionPattern<memref::SubViewOp> {
     }
 
     // Transform the offsets, sizes and strides according to the emulation.
-    auto stridedMetadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+    auto stridedMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
         loc, subViewOp.getViewSource());
 
     OpFoldResult linearizedIndices;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
index a617029ce470f..64e117fb77bcf 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
@@ -51,15 +51,15 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
       Value size;
       // Load dynamic sizes from the shape input, use constants for static dims.
       if (op.getType().isDynamicDim(i)) {
-        Value index = rewriter.create<arith::ConstantIndexOp>(loc, i);
-        size = rewriter.create<memref::LoadOp>(loc, op.getShape(), index);
+        Value index = arith::ConstantIndexOp::create(rewriter, loc, i);
+        size = memref::LoadOp::create(rewriter, loc, op.getShape(), index);
         if (!isa<IndexType>(size.getType()))
-          size = rewriter.create<arith::IndexCastOp>(
+          size = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getIndexType(), size);
         sizes[i] = size;
       } else {
         auto sizeAttr = rewriter.getIndexAttr(op.getType().getDimSize(i));
-        size = rewriter.create<arith::ConstantOp>(loc, sizeAttr);
+        size = arith::ConstantOp::create(rewriter, loc, sizeAttr);
         sizes[i] = sizeAttr;
       }
       if (stride)
@@ -69,10 +69,10 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
 
       if (i > 0) {
         if (stride) {
-          stride = rewriter.create<arith::MulIOp>(loc, stride, size);
+          stride = arith::MulIOp::create(rewriter, loc, stride, size);
         } else if (op.getType().isDynamicDim(i)) {
-          stride = rewriter.create<arith::MulIOp>(
-              loc, rewriter.create<arith::ConstantIndexOp>(loc, staticStride),
+          stride = arith::MulIOp::create(rewriter,
+              loc, arith::ConstantIndexOp::create(rewriter, loc, staticStride),
               size);
         } else {
           staticStride *= op.getType().getDimSize(i);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
index 7475d442b7b9a..b54fa61714323 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
@@ -73,7 +73,7 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     if (ShapedType::isDynamic(inputSize)) {
       Value dimZero = getValueOrCreateConstantIndexOp(rewriter, loc,
                                                       rewriter.getIndexAttr(0));
-      currSize = rewriter.create<memref::DimOp>(loc, op.getSource(), dimZero)
+      currSize = memref::DimOp::create(rewriter, loc, op.getSource(), dimZero)
                      .getResult();
     }
 
@@ -88,9 +88,9 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     // the old buffer is smaller than the requested size.
     Value lhs = getValueOrCreateConstantIndexOp(rewriter, loc, currSize);
     Value rhs = getValueOrCreateConstantIndexOp(rewriter, loc, targetSize);
-    Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                                 lhs, rhs);
-    auto ifOp = rewriter.create<scf::IfOp>(
+    auto ifOp = scf::IfOp::create(rewriter,
         loc, cond,
         [&](OpBuilder &builder, Location loc) {
           // Allocate the new buffer. If it is a dynamic memref we need to pass
@@ -100,25 +100,25 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           if (op.getDynamicResultSize())
             dynamicSizeOperands.push_back(op.getDynamicResultSize());
 
-          Value newAlloc = builder.create<memref::AllocOp>(
+          Value newAlloc = memref::AllocOp::create(builder,
               loc, op.getResult().getType(), dynamicSizeOperands,
               op.getAlignmentAttr());
 
           // Take a subview of the new (bigger) buffer such that we can copy the
           // old values over (the copy operation requires both operands to have
           // the same shape).
-          Value subview = builder.create<memref::SubViewOp>(
+          Value subview = memref::SubViewOp::create(builder,
               loc, newAlloc, ArrayRef<OpFoldResult>{rewriter.getIndexAttr(0)},
               ArrayRef<OpFoldResult>{currSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
-          builder.create<memref::CopyOp>(loc, op.getSource(), subview);
+          memref::CopyOp::create(builder, loc, op.getSource(), subview);
 
           // Insert the deallocation of the old buffer only if requested
           // (enabled by default).
           if (emitDeallocs)
-            builder.create<memref::DeallocOp>(loc, op.getSource());
+            memref::DeallocOp::create(builder, loc, op.getSource());
 
-          builder.create<scf::YieldOp>(loc, newAlloc);
+          scf::YieldOp::create(builder, loc, newAlloc);
         },
         [&](OpBuilder &builder, Location loc) {
           // We need to reinterpret-cast here because either the input or output
@@ -126,11 +126,11 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           // dynamic or vice-versa. If both are static and the original buffer
           // is already bigger than the requested size, the cast represents a
           // subview operation.
-          Value casted = builder.create<memref::ReinterpretCastOp>(
+          Value casted = memref::ReinterpretCastOp::create(builder,
               loc, cast<MemRefType>(op.getResult().getType()), op.getSource(),
               rewriter.getIndexAttr(0), ArrayRef<OpFoldResult>{targetSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
-          builder.create<scf::YieldOp>(loc, casted);
+          scf::YieldOp::create(builder, loc, casted);
         });
 
     rewriter.replaceOp(op, ifOp.getResult(0));
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index 2ba798f48ac7c..f581b6490fadb 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -66,7 +66,7 @@ resolveSubviewStridedMetadata(RewriterBase &rewriter,
   unsigned sourceRank = sourceType.getRank();
 
   auto newExtractStridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(origLoc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, origLoc, source);
 
   auto [sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
 #ifndef NDEBUG
@@ -577,7 +577,7 @@ static FailureOr<StridedMetadata> resolveReshapeStridedMetadata(
   unsigned sourceRank = sourceType.getRank();
 
   auto newExtractStridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(origLoc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, origLoc, source);
 
   // Collect statically known information.
   auto [strides, offset] = sourceType.getStridesAndOffset();
@@ -828,14 +828,14 @@ struct ExtractStridedMetadataOpAllocFolder
       if (allocLikeOp.getType() == baseBufferType)
         results.push_back(allocLikeOp);
       else
-        results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+        results.push_back(memref::ReinterpretCastOp::create(rewriter,
             loc, baseBufferType, allocLikeOp, offset,
             /*sizes=*/ArrayRef<int64_t>(),
             /*strides=*/ArrayRef<int64_t>()));
     }
 
     // Offset.
-    results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    results.push_back(arith::ConstantIndexOp::create(rewriter, loc, offset));
 
     for (OpFoldResult size : sizes)
       results.push_back(getValueOrCreateConstantIndexOp(rewriter, loc, size));
@@ -900,19 +900,19 @@ struct ExtractStridedMetadataOpGetGlobalFolder
     if (getGlobalOp.getType() == baseBufferType)
       results.push_back(getGlobalOp);
     else
-      results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+      results.push_back(memref::ReinterpretCastOp::create(rewriter,
           loc, baseBufferType, getGlobalOp, offset,
           /*sizes=*/ArrayRef<int64_t>(),
           /*strides=*/ArrayRef<int64_t>()));
 
     // Offset.
-    results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    results.push_back(arith::ConstantIndexOp::create(rewriter, loc, offset));
 
     for (auto size : sizes)
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, size));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, size));
 
     for (auto stride : strides)
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, stride));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, stride));
 
     rewriter.replaceOp(op, results);
     return success();
@@ -1009,7 +1009,7 @@ class ExtractStridedMetadataOpReinterpretCastFolder
     results.resize_for_overwrite(rank * 2 + 2);
 
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(
+        memref::ExtractStridedMetadataOp::create(rewriter,
             loc, reinterpretCastOp.getSource());
 
     // Register the base_buffer.
@@ -1083,7 +1083,7 @@ class ExtractStridedMetadataOpCastFolder
     results.resize_for_overwrite(rank * 2 + 2);
 
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc,
+        memref::ExtractStridedMetadataOp::create(rewriter, loc,
                                                           castOp.getSource());
 
     // Register the base_buffer.
@@ -1143,7 +1143,7 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
     if (!memSpaceCastOp)
       return failure();
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(
+        memref::ExtractStridedMetadataOp::create(rewriter,
             loc, memSpaceCastOp.getSource());
     SmallVector<Value> results(newExtractStridedMetadata.getResults());
     // As with most other strided metadata rewrite patterns, don't introduce
@@ -1158,7 +1158,7 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
       MemRefType::Builder newTypeBuilder(baseBufferType);
       newTypeBuilder.setMemorySpace(
           memSpaceCastOp.getResult().getType().getMemorySpace());
-      results[0] = rewriter.create<memref::MemorySpaceCastOp>(
+      results[0] = memref::MemorySpaceCastOp::create(rewriter,
           loc, Type{newTypeBuilder}, baseBuffer);
     } else {
       results[0] = nullptr;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
index 2f5c9436fb8c7..65f398f6dc5a8 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
@@ -42,7 +42,7 @@ static memref::LoadOp rebuildLoadOp(RewriterBase &rewriter,
                                     memref::LoadOp loadOp, Value srcMemRef,
                                     ArrayRef<Value> indices) {
   Location loc = loadOp.getLoc();
-  return rewriter.create<memref::LoadOp>(loc, srcMemRef, indices,
+  return memref::LoadOp::create(rewriter, loc, srcMemRef, indices,
                                          loadOp.getNontemporal());
 }
 
@@ -72,7 +72,7 @@ static memref::StoreOp rebuildStoreOp(RewriterBase &rewriter,
                                       memref::StoreOp storeOp, Value srcMemRef,
                                       ArrayRef<Value> indices) {
   Location loc = storeOp.getLoc();
-  return rewriter.create<memref::StoreOp>(loc, storeOp.getValueToStore(),
+  return memref::StoreOp::create(rewriter, loc, storeOp.getValueToStore(),
                                           srcMemRef, indices,
                                           storeOp.getNontemporal());
 }
@@ -104,7 +104,7 @@ static nvgpu::LdMatrixOp rebuildLdMatrixOp(RewriterBase &rewriter,
                                            Value srcMemRef,
                                            ArrayRef<Value> indices) {
   Location loc = ldMatrixOp.getLoc();
-  return rewriter.create<nvgpu::LdMatrixOp>(
+  return nvgpu::LdMatrixOp::create(rewriter,
       loc, ldMatrixOp.getResult().getType(), srcMemRef, indices,
       ldMatrixOp.getTranspose(), ldMatrixOp.getNumTiles());
 }
@@ -132,7 +132,7 @@ rebuildTransferReadOp(RewriterBase &rewriter,
                       vector::TransferReadOp transferReadOp, Value srcMemRef,
                       ArrayRef<Value> indices) {
   Location loc = transferReadOp.getLoc();
-  return rewriter.create<vector::TransferReadOp>(
+  return vector::TransferReadOp::create(rewriter,
       loc, transferReadOp.getResult().getType(), srcMemRef, indices,
       transferReadOp.getPermutationMap(), transferReadOp.getPadding(),
       transferReadOp.getMask(), transferReadOp.getInBoundsAttr());
@@ -150,7 +150,7 @@ rebuildTransferWriteOp(RewriterBase &rewriter,
                        vector::TransferWriteOp transferWriteOp, Value srcMemRef,
                        ArrayRef<Value> indices) {
   Location loc = transferWriteOp.getLoc();
-  return rewriter.create<vector::TransferWriteOp>(
+  return vector::TransferWriteOp::create(rewriter,
       loc, transferWriteOp.getValue(), srcMemRef, indices,
       transferWriteOp.getPermutationMapAttr(), transferWriteOp.getMask(),
       transferWriteOp.getInBoundsAttr());
@@ -183,7 +183,7 @@ getGenericOpViewSizeForEachDim(RewriterBase &rewriter,
                                LoadStoreLikeOp loadStoreLikeOp) {
   Location loc = loadStoreLikeOp.getLoc();
   auto extractStridedMetadataOp =
-      rewriter.create<memref::ExtractStridedMetadataOp>(
+      memref::ExtractStridedMetadataOp::create(rewriter,
           loc, getSrcMemRef(loadStoreLikeOp));
   SmallVector<OpFoldResult> srcSizes =
       extractStridedMetadataOp.getConstifiedMixedSizes();
@@ -267,12 +267,12 @@ struct LoadStoreLikeOpRewriter : public OpRewritePattern<LoadStoreLikeOp> {
     // apply them properly to the input indices.
     // Therefore the strides multipliers are simply ones.
     auto subview =
-        rewriter.create<memref::SubViewOp>(loc, /*source=*/srcMemRef,
+        memref::SubViewOp::create(rewriter, loc, /*source=*/srcMemRef,
                                            /*offsets=*/indices,
                                            /*sizes=*/sizes, /*strides=*/ones);
     // Rewrite the load/store with the subview as the base pointer.
     SmallVector<Value> zeros(loadStoreRank,
-                             rewriter.create<arith::ConstantIndexOp>(loc, 0));
+                             arith::ConstantIndexOp::create(rewriter, loc, 0));
     LoadStoreLikeOp newLoadStore = rebuildOpFromAddressAndIndices(
         rewriter, loadStoreLikeOp, subview.getResult(), zeros);
     rewriter.replaceOp(loadStoreLikeOp, newLoadStore->getResults());
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
index e9729a4766a0a..1a73cc6dd948a 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
@@ -45,7 +45,7 @@ using namespace mlir;
 static Value getValueFromOpFoldResult(OpBuilder &rewriter, Location loc,
                                       OpFoldResult in) {
   if (Attribute offsetAttr = dyn_cast<Attribute>(in)) {
-    return rewriter.create<arith::ConstantIndexOp>(
+    return arith::ConstantIndexOp::create(rewriter,
         loc, cast<IntegerAttr>(offsetAttr).getInt());
   }
   return cast<Value>(in);
@@ -65,7 +65,7 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
   }
 
   memref::ExtractStridedMetadataOp stridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(loc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, loc, source);
 
   auto typeBit = sourceType.getElementType().getIntOrFloatBitWidth();
   OpFoldResult linearizedIndices;
@@ -79,7 +79,7 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
           getAsOpFoldResult(indices));
 
   return std::make_pair(
-      rewriter.create<memref::ReinterpretCastOp>(
+      memref::ReinterpretCastOp::create(rewriter,
           loc, source,
           /* offset = */ linearizedInfo.linearizedOffset,
           /* shapes = */
@@ -116,7 +116,7 @@ template <typename T>
 static void castAllocResult(T oper, T newOper, Location loc,
                             PatternRewriter &rewriter) {
   memref::ExtractStridedMetadataOp stridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(loc, oper);
+      memref::ExtractStridedMetadataOp::create(rewriter, loc, oper);
   rewriter.replaceOpWithNewOp<memref::ReinterpretCastOp>(
       oper, cast<MemRefType>(oper.getType()), newOper,
       /*offset=*/rewriter.getIndexAttr(0),
@@ -130,62 +130,62 @@ static void replaceOp(T op, PatternRewriter &rewriter, Value flatMemref,
   Location loc = op->getLoc();
   llvm::TypeSwitch<Operation *>(op.getOperation())
       .template Case<memref::AllocOp>([&](auto oper) {
-        auto newAlloc = rewriter.create<memref::AllocOp>(
+        auto newAlloc = memref::AllocOp::create(rewriter,
             loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloc, loc, rewriter);
       })
       .template Case<memref::AllocaOp>([&](auto oper) {
-        auto newAlloca = rewriter.create<memref::AllocaOp>(
+        auto newAlloca = memref::AllocaOp::create(rewriter,
             loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloca, loc, rewriter);
       })
       .template Case<memref::LoadOp>([&](auto op) {
-        auto newLoad = rewriter.create<memref::LoadOp>(
+        auto newLoad = memref::LoadOp::create(rewriter,
             loc, op->getResultTypes(), flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<memref::StoreOp>([&](auto op) {
-        auto newStore = rewriter.create<memref::StoreOp>(
+        auto newStore = memref::StoreOp::create(rewriter,
             loc, op->getOperands().front(), flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::LoadOp>([&](auto op) {
-        auto newLoad = rewriter.create<vector::LoadOp>(
+        auto newLoad = vector::LoadOp::create(rewriter,
             loc, op->getResultTypes(), flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<vector::StoreOp>([&](auto op) {
-        auto newStore = rewriter.create<vector::StoreOp>(
+        auto newStore = vector::StoreOp::create(rewriter,
             loc, op->getOperands().front(), flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::MaskedLoadOp>([&](auto op) {
-        auto newMaskedLoad = rewriter.create<vector::MaskedLoadOp>(
+        auto newMaskedLoad = vector::MaskedLoadOp::create(rewriter,
             loc, op.getType(), flatMemref, ValueRange{offset}, op.getMask(),
             op.getPassThru());
         newMaskedLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedLoad.getResult());
       })
       .template Case<vector::MaskedStoreOp>([&](auto op) {
-        auto newMaskedStore = rewriter.create<vector::MaskedStoreOp>(
+        auto newMaskedStore = vector::MaskedStoreOp::create(rewriter,
             loc, flatMemref, ValueRange{offset}, op.getMask(),
             op.getValueToStore());
         newMaskedStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedStore);
       })
       .template Case<vector::TransferReadOp>([&](auto op) {
-        auto newTransferRead = rewriter.create<vector::TransferReadOp>(
+        auto newTransferRead = vector::TransferReadOp::create(rewriter,
             loc, op.getType(), flatMemref, ValueRange{offset}, op.getPadding());
         rewriter.replaceOp(op, newTransferRead.getResult());
       })
       .template Case<vector::TransferWriteOp>([&](auto op) {
-        auto newTransferWrite = rewriter.create<vector::TransferWriteOp>(
+        auto newTransferWrite = vector::TransferWriteOp::create(rewriter,
             loc, op.getVector(), flatMemref, ValueRange{offset});
         rewriter.replaceOp(op, newTransferWrite);
       })
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index 42c43ba8553a3..8052115610eaa 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -78,7 +78,7 @@ static LogicalResult resolveSourceIndicesExpandShape(
         llvm::map_to_vector(group, [&](int64_t d) { return destShape[d]; });
     SmallVector<Value> groupIndices =
         llvm::map_to_vector(group, [&](int64_t d) { return indices[d]; });
-    Value collapsedIndex = rewriter.create<affine::AffineLinearizeIndexOp>(
+    Value collapsedIndex = affine::AffineLinearizeIndexOp::create(rewriter,
         loc, groupIndices, groupBasis, /*disjoint=*/startsInbounds);
     sourceIndices.push_back(collapsedIndex);
   }
@@ -104,7 +104,7 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
                                   ValueRange indices,
                                   SmallVectorImpl<Value> &sourceIndices) {
   // Note: collapse_shape requires a strided memref, we can do this.
-  auto metadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+  auto metadata = memref::ExtractStridedMetadataOp::create(rewriter,
       loc, collapseShapeOp.getSrc());
   SmallVector<OpFoldResult> sourceSizes = metadata.getConstifiedMixedSizes();
   for (auto [index, group] :
@@ -119,7 +119,7 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
 
     SmallVector<OpFoldResult> basis =
         llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-    auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
         loc, index, basis, /*hasOuterBound=*/true);
     llvm::append_range(sourceIndices, delinearize.getResults());
   }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
index 21361d2e9a2d7..33be43da47679 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
@@ -53,7 +53,7 @@ FailureOr<Value> memref::buildIndependentOp(OpBuilder &b,
 
   // Create a new memref::AllocaOp.
   Value newAllocaOp =
-      b.create<AllocaOp>(loc, newSizes, allocaOp.getType().getElementType());
+      AllocaOp::create(b, loc, newSizes, allocaOp.getType().getElementType());
 
   // Create a memref::SubViewOp.
   SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
@@ -73,10 +73,10 @@ propagateSubViewOp(RewriterBase &rewriter,
   MemRefType newResultType = SubViewOp::inferRankReducedResultType(
       op.getType().getShape(), op.getSourceType(), op.getMixedOffsets(),
       op.getMixedSizes(), op.getMixedStrides());
-  Value newSubview = rewriter.create<SubViewOp>(
+  Value newSubview = SubViewOp::create(rewriter,
       op.getLoc(), newResultType, conversionOp.getOperand(0),
       op.getMixedOffsets(), op.getMixedSizes(), op.getMixedStrides());
-  auto newConversionOp = rewriter.create<UnrealizedConversionCastOp>(
+  auto newConversionOp = UnrealizedConversionCastOp::create(rewriter,
       op.getLoc(), op.getType(), newSubview);
   rewriter.replaceAllUsesWith(op.getResult(), newConversionOp->getResult(0));
   return newConversionOp;
@@ -108,7 +108,7 @@ static void replaceAndPropagateMemRefType(RewriterBase &rewriter,
   SmallVector<UnrealizedConversionCastOp> unrealizedConversions;
   for (const auto &it :
        llvm::enumerate(llvm::zip(from->getResults(), to->getResults()))) {
-    unrealizedConversions.push_back(rewriter.create<UnrealizedConversionCastOp>(
+    unrealizedConversions.push_back(UnrealizedConversionCastOp::create(rewriter,
         to->getLoc(), std::get<0>(it.value()).getType(),
         std::get<1>(it.value())));
     rewriter.replaceAllUsesWith(from->getResult(it.index()),
diff --git a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
index c475d92e0658e..a1b8821f38e7e 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
@@ -64,7 +64,7 @@ static void replaceUsesAndPropagateType(RewriterBase &rewriter,
         subviewUse.getType().getShape(), cast<MemRefType>(val.getType()),
         subviewUse.getStaticOffsets(), subviewUse.getStaticSizes(),
         subviewUse.getStaticStrides());
-    Value newSubview = rewriter.create<memref::SubViewOp>(
+    Value newSubview = memref::SubViewOp::create(rewriter,
         subviewUse->getLoc(), newType, val, subviewUse.getMixedOffsets(),
         subviewUse.getMixedSizes(), subviewUse.getMixedStrides());
 
@@ -178,7 +178,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   Location loc = allocOp->getLoc();
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(allocOp);
-  auto mbAlloc = rewriter.create<memref::AllocOp>(
+  auto mbAlloc = memref::AllocOp::create(rewriter,
       loc, mbMemRefType, ValueRange{}, allocOp->getAttrs());
   LLVM_DEBUG(DBGS() << "--multi-buffered alloc: " << mbAlloc << "\n");
 
@@ -212,7 +212,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   // Strides is [1, 1 ... 1 ].
   MemRefType dstMemref = memref::SubViewOp::inferRankReducedResultType(
       originalShape, mbMemRefType, offsets, sizes, strides);
-  Value subview = rewriter.create<memref::SubViewOp>(loc, dstMemref, mbAlloc,
+  Value subview = memref::SubViewOp::create(rewriter, loc, dstMemref, mbAlloc,
                                                      offsets, sizes, strides);
   LLVM_DEBUG(DBGS() << "--multi-buffered slice: " << subview << "\n");
 
@@ -225,7 +225,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPoint(deallocOp);
     auto newDeallocOp =
-        rewriter.create<memref::DeallocOp>(deallocOp->getLoc(), mbAlloc);
+        memref::DeallocOp::create(rewriter, deallocOp->getLoc(), mbAlloc);
     (void)newDeallocOp;
     LLVM_DEBUG(DBGS() << "----Created dealloc: " << newDeallocOp << "\n");
     rewriter.eraseOp(deallocOp);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index d6fcb8d9f0501..40dfad91b8959 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -278,7 +278,7 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
     if (!callOp)
       continue;
     Operation *newCallOp =
-        builder.create<func::CallOp>(userOp->getLoc(), callOp.getCalleeAttr(),
+        func::CallOp::create(builder, userOp->getLoc(), callOp.getCalleeAttr(),
                                      resultTypes, userOp->getOperands());
     bool replacingMemRefUsesFailed = false;
     bool returnTypeChanged = false;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
index e6b9e2f7e8213..ea7f394f48ba4 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
@@ -116,10 +116,10 @@ static LogicalResult reifyOpResultShapes(RewriterBase &rewriter,
     // Update the type.
     newRes.setType(reifiedTy);
     if (isa<RankedTensorType>(reifiedTy)) {
-      newResults.push_back(rewriter.create<tensor::CastOp>(loc, oldTy, newRes));
+      newResults.push_back(tensor::CastOp::create(rewriter, loc, oldTy, newRes));
     } else {
       assert(isa<MemRefType>(reifiedTy) && "expected a memref type");
-      newResults.push_back(rewriter.create<memref::CastOp>(loc, oldTy, newRes));
+      newResults.push_back(memref::CastOp::create(rewriter, loc, oldTy, newRes));
     }
   }
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
index 89a3895d06ba5..6a81a15f30e47 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
@@ -69,7 +69,7 @@ struct DimOfShapedTypeOpInterface : public OpRewritePattern<OpTy> {
     Location loc = dimOp->getLoc();
     rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
         dimOp, resultShape,
-        rewriter.create<arith::ConstantIndexOp>(loc, *dimIndex).getResult());
+        arith::ConstantIndexOp::create(rewriter, loc, *dimIndex).getResult());
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
index 7bf7c7b8e024c..e15bf379840b4 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
@@ -41,15 +41,15 @@ struct AssumeAlignmentOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto assumeOp = cast<AssumeAlignmentOp>(op);
-    Value ptr = builder.create<ExtractAlignedPointerAsIndexOp>(
+    Value ptr = ExtractAlignedPointerAsIndexOp::create(builder,
         loc, assumeOp.getMemref());
-    Value rest = builder.create<arith::RemUIOp>(
+    Value rest = arith::RemUIOp::create(builder,
         loc, ptr,
-        builder.create<arith::ConstantIndexOp>(loc, assumeOp.getAlignment()));
-    Value isAligned = builder.create<arith::CmpIOp>(
+        arith::ConstantIndexOp::create(builder, loc, assumeOp.getAlignment()));
+    Value isAligned = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::eq, rest,
-        builder.create<arith::ConstantIndexOp>(loc, 0));
-    builder.create<cf::AssertOp>(
+        arith::ConstantIndexOp::create(builder, loc, 0));
+    cf::AssertOp::create(builder,
         loc, isAligned,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "memref is not aligned to " +
@@ -72,12 +72,12 @@ struct CastOpInterface
 
     if (isa<UnrankedMemRefType>(srcType)) {
       // Check rank.
-      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
-          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
-      Value isSameRank = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
+      Value isSameRank = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameRank,
           RuntimeVerifiableOpInterface::generateErrorMessage(op,
                                                              "rank mismatch"));
@@ -96,8 +96,8 @@ struct CastOpInterface
         MemRefType::get(dynamicShape, resultType.getElementType(),
                         stridedLayout, resultType.getMemorySpace());
     Value helperCast =
-        builder.create<CastOp>(loc, dynStridesType, castOp.getSource());
-    auto metadataOp = builder.create<ExtractStridedMetadataOp>(loc, helperCast);
+        CastOp::create(builder, loc, dynStridesType, castOp.getSource());
+    auto metadataOp = ExtractStridedMetadataOp::create(builder, loc, helperCast);
 
     // Check dimension sizes.
     for (const auto &it : llvm::enumerate(resultType.getShape())) {
@@ -111,12 +111,12 @@ struct CastOpInterface
         continue;
 
       Value srcDimSz =
-          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+          DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameSz = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameSz = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
@@ -133,10 +133,10 @@ struct CastOpInterface
       // Static/dynamic offset -> dynamic offset does not need verification.
       Value srcOffset = metadataOp.getResult(1);
       Value resultOffsetVal =
-          builder.create<arith::ConstantIndexOp>(loc, resultOffset);
-      Value isSameOffset = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultOffset);
+      Value isSameOffset = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcOffset, resultOffsetVal);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameOffset,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset mismatch"));
@@ -151,10 +151,10 @@ struct CastOpInterface
       Value srcStride =
           metadataOp.getResult(2 + resultType.getRank() + it.index());
       Value resultStrideVal =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameStride = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameStride = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcStride, resultStrideVal);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameStride,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "stride mismatch of dim " + std::to_string(it.index())));
@@ -187,7 +187,7 @@ struct CopyOpInterface
       auto getDimSize = [&](Value memRef, MemRefType type,
                             int64_t dim) -> Value {
         return type.isDynamicDim(dim)
-                   ? builder.create<DimOp>(loc, memRef, dim).getResult()
+                   ? DimOp::create(builder, loc, memRef, dim).getResult()
                    : builder
                          .create<arith::ConstantIndexOp>(loc,
                                                          type.getDimSize(dim))
@@ -195,9 +195,9 @@ struct CopyOpInterface
       };
       Value sourceDim = getDimSize(copyOp.getSource(), rankedSourceType, i);
       Value targetDim = getDimSize(copyOp.getTarget(), rankedTargetType, i);
-      Value sameDimSize = builder.create<arith::CmpIOp>(
+      Value sameDimSize = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, sameDimSize,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size of " + std::to_string(i) +
@@ -212,9 +212,9 @@ struct DimOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto dimOp = cast<DimOp>(op);
-    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    builder.create<cf::AssertOp>(
+    Value rank = RankOp::create(builder, loc, dimOp.getSource());
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    cf::AssertOp::create(builder,
         loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
@@ -238,7 +238,7 @@ struct LoadStoreOpInterface
     }
     auto indices = loadStoreOp.getIndices();
 
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     Value assertCond;
     for (auto i : llvm::seq<int64_t>(0, rank)) {
       Value dimOp = builder.createOrFold<memref::DimOp>(loc, memref, i);
@@ -248,7 +248,7 @@ struct LoadStoreOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    builder.create<cf::AssertOp>(
+    cf::AssertOp::create(builder,
         loc, assertCond,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "out-of-bounds access"));
@@ -266,10 +266,10 @@ struct SubViewOpInterface
     // For each dimension, assert that:
     // 0 <= offset < dim_size
     // 0 <= offset + (size - 1) * stride < dim_size
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value one = arith::ConstantIndexOp::create(builder, loc, 1);
     auto metadataOp =
-        builder.create<ExtractStridedMetadataOp>(loc, subView.getSource());
+        ExtractStridedMetadataOp::create(builder, loc, subView.getSource());
     for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
       Value offset = getValueOrCreateConstantIndexOp(
           builder, loc, subView.getMixedOffsets()[i]);
@@ -282,20 +282,20 @@ struct SubViewOpInterface
       Value dimSize = metadataOp.getSizes()[i];
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
       // Verify that slice does not run out-of-bounds.
-      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOne = arith::SubIOp::create(builder, loc, size, one);
       Value sizeMinusOneTimesStride =
-          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+          arith::MulIOp::create(builder, loc, sizeMinusOne, stride);
       Value lastPos =
-          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+          arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "subview runs out-of-bounds along dimension " +
@@ -316,7 +316,7 @@ struct ExpandShapeOpInterface
     for (const auto &it :
          llvm::enumerate(expandShapeOp.getReassociationIndices())) {
       Value srcDimSz =
-          builder.create<DimOp>(loc, expandShapeOp.getSrc(), it.index());
+          DimOp::create(builder, loc, expandShapeOp.getSrc(), it.index());
       int64_t groupSz = 1;
       bool foundDynamicDim = false;
       for (int64_t resultDim : it.value()) {
@@ -331,14 +331,14 @@ struct ExpandShapeOpInterface
         groupSz *= expandShapeOp.getResultType().getDimSize(resultDim);
       }
       Value staticResultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, groupSz);
+          arith::ConstantIndexOp::create(builder, loc, groupSz);
       // staticResultDimSz must divide srcDimSz evenly.
       Value mod =
-          builder.create<arith::RemSIOp>(loc, srcDimSz, staticResultDimSz);
-      Value isModZero = builder.create<arith::CmpIOp>(
+          arith::RemSIOp::create(builder, loc, srcDimSz, staticResultDimSz);
+      Value isModZero = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, mod,
-          builder.create<arith::ConstantIndexOp>(loc, 0));
-      builder.create<cf::AssertOp>(
+          arith::ConstantIndexOp::create(builder, loc, 0));
+      cf::AssertOp::create(builder,
           loc, isModZero,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "static result dims in reassoc group do not "
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index 5a36984c9013c..5fd0d3c8a7f1b 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -92,7 +92,7 @@ SmallVector<Value> mlir::mesh::getMixedAsValues(OpBuilder b,
       values.emplace_back(*(dyn++));
     } else {
       TypedAttr val = type == i64 ? b.getI64IntegerAttr(s) : b.getIndexAttr(s);
-      values.emplace_back(b.create<arith::ConstantOp>(loc, type, val));
+      values.emplace_back(arith::ConstantOp::create(b, loc, type, val));
     }
   }
   return values;
@@ -317,9 +317,9 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
 
   if (!newShardOp) {
     auto shardingOp =
-        builder.create<ShardingOp>(operandValue.getLoc(), sharding);
+        ShardingOp::create(builder, operandValue.getLoc(), sharding);
     newShardOp =
-        builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+        ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                                 /*annotate_for_users*/ false);
   }
   operandValue.replaceUsesWithIf(
@@ -331,7 +331,7 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
     return;
   }
 
-  auto newShardOp2 = builder.create<ShardOp>(operandValue.getLoc(), newShardOp,
+  auto newShardOp2 = ShardOp::create(builder, operandValue.getLoc(), newShardOp,
                                              newShardOp.getSharding(),
                                              /*annotate_for_users*/ true);
   newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
@@ -379,9 +379,9 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
 
   builder.setInsertionPoint(operandOp);
   auto shardingOp =
-      builder.create<ShardingOp>(operand.get().getLoc(), sharding);
+      ShardingOp::create(builder, operand.get().getLoc(), sharding);
   auto newShardOp =
-      builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+      ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                               /*annotate_for_users*/ true);
   IRRewriter rewriter(builder);
   rewriter.replaceUsesWithIf(
@@ -396,7 +396,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
 
   builder.setInsertionPoint(newShardOp);
   auto newPreceedingShardOp =
-      builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+      ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                               /*annotate_for_users*/ false);
   rewriter.replaceUsesWithIf(
       newShardOp.getSrc(), newPreceedingShardOp, [&newShardOp](OpOperand &use) {
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
index 4c14b1c0ea4bb..b5b3a4d21f2af 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
@@ -92,7 +92,7 @@ struct MeshShapeFolder
         newShapeOpMeshAxes.push_back(opMeshAxes[i]);
       } else {
         // Fold static mesh axes.
-        newResults[i] = builder.create<arith::ConstantOp>(
+        newResults[i] = arith::ConstantOp::create(builder,
             builder.getIndexAttr(meshAxisSize));
       }
     }
@@ -100,7 +100,7 @@ struct MeshShapeFolder
     // Leave only the dynamic mesh axes to be queried.
     if (!newShapeOpMeshAxes.empty()) {
       MeshShapeOp newShapeOp =
-          builder.create<MeshShapeOp>(mesh.getSymName(), newShapeOpMeshAxes);
+          MeshShapeOp::create(builder, mesh.getSymName(), newShapeOpMeshAxes);
       for (size_t i = 0; i < newShapeOp->getResults().size(); ++i) {
         newResults[newToOldResultsIndexMap[i]] = newShapeOp->getResults()[i];
       }
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index a0284b093da94..6476e8b689baa 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -267,7 +267,7 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
       targetShardingInUnsplitLastAxis(ctx, sourceSharding, splitTensorAxis);
   ShapedType allGatherResultShape = allGatherResultShapeInUnsplitLastAxis(
       sourceShard.getType(), mesh.getShape()[splitMeshAxis], splitTensorAxis);
-  Value allGatherResult = builder.create<AllGatherOp>(
+  Value allGatherResult = AllGatherOp::create(builder,
       RankedTensorType::get(allGatherResultShape.getShape(),
                             allGatherResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({splitMeshAxis}), sourceShard,
@@ -275,7 +275,7 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      builder.create<tensor::CastOp>(targetShape, allGatherResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allGatherResult).getResult());
   return {targetShard, targetSharding};
 }
 
@@ -400,7 +400,7 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType allToAllResultShape = allToAllResultShapeInMoveLastAxis(
       sourceShard.getType(), mesh.getShape()[meshAxis], sourceTensorAxis,
       targetTensorAxis);
-  Value allToAllResult = builder.create<AllToAllOp>(
+  Value allToAllResult = AllToAllOp::create(builder,
       RankedTensorType::get(allToAllResultShape.getShape(),
                             allToAllResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({meshAxis}), sourceShard,
@@ -408,7 +408,7 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      builder.create<tensor::CastOp>(targetShape, allToAllResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allToAllResult).getResult());
   return {targetShard, targetSharding};
 }
 
@@ -479,13 +479,13 @@ tryUpdateHaloInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
 
   // Extract core from source and copy into destination core.
   auto noVals = ValueRange{};
-  auto initVal = builder.create<tensor::EmptyOp>(
+  auto initVal = tensor::EmptyOp::create(builder,
       sourceShard.getLoc(), outShape, sourceShard.getType().getElementType());
-  auto core = builder.create<tensor::ExtractSliceOp>(
+  auto core = tensor::ExtractSliceOp::create(builder,
       sourceShard.getLoc(),
       RankedTensorType::get(coreShape, sourceShard.getType().getElementType()),
       sourceShard, noVals, noVals, noVals, srcCoreOffs, coreShape, strides);
-  auto initOprnd = builder.create<tensor::InsertSliceOp>(
+  auto initOprnd = tensor::InsertSliceOp::create(builder,
       sourceShard.getLoc(), core, initVal, noVals, noVals, noVals, tgtCoreOffs,
       coreShape, strides);
 
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
index f08ef75d8a004..c85394924608d 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
@@ -49,10 +49,10 @@ struct ProcessMultiIndexOpLowering
 
     ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     builder.setInsertionPointAfter(op.getOperation());
-    Value linearIndex = builder.create<ProcessLinearIndexOp>(mesh);
-    ValueRange meshShape = builder.create<MeshShapeOp>(mesh).getResults();
+    Value linearIndex = ProcessLinearIndexOp::create(builder, mesh);
+    ValueRange meshShape = MeshShapeOp::create(builder, mesh).getResults();
     SmallVector<Value> completeMultiIndex =
-        builder.create<affine::AffineDelinearizeIndexOp>(linearIndex, meshShape)
+        affine::AffineDelinearizeIndexOp::create(builder, linearIndex, meshShape)
             .getMultiIndex();
     SmallVector<Value> multiIndex;
     ArrayRef<MeshAxis> opMeshAxes = op.getAxes();
@@ -101,32 +101,32 @@ struct AllSliceOpLowering
     ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     builder.setInsertionPointAfter(op.getOperation());
 
-    Value zero = builder.create<arith::ConstantOp>(builder.getIndexAttr(0));
+    Value zero = arith::ConstantOp::create(builder, builder.getIndexAttr(0));
 
     Operation::result_range processInGroupMultiIndex =
-        builder.create<ProcessMultiIndexOp>(mesh.getSymName(), op.getMeshAxes())
+        ProcessMultiIndexOp::create(builder, mesh.getSymName(), op.getMeshAxes())
             .getResults();
 
     Operation::result_range processGroupShape =
-        builder.create<MeshShapeOp>(mesh.getSymName(), op.getMeshAxes())
+        MeshShapeOp::create(builder, mesh.getSymName(), op.getMeshAxes())
             .getResult();
     Value processGroupSize =
         createCollectiveProcessGroupSize(mesh, op.getMeshAxes(), builder);
 
     int64_t sliceAxis = op.getSliceAxis().getSExtValue();
     Value operandSliceAxisSize =
-        builder.create<tensor::DimOp>(op.getOperand(), sliceAxis);
+        tensor::DimOp::create(builder, op.getOperand(), sliceAxis);
     Value operandSliceAxisSizeModProcessGroupSize =
-        builder.create<arith::RemUIOp>(operandSliceAxisSize, processGroupSize);
-    Value isTargetShapeExactlyDivisible = builder.create<arith::CmpIOp>(
+        arith::RemUIOp::create(builder, operandSliceAxisSize, processGroupSize);
+    Value isTargetShapeExactlyDivisible = arith::CmpIOp::create(builder,
         arith::CmpIPredicate::eq, operandSliceAxisSizeModProcessGroupSize,
         zero);
-    builder.create<cf::AssertOp>(isTargetShapeExactlyDivisible,
+    cf::AssertOp::create(builder, isTargetShapeExactlyDivisible,
                                  "Slicing a tensor with axis size that is "
                                  "not exactly divisible by the "
                                  "mesh process group size is not supported.");
     Value resultSliceAxisSize =
-        builder.create<arith::DivUIOp>(operandSliceAxisSize, processGroupSize);
+        arith::DivUIOp::create(builder, operandSliceAxisSize, processGroupSize);
     OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
         llvm::to_vector_of<OpFoldResult>(processInGroupMultiIndex),
         llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
@@ -139,7 +139,7 @@ struct AllSliceOpLowering
       if (i == sliceAxis) {
         sizes.emplace_back(resultSliceAxisSize);
       } else {
-        Value dimSize = builder.create<tensor::DimOp>(op.getOperand(), i);
+        Value dimSize = tensor::DimOp::create(builder, op.getOperand(), i);
         sizes.emplace_back(dimSize);
       }
     }
@@ -152,10 +152,10 @@ struct AllSliceOpLowering
                  resultSliceAxisSize);
     SmallVector<OpFoldResult> strides(
         operandType.getRank(), getAsIndexOpFoldResult(builder.getContext(), 1));
-    Value slice = builder.create<tensor::ExtractSliceOp>(
+    Value slice = tensor::ExtractSliceOp::create(builder,
         op.getOperand(), offsets, sizes, strides);
     Value newResult =
-        builder.create<tensor::CastOp>(op.getResult().getType(), slice);
+        tensor::CastOp::create(builder, op.getResult().getType(), slice);
     rewriter.replaceAllUsesWith(op.getResult(), newResult);
 
     return success();
@@ -201,7 +201,7 @@ TypedValue<IndexType>
 createCollectiveProcessGroupSize(MeshOp mesh, ArrayRef<MeshAxis> axes,
                                  ImplicitLocOpBuilder &builder) {
   Operation::result_range meshShape =
-      builder.create<mesh::MeshShapeOp>(mesh, axes).getResults();
+      mesh::MeshShapeOp::create(builder, mesh, axes).getResults();
   return cast<TypedValue<IndexType>>(arith::createProduct(
       builder, builder.getLoc(), llvm::to_vector_of<Value>(meshShape),
       builder.getIndexType()));
@@ -212,13 +212,13 @@ createProcessLinearIndex(StringRef mesh, ValueRange processInGroupMultiIndex,
                          ArrayRef<MeshAxis> meshAxes,
                          ImplicitLocOpBuilder &builder) {
   Operation::result_range processGroupShape =
-      builder.create<MeshShapeOp>(mesh, meshAxes).getResult();
+      MeshShapeOp::create(builder, mesh, meshAxes).getResult();
   OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
       llvm::to_vector_of<OpFoldResult>(processInGroupMultiIndex),
       llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
   auto res = dyn_cast<Value>(processInGroupLinearIndex);
   if (!res)
-    res = builder.create<arith::ConstantIndexOp>(
+    res = arith::ConstantIndexOp::create(builder,
         cast<IntegerAttr>(cast<Attribute>(processInGroupLinearIndex)).getInt());
   return cast<TypedValue<IndexType>>(res);
 }
@@ -227,7 +227,7 @@ TypedValue<IndexType> createProcessLinearIndex(StringRef mesh,
                                                ArrayRef<MeshAxis> meshAxes,
                                                ImplicitLocOpBuilder &builder) {
   return createProcessLinearIndex(
-      mesh, builder.create<ProcessMultiIndexOp>(mesh, meshAxes).getResults(),
+      mesh, ProcessMultiIndexOp::create(builder, mesh, meshAxes).getResults(),
       meshAxes, builder);
 }
 } // namespace mlir::mesh
diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
index d2c94b124cdfb..85317a68c9f00 100644
--- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
+++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
@@ -334,13 +334,13 @@ static Operation *replaceOpWithPredicatedOp(RewriterBase &rewriter,
   //
   Location loc = asyncCopyOp->getLoc();
   Value dstElements =
-      rewriter.create<arith::ConstantOp>(loc, asyncCopyOp.getDstElementsAttr());
+      arith::ConstantOp::create(rewriter, loc, asyncCopyOp.getDstElementsAttr());
   Value originalSrcElement =
       asyncCopyOp.getSrcElements() ? asyncCopyOp.getSrcElements() : dstElements;
-  Value c0Index = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto srcElements = rewriter.create<arith::SelectOp>(
+  Value c0Index = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto srcElements = arith::SelectOp::create(rewriter,
       loc, predicate, originalSrcElement, c0Index);
-  auto asyncCopyZeroFillOp = rewriter.create<nvgpu::DeviceAsyncCopyOp>(
+  auto asyncCopyZeroFillOp = nvgpu::DeviceAsyncCopyOp::create(rewriter,
       loc, nvgpu::DeviceAsyncTokenType::get(asyncCopyOp.getContext()),
       asyncCopyOp.getDst(), asyncCopyOp.getDstIndices(), asyncCopyOp.getSrc(),
       asyncCopyOp.getSrcIndices(), asyncCopyOp.getDstElements(), srcElements,
@@ -675,7 +675,7 @@ MmaSyncBuilder::buildMemRefLoads(OpBuilder &b, Location loc,
   for (auto indexing : indexings) {
     Value row = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.row()));
     Value col = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.col()));
-    auto load = b.create<memref::LoadOp>(loc, memref, ValueRange{row, col});
+    auto load = memref::LoadOp::create(b, loc, memref, ValueRange{row, col});
     res.push_back(load);
   }
   return res;
@@ -688,7 +688,7 @@ Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
 
   Type elementType = getElementTypeOrSelf(memref.getType());
   auto vt = VectorType::get(vectorShape, elementType);
-  Value res = b.create<vector::SplatOp>(loc, vt, loads[0]);
+  Value res = vector::SplatOp::create(b, loc, vt, loads[0]);
   foreachIndividualVectorElement(
       res,
       /*applyFn=*/
@@ -697,7 +697,7 @@ Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
       },
       /*reduceFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
-        res = b.create<vector::InsertOp>(loc, v, res, indices);
+        res = vector::InsertOp::create(b, loc, v, res, indices);
       });
 
   return res;
@@ -715,7 +715,7 @@ SmallVector<Operation *> MmaSyncBuilder::buildMemRefStores(
     Value row = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.row()));
     Value col = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.col()));
     Operation *store =
-        b.create<memref::StoreOp>(loc, val, memref, ValueRange{row, col});
+        memref::StoreOp::create(b, loc, val, memref, ValueRange{row, col});
     res.push_back(store);
   }
   return res;
@@ -730,7 +730,7 @@ SmallVector<Operation *> MmaSyncBuilder::buildMmaSyncMemRefStoreOperand(
       vectorToStore,
       /*applyFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
-        return b.create<vector::ExtractOp>(loc, vectorToStore, indices);
+        return vector::ExtractOp::create(b, loc, vectorToStore, indices);
       },
       /*reduceFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
@@ -810,7 +810,7 @@ FailureOr<Operation *> MmaSyncBuilder::buildMmaSync(LinalgOp linalgOp) {
                                             rhsIndexFn, rhsShape);
   Value res = buildMmaSyncMemRefLoadOperand(b, loc, laneId, resMemRef,
                                             resIndexFn, resShape);
-  res = b.create<nvgpu::MmaSyncOp>(loc, lhs, rhs, res, info.mmaShape,
+  res = nvgpu::MmaSyncOp::create(b, loc, lhs, rhs, res, info.mmaShape,
                                    info.tf32Enabled);
   buildMmaSyncMemRefStoreOperand(b, loc, res, laneId, resMemRef, resIndexFn,
                                  resShape);
@@ -832,7 +832,7 @@ DiagnosedSilenceableFailure transform::RewriteMatmulAsMmaSyncOp::applyToOne(
     }
     Location loc = linalgOp.getLoc();
     // TODO: more robust computation of laneId, for now assume a single warp.
-    Value laneId = rewriter.create<gpu::ThreadIdOp>(
+    Value laneId = gpu::ThreadIdOp::create(rewriter,
         loc, rewriter.getIndexType(), gpu::Dimension::x);
     if (succeeded(MmaSyncBuilder(rewriter, loc, laneId).buildMmaSync(linalgOp)))
       fail = false;
@@ -897,12 +897,12 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
     ArrayRef<TypedValue<MemRefType>> sharedMemBuffers,
     TypedValue<nvgpu::MBarrierGroupType> barrier) {
   SmallVector<Operation *> loadOps;
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Value tidx = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Value tidx = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
   Value cond =
-      rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, tidx, zero);
+      arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, tidx, zero);
   // clang-format off
-  rewriter.create<scf::IfOp>(
+  scf::IfOp::create(rewriter,
     /*location=*/loc,
     /*conditional=*/cond,
     /*thenBuilder=*/
@@ -917,14 +917,14 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
       // TODO: Note that cutlass predeclares the barrier arrive tx before the tma.async.load.
       // This may or may not have perf implications.
       buildBarrierArriveTx(barrier, sizes);
-      rewriter.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(rewriter, loc);
     },
     /*elseBuilder=*/
     [&](OpBuilder &lb, Location loc) {
       // TODO: is this for no-thread divergence?
       // Should we just yield the size and hoist?
       buildBarrierArriveTx(barrier, getAsIndexOpFoldResult(rewriter.getContext(), 0));
-      rewriter.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(rewriter, loc);
     });
   // clang-format on
   return loadOps;
@@ -939,14 +939,14 @@ static Attribute getSharedAddressSpaceAttribute(OpBuilder &b) {
 TypedValue<nvgpu::MBarrierGroupType>
 HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) {
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value barrier = rewriter.create<nvgpu::MBarrierCreateOp>(
+  Value barrier = nvgpu::MBarrierCreateOp::create(rewriter,
       loc,
       nvgpu::MBarrierGroupType::get(rewriter.getContext(), sharedMemorySpace));
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierInitOp>(
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierInitOp::create(rewriter,
       loc, barrier, getValueOrCreateConstantIndexOp(rewriter, loc, numThreads),
       zero, Value());
-  rewriter.create<gpu::BarrierOp>(loc);
+  gpu::BarrierOp::create(rewriter, loc);
   return cast<TypedValue<nvgpu::MBarrierGroupType>>(barrier);
 }
 
@@ -955,7 +955,7 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
                                            gpu::LaunchOp launchOp) {
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(launchOp);
-  Value unrankedMemRef = rewriter.create<memref::CastOp>(
+  Value unrankedMemRef = memref::CastOp::create(rewriter,
       loc,
       UnrankedMemRefType::get(memref.getType().getElementType(),
                               memref.getType().getMemorySpace()),
@@ -966,7 +966,7 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
       getValueOrCreateConstantIndexOp(rewriter, loc, mixedSizes);
 
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value desc = rewriter.create<nvgpu::TmaCreateDescriptorOp>(
+  Value desc = nvgpu::TmaCreateDescriptorOp::create(rewriter,
       loc,
       nvgpu::TensorMapDescriptorType::get(
           rewriter.getContext(),
@@ -985,8 +985,8 @@ OpFoldResult HopperBuilder::buildTmaAsyncLoad(
     TypedValue<nvgpu::MBarrierGroupType> barrier,
     SmallVectorImpl<Operation *> &loadOps) {
   MLIRContext *ctx = rewriter.getContext();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Operation *loadOp = rewriter.create<nvgpu::TmaAsyncLoadOp>(
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Operation *loadOp = nvgpu::TmaAsyncLoadOp::create(rewriter,
       loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero,
       Value(), Value());
   loadOps.push_back(loadOp);
@@ -1012,22 +1012,22 @@ void HopperBuilder::buildBarrierArriveTx(
   OpFoldResult size =
       affine::makeComposedFoldedAffineApply(rewriter, loc, sumExpr, mixedSizes);
   Value sizeVal = getValueOrCreateConstantIndexOp(rewriter, loc, size);
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierArriveExpectTxOp>(loc, barrier, sizeVal, zero,
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierArriveExpectTxOp::create(rewriter, loc, barrier, sizeVal, zero,
                                                    Value());
 }
 
 void HopperBuilder::buildTryWaitParity(
     TypedValue<nvgpu::MBarrierGroupType> barrier) {
   Type i1 = rewriter.getI1Type();
-  Value parity = rewriter.create<LLVM::ConstantOp>(loc, i1, 0);
+  Value parity = LLVM::ConstantOp::create(rewriter, loc, i1, 0);
   // 10M is an arbitrary, not too small or too big number to specify the number
   // of ticks before retry.
   // TODO: hoist this in a default dialect constant.
   Value ticksBeforeRetry =
-      rewriter.create<arith::ConstantIndexOp>(loc, 10000000);
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierTryWaitParityOp>(loc, barrier, parity,
+      arith::ConstantIndexOp::create(rewriter, loc, 10000000);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierTryWaitParityOp::create(rewriter, loc, barrier, parity,
                                                   ticksBeforeRetry, zero);
 }
 
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
index 10bc1993ffd96..6a167b1cb157b 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
@@ -110,17 +110,17 @@ static Value buildNumReadElements(OpBuilder &b, Location loc,
   for (auto [pos, sz] : llvm::zip(transferMask->extractPosition,
                                   transferMask->createMaskOp->getOperands())) {
     Value cmp =
-        b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
-                                b.create<arith::ConstantIndexOp>(loc, pos), sz);
+        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt,
+                                arith::ConstantIndexOp::create(b, loc, pos), sz);
     if (!cond) {
       cond = cmp;
       continue;
     }
-    cond = b.create<arith::AndIOp>(loc, cmp, cond);
+    cond = arith::AndIOp::create(b, loc, cmp, cond);
   }
-  return b.create<arith::SelectOp>(
+  return arith::SelectOp::create(b,
       loc, cond, transferMask->createMaskOp->getOperands().back(),
-      b.create<arith::ConstantIndexOp>(loc, 0));
+      arith::ConstantIndexOp::create(b, loc, 0));
 }
 
 /// Return "true" if the conversion to async copy is supported by "async copy".
@@ -252,7 +252,7 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
       int64_t sizeInBytes =
           (dstMemref.getElementTypeBitWidth() * numElements) / 8;
       // bypass_l1 only possible with 16 byte transfer.
-      Value token = rewriter.create<nvgpu::DeviceAsyncCopyOp>(
+      Value token = nvgpu::DeviceAsyncCopyOp::create(rewriter,
           writeOp->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
           /*dst=*/storeBase, /*dstIndices=*/nvgpu::getIndices(writeOp),
           /*src=*/loadBase,
@@ -265,10 +265,10 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
     }
 
     // Create the group and wait for it right after.
-    Value groupToken = rewriter.create<nvgpu::DeviceAsyncCreateGroupOp>(
+    Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(rewriter,
         op->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
         tokens);
-    rewriter.create<nvgpu::DeviceAsyncWaitOp>(op->getLoc(), groupToken,
+    nvgpu::DeviceAsyncWaitOp::create(rewriter, op->getLoc(), groupToken,
                                               nullptr);
     // Clean up old stores.
     for (Operation *writeOp : group)
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
index 72f7296a865f8..3c6b189606c6f 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
@@ -75,27 +75,27 @@ static Value permuteVectorOffset(OpBuilder &b, Location loc,
   int64_t mask = (1LL << (m - n)) - 1;
   if (permuteEveryN > 1)
     mask = mask << llvm::Log2_64(permuteEveryN);
-  Value srcBits = b.create<arith::ConstantIndexOp>(loc, mask);
-  srcBits = b.create<arith::AndIOp>(loc, src, srcBits);
+  Value srcBits = arith::ConstantIndexOp::create(b, loc, mask);
+  srcBits = arith::AndIOp::create(b, loc, src, srcBits);
 
   // Use the src bits to permute the target bits b[N:M] containing the
   // vector offset.
   if (permuteEveryN > 1) {
     int64_t shlBits = n - llvm::Log2_64(permuteEveryN);
     if (shlBits > 0) {
-      Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, shlBits);
+      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, shlBits);
       srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
     } else if (shlBits < 0) {
-      Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, -1 * shlBits);
+      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, -1 * shlBits);
       srcBits = b.createOrFold<arith::ShRUIOp>(loc, srcBits, finalShiftVal);
     }
   } else {
-    Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, n);
+    Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, n);
     srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
   }
 
   Value permutedVectorIdx =
-      b.create<arith::XOrIOp>(loc, indices[tgtDim], srcBits);
+      arith::XOrIOp::create(b, loc, indices[tgtDim], srcBits);
   return permutedVectorIdx;
 }
 
diff --git a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
index 793db73575b4f..a378b09c27c41 100644
--- a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
@@ -72,7 +72,7 @@ Value getScalarOrTensorConstant(OpBuilder &builder, Location loc, Value scalar,
 
   // Create tensor splat
   auto tensorConstant =
-      builder.create<tensor::SplatOp>(loc, scalar, referenceShape);
+      tensor::SplatOp::create(builder, loc, scalar, referenceShape);
   return tensorConstant;
 }
 
@@ -94,21 +94,21 @@ std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
   // Get unranked input shape and total size
   auto *context = builder.getContext();
   auto shapeType = shape::getExtentTensorType(context);
-  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
-  Value inputSize = builder.create<shape::NumElementsOp>(
+  auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
+  Value inputSize = shape::NumElementsOp::create(builder,
       loc, builder.getIndexType(), inputShape);
 
   // Turn input size into 1D tensor
   auto flatShapeType = shape::getExtentTensorType(context, 1);
   auto flatInputShape =
-      builder.create<tensor::FromElementsOp>(loc, flatShapeType, inputSize);
+      tensor::FromElementsOp::create(builder, loc, flatShapeType, inputSize);
 
   // Reshape input tensor into 1D
   auto inputType = cast<UnrankedTensorType>(input.getType());
   auto elementType = inputType.getElementType();
   auto flatInputType =
       RankedTensorType::get({ShapedType::kDynamic}, elementType);
-  auto flatInput = builder.create<tensor::ReshapeOp>(loc, flatInputType, input,
+  auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
                                                      flatInputShape);
   return std::make_pair(flatInput, inputShape);
 }
@@ -142,30 +142,30 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto *context = builder.getContext();
   auto indexType = builder.getIndexType();
   auto shapeType = shape::getExtentTensorType(context);
-  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
+  auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
 
   // Get shape and sizes on left and right of axis
-  auto axisValue = builder.create<arith::ConstantIndexOp>(loc, axis);
-  auto axisNextValue = builder.create<arith::ConstantIndexOp>(loc, axis + 1);
+  auto axisValue = arith::ConstantIndexOp::create(builder, loc, axis);
+  auto axisNextValue = arith::ConstantIndexOp::create(builder, loc, axis + 1);
   auto shapeLeft =
       builder
           .create<shape::SplitAtOp>(loc, TypeRange{shapeType, shapeType},
                                     inputShape, axisValue)
           .getResult(0);
   auto sizeLeft =
-      builder.create<shape::NumElementsOp>(loc, indexType, shapeLeft);
+      shape::NumElementsOp::create(builder, loc, indexType, shapeLeft);
   auto shapeRight =
       builder
           .create<shape::SplitAtOp>(loc, TypeRange{shapeType, shapeType},
                                     inputShape, axisNextValue)
           .getResult(1);
   auto sizeRight =
-      builder.create<shape::NumElementsOp>(loc, indexType, shapeRight);
+      shape::NumElementsOp::create(builder, loc, indexType, shapeRight);
 
   // Compute flat input shape as a 3-element 1D tensor
-  auto axisSizeValue = builder.create<arith::ConstantIndexOp>(loc, axisSize);
+  auto axisSizeValue = arith::ConstantIndexOp::create(builder, loc, axisSize);
   auto flatShapeType = shape::getExtentTensorType(context, 3);
-  auto flatInputShape = builder.create<tensor::FromElementsOp>(
+  auto flatInputShape = tensor::FromElementsOp::create(builder,
       loc, flatShapeType, ValueRange{sizeLeft, axisSizeValue, sizeRight});
 
   // Reshape input to 3D tensor
@@ -173,7 +173,7 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto elementType = inputType.getElementType();
   auto flatInputType = RankedTensorType::get(
       {ShapedType::kDynamic, axisSize, ShapedType::kDynamic}, elementType);
-  auto flatInput = builder.create<tensor::ReshapeOp>(loc, flatInputType, input,
+  auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
                                                      flatInputShape);
 
   return std::make_pair(flatInput, inputShape);
@@ -192,7 +192,7 @@ Value restoreUnrankedTensorShape(OpBuilder &builder, Location loc, Value input,
   auto inputType = cast<RankedTensorType>(input.getType());
   auto elementType = inputType.getElementType();
   auto unrankedType = UnrankedTensorType::get(elementType);
-  return builder.create<tensor::ReshapeOp>(loc, unrankedType, input,
+  return tensor::ReshapeOp::create(builder, loc, unrankedType, input,
                                            inputShape);
 }
 
@@ -215,7 +215,7 @@ Value materializePerChannelScales(OpBuilder &builder, Location loc,
   auto tensorType =
       RankedTensorType::get({(int64_t)scales.size()}, expressedType);
   auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, scalesAttr);
 }
 
 // Create a tensor constant containing all zero points in a per-channel
@@ -239,7 +239,7 @@ Value materializePerChannelZeroPoints(
   auto tensorType =
       RankedTensorType::get({(int64_t)zeroPoints.size()}, storageType);
   auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, zeroPointsAttr);
 }
 
 // Create a tensor constant containing all scales in a sub-channel quantized
@@ -263,7 +263,7 @@ Value materializeSubChannelScales(
   auto tensorType =
       RankedTensorType::get(scales.getType().getShape(), expressedType);
   auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, scalesAttr);
 }
 
 // Create a tensor constant containing all zero points in a sub-channel
@@ -287,7 +287,7 @@ Value materializeSubChannelZeroPoints(
   auto tensorType =
       RankedTensorType::get(zeroPoints.getType().getShape(), storageType);
   auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, zeroPointsAttr);
 }
 
 // Clamp the given scalar or tensor input using the storage bounds encoded in
@@ -314,9 +314,9 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
   // Materialize bounds
   auto inputType = input.getType();
   auto storageType = quantizedType.getStorageType();
-  auto storageMinScalar = builder.create<arith::ConstantIntOp>(
+  auto storageMinScalar = arith::ConstantIntOp::create(builder,
       loc, storageType, quantizedType.getStorageTypeMin());
-  auto storageMaxScalar = builder.create<arith::ConstantIntOp>(
+  auto storageMaxScalar = arith::ConstantIntOp::create(builder,
       loc, storageType, quantizedType.getStorageTypeMax());
   auto storageMin = getScalarOrTensorConstant(builder, loc, storageMinScalar,
                                               inputType, inputShape);
@@ -325,11 +325,11 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
 
   // Clamp
   if (quantizedType.isSigned()) {
-    input = builder.create<arith::MaxSIOp>(loc, input, storageMin);
-    input = builder.create<arith::MinSIOp>(loc, input, storageMax);
+    input = arith::MaxSIOp::create(builder, loc, input, storageMin);
+    input = arith::MinSIOp::create(builder, loc, input, storageMax);
   } else {
-    input = builder.create<arith::MaxUIOp>(loc, input, storageMin);
-    input = builder.create<arith::MinUIOp>(loc, input, storageMax);
+    input = arith::MaxUIOp::create(builder, loc, input, storageMin);
+    input = arith::MinUIOp::create(builder, loc, input, storageMax);
   }
   return input;
 }
@@ -338,16 +338,16 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
 Value convertFloatToInteger(OpBuilder &builder, Location loc, Value input,
                             Type resultType, bool isSigned) {
   if (isSigned)
-    return builder.create<arith::FPToSIOp>(loc, resultType, input);
-  return builder.create<arith::FPToUIOp>(loc, resultType, input);
+    return arith::FPToSIOp::create(builder, loc, resultType, input);
+  return arith::FPToUIOp::create(builder, loc, resultType, input);
 }
 
 // Emit op 'arith.sitofp' or 'arith.uitofp'.
 Value convertIntegerToFloat(OpBuilder &builder, Location loc, Value input,
                             Type resultType, bool isSigned) {
   if (isSigned)
-    return builder.create<arith::SIToFPOp>(loc, resultType, input);
-  return builder.create<arith::UIToFPOp>(loc, resultType, input);
+    return arith::SIToFPOp::create(builder, loc, resultType, input);
+  return arith::UIToFPOp::create(builder, loc, resultType, input);
 }
 
 // Quantize a scalar or ranked tensor value. The stored value is clamped using
@@ -362,7 +362,7 @@ Value quantizeValue(OpBuilder &builder, Location loc, Value input,
   scale = getScalarOrTensorConstant(builder, loc, scale, inputType, inputShape);
 
   // Scale input
-  auto scaledValue = builder.create<arith::DivFOp>(loc, input, scale);
+  auto scaledValue = arith::DivFOp::create(builder, loc, input, scale);
 
   // Skip unnecessary computations if no zero point is given
   Value storedValueFloat = scaledValue;
@@ -377,7 +377,7 @@ Value quantizeValue(OpBuilder &builder, Location loc, Value input,
 
     // Add zero point to stored value
     storedValueFloat =
-        builder.create<arith::AddFOp>(loc, scaledValue, zeroPoint);
+        arith::AddFOp::create(builder, loc, scaledValue, zeroPoint);
   }
 
   // Convert stored value to storage type
@@ -418,11 +418,11 @@ Value dequantizeValue(OpBuilder &builder, Location loc, Value input,
                                       quantizedType.isSigned());
 
     // Subtract zero point to stored value
-    result = builder.create<arith::SubFOp>(loc, result, zeroPoint);
+    result = arith::SubFOp::create(builder, loc, result, zeroPoint);
   }
 
   // Multiply by scale
-  result = builder.create<arith::MulFOp>(loc, result, scale);
+  result = arith::MulFOp::create(builder, loc, result, scale);
   return result;
 }
 
@@ -477,11 +477,11 @@ Value convertPerLayerRanked(OpBuilder &builder, Location loc, Operation *op,
   auto storageType = quantizedType.getStorageType();
   auto scaleAttr =
       builder.getFloatAttr(expressedType, quantizedType.getScale());
-  auto scale = builder.create<arith::ConstantOp>(loc, expressedType, scaleAttr);
+  auto scale = arith::ConstantOp::create(builder, loc, expressedType, scaleAttr);
   auto zeroPointAttr =
       builder.getIntegerAttr(storageType, quantizedType.getZeroPoint());
   auto zeroPoint =
-      builder.create<arith::ConstantOp>(loc, storageType, zeroPointAttr);
+      arith::ConstantOp::create(builder, loc, storageType, zeroPointAttr);
 
   auto inputShape = getScalarOrTensorShape(builder, loc, input);
   return convertRanked(builder, loc, op, input, inputShape, scale, zeroPoint,
@@ -546,7 +546,7 @@ Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
                          ? quantizedType.getStorageType()
                          : quantizedType.getExpressedType();
   auto initShape = tensor::getMixedSizes(builder, loc, input);
-  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+  Value init = tensor::EmptyOp::create(builder, loc, initShape, elementType);
 
   SmallVector<utils::IteratorType> iteratorTypes(inputRank,
                                                  utils::IteratorType::parallel);
@@ -572,7 +572,7 @@ Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
                               convertRanked(builder, loc, op, input, {}, scale,
                                             zeroPoint, quantizedType);
 
-                          builder.create<linalg::YieldOp>(loc, result);
+                          linalg::YieldOp::create(builder, loc, result);
                         })
                     .getResult(0);
 
@@ -642,7 +642,7 @@ Value convertSubChannel(OpBuilder &builder, Location loc, Operation *op,
                          ? quantizedType.getStorageType()
                          : quantizedType.getExpressedType();
   auto initShape = tensor::getMixedSizes(builder, loc, input);
-  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+  Value init = tensor::EmptyOp::create(builder, loc, initShape, elementType);
 
   SmallVector<utils::IteratorType> iteratorTypes(inputRank,
                                                  utils::IteratorType::parallel);
@@ -675,7 +675,7 @@ Value convertSubChannel(OpBuilder &builder, Location loc, Operation *op,
                               convertRanked(builder, loc, op, input, {}, scale,
                                             zeroPoint, quantizedType);
 
-                          builder.create<linalg::YieldOp>(loc, result);
+                          linalg::YieldOp::create(builder, loc, result);
                         })
                     .getResult(0);
 
@@ -729,7 +729,7 @@ struct DequantizeCastOpConversion
     // Convert quantized input to storage type
     auto storageScalarOrTensorType =
         getScalarOrTensorType(quantizedType.getStorageType(), input.getType());
-    input = rewriter.create<quant::StorageCastOp>(
+    input = quant::StorageCastOp::create(rewriter,
         loc, storageScalarOrTensorType, input);
 
     auto result = convertQuantized(rewriter, loc, op, input, quantizedType);
diff --git a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
index 4009faa21576d..f5e0aef365ff7 100644
--- a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
@@ -46,7 +46,7 @@ class QuantizedTypeConverter : public TypeConverter {
 
   static Value materializeConversion(OpBuilder &builder, Type type,
                                      ValueRange inputs, Location loc) {
-    return builder.create<quant::StorageCastOp>(loc, type,
+    return quant::StorageCastOp::create(builder, loc, type,
                                                 llvm::getSingleElement(inputs));
   }
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 5a3bd984530db..82fa50a6ac6e0 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -85,7 +85,7 @@ void SCFDialect::initialize() {
 
 /// Default callback for IfOp builders. Inserts a yield without arguments.
 void mlir::scf::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<scf::YieldOp>(loc);
+  scf::YieldOp::create(builder, loc);
 }
 
 /// Verifies that the first block of the given `region` is terminated by a
@@ -241,12 +241,12 @@ struct MultiBlockExecuteInliner : public OpRewritePattern<ExecuteRegionOp> {
     Block *postBlock = rewriter.splitBlock(prevBlock, op->getIterator());
     rewriter.setInsertionPointToEnd(prevBlock);
 
-    rewriter.create<cf::BranchOp>(op.getLoc(), &op.getRegion().front());
+    cf::BranchOp::create(rewriter, op.getLoc(), &op.getRegion().front());
 
     for (Block &blk : op.getRegion()) {
       if (YieldOp yieldOp = dyn_cast<YieldOp>(blk.getTerminator())) {
         rewriter.setInsertionPoint(yieldOp);
-        rewriter.create<cf::BranchOp>(yieldOp.getLoc(), postBlock,
+        cf::BranchOp::create(rewriter, yieldOp.getLoc(), postBlock,
                                       yieldOp.getResults());
         rewriter.eraseOp(yieldOp);
       }
@@ -557,7 +557,7 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
       [](OpBuilder &, Location, Value, ValueRange) {});
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
@@ -673,7 +673,7 @@ void mlir::scf::promote(RewriterBase &rewriter, scf::ForallOp forallOp) {
     Value dst = parallelInsertSliceOp.getDest();
     Value src = parallelInsertSliceOp.getSource();
     if (llvm::isa<TensorType>(src.getType())) {
-      results.push_back(rewriter.create<tensor::InsertSliceOp>(
+      results.push_back(tensor::InsertSliceOp::create(rewriter,
           forallOp.getLoc(), dst.getType(), src, dst,
           parallelInsertSliceOp.getOffsets(), parallelInsertSliceOp.getSizes(),
           parallelInsertSliceOp.getStrides(),
@@ -722,7 +722,7 @@ LoopNest mlir::scf::buildLoopNest(
   ValueRange currentIterArgs = iterArgs;
   Location currentLoc = loc;
   for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
-    auto loop = builder.create<scf::ForOp>(
+    auto loop = scf::ForOp::create(builder,
         currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
             ValueRange args) {
@@ -742,7 +742,7 @@ LoopNest mlir::scf::buildLoopNest(
   // For all loops but the innermost, yield the results of the nested loop.
   for (unsigned i = 0, e = loops.size() - 1; i < e; ++i) {
     builder.setInsertionPointToEnd(loops[i].getBody());
-    builder.create<scf::YieldOp>(loc, loops[i + 1].getResults());
+    scf::YieldOp::create(builder, loc, loops[i + 1].getResults());
   }
 
   // In the body of the innermost loop, call the body building function if any
@@ -756,7 +756,7 @@ LoopNest mlir::scf::buildLoopNest(
          "loop nest body must return as many values as loop has iteration "
          "arguments");
   builder.setInsertionPointToEnd(loops.back().getBody());
-  builder.create<scf::YieldOp>(loc, results);
+  scf::YieldOp::create(builder, loc, results);
 
   // Return the loops.
   ValueVector nestResults;
@@ -801,7 +801,7 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   }
 
   // 2. Create the new forOp shell.
-  scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+  scf::ForOp newForOp = scf::ForOp::create(rewriter,
       forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
       forOp.getStep(), newIterOperands);
   newForOp->setAttrs(forOp->getAttrs());
@@ -831,7 +831,7 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
                          clonedYieldOp.getOperand(yieldIdx));
   SmallVector<Value> newYieldOperands = clonedYieldOp.getOperands();
   newYieldOperands[yieldIdx] = castOut;
-  rewriter.create<scf::YieldOp>(newForOp.getLoc(), newYieldOperands);
+  scf::YieldOp::create(rewriter, newForOp.getLoc(), newYieldOperands);
   rewriter.eraseOp(clonedYieldOp);
 
   // 6. Inject an outgoing cast op after the forOp.
@@ -926,7 +926,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     if (!canonicalize)
       return failure();
 
-    scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+    scf::ForOp newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), newIterArgs);
     newForOp->setAttrs(forOp->getAttrs());
@@ -970,7 +970,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
       for (unsigned idx = 0, e = keepMask.size(); idx < e; ++idx)
         if (keepMask[idx])
           filteredOperands.push_back(mergedTerminator.getOperand(idx));
-      rewriter.create<scf::YieldOp>(mergedTerminator.getLoc(),
+      scf::YieldOp::create(rewriter, mergedTerminator.getLoc(),
                                     filteredOperands);
     };
 
@@ -1111,7 +1111,7 @@ struct ForOpTensorCastFolder : public OpRewritePattern<ForOp> {
           op, replaceAndCastForOpIterArg(
                   rewriter, op, iterOpOperand, incomingCast.getSource(),
                   [](OpBuilder &b, Location loc, Type type, Value source) {
-                    return b.create<tensor::CastOp>(loc, type, source);
+                    return tensor::CastOp::create(b, loc, type, source);
                   }));
       return success();
     }
@@ -1685,7 +1685,7 @@ struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
 
     // Step 3. Create a new scf.forall op with the new shared_outs' operands
     //         fetched earlier
-    auto newForallOp = rewriter.create<scf::ForallOp>(
+    auto newForallOp = scf::ForallOp::create(rewriter,
         forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
         forallOp.getMapping(),
@@ -1782,7 +1782,7 @@ struct ForallOpSingleOrZeroIterationDimsFolder
 
     // Replace the loop by a lower-dimensional loop.
     ForallOp newOp;
-    newOp = rewriter.create<ForallOp>(loc, newMixedLowerBounds,
+    newOp = ForallOp::create(rewriter, loc, newMixedLowerBounds,
                                       newMixedUpperBounds, newMixedSteps,
                                       op.getOutputs(), std::nullopt, nullptr);
     newOp.getBodyRegion().getBlocks().clear();
@@ -1867,7 +1867,7 @@ struct FoldTensorCastOfOutputIntoForallOp
 
     // Create new loop.
     Location loc = forallOp.getLoc();
-    auto newForallOp = rewriter.create<ForallOp>(
+    auto newForallOp = ForallOp::create(rewriter,
         loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
         forallOp.getMixedStep(), newOutputTensors, forallOp.getMapping(),
         [&](OpBuilder nestedBuilder, Location nestedLoc, ValueRange bbArgs) {
@@ -1875,7 +1875,7 @@ struct FoldTensorCastOfOutputIntoForallOp
               llvm::to_vector(bbArgs.take_back(forallOp->getNumResults()));
           for (auto [index, cast] : tensorCastProducers) {
             Value &oldTypeBBArg = castBlockArgs[index];
-            oldTypeBBArg = nestedBuilder.create<tensor::CastOp>(
+            oldTypeBBArg = tensor::CastOp::create(nestedBuilder,
                 nestedLoc, cast.dstType, oldTypeBBArg);
           }
 
@@ -1902,7 +1902,7 @@ struct FoldTensorCastOfOutputIntoForallOp
     SmallVector<Value> castResults = newForallOp.getResults();
     for (auto &item : tensorCastProducers) {
       Value &oldTypeResult = castResults[item.first];
-      oldTypeResult = rewriter.create<tensor::CastOp>(loc, item.second.dstType,
+      oldTypeResult = tensor::CastOp::create(rewriter, loc, item.second.dstType,
                                                       oldTypeResult);
     }
     rewriter.replaceOp(forallOp, castResults);
@@ -2309,7 +2309,7 @@ struct RemoveUnusedResults : public OpRewritePattern<IfOp> {
 
     // Create a replacement operation with empty then and else regions.
     auto newOp =
-        rewriter.create<IfOp>(op.getLoc(), newTypes, op.getCondition());
+        IfOp::create(rewriter, op.getLoc(), newTypes, op.getCondition());
     rewriter.createBlock(&newOp.getThenRegion());
     rewriter.createBlock(&newOp.getElseRegion());
 
@@ -2372,7 +2372,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
     if (nonHoistable.size() == op->getNumResults())
       return failure();
 
-    IfOp replacement = rewriter.create<IfOp>(op.getLoc(), nonHoistable, cond,
+    IfOp replacement = IfOp::create(rewriter, op.getLoc(), nonHoistable, cond,
                                              /*withElseRegion=*/false);
     if (replacement.thenBlock())
       rewriter.eraseBlock(replacement.thenBlock());
@@ -2398,7 +2398,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       } else if (trueVal == falseVal)
         results[it.index()] = trueVal;
       else
-        results[it.index()] = rewriter.create<arith::SelectOp>(
+        results[it.index()] = arith::SelectOp::create(rewriter,
             op.getLoc(), cond, trueVal, falseVal);
     }
 
@@ -2450,7 +2450,7 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantTrue)
-          constantTrue = rewriter.create<arith::ConstantOp>(
+          constantTrue = arith::ConstantOp::create(rewriter,
               op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
 
         rewriter.modifyOpInPlace(use.getOwner(),
@@ -2460,7 +2460,7 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantFalse)
-          constantFalse = rewriter.create<arith::ConstantOp>(
+          constantFalse = arith::ConstantOp::create(rewriter,
               op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
 
         rewriter.modifyOpInPlace(use.getOwner(),
@@ -2547,7 +2547,7 @@ struct ReplaceIfYieldWithConditionOrValue : public OpRewritePattern<IfOp> {
       if (!trueVal && falseVal) {
         if (!opResult.use_empty()) {
           Dialect *constDialect = trueResult.getDefiningOp()->getDialect();
-          Value notCond = rewriter.create<arith::XOrIOp>(
+          Value notCond = arith::XOrIOp::create(rewriter,
               op.getLoc(), op.getCondition(),
               constDialect
                   ->materializeConstant(rewriter,
@@ -2661,7 +2661,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
     SmallVector<Type> mergedTypes(prevIf.getResultTypes());
     llvm::append_range(mergedTypes, nextIf.getResultTypes());
 
-    IfOp combinedIf = rewriter.create<IfOp>(
+    IfOp combinedIf = IfOp::create(rewriter,
         nextIf.getLoc(), mergedTypes, prevIf.getCondition(), /*hasElse=*/false);
     rewriter.eraseBlock(&combinedIf.getThenRegion().back());
 
@@ -2677,7 +2677,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
 
       SmallVector<Value> mergedYields(thenYield.getOperands());
       llvm::append_range(mergedYields, thenYield2.getOperands());
-      rewriter.create<YieldOp>(thenYield2.getLoc(), mergedYields);
+      YieldOp::create(rewriter, thenYield2.getLoc(), mergedYields);
       rewriter.eraseOp(thenYield);
       rewriter.eraseOp(thenYield2);
     }
@@ -2701,7 +2701,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
         SmallVector<Value> mergedElseYields(elseYield.getOperands());
         llvm::append_range(mergedElseYields, elseYield2.getOperands());
 
-        rewriter.create<YieldOp>(elseYield2.getLoc(), mergedElseYields);
+        YieldOp::create(rewriter, elseYield2.getLoc(), mergedElseYields);
         rewriter.eraseOp(elseYield);
         rewriter.eraseOp(elseYield2);
       }
@@ -2823,9 +2823,9 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     }
 
     Location loc = op.getLoc();
-    Value newCondition = rewriter.create<arith::AndIOp>(
+    Value newCondition = arith::AndIOp::create(rewriter,
         loc, op.getCondition(), nestedIf.getCondition());
-    auto newIf = rewriter.create<IfOp>(loc, op.getResultTypes(), newCondition);
+    auto newIf = IfOp::create(rewriter, loc, op.getResultTypes(), newCondition);
     Block *newIfBlock = rewriter.createBlock(&newIf.getThenRegion());
 
     SmallVector<Value> results;
@@ -2833,7 +2833,7 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     rewriter.setInsertionPoint(newIf);
 
     for (auto idx : elseYieldsToUpgradeToSelect)
-      results[idx] = rewriter.create<arith::SelectOp>(
+      results[idx] = arith::SelectOp::create(rewriter,
           op.getLoc(), op.getCondition(), thenYield[idx], elseYield[idx]);
 
     rewriter.mergeBlocks(nestedIf.thenBlock(), newIfBlock);
@@ -2842,7 +2842,7 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     if (!elseYield.empty()) {
       rewriter.createBlock(&newIf.getElseRegion());
       rewriter.setInsertionPointToEnd(newIf.elseBlock());
-      rewriter.create<YieldOp>(loc, elseYield);
+      YieldOp::create(rewriter, loc, elseYield);
     }
     rewriter.replaceOp(op, results);
     return success();
@@ -3160,7 +3160,7 @@ struct ParallelOpSingleOrZeroIterationDimsFolder
     }
     // Replace the parallel loop by lower-dimensional parallel loop.
     auto newOp =
-        rewriter.create<ParallelOp>(op.getLoc(), newLowerBounds, newUpperBounds,
+        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds, newUpperBounds,
                                     newSteps, op.getInitVals(), nullptr);
     // Erase the empty block that was inserted by the builder.
     rewriter.eraseBlock(newOp.getBody());
@@ -3541,7 +3541,7 @@ struct WhileConditionTruth : public OpRewritePattern<WhileOp> {
       if (std::get<0>(yieldedAndBlockArgs) == term.getCondition()) {
         if (!std::get<1>(yieldedAndBlockArgs).use_empty()) {
           if (!constantTrue)
-            constantTrue = rewriter.create<arith::ConstantOp>(
+            constantTrue = arith::ConstantOp::create(rewriter,
                 op.getLoc(), term.getCondition().getType(),
                 rewriter.getBoolAttr(true));
 
@@ -3685,7 +3685,7 @@ struct RemoveLoopInvariantArgsFromBeforeBlock
     }
 
     auto newWhile =
-        rewriter.create<WhileOp>(op.getLoc(), op.getResultTypes(), newInitArgs);
+        WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(), newInitArgs);
 
     Block &newBeforeBlock = *rewriter.createBlock(
         &newWhile.getBefore(), /*insertPt*/ {},
@@ -3807,7 +3807,7 @@ struct RemoveLoopInvariantValueYielded : public OpRewritePattern<WhileOp> {
                                                newCondOpArgs);
     }
 
-    auto newWhile = rewriter.create<WhileOp>(op.getLoc(), newAfterBlockType,
+    auto newWhile = WhileOp::create(rewriter, op.getLoc(), newAfterBlockType,
                                              op.getOperands());
 
     Block &newAfterBlock =
@@ -3914,7 +3914,7 @@ struct WhileUnusedResult : public OpRewritePattern<WhileOp> {
     }
 
     auto newWhile =
-        rewriter.create<WhileOp>(op.getLoc(), newResultTypes, op.getInits());
+        WhileOp::create(rewriter, op.getLoc(), newResultTypes, op.getInits());
 
     Block &newAfterBlock = *rewriter.createBlock(
         &newWhile.getAfter(), /*insertPt*/ {}, newResultTypes, newArgLocs);
@@ -4043,7 +4043,7 @@ struct WhileRemoveUnusedArgs : public OpRewritePattern<WhileOp> {
 
     Location loc = op.getLoc();
     auto newWhileOp =
-        rewriter.create<WhileOp>(loc, op.getResultTypes(), newInits,
+        WhileOp::create(rewriter, loc, op.getResultTypes(), newInits,
                                  /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
@@ -4091,7 +4091,7 @@ struct WhileRemoveDuplicatedResults : public OpRewritePattern<WhileOp> {
     ValueRange argsRange(newArgs);
 
     Location loc = op.getLoc();
-    auto newWhileOp = rewriter.create<scf::WhileOp>(
+    auto newWhileOp = scf::WhileOp::create(rewriter,
         loc, argsRange.getTypes(), op.getInits(), /*beforeBody*/ nullptr,
         /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
@@ -4187,7 +4187,7 @@ struct WhileOpAlignBeforeArgs : public OpRewritePattern<WhileOp> {
     for (auto &&[i, j] : llvm::enumerate(*mapping))
       newResultTypes[j] = loop.getResult(i).getType();
 
-    auto newLoop = rewriter.create<WhileOp>(
+    auto newLoop = WhileOp::create(rewriter,
         loop.getLoc(), newResultTypes, loop.getInits(),
         /*beforeBuilder=*/nullptr, /*afterBuilder=*/nullptr);
     auto newBefore = newLoop.getBeforeBody();
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index 57c27231f2144..e9943a3489d46 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -163,7 +163,7 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
   OpBuilder::InsertionGuard g(b);
   b.setInsertionPoint(op);
   scf::ExecuteRegionOp executeRegionOp =
-      b.create<scf::ExecuteRegionOp>(op->getLoc(), op->getResultTypes());
+      scf::ExecuteRegionOp::create(b, op->getLoc(), op->getResultTypes());
   {
     OpBuilder::InsertionGuard g(b);
     b.setInsertionPointToStart(&executeRegionOp.getRegion().emplaceBlock());
@@ -172,7 +172,7 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
     assert(clonedRegion.empty() && "expected empty region");
     b.inlineRegionBefore(op->getRegions().front(), clonedRegion,
                          clonedRegion.end());
-    b.create<scf::YieldOp>(op->getLoc(), clonedOp->getResults());
+    scf::YieldOp::create(b, op->getLoc(), clonedOp->getResults());
   }
   b.replaceOp(op, executeRegionOp.getResults());
   return executeRegionOp;
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index d36d91249ed36..0214ce3090656 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -42,7 +42,7 @@ static Value castBuffer(OpBuilder &b, Value buffer, Type type) {
   // iter_arg's layout map must be changed (see uses of `castBuffer`).
   assert(memref::CastOp::areCastCompatible(buffer.getType(), type) &&
          "scf.while op bufferization: cast incompatible");
-  return b.create<memref::CastOp>(buffer.getLoc(), type, buffer).getResult();
+  return memref::CastOp::create(b, buffer.getLoc(), type, buffer).getResult();
 }
 
 /// Helper function for loop bufferization. Return "true" if the given value
@@ -191,7 +191,7 @@ struct ExecuteRegionOpInterface
 
     // Create new op and move over region.
     auto newOp =
-        rewriter.create<scf::ExecuteRegionOp>(op->getLoc(), newResultTypes);
+        scf::ExecuteRegionOp::create(rewriter, op->getLoc(), newResultTypes);
     newOp.getRegion().takeBody(executeRegionOp.getRegion());
 
     // Bufferize every block.
@@ -205,7 +205,7 @@ struct ExecuteRegionOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(executeRegionOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
+        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
             executeRegionOp.getLoc(), it.value(),
             newOp->getResult(it.index())));
       } else {
@@ -261,7 +261,7 @@ struct IfOpInterface
     // Create new op.
     rewriter.setInsertionPoint(ifOp);
     auto newIfOp =
-        rewriter.create<scf::IfOp>(ifOp.getLoc(), newTypes, ifOp.getCondition(),
+        scf::IfOp::create(rewriter, ifOp.getLoc(), newTypes, ifOp.getCondition(),
                                    /*withElseRegion=*/true);
 
     // Move over then/else blocks.
@@ -374,7 +374,7 @@ struct IndexSwitchOpInterface
 
     // Create new op.
     rewriter.setInsertionPoint(switchOp);
-    auto newSwitchOp = rewriter.create<scf::IndexSwitchOp>(
+    auto newSwitchOp = scf::IndexSwitchOp::create(rewriter,
         switchOp.getLoc(), newTypes, switchOp.getArg(), switchOp.getCases(),
         switchOp.getCases().size());
 
@@ -769,7 +769,7 @@ struct ForOpInterface
     }
 
     // Construct a new scf.for op with memref instead of tensor values.
-    auto newForOp = rewriter.create<scf::ForOp>(
+    auto newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), castedInitArgs);
     newForOp->setAttrs(forOp->getAttrs());
@@ -1005,7 +1005,7 @@ struct WhileOpInterface
     // Construct a new scf.while op with memref instead of tensor values.
     ValueRange argsRangeBefore(castedInitArgs);
     TypeRange argsTypesBefore(argsRangeBefore);
-    auto newWhileOp = rewriter.create<scf::WhileOp>(
+    auto newWhileOp = scf::WhileOp::create(rewriter,
         whileOp.getLoc(), argsTypesAfter, castedInitArgs);
 
     // Add before/after regions to the new op.
@@ -1265,7 +1265,7 @@ struct ForallOpInterface
              forallOp.getBody()->getArguments().drop_front(rank), buffers)) {
       BlockArgument bbArg = std::get<0>(it);
       Value buffer = std::get<1>(it);
-      Value bufferAsTensor = rewriter.create<ToTensorOp>(
+      Value bufferAsTensor = ToTensorOp::create(rewriter,
           forallOp.getLoc(), bbArg.getType(), buffer);
       bbArg.replaceAllUsesWith(bufferAsTensor);
     }
@@ -1274,7 +1274,7 @@ struct ForallOpInterface
     // introduced terminator.
     rewriter.setInsertionPoint(forallOp);
     ForallOp newForallOp;
-    newForallOp = rewriter.create<ForallOp>(
+    newForallOp = ForallOp::create(rewriter,
         forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
         /*outputs=*/ValueRange(), forallOp.getMapping());
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
index 3e93dc80b18ec..bf4eba3319a4d 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
@@ -50,7 +50,7 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     SmallVector<Value> initArgs;
     initArgs.push_back(forOp.getLowerBound());
     llvm::append_range(initArgs, forOp.getInitArgs());
-    auto whileOp = rewriter.create<WhileOp>(forOp.getLoc(), lcvTypes, initArgs,
+    auto whileOp = WhileOp::create(rewriter, forOp.getLoc(), lcvTypes, initArgs,
                                             forOp->getAttrs());
 
     // 'before' region contains the loop condition and forwarding of iteration
@@ -58,10 +58,10 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     auto *beforeBlock = rewriter.createBlock(
         &whileOp.getBefore(), whileOp.getBefore().begin(), lcvTypes, lcvLocs);
     rewriter.setInsertionPointToStart(whileOp.getBeforeBody());
-    auto cmpOp = rewriter.create<arith::CmpIOp>(
+    auto cmpOp = arith::CmpIOp::create(rewriter,
         whileOp.getLoc(), arith::CmpIPredicate::slt,
         beforeBlock->getArgument(0), forOp.getUpperBound());
-    rewriter.create<scf::ConditionOp>(whileOp.getLoc(), cmpOp.getResult(),
+    scf::ConditionOp::create(rewriter, whileOp.getLoc(), cmpOp.getResult(),
                                       beforeBlock->getArguments());
 
     // Inline for-loop body into an executeRegion operation in the "after"
@@ -72,7 +72,7 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
 
     // Add induction variable incrementation
     rewriter.setInsertionPointToEnd(afterBlock);
-    auto ivIncOp = rewriter.create<arith::AddIOp>(
+    auto ivIncOp = arith::AddIOp::create(rewriter,
         whileOp.getLoc(), afterBlock->getArgument(0), forOp.getStep());
 
     // Rewrite uses of the for-loop block arguments to the new while-loop
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
index 44e6840b03a3d..b95604fa44cb9 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
@@ -40,7 +40,7 @@ LogicalResult mlir::scf::forallToParallelLoop(RewriterBase &rewriter,
   SmallVector<Value> steps = forallOp.getStep(rewriter);
 
   // Create empty scf.parallel op.
-  auto parallelOp = rewriter.create<scf::ParallelOp>(loc, lbs, ubs, steps);
+  auto parallelOp = scf::ParallelOp::create(rewriter, loc, lbs, ubs, steps);
   rewriter.eraseBlock(&parallelOp.getRegion().front());
   rewriter.inlineRegionBefore(forallOp.getRegion(), parallelOp.getRegion(),
                               parallelOp.getRegion().begin());
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index bcecef5e6e0a9..f1ede64d01e0f 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -279,24 +279,24 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
     if (dynamicLoop) {
       Type t = ub.getType();
       // pred = ub > lb + (i * step)
-      Value iv = rewriter.create<arith::AddIOp>(
+      Value iv = arith::AddIOp::create(rewriter,
           loc, lb,
-          rewriter.create<arith::MulIOp>(
+          arith::MulIOp::create(rewriter,
               loc, step,
-              rewriter.create<arith::ConstantOp>(
+              arith::ConstantOp::create(rewriter,
                   loc, rewriter.getIntegerAttr(t, i))));
-      predicates[i] = rewriter.create<arith::CmpIOp>(
+      predicates[i] = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::slt, iv, ub);
     }
 
     // special handling for induction variable as the increment is implicit.
     // iv = lb + i * step
     Type t = lb.getType();
-    Value iv = rewriter.create<arith::AddIOp>(
+    Value iv = arith::AddIOp::create(rewriter,
         loc, lb,
-        rewriter.create<arith::MulIOp>(
+        arith::MulIOp::create(rewriter,
             loc, step,
-            rewriter.create<arith::ConstantOp>(loc,
+            arith::ConstantOp::create(rewriter, loc,
                                                rewriter.getIntegerAttr(t, i))));
     setValueMapping(forOp.getInductionVar(), iv, i);
     for (Operation *op : opOrder) {
@@ -332,7 +332,7 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
             Value prevValue = valueMapping
                 [forOp.getRegionIterArgs()[operand.getOperandNumber()]]
                 [i - stages[op]];
-            source = rewriter.create<arith::SelectOp>(
+            source = arith::SelectOp::create(rewriter,
                 loc, predicates[predicateIdx], source, prevValue);
           }
           setValueMapping(forOp.getRegionIterArgs()[operand.getOperandNumber()],
@@ -444,14 +444,14 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop(
     Type t = ub.getType();
     Location loc = forOp.getLoc();
     // newUb = ub - maxStage * step
-    Value maxStageValue = rewriter.create<arith::ConstantOp>(
+    Value maxStageValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(t, maxStage));
     Value maxStageByStep =
-        rewriter.create<arith::MulIOp>(loc, step, maxStageValue);
-    newUb = rewriter.create<arith::SubIOp>(loc, ub, maxStageByStep);
+        arith::MulIOp::create(rewriter, loc, step, maxStageValue);
+    newUb = arith::SubIOp::create(rewriter, loc, ub, maxStageByStep);
   }
   auto newForOp =
-      rewriter.create<scf::ForOp>(forOp.getLoc(), forOp.getLowerBound(), newUb,
+      scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(), newUb,
                                   forOp.getStep(), newLoopArg);
   // When there are no iter args, the loop body terminator will be created.
   // Since we always create it below, remove the terminator if it was created.
@@ -483,14 +483,14 @@ LogicalResult LoopPipelinerInternal::createKernel(
     Type t = ub.getType();
     for (unsigned i = 0; i < maxStage; i++) {
       // c = ub - (maxStage - i) * step
-      Value c = rewriter.create<arith::SubIOp>(
+      Value c = arith::SubIOp::create(rewriter,
           loc, ub,
-          rewriter.create<arith::MulIOp>(
+          arith::MulIOp::create(rewriter,
               loc, step,
-              rewriter.create<arith::ConstantOp>(
+              arith::ConstantOp::create(rewriter,
                   loc, rewriter.getIntegerAttr(t, int64_t(maxStage - i)))));
 
-      Value pred = rewriter.create<arith::CmpIOp>(
+      Value pred = arith::CmpIOp::create(rewriter,
           newForOp.getLoc(), arith::CmpIPredicate::slt,
           newForOp.getInductionVar(), c);
       predicates[i] = pred;
@@ -515,12 +515,12 @@ LogicalResult LoopPipelinerInternal::createKernel(
 
         // offset = (maxStage - stages[op]) * step
         Type t = step.getType();
-        Value offset = rewriter.create<arith::MulIOp>(
+        Value offset = arith::MulIOp::create(rewriter,
             forOp.getLoc(), step,
-            rewriter.create<arith::ConstantOp>(
+            arith::ConstantOp::create(rewriter,
                 forOp.getLoc(),
                 rewriter.getIntegerAttr(t, maxStage - stages[op])));
-        Value iv = rewriter.create<arith::AddIOp>(
+        Value iv = arith::AddIOp::create(rewriter,
             forOp.getLoc(), newForOp.getInductionVar(), offset);
         nestedNewOp->setOperand(operand->getOperandNumber(), iv);
         rewriter.setInsertionPointAfter(newOp);
@@ -594,7 +594,7 @@ LogicalResult LoopPipelinerInternal::createKernel(
         auto defStage = stages.find(def);
         if (defStage != stages.end() && defStage->second < maxStage) {
           Value pred = predicates[defStage->second];
-          source = rewriter.create<arith::SelectOp>(
+          source = arith::SelectOp::create(rewriter,
               pred.getLoc(), pred, source,
               newForOp.getBody()
                   ->getArguments()[yieldOperand.getOperandNumber() + 1]);
@@ -638,7 +638,7 @@ LogicalResult LoopPipelinerInternal::createKernel(
                       maxStage - defStage->second + 1);
     }
   }
-  rewriter.create<scf::YieldOp>(forOp.getLoc(), yieldOperands);
+  scf::YieldOp::create(rewriter, forOp.getLoc(), yieldOperands);
   return success();
 }
 
@@ -652,7 +652,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // removed by dead code if not used.
 
   auto createConst = [&](int v) {
-    return rewriter.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(rewriter, loc,
                                               rewriter.getIntegerAttr(t, v));
   };
 
@@ -661,41 +661,41 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // - total_iterations = (range_diff + step + (step < 0 ? 1 : -1)) / step
   Value zero = createConst(0);
   Value one = createConst(1);
-  Value stepLessZero = rewriter.create<arith::CmpIOp>(
+  Value stepLessZero = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::slt, step, zero);
   Value stepDecr =
-      rewriter.create<arith::SelectOp>(loc, stepLessZero, one, createConst(-1));
+      arith::SelectOp::create(rewriter, loc, stepLessZero, one, createConst(-1));
 
-  Value rangeDiff = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  Value rangeIncrStep = rewriter.create<arith::AddIOp>(loc, rangeDiff, step);
+  Value rangeDiff = arith::SubIOp::create(rewriter, loc, ub, lb);
+  Value rangeIncrStep = arith::AddIOp::create(rewriter, loc, rangeDiff, step);
   Value rangeDecr =
-      rewriter.create<arith::AddIOp>(loc, rangeIncrStep, stepDecr);
-  Value totalIterations = rewriter.create<arith::DivSIOp>(loc, rangeDecr, step);
+      arith::AddIOp::create(rewriter, loc, rangeIncrStep, stepDecr);
+  Value totalIterations = arith::DivSIOp::create(rewriter, loc, rangeDecr, step);
 
   // If total_iters < max_stage, start the epilogue at zero to match the
   // ramp-up in the prologue.
   // start_iter = max(0, total_iters - max_stage)
-  Value iterI = rewriter.create<arith::SubIOp>(loc, totalIterations,
+  Value iterI = arith::SubIOp::create(rewriter, loc, totalIterations,
                                                createConst(maxStage));
-  iterI = rewriter.create<arith::MaxSIOp>(loc, zero, iterI);
+  iterI = arith::MaxSIOp::create(rewriter, loc, zero, iterI);
 
   // Capture predicates for dynamic loops.
   SmallVector<Value> predicates(maxStage + 1);
 
   for (int64_t i = 1; i <= maxStage; i++) {
     // newLastIter = lb + step * iterI
-    Value newlastIter = rewriter.create<arith::AddIOp>(
-        loc, lb, rewriter.create<arith::MulIOp>(loc, step, iterI));
+    Value newlastIter = arith::AddIOp::create(rewriter,
+        loc, lb, arith::MulIOp::create(rewriter, loc, step, iterI));
 
     setValueMapping(forOp.getInductionVar(), newlastIter, i);
 
     // increment to next iterI
-    iterI = rewriter.create<arith::AddIOp>(loc, iterI, one);
+    iterI = arith::AddIOp::create(rewriter, loc, iterI, one);
 
     if (dynamicLoop) {
       // Disable stages when `i` is greater than total_iters.
       // pred = total_iters >= i
-      predicates[i] = rewriter.create<arith::CmpIOp>(
+      predicates[i] = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, totalIterations, createConst(i));
     }
   }
@@ -758,7 +758,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
           unsigned nextVersion = currentVersion + 1;
           Value pred = predicates[currentVersion];
           Value prevValue = valueMapping[mapVal][currentVersion];
-          auto selOp = rewriter.create<arith::SelectOp>(loc, pred, pair.value(),
+          auto selOp = arith::SelectOp::create(rewriter, loc, pred, pair.value(),
                                                         prevValue);
           returnValues[ri] = selOp;
           if (nextVersion <= maxStage)
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index b71ec985fa6a1..b24e276272c2c 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -64,13 +64,13 @@ static void specializeParallelLoopForUnrolling(ParallelOp op) {
   Value cond;
   for (auto bound : llvm::zip(op.getUpperBound(), constantIndices)) {
     Value constant =
-        b.create<arith::ConstantIndexOp>(op.getLoc(), std::get<1>(bound));
-    Value cmp = b.create<arith::CmpIOp>(op.getLoc(), arith::CmpIPredicate::eq,
+        arith::ConstantIndexOp::create(b, op.getLoc(), std::get<1>(bound));
+    Value cmp = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
                                         std::get<0>(bound), constant);
-    cond = cond ? b.create<arith::AndIOp>(op.getLoc(), cond, cmp) : cmp;
+    cond = cond ? arith::AndIOp::create(b, op.getLoc(), cond, cmp) : cmp;
     map.map(std::get<0>(bound), constant);
   }
-  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
   ifOp.getElseBodyBuilder().clone(*op.getOperation());
   op.erase();
@@ -95,11 +95,11 @@ static void specializeForLoopForUnrolling(ForOp op) {
 
   OpBuilder b(op);
   IRMapping map;
-  Value constant = b.create<arith::ConstantIndexOp>(op.getLoc(), minConstant);
-  Value cond = b.create<arith::CmpIOp>(op.getLoc(), arith::CmpIPredicate::eq,
+  Value constant = arith::ConstantIndexOp::create(b, op.getLoc(), minConstant);
+  Value cond = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
                                        bound, constant);
   map.map(bound, constant);
-  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
   ifOp.getElseBodyBuilder().clone(*op.getOperation());
   op.erase();
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index ad1267381c4f2..013c3b69620e2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -190,7 +190,7 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
 
   IRRewriter b(builder);
   b.setInsertionPoint(secondPloop);
-  auto newSecondPloop = b.create<ParallelOp>(
+  auto newSecondPloop = ParallelOp::create(b,
       secondPloop.getLoc(), secondPloop.getLowerBound(),
       secondPloop.getUpperBound(), secondPloop.getStep(), newInitVars);
 
@@ -212,7 +212,7 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
     SmallVector<Value> newReduceArgs(reduceArgs1.begin(), reduceArgs1.end());
     newReduceArgs.append(reduceArgs2.begin(), reduceArgs2.end());
 
-    auto newReduceOp = b.create<scf::ReduceOp>(term2.getLoc(), newReduceArgs);
+    auto newReduceOp = scf::ReduceOp::create(b, term2.getLoc(), newReduceArgs);
 
     for (auto &&[i, reg] : llvm::enumerate(llvm::concat<Region>(
              term1.getReductions(), term2.getReductions()))) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
index 66f7bc27f82ff..2ca3b1e366bdf 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
@@ -58,27 +58,27 @@ std::pair<ParallelOp, ParallelOp>
 mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
                             bool noMinMaxBounds) {
   OpBuilder b(op);
-  auto zero = b.create<arith::ConstantIndexOp>(op.getLoc(), 0);
+  auto zero = arith::ConstantIndexOp::create(b, op.getLoc(), 0);
   SmallVector<Value, 2> tileSizeConstants;
   tileSizeConstants.reserve(op.getUpperBound().size());
   for (size_t i = 0, end = op.getUpperBound().size(); i != end; ++i) {
     if (i < tileSizes.size())
       tileSizeConstants.push_back(
-          b.create<arith::ConstantIndexOp>(op.getLoc(), tileSizes[i]));
+          arith::ConstantIndexOp::create(b, op.getLoc(), tileSizes[i]));
     else
       // Just pick 1 for the remaining dimensions.
       tileSizeConstants.push_back(
-          b.create<arith::ConstantIndexOp>(op.getLoc(), 1));
+          arith::ConstantIndexOp::create(b, op.getLoc(), 1));
   }
 
   // Create the outer loop with adjusted steps.
   SmallVector<Value, 2> newSteps;
   newSteps.reserve(op.getStep().size());
   for (auto step : llvm::zip(op.getStep(), tileSizeConstants)) {
-    newSteps.push_back(b.create<arith::MulIOp>(op.getLoc(), std::get<0>(step),
+    newSteps.push_back(arith::MulIOp::create(b, op.getLoc(), std::get<0>(step),
                                                std::get<1>(step)));
   }
-  auto outerLoop = b.create<ParallelOp>(op.getLoc(), op.getLowerBound(),
+  auto outerLoop = ParallelOp::create(b, op.getLoc(), op.getLowerBound(),
                                         op.getUpperBound(), newSteps);
   b.setInsertionPointToStart(outerLoop.getBody());
 
@@ -130,10 +130,10 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // Otherwise, we dynamically compute the bound for
     // each iteration of the outer loop.
     newBounds.push_back(
-        b.create<affine::AffineMinOp>(op.getLoc(), b.getIndexType(), minMap,
+        affine::AffineMinOp::create(b, op.getLoc(), b.getIndexType(), minMap,
                                       ValueRange{newStep, upperBound, iv}));
   }
-  auto innerLoop = b.create<ParallelOp>(
+  auto innerLoop = ParallelOp::create(b,
       op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
       op.getStep());
 
@@ -141,20 +141,20 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     b.setInsertionPointToStart(innerLoop.getBody());
     // Insert in-bound check
     Value inbound =
-        b.create<arith::ConstantIntOp>(op.getLoc(), b.getIntegerType(1), 1);
+        arith::ConstantIntOp::create(b, op.getLoc(), b.getIntegerType(1), 1);
     for (auto [outerUpperBound, outerIV, innerIV, innerStep] :
          llvm::zip(outerLoop.getUpperBound(), outerLoop.getInductionVars(),
                    innerLoop.getInductionVars(), innerLoop.getStep())) {
       // %in_bound = %in_bound &&
       //             (%inner_iv * %inner_step + %outer_iv < %outer_upper_bound)
-      Value index = b.create<arith::AddIOp>(
-          op.getLoc(), b.create<arith::MulIOp>(op.getLoc(), innerIV, innerStep),
+      Value index = arith::AddIOp::create(b,
+          op.getLoc(), arith::MulIOp::create(b, op.getLoc(), innerIV, innerStep),
           outerIV);
-      Value dimInbound = b.create<arith::CmpIOp>(
+      Value dimInbound = arith::CmpIOp::create(b,
           op.getLoc(), arith::CmpIPredicate::ult, index, outerUpperBound);
-      inbound = b.create<arith::AndIOp>(op.getLoc(), inbound, dimInbound);
+      inbound = arith::AndIOp::create(b, op.getLoc(), inbound, dimInbound);
     }
-    auto ifInbound = b.create<IfOp>(op.getLoc(),
+    auto ifInbound = IfOp::create(b, op.getLoc(),
                                     /*resultTypes*/ ArrayRef<Type>{}, inbound,
                                     /*hasElseRegion*/ false);
     ifInbound.getThenRegion().takeBody(op.getRegion());
@@ -162,12 +162,12 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // Replace the scf.reduce terminator with an scf.yield terminator.
     Operation *reduceOp = thenBlock.getTerminator();
     b.setInsertionPointToEnd(&thenBlock);
-    b.create<scf::YieldOp>(reduceOp->getLoc());
+    scf::YieldOp::create(b, reduceOp->getLoc());
     reduceOp->erase();
     b.setInsertionPointToStart(innerLoop.getBody());
     for (const auto &ivs : llvm::enumerate(llvm::zip(
              innerLoop.getInductionVars(), outerLoop.getInductionVars()))) {
-      auto newIndex = b.create<arith::AddIOp>(
+      auto newIndex = arith::AddIOp::create(b,
           op.getLoc(), std::get<0>(ivs.value()), std::get<1>(ivs.value()));
       thenBlock.getArgument(ivs.index())
           .replaceAllUsesExcept(newIndex, newIndex);
@@ -179,7 +179,7 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     for (auto ivs : llvm::zip(innerLoop.getInductionVars(),
                               outerLoop.getInductionVars())) {
       Value innerIndex = std::get<0>(ivs);
-      auto newIndex = b.create<arith::AddIOp>(op.getLoc(), std::get<0>(ivs),
+      auto newIndex = arith::AddIOp::create(b, op.getLoc(), std::get<0>(ivs),
                                               std::get<1>(ivs));
       innerIndex.replaceAllUsesExcept(newIndex, newIndex);
     }
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 09326242eec2a..41895e46d953d 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -112,7 +112,7 @@ class ConvertForOpTypes
 
     // We can not do clone as the number of result types after conversion
     // might be different.
-    ForOp newOp = rewriter.create<ForOp>(
+    ForOp newOp = ForOp::create(rewriter,
         op.getLoc(), llvm::getSingleElement(adaptor.getLowerBound()),
         llvm::getSingleElement(adaptor.getUpperBound()),
         llvm::getSingleElement(adaptor.getStep()),
@@ -142,7 +142,7 @@ class ConvertIfOpTypes
                                       ConversionPatternRewriter &rewriter,
                                       TypeRange dstTypes) const {
 
-    IfOp newOp = rewriter.create<IfOp>(
+    IfOp newOp = IfOp::create(rewriter,
         op.getLoc(), dstTypes, llvm::getSingleElement(adaptor.getCondition()),
         true);
     newOp->setAttrs(op->getAttrs());
@@ -171,7 +171,7 @@ class ConvertWhileOpTypes
   std::optional<WhileOp> convertSourceOp(WhileOp op, OneToNOpAdaptor adaptor,
                                          ConversionPatternRewriter &rewriter,
                                          TypeRange dstTypes) const {
-    auto newOp = rewriter.create<WhileOp>(op.getLoc(), dstTypes,
+    auto newOp = WhileOp::create(rewriter, op.getLoc(), dstTypes,
                                           flattenValues(adaptor.getOperands()));
 
     for (auto i : {0u, 1u}) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index 995120ad8680e..f995f324a0dbd 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -450,7 +450,7 @@ static LogicalResult generateLoopNestUsingForOp(
   SmallVector<Value> ivs;
   for (auto [lb, ub, step] : llvm::zip_equal(lbVals, ubVals, stepVals)) {
     auto loop =
-        rewriter.create<scf::ForOp>(loc, lb, ub, step, destinationTensors,
+        scf::ForOp::create(rewriter, loc, lb, ub, step, destinationTensors,
                                     [](OpBuilder &bodyBuilder, Location bodyLoc,
                                        Value iv, ValueRange /*iterArgs*/) {});
     loops.push_back(loop);
@@ -479,12 +479,12 @@ static LogicalResult generateLoopNestUsingForOp(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    auto insertSlice = rewriter.create<tensor::InsertSliceOp>(
+    auto insertSlice = tensor::InsertSliceOp::create(rewriter,
         loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
     yieldedValues.push_back(insertSlice);
   }
-  rewriter.create<scf::YieldOp>(loc, yieldedValues);
+  scf::YieldOp::create(rewriter, loc, yieldedValues);
 
   // Add the scf.yield operations for all the outer loops.
   for (auto [outerLoop, innerLoop] :
@@ -492,7 +492,7 @@ static LogicalResult generateLoopNestUsingForOp(
                        MutableArrayRef(loops).drop_front())) {
     rewriter.setInsertionPointToEnd(
         cast<scf::ForOp>(outerLoop.getOperation()).getBody());
-    rewriter.create<scf::YieldOp>(outerLoop.getLoc(), innerLoop->getResults());
+    scf::YieldOp::create(rewriter, outerLoop.getLoc(), innerLoop->getResults());
   }
   return success();
 }
@@ -533,13 +533,13 @@ static LogicalResult generateLoopNestUsingForallOp(
         continue;
       nonZeroNumThreads.push_back(nt);
     }
-    forallOp = rewriter.create<scf::ForallOp>(loc, nonZeroNumThreads,
+    forallOp = scf::ForallOp::create(rewriter, loc, nonZeroNumThreads,
                                               destinationTensors, mappingAttr);
   } else {
     SmallVector<OpFoldResult> lbs, ubs, steps;
     std::tie(lbs, ubs, steps) =
         getLoopBounds(rewriter, loc, loopRanges, tileSizes);
-    forallOp = rewriter.create<scf::ForallOp>(loc, lbs, ubs, steps,
+    forallOp = scf::ForallOp::create(rewriter, loc, lbs, ubs, steps,
                                               destinationTensors, mappingAttr);
   }
   loops.push_back(forallOp);
@@ -561,7 +561,7 @@ static LogicalResult generateLoopNestUsingForallOp(
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
 
-    rewriter.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(rewriter,
         loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
   }
@@ -798,7 +798,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
 
   auto inits = llvm::to_vector(loopOp.getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = rewriter.create<scf::ForOp>(
+  auto newLoop = scf::ForOp::create(rewriter,
       loc, loopOp.getLowerBound(), loopOp.getUpperBound(), loopOp.getStep(),
       inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
@@ -829,7 +829,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    Value insert = rewriter.create<tensor::InsertSliceOp>(
+    Value insert = tensor::InsertSliceOp::create(rewriter,
         yieldOp->getLoc(), tiledValue, regionIterArg, resultOffset, resultSize,
         resultStride);
     newYieldValues.push_back(insert);
@@ -851,7 +851,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
   rewriter.setInsertionPoint(loopOp);
   auto inits = llvm::to_vector(loopOp.getOutputs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = rewriter.create<scf::ForallOp>(
+  auto newLoop = scf::ForallOp::create(rewriter,
       loc, loopOp.getMixedLowerBound(), loopOp.getMixedUpperBound(),
       loopOp.getMixedStep(), inits, loopOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
@@ -884,7 +884,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
            tiledValues, regionIterArgs, resultOffsets, resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    rewriter.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(rewriter,
         terminator.getLoc(), tiledValue, iterArg, resultOffset, resultSize,
         resultStride);
   }
@@ -935,7 +935,7 @@ static LogicalResult addInitOperandsToLoopNest(
     // Create a new loop with the new init values for this loop.
     SmallVector<Value> newInits = llvm::to_vector(forLoop.getInitArgs());
     newInits.append(newInitValues.begin(), newInitValues.end());
-    auto newLoop = rewriter.create<scf::ForOp>(
+    auto newLoop = scf::ForOp::create(rewriter,
         forLoop.getLoc(), forLoop.getLowerBound(), forLoop.getUpperBound(),
         forLoop.getStep(), newInits,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {});
@@ -1419,7 +1419,7 @@ FailureOr<SmallVector<Operation *>> mlir::scf::yieldReplacementForFusedProducer(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = rewriter.create<tensor::ExtractSliceOp>(
+        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
             loc, newRegionArg, offsetList[index], sizesList[index],
             SmallVector<OpFoldResult>(offsetList[index].size(),
                                       rewriter.getIndexAttr(1)));
@@ -2092,7 +2092,7 @@ cloneAsInsertSlice<tensor::InsertSliceOp>(RewriterBase &rewriter,
 template <>
 tensor::InsertSliceOp cloneAsInsertSlice<tensor::ParallelInsertSliceOp>(
     RewriterBase &rewriter, tensor::ParallelInsertSliceOp insertSliceOp) {
-  return rewriter.create<tensor::InsertSliceOp>(
+  return tensor::InsertSliceOp::create(rewriter,
       insertSliceOp->getLoc(), insertSliceOp.getSource(),
       insertSliceOp.getDest(), insertSliceOp.getMixedOffsets(),
       insertSliceOp.getMixedSizes(), insertSliceOp.getMixedStrides());
@@ -2314,7 +2314,7 @@ mlir::scf::tileAndFuseConsumerOfSlices(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = rewriter.create<tensor::ExtractSliceOp>(
+        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
             loc, newRegionArg, resultOffsets[index], resultSizes[index],
             SmallVector<OpFoldResult>(resultOffsets[index].size(),
                                       rewriter.getIndexAttr(1)));
@@ -2391,7 +2391,7 @@ mlir::scf::lowerToLoopsUsingSCFForOp(RewriterBase &rewriter,
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.size);
     Value strideVal =
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.stride);
-    auto loop = rewriter.create<scf::ForOp>(op.getLoc(), offsetVal, sizeVal,
+    auto loop = scf::ForOp::create(rewriter, op.getLoc(), offsetVal, sizeVal,
                                             strideVal, ValueRange{});
     loops.push_back(loop);
     ivs.push_back(loop.getInductionVar());
diff --git a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
index 29d6d2574a2be..14ded96516eab 100644
--- a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
@@ -191,7 +191,7 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
   // dummy builder instead.
   auto emptyBuilder = [](OpBuilder &, Location, Value, ValueRange) {};
   auto newLoop =
-      rewriter.create<scf::ForOp>(loc, lb, ub, step, newArgs, emptyBuilder);
+      scf::ForOp::create(rewriter, loc, lb, ub, step, newArgs, emptyBuilder);
 
   Block *newBody = newLoop.getBody();
 
@@ -238,18 +238,18 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
   rewriter.setInsertionPointAfter(newLoop);
   Value one;
   if (isa<IndexType>(step.getType())) {
-    one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    one = arith::ConstantIndexOp::create(rewriter, loc, 1);
   } else {
-    one = rewriter.create<arith::ConstantIntOp>(loc, step.getType(), 1);
+    one = arith::ConstantIntOp::create(rewriter, loc, step.getType(), 1);
   }
 
-  Value stepDec = rewriter.create<arith::SubIOp>(loc, step, one);
-  Value len = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  len = rewriter.create<arith::AddIOp>(loc, len, stepDec);
-  len = rewriter.create<arith::DivSIOp>(loc, len, step);
-  len = rewriter.create<arith::SubIOp>(loc, len, one);
-  Value res = rewriter.create<arith::MulIOp>(loc, len, step);
-  res = rewriter.create<arith::AddIOp>(loc, lb, res);
+  Value stepDec = arith::SubIOp::create(rewriter, loc, step, one);
+  Value len = arith::SubIOp::create(rewriter, loc, ub, lb);
+  len = arith::AddIOp::create(rewriter, loc, len, stepDec);
+  len = arith::DivSIOp::create(rewriter, loc, len, step);
+  len = arith::SubIOp::create(rewriter, loc, len, one);
+  Value res = arith::MulIOp::create(rewriter, loc, len, step);
+  res = arith::AddIOp::create(rewriter, loc, lb, res);
 
   // Reconstruct `scf.while` results, inserting final induction var value
   // into proper place.
diff --git a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
index f829208ce8798..707ed836cf63e 100644
--- a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
@@ -96,7 +96,7 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       condOp.getArgs(), [&](Value arg) { return mapper.lookupOrDefault(arg); });
 
   // Create rotated while loop.
-  auto newLoopOp = rewriter.create<scf::WhileOp>(
+  auto newLoopOp = scf::WhileOp::create(rewriter,
       whileOp.getLoc(), whileOp.getResultTypes(), clonedCondArgs,
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Rotate and move the loop body into before block.
@@ -109,21 +109,21 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       },
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Pass through values.
-        builder.create<scf::YieldOp>(loc, args);
+        scf::YieldOp::create(builder, loc, args);
       });
 
   // Create zero-trip-check and move the while loop in.
-  auto ifOp = rewriter.create<scf::IfOp>(
+  auto ifOp = scf::IfOp::create(rewriter,
       whileOp.getLoc(), clonedCondition,
       [&](OpBuilder &builder, Location loc) {
         // Then runs the while loop.
         rewriter.moveOpBefore(newLoopOp, builder.getInsertionBlock(),
                               builder.getInsertionPoint());
-        builder.create<scf::YieldOp>(loc, newLoopOp.getResults());
+        scf::YieldOp::create(builder, loc, newLoopOp.getResults());
       },
       [&](OpBuilder &builder, Location loc) {
         // Else returns the results from precondition.
-        builder.create<scf::YieldOp>(loc, clonedCondArgs);
+        scf::YieldOp::create(builder, loc, clonedCondArgs);
       });
 
   rewriter.replaceOp(whileOp, ifOp);
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index f4047be68ccf2..6a387ddf8908a 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -153,7 +153,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
       FunctionType::get(rewriter.getContext(), outlinedFuncArgTypes,
                         originalTerminator->getOperandTypes());
   auto outlinedFunc =
-      rewriter.create<func::FuncOp>(loc, funcName, outlinedFuncType);
+      func::FuncOp::create(rewriter, loc, funcName, outlinedFuncType);
   Block *outlinedFuncBody = outlinedFunc.addEntryBlock();
 
   // Merge blocks while replacing the original block operands.
@@ -168,7 +168,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
         outlinedFuncBlockArgs.take_front(numOriginalBlockArguments));
     // Explicitly set up a new ReturnOp terminator.
     rewriter.setInsertionPointToEnd(outlinedFuncBody);
-    rewriter.create<func::ReturnOp>(loc, originalTerminator->getResultTypes(),
+    func::ReturnOp::create(rewriter, loc, originalTerminator->getResultTypes(),
                                     originalTerminator->getOperands());
   }
 
@@ -185,7 +185,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
     SmallVector<Value> callValues;
     llvm::append_range(callValues, newBlock->getArguments());
     llvm::append_range(callValues, outlinedValues);
-    auto call = rewriter.create<func::CallOp>(loc, outlinedFunc, callValues);
+    auto call = func::CallOp::create(rewriter, loc, outlinedFunc, callValues);
     if (callOp)
       *callOp = call;
 
@@ -274,12 +274,12 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
 
-  Value divisorMinusOneCst = builder.create<arith::ConstantOp>(
+  Value divisorMinusOneCst = arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(dividend.getType(), divisor - 1));
-  Value divisorCst = builder.create<arith::ConstantOp>(
+  Value divisorCst = arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(dividend.getType(), divisor));
-  Value sum = builder.create<arith::AddIOp>(loc, dividend, divisorMinusOneCst);
-  return builder.create<arith::DivUIOp>(loc, sum, divisorCst);
+  Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOneCst);
+  return arith::DivUIOp::create(builder, loc, sum, divisorCst);
 }
 
 // Build the IR that performs ceil division of a positive value by another
@@ -290,11 +290,11 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
                              Value divisor) {
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
-  Value cstOne = builder.create<arith::ConstantOp>(
+  Value cstOne = arith::ConstantOp::create(builder,
       loc, builder.getOneAttr(dividend.getType()));
-  Value divisorMinusOne = builder.create<arith::SubIOp>(loc, divisor, cstOne);
-  Value sum = builder.create<arith::AddIOp>(loc, dividend, divisorMinusOne);
-  return builder.create<arith::DivUIOp>(loc, sum, divisor);
+  Value divisorMinusOne = arith::SubIOp::create(builder, loc, divisor, cstOne);
+  Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOne);
+  return arith::DivUIOp::create(builder, loc, sum, divisor);
 }
 
 /// Returns the trip count of `forOp` if its' low bound, high bound and step are
@@ -404,7 +404,7 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'upperBoundUnrolled' and set epilogue loop flag.
     generateEpilogueLoop = upperBoundUnrolledCst < ubCst;
     if (generateEpilogueLoop)
-      upperBoundUnrolled = boundsBuilder.create<arith::ConstantOp>(
+      upperBoundUnrolled = arith::ConstantOp::create(boundsBuilder,
           loc, boundsBuilder.getIntegerAttr(forOp.getUpperBound().getType(),
                                             upperBoundUnrolledCst));
     else
@@ -413,7 +413,7 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'stepUnrolled'.
     stepUnrolled = stepCst == stepUnrolledCst
                        ? step
-                       : boundsBuilder.create<arith::ConstantOp>(
+                       : arith::ConstantOp::create(boundsBuilder,
                              loc, boundsBuilder.getIntegerAttr(
                                       step.getType(), stepUnrolledCst));
   } else {
@@ -423,22 +423,22 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     auto lowerBound = forOp.getLowerBound();
     auto upperBound = forOp.getUpperBound();
     Value diff =
-        boundsBuilder.create<arith::SubIOp>(loc, upperBound, lowerBound);
+        arith::SubIOp::create(boundsBuilder, loc, upperBound, lowerBound);
     Value tripCount = ceilDivPositive(boundsBuilder, loc, diff, step);
-    Value unrollFactorCst = boundsBuilder.create<arith::ConstantOp>(
+    Value unrollFactorCst = arith::ConstantOp::create(boundsBuilder,
         loc, boundsBuilder.getIntegerAttr(tripCount.getType(), unrollFactor));
     Value tripCountRem =
-        boundsBuilder.create<arith::RemSIOp>(loc, tripCount, unrollFactorCst);
+        arith::RemSIOp::create(boundsBuilder, loc, tripCount, unrollFactorCst);
     // Compute tripCountEvenMultiple = tripCount - (tripCount % unrollFactor)
     Value tripCountEvenMultiple =
-        boundsBuilder.create<arith::SubIOp>(loc, tripCount, tripCountRem);
+        arith::SubIOp::create(boundsBuilder, loc, tripCount, tripCountRem);
     // Compute upperBoundUnrolled = lowerBound + tripCountEvenMultiple * step
-    upperBoundUnrolled = boundsBuilder.create<arith::AddIOp>(
+    upperBoundUnrolled = arith::AddIOp::create(boundsBuilder,
         loc, lowerBound,
-        boundsBuilder.create<arith::MulIOp>(loc, tripCountEvenMultiple, step));
+        arith::MulIOp::create(boundsBuilder, loc, tripCountEvenMultiple, step));
     // Scale 'step' by 'unrollFactor'.
     stepUnrolled =
-        boundsBuilder.create<arith::MulIOp>(loc, step, unrollFactorCst);
+        arith::MulIOp::create(boundsBuilder, loc, step, unrollFactorCst);
   }
 
   UnrolledLoopInfo resultLoops;
@@ -474,11 +474,11 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
       forOp.getBody(), forOp.getInductionVar(), unrollFactor,
       [&](unsigned i, Value iv, OpBuilder b) {
         // iv' = iv + step * i;
-        auto stride = b.create<arith::MulIOp>(
+        auto stride = arith::MulIOp::create(b,
             loc, step,
-            b.create<arith::ConstantOp>(loc,
+            arith::ConstantOp::create(b, loc,
                                         b.getIntegerAttr(iv.getType(), i)));
-        return b.create<arith::AddIOp>(loc, iv, stride);
+        return arith::AddIOp::create(b, loc, iv, stride);
       },
       annotateFn, iterArgs, yieldedValues);
   // Promote the loop body up if this has turned into a single iteration loop.
@@ -781,13 +781,13 @@ void mlir::denormalizeInductionVariable(RewriterBase &rewriter, Location loc,
   if (!isStepOne) {
     Value origStepValue =
         getValueOrCreateConstantIntOp(rewriter, loc, origStep);
-    scaled = rewriter.create<arith::MulIOp>(loc, normalizedIv, origStepValue);
+    scaled = arith::MulIOp::create(rewriter, loc, normalizedIv, origStepValue);
     preserve.insert(scaled.getDefiningOp());
   }
   denormalizedIv = scaled;
   if (!isZeroBased) {
     Value origLbValue = getValueOrCreateConstantIntOp(rewriter, loc, origLb);
-    denormalizedIv = rewriter.create<arith::AddIOp>(loc, scaled, origLbValue);
+    denormalizedIv = arith::AddIOp::create(rewriter, loc, scaled, origLbValue);
     preserve.insert(denormalizedIv.getDefiningOp());
   }
 
@@ -824,7 +824,7 @@ static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc,
       continue;
     if (productOf)
       productOf =
-          rewriter.create<arith::MulIOp>(loc, productOf.value(), v).getResult();
+          arith::MulIOp::create(rewriter, loc, productOf.value(), v).getResult();
     else
       productOf = v;
   }
@@ -851,7 +851,7 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
 
   if (linearizedIv.getType().isIndex()) {
     Operation *delinearizedOp =
-        rewriter.create<affine::AffineDelinearizeIndexOp>(loc, linearizedIv,
+        affine::AffineDelinearizeIndexOp::create(rewriter, loc, linearizedIv,
                                                           ubs);
     auto resultVals = llvm::map_to_vector(
         delinearizedOp->getResults(), [](OpResult r) -> Value { return r; });
@@ -874,7 +874,7 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
     if (!isUbOne.test(index)) {
       break;
     }
-    delinearizedIvs[index] = rewriter.create<arith::ConstantOp>(
+    delinearizedIvs[index] = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(ub.getType()));
     numLeadingOneUbs++;
   }
@@ -883,16 +883,16 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
   for (unsigned i = numLeadingOneUbs, e = ubs.size(); i < e; ++i) {
     unsigned idx = ubs.size() - (i - numLeadingOneUbs) - 1;
     if (i != numLeadingOneUbs && !isUbOne.test(idx + 1)) {
-      previous = rewriter.create<arith::DivSIOp>(loc, previous, ubs[idx + 1]);
+      previous = arith::DivSIOp::create(rewriter, loc, previous, ubs[idx + 1]);
       preservedUsers.insert(previous.getDefiningOp());
     }
     Value iv = previous;
     if (i != e - 1) {
       if (!isUbOne.test(idx)) {
-        iv = rewriter.create<arith::RemSIOp>(loc, previous, ubs[idx]);
+        iv = arith::RemSIOp::create(rewriter, loc, previous, ubs[idx]);
         preservedUsers.insert(iv.getDefiningOp());
       } else {
-        iv = rewriter.create<arith::ConstantOp>(
+        iv = arith::ConstantOp::create(rewriter,
             loc, rewriter.getZeroAttr(ubs[idx].getType()));
       }
     }
@@ -1093,12 +1093,12 @@ void mlir::collapseParallelLoops(
 
   // Combine iteration spaces.
   SmallVector<Value, 3> lowerBounds, upperBounds, steps;
-  auto cst0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto cst1 = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto cst0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto cst1 = arith::ConstantIndexOp::create(rewriter, loc, 1);
   for (auto &sortedDimension : sortedDimensions) {
-    Value newUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value newUpperBound = arith::ConstantIndexOp::create(rewriter, loc, 1);
     for (auto idx : sortedDimension) {
-      newUpperBound = rewriter.create<arith::MulIOp>(
+      newUpperBound = arith::MulIOp::create(rewriter,
           loc, newUpperBound, normalizedUpperBounds[idx]);
     }
     lowerBounds.push_back(cst0);
@@ -1112,7 +1112,7 @@ void mlir::collapseParallelLoops(
   // value. The remainders then determine based on that range, which iteration
   // of the original induction value this represents. This is a normalized value
   // that is un-normalized already by the previous logic.
-  auto newPloop = rewriter.create<scf::ParallelOp>(
+  auto newPloop = scf::ParallelOp::create(rewriter,
       loc, lowerBounds, upperBounds, steps,
       [&](OpBuilder &insideBuilder, Location, ValueRange ploopIVs) {
         for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) {
@@ -1123,14 +1123,14 @@ void mlir::collapseParallelLoops(
             unsigned idx = combinedDimensions[i][j];
 
             // Determine the current induction value's current loop iteration
-            Value iv = insideBuilder.create<arith::RemSIOp>(
+            Value iv = arith::RemSIOp::create(insideBuilder,
                 loc, previous, normalizedUpperBounds[idx]);
             replaceAllUsesInRegionWith(loops.getBody()->getArgument(idx), iv,
                                        loops.getRegion());
 
             // Remove the effect of the current induction value to prepare for
             // the next value.
-            previous = insideBuilder.create<arith::DivSIOp>(
+            previous = arith::DivSIOp::create(insideBuilder,
                 loc, previous, normalizedUpperBounds[idx]);
           }
 
@@ -1241,7 +1241,7 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
   auto iv = forOp.getInductionVar();
 
   OpBuilder b(forOp);
-  forOp.setStep(b.create<arith::MulIOp>(forOp.getLoc(), originalStep, factor));
+  forOp.setStep(arith::MulIOp::create(b, forOp.getLoc(), originalStep, factor));
 
   Loops innerLoops;
   for (auto t : targets) {
@@ -1251,12 +1251,12 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
 
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator((t.getBody()));
-    Value stepped = b.create<arith::AddIOp>(t.getLoc(), iv, forOp.getStep());
+    Value stepped = arith::AddIOp::create(b, t.getLoc(), iv, forOp.getStep());
     Value ub =
-        b.create<arith::MinSIOp>(t.getLoc(), forOp.getUpperBound(), stepped);
+        arith::MinSIOp::create(b, t.getLoc(), forOp.getUpperBound(), stepped);
 
     // Splice [begin, begin + nOps - 1) into `newForOp` and replace uses.
-    auto newForOp = b.create<scf::ForOp>(t.getLoc(), iv, ub, originalStep);
+    auto newForOp = scf::ForOp::create(b, t.getLoc(), iv, ub, originalStep);
     newForOp.getBody()->getOperations().splice(
         newForOp.getBody()->getOperations().begin(),
         t.getBody()->getOperations(), begin, std::next(begin, nOps - 1));
@@ -1343,7 +1343,7 @@ TileLoops mlir::extractFixedOuterLoops(scf::ForOp rootForOp,
     auto forOp = forOps[i];
     OpBuilder builder(forOp);
     auto loc = forOp.getLoc();
-    Value diff = builder.create<arith::SubIOp>(loc, forOp.getUpperBound(),
+    Value diff = arith::SubIOp::create(builder, loc, forOp.getUpperBound(),
                                                forOp.getLowerBound());
     Value numIterations = ceilDivPositive(builder, loc, diff, forOp.getStep());
     Value iterationsPerBlock =
@@ -1376,7 +1376,7 @@ scf::ForallOp mlir::fuseIndependentSiblingForallLoops(scf::ForallOp target,
 
   // Create a new scf.forall op after the source loop.
   rewriter.setInsertionPointAfter(source);
-  scf::ForallOp fusedLoop = rewriter.create<scf::ForallOp>(
+  scf::ForallOp fusedLoop = scf::ForallOp::create(rewriter,
       source.getLoc(), source.getMixedLowerBound(), source.getMixedUpperBound(),
       source.getMixedStep(), fusedOuts, source.getMapping());
 
@@ -1429,7 +1429,7 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   // Create a new scf.for op after the source loop (with scf.yield terminator
   // (without arguments) only in case its init_args is empty).
   rewriter.setInsertionPointAfter(source);
-  scf::ForOp fusedLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp fusedLoop = scf::ForOp::create(rewriter,
       source.getLoc(), source.getLowerBound(), source.getUpperBound(),
       source.getStep(), fusedInitArgs);
 
@@ -1456,7 +1456,7 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   for (Value operand : source.getBody()->getTerminator()->getOperands())
     yieldResults.push_back(mapping.lookupOrDefault(operand));
   if (!yieldResults.empty())
-    rewriter.create<scf::YieldOp>(source.getLoc(), yieldResults);
+    scf::YieldOp::create(rewriter, source.getLoc(), yieldResults);
 
   // Replace old loops by substituting their uses by results of the fused loop.
   rewriter.replaceOp(target, fusedLoop.getResults().take_front(numTargetOuts));
@@ -1487,7 +1487,7 @@ FailureOr<scf::ForallOp> mlir::normalizeForallOp(RewriterBase &rewriter,
 
   // Use the normalized builder since the lower bounds are always 0 and the
   // steps are always 1.
-  auto normalizedForallOp = rewriter.create<scf::ForallOp>(
+  auto normalizedForallOp = scf::ForallOp::create(rewriter,
       loc, newUbs, forallOp.getOutputs(), forallOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
 
diff --git a/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp b/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
index 66eed861b2bb7..48c0b1ed8e14f 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
@@ -30,14 +30,14 @@ Operation *SMTDialect::materializeConstant(OpBuilder &builder, Attribute value,
     if (auto attrValue = dyn_cast<BitVectorAttr>(value)) {
       assert(bvType == attrValue.getType() &&
              "attribute and desired result types have to match");
-      return builder.create<BVConstantOp>(loc, attrValue);
+      return BVConstantOp::create(builder, loc, attrValue);
     }
   }
 
   // BoolType constants can materialize into smt.constant
   if (auto boolType = dyn_cast<BoolType>(type)) {
     if (auto attrValue = dyn_cast<BoolAttr>(value))
-      return builder.create<BoolConstantOp>(loc, attrValue);
+      return BoolConstantOp::create(builder, loc, attrValue);
   }
 
   return nullptr;
diff --git a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
index 8977a3abc125d..c517ef2b18f6f 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
@@ -405,7 +405,7 @@ static void buildQuantifier(
         SmallVector<Location>(boundVarTypes.size(), odsState.location));
     Value returnVal =
         bodyBuilder(odsBuilder, odsState.location, block->getArguments());
-    odsBuilder.create<smt::YieldOp>(odsState.location, returnVal);
+    smt::YieldOp::create(odsBuilder, odsState.location, returnVal);
   }
   if (patternBuilder) {
     Region *region = odsState.addRegion();
@@ -416,7 +416,7 @@ static void buildQuantifier(
         SmallVector<Location>(boundVarTypes.size(), odsState.location));
     ValueRange returnVals =
         patternBuilder(odsBuilder, odsState.location, block->getArguments());
-    odsBuilder.create<smt::YieldOp>(odsState.location, returnVals);
+    smt::YieldOp::create(odsBuilder, odsState.location, returnVals);
   }
 }
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
index 371456552b5b5..cacc52d0feaf0 100644
--- a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
@@ -391,7 +391,7 @@ void LoopOp::addEntryAndMergeBlock(OpBuilder &builder) {
   builder.createBlock(&getBody());
 
   // Add a spirv.mlir.merge op into the merge block.
-  builder.create<spirv::MergeOp>(getLoc());
+  spirv::MergeOp::create(builder, getLoc());
 }
 
 //===----------------------------------------------------------------------===//
@@ -543,7 +543,7 @@ void SelectionOp::addMergeBlock(OpBuilder &builder) {
   builder.createBlock(&getBody());
 
   // Add a spirv.mlir.merge op into the merge block.
-  builder.create<spirv::MergeOp>(getLoc());
+  spirv::MergeOp::create(builder, getLoc());
 }
 
 SelectionOp
@@ -551,7 +551,7 @@ SelectionOp::createIfThen(Location loc, Value condition,
                           function_ref<void(OpBuilder &builder)> thenBody,
                           OpBuilder &builder) {
   auto selectionOp =
-      builder.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
+      spirv::SelectionOp::create(builder, loc, spirv::SelectionControl::None);
 
   selectionOp.addMergeBlock(builder);
   Block *mergeBlock = selectionOp.getMergeBlock();
@@ -562,14 +562,14 @@ SelectionOp::createIfThen(Location loc, Value condition,
     OpBuilder::InsertionGuard guard(builder);
     thenBlock = builder.createBlock(mergeBlock);
     thenBody(builder);
-    builder.create<spirv::BranchOp>(loc, mergeBlock);
+    spirv::BranchOp::create(builder, loc, mergeBlock);
   }
 
   // Build the header block.
   {
     OpBuilder::InsertionGuard guard(builder);
     builder.createBlock(thenBlock);
-    builder.create<spirv::BranchConditionalOp>(
+    spirv::BranchConditionalOp::create(builder,
         loc, condition, thenBlock,
         /*trueArguments=*/ArrayRef<Value>(), mergeBlock,
         /*falseArguments=*/ArrayRef<Value>());
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index 03af61c81ae6c..b1bf37092d2b6 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -179,16 +179,16 @@ struct IAddCarryFold final : OpRewritePattern<spirv::IAddCarryOp> {
       return failure();
 
     Value addsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, adds);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, adds);
 
     Value carrysVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, carrys);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, carrys);
 
     // Create empty struct
-    Value undef = rewriter.create<spirv::UndefOp>(loc, op.getType());
+    Value undef = spirv::UndefOp::create(rewriter, loc, op.getType());
     // Fill in adds at id 0
     Value intermediate =
-        rewriter.create<spirv::CompositeInsertOp>(loc, addsVal, undef, 0);
+        spirv::CompositeInsertOp::create(rewriter, loc, addsVal, undef, 0);
     // Fill in carrys at id 1
     rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(op, carrysVal,
                                                           intermediate, 1);
@@ -261,16 +261,16 @@ struct MulExtendedFold final : OpRewritePattern<MulOp> {
       return failure();
 
     Value lowBitsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, lowBits);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, lowBits);
 
     Value highBitsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, highBits);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, highBits);
 
     // Create empty struct
-    Value undef = rewriter.create<spirv::UndefOp>(loc, op.getType());
+    Value undef = spirv::UndefOp::create(rewriter, loc, op.getType());
     // Fill in lowBits at id 0
     Value intermediate =
-        rewriter.create<spirv::CompositeInsertOp>(loc, lowBitsVal, undef, 0);
+        spirv::CompositeInsertOp::create(rewriter, loc, lowBitsVal, undef, 0);
     // Fill in highBits at id 1
     rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(op, highBitsVal,
                                                           intermediate, 1);
@@ -1310,10 +1310,10 @@ struct ConvertSelectionOpToSelect final : OpRewritePattern<spirv::SelectionOp> {
     auto storeOpAttributes =
         cast<spirv::StoreOp>(trueBlock->front())->getAttrs();
 
-    auto selectOp = rewriter.create<spirv::SelectOp>(
+    auto selectOp = spirv::SelectOp::create(rewriter,
         selectionOp.getLoc(), trueValue.getType(),
         brConditionalOp.getCondition(), trueValue, falseValue);
-    rewriter.create<spirv::StoreOp>(selectOp.getLoc(), ptrValue,
+    spirv::StoreOp::create(rewriter, selectOp.getLoc(), ptrValue,
                                     selectOp.getResult(), storeOpAttributes);
 
     // `spirv.mlir.selection` is not needed anymore.
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
index 88c7adf3dfcb3..e85ea3455248f 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
@@ -944,12 +944,12 @@ Operation *SPIRVDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (!spirv::ConstantOp::isBuildableWith(type))
     return nullptr;
 
-  return builder.create<spirv::ConstantOp>(loc, type, value);
+  return spirv::ConstantOp::create(builder, loc, type, value);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index eb2974d62fdd1..f1299c8a926cd 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -653,25 +653,25 @@ spirv::ConstantOp spirv::ConstantOp::getZero(Type type, Location loc,
   if (auto intType = llvm::dyn_cast<IntegerType>(type)) {
     unsigned width = intType.getWidth();
     if (width == 1)
-      return builder.create<spirv::ConstantOp>(loc, type,
+      return spirv::ConstantOp::create(builder, loc, type,
                                                builder.getBoolAttr(false));
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getFloatAttr(floatType, 0.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 0).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 0.0).getValue()));
@@ -686,25 +686,25 @@ spirv::ConstantOp spirv::ConstantOp::getOne(Type type, Location loc,
   if (auto intType = llvm::dyn_cast<IntegerType>(type)) {
     unsigned width = intType.getWidth();
     if (width == 1)
-      return builder.create<spirv::ConstantOp>(loc, type,
+      return spirv::ConstantOp::create(builder, loc, type,
                                                builder.getBoolAttr(true));
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getFloatAttr(floatType, 1.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 1).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 1.0).getValue()));
@@ -1886,7 +1886,7 @@ ParseResult spirv::SpecConstantOperationOp::parse(OpAsmParser &parser,
 
   OpBuilder builder(parser.getContext());
   builder.setInsertionPointToEnd(&block);
-  builder.create<spirv::YieldOp>(wrappedOp->getLoc(), wrappedOp->getResult(0));
+  spirv::YieldOp::create(builder, wrappedOp->getLoc(), wrappedOp->getResult(0));
   result.location = wrappedOp->getLoc();
 
   result.addTypes(wrappedOp->getResult(0).getType());
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 71122f8e20512..4e53cef31d6ae 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -109,7 +109,7 @@ OwningOpRef<spirv::ModuleOp> combine(ArrayRef<spirv::ModuleOp> inputModules,
     }
   }
 
-  auto combinedModule = combinedModuleBuilder.create<spirv::ModuleOp>(
+  auto combinedModule = spirv::ModuleOp::create(combinedModuleBuilder,
       firstModule.getLoc(), addressingModel, memoryModel, vceTriple);
   combinedModuleBuilder.setInsertionPointToStart(combinedModule.getBody());
 
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 6fd20466e36e3..735af7ab80ef3 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -71,7 +71,7 @@ createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
   varType =
       spirv::PointerType::get(varPointeeType, varPtrType.getStorageClass());
 
-  return builder.create<spirv::GlobalVariableOp>(
+  return spirv::GlobalVariableOp::create(builder,
       funcOp.getLoc(), varType, varName, abiInfo.getDescriptorSet(),
       abiInfo.getBinding());
 }
@@ -147,7 +147,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     return funcOp.emitRemark("lower entry point failure: could not select "
                              "execution model based on 'spirv.target_env'");
 
-  builder.create<spirv::EntryPointOp>(funcOp.getLoc(), *executionModel, funcOp,
+  spirv::EntryPointOp::create(builder, funcOp.getLoc(), *executionModel, funcOp,
                                       interfaceVars);
 
   // Specifies the spirv.ExecutionModeOp.
@@ -155,7 +155,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::LocalSize);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
                                              spirv::ExecutionMode::LocalSize,
                                              workgroupSizeAttr.asArrayRef());
       // Erase workgroup size.
@@ -168,7 +168,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SubgroupSize);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
                                              spirv::ExecutionMode::SubgroupSize,
                                              *subgroupSize);
       // Erase subgroup size.
@@ -181,7 +181,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SignedZeroInfNanPreserve);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(
+      spirv::ExecutionModeOp::create(builder,
           funcOp.getLoc(), funcOp,
           spirv::ExecutionMode::SignedZeroInfNanPreserve, *targetWidth);
       // Erase target width.
@@ -260,7 +260,7 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
 
     // Insert spirv::AddressOf and spirv::AccessChain operations.
     Value replacement =
-        rewriter.create<spirv::AddressOfOp>(funcOp.getLoc(), var);
+        spirv::AddressOfOp::create(rewriter, funcOp.getLoc(), var);
     // Check if the arg is a scalar or vector type. In that case, the value
     // needs to be loaded into registers.
     // TODO: This is loading value of the scalar into registers
@@ -270,9 +270,9 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     if (cast<spirv::SPIRVType>(argType.value()).isScalarOrVector()) {
       auto zero =
           spirv::ConstantOp::getZero(indexType, funcOp.getLoc(), rewriter);
-      auto loadPtr = rewriter.create<spirv::AccessChainOp>(
+      auto loadPtr = spirv::AccessChainOp::create(rewriter,
           funcOp.getLoc(), replacement, zero.getConstant());
-      replacement = rewriter.create<spirv::LoadOp>(funcOp.getLoc(), loadPtr);
+      replacement = spirv::LoadOp::create(rewriter, funcOp.getLoc(), loadPtr);
     }
     signatureConverter.remapInput(argType.index(), replacement);
   }
@@ -309,7 +309,7 @@ void LowerABIAttributesPass::runOnOperation() {
                                             ValueRange inputs, Location loc) {
     if (inputs.size() != 1 || !isa<spirv::PointerType>(inputs[0].getType()))
       return Value();
-    return builder.create<spirv::BitcastOp>(loc, type, inputs[0]).getResult();
+    return spirv::BitcastOp::create(builder, loc, type, inputs[0]).getResult();
   });
 
   RewritePatternSet patterns(context);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
index 2e31172ab940b..637be91a82e33 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
@@ -66,7 +66,7 @@ void RewriteInsertsPass::runOnOperation() {
       operands.push_back(insertionOp.getObject());
 
     OpBuilder builder(lastCompositeInsertOp);
-    auto compositeConstructOp = builder.create<spirv::CompositeConstructOp>(
+    auto compositeConstructOp = spirv::CompositeConstructOp::create(builder,
         location, compositeType, operands);
 
     lastCompositeInsertOp.replaceAllUsesWith(
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index 1e7bb046d3752..4e398ca92be0e 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -672,21 +672,21 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
                               Location loc) {
   // We can only cast one value in SPIR-V.
   if (inputs.size() != 1) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   Value input = inputs.front();
 
   // Only support integer types for now. Floating point types to be implemented.
   if (!isa<IntegerType>(type)) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   auto inputType = cast<IntegerType>(input.getType());
 
   auto scalarType = dyn_cast<spirv::ScalarType>(type);
   if (!scalarType) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -694,14 +694,14 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // truncating to go back so we don't need to worry about the signedness.
   // For extension, we cannot have enough signal here to decide which op to use.
   if (inputType.getIntOrFloatBitWidth() < scalarType.getIntOrFloatBitWidth()) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
   // Boolean values would need to use different ops than normal integer values.
   if (type.isInteger(1)) {
     Value one = spirv::ConstantOp::getOne(inputType, loc, builder);
-    return builder.create<spirv::IEqualOp>(loc, input, one);
+    return spirv::IEqualOp::create(builder, loc, input, one);
   }
 
   // Check that the source integer type is supported by the environment.
@@ -711,7 +711,7 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   scalarType.getCapabilities(caps);
   if (failed(checkCapabilityRequirements(type, targetEnv, caps)) ||
       failed(checkExtensionRequirements(type, targetEnv, exts))) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -719,9 +719,9 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // care about signedness here. Still try to use a corresponding op for better
   // consistency though.
   if (type.isSignedInteger()) {
-    return builder.create<spirv::SConvertOp>(loc, type, input);
+    return spirv::SConvertOp::create(builder, loc, type, input);
   }
-  return builder.create<spirv::UConvertOp>(loc, type, input);
+  return spirv::UConvertOp::create(builder, loc, type, input);
 }
 
 //===----------------------------------------------------------------------===//
@@ -773,7 +773,7 @@ getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
                                            spirv::StorageClass::Input);
     std::string name = getBuiltinVarName(builtin, prefix, suffix);
     newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+        spirv::GlobalVariableOp::create(builder, loc, ptrType, name, builtin);
     break;
   }
   case spirv::BuiltIn::SubgroupId:
@@ -784,7 +784,7 @@ getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
         spirv::PointerType::get(integerType, spirv::StorageClass::Input);
     std::string name = getBuiltinVarName(builtin, prefix, suffix);
     newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+        spirv::GlobalVariableOp::create(builder, loc, ptrType, name, builtin);
     break;
   }
   default:
@@ -845,7 +845,7 @@ getOrInsertPushConstantVariable(Location loc, Block &block,
   auto builder = OpBuilder::atBlockBegin(&block, b.getListener());
   auto type = getPushConstantStorageType(elementCount, builder, indexType);
   const char *name = "__push_constant_var__";
-  return builder.create<spirv::GlobalVariableOp>(loc, type, name,
+  return spirv::GlobalVariableOp::create(builder, loc, type, name,
                                                  /*initializer=*/nullptr);
 }
 
@@ -882,7 +882,7 @@ struct FuncOpConversion final : OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted spirv.func op.
-    auto newFuncOp = rewriter.create<spirv::FuncOp>(
+    auto newFuncOp = spirv::FuncOp::create(rewriter,
         funcOp.getLoc(), funcOp.getName(),
         rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
                                  resultType ? TypeRange(resultType)
@@ -922,7 +922,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
     }
 
     // Create a new func op with the original type and copy the function body.
-    auto newFuncOp = rewriter.create<func::FuncOp>(funcOp.getLoc(),
+    auto newFuncOp = func::FuncOp::create(rewriter, funcOp.getLoc(),
                                                    funcOp.getName(), fnType);
     rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
                                 newFuncOp.end());
@@ -957,7 +957,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto origVecType = dyn_cast<VectorType>(origType);
       if (!origVecType) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = rewriter.create<arith::ConstantOp>(
+        Value result = arith::ConstantOp::create(rewriter,
             loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
@@ -970,7 +970,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto targetShape = getTargetShape(origVecType);
       if (!targetShape) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = rewriter.create<arith::ConstantOp>(
+        Value result = arith::ConstantOp::create(rewriter,
             loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
@@ -985,11 +985,11 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
           llvm::to_vector_of<int64_t, 4>(origVecType.getShape());
 
       // Prepare the result vector.
-      Value result = rewriter.create<arith::ConstantOp>(
+      Value result = arith::ConstantOp::create(rewriter,
           loc, origVecType, rewriter.getZeroAttr(origVecType));
       ++newOpCount;
       // Prepare the placeholder for the new arguments that will be added later.
-      Value dummy = rewriter.create<arith::ConstantOp>(
+      Value dummy = arith::ConstantOp::create(rewriter,
           loc, unrolledType, rewriter.getZeroAttr(unrolledType));
       ++newOpCount;
 
@@ -998,7 +998,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       SmallVector<Type> newTypes;
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        result = rewriter.create<vector::InsertStridedSliceOp>(
+        result = vector::InsertStridedSliceOp::create(rewriter,
             loc, dummy, result, offsets, strides);
         newTypes.push_back(unrolledType);
         unrolledInputNums.push_back(newInputNo);
@@ -1112,12 +1112,12 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
       Value returnValue = returnOp.getOperand(origResultNo);
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        Value result = rewriter.create<vector::ExtractStridedSliceOp>(
+        Value result = vector::ExtractStridedSliceOp::create(rewriter,
             loc, returnValue, offsets, extractShape, strides);
         if (originalShape.size() > 1) {
           SmallVector<int64_t> extractIndices(originalShape.size() - 1, 0);
           result =
-              rewriter.create<vector::ExtractOp>(loc, result, extractIndices);
+              vector::ExtractOp::create(rewriter, loc, result, extractIndices);
         }
         newOperands.push_back(result);
         newTypes.push_back(unrolledType);
@@ -1135,7 +1135,7 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
     // Replace the return op using the new operands. This will automatically
     // update the entry block as well.
     rewriter.replaceOp(returnOp,
-                       rewriter.create<func::ReturnOp>(loc, newOperands));
+                       func::ReturnOp::create(rewriter, loc, newOperands));
 
     return success();
   }
@@ -1160,8 +1160,8 @@ Value mlir::spirv::getBuiltinVariableValue(Operation *op,
   spirv::GlobalVariableOp varOp =
       getOrInsertBuiltinVariable(*parent->getRegion(0).begin(), op->getLoc(),
                                  builtin, integerType, builder, prefix, suffix);
-  Value ptr = builder.create<spirv::AddressOfOp>(op->getLoc(), varOp);
-  return builder.create<spirv::LoadOp>(op->getLoc(), ptr);
+  Value ptr = spirv::AddressOfOp::create(builder, op->getLoc(), varOp);
+  return spirv::LoadOp::create(builder, op->getLoc(), ptr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1182,12 +1182,12 @@ Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
       loc, parent->getRegion(0).front(), elementCount, builder, integerType);
 
   Value zeroOp = spirv::ConstantOp::getZero(integerType, loc, builder);
-  Value offsetOp = builder.create<spirv::ConstantOp>(
+  Value offsetOp = spirv::ConstantOp::create(builder,
       loc, integerType, builder.getI32IntegerAttr(offset));
-  auto addrOp = builder.create<spirv::AddressOfOp>(loc, varOp);
-  auto acOp = builder.create<spirv::AccessChainOp>(
+  auto addrOp = spirv::AddressOfOp::create(builder, loc, varOp);
+  auto acOp = spirv::AccessChainOp::create(builder,
       loc, addrOp, llvm::ArrayRef({zeroOp, offsetOp}));
-  return builder.create<spirv::LoadOp>(loc, acOp);
+  return spirv::LoadOp::create(builder, loc, acOp);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1247,7 +1247,7 @@ Value mlir::spirv::getVulkanElementPtr(const SPIRVTypeConverter &typeConverter,
     linearizedIndices.push_back(
         linearizeIndex(indices, strides, offset, indexType, loc, builder));
   }
-  return builder.create<spirv::AccessChainOp>(loc, basePtr, linearizedIndices);
+  return spirv::AccessChainOp::create(builder, loc, basePtr, linearizedIndices);
 }
 
 Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
@@ -1278,10 +1278,10 @@ Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
       cast<spirv::PointerType>(basePtr.getType()).getPointeeType();
   if (isa<spirv::ArrayType>(pointeeType)) {
     linearizedIndices.push_back(linearIndex);
-    return builder.create<spirv::AccessChainOp>(loc, basePtr,
+    return spirv::AccessChainOp::create(builder, loc, basePtr,
                                                 linearizedIndices);
   }
-  return builder.create<spirv::PtrAccessChainOp>(loc, basePtr, linearIndex,
+  return spirv::PtrAccessChainOp::create(builder, loc, basePtr, linearIndex,
                                                  linearizedIndices);
 }
 
@@ -1468,7 +1468,7 @@ SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr,
       });
   addTargetMaterialization([](OpBuilder &builder, Type type, ValueRange inputs,
                               Location loc) {
-    auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return cast.getResult(0);
   });
 }
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
index af1cf2a1373e3..17f79780afa20 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
@@ -64,16 +64,16 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   //     and 4 additions after constant folding.
   //   - With sign-extended arguments, we end up emitting 8 multiplications and
   //     and 12 additions after CSE.
-  Value cstLowMask = rewriter.create<ConstantOp>(
+  Value cstLowMask = ConstantOp::create(rewriter,
       loc, lhs.getType(), getScalarOrSplatAttr(argTy, (1 << 16) - 1));
   auto getLowDigit = [&rewriter, loc, cstLowMask](Value val) {
-    return rewriter.create<BitwiseAndOp>(loc, val, cstLowMask);
+    return BitwiseAndOp::create(rewriter, loc, val, cstLowMask);
   };
 
-  Value cst16 = rewriter.create<ConstantOp>(loc, lhs.getType(),
+  Value cst16 = ConstantOp::create(rewriter, loc, lhs.getType(),
                                             getScalarOrSplatAttr(argTy, 16));
   auto getHighDigit = [&rewriter, loc, cst16](Value val) {
-    return rewriter.create<ShiftRightLogicalOp>(loc, val, cst16);
+    return ShiftRightLogicalOp::create(rewriter, loc, val, cst16);
   };
 
   auto getSignDigit = [&rewriter, loc, cst16, &getHighDigit](Value val) {
@@ -82,10 +82,10 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
     // fine. We do not have to introduce an extra constant since any
     // value in [15, 32) would do.
     return getHighDigit(
-        rewriter.create<ShiftRightArithmeticOp>(loc, val, cst16));
+        ShiftRightArithmeticOp::create(rewriter, loc, val, cst16));
   };
 
-  Value cst0 = rewriter.create<ConstantOp>(loc, lhs.getType(),
+  Value cst0 = ConstantOp::create(rewriter, loc, lhs.getType(),
                                            getScalarOrSplatAttr(argTy, 0));
 
   Value lhsLow = getLowDigit(lhs);
@@ -108,7 +108,7 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
         continue;
 
       Value &thisResDigit = resultDigits[i + j];
-      Value mul = rewriter.create<IMulOp>(loc, lhsDigit, rhsDigit);
+      Value mul = IMulOp::create(rewriter, loc, lhsDigit, rhsDigit);
       Value current = rewriter.createOrFold<IAddOp>(loc, thisResDigit, mul);
       thisResDigit = getLowDigit(current);
 
@@ -122,13 +122,13 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   }
 
   auto combineDigits = [loc, cst16, &rewriter](Value low, Value high) {
-    Value highBits = rewriter.create<ShiftLeftLogicalOp>(loc, high, cst16);
-    return rewriter.create<BitwiseOrOp>(loc, low, highBits);
+    Value highBits = ShiftLeftLogicalOp::create(rewriter, loc, high, cst16);
+    return BitwiseOrOp::create(rewriter, loc, low, highBits);
   };
   Value low = combineDigits(resultDigits[0], resultDigits[1]);
   Value high = combineDigits(resultDigits[2], resultDigits[3]);
 
-  return rewriter.create<CompositeConstructOp>(
+  return CompositeConstructOp::create(rewriter,
       loc, mulOp->getResultTypes().front(), llvm::ArrayRef({low, high}));
 }
 
@@ -185,16 +185,16 @@ struct ExpandAddCarryPattern final : OpRewritePattern<IAddCarryOp> {
           llvm::formatv("Unexpected integer type for WebGPU: '{0}'", elemTy));
 
     Value one =
-        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 1));
+        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 1));
     Value zero =
-        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 0));
+        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 0));
 
     // Calculate the carry by checking if the addition resulted in an overflow.
-    Value out = rewriter.create<IAddOp>(loc, lhs, rhs);
-    Value cmp = rewriter.create<ULessThanOp>(loc, out, lhs);
-    Value carry = rewriter.create<SelectOp>(loc, cmp, one, zero);
+    Value out = IAddOp::create(rewriter, loc, lhs, rhs);
+    Value cmp = ULessThanOp::create(rewriter, loc, out, lhs);
+    Value carry = SelectOp::create(rewriter, loc, cmp, one, zero);
 
-    Value add = rewriter.create<CompositeConstructOp>(
+    Value add = CompositeConstructOp::create(rewriter,
         loc, op->getResultTypes().front(), llvm::ArrayRef({out, carry}));
 
     rewriter.replaceOp(op, add);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
index 07cf26926a1df..7fe2bd79403f9 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
@@ -384,13 +384,13 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = dstNumBytes / srcNumBytes;
-      auto ratioValue = rewriter.create<spirv::ConstantOp>(
+      auto ratioValue = spirv::ConstantOp::create(rewriter,
           loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
-          rewriter.create<spirv::SDivOp>(loc, indexType, oldIndex, ratioValue);
+          spirv::SDivOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
       indices.push_back(
-          rewriter.create<spirv::SModOp>(loc, indexType, oldIndex, ratioValue));
+          spirv::SModOp::create(rewriter, loc, indexType, oldIndex, ratioValue));
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -411,11 +411,11 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = srcNumBytes / dstNumBytes;
-      auto ratioValue = rewriter.create<spirv::ConstantOp>(
+      auto ratioValue = spirv::ConstantOp::create(rewriter,
           loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
-          rewriter.create<spirv::IMulOp>(loc, indexType, oldIndex, ratioValue);
+          spirv::IMulOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -439,14 +439,14 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
     auto dstElemType = cast<spirv::SPIRVType>(dstPtrType.getPointeeType());
 
     Location loc = loadOp.getLoc();
-    auto newLoadOp = rewriter.create<spirv::LoadOp>(loc, adaptor.getPtr());
+    auto newLoadOp = spirv::LoadOp::create(rewriter, loc, adaptor.getPtr());
     if (srcElemType == dstElemType) {
       rewriter.replaceOp(loadOp, newLoadOp->getResults());
       return success();
     }
 
     if (areSameBitwidthScalarType(srcElemType, dstElemType)) {
-      auto castOp = rewriter.create<spirv::BitcastOp>(loc, srcElemType,
+      auto castOp = spirv::BitcastOp::create(rewriter, loc, srcElemType,
                                                       newLoadOp.getValue());
       rewriter.replaceOp(loadOp, castOp->getResults());
 
@@ -479,14 +479,14 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
       auto indices = llvm::to_vector<4>(acOp.getIndices());
       for (int i = 1; i < ratio; ++i) {
         // Load all subsequent components belonging to this element.
-        indices.back() = rewriter.create<spirv::IAddOp>(
+        indices.back() = spirv::IAddOp::create(rewriter,
             loc, i32Type, indices.back(), oneValue);
-        auto componentAcOp = rewriter.create<spirv::AccessChainOp>(
+        auto componentAcOp = spirv::AccessChainOp::create(rewriter,
             loc, acOp.getBasePtr(), indices);
         // Assuming little endian, this reads lower-ordered bits of the number
         // to lower-numbered components of the vector.
         components.push_back(
-            rewriter.create<spirv::LoadOp>(loc, componentAcOp));
+            spirv::LoadOp::create(rewriter, loc, componentAcOp));
       }
 
       // Create a vector of the components and then cast back to the larger
@@ -514,15 +514,15 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
               castType = VectorType::get({count}, castType);
 
             for (Value &c : components)
-              c = rewriter.create<spirv::BitcastOp>(loc, castType, c);
+              c = spirv::BitcastOp::create(rewriter, loc, castType, c);
           }
         }
-      Value vectorValue = rewriter.create<spirv::CompositeConstructOp>(
+      Value vectorValue = spirv::CompositeConstructOp::create(rewriter,
           loc, vectorType, components);
 
       if (!isa<VectorType>(srcElemType))
         vectorValue =
-            rewriter.create<spirv::BitcastOp>(loc, srcElemType, vectorValue);
+            spirv::BitcastOp::create(rewriter, loc, srcElemType, vectorValue);
       rewriter.replaceOp(loadOp, vectorValue);
       return success();
     }
@@ -550,7 +550,7 @@ struct ConvertStore : public ConvertAliasResource<spirv::StoreOp> {
     Location loc = storeOp.getLoc();
     Value value = adaptor.getValue();
     if (srcElemType != dstElemType)
-      value = rewriter.create<spirv::BitcastOp>(loc, dstElemType, value);
+      value = spirv::BitcastOp::create(rewriter, loc, dstElemType, value);
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, adaptor.getPtr(),
                                                 value, storeOp->getAttrs());
     return success();
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 5f395eef9d601..7339ba5c2379c 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -151,16 +151,16 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (llvm::isa<ShapeType>(type) || isExtentTensorType(type))
-    return builder.create<ConstShapeOp>(
+    return ConstShapeOp::create(builder,
         loc, type, llvm::cast<DenseIntElementsAttr>(value));
   if (llvm::isa<SizeType>(type))
-    return builder.create<ConstSizeOp>(loc, type,
+    return ConstSizeOp::create(builder, loc, type,
                                        llvm::cast<IntegerAttr>(value));
   if (llvm::isa<WitnessType>(type))
-    return builder.create<ConstWitnessOp>(loc, type,
+    return ConstWitnessOp::create(builder, loc, type,
                                           llvm::cast<BoolAttr>(value));
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
@@ -316,7 +316,7 @@ struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
     auto newYieldOp =
         rewriter.replaceOpWithNewOp<AssumingYieldOp>(yieldOp, newYieldOperands);
     rewriter.setInsertionPoint(op);
-    auto newOp = rewriter.create<AssumingOp>(
+    auto newOp = AssumingOp::create(rewriter,
         op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
     newOp.getDoRegion().takeBody(op.getDoRegion());
 
@@ -385,7 +385,7 @@ void AssumingOp::build(
 
   // Build body.
   SmallVector<Value, 2> yieldValues = bodyBuilder(builder, result.location);
-  builder.create<AssumingYieldOp>(result.location, yieldValues);
+  AssumingYieldOp::create(builder, result.location, yieldValues);
 
   SmallVector<Type, 2> assumingTypes;
   for (Value v : yieldValues)
@@ -736,13 +736,13 @@ struct BroadcastForwardSingleOperandPattern
     if (replacement.getType() != op.getType()) {
       auto loc = op.getLoc();
       if (llvm::isa<ShapeType>(op.getType())) {
-        replacement = rewriter.create<FromExtentTensorOp>(loc, replacement);
+        replacement = FromExtentTensorOp::create(rewriter, loc, replacement);
       } else {
         assert(!llvm::isa<ShapeType>(op.getType()) &&
                !llvm::isa<ShapeType>(replacement.getType()) &&
                "expect extent tensor cast");
         replacement =
-            rewriter.create<tensor::CastOp>(loc, op.getType(), replacement);
+            tensor::CastOp::create(rewriter, loc, op.getType(), replacement);
       }
     }
 
@@ -780,7 +780,7 @@ struct BroadcastFoldConstantOperandsPattern
     auto foldedConstantOperandsTy = RankedTensorType::get(
         {static_cast<int64_t>(foldedConstantShape.size())},
         rewriter.getIndexType());
-    newShapeOperands.push_back(rewriter.create<ConstShapeOp>(
+    newShapeOperands.push_back(ConstShapeOp::create(rewriter,
         op.getLoc(), foldedConstantOperandsTy,
         rewriter.getIndexTensorAttr(foldedConstantShape)));
     rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(),
@@ -845,7 +845,7 @@ struct BroadcastConcretizeResultTypePattern
       }
     }
 
-    auto newOp = rewriter.create<BroadcastOp>(
+    auto newOp = BroadcastOp::create(rewriter,
         op.getLoc(), getExtentTensorType(getContext(), maxRank),
         op.getShapes());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
@@ -1354,11 +1354,11 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
   auto loc = result.location;
   auto dimAttr = builder.getIndexAttr(dim);
   if (llvm::isa<ShapeType>(shape.getType())) {
-    Value dim = builder.create<ConstSizeOp>(loc, dimAttr);
+    Value dim = ConstSizeOp::create(builder, loc, dimAttr);
     build(builder, result, builder.getType<SizeType>(), shape, dim);
   } else {
     Value dim =
-        builder.create<arith::ConstantOp>(loc, builder.getIndexType(), dimAttr);
+        arith::ConstantOp::create(builder, loc, builder.getIndexType(), dimAttr);
     build(builder, result, builder.getIndexType(), shape, dim);
   }
 }
@@ -1708,7 +1708,7 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
                                   rewriter.getIndexTensorAttr(type.getShape()))
             .getResult();
     if (constShape.getType() != op.getResult().getType())
-      constShape = rewriter.create<tensor::CastOp>(
+      constShape = tensor::CastOp::create(rewriter,
           loc, op.getResult().getType(), constShape);
     rewriter.replaceOp(op, constShape);
     return success();
@@ -1751,10 +1751,10 @@ struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
 
     if (opTensorTy != shapeTensorTy) {
       if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
-        shape = rewriter.create<tensor::CastOp>(op.getLoc(), opTensorTy, shape);
+        shape = tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
       else if (!isExtentTensorType(shapeTensorTy))
         shape =
-            rewriter.create<arith::IndexCastOp>(op.getLoc(), opTensorTy, shape);
+            arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
     }
 
     rewriter.replaceOp(op, shape);
diff --git a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
index 8a471c12d21e4..5533a7e914bf0 100644
--- a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -57,7 +57,7 @@ struct AssumingOpInterface
 
     // Create new op and move over region.
     TypeRange newResultTypes(yieldOp.getOperands());
-    auto newOp = rewriter.create<shape::AssumingOp>(
+    auto newOp = shape::AssumingOp::create(rewriter,
         op->getLoc(), newResultTypes, assumingOp.getWitness());
     newOp.getDoRegion().takeBody(assumingOp.getRegion());
 
@@ -66,7 +66,7 @@ struct AssumingOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(assumingOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
+        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
             assumingOp.getLoc(), it.value(), newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
diff --git a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
index ae06a34b65709..522347ff2b993 100644
--- a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
@@ -69,7 +69,7 @@ createFuncFromCluster(OpBuilder &b, const SmallVector<Operation *, 8> &cluster,
       cluster.empty()
           ? b.getFunctionType(shape.getType(), shape.getType())
           : b.getFunctionType(ValueRange(inputs).getTypes(), shape.getType());
-  shape::FuncOp fnOp = b.create<shape::FuncOp>(loc, fnName, fnType);
+  shape::FuncOp fnOp = shape::FuncOp::create(b, loc, fnName, fnType);
   Block *block = fnOp.addEntryBlock();
   b.setInsertionPointToEnd(block);
   IRMapping bvm;
@@ -85,7 +85,7 @@ createFuncFromCluster(OpBuilder &b, const SmallVector<Operation *, 8> &cluster,
   llvm::SmallVector<Value, 4> fnReturns;
   fnReturns.push_back(bvm.lookupOrDefault(shape));
 
-  b.create<shape::ReturnOp>(loc, fnReturns);
+  shape::ReturnOp::create(b, loc, fnReturns);
   fnOp.setPrivate();
   return std::make_pair(fnOp, inputs);
 }
@@ -187,7 +187,7 @@ class TensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
   LogicalResult matchAndRewrite(tensor::DimOp op,
                                 PatternRewriter &rewriter) const override {
     auto shapeOf =
-        rewriter.create<shape::ShapeOfOp>(op.getLoc(), op.getSource());
+        shape::ShapeOfOp::create(rewriter, op.getLoc(), op.getSource());
     rewriter.replaceOpWithNewOp<shape::GetExtentOp>(op, op.getType(), shapeOf,
                                                     op.getIndex());
     return success();
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index 121e0cc133e19..b5f6230bc2362 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -44,14 +44,14 @@ NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
                    ->materializeConstant(rewriter, rewriter.getIndexAttr(1),
                                          valueType, loc)
                    ->getResult(0);
-  ReduceOp reduce = rewriter.create<ReduceOp>(loc, op.getShape(), init);
+  ReduceOp reduce = ReduceOp::create(rewriter, loc, op.getShape(), init);
 
   // Generate reduce operator.
   Block *body = reduce.getBody();
   OpBuilder b = OpBuilder::atBlockEnd(body);
-  Value product = b.create<MulOp>(loc, valueType, body->getArgument(1),
+  Value product = MulOp::create(b, loc, valueType, body->getArgument(1),
                                   body->getArgument(2));
-  b.create<shape::YieldOp>(loc, product);
+  shape::YieldOp::create(b, loc, product);
 
   rewriter.replaceOp(op, reduce.getResult());
   return success();
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index ea7918fa14f23..d5e709bbed221 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -561,7 +561,7 @@ SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
   SmallVector<Type> retType(
       dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
       builder.getIndexType());
-  auto transOp = builder.create<CrdTranslateOp>(loc, retType, crds, dir, *this);
+  auto transOp = CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
   return transOp.getOutCrds();
 }
 
@@ -1483,7 +1483,7 @@ LogicalResult CrdTranslateOp::fold(FoldAdaptor adaptor,
 
 void LvlOp::build(OpBuilder &builder, OperationState &state, Value source,
                   int64_t index) {
-  Value val = builder.create<arith::ConstantIndexOp>(state.location, index);
+  Value val = arith::ConstantIndexOp::create(builder, state.location, index);
   return build(builder, state, source, val);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
index 9c84f4c25866f..74842cc1f310b 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
@@ -41,7 +41,7 @@ LogicalResult sparse_tensor::detail::stageWithSortImpl(
 
   // -> sort
   Type dstCOOTp = dstStt.getCOOType(/*ordered=*/true);
-  Value dstCOO = rewriter.create<ReorderCOOOp>(
+  Value dstCOO = ReorderCOOOp::create(rewriter,
       loc, dstCOOTp, srcCOO, SparseTensorSortKind::HybridQuickSort);
 
   // -> dest.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
index 5461987fb49d9..dce6e438d55d0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
@@ -90,13 +90,13 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
         } else if (directOut) {
           Value mem;
           if (kind == SparseTensorFieldKind::PosMemRef)
-            mem = builder.create<sparse_tensor::ToPositionsOp>(loc, inputs[0],
+            mem = sparse_tensor::ToPositionsOp::create(builder, loc, inputs[0],
                                                                lv);
           else if (kind == SparseTensorFieldKind::CrdMemRef)
-            mem = builder.create<sparse_tensor::ToCoordinatesOp>(loc, inputs[0],
+            mem = sparse_tensor::ToCoordinatesOp::create(builder, loc, inputs[0],
                                                                  lv);
           else
-            mem = builder.create<sparse_tensor::ToValuesOp>(loc, inputs[0]);
+            mem = sparse_tensor::ToValuesOp::create(builder, loc, inputs[0]);
           toVals.push_back(mem);
         } else {
           ShapedType rtp = cast<ShapedType>(t);
@@ -111,7 +111,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
 
     if (isIn) {
       // Assemble multiple inputs into a single sparse tensor.
-      auto a = builder.create<sparse_tensor::AssembleOp>(loc, rtp, inputs);
+      auto a = sparse_tensor::AssembleOp::create(builder, loc, rtp, inputs);
       toVals.push_back(a.getResult());
     } else if (!directOut) {
       // Disassemble a single sparse input into multiple outputs.
@@ -119,7 +119,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
       unsigned len = retTypes.size();
       retTypes.append(cntTypes);
       auto d =
-          builder.create<sparse_tensor::DisassembleOp>(loc, retTypes, inputs);
+          sparse_tensor::DisassembleOp::create(builder, loc, retTypes, inputs);
       for (unsigned i = 0; i < len; i++)
         toVals.push_back(d.getResult(i));
     }
@@ -201,7 +201,7 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     OpBuilder moduleBuilder(modOp.getBodyRegion());
     unsigned extra = inputTypes.size();
     inputTypes.append(extraTypes);
-    auto func = moduleBuilder.create<func::FuncOp>(
+    auto func = func::FuncOp::create(moduleBuilder,
         loc, orgName, FunctionType::get(context, inputTypes, outputTypes));
     func.setPublic();
 
@@ -218,14 +218,14 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     // Call the original, now private method. A subsequent inlining pass can
     // determine whether cloning the method body in place is worthwhile.
     auto org = SymbolRefAttr::get(context, wrapper);
-    auto call = rewriter.create<func::CallOp>(loc, funcOp.getResultTypes(), org,
+    auto call = func::CallOp::create(rewriter, loc, funcOp.getResultTypes(), org,
                                               inputs);
 
     // Convert outputs and return.
     SmallVector<Value> outputs;
     convVals(rewriter, loc, funcOp.getResultTypes(), call.getResults(),
              body->getArguments(), outputs, extra, /*isIn=*/false, directOut);
-    rewriter.create<func::ReturnOp>(loc, outputs);
+    func::ReturnOp::create(rewriter, loc, outputs);
 
     // Finally, migrate a potential c-interface property.
     if (funcOp->getAttrOfType<UnitAttr>(
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
index 0c5912bb73772..12507ff1c5251 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
@@ -94,7 +94,7 @@ static FlatSymbolRefAttr getMangledSortHelperFunc(
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPoint(insertPoint);
     Location loc = insertPoint.getLoc();
-    func = builder.create<func::FuncOp>(
+    func = func::FuncOp::create(builder,
         loc, nameOstream.str(),
         FunctionType::get(context, operands.getTypes(), resultTypes));
     func.setPrivate();
@@ -111,13 +111,13 @@ static void forEachIJPairInXs(
     uint64_t ny,
     function_ref<void(uint64_t, Value, Value, Value)> bodyBuilder) {
   Value cstep = constantIndex(builder, loc, xPerm.getNumResults() + ny);
-  Value iOffset = builder.create<arith::MulIOp>(loc, args[0], cstep);
-  Value jOffset = builder.create<arith::MulIOp>(loc, args[1], cstep);
+  Value iOffset = arith::MulIOp::create(builder, loc, args[0], cstep);
+  Value jOffset = arith::MulIOp::create(builder, loc, args[1], cstep);
   for (unsigned k = 0, e = xPerm.getNumResults(); k < e; k++) {
     unsigned actualK = cast<AffineDimExpr>(xPerm.getResult(k)).getPosition();
     Value ak = constantIndex(builder, loc, actualK);
-    Value i = builder.create<arith::AddIOp>(loc, ak, iOffset);
-    Value j = builder.create<arith::AddIOp>(loc, ak, jOffset);
+    Value i = arith::AddIOp::create(builder, loc, ak, iOffset);
+    Value j = arith::AddIOp::create(builder, loc, ak, jOffset);
     Value buffer = args[xStartIdx];
 
     bodyBuilder(k, i, j, buffer);
@@ -165,10 +165,10 @@ static void forEachIJPairInAllBuffers(
 static void createSwap(OpBuilder &builder, Location loc, ValueRange args,
                        AffineMap xPerm, uint64_t ny) {
   auto swapOnePair = [&](uint64_t unused, Value i, Value j, Value buffer) {
-    Value vi = builder.create<memref::LoadOp>(loc, buffer, i);
-    Value vj = builder.create<memref::LoadOp>(loc, buffer, j);
-    builder.create<memref::StoreOp>(loc, vj, buffer, i);
-    builder.create<memref::StoreOp>(loc, vi, buffer, j);
+    Value vi = memref::LoadOp::create(builder, loc, buffer, i);
+    Value vj = memref::LoadOp::create(builder, loc, buffer, j);
+    memref::StoreOp::create(builder, loc, vj, buffer, i);
+    memref::StoreOp::create(builder, loc, vi, buffer, j);
   };
 
   forEachIJPairInAllBuffers(builder, loc, args, xPerm, ny, swapOnePair);
@@ -193,7 +193,7 @@ static Value createInlinedCompareImplementation(
       OpBuilder::InsertionGuard insertionGuard(builder);
       auto ifOp = cast<scf::IfOp>(val.getDefiningOp());
       builder.setInsertionPointAfter(ifOp);
-      builder.create<scf::YieldOp>(loc, ifOp.getResult(0));
+      scf::YieldOp::create(builder, loc, ifOp.getResult(0));
     }
   };
 
@@ -207,25 +207,25 @@ static Value createInlinedCompareImplementation(
 /// result of the comparison.
 static Value createEqCompare(OpBuilder &builder, Location loc, Value i, Value j,
                              Value x, bool isFirstDim, bool isLastDim) {
-  Value vi = builder.create<memref::LoadOp>(loc, x, i);
-  Value vj = builder.create<memref::LoadOp>(loc, x, j);
+  Value vi = memref::LoadOp::create(builder, loc, x, i);
+  Value vj = memref::LoadOp::create(builder, loc, x, j);
 
   Value res;
   if (isLastDim) {
-    res = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, vi, vj);
+    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
-      builder.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(builder, loc, res);
   } else {
     Value ne =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, vi, vj);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIntegerType(1),
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
                                                ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value f = constantI1(builder, loc, false);
-    builder.create<scf::YieldOp>(loc, f);
+    scf::YieldOp::create(builder, loc, f);
 
     // If (x[i] == x[j]). Set up the insertion point for the nested if-stmt that
     // checks the remaining dimensions.
@@ -261,26 +261,26 @@ static Value createInlinedEqCompare(OpBuilder &builder, Location loc,
 static Value createLessThanCompare(OpBuilder &builder, Location loc, Value i,
                                    Value j, Value x, bool isFirstDim,
                                    bool isLastDim) {
-  Value vi = builder.create<memref::LoadOp>(loc, x, i);
-  Value vj = builder.create<memref::LoadOp>(loc, x, j);
+  Value vi = memref::LoadOp::create(builder, loc, x, i);
+  Value vj = memref::LoadOp::create(builder, loc, x, j);
 
   Value res;
   if (isLastDim) {
-    res = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, vi, vj);
+    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
-      builder.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(builder, loc, res);
   } else {
     Value ne =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, vi, vj);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIntegerType(1),
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
                                                ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value lt =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, vi, vj);
-    builder.create<scf::YieldOp>(loc, lt);
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
+    scf::YieldOp::create(builder, loc, lt);
 
     // If (x[i] == x[j]). Set up the insertion point for the nested if-stmt that
     // checks the remaining dimensions.
@@ -337,17 +337,17 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value p = args[hiIdx];
   SmallVector<Type, 2> types(2, p.getType()); // Only two types.
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(
+  scf::WhileOp whileOp = scf::WhileOp::create(builder,
       loc, types, SmallVector<Value, 2>{args[loIdx], args[hiIdx]});
 
   // The before-region of the WhileOp.
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, types, {loc, loc});
   builder.setInsertionPointToEnd(before);
-  Value cond1 = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                               before->getArgument(0),
                                               before->getArgument(1));
-  builder.create<scf::ConditionOp>(loc, cond1, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond1, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -357,9 +357,9 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   Value hi = after->getArgument(1);
   // Compute mid = (lo + hi) >> 1.
   Value c1 = constantIndex(builder, loc, 1);
-  Value mid = builder.create<arith::ShRUIOp>(
-      loc, builder.create<arith::AddIOp>(loc, lo, hi), c1);
-  Value midp1 = builder.create<arith::AddIOp>(loc, mid, c1);
+  Value mid = arith::ShRUIOp::create(builder,
+      loc, arith::AddIOp::create(builder, loc, lo, hi), c1);
+  Value midp1 = arith::AddIOp::create(builder, loc, mid, c1);
 
   // Compare xs[p] < xs[mid].
   SmallVector<Value> compareOperands{p, mid};
@@ -372,12 +372,12 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   //     hi = mid;
   //   else
   //     lo = mid + 1;
-  Value newLo = builder.create<arith::SelectOp>(loc, cond2, lo, midp1);
-  Value newHi = builder.create<arith::SelectOp>(loc, cond2, mid, hi);
-  builder.create<scf::YieldOp>(loc, ValueRange{newLo, newHi});
+  Value newLo = arith::SelectOp::create(builder, loc, cond2, lo, midp1);
+  Value newHi = arith::SelectOp::create(builder, loc, cond2, mid, hi);
+  scf::YieldOp::create(builder, loc, ValueRange{newLo, newHi});
 
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc, whileOp.getResult(0));
+  func::ReturnOp::create(builder, loc, whileOp.getResult(0));
 }
 
 /// Creates code to advance i in a loop based on xs[p] as follows:
@@ -393,7 +393,7 @@ static std::pair<Value, Value> createScanLoop(OpBuilder &builder,
                                               uint64_t ny, int step) {
   Location loc = func.getLoc();
   scf::WhileOp whileOp =
-      builder.create<scf::WhileOp>(loc, TypeRange{i.getType()}, ValueRange{i});
+      scf::WhileOp::create(builder, loc, TypeRange{i.getType()}, ValueRange{i});
 
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, {i.getType()}, {loc});
@@ -409,14 +409,14 @@ static std::pair<Value, Value> createScanLoop(OpBuilder &builder,
   }
   compareOperands.append(xs.begin(), xs.end());
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  builder.create<scf::ConditionOp>(loc, cond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond, before->getArguments());
 
   Block *after =
       builder.createBlock(&whileOp.getAfter(), {}, {i.getType()}, {loc});
   builder.setInsertionPointToEnd(after);
   Value cs = constantIndex(builder, loc, step);
-  i = builder.create<arith::AddIOp>(loc, after->getArgument(0), cs);
-  builder.create<scf::YieldOp>(loc, ValueRange{i});
+  i = arith::AddIOp::create(builder, loc, after->getArgument(0), cs);
+  scf::YieldOp::create(builder, loc, ValueRange{i});
   i = whileOp.getResult(0);
 
   builder.setInsertionPointAfter(whileOp);
@@ -440,7 +440,7 @@ static scf::IfOp createCompareThenSwap(OpBuilder &builder, Location loc,
   compareOperands[0] = b;
   compareOperands[1] = a;
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else=*/false);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, cond, /*else=*/false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   swapOperands[0] = b;
   swapOperands[1] = a;
@@ -517,12 +517,12 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
   swapOperands.append(args.begin() + xStartIdx, args.end());
   Location loc = func.getLoc();
   Value c1 = constantIndex(builder, loc, 1);
-  Value hiP1 = builder.create<arith::AddIOp>(loc, hi, c1);
-  Value len = builder.create<arith::SubIOp>(loc, hiP1, lo);
+  Value hiP1 = arith::AddIOp::create(builder, loc, hi, c1);
+  Value len = arith::SubIOp::create(builder, loc, hiP1, lo);
   Value lenThreshold = constantIndex(builder, loc, 1000);
-  Value lenCond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value lenCond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                                 len, lenThreshold);
-  scf::IfOp lenIf = builder.create<scf::IfOp>(loc, lenCond, /*else=*/true);
+  scf::IfOp lenIf = scf::IfOp::create(builder, loc, lenCond, /*else=*/true);
 
   // When len < 1000, choose pivot from median of 3 values.
   builder.setInsertionPointToStart(&lenIf.getThenRegion().front());
@@ -531,13 +531,13 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
 
   // When len >= 1000, choose pivot from median of 5 values.
   builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
-  Value miP1 = builder.create<arith::AddIOp>(loc, hi, c1);
-  Value a = builder.create<arith::AddIOp>(loc, lo, miP1);
+  Value miP1 = arith::AddIOp::create(builder, loc, hi, c1);
+  Value a = arith::AddIOp::create(builder, loc, lo, miP1);
   // Value a is the middle between [loc, mi].
-  a = builder.create<arith::ShRUIOp>(loc, a, c1);
-  Value b = builder.create<arith::AddIOp>(loc, mi, hiP1);
+  a = arith::ShRUIOp::create(builder, loc, a, c1);
+  Value b = arith::AddIOp::create(builder, loc, mi, hiP1);
   // Value b is the middle between [mi, hi].
-  b = builder.create<arith::ShRUIOp>(loc, b, c1);
+  b = arith::ShRUIOp::create(builder, loc, b, c1);
   createSort5(builder, loc, xPerm, ny, swapOperands, compareOperands, lo, a, mi,
               b, hi);
 
@@ -589,24 +589,24 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value sum = builder.create<arith::AddIOp>(loc, lo, hi);
+  Value sum = arith::AddIOp::create(builder, loc, lo, hi);
   Value c1 = constantIndex(builder, loc, 1);
-  Value p = builder.create<arith::ShRUIOp>(loc, sum, c1);
+  Value p = arith::ShRUIOp::create(builder, loc, sum, c1);
 
   Value i = lo;
-  Value j = builder.create<arith::SubIOp>(loc, hi, c1);
+  Value j = arith::SubIOp::create(builder, loc, hi, c1);
   createChoosePivot(builder, module, func, xPerm, ny, i, j, p, args);
   Value trueVal = constantI1(builder, loc, true); // The value for while (true)
   SmallVector<Value, 4> operands{i, j, p, trueVal}; // Exactly four values.
   SmallVector<Type, 4> types{i.getType(), j.getType(), p.getType(),
                              trueVal.getType()};
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(loc, types, operands);
+  scf::WhileOp whileOp = scf::WhileOp::create(builder, loc, types, operands);
 
   // The before-region of the WhileOp.
   Block *before = builder.createBlock(&whileOp.getBefore(), {}, types,
                                       {loc, loc, loc, loc});
   builder.setInsertionPointToEnd(before);
-  builder.create<scf::ConditionOp>(loc, before->getArgument(3),
+  scf::ConditionOp::create(builder, loc, before->getArgument(3),
                                    before->getArguments());
 
   // The after-region of the WhileOp.
@@ -629,70 +629,70 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
 
   // If i < j:
   Value cond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, i, j);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, i, j);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   SmallVector<Value> swapOperands{i, j};
   swapOperands.append(args.begin() + xStartIdx, args.end());
   createSwap(builder, loc, swapOperands, xPerm, ny);
   // If the pivot is moved, update p with the new pivot.
   Value icond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, i, p);
-  scf::IfOp ifOpI = builder.create<scf::IfOp>(loc, TypeRange{p.getType()},
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, i, p);
+  scf::IfOp ifOpI = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
                                               icond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpI.getThenRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{j});
+  scf::YieldOp::create(builder, loc, ValueRange{j});
   builder.setInsertionPointToStart(&ifOpI.getElseRegion().front());
   Value jcond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, j, p);
-  scf::IfOp ifOpJ = builder.create<scf::IfOp>(loc, TypeRange{p.getType()},
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, j, p);
+  scf::IfOp ifOpJ = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
                                               jcond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpJ.getThenRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{i});
+  scf::YieldOp::create(builder, loc, ValueRange{i});
   builder.setInsertionPointToStart(&ifOpJ.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{p});
+  scf::YieldOp::create(builder, loc, ValueRange{p});
   builder.setInsertionPointAfter(ifOpJ);
-  builder.create<scf::YieldOp>(loc, ifOpJ.getResults());
+  scf::YieldOp::create(builder, loc, ifOpJ.getResults());
   builder.setInsertionPointAfter(ifOpI);
   Value compareEqIJ =
-      builder.create<arith::AndIOp>(loc, iCompareEq, jCompareEq);
-  scf::IfOp ifOp2 = builder.create<scf::IfOp>(
+      arith::AndIOp::create(builder, loc, iCompareEq, jCompareEq);
+  scf::IfOp ifOp2 = scf::IfOp::create(builder,
       loc, TypeRange{i.getType(), j.getType()}, compareEqIJ, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
-  Value i2 = builder.create<arith::AddIOp>(loc, i, c1);
-  Value j2 = builder.create<arith::SubIOp>(loc, j, c1);
-  builder.create<scf::YieldOp>(loc, ValueRange{i2, j2});
+  Value i2 = arith::AddIOp::create(builder, loc, i, c1);
+  Value j2 = arith::SubIOp::create(builder, loc, j, c1);
+  scf::YieldOp::create(builder, loc, ValueRange{i2, j2});
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{i, j});
+  scf::YieldOp::create(builder, loc, ValueRange{i, j});
   builder.setInsertionPointAfter(ifOp2);
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc,
       ValueRange{ifOp2.getResult(0), ifOp2.getResult(1), ifOpI.getResult(0),
                  /*cont=*/constantI1(builder, loc, true)});
 
   // False branch for if i < j (i.e., i >= j):
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  p = builder.create<arith::AddIOp>(loc, j,
+  p = arith::AddIOp::create(builder, loc, j,
                                     constantOne(builder, loc, j.getType()));
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc, ValueRange{i, j, p, /*cont=*/constantI1(builder, loc, false)});
 
   // Return for the whileOp.
   builder.setInsertionPointAfter(ifOp);
-  builder.create<scf::YieldOp>(loc, ifOp.getResults());
+  scf::YieldOp::create(builder, loc, ifOp.getResults());
 
   // Return for the function.
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc, whileOp.getResult(2));
+  func::ReturnOp::create(builder, loc, whileOp.getResult(2));
 }
 
 /// Computes (n-2)/n, assuming n has index type.
 static Value createSubTwoDividedByTwo(OpBuilder &builder, Location loc,
                                       Value n) {
   Value i2 = constantIndex(builder, loc, 2);
-  Value res = builder.create<arith::SubIOp>(loc, n, i2);
+  Value res = arith::SubIOp::create(builder, loc, n, i2);
   Value i1 = constantIndex(builder, loc, 1);
-  return builder.create<arith::ShRUIOp>(loc, res, i1);
+  return arith::ShRUIOp::create(builder, loc, res, i1);
 }
 
 /// Creates a function to heapify the subtree with root `start` within the full
@@ -743,16 +743,16 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   // If (n >= 2).
   Value c2 = constantIndex(builder, loc, 2);
   Value condN =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, n, c2);
-  scf::IfOp ifN = builder.create<scf::IfOp>(loc, condN, /*else=*/false);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, n, c2);
+  scf::IfOp ifN = scf::IfOp::create(builder, loc, condN, /*else=*/false);
   builder.setInsertionPointToStart(&ifN.getThenRegion().front());
-  Value child = builder.create<arith::SubIOp>(loc, start, first);
+  Value child = arith::SubIOp::create(builder, loc, start, first);
 
   // If ((n-2)/2 >= child).
   Value t = createSubTwoDividedByTwo(builder, loc, n);
   Value condNc =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp ifNc = builder.create<scf::IfOp>(loc, condNc, /*else=*/false);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
+  scf::IfOp ifNc = scf::IfOp::create(builder, loc, condNc, /*else=*/false);
 
   builder.setInsertionPointToStart(&ifNc.getThenRegion().front());
   Value c1 = constantIndex(builder, loc, 1);
@@ -768,32 +768,32 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   //   if (child+1 < n && data[childIndex] < data[childIndex+1])
   //     childIndex ++; child ++ // Right child is bigger.
   auto getLargerChild = [&](Value r) -> std::pair<Value, Value> {
-    Value lChild = builder.create<arith::ShLIOp>(loc, r, c1);
-    lChild = builder.create<arith::AddIOp>(loc, lChild, c1);
-    Value lChildIdx = builder.create<arith::AddIOp>(loc, lChild, first);
-    Value rChild = builder.create<arith::AddIOp>(loc, lChild, c1);
-    Value cond1 = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value lChild = arith::ShLIOp::create(builder, loc, r, c1);
+    lChild = arith::AddIOp::create(builder, loc, lChild, c1);
+    Value lChildIdx = arith::AddIOp::create(builder, loc, lChild, first);
+    Value rChild = arith::AddIOp::create(builder, loc, lChild, c1);
+    Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                                 rChild, n);
     SmallVector<Type, 2> ifTypes(2, r.getType());
     scf::IfOp if1 =
-        builder.create<scf::IfOp>(loc, ifTypes, cond1, /*else=*/true);
+        scf::IfOp::create(builder, loc, ifTypes, cond1, /*else=*/true);
     builder.setInsertionPointToStart(&if1.getThenRegion().front());
-    Value rChildIdx = builder.create<arith::AddIOp>(loc, rChild, first);
+    Value rChildIdx = arith::AddIOp::create(builder, loc, rChild, first);
     // Compare data[left] < data[right].
     compareOperands[0] = lChildIdx;
     compareOperands[1] = rChildIdx;
     Value cond2 =
         createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
     scf::IfOp if2 =
-        builder.create<scf::IfOp>(loc, ifTypes, cond2, /*else=*/true);
+        scf::IfOp::create(builder, loc, ifTypes, cond2, /*else=*/true);
     builder.setInsertionPointToStart(&if2.getThenRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{rChild, rChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{rChild, rChildIdx});
     builder.setInsertionPointToStart(&if2.getElseRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{lChild, lChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{lChild, lChildIdx});
     builder.setInsertionPointAfter(if2);
-    builder.create<scf::YieldOp>(loc, if2.getResults());
+    scf::YieldOp::create(builder, loc, if2.getResults());
     builder.setInsertionPointToStart(&if1.getElseRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{lChild, lChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{lChild, lChildIdx});
     builder.setInsertionPointAfter(if1);
     return std::make_pair(if1.getResult(0), if1.getResult(1));
   };
@@ -803,7 +803,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
 
   // While (data[start] < data[childIndex]).
   SmallVector<Type, 3> types(3, child.getType());
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(
+  scf::WhileOp whileOp = scf::WhileOp::create(builder,
       loc, types, SmallVector<Value, 2>{start, child, childIdx});
 
   // The before-region of the WhileOp.
@@ -815,7 +815,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   compareOperands[0] = start;
   compareOperands[1] = childIdx;
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  builder.create<scf::ConditionOp>(loc, cond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after = builder.createBlock(&whileOp.getAfter(), {}, types, locs);
@@ -827,20 +827,20 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   createSwap(builder, loc, swapOperands, xPerm, ny);
   start = childIdx;
   Value cond2 =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp if2 = builder.create<scf::IfOp>(
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
+  scf::IfOp if2 = scf::IfOp::create(builder,
       loc, TypeRange{child.getType(), child.getType()}, cond2, /*else=*/true);
   builder.setInsertionPointToStart(&if2.getThenRegion().front());
   auto [newChild, newChildIdx] = getLargerChild(child);
-  builder.create<scf::YieldOp>(loc, ValueRange{newChild, newChildIdx});
+  scf::YieldOp::create(builder, loc, ValueRange{newChild, newChildIdx});
   builder.setInsertionPointToStart(&if2.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{child, childIdx});
+  scf::YieldOp::create(builder, loc, ValueRange{child, childIdx});
   builder.setInsertionPointAfter(if2);
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc, ValueRange{start, if2.getResult(0), if2.getResult(1)});
 
   builder.setInsertionPointAfter(ifN);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Creates a function to perform heap sort on the values in the range of index
@@ -870,45 +870,45 @@ static void createHeapSortFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value n = builder.create<arith::SubIOp>(loc, hi, lo);
+  Value n = arith::SubIOp::create(builder, loc, hi, lo);
 
   // For i = (n-2)/2 downto 0.
   Value c0 = constantIndex(builder, loc, 0);
   Value c1 = constantIndex(builder, loc, 1);
   Value s = createSubTwoDividedByTwo(builder, loc, n);
-  Value up = builder.create<arith::AddIOp>(loc, s, c1);
-  scf::ForOp forI = builder.create<scf::ForOp>(loc, c0, up, c1);
+  Value up = arith::AddIOp::create(builder, loc, s, c1);
+  scf::ForOp forI = scf::ForOp::create(builder, loc, c0, up, c1);
   builder.setInsertionPointToStart(forI.getBody());
-  Value i = builder.create<arith::SubIOp>(loc, s, forI.getInductionVar());
-  Value lopi = builder.create<arith::AddIOp>(loc, lo, i);
+  Value i = arith::SubIOp::create(builder, loc, s, forI.getInductionVar());
+  Value lopi = arith::AddIOp::create(builder, loc, lo, i);
   SmallVector<Value> shiftDownOperands = {lo, lopi};
   shiftDownOperands.append(args.begin() + xStartIdx, args.end());
   shiftDownOperands.push_back(n);
   FlatSymbolRefAttr shiftDownFunc = getMangledSortHelperFunc(
       builder, func, TypeRange(), kShiftDownFuncNamePrefix, xPerm, ny,
       shiftDownOperands, createShiftDownFunc, /*nTrailingP=*/1);
-  builder.create<func::CallOp>(loc, shiftDownFunc, TypeRange(),
+  func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
                                shiftDownOperands);
 
   builder.setInsertionPointAfter(forI);
   // For l = n downto 2.
-  up = builder.create<arith::SubIOp>(loc, n, c1);
-  scf::ForOp forL = builder.create<scf::ForOp>(loc, c0, up, c1);
+  up = arith::SubIOp::create(builder, loc, n, c1);
+  scf::ForOp forL = scf::ForOp::create(builder, loc, c0, up, c1);
   builder.setInsertionPointToStart(forL.getBody());
-  Value l = builder.create<arith::SubIOp>(loc, n, forL.getInductionVar());
-  Value loplm1 = builder.create<arith::AddIOp>(loc, lo, l);
-  loplm1 = builder.create<arith::SubIOp>(loc, loplm1, c1);
+  Value l = arith::SubIOp::create(builder, loc, n, forL.getInductionVar());
+  Value loplm1 = arith::AddIOp::create(builder, loc, lo, l);
+  loplm1 = arith::SubIOp::create(builder, loc, loplm1, c1);
   SmallVector<Value> swapOperands{lo, loplm1};
   swapOperands.append(args.begin() + xStartIdx, args.end());
   createSwap(builder, loc, swapOperands, xPerm, ny);
   shiftDownOperands[1] = lo;
   shiftDownOperands[shiftDownOperands.size() - 1] =
-      builder.create<arith::SubIOp>(loc, l, c1);
-  builder.create<func::CallOp>(loc, shiftDownFunc, TypeRange(),
+      arith::SubIOp::create(builder, loc, l, c1);
+  func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
                                shiftDownOperands);
 
   builder.setInsertionPointAfter(forL);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// A helper for generating code to perform quick sort. It partitions [lo, hi),
@@ -933,35 +933,35 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
                                       args.drop_back(nTrailingP))
                 .getResult(0);
 
-  Value lenLow = builder.create<arith::SubIOp>(loc, p, lo);
-  Value lenHigh = builder.create<arith::SubIOp>(loc, hi, p);
+  Value lenLow = arith::SubIOp::create(builder, loc, p, lo);
+  Value lenHigh = arith::SubIOp::create(builder, loc, hi, p);
   // Partition already sorts array with len <= 2
   Value c2 = constantIndex(builder, loc, 2);
-  Value len = builder.create<arith::SubIOp>(loc, hi, lo);
+  Value len = arith::SubIOp::create(builder, loc, hi, lo);
   Value lenGtTwo =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ugt, len, c2);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ugt, len, c2);
   scf::IfOp ifLenGtTwo =
-      builder.create<scf::IfOp>(loc, types, lenGtTwo, /*else=*/true);
+      scf::IfOp::create(builder, loc, types, lenGtTwo, /*else=*/true);
   builder.setInsertionPointToStart(&ifLenGtTwo.getElseRegion().front());
   // Returns an empty range to mark the entire region is fully sorted.
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
   // Else len > 2, need recursion.
   builder.setInsertionPointToStart(&ifLenGtTwo.getThenRegion().front());
-  Value cond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule,
+  Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
                                              lenLow, lenHigh);
 
   Value c0 = constantIndex(builder, loc, 0);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
 
   auto mayRecursion = [&](Value low, Value high, Value len) {
     Value cond =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, len, c0);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else=*/false);
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, len, c0);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, cond, /*else=*/false);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     SmallVector<Value> operands{low, high};
     operands.append(args.begin() + xStartIdx, args.end());
-    builder.create<func::CallOp>(loc, func, operands);
+    func::CallOp::create(builder, loc, func, operands);
     builder.setInsertionPointAfter(ifOp);
   };
 
@@ -969,14 +969,14 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
   // the bigger partition to be processed by the enclosed while-loop.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   mayRecursion(lo, p, lenLow);
-  builder.create<scf::YieldOp>(loc, ValueRange{p, hi});
+  scf::YieldOp::create(builder, loc, ValueRange{p, hi});
 
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
   mayRecursion(p, hi, lenHigh);
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, p});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, p});
 
   builder.setInsertionPointAfter(ifOp);
-  builder.create<scf::YieldOp>(loc, ifOp.getResults());
+  scf::YieldOp::create(builder, loc, ifOp.getResults());
 
   builder.setInsertionPointAfter(ifLenGtTwo);
   return std::make_pair(ifLenGtTwo.getResult(0), ifLenGtTwo.getResult(1));
@@ -1011,10 +1011,10 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   Value c1 = constantIndex(builder, loc, 1);
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value lop1 = builder.create<arith::AddIOp>(loc, lo, c1);
+  Value lop1 = arith::AddIOp::create(builder, loc, lo, c1);
 
   // Start the outer for-stmt with induction variable i.
-  scf::ForOp forOpI = builder.create<scf::ForOp>(loc, lop1, hi, c1);
+  scf::ForOp forOpI = scf::ForOp::create(builder, loc, lop1, hi, c1);
   builder.setInsertionPointToStart(forOpI.getBody());
   Value i = forOpI.getInductionVar();
 
@@ -1035,24 +1035,24 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t unused, Value i, Value unused2, Value buffer) {
-        d.push_back(builder.create<memref::LoadOp>(loc, buffer, i));
+        d.push_back(memref::LoadOp::create(builder, loc, buffer, i));
       });
 
   // Start the inner for-stmt with induction variable j, for moving data[p..i)
   // to data[p+1..i+1).
-  Value imp = builder.create<arith::SubIOp>(loc, i, p);
+  Value imp = arith::SubIOp::create(builder, loc, i, p);
   Value c0 = constantIndex(builder, loc, 0);
-  scf::ForOp forOpJ = builder.create<scf::ForOp>(loc, c0, imp, c1);
+  scf::ForOp forOpJ = scf::ForOp::create(builder, loc, c0, imp, c1);
   builder.setInsertionPointToStart(forOpJ.getBody());
   Value j = forOpJ.getInductionVar();
-  Value imj = builder.create<arith::SubIOp>(loc, i, j);
+  Value imj = arith::SubIOp::create(builder, loc, i, j);
   operands[1] = imj;
-  operands[0] = builder.create<arith::SubIOp>(loc, imj, c1);
+  operands[0] = arith::SubIOp::create(builder, loc, imj, c1);
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t unused, Value imjm1, Value imj, Value buffer) {
-        Value t = builder.create<memref::LoadOp>(loc, buffer, imjm1);
-        builder.create<memref::StoreOp>(loc, t, buffer, imj);
+        Value t = memref::LoadOp::create(builder, loc, buffer, imjm1);
+        memref::StoreOp::create(builder, loc, t, buffer, imj);
       });
 
   // Store the value at data[i] to data[p].
@@ -1061,11 +1061,11 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t k, Value p, Value usused, Value buffer) {
-        builder.create<memref::StoreOp>(loc, d[k], buffer, p);
+        memref::StoreOp::create(builder, loc, d[k], buffer, p);
       });
 
   builder.setInsertionPointAfter(forOpI);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Creates a function to perform quick sort or a hybrid quick sort on the
@@ -1127,7 +1127,7 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   Value hi = args[hiIdx];
   SmallVector<Type, 2> types(2, lo.getType()); // Only two types.
   scf::WhileOp whileOp =
-      builder.create<scf::WhileOp>(loc, types, SmallVector<Value, 2>{lo, hi});
+      scf::WhileOp::create(builder, loc, types, SmallVector<Value, 2>{lo, hi});
 
   // The before-region of the WhileOp.
   Block *before =
@@ -1136,10 +1136,10 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   lo = before->getArgument(0);
   hi = before->getArgument(1);
   Value loP1 =
-      builder.create<arith::AddIOp>(loc, lo, constantIndex(builder, loc, 1));
+      arith::AddIOp::create(builder, loc, lo, constantIndex(builder, loc, 1));
   Value needSort =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, loP1, hi);
-  builder.create<scf::ConditionOp>(loc, needSort, before->getArguments());
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, loP1, hi);
+  scf::ConditionOp::create(builder, loc, needSort, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -1151,53 +1151,53 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   args[1] = hi;
 
   if (isHybrid) {
-    Value len = builder.create<arith::SubIOp>(loc, hi, lo);
+    Value len = arith::SubIOp::create(builder, loc, hi, lo);
     Value lenLimit = constantIndex(builder, loc, 30);
-    Value lenCond = builder.create<arith::CmpIOp>(
+    Value lenCond = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::ule, len, lenLimit);
     scf::IfOp lenIf =
-        builder.create<scf::IfOp>(loc, types, lenCond, /*else=*/true);
+        scf::IfOp::create(builder, loc, types, lenCond, /*else=*/true);
 
     // When len <= limit.
     builder.setInsertionPointToStart(&lenIf.getThenRegion().front());
     FlatSymbolRefAttr insertionSortFunc = getMangledSortHelperFunc(
         builder, func, TypeRange(), kSortStableFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createSortStableFunc);
-    builder.create<func::CallOp>(loc, insertionSortFunc, TypeRange(),
+    func::CallOp::create(builder, loc, insertionSortFunc, TypeRange(),
                                  ValueRange(args).drop_back(nTrailingP));
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When len > limit.
     builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
     Value depthLimit = args.back();
-    depthLimit = builder.create<arith::SubIOp>(loc, depthLimit,
+    depthLimit = arith::SubIOp::create(builder, loc, depthLimit,
                                                constantI64(builder, loc, 1));
     Value depthCond =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule,
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
                                       depthLimit, constantI64(builder, loc, 0));
     scf::IfOp depthIf =
-        builder.create<scf::IfOp>(loc, types, depthCond, /*else=*/true);
+        scf::IfOp::create(builder, loc, types, depthCond, /*else=*/true);
 
     // When depth exceeds limit.
     builder.setInsertionPointToStart(&depthIf.getThenRegion().front());
     FlatSymbolRefAttr heapSortFunc = getMangledSortHelperFunc(
         builder, func, TypeRange(), kHeapSortFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createHeapSortFunc);
-    builder.create<func::CallOp>(loc, heapSortFunc, TypeRange(),
+    func::CallOp::create(builder, loc, heapSortFunc, TypeRange(),
                                  ValueRange(args).drop_back(nTrailingP));
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When depth doesn't exceed limit.
     builder.setInsertionPointToStart(&depthIf.getElseRegion().front());
     args.back() = depthLimit;
     std::tie(lo, hi) =
         createQuickSort(builder, module, func, args, xPerm, ny, nTrailingP);
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
     builder.setInsertionPointAfter(depthIf);
     lo = depthIf.getResult(0);
     hi = depthIf.getResult(1);
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
     builder.setInsertionPointAfter(lenIf);
     lo = lenIf.getResult(0);
@@ -1208,11 +1208,11 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   }
 
   // New [lo, hi) for the next while-loop iteration.
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
   // After the while-loop.
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Implements the rewriting for operator sort and sort_coo.
@@ -1228,7 +1228,7 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     if (!mtp.isDynamicDim(0)) {
       auto newMtp =
           MemRefType::get({ShapedType::kDynamic}, mtp.getElementType());
-      v = rewriter.create<memref::CastOp>(loc, newMtp, v);
+      v = memref::CastOp::create(rewriter, loc, newMtp, v);
     }
     operands.push_back(v);
   }
@@ -1248,12 +1248,12 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     // As a heuristics, set depthLimit = 2 * log2(n).
     Value lo = operands[loIdx];
     Value hi = operands[hiIdx];
-    Value len = rewriter.create<arith::IndexCastOp>(
+    Value len = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(),
-        rewriter.create<arith::SubIOp>(loc, hi, lo));
-    Value depthLimit = rewriter.create<arith::SubIOp>(
+        arith::SubIOp::create(rewriter, loc, hi, lo));
+    Value depthLimit = arith::SubIOp::create(rewriter,
         loc, constantI64(rewriter, loc, 64),
-        rewriter.create<math::CountLeadingZerosOp>(loc, len));
+        math::CountLeadingZerosOp::create(rewriter, loc, len));
     operands.push_back(depthLimit);
     break;
   }
@@ -1307,33 +1307,33 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
     Location loc = op->getLoc();
     Value c0 = constantIndex(rewriter, loc, 0);
     Value buffer = op.getInBuffer();
-    Value capacity = rewriter.create<memref::DimOp>(loc, buffer, c0);
+    Value capacity = memref::DimOp::create(rewriter, loc, buffer, c0);
     Value size = op.getCurSize();
     Value value = op.getValue();
 
     Value n = op.getN() ? op.getN() : constantIndex(rewriter, loc, 1);
-    Value newSize = rewriter.create<arith::AddIOp>(loc, size, n);
+    Value newSize = arith::AddIOp::create(rewriter, loc, size, n);
     auto nValue = dyn_cast_or_null<arith::ConstantIndexOp>(n.getDefiningOp());
     bool nIsOne = (nValue && nValue.value() == 1);
 
     if (!op.getInbounds()) {
-      Value cond = rewriter.create<arith::CmpIOp>(
+      Value cond = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ugt, newSize, capacity);
 
       Value c2 = constantIndex(rewriter, loc, 2);
       auto bufferType =
           MemRefType::get({ShapedType::kDynamic}, value.getType());
-      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, bufferType, cond,
+      scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, bufferType, cond,
                                                   /*else=*/true);
       // True branch.
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       if (nIsOne) {
-        capacity = rewriter.create<arith::MulIOp>(loc, capacity, c2);
+        capacity = arith::MulIOp::create(rewriter, loc, capacity, c2);
       } else {
         // Use a do-while loop to calculate the new capacity as follows:
         //   do { new_capacity *= 2 } while (size > new_capacity)
         scf::WhileOp whileOp =
-            rewriter.create<scf::WhileOp>(loc, capacity.getType(), capacity);
+            scf::WhileOp::create(rewriter, loc, capacity.getType(), capacity);
 
         // The before-region of the WhileOp.
         Block *before = rewriter.createBlock(&whileOp.getBefore(), {},
@@ -1341,36 +1341,36 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
         rewriter.setInsertionPointToEnd(before);
 
         capacity =
-            rewriter.create<arith::MulIOp>(loc, before->getArgument(0), c2);
-        cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ugt,
+            arith::MulIOp::create(rewriter, loc, before->getArgument(0), c2);
+        cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ugt,
                                               newSize, capacity);
-        rewriter.create<scf::ConditionOp>(loc, cond, ValueRange{capacity});
+        scf::ConditionOp::create(rewriter, loc, cond, ValueRange{capacity});
         // The after-region of the WhileOp.
         Block *after = rewriter.createBlock(&whileOp.getAfter(), {},
                                             {capacity.getType()}, {loc});
         rewriter.setInsertionPointToEnd(after);
-        rewriter.create<scf::YieldOp>(loc, after->getArguments());
+        scf::YieldOp::create(rewriter, loc, after->getArguments());
 
         rewriter.setInsertionPointAfter(whileOp);
         capacity = whileOp.getResult(0);
       }
 
       Value newBuffer =
-          rewriter.create<memref::ReallocOp>(loc, bufferType, buffer, capacity);
+          memref::ReallocOp::create(rewriter, loc, bufferType, buffer, capacity);
       if (enableBufferInitialization) {
-        Value fillSize = rewriter.create<arith::SubIOp>(loc, capacity, newSize);
+        Value fillSize = arith::SubIOp::create(rewriter, loc, capacity, newSize);
         Value fillValue = constantZero(rewriter, loc, value.getType());
-        Value subBuffer = rewriter.create<memref::SubViewOp>(
+        Value subBuffer = memref::SubViewOp::create(rewriter,
             loc, newBuffer, /*offset=*/ValueRange{newSize},
             /*size=*/ValueRange{fillSize},
             /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
-        rewriter.create<linalg::FillOp>(loc, fillValue, subBuffer);
+        linalg::FillOp::create(rewriter, loc, fillValue, subBuffer);
       }
-      rewriter.create<scf::YieldOp>(loc, newBuffer);
+      scf::YieldOp::create(rewriter, loc, newBuffer);
 
       // False branch.
       rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      rewriter.create<scf::YieldOp>(loc, buffer);
+      scf::YieldOp::create(rewriter, loc, buffer);
 
       // Prepare for adding the value to the end of the buffer.
       rewriter.setInsertionPointAfter(ifOp);
@@ -1379,12 +1379,12 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
 
     // Add the value to the end of the buffer.
     if (nIsOne) {
-      rewriter.create<memref::StoreOp>(loc, value, buffer, size);
+      memref::StoreOp::create(rewriter, loc, value, buffer, size);
     } else {
-      Value subBuffer = rewriter.create<memref::SubViewOp>(
+      Value subBuffer = memref::SubViewOp::create(rewriter,
           loc, buffer, /*offset=*/ValueRange{size}, /*size=*/ValueRange{n},
           /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
-      rewriter.create<linalg::FillOp>(loc, value, subBuffer);
+      linalg::FillOp::create(rewriter, loc, value, subBuffer);
     }
 
     // Update the buffer size.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index e89b34d457ff8..001754b5531be 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -59,7 +59,7 @@ static gpu::GPUModuleOp genGPUModule(OpBuilder &builder, ModuleOp topModule) {
     return op; // existing
   markAsGPUContainer(topModule);
   builder.setInsertionPointToStart(topModule.getBody());
-  return builder.create<gpu::GPUModuleOp>(topModule->getLoc(),
+  return gpu::GPUModuleOp::create(builder, topModule->getLoc(),
                                           "sparse_kernels");
 }
 
@@ -81,7 +81,7 @@ static gpu::GPUFuncOp genGPUFunc(OpBuilder &builder, gpu::GPUModuleOp gpuModule,
     argsTp.push_back(arg.getType());
   FunctionType type = FunctionType::get(gpuModule->getContext(), argsTp, {});
   auto gpuFunc =
-      builder.create<gpu::GPUFuncOp>(gpuModule->getLoc(), kernelName, type);
+      gpu::GPUFuncOp::create(builder, gpuModule->getLoc(), kernelName, type);
   gpuFunc->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                    builder.getUnitAttr());
   return gpuFunc;
@@ -115,28 +115,28 @@ static Value genHostRegisterMemref(OpBuilder &builder, Location loc,
   MemRefType memTp = cast<MemRefType>(mem.getType());
   UnrankedMemRefType resTp =
       UnrankedMemRefType::get(memTp.getElementType(), /*memorySpace=*/0);
-  Value cast = builder.create<memref::CastOp>(loc, resTp, mem);
-  builder.create<gpu::HostRegisterOp>(loc, cast);
+  Value cast = memref::CastOp::create(builder, loc, resTp, mem);
+  gpu::HostRegisterOp::create(builder, loc, cast);
   return cast;
 }
 
 /// Unmaps the provided buffer, expecting the casted buffer.
 static void genHostUnregisterMemref(OpBuilder &builder, Location loc,
                                     Value cast) {
-  builder.create<gpu::HostUnregisterOp>(loc, cast);
+  gpu::HostUnregisterOp::create(builder, loc, cast);
 }
 
 /// Generates first wait in an asynchronous chain.
 static Value genFirstWait(OpBuilder &builder, Location loc) {
   Type tokenType = builder.getType<gpu::AsyncTokenType>();
-  return builder.create<gpu::WaitOp>(loc, tokenType, ValueRange())
+  return gpu::WaitOp::create(builder, loc, tokenType, ValueRange())
       .getAsyncToken();
 }
 
 /// Generates last, blocking wait in an asynchronous chain.
 static void genBlockingWait(OpBuilder &builder, Location loc,
                             ValueRange operands) {
-  builder.create<gpu::WaitOp>(loc, Type(), operands);
+  gpu::WaitOp::create(builder, loc, Type(), operands);
 }
 
 /// Allocates memory on the device.
@@ -156,7 +156,7 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
       dynamicSizes.push_back(dimOp);
     }
   }
-  return builder.create<gpu::AllocOp>(loc, TypeRange({memTp, token.getType()}),
+  return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
                                       token, dynamicSizes, ValueRange());
 }
 
@@ -164,14 +164,14 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
 static Value genHostBuffer(OpBuilder &builder, Location loc, Type type,
                            Value size) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
-  return builder.create<memref::AllocOp>(loc, memTp, size).getResult();
+  return memref::AllocOp::create(builder, loc, memTp, size).getResult();
 }
 
 // Allocates a typed buffer on the device with given size.
 static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Type type,
                                    Value size, Value token) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
-  return builder.create<gpu::AllocOp>(loc, TypeRange({memTp, token.getType()}),
+  return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
                                       token, size, ValueRange());
 }
 
@@ -184,14 +184,14 @@ static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Value size,
 /// Deallocates memory from the device.
 static Value genDeallocMemRef(OpBuilder &builder, Location loc, Value mem,
                               Value token) {
-  return builder.create<gpu::DeallocOp>(loc, token.getType(), token, mem)
+  return gpu::DeallocOp::create(builder, loc, token.getType(), token, mem)
       .getAsyncToken();
 }
 
 /// Copies memory between host and device (direction is implicit).
 static Value genCopyMemRef(OpBuilder &builder, Location loc, Value dst,
                            Value src, Value token) {
-  return builder.create<gpu::MemcpyOp>(loc, token.getType(), token, dst, src)
+  return gpu::MemcpyOp::create(builder, loc, token.getType(), token, dst, src)
       .getAsyncToken();
 }
 
@@ -212,7 +212,7 @@ static Value genTensorToMemref(PatternRewriter &rewriter, Location loc,
   auto tensorType = llvm::cast<ShapedType>(tensor.getType());
   auto memrefType =
       MemRefType::get(tensorType.getShape(), tensorType.getElementType());
-  return rewriter.create<bufferization::ToBufferOp>(loc, memrefType, tensor);
+  return bufferization::ToBufferOp::create(rewriter, loc, memrefType, tensor);
 }
 
 /// Prepares the outlined arguments, passing scalars and buffers in. Here we
@@ -293,13 +293,13 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
   // so that:
   //   row = blockIdx.x * blockDim.x + threadIdx.x
   //   inc = blockDim.x * gridDim.x
-  Value bid = rewriter.create<gpu::BlockIdOp>(loc, gpu::Dimension::x);
-  Value bsz = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::x);
-  Value tid = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
-  Value gsz = rewriter.create<gpu::GridDimOp>(loc, gpu::Dimension::x);
-  Value mul = rewriter.create<arith::MulIOp>(loc, bid, bsz);
-  Value row = rewriter.create<arith::AddIOp>(loc, mul, tid);
-  Value inc = rewriter.create<arith::MulIOp>(loc, bsz, gsz);
+  Value bid = gpu::BlockIdOp::create(rewriter, loc, gpu::Dimension::x);
+  Value bsz = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::x);
+  Value tid = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
+  Value gsz = gpu::GridDimOp::create(rewriter, loc, gpu::Dimension::x);
+  Value mul = arith::MulIOp::create(rewriter, loc, bid, bsz);
+  Value row = arith::AddIOp::create(rewriter, loc, mul, tid);
+  Value inc = arith::MulIOp::create(rewriter, loc, bsz, gsz);
 
   // Construct the iteration over the computational space that
   // accounts for the fact that the total number of threads and
@@ -308,7 +308,7 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
   //     <loop-body>
   //   }
   Value upper = irMap.lookup(forallOp.getUpperBound()[0]);
-  scf::ForOp forOp = rewriter.create<scf::ForOp>(loc, row, upper, inc);
+  scf::ForOp forOp = scf::ForOp::create(rewriter, loc, row, upper, inc);
   // The scf.for builder creates an empty block. scf.for does not allow multiple
   // blocks in its region, so delete the block before `cloneRegionBefore` adds
   // an additional block.
@@ -321,7 +321,7 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
 
   // Done.
   rewriter.setInsertionPointAfter(forOp);
-  rewriter.create<gpu::ReturnOp>(gpuFunc->getLoc());
+  gpu::ReturnOp::create(rewriter, gpuFunc->getLoc());
 }
 
 //===----------------------------------------------------------------------===//
@@ -496,11 +496,11 @@ static Value genFirstPosOrCrds(OpBuilder &builder, Location loc, Value a,
   if (format == CuSparseFormat::kCOO) {
     // Library uses SoA COO, direct IR uses AoS COO.
     if (enableRT)
-      return builder.create<ToCoordinatesOp>(loc, a, 0);
-    return builder.create<ToCoordinatesBufferOp>(loc, a);
+      return ToCoordinatesOp::create(builder, loc, a, 0);
+    return ToCoordinatesBufferOp::create(builder, loc, a);
   }
   // Formats CSR/CSC and BSR use positions at 1.
-  return builder.create<ToPositionsOp>(loc, a, 1);
+  return ToPositionsOp::create(builder, loc, a, 1);
 }
 
 /// Generates the second coordinates of a sparse matrix.
@@ -510,7 +510,7 @@ static Value genSecondCrds(OpBuilder &builder, Location loc, Value a,
   if (isCOO && !enableRT)
     return Value(); // nothing needed
   // Formats CSR/CSC and BSR use coordinates at 1.
-  return builder.create<ToCoordinatesOp>(loc, a, 1);
+  return ToCoordinatesOp::create(builder, loc, a, 1);
 }
 
 /// Generates the sparse matrix handle.
@@ -523,12 +523,12 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
     // Library uses SoA COO, direct IR uses AoS COO.
     if (enableRT) {
       assert(colA);
-      return builder.create<gpu::CreateCooOp>(loc, handleTp, tokenTp, token,
+      return gpu::CreateCooOp::create(builder, loc, handleTp, tokenTp, token,
                                               sz1, sz2, nseA, rowA, colA, valA);
     }
 #ifdef CUSPARSE_COO_AOS
     assert(!colA);
-    return builder.create<gpu::CreateCooAoSOp>(loc, handleTp, tokenTp, token,
+    return gpu::CreateCooAoSOp::create(builder, loc, handleTp, tokenTp, token,
                                                sz1, sz2, nseA, rowA, valA);
 #else
     llvm_unreachable("gpu::CreateCooAoSOp is deprecated");
@@ -536,10 +536,10 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   }
   assert(colA);
   if (format == CuSparseFormat::kCSR)
-    return builder.create<gpu::CreateCsrOp>(loc, handleTp, tokenTp, token, sz1,
+    return gpu::CreateCsrOp::create(builder, loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
   if (format == CuSparseFormat::kCSC)
-    return builder.create<gpu::CreateCscOp>(loc, handleTp, tokenTp, token, sz1,
+    return gpu::CreateCscOp::create(builder, loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
   // BSR requires a bit more work since we need to pass in the block size
   // and all others sizes in terms of blocks (#block-rows, #block-cols,
@@ -549,11 +549,11 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   assert(dims.size() == 2 && dims[0] == dims[1]);
   uint64_t b = dims[0];
   Value bSz = constantIndex(builder, loc, b);
-  Value bRows = builder.create<arith::DivUIOp>(loc, sz1, bSz);
-  Value bCols = builder.create<arith::DivUIOp>(loc, sz2, bSz);
-  Value bNum = builder.create<arith::DivUIOp>(
+  Value bRows = arith::DivUIOp::create(builder, loc, sz1, bSz);
+  Value bCols = arith::DivUIOp::create(builder, loc, sz2, bSz);
+  Value bNum = arith::DivUIOp::create(builder,
       loc, nseA, constantIndex(builder, loc, b * b));
-  return builder.create<gpu::CreateBsrOp>(loc, handleTp, tokenTp, token, bRows,
+  return gpu::CreateBsrOp::create(builder, loc, handleTp, tokenTp, token, bRows,
                                           bCols, bNum, bSz, bSz, rowA, colA,
                                           valA);
 }
@@ -579,12 +579,12 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   //   a : memR/memC/memV -> rowA,colA,valA
   //   x : memX           -> vecX
   //   y : memY           -> vecY
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
   Value szY = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szX = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value memR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, a, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, a);
+  Value memV = ToValuesOp::create(rewriter, loc, a);
   Value rowA = genAllocCopy(rewriter, loc, memR, tokens);
   Value colA = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valA = genAllocCopy(rewriter, loc, memV, tokens);
@@ -606,18 +606,18 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dvecX = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dvecX = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, vecX, szX);
   Value dnX = dvecX.getResult(0);
   token = dvecX.getAsyncToken();
-  auto dvecY = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dvecY = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, vecY, szY);
   Value dnY = dvecY.getResult(0);
   token = dvecY.getAsyncToken();
   auto dnYType = llvm::cast<ShapedType>(y.getType()).getElementType();
 
   // Precompute buffersize for SpMV.
-  auto bufferComp = rewriter.create<gpu::SpMVBufferSizeOp>(
+  auto bufferComp = gpu::SpMVBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, spMatA, dnX, dnY,
       /*computeType=*/dnYType);
   Value bufferSz = bufferComp.getResult(0);
@@ -627,16 +627,16 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SpMV.
-  auto spmvComp = rewriter.create<gpu::SpMVOp>(
+  auto spmvComp = gpu::SpMVOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnX, dnY, /*computeType=*/dnYType, buffer);
   token = spmvComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnX)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnX)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnY)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnY)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, rowA, token);
   if (colA)
@@ -676,13 +676,13 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   //   a : memR/memC/memV -> rowA,colA,valA
   //   b : bufB           -> matB
   //   c : bufC           -> matC
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
   Value memR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, a, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, a);
+  Value memV = ToValuesOp::create(rewriter, loc, a);
   Value rowA = genAllocCopy(rewriter, loc, memR, tokens);
   Value colA = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valA = genAllocCopy(rewriter, loc, memV, tokens);
@@ -704,12 +704,12 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matB,
       SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matC,
       SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
@@ -717,7 +717,7 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dmatCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SpMM.
-  auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
   Value bufferSz = bufferComp.getResult(0);
@@ -728,16 +728,16 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Perform the SpMM.
-  auto spmmComp = rewriter.create<gpu::SpMMOp>(
+  auto spmmComp = gpu::SpMMOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType, buffer);
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnC)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, rowA, token);
   if (colA)
@@ -778,17 +778,17 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   //   b : bmemR/bmemC/bmemV -> rowB,colB,valB
   //   c : materializes
   auto dnCType = cTp.getElementType();
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
-  Value nseB = rewriter.create<NumberOfEntriesOp>(loc, b);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
+  Value nseB = NumberOfEntriesOp::create(rewriter, loc, b);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
   Value amemR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value amemC = genSecondCrds(rewriter, loc, a, format, enableRT); // not empty
-  Value amemV = rewriter.create<ToValuesOp>(loc, a);
+  Value amemV = ToValuesOp::create(rewriter, loc, a);
   Value bmemR = genFirstPosOrCrds(rewriter, loc, b, format, enableRT);
   Value bmemC = genSecondCrds(rewriter, loc, b, format, enableRT); // not empty
-  Value bmemV = rewriter.create<ToValuesOp>(loc, b);
+  Value bmemV = ToValuesOp::create(rewriter, loc, b);
   Value rowA = genAllocCopy(rewriter, loc, amemR, tokens);
   Value colA = genAllocCopy(rewriter, loc, amemC, tokens);
   Value valA = genAllocCopy(rewriter, loc, amemV, tokens);
@@ -818,7 +818,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   // Sparse matrix C materializes (also assumes beta == 0).
   Value zero = constantIndex(rewriter, loc, 0);
   Value one = constantIndex(rewriter, loc, 1);
-  Value mplus1 = rewriter.create<arith::AddIOp>(loc, szm, one);
+  Value mplus1 = arith::AddIOp::create(rewriter, loc, szm, one);
   auto e1 = genAllocBuffer(rewriter, loc, cTp.getPosType(), mplus1, token);
   Value rowC = e1.getResult(0);
   token = e1.getAsyncToken();
@@ -836,10 +836,10 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
 
   // Precompute buffersizes for SpGEMM.
   Operation *descOp =
-      rewriter.create<gpu::SpGEMMCreateDescrOp>(loc, descTp, tokenTp, token);
+      gpu::SpGEMMCreateDescrOp::create(rewriter, loc, descTp, tokenTp, token);
   Value desc = descOp->getResult(0);
   token = descOp->getResult(1);
-  Operation *work1 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
       valC, gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
@@ -848,7 +848,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   auto buf1 = genAllocBuffer(rewriter, loc, bufferSz1, token);
   Value buffer1 = buf1.getResult(0);
   token = buf1.getAsyncToken();
-  Operation *work2 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
       bufferSz1, buffer1,
@@ -856,7 +856,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = work2->getResult(1);
 
   // Compute step.
-  Operation *compute1 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
       valC, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
@@ -865,14 +865,14 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   auto buf2 = genAllocBuffer(rewriter, loc, bufferSz2, token);
   Value buffer2 = buf2.getResult(0);
   token = buf2.getAsyncToken();
-  Operation *compute2 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
       bufferSz2, buffer2, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
   token = compute2->getResult(1);
 
   // Get sizes.
-  Operation *sizes = rewriter.create<gpu::SpMatGetSizeOp>(
+  Operation *sizes = gpu::SpMatGetSizeOp::create(rewriter,
       loc, indexTp, indexTp, indexTp, tokenTp, token, spMatC);
   Value nnz = sizes->getResult(2);
   token = sizes->getResult(3);
@@ -884,10 +884,10 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = a3.getAsyncToken();
 
   // Update C with new pointers and copy final product back into C.
-  Operation *update = rewriter.create<gpu::SetCsrPointersOp>(
+  Operation *update = gpu::SetCsrPointersOp::create(rewriter,
       loc, tokenTp, token, spMatC, rowC, colC, valC);
   token = update->getResult(0);
-  Operation *copy = rewriter.create<gpu::SpGEMMCopyOp>(
+  Operation *copy = gpu::SpGEMMCopyOp::create(rewriter,
       loc, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType);
   token = copy->getResult(0);
@@ -898,13 +898,13 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   Value valH = genHostBuffer(rewriter, loc, dnCType, nnz);
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::SpGEMMDestroyDescrOp>(loc, tokenTp, token, desc)
+  token = gpu::SpGEMMDestroyDescrOp::create(rewriter, loc, tokenTp, token, desc)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatB)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatC)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatC)
               .getAsyncToken();
   token = genCopyMemRef(rewriter, loc, rowH, rowC, token);
   token = genCopyMemRef(rewriter, loc, colH, colC, token);
@@ -925,11 +925,11 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  Value vt = rewriter.create<bufferization::ToTensorOp>(
+  Value vt = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
-  Value rt = rewriter.create<bufferization::ToTensorOp>(
+  Value rt = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
-  Value ct = rewriter.create<bufferization::ToTensorOp>(
+  Value ct = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
   rewriter.replaceOpWithNewOp<AssembleOp>(op, c.getType(), ValueRange{rt, ct},
                                           vt);
@@ -980,17 +980,17 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  Operation *spGenA = rewriter.create<gpu::Create2To4SpMatOp>(
+  Operation *spGenA = gpu::Create2To4SpMatOp::create(rewriter,
       loc, spMatHandleTp, tokenTp, token, szm, szk,
       gpu::Prune2To4SpMatFlag::PRUNE_AND_CHECK, matA);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matB,
       SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matC,
       SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
@@ -1000,7 +1000,7 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   // Precompute buffersize for SpMM.
   SmallVector<Type> bufferTypes_{indexTp, indexTp, indexTp};
   TypeRange bufferTypes(bufferTypes_);
-  auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
       loc, bufferTypes, tokenTp, token, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
@@ -1022,17 +1022,17 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
 
   // Perform the SpMM.
   auto dnCType = llvm::cast<ShapedType>(matC.getType()).getElementType();
-  auto spmmComp = rewriter.create<gpu::SpMMOp>(
+  auto spmmComp = gpu::SpMMOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType,
       SmallVector<Value>{buffer1, buffer2, buffer3});
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resources.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnC)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, buffer1, token);
   token = genDeallocMemRef(rewriter, loc, buffer2, token);
@@ -1073,7 +1073,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   //   a : bufA           -> matA
   //   b : bufB           -> matB
   //   c : memR/memC/memV -> rowC,colC,valC
-  Value nseC = rewriter.create<NumberOfEntriesOp>(loc, c);
+  Value nseC = NumberOfEntriesOp::create(rewriter, loc, c);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
@@ -1083,7 +1083,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Value matB = genAllocCopy(rewriter, loc, bufB, tokens);
   Value memR = genFirstPosOrCrds(rewriter, loc, c, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, c, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, c);
+  Value memV = ToValuesOp::create(rewriter, loc, c);
   Value rowC = genAllocCopy(rewriter, loc, memR, tokens);
   Value colC = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valC = genAllocCopy(rewriter, loc, memV, tokens);
@@ -1096,11 +1096,11 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  auto dmatA = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatA = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnMatHandleTp, tokenTp, token, matA, SmallVector<Value>{szm, szk});
   Value dnA = dmatA.getResult(0);
   token = dmatA.getAsyncToken();
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnMatHandleTp, tokenTp, token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
@@ -1112,7 +1112,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SDDMM.
-  auto bufferComp = rewriter.create<gpu::SDDMMBufferSizeOp>(
+  auto bufferComp = gpu::SDDMMBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, dnA, dnB, spMatC, dnCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
@@ -1121,16 +1121,16 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SDDMM.
-  auto sddmmComp = rewriter.create<gpu::SDDMMOp>(loc, tokenTp, token, dnA, dnB,
+  auto sddmmComp = gpu::SDDMMOp::create(rewriter, loc, tokenTp, token, dnA, dnB,
                                                  spMatC, dnCType, buffer);
   token = sddmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnA)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatC)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, buffer, token);
   token = genDeallocMemRef(rewriter, loc, matA, token);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
index 2f68008e68b5f..0986a0f2ea0f0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
@@ -67,11 +67,11 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
       op.getRegionDefinedSpace(newBlock->getParent()->getRegionNumber());
   for (unsigned i : caseBits.bits()) {
     SparseIterator *it = iters[i].get();
-    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                 it->getCrd(), loopCrd);
-    casePred = rewriter.create<arith::AndIOp>(loc, casePred, pred);
+    casePred = arith::AndIOp::create(rewriter, loc, casePred, pred);
   }
-  scf::IfOp ifOp = rewriter.create<scf::IfOp>(
+  scf::IfOp ifOp = scf::IfOp::create(rewriter,
       loc, ValueRange(userReduc).getTypes(), casePred, /*else=*/true);
   rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
@@ -103,7 +103,7 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
   ValueRange yields = spY.getResults();
   rewriter.eraseOp(spY);
   rewriter.setInsertionPointToEnd(&ifOp.getThenRegion().front());
-  rewriter.create<scf::YieldOp>(loc, yields);
+  scf::YieldOp::create(rewriter, loc, yields);
 
   // Generates remaining case recursively.
   rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -111,7 +111,7 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
                                           newBlocks.drop_front(),
                                           oldBlocks.drop_front(), userReduc);
   if (!res.empty())
-    rewriter.create<scf::YieldOp>(loc, res);
+    scf::YieldOp::create(rewriter, loc, res);
 
   rewriter.setInsertionPointAfter(ifOp);
   return ifOp.getResults();
@@ -127,7 +127,7 @@ static ValueRange genLoopWithIterator(
   if (it->iteratableByFor()) {
     auto [lo, hi] = it->genForCond(rewriter, loc);
     Value step = constantIndex(rewriter, loc, 1);
-    scf::ForOp forOp = rewriter.create<scf::ForOp>(
+    scf::ForOp forOp = scf::ForOp::create(rewriter,
         loc, lo, hi, step, reduc,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
           // Empty builder function to ensure that no terminator is created.
@@ -140,7 +140,7 @@ static ValueRange genLoopWithIterator(
                                            it, forOp.getRegionIterArgs());
 
       rewriter.setInsertionPointToEnd(forOp.getBody());
-      rewriter.create<scf::YieldOp>(loc, ret);
+      scf::YieldOp::create(rewriter, loc, ret);
     }
     return forOp.getResults();
   }
@@ -149,7 +149,7 @@ static ValueRange genLoopWithIterator(
   llvm::append_range(ivs, it->getCursor());
 
   TypeRange types = ValueRange(ivs).getTypes();
-  auto whileOp = rewriter.create<scf::WhileOp>(loc, types, ivs);
+  auto whileOp = scf::WhileOp::create(rewriter, loc, types, ivs);
   {
     OpBuilder::InsertionGuard guard(rewriter);
     // Generates loop conditions.
@@ -158,7 +158,7 @@ static ValueRange genLoopWithIterator(
     rewriter.setInsertionPointToStart(before);
     ValueRange bArgs = before->getArguments();
     auto [whileCond, remArgs] = it->genWhileCond(rewriter, loc, bArgs);
-    rewriter.create<scf::ConditionOp>(loc, whileCond, before->getArguments());
+    scf::ConditionOp::create(rewriter, loc, whileCond, before->getArguments());
 
     // Delegates loop body generation.
     Region &dstRegion = whileOp.getAfter();
@@ -175,7 +175,7 @@ static ValueRange genLoopWithIterator(
     SmallVector<Value> yields;
     llvm::append_range(yields, ret);
     llvm::append_range(yields, it->forward(rewriter, loc));
-    rewriter.create<scf::YieldOp>(loc, yields);
+    scf::YieldOp::create(rewriter, loc, yields);
   }
   return whileOp.getResults().drop_front(it->getCursor().size());
 }
@@ -212,7 +212,7 @@ class ExtractValOpConverter : public OpConversionPattern<ExtractValOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value pos = adaptor.getIterator().back();
-    Value valBuf = rewriter.create<ToValuesOp>(
+    Value valBuf = ToValuesOp::create(rewriter,
         loc, llvm::getSingleElement(adaptor.getTensor()));
     rewriter.replaceOpWithNewOp<memref::LoadOp>(op, valBuf, pos);
     return success();
@@ -385,12 +385,12 @@ class SparseCoIterateOpConverter : public OpConversionPattern<CoIterateOp> {
         SmallVector<Value> nextIterYields(res);
         // 2nd. foward the loop.
         for (SparseIterator *it : validIters) {
-          Value cmp = rewriter.create<arith::CmpIOp>(
+          Value cmp = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::eq, it->getCrd(), loopCrd);
           it->forwardIf(rewriter, loc, cmp);
           llvm::append_range(nextIterYields, it->getCursor());
         }
-        rewriter.create<scf::YieldOp>(loc, nextIterYields);
+        scf::YieldOp::create(rewriter, loc, nextIterYields);
 
         // Exit the loop, relink the iterator SSA value.
         rewriter.setInsertionPointAfter(loop);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
index 0473e058646b9..fd1bbb300b6a6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
@@ -44,7 +44,7 @@ struct DemapInsRewriter : public OpRewritePattern<SourceOp> {
     SmallVector<Value> deMappedIns(op->getOperands());
     for (Value &in : deMappedIns) {
       if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) {
-        in = rewriter.create<ReinterpretMapOp>(loc, stt->getDemappedType(), in);
+        in = ReinterpretMapOp::create(rewriter, loc, stt->getDemappedType(), in);
         changed = true;
       }
     }
@@ -338,14 +338,14 @@ translateMap(linalg::GenericOp op, PatternRewriter &rewriter) {
 // Generates a "de"mapping reinterpretation of the map.
 static Value genDemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
-  return builder.create<ReinterpretMapOp>(val.getLoc(), enc.withoutDimToLvl(),
+  return ReinterpretMapOp::create(builder, val.getLoc(), enc.withoutDimToLvl(),
                                           val);
 }
 
 // Generates a "re"mapping reinterpretation of the map.
 static Value genRemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
-  return builder.create<ReinterpretMapOp>(val.getLoc(), enc, val);
+  return ReinterpretMapOp::create(builder, val.getLoc(), enc, val);
 }
 
 static SmallVector<Value> remapValueRange(OpBuilder &rewriter, TypeRange types,
@@ -354,7 +354,7 @@ static SmallVector<Value> remapValueRange(OpBuilder &rewriter, TypeRange types,
   assert(outs.size() == types.size());
   for (auto [r, t] : llvm::zip(ret, types))
     if (r.getType() != t)
-      r = rewriter.create<ReinterpretMapOp>(r.getLoc(), t, r);
+      r = ReinterpretMapOp::create(rewriter, r.getLoc(), t, r);
   return ret;
 }
 
@@ -567,7 +567,7 @@ struct GenericOpScheduler : public OpRewritePattern<linalg::GenericOp> {
       // Inserting the transpose
       rewriter.setInsertionPoint(linalgOp);
       RankedTensorType dstTp = stt.withDimToLvl(dimToLvl).getRankedTensorType();
-      Value dst = rewriter.create<ConvertOp>(tval.getLoc(), dstTp, tval);
+      Value dst = ConvertOp::create(rewriter, tval.getLoc(), dstTp, tval);
       rewriter.modifyOpInPlace(linalgOp, [&]() {
         linalgOp->setOperand(t->getOperandNumber(), dst);
       });
@@ -575,7 +575,7 @@ struct GenericOpScheduler : public OpRewritePattern<linalg::GenericOp> {
       // Release the transposed form afterwards.
       // TODO: CSE when used in more than one following op?
       rewriter.setInsertionPointAfter(linalgOp);
-      rewriter.create<bufferization::DeallocTensorOp>(dst.getLoc(), dst);
+      bufferization::DeallocTensorOp::create(rewriter, dst.getLoc(), dst);
 
       return success();
     }
@@ -605,7 +605,7 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     ValueRange dynSz = op.getDynamicSizes();
     for (int64_t dimSz : stt.getDimShape()) {
       if (ShapedType::isDynamic(dimSz)) {
-        Value maxCrd = rewriter.create<arith::SubIOp>(
+        Value maxCrd = arith::SubIOp::create(rewriter,
             loc, dynSz.front(), constantIndex(rewriter, loc, 1));
         maxDimCrds.push_back(maxCrd);
         dynSz = dynSz.drop_front();
@@ -620,7 +620,7 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     SmallVector<Value> dynLvlSzs;
     for (unsigned i = 0, e = lvlShape.size(); i < e; i++) {
       if (ShapedType::isDynamic(lvlShape[i])) {
-        Value sz = rewriter.create<arith::AddIOp>(
+        Value sz = arith::AddIOp::create(rewriter,
             loc, maxLvlCrds[i], constantIndex(rewriter, loc, 1));
         dynLvlSzs.push_back(sz);
       }
@@ -651,7 +651,7 @@ struct TensorInsertDemapper
     auto stt = getSparseTensorType(op.getResult());
     ValueRange lvlCrd = stt.translateCrds(rewriter, loc, op.getIndices(),
                                           CrdTransDirectionKind::dim2lvl);
-    auto insertOp = rewriter.create<tensor::InsertOp>(
+    auto insertOp = tensor::InsertOp::create(rewriter,
         loc, op.getScalar(), adaptor.getDest(), lvlCrd);
 
     Value out = genRemap(rewriter, stt.getEncoding(), insertOp.getResult());
@@ -766,7 +766,7 @@ struct ForeachOpDemapper
           stt && !stt->isIdentity()) {
         Value y =
             genDemap(rewriter, stt->getEncoding(), yield.getSingleResult());
-        rewriter.create<YieldOp>(loc, y);
+        YieldOp::create(rewriter, loc, y);
         rewriter.eraseOp(yield);
       }
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
index f85c4761a8d52..7b662c2fb0db0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
@@ -126,7 +126,7 @@ void collapseSparseSpace(MutableArrayRef<CollapseSpaceInfo> toCollapse) {
   OpBuilder builder(root);
 
   // Construct the collapsed iteration space.
-  auto collapsedSpace = builder.create<ExtractIterSpaceOp>(
+  auto collapsedSpace = ExtractIterSpaceOp::create(builder,
       loc, root.getTensor(), root.getParentIter(), root.getLoLvl(),
       leaf.getHiLvl());
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
index 01028f71c20bb..a1b9035f3b6e6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
@@ -69,13 +69,13 @@ class SpecifierStructBuilder : public StructBuilder {
   Value extractField(OpBuilder &builder, Location loc,
                      ArrayRef<int64_t> indices) const {
     return genCast(builder, loc,
-                   builder.create<LLVM::ExtractValueOp>(loc, value, indices),
+                   LLVM::ExtractValueOp::create(builder, loc, value, indices),
                    builder.getIndexType());
   }
 
   void insertField(OpBuilder &builder, Location loc, ArrayRef<int64_t> indices,
                    Value v) {
-    value = builder.create<LLVM::InsertValueOp>(
+    value = LLVM::InsertValueOp::create(builder,
         loc, value, genCast(builder, loc, v, builder.getIntegerType(64)),
         indices);
   }
@@ -110,7 +110,7 @@ class SpecifierStructBuilder : public StructBuilder {
 
 Value SpecifierStructBuilder::getInitValue(OpBuilder &builder, Location loc,
                                            Type structType, Value source) {
-  Value metaData = builder.create<LLVM::PoisonOp>(loc, structType);
+  Value metaData = LLVM::PoisonOp::create(builder, loc, structType);
   SpecifierStructBuilder md(metaData);
   if (!source) {
     auto memSizeArrayType =
@@ -204,14 +204,14 @@ void SpecifierStructBuilder::setMemSize(OpBuilder &builder, Location loc,
 /// Builds IR extracting the memory size array from the descriptor.
 Value SpecifierStructBuilder::memSizeArray(OpBuilder &builder,
                                            Location loc) const {
-  return builder.create<LLVM::ExtractValueOp>(loc, value,
+  return LLVM::ExtractValueOp::create(builder, loc, value,
                                               kMemSizePosInSpecifier);
 }
 
 /// Builds IR inserting the memory size array into the descriptor.
 void SpecifierStructBuilder::setMemSizeArray(OpBuilder &builder, Location loc,
                                              Value array) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, array,
+  value = LLVM::InsertValueOp::create(builder, loc, value, array,
                                               kMemSizePosInSpecifier);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 001ea62b07360..0450e1d6e8775 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -50,7 +50,7 @@ static SmallVector<Value> flattenValues(ArrayRef<ValueRange> values) {
 /// Generates a load with proper `index` typing.
 static Value genLoad(OpBuilder &builder, Location loc, Value mem, Value idx) {
   idx = genCast(builder, loc, idx, builder.getIndexType());
-  return builder.create<memref::LoadOp>(loc, mem, idx);
+  return memref::LoadOp::create(builder, loc, mem, idx);
 }
 
 /// Generates a store with proper `index` typing and proper value.
@@ -59,7 +59,7 @@ static void genStore(OpBuilder &builder, Location loc, Value val, Value mem,
   idx = genCast(builder, loc, idx, builder.getIndexType());
   val = genCast(builder, loc, val,
                 cast<ShapedType>(mem.getType()).getElementType());
-  builder.create<memref::StoreOp>(loc, val, mem, idx);
+  memref::StoreOp::create(builder, loc, val, mem, idx);
 }
 
 /// Creates a straightforward counting for-loop.
@@ -70,7 +70,7 @@ static scf::ForOp createFor(OpBuilder &builder, Location loc, Value upper,
   if (!lower)
     lower = constantZero(builder, loc, indexType);
   Value one = constantOne(builder, loc, indexType);
-  scf::ForOp forOp = builder.create<scf::ForOp>(loc, lower, upper, one, fields);
+  scf::ForOp forOp = scf::ForOp::create(builder, loc, lower, upper, one, fields);
   for (unsigned i = 0, e = fields.size(); i < e; i++)
     fields[i] = forOp.getRegionIterArg(i);
   builder.setInsertionPointToStart(forOp.getBody());
@@ -86,7 +86,7 @@ static void createPushback(OpBuilder &builder, Location loc,
   Value field = desc.getMemRefField(kind, lvl);
   StorageSpecifierKind specFieldKind = toSpecifierKind(kind);
 
-  auto pushBackOp = builder.create<PushBackOp>(
+  auto pushBackOp = PushBackOp::create(builder,
       loc, desc.getSpecifierField(builder, loc, specFieldKind, lvl), field,
       genCast(builder, loc, value, etp), repeat);
 
@@ -112,7 +112,7 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
       Value posZero = constantZero(builder, loc, stt.getPosType());
       if (isLooseCompressedLT(lt)) {
         Value two = constantIndex(builder, loc, 2);
-        linear = builder.create<arith::MulIOp>(loc, linear, two);
+        linear = arith::MulIOp::create(builder, loc, linear, two);
       }
       createPushback(builder, loc, desc, SparseTensorFieldKind::PosMemRef, lvl,
                      /*value=*/posZero, /*repeat=*/linear);
@@ -125,7 +125,7 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
     // otherwise the values array for the from-here "all-dense" case.
     assert(isDenseLT(lt));
     Value size = desc.getLvlSize(builder, loc, lvl);
-    linear = builder.create<arith::MulIOp>(loc, linear, size);
+    linear = arith::MulIOp::create(builder, loc, linear, size);
   }
   // Reached values array so prepare for an insertion.
   Value valZero = constantZero(builder, loc, stt.getElementType());
@@ -137,11 +137,11 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
 static Value createAllocation(OpBuilder &builder, Location loc,
                               MemRefType memRefType, Value sz,
                               bool enableInit) {
-  Value buffer = builder.create<memref::AllocOp>(loc, memRefType, sz);
+  Value buffer = memref::AllocOp::create(builder, loc, memRefType, sz);
   Type elemType = memRefType.getElementType();
   if (enableInit) {
     Value fillValue = constantZero(builder, loc, elemType);
-    builder.create<linalg::FillOp>(loc, fillValue, buffer);
+    linalg::FillOp::create(builder, loc, fillValue, buffer);
   }
   return buffer;
 }
@@ -179,14 +179,14 @@ static void createAllocFields(OpBuilder &builder, Location loc,
     valHeuristic = lvlSizesValues[0];
     for (Level lvl = 1; lvl < lvlRank; lvl++)
       valHeuristic =
-          builder.create<arith::MulIOp>(loc, valHeuristic, lvlSizesValues[lvl]);
+          arith::MulIOp::create(builder, loc, valHeuristic, lvlSizesValues[lvl]);
   } else if (sizeHint) {
     if (stt.getAoSCOOStart() == 0) {
       posHeuristic = constantIndex(builder, loc, 2);
-      crdHeuristic = builder.create<arith::MulIOp>(
+      crdHeuristic = arith::MulIOp::create(builder,
           loc, constantIndex(builder, loc, lvlRank), sizeHint); // AOS
     } else if (lvlRank == 2 && stt.isDenseLvl(0) && stt.isCompressedLvl(1)) {
-      posHeuristic = builder.create<arith::AddIOp>(
+      posHeuristic = arith::AddIOp::create(builder,
           loc, sizeHint, constantIndex(builder, loc, 1));
       crdHeuristic = sizeHint;
     } else {
@@ -280,7 +280,7 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   unsigned crdStride;
   std::tie(crdFidx, crdStride) = desc.getCrdMemRefIndexAndStride(lvl);
   const Value one = constantIndex(builder, loc, 1);
-  const Value pp1 = builder.create<arith::AddIOp>(loc, parentPos, one);
+  const Value pp1 = arith::AddIOp::create(builder, loc, parentPos, one);
   const Value positionsAtLvl = desc.getPosMemRef(lvl);
   const Value pstart = genLoad(builder, loc, positionsAtLvl, parentPos);
   const Value pstop = genLoad(builder, loc, positionsAtLvl, pp1);
@@ -288,29 +288,29 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   const Value crdStrideC =
       crdStride > 1 ? constantIndex(builder, loc, crdStride) : Value();
   const Value msz =
-      crdStrideC ? builder.create<arith::DivUIOp>(loc, crdMsz, crdStrideC)
+      crdStrideC ? arith::DivUIOp::create(builder, loc, crdMsz, crdStrideC)
                  : crdMsz;
-  const Value plast = builder.create<arith::SubIOp>(
+  const Value plast = arith::SubIOp::create(builder,
       loc, genCast(builder, loc, pstop, indexType), one);
   // Conditional expression.
-  Value lt = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value lt = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                            pstart, pstop);
   types.push_back(boolType);
-  scf::IfOp ifOp1 = builder.create<scf::IfOp>(loc, types, lt, /*else*/ true);
+  scf::IfOp ifOp1 = scf::IfOp::create(builder, loc, types, lt, /*else*/ true);
   types.pop_back();
   builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
   Value crd =
       genLoad(builder, loc, desc.getMemRefField(crdFidx),
-              crdStrideC ? builder.create<arith::MulIOp>(loc, plast, crdStrideC)
+              crdStrideC ? arith::MulIOp::create(builder, loc, plast, crdStrideC)
                          : plast);
-  Value eq = builder.create<arith::CmpIOp>(
+  Value eq = arith::CmpIOp::create(builder,
       loc, arith::CmpIPredicate::eq, genCast(builder, loc, crd, indexType),
       lvlCoords[lvl]);
-  builder.create<scf::YieldOp>(loc, eq);
+  scf::YieldOp::create(builder, loc, eq);
   builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
   if (lvl > 0)
     genStore(builder, loc, msz, positionsAtLvl, parentPos);
-  builder.create<scf::YieldOp>(loc, constantI1(builder, loc, false));
+  scf::YieldOp::create(builder, loc, constantI1(builder, loc, false));
   builder.setInsertionPointAfter(ifOp1);
   // If present construct. Note that for a non-unique dimension level, we
   // simply set the condition to false and rely on CSE/DCE to clean up the IR.
@@ -322,19 +322,19 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   types.push_back(indexType);
   const Value p = stt.isUniqueLvl(lvl) ? ifOp1.getResult(0)
                                        : constantI1(builder, loc, false);
-  scf::IfOp ifOp2 = builder.create<scf::IfOp>(loc, types, p, /*else*/ true);
+  scf::IfOp ifOp2 = scf::IfOp::create(builder, loc, types, p, /*else*/ true);
   // If present (fields unaffected, update pnext to plast).
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
 
   // FIXME: This does not looks like a clean way, but probably the most
   // efficient way.
   desc.getFields().push_back(plast);
-  builder.create<scf::YieldOp>(loc, desc.getFields());
+  scf::YieldOp::create(builder, loc, desc.getFields());
   desc.getFields().pop_back();
 
   // If !present (changes fields, update pnext).
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
-  Value mszp1 = builder.create<arith::AddIOp>(loc, msz, one);
+  Value mszp1 = arith::AddIOp::create(builder, loc, msz, one);
   genStore(builder, loc, mszp1, positionsAtLvl, pp1);
   createPushback(builder, loc, desc, SparseTensorFieldKind::CrdMemRef, lvl,
                  /*value=*/lvlCoords[lvl]);
@@ -343,7 +343,7 @@ static Value genCompressed(OpBuilder &builder, Location loc,
     allocSchemeForRank(builder, loc, desc, lvl + 1);
 
   desc.getFields().push_back(msz);
-  builder.create<scf::YieldOp>(loc, desc.getFields());
+  scf::YieldOp::create(builder, loc, desc.getFields());
   desc.getFields().pop_back();
 
   // Update fields and return next pos.
@@ -381,17 +381,17 @@ static void genEndInsert(OpBuilder &builder, Location loc,
         Value oldv = loop.getRegionIterArg(0);
         Value newv = genLoad(builder, loc, posMemRef, i);
         Value posZero = constantZero(builder, loc, posType);
-        Value cond = builder.create<arith::CmpIOp>(
+        Value cond = arith::CmpIOp::create(builder,
             loc, arith::CmpIPredicate::eq, newv, posZero);
-        scf::IfOp ifOp = builder.create<scf::IfOp>(loc, TypeRange(posType),
+        scf::IfOp ifOp = scf::IfOp::create(builder, loc, TypeRange(posType),
                                                    cond, /*else*/ true);
         builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         genStore(builder, loc, oldv, posMemRef, i);
-        builder.create<scf::YieldOp>(loc, oldv);
+        scf::YieldOp::create(builder, loc, oldv);
         builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-        builder.create<scf::YieldOp>(loc, newv);
+        scf::YieldOp::create(builder, loc, newv);
         builder.setInsertionPointAfter(ifOp);
-        builder.create<scf::YieldOp>(loc, ifOp.getResult(0));
+        scf::YieldOp::create(builder, loc, ifOp.getResult(0));
         builder.setInsertionPointAfter(loop);
       }
     } else {
@@ -484,7 +484,7 @@ class SparseInsertGenerator
         //   <insert @ positions[lvl] at next level lvl + 1>
         if (isLooseCompressedLT(lt)) {
           Value two = constantIndex(builder, loc, 2);
-          parentPos = builder.create<arith::MulIOp>(loc, parentPos, two);
+          parentPos = arith::MulIOp::create(builder, loc, parentPos, two);
         }
         parentPos =
             genCompressed(builder, loc, desc, coords, value, parentPos, lvl);
@@ -501,8 +501,8 @@ class SparseInsertGenerator
         //   positions[lvl] = size * positions[lvl-1] + coords[lvl]
         //   <insert @ positions[lvl] at next level lvl + 1>
         Value size = desc.getLvlSize(builder, loc, lvl);
-        Value mult = builder.create<arith::MulIOp>(loc, size, parentPos);
-        parentPos = builder.create<arith::AddIOp>(loc, mult, coords[lvl]);
+        Value mult = arith::MulIOp::create(builder, loc, size, parentPos);
+        parentPos = arith::AddIOp::create(builder, loc, mult, coords[lvl]);
       }
     }
     // Reached the actual value append/insert.
@@ -582,7 +582,7 @@ class SparseCallConverter : public OpConversionPattern<func::CallOp> {
       return failure();
 
     // (1) Generates new call with flattened return value.
-    auto newCall = rewriter.create<func::CallOp>(
+    auto newCall = func::CallOp::create(rewriter,
         loc, op.getCallee(), finalRetTy, flattenValues(adaptor.getOperands()));
     // (2) Gather sparse tensor returns.
     SmallVector<SmallVector<Value>> packedResultVals;
@@ -671,7 +671,7 @@ struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
 
     auto id = AffineMap::getMultiDimIdentityMap(srcStt.getLvlRank(), ctx);
 
-    rewriter.create<SortOp>(loc, nnz, crd, ValueRange{val}, id,
+    SortOp::create(rewriter, loc, nnz, crd, ValueRange{val}, id,
                             rewriter.getIndexAttr(0), op.getAlgorithm());
 
     // Since we do in-place sorting, the destinate tensor will have the same set
@@ -757,10 +757,10 @@ class SparseTensorAllocConverter
       // Memcpy on memref fields.
       for (auto field : desc.getMemRefFields()) {
         auto memrefTp = cast<MemRefType>(field.getType());
-        auto size = rewriter.create<memref::DimOp>(loc, field, 0);
+        auto size = memref::DimOp::create(rewriter, loc, field, 0);
         auto copied =
-            rewriter.create<memref::AllocOp>(loc, memrefTp, ValueRange{size});
-        rewriter.create<memref::CopyOp>(loc, field, copied);
+            memref::AllocOp::create(rewriter, loc, memrefTp, ValueRange{size});
+        memref::CopyOp::create(rewriter, loc, field, copied);
         fields.push_back(copied);
       }
       // Reuses specifier.
@@ -863,7 +863,7 @@ class SparseTensorDeallocConverter
           cast<RankedTensorType>(op.getTensor().getType()));
       for (auto input : desc.getMemRefFields())
         // Deallocate every buffer used to store the sparse tensor handler.
-        rewriter.create<memref::DeallocOp>(loc, input);
+        memref::DeallocOp::create(rewriter, loc, input);
     }
     rewriter.eraseOp(op);
     return success();
@@ -917,7 +917,7 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // Generate a memref for `sz` elements of type `t`.
     const auto genAlloc = [&](Type t) {
       const auto memTp = MemRefType::get({ShapedType::kDynamic}, t);
-      return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
+      return memref::AllocOp::create(rewriter, loc, memTp, ValueRange{sz});
     };
     // Allocate temporary buffers for values/filled-switch and added.
     // We do not use stack buffers for this, since the expanded size may
@@ -931,10 +931,10 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, eltType)},
         ValueRange{values});
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, boolType)},
         ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
@@ -965,7 +965,7 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     // If the innermost level is ordered, we need to sort the coordinates
     // in the "added" array prior to applying the compression.
     if (dstType.isOrderedLvl(dstType.getLvlRank() - 1))
-      rewriter.create<SortOp>(
+      SortOp::create(rewriter,
           loc, count, added, ValueRange{}, rewriter.getMultiDimIdentityMap(1),
           rewriter.getIndexAttr(0), SparseTensorSortKind::HybridQuickSort);
     // While performing the insertions, we also need to reset the elements
@@ -1000,15 +1000,15 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     SmallVector<Value> insertRet = insertGen.genCallOrInline(rewriter, loc);
     genStore(rewriter, loc, constantZero(rewriter, loc, eltType), values, crd);
     genStore(rewriter, loc, constantI1(rewriter, loc, false), filled, crd);
-    rewriter.create<scf::YieldOp>(loc, insertRet);
+    scf::YieldOp::create(rewriter, loc, insertRet);
 
     rewriter.setInsertionPointAfter(loop);
     // Deallocate the buffers on exit of the full loop nest.
     Operation *parent = getTop(op);
     rewriter.setInsertionPointAfter(parent);
-    rewriter.create<memref::DeallocOp>(loc, values);
-    rewriter.create<memref::DeallocOp>(loc, filled);
-    rewriter.create<memref::DeallocOp>(loc, added);
+    memref::DeallocOp::create(rewriter, loc, values);
+    memref::DeallocOp::create(rewriter, loc, filled);
+    memref::DeallocOp::create(rewriter, loc, added);
     // Replace operation with resulting memrefs.
     rewriter.replaceOpWithMultiple(op, {loop->getResults()});
     return success();
@@ -1192,7 +1192,7 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
             // would require a subViewOp to avoid overflow when copying
             // values.
             Value sz = linalg::createOrFoldDimOp(rewriter, loc, srcMem, 0);
-            auto dstMem = rewriter.create<memref::AllocOp>(
+            auto dstMem = memref::AllocOp::create(rewriter,
                 loc, cast<MemRefType>(fTp), sz);
             if (fTp != srcMem.getType()) {
               // Converts elements type.
@@ -1201,16 +1201,16 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
                   constantIndex(rewriter, loc, 1),
                   [srcMem, &dstMem](OpBuilder &builder, Location loc,
                                     ValueRange ivs) {
-                    Value v = builder.create<memref::LoadOp>(loc, srcMem, ivs);
+                    Value v = memref::LoadOp::create(builder, loc, srcMem, ivs);
                     Value casted = genCast(builder, loc, v,
                                            dstMem.getType().getElementType());
-                    builder.create<memref::StoreOp>(loc, casted, dstMem, ivs);
+                    memref::StoreOp::create(builder, loc, casted, dstMem, ivs);
                   });
             } else {
               // TODO: We can even reuse the same memref for the new tensor,
               // but that requires a `ref-counting` based memory management
               // for shared memrefs between multiple sparse tensors.
-              rewriter.create<memref::CopyOp>(loc, srcMem, dstMem);
+              memref::CopyOp::create(rewriter, loc, srcMem, dstMem);
             }
             fields.push_back(dstMem);
           }
@@ -1242,7 +1242,7 @@ class SparseExtractSliceConverter
     auto desc = getMutDescriptorFromTensorTuple(adaptor.getSource(), fields,
                                                 op.getSource().getType());
 
-    auto newSpec = rewriter.create<StorageSpecifierInitOp>(
+    auto newSpec = StorageSpecifierInitOp::create(rewriter,
         loc, StorageSpecifierType::get(ctx, dstEnc), desc.getSpecifier());
     desc.setSpecifier(newSpec);
 
@@ -1326,11 +1326,11 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
               // Flattens the buffer to batchLvlRank.
               auto reassoc = getReassociationForFlattening(
                   mem.getType(), stt.getBatchLvlRank());
-              mem = rewriter.create<memref::CastOp>(
+              mem = memref::CastOp::create(rewriter,
                   loc, fType,
-                  rewriter.create<memref::CollapseShapeOp>(loc, mem, reassoc));
+                  memref::CollapseShapeOp::create(rewriter, loc, mem, reassoc));
             } else {
-              mem = rewriter.create<memref::CastOp>(loc, fType, mem);
+              mem = memref::CastOp::create(rewriter, loc, fType, mem);
             }
             fields.push_back(mem);
           }
@@ -1362,8 +1362,8 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       LevelType lt = stt.getLvlType(lvl);
       // Simply forwards the position index when this is a dense level.
       if (lt.isa<LevelFormat::Dense>()) {
-        memSize = rewriter.create<arith::MulIOp>(loc, lvlSize, memSize);
-        posBack = rewriter.create<arith::SubIOp>(loc, memSize, c1);
+        memSize = arith::MulIOp::create(rewriter, loc, lvlSize, memSize);
+        posBack = arith::SubIOp::create(rewriter, loc, memSize, c1);
         continue;
       }
       if (lt.isa<LevelFormat::Batch>()) {
@@ -1376,12 +1376,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       if (isWithPosLT(lt)) {
         assert(isCompressedLT(lt) || isLooseCompressedLT(lt));
         if (isLooseCompressedLT(lt)) {
-          memSize = rewriter.create<arith::MulIOp>(loc, memSize, c2);
-          posBack = rewriter.create<arith::SubIOp>(loc, memSize, c1);
+          memSize = arith::MulIOp::create(rewriter, loc, memSize, c2);
+          posBack = arith::SubIOp::create(rewriter, loc, memSize, c1);
         } else {
           assert(isCompressedLT(lt));
           posBack = memSize;
-          memSize = rewriter.create<arith::AddIOp>(loc, memSize, c1);
+          memSize = arith::AddIOp::create(rewriter, loc, memSize, c1);
         }
         desc.setPosMemSize(rewriter, loc, lvl, memSize);
         // The last value in position array is the memory size for next level.
@@ -1391,12 +1391,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
                                    constantIndex(rewriter, loc, 0));
         batched.push_back(posBack);
         memSize = genIndexLoad(rewriter, loc, desc.getPosMemRef(lvl), batched);
-        posBack = rewriter.create<arith::SubIOp>(loc, posBack, c1);
+        posBack = arith::SubIOp::create(rewriter, loc, posBack, c1);
       }
       assert(isWithCrdLT(lt) && lvl <= trailCOOStart);
       // FIXME: This seems to be unnecessarily complex, can we simplify it?
       if (lvl == trailCOOStart) {
-        Value cooSz = rewriter.create<arith::MulIOp>(
+        Value cooSz = arith::MulIOp::create(rewriter,
             loc, memSize, constantIndex(rewriter, loc, trailCOORank));
         desc.setCrdMemSize(rewriter, loc, lvl, cooSz);
       } else {
@@ -1460,18 +1460,18 @@ struct SparseDisassembleOpConverter
       if (dst.getType().getRank() > stt.getBatchLvlRank() + 1) {
         auto reassoc =
             getReassociationForFlattening(dst.getType(), stt.getBatchLvlRank());
-        flatOut = rewriter.create<memref::CollapseShapeOp>(loc, dst, reassoc);
+        flatOut = memref::CollapseShapeOp::create(rewriter, loc, dst, reassoc);
       }
       Value dstMem = genSliceToSize(rewriter, loc, flatOut, sz);
       Value srcMem = genSliceToSize(rewriter, loc, src, sz);
-      rewriter.create<memref::CopyOp>(loc, srcMem, dstMem);
+      memref::CopyOp::create(rewriter, loc, srcMem, dstMem);
       return true;
     });
 
     // Converts MemRefs back to Tensors.
     SmallVector<Value> retValues = llvm::to_vector(
         llvm::map_range(retMem, [&rewriter, loc](Value v) -> Value {
-          return rewriter.create<bufferization::ToTensorOp>(
+          return bufferization::ToTensorOp::create(rewriter,
               loc, memref::getTensorTypeFromMemRefType(v.getType()), v);
         }));
     // Appends the actual memory length used in each buffer returned.
@@ -1549,13 +1549,13 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Level lvlRank = dstTp.getLvlRank();
     if (dstTp.isOrderedLvl(lvlRank - 1)) {
       Value kFalse = constantI1(rewriter, loc, false);
-      Value notSorted = rewriter.create<arith::CmpIOp>(
+      Value notSorted = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, isSorted, kFalse);
       scf::IfOp ifOp =
-          rewriter.create<scf::IfOp>(loc, notSorted, /*else*/ false);
+          scf::IfOp::create(rewriter, loc, notSorted, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       auto xPerm = rewriter.getMultiDimIdentityMap(lvlRank);
-      rewriter.create<SortOp>(loc, nse, xs, ValueRange{ys}, xPerm,
+      SortOp::create(rewriter, loc, nse, xs, ValueRange{ys}, xPerm,
                               rewriter.getIndexAttr(0),
                               SparseTensorSortKind::HybridQuickSort);
       rewriter.setInsertionPointAfter(ifOp);
@@ -1566,10 +1566,10 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Value posMemref0 = desc.getPosMemRef(0);
     const Type posTp = dstTp.getPosType();
     const Value posNse = genCast(rewriter, loc, nse, posTp);
-    rewriter.create<memref::StoreOp>(loc, posNse, posMemref0, c1);
+    memref::StoreOp::create(rewriter, loc, posNse, posMemref0, c1);
 
     // Update storage specifier.
-    Value coordinatesSize = rewriter.create<arith::MulIOp>(
+    Value coordinatesSize = arith::MulIOp::create(rewriter,
         loc, nse, constantIndex(rewriter, loc, lvlRank));
     desc.setSpecifierField(rewriter, loc, StorageSpecifierKind::CrdMemSize, 0,
                            coordinatesSize);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 50ccb43d432b6..54ea1843b457f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -137,7 +137,7 @@ static SmallVector<Value> getDimSizes(OpBuilder &builder, Location loc,
 /// this buffer must be explicitly deallocated by client.
 static Value genAlloc(RewriterBase &rewriter, Location loc, Value sz, Type tp) {
   auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
-  return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
+  return memref::AllocOp::create(rewriter, loc, memTp, ValueRange{sz});
 }
 
 /// Generates a temporary buffer for the level-types of the given encoding.
@@ -154,7 +154,7 @@ static Value genLvlTypesBuffer(OpBuilder &builder, Location loc,
 static Value extractBarePtrFromTensor(OpBuilder &builder, Location loc,
                                       Value tensor) {
   auto buf = genToMemref(builder, loc, tensor);
-  return builder.create<memref::ExtractAlignedPointerAsIndexOp>(loc, buf);
+  return memref::ExtractAlignedPointerAsIndexOp::create(builder, loc, buf);
 }
 
 /// Generates a temporary buffer for the level-types of the given encoding.
@@ -168,11 +168,11 @@ static Value genLvlPtrsBuffers(OpBuilder &builder, Location loc,
 
   // Passing in value buffer pointers.
   lvlBarePtrs.push_back(extractBarePtrFromTensor(builder, loc, valTensor));
-  Value idxPtr = builder.create<memref::ExtractAlignedPointerAsIndexOp>(
+  Value idxPtr = memref::ExtractAlignedPointerAsIndexOp::create(builder,
       loc, allocaBuffer(builder, loc, lvlBarePtrs));
   Value idxCast =
-      builder.create<arith::IndexCastOp>(loc, builder.getI64Type(), idxPtr);
-  return builder.create<LLVM::IntToPtrOp>(loc, getOpaquePointerType(builder),
+      arith::IndexCastOp::create(builder, loc, builder.getI64Type(), idxPtr);
+  return LLVM::IntToPtrOp::create(builder, loc, getOpaquePointerType(builder),
                                           idxCast);
 }
 
@@ -227,7 +227,7 @@ class NewCallParams final {
     assert(isInitialized() && "Must initialize before genNewCall");
     StringRef name = "newSparseTensor";
     params[kParamAction] = constantAction(builder, loc, action);
-    params[kParamPtr] = ptr ? ptr : builder.create<LLVM::ZeroOp>(loc, pTp);
+    params[kParamPtr] = ptr ? ptr : LLVM::ZeroOp::create(builder, loc, pTp);
     return createFuncCall(builder, loc, name, pTp, params, EmitCInterface::On)
         .getResult(0);
   }
@@ -539,7 +539,7 @@ class SparseTensorToCoordinatesConverter
     // Cast the MemRef type to the type expected by the users, though these
     // two types should be compatible at runtime.
     if (op.getType() != crds.getType())
-      crds = rewriter.create<memref::CastOp>(loc, op.getType(), crds);
+      crds = memref::CastOp::create(rewriter, loc, op.getType(), crds);
     rewriter.replaceOp(op, crds);
     return success();
   }
@@ -560,7 +560,7 @@ class SparseToCoordinatesBufferConverter
     // Cast the MemRef type to the type expected by the users, though these
     // two types should be compatible at runtime.
     if (op.getType() != crds.getType())
-      crds = rewriter.create<memref::CastOp>(loc, op.getType(), crds);
+      crds = memref::CastOp::create(rewriter, loc, op.getType(), crds);
     rewriter.replaceOp(op, crds);
     return success();
   }
@@ -652,7 +652,7 @@ class SparseTensorInsertConverter
       vref = genAllocaScalar(rewriter, loc, elemTp);
     }
     storeAll(rewriter, loc, lvlCoords, adaptor.getIndices());
-    rewriter.create<memref::StoreOp>(loc, adaptor.getScalar(), vref);
+    memref::StoreOp::create(rewriter, loc, adaptor.getScalar(), vref);
     SmallString<12> name{"lexInsert", primaryTypeFunctionSuffix(elemTp)};
     createFuncCall(rewriter, loc, name, {},
                    {adaptor.getDest(), lvlCoords, vref}, EmitCInterface::On);
@@ -690,10 +690,10 @@ class SparseTensorExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, eltType)},
         ValueRange{values});
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, boolType)},
         ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
@@ -733,9 +733,9 @@ class SparseTensorCompressConverter : public OpConversionPattern<CompressOp> {
     rewriter.replaceOp(op, adaptor.getTensor());
     // Deallocate the buffers on exit of the loop nest.
     rewriter.setInsertionPointAfter(parent);
-    rewriter.create<memref::DeallocOp>(loc, values);
-    rewriter.create<memref::DeallocOp>(loc, filled);
-    rewriter.create<memref::DeallocOp>(loc, added);
+    memref::DeallocOp::create(rewriter, loc, values);
+    memref::DeallocOp::create(rewriter, loc, filled);
+    memref::DeallocOp::create(rewriter, loc, added);
     return success();
   }
 };
@@ -837,21 +837,21 @@ class SparseTensorDisassembleConverter
                                       cooStartLvl + 1);
       auto crdLen = linalg::createOrFoldDimOp(rewriter, loc, crds0, 0);
       auto two = constantIndex(rewriter, loc, 2);
-      auto bufLen = rewriter.create<arith::MulIOp>(loc, crdLen, two);
+      auto bufLen = arith::MulIOp::create(rewriter, loc, crdLen, two);
       Type indexType = rewriter.getIndexType();
       auto zero = constantZero(rewriter, loc, indexType);
       auto one = constantOne(rewriter, loc, indexType);
-      scf::ForOp forOp = rewriter.create<scf::ForOp>(loc, zero, crdLen, one);
+      scf::ForOp forOp = scf::ForOp::create(rewriter, loc, zero, crdLen, one);
       auto idx = forOp.getInductionVar();
       rewriter.setInsertionPointToStart(forOp.getBody());
-      auto c0 = rewriter.create<memref::LoadOp>(loc, crds0, idx);
-      auto c1 = rewriter.create<memref::LoadOp>(loc, crds1, idx);
+      auto c0 = memref::LoadOp::create(rewriter, loc, crds0, idx);
+      auto c1 = memref::LoadOp::create(rewriter, loc, crds1, idx);
       SmallVector<Value> args;
       args.push_back(idx);
       args.push_back(zero);
-      rewriter.create<memref::StoreOp>(loc, c0, buf, args);
+      memref::StoreOp::create(rewriter, loc, c0, buf, args);
       args[1] = one;
-      rewriter.create<memref::StoreOp>(loc, c1, buf, args);
+      memref::StoreOp::create(rewriter, loc, c1, buf, args);
       rewriter.setInsertionPointAfter(forOp);
       auto bufLenTp = op.getLvlLens().getTypes()[retLen.size()];
       retVal.push_back(buf);
@@ -867,11 +867,11 @@ class SparseTensorDisassembleConverter
     // Converts MemRefs back to Tensors.
     assert(retVal.size() + retLen.size() == op.getNumResults());
     for (unsigned i = 0, sz = retVal.size(); i < sz; i++) {
-      auto tensor = rewriter.create<bufferization::ToTensorOp>(
+      auto tensor = bufferization::ToTensorOp::create(rewriter,
           loc, memref::getTensorTypeFromMemRefType(retVal[i].getType()),
           retVal[i]);
       retVal[i] =
-          rewriter.create<tensor::CastOp>(loc, op.getResultTypes()[i], tensor);
+          tensor::CastOp::create(rewriter, loc, op.getResultTypes()[i], tensor);
     }
 
     // Appends the actual memory length used in each buffer returned.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index d4a02bf7a70b6..ee963ab5d1ffb 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -127,7 +127,7 @@ static void sizesForTensor(OpBuilder &builder, SmallVectorImpl<Value> &sizes,
   for (const auto &d : enumerate(stp.getShape())) {
     Value dim;
     if (d.value() == ShapedType::kDynamic)
-      dim = builder.create<tensor::DimOp>(loc, tensor, d.index());
+      dim = tensor::DimOp::create(builder, loc, tensor, d.index());
     else
       dim = constantIndex(builder, loc, d.value());
     sizes.push_back(dim);
@@ -198,7 +198,7 @@ static void concatSizesFromInputs(OpBuilder &builder,
     for (const auto &src : srcs.drop_front()) {
       Value srcSz = linalg::createOrFoldDimOp(builder, loc, src, dim);
       // Sum up all the sizes.
-      sizes[dim] = builder.create<arith::AddIOp>(loc, sizes[dim], srcSz);
+      sizes[dim] = arith::AddIOp::create(builder, loc, sizes[dim], srcSz);
     }
   }
 }
@@ -405,7 +405,7 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     inputOps.push_back(op.getDpsInputOperand(1 - other)->get());
     fusedIndexMaps.push_back(fusedIndexMaps.back()); // mimic other
     // Fuse producer and consumer into a new generic op.
-    auto fusedOp = rewriter.create<GenericOp>(
+    auto fusedOp = GenericOp::create(rewriter,
         loc, op.getResult(0).getType(), inputOps, outputOps,
         rewriter.getAffineMapArrayAttr(fusedIndexMaps), prod.getIteratorTypes(),
         /*doc=*/nullptr, /*library_call=*/nullptr);
@@ -430,7 +430,7 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     mapper.map(consBlock.getArgument(other), fusedBlock->back().getResult(0));
     mapper.map(last, rewriter.clone(*sampler, mapper)->getResult(0));
     last = rewriter.clone(*acc, mapper)->getResult(0);
-    rewriter.create<linalg::YieldOp>(loc, last);
+    linalg::YieldOp::create(rewriter, loc, last);
     // Force initial value on merged allocation for dense outputs.
     // TODO: deal with non alloc tensor here one day
     if (!getSparseTensorEncoding(op.getResult(0).getType())) {
@@ -534,7 +534,7 @@ struct GenSemiRingSelect : public OpRewritePattern<GenericOp> {
       assert(t.getType() == f.getType());
       auto selTp = t.getType();
       auto c0 = constantZero(rewriter, loc, selTp);
-      auto binOp = rewriter.create<sparse_tensor::BinaryOp>(loc, selTp, t, f);
+      auto binOp = sparse_tensor::BinaryOp::create(rewriter, loc, selTp, t, f);
       // Initializes all the blocks.
       rewriter.createBlock(&binOp.getOverlapRegion(), {}, {selTp, selTp},
                            {t.getLoc(), f.getLoc()});
@@ -564,7 +564,7 @@ struct GenSemiRingSelect : public OpRewritePattern<GenericOp> {
           irMap.map(f, b->getArgument(1));
         }
         auto y = rewriter.clone(inst, irMap)->getResult(0);
-        rewriter.create<sparse_tensor::YieldOp>(loc, y);
+        sparse_tensor::YieldOp::create(rewriter, loc, y);
       }
 
       // We successfully rewrited a operation. We can not do replacement here
@@ -674,28 +674,28 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     // Identity.
     Location loc = op.getLoc();
     Value identity =
-        rewriter.create<tensor::ExtractOp>(loc, init->get(), ValueRange());
+        tensor::ExtractOp::create(rewriter, loc, init->get(), ValueRange());
     // Unary {
     //    present -> value
     //    absent  -> zero.
     // }
     Type rtp = s0.getType();
     rewriter.setInsertionPointToStart(&op.getRegion().front());
-    auto semiring = rewriter.create<sparse_tensor::UnaryOp>(loc, rtp, s0);
+    auto semiring = sparse_tensor::UnaryOp::create(rewriter, loc, rtp, s0);
     Block *present =
         rewriter.createBlock(&semiring.getPresentRegion(), {}, rtp, loc);
     rewriter.setInsertionPointToStart(&semiring.getPresentRegion().front());
-    rewriter.create<sparse_tensor::YieldOp>(loc, present->getArgument(0));
+    sparse_tensor::YieldOp::create(rewriter, loc, present->getArgument(0));
     rewriter.createBlock(&semiring.getAbsentRegion(), {}, {}, {});
     rewriter.setInsertionPointToStart(&semiring.getAbsentRegion().front());
     auto zero =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(rtp));
-    rewriter.create<sparse_tensor::YieldOp>(loc, zero);
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(rtp));
+    sparse_tensor::YieldOp::create(rewriter, loc, zero);
     rewriter.setInsertionPointAfter(semiring);
     // CustomReduce {
     //    x = x REDUC y, identity
     // }
-    auto custom = rewriter.create<sparse_tensor::ReduceOp>(
+    auto custom = sparse_tensor::ReduceOp::create(rewriter,
         loc, rtp, semiring.getResult(), s1, identity);
     Block *region =
         rewriter.createBlock(&custom.getRegion(), {}, {rtp, rtp}, {loc, loc});
@@ -704,7 +704,7 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     irMap.map(red->getOperand(0), region->getArgument(0));
     irMap.map(red->getOperand(1), region->getArgument(1));
     auto *cloned = rewriter.clone(*red, irMap);
-    rewriter.create<sparse_tensor::YieldOp>(loc, cloned->getResult(0));
+    sparse_tensor::YieldOp::create(rewriter, loc, cloned->getResult(0));
     rewriter.setInsertionPointAfter(custom);
     rewriter.replaceOp(red, custom.getResult());
     return success();
@@ -723,14 +723,14 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto tensor = op.getTensor();
     auto stt = getSparseTensorType(tensor);
     // Header with NSE.
-    auto nse = rewriter.create<NumberOfEntriesOp>(loc, tensor);
-    rewriter.create<vector::PrintOp>(
+    auto nse = NumberOfEntriesOp::create(rewriter, loc, tensor);
+    vector::PrintOp::create(rewriter,
         loc, rewriter.getStringAttr("---- Sparse Tensor ----\nnse = "));
-    rewriter.create<vector::PrintOp>(loc, nse);
+    vector::PrintOp::create(rewriter, loc, nse);
     // Print run-time contents for dim/lvl sizes.
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("dim = "));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("dim = "));
     printSizes(rewriter, loc, tensor, stt.getDimRank(), /*isDim=*/true);
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("lvl = "));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("lvl = "));
     printSizes(rewriter, loc, tensor, stt.getLvlRank(), /*isDim=*/false);
     // Use the "codegen" foreach loop construct to iterate over
     // all typical sparse tensor components for printing.
@@ -744,42 +744,42 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       }
       case SparseTensorFieldKind::PosMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("pos["));
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("pos["));
+        vector::PrintOp::create(rewriter,
             loc, lvl, vector::PrintPunctuation::NoPunctuation);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("] : "));
-        auto pos = rewriter.create<ToPositionsOp>(loc, tensor, l);
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
+        auto pos = ToPositionsOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, pos);
         break;
       }
       case SparseTensorFieldKind::CrdMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("crd["));
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("crd["));
+        vector::PrintOp::create(rewriter,
             loc, lvl, vector::PrintPunctuation::NoPunctuation);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("] : "));
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
         Value crd = nullptr;
         // For COO AoS storage, we want to print a single, linear view of
         // the full coordinate storage at this level. For any other storage,
         // we show the coordinate storage for every indivual level.
         if (stt.getAoSCOOStart() == l)
-          crd = rewriter.create<ToCoordinatesBufferOp>(loc, tensor);
+          crd = ToCoordinatesBufferOp::create(rewriter, loc, tensor);
         else
-          crd = rewriter.create<ToCoordinatesOp>(loc, tensor, l);
+          crd = ToCoordinatesOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, crd);
         break;
       }
       case SparseTensorFieldKind::ValMemRef: {
-        rewriter.create<vector::PrintOp>(loc,
+        vector::PrintOp::create(rewriter, loc,
                                          rewriter.getStringAttr("values : "));
-        auto val = rewriter.create<ToValuesOp>(loc, tensor);
+        auto val = ToValuesOp::create(rewriter, loc, tensor);
         printContents(rewriter, loc, val);
         break;
       }
       }
       return true;
     });
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("----\n"));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("----\n"));
     rewriter.eraseOp(op);
     return success();
   }
@@ -797,7 +797,7 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto shape = cast<ShapedType>(vec.getType()).getShape();
     SmallVector<Value> idxs;
     printContentsLevel(rewriter, loc, vec, 0, shape, idxs);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::NewLine);
   }
 
   // Helper to the helper.
@@ -805,13 +805,13 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
                                  Value vec, unsigned i, ArrayRef<int64_t> shape,
                                  SmallVectorImpl<Value> &idxs) {
     // Open bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
     // Generate for loop.
     auto zero = constantIndex(rewriter, loc, 0);
     auto index = constantIndex(rewriter, loc, i);
-    auto size = rewriter.create<memref::DimOp>(loc, vec, index);
+    auto size = memref::DimOp::create(rewriter, loc, vec, index);
     auto step = constantIndex(rewriter, loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, zero, size, step);
+    auto forOp = scf::ForOp::create(rewriter, loc, zero, size, step);
     idxs.push_back(forOp.getInductionVar());
     rewriter.setInsertionPointToStart(forOp.getBody());
     if (i < shape.size() - 1) {
@@ -819,56 +819,56 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       printContentsLevel(rewriter, loc, vec, i + 1, shape, idxs);
     } else {
       // Actual contents printing.
-      auto val = rewriter.create<memref::LoadOp>(loc, vec, idxs);
+      auto val = memref::LoadOp::create(rewriter, loc, vec, idxs);
       if (llvm::isa<ComplexType>(val.getType())) {
         // Since the vector dialect does not support complex types in any op,
         // we split those into (real, imag) pairs here.
-        Value real = rewriter.create<complex::ReOp>(loc, val);
-        Value imag = rewriter.create<complex::ImOp>(loc, val);
-        rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
-        rewriter.create<vector::PrintOp>(loc, real,
+        Value real = complex::ReOp::create(rewriter, loc, val);
+        Value imag = complex::ImOp::create(rewriter, loc, val);
+        vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
+        vector::PrintOp::create(rewriter, loc, real,
                                          vector::PrintPunctuation::Comma);
-        rewriter.create<vector::PrintOp>(loc, imag,
+        vector::PrintOp::create(rewriter, loc, imag,
                                          vector::PrintPunctuation::Close);
       } else {
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter,
             loc, val, vector::PrintPunctuation::NoPunctuation);
       }
       // Terminating comma (except at end).
-      auto bound = rewriter.create<arith::AddIOp>(loc, idxs.back(), step);
-      Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+      auto bound = arith::AddIOp::create(rewriter, loc, idxs.back(), step);
+      Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
                                                   bound, size);
-      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, cond, /*else*/ false);
+      scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, cond, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Comma);
+      vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Comma);
     }
     idxs.pop_back();
     rewriter.setInsertionPointAfter(forOp);
     // Close bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
   }
 
   // Helper method to print run-time lvl/dim sizes.
   static void printSizes(PatternRewriter &rewriter, Location loc, Value tensor,
                          unsigned size, bool isDim) {
     // Open bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
     // Print unrolled contents (dimop requires constant value).
     for (unsigned i = 0; i < size; i++) {
       auto idx = constantIndex(rewriter, loc, i);
       Value val;
       if (isDim)
-        val = rewriter.create<tensor::DimOp>(loc, tensor, idx);
+        val = tensor::DimOp::create(rewriter, loc, tensor, idx);
       else
-        val = rewriter.create<LvlOp>(loc, tensor, idx);
-      rewriter.create<vector::PrintOp>(
+        val = LvlOp::create(rewriter, loc, tensor, idx);
+      vector::PrintOp::create(rewriter,
           loc, val,
           i != size - 1 ? vector::PrintPunctuation::Comma
                         : vector::PrintPunctuation::NoPunctuation);
     }
     // Close bracket and end of line.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::NewLine);
   }
 };
 
@@ -896,7 +896,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     for (Dimension d : dstTp->getDimShape())
       dstSizes.push_back(constantIndex(rewriter, loc, d));
 
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, srcTensor);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, srcTensor);
     // Only need an unordered COO buffer if input and output are not sorted
     // in the same way.
     Type bufferTp = getBufferType(
@@ -920,7 +920,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp->getEncoding();
-    ForeachOp foreachOp = rewriter.create<ForeachOp>(
+    ForeachOp foreachOp = ForeachOp::create(rewriter,
         loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
@@ -935,7 +935,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
           Value collapseSize = constantIndex(builder, loc, 1);
           for (Dimension d = 0; d < srcRank; d++)
             collapseSize =
-                builder.create<arith::MulIOp>(loc, collapseSize, srcSizes[d]);
+                arith::MulIOp::create(builder, loc, collapseSize, srcSizes[d]);
           SmallVector<Value, 1> collapsedSizes = {collapseSize};
 
           ReassociationIndices collapseIdx;
@@ -955,15 +955,15 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
                      dstSizes, dstDcvs);
 
           auto t =
-              builder.create<tensor::InsertOp>(loc, v, reduc.front(), dstDcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+              tensor::InsertOp::create(builder, loc, v, reduc.front(), dstDcvs);
+          sparse_tensor::YieldOp::create(builder, loc, t);
         });
 
-    Value t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
+    Value t = LoadOp::create(rewriter, loc, foreachOp.getResult(0), true);
     if (bufferTp != *dstTp) {
       auto dstRTT = dstTp->getRankedTensorType();
-      Value converted = rewriter.create<ConvertOp>(loc, dstRTT, t).getResult();
-      rewriter.create<DeallocTensorOp>(loc, t);
+      Value converted = ConvertOp::create(rewriter, loc, dstRTT, t).getResult();
+      DeallocTensorOp::create(rewriter, loc, t);
       t = converted;
     }
     rewriter.replaceOp(op, t);
@@ -1004,7 +1004,7 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           dstDynSizes.push_back(dstSizes[idx]);
       }
     }
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, srcTensor);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, srcTensor);
     // Only need a unordered COO buffer if input and output are not sorted
     // in the same way.
     Type bufferTp = getBufferType(
@@ -1025,7 +1025,7 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp.getEncoding();
-    ForeachOp foreachOp = rewriter.create<ForeachOp>(
+    ForeachOp foreachOp = ForeachOp::create(rewriter,
         loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
@@ -1040,15 +1040,15 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           reshapeCvs(builder, loc, op.getReassociationIndices(), srcSizes,
                      srcDcvs, dstSizes, dstDcvs);
           auto t =
-              builder.create<tensor::InsertOp>(loc, v, reduc.front(), dstDcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+              tensor::InsertOp::create(builder, loc, v, reduc.front(), dstDcvs);
+          sparse_tensor::YieldOp::create(builder, loc, t);
         });
 
-    Value t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
+    Value t = LoadOp::create(rewriter, loc, foreachOp.getResult(0), true);
     if (bufferTp != dstTp) {
       auto dstRTT = dstTp.getRankedTensorType();
-      Value converted = rewriter.create<ConvertOp>(loc, dstRTT, t).getResult();
-      rewriter.create<DeallocTensorOp>(loc, t);
+      Value converted = ConvertOp::create(rewriter, loc, dstRTT, t).getResult();
+      DeallocTensorOp::create(rewriter, loc, t);
       t = converted;
     }
     rewriter.replaceOp(op, t);
@@ -1079,7 +1079,7 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
       auto rtp = getRankedTensorType(op.getSrc());
       auto denseTp =
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
-      auto convert = rewriter.create<ConvertOp>(loc, denseTp, op.getSrc());
+      auto convert = ConvertOp::create(rewriter, loc, denseTp, op.getSrc());
       rewriter.modifyOpInPlace(op, [&]() { op->setOperand(0, convert); });
       return success();
     }
@@ -1089,14 +1089,14 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
       ReshapeOp reshape;
       if constexpr (std::is_same<ReshapeOp, tensor::ExpandShapeOp>::value) {
-        reshape = rewriter.create<ReshapeOp>(
+        reshape = ReshapeOp::create(rewriter,
             loc, denseTp, op.getSrc(), op.getReassociation(),
             op.getOutputShape(), op.getStaticOutputShape());
       } else {
-        reshape = rewriter.create<ReshapeOp>(loc, denseTp, op.getSrc(),
+        reshape = ReshapeOp::create(rewriter, loc, denseTp, op.getSrc(),
                                              op.getReassociation());
       }
-      Value convert = rewriter.create<ConvertOp>(loc, rtp, reshape);
+      Value convert = ConvertOp::create(rewriter, loc, rtp, reshape);
       rewriter.replaceOp(op, convert);
       return success();
     }
@@ -1112,20 +1112,20 @@ struct TensorLike {
     SmallVector<Value> dynSzs;
     getDynamicSizes(rtt, sizes, dynSzs);
 
-    val = builder.create<AllocTensorOp>(loc, rtt, dynSzs);
+    val = AllocTensorOp::create(builder, loc, rtt, dynSzs);
     if (!isSparse()) {
       Value c0 = constantZero(builder, loc, rtt.getElementType());
-      val = builder.create<linalg::FillOp>(loc, c0, val).getResult(0);
+      val = linalg::FillOp::create(builder, loc, c0, val).getResult(0);
     }
   }
 
   void insert(OpBuilder &builder, Location loc, Value v, ValueRange crds) {
-    val = builder.create<tensor::InsertOp>(loc, v, val, crds);
+    val = tensor::InsertOp::create(builder, loc, v, val, crds);
   }
 
   Value finalize(OpBuilder &builder, Location loc, RankedTensorType rtp) const {
     if (isSparse())
-      return builder.create<LoadOp>(loc, val, true);
+      return LoadOp::create(builder, loc, val, true);
     return val;
   }
 
@@ -1160,18 +1160,18 @@ struct SparseTensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
     Location loc = op.getLoc();
     SmallVector<Value> maxLvlCrds;
     for (Level l = 0; l < stt->getLvlRank(); l++) {
-      Value lvlSz = rewriter.create<LvlOp>(loc, op.getSource(), l);
-      Value maxLvlCrd = rewriter.create<arith::SubIOp>(
+      Value lvlSz = LvlOp::create(rewriter, loc, op.getSource(), l);
+      Value maxLvlCrd = arith::SubIOp::create(rewriter,
           loc, lvlSz, constantOne(rewriter, loc, rewriter.getIndexType()));
       maxLvlCrds.push_back(maxLvlCrd);
     }
 
     AffineExpr lvl2DimExp = stt->getLvlToDim().getResult(*dim);
-    Value maxDimCrd = rewriter.create<affine::AffineApplyOp>(
+    Value maxDimCrd = affine::AffineApplyOp::create(rewriter,
         op.getLoc(), AffineMap::get(stt->getLvlRank(), 0, lvl2DimExp),
         maxLvlCrds);
 
-    Value dimSz = rewriter.create<arith::AddIOp>(
+    Value dimSz = arith::AddIOp::create(rewriter,
         loc, maxDimCrd, constantOne(rewriter, loc, rewriter.getIndexType()));
     rewriter.replaceOp(op, dimSz);
     return success();
@@ -1212,26 +1212,26 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
     for (Value input : op.getInputs()) {
       // Builds a for op for each input tensor to append new values into the
       // output tensor.
-      foreachOp = rewriter.create<ForeachOp>(
+      foreachOp = ForeachOp::create(rewriter,
           loc, input, iterArg,
           [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
               ValueRange reduc) {
             SmallVector<Value> offDimCrd(dcvs);
             offDimCrd[conDim] =
-                builder.create<arith::AddIOp>(loc, offDimCrd[conDim], offset);
+                arith::AddIOp::create(builder, loc, offDimCrd[conDim], offset);
 
             // Enters foreach, updates the SSA chain.
             dstBuf.val = reduc.front();
             if (!dstTp.isAllDense()) {
               Value cond = genIsNonzero(builder, loc, v);
-              auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+              auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
                                                     /*else*/ true);
               builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-              builder.create<scf::YieldOp>(loc, dstBuf.val);
+              scf::YieldOp::create(builder, loc, dstBuf.val);
 
               builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
               dstBuf.insert(builder, loc, v, offDimCrd);
-              builder.create<scf::YieldOp>(loc, dstBuf.val);
+              scf::YieldOp::create(builder, loc, dstBuf.val);
 
               // Exits the ifOp, update the sparse tensor SSA value.
               builder.setInsertionPointAfter(ifOp);
@@ -1239,14 +1239,14 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
             } else {
               dstBuf.insert(builder, loc, v, offDimCrd);
             }
-            builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
+            sparse_tensor::YieldOp::create(builder, loc, dstBuf.val);
           });
       // Accumulates the offset. Note that only static-shaped inputs are allowed
       // by concatenate op verifier, which saves us from computing the offset
       // dynamically.
       const Size sz = getSparseTensorType(input).getDynamicDimSize(conDim);
       assert(ShapedType::isStatic(sz));
-      offset = rewriter.create<arith::AddIOp>(loc, offset,
+      offset = arith::AddIOp::create(rewriter, loc, offset,
                                               constantIndex(rewriter, loc, sz));
       iterArg = foreachOp.getResult(0);
       dstBuf.val = iterArg;
@@ -1299,7 +1299,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
     ValueRange vs;
     TensorLike dstBuf(rewriter, loc, dstStt.getRankedTensorType(), sizes);
 
-    auto foreachOp = rewriter.create<ForeachOp>(
+    auto foreachOp = ForeachOp::create(rewriter,
         loc, src, dstBuf.val, foreachOrder,
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
@@ -1307,14 +1307,14 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           dstBuf.val = reduc.front();
           if (!skipZeroCheck) {
             Value cond = genIsNonzero(builder, loc, v);
-            auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+            auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
                                                   /*else*/ true);
             builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-            builder.create<scf::YieldOp>(loc, dstBuf.val);
+            scf::YieldOp::create(builder, loc, dstBuf.val);
 
             builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
             dstBuf.insert(builder, loc, v, dcvs);
-            builder.create<scf::YieldOp>(loc, dstBuf.val);
+            scf::YieldOp::create(builder, loc, dstBuf.val);
 
             // Exits the ifOp, update the sparse tensor SSA value.
             builder.setInsertionPointAfter(ifOp);
@@ -1322,7 +1322,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           } else {
             dstBuf.insert(builder, loc, v, dcvs);
           }
-          builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
+          sparse_tensor::YieldOp::create(builder, loc, dstBuf.val);
         });
 
     rewriter.setInsertionPointAfter(foreachOp);
@@ -1349,7 +1349,7 @@ struct CrdTranslateRewriter : public OpRewritePattern<CrdTranslateOp> {
       // TODO: we should probably expand the affine map to IR using our own
       // rules, since affine.apply assume signed value, while the cooridinates
       // we provided must always be signless.
-      Value trans = rewriter.create<affine::AffineApplyOp>(
+      Value trans = affine::AffineApplyOp::create(rewriter,
           op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
           op.getInCrds());
       outCrds.push_back(trans);
@@ -1412,8 +1412,8 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
     SmallVector<Value> pos = loopEmitter.getValPosits(0);
     // Loads the value from sparse tensor using position-index;
     // loads the value from dense tensor using coords.
-    Value val = enc ? rewriter.create<memref::LoadOp>(loc, vals, pos)
-                    : rewriter.create<memref::LoadOp>(loc, vals, lcvs);
+    Value val = enc ? memref::LoadOp::create(rewriter, loc, vals, pos)
+                    : memref::LoadOp::create(rewriter, loc, vals, lcvs);
 
     // 2. Inline the block in the foreach operator.
     Block *srcBlock = op.getBody();
@@ -1472,22 +1472,22 @@ struct NewRewriter : public OpRewritePattern<NewOp> {
     // with enveloping reinterpreted_map ops for non-permutations.
     RankedTensorType dstTp = stt.getRankedTensorType();
     RankedTensorType cooTp = stt.getCOOType(/*ordered=*/true);
-    Value cooTensor = rewriter.create<NewOp>(loc, cooTp, op.getSource());
+    Value cooTensor = NewOp::create(rewriter, loc, cooTp, op.getSource());
     Value convert = cooTensor;
     auto enc = stt.getEncoding();
     if (!stt.isPermutation()) { // demap coo, demap dstTp
       auto coo = getSparseTensorType(cooTensor).getEncoding().withoutDimToLvl();
-      convert = rewriter.create<ReinterpretMapOp>(loc, coo, convert);
+      convert = ReinterpretMapOp::create(rewriter, loc, coo, convert);
       dstTp = getSparseTensorType(convert).withEncoding(enc.withoutDimToLvl());
     }
-    convert = rewriter.create<ConvertOp>(loc, dstTp, convert);
+    convert = ConvertOp::create(rewriter, loc, dstTp, convert);
     if (!stt.isPermutation()) // remap to original enc
-      convert = rewriter.create<ReinterpretMapOp>(loc, enc, convert);
+      convert = ReinterpretMapOp::create(rewriter, loc, enc, convert);
     rewriter.replaceOp(op, convert);
 
     // Release the temporary ordered COO tensor.
     rewriter.setInsertionPointAfterValue(convert);
-    rewriter.create<DeallocTensorOp>(loc, cooTensor);
+    DeallocTensorOp::create(rewriter, loc, cooTensor);
 
     return success();
   }
@@ -1501,7 +1501,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     Location loc = op.getLoc();
     // Calculate NNZ.
     Value src = op.getTensor();
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, src);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, src);
 
     // Allocate a temporary buffer for storing dimension-sizes/coordinates.
     const auto srcTp = getSparseTensorType(src);
@@ -1514,7 +1514,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     SmallVector<Value> dims;
     sizesForTensor(rewriter, dims, loc, srcTp, src);
     for (Dimension d = 0; d < dimRank; d++) {
-      rewriter.create<memref::StoreOp>(loc, dims[d], dimSizes,
+      memref::StoreOp::create(rewriter, loc, dims[d], dimSizes,
                                        constantIndex(rewriter, loc, d));
     }
 
@@ -1536,20 +1536,20 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     ModuleOp module = op->getParentOfType<ModuleOp>();
 
     // For each element in the source tensor, output the element.
-    rewriter.create<ForeachOp>(
+    ForeachOp::create(rewriter,
         loc, src, ValueRange(),
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
           for (Dimension d = 0; d < dimRank; d++) {
-            rewriter.create<memref::StoreOp>(loc, dcvs[d], dimCoords,
+            memref::StoreOp::create(rewriter, loc, dcvs[d], dimCoords,
                                              constantIndex(builder, loc, d));
           }
-          rewriter.create<memref::StoreOp>(loc, v, value);
+          memref::StoreOp::create(rewriter, loc, v, value);
           SmallVector<Value> operands{writer, rankValue, dimCoords, value};
           FlatSymbolRefAttr fn = getFunc(module, outNextFuncName, {}, operands,
                                          EmitCInterface::On);
-          builder.create<func::CallOp>(loc, TypeRange(), fn, operands);
-          builder.create<sparse_tensor::YieldOp>(loc);
+          func::CallOp::create(builder, loc, TypeRange(), fn, operands);
+          sparse_tensor::YieldOp::create(builder, loc);
         });
 
     // Release the writer.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 359590f2434dc..5e1f872d5b185 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -79,7 +79,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
       matchPattern(step, m_Constant(&stepInt))) {
     if (((hiInt.getInt() - loInt.getInt()) % stepInt.getInt()) == 0) {
       Value trueVal = constantI1(rewriter, loc, true);
-      return rewriter.create<vector::BroadcastOp>(loc, mtp, trueVal);
+      return vector::BroadcastOp::create(rewriter, loc, mtp, trueVal);
     }
   }
   // Otherwise, generate a vector mask that avoids overrunning the upperbound
@@ -93,7 +93,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
       rewriter.getContext());
   Value end = rewriter.createOrFold<affine::AffineMinOp>(
       loc, min, ValueRange{hi, iv, step});
-  return rewriter.create<vector::CreateMaskOp>(loc, mtp, end);
+  return vector::CreateMaskOp::create(rewriter, loc, mtp, end);
 }
 
 /// Generates a vectorized invariant. Here we rely on subsequent loop
@@ -101,7 +101,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
 static Value genVectorInvariantValue(PatternRewriter &rewriter, VL vl,
                                      Value val) {
   VectorType vtp = vectorType(vl, val.getType());
-  return rewriter.create<vector::BroadcastOp>(val.getLoc(), vtp, val);
+  return vector::BroadcastOp::create(rewriter, val.getLoc(), vtp, val);
 }
 
 /// Generates a vectorized load lhs = a[ind[lo:hi]] or lhs = a[lo:hi],
@@ -116,10 +116,10 @@ static Value genVectorLoad(PatternRewriter &rewriter, Location loc, VL vl,
     SmallVector<Value> scalarArgs(idxs);
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
-    return rewriter.create<vector::GatherOp>(loc, vtp, mem, scalarArgs,
+    return vector::GatherOp::create(rewriter, loc, vtp, mem, scalarArgs,
                                              indexVec, vmask, pass);
   }
-  return rewriter.create<vector::MaskedLoadOp>(loc, vtp, mem, idxs, vmask,
+  return vector::MaskedLoadOp::create(rewriter, loc, vtp, mem, idxs, vmask,
                                                pass);
 }
 
@@ -133,11 +133,11 @@ static void genVectorStore(PatternRewriter &rewriter, Location loc, Value mem,
     SmallVector<Value> scalarArgs(idxs);
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
-    rewriter.create<vector::ScatterOp>(loc, mem, scalarArgs, indexVec, vmask,
+    vector::ScatterOp::create(rewriter, loc, mem, scalarArgs, indexVec, vmask,
                                        rhs);
     return;
   }
-  rewriter.create<vector::MaskedStoreOp>(loc, mem, idxs, vmask, rhs);
+  vector::MaskedStoreOp::create(rewriter, loc, mem, idxs, vmask, rhs);
 }
 
 /// Detects a vectorizable reduction operations and returns the
@@ -198,18 +198,18 @@ static Value genVectorReducInit(PatternRewriter &rewriter, Location loc,
   case vector::CombiningKind::ADD:
   case vector::CombiningKind::XOR:
     // Initialize reduction vector to: | 0 | .. | 0 | r |
-    return rewriter.create<vector::InsertOp>(loc, r,
+    return vector::InsertOp::create(rewriter, loc, r,
                                              constantZero(rewriter, loc, vtp),
                                              constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::MUL:
     // Initialize reduction vector to: | 1 | .. | 1 | r |
-    return rewriter.create<vector::InsertOp>(loc, r,
+    return vector::InsertOp::create(rewriter, loc, r,
                                              constantOne(rewriter, loc, vtp),
                                              constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::AND:
   case vector::CombiningKind::OR:
     // Initialize reduction vector to: | r | .. | r | r |
-    return rewriter.create<vector::BroadcastOp>(loc, vtp, r);
+    return vector::BroadcastOp::create(rewriter, loc, vtp, r);
   default:
     break;
   }
@@ -301,10 +301,10 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
         Type etp = llvm::cast<VectorType>(vload.getType()).getElementType();
         if (!llvm::isa<IndexType>(etp)) {
           if (etp.getIntOrFloatBitWidth() < 32)
-            vload = rewriter.create<arith::ExtUIOp>(
+            vload = arith::ExtUIOp::create(rewriter,
                 loc, vectorType(vl, rewriter.getI32Type()), vload);
           else if (etp.getIntOrFloatBitWidth() < 64 && !vl.enableSIMDIndex32)
-            vload = rewriter.create<arith::ExtUIOp>(
+            vload = arith::ExtUIOp::create(rewriter,
                 loc, vectorType(vl, rewriter.getI64Type()), vload);
         }
         idxs.push_back(vload);
@@ -329,7 +329,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
             return false;
           if (codegen)
             idxs.push_back(
-                rewriter.create<arith::AddIOp>(forOp.getLoc(), inv, idx));
+                arith::AddIOp::create(rewriter, forOp.getLoc(), inv, idx));
           continue; // success so far
         }
       }
@@ -342,7 +342,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define UNAOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = rewriter.create<xxx>(loc, vx);                                    \
+      vexp = xxx::create(rewriter, loc, vx);                                    \
     return true;                                                               \
   }
 
@@ -350,7 +350,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
   if (auto x = dyn_cast<xxx>(def)) {                                           \
     if (codegen) {                                                             \
       VectorType vtp = vectorType(vl, x.getType());                            \
-      vexp = rewriter.create<xxx>(loc, vtp, vx);                               \
+      vexp = xxx::create(rewriter, loc, vtp, vx);                               \
     }                                                                          \
     return true;                                                               \
   }
@@ -358,7 +358,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define BINOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = rewriter.create<xxx>(loc, vx, vy);                                \
+      vexp = xxx::create(rewriter, loc, vx, vy);                                \
     return true;                                                               \
   }
 
@@ -381,9 +381,9 @@ static bool vectorizeExpr(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       // such as a[i] = i, which must convert to [i, i+1, ...].
       if (codegen) {
         VectorType vtp = vectorType(vl, arg.getType());
-        Value veci = rewriter.create<vector::BroadcastOp>(loc, vtp, arg);
-        Value incr = rewriter.create<vector::StepOp>(loc, vtp);
-        vexp = rewriter.create<arith::AddIOp>(loc, veci, incr);
+        Value veci = vector::BroadcastOp::create(rewriter, loc, vtp, arg);
+        Value incr = vector::StepOp::create(rewriter, loc, vtp);
+        vexp = arith::AddIOp::create(rewriter, loc, veci, incr);
       }
       return true;
     }
@@ -526,15 +526,15 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
     Value step = constantIndex(rewriter, loc, vl.vectorLength);
     if (vl.enableVLAVectorization) {
       Value vscale =
-          rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-      step = rewriter.create<arith::MulIOp>(loc, vscale, step);
+          vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+      step = arith::MulIOp::create(rewriter, loc, vscale, step);
     }
     if (!yield.getResults().empty()) {
       Value init = forOp.getInitArgs()[0];
       VectorType vtp = vectorType(vl, init.getType());
       Value vinit = genVectorReducInit(rewriter, loc, yield->getOperand(0),
                                        forOp.getRegionIterArg(0), init, vtp);
-      forOpNew = rewriter.create<scf::ForOp>(
+      forOpNew = scf::ForOp::create(rewriter,
           loc, forOp.getLowerBound(), forOp.getUpperBound(), step, vinit);
       forOpNew->setAttr(
           LoopEmitter::getLoopEmitterLoopAttrName(),
@@ -563,10 +563,10 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       if (codegen) {
         Value partial = forOpNew.getResult(0);
         Value vpass = genVectorInvariantValue(rewriter, vl, iter);
-        Value vred = rewriter.create<arith::SelectOp>(loc, vmask, vrhs, vpass);
-        rewriter.create<scf::YieldOp>(loc, vred);
+        Value vred = arith::SelectOp::create(rewriter, loc, vmask, vrhs, vpass);
+        scf::YieldOp::create(rewriter, loc, vred);
         rewriter.setInsertionPointAfter(forOpNew);
-        Value vres = rewriter.create<vector::ReductionOp>(loc, kind, partial);
+        Value vres = vector::ReductionOp::create(rewriter, loc, kind, partial);
         // Now do some relinking (last one is not completely type safe
         // but all bad ones are removed right away). This also folds away
         // nop broadcast operations.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
index 2d604ed7a8ffc..0cabe5498d7f4 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
@@ -322,7 +322,7 @@ static void genBuffers(CodegenEnv &env, OpBuilder &builder) {
         if (!isInit) {
           Value zero = constantZero(builder, loc,
                                     getElementTypeOrSelf(tensor.getType()));
-          builder.create<linalg::FillOp>(loc, ValueRange{zero},
+          linalg::FillOp::create(builder, loc, ValueRange{zero},
                                          ValueRange{init});
         }
         return init;
@@ -385,7 +385,7 @@ static Value genInsertionLoad(CodegenEnv &env, OpBuilder &builder,
   }
   // Load from expanded access pattern.
   Value index = genIndex(env, t);
-  return builder.create<memref::LoadOp>(loc, env.getExpandValues(), index);
+  return memref::LoadOp::create(builder, loc, env.getExpandValues(), index);
 }
 
 /// Generates insertion code to implement dynamic tensor load for reduction.
@@ -401,22 +401,22 @@ static Value genInsertionLoadReduce(CodegenEnv &env, OpBuilder &builder,
   Value values = env.getExpandValues();
   Value filled = env.getExpandFilled();
   Value index = genIndex(env, t);
-  Value isFilled = builder.create<memref::LoadOp>(loc, filled, index);
-  Value valAtIndex = builder.create<memref::LoadOp>(loc, values, index);
-  return builder.create<arith::SelectOp>(loc, isFilled, valAtIndex, identity);
+  Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
+  Value valAtIndex = memref::LoadOp::create(builder, loc, values, index);
+  return arith::SelectOp::create(builder, loc, isFilled, valAtIndex, identity);
 }
 
 static Value genConditionalInsert(Location loc, OpBuilder &builder, Value cond,
                                   Value sparseOut, ValueRange ivs, Value v) {
   scf::IfOp condInsert =
-      builder.create<scf::IfOp>(loc, sparseOut.getType(), cond, true);
+      scf::IfOp::create(builder, loc, sparseOut.getType(), cond, true);
   // True branch.
   builder.setInsertionPointToStart(condInsert.thenBlock());
-  Value res = builder.create<tensor::InsertOp>(loc, v, sparseOut, ivs);
-  builder.create<scf::YieldOp>(loc, res);
+  Value res = tensor::InsertOp::create(builder, loc, v, sparseOut, ivs);
+  scf::YieldOp::create(builder, loc, res);
   // False branch.
   builder.setInsertionPointToStart(condInsert.elseBlock());
-  builder.create<scf::YieldOp>(loc, sparseOut);
+  scf::YieldOp::create(builder, loc, sparseOut);
   // Value assignment.
   builder.setInsertionPointAfter(condInsert);
   return condInsert.getResult(0);
@@ -453,7 +453,7 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
         Value nz = genIsNonzero(builder, loc, rhs);
         sparseOut = genConditionalInsert(loc, builder, nz, chain, ivs, rhs);
       } else {
-        sparseOut = builder.create<tensor::InsertOp>(loc, rhs, chain, ivs);
+        sparseOut = tensor::InsertOp::create(builder, loc, rhs, chain, ivs);
       }
       // Generates regular insertion chain.
       env.updateInsertionChain(sparseOut);
@@ -474,25 +474,25 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
   Value fval = constantI1(builder, loc, false);
   Value tval = constantI1(builder, loc, true);
   // If statement.
-  Value isFilled = builder.create<memref::LoadOp>(loc, filled, index);
-  Value cond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+  Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
+  Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
                                              isFilled, fval);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIndexType(), cond,
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIndexType(), cond,
                                              /*else=*/true);
   // True branch.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  builder.create<memref::StoreOp>(loc, tval, filled, index);
-  builder.create<memref::StoreOp>(loc, index, added, count);
+  memref::StoreOp::create(builder, loc, tval, filled, index);
+  memref::StoreOp::create(builder, loc, index, added, count);
   Value one = constantIndex(builder, loc, 1);
-  Value add = builder.create<arith::AddIOp>(loc, count, one);
-  builder.create<scf::YieldOp>(loc, add);
+  Value add = arith::AddIOp::create(builder, loc, count, one);
+  scf::YieldOp::create(builder, loc, add);
   // False branch.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, count);
+  scf::YieldOp::create(builder, loc, count);
   builder.setInsertionPointAfter(ifOp);
   // Value assignment.
   env.updateExpandCount(ifOp.getResult(0));
-  builder.create<memref::StoreOp>(loc, rhs, values, index);
+  memref::StoreOp::create(builder, loc, rhs, values, index);
 }
 
 /// Generates a load on a dense or sparse tensor.
@@ -522,9 +522,9 @@ static Value genTensorLoad(CodegenEnv &env, OpBuilder &builder, ExprId exp) {
   if (llvm::isa<TensorType>(ptr.getType())) {
     assert(env.options().sparseEmitStrategy ==
            SparseEmitStrategy::kSparseIterator);
-    return builder.create<ExtractValOp>(loc, ptr, llvm::getSingleElement(args));
+    return ExtractValOp::create(builder, loc, ptr, llvm::getSingleElement(args));
   }
-  return builder.create<memref::LoadOp>(loc, ptr, args);
+  return memref::LoadOp::create(builder, loc, ptr, args);
 }
 
 /// Generates a store on a dense or sparse tensor.
@@ -551,7 +551,7 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   if (!env.isSparseOutput(t)) {
     SmallVector<Value> args;
     Value ptr = genSubscript(env, builder, t, args);
-    builder.create<memref::StoreOp>(loc, rhs, ptr, args);
+    memref::StoreOp::create(builder, loc, rhs, ptr, args);
     return;
   }
   // Store during sparse insertion.
@@ -562,7 +562,7 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   // Select operation insertion.
   Value chain = env.getInsertionChain();
   scf::IfOp ifOp =
-      builder.create<scf::IfOp>(loc, chain.getType(), rhs, /*else=*/true);
+      scf::IfOp::create(builder, loc, chain.getType(), rhs, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   // Existing value was preserved to be used here.
   assert(env.exp(exp).val);
@@ -571,10 +571,10 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   env.merger().clearExprValue(exp);
   // Yield modified insertion chain along true branch.
   Value mchain = env.getInsertionChain();
-  builder.create<scf::YieldOp>(op.getLoc(), mchain);
+  scf::YieldOp::create(builder, op.getLoc(), mchain);
   // Yield original insertion chain along false branch.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, chain);
+  scf::YieldOp::create(builder, loc, chain);
   // Done with if statement.
   env.updateInsertionChain(ifOp->getResult(0));
   builder.setInsertionPointAfter(ifOp);
@@ -603,7 +603,7 @@ static Value relinkBranch(CodegenEnv &env, RewriterBase &rewriter, Block *block,
       assert(!getSparseTensorType(t->get()).hasEncoding()); // dense!
       SmallVector<Value> args;
       Value ptr = genSubscript(env, rewriter, t, args);
-      return rewriter.create<memref::LoadOp>(op.getLoc(), ptr, args);
+      return memref::LoadOp::create(rewriter, op.getLoc(), ptr, args);
     }
   } else if (Operation *def = e.getDefiningOp()) {
     // Handle index computation.
@@ -774,7 +774,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Type t2 = MemRefType::get(dynShape, builder.getI1Type());
     Type t3 = MemRefType::get(dynShape, builder.getIndexType());
     Type t4 = builder.getIndexType();
-    auto r = builder.create<ExpandOp>(loc, TypeRange({t1, t2, t3, t4}), tensor);
+    auto r = ExpandOp::create(builder, loc, TypeRange({t1, t2, t3, t4}), tensor);
     assert(r.getNumResults() == 4);
     env.startExpand(r.getResult(0), r.getResult(1), r.getResult(2),
                     r.getResult(3));
@@ -787,7 +787,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Value added = env.getExpandAdded();
     Value count = env.getExpandCount();
     Value chain = env.getInsertionChain();
-    Value compress = builder.create<CompressOp>(loc, values, filled, added,
+    Value compress = CompressOp::create(builder, loc, values, filled, added,
                                                 count, chain, indices);
     env.updateInsertionChain(compress);
     env.endExpand();
@@ -895,7 +895,7 @@ static void finalizeWhileOp(CodegenEnv &env, OpBuilder &builder,
         env.updateInsertionChain(ifOp->getResult(y++));
       }
       assert(y == yields.size());
-      builder.create<scf::YieldOp>(loc, yields);
+      scf::YieldOp::create(builder, loc, yields);
       builder.setInsertionPointAfter(ifOp);
     }
   }
@@ -948,13 +948,13 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
           assert(lvl.has_value());
           const Value crd = env.emitter().getCoord(tid, *lvl);
           const Value lvar = env.getLoopVar(curr);
-          clause = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+          clause = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
                                                  crd, lvar);
         } else {
           assert(lt.hasDenseSemantic() || isUndefLT(lt));
           clause = constantI1(builder, loc, true);
         }
-        cond = cond ? builder.create<arith::AndIOp>(loc, cond, clause) : clause;
+        cond = cond ? arith::AndIOp::create(builder, loc, cond, clause) : clause;
       });
   if (env.isReduc()) {
     types.push_back(env.getReduc().getType());
@@ -965,7 +965,7 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     types.push_back(builder.getIndexType());
   if (env.getInsertionChain())
     types.push_back(env.getInsertionChain().getType());
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   return ifOp;
 }
@@ -993,7 +993,7 @@ static void endIf(CodegenEnv &env, OpBuilder &builder, scf::IfOp ifOp,
     env.updateInsertionChain(insInput);
   }
   if (!operands.empty())
-    builder.create<scf::YieldOp>(env.op().getLoc(), operands);
+    scf::YieldOp::create(builder, env.op().getLoc(), operands);
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
 }
 
@@ -1307,7 +1307,7 @@ static void genStmt(CodegenEnv &env, RewriterBase &rewriter, ExprId exp,
           genStmt(env, rewriter, ej, curr + 1);
           // TODO: handle yield values.
           assert(reduc.empty() && "Not Implemented");
-          rewriter.create<sparse_tensor::YieldOp>(env.op().getLoc());
+          sparse_tensor::YieldOp::create(rewriter, env.op().getLoc());
           return std::nullopt;
         });
         // endIf(env, rewriter, ifOp, redInput, cntInput, insInput, validIns);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
index c370d104e0985..79c8d6c171249 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
@@ -42,7 +42,7 @@ struct GuardSparseAlloc
     // operation that leaves the underlying storage in a proper state
     // before the tensor escapes across the method boundary.
     rewriter.setInsertionPointAfter(op);
-    auto load = rewriter.create<LoadOp>(op.getLoc(), op.getResult(), true);
+    auto load = LoadOp::create(rewriter, op.getLoc(), op.getResult(), true);
     rewriter.replaceAllUsesExcept(op, load, load);
     return success();
   }
@@ -61,7 +61,7 @@ struct StageUnorderedSparseOps : public OpRewritePattern<StageWithSortOp> {
     // Deallocate tmpBuf.
     // TODO: Delegate to buffer deallocation pass in the future.
     if (succeeded(stageResult) && tmpBuf)
-      rewriter.create<bufferization::DeallocTensorOp>(loc, tmpBuf);
+      bufferization::DeallocTensorOp::create(rewriter, loc, tmpBuf);
 
     return stageResult;
   }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
index 1bd9563b3db07..7a735474b6d57 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
@@ -156,7 +156,7 @@ Value sparse_tensor::genCast(OpBuilder &builder, Location loc, Value value,
 
   // int <=> index
   if (isa<IndexType>(srcTp) || isa<IndexType>(dstTp))
-    return builder.create<arith::IndexCastOp>(loc, dstTp, value);
+    return arith::IndexCastOp::create(builder, loc, dstTp, value);
 
   const auto srcIntTp = dyn_cast_or_null<IntegerType>(srcTp);
   const bool isUnsignedCast = srcIntTp ? srcIntTp.isUnsigned() : false;
@@ -169,19 +169,19 @@ Value sparse_tensor::genScalarToTensor(OpBuilder &builder, Location loc,
     // Scalars can only be converted to 0-ranked tensors.
     assert(rtp.getRank() == 0);
     elem = sparse_tensor::genCast(builder, loc, elem, rtp.getElementType());
-    return builder.create<tensor::FromElementsOp>(loc, rtp, elem);
+    return tensor::FromElementsOp::create(builder, loc, rtp, elem);
   }
   return sparse_tensor::genCast(builder, loc, elem, dstTp);
 }
 
 Value sparse_tensor::genIndexLoad(OpBuilder &builder, Location loc, Value mem,
                                   ValueRange s) {
-  Value load = builder.create<memref::LoadOp>(loc, mem, s);
+  Value load = memref::LoadOp::create(builder, loc, mem, s);
   if (!isa<IndexType>(load.getType())) {
     if (load.getType().getIntOrFloatBitWidth() < 64)
-      load = builder.create<arith::ExtUIOp>(loc, builder.getI64Type(), load);
+      load = arith::ExtUIOp::create(builder, loc, builder.getI64Type(), load);
     load =
-        builder.create<arith::IndexCastOp>(loc, builder.getIndexType(), load);
+        arith::IndexCastOp::create(builder, loc, builder.getIndexType(), load);
   }
   return load;
 }
@@ -206,13 +206,13 @@ Value mlir::sparse_tensor::genIsNonzero(OpBuilder &builder, mlir::Location loc,
   Type tp = v.getType();
   Value zero = constantZero(builder, loc, tp);
   if (isa<FloatType>(tp))
-    return builder.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNE, v,
+    return arith::CmpFOp::create(builder, loc, arith::CmpFPredicate::UNE, v,
                                          zero);
   if (tp.isIntOrIndex())
-    return builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, v,
+    return arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, v,
                                          zero);
   if (isa<ComplexType>(tp))
-    return builder.create<complex::NotEqualOp>(loc, v, zero);
+    return complex::NotEqualOp::create(builder, loc, v, zero);
   llvm_unreachable("Non-numeric type");
 }
 
@@ -226,7 +226,7 @@ void mlir::sparse_tensor::genReshapeDstShape(
     for (const auto &map : llvm::enumerate(reassociation)) {
       auto dstDim = constantIndex(builder, loc, 1);
       for (unsigned i = start; i < start + map.value().size(); i++) {
-        dstDim = builder.create<arith::MulIOp>(loc, dstDim, srcShape[i]);
+        dstDim = arith::MulIOp::create(builder, loc, dstDim, srcShape[i]);
       }
       dstShape.push_back(dstDim);
       start = start + map.value().size();
@@ -260,7 +260,7 @@ void mlir::sparse_tensor::genReshapeDstShape(
         // Compute the dynamic dimension size.
         Value productVal = constantIndex(builder, loc, product);
         Value dynamicSize =
-            builder.create<arith::DivUIOp>(loc, srcDim, productVal);
+            arith::DivUIOp::create(builder, loc, srcDim, productVal);
         dstShape.push_back(dynamicSize);
       } else {
         // The expanded dimension is statically known.
@@ -289,7 +289,7 @@ void mlir::sparse_tensor::reshapeCvs(
     // Prepare strides information in dimension slice.
     Value linear = constantIndex(builder, loc, 1);
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
-      linear = builder.create<arith::MulIOp>(loc, linear, sizes[j]);
+      linear = arith::MulIOp::create(builder, loc, linear, sizes[j]);
     }
     // Start expansion.
     Value val;
@@ -297,16 +297,16 @@ void mlir::sparse_tensor::reshapeCvs(
       val = srcCvs[i];
     // Iterate over dimension slice.
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
-      linear = builder.create<arith::DivUIOp>(loc, linear, sizes[j]);
+      linear = arith::DivUIOp::create(builder, loc, linear, sizes[j]);
       if (isCollapse) {
-        const Value mul = builder.create<arith::MulIOp>(loc, srcCvs[j], linear);
-        val = val ? builder.create<arith::AddIOp>(loc, val, mul) : mul;
+        const Value mul = arith::MulIOp::create(builder, loc, srcCvs[j], linear);
+        val = val ? arith::AddIOp::create(builder, loc, val, mul) : mul;
       } else {
         const Value old = val;
-        val = builder.create<arith::DivUIOp>(loc, val, linear);
+        val = arith::DivUIOp::create(builder, loc, val, linear);
         assert(dstCvs.size() == j);
         dstCvs.push_back(val);
-        val = builder.create<arith::RemUIOp>(loc, old, linear);
+        val = arith::RemUIOp::create(builder, loc, old, linear);
       }
     }
     // Finalize collapse.
@@ -329,7 +329,7 @@ FlatSymbolRefAttr mlir::sparse_tensor::getFunc(ModuleOp module, StringRef name,
   auto func = module.lookupSymbol<func::FuncOp>(result.getAttr());
   if (!func) {
     OpBuilder moduleBuilder(module.getBodyRegion());
-    func = moduleBuilder.create<func::FuncOp>(
+    func = func::FuncOp::create(moduleBuilder,
         module.getLoc(), name,
         FunctionType::get(context, operands.getTypes(), resultType));
     func.setPrivate();
@@ -346,7 +346,7 @@ func::CallOp mlir::sparse_tensor::createFuncCall(
   auto module = builder.getBlock()->getParentOp()->getParentOfType<ModuleOp>();
   FlatSymbolRefAttr fn =
       getFunc(module, name, resultType, operands, emitCInterface);
-  return builder.create<func::CallOp>(loc, resultType, fn, operands);
+  return func::CallOp::create(builder, loc, resultType, fn, operands);
 }
 
 Type mlir::sparse_tensor::getOpaquePointerType(MLIRContext *ctx) {
@@ -361,7 +361,7 @@ Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
                                      unsigned sz, Type tp, bool staticShape) {
   if (staticShape) {
     auto memTp = MemRefType::get({sz}, tp);
-    return builder.create<memref::AllocaOp>(loc, memTp);
+    return memref::AllocaOp::create(builder, loc, memTp);
   }
   return genAlloca(builder, loc, constantIndex(builder, loc, sz), tp);
 }
@@ -369,12 +369,12 @@ Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
 Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc, Value sz,
                                      Type tp) {
   auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
-  return builder.create<memref::AllocaOp>(loc, memTp, ValueRange{sz});
+  return memref::AllocaOp::create(builder, loc, memTp, ValueRange{sz});
 }
 
 Value mlir::sparse_tensor::genAllocaScalar(OpBuilder &builder, Location loc,
                                            Type tp) {
-  return builder.create<memref::AllocaOp>(loc, MemRefType::get({}, tp));
+  return memref::AllocaOp::create(builder, loc, MemRefType::get({}, tp));
 }
 
 Value mlir::sparse_tensor::allocaBuffer(OpBuilder &builder, Location loc,
@@ -384,7 +384,7 @@ Value mlir::sparse_tensor::allocaBuffer(OpBuilder &builder, Location loc,
   Value buffer = genAlloca(builder, loc, sz, values[0].getType());
   for (unsigned i = 0; i < sz; i++) {
     Value idx = constantIndex(builder, loc, i);
-    builder.create<memref::StoreOp>(loc, values[i], buffer, idx);
+    memref::StoreOp::create(builder, loc, values[i], buffer, idx);
   }
   return buffer;
 }
@@ -400,15 +400,15 @@ Value mlir::sparse_tensor::allocDenseTensor(OpBuilder &builder, Location loc,
     if (shape[i] == ShapedType::kDynamic)
       dynamicSizes.push_back(sizes[i]);
   }
-  Value mem = builder.create<memref::AllocOp>(loc, memTp, dynamicSizes);
+  Value mem = memref::AllocOp::create(builder, loc, memTp, dynamicSizes);
   Value zero = constantZero(builder, loc, elemTp);
-  builder.create<linalg::FillOp>(loc, ValueRange{zero}, ValueRange{mem});
+  linalg::FillOp::create(builder, loc, ValueRange{zero}, ValueRange{mem});
   return mem;
 }
 
 void mlir::sparse_tensor::deallocDenseTensor(OpBuilder &builder, Location loc,
                                              Value buffer) {
-  builder.create<memref::DeallocOp>(loc, buffer);
+  memref::DeallocOp::create(builder, loc, buffer);
 }
 
 void mlir::sparse_tensor::sizesFromSrc(OpBuilder &builder,
@@ -486,17 +486,17 @@ void sparse_tensor::foreachInSparseConstant(
     cvs.clear();
     for (Dimension d = 0; d < dimRank; d++) {
       auto crd = elems[i].first[d].getInt();
-      cvs.push_back(builder.create<arith::ConstantIndexOp>(loc, crd));
+      cvs.push_back(arith::ConstantIndexOp::create(builder, loc, crd));
     }
     // Remap value.
     Value val;
     if (isa<ComplexType>(attr.getElementType())) {
       auto valAttr = cast<ArrayAttr>(elems[i].second);
-      val = builder.create<complex::ConstantOp>(loc, attr.getElementType(),
+      val = complex::ConstantOp::create(builder, loc, attr.getElementType(),
                                                 valAttr);
     } else {
       auto valAttr = cast<TypedAttr>(elems[i].second);
-      val = builder.create<arith::ConstantOp>(loc, valAttr);
+      val = arith::ConstantOp::create(builder, loc, valAttr);
     }
     assert(val);
     callback(cvs, val);
@@ -516,10 +516,10 @@ SmallVector<Value> sparse_tensor::loadAll(OpBuilder &builder, Location loc,
   SmallVector<Value> vs;
   vs.reserve(size);
   for (unsigned i = 0; i < size; i++) {
-    Value v = builder.create<memref::LoadOp>(loc, mem,
+    Value v = memref::LoadOp::create(builder, loc, mem,
                                              constantIndex(builder, loc, i));
     if (i == offsetIdx && offsetVal)
-      v = builder.create<arith::AddIOp>(loc, v, offsetVal);
+      v = arith::AddIOp::create(builder, loc, v, offsetVal);
     vs.push_back(v);
   }
   return vs;
@@ -538,9 +538,9 @@ void sparse_tensor::storeAll(OpBuilder &builder, Location loc, Value mem,
   for (const auto &v : llvm::enumerate(vs)) {
     const Value w =
         (offsetIdx == v.index() && offsetVal)
-            ? builder.create<arith::AddIOp>(loc, v.value(), offsetVal)
+            ? arith::AddIOp::create(builder, loc, v.value(), offsetVal)
             : v.value();
-    builder.create<memref::StoreOp>(loc, w, mem,
+    memref::StoreOp::create(builder, loc, w, mem,
                                     constantIndex(builder, loc, v.index()));
   }
 }
@@ -550,7 +550,7 @@ sparse_tensor::genToMemref(OpBuilder &builder, Location loc, Value tensor) {
   auto tTp = llvm::cast<TensorType>(tensor.getType());
   auto mTp = MemRefType::get(tTp.getShape(), tTp.getElementType());
   return cast<TypedValue<BaseMemRefType>>(
-      builder.create<bufferization::ToBufferOp>(loc, mTp, tensor).getResult());
+      bufferization::ToBufferOp::create(builder, loc, mTp, tensor).getResult());
 }
 
 Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
@@ -560,7 +560,7 @@ Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
   std::optional<unsigned> offset = enc.getStaticDimSliceOffset(dim);
   if (offset.has_value())
     return constantIndex(builder, loc, *offset);
-  return builder.create<ToSliceOffsetOp>(loc, tensor, APInt(64, dim));
+  return ToSliceOffsetOp::create(builder, loc, tensor, APInt(64, dim));
 }
 
 Value sparse_tensor::createOrFoldSliceStrideOp(OpBuilder &builder, Location loc,
@@ -570,7 +570,7 @@ Value sparse_tensor::createOrFoldSliceStrideOp(OpBuilder &builder, Location loc,
   std::optional<unsigned> stride = enc.getStaticDimSliceStride(dim);
   if (stride.has_value())
     return constantIndex(builder, loc, *stride);
-  return builder.create<ToSliceStrideOp>(loc, tensor, APInt(64, dim));
+  return ToSliceStrideOp::create(builder, loc, tensor, APInt(64, dim));
 }
 
 Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
@@ -612,7 +612,7 @@ Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
     // subsequent clients need the values (DCE will remove unused).
     for (Dimension d = 0; d < dimRank; d++) {
       if (stt.isDynamicDim(d))
-        dimSizesValues[d] = builder.create<memref::LoadOp>(
+        dimSizesValues[d] = memref::LoadOp::create(builder,
             loc, dimSizesBuffer, constantIndex(builder, loc, d));
     }
   }
@@ -689,7 +689,7 @@ Value sparse_tensor::genMapBuffers(
     if (cm == 0) {
       lvlSz = dimSizesValues[d];
       if (cf != 0)
-        lvlSz = builder.create<arith::DivUIOp>(loc, lvlSz,
+        lvlSz = arith::DivUIOp::create(builder, loc, lvlSz,
                                                constantIndex(builder, loc, cf));
     } else {
       lvlSz = constantIndex(builder, loc, cm);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
index dc017e6baa6dc..400be54aeb86f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
@@ -97,7 +97,7 @@ class FuncCallOrInlineGenerator {
       // Create the function if not already exist.
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(getParentOpOf<func::FuncOp>(builder));
-      func = builder.create<func::FuncOp>(
+      func = func::FuncOp::create(builder,
           loc, funcName,
           FunctionType::get(context, params.getTypes(), retTypes));
       func.setPrivate();
@@ -108,10 +108,10 @@ class FuncCallOrInlineGenerator {
       // Delegates to user to generate the actually implementation.
       SmallVector<Value> result =
           genImplementation(retTypes, args, builder, loc);
-      builder.create<func::ReturnOp>(loc, result);
+      func::ReturnOp::create(builder, loc, result);
     }
     // Returns the CallOp result.
-    func::CallOp call = builder.create<func::CallOp>(loc, func, params);
+    func::CallOp call = func::CallOp::create(builder, loc, func, params);
     return call.getResults();
   }
 
@@ -310,9 +310,9 @@ inline Value constantZero(OpBuilder &builder, Location loc, Type tp) {
   if (auto ctp = dyn_cast<ComplexType>(tp)) {
     auto zeroe = builder.getZeroAttr(ctp.getElementType());
     auto zeroa = builder.getArrayAttr({zeroe, zeroe});
-    return builder.create<complex::ConstantOp>(loc, tp, zeroa);
+    return complex::ConstantOp::create(builder, loc, tp, zeroa);
   }
-  return builder.create<arith::ConstantOp>(loc, tp, builder.getZeroAttr(tp));
+  return arith::ConstantOp::create(builder, loc, tp, builder.getZeroAttr(tp));
 }
 
 /// Generates a 1-valued constant of the given type.  This supports all
@@ -322,39 +322,39 @@ inline Value constantOne(OpBuilder &builder, Location loc, Type tp) {
     auto zeroe = builder.getZeroAttr(ctp.getElementType());
     auto onee = getOneAttr(builder, ctp.getElementType());
     auto zeroa = builder.getArrayAttr({onee, zeroe});
-    return builder.create<complex::ConstantOp>(loc, tp, zeroa);
+    return complex::ConstantOp::create(builder, loc, tp, zeroa);
   }
-  return builder.create<arith::ConstantOp>(loc, tp, getOneAttr(builder, tp));
+  return arith::ConstantOp::create(builder, loc, tp, getOneAttr(builder, tp));
 }
 
 /// Generates a constant of `index` type.
 inline Value constantIndex(OpBuilder &builder, Location loc, int64_t i) {
-  return builder.create<arith::ConstantIndexOp>(loc, i);
+  return arith::ConstantIndexOp::create(builder, loc, i);
 }
 
 /// Generates a constant of `i64` type.
 inline Value constantI64(OpBuilder &builder, Location loc, int64_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 64);
+  return arith::ConstantIntOp::create(builder, loc, i, 64);
 }
 
 /// Generates a constant of `i32` type.
 inline Value constantI32(OpBuilder &builder, Location loc, int32_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 32);
+  return arith::ConstantIntOp::create(builder, loc, i, 32);
 }
 
 /// Generates a constant of `i16` type.
 inline Value constantI16(OpBuilder &builder, Location loc, int16_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 16);
+  return arith::ConstantIntOp::create(builder, loc, i, 16);
 }
 
 /// Generates a constant of `i8` type.
 inline Value constantI8(OpBuilder &builder, Location loc, int8_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 8);
+  return arith::ConstantIntOp::create(builder, loc, i, 8);
 }
 
 /// Generates a constant of `i1` type.
 inline Value constantI1(OpBuilder &builder, Location loc, bool b) {
-  return builder.create<arith::ConstantIntOp>(loc, b, 1);
+  return arith::ConstantIntOp::create(builder, loc, b, 1);
 }
 
 /// Generates a constant of the given `Action`.
@@ -400,12 +400,12 @@ inline Value constantLevelTypeEncoding(OpBuilder &builder, Location loc,
 inline Value genValFromAttr(OpBuilder &builder, Location loc, Attribute attr) {
   if (auto complexAttr = dyn_cast<complex::NumberAttr>(attr)) {
     Type tp = cast<ComplexType>(complexAttr.getType()).getElementType();
-    return builder.create<complex::ConstantOp>(
+    return complex::ConstantOp::create(builder,
         loc, complexAttr.getType(),
         builder.getArrayAttr({FloatAttr::get(tp, complexAttr.getReal()),
                               FloatAttr::get(tp, complexAttr.getImag())}));
   }
-  return builder.create<arith::ConstantOp>(loc, cast<TypedAttr>(attr));
+  return arith::ConstantOp::create(builder, loc, cast<TypedAttr>(attr));
 }
 
 // TODO: is this at the right place?
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
index 9e41c8ec19bcd..6d795f91b1ab2 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
@@ -27,18 +27,18 @@ using namespace mlir::sparse_tensor;
 //===----------------------------------------------------------------------===//
 
 #define CMPI(p, l, r)                                                          \
-  (builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::p, (l), (r))       \
+  (arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::p, (l), (r))       \
        .getResult())
 
 #define C_IDX(v) (constantIndex(builder, loc, (v)))
-#define YIELD(vs) (builder.create<scf::YieldOp>(loc, (vs)))
-#define ADDI(lhs, rhs) (builder.create<arith::AddIOp>(loc, (lhs), (rhs)))
-#define ANDI(lhs, rhs) (builder.create<arith::AndIOp>(loc, (lhs), (rhs)))
-#define SUBI(lhs, rhs) (builder.create<arith::SubIOp>(loc, (lhs), (rhs)))
-#define MULI(lhs, rhs) (builder.create<arith::MulIOp>(loc, (lhs), (rhs)))
-#define REMUI(lhs, rhs) (builder.create<arith::RemUIOp>(loc, (lhs), (rhs)))
-#define DIVUI(lhs, rhs) (builder.create<arith::DivUIOp>(loc, (lhs), (rhs)))
-#define SELECT(c, l, r) (builder.create<arith::SelectOp>(loc, (c), (l), (r)))
+#define YIELD(vs) (scf::YieldOp::create(builder, loc, (vs)))
+#define ADDI(lhs, rhs) (arith::AddIOp::create(builder, loc, (lhs), (rhs)))
+#define ANDI(lhs, rhs) (arith::AndIOp::create(builder, loc, (lhs), (rhs)))
+#define SUBI(lhs, rhs) (arith::SubIOp::create(builder, loc, (lhs), (rhs)))
+#define MULI(lhs, rhs) (arith::MulIOp::create(builder, loc, (lhs), (rhs)))
+#define REMUI(lhs, rhs) (arith::RemUIOp::create(builder, loc, (lhs), (rhs)))
+#define DIVUI(lhs, rhs) (arith::DivUIOp::create(builder, loc, (lhs), (rhs)))
+#define SELECT(c, l, r) (arith::SelectOp::create(builder, loc, (c), (l), (r)))
 
 //===----------------------------------------------------------------------===//
 // Debugging utils
@@ -47,7 +47,7 @@ using namespace mlir::sparse_tensor;
 #ifndef NDEBUG
 LLVM_ATTRIBUTE_UNUSED static void dumpIndexMemRef(OpBuilder &builder,
                                                   Location loc, Value memref) {
-  memref = builder.create<memref::CastOp>(
+  memref = memref::CastOp::create(builder,
       loc, UnrankedMemRefType::get(builder.getIndexType(), 0), memref);
   createFuncCall(builder, loc, "printMemrefInd", TypeRange{},
                  ValueRange{memref}, EmitCInterface::On);
@@ -263,7 +263,7 @@ void LoopEmitter::initializeLoopEmit(
         denseTp = bufferization::getMemRefTypeWithFullyDynamicLayout(rtp);
 
       Value denseVal =
-          builder.create<bufferization::ToBufferOp>(loc, denseTp, tensor);
+          bufferization::ToBufferOp::create(builder, loc, denseTp, tensor);
       // Dense outputs need special handling.
       if (isOutput && updater)
         denseVal = updater(builder, loc, denseVal, tensor);
@@ -273,7 +273,7 @@ void LoopEmitter::initializeLoopEmit(
       // Annotated sparse tensors.
       // We also need the value buffer for all-dense annotated "sparse"
       // tensors.
-      valBuffer[t] = builder.create<ToValuesOp>(loc, tensor);
+      valBuffer[t] = ToValuesOp::create(builder, loc, tensor);
     }
   }
 
@@ -481,7 +481,7 @@ std::pair<Operation *, Value> LoopEmitter::emitForLoopOverTensorAtLvl(
   Value iv;
   if (isParallel) {
     scf::ParallelOp parOp =
-        builder.create<scf::ParallelOp>(loc, lo, hi, step, reduc);
+        scf::ParallelOp::create(builder, loc, lo, hi, step, reduc);
     builder.setInsertionPointToStart(parOp.getBody());
     assert(parOp.getNumReductions() == reduc.size());
     iv = parOp.getInductionVars()[0];
@@ -497,7 +497,7 @@ std::pair<Operation *, Value> LoopEmitter::emitForLoopOverTensorAtLvl(
       reduc[i] = parOp.getInitVals()[i];
     loop = parOp;
   } else {
-    scf::ForOp forOp = builder.create<scf::ForOp>(loc, lo, hi, step, reduc);
+    scf::ForOp forOp = scf::ForOp::create(builder, loc, lo, hi, step, reduc);
     builder.setInsertionPointToStart(forOp.getBody());
     iv = forOp.getInductionVar();
 
@@ -605,11 +605,11 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
 
       // Extract and iterate over the iteration space.
       ExtractIterSpaceOp extractSpaceOp =
-          lvl == 0 ? builder.create<ExtractIterSpaceOp>(loc, t)
-                   : builder.create<ExtractIterSpaceOp>(
+          lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
+                   : ExtractIterSpaceOp::create(builder,
                          loc, t, spIterVals[tid][lvl - 1], lvl);
 
-      IterateOp iterOp = builder.create<IterateOp>(
+      IterateOp iterOp = IterateOp::create(builder,
           loc, extractSpaceOp.getExtractedSpace(), reduc);
       spIterVals[tid][lvl] = iterOp.getIterator();
 
@@ -627,12 +627,12 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
     for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
       Value t = tensors[tid];
       ExtractIterSpaceOp extractSpaceOp =
-          lvl == 0 ? builder.create<ExtractIterSpaceOp>(loc, t)
-                   : builder.create<ExtractIterSpaceOp>(
+          lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
+                   : ExtractIterSpaceOp::create(builder,
                          loc, t, spIterVals[tid][lvl - 1], lvl);
       spaces.push_back(extractSpaceOp.getExtractedSpace());
     }
-    auto coIterOp = builder.create<CoIterateOp>(loc, spaces, reduc, numCases);
+    auto coIterOp = CoIterateOp::create(builder, loc, spaces, reduc, numCases);
     // The CoIterationOp does not have insertion block nor induction variable.
     // TODO: the `struct LoopInfo` should be simplied after full migration.
     loopStack.emplace_back(tidLvls, coIterOp, /*insertion block*/ nullptr,
@@ -730,7 +730,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   if (emitStrategy == SparseEmitStrategy::kSparseIterator) {
     auto iterateOp = llvm::cast<IterateOp>(loopInfo.loop);
     assert(reduc.size() == iterateOp.getNumResults());
-    rewriter.create<sparse_tensor::YieldOp>(loc, reduc);
+    sparse_tensor::YieldOp::create(rewriter, loc, reduc);
     // Exit the loop.
     rewriter.setInsertionPointAfter(iterateOp);
     // In-place update reduction variables.
@@ -740,7 +740,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   if (auto forOp = llvm::dyn_cast<scf::ForOp>(loopInfo.loop)) {
     if (!reduc.empty()) {
       assert(reduc.size() == forOp.getNumResults());
-      rewriter.create<scf::YieldOp>(loc, reduc);
+      scf::YieldOp::create(rewriter, loc, reduc);
     }
     // Exit the loop.
     rewriter.setInsertionPointAfter(forOp);
@@ -779,7 +779,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
 #endif // NDEBUG
 
       rewriter.setInsertionPointAfter(redExp);
-      auto redOp = rewriter.create<scf::ReduceOp>(loc, curVal);
+      auto redOp = scf::ReduceOp::create(rewriter, loc, curVal);
       // Attach to the reduction op.
       Block *redBlock = &redOp.getReductions().front().front();
       rewriter.setInsertionPointToEnd(redBlock);
@@ -791,7 +791,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
       // Erases the out-dated reduction expression.
       rewriter.eraseOp(redExp);
       rewriter.setInsertionPointToEnd(redBlock);
-      rewriter.create<scf::ReduceReturnOp>(loc, newRed->getResult(0));
+      scf::ReduceReturnOp::create(rewriter, loc, newRed->getResult(0));
     }
     rewriter.setInsertionPointAfter(parOp);
     // In-place update reduction variables.
@@ -865,7 +865,7 @@ void LoopEmitter::exitCurrentLoop(RewriterBase &rewriter, Location loc,
   if (emitStrategy == SparseEmitStrategy::kSparseIterator) {
     Operation *p = loopInfo.loop;
     if (isa<IterateOp>(p))
-      rewriter.create<sparse_tensor::YieldOp>(loc, reduc);
+      sparse_tensor::YieldOp::create(rewriter, loc, reduc);
 
     // Exit the loop.
     rewriter.setInsertionPointAfter(p);
@@ -931,7 +931,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
   // Ensures all operands are valid.
   assert(!llvm::is_contained(ivs, nullptr));
   TypeRange types = ValueRange(ivs).getTypes();
-  auto whileOp = builder.create<scf::WhileOp>(loc, types, ivs);
+  auto whileOp = scf::WhileOp::create(builder, loc, types, ivs);
 
   SmallVector<Location> locs(types.size(), loc);
   Block *before = builder.createBlock(&whileOp.getBefore(), {}, types, locs);
@@ -950,7 +950,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
   // The remaining block arguments are user-provided reduction values and an
   // optional universal index. Make sure their sizes match.
   assert(bArgs.size() == reduc.size() + (uniIdx ? 1 : 0));
-  builder.create<scf::ConditionOp>(loc, whileCond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, whileCond, before->getArguments());
 
   // Generates loop body.
   builder.setInsertionPointToStart(after);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
index cf99117065c5f..3b6976f8b7341 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
@@ -79,14 +79,14 @@ SparseTensorTypeToBufferConverter::SparseTensorTypeToBufferConverter() {
 
 Value SparseTensorSpecifier::getInitValue(OpBuilder &builder, Location loc,
                                           SparseTensorType stt) {
-  return builder.create<StorageSpecifierInitOp>(
+  return StorageSpecifierInitOp::create(builder,
       loc, StorageSpecifierType::get(stt.getEncoding()));
 }
 
 Value SparseTensorSpecifier::getSpecifierField(OpBuilder &builder, Location loc,
                                                StorageSpecifierKind kind,
                                                std::optional<Level> lvl) {
-  return builder.create<GetStorageSpecifierOp>(
+  return GetStorageSpecifierOp::create(builder,
       loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl));
 }
 
@@ -96,7 +96,7 @@ void SparseTensorSpecifier::setSpecifierField(OpBuilder &builder, Location loc,
                                               std::optional<Level> lvl) {
   // TODO: make `v` have type `TypedValue<IndexType>` instead.
   assert(v.getType().isIndex());
-  specifier = builder.create<SetStorageSpecifierOp>(
+  specifier = SetStorageSpecifierOp::create(builder,
       loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl), v);
 }
 
@@ -112,8 +112,8 @@ Value sparse_tensor::SparseTensorDescriptor::getCrdMemRefOrView(
 
   Value stride = constantIndex(builder, loc, rType.getLvlRank() - cooStart);
   Value size = getCrdMemSize(builder, loc, cooStart);
-  size = builder.create<arith::DivUIOp>(loc, size, stride);
-  return builder.create<memref::SubViewOp>(
+  size = arith::DivUIOp::create(builder, loc, size, stride);
+  return memref::SubViewOp::create(builder,
       loc, getMemRefField(SparseTensorFieldKind::CrdMemRef, cooStart),
       /*offset=*/ValueRange{constantIndex(builder, loc, lvl - cooStart)},
       /*size=*/ValueRange{size},
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
index 869c7864d7535..45d142a807c36 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
@@ -231,7 +231,7 @@ class MutSparseTensorDescriptor
 /// Packs the given values as a "tuple" value.
 inline Value genTuple(OpBuilder &builder, Location loc, Type tp,
                       ValueRange values) {
-  return builder.create<UnrealizedConversionCastOp>(loc, TypeRange(tp), values)
+  return UnrealizedConversionCastOp::create(builder, loc, TypeRange(tp), values)
       .getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
index aad5e97ed14ab..212ee99647167 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
@@ -22,23 +22,23 @@ using ValueTuple = std::tuple<Value, Value, Value>;
 // File local helper functions/macros.
 //===----------------------------------------------------------------------===//
 #define CMPI(p, lhs, rhs)                                                      \
-  (b.create<arith::CmpIOp>(l, arith::CmpIPredicate::p, (lhs), (rhs))           \
+  (arith::CmpIOp::create(b, l, arith::CmpIPredicate::p, (lhs), (rhs))           \
        .getResult())
 
 #define C_FALSE (constantI1(b, l, false))
 #define C_TRUE (constantI1(b, l, true))
 #define C_IDX(v) (constantIndex(b, l, (v)))
-#define YIELD(vs) (b.create<scf::YieldOp>(l, (vs)))
-#define ADDI(lhs, rhs) (b.create<arith::AddIOp>(l, (lhs), (rhs)).getResult())
-#define ORI(lhs, rhs) (b.create<arith::OrIOp>(l, (lhs), (rhs)).getResult())
-#define ANDI(lhs, rhs) (b.create<arith::AndIOp>(l, (lhs), (rhs)).getResult())
-#define SUBI(lhs, rhs) (b.create<arith::SubIOp>(l, (lhs), (rhs)).getResult())
-#define MULI(lhs, rhs) (b.create<arith::MulIOp>(l, (lhs), (rhs)).getResult())
-#define MINUI(lhs, rhs) (b.create<arith::MinUIOp>(l, (lhs), (rhs)).getResult())
-#define REMUI(lhs, rhs) (b.create<arith::RemUIOp>(l, (lhs), (rhs)).getResult())
-#define DIVUI(lhs, rhs) (b.create<arith::DivUIOp>(l, (lhs), (rhs)).getResult())
+#define YIELD(vs) (scf::YieldOp::create(b, l, (vs)))
+#define ADDI(lhs, rhs) (arith::AddIOp::create(b, l, (lhs), (rhs)).getResult())
+#define ORI(lhs, rhs) (arith::OrIOp::create(b, l, (lhs), (rhs)).getResult())
+#define ANDI(lhs, rhs) (arith::AndIOp::create(b, l, (lhs), (rhs)).getResult())
+#define SUBI(lhs, rhs) (arith::SubIOp::create(b, l, (lhs), (rhs)).getResult())
+#define MULI(lhs, rhs) (arith::MulIOp::create(b, l, (lhs), (rhs)).getResult())
+#define MINUI(lhs, rhs) (arith::MinUIOp::create(b, l, (lhs), (rhs)).getResult())
+#define REMUI(lhs, rhs) (arith::RemUIOp::create(b, l, (lhs), (rhs)).getResult())
+#define DIVUI(lhs, rhs) (arith::DivUIOp::create(b, l, (lhs), (rhs)).getResult())
 #define SELECT(c, lhs, rhs)                                                    \
-  (b.create<arith::SelectOp>(l, (c), (lhs), (rhs)).getResult())
+  (arith::SelectOp::create(b, l, (c), (lhs), (rhs)).getResult())
 
 //===----------------------------------------------------------------------===//
 // SparseTensorLevel derived classes.
@@ -150,19 +150,19 @@ class CompressedLevel : public SparseLevel</*hasPosBuf=*/true> {
       return loadRange();
 
     SmallVector<Type, 2> types{b.getIndexType(), b.getIndexType()};
-    scf::IfOp posRangeIf = b.create<scf::IfOp>(l, types, inPadZone, true);
+    scf::IfOp posRangeIf = scf::IfOp::create(b, l, types, inPadZone, true);
     // True branch, returns a "fake" empty range [0, 0) if parent
     // iterator is in pad zone.
     b.setInsertionPointToStart(posRangeIf.thenBlock());
 
     SmallVector<Value, 2> emptyRange{C_IDX(0), C_IDX(0)};
-    b.create<scf::YieldOp>(l, emptyRange);
+    scf::YieldOp::create(b, l, emptyRange);
 
     // False branch, returns the actual range.
     b.setInsertionPointToStart(posRangeIf.elseBlock());
     auto [pLo, pHi] = loadRange();
     SmallVector<Value, 2> loadedRange{pLo, pHi};
-    b.create<scf::YieldOp>(l, loadedRange);
+    scf::YieldOp::create(b, l, loadedRange);
 
     b.setInsertionPointAfter(posRangeIf);
     ValueRange posRange = posRangeIf.getResults();
@@ -248,7 +248,7 @@ static scf::ValueVector genWhenInBound(
     llvm::function_ref<scf::ValueVector(OpBuilder &, Location, Value)>
         builder) {
   TypeRange ifRetTypes = elseRet.getTypes();
-  auto ifOp = b.create<scf::IfOp>(l, ifRetTypes, it.genNotEnd(b, l), true);
+  auto ifOp = scf::IfOp::create(b, l, ifRetTypes, it.genNotEnd(b, l), true);
 
   b.setInsertionPointToStart(ifOp.thenBlock());
   Value crd = it.deref(b, l);
@@ -732,7 +732,7 @@ class NonEmptySubSectIterator : public SparseIterator {
   //      [itVal0, itVal1, ..., pNx0],
   //      ...]
   Value allocSubSectPosBuf(OpBuilder &b, Location l) {
-    return b.create<memref::AllocaOp>(
+    return memref::AllocaOp::create(b,
         l,
         MemRefType::get({ShapedType::kDynamic, tupleSz + 1}, b.getIndexType()),
         maxTupleCnt);
@@ -740,12 +740,12 @@ class NonEmptySubSectIterator : public SparseIterator {
 
   void storeNxLvlStart(OpBuilder &b, Location l, Value tupleId,
                        Value start) const {
-    b.create<memref::StoreOp>(l, start, subSectPosBuf,
+    memref::StoreOp::create(b, l, start, subSectPosBuf,
                               ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
   Value loadNxLvlStart(OpBuilder &b, Location l, Value tupleId) const {
-    return b.create<memref::LoadOp>(l, subSectPosBuf,
+    return memref::LoadOp::create(b, l, subSectPosBuf,
                                     ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
@@ -753,7 +753,7 @@ class NonEmptySubSectIterator : public SparseIterator {
                        ValueRange itVals) const {
     assert(itVals.size() == tupleSz);
     for (unsigned i = 0; i < tupleSz; i++) {
-      b.create<memref::StoreOp>(l, itVals[i], subSectPosBuf,
+      memref::StoreOp::create(b, l, itVals[i], subSectPosBuf,
                                 ValueRange{tupleId, C_IDX(i)});
     }
   }
@@ -762,7 +762,7 @@ class NonEmptySubSectIterator : public SparseIterator {
                                     Value tupleId) const {
     SmallVector<Value> ret;
     for (unsigned i = 0; i < tupleSz; i++) {
-      Value v = b.create<memref::LoadOp>(l, subSectPosBuf,
+      Value v = memref::LoadOp::create(b, l, subSectPosBuf,
                                          ValueRange{tupleId, C_IDX(i)});
       ret.push_back(v);
     }
@@ -1043,7 +1043,7 @@ ValueRange SparseIterator::forward(OpBuilder &b, Location l) {
 }
 
 ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
-  auto ifOp = b.create<scf::IfOp>(l, getCursor().getTypes(), cond, true);
+  auto ifOp = scf::IfOp::create(b, l, getCursor().getTypes(), cond, true);
   // Generate else branch first, otherwise iterator values will be updated by
   // `forward()`.
   b.setInsertionPointToStart(ifOp.elseBlock());
@@ -1058,12 +1058,12 @@ ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
 }
 
 Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
-  auto whileOp = b.create<scf::WhileOp>(
+  auto whileOp = scf::WhileOp::create(b,
       l, pos.getType(), pos,
       /*beforeBuilder=*/
       [this, pos](OpBuilder &b, Location l, ValueRange ivs) {
         Value inBound = CMPI(ult, ivs.front(), posHi);
-        auto ifInBound = b.create<scf::IfOp>(l, b.getI1Type(), inBound, true);
+        auto ifInBound = scf::IfOp::create(b, l, b.getI1Type(), inBound, true);
         {
           OpBuilder::InsertionGuard guard(b);
           // If in bound, load the next coordinates and check duplication.
@@ -1076,7 +1076,7 @@ Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
           b.setInsertionPointToStart(ifInBound.elseBlock());
           YIELD(constantI1(b, l, false));
         }
-        b.create<scf::ConditionOp>(l, ifInBound.getResults()[0], ivs);
+        scf::ConditionOp::create(b, l, ifInBound.getResults()[0], ivs);
       },
       /*afterBuilder=*/
       [](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1137,7 +1137,7 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
 
   SmallVector<Value> whileArgs(getCursor().begin(), getCursor().end());
   whileArgs.push_back(isFirst);
-  auto whileOp = b.create<scf::WhileOp>(
+  auto whileOp = scf::WhileOp::create(b,
       l, ValueRange(whileArgs).getTypes(), whileArgs,
       /*beforeBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1154,7 +1154,7 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
                              ret = ORI(ret, llvm::getSingleElement(isFirst));
                              return {ret};
                            });
-        b.create<scf::ConditionOp>(l, cont.front(), ivs);
+        scf::ConditionOp::create(b, l, cont.front(), ivs);
       },
       /*afterBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1219,7 +1219,7 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
     SmallVector<Value> iterArgs;
     iterArgs.push_back(C_IDX(0));
     iterArgs.append(reduc.begin(), reduc.end());
-    auto forEachLeaf = b.create<scf::ForOp>(
+    auto forEachLeaf = scf::ForOp::create(b,
         l, /*lb=*/C_IDX(0), /*ub=*/tupleCnt, /*step=*/C_IDX(1), iterArgs,
         [&helper, &builder](OpBuilder &b, Location l, Value tupleId,
                             ValueRange iterArgs) {
@@ -1235,12 +1235,12 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
           SmallVector<Value> whileArgs(helper.wrap.getCursor());
           whileArgs.append(iterArgs.begin(), iterArgs.end());
 
-          auto whileOp = b.create<scf::WhileOp>(
+          auto whileOp = scf::WhileOp::create(b,
               l, ValueRange(whileArgs).getTypes(), whileArgs,
               /*beforeBuilder=*/
               [&helper](OpBuilder &b, Location l, ValueRange ivs) {
                 helper.wrap.linkNewScope(ivs);
-                b.create<scf::ConditionOp>(l, helper.genNotEnd(b, l), ivs);
+                scf::ConditionOp::create(b, l, helper.genNotEnd(b, l), ivs);
               },
               /*afterBuilder=*/
               [&helper, &builder](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1267,7 +1267,7 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
                                      ValueRange reduc) {
     assert(!parent || parent->lvl + 1 == lvl);
     delegate->genInit(b, l, parent);
-    auto forOp = b.create<scf::ForOp>(
+    auto forOp = scf::ForOp::create(b,
         l, /*lb=*/C_IDX(0), /*ub=*/subSectSz, /*step=*/C_IDX(1), reduc,
         [&](OpBuilder &b, Location l, Value crd, ValueRange iterArgs) {
           helper.locate(b, l, crd);
@@ -1411,7 +1411,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
   // if (offset + size > parents.size)
   //   isNonEmpty = false;
   Value fastPathP = CMPI(ugt, getMinCrd(), getAbsOff());
-  auto ifOp = b.create<scf::IfOp>(l, getCursor().getTypes(), fastPathP, true);
+  auto ifOp = scf::IfOp::create(b, l, getCursor().getTypes(), fastPathP, true);
   {
     OpBuilder::InsertionGuard guard(b);
     // Take the fast path
@@ -1448,7 +1448,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 Value isMin = CMPI(eq, crd, getMinCrd());
                 delegate->forwardIf(b, l, isMin);
                 // Update the forwarded iterator values if needed.
-                auto ifIsMin = b.create<scf::IfOp>(l, isMin, false);
+                auto ifIsMin = scf::IfOp::create(b, l, isMin, false);
                 b.setInsertionPointToStart(&ifIsMin.getThenRegion().front());
                 storeCursorVals(b, l, tupleId, delegate->serialize());
                 b.setInsertionPointAfter(ifIsMin);
@@ -1458,7 +1458,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 return genWhenInBound(b, l, *delegate, /*elseRet=*/iterArgs,
                                       [nxMin](OpBuilder &b, Location l,
                                               Value crd) -> scf::ValueVector {
-                                        Value nx = b.create<arith::MinUIOp>(
+                                        Value nx = arith::MinUIOp::create(b,
                                             l, crd, nxMin);
                                         return {nx, C_TRUE};
                                       });
@@ -1480,7 +1480,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
 
   // We should at least forward the offset by one.
   Value minAbsOff = ADDI(getAbsOff(), c1);
-  nxAbsOff = b.create<arith::MaxUIOp>(l, minAbsOff, nxAbsOff);
+  nxAbsOff = arith::MaxUIOp::create(b, l, minAbsOff, nxAbsOff);
 
   seek(ValueRange{nxMinCrd, nxAbsOff, nxNotEnd});
   // The coordinate should not exceeds the space upper bound.
@@ -1581,16 +1581,16 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &b, Location l, Value t,
   auto stt = getSparseTensorType(t);
 
   LevelType lt = stt.getLvlType(lvl);
-  Value sz = stt.hasEncoding() ? b.create<LvlOp>(l, t, lvl).getResult()
-                               : b.create<tensor::DimOp>(l, t, lvl).getResult();
+  Value sz = stt.hasEncoding() ? LvlOp::create(b, l, t, lvl).getResult()
+                               : tensor::DimOp::create(b, l, t, lvl).getResult();
 
   SmallVector<Value, 2> buffers;
   if (lt.isWithPosLT()) {
-    Value pos = b.create<ToPositionsOp>(l, t, lvl);
+    Value pos = ToPositionsOp::create(b, l, t, lvl);
     buffers.push_back(pos);
   }
   if (lt.isWithCrdLT()) {
-    Value pos = b.create<ToCoordinatesOp>(l, t, lvl);
+    Value pos = ToCoordinatesOp::create(b, l, t, lvl);
     buffers.push_back(pos);
   }
   return makeSparseTensorLevel(lt, sz, buffers, tid, lvl);
diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
index 0258f797143cb..fb68620c378e5 100644
--- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
@@ -1563,7 +1563,7 @@ static Value insertYieldOp(RewriterBase &rewriter, Location loc, Region &region,
   Block &clonedBlock = tmpRegion.front();
   YieldOp clonedYield = cast<YieldOp>(clonedBlock.getTerminator());
   // Merge cloned block and return yield value.
-  Operation *placeholder = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Operation *placeholder = arith::ConstantIndexOp::create(rewriter, loc, 0);
   rewriter.inlineBlockBefore(&tmpRegion.front(), placeholder, vals);
   Value val = clonedYield.getSingleResult();
   rewriter.eraseOp(clonedYield);
@@ -1603,16 +1603,16 @@ static Value buildRelu(RewriterBase &rewriter, Location loc, Value v0,
                        Attribute attr) {
   Type tp = v0.getType();
   auto zero =
-      rewriter.create<arith::ConstantOp>(loc, tp, rewriter.getZeroAttr(tp));
+      arith::ConstantOp::create(rewriter, loc, tp, rewriter.getZeroAttr(tp));
   Value cmp;
   if (isa<FloatType>(tp)) {
     auto pred = llvm::cast<arith::CmpFPredicateAttr>(attr);
-    cmp = rewriter.create<arith::CmpFOp>(loc, pred, v0, zero);
+    cmp = arith::CmpFOp::create(rewriter, loc, pred, v0, zero);
   } else {
     auto pred = llvm::cast<arith::CmpIPredicateAttr>(attr);
-    cmp = rewriter.create<arith::CmpIOp>(loc, pred, v0, zero);
+    cmp = arith::CmpIOp::create(rewriter, loc, pred, v0, zero);
   }
-  return rewriter.create<arith::SelectOp>(loc, cmp, v0, zero);
+  return arith::SelectOp::create(rewriter, loc, cmp, v0, zero);
 }
 
 Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
@@ -1627,128 +1627,128 @@ Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
     llvm_unreachable("unexpected non-op");
   // Unary operations.
   case TensorExp::Kind::kAbsF:
-    return rewriter.create<math::AbsFOp>(loc, v0);
+    return math::AbsFOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kAbsC: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::AbsOp>(loc, eltType, v0);
+    return complex::AbsOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kAbsI:
-    return rewriter.create<math::AbsIOp>(loc, v0);
+    return math::AbsIOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kCeilF:
-    return rewriter.create<math::CeilOp>(loc, v0);
+    return math::CeilOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kFloorF:
-    return rewriter.create<math::FloorOp>(loc, v0);
+    return math::FloorOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSqrtF:
-    return rewriter.create<math::SqrtOp>(loc, v0);
+    return math::SqrtOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSqrtC:
-    return rewriter.create<complex::SqrtOp>(loc, v0);
+    return complex::SqrtOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kExpm1F:
-    return rewriter.create<math::ExpM1Op>(loc, v0);
+    return math::ExpM1Op::create(rewriter, loc, v0);
   case TensorExp::Kind::kExpm1C:
-    return rewriter.create<complex::Expm1Op>(loc, v0);
+    return complex::Expm1Op::create(rewriter, loc, v0);
   case TensorExp::Kind::kLog1pF:
-    return rewriter.create<math::Log1pOp>(loc, v0);
+    return math::Log1pOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kLog1pC:
-    return rewriter.create<complex::Log1pOp>(loc, v0);
+    return complex::Log1pOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kRelu:
     return buildRelu(rewriter, loc, v0, expr.attr);
   case TensorExp::Kind::kSinF:
-    return rewriter.create<math::SinOp>(loc, v0);
+    return math::SinOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSinC:
-    return rewriter.create<complex::SinOp>(loc, v0);
+    return complex::SinOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kTanhF:
-    return rewriter.create<math::TanhOp>(loc, v0);
+    return math::TanhOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kTanhC:
-    return rewriter.create<complex::TanhOp>(loc, v0);
+    return complex::TanhOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegF:
-    return rewriter.create<arith::NegFOp>(loc, v0);
+    return arith::NegFOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegC:
-    return rewriter.create<complex::NegOp>(loc, v0);
+    return complex::NegOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegI: // no negi in std
-    return rewriter.create<arith::SubIOp>(
+    return arith::SubIOp::create(rewriter,
         loc,
-        rewriter.create<arith::ConstantOp>(loc, v0.getType(),
+        arith::ConstantOp::create(rewriter, loc, v0.getType(),
                                            rewriter.getZeroAttr(v0.getType())),
         v0);
   case TensorExp::Kind::kTruncF:
-    return rewriter.create<arith::TruncFOp>(loc, inferType(e, v0), v0);
+    return arith::TruncFOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kExtF:
-    return rewriter.create<arith::ExtFOp>(loc, inferType(e, v0), v0);
+    return arith::ExtFOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastFS:
-    return rewriter.create<arith::FPToSIOp>(loc, inferType(e, v0), v0);
+    return arith::FPToSIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastFU:
-    return rewriter.create<arith::FPToUIOp>(loc, inferType(e, v0), v0);
+    return arith::FPToUIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastSF:
-    return rewriter.create<arith::SIToFPOp>(loc, inferType(e, v0), v0);
+    return arith::SIToFPOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastUF:
-    return rewriter.create<arith::UIToFPOp>(loc, inferType(e, v0), v0);
+    return arith::UIToFPOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastS:
-    return rewriter.create<arith::ExtSIOp>(loc, inferType(e, v0), v0);
+    return arith::ExtSIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastU:
-    return rewriter.create<arith::ExtUIOp>(loc, inferType(e, v0), v0);
+    return arith::ExtUIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastIdx:
-    return rewriter.create<arith::IndexCastOp>(loc, inferType(e, v0), v0);
+    return arith::IndexCastOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kTruncI:
-    return rewriter.create<arith::TruncIOp>(loc, inferType(e, v0), v0);
+    return arith::TruncIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCIm: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::ImOp>(loc, eltType, v0);
+    return complex::ImOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kCRe: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::ReOp>(loc, eltType, v0);
+    return complex::ReOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kBitCast:
-    return rewriter.create<arith::BitcastOp>(loc, inferType(e, v0), v0);
+    return arith::BitcastOp::create(rewriter, loc, inferType(e, v0), v0);
   // Binary operations.
   case TensorExp::Kind::kMulF:
-    return rewriter.create<arith::MulFOp>(loc, v0, v1);
+    return arith::MulFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kMulC:
-    return rewriter.create<complex::MulOp>(loc, v0, v1);
+    return complex::MulOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kMulI:
-    return rewriter.create<arith::MulIOp>(loc, v0, v1);
+    return arith::MulIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivF:
-    return rewriter.create<arith::DivFOp>(loc, v0, v1);
+    return arith::DivFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivC:
-    return rewriter.create<complex::DivOp>(loc, v0, v1);
+    return complex::DivOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivS:
-    return rewriter.create<arith::DivSIOp>(loc, v0, v1);
+    return arith::DivSIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivU:
-    return rewriter.create<arith::DivUIOp>(loc, v0, v1);
+    return arith::DivUIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddF:
-    return rewriter.create<arith::AddFOp>(loc, v0, v1);
+    return arith::AddFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddC:
-    return rewriter.create<complex::AddOp>(loc, v0, v1);
+    return complex::AddOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddI:
-    return rewriter.create<arith::AddIOp>(loc, v0, v1);
+    return arith::AddIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubF:
-    return rewriter.create<arith::SubFOp>(loc, v0, v1);
+    return arith::SubFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubC:
-    return rewriter.create<complex::SubOp>(loc, v0, v1);
+    return complex::SubOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubI:
-    return rewriter.create<arith::SubIOp>(loc, v0, v1);
+    return arith::SubIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAndI:
-    return rewriter.create<arith::AndIOp>(loc, v0, v1);
+    return arith::AndIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kOrI:
-    return rewriter.create<arith::OrIOp>(loc, v0, v1);
+    return arith::OrIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kXorI:
-    return rewriter.create<arith::XOrIOp>(loc, v0, v1);
+    return arith::XOrIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShrS:
-    return rewriter.create<arith::ShRSIOp>(loc, v0, v1);
+    return arith::ShRSIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShrU:
-    return rewriter.create<arith::ShRUIOp>(loc, v0, v1);
+    return arith::ShRUIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShlI:
-    return rewriter.create<arith::ShLIOp>(loc, v0, v1);
+    return arith::ShLIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kCmpI: {
     auto predicate = llvm::cast<arith::CmpIPredicateAttr>(expr.attr);
-    return rewriter.create<arith::CmpIOp>(loc, predicate, v0, v1);
+    return arith::CmpIOp::create(rewriter, loc, predicate, v0, v1);
   }
   case TensorExp::Kind::kCmpF: {
     auto predicate = llvm::cast<arith::CmpFPredicateAttr>(expr.attr);
-    return rewriter.create<arith::CmpFOp>(loc, predicate, v0, v1);
+    return arith::CmpFOp::create(rewriter, loc, predicate, v0, v1);
   }
   case TensorExp::Kind::kBinaryBranch: // semi-ring ops with custom logic.
     return insertYieldOp(rewriter, loc, *expr.op->getBlock()->getParent(),
diff --git a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
index fc93f1c1c9220..5e6e70987f6bd 100644
--- a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
+++ b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
@@ -78,11 +78,11 @@ struct CreatorOpShardingInterface
         if (!oldType.isDynamicDim(i) && shardType.isDynamicDim(i)) {
           if (!newSharding) {
             newSharding =
-                builder.create<ShardingOp>(op->getLoc(), resultShardings[0]);
+                ShardingOp::create(builder, op->getLoc(), resultShardings[0]);
             device =
-                builder.create<mesh::ProcessMultiIndexOp>(op->getLoc(), mesh)
+                mesh::ProcessMultiIndexOp::create(builder, op->getLoc(), mesh)
                     .getResults();
-            shapeForDevice = builder.create<mesh::ShardShapeOp>(
+            shapeForDevice = mesh::ShardShapeOp::create(builder,
                 op->getLoc(), oldType.getShape(), spmdizedOperands,
                 newSharding->getResult(0), device);
           }
@@ -92,7 +92,7 @@ struct CreatorOpShardingInterface
           newOperands.emplace_back(spmdizedOperands[++currOldOprndNum]);
         }
       }
-      newOp = builder.create<OpTy>(op->getLoc(), shardType, newOperands);
+      newOp = OpTy::create(builder, op->getLoc(), shardType, newOperands);
       spmdizationMap.map(op->getResult(0), newOp->getResult(0));
     } else {
       // `clone` will populate the mapping of old to new results.
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index a3e863254405c..79bddee1e7b84 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -56,7 +56,7 @@ Operation *TensorDialect::materializeConstant(OpBuilder &builder,
   if (auto op = arith::ConstantOp::materialize(builder, value, type, loc))
     return op;
   if (complex::ConstantOp::isBuildableWith(value, type))
-    return builder.create<complex::ConstantOp>(loc, type,
+    return complex::ConstantOp::create(builder, loc, type,
                                                llvm::cast<ArrayAttr>(value));
   return nullptr;
 }
@@ -110,7 +110,7 @@ FailureOr<Value> tensor::getOrCreateDestination(OpBuilder &b, Location loc,
 
   // Create empty tensor.
   Value emptyTensor =
-      b.create<tensor::EmptyOp>(loc, mixedSizes, tensorType.getElementType());
+      tensor::EmptyOp::create(b, loc, mixedSizes, tensorType.getElementType());
   return emptyTensor;
 }
 
@@ -681,7 +681,7 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
     inputShapes.emplace_back(std::move(inputShape));
   }
 
-  Value replacement = builder.create<tensor::EmptyOp>(
+  Value replacement = tensor::EmptyOp::create(builder,
       loc, outputShape, getType().getElementType());
 
   int64_t rank = getType().getRank();
@@ -690,12 +690,12 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
   SmallVector<OpFoldResult> offsets(rank, zero);
   for (auto [index, input] : llvm::enumerate(getInputs())) {
     offsets[concatDim] = concatOffsets[index];
-    auto insertSlice = builder.create<tensor::InsertSliceOp>(
+    auto insertSlice = tensor::InsertSliceOp::create(builder,
         loc, input, replacement, offsets, inputShapes[index], strides);
     replacement = insertSlice.getResult();
   }
   if (replacement.getType() != getType()) {
-    replacement = builder.create<tensor::CastOp>(loc, getType(), replacement);
+    replacement = tensor::CastOp::create(builder, loc, getType(), replacement);
   }
   return SmallVector<Value>{replacement};
 }
@@ -726,7 +726,7 @@ ConcatOp::reifyResultShapes(OpBuilder &builder,
           builder.getIndexAttr(inferredResultType.getDimSize(i)));
     } else {
       reifiedReturnShapes[0][i] =
-          builder.create<tensor::DimOp>(init.getLoc(), init, i).getResult();
+          tensor::DimOp::create(builder, init.getLoc(), init, i).getResult();
     }
   }
 
@@ -826,7 +826,7 @@ struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
 
         // Use refined operand type and create cast from original operand.
         auto castOp =
-            rewriter.create<CastOp>(concatOp->getLoc(), inferredOperandType,
+            CastOp::create(rewriter, concatOp->getLoc(), inferredOperandType,
                                     concatOp.getOperand(operandIdx));
         rewriter.modifyOpInPlace(concatOp, [=, operandIdx = operandIdx] {
           concatOp->setOperand(operandIdx, castOp->getResult(0));
@@ -867,7 +867,7 @@ struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
       return failure();
     }
 
-    auto newConcatOp = rewriter.create<ConcatOp>(
+    auto newConcatOp = ConcatOp::create(rewriter,
         concatOp->getLoc(), inferredResultType, dim, concatOp->getOperands());
     rewriter.replaceOpWithNewOp<CastOp>(concatOp, concatOp.getResultType(),
                                         newConcatOp);
@@ -895,7 +895,7 @@ void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
 void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
                   int64_t index) {
   auto loc = result.location;
-  Value indexValue = builder.create<arith::ConstantIndexOp>(loc, index);
+  Value indexValue = arith::ConstantIndexOp::create(builder, loc, index);
   build(builder, result, source, indexValue);
 }
 
@@ -1039,10 +1039,10 @@ struct DimOfReshapeOp : public OpRewritePattern<DimOp> {
     rewriter.setInsertionPointAfter(dim);
     Location loc = dim.getLoc();
     Value extract =
-        rewriter.create<ExtractOp>(loc, reshape.getShape(), dim.getIndex());
+        ExtractOp::create(rewriter, loc, reshape.getShape(), dim.getIndex());
     if (extract.getType() != dim.getType())
       extract =
-          rewriter.create<arith::IndexCastOp>(loc, dim.getType(), extract);
+          arith::IndexCastOp::create(rewriter, loc, dim.getType(), extract);
     rewriter.replaceOp(dim, extract);
     return success();
   }
@@ -1153,7 +1153,7 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
     if (foldedTensorType == op.getType())
       return failure();
 
-    auto newOp = rewriter.create<EmptyOp>(op.getLoc(), foldedTensorType,
+    auto newOp = EmptyOp::create(rewriter, op.getLoc(), foldedTensorType,
                                           foldedDynamicSizes);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
@@ -1329,7 +1329,7 @@ struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
 
       SmallVector<int64_t> basis =
           llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-      auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+      auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
           extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
       llvm::append_range(sourceIndices, delinearize.getResults());
     }
@@ -1501,7 +1501,7 @@ struct ExtractElementFromIndexCast
 
     Type elementTy = getElementTypeOrSelf(indexCast.getIn());
 
-    auto newExtract = rewriter.create<tensor::ExtractOp>(
+    auto newExtract = tensor::ExtractOp::create(rewriter,
         loc, elementTy, indexCast.getIn(), extract.getIndices());
 
     rewriter.replaceOpWithNewOp<arith::IndexCastOp>(extract, extract.getType(),
@@ -1739,7 +1739,7 @@ struct StaticTensorGenerate : public OpRewritePattern<GenerateOp> {
 
     auto loc = generateOp.getLoc();
     auto newOp =
-        rewriter.create<GenerateOp>(loc, foldedTensorType, foldedDynamicSizes);
+        GenerateOp::create(rewriter, loc, foldedTensorType, foldedDynamicSizes);
     rewriter.inlineRegionBefore(generateOp.getBody(), newOp.getBody(),
                                 newOp.getBody().begin());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(generateOp,
@@ -2164,7 +2164,7 @@ struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
         collapseShapeOp.getSrcMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = rewriter.create<CollapseShapeOp>(
+      auto newOp = CollapseShapeOp::create(rewriter,
           collapseShapeOp.getLoc(), newResultType, castOp.getSource(),
           collapseShapeOp.getReassociation());
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
@@ -2243,9 +2243,9 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
         newInputShape, expandOp.getSrcType().getElementType());
     auto outputType = RankedTensorType::get(
         newOutputShape, expandOp.getSrcType().getElementType());
-    auto inputCast = rewriter.create<CastOp>(expandOp.getLoc(), inputType,
+    auto inputCast = CastOp::create(rewriter, expandOp.getLoc(), inputType,
                                              expandOp.getSrc());
-    auto newExpand = rewriter.create<ExpandShapeOp>(
+    auto newExpand = ExpandShapeOp::create(rewriter,
         expandOp.getLoc(), outputType, inputCast.getResult(),
         expandOp.getReassociationIndices(), outputOfr);
     rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
@@ -2558,7 +2558,7 @@ class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
 
     // Create folded extract.
     Location loc = sliceOp.getLoc();
-    Value newResult = rewriter.create<ExtractSliceOp>(
+    Value newResult = ExtractSliceOp::create(rewriter,
         loc, sliceOp.getType(), castOp.getSource(), sliceOp.getOffsets(),
         sliceOp.getSizes(), sliceOp.getStrides(), sliceOp.getStaticOffsets(),
         sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
@@ -2712,7 +2712,7 @@ struct SliceCanonicalizer {
                   ExtractSliceOp newOp) {
     Value replacement = newOp.getResult();
     if (replacement.getType() != op.getType())
-      replacement = rewriter.create<tensor::CastOp>(op.getLoc(), op.getType(),
+      replacement = tensor::CastOp::create(rewriter, op.getLoc(), op.getType(),
                                                     replacement);
     rewriter.replaceOp(op, replacement);
   }
@@ -2981,7 +2981,7 @@ class InsertSliceOpConstantArgumentFolder final
       // the parallel case.
       if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      toInsert = rewriter.create<tensor::CastOp>(insertSliceOp.getLoc(),
+      toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
                                                  sourceType, toInsert);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
@@ -3078,7 +3078,7 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     if (!sliceResult.isValid)
       return failure();
 
-    Operation *replacement = rewriter.create<InsertOpTy>(
+    Operation *replacement = InsertOpTy::create(rewriter,
         insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
         mixedSizes, insertSliceOp.getMixedStrides());
 
@@ -3086,7 +3086,7 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     bool isParallelInsert =
         std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
     if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
-      replacement = rewriter.create<tensor::CastOp>(insertSliceOp.getLoc(),
+      replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
                                                     insertSliceOp.getDestType(),
                                                     replacement->getResult(0));
     }
@@ -3157,7 +3157,7 @@ struct InsertSliceOpSourceCastInserter final
     // parallel case.
     if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
       rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-    Value cast = rewriter.create<tensor::CastOp>(
+    Value cast = tensor::CastOp::create(rewriter,
         insertSliceOp.getLoc(), newSrcType, insertSliceOp.getSource());
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, cast, insertSliceOp.getDest(),
@@ -3356,7 +3356,7 @@ void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
   // a guard to reset the insertion point of the builder after it is destroyed.
   OpBuilder::InsertionGuard guard(b);
   b.createBlock(region, region->end(), blockArgTypes, blockArgLocs);
-  b.create<tensor::YieldOp>(result.location, constantPadValue);
+  tensor::YieldOp::create(b, result.location, constantPadValue);
 }
 
 llvm::SmallBitVector PadOp::getPaddedDims() {
@@ -3410,7 +3410,7 @@ struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
         padTensorOp.getSourceMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = rewriter.create<PadOp>(
+      auto newOp = PadOp::create(rewriter,
           padTensorOp->getLoc(), newResultType, padTensorOp.getSource(),
           padTensorOp.getStaticLow(), padTensorOp.getStaticHigh(),
           padTensorOp.getLow(), padTensorOp.getHigh(), padTensorOp.getNofold(),
@@ -3442,7 +3442,7 @@ struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
                                             tensorCastOp.getDest().getType()))
       return failure();
 
-    auto replacementOp = rewriter.create<PadOp>(
+    auto replacementOp = PadOp::create(rewriter,
         padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
         padTensorOp.getSource(), padTensorOp.getStaticLow(),
         padTensorOp.getStaticHigh(), padTensorOp.getLow(),
@@ -3600,10 +3600,10 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
 
     // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
     // the two paddings in one step.
-    auto newSliceOp = rewriter.create<ExtractSliceOp>(
+    auto newSliceOp = ExtractSliceOp::create(rewriter,
         padOp.getLoc(), outerSliceOp.getSource(), newOffsets, newSizes,
         innerSliceOp.getMixedStrides());
-    auto newPadOp = rewriter.create<PadOp>(
+    auto newPadOp = PadOp::create(rewriter,
         padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
         padOp.getMixedLowPad(), newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
@@ -3700,7 +3700,7 @@ struct FoldStaticPadding : public OpRewritePattern<PadOp> {
     // Rewrite the op using the new static type.
     auto newResultType = RankedTensorType::get(
         newOutDims, padTensorOp.getType().getElementType());
-    auto newOp = rewriter.create<PadOp>(
+    auto newOp = PadOp::create(rewriter,
         padTensorOp->getLoc(), newResultType, input, staticLow, staticHigh,
         newLows, newHighs, padTensorOp.getNofold(),
         getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
@@ -3780,7 +3780,7 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
     SmallVector<OpFoldResult> newLowPad =
         addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         padOp.getLoc(), padOp.getResultType(), producerPad.getSource(),
         newLowPad, newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, tensor::PadOp::getAttributeNames()));
@@ -3806,7 +3806,7 @@ PadOp::reifyResultShapes(OpBuilder &b,
     }
     Location loc = getLoc();
     Value dim = b.createOrFold<tensor::DimOp>(
-        loc, getSource(), b.create<arith::ConstantIndexOp>(loc, i));
+        loc, getSource(), arith::ConstantIndexOp::create(b, loc, i));
 
     AffineExpr d0, d1, d2;
     bindDims(b.getContext(), d0, d1, d2);
@@ -4111,7 +4111,7 @@ struct FoldTensorCastProducerOp
     for (auto [oldResult, newResult] :
          llvm::zip(op->getResults(), newOp->getResults())) {
       if (newResult.getType() != oldResult.getType()) {
-        replacements.push_back(rewriter.create<tensor::CastOp>(
+        replacements.push_back(tensor::CastOp::create(rewriter,
             op->getLoc(), oldResult.getType(), newResult));
       } else {
         replacements.push_back(newResult);
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 92540bd56ecbc..e6cb8bdbc472a 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -211,13 +211,13 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     if (isZeroInteger(newLength)) {
       hasZeroLen = true;
     } else if (!hasZeroLen) {
-      Value check = b.create<arith::CmpIOp>(
+      Value check = arith::CmpIOp::create(b,
           loc, arith::CmpIPredicate::eq,
           getValueOrCreateConstantIndexOp(b, loc, newLength),
           getValueOrCreateConstantIndexOp(b, loc, zero));
       dynHasZeroLenCond =
           dynHasZeroLenCond
-              ? b.create<arith::OrIOp>(loc, check, dynHasZeroLenCond)
+              ? arith::OrIOp::create(b, loc, check, dynHasZeroLenCond)
               : check;
     }
 
@@ -241,7 +241,7 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   auto castResult = [&](Value val) -> Value {
     if (resultType == val.getType())
       return val;
-    return b.create<tensor::CastOp>(loc, resultType, val);
+    return tensor::CastOp::create(b, loc, resultType, val);
   };
 
   // In cases where the original data source is unused: Emit a GenerateOp and
@@ -249,10 +249,10 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // have a dimension of size 0, the semantics of which is unclear.)
   auto createGenerateOp = [&]() {
     // Create GenerateOp.
-    auto generateOp = b.create<tensor::GenerateOp>(
+    auto generateOp = tensor::GenerateOp::create(b,
         loc, resultType, dynDims,
         [&](OpBuilder &builder, Location gLoc, ValueRange indices) {
-          builder.create<tensor::YieldOp>(gLoc, padValue);
+          tensor::YieldOp::create(builder, gLoc, padValue);
         });
     return generateOp;
   };
@@ -261,9 +261,9 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // the result shape of the new SliceOp has a zero dimension.
   auto createPadOfExtractSlice = [&]() {
     // Create pad(extract_slice(x)).
-    auto newSliceOp = b.create<tensor::ExtractSliceOp>(
+    auto newSliceOp = tensor::ExtractSliceOp::create(b,
         loc, padOp.getSource(), newOffsets, newLengths, newStrides);
-    auto newPadOp = b.create<PadOp>(
+    auto newPadOp = PadOp::create(b,
         loc, Type(), newSliceOp, newLows, newHighs,
         /*nofold=*/padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
@@ -291,17 +291,17 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     Operation *thenOp;
     Operation *elseOp;
     Operation *sliceOp;
-    auto result = b.create<scf::IfOp>(
+    auto result = scf::IfOp::create(b,
         loc, dynHasZeroLenCond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
           thenOp = createGenerateOp();
-          b.create<scf::YieldOp>(loc, castResult(thenOp->getResult(0)));
+          scf::YieldOp::create(b, loc, castResult(thenOp->getResult(0)));
         },
         /*elseBuilder=*/
         [&](OpBuilder &b, Location loc) {
           std::tie(elseOp, sliceOp) = createPadOfExtractSlice();
-          b.create<scf::YieldOp>(loc, castResult(elseOp->getResult(0)));
+          scf::YieldOp::create(b, loc, castResult(elseOp->getResult(0)));
         });
     return TilingResult{
         {elseOp}, SmallVector<Value>(result->getResults()), {sliceOp}};
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index 723731b8bed61..ba258402b61b3 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -165,7 +165,7 @@ void transform::TypeConversionCastShapeDynamicDimsOp::
     if (!tensor::CastOp::areCastCompatible(input.getType(), resultType)) {
       return Value();
     }
-    return builder.create<tensor::CastOp>(loc, resultType, input).getResult();
+    return tensor::CastOp::create(builder, loc, resultType, input).getResult();
   });
   converter.addTargetMaterialization([](OpBuilder &builder, Type resultType,
                                         ValueRange inputs,
@@ -177,7 +177,7 @@ void transform::TypeConversionCastShapeDynamicDimsOp::
     if (!tensor::CastOp::areCastCompatible(input.getType(), resultType)) {
       return Value();
     }
-    return builder.create<tensor::CastOp>(loc, resultType, input).getResult();
+    return tensor::CastOp::create(builder, loc, resultType, input).getResult();
   });
 }
 
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index 829b2ab92ac24..381cea7b93e41 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -223,7 +223,7 @@ struct CollapseShapeOpInterface
           MemRefType::get(collapseShapeOp.getSrcType().getShape(),
                           collapseShapeOp.getSrcType().getElementType(),
                           AffineMap(), bufferType.getMemorySpace());
-      buffer = rewriter.create<bufferization::ToBufferOp>(
+      buffer = bufferization::ToBufferOp::create(rewriter,
           op->getLoc(), memrefType, *tensorAlloc);
     }
 
@@ -350,7 +350,7 @@ struct ExpandShapeOpInterface
     if (failed(buffer))
       return failure();
 
-    auto memrefExpandShape = rewriter.create<memref::ExpandShapeOp>(
+    auto memrefExpandShape = memref::ExpandShapeOp::create(rewriter,
         op->getLoc(), tensorResultType.getShape(), *buffer,
         expandShapeOp.getReassociationIndices(),
         expandShapeOp.getMixedOutputShape());
@@ -399,7 +399,7 @@ struct ExtractSliceOpInterface
         extractSliceOp.getResult(), options, state);
     if (failed(resultMemrefType))
       return failure();
-    Value subView = rewriter.create<memref::SubViewOp>(
+    Value subView = memref::SubViewOp::create(rewriter,
         loc, llvm::cast<MemRefType>(*resultMemrefType), *srcMemref,
         mixedOffsets, mixedSizes, mixedStrides);
 
@@ -470,7 +470,7 @@ static void createStores(RewriterBase &rewriter, Location loc, int dim,
   if (dim == static_cast<int>(shape.size()) - 1) {
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
-      rewriter.create<memref::StoreOp>(loc, *elementIt, buffer, indices);
+      memref::StoreOp::create(rewriter, loc, *elementIt, buffer, indices);
       ++elementIt;
     }
     return;
@@ -508,7 +508,7 @@ struct FromElementsOpInterface
         bufferization::getBufferType(*tensorAlloc, options, state);
     if (failed(memrefType))
       return failure();
-    Value buffer = rewriter.create<bufferization::ToBufferOp>(
+    Value buffer = bufferization::ToBufferOp::create(rewriter,
         op->getLoc(), *memrefType, *tensorAlloc);
 
     // Case: tensor<0xelem_type>.
@@ -519,7 +519,7 @@ struct FromElementsOpInterface
 
     // Case: tensor<elem_type>.
     if (shape.empty()) {
-      rewriter.create<memref::StoreOp>(
+      memref::StoreOp::create(rewriter,
           loc, fromElementsOp.getElements().front(), buffer);
       replaceOpWithBufferizedValues(rewriter, op, buffer);
       return success();
@@ -530,7 +530,7 @@ struct FromElementsOpInterface
     SmallVector<Value, 2> constants;
     constants.reserve(maxDim);
     for (int i = 0; i < maxDim; ++i)
-      constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
+      constants.push_back(arith::ConstantIndexOp::create(rewriter, loc, i));
 
     // Traverse all `elements` and create `memref.store` ops.
     auto elementIt = fromElementsOp.getElements().begin();
@@ -577,7 +577,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   // Create linalg::MapOp.
   OpBuilder::InsertionGuard g(rewriter);
   auto linalgOp =
-      rewriter.create<linalg::MapOp>(loc, tensorType, /*inputs=*/ValueRange(),
+      linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
                                      /*init=*/tensorDestination);
   Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
@@ -585,7 +585,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   rewriter.setInsertionPointToStart(&linalgBody);
   SmallVector<Value> indices;
   for (int64_t dim = 0; dim < tensorType.getRank(); ++dim)
-    indices.push_back(rewriter.create<linalg::IndexOp>(loc, dim));
+    indices.push_back(linalg::IndexOp::create(rewriter, loc, dim));
 
   // Move over body.
   rewriter.mergeBlocks(&generateBody.front(), &linalgBody, indices);
@@ -645,7 +645,7 @@ struct InsertOpInterface
         getBuffer(rewriter, insertOp.getDest(), options, state);
     if (failed(destMemref))
       return failure();
-    rewriter.create<memref::StoreOp>(insertOp.getLoc(), insertOp.getScalar(),
+    memref::StoreOp::create(rewriter, insertOp.getLoc(), insertOp.getScalar(),
                                      *destMemref, insertOp.getIndices());
     replaceOpWithBufferizedValues(rewriter, op, *destMemref);
     return success();
@@ -714,7 +714,7 @@ struct InsertSliceOpInterface
         memref::SubViewOp::inferRankReducedResultType(
             insertSliceOp.getSourceType().getShape(), dstMemrefType,
             mixedOffsets, mixedSizes, mixedStrides);
-    Value subView = rewriter.create<memref::SubViewOp>(
+    Value subView = memref::SubViewOp::create(rewriter,
         loc, subviewMemRefType, *dstMemref, mixedOffsets, mixedSizes,
         mixedStrides);
 
@@ -797,13 +797,13 @@ struct PadOpInterface
     for (int64_t i = 0; i < resultType.getRank(); ++i) {
       if (!resultType.isDynamicDim(i))
         continue;
-      Value srcDim = rewriter.create<tensor::DimOp>(loc, padOp.getSource(), i);
+      Value srcDim = tensor::DimOp::create(rewriter, loc, padOp.getSource(), i);
       Value lowPad = toValue(mixedLowPad[i]);
       Value highPad = toValue(mixedHighPad[i]);
       AffineExpr s0, s1, s2;
       bindSymbols(op->getContext(), s0, s1, s2);
       AffineExpr sumExpr = s0 + s1 + s2;
-      Value sum = rewriter.create<affine::AffineApplyOp>(
+      Value sum = affine::AffineApplyOp::create(rewriter,
           loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
       dynamicSizes.push_back(sum);
     }
@@ -996,7 +996,7 @@ struct ParallelInsertSliceOpInterface
             parallelInsertSliceOp.getMixedOffsets(),
             parallelInsertSliceOp.getMixedSizes(),
             parallelInsertSliceOp.getMixedStrides());
-    Value subview = rewriter.create<memref::SubViewOp>(
+    Value subview = memref::SubViewOp::create(rewriter,
         parallelInsertSliceOp.getLoc(), subviewMemRefType, *destBuffer,
         parallelInsertSliceOp.getMixedOffsets(),
         parallelInsertSliceOp.getMixedSizes(),
@@ -1067,13 +1067,13 @@ struct SplatOpInterface
       return op->emitError("memory space not implemented yet");
 
     auto linalgOp =
-        rewriter.create<linalg::MapOp>(loc, tensorType, /*inputs=*/ValueRange(),
+        linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
                                        /*init=*/*tensorAlloc);
     Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
     // Create linalg::IndexOps.
     rewriter.setInsertionPointToStart(&linalgBody);
-    rewriter.create<linalg::YieldOp>(loc, splatOp.getInput());
+    linalg::YieldOp::create(rewriter, loc, splatOp.getInput());
     rewriter.replaceOp(splatOp, linalgOp.getResult()[0]);
 
     return success();
@@ -1127,7 +1127,7 @@ struct ConcatOpInterface
     MemRefType memrefType =
         MemRefType::get(concatOp.getResultType().getShape(),
                         concatOp.getResultType().getElementType(), layout);
-    Value dstBuffer = rewriter.create<bufferization::ToBufferOp>(
+    Value dstBuffer = bufferization::ToBufferOp::create(rewriter,
         op->getLoc(), memrefType, *tensorAlloc);
 
     // Extract the dimension for the concat op
@@ -1143,7 +1143,7 @@ struct ConcatOpInterface
     for (const auto &[dimIdx, dimSize] :
          llvm::enumerate(tensorType.getShape())) {
       if (dimSize == ShapedType::kDynamic) {
-        auto dimOp = rewriter.create<memref::DimOp>(loc, dstBuffer, dimIdx);
+        auto dimOp = memref::DimOp::create(rewriter, loc, dstBuffer, dimIdx);
         sizes.push_back(dimOp.getResult());
         if (dimIdx == concatDim)
           dynamicConcatDim = true;
@@ -1158,7 +1158,7 @@ struct ConcatOpInterface
     if (dynamicConcatDim) {
       // One or more operands have dynamic size, so we must accumulate the
       // offset with arith ops.
-      dynamicOffset = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+      dynamicOffset = arith::ConstantIndexOp::create(rewriter, loc, 0);
     }
 
     for (auto operand : concatOp.getInputs()) {
@@ -1175,7 +1175,7 @@ struct ConcatOpInterface
 
       if (dynamicConcatDim) {
         offsets[concatDim] = dynamicOffset.value();
-        dynamicSize = rewriter.create<memref::DimOp>(loc, *srcBuffer, concatDim)
+        dynamicSize = memref::DimOp::create(rewriter, loc, *srcBuffer, concatDim)
                           .getResult();
         sizes[concatDim] = dynamicSize.value();
       } else {
@@ -1189,7 +1189,7 @@ struct ConcatOpInterface
           memref::SubViewOp::inferRankReducedResultType(
               operandTensorType.getShape(), dstMemrefType, offsets, sizes,
               strides);
-      Value subview = rewriter.create<memref::SubViewOp>(
+      Value subview = memref::SubViewOp::create(rewriter,
           loc, subviewMemRefType, dstBuffer, offsets, sizes, strides);
 
       // Copy the source buffer into the destination subview.
@@ -1197,7 +1197,7 @@ struct ConcatOpInterface
         return failure();
 
       if (dynamicConcatDim) {
-        dynamicOffset = rewriter.create<arith::AddIOp>(
+        dynamicOffset = arith::AddIOp::create(rewriter,
             loc, dynamicOffset.value(), dynamicSize.value());
       } else {
         concatDimOffset += operandConcatDimSize;
diff --git a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
index fa748cf01977f..5b28791126bd7 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
@@ -43,7 +43,7 @@ struct FoldEmptyTensorWithReshapeOp : public OpRewritePattern<ReshapeOp> {
 
     // Create new tensor.empty op.
     // TODO: Do not drop tensor type encoding.
-    Value emptyTensor = rewriter.create<EmptyOp>(
+    Value emptyTensor = EmptyOp::create(rewriter,
         loc, resultShapes[0], reshapeOp.getResultType().getElementType());
     if (emptyTensor.getType() != reshapeOp.getResultType()) {
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
index e0acaee9f6626..50c72c329ed95 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
@@ -55,7 +55,7 @@ static ValueRange invertCollapseShapeIndexing(
   for (int64_t i : reassociation[dim])
     basis.push_back(reshapeSourceShape[i]);
   auto delinearized =
-      b.create<AffineDelinearizeIndexOp>(loc, indexValue, basis);
+      AffineDelinearizeIndexOp::create(b, loc, indexValue, basis);
   return delinearized->getResults();
 }
 
@@ -144,14 +144,14 @@ tensor::ExtractSliceFromCollapseHelper::emitLoopNestBody(
   SmallVector<Range> extractParams =
       helper.getExtractSliceParams(builder.getContext(), multiIndices);
 
-  Value subTileResult = builder.create<tensor::ExtractSliceOp>(
+  Value subTileResult = tensor::ExtractSliceOp::create(builder,
       loc, collapseShapeOp.getSrc(), extractParams);
 
   SmallVector<Range> insertParams =
       helper.getInsertSliceParams(builder.getContext(), tileInductionVars);
 
   // Collapse the dimensions of the source slice back down.
-  Value collapsedResult = builder.create<tensor::CollapseShapeOp>(
+  Value collapsedResult = tensor::CollapseShapeOp::create(builder,
       loc, subTileResult, reassociationIndices);
   return std::make_pair(collapsedResult, insertParams);
 }
@@ -175,7 +175,7 @@ tensor::simplifyCollapseShapeWithRankReducingExtractSlice(
   SmallVector<OpFoldResult> sizes =
       tensor::getMixedSizes(rewriter, op.getLoc(), op.getSrc());
   SmallVector<OpFoldResult> strides(sourceType.getRank(), one);
-  auto sliceOp = rewriter.create<tensor::ExtractSliceOp>(
+  auto sliceOp = tensor::ExtractSliceOp::create(rewriter,
       op.getLoc(), info->sliceResultType, op.getSrc(), offsets, sizes, strides);
 
   if (!info->newReassociationIndices.has_value()) {
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
index a787b485f7162..fd40ece63276e 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
@@ -115,7 +115,7 @@ TransferReadOfExtractSliceOpFolder::matchAndRewriteMaskableOp(
       extractSliceOp.getMixedStrides(), extractSliceOp.getDroppedDims(),
       indices, sourceIndices);
 
-  Operation *newOp = rewriter.create<vector::TransferReadOp>(
+  Operation *newOp = vector::TransferReadOp::create(rewriter,
       readOp.getLoc(), readOp.getVectorType(), extractSliceOp.getSource(),
       sourceIndices,
       AffineMapAttr::get(expandDimsToRank(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
index 4655fa3cf0d23..e3a2854038bdd 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
@@ -65,7 +65,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     return padOp.getResult();
 
   // Create a new tensor::PadOp.
-  auto newPadOp = b.create<PadOp>(
+  auto newPadOp = PadOp::create(b,
       loc, padOp.getResultType(), padOp.getSource(), newMixedLow, newMixedHigh,
       constantPadding, padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
 
@@ -84,7 +84,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
       offsets.push_back(b.getIndexAttr(0));
     } else {
       offsets.push_back(
-          b.create<affine::AffineApplyOp>(
+          affine::AffineApplyOp::create(b,
                loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
                std::initializer_list<Value>{cast<Value>(newMixedLow[i]),
                                             cast<Value>(prevLow)})
@@ -100,7 +100,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     strides.push_back(b.getIndexAttr(1));
   }
 
-  return b.create<ExtractSliceOp>(loc, newPadOp, offsets, sizes, strides)
+  return ExtractSliceOp::create(b, loc, newPadOp, offsets, sizes, strides)
       .getResult();
 }
 
@@ -125,7 +125,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b,
 
   // Create a new tensor::EmptyOp.
   Value newEmptyOp =
-      b.create<EmptyOp>(loc, newSizes, emptyOp.getType().getElementType());
+      EmptyOp::create(b, loc, newSizes, emptyOp.getType().getElementType());
 
   // Create a tensor::ExtractSliceOp.
   SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index 657624b817af2..247ea15640b39 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -77,7 +77,7 @@ struct FoldUnPaddingCollapseIntoExtract
       return rewriter.notifyMatchFailure(collapseShapeOp,
                                          "expected unpadding collapse");
 
-    Value unPaddedExtractSlice = rewriter.create<tensor::ExtractSliceOp>(
+    Value unPaddedExtractSlice = tensor::ExtractSliceOp::create(rewriter,
         extractSliceOp.getLoc(), collapseShapeOp.getResultType(),
         extractSliceOp.getSource(), extractSliceOp.getMixedOffsets(),
         extractSliceOp.getMixedSizes(), extractSliceOp.getMixedStrides());
@@ -271,7 +271,7 @@ struct BubbleUpExpandThroughParallelCollapse
     // matches the number of dimensions of the result, then the expand_shape
     // is a no-op.
     if (newExpandReInds.size() != newExpandSizes.size()) {
-      newCollapseSrc = rewriter.create<tensor::ExpandShapeOp>(
+      newCollapseSrc = tensor::ExpandShapeOp::create(rewriter,
           loc, expandResultType, newCollapseSrc, newExpandReInds,
           newExpandSizes);
     }
@@ -281,7 +281,7 @@ struct BubbleUpExpandThroughParallelCollapse
     // is a no-op.
     Value replacement = newCollapseSrc;
     if (newCollapseReInds.size() != newExpandSizes.size()) {
-      replacement = rewriter.create<tensor::CollapseShapeOp>(
+      replacement = tensor::CollapseShapeOp::create(rewriter,
           loc, newCollapseSrc, newCollapseReInds);
     }
     rewriter.replaceOp(expandOp, replacement);
@@ -406,7 +406,7 @@ struct BubbleUpExpandShapeThroughExtractSlice
         shape, expandShapeOp.getResultType().getElementType());
 
     // Create a new ExtractSliceOp and ExpandShapeOp.
-    Value newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
         collapsedStrides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -736,7 +736,7 @@ struct BubbleUpCollapseShapeThroughExtractSlice
                              groupExpandedOffsets.rend());
     }
 
-    Value newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         collapseShapeOp->getLoc(), collapseShapeOp.getSrc(), expandedOffsets,
         expandedSizes, expandedStrides);
     rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
index 7c9fced540adb..69e649d2eebe8 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
@@ -196,7 +196,7 @@ struct PadOpToConstant final : public OpRewritePattern<PadOp> {
                                          "tensor type not supported");
 
     if (newOp.getType() != resultType)
-      newOp = rewriter.create<tensor::CastOp>(loc, resultType, newOp);
+      newOp = tensor::CastOp::create(rewriter, loc, resultType, newOp);
 
     rewriter.replaceOp(padTensorOp, newOp);
     return success();
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
index 6138821ee8c61..8522c3ed28745 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
@@ -48,12 +48,12 @@ struct CastOpInterface
 
     if (isa<UnrankedTensorType>(srcType)) {
       // Check rank.
-      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
-          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
-      Value isSameRank = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
+      Value isSameRank = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameRank,
           RuntimeVerifiableOpInterface::generateErrorMessage(op,
                                                              "rank mismatch"));
@@ -71,12 +71,12 @@ struct CastOpInterface
         continue;
 
       Value srcDimSz =
-          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+          DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameSz = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameSz = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
@@ -90,9 +90,9 @@ struct DimOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto dimOp = cast<DimOp>(op);
-    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    builder.create<cf::AssertOp>(
+    Value rank = RankOp::create(builder, loc, dimOp.getSource());
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    cf::AssertOp::create(builder,
         loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
@@ -125,7 +125,7 @@ struct ExtractInsertOpInterface
     }
 
     auto indices = extractInsertOp.getIndices();
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     Value assertCond;
     for (auto i : llvm::seq<int64_t>(0, rank)) {
       Value dimOp = builder.createOrFold<tensor::DimOp>(loc, tensor, i);
@@ -135,7 +135,7 @@ struct ExtractInsertOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    builder.create<cf::AssertOp>(
+    cf::AssertOp::create(builder,
         loc, assertCond,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "out-of-bounds access"));
@@ -153,8 +153,8 @@ struct ExtractSliceOpInterface
     // For each dimension, assert that:
     // 0 <= offset < dim_size
     // 0 <= offset + (size - 1) * stride < dim_size
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value one = arith::ConstantIndexOp::create(builder, loc, 1);
     for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
       Value offset = getValueOrCreateConstantIndexOp(
           builder, loc, extractSliceOp.getMixedOffsets()[i]);
@@ -168,20 +168,20 @@ struct ExtractSliceOpInterface
           loc, extractSliceOp.getSource(), i);
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
       // Verify that slice does not run out-of-bounds.
-      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOne = arith::SubIOp::create(builder, loc, size, one);
       Value sizeMinusOneTimesStride =
-          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+          arith::MulIOp::create(builder, loc, sizeMinusOne, stride);
       Value lastPos =
-          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+          arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "extract_slice runs out-of-bounds along dimension " +
diff --git a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
index d50d7c62b789c..21f70e52b8706 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
@@ -59,7 +59,7 @@ struct InsertSliceLikeOpSubsetInsertionOpInterface
   Value buildSubsetExtraction(Operation *op, OpBuilder &builder,
                               Location loc) const {
     auto insertSliceOp = cast<OpTy>(op);
-    auto extractOp = builder.create<tensor::ExtractSliceOp>(
+    auto extractOp = tensor::ExtractSliceOp::create(builder,
         loc, insertSliceOp.getSourceType(), insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
         insertSliceOp.getMixedStrides());
diff --git a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
index 289296a07d9d3..728feab6a6d13 100644
--- a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
@@ -58,7 +58,7 @@ PadOp mlir::tensor::createPadHighOp(RankedTensorType resType, Value source,
     high[idx] = affine::makeComposedFoldedAffineApply(b, loc, d0 - d1,
                                                       {outDim, sourceDim});
   }
-  return b.create<PadOp>(loc, resType, source, low, high, pad, nofold);
+  return PadOp::create(b, loc, resType, source, low, high, pad, nofold);
 }
 
 SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
@@ -69,7 +69,7 @@ SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
   for (const auto &en : llvm::enumerate(tensorTy.getShape())) {
     if (en.value() == ShapedType::kDynamic)
       dynamicDims.push_back(
-          b.create<tensor::DimOp>(loc, rankedTensor, en.index()));
+          tensor::DimOp::create(b, loc, rankedTensor, en.index()));
   }
   return dynamicDims;
 }
@@ -121,7 +121,7 @@ mlir::tensor::dropGivenUnitDims(OpBuilder &b, Location loc, Value src,
     reassocMaps.emplace_back(llvm::make_range(seq.begin(), seq.end()));
     nextDimToGroup = setBit + 1;
   }
-  return b.create<tensor::CollapseShapeOp>(loc, src, reassocMaps);
+  return tensor::CollapseShapeOp::create(b, loc, src, reassocMaps);
 }
 
 bool mlir::tensor::isCastLikeInsertSliceOp(InsertSliceOp op) {
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 1d21096e8920b..477d35fc8d9c5 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -851,7 +851,7 @@ struct PadSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
         getTosaConstShape(rewriter, sliceOp.getLoc(), newPadPaddings);
     auto newPadTy =
         RankedTensorType::get(newPadShape, inputTy.getElementType());
-    auto newPadOp = rewriter.create<tosa::PadOp>(
+    auto newPadOp = tosa::PadOp::create(rewriter,
         padOp.getLoc(), newPadTy, padOp.getInput1(), newPaddingsOp,
         padOp.getPadConst());
 
@@ -903,7 +903,7 @@ struct SliceDynamicSizeCanonicalization
     }
 
     auto size_op = getTosaConstShape(rewriter, sliceOp.getLoc(), sliceSizes);
-    auto newSliceOp = rewriter.create<tosa::SliceOp>(
+    auto newSliceOp = tosa::SliceOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getType(), sliceOp.getInput1(),
         sliceOp.getStart(), size_op);
 
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 5170a11523845..48ae70c51c279 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -182,11 +182,11 @@ Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // Tosa dialect constants only support ElementsAttr unlike standard dialect
   // constant which supports all attributes.
   if (llvm::isa<shapeType>(type) && llvm::isa<DenseIntElementsAttr>(value)) {
-    return builder.create<tosa::ConstShapeOp>(
+    return tosa::ConstShapeOp::create(builder,
         loc, type, llvm::cast<DenseIntElementsAttr>(value));
   }
   if (llvm::isa<ElementsAttr>(value))
-    return builder.create<tosa::ConstOp>(loc, type,
+    return tosa::ConstOp::create(builder, loc, type,
                                          llvm::cast<ElementsAttr>(value));
   return nullptr;
 }
@@ -325,7 +325,7 @@ Value mlir::tosa::createPadConstTensor(OpBuilder &builder, Location loc,
                                    builder.getFloatAttr(srcElemType, val))
           : DenseElementsAttr::get(padConstEType,
                                    builder.getIntegerAttr(srcElemType, val))};
-  return builder.create<tosa::ConstOp>(loc, padConstType, padConstAttr);
+  return tosa::ConstOp::create(builder, loc, padConstType, padConstAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2417,7 +2417,7 @@ LogicalResult TransposeOp::reifyResultShapes(
     int32_t dimInInput = transposePerms[dim];
     if (inputType.isDynamicDim(dimInInput))
       returnedDims[dim] =
-          builder.create<tensor::DimOp>(getLoc(), input, dimInInput)
+          tensor::DimOp::create(builder, getLoc(), input, dimInInput)
               .getResult();
     else
       returnedDims[dim] =
@@ -3948,12 +3948,12 @@ std::optional<Value> mlir::tosa::createZeroPointTensor(OpBuilder &builder,
   if (llvm::isa<FloatType>(srcElemType)) {
     auto zpAttr = DenseElementsAttr::get(
         zpType, builder.getFloatAttr(srcElemType, static_cast<double>(zp)));
-    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+    return tosa::ConstOp::create(builder, loc, zpType, zpAttr);
   }
   if (llvm::isa<IntegerType>(srcElemType)) {
     auto zpAttr =
         DenseElementsAttr::get(zpType, builder.getIntegerAttr(srcElemType, zp));
-    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+    return tosa::ConstOp::create(builder, loc, zpType, zpAttr);
   }
   llvm::errs() << "zero point is not allowed for unsupported data types\n";
   return std::nullopt;
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
index 9b4cf85c480d3..72e3c0590183c 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
@@ -91,12 +91,12 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
 
     if (inputETy != resultETy) {
       inputType = inputType.clone(resultETy);
-      input = rewriter.create<tosa::CastOp>(op.getLoc(), inputType, input);
+      input = tosa::CastOp::create(rewriter, op.getLoc(), inputType, input);
     }
 
     if (weightETy != resultETy) {
       weightType = weightType.clone(resultETy);
-      weight = rewriter.create<tosa::CastOp>(op.getLoc(), weightType, weight);
+      weight = tosa::CastOp::create(rewriter, op.getLoc(), weightType, weight);
     }
 
     if (iZp != 0 || wZp != 0) {
@@ -110,8 +110,8 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         auto zpTy = RankedTensorType::get(shape, ety);
         auto zpAttr =
             DenseElementsAttr::get(zpTy, rewriter.getIntegerAttr(ety, zp));
-        auto zpVal = rewriter.create<tosa::ConstOp>(op.getLoc(), zpTy, zpAttr);
-        return rewriter.create<tosa::SubOp>(op.getLoc(), val.getType(), val,
+        auto zpVal = tosa::ConstOp::create(rewriter, op.getLoc(), zpTy, zpAttr);
+        return tosa::SubOp::create(rewriter, op.getLoc(), val.getType(), val,
                                             zpVal);
       };
 
@@ -139,9 +139,9 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
       auto padTy = RankedTensorType::get({1}, inputETy);
       auto padAttr = DenseElementsAttr::get(padTy, zeroAttr);
       Value padVal =
-          rewriter.create<tosa::ConstOp>(op->getLoc(), padTy, padAttr);
+          tosa::ConstOp::create(rewriter, op->getLoc(), padTy, padAttr);
       inputType = RankedTensorType::get(newShape, inputETy);
-      input = rewriter.create<tosa::PadOp>(op->getLoc(), inputType, input,
+      input = tosa::PadOp::create(rewriter, op->getLoc(), inputType, input,
                                            padSizeVal, padVal);
     }
 
@@ -162,7 +162,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
     auto shiftZeroAttr = DenseElementsAttr::get(
         shiftType, rewriter.getIntegerAttr(shiftElementType, 0));
     Value constZero =
-        rewriter.create<tosa::ConstOp>(op.getLoc(), shiftType, shiftZeroAttr);
+        tosa::ConstOp::create(rewriter, op.getLoc(), shiftType, shiftZeroAttr);
     Value mulValue = rewriter
                          .create<tosa::MulOp>(op.getLoc(), mulShapeType, input,
                                               weight, constZero)
@@ -175,7 +175,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         dyn_cast<RankedTensorType>(input.getType()).getElementType());
     auto outputShapeValue =
         getTosaConstShape(rewriter, op->getLoc(), outputShape);
-    Value outputValue = rewriter.create<tosa::ReshapeOp>(
+    Value outputValue = tosa::ReshapeOp::create(rewriter,
         op.getLoc(), outputShapeType, mulValue, outputShapeValue);
 
     Value bias = op.getBias();
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
index ea6ac981b53cc..12af0ce56a8bc 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
@@ -64,12 +64,12 @@ class TransposeConvNonStridedConverter
     convPad[2] = kernelWidth - 1 + pad[2];
     convPad[3] = kernelWidth - 1 + pad[3];
 
-    auto reverse1 = rewriter.create<tosa::ReverseOp>(
+    auto reverse1 = tosa::ReverseOp::create(rewriter,
         loc, weightTy, weight, /* axis = */ rewriter.getI32IntegerAttr(1));
-    auto reverse2 = rewriter.create<tosa::ReverseOp>(
+    auto reverse2 = tosa::ReverseOp::create(rewriter,
         loc, weightTy, reverse1, /* axis = */ rewriter.getI32IntegerAttr(2));
 
-    Value conv2d = rewriter.create<tosa::Conv2DOp>(
+    Value conv2d = tosa::Conv2DOp::create(rewriter,
         loc, resultTy, input, reverse2, bias, op.getInputZp(), op.getWeightZp(),
         rewriter.getDenseI64ArrayAttr(convPad),
         rewriter.getDenseI64ArrayAttr(stride),
@@ -218,7 +218,7 @@ class TransposeConvStridedConverter
         inputPaddingVal, inputPadConst);
 
     // We use a zero bias as we need to broadcast the bias.
-    auto zeroBias = rewriter.create<tosa::ConstOp>(
+    auto zeroBias = tosa::ConstOp::create(rewriter,
         loc,
         RankedTensorType::get({outputChannels * stride[0] * stride[1]},
                               biasETy),
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index 29ec9f8db2615..9a648015d23df 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -114,7 +114,7 @@ class TypeModificationState {
           OpBuilder builder{value.getContext()};
           builder.setInsertionPointAfter(value.getDefiningOp());
           castValue =
-              builder.create<tensor::CastOp>(value.getLoc(), oldType, value);
+              tensor::CastOp::create(builder, value.getLoc(), oldType, value);
         }
 
         use->set(castValue);
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index 7f85cd52f6bde..51a093bd0cc38 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -405,7 +405,7 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
     return std::nullopt;
   }
   ImplicitLocOpBuilder builder(reshapeOp.getLoc(), rewriter);
-  auto foldedReshape = rewriter.create<ReshapeOp>(
+  auto foldedReshape = ReshapeOp::create(rewriter,
       reshapeOp.getLoc(),
       RankedTensorType::get(applyTOSAPermutation(shape, hoistedPerms),
                             reshapeOutputType.getElementType()),
@@ -425,7 +425,7 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
   if (!maybeNewDenseAttr.has_value())
     return std::nullopt;
   auto newDenseAttr = maybeNewDenseAttr.value();
-  auto newConstOp = rewriter.create<ConstOp>(
+  auto newConstOp = ConstOp::create(rewriter,
       constOp.getLoc(), newDenseAttr.getType(), newDenseAttr);
   return newConstOp->getResult(0);
 }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
index 3b697a2ee3e47..677d8e9904a67 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
@@ -37,7 +37,7 @@ void mlir::tosa::populateTosaTypeConversion(TypeConverter &converter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
   converter.addTargetMaterialization([&](OpBuilder &builder, Type resultType,
@@ -46,7 +46,7 @@ void mlir::tosa::populateTosaTypeConversion(TypeConverter &converter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
 }
diff --git a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
index 9844abcc34cb1..69ef0ba40d72e 100644
--- a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
@@ -33,18 +33,18 @@ mlir::tosa::condenseValues(const SmallVector<Value> &values) {
 
 Value mlir::tosa::clampFloatHelper(Location loc, Value arg, Value min,
                                    Value max, OpBuilder &rewriter) {
-  Value minValue = rewriter.create<arith::MinimumFOp>(loc, arg, max);
-  return rewriter.create<arith::MaximumFOp>(loc, minValue, min);
+  Value minValue = arith::MinimumFOp::create(rewriter, loc, arg, max);
+  return arith::MaximumFOp::create(rewriter, loc, minValue, min);
 }
 
 Value mlir::tosa::clampIntHelper(Location loc, Value arg, Value min, Value max,
                                  OpBuilder &rewriter, bool isUnsigned) {
   if (isUnsigned) {
-    auto minOrArg = rewriter.create<arith::MaxUIOp>(loc, min, arg);
-    return rewriter.create<arith::MinUIOp>(loc, max, minOrArg);
+    auto minOrArg = arith::MaxUIOp::create(rewriter, loc, min, arg);
+    return arith::MinUIOp::create(rewriter, loc, max, minOrArg);
   }
-  auto minOrArg = rewriter.create<arith::MaxSIOp>(loc, min, arg);
-  return rewriter.create<arith::MinSIOp>(loc, max, minOrArg);
+  auto minOrArg = arith::MaxSIOp::create(rewriter, loc, min, arg);
+  return arith::MinSIOp::create(rewriter, loc, max, minOrArg);
 }
 
 bool mlir::tosa::validIntegerRange(IntegerType ty, int64_t value) {
@@ -144,7 +144,7 @@ LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder,
       ArrayRef<int64_t>(reshapeOutputShape), reshapeInputType.getElementType());
   auto reshapeOutputShapeValue = getTosaConstShape(builder, reshapeOutputShape);
 
-  auto reshapeLower = builder.create<tosa::ReshapeOp>(
+  auto reshapeLower = tosa::ReshapeOp::create(builder,
       reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
 
   if (input1Rank > input2Rank) {
@@ -162,7 +162,7 @@ Value mlir::tosa::getTosaConstShape(ImplicitLocOpBuilder &builder,
                                     llvm::ArrayRef<int64_t> shape) {
   auto attr = builder.getIndexTensorAttr(convertFromMlirShape(shape));
   auto type = mlir::tosa::shapeType::get(builder.getContext(), shape.size());
-  mlir::Operation *mlir_op = builder.create<tosa::ConstShapeOp>(type, attr);
+  mlir::Operation *mlir_op = tosa::ConstShapeOp::create(builder, type, attr);
   return mlir_op->getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/UB/IR/UBOps.cpp b/mlir/lib/Dialect/UB/IR/UBOps.cpp
index 5b2cfe7bf4264..ee523f9522953 100644
--- a/mlir/lib/Dialect/UB/IR/UBOps.cpp
+++ b/mlir/lib/Dialect/UB/IR/UBOps.cpp
@@ -52,7 +52,7 @@ void UBDialect::initialize() {
 Operation *UBDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                           Type type, Location loc) {
   if (auto attr = dyn_cast<PoisonAttr>(value))
-    return builder.create<PoisonOp>(loc, type, attr);
+    return PoisonOp::create(builder, loc, type, attr);
 
   return nullptr;
 }
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 14e626a6b23e3..e32b2ca8d76b4 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -125,7 +125,7 @@ static MaskFormat getMaskFormat(Value mask) {
 /// Default callback to build a region with a 'vector.yield' terminator with no
 /// arguments.
 void mlir::vector::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<vector::YieldOp>(loc);
+  vector::YieldOp::create(builder, loc);
 }
 
 // Helper for verifying combining kinds in contractions and reductions.
@@ -597,16 +597,16 @@ struct ElideUnitDimsInMultiDimReduction
         VectorType newMaskType =
             VectorType::get(dstVecType.getShape(), rewriter.getI1Type(),
                             dstVecType.getScalableDims());
-        mask = rewriter.create<vector::ShapeCastOp>(loc, newMaskType, mask);
+        mask = vector::ShapeCastOp::create(rewriter, loc, newMaskType, mask);
       }
-      cast = rewriter.create<vector::ShapeCastOp>(
+      cast = vector::ShapeCastOp::create(rewriter,
           loc, reductionOp.getDestType(), reductionOp.getSource());
     } else {
       // This means we are reducing all the dimensions, and all reduction
       // dimensions are of size 1. So a simple extraction would do.
       if (mask)
-        mask = rewriter.create<vector::ExtractOp>(loc, mask);
-      cast = rewriter.create<vector::ExtractOp>(loc, reductionOp.getSource());
+        mask = vector::ExtractOp::create(rewriter, loc, mask);
+      cast = vector::ExtractOp::create(rewriter, loc, reductionOp.getSource());
     }
 
     Value result =
@@ -673,35 +673,35 @@ Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
   switch (op) {
   case arith::AtomicRMWKind::addf:
   case arith::AtomicRMWKind::addi:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::ADD, vector);
   case arith::AtomicRMWKind::mulf:
   case arith::AtomicRMWKind::muli:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MUL, vector);
   case arith::AtomicRMWKind::minimumf:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINIMUMF, vector);
   case arith::AtomicRMWKind::mins:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINSI, vector);
   case arith::AtomicRMWKind::minu:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINUI, vector);
   case arith::AtomicRMWKind::maximumf:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXIMUMF, vector);
   case arith::AtomicRMWKind::maxs:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXSI, vector);
   case arith::AtomicRMWKind::maxu:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXUI, vector);
   case arith::AtomicRMWKind::andi:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::AND, vector);
   case arith::AtomicRMWKind::ori:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::OR, vector);
   // TODO: Add remaining reduction operations.
   default:
@@ -741,8 +741,8 @@ struct ElideSingleElementReduction : public OpRewritePattern<ReductionOp> {
 
     Location loc = reductionOp.getLoc();
     if (mask)
-      mask = rewriter.create<ExtractOp>(loc, mask);
-    Value result = rewriter.create<ExtractOp>(loc, reductionOp.getVector());
+      mask = ExtractOp::create(rewriter, loc, mask);
+    Value result = ExtractOp::create(rewriter, loc, reductionOp.getVector());
 
     if (Value acc = reductionOp.getAcc())
       result = vector::makeArithReduction(rewriter, loc, reductionOp.getKind(),
@@ -4171,7 +4171,7 @@ class StridedSliceBroadcast final
     // just a single scalar.
     bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
     if (!lowerDimMatch && !isScalarSrc) {
-      source = rewriter.create<ExtractStridedSliceOp>(
+      source = ExtractStridedSliceOp::create(rewriter,
           op->getLoc(), source,
           getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
@@ -4266,7 +4266,7 @@ class ContiguousExtractStridedSliceToExtract final
 
     SmallVector<int64_t> offsets = getI64SubArray(op.getOffsets());
     auto extractOffsets = ArrayRef(offsets).take_front(numOffsets);
-    Value extract = rewriter.create<vector::ExtractOp>(op->getLoc(), source,
+    Value extract = vector::ExtractOp::create(rewriter, op->getLoc(), source,
                                                        extractOffsets);
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(), extract);
     return success();
@@ -4297,7 +4297,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
 
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
         *padding, /*mask=*/Value(), inBoundsAttr);
 }
@@ -4315,7 +4315,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                                 SmallVector<bool>(vectorType.getRank(), false));
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, *padding,
         permutationMapAttr, inBoundsAttr);
 }
@@ -4334,7 +4334,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                                 SmallVector<bool>(vectorType.getRank(), false));
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
         *padding,
         /*mask=*/Value(), inBoundsAttr);
@@ -4859,7 +4859,7 @@ struct TransferReadAfterWriteToBroadcast
     VectorType broadcastedType = VectorType::get(
         broadcastShape, defWrite.getVectorType().getElementType(),
         broadcastScalableFlags);
-    vec = rewriter.create<vector::BroadcastOp>(loc, broadcastedType, vec);
+    vec = vector::BroadcastOp::create(rewriter, loc, broadcastedType, vec);
     SmallVector<int64_t> transposePerm(permutation.begin(), permutation.end());
     rewriter.replaceOpWithNewOp<vector::TransposeOp>(readOp, vec,
                                                      transposePerm);
@@ -5337,11 +5337,11 @@ struct SwapExtractSliceOfTransferWrite
     // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
     // Set all in_bounds to false and let the folder infer them.
     SmallVector<bool> newInBounds(vectorShape.size(), false);
-    auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
         extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
         insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
         insertOp.getMixedStrides());
-    auto newTransferWriteOp = rewriter.create<TransferWriteOp>(
+    auto newTransferWriteOp = TransferWriteOp::create(rewriter,
         transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
         transferOp.getIndices(), transferOp.getPermutationMapAttr(),
         rewriter.getBoolArrayAttr(newInBounds));
@@ -6867,7 +6867,7 @@ void MaskOp::ensureTerminator(Region &region, Builder &builder, Location loc) {
   OpBuilder opBuilder(builder.getContext());
   Operation *maskedOp = &block.front();
   opBuilder.setInsertionPointToEnd(&block);
-  opBuilder.create<vector::YieldOp>(loc, maskedOp->getResults());
+  vector::YieldOp::create(opBuilder, loc, maskedOp->getResults());
 }
 
 LogicalResult MaskOp::verify() {
@@ -7202,7 +7202,7 @@ void mlir::vector::createMaskOpRegion(OpBuilder &builder,
   // Create a block and move the op to that block.
   insBlock->getOperations().splice(
       insBlock->begin(), maskableOp->getBlock()->getOperations(), maskableOp);
-  builder.create<YieldOp>(maskableOp->getLoc(), maskableOp->getResults());
+  YieldOp::create(builder, maskableOp->getLoc(), maskableOp->getResults());
 }
 
 /// Creates a vector.mask operation around a maskable operation. Returns the
@@ -7214,10 +7214,10 @@ Operation *mlir::vector::maskOperation(OpBuilder &builder,
   if (!mask)
     return maskableOp;
   if (passthru)
-    return builder.create<MaskOp>(maskableOp->getLoc(),
+    return MaskOp::create(builder, maskableOp->getLoc(),
                                   maskableOp->getResultTypes(), mask, passthru,
                                   maskableOp, createMaskOpRegion);
-  return builder.create<MaskOp>(maskableOp->getLoc(),
+  return MaskOp::create(builder, maskableOp->getLoc(),
                                 maskableOp->getResultTypes(), mask, maskableOp,
                                 createMaskOpRegion);
 }
@@ -7234,7 +7234,7 @@ Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
   if (!mask)
     return newValue;
 
-  return builder.create<arith::SelectOp>(newValue.getLoc(), newValue.getType(),
+  return arith::SelectOp::create(builder, newValue.getLoc(), newValue.getType(),
                                          mask, newValue, passthru);
 }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
index 9da051150e409..c750017a6b504 100644
--- a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -116,7 +116,7 @@ struct TransferWriteOpInterface
         getBuffer(rewriter, writeOp.getBase(), options, state);
     if (failed(resultBuffer))
       return failure();
-    rewriter.create<vector::TransferWriteOp>(
+    vector::TransferWriteOp::create(rewriter,
         writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
         writeOp.getIndices(), writeOp.getPermutationMapAttr(),
         writeOp.getMask(), writeOp.getInBoundsAttr());
@@ -241,7 +241,7 @@ struct MaskOpInterface
     // Create a new vector.mask op.
     ValueRange newYieldedValuesRange(newYieldedValues);
     TypeRange newResultTypes(newYieldedValuesRange);
-    auto newOp = rewriter.create<vector::MaskOp>(
+    auto newOp = vector::MaskOp::create(rewriter,
         op->getLoc(), newResultTypes, maskOp.getMask(), maskOp.getPassthru(),
         /*maskableOp=*/nullptr,
         /*maskRegionBuilder=*/[](OpBuilder &b, Operation *) {});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
index 89930a6bd35fa..4c3a04cfb5bfa 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
@@ -64,14 +64,14 @@ class UnrollBitCastOp final : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(shape, resultType.getElementType(), scalableDims);
 
     Location loc = op.getLoc();
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
     for (auto position : *unrollIterator) {
       Value extract =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+          vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       Value bitcast =
-          rewriter.create<vector::BitCastOp>(loc, bitcastResType, extract);
+          vector::BitCastOp::create(rewriter, loc, bitcastResType, extract);
       result =
-          rewriter.create<vector::InsertOp>(loc, bitcast, result, position);
+          vector::InsertOp::create(rewriter, loc, bitcast, result, position);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
index 11dcfe421e0c4..cb8e566869cfd 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
@@ -52,7 +52,7 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
 
     // Stretching scalar inside vector (e.g. vector<1xf32>) can use splat.
     if (srcRank <= 1 && dstRank == 1) {
-      Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource());
+      Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource());
       rewriter.replaceOpWithNewOp<vector::SplatOp>(op, dstType, ext);
       return success();
     }
@@ -70,10 +70,10 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
       // Duplication.
       VectorType resType = VectorType::Builder(dstType).dropDim(0);
       Value bcst =
-          rewriter.create<vector::BroadcastOp>(loc, resType, op.getSource());
-      Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
+          vector::BroadcastOp::create(rewriter, loc, resType, op.getSource());
+      Value result = ub::PoisonOp::create(rewriter, loc, dstType);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
       rewriter.replaceOp(op, result);
       return success();
     }
@@ -111,13 +111,13 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType =
         VectorType::get(dstType.getShape().drop_front(), eltType,
                         dstType.getScalableDims().drop_front());
-    Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
+    Value result = ub::PoisonOp::create(rewriter, loc, dstType);
     if (m == 0) {
       // Stetch at start.
-      Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), 0);
-      Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
+      Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource(), 0);
+      Value bcst = vector::BroadcastOp::create(rewriter, loc, resType, ext);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
     } else {
       // Stetch not at start.
       if (dstType.getScalableDims()[0]) {
@@ -125,9 +125,9 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
         return failure();
       }
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d) {
-        Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), d);
-        Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource(), d);
+        Value bcst = vector::BroadcastOp::create(rewriter, loc, resType, ext);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
       }
     }
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
index c6627b5ec0d77..b2486c5cbbcf8 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
@@ -90,17 +90,17 @@ static Value reshapeLoad(Location loc, Value val, VectorType type,
 
   // At extraction dimension?
   if (index == 0)
-    return rewriter.create<vector::ExtractOp>(loc, val, pos);
+    return vector::ExtractOp::create(rewriter, loc, val, pos);
 
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   VectorType resType = VectorType::Builder(type).dropDim(index);
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, resType, rewriter.getZeroAttr(resType));
   for (int64_t d = 0, e = resType.getDimSize(0); d < e; d++) {
-    Value ext = rewriter.create<vector::ExtractOp>(loc, val, d);
+    Value ext = vector::ExtractOp::create(rewriter, loc, val, d);
     Value load = reshapeLoad(loc, ext, vType, index - 1, pos, rewriter);
-    result = rewriter.create<vector::InsertOp>(loc, load, result, d);
+    result = vector::InsertOp::create(rewriter, loc, load, result, d);
   }
   return result;
 }
@@ -115,15 +115,15 @@ static Value reshapeStore(Location loc, Value val, Value result,
     return val;
   // At insertion dimension?
   if (index == 0)
-    return rewriter.create<vector::InsertOp>(loc, val, result, pos);
+    return vector::InsertOp::create(rewriter, loc, val, result, pos);
 
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   for (int64_t d = 0, e = type.getDimSize(0); d < e; d++) {
-    Value ext = rewriter.create<vector::ExtractOp>(loc, result, d);
-    Value ins = rewriter.create<vector::ExtractOp>(loc, val, d);
+    Value ext = vector::ExtractOp::create(rewriter, loc, result, d);
+    Value ins = vector::ExtractOp::create(rewriter, loc, val, d);
     Value sto = reshapeStore(loc, ins, ext, vType, index - 1, pos, rewriter);
-    result = rewriter.create<vector::InsertOp>(loc, sto, result, d);
+    result = vector::InsertOp::create(rewriter, loc, sto, result, d);
   }
   return result;
 }
@@ -141,7 +141,7 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
         kind == CombiningKind::MINIMUMF || kind == CombiningKind::MAXIMUMF)
       // Only valid for floating point types.
       return std::nullopt;
-    mul = rewriter.create<arith::MulIOp>(loc, x, y);
+    mul = arith::MulIOp::create(rewriter, loc, x, y);
   } else {
     // Float case.
     if (kind == CombiningKind::AND || kind == CombiningKind::MINUI ||
@@ -152,14 +152,14 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
       return std::nullopt;
     // Special case for fused multiply-add.
     if (acc && isa<VectorType>(acc.getType()) && kind == CombiningKind::ADD) {
-      Value fma = rewriter.create<vector::FMAOp>(loc, x, y, acc);
+      Value fma = vector::FMAOp::create(rewriter, loc, x, y, acc);
       if (mask)
         // The fma op doesn't need explicit masking. However, fma ops used in
         // reductions must preserve previous 'acc' values for masked-out lanes.
         fma = selectPassthru(rewriter, mask, fma, acc);
       return fma;
     }
-    mul = rewriter.create<arith::MulFOp>(loc, x, y);
+    mul = arith::MulFOp::create(rewriter, loc, x, y);
   }
 
   if (!acc)
@@ -195,8 +195,8 @@ static std::optional<unsigned> getDimPosition(AffineMap map, unsigned dim) {
 static Value createAdd(Location loc, Value x, Value y, bool isInt,
                        PatternRewriter &rewriter) {
   if (isInt)
-    return rewriter.create<arith::AddIOp>(loc, x, y);
-  return rewriter.create<arith::AddFOp>(loc, x, y);
+    return arith::AddIOp::create(rewriter, loc, x, y);
+  return arith::AddFOp::create(rewriter, loc, x, y);
 }
 
 /// Creates a MulIOp if `isInt` is true otherwise create an MulFOp using
@@ -204,8 +204,8 @@ static Value createAdd(Location loc, Value x, Value y, bool isInt,
 static Value createMul(Location loc, Value x, Value y, bool isInt,
                        PatternRewriter &rewriter) {
   if (isInt)
-    return rewriter.create<arith::MulIOp>(loc, x, y);
-  return rewriter.create<arith::MulFOp>(loc, x, y);
+    return arith::MulIOp::create(rewriter, loc, x, y);
+  return arith::MulFOp::create(rewriter, loc, x, y);
 }
 
 namespace {
@@ -411,7 +411,7 @@ struct UnrolledOuterProductGenerator
   Value t(Value v, ArrayRef<int64_t> perm = {1, 0}) {
     if (!v)
       return v;
-    return rewriter.create<vector::TransposeOp>(loc, v, perm);
+    return vector::TransposeOp::create(rewriter, loc, v, perm);
   }
 
   Value promote(Value v, Type dstElementType) {
@@ -425,8 +425,8 @@ struct UnrolledOuterProductGenerator
     if (vecType)
       promotedType = vecType.clone(promotedType);
     if (isa<FloatType>(dstElementType))
-      return rewriter.create<arith::ExtFOp>(loc, promotedType, v);
-    return rewriter.create<arith::ExtSIOp>(loc, promotedType, v);
+      return arith::ExtFOp::create(rewriter, loc, promotedType, v);
+    return arith::ExtSIOp::create(rewriter, loc, promotedType, v);
   }
 
   FailureOr<Value> outerProd(Value lhs, Value rhs, Value res,
@@ -438,16 +438,16 @@ struct UnrolledOuterProductGenerator
 
     Type resElementType = cast<VectorType>(res.getType()).getElementType();
     for (int64_t k = 0; k < reductionSize; ++k) {
-      Value extractA = rewriter.create<vector::ExtractOp>(loc, lhs, k);
-      Value extractB = rewriter.create<vector::ExtractOp>(loc, rhs, k);
+      Value extractA = vector::ExtractOp::create(rewriter, loc, lhs, k);
+      Value extractB = vector::ExtractOp::create(rewriter, loc, rhs, k);
       extractA = promote(extractA, resElementType);
       extractB = promote(extractB, resElementType);
       Value extractMask;
       if (maybeMask.has_value() && maybeMask.value())
         extractMask =
-            rewriter.create<vector::ExtractOp>(loc, maybeMask.value(), k);
+            vector::ExtractOp::create(rewriter, loc, maybeMask.value(), k);
 
-      Operation *outerProdOp = rewriter.create<vector::OuterProductOp>(
+      Operation *outerProdOp = vector::OuterProductOp::create(rewriter,
           loc, res.getType(), extractA, extractB, res, kind);
       res = maskOperation(rewriter, outerProdOp, extractMask)->getResult(0);
     }
@@ -698,28 +698,28 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
     // Two outer parallel, one inner reduction (matmat flavor).
     //
     if (maps == infer({{m, k}, {k, n}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{m, k}, {n, k}, {m, n}})) {
       // No need to permute anything.
     } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
       // This is the classical row-major matmul. Just permute the lhs.
       Value tmp = lhs;
-      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
       rhs = tmp;
     } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
       std::swap(lhs, rhs);
     } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
       Value tmp = lhs;
-      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      rhs = rewriter.create<vector::TransposeOp>(loc, tmp, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, tmp, perm);
     } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
       Value tmp = rhs;
-      rhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
       lhs = tmp;
     } else {
       return failure();
@@ -732,12 +732,12 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
     if (maps == infer({{m, n}, {n}, {m}})) {
       // No need to permute anything.
     } else if (maps == infer({{n, m}, {n}, {m}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{n}, {m, n}, {m}})) {
       std::swap(lhs, rhs);
     } else if (maps == infer({{n}, {n, m}, {m}})) {
       std::swap(lhs, rhs);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else {
       return failure();
     }
@@ -754,31 +754,31 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
   unsigned dstColumns = rank == 1 ? 1 : dstType.getShape()[1];
 
   // ExtractOp does not allow dynamic indexing, we must unroll explicitly.
-  Value res = rewriter.create<arith::ConstantOp>(loc, dstType,
+  Value res = arith::ConstantOp::create(rewriter, loc, dstType,
                                                  rewriter.getZeroAttr(dstType));
   bool isInt = isa<IntegerType>(dstType.getElementType());
   llvm::SmallVector<Value> extractedCols;
   extractedCols.reserve(dstColumns);
   for (unsigned r = 0; r < dstRows; ++r) {
-    Value rowLhs = rewriter.create<vector::ExtractOp>(op.getLoc(), lhs, r);
+    Value rowLhs = vector::ExtractOp::create(rewriter, op.getLoc(), lhs, r);
     for (unsigned c = 0; c < dstColumns; ++c) {
       // Extract each respective row and column of the LHS and RHS once to
       // avoid having duplicate SSA values pointing to the same rows/columns.
       if (r == 0) {
         Value colRhs =
             rank == 1 ? rhs
-                      : rewriter.create<vector::ExtractOp>(op.getLoc(), rhs, c);
+                      : vector::ExtractOp::create(rewriter, op.getLoc(), rhs, c);
         extractedCols.push_back(colRhs);
       }
       Value extractedColRhs = extractedCols[c];
       Value product =
           createMul(op.getLoc(), rowLhs, extractedColRhs, isInt, rewriter);
-      Value sum = rewriter.create<vector::ReductionOp>(
+      Value sum = vector::ReductionOp::create(rewriter,
           op.getLoc(), vector::CombiningKind::ADD, product);
 
       SmallVector<int64_t, 2> pos = rank == 1 ? SmallVector<int64_t, 2>{r}
                                               : SmallVector<int64_t, 2>{r, c};
-      res = rewriter.create<vector::InsertOp>(op.getLoc(), sum, res, pos);
+      res = vector::InsertOp::create(rewriter, op.getLoc(), sum, res, pos);
     }
   }
   if (auto acc = op.getAcc())
@@ -879,21 +879,21 @@ struct ContractOpToElementwise
       lhsDims.append(lhsShape.begin(), lhsShape.end());
       auto expandedType =
           VectorType::get(lhsDims, contractOp.getLhsType().getElementType());
-      newLhs = rewriter.create<vector::BroadcastOp>(loc, expandedType, newLhs);
+      newLhs = vector::BroadcastOp::create(rewriter, loc, expandedType, newLhs);
     }
     if (!rhsDims.empty()) {
       rhsDims.append(rhsShape.begin(), rhsShape.end());
       auto expandedType =
           VectorType::get(rhsDims, contractOp.getRhsType().getElementType());
-      newRhs = rewriter.create<vector::BroadcastOp>(loc, expandedType, newRhs);
+      newRhs = vector::BroadcastOp::create(rewriter, loc, expandedType, newRhs);
     }
     bool isInt = contractOp.getLhsType().getElementType().isIntOrIndex();
-    newLhs = rewriter.create<vector::TransposeOp>(loc, newLhs, lhsTranspose);
-    newRhs = rewriter.create<vector::TransposeOp>(loc, newRhs, rhsTranspose);
+    newLhs = vector::TransposeOp::create(rewriter, loc, newLhs, lhsTranspose);
+    newRhs = vector::TransposeOp::create(rewriter, loc, newRhs, rhsTranspose);
     SmallVector<int64_t> lhsOffsets(lhsReductionDims.size(), 0);
     SmallVector<int64_t> rhsOffsets(rhsReductionDims.size(), 0);
-    newLhs = rewriter.create<vector::ExtractOp>(loc, newLhs, lhsOffsets);
-    newRhs = rewriter.create<vector::ExtractOp>(loc, newRhs, rhsOffsets);
+    newLhs = vector::ExtractOp::create(rewriter, loc, newLhs, lhsOffsets);
+    newRhs = vector::ExtractOp::create(rewriter, loc, newRhs, rhsOffsets);
     std::optional<Value> result =
         createContractArithOp(loc, newLhs, newRhs, contractOp.getAcc(),
                               contractOp.getKind(), rewriter, isInt);
@@ -1097,7 +1097,7 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       rewriter.getArrayAttr(adjustIter(op.getIteratorTypes(), iterIndex));
   // Unroll into a series of lower dimensional vector.contract ops.
   Location loc = op.getLoc();
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, resType, rewriter.getZeroAttr(resType));
 
   for (int64_t d = 0; d < dimSize; ++d) {
@@ -1110,7 +1110,7 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       lowMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *lowContract = rewriter.create<vector::ContractionOp>(
+    Operation *lowContract = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, acc, lowAffine, lowIter);
     lowContract = maskOperation(rewriter, lowContract, lowMask);
     result = reshapeStore(loc, lowContract->getResult(0), result, resType,
@@ -1161,8 +1161,8 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
 
     Value acc = op.getAcc();
     Operation *reductionOp =
-        acc ? rewriter.create<vector::ReductionOp>(loc, kind, m, acc)
-            : rewriter.create<vector::ReductionOp>(loc, kind, m);
+        acc ? vector::ReductionOp::create(rewriter, loc, kind, m, acc)
+            : vector::ReductionOp::create(rewriter, loc, kind, m);
     return maskOperation(rewriter, reductionOp, mask)->getResult(0);
   }
   // Construct new iterator types and affine map array attribute.
@@ -1186,7 +1186,7 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
       newMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *newContract = rewriter.create<vector::ContractionOp>(
+    Operation *newContract = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, result, lowAffine, lowIter);
     result = maskOperation(rewriter, newContract, newMask)->getResult(0);
   }
@@ -1240,7 +1240,7 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
 
     if (!rhsType) {
       // Special case: AXPY operation.
-      Value b = rewriter.create<vector::BroadcastOp>(loc, lhsType, op.getRhs());
+      Value b = vector::BroadcastOp::create(rewriter, loc, lhsType, op.getRhs());
       std::optional<Value> mult = createContractArithOp(
           loc, op.getLhs(), b, acc, kind, rewriter, isInt, mask);
       if (!mult.has_value())
@@ -1249,23 +1249,23 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
       return success();
     }
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
     for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
-      Value x = rewriter.create<vector::ExtractOp>(loc, op.getLhs(), d);
-      Value a = rewriter.create<vector::BroadcastOp>(loc, rhsType, x);
+      Value x = vector::ExtractOp::create(rewriter, loc, op.getLhs(), d);
+      Value a = vector::BroadcastOp::create(rewriter, loc, rhsType, x);
       Value r = nullptr;
       if (acc)
-        r = rewriter.create<vector::ExtractOp>(loc, acc, d);
+        r = vector::ExtractOp::create(rewriter, loc, acc, d);
       Value extrMask;
       if (mask)
-        extrMask = rewriter.create<vector::ExtractOp>(loc, mask, d);
+        extrMask = vector::ExtractOp::create(rewriter, loc, mask, d);
 
       std::optional<Value> m = createContractArithOp(
           loc, a, op.getRhs(), r, kind, rewriter, isInt, extrMask);
       if (!m.has_value())
         return failure();
-      result = rewriter.create<vector::InsertOp>(loc, *m, result, d);
+      result = vector::InsertOp::create(rewriter, loc, *m, result, d);
     }
 
     rewriter.replaceOp(rootOp, result);
@@ -1335,7 +1335,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   Value lhs = op.getLhs();
   auto lhsMap = op.getIndexingMapsArray()[0];
   if (lhsMap == AffineMap::get(3, 0, {k, m}, ctx))
-    lhs = rew.create<vector::TransposeOp>(loc, lhs, ArrayRef<int64_t>{1, 0});
+    lhs = vector::TransposeOp::create(rew, loc, lhs, ArrayRef<int64_t>{1, 0});
   else if (lhsMap != AffineMap::get(3, 0, {m, k}, ctx))
     return failure();
 
@@ -1343,7 +1343,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   Value rhs = op.getRhs();
   auto rhsMap = op.getIndexingMapsArray()[1];
   if (rhsMap == AffineMap::get(3, 0, {n, k}, ctx))
-    rhs = rew.create<vector::TransposeOp>(loc, rhs, ArrayRef<int64_t>{1, 0});
+    rhs = vector::TransposeOp::create(rew, loc, rhs, ArrayRef<int64_t>{1, 0});
   else if (rhsMap != AffineMap::get(3, 0, {k, n}, ctx))
     return failure();
 
@@ -1356,15 +1356,15 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
 
   Type flattenedLHSType =
       VectorType::get(lhsType.getNumElements(), lhsType.getElementType());
-  lhs = rew.create<vector::ShapeCastOp>(loc, flattenedLHSType, lhs);
+  lhs = vector::ShapeCastOp::create(rew, loc, flattenedLHSType, lhs);
 
   Type flattenedRHSType =
       VectorType::get(rhsType.getNumElements(), rhsType.getElementType());
-  rhs = rew.create<vector::ShapeCastOp>(loc, flattenedRHSType, rhs);
+  rhs = vector::ShapeCastOp::create(rew, loc, flattenedRHSType, rhs);
 
-  Value mul = rew.create<vector::MatmulOp>(loc, lhs, rhs, lhsRows, lhsColumns,
+  Value mul = vector::MatmulOp::create(rew, loc, lhs, rhs, lhsRows, lhsColumns,
                                            rhsColumns);
-  mul = rew.create<vector::ShapeCastOp>(
+  mul = vector::ShapeCastOp::create(rew,
       loc,
       VectorType::get({lhsRows, rhsColumns},
                       getElementTypeOrSelf(op.getAcc().getType())),
@@ -1373,15 +1373,15 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   // ACC must be C(m, n) or C(n, m).
   auto accMap = op.getIndexingMapsArray()[2];
   if (accMap == AffineMap::get(3, 0, {n, m}, ctx))
-    mul = rew.create<vector::TransposeOp>(loc, mul, ArrayRef<int64_t>{1, 0});
+    mul = vector::TransposeOp::create(rew, loc, mul, ArrayRef<int64_t>{1, 0});
   else if (accMap != AffineMap::get(3, 0, {m, n}, ctx))
     llvm_unreachable("invalid contraction semantics");
 
   Value res =
       isa<IntegerType>(elementType)
-          ? static_cast<Value>(rew.create<arith::AddIOp>(loc, op.getAcc(), mul))
+          ? static_cast<Value>(arith::AddIOp::create(rew, loc, op.getAcc(), mul))
           : static_cast<Value>(
-                rew.create<arith::AddFOp>(loc, op.getAcc(), mul));
+                arith::AddFOp::create(rew, loc, op.getAcc(), mul));
 
   return res;
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index 3000204c8ce17..c586059ca55bc 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -75,7 +75,7 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
     Value maskVec = op.getMask();
     Value passThruVec = op.getPassThru();
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultTy, rewriter.getZeroAttr(resultTy));
 
     VectorType subTy = VectorType::Builder(resultTy).dropDim(0);
@@ -84,16 +84,16 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
       int64_t thisIdx[1] = {i};
 
       Value indexSubVec =
-          rewriter.create<vector::ExtractOp>(loc, indexVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
       Value maskSubVec =
-          rewriter.create<vector::ExtractOp>(loc, maskVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, maskVec, thisIdx);
       Value passThruSubVec =
-          rewriter.create<vector::ExtractOp>(loc, passThruVec, thisIdx);
-      Value subGather = rewriter.create<vector::GatherOp>(
+          vector::ExtractOp::create(rewriter, loc, passThruVec, thisIdx);
+      Value subGather = vector::GatherOp::create(rewriter,
           loc, subTy, op.getBase(), op.getIndices(), indexSubVec, maskSubVec,
           passThruSubVec);
       result =
-          rewriter.create<vector::InsertOp>(loc, subGather, result, thisIdx);
+          vector::InsertOp::create(rewriter, loc, subGather, result, thisIdx);
     }
 
     rewriter.replaceOp(op, result);
@@ -159,22 +159,22 @@ struct RemoveStrideFromGatherSource : OpRewritePattern<vector::GatherOp> {
 
     // 1. Collapse the input memref so that it's "flat".
     SmallVector<ReassociationIndices> reassoc = {{0, 1}};
-    Value collapsed = rewriter.create<memref::CollapseShapeOp>(
+    Value collapsed = memref::CollapseShapeOp::create(rewriter,
         op.getLoc(), subview.getSource(), reassoc);
 
     // 2. Generate new gather indices that will model the
     // strided access.
     IntegerAttr stride = rewriter.getIndexAttr(srcTrailingDim);
     VectorType vType = op.getIndexVec().getType();
-    Value mulCst = rewriter.create<arith::ConstantOp>(
+    Value mulCst = arith::ConstantOp::create(rewriter,
         op.getLoc(), vType, DenseElementsAttr::get(vType, stride));
 
     Value newIdxs =
-        rewriter.create<arith::MulIOp>(op.getLoc(), op.getIndexVec(), mulCst);
+        arith::MulIOp::create(rewriter, op.getLoc(), op.getIndexVec(), mulCst);
 
     // 3. Create an updated gather op with the collapsed input memref and the
     // updated indices.
-    Value newGather = rewriter.create<vector::GatherOp>(
+    Value newGather = vector::GatherOp::create(rewriter,
         op.getLoc(), op.getResult().getType(), collapsed, op.getIndices(),
         newIdxs, op.getMask(), op.getPassThru());
     rewriter.replaceOp(op, newGather);
@@ -229,8 +229,8 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
     for (int64_t i = 0, e = resultTy.getNumElements(); i < e; ++i) {
       int64_t thisIdx[1] = {i};
       Value condition =
-          rewriter.create<vector::ExtractOp>(loc, condMask, thisIdx);
-      Value index = rewriter.create<vector::ExtractOp>(loc, indexVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, condMask, thisIdx);
+      Value index = vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
       baseOffsets.back() =
           rewriter.createOrFold<arith::AddIOp>(loc, lastBaseOffset, index);
 
@@ -240,19 +240,19 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
           // `vector.load` does not support scalar result; emit a vector load
           // and extract the single result instead.
           Value load =
-              b.create<vector::LoadOp>(loc, elemVecTy, base, baseOffsets);
+              vector::LoadOp::create(b, loc, elemVecTy, base, baseOffsets);
           int64_t zeroIdx[1] = {0};
-          extracted = b.create<vector::ExtractOp>(loc, load, zeroIdx);
+          extracted = vector::ExtractOp::create(b, loc, load, zeroIdx);
         } else {
-          extracted = b.create<tensor::ExtractOp>(loc, base, baseOffsets);
+          extracted = tensor::ExtractOp::create(b, loc, base, baseOffsets);
         }
 
         Value newResult =
-            b.create<vector::InsertOp>(loc, extracted, result, thisIdx);
-        b.create<scf::YieldOp>(loc, newResult);
+            vector::InsertOp::create(b, loc, extracted, result, thisIdx);
+        scf::YieldOp::create(b, loc, newResult);
       };
       auto passThruBuilder = [result](OpBuilder &b, Location loc) {
-        b.create<scf::YieldOp>(loc, result);
+        scf::YieldOp::create(b, loc, result);
       };
 
       result =
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index cab0f213b14a9..b0affbb699d9e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -60,14 +60,14 @@ class UnrollInterleaveOp final : public OpRewritePattern<vector::InterleaveOp> {
       return failure();
 
     auto loc = op.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
     for (auto position : *unrollIterator) {
-      Value extractLhs = rewriter.create<ExtractOp>(loc, op.getLhs(), position);
-      Value extractRhs = rewriter.create<ExtractOp>(loc, op.getRhs(), position);
+      Value extractLhs = ExtractOp::create(rewriter, loc, op.getLhs(), position);
+      Value extractRhs = ExtractOp::create(rewriter, loc, op.getRhs(), position);
       Value interleave =
-          rewriter.create<InterleaveOp>(loc, extractLhs, extractRhs);
-      result = rewriter.create<InsertOp>(loc, interleave, result, position);
+          InterleaveOp::create(rewriter, loc, extractLhs, extractRhs);
+      result = InsertOp::create(rewriter, loc, interleave, result, position);
     }
 
     rewriter.replaceOp(op, result);
@@ -123,19 +123,19 @@ class UnrollDeinterleaveOp final
       return failure();
 
     auto loc = op.getLoc();
-    Value emptyResult = rewriter.create<arith::ConstantOp>(
+    Value emptyResult = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
     Value evenResult = emptyResult;
     Value oddResult = emptyResult;
 
     for (auto position : *unrollIterator) {
       auto extractSrc =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+          vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       auto deinterleave =
-          rewriter.create<vector::DeinterleaveOp>(loc, extractSrc);
-      evenResult = rewriter.create<vector::InsertOp>(
+          vector::DeinterleaveOp::create(rewriter, loc, extractSrc);
+      evenResult = vector::InsertOp::create(rewriter,
           loc, deinterleave.getRes1(), evenResult, position);
-      oddResult = rewriter.create<vector::InsertOp>(loc, deinterleave.getRes2(),
+      oddResult = vector::InsertOp::create(rewriter, loc, deinterleave.getRes2(),
                                                     oddResult, position);
     }
     rewriter.replaceOp(op, ValueRange{evenResult, oddResult});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
index ba21092d2af3c..edd840d8fbc7b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
@@ -67,19 +67,19 @@ class CreateMaskOpLowering : public OpRewritePattern<vector::CreateMaskOp> {
     Value idx = op.getOperand(0);
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = rewriter.create<vector::CreateMaskOp>(
+    Value trueVal = vector::CreateMaskOp::create(rewriter,
         loc, lowType, op.getOperands().drop_front());
-    Value falseVal = rewriter.create<arith::ConstantOp>(
+    Value falseVal = arith::ConstantOp::create(rewriter,
         loc, lowType, rewriter.getZeroAttr(lowType));
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstType, rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < dim; d++) {
       Value bnd =
-          rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(d));
-      Value val = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+          arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(d));
+      Value val = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                  bnd, idx);
-      Value sel = rewriter.create<arith::SelectOp>(loc, val, trueVal, falseVal);
-      result = rewriter.create<vector::InsertOp>(loc, sel, result, d);
+      Value sel = arith::SelectOp::create(rewriter, loc, val, trueVal, falseVal);
+      result = vector::InsertOp::create(rewriter, loc, sel, result, d);
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -146,12 +146,12 @@ class ConstantMaskOpLowering : public OpRewritePattern<vector::ConstantMaskOp> {
           op, "Cannot unroll leading scalable dim in dstType");
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = rewriter.create<vector::ConstantMaskOp>(
+    Value trueVal = vector::ConstantMaskOp::create(rewriter,
         loc, lowType, dimSizes.drop_front());
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstType, rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < trueDimSize; d++)
-      result = rewriter.create<vector::InsertOp>(loc, trueVal, result, d);
+      result = vector::InsertOp::create(rewriter, loc, trueVal, result, d);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -261,7 +261,7 @@ struct MaskedGatherOpPattern : public MaskOpRewritePattern<GatherOp> {
                             PatternRewriter &rewriter) const override {
     Value passthru = maskingOp.hasPassthru()
                          ? maskingOp.getPassthru()
-                         : rewriter.create<arith::ConstantOp>(
+                         : arith::ConstantOp::create(rewriter,
                                gatherOp.getLoc(),
                                rewriter.getZeroAttr(gatherOp.getVectorType()));
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
index ce524b259d8d4..c5631ba00bbc6 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
@@ -103,12 +103,12 @@ class InnerOuterDimReductionConversion
     // If masked, transpose the original mask.
     Value transposedMask;
     if (maskableOp.isMasked()) {
-      transposedMask = rewriter.create<vector::TransposeOp>(
+      transposedMask = vector::TransposeOp::create(rewriter,
           loc, maskableOp.getMaskingOp().getMask(), indices);
     }
 
     // Transpose reduction source.
-    auto transposeOp = rewriter.create<vector::TransposeOp>(loc, src, indices);
+    auto transposeOp = vector::TransposeOp::create(rewriter, loc, src, indices);
     SmallVector<bool> reductionMask(srcRank, false);
     for (int i = 0; i < reductionSize; ++i) {
       if (useInnerDimsForReduction)
@@ -117,7 +117,7 @@ class InnerOuterDimReductionConversion
         reductionMask[i] = true;
     }
 
-    Operation *newMultiRedOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newMultiRedOp = vector::MultiDimReductionOp::create(rewriter,
         multiReductionOp.getLoc(), transposeOp.getResult(),
         multiReductionOp.getAcc(), reductionMask, multiReductionOp.getKind());
     newMultiRedOp =
@@ -256,13 +256,13 @@ class ReduceMultiDimReductionRank
           vectorShape,
           llvm::cast<VectorType>(vectorMask.getType()).getElementType());
       newVectorMask =
-          rewriter.create<vector::ShapeCastOp>(loc, maskCastedType, vectorMask);
+          vector::ShapeCastOp::create(rewriter, loc, maskCastedType, vectorMask);
     }
 
     auto castedType = VectorType::get(
         vectorShape, multiReductionOp.getSourceVectorType().getElementType(),
         scalableDims);
-    Value cast = rewriter.create<vector::ShapeCastOp>(
+    Value cast = vector::ShapeCastOp::create(rewriter,
         loc, castedType, multiReductionOp.getSource());
 
     Value acc = multiReductionOp.getAcc();
@@ -271,11 +271,11 @@ class ReduceMultiDimReductionRank
           {flattenedParallelDim},
           multiReductionOp.getSourceVectorType().getElementType(),
           /*scalableDims=*/{isParallelDimScalable});
-      acc = rewriter.create<vector::ShapeCastOp>(loc, accType, acc);
+      acc = vector::ShapeCastOp::create(rewriter, loc, accType, acc);
     }
     // 6. Creates the flattened form of vector.multi_reduction with inner/outer
     // most dim as reduction.
-    Operation *newMultiDimRedOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(rewriter,
         loc, cast, acc, mask, multiReductionOp.getKind());
     newMultiDimRedOp =
         mlir::vector::maskOperation(rewriter, newMultiDimRedOp, newVectorMask);
@@ -339,11 +339,11 @@ struct TwoDimMultiReductionToElementWise
 
     Value result = multiReductionOp.getAcc();
     for (int64_t i = 0; i < srcShape[0]; i++) {
-      auto operand = rewriter.create<vector::ExtractOp>(
+      auto operand = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getSource(), i);
       Value extractMask = nullptr;
       if (mask) {
-        extractMask = rewriter.create<vector::ExtractOp>(loc, mask, i);
+        extractMask = vector::ExtractOp::create(rewriter, loc, mask, i);
       }
       result =
           makeArithReduction(rewriter, loc, multiReductionOp.getKind(), operand,
@@ -383,27 +383,27 @@ struct TwoDimMultiReductionToReduction
     }
 
     auto loc = multiReductionOp.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, multiReductionOp.getDestType(),
         rewriter.getZeroAttr(multiReductionOp.getDestType()));
     int outerDim = multiReductionOp.getSourceVectorType().getShape()[0];
 
     for (int i = 0; i < outerDim; ++i) {
-      auto v = rewriter.create<vector::ExtractOp>(
+      auto v = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
-      auto acc = rewriter.create<vector::ExtractOp>(
+      auto acc = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getAcc(), ArrayRef<int64_t>{i});
-      Operation *reductionOp = rewriter.create<vector::ReductionOp>(
+      Operation *reductionOp = vector::ReductionOp::create(rewriter,
           loc, multiReductionOp.getKind(), v, acc);
 
       // If masked, slice the mask and mask the new reduction operation.
       if (maskableOp.isMasked()) {
-        Value mask = rewriter.create<vector::ExtractOp>(
+        Value mask = vector::ExtractOp::create(rewriter,
             loc, maskableOp.getMaskingOp().getMask(), ArrayRef<int64_t>{i});
         reductionOp = mlir::vector::maskOperation(rewriter, reductionOp, mask);
       }
 
-      result = rewriter.create<vector::InsertOp>(loc, reductionOp->getResult(0),
+      result = vector::InsertOp::create(rewriter, loc, reductionOp->getResult(0),
                                                  result, i);
     }
 
@@ -459,9 +459,9 @@ struct OneDimMultiReductionToTwoDim
     SmallVector<bool, 2> reductionMask{false, true};
 
     /// vector.extract(vector.multi_reduce(vector.shape_cast(v, 1xk)), 0)
-    Value cast = rewriter.create<vector::ShapeCastOp>(
+    Value cast = vector::ShapeCastOp::create(rewriter,
         loc, castedType, multiReductionOp.getSource());
-    Value castAcc = rewriter.create<vector::BroadcastOp>(
+    Value castAcc = vector::BroadcastOp::create(rewriter,
         loc, accType, multiReductionOp.getAcc());
     Value castMask;
     if (maskableOp.isMasked()) {
@@ -470,10 +470,10 @@ struct OneDimMultiReductionToTwoDim
           ArrayRef<int64_t>{1, maskType.getShape().back()},
           maskType.getElementType(),
           ArrayRef<bool>{false, maskType.getScalableDims().back()});
-      castMask = rewriter.create<vector::BroadcastOp>(loc, castMaskType, mask);
+      castMask = vector::BroadcastOp::create(rewriter, loc, castMaskType, mask);
     }
 
-    Operation *newOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newOp = vector::MultiDimReductionOp::create(rewriter,
         loc, cast, castAcc, reductionMask, multiReductionOp.getKind());
     newOp = vector::maskOperation(rewriter, newOp, castMask);
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
index a1f67bd0e9ed3..a6985b96b59df 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
@@ -122,7 +122,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
       return failure();
 
     VectorType resType = VectorType::get(destShape, elType);
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
     int64_t reductionDim = scanOp.getReductionDim();
     bool inclusive = scanOp.getInclusive();
@@ -144,7 +144,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
     for (int i = 0; i < destShape[reductionDim]; i++) {
       offsets[reductionDim] = i;
       ArrayAttr scanOffsets = rewriter.getI64ArrayAttr(offsets);
-      Value input = rewriter.create<vector::ExtractStridedSliceOp>(
+      Value input = vector::ExtractStridedSliceOp::create(rewriter,
           loc, reductionType, scanOp.getSource(), scanOffsets, scanSizes,
           scanStrides);
       Value output;
@@ -154,10 +154,10 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         } else {
           if (initialValueRank == 0) {
             // ShapeCastOp cannot handle 0-D vectors
-            output = rewriter.create<vector::BroadcastOp>(
+            output = vector::BroadcastOp::create(rewriter,
                 loc, input.getType(), scanOp.getInitialValue());
           } else {
-            output = rewriter.create<vector::ShapeCastOp>(
+            output = vector::ShapeCastOp::create(rewriter,
                 loc, input.getType(), scanOp.getInitialValue());
           }
         }
@@ -166,7 +166,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         output = vector::makeArithReduction(rewriter, loc, scanOp.getKind(),
                                             lastOutput, y);
       }
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = vector::InsertStridedSliceOp::create(rewriter,
           loc, output, result, offsets, strides);
       lastOutput = output;
       lastInput = input;
@@ -174,11 +174,11 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
 
     Value reduction;
     if (initialValueRank == 0) {
-      Value v = rewriter.create<vector::ExtractOp>(loc, lastOutput, 0);
+      Value v = vector::ExtractOp::create(rewriter, loc, lastOutput, 0);
       reduction =
-          rewriter.create<vector::BroadcastOp>(loc, initialValueType, v);
+          vector::BroadcastOp::create(rewriter, loc, initialValueType, v);
     } else {
-      reduction = rewriter.create<vector::ShapeCastOp>(loc, initialValueType,
+      reduction = vector::ShapeCastOp::create(rewriter, loc, initialValueType,
                                                        lastOutput);
     }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
index 39c16fab21c4e..69d3666a13879 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
@@ -137,10 +137,10 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     const int64_t resultLeading = delta > 0 ? 0 : -delta;
 
     const Value source = shapeCast.getSource();
-    const Value poison = rewriter.create<ub::PoisonOp>(loc, resultType);
-    const Value extracted = rewriter.create<vector::ExtractOp>(
+    const Value poison = ub::PoisonOp::create(rewriter, loc, resultType);
+    const Value extracted = vector::ExtractOp::create(rewriter,
         loc, source, SmallVector<int64_t>(sourceLeading, 0));
-    const Value result = rewriter.create<vector::InsertOp>(
+    const Value result = vector::InsertOp::create(rewriter,
         loc, extracted, poison, SmallVector<int64_t>(resultLeading, 0));
 
     rewriter.replaceOp(shapeCast, result);
@@ -171,13 +171,13 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 
     SmallVector<int64_t> extractIndex(sourceDim, 0);
     SmallVector<int64_t> insertIndex(resultDim, 0);
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
 
     for (int i = 0; i < nSlices; ++i) {
       Value extracted =
-          rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+          vector::ExtractOp::create(rewriter, loc, source, extractIndex);
 
-      result = rewriter.create<vector::InsertOp>(loc, extracted, result,
+      result = vector::InsertOp::create(rewriter, loc, extracted, result,
                                                  insertIndex);
 
       inplaceAdd(1, sourceShape.take_front(sourceDim), extractIndex);
@@ -276,9 +276,9 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     Value extracted = {};
     Value extractedStrided = {};
     Value insertedSlice = {};
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
     const Value partResult =
-        rewriter.create<ub::PoisonOp>(loc, insertStridedType);
+        ub::PoisonOp::create(rewriter, loc, insertStridedType);
 
     for (size_t i = 0; i < nAtomicSlices; ++i) {
 
@@ -288,14 +288,14 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
       // vector.extract
       if (extractStridedPhase == 0) {
         extracted =
-            rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+            vector::ExtractOp::create(rewriter, loc, source, extractIndex);
         inplaceAdd(1, sourceShape.take_front(sourceSuffixStartDim),
                    extractIndex);
       }
 
       // vector.extract_strided_slice
       extractOffsets[0] = extractStridedPhase * greatestCommonDivisor;
-      extractedStrided = rewriter.create<vector::ExtractStridedSliceOp>(
+      extractedStrided = vector::ExtractStridedSliceOp::create(rewriter,
           loc, extracted, extractOffsets, atomicShape, sizes);
 
       // vector.insert_strided_slice
@@ -303,12 +303,12 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
         insertedSlice = partResult;
       }
       insertOffsets[0] = insertStridedPhase * greatestCommonDivisor;
-      insertedSlice = rewriter.create<vector::InsertStridedSliceOp>(
+      insertedSlice = vector::InsertStridedSliceOp::create(rewriter,
           loc, extractedStrided, insertedSlice, insertOffsets, sizes);
 
       // vector.insert
       if (insertStridedPhase + 1 == insertPeriod) {
-        result = rewriter.create<vector::InsertOp>(loc, insertedSlice, result,
+        result = vector::InsertOp::create(rewriter, loc, insertedSlice, result,
                                                    insertIndex);
         inplaceAdd(1, resultType.getShape().take_front(resultSuffixStartDim),
                    insertIndex);
@@ -394,7 +394,7 @@ class ScalableShapeCastOpRewritePattern
     auto extractionVectorType = VectorType::get(
         {minExtractionSize}, sourceVectorType.getElementType(), {true});
 
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultVectorType);
     SmallVector<int64_t> srcIdx(srcRank, 0);
     SmallVector<int64_t> resIdx(resRank, 0);
 
@@ -406,7 +406,7 @@ class ScalableShapeCastOpRewritePattern
       // 1. Extract a scalable subvector from the source vector.
       if (!currentSourceScalableVector) {
         if (srcRank != 1) {
-          currentSourceScalableVector = rewriter.create<vector::ExtractOp>(
+          currentSourceScalableVector = vector::ExtractOp::create(rewriter,
               loc, op.getSource(), llvm::ArrayRef(srcIdx).drop_back());
         } else {
           currentSourceScalableVector = op.getSource();
@@ -414,7 +414,7 @@ class ScalableShapeCastOpRewritePattern
       }
       Value sourceSubVector = currentSourceScalableVector;
       if (minExtractionSize < minSourceTrailingSize) {
-        sourceSubVector = rewriter.create<vector::ScalableExtractOp>(
+        sourceSubVector = vector::ScalableExtractOp::create(rewriter,
             loc, extractionVectorType, sourceSubVector, srcIdx.back());
       }
 
@@ -423,14 +423,14 @@ class ScalableShapeCastOpRewritePattern
         if (minExtractionSize == minResultTrailingSize) {
           currentResultScalableVector = sourceSubVector;
         } else if (resRank != 1) {
-          currentResultScalableVector = rewriter.create<vector::ExtractOp>(
+          currentResultScalableVector = vector::ExtractOp::create(rewriter,
               loc, result, llvm::ArrayRef(resIdx).drop_back());
         } else {
           currentResultScalableVector = result;
         }
       }
       if (minExtractionSize < minResultTrailingSize) {
-        currentResultScalableVector = rewriter.create<vector::ScalableInsertOp>(
+        currentResultScalableVector = vector::ScalableInsertOp::create(rewriter,
             loc, sourceSubVector, currentResultScalableVector, resIdx.back());
       }
 
@@ -439,7 +439,7 @@ class ScalableShapeCastOpRewritePattern
           currentResultScalableVector != result) {
         // Finished row of result. Insert complete scalable vector into result
         // (n-D) vector.
-        result = rewriter.create<vector::InsertOp>(
+        result = vector::InsertOp::create(rewriter,
             loc, currentResultScalableVector, result,
             llvm::ArrayRef(resIdx).drop_back());
         currentResultScalableVector = {};
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
index 5b81d0d33d484..fe02cd5656571 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
@@ -47,7 +47,7 @@ static Value extendVectorRank(OpBuilder &builder, Location loc, Value vec,
                          originalVecType.getScalableDims().end());
   VectorType newVecType = VectorType::get(
       newShape, originalVecType.getElementType(), newScalableDims);
-  return builder.create<vector::BroadcastOp>(loc, newVecType, vec);
+  return vector::BroadcastOp::create(builder, loc, newVecType, vec);
 }
 
 /// Extend the rank of a vector Value by `addedRanks` by adding inner unit
@@ -62,7 +62,7 @@ static Value extendMaskRank(OpBuilder &builder, Location loc, Value vec,
     permutation.push_back(i);
   for (int64_t i = 0; i < addedRank; ++i)
     permutation.push_back(i);
-  return builder.create<vector::TransposeOp>(loc, broadcasted, permutation);
+  return vector::TransposeOp::create(builder, loc, broadcasted, permutation);
 }
 
 //===----------------------------------------------------------------------===//
@@ -138,7 +138,7 @@ struct TransferReadPermutationLowering
     // Generate new transfer_read operation.
     VectorType newReadType = VectorType::get(
         newVectorShape, op.getVectorType().getElementType(), newScalableDims);
-    Value newRead = rewriter.create<vector::TransferReadOp>(
+    Value newRead = vector::TransferReadOp::create(rewriter,
         op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
@@ -209,11 +209,11 @@ struct TransferWritePermutationLowering
         inverseTransposeInBoundsAttr(rewriter, op.getInBounds(), permutation);
 
     // Generate new transfer_write operation.
-    Value newVec = rewriter.create<vector::TransposeOp>(
+    Value newVec = vector::TransposeOp::create(rewriter,
         op.getLoc(), op.getVector(), indices);
     auto newMap = AffineMap::getMinorIdentityMap(
         map.getNumDims(), map.getNumResults(), rewriter.getContext());
-    auto newWrite = rewriter.create<vector::TransferWriteOp>(
+    auto newWrite = vector::TransferWriteOp::create(rewriter,
         op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getMask(), newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
@@ -299,7 +299,7 @@ struct TransferWriteNonPermutationLowering
       newInBoundsValues.push_back(op.isDimInBounds(i));
     }
     ArrayAttr newInBoundsAttr = rewriter.getBoolArrayAttr(newInBoundsValues);
-    auto newWrite = rewriter.create<vector::TransferWriteOp>(
+    auto newWrite = vector::TransferWriteOp::create(rewriter,
         op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), newMask, newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
@@ -370,7 +370,7 @@ struct TransferOpReduceRank
             ? rewriter.getArrayAttr(
                   op.getInBoundsAttr().getValue().take_back(reducedShapeRank))
             : ArrayAttr();
-    Value newRead = rewriter.create<vector::TransferReadOp>(
+    Value newRead = vector::TransferReadOp::create(rewriter,
         op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
@@ -471,20 +471,20 @@ struct TransferReadToVectorLoadLowering
             read, "vector type is not rank 1, can't create masked load, needs "
                   "VectorToSCF");
 
-      Value fill = rewriter.create<vector::SplatOp>(
+      Value fill = vector::SplatOp::create(rewriter,
           read.getLoc(), unbroadcastedVectorType, read.getPadding());
-      res = rewriter.create<vector::MaskedLoadOp>(
+      res = vector::MaskedLoadOp::create(rewriter,
           read.getLoc(), unbroadcastedVectorType, read.getBase(),
           read.getIndices(), read.getMask(), fill);
     } else {
-      res = rewriter.create<vector::LoadOp>(read.getLoc(),
+      res = vector::LoadOp::create(rewriter, read.getLoc(),
                                             unbroadcastedVectorType,
                                             read.getBase(), read.getIndices());
     }
 
     // Insert a broadcasting op if required.
     if (!broadcastedDims.empty())
-      res = rewriter.create<vector::BroadcastOp>(
+      res = vector::BroadcastOp::create(rewriter,
           read.getLoc(), read.getVectorType(), res->getResult(0));
     return res->getResult(0);
   }
@@ -569,11 +569,11 @@ struct TransferWriteToVectorStoreLowering
                    << write;
             });
 
-      rewriter.create<vector::MaskedStoreOp>(
+      vector::MaskedStoreOp::create(rewriter,
           write.getLoc(), write.getBase(), write.getIndices(), write.getMask(),
           write.getVector());
     } else {
-      rewriter.create<vector::StoreOp>(write.getLoc(), write.getVector(),
+      vector::StoreOp::create(rewriter, write.getLoc(), write.getVector(),
                                        write.getBase(), write.getIndices());
     }
     // There's no return value for StoreOps. Use Value() to signal success to
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index 732e316c93381..e9591bfcf6feb 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -80,7 +80,7 @@ getUnpackShufflePermFor128Lane(ArrayRef<int64_t> vals, int numBits) {
 static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({0, 1, numElem, numElem + 1}, numBits));
 }
@@ -94,7 +94,7 @@ static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({2, 3, numElem + 2, numElem + 3},
                                      numBits));
@@ -109,7 +109,7 @@ static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  auto shuffle = b.create<vector::ShuffleOp>(
+  auto shuffle = vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({0, numElem, 1, numElem + 1}, numBits));
   return shuffle;
@@ -124,7 +124,7 @@ static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({2, numElem + 2, 3, numElem + 3},
                                      numBits));
@@ -181,7 +181,7 @@ static Value create4x128BitSuffle(ImplicitLocOpBuilder &b, Value v1, Value v2,
   appendToMask(0, b23);
   appendToMask(16, b45);
   appendToMask(16, b67);
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// Lowers the value to a vector.shuffle op. The `source` is expected to be a
@@ -192,7 +192,7 @@ static Value transposeToShuffle1D(OpBuilder &b, Value source, int m, int n) {
   for (int64_t j = 0; j < n; ++j)
     for (int64_t i = 0; i < m; ++i)
       mask.push_back(i * n + j);
-  return b.create<vector::ShuffleOp>(source.getLoc(), source, source, mask);
+  return vector::ShuffleOp::create(b, source.getLoc(), source, source, mask);
 }
 
 /// Lowers the value to a sequence of vector.shuffle ops. The `source` is
@@ -284,9 +284,9 @@ static Value transposeToShuffle16x16(OpBuilder &builder, Value source, int m,
 
   auto reshInputType = VectorType::get(
       {m, n}, cast<VectorType>(source.getType()).getElementType());
-  Value res = b.create<ub::PoisonOp>(reshInputType);
+  Value res = ub::PoisonOp::create(b, reshInputType);
   for (int64_t i = 0; i < m; ++i)
-    res = b.create<vector::InsertOp>(vs[i], res, i);
+    res = vector::InsertOp::create(b, vs[i], res, i);
   return res;
 }
 
@@ -335,10 +335,10 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
       Type flattenedType =
           VectorType::get(resType.getNumElements(), resType.getElementType());
       auto matrix =
-          rewriter.create<vector::ShapeCastOp>(loc, flattenedType, input);
+          vector::ShapeCastOp::create(rewriter, loc, flattenedType, input);
       auto rows = rewriter.getI32IntegerAttr(resType.getShape()[0]);
       auto columns = rewriter.getI32IntegerAttr(resType.getShape()[1]);
-      Value trans = rewriter.create<vector::FlatTransposeOp>(
+      Value trans = vector::FlatTransposeOp::create(rewriter,
           loc, flattenedType, matrix, rows, columns);
       rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, trans);
       return success();
@@ -359,7 +359,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
     // of the leftmost transposed dimensions. We traverse every transpose
     // element using a linearized index that we delinearize to generate the
     // appropriate indices for the extract/insert operations.
-    Value result = rewriter.create<ub::PoisonOp>(loc, resType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resType);
     int64_t numTransposedElements = ShapedType::getNumElements(prunedInShape);
 
     for (int64_t linearIdx = 0; linearIdx < numTransposedElements;
@@ -482,14 +482,14 @@ class TransposeOp2DToShuffleLowering
     Location loc = op.getLoc();
     auto flattenedType = VectorType::get({n * m}, srcType.getElementType());
     auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
-    auto reshInput = rewriter.create<vector::ShapeCastOp>(loc, flattenedType,
+    auto reshInput = vector::ShapeCastOp::create(rewriter, loc, flattenedType,
                                                           op.getVector());
 
     Value res;
     if (vectorTransposeLowering == VectorTransposeLowering::Shuffle16x16 &&
         m == 16 && n == 16) {
       reshInput =
-          rewriter.create<vector::ShapeCastOp>(loc, reshInputType, reshInput);
+          vector::ShapeCastOp::create(rewriter, loc, reshInputType, reshInput);
       res = transposeToShuffle16x16(rewriter, reshInput, m, n);
     } else {
       // Fallback to shuffle on 1D approach.
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index fb99e22c77ea0..b555c3f6d9f85 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -114,7 +114,7 @@ struct DistributedLoadStoreHelper {
            "preregistered sequential value.");
     // Scalar case can directly use memref.store.
     if (!isa<VectorType>(val.getType()))
-      return b.create<memref::StoreOp>(loc, val, buffer, zero);
+      return memref::StoreOp::create(b, loc, val, buffer, zero);
 
     // Vector case must use vector::TransferWriteOp which will later lower to
     //   vector.store of memref.store depending on further lowerings.
@@ -127,7 +127,7 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return b.create<vector::TransferWriteOp>(
+    return vector::TransferWriteOp::create(b,
         loc, val, buffer, indices,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
@@ -156,7 +156,7 @@ struct DistributedLoadStoreHelper {
 
     // Scalar case can directly use memref.store.
     if (!isa<VectorType>(type))
-      return b.create<memref::LoadOp>(loc, buffer, zero);
+      return memref::LoadOp::create(b, loc, buffer, zero);
 
     // Other cases must be vector atm.
     // Vector case must use vector::TransferReadOp which will later lower to
@@ -172,7 +172,7 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return b.create<vector::TransferReadOp>(
+    return vector::TransferReadOp::create(b,
         loc, cast<VectorType>(type), buffer, indices, /*padding=*/std::nullopt,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
@@ -243,10 +243,10 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
     rewriter.setInsertionPoint(warpOp);
 
     // Step 1: Create scf.if op.
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    Value isLane0 = rewriter.create<arith::CmpIOp>(
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    Value isLane0 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, warpOp.getLaneid(), c0);
-    auto ifOp = rewriter.create<scf::IfOp>(loc, isLane0,
+    auto ifOp = scf::IfOp::create(rewriter, loc, isLane0,
                                            /*withElseRegion=*/false);
     rewriter.eraseOp(ifOp.thenBlock()->getTerminator());
 
@@ -325,7 +325,7 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
     // Step 7. Delete terminator and add empty scf.yield.
     rewriter.eraseOp(yieldOp);
     rewriter.setInsertionPointToEnd(ifOp.thenBlock());
-    rewriter.create<scf::YieldOp>(yieldLoc);
+    scf::YieldOp::create(rewriter, yieldLoc);
 
     // Compute replacements for WarpOp results.
     rewriter.replaceOp(warpOp, replacements);
@@ -512,7 +512,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
 
     // Create a second warp op that contains only writeOp.
-    auto secondWarpOp = rewriter.create<WarpExecuteOnLane0Op>(
+    auto secondWarpOp = WarpExecuteOnLane0Op::create(rewriter,
         loc, TypeRange(), newWarpOp.getLaneid(), newWarpOp.getWarpSize());
     Block &body = secondWarpOp.getBodyRegion().front();
     rewriter.setInsertionPointToStart(&body);
@@ -521,7 +521,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     newWriteOp.getValueToStoreMutable().assign(
         newWarpOp.getResult(newRetIndices[0]));
     rewriter.eraseOp(writeOp);
-    rewriter.create<gpu::YieldOp>(newWarpOp.getLoc());
+    gpu::YieldOp::create(rewriter, newWarpOp.getLoc());
     return success();
   }
 
@@ -698,7 +698,7 @@ struct WarpOpConstant : public WarpDistributionPattern {
         cast<ShapedType>(warpOp.getResult(operandIndex).getType()), scalarAttr);
     Location loc = warpOp.getLoc();
     rewriter.setInsertionPointAfter(warpOp);
-    Value distConstant = rewriter.create<arith::ConstantOp>(loc, newAttr);
+    Value distConstant = arith::ConstantOp::create(rewriter, loc, newAttr);
     rewriter.replaceAllUsesWith(warpOp.getResult(operandIndex), distConstant);
     rewriter.finalizeOpModification(warpOp);
     return success();
@@ -823,7 +823,7 @@ struct WarpOpTransferRead : public WarpDistributionPattern {
     Value newMask =
         hasMask ? newWarpOp.getResult(newRetIndices[newRetIndices.size() - 1])
                 : Value();
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         read.getLoc(), distributedVal.getType(), read.getBase(), newIndices,
         read.getPermutationMapAttr(), newPadding, newMask,
         read.getInBoundsAttr());
@@ -965,7 +965,7 @@ struct WarpOpBroadcast : public WarpDistributionPattern {
     WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
         rewriter, warpOp, {broadcastSrc}, {broadcastSrcType}, newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value broadcasted = rewriter.create<vector::BroadcastOp>(
+    Value broadcasted = vector::BroadcastOp::create(rewriter,
         loc, destVecType, newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 broadcasted);
@@ -1008,7 +1008,7 @@ struct WarpOpShapeCast : public WarpDistributionPattern {
         rewriter, warpOp, {oldCastOp.getSource()}, {castDistributedType},
         newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value newCast = rewriter.create<vector::ShapeCastOp>(
+    Value newCast = vector::ShapeCastOp::create(rewriter,
         oldCastOp.getLoc(), castResultType,
         newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newCast);
@@ -1091,7 +1091,7 @@ struct WarpOpCreateMask : public WarpDistributionPattern {
     }
 
     auto newMask =
-        rewriter.create<vector::CreateMaskOp>(loc, distType, newOperands);
+        vector::CreateMaskOp::create(rewriter, loc, distType, newOperands);
     rewriter.replaceAllUsesWith(warpOp.getResult(operandIndex), newMask);
     rewriter.finalizeOpModification(warpOp);
     return success();
@@ -1182,7 +1182,7 @@ struct WarpOpInsertStridedSlice : public WarpDistributionPattern {
     Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
     // Create a new insert strided slice op that inserts distributed source into
     // distributed dest.
-    Value newInsert = rewriter.create<vector::InsertStridedSliceOp>(
+    Value newInsert = vector::InsertStridedSliceOp::create(rewriter,
         insertOp.getLoc(), distributedDest.getType(), distributedSource,
         distributedDest, insertOp.getOffsets(), insertOp.getStrides());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newInsert);
@@ -1277,7 +1277,7 @@ struct WarpOpExtractStridedSlice : public WarpDistributionPattern {
     // Create a new extract strided slice op that extracts from the
     // distributed vector.
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
-    Value newExtract = rewriter.create<vector::ExtractStridedSliceOp>(
+    Value newExtract = vector::ExtractStridedSliceOp::create(rewriter,
         extractOp.getLoc(), distributedType, distributedVec,
         extractOp.getOffsets(),
         ArrayAttr::get(rewriter.getContext(), distributedSizes),
@@ -1323,7 +1323,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
       // Extract from distributed vector.
-      Value newExtract = rewriter.create<vector::ExtractOp>(
+      Value newExtract = vector::ExtractOp::create(rewriter,
           loc, distributedVec, extractOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
@@ -1352,7 +1352,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
     // Extract from distributed vector.
-    Value newExtract = rewriter.create<vector::ExtractOp>(
+    Value newExtract = vector::ExtractOp::create(rewriter,
         loc, distributedVec, extractOp.getMixedPosition());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 newExtract);
@@ -1422,7 +1422,7 @@ struct WarpOpExtractScalar : public WarpDistributionPattern {
       Value newExtract;
       SmallVector<int64_t> indices(extractSrcType.getRank(), 0);
       newExtract =
-          rewriter.create<vector::ExtractOp>(loc, distributedVec, indices);
+          vector::ExtractOp::create(rewriter, loc, distributedVec, indices);
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
       return success();
@@ -1442,11 +1442,11 @@ struct WarpOpExtractScalar : public WarpDistributionPattern {
     // Extract at position: pos % elementsPerLane
     Value newPos =
         elementsPerLane == 1
-            ? rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult()
+            ? arith::ConstantIndexOp::create(rewriter, loc, 0).getResult()
             : affine::makeComposedAffineApply(rewriter, loc,
                                               sym0 % elementsPerLane, pos);
     Value extracted =
-        rewriter.create<vector::ExtractOp>(loc, distributedVec, newPos);
+        vector::ExtractOp::create(rewriter, loc, distributedVec, newPos);
 
     // Shuffle the extracted value to all lanes.
     Value shuffled = warpShuffleFromIdxFn(
@@ -1535,7 +1535,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
       if (pos) {
         indices.push_back(pos);
       }
-      newInsert = rewriter.create<vector::InsertOp>(loc, newSource,
+      newInsert = vector::InsertOp::create(rewriter, loc, newSource,
                                                     distributedVec, indices);
       // Broadcast: Simply move the vector.insert op out.
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
@@ -1552,7 +1552,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
     // Insert position: pos % elementsPerLane
     OpFoldResult newPos = affine::makeComposedFoldedAffineApply(
         rewriter, loc, sym0 % elementsPerLane, pos);
-    Value isInsertingLane = rewriter.create<arith::CmpIOp>(
+    Value isInsertingLane = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
     Value newResult =
         rewriter
@@ -1560,13 +1560,13 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
                 loc, isInsertingLane,
                 /*thenBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  Value newInsert = builder.create<vector::InsertOp>(
+                  Value newInsert = vector::InsertOp::create(builder,
                       loc, newSource, distributedVec, newPos);
-                  builder.create<scf::YieldOp>(loc, newInsert);
+                  scf::YieldOp::create(builder, loc, newInsert);
                 },
                 /*elseBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  builder.create<scf::YieldOp>(loc, distributedVec);
+                  scf::YieldOp::create(builder, loc, distributedVec);
                 })
             .getResult(0);
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newResult);
@@ -1603,7 +1603,7 @@ struct WarpOpInsert : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedSrc = newWarpOp->getResult(newRetIndices[0]);
       Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
-      Value newResult = rewriter.create<vector::InsertOp>(
+      Value newResult = vector::InsertOp::create(rewriter,
           loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newResult);
@@ -1653,7 +1653,7 @@ struct WarpOpInsert : public WarpDistributionPattern {
     Value newResult;
     if (distrSrcDim >= 0) {
       // Every lane inserts a small piece.
-      newResult = rewriter.create<vector::InsertOp>(
+      newResult = vector::InsertOp::create(rewriter,
           loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
     } else {
       // One lane inserts the entire source vector.
@@ -1661,19 +1661,19 @@ struct WarpOpInsert : public WarpDistributionPattern {
       SmallVector<OpFoldResult> pos = insertOp.getMixedPosition();
       SmallVector<int64_t> newPos = getAsIntegers(pos);
       // tid of inserting lane: pos / elementsPerLane
-      Value insertingLane = rewriter.create<arith::ConstantIndexOp>(
+      Value insertingLane = arith::ConstantIndexOp::create(rewriter,
           loc, newPos[distrDestDim] / elementsPerLane);
-      Value isInsertingLane = rewriter.create<arith::CmpIOp>(
+      Value isInsertingLane = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
       // Insert position: pos % elementsPerLane
       newPos[distrDestDim] %= elementsPerLane;
       auto insertingBuilder = [&](OpBuilder &builder, Location loc) {
-        Value newInsert = builder.create<vector::InsertOp>(
+        Value newInsert = vector::InsertOp::create(builder,
             loc, distributedSrc, distributedDest, newPos);
-        builder.create<scf::YieldOp>(loc, newInsert);
+        scf::YieldOp::create(builder, loc, newInsert);
       };
       auto nonInsertingBuilder = [&](OpBuilder &builder, Location loc) {
-        builder.create<scf::YieldOp>(loc, distributedDest);
+        scf::YieldOp::create(builder, loc, distributedDest);
       };
       newResult = rewriter
                       .create<scf::IfOp>(loc, isInsertingLane,
@@ -1802,7 +1802,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
 
     // Create a new for op outside the region with a WarpExecuteOnLane0Op
     // region inside.
-    auto newForOp = rewriter.create<scf::ForOp>(
+    auto newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), newOperands);
     rewriter.setInsertionPointToStart(newForOp.getBody());
@@ -1817,7 +1817,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
       argIndexMapping[escapingValues[i]] = warpInputType.size();
       warpInputType.push_back(inputTypes[i]);
     }
-    auto innerWarp = rewriter.create<WarpExecuteOnLane0Op>(
+    auto innerWarp = WarpExecuteOnLane0Op::create(rewriter,
         newWarpOp.getLoc(), newForOp.getResultTypes(), newWarpOp.getLaneid(),
         newWarpOp.getWarpSize(), warpInput, warpInputType);
 
@@ -1833,10 +1833,10 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
     rewriter.eraseOp(forOp.getBody()->getTerminator());
     rewriter.mergeBlocks(forOp.getBody(), innerWarp.getBody(), argMapping);
     rewriter.setInsertionPointToEnd(innerWarp.getBody());
-    rewriter.create<gpu::YieldOp>(innerWarp.getLoc(), yieldOperands);
+    gpu::YieldOp::create(rewriter, innerWarp.getLoc(), yieldOperands);
     rewriter.setInsertionPointAfter(innerWarp);
     if (!innerWarp.getResults().empty())
-      rewriter.create<scf::YieldOp>(forOp.getLoc(), innerWarp.getResults());
+      scf::YieldOp::create(rewriter, forOp.getLoc(), innerWarp.getResults());
     rewriter.eraseOp(forOp);
     // Replace the warpOp result coming from the original ForOp.
     for (const auto &res : llvm::enumerate(resultIdx)) {
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
index 067d4e3491391..e9718a7795fc5 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
@@ -77,7 +77,7 @@ struct CastAwayExtractStridedSliceLeadingOneDim
 
     Location loc = extractOp.getLoc();
 
-    Value newSrcVector = rewriter.create<vector::ExtractOp>(
+    Value newSrcVector = vector::ExtractOp::create(rewriter,
         loc, extractOp.getVector(), splatZero(dropCount));
 
     // The offsets/sizes/strides attribute can have a less number of elements
@@ -89,7 +89,7 @@ struct CastAwayExtractStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         extractOp.getStrides().getValue().drop_front(dropCount));
 
-    auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
         loc, newDstType, newSrcVector, newOffsets, newSizes, newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(extractOp, oldDstType,
@@ -120,9 +120,9 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     // Trim leading one dimensions from both operands.
     Location loc = insertOp.getLoc();
 
-    Value newSrcVector = rewriter.create<vector::ExtractOp>(
+    Value newSrcVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getValueToStore(), splatZero(srcDropCount));
-    Value newDstVector = rewriter.create<vector::ExtractOp>(
+    Value newDstVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getDest(), splatZero(dstDropCount));
 
     auto newOffsets = rewriter.getArrayAttr(
@@ -130,7 +130,7 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         insertOp.getStrides().getValue().take_back(newSrcType.getRank()));
 
-    auto newInsertOp = rewriter.create<vector::InsertStridedSliceOp>(
+    auto newInsertOp = vector::InsertStridedSliceOp::create(rewriter,
         loc, newDstType, newSrcVector, newDstVector, newOffsets, newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
@@ -169,10 +169,10 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
 
     Value newSrcVector = insertOp.getValueToStore();
     if (oldSrcRank != 0) {
-      newSrcVector = rewriter.create<vector::ExtractOp>(
+      newSrcVector = vector::ExtractOp::create(rewriter,
           loc, insertOp.getValueToStore(), splatZero(srcDropCount));
     }
-    Value newDstVector = rewriter.create<vector::ExtractOp>(
+    Value newDstVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getDest(), splatZero(dstDropCount));
 
     // New position rank needs to be computed in two steps: (1) if destination
@@ -187,7 +187,7 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
     newPosition.resize(newDstType.getRank() - newSrcRank,
                        rewriter.getI64IntegerAttr(0));
 
-    auto newInsertOp = rewriter.create<vector::InsertOp>(
+    auto newInsertOp = vector::InsertOp::create(rewriter,
         loc, newSrcVector, newDstVector, newPosition);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
@@ -209,9 +209,9 @@ static Value dropUnitDimsFromMask(OpBuilder &b, Location loc, Value mask,
   if (vector::isBroadcastableTo(newMaskType, oldMaskType) ==
       BroadcastableToResult::Success) {
     int64_t dropDim = oldMaskType.getRank() - newMaskType.getRank();
-    return b.create<vector::ExtractOp>(loc, mask, splatZero(dropDim));
+    return vector::ExtractOp::create(b, loc, mask, splatZero(dropDim));
   }
-  return b.create<vector::ShapeCastOp>(loc, newMaskType, mask);
+  return vector::ShapeCastOp::create(b, loc, newMaskType, mask);
 }
 
 // Turns vector.transfer_read on vector with leading 1 dimensions into
@@ -259,7 +259,7 @@ struct CastAwayTransferReadLeadingOneDim
                                   newType, newMap, maskType);
     }
 
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         read.getLoc(), newType, read.getBase(), read.getIndices(),
         AffineMapAttr::get(newMap), read.getPadding(), mask, inBoundsAttr);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(read, oldType, newRead);
@@ -306,7 +306,7 @@ struct CastAwayTransferWriteLeadingOneDim
       inBoundsAttr = rewriter.getArrayAttr(
           write.getInBoundsAttr().getValue().take_back(newType.getRank()));
 
-    auto newVector = rewriter.create<vector::ExtractOp>(
+    auto newVector = vector::ExtractOp::create(rewriter,
         write.getLoc(), write.getVector(), splatZero(dropDim));
 
     if (write.getMask()) {
@@ -444,20 +444,20 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
     // Extract if its a valid extraction, otherwise use the operand
     // without extraction.
     newOperands.push_back(
-        validExtract ? rewriter.create<vector::ExtractOp>(
+        validExtract ? vector::ExtractOp::create(rewriter,
                            loc, operands[it.index()], splatZero(dropDim))
                      : operands[it.index()]);
   }
 
   // Depending on whether this vector.contract is masked, the replacing Op
   // should either be a new vector.contract Op or vector.mask Op.
-  Operation *newOp = rewriter.create<vector::ContractionOp>(
+  Operation *newOp = vector::ContractionOp::create(rewriter,
       loc, newOperands[0], newOperands[1], newOperands[2],
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind());
 
   if (maskingOp) {
-    auto newMask = rewriter.create<vector::ExtractOp>(loc, maskingOp.getMask(),
+    auto newMask = vector::ExtractOp::create(rewriter, loc, maskingOp.getMask(),
                                                       splatZero(dropDim));
 
     newOp = mlir::vector::maskOperation(rewriter, newOp, newMask);
@@ -519,7 +519,7 @@ class CastAwayElementwiseLeadingOneDim : public RewritePattern {
     SmallVector<Value, 4> newOperands;
     for (Value operand : op->getOperands()) {
       if (auto opVecType = dyn_cast<VectorType>(operand.getType())) {
-        newOperands.push_back(rewriter.create<vector::ExtractOp>(
+        newOperands.push_back(vector::ExtractOp::create(rewriter,
             op->getLoc(), operand, splatZero(dropDim)));
       } else {
         newOperands.push_back(operand);
@@ -559,7 +559,7 @@ struct CastAwayConstantMaskLeadingOneDim
     SmallVector<int64_t> newDimSizes = {flatLeadingSize};
     newDimSizes.append(dimSizes.begin() + dropDim + 1, dimSizes.end());
 
-    auto newMask = rewriter.create<vector::ConstantMaskOp>(
+    auto newMask = vector::ConstantMaskOp::create(rewriter,
         mask.getLoc(), newType, newDimSizes);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(mask, oldType, newMask);
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
index 8cc7008d80b3e..3411e010f3499 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
@@ -65,26 +65,26 @@ struct VectorMaskedLoadOpConverter final
     Value base = maskedLoadOp.getBase();
     Value iValue = maskedLoadOp.getPassThru();
     auto indices = llvm::to_vector_of<Value>(maskedLoadOp.getIndices());
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, indexType, IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
-      auto maskBit = rewriter.create<vector::ExtractOp>(loc, mask, i);
+      auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = rewriter.create<scf::IfOp>(
+      auto ifOp = scf::IfOp::create(rewriter,
           loc, maskBit,
           [&](OpBuilder &builder, Location loc) {
             auto loadedValue =
-                builder.create<memref::LoadOp>(loc, base, indices);
+                memref::LoadOp::create(builder, loc, base, indices);
             auto combinedValue =
-                builder.create<vector::InsertOp>(loc, loadedValue, iValue, i);
-            builder.create<scf::YieldOp>(loc, combinedValue.getResult());
+                vector::InsertOp::create(builder, loc, loadedValue, iValue, i);
+            scf::YieldOp::create(builder, loc, combinedValue.getResult());
           },
           [&](OpBuilder &builder, Location loc) {
-            builder.create<scf::YieldOp>(loc, iValue);
+            scf::YieldOp::create(builder, loc, iValue);
           });
       iValue = ifOp.getResult(0);
 
-      indices.back() = rewriter.create<arith::AddIOp>(loc, indices.back(), one);
+      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.replaceOp(maskedLoadOp, iValue);
@@ -132,18 +132,18 @@ struct VectorMaskedStoreOpConverter final
     Value base = maskedStoreOp.getBase();
     Value value = maskedStoreOp.getValueToStore();
     auto indices = llvm::to_vector_of<Value>(maskedStoreOp.getIndices());
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, indexType, IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
-      auto maskBit = rewriter.create<vector::ExtractOp>(loc, mask, i);
+      auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = rewriter.create<scf::IfOp>(loc, maskBit, /*else=*/false);
+      auto ifOp = scf::IfOp::create(rewriter, loc, maskBit, /*else=*/false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      auto extractedValue = rewriter.create<vector::ExtractOp>(loc, value, i);
-      rewriter.create<memref::StoreOp>(loc, extractedValue, base, indices);
+      auto extractedValue = vector::ExtractOp::create(rewriter, loc, value, i);
+      memref::StoreOp::create(rewriter, loc, extractedValue, base, indices);
 
       rewriter.setInsertionPointAfter(ifOp);
-      indices.back() = rewriter.create<arith::AddIOp>(loc, indices.back(), one);
+      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.eraseOp(maskedStoreOp);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 004beadc9ec7d..12b548bcee646 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -132,7 +132,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 SmallVector<Value> newMaskOperands(maskOperands.drop_back());
                 newMaskOperands.push_back(
                     getValueOrCreateConstantIndexOp(rewriter, loc, maskIndex));
-                return rewriter.create<vector::CreateMaskOp>(loc, newMaskType,
+                return vector::CreateMaskOp::create(rewriter, loc, newMaskType,
                                                              newMaskOperands);
               })
           .Case<vector::ConstantMaskOp>(
@@ -143,7 +143,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 int64_t &maskIndex = maskDimSizes.back();
                 maskIndex = llvm::divideCeil(numFrontPadElems + maskIndex,
                                              numSrcElemsPerDest);
-                return rewriter.create<vector::ConstantMaskOp>(loc, newMaskType,
+                return vector::ConstantMaskOp::create(rewriter, loc, newMaskType,
                                                                maskDimSizes);
               })
           .Case<arith::ConstantOp>([&](auto constantOp)
@@ -182,7 +182,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
               }
               compressedMaskValues.push_back(combinedValue);
             }
-            return rewriter.create<arith::ConstantOp>(
+            return arith::ConstantOp::create(rewriter,
                 loc, DenseElementsAttr::get(newMaskType, compressedMaskValues));
           });
 
@@ -190,7 +190,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
     return failure();
 
   while (!extractOps.empty()) {
-    newMask = rewriter.create<vector::ExtractOp>(
+    newMask = vector::ExtractOp::create(rewriter,
         loc, (*newMask)->getResults()[0], extractOps.back().getMixedPosition());
     extractOps.pop_back();
   }
@@ -258,7 +258,7 @@ static Value staticallyInsertSubvector(OpBuilder &rewriter, Location loc,
 
   auto offsets = rewriter.getI64ArrayAttr({offset});
   auto strides = rewriter.getI64ArrayAttr({1});
-  return rewriter.create<vector::InsertStridedSliceOp>(loc, destVecTy, src,
+  return vector::InsertStridedSliceOp::create(rewriter, loc, destVecTy, src,
                                                        dest, offsets, strides);
 }
 
@@ -301,11 +301,11 @@ static Value dynamicallyExtractSubVector(OpBuilder &rewriter, Location loc,
   for (int i = 0; i < numElemsToExtract; ++i) {
     Value extractLoc =
         (i == 0) ? dyn_cast<Value>(offset)
-                 : rewriter.create<arith::AddIOp>(
+                 : arith::AddIOp::create(rewriter,
                        loc, rewriter.getIndexType(), dyn_cast<Value>(offset),
-                       rewriter.create<arith::ConstantIndexOp>(loc, i));
-    auto extractOp = rewriter.create<vector::ExtractOp>(loc, src, extractLoc);
-    dest = rewriter.create<vector::InsertOp>(loc, extractOp, dest, i);
+                       arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto extractOp = vector::ExtractOp::create(rewriter, loc, src, extractLoc);
+    dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, i);
   }
   return dest;
 }
@@ -346,11 +346,11 @@ static Value dynamicallyInsertSubVector(RewriterBase &rewriter, Location loc,
   for (int64_t i = 0; i < numElemsToInsert; ++i) {
     auto insertLoc = i == 0
                          ? destOffsetVal
-                         : rewriter.create<arith::AddIOp>(
+                         : arith::AddIOp::create(rewriter,
                                loc, rewriter.getIndexType(), destOffsetVal,
-                               rewriter.create<arith::ConstantIndexOp>(loc, i));
-    auto extractOp = rewriter.create<vector::ExtractOp>(loc, src, i);
-    dest = rewriter.create<vector::InsertOp>(loc, extractOp, dest, insertLoc);
+                               arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto extractOp = vector::ExtractOp::create(rewriter, loc, src, i);
+    dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, insertLoc);
   }
   return dest;
 }
@@ -369,10 +369,10 @@ static VectorValue emulatedVectorLoad(OpBuilder &rewriter, Location loc,
                                       Type containerElemTy) {
   auto emulatedPerContainerElem = containerElemTy.getIntOrFloatBitWidth() /
                                   emulatedElemTy.getIntOrFloatBitWidth();
-  auto newLoad = rewriter.create<vector::LoadOp>(
+  auto newLoad = vector::LoadOp::create(rewriter,
       loc, VectorType::get(numContainerElemsToLoad, containerElemTy), base,
       getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
-  return rewriter.create<vector::BitCastOp>(
+  return vector::BitCastOp::create(rewriter,
       loc,
       VectorType::get(numContainerElemsToLoad * emulatedPerContainerElem,
                       emulatedElemTy),
@@ -390,16 +390,16 @@ static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
           upcastType.getNumElements() * upcastType.getElementTypeBitWidth() &&
       "expected input and output number of bits to match");
   if (trueValue.getType() != downcastType) {
-    trueValue = builder.create<vector::BitCastOp>(loc, downcastType, trueValue);
+    trueValue = vector::BitCastOp::create(builder, loc, downcastType, trueValue);
   }
   if (falseValue.getType() != downcastType) {
     falseValue =
-        builder.create<vector::BitCastOp>(loc, downcastType, falseValue);
+        vector::BitCastOp::create(builder, loc, downcastType, falseValue);
   }
   Value selectedType =
-      builder.create<arith::SelectOp>(loc, mask, trueValue, falseValue);
+      arith::SelectOp::create(builder, loc, mask, trueValue, falseValue);
   // Upcast the selected value to the new type.
-  return builder.create<vector::BitCastOp>(loc, upcastType, selectedType);
+  return vector::BitCastOp::create(builder, loc, upcastType, selectedType);
 }
 
 /// Emits `memref.generic_atomic_rmw` op to store a subbyte-sized value to a
@@ -422,7 +422,7 @@ static void atomicRMW(OpBuilder &builder, Location loc,
 
   // Create an atomic load-modify-write region using
   // `memref.generic_atomic_rmw`.
-  auto atomicOp = builder.create<memref::GenericAtomicRMWOp>(
+  auto atomicOp = memref::GenericAtomicRMWOp::create(builder,
       loc, linearizedMemref, ValueRange{storeIdx});
   Value origValue = atomicOp.getCurrentValue();
 
@@ -432,7 +432,7 @@ static void atomicRMW(OpBuilder &builder, Location loc,
   // Load the original value from memory, and cast it to the original element
   // type.
   auto oneElemVecType = VectorType::get({1}, origValue.getType());
-  Value origVecValue = builder.create<vector::FromElementsOp>(
+  Value origVecValue = vector::FromElementsOp::create(builder,
       loc, oneElemVecType, ValueRange{origValue});
 
   // Construct the final masked value and yield it.
@@ -440,8 +440,8 @@ static void atomicRMW(OpBuilder &builder, Location loc,
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
   auto scalarMaskedValue =
-      builder.create<vector::ExtractOp>(loc, maskedValue, 0);
-  builder.create<memref::AtomicYieldOp>(loc, scalarMaskedValue);
+      vector::ExtractOp::create(builder, loc, maskedValue, 0);
+  memref::AtomicYieldOp::create(builder, loc, scalarMaskedValue);
 }
 
 /// Generate a non-atomic read-modify-write sequence for storing to the emulated
@@ -453,15 +453,15 @@ static void nonAtomicRMW(OpBuilder &builder, Location loc,
 
   auto oneElemVecType =
       VectorType::get({1}, linearizedMemref.getType().getElementType());
-  Value origVecValue = builder.create<vector::LoadOp>(
+  Value origVecValue = vector::LoadOp::create(builder,
       loc, oneElemVecType, linearizedMemref, ValueRange{linearizedIndex});
-  origVecValue = builder.create<vector::BitCastOp>(loc, valueToStore.getType(),
+  origVecValue = vector::BitCastOp::create(builder, loc, valueToStore.getType(),
                                                    origVecValue);
 
   Value maskedValue =
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
-  builder.create<vector::StoreOp>(loc, maskedValue, linearizedMemref,
+  vector::StoreOp::create(builder, loc, maskedValue, linearizedMemref,
                                   linearizedIndex);
 }
 
@@ -489,7 +489,7 @@ static Value extractSliceIntoByte(ConversionPatternRewriter &rewriter,
   assert(8 % vectorElementType.getIntOrFloatBitWidth() == 0 &&
          "vector element must be a valid sub-byte type");
   auto emulatedPerContainerElem = 8 / vectorElementType.getIntOrFloatBitWidth();
-  auto emptyByteVector = rewriter.create<arith::ConstantOp>(
+  auto emptyByteVector = arith::ConstantOp::create(rewriter,
       loc, VectorType::get({emulatedPerContainerElem}, vectorElementType),
       rewriter.getZeroAttr(
           VectorType::get({emulatedPerContainerElem}, vectorElementType)));
@@ -602,7 +602,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         ShapedType::isDynamic(trailingDim) || trailingDim == origElements;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     // FIXME: ATM, we do not test cases where offsets, sizes, or strides are
     // non-zero. As such, this is not needed.
@@ -664,7 +664,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     if (!emulationRequiresPartialStores) {
       // Basic case: storing full bytes.
       auto numElements = origElements / emulatedPerContainerElem;
-      auto bitCast = rewriter.create<vector::BitCastOp>(
+      auto bitCast = vector::BitCastOp::create(rewriter,
           loc, VectorType::get(numElements, containerElemTy),
           op.getValueToStore());
       rewriter.replaceOpWithNewOp<vector::StoreOp>(
@@ -732,7 +732,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         std::fill_n(frontMaskValues.end() - frontSubWidthStoreElem,
                     *foldedNumFrontPadElems, true);
       }
-      auto frontMask = rewriter.create<arith::ConstantOp>(
+      auto frontMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
 
       currentSourceIndex = emulatedPerContainerElem - (*foldedNumFrontPadElems);
@@ -751,8 +751,8 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
 
     // Increment the destination index by 1 to align to the emulated width
     // boundary.
-    auto constantOne = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    currentDestIndex = rewriter.create<arith::AddIOp>(
+    auto constantOne = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    currentDestIndex = arith::AddIOp::create(rewriter,
         loc, rewriter.getIndexType(), currentDestIndex, constantOne);
 
     // 2. Full width store for the inner output bytes.
@@ -772,15 +772,15 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       auto storeType = VectorType::get(
           {originType.getNumElements() / emulatedPerContainerElem},
           memrefElemType);
-      auto bitCast = rewriter.create<vector::BitCastOp>(loc, storeType,
+      auto bitCast = vector::BitCastOp::create(rewriter, loc, storeType,
                                                         fullWidthStorePart);
-      rewriter.create<vector::StoreOp>(loc, bitCast.getResult(), memrefBase,
+      vector::StoreOp::create(rewriter, loc, bitCast.getResult(), memrefBase,
                                        currentDestIndex);
 
       currentSourceIndex += numNonFullWidthElements;
-      currentDestIndex = rewriter.create<arith::AddIOp>(
+      currentDestIndex = arith::AddIOp::create(rewriter,
           loc, rewriter.getIndexType(), currentDestIndex,
-          rewriter.create<arith::ConstantIndexOp>(loc, fullWidthStoreSize));
+          arith::ConstantIndexOp::create(rewriter, loc, fullWidthStoreSize));
     }
 
     // 3. Partial width store for the trailing output byte.
@@ -795,7 +795,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       // Generate back mask.
       auto maskValues = SmallVector<bool>(emulatedPerContainerElem, 0);
       std::fill_n(maskValues.begin(), remainingElements, 1);
-      auto backMask = rewriter.create<arith::ConstantOp>(
+      auto backMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
 
       storeFunc(rewriter, loc, memrefBase, currentDestIndex,
@@ -848,7 +848,7 @@ struct ConvertVectorMaskedStore final
       return failure();
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
     OpFoldResult linearizedIndicesOfr;
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndicesOfr) =
@@ -901,21 +901,21 @@ struct ConvertVectorMaskedStore final
     auto numElements = (origElements + emulatedPerContainerElem - 1) /
                        emulatedPerContainerElem;
     auto newType = VectorType::get(numElements, containerElemTy);
-    auto passThru = rewriter.create<arith::ConstantOp>(
+    auto passThru = arith::ConstantOp::create(rewriter,
         loc, newType, rewriter.getZeroAttr(newType));
 
-    auto newLoad = rewriter.create<vector::MaskedLoadOp>(
+    auto newLoad = vector::MaskedLoadOp::create(rewriter,
         loc, newType, adaptor.getBase(), linearizedIndices,
         newMask.value()->getResult(0), passThru);
 
     auto newBitCastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
     Value valueToStore =
-        rewriter.create<vector::BitCastOp>(loc, newBitCastType, newLoad);
-    valueToStore = rewriter.create<arith::SelectOp>(
+        vector::BitCastOp::create(rewriter, loc, newBitCastType, newLoad);
+    valueToStore = arith::SelectOp::create(rewriter,
         loc, op.getMask(), op.getValueToStore(), valueToStore);
     valueToStore =
-        rewriter.create<vector::BitCastOp>(loc, newType, valueToStore);
+        vector::BitCastOp::create(rewriter, loc, newType, valueToStore);
 
     rewriter.replaceOpWithNewOp<vector::MaskedStoreOp>(
         op, adaptor.getBase(), linearizedIndices, newMask.value()->getResult(0),
@@ -990,7 +990,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
     bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
@@ -1016,7 +1016,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
                            numElements, emulatedElemTy, containerElemTy);
 
     if (!foldedIntraVectorOffset) {
-      auto resultVector = rewriter.create<arith::ConstantOp>(
+      auto resultVector = arith::ConstantOp::create(rewriter,
           loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(
           rewriter, loc, dyn_cast<TypedValue<VectorType>>(result), resultVector,
@@ -1111,7 +1111,7 @@ struct ConvertVectorMaskedLoad final
     bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndices) =
@@ -1142,7 +1142,7 @@ struct ConvertVectorMaskedLoad final
     auto newBitcastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
 
-    auto emptyVector = rewriter.create<arith::ConstantOp>(
+    auto emptyVector = arith::ConstantOp::create(rewriter,
         loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
     if (!foldedIntraVectorOffset) {
       passthru = dynamicallyInsertSubVector(
@@ -1153,10 +1153,10 @@ struct ConvertVectorMaskedLoad final
                                            *foldedIntraVectorOffset);
     }
     auto newPassThru =
-        rewriter.create<vector::BitCastOp>(loc, loadType, passthru);
+        vector::BitCastOp::create(rewriter, loc, loadType, passthru);
 
     // Generating the new masked load.
-    auto newLoad = rewriter.create<vector::MaskedLoadOp>(
+    auto newLoad = vector::MaskedLoadOp::create(rewriter,
         loc, loadType, adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newMask.value()->getResult(0), newPassThru);
@@ -1164,13 +1164,13 @@ struct ConvertVectorMaskedLoad final
     // Setting the part that originally was not effectively loaded from memory
     // to pass through.
     auto bitCast =
-        rewriter.create<vector::BitCastOp>(loc, newBitcastType, newLoad);
+        vector::BitCastOp::create(rewriter, loc, newBitcastType, newLoad);
 
     Value mask = op.getMask();
     auto newSelectMaskType = VectorType::get(
         numElements * emulatedPerContainerElem, rewriter.getI1Type());
     // TODO: try to fold if op's mask is constant
-    auto emptyMask = rewriter.create<arith::ConstantOp>(
+    auto emptyMask = arith::ConstantOp::create(rewriter,
         loc, newSelectMaskType, rewriter.getZeroAttr(newSelectMaskType));
     if (!foldedIntraVectorOffset) {
       mask = dynamicallyInsertSubVector(rewriter, loc, mask, emptyMask,
@@ -1182,7 +1182,7 @@ struct ConvertVectorMaskedLoad final
     }
 
     Value result =
-        rewriter.create<arith::SelectOp>(loc, mask, bitCast, passthru);
+        arith::SelectOp::create(rewriter, loc, mask, bitCast, passthru);
     if (!foldedIntraVectorOffset) {
       result = dynamicallyExtractSubVector(
           rewriter, loc, result, op.getPassThru(),
@@ -1268,11 +1268,11 @@ struct ConvertVectorTransferRead final
     bool isDivisibleInSize =
         fitsInMultiByteContainerTy(op.getVectorType(), containerElemTy);
 
-    auto newPadding = rewriter.create<arith::ExtUIOp>(loc, containerElemTy,
+    auto newPadding = arith::ExtUIOp::create(rewriter, loc, containerElemTy,
                                                       adaptor.getPadding());
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
@@ -1293,19 +1293,19 @@ struct ConvertVectorTransferRead final
     auto numElements = llvm::divideCeil(maxIntraDataOffset + origElements,
                                         emulatedPerContainerElem);
 
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         loc, VectorType::get(numElements, containerElemTy), adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newPadding);
 
-    auto bitCast = rewriter.create<vector::BitCastOp>(
+    auto bitCast = vector::BitCastOp::create(rewriter,
         loc,
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy),
         newRead);
 
     Value result = bitCast->getResult(0);
     if (!foldedIntraVectorOffset) {
-      auto zeros = rewriter.create<arith::ConstantOp>(
+      auto zeros = arith::ConstantOp::create(rewriter,
           loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(rewriter, loc, bitCast, zeros,
                                            linearizedInfo.intraDataOffset,
@@ -1679,32 +1679,32 @@ Value BitCastRewriter::genericRewriteStep(
     PatternRewriter &rewriter, Location loc, Value initialValue,
     Value runningResult, const BitCastRewriter::Metadata &metadata) {
   // Create vector.shuffle from the metadata.
-  auto shuffleOp = rewriter.create<vector::ShuffleOp>(
+  auto shuffleOp = vector::ShuffleOp::create(rewriter,
       loc, initialValue, initialValue, metadata.shuffles);
 
   // Intersect with the mask.
   VectorType shuffledVectorType = shuffleOp.getResultVectorType();
-  auto constOp = rewriter.create<arith::ConstantOp>(
+  auto constOp = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(shuffledVectorType, metadata.masks));
-  Value andValue = rewriter.create<arith::AndIOp>(loc, shuffleOp, constOp);
+  Value andValue = arith::AndIOp::create(rewriter, loc, shuffleOp, constOp);
 
   // Align right on 0.
-  auto shiftRightConstantOp = rewriter.create<arith::ConstantOp>(
+  auto shiftRightConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftRightAmounts));
   Value shiftedRight =
-      rewriter.create<arith::ShRUIOp>(loc, andValue, shiftRightConstantOp);
+      arith::ShRUIOp::create(rewriter, loc, andValue, shiftRightConstantOp);
 
   // Shift bits left into their final position.
-  auto shiftLeftConstantOp = rewriter.create<arith::ConstantOp>(
+  auto shiftLeftConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftLeftAmounts));
   Value shiftedLeft =
-      rewriter.create<arith::ShLIOp>(loc, shiftedRight, shiftLeftConstantOp);
+      arith::ShLIOp::create(rewriter, loc, shiftedRight, shiftLeftConstantOp);
 
   runningResult =
       runningResult
-          ? rewriter.create<arith::OrIOp>(loc, runningResult, shiftedLeft)
+          ? arith::OrIOp::create(rewriter, loc, runningResult, shiftedLeft)
           : shiftedLeft;
 
   return runningResult;
@@ -1727,7 +1727,7 @@ static Value bitcastSubByteVectorToI8(PatternRewriter &rewriter, Location loc,
   // Adjust last dimension of the vector, so the total size remains the same.
   vecShape.back() = vecShape.back() / numSrcElemsPerByte;
   auto i8VecType = VectorType::get(vecShape, rewriter.getI8Type());
-  return rewriter.create<vector::BitCastOp>(loc, i8VecType, subByteVec);
+  return vector::BitCastOp::create(rewriter, loc, i8VecType, subByteVec);
 }
 
 /// Extracts a signed N-bit sequence from each element of a vector of bytes,
@@ -1755,15 +1755,15 @@ static Value extractNBitsPerByteAndSignExtendToI8(PatternRewriter &rewriter,
   assert(bitIdx >= 0 && bitsToShiftLeft >= 0 && numBits > 0 && numBits <= 8 &&
          "Invalid bitIdx range");
   if (bitsToShiftLeft != 0) {
-    Value shiftLeftValues = rewriter.create<arith::ConstantOp>(
+    Value shiftLeftValues = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(srcType, bitsToShiftLeft));
-    shl = rewriter.create<arith::ShLIOp>(loc, src, shiftLeftValues);
+    shl = arith::ShLIOp::create(rewriter, loc, src, shiftLeftValues);
   }
 
   int8_t bitsToShiftRight = 8 - numBits;
-  Value shiftRightValues = rewriter.create<arith::ConstantOp>(
+  Value shiftRightValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
-  Value shr = rewriter.create<arith::ShRSIOp>(loc, shl, shiftRightValues);
+  Value shr = arith::ShRSIOp::create(rewriter, loc, shl, shiftRightValues);
   return shr;
 }
 
@@ -1797,17 +1797,17 @@ static Value extractNBitsPerByteAndExtendToI8(PatternRewriter &rewriter,
   int8_t bitsToShiftRight = bitIdx;
   Value shr = src;
   if (bitsToShiftRight != 0) {
-    Value shiftRightValues = rewriter.create<arith::ConstantOp>(
+    Value shiftRightValues = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
-    shr = rewriter.create<arith::ShRUIOp>(loc, src, shiftRightValues);
+    shr = arith::ShRUIOp::create(rewriter, loc, src, shiftRightValues);
   }
   if (bitIdx + numBits == 8) {
     return shr;
   }
   uint8_t lowBitsMask = (1 << numBits) - 1;
-  Value lowBitsMaskValues = rewriter.create<arith::ConstantOp>(
+  Value lowBitsMaskValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(srcType, lowBitsMask));
-  return rewriter.create<arith::AndIOp>(loc, shr, lowBitsMaskValues);
+  return arith::AndIOp::create(rewriter, loc, shr, lowBitsMaskValues);
 }
 
 using ExtractNBitsFn =
@@ -1830,7 +1830,7 @@ static Value rewriteI4ToI8Ext(PatternRewriter &rewriter, Location loc,
   Value high = extFn(rewriter, loc, i8Vector, 4, 4);
 
   // 3. Interleave low and high i8 elements.
-  return rewriter.create<vector::InterleaveOp>(loc, low, high);
+  return vector::InterleaveOp::create(rewriter, loc, low, high);
 }
 
 /// Rewrite the i2 -> i8  extension into a sequence of shuffles and
@@ -1863,9 +1863,9 @@ static Value rewriteI2ToI8Ext(PatternRewriter &rewriter, Location loc,
   // 02    = [0,2,0,2,0,2,0,2],...
   // 13    = [1,3,1,3,1,3,1,3],...
   // 0213  = [0,1,2,3,...],...
-  Value interleave02 = rewriter.create<vector::InterleaveOp>(loc, vec0, vec2);
-  Value interleave13 = rewriter.create<vector::InterleaveOp>(loc, vec1, vec3);
-  return rewriter.create<vector::InterleaveOp>(loc, interleave02, interleave13);
+  Value interleave02 = vector::InterleaveOp::create(rewriter, loc, vec0, vec2);
+  Value interleave13 = vector::InterleaveOp::create(rewriter, loc, vec1, vec3);
+  return vector::InterleaveOp::create(rewriter, loc, interleave02, interleave13);
 }
 
 /// Rewrite the i8 -> i4 truncation into a deinterleave and series of bitwise
@@ -1877,29 +1877,29 @@ static Value rewriteI8ToI4Trunc(PatternRewriter &rewriter, Location loc,
          "Expected i8 type");
 
   // 1. De-interleave low and high i8 elements.
-  auto deinterleaveOp = rewriter.create<vector::DeinterleaveOp>(loc, srcValue);
+  auto deinterleaveOp = vector::DeinterleaveOp::create(rewriter, loc, srcValue);
 
   // 2. Zero out the upper side of each low i8 element.
   constexpr int8_t i8LowBitMask = 0x0F;
   VectorType deinterI8VecType = deinterleaveOp.getResultVectorType();
-  Value zeroOutMask = rewriter.create<arith::ConstantOp>(
+  Value zeroOutMask = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(deinterI8VecType, i8LowBitMask));
-  Value zeroOutLow = rewriter.create<arith::AndIOp>(
+  Value zeroOutLow = arith::AndIOp::create(rewriter,
       loc, deinterleaveOp.getRes1(), zeroOutMask);
 
   // 3. Move high i4 values to upper side of the byte.
   constexpr int8_t bitsToShift = 4;
-  auto shiftValues = rewriter.create<arith::ConstantOp>(
+  auto shiftValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(deinterI8VecType, bitsToShift));
-  Value shlHigh = rewriter.create<arith::ShLIOp>(loc, deinterleaveOp.getRes2(),
+  Value shlHigh = arith::ShLIOp::create(rewriter, loc, deinterleaveOp.getRes2(),
                                                  shiftValues);
 
   // 4. Merge high and low i4 values.
-  auto mergedHiLowOp = rewriter.create<arith::OrIOp>(loc, zeroOutLow, shlHigh);
+  auto mergedHiLowOp = arith::OrIOp::create(rewriter, loc, zeroOutLow, shlHigh);
 
   // 5. Generate a bitcast vector<Xxi8> -> vector<2Xxi4>.
   auto i4VecType = srcVecType.cloneWith(std::nullopt, rewriter.getI4Type());
-  return rewriter.create<vector::BitCastOp>(loc, i4VecType, mergedHiLowOp);
+  return vector::BitCastOp::create(rewriter, loc, i4VecType, mergedHiLowOp);
 }
 
 namespace {
@@ -2141,7 +2141,7 @@ struct RewriteAlignedSubByteIntTrunc : OpRewritePattern<arith::TruncIOp> {
     Location loc = truncOp.getLoc();
     auto i8VecType = srcVecType.cloneWith(std::nullopt, rewriter.getI8Type());
     Value i8TruncVal =
-        rewriter.create<arith::TruncIOp>(loc, i8VecType, srcValue);
+        arith::TruncIOp::create(rewriter, loc, i8VecType, srcValue);
 
     // Rewrite the i8 -> i4 truncation part.
     Value subByteTrunc = rewriteI8ToI4Trunc(rewriter, loc, i8TruncVal);
@@ -2189,9 +2189,9 @@ struct RewriteVectorTranspose : OpRewritePattern<vector::TransposeOp> {
     // support is available.
     auto srcNativeVecType = srcSubByteVecType.cloneWith(
         std::nullopt, rewriter.getIntegerType(minNativeBitwidth));
-    Value extOp = rewriter.create<arith::ExtSIOp>(loc, srcNativeVecType,
+    Value extOp = arith::ExtSIOp::create(rewriter, loc, srcNativeVecType,
                                                   transposeOp.getVector());
-    Value newTranspose = rewriter.create<vector::TransposeOp>(
+    Value newTranspose = vector::TransposeOp::create(rewriter,
         loc, extOp, transposeOp.getPermutation());
     VectorType dstSubByteVecType = transposeOp.getResultVectorType();
     rewriter.replaceOpWithNewOp<arith::TruncIOp>(transposeOp, dstSubByteVecType,
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
index d834a99076834..baeca29298334 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
@@ -53,14 +53,14 @@ class DecomposeDifferentRankInsertStridedSlice
     int64_t rankRest = dstType.getRank() - rankDiff;
     // Extract / insert the subvector of matching rank and InsertStridedSlice
     // on it.
-    Value extracted = rewriter.create<ExtractOp>(
+    Value extracted = ExtractOp::create(rewriter,
         loc, op.getDest(),
         getI64SubArray(op.getOffsets(), /*dropFront=*/0,
                        /*dropBack=*/rankRest));
 
     // A different pattern will kick in for InsertStridedSlice with matching
     // ranks.
-    auto stridedSliceInnerOp = rewriter.create<InsertStridedSliceOp>(
+    auto stridedSliceInnerOp = InsertStridedSliceOp::create(rewriter,
         loc, op.getValueToStore(), extracted,
         getI64SubArray(op.getOffsets(), /*dropFront=*/rankDiff),
         getI64SubArray(op.getStrides(), /*dropFront=*/0));
@@ -131,7 +131,7 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
       SmallVector<int64_t> offsets(nDest, 0);
       for (int64_t i = 0; i < nSrc; ++i)
         offsets[i] = i;
-      Value scaledSource = rewriter.create<ShuffleOp>(
+      Value scaledSource = ShuffleOp::create(rewriter,
           loc, op.getValueToStore(), op.getValueToStore(), offsets);
 
       // 2. Create a mask where we take the value from scaledSource of dest
@@ -156,21 +156,21 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
          off += stride, ++idx) {
       // 1. extract the proper subvector (or element) from source
       Value extractedSource =
-          rewriter.create<ExtractOp>(loc, op.getValueToStore(), idx);
+          ExtractOp::create(rewriter, loc, op.getValueToStore(), idx);
       if (isa<VectorType>(extractedSource.getType())) {
         // 2. If we have a vector, extract the proper subvector from destination
         // Otherwise we are at the element level and no need to recurse.
         Value extractedDest =
-            rewriter.create<ExtractOp>(loc, op.getDest(), off);
+            ExtractOp::create(rewriter, loc, op.getDest(), off);
         // 3. Reduce the problem to lowering a new InsertStridedSlice op with
         // smaller rank.
-        extractedSource = rewriter.create<InsertStridedSliceOp>(
+        extractedSource = InsertStridedSliceOp::create(rewriter,
             loc, extractedSource, extractedDest,
             getI64SubArray(op.getOffsets(), /* dropFront=*/1),
             getI64SubArray(op.getStrides(), /* dropFront=*/1));
       }
       // 4. Insert the extractedSource into the res vector.
-      res = rewriter.create<InsertOp>(loc, extractedSource, res, off);
+      res = InsertOp::create(rewriter, loc, extractedSource, res, off);
     }
 
     rewriter.replaceOp(op, res);
@@ -250,12 +250,12 @@ class Convert1DExtractStridedSliceIntoExtractInsertChain final
     SmallVector<Value> elements;
     elements.reserve(size);
     for (int64_t i = offset, e = offset + size * stride; i < e; i += stride)
-      elements.push_back(rewriter.create<ExtractOp>(loc, op.getVector(), i));
+      elements.push_back(ExtractOp::create(rewriter, loc, op.getVector(), i));
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(op.getType()));
     for (int64_t i = 0; i < size; ++i)
-      result = rewriter.create<InsertOp>(loc, elements[i], result, i);
+      result = InsertOp::create(rewriter, loc, elements[i], result, i);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -301,17 +301,17 @@ class DecomposeNDExtractStridedSlice
       return failure();
 
     // Extract/insert on a lower ranked extract strided slice op.
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value res = rewriter.create<SplatOp>(loc, dstType, zero);
+    Value res = SplatOp::create(rewriter, loc, dstType, zero);
     for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
          off += stride, ++idx) {
-      Value one = rewriter.create<ExtractOp>(loc, op.getVector(), off);
-      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+      Value one = ExtractOp::create(rewriter, loc, op.getVector(), off);
+      Value extracted = ExtractStridedSliceOp::create(rewriter,
           loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
           getI64SubArray(op.getSizes(), /* dropFront=*/1),
           getI64SubArray(op.getStrides(), /* dropFront=*/1));
-      res = rewriter.create<InsertOp>(loc, extracted, res, idx);
+      res = InsertOp::create(rewriter, loc, extracted, res, idx);
     }
     rewriter.replaceOp(op, res);
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
index 7cac1cbafdd64..880506142bfdd 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
@@ -634,7 +634,7 @@ struct LinearizeVectorCreateMask final
     // The result of the comparison is then multiplied with
     // the second operand of create_mask to get the 1D mask.
     auto firstOperand = adaptor.getOperands().front();
-    auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto isNonZero = rewriter.createOrFold<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::sgt, firstOperand, zero);
     auto isNonZeroIndex = rewriter.createOrFold<mlir::arith::IndexCastOp>(
@@ -644,7 +644,7 @@ struct LinearizeVectorCreateMask final
         loc, rewriter.getIndexType(), isNonZeroIndex, secondOperand);
 
     auto newMask =
-        rewriter.create<mlir::vector::CreateMaskOp>(loc, dstTy, maskSize);
+        mlir::vector::CreateMaskOp::create(rewriter, loc, dstTy, maskSize);
     rewriter.replaceOp(createMaskOp, newMask);
     return success();
   }
@@ -721,7 +721,7 @@ void mlir::vector::populateForVectorLinearize(TypeConverter &typeConverter,
     if (!isa<VectorType>(type) || !isa<VectorType>(value.getType()))
       return nullptr;
 
-    return builder.create<vector::ShapeCastOp>(loc, type, value);
+    return vector::ShapeCastOp::create(builder, loc, type, value);
   };
   typeConverter.addSourceMaterialization(materializeCast);
   typeConverter.addTargetMaterialization(materializeCast);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
index 363108238e596..70d786a516859 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
@@ -84,7 +84,7 @@ LogicalResult resolveAllTrueCreateMaskOp(IRRewriter &rewriter,
   // Replace createMaskOp with an all-true constant. This should result in the
   // mask being removed in most cases (as xfer ops + vector.mask have folds to
   // remove all-true masks).
-  auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+  auto allTrue = vector::ConstantMaskOp::create(rewriter,
       createMaskOp.getLoc(), maskType, ConstantMaskKind::AllTrue);
   rewriter.replaceAllUsesWith(createMaskOp, allTrue);
   return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index efdae93e730bd..aa38fc20dfb70 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -286,7 +286,7 @@ static Value rankReducingSubviewDroppingUnitDims(PatternRewriter &rewriter,
   if (resultType.canonicalizeStridedLayout() ==
       inputType.canonicalizeStridedLayout())
     return input;
-  return rewriter.create<memref::SubViewOp>(loc, resultType, input, offsets,
+  return memref::SubViewOp::create(rewriter, loc, resultType, input, offsets,
                                             sizes, strides);
 }
 
@@ -395,11 +395,11 @@ class TransferReadDropUnitDimsPattern
 
     Value reducedShapeSource =
         rankReducingSubviewDroppingUnitDims(rewriter, loc, source);
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
-    Operation *newTransferReadOp = rewriter.create<vector::TransferReadOp>(
+    Operation *newTransferReadOp = vector::TransferReadOp::create(rewriter,
         loc, reducedVectorType, reducedShapeSource, zeros, identityMap,
         transferReadOp.getPadding(), maskOp,
         rewriter.getBoolArrayAttr(inBounds));
@@ -477,13 +477,13 @@ class TransferWriteDropUnitDimsPattern
     }
     Value reducedShapeSource =
         rankReducingSubviewDroppingUnitDims(rewriter, loc, source);
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
     auto shapeCastSrc = rewriter.createOrFold<vector::ShapeCastOp>(
         loc, reducedVectorType, vector);
-    Operation *newXferWrite = rewriter.create<vector::TransferWriteOp>(
+    Operation *newXferWrite = vector::TransferWriteOp::create(rewriter,
         loc, Type(), shapeCastSrc, reducedShapeSource, zeros, identityMap,
         maskOp, rewriter.getBoolArrayAttr(inBounds));
 
@@ -520,7 +520,7 @@ static Value collapseInnerDims(PatternRewriter &rewriter, mlir::Location loc,
   for (int64_t i = firstDimToCollapse; i < inputType.getRank(); ++i)
     collapsedIndices.push_back(i);
   reassociation.push_back(collapsedIndices);
-  return rewriter.create<memref::CollapseShapeOp>(loc, input, reassociation);
+  return memref::CollapseShapeOp::create(rewriter, loc, input, reassociation);
 }
 
 /// Returns the new indices that collapses the inner dimensions starting from
@@ -559,7 +559,7 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   // one would get the following offset:
   //    %offset = %arg0 * 43
   OpFoldResult collapsedOffset =
-      rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult();
+      arith::ConstantIndexOp::create(rewriter, loc, 0).getResult();
 
   auto collapsedStrides = computeSuffixProduct(
       ArrayRef<int64_t>(shape.begin() + firstDimToCollapse, shape.end()));
@@ -573,7 +573,7 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   if (auto value = dyn_cast<Value>(collapsedOffset)) {
     indicesAfterCollapsing.push_back(value);
   } else {
-    indicesAfterCollapsing.push_back(rewriter.create<arith::ConstantIndexOp>(
+    indicesAfterCollapsing.push_back(arith::ConstantIndexOp::create(rewriter,
         loc, *getConstantIntValue(collapsedOffset)));
   }
 
@@ -659,7 +659,7 @@ class FlattenContiguousRowMajorTransferReadPattern
     // 3. Create new vector.transfer_read that reads from the collapsed memref
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
-    vector::TransferReadOp flatRead = rewriter.create<vector::TransferReadOp>(
+    vector::TransferReadOp flatRead = vector::TransferReadOp::create(rewriter,
         loc, flatVectorType, collapsedSource, collapsedIndices,
         transferReadOp.getPadding(), collapsedMap);
     flatRead.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
@@ -757,9 +757,9 @@ class FlattenContiguousRowMajorTransferWritePattern
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
     Value flatVector =
-        rewriter.create<vector::ShapeCastOp>(loc, flatVectorType, vector);
+        vector::ShapeCastOp::create(rewriter, loc, flatVectorType, vector);
     vector::TransferWriteOp flatWrite =
-        rewriter.create<vector::TransferWriteOp>(
+        vector::TransferWriteOp::create(rewriter,
             loc, flatVector, collapsedSource, collapsedIndices, collapsedMap);
     flatWrite.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
@@ -855,7 +855,7 @@ class RewriteScalarExtractElementOfTransferRead
         newIndices[newIndices.size() - 1] = value;
       } else {
         newIndices[newIndices.size() - 1] =
-            rewriter.create<arith::ConstantIndexOp>(loc,
+            arith::ConstantIndexOp::create(rewriter, loc,
                                                     *getConstantIntValue(ofr));
       }
     }
@@ -917,7 +917,7 @@ class RewriteScalarExtractOfTransferRead
       if (auto value = dyn_cast<Value>(composedIdx)) {
         newIndices[idx] = value;
       } else {
-        newIndices[idx] = rewriter.create<arith::ConstantIndexOp>(
+        newIndices[idx] = arith::ConstantIndexOp::create(rewriter,
             extractOp.getLoc(), *getConstantIntValue(composedIdx));
       }
     }
@@ -952,7 +952,7 @@ class RewriteScalarWrite : public OpRewritePattern<vector::TransferWriteOp> {
       return failure();
     // Only float and integer element types are supported.
     Value scalar =
-        rewriter.create<vector::ExtractOp>(xferOp.getLoc(), xferOp.getVector());
+        vector::ExtractOp::create(rewriter, xferOp.getLoc(), xferOp.getVector());
     // Construct a scalar store.
     if (isa<MemRefType>(xferOp.getBase().getType())) {
       rewriter.replaceOpWithNewOp<memref::StoreOp>(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
index 256c8cb69b1ba..a1f36a9744932 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
@@ -64,12 +64,12 @@ static Value createInBoundsCond(RewriterBase &b,
     if (maybeCstSum && maybeCstDimSz && *maybeCstSum <= *maybeCstDimSz)
       return;
     Value cond =
-        b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sle,
+        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sle,
                                 getValueOrCreateConstantIndexOp(b, loc, sum),
                                 getValueOrCreateConstantIndexOp(b, loc, dimSz));
     // Conjunction over all dims for which we are in-bounds.
     if (inBoundsCond)
-      inBoundsCond = b.create<arith::AndIOp>(loc, inBoundsCond, cond);
+      inBoundsCond = arith::AndIOp::create(b, loc, inBoundsCond, cond);
     else
       inBoundsCond = cond;
   });
@@ -177,11 +177,11 @@ static Value castToCompatibleMemRefType(OpBuilder &b, Value memref,
     sourceType = MemRefType::get(
         sourceType.getShape(), sourceType.getElementType(),
         sourceType.getLayout(), compatibleMemRefType.getMemorySpace());
-    res = b.create<memref::MemorySpaceCastOp>(memref.getLoc(), sourceType, res);
+    res = memref::MemorySpaceCastOp::create(b, memref.getLoc(), sourceType, res);
   }
   if (sourceType == compatibleMemRefType)
     return res;
-  return b.create<memref::CastOp>(memref.getLoc(), compatibleMemRefType, res);
+  return memref::CastOp::create(b, memref.getLoc(), compatibleMemRefType, res);
 }
 
 /// Operates under a scoped context to build the intersection between the
@@ -203,15 +203,15 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
   xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
     using MapList = ArrayRef<ArrayRef<AffineExpr>>;
     Value dimMemRef =
-        b.create<memref::DimOp>(xferOp.getLoc(), xferOp.getBase(), indicesIdx);
-    Value dimAlloc = b.create<memref::DimOp>(loc, alloc, resultIdx);
+        memref::DimOp::create(b, xferOp.getLoc(), xferOp.getBase(), indicesIdx);
+    Value dimAlloc = memref::DimOp::create(b, loc, alloc, resultIdx);
     Value index = xferOp.getIndices()[indicesIdx];
     AffineExpr i, j, k;
     bindDims(xferOp.getContext(), i, j, k);
     SmallVector<AffineMap, 4> maps =
         AffineMap::inferFromExprList(MapList{{i - j, k}}, b.getContext());
     // affine_min(%dimMemRef - %index, %dimAlloc)
-    Value affineMin = b.create<affine::AffineMinOp>(
+    Value affineMin = affine::AffineMinOp::create(b,
         loc, index.getType(), maps[0], ValueRange{dimMemRef, index, dimAlloc});
     sizes.push_back(affineMin);
   });
@@ -220,9 +220,9 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
       xferOp.getIndices(), [](Value idx) -> OpFoldResult { return idx; }));
   SmallVector<OpFoldResult> destIndices(memrefRank, b.getIndexAttr(0));
   SmallVector<OpFoldResult> strides(memrefRank, b.getIndexAttr(1));
-  auto copySrc = b.create<memref::SubViewOp>(
+  auto copySrc = memref::SubViewOp::create(b,
       loc, isaWrite ? alloc : xferOp.getBase(), srcIndices, sizes, strides);
-  auto copyDest = b.create<memref::SubViewOp>(
+  auto copyDest = memref::SubViewOp::create(b,
       loc, isaWrite ? xferOp.getBase() : alloc, destIndices, sizes, strides);
   return std::make_pair(copySrc, copyDest);
 }
@@ -251,18 +251,18 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
                             TypeRange returnTypes, Value inBoundsCond,
                             MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return b.create<scf::IfOp>(
+  return scf::IfOp::create(b,
       loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
         llvm::append_range(viewAndIndices, xferOp.getIndices());
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       },
       [&](OpBuilder &b, Location loc) {
-        b.create<linalg::FillOp>(loc, ValueRange{xferOp.getPadding()},
+        linalg::FillOp::create(b, loc, ValueRange{xferOp.getPadding()},
                                  ValueRange{alloc});
         // Take partial subview of memref which guarantees no dimension
         // overflows.
@@ -270,13 +270,13 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
         std::pair<Value, Value> copyArgs = createSubViewIntersection(
             rewriter, cast<VectorTransferOpInterface>(xferOp.getOperation()),
             alloc);
-        b.create<memref::CopyOp>(loc, copyArgs.first, copyArgs.second);
+        memref::CopyOp::create(b, loc, copyArgs.first, copyArgs.second);
         Value casted =
             castToCompatibleMemRefType(b, alloc, compatibleMemRefType);
         scf::ValueVector viewAndIndices{casted};
         viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
                               zero);
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       });
 }
 
@@ -304,22 +304,22 @@ static scf::IfOp createFullPartialVectorTransferRead(
     Value inBoundsCond, MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
   scf::IfOp fullPartialIfOp;
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return b.create<scf::IfOp>(
+  return scf::IfOp::create(b,
       loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
         llvm::append_range(viewAndIndices, xferOp.getIndices());
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       },
       [&](OpBuilder &b, Location loc) {
         Operation *newXfer = b.clone(*xferOp.getOperation());
         Value vector = cast<VectorTransferOpInterface>(newXfer).getVector();
-        b.create<memref::StoreOp>(
+        memref::StoreOp::create(b,
             loc, vector,
-            b.create<vector::TypeCastOp>(
+            vector::TypeCastOp::create(b,
                 loc, MemRefType::get({}, vector.getType()), alloc));
 
         Value casted =
@@ -327,7 +327,7 @@ static scf::IfOp createFullPartialVectorTransferRead(
         scf::ValueVector viewAndIndices{casted};
         viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
                               zero);
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       });
 }
 
@@ -351,7 +351,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
                           TypeRange returnTypes, Value inBoundsCond,
                           MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
   return b
       .create<scf::IfOp>(
@@ -361,7 +361,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
                 castToCompatibleMemRefType(b, memref, compatibleMemRefType);
             scf::ValueVector viewAndIndices{res};
             llvm::append_range(viewAndIndices, xferOp.getIndices());
-            b.create<scf::YieldOp>(loc, viewAndIndices);
+            scf::YieldOp::create(b, loc, viewAndIndices);
           },
           [&](OpBuilder &b, Location loc) {
             Value casted =
@@ -369,7 +369,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
             scf::ValueVector viewAndIndices{casted};
             viewAndIndices.insert(viewAndIndices.end(),
                                   xferOp.getTransferRank(), zero);
-            b.create<scf::YieldOp>(loc, viewAndIndices);
+            scf::YieldOp::create(b, loc, viewAndIndices);
           })
       ->getResults();
 }
@@ -391,15 +391,15 @@ static void createFullPartialLinalgCopy(RewriterBase &b,
                                         vector::TransferWriteOp xferOp,
                                         Value inBoundsCond, Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = b.create<arith::XOrIOp>(
-      loc, inBoundsCond, b.create<arith::ConstantIntOp>(loc, true, 1));
-  b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
+  auto notInBounds = arith::XOrIOp::create(b,
+      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRRewriter rewriter(b);
     std::pair<Value, Value> copyArgs = createSubViewIntersection(
         rewriter, cast<VectorTransferOpInterface>(xferOp.getOperation()),
         alloc);
-    b.create<memref::CopyOp>(loc, copyArgs.first, copyArgs.second);
-    b.create<scf::YieldOp>(loc, ValueRange{});
+    memref::CopyOp::create(b, loc, copyArgs.first, copyArgs.second);
+    scf::YieldOp::create(b, loc, ValueRange{});
   });
 }
 
@@ -420,18 +420,18 @@ static void createFullPartialVectorTransferWrite(RewriterBase &b,
                                                  Value inBoundsCond,
                                                  Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = b.create<arith::XOrIOp>(
-      loc, inBoundsCond, b.create<arith::ConstantIntOp>(loc, true, 1));
-  b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
+  auto notInBounds = arith::XOrIOp::create(b,
+      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRMapping mapping;
-    Value load = b.create<memref::LoadOp>(
+    Value load = memref::LoadOp::create(b,
         loc,
-        b.create<vector::TypeCastOp>(
+        vector::TypeCastOp::create(b,
             loc, MemRefType::get({}, xferOp.getVector().getType()), alloc),
         ValueRange());
     mapping.map(xferOp.getVector(), load);
     b.clone(*xferOp.getOperation(), mapping);
-    b.create<scf::YieldOp>(loc, ValueRange{});
+    scf::YieldOp::create(b, loc, ValueRange{});
   });
 }
 
@@ -561,7 +561,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
     b.setInsertionPointToStart(&scope->getRegion(0).front());
     auto shape = xferOp.getVectorType().getShape();
     Type elementType = xferOp.getVectorType().getElementType();
-    alloc = b.create<memref::AllocaOp>(scope->getLoc(),
+    alloc = memref::AllocaOp::create(b, scope->getLoc(),
                                        MemRefType::get(shape, elementType),
                                        ValueRange{}, b.getI64IntegerAttr(32));
   }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index bcaea1c79471f..2e54c89df7800 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -381,7 +381,7 @@ FailureOr<Value> combineContractAndBroadcast(vector::ContractionOp contractOp,
   if (getUnusedDimsBitVector({maps[0], maps[1]}).any())
     return failure();
 
-  Operation *newOp = rewriter.create<vector::ContractionOp>(
+  Operation *newOp = vector::ContractionOp::create(rewriter,
       contractOp.getLoc(), lhs, rhs, contractOp.getAcc(),
       rewriter.getAffineMapArrayAttr(maps), rewriter.getArrayAttr(iterators));
 
@@ -534,7 +534,7 @@ struct ReorderElementwiseOpsOnTranspose final
         // This is a constant. Create a reverse transpose op for it.
         auto vectorType =
             srcType.clone(cast<VectorType>(operand.getType()).getElementType());
-        srcValues.push_back(rewriter.create<vector::TransposeOp>(
+        srcValues.push_back(vector::TransposeOp::create(rewriter,
             operand.getLoc(), vectorType, operand, invOrder));
       }
     }
@@ -608,12 +608,12 @@ struct BubbleDownVectorBitCastForExtract
     // Get the single scalar (as a vector) in the source value that packs the
     // desired scalar. E.g. extract vector<1xf32> from vector<4xf32>
     Location loc = extractOp.getLoc();
-    Value packedValue = rewriter.create<vector::ExtractOp>(
+    Value packedValue = vector::ExtractOp::create(rewriter,
         loc, castOp.getSource(), index / expandRatio);
     Type packedVecType = VectorType::get(/*shape=*/{1}, packedValue.getType());
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, packedVecType, rewriter.getZeroAttr(packedVecType));
-    packedValue = rewriter.create<vector::InsertOp>(loc, packedValue, zero,
+    packedValue = vector::InsertOp::create(rewriter, loc, packedValue, zero,
                                                     /*position=*/0);
 
     // Cast it to a vector with the desired scalar's type.
@@ -621,7 +621,7 @@ struct BubbleDownVectorBitCastForExtract
     VectorType packedType =
         VectorType::get({expandRatio}, castDstType.getElementType());
     Value castedValue =
-        rewriter.create<vector::BitCastOp>(loc, packedType, packedValue);
+        vector::BitCastOp::create(rewriter, loc, packedType, packedValue);
 
     // Finally extract the desired scalar.
     rewriter.replaceOpWithNewOp<vector::ExtractOp>(extractOp, castedValue,
@@ -700,7 +700,7 @@ struct BubbleDownBitCastForStridedSliceExtract
     VectorType newExtractType =
         VectorType::get(dims, castSrcType.getElementType());
 
-    auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
         extractOp.getLoc(), newExtractType, castOp.getSource(), newOffsets,
         newSizes, extractOp.getStrides());
 
@@ -761,7 +761,7 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         isNumElemsShrink ? srcDims.back() / ratio : srcDims.back() * ratio;
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
-    auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
+    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims(insertOp.getDestVectorType().getShape());
@@ -771,7 +771,7 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(dstDims, castDstType.getElementType());
 
     // Bitcast the destination.
-    auto newCastDstOp = rewriter.create<vector::BitCastOp>(
+    auto newCastDstOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     // Generate new insert.
@@ -852,7 +852,7 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
 
-    auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
+    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims =
@@ -861,7 +861,7 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastDstType =
         VectorType::get(dstDims, castDstType.getElementType());
 
-    auto newCastDstOp = rewriter.create<vector::BitCastOp>(
+    auto newCastDstOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     rewriter.replaceOpWithNewOp<vector::InsertStridedSliceOp>(
@@ -936,9 +936,9 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
     Type elemType = castDstType.getElementType();
     assert(elemType.isSignlessIntOrIndexOrFloat());
 
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value res = rewriter.create<SplatOp>(loc, castDstType, zero);
+    Value res = SplatOp::create(rewriter, loc, castDstType, zero);
 
     SmallVector<int64_t> sliceShape = {castDstLastDim};
     SmallVector<int64_t> strides = {1};
@@ -947,12 +947,12 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
                         castDstType.getElementType());
 
     for (int i = 0, e = shrinkRatio; i < e; ++i) {
-      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+      Value extracted = ExtractStridedSliceOp::create(rewriter,
           loc, bitcastOp.getSource(), ArrayRef<int64_t>{i * castDstLastDim},
           sliceShape, strides);
       Value bitcast =
-          rewriter.create<BitCastOp>(loc, newCastDstType, extracted);
-      res = rewriter.create<InsertStridedSliceOp>(
+          BitCastOp::create(rewriter, loc, newCastDstType, extracted);
+      res = InsertStridedSliceOp::create(rewriter,
           loc, bitcast, res,
           ArrayRef<int64_t>{i * castDstLastDim / shrinkRatio}, strides);
     }
@@ -1097,7 +1097,7 @@ class ExtractOpFromElementwise final
     Location loc = eltwise->getLoc();
     SmallVector<OpFoldResult> pos = op.getMixedPosition();
     for (Value arg : eltwise->getOperands()) {
-      Value newArg = rewriter.create<vector::ExtractOp>(loc, arg, pos);
+      Value newArg = vector::ExtractOp::create(rewriter, loc, arg, pos);
       mapping.map(arg, newArg);
     }
 
@@ -1286,18 +1286,18 @@ static Value buildVectorComparison(PatternRewriter &rewriter, Operation *op,
     indicesAttr = rewriter.getI64VectorAttr(
         llvm::to_vector<4>(llvm::seq<int64_t>(0, dim)));
   }
-  Value indices = rewriter.create<arith::ConstantOp>(loc, indicesAttr);
+  Value indices = arith::ConstantOp::create(rewriter, loc, indicesAttr);
   // Add in an offset if requested.
   if (off) {
     Value o = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, *off);
-    Value ov = rewriter.create<vector::SplatOp>(loc, indices.getType(), o);
-    indices = rewriter.create<arith::AddIOp>(loc, ov, indices);
+    Value ov = vector::SplatOp::create(rewriter, loc, indices.getType(), o);
+    indices = arith::AddIOp::create(rewriter, loc, ov, indices);
   }
   // Construct the vector comparison.
   Value bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, b);
   Value bounds =
-      rewriter.create<vector::SplatOp>(loc, indices.getType(), bound);
-  return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, indices,
+      vector::SplatOp::create(rewriter, loc, indices.getType(), bound);
+  return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, indices,
                                         bounds);
 }
 
@@ -1329,15 +1329,15 @@ struct MaterializeTransferMask : public OpRewritePattern<ConcreteOp> {
     Value off = xferOp.getIndices()[lastIndex];
     Value dim =
         vector::createOrFoldDimOp(rewriter, loc, xferOp.getBase(), lastIndex);
-    Value b = rewriter.create<arith::SubIOp>(loc, dim.getType(), dim, off);
-    Value mask = rewriter.create<vector::CreateMaskOp>(
+    Value b = arith::SubIOp::create(rewriter, loc, dim.getType(), dim, off);
+    Value mask = vector::CreateMaskOp::create(rewriter,
         loc,
         VectorType::get(vtp.getShape(), rewriter.getI1Type(),
                         vtp.getScalableDims()),
         b);
     if (xferOp.getMask()) {
       // Intersect the in-bounds with the mask specified as an op parameter.
-      mask = rewriter.create<arith::AndIOp>(loc, mask, xferOp.getMask());
+      mask = arith::AndIOp::create(rewriter, loc, mask, xferOp.getMask());
     }
 
     rewriter.modifyOpInPlace(xferOp, [&]() {
@@ -1542,11 +1542,11 @@ class DropInnerMostUnitDimsTransferRead
         strides);
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         readOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
-    Value rankedReducedView = rewriter.create<memref::SubViewOp>(
+    Value rankedReducedView = memref::SubViewOp::create(rewriter,
         loc, resultMemrefType, readOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
-    Value result = rewriter.create<vector::TransferReadOp>(
+    Value result = vector::TransferReadOp::create(rewriter,
         loc, resultTargetVecType, rankedReducedView,
         readOp.getIndices().drop_back(dimsToDrop), AffineMapAttr::get(permMap),
         readOp.getPadding(),
@@ -1633,7 +1633,7 @@ class DropInnerMostUnitDimsTransferWrite
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         writeOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
 
-    Value rankedReducedView = rewriter.create<memref::SubViewOp>(
+    Value rankedReducedView = memref::SubViewOp::create(rewriter,
         loc, resultMemrefType, writeOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
@@ -1702,21 +1702,21 @@ struct CanonicalizeContractMatmulToMMT final
     auto createTranspose = [&rewriter, loc](Value mat) -> Value {
       if (auto sext = mat.getDefiningOp<arith::ExtSIOp>()) {
         Value trans =
-            rewriter.create<vector::TransposeOp>(loc, sext.getIn(), perm);
+            vector::TransposeOp::create(rewriter, loc, sext.getIn(), perm);
         VectorType newType =
             cast<VectorType>(trans.getType())
                 .clone(cast<VectorType>(mat.getType()).getElementType());
-        return rewriter.create<arith::ExtSIOp>(loc, newType, trans);
+        return arith::ExtSIOp::create(rewriter, loc, newType, trans);
       }
       if (auto zext = mat.getDefiningOp<arith::ExtUIOp>()) {
         Value trans =
-            rewriter.create<vector::TransposeOp>(loc, zext.getIn(), perm);
+            vector::TransposeOp::create(rewriter, loc, zext.getIn(), perm);
         VectorType newType =
             VectorType::get(cast<VectorType>(trans.getType()).getShape(),
                             cast<VectorType>(mat.getType()).getElementType());
-        return rewriter.create<arith::ExtUIOp>(loc, newType, trans);
+        return arith::ExtUIOp::create(rewriter, loc, newType, trans);
       }
-      return rewriter.create<vector::TransposeOp>(loc, mat, perm);
+      return vector::TransposeOp::create(rewriter, loc, mat, perm);
     };
 
     if (maps == infer({{m, k}, {k, n}, {m, n}})) {
@@ -1830,7 +1830,7 @@ struct ChainedReduction final : OpRewritePattern<vector::ReductionOp> {
       vAdd = rewriter.createOrFold<arith::AddIOp>(
           loc, parentReduction.getVector(), op.getVector());
     } else {
-      vAdd = rewriter.create<arith::AddFOp>(loc, parentReduction.getVector(),
+      vAdd = arith::AddFOp::create(rewriter, loc, parentReduction.getVector(),
                                             op.getVector());
     }
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), vAdd,
@@ -1919,7 +1919,7 @@ struct DropUnitDimFromElementwiseOps final
       if (newVType == opVectorType)
         return rewriter.notifyMatchFailure(op, "No unit dimension to remove.");
 
-      auto opSC = rewriter.create<vector::ShapeCastOp>(loc, newVType, operand);
+      auto opSC = vector::ShapeCastOp::create(rewriter, loc, newVType, operand);
       newOperands.push_back(opSC);
     }
 
@@ -1998,11 +1998,11 @@ struct DropUnitDimsFromTransposeOp final
 
     Location loc = op.getLoc();
     // Drop the unit dims via shape_cast.
-    auto dropDimsShapeCast = rewriter.create<vector::ShapeCastOp>(
+    auto dropDimsShapeCast = vector::ShapeCastOp::create(rewriter,
         loc, sourceTypeWithoutUnitDims, op.getVector());
     // Create the new transpose.
     auto transposeWithoutUnitDims =
-        rewriter.create<vector::TransposeOp>(loc, dropDimsShapeCast, newPerm);
+        vector::TransposeOp::create(rewriter, loc, dropDimsShapeCast, newPerm);
     // Restore the unit dims via shape cast.
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
         op, op.getResultVectorType(), transposeWithoutUnitDims);
@@ -2053,7 +2053,7 @@ struct DropUnitDimsFromScfForOp final : OpRewritePattern<scf::ForOp> {
 
       // Create a new ForOp with that iter operand replaced.
       auto castFn = [](OpBuilder &b, Location loc, Type type, Value source) {
-        return b.create<vector::ShapeCastOp>(loc, type, source);
+        return vector::ShapeCastOp::create(b, loc, type, source);
       };
 
       Value replacement =
@@ -2105,7 +2105,7 @@ struct ReduceRedundantZero final : OpRewritePattern<vector::ReductionOp> {
     if (!matchPattern(addLhs.getRhs(), m_AnyZeroFloat()))
       return failure();
 
-    auto newAdd = rewriter.create<arith::AddFOp>(vAdd.getLoc(), addLhs.getLhs(),
+    auto newAdd = arith::AddFOp::create(rewriter, vAdd.getLoc(), addLhs.getLhs(),
                                                  vAdd.getRhs());
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), newAdd,
                                                      op.getAcc());
@@ -2148,7 +2148,7 @@ struct BreakDownVectorReduction final : OpRewritePattern<vector::ReductionOp> {
     Location loc = op.getLoc();
     SmallVector<Value> extracted(numElems, nullptr);
     for (auto [idx, extractedElem] : llvm::enumerate(extracted))
-      extractedElem = rewriter.create<vector::ExtractOp>(
+      extractedElem = vector::ExtractOp::create(rewriter,
           loc, op.getVector(), static_cast<int64_t>(idx));
 
     Value res = extracted.front();
@@ -2228,7 +2228,7 @@ struct FoldArithToVectorOuterProduct : public OpRewritePattern<MulOpType> {
       if (!broadcastedRhs || !isValidBroadcastSource(broadcastedRhs))
         return failure();
 
-      return rewriter.create<vector::OuterProductOp>(
+      return vector::OuterProductOp::create(rewriter,
           mulOp->getLoc(), resType, broadcastedLhs.getSource(),
           broadcastedRhs.getSource(), Value(), vector::CombiningKind::ADD);
     };
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 693f4f955994d..9fa01061c1bed 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -49,7 +49,7 @@ static SmallVector<Value> sliceTransferIndices(ArrayRef<int64_t> elementOffsets,
                 getAffineConstantExpr(elementOffsets[dim.index()], ctx);
     auto map = AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, expr);
     slicedIndices[pos] =
-        builder.create<affine::AffineApplyOp>(loc, map, indices[pos]);
+        affine::AffineApplyOp::create(builder, loc, map, indices[pos]);
   }
   return slicedIndices;
 }
@@ -68,9 +68,9 @@ static SmallVector<Value> sliceLoadStoreIndices(PatternRewriter &rewriter,
   auto start = indices.size() - offsets.size();
   for (auto [i, offset] : llvm::enumerate(offsets)) {
     if (offset != 0) {
-      indices[start + i] = rewriter.create<arith::AddIOp>(
+      indices[start + i] = arith::AddIOp::create(rewriter,
           loc, originalIndices[start + i],
-          rewriter.create<arith::ConstantIndexOp>(loc, offset));
+          arith::ConstantIndexOp::create(rewriter, loc, offset));
     }
   }
   return indices;
@@ -172,7 +172,7 @@ struct UnrollTransferReadPattern
     ArrayRef<int64_t> originalSize = readOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
@@ -185,7 +185,7 @@ struct UnrollTransferReadPattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                readOp.getPermutationMap(), loc, rewriter);
-      auto slicedRead = rewriter.create<vector::TransferReadOp>(
+      auto slicedRead = vector::TransferReadOp::create(rewriter,
           loc, targetType, readOp.getBase(), indices,
           readOp.getPermutationMapAttr(), readOp.getPadding(), readOp.getMask(),
           readOp.getInBoundsAttr());
@@ -236,7 +236,7 @@ struct UnrollTransferWritePattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                writeOp.getPermutationMap(), loc, rewriter);
-      Operation *slicedWrite = rewriter.create<vector::TransferWriteOp>(
+      Operation *slicedWrite = vector::TransferWriteOp::create(rewriter,
           loc, slicedVector, resultTensor ? resultTensor : writeOp.getBase(),
           indices, writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
       // For the tensor case update the destination for the next transfer write.
@@ -348,7 +348,7 @@ struct UnrollContractionPattern
       accCache[dstOffets] = newOp->getResult(0);
     }
     // Assemble back the accumulator into a single vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
@@ -427,7 +427,7 @@ struct UnrollMultiReductionPattern
       accCache[destOffset] = result;
     }
     // Assemble back the accumulator into a single vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, reductionOp.getDestType(),
         rewriter.getZeroAttr(reductionOp.getDestType()));
     for (const auto &it : accCache) {
@@ -468,7 +468,7 @@ struct UnrollElementwisePattern : public RewritePattern {
           op, "expected input vector rank to match target shape rank");
     Location loc = op->getLoc();
     // Prepare the result vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     SmallVector<int64_t> strides(targetShape->size(), 1);
     VectorType newVecType =
@@ -567,7 +567,7 @@ struct UnrollTransposePattern : public OpRewritePattern<vector::TransposeOp> {
     ArrayRef<int64_t> originalSize = originalVectorType.getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, originalVectorType, rewriter.getZeroAttr(originalVectorType));
     ArrayRef<int64_t> permutation = transposeOp.getPermutation();
 
@@ -618,7 +618,7 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
     ArrayRef<int64_t> originalSize = gatherOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
@@ -638,7 +638,7 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
           rewriter.createOrFold<vector::ExtractStridedSliceOp>(
               loc, gatherOp.getPassThru(), elementOffsets, *targetShape,
               strides);
-      auto slicedGather = rewriter.create<vector::GatherOp>(
+      auto slicedGather = vector::GatherOp::create(rewriter,
           loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
           indexSubVec, maskSubVec, passThruSubVec);
 
@@ -671,7 +671,7 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
     ArrayRef<int64_t> originalShape = vecType.getShape();
     SmallVector<int64_t> strides(targetShape->size(), 1);
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, vecType, rewriter.getZeroAttr(vecType));
 
     SmallVector<int64_t> loopOrder =
@@ -684,7 +684,7 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
          StaticTileOffsetRange(originalShape, *targetShape, loopOrder)) {
       SmallVector<Value> indices =
           sliceLoadStoreIndices(rewriter, loc, loadOp.getIndices(), offsets);
-      Value slicedLoad = rewriter.create<vector::LoadOp>(
+      Value slicedLoad = vector::LoadOp::create(rewriter,
           loc, targetVecType, loadOp.getBase(), indices);
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, slicedLoad, result, offsets, strides);
@@ -727,7 +727,7 @@ struct UnrollStorePattern : public OpRewritePattern<vector::StoreOp> {
           sliceLoadStoreIndices(rewriter, loc, storeOp.getIndices(), offsets);
       Value slice = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
           loc, vector, offsets, *targetShape, strides);
-      rewriter.create<vector::StoreOp>(loc, slice, base, indices);
+      vector::StoreOp::create(rewriter, loc, slice, base, indices);
     }
     rewriter.eraseOp(storeOp);
     return success();
@@ -755,7 +755,7 @@ struct UnrollBroadcastPattern : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType = broadcastOp.getResultVectorType();
     VectorType targetType =
         resType.cloneWith(*targetShape, resType.getElementType());
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
 
     SmallVector<int64_t> originalShape = *broadcastOp.getShapeForUnroll();
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 7e4984582b373..7c31ada1644c5 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -331,7 +331,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
   assert(padValue.getType() == sourceShapedType.getElementType() &&
          "expected same pad element type to match source element type");
   int64_t readRank = inputVectorSizes.size();
-  auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+  auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
   SmallVector<bool> inBoundsVal(readRank, true);
 
   if (useInBoundsInsteadOfMasking) {
@@ -341,7 +341,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
       inBoundsVal[i] = (sourceShape[i] == inputVectorSizes[i]) &&
                        ShapedType::isStatic(sourceShape[i]);
   }
-  auto transferReadOp = builder.create<vector::TransferReadOp>(
+  auto transferReadOp = vector::TransferReadOp::create(builder,
       loc,
       /*vectorType=*/vectorType,
       /*source=*/source,
@@ -356,7 +356,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
 
   auto maskType = VectorType::get(inputVectorSizes, builder.getI1Type());
   Value mask =
-      builder.create<vector::CreateMaskOp>(loc, maskType, mixedSourceDims);
+      vector::CreateMaskOp::create(builder, loc, maskType, mixedSourceDims);
   return mlir::vector::maskOperation(builder, transferReadOp, mask)
       ->getResult(0);
 }
diff --git a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
index cc7ab7f3f3895..eafe3a97758c1 100644
--- a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
+++ b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
@@ -63,11 +63,11 @@ SmallVector<Value> x86vector::MaskCompressOp::getIntrinsicOperands(
   if (adaptor.getSrc()) {
     src = adaptor.getSrc();
   } else if (adaptor.getConstantSrc()) {
-    src = rewriter.create<LLVM::ConstantOp>(loc, opType,
+    src = LLVM::ConstantOp::create(rewriter, loc, opType,
                                             adaptor.getConstantSrcAttr());
   } else {
     auto zeroAttr = rewriter.getZeroAttr(opType);
-    src = rewriter.create<LLVM::ConstantOp>(loc, opType, zeroAttr);
+    src = LLVM::ConstantOp::create(rewriter, loc, opType, zeroAttr);
   }
 
   return SmallVector<Value>{adaptor.getA(), src, adaptor.getK()};
@@ -80,7 +80,7 @@ x86vector::DotOp::getIntrinsicOperands(ArrayRef<Value> operands,
   SmallVector<Value> intrinsicOperands(operands);
   // Dot product of all elements, broadcasted to all elements.
   Value scale =
-      rewriter.create<LLVM::ConstantOp>(getLoc(), rewriter.getI8Type(), 0xff);
+      LLVM::ConstantOp::create(rewriter, getLoc(), rewriter.getI8Type(), 0xff);
   intrinsicOperands.push_back(scale);
 
   return intrinsicOperands;
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
index 63f725084cabf..8f87661a68d96 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
+++ b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
@@ -38,7 +38,7 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
       "=x,x,x"; // Careful: constraint parser is very brittle: no ws!
   SmallVector<Value> asmVals{v1, v2};
   auto asmStr = llvm::formatv(asmTp, llvm::format_hex(mask, /*width=*/2)).str();
-  auto asmOp = b.create<LLVM::InlineAsmOp>(
+  auto asmOp = LLVM::InlineAsmOp::create(b,
       v1.getType(), /*operands=*/asmVals, /*asm_string=*/asmStr,
       /*constraints=*/asmCstr, /*has_side_effects=*/false,
       /*is_align_stack=*/false, LLVM::TailCallKind::None,
@@ -49,13 +49,13 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackLoPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2, ArrayRef<int64_t>{0, 8, 1, 9, 4, 12, 5, 13});
 }
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackHiPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2, ArrayRef<int64_t>{2, 10, 3, 11, 6, 14, 7, 15});
 }
 ///                            a  a   b   b  a  a   b   b
@@ -69,7 +69,7 @@ Value mlir::x86vector::avx2::intrin::mm256ShufflePs(ImplicitLocOpBuilder &b,
   MaskHelper::extractShuffle(mask, b01, b23, b45, b67);
   SmallVector<int64_t> shuffleMask = {
       b01, b23, b45 + 8, b67 + 8, b01 + 4, b23 + 4, b45 + 8 + 4, b67 + 8 + 4};
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 // imm[0:1] out of imm[0:3] is:
@@ -97,7 +97,7 @@ Value mlir::x86vector::avx2::intrin::mm256Permute2f128Ps(
   MaskHelper::extractPermute(mask, b03, b47);
   appendToMask(b03);
   appendToMask(b47);
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// If bit i of `mask` is zero, take f32 at i from v1 else take it from v2.
@@ -109,7 +109,7 @@ Value mlir::x86vector::avx2::intrin::mm256BlendPs(ImplicitLocOpBuilder &b,
     bool isSet = mask & (1 << i);
     shuffleMask.push_back(!isSet ? i : i + 8);
   }
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// AVX2 4x8xf32-specific transpose lowering using a "C intrinsics" model.
@@ -245,13 +245,13 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
           VectorType::get({n * m}, op.getSourceVectorType().getElementType());
       auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
       auto reshInput =
-          ib.create<vector::ShapeCastOp>(flattenedType, op.getVector());
-      reshInput = ib.create<vector::ShapeCastOp>(reshInputType, reshInput);
+          vector::ShapeCastOp::create(ib, flattenedType, op.getVector());
+      reshInput = vector::ShapeCastOp::create(ib, reshInputType, reshInput);
 
       // Extract 1-D vectors from the higher-order dimension of the input
       // vector.
       for (int64_t i = 0; i < m; ++i)
-        vs.push_back(ib.create<vector::ExtractOp>(reshInput, i));
+        vs.push_back(vector::ExtractOp::create(ib, reshInput, i));
 
       // Transpose set of 1-D vectors.
       if (m == 4)
@@ -261,16 +261,16 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
 
       // Insert transposed 1-D vectors into the higher-order dimension of the
       // output vector.
-      Value res = ib.create<arith::ConstantOp>(reshInputType,
+      Value res = arith::ConstantOp::create(ib, reshInputType,
                                                ib.getZeroAttr(reshInputType));
       for (int64_t i = 0; i < m; ++i)
-        res = ib.create<vector::InsertOp>(vs[i], res, i);
+        res = vector::InsertOp::create(ib, vs[i], res, i);
 
       // The output vector still has the shape of the input vector (e.g., 4x8).
       // We have to transpose their dimensions and retrieve its original rank
       // (e.g., 1x8x1x4x1).
-      res = ib.create<vector::ShapeCastOp>(flattenedType, res);
-      res = ib.create<vector::ShapeCastOp>(op.getResultVectorType(), res);
+      res = vector::ShapeCastOp::create(ib, flattenedType, res);
+      res = vector::ShapeCastOp::create(ib, op.getResultVectorType(), res);
       rewriter.replaceOp(op, res);
       return success();
     };
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 3bea6cb50ff7b..453d53ee41d54 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -416,7 +416,7 @@ void CreateDescOp::build(OpBuilder &builder, OperationState &state,
   int64_t size = static_cast<int64_t>(offsets.size());
   auto type = VectorType::get(size, builder.getIndexType());
   auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
-  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  auto offset = vector::FromElementsOp::create(builder, loc, type, values);
   build(builder, state, TensorDesc, source, offset);
 }
 
@@ -553,7 +553,7 @@ void UpdateOffsetOp::build(OpBuilder &builder, OperationState &state,
   int64_t size = static_cast<int64_t>(offsets.size());
   auto type = VectorType::get({size}, builder.getIndexType());
   auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
-  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  auto offset = vector::FromElementsOp::create(builder, loc, type, values);
   build(builder, state, tdescTy, tensorDesc, offset);
 }
 
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index c072557c2bd22..613a33fe63d87 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -120,13 +120,13 @@ static Value resolveDistributedTy(Value orig, T expected,
   // If orig is a vector type, create a shape cast op to reconcile the types.
   if (isa<VectorType>(orig.getType())) {
     auto castOp =
-        rewriter.create<vector::ShapeCastOp>(orig.getLoc(), expected, orig);
+        vector::ShapeCastOp::create(rewriter, orig.getLoc(), expected, orig);
     return castOp.getResult();
   }
   // If orig is a tensor descriptor type, create an unrealized conversion cast
   // op to reconcile the types.
   if (isa<xegpu::TensorDescType>(orig.getType())) {
-    auto castOp = rewriter.create<UnrealizedConversionCastOp>(orig.getLoc(),
+    auto castOp = UnrealizedConversionCastOp::create(rewriter, orig.getLoc(),
                                                               expected, orig);
     castOp->setAttr(resolveSIMTTypeMismatch, rewriter.getUnitAttr());
     return castOp.getResult(0);
@@ -198,16 +198,16 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         }))
       return failure();
     // Create a new function with the same signature.
-    auto newGpuFunc = rewriter.create<gpu::GPUFuncOp>(
+    auto newGpuFunc = gpu::GPUFuncOp::create(rewriter,
         gpuFuncOp.getLoc(), gpuFuncOp.getName(), gpuFuncOp.getFunctionType());
     // Create a WarpExecuteOnLane0Op with same arguments and results as the
     // original gpuFuncOp.
     rewriter.setInsertionPointToEnd(&newGpuFunc.getFunctionBody().front());
-    auto laneId = rewriter.create<gpu::LaneIdOp>(
+    auto laneId = gpu::LaneIdOp::create(rewriter,
         newGpuFunc.getLoc(), rewriter.getIndexType(),
         /** upperBound = **/ mlir::IntegerAttr());
     ArrayRef<Type> gpuFuncResultType = gpuFuncOp.getFunctionType().getResults();
-    auto warpOp = rewriter.create<gpu::WarpExecuteOnLane0Op>(
+    auto warpOp = gpu::WarpExecuteOnLane0Op::create(rewriter,
         laneId.getLoc(), gpuFuncResultType, laneId,
         xegpu::targetinfo::subgroupSize, newGpuFunc.getArguments(),
         newGpuFunc.getArgumentTypes());
@@ -216,7 +216,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
     auto origRetunOp =
         cast<gpu::ReturnOp>(gpuFuncOp.getBlocks().back().getTerminator());
     rewriter.setInsertionPointAfter(origRetunOp);
-    rewriter.create<gpu::YieldOp>(origRetunOp.getLoc(),
+    gpu::YieldOp::create(rewriter, origRetunOp.getLoc(),
                                   origRetunOp.getOperands());
     rewriter.eraseOp(origRetunOp);
     // Move the original function body to the WarpExecuteOnLane0Op body.
@@ -225,7 +225,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
     rewriter.eraseBlock(&warpBodyBlock);
     // Insert a new ReturnOp after the WarpExecuteOnLane0Op.
     rewriter.setInsertionPointAfter(warpOp);
-    rewriter.create<gpu::ReturnOp>(newGpuFunc.getLoc(), warpOp.getResults());
+    gpu::ReturnOp::create(rewriter, newGpuFunc.getLoc(), warpOp.getResults());
     rewriter.replaceOp(gpuFuncOp, newGpuFunc);
     return success();
   }
@@ -301,7 +301,7 @@ struct CreateNdDescDistribution final : public gpu::WarpDistributionPattern {
     xegpu::TensorDescType distributedTensorDescTy =
         descOp.getType().dropLayouts(); // Distributed tensor descriptor type
                                         // does not contain layout info.
-    Value newDescOp = rewriter.create<xegpu::CreateNdDescOp>(
+    Value newDescOp = xegpu::CreateNdDescOp::create(rewriter,
         newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
         descOp->getAttrs());
 
@@ -403,7 +403,7 @@ struct StoreNdDistribution final : public gpu::WarpDistributionPattern {
         resolveDistributedTy(newWarpOp.getResult(newRetIndices[1]),
                              distributedTensorDescTy, rewriter));
 
-    rewriter.create<xegpu::StoreNdOp>(
+    xegpu::StoreNdOp::create(rewriter,
         newWarpOp.getLoc(), TypeRange{}, newStoreOperands,
         removeTemporaryLayoutAttributes(storeOp->getAttrs()));
     rewriter.eraseOp(storeOp);
@@ -494,7 +494,7 @@ struct LoadNdDistribution final : public gpu::WarpDistributionPattern {
         loadOp.getTensorDescType().dropLayouts(); // Distributed tensor
                                                   // descriptor type does not
                                                   // contain layout info.
-    auto newLoadOp = rewriter.create<xegpu::LoadNdOp>(
+    auto newLoadOp = xegpu::LoadNdOp::create(rewriter,
         newWarpOp.getLoc(), loadNdDistValueTyOrFailure.value(),
         resolveDistributedTy(newWarpOp->getResult(newRetIndices[0]),
                              distributedTensorDescTy, rewriter),
@@ -630,7 +630,7 @@ struct DpasDistribution final : public gpu::WarpDistributionPattern {
           resolveDistributedTy(newWarpOp.getResult(newRetIndices[i]),
                                newDpasOperandExpectedTypes[i], rewriter));
     }
-    Value newDpasOp = rewriter.create<xegpu::DpasOp>(
+    Value newDpasOp = xegpu::DpasOp::create(rewriter,
         newWarpOp->getLoc(), distributedResultTy, newDpasOperands,
         removeTemporaryLayoutAttributes(dpasOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -717,7 +717,7 @@ struct UpdateNdOffsetDistribution final : public gpu::WarpDistributionPattern {
       }
     }
     // Create a new update op outside the warp op.
-    Value newUpdateOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+    Value newUpdateOp = xegpu::UpdateNdOffsetOp::create(rewriter,
         newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
         removeTemporaryLayoutAttributes(updateOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -783,7 +783,7 @@ struct PrefetchNdDistribution final : public gpu::WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     SmallVector<Value> newPrefetchOperands = {resolveDistributedTy(
         newWarpOp.getResult(newRetIndices[0]), newTensorDescTy, rewriter)};
-    rewriter.create<xegpu::PrefetchNdOp>(
+    xegpu::PrefetchNdOp::create(rewriter,
         newWarpOp.getLoc(), TypeRange{}, newPrefetchOperands,
         removeTemporaryLayoutAttributes(prefetchOp->getAttrs()));
     rewriter.eraseOp(prefetchOp);
@@ -806,7 +806,7 @@ struct GpuBarrierDistribution final : public gpu::WarpDistributionPattern {
       return failure();
     // Move the barrier op outside of the warp op.
     rewriter.setInsertionPointAfter(subgroupOp);
-    rewriter.create<gpu::BarrierOp>(
+    gpu::BarrierOp::create(rewriter,
         barrierOp.getLoc(), barrierOp->getResultTypes(),
         barrierOp->getOperands(), barrierOp->getAttrs());
     rewriter.eraseOp(barrierOp);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
index 2c48a735bf956..1d9f4dcc6b9e6 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
@@ -83,7 +83,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = rewriter.create<UnrealizedConversionCastOp>(
+      auto castOp = UnrealizedConversionCastOp::create(rewriter,
           loc, destTy, srcs, ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResult(0);
     }
@@ -109,7 +109,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = rewriter.create<UnrealizedConversionCastOp>(
+      auto castOp = UnrealizedConversionCastOp::create(rewriter,
           loc, destTypes, src, ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResults();
     }
@@ -144,10 +144,10 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
     auto addi = [&](OpFoldResult a, int64_t b) -> Value {
       std::optional<int64_t> maybeInt = getConstantIntValue(a);
       if (maybeInt) {
-        return rewriter.create<arith::ConstantIndexOp>(loc, *maybeInt + b);
+        return arith::ConstantIndexOp::create(rewriter, loc, *maybeInt + b);
       } else {
         auto aV = llvm::cast<Value>(a);
-        auto bV = rewriter.create<arith::ConstantIndexOp>(loc, b);
+        auto bV = arith::ConstantIndexOp::create(rewriter, loc, b);
         return rewriter.createOrFold<arith::AddIOp>(loc, aV, bV);
       }
     };
@@ -169,7 +169,7 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
            llvm::zip(validIdxes, oldOffsets, offsets))
         mixedOffsets[idx] = addi(oldOff, offset);
 
-      auto newOp = rewriter.create<xegpu::CreateNdDescOp>(
+      auto newOp = xegpu::CreateNdDescOp::create(rewriter,
           loc, newTdescTy, op.getSource(), mixedOffsets, op.getMixedSizes(),
           op.getMixedStrides());
       newOps.push_back(newOp);
@@ -199,7 +199,7 @@ struct UnrollUpdateNdOffsetOp : public UnrollPattern<xegpu::UpdateNdOffsetOp> {
 
     SmallVector<Value> newOps;
     for (auto t : convertedTdesc) {
-      auto newOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+      auto newOp = xegpu::UpdateNdOffsetOp::create(rewriter,
           loc, t.getType(), t, op.getOffsets(), op.getConstOffsets());
       newOps.push_back(newOp);
     }
@@ -226,7 +226,7 @@ struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      rewriter.create<xegpu::PrefetchNdOp>(loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchNdOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -257,7 +257,7 @@ struct UnrollLoadNdOp : public UnrollPattern<xegpu::LoadNdOp> {
     SmallVector<Value> newOps;
     for (auto t : convertedTdescs) {
       auto newOp =
-          rewriter.create<xegpu::LoadNdOp>(loc, newValueTy, t, op->getAttrs());
+          xegpu::LoadNdOp::create(rewriter, loc, newValueTy, t, op->getAttrs());
       newOps.push_back(newOp);
     }
 
@@ -291,7 +291,7 @@ struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto [v, t] : llvm::zip(convertedValues, convertedTdescs))
-      rewriter.create<xegpu::StoreNdOp>(loc, v, t, op.getL1HintAttr(),
+      xegpu::StoreNdOp::create(rewriter, loc, v, t, op.getL1HintAttr(),
                                         op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
@@ -384,7 +384,7 @@ struct UnrollDpasOp : public UnrollPattern<xegpu::DpasOp> {
           if (tmpC)
             operands.push_back(tmpC);
 
-          tmpC = rewriter.create<xegpu::DpasOp>(loc, vecTy, operands,
+          tmpC = xegpu::DpasOp::create(rewriter, loc, vecTy, operands,
                                                 op->getAttrs());
         }
         newOps.push_back(tmpC);
@@ -436,13 +436,13 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
            llvm::zip(convertedIndiceVec, convertedIndiceTypes)) {
         for (int64_t i = 0; i < numNewChunks; ++i) {
           // Compute the offset
-          Value inc = rewriter.create<arith::ConstantIndexOp>(
+          Value inc = arith::ConstantIndexOp::create(rewriter,
               loc, i * blockedChunkSize);
-          Value incVec = rewriter.create<vector::SplatOp>(loc, indiceType, inc);
+          Value incVec = vector::SplatOp::create(rewriter, loc, indiceType, inc);
           Value offsetIndice =
-              rewriter.create<arith::AddIOp>(loc, indice, incVec);
+              arith::AddIOp::create(rewriter, loc, indice, incVec);
 
-          auto newOp = rewriter.create<xegpu::CreateDescOp>(
+          auto newOp = xegpu::CreateDescOp::create(rewriter,
               loc, newTdescTy, op.getSource(), offsetIndice);
 
           newOps.push_back(newOp);
@@ -450,7 +450,7 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
       }
     } else {
       for (auto indice : convertedIndiceVec) {
-        auto newOp = rewriter.create<xegpu::CreateDescOp>(
+        auto newOp = xegpu::CreateDescOp::create(rewriter,
             loc, newTdescTy, op.getSource(), indice);
         newOps.push_back(newOp);
       }
@@ -515,7 +515,7 @@ struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
 
     SmallVector<Value> newOps;
     for (auto [t, m] : llvm::zip(convertedTdescs, convertedMasks)) {
-      auto newOp = rewriter.create<xegpu::LoadGatherOp>(
+      auto newOp = xegpu::LoadGatherOp::create(rewriter,
           loc, newValueTy, t, m, op.getL1HintAttr(), op.getL2HintAttr(),
           op.getL3HintAttr());
       newOps.push_back(newOp);
@@ -547,7 +547,7 @@ struct UnrollPrefetchOp : public UnrollPattern<xegpu::PrefetchOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      rewriter.create<xegpu::PrefetchOp>(loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -608,7 +608,7 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
       Value v = convertedValues[i];
       Value t = convertedTdescs[i];
       Value m = op.getMask() ? convertedMasks[i] : nullptr;
-      rewriter.create<xegpu::StoreScatterOp>(loc, v, t, m, op.getL1HintAttr(),
+      xegpu::StoreScatterOp::create(rewriter, loc, v, t, m, op.getL1HintAttr(),
                                              op.getL2HintAttr(),
                                              op.getL3HintAttr());
     }
@@ -665,7 +665,7 @@ struct UnrollUpdateOffsetOp : public UnrollPattern<xegpu::UpdateOffsetOp> {
 
     for (auto [t, o] : llvm::zip(convertedTdesc, convertedOffsetVec)) {
       auto newOp =
-          rewriter.create<xegpu::UpdateOffsetOp>(loc, t.getType(), t, o);
+          xegpu::UpdateOffsetOp::create(rewriter, loc, t.getType(), t, o);
       newOps.push_back(newOp);
     }
     Value castOp = unpack(newOps, op.getType(), *targetShape, loc, rewriter);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index be7b860dd1729..aa2fc78c77f39 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -121,11 +121,11 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
     for (size_t i = 0; i < rank; ++i) {
       size_t dimIdx = originalOffsets.size() - rank + i;
       Value constOffset =
-          rewriter.create<arith::ConstantIndexOp>(loc, distUnitBaseAddr[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, distUnitBaseAddr[i]);
       Value offset =
           rewriter.createOrFold<index::AddOp>(loc, localOffset[i], constOffset);
       Value modValue =
-          rewriter.create<arith::ConstantIndexOp>(loc, distUnitShape[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, distUnitShape[i]);
       Value offsetMod =
           rewriter.createOrFold<index::RemUOp>(loc, offset, modValue);
       Value origOffset = getValueOrCreateConstantIndexOp(
@@ -162,7 +162,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
     // TODO : Handle order attribute
     // Get the subgroup ID
     auto linearSgId =
-        rewriter.create<gpu::SubgroupIdOp>(loc, /*upper_bound=*/nullptr);
+        gpu::SubgroupIdOp::create(rewriter, loc, /*upper_bound=*/nullptr);
 
     // Create constants for layout dimensions
     SmallVector<Value> sgLayoutDim(sgLayout.size());
@@ -170,8 +170,8 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
 
     for (size_t i = 0; i < sgLayout.size(); i++) {
       sgLayoutDim[i] =
-          rewriter.create<arith::ConstantIndexOp>(loc, sgLayout[i]);
-      sgDataDim[i] = rewriter.create<arith::ConstantIndexOp>(loc, sgShape[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, sgLayout[i]);
+      sgDataDim[i] = arith::ConstantIndexOp::create(rewriter, loc, sgShape[i]);
     }
 
     auto deLinearizeSgId =
@@ -201,7 +201,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
           calculateGlobalOffsets(rewriter, loc, originalOffsets, localOffset,
                                  distUnitBaseAddr, distUnitShape);
 
-      auto newCreateNdOp = rewriter.create<xegpu::CreateNdDescOp>(
+      auto newCreateNdOp = xegpu::CreateNdDescOp::create(rewriter,
           loc, newTdescTy, op.getSource(), globalOffsets, op.getMixedSizes(),
           op.getMixedStrides());
       newCreateNdOps.push_back(newCreateNdOp);
@@ -224,7 +224,7 @@ struct WgToSgLoadNdOp : public OpConversionPattern<xegpu::LoadNdOp> {
           dyn_cast<xegpu::TensorDescType>(src.getType());
       ArrayRef<int64_t> srcShape = tdescTy.getShape();
       VectorType newResTy = VectorType::get(srcShape, tdescTy.getElementType());
-      auto newLoadOp = rewriter.create<xegpu::LoadNdOp>(op.getLoc(), newResTy,
+      auto newLoadOp = xegpu::LoadNdOp::create(rewriter, op.getLoc(), newResTy,
                                                         src, op->getAttrs());
       newLoadOps.push_back(newLoadOp);
     }
@@ -242,7 +242,7 @@ struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
   matchAndRewrite(xegpu::StoreNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     for (auto [v, t] : llvm::zip(adaptor.getValue(), adaptor.getTensorDesc()))
-      rewriter.create<xegpu::StoreNdOp>(op.getLoc(), v, t, op.getL1HintAttr(),
+      xegpu::StoreNdOp::create(rewriter, op.getLoc(), v, t, op.getL1HintAttr(),
                                         op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
@@ -261,7 +261,7 @@ struct WgToSgUpdateNdOffsetOp
                   ConversionPatternRewriter &rewriter) const override {
     llvm::SmallVector<Value> newUpdateTileOffsetOps;
     for (auto tDesc : adaptor.getTensorDesc()) {
-      auto newUpdateTileOffsetOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+      auto newUpdateTileOffsetOp = xegpu::UpdateNdOffsetOp::create(rewriter,
           op.getLoc(), tDesc.getType(), tDesc, op.getOffsets(),
           op.getConstOffsets());
       newUpdateTileOffsetOps.push_back(newUpdateTileOffsetOp);
@@ -305,7 +305,7 @@ struct WgToSgDpasOp : public OpConversionPattern<xegpu::DpasOp> {
             llvm::cast<VectorType>(bVec.getType()).getShape();
         VectorType resTy = VectorType::get({aVecShape[0], bVecShape[1]},
                                            resultTy.getElementType());
-        tmpC = rewriter.create<xegpu::DpasOp>(loc, resTy, operands);
+        tmpC = xegpu::DpasOp::create(rewriter, loc, resTy, operands);
         xegpu::setLayoutAttr(cast<OpResult>(tmpC),
                              originalLayout.dropSgLayoutAndData());
 
@@ -324,7 +324,7 @@ struct WgToSgPrefetchNdOp : public OpConversionPattern<xegpu::PrefetchNdOp> {
   matchAndRewrite(xegpu::PrefetchNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     for (auto src : adaptor.getTensorDesc())
-      rewriter.create<xegpu::PrefetchNdOp>(op.getLoc(), TypeRange(), src,
+      xegpu::PrefetchNdOp::create(rewriter, op.getLoc(), TypeRange(), src,
                                            op->getAttrs());
     rewriter.eraseOp(op);
     return success();
diff --git a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index 6b85a66a8bd36..9acb209d031bd 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -198,7 +198,7 @@ xegpu::extractVectorsWithShapeFromValue(OpBuilder &builder, Location loc,
   SmallVector<Value> result;
   for (SmallVector<int64_t> offsets : StaticTileOffsetRange(srcShape, shape)) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result.push_back(builder.create<vector::ExtractStridedSliceOp>(
+    result.push_back(vector::ExtractStridedSliceOp::create(builder,
         loc, value, offsets, shape, staticStrides));
   }
 
@@ -218,13 +218,13 @@ Value xegpu::createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
 
   VectorType resultTy = VectorType::get(shape, elemTy);
   auto zeroAttr = builder.getZeroAttr(elemTy);
-  Value result = builder.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(builder,
       loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
 
   for (auto [src, offsets] :
        llvm::zip_equal(values, StaticTileOffsetRange(shape, tileShape))) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result = builder.create<vector::InsertStridedSliceOp>(
+    result = vector::InsertStridedSliceOp::create(builder,
         loc, src, result, offsets, staticStrides);
   }
   return result;
@@ -236,7 +236,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
   auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
                             Location loc) -> Value {
-    return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
         .getResult(0);
   };
 
@@ -343,7 +343,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
         if (isa<RankedTensorType>(inputTy) && isa<VectorType>(outputTy)) {
           SmallVector<Value> values = xegpu::flattenValues(adaptor.getInputs());
-          auto newOp = rewriter.create<UnrealizedConversionCastOp>(
+          auto newOp = UnrealizedConversionCastOp::create(rewriter,
               op.getLoc(), outputTy, values);
           rewriter.replaceOp(op, newOp);
           return success();
@@ -355,7 +355,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
     converter.addSourceMaterialization(materializeCast);
     converter.addTargetMaterialization([&](OpBuilder &builder, TypeRange type,
                                            ValueRange inputs, Location loc) {
-      return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+      return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
           .getResults();
     });
 
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 6d7e2aa0ece7d..c88b328282275 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -132,7 +132,7 @@ void ModuleOp::build(OpBuilder &builder, OperationState &state,
 /// Construct a module from the given context.
 ModuleOp ModuleOp::create(Location loc, std::optional<StringRef> name) {
   OpBuilder builder(loc->getContext());
-  return builder.create<ModuleOp>(loc, name);
+  return ModuleOp::create(builder, loc, name);
 }
 
 DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {
diff --git a/mlir/lib/Query/Query.cpp b/mlir/lib/Query/Query.cpp
index 637e1f3cdef87..8ddb35d75734b 100644
--- a/mlir/lib/Query/Query.cpp
+++ b/mlir/lib/Query/Query.cpp
@@ -78,7 +78,7 @@ static Operation *extractFunction(std::vector<Operation *> &ops,
                       clonedOp->result_end());
   }
   // Add return operation
-  builder.create<func::ReturnOp>(loc, clonedVals);
+  func::ReturnOp::create(builder, loc, clonedVals);
 
   // Remove unused function arguments
   size_t currentIndex = 0;
diff --git a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
index 4e7f1d3185129..c0f19f6dce600 100644
--- a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
+++ b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
@@ -37,7 +37,7 @@ LogicalResult mlir::LLVMImportInterface::convertUnregisteredIntrinsic(
     return failure();
 
   Type resultType = moduleImport.convertType(inst->getType());
-  auto op = builder.create<::mlir::LLVM::CallIntrinsicOp>(
+  auto op = ::mlir::LLVM::CallIntrinsicOp::create(builder,
       moduleImport.translateLoc(inst->getDebugLoc()),
       isa<LLVMVoidType>(resultType) ? TypeRange{} : TypeRange{resultType},
       StringAttr::get(builder.getContext(), intrinName),
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index bfda223fe0f5f..6e47db6d1dc6a 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -185,7 +185,7 @@ ComdatOp ModuleImport::getGlobalComdatOp() {
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(mlirModule.getBody());
   globalComdatOp =
-      builder.create<ComdatOp>(mlirModule.getLoc(), getGlobalComdatOpName());
+      ComdatOp::create(builder, mlirModule.getLoc(), getGlobalComdatOpName());
   globalInsertionOp = globalComdatOp;
   return globalComdatOp;
 }
@@ -864,7 +864,7 @@ LogicalResult ModuleImport::convertModuleFlagsMetadata() {
   }
 
   if (!moduleFlags.empty())
-    builder.create<LLVM::ModuleFlagsOp>(mlirModule.getLoc(),
+    LLVM::ModuleFlagsOp::create(builder, mlirModule.getLoc(),
                                         builder.getArrayAttr(moduleFlags));
 
   return success();
@@ -880,7 +880,7 @@ LogicalResult ModuleImport::convertLinkerOptionsMetadata() {
       options.reserve(node->getNumOperands());
       for (const llvm::MDOperand &option : node->operands())
         options.push_back(cast<llvm::MDString>(option)->getString());
-      builder.create<LLVM::LinkerOptionsOp>(mlirModule.getLoc(),
+      LLVM::LinkerOptionsOp::create(builder, mlirModule.getLoc(),
                                             builder.getStrArrayAttr(options));
     }
   }
@@ -984,7 +984,7 @@ void ModuleImport::processComdat(const llvm::Comdat *comdat) {
   ComdatOp comdatOp = getGlobalComdatOp();
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(&comdatOp.getBody().back());
-  auto selectorOp = builder.create<ComdatSelectorOp>(
+  auto selectorOp = ComdatSelectorOp::create(builder,
       mlirModule.getLoc(), comdat->getName(),
       convertComdatFromLLVM(comdat->getSelectionKind()));
   auto symbolRef =
@@ -1346,7 +1346,7 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   Type type = convertType(alias->getValueType());
-  AliasOp aliasOp = builder.create<AliasOp>(
+  AliasOp aliasOp = AliasOp::create(builder,
       mlirModule.getLoc(), type, convertLinkageFromLLVM(alias->getLinkage()),
       alias->getName(),
       /*dso_local=*/alias->isDSOLocal(),
@@ -1360,7 +1360,7 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   FailureOr<Value> initializer = convertConstantExpr(alias->getAliasee());
   if (failed(initializer))
     return failure();
-  builder.create<ReturnOp>(aliasOp.getLoc(), *initializer);
+  ReturnOp::create(builder, aliasOp.getLoc(), *initializer);
 
   if (alias->hasAtLeastLocalUnnamedAddr())
     aliasOp.setUnnamedAddr(convertUnnamedAddrFromLLVM(alias->getUnnamedAddr()));
@@ -1403,7 +1403,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
   if (globalName.empty())
     globalName = getOrCreateNamelessSymbolName(globalVar).getValue();
 
-  GlobalOp globalOp = builder.create<GlobalOp>(
+  GlobalOp globalOp = GlobalOp::create(builder,
       mlirModule.getLoc(), type, globalVar->isConstant(),
       convertLinkageFromLLVM(globalVar->getLinkage()), StringRef(globalName),
       valueAttr, alignment, /*addr_space=*/globalVar->getAddressSpace(),
@@ -1420,7 +1420,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
         convertConstantExpr(globalVar->getInitializer());
     if (failed(initializer))
       return failure();
-    builder.create<ReturnOp>(globalOp.getLoc(), *initializer);
+    ReturnOp::create(builder, globalOp.getLoc(), *initializer);
   }
   if (globalVar->hasAtLeastLocalUnnamedAddr()) {
     globalOp.setUnnamedAddr(
@@ -1488,12 +1488,12 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   if (globalVar->getName() == getGlobalCtorsVarName()) {
-    globalInsertionOp = builder.create<LLVM::GlobalCtorsOp>(
+    globalInsertionOp = LLVM::GlobalCtorsOp::create(builder,
         mlirModule.getLoc(), builder.getArrayAttr(funcs),
         builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
     return success();
   }
-  globalInsertionOp = builder.create<LLVM::GlobalDtorsOp>(
+  globalInsertionOp = LLVM::GlobalDtorsOp::create(builder,
       mlirModule.getLoc(), builder.getArrayAttr(funcs),
       builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
   return success();
@@ -1569,33 +1569,33 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
   if (Attribute attr = getConstantAsAttr(constant)) {
     Type type = convertType(constant->getType());
     if (auto symbolRef = dyn_cast<FlatSymbolRefAttr>(attr)) {
-      return builder.create<AddressOfOp>(loc, type, symbolRef.getValue())
+      return AddressOfOp::create(builder, loc, type, symbolRef.getValue())
           .getResult();
     }
-    return builder.create<ConstantOp>(loc, type, attr).getResult();
+    return ConstantOp::create(builder, loc, type, attr).getResult();
   }
 
   // Convert null pointer constants.
   if (auto *nullPtr = dyn_cast<llvm::ConstantPointerNull>(constant)) {
     Type type = convertType(nullPtr->getType());
-    return builder.create<ZeroOp>(loc, type).getResult();
+    return ZeroOp::create(builder, loc, type).getResult();
   }
 
   // Convert none token constants.
   if (isa<llvm::ConstantTokenNone>(constant)) {
-    return builder.create<NoneTokenOp>(loc).getResult();
+    return NoneTokenOp::create(builder, loc).getResult();
   }
 
   // Convert poison.
   if (auto *poisonVal = dyn_cast<llvm::PoisonValue>(constant)) {
     Type type = convertType(poisonVal->getType());
-    return builder.create<PoisonOp>(loc, type).getResult();
+    return PoisonOp::create(builder, loc, type).getResult();
   }
 
   // Convert undef.
   if (auto *undefVal = dyn_cast<llvm::UndefValue>(constant)) {
     Type type = convertType(undefVal->getType());
-    return builder.create<UndefOp>(loc, type).getResult();
+    return UndefOp::create(builder, loc, type).getResult();
   }
 
   // Convert dso_local_equivalent.
@@ -1621,7 +1621,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
           getOrCreateNamelessSymbolName(cast<llvm::GlobalVariable>(globalObj));
     else
       symbolRef = FlatSymbolRefAttr::get(context, globalName);
-    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+    return AddressOfOp::create(builder, loc, type, symbolRef).getResult();
   }
 
   // Convert global alias accesses.
@@ -1629,7 +1629,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     Type type = convertType(globalAliasObj->getType());
     StringRef aliaseeName = globalAliasObj->getName();
     FlatSymbolRefAttr symbolRef = FlatSymbolRefAttr::get(context, aliaseeName);
-    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+    return AddressOfOp::create(builder, loc, type, symbolRef).getResult();
   }
 
   // Convert constant expressions.
@@ -1680,15 +1680,15 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     bool isArrayOrStruct = isa<LLVMArrayType, LLVMStructType>(rootType);
     assert((isArrayOrStruct || LLVM::isCompatibleVectorType(rootType)) &&
            "unrecognized aggregate type");
-    Value root = builder.create<UndefOp>(loc, rootType);
+    Value root = UndefOp::create(builder, loc, rootType);
     for (const auto &it : llvm::enumerate(elementValues)) {
       if (isArrayOrStruct) {
-        root = builder.create<InsertValueOp>(loc, root, it.value(), it.index());
+        root = InsertValueOp::create(builder, loc, root, it.value(), it.index());
       } else {
         Attribute indexAttr = builder.getI32IntegerAttr(it.index());
         Value indexValue =
-            builder.create<ConstantOp>(loc, builder.getI32Type(), indexAttr);
-        root = builder.create<InsertElementOp>(loc, rootType, root, it.value(),
+            ConstantOp::create(builder, loc, builder.getI32Type(), indexAttr);
+        root = InsertElementOp::create(builder, loc, rootType, root, it.value(),
                                                indexValue);
       }
     }
@@ -1702,7 +1702,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
            "target extension type does not support zero-initialization");
     // Create llvm.mlir.zero operation to represent zero-initialization of
     // target extension type.
-    return builder.create<LLVM::ZeroOp>(loc, targetExtType).getRes();
+    return LLVM::ZeroOp::create(builder, loc, targetExtType).getRes();
   }
 
   if (auto *blockAddr = dyn_cast<llvm::BlockAddress>(constant)) {
@@ -2124,7 +2124,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     if (!brInst->isConditional()) {
-      auto brOp = builder.create<LLVM::BrOp>(loc, succBlockArgs.front(),
+      auto brOp = LLVM::BrOp::create(builder, loc, succBlockArgs.front(),
                                              succBlocks.front());
       mapNoResultOp(inst, brOp);
       return success();
@@ -2132,7 +2132,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     FailureOr<Value> condition = convertValue(brInst->getCondition());
     if (failed(condition))
       return failure();
-    auto condBrOp = builder.create<LLVM::CondBrOp>(
+    auto condBrOp = LLVM::CondBrOp::create(builder,
         loc, *condition, succBlocks.front(), succBlockArgs.front(),
         succBlocks.back(), succBlockArgs.back());
     mapNoResultOp(inst, condBrOp);
@@ -2166,7 +2166,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       caseBlocks[it.index()] = lookupBlock(succBB);
     }
 
-    auto switchOp = builder.create<SwitchOp>(
+    auto switchOp = SwitchOp::create(builder,
         loc, *condition, lookupBlock(defaultBB), defaultBlockArgs, caseValues,
         caseBlocks, caseOperandRefs);
     mapNoResultOp(inst, switchOp);
@@ -2218,14 +2218,14 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
         // IR). Build the indirect call by passing an empty `callee` operand and
         // insert into `operands` to include the indirect call target.
         FlatSymbolRefAttr calleeSym = convertCalleeName(callInst);
-        Value indirectCallVal = builder.create<LLVM::AddressOfOp>(
+        Value indirectCallVal = LLVM::AddressOfOp::create(builder,
             loc, LLVM::LLVMPointerType::get(context), calleeSym);
         operands->insert(operands->begin(), indirectCallVal);
       } else {
         // Regular direct call using callee name.
         callee = convertCalleeName(callInst);
       }
-      CallOp callOp = builder.create<CallOp>(loc, *funcTy, callee, *operands);
+      CallOp callOp = CallOp::create(builder, loc, *funcTy, callee, *operands);
 
       if (failed(convertCallAttributes(callInst, callOp)))
         return failure();
@@ -2260,7 +2260,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
 
     Type type = convertType(lpInst->getType());
     auto lpOp =
-        builder.create<LandingpadOp>(loc, type, lpInst->isCleanup(), operands);
+        LandingpadOp::create(builder, loc, type, lpInst->isCleanup(), operands);
     mapValue(inst, lpOp);
     return success();
   }
@@ -2310,7 +2310,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // IR). Build the indirect invoke by passing an empty `callee` operand and
       // insert into `operands` to include the indirect invoke target.
       FlatSymbolRefAttr calleeSym = convertCalleeName(invokeInst);
-      Value indirectInvokeVal = builder.create<LLVM::AddressOfOp>(
+      Value indirectInvokeVal = LLVM::AddressOfOp::create(builder,
           loc, LLVM::LLVMPointerType::get(context), calleeSym);
       operands->insert(operands->begin(), indirectInvokeVal);
     } else {
@@ -2320,7 +2320,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     // Create the invoke operation. Normal destination block arguments will be
     // added later on to handle the case in which the operation result is
     // included in this list.
-    auto invokeOp = builder.create<InvokeOp>(
+    auto invokeOp = InvokeOp::create(builder,
         loc, *funcTy, calleeName, *operands, directNormalDest, ValueRange(),
         lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
 
@@ -2348,7 +2348,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // arguments (including the invoke operation's result).
       OpBuilder::InsertionGuard g(builder);
       builder.setInsertionPointToStart(directNormalDest);
-      builder.create<LLVM::BrOp>(loc, normalArgs, normalDest);
+      LLVM::BrOp::create(builder, loc, normalArgs, normalDest);
     } else {
       // If the invoke operation's result is not a block argument to the normal
       // destination block, just add the block arguments as usual.
@@ -2382,7 +2382,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     Type type = convertType(inst->getType());
-    auto gepOp = builder.create<GEPOp>(
+    auto gepOp = GEPOp::create(builder,
         loc, type, sourceElementType, *basePtr, indices,
         static_cast<GEPNoWrapFlags>(gepInst->getNoWrapFlags().getRaw()));
     mapValue(inst, gepOp);
@@ -2409,7 +2409,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     SmallVector<ValueRange> succBlockArgsRange =
         llvm::to_vector_of<ValueRange>(succBlockArgs);
     Location loc = translateLoc(inst->getDebugLoc());
-    auto indBrOp = builder.create<LLVM::IndirectBrOp>(
+    auto indBrOp = LLVM::IndirectBrOp::create(builder,
         loc, *basePtr, succBlockArgsRange, succBlocks);
 
     mapNoResultOp(inst, indBrOp);
@@ -2854,7 +2854,7 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) {
   builder.setInsertionPointToEnd(mlirModule.getBody());
 
   Location loc = debugImporter->translateFuncLocation(func);
-  LLVMFuncOp funcOp = builder.create<LLVMFuncOp>(
+  LLVMFuncOp funcOp = LLVMFuncOp::create(builder,
       loc, func->getName(), functionType,
       convertLinkageFromLLVM(func->getLinkage()), dsoLocal, cconv);
 
@@ -3032,11 +3032,11 @@ ModuleImport::processDebugIntrinsic(llvm::DbgVariableIntrinsic *dbgIntr,
   Operation *op =
       llvm::TypeSwitch<llvm::DbgVariableIntrinsic *, Operation *>(dbgIntr)
           .Case([&](llvm::DbgDeclareInst *) {
-            return builder.create<LLVM::DbgDeclareOp>(
+            return LLVM::DbgDeclareOp::create(builder,
                 loc, *argOperand, localVariableAttr, locationExprAttr);
           })
           .Case([&](llvm::DbgValueInst *) {
-            return builder.create<LLVM::DbgValueOp>(
+            return LLVM::DbgValueOp::create(builder,
                 loc, *argOperand, localVariableAttr, locationExprAttr);
           });
   mapNoResultOp(dbgIntr, op);
@@ -3082,7 +3082,7 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb,
   if (bb->hasAddressTaken()) {
     OpBuilder::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(block);
-    builder.create<BlockTagOp>(block->getParentOp()->getLoc(),
+    BlockTagOp::create(builder, block->getParentOp()->getLoc(),
                                BlockTagAttr::get(context, bb->getNumber()));
   }
   return success();
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index 55d6a380d0bff..6e14b94e87407 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -42,22 +42,22 @@ static inline spirv::Opcode extractOpcode(uint32_t word) {
 Value spirv::Deserializer::getValue(uint32_t id) {
   if (auto constInfo = getConstant(id)) {
     // Materialize a `spirv.Constant` op at every use site.
-    return opBuilder.create<spirv::ConstantOp>(unknownLoc, constInfo->second,
+    return spirv::ConstantOp::create(opBuilder, unknownLoc, constInfo->second,
                                                constInfo->first);
   }
   if (auto varOp = getGlobalVariable(id)) {
-    auto addressOfOp = opBuilder.create<spirv::AddressOfOp>(
+    auto addressOfOp = spirv::AddressOfOp::create(opBuilder,
         unknownLoc, varOp.getType(), SymbolRefAttr::get(varOp.getOperation()));
     return addressOfOp.getPointer();
   }
   if (auto constOp = getSpecConstant(id)) {
-    auto referenceOfOp = opBuilder.create<spirv::ReferenceOfOp>(
+    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
         unknownLoc, constOp.getDefaultValue().getType(),
         SymbolRefAttr::get(constOp.getOperation()));
     return referenceOfOp.getReference();
   }
   if (auto constCompositeOp = getSpecConstantComposite(id)) {
-    auto referenceOfOp = opBuilder.create<spirv::ReferenceOfOp>(
+    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
         unknownLoc, constCompositeOp.getType(),
         SymbolRefAttr::get(constCompositeOp.getOperation()));
     return referenceOfOp.getReference();
@@ -69,7 +69,7 @@ Value spirv::Deserializer::getValue(uint32_t id) {
         specConstOperationInfo->enclosedOpOperands);
   }
   if (auto undef = getUndefType(id)) {
-    return opBuilder.create<spirv::UndefOp>(unknownLoc, undef);
+    return spirv::UndefOp::create(opBuilder, unknownLoc, undef);
   }
   return valueMap.lookup(id);
 }
@@ -369,7 +369,7 @@ Deserializer::processOp<spirv::EntryPointOp>(ArrayRef<uint32_t> words) {
     interface.push_back(SymbolRefAttr::get(arg.getOperation()));
     wordIndex++;
   }
-  opBuilder.create<spirv::EntryPointOp>(
+  spirv::EntryPointOp::create(opBuilder,
       unknownLoc, execModel, SymbolRefAttr::get(opBuilder.getContext(), fnName),
       opBuilder.getArrayAttr(interface));
   return success();
@@ -402,7 +402,7 @@ Deserializer::processOp<spirv::ExecutionModeOp>(ArrayRef<uint32_t> words) {
     attrListElems.push_back(opBuilder.getI32IntegerAttr(words[wordIndex++]));
   }
   auto values = opBuilder.getArrayAttr(attrListElems);
-  opBuilder.create<spirv::ExecutionModeOp>(
+  spirv::ExecutionModeOp::create(opBuilder,
       unknownLoc, SymbolRefAttr::get(opBuilder.getContext(), fn.getName()),
       execMode, values);
   return success();
@@ -441,7 +441,7 @@ Deserializer::processOp<spirv::FunctionCallOp>(ArrayRef<uint32_t> operands) {
     arguments.push_back(value);
   }
 
-  auto opFunctionCall = opBuilder.create<spirv::FunctionCallOp>(
+  auto opFunctionCall = spirv::FunctionCallOp::create(opBuilder,
       unknownLoc, resultType,
       SymbolRefAttr::get(opBuilder.getContext(), functionName), arguments);
 
@@ -518,7 +518,7 @@ Deserializer::processOp<spirv::CopyMemoryOp>(ArrayRef<uint32_t> words) {
   }
 
   Location loc = createFileLineColLoc(opBuilder);
-  opBuilder.create<spirv::CopyMemoryOp>(loc, resultTypes, operands, attributes);
+  spirv::CopyMemoryOp::create(opBuilder, loc, resultTypes, operands, attributes);
 
   return success();
 }
@@ -549,7 +549,7 @@ LogicalResult Deserializer::processOp<spirv::GenericCastToPtrExplicitOp>(
   operands.push_back(arg);
 
   Location loc = createFileLineColLoc(opBuilder);
-  Operation *op = opBuilder.create<spirv::GenericCastToPtrExplicitOp>(
+  Operation *op = spirv::GenericCastToPtrExplicitOp::create(opBuilder,
       loc, resultTypes, operands);
   valueMap[valueID] = op->getResult(0);
   return success();
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index b1abd8b3dffe9..7b1f8055fadce 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -518,7 +518,7 @@ spirv::Deserializer::processFunction(ArrayRef<uint32_t> operands) {
   }
 
   std::string fnName = getFunctionSymbol(fnID);
-  auto funcOp = opBuilder.create<spirv::FuncOp>(
+  auto funcOp = spirv::FuncOp::create(opBuilder,
       unknownLoc, fnName, functionType, fnControl.value());
   // Processing other function attributes.
   if (decorations.count(fnID)) {
@@ -706,7 +706,7 @@ spirv::SpecConstantOp
 spirv::Deserializer::createSpecConstant(Location loc, uint32_t resultID,
                                         TypedAttr defaultValue) {
   auto symName = opBuilder.getStringAttr(getSpecConstantSymbol(resultID));
-  auto op = opBuilder.create<spirv::SpecConstantOp>(unknownLoc, symName,
+  auto op = spirv::SpecConstantOp::create(opBuilder, unknownLoc, symName,
                                                     defaultValue);
   if (decorations.count(resultID)) {
     for (auto attr : decorations[resultID].getAttrs())
@@ -782,7 +782,7 @@ spirv::Deserializer::processGlobalVariable(ArrayRef<uint32_t> operands) {
            << wordIndex << " of " << operands.size() << " processed";
   }
   auto loc = createFileLineColLoc(opBuilder);
-  auto varOp = opBuilder.create<spirv::GlobalVariableOp>(
+  auto varOp = spirv::GlobalVariableOp::create(opBuilder,
       loc, TypeAttr::get(type), opBuilder.getStringAttr(variableName),
       initializer);
 
@@ -1581,7 +1581,7 @@ spirv::Deserializer::processSpecConstantComposite(ArrayRef<uint32_t> operands) {
     elements.push_back(SymbolRefAttr::get(elementInfo));
   }
 
-  auto op = opBuilder.create<spirv::SpecConstantCompositeOp>(
+  auto op = spirv::SpecConstantCompositeOp::create(opBuilder,
       unknownLoc, TypeAttr::get(resultType), symName,
       opBuilder.getArrayAttr(elements));
   specConstCompositeMap[resultID] = op;
@@ -1656,7 +1656,7 @@ Value spirv::Deserializer::materializeSpecConstantOperation(
 
   auto loc = createFileLineColLoc(opBuilder);
   auto specConstOperationOp =
-      opBuilder.create<spirv::SpecConstantOperationOp>(loc, resultType);
+      spirv::SpecConstantOperationOp::create(opBuilder, loc, resultType);
 
   Region &body = specConstOperationOp.getBody();
   // Move the new block into SpecConstantOperation's body.
@@ -1669,7 +1669,7 @@ Value spirv::Deserializer::materializeSpecConstantOperation(
   OpBuilder::InsertionGuard moduleInsertionGuard(opBuilder);
   opBuilder.setInsertionPointToEnd(&block);
 
-  opBuilder.create<spirv::YieldOp>(loc, block.front().getResult(0));
+  spirv::YieldOp::create(opBuilder, loc, block.front().getResult(0));
   return specConstOperationOp.getResult();
 }
 
@@ -1733,7 +1733,7 @@ LogicalResult spirv::Deserializer::processBranch(ArrayRef<uint32_t> operands) {
   // The preceding instruction for the OpBranch instruction could be an
   // OpLoopMerge or an OpSelectionMerge instruction, in this case they will have
   // the same OpLine information.
-  opBuilder.create<spirv::BranchOp>(loc, target);
+  spirv::BranchOp::create(opBuilder, loc, target);
 
   clearDebugLine();
   return success();
@@ -1764,7 +1764,7 @@ spirv::Deserializer::processBranchConditional(ArrayRef<uint32_t> operands) {
   // an OpSelectionMerge instruction, in this case they will have the same
   // OpLine information.
   auto loc = createFileLineColLoc(opBuilder);
-  opBuilder.create<spirv::BranchConditionalOp>(
+  spirv::BranchConditionalOp::create(opBuilder,
       loc, condition, trueBlock,
       /*trueArguments=*/ArrayRef<Value>(), falseBlock,
       /*falseArguments=*/ArrayRef<Value>(), weights);
@@ -1947,7 +1947,7 @@ ControlFlowStructurizer::createSelectionOp(uint32_t selectionControl) {
   OpBuilder builder(&mergeBlock->front());
 
   auto control = static_cast<spirv::SelectionControl>(selectionControl);
-  auto selectionOp = builder.create<spirv::SelectionOp>(location, control);
+  auto selectionOp = spirv::SelectionOp::create(builder, location, control);
   selectionOp.addMergeBlock(builder);
 
   return selectionOp;
@@ -1959,7 +1959,7 @@ spirv::LoopOp ControlFlowStructurizer::createLoopOp(uint32_t loopControl) {
   OpBuilder builder(&mergeBlock->front());
 
   auto control = static_cast<spirv::LoopControl>(loopControl);
-  auto loopOp = builder.create<spirv::LoopOp>(location, control);
+  auto loopOp = spirv::LoopOp::create(builder, location, control);
   loopOp.addEntryAndMergeBlock(builder);
 
   return loopOp;
@@ -2092,7 +2092,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
     // The loop entry block should have a unconditional branch jumping to the
     // loop header block.
     builder.setInsertionPointToEnd(&body.front());
-    builder.create<spirv::BranchOp>(location, mapper.lookupOrNull(headerBlock),
+    spirv::BranchOp::create(builder, location, mapper.lookupOrNull(headerBlock),
                                     ArrayRef<Value>(blockArgs));
   }
 
@@ -2177,11 +2177,11 @@ LogicalResult ControlFlowStructurizer::structurize() {
     Operation *newOp = nullptr;
 
     if (isLoop)
-      newOp = builder.create<spirv::LoopOp>(
+      newOp = spirv::LoopOp::create(builder,
           location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::LoopControl>(control));
     else
-      newOp = builder.create<spirv::SelectionOp>(
+      newOp = spirv::SelectionOp::create(builder,
           location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::SelectionControl>(control));
 
@@ -2308,7 +2308,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
       // but replace all ops inside with a branch to the merge block.
       block->clear();
       builder.setInsertionPointToEnd(block);
-      builder.create<spirv::BranchOp>(location, mergeBlock);
+      spirv::BranchOp::create(builder, location, mergeBlock);
     } else {
       LLVM_DEBUG(logger.startLine() << "[cf] erasing block " << block << "\n");
       block->erase();
@@ -2362,7 +2362,7 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
 
     if (auto branchOp = dyn_cast<spirv::BranchOp>(op)) {
       // Replace the previous branch op with a new one with block arguments.
-      opBuilder.create<spirv::BranchOp>(branchOp.getLoc(), branchOp.getTarget(),
+      spirv::BranchOp::create(opBuilder, branchOp.getLoc(), branchOp.getTarget(),
                                         blockArgs);
       branchOp.erase();
     } else if (auto branchCondOp = dyn_cast<spirv::BranchConditionalOp>(op)) {
@@ -2370,13 +2370,13 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
               branchCondOp.getFalseBlock() == target) &&
              "expected target to be either the true or false target");
       if (target == branchCondOp.getTrueTarget())
-        opBuilder.create<spirv::BranchConditionalOp>(
+        spirv::BranchConditionalOp::create(opBuilder,
             branchCondOp.getLoc(), branchCondOp.getCondition(), blockArgs,
             branchCondOp.getFalseBlockArguments(),
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueTarget(),
             branchCondOp.getFalseTarget());
       else
-        opBuilder.create<spirv::BranchConditionalOp>(
+        spirv::BranchConditionalOp::create(opBuilder,
             branchCondOp.getLoc(), branchCondOp.getCondition(),
             branchCondOp.getTrueBlockArguments(), blockArgs,
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueBlock(),
@@ -2437,7 +2437,7 @@ LogicalResult spirv::Deserializer::splitConditionalBlocks() {
     if (!llvm::hasSingleElement(*block) || splitHeaderMergeBlock) {
       Block *newBlock = block->splitBlock(terminator);
       OpBuilder builder(block, block->end());
-      builder.create<spirv::BranchOp>(block->getParent()->getLoc(), newBlock);
+      spirv::BranchOp::create(builder, block->getParent()->getLoc(), newBlock);
 
       // After splitting we need to update the map to use the new block as a
       // header.
diff --git a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
index 824201d17b5ab..3f4e7762fed0e 100644
--- a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
+++ b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
@@ -134,7 +134,7 @@ class CodeGen {
 
 OwningOpRef<ModuleOp> CodeGen::generate(const ast::Module &module) {
   OwningOpRef<ModuleOp> mlirModule =
-      builder.create<ModuleOp>(genLoc(module.getLoc()));
+      ModuleOp::create(builder, genLoc(module.getLoc()));
   builder.setInsertionPointToStart(mlirModule->getBody());
 
   // Generate code for each of the decls within the module.
@@ -205,7 +205,7 @@ static void checkAndNestUnderRewriteOp(OpBuilder &builder, Value rootExpr,
                                        Location loc) {
   if (isa<pdl::PatternOp>(builder.getInsertionBlock()->getParentOp())) {
     pdl::RewriteOp rewrite =
-        builder.create<pdl::RewriteOp>(loc, rootExpr, /*name=*/StringAttr(),
+        pdl::RewriteOp::create(builder, loc, rootExpr, /*name=*/StringAttr(),
                                        /*externalArgs=*/ValueRange());
     builder.createBlock(&rewrite.getBodyRegion());
   }
@@ -219,7 +219,7 @@ void CodeGen::genImpl(const ast::EraseStmt *stmt) {
   // Make sure we are nested in a RewriteOp.
   OpBuilder::InsertionGuard guard(builder);
   checkAndNestUnderRewriteOp(builder, rootExpr, loc);
-  builder.create<pdl::EraseOp>(loc, rootExpr);
+  pdl::EraseOp::create(builder, loc, rootExpr);
 }
 
 void CodeGen::genImpl(const ast::LetStmt *stmt) { genVar(stmt->getVarDecl()); }
@@ -242,7 +242,7 @@ void CodeGen::genImpl(const ast::ReplaceStmt *stmt) {
   bool usesReplOperation =
       replValues.size() == 1 &&
       isa<pdl::OperationType>(replValues.front().getType());
-  builder.create<pdl::ReplaceOp>(
+  pdl::ReplaceOp::create(builder,
       loc, rootExpr, usesReplOperation ? replValues[0] : Value(),
       usesReplOperation ? ValueRange() : ValueRange(replValues));
 }
@@ -283,7 +283,7 @@ void CodeGen::genImpl(const ast::PatternDecl *decl) {
 
   // FIXME: Properly model HasBoundedRecursion in PDL so that we don't drop it
   // here.
-  pdl::PatternOp pattern = builder.create<pdl::PatternOp>(
+  pdl::PatternOp pattern = pdl::PatternOp::create(builder,
       genLoc(decl->getLoc()), decl->getBenefit(),
       name ? std::optional<StringRef>(name->getName())
            : std::optional<StringRef>());
@@ -338,19 +338,19 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   ast::Type type = varDecl->getType();
   Type mlirType = genType(type);
   if (isa<ast::ValueType>(type))
-    return builder.create<pdl::OperandOp>(loc, mlirType, getTypeConstraint());
+    return pdl::OperandOp::create(builder, loc, mlirType, getTypeConstraint());
   if (isa<ast::TypeType>(type))
-    return builder.create<pdl::TypeOp>(loc, mlirType, /*type=*/TypeAttr());
+    return pdl::TypeOp::create(builder, loc, mlirType, /*type=*/TypeAttr());
   if (isa<ast::AttributeType>(type))
-    return builder.create<pdl::AttributeOp>(loc, getTypeConstraint());
+    return pdl::AttributeOp::create(builder, loc, getTypeConstraint());
   if (ast::OperationType opType = dyn_cast<ast::OperationType>(type)) {
-    Value operands = builder.create<pdl::OperandsOp>(
+    Value operands = pdl::OperandsOp::create(builder,
         loc, pdl::RangeType::get(builder.getType<pdl::ValueType>()),
         /*type=*/Value());
-    Value results = builder.create<pdl::TypesOp>(
+    Value results = pdl::TypesOp::create(builder,
         loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
         /*types=*/ArrayAttr());
-    return builder.create<pdl::OperationOp>(loc, opType.getName(), operands,
+    return pdl::OperationOp::create(builder, loc, opType.getName(), operands,
                                             ArrayRef<StringRef>(), ValueRange(),
                                             results);
   }
@@ -358,10 +358,10 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   if (ast::RangeType rangeTy = dyn_cast<ast::RangeType>(type)) {
     ast::Type eleTy = rangeTy.getElementType();
     if (isa<ast::ValueType>(eleTy))
-      return builder.create<pdl::OperandsOp>(loc, mlirType,
+      return pdl::OperandsOp::create(builder, loc, mlirType,
                                              getTypeConstraint());
     if (isa<ast::TypeType>(eleTy))
-      return builder.create<pdl::TypesOp>(loc, mlirType, /*types=*/ArrayAttr());
+      return pdl::TypesOp::create(builder, loc, mlirType, /*types=*/ArrayAttr());
   }
 
   llvm_unreachable("invalid non-initialized variable type");
@@ -404,7 +404,7 @@ SmallVector<Value> CodeGen::genExpr(const ast::Expr *expr) {
 Value CodeGen::genExprImpl(const ast::AttributeExpr *expr) {
   Attribute attr = parseAttribute(expr->getValue(), builder.getContext());
   assert(attr && "invalid MLIR attribute data");
-  return builder.create<pdl::AttributeOp>(genLoc(expr->getLoc()), attr);
+  return pdl::AttributeOp::create(builder, genLoc(expr->getLoc()), attr);
 }
 
 SmallVector<Value> CodeGen::genExprImpl(const ast::CallExpr *expr) {
@@ -443,9 +443,9 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
     if (isa<ast::AllResultsMemberAccessExpr>(expr)) {
       Type mlirType = genType(expr->getType());
       if (isa<pdl::ValueType>(mlirType))
-        return builder.create<pdl::ResultOp>(loc, mlirType, parentExprs[0],
+        return pdl::ResultOp::create(builder, loc, mlirType, parentExprs[0],
                                              builder.getI32IntegerAttr(0));
-      return builder.create<pdl::ResultsOp>(loc, mlirType, parentExprs[0]);
+      return pdl::ResultsOp::create(builder, loc, mlirType, parentExprs[0]);
     }
 
     const ods::Operation *odsOp = opType.getODSOperation();
@@ -455,7 +455,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
       unsigned resultIndex;
       name.getAsInteger(/*Radix=*/10, resultIndex);
       IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
-      return builder.create<pdl::ResultOp>(loc, genType(expr->getType()),
+      return pdl::ResultOp::create(builder, loc, genType(expr->getType()),
                                            parentExprs[0], index);
     }
 
@@ -474,7 +474,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
 
     // Generate the result access.
     IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
-    return builder.create<pdl::ResultsOp>(loc, genType(expr->getType()),
+    return pdl::ResultsOp::create(builder, loc, genType(expr->getType()),
                                           parentExprs[0], index);
   }
 
@@ -518,7 +518,7 @@ Value CodeGen::genExprImpl(const ast::OperationExpr *expr) {
   for (const ast::Expr *result : expr->getResultTypes())
     results.push_back(genSingleExpr(result));
 
-  return builder.create<pdl::OperationOp>(loc, opName, operands, attrNames,
+  return pdl::OperationOp::create(builder, loc, opName, operands, attrNames,
                                           attrValues, results);
 }
 
@@ -527,7 +527,7 @@ Value CodeGen::genExprImpl(const ast::RangeExpr *expr) {
   for (const ast::Expr *element : expr->getElements())
     llvm::append_range(elements, genExpr(element));
 
-  return builder.create<pdl::RangeOp>(genLoc(expr->getLoc()),
+  return pdl::RangeOp::create(builder, genLoc(expr->getLoc()),
                                       genType(expr->getType()), elements);
 }
 
@@ -541,7 +541,7 @@ SmallVector<Value> CodeGen::genExprImpl(const ast::TupleExpr *expr) {
 Value CodeGen::genExprImpl(const ast::TypeExpr *expr) {
   Type type = parseType(expr->getValue(), builder.getContext());
   assert(type && "invalid MLIR type data");
-  return builder.create<pdl::TypeOp>(genLoc(expr->getLoc()),
+  return pdl::TypeOp::create(builder, genLoc(expr->getLoc()),
                                      builder.getType<pdl::TypeType>(),
                                      TypeAttr::get(type));
 }
@@ -586,7 +586,7 @@ CodeGen::genConstraintOrRewriteCall(const T *decl, Location loc,
     } else {
       resultTypes.push_back(genType(declResultType));
     }
-    PDLOpT pdlOp = builder.create<PDLOpT>(loc, resultTypes,
+    PDLOpT pdlOp = PDLOpT::create(builder, loc, resultTypes,
                                           decl->getName().getName(), inputs);
     if (isNegated && std::is_same_v<PDLOpT, pdl::ApplyNativeConstraintOp>)
       cast<pdl::ApplyNativeConstraintOp>(pdlOp).setIsNegated(true);
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index d118fe422f2f2..68b8335ed9262 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -1505,7 +1505,7 @@ ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
   OpBuilder builder(outputTypes.front().getContext());
   builder.setInsertionPoint(ip.getBlock(), ip.getPoint());
   auto convertOp =
-      builder.create<UnrealizedConversionCastOp>(loc, outputTypes, inputs);
+      UnrealizedConversionCastOp::create(builder, loc, outputTypes, inputs);
   if (!valuesToMap.empty())
     mapping.map(valuesToMap, convertOp.getResults());
   if (castOp)

>From e8d226d756df2460ca4343ad034fd585dc2b619c Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 12:56:56 -0400
Subject: [PATCH 3/5] fix cpp custom builders

---
 mlir/include/mlir/Dialect/Arith/IR/Arith.h | 10 +++
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp     | 79 ++++++++++++++++++----
 mlir/lib/IR/BuiltinDialect.cpp             |  2 +-
 3 files changed, 76 insertions(+), 15 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 7c50c2036ffdc..79b09b5fcaccf 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -59,15 +59,21 @@ class ConstantIntOp : public arith::ConstantOp {
   /// Build a constant int op that produces an integer of the specified width.
   static void build(OpBuilder &builder, OperationState &result, int64_t value,
                     unsigned width);
+  static ConstantIntOp create(OpBuilder &builder, Location location,
+                              int64_t value, unsigned width);
 
   /// Build a constant int op that produces an integer of the specified type,
   /// which must be an integer type.
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     int64_t value);
+  static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
+                              int64_t value);
 
   /// Build a constant int op that produces an integer from an APInt
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     const APInt &value);
+  static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
+                              const APInt &value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
@@ -85,6 +91,8 @@ class ConstantFloatOp : public arith::ConstantOp {
   /// Build a constant float op that produces a float of the specified type.
   static void build(OpBuilder &builder, OperationState &result, FloatType type,
                     const APFloat &value);
+  static ConstantFloatOp create(OpBuilder &builder, Location location,
+                                FloatType type, const APFloat &value);
 
   inline APFloat value() {
     return cast<FloatAttr>(arith::ConstantOp::getValue()).getValue();
@@ -100,6 +108,8 @@ class ConstantIndexOp : public arith::ConstantOp {
   static ::mlir::TypeID resolveTypeID() { return TypeID::get<ConstantOp>(); }
   /// Build a constant int op that produces an index.
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
+  static ConstantIndexOp create(OpBuilder &builder, Location location,
+                                int64_t value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index c9b0c8414b775..aa623c19a9218 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -256,18 +256,49 @@ void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location,
+                                                  int64_t value,
+                                                  unsigned width) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, value, width);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, int64_t value) {
   arith::ConstantOp::build(builder, result, type,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location, Type type,
+                                                  int64_t value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, const APInt &value) {
   arith::ConstantOp::build(builder, result, type,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location, Type type,
+                                                  const APInt &value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -280,6 +311,17 @@ void arith::ConstantFloatOp::build(OpBuilder &builder, OperationState &result,
                            builder.getFloatAttr(type, value));
 }
 
+arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
+                                                      Location location,
+                                                      FloatType type,
+                                                      const APFloat &value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantFloatOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantFloatOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return llvm::isa<FloatType>(constOp.getType());
@@ -292,6 +334,16 @@ void arith::ConstantIndexOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIndexAttr(value));
 }
 
+arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
+                                                      Location location,
+                                                      int64_t value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, value);
+  auto result = llvm::dyn_cast<ConstantIndexOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantIndexOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isIndex();
@@ -2335,9 +2387,8 @@ class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
     // comparison.
     rewriter.replaceOpWithNewOp<CmpIOp>(
         op, pred, intVal,
-        ConstantOp::create(rewriter,
-            op.getLoc(), intVal.getType(),
-            rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
+        ConstantOp::create(rewriter, op.getLoc(), intVal.getType(),
+                           rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
     return success();
   }
 };
@@ -2374,10 +2425,10 @@ struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
         matchPattern(op.getFalseValue(), m_One())) {
       rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
           op, op.getType(),
-          arith::XOrIOp::create(rewriter,
-              op.getLoc(), op.getCondition(),
-              arith::ConstantIntOp::create(rewriter,
-                  op.getLoc(), op.getCondition().getType(), 1)));
+          arith::XOrIOp::create(
+              rewriter, op.getLoc(), op.getCondition(),
+              arith::ConstantIntOp::create(rewriter, op.getLoc(),
+                                           op.getCondition().getType(), 1)));
       return success();
     }
 
@@ -2440,12 +2491,12 @@ OpFoldResult arith::SelectOp::fold(FoldAdaptor adaptor) {
 
   // Constant-fold constant operands over non-splat constant condition.
   // select %cst_vec, %cst0, %cst1 => %cst2
-  if (auto cond =
-          llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getCondition())) {
-    if (auto lhs =
-            llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getTrueValue())) {
-      if (auto rhs =
-              llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getFalseValue())) {
+  if (auto cond = llvm::dyn_cast_if_present<DenseElementsAttr>(
+          adaptor.getCondition())) {
+    if (auto lhs = llvm::dyn_cast_if_present<DenseElementsAttr>(
+            adaptor.getTrueValue())) {
+      if (auto rhs = llvm::dyn_cast_if_present<DenseElementsAttr>(
+              adaptor.getFalseValue())) {
         SmallVector<Attribute> results;
         results.reserve(static_cast<size_t>(cond.getNumElements()));
         auto condVals = llvm::make_range(cond.value_begin<BoolAttr>(),
@@ -2713,7 +2764,7 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
     return arith::MaximumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minimumf:
     return arith::MinimumFOp::create(builder, loc, lhs, rhs);
-   case AtomicRMWKind::maxnumf:
+  case AtomicRMWKind::maxnumf:
     return arith::MaxNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minnumf:
     return arith::MinNumFOp::create(builder, loc, lhs, rhs);
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index c88b328282275..6d7e2aa0ece7d 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -132,7 +132,7 @@ void ModuleOp::build(OpBuilder &builder, OperationState &state,
 /// Construct a module from the given context.
 ModuleOp ModuleOp::create(Location loc, std::optional<StringRef> name) {
   OpBuilder builder(loc->getContext());
-  return ModuleOp::create(builder, loc, name);
+  return builder.create<ModuleOp>(loc, name);
 }
 
 DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {

>From 949f3a13eb07f55418bebe6697105179d74988d3 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 17:06:46 -0400
Subject: [PATCH 4/5] fix arith+smt+create impmlicit

---
 .../mlir/Dialect/Affine/IR/AffineOps.h        |  21 ++++
 mlir/include/mlir/Dialect/Arith/IR/Arith.h    |   9 ++
 .../Linalg/TransformOps/LinalgTransformOps.td |  80 ++++++------
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   6 +-
 mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td    |   6 +-
 mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td |   6 +-
 .../mlir/Dialect/Vector/IR/VectorOps.td       |  12 +-
 mlir/include/mlir/IR/OpDefinition.h           |   1 +
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    | 116 +++++++++---------
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    |   2 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      |   2 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  98 ++++++++++-----
 .../Affine/Transforms/SuperVectorize.cpp      |  49 ++++----
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |  28 +++++
 .../Async/Transforms/AsyncParallelFor.cpp     | 105 ++++++++--------
 .../GPU/Transforms/MemoryPromotion.cpp        |   6 +-
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp   |  38 ++++--
 17 files changed, 352 insertions(+), 233 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
index 2091faa6b0b02..333de6bbd8a05 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
@@ -114,6 +114,21 @@ class AffineDmaStartOp
                     AffineMap tagMap, ValueRange tagIndices, Value numElements,
                     Value stride = nullptr, Value elementsPerStride = nullptr);
 
+  static AffineDmaStartOp
+  create(OpBuilder &builder, Location location, Value srcMemRef,
+         AffineMap srcMap, ValueRange srcIndices, Value destMemRef,
+         AffineMap dstMap, ValueRange destIndices, Value tagMemRef,
+         AffineMap tagMap, ValueRange tagIndices, Value numElements,
+         Value stride = nullptr, Value elementsPerStride = nullptr);
+
+  static AffineDmaStartOp create(ImplicitLocOpBuilder &builder, Value srcMemRef,
+                                 AffineMap srcMap, ValueRange srcIndices,
+                                 Value destMemRef, AffineMap dstMap,
+                                 ValueRange destIndices, Value tagMemRef,
+                                 AffineMap tagMap, ValueRange tagIndices,
+                                 Value numElements, Value stride = nullptr,
+                                 Value elementsPerStride = nullptr);
+
   /// Returns the operand index of the source memref.
   unsigned getSrcMemRefOperandIndex() { return 0; }
 
@@ -319,6 +334,12 @@ class AffineDmaWaitOp
 
   static void build(OpBuilder &builder, OperationState &result, Value tagMemRef,
                     AffineMap tagMap, ValueRange tagIndices, Value numElements);
+  static AffineDmaWaitOp create(OpBuilder &builder, Location location,
+                                Value tagMemRef, AffineMap tagMap,
+                                ValueRange tagIndices, Value numElements);
+  static AffineDmaWaitOp create(ImplicitLocOpBuilder &builder, Value tagMemRef,
+                                AffineMap tagMap, ValueRange tagIndices,
+                                Value numElements);
 
   static StringRef getOperationName() { return "affine.dma_wait"; }
 
diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 79b09b5fcaccf..0fc3db8e993d8 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -61,6 +61,8 @@ class ConstantIntOp : public arith::ConstantOp {
                     unsigned width);
   static ConstantIntOp create(OpBuilder &builder, Location location,
                               int64_t value, unsigned width);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, int64_t value,
+                              unsigned width);
 
   /// Build a constant int op that produces an integer of the specified type,
   /// which must be an integer type.
@@ -68,12 +70,16 @@ class ConstantIntOp : public arith::ConstantOp {
                     int64_t value);
   static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
                               int64_t value);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, Type type,
+                              int64_t value);
 
   /// Build a constant int op that produces an integer from an APInt
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     const APInt &value);
   static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
                               const APInt &value);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, Type type,
+                              const APInt &value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
@@ -93,6 +99,8 @@ class ConstantFloatOp : public arith::ConstantOp {
                     const APFloat &value);
   static ConstantFloatOp create(OpBuilder &builder, Location location,
                                 FloatType type, const APFloat &value);
+  static ConstantFloatOp create(ImplicitLocOpBuilder &builder, FloatType type,
+                                const APFloat &value);
 
   inline APFloat value() {
     return cast<FloatAttr>(arith::ConstantOp::getValue()).getValue();
@@ -110,6 +118,7 @@ class ConstantIndexOp : public arith::ConstantOp {
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
   static ConstantIndexOp create(OpBuilder &builder, Location location,
                                 int64_t value);
+  static ConstantIndexOp create(ImplicitLocOpBuilder &builder, int64_t value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 4360055e78691..c5c984e09bf67 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -582,9 +582,9 @@ def LinalgCopyToMemrefOp :
   let assemblyFormat = "$target attr-dict `:` "
                        "functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -997,12 +997,12 @@ def PackGreedilyOp : Op<Transform_Dialect, "structured.pack_greedily", [
                                  [DenseArrayCount<3>]>:$matmul_inner_dims_order);
   let results = (outs TransformHandleTypeInterface:$packed_op);
 
-  let builders = [
-    OpBuilder<(ins "Value":$target,
-                   "ArrayRef<OpFoldResult>":$mixedMatmulPackedSizes,
-                   "ArrayRef<int64_t>":$matmulPaddededSizesNextMultipleOf,
-                   CArg<"ArrayRef<int64_t>", "{}">:$matmulDimsInnerDimsOrder)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target,
+  //                  "ArrayRef<OpFoldResult>":$mixedMatmulPackedSizes,
+  //                  "ArrayRef<int64_t>":$matmulPaddededSizesNextMultipleOf,
+  //                  CArg<"ArrayRef<int64_t>", "{}">:$matmulDimsInnerDimsOrder)>
+  // ];
 
   let assemblyFormat = [{
     $target
@@ -2509,10 +2509,10 @@ def HoistRedundantVectorTransfersOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target,
-               CArg<"bool", "false">:$verify_non_zero_trip)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target,
+  //              CArg<"bool", "false">:$verify_non_zero_trip)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
          ::mlir::transform::TransformRewriter &rewriter,
@@ -2546,9 +2546,9 @@ def HoistRedundantVectorBroadcastsOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
          ::mlir::transform::TransformRewriter &rewriter,
@@ -2623,9 +2623,9 @@ def ConvertConv2DToImg2ColOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2666,9 +2666,9 @@ def FlattenElementwiseLinalgOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2715,9 +2715,9 @@ def TransposeConv2DOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2761,9 +2761,9 @@ def TransposeMatmulOp : Op<Transform_Dialect,
     attr-dict `:` functional-type($target, results)
   }];
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2801,9 +2801,9 @@ def InsertSliceToCopyOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2859,9 +2859,9 @@ def MapCopyToThreadsOp :
     `:` functional-type(operands, results)
   }];
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>,
+  // ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2910,9 +2910,9 @@ def WinogradConv2DOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
@@ -2947,9 +2947,9 @@ def DecomposeWinogradOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$target)>
+  // ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..118877ea145fd 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1751,9 +1751,9 @@ def DeclareMapperInfoOp : OpenMP_Op<"declare_mapper.info", [
     parent DeclareMapperOp.
   }] # clausesDescription;
 
-  let builders = [
-    OpBuilder<(ins CArg<"const DeclareMapperInfoOperands &">:$clauses)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins CArg<"const DeclareMapperInfoOperands &">:$clauses)>
+  // ];
 
   let extraClassDeclaration = [{
     // Override BlockArgOpenMPOpInterface method because `map` clauses have no
diff --git a/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td b/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
index 1523762efc18f..7fecf89605f06 100644
--- a/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
+++ b/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
@@ -171,9 +171,9 @@ def Ptr_TypeOffsetOp : Pointer_Op<"type_offset", [Pure]> {
 
   let arguments = (ins TypeAttr:$elementType);
   let results = (outs AnySignlessIntegerOrIndex:$result);
-  let builders = [
-    OpBuilder<(ins "Type":$elementType)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Type":$elementType)>
+  // ];
   let assemblyFormat = [{
     $elementType attr-dict `:` type($result)
   }];
diff --git a/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td b/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
index ea94dfd8fbd2a..5a919395c0954 100644
--- a/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
+++ b/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
@@ -54,9 +54,9 @@ class VariadicIntOp<string mnemonic> : SMTIntOp<mnemonic, [Pure, Commutative]> {
   let assemblyFormat = "$inputs attr-dict";
 
   let builders = [
-    OpBuilder<(ins "mlir::ValueRange":$inputs), [{
-      build($_builder, $_state, $_builder.getType<smt::IntType>(), inputs);
-    }]>,
+    // OpBuilder<(ins "mlir::ValueRange":$inputs), [{
+    //   build($_builder, $_state, $_builder.getType<smt::IntType>(), inputs);
+    // }]>,
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index ec2c87ca1cf44..5d0c34ff8da34 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -3011,12 +3011,12 @@ def Vector_ScanOp :
       vector<4x8x16x32xf32>, vector<4x16x32xf32>
     ```
   }];
-  let builders = [
-    OpBuilder<(ins "Value":$source, "Value":$initial_value,
-                   "CombiningKind":$kind,
-                   CArg<"int64_t", "0">:$reduction_dim,
-                   CArg<"bool", "true">:$inclusive)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins "Value":$source, "Value":$initial_value,
+  //                  "CombiningKind":$kind,
+  //                  CArg<"int64_t", "0">:$reduction_dim,
+  //                  CArg<"bool", "true">:$inclusive)>
+  // ];
   let extraClassDeclaration = [{
     VectorType getSourceType() {
       return ::llvm::cast<VectorType>(getSource().getType());
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index 663c256c848df..75c3aea0792ac 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -30,6 +30,7 @@
 namespace mlir {
 class Builder;
 class OpBuilder;
+class ImplicitLocOpBuilder;
 
 /// This class implements `Optional` functionality for ParseResult. We don't
 /// directly use Optional here, because it provides an implicit conversion
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 3f82342001ea5..69eec9d9bbf3e 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -20,8 +20,8 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
+#include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
@@ -114,7 +114,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
   auto makeConst = [&](int32_t index) -> Value {
     return LLVM::ConstantOp::create(rewriter, loc, IntegerType::get(ctx, 32),
-                                             rewriter.getI32IntegerAttr(index));
+                                    rewriter.getI32IntegerAttr(index));
   };
 
   if (arrayType) {
@@ -147,11 +147,11 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
         Value x1 =
             LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i * 2);
         Value x2 = LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult,
-                                                         i * 2 + 1);
+                                                i * 2 + 1);
         vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
-                                                     x1, makeConst(0));
+                                            x1, makeConst(0));
         vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
-                                                     x2, makeConst(1));
+                                            x2, makeConst(1));
         elements.push_back(vec);
       }
     }
@@ -160,7 +160,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
     Value result = LLVM::PoisonOp::create(rewriter, loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
       result = LLVM::InsertValueOp::create(rewriter, loc, result, el.value(),
-                                                    el.index());
+                                           el.index());
     }
     return result;
   }
@@ -208,8 +208,8 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
                          innerArrayTy.getElementType() == f32Ty)) {
       for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
            idx < innerSize; idx++) {
-        result.push_back(LLVM::ExtractElementOp::create(b,
-            toUse,
+        result.push_back(LLVM::ExtractElementOp::create(
+            b, toUse,
             LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(idx))));
       }
       continue;
@@ -285,8 +285,8 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
-    Value ldMatrixResult = NVVM::LdMatrixOp::create(b,
-        ldMatrixResultType, srcPtr,
+    Value ldMatrixResult = NVVM::LdMatrixOp::create(
+        b, ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
                                      : NVVM::MMALayout::row);
@@ -375,16 +375,16 @@ struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
     Type desiredRetTy = typeConverter->convertType(op->getResultTypes()[0]);
     Type intrinsicResTy = inferIntrinsicResultType(
         typeConverter->convertType(op->getResultTypes()[0]));
-    Value intrinsicResult = NVVM::MmaOp::create(b,
-        intrinsicResTy, matA, matB, matC,
-        /*shape=*/gemmShape,
-        /*b1Op=*/std::nullopt,
-        /*intOverflow=*/overflow,
-        /*multiplicandPtxTypes=*/
-        std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
-        /*multiplicandLayouts=*/
-        std::array<NVVM::MMALayout, 2>{NVVM::MMALayout::row,
-                                       NVVM::MMALayout::col});
+    Value intrinsicResult =
+        NVVM::MmaOp::create(b, intrinsicResTy, matA, matB, matC,
+                            /*shape=*/gemmShape,
+                            /*b1Op=*/std::nullopt,
+                            /*intOverflow=*/overflow,
+                            /*multiplicandPtxTypes=*/
+                            std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
+                            /*multiplicandLayouts=*/
+                            std::array<NVVM::MMALayout, 2>{
+                                NVVM::MMALayout::row, NVVM::MMALayout::col});
     rewriter.replaceOp(op, convertIntrinsicResult(op.getLoc(), intrinsicResTy,
                                                   desiredRetTy, intrinsicResult,
                                                   rewriter));
@@ -566,14 +566,15 @@ static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
   asmVals.push_back(indexData);
 
   return LLVM::InlineAsmOp::create(b,
-      /*resultTypes=*/intrinsicResultType,
-      /*operands=*/asmVals,
-      /*asm_string=*/asmStr,
-      /*constraints=*/constraintStr,
-      /*has_side_effects=*/true,
-      /*is_align_stack=*/false, LLVM::TailCallKind::None,
-      /*asm_dialect=*/asmDialectAttr,
-      /*operand_attrs=*/ArrayAttr());
+                                   /*resultTypes=*/intrinsicResultType,
+                                   /*operands=*/asmVals,
+                                   /*asm_string=*/asmStr,
+                                   /*constraints=*/constraintStr,
+                                   /*has_side_effects=*/true,
+                                   /*is_align_stack=*/false,
+                                   LLVM::TailCallKind::None,
+                                   /*asm_dialect=*/asmDialectAttr,
+                                   /*operand_attrs=*/ArrayAttr());
 }
 
 /// Lowers `nvgpu.mma.sp.sync` to inline assembly.
@@ -698,12 +699,12 @@ struct NVGPUAsyncCopyLowering
       // filled with zeros.
       Value c3I32 =
           LLVM::ConstantOp::create(b, b.getI32Type(), b.getI32IntegerAttr(3));
-      Value bitwidth = LLVM::ConstantOp::create(b,
-          b.getI32Type(),
+      Value bitwidth = LLVM::ConstantOp::create(
+          b, b.getI32Type(),
           b.getI32IntegerAttr(srcMemrefType.getElementTypeBitWidth()));
       Value srcElementsI32 = LLVM::TruncOp::create(b, b.getI32Type(), srcBytes);
-      srcBytes = LLVM::LShrOp::create(b,
-          LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
+      srcBytes = LLVM::LShrOp::create(
+          b, LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
     }
     // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
     // 16 dst bytes.
@@ -712,14 +713,15 @@ struct NVGPUAsyncCopyLowering
             ? NVVM::LoadCacheModifierKind::CG
             : NVVM::LoadCacheModifierKind::CA;
 
-    NVVM::CpAsyncOp::create(b,
-        dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
+    NVVM::CpAsyncOp::create(
+        b, dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
         NVVM::LoadCacheModifierKindAttr::get(op->getContext(), cacheModifier),
         srcBytes);
 
     // Drop the result token.
-    Value zero = LLVM::ConstantOp::create(b,
-        IntegerType::get(op.getContext(), 32), rewriter.getI32IntegerAttr(0));
+    Value zero =
+        LLVM::ConstantOp::create(b, IntegerType::get(op.getContext(), 32),
+                                 rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
   }
@@ -735,9 +737,9 @@ struct NVGPUAsyncCreateGroupLowering
                   ConversionPatternRewriter &rewriter) const override {
     NVVM::CpAsyncCommitGroupOp::create(rewriter, op.getLoc());
     // Drop the result token.
-    Value zero = LLVM::ConstantOp::create(rewriter,
-        op->getLoc(), IntegerType::get(op.getContext(), 32),
-        rewriter.getI32IntegerAttr(0));
+    Value zero = LLVM::ConstantOp::create(rewriter, op->getLoc(),
+                                          IntegerType::get(op.getContext(), 32),
+                                          rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
   }
@@ -771,8 +773,8 @@ struct NVGPUMBarrierCreateLowering
     SymbolTable symbolTable(moduleOp);
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(&moduleOp.front());
-    auto global = memref::GlobalOp::create(rewriter,
-        funcOp->getLoc(), "__mbarrier",
+    auto global = memref::GlobalOp::create(
+        rewriter, funcOp->getLoc(), "__mbarrier",
         /*sym_visibility=*/rewriter.getStringAttr("private"),
         /*type=*/barrierType,
         /*initial_value=*/ElementsAttr(),
@@ -1119,7 +1121,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 
 static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
   return LLVM::ConstantOp::create(b, b.getIntegerType(64),
-                                    b.getI32IntegerAttr(index));
+                                  b.getI32IntegerAttr(index));
 }
 
 /// Returns a Value that holds data type enum that is expected by CUDA driver.
@@ -1182,11 +1184,11 @@ struct NVGPUTmaCreateDescriptorOpLowering
     auto promotedOperands = getTypeConverter()->promoteOperands(
         b.getLoc(), op->getOperands(), adaptor.getOperands(), b);
 
-    Value boxArrayPtr = LLVM::AllocaOp::create(b, llvmPointerType, llvmInt64Type,
-                                                 makeI64Const(b, 5));
+    Value boxArrayPtr = LLVM::AllocaOp::create(
+        b, llvmPointerType, llvmInt64Type, makeI64Const(b, 5));
     for (auto [index, value] : llvm::enumerate(adaptor.getBoxDimensions())) {
       Value gep = LLVM::GEPOp::create(b, llvmPointerType, llvmPointerType,
-                                        boxArrayPtr, makeI64Const(b, index));
+                                      boxArrayPtr, makeI64Const(b, index));
       LLVM::StoreOp::create(b, value, gep);
     }
 
@@ -1430,9 +1432,9 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      return NVVM::WgmmaMmaAsyncOp::create(b,
-          matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
-          itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
+      return NVVM::WgmmaMmaAsyncOp::create(
+          b, matrixC.getType(), matrixC, descriptorA, descriptorB, shape,
+          itypeA, itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
           overflow);
     }
 
@@ -1444,7 +1446,8 @@ struct NVGPUWarpgroupMmaOpLowering
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
+        Value matrixC =
+            LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
             matrixC = generateWgmma(i, j, k, matrixC);
@@ -1452,7 +1455,7 @@ struct NVGPUWarpgroupMmaOpLowering
       }
       for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
         wgmmaResult = LLVM::InsertValueOp::create(b, wgmmaResult.getType(),
-                                                    wgmmaResult, matrix, idx);
+                                                  wgmmaResult, matrix, idx);
       }
       return wgmmaResult;
     }
@@ -1486,9 +1489,9 @@ struct NVGPUWarpgroupMmaOpLowering
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
     Value generateWarpgroupMma() {
-      NVVM::WgmmaFenceAlignedOp::create(b, );
+      NVVM::WgmmaFenceAlignedOp::create(b);
       Value wgmmaResult = generateWgmmaGroup();
-      NVVM::WgmmaGroupSyncAlignedOp::create(b, );
+      NVVM::WgmmaGroupSyncAlignedOp::create(b);
       NVVM::WgmmaWaitGroupSyncOp::create(b, op.getWaitGroup());
       return wgmmaResult;
     }
@@ -1626,7 +1629,8 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     auto stype = cast<LLVM::LLVMStructType>(matriDValue.getType());
     for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(matrixD);
-      Value innerStructValue = LLVM::ExtractValueOp::create(b, matriDValue, idx);
+      Value innerStructValue =
+          LLVM::ExtractValueOp::create(b, matriDValue, idx);
       storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
@@ -1656,15 +1660,15 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
       auto structType = cast<LLVM::LLVMStructType>(s);
       Value structValue = LLVM::ExtractValueOp::create(b, packStruct, idx);
       for (unsigned i = 0; i < structType.getBody().size(); ++i) {
-        structValue = LLVM::InsertValueOp::create(b,
-            structType, structValue, zero, ArrayRef<int64_t>({i}));
+        structValue = LLVM::InsertValueOp::create(b, structType, structValue,
+                                                  zero, ArrayRef<int64_t>({i}));
       }
       innerStructs.push_back(structValue);
     }
     // Pack the inner structs into a single struct
     for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
       packStruct = LLVM::InsertValueOp::create(b, packStruct.getType(),
-                                                 packStruct, matrix, idx);
+                                               packStruct, matrix, idx);
     }
     rewriter.replaceOp(op, packStruct);
     return success();
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 901b1eb265e32..4e8468078b9ad 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -709,7 +709,7 @@ class ExecutionModePattern
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
-    structValue = LLVM::InsertValueOp::create(rewriter, loc, structValue,
+    structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
                                                        executionMode, 0);
 
     // Insert extra operands if they exist into execution mode info struct.
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 1b31d391cb0bc..d21833736a718 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -670,7 +670,7 @@ static Value createFPReductionComparisonOpLowering(
 
   if (accumulator) {
     result =
-        typename VectorToScalarMapper<LLVMRedIntrinOp>::Type::create(rewriter,
+        rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
             loc, result, accumulator);
   }
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 263663d041f3a..95cca386d2360 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -1749,6 +1749,32 @@ void AffineDmaStartOp::build(OpBuilder &builder, OperationState &result,
   }
 }
 
+AffineDmaStartOp AffineDmaStartOp::create(
+    OpBuilder &builder, Location location, Value srcMemRef, AffineMap srcMap,
+    ValueRange srcIndices, Value destMemRef, AffineMap dstMap,
+    ValueRange destIndices, Value tagMemRef, AffineMap tagMap,
+    ValueRange tagIndices, Value numElements, Value stride,
+    Value elementsPerStride) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, srcMemRef, srcMap, srcIndices, destMemRef, dstMap,
+        destIndices, tagMemRef, tagMap, tagIndices, numElements, stride,
+        elementsPerStride);
+  auto result = llvm::dyn_cast<AffineDmaStartOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
+AffineDmaStartOp AffineDmaStartOp::create(
+    ImplicitLocOpBuilder &builder, Value srcMemRef, AffineMap srcMap,
+    ValueRange srcIndices, Value destMemRef, AffineMap dstMap,
+    ValueRange destIndices, Value tagMemRef, AffineMap tagMap,
+    ValueRange tagIndices, Value numElements, Value stride,
+    Value elementsPerStride) {
+  return create(builder, builder.getLoc(), srcMemRef, srcMap, srcIndices,
+                destMemRef, dstMap, destIndices, tagMemRef, tagMap, tagIndices,
+                numElements, stride, elementsPerStride);
+}
+
 void AffineDmaStartOp::print(OpAsmPrinter &p) {
   p << " " << getSrcMemRef() << '[';
   p.printAffineMapOfSSAIds(getSrcMapAttr(), getSrcIndices());
@@ -1919,6 +1945,25 @@ void AffineDmaWaitOp::build(OpBuilder &builder, OperationState &result,
   result.addOperands(numElements);
 }
 
+AffineDmaWaitOp AffineDmaWaitOp::create(OpBuilder &builder, Location location,
+                                        Value tagMemRef, AffineMap tagMap,
+                                        ValueRange tagIndices,
+                                        Value numElements) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, tagMemRef, tagMap, tagIndices, numElements);
+  auto result = llvm::dyn_cast<AffineDmaWaitOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
+AffineDmaWaitOp AffineDmaWaitOp::create(ImplicitLocOpBuilder &builder,
+                                        Value tagMemRef, AffineMap tagMap,
+                                        ValueRange tagIndices,
+                                        Value numElements) {
+  return create(builder, builder.getLoc(), tagMemRef, tagMap, tagIndices,
+                numElements);
+}
+
 void AffineDmaWaitOp::print(OpAsmPrinter &p) {
   p << " " << getTagMemRef() << '[';
   SmallVector<Value, 2> operands(getTagIndices());
@@ -2690,8 +2735,8 @@ FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInits());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  AffineForOp newLoop = AffineForOp::create(rewriter,
-      getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
+  AffineForOp newLoop = AffineForOp::create(
+      rewriter, getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
       getUpperBoundOperands(), getUpperBoundMap(), getStepAsInt(), inits);
 
   // Generate the new yield values and append them to the scf.yield operation.
@@ -2849,7 +2894,7 @@ buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb,
                              int64_t ub, int64_t step,
                              AffineForOp::BodyBuilderFn bodyBuilderFn) {
   return AffineForOp::create(builder, loc, lb, ub, step,
-                                     /*iterArgs=*/ValueRange(), bodyBuilderFn);
+                             /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
 /// Creates an affine loop from the bounds that may or may not be constants.
@@ -2863,8 +2908,8 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
     return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
                                         ubConst.value(), step, bodyBuilderFn);
   return AffineForOp::create(builder, loc, lb, builder.getDimIdentityMap(), ub,
-                                     builder.getDimIdentityMap(), step,
-                                     /*iterArgs=*/ValueRange(), bodyBuilderFn);
+                             builder.getDimIdentityMap(), step,
+                             /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
 void mlir::affine::buildAffineLoopNest(
@@ -4908,8 +4953,8 @@ struct DropUnitExtentBasis
 
     if (!newBasis.empty()) {
       // Will drop the leading nullptr from `basis` if there was no outer bound.
-      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(rewriter,
-          loc, delinearizeOp.getLinearIndex(), newBasis);
+      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(
+          rewriter, loc, delinearizeOp.getLinearIndex(), newBasis);
       int newIndex = 0;
       // Map back the new delinearized indices to the values they replace.
       for (auto &replacement : replacements) {
@@ -4973,12 +5018,12 @@ struct CancelDelinearizeOfLinearizeDisjointExactTail
       return success();
     }
 
-    Value newLinearize = affine::AffineLinearizeIndexOp::create(rewriter,
-        linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
+    Value newLinearize = affine::AffineLinearizeIndexOp::create(
+        rewriter, linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
         ArrayRef<OpFoldResult>{linearizeBasis}.drop_back(numMatches),
         linearizeOp.getDisjoint());
-    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-        delinearizeOp.getLoc(), newLinearize,
+    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), newLinearize,
         ArrayRef<OpFoldResult>{delinearizeBasis}.drop_back(numMatches),
         delinearizeOp.hasOuterBound());
     SmallVector<Value> mergedResults(newDelinearize.getResults());
@@ -5050,19 +5095,16 @@ struct SplitDelinearizeSpanningLastLinearizeArg final
           delinearizeOp,
           "need at least two elements to form the basis product");
 
-    Value linearizeWithoutBack =
-        affine::AffineLinearizeIndexOp::create(rewriter,
-            linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
-            linearizeOp.getDynamicBasis(),
-            linearizeOp.getStaticBasis().drop_back(),
-            linearizeOp.getDisjoint());
-    auto delinearizeWithoutSplitPart =
-        affine::AffineDelinearizeIndexOp::create(rewriter,
-            delinearizeOp.getLoc(), linearizeWithoutBack,
-            delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
-            delinearizeOp.hasOuterBound());
-    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(rewriter,
-        delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
+    Value linearizeWithoutBack = affine::AffineLinearizeIndexOp::create(
+        rewriter, linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
+        linearizeOp.getDynamicBasis(), linearizeOp.getStaticBasis().drop_back(),
+        linearizeOp.getDisjoint());
+    auto delinearizeWithoutSplitPart = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), linearizeWithoutBack,
+        delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
+        delinearizeOp.hasOuterBound());
+    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
         basis.take_back(elemsToSplit), /*hasOuterBound=*/true);
     SmallVector<Value> results = llvm::to_vector(
         llvm::concat<Value>(delinearizeWithoutSplitPart.getResults(),
@@ -5439,16 +5481,16 @@ struct CancelLinearizeOfDelinearizePortion final
       newDelinBasis.erase(newDelinBasis.begin() + m.delinStart,
                           newDelinBasis.begin() + m.delinStart + m.length);
       newDelinBasis.insert(newDelinBasis.begin() + m.delinStart, newSize);
-      auto newDelinearize = AffineDelinearizeIndexOp::create(rewriter,
-          m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
+      auto newDelinearize = AffineDelinearizeIndexOp::create(
+          rewriter, m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
           newDelinBasis);
 
       // Since there may be other uses of the indices we just merged together,
       // create a residual affine.delinearize_index that delinearizes the
       // merged output into its component parts.
       Value combinedElem = newDelinearize.getResult(m.delinStart);
-      auto residualDelinearize = AffineDelinearizeIndexOp::create(rewriter,
-          m.delinearize.getLoc(), combinedElem, basisToMerge);
+      auto residualDelinearize = AffineDelinearizeIndexOp::create(
+          rewriter, m.delinearize.getLoc(), combinedElem, basisToMerge);
 
       // Swap all the uses of the unaffected delinearize outputs to the new
       // delinearization so that the old code can be removed if this
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index b078514cbf978..827e1dff9f319 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -905,8 +905,8 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   for (auto resultExpr : map.getResults()) {
     auto singleResMap =
         AffineMap::get(map.getNumDims(), map.getNumSymbols(), resultExpr);
-    auto afOp = state.AffineApplyOp::create(builder, op->getLoc(), singleResMap,
-                                                    mapOperands);
+    auto afOp = AffineApplyOp::create(state.builder, op->getLoc(), singleResMap,
+                                      mapOperands);
     results.push_back(afOp);
   }
 }
@@ -961,7 +961,7 @@ static arith::ConstantOp vectorizeConstant(arith::ConstantOp constOp,
   auto vecForOp = cast<AffineForOp>(parentOp);
   state.builder.setInsertionPointToStart(vecForOp.getBody());
   auto newConstOp =
-      state.arith::ConstantOp::create(builder, constOp.getLoc(), vecAttr);
+      arith::ConstantOp::create(state.builder, constOp.getLoc(), vecAttr);
 
   // Register vector replacement for future uses in the scope.
   state.registerOpVectorReplacement(constOp, newConstOp);
@@ -986,8 +986,8 @@ static Operation *vectorizeAffineApplyOp(AffineApplyOp applyOp,
     }
   }
 
-  auto newApplyOp = state.AffineApplyOp::create(builder,
-      applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
+  auto newApplyOp = AffineApplyOp::create(
+      state.builder, applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
 
   // Register the new affine.apply result.
   state.registerValueScalarReplacement(applyOp.getResult(),
@@ -1010,7 +1010,7 @@ static arith::ConstantOp createInitialVector(arith::AtomicRMWKind reductionKind,
   auto vecTy = getVectorType(scalarTy, state.strategy);
   auto vecAttr = DenseElementsAttr::get(vecTy, valueAttr);
   auto newConstOp =
-      state.arith::ConstantOp::create(builder, oldOperand.getLoc(), vecAttr);
+      arith::ConstantOp::create(state.builder, oldOperand.getLoc(), vecAttr);
 
   return newConstOp;
 }
@@ -1062,11 +1062,11 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   AffineMap ubMap = vecForOp.getUpperBoundMap();
   Value ub;
   if (ubMap.getNumResults() == 1)
-    ub = state.AffineApplyOp::create(builder, loc, vecForOp.getUpperBoundMap(),
-                                             vecForOp.getUpperBoundOperands());
+    ub = AffineApplyOp::create(state.builder, loc, vecForOp.getUpperBoundMap(),
+                               vecForOp.getUpperBoundOperands());
   else
-    ub = state.AffineMinOp::create(builder, loc, vecForOp.getUpperBoundMap(),
-                                           vecForOp.getUpperBoundOperands());
+    ub = AffineMinOp::create(state.builder, loc, vecForOp.getUpperBoundMap(),
+                             vecForOp.getUpperBoundOperands());
   // Then we compute the number of (original) iterations left in the loop.
   AffineExpr subExpr =
       state.builder.getAffineDimExpr(0) - state.builder.getAffineDimExpr(1);
@@ -1080,7 +1080,7 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   Type maskTy = VectorType::get(state.strategy->vectorSizes,
                                 state.builder.getIntegerType(1));
   Value mask =
-      state.vector::CreateMaskOp::create(builder, loc, maskTy, itersLeft);
+      vector::CreateMaskOp::create(state.builder, loc, maskTy, itersLeft);
 
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ creating a mask:\n"
                     << itersLeft << "\n"
@@ -1123,8 +1123,8 @@ static Operation *vectorizeUniform(Value uniformVal,
   state.builder.setInsertionPointAfterValue(uniformScalarRepl);
 
   auto vectorTy = getVectorType(uniformVal.getType(), state.strategy);
-  auto bcastOp = state.BroadcastOp::create(builder, uniformVal.getLoc(),
-                                                   vectorTy, uniformScalarRepl);
+  auto bcastOp = BroadcastOp::create(state.builder, uniformVal.getLoc(),
+                                     vectorTy, uniformScalarRepl);
   state.registerValueVectorReplacement(uniformVal, bcastOp);
   return bcastOp;
 }
@@ -1256,8 +1256,8 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.vector::TransferReadOp::create(builder,
-      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
+  auto transfer = vector::TransferReadOp::create(
+      state.builder, loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
       /*padding=*/std::nullopt, permutationMap);
 
   // Register replacement for future uses in the scope.
@@ -1303,9 +1303,9 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.vector::TransferWriteOp::create(builder,
-      storeOp.getLoc(), vectorValue, storeOp.getMemRef(), indices,
-      permutationMap);
+  auto transfer = vector::TransferWriteOp::create(
+      state.builder, storeOp.getLoc(), vectorValue, storeOp.getMemRef(),
+      indices, permutationMap);
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
 
   // Register replacement for future uses in the scope.
@@ -1387,10 +1387,10 @@ static Operation *vectorizeAffineForOp(AffineForOp forOp,
     }
   }
 
-  auto vecForOp = state.AffineForOp::create(builder,
-      forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(),
-      forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep,
-      vecIterOperands,
+  auto vecForOp = AffineForOp::create(
+      state.builder, forOp.getLoc(), forOp.getLowerBoundOperands(),
+      forOp.getLowerBoundMap(), forOp.getUpperBoundOperands(),
+      forOp.getUpperBoundMap(), newStep, vecIterOperands,
       /*bodyBuilder=*/[](OpBuilder &, Location, Value, ValueRange) {
         // Make sure we don't create a default terminator in the loop body as
         // the proper terminator will be added during vectorization.
@@ -1512,8 +1512,8 @@ static Operation *vectorizeAffineYieldOp(AffineYieldOp yieldOp,
       // IterOperands are neutral element vectors.
       Value neutralVal = cast<AffineForOp>(newParentOp).getInits()[i];
       state.builder.setInsertionPoint(combinerOps.back());
-      Value maskedReducedVal = state.arith::SelectOp::create(builder,
-          reducedVal.getLoc(), mask, reducedVal, neutralVal);
+      Value maskedReducedVal = arith::SelectOp::create(
+          state.builder, reducedVal.getLoc(), mask, reducedVal, neutralVal);
       LLVM_DEBUG(
           dbgs() << "\n[early-vect]+++++ masking an input to a binary op that"
                     "produces value for a yield Op: "
@@ -1865,7 +1865,6 @@ verifyLoopNesting(const std::vector<SmallVector<AffineForOp, 2>> &loops) {
   return success();
 }
 
-
 /// External utility to vectorize affine loops in 'loops' using the n-D
 /// vectorization factors in 'vectorSizes'. By default, each vectorization
 /// factor is applied inner-to-outer to the loops of each loop nest.
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index aa623c19a9218..164489422a3e4 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -267,6 +267,12 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  int64_t value,
+                                                  unsigned width) {
+  return create(builder, builder.getLoc(), value, width);
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, int64_t value) {
   arith::ConstantOp::build(builder, result, type,
@@ -283,6 +289,11 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  Type type, int64_t value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, const APInt &value) {
   arith::ConstantOp::build(builder, result, type,
@@ -299,6 +310,12 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  Type type,
+                                                  const APInt &value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -322,6 +339,12 @@ arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantFloatOp
+arith::ConstantFloatOp::create(ImplicitLocOpBuilder &builder, FloatType type,
+                               const APFloat &value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 bool arith::ConstantFloatOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return llvm::isa<FloatType>(constOp.getType());
@@ -344,6 +367,11 @@ arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIndexOp
+arith::ConstantIndexOp::create(ImplicitLocOpBuilder &builder, int64_t value) {
+  return create(builder, builder.getLoc(), value);
+}
+
 bool arith::ConstantIndexOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isIndex();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index 447411ad989ba..09367ab544ff8 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -369,21 +369,22 @@ static ParallelComputeFunction createParallelComputeFunction(
       ImplicitLocOpBuilder b(loc, nestedBuilder);
 
       // Compute induction variable for `loopIdx`.
-      computeBlockInductionVars[loopIdx] = arith::AddIOp::create(b,
-          lowerBounds[loopIdx], arith::MulIOp::create(b, iv, steps[loopIdx]));
+      computeBlockInductionVars[loopIdx] =
+          arith::AddIOp::create(b, lowerBounds[loopIdx],
+                                arith::MulIOp::create(b, iv, steps[loopIdx]));
 
       // Check if we are inside first or last iteration of the loop.
-      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
-      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
+      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
+      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
 
       // Check if the previous loop is in its first or last iteration.
       if (loopIdx > 0) {
-        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(b,
-            isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
-        isBlockLastCoord[loopIdx] = arith::AndIOp::create(b,
-            isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
+        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(
+            b, isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
+        isBlockLastCoord[loopIdx] = arith::AndIOp::create(
+            b, isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
       }
 
       // Keep building loop nest.
@@ -392,20 +393,20 @@ static ParallelComputeFunction createParallelComputeFunction(
           // For block aligned loops we always iterate starting from 0 up to
           // the loop trip counts.
           scf::ForOp::create(b, c0, tripCounts[loopIdx + 1], c1, ValueRange(),
-                               workLoopBuilder(loopIdx + 1));
+                             workLoopBuilder(loopIdx + 1));
 
         } else {
           // Select nested loop lower/upper bounds depending on our position in
           // the multi-dimensional iteration space.
           auto lb = arith::SelectOp::create(b, isBlockFirstCoord[loopIdx],
-                                              blockFirstCoord[loopIdx + 1], c0);
+                                            blockFirstCoord[loopIdx + 1], c0);
 
           auto ub = arith::SelectOp::create(b, isBlockLastCoord[loopIdx],
-                                              blockEndCoord[loopIdx + 1],
-                                              tripCounts[loopIdx + 1]);
+                                            blockEndCoord[loopIdx + 1],
+                                            tripCounts[loopIdx + 1]);
 
           scf::ForOp::create(b, lb, ub, c1, ValueRange(),
-                               workLoopBuilder(loopIdx + 1));
+                             workLoopBuilder(loopIdx + 1));
         }
 
         scf::YieldOp::create(b, loc);
@@ -424,7 +425,7 @@ static ParallelComputeFunction createParallelComputeFunction(
   };
 
   scf::ForOp::create(b, blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
-                       workLoopBuilder(0));
+                     workLoopBuilder(0));
   func::ReturnOp::create(b, ValueRange());
 
   return {op.getNumLoops(), func, std::move(computeFuncType.captures)};
@@ -537,13 +538,13 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
       operands[2] = end;
 
       func::CallOp::create(executeBuilder, executeLoc, func.getSymName(),
-                                          func.getResultTypes(), operands);
+                           func.getResultTypes(), operands);
       async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to dispatch half of the block range.
     auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
-                                       executeBodyBuilder);
+                                     executeBodyBuilder);
     AddToGroupOp::create(b, indexTy, execute.getToken(), group);
     scf::YieldOp::create(b, ValueRange({start, midIndex}));
   }
@@ -558,8 +559,7 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   computeFuncOperands.append(forwardedInputs.begin(), forwardedInputs.end());
 
   func::CallOp::create(b, computeFunc.func.getSymName(),
-                         computeFunc.func.getResultTypes(),
-                         computeFuncOperands);
+                       computeFunc.func.getResultTypes(), computeFuncOperands);
   func::ReturnOp::create(b, ValueRange());
 
   return func;
@@ -605,9 +605,9 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     appendBlockComputeOperands(operands);
 
     func::CallOp::create(b, parallelComputeFunction.func.getSymName(),
-                           parallelComputeFunction.func.getResultTypes(),
-                           operands);
-    scf::YieldOp::create(b, );
+                         parallelComputeFunction.func.getResultTypes(),
+                         operands);
+    scf::YieldOp::create(b);
   };
 
   auto asyncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -624,12 +624,12 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     appendBlockComputeOperands(operands);
 
     func::CallOp::create(b, asyncDispatchFunction.getSymName(),
-                           asyncDispatchFunction.getResultTypes(), operands);
+                         asyncDispatchFunction.getResultTypes(), operands);
 
     // Wait for the completion of all parallel compute operations.
     AwaitAllOp::create(b, group);
 
-    scf::YieldOp::create(b, );
+    scf::YieldOp::create(b);
   };
 
   // Dispatch either single block compute function, or launch async dispatch.
@@ -682,16 +682,15 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
                                   Location executeLoc, ValueRange executeArgs) {
       func::CallOp::create(executeBuilder, executeLoc, compute.getSymName(),
-                                          compute.getResultTypes(),
-                                          computeFuncOperands(iv));
+                           compute.getResultTypes(), computeFuncOperands(iv));
       async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to launch parallel computate function.
     auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
-                                       executeBodyBuilder);
+                                     executeBodyBuilder);
     AddToGroupOp::create(b, rewriter.getIndexType(), execute.getToken(), group);
-    scf::YieldOp::create(b, );
+    scf::YieldOp::create(b);
   };
 
   // Iterate over all compute blocks and launch parallel compute operations.
@@ -699,7 +698,7 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
 
   // Call parallel compute function for the first block in the caller thread.
   func::CallOp::create(b, compute.getSymName(), compute.getResultTypes(),
-                         computeFuncOperands(c0));
+                       computeFuncOperands(c0));
 
   // Wait for the completion of all async compute operations.
   AwaitAllOp::create(b, group);
@@ -800,7 +799,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     if (numWorkerThreads >= 0)
       numWorkerThreadsVal = arith::ConstantIndexOp::create(b, numWorkerThreads);
     else
-      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b, );
+      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b);
 
     // With large number of threads the value of creating many compute blocks
     // is reduced because the problem typically becomes memory bound. For this
@@ -819,16 +818,16 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
         {4, 4.0f}, {8, 2.0f}, {16, 1.0f}, {32, 0.8f}, {64, 0.6f}};
     const float initialOvershardingFactor = 8.0f;
 
-    Value scalingFactor = arith::ConstantFloatOp::create(b,
-        b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
+    Value scalingFactor = arith::ConstantFloatOp::create(
+        b, b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
     for (const std::pair<int, float> &p : overshardingBrackets) {
       Value bracketBegin = arith::ConstantIndexOp::create(b, p.first);
-      Value inBracket = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
-      Value bracketScalingFactor = arith::ConstantFloatOp::create(b,
-          b.getF32Type(), llvm::APFloat(p.second));
-      scalingFactor = arith::SelectOp::create(b, inBracket, bracketScalingFactor,
-                                                scalingFactor);
+      Value inBracket = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
+      Value bracketScalingFactor = arith::ConstantFloatOp::create(
+          b, b.getF32Type(), llvm::APFloat(p.second));
+      scalingFactor = arith::SelectOp::create(
+          b, inBracket, bracketScalingFactor, scalingFactor);
     }
     Value numWorkersIndex =
         arith::IndexCastOp::create(b, b.getI32Type(), numWorkerThreadsVal);
@@ -841,8 +840,8 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     Value scaledWorkers =
         arith::IndexCastOp::create(b, b.getIndexType(), scaledNumInt);
 
-    Value maxComputeBlocks = arith::MaxSIOp::create(b,
-        arith::ConstantIndexOp::create(b, 1), scaledWorkers);
+    Value maxComputeBlocks = arith::MaxSIOp::create(
+        b, arith::ConstantIndexOp::create(b, 1), scaledWorkers);
 
     // Compute parallel block size from the parallel problem size:
     //   blockSize = min(tripCount,
@@ -869,7 +868,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       doDispatch(b, rewriter, compute, op, blockSize, blockCount, tripCounts);
-      scf::YieldOp::create(b, );
+      scf::YieldOp::create(b);
     };
 
     // Dispatch parallel compute function with hints for unrolling inner loops.
@@ -880,27 +879,27 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       // Align the block size to be a multiple of the statically known
       // number of iterations in the inner loops.
-      Value numIters = arith::ConstantIndexOp::create(b,
-          numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value alignedBlockSize = arith::MulIOp::create(b,
-          arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
+      Value numIters = arith::ConstantIndexOp::create(
+          b, numIterations[op.getNumLoops() - numUnrollableLoops]);
+      Value alignedBlockSize = arith::MulIOp::create(
+          b, arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
       doDispatch(b, rewriter, compute, op, alignedBlockSize, blockCount,
                  tripCounts);
-      scf::YieldOp::create(b, );
+      scf::YieldOp::create(b);
     };
 
     // Dispatch to block aligned compute function only if the computed block
     // size is larger than the number of iterations in the unrollable inner
     // loops, because otherwise it can reduce the available parallelism.
     if (numUnrollableLoops > 0) {
-      Value numIters = arith::ConstantIndexOp::create(b,
-          numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value useBlockAlignedComputeFn = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::sge, blockSize, numIters);
+      Value numIters = arith::ConstantIndexOp::create(
+          b, numIterations[op.getNumLoops() - numUnrollableLoops]);
+      Value useBlockAlignedComputeFn = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::sge, blockSize, numIters);
 
       scf::IfOp::create(b, useBlockAlignedComputeFn, dispatchBlockAligned,
-                          dispatchDefault);
-      scf::YieldOp::create(b, );
+                        dispatchDefault);
+      scf::YieldOp::create(b);
     } else {
       dispatchDefault(b, loc);
     }
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 2bee1a1619665..4af08dad08462 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -18,7 +18,7 @@
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
+#include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
@@ -131,10 +131,10 @@ static void insertCopies(Region &region, Location loc, Value from, Value to) {
 
   auto b = ImplicitLocOpBuilder::atBlockBegin(loc, &region.front());
   insertCopyLoops(b, from, to);
-  gpu::BarrierOp::create(b, );
+  gpu::BarrierOp::create(b);
 
   b.setInsertionPoint(&region.front().back());
-  gpu::BarrierOp::create(b, );
+  gpu::BarrierOp::create(b);
   insertCopyLoops(b, to, from);
 }
 
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 20c483e52da04..f35cfa6826388 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -238,6 +238,10 @@ static const char *const inlineCreateBody = R"(
   return __res__;
 )";
 
+static const char *const inlineCreateBodyImplicitLoc = R"(
+  return create(builder, builder.getLoc(){0});
+)";
+
 //===----------------------------------------------------------------------===//
 // Utility structs and functions
 //===----------------------------------------------------------------------===//
@@ -2579,35 +2583,47 @@ static bool canInferType(const Operator &op) {
 
 void OpEmitter::genInlineCreateBody(
     const SmallVector<MethodParameter> &paramList) {
-  SmallVector<MethodParameter> createParamList;
+  SmallVector<MethodParameter> createParamListOpBuilder;
+  SmallVector<MethodParameter> createParamListImplicitLocOpBuilder;
   SmallVector<llvm::StringRef, 4> nonBuilderStateArgsList;
-  createParamList.emplace_back("::mlir::OpBuilder &", "builder");
+  createParamListOpBuilder.emplace_back("::mlir::OpBuilder &", "builder");
+  createParamListImplicitLocOpBuilder.emplace_back(
+      "::mlir::ImplicitLocOpBuilder &", "builder");
   std::string locParamName = "location";
   while (llvm::find_if(paramList, [&locParamName](const MethodParameter &p) {
            return p.getName() == locParamName;
          }) != paramList.end()) {
     locParamName += "_";
   }
-  createParamList.emplace_back("::mlir::Location", locParamName);
+  createParamListOpBuilder.emplace_back("::mlir::Location", locParamName);
 
   for (auto &param : paramList) {
     if (param.getType() == "::mlir::OpBuilder &" ||
         param.getType() == "::mlir::OperationState &")
       continue;
-    createParamList.emplace_back(param.getType(), param.getName(),
-                                 param.getDefaultValue(), param.isOptional());
+    createParamListOpBuilder.emplace_back(param.getType(), param.getName(),
+                                          param.getDefaultValue(),
+                                          param.isOptional());
+    createParamListImplicitLocOpBuilder.emplace_back(
+        param.getType(), param.getName(), param.getDefaultValue(),
+        param.isOptional());
     nonBuilderStateArgsList.push_back(param.getName());
   }
-  auto *c = opClass.addStaticMethod(opClass.getClassName(), "create",
-                                    createParamList);
+  auto *cWithLoc = opClass.addStaticMethod(opClass.getClassName(), "create",
+                                           createParamListOpBuilder);
+  auto *cImplicitLoc = opClass.addStaticMethod(
+      opClass.getClassName(), "create", createParamListImplicitLocOpBuilder);
   std::string nonBuilderStateArgs = "";
   if (!nonBuilderStateArgsList.empty()) {
     llvm::raw_string_ostream nonBuilderStateArgsOS(nonBuilderStateArgs);
     interleaveComma(nonBuilderStateArgsList, nonBuilderStateArgsOS);
     nonBuilderStateArgs = ", " + nonBuilderStateArgs;
   }
-  c->body() << llvm::formatv(inlineCreateBody, locParamName,
-                             nonBuilderStateArgs, opClass.getClassName());
+  cWithLoc->body() << llvm::formatv(inlineCreateBody, locParamName,
+                                    nonBuilderStateArgs,
+                                    opClass.getClassName());
+  cImplicitLoc->body() << llvm::formatv(inlineCreateBodyImplicitLoc,
+                                        nonBuilderStateArgs);
 }
 
 void OpEmitter::genSeparateArgParamBuilder() {
@@ -3087,8 +3103,7 @@ void OpEmitter::genBuilder() {
 
     std::optional<StringRef> body = builder.getBody();
     auto properties = body ? Method::Static : Method::StaticDeclaration;
-    auto *method =
-        opClass.addMethod("void", "build", properties, std::move(arguments));
+    auto *method = opClass.addMethod("void", "build", properties, arguments);
     if (body)
       ERROR_IF_PRUNED(method, "build", op);
 
@@ -3100,6 +3115,7 @@ void OpEmitter::genBuilder() {
     fctx.addSubst("_state", builderOpState);
     if (body)
       method->body() << tgfmt(*body, &fctx);
+    genInlineCreateBody(arguments);
   }
 
   // Generate default builders that requires all result type, operands, and

>From d0ddc3c6fbaa57667325c9ba9fd19428916292b4 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 18:26:17 -0400
Subject: [PATCH 5/5] implicitbuilder

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  10 +-
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    |   1 +
 flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp   |   1 +
 flang/lib/Optimizer/Dialect/FIROps.cpp        |   1 +
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp     |   1 +
 .../lib/Standalone/StandaloneDialect.cpp      |   1 +
 .../lib/Standalone/StandaloneOps.cpp          |   1 +
 mlir/examples/toy/Ch2/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch3/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch4/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch5/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch6/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch7/mlir/Dialect.cpp        |   4 +-
 .../transform/Ch2/lib/MyExtension.cpp         |   1 +
 .../transform/Ch3/lib/MyExtension.cpp         |   1 +
 .../transform/Ch4/lib/MyExtension.cpp         |   1 +
 mlir/lib/CAPI/Dialect/PDL.cpp                 |   1 +
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    |   1 +
 mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp  |   1 +
 mlir/lib/Dialect/AMX/IR/AMXDialect.cpp        |   1 +
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |   1 +
 .../TransformOps/AffineTransformOps.cpp       |   1 +
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |   1 +
 .../lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp |   1 +
 .../ArmNeonVectorTransformOps.cpp             |   1 +
 mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp         |   1 +
 mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp  |   1 +
 .../TransformOps/ArmSVEVectorTransformOps.cpp |   1 +
 mlir/lib/Dialect/Async/IR/Async.cpp           |   1 +
 .../Bufferization/IR/BufferizationOps.cpp     |  20 ++--
 mlir/lib/Dialect/Complex/IR/ComplexOps.cpp    |   5 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.cpp |  16 +--
 .../DLTI/TransformOps/DLTITransformOps.cpp    |   1 +
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp           |   1 +
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |   7 +-
 .../Func/TransformOps/FuncTransformOps.cpp    |   3 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |   9 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |   5 +-
 .../GPU/Transforms/MemoryPromotion.cpp        |   1 +
 mlir/lib/Dialect/IRDL/IR/IRDL.cpp             |   1 +
 mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp          |   1 +
 mlir/lib/Dialect/Index/IR/IndexOps.cpp        |  16 ++-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |   7 +-
 mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp    |   1 +
 mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp   |   1 +
 mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp    |   1 +
 mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp    |   1 +
 .../lib/Dialect/MLProgram/IR/MLProgramOps.cpp |   1 +
 mlir/lib/Dialect/MPI/IR/MPIOps.cpp            |   5 +-
 mlir/lib/Dialect/Math/IR/MathOps.cpp          |   1 +
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |  27 +++--
 .../TransformOps/MemRefTransformOps.cpp       |   8 +-
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |  15 +--
 mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp    |   1 +
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       |   1 +
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |   1 +
 mlir/lib/Dialect/PDL/IR/PDL.cpp               |   1 +
 mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp   |   1 +
 mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp        |   1 +
 mlir/lib/Dialect/Quant/IR/QuantOps.cpp        |   1 +
 mlir/lib/Dialect/SCF/IR/SCF.cpp               | 106 ++++++++---------
 .../SCF/TransformOps/SCFTransformOps.cpp      |   1 +
 mlir/lib/Dialect/SMT/IR/SMTOps.cpp            |   1 +
 .../Dialect/SPIRV/IR/SPIRVOpDefinition.cpp    |   1 +
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  64 ++++++-----
 .../SparseTensor/IR/SparseTensorDialect.cpp   |   4 +-
 .../TransformOps/SparseTensorTransformOps.cpp |   1 +
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      | 107 +++++++++---------
 .../TransformOps/TensorTransformOps.cpp       |   1 +
 .../DebugExtension/DebugExtension.cpp         |   1 +
 .../DebugExtension/DebugExtensionOps.cpp      |   1 +
 .../lib/Dialect/Transform/IR/TransformOps.cpp |   1 +
 .../IRDLExtension/IRDLExtensionOps.cpp        |   1 +
 .../LoopExtension/LoopExtensionOps.cpp        |   1 +
 .../PDLExtension/PDLExtensionOps.cpp          |   1 +
 mlir/lib/Dialect/UB/IR/UBOps.cpp              |   1 +
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  56 ++++-----
 .../TransformOps/VectorTransformOps.cpp       |   1 +
 .../Dialect/X86Vector/IR/X86VectorDialect.cpp |   3 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   1 +
 mlir/lib/IR/BuiltinDialect.cpp                |   1 +
 mlir/test/lib/Dialect/Test/TestOps.td         |   2 +-
 .../TestTransformDialectExtension.cpp         |   1 +
 .../TestTilingInterfaceTransformOps.cpp       |   1 +
 .../test/lib/Transforms/TestTransformsOps.cpp |   1 +
 mlir/test/python/lib/PythonTestDialect.cpp    |   1 +
 86 files changed, 353 insertions(+), 240 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index b96c0360316ad..39b36587e559b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3624,11 +3624,11 @@ def fir_LocalitySpecifierOp : fir_Op<"local", [IsolatedFromAbove]> {
       attr-dict
   }];
 
-  let builders = [
-    OpBuilder<(ins CArg<"mlir::TypeRange">:$result,
-                   CArg<"mlir::StringAttr">:$sym_name,
-                   CArg<"mlir::TypeAttr">:$type)>
-  ];
+  // let builders = [
+  //   OpBuilder<(ins CArg<"mlir::TypeRange">:$result,
+  //                  CArg<"mlir::StringAttr">:$sym_name,
+  //                  CArg<"mlir::TypeAttr">:$type)>
+  // ];
 
   let extraClassDeclaration = [{
     mlir::BlockArgument getInitMoldArg() {
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index 687007d957225..cef95a13c5a50 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
index 19ad6bed512c7..d7d294ab621d8 100644
--- a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Optimizer/Dialect/FIRCG/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..2381d44e7bb77 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -26,6 +26,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index df6ce12215d26..7f2ec418af40a 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/APInt.h"
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
index 1ea69f9059321..3da966ef9cc85 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
@@ -8,6 +8,7 @@
 
 #include "Standalone/StandaloneDialect.h"
 #include "Standalone/StandaloneOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "Standalone/StandaloneTypes.h"
 
 using namespace mlir;
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
index 55b66b51232f2..ef20049f1b76b 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
@@ -8,6 +8,7 @@
 
 #include "Standalone/StandaloneOps.h"
 #include "Standalone/StandaloneDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 #define GET_OP_CLASSES
 #include "Standalone/StandaloneOps.cpp.inc"
diff --git a/mlir/examples/toy/Ch2/mlir/Dialect.cpp b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
index 489f348c8be52..fc4c3c75dcfda 100644
--- a/mlir/examples/toy/Ch2/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -144,7 +145,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -280,7 +282,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch3/mlir/Dialect.cpp b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
index 708855f18cf45..af3cbb385d332 100644
--- a/mlir/examples/toy/Ch3/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -144,7 +145,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -280,7 +282,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch4/mlir/Dialect.cpp b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
index 076a75a26619b..e3ab656c193a3 100644
--- a/mlir/examples/toy/Ch4/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch5/mlir/Dialect.cpp b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
index fb7c742a01802..81c9e8f9a143f 100644
--- a/mlir/examples/toy/Ch5/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch6/mlir/Dialect.cpp b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
index fb7c742a01802..81c9e8f9a143f 100644
--- a/mlir/examples/toy/Ch6/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch7/mlir/Dialect.cpp b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
index 52881db87d86b..64aa4d8e8995d 100644
--- a/mlir/examples/toy/Ch7/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpImplementation.h"
@@ -429,7 +430,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/transform/Ch2/lib/MyExtension.cpp b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
index b4b27e97d266e..e24714b408e29 100644
--- a/mlir/examples/transform/Ch2/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/examples/transform/Ch3/lib/MyExtension.cpp b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
index 4b2123fa71d31..5af5cc8ac5052 100644
--- a/mlir/examples/transform/Ch3/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
@@ -17,6 +17,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/examples/transform/Ch4/lib/MyExtension.cpp b/mlir/examples/transform/Ch4/lib/MyExtension.cpp
index fa0ffc9dc2e8a..86884d3bee7a0 100644
--- a/mlir/examples/transform/Ch4/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch4/lib/MyExtension.cpp
@@ -13,6 +13,7 @@
 
 #include "MyExtension.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE_MATCHER "transform-matcher"
diff --git a/mlir/lib/CAPI/Dialect/PDL.cpp b/mlir/lib/CAPI/Dialect/PDL.cpp
index bd8b13c6516e2..d2a83da643a89 100644
--- a/mlir/lib/CAPI/Dialect/PDL.cpp
+++ b/mlir/lib/CAPI/Dialect/PDL.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 69eec9d9bbf3e..cd92e1dbba090 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -22,6 +22,7 @@
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index 4613d14461969..79859f0513e84 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 748ff1edbfeb2..111d93e39c15e 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 95cca386d2360..91f85f2f366e8 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
diff --git a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
index b1e40d9b289ec..267179f26c52e 100644
--- a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
+++ b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 164489422a3e4..c4c2a043a8024 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
index 1ded35af78052..5f9cd148f5e0a 100644
--- a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
+++ b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
index d07e6a52d8b5f..f0b7a8a4b0954 100644
--- a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/ArmNeon/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
index cb3a665844872..2d8903df00055 100644
--- a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp b/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
index 594c9b4c270f2..bd7130b667b6b 100644
--- a/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
+++ b/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp b/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
index b2ca4fc1eaa8c..b50679239a840 100644
--- a/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index 71ee198fc0c23..e7d95575c6edf 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -10,6 +10,7 @@
 
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 4308f46bee6ff..b4910d45e1ae6 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include <optional>
 
@@ -322,8 +323,8 @@ struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
         newShape, op.getType().getElementType(), op.getType().getEncoding());
     if (newType == op.getType())
       return failure();
-    auto newOp = AllocTensorOp::create(rewriter,
-        op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
+    auto newOp = AllocTensorOp::create(rewriter, op.getLoc(), newType,
+                                       newDynamicSizes, /*copy=*/Value());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -515,7 +516,7 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
 
     if (source.getType() != cloneOp.getType())
       source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
-                                               cloneOp.getType(), source);
+                                      cloneOp.getType(), source);
     rewriter.replaceOp(cloneOp, source);
     rewriter.eraseOp(redundantDealloc);
     return success();
@@ -644,8 +645,9 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return ToTensorOp::create(builder,
-      loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
+  return ToTensorOp::create(
+      builder, loc, memref::getTensorTypeFromMemRefType(getDest().getType()),
+      getDest(),
       /*restrict=*/true, getWritable());
 }
 
@@ -808,7 +810,7 @@ struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
     auto memrefType = MemRefType::get(srcTensorType.getShape(),
                                       srcTensorType.getElementType());
     Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
-                                               tensorCastOperand.getOperand());
+                                      tensorCastOperand.getOperand());
     rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                 memref);
     return success();
@@ -1016,7 +1018,7 @@ struct DeallocRemoveDuplicateRetainedMemrefs
     // same as the number of condition operands.
     auto newDeallocOp =
         DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
-                                   deallocOp.getConditions(), newRetained);
+                          deallocOp.getConditions(), newRetained);
     SmallVector<Value> replacements(
         llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
           return newDeallocOp.getUpdatedConditions()[idx];
@@ -1037,8 +1039,8 @@ struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     if (deallocOp.getMemrefs().empty()) {
-      Value constFalse = arith::ConstantOp::create(rewriter,
-          deallocOp.getLoc(), rewriter.getBoolAttr(false));
+      Value constFalse = arith::ConstantOp::create(rewriter, deallocOp.getLoc(),
+                                                   rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
                                         constFalse));
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
index fb97045687d65..34008aa5c4d6a 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 
@@ -20,9 +21,7 @@ using namespace mlir::complex;
 // ConstantOp
 //===----------------------------------------------------------------------===//
 
-OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) {
-  return getValue();
-}
+OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValue(); }
 
 void ConstantOp::getAsmResultNames(
     function_ref<void(Value, StringRef)> setNameFn) {
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 38e47cc5e3714..66593783a14f1 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -89,7 +90,7 @@ LogicalResult AssertOp::canonicalize(AssertOp op, PatternRewriter &rewriter) {
   return failure();
 }
 
-// This side effect models "program termination". 
+// This side effect models "program termination".
 void AssertOp::getEffects(
     SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
         &effects) {
@@ -312,8 +313,9 @@ struct SimplifyCondBranchIdenticalSuccessors
       if (std::get<0>(it) == std::get<1>(it))
         mergedOperands.push_back(std::get<0>(it));
       else
-        mergedOperands.push_back(arith::SelectOp::create(rewriter,
-            condbr.getLoc(), condition, std::get<0>(it), std::get<1>(it)));
+        mergedOperands.push_back(
+            arith::SelectOp::create(rewriter, condbr.getLoc(), condition,
+                                    std::get<0>(it), std::get<1>(it)));
     }
 
     rewriter.replaceOpWithNewOp<BranchOp>(condbr, trueDest, mergedOperands);
@@ -412,8 +414,8 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantTrue)
-            constantTrue = arith::ConstantOp::create(rewriter,
-                condbr.getLoc(), ty, rewriter.getBoolAttr(true));
+            constantTrue = arith::ConstantOp::create(
+                rewriter, condbr.getLoc(), ty, rewriter.getBoolAttr(true));
 
           rewriter.modifyOpInPlace(use.getOwner(),
                                    [&] { use.set(constantTrue); });
@@ -427,8 +429,8 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantFalse)
-            constantFalse = arith::ConstantOp::create(rewriter,
-                condbr.getLoc(), ty, rewriter.getBoolAttr(false));
+            constantFalse = arith::ConstantOp::create(
+                rewriter, condbr.getLoc(), ty, rewriter.getBoolAttr(false));
 
           rewriter.modifyOpInPlace(use.getOwner(),
                                    [&] { use.set(constantFalse); });
diff --git a/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp b/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
index 02c41b4fe8113..4dbd1659e24dc 100644
--- a/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
+++ b/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/Utils/Utils.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index f9852257a4a7c..432c90836c66d 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Types.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index 1c0d3cd338006..1ac11f6a774f3 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -55,7 +56,7 @@ Operation *FuncDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                             Type type, Location loc) {
   if (ConstantOp::isBuildableWith(value, type))
     return ConstantOp::create(builder, loc, type,
-                                      llvm::cast<FlatSymbolRefAttr>(value));
+                              llvm::cast<FlatSymbolRefAttr>(value));
   return nullptr;
 }
 
@@ -143,9 +144,7 @@ LogicalResult ConstantOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   return success();
 }
 
-OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) {
-  return getValueAttr();
-}
+OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValueAttr(); }
 
 void ConstantOp::getAsmResultNames(
     function_ref<void(Value, StringRef)> setNameFn) {
diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
index 2ece1bcffb1b0..822c21e90ca21 100644
--- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
+++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
@@ -172,7 +173,7 @@ transform::CastAndCallOp::apply(transform::TransformRewriter &rewriter,
   }
 
   auto callOp = func::CallOp::create(rewriter, insertionPoint->getLoc(),
-                                              targetFunction, inputs);
+                                     targetFunction, inputs);
 
   // Cast the call results back to the expected types. If any conversions fail
   // this is a definite failure as the call has been constructed at this point.
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index f7eb41be287e4..8d6962d2d2be8 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -23,6 +23,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -1382,9 +1383,9 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
                       int32_t offset, int32_t width, ShuffleMode mode) {
   build(builder, result, value,
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(offset)),
+                                  builder.getI32IntegerAttr(offset)),
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(width)),
+                                  builder.getI32IntegerAttr(width)),
         mode);
 }
 
@@ -1396,9 +1397,9 @@ void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
                      int32_t offset, int32_t width) {
   build(builder, result, value,
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(offset)),
+                                  builder.getI32IntegerAttr(offset)),
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(width)));
+                                  builder.getI32IntegerAttr(width)));
 }
 
 LogicalResult RotateOp::verify() {
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index 2035f371694a7..26adfade879d8 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -31,6 +31,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/Visitors.h"
@@ -561,7 +562,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   if (originalBasisWasProvided) {
     for (Value tmpPredicate : builderResult.predicateOps) {
       predicate = predicate ? arith::AndIOp::create(rewriter, loc, predicate,
-                                                             tmpPredicate)
+                                                    tmpPredicate)
                             : tmpPredicate;
     }
   }
@@ -574,7 +575,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   if (predicate) {
     // Step 6.a. If predicated, move at the beginning.
     auto ifOp = scf::IfOp::create(rewriter, loc, predicate,
-                                           /*withElseRegion=*/false);
+                                  /*withElseRegion=*/false);
     targetBlock = ifOp.thenBlock();
     insertionPoint = ifOp.thenBlock()->begin();
   } else {
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 4af08dad08462..4a670373272a7 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
index bcc9f0b109ac2..d7bc15b866686 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
index 7ec3aa2741023..c62e5c9c50183 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/IRDL/IR/IRDL.h"
 #include "mlir/Dialect/IRDL/IRDLSymbols.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/ValueRange.h"
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Index/IR/IndexOps.cpp b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
index 2b32962f9eabb..61089d7665e6c 100644
--- a/mlir/lib/Dialect/Index/IR/IndexOps.cpp
+++ b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Index/IR/IndexAttrs.h"
 #include "mlir/Dialect/Index/IR/IndexDialect.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -130,15 +131,18 @@ canonicalizeAssociativeCommutativeBinaryOp(BinaryOp op,
 
   auto lhsOp = op.getLhs().template getDefiningOp<BinaryOp>();
   if (!lhsOp)
-    return rewriter.notifyMatchFailure(op.getLoc(), "LHS is not the same BinaryOp");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "LHS is not the same BinaryOp");
 
   if (!mlir::matchPattern(lhsOp.getRhs(), mlir::m_Constant()))
-    return rewriter.notifyMatchFailure(op.getLoc(), "RHS of LHS op is not a constant");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "RHS of LHS op is not a constant");
 
   Value c = rewriter.createOrFold<BinaryOp>(op->getLoc(), op.getRhs(),
-                                           lhsOp.getRhs());
+                                            lhsOp.getRhs());
   if (c.getDefiningOp<BinaryOp>())
-    return rewriter.notifyMatchFailure(op.getLoc(), "new BinaryOp was not folded");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "new BinaryOp was not folded");
 
   rewriter.replaceOpWithNewOp<BinaryOp>(op, lhsOp.getLhs(), c);
   return success();
@@ -717,10 +721,10 @@ LogicalResult CmpOp::canonicalize(CmpOp op, PatternRewriter &rewriter) {
   index::CmpOp newCmp;
   if (rhsIsZero)
     newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
-                                           subOp.getLhs(), subOp.getRhs());
+                                  subOp.getLhs(), subOp.getRhs());
   else
     newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
-                                           subOp.getRhs(), subOp.getLhs());
+                                  subOp.getRhs(), subOp.getLhs());
   rewriter.replaceOp(op, newCmp);
   return success();
 }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 546704e903f47..e25a90b1d276e 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
@@ -4313,8 +4314,8 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
   MLIRContext *ctx = builder.getContext();
   auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
-  auto global = LLVM::GlobalOp::create(moduleBuilder,
-      loc, type, /*isConstant=*/true, linkage, name,
+  auto global = LLVM::GlobalOp::create(
+      moduleBuilder, loc, type, /*isConstant=*/true, linkage, name,
       builder.getStringAttr(value), /*alignment=*/0);
 
   LLVMPointerType ptrType = LLVMPointerType::get(ctx);
@@ -4322,7 +4323,7 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   Value globalPtr =
       LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
   return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
-                                     ArrayRef<GEPArg>{0, 0});
+                             ArrayRef<GEPArg>{0, 0});
 }
 
 bool mlir::LLVM::satisfiesLLVMModule(Operation *op) {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index aaf6b0593c2e6..d95b146d27e5c 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -25,6 +25,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
index 9671afd52fa77..0bb8ecf98c533 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
index bd9d3528ceb74..b1676b4e58c39 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
index 9e497829ba723..fbedc714c200b 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp b/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
index 37ddca101f64b..877f8e72caa31 100644
--- a/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
+++ b/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/MLProgram/IR/MLProgram.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index 749cca0575581..da2317cd4eb97 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 
 using namespace mlir;
@@ -60,8 +61,8 @@ struct FoldRank final : public mlir::OpRewritePattern<mlir::mpi::CommRankOp> {
     if (!isa<IntegerAttr>(dltiAttr.value()))
       return op->emitError()
              << "Expected an integer attribute for MPI:comm_world_rank";
-    Value res = arith::ConstantIndexOp::create(b,
-        op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
+    Value res = arith::ConstantIndexOp::create(
+        b, op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
     if (Value retVal = op.getRetval())
       b.replaceOp(op, {retVal, res});
     else
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index a21631cbf8510..b09a67818fff6 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 706ac4b4867f4..cb5fe132494b3 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -213,9 +214,9 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
 
     // Create and insert the alloc op for the new memref.
-    auto newAlloc = AllocLikeOp::create(rewriter,
-        alloc.getLoc(), newMemRefType, dynamicSizes, alloc.getSymbolOperands(),
-        alloc.getAlignmentAttr());
+    auto newAlloc = AllocLikeOp::create(rewriter, alloc.getLoc(), newMemRefType,
+                                        dynamicSizes, alloc.getSymbolOperands(),
+                                        alloc.getAlignmentAttr());
     // Insert a cast so we have the same type as the old alloc.
     rewriter.replaceOpWithNewOp<CastOp>(alloc, alloc.getType(), newAlloc);
     return success();
@@ -1358,8 +1359,9 @@ static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
     assert(isa<Attribute>(maybeConstant) &&
            "The constified value should be either unchanged (i.e., == result) "
            "or a constant");
-    Value constantVal = arith::ConstantIndexOp::create(rewriter,
-        loc, llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
+    Value constantVal = arith::ConstantIndexOp::create(
+        rewriter, loc,
+        llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
     for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
       // modifyOpInPlace: lambda cannot capture structured bindings in C++17
       // yet.
@@ -2587,8 +2589,9 @@ struct CollapseShapeOpMemRefCastFolder
       rewriter.modifyOpInPlace(
           op, [&]() { op.getSrcMutable().assign(cast.getSource()); });
     } else {
-      Value newOp = CollapseShapeOp::create(rewriter,
-          op->getLoc(), cast.getSource(), op.getReassociationIndices());
+      Value newOp =
+          CollapseShapeOp::create(rewriter, op->getLoc(), cast.getSource(),
+                                  op.getReassociationIndices());
       rewriter.replaceOpWithNewOp<CastOp>(op, op.getType(), newOp);
     }
     return success();
@@ -3212,8 +3215,8 @@ class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
     if (!resultType)
       return failure();
 
-    Value newSubView = SubViewOp::create(rewriter,
-        subViewOp.getLoc(), resultType, castOp.getSource(),
+    Value newSubView = SubViewOp::create(
+        rewriter, subViewOp.getLoc(), resultType, castOp.getSource(),
         subViewOp.getOffsets(), subViewOp.getSizes(), subViewOp.getStrides(),
         subViewOp.getStaticOffsets(), subViewOp.getStaticSizes(),
         subViewOp.getStaticStrides());
@@ -3534,9 +3537,9 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
       return failure();
 
     // Create new ViewOp.
-    auto newViewOp = ViewOp::create(rewriter,
-        viewOp.getLoc(), newMemRefType, viewOp.getOperand(0),
-        viewOp.getByteShift(), newOperands);
+    auto newViewOp = ViewOp::create(rewriter, viewOp.getLoc(), newMemRefType,
+                                    viewOp.getOperand(0), viewOp.getByteShift(),
+                                    newOperands);
     // Insert a cast so we have the same type as the old memref type.
     rewriter.replaceOpWithNewOp<CastOp>(viewOp, viewOp.getType(), newViewOp);
     return success();
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 8176ed0f02dce..525ae1e15bca6 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -23,6 +23,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/Support/Debug.h"
@@ -156,9 +157,10 @@ transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
       Type resultType = alloca.getResult().getType();
       OpBuilder builder(rewriter.getContext());
       // TODO: Add a better builder for this.
-      globalOp = memref::GlobalOp::create(builder,
-          loc, StringAttr::get(ctx, "alloca"), StringAttr::get(ctx, "private"),
-          TypeAttr::get(resultType), Attribute{}, UnitAttr{}, IntegerAttr{});
+      globalOp = memref::GlobalOp::create(
+          builder, loc, StringAttr::get(ctx, "alloca"),
+          StringAttr::get(ctx, "private"), TypeAttr::get(resultType),
+          Attribute{}, UnitAttr{}, IntegerAttr{});
       symbolTable.insert(globalOp);
     }
 
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index 5fd0d3c8a7f1b..7167289ec0e14 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -318,9 +319,9 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
   if (!newShardOp) {
     auto shardingOp =
         ShardingOp::create(builder, operandValue.getLoc(), sharding);
-    newShardOp =
-        ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                                /*annotate_for_users*/ false);
+    newShardOp = ShardOp::create(builder, operandValue.getLoc(), operandValue,
+                                 shardingOp,
+                                 /*annotate_for_users*/ false);
   }
   operandValue.replaceUsesWithIf(
       newShardOp, [operandOp, operandValue](OpOperand &use) {
@@ -332,8 +333,8 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
   }
 
   auto newShardOp2 = ShardOp::create(builder, operandValue.getLoc(), newShardOp,
-                                             newShardOp.getSharding(),
-                                             /*annotate_for_users*/ true);
+                                     newShardOp.getSharding(),
+                                     /*annotate_for_users*/ true);
   newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
 }
 
@@ -382,7 +383,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
       ShardingOp::create(builder, operand.get().getLoc(), sharding);
   auto newShardOp =
       ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                              /*annotate_for_users*/ true);
+                      /*annotate_for_users*/ true);
   IRRewriter rewriter(builder);
   rewriter.replaceUsesWithIf(
       operandValue, newShardOp, [operandOp, operandValue](OpOperand &use) {
@@ -397,7 +398,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
   builder.setInsertionPoint(newShardOp);
   auto newPreceedingShardOp =
       ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                              /*annotate_for_users*/ false);
+                      /*annotate_for_users*/ false);
   rewriter.replaceUsesWithIf(
       newShardOp.getSrc(), newPreceedingShardOp, [&newShardOp](OpOperand &use) {
         return use.getOwner() == newShardOp.getOperation();
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index abbdb6a0f53ec..4a89699b80023 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 80c807e774a7e..d4707619fa273 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index ffc84781f77ff..8fa59488678ee 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/SymbolTable.h"
diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp
index d3f7c9798b9b8..ffe53bbe80607 100644
--- a/mlir/lib/Dialect/PDL/IR/PDL.cpp
+++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
index 9b1f11d835282..5cc84ab29e246 100644
--- a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
+++ b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
index c488144508128..3695262155387 100644
--- a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
+++ b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/Dialect/Ptr/IR/PtrOps.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
diff --git a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
index e23a0d6aba825..cd4198b431669 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Quant/IR/Quant.h"
 #include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 82fa50a6ac6e0..8ead566703ab4 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
@@ -247,7 +248,7 @@ struct MultiBlockExecuteInliner : public OpRewritePattern<ExecuteRegionOp> {
       if (YieldOp yieldOp = dyn_cast<YieldOp>(blk.getTerminator())) {
         rewriter.setInsertionPoint(yieldOp);
         cf::BranchOp::create(rewriter, yieldOp.getLoc(), postBlock,
-                                      yieldOp.getResults());
+                             yieldOp.getResults());
         rewriter.eraseOp(yieldOp);
       }
     }
@@ -557,8 +558,8 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  scf::ForOp newLoop = scf::ForOp::create(rewriter,
-      getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
+  scf::ForOp newLoop = scf::ForOp::create(
+      rewriter, getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
       [](OpBuilder &, Location, Value, ValueRange) {});
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
 
@@ -673,8 +674,8 @@ void mlir::scf::promote(RewriterBase &rewriter, scf::ForallOp forallOp) {
     Value dst = parallelInsertSliceOp.getDest();
     Value src = parallelInsertSliceOp.getSource();
     if (llvm::isa<TensorType>(src.getType())) {
-      results.push_back(tensor::InsertSliceOp::create(rewriter,
-          forallOp.getLoc(), dst.getType(), src, dst,
+      results.push_back(tensor::InsertSliceOp::create(
+          rewriter, forallOp.getLoc(), dst.getType(), src, dst,
           parallelInsertSliceOp.getOffsets(), parallelInsertSliceOp.getSizes(),
           parallelInsertSliceOp.getStrides(),
           parallelInsertSliceOp.getStaticOffsets(),
@@ -722,8 +723,8 @@ LoopNest mlir::scf::buildLoopNest(
   ValueRange currentIterArgs = iterArgs;
   Location currentLoc = loc;
   for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
-    auto loop = scf::ForOp::create(builder,
-        currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
+    auto loop = scf::ForOp::create(
+        builder, currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
             ValueRange args) {
           ivs.push_back(iv);
@@ -801,8 +802,8 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   }
 
   // 2. Create the new forOp shell.
-  scf::ForOp newForOp = scf::ForOp::create(rewriter,
-      forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
+  scf::ForOp newForOp = scf::ForOp::create(
+      rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
       forOp.getStep(), newIterOperands);
   newForOp->setAttrs(forOp->getAttrs());
   Block &newBlock = newForOp.getRegion().front();
@@ -926,9 +927,9 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     if (!canonicalize)
       return failure();
 
-    scf::ForOp newForOp = scf::ForOp::create(rewriter,
-        forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-        forOp.getStep(), newIterArgs);
+    scf::ForOp newForOp =
+        scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(),
+                           forOp.getUpperBound(), forOp.getStep(), newIterArgs);
     newForOp->setAttrs(forOp->getAttrs());
     Block &newBlock = newForOp.getRegion().front();
 
@@ -971,7 +972,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
         if (keepMask[idx])
           filteredOperands.push_back(mergedTerminator.getOperand(idx));
       scf::YieldOp::create(rewriter, mergedTerminator.getLoc(),
-                                    filteredOperands);
+                           filteredOperands);
     };
 
     rewriter.mergeBlocks(&oldBlock, &newBlock, newBlockTransferArgs);
@@ -1685,8 +1686,8 @@ struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
 
     // Step 3. Create a new scf.forall op with the new shared_outs' operands
     //         fetched earlier
-    auto newForallOp = scf::ForallOp::create(rewriter,
-        forallOp.getLoc(), forallOp.getMixedLowerBound(),
+    auto newForallOp = scf::ForallOp::create(
+        rewriter, forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
         forallOp.getMapping(),
         /*bodyBuilderFn =*/[](OpBuilder &, Location, ValueRange) {});
@@ -1783,8 +1784,8 @@ struct ForallOpSingleOrZeroIterationDimsFolder
     // Replace the loop by a lower-dimensional loop.
     ForallOp newOp;
     newOp = ForallOp::create(rewriter, loc, newMixedLowerBounds,
-                                      newMixedUpperBounds, newMixedSteps,
-                                      op.getOutputs(), std::nullopt, nullptr);
+                             newMixedUpperBounds, newMixedSteps,
+                             op.getOutputs(), std::nullopt, nullptr);
     newOp.getBodyRegion().getBlocks().clear();
     // The new loop needs to keep all attributes from the old one, except for
     // "operandSegmentSizes" and static loop bound attributes which capture
@@ -1867,16 +1868,17 @@ struct FoldTensorCastOfOutputIntoForallOp
 
     // Create new loop.
     Location loc = forallOp.getLoc();
-    auto newForallOp = ForallOp::create(rewriter,
-        loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
-        forallOp.getMixedStep(), newOutputTensors, forallOp.getMapping(),
+    auto newForallOp = ForallOp::create(
+        rewriter, loc, forallOp.getMixedLowerBound(),
+        forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
+        newOutputTensors, forallOp.getMapping(),
         [&](OpBuilder nestedBuilder, Location nestedLoc, ValueRange bbArgs) {
           auto castBlockArgs =
               llvm::to_vector(bbArgs.take_back(forallOp->getNumResults()));
           for (auto [index, cast] : tensorCastProducers) {
             Value &oldTypeBBArg = castBlockArgs[index];
-            oldTypeBBArg = tensor::CastOp::create(nestedBuilder,
-                nestedLoc, cast.dstType, oldTypeBBArg);
+            oldTypeBBArg = tensor::CastOp::create(nestedBuilder, nestedLoc,
+                                                  cast.dstType, oldTypeBBArg);
           }
 
           // Move old body into new parallel loop.
@@ -1903,7 +1905,7 @@ struct FoldTensorCastOfOutputIntoForallOp
     for (auto &item : tensorCastProducers) {
       Value &oldTypeResult = castResults[item.first];
       oldTypeResult = tensor::CastOp::create(rewriter, loc, item.second.dstType,
-                                                      oldTypeResult);
+                                             oldTypeResult);
     }
     rewriter.replaceOp(forallOp, castResults);
     return success();
@@ -2373,7 +2375,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       return failure();
 
     IfOp replacement = IfOp::create(rewriter, op.getLoc(), nonHoistable, cond,
-                                             /*withElseRegion=*/false);
+                                    /*withElseRegion=*/false);
     if (replacement.thenBlock())
       rewriter.eraseBlock(replacement.thenBlock());
     replacement.getThenRegion().takeBody(op.getThenRegion());
@@ -2398,8 +2400,8 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       } else if (trueVal == falseVal)
         results[it.index()] = trueVal;
       else
-        results[it.index()] = arith::SelectOp::create(rewriter,
-            op.getLoc(), cond, trueVal, falseVal);
+        results[it.index()] = arith::SelectOp::create(rewriter, op.getLoc(),
+                                                      cond, trueVal, falseVal);
     }
 
     rewriter.setInsertionPointToEnd(replacement.thenBlock());
@@ -2450,8 +2452,8 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantTrue)
-          constantTrue = arith::ConstantOp::create(rewriter,
-              op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
+          constantTrue = arith::ConstantOp::create(
+              rewriter, op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
 
         rewriter.modifyOpInPlace(use.getOwner(),
                                  [&]() { use.set(constantTrue); });
@@ -2460,8 +2462,8 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantFalse)
-          constantFalse = arith::ConstantOp::create(rewriter,
-              op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
+          constantFalse = arith::ConstantOp::create(
+              rewriter, op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
 
         rewriter.modifyOpInPlace(use.getOwner(),
                                  [&]() { use.set(constantFalse); });
@@ -2547,8 +2549,8 @@ struct ReplaceIfYieldWithConditionOrValue : public OpRewritePattern<IfOp> {
       if (!trueVal && falseVal) {
         if (!opResult.use_empty()) {
           Dialect *constDialect = trueResult.getDefiningOp()->getDialect();
-          Value notCond = arith::XOrIOp::create(rewriter,
-              op.getLoc(), op.getCondition(),
+          Value notCond = arith::XOrIOp::create(
+              rewriter, op.getLoc(), op.getCondition(),
               constDialect
                   ->materializeConstant(rewriter,
                                         rewriter.getIntegerAttr(i1Ty, 1), i1Ty,
@@ -2661,8 +2663,8 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
     SmallVector<Type> mergedTypes(prevIf.getResultTypes());
     llvm::append_range(mergedTypes, nextIf.getResultTypes());
 
-    IfOp combinedIf = IfOp::create(rewriter,
-        nextIf.getLoc(), mergedTypes, prevIf.getCondition(), /*hasElse=*/false);
+    IfOp combinedIf = IfOp::create(rewriter, nextIf.getLoc(), mergedTypes,
+                                   prevIf.getCondition(), /*hasElse=*/false);
     rewriter.eraseBlock(&combinedIf.getThenRegion().back());
 
     rewriter.inlineRegionBefore(prevIf.getThenRegion(),
@@ -2823,8 +2825,8 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     }
 
     Location loc = op.getLoc();
-    Value newCondition = arith::AndIOp::create(rewriter,
-        loc, op.getCondition(), nestedIf.getCondition());
+    Value newCondition = arith::AndIOp::create(rewriter, loc, op.getCondition(),
+                                               nestedIf.getCondition());
     auto newIf = IfOp::create(rewriter, loc, op.getResultTypes(), newCondition);
     Block *newIfBlock = rewriter.createBlock(&newIf.getThenRegion());
 
@@ -2833,8 +2835,9 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     rewriter.setInsertionPoint(newIf);
 
     for (auto idx : elseYieldsToUpgradeToSelect)
-      results[idx] = arith::SelectOp::create(rewriter,
-          op.getLoc(), op.getCondition(), thenYield[idx], elseYield[idx]);
+      results[idx] =
+          arith::SelectOp::create(rewriter, op.getLoc(), op.getCondition(),
+                                  thenYield[idx], elseYield[idx]);
 
     rewriter.mergeBlocks(nestedIf.thenBlock(), newIfBlock);
     rewriter.setInsertionPointToEnd(newIf.thenBlock());
@@ -3160,8 +3163,8 @@ struct ParallelOpSingleOrZeroIterationDimsFolder
     }
     // Replace the parallel loop by lower-dimensional parallel loop.
     auto newOp =
-        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds, newUpperBounds,
-                                    newSteps, op.getInitVals(), nullptr);
+        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds,
+                           newUpperBounds, newSteps, op.getInitVals(), nullptr);
     // Erase the empty block that was inserted by the builder.
     rewriter.eraseBlock(newOp.getBody());
     // Clone the loop body and remap the block arguments of the collapsed loops
@@ -3541,8 +3544,8 @@ struct WhileConditionTruth : public OpRewritePattern<WhileOp> {
       if (std::get<0>(yieldedAndBlockArgs) == term.getCondition()) {
         if (!std::get<1>(yieldedAndBlockArgs).use_empty()) {
           if (!constantTrue)
-            constantTrue = arith::ConstantOp::create(rewriter,
-                op.getLoc(), term.getCondition().getType(),
+            constantTrue = arith::ConstantOp::create(
+                rewriter, op.getLoc(), term.getCondition().getType(),
                 rewriter.getBoolAttr(true));
 
           rewriter.replaceAllUsesWith(std::get<1>(yieldedAndBlockArgs),
@@ -3684,8 +3687,8 @@ struct RemoveLoopInvariantArgsFromBeforeBlock
       rewriter.replaceOpWithNewOp<YieldOp>(yieldOp, newYieldOpArgs);
     }
 
-    auto newWhile =
-        WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(), newInitArgs);
+    auto newWhile = WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(),
+                                    newInitArgs);
 
     Block &newBeforeBlock = *rewriter.createBlock(
         &newWhile.getBefore(), /*insertPt*/ {},
@@ -3808,7 +3811,7 @@ struct RemoveLoopInvariantValueYielded : public OpRewritePattern<WhileOp> {
     }
 
     auto newWhile = WhileOp::create(rewriter, op.getLoc(), newAfterBlockType,
-                                             op.getOperands());
+                                    op.getOperands());
 
     Block &newAfterBlock =
         *rewriter.createBlock(&newWhile.getAfter(), /*insertPt*/ {},
@@ -4044,7 +4047,7 @@ struct WhileRemoveUnusedArgs : public OpRewritePattern<WhileOp> {
     Location loc = op.getLoc();
     auto newWhileOp =
         WhileOp::create(rewriter, loc, op.getResultTypes(), newInits,
-                                 /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
+                        /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
 
@@ -4091,9 +4094,10 @@ struct WhileRemoveDuplicatedResults : public OpRewritePattern<WhileOp> {
     ValueRange argsRange(newArgs);
 
     Location loc = op.getLoc();
-    auto newWhileOp = scf::WhileOp::create(rewriter,
-        loc, argsRange.getTypes(), op.getInits(), /*beforeBody*/ nullptr,
-        /*afterBody*/ nullptr);
+    auto newWhileOp =
+        scf::WhileOp::create(rewriter, loc, argsRange.getTypes(), op.getInits(),
+                             /*beforeBody*/ nullptr,
+                             /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
 
@@ -4187,8 +4191,8 @@ struct WhileOpAlignBeforeArgs : public OpRewritePattern<WhileOp> {
     for (auto &&[i, j] : llvm::enumerate(*mapping))
       newResultTypes[j] = loop.getResult(i).getType();
 
-    auto newLoop = WhileOp::create(rewriter,
-        loop.getLoc(), newResultTypes, loop.getInits(),
+    auto newLoop = WhileOp::create(
+        rewriter, loop.getLoc(), newResultTypes, loop.getInits(),
         /*beforeBuilder=*/nullptr, /*afterBuilder=*/nullptr);
     auto newBefore = newLoop.getBeforeBody();
     auto newAfter = newLoop.getAfterBody();
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index e9943a3489d46..145afac6df15e 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -25,6 +25,7 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
index c517ef2b18f6f..6af87afd8b83e 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/SMT/IR/SMTOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/APSInt.h"
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
index d8dfe164458e2..8ba2af581e00d 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 #include "SPIRVParsingUtils.h"
 
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 7339ba5c2379c..19dba20f43fa9 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -154,14 +155,14 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
     return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (llvm::isa<ShapeType>(type) || isExtentTensorType(type))
-    return ConstShapeOp::create(builder,
-        loc, type, llvm::cast<DenseIntElementsAttr>(value));
+    return ConstShapeOp::create(builder, loc, type,
+                                llvm::cast<DenseIntElementsAttr>(value));
   if (llvm::isa<SizeType>(type))
     return ConstSizeOp::create(builder, loc, type,
-                                       llvm::cast<IntegerAttr>(value));
+                               llvm::cast<IntegerAttr>(value));
   if (llvm::isa<WitnessType>(type))
     return ConstWitnessOp::create(builder, loc, type,
-                                          llvm::cast<BoolAttr>(value));
+                                  llvm::cast<BoolAttr>(value));
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
@@ -316,8 +317,8 @@ struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
     auto newYieldOp =
         rewriter.replaceOpWithNewOp<AssumingYieldOp>(yieldOp, newYieldOperands);
     rewriter.setInsertionPoint(op);
-    auto newOp = AssumingOp::create(rewriter,
-        op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
+    auto newOp = AssumingOp::create(
+        rewriter, op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
     newOp.getDoRegion().takeBody(op.getDoRegion());
 
     // Use the new results to replace the previously used ones.
@@ -780,9 +781,9 @@ struct BroadcastFoldConstantOperandsPattern
     auto foldedConstantOperandsTy = RankedTensorType::get(
         {static_cast<int64_t>(foldedConstantShape.size())},
         rewriter.getIndexType());
-    newShapeOperands.push_back(ConstShapeOp::create(rewriter,
-        op.getLoc(), foldedConstantOperandsTy,
-        rewriter.getIndexTensorAttr(foldedConstantShape)));
+    newShapeOperands.push_back(
+        ConstShapeOp::create(rewriter, op.getLoc(), foldedConstantOperandsTy,
+                             rewriter.getIndexTensorAttr(foldedConstantShape)));
     rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(),
                                              newShapeOperands);
     return success();
@@ -845,9 +846,9 @@ struct BroadcastConcretizeResultTypePattern
       }
     }
 
-    auto newOp = BroadcastOp::create(rewriter,
-        op.getLoc(), getExtentTensorType(getContext(), maxRank),
-        op.getShapes());
+    auto newOp = BroadcastOp::create(rewriter, op.getLoc(),
+                                     getExtentTensorType(getContext(), maxRank),
+                                     op.getShapes());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -1338,7 +1339,8 @@ std::optional<int64_t> GetExtentOp::getConstantDim() {
 }
 
 OpFoldResult GetExtentOp::fold(FoldAdaptor adaptor) {
-  auto elements = llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
+  auto elements =
+      llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
   if (!elements)
     return nullptr;
   std::optional<int64_t> dim = getConstantDim();
@@ -1357,8 +1359,8 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
     Value dim = ConstSizeOp::create(builder, loc, dimAttr);
     build(builder, result, builder.getType<SizeType>(), shape, dim);
   } else {
-    Value dim =
-        arith::ConstantOp::create(builder, loc, builder.getIndexType(), dimAttr);
+    Value dim = arith::ConstantOp::create(builder, loc, builder.getIndexType(),
+                                          dimAttr);
     build(builder, result, builder.getIndexType(), shape, dim);
   }
 }
@@ -1480,7 +1482,8 @@ bool mlir::shape::MeetOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
 //===----------------------------------------------------------------------===//
 
 OpFoldResult shape::RankOp::fold(FoldAdaptor adaptor) {
-  auto shape = llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
+  auto shape =
+      llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
   if (!shape)
     return {};
   int64_t rank = shape.getNumElements();
@@ -1708,8 +1711,8 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
                                   rewriter.getIndexTensorAttr(type.getShape()))
             .getResult();
     if (constShape.getType() != op.getResult().getType())
-      constShape = tensor::CastOp::create(rewriter,
-          loc, op.getResult().getType(), constShape);
+      constShape = tensor::CastOp::create(rewriter, loc,
+                                          op.getResult().getType(), constShape);
     rewriter.replaceOp(op, constShape);
     return success();
   }
@@ -1717,13 +1720,13 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
 
 // Canonicalize
 //
-// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
-// %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
+// tensor<*xf32> %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
 //
 // to
 //
-// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
-// %1 = %shape
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
+// tensor<*xf32> %1 = %shape
 //
 struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
   using OpRewritePattern<shape::ShapeOfOp>::OpRewritePattern;
@@ -1751,10 +1754,11 @@ struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
 
     if (opTensorTy != shapeTensorTy) {
       if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
-        shape = tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
-      else if (!isExtentTensorType(shapeTensorTy))
         shape =
-            arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
+            tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
+      else if (!isExtentTensorType(shapeTensorTy))
+        shape = arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy,
+                                           shape);
     }
 
     rewriter.replaceOp(op, shape);
@@ -1895,8 +1899,9 @@ LogicalResult SplitAtOp::fold(FoldAdaptor adaptor,
                               SmallVectorImpl<OpFoldResult> &results) {
   if (!adaptor.getOperand() || !adaptor.getIndex())
     return failure();
-  auto shapeVec = llvm::to_vector<6>(
-      llvm::cast<DenseIntElementsAttr>(adaptor.getOperand()).getValues<int64_t>());
+  auto shapeVec =
+      llvm::to_vector<6>(llvm::cast<DenseIntElementsAttr>(adaptor.getOperand())
+                             .getValues<int64_t>());
   auto shape = llvm::ArrayRef(shapeVec);
   auto splitPoint = llvm::cast<IntegerAttr>(adaptor.getIndex()).getInt();
   // Verify that the split point is in the correct range.
@@ -1920,8 +1925,9 @@ OpFoldResult ToExtentTensorOp::fold(FoldAdaptor adaptor) {
   if (!adaptor.getInput())
     return OpFoldResult();
   Builder builder(getContext());
-  auto shape = llvm::to_vector<6>(
-      llvm::cast<DenseIntElementsAttr>(adaptor.getInput()).getValues<int64_t>());
+  auto shape =
+      llvm::to_vector<6>(llvm::cast<DenseIntElementsAttr>(adaptor.getInput())
+                             .getValues<int64_t>());
   auto type = RankedTensorType::get({static_cast<int64_t>(shape.size())},
                                     builder.getIndexType());
   return DenseIntElementsAttr::get(type, shape);
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index d5e709bbed221..eb42335fa8c94 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -561,7 +562,8 @@ SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
   SmallVector<Type> retType(
       dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
       builder.getIndexType());
-  auto transOp = CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
+  auto transOp =
+      CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
   return transOp.getOutCrds();
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp b/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
index bdec43825ddc2..33249368f7ed3 100644
--- a/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
+++ b/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h"
 #include "mlir/Dialect/Linalg/TransformOps/Syntax.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 using namespace mlir::sparse_tensor;
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 79bddee1e7b84..b19a413d6d9fd 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinTypeInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -57,7 +58,7 @@ Operation *TensorDialect::materializeConstant(OpBuilder &builder,
     return op;
   if (complex::ConstantOp::isBuildableWith(value, type))
     return complex::ConstantOp::create(builder, loc, type,
-                                               llvm::cast<ArrayAttr>(value));
+                                       llvm::cast<ArrayAttr>(value));
   return nullptr;
 }
 
@@ -681,8 +682,8 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
     inputShapes.emplace_back(std::move(inputShape));
   }
 
-  Value replacement = tensor::EmptyOp::create(builder,
-      loc, outputShape, getType().getElementType());
+  Value replacement = tensor::EmptyOp::create(builder, loc, outputShape,
+                                              getType().getElementType());
 
   int64_t rank = getType().getRank();
   OpFoldResult one = builder.getIndexAttr(1);
@@ -690,8 +691,8 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
   SmallVector<OpFoldResult> offsets(rank, zero);
   for (auto [index, input] : llvm::enumerate(getInputs())) {
     offsets[concatDim] = concatOffsets[index];
-    auto insertSlice = tensor::InsertSliceOp::create(builder,
-        loc, input, replacement, offsets, inputShapes[index], strides);
+    auto insertSlice = tensor::InsertSliceOp::create(
+        builder, loc, input, replacement, offsets, inputShapes[index], strides);
     replacement = insertSlice.getResult();
   }
   if (replacement.getType() != getType()) {
@@ -827,7 +828,7 @@ struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
         // Use refined operand type and create cast from original operand.
         auto castOp =
             CastOp::create(rewriter, concatOp->getLoc(), inferredOperandType,
-                                    concatOp.getOperand(operandIdx));
+                           concatOp.getOperand(operandIdx));
         rewriter.modifyOpInPlace(concatOp, [=, operandIdx = operandIdx] {
           concatOp->setOperand(operandIdx, castOp->getResult(0));
         });
@@ -867,8 +868,9 @@ struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
       return failure();
     }
 
-    auto newConcatOp = ConcatOp::create(rewriter,
-        concatOp->getLoc(), inferredResultType, dim, concatOp->getOperands());
+    auto newConcatOp =
+        ConcatOp::create(rewriter, concatOp->getLoc(), inferredResultType, dim,
+                         concatOp->getOperands());
     rewriter.replaceOpWithNewOp<CastOp>(concatOp, concatOp.getResultType(),
                                         newConcatOp);
 
@@ -1154,7 +1156,7 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
       return failure();
 
     auto newOp = EmptyOp::create(rewriter, op.getLoc(), foldedTensorType,
-                                          foldedDynamicSizes);
+                                 foldedDynamicSizes);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -1329,8 +1331,8 @@ struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
 
       SmallVector<int64_t> basis =
           llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-      auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-          extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
+      auto delinearize = affine::AffineDelinearizeIndexOp::create(
+          rewriter, extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
       llvm::append_range(sourceIndices, delinearize.getResults());
     }
     if (collapseOp.getReassociationIndices().empty()) {
@@ -1501,8 +1503,8 @@ struct ExtractElementFromIndexCast
 
     Type elementTy = getElementTypeOrSelf(indexCast.getIn());
 
-    auto newExtract = tensor::ExtractOp::create(rewriter,
-        loc, elementTy, indexCast.getIn(), extract.getIndices());
+    auto newExtract = tensor::ExtractOp::create(
+        rewriter, loc, elementTy, indexCast.getIn(), extract.getIndices());
 
     rewriter.replaceOpWithNewOp<arith::IndexCastOp>(extract, extract.getType(),
                                                     newExtract);
@@ -2164,9 +2166,9 @@ struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
         collapseShapeOp.getSrcMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = CollapseShapeOp::create(rewriter,
-          collapseShapeOp.getLoc(), newResultType, castOp.getSource(),
-          collapseShapeOp.getReassociation());
+      auto newOp = CollapseShapeOp::create(rewriter, collapseShapeOp.getLoc(),
+                                           newResultType, castOp.getSource(),
+                                           collapseShapeOp.getReassociation());
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
           collapseShapeOp, collapseShapeOp.getResultType(), newOp);
     }
@@ -2244,9 +2246,9 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
     auto outputType = RankedTensorType::get(
         newOutputShape, expandOp.getSrcType().getElementType());
     auto inputCast = CastOp::create(rewriter, expandOp.getLoc(), inputType,
-                                             expandOp.getSrc());
-    auto newExpand = ExpandShapeOp::create(rewriter,
-        expandOp.getLoc(), outputType, inputCast.getResult(),
+                                    expandOp.getSrc());
+    auto newExpand = ExpandShapeOp::create(
+        rewriter, expandOp.getLoc(), outputType, inputCast.getResult(),
         expandOp.getReassociationIndices(), outputOfr);
     rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
                                         newExpand.getResult());
@@ -2558,10 +2560,11 @@ class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
 
     // Create folded extract.
     Location loc = sliceOp.getLoc();
-    Value newResult = ExtractSliceOp::create(rewriter,
-        loc, sliceOp.getType(), castOp.getSource(), sliceOp.getOffsets(),
-        sliceOp.getSizes(), sliceOp.getStrides(), sliceOp.getStaticOffsets(),
-        sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
+    Value newResult = ExtractSliceOp::create(
+        rewriter, loc, sliceOp.getType(), castOp.getSource(),
+        sliceOp.getOffsets(), sliceOp.getSizes(), sliceOp.getStrides(),
+        sliceOp.getStaticOffsets(), sliceOp.getStaticSizes(),
+        sliceOp.getStaticStrides());
     rewriter.replaceOp(sliceOp, newResult);
     return success();
   }
@@ -2713,7 +2716,7 @@ struct SliceCanonicalizer {
     Value replacement = newOp.getResult();
     if (replacement.getType() != op.getType())
       replacement = tensor::CastOp::create(rewriter, op.getLoc(), op.getType(),
-                                                    replacement);
+                                           replacement);
     rewriter.replaceOp(op, replacement);
   }
 };
@@ -2982,7 +2985,7 @@ class InsertSliceOpConstantArgumentFolder final
       if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
       toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
-                                                 sourceType, toInsert);
+                                        sourceType, toInsert);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, toInsert, insertSliceOp.getDest(), mixedOffsets,
@@ -3078,17 +3081,18 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     if (!sliceResult.isValid)
       return failure();
 
-    Operation *replacement = InsertOpTy::create(rewriter,
-        insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
-        mixedSizes, insertSliceOp.getMixedStrides());
+    Operation *replacement =
+        InsertOpTy::create(rewriter, insertSliceOp.getLoc(), src, dst,
+                           insertSliceOp.getMixedOffsets(), mixedSizes,
+                           insertSliceOp.getMixedStrides());
 
     // In the parallel case there is no result and so nothing to cast.
     bool isParallelInsert =
         std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
     if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
       replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
-                                                    insertSliceOp.getDestType(),
-                                                    replacement->getResult(0));
+                                           insertSliceOp.getDestType(),
+                                           replacement->getResult(0));
     }
     rewriter.replaceOp(insertSliceOp, replacement->getResults());
     return success();
@@ -3157,8 +3161,8 @@ struct InsertSliceOpSourceCastInserter final
     // parallel case.
     if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
       rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-    Value cast = tensor::CastOp::create(rewriter,
-        insertSliceOp.getLoc(), newSrcType, insertSliceOp.getSource());
+    Value cast = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
+                                        newSrcType, insertSliceOp.getSource());
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, cast, insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
@@ -3410,10 +3414,11 @@ struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
         padTensorOp.getSourceMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = PadOp::create(rewriter,
-          padTensorOp->getLoc(), newResultType, padTensorOp.getSource(),
-          padTensorOp.getStaticLow(), padTensorOp.getStaticHigh(),
-          padTensorOp.getLow(), padTensorOp.getHigh(), padTensorOp.getNofold(),
+      auto newOp = PadOp::create(
+          rewriter, padTensorOp->getLoc(), newResultType,
+          padTensorOp.getSource(), padTensorOp.getStaticLow(),
+          padTensorOp.getStaticHigh(), padTensorOp.getLow(),
+          padTensorOp.getHigh(), padTensorOp.getNofold(),
           getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
       IRMapping mapper;
       padTensorOp.getRegion().cloneInto(&newOp.getRegion(), mapper);
@@ -3442,8 +3447,8 @@ struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
                                             tensorCastOp.getDest().getType()))
       return failure();
 
-    auto replacementOp = PadOp::create(rewriter,
-        padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
+    auto replacementOp = PadOp::create(
+        rewriter, padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
         padTensorOp.getSource(), padTensorOp.getStaticLow(),
         padTensorOp.getStaticHigh(), padTensorOp.getLow(),
         padTensorOp.getHigh(), padTensorOp.getNofold(),
@@ -3600,11 +3605,11 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
 
     // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
     // the two paddings in one step.
-    auto newSliceOp = ExtractSliceOp::create(rewriter,
-        padOp.getLoc(), outerSliceOp.getSource(), newOffsets, newSizes,
-        innerSliceOp.getMixedStrides());
-    auto newPadOp = PadOp::create(rewriter,
-        padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
+    auto newSliceOp = ExtractSliceOp::create(
+        rewriter, padOp.getLoc(), outerSliceOp.getSource(), newOffsets,
+        newSizes, innerSliceOp.getMixedStrides());
+    auto newPadOp = PadOp::create(
+        rewriter, padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
         padOp.getMixedLowPad(), newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
     rewriter.inlineRegionBefore(padOp.getRegion(), newPadOp.getRegion(),
@@ -3700,9 +3705,9 @@ struct FoldStaticPadding : public OpRewritePattern<PadOp> {
     // Rewrite the op using the new static type.
     auto newResultType = RankedTensorType::get(
         newOutDims, padTensorOp.getType().getElementType());
-    auto newOp = PadOp::create(rewriter,
-        padTensorOp->getLoc(), newResultType, input, staticLow, staticHigh,
-        newLows, newHighs, padTensorOp.getNofold(),
+    auto newOp = PadOp::create(
+        rewriter, padTensorOp->getLoc(), newResultType, input, staticLow,
+        staticHigh, newLows, newHighs, padTensorOp.getNofold(),
         getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
 
     IRMapping mapper;
@@ -3780,9 +3785,9 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
     SmallVector<OpFoldResult> newLowPad =
         addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        padOp.getLoc(), padOp.getResultType(), producerPad.getSource(),
-        newLowPad, newHighPad, padOp.getNofold(),
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, padOp.getLoc(), padOp.getResultType(),
+        producerPad.getSource(), newLowPad, newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, tensor::PadOp::getAttributeNames()));
     rewriter.inlineRegionBefore(padOp.getRegion(), newPadOp.getRegion(),
                                 newPadOp.getRegion().begin());
@@ -4111,8 +4116,8 @@ struct FoldTensorCastProducerOp
     for (auto [oldResult, newResult] :
          llvm::zip(op->getResults(), newOp->getResults())) {
       if (newResult.getType() != oldResult.getType()) {
-        replacements.push_back(tensor::CastOp::create(rewriter,
-            op->getLoc(), oldResult.getType(), newResult));
+        replacements.push_back(tensor::CastOp::create(
+            rewriter, op->getLoc(), oldResult.getType(), newResult));
       } else {
         replacements.push_back(newResult);
       }
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index ba258402b61b3..b8b1a592fdca2 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
index d69535169f956..003d81fbc7771 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
index 12257da878a40..2d6d4e17aff66 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
@@ -10,6 +10,7 @@
 
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/Support/InterleavedRange.h"
 
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 0db0317461c03..d4a9d4ef1d656 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp b/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
index 93b0bc591ca02..15c2ae22f56ea 100644
--- a/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/ADT/STLExtras.h"
 
diff --git a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
index 34d6221d15fb0..fc6ea93b49218 100644
--- a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h"
 
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
diff --git a/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp b/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
index 85f61245eb734..9d5c749f02dfd 100644
--- a/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h"
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Rewrite/FrozenRewritePatternSet.h"
 #include "mlir/Rewrite/PatternApplicator.h"
diff --git a/mlir/lib/Dialect/UB/IR/UBOps.cpp b/mlir/lib/Dialect/UB/IR/UBOps.cpp
index ee523f9522953..442a53ca8162a 100644
--- a/mlir/lib/Dialect/UB/IR/UBOps.cpp
+++ b/mlir/lib/Dialect/UB/IR/UBOps.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "mlir/Dialect/UB/IR/UBOpsDialect.cpp.inc"
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index e32b2ca8d76b4..a8277a2a805e9 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -30,6 +30,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -599,8 +600,8 @@ struct ElideUnitDimsInMultiDimReduction
                             dstVecType.getScalableDims());
         mask = vector::ShapeCastOp::create(rewriter, loc, newMaskType, mask);
       }
-      cast = vector::ShapeCastOp::create(rewriter,
-          loc, reductionOp.getDestType(), reductionOp.getSource());
+      cast = vector::ShapeCastOp::create(
+          rewriter, loc, reductionOp.getDestType(), reductionOp.getSource());
     } else {
       // This means we are reducing all the dimensions, and all reduction
       // dimensions are of size 1. So a simple extraction would do.
@@ -674,35 +675,35 @@ Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
   case arith::AtomicRMWKind::addf:
   case arith::AtomicRMWKind::addi:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::ADD, vector);
+                                       CombiningKind::ADD, vector);
   case arith::AtomicRMWKind::mulf:
   case arith::AtomicRMWKind::muli:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MUL, vector);
+                                       CombiningKind::MUL, vector);
   case arith::AtomicRMWKind::minimumf:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINIMUMF, vector);
+                                       CombiningKind::MINIMUMF, vector);
   case arith::AtomicRMWKind::mins:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINSI, vector);
+                                       CombiningKind::MINSI, vector);
   case arith::AtomicRMWKind::minu:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINUI, vector);
+                                       CombiningKind::MINUI, vector);
   case arith::AtomicRMWKind::maximumf:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXIMUMF, vector);
+                                       CombiningKind::MAXIMUMF, vector);
   case arith::AtomicRMWKind::maxs:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXSI, vector);
+                                       CombiningKind::MAXSI, vector);
   case arith::AtomicRMWKind::maxu:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXUI, vector);
+                                       CombiningKind::MAXUI, vector);
   case arith::AtomicRMWKind::andi:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::AND, vector);
+                                       CombiningKind::AND, vector);
   case arith::AtomicRMWKind::ori:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::OR, vector);
+                                       CombiningKind::OR, vector);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
@@ -4171,8 +4172,8 @@ class StridedSliceBroadcast final
     // just a single scalar.
     bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
     if (!lowerDimMatch && !isScalarSrc) {
-      source = ExtractStridedSliceOp::create(rewriter,
-          op->getLoc(), source,
+      source = ExtractStridedSliceOp::create(
+          rewriter, op->getLoc(), source,
           getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getStrides(), /* dropFront=*/rankDiff));
@@ -4267,7 +4268,7 @@ class ContiguousExtractStridedSliceToExtract final
     SmallVector<int64_t> offsets = getI64SubArray(op.getOffsets());
     auto extractOffsets = ArrayRef(offsets).take_front(numOffsets);
     Value extract = vector::ExtractOp::create(rewriter, op->getLoc(), source,
-                                                       extractOffsets);
+                                              extractOffsets);
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(), extract);
     return success();
   }
@@ -5337,13 +5338,14 @@ struct SwapExtractSliceOfTransferWrite
     // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
     // Set all in_bounds to false and let the folder infer them.
     SmallVector<bool> newInBounds(vectorShape.size(), false);
-    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
-        extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
-        insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
-        insertOp.getMixedStrides());
-    auto newTransferWriteOp = TransferWriteOp::create(rewriter,
-        transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
-        transferOp.getIndices(), transferOp.getPermutationMapAttr(),
+    auto newExtractOp = tensor::ExtractSliceOp::create(
+        rewriter, extractOp.getLoc(), insertOp.getSourceType(),
+        insertOp.getDest(), insertOp.getMixedOffsets(),
+        insertOp.getMixedSizes(), insertOp.getMixedStrides());
+    auto newTransferWriteOp = TransferWriteOp::create(
+        rewriter, transferOp.getLoc(), transferOp.getVector(),
+        newExtractOp.getResult(), transferOp.getIndices(),
+        transferOp.getPermutationMapAttr(),
         rewriter.getBoolArrayAttr(newInBounds));
     rewriter.modifyOpInPlace(insertOp, [&]() {
       insertOp.getSourceMutable().assign(newTransferWriteOp.getResult());
@@ -7215,11 +7217,11 @@ Operation *mlir::vector::maskOperation(OpBuilder &builder,
     return maskableOp;
   if (passthru)
     return MaskOp::create(builder, maskableOp->getLoc(),
-                                  maskableOp->getResultTypes(), mask, passthru,
-                                  maskableOp, createMaskOpRegion);
+                          maskableOp->getResultTypes(), mask, passthru,
+                          maskableOp, createMaskOpRegion);
   return MaskOp::create(builder, maskableOp->getLoc(),
-                                maskableOp->getResultTypes(), mask, maskableOp,
-                                createMaskOpRegion);
+                        maskableOp->getResultTypes(), mask, maskableOp,
+                        createMaskOpRegion);
 }
 
 /// Creates a vector select operation that picks values from `newValue` or
@@ -7235,7 +7237,7 @@ Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
     return newValue;
 
   return arith::SelectOp::create(builder, newValue.getLoc(), newValue.getType(),
-                                         mask, newValue, passthru);
+                                 mask, newValue, passthru);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 125c3d918284c..bb051650897b4 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
 #include "mlir/Dialect/X86Vector/Transforms.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
index eafe3a97758c1..f7b1c0966fe27 100644
--- a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
+++ b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
@@ -64,7 +65,7 @@ SmallVector<Value> x86vector::MaskCompressOp::getIntrinsicOperands(
     src = adaptor.getSrc();
   } else if (adaptor.getConstantSrc()) {
     src = LLVM::ConstantOp::create(rewriter, loc, opType,
-                                            adaptor.getConstantSrcAttr());
+                                   adaptor.getConstantSrcAttr());
   } else {
     auto zeroAttr = rewriter.getZeroAttr(opType);
     src = LLVM::ConstantOp::create(rewriter, loc, opType, zeroAttr);
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 453d53ee41d54..81687a6fc37f2 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
 
 #include "llvm/Support/Debug.h"
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 6d7e2aa0ece7d..23988a00c7a2c 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectResourceBlobManager.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeRange.h"
 
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 535f5e9b4a15d..73366110b104e 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -1192,7 +1192,7 @@ def OpFuncRef : TEST_Op<"op_funcref"> {
   let description = [{
     The "test.op_funcref" is a test op with a reference to a function symbol.
   }];
-  let builders = [OpBuilder<(ins "::mlir::func::FuncOp":$function)>];
+  // let builders = [OpBuilder<(ins "::mlir::func::FuncOp":$function)>];
 }
 
 // Pattern add the argument plus a increasing static number hidden in
diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
index cdf44c2959d50..9753ac00af88a 100644
--- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
+++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
index ee3eb9522db7e..44f117e93da6b 100644
--- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
+++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "llvm/Support/Debug.h"
diff --git a/mlir/test/lib/Transforms/TestTransformsOps.cpp b/mlir/test/lib/Transforms/TestTransformsOps.cpp
index 9a5632bb99c06..3d7b6be8a78e9 100644
--- a/mlir/test/lib/Transforms/TestTransformsOps.cpp
+++ b/mlir/test/lib/Transforms/TestTransformsOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Transforms/RegionUtils.h"
 
diff --git a/mlir/test/python/lib/PythonTestDialect.cpp b/mlir/test/python/lib/PythonTestDialect.cpp
index a0ff31504c691..e0a76f1b75d38 100644
--- a/mlir/test/python/lib/PythonTestDialect.cpp
+++ b/mlir/test/python/lib/PythonTestDialect.cpp
@@ -9,6 +9,7 @@
 #include "PythonTestDialect.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "PythonTestDialect.cpp.inc"


From flang-commits at lists.llvm.org  Mon Jul  7 18:08:06 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 18:08:06 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
Message-ID: <llvm.org/llvm/llvm-project/pull/147442@github.com>


https://github.com/agozillon created https://github.com/llvm/llvm-project/pull/147442

Currently, we indicate to the runtime that implicit scalar captures are firstprivate (via map and
capture types), enough for the runtime trace to treat it as such, but we do not CodeGen the IR
in such a way that we can take full advantage of this aspect of the OpenMP specification.

This patch seeks to change that by applying the correct symbol flags (firstprivate/implicit) to the
implicitly captured scalars within target regions, which then triggers the delayed privitization code
generation for these symbols, bringing the code generation in-line with the explicit firstpriviate
clause. Currently, similarly to the delayed privitization I have sheltered this segment of code
behind the EnabledDelayedPrivitization flag, as without it, we'll trigger an compiler error for
firstprivate not being supported any time we implicitly capture a scalar and try to firstprivitize
it, in future when this flag is removed it can also be removed here. So, for now, you need to
enable this via providing the compiler the flag on compilation of any programs.

>From 9314e8e3a212eccb473c2269b6402f84fbcef0dd Mon Sep 17 00:00:00 2001
From: agozillon <Andrew.Gozillon at amd.com>
Date: Mon, 7 Jul 2025 20:01:28 -0500
Subject: [PATCH]     [Flang][OpenMP] Make implicitly captured scalars fully
 firstprivatized

    Currently, we indicate to the runtime that implicit scalar captures are firstprivate (via map and
    capture types), enough for the runtime trace to treat it as such, but we do not CodeGen the IR
    in such a way that we can take full advantage of this aspect of the OpenMP specification.

    This patch seeks to change that by applying the correct symbol flags (firstprivate/implicit) to the
    implicitly captured scalars within target regions, which then triggers the delayed privitization code
    generation for these symbols, bringing the code generation in-line with the explicit firstpriviate
    clause. Currently, similarly to the delayed privitization I have sheltered this segment of code
    behind the EnabledDelayedPrivitization flag, as without it, we'll trigger an compiler error for
    firstprivate not being supported any time we implicitly capture a scalar and try to firstprivitize
    it, in future when this flag is removed it can also be removed here. So, for now, you need to
    enable this via providing the compiler the flag on compilation of any programs.
---
 flang/lib/Semantics/resolve-directives.cpp    | 39 ++++++++++++++++-
 .../target-private-implicit-scalar-map.f90    | 42 +++++++++++++++++++
 .../target-private-multiple-variables.f90     | 16 +++----
 3 files changed, 88 insertions(+), 9 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 299bb6ff876e7..f7aa2df4c25b1 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -23,6 +23,7 @@
 #include "flang/Semantics/openmp-modifiers.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Flags.h"
 #include "llvm/Frontend/OpenMP/OMP.h.inc"
 #include "llvm/Support/Debug.h"
 #include <list>
@@ -2297,6 +2298,34 @@ static bool IsSymbolStaticStorageDuration(const Symbol &symbol) {
       (ultSym.flags().test(Symbol::Flag::InCommonBlock));
 }
 
+static bool IsTargetCaptureImplicitlyFirstPrivatizeable(
+    const Symbol &symbol, const Symbol::Flags &dsa) {
+  // if we're associated with any other flags we skip implicit privitization
+  // for now. If we're an allocatable, pointer or declare target, we're not
+  // implicitly firstprivitizeable under OpenMP restrictions.
+  // TODO: Relax restriction as we progress privitization and further
+  // investigate the flags we can intermix with.
+  if (!dsa.none() || !symbol.flags().none() ||
+      semantics::IsAllocatableOrPointer(symbol)) {
+    return false;
+  }
+
+  // It is default firstprivatizeable as far as the OpenMP specification is
+  // concerned if it is a non-array scalar type that has been implicitly
+  // captured in a target region
+  const auto *type{symbol.GetType()};
+  if ((!symbol.GetShape() || symbol.GetShape()->empty()) &&
+      (type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Numeric ||
+          type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Logical ||
+          type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Character)) {
+    return true;
+  }
+  return false;
+}
+
 void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
   if (!IsPrivatizable(symbol)) {
     return;
@@ -2444,7 +2473,15 @@ void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
       useLastDeclSymbol();
       PRINT_IMPLICIT_RULE("3) enclosing context");
     } else if (targetDir) {
-      // TODO 4) not mapped target variable -> firstprivate
+      // 4) not mapped target variable  -> firstprivate
+      //    - i.e. implicit, but meets OpenMP specification rules for
+      //    firstprivate "promotion"
+      if (enableDelayedPrivatizationStaging && symbol &&
+          IsTargetCaptureImplicitlyFirstPrivatizeable(*symbol, prevDSA)) {
+        prevDSA.set(Symbol::Flag::OmpImplicit);
+        prevDSA.set(Symbol::Flag::OmpFirstPrivate);
+        makeSymbol(prevDSA);
+      }
       dsa = prevDSA;
     } else if (taskGenDir) {
       // TODO 5) dummy arg in orphaned taskgen construct -> firstprivate
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
new file mode 100644
index 0000000000000..6cd14e7d3a2b4
--- /dev/null
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
@@ -0,0 +1,42 @@
+! Tests delayed privatization works for implicit capture of scalars similarly to
+! the way it works for explicitly firstprivitized scalars.
+
+! RUN: %flang_fc1 -emit-mlir -fopenmp -mmlir --enable-delayed-privatization-staging \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+
+! CHECK-LABEL:   omp.private {type = firstprivate} @_QFExdgfx_firstprivate_i32 : i32 copy {
+! CHECK:         ^bb0(%{{.*}}: !fir.ref<i32>, %{{.*}}: !fir.ref<i32>):
+! CHECK:           %{{.*}} = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:           fir.store %{{.*}} to %{{.*}} : !fir.ref<i32>
+! CHECK:           omp.yield(%{{.*}} : !fir.ref<i32>)
+! CHECK:         }
+
+! CHECK-LABEL:   omp.private {type = firstprivate} @_QFExfpvx_firstprivate_i32 : i32 copy {
+! CHECK:         ^bb0(%{{.*}}: !fir.ref<i32>, %{{.*}}: !fir.ref<i32>):
+! CHECK:           %{{.*}} = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:           fir.store %{{.*}} to %{{.*}} : !fir.ref<i32>
+! CHECK:           omp.yield(%{{.*}} : !fir.ref<i32>)
+! CHECK:         }
+
+! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "xdgfx", uniq_name = "_QFExdgfx"}
+! CHECK:  %[[VAL_1:.*]] = fir.declare %[[VAL_0]] {uniq_name = "_QFExdgfx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "xfpvx", uniq_name = "_QFExfpvx"}
+! CHECK:  %[[VAL_3:.*]] = fir.declare %[[VAL_2]] {uniq_name = "_QFExfpvx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_3]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32>
+! CHECK:  %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32>
+
+! CHECK:  omp.target map_entries(%[[VAL_4]] -> %{{.*}}, %[[VAL_5]] -> %{{.*}} : !fir.ref<i32>, !fir.ref<i32>) private(@_QFExfpvx_firstprivate_i32 %[[VAL_3]] -> %[[VAL_6:.*]] [map_idx=0], @_QFExdgfx_firstprivate_i32 %[[VAL_1]] -> %[[VAL_7:.*]] [map_idx=1] : !fir.ref<i32>, !fir.ref<i32>) {
+! CHECK:  %{{.*}} = fir.declare %[[VAL_6]] {uniq_name = "_QFExfpvx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %{{.*}} = fir.declare %[[VAL_7]] {uniq_name = "_QFExdgfx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+
+program test_default_implicit_firstprivate
+  implicit none
+  integer :: xdgfx, xfpvx
+  xdgfx = 1
+  xfpvx = 2
+  !$omp target firstprivate(xfpvx)
+  xdgfx = 42
+  xfpvx = 43
+  !$omp end target
+  write(*,*) xdgfx, xfpvx
+end program
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
index 217ac5638a3ea..e7e3fb2097308 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
@@ -139,28 +139,28 @@ end subroutine target_allocatable
 ! CHECK:        %[[REAL_ARR_ALLOC:.*]] = fir.alloca !fir.array<?xf32>, {{.*}} {bindc_name = "real_arr", {{.*}}}
 ! CHECK:        %[[REAL_ARR_DECL:.*]]:2 = hlfir.declare %[[REAL_ARR_ALLOC]]({{.*}})
 ! CHECK:        fir.store %[[REAL_ARR_DECL]]#0 to %[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
-! CHECK:        %[[MAPPED_MI0:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#1 : !fir.ref<i32>, i32) {{.*}}
 ! CHECK:        %[[ALLOC_VAR_MEMBER:.*]] = omp.map.info var_ptr(%[[ALLOC_VAR_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, i32)
 ! CHECK:        %[[ALLOC_VAR_MAP:.*]] = omp.map.info var_ptr(%[[ALLOC_VAR_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) {{.*}} members(%[[ALLOC_VAR_MEMBER]] :
 ! CHECK:        %[[REAL_ARR_MEMBER:.*]] = omp.map.info var_ptr(%[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, f32)
 ! CHECK:        %[[REAL_ARR_DESC_MAP:.*]] = omp.map.info var_ptr(%[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) {{.*}} members(%[[REAL_ARR_MEMBER]] :
 ! CHECK:        fir.store %[[CHAR_VAR_DECL]]#0 to %[[CHAR_VAR_DESC_ALLOCA]] : !fir.ref<!fir.boxchar<1>>
 ! CHECK:        %[[CHAR_VAR_DESC_MAP:.*]] = omp.map.info var_ptr(%[[CHAR_VAR_DESC_ALLOCA]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>)
+! CHECK:        %[[MAPPED_MI0:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#0 : !fir.ref<i32>, i32) {{.*}}
 ! CHECK:        omp.target
 ! CHECK-SAME:     map_entries(
-! CHECK-SAME:       %[[MAPPED_MI0]] -> %[[MAPPED_ARG0:[^,]+]],
-! CHECK-SAME:       %[[ALLOC_VAR_MAP]] -> %[[MAPPED_ARG1:[^,]+]]
+! CHECK-SAME:       %[[ALLOC_VAR_MAP]] -> %[[MAPPED_ARG1:[^,]+]],
 ! CHECK-SAME:       %[[REAL_ARR_DESC_MAP]] -> %[[MAPPED_ARG2:[^,]+]]
 ! CHECK-SAME:       %[[CHAR_VAR_DESC_MAP]] -> %[[MAPPED_ARG3:.[^,]+]]
-! CHECK-SAME:       !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.boxchar<1>>, !fir.llvm_ptr<!fir.ref<i32>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
+! CHECK-SAME:       %[[MAPPED_MI0]] -> %[[MAPPED_ARG0:[^,]+]]
+! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.boxchar<1>>, !fir.ref<i32>, !fir.llvm_ptr<!fir.ref<i32>>
 ! CHECK-SAME:     private(
-! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]] [map_idx=1],
+! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]] [map_idx=0],
 ! CHECK-SAME:       @[[REAL_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[REAL_ARG:[^,]+]],
 ! CHECK-SAME:       @[[LB_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[LB_ARG:[^,]+]],
-! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}} -> %[[ARR_ARG:[^,]+]] [map_idx=2],
+! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}} -> %[[ARR_ARG:[^,]+]] [map_idx=1],
 ! CHECK-SAME:       @[[COMP_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[COMP_ARG:[^,]+]],
-! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] [map_idx=3] :
-! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<complex<f32>>, !fir.boxchar<1>) {
+! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] [map_idx=2],
+! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<complex<f32>>, !fir.boxchar<1>, !fir.ref<i32>) {
 ! CHECK-NOT:      fir.alloca
 ! CHECK:          hlfir.declare %[[ALLOC_ARG]]
 ! CHECK:          hlfir.declare %[[REAL_ARG]]


From flang-commits at lists.llvm.org  Mon Jul  7 18:08:17 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 18:08:17 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147442@github.com>
Message-ID: <686c6f81.170a0220.321fe5.aa25@mx.google.com>


https://github.com/agozillon edited https://github.com/llvm/llvm-project/pull/147442

From flang-commits at lists.llvm.org  Mon Jul  7 18:08:34 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 18:08:34 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147442@github.com>
Message-ID: <686c6f92.630a0220.1b1056.baf9@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: None (agozillon)

<details>
<summary>Changes</summary>

Currently, we indicate to the runtime that implicit scalar captures are firstprivate (via map and
capture types), enough for the runtime trace to treat it as such, but we do not CodeGen the IR
in such a way that we can take full advantage of this aspect of the OpenMP specification.

This patch seeks to change that by applying the correct symbol flags (firstprivate/implicit) to the
implicitly captured scalars within target regions, which then triggers the delayed privitization code
generation for these symbols, bringing the code generation in-line with the explicit firstpriviate
clause. Currently, similarly to the delayed privitization I have sheltered this segment of code
behind the EnabledDelayedPrivitization flag, as without it, we'll trigger an compiler error for
firstprivate not being supported any time we implicitly capture a scalar and try to firstprivitize
it, in future when this flag is removed it can also be removed here. So, for now, you need to
enable this via providing the compiler the flag on compilation of any programs.

---
Full diff: https://github.com/llvm/llvm-project/pull/147442.diff


3 Files Affected:

- (modified) flang/lib/Semantics/resolve-directives.cpp (+38-1) 
- (added) flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90 (+42) 
- (modified) flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 (+8-8) 


``````````diff
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 299bb6ff876e7..f7aa2df4c25b1 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -23,6 +23,7 @@
 #include "flang/Semantics/openmp-modifiers.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Flags.h"
 #include "llvm/Frontend/OpenMP/OMP.h.inc"
 #include "llvm/Support/Debug.h"
 #include <list>
@@ -2297,6 +2298,34 @@ static bool IsSymbolStaticStorageDuration(const Symbol &symbol) {
       (ultSym.flags().test(Symbol::Flag::InCommonBlock));
 }
 
+static bool IsTargetCaptureImplicitlyFirstPrivatizeable(
+    const Symbol &symbol, const Symbol::Flags &dsa) {
+  // if we're associated with any other flags we skip implicit privitization
+  // for now. If we're an allocatable, pointer or declare target, we're not
+  // implicitly firstprivitizeable under OpenMP restrictions.
+  // TODO: Relax restriction as we progress privitization and further
+  // investigate the flags we can intermix with.
+  if (!dsa.none() || !symbol.flags().none() ||
+      semantics::IsAllocatableOrPointer(symbol)) {
+    return false;
+  }
+
+  // It is default firstprivatizeable as far as the OpenMP specification is
+  // concerned if it is a non-array scalar type that has been implicitly
+  // captured in a target region
+  const auto *type{symbol.GetType()};
+  if ((!symbol.GetShape() || symbol.GetShape()->empty()) &&
+      (type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Numeric ||
+          type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Logical ||
+          type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Character)) {
+    return true;
+  }
+  return false;
+}
+
 void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
   if (!IsPrivatizable(symbol)) {
     return;
@@ -2444,7 +2473,15 @@ void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
       useLastDeclSymbol();
       PRINT_IMPLICIT_RULE("3) enclosing context");
     } else if (targetDir) {
-      // TODO 4) not mapped target variable -> firstprivate
+      // 4) not mapped target variable  -> firstprivate
+      //    - i.e. implicit, but meets OpenMP specification rules for
+      //    firstprivate "promotion"
+      if (enableDelayedPrivatizationStaging && symbol &&
+          IsTargetCaptureImplicitlyFirstPrivatizeable(*symbol, prevDSA)) {
+        prevDSA.set(Symbol::Flag::OmpImplicit);
+        prevDSA.set(Symbol::Flag::OmpFirstPrivate);
+        makeSymbol(prevDSA);
+      }
       dsa = prevDSA;
     } else if (taskGenDir) {
       // TODO 5) dummy arg in orphaned taskgen construct -> firstprivate
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
new file mode 100644
index 0000000000000..6cd14e7d3a2b4
--- /dev/null
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
@@ -0,0 +1,42 @@
+! Tests delayed privatization works for implicit capture of scalars similarly to
+! the way it works for explicitly firstprivitized scalars.
+
+! RUN: %flang_fc1 -emit-mlir -fopenmp -mmlir --enable-delayed-privatization-staging \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+
+! CHECK-LABEL:   omp.private {type = firstprivate} @_QFExdgfx_firstprivate_i32 : i32 copy {
+! CHECK:         ^bb0(%{{.*}}: !fir.ref<i32>, %{{.*}}: !fir.ref<i32>):
+! CHECK:           %{{.*}} = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:           fir.store %{{.*}} to %{{.*}} : !fir.ref<i32>
+! CHECK:           omp.yield(%{{.*}} : !fir.ref<i32>)
+! CHECK:         }
+
+! CHECK-LABEL:   omp.private {type = firstprivate} @_QFExfpvx_firstprivate_i32 : i32 copy {
+! CHECK:         ^bb0(%{{.*}}: !fir.ref<i32>, %{{.*}}: !fir.ref<i32>):
+! CHECK:           %{{.*}} = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:           fir.store %{{.*}} to %{{.*}} : !fir.ref<i32>
+! CHECK:           omp.yield(%{{.*}} : !fir.ref<i32>)
+! CHECK:         }
+
+! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "xdgfx", uniq_name = "_QFExdgfx"}
+! CHECK:  %[[VAL_1:.*]] = fir.declare %[[VAL_0]] {uniq_name = "_QFExdgfx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "xfpvx", uniq_name = "_QFExfpvx"}
+! CHECK:  %[[VAL_3:.*]] = fir.declare %[[VAL_2]] {uniq_name = "_QFExfpvx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_3]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32>
+! CHECK:  %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32>
+
+! CHECK:  omp.target map_entries(%[[VAL_4]] -> %{{.*}}, %[[VAL_5]] -> %{{.*}} : !fir.ref<i32>, !fir.ref<i32>) private(@_QFExfpvx_firstprivate_i32 %[[VAL_3]] -> %[[VAL_6:.*]] [map_idx=0], @_QFExdgfx_firstprivate_i32 %[[VAL_1]] -> %[[VAL_7:.*]] [map_idx=1] : !fir.ref<i32>, !fir.ref<i32>) {
+! CHECK:  %{{.*}} = fir.declare %[[VAL_6]] {uniq_name = "_QFExfpvx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %{{.*}} = fir.declare %[[VAL_7]] {uniq_name = "_QFExdgfx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+
+program test_default_implicit_firstprivate
+  implicit none
+  integer :: xdgfx, xfpvx
+  xdgfx = 1
+  xfpvx = 2
+  !$omp target firstprivate(xfpvx)
+  xdgfx = 42
+  xfpvx = 43
+  !$omp end target
+  write(*,*) xdgfx, xfpvx
+end program
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
index 217ac5638a3ea..e7e3fb2097308 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
@@ -139,28 +139,28 @@ end subroutine target_allocatable
 ! CHECK:        %[[REAL_ARR_ALLOC:.*]] = fir.alloca !fir.array<?xf32>, {{.*}} {bindc_name = "real_arr", {{.*}}}
 ! CHECK:        %[[REAL_ARR_DECL:.*]]:2 = hlfir.declare %[[REAL_ARR_ALLOC]]({{.*}})
 ! CHECK:        fir.store %[[REAL_ARR_DECL]]#0 to %[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
-! CHECK:        %[[MAPPED_MI0:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#1 : !fir.ref<i32>, i32) {{.*}}
 ! CHECK:        %[[ALLOC_VAR_MEMBER:.*]] = omp.map.info var_ptr(%[[ALLOC_VAR_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, i32)
 ! CHECK:        %[[ALLOC_VAR_MAP:.*]] = omp.map.info var_ptr(%[[ALLOC_VAR_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) {{.*}} members(%[[ALLOC_VAR_MEMBER]] :
 ! CHECK:        %[[REAL_ARR_MEMBER:.*]] = omp.map.info var_ptr(%[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, f32)
 ! CHECK:        %[[REAL_ARR_DESC_MAP:.*]] = omp.map.info var_ptr(%[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) {{.*}} members(%[[REAL_ARR_MEMBER]] :
 ! CHECK:        fir.store %[[CHAR_VAR_DECL]]#0 to %[[CHAR_VAR_DESC_ALLOCA]] : !fir.ref<!fir.boxchar<1>>
 ! CHECK:        %[[CHAR_VAR_DESC_MAP:.*]] = omp.map.info var_ptr(%[[CHAR_VAR_DESC_ALLOCA]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>)
+! CHECK:        %[[MAPPED_MI0:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#0 : !fir.ref<i32>, i32) {{.*}}
 ! CHECK:        omp.target
 ! CHECK-SAME:     map_entries(
-! CHECK-SAME:       %[[MAPPED_MI0]] -> %[[MAPPED_ARG0:[^,]+]],
-! CHECK-SAME:       %[[ALLOC_VAR_MAP]] -> %[[MAPPED_ARG1:[^,]+]]
+! CHECK-SAME:       %[[ALLOC_VAR_MAP]] -> %[[MAPPED_ARG1:[^,]+]],
 ! CHECK-SAME:       %[[REAL_ARR_DESC_MAP]] -> %[[MAPPED_ARG2:[^,]+]]
 ! CHECK-SAME:       %[[CHAR_VAR_DESC_MAP]] -> %[[MAPPED_ARG3:.[^,]+]]
-! CHECK-SAME:       !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.boxchar<1>>, !fir.llvm_ptr<!fir.ref<i32>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
+! CHECK-SAME:       %[[MAPPED_MI0]] -> %[[MAPPED_ARG0:[^,]+]]
+! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.boxchar<1>>, !fir.ref<i32>, !fir.llvm_ptr<!fir.ref<i32>>
 ! CHECK-SAME:     private(
-! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]] [map_idx=1],
+! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]] [map_idx=0],
 ! CHECK-SAME:       @[[REAL_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[REAL_ARG:[^,]+]],
 ! CHECK-SAME:       @[[LB_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[LB_ARG:[^,]+]],
-! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}} -> %[[ARR_ARG:[^,]+]] [map_idx=2],
+! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}} -> %[[ARR_ARG:[^,]+]] [map_idx=1],
 ! CHECK-SAME:       @[[COMP_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[COMP_ARG:[^,]+]],
-! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] [map_idx=3] :
-! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<complex<f32>>, !fir.boxchar<1>) {
+! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] [map_idx=2],
+! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<complex<f32>>, !fir.boxchar<1>, !fir.ref<i32>) {
 ! CHECK-NOT:      fir.alloca
 ! CHECK:          hlfir.declare %[[ALLOC_ARG]]
 ! CHECK:          hlfir.declare %[[REAL_ARG]]

``````````

</details>


https://github.com/llvm/llvm-project/pull/147442

From flang-commits at lists.llvm.org  Mon Jul  7 18:08:35 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 18:08:35 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147442@github.com>
Message-ID: <686c6f93.630a0220.1b98e0.b707@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: None (agozillon)

<details>
<summary>Changes</summary>

Currently, we indicate to the runtime that implicit scalar captures are firstprivate (via map and
capture types), enough for the runtime trace to treat it as such, but we do not CodeGen the IR
in such a way that we can take full advantage of this aspect of the OpenMP specification.

This patch seeks to change that by applying the correct symbol flags (firstprivate/implicit) to the
implicitly captured scalars within target regions, which then triggers the delayed privitization code
generation for these symbols, bringing the code generation in-line with the explicit firstpriviate
clause. Currently, similarly to the delayed privitization I have sheltered this segment of code
behind the EnabledDelayedPrivitization flag, as without it, we'll trigger an compiler error for
firstprivate not being supported any time we implicitly capture a scalar and try to firstprivitize
it, in future when this flag is removed it can also be removed here. So, for now, you need to
enable this via providing the compiler the flag on compilation of any programs.

---
Full diff: https://github.com/llvm/llvm-project/pull/147442.diff


3 Files Affected:

- (modified) flang/lib/Semantics/resolve-directives.cpp (+38-1) 
- (added) flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90 (+42) 
- (modified) flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 (+8-8) 


``````````diff
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 299bb6ff876e7..f7aa2df4c25b1 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -23,6 +23,7 @@
 #include "flang/Semantics/openmp-modifiers.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Flags.h"
 #include "llvm/Frontend/OpenMP/OMP.h.inc"
 #include "llvm/Support/Debug.h"
 #include <list>
@@ -2297,6 +2298,34 @@ static bool IsSymbolStaticStorageDuration(const Symbol &symbol) {
       (ultSym.flags().test(Symbol::Flag::InCommonBlock));
 }
 
+static bool IsTargetCaptureImplicitlyFirstPrivatizeable(
+    const Symbol &symbol, const Symbol::Flags &dsa) {
+  // if we're associated with any other flags we skip implicit privitization
+  // for now. If we're an allocatable, pointer or declare target, we're not
+  // implicitly firstprivitizeable under OpenMP restrictions.
+  // TODO: Relax restriction as we progress privitization and further
+  // investigate the flags we can intermix with.
+  if (!dsa.none() || !symbol.flags().none() ||
+      semantics::IsAllocatableOrPointer(symbol)) {
+    return false;
+  }
+
+  // It is default firstprivatizeable as far as the OpenMP specification is
+  // concerned if it is a non-array scalar type that has been implicitly
+  // captured in a target region
+  const auto *type{symbol.GetType()};
+  if ((!symbol.GetShape() || symbol.GetShape()->empty()) &&
+      (type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Numeric ||
+          type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Logical ||
+          type->category() ==
+              Fortran::semantics::DeclTypeSpec::Category::Character)) {
+    return true;
+  }
+  return false;
+}
+
 void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
   if (!IsPrivatizable(symbol)) {
     return;
@@ -2444,7 +2473,15 @@ void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
       useLastDeclSymbol();
       PRINT_IMPLICIT_RULE("3) enclosing context");
     } else if (targetDir) {
-      // TODO 4) not mapped target variable -> firstprivate
+      // 4) not mapped target variable  -> firstprivate
+      //    - i.e. implicit, but meets OpenMP specification rules for
+      //    firstprivate "promotion"
+      if (enableDelayedPrivatizationStaging && symbol &&
+          IsTargetCaptureImplicitlyFirstPrivatizeable(*symbol, prevDSA)) {
+        prevDSA.set(Symbol::Flag::OmpImplicit);
+        prevDSA.set(Symbol::Flag::OmpFirstPrivate);
+        makeSymbol(prevDSA);
+      }
       dsa = prevDSA;
     } else if (taskGenDir) {
       // TODO 5) dummy arg in orphaned taskgen construct -> firstprivate
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
new file mode 100644
index 0000000000000..6cd14e7d3a2b4
--- /dev/null
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
@@ -0,0 +1,42 @@
+! Tests delayed privatization works for implicit capture of scalars similarly to
+! the way it works for explicitly firstprivitized scalars.
+
+! RUN: %flang_fc1 -emit-mlir -fopenmp -mmlir --enable-delayed-privatization-staging \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+
+! CHECK-LABEL:   omp.private {type = firstprivate} @_QFExdgfx_firstprivate_i32 : i32 copy {
+! CHECK:         ^bb0(%{{.*}}: !fir.ref<i32>, %{{.*}}: !fir.ref<i32>):
+! CHECK:           %{{.*}} = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:           fir.store %{{.*}} to %{{.*}} : !fir.ref<i32>
+! CHECK:           omp.yield(%{{.*}} : !fir.ref<i32>)
+! CHECK:         }
+
+! CHECK-LABEL:   omp.private {type = firstprivate} @_QFExfpvx_firstprivate_i32 : i32 copy {
+! CHECK:         ^bb0(%{{.*}}: !fir.ref<i32>, %{{.*}}: !fir.ref<i32>):
+! CHECK:           %{{.*}} = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:           fir.store %{{.*}} to %{{.*}} : !fir.ref<i32>
+! CHECK:           omp.yield(%{{.*}} : !fir.ref<i32>)
+! CHECK:         }
+
+! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "xdgfx", uniq_name = "_QFExdgfx"}
+! CHECK:  %[[VAL_1:.*]] = fir.declare %[[VAL_0]] {uniq_name = "_QFExdgfx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "xfpvx", uniq_name = "_QFExfpvx"}
+! CHECK:  %[[VAL_3:.*]] = fir.declare %[[VAL_2]] {uniq_name = "_QFExfpvx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_3]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32>
+! CHECK:  %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) -> !fir.ref<i32>
+
+! CHECK:  omp.target map_entries(%[[VAL_4]] -> %{{.*}}, %[[VAL_5]] -> %{{.*}} : !fir.ref<i32>, !fir.ref<i32>) private(@_QFExfpvx_firstprivate_i32 %[[VAL_3]] -> %[[VAL_6:.*]] [map_idx=0], @_QFExdgfx_firstprivate_i32 %[[VAL_1]] -> %[[VAL_7:.*]] [map_idx=1] : !fir.ref<i32>, !fir.ref<i32>) {
+! CHECK:  %{{.*}} = fir.declare %[[VAL_6]] {uniq_name = "_QFExfpvx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:  %{{.*}} = fir.declare %[[VAL_7]] {uniq_name = "_QFExdgfx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+
+program test_default_implicit_firstprivate
+  implicit none
+  integer :: xdgfx, xfpvx
+  xdgfx = 1
+  xfpvx = 2
+  !$omp target firstprivate(xfpvx)
+  xdgfx = 42
+  xfpvx = 43
+  !$omp end target
+  write(*,*) xdgfx, xfpvx
+end program
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
index 217ac5638a3ea..e7e3fb2097308 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
@@ -139,28 +139,28 @@ end subroutine target_allocatable
 ! CHECK:        %[[REAL_ARR_ALLOC:.*]] = fir.alloca !fir.array<?xf32>, {{.*}} {bindc_name = "real_arr", {{.*}}}
 ! CHECK:        %[[REAL_ARR_DECL:.*]]:2 = hlfir.declare %[[REAL_ARR_ALLOC]]({{.*}})
 ! CHECK:        fir.store %[[REAL_ARR_DECL]]#0 to %[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
-! CHECK:        %[[MAPPED_MI0:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#1 : !fir.ref<i32>, i32) {{.*}}
 ! CHECK:        %[[ALLOC_VAR_MEMBER:.*]] = omp.map.info var_ptr(%[[ALLOC_VAR_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, i32)
 ! CHECK:        %[[ALLOC_VAR_MAP:.*]] = omp.map.info var_ptr(%[[ALLOC_VAR_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) {{.*}} members(%[[ALLOC_VAR_MEMBER]] :
 ! CHECK:        %[[REAL_ARR_MEMBER:.*]] = omp.map.info var_ptr(%[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, f32)
 ! CHECK:        %[[REAL_ARR_DESC_MAP:.*]] = omp.map.info var_ptr(%[[REAL_ARR_DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) {{.*}} members(%[[REAL_ARR_MEMBER]] :
 ! CHECK:        fir.store %[[CHAR_VAR_DECL]]#0 to %[[CHAR_VAR_DESC_ALLOCA]] : !fir.ref<!fir.boxchar<1>>
 ! CHECK:        %[[CHAR_VAR_DESC_MAP:.*]] = omp.map.info var_ptr(%[[CHAR_VAR_DESC_ALLOCA]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>)
+! CHECK:        %[[MAPPED_MI0:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#0 : !fir.ref<i32>, i32) {{.*}}
 ! CHECK:        omp.target
 ! CHECK-SAME:     map_entries(
-! CHECK-SAME:       %[[MAPPED_MI0]] -> %[[MAPPED_ARG0:[^,]+]],
-! CHECK-SAME:       %[[ALLOC_VAR_MAP]] -> %[[MAPPED_ARG1:[^,]+]]
+! CHECK-SAME:       %[[ALLOC_VAR_MAP]] -> %[[MAPPED_ARG1:[^,]+]],
 ! CHECK-SAME:       %[[REAL_ARR_DESC_MAP]] -> %[[MAPPED_ARG2:[^,]+]]
 ! CHECK-SAME:       %[[CHAR_VAR_DESC_MAP]] -> %[[MAPPED_ARG3:.[^,]+]]
-! CHECK-SAME:       !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.boxchar<1>>, !fir.llvm_ptr<!fir.ref<i32>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
+! CHECK-SAME:       %[[MAPPED_MI0]] -> %[[MAPPED_ARG0:[^,]+]]
+! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.boxchar<1>>, !fir.ref<i32>, !fir.llvm_ptr<!fir.ref<i32>>
 ! CHECK-SAME:     private(
-! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]] [map_idx=1],
+! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]] [map_idx=0],
 ! CHECK-SAME:       @[[REAL_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[REAL_ARG:[^,]+]],
 ! CHECK-SAME:       @[[LB_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[LB_ARG:[^,]+]],
-! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}} -> %[[ARR_ARG:[^,]+]] [map_idx=2],
+! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}} -> %[[ARR_ARG:[^,]+]] [map_idx=1],
 ! CHECK-SAME:       @[[COMP_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[COMP_ARG:[^,]+]],
-! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] [map_idx=3] :
-! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<complex<f32>>, !fir.boxchar<1>) {
+! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] [map_idx=2],
+! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<complex<f32>>, !fir.boxchar<1>, !fir.ref<i32>) {
 ! CHECK-NOT:      fir.alloca
 ! CHECK:          hlfir.declare %[[ALLOC_ARG]]
 ! CHECK:          hlfir.declare %[[REAL_ARG]]

``````````

</details>


https://github.com/llvm/llvm-project/pull/147442

From flang-commits at lists.llvm.org  Mon Jul  7 18:09:25 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 07 Jul 2025 18:09:25 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147442@github.com>
Message-ID: <686c6fc5.170a0220.2efb89.aec4@mx.google.com>


agozillon wrote:

Please feel free to add more people to the review list (or add/remove yourself if you wish), wasn't sure who were the best people to review this as it's not my usual little segment!

https://github.com/llvm/llvm-project/pull/147442

From flang-commits at lists.llvm.org  Mon Jul  7 18:13:03 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 07 Jul 2025 18:13:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not produce data transfer
 in offloaded do concurrent (PR #147435)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147435@github.com>
Message-ID: <686c709f.170a0220.1fccb8.a8d0@mx.google.com>


================
@@ -55,7 +55,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, StdPar)
----------------
klausler wrote:

Can we please have a better name than "StdPar", like maybe "OffloadDoConcurrent"?

https://github.com/llvm/llvm-project/pull/147435

From flang-commits at lists.llvm.org  Mon Jul  7 18:13:03 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 07 Jul 2025 18:13:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not produce data transfer
 in offloaded do concurrent (PR #147435)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147435@github.com>
Message-ID: <686c709f.170a0220.94668.0d16@mx.google.com>


================
@@ -197,11 +197,11 @@ static void WarnUndefinedFunctionResult(
 
 using StatementSemanticsPass1 = ExprChecker;
 using StatementSemanticsPass2 = SemanticsVisitor<AllocateChecker,
-    ArithmeticIfStmtChecker, AssignmentChecker, CaseChecker, CoarrayChecker,
-    DataChecker, DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker,
-    MiscChecker, NamelistChecker, NullifyChecker, PurityChecker,
-    ReturnStmtChecker, SelectRankConstructChecker, SelectTypeChecker,
-    StopChecker>;
+    ArithmeticIfStmtChecker, CaseChecker, CoarrayChecker, DataChecker,
+    DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker, MiscChecker,
+    NamelistChecker, NullifyChecker, PurityChecker, ReturnStmtChecker,
+    SelectRankConstructChecker, SelectTypeChecker, StopChecker>;
+using StatementSemanticsPass3 = SemanticsVisitor<AssignmentChecker>;
----------------
klausler wrote:

This is a whole additional parse tree walk!  Maybe you could put the assignment checker into the first pass with the expressions instead.

https://github.com/llvm/llvm-project/pull/147435

From flang-commits at lists.llvm.org  Mon Jul  7 18:15:31 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Mon, 07 Jul 2025 18:15:31 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not produce data transfer
 in offloaded do concurrent (PR #147435)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147435@github.com>
Message-ID: <686c7133.050a0220.4c65f.c648@mx.google.com>


================
@@ -55,7 +55,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, StdPar)
----------------
clementval wrote:

Ok I'll put a better name

https://github.com/llvm/llvm-project/pull/147435

From flang-commits at lists.llvm.org  Mon Jul  7 18:15:58 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Mon, 07 Jul 2025 18:15:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not produce data transfer
 in offloaded do concurrent (PR #147435)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147435@github.com>
Message-ID: <686c714e.170a0220.2f9934.a70e@mx.google.com>


================
@@ -197,11 +197,11 @@ static void WarnUndefinedFunctionResult(
 
 using StatementSemanticsPass1 = ExprChecker;
 using StatementSemanticsPass2 = SemanticsVisitor<AllocateChecker,
-    ArithmeticIfStmtChecker, AssignmentChecker, CaseChecker, CoarrayChecker,
-    DataChecker, DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker,
-    MiscChecker, NamelistChecker, NullifyChecker, PurityChecker,
-    ReturnStmtChecker, SelectRankConstructChecker, SelectTypeChecker,
-    StopChecker>;
+    ArithmeticIfStmtChecker, CaseChecker, CoarrayChecker, DataChecker,
+    DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker, MiscChecker,
+    NamelistChecker, NullifyChecker, PurityChecker, ReturnStmtChecker,
+    SelectRankConstructChecker, SelectTypeChecker, StopChecker>;
+using StatementSemanticsPass3 = SemanticsVisitor<AssignmentChecker>;
----------------
clementval wrote:

Yeah I was worried about that. I'll try to move it in the first pass.

https://github.com/llvm/llvm-project/pull/147435

From flang-commits at lists.llvm.org  Mon Jul  7 19:02:11 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Mon, 07 Jul 2025 19:02:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686c7c23.050a0220.23eff.5255@mx.google.com>


================
@@ -1173,6 +1173,22 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
   return HasVectorSubscriptHelper{}(expr);
 }
 
+// HasTriplet()
----------------
eugeneepshteyn wrote:

@klausler , could you please review `HasTriplet` part? I'll add the other reviewers for lowering, once I add the test.

https://github.com/llvm/llvm-project/pull/147419

From flang-commits at lists.llvm.org  Mon Jul  7 21:01:02 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Mon, 07 Jul 2025 21:01:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not produce data transfer
 in offloaded do concurrent (PR #147435)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147435@github.com>
Message-ID: <686c97fe.050a0220.c1c64.d14d@mx.google.com>


================
@@ -197,11 +197,11 @@ static void WarnUndefinedFunctionResult(
 
 using StatementSemanticsPass1 = ExprChecker;
 using StatementSemanticsPass2 = SemanticsVisitor<AllocateChecker,
-    ArithmeticIfStmtChecker, AssignmentChecker, CaseChecker, CoarrayChecker,
-    DataChecker, DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker,
-    MiscChecker, NamelistChecker, NullifyChecker, PurityChecker,
-    ReturnStmtChecker, SelectRankConstructChecker, SelectTypeChecker,
-    StopChecker>;
+    ArithmeticIfStmtChecker, CaseChecker, CoarrayChecker, DataChecker,
+    DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker, MiscChecker,
+    NamelistChecker, NullifyChecker, PurityChecker, ReturnStmtChecker,
+    SelectRankConstructChecker, SelectTypeChecker, StopChecker>;
+using StatementSemanticsPass3 = SemanticsVisitor<AssignmentChecker>;
----------------
clementval wrote:

I don't think we can combine these as they are not the same Checker. I'll move these checks to the CUDAChecker so I don't need to add an extra pass. 

https://github.com/llvm/llvm-project/pull/147435

From flang-commits at lists.llvm.org  Mon Jul  7 21:57:22 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Mon, 07 Jul 2025 21:57:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not produce data transfer
 in offloaded do concurrent (PR #147435)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147435@github.com>
Message-ID: <686ca532.170a0220.368c08.b8f3@mx.google.com>


https://github.com/clementval updated https://github.com/llvm/llvm-project/pull/147435

>From f6ae5598ac9967e407f74245cc47fff4e35ebe0f Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Mon, 7 Jul 2025 17:21:42 -0700
Subject: [PATCH 1/3] [flang][cuda] Do not produce data transfer in offloaded
 do concurrent

---
 .../include/flang/Optimizer/Builder/CUFCommon.h  |  2 +-
 flang/include/flang/Support/Fortran-features.h   |  2 +-
 flang/lib/Lower/Bridge.cpp                       |  4 +++-
 flang/lib/Optimizer/Builder/CUFCommon.cpp        |  4 +++-
 flang/lib/Semantics/assignment.cpp               | 14 ++++++++++++++
 flang/lib/Semantics/assignment.h                 |  3 +++
 flang/lib/Semantics/semantics.cpp                | 11 ++++++-----
 flang/test/Lower/CUDA/cuda-data-transfer.cuf     | 16 ++++++++++++++++
 flang/tools/bbc/bbc.cpp                          |  8 ++++++++
 9 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/flang/include/flang/Optimizer/Builder/CUFCommon.h b/flang/include/flang/Optimizer/Builder/CUFCommon.h
index 65b9cce1d2021..cfd8a2305d031 100644
--- a/flang/include/flang/Optimizer/Builder/CUFCommon.h
+++ b/flang/include/flang/Optimizer/Builder/CUFCommon.h
@@ -27,7 +27,7 @@ mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
                                             mlir::SymbolTable &symTab);
 
 bool isCUDADeviceContext(mlir::Operation *op);
-bool isCUDADeviceContext(mlir::Region &);
+bool isCUDADeviceContext(mlir::Region &, bool isStdParEnabled = false);
 bool isRegisteredDeviceGlobal(fir::GlobalOp op);
 bool isRegisteredDeviceAttr(std::optional<cuf::DataAttribute> attr);
 
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 8a2b7b29a5233..393ee7d0687c6 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, StdPar)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..1be64ea26d7ea 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4886,7 +4886,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::Location loc = getCurrentLocation();
     fir::FirOpBuilder &builder = getFirOpBuilder();
 
-    bool isInDeviceContext = cuf::isCUDADeviceContext(builder.getRegion());
+    bool isInDeviceContext = cuf::isCUDADeviceContext(
+        builder.getRegion(), getFoldingContext().languageFeatures().IsEnabled(
+                                 Fortran::common::LanguageFeature::StdPar));
 
     bool isCUDATransfer =
         IsCUDADataTransfer(assign.lhs, assign.rhs) && !isInDeviceContext;
diff --git a/flang/lib/Optimizer/Builder/CUFCommon.cpp b/flang/lib/Optimizer/Builder/CUFCommon.cpp
index 5f286c04a7ca0..d40ae70503350 100644
--- a/flang/lib/Optimizer/Builder/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Builder/CUFCommon.cpp
@@ -43,7 +43,7 @@ bool cuf::isCUDADeviceContext(mlir::Operation *op) {
 // for it.
 // If the insertion point is inside an OpenACC region op, it is considered
 // device context.
-bool cuf::isCUDADeviceContext(mlir::Region &region) {
+bool cuf::isCUDADeviceContext(mlir::Region &region, bool isStdParEnabled) {
   if (region.getParentOfType<cuf::KernelOp>())
     return true;
   if (region.getParentOfType<mlir::acc::ComputeRegionOpInterface>())
@@ -56,6 +56,8 @@ bool cuf::isCUDADeviceContext(mlir::Region &region) {
              cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
     }
   }
+  if (isStdParEnabled && region.getParentOfType<fir::DoConcurrentLoopOp>())
+    return true;
   return false;
 }
 
diff --git a/flang/lib/Semantics/assignment.cpp b/flang/lib/Semantics/assignment.cpp
index 43e23a9d8f60b..8bd123ba1e806 100644
--- a/flang/lib/Semantics/assignment.cpp
+++ b/flang/lib/Semantics/assignment.cpp
@@ -260,6 +260,20 @@ void AssignmentChecker::Enter(const parser::CUFKernelDoConstruct &x) {
 void AssignmentChecker::Leave(const parser::CUFKernelDoConstruct &) {
   --context_.value().deviceConstructDepth_;
 }
+void AssignmentChecker::Enter(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context().foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::StdPar)) {
+    ++context_.value().deviceConstructDepth_;
+  }
+}
+void AssignmentChecker::Leave(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context().foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::StdPar)) {
+    --context_.value().deviceConstructDepth_;
+  }
+}
 static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
   const auto &beginBlockDirective =
       std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
diff --git a/flang/lib/Semantics/assignment.h b/flang/lib/Semantics/assignment.h
index 4a1bb92037119..fb703c71c989a 100644
--- a/flang/lib/Semantics/assignment.h
+++ b/flang/lib/Semantics/assignment.h
@@ -21,6 +21,7 @@ struct MaskedElsewhereStmt;
 struct PointerAssignmentStmt;
 struct WhereConstructStmt;
 struct WhereStmt;
+struct DoConstruct;
 } // namespace Fortran::parser
 
 namespace Fortran::semantics {
@@ -54,6 +55,8 @@ class AssignmentChecker : public virtual BaseChecker {
   void Leave(const parser::OpenACCCombinedConstruct &);
   void Enter(const parser::OpenACCLoopConstruct &);
   void Leave(const parser::OpenACCLoopConstruct &);
+  void Enter(const parser::DoConstruct &);
+  void Leave(const parser::DoConstruct &);
 
   SemanticsContext &context();
 
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ed41c6fb16892..f6e6272cbcd7b 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -197,11 +197,11 @@ static void WarnUndefinedFunctionResult(
 
 using StatementSemanticsPass1 = ExprChecker;
 using StatementSemanticsPass2 = SemanticsVisitor<AllocateChecker,
-    ArithmeticIfStmtChecker, AssignmentChecker, CaseChecker, CoarrayChecker,
-    DataChecker, DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker,
-    MiscChecker, NamelistChecker, NullifyChecker, PurityChecker,
-    ReturnStmtChecker, SelectRankConstructChecker, SelectTypeChecker,
-    StopChecker>;
+    ArithmeticIfStmtChecker, CaseChecker, CoarrayChecker, DataChecker,
+    DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker, MiscChecker,
+    NamelistChecker, NullifyChecker, PurityChecker, ReturnStmtChecker,
+    SelectRankConstructChecker, SelectTypeChecker, StopChecker>;
+using StatementSemanticsPass3 = SemanticsVisitor<AssignmentChecker>;
 
 static bool PerformStatementSemantics(
     SemanticsContext &context, parser::Program &program) {
@@ -212,6 +212,7 @@ static bool PerformStatementSemantics(
   StatementSemanticsPass1{context}.Walk(program);
   StatementSemanticsPass2 pass2{context};
   pass2.Walk(program);
+  StatementSemanticsPass3{context}.Walk(program);
   if (context.languageFeatures().IsEnabled(common::LanguageFeature::OpenACC)) {
     SemanticsVisitor<AccStructureChecker>{context}.Walk(program);
   }
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index d0032af415604..68a0202f951fe 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -403,3 +403,19 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPsub20()
 ! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
+
+subroutine sub21()
+  real, allocatable,device:: a(:,:), b(:,:)
+  real:: s
+  integer:: i,j,N=16
+  allocate(a(N,N),b(N,N))
+  do concurrent(i=1:N, j=1:N) reduce(+:s)
+    b(i,j)=a(i,j)**2
+    s=s+b(i,j)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub21()
+! CHECK: fir.do_concurrent.loop
+! CHECK-NOT: cuf.data_transfer
+! CHECK: hlfir.assign
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 59372a8eb58ed..25d072c93f22c 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -223,6 +223,10 @@ static llvm::cl::opt<bool> enableCUDA("fcuda",
                                       llvm::cl::desc("enable CUDA Fortran"),
                                       llvm::cl::init(false));
 
+static llvm::cl::opt<bool> enableStdPar("stdpar",
+                                        llvm::cl::desc("enable stdpar"),
+                                        llvm::cl::init(false));
+
 static llvm::cl::opt<bool>
     disableCUDAWarpFunction("fcuda-disable-warp-function",
                             llvm::cl::desc("Disable CUDA Warp Function"),
@@ -608,6 +612,10 @@ int main(int argc, char **argv) {
     options.features.Enable(Fortran::common::LanguageFeature::CUDA);
   }
 
+  if (enableStdPar) {
+    options.features.Enable(Fortran::common::LanguageFeature::StdPar);
+  }
+
   if (disableCUDAWarpFunction) {
     options.features.Enable(
         Fortran::common::LanguageFeature::CudaWarpMatchFunction, false);

>From c4844a45b3327af57574a338c280f965b3c4f49f Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Mon, 7 Jul 2025 21:35:13 -0700
Subject: [PATCH 2/3] Move check to CUDAChecker

---
 flang/lib/Semantics/assignment.cpp | 70 ------------------------------
 flang/lib/Semantics/assignment.h   | 10 -----
 flang/lib/Semantics/check-cuda.cpp | 64 +++++++++++++++++++++++++--
 flang/lib/Semantics/check-cuda.h   | 10 ++++-
 flang/lib/Semantics/semantics.cpp  | 11 +++--
 5 files changed, 75 insertions(+), 90 deletions(-)

diff --git a/flang/lib/Semantics/assignment.cpp b/flang/lib/Semantics/assignment.cpp
index 8bd123ba1e806..88e08887160d9 100644
--- a/flang/lib/Semantics/assignment.cpp
+++ b/flang/lib/Semantics/assignment.cpp
@@ -42,7 +42,6 @@ class AssignmentContext {
   void Analyze(const parser::AssignmentStmt &);
   void Analyze(const parser::PointerAssignmentStmt &);
   void Analyze(const parser::ConcurrentControl &);
-  int deviceConstructDepth_{0};
   SemanticsContext &context() { return context_; }
 
 private:
@@ -97,21 +96,6 @@ void AssignmentContext::Analyze(const parser::AssignmentStmt &stmt) {
     if (whereDepth_ > 0) {
       CheckShape(lhsLoc, &lhs);
     }
-    if (context_.foldingContext().languageFeatures().IsEnabled(
-            common::LanguageFeature::CUDA)) {
-      const auto &scope{context_.FindScope(lhsLoc)};
-      const Scope &progUnit{GetProgramUnitContaining(scope)};
-      if (!IsCUDADeviceContext(&progUnit) && deviceConstructDepth_ == 0) {
-        if (Fortran::evaluate::HasCUDADeviceAttrs(lhs) &&
-            Fortran::evaluate::HasCUDAImplicitTransfer(rhs)) {
-          if (GetNbOfCUDAManagedOrUnifiedSymbols(lhs) == 1 &&
-              GetNbOfCUDAManagedOrUnifiedSymbols(rhs) == 1 &&
-              GetNbOfCUDADeviceSymbols(rhs) == 1)
-            return; // This is a special case handled on the host.
-          context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
-        }
-      }
-    }
   }
 }
 
@@ -254,60 +238,6 @@ void AssignmentChecker::Enter(const parser::MaskedElsewhereStmt &x) {
 void AssignmentChecker::Leave(const parser::MaskedElsewhereStmt &) {
   context_.value().PopWhereContext();
 }
-void AssignmentChecker::Enter(const parser::CUFKernelDoConstruct &x) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::CUFKernelDoConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Enter(const parser::DoConstruct &x) {
-  if (x.IsDoConcurrent() &&
-      context().foldingContext().languageFeatures().IsEnabled(
-          common::LanguageFeature::StdPar)) {
-    ++context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Leave(const parser::DoConstruct &x) {
-  if (x.IsDoConcurrent() &&
-      context().foldingContext().languageFeatures().IsEnabled(
-          common::LanguageFeature::StdPar)) {
-    --context_.value().deviceConstructDepth_;
-  }
-}
-static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
-  const auto &beginBlockDirective =
-      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
-  const auto &blockDirective =
-      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
-  if (blockDirective.v == llvm::acc::ACCD_parallel ||
-      blockDirective.v == llvm::acc::ACCD_serial ||
-      blockDirective.v == llvm::acc::ACCD_kernels) {
-    return true;
-  }
-  return false;
-}
-void AssignmentChecker::Enter(const parser::OpenACCBlockConstruct &x) {
-  if (IsOpenACCComputeConstruct(x)) {
-    ++context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Leave(const parser::OpenACCBlockConstruct &x) {
-  if (IsOpenACCComputeConstruct(x)) {
-    --context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Enter(const parser::OpenACCCombinedConstruct &) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::OpenACCCombinedConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Enter(const parser::OpenACCLoopConstruct &) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::OpenACCLoopConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
 
 } // namespace Fortran::semantics
 template class Fortran::common::Indirection<
diff --git a/flang/lib/Semantics/assignment.h b/flang/lib/Semantics/assignment.h
index fb703c71c989a..54b3be2da06e8 100644
--- a/flang/lib/Semantics/assignment.h
+++ b/flang/lib/Semantics/assignment.h
@@ -47,16 +47,6 @@ class AssignmentChecker : public virtual BaseChecker {
   void Leave(const parser::EndWhereStmt &);
   void Enter(const parser::MaskedElsewhereStmt &);
   void Leave(const parser::MaskedElsewhereStmt &);
-  void Enter(const parser::CUFKernelDoConstruct &);
-  void Leave(const parser::CUFKernelDoConstruct &);
-  void Enter(const parser::OpenACCBlockConstruct &);
-  void Leave(const parser::OpenACCBlockConstruct &);
-  void Enter(const parser::OpenACCCombinedConstruct &);
-  void Leave(const parser::OpenACCCombinedConstruct &);
-  void Enter(const parser::OpenACCLoopConstruct &);
-  void Leave(const parser::OpenACCLoopConstruct &);
-  void Enter(const parser::DoConstruct &);
-  void Leave(const parser::DoConstruct &);
 
   SemanticsContext &context();
 
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index 8decfb0149829..c399fdb5a48de 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -685,18 +685,67 @@ void CUDAChecker::Enter(const parser::CUFKernelDoConstruct &x) {
       std::get<std::list<parser::CUFReduction>>(directive.t)) {
     CheckReduce(context_, reduce);
   }
-  inCUFKernelDoConstruct_ = true;
+  ++deviceConstructDepth_;
+}
+
+static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
+  const auto &beginBlockDirective =
+      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
+  const auto &blockDirective =
+      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
+  if (blockDirective.v == llvm::acc::ACCD_parallel ||
+      blockDirective.v == llvm::acc::ACCD_serial ||
+      blockDirective.v == llvm::acc::ACCD_kernels) {
+    return true;
+  }
+  return false;
 }
 
 void CUDAChecker::Leave(const parser::CUFKernelDoConstruct &) {
-  inCUFKernelDoConstruct_ = false;
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    ++deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Leave(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    --deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Enter(const parser::OpenACCCombinedConstruct &) {
+  ++deviceConstructDepth_;
+}
+void CUDAChecker::Leave(const parser::OpenACCCombinedConstruct &) {
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::OpenACCLoopConstruct &) {
+  ++deviceConstructDepth_;
+}
+void CUDAChecker::Leave(const parser::OpenACCLoopConstruct &) {
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context_.foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::StdPar)) {
+    ++deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Leave(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context_.foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::StdPar)) {
+    --deviceConstructDepth_;
+  }
 }
 
 void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
   auto lhsLoc{std::get<parser::Variable>(x.t).GetSource()};
   const auto &scope{context_.FindScope(lhsLoc)};
   const Scope &progUnit{GetProgramUnitContaining(scope)};
-  if (IsCUDADeviceContext(&progUnit) || inCUFKernelDoConstruct_) {
+  if (IsCUDADeviceContext(&progUnit) || deviceConstructDepth_ > 0) {
     return; // Data transfer with assignment is only perform on host.
   }
 
@@ -714,6 +763,15 @@ void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
     context_.Say(lhsLoc,
         "More than one reference to a CUDA object on the right hand side of the assigment"_err_en_US);
   }
+
+  if (Fortran::evaluate::HasCUDADeviceAttrs(assign->lhs) &&
+      Fortran::evaluate::HasCUDAImplicitTransfer(assign->rhs)) {
+    if (GetNbOfCUDAManagedOrUnifiedSymbols(assign->lhs) == 1 &&
+        GetNbOfCUDAManagedOrUnifiedSymbols(assign->rhs) == 1 &&
+        GetNbOfCUDADeviceSymbols(assign->rhs) == 1)
+      return; // This is a special case handled on the host.
+    context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
+  }
 }
 
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-cuda.h b/flang/lib/Semantics/check-cuda.h
index 222a2ee04b57c..10000253ffe5a 100644
--- a/flang/lib/Semantics/check-cuda.h
+++ b/flang/lib/Semantics/check-cuda.h
@@ -41,10 +41,18 @@ class CUDAChecker : public virtual BaseChecker {
   void Enter(const parser::CUFKernelDoConstruct &);
   void Leave(const parser::CUFKernelDoConstruct &);
   void Enter(const parser::AssignmentStmt &);
+  void Enter(const parser::OpenACCBlockConstruct &);
+  void Leave(const parser::OpenACCBlockConstruct &);
+  void Enter(const parser::OpenACCCombinedConstruct &);
+  void Leave(const parser::OpenACCCombinedConstruct &);
+  void Enter(const parser::OpenACCLoopConstruct &);
+  void Leave(const parser::OpenACCLoopConstruct &);
+  void Enter(const parser::DoConstruct &);
+  void Leave(const parser::DoConstruct &);
 
 private:
   SemanticsContext &context_;
-  bool inCUFKernelDoConstruct_ = false;
+  int deviceConstructDepth_{0};
 };
 
 bool CanonicalizeCUDA(parser::Program &);
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index f6e6272cbcd7b..ed41c6fb16892 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -197,11 +197,11 @@ static void WarnUndefinedFunctionResult(
 
 using StatementSemanticsPass1 = ExprChecker;
 using StatementSemanticsPass2 = SemanticsVisitor<AllocateChecker,
-    ArithmeticIfStmtChecker, CaseChecker, CoarrayChecker, DataChecker,
-    DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker, MiscChecker,
-    NamelistChecker, NullifyChecker, PurityChecker, ReturnStmtChecker,
-    SelectRankConstructChecker, SelectTypeChecker, StopChecker>;
-using StatementSemanticsPass3 = SemanticsVisitor<AssignmentChecker>;
+    ArithmeticIfStmtChecker, AssignmentChecker, CaseChecker, CoarrayChecker,
+    DataChecker, DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker,
+    MiscChecker, NamelistChecker, NullifyChecker, PurityChecker,
+    ReturnStmtChecker, SelectRankConstructChecker, SelectTypeChecker,
+    StopChecker>;
 
 static bool PerformStatementSemantics(
     SemanticsContext &context, parser::Program &program) {
@@ -212,7 +212,6 @@ static bool PerformStatementSemantics(
   StatementSemanticsPass1{context}.Walk(program);
   StatementSemanticsPass2 pass2{context};
   pass2.Walk(program);
-  StatementSemanticsPass3{context}.Walk(program);
   if (context.languageFeatures().IsEnabled(common::LanguageFeature::OpenACC)) {
     SemanticsVisitor<AccStructureChecker>{context}.Walk(program);
   }

>From bbdd46de5f2a0fd741ac7cd5ef72e27a15208a77 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Mon, 7 Jul 2025 21:38:01 -0700
Subject: [PATCH 3/3] Rename language feature

---
 flang/include/flang/Optimizer/Builder/CUFCommon.h |  3 ++-
 flang/include/flang/Support/Fortran-features.h    |  2 +-
 flang/lib/Lower/Bridge.cpp                        |  5 +++--
 flang/lib/Optimizer/Builder/CUFCommon.cpp         |  6 ++++--
 flang/lib/Semantics/check-cuda.cpp                |  4 ++--
 flang/tools/bbc/bbc.cpp                           | 12 +++++++-----
 6 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/flang/include/flang/Optimizer/Builder/CUFCommon.h b/flang/include/flang/Optimizer/Builder/CUFCommon.h
index cfd8a2305d031..5c56dd6b695f8 100644
--- a/flang/include/flang/Optimizer/Builder/CUFCommon.h
+++ b/flang/include/flang/Optimizer/Builder/CUFCommon.h
@@ -27,7 +27,8 @@ mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
                                             mlir::SymbolTable &symTab);
 
 bool isCUDADeviceContext(mlir::Operation *op);
-bool isCUDADeviceContext(mlir::Region &, bool isStdParEnabled = false);
+bool isCUDADeviceContext(mlir::Region &,
+                         bool isDoConcurrentOffloadEnabled = false);
 bool isRegisteredDeviceGlobal(fir::GlobalOp op);
 bool isRegisteredDeviceAttr(std::optional<cuf::DataAttribute> attr);
 
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 393ee7d0687c6..857de9479e4e5 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction, StdPar)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 1be64ea26d7ea..696473605a4e0 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4887,8 +4887,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     fir::FirOpBuilder &builder = getFirOpBuilder();
 
     bool isInDeviceContext = cuf::isCUDADeviceContext(
-        builder.getRegion(), getFoldingContext().languageFeatures().IsEnabled(
-                                 Fortran::common::LanguageFeature::StdPar));
+        builder.getRegion(),
+        getFoldingContext().languageFeatures().IsEnabled(
+            Fortran::common::LanguageFeature::DoConcurrentOffload));
 
     bool isCUDATransfer =
         IsCUDADataTransfer(assign.lhs, assign.rhs) && !isInDeviceContext;
diff --git a/flang/lib/Optimizer/Builder/CUFCommon.cpp b/flang/lib/Optimizer/Builder/CUFCommon.cpp
index d40ae70503350..dcbf4991907bf 100644
--- a/flang/lib/Optimizer/Builder/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Builder/CUFCommon.cpp
@@ -43,7 +43,8 @@ bool cuf::isCUDADeviceContext(mlir::Operation *op) {
 // for it.
 // If the insertion point is inside an OpenACC region op, it is considered
 // device context.
-bool cuf::isCUDADeviceContext(mlir::Region &region, bool isStdParEnabled) {
+bool cuf::isCUDADeviceContext(mlir::Region &region,
+                              bool isDoConcurrentOffloadEnabled) {
   if (region.getParentOfType<cuf::KernelOp>())
     return true;
   if (region.getParentOfType<mlir::acc::ComputeRegionOpInterface>())
@@ -56,7 +57,8 @@ bool cuf::isCUDADeviceContext(mlir::Region &region, bool isStdParEnabled) {
              cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
     }
   }
-  if (isStdParEnabled && region.getParentOfType<fir::DoConcurrentLoopOp>())
+  if (isDoConcurrentOffloadEnabled &&
+      region.getParentOfType<fir::DoConcurrentLoopOp>())
     return true;
   return false;
 }
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index c399fdb5a48de..8103402598f5f 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -729,14 +729,14 @@ void CUDAChecker::Leave(const parser::OpenACCLoopConstruct &) {
 void CUDAChecker::Enter(const parser::DoConstruct &x) {
   if (x.IsDoConcurrent() &&
       context_.foldingContext().languageFeatures().IsEnabled(
-          common::LanguageFeature::StdPar)) {
+          common::LanguageFeature::DoConcurrentOffload)) {
     ++deviceConstructDepth_;
   }
 }
 void CUDAChecker::Leave(const parser::DoConstruct &x) {
   if (x.IsDoConcurrent() &&
       context_.foldingContext().languageFeatures().IsEnabled(
-          common::LanguageFeature::StdPar)) {
+          common::LanguageFeature::DoConcurrentOffload)) {
     --deviceConstructDepth_;
   }
 }
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 25d072c93f22c..d216069654bc4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -223,9 +223,10 @@ static llvm::cl::opt<bool> enableCUDA("fcuda",
                                       llvm::cl::desc("enable CUDA Fortran"),
                                       llvm::cl::init(false));
 
-static llvm::cl::opt<bool> enableStdPar("stdpar",
-                                        llvm::cl::desc("enable stdpar"),
-                                        llvm::cl::init(false));
+static llvm::cl::opt<bool>
+    enableDoConcurrentOffload("fdoconcurrent-offload",
+                              llvm::cl::desc("enable do concurrent offload"),
+                              llvm::cl::init(false));
 
 static llvm::cl::opt<bool>
     disableCUDAWarpFunction("fcuda-disable-warp-function",
@@ -612,8 +613,9 @@ int main(int argc, char **argv) {
     options.features.Enable(Fortran::common::LanguageFeature::CUDA);
   }
 
-  if (enableStdPar) {
-    options.features.Enable(Fortran::common::LanguageFeature::StdPar);
+  if (enableDoConcurrentOffload) {
+    options.features.Enable(
+        Fortran::common::LanguageFeature::DoConcurrentOffload);
   }
 
   if (disableCUDAWarpFunction) {


From flang-commits at lists.llvm.org  Mon Jul  7 21:58:25 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Mon, 07 Jul 2025 21:58:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not produce data transfer
 in offloaded do concurrent (PR #147435)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147435@github.com>
Message-ID: <686ca571.170a0220.1a9c27.1779@mx.google.com>


https://github.com/clementval updated https://github.com/llvm/llvm-project/pull/147435

>From f6ae5598ac9967e407f74245cc47fff4e35ebe0f Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Mon, 7 Jul 2025 17:21:42 -0700
Subject: [PATCH 1/4] [flang][cuda] Do not produce data transfer in offloaded
 do concurrent

---
 .../include/flang/Optimizer/Builder/CUFCommon.h  |  2 +-
 flang/include/flang/Support/Fortran-features.h   |  2 +-
 flang/lib/Lower/Bridge.cpp                       |  4 +++-
 flang/lib/Optimizer/Builder/CUFCommon.cpp        |  4 +++-
 flang/lib/Semantics/assignment.cpp               | 14 ++++++++++++++
 flang/lib/Semantics/assignment.h                 |  3 +++
 flang/lib/Semantics/semantics.cpp                | 11 ++++++-----
 flang/test/Lower/CUDA/cuda-data-transfer.cuf     | 16 ++++++++++++++++
 flang/tools/bbc/bbc.cpp                          |  8 ++++++++
 9 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/flang/include/flang/Optimizer/Builder/CUFCommon.h b/flang/include/flang/Optimizer/Builder/CUFCommon.h
index 65b9cce1d2021..cfd8a2305d031 100644
--- a/flang/include/flang/Optimizer/Builder/CUFCommon.h
+++ b/flang/include/flang/Optimizer/Builder/CUFCommon.h
@@ -27,7 +27,7 @@ mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
                                             mlir::SymbolTable &symTab);
 
 bool isCUDADeviceContext(mlir::Operation *op);
-bool isCUDADeviceContext(mlir::Region &);
+bool isCUDADeviceContext(mlir::Region &, bool isStdParEnabled = false);
 bool isRegisteredDeviceGlobal(fir::GlobalOp op);
 bool isRegisteredDeviceAttr(std::optional<cuf::DataAttribute> attr);
 
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 8a2b7b29a5233..393ee7d0687c6 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, StdPar)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..1be64ea26d7ea 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4886,7 +4886,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::Location loc = getCurrentLocation();
     fir::FirOpBuilder &builder = getFirOpBuilder();
 
-    bool isInDeviceContext = cuf::isCUDADeviceContext(builder.getRegion());
+    bool isInDeviceContext = cuf::isCUDADeviceContext(
+        builder.getRegion(), getFoldingContext().languageFeatures().IsEnabled(
+                                 Fortran::common::LanguageFeature::StdPar));
 
     bool isCUDATransfer =
         IsCUDADataTransfer(assign.lhs, assign.rhs) && !isInDeviceContext;
diff --git a/flang/lib/Optimizer/Builder/CUFCommon.cpp b/flang/lib/Optimizer/Builder/CUFCommon.cpp
index 5f286c04a7ca0..d40ae70503350 100644
--- a/flang/lib/Optimizer/Builder/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Builder/CUFCommon.cpp
@@ -43,7 +43,7 @@ bool cuf::isCUDADeviceContext(mlir::Operation *op) {
 // for it.
 // If the insertion point is inside an OpenACC region op, it is considered
 // device context.
-bool cuf::isCUDADeviceContext(mlir::Region &region) {
+bool cuf::isCUDADeviceContext(mlir::Region &region, bool isStdParEnabled) {
   if (region.getParentOfType<cuf::KernelOp>())
     return true;
   if (region.getParentOfType<mlir::acc::ComputeRegionOpInterface>())
@@ -56,6 +56,8 @@ bool cuf::isCUDADeviceContext(mlir::Region &region) {
              cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
     }
   }
+  if (isStdParEnabled && region.getParentOfType<fir::DoConcurrentLoopOp>())
+    return true;
   return false;
 }
 
diff --git a/flang/lib/Semantics/assignment.cpp b/flang/lib/Semantics/assignment.cpp
index 43e23a9d8f60b..8bd123ba1e806 100644
--- a/flang/lib/Semantics/assignment.cpp
+++ b/flang/lib/Semantics/assignment.cpp
@@ -260,6 +260,20 @@ void AssignmentChecker::Enter(const parser::CUFKernelDoConstruct &x) {
 void AssignmentChecker::Leave(const parser::CUFKernelDoConstruct &) {
   --context_.value().deviceConstructDepth_;
 }
+void AssignmentChecker::Enter(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context().foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::StdPar)) {
+    ++context_.value().deviceConstructDepth_;
+  }
+}
+void AssignmentChecker::Leave(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context().foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::StdPar)) {
+    --context_.value().deviceConstructDepth_;
+  }
+}
 static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
   const auto &beginBlockDirective =
       std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
diff --git a/flang/lib/Semantics/assignment.h b/flang/lib/Semantics/assignment.h
index 4a1bb92037119..fb703c71c989a 100644
--- a/flang/lib/Semantics/assignment.h
+++ b/flang/lib/Semantics/assignment.h
@@ -21,6 +21,7 @@ struct MaskedElsewhereStmt;
 struct PointerAssignmentStmt;
 struct WhereConstructStmt;
 struct WhereStmt;
+struct DoConstruct;
 } // namespace Fortran::parser
 
 namespace Fortran::semantics {
@@ -54,6 +55,8 @@ class AssignmentChecker : public virtual BaseChecker {
   void Leave(const parser::OpenACCCombinedConstruct &);
   void Enter(const parser::OpenACCLoopConstruct &);
   void Leave(const parser::OpenACCLoopConstruct &);
+  void Enter(const parser::DoConstruct &);
+  void Leave(const parser::DoConstruct &);
 
   SemanticsContext &context();
 
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ed41c6fb16892..f6e6272cbcd7b 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -197,11 +197,11 @@ static void WarnUndefinedFunctionResult(
 
 using StatementSemanticsPass1 = ExprChecker;
 using StatementSemanticsPass2 = SemanticsVisitor<AllocateChecker,
-    ArithmeticIfStmtChecker, AssignmentChecker, CaseChecker, CoarrayChecker,
-    DataChecker, DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker,
-    MiscChecker, NamelistChecker, NullifyChecker, PurityChecker,
-    ReturnStmtChecker, SelectRankConstructChecker, SelectTypeChecker,
-    StopChecker>;
+    ArithmeticIfStmtChecker, CaseChecker, CoarrayChecker, DataChecker,
+    DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker, MiscChecker,
+    NamelistChecker, NullifyChecker, PurityChecker, ReturnStmtChecker,
+    SelectRankConstructChecker, SelectTypeChecker, StopChecker>;
+using StatementSemanticsPass3 = SemanticsVisitor<AssignmentChecker>;
 
 static bool PerformStatementSemantics(
     SemanticsContext &context, parser::Program &program) {
@@ -212,6 +212,7 @@ static bool PerformStatementSemantics(
   StatementSemanticsPass1{context}.Walk(program);
   StatementSemanticsPass2 pass2{context};
   pass2.Walk(program);
+  StatementSemanticsPass3{context}.Walk(program);
   if (context.languageFeatures().IsEnabled(common::LanguageFeature::OpenACC)) {
     SemanticsVisitor<AccStructureChecker>{context}.Walk(program);
   }
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index d0032af415604..68a0202f951fe 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -403,3 +403,19 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPsub20()
 ! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
+
+subroutine sub21()
+  real, allocatable,device:: a(:,:), b(:,:)
+  real:: s
+  integer:: i,j,N=16
+  allocate(a(N,N),b(N,N))
+  do concurrent(i=1:N, j=1:N) reduce(+:s)
+    b(i,j)=a(i,j)**2
+    s=s+b(i,j)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub21()
+! CHECK: fir.do_concurrent.loop
+! CHECK-NOT: cuf.data_transfer
+! CHECK: hlfir.assign
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 59372a8eb58ed..25d072c93f22c 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -223,6 +223,10 @@ static llvm::cl::opt<bool> enableCUDA("fcuda",
                                       llvm::cl::desc("enable CUDA Fortran"),
                                       llvm::cl::init(false));
 
+static llvm::cl::opt<bool> enableStdPar("stdpar",
+                                        llvm::cl::desc("enable stdpar"),
+                                        llvm::cl::init(false));
+
 static llvm::cl::opt<bool>
     disableCUDAWarpFunction("fcuda-disable-warp-function",
                             llvm::cl::desc("Disable CUDA Warp Function"),
@@ -608,6 +612,10 @@ int main(int argc, char **argv) {
     options.features.Enable(Fortran::common::LanguageFeature::CUDA);
   }
 
+  if (enableStdPar) {
+    options.features.Enable(Fortran::common::LanguageFeature::StdPar);
+  }
+
   if (disableCUDAWarpFunction) {
     options.features.Enable(
         Fortran::common::LanguageFeature::CudaWarpMatchFunction, false);

>From c4844a45b3327af57574a338c280f965b3c4f49f Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Mon, 7 Jul 2025 21:35:13 -0700
Subject: [PATCH 2/4] Move check to CUDAChecker

---
 flang/lib/Semantics/assignment.cpp | 70 ------------------------------
 flang/lib/Semantics/assignment.h   | 10 -----
 flang/lib/Semantics/check-cuda.cpp | 64 +++++++++++++++++++++++++--
 flang/lib/Semantics/check-cuda.h   | 10 ++++-
 flang/lib/Semantics/semantics.cpp  | 11 +++--
 5 files changed, 75 insertions(+), 90 deletions(-)

diff --git a/flang/lib/Semantics/assignment.cpp b/flang/lib/Semantics/assignment.cpp
index 8bd123ba1e806..88e08887160d9 100644
--- a/flang/lib/Semantics/assignment.cpp
+++ b/flang/lib/Semantics/assignment.cpp
@@ -42,7 +42,6 @@ class AssignmentContext {
   void Analyze(const parser::AssignmentStmt &);
   void Analyze(const parser::PointerAssignmentStmt &);
   void Analyze(const parser::ConcurrentControl &);
-  int deviceConstructDepth_{0};
   SemanticsContext &context() { return context_; }
 
 private:
@@ -97,21 +96,6 @@ void AssignmentContext::Analyze(const parser::AssignmentStmt &stmt) {
     if (whereDepth_ > 0) {
       CheckShape(lhsLoc, &lhs);
     }
-    if (context_.foldingContext().languageFeatures().IsEnabled(
-            common::LanguageFeature::CUDA)) {
-      const auto &scope{context_.FindScope(lhsLoc)};
-      const Scope &progUnit{GetProgramUnitContaining(scope)};
-      if (!IsCUDADeviceContext(&progUnit) && deviceConstructDepth_ == 0) {
-        if (Fortran::evaluate::HasCUDADeviceAttrs(lhs) &&
-            Fortran::evaluate::HasCUDAImplicitTransfer(rhs)) {
-          if (GetNbOfCUDAManagedOrUnifiedSymbols(lhs) == 1 &&
-              GetNbOfCUDAManagedOrUnifiedSymbols(rhs) == 1 &&
-              GetNbOfCUDADeviceSymbols(rhs) == 1)
-            return; // This is a special case handled on the host.
-          context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
-        }
-      }
-    }
   }
 }
 
@@ -254,60 +238,6 @@ void AssignmentChecker::Enter(const parser::MaskedElsewhereStmt &x) {
 void AssignmentChecker::Leave(const parser::MaskedElsewhereStmt &) {
   context_.value().PopWhereContext();
 }
-void AssignmentChecker::Enter(const parser::CUFKernelDoConstruct &x) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::CUFKernelDoConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Enter(const parser::DoConstruct &x) {
-  if (x.IsDoConcurrent() &&
-      context().foldingContext().languageFeatures().IsEnabled(
-          common::LanguageFeature::StdPar)) {
-    ++context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Leave(const parser::DoConstruct &x) {
-  if (x.IsDoConcurrent() &&
-      context().foldingContext().languageFeatures().IsEnabled(
-          common::LanguageFeature::StdPar)) {
-    --context_.value().deviceConstructDepth_;
-  }
-}
-static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
-  const auto &beginBlockDirective =
-      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
-  const auto &blockDirective =
-      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
-  if (blockDirective.v == llvm::acc::ACCD_parallel ||
-      blockDirective.v == llvm::acc::ACCD_serial ||
-      blockDirective.v == llvm::acc::ACCD_kernels) {
-    return true;
-  }
-  return false;
-}
-void AssignmentChecker::Enter(const parser::OpenACCBlockConstruct &x) {
-  if (IsOpenACCComputeConstruct(x)) {
-    ++context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Leave(const parser::OpenACCBlockConstruct &x) {
-  if (IsOpenACCComputeConstruct(x)) {
-    --context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Enter(const parser::OpenACCCombinedConstruct &) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::OpenACCCombinedConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Enter(const parser::OpenACCLoopConstruct &) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::OpenACCLoopConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
 
 } // namespace Fortran::semantics
 template class Fortran::common::Indirection<
diff --git a/flang/lib/Semantics/assignment.h b/flang/lib/Semantics/assignment.h
index fb703c71c989a..54b3be2da06e8 100644
--- a/flang/lib/Semantics/assignment.h
+++ b/flang/lib/Semantics/assignment.h
@@ -47,16 +47,6 @@ class AssignmentChecker : public virtual BaseChecker {
   void Leave(const parser::EndWhereStmt &);
   void Enter(const parser::MaskedElsewhereStmt &);
   void Leave(const parser::MaskedElsewhereStmt &);
-  void Enter(const parser::CUFKernelDoConstruct &);
-  void Leave(const parser::CUFKernelDoConstruct &);
-  void Enter(const parser::OpenACCBlockConstruct &);
-  void Leave(const parser::OpenACCBlockConstruct &);
-  void Enter(const parser::OpenACCCombinedConstruct &);
-  void Leave(const parser::OpenACCCombinedConstruct &);
-  void Enter(const parser::OpenACCLoopConstruct &);
-  void Leave(const parser::OpenACCLoopConstruct &);
-  void Enter(const parser::DoConstruct &);
-  void Leave(const parser::DoConstruct &);
 
   SemanticsContext &context();
 
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index 8decfb0149829..c399fdb5a48de 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -685,18 +685,67 @@ void CUDAChecker::Enter(const parser::CUFKernelDoConstruct &x) {
       std::get<std::list<parser::CUFReduction>>(directive.t)) {
     CheckReduce(context_, reduce);
   }
-  inCUFKernelDoConstruct_ = true;
+  ++deviceConstructDepth_;
+}
+
+static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
+  const auto &beginBlockDirective =
+      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
+  const auto &blockDirective =
+      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
+  if (blockDirective.v == llvm::acc::ACCD_parallel ||
+      blockDirective.v == llvm::acc::ACCD_serial ||
+      blockDirective.v == llvm::acc::ACCD_kernels) {
+    return true;
+  }
+  return false;
 }
 
 void CUDAChecker::Leave(const parser::CUFKernelDoConstruct &) {
-  inCUFKernelDoConstruct_ = false;
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    ++deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Leave(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    --deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Enter(const parser::OpenACCCombinedConstruct &) {
+  ++deviceConstructDepth_;
+}
+void CUDAChecker::Leave(const parser::OpenACCCombinedConstruct &) {
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::OpenACCLoopConstruct &) {
+  ++deviceConstructDepth_;
+}
+void CUDAChecker::Leave(const parser::OpenACCLoopConstruct &) {
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context_.foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::StdPar)) {
+    ++deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Leave(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context_.foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::StdPar)) {
+    --deviceConstructDepth_;
+  }
 }
 
 void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
   auto lhsLoc{std::get<parser::Variable>(x.t).GetSource()};
   const auto &scope{context_.FindScope(lhsLoc)};
   const Scope &progUnit{GetProgramUnitContaining(scope)};
-  if (IsCUDADeviceContext(&progUnit) || inCUFKernelDoConstruct_) {
+  if (IsCUDADeviceContext(&progUnit) || deviceConstructDepth_ > 0) {
     return; // Data transfer with assignment is only perform on host.
   }
 
@@ -714,6 +763,15 @@ void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
     context_.Say(lhsLoc,
         "More than one reference to a CUDA object on the right hand side of the assigment"_err_en_US);
   }
+
+  if (Fortran::evaluate::HasCUDADeviceAttrs(assign->lhs) &&
+      Fortran::evaluate::HasCUDAImplicitTransfer(assign->rhs)) {
+    if (GetNbOfCUDAManagedOrUnifiedSymbols(assign->lhs) == 1 &&
+        GetNbOfCUDAManagedOrUnifiedSymbols(assign->rhs) == 1 &&
+        GetNbOfCUDADeviceSymbols(assign->rhs) == 1)
+      return; // This is a special case handled on the host.
+    context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
+  }
 }
 
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-cuda.h b/flang/lib/Semantics/check-cuda.h
index 222a2ee04b57c..10000253ffe5a 100644
--- a/flang/lib/Semantics/check-cuda.h
+++ b/flang/lib/Semantics/check-cuda.h
@@ -41,10 +41,18 @@ class CUDAChecker : public virtual BaseChecker {
   void Enter(const parser::CUFKernelDoConstruct &);
   void Leave(const parser::CUFKernelDoConstruct &);
   void Enter(const parser::AssignmentStmt &);
+  void Enter(const parser::OpenACCBlockConstruct &);
+  void Leave(const parser::OpenACCBlockConstruct &);
+  void Enter(const parser::OpenACCCombinedConstruct &);
+  void Leave(const parser::OpenACCCombinedConstruct &);
+  void Enter(const parser::OpenACCLoopConstruct &);
+  void Leave(const parser::OpenACCLoopConstruct &);
+  void Enter(const parser::DoConstruct &);
+  void Leave(const parser::DoConstruct &);
 
 private:
   SemanticsContext &context_;
-  bool inCUFKernelDoConstruct_ = false;
+  int deviceConstructDepth_{0};
 };
 
 bool CanonicalizeCUDA(parser::Program &);
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index f6e6272cbcd7b..ed41c6fb16892 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -197,11 +197,11 @@ static void WarnUndefinedFunctionResult(
 
 using StatementSemanticsPass1 = ExprChecker;
 using StatementSemanticsPass2 = SemanticsVisitor<AllocateChecker,
-    ArithmeticIfStmtChecker, CaseChecker, CoarrayChecker, DataChecker,
-    DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker, MiscChecker,
-    NamelistChecker, NullifyChecker, PurityChecker, ReturnStmtChecker,
-    SelectRankConstructChecker, SelectTypeChecker, StopChecker>;
-using StatementSemanticsPass3 = SemanticsVisitor<AssignmentChecker>;
+    ArithmeticIfStmtChecker, AssignmentChecker, CaseChecker, CoarrayChecker,
+    DataChecker, DeallocateChecker, DoForallChecker, IfStmtChecker, IoChecker,
+    MiscChecker, NamelistChecker, NullifyChecker, PurityChecker,
+    ReturnStmtChecker, SelectRankConstructChecker, SelectTypeChecker,
+    StopChecker>;
 
 static bool PerformStatementSemantics(
     SemanticsContext &context, parser::Program &program) {
@@ -212,7 +212,6 @@ static bool PerformStatementSemantics(
   StatementSemanticsPass1{context}.Walk(program);
   StatementSemanticsPass2 pass2{context};
   pass2.Walk(program);
-  StatementSemanticsPass3{context}.Walk(program);
   if (context.languageFeatures().IsEnabled(common::LanguageFeature::OpenACC)) {
     SemanticsVisitor<AccStructureChecker>{context}.Walk(program);
   }

>From bbdd46de5f2a0fd741ac7cd5ef72e27a15208a77 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Mon, 7 Jul 2025 21:38:01 -0700
Subject: [PATCH 3/4] Rename language feature

---
 flang/include/flang/Optimizer/Builder/CUFCommon.h |  3 ++-
 flang/include/flang/Support/Fortran-features.h    |  2 +-
 flang/lib/Lower/Bridge.cpp                        |  5 +++--
 flang/lib/Optimizer/Builder/CUFCommon.cpp         |  6 ++++--
 flang/lib/Semantics/check-cuda.cpp                |  4 ++--
 flang/tools/bbc/bbc.cpp                           | 12 +++++++-----
 6 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/flang/include/flang/Optimizer/Builder/CUFCommon.h b/flang/include/flang/Optimizer/Builder/CUFCommon.h
index cfd8a2305d031..5c56dd6b695f8 100644
--- a/flang/include/flang/Optimizer/Builder/CUFCommon.h
+++ b/flang/include/flang/Optimizer/Builder/CUFCommon.h
@@ -27,7 +27,8 @@ mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
                                             mlir::SymbolTable &symTab);
 
 bool isCUDADeviceContext(mlir::Operation *op);
-bool isCUDADeviceContext(mlir::Region &, bool isStdParEnabled = false);
+bool isCUDADeviceContext(mlir::Region &,
+                         bool isDoConcurrentOffloadEnabled = false);
 bool isRegisteredDeviceGlobal(fir::GlobalOp op);
 bool isRegisteredDeviceAttr(std::optional<cuf::DataAttribute> attr);
 
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 393ee7d0687c6..857de9479e4e5 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction, StdPar)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 1be64ea26d7ea..696473605a4e0 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4887,8 +4887,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     fir::FirOpBuilder &builder = getFirOpBuilder();
 
     bool isInDeviceContext = cuf::isCUDADeviceContext(
-        builder.getRegion(), getFoldingContext().languageFeatures().IsEnabled(
-                                 Fortran::common::LanguageFeature::StdPar));
+        builder.getRegion(),
+        getFoldingContext().languageFeatures().IsEnabled(
+            Fortran::common::LanguageFeature::DoConcurrentOffload));
 
     bool isCUDATransfer =
         IsCUDADataTransfer(assign.lhs, assign.rhs) && !isInDeviceContext;
diff --git a/flang/lib/Optimizer/Builder/CUFCommon.cpp b/flang/lib/Optimizer/Builder/CUFCommon.cpp
index d40ae70503350..dcbf4991907bf 100644
--- a/flang/lib/Optimizer/Builder/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Builder/CUFCommon.cpp
@@ -43,7 +43,8 @@ bool cuf::isCUDADeviceContext(mlir::Operation *op) {
 // for it.
 // If the insertion point is inside an OpenACC region op, it is considered
 // device context.
-bool cuf::isCUDADeviceContext(mlir::Region &region, bool isStdParEnabled) {
+bool cuf::isCUDADeviceContext(mlir::Region &region,
+                              bool isDoConcurrentOffloadEnabled) {
   if (region.getParentOfType<cuf::KernelOp>())
     return true;
   if (region.getParentOfType<mlir::acc::ComputeRegionOpInterface>())
@@ -56,7 +57,8 @@ bool cuf::isCUDADeviceContext(mlir::Region &region, bool isStdParEnabled) {
              cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
     }
   }
-  if (isStdParEnabled && region.getParentOfType<fir::DoConcurrentLoopOp>())
+  if (isDoConcurrentOffloadEnabled &&
+      region.getParentOfType<fir::DoConcurrentLoopOp>())
     return true;
   return false;
 }
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index c399fdb5a48de..8103402598f5f 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -729,14 +729,14 @@ void CUDAChecker::Leave(const parser::OpenACCLoopConstruct &) {
 void CUDAChecker::Enter(const parser::DoConstruct &x) {
   if (x.IsDoConcurrent() &&
       context_.foldingContext().languageFeatures().IsEnabled(
-          common::LanguageFeature::StdPar)) {
+          common::LanguageFeature::DoConcurrentOffload)) {
     ++deviceConstructDepth_;
   }
 }
 void CUDAChecker::Leave(const parser::DoConstruct &x) {
   if (x.IsDoConcurrent() &&
       context_.foldingContext().languageFeatures().IsEnabled(
-          common::LanguageFeature::StdPar)) {
+          common::LanguageFeature::DoConcurrentOffload)) {
     --deviceConstructDepth_;
   }
 }
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 25d072c93f22c..d216069654bc4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -223,9 +223,10 @@ static llvm::cl::opt<bool> enableCUDA("fcuda",
                                       llvm::cl::desc("enable CUDA Fortran"),
                                       llvm::cl::init(false));
 
-static llvm::cl::opt<bool> enableStdPar("stdpar",
-                                        llvm::cl::desc("enable stdpar"),
-                                        llvm::cl::init(false));
+static llvm::cl::opt<bool>
+    enableDoConcurrentOffload("fdoconcurrent-offload",
+                              llvm::cl::desc("enable do concurrent offload"),
+                              llvm::cl::init(false));
 
 static llvm::cl::opt<bool>
     disableCUDAWarpFunction("fcuda-disable-warp-function",
@@ -612,8 +613,9 @@ int main(int argc, char **argv) {
     options.features.Enable(Fortran::common::LanguageFeature::CUDA);
   }
 
-  if (enableStdPar) {
-    options.features.Enable(Fortran::common::LanguageFeature::StdPar);
+  if (enableDoConcurrentOffload) {
+    options.features.Enable(
+        Fortran::common::LanguageFeature::DoConcurrentOffload);
   }
 
   if (disableCUDAWarpFunction) {

>From eb2da92eed771127559701d59a270f08e6bd895c Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Mon, 7 Jul 2025 21:58:12 -0700
Subject: [PATCH 4/4] Revert code

---
 flang/lib/Semantics/assignment.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/lib/Semantics/assignment.h b/flang/lib/Semantics/assignment.h
index 54b3be2da06e8..ba537744bfaaa 100644
--- a/flang/lib/Semantics/assignment.h
+++ b/flang/lib/Semantics/assignment.h
@@ -21,7 +21,6 @@ struct MaskedElsewhereStmt;
 struct PointerAssignmentStmt;
 struct WhereConstructStmt;
 struct WhereStmt;
-struct DoConstruct;
 } // namespace Fortran::parser
 
 namespace Fortran::semantics {


From flang-commits at lists.llvm.org  Mon Jul  7 23:38:02 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Mon, 07 Jul 2025 23:38:02 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686cbcca.630a0220.125127.0bbb@mx.google.com>


================
@@ -3409,3 +3409,30 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
 
   return Out;
 }
+
+std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
----------------
s-watanabe314 wrote:

Thank you for the review. I have corrected the function names.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Tue Jul  8 00:52:39 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 00:52:39 -0700 (PDT)
Subject: [flang-commits] [flang] d889a74 - [flang] Avoid UB in CharBlock
	Compare to C string (#147329)
Message-ID: <686cce47.170a0220.2f9934.b678@mx.google.com>


Author: David Spickett
Date: 2025-07-08T08:52:36+01:00
New Revision: d889a7485f77c214ee57d0a74bf164237fdfa423

URL: https://github.com/llvm/llvm-project/commit/d889a7485f77c214ee57d0a74bf164237fdfa423
DIFF: https://github.com/llvm/llvm-project/commit/d889a7485f77c214ee57d0a74bf164237fdfa423.diff

LOG: [flang] Avoid UB in CharBlock Compare to C string (#147329)

The behaviour of strncmp is undefined if either string pointer is null
(https://en.cppreference.com/w/cpp/string/byte/strncmp.html).

I've copied the logic over from Compare to another CharBlock, which had
code to avoid UB in memcmp.

The test Preprocessing/kind-suffix.F90 was failing with UBSAN enabled,
and now passes.

Added: 
    

Modified: 
    flang/include/flang/Parser/char-block.h

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/char-block.h b/flang/include/flang/Parser/char-block.h
index 38f4f7b82e1ea..b3b1f04034d3c 100644
--- a/flang/include/flang/Parser/char-block.h
+++ b/flang/include/flang/Parser/char-block.h
@@ -150,7 +150,12 @@ class CharBlock {
 
   int Compare(const char *that) const {
     std::size_t bytes{size()};
-    if (int cmp{std::strncmp(begin(), that, bytes)}) {
+    // strncmp is undefined if either pointer is null.
+    if (!bytes) {
+      return that == nullptr ? 0 : -1;
+    } else if (!that) {
+      return 1;
+    } else if (int cmp{std::strncmp(begin(), that, bytes)}) {
       return cmp;
     }
     return that[bytes] == '\0' ? 0 : -1;


        

From flang-commits at lists.llvm.org  Tue Jul  8 00:52:42 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 00:52:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid UB in CharBlock Compare to C
	string (PR #147329)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147329@github.com>
Message-ID: <686cce4a.630a0220.26c885.c82c@mx.google.com>


https://github.com/DavidSpickett closed https://github.com/llvm/llvm-project/pull/147329

From flang-commits at lists.llvm.org  Mon Jul  7 23:34:36 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Mon, 07 Jul 2025 23:34:36 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686cbbfc.170a0220.db54d.b34d@mx.google.com>


https://github.com/s-watanabe314 updated https://github.com/llvm/llvm-project/pull/146641

>From baf87a6002d740868daeeba0f89d55c05e85b712 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 18 Apr 2025 09:04:07 +0900
Subject: [PATCH 1/7] [flang] Add -fcomplex-arithmetic= option and select
 complex division algorithm

This patch adds an option to select the method for computing complex
number division. It uses `LoweringOptions` to determine whether to lower
complex division to a runtime function call or to MLIR's `complex.div`,
and `CodeGenOptions` to select the computation algorithm for
`complex.div`. The available option values and their corresponding
algorithms are as follows:
- `full`: Lower to a runtime function call. (Default behavior)
- `improved`: Lower to `complex.div` and expand to Smith's algorithm.
- `basic`: Lower to `complex.div` and expand to the algebraic algorithm.

See also the discussion in the following discourse post:
https://discourse.llvm.org/t/optimization-of-complex-number-division/83468
---
 clang/include/clang/Driver/Options.td         |   4 +-
 clang/lib/Driver/ToolChains/Flang.cpp         |  47 ++
 .../include/flang/Frontend/CodeGenOptions.def |   1 +
 flang/include/flang/Frontend/CodeGenOptions.h |  25 +
 flang/include/flang/Lower/LoweringOptions.def |   4 +
 .../flang/Optimizer/Builder/FIRBuilder.h      |  15 +
 .../include/flang/Optimizer/CodeGen/CodeGen.h |   6 +
 flang/include/flang/Tools/CrossToolHelpers.h  |   3 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  21 +
 flang/lib/Frontend/FrontendActions.cpp        |   2 +
 flang/lib/Lower/Bridge.cpp                    |   2 +
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |  12 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  15 +-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |   1 +
 flang/test/Driver/complex-range.f90           |  23 +
 .../complex-div-to-llvm-kind10.f90            | 131 +++++
 .../complex-div-to-llvm-kind16.f90            | 131 +++++
 .../test/Integration/complex-div-to-llvm.f90  | 508 ++++++++++++++++++
 .../HLFIR/complex-div-to-hlfir-kind10.f90     |  34 ++
 .../HLFIR/complex-div-to-hlfir-kind16.f90     |  35 ++
 .../test/Lower/HLFIR/complex-div-to-hlfir.f90 |  94 ++++
 flang/tools/bbc/bbc.cpp                       |   8 +
 22 files changed, 1117 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Driver/complex-range.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm-kind10.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm-kind16.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..58209ceb5dc54 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
 
 def complex_range_EQ : Joined<["-"], "complex-range=">, Group<f_Group>,
-  Visibility<[CC1Option]>,
+  Visibility<[CC1Option, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>,
   MarshallingInfoEnum<LangOpts<"ComplexRange">, "CX_Full">;
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index e4e321ba1e195..ba5db7e3aba43 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -595,6 +595,30 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
+static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  default:
+    return "";
+  }
+}
+
+static std::string
+RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
+
 static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   StringRef FPContract;
@@ -605,6 +629,8 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
   bool AssociativeMath = false;
   bool ReciprocalMath = false;
 
+  LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_None;
+
   if (const Arg *A = Args.getLastArg(options::OPT_ffp_contract)) {
     const StringRef Val = A->getValue();
     if (Val == "fast" || Val == "off") {
@@ -629,6 +655,20 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
     default:
       continue;
 
+    case options::OPT_fcomplex_arithmetic_EQ: {
+      StringRef Val = A->getValue();
+      if (Val == "full")
+        Range = LangOptions::ComplexRangeKind::CX_Full;
+      else if (Val == "improved")
+        Range = LangOptions::ComplexRangeKind::CX_Improved;
+      else if (Val == "basic")
+        Range = LangOptions::ComplexRangeKind::CX_Basic;
+      else {
+        D.Diag(diag::err_drv_unsupported_option_argument)
+            << A->getSpelling() << Val;
+      }
+      break;
+    }
     case options::OPT_fhonor_infinities:
       HonorINFs = true;
       break;
@@ -699,6 +739,13 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
   if (!Recip.empty())
     CmdArgs.push_back(Args.MakeArgString("-mrecip=" + Recip));
 
+  if (Range != LangOptions::ComplexRangeKind::CX_None) {
+    std::string ComplexRangeStr = RenderComplexRangeOption(Range);
+    CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
+    CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" +
+                                         ComplexRangeKindToStr(Range)));
+  }
+
   if (!HonorINFs && !HonorNaNs && AssociativeMath && ReciprocalMath &&
       ApproxFunc && !SignedZeros &&
       (FPContract == "fast" || FPContract.empty())) {
diff --git a/flang/include/flang/Frontend/CodeGenOptions.def b/flang/include/flang/Frontend/CodeGenOptions.def
index ae12aec518108..cdeea93c9aecb 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.def
+++ b/flang/include/flang/Frontend/CodeGenOptions.def
@@ -52,6 +52,7 @@ ENUM_CODEGENOPT(RelocationModel, llvm::Reloc::Model, 3, llvm::Reloc::PIC_) ///<
 ENUM_CODEGENOPT(DebugInfo,  llvm::codegenoptions::DebugInfoKind, 4,  llvm::codegenoptions::NoDebugInfo) ///< Level of debug info to generate
 ENUM_CODEGENOPT(VecLib, llvm::driver::VectorLibrary, 4, llvm::driver::VectorLibrary::NoLibrary) ///< Vector functions library to use
 ENUM_CODEGENOPT(FramePointer, llvm::FramePointerKind, 2, llvm::FramePointerKind::None) ///< Enable the usage of frame pointers
+ENUM_CODEGENOPT(ComplexRange, ComplexRangeKind, 3, ComplexRangeKind::CX_Full) ///< Method for calculating complex number division
 
 ENUM_CODEGENOPT(DoConcurrentMapping, DoConcurrentMappingKind, 2, DoConcurrentMappingKind::DCMK_None) ///< Map `do concurrent` to OpenMP
 
diff --git a/flang/include/flang/Frontend/CodeGenOptions.h b/flang/include/flang/Frontend/CodeGenOptions.h
index bad17c8309eb8..df6063cc90340 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.h
+++ b/flang/include/flang/Frontend/CodeGenOptions.h
@@ -192,6 +192,31 @@ class CodeGenOptions : public CodeGenOptionsBase {
     return getProfileUse() == llvm::driver::ProfileCSIRInstr;
   }
 
+  /// Controls the various implementations for complex division.
+  enum ComplexRangeKind {
+    /// Implementation of complex division using a call to runtime library
+    /// functions. Overflow and non-finite values are handled by the library
+    /// implementation. This is the default value.
+    CX_Full,
+
+    /// Implementation of complex division offering an improved handling
+    /// for overflow in intermediate calculations. Overflow and non-finite
+    /// values are handled by MLIR's implementation of "complex.div", but this
+    /// may change in the future.
+    CX_Improved,
+
+    /// Implementation of complex division using algebraic formulas at source
+    /// precision. No special handling to avoid overflow. NaN and infinite
+    /// values are not handled.
+    CX_Basic,
+
+    /// No range rule is enabled.
+    CX_None
+
+    /// TODO: Implemention of other values as needed. In Clang, "CX_Promoted"
+    /// is implemented. (See clang/Basic/LangOptions.h)
+  };
+
   // Define accessors/mutators for code generation options of enumeration type.
 #define CODEGENOPT(Name, Bits, Default)
 #define ENUM_CODEGENOPT(Name, Type, Bits, Default)                             \
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index 3263ab129d076..8135704971aa4 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -70,5 +70,9 @@ ENUM_LOWERINGOPT(CUDARuntimeCheck, unsigned, 1, 0)
 /// derived types defined in other compilation units.
 ENUM_LOWERINGOPT(SkipExternalRttiDefinition, unsigned, 1, 0)
 
+/// If true, convert complex number division to runtime on the frontend.
+/// If false, lower to the complex dialect of MLIR.
+/// On by default.
+ENUM_LOWERINGOPT(ComplexDivisionToRuntime, unsigned, 1, 1)
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index e1eaab3346901..b1513850a9048 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -609,6 +609,17 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return integerOverflowFlags;
   }
 
+  /// Set ComplexDivisionToRuntimeFlag value for whether complex number division
+  /// is lowered to a runtime function by this builder.
+  void setComplexDivisionToRuntimeFlag(bool flag) {
+    complexDivisionToRuntimeFlag = flag;
+  }
+
+  /// Get current ComplexDivisionToRuntimeFlag value.
+  bool getComplexDivisionToRuntimeFlag() const {
+    return complexDivisionToRuntimeFlag;
+  }
+
   /// Dump the current function. (debug)
   LLVM_DUMP_METHOD void dumpFunc();
 
@@ -673,6 +684,10 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   /// mlir::arith::IntegerOverflowFlagsAttr.
   mlir::arith::IntegerOverflowFlags integerOverflowFlags{};
 
+  /// Flag to control whether complex number division is lowered to a runtime
+  /// function or to the MLIR complex dialect.
+  bool complexDivisionToRuntimeFlag = true;
+
   /// fir::GlobalOp and func::FuncOp symbol table to speed-up
   /// lookups.
   mlir::SymbolTable *symbolTable = nullptr;
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 93f07d8d5d4d9..e9a07a8dde5cd 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 #define FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 
+#include "flang/Frontend/CodeGenOptions.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassRegistry.h"
@@ -58,6 +59,11 @@ struct FIRToLLVMPassOptions {
   // the name of the global variable corresponding to a derived
   // type's descriptor.
   bool typeDescriptorsRenamedForAssembly = false;
+
+  // Specify the calculation method for complex number division used by the
+  // Conversion pass of the MLIR complex dialect.
+  Fortran::frontend::CodeGenOptions::ComplexRangeKind ComplexRange =
+      Fortran::frontend::CodeGenOptions::ComplexRangeKind::CX_Full;
 };
 
 /// Convert FIR to the LLVM IR dialect with default options.
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 337685c82af5f..df1da27058552 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -140,6 +140,9 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   std::string InstrumentFunctionExit =
       ""; ///< Name of the instrument-function that is called on each
           ///< function-exit
+  Fortran::frontend::CodeGenOptions::ComplexRangeKind ComplexRange =
+      Fortran::frontend::CodeGenOptions::ComplexRangeKind::
+          CX_Full; ///< Method for calculating complex number division
 };
 
 struct OffloadModuleOpts {
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..86ec410b1f70f 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -484,6 +484,21 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
   }
 
   parseDoConcurrentMapping(opts, args, diags);
+
+  if (const auto *arg =
+          args.getLastArg(clang::driver::options::OPT_complex_range_EQ)) {
+    llvm::StringRef argValue = llvm::StringRef(arg->getValue());
+    if (argValue == "full") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Full);
+    } else if (argValue == "improved") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Improved);
+    } else if (argValue == "basic") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Basic);
+    } else {
+      diags.Report(clang::diag::err_drv_invalid_value)
+          << arg->getAsString(args) << arg->getValue();
+    }
+  }
 }
 
 /// Parses all target input arguments and populates the target
@@ -1811,4 +1826,10 @@ void CompilerInvocation::setLoweringOptions() {
       .setNoSignedZeros(langOptions.NoSignedZeros)
       .setAssociativeMath(langOptions.AssociativeMath)
       .setReciprocalMath(langOptions.ReciprocalMath);
+
+  if (codegenOpts.getComplexRange() ==
+          CodeGenOptions::ComplexRangeKind::CX_Improved ||
+      codegenOpts.getComplexRange() ==
+          CodeGenOptions::ComplexRangeKind::CX_Basic)
+    loweringOpts.setComplexDivisionToRuntime(false);
 }
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index bf15def3f3b2e..b5f4f9421f633 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -750,6 +750,8 @@ void CodeGenAction::generateLLVMIR() {
   if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
     config.NSWOnLoopVarInc = false;
 
+  config.ComplexRange = opts.getComplexRange();
+
   // Create the pass pipeline
   fir::createMLIRToLLVMPassPipeline(pm, config, getCurrentFile());
   (void)mlir::applyPassManagerCLOptions(pm);
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..7e95c640e73b0 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5746,6 +5746,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     builder =
         new fir::FirOpBuilder(func, bridge.getKindMap(), &mlirSymbolTable);
     assert(builder && "FirOpBuilder did not instantiate");
+    builder->setComplexDivisionToRuntimeFlag(
+        bridge.getLoweringOptions().getComplexDivisionToRuntime());
     builder->setFastMathFlags(bridge.getLoweringOptions().getMathOptions());
     builder->setInsertionPointToStart(&func.front());
     if (funit.parent.isA<Fortran::lower::pft::FunctionLikeUnit>()) {
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index df8dfbc72c030..cb338618dbf3b 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1066,8 +1066,16 @@ struct BinaryOp<Fortran::evaluate::Divide<
     mlir::Type ty = Fortran::lower::getFIRType(
         builder.getContext(), Fortran::common::TypeCategory::Complex, KIND,
         /*params=*/{});
-    return hlfir::EntityWithAttributes{
-        fir::genDivC(builder, loc, ty, lhs, rhs)};
+
+    // TODO: Ideally, complex number division operations should always be
+    // lowered to MLIR. However, converting them to the runtime via MLIR causes
+    // ABI issues.
+    if (builder.getComplexDivisionToRuntimeFlag())
+      return hlfir::EntityWithAttributes{
+          fir::genDivC(builder, loc, ty, lhs, rhs)};
+    else
+      return hlfir::EntityWithAttributes{
+          builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
   }
 };
 
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 2b018912b40e4..4cf533f195e69 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -4119,7 +4119,20 @@ class FIRToLLVMLowering
     mathToFuncsOptions.minWidthOfFPowIExponent = 33;
     mathConvertionPM.addPass(
         mlir::createConvertMathToFuncs(mathToFuncsOptions));
-    mathConvertionPM.addPass(mlir::createConvertComplexToStandardPass());
+
+    mlir::ConvertComplexToStandardPassOptions complexToStandardOptions{};
+    if (options.ComplexRange ==
+        Fortran::frontend::CodeGenOptions::ComplexRangeKind::CX_Basic) {
+      complexToStandardOptions.complexRange =
+          mlir::complex::ComplexRangeFlags::basic;
+    } else if (options.ComplexRange == Fortran::frontend::CodeGenOptions::
+                                           ComplexRangeKind::CX_Improved) {
+      complexToStandardOptions.complexRange =
+          mlir::complex::ComplexRangeFlags::improved;
+    }
+    mathConvertionPM.addPass(
+        mlir::createConvertComplexToStandardPass(complexToStandardOptions));
+
     // Convert Math dialect operations into LLVM dialect operations.
     // There is no way to prefer MathToLLVM patterns over MathToLibm
     // patterns (applied below), so we have to run MathToLLVM conversion here.
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 42d9e7ba2418f..d940934f9f6ad 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -113,6 +113,7 @@ void addFIRToLLVMPass(mlir::PassManager &pm,
   options.forceUnifiedTBAATree = useOldAliasTags;
   options.typeDescriptorsRenamedForAssembly =
       !disableCompilerGeneratedNamesConversion;
+  options.ComplexRange = config.ComplexRange;
   addPassConditionally(pm, disableFirToLlvmIr,
                        [&]() { return fir::createFIRToLLVMPass(options); });
   // The dialect conversion framework may leave dead unrealized_conversion_cast
diff --git a/flang/test/Driver/complex-range.f90 b/flang/test/Driver/complex-range.f90
new file mode 100644
index 0000000000000..e5a1ba9068ac9
--- /dev/null
+++ b/flang/test/Driver/complex-range.f90
@@ -0,0 +1,23 @@
+! Test range options for complex multiplication and division.
+
+! RUN: %flang -### -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=RANGE
+
+! RUN: %flang -### -fcomplex-arithmetic=full -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=FULL
+
+! RUN: %flang -### -fcomplex-arithmetic=improved -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=IMPRVD
+
+! RUN: %flang -### -fcomplex-arithmetic=basic -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=BASIC
+
+! RUN: not %flang -### -fcomplex-arithmetic=foo -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=ERR
+
+! RANGE-NOT: -complex-range=
+! FULL: -complex-range=full
+! IMPRVD: -complex-range=improved
+! BASIC: -complex-range=basic
+
+! ERR: error: unsupported argument 'foo' to option '-fcomplex-arithmetic='
diff --git a/flang/test/Integration/complex-div-to-llvm-kind10.f90 b/flang/test/Integration/complex-div-to-llvm-kind10.f90
new file mode 100644
index 0000000000000..04d1f7ed9b024
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm-kind10.f90
@@ -0,0 +1,131 @@
+! Test lowering complex division to llvm ir according to options
+
+! REQUIRES: target=x86_64{{.*}}
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_extended
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { x86_fp80, x86_fp80 }, ptr %[[LHS]], align 16
+! CHECK: %[[LOAD_RHS:.*]] = load { x86_fp80, x86_fp80 }, ptr %[[RHS]], align 16
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract x86_fp80 %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract x86_fp80 %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract x86_fp80 %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract x86_fp80 %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract x86_fp80 %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract x86_fp80 %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract x86_fp80 %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract x86_fp80 %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract x86_fp80 %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract x86_fp80 %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract x86_fp80 %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract x86_fp80 %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq x86_fp80 %[[RHS_REAL_ABS]], 0xK00000000000000000000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq x86_fp80 %[[RHS_IMAG_ABS]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord x86_fp80 %[[LHS_REAL]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord x86_fp80 %[[LHS_IMAG]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 0xK7FFF8000000000000000, x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract x86_fp80 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract x86_fp80 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one x86_fp80 %[[RHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one x86_fp80 %[[RHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq x86_fp80 %[[LHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq x86_fp80 %[[LHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[LHS_REAL_IS_INF]], x86_fp80 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[LHS_IMAG_IS_INF]], x86_fp80 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract x86_fp80 %[[INF_MULTIPLICATOR_1]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract x86_fp80 %[[INF_MULTIPLICATOR_2]], 0xK7FFF8000000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one x86_fp80 %[[LHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one x86_fp80 %[[LHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq x86_fp80 %[[RHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq x86_fp80 %[[RHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[RHS_REAL_IS_INF]], x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[RHS_IMAG_IS_INF]], x86_fp80 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract x86_fp80 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract x86_fp80 %[[ZERO_MULTIPLICATOR_1]], 0xK00000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract x86_fp80 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract x86_fp80 %[[ZERO_MULTIPLICATOR_2]], 0xK00000000000000000000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt x86_fp80 %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], x86_fp80 %[[RESULT_REAL_1]], x86_fp80 %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], x86_fp80 %[[RESULT_IMAG_1]], x86_fp80 %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], x86_fp80 %[[RESULT_REAL_4]], x86_fp80 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], x86_fp80 %[[RESULT_IMAG_4]], x86_fp80 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], x86_fp80 %[[RESULT_REAL_3]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], x86_fp80 %[[RESULT_IMAG_3]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], x86_fp80 %[[INFINITY_RESULT_REAL]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], x86_fp80 %[[INFINITY_RESULT_IMAG]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno x86_fp80 %[[RESULT_REAL]], 0xK00000000000000000000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno x86_fp80 %[[RESULT_IMAG]], 0xK00000000000000000000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_1]], x86_fp80 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_1]], x86_fp80 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { x86_fp80, x86_fp80 } %[[RESULT_1]], x86_fp80 %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { x86_fp80, x86_fp80 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract x86_fp80 %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract x86_fp80 %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract x86_fp80 %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract x86_fp80 %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract x86_fp80 %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract x86_fp80 %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract x86_fp80 %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { x86_fp80, x86_fp80 } %[[RESULT_1]], x86_fp80 %[[IMAG]], 1
+! BASIC: store { x86_fp80, x86_fp80 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! CHECK: ret void
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
diff --git a/flang/test/Integration/complex-div-to-llvm-kind16.f90 b/flang/test/Integration/complex-div-to-llvm-kind16.f90
new file mode 100644
index 0000000000000..887a7972dc1be
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm-kind16.f90
@@ -0,0 +1,131 @@
+! Test lowering complex division to llvm ir according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_quad
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { fp128, fp128 }, ptr %[[LHS]], align 16
+! CHECK: %[[LOAD_RHS:.*]] = load { fp128, fp128 }, ptr %[[RHS]], align 16
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { fp128, fp128 } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { fp128, fp128 } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { fp128, fp128 } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { fp128, fp128 } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract fp128 %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract fp128 %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract fp128 %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract fp128 %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract fp128 %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract fp128 %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract fp128 %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract fp128 %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract fp128 %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract fp128 %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract fp128 %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract fp128 %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq fp128 %[[RHS_REAL_ABS]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord fp128 %[[LHS_REAL]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord fp128 %[[LHS_IMAG]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call fp128 @llvm.copysign.f128(fp128 0xL00000000000000007FFF000000000000, fp128 %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract fp128 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract fp128 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one fp128 %[[RHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq fp128 %[[LHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq fp128 %[[LHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[LHS_REAL_IS_INF]], fp128 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[LHS_IMAG_IS_INF]], fp128 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract fp128 %[[INF_MULTIPLICATOR_1]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract fp128 %[[INF_MULTIPLICATOR_2]], 0xL00000000000000007FFF000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one fp128 %[[LHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one fp128 %[[LHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq fp128 %[[RHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[RHS_REAL_IS_INF]], fp128 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[RHS_IMAG_IS_INF]], fp128 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract fp128 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract fp128 %[[ZERO_MULTIPLICATOR_1]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract fp128 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract fp128 %[[ZERO_MULTIPLICATOR_2]], 0xL00000000000000000000000000000000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt fp128 %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], fp128 %[[RESULT_REAL_1]], fp128 %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], fp128 %[[RESULT_IMAG_1]], fp128 %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], fp128 %[[RESULT_REAL_4]], fp128 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], fp128 %[[RESULT_IMAG_4]], fp128 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], fp128 %[[RESULT_REAL_3]], fp128 %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], fp128 %[[RESULT_IMAG_3]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], fp128 %[[INFINITY_RESULT_REAL]], fp128 %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], fp128 %[[INFINITY_RESULT_IMAG]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno fp128 %[[RESULT_REAL]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno fp128 %[[RESULT_IMAG]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], fp128 %[[RESULT_REAL_SPECIAL_CASE_1]], fp128 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_1]], fp128 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { fp128, fp128 } poison, fp128 %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { fp128, fp128 } %[[RESULT_1]], fp128 %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { fp128, fp128 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract fp128 %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract fp128 %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract fp128 %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract fp128 %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract fp128 %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract fp128 %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract fp128 %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { fp128, fp128 } poison, fp128 %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { fp128, fp128 } %[[RESULT_1]], fp128 %[[IMAG]], 1
+! BASIC: store { fp128, fp128 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! CHECK: ret void
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
diff --git a/flang/test/Integration/complex-div-to-llvm.f90 b/flang/test/Integration/complex-div-to-llvm.f90
new file mode 100644
index 0000000000000..01782a565f24c
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm.f90
@@ -0,0 +1,508 @@
+! Test lowering complex division to llvm ir according to options
+
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_half
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { half, half }, ptr %[[LHS]], align 2
+! CHECK: %[[LOAD_RHS:.*]] = load { half, half }, ptr %[[RHS]], align 2
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { half, half } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { half, half } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { half, half } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { half, half } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract half %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract half %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract half %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract half %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract half %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract half %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract half %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract half %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract half %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract half %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract half %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract half %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq half %[[RHS_REAL_ABS]], 0xH0000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq half %[[RHS_IMAG_ABS]], 0xH0000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord half %[[LHS_REAL]], 0xH0000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord half %[[LHS_IMAG]], 0xH0000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call half @llvm.copysign.f16(half 0xH7C00, half %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract half %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract half %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one half %[[RHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one half %[[RHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq half %[[LHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq half %[[LHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[LHS_REAL_IS_INF]], half %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[LHS_IMAG_IS_INF]], half %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract half %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract half %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract half %[[INF_MULTIPLICATOR_1]], 0xH7C00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract half %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract half %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract half %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract half %[[INF_MULTIPLICATOR_2]], 0xH7C00
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one half %[[LHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one half %[[LHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq half %[[RHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq half %[[RHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[RHS_REAL_IS_INF]], half %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[RHS_IMAG_IS_INF]], half %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract half %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract half %[[ZERO_MULTIPLICATOR_1]], 0xH0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract half %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract half %[[ZERO_MULTIPLICATOR_2]], 0xH0000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt half %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], half %[[RESULT_REAL_1]], half %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], half %[[RESULT_IMAG_1]], half %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], half %[[RESULT_REAL_4]], half %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], half %[[RESULT_IMAG_4]], half %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], half %[[RESULT_REAL_3]], half %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], half %[[RESULT_IMAG_3]], half %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], half %[[INFINITY_RESULT_REAL]], half %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], half %[[INFINITY_RESULT_IMAG]], half %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno half %[[RESULT_REAL]], 0xH0000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno half %[[RESULT_IMAG]], 0xH0000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], half %[[RESULT_REAL_SPECIAL_CASE_1]], half %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], half %[[RESULT_IMAG_SPECIAL_CASE_1]], half %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { half, half } poison, half %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { half, half } %[[RESULT_1]], half %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { half, half } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract half %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract half %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract half %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract half %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract half %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract half %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract half %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { half, half } poison, half %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { half, half } %[[RESULT_1]], half %[[IMAG]], 1
+! BASIC: store { half, half } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! CHECK: ret void
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+
+! CHECK-LABEL: @div_test_bfloat
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { bfloat, bfloat }, ptr %[[LHS]], align 2
+! CHECK: %[[LOAD_RHS:.*]] = load { bfloat, bfloat }, ptr %[[RHS]], align 2
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract bfloat %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract bfloat %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract bfloat %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract bfloat %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract bfloat %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract bfloat %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract bfloat %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract bfloat %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract bfloat %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract bfloat %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract bfloat %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract bfloat %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq bfloat %[[RHS_REAL_ABS]], 0xR0000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq bfloat %[[RHS_IMAG_ABS]], 0xR0000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord bfloat %[[LHS_REAL]], 0xR0000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord bfloat %[[LHS_IMAG]], 0xR0000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call bfloat @llvm.copysign.bf16(bfloat 0xR7F80, bfloat %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract bfloat %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract bfloat %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one bfloat %[[RHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one bfloat %[[RHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq bfloat %[[LHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq bfloat %[[LHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[LHS_REAL_IS_INF]], bfloat %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[LHS_IMAG_IS_INF]], bfloat %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract bfloat %[[INF_MULTIPLICATOR_1]], 0xR7F80
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract bfloat %[[INF_MULTIPLICATOR_2]], 0xR7F80
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one bfloat %[[LHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one bfloat %[[LHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq bfloat %[[RHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq bfloat %[[RHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[RHS_REAL_IS_INF]], bfloat %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[RHS_IMAG_IS_INF]], bfloat %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract bfloat %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract bfloat %[[ZERO_MULTIPLICATOR_1]], 0xR0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract bfloat %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract bfloat %[[ZERO_MULTIPLICATOR_2]], 0xR0000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt bfloat %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], bfloat %[[RESULT_REAL_1]], bfloat %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], bfloat %[[RESULT_IMAG_1]], bfloat %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], bfloat %[[RESULT_REAL_4]], bfloat %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], bfloat %[[RESULT_IMAG_4]], bfloat %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], bfloat %[[RESULT_REAL_3]], bfloat %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], bfloat %[[RESULT_IMAG_3]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], bfloat %[[INFINITY_RESULT_REAL]], bfloat %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], bfloat %[[INFINITY_RESULT_IMAG]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno bfloat %[[RESULT_REAL]], 0xR0000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno bfloat %[[RESULT_IMAG]], 0xR0000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], bfloat %[[RESULT_REAL_SPECIAL_CASE_1]], bfloat %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_1]], bfloat %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { bfloat, bfloat } poison, bfloat %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { bfloat, bfloat } %[[RESULT_1]], bfloat %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { bfloat, bfloat } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract bfloat %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract bfloat %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract bfloat %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract bfloat %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract bfloat %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract bfloat %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract bfloat %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { bfloat, bfloat } poison, bfloat %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { bfloat, bfloat } %[[RESULT_1]], bfloat %[[IMAG]], 1
+! BASIC: store { bfloat, bfloat } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! CHECK: ret void
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+
+! CHECK-LABEL: @div_test_single
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { float, float }, ptr %[[LHS]], align 4
+! CHECK: %[[LOAD_RHS:.*]] = load { float, float }, ptr %[[RHS]], align 4
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { float, float } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { float, float } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { float, float } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { float, float } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract float %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract float %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract float %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract float %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract float %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract float %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract float %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract float %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract float %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract float %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract float %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract float %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq float %[[RHS_REAL_ABS]], 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq float %[[RHS_IMAG_ABS]], 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord float %[[LHS_REAL]], 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord float %[[LHS_IMAG]], 0.000000e+00
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call float @llvm.copysign.f32(float 0x7FF0000000000000, float %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract float %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract float %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one float %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one float %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq float %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq float %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[LHS_REAL_IS_INF]], float %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[LHS_IMAG_IS_INF]], float %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract float %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract float %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract float %[[INF_MULTIPLICATOR_1]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract float %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract float %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract float %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract float %[[INF_MULTIPLICATOR_2]], 0x7FF0000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one float %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one float %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq float %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq float %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[RHS_REAL_IS_INF]], float %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[RHS_IMAG_IS_INF]], float %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract float %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract float %[[ZERO_MULTIPLICATOR_1]], 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract float %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract float %[[ZERO_MULTIPLICATOR_2]], 0.000000e+00
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt float %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], float %[[RESULT_REAL_1]], float %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], float %[[RESULT_IMAG_1]], float %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], float %[[RESULT_REAL_4]], float %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], float %[[RESULT_IMAG_4]], float %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], float %[[RESULT_REAL_3]], float %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], float %[[RESULT_IMAG_3]], float %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], float %[[INFINITY_RESULT_REAL]], float %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], float %[[INFINITY_RESULT_IMAG]], float %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno float %[[RESULT_REAL]], 0.000000e+00
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno float %[[RESULT_IMAG]], 0.000000e+00
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], float %[[RESULT_REAL_SPECIAL_CASE_1]], float %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], float %[[RESULT_IMAG_SPECIAL_CASE_1]], float %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { float, float } poison, float %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT_1]], float %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { float, float } %[[RESULT_2]], ptr %[[RET]], align 4
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract float %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract float %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract float %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract float %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract float %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract float %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract float %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { float, float } poison, float %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT_1]], float %[[IMAG]], 1
+! BASIC: store { float, float } %[[RESULT_2]], ptr %[[RET]], align 4
+
+! CHECK: ret void
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+
+! CHECK-LABEL: @div_test_double
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { double, double }, ptr %[[LHS]], align 8
+! CHECK: %[[LOAD_RHS:.*]] = load { double, double }, ptr %[[RHS]], align 8
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { double, double } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { double, double } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { double, double } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { double, double } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract double %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract double %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract double %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract double %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract double %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract double %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract double %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract double %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract double %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract double %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract double %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract double %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq double %[[RHS_REAL_ABS]], 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq double %[[RHS_IMAG_ABS]], 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord double %[[LHS_REAL]], 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord double %[[LHS_IMAG]], 0.000000e+00
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call double @llvm.copysign.f64(double 0x7FF0000000000000, double %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract double %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract double %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one double %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one double %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq double %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq double %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[LHS_REAL_IS_INF]], double %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[LHS_IMAG_IS_INF]], double %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract double %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract double %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract double %[[INF_MULTIPLICATOR_1]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract double %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract double %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract double %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract double %[[INF_MULTIPLICATOR_2]], 0x7FF0000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one double %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one double %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq double %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq double %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[RHS_REAL_IS_INF]], double %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[RHS_IMAG_IS_INF]], double %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract double %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract double %[[ZERO_MULTIPLICATOR_1]], 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract double %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract double %[[ZERO_MULTIPLICATOR_2]], 0.000000e+00
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt double %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], double %[[RESULT_REAL_1]], double %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], double %[[RESULT_IMAG_1]], double %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], double %[[RESULT_REAL_4]], double %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], double %[[RESULT_IMAG_4]], double %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], double %[[RESULT_REAL_3]], double %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], double %[[RESULT_IMAG_3]], double %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], double %[[INFINITY_RESULT_REAL]], double %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], double %[[INFINITY_RESULT_IMAG]], double %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno double %[[RESULT_REAL]], 0.000000e+00
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno double %[[RESULT_IMAG]], 0.000000e+00
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], double %[[RESULT_REAL_SPECIAL_CASE_1]], double %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], double %[[RESULT_IMAG_SPECIAL_CASE_1]], double %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { double, double } poison, double %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { double, double } %[[RESULT_1]], double %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { double, double } %[[RESULT_2]], ptr %[[RET]], align 8
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract double %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract double %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract double %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract double %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract double %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract double %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract double %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { double, double } poison, double %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { double, double } %[[RESULT_1]], double %[[IMAG]], 1
+! BASIC: store { double, double } %[[RESULT_2]], ptr %[[RET]], align 8
+
+! CHECK: ret void
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
new file mode 100644
index 0000000000000..0e219e30ec94b
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
@@ -0,0 +1,34 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: target=x86_64{{.*}}
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+! CHECK-LABEL: @_QPdiv_test_extended
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f80>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEa"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEb"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEc"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f80>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f80>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_RET:.*]] = fir.call @__divxc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f80, f80, f80, f80) -> complex<f80>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f80>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f80>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f80>, !fir.ref<complex<f80>>
+! CHECK: return
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
new file mode 100644
index 0000000000000..3e158b0bed998
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
@@ -0,0 +1,35 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_quad
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f128>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEa"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEb"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEc"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f128>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f128>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_RET:.*]] = fir.call @__divtc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f128, f128, f128, f128) -> complex<f128>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f128>, !fir.ref<complex<f128>>
+! CHECK: return
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
\ No newline at end of file
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90
new file mode 100644
index 0000000000000..b488bfde4ee85
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90
@@ -0,0 +1,94 @@
+! Test lowering of complex division according to options
+
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_half
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f16>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEa"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEb"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEc"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f16>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f16>>
+! CHECK: %[[VAL_9:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f16>
+! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : complex<f16>, !fir.ref<complex<f16>>
+! CHECK: return
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+
+! CHECK-LABEL: @_QPdiv_test_bfloat
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<bf16>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEa"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEb"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEc"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<bf16>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<bf16>>
+! CHECK: %[[VAL_9:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<bf16>
+! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : complex<bf16>, !fir.ref<complex<bf16>>
+! CHECK: return
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+
+! CHECK-LABEL: @_QPdiv_test_single
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f32>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEa"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEb"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEc"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f32>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_RET:.*]] = fir.call @__divsc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f32, f32, f32, f32) -> complex<f32>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f32>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f32>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f32>, !fir.ref<complex<f32>>
+! CHECK: return
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+
+! CHECK-LABEL: @_QPdiv_test_double
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f64>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEa"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEb"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEc"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f64>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f64>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_RET:.*]] = fir.call @__divdc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f64, f64, f64, f64) -> complex<f64>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f64>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f64>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f64>, !fir.ref<complex<f64>>
+! CHECK: return
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 59372a8eb58ed..2ccbe8b96d805 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -276,6 +276,12 @@ static llvm::cl::opt<bool>
                                      "leading dimension will be repacked"),
                       llvm::cl::init(true));
 
+static llvm::cl::opt<std::string> complexRange(
+    "complex-range",
+    llvm::cl::desc("Controls the various implementations for complex "
+                   "multiplication and division [full|improved|basic]"),
+    llvm::cl::init(""));
+
 #define FLANG_EXCLUDE_CODEGEN
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
@@ -437,6 +443,8 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setSkipExternalRttiDefinition(skipExternalRttiDefinition);
   if (enableCUDA)
     loweringOptions.setCUDARuntimeCheck(true);
+  if (complexRange == "improved" || complexRange == "basic")
+    loweringOptions.setComplexDivisionToRuntime(false);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   Fortran::frontend::TargetOptions targetOpts;
   Fortran::frontend::CodeGenOptions cgOpts;

>From 60e0cd9ece16f0f29ce2525d72e2ba37c0ae8447 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 10:19:12 +0900
Subject: [PATCH 2/7] Add help text

---
 clang/include/clang/Driver/Options.td | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 58209ceb5dc54..a1350e69a202e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1025,7 +1025,8 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
-  NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
+  NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>,
+  HelpText<"Controls the calculation methods of complex number multiplication and division.">;
 
 def complex_range_EQ : Joined<["-"], "complex-range=">, Group<f_Group>,
   Visibility<[CC1Option, FC1Option]>,

>From bddb2d33369ca4bdec50c3e927004555a7940d9a Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 12:01:16 +0900
Subject: [PATCH 3/7] Move common functions to CommonArgs.cpp

---
 clang/include/clang/Driver/CommonArgs.h    |  6 +++++
 clang/lib/Driver/ToolChains/Clang.cpp      | 27 ----------------------
 clang/lib/Driver/ToolChains/CommonArgs.cpp | 27 ++++++++++++++++++++++
 clang/lib/Driver/ToolChains/Flang.cpp      | 24 -------------------
 4 files changed, 33 insertions(+), 51 deletions(-)

diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index 26aa3ccf84786..cd1f3b2eb36ea 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -283,6 +283,12 @@ StringRef parseMPreferVectorWidthOption(clang::DiagnosticsEngine &Diags,
 StringRef parseMRecipOption(clang::DiagnosticsEngine &Diags,
                             const llvm::opt::ArgList &Args);
 
+// Convert ComplexRangeKind to a string that can be passed as a frontend option.
+std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range);
+
+// Render a frontend option corresponding to ComplexRangeKind.
+std::string RenderComplexRangeOption(LangOptions::ComplexRangeKind Range);
+
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index fea4ee909ff46..5d42ce56926d8 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2740,25 +2740,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
   }
 }
 
-static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
-  switch (Range) {
-  case LangOptions::ComplexRangeKind::CX_Full:
-    return "full";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Basic:
-    return "basic";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Improved:
-    return "improved";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Promoted:
-    return "promoted";
-    break;
-  default:
-    return "";
-  }
-}
-
 static std::string ComplexArithmeticStr(LangOptions::ComplexRangeKind Range) {
   return (Range == LangOptions::ComplexRangeKind::CX_None)
              ? ""
@@ -2772,14 +2753,6 @@ static void EmitComplexRangeDiag(const Driver &D, std::string str1,
   }
 }
 
-static std::string
-RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
-  if (!ComplexRangeStr.empty())
-    return "-complex-range=" + ComplexRangeStr;
-  return ComplexRangeStr;
-}
-
 static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
                                        bool OFastEnabled, const ArgList &Args,
                                        ArgStringList &CmdArgs,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..a6735e5e412d4 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -3409,3 +3409,30 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
 
   return Out;
 }
+
+std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Promoted:
+    return "promoted";
+    break;
+  default:
+    return "";
+  }
+}
+
+std::string
+tools::RenderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index ba5db7e3aba43..b5767918b86d1 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -595,30 +595,6 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
-static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
-  switch (Range) {
-  case LangOptions::ComplexRangeKind::CX_Full:
-    return "full";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Improved:
-    return "improved";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Basic:
-    return "basic";
-    break;
-  default:
-    return "";
-  }
-}
-
-static std::string
-RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
-  if (!ComplexRangeStr.empty())
-    return "-complex-range=" + ComplexRangeStr;
-  return ComplexRangeStr;
-}
-
 static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   StringRef FPContract;

>From 65c522ccd201fd48c481990ac0b1ec9b8be6d163 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 12:01:54 +0900
Subject: [PATCH 4/7] Minor fixes

---
 flang/lib/Frontend/CompilerInvocation.cpp              | 2 +-
 flang/lib/Lower/ConvertExprToHLFIR.cpp                 | 5 +++--
 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 86ec410b1f70f..910e0f5dcebd5 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -485,7 +485,7 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
 
   parseDoConcurrentMapping(opts, args, diags);
 
-  if (const auto *arg =
+  if (const llvm::opt::Arg *arg =
           args.getLastArg(clang::driver::options::OPT_complex_range_EQ)) {
     llvm::StringRef argValue = llvm::StringRef(arg->getValue());
     if (argValue == "full") {
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index cb338618dbf3b..9689f920840fb 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1070,12 +1070,13 @@ struct BinaryOp<Fortran::evaluate::Divide<
     // TODO: Ideally, complex number division operations should always be
     // lowered to MLIR. However, converting them to the runtime via MLIR causes
     // ABI issues.
-    if (builder.getComplexDivisionToRuntimeFlag())
+    if (builder.getComplexDivisionToRuntimeFlag()) {
       return hlfir::EntityWithAttributes{
           fir::genDivC(builder, loc, ty, lhs, rhs)};
-    else
+    } else {
       return hlfir::EntityWithAttributes{
           builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
+    }
   }
 };
 
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
index 3e158b0bed998..fe4a7256a4a16 100644
--- a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
@@ -32,4 +32,4 @@
 subroutine div_test_quad(a,b,c)
   complex(kind=16) :: a, b, c
   a = b / c
-end subroutine div_test_quad
\ No newline at end of file
+end subroutine div_test_quad

>From 528e1defac6714baad3f80a6e031fe02fb18011c Mon Sep 17 00:00:00 2001
From: Shunsuke Watanabe <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 17:41:45 +0900
Subject: [PATCH 5/7] Update flang/include/flang/Optimizer/Builder/FIRBuilder.h

Co-authored-by: Tarun Prabhu <tarunprabhu at gmail.com>
---
 flang/include/flang/Optimizer/Builder/FIRBuilder.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index b1513850a9048..004ad00b6f470 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -609,8 +609,8 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return integerOverflowFlags;
   }
 
-  /// Set ComplexDivisionToRuntimeFlag value for whether complex number division
-  /// is lowered to a runtime function by this builder.
+  /// Set ComplexDivisionToRuntimeFlag value. If set to true, complex number
+  /// division is lowered to a runtime function by this builder.
   void setComplexDivisionToRuntimeFlag(bool flag) {
     complexDivisionToRuntimeFlag = flag;
   }

>From eea4a5f402cddae02c31a1217af6930fa4981874 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Mon, 7 Jul 2025 18:27:04 +0900
Subject: [PATCH 6/7] Add lowering information to ComplexOperations.md

---
 flang/docs/ComplexOperations.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/flang/docs/ComplexOperations.md b/flang/docs/ComplexOperations.md
index 03566035c28a3..b7d2ecaa85e1c 100644
--- a/flang/docs/ComplexOperations.md
+++ b/flang/docs/ComplexOperations.md
@@ -74,4 +74,26 @@ The ComplexToStandard dialect does still call into libm for some floating
 point math operations, however these don't have the same ABI issues as the
 complex libm functions.
 
+The flang driver option `-fcomplex-arithmetic=` allows you to select whether
+complex number division is lowered to function calls or to the `complex.div`
+operation in the MLIR complex dialect. To avoid the ABI issues mentioned above,
+the selection between function calls and the `complex.div` is made during the
+lowering phase. The behavior of this option is as follows:
+
+- `basic`: Lowers to `complex.div` and is converted to algebraic formulas. No
+special handling to avoid overflow. NaN and infinite values are not handled.
+- `improved`: Lowers to `complex.div` and is converted to Smith's algorithm. See
+SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962). This
+value offers improved handling for overflow in intermediate calculations, but
+overflow may occur. NaN and infinite values are handled.
+- `full`: Lowers to a call to runtime library functions. Overflow and non-finite
+values are handled by the library implementation. This is the default value.
+
+While [the same option in clang][2] allows specifying `promoted`, this is not
+implemented in Flang. Also, in the case of `improved`, clang does not handle NaN
+and infinite values, but Flang does. These behavioral differences arise because
+the transformation of complex division calculations depends on the implementation
+of ComplexToStandard, which may change in the future.
+
 [1]: https://discourse.llvm.org/t/rfc-change-lowering-of-fortran-math-intrinsics/63971
+[2]: https://clang.llvm.org/docs/UsersManual.html#cmdoption-fcomplex-arithmetic

>From e94dad223f75882bfed40bd7dce651549db31523 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Tue, 8 Jul 2025 10:30:06 +0900
Subject: [PATCH 7/7] fix function names

---
 clang/include/clang/Driver/CommonArgs.h    |  4 ++--
 clang/lib/Driver/ToolChains/Clang.cpp      | 14 +++++++-------
 clang/lib/Driver/ToolChains/CommonArgs.cpp |  6 +++---
 clang/lib/Driver/ToolChains/Flang.cpp      |  4 ++--
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index cd1f3b2eb36ea..49c7149380fba 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -284,10 +284,10 @@ StringRef parseMRecipOption(clang::DiagnosticsEngine &Diags,
                             const llvm::opt::ArgList &Args);
 
 // Convert ComplexRangeKind to a string that can be passed as a frontend option.
-std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range);
+std::string complexRangeKindToStr(LangOptions::ComplexRangeKind Range);
 
 // Render a frontend option corresponding to ComplexRangeKind.
-std::string RenderComplexRangeOption(LangOptions::ComplexRangeKind Range);
+std::string renderComplexRangeOption(LangOptions::ComplexRangeKind Range);
 
 } // end namespace tools
 } // end namespace driver
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 5d42ce56926d8..df80565867f48 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2743,7 +2743,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
 static std::string ComplexArithmeticStr(LangOptions::ComplexRangeKind Range) {
   return (Range == LangOptions::ComplexRangeKind::CX_None)
              ? ""
-             : "-fcomplex-arithmetic=" + ComplexRangeKindToStr(Range);
+             : "-fcomplex-arithmetic=" + complexRangeKindToStr(Range);
 }
 
 static void EmitComplexRangeDiag(const Driver &D, std::string str1,
@@ -2885,7 +2885,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     case options::OPT_fcx_limited_range:
       if (GccRangeComplexOption.empty()) {
         if (Range != LangOptions::ComplexRangeKind::CX_Basic)
-          EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
+          EmitComplexRangeDiag(D, renderComplexRangeOption(Range),
                                "-fcx-limited-range");
       } else {
         if (GccRangeComplexOption != "-fno-cx-limited-range")
@@ -2897,7 +2897,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       break;
     case options::OPT_fno_cx_limited_range:
       if (GccRangeComplexOption.empty()) {
-        EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
+        EmitComplexRangeDiag(D, renderComplexRangeOption(Range),
                              "-fno-cx-limited-range");
       } else {
         if (GccRangeComplexOption != "-fcx-limited-range" &&
@@ -2911,7 +2911,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       break;
     case options::OPT_fcx_fortran_rules:
       if (GccRangeComplexOption.empty())
-        EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
+        EmitComplexRangeDiag(D, renderComplexRangeOption(Range),
                              "-fcx-fortran-rules");
       else
         EmitComplexRangeDiag(D, GccRangeComplexOption, "-fcx-fortran-rules");
@@ -2921,7 +2921,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       break;
     case options::OPT_fno_cx_fortran_rules:
       if (GccRangeComplexOption.empty()) {
-        EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
+        EmitComplexRangeDiag(D, renderComplexRangeOption(Range),
                              "-fno-cx-fortran-rules");
       } else {
         if (GccRangeComplexOption != "-fno-cx-limited-range")
@@ -3389,12 +3389,12 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     CmdArgs.push_back("-fno-strict-float-cast-overflow");
 
   if (Range != LangOptions::ComplexRangeKind::CX_None)
-    ComplexRangeStr = RenderComplexRangeOption(Range);
+    ComplexRangeStr = renderComplexRangeOption(Range);
   if (!ComplexRangeStr.empty()) {
     CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
     if (Args.hasArg(options::OPT_fcomplex_arithmetic_EQ))
       CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" +
-                                           ComplexRangeKindToStr(Range)));
+                                           complexRangeKindToStr(Range)));
   }
   if (Args.hasArg(options::OPT_fcx_limited_range))
     CmdArgs.push_back("-fcx-limited-range");
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index a6735e5e412d4..b5543a7eb738d 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -3410,7 +3410,7 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
   return Out;
 }
 
-std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+std::string tools::complexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
   switch (Range) {
   case LangOptions::ComplexRangeKind::CX_Full:
     return "full";
@@ -3430,8 +3430,8 @@ std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
 }
 
 std::string
-tools::RenderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+tools::renderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = complexRangeKindToStr(Range);
   if (!ComplexRangeStr.empty())
     return "-complex-range=" + ComplexRangeStr;
   return ComplexRangeStr;
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index b5767918b86d1..3c73f73ac9acf 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -716,10 +716,10 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
     CmdArgs.push_back(Args.MakeArgString("-mrecip=" + Recip));
 
   if (Range != LangOptions::ComplexRangeKind::CX_None) {
-    std::string ComplexRangeStr = RenderComplexRangeOption(Range);
+    std::string ComplexRangeStr = renderComplexRangeOption(Range);
     CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
     CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" +
-                                         ComplexRangeKindToStr(Range)));
+                                         complexRangeKindToStr(Range)));
   }
 
   if (!HonorINFs && !HonorNaNs && AssociativeMath && ReciprocalMath &&


From flang-commits at lists.llvm.org  Mon Jul  7 23:35:24 2025
From: flang-commits at lists.llvm.org (Shunsuke Watanabe via flang-commits)
Date: Mon, 07 Jul 2025 23:35:24 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686cbc2c.630a0220.67923.cacd@mx.google.com>


https://github.com/s-watanabe314 updated https://github.com/llvm/llvm-project/pull/146641

>From baf87a6002d740868daeeba0f89d55c05e85b712 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 18 Apr 2025 09:04:07 +0900
Subject: [PATCH 1/8] [flang] Add -fcomplex-arithmetic= option and select
 complex division algorithm

This patch adds an option to select the method for computing complex
number division. It uses `LoweringOptions` to determine whether to lower
complex division to a runtime function call or to MLIR's `complex.div`,
and `CodeGenOptions` to select the computation algorithm for
`complex.div`. The available option values and their corresponding
algorithms are as follows:
- `full`: Lower to a runtime function call. (Default behavior)
- `improved`: Lower to `complex.div` and expand to Smith's algorithm.
- `basic`: Lower to `complex.div` and expand to the algebraic algorithm.

See also the discussion in the following discourse post:
https://discourse.llvm.org/t/optimization-of-complex-number-division/83468
---
 clang/include/clang/Driver/Options.td         |   4 +-
 clang/lib/Driver/ToolChains/Flang.cpp         |  47 ++
 .../include/flang/Frontend/CodeGenOptions.def |   1 +
 flang/include/flang/Frontend/CodeGenOptions.h |  25 +
 flang/include/flang/Lower/LoweringOptions.def |   4 +
 .../flang/Optimizer/Builder/FIRBuilder.h      |  15 +
 .../include/flang/Optimizer/CodeGen/CodeGen.h |   6 +
 flang/include/flang/Tools/CrossToolHelpers.h  |   3 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  21 +
 flang/lib/Frontend/FrontendActions.cpp        |   2 +
 flang/lib/Lower/Bridge.cpp                    |   2 +
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |  12 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  15 +-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |   1 +
 flang/test/Driver/complex-range.f90           |  23 +
 .../complex-div-to-llvm-kind10.f90            | 131 +++++
 .../complex-div-to-llvm-kind16.f90            | 131 +++++
 .../test/Integration/complex-div-to-llvm.f90  | 508 ++++++++++++++++++
 .../HLFIR/complex-div-to-hlfir-kind10.f90     |  34 ++
 .../HLFIR/complex-div-to-hlfir-kind16.f90     |  35 ++
 .../test/Lower/HLFIR/complex-div-to-hlfir.f90 |  94 ++++
 flang/tools/bbc/bbc.cpp                       |   8 +
 22 files changed, 1117 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Driver/complex-range.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm-kind10.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm-kind16.f90
 create mode 100644 flang/test/Integration/complex-div-to-llvm.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
 create mode 100644 flang/test/Lower/HLFIR/complex-div-to-hlfir.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9911d752966e3..58209ceb5dc54 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1023,12 +1023,12 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
   BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>;
 
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
 
 def complex_range_EQ : Joined<["-"], "complex-range=">, Group<f_Group>,
-  Visibility<[CC1Option]>,
+  Visibility<[CC1Option, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
   NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>,
   MarshallingInfoEnum<LangOpts<"ComplexRange">, "CX_Full">;
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index e4e321ba1e195..ba5db7e3aba43 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -595,6 +595,30 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
+static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  default:
+    return "";
+  }
+}
+
+static std::string
+RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
+
 static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   StringRef FPContract;
@@ -605,6 +629,8 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
   bool AssociativeMath = false;
   bool ReciprocalMath = false;
 
+  LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_None;
+
   if (const Arg *A = Args.getLastArg(options::OPT_ffp_contract)) {
     const StringRef Val = A->getValue();
     if (Val == "fast" || Val == "off") {
@@ -629,6 +655,20 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
     default:
       continue;
 
+    case options::OPT_fcomplex_arithmetic_EQ: {
+      StringRef Val = A->getValue();
+      if (Val == "full")
+        Range = LangOptions::ComplexRangeKind::CX_Full;
+      else if (Val == "improved")
+        Range = LangOptions::ComplexRangeKind::CX_Improved;
+      else if (Val == "basic")
+        Range = LangOptions::ComplexRangeKind::CX_Basic;
+      else {
+        D.Diag(diag::err_drv_unsupported_option_argument)
+            << A->getSpelling() << Val;
+      }
+      break;
+    }
     case options::OPT_fhonor_infinities:
       HonorINFs = true;
       break;
@@ -699,6 +739,13 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
   if (!Recip.empty())
     CmdArgs.push_back(Args.MakeArgString("-mrecip=" + Recip));
 
+  if (Range != LangOptions::ComplexRangeKind::CX_None) {
+    std::string ComplexRangeStr = RenderComplexRangeOption(Range);
+    CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
+    CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" +
+                                         ComplexRangeKindToStr(Range)));
+  }
+
   if (!HonorINFs && !HonorNaNs && AssociativeMath && ReciprocalMath &&
       ApproxFunc && !SignedZeros &&
       (FPContract == "fast" || FPContract.empty())) {
diff --git a/flang/include/flang/Frontend/CodeGenOptions.def b/flang/include/flang/Frontend/CodeGenOptions.def
index ae12aec518108..cdeea93c9aecb 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.def
+++ b/flang/include/flang/Frontend/CodeGenOptions.def
@@ -52,6 +52,7 @@ ENUM_CODEGENOPT(RelocationModel, llvm::Reloc::Model, 3, llvm::Reloc::PIC_) ///<
 ENUM_CODEGENOPT(DebugInfo,  llvm::codegenoptions::DebugInfoKind, 4,  llvm::codegenoptions::NoDebugInfo) ///< Level of debug info to generate
 ENUM_CODEGENOPT(VecLib, llvm::driver::VectorLibrary, 4, llvm::driver::VectorLibrary::NoLibrary) ///< Vector functions library to use
 ENUM_CODEGENOPT(FramePointer, llvm::FramePointerKind, 2, llvm::FramePointerKind::None) ///< Enable the usage of frame pointers
+ENUM_CODEGENOPT(ComplexRange, ComplexRangeKind, 3, ComplexRangeKind::CX_Full) ///< Method for calculating complex number division
 
 ENUM_CODEGENOPT(DoConcurrentMapping, DoConcurrentMappingKind, 2, DoConcurrentMappingKind::DCMK_None) ///< Map `do concurrent` to OpenMP
 
diff --git a/flang/include/flang/Frontend/CodeGenOptions.h b/flang/include/flang/Frontend/CodeGenOptions.h
index bad17c8309eb8..df6063cc90340 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.h
+++ b/flang/include/flang/Frontend/CodeGenOptions.h
@@ -192,6 +192,31 @@ class CodeGenOptions : public CodeGenOptionsBase {
     return getProfileUse() == llvm::driver::ProfileCSIRInstr;
   }
 
+  /// Controls the various implementations for complex division.
+  enum ComplexRangeKind {
+    /// Implementation of complex division using a call to runtime library
+    /// functions. Overflow and non-finite values are handled by the library
+    /// implementation. This is the default value.
+    CX_Full,
+
+    /// Implementation of complex division offering an improved handling
+    /// for overflow in intermediate calculations. Overflow and non-finite
+    /// values are handled by MLIR's implementation of "complex.div", but this
+    /// may change in the future.
+    CX_Improved,
+
+    /// Implementation of complex division using algebraic formulas at source
+    /// precision. No special handling to avoid overflow. NaN and infinite
+    /// values are not handled.
+    CX_Basic,
+
+    /// No range rule is enabled.
+    CX_None
+
+    /// TODO: Implemention of other values as needed. In Clang, "CX_Promoted"
+    /// is implemented. (See clang/Basic/LangOptions.h)
+  };
+
   // Define accessors/mutators for code generation options of enumeration type.
 #define CODEGENOPT(Name, Bits, Default)
 #define ENUM_CODEGENOPT(Name, Type, Bits, Default)                             \
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index 3263ab129d076..8135704971aa4 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -70,5 +70,9 @@ ENUM_LOWERINGOPT(CUDARuntimeCheck, unsigned, 1, 0)
 /// derived types defined in other compilation units.
 ENUM_LOWERINGOPT(SkipExternalRttiDefinition, unsigned, 1, 0)
 
+/// If true, convert complex number division to runtime on the frontend.
+/// If false, lower to the complex dialect of MLIR.
+/// On by default.
+ENUM_LOWERINGOPT(ComplexDivisionToRuntime, unsigned, 1, 1)
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index e1eaab3346901..b1513850a9048 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -609,6 +609,17 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return integerOverflowFlags;
   }
 
+  /// Set ComplexDivisionToRuntimeFlag value for whether complex number division
+  /// is lowered to a runtime function by this builder.
+  void setComplexDivisionToRuntimeFlag(bool flag) {
+    complexDivisionToRuntimeFlag = flag;
+  }
+
+  /// Get current ComplexDivisionToRuntimeFlag value.
+  bool getComplexDivisionToRuntimeFlag() const {
+    return complexDivisionToRuntimeFlag;
+  }
+
   /// Dump the current function. (debug)
   LLVM_DUMP_METHOD void dumpFunc();
 
@@ -673,6 +684,10 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   /// mlir::arith::IntegerOverflowFlagsAttr.
   mlir::arith::IntegerOverflowFlags integerOverflowFlags{};
 
+  /// Flag to control whether complex number division is lowered to a runtime
+  /// function or to the MLIR complex dialect.
+  bool complexDivisionToRuntimeFlag = true;
+
   /// fir::GlobalOp and func::FuncOp symbol table to speed-up
   /// lookups.
   mlir::SymbolTable *symbolTable = nullptr;
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 93f07d8d5d4d9..e9a07a8dde5cd 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 #define FORTRAN_OPTIMIZER_CODEGEN_CODEGEN_H
 
+#include "flang/Frontend/CodeGenOptions.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassRegistry.h"
@@ -58,6 +59,11 @@ struct FIRToLLVMPassOptions {
   // the name of the global variable corresponding to a derived
   // type's descriptor.
   bool typeDescriptorsRenamedForAssembly = false;
+
+  // Specify the calculation method for complex number division used by the
+  // Conversion pass of the MLIR complex dialect.
+  Fortran::frontend::CodeGenOptions::ComplexRangeKind ComplexRange =
+      Fortran::frontend::CodeGenOptions::ComplexRangeKind::CX_Full;
 };
 
 /// Convert FIR to the LLVM IR dialect with default options.
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 337685c82af5f..df1da27058552 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -140,6 +140,9 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   std::string InstrumentFunctionExit =
       ""; ///< Name of the instrument-function that is called on each
           ///< function-exit
+  Fortran::frontend::CodeGenOptions::ComplexRangeKind ComplexRange =
+      Fortran::frontend::CodeGenOptions::ComplexRangeKind::
+          CX_Full; ///< Method for calculating complex number division
 };
 
 struct OffloadModuleOpts {
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..86ec410b1f70f 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -484,6 +484,21 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
   }
 
   parseDoConcurrentMapping(opts, args, diags);
+
+  if (const auto *arg =
+          args.getLastArg(clang::driver::options::OPT_complex_range_EQ)) {
+    llvm::StringRef argValue = llvm::StringRef(arg->getValue());
+    if (argValue == "full") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Full);
+    } else if (argValue == "improved") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Improved);
+    } else if (argValue == "basic") {
+      opts.setComplexRange(CodeGenOptions::ComplexRangeKind::CX_Basic);
+    } else {
+      diags.Report(clang::diag::err_drv_invalid_value)
+          << arg->getAsString(args) << arg->getValue();
+    }
+  }
 }
 
 /// Parses all target input arguments and populates the target
@@ -1811,4 +1826,10 @@ void CompilerInvocation::setLoweringOptions() {
       .setNoSignedZeros(langOptions.NoSignedZeros)
       .setAssociativeMath(langOptions.AssociativeMath)
       .setReciprocalMath(langOptions.ReciprocalMath);
+
+  if (codegenOpts.getComplexRange() ==
+          CodeGenOptions::ComplexRangeKind::CX_Improved ||
+      codegenOpts.getComplexRange() ==
+          CodeGenOptions::ComplexRangeKind::CX_Basic)
+    loweringOpts.setComplexDivisionToRuntime(false);
 }
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index bf15def3f3b2e..b5f4f9421f633 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -750,6 +750,8 @@ void CodeGenAction::generateLLVMIR() {
   if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
     config.NSWOnLoopVarInc = false;
 
+  config.ComplexRange = opts.getComplexRange();
+
   // Create the pass pipeline
   fir::createMLIRToLLVMPassPipeline(pm, config, getCurrentFile());
   (void)mlir::applyPassManagerCLOptions(pm);
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..7e95c640e73b0 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5746,6 +5746,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     builder =
         new fir::FirOpBuilder(func, bridge.getKindMap(), &mlirSymbolTable);
     assert(builder && "FirOpBuilder did not instantiate");
+    builder->setComplexDivisionToRuntimeFlag(
+        bridge.getLoweringOptions().getComplexDivisionToRuntime());
     builder->setFastMathFlags(bridge.getLoweringOptions().getMathOptions());
     builder->setInsertionPointToStart(&func.front());
     if (funit.parent.isA<Fortran::lower::pft::FunctionLikeUnit>()) {
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index df8dfbc72c030..cb338618dbf3b 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1066,8 +1066,16 @@ struct BinaryOp<Fortran::evaluate::Divide<
     mlir::Type ty = Fortran::lower::getFIRType(
         builder.getContext(), Fortran::common::TypeCategory::Complex, KIND,
         /*params=*/{});
-    return hlfir::EntityWithAttributes{
-        fir::genDivC(builder, loc, ty, lhs, rhs)};
+
+    // TODO: Ideally, complex number division operations should always be
+    // lowered to MLIR. However, converting them to the runtime via MLIR causes
+    // ABI issues.
+    if (builder.getComplexDivisionToRuntimeFlag())
+      return hlfir::EntityWithAttributes{
+          fir::genDivC(builder, loc, ty, lhs, rhs)};
+    else
+      return hlfir::EntityWithAttributes{
+          builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
   }
 };
 
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 2b018912b40e4..4cf533f195e69 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -4119,7 +4119,20 @@ class FIRToLLVMLowering
     mathToFuncsOptions.minWidthOfFPowIExponent = 33;
     mathConvertionPM.addPass(
         mlir::createConvertMathToFuncs(mathToFuncsOptions));
-    mathConvertionPM.addPass(mlir::createConvertComplexToStandardPass());
+
+    mlir::ConvertComplexToStandardPassOptions complexToStandardOptions{};
+    if (options.ComplexRange ==
+        Fortran::frontend::CodeGenOptions::ComplexRangeKind::CX_Basic) {
+      complexToStandardOptions.complexRange =
+          mlir::complex::ComplexRangeFlags::basic;
+    } else if (options.ComplexRange == Fortran::frontend::CodeGenOptions::
+                                           ComplexRangeKind::CX_Improved) {
+      complexToStandardOptions.complexRange =
+          mlir::complex::ComplexRangeFlags::improved;
+    }
+    mathConvertionPM.addPass(
+        mlir::createConvertComplexToStandardPass(complexToStandardOptions));
+
     // Convert Math dialect operations into LLVM dialect operations.
     // There is no way to prefer MathToLLVM patterns over MathToLibm
     // patterns (applied below), so we have to run MathToLLVM conversion here.
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 42d9e7ba2418f..d940934f9f6ad 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -113,6 +113,7 @@ void addFIRToLLVMPass(mlir::PassManager &pm,
   options.forceUnifiedTBAATree = useOldAliasTags;
   options.typeDescriptorsRenamedForAssembly =
       !disableCompilerGeneratedNamesConversion;
+  options.ComplexRange = config.ComplexRange;
   addPassConditionally(pm, disableFirToLlvmIr,
                        [&]() { return fir::createFIRToLLVMPass(options); });
   // The dialect conversion framework may leave dead unrealized_conversion_cast
diff --git a/flang/test/Driver/complex-range.f90 b/flang/test/Driver/complex-range.f90
new file mode 100644
index 0000000000000..e5a1ba9068ac9
--- /dev/null
+++ b/flang/test/Driver/complex-range.f90
@@ -0,0 +1,23 @@
+! Test range options for complex multiplication and division.
+
+! RUN: %flang -### -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=RANGE
+
+! RUN: %flang -### -fcomplex-arithmetic=full -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=FULL
+
+! RUN: %flang -### -fcomplex-arithmetic=improved -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=IMPRVD
+
+! RUN: %flang -### -fcomplex-arithmetic=basic -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=BASIC
+
+! RUN: not %flang -### -fcomplex-arithmetic=foo -c %s 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=ERR
+
+! RANGE-NOT: -complex-range=
+! FULL: -complex-range=full
+! IMPRVD: -complex-range=improved
+! BASIC: -complex-range=basic
+
+! ERR: error: unsupported argument 'foo' to option '-fcomplex-arithmetic='
diff --git a/flang/test/Integration/complex-div-to-llvm-kind10.f90 b/flang/test/Integration/complex-div-to-llvm-kind10.f90
new file mode 100644
index 0000000000000..04d1f7ed9b024
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm-kind10.f90
@@ -0,0 +1,131 @@
+! Test lowering complex division to llvm ir according to options
+
+! REQUIRES: target=x86_64{{.*}}
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_extended
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { x86_fp80, x86_fp80 }, ptr %[[LHS]], align 16
+! CHECK: %[[LOAD_RHS:.*]] = load { x86_fp80, x86_fp80 }, ptr %[[RHS]], align 16
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { x86_fp80, x86_fp80 } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract x86_fp80 %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract x86_fp80 %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract x86_fp80 %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract x86_fp80 %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract x86_fp80 %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract x86_fp80 %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract x86_fp80 %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract x86_fp80 %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract x86_fp80 %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract x86_fp80 %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract x86_fp80 %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract x86_fp80 %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq x86_fp80 %[[RHS_REAL_ABS]], 0xK00000000000000000000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq x86_fp80 %[[RHS_IMAG_ABS]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord x86_fp80 %[[LHS_REAL]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord x86_fp80 %[[LHS_IMAG]], 0xK00000000000000000000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 0xK7FFF8000000000000000, x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract x86_fp80 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract x86_fp80 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one x86_fp80 %[[RHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one x86_fp80 %[[RHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq x86_fp80 %[[LHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract x86_fp80 @llvm.fabs.f80(x86_fp80 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq x86_fp80 %[[LHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[LHS_REAL_IS_INF]], x86_fp80 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[LHS_IMAG_IS_INF]], x86_fp80 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract x86_fp80 %[[INF_MULTIPLICATOR_1]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract x86_fp80 %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract x86_fp80 %[[INF_MULTIPLICATOR_2]], 0xK7FFF8000000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one x86_fp80 %[[LHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one x86_fp80 %[[LHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq x86_fp80 %[[RHS_REAL_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq x86_fp80 %[[RHS_IMAG_ABS]], 0xK7FFF8000000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[RHS_REAL_IS_INF]], x86_fp80 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call x86_fp80 @llvm.copysign.f80(x86_fp80 %[[RHS_IMAG_IS_INF]], x86_fp80 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract x86_fp80 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract x86_fp80 %[[ZERO_MULTIPLICATOR_1]], 0xK00000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract x86_fp80 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract x86_fp80 %[[ZERO_MULTIPLICATOR_2]], 0xK00000000000000000000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt x86_fp80 %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], x86_fp80 %[[RESULT_REAL_1]], x86_fp80 %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], x86_fp80 %[[RESULT_IMAG_1]], x86_fp80 %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], x86_fp80 %[[RESULT_REAL_4]], x86_fp80 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], x86_fp80 %[[RESULT_IMAG_4]], x86_fp80 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], x86_fp80 %[[RESULT_REAL_3]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], x86_fp80 %[[RESULT_IMAG_3]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], x86_fp80 %[[INFINITY_RESULT_REAL]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], x86_fp80 %[[INFINITY_RESULT_IMAG]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno x86_fp80 %[[RESULT_REAL]], 0xK00000000000000000000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno x86_fp80 %[[RESULT_IMAG]], 0xK00000000000000000000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], x86_fp80 %[[RESULT_REAL_SPECIAL_CASE_1]], x86_fp80 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], x86_fp80 %[[RESULT_IMAG_SPECIAL_CASE_1]], x86_fp80 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { x86_fp80, x86_fp80 } %[[RESULT_1]], x86_fp80 %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { x86_fp80, x86_fp80 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract x86_fp80 %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract x86_fp80 %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract x86_fp80 %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract x86_fp80 %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract x86_fp80 %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract x86_fp80 %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract x86_fp80 %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract x86_fp80 %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract x86_fp80 %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { x86_fp80, x86_fp80 } %[[RESULT_1]], x86_fp80 %[[IMAG]], 1
+! BASIC: store { x86_fp80, x86_fp80 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! CHECK: ret void
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
diff --git a/flang/test/Integration/complex-div-to-llvm-kind16.f90 b/flang/test/Integration/complex-div-to-llvm-kind16.f90
new file mode 100644
index 0000000000000..887a7972dc1be
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm-kind16.f90
@@ -0,0 +1,131 @@
+! Test lowering complex division to llvm ir according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_quad
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { fp128, fp128 }, ptr %[[LHS]], align 16
+! CHECK: %[[LOAD_RHS:.*]] = load { fp128, fp128 }, ptr %[[RHS]], align 16
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { fp128, fp128 } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { fp128, fp128 } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { fp128, fp128 } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { fp128, fp128 } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract fp128 %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract fp128 %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract fp128 %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract fp128 %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract fp128 %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract fp128 %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract fp128 %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract fp128 %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract fp128 %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract fp128 %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract fp128 %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract fp128 %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq fp128 %[[RHS_REAL_ABS]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord fp128 %[[LHS_REAL]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord fp128 %[[LHS_IMAG]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call fp128 @llvm.copysign.f128(fp128 0xL00000000000000007FFF000000000000, fp128 %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract fp128 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract fp128 %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one fp128 %[[RHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq fp128 %[[LHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract fp128 @llvm.fabs.f128(fp128 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq fp128 %[[LHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[LHS_REAL_IS_INF]], fp128 %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[LHS_IMAG_IS_INF]], fp128 %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract fp128 %[[INF_MULTIPLICATOR_1]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract fp128 %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract fp128 %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract fp128 %[[INF_MULTIPLICATOR_2]], 0xL00000000000000007FFF000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one fp128 %[[LHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one fp128 %[[LHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq fp128 %[[RHS_REAL_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq fp128 %[[RHS_IMAG_ABS]], 0xL00000000000000007FFF000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[RHS_REAL_IS_INF]], fp128 %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call fp128 @llvm.copysign.f128(fp128 %[[RHS_IMAG_IS_INF]], fp128 %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract fp128 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract fp128 %[[ZERO_MULTIPLICATOR_1]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract fp128 %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract fp128 %[[ZERO_MULTIPLICATOR_2]], 0xL00000000000000000000000000000000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt fp128 %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], fp128 %[[RESULT_REAL_1]], fp128 %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], fp128 %[[RESULT_IMAG_1]], fp128 %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], fp128 %[[RESULT_REAL_4]], fp128 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], fp128 %[[RESULT_IMAG_4]], fp128 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], fp128 %[[RESULT_REAL_3]], fp128 %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], fp128 %[[RESULT_IMAG_3]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], fp128 %[[INFINITY_RESULT_REAL]], fp128 %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], fp128 %[[INFINITY_RESULT_IMAG]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno fp128 %[[RESULT_REAL]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno fp128 %[[RESULT_IMAG]], 0xL00000000000000000000000000000000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], fp128 %[[RESULT_REAL_SPECIAL_CASE_1]], fp128 %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], fp128 %[[RESULT_IMAG_SPECIAL_CASE_1]], fp128 %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { fp128, fp128 } poison, fp128 %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { fp128, fp128 } %[[RESULT_1]], fp128 %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { fp128, fp128 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract fp128 %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract fp128 %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract fp128 %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract fp128 %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract fp128 %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract fp128 %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract fp128 %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract fp128 %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract fp128 %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { fp128, fp128 } poison, fp128 %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { fp128, fp128 } %[[RESULT_1]], fp128 %[[IMAG]], 1
+! BASIC: store { fp128, fp128 } %[[RESULT_2]], ptr %[[RET]], align 16
+
+! CHECK: ret void
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
diff --git a/flang/test/Integration/complex-div-to-llvm.f90 b/flang/test/Integration/complex-div-to-llvm.f90
new file mode 100644
index 0000000000000..01782a565f24c
--- /dev/null
+++ b/flang/test/Integration/complex-div-to-llvm.f90
@@ -0,0 +1,508 @@
+! Test lowering complex division to llvm ir according to options
+
+! RUN: %flang -fcomplex-arithmetic=improved -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang -fcomplex-arithmetic=basic -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @div_test_half
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { half, half }, ptr %[[LHS]], align 2
+! CHECK: %[[LOAD_RHS:.*]] = load { half, half }, ptr %[[RHS]], align 2
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { half, half } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { half, half } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { half, half } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { half, half } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract half %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract half %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract half %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract half %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract half %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract half %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract half %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract half %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract half %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract half %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract half %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract half %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq half %[[RHS_REAL_ABS]], 0xH0000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq half %[[RHS_IMAG_ABS]], 0xH0000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord half %[[LHS_REAL]], 0xH0000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord half %[[LHS_IMAG]], 0xH0000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call half @llvm.copysign.f16(half 0xH7C00, half %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract half %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract half %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one half %[[RHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one half %[[RHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq half %[[LHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract half @llvm.fabs.f16(half %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq half %[[LHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[LHS_REAL_IS_INF]], half %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[LHS_IMAG_IS_INF]], half %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract half %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract half %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract half %[[INF_MULTIPLICATOR_1]], 0xH7C00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract half %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract half %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract half %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract half %[[INF_MULTIPLICATOR_2]], 0xH7C00
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one half %[[LHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one half %[[LHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq half %[[RHS_REAL_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq half %[[RHS_IMAG_ABS]], 0xH7C00
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[RHS_REAL_IS_INF]], half %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], half 0xH3C00, half 0xH0000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call half @llvm.copysign.f16(half %[[RHS_IMAG_IS_INF]], half %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract half %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract half %[[ZERO_MULTIPLICATOR_1]], 0xH0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract half %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract half %[[ZERO_MULTIPLICATOR_2]], 0xH0000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt half %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], half %[[RESULT_REAL_1]], half %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], half %[[RESULT_IMAG_1]], half %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], half %[[RESULT_REAL_4]], half %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], half %[[RESULT_IMAG_4]], half %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], half %[[RESULT_REAL_3]], half %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], half %[[RESULT_IMAG_3]], half %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], half %[[INFINITY_RESULT_REAL]], half %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], half %[[INFINITY_RESULT_IMAG]], half %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno half %[[RESULT_REAL]], 0xH0000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno half %[[RESULT_IMAG]], 0xH0000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], half %[[RESULT_REAL_SPECIAL_CASE_1]], half %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], half %[[RESULT_IMAG_SPECIAL_CASE_1]], half %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { half, half } poison, half %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { half, half } %[[RESULT_1]], half %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { half, half } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract half %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract half %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract half %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract half %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract half %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract half %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract half %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract half %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract half %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { half, half } poison, half %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { half, half } %[[RESULT_1]], half %[[IMAG]], 1
+! BASIC: store { half, half } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! CHECK: ret void
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+
+! CHECK-LABEL: @div_test_bfloat
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { bfloat, bfloat }, ptr %[[LHS]], align 2
+! CHECK: %[[LOAD_RHS:.*]] = load { bfloat, bfloat }, ptr %[[RHS]], align 2
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { bfloat, bfloat } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract bfloat %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract bfloat %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract bfloat %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract bfloat %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract bfloat %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract bfloat %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract bfloat %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract bfloat %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract bfloat %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract bfloat %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract bfloat %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract bfloat %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq bfloat %[[RHS_REAL_ABS]], 0xR0000
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq bfloat %[[RHS_IMAG_ABS]], 0xR0000
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord bfloat %[[LHS_REAL]], 0xR0000
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord bfloat %[[LHS_IMAG]], 0xR0000
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call bfloat @llvm.copysign.bf16(bfloat 0xR7F80, bfloat %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract bfloat %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract bfloat %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one bfloat %[[RHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one bfloat %[[RHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq bfloat %[[LHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract bfloat @llvm.fabs.bf16(bfloat %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq bfloat %[[LHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[LHS_REAL_IS_INF]], bfloat %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[LHS_IMAG_IS_INF]], bfloat %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract bfloat %[[INF_MULTIPLICATOR_1]], 0xR7F80
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract bfloat %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract bfloat %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract bfloat %[[INF_MULTIPLICATOR_2]], 0xR7F80
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one bfloat %[[LHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one bfloat %[[LHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq bfloat %[[RHS_REAL_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq bfloat %[[RHS_IMAG_ABS]], 0xR7F80
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[RHS_REAL_IS_INF]], bfloat %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], bfloat 0xR3F80, bfloat 0xR0000
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call bfloat @llvm.copysign.bf16(bfloat %[[RHS_IMAG_IS_INF]], bfloat %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract bfloat %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract bfloat %[[ZERO_MULTIPLICATOR_1]], 0xR0000
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract bfloat %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract bfloat %[[ZERO_MULTIPLICATOR_2]], 0xR0000
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt bfloat %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], bfloat %[[RESULT_REAL_1]], bfloat %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], bfloat %[[RESULT_IMAG_1]], bfloat %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], bfloat %[[RESULT_REAL_4]], bfloat %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], bfloat %[[RESULT_IMAG_4]], bfloat %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], bfloat %[[RESULT_REAL_3]], bfloat %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], bfloat %[[RESULT_IMAG_3]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], bfloat %[[INFINITY_RESULT_REAL]], bfloat %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], bfloat %[[INFINITY_RESULT_IMAG]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno bfloat %[[RESULT_REAL]], 0xR0000
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno bfloat %[[RESULT_IMAG]], 0xR0000
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], bfloat %[[RESULT_REAL_SPECIAL_CASE_1]], bfloat %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], bfloat %[[RESULT_IMAG_SPECIAL_CASE_1]], bfloat %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { bfloat, bfloat } poison, bfloat %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { bfloat, bfloat } %[[RESULT_1]], bfloat %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { bfloat, bfloat } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract bfloat %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract bfloat %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract bfloat %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract bfloat %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract bfloat %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract bfloat %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract bfloat %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract bfloat %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract bfloat %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { bfloat, bfloat } poison, bfloat %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { bfloat, bfloat } %[[RESULT_1]], bfloat %[[IMAG]], 1
+! BASIC: store { bfloat, bfloat } %[[RESULT_2]], ptr %[[RET]], align 2
+
+! CHECK: ret void
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+
+! CHECK-LABEL: @div_test_single
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { float, float }, ptr %[[LHS]], align 4
+! CHECK: %[[LOAD_RHS:.*]] = load { float, float }, ptr %[[RHS]], align 4
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { float, float } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { float, float } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { float, float } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { float, float } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract float %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract float %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract float %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract float %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract float %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract float %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract float %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract float %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract float %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract float %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract float %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract float %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq float %[[RHS_REAL_ABS]], 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq float %[[RHS_IMAG_ABS]], 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord float %[[LHS_REAL]], 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord float %[[LHS_IMAG]], 0.000000e+00
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call float @llvm.copysign.f32(float 0x7FF0000000000000, float %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract float %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract float %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one float %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one float %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq float %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract float @llvm.fabs.f32(float %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq float %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[LHS_REAL_IS_INF]], float %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[LHS_IMAG_IS_INF]], float %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract float %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract float %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract float %[[INF_MULTIPLICATOR_1]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract float %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract float %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract float %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract float %[[INF_MULTIPLICATOR_2]], 0x7FF0000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one float %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one float %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq float %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq float %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[RHS_REAL_IS_INF]], float %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], float 1.000000e+00, float 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call float @llvm.copysign.f32(float %[[RHS_IMAG_IS_INF]], float %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract float %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract float %[[ZERO_MULTIPLICATOR_1]], 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract float %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract float %[[ZERO_MULTIPLICATOR_2]], 0.000000e+00
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt float %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], float %[[RESULT_REAL_1]], float %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], float %[[RESULT_IMAG_1]], float %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], float %[[RESULT_REAL_4]], float %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], float %[[RESULT_IMAG_4]], float %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], float %[[RESULT_REAL_3]], float %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], float %[[RESULT_IMAG_3]], float %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], float %[[INFINITY_RESULT_REAL]], float %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], float %[[INFINITY_RESULT_IMAG]], float %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno float %[[RESULT_REAL]], 0.000000e+00
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno float %[[RESULT_IMAG]], 0.000000e+00
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], float %[[RESULT_REAL_SPECIAL_CASE_1]], float %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], float %[[RESULT_IMAG_SPECIAL_CASE_1]], float %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { float, float } poison, float %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT_1]], float %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { float, float } %[[RESULT_2]], ptr %[[RET]], align 4
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract float %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract float %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract float %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract float %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract float %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract float %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract float %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract float %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract float %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { float, float } poison, float %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT_1]], float %[[IMAG]], 1
+! BASIC: store { float, float } %[[RESULT_2]], ptr %[[RET]], align 4
+
+! CHECK: ret void
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+
+! CHECK-LABEL: @div_test_double
+! CHECK-SAME: ptr %[[RET:.*]], ptr %[[LHS:.*]], ptr %[[RHS:.*]])
+! CHECK: %[[LOAD_LHS:.*]] = load { double, double }, ptr %[[LHS]], align 8
+! CHECK: %[[LOAD_RHS:.*]] = load { double, double }, ptr %[[RHS]], align 8
+! CHECK: %[[LHS_REAL:.*]] = extractvalue { double, double } %[[LOAD_LHS]], 0
+! CHECK: %[[LHS_IMAG:.*]] = extractvalue { double, double } %[[LOAD_LHS]], 1
+! CHECK: %[[RHS_REAL:.*]] = extractvalue { double, double } %[[LOAD_RHS]], 0
+! CHECK: %[[RHS_IMAG:.*]] = extractvalue { double, double } %[[LOAD_RHS]], 1
+
+! IMPRVD: %[[RHS_REAL_IMAG_RATIO:.*]] = fdiv contract double %[[RHS_REAL]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_REAL_IMAG_DENOM:.*]] = fadd contract double %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_1:.*]] = fadd contract double %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_1:.*]] = fdiv contract double %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_1:.*]] = fsub contract double %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_1:.*]] = fdiv contract double %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]]
+! IMPRVD: %[[RHS_IMAG_REAL_RATIO:.*]] = fdiv contract double %[[RHS_IMAG]], %[[RHS_REAL]]
+! IMPRVD: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]]
+! IMPRVD: %[[RHS_IMAG_REAL_DENOM:.*]] = fadd contract double %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[REAL_NUMERATOR_2:.*]] = fadd contract double %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_REAL_2:.*]] = fdiv contract double %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+! IMPRVD: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[IMAG_NUMERATOR_2:.*]] = fsub contract double %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]]
+! IMPRVD: %[[RESULT_IMAG_2:.*]] = fdiv contract double %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]]
+
+! Case 1. Zero denominator, numerator contains at most one NaN value.
+! IMPRVD: %[[RHS_REAL_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[RHS_REAL]])
+! IMPRVD: %[[RHS_REAL_ABS_IS_ZERO:.*]] = fcmp oeq double %[[RHS_REAL_ABS]], 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = fcmp oeq double %[[RHS_IMAG_ABS]], 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_NOT_NAN:.*]] = fcmp ord double %[[LHS_REAL]], 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_NOT_NAN:.*]] = fcmp ord double %[[LHS_IMAG]], 0.000000e+00
+! IMPRVD: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = or i1 %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]]
+! IMPRVD: %[[RHS_IS_ZERO:.*]] = and i1 %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]]
+! IMPRVD: %[[RESULT_IS_INFINITY:.*]] = and i1 %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]]
+! IMPRVD: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = call double @llvm.copysign.f64(double 0x7FF0000000000000, double %[[RHS_REAL]])
+! IMPRVD: %[[INFINITY_RESULT_REAL:.*]] = fmul contract double %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]]
+! IMPRVD: %[[INFINITY_RESULT_IMAG:.*]] = fmul contract double %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]]
+
+! Case 2. Infinite numerator, finite denominator.
+! IMPRVD: %[[RHS_REAL_FINITE:.*]] = fcmp one double %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_FINITE:.*]] = fcmp one double %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_FINITE:.*]] = and i1 %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]]
+! IMPRVD: %[[LHS_REAL_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[LHS_REAL]])
+! IMPRVD: %[[LHS_REAL_INFINITE:.*]] = fcmp oeq double %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_ABS:.*]] = call contract double @llvm.fabs.f64(double %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_IMAG_INFINITE:.*]] = fcmp oeq double %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_INFINITE:.*]] = or i1 %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]]
+! IMPRVD: %[[INF_NUM_FINITE_DENOM:.*]] = and i1 %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]]
+! IMPRVD: %[[LHS_REAL_IS_INF:.*]] = select i1 %[[LHS_REAL_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[LHS_REAL_IS_INF]], double %[[LHS_REAL]])
+! IMPRVD: %[[LHS_IMAG_IS_INF:.*]] = select i1 %[[LHS_IMAG_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[LHS_IMAG_IS_INF]], double %[[LHS_IMAG]])
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract double %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[INF_MULTIPLICATOR_1:.*]] = fadd contract double %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_3:.*]] = fmul contract double %[[INF_MULTIPLICATOR_1]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = fmul contract double %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]]
+! IMPRVD: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = fmul contract double %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]]
+! IMPRVD: %[[INF_MULTIPLICATOR_2:.*]] = fsub contract double %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]]
+! IMPRVD: %[[RESULT_IMAG_3:.*]] = fmul contract double %[[INF_MULTIPLICATOR_2]], 0x7FF0000000000000
+
+! Case 3. Finite numerator, infinite denominator.
+! IMPRVD: %[[LHS_REAL_FINITE:.*]] = fcmp one double %[[LHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IMAG_FINITE:.*]] = fcmp one double %[[LHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[LHS_IS_FINITE:.*]] = and i1 %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]]
+! IMPRVD: %[[RHS_REAL_INFINITE:.*]] = fcmp oeq double %[[RHS_REAL_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IMAG_INFINITE:.*]] = fcmp oeq double %[[RHS_IMAG_ABS]], 0x7FF0000000000000
+! IMPRVD: %[[RHS_IS_INFINITE:.*]] = or i1 %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]]
+! IMPRVD: %[[FINITE_NUM_INFINITE_DENOM:.*]] = and i1 %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]]
+! IMPRVD: %[[RHS_REAL_IS_INF:.*]] = select i1 %[[RHS_REAL_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[RHS_REAL_IS_INF]], double %[[RHS_REAL]])
+! IMPRVD: %[[RHS_IMAG_IS_INF:.*]] = select i1 %[[RHS_IMAG_INFINITE]], double 1.000000e+00, double 0.000000e+00
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = call double @llvm.copysign.f64(double %[[RHS_IMAG_IS_INF]], double %[[RHS_IMAG]])
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_1:.*]] = fadd contract double %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]]
+! IMPRVD: %[[RESULT_REAL_4:.*]] = fmul contract double %[[ZERO_MULTIPLICATOR_1]], 0.000000e+00
+! IMPRVD: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]]
+! IMPRVD: %[[ZERO_MULTIPLICATOR_2:.*]] = fsub contract double %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]]
+! IMPRVD: %[[RESULT_IMAG_4:.*]] = fmul contract double %[[ZERO_MULTIPLICATOR_2]], 0.000000e+00
+
+! IMPRVD: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = fcmp olt double %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]]
+! IMPRVD: %[[RESULT_REAL:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], double %[[RESULT_REAL_1]], double %[[RESULT_REAL_2]]
+! IMPRVD: %[[RESULT_IMAG:.*]] = select i1 %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], double %[[RESULT_IMAG_1]], double %[[RESULT_IMAG_2]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], double %[[RESULT_REAL_4]], double %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = select i1 %[[FINITE_NUM_INFINITE_DENOM]], double %[[RESULT_IMAG_4]], double %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], double %[[RESULT_REAL_3]], double %[[RESULT_REAL_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = select i1 %[[INF_NUM_FINITE_DENOM]], double %[[RESULT_IMAG_3]], double %[[RESULT_IMAG_SPECIAL_CASE_3]]
+! IMPRVD: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], double %[[INFINITY_RESULT_REAL]], double %[[RESULT_REAL_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = select i1 %[[RESULT_IS_INFINITY]], double %[[INFINITY_RESULT_IMAG]], double %[[RESULT_IMAG_SPECIAL_CASE_2]]
+! IMPRVD: %[[RESULT_REAL_IS_NAN:.*]] = fcmp uno double %[[RESULT_REAL]], 0.000000e+00
+! IMPRVD: %[[RESULT_IMAG_IS_NAN:.*]] = fcmp uno double %[[RESULT_IMAG]], 0.000000e+00
+! IMPRVD: %[[RESULT_IS_NAN:.*]] = and i1 %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]]
+! IMPRVD: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], double %[[RESULT_REAL_SPECIAL_CASE_1]], double %[[RESULT_REAL]]
+! IMPRVD: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = select i1 %[[RESULT_IS_NAN]], double %[[RESULT_IMAG_SPECIAL_CASE_1]], double %[[RESULT_IMAG]]
+! IMPRVD: %[[RESULT_1:.*]] = insertvalue { double, double } poison, double %[[RESULT_REAL_WITH_SPECIAL_CASES]], 0
+! IMPRVD: %[[RESULT_2:.*]] = insertvalue { double, double } %[[RESULT_1]], double %[[RESULT_IMAG_WITH_SPECIAL_CASES]], 1
+! IMPRVD: store { double, double } %[[RESULT_2]], ptr %[[RET]], align 8
+
+! BASIC-DAG: %[[RHS_REAL_SQ:.*]] = fmul contract double %[[RHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[RHS_IMAG_SQ:.*]] = fmul contract double %[[RHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[SQ_NORM:.*]] = fadd contract double %[[RHS_REAL_SQ]], %[[RHS_IMAG_SQ]]
+! BASIC-DAG: %[[REAL_TMP_0:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_REAL]]
+! BASIC-DAG: %[[REAL_TMP_1:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_IMAG]]
+! BASIC: %[[REAL_TMP_2:.*]] = fadd contract double %[[REAL_TMP_0]], %[[REAL_TMP_1]]
+! BASIC-DAG: %[[IMAG_TMP_0:.*]] = fmul contract double %[[LHS_IMAG]], %[[RHS_REAL]]
+! BASIC-DAG: %[[IMAG_TMP_1:.*]] = fmul contract double %[[LHS_REAL]], %[[RHS_IMAG]]
+! BASIC: %[[IMAG_TMP_2:.*]] = fsub contract double %[[IMAG_TMP_0]], %[[IMAG_TMP_1]]
+! BASIC: %[[REAL:.*]] = fdiv contract double %[[REAL_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[IMAG:.*]] = fdiv contract double %[[IMAG_TMP_2]], %[[SQ_NORM]]
+! BASIC: %[[RESULT_1:.*]] = insertvalue { double, double } poison, double %[[REAL]], 0
+! BASIC: %[[RESULT_2:.*]] = insertvalue { double, double } %[[RESULT_1]], double %[[IMAG]], 1
+! BASIC: store { double, double } %[[RESULT_2]], ptr %[[RET]], align 8
+
+! CHECK: ret void
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
new file mode 100644
index 0000000000000..0e219e30ec94b
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind10.f90
@@ -0,0 +1,34 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: target=x86_64{{.*}}
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+! CHECK-LABEL: @_QPdiv_test_extended
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f80>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEa"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEb"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_extendedEc"} : (!fir.ref<complex<f80>>, !fir.dscope) -> (!fir.ref<complex<f80>>, !fir.ref<complex<f80>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f80>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f80>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f80>) -> f80
+! FULL: %[[VAL_RET:.*]] = fir.call @__divxc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f80, f80, f80, f80) -> complex<f80>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f80>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f80>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f80>, !fir.ref<complex<f80>>
+! CHECK: return
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
new file mode 100644
index 0000000000000..3e158b0bed998
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
@@ -0,0 +1,35 @@
+! Test lowering of complex division according to options
+
+! REQUIRES: flang-supports-f128-math
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_quad
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f128>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEa"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEb"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_quadEc"} : (!fir.ref<complex<f128>>, !fir.dscope) -> (!fir.ref<complex<f128>>, !fir.ref<complex<f128>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f128>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f128>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f128>) -> f128
+! FULL: %[[VAL_RET:.*]] = fir.call @__divtc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f128, f128, f128, f128) -> complex<f128>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f128>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f128>, !fir.ref<complex<f128>>
+! CHECK: return
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
+  a = b / c
+end subroutine div_test_quad
\ No newline at end of file
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90
new file mode 100644
index 0000000000000..b488bfde4ee85
--- /dev/null
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir.f90
@@ -0,0 +1,94 @@
+! Test lowering of complex division according to options
+
+! RUN: bbc -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: bbc -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: bbc -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+! RUN: %flang_fc1 -complex-range=full -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,FULL
+! RUN: %flang_fc1 -complex-range=improved -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,IMPRVD
+! RUN: %flang_fc1 -complex-range=basic -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,BASIC
+
+
+! CHECK-LABEL: @_QPdiv_test_half
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f16>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEa"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEb"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_halfEc"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f16>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f16>>
+! CHECK: %[[VAL_9:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f16>
+! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : complex<f16>, !fir.ref<complex<f16>>
+! CHECK: return
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+
+! CHECK-LABEL: @_QPdiv_test_bfloat
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<bf16>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEa"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEb"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_bfloatEc"} : (!fir.ref<complex<bf16>>, !fir.dscope) -> (!fir.ref<complex<bf16>>, !fir.ref<complex<bf16>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<bf16>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<bf16>>
+! CHECK: %[[VAL_9:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<bf16>
+! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : complex<bf16>, !fir.ref<complex<bf16>>
+! CHECK: return
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+
+! CHECK-LABEL: @_QPdiv_test_single
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f32>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEa"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEb"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_singleEc"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f32>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f32>) -> f32
+! FULL: %[[VAL_RET:.*]] = fir.call @__divsc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f32, f32, f32, f32) -> complex<f32>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f32>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f32>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f32>, !fir.ref<complex<f32>>
+! CHECK: return
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+
+! CHECK-LABEL: @_QPdiv_test_double
+! CHECK-SAME: %[[REF_0:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[REF_1:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[REF_2:.*]]: !fir.ref<complex<f64>> {{.*}})
+! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[REF_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEa"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[REF_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEb"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[REF_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFdiv_test_doubleEc"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f64>>
+! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f64>>
+
+! FULL: %[[VAL_9:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_11:.*]] = fir.extract_value %[[VAL_8]], [0 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_12:.*]] = fir.extract_value %[[VAL_8]], [1 : index] : (complex<f64>) -> f64
+! FULL: %[[VAL_RET:.*]] = fir.call @__divdc3(%[[VAL_9]], %[[VAL_10]], %[[VAL_11]], %[[VAL_12]]) fastmath<contract> : (f64, f64, f64, f64) -> complex<f64>
+
+! IMPRVD: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f64>
+! BASIC: %[[VAL_RET:.*]] = complex.div %[[VAL_7]], %[[VAL_8]] fastmath<contract> : complex<f64>
+
+! CHECK: hlfir.assign %[[VAL_RET]] to %[[VAL_4]]#0 : complex<f64>, !fir.ref<complex<f64>>
+! CHECK: return
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 59372a8eb58ed..2ccbe8b96d805 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -276,6 +276,12 @@ static llvm::cl::opt<bool>
                                      "leading dimension will be repacked"),
                       llvm::cl::init(true));
 
+static llvm::cl::opt<std::string> complexRange(
+    "complex-range",
+    llvm::cl::desc("Controls the various implementations for complex "
+                   "multiplication and division [full|improved|basic]"),
+    llvm::cl::init(""));
+
 #define FLANG_EXCLUDE_CODEGEN
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
@@ -437,6 +443,8 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setSkipExternalRttiDefinition(skipExternalRttiDefinition);
   if (enableCUDA)
     loweringOptions.setCUDARuntimeCheck(true);
+  if (complexRange == "improved" || complexRange == "basic")
+    loweringOptions.setComplexDivisionToRuntime(false);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   Fortran::frontend::TargetOptions targetOpts;
   Fortran::frontend::CodeGenOptions cgOpts;

>From 60e0cd9ece16f0f29ce2525d72e2ba37c0ae8447 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 10:19:12 +0900
Subject: [PATCH 2/8] Add help text

---
 clang/include/clang/Driver/Options.td | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 58209ceb5dc54..a1350e69a202e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1025,7 +1025,8 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block",
 def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">,
-  NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>;
+  NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>,
+  HelpText<"Controls the calculation methods of complex number multiplication and division.">;
 
 def complex_range_EQ : Joined<["-"], "complex-range=">, Group<f_Group>,
   Visibility<[CC1Option, FC1Option]>,

>From bddb2d33369ca4bdec50c3e927004555a7940d9a Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 12:01:16 +0900
Subject: [PATCH 3/8] Move common functions to CommonArgs.cpp

---
 clang/include/clang/Driver/CommonArgs.h    |  6 +++++
 clang/lib/Driver/ToolChains/Clang.cpp      | 27 ----------------------
 clang/lib/Driver/ToolChains/CommonArgs.cpp | 27 ++++++++++++++++++++++
 clang/lib/Driver/ToolChains/Flang.cpp      | 24 -------------------
 4 files changed, 33 insertions(+), 51 deletions(-)

diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index 26aa3ccf84786..cd1f3b2eb36ea 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -283,6 +283,12 @@ StringRef parseMPreferVectorWidthOption(clang::DiagnosticsEngine &Diags,
 StringRef parseMRecipOption(clang::DiagnosticsEngine &Diags,
                             const llvm::opt::ArgList &Args);
 
+// Convert ComplexRangeKind to a string that can be passed as a frontend option.
+std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range);
+
+// Render a frontend option corresponding to ComplexRangeKind.
+std::string RenderComplexRangeOption(LangOptions::ComplexRangeKind Range);
+
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index fea4ee909ff46..5d42ce56926d8 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2740,25 +2740,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
   }
 }
 
-static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
-  switch (Range) {
-  case LangOptions::ComplexRangeKind::CX_Full:
-    return "full";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Basic:
-    return "basic";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Improved:
-    return "improved";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Promoted:
-    return "promoted";
-    break;
-  default:
-    return "";
-  }
-}
-
 static std::string ComplexArithmeticStr(LangOptions::ComplexRangeKind Range) {
   return (Range == LangOptions::ComplexRangeKind::CX_None)
              ? ""
@@ -2772,14 +2753,6 @@ static void EmitComplexRangeDiag(const Driver &D, std::string str1,
   }
 }
 
-static std::string
-RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
-  if (!ComplexRangeStr.empty())
-    return "-complex-range=" + ComplexRangeStr;
-  return ComplexRangeStr;
-}
-
 static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
                                        bool OFastEnabled, const ArgList &Args,
                                        ArgStringList &CmdArgs,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 070901f037823..a6735e5e412d4 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -3409,3 +3409,30 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
 
   return Out;
 }
+
+std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    return "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Basic:
+    return "basic";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Improved:
+    return "improved";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Promoted:
+    return "promoted";
+    break;
+  default:
+    return "";
+  }
+}
+
+std::string
+tools::RenderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+  if (!ComplexRangeStr.empty())
+    return "-complex-range=" + ComplexRangeStr;
+  return ComplexRangeStr;
+}
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index ba5db7e3aba43..b5767918b86d1 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -595,30 +595,6 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
   addOpenMPHostOffloadingArgs(C, JA, Args, CmdArgs);
 }
 
-static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
-  switch (Range) {
-  case LangOptions::ComplexRangeKind::CX_Full:
-    return "full";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Improved:
-    return "improved";
-    break;
-  case LangOptions::ComplexRangeKind::CX_Basic:
-    return "basic";
-    break;
-  default:
-    return "";
-  }
-}
-
-static std::string
-RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
-  if (!ComplexRangeStr.empty())
-    return "-complex-range=" + ComplexRangeStr;
-  return ComplexRangeStr;
-}
-
 static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
                                     ArgStringList &CmdArgs) {
   StringRef FPContract;

>From 65c522ccd201fd48c481990ac0b1ec9b8be6d163 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 12:01:54 +0900
Subject: [PATCH 4/8] Minor fixes

---
 flang/lib/Frontend/CompilerInvocation.cpp              | 2 +-
 flang/lib/Lower/ConvertExprToHLFIR.cpp                 | 5 +++--
 flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 86ec410b1f70f..910e0f5dcebd5 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -485,7 +485,7 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
 
   parseDoConcurrentMapping(opts, args, diags);
 
-  if (const auto *arg =
+  if (const llvm::opt::Arg *arg =
           args.getLastArg(clang::driver::options::OPT_complex_range_EQ)) {
     llvm::StringRef argValue = llvm::StringRef(arg->getValue());
     if (argValue == "full") {
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index cb338618dbf3b..9689f920840fb 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1070,12 +1070,13 @@ struct BinaryOp<Fortran::evaluate::Divide<
     // TODO: Ideally, complex number division operations should always be
     // lowered to MLIR. However, converting them to the runtime via MLIR causes
     // ABI issues.
-    if (builder.getComplexDivisionToRuntimeFlag())
+    if (builder.getComplexDivisionToRuntimeFlag()) {
       return hlfir::EntityWithAttributes{
           fir::genDivC(builder, loc, ty, lhs, rhs)};
-    else
+    } else {
       return hlfir::EntityWithAttributes{
           builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
+    }
   }
 };
 
diff --git a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90 b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
index 3e158b0bed998..fe4a7256a4a16 100644
--- a/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
+++ b/flang/test/Lower/HLFIR/complex-div-to-hlfir-kind16.f90
@@ -32,4 +32,4 @@
 subroutine div_test_quad(a,b,c)
   complex(kind=16) :: a, b, c
   a = b / c
-end subroutine div_test_quad
\ No newline at end of file
+end subroutine div_test_quad

>From 528e1defac6714baad3f80a6e031fe02fb18011c Mon Sep 17 00:00:00 2001
From: Shunsuke Watanabe <watanabe.shu-06 at fujitsu.com>
Date: Fri, 4 Jul 2025 17:41:45 +0900
Subject: [PATCH 5/8] Update flang/include/flang/Optimizer/Builder/FIRBuilder.h

Co-authored-by: Tarun Prabhu <tarunprabhu at gmail.com>
---
 flang/include/flang/Optimizer/Builder/FIRBuilder.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index b1513850a9048..004ad00b6f470 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -609,8 +609,8 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return integerOverflowFlags;
   }
 
-  /// Set ComplexDivisionToRuntimeFlag value for whether complex number division
-  /// is lowered to a runtime function by this builder.
+  /// Set ComplexDivisionToRuntimeFlag value. If set to true, complex number
+  /// division is lowered to a runtime function by this builder.
   void setComplexDivisionToRuntimeFlag(bool flag) {
     complexDivisionToRuntimeFlag = flag;
   }

>From eea4a5f402cddae02c31a1217af6930fa4981874 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Mon, 7 Jul 2025 18:27:04 +0900
Subject: [PATCH 6/8] Add lowering information to ComplexOperations.md

---
 flang/docs/ComplexOperations.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/flang/docs/ComplexOperations.md b/flang/docs/ComplexOperations.md
index 03566035c28a3..b7d2ecaa85e1c 100644
--- a/flang/docs/ComplexOperations.md
+++ b/flang/docs/ComplexOperations.md
@@ -74,4 +74,26 @@ The ComplexToStandard dialect does still call into libm for some floating
 point math operations, however these don't have the same ABI issues as the
 complex libm functions.
 
+The flang driver option `-fcomplex-arithmetic=` allows you to select whether
+complex number division is lowered to function calls or to the `complex.div`
+operation in the MLIR complex dialect. To avoid the ABI issues mentioned above,
+the selection between function calls and the `complex.div` is made during the
+lowering phase. The behavior of this option is as follows:
+
+- `basic`: Lowers to `complex.div` and is converted to algebraic formulas. No
+special handling to avoid overflow. NaN and infinite values are not handled.
+- `improved`: Lowers to `complex.div` and is converted to Smith's algorithm. See
+SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962). This
+value offers improved handling for overflow in intermediate calculations, but
+overflow may occur. NaN and infinite values are handled.
+- `full`: Lowers to a call to runtime library functions. Overflow and non-finite
+values are handled by the library implementation. This is the default value.
+
+While [the same option in clang][2] allows specifying `promoted`, this is not
+implemented in Flang. Also, in the case of `improved`, clang does not handle NaN
+and infinite values, but Flang does. These behavioral differences arise because
+the transformation of complex division calculations depends on the implementation
+of ComplexToStandard, which may change in the future.
+
 [1]: https://discourse.llvm.org/t/rfc-change-lowering-of-fortran-math-intrinsics/63971
+[2]: https://clang.llvm.org/docs/UsersManual.html#cmdoption-fcomplex-arithmetic

>From e94dad223f75882bfed40bd7dce651549db31523 Mon Sep 17 00:00:00 2001
From: s-watanabe314 <watanabe.shu-06 at fujitsu.com>
Date: Tue, 8 Jul 2025 10:30:06 +0900
Subject: [PATCH 7/8] fix function names

---
 clang/include/clang/Driver/CommonArgs.h    |  4 ++--
 clang/lib/Driver/ToolChains/Clang.cpp      | 14 +++++++-------
 clang/lib/Driver/ToolChains/CommonArgs.cpp |  6 +++---
 clang/lib/Driver/ToolChains/Flang.cpp      |  4 ++--
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index cd1f3b2eb36ea..49c7149380fba 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -284,10 +284,10 @@ StringRef parseMRecipOption(clang::DiagnosticsEngine &Diags,
                             const llvm::opt::ArgList &Args);
 
 // Convert ComplexRangeKind to a string that can be passed as a frontend option.
-std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range);
+std::string complexRangeKindToStr(LangOptions::ComplexRangeKind Range);
 
 // Render a frontend option corresponding to ComplexRangeKind.
-std::string RenderComplexRangeOption(LangOptions::ComplexRangeKind Range);
+std::string renderComplexRangeOption(LangOptions::ComplexRangeKind Range);
 
 } // end namespace tools
 } // end namespace driver
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 5d42ce56926d8..df80565867f48 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2743,7 +2743,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
 static std::string ComplexArithmeticStr(LangOptions::ComplexRangeKind Range) {
   return (Range == LangOptions::ComplexRangeKind::CX_None)
              ? ""
-             : "-fcomplex-arithmetic=" + ComplexRangeKindToStr(Range);
+             : "-fcomplex-arithmetic=" + complexRangeKindToStr(Range);
 }
 
 static void EmitComplexRangeDiag(const Driver &D, std::string str1,
@@ -2885,7 +2885,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     case options::OPT_fcx_limited_range:
       if (GccRangeComplexOption.empty()) {
         if (Range != LangOptions::ComplexRangeKind::CX_Basic)
-          EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
+          EmitComplexRangeDiag(D, renderComplexRangeOption(Range),
                                "-fcx-limited-range");
       } else {
         if (GccRangeComplexOption != "-fno-cx-limited-range")
@@ -2897,7 +2897,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       break;
     case options::OPT_fno_cx_limited_range:
       if (GccRangeComplexOption.empty()) {
-        EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
+        EmitComplexRangeDiag(D, renderComplexRangeOption(Range),
                              "-fno-cx-limited-range");
       } else {
         if (GccRangeComplexOption != "-fcx-limited-range" &&
@@ -2911,7 +2911,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       break;
     case options::OPT_fcx_fortran_rules:
       if (GccRangeComplexOption.empty())
-        EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
+        EmitComplexRangeDiag(D, renderComplexRangeOption(Range),
                              "-fcx-fortran-rules");
       else
         EmitComplexRangeDiag(D, GccRangeComplexOption, "-fcx-fortran-rules");
@@ -2921,7 +2921,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       break;
     case options::OPT_fno_cx_fortran_rules:
       if (GccRangeComplexOption.empty()) {
-        EmitComplexRangeDiag(D, RenderComplexRangeOption(Range),
+        EmitComplexRangeDiag(D, renderComplexRangeOption(Range),
                              "-fno-cx-fortran-rules");
       } else {
         if (GccRangeComplexOption != "-fno-cx-limited-range")
@@ -3389,12 +3389,12 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     CmdArgs.push_back("-fno-strict-float-cast-overflow");
 
   if (Range != LangOptions::ComplexRangeKind::CX_None)
-    ComplexRangeStr = RenderComplexRangeOption(Range);
+    ComplexRangeStr = renderComplexRangeOption(Range);
   if (!ComplexRangeStr.empty()) {
     CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
     if (Args.hasArg(options::OPT_fcomplex_arithmetic_EQ))
       CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" +
-                                           ComplexRangeKindToStr(Range)));
+                                           complexRangeKindToStr(Range)));
   }
   if (Args.hasArg(options::OPT_fcx_limited_range))
     CmdArgs.push_back("-fcx-limited-range");
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index a6735e5e412d4..b5543a7eb738d 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -3410,7 +3410,7 @@ StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
   return Out;
 }
 
-std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
+std::string tools::complexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
   switch (Range) {
   case LangOptions::ComplexRangeKind::CX_Full:
     return "full";
@@ -3430,8 +3430,8 @@ std::string tools::ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) {
 }
 
 std::string
-tools::RenderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
-  std::string ComplexRangeStr = ComplexRangeKindToStr(Range);
+tools::renderComplexRangeOption(LangOptionsBase::ComplexRangeKind Range) {
+  std::string ComplexRangeStr = complexRangeKindToStr(Range);
   if (!ComplexRangeStr.empty())
     return "-complex-range=" + ComplexRangeStr;
   return ComplexRangeStr;
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index b5767918b86d1..3c73f73ac9acf 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -716,10 +716,10 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
     CmdArgs.push_back(Args.MakeArgString("-mrecip=" + Recip));
 
   if (Range != LangOptions::ComplexRangeKind::CX_None) {
-    std::string ComplexRangeStr = RenderComplexRangeOption(Range);
+    std::string ComplexRangeStr = renderComplexRangeOption(Range);
     CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
     CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" +
-                                         ComplexRangeKindToStr(Range)));
+                                         complexRangeKindToStr(Range)));
   }
 
   if (!HonorINFs && !HonorNaNs && AssociativeMath && ReciprocalMath &&

>From af90b7a6ac72ca08a37312f1b7362ef0142faeac Mon Sep 17 00:00:00 2001
From: Shunsuke Watanabe <watanabe.shu-06 at fujitsu.com>
Date: Tue, 8 Jul 2025 15:35:11 +0900
Subject: [PATCH 8/8] Update flang/docs/ComplexOperations.md

Co-authored-by: Tarun Prabhu <tarunprabhu at gmail.com>
---
 flang/docs/ComplexOperations.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/docs/ComplexOperations.md b/flang/docs/ComplexOperations.md
index b7d2ecaa85e1c..3ebeea5e0a540 100644
--- a/flang/docs/ComplexOperations.md
+++ b/flang/docs/ComplexOperations.md
@@ -77,7 +77,7 @@ complex libm functions.
 The flang driver option `-fcomplex-arithmetic=` allows you to select whether
 complex number division is lowered to function calls or to the `complex.div`
 operation in the MLIR complex dialect. To avoid the ABI issues mentioned above,
-the selection between function calls and the `complex.div` is made during the
+the choice of function calls or the `complex.div` operation is made during the
 lowering phase. The behavior of this option is as follows:
 
 - `basic`: Lowers to `complex.div` and is converted to algebraic formulas. No


From flang-commits at lists.llvm.org  Tue Jul  8 00:54:06 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 00:54:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686cce9e.170a0220.23dfe0.18da@mx.google.com>


================
@@ -805,13 +805,12 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
+      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
----------------
jeanPerier wrote:

I think it is slightly unsafe to assume this cannot be a fir.heap/fir.ptr. Nothing prevents these types from describing constant shaped compiler generated objects. Please use `fir::unwrapRefType()` instead. 

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Tue Jul  8 00:54:06 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 00:54:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686cce9e.170a0220.b3c0c.b4e6@mx.google.com>


https://github.com/jeanPerier approved this pull request.


https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Tue Jul  8 02:02:03 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 02:02:03 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686cde8b.050a0220.171fcb.d7ed@mx.google.com>


Stylie777 wrote:

Apologies @kparzysz @ronlieb I am so far unable to reproduce this issue. I ran the reproducer with the command provided 120 times and it has not crashed.

 I agree that where it sets `nextIt` could be an issue, and a check should be added here that if the end of the `block` has been reached, the process returns. Something like this after each time `block.erase` is used.

```
if (nextIt == block.end()) {
  return;
}
```

Is there any other changes you have applied ontop of mine that could be causing this?

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Tue Jul  8 03:52:31 2025
From: flang-commits at lists.llvm.org (Mehdi Amini via flang-commits)
Date: Tue, 08 Jul 2025 03:52:31 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][tblgen] add concrete create
	methods (PR #147168)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147168@github.com>
Message-ID: <686cf86f.170a0220.12d334.bfd7@mx.google.com>


joker-eph wrote:

> Note the removed let builders = are dangling (they have no impls).

Do you mind just pushing this already? This is general cleanup we should lock-in independently of the rest of the work here.

https://github.com/llvm/llvm-project/pull/147168

From flang-commits at lists.llvm.org  Tue Jul  8 04:01:58 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 04:01:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour in
	Interval::Contains (PR #147505)
Message-ID: <llvm.org/llvm/llvm-project/pull/147505@github.com>


https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/147505

If the size of the other Interval was 0, (that.size_ - 1) would wrap below zero.

I've fixed this so that a zero size interval A is within interval B if the start of A is within B. There's a few ways you could handle zero sized intervals in theory but this one passes all tests so I assume it's the intention.

This fixes the following tests when ubsan is enabled:
  Flang :: Lower/OpenMP/PFT/sections-pft.f90
  Flang :: Lower/OpenMP/derived-type-allocatable.f90
  Flang :: Lower/OpenMP/privatization-proc-ptr.f90
  Flang :: Lower/OpenMP/sections.f90
  Flang :: Parser/OpenMP/sections.f90
  Flang :: Semantics/OpenMP/clause-validity01.f90
  Flang :: Semantics/OpenMP/if-clause.f90
  Flang :: Semantics/OpenMP/parallel-sections01.f90
  Flang :: Semantics/OpenMP/private-assoc.f90

>From 128db9d580e102282cf820c3e21ffb62af8e217c Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Tue, 8 Jul 2025 10:57:14 +0000
Subject: [PATCH] [flang] Avoid undefined behaviour in Interval::Contains

If the size of the other Interval was 0, (that.size_ - 1)
would wrap below zero.

I've fixed this so that a zero size interval A is within
interval B if the start of A is within B.

There's a few ways you could handle zero sized intervals
in theory but this one passes all tests so I assume it's
the intention.

This fixes the following tests when ubsan is enabled:
  Flang :: Lower/OpenMP/PFT/sections-pft.f90
  Flang :: Lower/OpenMP/derived-type-allocatable.f90
  Flang :: Lower/OpenMP/privatization-proc-ptr.f90
  Flang :: Lower/OpenMP/sections.f90
  Flang :: Parser/OpenMP/sections.f90
  Flang :: Semantics/OpenMP/clause-validity01.f90
  Flang :: Semantics/OpenMP/if-clause.f90
  Flang :: Semantics/OpenMP/parallel-sections01.f90
  Flang :: Semantics/OpenMP/private-assoc.f90
---
 flang/include/flang/Common/interval.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Common/interval.h b/flang/include/flang/Common/interval.h
index c4cab0ccf1130..2f0b6d9b55fe5 100644
--- a/flang/include/flang/Common/interval.h
+++ b/flang/include/flang/Common/interval.h
@@ -60,7 +60,8 @@ template <typename A> class Interval {
     return start_ <= x && x < start_ + size_;
   }
   constexpr bool Contains(const Interval &that) const {
-    return Contains(that.start_) && Contains(that.start_ + (that.size_ - 1));
+    return Contains(that.start_) &&
+        ((that.size_ == 0) || Contains(that.start_ + (that.size_ - 1)));
   }
   constexpr bool IsDisjointWith(const Interval &that) const {
     return that.NextAfter() <= start_ || NextAfter() <= that.start_;


From flang-commits at lists.llvm.org  Tue Jul  8 05:29:31 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 08 Jul 2025 05:29:31 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686d0f2b.050a0220.21ac2c.d265@mx.google.com>


kparzysz wrote:

Add `assert(nextIt != block.end())` after the line I showed and it should fail with the short testcase.

I'm pretty sure that all other cases where nextIt changes go back to the beginning of the loop where nextIt != block.end() is checked.  I wrapped the following "if" in "if (nextIt != block.end())" and that fixed the problem.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Tue Jul  8 05:46:15 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 05:46:15 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686d1317.050a0220.2b20ec.f008@mx.google.com>


DavidSpickett wrote:

I think this is the cause of https://github.com/llvm/llvm-project/issues/147309.

On Windows on Arm a few tests fail pretty consistently, and the backtrace points here. @kparzysz's assessment seems to be correct.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Tue Jul  8 05:49:05 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 05:49:05 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686d13c1.170a0220.19bdfc.cc3f@mx.google.com>


Stylie777 wrote:

Ok, I will post a fix up or review and tag you both. Thanks all.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Tue Jul  8 05:49:33 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 05:49:33 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686d13dd.170a0220.28af06.d99e@mx.google.com>


DavidSpickett wrote:

> Apologies @kparzysz @ronlieb I am so far unable to reproduce this issue. I ran the reproducer with the command provided 120 times and it has not crashed.

I also hammered the tests on AArch64 Ubuntu Linux and never found a failure but it would fail within a few runs on Windows on Arm. So there is some unknown factor, perhaps memory allocation if the list has to reallocate as a result of the erase.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Tue Jul  8 06:00:51 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 06:00:51 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686d1683.630a0220.2a2123.26e6@mx.google.com>


DavidSpickett wrote:

> In this one case I wrapped the following "if" in "if (nextIt != block.end())" and that fixed the problem.

On Windows on Arm, the following hack passes all tests and hammering the specific tests produced no failures:
```
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 1edcb376596b..f33d161d6870 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -151,12 +151,14 @@ private:
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
+          if (nextIt != block.end()) {
           if (auto *endDir{
                   GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
             std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
                 std::move(*endDir);
             nextIt = block.erase(nextIt);
           }
+          }
         } else {
           messages_.Say(dir.source,
               "DO loop after the %s directive must have loop control"_err_en_US,
```

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Tue Jul  8 06:02:49 2025
From: flang-commits at lists.llvm.org (Mehdi Amini via flang-commits)
Date: Tue, 08 Jul 2025 06:02:49 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][tblgen] add concrete create
	methods (PR #147168)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147168@github.com>
Message-ID: <686d16f9.170a0220.34d2ee.3b8d@mx.google.com>


joker-eph wrote:

I'm OK with this direction. We need to make a call on what to do about `ImplicitLocOpBuilder.h`.

Overall I'd like to hear more about this from @jpienaar @ftynse @j2kun (maybe @Mogball and @River707 ?) : are we OK to transition from `builder.create<Op>(...)` to `Op::create(builder, ...)`  for the benefit of better IDE support?

As far as I understand this is now perfectly backward compatible with existing hand-written `build` methods, so it can be a scripted mass replacement.

https://github.com/llvm/llvm-project/pull/147168

From flang-commits at lists.llvm.org  Tue Jul  8 06:08:12 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Tue, 08 Jul 2025 06:08:12 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][tblgen] add concrete create
	methods (PR #147168)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147168@github.com>
Message-ID: <686d183c.170a0220.e1701.d4d7@mx.google.com>


makslevental wrote:

> I'm OK with this direction. We need to make a call on what to do about `ImplicitLocOpBuilder.h`.

Some reasonable options in here I think? https://stackoverflow.com/questions/68306977/how-can-i-deprecate-a-c-header

https://github.com/llvm/llvm-project/pull/147168

From flang-commits at lists.llvm.org  Tue Jul  8 06:13:55 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 06:13:55 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686d1993.630a0220.24449b.f584@mx.google.com>


Stylie777 wrote:

> I also hammered the tests on AArch64 Ubuntu Linux and never found a failure but it would fail within a few runs on Windows on Arm.

I tested it on X86 Ubuntu and found the same. I don't have access to Windows on Arm but that may explain why I could not reproduce the issue.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Tue Jul  8 06:20:38 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 06:20:38 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
Message-ID: <llvm.org/llvm/llvm-project/pull/147519@github.com>


https://github.com/Stylie777 created https://github.com/llvm/llvm-project/pull/147519

As reported in #145917 and #147309, there are situation's where flang may crash. This is because `nextIt` in
`RewriteOpenMPLoopConstruct` gets re-assigned when an iterator is erased from the block. If this is missed, Flang may attempt to access a location in memory that is not accessable and cause a compiler crash.

This adds protection where the crash can occur, and a test with a reproducer that can trigger the crash.

Fixes #147309

>From fc212fa8385aefb133a93fbb88a64a44f7d27aef Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 8 Jul 2025 14:10:04 +0100
Subject: [PATCH] [Flang][OpenMP] Fix crash when block.end() is missed

As reported in #145917 and #147309, there are situation's where
flang may crash. This is because `nextIt` in
`RewriteOpenMPLoopConstruct` gets re-assigned when an iterator is
erased from the block. If this is missed, Flang may attempt to access
a location in memory that is not accessable and cause a compiler crash.

This adds protection where the crash can occur, and a test with a
reproducer that can trigger the crash.

Fixes #147309
---
 flang/lib/Semantics/canonicalize-omp.cpp      | 12 +--
 .../loop-transformation-construct03.f90       | 75 +++++++++++++++++++
 2 files changed, 82 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/loop-transformation-construct03.f90

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 1edcb376596b0..f6e21babc1186 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -151,11 +151,13 @@ class CanonicalizationOfOmp {
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if (auto *endDir{
-                  GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
-            std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
-                std::move(*endDir);
-            nextIt = block.erase(nextIt);
+          if(nextIt != block.end()) {
+            if (auto *endDir{
+                    GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
+              std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                  std::move(*endDir);
+              nextIt = block.erase(nextIt);
+            }
           }
         } else {
           messages_.Say(dir.source,
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct03.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct03.f90
new file mode 100644
index 0000000000000..81f0de1b76263
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct03.f90
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = target teams distribute parallel do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList -> OmpClause -> Collapse -> Scalar -> Integer -> Constant -> Expr = '2_4'
+!CHECK-PARSE-NEXT: | | | | | LiteralConstant -> IntLiteralConstant = '2'
+!CHECK-PARSE-NEXT: | | | | OmpClause -> Private -> OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Expr = '10_4'
+!CHECK-PARSE-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '10'
+!CHECK-PARSE-NEXT: | | | | Block
+!CHECK-PARSE-NEXT: | | | | | ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '10_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '10'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'a(int(b,kind=8),2_8)=a(int(c,kind=8),1_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'a(int(b,kind=8),2_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'a'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'b'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = '2_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '2'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = 'a(int(c,kind=8),1_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'a'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | EndSubroutineStmt ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct7
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  REAL(KIND=8_4), DIMENSION(1_4:10_4,2_4) :: a
+!CHECK-UNPARSE-NEXT:  INTEGER b, c
+!CHECK-UNPARSE-NEXT: !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO  COLLAPSE(2_4) PRIVATE(b)
+!CHECK-UNPARSE-NEXT:  DO b=1_4,10_4
+!CHECK-UNPARSE-NEXT:   DO c=1_4,10_4
+!CHECK-UNPARSE-NEXT:     a(int(b,kind=8),2_8)=a(int(c,kind=8),1_8)
+!CHECK-UNPARSE-NEXT:   END DO
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE


From flang-commits at lists.llvm.org  Tue Jul  8 06:21:07 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 06:21:07 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d1b43.050a0220.3cf462.f978@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-parser

@llvm/pr-subscribers-flang-openmp

Author: Jack Styles (Stylie777)

<details>
<summary>Changes</summary>

As reported in #<!-- -->145917 and #<!-- -->147309, there are situation's where flang may crash. This is because `nextIt` in
`RewriteOpenMPLoopConstruct` gets re-assigned when an iterator is erased from the block. If this is missed, Flang may attempt to access a location in memory that is not accessable and cause a compiler crash.

This adds protection where the crash can occur, and a test with a reproducer that can trigger the crash.

Fixes #<!-- -->147309

---
Full diff: https://github.com/llvm/llvm-project/pull/147519.diff


2 Files Affected:

- (modified) flang/lib/Semantics/canonicalize-omp.cpp (+7-5) 
- (added) flang/test/Parser/OpenMP/loop-transformation-construct03.f90 (+75) 


``````````diff
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 1edcb376596b0..f6e21babc1186 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -151,11 +151,13 @@ class CanonicalizationOfOmp {
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if (auto *endDir{
-                  GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
-            std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
-                std::move(*endDir);
-            nextIt = block.erase(nextIt);
+          if(nextIt != block.end()) {
+            if (auto *endDir{
+                    GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
+              std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                  std::move(*endDir);
+              nextIt = block.erase(nextIt);
+            }
           }
         } else {
           messages_.Say(dir.source,
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct03.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct03.f90
new file mode 100644
index 0000000000000..81f0de1b76263
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct03.f90
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = target teams distribute parallel do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList -> OmpClause -> Collapse -> Scalar -> Integer -> Constant -> Expr = '2_4'
+!CHECK-PARSE-NEXT: | | | | | LiteralConstant -> IntLiteralConstant = '2'
+!CHECK-PARSE-NEXT: | | | | OmpClause -> Private -> OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Expr = '10_4'
+!CHECK-PARSE-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '10'
+!CHECK-PARSE-NEXT: | | | | Block
+!CHECK-PARSE-NEXT: | | | | | ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '10_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '10'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'a(int(b,kind=8),2_8)=a(int(c,kind=8),1_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'a(int(b,kind=8),2_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'a'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'b'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = '2_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '2'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = 'a(int(c,kind=8),1_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'a'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | EndSubroutineStmt ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct7
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  REAL(KIND=8_4), DIMENSION(1_4:10_4,2_4) :: a
+!CHECK-UNPARSE-NEXT:  INTEGER b, c
+!CHECK-UNPARSE-NEXT: !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO  COLLAPSE(2_4) PRIVATE(b)
+!CHECK-UNPARSE-NEXT:  DO b=1_4,10_4
+!CHECK-UNPARSE-NEXT:   DO c=1_4,10_4
+!CHECK-UNPARSE-NEXT:     a(int(b,kind=8),2_8)=a(int(c,kind=8),1_8)
+!CHECK-UNPARSE-NEXT:   END DO
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE

``````````

</details>


https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:21:45 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 06:21:45 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Add Semantics support for
 Nested OpenMPLoopConstructs (PR #145917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145917@github.com>
Message-ID: <686d1b69.a70a0220.248669.f711@mx.google.com>


Stylie777 wrote:

https://github.com/llvm/llvm-project/pull/147519

Fix is up for review. Apologies everyone for the disruption and thanks for the collaboration.

https://github.com/llvm/llvm-project/pull/145917

From flang-commits at lists.llvm.org  Tue Jul  8 06:23:03 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 06:23:03 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d1bb7.630a0220.39ee07.0155@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions cpp -- flang/lib/Semantics/canonicalize-omp.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index f6e21babc..cf05d8463 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -151,7 +151,7 @@ private:
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if(nextIt != block.end()) {
+          if (nextIt != block.end()) {
             if (auto *endDir{
                     GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
               std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =

``````````

</details>


https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:23:22 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 06:23:22 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d1bca.170a0220.1a83c4.e406@mx.google.com>


https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/147519

>From fc212fa8385aefb133a93fbb88a64a44f7d27aef Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 8 Jul 2025 14:10:04 +0100
Subject: [PATCH 1/2] [Flang][OpenMP] Fix crash when block.end() is missed

As reported in #145917 and #147309, there are situation's where
flang may crash. This is because `nextIt` in
`RewriteOpenMPLoopConstruct` gets re-assigned when an iterator is
erased from the block. If this is missed, Flang may attempt to access
a location in memory that is not accessable and cause a compiler crash.

This adds protection where the crash can occur, and a test with a
reproducer that can trigger the crash.

Fixes #147309
---
 flang/lib/Semantics/canonicalize-omp.cpp      | 12 +--
 .../loop-transformation-construct03.f90       | 75 +++++++++++++++++++
 2 files changed, 82 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/loop-transformation-construct03.f90

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 1edcb376596b0..f6e21babc1186 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -151,11 +151,13 @@ class CanonicalizationOfOmp {
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if (auto *endDir{
-                  GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
-            std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
-                std::move(*endDir);
-            nextIt = block.erase(nextIt);
+          if(nextIt != block.end()) {
+            if (auto *endDir{
+                    GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
+              std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                  std::move(*endDir);
+              nextIt = block.erase(nextIt);
+            }
           }
         } else {
           messages_.Say(dir.source,
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct03.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct03.f90
new file mode 100644
index 0000000000000..81f0de1b76263
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct03.f90
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = target teams distribute parallel do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList -> OmpClause -> Collapse -> Scalar -> Integer -> Constant -> Expr = '2_4'
+!CHECK-PARSE-NEXT: | | | | | LiteralConstant -> IntLiteralConstant = '2'
+!CHECK-PARSE-NEXT: | | | | OmpClause -> Private -> OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Expr = '10_4'
+!CHECK-PARSE-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '10'
+!CHECK-PARSE-NEXT: | | | | Block
+!CHECK-PARSE-NEXT: | | | | | ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '10_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '10'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'a(int(b,kind=8),2_8)=a(int(c,kind=8),1_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'a(int(b,kind=8),2_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'a'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'b'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = '2_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '2'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = 'a(int(c,kind=8),1_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'a'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | EndSubroutineStmt ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct7
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  REAL(KIND=8_4), DIMENSION(1_4:10_4,2_4) :: a
+!CHECK-UNPARSE-NEXT:  INTEGER b, c
+!CHECK-UNPARSE-NEXT: !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO  COLLAPSE(2_4) PRIVATE(b)
+!CHECK-UNPARSE-NEXT:  DO b=1_4,10_4
+!CHECK-UNPARSE-NEXT:   DO c=1_4,10_4
+!CHECK-UNPARSE-NEXT:     a(int(b,kind=8),2_8)=a(int(c,kind=8),1_8)
+!CHECK-UNPARSE-NEXT:   END DO
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE

>From 12a5cf1bc0ce4c0aa685c14ba4dead0eb99e9909 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 8 Jul 2025 14:22:24 +0100
Subject: [PATCH 2/2] fix formatting

---
 flang/lib/Semantics/canonicalize-omp.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index f6e21babc1186..cf05d8463277f 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -151,7 +151,7 @@ class CanonicalizationOfOmp {
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if(nextIt != block.end()) {
+          if (nextIt != block.end()) {
             if (auto *endDir{
                     GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
               std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =


From flang-commits at lists.llvm.org  Tue Jul  8 06:29:24 2025
From: flang-commits at lists.llvm.org (David Truby via flang-commits)
Date: Tue, 08 Jul 2025 06:29:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour in
	Interval::Contains (PR #147505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147505@github.com>
Message-ID: <686d1d34.630a0220.bd747.4afa@mx.google.com>


https://github.com/DavidTruby approved this pull request.

LGTM thanks!

https://github.com/llvm/llvm-project/pull/147505

From flang-commits at lists.llvm.org  Tue Jul  8 06:33:36 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Tue, 08 Jul 2025 06:33:36 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d1e30.170a0220.3c27d1.4b77@mx.google.com>


https://github.com/mjklemm approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:39:12 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 06:39:12 -0700 (PDT)
Subject: [flang-commits] [flang] 31786ee - [flang] Avoid undefined behaviour
 in Interval::Contains (#147505)
Message-ID: <686d1f80.050a0220.169cef.f742@mx.google.com>


Author: David Spickett
Date: 2025-07-08T14:39:08+01:00
New Revision: 31786ee89f683721248ef6c41d347a00a5e619f6

URL: https://github.com/llvm/llvm-project/commit/31786ee89f683721248ef6c41d347a00a5e619f6
DIFF: https://github.com/llvm/llvm-project/commit/31786ee89f683721248ef6c41d347a00a5e619f6.diff

LOG: [flang] Avoid undefined behaviour in Interval::Contains (#147505)

If the size of the other Interval was 0, (that.size_ - 1) would wrap
below zero.

I've fixed this so that a zero size interval A is within interval B if
the start of A is within B. There's a few ways you could handle zero
sized intervals in theory but this one passes all tests so I assume it's
the intention.

This fixes the following tests when ubsan is enabled:
  Flang :: Lower/OpenMP/PFT/sections-pft.f90
  Flang :: Lower/OpenMP/derived-type-allocatable.f90
  Flang :: Lower/OpenMP/privatization-proc-ptr.f90
  Flang :: Lower/OpenMP/sections.f90
  Flang :: Parser/OpenMP/sections.f90
  Flang :: Semantics/OpenMP/clause-validity01.f90
  Flang :: Semantics/OpenMP/if-clause.f90
  Flang :: Semantics/OpenMP/parallel-sections01.f90
  Flang :: Semantics/OpenMP/private-assoc.f90

Added: 
    

Modified: 
    flang/include/flang/Common/interval.h

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Common/interval.h b/flang/include/flang/Common/interval.h
index c4cab0ccf1130..2f0b6d9b55fe5 100644
--- a/flang/include/flang/Common/interval.h
+++ b/flang/include/flang/Common/interval.h
@@ -60,7 +60,8 @@ template <typename A> class Interval {
     return start_ <= x && x < start_ + size_;
   }
   constexpr bool Contains(const Interval &that) const {
-    return Contains(that.start_) && Contains(that.start_ + (that.size_ - 1));
+    return Contains(that.start_) &&
+        ((that.size_ == 0) || Contains(that.start_ + (that.size_ - 1)));
   }
   constexpr bool IsDisjointWith(const Interval &that) const {
     return that.NextAfter() <= start_ || NextAfter() <= that.start_;


        

From flang-commits at lists.llvm.org  Tue Jul  8 06:39:15 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 06:39:15 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour in
	Interval::Contains (PR #147505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147505@github.com>
Message-ID: <686d1f83.050a0220.1c1b04.eeff@mx.google.com>


https://github.com/DavidSpickett closed https://github.com/llvm/llvm-project/pull/147505

From flang-commits at lists.llvm.org  Tue Jul  8 06:44:46 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 06:44:46 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d20ce.170a0220.312d9a.ecd1@mx.google.com>


================
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
----------------
DavidSpickett wrote:

Also, is it worth trying a few invalid examples like that to see if flang can handle them? Maybe this is caught earlier than what this test is checking.

Like would a missing `end do` be diagnosed before the code in this PR even runs? If so then no need to worry about that.

https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:44:46 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 06:44:46 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d20ce.170a0220.172838.e76f@mx.google.com>


https://github.com/DavidSpickett approved this pull request.

LGTM, I've tested an equivalent patch on Windows On Arm and it fixed the issue.

https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:44:46 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 06:44:46 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d20ce.170a0220.34cee1.54fc@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:44:47 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 06:44:47 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d20cf.170a0220.35bb2f.e56c@mx.google.com>


================
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
----------------
DavidSpickett wrote:

If the crash happens because there is something missing from this function, then add a comment to say where that thing would be. If this function is valid as is, then no change needed.

(one of my first suspicions was something like `begin bla; stuff;` missing some `end` statement)

https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:52:09 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 06:52:09 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d2289.170a0220.1e7245.58ac@mx.google.com>


================
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
----------------
DavidSpickett wrote:

I suppose it would. A missing `end do` would be invalid Fortran, regardless of OpenMP. So I'd expect that error to be caught earlier.

https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:58:18 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 06:58:18 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d23fa.170a0220.3a56bc.f734@mx.google.com>


================
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
----------------
Stylie777 wrote:

There is not anything missing from the function, it should be able to parse. A missing `end do` is also diagnosed before this stage yes.

Using `block.erase()` would advance the `nextIt` value to the next iterator. This could be `block.end()`. There is then potential for the while loop to never be equal to `block.end()`, and as such cause the compiler crash we saw. This test should cover it, as it should be possible to parse already.

https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 06:58:35 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 06:58:35 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d240b.170a0220.1d49e5.fc6a@mx.google.com>


https://github.com/Stylie777 edited https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 07:00:21 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Tue, 08 Jul 2025 07:00:21 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d2475.170a0220.7ad8b.fab4@mx.google.com>


https://github.com/Stylie777 edited https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 07:03:36 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Tue, 08 Jul 2025 07:03:36 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686d2538.050a0220.a3c14.1a5a@mx.google.com>


https://github.com/tarunprabhu approved this pull request.


https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Tue Jul  8 07:09:27 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 07:09:27 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d2697.050a0220.3d5b4f.2caf@mx.google.com>


================
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
----------------
DavidSpickett wrote:

> There is then potential for the while loop to never be equal to block.end() as it advances the nextIt value before checking if it is block.end().

Ok this is the key part, I understand now, thanks.

https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 05:02:54 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 05:02:54 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [TargetVerifier][AMDGPU] Add
 TargetVerifier. (PR #123609)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/123609@github.com>
Message-ID: <686d08ee.050a0220.1410e1.d640@mx.google.com>


https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/123609

>From d9b0a5df811451d4d4a2e37fc4933c8071022317 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 20 Jan 2025 04:51:26 -0800
Subject: [PATCH 01/33] [TargetVerifier][AMDGPU] Add TargetVerifier.

This pass verifies the IR for an individual backend. This is
different than Lint because it consolidates all checks for a
given backend in a single pass. A check for Lint may be
undefined behavior across all targets, whereas a check in
TargetVerifier would only pertain to the specified target
but can check more than just undefined behavior such are IR
validity. A use case of this would be to reject programs
with invalid IR while fuzzing.
---
 llvm/include/llvm/IR/Module.h                 |   9 +
 llvm/include/llvm/Target/TargetVerifier.h     |  82 +++++++
 .../TargetVerify/AMDGPUTargetVerifier.h       |  36 +++
 llvm/lib/IR/Verifier.cpp                      |  18 +-
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 213 ++++++++++++++++++
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        |  62 +++++
 llvm/tools/llvm-tgt-verify/CMakeLists.txt     |  34 +++
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 172 ++++++++++++++
 9 files changed, 623 insertions(+), 4 deletions(-)
 create mode 100644 llvm/include/llvm/Target/TargetVerifier.h
 create mode 100644 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify.ll
 create mode 100644 llvm/tools/llvm-tgt-verify/CMakeLists.txt
 create mode 100644 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp

diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index a99937a90cbb7..3037b2c1644d9 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,6 +215,15 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
+  /// Is this Module valid as determined by one of the verification passes
+  /// i.e. Lint, Verifier, TargetVerifier.
+  bool IsValid = true;
+
+  /// Is this Module using intrinsics to record the position of debugging
+  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
+  /// \ref BasicBlock.
+  bool IsNewDbgInfoFormat;
+
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
new file mode 100644
index 0000000000000..e00c6a7b260c9
--- /dev/null
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -0,0 +1,82 @@
+//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at TargetVerifier.cpp or an
+// individual backend's TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_VERIFIER_H
+#define LLVM_TARGET_VERIFIER_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+
+class Function;
+
+class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {}
+};
+
+class TargetVerify {
+protected:
+  void WriteValues(ArrayRef<const Value *> Vs) {
+    for (const Value *V : Vs) {
+      if (!V)
+        continue;
+      if (isa<Instruction>(V)) {
+        MessagesStr << *V << '\n';
+      } else {
+        V->printAsOperand(MessagesStr, true, Mod);
+        MessagesStr << '\n';
+      }
+    }
+  }
+
+  /// A check failed, so printout out the condition and the message.
+  ///
+  /// This provides a nice place to put a breakpoint if you want to see why
+  /// something is not correct.
+  void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
+
+  /// A check failed (with values to print).
+  ///
+  /// This calls the Message-only version so that the above is easier to set
+  /// a breakpoint on.
+  template <typename T1, typename... Ts>
+  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) {
+    CheckFailed(Message);
+    WriteValues({V1, Vs...});
+  }
+public:
+  Module *Mod;
+  Triple TT;
+
+  std::string Messages;
+  raw_string_ostream MessagesStr;
+
+  TargetVerify(Module *Mod)
+      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
+        MessagesStr(Messages) {}
+
+  void run(Function &F) {};
+};
+
+} // namespace llvm
+
+#endif // LLVM_TARGET_VERIFIER_H
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
new file mode 100644
index 0000000000000..e6ff57629b141
--- /dev/null
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -0,0 +1,36 @@
+//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU ---*- C++ -*-===//
+////
+//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//// See https://llvm.org/LICENSE.txt for license information.
+//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+////
+////===----------------------------------------------------------------------===//
+////
+//// This file defines target verifier interfaces that can be used for some
+//// validation of input to the system, and for checking that transformations
+//// haven't done something bad. In contrast to the Verifier or Lint, the
+//// TargetVerifier looks for constructions invalid to a particular target
+//// machine.
+////
+//// To see what specifically is checked, look at an individual backend's
+//// TargetVerifier.
+////
+////===----------------------------------------------------------------------===//
+
+#ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
+#define LLVM_AMDGPU_TARGET_VERIFIER_H
+
+#include "llvm/Target/TargetVerifier.h"
+
+namespace llvm {
+
+class Function;
+
+class AMDGPUTargetVerifierPass : public TargetVerifierPass {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1f1041b259736..18e9760f9c617 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -135,6 +135,10 @@ static cl::opt<bool> VerifyNoAliasScopeDomination(
     cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
              "scopes are not dominating"));
 
+static cl::opt<bool>
+    VerifyAbortOnError("verifier-abort-on-error", cl::init(false),
+                       cl::desc("In the Verifier pass, abort on errors."));
+
 namespace llvm {
 
 struct VerifierSupport {
@@ -7865,16 +7869,22 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
-  if (FatalErrors && (Res.IRBroken || Res.DebugInfoBroken))
-    report_fatal_error("Broken module found, compilation aborted!");
+  if (Res.IRBroken || Res.DebugInfoBroken) {
+    M.IsValid = false;
+    if (VerifyAbortOnError && FatalErrors)
+      report_fatal_error("Broken module found, compilation aborted!");
+  }
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
-  if (res.IRBroken && FatalErrors)
-    report_fatal_error("Broken function found, compilation aborted!");
+  if (res.IRBroken) {
+    F.getParent()->IsValid = false;
+    if (VerifyAbortOnError && FatalErrors)
+      report_fatal_error("Broken function found, compilation aborted!");
+  }
 
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
new file mode 100644
index 0000000000000..585b19065c142
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -0,0 +1,213 @@
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<bool>
+MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
+
+// Check - We know that cond should be true, if not print an error message.
+#define Check(C, ...)                                                          \
+  do {                                                                         \
+    if (!(C)) {                                                                \
+      TargetVerify::CheckFailed(__VA_ARGS__);                                  \
+      return;                                                                  \
+    }                                                                          \
+  } while (false)
+
+static bool isMFMA(unsigned IID) {
+  switch (IID) {
+    case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
+    case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
+    case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
+    case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
+    case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
+    case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
+    case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
+    case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
+    case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
+    case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
+    case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
+
+    case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
+    case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
+
+    case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
+    case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
+
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
+
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
+      return true;
+    default:
+      return false;
+  }
+}
+
+namespace llvm {
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  Module *Mod;
+
+  DominatorTree *DT;
+  PostDominatorTree *PDT;
+  UniformityInfo *UA;
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  void run(Function &F);
+};
+
+static bool IsValidInt(const Type *Ty) {
+  return Ty->isIntegerTy(1) ||
+         Ty->isIntegerTy(8) ||
+         Ty->isIntegerTy(16) ||
+         Ty->isIntegerTy(32) ||
+         Ty->isIntegerTy(64) ||
+         Ty->isIntegerTy(128);
+}
+
+static bool isShader(CallingConv::ID CC) {
+  switch(CC) {
+    case CallingConv::AMDGPU_VS:
+    case CallingConv::AMDGPU_LS:
+    case CallingConv::AMDGPU_HS:
+    case CallingConv::AMDGPU_ES:
+    case CallingConv::AMDGPU_GS:
+    case CallingConv::AMDGPU_PS:
+    case CallingConv::AMDGPU_CS_Chain:
+    case CallingConv::AMDGPU_CS_ChainPreserve:
+    case CallingConv::AMDGPU_CS:
+      return true;
+    default:
+      return false;
+  }
+}
+
+void AMDGPUTargetVerify::run(Function &F) {
+  // Ensure shader calling convention returns void
+  if (isShader(F.getCallingConv()))
+    Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
+
+  for (auto &BB : F) {
+
+    for (auto &I : BB) {
+      if (MarkUniform)
+        outs() << UA->isUniform(&I) << ' ' << I << '\n';
+
+      // Ensure integral types are valid: i8, i16, i32, i64, i128
+      if (I.getType()->isIntegerTy())
+        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
+      for (unsigned i = 0; i < I.getNumOperands(); ++i)
+        if (I.getOperand(i)->getType()->isIntegerTy())
+          Check(IsValidInt(I.getOperand(i)->getType()),
+                "Int type is invalid.", I.getOperand(i));
+
+      // Ensure no store to const memory
+      if (auto *SI = dyn_cast<StoreInst>(&I))
+      {
+        unsigned AS = SI->getPointerAddressSpace();
+        Check(AS != 4, "Write to const memory", SI);
+      }
+
+      // Ensure no kernel to kernel calls.
+      if (auto *CI = dyn_cast<CallInst>(&I))
+      {
+        CallingConv::ID CalleeCC = CI->getCallingConv();
+        if (CalleeCC == CallingConv::AMDGPU_KERNEL)
+        {
+          CallingConv::ID CallerCC = CI->getParent()->getParent()->getCallingConv();
+          Check(CallerCC != CallingConv::AMDGPU_KERNEL,
+            "A kernel may not call a kernel", CI->getParent()->getParent());
+        }
+      }
+
+      // Ensure MFMA is not in control flow with diverging operands
+      if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+        if (isMFMA(II->getIntrinsicID())) {
+          bool InControlFlow = false;
+          for (const auto &P : predecessors(&BB))
+            if (!PDT->dominates(&BB, P)) {
+              InControlFlow = true;
+              break;
+            }
+          for (const auto &S : successors(&BB))
+            if (!DT->dominates(&BB, S)) {
+              InControlFlow = true;
+              break;
+            }
+          if (InControlFlow) {
+            // If operands to MFMA are not uniform, MFMA cannot be in control flow
+            bool hasUniformOperands = true;
+            for (unsigned i = 0; i < II->getNumOperands(); i++) {
+              if (!UA->isUniform(II->getOperand(i))) {
+                dbgs() << "Not uniform: " << *II->getOperand(i) << '\n';
+                hasUniformOperands = false;
+              }
+            }
+            if (!hasUniformOperands) Check(false, "MFMA in control flow", II);
+            //else Check(false, "MFMA in control flow (uniform operands)", II);
+          }
+          //else Check(false, "MFMA not in control flow", II);
+        }
+      }
+    }
+  }
+}
+
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+
+  auto *Mod = F.getParent();
+
+  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  TV.run(F);
+
+  dbgs() << TV.MessagesStr.str();
+  if (!TV.MessagesStr.str().empty()) {
+    F.getParent()->IsValid = false;
+  }
+
+  return PreservedAnalyses::all();
+}
+} // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index c6d70ee39202e..a47b5f2732f91 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -111,6 +111,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUTargetMachine.cpp
   AMDGPUTargetObjectFile.cpp
   AMDGPUTargetTransformInfo.cpp
+  AMDGPUTargetVerifier.cpp
   AMDGPUWaitSGPRHazards.cpp
   AMDGPUUnifyDivergentExitNodes.cpp
   AMDGPUUnifyMetadata.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
new file mode 100644
index 0000000000000..f56ff992a56c2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -0,0 +1,62 @@
+; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
+
+define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg) #0 {
+; CHECK: Not uniform: %in.f32 = load <32 x float>, ptr addrspace(1) %gep, align 128
+; CHECK-NEXT: MFMA in control flow
+; CHECK-NEXT:   %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
+s:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
+  %in.i32 = load <32 x i32>, ptr addrspace(1) %gep
+  %in.f32 = load <32 x float>, ptr addrspace(1) %gep
+
+  %0 = icmp eq <32 x i32> %in.i32, zeroinitializer
+  %div.br = extractelement <32 x i1> %0, i32 0
+  br i1 %div.br, label %if.3, label %else.0
+
+if.3:
+  br label %join
+
+else.0:
+  %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
+  br label %join
+
+join:
+  ret void
+}
+
+define amdgpu_cs i32 @shader() {
+; CHECK: Shaders must return void
+  ret i32 0
+}
+
+define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
+; CHECK: Undefined behavior: Write to memory in const addrspace
+; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
+; CHECK-NEXT: Write to const memory
+; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
+  %r = add i32 %a, %b
+  store i32 %r, ptr addrspace(4) %out
+  ret void
+}
+
+define amdgpu_kernel void @kernel_callee(ptr %x) {
+  ret void
+}
+
+define amdgpu_kernel void @kernel_caller(ptr %x) {
+; CHECK: A kernel may not call a kernel
+; CHECK-NEXT: ptr @kernel_caller
+  call amdgpu_kernel void @kernel_callee(ptr %x)
+  ret void
+}
+
+
+; Function Attrs: nounwind
+define i65 @invalid_type(i65 %x) #0 {
+; CHECK: Int type is invalid.
+; CHECK-NEXT: %tmp2 = ashr i65 %x, 64
+entry:
+  %tmp2 = ashr i65 %x, 64
+  ret i65 %tmp2
+}
diff --git a/llvm/tools/llvm-tgt-verify/CMakeLists.txt b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
new file mode 100644
index 0000000000000..fe47c85e6cdce
--- /dev/null
+++ b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
@@ -0,0 +1,34 @@
+set(LLVM_LINK_COMPONENTS
+  AllTargetsAsmParsers
+  AllTargetsCodeGens
+  AllTargetsDescs
+  AllTargetsInfos
+  Analysis
+  AsmPrinter
+  CodeGen
+  CodeGenTypes
+  Core
+  IRPrinter
+  IRReader
+  MC
+  MIRParser
+  Passes
+  Remarks
+  ScalarOpts
+  SelectionDAG
+  Support
+  Target
+  TargetParser
+  TransformUtils
+  Vectorize
+  )
+
+add_llvm_tool(llvm-tgt-verify
+  llvm-tgt-verify.cpp
+
+  DEPENDS
+  intrinsics_gen
+  SUPPORT_PLUGINS
+  )
+
+export_executable_symbols_for_plugins(llc)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
new file mode 100644
index 0000000000000..68422abd6f4cc
--- /dev/null
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -0,0 +1,172 @@
+//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tool to fuzz instruction selection using libFuzzer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/InitializePasses.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/Lint.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/CodeGen/CommandFlags.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetVerifier.h"
+
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#define DEBUG_TYPE "isel-fuzzer"
+
+using namespace llvm;
+
+static codegen::RegisterCodeGenFlags CGF;
+
+static cl::opt<std::string>
+InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
+
+static cl::opt<bool>
+    StacktraceAbort("stacktrace-abort",
+        cl::desc("Turn on stacktrace"), cl::init(false));
+
+static cl::opt<bool>
+    NoLint("no-lint",
+        cl::desc("Turn off Lint"), cl::init(false));
+
+static cl::opt<bool>
+    NoVerify("no-verifier",
+        cl::desc("Turn off Verifier"), cl::init(false));
+
+static cl::opt<char>
+    OptLevel("O",
+             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
+                      "(default = '-O2')"),
+             cl::Prefix, cl::init('2'));
+
+static cl::opt<std::string>
+    TargetTriple("mtriple", cl::desc("Override target triple for module"));
+
+static std::unique_ptr<TargetMachine> TM;
+
+static void handleLLVMFatalError(void *, const char *Message, bool) {
+  if (StacktraceAbort) {
+    dbgs() << "LLVM ERROR: " << Message << "\n"
+           << "Aborting.\n";
+    abort();
+  }
+}
+
+int main(int argc, char **argv) {
+  StringRef ExecName = argv[0];
+  InitLLVM X(argc, argv);
+
+  InitializeAllTargets();
+  InitializeAllTargetMCs();
+  InitializeAllAsmPrinters();
+  InitializeAllAsmParsers();
+
+  PassRegistry *Registry = PassRegistry::getPassRegistry();
+  initializeCore(*Registry);
+  initializeCodeGen(*Registry);
+  initializeAnalysis(*Registry);
+  initializeTarget(*Registry);
+
+  cl::ParseCommandLineOptions(argc, argv);
+
+  if (TargetTriple.empty()) {
+    errs() << ExecName << ": -mtriple must be specified\n";
+    exit(1);
+  }
+
+  CodeGenOptLevel OLvl;
+  if (auto Level = CodeGenOpt::parseLevel(OptLevel)) {
+    OLvl = *Level;
+  } else {
+    errs() << ExecName << ": invalid optimization level.\n";
+    return 1;
+  }
+  ExitOnError ExitOnErr(std::string(ExecName) + ": error:");
+  TM = ExitOnErr(codegen::createTargetMachineForTriple(
+      Triple::normalize(TargetTriple), OLvl));
+  assert(TM && "Could not allocate target machine!");
+
+  // Make sure we print the summary and the current unit when LLVM errors out.
+  install_fatal_error_handler(handleLLVMFatalError, nullptr);
+
+  LLVMContext Context;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
+  if (!M) {
+    errs() << "Invalid mod\n";
+    return 1;
+  }
+  auto S = Triple::normalize(TargetTriple);
+  M->setTargetTriple(S);
+
+  PassInstrumentationCallbacks PIC;
+  StandardInstrumentations SI(Context, false/*debug PM*/,
+                              false);
+  registerCodeGenCallback(PIC, *TM);
+
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+  //TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
+
+  MachineFunctionAnalysisManager MFAM;
+  LoopAnalysisManager LAM;
+  FunctionAnalysisManager FAM;
+  CGSCCAnalysisManager CGAM;
+  ModuleAnalysisManager MAM;
+  PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.registerMachineFunctionAnalyses(MFAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
+
+  SI.registerCallbacks(PIC, &MAM);
+
+  //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
+
+  Triple TT(M->getTargetTriple());
+  if (!NoLint)
+    FPM.addPass(LintPass());
+  if (!NoVerify)
+    MPM.addPass(VerifierPass());
+  if (TT.isAMDGPU())
+    FPM.addPass(AMDGPUTargetVerifierPass());
+  else if (false) {} // ...
+  else
+    FPM.addPass(TargetVerifierPass());
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
+  MPM.run(*M, MAM);
+
+  if (!M->IsValid)
+    return 1;
+
+  return 0;
+}

>From 3b69ade6d8880cadc883698b64a06bd9207d084b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 3 Feb 2025 07:15:12 -0800
Subject: [PATCH 02/33] Add hook for target verifier in llc,opt

---
 .../llvm/Passes/StandardInstrumentations.h    |  6 ++++--
 llvm/include/llvm/Target/TargetVerifier.h     |  1 +
 .../TargetVerify/AMDGPUTargetVerifier.h       | 18 ++++++++++++++++++
 llvm/lib/LTO/LTOBackend.cpp                   |  2 +-
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp         |  2 +-
 llvm/lib/Passes/CMakeLists.txt                |  1 +
 llvm/lib/Passes/PassBuilderBindings.cpp       |  2 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  | 19 +++++++++++++++----
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 12 ++++++------
 llvm/lib/Target/CMakeLists.txt                |  2 ++
 .../CodeGen/AMDGPU/tgt-verify-llc-fail.ll     |  6 ++++++
 .../CodeGen/AMDGPU/tgt-verify-llc-pass.ll     |  6 ++++++
 llvm/tools/llc/NewPMDriver.cpp                |  2 +-
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 +-
 llvm/tools/opt/NewPMDriver.cpp                |  2 +-
 llvm/unittests/IR/PassManagerTest.cpp         |  6 +++---
 16 files changed, 68 insertions(+), 21 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll

diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 4ee5ab2554868..7d7b858b447cc 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,7 +478,8 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM);
+                         ModuleAnalysisManager *MAM,
+			 FunctionAnalysisManager *FAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -624,7 +625,8 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM = nullptr);
+                         ModuleAnalysisManager *MAM,
+			 FunctionAnalysisManager *FAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index e00c6a7b260c9..ad5aeb895953d 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -75,6 +75,7 @@ class TargetVerify {
         MessagesStr(Messages) {}
 
   void run(Function &F) {};
+  void run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index e6ff57629b141..d8a3fda4f87dc 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -22,6 +22,10 @@
 
 #include "llvm/Target/TargetVerifier.h"
 
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/IR/Dominators.h"
+
 namespace llvm {
 
 class Function;
@@ -31,6 +35,20 @@ class AMDGPUTargetVerifierPass : public TargetVerifierPass {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  Module *Mod;
+
+  DominatorTree *DT;
+  PostDominatorTree *PDT;
+  UniformityInfo *UA;
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  void run(Function &F);
+};
+
 } // namespace llvm
 
 #endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 5e8cd12fe040b..7ae23ef4e7df4 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -273,7 +273,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
                               Conf.VerifyEach);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 
   RegisterPassPlugins(Conf.PassPlugins, PB);
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 5b333cda31fb8..8db92e70d5a96 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -241,7 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PipelineTuningOptions PTO;
   PTO.LoopVectorization = true;
   PTO.SLPVectorization = true;
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 91c8c4f67074d..e5912f23903b6 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,6 +30,7 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
+  TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index 933fe89e53a94..f0e1abb8cebc4 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -76,7 +76,7 @@ static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   // Run the pipeline.
   if (Fun) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 4e8abc43572d7..e2764b162fddb 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,6 +40,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
+#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -1449,9 +1450,10 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 }
 
 void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                              ModuleAnalysisManager *MAM) {
+                                              ModuleAnalysisManager *MAM,
+					      FunctionAnalysisManager *FAM) {
   PIC.registerAfterPassCallback(
-      [this, MAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+      [this, MAM, FAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
         const auto *F = unwrapIR<Function>(IR);
@@ -1468,6 +1470,15 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
             report_fatal_error(formatv("Broken function found after pass "
                                        "\"{0}\", compilation aborted!",
                                        P));
+
+          if (FAM) {
+            TargetVerify TV(const_cast<Module*>(F->getParent()));
+            TV.run(*const_cast<Function*>(F), *FAM);
+	    if (!F->getParent()->IsValid)
+              report_fatal_error(formatv("Broken function found after pass "
+                                         "\"{0}\", compilation aborted!",
+                                         P));
+          }
         } else {
           const auto *M = unwrapIR<Module>(IR);
           if (!M) {
@@ -2507,7 +2518,7 @@ void PrintCrashIRInstrumentation::registerCallbacks(
 }
 
 void StandardInstrumentations::registerCallbacks(
-    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM) {
+    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM, FunctionAnalysisManager *FAM) {
   PrintIR.registerCallbacks(PIC);
   PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
@@ -2516,7 +2527,7 @@ void StandardInstrumentations::registerCallbacks(
   PrintChangedIR.registerCallbacks(PIC);
   PseudoProbeVerification.registerCallbacks(PIC);
   if (VerifyEach)
-    Verify.registerCallbacks(PIC, MAM);
+    Verify.registerCallbacks(PIC, MAM, FAM);
   PrintChangedDiff.registerCallbacks(PIC);
   WebsiteChangeReporter.registerCallbacks(PIC);
   ChangeTester.registerCallbacks(PIC);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 585b19065c142..e6cdec7160229 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -14,8 +14,8 @@
 
 using namespace llvm;
 
-static cl::opt<bool>
-MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
+//static cl::opt<bool>
+//MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
 
 // Check - We know that cond should be true, if not print an error message.
 #define Check(C, ...)                                                          \
@@ -81,7 +81,7 @@ static bool isMFMA(unsigned IID) {
 }
 
 namespace llvm {
-class AMDGPUTargetVerify : public TargetVerify {
+/*class AMDGPUTargetVerify : public TargetVerify {
 public:
   Module *Mod;
 
@@ -93,7 +93,7 @@ class AMDGPUTargetVerify : public TargetVerify {
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   void run(Function &F);
-};
+};*/
 
 static bool IsValidInt(const Type *Ty) {
   return Ty->isIntegerTy(1) ||
@@ -129,8 +129,8 @@ void AMDGPUTargetVerify::run(Function &F) {
   for (auto &BB : F) {
 
     for (auto &I : BB) {
-      if (MarkUniform)
-        outs() << UA->isUniform(&I) << ' ' << I << '\n';
+      //if (MarkUniform)
+        //outs() << UA->isUniform(&I) << ' ' << I << '\n';
 
       // Ensure integral types are valid: i8, i16, i32, i64, i128
       if (I.getType()->isIntegerTy())
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index 9472288229cac..f2a5d545ce84f 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -7,6 +7,8 @@ add_llvm_component_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
+  TargetVerifier.cpp
+  AMDGPU/AMDGPUTargetVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Target
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
new file mode 100644
index 0000000000000..584097d7bc134
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -0,0 +1,6 @@
+; RUN: not not llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each -o - < %s 2>&1 | FileCheck %s
+
+define amdgpu_cs i32 @nonvoid_shader() {
+; CHECK: LLVM ERROR
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
new file mode 100644
index 0000000000000..0c3a5fe5ac4a5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -0,0 +1,6 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each %s -o - 2>&1 | FileCheck %s
+
+define amdgpu_cs void @void_shader() {
+; CHECK: ModuleToFunctionPassAdaptor
+  ret void
+}
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index fa82689ecf9ae..a060d16e74958 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -126,7 +126,7 @@ int llvm::compileModuleWithNewPM(
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 68422abd6f4cc..3352d07deff2f 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -147,7 +147,7 @@ int main(int argc, char **argv) {
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
 
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index 7d168a6ceb17c..a8977d80bdf44 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -423,7 +423,7 @@ bool llvm::runPassPipeline(
   PrintPassOpts.SkipAnalyses = DebugPM == DebugLogging::Quiet;
   StandardInstrumentations SI(M.getContext(), DebugPM != DebugLogging::None,
                               VK == VerifierKind::EachPass, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   DebugifyEachInstrumentation Debugify;
   DebugifyStatsMap DIStatsMap;
   DebugInfoPerPass DebugInfoBeforePass;
diff --git a/llvm/unittests/IR/PassManagerTest.cpp b/llvm/unittests/IR/PassManagerTest.cpp
index a6487169224c2..bb4db6120035f 100644
--- a/llvm/unittests/IR/PassManagerTest.cpp
+++ b/llvm/unittests/IR/PassManagerTest.cpp
@@ -828,7 +828,7 @@ TEST_F(PassManagerTest, FunctionPassCFGChecker) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -877,7 +877,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerInvalidateAnalysis) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -945,7 +945,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerWrapped) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });

>From a89ff83fbb4a39d6e91ae3bccbb6244c32433a4b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 16 Apr 2025 10:19:00 -0400
Subject: [PATCH 03/33] Run AMDGPUTargetVerifier within AMDGPU pipeline. Move
 IsValid from Module to TargetVerify.

---
 clang/lib/CodeGen/BackendUtil.cpp               | 2 +-
 llvm/include/llvm/IR/Module.h                   | 4 ----
 llvm/include/llvm/Target/TargetVerifier.h       | 2 ++
 llvm/lib/IR/Verifier.cpp                        | 4 ++--
 llvm/lib/Passes/StandardInstrumentations.cpp    | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  | 5 +++++
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 3 ++-
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp  | 6 +++---
 8 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 7e0a3cf5591ce..560129f871026 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -917,7 +917,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       TheModule->getContext(),
       (CodeGenOpts.DebugPassManager || DebugPassStructure),
       CodeGenOpts.VerifyEach, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
   // Handle the assignment tracking feature options.
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 3037b2c1644d9..34ac47a2b8be4 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,10 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module valid as determined by one of the verification passes
-  /// i.e. Lint, Verifier, TargetVerifier.
-  bool IsValid = true;
-
   /// Is this Module using intrinsics to record the position of debugging
   /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
   /// \ref BasicBlock.
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index ad5aeb895953d..2d0c039132c35 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -70,6 +70,8 @@ class TargetVerify {
   std::string Messages;
   raw_string_ostream MessagesStr;
 
+  bool IsValid = true;
+
   TargetVerify(Module *Mod)
       : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
         MessagesStr(Messages) {}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 18e9760f9c617..0054dd26327ae 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7870,7 +7870,7 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
   if (Res.IRBroken || Res.DebugInfoBroken) {
-    M.IsValid = false;
+    //M.IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken module found, compilation aborted!");
   }
@@ -7881,7 +7881,7 @@ PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
   if (res.IRBroken) {
-    F.getParent()->IsValid = false;
+    //F.getParent()->IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken function found, compilation aborted!");
   }
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index e2764b162fddb..2bc54cf7d4891 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -1474,7 +1474,7 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
           if (FAM) {
             TargetVerify TV(const_cast<Module*>(F->getParent()));
             TV.run(*const_cast<Function*>(F), *FAM);
-	    if (!F->getParent()->IsValid)
+	    if (!TV.IsValid)
               report_fatal_error(formatv("Broken function found after pass "
                                          "\"{0}\", compilation aborted!",
                                          P));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index d59087839b0e1..dc88042a66df9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,6 +90,7 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
@@ -1328,6 +1329,8 @@ void AMDGPUPassConfig::addIRPasses() {
       addPass(createLICMPass());
   }
 
+  //addPass(AMDGPUTargetVerifierPass());
+
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
@@ -2071,6 +2074,8 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   // but EarlyCSE can do neither of them.
   if (isPassEnabled(EnableScalarIRPasses))
     addEarlyCSEOrGVNPass(addPass);
+
+  addPass(AMDGPUTargetVerifierPass());
 }
 
 void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index e6cdec7160229..c70a6d1b6fa66 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -205,7 +205,8 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
 
   dbgs() << TV.MessagesStr.str();
   if (!TV.MessagesStr.str().empty()) {
-    F.getParent()->IsValid = false;
+    TV.IsValid = false;
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 3352d07deff2f..fbe7f6089ff18 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -163,9 +163,9 @@ int main(int argc, char **argv) {
     FPM.addPass(TargetVerifierPass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
-  MPM.run(*M, MAM);
-
-  if (!M->IsValid)
+  auto PA = MPM.run(*M, MAM);
+  auto PAC = PA.getChecker<VerifierAnalysis>();
+  if (!PAC.preserved())
     return 1;
 
   return 0;

>From df507f9bfc4caa5578b48be68fb6c362808ec4af Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 16 Apr 2025 11:08:20 -0400
Subject: [PATCH 04/33] Remove cmd line options that aren't required. Make
 error message explicit.

---
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll | 4 ++--
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index 584097d7bc134..c5e59d4a2369e 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,6 +1,6 @@
-; RUN: not not llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
-; CHECK: LLVM ERROR
+; CHECK: Shaders must return void
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index 0c3a5fe5ac4a5..8a503b7624a73 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each %s -o - 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
-; CHECK: ModuleToFunctionPassAdaptor
+; CHECK-NOT: Shaders must return void
   ret void
 }

>From 942312d57ea09d3dc9224d5cb3753390db13b2ba Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 19:36:34 -0400
Subject: [PATCH 05/33] Return Verifier none status through PreservedAnalyses
 on fail.

---
 llvm/lib/Analysis/Lint.cpp                     | 4 +++-
 llvm/lib/IR/Verifier.cpp                       | 2 ++
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 8 +++++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 1168005f48c0e..c8627e6db0692 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -735,9 +735,11 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) {
   Lint L(Mod, DL, AA, AC, DT, TLI);
   L.visit(F);
   dbgs() << L.MessagesStr.str();
-  if (AbortOnError && !L.MessagesStr.str().empty())
+  if (AbortOnError && !L.MessagesStr.str().empty()) {
     report_fatal_error(
         "linter found errors, aborting. (enabled by abort-on-error)", false);
+    return PreservedAnalyses::none();
+  }
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 0054dd26327ae..439823ba8b682 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7873,6 +7873,7 @@ PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
     //M.IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken module found, compilation aborted!");
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
@@ -7884,6 +7885,7 @@ PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
     //F.getParent()->IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken function found, compilation aborted!");
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index fbe7f6089ff18..042824ac37fea 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -164,9 +164,11 @@ int main(int argc, char **argv) {
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
   auto PA = MPM.run(*M, MAM);
-  auto PAC = PA.getChecker<VerifierAnalysis>();
-  if (!PAC.preserved())
-    return 1;
+  {
+    auto PAC = PA.getChecker<VerifierAnalysis>();
+    if (!PAC.preserved())
+      return 1;
+  }
 
   return 0;
 }

>From 6216a104469edfa1da128ce64be2cba2c2421b00 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 21:16:02 -0400
Subject: [PATCH 06/33] Rebase update.

---
 llvm/include/llvm/Target/TargetVerifier.h      | 2 +-
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 2d0c039132c35..fe683311b901c 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -73,7 +73,7 @@ class TargetVerify {
   bool IsValid = true;
 
   TargetVerify(Module *Mod)
-      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
+      : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
   void run(Function &F) {};
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 042824ac37fea..627bc51ef3a43 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -123,7 +123,7 @@ int main(int argc, char **argv) {
     return 1;
   }
   auto S = Triple::normalize(TargetTriple);
-  M->setTargetTriple(S);
+  M->setTargetTriple(Triple(S));
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Context, false/*debug PM*/,
@@ -153,7 +153,7 @@ int main(int argc, char **argv) {
 
   Triple TT(M->getTargetTriple());
   if (!NoLint)
-    FPM.addPass(LintPass());
+    FPM.addPass(LintPass(false));
   if (!NoVerify)
     MPM.addPass(VerifierPass());
   if (TT.isAMDGPU())

>From 4b7068f49c24f28d4fac762ea6f1f8420028d525 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 22:49:52 -0400
Subject: [PATCH 07/33] Add generic TargetVerifier.

---
 llvm/lib/Target/TargetVerifier.cpp | 32 ++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 llvm/lib/Target/TargetVerifier.cpp

diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
new file mode 100644
index 0000000000000..de3ff749e7c3c
--- /dev/null
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -0,0 +1,32 @@
+#include "llvm/Target/TargetVerifier.h"
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+
+namespace llvm {
+
+void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
+  if (TT.isAMDGPU()) {
+    auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
+    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+    }
+  }
+}
+
+} // namespace llvm

>From 86e806635cdf1f1606af654817aefa8d797b8cf5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:13:14 -0400
Subject: [PATCH 08/33] Remove store to const check since it is in Lint already

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 8 --------
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll          | 2 --
 2 files changed, 10 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index c70a6d1b6fa66..1cf2b277bee26 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -140,14 +140,6 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(IsValidInt(I.getOperand(i)->getType()),
                 "Int type is invalid.", I.getOperand(i));
 
-      // Ensure no store to const memory
-      if (auto *SI = dyn_cast<StoreInst>(&I))
-      {
-        unsigned AS = SI->getPointerAddressSpace();
-        Check(AS != 4, "Write to const memory", SI);
-      }
-
-      // Ensure no kernel to kernel calls.
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
         CallingConv::ID CalleeCC = CI->getCallingConv();
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index f56ff992a56c2..c628abbde11d1 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -32,8 +32,6 @@ define amdgpu_cs i32 @shader() {
 
 define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
 ; CHECK: Undefined behavior: Write to memory in const addrspace
-; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
-; CHECK-NEXT: Write to const memory
 ; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
   %r = add i32 %a, %b
   store i32 %r, ptr addrspace(4) %out

>From 17fc5c6be3406d4360e25b2de595d31afe7f77bc Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:14:38 -0400
Subject: [PATCH 09/33] Add chain followed by unreachable check

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp |  6 ++++++
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll          | 10 ++++++++++
 2 files changed, 16 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 1cf2b277bee26..8ea773bc0e66f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -142,6 +142,7 @@ void AMDGPUTargetVerify::run(Function &F) {
 
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
+        // Ensure no kernel to kernel calls.
         CallingConv::ID CalleeCC = CI->getCallingConv();
         if (CalleeCC == CallingConv::AMDGPU_KERNEL)
         {
@@ -149,6 +150,11 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(CallerCC != CallingConv::AMDGPU_KERNEL,
             "A kernel may not call a kernel", CI->getParent()->getParent());
         }
+
+        // Ensure chain intrinsics are followed by unreachables.
+        if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
+          Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
+            "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
 
       // Ensure MFMA is not in control flow with diverging operands
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index c628abbde11d1..e620df94ccde4 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -58,3 +58,13 @@ entry:
   %tmp2 = ashr i65 %x, 64
   ret i65 %tmp2
 }
+
+declare void @llvm.amdgcn.cs.chain.v3i32(ptr, i32, <3 x i32>, <3 x i32>, i32, ...)
+declare amdgpu_cs_chain void @chain_callee(<3 x i32> inreg, <3 x i32>)
+
+define amdgpu_cs void @no_unreachable(<3 x i32> inreg %a, <3 x i32> %b) {
+; CHECK: llvm.amdgcn.cs.chain must be followed by unreachable
+; CHECK-NEXT: call void (ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.v3i32.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  ret void
+}

>From 46e662cce2d1f3716fcf5c23be39fdb27a2d7ecb Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:26:19 -0400
Subject: [PATCH 10/33] Remove mfma check

---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 89 -------------------
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        | 25 ------
 2 files changed, 114 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 8ea773bc0e66f..684ced5bba574 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -14,9 +14,6 @@
 
 using namespace llvm;
 
-//static cl::opt<bool>
-//MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
-
 // Check - We know that cond should be true, if not print an error message.
 #define Check(C, ...)                                                          \
   do {                                                                         \
@@ -26,60 +23,6 @@ using namespace llvm;
     }                                                                          \
   } while (false)
 
-static bool isMFMA(unsigned IID) {
-  switch (IID) {
-    case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
-    case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
-    case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
-    case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
-    case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
-    case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
-    case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
-    case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
-    case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
-    case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
-    case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
-
-    case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
-    case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
-
-    case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
-    case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
-
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
-
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
-      return true;
-    default:
-      return false;
-  }
-}
-
 namespace llvm {
 /*class AMDGPUTargetVerify : public TargetVerify {
 public:
@@ -129,8 +72,6 @@ void AMDGPUTargetVerify::run(Function &F) {
   for (auto &BB : F) {
 
     for (auto &I : BB) {
-      //if (MarkUniform)
-        //outs() << UA->isUniform(&I) << ' ' << I << '\n';
 
       // Ensure integral types are valid: i8, i16, i32, i64, i128
       if (I.getType()->isIntegerTy())
@@ -156,36 +97,6 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
             "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
-
-      // Ensure MFMA is not in control flow with diverging operands
-      if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
-        if (isMFMA(II->getIntrinsicID())) {
-          bool InControlFlow = false;
-          for (const auto &P : predecessors(&BB))
-            if (!PDT->dominates(&BB, P)) {
-              InControlFlow = true;
-              break;
-            }
-          for (const auto &S : successors(&BB))
-            if (!DT->dominates(&BB, S)) {
-              InControlFlow = true;
-              break;
-            }
-          if (InControlFlow) {
-            // If operands to MFMA are not uniform, MFMA cannot be in control flow
-            bool hasUniformOperands = true;
-            for (unsigned i = 0; i < II->getNumOperands(); i++) {
-              if (!UA->isUniform(II->getOperand(i))) {
-                dbgs() << "Not uniform: " << *II->getOperand(i) << '\n';
-                hasUniformOperands = false;
-              }
-            }
-            if (!hasUniformOperands) Check(false, "MFMA in control flow", II);
-            //else Check(false, "MFMA in control flow (uniform operands)", II);
-          }
-          //else Check(false, "MFMA not in control flow", II);
-        }
-      }
     }
   }
 }
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index e620df94ccde4..62b220d7d9f49 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -1,30 +1,5 @@
 ; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
 
-define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg) #0 {
-; CHECK: Not uniform: %in.f32 = load <32 x float>, ptr addrspace(1) %gep, align 128
-; CHECK-NEXT: MFMA in control flow
-; CHECK-NEXT:   %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
-s:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
-  %in.i32 = load <32 x i32>, ptr addrspace(1) %gep
-  %in.f32 = load <32 x float>, ptr addrspace(1) %gep
-
-  %0 = icmp eq <32 x i32> %in.i32, zeroinitializer
-  %div.br = extractelement <32 x i1> %0, i32 0
-  br i1 %div.br, label %if.3, label %else.0
-
-if.3:
-  br label %join
-
-else.0:
-  %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
-  br label %join
-
-join:
-  ret void
-}
-
 define amdgpu_cs i32 @shader() {
 ; CHECK: Shaders must return void
   ret i32 0

>From 7e471c9111d6a3a33f0de6bac883353669a69b0a Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 21 Apr 2025 20:54:10 -0400
Subject: [PATCH 11/33] Add registerVerifierPasses to PassBuilder and add the
 verifier passes to PassRegistry.

---
 llvm/include/llvm/InitializePasses.h          |  1 +
 llvm/include/llvm/Passes/PassBuilder.h        | 21 +++++++
 .../llvm/Passes/TargetPassRegistry.inc        | 12 ++++
 .../TargetVerify/AMDGPUTargetVerifier.h       | 11 ++--
 llvm/lib/Passes/PassBuilder.cpp               |  7 +++
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 11 ++++
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 56 ++++++++++++++++++-
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  1 +
 8 files changed, 114 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 1b5b1d5888824..662f6b71aa46a 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,6 +333,7 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
+LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index f13b5c678a894..c134f2d0c1112 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -173,6 +173,13 @@ class PassBuilder {
   /// additional analyses.
   LLVM_ABI void registerLoopAnalyses(LoopAnalysisManager &LAM);
 
+  /// Registers all available verifier passes.
+  ///
+  /// This is an interface that can be used to populate a
+  /// \c ModuleAnalysisManager with all registered loop analyses. Callers can
+  /// still manually register any additional analyses.
+  void registerVerifierPasses(ModulePassManager &PM, FunctionPassManager &);
+
   /// Registers all available machine function analysis passes.
   ///
   /// This is an interface that can be used to populate a \c
@@ -576,6 +583,15 @@ class PassBuilder {
   }
   /// @}}
 
+  /// Register a callback for parsing an Verifier Name to populate
+  /// the given managers.
+  void registerVerifierCallback(
+      const std::function<bool(ModulePassManager &MPM)> &C,
+      const std::function<bool(FunctionPassManager &MPM)> &CF) {
+    VerifierCallbacks.push_back(C);
+    FnVerifierCallbacks.push_back(CF);
+  }
+
   /// {{@ Register pipeline parsing callbacks with this pass builder instance.
   /// Using these callbacks, callers can parse both a single pass name, as well
   /// as entire sub-pipelines, and populate the PassManager instance
@@ -851,6 +867,11 @@ class PassBuilder {
   // Callbacks to parse `filter` parameter in register allocation passes
   SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
+  // Verifier callbacks
+  SmallVector<std::function<bool(ModulePassManager &)>, 2>
+      VerifierCallbacks;
+  SmallVector<std::function<bool(FunctionPassManager &)>, 2>
+      FnVerifierCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc
index 521913cb25a4a..2d04b874cf360 100644
--- a/llvm/include/llvm/Passes/TargetPassRegistry.inc
+++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc
@@ -151,6 +151,18 @@ PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM,
   return false;
 });
 
+PB.registerVerifierCallback([](ModulePassManager &PM) {
+#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS) PM.addPass(CREATE_PASS)
+#include GET_PASS_REGISTRY
+#undef VERIFIER_MODULE_ANALYSIS
+  return false;
+}, [](FunctionPassManager &FPM) {
+#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS) FPM.addPass(CREATE_PASS)
+#include GET_PASS_REGISTRY
+#undef VERIFIER_FUNCTION_ANALYSIS
+  return false;
+});
+
 #undef ADD_PASS
 #undef ADD_PASS_WITH_PARAMS
 
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index d8a3fda4f87dc..b6a7412e8c1ef 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -39,14 +39,17 @@ class AMDGPUTargetVerify : public TargetVerify {
 public:
   Module *Mod;
 
-  DominatorTree *DT;
-  PostDominatorTree *PDT;
-  UniformityInfo *UA;
+  DominatorTree *DT = nullptr;
+  PostDominatorTree *PDT = nullptr;
+  UniformityInfo *UA = nullptr;
+
+  AMDGPUTargetVerify(Module *Mod)
+    : TargetVerify(Mod), Mod(Mod) {}
 
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
-  void run(Function &F);
+  bool run(Function &F);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 4603eaff8ade9..561996d0c8980 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -576,6 +576,13 @@ void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     C(LAM);
 }
 
+void PassBuilder::registerVerifierPasses(ModulePassManager &MPM, FunctionPassManager &FPM) {
+  for (auto &C : VerifierCallbacks)
+    C(MPM);
+  for (auto &C : FnVerifierCallbacks)
+    C(FPM);
+}
+
 static std::optional<std::pair<bool, bool>>
 parseFunctionPipelineName(StringRef Name) {
   std::pair<bool, bool> Params;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 13453963eec6d..652cd23f62443 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -82,6 +82,17 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
+#ifndef VERIFIER_MODULE_ANALYSIS
+#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS)
+#endif
+#ifndef VERIFIER_FUNCTION_ANALYSIS
+#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
+#endif
+VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
+#undef VERIFIER_MODULE_ANALYSIS
+#undef VERIFIER_FUNCTION_ANALYSIS
+
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 684ced5bba574..63a7526b9abdc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -5,6 +5,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -19,7 +20,7 @@ using namespace llvm;
   do {                                                                         \
     if (!(C)) {                                                                \
       TargetVerify::CheckFailed(__VA_ARGS__);                                  \
-      return;                                                                  \
+      return false;                                                                  \
     }                                                                          \
   } while (false)
 
@@ -64,7 +65,7 @@ static bool isShader(CallingConv::ID CC) {
   }
 }
 
-void AMDGPUTargetVerify::run(Function &F) {
+bool AMDGPUTargetVerify::run(Function &F) {
   // Ensure shader calling convention returns void
   if (isShader(F.getCallingConv()))
     Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
@@ -99,6 +100,10 @@ void AMDGPUTargetVerify::run(Function &F) {
       }
     }
   }
+
+  if (!MessagesStr.str().empty())
+    return false;
+  return true;
 }
 
 PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
@@ -120,4 +125,51 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
 
   return PreservedAnalyses::all();
 }
+
+struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<AMDGPUTargetVerify> TV;
+  bool FatalErrors = true;
+
+  AMDGPUTargetVerifierLegacyPass() : FunctionPass(ID) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
+      : FunctionPass(ID),
+        FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<AMDGPUTargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (TV->run(F) && FatalErrors) {
+      errs() << "in function " << F.getName() << '\n';
+      report_fatal_error("Broken function found, compilation aborted!");
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    //IsValid &= TV->run();
+    if (FatalErrors && !IsValid)
+      report_fatal_error("Broken module found, compilation aborted!");
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char AMDGPUTargetVerifierLegacyPass::ID = 0;
 } // namespace llvm
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverify", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 627bc51ef3a43..503db7b1f8d18 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -144,6 +144,7 @@ int main(int argc, char **argv) {
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
+  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 

>From 1538985ecdd840e68207f83ca7b159bccef98ff5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Tue, 22 Apr 2025 14:57:31 -0400
Subject: [PATCH 12/33] Remove leftovers. Add titles. Add call to
 registerVerifierCallbacks in llc.

---
 llvm/lib/Passes/CMakeLists.txt                |  2 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  4 ---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 35 +++++++++++--------
 llvm/lib/Target/TargetVerifier.cpp            | 19 ++++++++++
 llvm/tools/llc/NewPMDriver.cpp                |  6 ++--
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  7 ++--
 6 files changed, 45 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index e5912f23903b6..b8d25b15186e8 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,7 +30,7 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
-  TargetParser
+  #TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index dc88042a66df9..0c41c633dcb3d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1329,8 +1329,6 @@ void AMDGPUPassConfig::addIRPasses() {
       addPass(createLICMPass());
   }
 
-  //addPass(AMDGPUTargetVerifierPass());
-
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
@@ -2074,8 +2072,6 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   // but EarlyCSE can do neither of them.
   if (isPassEnabled(EnableScalarIRPasses))
     addEarlyCSEOrGVNPass(addPass);
-
-  addPass(AMDGPUTargetVerifierPass());
 }
 
 void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 63a7526b9abdc..0eecedaebc7ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -1,3 +1,22 @@
+//===-- AMDGPUTargetVerifier.cpp - AMDGPU -------------------------*- C++ -*-===//
+////
+//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//// See https://llvm.org/LICENSE.txt for license information.
+//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+////
+////===----------------------------------------------------------------------===//
+////
+//// This file defines target verifier interfaces that can be used for some
+//// validation of input to the system, and for checking that transformations
+//// haven't done something bad. In contrast to the Verifier or Lint, the
+//// TargetVerifier looks for constructions invalid to a particular target
+//// machine.
+////
+//// To see what specifically is checked, look at an individual backend's
+//// TargetVerifier.
+////
+////===----------------------------------------------------------------------===//
+
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
 #include "llvm/Analysis/UniformityAnalysis.h"
@@ -25,19 +44,6 @@ using namespace llvm;
   } while (false)
 
 namespace llvm {
-/*class AMDGPUTargetVerify : public TargetVerify {
-public:
-  Module *Mod;
-
-  DominatorTree *DT;
-  PostDominatorTree *PDT;
-  UniformityInfo *UA;
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
-    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
-  void run(Function &F);
-};*/
 
 static bool IsValidInt(const Type *Ty) {
   return Ty->isIntegerTy(1) ||
@@ -147,7 +153,7 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   }
 
   bool runOnFunction(Function &F) override {
-    if (TV->run(F) && FatalErrors) {
+    if (!TV->run(F) && FatalErrors) {
       errs() << "in function " << F.getName() << '\n';
       report_fatal_error("Broken function found, compilation aborted!");
     }
@@ -160,7 +166,6 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
       if (F.isDeclaration())
         IsValid &= TV->run(F);
 
-    //IsValid &= TV->run();
     if (FatalErrors && !IsValid)
       report_fatal_error("Broken module found, compilation aborted!");
     return false;
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index de3ff749e7c3c..992a0c91d93b1 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -1,3 +1,22 @@
+//===-- TargetVerifier.cpp - LLVM IR Target Verifier ----------------*- C++ -*-===//
+////
+///// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+///// See https://llvm.org/LICENSE.txt for license information.
+///// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+/////
+/////===----------------------------------------------------------------------===//
+/////
+///// This file defines target verifier interfaces that can be used for some
+///// validation of input to the system, and for checking that transformations
+///// haven't done something bad. In contrast to the Verifier or Lint, the
+///// TargetVerifier looks for constructions invalid to a particular target
+///// machine.
+/////
+///// To see what specifically is checked, look at TargetVerifier.cpp or an
+///// individual backend's TargetVerifier.
+/////
+/////===----------------------------------------------------------------------===//
+
 #include "llvm/Target/TargetVerifier.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index a060d16e74958..a8f6b999af06e 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -114,6 +114,8 @@ int llvm::compileModuleWithNewPM(
                               VK == VerifierKind::EachPass);
   registerCodeGenCallback(PIC, *Target);
 
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
   MachineFunctionAnalysisManager MFAM;
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
@@ -125,15 +127,13 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
+  PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
 
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
-
   if (!PassPipeline.empty()) {
     // Construct a custom pass pipeline that starts after instruction
     // selection.
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 503db7b1f8d18..b00bab66c6c3e 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -1,4 +1,4 @@
-//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
+//===--- llvm-tgt-verify.cpp - Target Verifier ----------------- ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Tool to fuzz instruction selection using libFuzzer.
+// Tool to verify a target.
 //
 //===----------------------------------------------------------------------===//
 
@@ -144,14 +144,11 @@ int main(int argc, char **argv) {
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
-  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
-  //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
-
   Triple TT(M->getTargetTriple());
   if (!NoLint)
     FPM.addPass(LintPass(false));

>From 32b0c0d5d74b3b78699b78736604e87369ca4217 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 24 Apr 2025 16:02:21 -0400
Subject: [PATCH 13/33] Add pass to legacy PM.

---
 llvm/include/llvm/CodeGen/Passes.h            |  2 +
 llvm/include/llvm/Target/TargetVerifier.h     |  6 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  |  4 +-
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  2 +-
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 45 ----------
 llvm/lib/Target/TargetVerifier.cpp            | 87 ++++++++++++++++++-
 llvm/tools/llc/NewPMDriver.cpp                |  6 +-
 llvm/tools/llc/llc.cpp                        |  4 +
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  1 +
 9 files changed, 105 insertions(+), 52 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 990452fa11fec..53bd70183e476 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -620,6 +620,8 @@ LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
+
+LLVM_ABI FunctionPass *createTargetVerifierLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index fe683311b901c..23ef2e0b8d4ef 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -30,7 +30,7 @@ class Function;
 
 class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 class TargetVerify {
@@ -76,8 +76,8 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  void run(Function &F) {};
-  void run(Function &F, FunctionAnalysisManager &AM);
+  bool run(Function &F);
+  bool run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 2bc54cf7d4891..539f4b24b5110 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -57,6 +57,8 @@ static cl::opt<bool> VerifyAnalysisInvalidation("verify-analysis-invalidation",
 #endif
 );
 
+static cl::opt<bool> VerifyTargetEach("verify-tgt-each");
+
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
 // of this option.  Note that this option has no effect without -print-changed.
@@ -1471,7 +1473,7 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
                                        "\"{0}\", compilation aborted!",
                                        P));
 
-          if (FAM) {
+          if (VerifyTargetEach && FAM) {
             TargetVerify TV(const_cast<Module*>(F->getParent()));
             TV.run(*const_cast<Function*>(F), *FAM);
 	    if (!TV.IsValid)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 652cd23f62443..ad4fa93d09e0c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -89,7 +89,7 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("tgtverifier", TargetVerifierPass())
 #undef VERIFIER_MODULE_ANALYSIS
 #undef VERIFIER_FUNCTION_ANALYSIS
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 0eecedaebc7ce..96bcaaf6f2ac9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -132,49 +132,4 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
   return PreservedAnalyses::all();
 }
 
-struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
-  static char ID;
-
-  std::unique_ptr<AMDGPUTargetVerify> TV;
-  bool FatalErrors = true;
-
-  AMDGPUTargetVerifierLegacyPass() : FunctionPass(ID) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
-      : FunctionPass(ID),
-        FatalErrors(FatalErrors) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool doInitialization(Module &M) override {
-    TV = std::make_unique<AMDGPUTargetVerify>(&M);
-    return false;
-  }
-
-  bool runOnFunction(Function &F) override {
-    if (!TV->run(F) && FatalErrors) {
-      errs() << "in function " << F.getName() << '\n';
-      report_fatal_error("Broken function found, compilation aborted!");
-    }
-    return false;
-  }
-
-  bool doFinalization(Module &M) override {
-    bool IsValid = true;
-    for (Function &F : M)
-      if (F.isDeclaration())
-        IsValid &= TV->run(F);
-
-    if (FatalErrors && !IsValid)
-      report_fatal_error("Broken module found, compilation aborted!");
-    return false;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-  }
-};
-char AMDGPUTargetVerifierLegacyPass::ID = 0;
 } // namespace llvm
-INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverify", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 992a0c91d93b1..170fc4769c1d8 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Target/TargetVerifier.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
+#include "llvm/InitializePasses.h"
 #include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Support/Debug.h"
@@ -32,7 +33,22 @@
 
 namespace llvm {
 
-void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
+bool TargetVerify::run(Function &F) {
+  if (TT.isAMDGPU()) {
+    AMDGPUTargetVerify TV(Mod);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+      return false;
+    }
+    return true;
+  }
+  report_fatal_error("Target has no verification method\n");
+}
+
+bool TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
   if (TT.isAMDGPU()) {
     auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
     auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
@@ -44,8 +60,77 @@ void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
     dbgs() << TV.MessagesStr.str();
     if (!TV.MessagesStr.str().empty()) {
       TV.IsValid = false;
+      return false;
+    }
+    return true;
+  }
+  report_fatal_error("Target has no verification method\n");
+}
+
+PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto TT = F.getParent()->getTargetTriple();
+
+  if (TT.isAMDGPU()) {
+    auto *Mod = F.getParent();
+
+    auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+      return PreservedAnalyses::none();
     }
+    return PreservedAnalyses::all();
   }
+  report_fatal_error("Target has no verification method\n");
 }
 
+struct TargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<TargetVerify> TV;
+
+  TargetVerifierLegacyPass() : FunctionPass(ID) {
+    initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<TargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (!TV->run(F)) {
+      errs() << "in function " << F.getName() << '\n';
+      report_fatal_error("broken function found, compilation aborted!");
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    if (!IsValid)
+      report_fatal_error("broken module found, compilation aborted!");
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char TargetVerifierLegacyPass::ID = 0;
+FunctionPass *createTargetVerifierLegacyPass() {
+  return new TargetVerifierLegacyPass();
+}
 } // namespace llvm
+using namespace llvm;
+INITIALIZE_PASS(TargetVerifierLegacyPass, "tgtverifier", "Target Verifier", false, false)
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index a8f6b999af06e..4b95977a10c5f 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -57,6 +57,9 @@ static cl::opt<bool>
     DebugPM("debug-pass-manager", cl::Hidden,
             cl::desc("Print pass management debugging information"));
 
+static cl::opt<bool> VerifyTarget("verify-tgt-new-pm",
+		                  cl::desc("Verify the target"));
+
 bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
   DiagnosticHandler::handleDiagnostics(DI);
   if (DI.getKind() == llvm::DK_SrcMgr) {
@@ -127,7 +130,8 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
-  PB.registerVerifierPasses(MPM, FPM);
+  if (VerifyTarget)
+    PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 140459ba2de21..1fd8a9f9cd9f8 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -209,6 +209,8 @@ static cl::opt<std::string> PassPipeline(
 static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
                                cl::desc("Alias for -passes"));
 
+static cl::opt<bool> VerifyTarget("verify-tgt", cl::desc("Verify the target"));
+
 namespace {
 
 std::vector<std::string> &getRunPassNames() {
@@ -658,6 +660,8 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
+  if (VerifyTarget)
+    PM.add(createTargetVerifierLegacyPass());
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index b00bab66c6c3e..b86c2318b45b7 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -141,6 +141,7 @@ int main(int argc, char **argv) {
   ModuleAnalysisManager MAM;
   PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
   PB.registerModuleAnalyses(MAM);
+  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);

>From 387b2d5936a699238ac775726b0b30860f68507f Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 00:13:42 -0400
Subject: [PATCH 14/33] Add fam in other projects.

---
 flang/lib/Frontend/FrontendActions.cpp      | 2 +-
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index d684eeb696755..c15151fc5c1b8 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -947,7 +947,7 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
 
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
-  si.registerCallbacks(pic, &mam);
+  si.registerCallbacks(pic, &mam, &fam);
   if (ci.isTimingEnabled())
     si.getTimePasses().setOutStream(ci.getTimingStreamLLVM());
   pto.LoopUnrolling = opts.UnrollLoops;
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index 0f58391c50667..f9664025f61f1 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -577,7 +577,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index 7117eaf4982b0..eae06d9f57467 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -851,7 +851,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index cb7b6cc8651c1..30ad79ef2fc58 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -970,7 +970,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 91b7191a07c6f..4a39bc33c5591 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1139,7 +1139,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Promote allocas to registers.

>From d2c9425f9e951768842ca9771148611bdba5e1bb Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 02:30:40 -0400
Subject: [PATCH 15/33] Avoid fatal errors in llc.

---
 llvm/lib/Target/TargetVerifier.cpp             | 18 +++++++++++++-----
 .../test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll |  2 +-
 .../test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll |  2 +-
 llvm/tools/llc/llc.cpp                         |  2 +-
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 170fc4769c1d8..3be50f4ef6da3 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -94,8 +94,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   static char ID;
 
   std::unique_ptr<TargetVerify> TV;
+  bool FatalErrors = false;
 
-  TargetVerifierLegacyPass() : FunctionPass(ID) {
+  TargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
+    FatalErrors(FatalErrors) {
     initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
@@ -107,7 +109,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   bool runOnFunction(Function &F) override {
     if (!TV->run(F)) {
       errs() << "in function " << F.getName() << '\n';
-      report_fatal_error("broken function found, compilation aborted!");
+      if (FatalErrors)
+        report_fatal_error("broken function found, compilation aborted!");
+      else
+        errs() << "broken function found, compilation aborted!\n";
     }
     return false;
   }
@@ -119,7 +124,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
         IsValid &= TV->run(F);
 
     if (!IsValid)
-      report_fatal_error("broken module found, compilation aborted!");
+      if (FatalErrors)
+        report_fatal_error("broken module found, compilation aborted!");
+      else
+        errs() << "broken module found, compilation aborted!\n";
     return false;
   }
 
@@ -128,8 +136,8 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   }
 };
 char TargetVerifierLegacyPass::ID = 0;
-FunctionPass *createTargetVerifierLegacyPass() {
-  return new TargetVerifierLegacyPass();
+FunctionPass *createTargetVerifierLegacyPass(bool FatalErrors) {
+  return new TargetVerifierLegacyPass(FatalErrors);
 }
 } // namespace llvm
 using namespace llvm;
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index c5e59d4a2369e..e2d9edda5d008 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
 ; CHECK: Shaders must return void
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index 8a503b7624a73..a2dab0ff47924 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm %s -o - 2>&1 | FileCheck %s --allow-empty
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
 ; CHECK-NOT: Shaders must return void
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 1fd8a9f9cd9f8..329d95826551f 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -661,7 +661,7 @@ static int compileModule(char **argv, LLVMContext &Context) {
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
   if (VerifyTarget)
-    PM.add(createTargetVerifierLegacyPass());
+    PM.add(createTargetVerifierLegacyPass(false));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {

>From 31a79a57d9d41365ffcd7b263b2a05153463b2f5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 03:40:47 -0400
Subject: [PATCH 16/33] Add tool to build/test.

---
 llvm/test/CMakeLists.txt                      |  1 +
 llvm/test/lit.cfg.py                          |  1 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |  1 +
 .../llvm/tools/llvm-tgt-verify/BUILD.gn       | 25 +++++++++++++++++++
 4 files changed, 28 insertions(+)
 create mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn

diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 2a6135da9a61e..a2bb112db8746 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -136,6 +136,7 @@ set(LLVM_TEST_DEPENDS
           llvm-strip
           llvm-symbolizer
           llvm-tblgen
+          llvm-tgt-verify
           llvm-readtapi
           llvm-tli-checker
           llvm-undname
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index bd6e37c848d8c..bf06e80dc7ae5 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -228,6 +228,7 @@ def get_asan_rtlib():
         "llvm-strings",
         "llvm-strip",
         "llvm-tblgen",
+        "llvm-tgt-verify",
         "llvm-readtapi",
         "llvm-undname",
         "llvm-windres",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 946b63f8a54fb..36cf956545caf 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -320,6 +320,7 @@ group("test") {
     "//llvm/tools/llvm-strings",
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-tli-checker",
+    "//llvm/tools/llvm-tgt-verify",
     "//llvm/tools/llvm-undname",
     "//llvm/tools/llvm-xray",
     "//llvm/tools/lto",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
new file mode 100644
index 0000000000000..b751bafc5052c
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tgtverifier("llvm-tgt-verify") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmPrinter",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGenTypes",
+    "//llvm/lib/Core",
+    "//llvm/lib/IRPrinter",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/MC",
+    "//llvm/lib/MIRParser",
+    "//llvm/lib/Passes",
+    "//llvm/lib/Remarks",
+    "//llvm/lib/ScalarOpts",
+    "//llvm/lib/SelectionDAG",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/TargetParser",
+    "//llvm/lib/TransformUtils",
+    "//llvm/lib/Vectorize",
+  ]
+  sources = [ "llvm-tgt-verify.cpp" ]
+}

>From b51b530c5672945e06a20b93b1f652d097386e61 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 28 Apr 2025 10:42:24 -0400
Subject: [PATCH 17/33] Cleanup of unrequired functions.

---
 llvm/include/llvm/Target/TargetVerifier.h     |  1 -
 .../TargetVerify/AMDGPUTargetVerifier.h       |  1 -
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 25 +++----------------
 llvm/lib/Target/TargetVerifier.cpp            | 22 ++--------------
 4 files changed, 6 insertions(+), 43 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 23ef2e0b8d4ef..427a05b2648a9 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -77,7 +77,6 @@ class TargetVerify {
         MessagesStr(Messages) {}
 
   bool run(Function &F);
-  bool run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index b6a7412e8c1ef..74e5b5f7a1efd 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -32,7 +32,6 @@ class Function;
 
 class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 class AMDGPUTargetVerify : public TargetVerify {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 96bcaaf6f2ac9..bda412f723242 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -107,29 +107,12 @@ bool AMDGPUTargetVerify::run(Function &F) {
     }
   }
 
-  if (!MessagesStr.str().empty())
+  //dbgs() << MessagesStr.str();
+  if (!MessagesStr.str().empty()) {
+    //IsValid = false;
     return false;
-  return true;
-}
-
-PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
-
-  auto *Mod = F.getParent();
-
-  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-  TV.run(F);
-
-  dbgs() << TV.MessagesStr.str();
-  if (!TV.MessagesStr.str().empty()) {
-    TV.IsValid = false;
-    return PreservedAnalyses::none();
   }
-
-  return PreservedAnalyses::all();
+  return true;
 }
 
 } // namespace llvm
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 3be50f4ef6da3..6b57c18ff9316 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -48,25 +48,6 @@ bool TargetVerify::run(Function &F) {
   report_fatal_error("Target has no verification method\n");
 }
 
-bool TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
-  if (TT.isAMDGPU()) {
-    auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
-    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return false;
-    }
-    return true;
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
 PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto TT = F.getParent()->getTargetTriple();
 
@@ -123,11 +104,12 @@ struct TargetVerifierLegacyPass : public FunctionPass {
       if (F.isDeclaration())
         IsValid &= TV->run(F);
 
-    if (!IsValid)
+    if (!IsValid) {
       if (FatalErrors)
         report_fatal_error("broken module found, compilation aborted!");
       else
         errs() << "broken module found, compilation aborted!\n";
+    }
     return false;
   }
 

>From d41d5f9e6a33b70f71a20d58642fd73c9327e1e5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 28 Apr 2025 10:43:32 -0400
Subject: [PATCH 18/33] Make virtual.

---
 llvm/include/llvm/Target/TargetVerifier.h                    | 2 +-
 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 427a05b2648a9..ade2676a64325 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -76,7 +76,7 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  bool run(Function &F);
+  virtual bool run(Function &F);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 74e5b5f7a1efd..b97fbc046e391 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -48,7 +48,7 @@ class AMDGPUTargetVerify : public TargetVerify {
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
-  bool run(Function &F);
+  bool run(Function &F) override;
 };
 
 } // namespace llvm

>From 7ece5e262a4f4aff350a511af5d33496ac2d5604 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 10:56:43 -0400
Subject: [PATCH 19/33] Remove from legacy PM. Add to target dependent
 pipeline.

---
 llvm/include/llvm/InitializePasses.h          |  2 +-
 llvm/include/llvm/Target/TargetVerifier.h     |  4 +-
 .../TargetVerify/AMDGPUTargetVerifier.h       |  1 +
 llvm/lib/Passes/StandardInstrumentations.cpp  | 10 +--
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  2 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  2 +
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 74 ++++++++++++++-
 llvm/lib/Target/TargetVerifier.cpp            | 90 -------------------
 llvm/tools/llc/llc.cpp                        |  2 -
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 -
 10 files changed, 84 insertions(+), 105 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 662f6b71aa46a..d718f92877276 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,7 +333,7 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
-LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+//LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index ade2676a64325..1d12eb55bbf0a 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -30,7 +30,7 @@ class Function;
 
 class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  virtual PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) = 0;
 };
 
 class TargetVerify {
@@ -76,7 +76,7 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  virtual bool run(Function &F);
+  virtual bool run(Function &F) = 0;
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index b97fbc046e391..49bcbc8849e3c 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -32,6 +32,7 @@ class Function;
 
 class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
 };
 
 class AMDGPUTargetVerify : public TargetVerify {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 539f4b24b5110..905ba667fac7a 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,7 +40,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
-#include "llvm/Target/TargetVerifier.h"
+//#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -1474,12 +1474,12 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
                                        P));
 
           if (VerifyTargetEach && FAM) {
-            TargetVerify TV(const_cast<Module*>(F->getParent()));
-            TV.run(*const_cast<Function*>(F), *FAM);
-	    if (!TV.IsValid)
+            //TargetVerify TV(const_cast<Module*>(F->getParent()));
+            //TV.run(*const_cast<Function*>(F), *FAM);
+	    /*if (!TV.IsValid)
               report_fatal_error(formatv("Broken function found after pass "
                                          "\"{0}\", compilation aborted!",
-                                         P));
+                                         P));*/
           }
         } else {
           const auto *M = unwrapIR<Module>(IR);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index ad4fa93d09e0c..652cd23f62443 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -89,7 +89,7 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("tgtverifier", TargetVerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
 #undef VERIFIER_MODULE_ANALYSIS
 #undef VERIFIER_FUNCTION_ANALYSIS
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 0c41c633dcb3d..718655b7ebf00 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -2010,6 +2010,8 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
 }
 
 void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
+  addPass(AMDGPUTargetVerifierPass());
+
   if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
     addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index bda412f723242..cedd9ddc78011 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -107,12 +107,82 @@ bool AMDGPUTargetVerify::run(Function &F) {
     }
   }
 
-  //dbgs() << MessagesStr.str();
+  dbgs() << MessagesStr.str();
   if (!MessagesStr.str().empty()) {
-    //IsValid = false;
+    IsValid = false;
     return false;
   }
   return true;
 }
 
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto *Mod = F.getParent();
+
+  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  TV.run(F);
+
+  dbgs() << TV.MessagesStr.str();
+  if (!TV.MessagesStr.str().empty()) {
+    TV.IsValid = false;
+    return PreservedAnalyses::none();
+  }
+  return PreservedAnalyses::all();
+}
+
+/*
+struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<AMDGPUTargetVerify> TV;
+  bool FatalErrors = false;
+
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
+    FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<AMDGPUTargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (!TV->run(F)) {
+      errs() << "in function " << F.getName() << '\n';
+      if (FatalErrors)
+        report_fatal_error("broken function found, compilation aborted!");
+      else
+        errs() << "broken function found, compilation aborted!\n";
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    if (!IsValid) {
+      if (FatalErrors)
+        report_fatal_error("broken module found, compilation aborted!");
+      else
+        errs() << "broken module found, compilation aborted!\n";
+    }
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char AMDGPUTargetVerifierLegacyPass::ID = 0;
+FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
+  return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
+}*/
 } // namespace llvm
+//INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 6b57c18ff9316..c63ae2a2c5daf 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -33,94 +33,4 @@
 
 namespace llvm {
 
-bool TargetVerify::run(Function &F) {
-  if (TT.isAMDGPU()) {
-    AMDGPUTargetVerify TV(Mod);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return false;
-    }
-    return true;
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
-PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
-  auto TT = F.getParent()->getTargetTriple();
-
-  if (TT.isAMDGPU()) {
-    auto *Mod = F.getParent();
-
-    auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return PreservedAnalyses::none();
-    }
-    return PreservedAnalyses::all();
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
-struct TargetVerifierLegacyPass : public FunctionPass {
-  static char ID;
-
-  std::unique_ptr<TargetVerify> TV;
-  bool FatalErrors = false;
-
-  TargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
-    FatalErrors(FatalErrors) {
-    initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool doInitialization(Module &M) override {
-    TV = std::make_unique<TargetVerify>(&M);
-    return false;
-  }
-
-  bool runOnFunction(Function &F) override {
-    if (!TV->run(F)) {
-      errs() << "in function " << F.getName() << '\n';
-      if (FatalErrors)
-        report_fatal_error("broken function found, compilation aborted!");
-      else
-        errs() << "broken function found, compilation aborted!\n";
-    }
-    return false;
-  }
-
-  bool doFinalization(Module &M) override {
-    bool IsValid = true;
-    for (Function &F : M)
-      if (F.isDeclaration())
-        IsValid &= TV->run(F);
-
-    if (!IsValid) {
-      if (FatalErrors)
-        report_fatal_error("broken module found, compilation aborted!");
-      else
-        errs() << "broken module found, compilation aborted!\n";
-    }
-    return false;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-  }
-};
-char TargetVerifierLegacyPass::ID = 0;
-FunctionPass *createTargetVerifierLegacyPass(bool FatalErrors) {
-  return new TargetVerifierLegacyPass(FatalErrors);
-}
 } // namespace llvm
-using namespace llvm;
-INITIALIZE_PASS(TargetVerifierLegacyPass, "tgtverifier", "Target Verifier", false, false)
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 329d95826551f..2e9e4837fe467 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -660,8 +660,6 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
-  if (VerifyTarget)
-    PM.add(createTargetVerifierLegacyPass(false));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index b86c2318b45b7..d832dcdff4ad0 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -158,8 +158,6 @@ int main(int argc, char **argv) {
   if (TT.isAMDGPU())
     FPM.addPass(AMDGPUTargetVerifierPass());
   else if (false) {} // ...
-  else
-    FPM.addPass(TargetVerifierPass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
   auto PA = MPM.run(*M, MAM);

>From 42b89826fa18b4796c9fdee25864e929a2351661 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 11:18:50 -0400
Subject: [PATCH 20/33] Add back to legacy PM.

---
 llvm/include/llvm/InitializePasses.h            | 1 -
 llvm/lib/Target/AMDGPU/AMDGPU.h                 | 3 +++
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  | 1 +
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 8 ++++----
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index d718f92877276..1b5b1d5888824 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,7 +333,6 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
-//LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 5a917734e9c74..26fb0a967fd76 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -549,6 +549,9 @@ extern char &GCNRewritePartialRegUsesID;
 void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
 extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
+FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
+void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 718655b7ebf00..8b0b8f6df740e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1411,6 +1411,7 @@ bool AMDGPUPassConfig::addGCPasses() {
 //===----------------------------------------------------------------------===//
 
 bool GCNPassConfig::addPreISel() {
+  addPass(createAMDGPUTargetVerifierLegacyPass(false));
   AMDGPUPassConfig::addPreISel();
 
   if (TM->getOptLevel() > CodeGenOptLevel::None)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index cedd9ddc78011..c4d303bee6ef8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -17,6 +17,7 @@
 ////
 ////===----------------------------------------------------------------------===//
 
+#include "AMDGPU.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
 #include "llvm/Analysis/UniformityAnalysis.h"
@@ -24,7 +25,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-#include "llvm/InitializePasses.h"
+//#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -133,7 +134,6 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
   return PreservedAnalyses::all();
 }
 
-/*
 struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   static char ID;
 
@@ -183,6 +183,6 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
 char AMDGPUTargetVerifierLegacyPass::ID = 0;
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
   return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
-}*/
+}
 } // namespace llvm
-//INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)

>From 230f33b8db0650a59c5656d2caf5c7be31f0aa49 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 14:04:10 -0400
Subject: [PATCH 21/33] Remove reference to FAM in registerCallbacks and
 VerifyEach for TargetVerify in instrumentation

---
 clang/lib/CodeGen/BackendUtil.cpp             |  2 +-
 flang/lib/Frontend/FrontendActions.cpp        |  2 +-
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp   |  2 +-
 .../llvm/Passes/StandardInstrumentations.h    |  6 ++----
 llvm/lib/LTO/LTOBackend.cpp                   |  2 +-
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp         |  2 +-
 llvm/lib/Passes/PassBuilderBindings.cpp       |  2 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  | 21 ++++---------------
 llvm/tools/llc/NewPMDriver.cpp                |  7 ++++---
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 +-
 llvm/tools/opt/NewPMDriver.cpp                |  2 +-
 llvm/unittests/IR/PassManagerTest.cpp         |  6 +++---
 15 files changed, 24 insertions(+), 38 deletions(-)

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 560129f871026..7e0a3cf5591ce 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -917,7 +917,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       TheModule->getContext(),
       (CodeGenOpts.DebugPassManager || DebugPassStructure),
       CodeGenOpts.VerifyEach, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
   // Handle the assignment tracking feature options.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index c15151fc5c1b8..d684eeb696755 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -947,7 +947,7 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
 
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
-  si.registerCallbacks(pic, &mam, &fam);
+  si.registerCallbacks(pic, &mam);
   if (ci.isTimingEnabled())
     si.getTimePasses().setOutStream(ci.getTimingStreamLLVM());
   pto.LoopUnrolling = opts.UnrollLoops;
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index f9664025f61f1..0f58391c50667 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -577,7 +577,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index eae06d9f57467..7117eaf4982b0 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -851,7 +851,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index 30ad79ef2fc58..cb7b6cc8651c1 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -970,7 +970,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 4a39bc33c5591..91b7191a07c6f 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1139,7 +1139,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Promote allocas to registers.
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 7d7b858b447cc..df9d8de8a025a 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,8 +478,7 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM,
-			 FunctionAnalysisManager *FAM);
+                         ModuleAnalysisManager *MAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -625,8 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM,
-			 FunctionAnalysisManager *FAM);
+                         ModuleAnalysisManager *MAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 7ae23ef4e7df4..5e8cd12fe040b 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -273,7 +273,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
                               Conf.VerifyEach);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 
   RegisterPassPlugins(Conf.PassPlugins, PB);
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 8db92e70d5a96..5b333cda31fb8 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -241,7 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PipelineTuningOptions PTO;
   PTO.LoopVectorization = true;
   PTO.SLPVectorization = true;
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index f0e1abb8cebc4..933fe89e53a94 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -76,7 +76,7 @@ static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   // Run the pipeline.
   if (Fun) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 905ba667fac7a..4e8abc43572d7 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,7 +40,6 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
-//#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -57,8 +56,6 @@ static cl::opt<bool> VerifyAnalysisInvalidation("verify-analysis-invalidation",
 #endif
 );
 
-static cl::opt<bool> VerifyTargetEach("verify-tgt-each");
-
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
 // of this option.  Note that this option has no effect without -print-changed.
@@ -1452,10 +1449,9 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 }
 
 void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                              ModuleAnalysisManager *MAM,
-					      FunctionAnalysisManager *FAM) {
+                                              ModuleAnalysisManager *MAM) {
   PIC.registerAfterPassCallback(
-      [this, MAM, FAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+      [this, MAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
         const auto *F = unwrapIR<Function>(IR);
@@ -1472,15 +1468,6 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
             report_fatal_error(formatv("Broken function found after pass "
                                        "\"{0}\", compilation aborted!",
                                        P));
-
-          if (VerifyTargetEach && FAM) {
-            //TargetVerify TV(const_cast<Module*>(F->getParent()));
-            //TV.run(*const_cast<Function*>(F), *FAM);
-	    /*if (!TV.IsValid)
-              report_fatal_error(formatv("Broken function found after pass "
-                                         "\"{0}\", compilation aborted!",
-                                         P));*/
-          }
         } else {
           const auto *M = unwrapIR<Module>(IR);
           if (!M) {
@@ -2520,7 +2507,7 @@ void PrintCrashIRInstrumentation::registerCallbacks(
 }
 
 void StandardInstrumentations::registerCallbacks(
-    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM, FunctionAnalysisManager *FAM) {
+    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM) {
   PrintIR.registerCallbacks(PIC);
   PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
@@ -2529,7 +2516,7 @@ void StandardInstrumentations::registerCallbacks(
   PrintChangedIR.registerCallbacks(PIC);
   PseudoProbeVerification.registerCallbacks(PIC);
   if (VerifyEach)
-    Verify.registerCallbacks(PIC, MAM, FAM);
+    Verify.registerCallbacks(PIC, MAM);
   PrintChangedDiff.registerCallbacks(PIC);
   WebsiteChangeReporter.registerCallbacks(PIC);
   ChangeTester.registerCallbacks(PIC);
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 4b95977a10c5f..863a555798dab 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -117,8 +117,6 @@ int llvm::compileModuleWithNewPM(
                               VK == VerifierKind::EachPass);
   registerCodeGenCallback(PIC, *Target);
 
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
   MachineFunctionAnalysisManager MFAM;
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
@@ -133,11 +131,14 @@ int llvm::compileModuleWithNewPM(
   if (VerifyTarget)
     PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
 
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+
   if (!PassPipeline.empty()) {
     // Construct a custom pass pipeline that starts after instruction
     // selection.
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index d832dcdff4ad0..50f4e56bb6af6 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -148,7 +148,7 @@ int main(int argc, char **argv) {
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   Triple TT(M->getTargetTriple());
   if (!NoLint)
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index a8977d80bdf44..7d168a6ceb17c 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -423,7 +423,7 @@ bool llvm::runPassPipeline(
   PrintPassOpts.SkipAnalyses = DebugPM == DebugLogging::Quiet;
   StandardInstrumentations SI(M.getContext(), DebugPM != DebugLogging::None,
                               VK == VerifierKind::EachPass, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   DebugifyEachInstrumentation Debugify;
   DebugifyStatsMap DIStatsMap;
   DebugInfoPerPass DebugInfoBeforePass;
diff --git a/llvm/unittests/IR/PassManagerTest.cpp b/llvm/unittests/IR/PassManagerTest.cpp
index bb4db6120035f..a6487169224c2 100644
--- a/llvm/unittests/IR/PassManagerTest.cpp
+++ b/llvm/unittests/IR/PassManagerTest.cpp
@@ -828,7 +828,7 @@ TEST_F(PassManagerTest, FunctionPassCFGChecker) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -877,7 +877,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerInvalidateAnalysis) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -945,7 +945,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerWrapped) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });

>From c680041d5134d6edd47e8dec65d4986feb63fb7d Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 14:44:28 -0400
Subject: [PATCH 22/33] Remove references to registry

---
 llvm/include/llvm/Passes/PassBuilder.h        | 21 -------------------
 .../llvm/Passes/StandardInstrumentations.h    |  2 +-
 .../llvm/Passes/TargetPassRegistry.inc        | 12 -----------
 llvm/lib/Passes/PassBuilder.cpp               |  7 -------
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 11 ----------
 llvm/tools/llc/NewPMDriver.cpp                |  5 -----
 llvm/tools/llc/llc.cpp                        |  2 --
 7 files changed, 1 insertion(+), 59 deletions(-)

diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index c134f2d0c1112..f13b5c678a894 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -173,13 +173,6 @@ class PassBuilder {
   /// additional analyses.
   LLVM_ABI void registerLoopAnalyses(LoopAnalysisManager &LAM);
 
-  /// Registers all available verifier passes.
-  ///
-  /// This is an interface that can be used to populate a
-  /// \c ModuleAnalysisManager with all registered loop analyses. Callers can
-  /// still manually register any additional analyses.
-  void registerVerifierPasses(ModulePassManager &PM, FunctionPassManager &);
-
   /// Registers all available machine function analysis passes.
   ///
   /// This is an interface that can be used to populate a \c
@@ -583,15 +576,6 @@ class PassBuilder {
   }
   /// @}}
 
-  /// Register a callback for parsing an Verifier Name to populate
-  /// the given managers.
-  void registerVerifierCallback(
-      const std::function<bool(ModulePassManager &MPM)> &C,
-      const std::function<bool(FunctionPassManager &MPM)> &CF) {
-    VerifierCallbacks.push_back(C);
-    FnVerifierCallbacks.push_back(CF);
-  }
-
   /// {{@ Register pipeline parsing callbacks with this pass builder instance.
   /// Using these callbacks, callers can parse both a single pass name, as well
   /// as entire sub-pipelines, and populate the PassManager instance
@@ -867,11 +851,6 @@ class PassBuilder {
   // Callbacks to parse `filter` parameter in register allocation passes
   SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
-  // Verifier callbacks
-  SmallVector<std::function<bool(ModulePassManager &)>, 2>
-      VerifierCallbacks;
-  SmallVector<std::function<bool(FunctionPassManager &)>, 2>
-      FnVerifierCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index df9d8de8a025a..d7c1bbf04c328 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -624,7 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc
index 2d04b874cf360..521913cb25a4a 100644
--- a/llvm/include/llvm/Passes/TargetPassRegistry.inc
+++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc
@@ -151,18 +151,6 @@ PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM,
   return false;
 });
 
-PB.registerVerifierCallback([](ModulePassManager &PM) {
-#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS) PM.addPass(CREATE_PASS)
-#include GET_PASS_REGISTRY
-#undef VERIFIER_MODULE_ANALYSIS
-  return false;
-}, [](FunctionPassManager &FPM) {
-#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS) FPM.addPass(CREATE_PASS)
-#include GET_PASS_REGISTRY
-#undef VERIFIER_FUNCTION_ANALYSIS
-  return false;
-});
-
 #undef ADD_PASS
 #undef ADD_PASS_WITH_PARAMS
 
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 561996d0c8980..4603eaff8ade9 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -576,13 +576,6 @@ void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     C(LAM);
 }
 
-void PassBuilder::registerVerifierPasses(ModulePassManager &MPM, FunctionPassManager &FPM) {
-  for (auto &C : VerifierCallbacks)
-    C(MPM);
-  for (auto &C : FnVerifierCallbacks)
-    C(FPM);
-}
-
 static std::optional<std::pair<bool, bool>>
 parseFunctionPipelineName(StringRef Name) {
   std::pair<bool, bool> Params;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 652cd23f62443..13453963eec6d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -82,17 +82,6 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
-#ifndef VERIFIER_MODULE_ANALYSIS
-#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS)
-#endif
-#ifndef VERIFIER_FUNCTION_ANALYSIS
-#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
-#endif
-VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
-#undef VERIFIER_MODULE_ANALYSIS
-#undef VERIFIER_FUNCTION_ANALYSIS
-
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 863a555798dab..fa82689ecf9ae 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -57,9 +57,6 @@ static cl::opt<bool>
     DebugPM("debug-pass-manager", cl::Hidden,
             cl::desc("Print pass management debugging information"));
 
-static cl::opt<bool> VerifyTarget("verify-tgt-new-pm",
-		                  cl::desc("Verify the target"));
-
 bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
   DiagnosticHandler::handleDiagnostics(DI);
   if (DI.getKind() == llvm::DK_SrcMgr) {
@@ -128,8 +125,6 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
-  if (VerifyTarget)
-    PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM);
 
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 2e9e4837fe467..140459ba2de21 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -209,8 +209,6 @@ static cl::opt<std::string> PassPipeline(
 static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
                                cl::desc("Alias for -passes"));
 
-static cl::opt<bool> VerifyTarget("verify-tgt", cl::desc("Verify the target"));
-
 namespace {
 
 std::vector<std::string> &getRunPassNames() {

>From 3f5634a532856668a42f44ea04d0ffcc862c96d4 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 19:05:53 -0400
Subject: [PATCH 23/33] Remove int check

---
 .../lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index c4d303bee6ef8..2ca0bbeb57653 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -25,7 +25,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-//#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -46,15 +45,6 @@ using namespace llvm;
 
 namespace llvm {
 
-static bool IsValidInt(const Type *Ty) {
-  return Ty->isIntegerTy(1) ||
-         Ty->isIntegerTy(8) ||
-         Ty->isIntegerTy(16) ||
-         Ty->isIntegerTy(32) ||
-         Ty->isIntegerTy(64) ||
-         Ty->isIntegerTy(128);
-}
-
 static bool isShader(CallingConv::ID CC) {
   switch(CC) {
     case CallingConv::AMDGPU_VS:
@@ -81,14 +71,6 @@ bool AMDGPUTargetVerify::run(Function &F) {
 
     for (auto &I : BB) {
 
-      // Ensure integral types are valid: i8, i16, i32, i64, i128
-      if (I.getType()->isIntegerTy())
-        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
-      for (unsigned i = 0; i < I.getNumOperands(); ++i)
-        if (I.getOperand(i)->getType()->isIntegerTy())
-          Check(IsValidInt(I.getOperand(i)->getType()),
-                "Int type is invalid.", I.getOperand(i));
-
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
         // Ensure no kernel to kernel calls.

>From 6007895007af8925e32b709009c0d6131141c282 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 21:56:28 -0400
Subject: [PATCH 24/33] Remove modifications to Lint/Verifier.

---
 llvm/lib/Analysis/Lint.cpp |  4 +---
 llvm/lib/IR/Verifier.cpp   | 20 ++++----------------
 2 files changed, 5 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index c8627e6db0692..1168005f48c0e 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -735,11 +735,9 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) {
   Lint L(Mod, DL, AA, AC, DT, TLI);
   L.visit(F);
   dbgs() << L.MessagesStr.str();
-  if (AbortOnError && !L.MessagesStr.str().empty()) {
+  if (AbortOnError && !L.MessagesStr.str().empty())
     report_fatal_error(
         "linter found errors, aborting. (enabled by abort-on-error)", false);
-    return PreservedAnalyses::none();
-  }
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 439823ba8b682..1f1041b259736 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -135,10 +135,6 @@ static cl::opt<bool> VerifyNoAliasScopeDomination(
     cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
              "scopes are not dominating"));
 
-static cl::opt<bool>
-    VerifyAbortOnError("verifier-abort-on-error", cl::init(false),
-                       cl::desc("In the Verifier pass, abort on errors."));
-
 namespace llvm {
 
 struct VerifierSupport {
@@ -7869,24 +7865,16 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
-  if (Res.IRBroken || Res.DebugInfoBroken) {
-    //M.IsValid = false;
-    if (VerifyAbortOnError && FatalErrors)
-      report_fatal_error("Broken module found, compilation aborted!");
-    return PreservedAnalyses::none();
-  }
+  if (FatalErrors && (Res.IRBroken || Res.DebugInfoBroken))
+    report_fatal_error("Broken module found, compilation aborted!");
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
-  if (res.IRBroken) {
-    //F.getParent()->IsValid = false;
-    if (VerifyAbortOnError && FatalErrors)
-      report_fatal_error("Broken function found, compilation aborted!");
-    return PreservedAnalyses::none();
-  }
+  if (res.IRBroken && FatalErrors)
+    report_fatal_error("Broken function found, compilation aborted!");
 
   return PreservedAnalyses::all();
 }

>From 756c3ebee47f59f69a267da6079edafce4f595ca Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 22:21:47 -0400
Subject: [PATCH 25/33] Remove llvm-tgt-verify tool.

---
 llvm/test/CMakeLists.txt                      |   1 -
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        |  45 -----
 llvm/test/lit.cfg.py                          |   1 -
 llvm/tools/llvm-tgt-verify/CMakeLists.txt     |  34 ----
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 171 ------------------
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |   1 -
 .../llvm/tools/llvm-tgt-verify/BUILD.gn       |  25 ---
 7 files changed, 278 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify.ll
 delete mode 100644 llvm/tools/llvm-tgt-verify/CMakeLists.txt
 delete mode 100644 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
 delete mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn

diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index a2bb112db8746..2a6135da9a61e 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -136,7 +136,6 @@ set(LLVM_TEST_DEPENDS
           llvm-strip
           llvm-symbolizer
           llvm-tblgen
-          llvm-tgt-verify
           llvm-readtapi
           llvm-tli-checker
           llvm-undname
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
deleted file mode 100644
index 62b220d7d9f49..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
-
-define amdgpu_cs i32 @shader() {
-; CHECK: Shaders must return void
-  ret i32 0
-}
-
-define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
-; CHECK: Undefined behavior: Write to memory in const addrspace
-; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
-  %r = add i32 %a, %b
-  store i32 %r, ptr addrspace(4) %out
-  ret void
-}
-
-define amdgpu_kernel void @kernel_callee(ptr %x) {
-  ret void
-}
-
-define amdgpu_kernel void @kernel_caller(ptr %x) {
-; CHECK: A kernel may not call a kernel
-; CHECK-NEXT: ptr @kernel_caller
-  call amdgpu_kernel void @kernel_callee(ptr %x)
-  ret void
-}
-
-
-; Function Attrs: nounwind
-define i65 @invalid_type(i65 %x) #0 {
-; CHECK: Int type is invalid.
-; CHECK-NEXT: %tmp2 = ashr i65 %x, 64
-entry:
-  %tmp2 = ashr i65 %x, 64
-  ret i65 %tmp2
-}
-
-declare void @llvm.amdgcn.cs.chain.v3i32(ptr, i32, <3 x i32>, <3 x i32>, i32, ...)
-declare amdgpu_cs_chain void @chain_callee(<3 x i32> inreg, <3 x i32>)
-
-define amdgpu_cs void @no_unreachable(<3 x i32> inreg %a, <3 x i32> %b) {
-; CHECK: llvm.amdgcn.cs.chain must be followed by unreachable
-; CHECK-NEXT: call void (ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.v3i32.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
-  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
-  ret void
-}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index bf06e80dc7ae5..bd6e37c848d8c 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -228,7 +228,6 @@ def get_asan_rtlib():
         "llvm-strings",
         "llvm-strip",
         "llvm-tblgen",
-        "llvm-tgt-verify",
         "llvm-readtapi",
         "llvm-undname",
         "llvm-windres",
diff --git a/llvm/tools/llvm-tgt-verify/CMakeLists.txt b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
deleted file mode 100644
index fe47c85e6cdce..0000000000000
--- a/llvm/tools/llvm-tgt-verify/CMakeLists.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  AllTargetsAsmParsers
-  AllTargetsCodeGens
-  AllTargetsDescs
-  AllTargetsInfos
-  Analysis
-  AsmPrinter
-  CodeGen
-  CodeGenTypes
-  Core
-  IRPrinter
-  IRReader
-  MC
-  MIRParser
-  Passes
-  Remarks
-  ScalarOpts
-  SelectionDAG
-  Support
-  Target
-  TargetParser
-  TransformUtils
-  Vectorize
-  )
-
-add_llvm_tool(llvm-tgt-verify
-  llvm-tgt-verify.cpp
-
-  DEPENDS
-  intrinsics_gen
-  SUPPORT_PLUGINS
-  )
-
-export_executable_symbols_for_plugins(llc)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
deleted file mode 100644
index 50f4e56bb6af6..0000000000000
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ /dev/null
@@ -1,171 +0,0 @@
-//===--- llvm-tgt-verify.cpp - Target Verifier ----------------- ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Tool to verify a target.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/InitializePasses.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/Lint.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Bitcode/BitcodeReader.h"
-#include "llvm/Bitcode/BitcodeWriter.h"
-#include "llvm/CodeGen/CommandFlags.h"
-#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Verifier.h"
-#include "llvm/IRReader/IRReader.h"
-#include "llvm/Passes/PassBuilder.h"
-#include "llvm/Passes/StandardInstrumentations.h"
-#include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/InitLLVM.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/TargetSelect.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetVerifier.h"
-
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
-
-#define DEBUG_TYPE "isel-fuzzer"
-
-using namespace llvm;
-
-static codegen::RegisterCodeGenFlags CGF;
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-static cl::opt<bool>
-    StacktraceAbort("stacktrace-abort",
-        cl::desc("Turn on stacktrace"), cl::init(false));
-
-static cl::opt<bool>
-    NoLint("no-lint",
-        cl::desc("Turn off Lint"), cl::init(false));
-
-static cl::opt<bool>
-    NoVerify("no-verifier",
-        cl::desc("Turn off Verifier"), cl::init(false));
-
-static cl::opt<char>
-    OptLevel("O",
-             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
-                      "(default = '-O2')"),
-             cl::Prefix, cl::init('2'));
-
-static cl::opt<std::string>
-    TargetTriple("mtriple", cl::desc("Override target triple for module"));
-
-static std::unique_ptr<TargetMachine> TM;
-
-static void handleLLVMFatalError(void *, const char *Message, bool) {
-  if (StacktraceAbort) {
-    dbgs() << "LLVM ERROR: " << Message << "\n"
-           << "Aborting.\n";
-    abort();
-  }
-}
-
-int main(int argc, char **argv) {
-  StringRef ExecName = argv[0];
-  InitLLVM X(argc, argv);
-
-  InitializeAllTargets();
-  InitializeAllTargetMCs();
-  InitializeAllAsmPrinters();
-  InitializeAllAsmParsers();
-
-  PassRegistry *Registry = PassRegistry::getPassRegistry();
-  initializeCore(*Registry);
-  initializeCodeGen(*Registry);
-  initializeAnalysis(*Registry);
-  initializeTarget(*Registry);
-
-  cl::ParseCommandLineOptions(argc, argv);
-
-  if (TargetTriple.empty()) {
-    errs() << ExecName << ": -mtriple must be specified\n";
-    exit(1);
-  }
-
-  CodeGenOptLevel OLvl;
-  if (auto Level = CodeGenOpt::parseLevel(OptLevel)) {
-    OLvl = *Level;
-  } else {
-    errs() << ExecName << ": invalid optimization level.\n";
-    return 1;
-  }
-  ExitOnError ExitOnErr(std::string(ExecName) + ": error:");
-  TM = ExitOnErr(codegen::createTargetMachineForTriple(
-      Triple::normalize(TargetTriple), OLvl));
-  assert(TM && "Could not allocate target machine!");
-
-  // Make sure we print the summary and the current unit when LLVM errors out.
-  install_fatal_error_handler(handleLLVMFatalError, nullptr);
-
-  LLVMContext Context;
-  SMDiagnostic Err;
-  std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
-  if (!M) {
-    errs() << "Invalid mod\n";
-    return 1;
-  }
-  auto S = Triple::normalize(TargetTriple);
-  M->setTargetTriple(Triple(S));
-
-  PassInstrumentationCallbacks PIC;
-  StandardInstrumentations SI(Context, false/*debug PM*/,
-                              false);
-  registerCodeGenCallback(PIC, *TM);
-
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
-  //TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
-
-  MachineFunctionAnalysisManager MFAM;
-  LoopAnalysisManager LAM;
-  FunctionAnalysisManager FAM;
-  CGSCCAnalysisManager CGAM;
-  ModuleAnalysisManager MAM;
-  PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
-  PB.registerModuleAnalyses(MAM);
-  //PB.registerVerifierPasses(MPM, FPM);
-  PB.registerCGSCCAnalyses(CGAM);
-  PB.registerFunctionAnalyses(FAM);
-  PB.registerLoopAnalyses(LAM);
-  PB.registerMachineFunctionAnalyses(MFAM);
-  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-
-  SI.registerCallbacks(PIC, &MAM);
-
-  Triple TT(M->getTargetTriple());
-  if (!NoLint)
-    FPM.addPass(LintPass(false));
-  if (!NoVerify)
-    MPM.addPass(VerifierPass());
-  if (TT.isAMDGPU())
-    FPM.addPass(AMDGPUTargetVerifierPass());
-  else if (false) {} // ...
-  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
-
-  auto PA = MPM.run(*M, MAM);
-  {
-    auto PAC = PA.getChecker<VerifierAnalysis>();
-    if (!PAC.preserved())
-      return 1;
-  }
-
-  return 0;
-}
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 36cf956545caf..946b63f8a54fb 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -320,7 +320,6 @@ group("test") {
     "//llvm/tools/llvm-strings",
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-tli-checker",
-    "//llvm/tools/llvm-tgt-verify",
     "//llvm/tools/llvm-undname",
     "//llvm/tools/llvm-xray",
     "//llvm/tools/lto",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
deleted file mode 100644
index b751bafc5052c..0000000000000
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
+++ /dev/null
@@ -1,25 +0,0 @@
-import("//llvm/utils/TableGen/tablegen.gni")
-
-tgtverifier("llvm-tgt-verify") {
-  deps = [
-    "//llvm/lib/Analysis",
-    "//llvm/lib/AsmPrinter",
-    "//llvm/lib/CodeGen",
-    "//llvm/lib/CodeGenTypes",
-    "//llvm/lib/Core",
-    "//llvm/lib/IRPrinter",
-    "//llvm/lib/IRReader",
-    "//llvm/lib/MC",
-    "//llvm/lib/MIRParser",
-    "//llvm/lib/Passes",
-    "//llvm/lib/Remarks",
-    "//llvm/lib/ScalarOpts",
-    "//llvm/lib/SelectionDAG",
-    "//llvm/lib/Support",
-    "//llvm/lib/Target",
-    "//llvm/lib/TargetParser",
-    "//llvm/lib/TransformUtils",
-    "//llvm/lib/Vectorize",
-  ]
-  sources = [ "llvm-tgt-verify.cpp" ]
-}

>From 3f1d72ff01d0303c0fc17d9a6c40e46661bf1fb0 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 22:23:15 -0400
Subject: [PATCH 26/33] Remove TargetVerifier.cpp

---
 llvm/lib/Passes/CMakeLists.txt     |  1 -
 llvm/lib/Target/CMakeLists.txt     |  2 --
 llvm/lib/Target/TargetVerifier.cpp | 36 ------------------------------
 3 files changed, 39 deletions(-)
 delete mode 100644 llvm/lib/Target/TargetVerifier.cpp

diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index b8d25b15186e8..91c8c4f67074d 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,7 +30,6 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
-  #TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index f2a5d545ce84f..9472288229cac 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -7,8 +7,6 @@ add_llvm_component_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
-  TargetVerifier.cpp
-  AMDGPU/AMDGPUTargetVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Target
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
deleted file mode 100644
index c63ae2a2c5daf..0000000000000
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-//===-- TargetVerifier.cpp - LLVM IR Target Verifier ----------------*- C++ -*-===//
-////
-///// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-///// See https://llvm.org/LICENSE.txt for license information.
-///// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-/////
-/////===----------------------------------------------------------------------===//
-/////
-///// This file defines target verifier interfaces that can be used for some
-///// validation of input to the system, and for checking that transformations
-///// haven't done something bad. In contrast to the Verifier or Lint, the
-///// TargetVerifier looks for constructions invalid to a particular target
-///// machine.
-/////
-///// To see what specifically is checked, look at TargetVerifier.cpp or an
-///// individual backend's TargetVerifier.
-/////
-/////===----------------------------------------------------------------------===//
-
-#include "llvm/Target/TargetVerifier.h"
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
-
-#include "llvm/InitializePasses.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/IR/Dominators.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Value.h"
-
-namespace llvm {
-
-} // namespace llvm

>From 1c721cf05616091cf3c42e4c244400a393abe755 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 1 May 2025 03:02:30 -0400
Subject: [PATCH 27/33] clang-format

---
 llvm/include/llvm/Target/TargetVerifier.h     | 10 +-
 .../TargetVerify/AMDGPUTargetVerifier.h       | 44 ++++-----
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 98 ++++++++++---------
 3 files changed, 77 insertions(+), 75 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 1d12eb55bbf0a..3f8c710a88768 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -1,4 +1,4 @@
-//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier ---*- C++ -*-===//
+//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier --*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -20,8 +20,8 @@
 #ifndef LLVM_TARGET_VERIFIER_H
 #define LLVM_TARGET_VERIFIER_H
 
-#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
@@ -59,10 +59,11 @@ class TargetVerify {
   /// This calls the Message-only version so that the above is easier to set
   /// a breakpoint on.
   template <typename T1, typename... Ts>
-  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) {
+  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
     CheckFailed(Message);
     WriteValues({V1, Vs...});
   }
+
 public:
   Module *Mod;
   Triple TT;
@@ -73,8 +74,7 @@ class TargetVerify {
   bool IsValid = true;
 
   TargetVerify(Module *Mod)
-      : Mod(Mod), TT(Mod->getTargetTriple()),
-        MessagesStr(Messages) {}
+      : Mod(Mod), TT(Mod->getTargetTriple()), MessagesStr(Messages) {}
 
   virtual bool run(Function &F) = 0;
 };
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 49bcbc8849e3c..5b8d9ec259b63 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -1,29 +1,29 @@
-//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU ---*- C++ -*-===//
-////
-//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//// See https://llvm.org/LICENSE.txt for license information.
-//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-////
-////===----------------------------------------------------------------------===//
-////
-//// This file defines target verifier interfaces that can be used for some
-//// validation of input to the system, and for checking that transformations
-//// haven't done something bad. In contrast to the Verifier or Lint, the
-//// TargetVerifier looks for constructions invalid to a particular target
-//// machine.
-////
-//// To see what specifically is checked, look at an individual backend's
-//// TargetVerifier.
-////
-////===----------------------------------------------------------------------===//
+//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU -- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
 
 #ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
 #define LLVM_AMDGPU_TARGET_VERIFIER_H
 
 #include "llvm/Target/TargetVerifier.h"
 
-#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/IR/Dominators.h"
 
 namespace llvm {
@@ -43,10 +43,10 @@ class AMDGPUTargetVerify : public TargetVerify {
   PostDominatorTree *PDT = nullptr;
   UniformityInfo *UA = nullptr;
 
-  AMDGPUTargetVerify(Module *Mod)
-    : TargetVerify(Mod), Mod(Mod) {}
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod), Mod(Mod) {}
 
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
+                     UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   bool run(Function &F) override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 2ca0bbeb57653..eb22eb2177f7f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -1,34 +1,34 @@
-//===-- AMDGPUTargetVerifier.cpp - AMDGPU -------------------------*- C++ -*-===//
-////
-//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//// See https://llvm.org/LICENSE.txt for license information.
-//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-////
-////===----------------------------------------------------------------------===//
-////
-//// This file defines target verifier interfaces that can be used for some
-//// validation of input to the system, and for checking that transformations
-//// haven't done something bad. In contrast to the Verifier or Lint, the
-//// TargetVerifier looks for constructions invalid to a particular target
-//// machine.
-////
-//// To see what specifically is checked, look at an individual backend's
-//// TargetVerifier.
-////
-////===----------------------------------------------------------------------===//
+//===-- AMDGPUTargetVerifier.cpp - AMDGPU -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
 
-#include "AMDGPU.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+#include "AMDGPU.h"
 
-#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Debug.h"
 
 #include "llvm/Support/raw_ostream.h"
 
@@ -39,53 +39,52 @@ using namespace llvm;
   do {                                                                         \
     if (!(C)) {                                                                \
       TargetVerify::CheckFailed(__VA_ARGS__);                                  \
-      return false;                                                                  \
     }                                                                          \
   } while (false)
 
 namespace llvm {
 
 static bool isShader(CallingConv::ID CC) {
-  switch(CC) {
-    case CallingConv::AMDGPU_VS:
-    case CallingConv::AMDGPU_LS:
-    case CallingConv::AMDGPU_HS:
-    case CallingConv::AMDGPU_ES:
-    case CallingConv::AMDGPU_GS:
-    case CallingConv::AMDGPU_PS:
-    case CallingConv::AMDGPU_CS_Chain:
-    case CallingConv::AMDGPU_CS_ChainPreserve:
-    case CallingConv::AMDGPU_CS:
-      return true;
-    default:
-      return false;
+  switch (CC) {
+  case CallingConv::AMDGPU_VS:
+  case CallingConv::AMDGPU_LS:
+  case CallingConv::AMDGPU_HS:
+  case CallingConv::AMDGPU_ES:
+  case CallingConv::AMDGPU_GS:
+  case CallingConv::AMDGPU_PS:
+  case CallingConv::AMDGPU_CS_Chain:
+  case CallingConv::AMDGPU_CS_ChainPreserve:
+  case CallingConv::AMDGPU_CS:
+    return true;
+  default:
+    return false;
   }
 }
 
 bool AMDGPUTargetVerify::run(Function &F) {
   // Ensure shader calling convention returns void
   if (isShader(F.getCallingConv()))
-    Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
+    Check(F.getReturnType() == Type::getVoidTy(F.getContext()),
+          "Shaders must return void");
 
   for (auto &BB : F) {
 
     for (auto &I : BB) {
 
-      if (auto *CI = dyn_cast<CallInst>(&I))
-      {
+      if (auto *CI = dyn_cast<CallInst>(&I)) {
         // Ensure no kernel to kernel calls.
         CallingConv::ID CalleeCC = CI->getCallingConv();
-        if (CalleeCC == CallingConv::AMDGPU_KERNEL)
-        {
-          CallingConv::ID CallerCC = CI->getParent()->getParent()->getCallingConv();
+        if (CalleeCC == CallingConv::AMDGPU_KERNEL) {
+          CallingConv::ID CallerCC =
+              CI->getParent()->getParent()->getCallingConv();
           Check(CallerCC != CallingConv::AMDGPU_KERNEL,
-            "A kernel may not call a kernel", CI->getParent()->getParent());
+                "A kernel may not call a kernel", CI->getParent()->getParent());
         }
 
         // Ensure chain intrinsics are followed by unreachables.
         if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
           Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
-            "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
+                "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
     }
   }
@@ -98,7 +97,8 @@ bool AMDGPUTargetVerify::run(Function &F) {
   return true;
 }
 
-PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F,
+                                                FunctionAnalysisManager &AM) {
   auto *Mod = F.getParent();
 
   auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
@@ -122,9 +122,10 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   std::unique_ptr<AMDGPUTargetVerify> TV;
   bool FatalErrors = false;
 
-  AMDGPUTargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
-    FatalErrors(FatalErrors) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
+      : FunctionPass(ID), FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(
+        *PassRegistry::getPassRegistry());
   }
 
   bool doInitialization(Module &M) override {
@@ -167,4 +168,5 @@ FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
   return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
 }
 } // namespace llvm
-INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier",
+                "AMDGPU Target Verifier", false, false)

>From f59b0f27beb766516186dcab8bde8463734ee64b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 1 May 2025 03:49:20 -0400
Subject: [PATCH 28/33] Add VerifyTarget option

---
 .../llvm/Target/TargetVerify/AMDGPUTargetVerifier.h      | 6 ++----
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp           | 9 +++++++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 5b8d9ec259b63..8ed7dd7ea2f69 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -37,17 +37,15 @@ class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 
 class AMDGPUTargetVerify : public TargetVerify {
 public:
-  Module *Mod;
-
   DominatorTree *DT = nullptr;
   PostDominatorTree *PDT = nullptr;
   UniformityInfo *UA = nullptr;
 
-  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod), Mod(Mod) {}
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
 
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
                      UniformityInfo *UA)
-    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   bool run(Function &F) override;
 };
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8b0b8f6df740e..be8bde1c1c0e0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -482,6 +482,9 @@ static cl::opt<bool> HasClosedWorldAssumption(
     cl::desc("Whether has closed-world assumption at link time"),
     cl::init(false), cl::Hidden);
 
+static cl::opt<bool> VerifyTarget("verify-tgt",
+                                  cl::desc("Enable the target verifier"));
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
@@ -1411,7 +1414,8 @@ bool AMDGPUPassConfig::addGCPasses() {
 //===----------------------------------------------------------------------===//
 
 bool GCNPassConfig::addPreISel() {
-  addPass(createAMDGPUTargetVerifierLegacyPass(false));
+  if (VerifyTarget)
+    addPass(createAMDGPUTargetVerifierLegacyPass(false));
   AMDGPUPassConfig::addPreISel();
 
   if (TM->getOptLevel() > CodeGenOptLevel::None)
@@ -2011,7 +2015,8 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
 }
 
 void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
-  addPass(AMDGPUTargetVerifierPass());
+  if (VerifyTarget)
+    addPass(AMDGPUTargetVerifierPass());
 
   if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
     addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));

>From f0f12f552a0a48eda3e1dc0b7db88e605067f945 Mon Sep 17 00:00:00 2001
From: Joey F <jofernau at amd.com>
Date: Wed, 7 May 2025 21:59:43 -0400
Subject: [PATCH 29/33] Remove AMDGPUTargetVerifier.h

---
 .../TargetVerify/AMDGPUTargetVerifier.h       | 55 -------------------
 llvm/lib/Target/AMDGPU/AMDGPU.h               |  5 ++
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  1 -
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 16 +++++-
 4 files changed, 20 insertions(+), 57 deletions(-)
 delete mode 100644 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h

diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
deleted file mode 100644
index 8ed7dd7ea2f69..0000000000000
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU -- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines target verifier interfaces that can be used for some
-// validation of input to the system, and for checking that transformations
-// haven't done something bad. In contrast to the Verifier or Lint, the
-// TargetVerifier looks for constructions invalid to a particular target
-// machine.
-//
-// To see what specifically is checked, look at an individual backend's
-// TargetVerifier.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
-#define LLVM_AMDGPU_TARGET_VERIFIER_H
-
-#include "llvm/Target/TargetVerifier.h"
-
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/IR/Dominators.h"
-
-namespace llvm {
-
-class Function;
-
-class AMDGPUTargetVerifierPass : public TargetVerifierPass {
-public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
-};
-
-class AMDGPUTargetVerify : public TargetVerify {
-public:
-  DominatorTree *DT = nullptr;
-  PostDominatorTree *PDT = nullptr;
-  UniformityInfo *UA = nullptr;
-
-  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
-                     UniformityInfo *UA)
-      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
-  bool run(Function &F) override;
-};
-
-} // namespace llvm
-
-#endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 26fb0a967fd76..f5f083ca066a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -15,6 +15,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Target/TargetVerifier.h"
 
 namespace llvm {
 
@@ -551,6 +552,10 @@ extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
 void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+class AMDGPUTargetVerifierPass : public TargetVerifierPass {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
+};
 
 namespace AMDGPU {
 enum TargetIndex {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index be8bde1c1c0e0..2009459aac329 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,7 +90,6 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/FormatVariadic.h"
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index eb22eb2177f7f..cc96164a1a558 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -17,7 +17,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "AMDGPU.h"
 
 #include "llvm/Analysis/PostDominators.h"
@@ -44,6 +43,21 @@ using namespace llvm;
 
 namespace llvm {
 
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  DominatorTree *DT = nullptr;
+  PostDominatorTree *PDT = nullptr;
+  UniformityInfo *UA = nullptr;
+
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
+                     UniformityInfo *UA)
+      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  bool run(Function &F) override;
+};
+
 static bool isShader(CallingConv::ID CC) {
   switch (CC) {
   case CallingConv::AMDGPU_VS:

>From 88a6d6b0faeff37cd98be19fd88632f9546720be Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 7 May 2025 22:47:55 -0400
Subject: [PATCH 30/33] Remove analyses.

---
 .../lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index cc96164a1a558..d07b5fb668c14 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -19,9 +19,6 @@
 
 #include "AMDGPU.h"
 
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -45,16 +42,7 @@ namespace llvm {
 
 class AMDGPUTargetVerify : public TargetVerify {
 public:
-  DominatorTree *DT = nullptr;
-  PostDominatorTree *PDT = nullptr;
-  UniformityInfo *UA = nullptr;
-
   AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
-                     UniformityInfo *UA)
-      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
   bool run(Function &F) override;
 };
 
@@ -115,11 +103,7 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F,
                                                 FunctionAnalysisManager &AM) {
   auto *Mod = F.getParent();
 
-  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  AMDGPUTargetVerify TV(Mod);
   TV.run(F);
 
   dbgs() << TV.MessagesStr.str();

>From 0d99a659957a043ec7dfa4762a1f1c5bd34e50b0 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 8 May 2025 12:59:01 -0400
Subject: [PATCH 31/33] function name style, TargetVerifier comment, option
 name generality

---
 llvm/include/llvm/Target/TargetVerifier.h       | 14 +++++++-------
 llvm/lib/Target/AMDGPU/AMDGPU.h                 |  3 +--
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp |  2 +-
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll |  2 +-
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll |  2 +-
 6 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 3f8c710a88768..eff846694ac10 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -12,8 +12,8 @@
 // TargetVerifier looks for constructions invalid to a particular target
 // machine.
 //
-// To see what specifically is checked, look at TargetVerifier.cpp or an
-// individual backend's TargetVerifier.
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
 //
 //===----------------------------------------------------------------------===//
 
@@ -35,7 +35,7 @@ class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 
 class TargetVerify {
 protected:
-  void WriteValues(ArrayRef<const Value *> Vs) {
+  void writeValues(ArrayRef<const Value *> Vs) {
     for (const Value *V : Vs) {
       if (!V)
         continue;
@@ -52,16 +52,16 @@ class TargetVerify {
   ///
   /// This provides a nice place to put a breakpoint if you want to see why
   /// something is not correct.
-  void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
+  void checkFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
 
   /// A check failed (with values to print).
   ///
   /// This calls the Message-only version so that the above is easier to set
   /// a breakpoint on.
   template <typename T1, typename... Ts>
-  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
-    CheckFailed(Message);
-    WriteValues({V1, Vs...});
+  void checkFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
+    checkFailed(Message);
+    writeValues({V1, Vs...});
   }
 
 public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index f5f083ca066a2..af09a7e4547f3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -552,8 +552,7 @@ extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
 void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
-class AMDGPUTargetVerifierPass : public TargetVerifierPass {
-public:
+struct AMDGPUTargetVerifierPass : public TargetVerifierPass {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
 };
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 2009459aac329..370fde6cce6ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -481,7 +481,7 @@ static cl::opt<bool> HasClosedWorldAssumption(
     cl::desc("Whether has closed-world assumption at link time"),
     cl::init(false), cl::Hidden);
 
-static cl::opt<bool> VerifyTarget("verify-tgt",
+static cl::opt<bool> VerifyTarget("amdgpu-verify-tgt",
                                   cl::desc("Enable the target verifier"));
 
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index d07b5fb668c14..3246b0ac0c624 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -34,7 +34,7 @@ using namespace llvm;
 #define Check(C, ...)                                                          \
   do {                                                                         \
     if (!(C)) {                                                                \
-      TargetVerify::CheckFailed(__VA_ARGS__);                                  \
+      TargetVerify::checkFailed(__VA_ARGS__);                                  \
     }                                                                          \
   } while (false)
 
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index e2d9edda5d008..819e66bb46b0a 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
 ; CHECK: Shaders must return void
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index a2dab0ff47924..08854df01acb2 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
 ; CHECK-NOT: Shaders must return void

>From 79215beef3f1330292ecee15b422f5cbfdeea9b5 Mon Sep 17 00:00:00 2001
From: Joey F <jofernau at amd.com>
Date: Mon, 16 Jun 2025 19:09:24 -0400
Subject: [PATCH 32/33] Fix after rebase

---
 llvm/include/llvm/CodeGen/Passes.h                  | 2 --
 llvm/include/llvm/IR/Module.h                       | 5 -----
 llvm/include/llvm/Passes/StandardInstrumentations.h | 4 ++--
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 53bd70183e476..990452fa11fec 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -620,8 +620,6 @@ LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
-
-LLVM_ABI FunctionPass *createTargetVerifierLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 34ac47a2b8be4..a99937a90cbb7 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,11 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module using intrinsics to record the position of debugging
-  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
-  /// \ref BasicBlock.
-  bool IsNewDbgInfoFormat;
-
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index d7c1bbf04c328..4ee5ab2554868 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,7 +478,7 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -624,7 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM = nullptr);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };

>From 374424f476d05d009195b6be97b25bc5f012e590 Mon Sep 17 00:00:00 2001
From: jofrn <jofernau at amd.com>
Date: Tue, 8 Jul 2025 08:02:10 -0400
Subject: [PATCH 33/33] Add back int check. These are invalid for AMDGPU.

---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 49 ++++++-------------
 1 file changed, 15 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 3246b0ac0c624..ae35322c905f1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -46,48 +46,29 @@ class AMDGPUTargetVerify : public TargetVerify {
   bool run(Function &F) override;
 };
 
-static bool isShader(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::AMDGPU_VS:
-  case CallingConv::AMDGPU_LS:
-  case CallingConv::AMDGPU_HS:
-  case CallingConv::AMDGPU_ES:
-  case CallingConv::AMDGPU_GS:
-  case CallingConv::AMDGPU_PS:
-  case CallingConv::AMDGPU_CS_Chain:
-  case CallingConv::AMDGPU_CS_ChainPreserve:
-  case CallingConv::AMDGPU_CS:
-    return true;
-  default:
-    return false;
-  }
+static bool IsValidInt(const Type *Ty) {
+  return Ty->isIntegerTy(1) ||
+         Ty->isIntegerTy(8) ||
+         Ty->isIntegerTy(16) ||
+         Ty->isIntegerTy(32) ||
+         Ty->isIntegerTy(64) ||
+         Ty->isIntegerTy(128);
 }
 
 bool AMDGPUTargetVerify::run(Function &F) {
-  // Ensure shader calling convention returns void
-  if (isShader(F.getCallingConv()))
-    Check(F.getReturnType() == Type::getVoidTy(F.getContext()),
-          "Shaders must return void");
 
   for (auto &BB : F) {
 
     for (auto &I : BB) {
 
-      if (auto *CI = dyn_cast<CallInst>(&I)) {
-        // Ensure no kernel to kernel calls.
-        CallingConv::ID CalleeCC = CI->getCallingConv();
-        if (CalleeCC == CallingConv::AMDGPU_KERNEL) {
-          CallingConv::ID CallerCC =
-              CI->getParent()->getParent()->getCallingConv();
-          Check(CallerCC != CallingConv::AMDGPU_KERNEL,
-                "A kernel may not call a kernel", CI->getParent()->getParent());
-        }
-
-        // Ensure chain intrinsics are followed by unreachables.
-        if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
-          Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
-                "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
-      }
+      // Ensure integral types are valid: i8, i16, i32, i64, i128
+      if (I.getType()->isIntegerTy())
+        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
+      for (unsigned i = 0; i < I.getNumOperands(); ++i)
+        if (I.getOperand(i)->getType()->isIntegerTy())
+          Check(IsValidInt(I.getOperand(i)->getType()),
+                "Int type is invalid.", I.getOperand(i));
+
     }
   }
 


From flang-commits at lists.llvm.org  Tue Jul  8 07:02:43 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Tue, 08 Jul 2025 07:02:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686d2503.170a0220.26e143.0558@mx.google.com>


================
@@ -805,13 +805,12 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
+      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
----------------
luporl wrote:

Thanks for the review.

On line 786 there is a call to `hlfir::derefPointersAndAllocatables`:
`lhs = hlfir::derefPointersAndAllocatables(loc, builder, lhs);`
Shouldn't it handle fir.heap/fir.ptr already?

Also, after getting the flat array reference type, `lhs` is converted to it. Would this convert be valid for fir.heap/fir.ptr?

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Tue Jul  8 07:40:54 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 07:40:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Do not invoke undefined behaviour
 when parsing format expressions (PR #147539)
Message-ID: <llvm.org/llvm/llvm-project/pull/147539@github.com>


https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/147539

The test flang/test/Semantics/io08.f90 was failing when UBSAN was enabled:
```
/home/david.spickett/llvm-project/flang/include/flang/Common/format.h:224:26: runtime error: signed integer overflow: 10 * 987654321098765432 cannot be represented in type 'int64_t' (aka 'long') SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/david.spickett/llvm-project/flang/include/flang/Common/format.h:224:26 error: Semantic errors in /home/david.spickett/llvm-project/flang/test/Semantics/io08.f90 /home/david.spickett/llvm-project/flang/test/Semantics/io08.f90:2:14: error: Integer overflow in format expression
    write(*,'(I98765432109876543210)')
               ^^^^^^^^^^^^^^^^^^^^
```
This is because the code was effectively:
* Take the risk of UB happening
* Check whether it happened or not

Which UBSAN is obviously not going to like. Instead of checking after the fact, use llvm's helpers that catch overflow without actually doing it.

>From d1e410a35bfd788e5062c566b7b031c6758a9c18 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Tue, 8 Jul 2025 14:36:35 +0000
Subject: [PATCH] [flang] Do not invoke undefined behaviour when parsing format
 expressions

The test flang/test/Semantics/io08.f90 was failing when UBSAN was enabled:
/home/david.spickett/llvm-project/flang/include/flang/Common/format.h:224:26: runtime error: signed integer overflow: 10 * 987654321098765432 cannot be represented in type 'int64_t' (aka 'long')
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/david.spickett/llvm-project/flang/include/flang/Common/format.h:224:26
error: Semantic errors in /home/david.spickett/llvm-project/flang/test/Semantics/io08.f90
/home/david.spickett/llvm-project/flang/test/Semantics/io08.f90:2:14: error: Integer overflow in format expression
    write(*,'(I98765432109876543210)')
               ^^^^^^^^^^^^^^^^^^^^

This is because the code was effectively:
* Take the risk of UB happening
* Check whether it happened or not

Which UBSAN is obviously not going to like. Instead of checking after the
fact, use llvm's helpers that catch overflow without actually doing it.
---
 flang/include/flang/Common/format.h | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 1650f56140b4d..11aa6f2c07797 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -11,6 +11,7 @@
 
 #include "Fortran-consts.h"
 #include "enum-set.h"
+#include "llvm/Support/MathExtras.h"
 #include <cstring>
 
 // Define a FormatValidator class template to validate a format expression
@@ -214,16 +215,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::MulOverflow(
+            static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
       }
       c = NextChar();
     } while (c >= '0' && c <= '9');


From flang-commits at lists.llvm.org  Tue Jul  8 07:42:30 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 07:42:30 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Do not invoke undefined behaviour
 when parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686d2e56.050a0220.10e156.587b@mx.google.com>


================
@@ -214,16 +215,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::MulOverflow(
+            static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
----------------
DavidSpickett wrote:

You could write this as `if (overflow = MulOverflow(...); !overflow) { overflow =`, or `overflow = MulOverflow(...) || AddOverflow`.

However when I tried those I forgot that we actually want to latch overflow, if its ever set to true it must stay true. So I think this multiple if is cumbersome but the most clear way to write it.

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Tue Jul  8 07:43:27 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 07:43:27 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Do not invoke undefined behaviour
 when parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686d2e8f.a70a0220.183b52.503a@mx.google.com>


DavidSpickett wrote:

Side note: This is the last test failing on AArch64 Linux with UBSAN enabled.

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Tue Jul  8 07:43:39 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 08 Jul 2025 07:43:39 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Fix crash when block.end()
	is missed (PR #147519)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147519@github.com>
Message-ID: <686d2e9b.170a0220.1ace94.39b5@mx.google.com>


https://github.com/kparzysz approved this pull request.


https://github.com/llvm/llvm-project/pull/147519

From flang-commits at lists.llvm.org  Tue Jul  8 07:47:37 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 07:47:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Do not invoke undefined behaviour
 when parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686d2f89.630a0220.6987c.3b8a@mx.google.com>


================
@@ -214,16 +215,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::MulOverflow(
+            static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
----------------
DavidSpickett wrote:

`overflow = overflow || MulOverflow() || AddOverflow()` would do the latching properly but is more work to comprehend.

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Tue Jul  8 07:48:17 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 07:48:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686d2fb1.050a0220.2b0a80.4b0e@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Tue Jul  8 07:49:53 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Tue, 08 Jul 2025 07:49:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686d3011.170a0220.12a3c7.3846@mx.google.com>


https://github.com/DavidSpickett edited https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Tue Jul  8 08:16:35 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 08 Jul 2025 08:16:35 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][OpenMP] Enable tiling
	(PR #143715)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143715@github.com>
Message-ID: <686d3653.170a0220.222bfa.46e3@mx.google.com>


================
@@ -38,14 +38,21 @@ namespace lower {
 namespace omp {
 
 int64_t getCollapseValue(const List<Clause> &clauses) {
-  auto iter = llvm::find_if(clauses, [](const Clause &clause) {
-    return clause.id == llvm::omp::Clause::OMPC_collapse;
-  });
-  if (iter != clauses.end()) {
-    const auto &collapse = std::get<clause::Collapse>(iter->u);
-    return evaluate::ToInt64(collapse.v).value();
+  int64_t collapseValue = 1;
+  int64_t numTileSizes = 0;
+  for (auto &clause : clauses) {
+    if (clause.id == llvm::omp::Clause::OMPC_collapse) {
+      const auto &collapse = std::get<clause::Collapse>(clause.u);
+      collapseValue = evaluate::ToInt64(collapse.v).value();
+    } else if (clause.id == llvm::omp::Clause::OMPC_sizes) {
+      const auto &sizes = std::get<clause::Sizes>(clause.u);
+      numTileSizes = sizes.v.size();
+    }
   }
-  return 1;
+
+  collapseValue = collapseValue - numTileSizes;
+  int64_t result = collapseValue > numTileSizes ? collapseValue : numTileSizes;
+  return result;
----------------
Meinersbur wrote:

Support for collapsing grid- and tile-loops have only been added in OpenMP 6.0. Before that, there was a restriction:
![image](https://github.com/user-attachments/assets/75a23fac-3cbb-4004-87f4-ef0c20a49310)

The value of the collapse clause should not really be modified implicitly. Is there is check in FortranSemantics that ensures collapse is not larger than the number of grid loops?

https://github.com/llvm/llvm-project/pull/143715

From flang-commits at lists.llvm.org  Tue Jul  8 08:16:35 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 08 Jul 2025 08:16:35 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][OpenMP] Enable tiling
	(PR #143715)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143715@github.com>
Message-ID: <686d3653.630a0220.1077fa.ceea@mx.google.com>


================
@@ -659,7 +756,6 @@ bool collectLoopRelatedInfo(
   } while (collapseValue > 0);
 
   convertLoopBounds(converter, currentLocation, result, loopVarTypeSize);
-
----------------
Meinersbur wrote:

[nit] unrelated change

https://github.com/llvm/llvm-project/pull/143715

From flang-commits at lists.llvm.org  Tue Jul  8 08:16:36 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 08 Jul 2025 08:16:36 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][OpenMP] Enable tiling
	(PR #143715)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143715@github.com>
Message-ID: <686d3654.170a0220.28af06.590b@mx.google.com>


================
@@ -38,14 +59,21 @@ namespace lower {
 namespace omp {
 
 int64_t getCollapseValue(const List<Clause> &clauses) {
-  auto iter = llvm::find_if(clauses, [](const Clause &clause) {
-    return clause.id == llvm::omp::Clause::OMPC_collapse;
-  });
-  if (iter != clauses.end()) {
-    const auto &collapse = std::get<clause::Collapse>(iter->u);
-    return evaluate::ToInt64(collapse.v).value();
+  int64_t collapseValue = 1;
+  int64_t numTileSizes = 0;
+  for (auto &clause : clauses) {
+    if (clause.id == llvm::omp::Clause::OMPC_collapse) {
+      const auto &collapse = std::get<clause::Collapse>(clause.u);
+      collapseValue = evaluate::ToInt64(collapse.v).value();
+    } else if (clause.id == llvm::omp::Clause::OMPC_sizes) {
+      const auto &sizes = std::get<clause::Sizes>(clause.u);
+      numTileSizes = sizes.v.size();
+    }
----------------
Meinersbur wrote:

This mixes the directive that collapse goes on (DO, DISTRIBUTE, ..) and those sizes goes on (TILE). During resolving merge conflicts of #145917 and #144785 the approach of handling tile like a compound construct already caused problems. E.g. it could be `tile` twice with different `sizes`, or `unroll partial` followed by `tile` vs the other way around, and there is no way to disambiguate them.

So it will need to be tangled apart eventually, and I would appreciate if we could avoid the assumption that there is just one `tile` which is always first.

https://github.com/llvm/llvm-project/pull/143715

From flang-commits at lists.llvm.org  Tue Jul  8 08:16:36 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 08 Jul 2025 08:16:36 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang][OpenMP] Enable tiling
	(PR #143715)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143715@github.com>
Message-ID: <686d3654.170a0220.c3424.b785@mx.google.com>


================
@@ -2992,18 +2991,59 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
     loopInfos.push_back(*loopResult);
   }
 
-  // Collapse loops. Store the insertion point because LoopInfos may get
-  // invalidated.
   llvm::OpenMPIRBuilder::InsertPointTy afterIP =
       loopInfos.front()->getAfterIP();
 
-  // Update the stack frame created for this loop to point to the resulting loop
-  // after applying transformations.
-  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
-      [&](OpenMPLoopInfoStackFrame &frame) {
-        frame.loopInfo = ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
-        return WalkResult::interrupt();
-      });
+  // Initialize the new loop info to the current one, in case there
+  // are no loop transformations done.
+  llvm::CanonicalLoopInfo *NewTopLoopInfo = nullptr;
+
+  // Do tiling
+  if (const auto &tiles = loopOp.getTileSizes()) {
+    llvm::Type *IVType = loopInfos.front()->getIndVarType();
+    SmallVector<llvm::Value *> TileSizes;
+
+    for (auto tile : tiles.value()) {
+      llvm::Value *TileVal = llvm::ConstantInt::get(IVType, tile);
+      TileSizes.push_back(TileVal);
+    }
+
+    std::vector<llvm::CanonicalLoopInfo *> NewLoops =
+        ompBuilder->tileLoops(ompLoc.DL, loopInfos, TileSizes);
+
+    // Update afterIP to get the correct insertion point after
+    // tiling.
+    llvm::BasicBlock *AfterBB = NewLoops.front()->getAfter();
+    llvm::BasicBlock *AfterAfterBB = AfterBB->getSingleSuccessor();
+    afterIP = {AfterAfterBB, AfterAfterBB->begin()};
+    NewTopLoopInfo = NewLoops[0];
+
+    // Update the loop infos
+    loopInfos.clear();
+    for (const auto &newLoop : NewLoops) {
+      loopInfos.push_back(newLoop);
+    }
+  } // Tiling done
+
+  // Do collapse
+  if (const auto &numCollapse = loopOp.getNumCollapse()) {
+    SmallVector<llvm::CanonicalLoopInfo *> collapseLoopInfos(
+        loopInfos.begin(), loopInfos.begin() + (numCollapse));
+
+    auto newLoopInfo =
+        ompBuilder->collapseLoops(ompLoc.DL, collapseLoopInfos, {});
----------------
Meinersbur wrote:

Previously this was called withing the OpenMPLoopInfoStackFrame callback. Does it make a difference?

(Doing it outside of it seems more logical to me, but I assume there was a reason)

https://github.com/llvm/llvm-project/pull/143715

From flang-commits at lists.llvm.org  Tue Jul  8 08:46:05 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 08 Jul 2025 08:46:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686d3d3d.170a0220.33802.6005@mx.google.com>


================
@@ -170,7 +170,7 @@ class CallInterface {
     /// Is the argument INTENT(OUT)
     bool isIntentOut() const;
     /// Does the argument have the CONTIGUOUS attribute or have explicit shape?
-    bool mustBeMadeContiguous() const;
+    bool mustBeMadeContiguous(const bool argHasTriplet = false) const;
----------------
klausler wrote:

`const` is never needed in a prototype argument that isn't a reference or a pointer

https://github.com/llvm/llvm-project/pull/147419

From flang-commits at lists.llvm.org  Tue Jul  8 08:46:06 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 08 Jul 2025 08:46:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686d3d3e.630a0220.118ec0.313d@mx.google.com>


================
@@ -1173,6 +1173,22 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
   return HasVectorSubscriptHelper{}(expr);
 }
 
+// HasTriplet()
----------------
klausler wrote:

Simply checking for triplets is going to give false positives.  `A(:)` is contiguous when `A` is; so are `A(J:K)` and `A(J:K:1)`.  There's a general contiguity checking facility in Evaluate that can handle questions of contiguity; have you tried using it?

https://github.com/llvm/llvm-project/pull/147419

From flang-commits at lists.llvm.org  Tue Jul  8 08:55:57 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 08 Jul 2025 08:55:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686d3f8d.170a0220.27246d.c32f@mx.google.com>


================
@@ -1173,6 +1173,22 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
   return HasVectorSubscriptHelper{}(expr);
 }
 
+// HasTriplet()
----------------
eugeneepshteyn wrote:

If `mustBeMadeContiguous()` returns true, then this eventually gets to `isSimplyContiguous()` which is a helper function that calls `Fortran::evaluate::IsSimplyContiguous()`. 

https://github.com/llvm/llvm-project/pull/147419

From flang-commits at lists.llvm.org  Tue Jul  8 09:03:19 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 08 Jul 2025 09:03:19 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686d4147.170a0220.21392b.cec2@mx.google.com>


================
@@ -1173,6 +1173,22 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
   return HasVectorSubscriptHelper{}(expr);
 }
 
+// HasTriplet()
----------------
klausler wrote:

And if `mustBeMadeContiguous` returns false, then `IsSimplyContiguous` is not called?  Won't that lead to false negatives from e.g. `A%X`, non-CONTIGUOUS pointers, &c.?

https://github.com/llvm/llvm-project/pull/147419

From flang-commits at lists.llvm.org  Tue Jul  8 10:47:30 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 10:47:30 -0700 (PDT)
Subject: [flang-commits] [flang] e976eaf - [flang] Fix optimization of array
 assignments after #146408 (#147371)
Message-ID: <686d59b2.050a0220.3c172d.91a6@mx.google.com>


Author: Leandro Lupori
Date: 2025-07-08T14:47:26-03:00
New Revision: e976eaf3037a1c0fe68432ae60a503bf144a1c30

URL: https://github.com/llvm/llvm-project/commit/e976eaf3037a1c0fe68432ae60a503bf144a1c30
DIFF: https://github.com/llvm/llvm-project/commit/e976eaf3037a1c0fe68432ae60a503bf144a1c30.diff

LOG: [flang] Fix optimization of array assignments after #146408 (#147371)

Host associated variables were not being handled properly.
For array references, get the fixed shape extents from the value
type instead, that works correctly in all cases.

Added: 
    

Modified: 
    flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
    flang/test/HLFIR/opt-scalar-assign.fir

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 54892ef99bf58..434973f5887e8 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -805,13 +805,12 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
+      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
+      auto seqTy = mlir::cast<fir::SequenceType>(refTy.getElementType());
+      llvm::ArrayRef<int64_t> fixedShape = seqTy.getShape();
       int64_t flatExtent = 1;
-      for (const mlir::Value &extent : extents) {
-        mlir::Operation *op = extent.getDefiningOp();
-        assert(op && "no defining operation for constant array extent");
-        flatExtent *= fir::toInt(mlir::cast<mlir::arith::ConstantOp>(*op));
-      }
-
+      for (int64_t extent : fixedShape)
+        flatExtent *= extent;
       flatArrayType =
           fir::ReferenceType::get(fir::SequenceType::get({flatExtent}, eleTy));
       flatArray = builder.createConvert(loc, flatArrayType, flatArray);

diff  --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 74cdcd9622adb..468a5dbf988d7 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -155,3 +155,32 @@ func.func @_QPtest6(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {f
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @_QQmain() {
+  return
+}
+
+func.func private @_QFPtest7(%arg0: !fir.ref<tuple<!fir.box<!fir.array<2x2xi32>>>> {fir.host_assoc}) attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>} {
+  %0 = fir.dummy_scope : !fir.dscope
+  %c0_i32 = arith.constant 0 : i32
+  %1 = fir.coordinate_of %arg0, %c0_i32 : (!fir.ref<tuple<!fir.box<!fir.array<2x2xi32>>>>, i32) -> !fir.ref<!fir.box<!fir.array<2x2xi32>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.array<2x2xi32>>>
+  %3 = fir.box_addr %2 : (!fir.box<!fir.array<2x2xi32>>) -> !fir.ref<!fir.array<2x2xi32>>
+  %c0 = arith.constant 0 : index
+  %4:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.array<2x2xi32>>, index) -> (index, index, index)
+  %c1 = arith.constant 1 : index
+  %5:3 = fir.box_dims %2, %c1 : (!fir.box<!fir.array<2x2xi32>>, index) -> (index, index, index)
+  %6 = fir.shape %4#1, %5#1 : (index, index) -> !fir.shape<2>
+  %7:2 = hlfir.declare %3(%6) {fortran_attrs = #fir.var_attrs<host_assoc>, uniq_name = "_QFEa"} : (!fir.ref<!fir.array<2x2xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<2x2xi32>>, !fir.ref<!fir.array<2x2xi32>>)
+  %c0_i32_0 = arith.constant 0 : i32
+  hlfir.assign %c0_i32_0 to %7#0 : i32, !fir.ref<!fir.array<2x2xi32>>
+  return
+}
+
+// CHECK-LABEL:   func.func private @_QFPtest7({{.*}}) {{.*}} {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : i32
+// CHECK:           fir.do_loop %[[VAL_1:.*]] = %{{.*}} to %{{.*}} step %{{.*}} unordered {
+// CHECK:             %[[VAL_2:.*]] = hlfir.designate %{{.*}} (%[[VAL_1]]) : (!fir.ref<!fir.array<4xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_0]] to %[[VAL_2]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:         }


        

From flang-commits at lists.llvm.org  Tue Jul  8 10:52:19 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 10:52:19 -0700 (PDT)
Subject: [flang-commits] [flang] 46caad5 - [flang][cuda] Do not produce data
 transfer in offloaded do concurrent (#147435)
Message-ID: <686d5ad3.170a0220.2c21a0.e44a@mx.google.com>


Author: Valentin Clement (バレンタイン クレメン)
Date: 2025-07-08T10:52:15-07:00
New Revision: 46caad52ac14cefd6f9cf3188863818e330f3844

URL: https://github.com/llvm/llvm-project/commit/46caad52ac14cefd6f9cf3188863818e330f3844
DIFF: https://github.com/llvm/llvm-project/commit/46caad52ac14cefd6f9cf3188863818e330f3844.diff

LOG: [flang][cuda] Do not produce data transfer in offloaded do concurrent (#147435)

If a `do concurrent` loop is offloaded then there should be no CUDA data
transfer in it. Update the semantic and lowering to take that into
account.

`AssignmentChecker` has to be put into a separate pass because the
checkers in `SemanticsVisitor` cannot have the same `Enter/Leave`
functions. The `DoForallChecker` already has `Eneter/Leave` functions
for the `DoConstruct`.

Added: 
    

Modified: 
    flang/include/flang/Optimizer/Builder/CUFCommon.h
    flang/include/flang/Support/Fortran-features.h
    flang/lib/Lower/Bridge.cpp
    flang/lib/Optimizer/Builder/CUFCommon.cpp
    flang/lib/Semantics/assignment.cpp
    flang/lib/Semantics/assignment.h
    flang/lib/Semantics/check-cuda.cpp
    flang/lib/Semantics/check-cuda.h
    flang/test/Lower/CUDA/cuda-data-transfer.cuf
    flang/tools/bbc/bbc.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Builder/CUFCommon.h b/flang/include/flang/Optimizer/Builder/CUFCommon.h
index 65b9cce1d2021..5c56dd6b695f8 100644
--- a/flang/include/flang/Optimizer/Builder/CUFCommon.h
+++ b/flang/include/flang/Optimizer/Builder/CUFCommon.h
@@ -27,7 +27,8 @@ mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
                                             mlir::SymbolTable &symTab);
 
 bool isCUDADeviceContext(mlir::Operation *op);
-bool isCUDADeviceContext(mlir::Region &);
+bool isCUDADeviceContext(mlir::Region &,
+                         bool isDoConcurrentOffloadEnabled = false);
 bool isRegisteredDeviceGlobal(fir::GlobalOp op);
 bool isRegisteredDeviceAttr(std::optional<cuf::DataAttribute> attr);
 

diff  --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 8a2b7b29a5233..857de9479e4e5 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,

diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..696473605a4e0 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4886,7 +4886,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::Location loc = getCurrentLocation();
     fir::FirOpBuilder &builder = getFirOpBuilder();
 
-    bool isInDeviceContext = cuf::isCUDADeviceContext(builder.getRegion());
+    bool isInDeviceContext = cuf::isCUDADeviceContext(
+        builder.getRegion(),
+        getFoldingContext().languageFeatures().IsEnabled(
+            Fortran::common::LanguageFeature::DoConcurrentOffload));
 
     bool isCUDATransfer =
         IsCUDADataTransfer(assign.lhs, assign.rhs) && !isInDeviceContext;

diff  --git a/flang/lib/Optimizer/Builder/CUFCommon.cpp b/flang/lib/Optimizer/Builder/CUFCommon.cpp
index 5f286c04a7ca0..dcbf4991907bf 100644
--- a/flang/lib/Optimizer/Builder/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Builder/CUFCommon.cpp
@@ -43,7 +43,8 @@ bool cuf::isCUDADeviceContext(mlir::Operation *op) {
 // for it.
 // If the insertion point is inside an OpenACC region op, it is considered
 // device context.
-bool cuf::isCUDADeviceContext(mlir::Region &region) {
+bool cuf::isCUDADeviceContext(mlir::Region &region,
+                              bool isDoConcurrentOffloadEnabled) {
   if (region.getParentOfType<cuf::KernelOp>())
     return true;
   if (region.getParentOfType<mlir::acc::ComputeRegionOpInterface>())
@@ -56,6 +57,9 @@ bool cuf::isCUDADeviceContext(mlir::Region &region) {
              cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
     }
   }
+  if (isDoConcurrentOffloadEnabled &&
+      region.getParentOfType<fir::DoConcurrentLoopOp>())
+    return true;
   return false;
 }
 

diff  --git a/flang/lib/Semantics/assignment.cpp b/flang/lib/Semantics/assignment.cpp
index 43e23a9d8f60b..88e08887160d9 100644
--- a/flang/lib/Semantics/assignment.cpp
+++ b/flang/lib/Semantics/assignment.cpp
@@ -42,7 +42,6 @@ class AssignmentContext {
   void Analyze(const parser::AssignmentStmt &);
   void Analyze(const parser::PointerAssignmentStmt &);
   void Analyze(const parser::ConcurrentControl &);
-  int deviceConstructDepth_{0};
   SemanticsContext &context() { return context_; }
 
 private:
@@ -97,21 +96,6 @@ void AssignmentContext::Analyze(const parser::AssignmentStmt &stmt) {
     if (whereDepth_ > 0) {
       CheckShape(lhsLoc, &lhs);
     }
-    if (context_.foldingContext().languageFeatures().IsEnabled(
-            common::LanguageFeature::CUDA)) {
-      const auto &scope{context_.FindScope(lhsLoc)};
-      const Scope &progUnit{GetProgramUnitContaining(scope)};
-      if (!IsCUDADeviceContext(&progUnit) && deviceConstructDepth_ == 0) {
-        if (Fortran::evaluate::HasCUDADeviceAttrs(lhs) &&
-            Fortran::evaluate::HasCUDAImplicitTransfer(rhs)) {
-          if (GetNbOfCUDAManagedOrUnifiedSymbols(lhs) == 1 &&
-              GetNbOfCUDAManagedOrUnifiedSymbols(rhs) == 1 &&
-              GetNbOfCUDADeviceSymbols(rhs) == 1)
-            return; // This is a special case handled on the host.
-          context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
-        }
-      }
-    }
   }
 }
 
@@ -254,46 +238,6 @@ void AssignmentChecker::Enter(const parser::MaskedElsewhereStmt &x) {
 void AssignmentChecker::Leave(const parser::MaskedElsewhereStmt &) {
   context_.value().PopWhereContext();
 }
-void AssignmentChecker::Enter(const parser::CUFKernelDoConstruct &x) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::CUFKernelDoConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
-static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
-  const auto &beginBlockDirective =
-      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
-  const auto &blockDirective =
-      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
-  if (blockDirective.v == llvm::acc::ACCD_parallel ||
-      blockDirective.v == llvm::acc::ACCD_serial ||
-      blockDirective.v == llvm::acc::ACCD_kernels) {
-    return true;
-  }
-  return false;
-}
-void AssignmentChecker::Enter(const parser::OpenACCBlockConstruct &x) {
-  if (IsOpenACCComputeConstruct(x)) {
-    ++context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Leave(const parser::OpenACCBlockConstruct &x) {
-  if (IsOpenACCComputeConstruct(x)) {
-    --context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Enter(const parser::OpenACCCombinedConstruct &) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::OpenACCCombinedConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Enter(const parser::OpenACCLoopConstruct &) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::OpenACCLoopConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
 
 } // namespace Fortran::semantics
 template class Fortran::common::Indirection<

diff  --git a/flang/lib/Semantics/assignment.h b/flang/lib/Semantics/assignment.h
index 4a1bb92037119..ba537744bfaaa 100644
--- a/flang/lib/Semantics/assignment.h
+++ b/flang/lib/Semantics/assignment.h
@@ -46,14 +46,6 @@ class AssignmentChecker : public virtual BaseChecker {
   void Leave(const parser::EndWhereStmt &);
   void Enter(const parser::MaskedElsewhereStmt &);
   void Leave(const parser::MaskedElsewhereStmt &);
-  void Enter(const parser::CUFKernelDoConstruct &);
-  void Leave(const parser::CUFKernelDoConstruct &);
-  void Enter(const parser::OpenACCBlockConstruct &);
-  void Leave(const parser::OpenACCBlockConstruct &);
-  void Enter(const parser::OpenACCCombinedConstruct &);
-  void Leave(const parser::OpenACCCombinedConstruct &);
-  void Enter(const parser::OpenACCLoopConstruct &);
-  void Leave(const parser::OpenACCLoopConstruct &);
 
   SemanticsContext &context();
 

diff  --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index 8decfb0149829..b01147606a99c 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -685,18 +685,67 @@ void CUDAChecker::Enter(const parser::CUFKernelDoConstruct &x) {
       std::get<std::list<parser::CUFReduction>>(directive.t)) {
     CheckReduce(context_, reduce);
   }
-  inCUFKernelDoConstruct_ = true;
+  ++deviceConstructDepth_;
+}
+
+static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
+  const auto &beginBlockDirective =
+      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
+  const auto &blockDirective =
+      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
+  if (blockDirective.v == llvm::acc::ACCD_parallel ||
+      blockDirective.v == llvm::acc::ACCD_serial ||
+      blockDirective.v == llvm::acc::ACCD_kernels) {
+    return true;
+  }
+  return false;
 }
 
 void CUDAChecker::Leave(const parser::CUFKernelDoConstruct &) {
-  inCUFKernelDoConstruct_ = false;
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    ++deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Leave(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    --deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Enter(const parser::OpenACCCombinedConstruct &) {
+  ++deviceConstructDepth_;
+}
+void CUDAChecker::Leave(const parser::OpenACCCombinedConstruct &) {
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::OpenACCLoopConstruct &) {
+  ++deviceConstructDepth_;
+}
+void CUDAChecker::Leave(const parser::OpenACCLoopConstruct &) {
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context_.foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::DoConcurrentOffload)) {
+    ++deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Leave(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context_.foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::DoConcurrentOffload)) {
+    --deviceConstructDepth_;
+  }
 }
 
 void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
   auto lhsLoc{std::get<parser::Variable>(x.t).GetSource()};
   const auto &scope{context_.FindScope(lhsLoc)};
   const Scope &progUnit{GetProgramUnitContaining(scope)};
-  if (IsCUDADeviceContext(&progUnit) || inCUFKernelDoConstruct_) {
+  if (IsCUDADeviceContext(&progUnit) || deviceConstructDepth_ > 0) {
     return; // Data transfer with assignment is only perform on host.
   }
 
@@ -714,6 +763,16 @@ void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
     context_.Say(lhsLoc,
         "More than one reference to a CUDA object on the right hand side of the assigment"_err_en_US);
   }
+
+  if (Fortran::evaluate::HasCUDADeviceAttrs(assign->lhs) &&
+      Fortran::evaluate::HasCUDAImplicitTransfer(assign->rhs)) {
+    if (GetNbOfCUDAManagedOrUnifiedSymbols(assign->lhs) == 1 &&
+        GetNbOfCUDAManagedOrUnifiedSymbols(assign->rhs) == 1 &&
+        GetNbOfCUDADeviceSymbols(assign->rhs) == 1) {
+      return; // This is a special case handled on the host.
+    }
+    context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
+  }
 }
 
 } // namespace Fortran::semantics

diff  --git a/flang/lib/Semantics/check-cuda.h b/flang/lib/Semantics/check-cuda.h
index 222a2ee04b57c..10000253ffe5a 100644
--- a/flang/lib/Semantics/check-cuda.h
+++ b/flang/lib/Semantics/check-cuda.h
@@ -41,10 +41,18 @@ class CUDAChecker : public virtual BaseChecker {
   void Enter(const parser::CUFKernelDoConstruct &);
   void Leave(const parser::CUFKernelDoConstruct &);
   void Enter(const parser::AssignmentStmt &);
+  void Enter(const parser::OpenACCBlockConstruct &);
+  void Leave(const parser::OpenACCBlockConstruct &);
+  void Enter(const parser::OpenACCCombinedConstruct &);
+  void Leave(const parser::OpenACCCombinedConstruct &);
+  void Enter(const parser::OpenACCLoopConstruct &);
+  void Leave(const parser::OpenACCLoopConstruct &);
+  void Enter(const parser::DoConstruct &);
+  void Leave(const parser::DoConstruct &);
 
 private:
   SemanticsContext &context_;
-  bool inCUFKernelDoConstruct_ = false;
+  int deviceConstructDepth_{0};
 };
 
 bool CanonicalizeCUDA(parser::Program &);

diff  --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index d0032af415604..68a0202f951fe 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -403,3 +403,19 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPsub20()
 ! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
+
+subroutine sub21()
+  real, allocatable,device:: a(:,:), b(:,:)
+  real:: s
+  integer:: i,j,N=16
+  allocate(a(N,N),b(N,N))
+  do concurrent(i=1:N, j=1:N) reduce(+:s)
+    b(i,j)=a(i,j)**2
+    s=s+b(i,j)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub21()
+! CHECK: fir.do_concurrent.loop
+! CHECK-NOT: cuf.data_transfer
+! CHECK: hlfir.assign

diff  --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 59372a8eb58ed..d216069654bc4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -223,6 +223,11 @@ static llvm::cl::opt<bool> enableCUDA("fcuda",
                                       llvm::cl::desc("enable CUDA Fortran"),
                                       llvm::cl::init(false));
 
+static llvm::cl::opt<bool>
+    enableDoConcurrentOffload("fdoconcurrent-offload",
+                              llvm::cl::desc("enable do concurrent offload"),
+                              llvm::cl::init(false));
+
 static llvm::cl::opt<bool>
     disableCUDAWarpFunction("fcuda-disable-warp-function",
                             llvm::cl::desc("Disable CUDA Warp Function"),
@@ -608,6 +613,11 @@ int main(int argc, char **argv) {
     options.features.Enable(Fortran::common::LanguageFeature::CUDA);
   }
 
+  if (enableDoConcurrentOffload) {
+    options.features.Enable(
+        Fortran::common::LanguageFeature::DoConcurrentOffload);
+  }
+
   if (disableCUDAWarpFunction) {
     options.features.Enable(
         Fortran::common::LanguageFeature::CudaWarpMatchFunction, false);


        

From flang-commits at lists.llvm.org  Tue Jul  8 10:52:24 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Tue, 08 Jul 2025 10:52:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Do not produce data transfer
 in offloaded do concurrent (PR #147435)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147435@github.com>
Message-ID: <686d5ad8.170a0220.35bfea.7bda@mx.google.com>


https://github.com/clementval closed https://github.com/llvm/llvm-project/pull/147435

From flang-commits at lists.llvm.org  Tue Jul  8 11:56:59 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 08 Jul 2025 11:56:59 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686d69fb.170a0220.e1701.805f@mx.google.com>


================
@@ -170,7 +170,7 @@ class CallInterface {
     /// Is the argument INTENT(OUT)
     bool isIntentOut() const;
     /// Does the argument have the CONTIGUOUS attribute or have explicit shape?
-    bool mustBeMadeContiguous() const;
+    bool mustBeMadeContiguous(const bool argHasTriplet = false) const;
----------------
eugeneepshteyn wrote:

Why "never"? Maybe I just want to make it obvious that the arg shouldn't be modified:
```
void f(const bool x = false)
{
  x = true;
}

$ clang++ -c const.cpp
const.cpp:3:5: error: cannot assign to variable 'x' with const-qualified type 'const bool'
    3 |   x = true;
      |   ~ ^
const.cpp:1:19: note: variable 'x' declared const here
    1 | void f(const bool x = false)
      |        ~~~~~~~~~~~^~~~~~~~~
1 error generated.
```
In any case, this is not important to my change, so I can remove it.

https://github.com/llvm/llvm-project/pull/147419

From flang-commits at lists.llvm.org  Tue Jul  8 11:56:59 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 08 Jul 2025 11:56:59 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686d69fb.170a0220.63aab.ec4c@mx.google.com>


================
@@ -1173,6 +1173,22 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
   return HasVectorSubscriptHelper{}(expr);
 }
 
+// HasTriplet()
----------------
eugeneepshteyn wrote:

That would also be a problem with the current source, without my modifications. I'll check what happens in the cases you mentioned.

https://github.com/llvm/llvm-project/pull/147419

From flang-commits at lists.llvm.org  Tue Jul  8 10:46:57 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Tue, 08 Jul 2025 10:46:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686d5991.170a0220.15b711.7518@mx.google.com>


================
@@ -805,13 +805,12 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
+      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
----------------
luporl wrote:

Since it is already approved and blocking testing, I believe it's OK to merge it now.
As mentioned, I can make any necessary changes in a follow-up PR.

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Tue Jul  8 10:47:33 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Tue, 08 Jul 2025 10:47:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686d59b5.170a0220.3708d5.7550@mx.google.com>


https://github.com/luporl closed https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Tue Jul  8 12:00:44 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 08 Jul 2025 12:00:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686d6adc.a70a0220.37cb87.8edc@mx.google.com>


================
@@ -170,7 +170,7 @@ class CallInterface {
     /// Is the argument INTENT(OUT)
     bool isIntentOut() const;
     /// Does the argument have the CONTIGUOUS attribute or have explicit shape?
-    bool mustBeMadeContiguous() const;
+    bool mustBeMadeContiguous(const bool argHasTriplet = false) const;
----------------
klausler wrote:

The prototype doesn't need the `const` here as part of the API, and the implementation is allowed to use `const` later if desired.  So the `const` in the prototype makes me have to stop and wonder whether a `&` or `*` was intended but omitted.

https://github.com/llvm/llvm-project/pull/147419

From flang-commits at lists.llvm.org  Tue Jul  8 13:04:26 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Tue, 08 Jul 2025 13:04:26 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147442@github.com>
Message-ID: <686d79ca.170a0220.2a0f5e.0499@mx.google.com>


================
@@ -2444,7 +2473,15 @@ void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
       useLastDeclSymbol();
       PRINT_IMPLICIT_RULE("3) enclosing context");
     } else if (targetDir) {
-      // TODO 4) not mapped target variable -> firstprivate
+      // 4) not mapped target variable  -> firstprivate
+      //    - i.e. implicit, but meets OpenMP specification rules for
+      //    firstprivate "promotion"
+      if (enableDelayedPrivatizationStaging && symbol &&
----------------
luporl wrote:

`symbol` can't be NULL, as that is one of the conditions checked before calling `CreateImplicitSymbols`.
In the future, it would be good to change its `symbol` parameter to a reference.

https://github.com/llvm/llvm-project/pull/147442

From flang-commits at lists.llvm.org  Tue Jul  8 13:04:26 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Tue, 08 Jul 2025 13:04:26 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147442@github.com>
Message-ID: <686d79ca.630a0220.2175a4.8f47@mx.google.com>


================
@@ -2444,7 +2473,15 @@ void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
       useLastDeclSymbol();
       PRINT_IMPLICIT_RULE("3) enclosing context");
     } else if (targetDir) {
-      // TODO 4) not mapped target variable -> firstprivate
+      // 4) not mapped target variable  -> firstprivate
+      //    - i.e. implicit, but meets OpenMP specification rules for
+      //    firstprivate "promotion"
+      if (enableDelayedPrivatizationStaging && symbol &&
+          IsTargetCaptureImplicitlyFirstPrivatizeable(*symbol, prevDSA)) {
+        prevDSA.set(Symbol::Flag::OmpImplicit);
+        prevDSA.set(Symbol::Flag::OmpFirstPrivate);
+        makeSymbol(prevDSA);
+      }
----------------
luporl wrote:

Please add this line, to keep debug messages consistent:
`PRINT_IMPLICIT_RULE("4) not mapped target variable  -> firstprivate");`

https://github.com/llvm/llvm-project/pull/147442

From flang-commits at lists.llvm.org  Tue Jul  8 13:04:27 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Tue, 08 Jul 2025 13:04:27 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147442@github.com>
Message-ID: <686d79cb.630a0220.52af0.95f8@mx.google.com>


================
@@ -2297,6 +2298,34 @@ static bool IsSymbolStaticStorageDuration(const Symbol &symbol) {
       (ultSym.flags().test(Symbol::Flag::InCommonBlock));
 }
 
+static bool IsTargetCaptureImplicitlyFirstPrivatizeable(
+    const Symbol &symbol, const Symbol::Flags &dsa) {
+  // if we're associated with any other flags we skip implicit privitization
+  // for now. If we're an allocatable, pointer or declare target, we're not
+  // implicitly firstprivitizeable under OpenMP restrictions.
+  // TODO: Relax restriction as we progress privitization and further
+  // investigate the flags we can intermix with.
+  if (!dsa.none() || !symbol.flags().none() ||
----------------
luporl wrote:

It's possible that other flags not related to DSA may be set.
If you want to consider only DSA and data-mapping related flags, you can use:
`(dsa & (dataSharingAttributeFlags | dataMappingAttributeFlags)).none()`

Note that `dataMappingAttributeFlags` would need to be moved from OmpAttributeVisitor::ResolveOmpObject() to OmpAttributeVisitor.

https://github.com/llvm/llvm-project/pull/147442

From flang-commits at lists.llvm.org  Tue Jul  8 13:49:09 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 08 Jul 2025 13:49:09 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't create bogus tokens from
 token pasting (##) (PR #147596)
Message-ID: <llvm.org/llvm/llvm-project/pull/147596@github.com>


https://github.com/klausler created https://github.com/llvm/llvm-project/pull/147596

When blank tokens arise from macro replacement in token sequences with token pasting (##), the preprocessor is producing some bogus tokens (e.g., "name(") that can lead to subtle bugs later when macro names are not recognized as such.

The fix is to not paste tokens together when the result would not be a valid Fortran or C token in the preprocessing context.

>From ac650e4d3c87822f0607e91c77b153e84a8d328e Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 8 Jul 2025 13:43:50 -0700
Subject: [PATCH] [flang] Don't create bogus tokens from token pasting (##)

When blank tokens arise from macro replacement in token sequences
with token pasting (##), the preprocessor is producing some bogus
tokens (e.g., "name(") that can lead to subtle bugs later when
macro names are not recognized as such.

The fix is to not paste tokens together when the result would not
be a valid Fortran or C token in the preprocessing context.
---
 flang/lib/Parser/preprocessor.cpp    | 43 ++++++++++++++++++++++------
 flang/test/Preprocessing/bug1126.F90 | 20 +++++++++++++
 2 files changed, 55 insertions(+), 8 deletions(-)
 create mode 100644 flang/test/Preprocessing/bug1126.F90

diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index 8ef9810463d5a..9f711a9001057 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -156,23 +156,50 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
   }
   TokenSequence result;
   std::size_t tokens{text.SizeInTokens()};
-  bool pasting{false};
+  std::optional<CharBlock> before; // last non-blank token before ##
   for (std::size_t j{0}; j < tokens; ++j) {
-    if (IsTokenPasting(text.TokenAt(j))) {
-      if (!pasting) {
+    CharBlock after{text.TokenAt(j)};
+    if (!before) {
+      if (IsTokenPasting(after)) {
         while (!result.empty() &&
             result.TokenAt(result.SizeInTokens() - 1).IsBlank()) {
           result.pop_back();
         }
         if (!result.empty()) {
-          result.ReopenLastToken();
-          pasting = true;
+          before = result.TokenAt(result.SizeInTokens() - 1);
         }
+      } else {
+        result.AppendRange(text, j, 1);
+      }
+    } else if (after.IsBlank() || IsTokenPasting(after)) {
+      // drop it
+    } else { // pasting before ## after
+      bool doPaste{false};
+      char last{before->end()[-1]};
+      char first{after.begin()[0]};
+      // Apply basic sanity checking to pasting so avoid constructing a bogus
+      // token that might cause macro replacement to fail, like "macro(".
+      if (IsLegalInIdentifier(last) && IsLegalInIdentifier(first)) {
+        doPaste = true;
+      } else if (IsDecimalDigit(first) &&
+          (last == '.' || last == '+' || last == '-')) {
+        doPaste = true; // 1. ## 0, - ## 1
+      } else if (before->size() == 1 && after.size() == 1) {
+        if (first == last &&
+            (last == '<' || last == '>' || last == '*' || last == '/' ||
+                last == '=' || last == '&' || last == '|' || last == ':')) {
+          // Fortran **, //, ==, ::
+          // C <<, >>, &&, || for use in #if expressions
+          doPaste = true;
+        } else if (first == '=' && (last == '!' || last == '/')) {
+          doPaste = true; // != and /=
+        }
+      }
+      if (doPaste) {
+        result.ReopenLastToken();
       }
-    } else if (pasting && text.TokenAt(j).IsBlank()) {
-    } else {
       result.AppendRange(text, j, 1);
-      pasting = false;
+      before.reset();
     }
   }
   return result;
diff --git a/flang/test/Preprocessing/bug1126.F90 b/flang/test/Preprocessing/bug1126.F90
new file mode 100644
index 0000000000000..ae5bb5633581d
--- /dev/null
+++ b/flang/test/Preprocessing/bug1126.F90
@@ -0,0 +1,20 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+#define PREFIX(x) prefix ## x
+#define NAME(x) PREFIX(foo ## x)
+#define AUGMENT(x) NAME(x ## suffix)
+
+! CHECK: subroutine prefixfoosuffix()
+! CHECK: print *, "prefixfoosuffix"
+! CHECK: end subroutine prefixfoosuffix
+subroutine AUGMENT()()
+  print *, TOSTRING(AUGMENT())
+end subroutine AUGMENT()
+
+! CHECK: subroutine prefixfoobarsuffix()
+! CHECK: print *, "prefixfoobarsuffix"
+! CHECK: end subroutine prefixfoobarsuffix
+subroutine AUGMENT(bar)()
+  print *, TOSTRING(AUGMENT(bar))
+end subroutine AUGMENT(bar)


From flang-commits at lists.llvm.org  Tue Jul  8 13:49:44 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 13:49:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't create bogus tokens from
 token pasting (##) (PR #147596)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147596@github.com>
Message-ID: <686d8468.050a0220.1fba11.aeed@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: Peter Klausler (klausler)

<details>
<summary>Changes</summary>

When blank tokens arise from macro replacement in token sequences with token pasting (##), the preprocessor is producing some bogus tokens (e.g., "name(") that can lead to subtle bugs later when macro names are not recognized as such.

The fix is to not paste tokens together when the result would not be a valid Fortran or C token in the preprocessing context.

---
Full diff: https://github.com/llvm/llvm-project/pull/147596.diff


2 Files Affected:

- (modified) flang/lib/Parser/preprocessor.cpp (+35-8) 
- (added) flang/test/Preprocessing/bug1126.F90 (+20) 


``````````diff
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index 8ef9810463d5a..9f711a9001057 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -156,23 +156,50 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
   }
   TokenSequence result;
   std::size_t tokens{text.SizeInTokens()};
-  bool pasting{false};
+  std::optional<CharBlock> before; // last non-blank token before ##
   for (std::size_t j{0}; j < tokens; ++j) {
-    if (IsTokenPasting(text.TokenAt(j))) {
-      if (!pasting) {
+    CharBlock after{text.TokenAt(j)};
+    if (!before) {
+      if (IsTokenPasting(after)) {
         while (!result.empty() &&
             result.TokenAt(result.SizeInTokens() - 1).IsBlank()) {
           result.pop_back();
         }
         if (!result.empty()) {
-          result.ReopenLastToken();
-          pasting = true;
+          before = result.TokenAt(result.SizeInTokens() - 1);
         }
+      } else {
+        result.AppendRange(text, j, 1);
+      }
+    } else if (after.IsBlank() || IsTokenPasting(after)) {
+      // drop it
+    } else { // pasting before ## after
+      bool doPaste{false};
+      char last{before->end()[-1]};
+      char first{after.begin()[0]};
+      // Apply basic sanity checking to pasting so avoid constructing a bogus
+      // token that might cause macro replacement to fail, like "macro(".
+      if (IsLegalInIdentifier(last) && IsLegalInIdentifier(first)) {
+        doPaste = true;
+      } else if (IsDecimalDigit(first) &&
+          (last == '.' || last == '+' || last == '-')) {
+        doPaste = true; // 1. ## 0, - ## 1
+      } else if (before->size() == 1 && after.size() == 1) {
+        if (first == last &&
+            (last == '<' || last == '>' || last == '*' || last == '/' ||
+                last == '=' || last == '&' || last == '|' || last == ':')) {
+          // Fortran **, //, ==, ::
+          // C <<, >>, &&, || for use in #if expressions
+          doPaste = true;
+        } else if (first == '=' && (last == '!' || last == '/')) {
+          doPaste = true; // != and /=
+        }
+      }
+      if (doPaste) {
+        result.ReopenLastToken();
       }
-    } else if (pasting && text.TokenAt(j).IsBlank()) {
-    } else {
       result.AppendRange(text, j, 1);
-      pasting = false;
+      before.reset();
     }
   }
   return result;
diff --git a/flang/test/Preprocessing/bug1126.F90 b/flang/test/Preprocessing/bug1126.F90
new file mode 100644
index 0000000000000..ae5bb5633581d
--- /dev/null
+++ b/flang/test/Preprocessing/bug1126.F90
@@ -0,0 +1,20 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+#define PREFIX(x) prefix ## x
+#define NAME(x) PREFIX(foo ## x)
+#define AUGMENT(x) NAME(x ## suffix)
+
+! CHECK: subroutine prefixfoosuffix()
+! CHECK: print *, "prefixfoosuffix"
+! CHECK: end subroutine prefixfoosuffix
+subroutine AUGMENT()()
+  print *, TOSTRING(AUGMENT())
+end subroutine AUGMENT()
+
+! CHECK: subroutine prefixfoobarsuffix()
+! CHECK: print *, "prefixfoobarsuffix"
+! CHECK: end subroutine prefixfoobarsuffix
+subroutine AUGMENT(bar)()
+  print *, TOSTRING(AUGMENT(bar))
+end subroutine AUGMENT(bar)

``````````

</details>


https://github.com/llvm/llvm-project/pull/147596

From flang-commits at lists.llvm.org  Tue Jul  8 14:56:13 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 14:56:13 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP] Make implicitly captured
 scalars fully firstprivatized (PR #147442)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147442@github.com>
Message-ID: <686d93fd.170a0220.15c9a7.875e@mx.google.com>


agozillon wrote:

Thank you very much for the quick review @luporl ! Ill update this PR in the next week or so with the review points.

https://github.com/llvm/llvm-project/pull/147442

From flang-commits at lists.llvm.org  Tue Jul  8 14:57:39 2025
From: flang-commits at lists.llvm.org (Jacques Pienaar via flang-commits)
Date: Tue, 08 Jul 2025 14:57:39 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][tblgen] add concrete create
	methods (PR #147168)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147168@github.com>
Message-ID: <686d9453.170a0220.37348f.8c23@mx.google.com>


jpienaar wrote:

Oh and I'd be pro merging ImplicitLocOpBuilder in to Builders.h - it is really privileged now. I mean one could just have ImplicitLocOpBuilder.h keep including Builders.h, then you also keep folks happy who get their include indirectly. And then add pragma, soak, delete. The update for others before deletion is then sed `s/ImplicitLocOpBuilder.h/Builder.s/` .

https://github.com/llvm/llvm-project/pull/147168

From flang-commits at lists.llvm.org  Tue Jul  8 16:49:33 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Tue, 08 Jul 2025 16:49:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] add -Wfatal-errors (PR
	#147614)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147614@github.com>
Message-ID: <686dae8d.170a0220.2e4a9d.841a@mx.google.com>


================
@@ -452,22 +452,31 @@ void Messages::ResolveProvenances(const AllCookedSources &allCooked) {
 }
 
 void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLines,
-    const common::LanguageFeatureControl *hintFlagPtr) const {
+    bool echoSourceLines, const common::LanguageFeatureControl *hintFlagPtr,
+    size_t maxErrorsToEmit) const {
   std::vector<const Message *> sorted;
   for (const auto &msg : messages_) {
     sorted.push_back(&msg);
   }
   std::stable_sort(sorted.begin(), sorted.end(),
       [](const Message *x, const Message *y) { return x->SortBefore(*y); });
   const Message *lastMsg{nullptr};
+  size_t errorsEmitted{0};
   for (const Message *msg : sorted) {
     if (lastMsg && *msg == *lastMsg) {
       // Don't emit two identical messages for the same location
       continue;
     }
     msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
+    if (msg->IsFatal()) {
+      ++errorsEmitted;
+    }
----------------
akuhlens wrote:

<img width="1920" alt="Screenshot 2025-07-08 at 4 48 37 PM" src="https://github.com/user-attachments/assets/45793879-b7a3-4766-9f78-83bf9cb3e1f2" />
gfortran also seems to emit the warnings before the first error.

https://github.com/llvm/llvm-project/pull/147614

From flang-commits at lists.llvm.org  Tue Jul  8 16:50:42 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 08 Jul 2025 16:50:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] add -Wfatal-errors (PR
	#147614)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147614@github.com>
Message-ID: <686daed2.170a0220.220d61.8fc3@mx.google.com>


https://github.com/klausler approved this pull request.


https://github.com/llvm/llvm-project/pull/147614

From flang-commits at lists.llvm.org  Tue Jul  8 16:51:11 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Tue, 08 Jul 2025 16:51:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
Message-ID: <llvm.org/llvm/llvm-project/pull/147618@github.com>


https://github.com/vzakhari created https://github.com/llvm/llvm-project/pull/147618

After #142609, repacking of assumed-type arrays under -frepack-arrays
become broken, because the code tried to pass a `!fir.box`
source_box to `fir.embox`, which is not allowed.
It might be okay to cast the original box to class for using it
in `fir.embox`.


>From e2ebd285b37cdf57e30f1a569eddf2c715704a67 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Tue, 8 Jul 2025 16:43:24 -0700
Subject: [PATCH] [flang] Fixed assumed-type temporary allocation for
 -frepack-arrays.

After #142609, repacking of assumed-type arrays under -frepack-arrays
become broken, because the code tried to pass a `!fir.box`
source_box to `fir.embox`, which is not allowed.
It might be okay to cast the original box to class for using it
in `fir.embox`.
---
 flang/lib/Optimizer/Builder/MutableBox.cpp    |  9 +++
 flang/test/Transforms/lower-repack-arrays.fir | 78 +++++++++++++++++++
 2 files changed, 87 insertions(+)

diff --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp
index d944a4c98473e..8c4e12913f714 100644
--- a/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -987,6 +987,15 @@ mlir::Value fir::factory::getAndEstablishBoxStorage(
   mlir::Value boxStorage = builder.createTemporary(loc, boxTy);
   mlir::Value nullAddr =
       builder.createNullConstant(loc, boxTy.getBaseAddressType());
+  if (polymorphicMold && fir::isAssumedType(polymorphicMold.getType())) {
+    // An assumed-type (!fir.box) entity cannot be used as a mold
+    // in fir.embox, so we have to represent it as an unlimited
+    // polymorphic entity (!fir.class).
+    mlir::Type newMoldType = fir::wrapInClassOrBoxType(
+        mlir::cast<fir::BaseBoxType>(polymorphicMold.getType()).getEleTy(),
+        /*isPolymorphic=*/true);
+    polymorphicMold = builder.createConvert(loc, newMoldType, polymorphicMold);
+  }
   mlir::Value box =
       builder.create<fir::EmboxOp>(loc, boxTy, nullAddr, shape,
                                    /*emptySlice=*/mlir::Value{},
diff --git a/flang/test/Transforms/lower-repack-arrays.fir b/flang/test/Transforms/lower-repack-arrays.fir
index 458869cce45fd..d96cfc609533a 100644
--- a/flang/test/Transforms/lower-repack-arrays.fir
+++ b/flang/test/Transforms/lower-repack-arrays.fir
@@ -1091,3 +1091,81 @@ func.func @_QPtest7_stack(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_nam
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+// Test assumed type array.
+// The temporary allocation requires creating a fir.box with the mold
+// being the !fir.box<!fir.array<?xnone>>. We have to cast it to
+// !fir.class<!fir.array<?xnone>> to make the fir.embox's source_box
+// operand valid.
+// CHECK-LABEL:   func.func @_QPrepack_assumed_type(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant
+// CHECK:           %[[VAL_1:.*]] = arith.constant
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant false
+// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]] = fir.is_present %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:           %[[VAL_7:.*]] = fir.if %[[VAL_6]] -> (!fir.box<!fir.array<?xnone>>) {
+// CHECK:             %[[VAL_8:.*]] = fir.is_contiguous_box %[[ARG0]] whole : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:             %[[VAL_9:.*]] = arith.cmpi eq, %[[VAL_8]], %[[VAL_3]] : i1
+// CHECK:             %[[VAL_10:.*]] = fir.box_addr %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.ref<!fir.box<!fir.array<?xnone>>>
+// CHECK:             %[[VAL_11:.*]] = fir.is_present %[[VAL_10]] : (!fir.ref<!fir.box<!fir.array<?xnone>>>) -> i1
+// CHECK:             %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_11]] : i1
+// CHECK:             %[[VAL_13:.*]] = fir.if %[[VAL_12]] weights([0, 1]) -> (!fir.box<!fir.array<?xnone>>) {
+// CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.box<!fir.array<?xnone>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_15:.*]] = fir.shape %[[VAL_14]]#1 : (index) -> !fir.shape<1>
+// CHECK:               %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?xnone>>
+// CHECK:               %[[VAL_17:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.array<?xnone>>
+// CHECK:               %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_15]]) source_box %[[VAL_17]] : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.class<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:               fir.store %[[VAL_18]] to %[[VAL_4]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>
+// CHECK:               %[[VAL_19:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:               %[[VAL_20:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_21:.*]] = fir.absent !fir.box<none>
+// CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               %[[VAL_25:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_22]], %[[VAL_23]], %[[VAL_3]], %[[VAL_21]], %[[VAL_24]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>
+// CHECK:               %[[VAL_27:.*]] = fir.declare %[[VAL_26]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?xnone>>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:               %[[VAL_28:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_27]] : (!fir.class<!fir.heap<!fir.array<?xnone>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_28]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_29]], %[[VAL_30]], %[[VAL_31]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               %[[VAL_32:.*]] = fir.shift %[[VAL_14]]#0 : (index) -> !fir.shift<1>
+// CHECK:               %[[VAL_33:.*]] = fir.rebox %[[VAL_27]](%[[VAL_32]]) : (!fir.class<!fir.heap<!fir.array<?xnone>>>, !fir.shift<1>) -> !fir.box<!fir.array<?xnone>>
+// CHECK:               fir.result %[[VAL_33]] : !fir.box<!fir.array<?xnone>>
+// CHECK:             } else {
+// CHECK:               fir.result %[[ARG0]] : !fir.box<!fir.array<?xnone>>
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_13]] : !fir.box<!fir.array<?xnone>>
+// CHECK:           } else {
+// CHECK:             fir.result %[[ARG0]] : !fir.box<!fir.array<?xnone>>
+// CHECK:           }
+// CHECK:           %[[VAL_34:.*]] = fir.declare %[[VAL_7]] dummy_scope %[[VAL_5]] {uniq_name = "_QFrepack_assumed_typeEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> !fir.box<!fir.array<?xnone>>
+// CHECK:           %[[VAL_35:.*]] = fir.is_present %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:           fir.if %[[VAL_35]] {
+// CHECK:             %[[VAL_36:.*]] = fir.box_addr %[[VAL_7]] : (!fir.box<!fir.array<?xnone>>) -> !fir.heap<!fir.array<?xnone>>
+// CHECK:             %[[VAL_37:.*]] = fir.box_addr %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.heap<!fir.array<?xnone>>
+// CHECK:             %[[VAL_38:.*]] = fir.convert %[[VAL_36]] : (!fir.heap<!fir.array<?xnone>>) -> index
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?xnone>>) -> index
+// CHECK:             %[[VAL_40:.*]] = arith.cmpi ne, %[[VAL_38]], %[[VAL_39]] : index
+// CHECK:             fir.if %[[VAL_40]] weights([0, 1]) {
+// CHECK:               %[[VAL_41:.*]] = fir.address_of(@_QQclX2b7d3b4eb2a9b0772619a406c28937e5) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_42:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_41]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_42]], %[[VAL_43]], %[[VAL_44]], %[[VAL_0]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               fir.freemem %[[VAL_36]] : !fir.heap<!fir.array<?xnone>>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @_QPrepack_assumed_type(%arg0: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.pack_array %arg0 heap whole : (!fir.box<!fir.array<?xnone>>) -> !fir.box<!fir.array<?xnone>>
+  %2 = fir.declare %1 dummy_scope %0 {uniq_name = "_QFrepack_assumed_typeEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> !fir.box<!fir.array<?xnone>>
+  fir.unpack_array %1 to %arg0 heap : !fir.box<!fir.array<?xnone>>
+  return
+}


From flang-commits at lists.llvm.org  Tue Jul  8 16:51:40 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 08 Jul 2025 16:51:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147618@github.com>
Message-ID: <686daf0c.170a0220.2f65d2.8f58@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Slava Zakharin (vzakhari)

<details>
<summary>Changes</summary>

After #<!-- -->142609, repacking of assumed-type arrays under -frepack-arrays
become broken, because the code tried to pass a `!fir.box`
source_box to `fir.embox`, which is not allowed.
It might be okay to cast the original box to class for using it
in `fir.embox`.


---
Full diff: https://github.com/llvm/llvm-project/pull/147618.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/Builder/MutableBox.cpp (+9) 
- (modified) flang/test/Transforms/lower-repack-arrays.fir (+78) 


``````````diff
diff --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp
index d944a4c98473e..8c4e12913f714 100644
--- a/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -987,6 +987,15 @@ mlir::Value fir::factory::getAndEstablishBoxStorage(
   mlir::Value boxStorage = builder.createTemporary(loc, boxTy);
   mlir::Value nullAddr =
       builder.createNullConstant(loc, boxTy.getBaseAddressType());
+  if (polymorphicMold && fir::isAssumedType(polymorphicMold.getType())) {
+    // An assumed-type (!fir.box) entity cannot be used as a mold
+    // in fir.embox, so we have to represent it as an unlimited
+    // polymorphic entity (!fir.class).
+    mlir::Type newMoldType = fir::wrapInClassOrBoxType(
+        mlir::cast<fir::BaseBoxType>(polymorphicMold.getType()).getEleTy(),
+        /*isPolymorphic=*/true);
+    polymorphicMold = builder.createConvert(loc, newMoldType, polymorphicMold);
+  }
   mlir::Value box =
       builder.create<fir::EmboxOp>(loc, boxTy, nullAddr, shape,
                                    /*emptySlice=*/mlir::Value{},
diff --git a/flang/test/Transforms/lower-repack-arrays.fir b/flang/test/Transforms/lower-repack-arrays.fir
index 458869cce45fd..d96cfc609533a 100644
--- a/flang/test/Transforms/lower-repack-arrays.fir
+++ b/flang/test/Transforms/lower-repack-arrays.fir
@@ -1091,3 +1091,81 @@ func.func @_QPtest7_stack(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_nam
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+// Test assumed type array.
+// The temporary allocation requires creating a fir.box with the mold
+// being the !fir.box<!fir.array<?xnone>>. We have to cast it to
+// !fir.class<!fir.array<?xnone>> to make the fir.embox's source_box
+// operand valid.
+// CHECK-LABEL:   func.func @_QPrepack_assumed_type(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant
+// CHECK:           %[[VAL_1:.*]] = arith.constant
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant false
+// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]] = fir.is_present %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:           %[[VAL_7:.*]] = fir.if %[[VAL_6]] -> (!fir.box<!fir.array<?xnone>>) {
+// CHECK:             %[[VAL_8:.*]] = fir.is_contiguous_box %[[ARG0]] whole : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:             %[[VAL_9:.*]] = arith.cmpi eq, %[[VAL_8]], %[[VAL_3]] : i1
+// CHECK:             %[[VAL_10:.*]] = fir.box_addr %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.ref<!fir.box<!fir.array<?xnone>>>
+// CHECK:             %[[VAL_11:.*]] = fir.is_present %[[VAL_10]] : (!fir.ref<!fir.box<!fir.array<?xnone>>>) -> i1
+// CHECK:             %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_11]] : i1
+// CHECK:             %[[VAL_13:.*]] = fir.if %[[VAL_12]] weights([0, 1]) -> (!fir.box<!fir.array<?xnone>>) {
+// CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.box<!fir.array<?xnone>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_15:.*]] = fir.shape %[[VAL_14]]#1 : (index) -> !fir.shape<1>
+// CHECK:               %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?xnone>>
+// CHECK:               %[[VAL_17:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.array<?xnone>>
+// CHECK:               %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_15]]) source_box %[[VAL_17]] : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.class<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:               fir.store %[[VAL_18]] to %[[VAL_4]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>
+// CHECK:               %[[VAL_19:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:               %[[VAL_20:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_21:.*]] = fir.absent !fir.box<none>
+// CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               %[[VAL_25:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_22]], %[[VAL_23]], %[[VAL_3]], %[[VAL_21]], %[[VAL_24]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>
+// CHECK:               %[[VAL_27:.*]] = fir.declare %[[VAL_26]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?xnone>>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:               %[[VAL_28:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_27]] : (!fir.class<!fir.heap<!fir.array<?xnone>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_28]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_29]], %[[VAL_30]], %[[VAL_31]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               %[[VAL_32:.*]] = fir.shift %[[VAL_14]]#0 : (index) -> !fir.shift<1>
+// CHECK:               %[[VAL_33:.*]] = fir.rebox %[[VAL_27]](%[[VAL_32]]) : (!fir.class<!fir.heap<!fir.array<?xnone>>>, !fir.shift<1>) -> !fir.box<!fir.array<?xnone>>
+// CHECK:               fir.result %[[VAL_33]] : !fir.box<!fir.array<?xnone>>
+// CHECK:             } else {
+// CHECK:               fir.result %[[ARG0]] : !fir.box<!fir.array<?xnone>>
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_13]] : !fir.box<!fir.array<?xnone>>
+// CHECK:           } else {
+// CHECK:             fir.result %[[ARG0]] : !fir.box<!fir.array<?xnone>>
+// CHECK:           }
+// CHECK:           %[[VAL_34:.*]] = fir.declare %[[VAL_7]] dummy_scope %[[VAL_5]] {uniq_name = "_QFrepack_assumed_typeEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> !fir.box<!fir.array<?xnone>>
+// CHECK:           %[[VAL_35:.*]] = fir.is_present %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:           fir.if %[[VAL_35]] {
+// CHECK:             %[[VAL_36:.*]] = fir.box_addr %[[VAL_7]] : (!fir.box<!fir.array<?xnone>>) -> !fir.heap<!fir.array<?xnone>>
+// CHECK:             %[[VAL_37:.*]] = fir.box_addr %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.heap<!fir.array<?xnone>>
+// CHECK:             %[[VAL_38:.*]] = fir.convert %[[VAL_36]] : (!fir.heap<!fir.array<?xnone>>) -> index
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?xnone>>) -> index
+// CHECK:             %[[VAL_40:.*]] = arith.cmpi ne, %[[VAL_38]], %[[VAL_39]] : index
+// CHECK:             fir.if %[[VAL_40]] weights([0, 1]) {
+// CHECK:               %[[VAL_41:.*]] = fir.address_of(@_QQclX2b7d3b4eb2a9b0772619a406c28937e5) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_42:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_41]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_42]], %[[VAL_43]], %[[VAL_44]], %[[VAL_0]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               fir.freemem %[[VAL_36]] : !fir.heap<!fir.array<?xnone>>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @_QPrepack_assumed_type(%arg0: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.pack_array %arg0 heap whole : (!fir.box<!fir.array<?xnone>>) -> !fir.box<!fir.array<?xnone>>
+  %2 = fir.declare %1 dummy_scope %0 {uniq_name = "_QFrepack_assumed_typeEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> !fir.box<!fir.array<?xnone>>
+  fir.unpack_array %1 to %arg0 heap : !fir.box<!fir.array<?xnone>>
+  return
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/147618

From flang-commits at lists.llvm.org  Tue Jul  8 16:51:51 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 08 Jul 2025 16:51:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't create bogus tokens from
 token pasting (##) (PR #147596)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147596@github.com>
Message-ID: <686daf17.050a0220.2b2157.a5ed@mx.google.com>


https://github.com/eugeneepshteyn approved this pull request.


https://github.com/llvm/llvm-project/pull/147596

From flang-commits at lists.llvm.org  Tue Jul  8 16:51:51 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 08 Jul 2025 16:51:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't create bogus tokens from
 token pasting (##) (PR #147596)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147596@github.com>
Message-ID: <686daf17.170a0220.28df35.8843@mx.google.com>


================
@@ -156,23 +156,50 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
   }
   TokenSequence result;
   std::size_t tokens{text.SizeInTokens()};
-  bool pasting{false};
+  std::optional<CharBlock> before; // last non-blank token before ##
   for (std::size_t j{0}; j < tokens; ++j) {
-    if (IsTokenPasting(text.TokenAt(j))) {
-      if (!pasting) {
+    CharBlock after{text.TokenAt(j)};
+    if (!before) {
+      if (IsTokenPasting(after)) {
         while (!result.empty() &&
             result.TokenAt(result.SizeInTokens() - 1).IsBlank()) {
           result.pop_back();
         }
         if (!result.empty()) {
-          result.ReopenLastToken();
-          pasting = true;
+          before = result.TokenAt(result.SizeInTokens() - 1);
         }
+      } else {
+        result.AppendRange(text, j, 1);
+      }
+    } else if (after.IsBlank() || IsTokenPasting(after)) {
+      // drop it
+    } else { // pasting before ## after
+      bool doPaste{false};
+      char last{before->end()[-1]};
+      char first{after.begin()[0]};
----------------
eugeneepshteyn wrote:

Just a suggestion: maybe there could be `char CharBlock::first()` and `char CharBlock::last()`. There are a few instances of  'begin()[0]` and `end()[-1]` in the code that could then be replaced by `first()` and `last()`.

https://github.com/llvm/llvm-project/pull/147596

From flang-commits at lists.llvm.org  Tue Jul  8 16:52:12 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 08 Jul 2025 16:52:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't create bogus tokens from
 token pasting (##) (PR #147596)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147596@github.com>
Message-ID: <686daf2c.170a0220.2bda82.0618@mx.google.com>


https://github.com/eugeneepshteyn edited https://github.com/llvm/llvm-project/pull/147596

From flang-commits at lists.llvm.org  Tue Jul  8 18:47:42 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Tue, 08 Jul 2025 18:47:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147618@github.com>
Message-ID: <686dca3e.050a0220.15a319.6270@mx.google.com>


https://github.com/vzakhari updated https://github.com/llvm/llvm-project/pull/147618

>From e2ebd285b37cdf57e30f1a569eddf2c715704a67 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Tue, 8 Jul 2025 16:43:24 -0700
Subject: [PATCH 1/2] [flang] Fixed assumed-type temporary allocation for
 -frepack-arrays.

After #142609, repacking of assumed-type arrays under -frepack-arrays
become broken, because the code tried to pass a `!fir.box`
source_box to `fir.embox`, which is not allowed.
It might be okay to cast the original box to class for using it
in `fir.embox`.
---
 flang/lib/Optimizer/Builder/MutableBox.cpp    |  9 +++
 flang/test/Transforms/lower-repack-arrays.fir | 78 +++++++++++++++++++
 2 files changed, 87 insertions(+)

diff --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp
index d944a4c98473e..8c4e12913f714 100644
--- a/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -987,6 +987,15 @@ mlir::Value fir::factory::getAndEstablishBoxStorage(
   mlir::Value boxStorage = builder.createTemporary(loc, boxTy);
   mlir::Value nullAddr =
       builder.createNullConstant(loc, boxTy.getBaseAddressType());
+  if (polymorphicMold && fir::isAssumedType(polymorphicMold.getType())) {
+    // An assumed-type (!fir.box) entity cannot be used as a mold
+    // in fir.embox, so we have to represent it as an unlimited
+    // polymorphic entity (!fir.class).
+    mlir::Type newMoldType = fir::wrapInClassOrBoxType(
+        mlir::cast<fir::BaseBoxType>(polymorphicMold.getType()).getEleTy(),
+        /*isPolymorphic=*/true);
+    polymorphicMold = builder.createConvert(loc, newMoldType, polymorphicMold);
+  }
   mlir::Value box =
       builder.create<fir::EmboxOp>(loc, boxTy, nullAddr, shape,
                                    /*emptySlice=*/mlir::Value{},
diff --git a/flang/test/Transforms/lower-repack-arrays.fir b/flang/test/Transforms/lower-repack-arrays.fir
index 458869cce45fd..d96cfc609533a 100644
--- a/flang/test/Transforms/lower-repack-arrays.fir
+++ b/flang/test/Transforms/lower-repack-arrays.fir
@@ -1091,3 +1091,81 @@ func.func @_QPtest7_stack(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_nam
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+// Test assumed type array.
+// The temporary allocation requires creating a fir.box with the mold
+// being the !fir.box<!fir.array<?xnone>>. We have to cast it to
+// !fir.class<!fir.array<?xnone>> to make the fir.embox's source_box
+// operand valid.
+// CHECK-LABEL:   func.func @_QPrepack_assumed_type(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant
+// CHECK:           %[[VAL_1:.*]] = arith.constant
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant false
+// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]] = fir.is_present %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:           %[[VAL_7:.*]] = fir.if %[[VAL_6]] -> (!fir.box<!fir.array<?xnone>>) {
+// CHECK:             %[[VAL_8:.*]] = fir.is_contiguous_box %[[ARG0]] whole : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:             %[[VAL_9:.*]] = arith.cmpi eq, %[[VAL_8]], %[[VAL_3]] : i1
+// CHECK:             %[[VAL_10:.*]] = fir.box_addr %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.ref<!fir.box<!fir.array<?xnone>>>
+// CHECK:             %[[VAL_11:.*]] = fir.is_present %[[VAL_10]] : (!fir.ref<!fir.box<!fir.array<?xnone>>>) -> i1
+// CHECK:             %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_11]] : i1
+// CHECK:             %[[VAL_13:.*]] = fir.if %[[VAL_12]] weights([0, 1]) -> (!fir.box<!fir.array<?xnone>>) {
+// CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.box<!fir.array<?xnone>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_15:.*]] = fir.shape %[[VAL_14]]#1 : (index) -> !fir.shape<1>
+// CHECK:               %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?xnone>>
+// CHECK:               %[[VAL_17:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.array<?xnone>>
+// CHECK:               %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_15]]) source_box %[[VAL_17]] : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.class<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:               fir.store %[[VAL_18]] to %[[VAL_4]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>
+// CHECK:               %[[VAL_19:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:               %[[VAL_20:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_21:.*]] = fir.absent !fir.box<none>
+// CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               %[[VAL_25:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_22]], %[[VAL_23]], %[[VAL_3]], %[[VAL_21]], %[[VAL_24]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>
+// CHECK:               %[[VAL_27:.*]] = fir.declare %[[VAL_26]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?xnone>>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+// CHECK:               %[[VAL_28:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_27]] : (!fir.class<!fir.heap<!fir.array<?xnone>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_28]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_29]], %[[VAL_30]], %[[VAL_31]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               %[[VAL_32:.*]] = fir.shift %[[VAL_14]]#0 : (index) -> !fir.shift<1>
+// CHECK:               %[[VAL_33:.*]] = fir.rebox %[[VAL_27]](%[[VAL_32]]) : (!fir.class<!fir.heap<!fir.array<?xnone>>>, !fir.shift<1>) -> !fir.box<!fir.array<?xnone>>
+// CHECK:               fir.result %[[VAL_33]] : !fir.box<!fir.array<?xnone>>
+// CHECK:             } else {
+// CHECK:               fir.result %[[ARG0]] : !fir.box<!fir.array<?xnone>>
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_13]] : !fir.box<!fir.array<?xnone>>
+// CHECK:           } else {
+// CHECK:             fir.result %[[ARG0]] : !fir.box<!fir.array<?xnone>>
+// CHECK:           }
+// CHECK:           %[[VAL_34:.*]] = fir.declare %[[VAL_7]] dummy_scope %[[VAL_5]] {uniq_name = "_QFrepack_assumed_typeEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> !fir.box<!fir.array<?xnone>>
+// CHECK:           %[[VAL_35:.*]] = fir.is_present %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> i1
+// CHECK:           fir.if %[[VAL_35]] {
+// CHECK:             %[[VAL_36:.*]] = fir.box_addr %[[VAL_7]] : (!fir.box<!fir.array<?xnone>>) -> !fir.heap<!fir.array<?xnone>>
+// CHECK:             %[[VAL_37:.*]] = fir.box_addr %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.heap<!fir.array<?xnone>>
+// CHECK:             %[[VAL_38:.*]] = fir.convert %[[VAL_36]] : (!fir.heap<!fir.array<?xnone>>) -> index
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?xnone>>) -> index
+// CHECK:             %[[VAL_40:.*]] = arith.cmpi ne, %[[VAL_38]], %[[VAL_39]] : index
+// CHECK:             fir.if %[[VAL_40]] weights([0, 1]) {
+// CHECK:               %[[VAL_41:.*]] = fir.address_of(@_QQclX2b7d3b4eb2a9b0772619a406c28937e5) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_42:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_41]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_42]], %[[VAL_43]], %[[VAL_44]], %[[VAL_0]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               fir.freemem %[[VAL_36]] : !fir.heap<!fir.array<?xnone>>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @_QPrepack_assumed_type(%arg0: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.pack_array %arg0 heap whole : (!fir.box<!fir.array<?xnone>>) -> !fir.box<!fir.array<?xnone>>
+  %2 = fir.declare %1 dummy_scope %0 {uniq_name = "_QFrepack_assumed_typeEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> !fir.box<!fir.array<?xnone>>
+  fir.unpack_array %1 to %arg0 heap : !fir.box<!fir.array<?xnone>>
+  return
+}

>From ec30a52600644d38fcfffe760a6c1f068e6f063d Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Tue, 8 Jul 2025 18:47:21 -0700
Subject: [PATCH 2/2] Fixed LIT test.

---
 flang/test/Transforms/lower-repack-arrays.fir | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Transforms/lower-repack-arrays.fir b/flang/test/Transforms/lower-repack-arrays.fir
index d96cfc609533a..07bc5eefff427 100644
--- a/flang/test/Transforms/lower-repack-arrays.fir
+++ b/flang/test/Transforms/lower-repack-arrays.fir
@@ -1152,7 +1152,7 @@ func.func @_QPtest7_stack(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_nam
 // CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?xnone>>) -> index
 // CHECK:             %[[VAL_40:.*]] = arith.cmpi ne, %[[VAL_38]], %[[VAL_39]] : index
 // CHECK:             fir.if %[[VAL_40]] weights([0, 1]) {
-// CHECK:               %[[VAL_41:.*]] = fir.address_of(@_QQclX2b7d3b4eb2a9b0772619a406c28937e5) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_41:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
 // CHECK:               %[[VAL_42:.*]] = fir.convert %[[ARG0]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
 // CHECK:               %[[VAL_43:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.array<?xnone>>) -> !fir.box<none>
 // CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_41]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>


From flang-commits at lists.llvm.org  Wed Jul  9 01:04:17 2025
From: flang-commits at lists.llvm.org (Martin Erhart via flang-commits)
Date: Wed, 09 Jul 2025 01:04:17 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir] remove dangling builders from
	td (PR #147619)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147619@github.com>
Message-ID: <686e2281.170a0220.1e9b.a0da@mx.google.com>


================
@@ -52,12 +52,6 @@ class VariadicIntOp<string mnemonic> : SMTIntOp<mnemonic, [Pure, Commutative]> {
   let arguments = (ins Variadic<IntType>:$inputs);
   let results = (outs IntType:$result);
   let assemblyFormat = "$inputs attr-dict";
-
-  let builders = [
-    OpBuilder<(ins "mlir::ValueRange":$inputs), [{
-      build($_builder, $_state, $_builder.getType<smt::IntType>(), inputs);
-    }]>,
----------------
maerhart wrote:

Looks good to merge from my side. Thanks!

https://github.com/llvm/llvm-project/pull/147619

From flang-commits at lists.llvm.org  Wed Jul  9 01:14:38 2025
From: flang-commits at lists.llvm.org (Mehdi Amini via flang-commits)
Date: Wed, 09 Jul 2025 01:14:38 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir] remove dangling builders from
	td (PR #147619)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147619@github.com>
Message-ID: <686e24ee.170a0220.ddf9b.1aa1@mx.google.com>


https://github.com/joker-eph approved this pull request.

Thanks for the cleanup!

https://github.com/llvm/llvm-project/pull/147619

From flang-commits at lists.llvm.org  Wed Jul  9 02:00:48 2025
From: flang-commits at lists.llvm.org (Oleksandr Alex Zinenko via flang-commits)
Date: Wed, 09 Jul 2025 02:00:48 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][tblgen] add concrete create
	methods (PR #147168)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147168@github.com>
Message-ID: <686e2fc0.050a0220.5fe11.b365@mx.google.com>


ftynse wrote:

> Overall I'd like to hear more about this from @jpienaar @ftynse @j2kun (maybe @Mogball and @River707 ?) : are we OK to transition from builder.create<Op>(...) to Op::create(builder, ...) for the benefit of better IDE support?

I'd be very happy to transition! I can offer some help if needed.

Strategically, I think this still needs a dedicated forum post for visibility, even though it appears there is consensus in this thread. The post should include instructions for downstreams on how to transition.

I'd also like to deprecate and remove the previous style of builders _eventually_, following the same consistency arguments as @joker-eph. At the very least for upstream usage. Given the importance of this particular API, I'd say we could deprecate after the next release branches, and remove one or two releases after that. Maybe there'a a way to have a GitHub action check new commits for the `create<` string and notify about the preferred upstream style as a way of soft deprecation. This is something to discuss on the forum, no particular rush IMO. 

https://github.com/llvm/llvm-project/pull/147168

From flang-commits at lists.llvm.org  Wed Jul  9 03:38:15 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 03:38:15 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [Flang][FlangRT][Runtime] Add
 RT_OFFLOAD_API_GROUP_BEGIN to missing symbols on AMDGPU (PR #147612)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147612@github.com>
Message-ID: <686e4697.170a0220.3c67e2.ab2a@mx.google.com>


agozillon wrote:

> Thank you for the changes!
> 
> I wish clang supported `declare target` properly for the explicit instantiations and did not require `declare target` for the class declaration. Since the function references are automatically `declare target` if they are referenced by other `declare target` functions according to OpenMP standard, it really does not make much sense to require `declare target` to be applied to the class declaration and its methods' declarations.
>

Yes, I actually thought we already had the implicit marking in Clang C++/OpenMP, but it seems to not be the case, or at least it doesn't handle more complicated cases unfortunately! :-) So sadly need to be a bit more generous with the RT_OFFLOAD_API_GROUP_BEGIN macros, which isn't the prettiest or best thing for compile times long term I imagine. 

> I guess a less intrusive way would be to expand `RT_API_ATTRS` into something like `[[omp::declare_target]]`, but this is not part of OpenMP standard :)

I'd personally prefer to keep it as close to OpenMP standard acceptable as possible, mainly so other OpenMP compilers can come along and compile the runtime if they'd like to (I do think OpenMP does support the attribute specifier syntax in the more recent standard, but I may be misremembering) ! But happy to make this change if you feel it'd be better! :-) 


https://github.com/llvm/llvm-project/pull/147612

From flang-commits at lists.llvm.org  Wed Jul  9 03:39:58 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 03:39:58 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [Flang][FlangRT][Runtime] Add
 RT_OFFLOAD_API_GROUP_BEGIN to missing symbols on AMDGPU (PR #147612)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147612@github.com>
Message-ID: <686e46fe.630a0220.325e69.bc3d@mx.google.com>


agozillon wrote:

> If I understand correctly, this is related to #145888 - can you please add a reference into the description?

I didn't actually notice this issue, I ran into the problem separately in some map tests I have! Thank you for pointing me to it, yes, I do believe this would fix that issue, at least the symbols referenced look similar, if it doesn't fix them all I'd be happy to create a subsequent PR to fix the rest. I'll alter the description just now to add the issue, thank you again :-)!

https://github.com/llvm/llvm-project/pull/147612

From flang-commits at lists.llvm.org  Wed Jul  9 03:40:23 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 03:40:23 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [Flang][FlangRT][Runtime] Add
 RT_OFFLOAD_API_GROUP_BEGIN to missing symbols on AMDGPU (PR #147612)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147612@github.com>
Message-ID: <686e4717.170a0220.220d61.ac24@mx.google.com>


https://github.com/agozillon edited https://github.com/llvm/llvm-project/pull/147612

From flang-commits at lists.llvm.org  Wed Jul  9 03:47:55 2025
From: flang-commits at lists.llvm.org (Mehdi Amini via flang-commits)
Date: Wed, 09 Jul 2025 03:47:55 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][tblgen] add concrete create
	methods (PR #147168)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147168@github.com>
Message-ID: <686e48db.170a0220.33cbb3.25cc@mx.google.com>


joker-eph wrote:

> are we OK to transition from builder.create(...) to Op::create(builder, ...) for the benefit of better IDE support?

For posterity I'll correct this, @Mogball identified on discourse that is more to it than better IDE support. Not having a template means that we can have user-defined implicit conversion as part of the call.
For example this does not compile today:

```
  auto acOp = builder.create<spirv::AccessChainOp>(loc, addrOp, {zeroOp, offsetOp});
```

It throw an error like:
```
mlir/include/mlir/IR/Builders.h:503:8: note: candidate template ignored: substitution failure [with OpTy = spirv::AccessChainOp]: deduced incomplete pack <mlir::spirv::AddressOfOp &, (no value)> for template parameter 'Args'
  502 |   template <typename OpTy, typename... Args>
      |                                        ~~~~
  503 |   OpTy create(Location location, Args &&...args) {
      |        ^
1 error generated.
```

We must be explicit and make the type perfectly match:

```
  auto acOp = builder.create<spirv::AccessChainOp>(loc, addrOp, llvm::ArrayRef({zeroOp, offsetOp}));
```

Whereas the new form will just work like this:

```
  auto acOp = spirv::AccessChainOp::create(builder, loc, addrOp, {zeroOp, offsetOp});
```


https://github.com/llvm/llvm-project/pull/147168

From flang-commits at lists.llvm.org  Wed Jul  9 05:04:05 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 05:04:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e5ab5.630a0220.adeee.ecf7@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks for another fix

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 05:04:06 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 05:04:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e5ab6.170a0220.25ad8.aa0a@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 05:04:06 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 05:04:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e5ab6.170a0220.1171d.313b@mx.google.com>


================
@@ -214,16 +215,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::MulOverflow(
+            static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
----------------
tblah wrote:

Personally I would find `overflow = overflow || MulOverflow() || AddOverflow()` easier to read but that is subjective. This is okay as it is.

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:24 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:24 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620c.170a0220.37348f.b1a0@mx.google.com>


https://github.com/ergawy commented:

Thanks for the updates. I have some further comments/suggestions.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:24 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:24 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620c.630a0220.9fc0e.c8dc@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:24 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:24 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620c.170a0220.279776.b22f@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
----------------
ergawy wrote:

It seems this function and some of the ones below are shared between `CodeGen.cpp` and `CodeGenOpenMP.cpp`. Can we move them to a shared location, e.g. `flang/Optimizer/Support/Utils.h/.cpp`?

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:25 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:25 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620d.a70a0220.19e60b.d8bb@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    mlir::Type llvmPtrTy =
+        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "omp.target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy,
+                                          lowerTy().getDataLayout());
+    if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(lowerTy(), loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
----------------
ergawy wrote:

I think this `if` condition is not covered by the introduced tests.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:25 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:25 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620d.170a0220.19195a.0204@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
----------------
ergawy wrote:

I think we are using a deprected conversion here (see: https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/Support/TypeSize.h#L360).

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:25 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:25 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620d.170a0220.375496.aa16@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    mlir::Type llvmPtrTy =
+        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
----------------
ergawy wrote:

```suggestion
        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext());
```
`0` is the default value for the address space.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:25 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:25 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620d.630a0220.125127.0899@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
----------------
ergawy wrote:

I think this conversion pattern should not replace `omp.target_allocmem`. Instead it should only convert the `fir`-specific type to `i64` and leave the lowering of `omp.target_allocmem` to `call ptr @omp_target_alloc(...)` to `convertTargetAllocMemOp` in `OpenMPToLLVMIRTranslation.cpp`. What we have now means that we do the same conversion in 2 different places. The preferred place would be `OpenMPToLLVMIRTranslation.cpp` since this is where all OpenMP to LLVM conversions are done.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:25 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:25 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620d.050a0220.95a1e.c3b7@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
----------------
ergawy wrote:

It seems this function does more than what the name suggests (i.e. copies old attributes, drops the old `operandSegmentSizes` and recreates it, and adds and empty `op_bundle_sizes`).

Should we do the attribute changes in-place (i.e. at the call site) instead? Since this function does not seem reusable in other scenarios.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:35:27 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:35:27 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e620f.630a0220.2ed42a.bd7e@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
----------------
ergawy wrote:

nit: move before use location below.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:36:18 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:36:18 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e6242.050a0220.2be776.c969@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
----------------
ergawy wrote:

I see this is adapted from `CodeGen.cpp` though, so see my comment below.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:37:24 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 05:37:24 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e6284.050a0220.3c98c5.c6f4@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul  9 05:49:11 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:49:11 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e6547.630a0220.4f15a.b886@mx.google.com>


https://github.com/NimishMishra updated https://github.com/llvm/llvm-project/pull/139386

>From 5177cad8b2e32e91b3aaaad08e4b4f19449a12dd Mon Sep 17 00:00:00 2001
From: Nimish Mishra <neelam.nimish at gmail.com>
Date: Wed, 9 Jul 2025 14:22:08 +0530
Subject: [PATCH 1/3] [flang][OpenMP] Support MLIR lowering of linear clause
 for omp.wsloop

---
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  7 +-
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |  2 +-
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 98 ++++++++++++-------
 3 files changed, 66 insertions(+), 41 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0a56e888ac44b..ef4c592709d4c 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1816,8 +1816,7 @@ static void genSimdClauses(
   cp.processReduction(loc, clauseOps, reductionSyms);
   cp.processSafelen(clauseOps);
   cp.processSimdlen(clauseOps);
-
-  cp.processTODO<clause::Linear>(loc, llvm::omp::Directive::OMPD_simd);
+  cp.processLinear(clauseOps);
 }
 
 static void genSingleClauses(lower::AbstractConverter &converter,
@@ -2007,9 +2006,9 @@ static void genWsloopClauses(
   cp.processOrdered(clauseOps);
   cp.processReduction(loc, clauseOps, reductionSyms);
   cp.processSchedule(stmtCtx, clauseOps);
+  cp.processLinear(clauseOps);
 
-  cp.processTODO<clause::Allocate, clause::Linear>(
-      loc, llvm::omp::Directive::OMPD_do);
+  cp.processTODO<clause::Allocate>(loc, llvm::omp::Directive::OMPD_do);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index ffc84781f77ff..c4b91348c9853 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -2608,7 +2608,7 @@ void SimdOp::build(OpBuilder &builder, OperationState &state,
   // TODO Store clauses in op: linearVars, linearStepVars
   SimdOp::build(builder, state, clauses.alignedVars,
                 makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,
-                /*linear_vars=*/{}, /*linear_step_vars=*/{},
+                clauses.linearVars, clauses.linearStepVars,
                 clauses.nontemporalVars, clauses.order, clauses.orderMod,
                 clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
                 clauses.privateNeedsBarrier, clauses.reductionMod,
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 336f71ccd06a3..897357472f4a4 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -147,9 +147,9 @@ class LinearClauseProcessor {
 
 public:
   // Allocate space for linear variabes
-  void createLinearVar(llvm::IRBuilderBase &builder,
-                       LLVM::ModuleTranslation &moduleTranslation,
-                       mlir::Value &linearVar) {
+  LogicalResult createLinearVar(llvm::IRBuilderBase &builder,
+                                LLVM::ModuleTranslation &moduleTranslation,
+                                mlir::Value &linearVar, Operation &op) {
     if (llvm::AllocaInst *linearVarAlloca = dyn_cast<llvm::AllocaInst>(
             moduleTranslation.lookupValue(linearVar))) {
       linearPreconditionVars.push_back(builder.CreateAlloca(
@@ -159,7 +159,12 @@ class LinearClauseProcessor {
       linearOrigVal.push_back(moduleTranslation.lookupValue(linearVar));
       linearLoopBodyTemps.push_back(linearLoopBodyTemp);
       linearOrigVars.push_back(linearVarAlloca);
+      return success();
     }
+
+    else
+      return op.emitError() << "not yet implemented: linear clause support"
+                            << " for non alloca linear variables";
   }
 
   // Initialize linear step
@@ -169,20 +174,15 @@ class LinearClauseProcessor {
   }
 
   // Emit IR for initialization of linear variables
-  llvm::OpenMPIRBuilder::InsertPointOrErrorTy
-  initLinearVar(llvm::IRBuilderBase &builder,
-                LLVM::ModuleTranslation &moduleTranslation,
-                llvm::BasicBlock *loopPreHeader) {
+  void initLinearVar(llvm::IRBuilderBase &builder,
+                     LLVM::ModuleTranslation &moduleTranslation,
+                     llvm::BasicBlock *loopPreHeader) {
     builder.SetInsertPoint(loopPreHeader->getTerminator());
     for (size_t index = 0; index < linearOrigVars.size(); index++) {
       llvm::LoadInst *linearVarLoad = builder.CreateLoad(
           linearOrigVars[index]->getAllocatedType(), linearOrigVars[index]);
       builder.CreateStore(linearVarLoad, linearPreconditionVars[index]);
     }
-    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
-        moduleTranslation.getOpenMPBuilder()->createBarrier(
-            builder.saveIP(), llvm::omp::OMPD_barrier);
-    return afterBarrierIP;
   }
 
   // Emit IR for updating Linear variables
@@ -193,18 +193,27 @@ class LinearClauseProcessor {
       // Emit increments for linear vars
       llvm::LoadInst *linearVarStart =
           builder.CreateLoad(linearOrigVars[index]->getAllocatedType(),
-
                              linearPreconditionVars[index]);
+
       auto mulInst = builder.CreateMul(loopInductionVar, linearSteps[index]);
-      auto addInst = builder.CreateAdd(linearVarStart, mulInst);
-      builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+      if (linearOrigVars[index]->getAllocatedType()->isIntegerTy()) {
+        auto addInst = builder.CreateAdd(linearVarStart, mulInst);
+        builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+      } else if (linearOrigVars[index]
+                     ->getAllocatedType()
+                     ->isFloatingPointTy()) {
+        auto cvt = builder.CreateSIToFP(
+            mulInst, linearOrigVars[index]->getAllocatedType());
+        auto addInst = builder.CreateFAdd(linearVarStart, cvt);
+        builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+      }
     }
   }
 
   // Linear variable finalization is conditional on the last logical iteration.
   // Create BB splits to manage the same.
-  void outlineLinearFinalizationBB(llvm::IRBuilderBase &builder,
-                                   llvm::BasicBlock *loopExit) {
+  void splitLinearFiniBB(llvm::IRBuilderBase &builder,
+                         llvm::BasicBlock *loopExit) {
     linearFinalizationBB = loopExit->splitBasicBlock(
         loopExit->getTerminator(), "omp_loop.linear_finalization");
     linearExitBB = linearFinalizationBB->splitBasicBlock(
@@ -339,10 +348,6 @@ static LogicalResult checkImplementationStatus(Operation &op) {
     if (!op.getIsDevicePtrVars().empty())
       result = todo("is_device_ptr");
   };
-  auto checkLinear = [&todo](auto op, LogicalResult &result) {
-    if (!op.getLinearVars().empty() || !op.getLinearStepVars().empty())
-      result = todo("linear");
-  };
   auto checkNowait = [&todo](auto op, LogicalResult &result) {
     if (op.getNowait())
       result = todo("nowait");
@@ -432,7 +437,6 @@ static LogicalResult checkImplementationStatus(Operation &op) {
       })
       .Case([&](omp::WsloopOp op) {
         checkAllocate(op, result);
-        checkLinear(op, result);
         checkOrder(op, result);
         checkReduction(op, result);
       })
@@ -440,10 +444,7 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         checkAllocate(op, result);
         checkReduction(op, result);
       })
-      .Case([&](omp::SimdOp op) {
-        checkLinear(op, result);
-        checkReduction(op, result);
-      })
+      .Case([&](omp::SimdOp op) { checkReduction(op, result); })
       .Case<omp::AtomicReadOp, omp::AtomicWriteOp, omp::AtomicUpdateOp,
             omp::AtomicCaptureOp>([&](auto op) { checkHint(op, result); })
       .Case<omp::TargetEnterDataOp, omp::TargetExitDataOp, omp::TargetUpdateOp>(
@@ -2587,13 +2588,13 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
 
   // Initialize linear variables and linear step
   LinearClauseProcessor linearClauseProcessor;
-  if (wsloopOp.getLinearVars().size()) {
-    for (mlir::Value linearVar : wsloopOp.getLinearVars())
-      linearClauseProcessor.createLinearVar(builder, moduleTranslation,
-                                            linearVar);
-    for (mlir::Value linearStep : wsloopOp.getLinearStepVars())
-      linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
+  for (mlir::Value linearVar : wsloopOp.getLinearVars()) {
+    if (failed(linearClauseProcessor.createLinearVar(builder, moduleTranslation,
+                                                     linearVar, opInst)))
+      return failure();
   }
+  for (mlir::Value linearStep : wsloopOp.getLinearStepVars())
+    linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
 
   llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
       wsloopOp.getRegion(), "omp.wsloop.region", builder, moduleTranslation);
@@ -2605,16 +2606,17 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
 
   // Emit Initialization and Update IR for linear variables
   if (wsloopOp.getLinearVars().size()) {
+    linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+                                        loopInfo->getPreheader());
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
-        linearClauseProcessor.initLinearVar(builder, moduleTranslation,
-                                            loopInfo->getPreheader());
+        moduleTranslation.getOpenMPBuilder()->createBarrier(
+            builder.saveIP(), llvm::omp::OMPD_barrier);
     if (failed(handleError(afterBarrierIP, *loopOp)))
       return failure();
     builder.restoreIP(*afterBarrierIP);
     linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
                                           loopInfo->getIndVar());
-    linearClauseProcessor.outlineLinearFinalizationBB(builder,
-                                                      loopInfo->getExit());
+    linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
   }
 
   builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
@@ -2882,6 +2884,17 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
       findAllocaInsertPoint(builder, moduleTranslation);
 
+  // Create linear variables and initialize linear step
+  LinearClauseProcessor linearClauseProcessor;
+
+  for (mlir::Value linearVar : simdOp.getLinearVars()) {
+    if (failed(linearClauseProcessor.createLinearVar(builder, moduleTranslation,
+                                                     linearVar, opInst)))
+      return failure();
+  }
+  for (mlir::Value linearStep : simdOp.getLinearStepVars())
+    linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
+
   llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
       builder, moduleTranslation, privateVarsInfo, allocaIP);
   if (handleError(afterAllocas, opInst).failed())
@@ -2945,14 +2958,27 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
   if (failed(handleError(regionBlock, opInst)))
     return failure();
 
-  builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
   llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
+
+  // Emit Initialization for linear variables
+  if (simdOp.getLinearVars().size()) {
+    linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+                                        loopInfo->getPreheader());
+    linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
+                                          loopInfo->getIndVar());
+  }
+  builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
+
   ompBuilder->applySimd(loopInfo, alignedVars,
                         simdOp.getIfExpr()
                             ? moduleTranslation.lookupValue(simdOp.getIfExpr())
                             : nullptr,
                         order, simdlen, safelen);
 
+  for (size_t index = 0; index < simdOp.getLinearVars().size(); index++)
+    linearClauseProcessor.rewriteInPlace(builder, "omp.loop_nest.region",
+                                         index);
+
   // We now need to reduce the per-simd-lane reduction variable into the
   // original variable. This works a bit differently to other reductions (e.g.
   // wsloop) because we don't need to call into the OpenMP runtime to handle

>From 29ba59e6fe9e5db346ecbbf4e311d1bf69df8412 Mon Sep 17 00:00:00 2001
From: Nimish Mishra <neelam.nimish at gmail.com>
Date: Wed, 9 Jul 2025 18:11:42 +0530
Subject: [PATCH 2/3] Tests

---
 flang/test/Lower/OpenMP/wsloop-linear.f90 |  57 ++++++++++
 mlir/test/Target/LLVMIR/openmp-llvm.mlir  | 133 ++++++++++++++++++++++
 mlir/test/Target/LLVMIR/openmp-todo.mlir  |  14 +--
 3 files changed, 191 insertions(+), 13 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/wsloop-linear.f90

diff --git a/flang/test/Lower/OpenMP/wsloop-linear.f90 b/flang/test/Lower/OpenMP/wsloop-linear.f90
new file mode 100644
index 0000000000000..b99677108be2f
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-linear.f90
@@ -0,0 +1,57 @@
+! This test checks lowering of OpenMP DO Directive (Worksharing)
+! with linear clause
+
+! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - 2>&1 | FileCheck %s
+
+!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_linearEx"}
+!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFsimple_linearEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[const:.*]] = arith.constant 1 : i32
+subroutine simple_linear
+    implicit none
+    integer :: x, y, i
+    !CHECK: omp.wsloop linear(%[[X]]#0 = %[[const]] : !fir.ref<i32>) {{.*}}
+    !$omp do linear(x)
+    !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
+    !CHECK: %[[const:.*]] = arith.constant 2 : i32
+    !CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32
+    do i = 1, 10
+        y = x + 2
+    end do
+    !$omp end do
+end subroutine
+
+
+!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFlinear_stepEx"}
+!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFlinear_stepEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+subroutine linear_step
+    implicit none
+    integer :: x, y, i
+    !CHECK: %[[const:.*]] = arith.constant 4 : i32
+    !CHECK: omp.wsloop linear(%[[X]]#0 = %[[const]] : !fir.ref<i32>) {{.*}}
+    !$omp do linear(x:4)
+    !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
+    !CHECK: %[[const:.*]] = arith.constant 2 : i32
+    !CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32   
+    do i = 1, 10
+        y = x + 2
+    end do
+    !$omp end do
+end subroutine
+
+!CHECK: %[[A_alloca:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFlinear_exprEa"}
+!CHECK: %[[A:.*]]:2 = hlfir.declare %[[A_alloca]] {uniq_name = "_QFlinear_exprEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFlinear_exprEx"}
+!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFlinear_exprEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+subroutine linear_expr
+    implicit none
+    integer :: x, y, i, a
+    !CHECK: %[[LOAD_A:.*]] = fir.load %[[A]]#0 : !fir.ref<i32>
+    !CHECK: %[[const:.*]] = arith.constant 4 : i32
+    !CHECK: %[[LINEAR_EXPR:.*]] = arith.addi %[[LOAD_A]], %[[const]] : i32
+    !CHECK: omp.wsloop linear(%[[X]]#0 = %[[LINEAR_EXPR]] : !fir.ref<i32>) {{.*}}
+    !$omp do linear(x:a+4)
+    do i = 1, 10
+        y = x + 2
+    end do
+    !$omp end do
+end subroutine
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 32f0ba5b105ff..583356e81a978 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -358,6 +358,97 @@ llvm.func @wsloop_simple(%arg0: !llvm.ptr) {
 
 // -----
 
+// CHECK-LABEL: wsloop_linear
+
+// CHECK: {{.*}} = alloca i32, i64 1, align 4
+// CHECK: %[[Y:.*]] = alloca i32, i64 1, align 4
+// CHECK: %[[X:.*]] = alloca i32, i64 1, align 4
+
+// CHECK: entry:
+// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
+// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
+// CHECK: br label %omp_loop.preheader
+
+
+// CHECK: omp_loop.preheader:
+// CHECK: %[[LOAD:.*]] = load i32, ptr %[[X]], align 4
+// CHECK: store i32 %[[LOAD]], ptr %[[LINEAR_VAR]], align 4
+// CHECK: %omp_global_thread_num = call i32 @__kmpc_global_thread_num(ptr @2)
+// CHECK: call void @__kmpc_barrier(ptr @1, i32 %omp_global_thread_num)
+
+// CHECK: omp_loop.body:
+// CHECK: %[[LOOP_IV:.*]] = add i32 %omp_loop.iv, {{.*}}
+// CHECK: %[[LINEAR_LOAD:.*]] = load i32, ptr %[[LINEAR_VAR]], align 4
+// CHECK: %[[MUL:.*]] = mul i32 %[[LOOP_IV]], 1
+// CHECK: %[[ADD:.*]] = add i32 %[[LINEAR_LOAD]], %[[MUL]]
+// CHECK: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]], align 4
+// CHECK: br label %omp.loop_nest.region
+
+// CHECK: omp.loop_nest.region:
+// CHECK: %[[LINEAR_LOAD:.*]] = load i32, ptr %[[LINEAR_RESULT]], align 4
+// CHECK: %[[ADD:.*]] = add i32 %[[LINEAR_LOAD]], 2
+// CHECK: store i32 %[[ADD]], ptr %[[Y]], align 4
+
+// CHECK: omp_loop.exit:
+// CHECK: call void @__kmpc_for_static_fini(ptr @2, i32 %omp_global_thread_num4)
+// CHECK: %omp_global_thread_num5 = call i32 @__kmpc_global_thread_num(ptr @2)
+// CHECK: call void @__kmpc_barrier(ptr @3, i32 %omp_global_thread_num5)
+// CHECK: br label %omp_loop.linear_finalization
+
+
+// CHECK: omp_loop.linear_finalization:
+// CHECK: %[[LAST_ITER:.*]] = load i32, ptr %p.lastiter, align 4
+// CHECK: %[[CMP:.*]] = icmp ne i32 %[[LAST_ITER]], 0
+// CHECK: br i1 %[[CMP]], label %omp_loop.linear_lastiter_exit, label %omp_loop.linear_exit
+
+// CHECK: omp_loop.linear_lastiter_exit:
+// CHECK: %[[LINEAR_RESULT_LOAD:.*]] = load i32, ptr %[[LINEAR_RESULT]], align 4
+// CHECK: store i32 %[[LINEAR_RESULT_LOAD]], ptr %[[X]], align 4
+// CHECK: br label %omp_loop.linear_exit
+
+
+// CHECK: omp_loop.linear_exit:
+// CHECK: %omp_global_thread_num6 = call i32 @__kmpc_global_thread_num(ptr @2)
+// CHECK: call void @__kmpc_barrier(ptr @1, i32 %omp_global_thread_num6)
+// CHECK: br label %omp_loop.after
+
+llvm.func @wsloop_linear() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
+  %2 = llvm.mlir.constant(1 : i64) : i64
+  %3 = llvm.alloca %2 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+  %4 = llvm.mlir.constant(1 : i64) : i64
+  %5 = llvm.alloca %4 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %6 = llvm.mlir.constant(1 : i64) : i64
+  %7 = llvm.alloca %6 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  %8 = llvm.mlir.constant(2 : i32) : i32
+  %9 = llvm.mlir.constant(10 : i32) : i32
+  %10 = llvm.mlir.constant(1 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i64) : i64
+  %12 = llvm.mlir.constant(1 : i64) : i64
+  %13 = llvm.mlir.constant(1 : i64) : i64
+  %14 = llvm.mlir.constant(1 : i64) : i64
+  omp.wsloop linear(%5 = %10 : !llvm.ptr) {
+    omp.loop_nest (%arg0) : i32 = (%10) to (%9) inclusive step (%10) {
+      llvm.store %arg0, %1 : i32, !llvm.ptr
+      %15 = llvm.load %5 : !llvm.ptr -> i32
+      %16 = llvm.add %15, %8 : i32
+      llvm.store %16, %3 : i32, !llvm.ptr
+      %17 = llvm.add %arg0, %10 : i32
+      %18 = llvm.icmp "sgt" %17, %9 : i32
+      llvm.cond_br %18, ^bb1, ^bb2
+    ^bb1:  // pred: ^bb0
+      llvm.store %17, %1 : i32, !llvm.ptr
+      llvm.br ^bb2
+    ^bb2:  // 2 preds: ^bb0, ^bb1
+      omp.yield
+     }
+   }
+  llvm.return
+}
+
+// -----
+
 // CHECK-LABEL: @wsloop_inclusive_1
 llvm.func @wsloop_inclusive_1(%arg0: !llvm.ptr) {
   %0 = llvm.mlir.constant(42 : index) : i64
@@ -695,6 +786,48 @@ llvm.func @simd_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) {
 
 // -----
 
+// CHECK-LABEL: @simd_linear
+
+// CHECK: entry:
+// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
+// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
+// CHECK: br label %omp.simd.region
+
+// CHECK: omp.simd.region:
+// CHECK: br label %omp_loop.preheader
+
+// CHECK: omp_loop.preheader:
+// CHECK: %[[ORIG_VAL:.*]] = load i32, ptr {{.*}}, align 4
+// CHECK: store i32 %[[ORIG_VAL]], ptr %[[LINEAR_VAR]], align 4
+// CHECK: br label %omp_loop.header
+
+// CHECK: omp_loop.body:
+// CHECK: %[[LINEAR_LOAD:.*]] = load i32, ptr %[[LINEAR_VAR]], align 4
+// CHECK: %[[MUL:.*]] = mul i32 %omp_loop.iv, 2
+// CHECK: %[[ADD:.*]] = add i32 %[[LINEAR_LOAD]], %[[MUL]]
+// CHECK: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]], align 4
+// CHECK: br label %omp.loop_nest.region
+
+llvm.func @simd_linear() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %2 = llvm.mlir.constant(1 : i64) : i64
+  %3 = llvm.alloca %2 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  %4 = llvm.mlir.constant(10 : i32) : i32
+  %5 = llvm.mlir.constant(1 : i32) : i32
+  %6 = llvm.mlir.constant(2 : i32) : i32
+  %7 = llvm.mlir.constant(1 : i64) : i64
+  %8 = llvm.mlir.constant(1 : i64) : i64
+  omp.simd linear(%1 = %6 : !llvm.ptr) {
+    omp.loop_nest (%arg1) : i32 = (%5) to (%4) inclusive step (%5) {
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 // CHECK-LABEL: @simd_simple_multiple
 llvm.func @simd_simple_multiple(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) {
   omp.simd {
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index e2c32b254c200..f3a61695b7401 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -117,7 +117,7 @@ llvm.func @sections_private(%x : !llvm.ptr) {
 // -----
 
 llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{not yet implemented: Unhandled clause linear in omp.simd operation}}
+  // expected-error at below {{not yet implemented: linear clause support for non alloca linear variables}}
   // expected-error at below {{LLVM Translation failed for operation: omp.simd}}
   omp.simd linear(%x = %step : !llvm.ptr) {
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
@@ -467,18 +467,6 @@ llvm.func @wsloop_allocate(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
 
 // -----
 
-llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{not yet implemented: Unhandled clause linear in omp.wsloop operation}}
-  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop linear(%x = %step : !llvm.ptr) {
-    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
-      omp.yield
-    }
-  }
-  llvm.return
-}
-
-// -----
 llvm.func @wsloop_order(%lb : i32, %ub : i32, %step : i32) {
   // expected-error at below {{not yet implemented: Unhandled clause order in omp.wsloop operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}

>From 39ca84043a543b64f47b0e0d2282c8bd715e12df Mon Sep 17 00:00:00 2001
From: Nimish Mishra <neelam.nimish at gmail.com>
Date: Wed, 9 Jul 2025 18:17:35 +0530
Subject: [PATCH 3/3] Add more tests

---
 .../Lower/OpenMP/Todo/omp-do-simd-linear.f90  | 14 -----
 flang/test/Lower/OpenMP/simd-linear.f90       | 54 +++++++++++++++++++
 2 files changed, 54 insertions(+), 14 deletions(-)
 delete mode 100644 flang/test/Lower/OpenMP/Todo/omp-do-simd-linear.f90
 create mode 100644 flang/test/Lower/OpenMP/simd-linear.f90

diff --git a/flang/test/Lower/OpenMP/Todo/omp-do-simd-linear.f90 b/flang/test/Lower/OpenMP/Todo/omp-do-simd-linear.f90
deleted file mode 100644
index 4caf12a0169c4..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/omp-do-simd-linear.f90
+++ /dev/null
@@ -1,14 +0,0 @@
-! This test checks lowering of OpenMP do simd linear() pragma
-
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-subroutine testDoSimdLinear(int_array)
-        integer :: int_array(*)
-!CHECK: not yet implemented: Unhandled clause LINEAR in SIMD construct
-!$omp do simd linear(int_array)
-        do index_ = 1, 10
-        end do
-!$omp end do simd
-
-end subroutine testDoSimdLinear
-
diff --git a/flang/test/Lower/OpenMP/simd-linear.f90 b/flang/test/Lower/OpenMP/simd-linear.f90
new file mode 100644
index 0000000000000..ccb5fbb8b0a24
--- /dev/null
+++ b/flang/test/Lower/OpenMP/simd-linear.f90
@@ -0,0 +1,54 @@
+! This test checks lowering of OpenMP DO Directive (Worksharing)
+! with linear clause
+
+! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - 2>&1 | FileCheck %s
+
+!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_linearEx"}
+!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFsimple_linearEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[const:.*]] = arith.constant 1 : i32
+subroutine simple_linear
+    implicit none
+    integer :: x, y, i
+    !CHECK: omp.simd linear(%[[X]]#0 = %[[const]] : !fir.ref<i32>) {{.*}}
+    !$omp simd linear(x)
+    !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
+    !CHECK: %[[const:.*]] = arith.constant 2 : i32
+    !CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32
+    do i = 1, 10
+        y = x + 2
+    end do
+end subroutine
+
+
+!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFlinear_stepEx"}
+!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFlinear_stepEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+subroutine linear_step
+    implicit none
+    integer :: x, y, i
+    !CHECK: %[[const:.*]] = arith.constant 4 : i32
+    !CHECK: omp.simd linear(%[[X]]#0 = %[[const]] : !fir.ref<i32>) {{.*}}
+    !$omp simd linear(x:4)
+    !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
+    !CHECK: %[[const:.*]] = arith.constant 2 : i32
+    !CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32   
+    do i = 1, 10
+        y = x + 2
+    end do
+end subroutine
+
+!CHECK: %[[A_alloca:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFlinear_exprEa"}
+!CHECK: %[[A:.*]]:2 = hlfir.declare %[[A_alloca]] {uniq_name = "_QFlinear_exprEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFlinear_exprEx"}
+!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFlinear_exprEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+subroutine linear_expr
+    implicit none
+    integer :: x, y, i, a
+    !CHECK: %[[LOAD_A:.*]] = fir.load %[[A]]#0 : !fir.ref<i32>
+    !CHECK: %[[const:.*]] = arith.constant 4 : i32
+    !CHECK: %[[LINEAR_EXPR:.*]] = arith.addi %[[LOAD_A]], %[[const]] : i32
+    !CHECK: omp.simd linear(%[[X]]#0 = %[[LINEAR_EXPR]] : !fir.ref<i32>) {{.*}}
+    !$omp simd linear(x:a+4)
+    do i = 1, 10
+        y = x + 2
+    end do
+end subroutine


From flang-commits at lists.llvm.org  Wed Jul  9 05:50:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:50:02 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e657a.170a0220.25ad8.aca4@mx.google.com>


https://github.com/NimishMishra edited https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 05:50:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:50:02 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e657a.050a0220.211dc7.c0cf@mx.google.com>


================
@@ -2423,15 +2562,40 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
                            llvm::omp::Directive::OMPD_for);
 
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+
+  // Initialize linear variables and linear step
+  LinearClauseProcessor linearClauseProcessor;
+  if (wsloopOp.getLinearVars().size()) {
----------------
NimishMishra wrote:

Done

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 05:50:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:50:02 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e657a.170a0220.160dc2.330e@mx.google.com>


================
@@ -124,6 +124,146 @@ class PreviouslyReportedError
 
 char PreviouslyReportedError::ID = 0;
 
+/*
+ * Custom class for processing linear clause for omp.wsloop
+ * and omp.simd. Linear clause translation requires setup,
+ * initialization, update, and finalization at varying
+ * basic blocks in the IR. This class helps maintain
+ * internal state to allow consistent translation in
+ * each of these stages.
+ */
+
+class LinearClauseProcessor {
+
+private:
+  SmallVector<llvm::Value *> linearPreconditionVars;
+  SmallVector<llvm::Value *> linearLoopBodyTemps;
+  SmallVector<llvm::AllocaInst *> linearOrigVars;
+  SmallVector<llvm::Value *> linearOrigVal;
+  SmallVector<llvm::Value *> linearSteps;
+  llvm::BasicBlock *linearFinalizationBB;
+  llvm::BasicBlock *linearExitBB;
+  llvm::BasicBlock *linearLastIterExitBB;
+
+public:
+  // Allocate space for linear variabes
+  void createLinearVar(llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation,
+                       mlir::Value &linearVar) {
+    if (llvm::AllocaInst *linearVarAlloca = dyn_cast<llvm::AllocaInst>(
+            moduleTranslation.lookupValue(linearVar))) {
+      linearPreconditionVars.push_back(builder.CreateAlloca(
+          linearVarAlloca->getAllocatedType(), nullptr, ".linear_var"));
+      llvm::Value *linearLoopBodyTemp = builder.CreateAlloca(
+          linearVarAlloca->getAllocatedType(), nullptr, ".linear_result");
+      linearOrigVal.push_back(moduleTranslation.lookupValue(linearVar));
+      linearLoopBodyTemps.push_back(linearLoopBodyTemp);
+      linearOrigVars.push_back(linearVarAlloca);
+    }
+  }
+
+  // Initialize linear step
+  inline void initLinearStep(LLVM::ModuleTranslation &moduleTranslation,
+                             mlir::Value &linearStep) {
+    linearSteps.push_back(moduleTranslation.lookupValue(linearStep));
+  }
+
+  // Emit IR for initialization of linear variables
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy
+  initLinearVar(llvm::IRBuilderBase &builder,
+                LLVM::ModuleTranslation &moduleTranslation,
+                llvm::BasicBlock *loopPreHeader) {
+    builder.SetInsertPoint(loopPreHeader->getTerminator());
+    for (size_t index = 0; index < linearOrigVars.size(); index++) {
+      llvm::LoadInst *linearVarLoad = builder.CreateLoad(
+          linearOrigVars[index]->getAllocatedType(), linearOrigVars[index]);
+      builder.CreateStore(linearVarLoad, linearPreconditionVars[index]);
+    }
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+        moduleTranslation.getOpenMPBuilder()->createBarrier(
+            builder.saveIP(), llvm::omp::OMPD_barrier);
+    return afterBarrierIP;
+  }
+
+  // Emit IR for updating Linear variables
+  void updateLinearVar(llvm::IRBuilderBase &builder, llvm::BasicBlock *loopBody,
+                       llvm::Value *loopInductionVar) {
+    builder.SetInsertPoint(loopBody->getTerminator());
+    for (size_t index = 0; index < linearPreconditionVars.size(); index++) {
+      // Emit increments for linear vars
+      llvm::LoadInst *linearVarStart =
+          builder.CreateLoad(linearOrigVars[index]->getAllocatedType(),
+
+                             linearPreconditionVars[index]);
+      auto mulInst = builder.CreateMul(loopInductionVar, linearSteps[index]);
+      auto addInst = builder.CreateAdd(linearVarStart, mulInst);
+      builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+    }
+  }
+
+  // Linear variable finalization is conditional on the last logical iteration.
+  // Create BB splits to manage the same.
+  void outlineLinearFinalizationBB(llvm::IRBuilderBase &builder,
+                                   llvm::BasicBlock *loopExit) {
+    linearFinalizationBB = loopExit->splitBasicBlock(
+        loopExit->getTerminator(), "omp_loop.linear_finalization");
+    linearExitBB = linearFinalizationBB->splitBasicBlock(
+        linearFinalizationBB->getTerminator(), "omp_loop.linear_exit");
+    linearLastIterExitBB = linearFinalizationBB->splitBasicBlock(
+        linearFinalizationBB->getTerminator(), "omp_loop.linear_lastiter_exit");
+  }
+
+  // Finalize the linear vars
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy
+  finalizeLinearVar(llvm::IRBuilderBase &builder,
+                    LLVM::ModuleTranslation &moduleTranslation,
+                    llvm::Value *lastIter) {
+    // Emit condition to check whether last logical iteration is being executed
+    builder.SetInsertPoint(linearFinalizationBB->getTerminator());
+    llvm::Value *loopLastIterLoad = builder.CreateLoad(
+        llvm::Type::getInt32Ty(builder.getContext()), lastIter);
----------------
NimishMishra wrote:

I checked the workshare loop allocas, %p.lastiter is `alloca i32, align 4`. So we can use a `load i32` too. Would that be ok?

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 05:50:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:50:02 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e657a.a70a0220.15ec24.c9b6@mx.google.com>


================
@@ -124,6 +124,146 @@ class PreviouslyReportedError
 
 char PreviouslyReportedError::ID = 0;
 
+/*
+ * Custom class for processing linear clause for omp.wsloop
+ * and omp.simd. Linear clause translation requires setup,
+ * initialization, update, and finalization at varying
+ * basic blocks in the IR. This class helps maintain
+ * internal state to allow consistent translation in
+ * each of these stages.
+ */
+
+class LinearClauseProcessor {
+
+private:
+  SmallVector<llvm::Value *> linearPreconditionVars;
+  SmallVector<llvm::Value *> linearLoopBodyTemps;
+  SmallVector<llvm::AllocaInst *> linearOrigVars;
+  SmallVector<llvm::Value *> linearOrigVal;
+  SmallVector<llvm::Value *> linearSteps;
+  llvm::BasicBlock *linearFinalizationBB;
+  llvm::BasicBlock *linearExitBB;
+  llvm::BasicBlock *linearLastIterExitBB;
+
+public:
+  // Allocate space for linear variabes
+  void createLinearVar(llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation,
+                       mlir::Value &linearVar) {
+    if (llvm::AllocaInst *linearVarAlloca = dyn_cast<llvm::AllocaInst>(
+            moduleTranslation.lookupValue(linearVar))) {
+      linearPreconditionVars.push_back(builder.CreateAlloca(
+          linearVarAlloca->getAllocatedType(), nullptr, ".linear_var"));
+      llvm::Value *linearLoopBodyTemp = builder.CreateAlloca(
+          linearVarAlloca->getAllocatedType(), nullptr, ".linear_result");
+      linearOrigVal.push_back(moduleTranslation.lookupValue(linearVar));
+      linearLoopBodyTemps.push_back(linearLoopBodyTemp);
+      linearOrigVars.push_back(linearVarAlloca);
+    }
+  }
+
+  // Initialize linear step
+  inline void initLinearStep(LLVM::ModuleTranslation &moduleTranslation,
+                             mlir::Value &linearStep) {
+    linearSteps.push_back(moduleTranslation.lookupValue(linearStep));
+  }
+
+  // Emit IR for initialization of linear variables
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy
+  initLinearVar(llvm::IRBuilderBase &builder,
+                LLVM::ModuleTranslation &moduleTranslation,
+                llvm::BasicBlock *loopPreHeader) {
+    builder.SetInsertPoint(loopPreHeader->getTerminator());
+    for (size_t index = 0; index < linearOrigVars.size(); index++) {
+      llvm::LoadInst *linearVarLoad = builder.CreateLoad(
+          linearOrigVars[index]->getAllocatedType(), linearOrigVars[index]);
+      builder.CreateStore(linearVarLoad, linearPreconditionVars[index]);
+    }
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+        moduleTranslation.getOpenMPBuilder()->createBarrier(
+            builder.saveIP(), llvm::omp::OMPD_barrier);
+    return afterBarrierIP;
+  }
+
+  // Emit IR for updating Linear variables
+  void updateLinearVar(llvm::IRBuilderBase &builder, llvm::BasicBlock *loopBody,
+                       llvm::Value *loopInductionVar) {
+    builder.SetInsertPoint(loopBody->getTerminator());
+    for (size_t index = 0; index < linearPreconditionVars.size(); index++) {
+      // Emit increments for linear vars
+      llvm::LoadInst *linearVarStart =
+          builder.CreateLoad(linearOrigVars[index]->getAllocatedType(),
+
+                             linearPreconditionVars[index]);
+      auto mulInst = builder.CreateMul(loopInductionVar, linearSteps[index]);
+      auto addInst = builder.CreateAdd(linearVarStart, mulInst);
+      builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+    }
+  }
+
+  // Linear variable finalization is conditional on the last logical iteration.
+  // Create BB splits to manage the same.
+  void outlineLinearFinalizationBB(llvm::IRBuilderBase &builder,
----------------
NimishMishra wrote:

Done.

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 05:50:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:50:02 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e657a.050a0220.2ac4a2.df07@mx.google.com>


================
@@ -124,6 +124,146 @@ class PreviouslyReportedError
 
 char PreviouslyReportedError::ID = 0;
 
+/*
+ * Custom class for processing linear clause for omp.wsloop
+ * and omp.simd. Linear clause translation requires setup,
+ * initialization, update, and finalization at varying
+ * basic blocks in the IR. This class helps maintain
+ * internal state to allow consistent translation in
+ * each of these stages.
+ */
+
+class LinearClauseProcessor {
+
+private:
+  SmallVector<llvm::Value *> linearPreconditionVars;
+  SmallVector<llvm::Value *> linearLoopBodyTemps;
+  SmallVector<llvm::AllocaInst *> linearOrigVars;
+  SmallVector<llvm::Value *> linearOrigVal;
+  SmallVector<llvm::Value *> linearSteps;
+  llvm::BasicBlock *linearFinalizationBB;
+  llvm::BasicBlock *linearExitBB;
+  llvm::BasicBlock *linearLastIterExitBB;
+
+public:
+  // Allocate space for linear variabes
+  void createLinearVar(llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation,
+                       mlir::Value &linearVar) {
+    if (llvm::AllocaInst *linearVarAlloca = dyn_cast<llvm::AllocaInst>(
+            moduleTranslation.lookupValue(linearVar))) {
+      linearPreconditionVars.push_back(builder.CreateAlloca(
+          linearVarAlloca->getAllocatedType(), nullptr, ".linear_var"));
+      llvm::Value *linearLoopBodyTemp = builder.CreateAlloca(
+          linearVarAlloca->getAllocatedType(), nullptr, ".linear_result");
+      linearOrigVal.push_back(moduleTranslation.lookupValue(linearVar));
+      linearLoopBodyTemps.push_back(linearLoopBodyTemp);
+      linearOrigVars.push_back(linearVarAlloca);
+    }
----------------
NimishMishra wrote:

I have added a failure to catch cases where the linear variable is non-alloca.

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 05:50:04 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:50:04 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e657c.050a0220.16e31f.d064@mx.google.com>


================
@@ -124,6 +124,146 @@ class PreviouslyReportedError
 
 char PreviouslyReportedError::ID = 0;
 
+/*
+ * Custom class for processing linear clause for omp.wsloop
+ * and omp.simd. Linear clause translation requires setup,
+ * initialization, update, and finalization at varying
+ * basic blocks in the IR. This class helps maintain
+ * internal state to allow consistent translation in
+ * each of these stages.
+ */
+
+class LinearClauseProcessor {
+
+private:
+  SmallVector<llvm::Value *> linearPreconditionVars;
+  SmallVector<llvm::Value *> linearLoopBodyTemps;
+  SmallVector<llvm::AllocaInst *> linearOrigVars;
+  SmallVector<llvm::Value *> linearOrigVal;
+  SmallVector<llvm::Value *> linearSteps;
+  llvm::BasicBlock *linearFinalizationBB;
+  llvm::BasicBlock *linearExitBB;
+  llvm::BasicBlock *linearLastIterExitBB;
+
+public:
+  // Allocate space for linear variabes
+  void createLinearVar(llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation,
+                       mlir::Value &linearVar) {
+    if (llvm::AllocaInst *linearVarAlloca = dyn_cast<llvm::AllocaInst>(
+            moduleTranslation.lookupValue(linearVar))) {
+      linearPreconditionVars.push_back(builder.CreateAlloca(
+          linearVarAlloca->getAllocatedType(), nullptr, ".linear_var"));
+      llvm::Value *linearLoopBodyTemp = builder.CreateAlloca(
+          linearVarAlloca->getAllocatedType(), nullptr, ".linear_result");
+      linearOrigVal.push_back(moduleTranslation.lookupValue(linearVar));
+      linearLoopBodyTemps.push_back(linearLoopBodyTemp);
+      linearOrigVars.push_back(linearVarAlloca);
+    }
+  }
+
+  // Initialize linear step
+  inline void initLinearStep(LLVM::ModuleTranslation &moduleTranslation,
+                             mlir::Value &linearStep) {
+    linearSteps.push_back(moduleTranslation.lookupValue(linearStep));
+  }
+
+  // Emit IR for initialization of linear variables
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy
+  initLinearVar(llvm::IRBuilderBase &builder,
+                LLVM::ModuleTranslation &moduleTranslation,
+                llvm::BasicBlock *loopPreHeader) {
+    builder.SetInsertPoint(loopPreHeader->getTerminator());
+    for (size_t index = 0; index < linearOrigVars.size(); index++) {
+      llvm::LoadInst *linearVarLoad = builder.CreateLoad(
+          linearOrigVars[index]->getAllocatedType(), linearOrigVars[index]);
+      builder.CreateStore(linearVarLoad, linearPreconditionVars[index]);
+    }
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+        moduleTranslation.getOpenMPBuilder()->createBarrier(
+            builder.saveIP(), llvm::omp::OMPD_barrier);
+    return afterBarrierIP;
+  }
+
+  // Emit IR for updating Linear variables
+  void updateLinearVar(llvm::IRBuilderBase &builder, llvm::BasicBlock *loopBody,
+                       llvm::Value *loopInductionVar) {
+    builder.SetInsertPoint(loopBody->getTerminator());
+    for (size_t index = 0; index < linearPreconditionVars.size(); index++) {
+      // Emit increments for linear vars
+      llvm::LoadInst *linearVarStart =
+          builder.CreateLoad(linearOrigVars[index]->getAllocatedType(),
+
+                             linearPreconditionVars[index]);
+      auto mulInst = builder.CreateMul(loopInductionVar, linearSteps[index]);
----------------
NimishMishra wrote:

I have added additional checks for this in `updateLinearVar`

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 05:50:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:50:02 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e657a.170a0220.31a373.29c2@mx.google.com>


https://github.com/NimishMishra commented:

Thanks @tblah for the comments. I have changed the implementation to address your comments as well as support linear clause on simd.

Please note that I am planning to raise another PR for implicit linearization in simd. So I am not planning to merge this PR unless the that other PR gets reviewed and accepted.

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 05:53:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 05:53:01 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop and omp.simd (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e662d.170a0220.15e77.b647@mx.google.com>


https://github.com/NimishMishra edited https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 06:16:28 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Wed, 09 Jul 2025 06:16:28 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Handle multiple spellings
 in OmpDirectiveNameParser (PR #147722)
Message-ID: <llvm.org/llvm/llvm-project/pull/147722@github.com>


https://github.com/kparzysz created https://github.com/llvm/llvm-project/pull/147722

Collect all spellings from all supported OpenMP versions before parsing. Break up the list of spellings by the initial letter to speed up parsing a little.

>From 1419c89670953b969fcc1c474a258c7475d8e7df Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 9 May 2025 07:45:20 -0500
Subject: [PATCH] [flang][OpenMP] Handle multiple spellings in
 OmpDirectiveNameParser

Collect all spellings from all supported OpenMP versions before parsing.
Break up the list of spellings by the initial letter to speed up parsing
a little.
---
 flang/lib/Parser/openmp-parsers.cpp | 49 +++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 30b5d4f9513a7..29701a616d4b0 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
 // OpenMP Directives and Clauses
@@ -104,8 +105,13 @@ struct OmpDirectiveNameParser {
   using Token = TokenStringMatch<false, false>;
 
   std::optional<resultType> Parse(ParseState &state) const {
+    if (state.BytesRemaining() == 0) {
+      return std::nullopt;
+    }
     auto begin{state.GetLocation()};
-    for (const NameWithId &nid : directives()) {
+    char next{static_cast<char>(std::tolower(*begin))};
+
+    for (const NameWithId &nid : directives_starting_with(next)) {
       if (attempt(Token(nid.first.data())).Parse(state)) {
         OmpDirectiveName n;
         n.v = nid.second;
@@ -118,30 +124,47 @@ struct OmpDirectiveNameParser {
 
 private:
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
+  using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<const NameWithId *> directives() const;
-  void initTokens(NameWithId *) const;
+  llvm::iterator_range<ConstIterator>
+  directives_starting_with(char initial) const;
+  void initTokens(std::vector<NameWithId>[]) const;
 };
 
-llvm::iterator_range<const OmpDirectiveNameParser::NameWithId *>
-OmpDirectiveNameParser::directives() const {
-  static NameWithId table[llvm::omp::Directive_enumSize];
+llvm::iterator_range<OmpDirectiveNameParser::ConstIterator>
+OmpDirectiveNameParser::directives_starting_with(char initial) const {
+  static const std::vector<NameWithId> empty{};
+  if (initial < 'a' || initial > 'z') {
+    return llvm::make_range(std::cbegin(empty), std::cend(empty));
+  }
+
+  static std::vector<NameWithId> table['z' - 'a' + 1];
   [[maybe_unused]] static bool init = (initTokens(table), true);
-  return llvm::make_range(std::cbegin(table), std::cend(table));
+
+  int index = initial - 'a';
+  return llvm::make_range(std::cbegin(table[index]), std::cend(table[index]));
 }
 
-void OmpDirectiveNameParser::initTokens(NameWithId *table) const {
+void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   for (size_t i{0}, e{llvm::omp::Directive_enumSize}; i != e; ++i) {
+    llvm::StringSet spellings;
     auto id{static_cast<llvm::omp::Directive>(i)};
-    llvm::StringRef name{
-        llvm::omp::getOpenMPDirectiveName(id, llvm::omp::FallbackVersion)};
-    table[i] = std::make_pair(name.str(), id);
+    for (unsigned version : llvm::omp::getOpenMPVersions()) {
+      spellings.insert(llvm::omp::getOpenMPDirectiveName(id, version));
+    }
+    for (auto &[name, _] : spellings) {
+      char initial{static_cast<char>(std::tolower(name.front()))};
+      table[initial - 'a'].emplace_back(name.str(), id);
+    }
   }
   // Sort the table with respect to the directive name length in a descending
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
-  std::sort(table, table + llvm::omp::Directive_enumSize,
-      [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
+  for (int initial{'a'}; initial != 'z' + 1; ++initial) {
+    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
+      return a.first.size() > b.first.size();
+    });
+  }
 }
 
 // --- Modifier helpers -----------------------------------------------


From flang-commits at lists.llvm.org  Wed Jul  9 06:17:06 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 06:17:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Handle multiple spellings
 in OmpDirectiveNameParser (PR #147722)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147722@github.com>
Message-ID: <686e6bd2.a70a0220.15ec24.cc0b@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: Krzysztof Parzyszek (kparzysz)

<details>
<summary>Changes</summary>

Collect all spellings from all supported OpenMP versions before parsing. Break up the list of spellings by the initial letter to speed up parsing a little.

---
Full diff: https://github.com/llvm/llvm-project/pull/147722.diff


1 Files Affected:

- (modified) flang/lib/Parser/openmp-parsers.cpp (+36-13) 


``````````diff
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 30b5d4f9513a7..29701a616d4b0 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
 // OpenMP Directives and Clauses
@@ -104,8 +105,13 @@ struct OmpDirectiveNameParser {
   using Token = TokenStringMatch<false, false>;
 
   std::optional<resultType> Parse(ParseState &state) const {
+    if (state.BytesRemaining() == 0) {
+      return std::nullopt;
+    }
     auto begin{state.GetLocation()};
-    for (const NameWithId &nid : directives()) {
+    char next{static_cast<char>(std::tolower(*begin))};
+
+    for (const NameWithId &nid : directives_starting_with(next)) {
       if (attempt(Token(nid.first.data())).Parse(state)) {
         OmpDirectiveName n;
         n.v = nid.second;
@@ -118,30 +124,47 @@ struct OmpDirectiveNameParser {
 
 private:
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
+  using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<const NameWithId *> directives() const;
-  void initTokens(NameWithId *) const;
+  llvm::iterator_range<ConstIterator>
+  directives_starting_with(char initial) const;
+  void initTokens(std::vector<NameWithId>[]) const;
 };
 
-llvm::iterator_range<const OmpDirectiveNameParser::NameWithId *>
-OmpDirectiveNameParser::directives() const {
-  static NameWithId table[llvm::omp::Directive_enumSize];
+llvm::iterator_range<OmpDirectiveNameParser::ConstIterator>
+OmpDirectiveNameParser::directives_starting_with(char initial) const {
+  static const std::vector<NameWithId> empty{};
+  if (initial < 'a' || initial > 'z') {
+    return llvm::make_range(std::cbegin(empty), std::cend(empty));
+  }
+
+  static std::vector<NameWithId> table['z' - 'a' + 1];
   [[maybe_unused]] static bool init = (initTokens(table), true);
-  return llvm::make_range(std::cbegin(table), std::cend(table));
+
+  int index = initial - 'a';
+  return llvm::make_range(std::cbegin(table[index]), std::cend(table[index]));
 }
 
-void OmpDirectiveNameParser::initTokens(NameWithId *table) const {
+void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   for (size_t i{0}, e{llvm::omp::Directive_enumSize}; i != e; ++i) {
+    llvm::StringSet spellings;
     auto id{static_cast<llvm::omp::Directive>(i)};
-    llvm::StringRef name{
-        llvm::omp::getOpenMPDirectiveName(id, llvm::omp::FallbackVersion)};
-    table[i] = std::make_pair(name.str(), id);
+    for (unsigned version : llvm::omp::getOpenMPVersions()) {
+      spellings.insert(llvm::omp::getOpenMPDirectiveName(id, version));
+    }
+    for (auto &[name, _] : spellings) {
+      char initial{static_cast<char>(std::tolower(name.front()))};
+      table[initial - 'a'].emplace_back(name.str(), id);
+    }
   }
   // Sort the table with respect to the directive name length in a descending
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
-  std::sort(table, table + llvm::omp::Directive_enumSize,
-      [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
+  for (int initial{'a'}; initial != 'z' + 1; ++initial) {
+    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
+      return a.first.size() > b.first.size();
+    });
+  }
 }
 
 // --- Modifier helpers -----------------------------------------------

``````````

</details>


https://github.com/llvm/llvm-project/pull/147722

From flang-commits at lists.llvm.org  Wed Jul  9 06:18:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 06:18:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Handle multiple spellings
 in OmpDirectiveNameParser (PR #147722)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147722@github.com>
Message-ID: <686e6c36.170a0220.3cfd2e.0717@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions cpp -- flang/lib/Parser/openmp-parsers.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 29701a616..fdf96dffe 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -126,8 +126,8 @@ private:
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
   using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<ConstIterator>
-  directives_starting_with(char initial) const;
+  llvm::iterator_range<ConstIterator> directives_starting_with(
+      char initial) const;
   void initTokens(std::vector<NameWithId>[]) const;
 };
 
@@ -161,9 +161,8 @@ void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
   for (int initial{'a'}; initial != 'z' + 1; ++initial) {
-    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
-      return a.first.size() > b.first.size();
-    });
+    llvm::stable_sort(table[initial - 'a'],
+        [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
   }
 }
 

``````````

</details>


https://github.com/llvm/llvm-project/pull/147722

From flang-commits at lists.llvm.org  Wed Jul  9 06:20:10 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Wed, 09 Jul 2025 06:20:10 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Handle multiple spellings
 in OmpDirectiveNameParser (PR #147722)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147722@github.com>
Message-ID: <686e6c8a.170a0220.379ec7.ad98@mx.google.com>


kparzysz wrote:

Stack:
- https://github.com/llvm/llvm-project/pull/147722 (this PR)
- https://github.com/llvm/llvm-project/pull/147723

https://github.com/llvm/llvm-project/pull/147722

From flang-commits at lists.llvm.org  Wed Jul  9 06:22:47 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Wed, 09 Jul 2025 06:22:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Handle multiple spellings
 in OmpDirectiveNameParser (PR #147722)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147722@github.com>
Message-ID: <686e6d27.050a0220.fafd1.e503@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/147722

>From 1419c89670953b969fcc1c474a258c7475d8e7df Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 9 May 2025 07:45:20 -0500
Subject: [PATCH 1/2] [flang][OpenMP] Handle multiple spellings in
 OmpDirectiveNameParser

Collect all spellings from all supported OpenMP versions before parsing.
Break up the list of spellings by the initial letter to speed up parsing
a little.
---
 flang/lib/Parser/openmp-parsers.cpp | 49 +++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 30b5d4f9513a7..29701a616d4b0 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
 // OpenMP Directives and Clauses
@@ -104,8 +105,13 @@ struct OmpDirectiveNameParser {
   using Token = TokenStringMatch<false, false>;
 
   std::optional<resultType> Parse(ParseState &state) const {
+    if (state.BytesRemaining() == 0) {
+      return std::nullopt;
+    }
     auto begin{state.GetLocation()};
-    for (const NameWithId &nid : directives()) {
+    char next{static_cast<char>(std::tolower(*begin))};
+
+    for (const NameWithId &nid : directives_starting_with(next)) {
       if (attempt(Token(nid.first.data())).Parse(state)) {
         OmpDirectiveName n;
         n.v = nid.second;
@@ -118,30 +124,47 @@ struct OmpDirectiveNameParser {
 
 private:
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
+  using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<const NameWithId *> directives() const;
-  void initTokens(NameWithId *) const;
+  llvm::iterator_range<ConstIterator>
+  directives_starting_with(char initial) const;
+  void initTokens(std::vector<NameWithId>[]) const;
 };
 
-llvm::iterator_range<const OmpDirectiveNameParser::NameWithId *>
-OmpDirectiveNameParser::directives() const {
-  static NameWithId table[llvm::omp::Directive_enumSize];
+llvm::iterator_range<OmpDirectiveNameParser::ConstIterator>
+OmpDirectiveNameParser::directives_starting_with(char initial) const {
+  static const std::vector<NameWithId> empty{};
+  if (initial < 'a' || initial > 'z') {
+    return llvm::make_range(std::cbegin(empty), std::cend(empty));
+  }
+
+  static std::vector<NameWithId> table['z' - 'a' + 1];
   [[maybe_unused]] static bool init = (initTokens(table), true);
-  return llvm::make_range(std::cbegin(table), std::cend(table));
+
+  int index = initial - 'a';
+  return llvm::make_range(std::cbegin(table[index]), std::cend(table[index]));
 }
 
-void OmpDirectiveNameParser::initTokens(NameWithId *table) const {
+void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   for (size_t i{0}, e{llvm::omp::Directive_enumSize}; i != e; ++i) {
+    llvm::StringSet spellings;
     auto id{static_cast<llvm::omp::Directive>(i)};
-    llvm::StringRef name{
-        llvm::omp::getOpenMPDirectiveName(id, llvm::omp::FallbackVersion)};
-    table[i] = std::make_pair(name.str(), id);
+    for (unsigned version : llvm::omp::getOpenMPVersions()) {
+      spellings.insert(llvm::omp::getOpenMPDirectiveName(id, version));
+    }
+    for (auto &[name, _] : spellings) {
+      char initial{static_cast<char>(std::tolower(name.front()))};
+      table[initial - 'a'].emplace_back(name.str(), id);
+    }
   }
   // Sort the table with respect to the directive name length in a descending
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
-  std::sort(table, table + llvm::omp::Directive_enumSize,
-      [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
+  for (int initial{'a'}; initial != 'z' + 1; ++initial) {
+    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
+      return a.first.size() > b.first.size();
+    });
+  }
 }
 
 // --- Modifier helpers -----------------------------------------------

>From 1dd6916ae6c7bd360e1c603b0cb9fd3b942233fc Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 9 Jul 2025 08:22:33 -0500
Subject: [PATCH 2/2] format

---
 flang/lib/Parser/openmp-parsers.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 29701a616d4b0..fdf96dffeb8ed 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -126,8 +126,8 @@ struct OmpDirectiveNameParser {
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
   using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<ConstIterator>
-  directives_starting_with(char initial) const;
+  llvm::iterator_range<ConstIterator> directives_starting_with(
+      char initial) const;
   void initTokens(std::vector<NameWithId>[]) const;
 };
 
@@ -161,9 +161,8 @@ void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
   for (int initial{'a'}; initial != 'z' + 1; ++initial) {
-    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
-      return a.first.size() > b.first.size();
-    });
+    llvm::stable_sort(table[initial - 'a'],
+        [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
   }
 }
 


From flang-commits at lists.llvm.org  Wed Jul  9 06:59:28 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 06:59:28 -0700 (PDT)
Subject: [flang-commits] [flang] 1770e9b - [mlir] remove dangling builders
	from td (#147619)
Message-ID: <686e75c0.170a0220.f60d7.d6fd@mx.google.com>


Author: Maksim Levental
Date: 2025-07-09T09:59:24-04:00
New Revision: 1770e9b5c6f9e95f253b0d8619de6f3a4290c96e

URL: https://github.com/llvm/llvm-project/commit/1770e9b5c6f9e95f253b0d8619de6f3a4290c96e
DIFF: https://github.com/llvm/llvm-project/commit/1770e9b5c6f9e95f253b0d8619de6f3a4290c96e.diff

LOG: [mlir] remove dangling builders from td (#147619)

These are "dangling" builders (decls are emitted but there are no defns
anywhere).

Added: 
    

Modified: 
    flang/include/flang/Optimizer/Dialect/FIROps.td
    mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
    mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
    mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
    mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
    mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
    mlir/test/lib/Dialect/Test/TestOps.td

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index b96c0360316ad..f440580f0878a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3624,12 +3624,6 @@ def fir_LocalitySpecifierOp : fir_Op<"local", [IsolatedFromAbove]> {
       attr-dict
   }];
 
-  let builders = [
-    OpBuilder<(ins CArg<"mlir::TypeRange">:$result,
-                   CArg<"mlir::StringAttr">:$sym_name,
-                   CArg<"mlir::TypeAttr">:$type)>
-  ];
-
   let extraClassDeclaration = [{
     mlir::BlockArgument getInitMoldArg() {
       auto &region = getInitRegion();

diff  --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 4360055e78691..b4dde776822a1 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -582,9 +582,6 @@ def LinalgCopyToMemrefOp :
   let assemblyFormat = "$target attr-dict `:` "
                        "functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -997,13 +994,6 @@ def PackGreedilyOp : Op<Transform_Dialect, "structured.pack_greedily", [
                                  [DenseArrayCount<3>]>:$matmul_inner_dims_order);
   let results = (outs TransformHandleTypeInterface:$packed_op);
 
-  let builders = [
-    OpBuilder<(ins "Value":$target,
-                   "ArrayRef<OpFoldResult>":$mixedMatmulPackedSizes,
-                   "ArrayRef<int64_t>":$matmulPaddededSizesNextMultipleOf,
-                   CArg<"ArrayRef<int64_t>", "{}">:$matmulDimsInnerDimsOrder)>
-  ];
-
   let assemblyFormat = [{
     $target
     oilist(
@@ -2509,10 +2499,6 @@ def HoistRedundantVectorTransfersOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target,
-               CArg<"bool", "false">:$verify_non_zero_trip)>,
-  ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
          ::mlir::transform::TransformRewriter &rewriter,
@@ -2546,9 +2532,6 @@ def HoistRedundantVectorBroadcastsOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
          ::mlir::transform::TransformRewriter &rewriter,
@@ -2623,10 +2606,6 @@ def ConvertConv2DToImg2ColOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
-
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2666,10 +2645,6 @@ def FlattenElementwiseLinalgOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
-
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2715,10 +2690,6 @@ def TransposeConv2DOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
-
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2761,10 +2732,6 @@ def TransposeMatmulOp : Op<Transform_Dialect,
     attr-dict `:` functional-type($target, results)
   }];
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
-
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2801,9 +2768,6 @@ def InsertSliceToCopyOp :
 
   let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) ";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2859,9 +2823,6 @@ def MapCopyToThreadsOp :
     `:` functional-type(operands, results)
   }];
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>,
-  ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2910,10 +2871,6 @@ def WinogradConv2DOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
-
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,
@@ -2947,10 +2904,6 @@ def DecomposeWinogradOp : Op<Transform_Dialect,
   let assemblyFormat =
     "$target attr-dict `:` functional-type($target, results)";
 
-  let builders = [
-    OpBuilder<(ins "Value":$target)>
-  ];
-
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
         ::mlir::transform::TransformRewriter &rewriter,

diff  --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..dffb3c5e14b62 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1751,10 +1751,6 @@ def DeclareMapperInfoOp : OpenMP_Op<"declare_mapper.info", [
     parent DeclareMapperOp.
   }] # clausesDescription;
 
-  let builders = [
-    OpBuilder<(ins CArg<"const DeclareMapperInfoOperands &">:$clauses)>
-  ];
-
   let extraClassDeclaration = [{
     // Override BlockArgOpenMPOpInterface method because `map` clauses have no
     // associated entry block arguments in this operation.

diff  --git a/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td b/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
index 1523762efc18f..440f6e571eec5 100644
--- a/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
+++ b/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
@@ -171,9 +171,6 @@ def Ptr_TypeOffsetOp : Pointer_Op<"type_offset", [Pure]> {
 
   let arguments = (ins TypeAttr:$elementType);
   let results = (outs AnySignlessIntegerOrIndex:$result);
-  let builders = [
-    OpBuilder<(ins "Type":$elementType)>
-  ];
   let assemblyFormat = [{
     $elementType attr-dict `:` type($result)
   }];

diff  --git a/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td b/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
index ea94dfd8fbd2a..4aa07a1283461 100644
--- a/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
+++ b/mlir/include/mlir/Dialect/SMT/IR/SMTIntOps.td
@@ -52,12 +52,6 @@ class VariadicIntOp<string mnemonic> : SMTIntOp<mnemonic, [Pure, Commutative]> {
   let arguments = (ins Variadic<IntType>:$inputs);
   let results = (outs IntType:$result);
   let assemblyFormat = "$inputs attr-dict";
-
-  let builders = [
-    OpBuilder<(ins "mlir::ValueRange":$inputs), [{
-      build($_builder, $_state, $_builder.getType<smt::IntType>(), inputs);
-    }]>,
-  ];
 }
 
 class BinaryIntOp<string mnemonic> : SMTIntOp<mnemonic, [Pure]> {

diff  --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index ec2c87ca1cf44..cbe490f6e4dd1 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -3011,12 +3011,7 @@ def Vector_ScanOp :
       vector<4x8x16x32xf32>, vector<4x16x32xf32>
     ```
   }];
-  let builders = [
-    OpBuilder<(ins "Value":$source, "Value":$initial_value,
-                   "CombiningKind":$kind,
-                   CArg<"int64_t", "0">:$reduction_dim,
-                   CArg<"bool", "true">:$inclusive)>
-  ];
+
   let extraClassDeclaration = [{
     VectorType getSourceType() {
       return ::llvm::cast<VectorType>(getSource().getType());

diff  --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 535f5e9b4a15d..ab3f847ca2acf 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -1192,7 +1192,6 @@ def OpFuncRef : TEST_Op<"op_funcref"> {
   let description = [{
     The "test.op_funcref" is a test op with a reference to a function symbol.
   }];
-  let builders = [OpBuilder<(ins "::mlir::func::FuncOp":$function)>];
 }
 
 // Pattern add the argument plus a increasing static number hidden in


        

From flang-commits at lists.llvm.org  Wed Jul  9 06:59:31 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Wed, 09 Jul 2025 06:59:31 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir] remove dangling builders from
	td (PR #147619)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147619@github.com>
Message-ID: <686e75c3.050a0220.2f7a30.f716@mx.google.com>


https://github.com/makslevental closed https://github.com/llvm/llvm-project/pull/147619

From flang-commits at lists.llvm.org  Wed Jul  9 07:15:56 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 09 Jul 2025 07:15:56 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add
	ComplexTOROCDLLibraryCalls pass (PR #144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <686e799c.170a0220.23dfe0.3de6@mx.google.com>


TIFitis wrote:

Polite reminder for review, thanks.

https://github.com/llvm/llvm-project/pull/144926

From flang-commits at lists.llvm.org  Wed Jul  9 07:31:50 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 07:31:50 -0700 (PDT)
Subject: [flang-commits] [flang] d0caf0d - [flang] Avoid undefined behaviour
 when parsing format expressions (#147539)
Message-ID: <686e7d56.170a0220.28df35.dc9f@mx.google.com>


Author: David Spickett
Date: 2025-07-09T15:31:45+01:00
New Revision: d0caf0d4857c2b00ba988f86703663685ec8697f

URL: https://github.com/llvm/llvm-project/commit/d0caf0d4857c2b00ba988f86703663685ec8697f
DIFF: https://github.com/llvm/llvm-project/commit/d0caf0d4857c2b00ba988f86703663685ec8697f.diff

LOG: [flang] Avoid undefined behaviour when parsing format expressions (#147539)

The test flang/test/Semantics/io08.f90 was failing when UBSAN was
enabled:
```
/home/david.spickett/llvm-project/flang/include/flang/Common/format.h:224:26: runtime error: signed integer overflow: 10 * 987654321098765432 cannot be represented in type 'int64_t' (aka 'long')
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/david.spickett/llvm-project/flang/include/flang/Common/format.h:224:26
```
This is because the code was effectively:
* Take the risk of UB happening
* Check whether it happened or not

Which UBSAN is obviously not going to like. Instead of checking after
the fact, use llvm's helpers that catch overflow without actually doing
it.

Added: 
    

Modified: 
    flang/include/flang/Common/format.h

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 1650f56140b4d..11aa6f2c07797 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -11,6 +11,7 @@
 
 #include "Fortran-consts.h"
 #include "enum-set.h"
+#include "llvm/Support/MathExtras.h"
 #include <cstring>
 
 // Define a FormatValidator class template to validate a format expression
@@ -214,16 +215,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::MulOverflow(
+            static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (LLVM_LIKELY(!overflow)) {
+        overflow = llvm::AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
       }
       c = NextChar();
     } while (c >= '0' && c <= '9');


        

From flang-commits at lists.llvm.org  Wed Jul  9 07:31:52 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Wed, 09 Jul 2025 07:31:52 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e7d58.170a0220.1cb24e.e55c@mx.google.com>


https://github.com/DavidSpickett closed https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 07:40:42 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 09 Jul 2025 07:40:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e7f6a.170a0220.bfd66.67a0@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `amdgpu-offload-rhel-8-cmake-build-only` running on `rocm-docker-rhel-8` while building `flang` at step 4 "annotate".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/204/builds/14933

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 4 (annotate) failure: '../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py --jobs=32' (failure)
...
[37/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libomptarget.so.21.0git
[38/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libomptarget.so
[39/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libLLVMOffload.so.21.0git
[40/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libLLVMOffload.so
[41/49] Generating /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/compile_commands.json
[42/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/bin/llvm-offload-device-info
[43/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/bin/llvm-omp-kernel-replay
[44/49] Building LLVM bitcode Workshare.cpp-amdgpu.o
[45/49] Building LLVM bitcode Parallelism.cpp-amdgpu.o
[46/49] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/./bin/clang++ --target=x86_64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/include -I/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/../flang/include -I/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/runtimes/runtimes-bins/flang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -std=gnu++17 -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp
In file included from /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp:9:
In file included from /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
[47/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libomptarget-amdgpu.bc
clang++: warning: joined argument expects additional value: '-mcpu=' [-Wunused-command-line-argument]
ninja: build stopped: subcommand failed.
FAILED: runtimes/runtimes-stamps/runtimes-build /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/runtimes/runtimes-stamps/runtimes-build 
cd /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/runtimes/runtimes-bins && /usr/bin/cmake --build .
ninja: build stopped: subcommand failed.
['ninja'] exited with return code 1.
The build step threw an exception...
Traceback (most recent call last):
  File "../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 50, in step
    yield
  File "../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 41, in main
    run_command(["ninja"])
  File "../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 63, in run_command
    util.report_run_cmd(cmd, cwd=directory)
  File "/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-zorg/zorg/buildbot/builders/annotated/util.py", line 49, in report_run_cmd
    subprocess.check_call(cmd, shell=shell, *args, **kwargs)
  File "/usr/lib64/python3.8/subprocess.py", line 364, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['ninja']' returned non-zero exit status 1.
@@@STEP_FAILURE@@@
Step 7 (build cmake config) failure: build cmake config (failure)
...
[37/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libomptarget.so.21.0git
[38/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libomptarget.so
[39/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libLLVMOffload.so.21.0git
[40/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libLLVMOffload.so
[41/49] Generating /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/compile_commands.json
[42/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/bin/llvm-offload-device-info
[43/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/bin/llvm-omp-kernel-replay
[44/49] Building LLVM bitcode Workshare.cpp-amdgpu.o
[45/49] Building LLVM bitcode Parallelism.cpp-amdgpu.o
[46/49] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/./bin/clang++ --target=x86_64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/include -I/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/../flang/include -I/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/runtimes/runtimes-bins/flang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -std=gnu++17 -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp
In file included from /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp:9:
In file included from /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-project/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
[47/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/lib/libomptarget-amdgpu.bc
clang++: warning: joined argument expects additional value: '-mcpu=' [-Wunused-command-line-argument]
ninja: build stopped: subcommand failed.
FAILED: runtimes/runtimes-stamps/runtimes-build /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/runtimes/runtimes-stamps/runtimes-build 
cd /home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/build/runtimes/runtimes-bins && /usr/bin/cmake --build .
ninja: build stopped: subcommand failed.
['ninja'] exited with return code 1.
The build step threw an exception...
Traceback (most recent call last):
  File "../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 50, in step
    yield
  File "../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 41, in main
    run_command(["ninja"])
  File "../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 63, in run_command
    util.report_run_cmd(cmd, cwd=directory)
  File "/home/botworker/bbot/amdgpu-offload-rhel-8-cmake-build-only/llvm-zorg/zorg/buildbot/builders/annotated/util.py", line 49, in report_run_cmd
    subprocess.check_call(cmd, shell=shell, *args, **kwargs)
  File "/usr/lib64/python3.8/subprocess.py", line 364, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['ninja']' returned non-zero exit status 1.
program finished with exit code 0
elapsedTime=393.671125

```

</details>

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 07:42:07 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 09 Jul 2025 07:42:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e7fbf.050a0220.1d1e70.f44c@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `openmp-offload-sles-build-only` running on `rocm-worker-hw-04-sles` while building `flang` at step 5 "compile-openmp".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/140/builds/26612

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 5 (compile-openmp) failure: build (failure)
...
6.668 [98/34/858] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/file.cpp.o
6.709 [97/34/859] Linking CXX executable /home/botworker/bbot/builds/openmp-offload-sles-build/llvm.build/lib/libomptarget-amdgpu.bc
clang++: warning: joined argument expects additional value: '-mcpu=' [-Wunused-command-line-argument]
6.719 [96/34/860] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/internal-unit.cpp.o
6.781 [95/34/861] Linking CXX static library /home/botworker/bbot/builds/openmp-offload-sles-build/llvm.build/lib/amdgcn-amd-amdhsa/libompdevice.a
6.787 [94/34/862] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/derived.cpp.o
6.803 [93/34/863] Building CXX object offload/tools/offload-tblgen/CMakeFiles/offload-tblgen.dir/MiscGen.cpp.o
6.804 [92/34/864] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/inquiry.cpp.o
6.890 [91/34/865] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/assign.cpp.o
6.894 [90/34/866] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/home/botworker/bbot/builds/openmp-offload-sles-build/llvm.build/./bin/clang++ --target=x86_64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/include -I/home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/../flang/include -Iflang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -std=gnu++17 -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/lib/runtime/format.cpp
In file included from /home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/lib/runtime/format.cpp:9:
In file included from /home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
6.938 [90/33/867] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/io-error.cpp.o
6.943 [90/32/868] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang/lib/Decimal/binary-to-decimal.cpp.o
6.973 [90/31/869] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/edit-input.cpp.o
6.992 [90/30/870] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/external-unit.cpp.o
7.097 [90/29/871] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/non-tbp-dio.cpp.o
7.149 [90/28/872] Building CXX object compiler-rt/lib/memprof/CMakeFiles/RTMemprof_dynamic.x86_64.dir/memprof_interceptors.cpp.o
7.188 [90/27/873] Building CXX object offload/tools/offload-tblgen/CMakeFiles/offload-tblgen.dir/PrintGen.cpp.o
7.198 [90/26/874] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/memory.cpp.o
7.205 [90/25/875] Building CXX object offload/tools/offload-tblgen/CMakeFiles/offload-tblgen.dir/EntryPointGen.cpp.o
7.281 [90/24/876] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/misc-intrinsic.cpp.o
7.340 [90/23/877] Building CXX object compiler-rt/lib/msan/CMakeFiles/clang_rt.msan-x86_64.dir/msan_interceptors.cpp.o
7.426 [90/22/878] Building CXX object offload/tools/offload-tblgen/CMakeFiles/offload-tblgen.dir/APIGen.cpp.o
7.485 [90/21/879] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/ragged.cpp.o
7.517 [90/20/880] Building CXX object compiler-rt/lib/tsan/rtl/CMakeFiles/clang_rt.tsan-x86_64.dir/tsan_interceptors_posix.cpp.o
7.542 [90/19/881] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/pointer.cpp.o
7.709 [90/18/882] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/numeric.cpp.o
7.742 [90/17/883] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/pseudo-unit.cpp.o
7.818 [90/16/884] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/io-api-minimal.cpp.o
7.944 [90/15/885] Building CXX object compiler-rt/lib/tsan/rtl/CMakeFiles/clang_rt.tsan-dynamic-x86_64.dir/tsan_interceptors_posix.cpp.o
7.982 [90/14/886] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/namelist.cpp.o
8.247 [90/13/887] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/edit-output.cpp.o
In file included from /home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/lib/runtime/edit-output.cpp:10:
/home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/include/flang-rt/runtime/emit-encoded.h:49:40: warning: implicit conversion from 'const UnsignedChar' (aka 'const char16_t') to 'char32_t' may change the meaning of the represented code unit [-Wcharacter-conversion]
   49 |       auto len{EncodeUTF8(buffer + at, *uData++)};
      |                ~~~~~~~~~~              ^~~~~~~~
/home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/lib/runtime/edit-output.cpp:865:18: note: in instantiation of function template specialization 'Fortran::runtime::io::EmitEncoded<Fortran::runtime::io::IoStatementState, char16_t, true>' requested here
  865 |       ok = ok && EmitEncoded(io, &ch, 1);
      |                  ^
In file included from /home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/lib/runtime/edit-output.cpp:10:
/home/botworker/bbot/builds/openmp-offload-sles-build/llvm.src/flang-rt/include/flang-rt/runtime/emit-encoded.h:49:40: warning: implicit conversion from 'const UnsignedChar' (aka 'const char16_t') to 'char32_t' may change the meaning of the represented code unit [-Wcharacter-conversion]
   49 |       auto len{EncodeUTF8(buffer + at, *uData++)};

```

</details>

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 07:44:53 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 09 Jul 2025 07:44:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e8065.170a0220.2a3312.ed37@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `amdgpu-offload-ubuntu-22-cmake-build-only` running on `rocm-docker-ubu-22` while building `flang` at step 4 "annotate".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/203/builds/16120

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 4 (annotate) failure: '../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py --jobs=32' (failure)
...
[37/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libomptarget.so.21.0git
[38/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libomptarget.so
[39/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libLLVMOffload.so.21.0git
[40/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libLLVMOffload.so
[41/49] Generating /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/compile_commands.json
[42/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/bin/llvm-offload-device-info
[43/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/bin/llvm-omp-kernel-replay
[44/49] Building LLVM bitcode Workshare.cpp-amdgpu.o
[45/49] Building LLVM bitcode Parallelism.cpp-amdgpu.o
[46/49] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/./bin/clang++ --target=x86_64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/include -I/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/../flang/include -I/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/runtimes/runtimes-bins/flang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -std=gnu++17 -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp
In file included from /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp:9:
In file included from /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
[47/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libomptarget-amdgpu.bc
clang++: warning: joined argument expects additional value: '-mcpu=' [-Wunused-command-line-argument]
ninja: build stopped: subcommand failed.
FAILED: runtimes/runtimes-stamps/runtimes-build /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/runtimes/runtimes-stamps/runtimes-build 
cd /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/runtimes/runtimes-bins && /usr/bin/cmake --build .
[7928/7931] Building CXX object tools/flang/tools/bbc/CMakeFiles/bbc.dir/bbc.cpp.o
ninja: build stopped: subcommand failed.
['ninja'] exited with return code 1.
The build step threw an exception...
Traceback (most recent call last):
  File "/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 50, in step
    yield
  File "/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 41, in main
    run_command(["ninja"])
  File "/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 63, in run_command
    util.report_run_cmd(cmd, cwd=directory)
  File "/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-zorg/zorg/buildbot/builders/annotated/util.py", line 49, in report_run_cmd
    subprocess.check_call(cmd, shell=shell, *args, **kwargs)
  File "/usr/lib/python3.10/subprocess.py", line 369, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['ninja']' returned non-zero exit status 1.
@@@STEP_FAILURE@@@
Step 7 (build cmake config) failure: build cmake config (failure)
...
[37/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libomptarget.so.21.0git
[38/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libomptarget.so
[39/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libLLVMOffload.so.21.0git
[40/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libLLVMOffload.so
[41/49] Generating /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/compile_commands.json
[42/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/bin/llvm-offload-device-info
[43/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/bin/llvm-omp-kernel-replay
[44/49] Building LLVM bitcode Workshare.cpp-amdgpu.o
[45/49] Building LLVM bitcode Parallelism.cpp-amdgpu.o
[46/49] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/./bin/clang++ --target=x86_64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/include -I/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/../flang/include -I/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/runtimes/runtimes-bins/flang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -std=gnu++17 -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp
In file included from /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp:9:
In file included from /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-project/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
[47/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/lib/libomptarget-amdgpu.bc
clang++: warning: joined argument expects additional value: '-mcpu=' [-Wunused-command-line-argument]
ninja: build stopped: subcommand failed.
FAILED: runtimes/runtimes-stamps/runtimes-build /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/runtimes/runtimes-stamps/runtimes-build 
cd /home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/runtimes/runtimes-bins && /usr/bin/cmake --build .
[7928/7931] Building CXX object tools/flang/tools/bbc/CMakeFiles/bbc.dir/bbc.cpp.o
ninja: build stopped: subcommand failed.
['ninja'] exited with return code 1.
The build step threw an exception...
Traceback (most recent call last):
  File "/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 50, in step
    yield
  File "/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 41, in main
    run_command(["ninja"])
  File "/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 63, in run_command
    util.report_run_cmd(cmd, cwd=directory)
  File "/home/botworker/bbot/amdgpu-offload-ubuntu-22-cmake-build-only/llvm-zorg/zorg/buildbot/builders/annotated/util.py", line 49, in report_run_cmd
    subprocess.check_call(cmd, shell=shell, *args, **kwargs)
  File "/usr/lib/python3.10/subprocess.py", line 369, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['ninja']' returned non-zero exit status 1.
program finished with exit code 0
elapsedTime=408.028063

```

</details>

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 07:46:22 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 09 Jul 2025 07:46:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e80be.170a0220.15e77.f435@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `amdgpu-offload-rhel-9-cmake-build-only` running on `rocm-docker-rhel-9` while building `flang` at step 4 "annotate".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/205/builds/14910

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 4 (annotate) failure: '../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py --jobs=32' (failure)
...
[37/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libomptarget.so.21.0git
[38/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libomptarget.so
[39/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libLLVMOffload.so.21.0git
[40/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libLLVMOffload.so
[41/49] Generating /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/compile_commands.json
[42/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/bin/llvm-offload-device-info
[43/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/bin/llvm-omp-kernel-replay
[44/49] Building LLVM bitcode Parallelism.cpp-amdgpu.o
[45/49] Building LLVM bitcode Workshare.cpp-amdgpu.o
[46/49] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/./bin/clang++ --target=x86_64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/include -I/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/../flang/include -I/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/runtimes/runtimes-bins/flang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -std=gnu++17 -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp
In file included from /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp:9:
In file included from /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
[47/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libomptarget-amdgpu.bc
clang++: warning: joined argument expects additional value: '-mcpu=' [-Wunused-command-line-argument]
ninja: build stopped: subcommand failed.
FAILED: runtimes/runtimes-stamps/runtimes-build /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/runtimes/runtimes-stamps/runtimes-build 
cd /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/runtimes/runtimes-bins && /usr/bin/cmake --build .
[7928/7931] Building CXX object tools/flang/tools/bbc/CMakeFiles/bbc.dir/bbc.cpp.o
ninja: build stopped: subcommand failed.
['ninja'] exited with return code 1.
The build step threw an exception...
Traceback (most recent call last):
  File "/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 50, in step
    yield
  File "/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 41, in main
    run_command(["ninja"])
  File "/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 63, in run_command
    util.report_run_cmd(cmd, cwd=directory)
  File "/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-zorg/zorg/buildbot/builders/annotated/util.py", line 49, in report_run_cmd
    subprocess.check_call(cmd, shell=shell, *args, **kwargs)
  File "/usr/lib64/python3.9/subprocess.py", line 373, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['ninja']' returned non-zero exit status 1.
@@@STEP_FAILURE@@@
Step 7 (build cmake config) failure: build cmake config (failure)
...
[37/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libomptarget.so.21.0git
[38/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libomptarget.so
[39/49] Linking CXX shared library /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libLLVMOffload.so.21.0git
[40/49] Creating library symlink /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libLLVMOffload.so
[41/49] Generating /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/compile_commands.json
[42/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/bin/llvm-offload-device-info
[43/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/bin/llvm-omp-kernel-replay
[44/49] Building LLVM bitcode Parallelism.cpp-amdgpu.o
[45/49] Building LLVM bitcode Workshare.cpp-amdgpu.o
[46/49] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/./bin/clang++ --target=x86_64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/include -I/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/../flang/include -I/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/runtimes/runtimes-bins/flang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -std=gnu++17 -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp
In file included from /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/lib/runtime/format.cpp:9:
In file included from /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-project/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
[47/49] Linking CXX executable /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/lib/libomptarget-amdgpu.bc
clang++: warning: joined argument expects additional value: '-mcpu=' [-Wunused-command-line-argument]
ninja: build stopped: subcommand failed.
FAILED: runtimes/runtimes-stamps/runtimes-build /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/runtimes/runtimes-stamps/runtimes-build 
cd /home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/runtimes/runtimes-bins && /usr/bin/cmake --build .
[7928/7931] Building CXX object tools/flang/tools/bbc/CMakeFiles/bbc.dir/bbc.cpp.o
ninja: build stopped: subcommand failed.
['ninja'] exited with return code 1.
The build step threw an exception...
Traceback (most recent call last):
  File "/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 50, in step
    yield
  File "/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 41, in main
    run_command(["ninja"])
  File "/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/build/../llvm-zorg/zorg/buildbot/builders/annotated/amdgpu-offload-cmake.py", line 63, in run_command
    util.report_run_cmd(cmd, cwd=directory)
  File "/home/botworker/bbot/amdgpu-offload-rhel-9-cmake-build-only/llvm-zorg/zorg/buildbot/builders/annotated/util.py", line 49, in report_run_cmd
    subprocess.check_call(cmd, shell=shell, *args, **kwargs)
  File "/usr/lib64/python3.9/subprocess.py", line 373, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['ninja']' returned non-zero exit status 1.
program finished with exit code 0
elapsedTime=412.352443

```

</details>

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 07:50:56 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Wed, 09 Jul 2025 07:50:56 -0700 (PDT)
Subject: [flang-commits] [flang] e8e5d07 - Revert "[flang] Avoid undefined
 behaviour when parsing format expressions (#147539)"
Message-ID: <686e81d0.170a0220.2a20c.f67e@mx.google.com>


Author: David Spickett
Date: 2025-07-09T14:49:58Z
New Revision: e8e5d07767c444913f837dd35846a92fcf520eab

URL: https://github.com/llvm/llvm-project/commit/e8e5d07767c444913f837dd35846a92fcf520eab
DIFF: https://github.com/llvm/llvm-project/commit/e8e5d07767c444913f837dd35846a92fcf520eab.diff

LOG: Revert "[flang] Avoid undefined behaviour when parsing format expressions (#147539)"

This reverts commit d0caf0d4857c2b00ba988f86703663685ec8697f.

MathExtras.h is not found in some builds.

Added: 
    

Modified: 
    flang/include/flang/Common/format.h

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 11aa6f2c07797..1650f56140b4d 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -11,7 +11,6 @@
 
 #include "Fortran-consts.h"
 #include "enum-set.h"
-#include "llvm/Support/MathExtras.h"
 #include <cstring>
 
 // Define a FormatValidator class template to validate a format expression
@@ -215,18 +214,16 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
+    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
+      lastValue = integerValue_;
       lastCursor = cursor_;
-      if (LLVM_LIKELY(!overflow)) {
-        overflow = llvm::MulOverflow(
-            static_cast<int64_t>(10), integerValue_, integerValue_);
-      }
-      if (LLVM_LIKELY(!overflow)) {
-        overflow = llvm::AddOverflow(
-            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
+      integerValue_ = 10 * integerValue_ + c - '0';
+      if (lastValue > integerValue_) {
+        overflow = true;
       }
       c = NextChar();
     } while (c >= '0' && c <= '9');


        

From flang-commits at lists.llvm.org  Wed Jul  9 07:52:35 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 09 Jul 2025 07:52:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e8233.170a0220.373d99.02b1@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `clang-aarch64-sve-vla` running on `linaro-g3-03` while building `flang` at step 6 "build stage 1".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/17/builds/9443

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (build stage 1) failure: 'ninja' (failure)
...
[3455/3458] Performing build step for 'runtimes'
[1/11] Generating Fortran dyndep file flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/Fortran.dd
[2/11] Building CXX object compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic_version_script_dummy.aarch64.dir/dummy.cpp.o
[3/11] Building CXX object compiler-rt/lib/hwasan/CMakeFiles/RTHwasan_dynamic_version_script_dummy.aarch64.dir/dummy.cpp.o
[4/11] Building CXX object compiler-rt/lib/ubsan/CMakeFiles/RTUbsan_dynamic_version_script_dummy.aarch64.dir/dummy.cpp.o
[5/11] Linking CXX shared library /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/stage1/lib/clang/21/lib/aarch64-unknown-linux-gnu/libclang_rt.hwasan.so
[6/11] Linking CXX shared library /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/stage1/lib/clang/21/lib/aarch64-unknown-linux-gnu/libclang_rt.ubsan_standalone.so
[7/11] Linking CXX shared library /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/stage1/lib/clang/21/lib/aarch64-unknown-linux-gnu/libclang_rt.asan.so
[8/11] Generating /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/stage1/compile_commands.json
[9/11] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/home/tcwg-buildbot/worker/clang-aarch64-sve-vla/stage1/./bin/clang++ --target=aarch64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/tcwg-buildbot/worker/clang-aarch64-sve-vla/llvm/flang-rt/include -I/home/tcwg-buildbot/worker/clang-aarch64-sve-vla/llvm/flang-rt/../flang/include -I/home/tcwg-buildbot/worker/clang-aarch64-sve-vla/stage1/runtimes/runtimes-bins/flang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -std=gnu++17 -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/llvm/flang-rt/lib/runtime/format.cpp
In file included from /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/llvm/flang-rt/lib/runtime/format.cpp:9:
In file included from /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/llvm/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/home/tcwg-buildbot/worker/clang-aarch64-sve-vla/llvm/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
[10/11] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/time-intrinsic.cpp.o
ninja: build stopped: subcommand failed.
FAILED: runtimes/runtimes-stamps/runtimes-build /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/stage1/runtimes/runtimes-stamps/runtimes-build 
cd /home/tcwg-buildbot/worker/clang-aarch64-sve-vla/stage1/runtimes/runtimes-bins && /usr/local/bin/cmake --build .
ninja: build stopped: subcommand failed.

```

</details>

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 08:01:51 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 08:01:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop and omp.simd (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e845f.050a0220.3928.2dba@mx.google.com>


================
@@ -1816,8 +1816,7 @@ static void genSimdClauses(
   cp.processReduction(loc, clauseOps, reductionSyms);
   cp.processSafelen(clauseOps);
   cp.processSimdlen(clauseOps);
-
-  cp.processTODO<clause::Linear>(loc, llvm::omp::Directive::OMPD_simd);
+  cp.processLinear(clauseOps);
----------------
tblah wrote:

The current implementation of `processLinear` does not seem to guarantee that the variable comes from an alloca. For example what if it is a function argument?

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 08:01:51 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 08:01:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop and omp.simd (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e845f.170a0220.32f909.2303@mx.google.com>


================
@@ -695,6 +786,48 @@ llvm.func @simd_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) {
 
 // -----
 
+// CHECK-LABEL: @simd_linear
+
+// CHECK: entry:
+// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
+// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
+// CHECK: br label %omp.simd.region
+
+// CHECK: omp.simd.region:
+// CHECK: br label %omp_loop.preheader
+
+// CHECK: omp_loop.preheader:
+// CHECK: %[[ORIG_VAL:.*]] = load i32, ptr {{.*}}, align 4
+// CHECK: store i32 %[[ORIG_VAL]], ptr %[[LINEAR_VAR]], align 4
+// CHECK: br label %omp_loop.header
+
+// CHECK: omp_loop.body:
+// CHECK: %[[LINEAR_LOAD:.*]] = load i32, ptr %[[LINEAR_VAR]], align 4
----------------
tblah wrote:

Please could you test that these loads and stores do get the llvm access group metadata

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 08:01:51 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 08:01:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop and omp.simd (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e845f.630a0220.b1ab6.38f2@mx.google.com>


================
@@ -358,6 +358,97 @@ llvm.func @wsloop_simple(%arg0: !llvm.ptr) {
 
 // -----
 
+// CHECK-LABEL: wsloop_linear
+
+// CHECK: {{.*}} = alloca i32, i64 1, align 4
+// CHECK: %[[Y:.*]] = alloca i32, i64 1, align 4
+// CHECK: %[[X:.*]] = alloca i32, i64 1, align 4
+
+// CHECK: entry:
+// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
+// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
+// CHECK: br label %omp_loop.preheader
+
+
+// CHECK: omp_loop.preheader:
+// CHECK: %[[LOAD:.*]] = load i32, ptr %[[X]], align 4
+// CHECK: store i32 %[[LOAD]], ptr %[[LINEAR_VAR]], align 4
+// CHECK: %omp_global_thread_num = call i32 @__kmpc_global_thread_num(ptr @2)
+// CHECK: call void @__kmpc_barrier(ptr @1, i32 %omp_global_thread_num)
+
+// CHECK: omp_loop.body:
+// CHECK: %[[LOOP_IV:.*]] = add i32 %omp_loop.iv, {{.*}}
+// CHECK: %[[LINEAR_LOAD:.*]] = load i32, ptr %[[LINEAR_VAR]], align 4
+// CHECK: %[[MUL:.*]] = mul i32 %[[LOOP_IV]], 1
+// CHECK: %[[ADD:.*]] = add i32 %[[LINEAR_LOAD]], %[[MUL]]
+// CHECK: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]], align 4
+// CHECK: br label %omp.loop_nest.region
+
+// CHECK: omp.loop_nest.region:
+// CHECK: %[[LINEAR_LOAD:.*]] = load i32, ptr %[[LINEAR_RESULT]], align 4
+// CHECK: %[[ADD:.*]] = add i32 %[[LINEAR_LOAD]], 2
+// CHECK: store i32 %[[ADD]], ptr %[[Y]], align 4
+
+// CHECK: omp_loop.exit:
+// CHECK: call void @__kmpc_for_static_fini(ptr @2, i32 %omp_global_thread_num4)
+// CHECK: %omp_global_thread_num5 = call i32 @__kmpc_global_thread_num(ptr @2)
+// CHECK: call void @__kmpc_barrier(ptr @3, i32 %omp_global_thread_num5)
+// CHECK: br label %omp_loop.linear_finalization
+
+
+// CHECK: omp_loop.linear_finalization:
+// CHECK: %[[LAST_ITER:.*]] = load i32, ptr %p.lastiter, align 4
+// CHECK: %[[CMP:.*]] = icmp ne i32 %[[LAST_ITER]], 0
+// CHECK: br i1 %[[CMP]], label %omp_loop.linear_lastiter_exit, label %omp_loop.linear_exit
+
+// CHECK: omp_loop.linear_lastiter_exit:
+// CHECK: %[[LINEAR_RESULT_LOAD:.*]] = load i32, ptr %[[LINEAR_RESULT]], align 4
+// CHECK: store i32 %[[LINEAR_RESULT_LOAD]], ptr %[[X]], align 4
+// CHECK: br label %omp_loop.linear_exit
+
+
+// CHECK: omp_loop.linear_exit:
+// CHECK: %omp_global_thread_num6 = call i32 @__kmpc_global_thread_num(ptr @2)
+// CHECK: call void @__kmpc_barrier(ptr @1, i32 %omp_global_thread_num6)
+// CHECK: br label %omp_loop.after
+
+llvm.func @wsloop_linear() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
+  %2 = llvm.mlir.constant(1 : i64) : i64
+  %3 = llvm.alloca %2 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+  %4 = llvm.mlir.constant(1 : i64) : i64
+  %5 = llvm.alloca %4 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %6 = llvm.mlir.constant(1 : i64) : i64
+  %7 = llvm.alloca %6 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  %8 = llvm.mlir.constant(2 : i32) : i32
+  %9 = llvm.mlir.constant(10 : i32) : i32
+  %10 = llvm.mlir.constant(1 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i64) : i64
+  %12 = llvm.mlir.constant(1 : i64) : i64
+  %13 = llvm.mlir.constant(1 : i64) : i64
+  %14 = llvm.mlir.constant(1 : i64) : i64
----------------
tblah wrote:

nit: these could be trivially cleaned up with something like `mlir-opt --canonicalize test_file.mlir`

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 08:01:51 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 08:01:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop and omp.simd (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e845f.170a0220.15e77.1ab2@mx.google.com>


================
@@ -358,6 +358,97 @@ llvm.func @wsloop_simple(%arg0: !llvm.ptr) {
 
 // -----
 
+// CHECK-LABEL: wsloop_linear
+
+// CHECK: {{.*}} = alloca i32, i64 1, align 4
+// CHECK: %[[Y:.*]] = alloca i32, i64 1, align 4
+// CHECK: %[[X:.*]] = alloca i32, i64 1, align 4
+
+// CHECK: entry:
+// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
+// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
+// CHECK: br label %omp_loop.preheader
+
+
+// CHECK: omp_loop.preheader:
+// CHECK: %[[LOAD:.*]] = load i32, ptr %[[X]], align 4
+// CHECK: store i32 %[[LOAD]], ptr %[[LINEAR_VAR]], align 4
+// CHECK: %omp_global_thread_num = call i32 @__kmpc_global_thread_num(ptr @2)
+// CHECK: call void @__kmpc_barrier(ptr @1, i32 %omp_global_thread_num)
+
+// CHECK: omp_loop.body:
+// CHECK: %[[LOOP_IV:.*]] = add i32 %omp_loop.iv, {{.*}}
+// CHECK: %[[LINEAR_LOAD:.*]] = load i32, ptr %[[LINEAR_VAR]], align 4
+// CHECK: %[[MUL:.*]] = mul i32 %[[LOOP_IV]], 1
+// CHECK: %[[ADD:.*]] = add i32 %[[LINEAR_LOAD]], %[[MUL]]
+// CHECK: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]], align 4
+// CHECK: br label %omp.loop_nest.region
+
+// CHECK: omp.loop_nest.region:
+// CHECK: %[[LINEAR_LOAD:.*]] = load i32, ptr %[[LINEAR_RESULT]], align 4
+// CHECK: %[[ADD:.*]] = add i32 %[[LINEAR_LOAD]], 2
+// CHECK: store i32 %[[ADD]], ptr %[[Y]], align 4
+
+// CHECK: omp_loop.exit:
+// CHECK: call void @__kmpc_for_static_fini(ptr @2, i32 %omp_global_thread_num4)
+// CHECK: %omp_global_thread_num5 = call i32 @__kmpc_global_thread_num(ptr @2)
+// CHECK: call void @__kmpc_barrier(ptr @3, i32 %omp_global_thread_num5)
----------------
tblah wrote:

I don't think we need both this barrier and the one after processing the linear variables.

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 08:01:52 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 08:01:52 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop and omp.simd (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686e8460.170a0220.788ca.92ce@mx.google.com>


================
@@ -2608,7 +2608,7 @@ void SimdOp::build(OpBuilder &builder, OperationState &state,
   // TODO Store clauses in op: linearVars, linearStepVars
----------------
tblah wrote:

```suggestion
```

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 08:03:02 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 08:03:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e84a6.050a0220.3beaba.46d7@mx.google.com>


tblah wrote:

It looks like this part of flang needs a dependency adding for the LLVM library

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 00:18:02 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 09 Jul 2025 00:18:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][do concurrent] Re-model `reduce` to
 match reductions are modelled in OpenMP and OpenACC (PR #145837)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145837@github.com>
Message-ID: <686e17aa.050a0220.1df27a.bb2d@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/145837

>From df68e3a21f393213d261492f31ed5f505aac7bb0 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Thu, 12 Jun 2025 06:14:21 -0500
Subject: [PATCH 1/3] [flang][do concurrent] Re-model `reduce` to match
 reductions are modelled in OpenMP and OpenACC

This PR proposes re-modelling `reduce` specifiers to match OpenMP and
OpenACC. In particular, this PR includes the following:

* A new `fir` op: `fir.delcare_reduction` which is identical to OpenMP's
  `omp.declare_reduction` op.
* Updating the `reduce` clause on `fir.do_concurrent.loop` to use the
  new op.
* Re-uses the `ReductionProcessor` component to emit reductions for `do
  concurrent` just like we do for OpenMP. To do this, the
  `ReductionProcessor` had to be refactored to be more generalized.
* Upates mapping `do concurrent` to `fir.loop ... unordered` nests using
  the new reduction model.

Unfortunately, this is a big PR that would be difficult to divide up in
smaller parts because the bottom of the changes are the `fir` table-gen
changes to `do concurrent`. However, doing these MLIR changes cascades
to the other parts that have to be modified to not break things.
---
 .../flang/Optimizer/Dialect/FIRAttr.td        |   2 +-
 .../include/flang/Optimizer/Dialect/FIROps.td | 136 +++++++-
 flang/lib/Lower/Bridge.cpp                    |  85 +++--
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  79 ++++-
 flang/lib/Lower/OpenMP/ClauseProcessor.h      |   1 -
 flang/lib/Lower/OpenMP/Clauses.h              |   1 +
 flang/lib/Lower/OpenMP/OpenMP.cpp             |   1 -
 flang/lib/Lower/OpenMP/ReductionProcessor.cpp | 281 +++++++++--------
 flang/lib/Lower/OpenMP/ReductionProcessor.h   |  30 +-
 flang/lib/Lower/Support/Utils.cpp             |   4 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  48 +--
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 293 ++++++++++++------
 .../Transforms/SimplifyFIROperations.cpp      |  10 +-
 flang/test/Fir/do_concurrent.fir              |  63 +++-
 flang/test/Fir/invalid.fir                    |  18 +-
 flang/test/Lower/do_concurrent_reduce.f90     |  41 +++
 flang/test/Lower/loops.f90                    |   2 +-
 flang/test/Lower/loops3.f90                   |   2 +-
 .../do_concurrent-to-do_loop-unodered.fir     |   2 +-
 19 files changed, 780 insertions(+), 319 deletions(-)
 create mode 100644 flang/test/Lower/do_concurrent_reduce.f90

diff --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.td b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
index 2845080030b92..7bd96ac3ea631 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRAttr.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
@@ -112,7 +112,7 @@ def fir_ReduceOperationEnum : I32BitEnumAttr<"ReduceOperationEnum",
       I32BitEnumAttrCaseBit<"MIN", 7, "min">,
       I32BitEnumAttrCaseBit<"IAND", 8, "iand">,
       I32BitEnumAttrCaseBit<"IOR", 9, "ior">,
-      I32BitEnumAttrCaseBit<"EIOR", 10, "eior">
+      I32BitEnumAttrCaseBit<"IEOR", 10, "ieor">
     ]> {
   let separator = ", ";
   let cppNamespace = "::fir";
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index b96c0360316ad..b217f1433767f 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3518,7 +3518,7 @@ def fir_BoxTotalElementsOp
 
 def YieldOp : fir_Op<"yield",
     [Pure, ReturnLike, Terminator,
-     ParentOneOf<["LocalitySpecifierOp"]>]> {
+     ParentOneOf<["LocalitySpecifierOp", "DeclareReductionOp"]>]> {
   let summary = "loop yield and termination operation";
   let description = [{
     "fir.yield" yields SSA values from a fir dialect op region and
@@ -3662,6 +3662,103 @@ def fir_LocalitySpecifierOp : fir_Op<"local", [IsolatedFromAbove]> {
   let hasRegionVerifier = 1;
 }
 
+def fir_DeclareReductionOp : fir_Op<"declare_reduction", [IsolatedFromAbove,
+                                                         Symbol]> {
+  let summary = "declares a reduction kind";
+  let description = [{
+    Note: this operation is adapted from omp::DeclareReductionOp. There is a lot
+    duplication at the moment. TODO Combined both ops into one. See:
+    https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
+
+    Declares an `do concurrent` reduction. This requires two mandatory and three
+    optional regions.
+
+      1. The optional alloc region specifies how to allocate the thread-local
+         reduction value. This region should not contain control flow and all
+         IR should be suitable for inlining straight into an entry block. In
+         the common case this is expected to contain only allocas. It is
+         expected to `fir.yield` the allocated value on all control paths.
+         If allocation is conditional (e.g. only allocate if the mold is
+         allocated), this should be done in the initilizer region and this
+         region not included. The alloc region is not used for by-value
+         reductions (where allocation is implicit).
+      2. The initializer region specifies how to initialize the thread-local
+         reduction value. This is usually the neutral element of the reduction.
+         For convenience, the region has an argument that contains the value
+         of the reduction accumulator at the start of the reduction. If an alloc
+         region is specified, there is a second block argument containing the
+         address of the allocated memory. The initializer region is expected to
+         `fir.yield` the new value on all control flow paths.
+      3. The reduction region specifies how to combine two values into one, i.e.
+         the reduction operator. It accepts the two values as arguments and is
+         expected to `fir.yield` the combined value on all control flow paths.
+      4. The atomic reduction region is optional and specifies how two values
+         can be combined atomically given local accumulator variables. It is
+         expected to store the combined value in the first accumulator variable.
+      5. The cleanup region is optional and specifies how to clean up any memory
+         allocated by the initializer region. The region has an argument that
+         contains the value of the thread-local reduction accumulator. This will
+         be executed after the reduction has completed.
+
+    Note that the MLIR type system does not allow for type-polymorphic
+    reductions. Separate reduction declarations should be created for different
+    element and accumulator types.
+
+    For initializer and reduction regions, the operand to `fir.yield` must
+    match the parent operation's results.
+  }];
+
+  let arguments = (ins SymbolNameAttr:$sym_name,
+                       TypeAttr:$type);
+
+  let regions = (region MaxSizedRegion<1>:$allocRegion,
+                        AnyRegion:$initializerRegion,
+                        AnyRegion:$reductionRegion,
+                        AnyRegion:$atomicReductionRegion,
+                        AnyRegion:$cleanupRegion);
+
+  let assemblyFormat = "$sym_name `:` $type attr-dict-with-keyword "
+                       "( `alloc` $allocRegion^ )? "
+                       "`init` $initializerRegion "
+                       "`combiner` $reductionRegion "
+                       "( `atomic` $atomicReductionRegion^ )? "
+                       "( `cleanup` $cleanupRegion^ )? ";
+
+  let extraClassDeclaration = [{
+    mlir::BlockArgument getAllocMoldArg() {
+      auto &region = getAllocRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    mlir::BlockArgument getInitializerMoldArg() {
+      return getInitializerRegion().getArgument(0);
+    }
+    mlir::BlockArgument getInitializerAllocArg() {
+      return getAllocRegion().empty() ?
+          nullptr : getInitializerRegion().getArgument(1);
+    }
+    mlir::BlockArgument getReductionLhsArg() {
+      return getReductionRegion().getArgument(0);
+    }
+    mlir::BlockArgument getReductionRhsArg() {
+      return getReductionRegion().getArgument(1);
+    }
+    mlir::BlockArgument getAtomicReductionLhsArg() {
+      auto &region = getAtomicReductionRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    mlir::BlockArgument getAtomicReductionRhsArg() {
+      auto &region = getAtomicReductionRegion();
+      return region.empty() ? nullptr : region.getArgument(1);
+    }
+    mlir::BlockArgument getCleanupAllocArg() {
+      auto &region = getCleanupRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+  }];
+
+  let hasRegionVerifier = 1;
+}
+
 def fir_DoConcurrentOp : fir_Op<"do_concurrent",
     [SingleBlock, AutomaticAllocationScope]> {
   let summary = "do concurrent loop wrapper";
@@ -3700,6 +3797,25 @@ def fir_LocalSpecifier {
   );
 }
 
+def fir_ReduceSpecifier {
+  dag arguments = (ins
+    Variadic<AnyType>:$reduce_vars,
+    OptionalAttr<DenseBoolArrayAttr>:$reduce_byref,
+
+    // This introduces redundency in how reductions are modelled. In particular,
+    // a single reduction is represented by 2 attributes:
+    //
+    // 1. `$reduce_syms` which is a list of `DeclareReductionOp`s.
+    // 2. `$reduce_attrs` which is an array of `fir::ReduceAttr` values.
+    //
+    // The first makes it easier to map `do concurrent` to parallization models
+    // (e.g. OpenMP and OpenACC) while the second makes it easier to map it to
+    // nests of `fir.do_loop ... unodered` ops.
+    OptionalAttr<SymbolRefArrayAttr>:$reduce_syms,
+    OptionalAttr<ArrayAttr>:$reduce_attrs
+  );
+}
+
 def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     [AttrSizedOperandSegments, DeclareOpInterfaceMethods<LoopLikeOpInterface,
                                                          ["getLoopInductionVars"]>,
@@ -3709,7 +3825,7 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
   let description = [{
     An operation that models a Fortran `do concurrent` loop's header and block.
     This is a single-region single-block terminator op that is expected to
-    terminate the region of a `omp.do_concurrent` wrapper op.
+    terminate the region of a `fir.do_concurrent` wrapper op.
 
     This op borrows from both `scf.parallel` and `fir.do_loop` ops. Similar to
     `scf.parallel`, a loop nest takes 3 groups of SSA values as operands that
@@ -3747,8 +3863,6 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     - `lowerBound`: The group of SSA values for the nest's lower bounds.
     - `upperBound`: The group of SSA values for the nest's upper bounds.
     - `step`: The group of SSA values for the nest's steps.
-    - `reduceOperands`: The reduction SSA values, if any.
-    - `reduceAttrs`: Attributes to store reduction operations, if any.
     - `loopAnnotation`: Loop metadata to be passed down the compiler pipeline to
       LLVM.
   }];
@@ -3757,12 +3871,12 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     Variadic<Index>:$lowerBound,
     Variadic<Index>:$upperBound,
     Variadic<Index>:$step,
-    Variadic<AnyType>:$reduceOperands,
-    OptionalAttr<ArrayAttr>:$reduceAttrs,
     OptionalAttr<LoopAnnotationAttr>:$loopAnnotation
   );
 
-  let arguments = !con(opArgs, fir_LocalSpecifier.arguments);
+  let arguments = !con(opArgs,
+    fir_LocalSpecifier.arguments,
+    fir_ReduceSpecifier.arguments);
 
   let regions = (region SizedRegion<1>:$region);
 
@@ -3783,12 +3897,18 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
                                              getNumLocalOperands());
     }
 
+    mlir::Block::BlockArgListType getRegionReduceArgs() {
+      return getBody()->getArguments().slice(getNumInductionVars()
+                                               + getNumLocalOperands(),
+                                             getNumReduceOperands());
+    }
+
     /// Number of operands controlling the loop
     unsigned getNumControlOperands() { return getLowerBound().size() * 3; }
 
     // Get Number of reduction operands
     unsigned getNumReduceOperands() {
-      return getReduceOperands().size();
+      return getReduceVars().size();
     }
 
     mlir::Operation::operand_range getLocalOperands() {
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ed062ecf07166..4f028bf558861 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Lower/Bridge.h"
 
+#include "OpenMP/ReductionProcessor.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -127,9 +128,8 @@ struct IncrementLoopInfo {
   bool isConcurrent;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localSymList;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localInitSymList;
-  llvm::SmallVector<
-      std::pair<fir::ReduceOperationEnum, const Fortran::semantics::Symbol *>>
-      reduceSymList;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> reduceSymList;
+  llvm::SmallVector<fir::ReduceOperationEnum> reduceOperatorList;
   llvm::SmallVector<const Fortran::semantics::Symbol *> sharedSymList;
   mlir::Value loopVariable = nullptr;
 
@@ -1993,7 +1993,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     case Fortran::parser::ReductionOperator::Operator::Ior:
       return fir::ReduceOperationEnum::IOR;
     case Fortran::parser::ReductionOperator::Operator::Ieor:
-      return fir::ReduceOperationEnum::EIOR;
+      return fir::ReduceOperationEnum::IEOR;
     }
     llvm_unreachable("illegal reduction operator");
   }
@@ -2027,8 +2027,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
               std::get<Fortran::parser::ReductionOperator>(reduceList->t));
           for (const Fortran::parser::Name &x :
                std::get<std::list<Fortran::parser::Name>>(reduceList->t)) {
-            info.reduceSymList.push_back(
-                std::make_pair(reduce_operation, x.symbol));
+            info.reduceSymList.push_back(x.symbol);
+            info.reduceOperatorList.push_back(reduce_operation);
           }
         }
       }
@@ -2089,6 +2089,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         assign.u = Fortran::evaluate::Assignment::BoundsSpec{};
       genAssignment(assign);
     }
+
     for (const Fortran::semantics::Symbol *sym : info.sharedSymList) {
       const auto *hostDetails =
           sym->detailsIf<Fortran::semantics::HostAssocDetails>();
@@ -2112,6 +2113,46 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       }
     }
 
+    llvm::SmallVector<bool> reduceVarByRef;
+    llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
+    llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
+
+    for (const auto &reduceOp : info.reduceOperatorList) {
+      nestReduceAttrs.push_back(
+          fir::ReduceAttr::get(builder->getContext(), reduceOp));
+    }
+
+    llvm::SmallVector<mlir::Value> reduceVars;
+    Fortran::lower::omp::ReductionProcessor rp;
+    rp.processReductionArguments<fir::DeclareReductionOp>(
+        toLocation(), *this, info.reduceOperatorList, reduceVars,
+        reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
+
+    doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
+    doConcurrentLoopOp.setReduceSymsAttr(
+        reductionDeclSymbols.empty()
+            ? nullptr
+            : mlir::ArrayAttr::get(builder->getContext(),
+                                   reductionDeclSymbols));
+    doConcurrentLoopOp.setReduceAttrsAttr(
+        nestReduceAttrs.empty()
+            ? nullptr
+            : mlir::ArrayAttr::get(builder->getContext(), nestReduceAttrs));
+    doConcurrentLoopOp.setReduceByrefAttr(
+        reduceVarByRef.empty() ? nullptr
+                               : mlir::DenseBoolArrayAttr::get(
+                                     builder->getContext(), reduceVarByRef));
+
+    for (auto [sym, reduceVar] :
+         llvm::zip_equal(info.reduceSymList, reduceVars)) {
+      auto arg = doConcurrentLoopOp.getRegion().begin()->addArgument(
+          reduceVar.getType(), doConcurrentLoopOp.getLoc());
+      bindSymbol(*sym, hlfir::translateToExtendedValue(
+                           reduceVar.getLoc(), *builder, hlfir::Entity{arg},
+                           /*contiguousHint=*/true)
+                           .first);
+    }
+
     // Note that allocatable, types with ultimate components, and type
     // requiring finalization are forbidden in LOCAL/LOCAL_INIT (F2023 C1130),
     // so no clean-up needs to be generated for these entities.
@@ -2203,6 +2244,12 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       }
     }
 
+    // Introduce a `do concurrent` scope to bind symbols corresponding to local,
+    // local_init, and reduce region arguments.
+    if (!incrementLoopNestInfo.empty() &&
+        incrementLoopNestInfo.back().isConcurrent)
+      localSymbols.pushScope();
+
     // Increment loop begin code. (Infinite/while code was already generated.)
     if (!infiniteLoop && !whileCondition)
       genFIRIncrementLoopBegin(incrementLoopNestInfo, doStmtEval.dirs);
@@ -2226,6 +2273,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     // This call may generate a branch in some contexts.
     genFIR(endDoEval, unstructuredContext);
+
+    if (!incrementLoopNestInfo.empty() &&
+        incrementLoopNestInfo.back().isConcurrent)
+      localSymbols.popScope();
   }
 
   /// Generate FIR to evaluate loop control values (lower, upper and step).
@@ -2408,19 +2459,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         info.stepVariable = builder->createTemporary(loc, stepValue.getType());
         builder->create<fir::StoreOp>(loc, stepValue, info.stepVariable);
       }
-
-      if (genDoConcurrent && nestReduceOperands.empty()) {
-        // Create DO CONCURRENT reduce operands and attributes
-        for (const auto &reduceSym : info.reduceSymList) {
-          const fir::ReduceOperationEnum reduceOperation = reduceSym.first;
-          const Fortran::semantics::Symbol *sym = reduceSym.second;
-          fir::ExtendedValue exv = getSymbolExtendedValue(*sym, nullptr);
-          nestReduceOperands.push_back(fir::getBase(exv));
-          auto reduceAttr =
-              fir::ReduceAttr::get(builder->getContext(), reduceOperation);
-          nestReduceAttrs.push_back(reduceAttr);
-        }
-      }
     }
 
     for (auto [info, lowerValue, upperValue, stepValue] :
@@ -2518,11 +2556,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
-          loc, nestLBs, nestUBs, nestSts, nestReduceOperands,
-          nestReduceAttrs.empty()
-              ? nullptr
-              : mlir::ArrayAttr::get(builder->getContext(), nestReduceAttrs),
-          nullptr, /*local_vars=*/std::nullopt, /*local_syms=*/nullptr);
+          loc, nestLBs, nestUBs, nestSts, nullptr, /*local_vars=*/std::nullopt,
+          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
+          /*reduce_attrs=*/nullptr);
 
       llvm::SmallVector<mlir::Type> loopBlockArgTypes(
           incrementLoopNestInfo.size(), builder->getIndexType());
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 7bea427099a28..5aebfc901e8ac 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -12,6 +12,7 @@
 
 #include "ClauseProcessor.h"
 #include "Clauses.h"
+#include "ReductionProcessor.h"
 #include "Utils.h"
 
 #include "flang/Lower/ConvertExprToHLFIR.h"
@@ -25,6 +26,21 @@ namespace Fortran {
 namespace lower {
 namespace omp {
 
+using ReductionModifier =
+    Fortran::lower::omp::clause::Reduction::ReductionModifier;
+
+mlir::omp::ReductionModifier translateReductionModifier(ReductionModifier mod) {
+  switch (mod) {
+  case ReductionModifier::Default:
+    return mlir::omp::ReductionModifier::defaultmod;
+  case ReductionModifier::Inscan:
+    return mlir::omp::ReductionModifier::inscan;
+  case ReductionModifier::Task:
+    return mlir::omp::ReductionModifier::task;
+  }
+  return mlir::omp::ReductionModifier::defaultmod;
+}
+
 /// Check for unsupported map operand types.
 static void checkMapType(mlir::Location location, mlir::Type type) {
   if (auto refType = mlir::dyn_cast<fir::ReferenceType>(type))
@@ -1076,6 +1092,18 @@ bool ClauseProcessor::processIf(
   });
   return found;
 }
+
+template <typename T>
+void collectReductionSyms(
+    const T &reduction,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
+  const auto &objectList{std::get<omp::ObjectList>(reduction.t)};
+  for (const Object &object : objectList) {
+    const semantics::Symbol *symbol = object.sym();
+    reductionSyms.push_back(symbol);
+  }
+}
+
 bool ClauseProcessor::processInReduction(
     mlir::Location currentLocation, mlir::omp::InReductionClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &outReductionSyms) const {
@@ -1085,10 +1113,14 @@ bool ClauseProcessor::processInReduction(
         llvm::SmallVector<bool> inReduceVarByRef;
         llvm::SmallVector<mlir::Attribute> inReductionDeclSymbols;
         llvm::SmallVector<const semantics::Symbol *> inReductionSyms;
+        collectReductionSyms(clause, inReductionSyms);
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::InReduction>(
-            currentLocation, converter, clause, inReductionVars,
-            inReduceVarByRef, inReductionDeclSymbols, inReductionSyms);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
+            inReductionSyms);
 
         // Copy local lists into the output.
         llvm::copy(inReductionVars, std::back_inserter(result.inReductionVars));
@@ -1416,10 +1448,23 @@ bool ClauseProcessor::processReduction(
         llvm::SmallVector<bool> reduceVarByRef;
         llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
         llvm::SmallVector<const semantics::Symbol *> reductionSyms;
+        collectReductionSyms(clause, reductionSyms);
+
+        auto mod = std::get<std::optional<ReductionModifier>>(clause.t);
+        if (mod.has_value()) {
+          if (mod.value() == ReductionModifier::Task)
+            TODO(currentLocation, "Reduction modifier `task` is not supported");
+          else
+            result.reductionMod = mlir::omp::ReductionModifierAttr::get(
+                converter.getFirOpBuilder().getContext(),
+                translateReductionModifier(mod.value()));
+        }
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::Reduction>(
-            currentLocation, converter, clause, reductionVars, reduceVarByRef,
-            reductionDeclSymbols, reductionSyms, &result.reductionMod);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            reductionVars, reduceVarByRef, reductionDeclSymbols, reductionSyms);
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
@@ -1435,21 +1480,25 @@ bool ClauseProcessor::processTaskReduction(
   return findRepeatableClause<omp::clause::TaskReduction>(
       [&](const omp::clause::TaskReduction &clause, const parser::CharBlock &) {
         llvm::SmallVector<mlir::Value> taskReductionVars;
-        llvm::SmallVector<bool> TaskReduceVarByRef;
-        llvm::SmallVector<mlir::Attribute> TaskReductionDeclSymbols;
-        llvm::SmallVector<const semantics::Symbol *> TaskReductionSyms;
+        llvm::SmallVector<bool> taskReduceVarByRef;
+        llvm::SmallVector<mlir::Attribute> taskReductionDeclSymbols;
+        llvm::SmallVector<const semantics::Symbol *> taskReductionSyms;
+        collectReductionSyms(clause, taskReductionSyms);
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::TaskReduction>(
-            currentLocation, converter, clause, taskReductionVars,
-            TaskReduceVarByRef, TaskReductionDeclSymbols, TaskReductionSyms);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
+            taskReductionSyms);
         // Copy local lists into the output.
         llvm::copy(taskReductionVars,
                    std::back_inserter(result.taskReductionVars));
-        llvm::copy(TaskReduceVarByRef,
+        llvm::copy(taskReduceVarByRef,
                    std::back_inserter(result.taskReductionByref));
-        llvm::copy(TaskReductionDeclSymbols,
+        llvm::copy(taskReductionDeclSymbols,
                    std::back_inserter(result.taskReductionSyms));
-        llvm::copy(TaskReductionSyms, std::back_inserter(outReductionSyms));
+        llvm::copy(taskReductionSyms, std::back_inserter(outReductionSyms));
       });
 }
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 3d8c4a337a4a4..46b749fb66c86 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -14,7 +14,6 @@
 
 #include "ClauseFinder.h"
 #include "Clauses.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/Bridge.h"
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index d7ab21d428e32..7f317f05f67b7 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -179,6 +179,7 @@ using IteratorSpecifier = tomp::type::IteratorSpecifierT<TypeTy, IdTy, ExprTy>;
 using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
 using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
 using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
+using ReductionOperatorList = List<ReductionOperator>;
 using DependenceType = tomp::type::DependenceType;
 using Prescriptiveness = tomp::type::Prescriptiveness;
 
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0a56e888ac44b..65e852cbcc911 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -17,7 +17,6 @@
 #include "Clauses.h"
 #include "DataSharingProcessor.h"
 #include "Decomposer.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Lower/Bridge.h"
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index 330cef7b54c74..d14fc1f7a52da 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -12,6 +12,7 @@
 
 #include "ReductionProcessor.h"
 
+#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/ConvertType.h"
 #include "flang/Lower/Support/PrivateReductionUtils.h"
@@ -21,8 +22,6 @@
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
-#include "flang/Optimizer/Support/FatalError.h"
-#include "flang/Parser/tools.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "llvm/Support/CommandLine.h"
 #include <type_traits>
@@ -40,35 +39,35 @@ namespace lower {
 namespace omp {
 
 // explicit template declarations
-template void
-ReductionProcessor::processReductionArguments<omp::clause::Reduction>(
+template void ReductionProcessor::processReductionArguments<
+    mlir::omp::DeclareReductionOp, omp::clause::ReductionOperatorList>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::Reduction &reduction,
+    const omp::clause::ReductionOperatorList &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void
-ReductionProcessor::processReductionArguments<omp::clause::TaskReduction>(
+template void ReductionProcessor::processReductionArguments<
+    fir::DeclareReductionOp, llvm::SmallVector<fir::ReduceOperationEnum>>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::TaskReduction &reduction,
+    const llvm::SmallVector<fir::ReduceOperationEnum> &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void
-ReductionProcessor::processReductionArguments<omp::clause::InReduction>(
-    mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::InReduction &reduction,
-    llvm::SmallVectorImpl<mlir::Value> &reductionVars,
-    llvm::SmallVectorImpl<bool> &reduceVarByRef,
-    llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+template mlir::omp::DeclareReductionOp
+ReductionProcessor::createDeclareReduction<mlir::omp::DeclareReductionOp>(
+    AbstractConverter &converter, llvm::StringRef reductionOpName,
+    const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
+    bool isByRef);
+
+template fir::DeclareReductionOp
+ReductionProcessor::createDeclareReduction<fir::DeclareReductionOp>(
+    AbstractConverter &converter, llvm::StringRef reductionOpName,
+    const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
+    bool isByRef);
 
 ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(
     const omp::clause::ProcedureDesignator &pd) {
@@ -106,6 +105,37 @@ ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(
   }
 }
 
+ReductionProcessor::ReductionIdentifier
+ReductionProcessor::getReductionType(const fir::ReduceOperationEnum &redOp) {
+  switch (redOp) {
+  case fir::ReduceOperationEnum::Add:
+    return ReductionIdentifier::ADD;
+  case fir::ReduceOperationEnum::Multiply:
+    return ReductionIdentifier::MULTIPLY;
+
+  case fir::ReduceOperationEnum::AND:
+    return ReductionIdentifier::AND;
+  case fir::ReduceOperationEnum::OR:
+    return ReductionIdentifier::OR;
+
+  case fir::ReduceOperationEnum::EQV:
+    return ReductionIdentifier::EQV;
+  case fir::ReduceOperationEnum::NEQV:
+    return ReductionIdentifier::NEQV;
+
+  case fir::ReduceOperationEnum::IAND:
+    return ReductionIdentifier::IAND;
+  case fir::ReduceOperationEnum::IEOR:
+    return ReductionIdentifier::IEOR;
+  case fir::ReduceOperationEnum::IOR:
+    return ReductionIdentifier::IOR;
+  case fir::ReduceOperationEnum::MAX:
+    return ReductionIdentifier::MAX;
+  case fir::ReduceOperationEnum::MIN:
+    return ReductionIdentifier::MIN;
+  }
+}
+
 bool ReductionProcessor::supportedIntrinsicProcReduction(
     const omp::clause::ProcedureDesignator &pd) {
   semantics::Symbol *sym = pd.v.sym();
@@ -136,28 +166,29 @@ ReductionProcessor::getReductionName(llvm::StringRef name,
   return fir::getTypeAsString(ty, kindMap, (name + byrefAddition).str());
 }
 
-std::string ReductionProcessor::getReductionName(
-    omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp,
-    const fir::KindMapping &kindMap, mlir::Type ty, bool isByRef) {
+std::string
+ReductionProcessor::getReductionName(ReductionIdentifier redId,
+                                     const fir::KindMapping &kindMap,
+                                     mlir::Type ty, bool isByRef) {
   std::string reductionName;
 
-  switch (intrinsicOp) {
-  case omp::clause::DefinedOperator::IntrinsicOperator::Add:
+  switch (redId) {
+  case ReductionIdentifier::ADD:
     reductionName = "add_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::Multiply:
+  case ReductionIdentifier::MULTIPLY:
     reductionName = "multiply_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::AND:
+  case ReductionIdentifier::AND:
     reductionName = "and_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::EQV:
+  case ReductionIdentifier::EQV:
     reductionName = "eqv_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::OR:
+  case ReductionIdentifier::OR:
     reductionName = "or_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:
+  case ReductionIdentifier::NEQV:
     reductionName = "neqv_reduction";
     break;
   default:
@@ -334,8 +365,18 @@ mlir::Value ReductionProcessor::createScalarCombiner(
   return reductionOp;
 }
 
+template <typename ParentDeclOpType>
+static void genYield(fir::FirOpBuilder &builder, mlir::Location loc,
+                     mlir::Value yieldedValue) {
+  if constexpr (std::is_same_v<ParentDeclOpType, mlir::omp::DeclareReductionOp>)
+    builder.create<mlir::omp::YieldOp>(loc, yieldedValue);
+  else
+    builder.create<fir::YieldOp>(loc, yieldedValue);
+}
+
 /// Create reduction combiner region for reduction variables which are boxed
 /// arrays
+template <typename DeclRedOpType>
 static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
                            ReductionProcessor::ReductionIdentifier redId,
                            fir::BaseBoxType boxTy, mlir::Value lhs,
@@ -369,7 +410,7 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value result = ReductionProcessor::createScalarCombiner(
         builder, loc, redId, eleTy, lhs, rhs);
     builder.create<fir::StoreOp>(loc, result, lhsValAddr);
-    builder.create<mlir::omp::YieldOp>(loc, lhsAddr);
+    genYield<DeclRedOpType>(builder, loc, lhsAddr);
     return;
   }
 
@@ -408,10 +449,11 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
   builder.create<fir::StoreOp>(loc, scalarReduction, lhsEleAddr);
 
   builder.setInsertionPointAfter(nest.outerOp);
-  builder.create<mlir::omp::YieldOp>(loc, lhsAddr);
+  genYield<DeclRedOpType>(builder, loc, lhsAddr);
 }
 
 // generate combiner region for reduction operations
+template <typename DeclRedOpType>
 static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
                         ReductionProcessor::ReductionIdentifier redId,
                         mlir::Type ty, mlir::Value lhs, mlir::Value rhs,
@@ -426,15 +468,15 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
         builder, loc, redId, ty, lhsLoaded, rhsLoaded);
     if (isByRef) {
       builder.create<fir::StoreOp>(loc, result, lhs);
-      builder.create<mlir::omp::YieldOp>(loc, lhs);
+      genYield<DeclRedOpType>(builder, loc, lhs);
     } else {
-      builder.create<mlir::omp::YieldOp>(loc, result);
+      genYield<DeclRedOpType>(builder, loc, result);
     }
     return;
   }
   // all arrays should have been boxed
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {
-    genBoxCombiner(builder, loc, redId, boxTy, lhs, rhs);
+    genBoxCombiner<DeclRedOpType>(builder, loc, redId, boxTy, lhs, rhs);
     return;
   }
 
@@ -454,15 +496,13 @@ static mlir::Type unwrapSeqOrBoxedType(mlir::Type ty) {
   return ty;
 }
 
+template <typename OpType>
 static void createReductionAllocAndInitRegions(
-    AbstractConverter &converter, mlir::Location loc,
-    mlir::omp::DeclareReductionOp &reductionDecl,
+    AbstractConverter &converter, mlir::Location loc, OpType &reductionDecl,
     const ReductionProcessor::ReductionIdentifier redId, mlir::Type type,
     bool isByRef) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  auto yield = [&](mlir::Value ret) {
-    builder.create<mlir::omp::YieldOp>(loc, ret);
-  };
+  auto yield = [&](mlir::Value ret) { genYield<OpType>(builder, loc, ret); };
 
   mlir::Block *allocBlock = nullptr;
   mlir::Block *initBlock = nullptr;
@@ -512,7 +552,8 @@ static void createReductionAllocAndInitRegions(
   yield(boxAlloca);
 }
 
-mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
+template <typename OpType>
+OpType ReductionProcessor::createDeclareReduction(
     AbstractConverter &converter, llvm::StringRef reductionOpName,
     const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
     bool isByRef) {
@@ -522,8 +563,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
 
   assert(!reductionOpName.empty());
 
-  auto decl =
-      module.lookupSymbol<mlir::omp::DeclareReductionOp>(reductionOpName);
+  auto decl = module.lookupSymbol<OpType>(reductionOpName);
   if (decl)
     return decl;
 
@@ -532,8 +572,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
   if (!isByRef)
     type = valTy;
 
-  decl = modBuilder.create<mlir::omp::DeclareReductionOp>(loc, reductionOpName,
-                                                          type);
+  decl = modBuilder.create<OpType>(loc, reductionOpName, type);
   createReductionAllocAndInitRegions(converter, loc, decl, redId, type,
                                      isByRef);
 
@@ -544,7 +583,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
   builder.setInsertionPointToEnd(&decl.getReductionRegion().back());
   mlir::Value op1 = decl.getReductionRegion().front().getArgument(0);
   mlir::Value op2 = decl.getReductionRegion().front().getArgument(1);
-  genCombiner(builder, loc, redId, type, op1, op2, isByRef);
+  genCombiner<OpType>(builder, loc, redId, type, op1, op2, isByRef);
 
   return decl;
 }
@@ -563,64 +602,41 @@ static bool doReductionByRef(mlir::Value reductionVar) {
   return false;
 }
 
-mlir::omp::ReductionModifier translateReductionModifier(ReductionModifier mod) {
-  switch (mod) {
-  case ReductionModifier::Default:
-    return mlir::omp::ReductionModifier::defaultmod;
-  case ReductionModifier::Inscan:
-    return mlir::omp::ReductionModifier::inscan;
-  case ReductionModifier::Task:
-    return mlir::omp::ReductionModifier::task;
-  }
-  return mlir::omp::ReductionModifier::defaultmod;
-}
-
-template <class T>
+template <typename OpType, typename RedOperatorListTy>
 void ReductionProcessor::processReductionArguments(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const T &reduction, llvm::SmallVectorImpl<mlir::Value> &reductionVars,
+    const RedOperatorListTy &redOperatorList,
+    llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
-  if constexpr (std::is_same_v<T, omp::clause::Reduction>) {
-    auto mod = std::get<std::optional<ReductionModifier>>(reduction.t);
-    if (mod.has_value()) {
-      if (mod.value() == ReductionModifier::Task)
-        TODO(currentLocation, "Reduction modifier `task` is not supported");
-      else
-        *reductionMod = mlir::omp::ReductionModifierAttr::get(
-            firOpBuilder.getContext(), translateReductionModifier(mod.value()));
-    }
-  }
-
-  mlir::omp::DeclareReductionOp decl;
-  const auto &redOperatorList{
-      std::get<typename T::ReductionIdentifiers>(reduction.t)};
-  assert(redOperatorList.size() == 1 && "Expecting single operator");
-  const auto &redOperator = redOperatorList.front();
-  const auto &objectList{std::get<omp::ObjectList>(reduction.t)};
-
-  if (!std::holds_alternative<omp::clause::DefinedOperator>(redOperator.u)) {
-    if (const auto *reductionIntrinsic =
-            std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
-      if (!ReductionProcessor::supportedIntrinsicProcReduction(
-              *reductionIntrinsic)) {
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols) {
+  if constexpr (std::is_same_v<RedOperatorListTy,
+                               omp::clause::ReductionOperatorList>) {
+    // For OpenMP reduction clauses, check if the reduction operator is
+    // supported.
+    assert(redOperatorList.size() == 1 && "Expecting single operator");
+    const Fortran::lower::omp::clause::ReductionOperator &redOperator =
+        redOperatorList.front();
+
+    if (!std::holds_alternative<omp::clause::DefinedOperator>(redOperator.u)) {
+      if (const auto *reductionIntrinsic =
+              std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
+        if (!ReductionProcessor::supportedIntrinsicProcReduction(
+                *reductionIntrinsic)) {
+          return;
+        }
+      } else {
         return;
       }
-    } else {
-      return;
     }
   }
 
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
   // Reduction variable processing common to both intrinsic operators and
   // procedure designators
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  for (const Object &object : objectList) {
-    const semantics::Symbol *symbol = object.sym();
-    reductionSymbols.push_back(symbol);
+  for (const semantics::Symbol *symbol : reductionSymbols) {
     mlir::Value symVal = converter.getSymbolAddress(*symbol);
     mlir::Type eleType;
     auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
@@ -672,52 +688,63 @@ void ReductionProcessor::processReductionArguments(
     reduceVarByRef.push_back(doReductionByRef(symVal));
   }
 
+  unsigned idx = 0;
   for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
     auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
     const auto &kindMap = firOpBuilder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
 
-    if (const auto &redDefinedOp =
-            std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
-      const auto &intrinsicOp{
-          std::get<omp::clause::DefinedOperator::IntrinsicOperator>(
-              redDefinedOp->u)};
-      redId = getReductionType(intrinsicOp);
-      switch (redId) {
-      case ReductionIdentifier::ADD:
-      case ReductionIdentifier::MULTIPLY:
-      case ReductionIdentifier::AND:
-      case ReductionIdentifier::EQV:
-      case ReductionIdentifier::OR:
-      case ReductionIdentifier::NEQV:
-        break;
-      default:
-        TODO(currentLocation,
-             "Reduction of some intrinsic operators is not supported");
-        break;
-      }
-
-      reductionName = getReductionName(intrinsicOp, kindMap, redType, isByRef);
-    } else if (const auto *reductionIntrinsic =
-                   std::get_if<omp::clause::ProcedureDesignator>(
-                       &redOperator.u)) {
-      if (!ReductionProcessor::supportedIntrinsicProcReduction(
-              *reductionIntrinsic)) {
-        TODO(currentLocation, "Unsupported intrinsic proc reduction");
+    if constexpr (std::is_same_v<RedOperatorListTy,
+                                 omp::clause::ReductionOperatorList>) {
+      const Fortran::lower::omp::clause::ReductionOperator &redOperator =
+          redOperatorList.front();
+      if (const auto &redDefinedOp =
+              std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
+        const auto &intrinsicOp{
+            std::get<omp::clause::DefinedOperator::IntrinsicOperator>(
+                redDefinedOp->u)};
+        redId = getReductionType(intrinsicOp);
+        switch (redId) {
+        case ReductionIdentifier::ADD:
+        case ReductionIdentifier::MULTIPLY:
+        case ReductionIdentifier::AND:
+        case ReductionIdentifier::EQV:
+        case ReductionIdentifier::OR:
+        case ReductionIdentifier::NEQV:
+          break;
+        default:
+          TODO(currentLocation,
+               "Reduction of some intrinsic operators is not supported");
+          break;
+        }
+
+        reductionName = getReductionName(redId, kindMap, redType, isByRef);
+      } else if (const auto *reductionIntrinsic =
+                     std::get_if<omp::clause::ProcedureDesignator>(
+                         &redOperator.u)) {
+        if (!ReductionProcessor::supportedIntrinsicProcReduction(
+                *reductionIntrinsic)) {
+          TODO(currentLocation, "Unsupported intrinsic proc reduction");
+        }
+        redId = getReductionType(*reductionIntrinsic);
+        reductionName =
+            getReductionName(getRealName(*reductionIntrinsic).ToString(),
+                             kindMap, redType, isByRef);
+      } else {
+        TODO(currentLocation, "Unexpected reduction type");
       }
-      redId = getReductionType(*reductionIntrinsic);
-      reductionName =
-          getReductionName(getRealName(*reductionIntrinsic).ToString(), kindMap,
-                           redType, isByRef);
     } else {
-      TODO(currentLocation, "Unexpected reduction type");
+      // `do concurrent` reductions
+      redId = getReductionType(redOperatorList[idx]);
+      reductionName = getReductionName(redId, kindMap, redType, isByRef);
     }
 
-    decl = createDeclareReduction(converter, reductionName, redId, redType,
-                                  currentLocation, isByRef);
+    OpType decl = createDeclareReduction<OpType>(
+        converter, reductionName, redId, redType, currentLocation, isByRef);
     reductionDeclSymbols.push_back(
         mlir::SymbolRefAttr::get(firOpBuilder.getContext(), decl.getSymName()));
+    ++idx;
   }
 }
 
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/lib/Lower/OpenMP/ReductionProcessor.h
index a7198b48f6b4e..95b4b077bdc46 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.h
@@ -19,7 +19,6 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/type.h"
-#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Types.h"
 
@@ -65,6 +64,9 @@ class ReductionProcessor {
   static ReductionIdentifier
   getReductionType(omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp);
 
+  static ReductionIdentifier
+  getReductionType(const fir::ReduceOperationEnum &pd);
+
   static bool
   supportedIntrinsicProcReduction(const omp::clause::ProcedureDesignator &pd);
 
@@ -78,10 +80,9 @@ class ReductionProcessor {
                                       const fir::KindMapping &kindMap,
                                       mlir::Type ty, bool isByRef);
 
-  static std::string
-  getReductionName(omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp,
-                   const fir::KindMapping &kindMap, mlir::Type ty,
-                   bool isByRef);
+  static std::string getReductionName(ReductionIdentifier redId,
+                                      const fir::KindMapping &kindMap,
+                                      mlir::Type ty, bool isByRef);
 
   /// This function returns the identity value of the operator \p
   /// reductionOpName. For example:
@@ -113,22 +114,23 @@ class ReductionProcessor {
   /// symbol table. The declaration has a constant initializer with the neutral
   /// value `initValue`, and the reduction combiner carried over from `reduce`.
   /// TODO: add atomic region.
-  static mlir::omp::DeclareReductionOp
-  createDeclareReduction(AbstractConverter &builder,
-                         llvm::StringRef reductionOpName,
-                         const ReductionIdentifier redId, mlir::Type type,
-                         mlir::Location loc, bool isByRef);
+  template <typename OpType>
+  static OpType createDeclareReduction(AbstractConverter &builder,
+                                       llvm::StringRef reductionOpName,
+                                       const ReductionIdentifier redId,
+                                       mlir::Type type, mlir::Location loc,
+                                       bool isByRef);
 
   /// Creates a reduction declaration and associates it with an OpenMP block
   /// directive.
-  template <class T>
+  template <typename OpType, typename RedOperatorListTy>
   static void processReductionArguments(
       mlir::Location currentLocation, lower::AbstractConverter &converter,
-      const T &reduction, llvm::SmallVectorImpl<mlir::Value> &reductionVars,
+      const RedOperatorListTy &redOperatorList,
+      llvm::SmallVectorImpl<mlir::Value> &reductionVars,
       llvm::SmallVectorImpl<bool> &reduceVarByRef,
       llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-      llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-      mlir::omp::ReductionModifierAttr *reductionMod = nullptr);
+      const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 };
 
 template <typename FloatOp, typename IntegerOp>
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index c65f51ce6cacd..b9d2574a76ad0 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -668,9 +668,7 @@ void privatizeSymbol(
 
   const semantics::Symbol *sym =
       isDoConcurrent ? &symToPrivatize->GetUltimate() : symToPrivatize;
-  const lower::SymbolBox hsb = isDoConcurrent
-                                   ? converter.shallowLookupSymbol(*sym)
-                                   : converter.lookupOneLevelUpSymbol(*sym);
+  const lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
   assert(hsb && "Host symbol box not found");
 
   mlir::Location symLoc = hsb.getAddr().getLoc();
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 1bc981516e226..3873eeaa6beea 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -3341,26 +3341,26 @@ struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
   }
 };
 
-struct LocalitySpecifierOpConversion
-    : public fir::FIROpConversion<fir::LocalitySpecifierOp> {
-  using FIROpConversion::FIROpConversion;
+template <typename OpTy>
+struct DoConcurrentSpecifierOpConversion : public fir::FIROpConversion<OpTy> {
+  using fir::FIROpConversion<OpTy>::FIROpConversion;
   llvm::LogicalResult
-  matchAndRewrite(fir::LocalitySpecifierOp localizer, OpAdaptor adaptor,
+  matchAndRewrite(OpTy specifier, typename OpTy::Adaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
 #ifdef EXPENSIVE_CHECKS
     auto uses = mlir::SymbolTable::getSymbolUses(
-        localizer, localizer->getParentOfType<mlir::ModuleOp>());
+        specifier, specifier->getParentOfType<mlir::ModuleOp>());
 
-    // `fir.local` ops are not supposed to have any uses at this point (i.e.
-    // during lowering to LLVM). In case of serialization, the
-    // `fir.do_concurrent` users are expected to have been lowered to
+    // `fir.local|fir.declare_reduction` ops are not supposed to have any uses
+    // at this point (i.e. during lowering to LLVM). In case of serialization,
+    // the `fir.do_concurrent` users are expected to have been lowered to
     // `fir.do_loop` nests. In case of parallelization, the `fir.do_concurrent`
     // users are expected to have been lowered to the target parallel model
     // (e.g. OpenMP).
     assert(uses && uses->empty());
 #endif
 
-    rewriter.eraseOp(localizer);
+    rewriter.eraseOp(specifier);
     return mlir::success();
   }
 };
@@ -4328,20 +4328,22 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, VolatileCastOpConversion, ConvertOpConversion,
       CoordinateOpConversion, CopyOpConversion, DTEntryOpConversion,
-      DeclareOpConversion, DivcOpConversion, EmboxOpConversion,
-      EmboxCharOpConversion, EmboxProcOpConversion, ExtractValueOpConversion,
-      FieldIndexOpConversion, FirEndOpConversion, FreeMemOpConversion,
-      GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
-      IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
-      LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
-      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
-      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
-      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
-      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
-      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
-      XArrayCoorOpConversion, XEmboxOpConversion, XReboxOpConversion,
-      ZeroOpConversion>(converter, options);
+      DeclareOpConversion,
+      DoConcurrentSpecifierOpConversion<fir::LocalitySpecifierOp>,
+      DoConcurrentSpecifierOpConversion<fir::DeclareReductionOp>,
+      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
+      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
+      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
+      GlobalOpConversion, InsertOnRangeOpConversion, IsPresentOpConversion,
+      LenParamIndexOpConversion, LoadOpConversion, MulcOpConversion,
+      NegcOpConversion, NoReassocOpConversion, SelectCaseOpConversion,
+      SelectOpConversion, SelectRankOpConversion, SelectTypeOpConversion,
+      ShapeOpConversion, ShapeShiftOpConversion, ShiftOpConversion,
+      SliceOpConversion, StoreOpConversion, StringLitOpConversion,
+      SubcOpConversion, TypeDescOpConversion, TypeInfoOpConversion,
+      UnboxCharOpConversion, UnboxProcOpConversion, UndefOpConversion,
+      UnreachableOpConversion, XArrayCoorOpConversion, XEmboxOpConversion,
+      XReboxOpConversion, ZeroOpConversion>(converter, options);
 
   // Patterns that are populated without a type converter do not trigger
   // target materializations for the operands of the root op.
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..674129f546e94 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -5041,6 +5041,9 @@ void fir::BoxTotalElementsOp::getCanonicalizationPatterns(
 // LocalitySpecifierOp
 //===----------------------------------------------------------------------===//
 
+// TODO This is a copy of omp::PrivateClauseOp::verifiyRegions(). Once we find a
+// solution to merge both ops into one this duplication will not be needed. See:
+// https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
 llvm::LogicalResult fir::LocalitySpecifierOp::verifyRegions() {
   mlir::Type argType = getArgType();
   auto verifyTerminator = [&](mlir::Operation *terminator,
@@ -5136,6 +5139,84 @@ llvm::LogicalResult fir::LocalitySpecifierOp::verifyRegions() {
   return llvm::success();
 }
 
+// TODO This is a copy of omp::DeclareReductionOp::verifiyRegions(). Once we
+// find a solution to merge both ops into one this duplication will not be
+// needed.
+mlir::LogicalResult fir::DeclareReductionOp::verifyRegions() {
+  if (!getAllocRegion().empty()) {
+    for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
+      if (yieldOp.getResults().size() != 1 ||
+          yieldOp.getResults().getTypes()[0] != getType())
+        return emitOpError() << "expects alloc region to yield a value "
+                                "of the reduction type";
+    }
+  }
+
+  if (getInitializerRegion().empty())
+    return emitOpError() << "expects non-empty initializer region";
+  mlir::Block &initializerEntryBlock = getInitializerRegion().front();
+
+  if (initializerEntryBlock.getNumArguments() == 1) {
+    if (!getAllocRegion().empty())
+      return emitOpError() << "expects two arguments to the initializer region "
+                              "when an allocation region is used";
+  } else if (initializerEntryBlock.getNumArguments() == 2) {
+    if (getAllocRegion().empty())
+      return emitOpError() << "expects one argument to the initializer region "
+                              "when no allocation region is used";
+  } else {
+    return emitOpError()
+           << "expects one or two arguments to the initializer region";
+  }
+
+  for (mlir::Value arg : initializerEntryBlock.getArguments())
+    if (arg.getType() != getType())
+      return emitOpError() << "expects initializer region argument to match "
+                              "the reduction type";
+
+  for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
+    if (yieldOp.getResults().size() != 1 ||
+        yieldOp.getResults().getTypes()[0] != getType())
+      return emitOpError() << "expects initializer region to yield a value "
+                              "of the reduction type";
+  }
+
+  if (getReductionRegion().empty())
+    return emitOpError() << "expects non-empty reduction region";
+  mlir::Block &reductionEntryBlock = getReductionRegion().front();
+  if (reductionEntryBlock.getNumArguments() != 2 ||
+      reductionEntryBlock.getArgumentTypes()[0] !=
+          reductionEntryBlock.getArgumentTypes()[1] ||
+      reductionEntryBlock.getArgumentTypes()[0] != getType())
+    return emitOpError() << "expects reduction region with two arguments of "
+                            "the reduction type";
+  for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
+    if (yieldOp.getResults().size() != 1 ||
+        yieldOp.getResults().getTypes()[0] != getType())
+      return emitOpError() << "expects reduction region to yield a value "
+                              "of the reduction type";
+  }
+
+  if (!getAtomicReductionRegion().empty()) {
+    mlir::Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();
+    if (atomicReductionEntryBlock.getNumArguments() != 2 ||
+        atomicReductionEntryBlock.getArgumentTypes()[0] !=
+            atomicReductionEntryBlock.getArgumentTypes()[1])
+      return emitOpError() << "expects atomic reduction region with two "
+                              "arguments of the same type";
+  }
+
+  if (getCleanupRegion().empty())
+    return mlir::success();
+  mlir::Block &cleanupEntryBlock = getCleanupRegion().front();
+  if (cleanupEntryBlock.getNumArguments() != 1 ||
+      cleanupEntryBlock.getArgument(0).getType() != getType())
+    return emitOpError() << "expects cleanup region with one argument "
+                            "of the reduction type";
+
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // DoConcurrentOp
 //===----------------------------------------------------------------------===//
@@ -5157,6 +5238,86 @@ llvm::LogicalResult fir::DoConcurrentOp::verify() {
 // DoConcurrentLoopOp
 //===----------------------------------------------------------------------===//
 
+static mlir::ParseResult parseSpecifierList(
+    mlir::OpAsmParser &parser, mlir::OperationState &result,
+    llvm::StringRef specifierKeyword, llvm::StringRef symsAttrName,
+    llvm::SmallVectorImpl<mlir::OpAsmParser::Argument> &regionArgs,
+    llvm::SmallVectorImpl<mlir::Type> &regionArgTypes,
+    int32_t &numSpecifierOperands, bool isReduce = false) {
+  auto &builder = parser.getBuilder();
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> specifierOperands;
+
+  if (failed(parser.parseOptionalKeyword(specifierKeyword)))
+    return mlir::success();
+
+  std::size_t oldArgTypesSize = regionArgTypes.size();
+  if (failed(parser.parseLParen()))
+    return mlir::failure();
+
+  llvm::SmallVector<mlir::SymbolRefAttr> spceifierSymbolVec;
+  llvm::SmallVector<fir::ReduceAttr> attributes;
+
+  if (failed(parser.parseCommaSeparatedList([&]() {
+        if (failed(parser.parseAttribute(spceifierSymbolVec.emplace_back())))
+          return mlir::failure();
+
+        if (isReduce &&
+            failed(parser.parseAttribute(attributes.emplace_back()))) {
+          return mlir::failure();
+        }
+
+        if (parser.parseOperand(specifierOperands.emplace_back()) ||
+            parser.parseArrow() ||
+            parser.parseArgument(regionArgs.emplace_back()))
+          return mlir::failure();
+
+        return mlir::success();
+      })))
+    return mlir::failure();
+
+  if (failed(parser.parseColon()))
+    return mlir::failure();
+
+  if (failed(parser.parseCommaSeparatedList([&]() {
+        if (failed(parser.parseType(regionArgTypes.emplace_back())))
+          return mlir::failure();
+
+        return mlir::success();
+      })))
+    return mlir::failure();
+
+  if (regionArgs.size() != regionArgTypes.size())
+    return parser.emitError(parser.getNameLoc(), "mismatch in number of " +
+                                                     specifierKeyword.str() +
+                                                     " arg and types");
+
+  if (failed(parser.parseRParen()))
+    return mlir::failure();
+
+  for (auto operandType :
+       llvm::zip_equal(specifierOperands,
+                       llvm::drop_begin(regionArgTypes, oldArgTypesSize)))
+    if (parser.resolveOperand(std::get<0>(operandType),
+                              std::get<1>(operandType), result.operands))
+      return mlir::failure();
+
+  llvm::SmallVector<mlir::Attribute> symbolAttrs(spceifierSymbolVec.begin(),
+                                                 spceifierSymbolVec.end());
+  result.addAttribute(symsAttrName, builder.getArrayAttr(symbolAttrs));
+
+  if (isReduce) {
+    llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
+                                                 attributes.end());
+    result.addAttribute(
+        fir::DoConcurrentLoopOp::getReduceAttrsAttrName(result.name),
+        builder.getArrayAttr(arrayAttr));
+  }
+
+  numSpecifierOperands = specifierOperands.size();
+
+  return mlir::success();
+}
+
 mlir::ParseResult fir::DoConcurrentLoopOp::parse(mlir::OpAsmParser &parser,
                                                  mlir::OperationState &result) {
   auto &builder = parser.getBuilder();
@@ -5192,90 +5353,26 @@ mlir::ParseResult fir::DoConcurrentLoopOp::parse(mlir::OpAsmParser &parser,
       parser.resolveOperands(steps, builder.getIndexType(), result.operands))
     return mlir::failure();
 
-  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> reduceOperands;
-  llvm::SmallVector<mlir::Type> reduceArgTypes;
-  if (succeeded(parser.parseOptionalKeyword("reduce"))) {
-    // Parse reduction attributes and variables.
-    llvm::SmallVector<fir::ReduceAttr> attributes;
-    if (failed(parser.parseCommaSeparatedList(
-            mlir::AsmParser::Delimiter::Paren, [&]() {
-              if (parser.parseAttribute(attributes.emplace_back()) ||
-                  parser.parseArrow() ||
-                  parser.parseOperand(reduceOperands.emplace_back()) ||
-                  parser.parseColonType(reduceArgTypes.emplace_back()))
-                return mlir::failure();
-              return mlir::success();
-            })))
-      return mlir::failure();
-    // Resolve input operands.
-    for (auto operand_type : llvm::zip(reduceOperands, reduceArgTypes))
-      if (parser.resolveOperand(std::get<0>(operand_type),
-                                std::get<1>(operand_type), result.operands))
-        return mlir::failure();
-    llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
-                                                 attributes.end());
-    result.addAttribute(getReduceAttrsAttrName(result.name),
-                        builder.getArrayAttr(arrayAttr));
-  }
-
-  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> localOperands;
-  if (succeeded(parser.parseOptionalKeyword("local"))) {
-    std::size_t oldArgTypesSize = argTypes.size();
-    if (failed(parser.parseLParen()))
-      return mlir::failure();
-
-    llvm::SmallVector<mlir::SymbolRefAttr> localSymbolVec;
-    if (failed(parser.parseCommaSeparatedList([&]() {
-          if (failed(parser.parseAttribute(localSymbolVec.emplace_back())))
-            return mlir::failure();
-
-          if (parser.parseOperand(localOperands.emplace_back()) ||
-              parser.parseArrow() ||
-              parser.parseArgument(regionArgs.emplace_back()))
-            return mlir::failure();
-
-          return mlir::success();
-        })))
-      return mlir::failure();
-
-    if (failed(parser.parseColon()))
-      return mlir::failure();
-
-    if (failed(parser.parseCommaSeparatedList([&]() {
-          if (failed(parser.parseType(argTypes.emplace_back())))
-            return mlir::failure();
-
-          return mlir::success();
-        })))
-      return mlir::failure();
-
-    if (regionArgs.size() != argTypes.size())
-      return parser.emitError(parser.getNameLoc(),
-                              "mismatch in number of local arg and types");
-
-    if (failed(parser.parseRParen()))
-      return mlir::failure();
-
-    for (auto operandType : llvm::zip_equal(
-             localOperands, llvm::drop_begin(argTypes, oldArgTypesSize)))
-      if (parser.resolveOperand(std::get<0>(operandType),
-                                std::get<1>(operandType), result.operands))
-        return mlir::failure();
+  int32_t numLocalOperands = 0;
+  if (failed(parseSpecifierList(parser, result, "local",
+                                getLocalSymsAttrName(result.name), regionArgs,
+                                argTypes, numLocalOperands)))
+    return mlir::failure();
 
-    llvm::SmallVector<mlir::Attribute> symbolAttrs(localSymbolVec.begin(),
-                                                   localSymbolVec.end());
-    result.addAttribute(getLocalSymsAttrName(result.name),
-                        builder.getArrayAttr(symbolAttrs));
-  }
+  int32_t numReduceOperands = 0;
+  if (failed(parseSpecifierList(
+          parser, result, "reduce", getReduceSymsAttrName(result.name),
+          regionArgs, argTypes, numReduceOperands, /*isReduce=*/true)))
+    return mlir::failure();
 
   // Set `operandSegmentSizes` attribute.
-  result.addAttribute(DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
-                      builder.getDenseI32ArrayAttr(
-                          {static_cast<int32_t>(lower.size()),
-                           static_cast<int32_t>(upper.size()),
-                           static_cast<int32_t>(steps.size()),
-                           static_cast<int32_t>(reduceOperands.size()),
-                           static_cast<int32_t>(localOperands.size())}));
+  result.addAttribute(
+      DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
+      builder.getDenseI32ArrayAttr({static_cast<int32_t>(lower.size()),
+                                    static_cast<int32_t>(upper.size()),
+                                    static_cast<int32_t>(steps.size()),
+                                    static_cast<int32_t>(numLocalOperands),
+                                    static_cast<int32_t>(numReduceOperands)}));
 
   // Now parse the body.
   for (auto [arg, type] : llvm::zip_equal(regionArgs, argTypes))
@@ -5297,17 +5394,6 @@ void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
     << ") = (" << getLowerBound() << ") to (" << getUpperBound() << ") step ("
     << getStep() << ")";
 
-  if (!getReduceOperands().empty()) {
-    p << " reduce(";
-    auto attrs = getReduceAttrsAttr();
-    auto operands = getReduceOperands();
-    llvm::interleaveComma(llvm::zip(attrs, operands), p, [&](auto it) {
-      p << std::get<0>(it) << " -> " << std::get<1>(it) << " : "
-        << std::get<1>(it).getType();
-    });
-    p << ')';
-  }
-
   if (!getLocalVars().empty()) {
     p << " local(";
     llvm::interleaveComma(llvm::zip_equal(getLocalSymsAttr(), getLocalVars(),
@@ -5322,13 +5408,30 @@ void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
     p << ")";
   }
 
+  if (!getReduceVars().empty()) {
+    p << " reduce(";
+    llvm::interleaveComma(
+        llvm::zip_equal(getReduceSymsAttr(), getReduceAttrsAttr(),
+                        getReduceVars(), getRegionReduceArgs()),
+        p, [&](auto it) {
+          p << std::get<0>(it) << " " << std::get<1>(it) << " "
+            << std::get<2>(it) << " -> " << std::get<3>(it);
+        });
+    p << " : ";
+    llvm::interleaveComma(getReduceVars(), p,
+                          [&](auto it) { p << it.getType(); });
+    p << ")";
+  }
+
   p << ' ';
   p.printRegion(getRegion(), /*printEntryBlockArgs=*/false);
   p.printOptionalAttrDict(
       (*this)->getAttrs(),
       /*elidedAttrs=*/{DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
+                       DoConcurrentLoopOp::getLocalSymsAttrName(),
+                       DoConcurrentLoopOp::getReduceSymsAttrName(),
                        DoConcurrentLoopOp::getReduceAttrsAttrName(),
-                       DoConcurrentLoopOp::getLocalSymsAttrName()});
+                       DoConcurrentLoopOp::getReduceByrefAttrName()});
 }
 
 llvm::SmallVector<mlir::Region *> fir::DoConcurrentLoopOp::getLoopRegions() {
@@ -5340,6 +5443,7 @@ llvm::LogicalResult fir::DoConcurrentLoopOp::verify() {
   mlir::Operation::operand_range ubValues = getUpperBound();
   mlir::Operation::operand_range stepValues = getStep();
   mlir::Operation::operand_range localVars = getLocalVars();
+  mlir::Operation::operand_range reduceVars = getReduceVars();
 
   if (lbValues.empty())
     return emitOpError(
@@ -5353,7 +5457,8 @@ llvm::LogicalResult fir::DoConcurrentLoopOp::verify() {
   // Check that the body defines the same number of block arguments as the
   // number of tuple elements in step.
   mlir::Block *body = getBody();
-  unsigned numIndVarArgs = body->getNumArguments() - localVars.size();
+  unsigned numIndVarArgs =
+      body->getNumArguments() - localVars.size() - reduceVars.size();
 
   if (numIndVarArgs != stepValues.size())
     return emitOpError() << "expects the same number of induction variables: "
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index e440852b3103a..506c8e66dbdfa 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -234,6 +234,10 @@ class DoConcurrentConversion
       loop.setLocalSymsAttr(nullptr);
     }
 
+    for (auto [reduceVar, reduceArg] :
+         llvm::zip_equal(loop.getReduceVars(), loop.getRegionReduceArgs()))
+      rewriter.replaceAllUsesWith(reduceArg, reduceVar);
+
     // Collect iteration variable(s) allocations so that we can move them
     // outside the `fir.do_concurrent` wrapper.
     llvm::SmallVector<mlir::Operation *> opsToMove;
@@ -257,12 +261,16 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceOperands(),
+          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
     }
 
+    loop.getRegion().front().eraseArguments(loop.getNumInductionVars() +
+                                                loop.getNumLocalOperands(),
+                                            loop.getNumReduceOperands());
+
     rewriter.inlineBlockBefore(
         &loopBlock, innermostUnorderdLoop.getBody()->getTerminator(), ivArgs);
     rewriter.eraseOp(doConcurentOp);
diff --git a/flang/test/Fir/do_concurrent.fir b/flang/test/Fir/do_concurrent.fir
index cc1197ba56bd7..c207ea5dea825 100644
--- a/flang/test/Fir/do_concurrent.fir
+++ b/flang/test/Fir/do_concurrent.fir
@@ -63,7 +63,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
     %j = fir.alloca i32
     fir.do_concurrent.loop
       (%i_iv, %j_iv) = (%i_lb, %j_lb) to (%i_ub, %j_ub) step (%i_st, %j_st) 
-      reduce(#fir.reduce_attr<add> -> %sum : !fir.ref<i32>) {
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %sum -> %sum_arg : !fir.ref<i32>) {
       %0 = fir.convert %i_iv : (index) -> i32
       fir.store %0 to %i : !fir.ref<i32>
 
@@ -83,7 +83,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
 // CHECK:           %[[I:.*]] = fir.alloca i32
 // CHECK:           %[[J:.*]] = fir.alloca i32
 // CHECK:           fir.do_concurrent.loop
-// CHECK-SAME:        (%[[I_IV:.*]], %[[J_IV:.*]]) = (%[[I_LB]], %[[J_LB]]) to (%[[I_UB]], %[[J_UB]]) step (%[[I_ST]], %[[J_ST]]) reduce(#fir.reduce_attr<add> -> %[[SUM]] : !fir.ref<i32>) {
+// CHECK-SAME:        (%[[I_IV:.*]], %[[J_IV:.*]]) = (%[[I_LB]], %[[J_LB]]) to (%[[I_UB]], %[[J_UB]]) step (%[[I_ST]], %[[J_ST]]) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[SUM]] -> %{{.*}} : !fir.ref<i32>) {
 // CHECK:             %[[I_IV_CVT:.*]] = fir.convert %[[I_IV]] : (index) -> i32
 // CHECK:             fir.store %[[I_IV_CVT]] to %[[I]] : !fir.ref<i32>
 // CHECK:             %[[J_IV_CVT:.*]] = fir.convert %[[J_IV]] : (index) -> i32
@@ -161,3 +161,62 @@ func.func @do_concurrent_with_locality_specs() {
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @dc_reduce() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c1 = arith.constant 1 : index 
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(@add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1 : !fir.ref<i32>) {
+    }
+  }
+  return
+}
+
+// CHECK-LABEL:  func.func @dc_reduce() {
+// CHECK:          %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+// CHECK:          %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:          fir.do_concurrent {
+// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
+// CHECK:            }
+// CHECK:          }
+// CHECK:          return
+// CHECK:        }
+
+func.func @dc_reduce_2() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  %5 = fir.alloca i32 {bindc_name = "m", uniq_name = "dc_reduce"}
+  %6:2 = hlfir.declare %5 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  %c1 = arith.constant 1 : index 
+
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) 
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1,
+             @mul_reduction_i32 #fir.reduce_attr<multiply> %6#0 -> %arg2
+             : !fir.ref<i32>, !fir.ref<i32>) {
+    }
+  }
+
+  return
+}
+
+// CHECK-LABEL:  func.func @dc_reduce_2() {
+// CHECK:          %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+// CHECK:          %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+// CHECK:          %[[M_ALLOC:.*]] = fir.alloca i32 {bindc_name = "m", uniq_name = "dc_reduce"}
+// CHECK:          %[[M_DECL:.*]]:2 = hlfir.declare %[[M_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:          fir.do_concurrent {
+// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{[^[:space:]]+}}) 
+// CHECK-SAME:         reduce(
+// CHECK-SAME:           @add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:[^,]+]],
+// CHECK-SAME:           @mul_reduction_i32 #fir.reduce_attr<multiply> %[[M_DECL]]#0 -> %[[M_ARG:[^[:space:]]+]]
+// CHECK-SAME:           : !fir.ref<i32>, !fir.ref<i32>) {
+// CHECK:            }
+// CHECK:          }
+// CHECK:          return
+// CHECK:        }
+
diff --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index aca0ecc1abdc1..e32ea7ad3c729 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -1256,8 +1256,8 @@ func.func @dc_invalid_reduction(%arg0: index, %arg1: index) {
   %sum = fir.alloca i32
   // expected-error at +2 {{'fir.do_concurrent.loop' op mismatch in number of reduction variables and reduction attributes}}
   fir.do_concurrent {
-    "fir.do_concurrent.loop"(%arg0, %arg1, %arg0, %sum) <{operandSegmentSizes = array<i32: 1, 1, 1, 1, 0>}> ({
-    ^bb0(%arg3: index):
+    "fir.do_concurrent.loop"(%arg0, %arg1, %arg0, %sum) <{operandSegmentSizes = array<i32: 1, 1, 1, 0, 1>}> ({
+    ^bb0(%arg3: index, %sum_arg: i32):
       %tmp = "fir.alloca"() <{in_type = i32, operandSegmentSizes = array<i32: 0, 0>}> : () -> !fir.ref<i32>
     }) : (index, index, index, !fir.ref<i32>) -> ()
   }
@@ -1266,6 +1266,20 @@ func.func @dc_invalid_reduction(%arg0: index, %arg1: index) {
 
 // -----
 
+func.func @dc_reduce_no_attr() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c1 = arith.constant 1 : index 
+  // expected-error at +2 {{expected attribute value}}
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(@add_reduction_i32 %4#0 -> %arg1 : !fir.ref<i32>) {
+    }
+  }
+  return
+}
+
+// -----
+
 // Should fail when volatility changes from a fir.convert
 func.func @bad_convert_volatile(%arg0: !fir.ref<i32>) -> !fir.ref<i32, volatile> {
   // expected-error at +1 {{op this conversion does not preserve volatility}}
diff --git a/flang/test/Lower/do_concurrent_reduce.f90 b/flang/test/Lower/do_concurrent_reduce.f90
new file mode 100644
index 0000000000000..8591a21e2b9e0
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_reduce.f90
@@ -0,0 +1,41 @@
+! RUN: %flang_fc1 -emit-hlfir -mmlir --enable-delayed-privatization-staging=true -o - %s | FileCheck %s
+
+subroutine do_concurrent_reduce
+  implicit none
+  integer :: s, i
+
+  do concurrent (i=1:10) reduce(+:s)
+    s = s + 1
+  end do
+end
+
+! CHECK-LABEL:  fir.declare_reduction @add_reduction_i32 : i32 init {
+! CHECK:        ^bb0(%[[ARG0:.*]]: i32):
+! CHECK:          %[[VAL_0:.*]] = arith.constant 0 : i32
+! CHECK:          fir.yield(%[[VAL_0]] : i32)
+! CHECK:        } combiner {
+! CHECK:          ^bb0(%[[VAL_1:.*]]: i32, %[[VAL_2:.*]]: i32):
+! CHECK:          %[[VAL_3:.*]] = arith.addi %[[VAL_1]], %[[VAL_2]] : i32
+! CHECK:          fir.yield(%[[VAL_3]] : i32)
+! CHECK:        }
+
+! CHECK-LABEL:  func.func @_QPdo_concurrent_reduce() {
+! CHECK:           %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "_QFdo_concurrent_reduceEs"}
+! CHECK:           %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "_QFdo_concurrent_reduceEs"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+! CHECK:           fir.do_concurrent {
+! CHECK:             %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i"}
+! CHECK:             %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFdo_concurrent_reduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:             fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{[^[:space:]]+}}) 
+! CHECK-SAME:          reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
+
+! CHECK:               %[[S_ARG_DECL:.*]]:2 = hlfir.declare %[[S_ARG]] {uniq_name = "_QFdo_concurrent_reduceEs"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:               %[[S_ARG_VAL:.*]] = fir.load %[[S_ARG_DECL]]#0 : !fir.ref<i32>
+! CHECK:               %[[C1:.*]] = arith.constant 1 : i32
+! CHECK:               %[[RED_UPDATE:.*]] = arith.addi %[[S_ARG_VAL]], %[[C1]] : i32
+! CHECK:               hlfir.assign %[[RED_UPDATE]] to %[[S_ARG_DECL]]#0 : i32, !fir.ref<i32>
+
+! CHECK:             }
+! CHECK:           }
+! CHECK:           return
+! CHECK:        }
diff --git a/flang/test/Lower/loops.f90 b/flang/test/Lower/loops.f90
index 60df27a591dc3..64f14ff972272 100644
--- a/flang/test/Lower/loops.f90
+++ b/flang/test/Lower/loops.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
+! RUN: bbc -emit-fir -hlfir=false --enable-delayed-privatization=false -o - %s | FileCheck %s
 
 ! CHECK-LABEL: loop_test
 subroutine loop_test
diff --git a/flang/test/Lower/loops3.f90 b/flang/test/Lower/loops3.f90
index 84db1972cca16..2965b954b49a8 100644
--- a/flang/test/Lower/loops3.f90
+++ b/flang/test/Lower/loops3.f90
@@ -12,7 +12,7 @@ subroutine loop_test
 
   ! CHECK: %[[VAL_0:.*]] = fir.alloca f32 {bindc_name = "m", uniq_name = "_QFloop_testEm"}
   ! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QFloop_testEsum) : !fir.ref<i32>
-  ! CHECK: fir.do_concurrent.loop ({{.*}}) = ({{.*}}) to ({{.*}}) step ({{.*}}) reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
+  ! CHECK: fir.do_concurrent.loop ({{.*}}) = ({{.*}}) to ({{.*}}) step ({{.*}}) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[VAL_1]] -> %{{.*}}, @other_reduction_f32 #fir.reduce_attr<max> %[[VAL_0]] -> %{{.*}} : {{.*}}) {
   do concurrent (i=1:5, j=1:5, k=1:5) local(tmp) reduce(+:sum) reduce(max:m)
     tmp = i + j + k
     sum = tmp + sum
diff --git a/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir b/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
index d9ef36b175598..c550ab8a97d4c 100644
--- a/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
+++ b/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
@@ -86,7 +86,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
     %j = fir.alloca i32
     fir.do_concurrent.loop
       (%i_iv, %j_iv) = (%i_lb, %j_lb) to (%i_ub, %j_ub) step (%i_st, %j_st) 
-      reduce(#fir.reduce_attr<add> -> %sum : !fir.ref<i32>) {
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %sum -> %sum_arg : !fir.ref<i32>) {
       %0 = fir.convert %i_iv : (index) -> i32
       fir.store %0 to %i : !fir.ref<i32>
 

>From acd5ea091b2657764276d4a3ccdfd22bad8c90d8 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Thu, 26 Jun 2025 23:17:37 -0500
Subject: [PATCH 2/3] review comments

---
 flang/include/flang/Optimizer/Dialect/FIROps.td | 4 ++--
 flang/lib/Lower/Bridge.cpp                      | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index b217f1433767f..41363234e5c6b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3667,10 +3667,10 @@ def fir_DeclareReductionOp : fir_Op<"declare_reduction", [IsolatedFromAbove,
   let summary = "declares a reduction kind";
   let description = [{
     Note: this operation is adapted from omp::DeclareReductionOp. There is a lot
-    duplication at the moment. TODO Combined both ops into one. See:
+    duplication at the moment. TODO Combine both ops into one. See:
     https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
 
-    Declares an `do concurrent` reduction. This requires two mandatory and three
+    Declares a `do concurrent` reduction. This requires two mandatory and three
     optional regions.
 
       1. The optional alloc region specifies how to allocate the thread-local
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4f028bf558861..4b00392a75891 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2117,10 +2117,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
     llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
 
-    for (const auto &reduceOp : info.reduceOperatorList) {
+    for (const auto &reduceOp : info.reduceOperatorList)
       nestReduceAttrs.push_back(
           fir::ReduceAttr::get(builder->getContext(), reduceOp));
-    }
 
     llvm::SmallVector<mlir::Value> reduceVars;
     Fortran::lower::omp::ReductionProcessor rp;
@@ -2556,7 +2555,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
-          loc, nestLBs, nestUBs, nestSts, nullptr, /*local_vars=*/std::nullopt,
+          loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
+          /*local_vars=*/std::nullopt,
           /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
           /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
           /*reduce_attrs=*/nullptr);

>From a4f7e239da4b3862a412e8f773cafb7e149f5d51 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Fri, 27 Jun 2025 01:54:06 -0500
Subject: [PATCH 3/3] byref printing and parsing

---
 flang/lib/Optimizer/Dialect/FIROps.cpp | 27 ++++++++++++++++++++------
 flang/test/Fir/do_concurrent.fir       |  4 ++--
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 674129f546e94..6b40e7015fdd8 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -5254,17 +5254,21 @@ static mlir::ParseResult parseSpecifierList(
   if (failed(parser.parseLParen()))
     return mlir::failure();
 
+  llvm::SmallVector<bool> isByRefVec;
   llvm::SmallVector<mlir::SymbolRefAttr> spceifierSymbolVec;
   llvm::SmallVector<fir::ReduceAttr> attributes;
 
   if (failed(parser.parseCommaSeparatedList([&]() {
+        if (isReduce)
+          isByRefVec.push_back(
+              parser.parseOptionalKeyword("byref").succeeded());
+
         if (failed(parser.parseAttribute(spceifierSymbolVec.emplace_back())))
           return mlir::failure();
 
         if (isReduce &&
-            failed(parser.parseAttribute(attributes.emplace_back()))) {
+            failed(parser.parseAttribute(attributes.emplace_back())))
           return mlir::failure();
-        }
 
         if (parser.parseOperand(specifierOperands.emplace_back()) ||
             parser.parseArrow() ||
@@ -5301,6 +5305,13 @@ static mlir::ParseResult parseSpecifierList(
                               std::get<1>(operandType), result.operands))
       return mlir::failure();
 
+  if (isReduce)
+    result.addAttribute(
+        fir::DoConcurrentLoopOp::getReduceByrefAttrName(result.name),
+        isByRefVec.empty()
+            ? nullptr
+            : mlir::DenseBoolArrayAttr::get(builder.getContext(), isByRefVec));
+
   llvm::SmallVector<mlir::Attribute> symbolAttrs(spceifierSymbolVec.begin(),
                                                  spceifierSymbolVec.end());
   result.addAttribute(symsAttrName, builder.getArrayAttr(symbolAttrs));
@@ -5411,11 +5422,15 @@ void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
   if (!getReduceVars().empty()) {
     p << " reduce(";
     llvm::interleaveComma(
-        llvm::zip_equal(getReduceSymsAttr(), getReduceAttrsAttr(),
-                        getReduceVars(), getRegionReduceArgs()),
+        llvm::zip_equal(getReduceByrefAttr().asArrayRef(), getReduceSymsAttr(),
+                        getReduceAttrsAttr(), getReduceVars(),
+                        getRegionReduceArgs()),
         p, [&](auto it) {
-          p << std::get<0>(it) << " " << std::get<1>(it) << " "
-            << std::get<2>(it) << " -> " << std::get<3>(it);
+          if (std::get<0>(it))
+            p << "byref ";
+
+          p << std::get<1>(it) << " " << std::get<2>(it) << " "
+            << std::get<3>(it) << " -> " << std::get<4>(it);
         });
     p << " : ";
     llvm::interleaveComma(getReduceVars(), p,
diff --git a/flang/test/Fir/do_concurrent.fir b/flang/test/Fir/do_concurrent.fir
index c207ea5dea825..6e2173447855e 100644
--- a/flang/test/Fir/do_concurrent.fir
+++ b/flang/test/Fir/do_concurrent.fir
@@ -167,7 +167,7 @@ func.func @dc_reduce() {
   %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   %c1 = arith.constant 1 : index 
   fir.do_concurrent {
-    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(@add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1 : !fir.ref<i32>) {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(byref @add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1 : !fir.ref<i32>) {
     }
   }
   return
@@ -177,7 +177,7 @@ func.func @dc_reduce() {
 // CHECK:          %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
 // CHECK:          %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 // CHECK:          fir.do_concurrent {
-// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
+// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) reduce(byref @add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
 // CHECK:            }
 // CHECK:          }
 // CHECK:          return


From flang-commits at lists.llvm.org  Wed Jul  9 03:50:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 03:50:45 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <686e4985.170a0220.3a56bc.adbf@mx.google.com>


https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/145464

>From 4915fb8f1d27847dc3d36899e233d5ac988f96c5 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 23 Jun 2025 16:39:55 +0530
Subject: [PATCH 1/6] [flang] Introduce omp_target_allocmem and
 omp_target_freemem fir ops.

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  58 ++++++++++
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +++++++++++++++++-
 2 files changed, 159 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 8ac847dd7dd0a..2dff0f05fade7 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,6 +517,64 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
+def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Creates a heap memory reference suitable for storing a value of the
+    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
+    The memory object is in an undefined state.  `omp_target_allocmem` operations must
+    be paired with `omp_target_freemem` operations to avoid memory leaks.
+
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyIntegerType>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyIntegerType>:$typeparams,
+    Variadic<AnyIntegerType>:$shape
+  );
+  let results = (outs fir_HeapType);
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+    bool hasLenParams() { return !getTypeparams().empty(); }
+    bool hasShapeOperands() { return !getShape().empty(); }
+    unsigned numLenParams() { return getTypeparams().size(); }
+    operand_range getLenParams() { return getTypeparams(); }
+    unsigned numShapeOperands() { return getShape().size(); }
+    operand_range getShapeOperands() { return getShape(); }
+    static mlir::Type getRefTy(mlir::Type ty);
+  }];
+}
+
+def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free a heap object";
+
+  let description = [{
+    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
+    The memory object that is deallocated is placed in an undefined state
+    after `fir.omp_target_freemem`.
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      ...
+      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyIntegerType, "", [MemFree]>:$device,
+  Arg<fir_HeapType, "", [MemFree]>:$heapref
+  );
+}
+
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..042ade6b1e0a1 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,6 +1168,105 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetAllocMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = heap.getType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
+    mlir::Location loc = heap.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(dataTy);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
+    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(loc, rewriter, mallocTy, size);
+    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+        heap, ::getLlvmPtrType(heap.getContext()),
+        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
+    return mlir::success();
+  }
+
+  /// Compute the allocation size in bytes of the element type of
+  /// \p llTy pointer type. The result is returned as a value of \p idxTy
+  /// integer type.
+  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy) const {
+    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+  }
+};
+} // namespace
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp freeFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
+    return freeFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_free",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMVoidType::get(module->getContext()),
+          {getLlvmPtrType(module->getContext()), i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetFreeMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
+    mlir::Location loc = freemem.getLoc();
+    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
+    rewriter.create<mlir::LLVM::CallOp>(
+        loc, mlir::TypeRange{},
+        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
+        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
+    rewriter.eraseOp(freemem);
+    return mlir::success();
+  }
+};
+} // namespace
+
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4274,7 +4373,8 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
+      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,

>From e41a2c76786538fc411e104542a0282cfebef4f7 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:20:36 +0530
Subject: [PATCH 2/6] [flang] Fix parsing and printing.

---
 .../include/flang/Optimizer/Dialect/FIROps.td | 13 ++-
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 90 ++++++++++++++++---
 flang/test/Fir/omp_target_allocmem.fir        | 28 ++++++
 flang/test/Fir/omp_target_freemem.fir         | 28 ++++++
 4 files changed, 145 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Fir/omp_target_allocmem.fir
 create mode 100644 flang/test/Fir/omp_target_freemem.fir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 2dff0f05fade7..666b66a8670d6 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -528,7 +528,8 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
     be paired with `omp_target_freemem` operations to avoid memory leaks.
 
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
     ```
   }];
 
@@ -542,6 +543,9 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
   );
   let results = (outs fir_HeapType);
 
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
   let extraClassDeclaration = [{
     mlir::Type getAllocatedType();
     bool hasLenParams() { return !getTypeparams().empty(); }
@@ -563,9 +567,9 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     The memory object that is deallocated is placed in an undefined state
     after `fir.omp_target_freemem`.
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
-      ...
-      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
     ```
   }];
 
@@ -573,6 +577,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   Arg<AnyIntegerType, "", [MemFree]>:$device,
   Arg<fir_HeapType, "", [MemFree]>:$heapref
   );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..9335a4b041ac8 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,24 +106,38 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca and Allocmem
-///
+/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
+/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
-                                            mlir::OpAsmParser &parser,
-                                            mlir::OperationState &result) {
+static mlir::ParseResult
+parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
+                   mlir::OperationState &result, bool isTargetOp = false) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+  // Parse device number as a new operand
+  if (isTargetOp) {
+    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+    mlir::Type deviceType;
+    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+      return mlir::failure();
+    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+      return mlir::failure();
+    if (parser.parseComma())
+      return mlir::failure();
+  }
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
-  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -147,13 +161,19 @@ static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
+
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
-                                                 {typeparamsSize, shapeSize}));
+  llvm::SmallVector<std::int32_t> segmentSizes;
+  if (isTargetOp)
+    segmentSizes.push_back(1);
+  segmentSizes.push_back(typeparamsSize);
+  segmentSizes.push_back(shapeSize);
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -385,6 +405,56 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// OmpTargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
+  return mlir::cast<fir::HeapType>(getType()).getEleTy();
+}
+
+mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
+  return fir::HeapType::get(ty);
+}
+
+mlir::ParseResult
+fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                mlir::OperationState &result) {
+  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
+}
+
+void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
+  llvm::SmallVector<llvm::StringRef> visited;
+  if (verifyInType(getInType(), visited, numShapeOperands()))
+    return emitOpError("invalid type for allocation");
+  if (verifyTypeParamCount(getInType(), numLenParams()))
+    return emitOpError("LEN params do not correspond to type");
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<fir::HeapType>(outType))
+    return emitOpError("must be a !fir.heap type");
+  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
+    return emitOpError("cannot allocate !fir.box of unknown rank or type");
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
new file mode 100644
index 0000000000000..5140c91c9510c
--- /dev/null
+++ b/flang/test/Fir/omp_target_allocmem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  return %1 : !fir.heap<!fir.array<3x3xi32>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_freemem.fir
new file mode 100644
index 0000000000000..02e136076a9cf
--- /dev/null
+++ b/flang/test/Fir/omp_target_freemem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  return
+}

>From a864094e5bd482499f82ef913c92c1f5b6bd070f Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:32:21 +0530
Subject: [PATCH 3/6] [flang] Fix doc in td

---
 flang/include/flang/Optimizer/Dialect/FIROps.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 666b66a8670d6..93d617027e30b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -560,7 +560,7 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
 
 def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object";
+  let summary = "free a heap object on an openmp device";
 
   let description = [{
     Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
@@ -569,7 +569,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     ```
       %device = arith.constant 0 : i32
       %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
     ```
   }];
 

>From 2f5b289464787d89ea00ab295ccb2ebee6b57329 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 12:22:55 +0530
Subject: [PATCH 4/6] [omp][mlir] Introduce TargetAllocMem and TargetFreeMem
 ops in openMP mlir dialect

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  63 -------
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +----------
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 161 ++++++++++++++++++
 flang/lib/Optimizer/Dialect/FIROps.cpp        |  88 +---------
 flang/test/Fir/omp_target_allocmem.fir        |  28 ---
 ...em.fir => omp_target_allocmem_freemem.fir} |  19 ++-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  64 +++++++
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 102 +++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  82 +++++++++
 .../ompenmp-target-allocmem-freemem.mlir      |  42 +++++
 10 files changed, 473 insertions(+), 278 deletions(-)
 delete mode 100644 flang/test/Fir/omp_target_allocmem.fir
 rename flang/test/Fir/{omp_target_freemem.fir => omp_target_allocmem_freemem.fir} (51%)
 create mode 100644 mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 93d617027e30b..8ac847dd7dd0a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,69 +517,6 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
-def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
-    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
-  let summary = "allocate storage on an openmp device for an object of a given type";
-
-  let description = [{
-    Creates a heap memory reference suitable for storing a value of the
-    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
-    The memory object is in an undefined state.  `omp_target_allocmem` operations must
-    be paired with `omp_target_freemem` operations to avoid memory leaks.
-
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-    ```
-  }];
-
-  let arguments = (ins
-    Arg<AnyIntegerType>:$device,
-    TypeAttr:$in_type,
-    OptionalAttr<StrAttr>:$uniq_name,
-    OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyIntegerType>:$typeparams,
-    Variadic<AnyIntegerType>:$shape
-  );
-  let results = (outs fir_HeapType);
-
-  let hasCustomAssemblyFormat = 1;
-  let hasVerifier = 1;
-
-  let extraClassDeclaration = [{
-    mlir::Type getAllocatedType();
-    bool hasLenParams() { return !getTypeparams().empty(); }
-    bool hasShapeOperands() { return !getShape().empty(); }
-    unsigned numLenParams() { return getTypeparams().size(); }
-    operand_range getLenParams() { return getTypeparams(); }
-    unsigned numShapeOperands() { return getShape().size(); }
-    operand_range getShapeOperands() { return getShape(); }
-    static mlir::Type getRefTy(mlir::Type ty);
-  }];
-}
-
-def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
-  [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object on an openmp device";
-
-  let description = [{
-    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
-    The memory object that is deallocated is placed in an undefined state
-    after `fir.omp_target_freemem`.
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
-    ```
-  }];
-
-  let arguments = (ins
-  Arg<AnyIntegerType, "", [MemFree]>:$device,
-  Arg<fir_HeapType, "", [MemFree]>:$heapref
-  );
-  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
-}
-
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 042ade6b1e0a1..a3de3ae9d116a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,105 +1168,6 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetAllocMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::Type heapTy = heap.getType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
-    mlir::Location loc = heap.getLoc();
-    auto ity = lowerTy().indexType();
-    mlir::Type dataTy = fir::unwrapRefType(heapTy);
-    mlir::Type llvmObjectTy = convertObjectType(dataTy);
-    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
-      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
-                "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
-      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands().drop_front())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
-    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
-    auto mallocTy =
-        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
-    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
-      size = integerCast(loc, rewriter, mallocTy, size);
-    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        heap, ::getLlvmPtrType(heap.getContext()),
-        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
-    return mlir::success();
-  }
-
-  /// Compute the allocation size in bytes of the element type of
-  /// \p llTy pointer type. The result is returned as a value of \p idxTy
-  /// integer type.
-  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
-  }
-};
-} // namespace
-
-static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp freeFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
-    return freeFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_free",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMVoidType::get(module->getContext()),
-          {getLlvmPtrType(module->getContext()), i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetFreeMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
-    mlir::Location loc = freemem.getLoc();
-    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
-    rewriter.create<mlir::LLVM::CallOp>(
-        loc, mlir::TypeRange{},
-        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
-        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
-    rewriter.eraseOp(freemem);
-    return mlir::success();
-  }
-};
-} // namespace
-
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4373,8 +4274,7 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
-      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 37f1c9f97e1ce..a04c5d7eb7ee7 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -125,10 +125,171 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type
+convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
+    mlir::Type ty, mlir::Value val, bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "omp.target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(lowerTy(), loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
+    allocmemOp->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    auto callOp = rewriter.create<mlir::LLVM::CallOp>(
+        loc, llvmPtrTy,
+        mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    return mlir::success();
+  }
+};
 } // namespace
 
 void fir::populateOpenMPFIRToLLVMConversionPatterns(
     const LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
   patterns.add<MapInfoOpConversion>(converter);
   patterns.add<PrivateClauseOpConversion>(converter);
+  patterns.add<TargetAllocMemOpConversion>(converter);
 }
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 9335a4b041ac8..f6c794b8fe9ae 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,38 +106,24 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// Parser shared by Alloca and Allocmem
 /// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
-/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
-///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
-///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult
-parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
-                   mlir::OperationState &result, bool isTargetOp = false) {
-  auto &builder = parser.getBuilder();
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
-  // Parse device number as a new operand
-  if (isTargetOp) {
-    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
-    mlir::Type deviceType;
-    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
-      return mlir::failure();
-    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
-      return mlir::failure();
-    if (parser.parseComma())
-      return mlir::failure();
-  }
+static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
+                                            mlir::OpAsmParser &parser,
+                                            mlir::OperationState &result) {
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
+  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -161,19 +147,13 @@ parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
-
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  llvm::SmallVector<std::int32_t> segmentSizes;
-  if (isTargetOp)
-    segmentSizes.push_back(1);
-  segmentSizes.push_back(typeparamsSize);
-  segmentSizes.push_back(shapeSize);
-  result.addAttribute("operandSegmentSizes",
-                      builder.getDenseI32ArrayAttr(segmentSizes));
+  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
+                                                 {typeparamsSize, shapeSize}));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -405,56 +385,6 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
-//===----------------------------------------------------------------------===//
-// OmpTargetAllocMemOp
-//===----------------------------------------------------------------------===//
-
-mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
-  return mlir::cast<fir::HeapType>(getType()).getEleTy();
-}
-
-mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
-  return fir::HeapType::get(ty);
-}
-
-mlir::ParseResult
-fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
-                                mlir::OperationState &result) {
-  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
-}
-
-void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
-  p << " ";
-  p.printOperand(getDevice());
-  p << " : ";
-  p << getDevice().getType();
-  p << ", ";
-  p << getInType();
-  if (!getTypeparams().empty()) {
-    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
-  }
-  for (auto sh : getShape()) {
-    p << ", ";
-    p.printOperand(sh);
-  }
-  p.printOptionalAttrDict((*this)->getAttrs(),
-                          {"in_type", "operandSegmentSizes"});
-}
-
-llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
-  llvm::SmallVector<llvm::StringRef> visited;
-  if (verifyInType(getInType(), visited, numShapeOperands()))
-    return emitOpError("invalid type for allocation");
-  if (verifyTypeParamCount(getInType(), numLenParams()))
-    return emitOpError("LEN params do not correspond to type");
-  mlir::Type outType = getType();
-  if (!mlir::dyn_cast<fir::HeapType>(outType))
-    return emitOpError("must be a !fir.heap type");
-  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
-    return emitOpError("cannot allocate !fir.box of unknown rank or type");
-  return mlir::success();
-}
-
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
deleted file mode 100644
index 5140c91c9510c..0000000000000
--- a/flang/test/Fir/omp_target_allocmem.fir
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
-// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
-func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  return %1 : !fir.heap<!fir.array<3x3xi32>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
-// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
-func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
-// CHECK-SAME: i32 %[[len:.*]])
-// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
-// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
-// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
-func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
-}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
similarity index 51%
rename from flang/test/Fir/omp_target_freemem.fir
rename to flang/test/Fir/omp_target_allocmem_freemem.fir
index 02e136076a9cf..9202202728454 100644
--- a/flang/test/Fir/omp_target_freemem.fir
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -1,28 +1,33 @@
 // RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_nonchar() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3xi32>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_char() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
 func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..bcc7c906041ac 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1887,4 +1887,68 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Allocates memory on the specified OpenMP device for an object of the given type.
+    Returns an integer value representing the device pointer to the allocated memory.
+    The memory is uninitialized after allocation. Operations must be paired with 
+    `omp.target_freemem` to avoid memory leaks.
+
+    ```mlir
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyInteger>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyInteger>:$typeparams,
+    Variadic<AnyInteger>:$shape
+  );
+  let results = (outs I64);
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// TargetFreeMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetFreeMemOp : OpenMP_Op<"target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free memory on an openmp device";
+
+  let description = [{
+    Deallocates memory on the specified OpenMP device that was previously 
+    allocated by an `omp.target_allocmem` operation. The memory is placed 
+    in an undefined state after deallocation.
+    ```
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+      omp.target_freemem %device, %ptr : i32, i64
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyInteger, "", [MemFree]>:$device,
+  Arg<I64, "", [MemFree]>:$heapref
+  );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..5e3ad6ccf3ffe 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3493,6 +3493,108 @@ LogicalResult ScanOp::verify() {
                    "reduction modifier");
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type omp::TargetAllocMemOp::getAllocatedType() {
+  return getInTypeAttr().getValue();
+}
+
+/// operation ::= %res = (`omp.target_alloc_mem`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
+static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+
+  // Parse device number as a new operand
+  mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+  mlir::Type deviceType;
+  if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+    return mlir::failure();
+  if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+    return mlir::failure();
+  if (parser.parseComma())
+    return mlir::failure();
+
+  mlir::Type intype;
+  if (parser.parseType(intype))
+    return mlir::failure();
+  result.addAttribute("in_type", mlir::TypeAttr::get(intype));
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
+  llvm::SmallVector<mlir::Type> typeVec;
+  if (!parser.parseOptionalLParen()) {
+    // parse the LEN params of the derived type. (<params> : <types>)
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
+        parser.parseColonTypeList(typeVec) || parser.parseRParen())
+      return mlir::failure();
+    typeparamsSize = operands.size();
+    hasOperands = true;
+  }
+  std::int32_t shapeSize = 0;
+  if (!parser.parseOptionalComma()) {
+    // parse size to scale by, vector of n dimensions of type index
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None))
+      return mlir::failure();
+    shapeSize = operands.size() - typeparamsSize;
+    auto idxTy = builder.getIndexType();
+    for (std::int32_t i = typeparamsSize, end = operands.size(); i != end; ++i)
+      typeVec.push_back(idxTy);
+    hasOperands = true;
+  }
+  if (hasOperands &&
+      parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
+                             result.operands))
+    return mlir::failure();
+
+  mlir::Type restype = builder.getIntegerType(64);
+  ;
+  if (!restype) {
+    parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
+    return mlir::failure();
+  }
+  llvm::SmallVector<std::int32_t> segmentSizes{1, typeparamsSize, shapeSize};
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
+  if (parser.parseOptionalAttrDict(result.attributes) ||
+      parser.addTypeToList(restype, result.types))
+    return mlir::failure();
+  return mlir::success();
+}
+
+mlir::ParseResult omp::TargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  return parseTargetAllocMemOp(parser, result);
+}
+
+void omp::TargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult omp::TargetAllocMemOp::verify() {
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<IntegerType>(outType))
+    return emitOpError("must be a integer type");
+  return mlir::success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index eece8573f00ec..63b41f3b13363 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5714,6 +5714,82 @@ static bool isTargetDeviceOp(Operation *op) {
   return false;
 }
 
+static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder,
+                                         llvm::Module *llvmModule) {
+  llvm::Type *i64Ty = builder.getInt64Ty();
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *returnType = builder.getPtrTy(0);
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(returnType, {i64Ty, i32Ty}, false);
+  llvm::Function *func = cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_alloc", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                        LLVM::ModuleTranslation &moduleTranslation) {
+  auto allocMemOp = cast<omp::TargetAllocMemOp>(opInst);
+  if (!allocMemOp)
+    return failure();
+
+  // Get "omp_target_alloc" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = allocMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the allocation size.
+  llvm::DataLayout dataLayout = llvmModule->getDataLayout();
+  mlir::Type heapTy = allocMemOp.getAllocatedType();
+  llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
+  llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
+  llvm::ConstantInt *allocSize = builder.getInt64(typeSize.getFixedValue());
+  // Create call to "omp_target_alloc" with the args as translated llvm values.
+  llvm::CallInst *call =
+      builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});
+  llvm::Value *resultI64 = builder.CreatePtrToInt(call, builder.getInt64Ty());
+
+  // Map the result
+  moduleTranslation.mapValue(allocMemOp.getResult(), resultI64);
+  return success();
+}
+
+static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder,
+                                        llvm::Module *llvmModule) {
+  llvm::Type *ptrTy = builder.getPtrTy(0);
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *voidTy = builder.getVoidTy();
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(voidTy, {ptrTy, i32Ty}, false);
+  llvm::Function *func = dyn_cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_free", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation) {
+  auto freeMemOp = cast<omp::TargetFreeMemOp>(opInst);
+  if (!freeMemOp)
+    return failure();
+
+  // Get "omp_target_free" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = freeMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the corresponding heapref value in llvm
+  mlir::Value heapref = freeMemOp.getHeapref();
+  llvm::Value *llvmHeapref = moduleTranslation.lookupValue(heapref);
+  // Convert heapref int to ptr and call "omp_target_free"
+  llvm::Value *intToPtr =
+      builder.CreateIntToPtr(llvmHeapref, builder.getPtrTy(0));
+  builder.CreateCall(ompTragetFreeFunc, {intToPtr, llvmDeviceNum});
+  return success();
+}
+
 /// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including
 /// OpenMP runtime calls).
 static LogicalResult
@@ -5871,6 +5947,12 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
                 // etc. and then discarded
                 return success();
               })
+          .Case([&](omp::TargetAllocMemOp) {
+            return convertTargetAllocMemOp(*op, builder, moduleTranslation);
+          })
+          .Case([&](omp::TargetFreeMemOp) {
+            return convertTargetFreeMemOp(*op, builder, moduleTranslation);
+          })
           .Default([&](Operation *inst) {
             return inst->emitError()
                    << "not yet implemented: " << inst->getName();
diff --git a/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
new file mode 100644
index 0000000000000..1bc97609ccff4
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt %s -convert-openmp-to-llvm | mlir-translate -mlir-to-llvmir | FileCheck %s
+
+// This file contains MLIR test cases for omp.target_allocmem and omp.target_freemem
+
+// CHECK-LABEL: test_alloc_free_i64
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 8, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_i64() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, i64
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_1d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 64, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_1d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_2d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 1024, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_2d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16x16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}

>From 1e7a216d3d9e07146ebb364025983e0f5000eefa Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 21:05:35 +0530
Subject: [PATCH 5/6] Fix comments

---
 flang/lib/Optimizer/Dialect/FIROps.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index f6c794b8fe9ae..ba5c81d826e5e 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -107,7 +107,6 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
 }
 
 /// Parser shared by Alloca and Allocmem
-/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword

>From 3879eb7271098d52281aca44e77fc82af9987f02 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 9 Jul 2025 15:49:06 +0530
Subject: [PATCH 6/6] clang format

---
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 24 ++++++++++++-------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index a04c5d7eb7ee7..14cc7bb511f0f 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -142,8 +142,8 @@ static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
           /*isVarArg=*/false));
 }
 
-static mlir::Type
-convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
     return converter.convertBoxTypeAsStruct(boxTy);
   return converter.convertType(firType);
@@ -189,8 +189,9 @@ computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
 }
 
 static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
   return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
 }
 
@@ -224,8 +225,10 @@ genAllocationScaleSize(OP op, mlir::Type ity,
 }
 
 static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
-    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
-    mlir::Type ty, mlir::Value val, bool fold = false) {
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
   auto valTy = val.getType();
   // If the value was not yet lowered, lower its type so that it can
   // be used in getPrimitiveTypeSizeInBits.
@@ -261,11 +264,13 @@ struct TargetAllocMemOpConversion
     auto ity = lowerTy().indexType();
     mlir::Type dataTy = fir::unwrapRefType(heapTy);
     mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
-    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    mlir::Type llvmPtrTy =
+        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "omp.target_allocmem codegen of derived type with length "
                 "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy,
+                                          lowerTy().getDataLayout());
     if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands().drop_front())
@@ -281,7 +286,8 @@ struct TargetAllocMemOpConversion
         loc, llvmPtrTy,
         mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
         addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(
+        allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
     return mlir::success();
   }
 };


From flang-commits at lists.llvm.org  Wed Jul  9 05:10:54 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 09 Jul 2025 05:10:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686e5c4e.170a0220.14187d.3127@mx.google.com>


eugeneepshteyn wrote:

@luporl , sorry to bother you, but it seems this fix caused regression in various Fujitsu tests at `-O1` or higher. For example, compiling https://github.com/fujitsu/compiler-test-suite/blob/main/Fortran/1032/1032_0022.f90 results in the following:
```
$ flang -c 1032_0022.f90 -O1
flang: llvm/include/llvm/Support/Casting.h:566: decltype(auto) llvm::cast(const From&) [with To = fir::ReferenceType; From = mlir::Type]: Assertion `isa<To>(Val) && "cast<Ty>() argument of incompatible type!"' failed.
...
#10 0x0000000005a38f49 (anonymous namespace)::BroadcastAssignBufferization::matchAndRewrite(hlfir::AssignOp, mlir::PatternRewrite
r&) const OptimizedBufferization.cpp:0:0
#11 0x00000000059dc3f3 mlir::detail::OpOrInterfaceRewritePatternBase<hlfir::AssignOp>::matchAndRewrite(mlir::Operation*, mlir::Pa
tternRewriter&) const ConvertToFIR.cpp:0:0          
#12 0x0000000008d2629c mlir::PatternApplicator::matchAndRewrite(mlir::Operation*, mlir::PatternRewriter&, llvm::function_ref<bool
 (mlir::Pattern const&)>, llvm::function_ref<void (mlir::Pattern const&)>, llvm::function_ref<llvm::LogicalResult (mlir::Pattern 
const&)>) (/proj/nv/llvm/Linux_x86_64/llvm-5497/bin/flang+0x8d2629c)
#13 0x0000000008cf153c (anonymous namespace)::GreedyPatternRewriteDriver::processWorklist() GreedyPatternRewriteDriver.cpp:0:0
#14 0x0000000008cf445c mlir::applyPatternsGreedily(mlir::Region&, mlir::FrozenRewritePatternSet const&, mlir::GreedyRewriteConfig
, bool*) (/proj/nv/llvm/Linux_x86_64/llvm-5497/bin/flang+0x8cf445c)
#15 0x0000000005a36317 (anonymous namespace)::OptimizedBufferizationPass::runOnOperation() OptimizedBufferization.cpp:0:0
#16 0x000000000a43d431 mlir::detail::OpToOpPassAdaptor::run(mlir::Pass*, mlir::Operation*, mlir::AnalysisManager, bool, unsigned 
int) (/proj/nv/llvm/Linux_x86_64/llvm-5497/bin/flang+0xa43d431)
#17 0x000000000a43d83a mlir::detail::OpToOpPassAdaptor::runPipeline(mlir::OpPassManager&, mlir::Operation*, mlir::AnalysisManager
, bool, unsigned int, mlir::PassInstrumentor*, mlir::PassInstrumentation::PipelineParentInfo const*) (/proj/nv/llvm/Linux_x86_64/
llvm-5497/bin/flang+0xa43d83a)
#18 0x000000000a43dbbe mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::'lambda'(mlir::detail::OpToOpPassAdaptor::
runOnOperationAsyncImpl(bool)::OpPMInfo&)::operator()(mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo&) 
const Pass.cpp:0:0
#19 0x000000000a43dd8f std::_Function_handler<void (), llvm::LogicalResult mlir::failableParallelForEach<__gnu_cxx::__normal_iter
ator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo*, std::vector<mlir::detail::OpToOpPassAdaptor::runO
nOperationAsyncImpl(bool)::OpPMInfo, std::allocator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo>>>, 
void mlir::parallelForEach<__gnu_cxx::__normal_iterator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo*
, std::vector<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo, std::allocator<mlir::detail::OpToOpPassAd
aptor::runOnOperationAsyncImpl(bool)::OpPMInfo>>>, mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::'lambda'(mlir:
:detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo&)>(mlir::MLIRContext*, __gnu_cxx::__normal_iterator<mlir::det
ail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo*, std::vector<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyn
cImpl(bool)::OpPMInfo, std::allocator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo>>>, __gnu_cxx::__n
ormal_iterator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo*, std::vector<mlir::detail::OpToOpPassAda
ptor::runOnOperationAsyncImpl(bool)::OpPMInfo, std::allocator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpP
MInfo>>>, mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::'lambda'(mlir::detail::OpToOpPassAdaptor::runOnOperatio
nAsyncImpl(bool)::OpPMInfo&)&&)::'lambda'(__gnu_cxx::__normal_iterator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(b
ool)::OpPMInfo*, std::vector<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo, std::allocator<mlir::detai
l::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo>>>&&)>(mlir::MLIRContext*, __gnu_cxx::__normal_iterator<mlir::detai
l::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo*, std::vector<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncI
mpl(bool)::OpPMInfo, std::allocator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo>>>, __gnu_cxx::__nor
mal_iterator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMInfo*, std::vector<mlir::detail::OpToOpPassAdapt
or::runOnOperationAsyncImpl(bool)::OpPMInfo, std::allocator<mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::OpPMI
nfo>>>, mlir::detail::OpToOpPassAdaptor::runOnOperationAsyncImpl(bool)::'lambda'(mlir::detail::OpToOpPassAdaptor::runOnOperationA
syncImpl(bool)::OpPMInfo&)&&)::'lambda'()>::_M_invoke(std::_Any_data const&) Pass.cpp:0:0
...
```

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Wed Jul  9 05:33:04 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 05:33:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Create unique reduction
 decls for different logical kinds (PR #146558)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146558@github.com>
Message-ID: <686e6180.170a0220.3a7d3e.2a7d@mx.google.com>


================

----------------
tblah wrote:

Thank you for the bug report. I don't think flang/test/Parser/OpenMP/declare-reduction-operator.f90 could have been effected by this PR because that is testing a much earlier stage of the compiler and will not even execute the changed lines. The stack trace has very little information but both look like they could be the same issue.

https://github.com/llvm/llvm-project/pull/146558

From flang-commits at lists.llvm.org  Wed Jul  9 06:25:00 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Wed, 09 Jul 2025 06:25:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686e6dac.170a0220.12d1f3.3066@mx.google.com>


================
@@ -805,13 +805,12 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
+      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
----------------
luporl wrote:

Now I understand @jeanPerier's comment.
Even after calling `hlfir::derefPointersAndAllocatables` the resulting type may still be a fir.heap/fir.ptr.
They can be converted to a fir.ref.

It seems `hlfir::derefPointersAndAllocatables` dereferences other pointer types, such as `!fir.ref<!fir.box<!fir.ptr<!fir.array...>>>`, generated from Fortran pointers, but not pointers directly to arrays, such as `!fir.ptr<!fir.array...>`, that may appear when `equivalence` is used.

I'll prepare a follow-up PR with this change.

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Wed Jul  9 07:02:08 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Wed, 09 Jul 2025 07:02:08 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Create unique reduction
 decls for different logical kinds (PR #146558)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146558@github.com>
Message-ID: <686e7660.630a0220.1b98e0.ec5b@mx.google.com>


================

----------------
luporl wrote:

This issue was also reported here: https://github.com/llvm/llvm-project/issues/147309.
It was caused by https://github.com/llvm/llvm-project/pull/145917 and fixed by https://github.com/llvm/llvm-project/pull/147519.

https://github.com/llvm/llvm-project/pull/146558

From flang-commits at lists.llvm.org  Wed Jul  9 07:34:31 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Wed, 09 Jul 2025 07:34:31 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix array assignment regression
 introduced by #147371 (PR #147761)
Message-ID: <llvm.org/llvm/llvm-project/pull/147761@github.com>


https://github.com/luporl created https://github.com/llvm/llvm-project/pull/147761

In some cases fixed shape arrays can be fir.heap/fir.ptr, even
after hlfir::derefPointersAndAllocatables() is called.


>From a3310779ce305d5b835c42aba81dbacb0dc7a8ca Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori at linaro.org>
Date: Wed, 9 Jul 2025 09:55:41 -0300
Subject: [PATCH] [flang] Fix array assignment regression introduced by #147371

In some cases fixed shape arrays can be fir.heap/fir.ptr, even
after hlfir::derefPointersAndAllocatables() is called.
---
 .../Transforms/OptimizedBufferization.cpp     |  4 +--
 flang/test/HLFIR/opt-scalar-assign.fir        | 31 +++++++++++++++++++
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 434973f5887e8..abcbf145410ab 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -805,8 +805,8 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
-      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
-      auto seqTy = mlir::cast<fir::SequenceType>(refTy.getElementType());
+      auto seqTy =
+          mlir::cast<fir::SequenceType>(fir::unwrapRefType(lhs.getType()));
       llvm::ArrayRef<int64_t> fixedShape = seqTy.getShape();
       int64_t flatExtent = 1;
       for (int64_t extent : fixedShape)
diff --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 468a5dbf988d7..256ccee6b9944 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -184,3 +184,34 @@ func.func private @_QFPtest7(%arg0: !fir.ref<tuple<!fir.box<!fir.array<2x2xi32>>
 // CHECK:             hlfir.assign %[[VAL_0]] to %[[VAL_2]] : i32, !fir.ref<i32>
 // CHECK:           }
 // CHECK:         }
+
+func.func @_QPtest8() {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.alloca !fir.array<120xi8> {uniq_name = "_QFtest8Ea"}
+  %c0 = arith.constant 0 : index
+  %2 = fir.coordinate_of %1, %c0 : (!fir.ref<!fir.array<120xi8>>, index) -> !fir.ref<i8>
+  %3 = fir.convert %2 : (!fir.ref<i8>) -> !fir.ptr<!fir.array<2x10xf32>>
+  %c2 = arith.constant 2 : index
+  %c10 = arith.constant 10 : index
+  %4 = fir.shape %c2, %c10 : (index, index) -> !fir.shape<2>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFtest8Ea"} : (!fir.ptr<!fir.array<2x10xf32>>, !fir.shape<2>) -> (!fir.ptr<!fir.array<2x10xf32>>, !fir.ptr<!fir.array<2x10xf32>>)
+  %c0_0 = arith.constant 0 : index
+  %6 = fir.coordinate_of %1, %c0_0 : (!fir.ref<!fir.array<120xi8>>, index) -> !fir.ref<i8>
+  %7 = fir.convert %6 : (!fir.ref<i8>) -> !fir.ptr<!fir.array<30xf32>>
+  %c30 = arith.constant 30 : index
+  %8 = fir.shape %c30 : (index) -> !fir.shape<1>
+  %9:2 = hlfir.declare %7(%8) {uniq_name = "_QFtest8Eb"} : (!fir.ptr<!fir.array<30xf32>>, !fir.shape<1>) -> (!fir.ptr<!fir.array<30xf32>>, !fir.ptr<!fir.array<30xf32>>)
+  %cst = arith.constant -1.000000e+00 : f32
+  hlfir.assign %cst to %5#0 : f32, !fir.ptr<!fir.array<2x10xf32>>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPtest8() {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest8Ea"} : (!fir.ptr<!fir.array<2x10xf32>>, !fir.shape<2>) -> (!fir.ptr<!fir.array<2x10xf32>>, !fir.ptr<!fir.array<2x10xf32>>)
+// CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ptr<!fir.array<2x10xf32>>) -> !fir.ref<!fir.array<20xf32>>
+// CHECK:           fir.do_loop %[[VAL_3:.*]] = %{{.*}} to %{{.*}} step %{{.*}} unordered {
+// CHECK:             %[[VAL_4:.*]] = hlfir.designate %[[VAL_2]] (%[[VAL_3]]) : (!fir.ref<!fir.array<20xf32>>, index) -> !fir.ref<f32>
+// CHECK:             hlfir.assign %[[VAL_0]] to %[[VAL_4]] : f32, !fir.ref<f32>
+// CHECK:           }
+// CHECK:         }


From flang-commits at lists.llvm.org  Wed Jul  9 07:35:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 07:35:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix array assignment regression
 introduced by #147371 (PR #147761)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147761@github.com>
Message-ID: <686e7e15.170a0220.3b69e0.5f35@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Leandro Lupori (luporl)

<details>
<summary>Changes</summary>

In some cases fixed shape arrays can be fir.heap/fir.ptr, even
after hlfir::derefPointersAndAllocatables() is called.


---
Full diff: https://github.com/llvm/llvm-project/pull/147761.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp (+2-2) 
- (modified) flang/test/HLFIR/opt-scalar-assign.fir (+31) 


``````````diff
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 434973f5887e8..abcbf145410ab 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -805,8 +805,8 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
-      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
-      auto seqTy = mlir::cast<fir::SequenceType>(refTy.getElementType());
+      auto seqTy =
+          mlir::cast<fir::SequenceType>(fir::unwrapRefType(lhs.getType()));
       llvm::ArrayRef<int64_t> fixedShape = seqTy.getShape();
       int64_t flatExtent = 1;
       for (int64_t extent : fixedShape)
diff --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 468a5dbf988d7..256ccee6b9944 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -184,3 +184,34 @@ func.func private @_QFPtest7(%arg0: !fir.ref<tuple<!fir.box<!fir.array<2x2xi32>>
 // CHECK:             hlfir.assign %[[VAL_0]] to %[[VAL_2]] : i32, !fir.ref<i32>
 // CHECK:           }
 // CHECK:         }
+
+func.func @_QPtest8() {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.alloca !fir.array<120xi8> {uniq_name = "_QFtest8Ea"}
+  %c0 = arith.constant 0 : index
+  %2 = fir.coordinate_of %1, %c0 : (!fir.ref<!fir.array<120xi8>>, index) -> !fir.ref<i8>
+  %3 = fir.convert %2 : (!fir.ref<i8>) -> !fir.ptr<!fir.array<2x10xf32>>
+  %c2 = arith.constant 2 : index
+  %c10 = arith.constant 10 : index
+  %4 = fir.shape %c2, %c10 : (index, index) -> !fir.shape<2>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFtest8Ea"} : (!fir.ptr<!fir.array<2x10xf32>>, !fir.shape<2>) -> (!fir.ptr<!fir.array<2x10xf32>>, !fir.ptr<!fir.array<2x10xf32>>)
+  %c0_0 = arith.constant 0 : index
+  %6 = fir.coordinate_of %1, %c0_0 : (!fir.ref<!fir.array<120xi8>>, index) -> !fir.ref<i8>
+  %7 = fir.convert %6 : (!fir.ref<i8>) -> !fir.ptr<!fir.array<30xf32>>
+  %c30 = arith.constant 30 : index
+  %8 = fir.shape %c30 : (index) -> !fir.shape<1>
+  %9:2 = hlfir.declare %7(%8) {uniq_name = "_QFtest8Eb"} : (!fir.ptr<!fir.array<30xf32>>, !fir.shape<1>) -> (!fir.ptr<!fir.array<30xf32>>, !fir.ptr<!fir.array<30xf32>>)
+  %cst = arith.constant -1.000000e+00 : f32
+  hlfir.assign %cst to %5#0 : f32, !fir.ptr<!fir.array<2x10xf32>>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPtest8() {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest8Ea"} : (!fir.ptr<!fir.array<2x10xf32>>, !fir.shape<2>) -> (!fir.ptr<!fir.array<2x10xf32>>, !fir.ptr<!fir.array<2x10xf32>>)
+// CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ptr<!fir.array<2x10xf32>>) -> !fir.ref<!fir.array<20xf32>>
+// CHECK:           fir.do_loop %[[VAL_3:.*]] = %{{.*}} to %{{.*}} step %{{.*}} unordered {
+// CHECK:             %[[VAL_4:.*]] = hlfir.designate %[[VAL_2]] (%[[VAL_3]]) : (!fir.ref<!fir.array<20xf32>>, index) -> !fir.ref<f32>
+// CHECK:             hlfir.assign %[[VAL_0]] to %[[VAL_4]] : f32, !fir.ref<f32>
+// CHECK:           }
+// CHECK:         }

``````````

</details>


https://github.com/llvm/llvm-project/pull/147761

From flang-commits at lists.llvm.org  Wed Jul  9 07:37:40 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Wed, 09 Jul 2025 07:37:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686e7eb4.170a0220.222bfa.e020@mx.google.com>


luporl wrote:

@eugeneepshteyn, sorry for introducing another regression.

https://github.com/llvm/llvm-project/pull/147761 should fix it.

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Wed Jul  9 07:41:35 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 09 Jul 2025 07:41:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix array assignment regression
 introduced by #147371 (PR #147761)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147761@github.com>
Message-ID: <686e7f9f.630a0220.35d5c3.09fe@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/147761

From flang-commits at lists.llvm.org  Wed Jul  9 07:42:18 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 09 Jul 2025 07:42:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix optimization of array
 assignments after #146408 (PR #147371)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147371@github.com>
Message-ID: <686e7fca.170a0220.1fccb8.eec3@mx.google.com>


================
@@ -805,13 +805,12 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
+      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
----------------
eugeneepshteyn wrote:

Ok, lesson learned: "always listen to Jean" :)

https://github.com/llvm/llvm-project/pull/147371

From flang-commits at lists.llvm.org  Wed Jul  9 07:43:47 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 09 Jul 2025 07:43:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix array assignment regression
 introduced by #147371 (PR #147761)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147761@github.com>
Message-ID: <686e8023.170a0220.278be5.eba3@mx.google.com>


https://github.com/eugeneepshteyn approved this pull request.


https://github.com/llvm/llvm-project/pull/147761

From flang-commits at lists.llvm.org  Wed Jul  9 08:21:12 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Wed, 09 Jul 2025 08:21:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686e88e8.050a0220.3ac50b.60c4@mx.google.com>


DavidSpickett wrote:

Only tests in flang-rt use `llvm/` includes right now. Is that because it deliberately doesn't use llvm headers or just because I'm the first to try. I'll find out.

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 08:44:33 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Wed, 09 Jul 2025 08:44:33 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686e8e61.050a0220.3c7cf2.3d39@mx.google.com>


https://github.com/rolfmorel updated https://github.com/llvm/llvm-project/pull/145899

>From 6523e512f69d38efe8218590b501eb28737d6522 Mon Sep 17 00:00:00 2001
From: Rolf Morel <rolf.morel at intel.com>
Date: Thu, 26 Jun 2025 07:16:09 -0700
Subject: [PATCH 1/7] [MLIR][LLVMIR][DLTI] Add LLVMTargetAttrInterface,
 #llvm.target and #llvm.data_layout

---
 .../mlir/Dialect/LLVMIR/CMakeLists.txt        |  13 +-
 .../mlir/Dialect}/LLVMIR/DataLayoutImporter.h |  29 ++-
 .../Dialect/LLVMIR/LLVMAttrAndEnumDefs.td     |  10 +
 .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td       | 180 +++++++++++-------
 mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h  |  10 +-
 .../mlir/Dialect/LLVMIR/LLVMDialect.td        |   8 +
 .../mlir/Dialect/LLVMIR/LLVMInterfaces.td     |  43 +++++
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   |   1 +
 .../LLVMIR/LLVMTargetFeaturesAttrDefs.td      |  82 ++++++++
 .../mlir/Dialect/LLVMIR/Transforms/Passes.td  |   8 +
 mlir/lib/Dialect/DLTI/DLTI.cpp                |   2 +-
 mlir/lib/Dialect/DLTI/Traits.cpp              |   2 +-
 mlir/lib/Dialect/LLVMIR/CMakeLists.txt        |  11 ++
 .../LLVMIR/IR}/DataLayoutImporter.cpp         |  18 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp      |  76 +++++++-
 .../Dialect/LLVMIR/Transforms/CMakeLists.txt  |   1 +
 .../Transforms/DataLayoutFromTarget.cpp       |  62 ++++++
 mlir/lib/Target/LLVMIR/CMakeLists.txt         |   2 -
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |   2 +-
 19 files changed, 462 insertions(+), 98 deletions(-)
 rename mlir/{lib/Target => include/mlir/Dialect}/LLVMIR/DataLayoutImporter.h (82%)
 create mode 100644 mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td
 create mode 100644 mlir/include/mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.td
 rename mlir/lib/{Target/LLVMIR => Dialect/LLVMIR/IR}/DataLayoutImporter.cpp (94%)
 create mode 100644 mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp

diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index cfad07e57021f..311f1489e4e7e 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -7,12 +7,21 @@ mlir_tablegen(LLVMOpsDialect.h.inc -gen-dialect-decls)
 mlir_tablegen(LLVMOpsDialect.cpp.inc -gen-dialect-defs)
 mlir_tablegen(LLVMOpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(LLVMOpsEnums.cpp.inc -gen-enum-defs)
-mlir_tablegen(LLVMOpsAttrDefs.h.inc -gen-attrdef-decls
-              -attrdefs-dialect=llvm)
 mlir_tablegen(LLVMOpsAttrDefs.cpp.inc -gen-attrdef-defs
               -attrdefs-dialect=llvm)
 add_public_tablegen_target(MLIRLLVMOpsIncGen)
 
+# FIXME: explain how this solves generated header recursive dependency
+set(LLVM_TARGET_DEFINITIONS LLVMAttrAndEnumDefs.td)
+mlir_tablegen(LLVMOpsAttrDefs.h.inc -gen-attrdef-decls -attrdefs-dialect=llvm)
+add_public_tablegen_target(MLIRLLVMAttrsIncGen)
+
+# FIXME: explain how this solves generated header recursive dependency
+set(LLVM_TARGET_DEFINITIONS LLVMTargetFeaturesAttrDefs.td)
+mlir_tablegen(LLVMTargetFeaturesAttrDefs.h.inc -gen-attrdef-decls)
+mlir_tablegen(LLVMTargetFeaturesAttrDefs.cpp.inc -gen-attrdef-defs)
+add_public_tablegen_target(MLIRLLVMTargetFeaturesAttrsIncGen)
+
 set(LLVM_TARGET_DEFINITIONS LLVMTypes.td)
 mlir_tablegen(LLVMTypes.h.inc -gen-typedef-decls -typedefs-dialect=llvm)
 mlir_tablegen(LLVMTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=llvm)
diff --git a/mlir/lib/Target/LLVMIR/DataLayoutImporter.h b/mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
similarity index 82%
rename from mlir/lib/Target/LLVMIR/DataLayoutImporter.h
rename to mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
index 88ceaf1a74e62..435528540508b 100644
--- a/mlir/lib/Target/LLVMIR/DataLayoutImporter.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
@@ -11,12 +11,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
-#define MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
+#ifndef MLIR_LLVMIR_DATALAYOUTIMPORTER_H_
+#define MLIR_LLVMIR_DATALAYOUTIMPORTER_H_
 
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
+#include "llvm/IR/DataLayout.h"
 #include "llvm/ADT/MapVector.h"
 
 namespace llvm {
@@ -38,6 +39,8 @@ namespace detail {
 /// null if the bit width is not supported.
 FloatType getFloatType(MLIRContext *context, unsigned width);
 
+} // namespace detail
+
 /// Helper class that translates an LLVM data layout to an MLIR data layout
 /// specification. Only integer, float, pointer, alloca memory space, stack
 /// alignment, and endianness entries are translated. The class also returns all
@@ -49,7 +52,22 @@ class DataLayoutImporter {
   DataLayoutImporter(MLIRContext *context,
                      const llvm::DataLayout &llvmDataLayout)
       : context(context) {
-    translateDataLayout(llvmDataLayout);
+    // Transform the data layout to its string representation and append the
+    // default data layout string specified in the language reference
+    // (https://llvm.org/docs/LangRef.html#data-layout). The translation then
+    // parses the string and ignores the default value if a specific kind occurs
+    // in both strings. Additionally, the following default values exist:
+    // - non-default address space pointer specifications default to the default
+    //   address space pointer specification
+    // - the alloca address space defaults to the default address space.
+    layoutStr = llvmDataLayout.getStringRepresentation();
+    translateDataLayoutFromStr();
+  }
+
+  DataLayoutImporter(MLIRContext *context,
+                     StringRef dataLayoutStr)
+      : layoutStr(dataLayoutStr), context(context) {
+    translateDataLayoutFromStr();
   }
 
   /// Returns the MLIR data layout specification translated from the LLVM
@@ -66,7 +84,7 @@ class DataLayoutImporter {
 
 private:
   /// Translates the LLVM `dataLayout` to an MLIR data layout specification.
-  void translateDataLayout(const llvm::DataLayout &llvmDataLayout);
+  void translateDataLayoutFromStr();
 
   /// Tries to parse the letter only prefix that identifies the specification
   /// and removes the consumed characters from the beginning of the string.
@@ -125,8 +143,7 @@ class DataLayoutImporter {
   DataLayoutSpecInterface dataLayout;
 };
 
-} // namespace detail
 } // namespace LLVM
 } // namespace mlir
 
-#endif // MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
+#endif // MLIR_LLVMIR_DATALAYOUTIMPORTER_H_
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td
new file mode 100644
index 0000000000000..2a01ab88116d8
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td
@@ -0,0 +1,10 @@
+//===-- LLVMAttrDefs.td - Solely LLVM Attribute and Enum definitions ----*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
+include "mlir/Dialect/LLVMIR/LLVMEnums.td"
\ No newline at end of file
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 790d2e77ea874..705be61e17e76 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -13,13 +13,118 @@ include "mlir/Dialect/LLVMIR/LLVMDialect.td"
 include "mlir/Dialect/LLVMIR/LLVMInterfaces.td"
 include "mlir/IR/AttrTypeBase.td"
 include "mlir/IR/CommonAttrConstraints.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
 
-// All of the attributes will extend this class.
-class LLVM_Attr<string name, string attrMnemonic,
-                list<Trait> traits = [],
-                string baseCppClass = "::mlir::Attribute">
-    : AttrDef<LLVM_Dialect, name, traits, baseCppClass> {
-  let mnemonic = attrMnemonic;
+//===----------------------------------------------------------------------===//
+// LLVM_TargetAttr
+//===----------------------------------------------------------------------===//
+
+def LLVM_TargetAttr : LLVM_Attr<"Target", "target", [LLVM_TargetAttrInterface]> {
+  let mnemonic = "target";
+  let summary = "TODO";
+  let description = [{
+    TODO
+  }];
+  let parameters = (ins "StringAttr":$triple,
+                        "StringAttr":$cpu,
+                        "TargetFeaturesAttr":$target_features);
+
+  let assemblyFormat = "`<` $triple `,` $cpu `,` qualified($target_features) `>`";
+
+  let extraClassDeclaration = [{
+    std::optional<llvm::TargetMachine *> targetMachine = std::nullopt;
+
+    FailureOr<llvm::TargetMachine *> getTargetMachine();
+
+    std::optional<llvm::DataLayout> dataLayout = std::nullopt;
+    FailureOr<llvm::DataLayout> getDataLayout();
+    FailureOr<Attribute> query(DataLayoutEntryKey key);
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// LLVM_DataLayoutAttr
+//===----------------------------------------------------------------------===//
+
+def LLVM_DataLayoutAttr
+    : LLVM_Attr<"DataLayout", "data_layout", [DataLayoutSpecInterface]> {
+  let summary = "TODO";
+  let description = [{
+    TODO
+  }];
+  let parameters = (ins "StringAttr":$data_layout_str,
+                         OptionalParameter<"DataLayoutSpecInterface", "{}">:$data_layout_spec);
+  let builders = [
+    AttrBuilder<(ins "llvm::StringRef":$data_layout_str), [{
+      auto importer = LLVM::DataLayoutImporter($_ctxt, data_layout_str);
+      auto dataLayoutSpec = importer.getDataLayout();
+      return $_get($_ctxt, mlir::StringAttr::get($_ctxt, data_layout_str), dataLayoutSpec);
+    }]>
+  ];
+  let assemblyFormat = "`<` $data_layout_str `>`";
+  let extraClassDeclaration = [{
+    template <typename Ty>
+    DataLayoutEntryList getSpecForType() {
+      return getDataLayoutSpec().getSpecForType(TypeID::get<Ty>());
+    }
+
+    inline ::mlir::FailureOr<::mlir::Attribute>
+    queryHelper(::mlir::DataLayoutEntryKey key) const {
+      return getDataLayoutSpec().queryHelper(key);
+    }
+
+    void bucketEntriesByType(
+        ::llvm::MapVector<::mlir::TypeID, ::mlir::DataLayoutEntryList> &types,
+        ::llvm::MapVector<::mlir::StringAttr,
+                          ::mlir::DataLayoutEntryInterface> &ids) {
+      getDataLayoutSpec().bucketEntriesByType(types, ids);
+    };
+
+    ::mlir::DataLayoutSpecInterface
+    combineWith(ArrayRef<::mlir::DataLayoutSpecInterface> specs) const {
+      return getDataLayoutSpec().combineWith(specs);
+    }
+    DataLayoutEntryListRef getEntries() const { return getDataLayoutSpec().getEntries(); }
+    LogicalResult verifySpec(Location loc) {
+      return getDataLayoutSpec().verifySpec(loc);
+    }
+    StringAttr getEndiannessIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getEndiannessIdentifier(context);
+    }
+    StringAttr getDefaultMemorySpaceIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getDefaultMemorySpaceIdentifier(context);
+    }
+    StringAttr getAllocaMemorySpaceIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getAllocaMemorySpaceIdentifier(context);
+    }
+    StringAttr getManglingModeIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getManglingModeIdentifier(context);
+    }
+    StringAttr getProgramMemorySpaceIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getProgramMemorySpaceIdentifier(context);
+    }
+    StringAttr getGlobalMemorySpaceIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getGlobalMemorySpaceIdentifier(context);
+    }
+    StringAttr getStackAlignmentIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getStackAlignmentIdentifier(context);
+    }
+    StringAttr getFunctionPointerAlignmentIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getFunctionPointerAlignmentIdentifier(context);
+    }
+    StringAttr getLegalIntWidthsIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getLegalIntWidthsIdentifier(context);
+    }
+    ::mlir::DataLayoutEntryList getSpecForType(TypeID type) const {
+      return getDataLayoutSpec().getSpecForType(type);
+    }
+    ::mlir::DataLayoutEntryInterface getSpecForIdentifier(StringAttr identifier) const {
+      return getDataLayoutSpec().getSpecForIdentifier(identifier);
+    }
+    FailureOr<Attribute> query(DataLayoutEntryKey key) const {
+      return getDataLayoutSpec().query(key);
+    }
+  }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -1241,69 +1346,6 @@ def LLVM_VScaleRangeAttr : LLVM_Attr<"VScaleRange", "vscale_range"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// TargetFeaturesAttr
-//===----------------------------------------------------------------------===//
-
-def LLVM_TargetFeaturesAttr : LLVM_Attr<"TargetFeatures", "target_features">
-{
-  let summary = "LLVM target features attribute";
-
-  let description = [{
-    Represents the LLVM target features as a list that can be checked within
-    passes/rewrites.
-
-    Example:
-    ```mlir
-    #llvm.target_features<["+sme", "+sve", "+sme-f64f64"]>
-    ```
-
-    Then within a pass or rewrite the features active at an op can be queried:
-
-    ```c++
-    auto targetFeatures = LLVM::TargetFeaturesAttr::featuresAt(op);
-
-    if (!targetFeatures.contains("+sme-f64f64"))
-      return failure();
-    ```
-  }];
-
-  let parameters = (ins OptionalArrayRefParameter<"StringAttr">:$features);
-
-  let builders = [
-    TypeBuilder<(ins "::llvm::StringRef":$features)>,
-    TypeBuilder<(ins "::llvm::ArrayRef<::llvm::StringRef>":$features)>
-  ];
-
-  let extraClassDeclaration = [{
-    /// Checks if a feature is contained within the features list.
-    /// Note: Using a StringAttr allows doing pointer-comparisons.
-    bool contains(::mlir::StringAttr feature) const;
-    bool contains(::llvm::StringRef feature) const;
-
-    bool nullOrEmpty() const {
-      // Checks if this attribute is null, or the features are empty.
-      return !bool(*this) || getFeatures().empty();
-    }
-
-    /// Returns the list of features as an LLVM-compatible string.
-    std::string getFeaturesString() const;
-
-    /// Finds the target features on the parent FunctionOpInterface.
-    /// Note: This assumes the attribute name matches the return value of
-    /// `getAttributeName()`.
-    static TargetFeaturesAttr featuresAt(Operation* op);
-
-    /// Canonical name for this attribute within MLIR.
-    static constexpr StringLiteral getAttributeName() {
-      return StringLiteral("target_features");
-    }
-  }];
-
-  let assemblyFormat = "`<` `[` (`]`) : ($features^ `]`)? `>`";
-  let genVerifyDecl = 1;
-}
-
 //===----------------------------------------------------------------------===//
 // UndefAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
index 3ede857733242..371c202374225 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
@@ -14,7 +14,9 @@
 #ifndef MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 #define MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 
-#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Target/TargetMachine.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
 #include "mlir/IR/OpImplementation.h"
 #include <optional>
 
@@ -94,6 +96,12 @@ using linkage::Linkage;
 } // namespace LLVM
 } // namespace mlir
 
+// First obtain TargetFeaturesAttr definitions as it is used both an LLVMIR
+// interface and that interface and this attribute are turn required by another
+// LLVMIR attribute.
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.h.inc"
+
 #include "mlir/Dialect/LLVMIR/LLVMAttrInterfaces.h.inc"
 
 #define GET_ATTRDEF_CLASSES
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
index b5ea8fc5da500..e924be32da10f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
@@ -10,6 +10,7 @@
 #define LLVMIR_DIALECT
 
 include "mlir/IR/DialectBase.td"
+include "mlir/IR/AttrTypeBase.td"
 
 def LLVM_Dialect : Dialect {
   let name = "llvm";
@@ -123,4 +124,11 @@ def LLVM_Dialect : Dialect {
   }];
 }
 
+class LLVM_Attr<string name, string attrMnemonic,
+                list<Trait> traits = [],
+                string baseCppClass = "::mlir::Attribute">
+    : AttrDef<LLVM_Dialect, name, traits, baseCppClass> {
+  let mnemonic = attrMnemonic;
+}
+
 #endif  // LLVMIR_DIALECT
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
index 138170f8c8762..b7c1c6766fa60 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
@@ -14,6 +14,7 @@
 #define LLVMIR_INTERFACES
 
 include "mlir/IR/OpBase.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
 
 def FastmathFlagsInterface : OpInterface<"FastmathFlagsInterface"> {
   let description = [{
@@ -532,4 +533,46 @@ def LLVM_DIRecursiveTypeAttrInterface
   ];
 }
 
+def LLVM_TargetAttrInterface
+  : AttrInterface<"TargetAttrInterface", [DLTIQueryInterface]> {
+  let description = [{
+    TODO
+
+    NEEDS TO document which TI keys it responds to.
+  }];
+  let cppNamespace = "::mlir::LLVM";
+  let methods = [
+    InterfaceMethod<
+      /*description=*/"Returns the target triple identifier.",
+      /*retTy=*/"::llvm::StringRef",
+      /*methodName=*/"getTriple",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the target cpu identifier.",
+      /*retTy=*/"::llvm::StringRef",
+      /*methodName=*/"getCpu",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the target features as a string.",
+      /*retTy=*/"::mlir::LLVM::TargetFeaturesAttr",
+      /*methodName=*/"getTargetFeatures",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the target machine.",
+      /*retTy=*/"FailureOr<::llvm::TargetMachine *>",
+      /*methodName=*/"getTargetMachine",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the data layout associated to the target machine.",
+      /*retTy=*/"FailureOr<::llvm::DataLayout>",
+      /*methodName=*/"getDataLayout",
+      /*args=*/(ins)
+    >
+  ];
+}
+
 #endif // LLVMIR_INTERFACES
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index f4c1640098320..36a6291b5e3a8 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -14,6 +14,7 @@
 #define LLVMIR_OPS
 
 include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
+include "mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.td"
 include "mlir/Dialect/LLVMIR/LLVMEnums.td"
 include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
 include "mlir/IR/EnumAttr.td"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.td
new file mode 100644
index 0000000000000..2183f083103f5
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.td
@@ -0,0 +1,82 @@
+//===-- LLVMAttrDefs.td - LLVM Attributes definition file --*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVMIR_TARGETFEATURESATTRDEFS
+#define LLVMIR_TARGETFEATURESATTRDEFS
+
+include "mlir/Dialect/LLVMIR/LLVMDialect.td"
+include "mlir/IR/CommonAttrConstraints.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
+
+//===----------------------------------------------------------------------===//
+// TargetFeaturesAttr
+//===----------------------------------------------------------------------===//
+
+def LLVM_TargetFeaturesAttr : LLVM_Attr<"TargetFeatures", "target_features",
+    [DLTIQueryInterface]> {
+  let summary = "LLVM target features attribute";
+
+  let description = [{
+    Represents the LLVM target features as a list that can be checked within
+    passes/rewrites.
+
+    Example:
+    ```mlir
+    #llvm.target_features<["+sme", "+sve", "+sme-f64f64"]>
+    ```
+
+    Then within a pass or rewrite the features active at an op can be queried:
+
+    ```c++
+    auto targetFeatures = LLVM::TargetFeaturesAttr::featuresAt(op);
+
+    if (!targetFeatures.contains("+sme-f64f64"))
+      return failure();
+    ```
+  }];
+
+  let parameters = (ins OptionalArrayRefParameter<"StringAttr">:$features);
+
+  let builders = [
+    TypeBuilder<(ins "::llvm::StringRef":$features)>,
+    TypeBuilder<(ins "::llvm::ArrayRef<::llvm::StringRef>":$features)>
+  ];
+
+  let extraClassDeclaration = [{
+    /// Checks if a feature is contained within the features list.
+    /// Note: Using a StringAttr allows doing pointer-comparisons.
+    bool contains(::mlir::StringAttr feature) const;
+    bool contains(::llvm::StringRef feature) const;
+
+    bool nullOrEmpty() const {
+      // Checks if this attribute is null, or the features are empty.
+      return !bool(*this) || getFeatures().empty();
+    }
+
+    /// Returns the list of features as an LLVM-compatible string.
+    std::string getFeaturesString() const;
+
+    /// Finds the target features on the parent FunctionOpInterface.
+    /// Note: This assumes the attribute name matches the return value of
+    /// `getAttributeName()`.
+    static TargetFeaturesAttr featuresAt(Operation* op);
+
+    /// Canonical name for this attribute within MLIR.
+    static constexpr StringLiteral getAttributeName() {
+      return StringLiteral("target_features");
+    }
+
+    /// Returns the attribute associated with the key.
+    FailureOr<Attribute> query(DataLayoutEntryKey key);
+  }];
+
+  let assemblyFormat = "`<` `[` (`]`) : ($features^ `]`)? `>`";
+  let genVerifyDecl = 1;
+}
+
+#endif // LLVMIR_TARGETFEATURESATTRDEFS
diff --git a/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.td b/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.td
index 961909d5c8d27..5920c3b7b0332 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.td
@@ -45,6 +45,14 @@ def LLVMRequestCWrappersPass
   }];
 }
 
+def LLVMDataLayoutFromTarget : Pass<"llvm-data-layout-from-target", "::mlir::ModuleOp"> {
+  let summary = "TODO";
+  let dependentDialects = ["mlir::DLTIDialect"];
+  let description = [{
+  TODO
+  }];
+}
+
 def NVVMOptimizeForTargetPass : Pass<"llvm-optimize-for-nvvm-target"> {
   let summary = "Optimize NVVM IR";
 }
diff --git a/mlir/lib/Dialect/DLTI/DLTI.cpp b/mlir/lib/Dialect/DLTI/DLTI.cpp
index 0c4fe27ffa58e..62ecf7d759959 100644
--- a/mlir/lib/Dialect/DLTI/DLTI.cpp
+++ b/mlir/lib/Dialect/DLTI/DLTI.cpp
@@ -657,7 +657,7 @@ void DLTIDialect::initialize() {
 LogicalResult DLTIDialect::verifyOperationAttribute(Operation *op,
                                                     NamedAttribute attr) {
   if (attr.getName() == DLTIDialect::kDataLayoutAttrName) {
-    if (!llvm::isa<DataLayoutSpecAttr>(attr.getValue())) {
+    if (!llvm::dyn_cast<DataLayoutSpecInterface>(attr.getValue())) {
       return op->emitError() << "'" << DLTIDialect::kDataLayoutAttrName
                              << "' is expected to be a #dlti.dl_spec attribute";
     }
diff --git a/mlir/lib/Dialect/DLTI/Traits.cpp b/mlir/lib/Dialect/DLTI/Traits.cpp
index 34f2dd5896083..3f6dd2900a915 100644
--- a/mlir/lib/Dialect/DLTI/Traits.cpp
+++ b/mlir/lib/Dialect/DLTI/Traits.cpp
@@ -24,7 +24,7 @@ LogicalResult mlir::impl::verifyHasDefaultDLTIDataLayoutTrait(Operation *op) {
 }
 
 DataLayoutSpecInterface mlir::impl::getDataLayoutSpec(Operation *op) {
-  return op->getAttrOfType<DataLayoutSpecAttr>(
+  return op->getAttrOfType<DataLayoutSpecInterface>(
       DLTIDialect::kDataLayoutAttrName);
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index d987b72e98354..c3dea62df14c2 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_subdirectory(Transforms)
 
 add_mlir_dialect_library(MLIRLLVMDialect
+  IR/DataLayoutImporter.cpp
   IR/FunctionCallUtils.cpp
   IR/LLVMAttrs.cpp
   IR/LLVMDialect.cpp
@@ -13,6 +14,7 @@ add_mlir_dialect_library(MLIRLLVMDialect
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
 
   DEPENDS
+  MLIRLLVMTargetFeaturesAttrsIncGen
   MLIRLLVMOpsIncGen
   MLIRLLVMTypesIncGen
   MLIRLLVMIntrinsicOpsIncGen
@@ -26,10 +28,19 @@ add_mlir_dialect_library(MLIRLLVMDialect
   BitReader
   BitWriter
   Core
+  MC
+  Target
+  Support
+  TargetParser
+  AllTargetsAsmParsers
+  AllTargetsCodeGens
+  AllTargetsDescs
+  AllTargetsInfos
 
   LINK_LIBS PUBLIC
   MLIRCallInterfaces
   MLIRControlFlowInterfaces
+  MLIRDLTIDialect
   MLIRDataLayoutInterfaces
   MLIRFunctionInterfaces
   MLIRInferTypeOpInterface
diff --git a/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp b/mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp
similarity index 94%
rename from mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
rename to mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp
index fbad5c2fb78d9..f5fbd81503806 100644
--- a/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp
@@ -6,7 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "DataLayoutImporter.h"
+//#include "DataLayoutImporter.h"
+//#include "mlir/Target/LLVMIR/DataLayoutImporter.h"
+#include "mlir/Dialect/LLVMIR/DataLayoutImporter.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
@@ -274,19 +276,7 @@ DataLayoutImporter::tryToEmplaceLegalIntWidthsEntry(StringRef token) {
   return success();
 }
 
-void DataLayoutImporter::translateDataLayout(
-    const llvm::DataLayout &llvmDataLayout) {
-  dataLayout = {};
-
-  // Transform the data layout to its string representation and append the
-  // default data layout string specified in the language reference
-  // (https://llvm.org/docs/LangRef.html#data-layout). The translation then
-  // parses the string and ignores the default value if a specific kind occurs
-  // in both strings. Additionally, the following default values exist:
-  // - non-default address space pointer specifications default to the default
-  //   address space pointer specification
-  // - the alloca address space defaults to the default address space.
-  layoutStr = llvmDataLayout.getStringRepresentation();
+void DataLayoutImporter::translateDataLayoutFromStr() {
   if (!layoutStr.empty())
     layoutStr += "-";
   layoutStr += kDefaultDataLayout;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
index 915954a7f5f51..5cc11516b5df2 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
+#include "mlir/Dialect/LLVMIR/DataLayoutImporter.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
@@ -18,8 +19,11 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfoMetadata.h"
-#include <optional>
+#include "llvm/Support/TargetSelect.h"
+
+#define DEBUG_TYPE "llvm-attrs"
 
 using namespace mlir;
 using namespace mlir::LLVM;
@@ -404,3 +408,73 @@ ModuleFlagAttr::verify(function_ref<InFlightDiagnostic()> emitError,
                         "supported for unknown key '"
                      << key << "'";
 }
+
+FailureOr<Attribute> TargetFeaturesAttr::query(DataLayoutEntryKey key) {
+  if (auto stringKey = dyn_cast<StringAttr>(key))
+    if (contains(stringKey))
+      return UnitAttr::get(getContext());
+  return failure();
+}
+
+//===----------------------------------------------------------------------===//
+// LLVM_TargetAttr
+//===----------------------------------------------------------------------===//
+
+FailureOr<llvm::TargetMachine *> TargetAttr::getTargetMachine() {
+  if (targetMachine.has_value()) {
+    llvm::TargetMachine *tm = targetMachine.value();
+    if (tm != nullptr)
+      return {tm};
+    return failure();
+  }
+  llvm::InitializeAllTargets();
+  llvm::InitializeAllTargetMCs();
+
+  std::string error;
+  const llvm::Target *target =
+      llvm::TargetRegistry::lookupTarget(getTriple(), error);
+  if (!error.empty()) {
+    LLVM_DEBUG({
+      llvm::dbgs() << "Failed to retrieve the target with: `" << error << "`\n";
+    });
+    targetMachine = {nullptr};
+    return failure();
+  }
+
+  targetMachine = {target->createTargetMachine(
+      llvm::Triple(getTriple().strref()), getCpu().strref(),
+      getTargetFeatures().getFeaturesString().c_str(), {}, {})};
+
+  return {targetMachine.value()};
+}
+
+FailureOr<llvm::DataLayout> TargetAttr::getDataLayout() {
+  if (dataLayout.has_value()) {
+    return dataLayout.value();
+  }
+
+  FailureOr<llvm::TargetMachine *> targetMachine = getTargetMachine();
+  if (failed(targetMachine)) {
+    LLVM_DEBUG({
+      llvm::dbgs()
+          << "Failed to retrieve the target machine for data layout.\n";
+    });
+    dataLayout = std::nullopt;
+    return failure();
+  }
+  dataLayout = (targetMachine.value())->createDataLayout();
+  return dataLayout.value();
+}
+
+FailureOr<::mlir::Attribute> TargetAttr::query(DataLayoutEntryKey key) {
+  if (auto stringAttrKey = dyn_cast<StringAttr>(key)) {
+    if (stringAttrKey.getValue() == "triple")
+      return getTriple();
+    if (stringAttrKey.getValue() == "cpu")
+      return getCpu();
+    if (stringAttrKey.getValue() == "features")
+      return getTargetFeatures();
+  }
+
+  return failure();
+}
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt
index d4ff0955c5d0e..cc6289b485c55 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt
@@ -7,6 +7,7 @@ add_mlir_dialect_library(MLIRLLVMIRTransforms
   LegalizeForExport.cpp
   OptimizeForNVVM.cpp
   RequestCWrappers.cpp
+  DataLayoutFromTarget.cpp
 
   DEPENDS
   MLIRLLVMPassIncGen
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp
new file mode 100644
index 0000000000000..c2cccd37dd720
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp
@@ -0,0 +1,62 @@
+//===- DataLayoutFromTarget.cpp - extract data layout from TargetMachine --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/DLTI/DLTI.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Dialect/LLVMIR/DataLayoutImporter.h"
+
+namespace mlir {
+namespace LLVM {
+#define GEN_PASS_DEF_LLVMDATALAYOUTFROMTARGET
+#include "mlir/Dialect/LLVMIR/Transforms/Passes.h.inc"
+} // namespace LLVM
+} // namespace mlir
+
+using namespace mlir;
+using namespace mlir::LLVM;
+
+struct DataLayoutFromTargetPass
+    : public LLVM::impl::LLVMDataLayoutFromTargetBase<DataLayoutFromTargetPass> {
+  void runOnOperation() override {
+    ModuleOp mod = getOperation();
+
+    bool passFailed = false;
+
+    mod->walk([&](ModuleOp mod) {
+      auto targetAttr =
+          mod->getAttrOfType<LLVM::TargetAttrInterface>("llvm.target");
+      if (!targetAttr)
+        return;
+
+      FailureOr<llvm::DataLayout> dataLayout = targetAttr.getDataLayout();
+      if (failed(dataLayout)) {
+        mod->emitError() << "failed to obtain llvm::DataLayout from "
+                         << targetAttr;
+        passFailed = true;
+      }
+      auto dataLayoutAttr = DataLayoutAttr::get(
+          &getContext(), dataLayout->getStringRepresentation());
+
+      StringRef dlSpecIdentifier = "dlti.dl_spec";
+      auto existingDlSpec =
+          mod->getAttrOfType<DataLayoutSpecInterface>(dlSpecIdentifier);
+      if (existingDlSpec) {
+        DataLayoutSpecInterface dataLayoutSpec =
+            existingDlSpec.combineWith({dataLayoutAttr.getDataLayoutSpec()});
+        mod->setAttr(dlSpecIdentifier, dataLayoutSpec);
+      } else {
+        mod->setAttr(dlSpecIdentifier, dataLayoutAttr);
+      }
+    });
+
+    if (passFailed) {
+      return signalPassFailure();
+    }
+  }
+};
diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt
index af22a7ff04bf0..f6d70fe05a163 100644
--- a/mlir/lib/Target/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt
@@ -3,7 +3,6 @@ add_subdirectory(Dialect)
 set(LLVM_OPTIONAL_SOURCES
   ConvertFromLLVMIR.cpp
   ConvertToLLVMIR.cpp
-  DataLayoutImporter.cpp
   DebugTranslation.cpp
   DebugImporter.cpp
   LoopAnnotationImporter.cpp
@@ -63,7 +62,6 @@ add_mlir_translation_library(MLIRToLLVMIRTranslationRegistration
   )
 
 add_mlir_translation_library(MLIRTargetLLVMIRImport
-  DataLayoutImporter.cpp
   DebugImporter.cpp
   LoopAnnotationImporter.cpp
   ModuleImport.cpp
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index bfda223fe0f5f..8b570d669a0f3 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -16,11 +16,11 @@
 #include "mlir/Target/LLVMIR/Import.h"
 
 #include "AttrKindDetail.h"
-#include "DataLayoutImporter.h"
 #include "DebugImporter.h"
 #include "LoopAnnotationImporter.h"
 
 #include "mlir/Dialect/DLTI/DLTI.h"
+#include "mlir/Dialect/LLVMIR/DataLayoutImporter.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Matchers.h"

>From 0a1903b187ac6af1f9a9b6400c17ac289bff96a1 Mon Sep 17 00:00:00 2001
From: Rolf Morel <rolf.morel at intel.com>
Date: Thu, 26 Jun 2025 09:15:32 -0700
Subject: [PATCH 2/7] Try to fix Flang...

---
 flang/include/flang/Optimizer/Dialect/FIRType.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/include/flang/Optimizer/Dialect/FIRType.h b/flang/include/flang/Optimizer/Dialect/FIRType.h
index ecab12de55d61..83077aef8d08d 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRType.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRType.h
@@ -13,6 +13,7 @@
 #ifndef FORTRAN_OPTIMIZER_DIALECT_FIRTYPE_H
 #define FORTRAN_OPTIMIZER_DIALECT_FIRTYPE_H
 
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"

>From eb1cbb569976729823fe9a27b2f1f100b742c8ea Mon Sep 17 00:00:00 2001
From: Rolf Morel <rolf.morel at intel.com>
Date: Thu, 26 Jun 2025 10:02:48 -0700
Subject: [PATCH 3/7] Fix formatting

---
 mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h       | 5 ++---
 mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td     | 2 +-
 mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h                | 6 +++---
 mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp           | 2 --
 mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp | 5 +++--
 5 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h b/mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
index 435528540508b..0f036f1c43492 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
@@ -17,8 +17,8 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
-#include "llvm/IR/DataLayout.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/IR/DataLayout.h"
 
 namespace llvm {
 class StringRef;
@@ -64,8 +64,7 @@ class DataLayoutImporter {
     translateDataLayoutFromStr();
   }
 
-  DataLayoutImporter(MLIRContext *context,
-                     StringRef dataLayoutStr)
+  DataLayoutImporter(MLIRContext *context, StringRef dataLayoutStr)
       : layoutStr(dataLayoutStr), context(context) {
     translateDataLayoutFromStr();
   }
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td
index 2a01ab88116d8..e34375076ffd1 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td
@@ -7,4 +7,4 @@
 //===----------------------------------------------------------------------===//
 
 include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
-include "mlir/Dialect/LLVMIR/LLVMEnums.td"
\ No newline at end of file
+include "mlir/Dialect/LLVMIR/LLVMEnums.td"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
index 371c202374225..14645d5dee95f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
@@ -14,10 +14,10 @@
 #ifndef MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 #define MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 
+#include "mlir/IR/OpImplementation.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Target/TargetMachine.h"
-#include "mlir/Interfaces/DataLayoutInterfaces.h"
-#include "mlir/IR/OpImplementation.h"
 #include <optional>
 
 #include "mlir/Dialect/LLVMIR/LLVMOpsEnums.h.inc"
@@ -91,8 +91,8 @@ class TBAANodeAttr : public Attribute {
 // TODO: this shouldn't be needed after we unify the attribute generation, i.e.
 // --gen-attr-* and --gen-attrdef-*.
 using cconv::CConv;
-using tailcallkind::TailCallKind;
 using linkage::Linkage;
+using tailcallkind::TailCallKind;
 } // namespace LLVM
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp b/mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp
index f5fbd81503806..988fe27e7cd8e 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp
@@ -6,8 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-//#include "DataLayoutImporter.h"
-//#include "mlir/Target/LLVMIR/DataLayoutImporter.h"
 #include "mlir/Dialect/LLVMIR/DataLayoutImporter.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/IR/Builders.h"
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp
index c2cccd37dd720..f43709ab1aca5 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/DLTI/DLTI.h"
+#include "mlir/Dialect/LLVMIR/DataLayoutImporter.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Pass/Pass.h"
-#include "mlir/Dialect/LLVMIR/DataLayoutImporter.h"
 
 namespace mlir {
 namespace LLVM {
@@ -22,7 +22,8 @@ using namespace mlir;
 using namespace mlir::LLVM;
 
 struct DataLayoutFromTargetPass
-    : public LLVM::impl::LLVMDataLayoutFromTargetBase<DataLayoutFromTargetPass> {
+    : public LLVM::impl::LLVMDataLayoutFromTargetBase<
+          DataLayoutFromTargetPass> {
   void runOnOperation() override {
     ModuleOp mod = getOperation();
 

>From b6a07f9559bec8dc3a3264f2f5da4178b34551f4 Mon Sep 17 00:00:00 2001
From: Rolf Morel <rolf.morel at intel.com>
Date: Thu, 26 Jun 2025 14:37:49 -0700
Subject: [PATCH 4/7] Minor fixes

---
 mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td  |  8 +++++---
 .../include/mlir/Dialect/LLVMIR/LLVMInterfaces.td |  6 +++---
 mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp          | 15 ++++++++-------
 .../LLVMIR/Transforms/DataLayoutFromTarget.cpp    |  1 +
 4 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 705be61e17e76..e0480f4be82ed 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -26,10 +26,12 @@ def LLVM_TargetAttr : LLVM_Attr<"Target", "target", [LLVM_TargetAttrInterface]>
     TODO
   }];
   let parameters = (ins "StringAttr":$triple,
-                        "StringAttr":$cpu,
-                        "TargetFeaturesAttr":$target_features);
+                        OptionalParameter<"StringAttr">:$chip,
+                        OptionalParameter<"TargetFeaturesAttr">:$features);
 
-  let assemblyFormat = "`<` $triple `,` $cpu `,` qualified($target_features) `>`";
+  let assemblyFormat = [{`<` `triple` `=` $triple
+                         (`,` `chip` `=` $chip^)?
+                         (`,` qualified($features)^)? `>`}];
 
   let extraClassDeclaration = [{
     std::optional<llvm::TargetMachine *> targetMachine = std::nullopt;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
index b7c1c6766fa60..72facd13994f7 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
@@ -549,15 +549,15 @@ def LLVM_TargetAttrInterface
       /*args=*/(ins)
     >,
     InterfaceMethod<
-      /*description=*/"Returns the target cpu identifier.",
+      /*description=*/"Returns the target chip (i.e. \"cpu\") identifier.",
       /*retTy=*/"::llvm::StringRef",
-      /*methodName=*/"getCpu",
+      /*methodName=*/"getChip",
       /*args=*/(ins)
     >,
     InterfaceMethod<
       /*description=*/"Returns the target features as a string.",
       /*retTy=*/"::mlir::LLVM::TargetFeaturesAttr",
-      /*methodName=*/"getTargetFeatures",
+      /*methodName=*/"getFeatures",
       /*args=*/(ins)
     >,
     InterfaceMethod<
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
index 5cc11516b5df2..dbe91e001f1cb 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
@@ -433,7 +433,7 @@ FailureOr<llvm::TargetMachine *> TargetAttr::getTargetMachine() {
   std::string error;
   const llvm::Target *target =
       llvm::TargetRegistry::lookupTarget(getTriple(), error);
-  if (!error.empty()) {
+  if (!target || !error.empty()) {
     LLVM_DEBUG({
       llvm::dbgs() << "Failed to retrieve the target with: `" << error << "`\n";
     });
@@ -442,8 +442,8 @@ FailureOr<llvm::TargetMachine *> TargetAttr::getTargetMachine() {
   }
 
   targetMachine = {target->createTargetMachine(
-      llvm::Triple(getTriple().strref()), getCpu().strref(),
-      getTargetFeatures().getFeaturesString().c_str(), {}, {})};
+      llvm::Triple(getTriple().strref()), getChip() ? getChip().strref() : "",
+      getFeatures() ? getFeatures().getFeaturesString() : "", {}, {})};
 
   return {targetMachine.value()};
 }
@@ -467,13 +467,14 @@ FailureOr<llvm::DataLayout> TargetAttr::getDataLayout() {
 }
 
 FailureOr<::mlir::Attribute> TargetAttr::query(DataLayoutEntryKey key) {
+  Attribute result;
   if (auto stringAttrKey = dyn_cast<StringAttr>(key)) {
     if (stringAttrKey.getValue() == "triple")
       return getTriple();
-    if (stringAttrKey.getValue() == "cpu")
-      return getCpu();
-    if (stringAttrKey.getValue() == "features")
-      return getTargetFeatures();
+    if (stringAttrKey.getValue() == "chip" && (result = getChip()))
+      return result;
+    if (stringAttrKey.getValue() == "features" && (result = getFeatures()))
+      return result;
   }
 
   return failure();
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp
index f43709ab1aca5..06990525d87ac 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp
@@ -40,6 +40,7 @@ struct DataLayoutFromTargetPass
         mod->emitError() << "failed to obtain llvm::DataLayout from "
                          << targetAttr;
         passFailed = true;
+        return;
       }
       auto dataLayoutAttr = DataLayoutAttr::get(
           &getContext(), dataLayout->getStringRepresentation());

>From ef55a799bf6fd30611de5df40d4036c445490c34 Mon Sep 17 00:00:00 2001
From: Rolf Morel <rolf.morel at intel.com>
Date: Thu, 26 Jun 2025 14:39:43 -0700
Subject: [PATCH 5/7] Add first two tests

---
 .../data-layout-from-target-invalid.mlir      |  8 ++++
 .../LLVMIR/data-layout-from-target.mlir       | 41 +++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 mlir/test/Dialect/LLVMIR/data-layout-from-target-invalid.mlir
 create mode 100644 mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir

diff --git a/mlir/test/Dialect/LLVMIR/data-layout-from-target-invalid.mlir b/mlir/test/Dialect/LLVMIR/data-layout-from-target-invalid.mlir
new file mode 100644
index 0000000000000..71526663abf96
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/data-layout-from-target-invalid.mlir
@@ -0,0 +1,8 @@
+// RUN: mlir-opt %s -llvm-data-layout-from-target --split-input-file --verify-diagnostics
+
+// expected-error @+1 {{failed to obtain llvm::DataLayout from #llvm.target}}
+module attributes { dlti.dl_spec = #dlti.dl_spec<index = 32>,
+llvm.target =
+    #llvm.target<triple="x64_86-unknown-linux",
+                 chip="lakesky"> } {
+}
diff --git a/mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir b/mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir
new file mode 100644
index 0000000000000..5983a34fe1e0d
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir
@@ -0,0 +1,41 @@
+// RUN: mlir-opt -llvm-data-layout-from-target -split-input-file %s | FileCheck %s
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #llvm.data_layout
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-NOT:    chip =
+// CHECK-NOT:    #llvm.target_features
+
+module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux"> } {
+}
+
+// -----
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #llvm.data_layout<"[[DL_STR:.*]]">
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-SAME:   chip = "skylake"
+// CHECK-SAME:   #llvm.target_features<["+mmx", "+sse"]>
+
+module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+                                               chip = "skylake",
+                                               #llvm.target_features<["+mmx", "+sse"]>> } {
+}
+
+// -----
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #dlti.dl_spec<"dlti.endianness" = "little"
+// CHECK-SAME:   index = 32
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-SAME:   chip = "skylake"
+// CHECK-SAME:   #llvm.target_features<["+mmx", "+sse"]>
+
+module attributes { dlti.dl_spec = #dlti.dl_spec<index = 32>,
+                    llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+                                               chip = "skylake",
+                                               #llvm.target_features<["+mmx", "+sse"]>> } {
+}

>From 9b67848c47996d6eef87338b136bc53c2734665b Mon Sep 17 00:00:00 2001
From: Rolf Morel <rolf.morel at intel.com>
Date: Tue, 8 Jul 2025 09:19:33 -0700
Subject: [PATCH 6/7] Update docs

---
 .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td       | 30 +++++++++++++++----
 .../mlir/Dialect/LLVMIR/LLVMInterfaces.td     | 16 ++++++++--
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index e0480f4be82ed..e563441d0102b 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -19,11 +19,24 @@ include "mlir/Interfaces/DataLayoutInterfaces.td"
 // LLVM_TargetAttr
 //===----------------------------------------------------------------------===//
 
-def LLVM_TargetAttr : LLVM_Attr<"Target", "target", [LLVM_TargetAttrInterface]> {
-  let mnemonic = "target";
-  let summary = "TODO";
+def LLVM_TargetAttr : LLVM_Attr<"Target", "target",
+                                [LLVM_TargetAttrInterface]> {
+  let summary = "LLVM target info: triple, chip, features";
   let description = [{
-    TODO
+    An attribute to hold LLVM target information, specifying LLVM's target
+    `triple` string, the target `chip` string (i.e. the `cpu` string), and
+    target `features` string as an attribute. The latter two are optional.
+
+    Has facilities to obtain the corresponding `llvm::TargetMachine` and
+    `llvm::DataLayout`, given the relevant LLVM backend is loaded.
+
+    ---
+
+    Responds to DLTI-queries on the keys:
+       * A query for `"triple"` returns the `StringAttr` for the `triple`.
+       * A query for `"chip"` returns the `StringAttr` for the `chip`/`cpu`, if provided.
+       * A query for `"features"` returns the `TargetFeaturesAttr`, if provided.
+         * Individual features can be queried for on this attribute.
   }];
   let parameters = (ins "StringAttr":$triple,
                         OptionalParameter<"StringAttr">:$chip,
@@ -50,9 +63,14 @@ def LLVM_TargetAttr : LLVM_Attr<"Target", "target", [LLVM_TargetAttrInterface]>
 
 def LLVM_DataLayoutAttr
     : LLVM_Attr<"DataLayout", "data_layout", [DataLayoutSpecInterface]> {
-  let summary = "TODO";
+  let summary = "LLVM data layout string, exposed through DLTI";
   let description = [{
-    TODO
+    An attribute to hold a LLVM data layout string.
+
+    The LLVM data layout string is parsed and mapped to the corresponding MLIR
+    data layout specification. The `#llvm.data_layout` attribute then serves as
+    a proxy, forwarding all DLTI queries to the underlying MLIR data layout
+    specification.
   }];
   let parameters = (ins "StringAttr":$data_layout_str,
                          OptionalParameter<"DataLayoutSpecInterface", "{}">:$data_layout_spec);
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
index 72facd13994f7..0d2603debbc28 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
@@ -536,9 +536,19 @@ def LLVM_DIRecursiveTypeAttrInterface
 def LLVM_TargetAttrInterface
   : AttrInterface<"TargetAttrInterface", [DLTIQueryInterface]> {
   let description = [{
-    TODO
-
-    NEEDS TO document which TI keys it responds to.
+    Interface for attributes that describe LLVM targets.
+
+    These attributes should be able to return the specified target
+    `triple`, `chip` and `features` and are expected to be able to
+    produce the corresponding `llvm::TargetMachine` and
+    `llvm::DataLayout`. These methods can fail in case the backend
+    is not available.
+
+    Implementing attributes should provide a
+    `DLTIQueryInterface::query()` implementation which responds to
+    keys `"triple"`, `"chip"` and `"features"` by returning an
+    appropriate `StringAttr`, `StringAttr` and
+    `LLVM_TargetFeaturesAttr`.
   }];
   let cppNamespace = "::mlir::LLVM";
   let methods = [

>From 91560a50ccddf3e5b93e2229fc0c7a960a2637a2 Mon Sep 17 00:00:00 2001
From: Rolf Morel <rolf.morel at intel.com>
Date: Wed, 9 Jul 2025 08:43:47 -0700
Subject: [PATCH 7/7] Add cmake comments and a test

---
 .../mlir/Dialect/LLVMIR/CMakeLists.txt        |  9 ++++--
 .../LLVMIR/data-layout-from-target.mlir       | 30 ++++++++++++++++++-
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index 311f1489e4e7e..f1385cdff62be 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -7,16 +7,21 @@ mlir_tablegen(LLVMOpsDialect.h.inc -gen-dialect-decls)
 mlir_tablegen(LLVMOpsDialect.cpp.inc -gen-dialect-defs)
 mlir_tablegen(LLVMOpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(LLVMOpsEnums.cpp.inc -gen-enum-defs)
+#For LLVMOpsAttrDefs.h.inc, see below.
 mlir_tablegen(LLVMOpsAttrDefs.cpp.inc -gen-attrdef-defs
               -attrdefs-dialect=llvm)
 add_public_tablegen_target(MLIRLLVMOpsIncGen)
 
-# FIXME: explain how this solves generated header recursive dependency
+# NB: Separate out LLVMOpsAttrDefs.h.inc generation as generating it
+#     through LLVMOps.td ends up defining LLVMTargetFeaturesAttr even
+#     though LLVMTargetFeaturesAttrDefs.* is responsible for that.
 set(LLVM_TARGET_DEFINITIONS LLVMAttrAndEnumDefs.td)
 mlir_tablegen(LLVMOpsAttrDefs.h.inc -gen-attrdef-decls -attrdefs-dialect=llvm)
 add_public_tablegen_target(MLIRLLVMAttrsIncGen)
 
-# FIXME: explain how this solves generated header recursive dependency
+# NB: LLVMTargetFeaturesAttr is split out into its own file
+#     to break a recursive dependency: LLVMInterfaces depends
+#     on it, and other LLVMAttrs depending on LLVMInterfaces.
 set(LLVM_TARGET_DEFINITIONS LLVMTargetFeaturesAttrDefs.td)
 mlir_tablegen(LLVMTargetFeaturesAttrDefs.h.inc -gen-attrdef-decls)
 mlir_tablegen(LLVMTargetFeaturesAttrDefs.cpp.inc -gen-attrdef-defs)
diff --git a/mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir b/mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir
index 5983a34fe1e0d..5eb698d6dd859 100644
--- a/mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir
+++ b/mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir
@@ -12,8 +12,36 @@ module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux">
 
 // -----
 
+// TODO: module attributes
+// TODO-SAME: dlti.dl_spec = #llvm.data_layout
+// TODO-SAME: llvm.target = #llvm.target<
+// TODO-SAME:   triple = "x86_64-unknown-linux"
+// TODO-NOT:   chip = "skylake"
+// TODO-SAME:   #llvm.target_features<["+mmx", "+sse"]>
+// TODO-NOT:   chip = "skylake"
+
+//module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+//                                               // Generated parser is dumb: expected 'chip'
+//                                               #llvm.target_features<["+mmx", "+sse"]>> } {
+//}
+
+// -----
+
 // CHECK: module attributes
-// CHECK-SAME: dlti.dl_spec = #llvm.data_layout<"[[DL_STR:.*]]">
+// CHECK-SAME: dlti.dl_spec = #llvm.data_layout
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-SAME:   chip = "skylake"
+// CHECK-NOT:    #llvm.target_features
+
+module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+                                               chip = "skylake"> } {
+}
+
+// -----
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #llvm.data_layout
 // CHECK-SAME: llvm.target = #llvm.target<
 // CHECK-SAME:   triple = "x86_64-unknown-linux"
 // CHECK-SAME:   chip = "skylake"


From flang-commits at lists.llvm.org  Wed Jul  9 08:59:39 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Wed, 09 Jul 2025 08:59:39 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang][OpenMP][Sema] Module support for
 REQUIRES directive (PR #77082)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/77082@github.com>
Message-ID: <686e91eb.170a0220.1b9909.bc50@mx.google.com>


kiranchandramohan wrote:

@skatrak Did this go in as another patch or is still waiting for review?

https://github.com/llvm/llvm-project/pull/77082

From flang-commits at lists.llvm.org  Wed Jul  9 09:21:36 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Wed, 09 Jul 2025 09:21:36 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686e9710.170a0220.3da77f.c51a@mx.google.com>


rolfmorel wrote:

Couple of pain points in need of suggestions/feedback:

1. As `LLVMTargetFeaturesAttr` exists we have the following:
   1. There's a circular dependency between `LLVMInterfaces` and `LLVMAttrs` in that the new `TargetAttrInterface` requires `LLVMTargetFeaturesAttr` (for the `features` of a target) and there are attrs in `LLVMAttrs` that require interfaces from `LLVMInterfaces`. I break the dependency by moving `LLVMTargetFeaturesAttr` to its own file and having a dedicated CMake-rule to generate `LLVMOpsAttrDefs.h.inc` by way of `LLVMAttrAndEnumDefs.td`.
        1. Is there a better way of dealing with this circular dependency? 
   3. Currently, `#nvvm.target` and `#rocdl.target` take `features` as a `StringAttr`. Seems to me they should take `LLVMTargetFeaturesAttr` which would make them eligible for/compatible with implementing `TargetAttrInterface`. I have not spent time on transitioning them yet.
4.  For some reason, the parser generated for ``let assemblyFormat = [{`<` `triple` `=` $triple (`,` `chip` `=` $chip^)? (`,` qualified($features)^)? `>`}];`` does not actually make the `chip` clause optional. That is, it greedily assumes that as it saw `,` from that clause and it is now committed to it. For example, `#llvm.target<triple = "x86_64-unknown-linux", #llvm.target_features<["+mmx", "+sse"]>>` yields an `expected 'chip'` parser error.
5. How do I properly hang a `unique_ptr<llvm::TargetMachine>` of off `TargetAttr`? The current bare pointer works but probably is not safe, e.g. in case the context gets deleted. See discussion with @rengolin above.
6. Had to move the `DataLayoutImporter` from `lib/Target/LLVMIR` to `Dialect/LLVMIR/IR`, in support of `#llvm.data_layout` being a wrapper around the corresponding `#dlti.dl_spec`, which causes a linking dependency on the DLTI-dialect.
7. What is the strategy around verification? See discussion with @rengolin above.
8. Test cases will need to be made aware of which backends the compiled `llvm-project` has available and be skipped if not applicable. 

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 09:25:13 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Wed, 09 Jul 2025 09:25:13 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686e97e9.050a0220.1a81d4.62cd@mx.google.com>


https://github.com/rolfmorel ready_for_review https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 09:25:44 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 09:25:44 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686e9808.170a0220.279776.4805@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-mlir-llvm

@llvm/pr-subscribers-mlir

Author: Rolf Morel (rolfmorel)

<details>
<summary>Changes</summary>



---

Patch is 38.47 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/145899.diff


22 Files Affected:

- (modified) flang/include/flang/Optimizer/Dialect/FIRType.h (+1) 
- (modified) mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt (+16-2) 
- (renamed) mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h (+22-6) 
- (added) mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td (+10) 
- (modified) mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td (+131-69) 
- (modified) mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h (+10-2) 
- (modified) mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td (+8) 
- (modified) mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td (+53) 
- (modified) mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td (+1) 
- (added) mlir/include/mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.td (+82) 
- (modified) mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.td (+8) 
- (modified) mlir/lib/Dialect/DLTI/DLTI.cpp (+1-1) 
- (modified) mlir/lib/Dialect/DLTI/Traits.cpp (+1-1) 
- (modified) mlir/lib/Dialect/LLVMIR/CMakeLists.txt (+11) 
- (renamed) mlir/lib/Dialect/LLVMIR/IR/DataLayoutImporter.cpp (+2-14) 
- (modified) mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp (+76-1) 
- (modified) mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt (+1) 
- (added) mlir/lib/Dialect/LLVMIR/Transforms/DataLayoutFromTarget.cpp (+64) 
- (modified) mlir/lib/Target/LLVMIR/CMakeLists.txt (-2) 
- (modified) mlir/lib/Target/LLVMIR/ModuleImport.cpp (+1-1) 
- (added) mlir/test/Dialect/LLVMIR/data-layout-from-target-invalid.mlir (+8) 
- (added) mlir/test/Dialect/LLVMIR/data-layout-from-target.mlir (+69) 


``````````diff
diff --git a/flang/include/flang/Optimizer/Dialect/FIRType.h b/flang/include/flang/Optimizer/Dialect/FIRType.h
index ecab12de55d61..83077aef8d08d 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRType.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRType.h
@@ -13,6 +13,7 @@
 #ifndef FORTRAN_OPTIMIZER_DIALECT_FIRTYPE_H
 #define FORTRAN_OPTIMIZER_DIALECT_FIRTYPE_H
 
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index cfad07e57021f..f1385cdff62be 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -7,12 +7,26 @@ mlir_tablegen(LLVMOpsDialect.h.inc -gen-dialect-decls)
 mlir_tablegen(LLVMOpsDialect.cpp.inc -gen-dialect-defs)
 mlir_tablegen(LLVMOpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(LLVMOpsEnums.cpp.inc -gen-enum-defs)
-mlir_tablegen(LLVMOpsAttrDefs.h.inc -gen-attrdef-decls
-              -attrdefs-dialect=llvm)
+#For LLVMOpsAttrDefs.h.inc, see below.
 mlir_tablegen(LLVMOpsAttrDefs.cpp.inc -gen-attrdef-defs
               -attrdefs-dialect=llvm)
 add_public_tablegen_target(MLIRLLVMOpsIncGen)
 
+# NB: Separate out LLVMOpsAttrDefs.h.inc generation as generating it
+#     through LLVMOps.td ends up defining LLVMTargetFeaturesAttr even
+#     though LLVMTargetFeaturesAttrDefs.* is responsible for that.
+set(LLVM_TARGET_DEFINITIONS LLVMAttrAndEnumDefs.td)
+mlir_tablegen(LLVMOpsAttrDefs.h.inc -gen-attrdef-decls -attrdefs-dialect=llvm)
+add_public_tablegen_target(MLIRLLVMAttrsIncGen)
+
+# NB: LLVMTargetFeaturesAttr is split out into its own file
+#     to break a recursive dependency: LLVMInterfaces depends
+#     on it, and other LLVMAttrs depending on LLVMInterfaces.
+set(LLVM_TARGET_DEFINITIONS LLVMTargetFeaturesAttrDefs.td)
+mlir_tablegen(LLVMTargetFeaturesAttrDefs.h.inc -gen-attrdef-decls)
+mlir_tablegen(LLVMTargetFeaturesAttrDefs.cpp.inc -gen-attrdef-defs)
+add_public_tablegen_target(MLIRLLVMTargetFeaturesAttrsIncGen)
+
 set(LLVM_TARGET_DEFINITIONS LLVMTypes.td)
 mlir_tablegen(LLVMTypes.h.inc -gen-typedef-decls -typedefs-dialect=llvm)
 mlir_tablegen(LLVMTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=llvm)
diff --git a/mlir/lib/Target/LLVMIR/DataLayoutImporter.h b/mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
similarity index 82%
rename from mlir/lib/Target/LLVMIR/DataLayoutImporter.h
rename to mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
index 88ceaf1a74e62..0f036f1c43492 100644
--- a/mlir/lib/Target/LLVMIR/DataLayoutImporter.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/DataLayoutImporter.h
@@ -11,13 +11,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
-#define MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
+#ifndef MLIR_LLVMIR_DATALAYOUTIMPORTER_H_
+#define MLIR_LLVMIR_DATALAYOUTIMPORTER_H_
 
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/IR/DataLayout.h"
 
 namespace llvm {
 class StringRef;
@@ -38,6 +39,8 @@ namespace detail {
 /// null if the bit width is not supported.
 FloatType getFloatType(MLIRContext *context, unsigned width);
 
+} // namespace detail
+
 /// Helper class that translates an LLVM data layout to an MLIR data layout
 /// specification. Only integer, float, pointer, alloca memory space, stack
 /// alignment, and endianness entries are translated. The class also returns all
@@ -49,7 +52,21 @@ class DataLayoutImporter {
   DataLayoutImporter(MLIRContext *context,
                      const llvm::DataLayout &llvmDataLayout)
       : context(context) {
-    translateDataLayout(llvmDataLayout);
+    // Transform the data layout to its string representation and append the
+    // default data layout string specified in the language reference
+    // (https://llvm.org/docs/LangRef.html#data-layout). The translation then
+    // parses the string and ignores the default value if a specific kind occurs
+    // in both strings. Additionally, the following default values exist:
+    // - non-default address space pointer specifications default to the default
+    //   address space pointer specification
+    // - the alloca address space defaults to the default address space.
+    layoutStr = llvmDataLayout.getStringRepresentation();
+    translateDataLayoutFromStr();
+  }
+
+  DataLayoutImporter(MLIRContext *context, StringRef dataLayoutStr)
+      : layoutStr(dataLayoutStr), context(context) {
+    translateDataLayoutFromStr();
   }
 
   /// Returns the MLIR data layout specification translated from the LLVM
@@ -66,7 +83,7 @@ class DataLayoutImporter {
 
 private:
   /// Translates the LLVM `dataLayout` to an MLIR data layout specification.
-  void translateDataLayout(const llvm::DataLayout &llvmDataLayout);
+  void translateDataLayoutFromStr();
 
   /// Tries to parse the letter only prefix that identifies the specification
   /// and removes the consumed characters from the beginning of the string.
@@ -125,8 +142,7 @@ class DataLayoutImporter {
   DataLayoutSpecInterface dataLayout;
 };
 
-} // namespace detail
 } // namespace LLVM
 } // namespace mlir
 
-#endif // MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
+#endif // MLIR_LLVMIR_DATALAYOUTIMPORTER_H_
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td
new file mode 100644
index 0000000000000..e34375076ffd1
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrAndEnumDefs.td
@@ -0,0 +1,10 @@
+//===-- LLVMAttrDefs.td - Solely LLVM Attribute and Enum definitions ----*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
+include "mlir/Dialect/LLVMIR/LLVMEnums.td"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 790d2e77ea874..e563441d0102b 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -13,13 +13,138 @@ include "mlir/Dialect/LLVMIR/LLVMDialect.td"
 include "mlir/Dialect/LLVMIR/LLVMInterfaces.td"
 include "mlir/IR/AttrTypeBase.td"
 include "mlir/IR/CommonAttrConstraints.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
 
-// All of the attributes will extend this class.
-class LLVM_Attr<string name, string attrMnemonic,
-                list<Trait> traits = [],
-                string baseCppClass = "::mlir::Attribute">
-    : AttrDef<LLVM_Dialect, name, traits, baseCppClass> {
-  let mnemonic = attrMnemonic;
+//===----------------------------------------------------------------------===//
+// LLVM_TargetAttr
+//===----------------------------------------------------------------------===//
+
+def LLVM_TargetAttr : LLVM_Attr<"Target", "target",
+                                [LLVM_TargetAttrInterface]> {
+  let summary = "LLVM target info: triple, chip, features";
+  let description = [{
+    An attribute to hold LLVM target information, specifying LLVM's target
+    `triple` string, the target `chip` string (i.e. the `cpu` string), and
+    target `features` string as an attribute. The latter two are optional.
+
+    Has facilities to obtain the corresponding `llvm::TargetMachine` and
+    `llvm::DataLayout`, given the relevant LLVM backend is loaded.
+
+    ---
+
+    Responds to DLTI-queries on the keys:
+       * A query for `"triple"` returns the `StringAttr` for the `triple`.
+       * A query for `"chip"` returns the `StringAttr` for the `chip`/`cpu`, if provided.
+       * A query for `"features"` returns the `TargetFeaturesAttr`, if provided.
+         * Individual features can be queried for on this attribute.
+  }];
+  let parameters = (ins "StringAttr":$triple,
+                        OptionalParameter<"StringAttr">:$chip,
+                        OptionalParameter<"TargetFeaturesAttr">:$features);
+
+  let assemblyFormat = [{`<` `triple` `=` $triple
+                         (`,` `chip` `=` $chip^)?
+                         (`,` qualified($features)^)? `>`}];
+
+  let extraClassDeclaration = [{
+    std::optional<llvm::TargetMachine *> targetMachine = std::nullopt;
+
+    FailureOr<llvm::TargetMachine *> getTargetMachine();
+
+    std::optional<llvm::DataLayout> dataLayout = std::nullopt;
+    FailureOr<llvm::DataLayout> getDataLayout();
+    FailureOr<Attribute> query(DataLayoutEntryKey key);
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// LLVM_DataLayoutAttr
+//===----------------------------------------------------------------------===//
+
+def LLVM_DataLayoutAttr
+    : LLVM_Attr<"DataLayout", "data_layout", [DataLayoutSpecInterface]> {
+  let summary = "LLVM data layout string, exposed through DLTI";
+  let description = [{
+    An attribute to hold a LLVM data layout string.
+
+    The LLVM data layout string is parsed and mapped to the corresponding MLIR
+    data layout specification. The `#llvm.data_layout` attribute then serves as
+    a proxy, forwarding all DLTI queries to the underlying MLIR data layout
+    specification.
+  }];
+  let parameters = (ins "StringAttr":$data_layout_str,
+                         OptionalParameter<"DataLayoutSpecInterface", "{}">:$data_layout_spec);
+  let builders = [
+    AttrBuilder<(ins "llvm::StringRef":$data_layout_str), [{
+      auto importer = LLVM::DataLayoutImporter($_ctxt, data_layout_str);
+      auto dataLayoutSpec = importer.getDataLayout();
+      return $_get($_ctxt, mlir::StringAttr::get($_ctxt, data_layout_str), dataLayoutSpec);
+    }]>
+  ];
+  let assemblyFormat = "`<` $data_layout_str `>`";
+  let extraClassDeclaration = [{
+    template <typename Ty>
+    DataLayoutEntryList getSpecForType() {
+      return getDataLayoutSpec().getSpecForType(TypeID::get<Ty>());
+    }
+
+    inline ::mlir::FailureOr<::mlir::Attribute>
+    queryHelper(::mlir::DataLayoutEntryKey key) const {
+      return getDataLayoutSpec().queryHelper(key);
+    }
+
+    void bucketEntriesByType(
+        ::llvm::MapVector<::mlir::TypeID, ::mlir::DataLayoutEntryList> &types,
+        ::llvm::MapVector<::mlir::StringAttr,
+                          ::mlir::DataLayoutEntryInterface> &ids) {
+      getDataLayoutSpec().bucketEntriesByType(types, ids);
+    };
+
+    ::mlir::DataLayoutSpecInterface
+    combineWith(ArrayRef<::mlir::DataLayoutSpecInterface> specs) const {
+      return getDataLayoutSpec().combineWith(specs);
+    }
+    DataLayoutEntryListRef getEntries() const { return getDataLayoutSpec().getEntries(); }
+    LogicalResult verifySpec(Location loc) {
+      return getDataLayoutSpec().verifySpec(loc);
+    }
+    StringAttr getEndiannessIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getEndiannessIdentifier(context);
+    }
+    StringAttr getDefaultMemorySpaceIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getDefaultMemorySpaceIdentifier(context);
+    }
+    StringAttr getAllocaMemorySpaceIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getAllocaMemorySpaceIdentifier(context);
+    }
+    StringAttr getManglingModeIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getManglingModeIdentifier(context);
+    }
+    StringAttr getProgramMemorySpaceIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getProgramMemorySpaceIdentifier(context);
+    }
+    StringAttr getGlobalMemorySpaceIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getGlobalMemorySpaceIdentifier(context);
+    }
+    StringAttr getStackAlignmentIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getStackAlignmentIdentifier(context);
+    }
+    StringAttr getFunctionPointerAlignmentIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getFunctionPointerAlignmentIdentifier(context);
+    }
+    StringAttr getLegalIntWidthsIdentifier(MLIRContext *context) const {
+      return getDataLayoutSpec().getLegalIntWidthsIdentifier(context);
+    }
+    ::mlir::DataLayoutEntryList getSpecForType(TypeID type) const {
+      return getDataLayoutSpec().getSpecForType(type);
+    }
+    ::mlir::DataLayoutEntryInterface getSpecForIdentifier(StringAttr identifier) const {
+      return getDataLayoutSpec().getSpecForIdentifier(identifier);
+    }
+    FailureOr<Attribute> query(DataLayoutEntryKey key) const {
+      return getDataLayoutSpec().query(key);
+    }
+  }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -1241,69 +1366,6 @@ def LLVM_VScaleRangeAttr : LLVM_Attr<"VScaleRange", "vscale_range"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// TargetFeaturesAttr
-//===----------------------------------------------------------------------===//
-
-def LLVM_TargetFeaturesAttr : LLVM_Attr<"TargetFeatures", "target_features">
-{
-  let summary = "LLVM target features attribute";
-
-  let description = [{
-    Represents the LLVM target features as a list that can be checked within
-    passes/rewrites.
-
-    Example:
-    ```mlir
-    #llvm.target_features<["+sme", "+sve", "+sme-f64f64"]>
-    ```
-
-    Then within a pass or rewrite the features active at an op can be queried:
-
-    ```c++
-    auto targetFeatures = LLVM::TargetFeaturesAttr::featuresAt(op);
-
-    if (!targetFeatures.contains("+sme-f64f64"))
-      return failure();
-    ```
-  }];
-
-  let parameters = (ins OptionalArrayRefParameter<"StringAttr">:$features);
-
-  let builders = [
-    TypeBuilder<(ins "::llvm::StringRef":$features)>,
-    TypeBuilder<(ins "::llvm::ArrayRef<::llvm::StringRef>":$features)>
-  ];
-
-  let extraClassDeclaration = [{
-    /// Checks if a feature is contained within the features list.
-    /// Note: Using a StringAttr allows doing pointer-comparisons.
-    bool contains(::mlir::StringAttr feature) const;
-    bool contains(::llvm::StringRef feature) const;
-
-    bool nullOrEmpty() const {
-      // Checks if this attribute is null, or the features are empty.
-      return !bool(*this) || getFeatures().empty();
-    }
-
-    /// Returns the list of features as an LLVM-compatible string.
-    std::string getFeaturesString() const;
-
-    /// Finds the target features on the parent FunctionOpInterface.
-    /// Note: This assumes the attribute name matches the return value of
-    /// `getAttributeName()`.
-    static TargetFeaturesAttr featuresAt(Operation* op);
-
-    /// Canonical name for this attribute within MLIR.
-    static constexpr StringLiteral getAttributeName() {
-      return StringLiteral("target_features");
-    }
-  }];
-
-  let assemblyFormat = "`<` `[` (`]`) : ($features^ `]`)? `>`";
-  let genVerifyDecl = 1;
-}
-
 //===----------------------------------------------------------------------===//
 // UndefAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
index 3ede857733242..14645d5dee95f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
@@ -14,8 +14,10 @@
 #ifndef MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 #define MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 
-#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Target/TargetMachine.h"
 #include <optional>
 
 #include "mlir/Dialect/LLVMIR/LLVMOpsEnums.h.inc"
@@ -89,11 +91,17 @@ class TBAANodeAttr : public Attribute {
 // TODO: this shouldn't be needed after we unify the attribute generation, i.e.
 // --gen-attr-* and --gen-attrdef-*.
 using cconv::CConv;
-using tailcallkind::TailCallKind;
 using linkage::Linkage;
+using tailcallkind::TailCallKind;
 } // namespace LLVM
 } // namespace mlir
 
+// First obtain TargetFeaturesAttr definitions as it is used both an LLVMIR
+// interface and that interface and this attribute are turn required by another
+// LLVMIR attribute.
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.h.inc"
+
 #include "mlir/Dialect/LLVMIR/LLVMAttrInterfaces.h.inc"
 
 #define GET_ATTRDEF_CLASSES
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
index b5ea8fc5da500..e924be32da10f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
@@ -10,6 +10,7 @@
 #define LLVMIR_DIALECT
 
 include "mlir/IR/DialectBase.td"
+include "mlir/IR/AttrTypeBase.td"
 
 def LLVM_Dialect : Dialect {
   let name = "llvm";
@@ -123,4 +124,11 @@ def LLVM_Dialect : Dialect {
   }];
 }
 
+class LLVM_Attr<string name, string attrMnemonic,
+                list<Trait> traits = [],
+                string baseCppClass = "::mlir::Attribute">
+    : AttrDef<LLVM_Dialect, name, traits, baseCppClass> {
+  let mnemonic = attrMnemonic;
+}
+
 #endif  // LLVMIR_DIALECT
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
index 138170f8c8762..0d2603debbc28 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
@@ -14,6 +14,7 @@
 #define LLVMIR_INTERFACES
 
 include "mlir/IR/OpBase.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
 
 def FastmathFlagsInterface : OpInterface<"FastmathFlagsInterface"> {
   let description = [{
@@ -532,4 +533,56 @@ def LLVM_DIRecursiveTypeAttrInterface
   ];
 }
 
+def LLVM_TargetAttrInterface
+  : AttrInterface<"TargetAttrInterface", [DLTIQueryInterface]> {
+  let description = [{
+    Interface for attributes that describe LLVM targets.
+
+    These attributes should be able to return the specified target
+    `triple`, `chip` and `features` and are expected to be able to
+    produce the corresponding `llvm::TargetMachine` and
+    `llvm::DataLayout`. These methods can fail in case the backend
+    is not available.
+
+    Implementing attributes should provide a
+    `DLTIQueryInterface::query()` implementation which responds to
+    keys `"triple"`, `"chip"` and `"features"` by returning an
+    appropriate `StringAttr`, `StringAttr` and
+    `LLVM_TargetFeaturesAttr`.
+  }];
+  let cppNamespace = "::mlir::LLVM";
+  let methods = [
+    InterfaceMethod<
+      /*description=*/"Returns the target triple identifier.",
+      /*retTy=*/"::llvm::StringRef",
+      /*methodName=*/"getTriple",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the target chip (i.e. \"cpu\") identifier.",
+      /*retTy=*/"::llvm::StringRef",
+      /*methodName=*/"getChip",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the target features as a string.",
+      /*retTy=*/"::mlir::LLVM::TargetFeaturesAttr",
+      /*methodName=*/"getFeatures",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the target machine.",
+      /*retTy=*/"FailureOr<::llvm::TargetMachine *>",
+      /*methodName=*/"getTargetMachine",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the data layout associated to the target machine.",
+      /*retTy=*/"FailureOr<::llvm::DataLayout>",
+      /*methodName=*/"getDataLayout",
+      /*args=*/(ins)
+    >
+  ];
+}
+
 #endif // LLVMIR_INTERFACES
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index f4c1640098320..36a6291b5e3a8 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -14,6 +14,7 @@
 #define LLVMIR_OPS
 
 include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
+include "mlir/Dialect/LLVMIR/LLVMTargetFeaturesAttrDefs.td"
 include "mlir/Dialect/LLVMIR/LLVMEnums.td"
 include...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 09:37:56 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Wed, 09 Jul 2025 09:37:56 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686e9ae4.050a0220.23e692.5695@mx.google.com>


https://github.com/rolfmorel edited https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 09:54:25 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Wed, 09 Jul 2025 09:54:25 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686e9ec1.170a0220.158062.47f5@mx.google.com>


https://github.com/rolfmorel edited https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 10:07:43 2025
From: flang-commits at lists.llvm.org (Fabian Mora via flang-commits)
Date: Wed, 09 Jul 2025 10:07:43 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686ea1df.630a0220.290746.7047@mx.google.com>


================
@@ -14,8 +14,10 @@
 #ifndef MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 #define MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 
-#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Target/TargetMachine.h"
----------------
fabianmcg wrote:

This is adding a lot of LLVM deps to the LLVM dialect that we don't want. The best way to remove this dep is add a promised interface and implement  `LLVM_TargetAttrInterface` for that attr as an external model in `mlir/lib/Target`.

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 10:07:43 2025
From: flang-commits at lists.llvm.org (Fabian Mora via flang-commits)
Date: Wed, 09 Jul 2025 10:07:43 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686ea1df.050a0220.1a4a94.5f03@mx.google.com>


https://github.com/fabianmcg commented:

Quick comment, see my comment on `LLVMAttrs.h`, that affects many things in this PR and should be fixed. I'll give it another look later in the week. 

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 10:07:44 2025
From: flang-commits at lists.llvm.org (Fabian Mora via flang-commits)
Date: Wed, 09 Jul 2025 10:07:44 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686ea1e0.170a0220.1b8ce.d3ba@mx.google.com>


https://github.com/fabianmcg edited https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 10:08:54 2025
From: flang-commits at lists.llvm.org (Fabian Mora via flang-commits)
Date: Wed, 09 Jul 2025 10:08:54 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686ea226.170a0220.3708d5.5e80@mx.google.com>


https://github.com/fabianmcg edited https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 10:10:57 2025
From: flang-commits at lists.llvm.org (Fabian Mora via flang-commits)
Date: Wed, 09 Jul 2025 10:10:57 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <686ea2a1.170a0220.958c7.4f96@mx.google.com>


https://github.com/fabianmcg edited https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Wed Jul  9 10:42:00 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 10:42:00 -0700 (PDT)
Subject: [flang-commits] [flang] a63846b - [flang] Fix array assignment
 regression introduced by #147371 (#147761)
Message-ID: <686ea9e8.170a0220.360f59.61bd@mx.google.com>


Author: Leandro Lupori
Date: 2025-07-09T14:41:56-03:00
New Revision: a63846b475bacfda49eb00016e0dc43c9ab1aa7d

URL: https://github.com/llvm/llvm-project/commit/a63846b475bacfda49eb00016e0dc43c9ab1aa7d
DIFF: https://github.com/llvm/llvm-project/commit/a63846b475bacfda49eb00016e0dc43c9ab1aa7d.diff

LOG: [flang] Fix array assignment regression introduced by #147371 (#147761)

In some cases fixed shape arrays can be fir.heap/fir.ptr, even
after hlfir::derefPointersAndAllocatables() is called.

Added: 
    

Modified: 
    flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
    flang/test/HLFIR/opt-scalar-assign.fir

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 434973f5887e8..abcbf145410ab 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -805,8 +805,8 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
-      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
-      auto seqTy = mlir::cast<fir::SequenceType>(refTy.getElementType());
+      auto seqTy =
+          mlir::cast<fir::SequenceType>(fir::unwrapRefType(lhs.getType()));
       llvm::ArrayRef<int64_t> fixedShape = seqTy.getShape();
       int64_t flatExtent = 1;
       for (int64_t extent : fixedShape)

diff  --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 468a5dbf988d7..256ccee6b9944 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -184,3 +184,34 @@ func.func private @_QFPtest7(%arg0: !fir.ref<tuple<!fir.box<!fir.array<2x2xi32>>
 // CHECK:             hlfir.assign %[[VAL_0]] to %[[VAL_2]] : i32, !fir.ref<i32>
 // CHECK:           }
 // CHECK:         }
+
+func.func @_QPtest8() {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.alloca !fir.array<120xi8> {uniq_name = "_QFtest8Ea"}
+  %c0 = arith.constant 0 : index
+  %2 = fir.coordinate_of %1, %c0 : (!fir.ref<!fir.array<120xi8>>, index) -> !fir.ref<i8>
+  %3 = fir.convert %2 : (!fir.ref<i8>) -> !fir.ptr<!fir.array<2x10xf32>>
+  %c2 = arith.constant 2 : index
+  %c10 = arith.constant 10 : index
+  %4 = fir.shape %c2, %c10 : (index, index) -> !fir.shape<2>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFtest8Ea"} : (!fir.ptr<!fir.array<2x10xf32>>, !fir.shape<2>) -> (!fir.ptr<!fir.array<2x10xf32>>, !fir.ptr<!fir.array<2x10xf32>>)
+  %c0_0 = arith.constant 0 : index
+  %6 = fir.coordinate_of %1, %c0_0 : (!fir.ref<!fir.array<120xi8>>, index) -> !fir.ref<i8>
+  %7 = fir.convert %6 : (!fir.ref<i8>) -> !fir.ptr<!fir.array<30xf32>>
+  %c30 = arith.constant 30 : index
+  %8 = fir.shape %c30 : (index) -> !fir.shape<1>
+  %9:2 = hlfir.declare %7(%8) {uniq_name = "_QFtest8Eb"} : (!fir.ptr<!fir.array<30xf32>>, !fir.shape<1>) -> (!fir.ptr<!fir.array<30xf32>>, !fir.ptr<!fir.array<30xf32>>)
+  %cst = arith.constant -1.000000e+00 : f32
+  hlfir.assign %cst to %5#0 : f32, !fir.ptr<!fir.array<2x10xf32>>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPtest8() {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest8Ea"} : (!fir.ptr<!fir.array<2x10xf32>>, !fir.shape<2>) -> (!fir.ptr<!fir.array<2x10xf32>>, !fir.ptr<!fir.array<2x10xf32>>)
+// CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ptr<!fir.array<2x10xf32>>) -> !fir.ref<!fir.array<20xf32>>
+// CHECK:           fir.do_loop %[[VAL_3:.*]] = %{{.*}} to %{{.*}} step %{{.*}} unordered {
+// CHECK:             %[[VAL_4:.*]] = hlfir.designate %[[VAL_2]] (%[[VAL_3]]) : (!fir.ref<!fir.array<20xf32>>, index) -> !fir.ref<f32>
+// CHECK:             hlfir.assign %[[VAL_0]] to %[[VAL_4]] : f32, !fir.ref<f32>
+// CHECK:           }
+// CHECK:         }


        

From flang-commits at lists.llvm.org  Wed Jul  9 10:47:46 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Wed, 09 Jul 2025 10:47:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686eab42.170a0220.10eab5.dad9@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `premerge-monolithic-linux` running on `premerge-linux-1` while building `flang` at step 6 "build-unified-tree".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/153/builds/37499

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (build-unified-tree) failure: build (failure)
...
0.023 [3/8/4] Building CXX object compiler-rt/lib/nsan/CMakeFiles/RTNsan_dynamic_version_script_dummy.x86_64.dir/dummy.cpp.o
0.023 [2/8/5] Building CXX object compiler-rt/lib/memprof/CMakeFiles/RTMemprof_dynamic_version_script_dummy.x86_64.dir/dummy.cpp.o
0.066 [1/8/6] Linking CXX shared library /build/buildbot/premerge-monolithic-linux/build/lib/clang/21/lib/x86_64-unknown-linux-gnu/libclang_rt.ubsan_standalone.so
0.067 [1/7/7] Linking CXX shared library /build/buildbot/premerge-monolithic-linux/build/lib/clang/21/lib/x86_64-unknown-linux-gnu/libclang_rt.nsan.so
0.068 [1/6/8] Linking CXX shared library /build/buildbot/premerge-monolithic-linux/build/lib/clang/21/lib/x86_64-unknown-linux-gnu/libclang_rt.hwasan_aliases.so
0.070 [1/5/9] Linking CXX shared library /build/buildbot/premerge-monolithic-linux/build/lib/clang/21/lib/x86_64-unknown-linux-gnu/libclang_rt.hwasan.so
0.074 [1/4/10] Linking CXX shared library /build/buildbot/premerge-monolithic-linux/build/lib/clang/21/lib/x86_64-unknown-linux-gnu/libclang_rt.memprof.so
0.081 [1/3/11] Linking CXX shared library /build/buildbot/premerge-monolithic-linux/build/lib/clang/21/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so
0.570 [1/2/12] Generating /build/buildbot/premerge-monolithic-linux/build/compile_commands.json
1.025 [1/1/13] Building CXX object flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o
FAILED: flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o 
/build/buildbot/premerge-monolithic-linux/build/./bin/clang++ --target=x86_64-unknown-linux-gnu -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/build/buildbot/premerge-monolithic-linux/llvm-project/flang-rt/include -I/build/buildbot/premerge-monolithic-linux/llvm-project/flang-rt/../flang/include -I/build/buildbot/premerge-monolithic-linux/build/runtimes/runtimes-bins/flang-rt -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -DNDEBUG -UNDEBUG -fno-lto -fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables -U_GLIBCXX_ASSERTIONS -U_LIBCPP_ENABLE_ASSERTIONS -std=gnu++17 -MD -MT flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -MF flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o.d -o flang-rt/lib/runtime/CMakeFiles/flang_rt.runtime.static.dir/format.cpp.o -c /build/buildbot/premerge-monolithic-linux/llvm-project/flang-rt/lib/runtime/format.cpp
In file included from /build/buildbot/premerge-monolithic-linux/llvm-project/flang-rt/lib/runtime/format.cpp:9:
In file included from /build/buildbot/premerge-monolithic-linux/llvm-project/flang-rt/include/flang-rt/runtime/format-implementation.h:18:
/build/buildbot/premerge-monolithic-linux/llvm-project/flang/include/flang/Common/format.h:14:10: fatal error: 'llvm/Support/MathExtras.h' file not found
   14 | #include "llvm/Support/MathExtras.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
ninja: build stopped: subcommand failed.
FAILED: runtimes/runtimes-stamps/runtimes-build /build/buildbot/premerge-monolithic-linux/build/runtimes/runtimes-stamps/runtimes-build 
cd /build/buildbot/premerge-monolithic-linux/build/runtimes/runtimes-bins && /etc/cmake/bin/cmake --build .
ninja: build stopped: subcommand failed.

```

</details>

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Wed Jul  9 10:51:43 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 09 Jul 2025 10:51:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Create temporaries for array
 sections passed to IGNORE_TKR dummy args (PR #147419)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147419@github.com>
Message-ID: <686eac2f.a70a0220.248669.80c1@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/147419

>From 2477076ba81ec61e8bbff36fb624e8bb76c27380 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 4 Jul 2025 16:26:17 -0400
Subject: [PATCH 1/7] [flang] Contiguity check on IGNORE_TKR argument

If argument is marked as IGNORE_TKR, enable contiguity check on it,
unless IGNORE_TKR explicitly specifies to ignore contiguity.
---
 flang/lib/Lower/CallInterface.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index 72431a9cfacc4..abe43dc9cb2b7 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -1426,6 +1426,13 @@ bool Fortran::lower::CallInterface<T>::PassedEntity::mustBeMadeContiguous()
           &characteristics->u);
   if (!dummy)
     return false;
+  if (dummy->ignoreTKR.test(common::IgnoreTKR::Contiguous))
+    return false;
+
+  // TODO: should this check ignore "device" or "managed"?
+  if (dummy->ignoreTKR.any())
+    return true;
+
   const auto &shapeAttrs = dummy->type.attrs();
   using ShapeAttrs = Fortran::evaluate::characteristics::TypeAndShape::Attr;
   if (shapeAttrs.test(ShapeAttrs::AssumedRank) ||

>From 50dd216ff0487709d52944006e35e6102015cef0 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Mon, 7 Jul 2025 10:20:38 -0400
Subject: [PATCH 2/7] [flang] Add -pedantic check for passing non-VOLATILE
 array section to VOLATILE dummy arg

Add the check, which partially addresses https://github.com/llvm/llvm-project/issues/137369

Implement HasTriplet().
---
 flang/include/flang/Evaluate/tools.h |  3 +++
 flang/lib/Evaluate/tools.cpp         | 19 +++++++++++++++++++
 flang/lib/Semantics/definable.cpp    |  3 +++
 3 files changed, 25 insertions(+)

diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index cad1b634f8924..0604049fd6a6d 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1123,6 +1123,9 @@ extern template semantics::UnorderedSymbolSet CollectCudaSymbols(
 // Predicate: does a variable contain a vector-valued subscript (not a triplet)?
 bool HasVectorSubscript(const Expr<SomeType> &);
 
+// Predicate: does a variable contain a triplet?
+bool HasTriplet(const Expr<SomeType> &);
+
 // Predicate: does an expression contain constant?
 bool HasConstant(const Expr<SomeType> &);
 
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index fcacdb93d662b..50a201dfcaced 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1173,6 +1173,25 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
   return HasVectorSubscriptHelper{}(expr);
 }
 
+// HasTriplet()
+struct HasTripletHelper
+    : public AnyTraverse<HasTripletHelper, bool,
+          /*TraverseAssocEntityDetails=*/false> {
+  using Base = AnyTraverse<HasTripletHelper, bool, false>;
+  HasTripletHelper() : Base{*this} {}
+  using Base::operator();
+  bool operator()(const Subscript &ss) const {
+    return std::holds_alternative<Triplet>(ss.u);
+  }
+  bool operator()(const ProcedureRef &) const {
+    return false; // don't descend into function call arguments
+  }
+};
+
+bool HasTriplet(const Expr<SomeType> &expr) {
+  return HasTripletHelper{}(expr);
+}
+
 // HasConstant()
 struct HasConstantHelper : public AnyTraverse<HasConstantHelper, bool,
                                /*TraverseAssocEntityDetails=*/false> {
diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp
index 08cb268b318ae..b2d2330887a53 100644
--- a/flang/lib/Semantics/definable.cpp
+++ b/flang/lib/Semantics/definable.cpp
@@ -371,6 +371,9 @@ std::optional<parser::Message> WhyNotDefinable(parser::CharBlock at,
         return parser::Message{at,
             "Variable '%s' has a vector subscript"_err_en_US, expr.AsFortran()};
       }
+    } else if (evaluate::HasTriplet(expr)) {
+      return parser::Message{at,
+          "Variable '%s' has array section"_err_en_US, expr.AsFortran()};
     }
     if (FindPureProcedureContaining(scope) &&
         evaluate::ExtractCoarrayRef(expr)) {

>From 28cb65fbc9ee572dfffac698f0853e3bf5932871 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Mon, 7 Jul 2025 13:29:46 -0400
Subject: [PATCH 3/7] Undid change to definable.cpp

---
 flang/lib/Semantics/definable.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp
index b2d2330887a53..08cb268b318ae 100644
--- a/flang/lib/Semantics/definable.cpp
+++ b/flang/lib/Semantics/definable.cpp
@@ -371,9 +371,6 @@ std::optional<parser::Message> WhyNotDefinable(parser::CharBlock at,
         return parser::Message{at,
             "Variable '%s' has a vector subscript"_err_en_US, expr.AsFortran()};
       }
-    } else if (evaluate::HasTriplet(expr)) {
-      return parser::Message{at,
-          "Variable '%s' has array section"_err_en_US, expr.AsFortran()};
     }
     if (FindPureProcedureContaining(scope) &&
         evaluate::ExtractCoarrayRef(expr)) {

>From a176d2d1a6aab8c3cc3c4ceba907462970d86565 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Mon, 7 Jul 2025 16:19:47 -0400
Subject: [PATCH 4/7] The current implementation seems to work, but I will
 likely move some code around

---
 flang/include/flang/Lower/CallInterface.h | 2 +-
 flang/lib/Lower/CallInterface.cpp         | 5 +++--
 flang/lib/Lower/ConvertCall.cpp           | 9 ++++++++-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/flang/include/flang/Lower/CallInterface.h b/flang/include/flang/Lower/CallInterface.h
index 72bc9dd890a94..f2c8db47c60b1 100644
--- a/flang/include/flang/Lower/CallInterface.h
+++ b/flang/include/flang/Lower/CallInterface.h
@@ -170,7 +170,7 @@ class CallInterface {
     /// Is the argument INTENT(OUT)
     bool isIntentOut() const;
     /// Does the argument have the CONTIGUOUS attribute or have explicit shape?
-    bool mustBeMadeContiguous() const;
+    bool mustBeMadeContiguous(const bool argHasTriplet = false) const;
     /// Does the dummy argument have the VALUE attribute?
     bool hasValueAttribute() const;
     /// Does the dummy argument have the ALLOCATABLE attribute?
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index abe43dc9cb2b7..2ffdd985e4f72 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -1416,8 +1416,9 @@ bool Fortran::lower::CallInterface<T>::PassedEntity::isIntentOut() const {
     return true;
   return characteristics->GetIntent() == Fortran::common::Intent::Out;
 }
+
 template <typename T>
-bool Fortran::lower::CallInterface<T>::PassedEntity::mustBeMadeContiguous()
+bool Fortran::lower::CallInterface<T>::PassedEntity::mustBeMadeContiguous(const bool argHasTriplet)
     const {
   if (!characteristics)
     return true;
@@ -1430,7 +1431,7 @@ bool Fortran::lower::CallInterface<T>::PassedEntity::mustBeMadeContiguous()
     return false;
 
   // TODO: should this check ignore "device" or "managed"?
-  if (dummy->ignoreTKR.any())
+  if (dummy->ignoreTKR.any() && argHasTriplet)
     return true;
 
   const auto &shapeAttrs = dummy->type.attrs();
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 6ed15df0de754..91e54b15fb869 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -1255,11 +1255,18 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
       passingPolymorphicToNonPolymorphic &&
       (actual.isArray() || mlir::isa<fir::BaseBoxType>(dummyType));
 
+  // Helper function to make it easier to unwrap and use expression
+  auto argHasTriplet = [](const Fortran::evaluate::ActualArgument &arg) -> bool {
+    if (const auto *expr = arg.UnwrapExpr())
+      return HasTriplet(*expr);
+    return false;
+  };
+
   // The simple contiguity of the actual is "lost" when passing a polymorphic
   // to a non polymorphic entity because the dummy dynamic type matters for
   // the contiguity.
   const bool mustDoCopyInOut =
-      actual.isArray() && arg.mustBeMadeContiguous() &&
+      actual.isArray() && arg.mustBeMadeContiguous(argHasTriplet(*arg.entity)) &&
       (passingPolymorphicToNonPolymorphic ||
        !isSimplyContiguous(*arg.entity, foldingContext));
 

>From 96daba4399d66cd16667592cae4e0aff5e332f63 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Mon, 7 Jul 2025 18:55:10 -0400
Subject: [PATCH 5/7] clang-format

---
 flang/lib/Evaluate/tools.cpp      | 9 +++------
 flang/lib/Lower/CallInterface.cpp | 4 ++--
 flang/lib/Lower/ConvertCall.cpp   | 6 ++++--
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 50a201dfcaced..1c43e475905d2 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1174,9 +1174,8 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
 }
 
 // HasTriplet()
-struct HasTripletHelper
-    : public AnyTraverse<HasTripletHelper, bool,
-          /*TraverseAssocEntityDetails=*/false> {
+struct HasTripletHelper : public AnyTraverse<HasTripletHelper, bool,
+                              /*TraverseAssocEntityDetails=*/false> {
   using Base = AnyTraverse<HasTripletHelper, bool, false>;
   HasTripletHelper() : Base{*this} {}
   using Base::operator();
@@ -1188,9 +1187,7 @@ struct HasTripletHelper
   }
 };
 
-bool HasTriplet(const Expr<SomeType> &expr) {
-  return HasTripletHelper{}(expr);
-}
+bool HasTriplet(const Expr<SomeType> &expr) { return HasTripletHelper{}(expr); }
 
 // HasConstant()
 struct HasConstantHelper : public AnyTraverse<HasConstantHelper, bool,
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index 2ffdd985e4f72..26bb1acfeea1b 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -1418,8 +1418,8 @@ bool Fortran::lower::CallInterface<T>::PassedEntity::isIntentOut() const {
 }
 
 template <typename T>
-bool Fortran::lower::CallInterface<T>::PassedEntity::mustBeMadeContiguous(const bool argHasTriplet)
-    const {
+bool Fortran::lower::CallInterface<T>::PassedEntity::mustBeMadeContiguous(
+    const bool argHasTriplet) const {
   if (!characteristics)
     return true;
   const auto *dummy =
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 91e54b15fb869..0da869adc5efa 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -1256,7 +1256,8 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
       (actual.isArray() || mlir::isa<fir::BaseBoxType>(dummyType));
 
   // Helper function to make it easier to unwrap and use expression
-  auto argHasTriplet = [](const Fortran::evaluate::ActualArgument &arg) -> bool {
+  auto argHasTriplet =
+      [](const Fortran::evaluate::ActualArgument &arg) -> bool {
     if (const auto *expr = arg.UnwrapExpr())
       return HasTriplet(*expr);
     return false;
@@ -1266,7 +1267,8 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
   // to a non polymorphic entity because the dummy dynamic type matters for
   // the contiguity.
   const bool mustDoCopyInOut =
-      actual.isArray() && arg.mustBeMadeContiguous(argHasTriplet(*arg.entity)) &&
+      actual.isArray() &&
+      arg.mustBeMadeContiguous(argHasTriplet(*arg.entity)) &&
       (passingPolymorphicToNonPolymorphic ||
        !isSimplyContiguous(*arg.entity, foldingContext));
 

>From aace0c2a95b6d8af56deb8f20f0aae92d64f3c36 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Mon, 7 Jul 2025 19:19:01 -0400
Subject: [PATCH 6/7] Some reformatting and comments

---
 flang/lib/Lower/CallInterface.cpp | 2 ++
 flang/lib/Lower/ConvertCall.cpp   | 8 ++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index 26bb1acfeea1b..80f8eb969797a 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -1417,6 +1417,8 @@ bool Fortran::lower::CallInterface<T>::PassedEntity::isIntentOut() const {
   return characteristics->GetIntent() == Fortran::common::Intent::Out;
 }
 
+/// Returning "true" from this function is a prerequisite for running
+/// contiguity check on the actual argument.
 template <typename T>
 bool Fortran::lower::CallInterface<T>::PassedEntity::mustBeMadeContiguous(
     const bool argHasTriplet) const {
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 0da869adc5efa..34f35e4fd4fb4 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -1256,19 +1256,19 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
       (actual.isArray() || mlir::isa<fir::BaseBoxType>(dummyType));
 
   // Helper function to make it easier to unwrap and use expression
-  auto argHasTriplet =
-      [](const Fortran::evaluate::ActualArgument &arg) -> bool {
+  auto argHasTriplet = [](const Fortran::evaluate::ActualArgument &arg) -> bool {
     if (const auto *expr = arg.UnwrapExpr())
       return HasTriplet(*expr);
     return false;
   };
 
+  const bool actualHasTriplet = argHasTriplet(*arg.entity);
+
   // The simple contiguity of the actual is "lost" when passing a polymorphic
   // to a non polymorphic entity because the dummy dynamic type matters for
   // the contiguity.
   const bool mustDoCopyInOut =
-      actual.isArray() &&
-      arg.mustBeMadeContiguous(argHasTriplet(*arg.entity)) &&
+      actual.isArray() && arg.mustBeMadeContiguous(actualHasTriplet) &&
       (passingPolymorphicToNonPolymorphic ||
        !isSimplyContiguous(*arg.entity, foldingContext));
 

>From 5fdfb455a4618a2bfb3b219dd51acab3da791d13 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Mon, 7 Jul 2025 19:20:01 -0400
Subject: [PATCH 7/7] clang-format

---
 flang/lib/Lower/ConvertCall.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 34f35e4fd4fb4..4d7d142c6b49d 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -1256,7 +1256,8 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
       (actual.isArray() || mlir::isa<fir::BaseBoxType>(dummyType));
 
   // Helper function to make it easier to unwrap and use expression
-  auto argHasTriplet = [](const Fortran::evaluate::ActualArgument &arg) -> bool {
+  auto argHasTriplet =
+      [](const Fortran::evaluate::ActualArgument &arg) -> bool {
     if (const auto *expr = arg.UnwrapExpr())
       return HasTriplet(*expr);
     return false;


From flang-commits at lists.llvm.org  Wed Jul  9 09:07:29 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 09 Jul 2025 09:07:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix array assignment regression
 introduced by #147371 (PR #147761)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147761@github.com>
Message-ID: <686e93c1.170a0220.20a151.b5ae@mx.google.com>


eugeneepshteyn wrote:

With this change I can compile both `ISO_Fortran_binding_1.f90` from the last regression and https://github.com/fujitsu/compiler-test-suite/blob/main/Fortran/1032/1032_0022.f90 from the latest regression.

Ship it! :)

https://github.com/llvm/llvm-project/pull/147761

From flang-commits at lists.llvm.org  Wed Jul  9 10:42:03 2025
From: flang-commits at lists.llvm.org (Leandro Lupori via flang-commits)
Date: Wed, 09 Jul 2025 10:42:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix array assignment regression
 introduced by #147371 (PR #147761)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147761@github.com>
Message-ID: <686ea9eb.170a0220.33cbb3.db08@mx.google.com>


https://github.com/luporl closed https://github.com/llvm/llvm-project/pull/147761

From flang-commits at lists.llvm.org  Wed Jul  9 11:46:40 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Wed, 09 Jul 2025 11:46:40 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][acc][flang] Lower nested ACC loops
 with tile clause as collapsed loops (PR #147801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147801@github.com>
Message-ID: <686eb910.170a0220.26e143.6687@mx.google.com>


================
@@ -4942,13 +4942,23 @@ void Fortran::lower::genEarlyReturnInOpenACCLoop(fir::FirOpBuilder &builder,
 
 int64_t Fortran::lower::getCollapseValue(
----------------
razvanlupusoru wrote:

Should we rename this to something like getLoopCountForCollapseAndTile?

https://github.com/llvm/llvm-project/pull/147801

From flang-commits at lists.llvm.org  Wed Jul  9 11:48:04 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Wed, 09 Jul 2025 11:48:04 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][acc][flang] Lower nested ACC loops
 with tile clause as collapsed loops (PR #147801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147801@github.com>
Message-ID: <686eb964.170a0220.c829a.b9d7@mx.google.com>


https://github.com/razvanlupusoru approved this pull request.

Looks great. Thank you!

https://github.com/llvm/llvm-project/pull/147801

From flang-commits at lists.llvm.org  Wed Jul  9 11:56:00 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Wed, 09 Jul 2025 11:56:00 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][acc][flang] Lower nested ACC loops
 with tile clause as collapsed loops (PR #147801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147801@github.com>
Message-ID: <686ebb40.050a0220.15a310.352a@mx.google.com>


razvanlupusoru wrote:

> Can you add a test with collapse and tile? If that's even allowed by the spec.

Vijay, would you mind looking into this? From a quick glance, I am myself not quite sure of the behavior when both clauses are present - I would lean towards checking for presence of both during semantics and generating an error. Also updating acc.loop verifier to not permit it. Unless you find a reasonable interpretation that would make both feasible.

https://github.com/llvm/llvm-project/pull/147801

From flang-commits at lists.llvm.org  Wed Jul  9 13:46:49 2025
From: flang-commits at lists.llvm.org (Peter Collingbourne via flang-commits)
Date: Wed, 09 Jul 2025 13:46:49 -0700 (PDT)
Subject: [flang-commits] [clang] [clang-tools-extra] [compiler-rt] [flang]
 [libc] [llvm] [mlir] AArch64: Relax x16/x17 constraint on AUT in certain
 cases. (PR #132857)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/132857@github.com>
Message-ID: <686ed539.170a0220.2c1076.01a2@mx.google.com>


https://github.com/pcc edited https://github.com/llvm/llvm-project/pull/132857

From flang-commits at lists.llvm.org  Wed Jul  9 13:46:52 2025
From: flang-commits at lists.llvm.org (Peter Collingbourne via flang-commits)
Date: Wed, 09 Jul 2025 13:46:52 -0700 (PDT)
Subject: [flang-commits] [clang] [clang-tools-extra] [compiler-rt] [flang]
 [libc] [llvm] [mlir] AArch64: Relax x16/x17 constraint on AUT in certain
 cases. (PR #132857)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/132857@github.com>
Message-ID: <686ed53c.170a0220.9b6ef.0216@mx.google.com>


https://github.com/pcc closed https://github.com/llvm/llvm-project/pull/132857

From flang-commits at lists.llvm.org  Wed Jul  9 13:46:59 2025
From: flang-commits at lists.llvm.org (Peter Collingbourne via flang-commits)
Date: Wed, 09 Jul 2025 13:46:59 -0700 (PDT)
Subject: [flang-commits] [clang] [clang-tools-extra] [compiler-rt] [flang]
 [libc] [llvm] [mlir] AArch64: Relax x16/x17 constraint on AUT in certain
 cases. (PR #132857)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/132857@github.com>
Message-ID: <686ed543.170a0220.16c5dd.015e@mx.google.com>


pcc wrote:

> LGTM, thanks!
> 
> A minor nit-picking is that the description of `emitPtrauthDiscriminator` in AsmPrinter is a bit out of sync with the implementation, as it does not mention that the requirement on certain registers being x16/x17 can be relaxed.

Fixed

https://github.com/llvm/llvm-project/pull/132857

From flang-commits at lists.llvm.org  Wed Jul  9 13:47:15 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 13:47:15 -0700 (PDT)
Subject: [flang-commits] [flang] c4138a2 - [mlir][acc][flang] Lower nested
 ACC loops with tile clause as collapsed loops (#147801)
Message-ID: <686ed553.630a0220.37dc76.0240@mx.google.com>


Author: Vijay Kandiah
Date: 2025-07-09T15:47:11-05:00
New Revision: c4138a24dc254783697f512f053e794fb1c68f88

URL: https://github.com/llvm/llvm-project/commit/c4138a24dc254783697f512f053e794fb1c68f88
DIFF: https://github.com/llvm/llvm-project/commit/c4138a24dc254783697f512f053e794fb1c68f88.diff

LOG: [mlir][acc][flang] Lower nested ACC loops with tile clause as collapsed loops (#147801)

In the case of nested loops, `acc.loop` is meant to subsume all of the
loops that it applies to (when explicitly described as doing so in the
OpenACC specification). So when there is a `acc loop tile(...)` present
on nested Fortran DO loops, `acc.loop` should apply to the `n` loops
that `tile` applies to. This change lowers such nested Fortran loops
with tile clause into a collapsed `acc.loop` with `n` IVs, loop bounds,
and step, in a similar fashion to the current lowering for acc loops
with `collapse` clause.

Added: 
    

Modified: 
    flang/include/flang/Lower/OpenACC.h
    flang/lib/Lower/Bridge.cpp
    flang/lib/Lower/OpenACC.cpp
    flang/test/Lower/OpenACC/acc-kernels-loop.f90
    flang/test/Lower/OpenACC/acc-loop.f90
    flang/test/Lower/OpenACC/acc-parallel-loop.f90
    flang/test/Lower/OpenACC/acc-serial-loop.f90

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Lower/OpenACC.h b/flang/include/flang/Lower/OpenACC.h
index dad841863ac00..af3451023e3df 100644
--- a/flang/include/flang/Lower/OpenACC.h
+++ b/flang/include/flang/Lower/OpenACC.h
@@ -114,7 +114,7 @@ void attachDeclarePostDeallocAction(AbstractConverter &, fir::FirOpBuilder &,
 void genOpenACCTerminator(fir::FirOpBuilder &, mlir::Operation *,
                           mlir::Location);
 
-int64_t getCollapseValue(const Fortran::parser::AccClauseList &);
+int64_t getLoopCountForCollapseAndTile(const Fortran::parser::AccClauseList &);
 
 bool isInOpenACCLoop(fir::FirOpBuilder &);
 

diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ed062ecf07166..2c8b3fb95e255 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -3083,25 +3083,25 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     Fortran::lower::pft::Evaluation *curEval = &getEval();
 
     if (accLoop || accCombined) {
-      int64_t collapseValue;
+      int64_t loopCount;
       if (accLoop) {
         const Fortran::parser::AccBeginLoopDirective &beginLoopDir =
             std::get<Fortran::parser::AccBeginLoopDirective>(accLoop->t);
         const Fortran::parser::AccClauseList &clauseList =
             std::get<Fortran::parser::AccClauseList>(beginLoopDir.t);
-        collapseValue = Fortran::lower::getCollapseValue(clauseList);
+        loopCount = Fortran::lower::getLoopCountForCollapseAndTile(clauseList);
       } else if (accCombined) {
         const Fortran::parser::AccBeginCombinedDirective &beginCombinedDir =
             std::get<Fortran::parser::AccBeginCombinedDirective>(
                 accCombined->t);
         const Fortran::parser::AccClauseList &clauseList =
             std::get<Fortran::parser::AccClauseList>(beginCombinedDir.t);
-        collapseValue = Fortran::lower::getCollapseValue(clauseList);
+        loopCount = Fortran::lower::getLoopCountForCollapseAndTile(clauseList);
       }
 
       if (curEval->lowerAsStructured()) {
         curEval = &curEval->getFirstNestedEvaluation();
-        for (int64_t i = 1; i < collapseValue; i++)
+        for (int64_t i = 1; i < loopCount; i++)
           curEval = &*std::next(curEval->getNestedEvaluations().begin());
       }
     }
@@ -6155,8 +6155,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
       Fortran::lower::defineModuleVariable(*this, var);
     }
-      for (auto &eval : mod.evaluationList)
-        genFIR(eval);
+    for (auto &eval : mod.evaluationList)
+      genFIR(eval);
   }
 
   /// Lower functions contained in a module.

diff  --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index e56d7f7ed9b6f..42842bcb41a74 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -2442,8 +2442,9 @@ static mlir::acc::LoopOp createLoopOp(
       inclusiveBounds.push_back(true);
     }
   } else {
-    int64_t collapseValue = Fortran::lower::getCollapseValue(accClauseList);
-    for (unsigned i = 0; i < collapseValue; ++i) {
+    int64_t loopCount =
+        Fortran::lower::getLoopCountForCollapseAndTile(accClauseList);
+    for (unsigned i = 0; i < loopCount; ++i) {
       const Fortran::parser::LoopControl *loopControl;
       if (i == 0) {
         loopControl = &*outerDoConstruct.GetLoopControl();
@@ -2478,7 +2479,7 @@ static mlir::acc::LoopOp createLoopOp(
 
       inclusiveBounds.push_back(true);
 
-      if (i < collapseValue - 1)
+      if (i < loopCount - 1)
         crtEval = &*std::next(crtEval->getNestedEvaluations().begin());
     }
   }
@@ -4940,15 +4941,25 @@ void Fortran::lower::genEarlyReturnInOpenACCLoop(fir::FirOpBuilder &builder,
   builder.create<mlir::acc::YieldOp>(loc, yieldValue);
 }
 
-int64_t Fortran::lower::getCollapseValue(
+int64_t Fortran::lower::getLoopCountForCollapseAndTile(
     const Fortran::parser::AccClauseList &clauseList) {
+  int64_t collapseLoopCount = 1;
+  int64_t tileLoopCount = 1;
   for (const Fortran::parser::AccClause &clause : clauseList.v) {
     if (const auto *collapseClause =
             std::get_if<Fortran::parser::AccClause::Collapse>(&clause.u)) {
       const parser::AccCollapseArg &arg = collapseClause->v;
       const auto &collapseValue{std::get<parser::ScalarIntConstantExpr>(arg.t)};
-      return *Fortran::semantics::GetIntValue(collapseValue);
+      collapseLoopCount = *Fortran::semantics::GetIntValue(collapseValue);
+    }
+    if (const auto *tileClause =
+            std::get_if<Fortran::parser::AccClause::Tile>(&clause.u)) {
+      const parser::AccTileExprList &tileExprList = tileClause->v;
+      const std::list<parser::AccTileExpr> &listTileExpr = tileExprList.v;
+      tileLoopCount = listTileExpr.size();
     }
   }
-  return 1;
+  if (tileLoopCount > collapseLoopCount)
+    return tileLoopCount;
+  return collapseLoopCount;
 }

diff  --git a/flang/test/Lower/OpenACC/acc-kernels-loop.f90 b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
index 4e968144399a8..8d95f35b186ee 100644
--- a/flang/test/Lower/OpenACC/acc-kernels-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
@@ -663,7 +663,7 @@ subroutine acc_kernels_loop
 ! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
 ! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
-! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) control(%arg0 : i32, %arg1 : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -689,7 +689,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      acc.kernels {{.*}} {
-! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) control(%arg0 : i32, %arg1 : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator

diff  --git a/flang/test/Lower/OpenACC/acc-loop.f90 b/flang/test/Lower/OpenACC/acc-loop.f90
index 5baa485534b2a..c6df28ec5e000 100644
--- a/flang/test/Lower/OpenACC/acc-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-loop.f90
@@ -9,7 +9,7 @@
 
 program acc_loop
 
-  integer :: i, j
+  integer :: i, j, k
   integer, parameter :: n = 10
   real, dimension(n) :: a, b
   real, dimension(n, n) :: c, d
@@ -209,9 +209,9 @@ program acc_loop
 
 ! CHECK:      [[TILESIZE1:%.*]] = arith.constant 2 : i32
 ! CHECK:      [[TILESIZE2:%.*]] = arith.constant 2 : i32
-! CHECK:      acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) control(%arg0 : i32, %arg1 : i32) = (%{{.*}} : i32, i32) to (%{{.*}} : i32, i32) step (%{{.*}} : i32, i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true, true>, independent = [#acc.device_type<none>]}
 
   !$acc loop tile(tileSize)
   DO i = 1, n
@@ -229,9 +229,9 @@ program acc_loop
     END DO
   END DO
 
-! CHECK:      acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) control(%arg0 : i32, %arg1 : i32) = (%{{.*}} : i32, i32) to (%{{.*}} : i32, i32) step (%{{.*}} : i32, i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true, true>, independent = [#acc.device_type<none>]}
 
   !$acc loop collapse(2)
   DO i = 1, n
@@ -246,6 +246,51 @@ program acc_loop
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>]{{.*}}}
 
+  !$acc loop collapse(2) tile(tileSize)
+  DO i = 1, n
+    DO j = 1, n
+      c(i, j) = d(i, j)
+    END DO
+  END DO
+
+! CHECK:      acc.loop {{.*}} tile({%{{.*}} : i32}) control(%arg0 : i32, %arg1 : i32) = (%{{.*}} : i32, i32) to (%{{.*}} : i32, i32) step (%{{.*}} : i32, i32) {
+! CHECK:        fir.store %arg0 to %{{.*}} : !fir.ref<i32>
+! CHECK:        fir.store %arg1 to %{{.*}} : !fir.ref<i32>
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>]{{.*}}}
+
+  !$acc loop collapse(2) tile(tileSize, tileSize, tileSize)
+  DO i = 1, n
+    DO j = 1, n
+      DO k = 1, n
+        c(i, j) = d(i, j)
+      END DO
+    END DO
+  END DO
+
+! CHECK:      acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32}) control(%arg0 : i32, %arg1 : i32, %arg2 : i32) = (%{{.*}} : i32, i32, i32) to (%{{.*}} : i32, i32, i32) step (%{{.*}} : i32, i32, i32) {
+! CHECK:        fir.store %arg0 to %{{.*}} : !fir.ref<i32>
+! CHECK:        fir.store %arg1 to %{{.*}} : !fir.ref<i32>
+! CHECK:        fir.store %arg2 to %{{.*}} : !fir.ref<i32>
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>]{{.*}}}
+
+!$acc loop collapse(3) tile(tileSize, tileSize)
+  DO i = 1, n
+    DO j = 1, n
+      DO k = 1, n
+        c(i, j) = d(i, j)
+      END DO
+    END DO
+  END DO
+
+! CHECK:      acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) control(%arg0 : i32, %arg1 : i32, %arg2 : i32) = (%{{.*}} : i32, i32, i32) to (%{{.*}} : i32, i32, i32) step (%{{.*}} : i32, i32, i32) {
+! CHECK:        fir.store %arg0 to %{{.*}} : !fir.ref<i32>
+! CHECK:        fir.store %arg1 to %{{.*}} : !fir.ref<i32>
+! CHECK:        fir.store %arg2 to %{{.*}} : !fir.ref<i32>
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {collapse = [3], collapseDeviceType = [#acc.device_type<none>]{{.*}}}
+
   !$acc loop
   DO i = 1, n
     !$acc loop

diff  --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
index 32060179acdf1..8086080bd3797 100644
--- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
@@ -681,7 +681,7 @@ subroutine acc_parallel_loop
 ! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
 ! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
-! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) control(%arg0 : i32, %arg1 : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -707,7 +707,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      acc.parallel {{.*}} {
-! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) control(%arg0 : i32, %arg1 : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield

diff  --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90
index af7bb0fac158c..cad0ee73f6cc5 100644
--- a/flang/test/Lower/OpenACC/acc-serial-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90
@@ -620,7 +620,7 @@ subroutine acc_serial_loop
 ! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
 ! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
-! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) control(%arg0 : i32, %arg1 : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -646,7 +646,7 @@ subroutine acc_serial_loop
   END DO
 
 ! CHECK:      acc.serial {{.*}} {
-! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) control(%arg0 : i32, %arg1 : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield


        

From flang-commits at lists.llvm.org  Wed Jul  9 13:47:19 2025
From: flang-commits at lists.llvm.org (Vijay Kandiah via flang-commits)
Date: Wed, 09 Jul 2025 13:47:19 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][acc][flang] Lower nested ACC loops
 with tile clause as collapsed loops (PR #147801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147801@github.com>
Message-ID: <686ed557.170a0220.3986bf.02bd@mx.google.com>


https://github.com/VijayKandiah closed https://github.com/llvm/llvm-project/pull/147801

From flang-commits at lists.llvm.org  Wed Jul  9 14:53:51 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Wed, 09 Jul 2025 14:53:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][acc] Update FIR ref, heap,
 and pointer to be MappableType (PR #147834)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147834@github.com>
Message-ID: <686ee4ef.050a0220.278bc5.a100@mx.google.com>


https://github.com/razvanlupusoru updated https://github.com/llvm/llvm-project/pull/147834

>From c6d5cf58fd920f8d5e5211337ce7d9aca8b424dd Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Wed, 9 Jul 2025 14:25:58 -0700
Subject: [PATCH 1/2] [flang][acc] Update FIR ref, heap, and pointer to be
 MappableType

The MappableType OpenACC type interface is a richer interface that
allows OpenACC dialect to be capable to better interact with a source
dialect, FIR in this case. fir.box and fir.class types already
implemented this interface. Now the same is being done with the other
FIR types that represent variables.

One additional notable change is that fir.array no longer implements
this interface. This is because MappableType is primarily intended
for variables - and FIR variables of this type have storage associated
and thus there's a pointer-like type (fir.ref/heap/pointer) that holds
the array type.

The end goal of promoting these FIR types to MappableType is that
we will soon implement ability to generate recipes outside of the
frontend via this interface.
---
 flang/lib/Lower/OpenACC.cpp                   |  13 +-
 flang/lib/Optimizer/Dialect/FIRType.cpp       |   4 +-
 .../OpenACC/FIROpenACCTypeInterfaces.cpp      | 398 ++++++++++--------
 .../OpenACC/RegisterOpenACCExtensions.cpp     |   8 +-
 flang/test/Fir/OpenACC/openacc-mappable.fir   |  11 +-
 .../OpenACC/openacc-type-categories-class.f90 |   4 +-
 .../Fir/OpenACC/openacc-type-categories.f90   |  20 +-
 .../lib/OpenACC/TestOpenACCInterfaces.cpp     |  14 +-
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       |  17 +-
 9 files changed, 277 insertions(+), 212 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index e56d7f7ed9b6f..387b0c67952a7 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -164,14 +164,13 @@ createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
   op.setStructured(structured);
   op.setImplicit(implicit);
   op.setDataClause(dataClause);
-  if (auto mappableTy =
-          mlir::dyn_cast<mlir::acc::MappableType>(baseAddr.getType())) {
-    op.setVarType(baseAddr.getType());
+  if (auto pointerLikeTy =
+          mlir::dyn_cast<mlir::acc::PointerLikeType>(baseAddr.getType())) {
+    op.setVarType(pointerLikeTy.getElementType());
   } else {
-    assert(mlir::isa<mlir::acc::PointerLikeType>(baseAddr.getType()) &&
-           "expected pointer-like");
-    op.setVarType(mlir::cast<mlir::acc::PointerLikeType>(baseAddr.getType())
-                      .getElementType());
+    assert(mlir::isa<mlir::acc::MappableType>(baseAddr.getType()) &&
+           "expected mappable");
+    op.setVarType(baseAddr.getType());
   }
 
   op->setAttr(Op::getOperandSegmentSizeAttr(),
diff --git a/flang/lib/Optimizer/Dialect/FIRType.cpp b/flang/lib/Optimizer/Dialect/FIRType.cpp
index 2ff1d6d945ba3..4a9579cfde37c 100644
--- a/flang/lib/Optimizer/Dialect/FIRType.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRType.cpp
@@ -1533,7 +1533,9 @@ std::optional<std::pair<uint64_t, unsigned short>>
 fir::getTypeSizeAndAlignment(mlir::Location loc, mlir::Type ty,
                              const mlir::DataLayout &dl,
                              const fir::KindMapping &kindMap) {
-  if (mlir::isa<mlir::IntegerType, mlir::FloatType, mlir::ComplexType>(ty)) {
+  if (ty.isIntOrIndexOrFloat() ||
+      mlir::isa<mlir::ComplexType, mlir::VectorType,
+                mlir::DataLayoutTypeInterface>(ty)) {
     llvm::TypeSize size = dl.getTypeSize(ty);
     unsigned short alignment = dl.getTypeABIAlignment(ty);
     return std::pair{size, alignment};
diff --git a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
index 317a41a2129c3..0767733f53728 100644
--- a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
@@ -29,8 +29,9 @@
 
 namespace fir::acc {
 
-static mlir::TypedValue<mlir::acc::PointerLikeType>
-getPtrFromVar(mlir::Value var) {
+template <typename Ty>
+mlir::TypedValue<mlir::acc::PointerLikeType>
+OpenACCMappableModel<Ty>::getVarPtr(mlir::Type type, mlir::Value var) const {
   if (auto ptr =
           mlir::dyn_cast<mlir::TypedValue<mlir::acc::PointerLikeType>>(var))
     return ptr;
@@ -44,34 +45,51 @@ getPtrFromVar(mlir::Value var) {
   return {};
 }
 
-template <>
-mlir::TypedValue<mlir::acc::PointerLikeType>
-OpenACCMappableModel<fir::SequenceType>::getVarPtr(mlir::Type type,
-                                                   mlir::Value var) const {
-  return getPtrFromVar(var);
-}
-
-template <>
-mlir::TypedValue<mlir::acc::PointerLikeType>
+template mlir::TypedValue<mlir::acc::PointerLikeType>
 OpenACCMappableModel<fir::BaseBoxType>::getVarPtr(mlir::Type type,
-                                                  mlir::Value var) const {
-  return getPtrFromVar(var);
-}
+                                                  mlir::Value var) const;
 
-template <>
-std::optional<llvm::TypeSize>
-OpenACCMappableModel<fir::SequenceType>::getSizeInBytes(
+template mlir::TypedValue<mlir::acc::PointerLikeType>
+OpenACCMappableModel<fir::ReferenceType>::getVarPtr(mlir::Type type,
+                                                    mlir::Value var) const;
+
+template mlir::TypedValue<mlir::acc::PointerLikeType>
+OpenACCMappableModel<fir::HeapType>::getVarPtr(mlir::Type type,
+                                               mlir::Value var) const;
+
+template mlir::TypedValue<mlir::acc::PointerLikeType>
+OpenACCMappableModel<fir::PointerType>::getVarPtr(mlir::Type type,
+                                                  mlir::Value var) const;
+
+template <typename Ty>
+std::optional<llvm::TypeSize> OpenACCMappableModel<Ty>::getSizeInBytes(
     mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
     const mlir::DataLayout &dataLayout) const {
-  // TODO: Bounds operation affect the total size - add support to take them
+  // TODO: Bounds operation affect the size - add support to take them
   // into account.
   if (!accBounds.empty())
     return {};
 
+  // Class-type is either a polymorphic or unlimited polymorphic. In the latter
+  // case, the size is not computable. But in the former it should be - however,
+  // fir::getTypeSizeAndAlignment does not support polymorphic types.
+  if (mlir::isa<fir::ClassType>(type)) {
+    return {};
+  }
+
+  // When requesting the size of a box entity or a reference, the intent
+  // is to get the size of the data that it is referring to.
+  mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(type);
+  assert(eleTy && "expect to be able to unwrap the element type");
+
+  // If the type enclosed is a mappable type, then have it provide the size.
+  if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
+    return mappableTy.getSizeInBytes(var, accBounds, dataLayout);
+
   // Dynamic extents or unknown ranks generally do not have compile-time
   // computable dimensions.
-  auto seqType = mlir::cast<fir::SequenceType>(type);
-  if (seqType.hasDynamicExtents() || seqType.hasUnknownShape())
+  auto seqType = mlir::dyn_cast<fir::SequenceType>(eleTy);
+  if (seqType && (seqType.hasDynamicExtents() || seqType.hasUnknownShape()))
     return {};
 
   // Attempt to find an operation that a lookup for KindMapping can be done
@@ -85,99 +103,113 @@ OpenACCMappableModel<fir::SequenceType>::getSizeInBytes(
   auto kindMap = fir::getKindMapping(kindMapSrcOp);
 
   auto sizeAndAlignment =
-      fir::getTypeSizeAndAlignment(var.getLoc(), type, dataLayout, kindMap);
+      fir::getTypeSizeAndAlignment(var.getLoc(), eleTy, dataLayout, kindMap);
   if (!sizeAndAlignment.has_value())
     return {};
 
   return {llvm::TypeSize::getFixed(sizeAndAlignment->first)};
 }
 
-template <>
-std::optional<llvm::TypeSize>
+template std::optional<llvm::TypeSize>
 OpenACCMappableModel<fir::BaseBoxType>::getSizeInBytes(
     mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
-    const mlir::DataLayout &dataLayout) const {
-  // If we have a box value instead of box reference, the intent is to
-  // get the size of the data not the box itself.
-  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(var.getType())) {
-    if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(
-            fir::unwrapRefType(boxTy.getEleTy()))) {
-      return mappableTy.getSizeInBytes(var, accBounds, dataLayout);
-    }
-  }
-  // Size for boxes is not computable until it gets materialized.
-  return {};
-}
+    const mlir::DataLayout &dataLayout) const;
 
-template <>
-std::optional<int64_t>
-OpenACCMappableModel<fir::SequenceType>::getOffsetInBytes(
+template std::optional<llvm::TypeSize>
+OpenACCMappableModel<fir::ReferenceType>::getSizeInBytes(
+    mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
+    const mlir::DataLayout &dataLayout) const;
+
+template std::optional<llvm::TypeSize>
+OpenACCMappableModel<fir::HeapType>::getSizeInBytes(
+    mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
+    const mlir::DataLayout &dataLayout) const;
+
+template std::optional<llvm::TypeSize>
+OpenACCMappableModel<fir::PointerType>::getSizeInBytes(
+    mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
+    const mlir::DataLayout &dataLayout) const;
+
+template <typename Ty>
+std::optional<int64_t> OpenACCMappableModel<Ty>::getOffsetInBytes(
     mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
     const mlir::DataLayout &dataLayout) const {
-  // TODO: Bounds operation affect the offset- add support to take them
+  // TODO: Bounds operation affect the offset - add support to take them
   // into account.
   if (!accBounds.empty())
     return {};
 
+  // Class-type does not behave like a normal box because it does not hold an
+  // element type. Thus special handle it here.
+  if (mlir::isa<fir::ClassType>(type)) {
+    // The pointer to the class-type is always at the start address.
+    return {0};
+  }
+
+  mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(type);
+  assert(eleTy && "expect to be able to unwrap the element type");
+
+  // If the type enclosed is a mappable type, then have it provide the offset.
+  if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
+    return mappableTy.getOffsetInBytes(var, accBounds, dataLayout);
+
   // Dynamic extents (aka descriptor-based arrays) - may have a offset.
   // For example, a negative stride may mean a negative offset to compute the
   // start of array.
-  auto seqType = mlir::cast<fir::SequenceType>(type);
-  if (seqType.hasDynamicExtents() || seqType.hasUnknownShape())
+  auto seqType = mlir::dyn_cast<fir::SequenceType>(eleTy);
+  if (seqType && (seqType.hasDynamicExtents() || seqType.hasUnknownShape()))
     return {};
 
-  // We have non-dynamic extents - but if for some reason the size is not
-  // computable - assume offset is not either. Otherwise, it is an offset of
-  // zero.
+  // If the size is computable and since there are no bounds or dynamic extents,
+  // then the offset relative to pointer must be zero.
   if (getSizeInBytes(type, var, accBounds, dataLayout).has_value()) {
     return {0};
   }
+
+  // The offset is not evident because it is relative to the pointer being held.
+  // And we don't have any further details about this type.
   return {};
 }
 
-template <>
-std::optional<int64_t> OpenACCMappableModel<fir::BaseBoxType>::getOffsetInBytes(
+template std::optional<int64_t>
+OpenACCMappableModel<fir::BaseBoxType>::getOffsetInBytes(
     mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
-    const mlir::DataLayout &dataLayout) const {
-  // If we have a box value instead of box reference, the intent is to
-  // get the offset of the data not the offset of the box itself.
-  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(var.getType())) {
-    if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(
-            fir::unwrapRefType(boxTy.getEleTy()))) {
-      return mappableTy.getOffsetInBytes(var, accBounds, dataLayout);
-    }
-  }
-  // Until boxes get materialized, the offset is not evident because it is
-  // relative to the pointer being held.
-  return {};
-}
+    const mlir::DataLayout &dataLayout) const;
 
-template <>
-llvm::SmallVector<mlir::Value>
-OpenACCMappableModel<fir::SequenceType>::generateAccBounds(
-    mlir::Type type, mlir::Value var, mlir::OpBuilder &builder) const {
+template std::optional<int64_t>
+OpenACCMappableModel<fir::ReferenceType>::getOffsetInBytes(
+    mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
+    const mlir::DataLayout &dataLayout) const;
+
+template std::optional<int64_t>
+OpenACCMappableModel<fir::HeapType>::getOffsetInBytes(
+    mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
+    const mlir::DataLayout &dataLayout) const;
+
+template std::optional<int64_t>
+OpenACCMappableModel<fir::PointerType>::getOffsetInBytes(
+    mlir::Type type, mlir::Value var, mlir::ValueRange accBounds,
+    const mlir::DataLayout &dataLayout) const;
+
+static llvm::SmallVector<mlir::Value>
+generateSeqTyAccBounds(fir::SequenceType seqType, mlir::Value var,
+                       mlir::OpBuilder &builder) {
   assert((mlir::isa<mlir::acc::PointerLikeType>(var.getType()) ||
           mlir::isa<mlir::acc::MappableType>(var.getType())) &&
          "must be pointer-like or mappable");
-
   fir::FirOpBuilder firBuilder(builder, var.getDefiningOp());
-  auto seqType = mlir::cast<fir::SequenceType>(type);
   mlir::Location loc = var.getLoc();
 
-  mlir::Value varPtr =
-      mlir::isa<mlir::acc::PointerLikeType>(var.getType())
-          ? var
-          : mlir::cast<mlir::acc::MappableType>(var.getType()).getVarPtr(var);
-
   if (seqType.hasDynamicExtents() || seqType.hasUnknownShape()) {
     if (auto boxAddr =
-            mlir::dyn_cast_if_present<fir::BoxAddrOp>(varPtr.getDefiningOp())) {
+            mlir::dyn_cast_if_present<fir::BoxAddrOp>(var.getDefiningOp())) {
       mlir::Value box = boxAddr.getVal();
       auto res =
           hlfir::translateToExtendedValue(loc, firBuilder, hlfir::Entity(box));
       fir::ExtendedValue exv = res.first;
       mlir::Value boxRef = box;
-      if (auto boxPtr = getPtrFromVar(box)) {
+      if (auto boxPtr = mlir::cast<mlir::acc::MappableType>(box.getType())
+                            .getVarPtr(box)) {
         boxRef = boxPtr;
       }
       // TODO: Handle Fortran optional.
@@ -189,7 +221,7 @@ OpenACCMappableModel<fir::SequenceType>::generateAccBounds(
           firBuilder, loc, exv, info);
     }
 
-    if (mlir::isa<hlfir::DeclareOp, fir::DeclareOp>(varPtr.getDefiningOp())) {
+    if (mlir::isa<hlfir::DeclareOp, fir::DeclareOp>(var.getDefiningOp())) {
       mlir::Value zero =
           firBuilder.createIntegerConstant(loc, builder.getIndexType(), 0);
       mlir::Value one =
@@ -197,10 +229,10 @@ OpenACCMappableModel<fir::SequenceType>::generateAccBounds(
 
       mlir::Value shape;
       if (auto declareOp =
-              mlir::dyn_cast_if_present<fir::DeclareOp>(varPtr.getDefiningOp()))
+              mlir::dyn_cast_if_present<fir::DeclareOp>(var.getDefiningOp()))
         shape = declareOp.getShape();
       else if (auto declareOp = mlir::dyn_cast_if_present<hlfir::DeclareOp>(
-                   varPtr.getDefiningOp()))
+                   var.getDefiningOp()))
         shape = declareOp.getShape();
 
       const bool strideIncludeLowerExtent = true;
@@ -265,9 +297,9 @@ OpenACCMappableModel<fir::SequenceType>::generateAccBounds(
 
   // TODO: Detect assumed-size case.
   const bool isAssumedSize = false;
-  auto valToCheck = varPtr;
+  auto valToCheck = var;
   if (auto boxAddr =
-          mlir::dyn_cast_if_present<fir::BoxAddrOp>(varPtr.getDefiningOp())) {
+          mlir::dyn_cast_if_present<fir::BoxAddrOp>(var.getDefiningOp())) {
     valToCheck = boxAddr.getVal();
   }
   auto res = hlfir::translateToExtendedValue(loc, firBuilder,
@@ -279,86 +311,34 @@ OpenACCMappableModel<fir::SequenceType>::generateAccBounds(
       /*isAssumedSize=*/isAssumedSize);
 }
 
-template <>
+template <typename Ty>
 llvm::SmallVector<mlir::Value>
-OpenACCMappableModel<fir::BaseBoxType>::generateAccBounds(
-    mlir::Type type, mlir::Value var, mlir::OpBuilder &builder) const {
-  // If we have a box value instead of box reference, the intent is to
-  // get the bounds of the data not the bounds of the box itself.
-  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(var.getType())) {
-    if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(
-            fir::unwrapRefType(boxTy.getEleTy()))) {
-      mlir::Value data = builder.create<fir::BoxAddrOp>(var.getLoc(), var);
-      return mappableTy.generateAccBounds(data, builder);
-    }
+OpenACCMappableModel<Ty>::generateAccBounds(mlir::Type type, mlir::Value var,
+                                            mlir::OpBuilder &builder) const {
+  // acc bounds only make sense for arrays - thus look for sequence type.
+  mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(type);
+  if (auto seqTy = mlir::dyn_cast_if_present<fir::SequenceType>(eleTy)) {
+    return generateSeqTyAccBounds(seqTy, var, builder);
   }
-  // Box references are not arrays - thus generating acc.bounds does not make
-  // sense.
-  return {};
-}
-
-static bool isScalarLike(mlir::Type type) {
-  return fir::isa_trivial(type) || fir::isa_ref_type(type);
-}
-
-static bool isArrayLike(mlir::Type type) {
-  return mlir::isa<fir::SequenceType>(type);
-}
 
-static bool isCompositeLike(mlir::Type type) {
-  // class(*) is not a composite type since it does not have a determined type.
-  if (fir::isUnlimitedPolymorphicType(type))
-    return false;
-
-  return mlir::isa<fir::RecordType, fir::ClassType, mlir::TupleType>(type);
-}
-
-template <>
-mlir::acc::VariableTypeCategory
-OpenACCMappableModel<fir::SequenceType>::getTypeCategory(
-    mlir::Type type, mlir::Value var) const {
-  return mlir::acc::VariableTypeCategory::array;
+  return {};
 }
 
-template <>
-mlir::acc::VariableTypeCategory
-OpenACCMappableModel<fir::BaseBoxType>::getTypeCategory(mlir::Type type,
-                                                        mlir::Value var) const {
-  // Class-type does not behave like a normal box because it does not hold an
-  // element type. Thus special handle it here.
-  if (mlir::isa<fir::ClassType>(type)) {
-    // class(*) is not a composite type since it does not have a determined
-    // type.
-    if (fir::isUnlimitedPolymorphicType(type))
-      return mlir::acc::VariableTypeCategory::uncategorized;
-    return mlir::acc::VariableTypeCategory::composite;
-  }
-
-  mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(type);
-  assert(eleTy && "expect to be able to unwrap the element type");
+template llvm::SmallVector<mlir::Value>
+OpenACCMappableModel<fir::BaseBoxType>::generateAccBounds(
+    mlir::Type type, mlir::Value var, mlir::OpBuilder &builder) const;
 
-  // If the type enclosed by the box is a mappable type, then have it
-  // provide the type category.
-  if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
-    return mappableTy.getTypeCategory(var);
+template llvm::SmallVector<mlir::Value>
+OpenACCMappableModel<fir::ReferenceType>::generateAccBounds(
+    mlir::Type type, mlir::Value var, mlir::OpBuilder &builder) const;
 
-  // For all arrays, despite whether they are allocatable, pointer, assumed,
-  // etc, we'd like to categorize them as "array".
-  if (isArrayLike(eleTy))
-    return mlir::acc::VariableTypeCategory::array;
-
-  // We got here because we don't have an array nor a mappable type. At this
-  // point, we know we have a type that fits the "aggregate" definition since it
-  // is a type with a descriptor. Try to refine it by checking if it matches the
-  // "composite" definition.
-  if (isCompositeLike(eleTy))
-    return mlir::acc::VariableTypeCategory::composite;
+template llvm::SmallVector<mlir::Value>
+OpenACCMappableModel<fir::HeapType>::generateAccBounds(
+    mlir::Type type, mlir::Value var, mlir::OpBuilder &builder) const;
 
-  // Even if we have a scalar type - simply because it is wrapped in a box
-  // we want to categorize it as "nonscalar". Anything else would've been
-  // non-scalar anyway.
-  return mlir::acc::VariableTypeCategory::nonscalar;
-}
+template llvm::SmallVector<mlir::Value>
+OpenACCMappableModel<fir::PointerType>::generateAccBounds(
+    mlir::Type type, mlir::Value var, mlir::OpBuilder &builder) const;
 
 static mlir::Value
 getBaseRef(mlir::TypedValue<mlir::acc::PointerLikeType> varPtr) {
@@ -389,33 +369,44 @@ getBaseRef(mlir::TypedValue<mlir::acc::PointerLikeType> varPtr) {
   return baseRef;
 }
 
-static mlir::acc::VariableTypeCategory
-categorizePointee(mlir::Type pointer,
-                  mlir::TypedValue<mlir::acc::PointerLikeType> varPtr,
-                  mlir::Type varType) {
-  // FIR uses operations to compute interior pointers.
-  // So for example, an array element or composite field access to a float
-  // value would both be represented as !fir.ref<f32>. We do not want to treat
-  // such a reference as a scalar. Thus unwrap interior pointer calculations.
-  auto baseRef = getBaseRef(varPtr);
+static bool isScalarLike(mlir::Type type) {
+  return fir::isa_trivial(type) || fir::isa_ref_type(type);
+}
 
-  if (auto mappableTy =
-          mlir::dyn_cast<mlir::acc::MappableType>(baseRef.getType()))
-    return mappableTy.getTypeCategory(baseRef);
+static bool isArrayLike(mlir::Type type) {
+  return mlir::isa<fir::SequenceType>(type);
+}
 
-  // It must be a pointer-like type since it is not a MappableType.
-  auto ptrLikeTy = mlir::cast<mlir::acc::PointerLikeType>(baseRef.getType());
-  mlir::Type eleTy = ptrLikeTy.getElementType();
+static bool isCompositeLike(mlir::Type type) {
+  // class(*) is not a composite type since it does not have a determined type.
+  if (fir::isUnlimitedPolymorphicType(type))
+    return false;
 
-  if (auto mappableEleTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
-    return mappableEleTy.getTypeCategory(varPtr);
+  return mlir::isa<fir::RecordType, fir::ClassType, mlir::TupleType>(type);
+}
 
-  if (isScalarLike(eleTy))
-    return mlir::acc::VariableTypeCategory::scalar;
+static mlir::acc::VariableTypeCategory
+categorizeElemType(mlir::Type enclosingTy, mlir::Type eleTy, mlir::Value var) {
+  // If the type enclosed is a mappable type, then have it provide the type
+  // category.
+  if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
+    return mappableTy.getTypeCategory(var);
+
+  // For all arrays, despite whether they are allocatable, pointer, assumed,
+  // etc, we'd like to categorize them as "array".
   if (isArrayLike(eleTy))
     return mlir::acc::VariableTypeCategory::array;
+
   if (isCompositeLike(eleTy))
     return mlir::acc::VariableTypeCategory::composite;
+  if (mlir::isa<fir::BoxType>(enclosingTy)) {
+    // Even if we have a scalar type - simply because it is wrapped in a box
+    // we want to categorize it as "nonscalar". Anything else would've been
+    // non-scalar anyway.
+    return mlir::acc::VariableTypeCategory::nonscalar;
+  }
+  if (isScalarLike(eleTy))
+    return mlir::acc::VariableTypeCategory::scalar;
   if (mlir::isa<fir::CharacterType, mlir::FunctionType>(eleTy))
     return mlir::acc::VariableTypeCategory::nonscalar;
   // Assumed-type (type(*))does not have a determined type that can be
@@ -431,6 +422,77 @@ categorizePointee(mlir::Type pointer,
   return mlir::acc::VariableTypeCategory::uncategorized;
 }
 
+template <typename Ty>
+mlir::acc::VariableTypeCategory
+OpenACCMappableModel<Ty>::getTypeCategory(mlir::Type type,
+                                          mlir::Value var) const {
+  // FIR uses operations to compute interior pointers.
+  // So for example, an array element or composite field access to a float
+  // value would both be represented as !fir.ref<f32>. We do not want to treat
+  // such a reference as a scalar. Thus unwrap interior pointer calculations.
+  mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(type);
+  if (eleTy && isScalarLike(eleTy)) {
+    if (auto ptrLikeVar = mlir::dyn_cast_if_present<
+            mlir::TypedValue<mlir::acc::PointerLikeType>>(var)) {
+      auto baseRef = getBaseRef(ptrLikeVar);
+      if (baseRef != var) {
+        type = baseRef.getType();
+        if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(type))
+          return mappableTy.getTypeCategory(baseRef);
+      }
+    }
+  }
+
+  // Class-type does not behave like a normal box because it does not hold an
+  // element type. Thus special handle it here.
+  if (mlir::isa<fir::ClassType>(type)) {
+    // class(*) is not a composite type since it does not have a determined
+    // type.
+    if (fir::isUnlimitedPolymorphicType(type))
+      return mlir::acc::VariableTypeCategory::uncategorized;
+    return mlir::acc::VariableTypeCategory::composite;
+  }
+
+  assert(eleTy && "expect to be able to unwrap the element type");
+  return categorizeElemType(type, eleTy, var);
+}
+
+template mlir::acc::VariableTypeCategory
+OpenACCMappableModel<fir::BaseBoxType>::getTypeCategory(mlir::Type type,
+                                                        mlir::Value var) const;
+
+template mlir::acc::VariableTypeCategory
+OpenACCMappableModel<fir::ReferenceType>::getTypeCategory(
+    mlir::Type type, mlir::Value var) const;
+
+template mlir::acc::VariableTypeCategory
+OpenACCMappableModel<fir::HeapType>::getTypeCategory(mlir::Type type,
+                                                     mlir::Value var) const;
+
+template mlir::acc::VariableTypeCategory
+OpenACCMappableModel<fir::PointerType>::getTypeCategory(mlir::Type type,
+                                                        mlir::Value var) const;
+
+static mlir::acc::VariableTypeCategory
+categorizePointee(mlir::Type pointer,
+                  mlir::TypedValue<mlir::acc::PointerLikeType> varPtr,
+                  mlir::Type varType) {
+  // FIR uses operations to compute interior pointers.
+  // So for example, an array element or composite field access to a float
+  // value would both be represented as !fir.ref<f32>. We do not want to treat
+  // such a reference as a scalar. Thus unwrap interior pointer calculations.
+  auto baseRef = getBaseRef(varPtr);
+
+  if (auto mappableTy =
+          mlir::dyn_cast<mlir::acc::MappableType>(baseRef.getType()))
+    return mappableTy.getTypeCategory(baseRef);
+
+  // It must be a pointer-like type since it is not a MappableType.
+  auto ptrLikeTy = mlir::cast<mlir::acc::PointerLikeType>(baseRef.getType());
+  mlir::Type eleTy = ptrLikeTy.getElementType();
+  return categorizeElemType(pointer, eleTy, varPtr);
+}
+
 template <>
 mlir::acc::VariableTypeCategory
 OpenACCPointerLikeModel<fir::ReferenceType>::getPointeeTypeCategory(
diff --git a/flang/lib/Optimizer/OpenACC/RegisterOpenACCExtensions.cpp b/flang/lib/Optimizer/OpenACC/RegisterOpenACCExtensions.cpp
index 5f174ad4b40fe..869f9c2429aa0 100644
--- a/flang/lib/Optimizer/OpenACC/RegisterOpenACCExtensions.cpp
+++ b/flang/lib/Optimizer/OpenACC/RegisterOpenACCExtensions.cpp
@@ -19,11 +19,14 @@ namespace fir::acc {
 void registerOpenACCExtensions(mlir::DialectRegistry &registry) {
   registry.addExtension(+[](mlir::MLIRContext *ctx,
                             fir::FIROpsDialect *dialect) {
-    fir::SequenceType::attachInterface<OpenACCMappableModel<fir::SequenceType>>(
-        *ctx);
     fir::BoxType::attachInterface<OpenACCMappableModel<fir::BaseBoxType>>(*ctx);
     fir::ClassType::attachInterface<OpenACCMappableModel<fir::BaseBoxType>>(
         *ctx);
+    fir::ReferenceType::attachInterface<
+        OpenACCMappableModel<fir::ReferenceType>>(*ctx);
+    fir::PointerType::attachInterface<OpenACCMappableModel<fir::PointerType>>(
+        *ctx);
+    fir::HeapType::attachInterface<OpenACCMappableModel<fir::HeapType>>(*ctx);
 
     fir::ReferenceType::attachInterface<
         OpenACCPointerLikeModel<fir::ReferenceType>>(*ctx);
@@ -31,6 +34,7 @@ void registerOpenACCExtensions(mlir::DialectRegistry &registry) {
         OpenACCPointerLikeModel<fir::PointerType>>(*ctx);
     fir::HeapType::attachInterface<OpenACCPointerLikeModel<fir::HeapType>>(
         *ctx);
+
     fir::LLVMPointerType::attachInterface<
         OpenACCPointerLikeModel<fir::LLVMPointerType>>(*ctx);
   });
diff --git a/flang/test/Fir/OpenACC/openacc-mappable.fir b/flang/test/Fir/OpenACC/openacc-mappable.fir
index 3e3e455469f69..71576f4b71075 100644
--- a/flang/test/Fir/OpenACC/openacc-mappable.fir
+++ b/flang/test/Fir/OpenACC/openacc-mappable.fir
@@ -23,7 +23,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<f16 = dense<16> : vector<2xi64>,
   // CHECK: Size: 40
 
   // CHECK: Visiting: %{{.*}} = acc.copyin varPtr(%{{.*}} : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "arr", structured = false}
-  // CHECK: Mappable: !fir.array<10xf32>
+  // CHECK: Pointer-like and Mappable: !fir.ref<!fir.array<10xf32>>
   // CHECK: Type category: array
   // CHECK: Size: 40
 
@@ -60,20 +60,17 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<f16 = dense<16> : vector<2xi64>,
   }
 
   // CHECK: Visiting: %{{.*}} = acc.copyin varPtr(%{{.*}} : !fir.ref<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>> {name = "arr1", structured = false}
-  // CHECK: Pointer-like: !fir.ref<!fir.array<?xf32>>
-  // CHECK: Mappable: !fir.array<?xf32>
+  // CHECK: Pointer-like and Mappable: !fir.ref<!fir.array<?xf32>>
   // CHECK: Type category: array
   // CHECK: Bound[0]: %{{.*}} = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%c1{{.*}} : index)
 
   // CHECK: Visiting: %{{.*}} = acc.copyin varPtr(%{{.*}} : !fir.ref<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>> {name = "arr2", structured = false}
-  // CHECK: Pointer-like: !fir.ref<!fir.array<?xf32>>
-  // CHECK: Mappable: !fir.array<?xf32>
+  // CHECK: Pointer-like and Mappable: !fir.ref<!fir.array<?xf32>>
   // CHECK: Type category: array
   // CHECK: Bound[0]: %{{.*}} = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%c2{{.*}} : index)
 
   // CHECK: Visiting: %{{.*}} = acc.copyin varPtr(%{{.*}} : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "arr3", structured = false}
-  // CHECK: Pointer-like: !fir.ref<!fir.array<10xf32>>
-  // CHECK: Mappable: !fir.array<10xf32>
+  // CHECK: Pointer-like and Mappable: !fir.ref<!fir.array<10xf32>>
   // CHECK: Type category: array
   // CHECK: Size: 40
   // CHECK: Offset: 0
diff --git a/flang/test/Fir/OpenACC/openacc-type-categories-class.f90 b/flang/test/Fir/OpenACC/openacc-type-categories-class.f90
index 58025bfa556a5..e8951cceeeaeb 100644
--- a/flang/test/Fir/OpenACC/openacc-type-categories-class.f90
+++ b/flang/test/Fir/OpenACC/openacc-type-categories-class.f90
@@ -29,13 +29,13 @@ subroutine init_unlimited(this)
 ! CHECK: Mappable: !fir.class<!fir.type<_QMmmTpolyty{field:f32}>>
 ! CHECK: Type category: composite
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "this%field", structured = false}
-! CHECK: Pointer-like: !fir.ref<f32>
+! CHECK: Pointer-like and Mappable: !fir.ref<f32>
 ! CHECK: Type category: composite
 
 ! For unlimited polymorphic entities and assumed types - they effectively have
 ! no declared type. Thus the type categorizer cannot categorize it.
 ! CHECK: Visiting: {{.*}} = acc.copyin {{.*}} {name = "var", structured = false}
-! CHECK: Pointer-like: !fir.ref<none>
+! CHECK: Pointer-like and Mappable: !fir.ref<none>
 ! CHECK: Type category: uncategorized
 ! CHECK: Visiting: {{.*}} = acc.copyin {{.*}} {name = "this", structured = false}
 ! CHECK: Mappable: !fir.class<none>
diff --git a/flang/test/Fir/OpenACC/openacc-type-categories.f90 b/flang/test/Fir/OpenACC/openacc-type-categories.f90
index c25c38422b755..3d6067db8224d 100644
--- a/flang/test/Fir/OpenACC/openacc-type-categories.f90
+++ b/flang/test/Fir/OpenACC/openacc-type-categories.f90
@@ -18,32 +18,32 @@ program main
 end program
 
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "scalar", structured = false}
-! CHECK: Pointer-like: !fir.ref<f32>
+! CHECK: Pointer-like and Mappable: !fir.ref<f32>
 ! CHECK: Type category: scalar
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "scalaralloc", structured = false}
-! CHECK: Pointer-like: !fir.ref<!fir.box<!fir.heap<f32>>>
+! CHECK: Pointer-like and Mappable: !fir.ref<!fir.box<!fir.heap<f32>>>
 ! CHECK: Type category: nonscalar
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "ttvar", structured = false}
-! CHECK: Pointer-like: !fir.ref<!fir.type<_QFTtt{field:f32,fieldarray:!fir.array<10xf32>}>>
+! CHECK: Pointer-like and Mappable: !fir.ref<!fir.type<_QFTtt{field:f32,fieldarray:!fir.array<10xf32>}>>
 ! CHECK: Type category: composite
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "arrayconstsize", structured = false}
-! CHECK: Pointer-like: !fir.ref<!fir.array<10xf32>>
+! CHECK: Pointer-like and Mappable: !fir.ref<!fir.array<10xf32>>
 ! CHECK: Type category: array
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "arrayalloc", structured = false}
-! CHECK: Pointer-like: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: Pointer-like and Mappable: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: Type category: array
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "complexvar", structured = false}
-! CHECK: Pointer-like: !fir.ref<complex<f32>>
+! CHECK: Pointer-like and Mappable: !fir.ref<complex<f32>>
 ! CHECK: Type category: scalar
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "charvar", structured = false}
-! CHECK: Pointer-like: !fir.ref<!fir.char<1>>
+! CHECK: Pointer-like and Mappable: !fir.ref<!fir.char<1>>
 ! CHECK: Type category: nonscalar
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "ttvar%field", structured = false}
-! CHECK: Pointer-like: !fir.ref<f32>
+! CHECK: Pointer-like and Mappable: !fir.ref<f32>
 ! CHECK: Type category: composite
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "ttvar%fieldarray", structured = false}
-! CHECK: Pointer-like: !fir.ref<!fir.array<10xf32>>
+! CHECK: Pointer-like and Mappable: !fir.ref<!fir.array<10xf32>>
 ! CHECK: Type category: array
 ! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "arrayconstsize(1)", structured = false}
-! CHECK: Pointer-like: !fir.ref<!fir.array<10xf32>>
+! CHECK: Pointer-like and Mappable: !fir.ref<!fir.array<10xf32>>
 ! CHECK: Type category: array
diff --git a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
index e72b96fe7cd10..1888893a4f8b8 100644
--- a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
+++ b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
@@ -58,8 +58,18 @@ struct TestFIROpenACCInterfaces
         llvm::errs() << "Visiting: " << *op << "\n";
         llvm::errs() << "\tVar: " << var << "\n";
 
-        if (auto ptrTy = dyn_cast_if_present<acc::PointerLikeType>(typeOfVar)) {
+        if (mlir::isa<acc::PointerLikeType>(typeOfVar) &&
+            mlir::isa<acc::MappableType>(typeOfVar)) {
+          llvm::errs() << "\tPointer-like and Mappable: " << typeOfVar << "\n";
+        } else if (mlir::isa<acc::PointerLikeType>(typeOfVar)) {
           llvm::errs() << "\tPointer-like: " << typeOfVar << "\n";
+        } else {
+          assert(mlir::isa<acc::MappableType>(typeOfVar) &&
+                 "expected mappable");
+          llvm::errs() << "\tMappable: " << typeOfVar << "\n";
+        }
+
+        if (auto ptrTy = dyn_cast_if_present<acc::PointerLikeType>(typeOfVar)) {
           // If the pointee is not mappable, print details about it. Otherwise,
           // we defer to the mappable printing below to print those details.
           if (!mappableTy) {
@@ -72,8 +82,6 @@ struct TestFIROpenACCInterfaces
         }
 
         if (mappableTy) {
-          llvm::errs() << "\tMappable: " << mappableTy << "\n";
-
           acc::VariableTypeCategory typeCategory =
               mappableTy.getTypeCategory(var);
           llvm::errs() << "\t\tType category: " << typeCategory << "\n";
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 80c807e774a7e..f2eab62b286af 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -293,22 +293,15 @@ static LogicalResult checkVarAndVarType(Op op) {
   if (!op.getVar())
     return op.emitError("must have var operand");
 
-  if (mlir::isa<mlir::acc::PointerLikeType>(op.getVar().getType()) &&
-      mlir::isa<mlir::acc::MappableType>(op.getVar().getType())) {
-    // TODO: If a type implements both interfaces (mappable and pointer-like),
-    // it is unclear which semantics to apply without additional info which
-    // would need captured in the data operation. For now restrict this case
-    // unless a compelling reason to support disambiguating between the two.
-    return op.emitError("var must be mappable or pointer-like (not both)");
-  }
-
+  // A variable must have a type that is either pointer-like or mappable.
   if (!mlir::isa<mlir::acc::PointerLikeType>(op.getVar().getType()) &&
       !mlir::isa<mlir::acc::MappableType>(op.getVar().getType()))
     return op.emitError("var must be mappable or pointer-like");
 
-  if (mlir::isa<mlir::acc::MappableType>(op.getVar().getType()) &&
-      op.getVarType() != op.getVar().getType())
-    return op.emitError("varType must match when var is mappable");
+  // When it is a pointer-like type, the varType must capture the target type.
+  if (mlir::isa<mlir::acc::PointerLikeType>(op.getVar().getType()) &&
+      op.getVarType() == op.getVar().getType())
+    return op.emitError("varType must capture the element type of var");
 
   return success();
 }

>From de232922767bb21a4a17651d73d6969a2e70ee40 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Wed, 9 Jul 2025 14:53:39 -0700
Subject: [PATCH 2/2] Fix format

---
 flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
index 1888893a4f8b8..de6cb1d09080d 100644
--- a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
+++ b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
@@ -64,8 +64,8 @@ struct TestFIROpenACCInterfaces
         } else if (mlir::isa<acc::PointerLikeType>(typeOfVar)) {
           llvm::errs() << "\tPointer-like: " << typeOfVar << "\n";
         } else {
-          assert(mlir::isa<acc::MappableType>(typeOfVar) &&
-                 "expected mappable");
+          assert(
+              mlir::isa<acc::MappableType>(typeOfVar) && "expected mappable");
           llvm::errs() << "\tMappable: " << typeOfVar << "\n";
         }
 


From flang-commits at lists.llvm.org  Wed Jul  9 15:01:22 2025
From: flang-commits at lists.llvm.org (Daniel Paoliello via flang-commits)
Date: Wed, 09 Jul 2025 15:01:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't check the '-mframe-pointer'
	flag (PR #147837)
Message-ID: <llvm.org/llvm/llvm-project/pull/147837@github.com>


https://github.com/dpaoliello created https://github.com/llvm/llvm-project/pull/147837

The `-mframe-pointer` flag is not explicitly set in the original `flang` invocation and so the value passed to `flang -fc1` can vary depending on the host machine, so don't verify it in the output.

`-mframe-pointer` forwarding is already verified by `flang/test/Driver/frame-pointer-forwarding.f90`.

>From 987bb9701d753a76277f0cc36c635794264de06e Mon Sep 17 00:00:00 2001
From: Daniel Paoliello <danpao at microsoft.com>
Date: Wed, 9 Jul 2025 14:59:15 -0700
Subject: [PATCH] [flang] Don't check the '-mframe-pointer' flag

---
 flang/test/Driver/frontend-forwarding.f90 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/test/Driver/frontend-forwarding.f90 b/flang/test/Driver/frontend-forwarding.f90
index 55a74ccf40467..952937168c95d 100644
--- a/flang/test/Driver/frontend-forwarding.f90
+++ b/flang/test/Driver/frontend-forwarding.f90
@@ -60,7 +60,6 @@
 ! CHECK: "-Reverything"
 ! CHECK: "-Rno-everything"
 ! CHECK: "-Rpass=inline"
-! CHECK: "-mframe-pointer=none"
 ! CHECK: "-mllvm" "-print-before-all"
 ! CHECK: "-fwrapv"
 ! CHECK: "-save-temps=obj"


From flang-commits at lists.llvm.org  Wed Jul  9 15:01:55 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 15:01:55 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't check the '-mframe-pointer'
	flag (PR #147837)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147837@github.com>
Message-ID: <686ee6d3.170a0220.825ef.0483@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-driver

Author: Daniel Paoliello (dpaoliello)

<details>
<summary>Changes</summary>

The `-mframe-pointer` flag is not explicitly set in the original `flang` invocation and so the value passed to `flang -fc1` can vary depending on the host machine, so don't verify it in the output.

`-mframe-pointer` forwarding is already verified by `flang/test/Driver/frame-pointer-forwarding.f90`.

---
Full diff: https://github.com/llvm/llvm-project/pull/147837.diff


1 Files Affected:

- (modified) flang/test/Driver/frontend-forwarding.f90 (-1) 


``````````diff
diff --git a/flang/test/Driver/frontend-forwarding.f90 b/flang/test/Driver/frontend-forwarding.f90
index 55a74ccf40467..952937168c95d 100644
--- a/flang/test/Driver/frontend-forwarding.f90
+++ b/flang/test/Driver/frontend-forwarding.f90
@@ -60,7 +60,6 @@
 ! CHECK: "-Reverything"
 ! CHECK: "-Rno-everything"
 ! CHECK: "-Rpass=inline"
-! CHECK: "-mframe-pointer=none"
 ! CHECK: "-mllvm" "-print-before-all"
 ! CHECK: "-fwrapv"
 ! CHECK: "-save-temps=obj"

``````````

</details>


https://github.com/llvm/llvm-project/pull/147837

From flang-commits at lists.llvm.org  Wed Jul  9 20:03:30 2025
From: flang-commits at lists.llvm.org (Kelvin Li via flang-commits)
Date: Wed, 09 Jul 2025 20:03:30 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang] Add -fcomplex-arithmetic=
 option and select complex division algorithm (PR #146641)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146641@github.com>
Message-ID: <686f2d82.050a0220.3d96c9.0ccd@mx.google.com>


kkwli wrote:

> LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `clang,flang` at step 5 "build-unified-tree".
> 
> Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/33205
> Here is the relevant piece of the build log for the reference

The failure is fixed in https://github.com/llvm/llvm-project/pull/147816.

https://github.com/llvm/llvm-project/pull/146641

From flang-commits at lists.llvm.org  Wed Jul  9 21:04:44 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 21:04:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147881@github.com>
Message-ID: <686f3bdc.170a0220.1d41b0.11a0@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-driver

@llvm/pr-subscribers-flang-fir-hlfir

Author: Slava Zakharin (vzakhari)

<details>
<summary>Changes</summary>

This patch allows optimizing redundant array repacking, when
the source array is statically known to be contiguous.
This is part of the implementation plan for the array repacking
feature, though, it does not affect any real life use case
as long as FIR inlining is not a thing. I experimented with
simple cases of FIR inling using `-inline-all`, and I recorded
these cases in optimize-array-repacking.fir tests.


---

Patch is 91.37 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/147881.diff


16 Files Affected:

- (modified) flang/include/flang/Optimizer/Builder/HLFIRTools.h (+8) 
- (modified) flang/include/flang/Optimizer/Dialect/FIROpsSupport.h (+13-1) 
- (modified) flang/include/flang/Optimizer/Transforms/Passes.h (+1-33) 
- (modified) flang/include/flang/Optimizer/Transforms/Passes.td (+10) 
- (modified) flang/lib/Optimizer/Builder/HLFIRTools.cpp (+37-3) 
- (modified) flang/lib/Optimizer/Dialect/FIROps.cpp (+129-28) 
- (modified) flang/lib/Optimizer/Passes/Pipelines.cpp (+4) 
- (modified) flang/lib/Optimizer/Transforms/CMakeLists.txt (+1) 
- (modified) flang/lib/Optimizer/Transforms/LoopVersioning.cpp (+3-1) 
- (added) flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp (+90) 
- (modified) flang/test/Driver/bbc-mlir-pass-pipeline.f90 (+2) 
- (modified) flang/test/Driver/mlir-pass-pipeline.f90 (+2) 
- (modified) flang/test/Fir/basic-program.fir (+2) 
- (modified) flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir (+6-30) 
- (modified) flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir (+14-93) 
- (added) flang/test/Transforms/optimize-array-repacking.fir (+660) 


``````````diff
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 2cbad6e268a38..49dfc85dc76e6 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -542,6 +542,14 @@ Entity gen1DSection(mlir::Location loc, fir::FirOpBuilder &builder,
 /// contiguous.
 bool designatePreservesContinuity(hlfir::DesignateOp op);
 
+/// Return true iff the given \p base desribes an object
+/// that is contiguous. If \p checkWhole is true, then
+/// the object must be contiguous in all dimensions,
+/// otherwise, it must be contiguous in the innermost dimension.
+/// This function is an extension of hlfir::Entity::isSimplyContiguous(),
+/// and it can be used on pure FIR representation as well as on HLFIR.
+bool isSimplyContiguous(mlir::Value base, bool checkWhole = true);
+
 } // namespace hlfir
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_HLFIRTOOLS_H
diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index 0a2337be7455e..ae471eb0d2e04 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -238,7 +238,19 @@ std::optional<int64_t> getAllocaByteSize(fir::AllocaOp alloca,
 /// When \p checkWhole is false, then the checking is only done
 /// for continuity in the innermost dimension, otherwise,
 /// the checking is done for continuity of the whole result of rebox.
-bool reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole = true);
+/// The caller may specify \p mayHaveNonDefaultLowerBounds, if it is known,
+/// to allow better handling of the rebox operations representing
+/// full array slices.
+bool reboxPreservesContinuity(fir::ReboxOp rebox,
+                              bool mayHaveNonDefaultLowerBounds = true,
+                              bool checkWhole = true);
+
+/// Return true, if \p embox operation produces a contiguous
+/// entity.
+/// When \p checkWhole is false, then the checking is only done
+/// for continuity in the innermost dimension, otherwise,
+/// the checking is done for continuity of the whole result of embox
+bool isContiguousEmbox(fir::EmboxOp embox, bool checkWhole = true);
 
 } // namespace fir
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index dc8a5b9141ad2..6f5dff4687cbb 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -31,39 +31,7 @@ namespace fir {
 // Passes defined in Passes.td
 //===----------------------------------------------------------------------===//
 
-#define GEN_PASS_DECL_ABSTRACTRESULTOPT
-#define GEN_PASS_DECL_AFFINEDIALECTPROMOTION
-#define GEN_PASS_DECL_AFFINEDIALECTDEMOTION
-#define GEN_PASS_DECL_ANNOTATECONSTANTOPERANDS
-#define GEN_PASS_DECL_ARRAYVALUECOPY
-#define GEN_PASS_DECL_ASSUMEDRANKOPCONVERSION
-#define GEN_PASS_DECL_CHARACTERCONVERSION
-#define GEN_PASS_DECL_CFGCONVERSION
-#define GEN_PASS_DECL_CUFADDCONSTRUCTOR
-#define GEN_PASS_DECL_CUFDEVICEGLOBAL
-#define GEN_PASS_DECL_CUFGPUTOLLVMCONVERSION
-#define GEN_PASS_DECL_CUFOPCONVERSION
-#define GEN_PASS_DECL_CUFCOMPUTESHAREDMEMORYOFFSETSANDSIZE
-#define GEN_PASS_DECL_EXTERNALNAMECONVERSION
-#define GEN_PASS_DECL_MEMREFDATAFLOWOPT
-#define GEN_PASS_DECL_SIMPLIFYINTRINSICS
-#define GEN_PASS_DECL_MEMORYALLOCATIONOPT
-#define GEN_PASS_DECL_SIMPLIFYREGIONLITE
-#define GEN_PASS_DECL_ALGEBRAICSIMPLIFICATION
-#define GEN_PASS_DECL_POLYMORPHICOPCONVERSION
-#define GEN_PASS_DECL_OPENACCDATAOPERANDCONVERSION
-#define GEN_PASS_DECL_ADDDEBUGINFO
-#define GEN_PASS_DECL_STACKARRAYS
-#define GEN_PASS_DECL_STACKRECLAIM
-#define GEN_PASS_DECL_LOOPVERSIONING
-#define GEN_PASS_DECL_ADDALIASTAGS
-#define GEN_PASS_DECL_VSCALEATTR
-#define GEN_PASS_DECL_FUNCTIONATTR
-#define GEN_PASS_DECL_CONSTANTARGUMENTGLOBALISATIONOPT
-#define GEN_PASS_DECL_COMPILERGENERATEDNAMESCONVERSION
-#define GEN_PASS_DECL_SETRUNTIMECALLATTRIBUTES
-#define GEN_PASS_DECL_GENRUNTIMECALLSFORTEST
-#define GEN_PASS_DECL_SIMPLIFYFIROPERATIONS
+#define GEN_PASS_DECL
 
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 34842f9785942..b230f60b4c59e 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -551,4 +551,14 @@ def SimplifyFIROperations : Pass<"simplify-fir-operations", "mlir::ModuleOp"> {
       "Prefer expanding without using Fortran runtime calls.">];
 }
 
+def OptimizeArrayRepacking
+    : Pass<"optimize-array-repacking", "mlir::func::FuncOp"> {
+  let summary = "Optimizes redundant array repacking operations";
+  let description = [{
+    If the source of fir.pack_array is known to be contiguous,
+    then this pass erases such operations. The corresponding
+    fir.unpack_array operations are also removed.
+  }];
+}
+
 #endif // FLANG_OPTIMIZER_TRANSFORMS_PASSES
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index b54cccbef14fa..e59a6bf2bf224 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -212,10 +212,17 @@ bool hlfir::Entity::mayHaveNonDefaultLowerBounds() const {
   if (auto varIface = getIfVariableInterface())
     return isShapeWithLowerBounds(varIface.getShape());
   // Go through chain of fir.box converts.
-  if (auto convert = getDefiningOp<fir::ConvertOp>())
+  if (auto convert = getDefiningOp<fir::ConvertOp>()) {
     return hlfir::Entity{convert.getValue()}.mayHaveNonDefaultLowerBounds();
-  // TODO: Embox and Rebox do not have hlfir variable interface, but are
-  // easy to reason about.
+  } else if (auto rebox = getDefiningOp<fir::ReboxOp>()) {
+    // If slicing is involved, then the resulting box has
+    // default lower bounds. If there is no slicing,
+    // then the result depends on the shape operand
+    // (whether it has non default lower bounds or not).
+    return !rebox.getSlice() && isShapeWithLowerBounds(rebox.getShape());
+  } else if (auto embox = getDefiningOp<fir::EmboxOp>()) {
+    return !embox.getSlice() && isShapeWithLowerBounds(embox.getShape());
+  }
   return true;
 }
 
@@ -1646,3 +1653,30 @@ bool hlfir::designatePreservesContinuity(hlfir::DesignateOp op) {
   }
   return true;
 }
+
+bool hlfir::isSimplyContiguous(mlir::Value base, bool checkWhole) {
+  hlfir::Entity entity{base};
+  if (entity.isSimplyContiguous())
+    return true;
+
+  // Look at the definition.
+  mlir::Operation *def = base.getDefiningOp();
+  if (!def)
+    return false;
+
+  return mlir::TypeSwitch<mlir::Operation *, bool>(def)
+      .Case<fir::EmboxOp>(
+          [&](auto op) { return fir::isContiguousEmbox(op, checkWhole); })
+      .Case<fir::ReboxOp>([&](auto op) {
+        hlfir::Entity box{op.getBox()};
+        return fir::reboxPreservesContinuity(
+                   op, box.mayHaveNonDefaultLowerBounds(), checkWhole) &&
+               isSimplyContiguous(box, checkWhole);
+      })
+      .Case<fir::DeclareOp, hlfir::DeclareOp>([&](auto op) {
+        return isSimplyContiguous(op.getMemref(), checkWhole);
+      })
+      .Case<fir::ConvertOp>(
+          [&](auto op) { return isSimplyContiguous(op.getValue()); })
+      .Default([](auto &&) { return false; });
+}
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..d04306a48e922 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -1944,6 +1944,128 @@ llvm::LogicalResult fir::EmboxOp::verify() {
   return mlir::success();
 }
 
+/// Returns true if \p extent matches the extent of the \p box's
+/// dimension \p dim.
+bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
+  if (auto op = extent.getDefiningOp<fir::BoxDimsOp>())
+    if (op.getVal() == box && op.getExtent() == extent)
+      if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
+        return *dimOperand == dim;
+  return false;
+}
+
+/// Returns true if \p lb matches the lower bound of the \p box's
+/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
+/// then \p lb may be an integer constant 1.
+bool isBoxLb(mlir::Value box, std::int64_t dim, mlir::Value lb,
+             bool mayHaveNonDefaultLowerBounds = true) {
+  if (auto op = lb.getDefiningOp<fir::BoxDimsOp>()) {
+    if (op.getVal() == box && op.getLowerBound() == lb)
+      if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
+        return *dimOperand == dim;
+  } else if (!mayHaveNonDefaultLowerBounds) {
+    if (auto constantLb = fir::getIntIfConstant(lb))
+      return *constantLb == 1;
+  }
+  return false;
+}
+
+/// Returns true if \p ub matches the upper bound of the \p box's
+/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
+/// then the dimension's lower bound may be an integer constant 1.
+/// Note that the upper bound is usually a result of computation
+/// involving the lower bound and the extent, and the function
+/// tries its best to recognize the computation pattern.
+/// The conservative result 'false' does not necessarily mean
+/// that \p ub is not an actual upper bound value.
+bool isBoxUb(mlir::Value box, std::int64_t dim, mlir::Value ub,
+             bool mayHaveNonDefaultLowerBounds = true) {
+  if (auto sub1 = ub.getDefiningOp<mlir::arith::SubIOp>()) {
+    auto one = fir::getIntIfConstant(sub1.getOperand(1));
+    if (!one || *one != 1)
+      return false;
+    if (auto add = sub1.getOperand(0).getDefiningOp<mlir::arith::AddIOp>())
+      if ((isBoxLb(box, dim, add.getOperand(0)) &&
+           isBoxExtent(box, dim, add.getOperand(1))) ||
+          (isBoxLb(box, dim, add.getOperand(1)) &&
+           isBoxExtent(box, dim, add.getOperand(0))))
+        return true;
+  } else if (!mayHaveNonDefaultLowerBounds) {
+    return isBoxExtent(box, dim, ub);
+  }
+  return false;
+}
+
+/// Checks if the given \p sliceOp specifies a contiguous
+/// array slice. If \p checkWhole is true, then the check
+/// is done for all dimensions, otherwise, only for the innermost
+/// dimension.
+/// The simplest way to prove that this is an contiguous slice
+/// is to check whether the slice stride(s) is 1.
+/// For more complex cases, extra information must be provided
+/// by the caller:
+///   * \p origBox - if not null, then the source array is represented
+///     with this !fir.box value. The box is used to recognize
+///     the full dimension slices, which are specified by the triplets
+///     computed from the dimensions' lower bounds and extents.
+///   * \p mayHaveNonDefaultLowerBounds may be set to false to indicate
+///     that the source entity has default lower bounds, so the full
+///     dimension slices computations may use 1 for the lower bound.
+static bool isContiguousArraySlice(fir::SliceOp sliceOp, bool checkWhole = true,
+                                   mlir::Value origBox = nullptr,
+                                   bool mayHaveNonDefaultLowerBounds = true) {
+  if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
+    // TODO: generalize code for the triples analysis with
+    // hlfir::designatePreservesContinuity, especially when
+    // recognition of the whole dimension slices is added.
+    auto triples = sliceOp.getTriples();
+    assert((triples.size() % 3) == 0 && "invalid triples size");
+
+    // A slice with step=1 in the innermost dimension preserves
+    // the continuity of the array in the innermost dimension.
+    // If checkWhole is false, then check only the innermost slice triples.
+    std::size_t checkUpTo = checkWhole ? triples.size() : 3;
+    checkUpTo = std::min(checkUpTo, triples.size());
+    for (std::size_t i = 0; i < checkUpTo; i += 3) {
+      if (triples[i] != triples[i + 1]) {
+        // This is a section of the dimension. Only allow it
+        // to be the first triple, if the source of the slice
+        // is a boxed array. If it is a raw pointer, then
+        // the result will still be contiguous, as long as
+        // the strides are all ones.
+        // When origBox is not null, we must prove that the triple
+        // covers the whole dimension and the stride is one,
+        // before claiming contiguity for this dimension.
+        if (i != 0 && origBox) {
+          std::int64_t dim = i / 3;
+          if (!isBoxLb(origBox, dim, triples[i],
+                       mayHaveNonDefaultLowerBounds) ||
+              !isBoxUb(origBox, dim, triples[i + 1],
+                       mayHaveNonDefaultLowerBounds))
+            return false;
+        }
+        auto constantStep = fir::getIntIfConstant(triples[i + 2]);
+        if (!constantStep || *constantStep != 1)
+          return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+bool fir::isContiguousEmbox(fir::EmboxOp embox, bool checkWhole) {
+  auto sliceArg = embox.getSlice();
+  if (!sliceArg)
+    return true;
+
+  if (auto sliceOp =
+          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
+    return isContiguousArraySlice(sliceOp, checkWhole);
+
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // EmboxCharOp
 //===----------------------------------------------------------------------===//
@@ -4794,7 +4916,9 @@ mlir::Type fir::applyPathToType(mlir::Type eleTy, mlir::ValueRange path) {
   return eleTy;
 }
 
-bool fir::reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole) {
+bool fir::reboxPreservesContinuity(fir::ReboxOp rebox,
+                                   bool mayHaveNonDefaultLowerBounds,
+                                   bool checkWhole) {
   // If slicing is not involved, then the rebox does not affect
   // the continuity of the array.
   auto sliceArg = rebox.getSlice();
@@ -4802,33 +4926,10 @@ bool fir::reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole) {
     return true;
 
   if (auto sliceOp =
-          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp())) {
-    if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
-      // TODO: generalize code for the triples analysis with
-      // hlfir::designatePreservesContinuity, especially when
-      // recognition of the whole dimension slices is added.
-      auto triples = sliceOp.getTriples();
-      assert((triples.size() % 3) == 0 && "invalid triples size");
-
-      // A slice with step=1 in the innermost dimension preserves
-      // the continuity of the array in the innermost dimension.
-      // If checkWhole is false, then check only the innermost slice triples.
-      std::size_t checkUpTo = checkWhole ? triples.size() : 3;
-      checkUpTo = std::min(checkUpTo, triples.size());
-      for (std::size_t i = 0; i < checkUpTo; i += 3) {
-        if (triples[i] != triples[i + 1]) {
-          // This is a section of the dimension. Only allow it
-          // to be the first triple.
-          if (i != 0)
-            return false;
-          auto constantStep = fir::getIntIfConstant(triples[i + 2]);
-          if (!constantStep || *constantStep != 1)
-            return false;
-        }
-      }
-      return true;
-    }
-  }
+          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
+    return isContiguousArraySlice(sliceOp, checkWhole, rebox.getBox(),
+                                  mayHaveNonDefaultLowerBounds);
+
   return false;
 }
 
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 333bfc8b7aed8..ca8e820608688 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -207,6 +207,10 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
   pm.addPass(fir::createPolymorphicOpConversion());
   pm.addPass(fir::createAssumedRankOpConversion());
 
+  // Optimize redundant array repacking operations,
+  // if the source is known to be contiguous.
+  if (pc.OptLevel.isOptimizingForSpeed())
+    pm.addPass(fir::createOptimizeArrayRepacking());
   pm.addPass(fir::createLowerRepackArraysPass());
   // Expand FIR operations that may use SCF dialect for their
   // implementation. This is a mandatory pass.
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index 846d6c64dbd04..a8812e08c1ccd 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -34,6 +34,7 @@ add_flang_library(FIRTransforms
   SetRuntimeCallAttributes.cpp
   GenRuntimeCallsForTest.cpp
   SimplifyFIROperations.cpp
+  OptimizeArrayRepacking.cpp
 
   DEPENDS
   CUFAttrs
diff --git a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
index 50e7ee5599ab1..056bdf63d914f 100644
--- a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
@@ -214,7 +214,9 @@ static mlir::Value unwrapPassThroughOps(mlir::Value val) {
 /// of the value, otherwise return the value
 static mlir::Value unwrapReboxOp(mlir::Value val) {
   while (fir::ReboxOp rebox = val.getDefiningOp<fir::ReboxOp>()) {
-    if (!fir::reboxPreservesContinuity(rebox, /*checkWhole=*/false)) {
+    if (!fir::reboxPreservesContinuity(rebox,
+                                       /*mayHaveNonDefaultLowerBounds=*/true,
+                                       /*checkWhole=*/false)) {
       LLVM_DEBUG(llvm::dbgs() << "REBOX may produce non-contiguous array: "
                               << rebox << '\n');
       break;
diff --git a/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
new file mode 100644
index 0000000000000..1688f2887a57a
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
@@ -0,0 +1,90 @@
+//===- OptimizeArrayRepacking.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+/// \file
+/// This pass removes redundant fir.pack_array operations, if it can prove
+/// that the source array is contiguous. In this case, it relink all uses
+/// of fir.pack_array result to the source. If such a rewrite happens,
+/// it may turn the using fir.unpack_array operation into one with the same
+/// temp and original operands - these are also removed as redundant.
+//===----------------------------------------------------------------------===//
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Support/Utils.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+namespace fir {
+#define GEN_PASS_DEF_OPTIMIZEARRAYREPACKING
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+} // namespace fir
+
+namespace {
+class OptimizeArrayRepackingPass
+    : public fir::impl::OptimizeArrayRepackingBase<OptimizeArrayRepackingPass> {
+public:
+  void runOnOperation() override;
+};
+
+/// Relinks all uses of redundant fir.pack_array to the source.
+class PackingOfContiguous : public mlir::OpRewritePattern<fir::PackArrayOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  mlir::LogicalResult matchAndRewrite(fir::PackArrayOp,
+                                      mlir::PatternRewriter &) const override;
+};
+
+/// Erases fir.unpack_array with have the matching temp and original
+/// operands.
+class NoopUnpacking : public mlir::OpRewritePattern<fir::UnpackArrayOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  mlir::LogicalResult matchAndRewrite(fir::UnpackArrayOp,
+                                      mlir::PatternRewriter &) const override;
+};
+} // namespace
+
+mlir::LogicalResult
+PackingOfContiguous::matchAndRewrite(fir::PackArrayOp op,
+                                     mlir::PatternRewriter &rewriter...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/147881

From flang-commits at lists.llvm.org  Wed Jul  9 21:17:22 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Wed, 09 Jul 2025 21:17:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Update implicit data transfer
 for device component (PR #147882)
Message-ID: <llvm.org/llvm/llvm-project/pull/147882@github.com>


https://github.com/clementval created https://github.com/llvm/llvm-project/pull/147882

Update the detection of implicit data transfer when a device resident allocatable derived-type component is involved  and remove the TODOs.

>From f45edd9dbf728cd3174f9890f3541e559e2bdd92 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Wed, 9 Jul 2025 21:15:38 -0700
Subject: [PATCH] [flang][cuda] Update implicit data transfer for device
 component

---
 flang/include/flang/Evaluate/tools.h         | 19 +--------------
 flang/lib/Evaluate/tools.cpp                 | 25 ++++++++++++++++++++
 flang/lib/Lower/Bridge.cpp                   |  2 --
 flang/test/Lower/CUDA/cuda-data-transfer.cuf | 17 +++++++++++++
 4 files changed, 43 insertions(+), 20 deletions(-)

diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 18c244f6f450f..96ed86f468350 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1359,24 +1359,7 @@ inline bool IsCUDADataTransfer(const A &lhs, const B &rhs) {
 
 /// Check if the expression is a mix of host and device variables that require
 /// implicit data transfer.
-inline bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
-  unsigned hostSymbols{0};
-  unsigned deviceSymbols{0};
-  for (const Symbol &sym : CollectCudaSymbols(expr)) {
-    if (IsCUDADeviceSymbol(sym)) {
-      ++deviceSymbols;
-    } else {
-      if (sym.owner().IsDerivedType()) {
-        if (IsCUDADeviceSymbol(sym.owner().GetSymbol()->GetUltimate())) {
-          ++deviceSymbols;
-        }
-      }
-      ++hostSymbols;
-    }
-  }
-  bool hasConstant{HasConstant(expr)};
-  return (hasConstant || (hostSymbols > 0)) && deviceSymbols > 0;
-}
+bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr);
 
 // Checks whether the symbol on the LHS is present in the RHS expression.
 bool CheckForSymbolMatch(const Expr<SomeType> *lhs, const Expr<SomeType> *rhs);
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 6a57d87a30e93..3d9f06308d8c1 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1154,6 +1154,31 @@ template semantics::UnorderedSymbolSet CollectCudaSymbols(
 template semantics::UnorderedSymbolSet CollectCudaSymbols(
     const Expr<SubscriptInteger> &);
 
+bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
+  semantics::UnorderedSymbolSet hostSymbols;
+  semantics::UnorderedSymbolSet deviceSymbols;
+
+  SymbolVector symbols{GetSymbolVector(expr)};
+  std::reverse(symbols.begin(), symbols.end());
+  bool skipNext{false};
+  for (const Symbol &sym : symbols) {
+    bool isComponent{sym.owner().IsDerivedType()};
+    bool skipComponent{false};
+    if (!skipNext) {
+      if (IsCUDADeviceSymbol(sym)) {
+        deviceSymbols.insert(sym);
+      } else if (isComponent) {
+        skipComponent = true; // Component is not device. Look on the base.
+      } else {
+        hostSymbols.insert(sym);
+      }
+    }
+    skipNext = isComponent && !skipComponent;
+  }
+  bool hasConstant{HasConstant(expr)};
+  return (hasConstant || (hostSymbols.size() > 0)) && deviceSymbols.size() > 0;
+}
+
 // HasVectorSubscript()
 struct HasVectorSubscriptHelper
     : public AnyTraverse<HasVectorSubscriptHelper, bool,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 696473605a4e0..ff26aa8e33649 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          if (sym.owner().IsDerivedType() && IsAllocatable(sym.GetUltimate()))
-            TODO(loc, "Device resident allocatable derived-type component");
           // TODO: This should probably being checked in semantic and give a
           // proper error.
           assert(
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index 68a0202f951fe..3a9b55996d9b1 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -7,6 +7,10 @@ module mod1
     integer :: i
   end type
 
+  type :: t2
+    integer, device, allocatable, dimension(:) :: x
+  end type
+
   integer, device, dimension(11:20) :: cdev
 
 contains
@@ -419,3 +423,16 @@ end subroutine
 ! CHECK: fir.do_concurrent.loop
 ! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
+
+
+subroutine sub22()
+  use mod1
+  type(t2) :: a
+  integer :: b(100)
+  allocate(a%x(100))
+
+  b = a%x
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub22()
+! CHECK: cuf.data_transfer


From flang-commits at lists.llvm.org  Wed Jul  9 21:17:53 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 21:17:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Update implicit data transfer
 for device component (PR #147882)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147882@github.com>
Message-ID: <686f3ef1.050a0220.34432a.0fe7@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Valentin Clement (バレンタイン クレメン) (clementval)

<details>
<summary>Changes</summary>

Update the detection of implicit data transfer when a device resident allocatable derived-type component is involved  and remove the TODOs.

---
Full diff: https://github.com/llvm/llvm-project/pull/147882.diff


4 Files Affected:

- (modified) flang/include/flang/Evaluate/tools.h (+1-18) 
- (modified) flang/lib/Evaluate/tools.cpp (+25) 
- (modified) flang/lib/Lower/Bridge.cpp (-2) 
- (modified) flang/test/Lower/CUDA/cuda-data-transfer.cuf (+17) 


``````````diff
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 18c244f6f450f..96ed86f468350 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1359,24 +1359,7 @@ inline bool IsCUDADataTransfer(const A &lhs, const B &rhs) {
 
 /// Check if the expression is a mix of host and device variables that require
 /// implicit data transfer.
-inline bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
-  unsigned hostSymbols{0};
-  unsigned deviceSymbols{0};
-  for (const Symbol &sym : CollectCudaSymbols(expr)) {
-    if (IsCUDADeviceSymbol(sym)) {
-      ++deviceSymbols;
-    } else {
-      if (sym.owner().IsDerivedType()) {
-        if (IsCUDADeviceSymbol(sym.owner().GetSymbol()->GetUltimate())) {
-          ++deviceSymbols;
-        }
-      }
-      ++hostSymbols;
-    }
-  }
-  bool hasConstant{HasConstant(expr)};
-  return (hasConstant || (hostSymbols > 0)) && deviceSymbols > 0;
-}
+bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr);
 
 // Checks whether the symbol on the LHS is present in the RHS expression.
 bool CheckForSymbolMatch(const Expr<SomeType> *lhs, const Expr<SomeType> *rhs);
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 6a57d87a30e93..3d9f06308d8c1 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1154,6 +1154,31 @@ template semantics::UnorderedSymbolSet CollectCudaSymbols(
 template semantics::UnorderedSymbolSet CollectCudaSymbols(
     const Expr<SubscriptInteger> &);
 
+bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
+  semantics::UnorderedSymbolSet hostSymbols;
+  semantics::UnorderedSymbolSet deviceSymbols;
+
+  SymbolVector symbols{GetSymbolVector(expr)};
+  std::reverse(symbols.begin(), symbols.end());
+  bool skipNext{false};
+  for (const Symbol &sym : symbols) {
+    bool isComponent{sym.owner().IsDerivedType()};
+    bool skipComponent{false};
+    if (!skipNext) {
+      if (IsCUDADeviceSymbol(sym)) {
+        deviceSymbols.insert(sym);
+      } else if (isComponent) {
+        skipComponent = true; // Component is not device. Look on the base.
+      } else {
+        hostSymbols.insert(sym);
+      }
+    }
+    skipNext = isComponent && !skipComponent;
+  }
+  bool hasConstant{HasConstant(expr)};
+  return (hasConstant || (hostSymbols.size() > 0)) && deviceSymbols.size() > 0;
+}
+
 // HasVectorSubscript()
 struct HasVectorSubscriptHelper
     : public AnyTraverse<HasVectorSubscriptHelper, bool,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 696473605a4e0..ff26aa8e33649 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          if (sym.owner().IsDerivedType() && IsAllocatable(sym.GetUltimate()))
-            TODO(loc, "Device resident allocatable derived-type component");
           // TODO: This should probably being checked in semantic and give a
           // proper error.
           assert(
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index 68a0202f951fe..3a9b55996d9b1 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -7,6 +7,10 @@ module mod1
     integer :: i
   end type
 
+  type :: t2
+    integer, device, allocatable, dimension(:) :: x
+  end type
+
   integer, device, dimension(11:20) :: cdev
 
 contains
@@ -419,3 +423,16 @@ end subroutine
 ! CHECK: fir.do_concurrent.loop
 ! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
+
+
+subroutine sub22()
+  use mod1
+  type(t2) :: a
+  integer :: b(100)
+  allocate(a%x(100))
+
+  b = a%x
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub22()
+! CHECK: cuf.data_transfer

``````````

</details>


https://github.com/llvm/llvm-project/pull/147882

From flang-commits at lists.llvm.org  Wed Jul  9 21:17:54 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 21:17:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Update implicit data transfer
 for device component (PR #147882)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147882@github.com>
Message-ID: <686f3ef2.630a0220.c1543.124e@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Valentin Clement (バレンタイン クレメン) (clementval)

<details>
<summary>Changes</summary>

Update the detection of implicit data transfer when a device resident allocatable derived-type component is involved  and remove the TODOs.

---
Full diff: https://github.com/llvm/llvm-project/pull/147882.diff


4 Files Affected:

- (modified) flang/include/flang/Evaluate/tools.h (+1-18) 
- (modified) flang/lib/Evaluate/tools.cpp (+25) 
- (modified) flang/lib/Lower/Bridge.cpp (-2) 
- (modified) flang/test/Lower/CUDA/cuda-data-transfer.cuf (+17) 


``````````diff
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 18c244f6f450f..96ed86f468350 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1359,24 +1359,7 @@ inline bool IsCUDADataTransfer(const A &lhs, const B &rhs) {
 
 /// Check if the expression is a mix of host and device variables that require
 /// implicit data transfer.
-inline bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
-  unsigned hostSymbols{0};
-  unsigned deviceSymbols{0};
-  for (const Symbol &sym : CollectCudaSymbols(expr)) {
-    if (IsCUDADeviceSymbol(sym)) {
-      ++deviceSymbols;
-    } else {
-      if (sym.owner().IsDerivedType()) {
-        if (IsCUDADeviceSymbol(sym.owner().GetSymbol()->GetUltimate())) {
-          ++deviceSymbols;
-        }
-      }
-      ++hostSymbols;
-    }
-  }
-  bool hasConstant{HasConstant(expr)};
-  return (hasConstant || (hostSymbols > 0)) && deviceSymbols > 0;
-}
+bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr);
 
 // Checks whether the symbol on the LHS is present in the RHS expression.
 bool CheckForSymbolMatch(const Expr<SomeType> *lhs, const Expr<SomeType> *rhs);
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 6a57d87a30e93..3d9f06308d8c1 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1154,6 +1154,31 @@ template semantics::UnorderedSymbolSet CollectCudaSymbols(
 template semantics::UnorderedSymbolSet CollectCudaSymbols(
     const Expr<SubscriptInteger> &);
 
+bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
+  semantics::UnorderedSymbolSet hostSymbols;
+  semantics::UnorderedSymbolSet deviceSymbols;
+
+  SymbolVector symbols{GetSymbolVector(expr)};
+  std::reverse(symbols.begin(), symbols.end());
+  bool skipNext{false};
+  for (const Symbol &sym : symbols) {
+    bool isComponent{sym.owner().IsDerivedType()};
+    bool skipComponent{false};
+    if (!skipNext) {
+      if (IsCUDADeviceSymbol(sym)) {
+        deviceSymbols.insert(sym);
+      } else if (isComponent) {
+        skipComponent = true; // Component is not device. Look on the base.
+      } else {
+        hostSymbols.insert(sym);
+      }
+    }
+    skipNext = isComponent && !skipComponent;
+  }
+  bool hasConstant{HasConstant(expr)};
+  return (hasConstant || (hostSymbols.size() > 0)) && deviceSymbols.size() > 0;
+}
+
 // HasVectorSubscript()
 struct HasVectorSubscriptHelper
     : public AnyTraverse<HasVectorSubscriptHelper, bool,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 696473605a4e0..ff26aa8e33649 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          if (sym.owner().IsDerivedType() && IsAllocatable(sym.GetUltimate()))
-            TODO(loc, "Device resident allocatable derived-type component");
           // TODO: This should probably being checked in semantic and give a
           // proper error.
           assert(
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index 68a0202f951fe..3a9b55996d9b1 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -7,6 +7,10 @@ module mod1
     integer :: i
   end type
 
+  type :: t2
+    integer, device, allocatable, dimension(:) :: x
+  end type
+
   integer, device, dimension(11:20) :: cdev
 
 contains
@@ -419,3 +423,16 @@ end subroutine
 ! CHECK: fir.do_concurrent.loop
 ! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
+
+
+subroutine sub22()
+  use mod1
+  type(t2) :: a
+  integer :: b(100)
+  allocate(a%x(100))
+
+  b = a%x
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub22()
+! CHECK: cuf.data_transfer

``````````

</details>


https://github.com/llvm/llvm-project/pull/147882

From flang-commits at lists.llvm.org  Wed Jul  9 21:04:12 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Wed, 09 Jul 2025 21:04:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
Message-ID: <llvm.org/llvm/llvm-project/pull/147881@github.com>


https://github.com/vzakhari created https://github.com/llvm/llvm-project/pull/147881

This patch allows optimizing redundant array repacking, when
the source array is statically known to be contiguous.
This is part of the implementation plan for the array repacking
feature, though, it does not affect any real life use case
as long as FIR inlining is not a thing. I experimented with
simple cases of FIR inling using `-inline-all`, and I recorded
these cases in optimize-array-repacking.fir tests.


>From 1064e297fbf1ff6142a5c9bcc856474028863b42 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Wed, 9 Jul 2025 17:21:48 -0700
Subject: [PATCH] [flang] Optimize redundant array repacking.

This patch allows optimizing redundant array repacking, when
the source array is statically known to be contiguous.
This is part of the implementation plan for the array repacking
feature, though, it does not affect any real life use case
as long as FIR inlining is not a thing. I experimented with
simple cases of FIR inling using `-inline-all`, and I recorded
these cases in optimize-array-repacking.fir tests.
---
 .../flang/Optimizer/Builder/HLFIRTools.h      |   8 +
 .../flang/Optimizer/Dialect/FIROpsSupport.h   |  14 +-
 .../flang/Optimizer/Transforms/Passes.h       |  34 +-
 .../flang/Optimizer/Transforms/Passes.td      |  10 +
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |  40 +-
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 157 ++++-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |   4 +
 flang/lib/Optimizer/Transforms/CMakeLists.txt |   1 +
 .../Optimizer/Transforms/LoopVersioning.cpp   |   4 +-
 .../Transforms/OptimizeArrayRepacking.cpp     |  90 +++
 flang/test/Driver/bbc-mlir-pass-pipeline.f90  |   2 +
 flang/test/Driver/mlir-pass-pipeline.f90      |   2 +
 flang/test/Fir/basic-program.fir              |   2 +
 .../simplify-hlfir-intrinsics-cshift.fir      |  36 +-
 .../simplify-hlfir-intrinsics-matmul.fir      | 107 +--
 .../Transforms/optimize-array-repacking.fir   | 660 ++++++++++++++++++
 16 files changed, 982 insertions(+), 189 deletions(-)
 create mode 100644 flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
 create mode 100644 flang/test/Transforms/optimize-array-repacking.fir

diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 2cbad6e268a38..49dfc85dc76e6 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -542,6 +542,14 @@ Entity gen1DSection(mlir::Location loc, fir::FirOpBuilder &builder,
 /// contiguous.
 bool designatePreservesContinuity(hlfir::DesignateOp op);
 
+/// Return true iff the given \p base desribes an object
+/// that is contiguous. If \p checkWhole is true, then
+/// the object must be contiguous in all dimensions,
+/// otherwise, it must be contiguous in the innermost dimension.
+/// This function is an extension of hlfir::Entity::isSimplyContiguous(),
+/// and it can be used on pure FIR representation as well as on HLFIR.
+bool isSimplyContiguous(mlir::Value base, bool checkWhole = true);
+
 } // namespace hlfir
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_HLFIRTOOLS_H
diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index 0a2337be7455e..ae471eb0d2e04 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -238,7 +238,19 @@ std::optional<int64_t> getAllocaByteSize(fir::AllocaOp alloca,
 /// When \p checkWhole is false, then the checking is only done
 /// for continuity in the innermost dimension, otherwise,
 /// the checking is done for continuity of the whole result of rebox.
-bool reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole = true);
+/// The caller may specify \p mayHaveNonDefaultLowerBounds, if it is known,
+/// to allow better handling of the rebox operations representing
+/// full array slices.
+bool reboxPreservesContinuity(fir::ReboxOp rebox,
+                              bool mayHaveNonDefaultLowerBounds = true,
+                              bool checkWhole = true);
+
+/// Return true, if \p embox operation produces a contiguous
+/// entity.
+/// When \p checkWhole is false, then the checking is only done
+/// for continuity in the innermost dimension, otherwise,
+/// the checking is done for continuity of the whole result of embox
+bool isContiguousEmbox(fir::EmboxOp embox, bool checkWhole = true);
 
 } // namespace fir
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index dc8a5b9141ad2..6f5dff4687cbb 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -31,39 +31,7 @@ namespace fir {
 // Passes defined in Passes.td
 //===----------------------------------------------------------------------===//
 
-#define GEN_PASS_DECL_ABSTRACTRESULTOPT
-#define GEN_PASS_DECL_AFFINEDIALECTPROMOTION
-#define GEN_PASS_DECL_AFFINEDIALECTDEMOTION
-#define GEN_PASS_DECL_ANNOTATECONSTANTOPERANDS
-#define GEN_PASS_DECL_ARRAYVALUECOPY
-#define GEN_PASS_DECL_ASSUMEDRANKOPCONVERSION
-#define GEN_PASS_DECL_CHARACTERCONVERSION
-#define GEN_PASS_DECL_CFGCONVERSION
-#define GEN_PASS_DECL_CUFADDCONSTRUCTOR
-#define GEN_PASS_DECL_CUFDEVICEGLOBAL
-#define GEN_PASS_DECL_CUFGPUTOLLVMCONVERSION
-#define GEN_PASS_DECL_CUFOPCONVERSION
-#define GEN_PASS_DECL_CUFCOMPUTESHAREDMEMORYOFFSETSANDSIZE
-#define GEN_PASS_DECL_EXTERNALNAMECONVERSION
-#define GEN_PASS_DECL_MEMREFDATAFLOWOPT
-#define GEN_PASS_DECL_SIMPLIFYINTRINSICS
-#define GEN_PASS_DECL_MEMORYALLOCATIONOPT
-#define GEN_PASS_DECL_SIMPLIFYREGIONLITE
-#define GEN_PASS_DECL_ALGEBRAICSIMPLIFICATION
-#define GEN_PASS_DECL_POLYMORPHICOPCONVERSION
-#define GEN_PASS_DECL_OPENACCDATAOPERANDCONVERSION
-#define GEN_PASS_DECL_ADDDEBUGINFO
-#define GEN_PASS_DECL_STACKARRAYS
-#define GEN_PASS_DECL_STACKRECLAIM
-#define GEN_PASS_DECL_LOOPVERSIONING
-#define GEN_PASS_DECL_ADDALIASTAGS
-#define GEN_PASS_DECL_VSCALEATTR
-#define GEN_PASS_DECL_FUNCTIONATTR
-#define GEN_PASS_DECL_CONSTANTARGUMENTGLOBALISATIONOPT
-#define GEN_PASS_DECL_COMPILERGENERATEDNAMESCONVERSION
-#define GEN_PASS_DECL_SETRUNTIMECALLATTRIBUTES
-#define GEN_PASS_DECL_GENRUNTIMECALLSFORTEST
-#define GEN_PASS_DECL_SIMPLIFYFIROPERATIONS
+#define GEN_PASS_DECL
 
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 34842f9785942..b230f60b4c59e 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -551,4 +551,14 @@ def SimplifyFIROperations : Pass<"simplify-fir-operations", "mlir::ModuleOp"> {
       "Prefer expanding without using Fortran runtime calls.">];
 }
 
+def OptimizeArrayRepacking
+    : Pass<"optimize-array-repacking", "mlir::func::FuncOp"> {
+  let summary = "Optimizes redundant array repacking operations";
+  let description = [{
+    If the source of fir.pack_array is known to be contiguous,
+    then this pass erases such operations. The corresponding
+    fir.unpack_array operations are also removed.
+  }];
+}
+
 #endif // FLANG_OPTIMIZER_TRANSFORMS_PASSES
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index b54cccbef14fa..e59a6bf2bf224 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -212,10 +212,17 @@ bool hlfir::Entity::mayHaveNonDefaultLowerBounds() const {
   if (auto varIface = getIfVariableInterface())
     return isShapeWithLowerBounds(varIface.getShape());
   // Go through chain of fir.box converts.
-  if (auto convert = getDefiningOp<fir::ConvertOp>())
+  if (auto convert = getDefiningOp<fir::ConvertOp>()) {
     return hlfir::Entity{convert.getValue()}.mayHaveNonDefaultLowerBounds();
-  // TODO: Embox and Rebox do not have hlfir variable interface, but are
-  // easy to reason about.
+  } else if (auto rebox = getDefiningOp<fir::ReboxOp>()) {
+    // If slicing is involved, then the resulting box has
+    // default lower bounds. If there is no slicing,
+    // then the result depends on the shape operand
+    // (whether it has non default lower bounds or not).
+    return !rebox.getSlice() && isShapeWithLowerBounds(rebox.getShape());
+  } else if (auto embox = getDefiningOp<fir::EmboxOp>()) {
+    return !embox.getSlice() && isShapeWithLowerBounds(embox.getShape());
+  }
   return true;
 }
 
@@ -1646,3 +1653,30 @@ bool hlfir::designatePreservesContinuity(hlfir::DesignateOp op) {
   }
   return true;
 }
+
+bool hlfir::isSimplyContiguous(mlir::Value base, bool checkWhole) {
+  hlfir::Entity entity{base};
+  if (entity.isSimplyContiguous())
+    return true;
+
+  // Look at the definition.
+  mlir::Operation *def = base.getDefiningOp();
+  if (!def)
+    return false;
+
+  return mlir::TypeSwitch<mlir::Operation *, bool>(def)
+      .Case<fir::EmboxOp>(
+          [&](auto op) { return fir::isContiguousEmbox(op, checkWhole); })
+      .Case<fir::ReboxOp>([&](auto op) {
+        hlfir::Entity box{op.getBox()};
+        return fir::reboxPreservesContinuity(
+                   op, box.mayHaveNonDefaultLowerBounds(), checkWhole) &&
+               isSimplyContiguous(box, checkWhole);
+      })
+      .Case<fir::DeclareOp, hlfir::DeclareOp>([&](auto op) {
+        return isSimplyContiguous(op.getMemref(), checkWhole);
+      })
+      .Case<fir::ConvertOp>(
+          [&](auto op) { return isSimplyContiguous(op.getValue()); })
+      .Default([](auto &&) { return false; });
+}
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..d04306a48e922 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -1944,6 +1944,128 @@ llvm::LogicalResult fir::EmboxOp::verify() {
   return mlir::success();
 }
 
+/// Returns true if \p extent matches the extent of the \p box's
+/// dimension \p dim.
+bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
+  if (auto op = extent.getDefiningOp<fir::BoxDimsOp>())
+    if (op.getVal() == box && op.getExtent() == extent)
+      if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
+        return *dimOperand == dim;
+  return false;
+}
+
+/// Returns true if \p lb matches the lower bound of the \p box's
+/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
+/// then \p lb may be an integer constant 1.
+bool isBoxLb(mlir::Value box, std::int64_t dim, mlir::Value lb,
+             bool mayHaveNonDefaultLowerBounds = true) {
+  if (auto op = lb.getDefiningOp<fir::BoxDimsOp>()) {
+    if (op.getVal() == box && op.getLowerBound() == lb)
+      if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
+        return *dimOperand == dim;
+  } else if (!mayHaveNonDefaultLowerBounds) {
+    if (auto constantLb = fir::getIntIfConstant(lb))
+      return *constantLb == 1;
+  }
+  return false;
+}
+
+/// Returns true if \p ub matches the upper bound of the \p box's
+/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
+/// then the dimension's lower bound may be an integer constant 1.
+/// Note that the upper bound is usually a result of computation
+/// involving the lower bound and the extent, and the function
+/// tries its best to recognize the computation pattern.
+/// The conservative result 'false' does not necessarily mean
+/// that \p ub is not an actual upper bound value.
+bool isBoxUb(mlir::Value box, std::int64_t dim, mlir::Value ub,
+             bool mayHaveNonDefaultLowerBounds = true) {
+  if (auto sub1 = ub.getDefiningOp<mlir::arith::SubIOp>()) {
+    auto one = fir::getIntIfConstant(sub1.getOperand(1));
+    if (!one || *one != 1)
+      return false;
+    if (auto add = sub1.getOperand(0).getDefiningOp<mlir::arith::AddIOp>())
+      if ((isBoxLb(box, dim, add.getOperand(0)) &&
+           isBoxExtent(box, dim, add.getOperand(1))) ||
+          (isBoxLb(box, dim, add.getOperand(1)) &&
+           isBoxExtent(box, dim, add.getOperand(0))))
+        return true;
+  } else if (!mayHaveNonDefaultLowerBounds) {
+    return isBoxExtent(box, dim, ub);
+  }
+  return false;
+}
+
+/// Checks if the given \p sliceOp specifies a contiguous
+/// array slice. If \p checkWhole is true, then the check
+/// is done for all dimensions, otherwise, only for the innermost
+/// dimension.
+/// The simplest way to prove that this is an contiguous slice
+/// is to check whether the slice stride(s) is 1.
+/// For more complex cases, extra information must be provided
+/// by the caller:
+///   * \p origBox - if not null, then the source array is represented
+///     with this !fir.box value. The box is used to recognize
+///     the full dimension slices, which are specified by the triplets
+///     computed from the dimensions' lower bounds and extents.
+///   * \p mayHaveNonDefaultLowerBounds may be set to false to indicate
+///     that the source entity has default lower bounds, so the full
+///     dimension slices computations may use 1 for the lower bound.
+static bool isContiguousArraySlice(fir::SliceOp sliceOp, bool checkWhole = true,
+                                   mlir::Value origBox = nullptr,
+                                   bool mayHaveNonDefaultLowerBounds = true) {
+  if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
+    // TODO: generalize code for the triples analysis with
+    // hlfir::designatePreservesContinuity, especially when
+    // recognition of the whole dimension slices is added.
+    auto triples = sliceOp.getTriples();
+    assert((triples.size() % 3) == 0 && "invalid triples size");
+
+    // A slice with step=1 in the innermost dimension preserves
+    // the continuity of the array in the innermost dimension.
+    // If checkWhole is false, then check only the innermost slice triples.
+    std::size_t checkUpTo = checkWhole ? triples.size() : 3;
+    checkUpTo = std::min(checkUpTo, triples.size());
+    for (std::size_t i = 0; i < checkUpTo; i += 3) {
+      if (triples[i] != triples[i + 1]) {
+        // This is a section of the dimension. Only allow it
+        // to be the first triple, if the source of the slice
+        // is a boxed array. If it is a raw pointer, then
+        // the result will still be contiguous, as long as
+        // the strides are all ones.
+        // When origBox is not null, we must prove that the triple
+        // covers the whole dimension and the stride is one,
+        // before claiming contiguity for this dimension.
+        if (i != 0 && origBox) {
+          std::int64_t dim = i / 3;
+          if (!isBoxLb(origBox, dim, triples[i],
+                       mayHaveNonDefaultLowerBounds) ||
+              !isBoxUb(origBox, dim, triples[i + 1],
+                       mayHaveNonDefaultLowerBounds))
+            return false;
+        }
+        auto constantStep = fir::getIntIfConstant(triples[i + 2]);
+        if (!constantStep || *constantStep != 1)
+          return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+bool fir::isContiguousEmbox(fir::EmboxOp embox, bool checkWhole) {
+  auto sliceArg = embox.getSlice();
+  if (!sliceArg)
+    return true;
+
+  if (auto sliceOp =
+          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
+    return isContiguousArraySlice(sliceOp, checkWhole);
+
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // EmboxCharOp
 //===----------------------------------------------------------------------===//
@@ -4794,7 +4916,9 @@ mlir::Type fir::applyPathToType(mlir::Type eleTy, mlir::ValueRange path) {
   return eleTy;
 }
 
-bool fir::reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole) {
+bool fir::reboxPreservesContinuity(fir::ReboxOp rebox,
+                                   bool mayHaveNonDefaultLowerBounds,
+                                   bool checkWhole) {
   // If slicing is not involved, then the rebox does not affect
   // the continuity of the array.
   auto sliceArg = rebox.getSlice();
@@ -4802,33 +4926,10 @@ bool fir::reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole) {
     return true;
 
   if (auto sliceOp =
-          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp())) {
-    if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
-      // TODO: generalize code for the triples analysis with
-      // hlfir::designatePreservesContinuity, especially when
-      // recognition of the whole dimension slices is added.
-      auto triples = sliceOp.getTriples();
-      assert((triples.size() % 3) == 0 && "invalid triples size");
-
-      // A slice with step=1 in the innermost dimension preserves
-      // the continuity of the array in the innermost dimension.
-      // If checkWhole is false, then check only the innermost slice triples.
-      std::size_t checkUpTo = checkWhole ? triples.size() : 3;
-      checkUpTo = std::min(checkUpTo, triples.size());
-      for (std::size_t i = 0; i < checkUpTo; i += 3) {
-        if (triples[i] != triples[i + 1]) {
-          // This is a section of the dimension. Only allow it
-          // to be the first triple.
-          if (i != 0)
-            return false;
-          auto constantStep = fir::getIntIfConstant(triples[i + 2]);
-          if (!constantStep || *constantStep != 1)
-            return false;
-        }
-      }
-      return true;
-    }
-  }
+          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
+    return isContiguousArraySlice(sliceOp, checkWhole, rebox.getBox(),
+                                  mayHaveNonDefaultLowerBounds);
+
   return false;
 }
 
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 333bfc8b7aed8..ca8e820608688 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -207,6 +207,10 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
   pm.addPass(fir::createPolymorphicOpConversion());
   pm.addPass(fir::createAssumedRankOpConversion());
 
+  // Optimize redundant array repacking operations,
+  // if the source is known to be contiguous.
+  if (pc.OptLevel.isOptimizingForSpeed())
+    pm.addPass(fir::createOptimizeArrayRepacking());
   pm.addPass(fir::createLowerRepackArraysPass());
   // Expand FIR operations that may use SCF dialect for their
   // implementation. This is a mandatory pass.
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index 846d6c64dbd04..a8812e08c1ccd 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -34,6 +34,7 @@ add_flang_library(FIRTransforms
   SetRuntimeCallAttributes.cpp
   GenRuntimeCallsForTest.cpp
   SimplifyFIROperations.cpp
+  OptimizeArrayRepacking.cpp
 
   DEPENDS
   CUFAttrs
diff --git a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
index 50e7ee5599ab1..056bdf63d914f 100644
--- a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
@@ -214,7 +214,9 @@ static mlir::Value unwrapPassThroughOps(mlir::Value val) {
 /// of the value, otherwise return the value
 static mlir::Value unwrapReboxOp(mlir::Value val) {
   while (fir::ReboxOp rebox = val.getDefiningOp<fir::ReboxOp>()) {
-    if (!fir::reboxPreservesContinuity(rebox, /*checkWhole=*/false)) {
+    if (!fir::reboxPreservesContinuity(rebox,
+                                       /*mayHaveNonDefaultLowerBounds=*/true,
+                                       /*checkWhole=*/false)) {
       LLVM_DEBUG(llvm::dbgs() << "REBOX may produce non-contiguous array: "
                               << rebox << '\n');
       break;
diff --git a/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
new file mode 100644
index 0000000000000..1688f2887a57a
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
@@ -0,0 +1,90 @@
+//===- OptimizeArrayRepacking.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+/// \file
+/// This pass removes redundant fir.pack_array operations, if it can prove
+/// that the source array is contiguous. In this case, it relink all uses
+/// of fir.pack_array result to the source. If such a rewrite happens,
+/// it may turn the using fir.unpack_array operation into one with the same
+/// temp and original operands - these are also removed as redundant.
+//===----------------------------------------------------------------------===//
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Support/Utils.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+namespace fir {
+#define GEN_PASS_DEF_OPTIMIZEARRAYREPACKING
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+} // namespace fir
+
+namespace {
+class OptimizeArrayRepackingPass
+    : public fir::impl::OptimizeArrayRepackingBase<OptimizeArrayRepackingPass> {
+public:
+  void runOnOperation() override;
+};
+
+/// Relinks all uses of redundant fir.pack_array to the source.
+class PackingOfContiguous : public mlir::OpRewritePattern<fir::PackArrayOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  mlir::LogicalResult matchAndRewrite(fir::PackArrayOp,
+                                      mlir::PatternRewriter &) const override;
+};
+
+/// Erases fir.unpack_array with have the matching temp and original
+/// operands.
+class NoopUnpacking : public mlir::OpRewritePattern<fir::UnpackArrayOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  mlir::LogicalResult matchAndRewrite(fir::UnpackArrayOp,
+                                      mlir::PatternRewriter &) const override;
+};
+} // namespace
+
+mlir::LogicalResult
+PackingOfContiguous::matchAndRewrite(fir::PackArrayOp op,
+                                     mlir::PatternRewriter &rewriter) const {
+  mlir::Value box = op.getArray();
+  if (hlfir::isSimplyContiguous(box, !op.getInnermost())) {
+    rewriter.replaceAllUsesWith(op, box);
+    rewriter.eraseOp(op);
+    return mlir::success();
+  }
+  return mlir::failure();
+}
+
+mlir::LogicalResult
+NoopUnpacking::matchAndRewrite(fir::UnpackArrayOp op,
+                               mlir::PatternRewriter &rewriter) const {
+  if (op.getTemp() == op.getOriginal()) {
+    rewriter.eraseOp(op);
+    return mlir::success();
+  }
+  return mlir::failure();
+}
+
+void OptimizeArrayRepackingPass::runOnOperation() {
+  mlir::func::FuncOp funcOp = getOperation();
+  mlir::MLIRContext *context = &getContext();
+  mlir::RewritePatternSet patterns(context);
+  mlir::GreedyRewriteConfig config;
+  config.setRegionSimplificationLevel(
+      mlir::GreedySimplifyRegionLevel::Disabled);
+  patterns.insert<PackingOfContiguous>(context);
+  patterns.insert<NoopUnpacking>(context);
+  if (mlir::failed(
+          mlir::applyPatternsGreedily(funcOp, std::move(patterns), config))) {
+    mlir::emitError(funcOp.getLoc(), "failure in array repacking optimization");
+    signalPassFailure();
+  }
+}
diff --git a/flang/test/Driver/bbc-mlir-pass-pipeline.f90 b/flang/test/Driver/bbc-mlir-pass-pipeline.f90
index 137c19608c38f..f3791fe9f8dc3 100644
--- a/flang/test/Driver/bbc-mlir-pass-pipeline.f90
+++ b/flang/test/Driver/bbc-mlir-pass-pipeline.f90
@@ -47,6 +47,8 @@
 
 ! CHECK-NEXT: PolymorphicOpConversion
 ! CHECK-NEXT: AssumedRankOpConversion
+! CHECK-NEXT: 'func.func' Pipeline
+! CHECK-NEXT:   OptimizeArrayRepacking
 ! CHECK-NEXT: LowerRepackArraysPass
 ! CHECK-NEXT: SimplifyFIROperations
 
diff --git a/flang/test/Driver/mlir-pass-pipeline.f90 b/flang/test/Driver/mlir-pass-pipeline.f90
index 99f192ce7aec2..0bcd055a84b87 100644
--- a/flang/test/Driver/mlir-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-pass-pipeline.f90
@@ -105,6 +105,8 @@
 
 ! ALL-NEXT: PolymorphicOpConversion
 ! ALL-NEXT: AssumedRankOpConversion
+! O2-NEXT:  'func.func' Pipeline
+! O2-NEXT:    OptimizeArrayRepacking
 ! ALL-NEXT: LowerRepackArraysPass
 ! ALL-NEXT: SimplifyFIROperations
 ! O2-NEXT:  AddAliasTags
diff --git a/flang/test/Fir/basic-program.fir b/flang/test/Fir/basic-program.fir
index 7ac8b92f48953..c9fe53bf093a1 100644
--- a/flang/test/Fir/basic-program.fir
+++ b/flang/test/Fir/basic-program.fir
@@ -103,6 +103,8 @@ func.func @_QQmain() {
 
 // PASSES-NEXT: PolymorphicOpConversion
 // PASSES-NEXT: AssumedRankOpConversion
+// PASSES-NEXT: 'func.func' Pipeline
+// PASSES-NEXT:   OptimizeArrayRepacking
 // PASSES-NEXT: LowerRepackArraysPass
 // PASSES-NEXT: SimplifyFIROperations
 // PASSES-NEXT: AddAliasTags
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
index 35530c66f4038..8684a429ea5b4 100644
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
@@ -41,19 +41,11 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_17]] : (i64) -> index
 // CHECK:               fir.do_loop %[[VAL_28:.*]] = %[[VAL_2]] to %[[VAL_27]] step %[[VAL_2]] unordered {
 // CHECK:                 %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (index) -> i64
-// CHECK:                 %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_31:.*]] = fir.convert %[[VAL_29]] : (i64) -> index
-// CHECK:                 %[[VAL_32:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_33:.*]] = arith.addi %[[VAL_31]], %[[VAL_32]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_34:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_33]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_34:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_29]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_35:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
 // CHECK:                 %[[VAL_36:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_37:.*]] = arith.addi %[[VAL_29]], %[[VAL_36]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (i64) -> index
-// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_39]], %[[VAL_40]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_42:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_41]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_42:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_37]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_35]] to %[[VAL_42]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:               %[[VAL_43:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
@@ -61,17 +53,9 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:               fir.do_loop %[[VAL_45:.*]] = %[[VAL_2]] to %[[VAL_44]] step %[[VAL_2]] unordered {
 // CHECK:                 %[[VAL_46:.*]] = fir.convert %[[VAL_45]] : (index) -> i64
 // CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_17]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_48:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_49:.*]] = fir.convert %[[VAL_47]] : (i64) -> index
-// CHECK:                 %[[VAL_50:.*]] = arith.subi %[[VAL_48]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_51:.*]] = arith.addi %[[VAL_49]], %[[VAL_50]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_52:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_51]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_52:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_47]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_53:.*]] = fir.load %[[VAL_52]] : !fir.ref<i32>
-// CHECK:                 %[[VAL_54:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_55:.*]] = fir.convert %[[VAL_46]] : (i64) -> index
-// CHECK:                 %[[VAL_56:.*]] = arith.subi %[[VAL_54]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_57:.*]] = arith.addi %[[VAL_55]], %[[VAL_56]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_58:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_57]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_58:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_46]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_53]] to %[[VAL_58]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:             } else {
@@ -86,11 +70,7 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:                 %[[VAL_67:.*]] = fir.load %[[VAL_66]] : !fir.ref<i32>
 // CHECK:                 %[[VAL_68:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_69:.*]] = arith.addi %[[VAL_61]], %[[VAL_68]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_70:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_71:.*]] = fir.convert %[[VAL_69]] : (i64) -> index
-// CHECK:                 %[[VAL_72:.*]] = arith.subi %[[VAL_70]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_73:.*]] = arith.addi %[[VAL_71]], %[[VAL_72]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_74:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_73]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_74:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_69]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_67]] to %[[VAL_74]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:               %[[VAL_75:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
@@ -104,11 +84,7 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:                 %[[VAL_83:.*]] = arith.addi %[[VAL_81]], %[[VAL_82]] overflow<nsw, nuw> : index
 // CHECK:                 %[[VAL_84:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_83]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_85:.*]] = fir.load %[[VAL_84]] : !fir.ref<i32>
-// CHECK:                 %[[VAL_86:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_87:.*]] = fir.convert %[[VAL_78]] : (i64) -> index
-// CHECK:                 %[[VAL_88:.*]] = arith.subi %[[VAL_86]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_89:.*]] = arith.addi %[[VAL_87]], %[[VAL_88]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_90:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_89]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_90:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_78]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_85]] to %[[VAL_90]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:             }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
index d29e9a26c20ba..73e6e5c2421ff 100644
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
@@ -13,7 +13,6 @@ func.func @matmul_matrix_matrix_integer(%arg0: !hlfir.expr<?x?xi16>, %arg1: !hlf
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0 : i32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi16>) -> !fir.shape<2>
@@ -27,26 +26,14 @@ func.func @matmul_matrix_matrix_integer(%arg0: !hlfir.expr<?x?xi16>, %arg1: !hlf
 // ANSE:             %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xi32>>
 // ANSE:             fir.do_loop %[[VAL_14:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_19:.*]] = arith.addi %[[VAL_15]], %[[VAL_18]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_14]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_15]], %[[VAL_14]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
 // ANSE:                 hlfir.assign %[[VAL_4]] to %[[VAL_22]] : i32, !fir.ref<i32>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_23:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:                 fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                   %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]] = arith.subi %[[VAL_26]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_29:.*]] = arith.addi %[[VAL_25]], %[[VAL_28]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.addi %[[VAL_24]], %[[VAL_30]] : index
-// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_25]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
 // ANSE:                   %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
 // ANSE:                   %[[VAL_34:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_25]], %[[VAL_23]] : (!hlfir.expr<?x?xi16>, index, index) -> i16
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_23]], %[[VAL_24]] : (!hlfir.expr<?x?xi32>, index, index) -> i32
@@ -94,7 +81,6 @@ func.func @matmul_matrix_matrix_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
@@ -108,26 +94,14 @@ func.func @matmul_matrix_matrix_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 // ANSE:             %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
 // ANSE:             fir.do_loop %[[VAL_14:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_19:.*]] = arith.addi %[[VAL_15]], %[[VAL_18]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_14]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_15]], %[[VAL_14]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
 // ANSE:                 hlfir.assign %[[VAL_4]] to %[[VAL_22]] : f32, !fir.ref<f32>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_23:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]] = arith.subi %[[VAL_26]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_29:.*]] = arith.addi %[[VAL_25]], %[[VAL_28]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.addi %[[VAL_24]], %[[VAL_30]] : index
-// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_25]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
 // ANSE:                   %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<f32>
 // ANSE:                   %[[VAL_34:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_25]], %[[VAL_23]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_23]], %[[VAL_24]] : (!hlfir.expr<?x?xf16>, index, index) -> f16
@@ -175,7 +149,6 @@ func.func @matmul_matrix_matrix_complex(%arg0: !hlfir.expr<?x?xcomplex<f32>>, %a
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>
@@ -192,26 +165,14 @@ func.func @matmul_matrix_matrix_complex(%arg0: !hlfir.expr<?x?xcomplex<f32>>, %a
 // ANSE:             %[[VAL_16:.*]] = fir.insert_value %[[VAL_15]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_18]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_24:.*]] = arith.addi %[[VAL_17]], %[[VAL_23]] : index
-// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_18]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                 hlfir.assign %[[VAL_16]] to %[[VAL_25]] : complex<f32>, !fir.ref<complex<f32>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_27:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_28:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_29:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_29]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_28]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_34:.*]] = arith.addi %[[VAL_27]], %[[VAL_33]] : index
-// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_32]], %[[VAL_34]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_28]], %[[VAL_27]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_37:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_28]], %[[VAL_26]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>
 // ANSE:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_26]], %[[VAL_27]] : (!hlfir.expr<?x?xcomplex<f16>>, index, index) -> complex<f16>
@@ -262,7 +223,6 @@ func.func @matmul_matrix_matrix_complex_real(%arg0: !hlfir.expr<?x?xcomplex<f32>
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>
@@ -279,26 +239,14 @@ func.func @matmul_matrix_matrix_complex_real(%arg0: !hlfir.expr<?x?xcomplex<f32>
 // ANSE:             %[[VAL_16:.*]] = fir.insert_value %[[VAL_15]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_18]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_24:.*]] = arith.addi %[[VAL_17]], %[[VAL_23]] : index
-// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_18]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                 hlfir.assign %[[VAL_16]] to %[[VAL_25]] : complex<f32>, !fir.ref<complex<f32>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_27:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_28:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_29:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_29]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_28]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_34:.*]] = arith.addi %[[VAL_27]], %[[VAL_33]] : index
-// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_32]], %[[VAL_34]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_28]], %[[VAL_27]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_37:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_28]], %[[VAL_26]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>
 // ANSE:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_26]], %[[VAL_27]] : (!hlfir.expr<?x?xf16>, index, index) -> f16
@@ -357,7 +305,6 @@ func.func @matmul_matrix_matrix_logical(%arg0: !hlfir.expr<?x?x!fir.logical<1>>,
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant false
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<1>>) -> !fir.shape<2>
@@ -372,26 +319,14 @@ func.func @matmul_matrix_matrix_logical(%arg0: !hlfir.expr<?x?x!fir.logical<1>>,
 // ANSE:             %[[VAL_14:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>
 // ANSE:             fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_16:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.addi %[[VAL_16]], %[[VAL_19]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_15]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_16]], %[[VAL_15]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
 // ANSE:                 hlfir.assign %[[VAL_14]] to %[[VAL_23]] : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:                 fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.addi %[[VAL_26]], %[[VAL_29]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_25]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// ANSE:                   %[[VAL_33:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_26]], %[[VAL_25]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
 // ANSE:                   %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<!fir.logical<4>>
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_26]], %[[VAL_24]] : (!hlfir.expr<?x?x!fir.logical<1>>, index, index) -> !fir.logical<1>
 // ANSE:                   %[[VAL_36:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_24]], %[[VAL_25]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
@@ -446,7 +381,6 @@ func.func @matmul_matrix_vector_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
@@ -457,18 +391,12 @@ func.func @matmul_matrix_vector_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 // ANSE:           ^bb0(%[[VAL_10:.*]]: !fir.ref<!fir.array<?xf32>>):
 // ANSE:             %[[VAL_11:.*]] = fir.embox %[[VAL_10]](%[[VAL_8]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 // ANSE:             fir.do_loop %[[VAL_12:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_11]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:               %[[VAL_14:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_3]] : index
-// ANSE:               %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_14]] : index
-// ANSE:               %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_15]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:               %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_12]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:               hlfir.assign %[[VAL_4]] to %[[VAL_16]] : f32, !fir.ref<f32>
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_11]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_18]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_21]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_18]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<f32>
 // ANSE:                 %[[VAL_24:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]], %[[VAL_17]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
 // ANSE:                 %[[VAL_25:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_17]] : (!hlfir.expr<?xf16>, index) -> f16
@@ -513,7 +441,6 @@ func.func @matmul_vector_matrix_real(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.ex
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
@@ -525,18 +452,12 @@ func.func @matmul_vector_matrix_real(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.ex
 // ANSE:           ^bb0(%[[VAL_11:.*]]: !fir.ref<!fir.array<?xf32>>):
 // ANSE:             %[[VAL_12:.*]] = fir.embox %[[VAL_11]](%[[VAL_9]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 // ANSE:             fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] unordered {
-// ANSE:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_12]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:               %[[VAL_15:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_3]] : index
-// ANSE:               %[[VAL_16:.*]] = arith.addi %[[VAL_13]], %[[VAL_15]] : index
-// ANSE:               %[[VAL_17:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_16]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:               %[[VAL_17:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_13]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:               hlfir.assign %[[VAL_4]] to %[[VAL_17]] : f32, !fir.ref<f32>
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_19:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] {
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_12]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_19]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_22]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f32>
 // ANSE:                 %[[VAL_25:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]] : (!hlfir.expr<?xf32>, index) -> f32
 // ANSE:                 %[[VAL_26:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_18]], %[[VAL_19]] : (!hlfir.expr<?x?xf16>, index, index) -> f16
diff --git a/flang/test/Transforms/optimize-array-repacking.fir b/flang/test/Transforms/optimize-array-repacking.fir
new file mode 100644
index 0000000000000..6269fa441fe44
--- /dev/null
+++ b/flang/test/Transforms/optimize-array-repacking.fir
@@ -0,0 +1,660 @@
+// Test that the redundant fir.[un]pack_array operations
+// are optimized away, when the source is statically known
+// to be contiguous.
+// RUN: fir-opt --optimize-array-repacking %s | FileCheck %s
+
+// FIR is produced by compiling the sources with -mllvm -inline-all.
+// module inner
+// contains
+//   subroutine inner_repack1(x)
+//     real :: x(:)
+//   end subroutine inner_repack1
+//   subroutine inner_repack2(x)
+//     real :: x(:,:)
+//   end subroutine inner_repack2
+// end module inner
+
+// subroutine test_explicit_shape_cst(x)
+//   real :: x(100)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_explicit_shape_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_cst(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_cstEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_explicit_shape_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_var(x, n, l, u)
+//   integer :: n, l, u
+//   real :: x(n)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_explicit_shape_var
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_var(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}, %arg2: !fir.ref<i32> {fir.bindc_name = "l"}, %arg3: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg3 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %4 = fir.load %2 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9 = fir.declare %arg0(%8) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %10 = fir.load %1 : !fir.ref<i32>
+  %11 = fir.load %3 : !fir.ref<i32>
+  %12 = fir.convert %10 : (i32) -> index
+  %13 = fir.convert %11 : (i32) -> index
+  %14 = fir.slice %12, %13, %c1 : (index, index, index) -> !fir.slice<1>
+  %15 = fir.embox %9(%8) [%14] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.dummy_scope : !fir.dscope
+  %17 = fir.pack_array %15 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.declare %17 dummy_scope %16 {uniq_name = "_QFtest_explicit_shape_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %17 to %15 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_cst(x)
+//   real :: x(*)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_assumed_size_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_cst(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_cstEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_assumed_size_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_var(x, l, u)
+//   integer :: l, u
+//   real :: x(*)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_assumed_size_var
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_var(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %4 = fir.declare %arg0(%3) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %10 = fir.embox %4(%3) [%9] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_assumed_size_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_cst(x)
+//   real, allocatable :: x(:)
+//   call repack(x(10:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_allocatable_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_cst(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c41 = arith.constant 41 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_cstEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %3 = fir.shape %c41 : (index) -> !fir.shape<1>
+  %4 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %5:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %6 = fir.shape_shift %5#0, %5#1 : (index, index) -> !fir.shapeshift<1>
+  %7 = fir.array_coor %4(%6) %c10 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, index) -> !fir.ref<f32>
+  %8 = fir.convert %7 : (!fir.ref<f32>) -> !fir.ref<!fir.array<41xf32>>
+  %9 = fir.embox %8(%3) : (!fir.ref<!fir.array<41xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<41xf32>>
+  %10 = fir.convert %9 : (!fir.box<!fir.array<41xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_allocatable_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_var(x, l, u)
+//   integer :: l, u
+//   real, allocatable :: x(:)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_allocatable_var
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_var(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_allocatable_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_allocatable_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_varEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.box_addr %4 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %10:3 = fir.box_dims %4, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %11 = fir.shape_shift %10#0, %10#1 : (index, index) -> !fir.shapeshift<1>
+  %12 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %13 = fir.embox %9(%11) [%12] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.dummy_scope : !fir.dscope
+  %15 = fir.pack_array %13 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.declare %15 dummy_scope %14 {uniq_name = "_QFtest_allocatable_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %15 to %13 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_full(x)
+//   real, allocatable :: x(:,:)
+//   call repack(x(:,:))
+// contains
+//   subroutine repack(x)
+//     real :: x(:,:)
+//   end subroutine repack
+// end subroutine test_allocatable_full
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_full(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_full(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_fullEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %4:3 = fir.box_dims %2, %c1 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %5 = arith.addi %3#0, %3#1 : index
+  %6 = arith.subi %5, %c1 : index
+  %7 = arith.addi %4#0, %4#1 : index
+  %8 = arith.subi %7, %c1 : index
+  %9 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.heap<!fir.array<?x?xf32>>
+  %10 = fir.shape_shift %3#0, %3#1, %4#0, %4#1 : (index, index, index, index) -> !fir.shapeshift<2>
+  %11 = fir.slice %3#0, %6, %c1, %4#0, %8, %c1 : (index, index, index, index, index, index) -> !fir.slice<2>
+  %12 = fir.embox %9(%10) [%11] : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>, !fir.slice<2>) -> !fir.box<!fir.array<?x?xf32>>
+  %13 = fir.dummy_scope : !fir.dscope
+  %14 = fir.pack_array %12 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %15 = fir.declare %14 dummy_scope %13 {uniq_name = "_QFtest_allocatable_fullFrepackEx"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %14 to %12 heap : !fir.box<!fir.array<?x?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_cst_chain(x)
+//   real :: x(100)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_explicit_shape_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_cst_chain(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_cst_chainEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_explicit_shape_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.rebox %10 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %12 = fir.dummy_scope : !fir.dscope
+  %13 = fir.pack_array %11 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.declare %13 dummy_scope %12 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %13 to %11 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_var_chain(x, n, l, u)
+//   integer :: n, l, u
+//   real :: x(n)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_explicit_shape_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_var_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}, %arg2: !fir.ref<i32> {fir.bindc_name = "l"}, %arg3: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg3 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %4 = fir.load %2 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9 = fir.declare %arg0(%8) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %10 = fir.load %1 : !fir.ref<i32>
+  %11 = fir.load %3 : !fir.ref<i32>
+  %12 = fir.convert %10 : (i32) -> index
+  %13 = fir.convert %11 : (i32) -> index
+  %14 = fir.slice %12, %13, %c1 : (index, index, index) -> !fir.slice<1>
+  %15 = fir.embox %9(%8) [%14] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.dummy_scope : !fir.dscope
+  %17 = fir.pack_array %15 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.declare %17 dummy_scope %16 {uniq_name = "_QFtest_explicit_shape_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %19 = fir.rebox %18 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %20 = fir.dummy_scope : !fir.dscope
+  %21 = fir.pack_array %19 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %22 = fir.declare %21 dummy_scope %20 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %21 to %19 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %17 to %15 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_cst_chain(x)
+//   real :: x(*)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_assumed_size_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_cst_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_cst_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_assumed_size_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.rebox %10 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %12 = fir.dummy_scope : !fir.dscope
+  %13 = fir.pack_array %11 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.declare %13 dummy_scope %12 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %13 to %11 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_var_chain(x, l, u)
+//   integer :: l, u
+//   real :: x(*)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_assumed_size_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_var_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %4 = fir.declare %arg0(%3) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %10 = fir.embox %4(%3) [%9] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_assumed_size_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.rebox %13 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %15 = fir.dummy_scope : !fir.dscope
+  %16 = fir.pack_array %14 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.declare %16 dummy_scope %15 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %16 to %14 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_cst_chain(x)
+//   real, allocatable :: x(:)
+//   call repack(x(10:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_allocatable_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_cst_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c41 = arith.constant 41 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_cst_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %3 = fir.shape %c41 : (index) -> !fir.shape<1>
+  %4 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %5:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %6 = fir.shape_shift %5#0, %5#1 : (index, index) -> !fir.shapeshift<1>
+  %7 = fir.array_coor %4(%6) %c10 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, index) -> !fir.ref<f32>
+  %8 = fir.convert %7 : (!fir.ref<f32>) -> !fir.ref<!fir.array<41xf32>>
+  %9 = fir.embox %8(%3) : (!fir.ref<!fir.array<41xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<41xf32>>
+  %10 = fir.convert %9 : (!fir.box<!fir.array<41xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_allocatable_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.rebox %13 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %15 = fir.dummy_scope : !fir.dscope
+  %16 = fir.pack_array %14 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.declare %16 dummy_scope %15 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %16 to %14 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_var_chain(x, l, u)
+//   integer :: l, u
+//   real, allocatable :: x(:)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_allocatable_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_var_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_allocatable_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_allocatable_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_var_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.box_addr %4 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %10:3 = fir.box_dims %4, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %11 = fir.shape_shift %10#0, %10#1 : (index, index) -> !fir.shapeshift<1>
+  %12 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %13 = fir.embox %9(%11) [%12] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.dummy_scope : !fir.dscope
+  %15 = fir.pack_array %13 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.declare %15 dummy_scope %14 {uniq_name = "_QFtest_allocatable_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.rebox %16 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.dummy_scope : !fir.dscope
+  %19 = fir.pack_array %17 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %20 = fir.declare %19 dummy_scope %18 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %19 to %17 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %15 to %13 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_full_chain(x)
+//   real, allocatable :: x(:,:)
+//   call repack(x(:,:))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:,:)
+//     call inner_repack2(x)
+//   end subroutine repack
+// end subroutine test_allocatable_full_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_full_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_full_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_full_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %4:3 = fir.box_dims %2, %c1 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %5 = arith.addi %3#0, %3#1 : index
+  %6 = arith.subi %5, %c1 : index
+  %7 = arith.addi %4#0, %4#1 : index
+  %8 = arith.subi %7, %c1 : index
+  %9 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.heap<!fir.array<?x?xf32>>
+  %10 = fir.shape_shift %3#0, %3#1, %4#0, %4#1 : (index, index, index, index) -> !fir.shapeshift<2>
+  %11 = fir.slice %3#0, %6, %c1, %4#0, %8, %c1 : (index, index, index, index, index, index) -> !fir.slice<2>
+  %12 = fir.embox %9(%10) [%11] : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>, !fir.slice<2>) -> !fir.box<!fir.array<?x?xf32>>
+  %13 = fir.dummy_scope : !fir.dscope
+  %14 = fir.pack_array %12 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %15 = fir.declare %14 dummy_scope %13 {uniq_name = "_QFtest_allocatable_full_chainFrepackEx"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  %16 = fir.rebox %15 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %17 = fir.dummy_scope : !fir.dscope
+  %18 = fir.pack_array %16 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %19 = fir.declare %18 dummy_scope %17 {uniq_name = "_QMinnerFinner_repack2Ex"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %18 to %16 heap : !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %14 to %12 heap : !fir.box<!fir.array<?x?xf32>>
+  return
+}
+
+// TODO: if both fir.pack_array have the same property,
+// then the second one is redundant, because the first
+// repack makes 'x' contiguous.
+// subroutine neg_test_assumed_shape(x)
+//   real :: x(:)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_assumed_shape
+//
+// CHECK-LABEL:   func.func @_QPneg_test_assumed_shape(
+// CHECK: fir.pack_array
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_assumed_shape(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.pack_array %arg0 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %2 = fir.declare %1 dummy_scope %0 {uniq_name = "_QFneg_test_assumed_shapeEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %3 = fir.rebox %2 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %4 = fir.slice %c1, %c50, %c1 : (index, index, index) -> !fir.slice<1>
+  %5 = fir.rebox %3 [%4] : (!fir.box<!fir.array<?xf32>>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
+  %6 = fir.convert %5 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %7 = fir.dummy_scope : !fir.dscope
+  %8 = fir.pack_array %6 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %9 = fir.declare %8 dummy_scope %7 {uniq_name = "_QFneg_test_assumed_shapeFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %8 to %6 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %1 to %arg0 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_non_contig_slice_cst(x)
+//   real :: x(100)
+//   call repack(x(1:50:2))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_non_contig_slice_cst
+//
+// CHECK-LABEL:   func.func @_QPneg_test_non_contig_slice_cst(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_non_contig_slice_cst(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c2 = arith.constant 2 : index
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_cstEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.slice %c1, %c50, %c2 : (index, index, index) -> !fir.slice<1>
+  %4 = fir.embox %2(%1) [%3] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<25xf32>>
+  %5 = fir.convert %4 : (!fir.box<!fir.array<25xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %6 = fir.dummy_scope : !fir.dscope
+  %7 = fir.pack_array %5 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.declare %7 dummy_scope %6 {uniq_name = "_QFneg_test_non_contig_slice_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %7 to %5 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_non_contig_slice_var(x, s)
+//   integer :: s
+//   real :: x(100)
+//   call repack(x(1:50:s))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_non_contig_slice_var
+//
+// CHECK-LABEL:   func.func @_QPneg_test_non_contig_slice_var(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_non_contig_slice_var(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "s"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_varEs"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %3 = fir.declare %arg0(%2) dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_varEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %4 = fir.load %1 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = fir.slice %c1, %c50, %5 : (index, index, index) -> !fir.slice<1>
+  %7 = fir.embox %3(%2) [%6] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFneg_test_non_contig_slice_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_pointer(x)
+//   real, pointer :: x(:)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_pointer
+//
+// CHECK-LABEL:   func.func @_QPneg_test_pointer(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_pointer(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFneg_test_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %4 = fir.shift %3#0 : (index) -> !fir.shift<1>
+  %5 = fir.slice %c1, %c50, %c1 : (index, index, index) -> !fir.slice<1>
+  %6 = fir.rebox %2(%4) [%5] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, !fir.shift<1>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFneg_test_pointerFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}


From flang-commits at lists.llvm.org  Wed Jul  9 22:55:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 22:55:45 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop and omp.simd (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686f55e1.050a0220.2fb0cf.15f0@mx.google.com>


================
@@ -1816,8 +1816,7 @@ static void genSimdClauses(
   cp.processReduction(loc, clauseOps, reductionSyms);
   cp.processSafelen(clauseOps);
   cp.processSimdlen(clauseOps);
-
-  cp.processTODO<clause::Linear>(loc, llvm::omp::Directive::OMPD_simd);
+  cp.processLinear(clauseOps);
----------------
NimishMishra wrote:

I ran some tests, and the semantic checks were able to catch those cases. For instance, if the linear variable is anything other than an `integer` type, we get a semantic error noting that a linear variable with the REF modifier needs to be of integer type.

Do you think we should nevertheless add a check during FIR gen? I added a check during the translation but skipped adding it during FIR gen because of this reason.

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Wed Jul  9 23:47:08 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 23:47:08 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <686f61ec.630a0220.23b7ba.195d@mx.google.com>


https://github.com/EbinJose2002 updated https://github.com/llvm/llvm-project/pull/146851

>From 6476107702e746e1fcd6a95a0e88fb8df6c4b97f Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Tue, 1 Jul 2025 18:11:05 +0530
Subject: [PATCH 1/4] Removed unnecessary handling of reference to boxchar
 conversion in BufferizeHLFIR.cpp

---
 flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 58f2b57712974..240b79138b0e1 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -446,7 +446,8 @@ struct AssociateOpConversion
             ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
               !mlir::isa<fir::BoxCharType>(assocType)))) {
           sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
-        } else {
+        } else if (!mlir::isa<fir::ReferenceType>(sourceVar.getType()) ||
+                   !mlir::isa<fir::BoxCharType>(assocType)) {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
         }
         return sourceVar;

>From ee425fb1685f8d783d0e50e76b15df59c921e3b2 Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Thu, 3 Jul 2025 15:58:59 +0530
Subject: [PATCH 2/4] Added test file for the issue

---
 flang/test/Lower/adjustr.f90 | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 flang/test/Lower/adjustr.f90

diff --git a/flang/test/Lower/adjustr.f90 b/flang/test/Lower/adjustr.f90
new file mode 100644
index 0000000000000..6add2f5456bbc
--- /dev/null
+++ b/flang/test/Lower/adjustr.f90
@@ -0,0 +1,14 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+! Checking no reference to boxchar conversion is in the emitted fir.
+! CHECK-NOT: (!fir.ref<!fir.char<1,4>>) -> !fir.boxchar<1>
+
+program main
+  logical         ,dimension(1):: mask = .true.
+  character(len=2),dimension(1):: d1 = "a "
+  character(len=4),dimension(1):: d4
+  where (mask)
+    d4 = adjustr(d1 // d1)
+  end where
+  write(6,*) "d4     =", d4
+end
\ No newline at end of file

>From 3928151da22c344c403585c6b4420a3097991ad0 Mon Sep 17 00:00:00 2001
From: Ebin-McW <ebin.jose at multicorewareinc.com>
Date: Thu, 3 Jul 2025 16:04:36 +0530
Subject: [PATCH 3/4] Update adjustr.f90

---
 flang/test/Lower/adjustr.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Lower/adjustr.f90 b/flang/test/Lower/adjustr.f90
index 6add2f5456bbc..4813834614bd6 100644
--- a/flang/test/Lower/adjustr.f90
+++ b/flang/test/Lower/adjustr.f90
@@ -11,4 +11,4 @@ program main
     d4 = adjustr(d1 // d1)
   end where
   write(6,*) "d4     =", d4
-end
\ No newline at end of file
+end

>From c1b32bfe401bd2cd718f742e4909a2ed67262f5f Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Thu, 10 Jul 2025 12:08:54 +0530
Subject: [PATCH 4/4] Create emboxchar

---
 .../Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp   | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 240b79138b0e1..9f8f652397a1b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -446,8 +446,17 @@ struct AssociateOpConversion
             ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
               !mlir::isa<fir::BoxCharType>(assocType)))) {
           sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
-        } else if (!mlir::isa<fir::ReferenceType>(sourceVar.getType()) ||
-                   !mlir::isa<fir::BoxCharType>(assocType)) {
+        } else if (mlir::isa<fir::ReferenceType>(sourceVar.getType()) &&
+                   mlir::isa<fir::BoxCharType>(assocType)) {
+          auto refTy = mlir::cast<fir::ReferenceType>(sourceVar.getType());
+          auto charTy = mlir::dyn_cast<fir::CharacterType>(refTy.getEleTy());
+          mlir::Value lenVal = builder.createIntegerConstant(
+              loc, builder.getCharacterLengthType(), charTy.getLen());
+          auto boxCharType =
+              fir::BoxCharType::get(builder.getContext(), charTy.getFKind());
+          sourceVar =
+              builder.create<fir::EmboxCharOp>(loc, boxCharType, sourceVar, lenVal);
+        } else {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
         }
         return sourceVar;


From flang-commits at lists.llvm.org  Wed Jul  9 23:50:00 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 23:50:00 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <686f6298.170a0220.22100e.198a@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions cpp -- flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 9f8f65239..72680f4a6 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -454,8 +454,8 @@ struct AssociateOpConversion
               loc, builder.getCharacterLengthType(), charTy.getLen());
           auto boxCharType =
               fir::BoxCharType::get(builder.getContext(), charTy.getFKind());
-          sourceVar =
-              builder.create<fir::EmboxCharOp>(loc, boxCharType, sourceVar, lenVal);
+          sourceVar = builder.create<fir::EmboxCharOp>(loc, boxCharType,
+                                                       sourceVar, lenVal);
         } else {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
         }

``````````

</details>


https://github.com/llvm/llvm-project/pull/146851

From flang-commits at lists.llvm.org  Wed Jul  9 23:51:20 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 09 Jul 2025 23:51:20 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <686f62e8.050a0220.2b6cc.1b8b@mx.google.com>


https://github.com/EbinJose2002 updated https://github.com/llvm/llvm-project/pull/146851

>From 6476107702e746e1fcd6a95a0e88fb8df6c4b97f Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Tue, 1 Jul 2025 18:11:05 +0530
Subject: [PATCH 1/4] Removed unnecessary handling of reference to boxchar
 conversion in BufferizeHLFIR.cpp

---
 flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 58f2b57712974..240b79138b0e1 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -446,7 +446,8 @@ struct AssociateOpConversion
             ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
               !mlir::isa<fir::BoxCharType>(assocType)))) {
           sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
-        } else {
+        } else if (!mlir::isa<fir::ReferenceType>(sourceVar.getType()) ||
+                   !mlir::isa<fir::BoxCharType>(assocType)) {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
         }
         return sourceVar;

>From ee425fb1685f8d783d0e50e76b15df59c921e3b2 Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Thu, 3 Jul 2025 15:58:59 +0530
Subject: [PATCH 2/4] Added test file for the issue

---
 flang/test/Lower/adjustr.f90 | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 flang/test/Lower/adjustr.f90

diff --git a/flang/test/Lower/adjustr.f90 b/flang/test/Lower/adjustr.f90
new file mode 100644
index 0000000000000..6add2f5456bbc
--- /dev/null
+++ b/flang/test/Lower/adjustr.f90
@@ -0,0 +1,14 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+! Checking no reference to boxchar conversion is in the emitted fir.
+! CHECK-NOT: (!fir.ref<!fir.char<1,4>>) -> !fir.boxchar<1>
+
+program main
+  logical         ,dimension(1):: mask = .true.
+  character(len=2),dimension(1):: d1 = "a "
+  character(len=4),dimension(1):: d4
+  where (mask)
+    d4 = adjustr(d1 // d1)
+  end where
+  write(6,*) "d4     =", d4
+end
\ No newline at end of file

>From 3928151da22c344c403585c6b4420a3097991ad0 Mon Sep 17 00:00:00 2001
From: Ebin-McW <ebin.jose at multicorewareinc.com>
Date: Thu, 3 Jul 2025 16:04:36 +0530
Subject: [PATCH 3/4] Update adjustr.f90

---
 flang/test/Lower/adjustr.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Lower/adjustr.f90 b/flang/test/Lower/adjustr.f90
index 6add2f5456bbc..4813834614bd6 100644
--- a/flang/test/Lower/adjustr.f90
+++ b/flang/test/Lower/adjustr.f90
@@ -11,4 +11,4 @@ program main
     d4 = adjustr(d1 // d1)
   end where
   write(6,*) "d4     =", d4
-end
\ No newline at end of file
+end

>From c92e34eb8edcc589806a85a66c74f08d2aa559a9 Mon Sep 17 00:00:00 2001
From: EbinJose2002 <ebin.jose at multicorewareinc.com>
Date: Thu, 10 Jul 2025 12:08:54 +0530
Subject: [PATCH 4/4] Create emboxchar

---
 .../Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp   | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 240b79138b0e1..72680f4a6e928 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -446,8 +446,17 @@ struct AssociateOpConversion
             ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
               !mlir::isa<fir::BoxCharType>(assocType)))) {
           sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
-        } else if (!mlir::isa<fir::ReferenceType>(sourceVar.getType()) ||
-                   !mlir::isa<fir::BoxCharType>(assocType)) {
+        } else if (mlir::isa<fir::ReferenceType>(sourceVar.getType()) &&
+                   mlir::isa<fir::BoxCharType>(assocType)) {
+          auto refTy = mlir::cast<fir::ReferenceType>(sourceVar.getType());
+          auto charTy = mlir::dyn_cast<fir::CharacterType>(refTy.getEleTy());
+          mlir::Value lenVal = builder.createIntegerConstant(
+              loc, builder.getCharacterLengthType(), charTy.getLen());
+          auto boxCharType =
+              fir::BoxCharType::get(builder.getContext(), charTy.getFKind());
+          sourceVar = builder.create<fir::EmboxCharOp>(loc, boxCharType,
+                                                       sourceVar, lenVal);
+        } else {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
         }
         return sourceVar;


From flang-commits at lists.llvm.org  Thu Jul 10 02:17:05 2025
From: flang-commits at lists.llvm.org (Paul Osmialowski via flang-commits)
Date: Thu, 10 Jul 2025 02:17:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't check the '-mframe-pointer'
	flag (PR #147837)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147837@github.com>
Message-ID: <686f8511.170a0220.27cfe6.22bd@mx.google.com>


https://github.com/pawosm-arm approved this pull request.

Indeed, this is already being checked by `flang/test/Driver/frame-pointer-forwarding.f90`

https://github.com/llvm/llvm-project/pull/147837

From flang-commits at lists.llvm.org  Thu Jul 10 03:11:56 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 03:11:56 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [llvm][mlir][OpenMP] Support
 translation for linear clause in omp.wsloop and omp.simd (PR #139386)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139386@github.com>
Message-ID: <686f91ec.050a0220.386937.2515@mx.google.com>


================
@@ -1816,8 +1816,7 @@ static void genSimdClauses(
   cp.processReduction(loc, clauseOps, reductionSyms);
   cp.processSafelen(clauseOps);
   cp.processSimdlen(clauseOps);
-
-  cp.processTODO<clause::Linear>(loc, llvm::omp::Directive::OMPD_simd);
+  cp.processLinear(clauseOps);
----------------
tblah wrote:

```
subroutine test(lin, lb, ub, step)
  integer :: lin, lb, ub, step, i

  !$omp do linear(lin)
  do i = lb,ub,step
    l = l + 2
  enddo
endsubroutine
```

https://github.com/llvm/llvm-project/pull/139386

From flang-commits at lists.llvm.org  Thu Jul 10 04:19:35 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 04:19:35 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [Flang][FlangRT][Runtime] Add
 RT_OFFLOAD_API_GROUP_BEGIN to missing symbols on AMDGPU (PR #147612)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147612@github.com>
Message-ID: <686fa1c7.050a0220.cb2d0.2b54@mx.google.com>


agozillon wrote:

> > I'd personally prefer to keep it as close to OpenMP standard acceptable as possible
> 
> I second this. Let's keep this PR as-is.

Sounds good! :-) 

> It would be worth "fixing" clang to apply `declare target` to explicit instantiations, but I am not sure who can do this.

It would indeed, I am unfortunately not the best person to do it I think unfortunately, aware of how it works in Flang, but not Clang as of yet... but I can bring it up internally and maybe someone will have the bandwidth to look at it. 

Thank you very much for the review @vzakhari 


https://github.com/llvm/llvm-project/pull/147612

From flang-commits at lists.llvm.org  Thu Jul 10 04:20:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 04:20:01 -0700 (PDT)
Subject: [flang-commits] [flang] 75f81de - [Flang][FlangRT][Runtime] Add
 RT_OFFLOAD_API_GROUP_BEGIN to missing symbols on AMDGPU (#147612)
Message-ID: <686fa1e1.170a0220.106db.2fb3@mx.google.com>


Author: agozillon
Date: 2025-07-10T13:19:58+02:00
New Revision: 75f81ded8f9d558c92d92641bce3cf7ef8e13e9c

URL: https://github.com/llvm/llvm-project/commit/75f81ded8f9d558c92d92641bce3cf7ef8e13e9c
DIFF: https://github.com/llvm/llvm-project/commit/75f81ded8f9d558c92d92641bce3cf7ef8e13e9c.diff

LOG: [Flang][FlangRT][Runtime]  Add RT_OFFLOAD_API_GROUP_BEGIN to missing symbols on AMDGPU (#147612)

After the recent move to work queues, in certain cases when linking in
the fortran runtime built for offload on AMDGPU as required in certain
cases, we'll get missing symbols when linking. This PR tries to address
this issue by encompassing more of the library in
RT_OFFLOAD_API_GROUP_BEGIN, which has the affect of compiling these
functions for AMDGPU, resolving the missing symbols.

This PR should address the following issue:
https://github.com/llvm/llvm-project/issues/145888

Added: 
    

Modified: 
    flang-rt/include/flang-rt/runtime/format.h
    flang-rt/include/flang-rt/runtime/internal-unit.h
    flang-rt/include/flang-rt/runtime/io-stmt.h
    flang-rt/include/flang-rt/runtime/non-tbp-dio.h
    flang-rt/include/flang-rt/runtime/work-queue.h
    flang-rt/lib/runtime/edit-input.h
    flang-rt/lib/runtime/edit-output.h
    flang-rt/lib/runtime/unit.h
    flang/include/flang/Decimal/decimal.h

Removed: 
    


################################################################################
diff  --git a/flang-rt/include/flang-rt/runtime/format.h b/flang-rt/include/flang-rt/runtime/format.h
index b169d63fa4999..9469cb0ee9a4c 100644
--- a/flang-rt/include/flang-rt/runtime/format.h
+++ b/flang-rt/include/flang-rt/runtime/format.h
@@ -25,6 +25,8 @@ class Descriptor;
 
 namespace Fortran::runtime::io {
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 class IoStatementState;
 
 enum EditingFlags {
@@ -200,5 +202,8 @@ template <typename CONTEXT> class FormatControl {
   // must be last, may be incomplete
   Iteration stack_[maxMaxHeight];
 };
+
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
 #endif // FLANG_RT_RUNTIME_FORMAT_H_

diff  --git a/flang-rt/include/flang-rt/runtime/internal-unit.h b/flang-rt/include/flang-rt/runtime/internal-unit.h
index 429d3489e0112..a046764153ad8 100644
--- a/flang-rt/include/flang-rt/runtime/internal-unit.h
+++ b/flang-rt/include/flang-rt/runtime/internal-unit.h
@@ -18,6 +18,8 @@
 
 namespace Fortran::runtime::io {
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 class IoErrorHandler;
 
 // Points to (but does not own) a CHARACTER scalar or array for internal I/O.
@@ -55,5 +57,8 @@ template <Direction DIR> class InternalDescriptorUnit : public ConnectionState {
 
 extern template class InternalDescriptorUnit<Direction::Output>;
 extern template class InternalDescriptorUnit<Direction::Input>;
+
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
 #endif // FLANG_RT_RUNTIME_INTERNAL_UNIT_H_

diff  --git a/flang-rt/include/flang-rt/runtime/io-stmt.h b/flang-rt/include/flang-rt/runtime/io-stmt.h
index 73c59b0b5ed29..bfd5f69c76337 100644
--- a/flang-rt/include/flang-rt/runtime/io-stmt.h
+++ b/flang-rt/include/flang-rt/runtime/io-stmt.h
@@ -28,6 +28,8 @@
 
 namespace Fortran::runtime::io {
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 class ExternalFileUnit;
 class ChildIo;
 
@@ -880,5 +882,7 @@ class ErroneousIoStatementState : public IoStatementBase {
   ExternalFileUnit *unit_{nullptr};
 };
 
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
 #endif // FLANG_RT_RUNTIME_IO_STMT_H_

diff  --git a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
index 2bbbfa7f97f79..99d4113b6c7a8 100644
--- a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
+++ b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
@@ -32,6 +32,8 @@ class DerivedType;
 
 namespace Fortran::runtime::io {
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 struct NonTbpDefinedIo {
   const typeInfo::DerivedType &derivedType;
   void (*subroutine)(); // null means no non-TBP defined I/O here
@@ -52,5 +54,7 @@ struct NonTbpDefinedIoTable {
   bool ignoreNonTbpEntries{false};
 };
 
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
 #endif // FLANG_RT_RUNTIME_NON_TBP_DIO_H_

diff  --git a/flang-rt/include/flang-rt/runtime/work-queue.h b/flang-rt/include/flang-rt/runtime/work-queue.h
index 0daa7bc4d3384..124a87a772bbe 100644
--- a/flang-rt/include/flang-rt/runtime/work-queue.h
+++ b/flang-rt/include/flang-rt/runtime/work-queue.h
@@ -74,6 +74,8 @@ namespace Fortran::runtime {
 class Terminator;
 class WorkQueue;
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 // Ticket worker base classes
 
 template <typename TICKET> class ImmediateTicketRunner {
@@ -361,6 +363,7 @@ class DescriptorIoTicket
       : ImmediateTicketRunner<DescriptorIoTicket>(*this),
         Elementwise{descriptor}, io_{io}, table_{table},
         anyIoTookPlace_{anyIoTookPlace} {}
+
   RT_API_ATTRS int Begin(WorkQueue &);
   RT_API_ATTRS int Continue(WorkQueue &);
   RT_API_ATTRS bool &anyIoTookPlace() { return anyIoTookPlace_; }
@@ -551,5 +554,7 @@ class WorkQueue {
   TicketList *firstFree_{static_};
 };
 
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime
 #endif // FLANG_RT_RUNTIME_WORK_QUEUE_H_

diff  --git a/flang-rt/lib/runtime/edit-input.h b/flang-rt/lib/runtime/edit-input.h
index 686cd461b3e34..961f06569aca3 100644
--- a/flang-rt/lib/runtime/edit-input.h
+++ b/flang-rt/lib/runtime/edit-input.h
@@ -15,6 +15,8 @@
 
 namespace Fortran::runtime::io {
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 RT_API_ATTRS bool EditIntegerInput(
     IoStatementState &, const DataEdit &, void *, int kind, bool isSigned);
 
@@ -49,5 +51,7 @@ extern template RT_API_ATTRS bool EditCharacterInput(
 extern template RT_API_ATTRS bool EditCharacterInput(
     IoStatementState &, const DataEdit &, char32_t *, std::size_t);
 
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
 #endif // FLANG_RT_RUNTIME_EDIT_INPUT_H_

diff  --git a/flang-rt/lib/runtime/edit-output.h b/flang-rt/lib/runtime/edit-output.h
index 51a47405e49e4..dbeccb291d363 100644
--- a/flang-rt/lib/runtime/edit-output.h
+++ b/flang-rt/lib/runtime/edit-output.h
@@ -25,6 +25,8 @@
 
 namespace Fortran::runtime::io {
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 // I, B, O, Z, and G output editing for INTEGER.
 // The DataEdit reference is const here (and elsewhere in this header) so that
 // one edit descriptor with a repeat factor may safely serve to edit
@@ -137,5 +139,7 @@ extern template class RealOutputEditing<10>;
 // TODO: double/double
 extern template class RealOutputEditing<16>;
 
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
 #endif // FLANG_RT_RUNTIME_EDIT_OUTPUT_H_

diff  --git a/flang-rt/lib/runtime/unit.h b/flang-rt/lib/runtime/unit.h
index 86e5639f1250e..9aec9b1659576 100644
--- a/flang-rt/lib/runtime/unit.h
+++ b/flang-rt/lib/runtime/unit.h
@@ -46,6 +46,8 @@ extern RT_VAR_ATTRS ExternalFileUnit *defaultOutput; // unit 6
 extern RT_VAR_ATTRS ExternalFileUnit *errorOutput; // unit 0 extension
 RT_OFFLOAD_VAR_GROUP_END
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 #if defined(RT_USE_PSEUDO_FILE_UNIT)
 // A flavor of OpenFile class that pretends to be a terminal,
 // and only provides basic buffering of the output
@@ -298,5 +300,7 @@ class ChildIo {
   Fortran::common::optional<IoStatementState> io_;
 };
 
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
 #endif // FLANG_RT_RUNTIME_UNIT_H_

diff  --git a/flang/include/flang/Decimal/decimal.h b/flang/include/flang/Decimal/decimal.h
index 443163d058e28..5e81e47a18268 100644
--- a/flang/include/flang/Decimal/decimal.h
+++ b/flang/include/flang/Decimal/decimal.h
@@ -65,6 +65,9 @@ enum DecimalConversionFlags {
 #define EXTRA_DECIMAL_CONVERSION_SPACE (1 + 1 + 2 * 16 - 1)
 
 #ifdef __cplusplus
+
+RT_OFFLOAD_API_GROUP_BEGIN
+
 template <int PREC>
 RT_API_ATTRS ConversionToDecimalResult ConvertToDecimal(char *, size_t,
     DecimalConversionFlags, int digits, enum FortranRounding rounding,
@@ -110,6 +113,9 @@ extern template RT_API_ATTRS ConversionToBinaryResult<64> ConvertToBinary<64>(
     const char *&, enum FortranRounding, const char *end);
 extern template RT_API_ATTRS ConversionToBinaryResult<113> ConvertToBinary<113>(
     const char *&, enum FortranRounding, const char *end);
+
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::decimal
 extern "C" {
 #define NS(x) Fortran::decimal::x


        

From flang-commits at lists.llvm.org  Thu Jul 10 04:20:05 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 04:20:05 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [Flang][FlangRT][Runtime] Add
 RT_OFFLOAD_API_GROUP_BEGIN to missing symbols on AMDGPU (PR #147612)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147612@github.com>
Message-ID: <686fa1e5.630a0220.c1543.2924@mx.google.com>


https://github.com/agozillon closed https://github.com/llvm/llvm-project/pull/147612

From flang-commits at lists.llvm.org  Thu Jul 10 05:04:49 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Thu, 10 Jul 2025 05:04:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Avoid undefined behaviour when
 parsing format expressions (PR #147539)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147539@github.com>
Message-ID: <686fac61.170a0220.1a48ca.2afc@mx.google.com>


DavidSpickett wrote:

The reason this didn't reproduce locally is I had an llvm installed from apt so it found the header in `/usr/local/include/llvm`.

https://github.com/llvm/llvm-project/pull/147539

From flang-commits at lists.llvm.org  Thu Jul 10 06:06:53 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 06:06:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fbaed.050a0220.13c502.3216@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/147765

>From 1419c89670953b969fcc1c474a258c7475d8e7df Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 9 May 2025 07:45:20 -0500
Subject: [PATCH 1/5] [flang][OpenMP] Handle multiple spellings in
 OmpDirectiveNameParser

Collect all spellings from all supported OpenMP versions before parsing.
Break up the list of spellings by the initial letter to speed up parsing
a little.
---
 flang/lib/Parser/openmp-parsers.cpp | 49 +++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 30b5d4f9513a7..29701a616d4b0 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
 // OpenMP Directives and Clauses
@@ -104,8 +105,13 @@ struct OmpDirectiveNameParser {
   using Token = TokenStringMatch<false, false>;
 
   std::optional<resultType> Parse(ParseState &state) const {
+    if (state.BytesRemaining() == 0) {
+      return std::nullopt;
+    }
     auto begin{state.GetLocation()};
-    for (const NameWithId &nid : directives()) {
+    char next{static_cast<char>(std::tolower(*begin))};
+
+    for (const NameWithId &nid : directives_starting_with(next)) {
       if (attempt(Token(nid.first.data())).Parse(state)) {
         OmpDirectiveName n;
         n.v = nid.second;
@@ -118,30 +124,47 @@ struct OmpDirectiveNameParser {
 
 private:
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
+  using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<const NameWithId *> directives() const;
-  void initTokens(NameWithId *) const;
+  llvm::iterator_range<ConstIterator>
+  directives_starting_with(char initial) const;
+  void initTokens(std::vector<NameWithId>[]) const;
 };
 
-llvm::iterator_range<const OmpDirectiveNameParser::NameWithId *>
-OmpDirectiveNameParser::directives() const {
-  static NameWithId table[llvm::omp::Directive_enumSize];
+llvm::iterator_range<OmpDirectiveNameParser::ConstIterator>
+OmpDirectiveNameParser::directives_starting_with(char initial) const {
+  static const std::vector<NameWithId> empty{};
+  if (initial < 'a' || initial > 'z') {
+    return llvm::make_range(std::cbegin(empty), std::cend(empty));
+  }
+
+  static std::vector<NameWithId> table['z' - 'a' + 1];
   [[maybe_unused]] static bool init = (initTokens(table), true);
-  return llvm::make_range(std::cbegin(table), std::cend(table));
+
+  int index = initial - 'a';
+  return llvm::make_range(std::cbegin(table[index]), std::cend(table[index]));
 }
 
-void OmpDirectiveNameParser::initTokens(NameWithId *table) const {
+void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   for (size_t i{0}, e{llvm::omp::Directive_enumSize}; i != e; ++i) {
+    llvm::StringSet spellings;
     auto id{static_cast<llvm::omp::Directive>(i)};
-    llvm::StringRef name{
-        llvm::omp::getOpenMPDirectiveName(id, llvm::omp::FallbackVersion)};
-    table[i] = std::make_pair(name.str(), id);
+    for (unsigned version : llvm::omp::getOpenMPVersions()) {
+      spellings.insert(llvm::omp::getOpenMPDirectiveName(id, version));
+    }
+    for (auto &[name, _] : spellings) {
+      char initial{static_cast<char>(std::tolower(name.front()))};
+      table[initial - 'a'].emplace_back(name.str(), id);
+    }
   }
   // Sort the table with respect to the directive name length in a descending
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
-  std::sort(table, table + llvm::omp::Directive_enumSize,
-      [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
+  for (int initial{'a'}; initial != 'z' + 1; ++initial) {
+    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
+      return a.first.size() > b.first.size();
+    });
+  }
 }
 
 // --- Modifier helpers -----------------------------------------------

>From 0e2c062d467d7d23ed88cdf14c2c3de740549ca6 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 7 Jul 2025 13:54:32 -0500
Subject: [PATCH 2/5] [flang][OpenMP] Recognize remaining OpenMP 6.0 spellings
 in parser

Parse OpenMP 6.0 spellings for directives that don't use
OmpDirectiveNameParser.
---
 flang/lib/Parser/openmp-parsers.cpp           |  17 +-
 .../OpenMP/openmp6-directive-spellings.f90    | 252 ++++++++++++++++++
 2 files changed, 263 insertions(+), 6 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/openmp6-directive-spellings.f90

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 29701a616d4b0..b25445b0aaef6 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1501,7 +1501,7 @@ TYPE_PARSER(
     // In this context "TARGET UPDATE" can be parsed as a TARGET directive
     // followed by an UPDATE clause. This is the only combination at the
     // moment, exclude it explicitly.
-    (!"TARGET UPDATE"_sptok) >=
+    (!("TARGET UPDATE"_sptok || "TARGET_UPDATE"_sptok)) >=
     construct<OmpBlockDirective>(first(
         "MASKED" >> pure(llvm::omp::Directive::OMPD_masked),
         "MASTER" >> pure(llvm::omp::Directive::OMPD_master),
@@ -1514,6 +1514,7 @@ TYPE_PARSER(
         "SCOPE" >> pure(llvm::omp::Directive::OMPD_scope),
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
+        "TARGET_DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
@@ -1534,12 +1535,13 @@ TYPE_PARSER(construct<OmpInitializerClause>(
 
 // OpenMP 5.2: 7.5.4 Declare Variant directive
 TYPE_PARSER(sourced(
-    construct<OmpDeclareVariantDirective>(verbatim("DECLARE VARIANT"_tok),
+    construct<OmpDeclareVariantDirective>(
+        verbatim("DECLARE VARIANT"_tok) || verbatim("DECLARE_VARIANT"_tok),
         "(" >> maybe(name / ":"), name / ")", Parser<OmpClauseList>{})))
 
 // 2.16 Declare Reduction Construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareReductionConstruct>(
-    verbatim("DECLARE REDUCTION"_tok),
+    verbatim("DECLARE REDUCTION"_tok) || verbatim("DECLARE_REDUCTION"_tok),
     "(" >> indirect(Parser<OmpReductionSpecifier>{}) / ")",
     maybe(Parser<OmpClauseList>{}))))
 
@@ -1558,7 +1560,8 @@ TYPE_PARSER(
 
 // 2.10.6 Declare Target Construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareTargetConstruct>(
-    verbatim("DECLARE TARGET"_tok), Parser<OmpDeclareTargetSpecifier>{})))
+    verbatim("DECLARE TARGET"_tok) || verbatim("DECLARE_TARGET"_tok),
+    Parser<OmpDeclareTargetSpecifier>{})))
 
 static OmpMapperSpecifier ConstructOmpMapperSpecifier(
     std::optional<Name> &&mapperName, TypeSpec &&typeSpec, Name &&varName) {
@@ -1586,7 +1589,8 @@ TYPE_PARSER(applyFunction<OmpMapperSpecifier>(ConstructOmpMapperSpecifier,
 
 // OpenMP 5.2: 5.8.8 Declare Mapper Construct
 TYPE_PARSER(sourced(
-    construct<OpenMPDeclareMapperConstruct>(verbatim("DECLARE MAPPER"_tok),
+    construct<OpenMPDeclareMapperConstruct>(
+        verbatim("DECLARE MAPPER"_tok) || verbatim("DECLARE_MAPPER"_tok),
         parenthesized(Parser<OmpMapperSpecifier>{}), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(construct<OmpReductionCombiner>(Parser<AssignmentStmt>{}) ||
@@ -1633,7 +1637,8 @@ TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
 
 // 2.8.2 Declare Simd construct
 TYPE_PARSER(
-    sourced(construct<OpenMPDeclareSimdConstruct>(verbatim("DECLARE SIMD"_tok),
+    sourced(construct<OpenMPDeclareSimdConstruct>(
+        verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
         maybe(parenthesized(name)), Parser<OmpClauseList>{})))
 
 // 2.4 Requires construct
diff --git a/flang/test/Parser/OpenMP/openmp6-directive-spellings.f90 b/flang/test/Parser/OpenMP/openmp6-directive-spellings.f90
new file mode 100644
index 0000000000000..6a1d565451220
--- /dev/null
+++ b/flang/test/Parser/OpenMP/openmp6-directive-spellings.f90
@@ -0,0 +1,252 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=60 %s -o - | FileCheck --check-prefix=UNPARSE %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=60 %s -o - | FileCheck --check-prefix=PARSE-TREE %s
+
+! The directives to check:
+!   cancellation_point
+!   declare_mapper
+!   declare_reduction
+!   declare_simd
+!   declare_target
+!   declare_variant
+!   target_data
+!   target_enter_data
+!   target_exit_data
+!   target_update
+
+subroutine f00
+  implicit none
+  integer :: i
+
+  !$omp parallel
+  do i = 1, 10
+    !$omp cancellation_point parallel
+  enddo
+  !$omp end parallel
+end
+
+!UNPARSE: SUBROUTINE f00
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP PARALLEL
+!UNPARSE:  DO i=1_4,10_4
+!UNPARSE: !$OMP CANCELLATION_POINT PARALLEL
+!UNPARSE:  END DO
+!UNPARSE: !$OMP END PARALLEL
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPCancellationPointConstruct -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = cancellation point
+!PARSE-TREE: | OmpClauseList -> OmpClause -> CancellationConstructType -> OmpCancellationConstructTypeClause
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = parallel
+!PARSE-TREE: | Flags = None
+
+subroutine f01
+  type :: t
+    integer :: x
+  end type
+  !$omp declare_mapper(t :: v) map(v%x)
+end
+
+!UNPARSE: SUBROUTINE f01
+!UNPARSE:  TYPE :: t
+!UNPARSE:   INTEGER :: x
+!UNPARSE:  END TYPE
+!UNPARSE: !$OMP DECLARE MAPPER (t::v) MAP(v%x)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareMapperConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpMapperSpecifier
+!PARSE-TREE: | | string = 't.omp.default.mapper'
+!PARSE-TREE: | | TypeSpec -> DerivedTypeSpec
+!PARSE-TREE: | | | Name = 't'
+!PARSE-TREE: | | Name = 'v'
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> StructureComponent
+!PARSE-TREE: | | | DataRef -> Name = 'v'
+!PARSE-TREE: | | | Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02
+  type :: t
+    integer :: x
+  end type
+  !$omp declare_reduction(+ : t : omp_out%x = omp_out%x + omp_in%x)
+end
+
+!UNPARSE: SUBROUTINE f02
+!UNPARSE:  TYPE :: t
+!UNPARSE:   INTEGER :: x
+!UNPARSE:  END TYPE
+!UNPARSE: !$OMP DECLARE REDUCTION (+:t: omp_out%x=omp_out%x+omp_in%x
+!UNPARSE: )
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareReductionConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpReductionSpecifier
+!PARSE-TREE: | | OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
+!PARSE-TREE: | | OmpTypeNameList -> OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
+!PARSE-TREE: | | | Name = 't'
+!PARSE-TREE: | | OmpReductionCombiner -> AssignmentStmt = 'omp_out%x=omp_out%x+omp_in%x'
+!PARSE-TREE: | | | Variable = 'omp_out%x'
+!PARSE-TREE: | | | | Designator -> DataRef -> StructureComponent
+!PARSE-TREE: | | | | | DataRef -> Name = 'omp_out'
+!PARSE-TREE: | | | | | Name = 'x'
+!PARSE-TREE: | | | Expr = 'omp_out%x+omp_in%x'
+!PARSE-TREE: | | | | Add
+!PARSE-TREE: | | | | | Expr = 'omp_out%x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> StructureComponent
+!PARSE-TREE: | | | | | | | DataRef -> Name = 'omp_out'
+!PARSE-TREE: | | | | | | | Name = 'x'
+!PARSE-TREE: | | | | | Expr = 'omp_in%x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> StructureComponent
+!PARSE-TREE: | | | | | | | DataRef -> Name = 'omp_in'
+!PARSE-TREE: | | | | | | | Name = 'x'
+!PARSE-TREE: | OmpClauseList ->
+
+subroutine f03
+  !$omp declare_simd
+end
+
+!UNPARSE: SUBROUTINE f03
+!UNPARSE: !$OMP DECLARE SIMD
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OpenMPDeclareSimdConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpClauseList ->
+
+subroutine f04
+  !$omp declare_target
+end
+
+!UNPARSE: SUBROUTINE f04
+!UNPARSE: !$OMP DECLARE TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OpenMPDeclareTargetConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpDeclareTargetSpecifier -> OmpDeclareTargetWithClause -> OmpClauseList ->
+
+subroutine f05
+  implicit none
+  interface
+    subroutine g05
+    end
+  end interface
+  !$omp declare_variant(g05) match(user={condition(.true.)})
+end
+
+!UNPARSE: SUBROUTINE f05
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTERFACE
+!UNPARSE:   SUBROUTINE g05
+!UNPARSE:   END SUBROUTINE
+!UNPARSE:  END INTERFACE
+!UNPARSE: !$OMP DECLARE VARIANT (g05) MATCH(USER={CONDITION(.true._4)})
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OmpDeclareVariantDirective
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | Name = 'g05'
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Match -> OmpMatchClause -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | OmpTraitSelector
+!PARSE-TREE: | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | Properties
+!PARSE-TREE: | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | bool = 'true'
+
+subroutine f06
+  implicit none
+  integer :: i
+  !$omp target_data map(tofrom: i)
+  i = 0
+  !$omp end target data
+end
+
+!UNPARSE: SUBROUTINE f06
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP TARGET DATA  MAP(TOFROM: i)
+!UNPARSE:   i=0_4
+!UNPARSE: !$OMP END TARGET DATA
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+!PARSE-TREE: | OmpBeginBlockDirective
+!PARSE-TREE: | | OmpBlockDirective -> llvm::omp::Directive = target data
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | | Modifier -> OmpMapType -> Value = Tofrom
+!PARSE-TREE: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | | bool = 'true'
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'i=0_4'
+!PARSE-TREE: | | | Variable = 'i'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | | Expr = '0_4'
+!PARSE-TREE: | | | | LiteralConstant -> IntLiteralConstant = '0'
+!PARSE-TREE: | OmpEndBlockDirective
+!PARSE-TREE: | | OmpBlockDirective -> llvm::omp::Directive = target data
+!PARSE-TREE: | | OmpClauseList ->
+
+subroutine f07
+  implicit none
+  integer :: i
+  !$omp target_enter_data map(to: i)
+end
+
+!UNPARSE: SUBROUTINE f07
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP TARGET_ENTER_DATA MAP(TO: i)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target enter data
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | bool = 'true'
+!PARSE-TREE: | Flags = None
+
+subroutine f08
+  implicit none
+  integer :: i
+  !$omp target_exit_data map(from: i)
+end
+
+!UNPARSE: SUBROUTINE f08
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP TARGET_EXIT_DATA MAP(FROM: i)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target exit data
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = From
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | bool = 'true'
+!PARSE-TREE: | Flags = None
+
+subroutine f09
+  implicit none
+  integer :: i
+  !$omp target_update to(i)
+end
+
+!UNPARSE: SUBROUTINE f09
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP TARGET_UPDATE TO(i)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target update
+!PARSE-TREE: | OmpClauseList -> OmpClause -> To -> OmpToClause
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | bool = 'true'
+!PARSE-TREE: | Flags = None

>From 1dd6916ae6c7bd360e1c603b0cb9fd3b942233fc Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 9 Jul 2025 08:22:33 -0500
Subject: [PATCH 3/5] format

---
 flang/lib/Parser/openmp-parsers.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 29701a616d4b0..fdf96dffeb8ed 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -126,8 +126,8 @@ struct OmpDirectiveNameParser {
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
   using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<ConstIterator>
-  directives_starting_with(char initial) const;
+  llvm::iterator_range<ConstIterator> directives_starting_with(
+      char initial) const;
   void initTokens(std::vector<NameWithId>[]) const;
 };
 
@@ -161,9 +161,8 @@ void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
   for (int initial{'a'}; initial != 'z' + 1; ++initial) {
-    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
-      return a.first.size() > b.first.size();
-    });
+    llvm::stable_sort(table[initial - 'a'],
+        [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
   }
 }
 

>From a69ffbae3a4eaa3cf4b0505808e693fcf416eb00 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 9 Jul 2025 08:24:58 -0500
Subject: [PATCH 4/5] format

---
 flang/lib/Parser/openmp-parsers.cpp | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index e8e1d5ac16e7a..ee88cce288cae 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1533,10 +1533,9 @@ TYPE_PARSER(construct<OmpInitializerClause>(
     construct<OmpInitializerClause>(Parser<OmpInitializerProc>{})))
 
 // OpenMP 5.2: 7.5.4 Declare Variant directive
-TYPE_PARSER(sourced(
-    construct<OmpDeclareVariantDirective>(
-        verbatim("DECLARE VARIANT"_tok) || verbatim("DECLARE_VARIANT"_tok),
-        "(" >> maybe(name / ":"), name / ")", Parser<OmpClauseList>{})))
+TYPE_PARSER(sourced(construct<OmpDeclareVariantDirective>(
+    verbatim("DECLARE VARIANT"_tok) || verbatim("DECLARE_VARIANT"_tok),
+    "(" >> maybe(name / ":"), name / ")", Parser<OmpClauseList>{})))
 
 // 2.16 Declare Reduction Construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareReductionConstruct>(
@@ -1587,10 +1586,9 @@ TYPE_PARSER(applyFunction<OmpMapperSpecifier>(ConstructOmpMapperSpecifier,
     maybe(name / ":" / !":"_tok), typeSpec / "::", name))
 
 // OpenMP 5.2: 5.8.8 Declare Mapper Construct
-TYPE_PARSER(sourced(
-    construct<OpenMPDeclareMapperConstruct>(
-        verbatim("DECLARE MAPPER"_tok) || verbatim("DECLARE_MAPPER"_tok),
-        parenthesized(Parser<OmpMapperSpecifier>{}), Parser<OmpClauseList>{})))
+TYPE_PARSER(sourced(construct<OpenMPDeclareMapperConstruct>(
+    verbatim("DECLARE MAPPER"_tok) || verbatim("DECLARE_MAPPER"_tok),
+    parenthesized(Parser<OmpMapperSpecifier>{}), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(construct<OmpReductionCombiner>(Parser<AssignmentStmt>{}) ||
     construct<OmpReductionCombiner>(Parser<FunctionReference>{}))
@@ -1635,10 +1633,9 @@ TYPE_PARSER(sourced(construct<OpenMPAllocatorsConstruct>(
 TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
 
 // 2.8.2 Declare Simd construct
-TYPE_PARSER(
-    sourced(construct<OpenMPDeclareSimdConstruct>(
-        verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
-        maybe(parenthesized(name)), Parser<OmpClauseList>{})))
+TYPE_PARSER(sourced(construct<OpenMPDeclareSimdConstruct>(
+    verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
+    maybe(parenthesized(name)), Parser<OmpClauseList>{})))
 
 // 2.4 Requires construct
 TYPE_PARSER(sourced(construct<OpenMPRequiresConstruct>(

>From 7b42bbe446f366fa10e1fb6ecf237169a56400b0 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 8 Jul 2025 18:06:48 -0500
Subject: [PATCH 5/5] [flang][OpenMP] Issue a warning when parsing future
 directive spelling

OpenMP 6.0 introduced alternative spelling for some directives, with the
previous spellings still allowed.

Warn the user when a new spelling is encountered with OpenMP version set
to an older value.
---
 flang/include/flang/Parser/parse-tree.h       |   2 +-
 flang/lib/Parser/openmp-parsers.cpp           |   4 +-
 flang/lib/Semantics/check-omp-structure.cpp   | 181 ++++++++++++++++++
 flang/lib/Semantics/check-omp-structure.h     |   2 +
 .../OpenMP/future-directive-spellings.f90     |  92 +++++++++
 5 files changed, 278 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Semantics/OpenMP/future-directive-spellings.f90

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 7e752eeb4dfe4..43954ff735361 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4612,7 +4612,7 @@ struct OmpDirectiveSpecification {
 
 struct OmpMetadirectiveDirective {
   TUPLE_CLASS_BOILERPLATE(OmpMetadirectiveDirective);
-  std::tuple<OmpClauseList> t;
+  std::tuple<Verbatim, OmpClauseList> t;
   CharBlock source;
 };
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index ee88cce288cae..3016ce4ccd2f8 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1194,7 +1194,7 @@ TYPE_PARSER(sourced(construct<OpenMPUtilityConstruct>(
         sourced(Parser<OmpNothingDirective>{}))))))
 
 TYPE_PARSER(sourced(construct<OmpMetadirectiveDirective>(
-    "METADIRECTIVE" >> Parser<OmpClauseList>{})))
+    verbatim("METADIRECTIVE"_tok), Parser<OmpClauseList>{})))
 
 // Omp directives enclosing do loop
 TYPE_PARSER(sourced(construct<OmpLoopDirective>(first(
@@ -1687,7 +1687,7 @@ TYPE_PARSER(sourced(construct<OmpAssumeDirective>(
     verbatim("ASSUME"_tok), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(sourced(construct<OmpEndAssumeDirective>(
-    verbatim(startOmpLine >> "END ASSUME"_tok))))
+    startOmpLine >> verbatim("END ASSUME"_tok))))
 
 TYPE_PARSER(sourced(
     construct<OpenMPAssumeConstruct>(Parser<OmpAssumeDirective>{} / endOmpLine,
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index aa4cdb9d27019..191f13cdeef65 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -259,6 +259,55 @@ void OmpStructureChecker::CheckVariableListItem(
   }
 }
 
+void OmpStructureChecker::CheckDirectiveSpelling(
+    parser::CharBlock spelling, llvm::omp::Directive id) {
+  // Directive names that contain spaces can be spelled in the source without
+  // any of the spaces. Because of that getOpenMPKind* is not guaranteed to
+  // work with the source spelling as the argument.
+  //
+  // To verify the source spellings, we have to get the spelling for a given
+  // version, remove spaces and compare it with the source spelling (also
+  // with spaces removed).
+  auto removeSpaces = [](llvm::StringRef s) {
+    std::string n{s.str()};
+    for (auto it{n.begin()}; it != n.end();) {
+      it = isspace(*it) ? n.erase(it) : std::next(it);
+    }
+    return n;
+  };
+
+  std::string lowerNoWS{removeSpaces(
+      parser::ToLowerCaseLetters({spelling.begin(), spelling.size()}))};
+  llvm::StringRef ref(lowerNoWS);
+  if (ref.starts_with("end")) {
+    ref = ref.drop_front(3);
+  }
+
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  // For every "future" version v, check if the check if the corresponding
+  // spelling of id was introduced later than the current version. If so,
+  // and if that spelling matches the source spelling, issue a warning.
+  for (unsigned v : llvm::omp::getOpenMPVersions()) {
+    if (v <= version) {
+      continue;
+    }
+    llvm::StringRef name{llvm::omp::getOpenMPDirectiveName(id, v)};
+    auto [kind, versions]{llvm::omp::getOpenMPDirectiveKindAndVersions(name)};
+    assert(kind == id && "Directive kind mismatch");
+
+    if (static_cast<int>(version) >= versions.Min) {
+      continue;
+    }
+    if (ref == removeSpaces(name)) {
+      context_.Say(spelling,
+          "Directive spelling '%s' is introduced in a later OpenMP version, %s"_warn_en_US,
+          parser::ToUpperCaseLetters(ref), TryVersion(versions.Min));
+      break;
+    }
+  }
+}
+
 void OmpStructureChecker::CheckMultipleOccurrence(
     semantics::UnorderedSymbolSet &listVars,
     const std::list<parser::Name> &nameList, const parser::CharBlock &item,
@@ -436,7 +485,133 @@ void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
   }
 }
 
+template <typename Checker> struct DirectiveSpellingVisitor {
+  using Directive = llvm::omp::Directive;
+
+  DirectiveSpellingVisitor(Checker &&checker) : checker_(std::move(checker)) {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::OmpSectionsDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclarativeAllocate &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocate);
+    return false;
+  }
+  bool Pre(const parser::OmpDispatchDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_dispatch);
+    return false;
+  }
+  bool Pre(const parser::OmpErrorDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_error);
+    return false;
+  }
+  bool Pre(const parser::OmpNothingDirective &x) {
+    checker_(x.source, Directive::OMPD_nothing);
+    return false;
+  }
+  bool Pre(const parser::OpenMPExecutableAllocate &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocate);
+    return false;
+  }
+  bool Pre(const parser::OpenMPAllocatorsConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocators);
+    return false;
+  }
+  bool Pre(const parser::OmpAssumeDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_assume);
+    return false;
+  }
+  bool Pre(const parser::OmpEndAssumeDirective &x) {
+    checker_(x.v.source, Directive::OMPD_assume);
+    return false;
+  }
+  bool Pre(const parser::OmpCriticalDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_critical);
+    return false;
+  }
+  bool Pre(const parser::OmpEndCriticalDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_critical);
+    return false;
+  }
+  bool Pre(const parser::OmpMetadirectiveDirective &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_metadirective);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclarativeAssumes &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_assumes);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareMapperConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_mapper);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareReductionConstruct &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source,
+        Directive::OMPD_declare_reduction);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareSimdConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_simd);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareTargetConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_target);
+    return false;
+  }
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source,
+        Directive::OMPD_declare_variant);
+    return false;
+  }
+  bool Pre(const parser::OpenMPThreadprivate &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_threadprivate);
+    return false;
+  }
+  bool Pre(const parser::OpenMPRequiresConstruct &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_requires);
+    return false;
+  }
+
+  bool Pre(const parser::OmpBlockDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+
+  bool Pre(const parser::OmpLoopDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+
+  bool Pre(const parser::OmpDirectiveSpecification &x) {
+    auto &name = std::get<parser::OmpDirectiveName>(x.t);
+    checker_(name.source, name.v);
+    return false;
+  }
+
+private:
+  Checker checker_;
+};
+
+template <typename T>
+DirectiveSpellingVisitor(T &&) -> DirectiveSpellingVisitor<T>;
+
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
+  DirectiveSpellingVisitor visitor(
+      [this](parser::CharBlock source, llvm::omp::Directive id) {
+        return CheckDirectiveSpelling(source, id);
+      });
+  parser::Walk(x, visitor);
+
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
   // PushContextAndClauseSets() has not been called yet, it is
@@ -461,6 +636,12 @@ void OmpStructureChecker::Leave(const parser::OpenMPConstruct &) {
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPDeclarativeConstruct &x) {
+  DirectiveSpellingVisitor visitor(
+      [this](parser::CharBlock source, llvm::omp::Directive id) {
+        return CheckDirectiveSpelling(source, id);
+      });
+  parser::Walk(x, visitor);
+
   EnterDirectiveNest(DeclarativeNest);
 }
 
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 2a3853335fd1c..6a877a5d0a7c0 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -163,6 +163,8 @@ class OmpStructureChecker
 private:
   bool CheckAllowedClause(llvmOmpClause clause);
   void CheckVariableListItem(const SymbolSourceMap &symbols);
+  void CheckDirectiveSpelling(
+      parser::CharBlock spelling, llvm::omp::Directive id);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
       const std::string &clauseName);
diff --git a/flang/test/Semantics/OpenMP/future-directive-spellings.f90 b/flang/test/Semantics/OpenMP/future-directive-spellings.f90
new file mode 100644
index 0000000000000..ed7c4d6782fbe
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/future-directive-spellings.f90
@@ -0,0 +1,92 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52 -Werror
+
+! The directives to check:
+!   cancellation_point
+!   declare_mapper
+!   declare_reduction
+!   declare_simd
+!   declare_target
+!   declare_variant
+!   target_data
+!   target_enter_data
+!   target_exit_data
+!   target_update
+
+subroutine f00
+  implicit none
+  integer :: i
+
+  !$omp parallel
+  do i = 1, 10
+!WARNING: Directive spelling 'CANCELLATION_POINT' is introduced in a later OpenMP version, try -fopenmp-version=60
+    !$omp cancellation_point parallel
+  enddo
+  !$omp end parallel
+end
+
+subroutine f01
+  type :: t
+    integer :: x
+  end type
+!WARNING: Directive spelling 'DECLARE_MAPPER' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_mapper(t :: v) map(v%x)
+end
+
+subroutine f02
+  type :: t
+    integer :: x
+  end type
+!WARNING: Directive spelling 'DECLARE_REDUCTION' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_reduction(+ : t : omp_out%x = omp_out%x + omp_in%x)
+end
+
+subroutine f03
+!WARNING: Directive spelling 'DECLARE_SIMD' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_simd
+end
+
+subroutine f04
+!WARNING: Directive spelling 'DECLARE_TARGET' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_target
+end
+
+subroutine f05
+  implicit none
+  interface
+    subroutine g05
+    end
+  end interface
+!WARNING: Directive spelling 'DECLARE_VARIANT' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_variant(g05) match(user={condition(.true.)})
+end
+
+subroutine f06
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_data map(tofrom: i)
+  i = 0
+!WARNING: Directive spelling 'TARGET_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp end target_data
+end
+
+subroutine f07
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_ENTER_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_enter_data map(to: i)
+end
+
+subroutine f08
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_EXIT_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_exit_data map(from: i)
+end
+
+subroutine f09
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_UPDATE' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_update to(i)
+end


From flang-commits at lists.llvm.org  Thu Jul 10 06:53:07 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 06:53:07 -0700 (PDT)
Subject: [flang-commits] [flang] 13ead00 - [Flang] Fix PowerPC build failure
 due to the deprecation of ArrayRef(std::nullopt_t) {}. (#147816)
Message-ID: <686fc5c3.630a0220.1cf9c9.48ad@mx.google.com>


Author: Daniel Chen
Date: 2025-07-10T09:53:03-04:00
New Revision: 13ead00049e2faea57dac2d9a43f2aef5bbe3370

URL: https://github.com/llvm/llvm-project/commit/13ead00049e2faea57dac2d9a43f2aef5bbe3370
DIFF: https://github.com/llvm/llvm-project/commit/13ead00049e2faea57dac2d9a43f2aef5bbe3370.diff

LOG: [Flang] Fix PowerPC build failure due to the deprecation of ArrayRef(std::nullopt_t) {}. (#147816)

Our local Flang build on PowerPC was broken as
```
llvm/flang/../mlir/include/mlir/IR/ValueRange.h:401:20: error: 'ArrayRef' is deprecated: Use {} or ArrayRef<T>() instead [-Werror,-Wdeprecated-declarations]
  401 |       : ValueRange(ArrayRef<Value>(std::forward<Arg>(arg))) {}
      |                    ^
llvm/flang/lib/Optimizer/CodeGen/CodeGen.cpp:2243:53: note: in instantiation of function template specialization 'mlir::ValueRange::ValueRange<const std::nullopt_t &, void>' requested here
 2243 |                              /*cstInteriorIndices=*/std::nullopt, fieldIndices,
      |                                                     ^
 llvm/include/llvm/ADT/ArrayRef.h:70:18: note: 'ArrayRef' has been explicitly marked deprecated here
   70 |     /*implicit*/ LLVM_DEPRECATED("Use {} or ArrayRef<T>() instead", "{}")
      |                  ^
llvm/include/llvm/Support/Compiler.h:244:50: note: expanded from macro 'LLVM_DEPRECATED'
  244 | #define LLVM_DEPRECATED(MSG, FIX) __attribute__((deprecated(MSG, FIX)))
      |                                                  ^
1 error generated.
```

This patch is to fix it.

Added: 
    

Modified: 
    flang/lib/Optimizer/CodeGen/CodeGen.cpp
    flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 1bc981516e226..3bbc32f23bcfa 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -2239,17 +2239,18 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
         getSubcomponentIndices(rebox, rebox.getBox(), operands, fieldIndices);
       if (!rebox.getSubstr().empty())
         substringOffset = operands[rebox.getSubstrOperandIndex()];
-      base = genBoxOffsetGep(rewriter, loc, base, llvmBaseObjectType, zero,
-                             /*cstInteriorIndices=*/std::nullopt, fieldIndices,
-                             substringOffset);
+      base =
+          genBoxOffsetGep(rewriter, loc, base, llvmBaseObjectType, zero,
+                          /*cstInteriorIndices=*/llvm::ArrayRef<mlir::Value>(),
+                          fieldIndices, substringOffset);
     }
 
     if (rebox.getSlice().empty())
       // The array section is of the form array[%component][substring], keep
       // the input array extents and strides.
       return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
-                           /*lbounds*/ std::nullopt, inputExtents, inputStrides,
-                           rewriter);
+                           /*lbounds*/ llvm::ArrayRef<mlir::Value>(),
+                           inputExtents, inputStrides, rewriter);
 
     // The slice is of the form array(i:j:k)[%component]. Compute new extents
     // and strides.
@@ -2297,8 +2298,8 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       }
     }
     return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
-                         /*lbounds*/ std::nullopt, slicedExtents, slicedStrides,
-                         rewriter);
+                         /*lbounds*/ llvm::ArrayRef<mlir::Value>(),
+                         slicedExtents, slicedStrides, rewriter);
   }
 
   /// Apply a new shape to the data described by a box given the base address,
@@ -3396,7 +3397,8 @@ static void genBrOp(A caseOp, mlir::Block *dest, std::optional<B> destOps,
   if (destOps)
     rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, *destOps, dest);
   else
-    rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, std::nullopt, dest);
+    rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(
+        caseOp, llvm::ArrayRef<mlir::Value>(), dest);
 }
 
 static void genCaseLadderStep(mlir::Location loc, mlir::Value cmp,

diff  --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index d09d7d397e8b7..eca2c7f7c942f 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -107,9 +107,11 @@ class EmboxConversion : public mlir::OpRewritePattern<fir::EmboxOp> {
       shapeOpers.push_back(extVal);
     }
     auto xbox = rewriter.create<fir::cg::XEmboxOp>(
-        loc, embox.getType(), embox.getMemref(), shapeOpers, std::nullopt,
-        std::nullopt, std::nullopt, std::nullopt, embox.getTypeparams(),
-        embox.getSourceBox(), embox.getAllocatorIdxAttr());
+        loc, embox.getType(), embox.getMemref(), shapeOpers,
+        llvm::ArrayRef<mlir::Value>(), llvm::ArrayRef<mlir::Value>(),
+        llvm::ArrayRef<mlir::Value>(), llvm::ArrayRef<mlir::Value>(),
+        embox.getTypeparams(), embox.getSourceBox(),
+        embox.getAllocatorIdxAttr());
     LLVM_DEBUG(llvm::dbgs() << "rewriting " << embox << " to " << xbox << '\n');
     rewriter.replaceOp(embox, xbox.getOperation()->getResults());
     return mlir::success();


        

From flang-commits at lists.llvm.org  Thu Jul 10 06:53:10 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Thu, 10 Jul 2025 06:53:10 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix PowerPC build failure due to
 the deprecation of ArrayRef(std::nullopt_t) {}. (PR #147816)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147816@github.com>
Message-ID: <686fc5c6.630a0220.27e319.4997@mx.google.com>


https://github.com/DanielCChen closed https://github.com/llvm/llvm-project/pull/147816

From flang-commits at lists.llvm.org  Thu Jul 10 06:59:24 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 06:59:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fc73c.a70a0220.7a46.5229@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/147765

>From 1419c89670953b969fcc1c474a258c7475d8e7df Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 9 May 2025 07:45:20 -0500
Subject: [PATCH 1/6] [flang][OpenMP] Handle multiple spellings in
 OmpDirectiveNameParser

Collect all spellings from all supported OpenMP versions before parsing.
Break up the list of spellings by the initial letter to speed up parsing
a little.
---
 flang/lib/Parser/openmp-parsers.cpp | 49 +++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 30b5d4f9513a7..29701a616d4b0 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
 // OpenMP Directives and Clauses
@@ -104,8 +105,13 @@ struct OmpDirectiveNameParser {
   using Token = TokenStringMatch<false, false>;
 
   std::optional<resultType> Parse(ParseState &state) const {
+    if (state.BytesRemaining() == 0) {
+      return std::nullopt;
+    }
     auto begin{state.GetLocation()};
-    for (const NameWithId &nid : directives()) {
+    char next{static_cast<char>(std::tolower(*begin))};
+
+    for (const NameWithId &nid : directives_starting_with(next)) {
       if (attempt(Token(nid.first.data())).Parse(state)) {
         OmpDirectiveName n;
         n.v = nid.second;
@@ -118,30 +124,47 @@ struct OmpDirectiveNameParser {
 
 private:
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
+  using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<const NameWithId *> directives() const;
-  void initTokens(NameWithId *) const;
+  llvm::iterator_range<ConstIterator>
+  directives_starting_with(char initial) const;
+  void initTokens(std::vector<NameWithId>[]) const;
 };
 
-llvm::iterator_range<const OmpDirectiveNameParser::NameWithId *>
-OmpDirectiveNameParser::directives() const {
-  static NameWithId table[llvm::omp::Directive_enumSize];
+llvm::iterator_range<OmpDirectiveNameParser::ConstIterator>
+OmpDirectiveNameParser::directives_starting_with(char initial) const {
+  static const std::vector<NameWithId> empty{};
+  if (initial < 'a' || initial > 'z') {
+    return llvm::make_range(std::cbegin(empty), std::cend(empty));
+  }
+
+  static std::vector<NameWithId> table['z' - 'a' + 1];
   [[maybe_unused]] static bool init = (initTokens(table), true);
-  return llvm::make_range(std::cbegin(table), std::cend(table));
+
+  int index = initial - 'a';
+  return llvm::make_range(std::cbegin(table[index]), std::cend(table[index]));
 }
 
-void OmpDirectiveNameParser::initTokens(NameWithId *table) const {
+void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   for (size_t i{0}, e{llvm::omp::Directive_enumSize}; i != e; ++i) {
+    llvm::StringSet spellings;
     auto id{static_cast<llvm::omp::Directive>(i)};
-    llvm::StringRef name{
-        llvm::omp::getOpenMPDirectiveName(id, llvm::omp::FallbackVersion)};
-    table[i] = std::make_pair(name.str(), id);
+    for (unsigned version : llvm::omp::getOpenMPVersions()) {
+      spellings.insert(llvm::omp::getOpenMPDirectiveName(id, version));
+    }
+    for (auto &[name, _] : spellings) {
+      char initial{static_cast<char>(std::tolower(name.front()))};
+      table[initial - 'a'].emplace_back(name.str(), id);
+    }
   }
   // Sort the table with respect to the directive name length in a descending
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
-  std::sort(table, table + llvm::omp::Directive_enumSize,
-      [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
+  for (int initial{'a'}; initial != 'z' + 1; ++initial) {
+    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
+      return a.first.size() > b.first.size();
+    });
+  }
 }
 
 // --- Modifier helpers -----------------------------------------------

>From 0e2c062d467d7d23ed88cdf14c2c3de740549ca6 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 7 Jul 2025 13:54:32 -0500
Subject: [PATCH 2/6] [flang][OpenMP] Recognize remaining OpenMP 6.0 spellings
 in parser

Parse OpenMP 6.0 spellings for directives that don't use
OmpDirectiveNameParser.
---
 flang/lib/Parser/openmp-parsers.cpp           |  17 +-
 .../OpenMP/openmp6-directive-spellings.f90    | 252 ++++++++++++++++++
 2 files changed, 263 insertions(+), 6 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/openmp6-directive-spellings.f90

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 29701a616d4b0..b25445b0aaef6 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1501,7 +1501,7 @@ TYPE_PARSER(
     // In this context "TARGET UPDATE" can be parsed as a TARGET directive
     // followed by an UPDATE clause. This is the only combination at the
     // moment, exclude it explicitly.
-    (!"TARGET UPDATE"_sptok) >=
+    (!("TARGET UPDATE"_sptok || "TARGET_UPDATE"_sptok)) >=
     construct<OmpBlockDirective>(first(
         "MASKED" >> pure(llvm::omp::Directive::OMPD_masked),
         "MASTER" >> pure(llvm::omp::Directive::OMPD_master),
@@ -1514,6 +1514,7 @@ TYPE_PARSER(
         "SCOPE" >> pure(llvm::omp::Directive::OMPD_scope),
         "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
+        "TARGET_DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
@@ -1534,12 +1535,13 @@ TYPE_PARSER(construct<OmpInitializerClause>(
 
 // OpenMP 5.2: 7.5.4 Declare Variant directive
 TYPE_PARSER(sourced(
-    construct<OmpDeclareVariantDirective>(verbatim("DECLARE VARIANT"_tok),
+    construct<OmpDeclareVariantDirective>(
+        verbatim("DECLARE VARIANT"_tok) || verbatim("DECLARE_VARIANT"_tok),
         "(" >> maybe(name / ":"), name / ")", Parser<OmpClauseList>{})))
 
 // 2.16 Declare Reduction Construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareReductionConstruct>(
-    verbatim("DECLARE REDUCTION"_tok),
+    verbatim("DECLARE REDUCTION"_tok) || verbatim("DECLARE_REDUCTION"_tok),
     "(" >> indirect(Parser<OmpReductionSpecifier>{}) / ")",
     maybe(Parser<OmpClauseList>{}))))
 
@@ -1558,7 +1560,8 @@ TYPE_PARSER(
 
 // 2.10.6 Declare Target Construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareTargetConstruct>(
-    verbatim("DECLARE TARGET"_tok), Parser<OmpDeclareTargetSpecifier>{})))
+    verbatim("DECLARE TARGET"_tok) || verbatim("DECLARE_TARGET"_tok),
+    Parser<OmpDeclareTargetSpecifier>{})))
 
 static OmpMapperSpecifier ConstructOmpMapperSpecifier(
     std::optional<Name> &&mapperName, TypeSpec &&typeSpec, Name &&varName) {
@@ -1586,7 +1589,8 @@ TYPE_PARSER(applyFunction<OmpMapperSpecifier>(ConstructOmpMapperSpecifier,
 
 // OpenMP 5.2: 5.8.8 Declare Mapper Construct
 TYPE_PARSER(sourced(
-    construct<OpenMPDeclareMapperConstruct>(verbatim("DECLARE MAPPER"_tok),
+    construct<OpenMPDeclareMapperConstruct>(
+        verbatim("DECLARE MAPPER"_tok) || verbatim("DECLARE_MAPPER"_tok),
         parenthesized(Parser<OmpMapperSpecifier>{}), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(construct<OmpReductionCombiner>(Parser<AssignmentStmt>{}) ||
@@ -1633,7 +1637,8 @@ TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
 
 // 2.8.2 Declare Simd construct
 TYPE_PARSER(
-    sourced(construct<OpenMPDeclareSimdConstruct>(verbatim("DECLARE SIMD"_tok),
+    sourced(construct<OpenMPDeclareSimdConstruct>(
+        verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
         maybe(parenthesized(name)), Parser<OmpClauseList>{})))
 
 // 2.4 Requires construct
diff --git a/flang/test/Parser/OpenMP/openmp6-directive-spellings.f90 b/flang/test/Parser/OpenMP/openmp6-directive-spellings.f90
new file mode 100644
index 0000000000000..6a1d565451220
--- /dev/null
+++ b/flang/test/Parser/OpenMP/openmp6-directive-spellings.f90
@@ -0,0 +1,252 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=60 %s -o - | FileCheck --check-prefix=UNPARSE %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=60 %s -o - | FileCheck --check-prefix=PARSE-TREE %s
+
+! The directives to check:
+!   cancellation_point
+!   declare_mapper
+!   declare_reduction
+!   declare_simd
+!   declare_target
+!   declare_variant
+!   target_data
+!   target_enter_data
+!   target_exit_data
+!   target_update
+
+subroutine f00
+  implicit none
+  integer :: i
+
+  !$omp parallel
+  do i = 1, 10
+    !$omp cancellation_point parallel
+  enddo
+  !$omp end parallel
+end
+
+!UNPARSE: SUBROUTINE f00
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP PARALLEL
+!UNPARSE:  DO i=1_4,10_4
+!UNPARSE: !$OMP CANCELLATION_POINT PARALLEL
+!UNPARSE:  END DO
+!UNPARSE: !$OMP END PARALLEL
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPCancellationPointConstruct -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = cancellation point
+!PARSE-TREE: | OmpClauseList -> OmpClause -> CancellationConstructType -> OmpCancellationConstructTypeClause
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = parallel
+!PARSE-TREE: | Flags = None
+
+subroutine f01
+  type :: t
+    integer :: x
+  end type
+  !$omp declare_mapper(t :: v) map(v%x)
+end
+
+!UNPARSE: SUBROUTINE f01
+!UNPARSE:  TYPE :: t
+!UNPARSE:   INTEGER :: x
+!UNPARSE:  END TYPE
+!UNPARSE: !$OMP DECLARE MAPPER (t::v) MAP(v%x)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareMapperConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpMapperSpecifier
+!PARSE-TREE: | | string = 't.omp.default.mapper'
+!PARSE-TREE: | | TypeSpec -> DerivedTypeSpec
+!PARSE-TREE: | | | Name = 't'
+!PARSE-TREE: | | Name = 'v'
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> StructureComponent
+!PARSE-TREE: | | | DataRef -> Name = 'v'
+!PARSE-TREE: | | | Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02
+  type :: t
+    integer :: x
+  end type
+  !$omp declare_reduction(+ : t : omp_out%x = omp_out%x + omp_in%x)
+end
+
+!UNPARSE: SUBROUTINE f02
+!UNPARSE:  TYPE :: t
+!UNPARSE:   INTEGER :: x
+!UNPARSE:  END TYPE
+!UNPARSE: !$OMP DECLARE REDUCTION (+:t: omp_out%x=omp_out%x+omp_in%x
+!UNPARSE: )
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareReductionConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpReductionSpecifier
+!PARSE-TREE: | | OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
+!PARSE-TREE: | | OmpTypeNameList -> OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
+!PARSE-TREE: | | | Name = 't'
+!PARSE-TREE: | | OmpReductionCombiner -> AssignmentStmt = 'omp_out%x=omp_out%x+omp_in%x'
+!PARSE-TREE: | | | Variable = 'omp_out%x'
+!PARSE-TREE: | | | | Designator -> DataRef -> StructureComponent
+!PARSE-TREE: | | | | | DataRef -> Name = 'omp_out'
+!PARSE-TREE: | | | | | Name = 'x'
+!PARSE-TREE: | | | Expr = 'omp_out%x+omp_in%x'
+!PARSE-TREE: | | | | Add
+!PARSE-TREE: | | | | | Expr = 'omp_out%x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> StructureComponent
+!PARSE-TREE: | | | | | | | DataRef -> Name = 'omp_out'
+!PARSE-TREE: | | | | | | | Name = 'x'
+!PARSE-TREE: | | | | | Expr = 'omp_in%x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> StructureComponent
+!PARSE-TREE: | | | | | | | DataRef -> Name = 'omp_in'
+!PARSE-TREE: | | | | | | | Name = 'x'
+!PARSE-TREE: | OmpClauseList ->
+
+subroutine f03
+  !$omp declare_simd
+end
+
+!UNPARSE: SUBROUTINE f03
+!UNPARSE: !$OMP DECLARE SIMD
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OpenMPDeclareSimdConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpClauseList ->
+
+subroutine f04
+  !$omp declare_target
+end
+
+!UNPARSE: SUBROUTINE f04
+!UNPARSE: !$OMP DECLARE TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OpenMPDeclareTargetConstruct
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | OmpDeclareTargetSpecifier -> OmpDeclareTargetWithClause -> OmpClauseList ->
+
+subroutine f05
+  implicit none
+  interface
+    subroutine g05
+    end
+  end interface
+  !$omp declare_variant(g05) match(user={condition(.true.)})
+end
+
+!UNPARSE: SUBROUTINE f05
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTERFACE
+!UNPARSE:   SUBROUTINE g05
+!UNPARSE:   END SUBROUTINE
+!UNPARSE:  END INTERFACE
+!UNPARSE: !$OMP DECLARE VARIANT (g05) MATCH(USER={CONDITION(.true._4)})
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OpenMPDeclarativeConstruct -> OmpDeclareVariantDirective
+!PARSE-TREE: | Verbatim
+!PARSE-TREE: | Name = 'g05'
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Match -> OmpMatchClause -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | OmpTraitSelector
+!PARSE-TREE: | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | Properties
+!PARSE-TREE: | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | bool = 'true'
+
+subroutine f06
+  implicit none
+  integer :: i
+  !$omp target_data map(tofrom: i)
+  i = 0
+  !$omp end target data
+end
+
+!UNPARSE: SUBROUTINE f06
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP TARGET DATA  MAP(TOFROM: i)
+!UNPARSE:   i=0_4
+!UNPARSE: !$OMP END TARGET DATA
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+!PARSE-TREE: | OmpBeginBlockDirective
+!PARSE-TREE: | | OmpBlockDirective -> llvm::omp::Directive = target data
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | | Modifier -> OmpMapType -> Value = Tofrom
+!PARSE-TREE: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | | bool = 'true'
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'i=0_4'
+!PARSE-TREE: | | | Variable = 'i'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | | Expr = '0_4'
+!PARSE-TREE: | | | | LiteralConstant -> IntLiteralConstant = '0'
+!PARSE-TREE: | OmpEndBlockDirective
+!PARSE-TREE: | | OmpBlockDirective -> llvm::omp::Directive = target data
+!PARSE-TREE: | | OmpClauseList ->
+
+subroutine f07
+  implicit none
+  integer :: i
+  !$omp target_enter_data map(to: i)
+end
+
+!UNPARSE: SUBROUTINE f07
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP TARGET_ENTER_DATA MAP(TO: i)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target enter data
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | bool = 'true'
+!PARSE-TREE: | Flags = None
+
+subroutine f08
+  implicit none
+  integer :: i
+  !$omp target_exit_data map(from: i)
+end
+
+!UNPARSE: SUBROUTINE f08
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP TARGET_EXIT_DATA MAP(FROM: i)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target exit data
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = From
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | bool = 'true'
+!PARSE-TREE: | Flags = None
+
+subroutine f09
+  implicit none
+  integer :: i
+  !$omp target_update to(i)
+end
+
+!UNPARSE: SUBROUTINE f09
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER i
+!UNPARSE: !$OMP TARGET_UPDATE TO(i)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target update
+!PARSE-TREE: | OmpClauseList -> OmpClause -> To -> OmpToClause
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'i'
+!PARSE-TREE: | | bool = 'true'
+!PARSE-TREE: | Flags = None

>From 1dd6916ae6c7bd360e1c603b0cb9fd3b942233fc Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 9 Jul 2025 08:22:33 -0500
Subject: [PATCH 3/6] format

---
 flang/lib/Parser/openmp-parsers.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 29701a616d4b0..fdf96dffeb8ed 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -126,8 +126,8 @@ struct OmpDirectiveNameParser {
   using NameWithId = std::pair<std::string, llvm::omp::Directive>;
   using ConstIterator = std::vector<NameWithId>::const_iterator;
 
-  llvm::iterator_range<ConstIterator>
-  directives_starting_with(char initial) const;
+  llvm::iterator_range<ConstIterator> directives_starting_with(
+      char initial) const;
   void initTokens(std::vector<NameWithId>[]) const;
 };
 
@@ -161,9 +161,8 @@ void OmpDirectiveNameParser::initTokens(std::vector<NameWithId> table[]) const {
   // order. This is to make sure that longer names are tried first, before
   // any potential prefix (e.g. "target update" before "target").
   for (int initial{'a'}; initial != 'z' + 1; ++initial) {
-    llvm::stable_sort(table[initial - 'a'], [](auto &a, auto &b) {
-      return a.first.size() > b.first.size();
-    });
+    llvm::stable_sort(table[initial - 'a'],
+        [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
   }
 }
 

>From a69ffbae3a4eaa3cf4b0505808e693fcf416eb00 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 9 Jul 2025 08:24:58 -0500
Subject: [PATCH 4/6] format

---
 flang/lib/Parser/openmp-parsers.cpp | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index e8e1d5ac16e7a..ee88cce288cae 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1533,10 +1533,9 @@ TYPE_PARSER(construct<OmpInitializerClause>(
     construct<OmpInitializerClause>(Parser<OmpInitializerProc>{})))
 
 // OpenMP 5.2: 7.5.4 Declare Variant directive
-TYPE_PARSER(sourced(
-    construct<OmpDeclareVariantDirective>(
-        verbatim("DECLARE VARIANT"_tok) || verbatim("DECLARE_VARIANT"_tok),
-        "(" >> maybe(name / ":"), name / ")", Parser<OmpClauseList>{})))
+TYPE_PARSER(sourced(construct<OmpDeclareVariantDirective>(
+    verbatim("DECLARE VARIANT"_tok) || verbatim("DECLARE_VARIANT"_tok),
+    "(" >> maybe(name / ":"), name / ")", Parser<OmpClauseList>{})))
 
 // 2.16 Declare Reduction Construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareReductionConstruct>(
@@ -1587,10 +1586,9 @@ TYPE_PARSER(applyFunction<OmpMapperSpecifier>(ConstructOmpMapperSpecifier,
     maybe(name / ":" / !":"_tok), typeSpec / "::", name))
 
 // OpenMP 5.2: 5.8.8 Declare Mapper Construct
-TYPE_PARSER(sourced(
-    construct<OpenMPDeclareMapperConstruct>(
-        verbatim("DECLARE MAPPER"_tok) || verbatim("DECLARE_MAPPER"_tok),
-        parenthesized(Parser<OmpMapperSpecifier>{}), Parser<OmpClauseList>{})))
+TYPE_PARSER(sourced(construct<OpenMPDeclareMapperConstruct>(
+    verbatim("DECLARE MAPPER"_tok) || verbatim("DECLARE_MAPPER"_tok),
+    parenthesized(Parser<OmpMapperSpecifier>{}), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(construct<OmpReductionCombiner>(Parser<AssignmentStmt>{}) ||
     construct<OmpReductionCombiner>(Parser<FunctionReference>{}))
@@ -1635,10 +1633,9 @@ TYPE_PARSER(sourced(construct<OpenMPAllocatorsConstruct>(
 TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
 
 // 2.8.2 Declare Simd construct
-TYPE_PARSER(
-    sourced(construct<OpenMPDeclareSimdConstruct>(
-        verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
-        maybe(parenthesized(name)), Parser<OmpClauseList>{})))
+TYPE_PARSER(sourced(construct<OpenMPDeclareSimdConstruct>(
+    verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
+    maybe(parenthesized(name)), Parser<OmpClauseList>{})))
 
 // 2.4 Requires construct
 TYPE_PARSER(sourced(construct<OpenMPRequiresConstruct>(

>From 7b42bbe446f366fa10e1fb6ecf237169a56400b0 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Tue, 8 Jul 2025 18:06:48 -0500
Subject: [PATCH 5/6] [flang][OpenMP] Issue a warning when parsing future
 directive spelling

OpenMP 6.0 introduced alternative spelling for some directives, with the
previous spellings still allowed.

Warn the user when a new spelling is encountered with OpenMP version set
to an older value.
---
 flang/include/flang/Parser/parse-tree.h       |   2 +-
 flang/lib/Parser/openmp-parsers.cpp           |   4 +-
 flang/lib/Semantics/check-omp-structure.cpp   | 181 ++++++++++++++++++
 flang/lib/Semantics/check-omp-structure.h     |   2 +
 .../OpenMP/future-directive-spellings.f90     |  92 +++++++++
 5 files changed, 278 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Semantics/OpenMP/future-directive-spellings.f90

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 7e752eeb4dfe4..43954ff735361 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4612,7 +4612,7 @@ struct OmpDirectiveSpecification {
 
 struct OmpMetadirectiveDirective {
   TUPLE_CLASS_BOILERPLATE(OmpMetadirectiveDirective);
-  std::tuple<OmpClauseList> t;
+  std::tuple<Verbatim, OmpClauseList> t;
   CharBlock source;
 };
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index ee88cce288cae..3016ce4ccd2f8 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1194,7 +1194,7 @@ TYPE_PARSER(sourced(construct<OpenMPUtilityConstruct>(
         sourced(Parser<OmpNothingDirective>{}))))))
 
 TYPE_PARSER(sourced(construct<OmpMetadirectiveDirective>(
-    "METADIRECTIVE" >> Parser<OmpClauseList>{})))
+    verbatim("METADIRECTIVE"_tok), Parser<OmpClauseList>{})))
 
 // Omp directives enclosing do loop
 TYPE_PARSER(sourced(construct<OmpLoopDirective>(first(
@@ -1687,7 +1687,7 @@ TYPE_PARSER(sourced(construct<OmpAssumeDirective>(
     verbatim("ASSUME"_tok), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(sourced(construct<OmpEndAssumeDirective>(
-    verbatim(startOmpLine >> "END ASSUME"_tok))))
+    startOmpLine >> verbatim("END ASSUME"_tok))))
 
 TYPE_PARSER(sourced(
     construct<OpenMPAssumeConstruct>(Parser<OmpAssumeDirective>{} / endOmpLine,
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index aa4cdb9d27019..191f13cdeef65 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -259,6 +259,55 @@ void OmpStructureChecker::CheckVariableListItem(
   }
 }
 
+void OmpStructureChecker::CheckDirectiveSpelling(
+    parser::CharBlock spelling, llvm::omp::Directive id) {
+  // Directive names that contain spaces can be spelled in the source without
+  // any of the spaces. Because of that getOpenMPKind* is not guaranteed to
+  // work with the source spelling as the argument.
+  //
+  // To verify the source spellings, we have to get the spelling for a given
+  // version, remove spaces and compare it with the source spelling (also
+  // with spaces removed).
+  auto removeSpaces = [](llvm::StringRef s) {
+    std::string n{s.str()};
+    for (auto it{n.begin()}; it != n.end();) {
+      it = isspace(*it) ? n.erase(it) : std::next(it);
+    }
+    return n;
+  };
+
+  std::string lowerNoWS{removeSpaces(
+      parser::ToLowerCaseLetters({spelling.begin(), spelling.size()}))};
+  llvm::StringRef ref(lowerNoWS);
+  if (ref.starts_with("end")) {
+    ref = ref.drop_front(3);
+  }
+
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  // For every "future" version v, check if the check if the corresponding
+  // spelling of id was introduced later than the current version. If so,
+  // and if that spelling matches the source spelling, issue a warning.
+  for (unsigned v : llvm::omp::getOpenMPVersions()) {
+    if (v <= version) {
+      continue;
+    }
+    llvm::StringRef name{llvm::omp::getOpenMPDirectiveName(id, v)};
+    auto [kind, versions]{llvm::omp::getOpenMPDirectiveKindAndVersions(name)};
+    assert(kind == id && "Directive kind mismatch");
+
+    if (static_cast<int>(version) >= versions.Min) {
+      continue;
+    }
+    if (ref == removeSpaces(name)) {
+      context_.Say(spelling,
+          "Directive spelling '%s' is introduced in a later OpenMP version, %s"_warn_en_US,
+          parser::ToUpperCaseLetters(ref), TryVersion(versions.Min));
+      break;
+    }
+  }
+}
+
 void OmpStructureChecker::CheckMultipleOccurrence(
     semantics::UnorderedSymbolSet &listVars,
     const std::list<parser::Name> &nameList, const parser::CharBlock &item,
@@ -436,7 +485,133 @@ void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
   }
 }
 
+template <typename Checker> struct DirectiveSpellingVisitor {
+  using Directive = llvm::omp::Directive;
+
+  DirectiveSpellingVisitor(Checker &&checker) : checker_(std::move(checker)) {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::OmpSectionsDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclarativeAllocate &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocate);
+    return false;
+  }
+  bool Pre(const parser::OmpDispatchDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_dispatch);
+    return false;
+  }
+  bool Pre(const parser::OmpErrorDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_error);
+    return false;
+  }
+  bool Pre(const parser::OmpNothingDirective &x) {
+    checker_(x.source, Directive::OMPD_nothing);
+    return false;
+  }
+  bool Pre(const parser::OpenMPExecutableAllocate &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocate);
+    return false;
+  }
+  bool Pre(const parser::OpenMPAllocatorsConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocators);
+    return false;
+  }
+  bool Pre(const parser::OmpAssumeDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_assume);
+    return false;
+  }
+  bool Pre(const parser::OmpEndAssumeDirective &x) {
+    checker_(x.v.source, Directive::OMPD_assume);
+    return false;
+  }
+  bool Pre(const parser::OmpCriticalDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_critical);
+    return false;
+  }
+  bool Pre(const parser::OmpEndCriticalDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_critical);
+    return false;
+  }
+  bool Pre(const parser::OmpMetadirectiveDirective &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_metadirective);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclarativeAssumes &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_assumes);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareMapperConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_mapper);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareReductionConstruct &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source,
+        Directive::OMPD_declare_reduction);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareSimdConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_simd);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareTargetConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_target);
+    return false;
+  }
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source,
+        Directive::OMPD_declare_variant);
+    return false;
+  }
+  bool Pre(const parser::OpenMPThreadprivate &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_threadprivate);
+    return false;
+  }
+  bool Pre(const parser::OpenMPRequiresConstruct &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_requires);
+    return false;
+  }
+
+  bool Pre(const parser::OmpBlockDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+
+  bool Pre(const parser::OmpLoopDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+
+  bool Pre(const parser::OmpDirectiveSpecification &x) {
+    auto &name = std::get<parser::OmpDirectiveName>(x.t);
+    checker_(name.source, name.v);
+    return false;
+  }
+
+private:
+  Checker checker_;
+};
+
+template <typename T>
+DirectiveSpellingVisitor(T &&) -> DirectiveSpellingVisitor<T>;
+
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
+  DirectiveSpellingVisitor visitor(
+      [this](parser::CharBlock source, llvm::omp::Directive id) {
+        return CheckDirectiveSpelling(source, id);
+      });
+  parser::Walk(x, visitor);
+
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
   // PushContextAndClauseSets() has not been called yet, it is
@@ -461,6 +636,12 @@ void OmpStructureChecker::Leave(const parser::OpenMPConstruct &) {
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPDeclarativeConstruct &x) {
+  DirectiveSpellingVisitor visitor(
+      [this](parser::CharBlock source, llvm::omp::Directive id) {
+        return CheckDirectiveSpelling(source, id);
+      });
+  parser::Walk(x, visitor);
+
   EnterDirectiveNest(DeclarativeNest);
 }
 
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 2a3853335fd1c..6a877a5d0a7c0 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -163,6 +163,8 @@ class OmpStructureChecker
 private:
   bool CheckAllowedClause(llvmOmpClause clause);
   void CheckVariableListItem(const SymbolSourceMap &symbols);
+  void CheckDirectiveSpelling(
+      parser::CharBlock spelling, llvm::omp::Directive id);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
       const std::string &clauseName);
diff --git a/flang/test/Semantics/OpenMP/future-directive-spellings.f90 b/flang/test/Semantics/OpenMP/future-directive-spellings.f90
new file mode 100644
index 0000000000000..ed7c4d6782fbe
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/future-directive-spellings.f90
@@ -0,0 +1,92 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52 -Werror
+
+! The directives to check:
+!   cancellation_point
+!   declare_mapper
+!   declare_reduction
+!   declare_simd
+!   declare_target
+!   declare_variant
+!   target_data
+!   target_enter_data
+!   target_exit_data
+!   target_update
+
+subroutine f00
+  implicit none
+  integer :: i
+
+  !$omp parallel
+  do i = 1, 10
+!WARNING: Directive spelling 'CANCELLATION_POINT' is introduced in a later OpenMP version, try -fopenmp-version=60
+    !$omp cancellation_point parallel
+  enddo
+  !$omp end parallel
+end
+
+subroutine f01
+  type :: t
+    integer :: x
+  end type
+!WARNING: Directive spelling 'DECLARE_MAPPER' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_mapper(t :: v) map(v%x)
+end
+
+subroutine f02
+  type :: t
+    integer :: x
+  end type
+!WARNING: Directive spelling 'DECLARE_REDUCTION' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_reduction(+ : t : omp_out%x = omp_out%x + omp_in%x)
+end
+
+subroutine f03
+!WARNING: Directive spelling 'DECLARE_SIMD' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_simd
+end
+
+subroutine f04
+!WARNING: Directive spelling 'DECLARE_TARGET' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_target
+end
+
+subroutine f05
+  implicit none
+  interface
+    subroutine g05
+    end
+  end interface
+!WARNING: Directive spelling 'DECLARE_VARIANT' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_variant(g05) match(user={condition(.true.)})
+end
+
+subroutine f06
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_data map(tofrom: i)
+  i = 0
+!WARNING: Directive spelling 'TARGET_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp end target_data
+end
+
+subroutine f07
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_ENTER_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_enter_data map(to: i)
+end
+
+subroutine f08
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_EXIT_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_exit_data map(from: i)
+end
+
+subroutine f09
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_UPDATE' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_update to(i)
+end

>From 2db790a01737ec31c5c8ed486e816d4cc7563e22 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Thu, 10 Jul 2025 08:58:24 -0500
Subject: [PATCH 6/6] Iterate backwards

---
 flang/lib/Semantics/check-omp-structure.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 191f13cdeef65..89c1565bf66aa 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -270,8 +270,10 @@ void OmpStructureChecker::CheckDirectiveSpelling(
   // with spaces removed).
   auto removeSpaces = [](llvm::StringRef s) {
     std::string n{s.str()};
-    for (auto it{n.begin()}; it != n.end();) {
-      it = isspace(*it) ? n.erase(it) : std::next(it);
+    for (size_t idx{n.size()}; idx > 0; --idx) {
+      if (isspace(n[idx - 1])) {
+        n.erase(idx - 1, 1);
+      }
     }
     return n;
   };


From flang-commits at lists.llvm.org  Thu Jul 10 07:01:37 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 07:01:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fc7c1.050a0220.318b22.5143@mx.google.com>


================
@@ -259,6 +259,55 @@ void OmpStructureChecker::CheckVariableListItem(
   }
 }
 
+void OmpStructureChecker::CheckDirectiveSpelling(
+    parser::CharBlock spelling, llvm::omp::Directive id) {
+  // Directive names that contain spaces can be spelled in the source without
+  // any of the spaces. Because of that getOpenMPKind* is not guaranteed to
+  // work with the source spelling as the argument.
+  //
+  // To verify the source spellings, we have to get the spelling for a given
+  // version, remove spaces and compare it with the source spelling (also
+  // with spaces removed).
+  auto removeSpaces = [](llvm::StringRef s) {
+    std::string n{s.str()};
+    for (auto it{n.begin()}; it != n.end();) {
+      it = isspace(*it) ? n.erase(it) : std::next(it);
+    }
----------------
kparzysz wrote:

Done

https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 07:11:14 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Thu, 10 Jul 2025 07:11:14 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fca02.170a0220.119941.5f55@mx.google.com>


================
@@ -259,6 +259,57 @@ void OmpStructureChecker::CheckVariableListItem(
   }
 }
 
+void OmpStructureChecker::CheckDirectiveSpelling(
+    parser::CharBlock spelling, llvm::omp::Directive id) {
+  // Directive names that contain spaces can be spelled in the source without
+  // any of the spaces. Because of that getOpenMPKind* is not guaranteed to
+  // work with the source spelling as the argument.
+  //
+  // To verify the source spellings, we have to get the spelling for a given
+  // version, remove spaces and compare it with the source spelling (also
+  // with spaces removed).
+  auto removeSpaces = [](llvm::StringRef s) {
+    std::string n{s.str()};
+    for (size_t idx{n.size()}; idx > 0; --idx) {
+      if (isspace(n[idx - 1])) {
+        n.erase(idx - 1, 1);
+      }
+    }
+    return n;
+  };
+
+  std::string lowerNoWS{removeSpaces(
+      parser::ToLowerCaseLetters({spelling.begin(), spelling.size()}))};
+  llvm::StringRef ref(lowerNoWS);
+  if (ref.starts_with("end")) {
+    ref = ref.drop_front(3);
+  }
+
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  // For every "future" version v, check if the check if the corresponding
+  // spelling of id was introduced later than the current version. If so,
+  // and if that spelling matches the source spelling, issue a warning.
+  for (unsigned v : llvm::omp::getOpenMPVersions()) {
+    if (v <= version) {
+      continue;
+    }
+    llvm::StringRef name{llvm::omp::getOpenMPDirectiveName(id, v)};
+    auto [kind, versions]{llvm::omp::getOpenMPDirectiveKindAndVersions(name)};
+    assert(kind == id && "Directive kind mismatch");
+
+    if (static_cast<int>(version) >= versions.Min) {
+      continue;
+    }
+    if (ref == removeSpaces(name)) {
+      context_.Say(spelling,
----------------
Stylie777 wrote:

What is the potential impact if we make this an error? I understand some users programs may no longer compile but they should be using the correct spelling for the version of OpenMP they intend to use - but I feel this may be worth considering.

https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 07:11:26 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Thu, 10 Jul 2025 07:11:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fca0e.630a0220.1011ce.622a@mx.google.com>


https://github.com/Stylie777 edited https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 07:11:37 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Thu, 10 Jul 2025 07:11:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fca19.050a0220.3a714.6233@mx.google.com>


https://github.com/Stylie777 edited https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 07:21:04 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 07:21:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147881@github.com>
Message-ID: <686fcc50.170a0220.335c2d.73f5@mx.google.com>


https://github.com/tblah approved this pull request.

This is great. LGTM

https://github.com/llvm/llvm-project/pull/147881

From flang-commits at lists.llvm.org  Thu Jul 10 07:21:04 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 07:21:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147881@github.com>
Message-ID: <686fcc50.050a0220.3cf967.7b86@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/147881

From flang-commits at lists.llvm.org  Thu Jul 10 07:21:04 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 07:21:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147881@github.com>
Message-ID: <686fcc50.170a0220.34474f.7465@mx.google.com>


================
@@ -1944,6 +1944,128 @@ llvm::LogicalResult fir::EmboxOp::verify() {
   return mlir::success();
 }
 
+/// Returns true if \p extent matches the extent of the \p box's
+/// dimension \p dim.
+bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
----------------
tblah wrote:

nit: can this be static? (and below)

https://github.com/llvm/llvm-project/pull/147881

From flang-commits at lists.llvm.org  Thu Jul 10 07:27:37 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 07:27:37 -0700 (PDT)
Subject: [flang-commits] [flang] 154de3e - [flang] Don't check the
	'-mframe-pointer' flag (#147837)
Message-ID: <686fcdd9.170a0220.2a233c.6e82@mx.google.com>


Author: Daniel Paoliello
Date: 2025-07-10T07:27:33-07:00
New Revision: 154de3e1bdd83ccaf7be51bb876db4593a064607

URL: https://github.com/llvm/llvm-project/commit/154de3e1bdd83ccaf7be51bb876db4593a064607
DIFF: https://github.com/llvm/llvm-project/commit/154de3e1bdd83ccaf7be51bb876db4593a064607.diff

LOG: [flang] Don't check the '-mframe-pointer' flag (#147837)

The `-mframe-pointer` flag is not explicitly set in the original `flang`
invocation and so the value passed to `flang -fc1` can vary depending on
the host machine, so don't verify it in the output.

`-mframe-pointer` forwarding is already verified by
`flang/test/Driver/frame-pointer-forwarding.f90`.

Added: 
    

Modified: 
    flang/test/Driver/frontend-forwarding.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Driver/frontend-forwarding.f90 b/flang/test/Driver/frontend-forwarding.f90
index 55a74ccf40467..952937168c95d 100644
--- a/flang/test/Driver/frontend-forwarding.f90
+++ b/flang/test/Driver/frontend-forwarding.f90
@@ -60,7 +60,6 @@
 ! CHECK: "-Reverything"
 ! CHECK: "-Rno-everything"
 ! CHECK: "-Rpass=inline"
-! CHECK: "-mframe-pointer=none"
 ! CHECK: "-mllvm" "-print-before-all"
 ! CHECK: "-fwrapv"
 ! CHECK: "-save-temps=obj"


        

From flang-commits at lists.llvm.org  Thu Jul 10 07:27:40 2025
From: flang-commits at lists.llvm.org (Daniel Paoliello via flang-commits)
Date: Thu, 10 Jul 2025 07:27:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't check the '-mframe-pointer'
	flag (PR #147837)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147837@github.com>
Message-ID: <686fcddc.050a0220.30ca58.8092@mx.google.com>


https://github.com/dpaoliello closed https://github.com/llvm/llvm-project/pull/147837

From flang-commits at lists.llvm.org  Thu Jul 10 07:31:50 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 07:31:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fced6.170a0220.435b6.825c@mx.google.com>


================
@@ -259,6 +259,57 @@ void OmpStructureChecker::CheckVariableListItem(
   }
 }
 
+void OmpStructureChecker::CheckDirectiveSpelling(
+    parser::CharBlock spelling, llvm::omp::Directive id) {
+  // Directive names that contain spaces can be spelled in the source without
+  // any of the spaces. Because of that getOpenMPKind* is not guaranteed to
+  // work with the source spelling as the argument.
+  //
+  // To verify the source spellings, we have to get the spelling for a given
+  // version, remove spaces and compare it with the source spelling (also
+  // with spaces removed).
+  auto removeSpaces = [](llvm::StringRef s) {
+    std::string n{s.str()};
+    for (size_t idx{n.size()}; idx > 0; --idx) {
+      if (isspace(n[idx - 1])) {
+        n.erase(idx - 1, 1);
+      }
+    }
+    return n;
+  };
+
+  std::string lowerNoWS{removeSpaces(
+      parser::ToLowerCaseLetters({spelling.begin(), spelling.size()}))};
+  llvm::StringRef ref(lowerNoWS);
+  if (ref.starts_with("end")) {
+    ref = ref.drop_front(3);
+  }
+
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  // For every "future" version v, check if the check if the corresponding
+  // spelling of id was introduced later than the current version. If so,
+  // and if that spelling matches the source spelling, issue a warning.
+  for (unsigned v : llvm::omp::getOpenMPVersions()) {
+    if (v <= version) {
+      continue;
+    }
+    llvm::StringRef name{llvm::omp::getOpenMPDirectiveName(id, v)};
+    auto [kind, versions]{llvm::omp::getOpenMPDirectiveKindAndVersions(name)};
+    assert(kind == id && "Directive kind mismatch");
+
+    if (static_cast<int>(version) >= versions.Min) {
+      continue;
+    }
+    if (ref == removeSpaces(name)) {
+      context_.Say(spelling,
----------------
kparzysz wrote:

This can be turned into an error with -Werror (although it turns all warnings into errors).  We do emit a warning for this in clang, this PR follows that behavior.

https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 07:34:20 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 07:34:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fcf6c.170a0220.1465bd.8179@mx.google.com>


================
@@ -259,6 +259,57 @@ void OmpStructureChecker::CheckVariableListItem(
   }
 }
 
+void OmpStructureChecker::CheckDirectiveSpelling(
+    parser::CharBlock spelling, llvm::omp::Directive id) {
+  // Directive names that contain spaces can be spelled in the source without
+  // any of the spaces. Because of that getOpenMPKind* is not guaranteed to
+  // work with the source spelling as the argument.
+  //
+  // To verify the source spellings, we have to get the spelling for a given
+  // version, remove spaces and compare it with the source spelling (also
+  // with spaces removed).
+  auto removeSpaces = [](llvm::StringRef s) {
+    std::string n{s.str()};
+    for (size_t idx{n.size()}; idx > 0; --idx) {
+      if (isspace(n[idx - 1])) {
+        n.erase(idx - 1, 1);
+      }
+    }
+    return n;
+  };
+
+  std::string lowerNoWS{removeSpaces(
+      parser::ToLowerCaseLetters({spelling.begin(), spelling.size()}))};
+  llvm::StringRef ref(lowerNoWS);
+  if (ref.starts_with("end")) {
+    ref = ref.drop_front(3);
+  }
+
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  // For every "future" version v, check if the check if the corresponding
+  // spelling of id was introduced later than the current version. If so,
+  // and if that spelling matches the source spelling, issue a warning.
+  for (unsigned v : llvm::omp::getOpenMPVersions()) {
+    if (v <= version) {
+      continue;
+    }
+    llvm::StringRef name{llvm::omp::getOpenMPDirectiveName(id, v)};
+    auto [kind, versions]{llvm::omp::getOpenMPDirectiveKindAndVersions(name)};
+    assert(kind == id && "Directive kind mismatch");
+
+    if (static_cast<int>(version) >= versions.Min) {
+      continue;
+    }
+    if (ref == removeSpaces(name)) {
+      context_.Say(spelling,
----------------
kparzysz wrote:

Also, we already emit warnings for similar cases---look at uses of `TryVersion` in messages.  There is only one case of an error (which I think was meant to be a warning).

https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 07:46:39 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Thu, 10 Jul 2025 07:46:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fd24f.170a0220.2a14bb.7a49@mx.google.com>


================
@@ -259,6 +259,57 @@ void OmpStructureChecker::CheckVariableListItem(
   }
 }
 
+void OmpStructureChecker::CheckDirectiveSpelling(
+    parser::CharBlock spelling, llvm::omp::Directive id) {
+  // Directive names that contain spaces can be spelled in the source without
+  // any of the spaces. Because of that getOpenMPKind* is not guaranteed to
+  // work with the source spelling as the argument.
+  //
+  // To verify the source spellings, we have to get the spelling for a given
+  // version, remove spaces and compare it with the source spelling (also
+  // with spaces removed).
+  auto removeSpaces = [](llvm::StringRef s) {
+    std::string n{s.str()};
+    for (size_t idx{n.size()}; idx > 0; --idx) {
+      if (isspace(n[idx - 1])) {
+        n.erase(idx - 1, 1);
+      }
+    }
+    return n;
+  };
+
+  std::string lowerNoWS{removeSpaces(
+      parser::ToLowerCaseLetters({spelling.begin(), spelling.size()}))};
+  llvm::StringRef ref(lowerNoWS);
+  if (ref.starts_with("end")) {
+    ref = ref.drop_front(3);
+  }
+
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  // For every "future" version v, check if the check if the corresponding
+  // spelling of id was introduced later than the current version. If so,
+  // and if that spelling matches the source spelling, issue a warning.
+  for (unsigned v : llvm::omp::getOpenMPVersions()) {
+    if (v <= version) {
+      continue;
+    }
+    llvm::StringRef name{llvm::omp::getOpenMPDirectiveName(id, v)};
+    auto [kind, versions]{llvm::omp::getOpenMPDirectiveKindAndVersions(name)};
+    assert(kind == id && "Directive kind mismatch");
+
+    if (static_cast<int>(version) >= versions.Min) {
+      continue;
+    }
+    if (ref == removeSpaces(name)) {
+      context_.Say(spelling,
----------------
Stylie777 wrote:

Makes sense, thanks for the explanation

https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 07:46:47 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Thu, 10 Jul 2025 07:46:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fd257.170a0220.cee89.8ea0@mx.google.com>


https://github.com/Stylie777 approved this pull request.


https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 07:57:07 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 07:57:07 -0700 (PDT)
Subject: [flang-commits] [flang] 9b0ae6c - [flang][OpenMP] Issue a warning
 when parsing future directive spelling (#147765)
Message-ID: <686fd4c3.170a0220.22ae60.86dd@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-10T09:57:03-05:00
New Revision: 9b0ae6ccd6bc9c4fe05064b6e1bb192f189e8f3b

URL: https://github.com/llvm/llvm-project/commit/9b0ae6ccd6bc9c4fe05064b6e1bb192f189e8f3b
DIFF: https://github.com/llvm/llvm-project/commit/9b0ae6ccd6bc9c4fe05064b6e1bb192f189e8f3b.diff

LOG: [flang][OpenMP] Issue a warning when parsing future directive spelling (#147765)

OpenMP 6.0 introduced alternative spelling for some directives, with the
previous spellings still allowed.

Warn the user when a new spelling is encountered with OpenMP version set
to an older value.

Added: 
    flang/test/Semantics/OpenMP/future-directive-spellings.f90

Modified: 
    flang/include/flang/Parser/parse-tree.h
    flang/lib/Parser/openmp-parsers.cpp
    flang/lib/Semantics/check-omp-structure.cpp
    flang/lib/Semantics/check-omp-structure.h

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 7e752eeb4dfe4..43954ff735361 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4612,7 +4612,7 @@ struct OmpDirectiveSpecification {
 
 struct OmpMetadirectiveDirective {
   TUPLE_CLASS_BOILERPLATE(OmpMetadirectiveDirective);
-  std::tuple<OmpClauseList> t;
+  std::tuple<Verbatim, OmpClauseList> t;
   CharBlock source;
 };
 

diff  --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index ee88cce288cae..3016ce4ccd2f8 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1194,7 +1194,7 @@ TYPE_PARSER(sourced(construct<OpenMPUtilityConstruct>(
         sourced(Parser<OmpNothingDirective>{}))))))
 
 TYPE_PARSER(sourced(construct<OmpMetadirectiveDirective>(
-    "METADIRECTIVE" >> Parser<OmpClauseList>{})))
+    verbatim("METADIRECTIVE"_tok), Parser<OmpClauseList>{})))
 
 // Omp directives enclosing do loop
 TYPE_PARSER(sourced(construct<OmpLoopDirective>(first(
@@ -1687,7 +1687,7 @@ TYPE_PARSER(sourced(construct<OmpAssumeDirective>(
     verbatim("ASSUME"_tok), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(sourced(construct<OmpEndAssumeDirective>(
-    verbatim(startOmpLine >> "END ASSUME"_tok))))
+    startOmpLine >> verbatim("END ASSUME"_tok))))
 
 TYPE_PARSER(sourced(
     construct<OpenMPAssumeConstruct>(Parser<OmpAssumeDirective>{} / endOmpLine,

diff  --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index aa4cdb9d27019..89c1565bf66aa 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -259,6 +259,57 @@ void OmpStructureChecker::CheckVariableListItem(
   }
 }
 
+void OmpStructureChecker::CheckDirectiveSpelling(
+    parser::CharBlock spelling, llvm::omp::Directive id) {
+  // Directive names that contain spaces can be spelled in the source without
+  // any of the spaces. Because of that getOpenMPKind* is not guaranteed to
+  // work with the source spelling as the argument.
+  //
+  // To verify the source spellings, we have to get the spelling for a given
+  // version, remove spaces and compare it with the source spelling (also
+  // with spaces removed).
+  auto removeSpaces = [](llvm::StringRef s) {
+    std::string n{s.str()};
+    for (size_t idx{n.size()}; idx > 0; --idx) {
+      if (isspace(n[idx - 1])) {
+        n.erase(idx - 1, 1);
+      }
+    }
+    return n;
+  };
+
+  std::string lowerNoWS{removeSpaces(
+      parser::ToLowerCaseLetters({spelling.begin(), spelling.size()}))};
+  llvm::StringRef ref(lowerNoWS);
+  if (ref.starts_with("end")) {
+    ref = ref.drop_front(3);
+  }
+
+  unsigned version{context_.langOptions().OpenMPVersion};
+
+  // For every "future" version v, check if the check if the corresponding
+  // spelling of id was introduced later than the current version. If so,
+  // and if that spelling matches the source spelling, issue a warning.
+  for (unsigned v : llvm::omp::getOpenMPVersions()) {
+    if (v <= version) {
+      continue;
+    }
+    llvm::StringRef name{llvm::omp::getOpenMPDirectiveName(id, v)};
+    auto [kind, versions]{llvm::omp::getOpenMPDirectiveKindAndVersions(name)};
+    assert(kind == id && "Directive kind mismatch");
+
+    if (static_cast<int>(version) >= versions.Min) {
+      continue;
+    }
+    if (ref == removeSpaces(name)) {
+      context_.Say(spelling,
+          "Directive spelling '%s' is introduced in a later OpenMP version, %s"_warn_en_US,
+          parser::ToUpperCaseLetters(ref), TryVersion(versions.Min));
+      break;
+    }
+  }
+}
+
 void OmpStructureChecker::CheckMultipleOccurrence(
     semantics::UnorderedSymbolSet &listVars,
     const std::list<parser::Name> &nameList, const parser::CharBlock &item,
@@ -436,7 +487,133 @@ void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
   }
 }
 
+template <typename Checker> struct DirectiveSpellingVisitor {
+  using Directive = llvm::omp::Directive;
+
+  DirectiveSpellingVisitor(Checker &&checker) : checker_(std::move(checker)) {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::OmpSectionsDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclarativeAllocate &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocate);
+    return false;
+  }
+  bool Pre(const parser::OmpDispatchDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_dispatch);
+    return false;
+  }
+  bool Pre(const parser::OmpErrorDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_error);
+    return false;
+  }
+  bool Pre(const parser::OmpNothingDirective &x) {
+    checker_(x.source, Directive::OMPD_nothing);
+    return false;
+  }
+  bool Pre(const parser::OpenMPExecutableAllocate &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocate);
+    return false;
+  }
+  bool Pre(const parser::OpenMPAllocatorsConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocators);
+    return false;
+  }
+  bool Pre(const parser::OmpAssumeDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_assume);
+    return false;
+  }
+  bool Pre(const parser::OmpEndAssumeDirective &x) {
+    checker_(x.v.source, Directive::OMPD_assume);
+    return false;
+  }
+  bool Pre(const parser::OmpCriticalDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_critical);
+    return false;
+  }
+  bool Pre(const parser::OmpEndCriticalDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_critical);
+    return false;
+  }
+  bool Pre(const parser::OmpMetadirectiveDirective &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_metadirective);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclarativeAssumes &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_assumes);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareMapperConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_mapper);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareReductionConstruct &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source,
+        Directive::OMPD_declare_reduction);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareSimdConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_simd);
+    return false;
+  }
+  bool Pre(const parser::OpenMPDeclareTargetConstruct &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_declare_target);
+    return false;
+  }
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source,
+        Directive::OMPD_declare_variant);
+    return false;
+  }
+  bool Pre(const parser::OpenMPThreadprivate &x) {
+    checker_(
+        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_threadprivate);
+    return false;
+  }
+  bool Pre(const parser::OpenMPRequiresConstruct &x) {
+    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_requires);
+    return false;
+  }
+
+  bool Pre(const parser::OmpBlockDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+
+  bool Pre(const parser::OmpLoopDirective &x) {
+    checker_(x.source, x.v);
+    return false;
+  }
+
+  bool Pre(const parser::OmpDirectiveSpecification &x) {
+    auto &name = std::get<parser::OmpDirectiveName>(x.t);
+    checker_(name.source, name.v);
+    return false;
+  }
+
+private:
+  Checker checker_;
+};
+
+template <typename T>
+DirectiveSpellingVisitor(T &&) -> DirectiveSpellingVisitor<T>;
+
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
+  DirectiveSpellingVisitor visitor(
+      [this](parser::CharBlock source, llvm::omp::Directive id) {
+        return CheckDirectiveSpelling(source, id);
+      });
+  parser::Walk(x, visitor);
+
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
   // PushContextAndClauseSets() has not been called yet, it is
@@ -461,6 +638,12 @@ void OmpStructureChecker::Leave(const parser::OpenMPConstruct &) {
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPDeclarativeConstruct &x) {
+  DirectiveSpellingVisitor visitor(
+      [this](parser::CharBlock source, llvm::omp::Directive id) {
+        return CheckDirectiveSpelling(source, id);
+      });
+  parser::Walk(x, visitor);
+
   EnterDirectiveNest(DeclarativeNest);
 }
 

diff  --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 2a3853335fd1c..6a877a5d0a7c0 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -163,6 +163,8 @@ class OmpStructureChecker
 private:
   bool CheckAllowedClause(llvmOmpClause clause);
   void CheckVariableListItem(const SymbolSourceMap &symbols);
+  void CheckDirectiveSpelling(
+      parser::CharBlock spelling, llvm::omp::Directive id);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
       const std::string &clauseName);

diff  --git a/flang/test/Semantics/OpenMP/future-directive-spellings.f90 b/flang/test/Semantics/OpenMP/future-directive-spellings.f90
new file mode 100644
index 0000000000000..ed7c4d6782fbe
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/future-directive-spellings.f90
@@ -0,0 +1,92 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52 -Werror
+
+! The directives to check:
+!   cancellation_point
+!   declare_mapper
+!   declare_reduction
+!   declare_simd
+!   declare_target
+!   declare_variant
+!   target_data
+!   target_enter_data
+!   target_exit_data
+!   target_update
+
+subroutine f00
+  implicit none
+  integer :: i
+
+  !$omp parallel
+  do i = 1, 10
+!WARNING: Directive spelling 'CANCELLATION_POINT' is introduced in a later OpenMP version, try -fopenmp-version=60
+    !$omp cancellation_point parallel
+  enddo
+  !$omp end parallel
+end
+
+subroutine f01
+  type :: t
+    integer :: x
+  end type
+!WARNING: Directive spelling 'DECLARE_MAPPER' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_mapper(t :: v) map(v%x)
+end
+
+subroutine f02
+  type :: t
+    integer :: x
+  end type
+!WARNING: Directive spelling 'DECLARE_REDUCTION' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_reduction(+ : t : omp_out%x = omp_out%x + omp_in%x)
+end
+
+subroutine f03
+!WARNING: Directive spelling 'DECLARE_SIMD' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_simd
+end
+
+subroutine f04
+!WARNING: Directive spelling 'DECLARE_TARGET' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_target
+end
+
+subroutine f05
+  implicit none
+  interface
+    subroutine g05
+    end
+  end interface
+!WARNING: Directive spelling 'DECLARE_VARIANT' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp declare_variant(g05) match(user={condition(.true.)})
+end
+
+subroutine f06
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_data map(tofrom: i)
+  i = 0
+!WARNING: Directive spelling 'TARGET_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp end target_data
+end
+
+subroutine f07
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_ENTER_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_enter_data map(to: i)
+end
+
+subroutine f08
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_EXIT_DATA' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_exit_data map(from: i)
+end
+
+subroutine f09
+  implicit none
+  integer :: i
+!WARNING: Directive spelling 'TARGET_UPDATE' is introduced in a later OpenMP version, try -fopenmp-version=60
+  !$omp target_update to(i)
+end


        

From flang-commits at lists.llvm.org  Thu Jul 10 07:57:12 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 07:57:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Issue a warning when
 parsing future directive spelling (PR #147765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147765@github.com>
Message-ID: <686fd4c8.170a0220.6bc42.868b@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/147765

From flang-commits at lists.llvm.org  Thu Jul 10 03:17:50 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 03:17:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix deprecation warning (PR #147932)
Message-ID: <llvm.org/llvm/llvm-project/pull/147932@github.com>


https://github.com/tblah created https://github.com/llvm/llvm-project/pull/147932

I started getting deprecation warnings from operations constructors which seem to be doing implicit construction of mlir::ValueRange from a std::nullopt by relying on implicit conversion from std::nullopt into llvm::ArrayRef. ArrayRef{std::nullopt} is what has been deprecated.

>From 86b7f2f9668b88b675e6f195b7d1933fc7aef874 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Thu, 10 Jul 2025 10:13:06 +0000
Subject: [PATCH] [flang] Fix deprecation warning

I started getting deprecation warnings from operations constructors
which seem to be doing implicit construction of mlir::ValueRange from a
std::nullopt by relying on implicit conversion from std::nullopt into
llvm::ArrayRef. ArrayRef{std::nullopt} is what has been deprecated.
---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp      | 8 ++++----
 flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp | 7 ++++---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 1bc981516e226..2fe726cbb00e8 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -2240,7 +2240,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       if (!rebox.getSubstr().empty())
         substringOffset = operands[rebox.getSubstrOperandIndex()];
       base = genBoxOffsetGep(rewriter, loc, base, llvmBaseObjectType, zero,
-                             /*cstInteriorIndices=*/std::nullopt, fieldIndices,
+                             /*cstInteriorIndices=*/{}, fieldIndices,
                              substringOffset);
     }
 
@@ -2248,7 +2248,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       // The array section is of the form array[%component][substring], keep
       // the input array extents and strides.
       return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
-                           /*lbounds*/ std::nullopt, inputExtents, inputStrides,
+                           /*lbounds*/ {}, inputExtents, inputStrides,
                            rewriter);
 
     // The slice is of the form array(i:j:k)[%component]. Compute new extents
@@ -2297,7 +2297,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       }
     }
     return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
-                         /*lbounds*/ std::nullopt, slicedExtents, slicedStrides,
+                         /*lbounds*/ {}, slicedExtents, slicedStrides,
                          rewriter);
   }
 
@@ -3396,7 +3396,7 @@ static void genBrOp(A caseOp, mlir::Block *dest, std::optional<B> destOps,
   if (destOps)
     rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, *destOps, dest);
   else
-    rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, std::nullopt, dest);
+    rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, B{}, dest);
 }
 
 static void genCaseLadderStep(mlir::Location loc, mlir::Value cmp,
diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index d09d7d397e8b7..b60ac11c7795a 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -107,9 +107,10 @@ class EmboxConversion : public mlir::OpRewritePattern<fir::EmboxOp> {
       shapeOpers.push_back(extVal);
     }
     auto xbox = rewriter.create<fir::cg::XEmboxOp>(
-        loc, embox.getType(), embox.getMemref(), shapeOpers, std::nullopt,
-        std::nullopt, std::nullopt, std::nullopt, embox.getTypeparams(),
-        embox.getSourceBox(), embox.getAllocatorIdxAttr());
+        loc, embox.getType(), embox.getMemref(), shapeOpers, mlir::ValueRange{},
+        mlir::ValueRange{}, mlir::ValueRange{}, mlir::ValueRange{},
+        embox.getTypeparams(), embox.getSourceBox(),
+        embox.getAllocatorIdxAttr());
     LLVM_DEBUG(llvm::dbgs() << "rewriting " << embox << " to " << xbox << '\n');
     rewriter.replaceOp(embox, xbox.getOperation()->getResults());
     return mlir::success();


From flang-commits at lists.llvm.org  Thu Jul 10 03:18:25 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 03:18:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix deprecation warning (PR #147932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147932@github.com>
Message-ID: <686f9371.050a0220.3b9246.24a1@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-fir-hlfir

@llvm/pr-subscribers-flang-codegen

Author: Tom Eccles (tblah)

<details>
<summary>Changes</summary>

I started getting deprecation warnings from operations constructors which seem to be doing implicit construction of mlir::ValueRange from a std::nullopt by relying on implicit conversion from std::nullopt into llvm::ArrayRef. ArrayRef{std::nullopt} is what has been deprecated.

---
Full diff: https://github.com/llvm/llvm-project/pull/147932.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/CodeGen/CodeGen.cpp (+4-4) 
- (modified) flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp (+4-3) 


``````````diff
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 1bc981516e226..2fe726cbb00e8 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -2240,7 +2240,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       if (!rebox.getSubstr().empty())
         substringOffset = operands[rebox.getSubstrOperandIndex()];
       base = genBoxOffsetGep(rewriter, loc, base, llvmBaseObjectType, zero,
-                             /*cstInteriorIndices=*/std::nullopt, fieldIndices,
+                             /*cstInteriorIndices=*/{}, fieldIndices,
                              substringOffset);
     }
 
@@ -2248,7 +2248,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       // The array section is of the form array[%component][substring], keep
       // the input array extents and strides.
       return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
-                           /*lbounds*/ std::nullopt, inputExtents, inputStrides,
+                           /*lbounds*/ {}, inputExtents, inputStrides,
                            rewriter);
 
     // The slice is of the form array(i:j:k)[%component]. Compute new extents
@@ -2297,7 +2297,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       }
     }
     return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
-                         /*lbounds*/ std::nullopt, slicedExtents, slicedStrides,
+                         /*lbounds*/ {}, slicedExtents, slicedStrides,
                          rewriter);
   }
 
@@ -3396,7 +3396,7 @@ static void genBrOp(A caseOp, mlir::Block *dest, std::optional<B> destOps,
   if (destOps)
     rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, *destOps, dest);
   else
-    rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, std::nullopt, dest);
+    rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, B{}, dest);
 }
 
 static void genCaseLadderStep(mlir::Location loc, mlir::Value cmp,
diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index d09d7d397e8b7..b60ac11c7795a 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -107,9 +107,10 @@ class EmboxConversion : public mlir::OpRewritePattern<fir::EmboxOp> {
       shapeOpers.push_back(extVal);
     }
     auto xbox = rewriter.create<fir::cg::XEmboxOp>(
-        loc, embox.getType(), embox.getMemref(), shapeOpers, std::nullopt,
-        std::nullopt, std::nullopt, std::nullopt, embox.getTypeparams(),
-        embox.getSourceBox(), embox.getAllocatorIdxAttr());
+        loc, embox.getType(), embox.getMemref(), shapeOpers, mlir::ValueRange{},
+        mlir::ValueRange{}, mlir::ValueRange{}, mlir::ValueRange{},
+        embox.getTypeparams(), embox.getSourceBox(),
+        embox.getAllocatorIdxAttr());
     LLVM_DEBUG(llvm::dbgs() << "rewriting " << embox << " to " << xbox << '\n');
     rewriter.replaceOp(embox, xbox.getOperation()->getResults());
     return mlir::success();

``````````

</details>


https://github.com/llvm/llvm-project/pull/147932

From flang-commits at lists.llvm.org  Thu Jul 10 03:33:38 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 03:33:38 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix deprecation warning (PR
	#147932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147932@github.com>
Message-ID: <686f9702.170a0220.2e4720.22f3@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/147932

From flang-commits at lists.llvm.org  Thu Jul 10 06:28:55 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Thu, 10 Jul 2025 06:28:55 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add
	ComplexTOROCDLLibraryCalls pass (PR #144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <686fc017.170a0220.1a48ca.3034@mx.google.com>


https://github.com/TIFitis updated https://github.com/llvm/llvm-project/pull/144926

>From c91b94829d5fc94dbc0f8122eee0b598128e4df1 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 19 Jun 2025 17:25:48 +0100
Subject: [PATCH 1/7] [MLIR] Add ComplexTOROCDL pass

This patch adds a new ComplexToROCDL pass to convert complex.abs operations to __ocml_cabs_f32/__ocml_cabs_f64 calls.
---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt    |  1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  5 +-
 .../ComplexToROCDL/ComplexToROCDL.h           | 19 ++++
 mlir/include/mlir/Conversion/Passes.h         |  1 +
 mlir/include/mlir/Conversion/Passes.td        | 12 +++
 mlir/lib/Conversion/CMakeLists.txt            |  1 +
 .../Conversion/ComplexToROCDL/CMakeLists.txt  | 18 ++++
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 95 +++++++++++++++++++
 .../ComplexToROCDL/complex-to-rocdl.mlir      | 13 +++
 9 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
 create mode 100644 mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
 create mode 100644 mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
 create mode 100644 mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 980307db315d9..8b4ac18fba527 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -40,6 +40,7 @@ add_flang_library(FIRCodeGen
   MLIRMathToLLVM
   MLIRMathToLibm
   MLIRMathToROCDL
+  MLIRComplexToROCDL
   MLIROpenMPToLLVM
   MLIROpenACCDialect
   MLIRBuiltinToLLVMIRTranslation
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 1bc981516e226..a83abed601a07 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -33,6 +33,7 @@
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
@@ -4152,8 +4153,10 @@ class FIRToLLVMLowering
     // GPU library calls, the rest can be converted to LLVM intrinsics, which
     // is handled in the mathToLLVM conversion. The lowering to libm calls is
     // not needed since all math operations are handled this way.
-    if (isAMDGCN)
+    if (isAMDGCN) {
       mathConvertionPM.addPass(mlir::createConvertMathToROCDL());
+      mathConvertionPM.addPass(mlir::createConvertComplexToROCDL());
+    }
 
     // Convert math::FPowI operations to inline implementation
     // only if the exponent's width is greater than 32, otherwise,
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
new file mode 100644
index 0000000000000..ed65be9980408
--- /dev/null
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -0,0 +1,19 @@
+#ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+class RewritePatternSet;
+
+#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDL
+#include "mlir/Conversion/Passes.h.inc"
+
+/// Populate the given list with patterns that convert from Complex to ROCDL
+/// calls.
+void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
+                                              PatternBenefit benefit);
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index c9d2a54433736..67e8f5b99b67b 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -23,6 +23,7 @@
 #include "mlir/Conversion/BufferizationToMemRef/BufferizationToMemRef.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
 #include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h"
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRVPass.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 5a864865adffc..8e0a941f4c07a 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -312,6 +312,18 @@ def ConvertComplexToLibm : Pass<"convert-complex-to-libm", "ModuleOp"> {
   let dependentDialects = ["func::FuncDialect"];
 }
 
+//===----------------------------------------------------------------------===//
+// ComplexToROCDL
+//===----------------------------------------------------------------------===//
+
+def ConvertComplexToROCDL : Pass<"convert-complex-to-rocdl", "ModuleOp"> {
+  let summary = "Convert Complex dialect to ROCDL calls";
+  let description = [{
+    This pass converts supported Complex ops to calls to the AMD device library.
+  }];
+  let dependentDialects = ["func::FuncDialect"];
+}
+
 //===----------------------------------------------------------------------===//
 // ComplexToSPIRV
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index e4b4974600577..4ad81553a4fa8 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(AsyncToLLVM)
 add_subdirectory(BufferizationToMemRef)
 add_subdirectory(ComplexCommon)
 add_subdirectory(ComplexToLibm)
+add_subdirectory(ComplexToROCDL)
 add_subdirectory(ComplexToLLVM)
 add_subdirectory(ComplexToSPIRV)
 add_subdirectory(ComplexToStandard)
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
new file mode 100644
index 0000000000000..54607250083d7
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_conversion_library(MLIRComplexToROCDL
+  ComplexToROCDL.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDL
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRComplexDialect
+  MLIRFuncDialect
+  MLIRPass
+  MLIRTransformUtils
+  )
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
new file mode 100644
index 0000000000000..cdfe2a6dfe874
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -0,0 +1,95 @@
+//===-- ComplexToROCDL.cpp - conversion from Complex to ROCDL calls -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+
+#include "mlir/Dialect/Complex/IR/Complex.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include <optional>
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+
+namespace {
+struct FloatTypeResolver {
+  std::optional<bool> operator()(Type type) const {
+    auto elementType = cast<FloatType>(type);
+    if (!isa<Float32Type, Float64Type>(elementType))
+      return {};
+    return elementType.getIntOrFloatBitWidth() == 64;
+  }
+};
+
+template <typename Op, typename TypeResolver = FloatTypeResolver>
+struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
+  using OpRewritePattern<Op>::OpRewritePattern;
+  ScalarOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
+                      StringRef doubleFunc, PatternBenefit benefit)
+      : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
+        doubleFunc(doubleFunc) {}
+
+  LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
+    auto module = SymbolTable::getNearestSymbolTable(op);
+    auto isDouble = TypeResolver()(op.getType());
+    if (!isDouble.has_value())
+      return failure();
+
+    auto name = *isDouble ? doubleFunc : floatFunc;
+
+    auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
+        SymbolTable::lookupSymbolIn(module, name));
+    if (!opFunc) {
+      OpBuilder::InsertionGuard guard(rewriter);
+      rewriter.setInsertionPointToStart(&module->getRegion(0).front());
+      auto funcTy = FunctionType::get(
+          rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
+      opFunc =
+          rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name, funcTy);
+      opFunc.setPrivate();
+    }
+    rewriter.replaceOpWithNewOp<func::CallOp>(op, name, op.getType(),
+                                              op->getOperands());
+    return success();
+  }
+
+private:
+  std::string floatFunc, doubleFunc;
+};
+} // namespace
+
+void mlir::populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
+                                                    PatternBenefit benefit) {
+  patterns.add<ScalarOpToROCDLCall<complex::AbsOp>>(
+      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64", benefit);
+}
+
+namespace {
+struct ConvertComplexToROCDLPass
+    : public impl::ConvertComplexToROCDLBase<ConvertComplexToROCDLPass> {
+  void runOnOperation() override;
+};
+} // namespace
+
+void ConvertComplexToROCDLPass::runOnOperation() {
+  auto module = getOperation();
+
+  RewritePatternSet patterns(&getContext());
+  populateComplexToROCDLConversionPatterns(patterns, /*benefit=*/1);
+
+  ConversionTarget target(getContext());
+  target.addLegalDialect<func::FuncDialect>();
+  target.addIllegalOp<complex::AbsOp>();
+  if (failed(applyPartialConversion(module, target, std::move(patterns))))
+    signalPassFailure();
+}
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
new file mode 100644
index 0000000000000..618e9c238378c
--- /dev/null
+++ b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
@@ -0,0 +1,13 @@
+// RUN: mlir-opt %s -convert-complex-to-rocdl -canonicalize | FileCheck %s
+
+// CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
+// CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64
+
+func.func @abs_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
+  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%[[F:.*]])
+  %rf = complex.abs %f : complex<f32>
+  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%[[D:.*]])
+  %rd = complex.abs %d : complex<f64>
+  // CHECK: return %[[RF]], %[[RD]]
+  return %rf, %rd : f32, f64
+}

>From 4061a93bc818dc77f8f49f9665e94a406ab45fd0 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 19 Jun 2025 17:38:09 +0100
Subject: [PATCH 2/7] Added license to header.

---
 .../mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h       | 8 ++++++++
 mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp     | 1 -
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
index ed65be9980408..96d5a352f54c8 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -1,3 +1,11 @@
+//===-- ComplexToROCDL.h - conversion from Complex to ROCDL calls ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
 #ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
 #define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
 
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index cdfe2a6dfe874..7c7510b5c4e10 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
-
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"

>From 1c1a06d929bf719cb80b53f05c4f974050903026 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Fri, 20 Jun 2025 16:22:07 +0100
Subject: [PATCH 3/7] Address reviewer changes. Add conversion for complex.exp.

---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 14 ++---
 .../ComplexToROCDL/ComplexToROCDL.h           |  3 +-
 .../Conversion/ComplexToROCDL/CMakeLists.txt  |  3 -
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 57 ++++++++++---------
 .../ComplexToROCDL/complex-to-rocdl.mlir      | 19 ++++++-
 5 files changed, 54 insertions(+), 42 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a83abed601a07..8be95e22d231a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -4146,7 +4146,7 @@ class FIRToLLVMLowering
     // conversions that affect the ModuleOp, e.g. create new
     // function operations in it. We have to run such conversions
     // as passes here.
-    mlir::OpPassManager mathConvertionPM("builtin.module");
+    mlir::OpPassManager mathConversionPM("builtin.module");
 
     bool isAMDGCN = fir::getTargetTriple(mod).isAMDGCN();
     // If compiling for AMD target some math operations must be lowered to AMD
@@ -4154,8 +4154,8 @@ class FIRToLLVMLowering
     // is handled in the mathToLLVM conversion. The lowering to libm calls is
     // not needed since all math operations are handled this way.
     if (isAMDGCN) {
-      mathConvertionPM.addPass(mlir::createConvertMathToROCDL());
-      mathConvertionPM.addPass(mlir::createConvertComplexToROCDL());
+      mathConversionPM.addPass(mlir::createConvertMathToROCDL());
+      mathConversionPM.addPass(mlir::createConvertComplexToROCDL());
     }
 
     // Convert math::FPowI operations to inline implementation
@@ -4163,7 +4163,7 @@ class FIRToLLVMLowering
     // it will be lowered to LLVM intrinsic operation by a later conversion.
     mlir::ConvertMathToFuncsOptions mathToFuncsOptions{};
     mathToFuncsOptions.minWidthOfFPowIExponent = 33;
-    mathConvertionPM.addPass(
+    mathConversionPM.addPass(
         mlir::createConvertMathToFuncs(mathToFuncsOptions));
 
     mlir::ConvertComplexToStandardPassOptions complexToStandardOptions{};
@@ -4176,15 +4176,15 @@ class FIRToLLVMLowering
       complexToStandardOptions.complexRange =
           mlir::complex::ComplexRangeFlags::improved;
     }
-    mathConvertionPM.addPass(
+    mathConversionPM.addPass(
         mlir::createConvertComplexToStandardPass(complexToStandardOptions));
 
     // Convert Math dialect operations into LLVM dialect operations.
     // There is no way to prefer MathToLLVM patterns over MathToLibm
     // patterns (applied below), so we have to run MathToLLVM conversion here.
-    mathConvertionPM.addNestedPass<mlir::func::FuncOp>(
+    mathConversionPM.addNestedPass<mlir::func::FuncOp>(
         mlir::createConvertMathToLLVMPass());
-    if (mlir::failed(runPipeline(mathConvertionPM, mod)))
+    if (mlir::failed(runPipeline(mathConversionPM, mod)))
       return signalPassFailure();
 
     std::optional<mlir::DataLayout> dl =
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
index 96d5a352f54c8..eb785080adab3 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
@@ -20,8 +20,7 @@ class RewritePatternSet;
 
 /// Populate the given list with patterns that convert from Complex to ROCDL
 /// calls.
-void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
-                                              PatternBenefit benefit);
+void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns);
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
index 54607250083d7..133809ac32f0f 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
+++ b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
@@ -7,9 +7,6 @@ add_mlir_conversion_library(MLIRComplexToROCDL
   DEPENDS
   MLIRConversionPassIncGen
 
-  LINK_COMPONENTS
-  Core
-
   LINK_LIBS PUBLIC
   MLIRComplexDialect
   MLIRFuncDialect
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index 7c7510b5c4e10..98adb9fb1f607 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -11,7 +11,6 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include <optional>
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
@@ -21,36 +20,38 @@ namespace mlir {
 using namespace mlir;
 
 namespace {
-struct FloatTypeResolver {
-  std::optional<bool> operator()(Type type) const {
-    auto elementType = cast<FloatType>(type);
-    if (!isa<Float32Type, Float64Type>(elementType))
-      return {};
-    return elementType.getIntOrFloatBitWidth() == 64;
-  }
-};
 
-template <typename Op, typename TypeResolver = FloatTypeResolver>
-struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
+template <typename Op>
+// Pattern to convert Complex ops to ROCDL function calls.
+struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ScalarOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
-                      StringRef doubleFunc, PatternBenefit benefit)
+  ComplexOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
+                       StringRef doubleFunc, PatternBenefit benefit = 1)
       : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
         doubleFunc(doubleFunc) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
-    auto module = SymbolTable::getNearestSymbolTable(op);
-    auto isDouble = TypeResolver()(op.getType());
-    if (!isDouble.has_value())
+    Operation *symTable = SymbolTable::getNearestSymbolTable(op);
+    Type resType = op.getType();
+    if (auto complexType = dyn_cast<ComplexType>(resType))
+      resType = complexType.getElementType();
+    FloatType floatTy = dyn_cast<FloatType>(resType);
+    if (!floatTy)
       return failure();
 
-    auto name = *isDouble ? doubleFunc : floatFunc;
+    StringRef name;
+    if (floatTy.isF64())
+      name = doubleFunc;
+    else if (floatTy.isF32())
+      name = floatFunc;
+    else
+      return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
-        SymbolTable::lookupSymbolIn(module, name));
+        SymbolTable::lookupSymbolIn(symTable, name));
     if (!opFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
-      rewriter.setInsertionPointToStart(&module->getRegion(0).front());
+      rewriter.setInsertionPointToStart(&symTable->getRegion(0).front());
       auto funcTy = FunctionType::get(
           rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
       opFunc =
@@ -67,10 +68,12 @@ struct ScalarOpToROCDLCall : public OpRewritePattern<Op> {
 };
 } // namespace
 
-void mlir::populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns,
-                                                    PatternBenefit benefit) {
-  patterns.add<ScalarOpToROCDLCall<complex::AbsOp>>(
-      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64", benefit);
+void mlir::populateComplexToROCDLConversionPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp>>(
+      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp>>(
+      patterns.getContext(), "__ocml_cexp_f32", "__ocml_cexp_f64");
 }
 
 namespace {
@@ -81,14 +84,14 @@ struct ConvertComplexToROCDLPass
 } // namespace
 
 void ConvertComplexToROCDLPass::runOnOperation() {
-  auto module = getOperation();
+  Operation *op = getOperation();
 
   RewritePatternSet patterns(&getContext());
-  populateComplexToROCDLConversionPatterns(patterns, /*benefit=*/1);
+  populateComplexToROCDLConversionPatterns(patterns);
 
   ConversionTarget target(getContext());
   target.addLegalDialect<func::FuncDialect>();
-  target.addIllegalOp<complex::AbsOp>();
-  if (failed(applyPartialConversion(module, target, std::move(patterns))))
+  target.addIllegalOp<complex::AbsOp, complex::ExpOp>();
+  if (failed(applyPartialConversion(op, target, std::move(patterns))))
     signalPassFailure();
 }
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
index 618e9c238378c..23631e25e4588 100644
--- a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
+++ b/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
@@ -1,13 +1,26 @@
-// RUN: mlir-opt %s -convert-complex-to-rocdl -canonicalize | FileCheck %s
+// RUN: mlir-opt %s -convert-complex-to-rocdl | FileCheck %s
 
 // CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
 // CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64
+// CHECK-DAG: @__ocml_cexp_f32(complex<f32>) -> complex<f32>
+// CHECK-DAG: @__ocml_cexp_f64(complex<f64>) -> complex<f64>
 
+//CHECK-LABEL: @abs_caller
 func.func @abs_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
-  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%[[F:.*]])
+  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%{{.*}})
   %rf = complex.abs %f : complex<f32>
-  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%[[D:.*]])
+  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%{{.*}})
   %rd = complex.abs %d : complex<f64>
   // CHECK: return %[[RF]], %[[RD]]
   return %rf, %rd : f32, f64
 }
+
+//CHECK-LABEL: @exp_caller
+func.func @exp_caller(%f: complex<f32>, %d: complex<f64>) -> (complex<f32>, complex<f64>) {
+  // CHECK: %[[EF:.*]] = call @__ocml_cexp_f32(%{{.*}})
+  %ef = complex.exp %f : complex<f32>
+  // CHECK: %[[ED:.*]] = call @__ocml_cexp_f64(%{{.*}})
+  %ed = complex.exp %d : complex<f64>
+  // CHECK: return %[[EF]], %[[ED]]
+  return %ef, %ed : complex<f32>, complex<f64>
+}

>From 012aa6c590564e923a4b6e2bc71c81f00f89cae7 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Fri, 20 Jun 2025 18:30:57 +0100
Subject: [PATCH 4/7] Correct alphabetical order for cmake.

---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 8b4ac18fba527..de8e1c5c3fa3f 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -34,13 +34,13 @@ add_flang_library(FIRCodeGen
 
   MLIR_LIBS
   MLIRComplexToLLVM
+  MLIRComplexToROCDL
   MLIRComplexToStandard
   MLIRGPUDialect
   MLIRMathToFuncs
   MLIRMathToLLVM
   MLIRMathToLibm
   MLIRMathToROCDL
-  MLIRComplexToROCDL
   MLIROpenMPToLLVM
   MLIROpenACCDialect
   MLIRBuiltinToLLVMIRTranslation

>From 0949d97ed14e700c5233f028d6882338b79ec878 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Tue, 24 Jun 2025 16:34:11 +0100
Subject: [PATCH 5/7] Add FloatTy as a template parameter.

---
 .../ComplexToROCDL/ComplexToROCDL.cpp         | 42 ++++++++-----------
 1 file changed, 18 insertions(+), 24 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
index 98adb9fb1f607..cfad9f5f6fa19 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
@@ -21,59 +21,53 @@ using namespace mlir;
 
 namespace {
 
-template <typename Op>
+template <typename Op, typename Ty>
 // Pattern to convert Complex ops to ROCDL function calls.
 struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ComplexOpToROCDLCall(MLIRContext *context, StringRef floatFunc,
-                       StringRef doubleFunc, PatternBenefit benefit = 1)
-      : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
-        doubleFunc(doubleFunc) {}
+  ComplexOpToROCDLCall(MLIRContext *context, StringRef funcName,
+                       PatternBenefit benefit = 1)
+      : OpRewritePattern<Op>(context, benefit), funcName(funcName) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
     Operation *symTable = SymbolTable::getNearestSymbolTable(op);
     Type resType = op.getType();
     if (auto complexType = dyn_cast<ComplexType>(resType))
       resType = complexType.getElementType();
-    FloatType floatTy = dyn_cast<FloatType>(resType);
-    if (!floatTy)
-      return failure();
-
-    StringRef name;
-    if (floatTy.isF64())
-      name = doubleFunc;
-    else if (floatTy.isF32())
-      name = floatFunc;
-    else
+    if (!isa<Ty>(resType))
       return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
-        SymbolTable::lookupSymbolIn(symTable, name));
+        SymbolTable::lookupSymbolIn(symTable, funcName));
     if (!opFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(&symTable->getRegion(0).front());
       auto funcTy = FunctionType::get(
           rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-      opFunc =
-          rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name, funcTy);
+      opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), funcName,
+                                             funcTy);
       opFunc.setPrivate();
     }
-    rewriter.replaceOpWithNewOp<func::CallOp>(op, name, op.getType(),
+    rewriter.replaceOpWithNewOp<func::CallOp>(op, funcName, op.getType(),
                                               op->getOperands());
     return success();
   }
 
 private:
-  std::string floatFunc, doubleFunc;
+  std::string funcName;
 };
 } // namespace
 
 void mlir::populateComplexToROCDLConversionPatterns(
     RewritePatternSet &patterns) {
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp>>(
-      patterns.getContext(), "__ocml_cabs_f32", "__ocml_cabs_f64");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp>>(
-      patterns.getContext(), "__ocml_cexp_f32", "__ocml_cexp_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float32Type>>(
+      patterns.getContext(), "__ocml_cabs_f32");
+  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float64Type>>(
+      patterns.getContext(), "__ocml_cabs_f64");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float32Type>>(
+      patterns.getContext(), "__ocml_cexp_f32");
+  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float64Type>>(
+      patterns.getContext(), "__ocml_cexp_f64");
 }
 
 namespace {

>From 8887213da7008d4d15a583cecd641d9eed1d0454 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 2 Jul 2025 14:48:34 +0100
Subject: [PATCH 6/7] Rename to ComplexToROCDLLibraryCalls

---
 flang/lib/Optimizer/CodeGen/CMakeLists.txt    |  2 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  4 +--
 .../ComplexToROCDLLibraryCalls.h}             | 14 ++++----
 mlir/include/mlir/Conversion/Passes.h         |  2 +-
 mlir/include/mlir/Conversion/Passes.td        |  6 ++--
 mlir/lib/Conversion/CMakeLists.txt            |  2 +-
 .../Conversion/ComplexToROCDL/CMakeLists.txt  | 15 --------
 .../ComplexToROCDLLibraryCalls/CMakeLists.txt | 18 ++++++++++
 .../ComplexToROCDLLibraryCalls.cpp}           | 35 ++++++++++---------
 .../complex-to-rocdl-library-calls.mlir}      |  2 +-
 10 files changed, 52 insertions(+), 48 deletions(-)
 rename mlir/include/mlir/Conversion/{ComplexToROCDL/ComplexToROCDL.h => ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h} (52%)
 delete mode 100644 mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
 create mode 100644 mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
 rename mlir/lib/Conversion/{ComplexToROCDL/ComplexToROCDL.cpp => ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp} (68%)
 rename mlir/test/Conversion/{ComplexToROCDL/complex-to-rocdl.mlir => ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir} (92%)

diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index de8e1c5c3fa3f..16c7944a885a1 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -34,7 +34,7 @@ add_flang_library(FIRCodeGen
 
   MLIR_LIBS
   MLIRComplexToLLVM
-  MLIRComplexToROCDL
+  MLIRComplexToROCDLLibraryCalls
   MLIRComplexToStandard
   MLIRGPUDialect
   MLIRMathToFuncs
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 8be95e22d231a..b0ab2a8f63b3b 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -33,7 +33,7 @@
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
@@ -4155,7 +4155,7 @@ class FIRToLLVMLowering
     // not needed since all math operations are handled this way.
     if (isAMDGCN) {
       mathConversionPM.addPass(mlir::createConvertMathToROCDL());
-      mathConversionPM.addPass(mlir::createConvertComplexToROCDL());
+      mathConversionPM.addPass(mlir::createConvertComplexToROCDLLibraryCalls());
     }
 
     // Convert math::FPowI operations to inline implementation
diff --git a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
similarity index 52%
rename from mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
rename to mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
index eb785080adab3..409b87747f437 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
@@ -1,4 +1,4 @@
-//===-- ComplexToROCDL.h - conversion from Complex to ROCDL calls ---------===//
+//===- ComplexToROCDLLibraryCalls.h - convert from Complex to ROCDL calls -===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,21 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
-#define MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#ifndef MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
+#define MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
 
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 class RewritePatternSet;
 
-#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDL
+#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDLLIBRARYCALLS
 #include "mlir/Conversion/Passes.h.inc"
 
 /// Populate the given list with patterns that convert from Complex to ROCDL
 /// calls.
-void populateComplexToROCDLConversionPatterns(RewritePatternSet &patterns);
+void populateComplexToROCDLLibraryCallsConversionPatterns(
+    RewritePatternSet &patterns);
 } // namespace mlir
 
-#endif // MLIR_CONVERSION_COMPLEXTOROCDL_COMPLEXTOROCDL_H_
+#endif // MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 67e8f5b99b67b..896fff277679e 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -23,7 +23,7 @@
 #include "mlir/Conversion/BufferizationToMemRef/BufferizationToMemRef.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
 #include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h"
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRVPass.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 8e0a941f4c07a..7a48783374dc2 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -313,11 +313,11 @@ def ConvertComplexToLibm : Pass<"convert-complex-to-libm", "ModuleOp"> {
 }
 
 //===----------------------------------------------------------------------===//
-// ComplexToROCDL
+// ComplexToROCDLLibraryCalls
 //===----------------------------------------------------------------------===//
 
-def ConvertComplexToROCDL : Pass<"convert-complex-to-rocdl", "ModuleOp"> {
-  let summary = "Convert Complex dialect to ROCDL calls";
+def ConvertComplexToROCDLLibraryCalls : Pass<"convert-complex-to-rocdl-library-calls", "ModuleOp"> {
+  let summary = "Convert Complex dialect to ROCDL library calls";
   let description = [{
     This pass converts supported Complex ops to calls to the AMD device library.
   }];
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index 4ad81553a4fa8..b731662dcebc7 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -13,7 +13,7 @@ add_subdirectory(AsyncToLLVM)
 add_subdirectory(BufferizationToMemRef)
 add_subdirectory(ComplexCommon)
 add_subdirectory(ComplexToLibm)
-add_subdirectory(ComplexToROCDL)
+add_subdirectory(ComplexToROCDLLibraryCalls)
 add_subdirectory(ComplexToLLVM)
 add_subdirectory(ComplexToSPIRV)
 add_subdirectory(ComplexToStandard)
diff --git a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
deleted file mode 100644
index 133809ac32f0f..0000000000000
--- a/mlir/lib/Conversion/ComplexToROCDL/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-add_mlir_conversion_library(MLIRComplexToROCDL
-  ComplexToROCDL.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDL
-
-  DEPENDS
-  MLIRConversionPassIncGen
-
-  LINK_LIBS PUBLIC
-  MLIRComplexDialect
-  MLIRFuncDialect
-  MLIRPass
-  MLIRTransformUtils
-  )
diff --git a/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
new file mode 100644
index 0000000000000..695bb2dd0a82c
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_conversion_library(MLIRComplexToROCDLLibraryCalls
+  ComplexToROCDLLibraryCalls.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDLLibraryCalls
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRComplexDialect
+  MLIRFuncDialect
+  MLIRPass
+  MLIRTransformUtils
+  )
diff --git a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
similarity index 68%
rename from mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
rename to mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
index cfad9f5f6fa19..99d5424aef79a 100644
--- a/mlir/lib/Conversion/ComplexToROCDL/ComplexToROCDL.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
@@ -1,4 +1,4 @@
-//===-- ComplexToROCDL.cpp - conversion from Complex to ROCDL calls -------===//
+//=== ComplexToROCDLLibraryCalls.cpp - convert from Complex to ROCDL calls ===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/ComplexToROCDL/ComplexToROCDL.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
-#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDL
+#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDLLIBRARYCALLS
 #include "mlir/Conversion/Passes.h.inc"
 } // namespace mlir
 
@@ -21,12 +21,12 @@ using namespace mlir;
 
 namespace {
 
-template <typename Op, typename Ty>
+template <typename Op, typename FloatTy>
 // Pattern to convert Complex ops to ROCDL function calls.
-struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
+struct ComplexOpToROCDLLibraryCalls : public OpRewritePattern<Op> {
   using OpRewritePattern<Op>::OpRewritePattern;
-  ComplexOpToROCDLCall(MLIRContext *context, StringRef funcName,
-                       PatternBenefit benefit = 1)
+  ComplexOpToROCDLLibraryCalls(MLIRContext *context, StringRef funcName,
+                               PatternBenefit benefit = 1)
       : OpRewritePattern<Op>(context, benefit), funcName(funcName) {}
 
   LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
@@ -34,7 +34,7 @@ struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
     Type resType = op.getType();
     if (auto complexType = dyn_cast<ComplexType>(resType))
       resType = complexType.getElementType();
-    if (!isa<Ty>(resType))
+    if (!isa<FloatTy>(resType))
       return failure();
 
     auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
@@ -58,30 +58,31 @@ struct ComplexOpToROCDLCall : public OpRewritePattern<Op> {
 };
 } // namespace
 
-void mlir::populateComplexToROCDLConversionPatterns(
+void mlir::populateComplexToROCDLLibraryCallsConversionPatterns(
     RewritePatternSet &patterns) {
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float32Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float32Type>>(
       patterns.getContext(), "__ocml_cabs_f32");
-  patterns.add<ComplexOpToROCDLCall<complex::AbsOp, Float64Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float64Type>>(
       patterns.getContext(), "__ocml_cabs_f64");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float32Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ExpOp, Float32Type>>(
       patterns.getContext(), "__ocml_cexp_f32");
-  patterns.add<ComplexOpToROCDLCall<complex::ExpOp, Float64Type>>(
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ExpOp, Float64Type>>(
       patterns.getContext(), "__ocml_cexp_f64");
 }
 
 namespace {
-struct ConvertComplexToROCDLPass
-    : public impl::ConvertComplexToROCDLBase<ConvertComplexToROCDLPass> {
+struct ConvertComplexToROCDLLibraryCallsPass
+    : public impl::ConvertComplexToROCDLLibraryCallsBase<
+          ConvertComplexToROCDLLibraryCallsPass> {
   void runOnOperation() override;
 };
 } // namespace
 
-void ConvertComplexToROCDLPass::runOnOperation() {
+void ConvertComplexToROCDLLibraryCallsPass::runOnOperation() {
   Operation *op = getOperation();
 
   RewritePatternSet patterns(&getContext());
-  populateComplexToROCDLConversionPatterns(patterns);
+  populateComplexToROCDLLibraryCallsConversionPatterns(patterns);
 
   ConversionTarget target(getContext());
   target.addLegalDialect<func::FuncDialect>();
diff --git a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
similarity index 92%
rename from mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
rename to mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
index 23631e25e4588..bae7c5986ef9e 100644
--- a/mlir/test/Conversion/ComplexToROCDL/complex-to-rocdl.mlir
+++ b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-complex-to-rocdl | FileCheck %s
+// RUN: mlir-opt %s -convert-complex-to-rocdl-library-calls | FileCheck %s
 
 // CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
 // CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64

>From cd8566373c97dbb7213c39d1905153d20bf7c9de Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 2 Jul 2025 16:43:10 +0100
Subject: [PATCH 7/7] Fix build error.

---
 .../ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
index 409b87747f437..daac2a99ed80f 100644
--- a/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
+++ b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
@@ -10,6 +10,7 @@
 #define MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
 
 #include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 class RewritePatternSet;


From flang-commits at lists.llvm.org  Thu Jul 10 07:22:58 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 07:22:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix deprecation warning (PR
	#147932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147932@github.com>
Message-ID: <686fccc2.630a0220.27c3c2.7c2c@mx.google.com>


https://github.com/tblah closed https://github.com/llvm/llvm-project/pull/147932

From flang-commits at lists.llvm.org  Thu Jul 10 07:22:59 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 07:22:59 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix deprecation warning (PR
	#147932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147932@github.com>
Message-ID: <686fccc3.050a0220.33b0d6.8410@mx.google.com>


tblah wrote:

Ahh this was fixed by https://github.com/llvm/llvm-project/pull/147816

https://github.com/llvm/llvm-project/pull/147932

From flang-commits at lists.llvm.org  Thu Jul 10 07:35:04 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Thu, 10 Jul 2025 07:35:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix deprecation warning (PR
	#147932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147932@github.com>
Message-ID: <686fcf98.050a0220.38c4b.8a63@mx.google.com>


DanielCChen wrote:

@tblah
Your fix in the PR is less verbose than mine in #147816, which is what @vzakhari prefers. 
I would like to switch to your fix if that is OK. Do you want to add on top of #147816 or you prefer me to revert it first?

https://github.com/llvm/llvm-project/pull/147932

From flang-commits at lists.llvm.org  Thu Jul 10 07:51:33 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Thu, 10 Jul 2025 07:51:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix deprecation warning (PR
	#147932)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147932@github.com>
Message-ID: <686fd375.050a0220.458d9.89fc@mx.google.com>


https://github.com/tblah updated https://github.com/llvm/llvm-project/pull/147932

>From 86b7f2f9668b88b675e6f195b7d1933fc7aef874 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Thu, 10 Jul 2025 10:13:06 +0000
Subject: [PATCH 1/2] [flang] Fix deprecation warning

I started getting deprecation warnings from operations constructors
which seem to be doing implicit construction of mlir::ValueRange from a
std::nullopt by relying on implicit conversion from std::nullopt into
llvm::ArrayRef. ArrayRef{std::nullopt} is what has been deprecated.
---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp      | 8 ++++----
 flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp | 7 ++++---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 1bc981516e226..2fe726cbb00e8 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -2240,7 +2240,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       if (!rebox.getSubstr().empty())
         substringOffset = operands[rebox.getSubstrOperandIndex()];
       base = genBoxOffsetGep(rewriter, loc, base, llvmBaseObjectType, zero,
-                             /*cstInteriorIndices=*/std::nullopt, fieldIndices,
+                             /*cstInteriorIndices=*/{}, fieldIndices,
                              substringOffset);
     }
 
@@ -2248,7 +2248,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       // The array section is of the form array[%component][substring], keep
       // the input array extents and strides.
       return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
-                           /*lbounds*/ std::nullopt, inputExtents, inputStrides,
+                           /*lbounds*/ {}, inputExtents, inputStrides,
                            rewriter);
 
     // The slice is of the form array(i:j:k)[%component]. Compute new extents
@@ -2297,7 +2297,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       }
     }
     return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
-                         /*lbounds*/ std::nullopt, slicedExtents, slicedStrides,
+                         /*lbounds*/ {}, slicedExtents, slicedStrides,
                          rewriter);
   }
 
@@ -3396,7 +3396,7 @@ static void genBrOp(A caseOp, mlir::Block *dest, std::optional<B> destOps,
   if (destOps)
     rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, *destOps, dest);
   else
-    rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, std::nullopt, dest);
+    rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(caseOp, B{}, dest);
 }
 
 static void genCaseLadderStep(mlir::Location loc, mlir::Value cmp,
diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index d09d7d397e8b7..b60ac11c7795a 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -107,9 +107,10 @@ class EmboxConversion : public mlir::OpRewritePattern<fir::EmboxOp> {
       shapeOpers.push_back(extVal);
     }
     auto xbox = rewriter.create<fir::cg::XEmboxOp>(
-        loc, embox.getType(), embox.getMemref(), shapeOpers, std::nullopt,
-        std::nullopt, std::nullopt, std::nullopt, embox.getTypeparams(),
-        embox.getSourceBox(), embox.getAllocatorIdxAttr());
+        loc, embox.getType(), embox.getMemref(), shapeOpers, mlir::ValueRange{},
+        mlir::ValueRange{}, mlir::ValueRange{}, mlir::ValueRange{},
+        embox.getTypeparams(), embox.getSourceBox(),
+        embox.getAllocatorIdxAttr());
     LLVM_DEBUG(llvm::dbgs() << "rewriting " << embox << " to " << xbox << '\n');
     rewriter.replaceOp(embox, xbox.getOperation()->getResults());
     return mlir::success();

>From ef8ca4aafc6905efc85fa19edd3d806dfa4a0eaa Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Thu, 10 Jul 2025 14:51:00 +0000
Subject: [PATCH 2/2] Clang format

This is what I get for trying to use the web interface for the merge
conflicts.
---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index e860632d325f0..2fe726cbb00e8 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -2299,7 +2299,6 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     return finalizeRebox(rebox, adaptor, destBoxTy, dest, base,
                          /*lbounds*/ {}, slicedExtents, slicedStrides,
                          rewriter);
-
   }
 
   /// Apply a new shape to the data described by a box given the base address,


From flang-commits at lists.llvm.org  Thu Jul 10 09:24:02 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Thu, 10 Jul 2025 09:24:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147881@github.com>
Message-ID: <686fe922.170a0220.49d1.9338@mx.google.com>


================
@@ -1944,6 +1944,128 @@ llvm::LogicalResult fir::EmboxOp::verify() {
   return mlir::success();
 }
 
+/// Returns true if \p extent matches the extent of the \p box's
+/// dimension \p dim.
+bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
----------------
vzakhari wrote:

Thanks for catching this! I made them external utilities elsewhere, initially, but then I got the components linking issue.  I am not sure yet where these utils belong to, so I will keep them static here for the time being.

https://github.com/llvm/llvm-project/pull/147881

From flang-commits at lists.llvm.org  Thu Jul 10 09:49:25 2025
From: flang-commits at lists.llvm.org (Zhen Wang via flang-commits)
Date: Thu, 10 Jul 2025 09:49:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Update implicit data transfer
 for device component (PR #147882)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147882@github.com>
Message-ID: <686fef15.170a0220.3c211f.97ce@mx.google.com>


https://github.com/wangzpgi approved this pull request.


https://github.com/llvm/llvm-project/pull/147882

From flang-commits at lists.llvm.org  Thu Jul 10 09:50:35 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 09:50:35 -0700 (PDT)
Subject: [flang-commits] [flang] 9a0e03f - [flang][cuda] Update implicit
 data transfer for device component (#147882)
Message-ID: <686fef5b.170a0220.2da2a7.9b20@mx.google.com>


Author: Valentin Clement (バレンタイン クレメン)
Date: 2025-07-10T09:50:31-07:00
New Revision: 9a0e03f430dec4634086fe8315c4c3b730bd7c66

URL: https://github.com/llvm/llvm-project/commit/9a0e03f430dec4634086fe8315c4c3b730bd7c66
DIFF: https://github.com/llvm/llvm-project/commit/9a0e03f430dec4634086fe8315c4c3b730bd7c66.diff

LOG: [flang][cuda] Update implicit data transfer for device component (#147882)

Update the detection of implicit data transfer when a device resident
allocatable derived-type component is involved and remove the TODOs.

Added: 
    

Modified: 
    flang/include/flang/Evaluate/tools.h
    flang/lib/Evaluate/tools.cpp
    flang/lib/Lower/Bridge.cpp
    flang/test/Lower/CUDA/cuda-data-transfer.cuf

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 18c244f6f450f..96ed86f468350 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1359,24 +1359,7 @@ inline bool IsCUDADataTransfer(const A &lhs, const B &rhs) {
 
 /// Check if the expression is a mix of host and device variables that require
 /// implicit data transfer.
-inline bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
-  unsigned hostSymbols{0};
-  unsigned deviceSymbols{0};
-  for (const Symbol &sym : CollectCudaSymbols(expr)) {
-    if (IsCUDADeviceSymbol(sym)) {
-      ++deviceSymbols;
-    } else {
-      if (sym.owner().IsDerivedType()) {
-        if (IsCUDADeviceSymbol(sym.owner().GetSymbol()->GetUltimate())) {
-          ++deviceSymbols;
-        }
-      }
-      ++hostSymbols;
-    }
-  }
-  bool hasConstant{HasConstant(expr)};
-  return (hasConstant || (hostSymbols > 0)) && deviceSymbols > 0;
-}
+bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr);
 
 // Checks whether the symbol on the LHS is present in the RHS expression.
 bool CheckForSymbolMatch(const Expr<SomeType> *lhs, const Expr<SomeType> *rhs);

diff  --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 6a57d87a30e93..3d9f06308d8c1 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1154,6 +1154,31 @@ template semantics::UnorderedSymbolSet CollectCudaSymbols(
 template semantics::UnorderedSymbolSet CollectCudaSymbols(
     const Expr<SubscriptInteger> &);
 
+bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
+  semantics::UnorderedSymbolSet hostSymbols;
+  semantics::UnorderedSymbolSet deviceSymbols;
+
+  SymbolVector symbols{GetSymbolVector(expr)};
+  std::reverse(symbols.begin(), symbols.end());
+  bool skipNext{false};
+  for (const Symbol &sym : symbols) {
+    bool isComponent{sym.owner().IsDerivedType()};
+    bool skipComponent{false};
+    if (!skipNext) {
+      if (IsCUDADeviceSymbol(sym)) {
+        deviceSymbols.insert(sym);
+      } else if (isComponent) {
+        skipComponent = true; // Component is not device. Look on the base.
+      } else {
+        hostSymbols.insert(sym);
+      }
+    }
+    skipNext = isComponent && !skipComponent;
+  }
+  bool hasConstant{HasConstant(expr)};
+  return (hasConstant || (hostSymbols.size() > 0)) && deviceSymbols.size() > 0;
+}
+
 // HasVectorSubscript()
 struct HasVectorSubscriptHelper
     : public AnyTraverse<HasVectorSubscriptHelper, bool,

diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 2c8b3fb95e255..987fd3095fdf6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          if (sym.owner().IsDerivedType() && IsAllocatable(sym.GetUltimate()))
-            TODO(loc, "Device resident allocatable derived-type component");
           // TODO: This should probably being checked in semantic and give a
           // proper error.
           assert(

diff  --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index 68a0202f951fe..3a9b55996d9b1 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -7,6 +7,10 @@ module mod1
     integer :: i
   end type
 
+  type :: t2
+    integer, device, allocatable, dimension(:) :: x
+  end type
+
   integer, device, dimension(11:20) :: cdev
 
 contains
@@ -419,3 +423,16 @@ end subroutine
 ! CHECK: fir.do_concurrent.loop
 ! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
+
+
+subroutine sub22()
+  use mod1
+  type(t2) :: a
+  integer :: b(100)
+  allocate(a%x(100))
+
+  b = a%x
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub22()
+! CHECK: cuf.data_transfer


        

From flang-commits at lists.llvm.org  Thu Jul 10 09:50:37 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 10 Jul 2025 09:50:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Update implicit data transfer
 for device component (PR #147882)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147882@github.com>
Message-ID: <686fef5d.170a0220.35673a.a612@mx.google.com>


https://github.com/clementval closed https://github.com/llvm/llvm-project/pull/147882

From flang-commits at lists.llvm.org  Thu Jul 10 09:30:23 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Thu, 10 Jul 2025 09:30:23 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147881@github.com>
Message-ID: <686fea9f.170a0220.351571.9448@mx.google.com>


https://github.com/vzakhari updated https://github.com/llvm/llvm-project/pull/147881

>From 1064e297fbf1ff6142a5c9bcc856474028863b42 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Wed, 9 Jul 2025 17:21:48 -0700
Subject: [PATCH 1/2] [flang] Optimize redundant array repacking.

This patch allows optimizing redundant array repacking, when
the source array is statically known to be contiguous.
This is part of the implementation plan for the array repacking
feature, though, it does not affect any real life use case
as long as FIR inlining is not a thing. I experimented with
simple cases of FIR inling using `-inline-all`, and I recorded
these cases in optimize-array-repacking.fir tests.
---
 .../flang/Optimizer/Builder/HLFIRTools.h      |   8 +
 .../flang/Optimizer/Dialect/FIROpsSupport.h   |  14 +-
 .../flang/Optimizer/Transforms/Passes.h       |  34 +-
 .../flang/Optimizer/Transforms/Passes.td      |  10 +
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |  40 +-
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 157 ++++-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |   4 +
 flang/lib/Optimizer/Transforms/CMakeLists.txt |   1 +
 .../Optimizer/Transforms/LoopVersioning.cpp   |   4 +-
 .../Transforms/OptimizeArrayRepacking.cpp     |  90 +++
 flang/test/Driver/bbc-mlir-pass-pipeline.f90  |   2 +
 flang/test/Driver/mlir-pass-pipeline.f90      |   2 +
 flang/test/Fir/basic-program.fir              |   2 +
 .../simplify-hlfir-intrinsics-cshift.fir      |  36 +-
 .../simplify-hlfir-intrinsics-matmul.fir      | 107 +--
 .../Transforms/optimize-array-repacking.fir   | 660 ++++++++++++++++++
 16 files changed, 982 insertions(+), 189 deletions(-)
 create mode 100644 flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
 create mode 100644 flang/test/Transforms/optimize-array-repacking.fir

diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 2cbad6e268a38..49dfc85dc76e6 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -542,6 +542,14 @@ Entity gen1DSection(mlir::Location loc, fir::FirOpBuilder &builder,
 /// contiguous.
 bool designatePreservesContinuity(hlfir::DesignateOp op);
 
+/// Return true iff the given \p base desribes an object
+/// that is contiguous. If \p checkWhole is true, then
+/// the object must be contiguous in all dimensions,
+/// otherwise, it must be contiguous in the innermost dimension.
+/// This function is an extension of hlfir::Entity::isSimplyContiguous(),
+/// and it can be used on pure FIR representation as well as on HLFIR.
+bool isSimplyContiguous(mlir::Value base, bool checkWhole = true);
+
 } // namespace hlfir
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_HLFIRTOOLS_H
diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index 0a2337be7455e..ae471eb0d2e04 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -238,7 +238,19 @@ std::optional<int64_t> getAllocaByteSize(fir::AllocaOp alloca,
 /// When \p checkWhole is false, then the checking is only done
 /// for continuity in the innermost dimension, otherwise,
 /// the checking is done for continuity of the whole result of rebox.
-bool reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole = true);
+/// The caller may specify \p mayHaveNonDefaultLowerBounds, if it is known,
+/// to allow better handling of the rebox operations representing
+/// full array slices.
+bool reboxPreservesContinuity(fir::ReboxOp rebox,
+                              bool mayHaveNonDefaultLowerBounds = true,
+                              bool checkWhole = true);
+
+/// Return true, if \p embox operation produces a contiguous
+/// entity.
+/// When \p checkWhole is false, then the checking is only done
+/// for continuity in the innermost dimension, otherwise,
+/// the checking is done for continuity of the whole result of embox
+bool isContiguousEmbox(fir::EmboxOp embox, bool checkWhole = true);
 
 } // namespace fir
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index dc8a5b9141ad2..6f5dff4687cbb 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -31,39 +31,7 @@ namespace fir {
 // Passes defined in Passes.td
 //===----------------------------------------------------------------------===//
 
-#define GEN_PASS_DECL_ABSTRACTRESULTOPT
-#define GEN_PASS_DECL_AFFINEDIALECTPROMOTION
-#define GEN_PASS_DECL_AFFINEDIALECTDEMOTION
-#define GEN_PASS_DECL_ANNOTATECONSTANTOPERANDS
-#define GEN_PASS_DECL_ARRAYVALUECOPY
-#define GEN_PASS_DECL_ASSUMEDRANKOPCONVERSION
-#define GEN_PASS_DECL_CHARACTERCONVERSION
-#define GEN_PASS_DECL_CFGCONVERSION
-#define GEN_PASS_DECL_CUFADDCONSTRUCTOR
-#define GEN_PASS_DECL_CUFDEVICEGLOBAL
-#define GEN_PASS_DECL_CUFGPUTOLLVMCONVERSION
-#define GEN_PASS_DECL_CUFOPCONVERSION
-#define GEN_PASS_DECL_CUFCOMPUTESHAREDMEMORYOFFSETSANDSIZE
-#define GEN_PASS_DECL_EXTERNALNAMECONVERSION
-#define GEN_PASS_DECL_MEMREFDATAFLOWOPT
-#define GEN_PASS_DECL_SIMPLIFYINTRINSICS
-#define GEN_PASS_DECL_MEMORYALLOCATIONOPT
-#define GEN_PASS_DECL_SIMPLIFYREGIONLITE
-#define GEN_PASS_DECL_ALGEBRAICSIMPLIFICATION
-#define GEN_PASS_DECL_POLYMORPHICOPCONVERSION
-#define GEN_PASS_DECL_OPENACCDATAOPERANDCONVERSION
-#define GEN_PASS_DECL_ADDDEBUGINFO
-#define GEN_PASS_DECL_STACKARRAYS
-#define GEN_PASS_DECL_STACKRECLAIM
-#define GEN_PASS_DECL_LOOPVERSIONING
-#define GEN_PASS_DECL_ADDALIASTAGS
-#define GEN_PASS_DECL_VSCALEATTR
-#define GEN_PASS_DECL_FUNCTIONATTR
-#define GEN_PASS_DECL_CONSTANTARGUMENTGLOBALISATIONOPT
-#define GEN_PASS_DECL_COMPILERGENERATEDNAMESCONVERSION
-#define GEN_PASS_DECL_SETRUNTIMECALLATTRIBUTES
-#define GEN_PASS_DECL_GENRUNTIMECALLSFORTEST
-#define GEN_PASS_DECL_SIMPLIFYFIROPERATIONS
+#define GEN_PASS_DECL
 
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 34842f9785942..b230f60b4c59e 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -551,4 +551,14 @@ def SimplifyFIROperations : Pass<"simplify-fir-operations", "mlir::ModuleOp"> {
       "Prefer expanding without using Fortran runtime calls.">];
 }
 
+def OptimizeArrayRepacking
+    : Pass<"optimize-array-repacking", "mlir::func::FuncOp"> {
+  let summary = "Optimizes redundant array repacking operations";
+  let description = [{
+    If the source of fir.pack_array is known to be contiguous,
+    then this pass erases such operations. The corresponding
+    fir.unpack_array operations are also removed.
+  }];
+}
+
 #endif // FLANG_OPTIMIZER_TRANSFORMS_PASSES
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index b54cccbef14fa..e59a6bf2bf224 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -212,10 +212,17 @@ bool hlfir::Entity::mayHaveNonDefaultLowerBounds() const {
   if (auto varIface = getIfVariableInterface())
     return isShapeWithLowerBounds(varIface.getShape());
   // Go through chain of fir.box converts.
-  if (auto convert = getDefiningOp<fir::ConvertOp>())
+  if (auto convert = getDefiningOp<fir::ConvertOp>()) {
     return hlfir::Entity{convert.getValue()}.mayHaveNonDefaultLowerBounds();
-  // TODO: Embox and Rebox do not have hlfir variable interface, but are
-  // easy to reason about.
+  } else if (auto rebox = getDefiningOp<fir::ReboxOp>()) {
+    // If slicing is involved, then the resulting box has
+    // default lower bounds. If there is no slicing,
+    // then the result depends on the shape operand
+    // (whether it has non default lower bounds or not).
+    return !rebox.getSlice() && isShapeWithLowerBounds(rebox.getShape());
+  } else if (auto embox = getDefiningOp<fir::EmboxOp>()) {
+    return !embox.getSlice() && isShapeWithLowerBounds(embox.getShape());
+  }
   return true;
 }
 
@@ -1646,3 +1653,30 @@ bool hlfir::designatePreservesContinuity(hlfir::DesignateOp op) {
   }
   return true;
 }
+
+bool hlfir::isSimplyContiguous(mlir::Value base, bool checkWhole) {
+  hlfir::Entity entity{base};
+  if (entity.isSimplyContiguous())
+    return true;
+
+  // Look at the definition.
+  mlir::Operation *def = base.getDefiningOp();
+  if (!def)
+    return false;
+
+  return mlir::TypeSwitch<mlir::Operation *, bool>(def)
+      .Case<fir::EmboxOp>(
+          [&](auto op) { return fir::isContiguousEmbox(op, checkWhole); })
+      .Case<fir::ReboxOp>([&](auto op) {
+        hlfir::Entity box{op.getBox()};
+        return fir::reboxPreservesContinuity(
+                   op, box.mayHaveNonDefaultLowerBounds(), checkWhole) &&
+               isSimplyContiguous(box, checkWhole);
+      })
+      .Case<fir::DeclareOp, hlfir::DeclareOp>([&](auto op) {
+        return isSimplyContiguous(op.getMemref(), checkWhole);
+      })
+      .Case<fir::ConvertOp>(
+          [&](auto op) { return isSimplyContiguous(op.getValue()); })
+      .Default([](auto &&) { return false; });
+}
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..d04306a48e922 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -1944,6 +1944,128 @@ llvm::LogicalResult fir::EmboxOp::verify() {
   return mlir::success();
 }
 
+/// Returns true if \p extent matches the extent of the \p box's
+/// dimension \p dim.
+bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
+  if (auto op = extent.getDefiningOp<fir::BoxDimsOp>())
+    if (op.getVal() == box && op.getExtent() == extent)
+      if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
+        return *dimOperand == dim;
+  return false;
+}
+
+/// Returns true if \p lb matches the lower bound of the \p box's
+/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
+/// then \p lb may be an integer constant 1.
+bool isBoxLb(mlir::Value box, std::int64_t dim, mlir::Value lb,
+             bool mayHaveNonDefaultLowerBounds = true) {
+  if (auto op = lb.getDefiningOp<fir::BoxDimsOp>()) {
+    if (op.getVal() == box && op.getLowerBound() == lb)
+      if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
+        return *dimOperand == dim;
+  } else if (!mayHaveNonDefaultLowerBounds) {
+    if (auto constantLb = fir::getIntIfConstant(lb))
+      return *constantLb == 1;
+  }
+  return false;
+}
+
+/// Returns true if \p ub matches the upper bound of the \p box's
+/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
+/// then the dimension's lower bound may be an integer constant 1.
+/// Note that the upper bound is usually a result of computation
+/// involving the lower bound and the extent, and the function
+/// tries its best to recognize the computation pattern.
+/// The conservative result 'false' does not necessarily mean
+/// that \p ub is not an actual upper bound value.
+bool isBoxUb(mlir::Value box, std::int64_t dim, mlir::Value ub,
+             bool mayHaveNonDefaultLowerBounds = true) {
+  if (auto sub1 = ub.getDefiningOp<mlir::arith::SubIOp>()) {
+    auto one = fir::getIntIfConstant(sub1.getOperand(1));
+    if (!one || *one != 1)
+      return false;
+    if (auto add = sub1.getOperand(0).getDefiningOp<mlir::arith::AddIOp>())
+      if ((isBoxLb(box, dim, add.getOperand(0)) &&
+           isBoxExtent(box, dim, add.getOperand(1))) ||
+          (isBoxLb(box, dim, add.getOperand(1)) &&
+           isBoxExtent(box, dim, add.getOperand(0))))
+        return true;
+  } else if (!mayHaveNonDefaultLowerBounds) {
+    return isBoxExtent(box, dim, ub);
+  }
+  return false;
+}
+
+/// Checks if the given \p sliceOp specifies a contiguous
+/// array slice. If \p checkWhole is true, then the check
+/// is done for all dimensions, otherwise, only for the innermost
+/// dimension.
+/// The simplest way to prove that this is an contiguous slice
+/// is to check whether the slice stride(s) is 1.
+/// For more complex cases, extra information must be provided
+/// by the caller:
+///   * \p origBox - if not null, then the source array is represented
+///     with this !fir.box value. The box is used to recognize
+///     the full dimension slices, which are specified by the triplets
+///     computed from the dimensions' lower bounds and extents.
+///   * \p mayHaveNonDefaultLowerBounds may be set to false to indicate
+///     that the source entity has default lower bounds, so the full
+///     dimension slices computations may use 1 for the lower bound.
+static bool isContiguousArraySlice(fir::SliceOp sliceOp, bool checkWhole = true,
+                                   mlir::Value origBox = nullptr,
+                                   bool mayHaveNonDefaultLowerBounds = true) {
+  if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
+    // TODO: generalize code for the triples analysis with
+    // hlfir::designatePreservesContinuity, especially when
+    // recognition of the whole dimension slices is added.
+    auto triples = sliceOp.getTriples();
+    assert((triples.size() % 3) == 0 && "invalid triples size");
+
+    // A slice with step=1 in the innermost dimension preserves
+    // the continuity of the array in the innermost dimension.
+    // If checkWhole is false, then check only the innermost slice triples.
+    std::size_t checkUpTo = checkWhole ? triples.size() : 3;
+    checkUpTo = std::min(checkUpTo, triples.size());
+    for (std::size_t i = 0; i < checkUpTo; i += 3) {
+      if (triples[i] != triples[i + 1]) {
+        // This is a section of the dimension. Only allow it
+        // to be the first triple, if the source of the slice
+        // is a boxed array. If it is a raw pointer, then
+        // the result will still be contiguous, as long as
+        // the strides are all ones.
+        // When origBox is not null, we must prove that the triple
+        // covers the whole dimension and the stride is one,
+        // before claiming contiguity for this dimension.
+        if (i != 0 && origBox) {
+          std::int64_t dim = i / 3;
+          if (!isBoxLb(origBox, dim, triples[i],
+                       mayHaveNonDefaultLowerBounds) ||
+              !isBoxUb(origBox, dim, triples[i + 1],
+                       mayHaveNonDefaultLowerBounds))
+            return false;
+        }
+        auto constantStep = fir::getIntIfConstant(triples[i + 2]);
+        if (!constantStep || *constantStep != 1)
+          return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+bool fir::isContiguousEmbox(fir::EmboxOp embox, bool checkWhole) {
+  auto sliceArg = embox.getSlice();
+  if (!sliceArg)
+    return true;
+
+  if (auto sliceOp =
+          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
+    return isContiguousArraySlice(sliceOp, checkWhole);
+
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // EmboxCharOp
 //===----------------------------------------------------------------------===//
@@ -4794,7 +4916,9 @@ mlir::Type fir::applyPathToType(mlir::Type eleTy, mlir::ValueRange path) {
   return eleTy;
 }
 
-bool fir::reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole) {
+bool fir::reboxPreservesContinuity(fir::ReboxOp rebox,
+                                   bool mayHaveNonDefaultLowerBounds,
+                                   bool checkWhole) {
   // If slicing is not involved, then the rebox does not affect
   // the continuity of the array.
   auto sliceArg = rebox.getSlice();
@@ -4802,33 +4926,10 @@ bool fir::reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole) {
     return true;
 
   if (auto sliceOp =
-          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp())) {
-    if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
-      // TODO: generalize code for the triples analysis with
-      // hlfir::designatePreservesContinuity, especially when
-      // recognition of the whole dimension slices is added.
-      auto triples = sliceOp.getTriples();
-      assert((triples.size() % 3) == 0 && "invalid triples size");
-
-      // A slice with step=1 in the innermost dimension preserves
-      // the continuity of the array in the innermost dimension.
-      // If checkWhole is false, then check only the innermost slice triples.
-      std::size_t checkUpTo = checkWhole ? triples.size() : 3;
-      checkUpTo = std::min(checkUpTo, triples.size());
-      for (std::size_t i = 0; i < checkUpTo; i += 3) {
-        if (triples[i] != triples[i + 1]) {
-          // This is a section of the dimension. Only allow it
-          // to be the first triple.
-          if (i != 0)
-            return false;
-          auto constantStep = fir::getIntIfConstant(triples[i + 2]);
-          if (!constantStep || *constantStep != 1)
-            return false;
-        }
-      }
-      return true;
-    }
-  }
+          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
+    return isContiguousArraySlice(sliceOp, checkWhole, rebox.getBox(),
+                                  mayHaveNonDefaultLowerBounds);
+
   return false;
 }
 
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 333bfc8b7aed8..ca8e820608688 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -207,6 +207,10 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
   pm.addPass(fir::createPolymorphicOpConversion());
   pm.addPass(fir::createAssumedRankOpConversion());
 
+  // Optimize redundant array repacking operations,
+  // if the source is known to be contiguous.
+  if (pc.OptLevel.isOptimizingForSpeed())
+    pm.addPass(fir::createOptimizeArrayRepacking());
   pm.addPass(fir::createLowerRepackArraysPass());
   // Expand FIR operations that may use SCF dialect for their
   // implementation. This is a mandatory pass.
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index 846d6c64dbd04..a8812e08c1ccd 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -34,6 +34,7 @@ add_flang_library(FIRTransforms
   SetRuntimeCallAttributes.cpp
   GenRuntimeCallsForTest.cpp
   SimplifyFIROperations.cpp
+  OptimizeArrayRepacking.cpp
 
   DEPENDS
   CUFAttrs
diff --git a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
index 50e7ee5599ab1..056bdf63d914f 100644
--- a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
@@ -214,7 +214,9 @@ static mlir::Value unwrapPassThroughOps(mlir::Value val) {
 /// of the value, otherwise return the value
 static mlir::Value unwrapReboxOp(mlir::Value val) {
   while (fir::ReboxOp rebox = val.getDefiningOp<fir::ReboxOp>()) {
-    if (!fir::reboxPreservesContinuity(rebox, /*checkWhole=*/false)) {
+    if (!fir::reboxPreservesContinuity(rebox,
+                                       /*mayHaveNonDefaultLowerBounds=*/true,
+                                       /*checkWhole=*/false)) {
       LLVM_DEBUG(llvm::dbgs() << "REBOX may produce non-contiguous array: "
                               << rebox << '\n');
       break;
diff --git a/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
new file mode 100644
index 0000000000000..1688f2887a57a
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
@@ -0,0 +1,90 @@
+//===- OptimizeArrayRepacking.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+/// \file
+/// This pass removes redundant fir.pack_array operations, if it can prove
+/// that the source array is contiguous. In this case, it relink all uses
+/// of fir.pack_array result to the source. If such a rewrite happens,
+/// it may turn the using fir.unpack_array operation into one with the same
+/// temp and original operands - these are also removed as redundant.
+//===----------------------------------------------------------------------===//
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Support/Utils.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+namespace fir {
+#define GEN_PASS_DEF_OPTIMIZEARRAYREPACKING
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+} // namespace fir
+
+namespace {
+class OptimizeArrayRepackingPass
+    : public fir::impl::OptimizeArrayRepackingBase<OptimizeArrayRepackingPass> {
+public:
+  void runOnOperation() override;
+};
+
+/// Relinks all uses of redundant fir.pack_array to the source.
+class PackingOfContiguous : public mlir::OpRewritePattern<fir::PackArrayOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  mlir::LogicalResult matchAndRewrite(fir::PackArrayOp,
+                                      mlir::PatternRewriter &) const override;
+};
+
+/// Erases fir.unpack_array with have the matching temp and original
+/// operands.
+class NoopUnpacking : public mlir::OpRewritePattern<fir::UnpackArrayOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  mlir::LogicalResult matchAndRewrite(fir::UnpackArrayOp,
+                                      mlir::PatternRewriter &) const override;
+};
+} // namespace
+
+mlir::LogicalResult
+PackingOfContiguous::matchAndRewrite(fir::PackArrayOp op,
+                                     mlir::PatternRewriter &rewriter) const {
+  mlir::Value box = op.getArray();
+  if (hlfir::isSimplyContiguous(box, !op.getInnermost())) {
+    rewriter.replaceAllUsesWith(op, box);
+    rewriter.eraseOp(op);
+    return mlir::success();
+  }
+  return mlir::failure();
+}
+
+mlir::LogicalResult
+NoopUnpacking::matchAndRewrite(fir::UnpackArrayOp op,
+                               mlir::PatternRewriter &rewriter) const {
+  if (op.getTemp() == op.getOriginal()) {
+    rewriter.eraseOp(op);
+    return mlir::success();
+  }
+  return mlir::failure();
+}
+
+void OptimizeArrayRepackingPass::runOnOperation() {
+  mlir::func::FuncOp funcOp = getOperation();
+  mlir::MLIRContext *context = &getContext();
+  mlir::RewritePatternSet patterns(context);
+  mlir::GreedyRewriteConfig config;
+  config.setRegionSimplificationLevel(
+      mlir::GreedySimplifyRegionLevel::Disabled);
+  patterns.insert<PackingOfContiguous>(context);
+  patterns.insert<NoopUnpacking>(context);
+  if (mlir::failed(
+          mlir::applyPatternsGreedily(funcOp, std::move(patterns), config))) {
+    mlir::emitError(funcOp.getLoc(), "failure in array repacking optimization");
+    signalPassFailure();
+  }
+}
diff --git a/flang/test/Driver/bbc-mlir-pass-pipeline.f90 b/flang/test/Driver/bbc-mlir-pass-pipeline.f90
index 137c19608c38f..f3791fe9f8dc3 100644
--- a/flang/test/Driver/bbc-mlir-pass-pipeline.f90
+++ b/flang/test/Driver/bbc-mlir-pass-pipeline.f90
@@ -47,6 +47,8 @@
 
 ! CHECK-NEXT: PolymorphicOpConversion
 ! CHECK-NEXT: AssumedRankOpConversion
+! CHECK-NEXT: 'func.func' Pipeline
+! CHECK-NEXT:   OptimizeArrayRepacking
 ! CHECK-NEXT: LowerRepackArraysPass
 ! CHECK-NEXT: SimplifyFIROperations
 
diff --git a/flang/test/Driver/mlir-pass-pipeline.f90 b/flang/test/Driver/mlir-pass-pipeline.f90
index 99f192ce7aec2..0bcd055a84b87 100644
--- a/flang/test/Driver/mlir-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-pass-pipeline.f90
@@ -105,6 +105,8 @@
 
 ! ALL-NEXT: PolymorphicOpConversion
 ! ALL-NEXT: AssumedRankOpConversion
+! O2-NEXT:  'func.func' Pipeline
+! O2-NEXT:    OptimizeArrayRepacking
 ! ALL-NEXT: LowerRepackArraysPass
 ! ALL-NEXT: SimplifyFIROperations
 ! O2-NEXT:  AddAliasTags
diff --git a/flang/test/Fir/basic-program.fir b/flang/test/Fir/basic-program.fir
index 7ac8b92f48953..c9fe53bf093a1 100644
--- a/flang/test/Fir/basic-program.fir
+++ b/flang/test/Fir/basic-program.fir
@@ -103,6 +103,8 @@ func.func @_QQmain() {
 
 // PASSES-NEXT: PolymorphicOpConversion
 // PASSES-NEXT: AssumedRankOpConversion
+// PASSES-NEXT: 'func.func' Pipeline
+// PASSES-NEXT:   OptimizeArrayRepacking
 // PASSES-NEXT: LowerRepackArraysPass
 // PASSES-NEXT: SimplifyFIROperations
 // PASSES-NEXT: AddAliasTags
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
index 35530c66f4038..8684a429ea5b4 100644
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
@@ -41,19 +41,11 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_17]] : (i64) -> index
 // CHECK:               fir.do_loop %[[VAL_28:.*]] = %[[VAL_2]] to %[[VAL_27]] step %[[VAL_2]] unordered {
 // CHECK:                 %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (index) -> i64
-// CHECK:                 %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_31:.*]] = fir.convert %[[VAL_29]] : (i64) -> index
-// CHECK:                 %[[VAL_32:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_33:.*]] = arith.addi %[[VAL_31]], %[[VAL_32]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_34:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_33]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_34:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_29]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_35:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
 // CHECK:                 %[[VAL_36:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_37:.*]] = arith.addi %[[VAL_29]], %[[VAL_36]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (i64) -> index
-// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_39]], %[[VAL_40]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_42:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_41]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_42:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_37]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_35]] to %[[VAL_42]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:               %[[VAL_43:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
@@ -61,17 +53,9 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:               fir.do_loop %[[VAL_45:.*]] = %[[VAL_2]] to %[[VAL_44]] step %[[VAL_2]] unordered {
 // CHECK:                 %[[VAL_46:.*]] = fir.convert %[[VAL_45]] : (index) -> i64
 // CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_17]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_48:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_49:.*]] = fir.convert %[[VAL_47]] : (i64) -> index
-// CHECK:                 %[[VAL_50:.*]] = arith.subi %[[VAL_48]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_51:.*]] = arith.addi %[[VAL_49]], %[[VAL_50]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_52:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_51]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_52:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_47]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_53:.*]] = fir.load %[[VAL_52]] : !fir.ref<i32>
-// CHECK:                 %[[VAL_54:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_55:.*]] = fir.convert %[[VAL_46]] : (i64) -> index
-// CHECK:                 %[[VAL_56:.*]] = arith.subi %[[VAL_54]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_57:.*]] = arith.addi %[[VAL_55]], %[[VAL_56]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_58:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_57]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_58:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_46]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_53]] to %[[VAL_58]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:             } else {
@@ -86,11 +70,7 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:                 %[[VAL_67:.*]] = fir.load %[[VAL_66]] : !fir.ref<i32>
 // CHECK:                 %[[VAL_68:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_69:.*]] = arith.addi %[[VAL_61]], %[[VAL_68]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_70:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_71:.*]] = fir.convert %[[VAL_69]] : (i64) -> index
-// CHECK:                 %[[VAL_72:.*]] = arith.subi %[[VAL_70]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_73:.*]] = arith.addi %[[VAL_71]], %[[VAL_72]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_74:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_73]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_74:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_69]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_67]] to %[[VAL_74]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:               %[[VAL_75:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
@@ -104,11 +84,7 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:                 %[[VAL_83:.*]] = arith.addi %[[VAL_81]], %[[VAL_82]] overflow<nsw, nuw> : index
 // CHECK:                 %[[VAL_84:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_83]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_85:.*]] = fir.load %[[VAL_84]] : !fir.ref<i32>
-// CHECK:                 %[[VAL_86:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_87:.*]] = fir.convert %[[VAL_78]] : (i64) -> index
-// CHECK:                 %[[VAL_88:.*]] = arith.subi %[[VAL_86]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_89:.*]] = arith.addi %[[VAL_87]], %[[VAL_88]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_90:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_89]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_90:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_78]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_85]] to %[[VAL_90]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:             }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
index d29e9a26c20ba..73e6e5c2421ff 100644
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
@@ -13,7 +13,6 @@ func.func @matmul_matrix_matrix_integer(%arg0: !hlfir.expr<?x?xi16>, %arg1: !hlf
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0 : i32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi16>) -> !fir.shape<2>
@@ -27,26 +26,14 @@ func.func @matmul_matrix_matrix_integer(%arg0: !hlfir.expr<?x?xi16>, %arg1: !hlf
 // ANSE:             %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xi32>>
 // ANSE:             fir.do_loop %[[VAL_14:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_19:.*]] = arith.addi %[[VAL_15]], %[[VAL_18]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_14]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_15]], %[[VAL_14]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
 // ANSE:                 hlfir.assign %[[VAL_4]] to %[[VAL_22]] : i32, !fir.ref<i32>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_23:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:                 fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                   %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]] = arith.subi %[[VAL_26]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_29:.*]] = arith.addi %[[VAL_25]], %[[VAL_28]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.addi %[[VAL_24]], %[[VAL_30]] : index
-// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_25]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
 // ANSE:                   %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
 // ANSE:                   %[[VAL_34:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_25]], %[[VAL_23]] : (!hlfir.expr<?x?xi16>, index, index) -> i16
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_23]], %[[VAL_24]] : (!hlfir.expr<?x?xi32>, index, index) -> i32
@@ -94,7 +81,6 @@ func.func @matmul_matrix_matrix_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
@@ -108,26 +94,14 @@ func.func @matmul_matrix_matrix_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 // ANSE:             %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
 // ANSE:             fir.do_loop %[[VAL_14:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_19:.*]] = arith.addi %[[VAL_15]], %[[VAL_18]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_14]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_15]], %[[VAL_14]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
 // ANSE:                 hlfir.assign %[[VAL_4]] to %[[VAL_22]] : f32, !fir.ref<f32>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_23:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]] = arith.subi %[[VAL_26]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_29:.*]] = arith.addi %[[VAL_25]], %[[VAL_28]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.addi %[[VAL_24]], %[[VAL_30]] : index
-// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_25]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
 // ANSE:                   %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<f32>
 // ANSE:                   %[[VAL_34:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_25]], %[[VAL_23]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_23]], %[[VAL_24]] : (!hlfir.expr<?x?xf16>, index, index) -> f16
@@ -175,7 +149,6 @@ func.func @matmul_matrix_matrix_complex(%arg0: !hlfir.expr<?x?xcomplex<f32>>, %a
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>
@@ -192,26 +165,14 @@ func.func @matmul_matrix_matrix_complex(%arg0: !hlfir.expr<?x?xcomplex<f32>>, %a
 // ANSE:             %[[VAL_16:.*]] = fir.insert_value %[[VAL_15]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_18]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_24:.*]] = arith.addi %[[VAL_17]], %[[VAL_23]] : index
-// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_18]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                 hlfir.assign %[[VAL_16]] to %[[VAL_25]] : complex<f32>, !fir.ref<complex<f32>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_27:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_28:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_29:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_29]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_28]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_34:.*]] = arith.addi %[[VAL_27]], %[[VAL_33]] : index
-// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_32]], %[[VAL_34]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_28]], %[[VAL_27]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_37:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_28]], %[[VAL_26]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>
 // ANSE:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_26]], %[[VAL_27]] : (!hlfir.expr<?x?xcomplex<f16>>, index, index) -> complex<f16>
@@ -262,7 +223,6 @@ func.func @matmul_matrix_matrix_complex_real(%arg0: !hlfir.expr<?x?xcomplex<f32>
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>
@@ -279,26 +239,14 @@ func.func @matmul_matrix_matrix_complex_real(%arg0: !hlfir.expr<?x?xcomplex<f32>
 // ANSE:             %[[VAL_16:.*]] = fir.insert_value %[[VAL_15]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_18]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_24:.*]] = arith.addi %[[VAL_17]], %[[VAL_23]] : index
-// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_18]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                 hlfir.assign %[[VAL_16]] to %[[VAL_25]] : complex<f32>, !fir.ref<complex<f32>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_27:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_28:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_29:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_29]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_28]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_34:.*]] = arith.addi %[[VAL_27]], %[[VAL_33]] : index
-// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_32]], %[[VAL_34]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_28]], %[[VAL_27]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_37:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_28]], %[[VAL_26]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>
 // ANSE:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_26]], %[[VAL_27]] : (!hlfir.expr<?x?xf16>, index, index) -> f16
@@ -357,7 +305,6 @@ func.func @matmul_matrix_matrix_logical(%arg0: !hlfir.expr<?x?x!fir.logical<1>>,
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant false
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<1>>) -> !fir.shape<2>
@@ -372,26 +319,14 @@ func.func @matmul_matrix_matrix_logical(%arg0: !hlfir.expr<?x?x!fir.logical<1>>,
 // ANSE:             %[[VAL_14:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>
 // ANSE:             fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_16:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.addi %[[VAL_16]], %[[VAL_19]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_15]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_16]], %[[VAL_15]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
 // ANSE:                 hlfir.assign %[[VAL_14]] to %[[VAL_23]] : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:                 fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.addi %[[VAL_26]], %[[VAL_29]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_25]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// ANSE:                   %[[VAL_33:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_26]], %[[VAL_25]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
 // ANSE:                   %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<!fir.logical<4>>
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_26]], %[[VAL_24]] : (!hlfir.expr<?x?x!fir.logical<1>>, index, index) -> !fir.logical<1>
 // ANSE:                   %[[VAL_36:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_24]], %[[VAL_25]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
@@ -446,7 +381,6 @@ func.func @matmul_matrix_vector_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
@@ -457,18 +391,12 @@ func.func @matmul_matrix_vector_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 // ANSE:           ^bb0(%[[VAL_10:.*]]: !fir.ref<!fir.array<?xf32>>):
 // ANSE:             %[[VAL_11:.*]] = fir.embox %[[VAL_10]](%[[VAL_8]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 // ANSE:             fir.do_loop %[[VAL_12:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_11]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:               %[[VAL_14:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_3]] : index
-// ANSE:               %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_14]] : index
-// ANSE:               %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_15]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:               %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_12]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:               hlfir.assign %[[VAL_4]] to %[[VAL_16]] : f32, !fir.ref<f32>
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_11]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_18]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_21]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_18]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<f32>
 // ANSE:                 %[[VAL_24:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]], %[[VAL_17]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
 // ANSE:                 %[[VAL_25:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_17]] : (!hlfir.expr<?xf16>, index) -> f16
@@ -513,7 +441,6 @@ func.func @matmul_vector_matrix_real(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.ex
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
@@ -525,18 +452,12 @@ func.func @matmul_vector_matrix_real(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.ex
 // ANSE:           ^bb0(%[[VAL_11:.*]]: !fir.ref<!fir.array<?xf32>>):
 // ANSE:             %[[VAL_12:.*]] = fir.embox %[[VAL_11]](%[[VAL_9]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 // ANSE:             fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] unordered {
-// ANSE:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_12]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:               %[[VAL_15:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_3]] : index
-// ANSE:               %[[VAL_16:.*]] = arith.addi %[[VAL_13]], %[[VAL_15]] : index
-// ANSE:               %[[VAL_17:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_16]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:               %[[VAL_17:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_13]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:               hlfir.assign %[[VAL_4]] to %[[VAL_17]] : f32, !fir.ref<f32>
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_19:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] {
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_12]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_19]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_22]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f32>
 // ANSE:                 %[[VAL_25:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]] : (!hlfir.expr<?xf32>, index) -> f32
 // ANSE:                 %[[VAL_26:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_18]], %[[VAL_19]] : (!hlfir.expr<?x?xf16>, index, index) -> f16
diff --git a/flang/test/Transforms/optimize-array-repacking.fir b/flang/test/Transforms/optimize-array-repacking.fir
new file mode 100644
index 0000000000000..6269fa441fe44
--- /dev/null
+++ b/flang/test/Transforms/optimize-array-repacking.fir
@@ -0,0 +1,660 @@
+// Test that the redundant fir.[un]pack_array operations
+// are optimized away, when the source is statically known
+// to be contiguous.
+// RUN: fir-opt --optimize-array-repacking %s | FileCheck %s
+
+// FIR is produced by compiling the sources with -mllvm -inline-all.
+// module inner
+// contains
+//   subroutine inner_repack1(x)
+//     real :: x(:)
+//   end subroutine inner_repack1
+//   subroutine inner_repack2(x)
+//     real :: x(:,:)
+//   end subroutine inner_repack2
+// end module inner
+
+// subroutine test_explicit_shape_cst(x)
+//   real :: x(100)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_explicit_shape_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_cst(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_cstEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_explicit_shape_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_var(x, n, l, u)
+//   integer :: n, l, u
+//   real :: x(n)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_explicit_shape_var
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_var(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}, %arg2: !fir.ref<i32> {fir.bindc_name = "l"}, %arg3: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg3 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %4 = fir.load %2 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9 = fir.declare %arg0(%8) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %10 = fir.load %1 : !fir.ref<i32>
+  %11 = fir.load %3 : !fir.ref<i32>
+  %12 = fir.convert %10 : (i32) -> index
+  %13 = fir.convert %11 : (i32) -> index
+  %14 = fir.slice %12, %13, %c1 : (index, index, index) -> !fir.slice<1>
+  %15 = fir.embox %9(%8) [%14] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.dummy_scope : !fir.dscope
+  %17 = fir.pack_array %15 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.declare %17 dummy_scope %16 {uniq_name = "_QFtest_explicit_shape_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %17 to %15 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_cst(x)
+//   real :: x(*)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_assumed_size_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_cst(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_cstEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_assumed_size_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_var(x, l, u)
+//   integer :: l, u
+//   real :: x(*)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_assumed_size_var
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_var(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %4 = fir.declare %arg0(%3) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %10 = fir.embox %4(%3) [%9] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_assumed_size_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_cst(x)
+//   real, allocatable :: x(:)
+//   call repack(x(10:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_allocatable_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_cst(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c41 = arith.constant 41 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_cstEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %3 = fir.shape %c41 : (index) -> !fir.shape<1>
+  %4 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %5:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %6 = fir.shape_shift %5#0, %5#1 : (index, index) -> !fir.shapeshift<1>
+  %7 = fir.array_coor %4(%6) %c10 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, index) -> !fir.ref<f32>
+  %8 = fir.convert %7 : (!fir.ref<f32>) -> !fir.ref<!fir.array<41xf32>>
+  %9 = fir.embox %8(%3) : (!fir.ref<!fir.array<41xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<41xf32>>
+  %10 = fir.convert %9 : (!fir.box<!fir.array<41xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_allocatable_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_var(x, l, u)
+//   integer :: l, u
+//   real, allocatable :: x(:)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_allocatable_var
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_var(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_allocatable_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_allocatable_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_varEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.box_addr %4 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %10:3 = fir.box_dims %4, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %11 = fir.shape_shift %10#0, %10#1 : (index, index) -> !fir.shapeshift<1>
+  %12 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %13 = fir.embox %9(%11) [%12] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.dummy_scope : !fir.dscope
+  %15 = fir.pack_array %13 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.declare %15 dummy_scope %14 {uniq_name = "_QFtest_allocatable_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %15 to %13 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_full(x)
+//   real, allocatable :: x(:,:)
+//   call repack(x(:,:))
+// contains
+//   subroutine repack(x)
+//     real :: x(:,:)
+//   end subroutine repack
+// end subroutine test_allocatable_full
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_full(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_full(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_fullEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %4:3 = fir.box_dims %2, %c1 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %5 = arith.addi %3#0, %3#1 : index
+  %6 = arith.subi %5, %c1 : index
+  %7 = arith.addi %4#0, %4#1 : index
+  %8 = arith.subi %7, %c1 : index
+  %9 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.heap<!fir.array<?x?xf32>>
+  %10 = fir.shape_shift %3#0, %3#1, %4#0, %4#1 : (index, index, index, index) -> !fir.shapeshift<2>
+  %11 = fir.slice %3#0, %6, %c1, %4#0, %8, %c1 : (index, index, index, index, index, index) -> !fir.slice<2>
+  %12 = fir.embox %9(%10) [%11] : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>, !fir.slice<2>) -> !fir.box<!fir.array<?x?xf32>>
+  %13 = fir.dummy_scope : !fir.dscope
+  %14 = fir.pack_array %12 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %15 = fir.declare %14 dummy_scope %13 {uniq_name = "_QFtest_allocatable_fullFrepackEx"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %14 to %12 heap : !fir.box<!fir.array<?x?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_cst_chain(x)
+//   real :: x(100)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_explicit_shape_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_cst_chain(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_cst_chainEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_explicit_shape_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.rebox %10 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %12 = fir.dummy_scope : !fir.dscope
+  %13 = fir.pack_array %11 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.declare %13 dummy_scope %12 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %13 to %11 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_var_chain(x, n, l, u)
+//   integer :: n, l, u
+//   real :: x(n)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_explicit_shape_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_var_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}, %arg2: !fir.ref<i32> {fir.bindc_name = "l"}, %arg3: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg3 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %4 = fir.load %2 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9 = fir.declare %arg0(%8) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %10 = fir.load %1 : !fir.ref<i32>
+  %11 = fir.load %3 : !fir.ref<i32>
+  %12 = fir.convert %10 : (i32) -> index
+  %13 = fir.convert %11 : (i32) -> index
+  %14 = fir.slice %12, %13, %c1 : (index, index, index) -> !fir.slice<1>
+  %15 = fir.embox %9(%8) [%14] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.dummy_scope : !fir.dscope
+  %17 = fir.pack_array %15 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.declare %17 dummy_scope %16 {uniq_name = "_QFtest_explicit_shape_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %19 = fir.rebox %18 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %20 = fir.dummy_scope : !fir.dscope
+  %21 = fir.pack_array %19 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %22 = fir.declare %21 dummy_scope %20 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %21 to %19 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %17 to %15 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_cst_chain(x)
+//   real :: x(*)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_assumed_size_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_cst_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_cst_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_assumed_size_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.rebox %10 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %12 = fir.dummy_scope : !fir.dscope
+  %13 = fir.pack_array %11 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.declare %13 dummy_scope %12 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %13 to %11 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_var_chain(x, l, u)
+//   integer :: l, u
+//   real :: x(*)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_assumed_size_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_var_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %4 = fir.declare %arg0(%3) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %10 = fir.embox %4(%3) [%9] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_assumed_size_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.rebox %13 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %15 = fir.dummy_scope : !fir.dscope
+  %16 = fir.pack_array %14 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.declare %16 dummy_scope %15 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %16 to %14 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_cst_chain(x)
+//   real, allocatable :: x(:)
+//   call repack(x(10:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_allocatable_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_cst_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c41 = arith.constant 41 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_cst_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %3 = fir.shape %c41 : (index) -> !fir.shape<1>
+  %4 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %5:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %6 = fir.shape_shift %5#0, %5#1 : (index, index) -> !fir.shapeshift<1>
+  %7 = fir.array_coor %4(%6) %c10 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, index) -> !fir.ref<f32>
+  %8 = fir.convert %7 : (!fir.ref<f32>) -> !fir.ref<!fir.array<41xf32>>
+  %9 = fir.embox %8(%3) : (!fir.ref<!fir.array<41xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<41xf32>>
+  %10 = fir.convert %9 : (!fir.box<!fir.array<41xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_allocatable_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.rebox %13 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %15 = fir.dummy_scope : !fir.dscope
+  %16 = fir.pack_array %14 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.declare %16 dummy_scope %15 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %16 to %14 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_var_chain(x, l, u)
+//   integer :: l, u
+//   real, allocatable :: x(:)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_allocatable_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_var_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_allocatable_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_allocatable_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_var_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.box_addr %4 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %10:3 = fir.box_dims %4, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %11 = fir.shape_shift %10#0, %10#1 : (index, index) -> !fir.shapeshift<1>
+  %12 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %13 = fir.embox %9(%11) [%12] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.dummy_scope : !fir.dscope
+  %15 = fir.pack_array %13 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.declare %15 dummy_scope %14 {uniq_name = "_QFtest_allocatable_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.rebox %16 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.dummy_scope : !fir.dscope
+  %19 = fir.pack_array %17 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %20 = fir.declare %19 dummy_scope %18 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %19 to %17 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %15 to %13 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_full_chain(x)
+//   real, allocatable :: x(:,:)
+//   call repack(x(:,:))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:,:)
+//     call inner_repack2(x)
+//   end subroutine repack
+// end subroutine test_allocatable_full_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_full_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_full_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_full_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %4:3 = fir.box_dims %2, %c1 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %5 = arith.addi %3#0, %3#1 : index
+  %6 = arith.subi %5, %c1 : index
+  %7 = arith.addi %4#0, %4#1 : index
+  %8 = arith.subi %7, %c1 : index
+  %9 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.heap<!fir.array<?x?xf32>>
+  %10 = fir.shape_shift %3#0, %3#1, %4#0, %4#1 : (index, index, index, index) -> !fir.shapeshift<2>
+  %11 = fir.slice %3#0, %6, %c1, %4#0, %8, %c1 : (index, index, index, index, index, index) -> !fir.slice<2>
+  %12 = fir.embox %9(%10) [%11] : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>, !fir.slice<2>) -> !fir.box<!fir.array<?x?xf32>>
+  %13 = fir.dummy_scope : !fir.dscope
+  %14 = fir.pack_array %12 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %15 = fir.declare %14 dummy_scope %13 {uniq_name = "_QFtest_allocatable_full_chainFrepackEx"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  %16 = fir.rebox %15 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %17 = fir.dummy_scope : !fir.dscope
+  %18 = fir.pack_array %16 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %19 = fir.declare %18 dummy_scope %17 {uniq_name = "_QMinnerFinner_repack2Ex"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %18 to %16 heap : !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %14 to %12 heap : !fir.box<!fir.array<?x?xf32>>
+  return
+}
+
+// TODO: if both fir.pack_array have the same property,
+// then the second one is redundant, because the first
+// repack makes 'x' contiguous.
+// subroutine neg_test_assumed_shape(x)
+//   real :: x(:)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_assumed_shape
+//
+// CHECK-LABEL:   func.func @_QPneg_test_assumed_shape(
+// CHECK: fir.pack_array
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_assumed_shape(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.pack_array %arg0 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %2 = fir.declare %1 dummy_scope %0 {uniq_name = "_QFneg_test_assumed_shapeEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %3 = fir.rebox %2 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %4 = fir.slice %c1, %c50, %c1 : (index, index, index) -> !fir.slice<1>
+  %5 = fir.rebox %3 [%4] : (!fir.box<!fir.array<?xf32>>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
+  %6 = fir.convert %5 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %7 = fir.dummy_scope : !fir.dscope
+  %8 = fir.pack_array %6 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %9 = fir.declare %8 dummy_scope %7 {uniq_name = "_QFneg_test_assumed_shapeFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %8 to %6 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %1 to %arg0 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_non_contig_slice_cst(x)
+//   real :: x(100)
+//   call repack(x(1:50:2))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_non_contig_slice_cst
+//
+// CHECK-LABEL:   func.func @_QPneg_test_non_contig_slice_cst(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_non_contig_slice_cst(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c2 = arith.constant 2 : index
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_cstEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.slice %c1, %c50, %c2 : (index, index, index) -> !fir.slice<1>
+  %4 = fir.embox %2(%1) [%3] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<25xf32>>
+  %5 = fir.convert %4 : (!fir.box<!fir.array<25xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %6 = fir.dummy_scope : !fir.dscope
+  %7 = fir.pack_array %5 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.declare %7 dummy_scope %6 {uniq_name = "_QFneg_test_non_contig_slice_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %7 to %5 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_non_contig_slice_var(x, s)
+//   integer :: s
+//   real :: x(100)
+//   call repack(x(1:50:s))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_non_contig_slice_var
+//
+// CHECK-LABEL:   func.func @_QPneg_test_non_contig_slice_var(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_non_contig_slice_var(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "s"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_varEs"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %3 = fir.declare %arg0(%2) dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_varEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %4 = fir.load %1 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = fir.slice %c1, %c50, %5 : (index, index, index) -> !fir.slice<1>
+  %7 = fir.embox %3(%2) [%6] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFneg_test_non_contig_slice_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_pointer(x)
+//   real, pointer :: x(:)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_pointer
+//
+// CHECK-LABEL:   func.func @_QPneg_test_pointer(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_pointer(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFneg_test_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %4 = fir.shift %3#0 : (index) -> !fir.shift<1>
+  %5 = fir.slice %c1, %c50, %c1 : (index, index, index) -> !fir.slice<1>
+  %6 = fir.rebox %2(%4) [%5] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, !fir.shift<1>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFneg_test_pointerFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}

>From be0a1596ee2380e699d697df5e1bbd079c7e7199 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Thu, 10 Jul 2025 09:29:08 -0700
Subject: [PATCH 2/2] Made utils static.

---
 flang/lib/Optimizer/Dialect/FIROps.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index d04306a48e922..372b7b4610e05 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -1946,7 +1946,7 @@ llvm::LogicalResult fir::EmboxOp::verify() {
 
 /// Returns true if \p extent matches the extent of the \p box's
 /// dimension \p dim.
-bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
+static bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
   if (auto op = extent.getDefiningOp<fir::BoxDimsOp>())
     if (op.getVal() == box && op.getExtent() == extent)
       if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
@@ -1957,8 +1957,8 @@ bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
 /// Returns true if \p lb matches the lower bound of the \p box's
 /// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
 /// then \p lb may be an integer constant 1.
-bool isBoxLb(mlir::Value box, std::int64_t dim, mlir::Value lb,
-             bool mayHaveNonDefaultLowerBounds = true) {
+static bool isBoxLb(mlir::Value box, std::int64_t dim, mlir::Value lb,
+                    bool mayHaveNonDefaultLowerBounds = true) {
   if (auto op = lb.getDefiningOp<fir::BoxDimsOp>()) {
     if (op.getVal() == box && op.getLowerBound() == lb)
       if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
@@ -1978,8 +1978,8 @@ bool isBoxLb(mlir::Value box, std::int64_t dim, mlir::Value lb,
 /// tries its best to recognize the computation pattern.
 /// The conservative result 'false' does not necessarily mean
 /// that \p ub is not an actual upper bound value.
-bool isBoxUb(mlir::Value box, std::int64_t dim, mlir::Value ub,
-             bool mayHaveNonDefaultLowerBounds = true) {
+static bool isBoxUb(mlir::Value box, std::int64_t dim, mlir::Value ub,
+                    bool mayHaveNonDefaultLowerBounds = true) {
   if (auto sub1 = ub.getDefiningOp<mlir::arith::SubIOp>()) {
     auto one = fir::getIntIfConstant(sub1.getOperand(1));
     if (!one || *one != 1)


From flang-commits at lists.llvm.org  Thu Jul 10 10:09:02 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 10:09:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
Message-ID: <llvm.org/llvm/llvm-project/pull/148005@github.com>


https://github.com/kparzysz created https://github.com/llvm/llvm-project/pull/148005

The ALLOCATORS construct is one of the few constructs that require a special form of the associated block.
Convert the AST node to use OmpDirectiveSpecification for the directive and the optional end directive, and to use parser::Block as the body: the form of the block is checked in the semantic checks (with a more meaningful message).

>From 0f4462c45fd80c5417a0970919e600b08ea6e2fa Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Thu, 10 Jul 2025 07:55:32 -0500
Subject: [PATCH] [flang][OpenMP] Convert AST node for ALLOCATORS to use Block
 as body

The ALLOCATORS construct is one of the few constructs that require a
special form of the associated block.
Convert the AST node to use OmpDirectiveSpecification for the directive
and the optional end directive, and to use parser::Block as the body:
the form of the block is checked in the semantic checks (with a more
meaningful message).
---
 flang/examples/FeatureList/FeatureList.cpp    |  1 -
 flang/include/flang/Parser/dump-parse-tree.h  |  1 -
 flang/include/flang/Parser/parse-tree.h       | 21 ++--
 flang/lib/Parser/openmp-parsers.cpp           | 36 +++++--
 flang/lib/Parser/unparse.cpp                  | 24 ++---
 flang/lib/Semantics/check-omp-atomic.cpp      | 31 ------
 flang/lib/Semantics/check-omp-structure.cpp   | 46 ++++++---
 flang/lib/Semantics/openmp-utils.cpp          | 20 ++++
 flang/lib/Semantics/openmp-utils.h            | 13 +++
 flang/lib/Semantics/resolve-directives.cpp    | 97 +++++++++++++------
 .../test/Parser/OpenMP/allocators-unparse.f90 | 80 ++++++++-------
 flang/test/Semantics/OpenMP/allocators07.f90  | 27 ++++++
 12 files changed, 253 insertions(+), 144 deletions(-)
 create mode 100644 flang/test/Semantics/OpenMP/allocators07.f90

diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index a36b8719e365d..e9aeed18ab0b7 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -472,7 +472,6 @@ struct NodeVisitor {
   READ_FEATURE(OmpIteration)
   READ_FEATURE(OmpIterationOffset)
   READ_FEATURE(OmpIterationVector)
-  READ_FEATURE(OmpEndAllocators)
   READ_FEATURE(OmpEndBlockDirective)
   READ_FEATURE(OmpEndCriticalDirective)
   READ_FEATURE(OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..73c224e3ad235 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -578,7 +578,6 @@ class ParseTreeDumper {
   NODE(parser, OmpDetachClause)
   NODE(parser, OmpDoacrossClause)
   NODE(parser, OmpDestroyClause)
-  NODE(parser, OmpEndAllocators)
   NODE(parser, OmpEndBlockDirective)
   NODE(parser, OmpEndCriticalDirective)
   NODE(parser, OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 43954ff735361..fbc39286a95bf 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4598,8 +4598,11 @@ struct OmpClauseList {
 struct OmpDirectiveSpecification {
   ENUM_CLASS(Flags, None, DeprecatedSyntax);
   TUPLE_CLASS_BOILERPLATE(OmpDirectiveSpecification);
+  const OmpDirectiveName &DirName() const {
+    return std::get<OmpDirectiveName>(t);
+  }
   llvm::omp::Directive DirId() const { //
-    return std::get<OmpDirectiveName>(t).v;
+    return DirName().v;
   }
   const OmpArgumentList &Arguments() const;
   const OmpClauseList &Clauses() const;
@@ -4839,17 +4842,17 @@ struct OpenMPExecutableAllocate {
       t;
 };
 
-EMPTY_CLASS(OmpEndAllocators);
-
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
+// Ref: [5.2:180-181], [6.0:315]
+//
+// allocators-construct ->
+//    ALLOCATORS [allocate-clause...]
+//    block
+//    [END ALLOCATORS]
 struct OpenMPAllocatorsConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPAllocatorsConstruct);
   CharBlock source;
-  std::tuple<Verbatim, OmpClauseList, Statement<AllocateStmt>,
-      std::optional<OmpEndAllocators>>
+  std::tuple<OmpDirectiveSpecification, Block,
+      std::optional<OmpDirectiveSpecification>>
       t;
 };
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 3016ce4ccd2f8..811ca2c855a6e 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1276,6 +1276,32 @@ struct OmpEndDirectiveParser {
   llvm::omp::Directive dir_;
 };
 
+struct OmpAllocatorsConstructParser {
+  using resultType = OpenMPAllocatorsConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_allocators) {
+      return std::nullopt;
+    }
+
+    // This should be an allocate-stmt. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_allocators}};
+    return OpenMPAllocatorsConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
+    construct<OpenMPAllocatorsConstruct>(
+        "ALLOCATORS"_tok >= OmpAllocatorsConstructParser{})))
+
 // Parser for an arbitrary OpenMP ATOMIC construct.
 //
 // Depending on circumstances, an ATOMIC construct applies to one or more
@@ -1622,16 +1648,6 @@ TYPE_PARSER(
         maybe(nonemptyList(Parser<OpenMPDeclarativeAllocate>{})) / endOmpLine,
         statement(allocateStmt))))
 
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
-TYPE_PARSER(sourced(construct<OpenMPAllocatorsConstruct>(
-    verbatim("ALLOCATORS"_tok), Parser<OmpClauseList>{} / endOmpLine,
-    statement(allocateStmt), maybe(Parser<OmpEndAllocators>{} / endOmpLine))))
-
-TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
-
 // 2.8.2 Declare Simd construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareSimdConstruct>(
     verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index dcd1ac165adc1..4f692f3e9084f 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2571,7 +2571,7 @@ class UnparseVisitor {
     Word(ToUpperCaseLetters(common::EnumToString(x)));
   }
 
-  void Unparse(const OpenMPAtomicConstruct &x) {
+  template <typename Construct> void UnparseBlockConstruct(const Construct &x) {
     BeginOpenMP();
     Word("!$OMP ");
     Walk(std::get<OmpDirectiveSpecification>(x.t));
@@ -2587,6 +2587,10 @@ class UnparseVisitor {
     }
   }
 
+  void Unparse(const OpenMPAtomicConstruct &x) { //
+    UnparseBlockConstruct(x);
+  }
+
   void Unparse(const OpenMPExecutableAllocate &x) {
     const auto &fields =
         std::get<std::optional<std::list<parser::OpenMPDeclarativeAllocate>>>(
@@ -2614,22 +2618,8 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
-  void Unparse(const OmpEndAllocators &x) {
-    BeginOpenMP();
-    Word("!$OMP END ALLOCATE");
-    Put("\n");
-    EndOpenMP();
-  }
-  void Unparse(const OpenMPAllocatorsConstruct &x) {
-    BeginOpenMP();
-    Word("!$OMP ALLOCATE");
-    Walk(std::get<OmpClauseList>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<Statement<AllocateStmt>>(x.t));
-    if (const auto &end = std::get<std::optional<OmpEndAllocators>>(x.t)) {
-      Walk(*end);
-    }
+  void Unparse(const OpenMPAllocatorsConstruct &x) { //
+    UnparseBlockConstruct(x);
   }
   void Unparse(const OmpAssumeDirective &x) {
     BeginOpenMP();
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index 047c604693460..c5ed8796f0c34 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -47,43 +47,12 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
   return !(e == f);
 }
 
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
 struct AnalyzedCondStmt {
   SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
   parser::CharBlock source;
   SourcedActionStmt ift, iff;
 };
 
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
 // Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
 // is that the ActionStmt will be either an assignment or a pointer-assignment,
 // otherwise return std::nullopt.
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 89c1565bf66aa..9b89d19a91f38 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -495,6 +495,12 @@ template <typename Checker> struct DirectiveSpellingVisitor {
   template <typename T> bool Pre(const T &) { return true; }
   template <typename T> void Post(const T &) {}
 
+  template <typename... Ts>
+  static const parser::OmpDirectiveName &GetDirName(
+      const std::tuple<Ts...> &t) {
+    return std::get<parser::OmpDirectiveSpecification>(t).DirName();
+  }
+
   bool Pre(const parser::OmpSectionsDirective &x) {
     checker_(x.source, x.v);
     return false;
@@ -520,8 +526,7 @@ template <typename Checker> struct DirectiveSpellingVisitor {
     return false;
   }
   bool Pre(const parser::OpenMPAllocatorsConstruct &x) {
-    checker_(
-        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocators);
+    checker_(GetDirName(x.t).source, Directive::OMPD_allocators);
     return false;
   }
   bool Pre(const parser::OmpAssumeDirective &x) {
@@ -1657,26 +1662,45 @@ void OmpStructureChecker::Leave(const parser::OpenMPExecutableAllocate &x) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPAllocatorsConstruct &x) {
   isPredefinedAllocator = true;
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+  PushContextAndClauseSets(
+      dirSpec.DirName().source, llvm::omp::Directive::OMPD_allocators);
+
+  if (block.empty()) {
+    context_.Say(dirSpec.source,
+        "The ALLOCATORS construct should contain a single ALLOCATE statement"_err_en_US);
+    return;
+  }
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const auto *allocate{
+      action ? parser::Unwrap<parser::AllocateStmt>(action.stmt) : nullptr};
+
+  if (!allocate) {
+    const parser::CharBlock &source = action ? action.source : x.source;
+    context_.Say(source,
+        "The body of the ALLOCATORS construct should be an ALLOCATE statement"_err_en_US);
+  }
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             parser::Unwrap<parser::OmpClause::Allocate>(clause)}) {
       CheckVarIsNotPartOfAnotherVar(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       CheckPredefinedAllocatorRestriction(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
   dirContext_.pop_back();
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index fd9596a09cd52..f43d2cc75620e 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -38,6 +38,26 @@
 
 namespace Fortran::semantics::omp {
 
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
 std::string ThisVersion(unsigned version) {
   std::string tv{
       std::to_string(version / 10) + "." + std::to_string(version % 10)};
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index dbb0565215357..a96c008fb26e7 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -29,6 +29,19 @@ class Symbol;
 
 // Add this namespace to avoid potential conflicts
 namespace omp {
+// There is no consistent way to get the source of an ActionStmt, but there
+// is "source" in Statement<T>. This structure keeps the ActionStmt with the
+// extracted source for further use.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x);
+SourcedActionStmt GetActionStmt(const parser::Block &block);
+
 std::string ThisVersion(unsigned version);
 std::string TryVersion(unsigned version);
 
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 299bb6ff876e7..bb3914ffa2b1e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -10,6 +10,7 @@
 
 #include "check-acc-structure.h"
 #include "check-omp-structure.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/fold.h"
@@ -353,12 +354,6 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return true;
   }
 
-  bool Pre(const parser::OmpDirectiveSpecification &x) {
-    PushContext(x.source, x.DirId());
-    return true;
-  }
-  void Post(const parser::OmpDirectiveSpecification &) { PopContext(); }
-
   bool Pre(const parser::OmpMetadirectiveDirective &x) {
     PushContext(x.source, llvm::omp::Directive::OMPD_metadirective);
     return true;
@@ -372,6 +367,29 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     GetContext().withinConstruct = true;
   }
 
+  bool Pre(const parser::OpenMPStandaloneConstruct &x) {
+    common::visit(
+        [&](auto &&s) {
+          using TypeS = llvm::remove_cvref_t<decltype(s)>;
+          // These two cases are handled individually.
+          if constexpr ( //
+              !std::is_same_v<TypeS, parser::OpenMPSimpleStandaloneConstruct> &&
+              !std::is_same_v<TypeS, parser::OmpMetadirectiveDirective>) {
+            PushContext(x.source, s.v.DirId());
+          }
+        },
+        x.u);
+    return true;
+  }
+
+  void Post(const parser::OpenMPStandaloneConstruct &x) {
+    // These two cases are handled individually.
+    if (!std::holds_alternative<parser::OpenMPSimpleStandaloneConstruct>(x.u) &&
+        !std::holds_alternative<parser::OmpMetadirectiveDirective>(x.u)) {
+      PopContext();
+    }
+  }
+
   bool Pre(const parser::OpenMPSimpleStandaloneConstruct &);
   void Post(const parser::OpenMPSimpleStandaloneConstruct &) { PopContext(); }
 
@@ -2149,9 +2167,10 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPExecutableAllocate &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPAllocatorsConstruct &x) {
-  PushContext(x.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  PushContext(x.source, dirSpec.DirId());
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       ResolveOmpObjectList(std::get<parser::OmpObjectList>(allocClause->v.t),
@@ -2234,31 +2253,47 @@ void OmpAttributeVisitor::Post(const parser::OpenMPExecutableAllocate &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
-    if (const auto *alloc{
-            std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
-      CheckAllNamesInAllocateStmt(dir.source,
-          std::get<parser::OmpObjectList>(alloc->v.t),
-          std::get<parser::Statement<parser::AllocateStmt>>(x.t).statement);
-
-      auto &modifiers{OmpGetModifiers(alloc->v)};
-      bool hasAllocator{
-          OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(modifiers) ||
-          OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(modifiers)};
-
-      // TODO: As with allocate directive, exclude the case when a requires
-      //       directive with the dynamic_allocators clause is present in
-      //       the same compilation unit (OMP5.0 2.11.3).
-      if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
-          !hasAllocator) {
-        context_.Say(x.source,
-            "ALLOCATORS directives that appear in a TARGET region "
-            "must specify an allocator"_err_en_US);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const parser::AllocateStmt *allocate{[&]() {
+    if (action) {
+      if (auto *alloc{std::get_if<common::Indirection<parser::AllocateStmt>>(
+              &action.stmt->u)}) {
+        return &alloc->value();
+      }
+    }
+    return static_cast<const parser::AllocateStmt *>(nullptr);
+  }()};
+
+  if (allocate) {
+    for (const auto &clause : dirSpec.Clauses().v) {
+      if (auto *alloc{std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
+        CheckAllNamesInAllocateStmt(
+            x.source, std::get<parser::OmpObjectList>(alloc->v.t), *allocate);
+
+        using OmpAllocatorSimpleModifier = parser::OmpAllocatorSimpleModifier;
+        using OmpAllocatorComplexModifier = parser::OmpAllocatorComplexModifier;
+
+        auto &modifiers{OmpGetModifiers(alloc->v)};
+        bool hasAllocator{
+            OmpGetUniqueModifier<OmpAllocatorSimpleModifier>(modifiers) ||
+            OmpGetUniqueModifier<OmpAllocatorComplexModifier>(modifiers)};
+
+        // TODO: As with allocate directive, exclude the case when a requires
+        //       directive with the dynamic_allocators clause is present in
+        //       the same compilation unit (OMP5.0 2.11.3).
+        if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
+            !hasAllocator) {
+          context_.Say(x.source,
+              "ALLOCATORS directives that appear in a TARGET region "
+              "must specify an allocator"_err_en_US);
+        }
       }
     }
   }
+
   PopContext();
 }
 
diff --git a/flang/test/Parser/OpenMP/allocators-unparse.f90 b/flang/test/Parser/OpenMP/allocators-unparse.f90
index 5cd0230471fc4..70feb7a6b527e 100644
--- a/flang/test/Parser/OpenMP/allocators-unparse.f90
+++ b/flang/test/Parser/OpenMP/allocators-unparse.f90
@@ -15,48 +15,62 @@ subroutine allocate()
 
   !$omp allocators allocate(align(32): arr2)
     allocate(arr2(5, 3))
+  !$omp end allocators
 end subroutine allocate
 
 !CHECK: INTEGER, ALLOCATABLE :: arr1(:), arr2(:,:)
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(omp_default_mem_alloc: arr1)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(omp_default_mem_alloc: arr1)
 !CHECK-NEXT: ALLOCATE(arr1(5))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) ALL&
-!CHECK-NEXT:!$OMP&OCATE(omp_default_mem_alloc: arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) A&
+!CHECK-NEXT:!$OMP&LLOCATE(omp_default_mem_alloc: arr2)
 !CHECK-NEXT: ALLOCATE(arr1(10), arr2(3,2))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALIGN(32): arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALIGN(32): arr2)
 !CHECK-NEXT: ALLOCATE(arr2(5,3))
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr1'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr1'
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorComplexModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
-!PARSE-TREE-NEXT: OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr1'
-!PARSE-TREE-NEXT: AllocateShapeSpec
-!PARSE-TREE-NEXT: Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '10'
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorComplexModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
+!PARSE-TREE-NEXT: | | OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr1'
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | | | AllocateShapeSpec
+!PARSE-TREE-NEXT: | | | | | Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '5'
+!PARSE-TREE-NEXT: | | | | AllocateShapeSpec
+!PARSE-TREE-NEXT: | | | | | Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '3'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList ->
+!PARSE-TREE-NEXT: | | Flags = None
diff --git a/flang/test/Semantics/OpenMP/allocators07.f90 b/flang/test/Semantics/OpenMP/allocators07.f90
new file mode 100644
index 0000000000000..a28f706965cb1
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/allocators07.f90
@@ -0,0 +1,27 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+subroutine f00
+  implicit none
+  integer, allocatable :: a(:)
+
+  !$omp allocators allocate(a)
+!ERROR: The body of the ALLOCATORS construct should be an ALLOCATE statement
+  continue
+end
+
+subroutine f01
+  implicit none
+  integer, allocatable :: a(:)
+
+!ERROR: The ALLOCATORS construct should contain a single ALLOCATE statement
+  !$omp allocators allocate(a)
+  !$omp end allocators
+end
+
+subroutine f02
+  implicit none
+  integer, allocatable :: a(:)
+
+!ERROR: The ALLOCATORS construct should contain a single ALLOCATE statement
+  !$omp allocators allocate(a)
+end


From flang-commits at lists.llvm.org  Thu Jul 10 10:09:36 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 10:09:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <686ff3d0.050a0220.221247.a816@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-openmp

@llvm/pr-subscribers-flang-semantics

Author: Krzysztof Parzyszek (kparzysz)

<details>
<summary>Changes</summary>

The ALLOCATORS construct is one of the few constructs that require a special form of the associated block.
Convert the AST node to use OmpDirectiveSpecification for the directive and the optional end directive, and to use parser::Block as the body: the form of the block is checked in the semantic checks (with a more meaningful message).

---

Patch is 25.97 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/148005.diff


12 Files Affected:

- (modified) flang/examples/FeatureList/FeatureList.cpp (-1) 
- (modified) flang/include/flang/Parser/dump-parse-tree.h (-1) 
- (modified) flang/include/flang/Parser/parse-tree.h (+12-9) 
- (modified) flang/lib/Parser/openmp-parsers.cpp (+26-10) 
- (modified) flang/lib/Parser/unparse.cpp (+7-17) 
- (modified) flang/lib/Semantics/check-omp-atomic.cpp (-31) 
- (modified) flang/lib/Semantics/check-omp-structure.cpp (+35-11) 
- (modified) flang/lib/Semantics/openmp-utils.cpp (+20) 
- (modified) flang/lib/Semantics/openmp-utils.h (+13) 
- (modified) flang/lib/Semantics/resolve-directives.cpp (+66-31) 
- (modified) flang/test/Parser/OpenMP/allocators-unparse.f90 (+47-33) 
- (added) flang/test/Semantics/OpenMP/allocators07.f90 (+27) 


``````````diff
diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index a36b8719e365d..e9aeed18ab0b7 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -472,7 +472,6 @@ struct NodeVisitor {
   READ_FEATURE(OmpIteration)
   READ_FEATURE(OmpIterationOffset)
   READ_FEATURE(OmpIterationVector)
-  READ_FEATURE(OmpEndAllocators)
   READ_FEATURE(OmpEndBlockDirective)
   READ_FEATURE(OmpEndCriticalDirective)
   READ_FEATURE(OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..73c224e3ad235 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -578,7 +578,6 @@ class ParseTreeDumper {
   NODE(parser, OmpDetachClause)
   NODE(parser, OmpDoacrossClause)
   NODE(parser, OmpDestroyClause)
-  NODE(parser, OmpEndAllocators)
   NODE(parser, OmpEndBlockDirective)
   NODE(parser, OmpEndCriticalDirective)
   NODE(parser, OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 43954ff735361..fbc39286a95bf 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4598,8 +4598,11 @@ struct OmpClauseList {
 struct OmpDirectiveSpecification {
   ENUM_CLASS(Flags, None, DeprecatedSyntax);
   TUPLE_CLASS_BOILERPLATE(OmpDirectiveSpecification);
+  const OmpDirectiveName &DirName() const {
+    return std::get<OmpDirectiveName>(t);
+  }
   llvm::omp::Directive DirId() const { //
-    return std::get<OmpDirectiveName>(t).v;
+    return DirName().v;
   }
   const OmpArgumentList &Arguments() const;
   const OmpClauseList &Clauses() const;
@@ -4839,17 +4842,17 @@ struct OpenMPExecutableAllocate {
       t;
 };
 
-EMPTY_CLASS(OmpEndAllocators);
-
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
+// Ref: [5.2:180-181], [6.0:315]
+//
+// allocators-construct ->
+//    ALLOCATORS [allocate-clause...]
+//    block
+//    [END ALLOCATORS]
 struct OpenMPAllocatorsConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPAllocatorsConstruct);
   CharBlock source;
-  std::tuple<Verbatim, OmpClauseList, Statement<AllocateStmt>,
-      std::optional<OmpEndAllocators>>
+  std::tuple<OmpDirectiveSpecification, Block,
+      std::optional<OmpDirectiveSpecification>>
       t;
 };
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 3016ce4ccd2f8..811ca2c855a6e 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1276,6 +1276,32 @@ struct OmpEndDirectiveParser {
   llvm::omp::Directive dir_;
 };
 
+struct OmpAllocatorsConstructParser {
+  using resultType = OpenMPAllocatorsConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_allocators) {
+      return std::nullopt;
+    }
+
+    // This should be an allocate-stmt. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_allocators}};
+    return OpenMPAllocatorsConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
+    construct<OpenMPAllocatorsConstruct>(
+        "ALLOCATORS"_tok >= OmpAllocatorsConstructParser{})))
+
 // Parser for an arbitrary OpenMP ATOMIC construct.
 //
 // Depending on circumstances, an ATOMIC construct applies to one or more
@@ -1622,16 +1648,6 @@ TYPE_PARSER(
         maybe(nonemptyList(Parser<OpenMPDeclarativeAllocate>{})) / endOmpLine,
         statement(allocateStmt))))
 
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
-TYPE_PARSER(sourced(construct<OpenMPAllocatorsConstruct>(
-    verbatim("ALLOCATORS"_tok), Parser<OmpClauseList>{} / endOmpLine,
-    statement(allocateStmt), maybe(Parser<OmpEndAllocators>{} / endOmpLine))))
-
-TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
-
 // 2.8.2 Declare Simd construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareSimdConstruct>(
     verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index dcd1ac165adc1..4f692f3e9084f 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2571,7 +2571,7 @@ class UnparseVisitor {
     Word(ToUpperCaseLetters(common::EnumToString(x)));
   }
 
-  void Unparse(const OpenMPAtomicConstruct &x) {
+  template <typename Construct> void UnparseBlockConstruct(const Construct &x) {
     BeginOpenMP();
     Word("!$OMP ");
     Walk(std::get<OmpDirectiveSpecification>(x.t));
@@ -2587,6 +2587,10 @@ class UnparseVisitor {
     }
   }
 
+  void Unparse(const OpenMPAtomicConstruct &x) { //
+    UnparseBlockConstruct(x);
+  }
+
   void Unparse(const OpenMPExecutableAllocate &x) {
     const auto &fields =
         std::get<std::optional<std::list<parser::OpenMPDeclarativeAllocate>>>(
@@ -2614,22 +2618,8 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
-  void Unparse(const OmpEndAllocators &x) {
-    BeginOpenMP();
-    Word("!$OMP END ALLOCATE");
-    Put("\n");
-    EndOpenMP();
-  }
-  void Unparse(const OpenMPAllocatorsConstruct &x) {
-    BeginOpenMP();
-    Word("!$OMP ALLOCATE");
-    Walk(std::get<OmpClauseList>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<Statement<AllocateStmt>>(x.t));
-    if (const auto &end = std::get<std::optional<OmpEndAllocators>>(x.t)) {
-      Walk(*end);
-    }
+  void Unparse(const OpenMPAllocatorsConstruct &x) { //
+    UnparseBlockConstruct(x);
   }
   void Unparse(const OmpAssumeDirective &x) {
     BeginOpenMP();
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index 047c604693460..c5ed8796f0c34 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -47,43 +47,12 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
   return !(e == f);
 }
 
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
 struct AnalyzedCondStmt {
   SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
   parser::CharBlock source;
   SourcedActionStmt ift, iff;
 };
 
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
 // Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
 // is that the ActionStmt will be either an assignment or a pointer-assignment,
 // otherwise return std::nullopt.
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 89c1565bf66aa..9b89d19a91f38 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -495,6 +495,12 @@ template <typename Checker> struct DirectiveSpellingVisitor {
   template <typename T> bool Pre(const T &) { return true; }
   template <typename T> void Post(const T &) {}
 
+  template <typename... Ts>
+  static const parser::OmpDirectiveName &GetDirName(
+      const std::tuple<Ts...> &t) {
+    return std::get<parser::OmpDirectiveSpecification>(t).DirName();
+  }
+
   bool Pre(const parser::OmpSectionsDirective &x) {
     checker_(x.source, x.v);
     return false;
@@ -520,8 +526,7 @@ template <typename Checker> struct DirectiveSpellingVisitor {
     return false;
   }
   bool Pre(const parser::OpenMPAllocatorsConstruct &x) {
-    checker_(
-        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocators);
+    checker_(GetDirName(x.t).source, Directive::OMPD_allocators);
     return false;
   }
   bool Pre(const parser::OmpAssumeDirective &x) {
@@ -1657,26 +1662,45 @@ void OmpStructureChecker::Leave(const parser::OpenMPExecutableAllocate &x) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPAllocatorsConstruct &x) {
   isPredefinedAllocator = true;
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+  PushContextAndClauseSets(
+      dirSpec.DirName().source, llvm::omp::Directive::OMPD_allocators);
+
+  if (block.empty()) {
+    context_.Say(dirSpec.source,
+        "The ALLOCATORS construct should contain a single ALLOCATE statement"_err_en_US);
+    return;
+  }
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const auto *allocate{
+      action ? parser::Unwrap<parser::AllocateStmt>(action.stmt) : nullptr};
+
+  if (!allocate) {
+    const parser::CharBlock &source = action ? action.source : x.source;
+    context_.Say(source,
+        "The body of the ALLOCATORS construct should be an ALLOCATE statement"_err_en_US);
+  }
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             parser::Unwrap<parser::OmpClause::Allocate>(clause)}) {
       CheckVarIsNotPartOfAnotherVar(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       CheckPredefinedAllocatorRestriction(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
   dirContext_.pop_back();
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index fd9596a09cd52..f43d2cc75620e 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -38,6 +38,26 @@
 
 namespace Fortran::semantics::omp {
 
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
 std::string ThisVersion(unsigned version) {
   std::string tv{
       std::to_string(version / 10) + "." + std::to_string(version % 10)};
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index dbb0565215357..a96c008fb26e7 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -29,6 +29,19 @@ class Symbol;
 
 // Add this namespace to avoid potential conflicts
 namespace omp {
+// There is no consistent way to get the source of an ActionStmt, but there
+// is "source" in Statement<T>. This structure keeps the ActionStmt with the
+// extracted source for further use.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x);
+SourcedActionStmt GetActionStmt(const parser::Block &block);
+
 std::string ThisVersion(unsigned version);
 std::string TryVersion(unsigned version);
 
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 299bb6ff876e7..bb3914ffa2b1e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -10,6 +10,7 @@
 
 #include "check-acc-structure.h"
 #include "check-omp-structure.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/fold.h"
@@ -353,12 +354,6 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return true;
   }
 
-  bool Pre(const parser::OmpDirectiveSpecification &x) {
-    PushContext(x.source, x.DirId());
-    return true;
-  }
-  void Post(const parser::OmpDirectiveSpecification &) { PopContext(); }
-
   bool Pre(const parser::OmpMetadirectiveDirective &x) {
     PushContext(x.source, llvm::omp::Directive::OMPD_metadirective);
     return true;
@@ -372,6 +367,29 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     GetContext().withinConstruct = true;
   }
 
+  bool Pre(const parser::OpenMPStandaloneConstruct &x) {
+    common::visit(
+        [&](auto &&s) {
+          using TypeS = llvm::remove_cvref_t<decltype(s)>;
+          // These two cases are handled individually.
+          if constexpr ( //
+              !std::is_same_v<TypeS, parser::OpenMPSimpleStandaloneConstruct> &&
+              !std::is_same_v<TypeS, parser::OmpMetadirectiveDirective>) {
+            PushContext(x.source, s.v.DirId());
+          }
+        },
+        x.u);
+    return true;
+  }
+
+  void Post(const parser::OpenMPStandaloneConstruct &x) {
+    // These two cases are handled individually.
+    if (!std::holds_alternative<parser::OpenMPSimpleStandaloneConstruct>(x.u) &&
+        !std::holds_alternative<parser::OmpMetadirectiveDirective>(x.u)) {
+      PopContext();
+    }
+  }
+
   bool Pre(const parser::OpenMPSimpleStandaloneConstruct &);
   void Post(const parser::OpenMPSimpleStandaloneConstruct &) { PopContext(); }
 
@@ -2149,9 +2167,10 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPExecutableAllocate &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPAllocatorsConstruct &x) {
-  PushContext(x.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  PushContext(x.source, dirSpec.DirId());
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       ResolveOmpObjectList(std::get<parser::OmpObjectList>(allocClause->v.t),
@@ -2234,31 +2253,47 @@ void OmpAttributeVisitor::Post(const parser::OpenMPExecutableAllocate &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
-    if (const auto *alloc{
-            std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
-      CheckAllNamesInAllocateStmt(dir.source,
-          std::get<parser::OmpObjectList>(alloc->v.t),
-          std::get<parser::Statement<parser::AllocateStmt>>(x.t).statement);
-
-      auto &modifiers{OmpGetModifiers(alloc->v)};
-      bool hasAllocator{
-          OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(modifiers) ||
-          OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(modifiers)};
-
-      // TODO: As with allocate directive, exclude the case when a requires
-      //       directive with the dynamic_allocators clause is present in
-      //       the same compilation unit (OMP5.0 2.11.3).
-      if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
-          !hasAllocator) {
-        context_.Say(x.source,
-            "ALLOCATORS directives that appear in a TARGET region "
-            "must specify an allocator"_err_en_US);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const parser::AllocateStmt *allocate{[&]() {
+    if (action) {
+      if (auto *alloc{std::get_if<common::Indirection<parser::AllocateStmt>>(
+              &action.stmt->u)}) {
+        return &alloc->value();
+      }
+    }
+    return static_cast<const parser::AllocateStmt *>(nullptr);
+  }()};
+
+  if (allocate) {
+    for (const auto &clause : dirSpec.Clauses().v) {
+      if (auto *alloc{std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
+        CheckAllNamesInAllocateStmt(
+            x.source, std::get<parser::OmpObjectList>(alloc->v.t), *allocate);
+
+        using OmpAllocatorSimpleModifier = parser::OmpAllocatorSimpleModifier;
+        using OmpAllocatorComplexModifier = parser::OmpAllocatorComplexModifier;
+
+        auto &modifiers{OmpGetModifiers(alloc->v)};
+        bool hasAllocator{
+            OmpGetUniqueModifier<OmpAllocatorSimpleModifier>(modifiers) ||
+            OmpGetUniqueModifier<OmpAllocatorComplexModifier>(modifiers)};
+
+        // TODO: As with allocate directive, exclude the case when a requires
+        //       directive with the dynamic_allocators clause is present in
+        //       the same compilation unit (OMP5.0 2.11.3).
+        if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
+            !hasAllocator) {
+          context_.Say(x.source,
+              "ALLOCATORS directives that appear in a TARGET region "
+              "must specify an allocator"_err_en_US);
+        }
       }
     }
   }
+
   PopContext();
 }
 
diff --git a/flang/test/Parser/OpenMP/allocators-unparse.f90 b/flang/test/Parser/OpenMP/allocators-unparse.f90
index 5cd0230471fc4..70feb7a6b527e 100644
--- a/flang/test/Parser/OpenMP/allocators-unparse.f90
+++ b/flang/test/Parser/OpenMP/allocators-unparse.f90
@@ -15,48 +15,62 @@ subroutine allocate()
 
   !$omp allocators allocate(align(32): arr2)
     allocate(arr2(5, 3))
+  !$omp end allocators
 end subroutine allocate
 
 !CHECK: INTEGER, ALLOCATABLE :: arr1(:), arr2(:,:)
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(omp_default_mem_alloc: arr1)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(omp_default_mem_alloc: arr1)
 !CHECK-NEXT: ALLOCATE(arr1(5))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) ALL&
-!CHECK-NEXT:!$OMP&OCATE(omp_default_mem_alloc: arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) A&
+!CHECK-NEXT:!$OMP&LLOCATE(omp_default_mem_alloc: arr2)
 !CHECK-NEXT: ALLOCATE(arr1(10), arr2(3,2))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALIGN(32): arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALIGN(32): arr2)
 !CHECK-NEXT: ALLOCATE(arr2(5,3))
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Thu Jul 10 10:09:36 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 10:09:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <686ff3d0.630a0220.327294.9fff@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: Krzysztof Parzyszek (kparzysz)

<details>
<summary>Changes</summary>

The ALLOCATORS construct is one of the few constructs that require a special form of the associated block.
Convert the AST node to use OmpDirectiveSpecification for the directive and the optional end directive, and to use parser::Block as the body: the form of the block is checked in the semantic checks (with a more meaningful message).

---

Patch is 25.97 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/148005.diff


12 Files Affected:

- (modified) flang/examples/FeatureList/FeatureList.cpp (-1) 
- (modified) flang/include/flang/Parser/dump-parse-tree.h (-1) 
- (modified) flang/include/flang/Parser/parse-tree.h (+12-9) 
- (modified) flang/lib/Parser/openmp-parsers.cpp (+26-10) 
- (modified) flang/lib/Parser/unparse.cpp (+7-17) 
- (modified) flang/lib/Semantics/check-omp-atomic.cpp (-31) 
- (modified) flang/lib/Semantics/check-omp-structure.cpp (+35-11) 
- (modified) flang/lib/Semantics/openmp-utils.cpp (+20) 
- (modified) flang/lib/Semantics/openmp-utils.h (+13) 
- (modified) flang/lib/Semantics/resolve-directives.cpp (+66-31) 
- (modified) flang/test/Parser/OpenMP/allocators-unparse.f90 (+47-33) 
- (added) flang/test/Semantics/OpenMP/allocators07.f90 (+27) 


``````````diff
diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index a36b8719e365d..e9aeed18ab0b7 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -472,7 +472,6 @@ struct NodeVisitor {
   READ_FEATURE(OmpIteration)
   READ_FEATURE(OmpIterationOffset)
   READ_FEATURE(OmpIterationVector)
-  READ_FEATURE(OmpEndAllocators)
   READ_FEATURE(OmpEndBlockDirective)
   READ_FEATURE(OmpEndCriticalDirective)
   READ_FEATURE(OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..73c224e3ad235 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -578,7 +578,6 @@ class ParseTreeDumper {
   NODE(parser, OmpDetachClause)
   NODE(parser, OmpDoacrossClause)
   NODE(parser, OmpDestroyClause)
-  NODE(parser, OmpEndAllocators)
   NODE(parser, OmpEndBlockDirective)
   NODE(parser, OmpEndCriticalDirective)
   NODE(parser, OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 43954ff735361..fbc39286a95bf 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4598,8 +4598,11 @@ struct OmpClauseList {
 struct OmpDirectiveSpecification {
   ENUM_CLASS(Flags, None, DeprecatedSyntax);
   TUPLE_CLASS_BOILERPLATE(OmpDirectiveSpecification);
+  const OmpDirectiveName &DirName() const {
+    return std::get<OmpDirectiveName>(t);
+  }
   llvm::omp::Directive DirId() const { //
-    return std::get<OmpDirectiveName>(t).v;
+    return DirName().v;
   }
   const OmpArgumentList &Arguments() const;
   const OmpClauseList &Clauses() const;
@@ -4839,17 +4842,17 @@ struct OpenMPExecutableAllocate {
       t;
 };
 
-EMPTY_CLASS(OmpEndAllocators);
-
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
+// Ref: [5.2:180-181], [6.0:315]
+//
+// allocators-construct ->
+//    ALLOCATORS [allocate-clause...]
+//    block
+//    [END ALLOCATORS]
 struct OpenMPAllocatorsConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPAllocatorsConstruct);
   CharBlock source;
-  std::tuple<Verbatim, OmpClauseList, Statement<AllocateStmt>,
-      std::optional<OmpEndAllocators>>
+  std::tuple<OmpDirectiveSpecification, Block,
+      std::optional<OmpDirectiveSpecification>>
       t;
 };
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 3016ce4ccd2f8..811ca2c855a6e 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1276,6 +1276,32 @@ struct OmpEndDirectiveParser {
   llvm::omp::Directive dir_;
 };
 
+struct OmpAllocatorsConstructParser {
+  using resultType = OpenMPAllocatorsConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_allocators) {
+      return std::nullopt;
+    }
+
+    // This should be an allocate-stmt. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_allocators}};
+    return OpenMPAllocatorsConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
+    construct<OpenMPAllocatorsConstruct>(
+        "ALLOCATORS"_tok >= OmpAllocatorsConstructParser{})))
+
 // Parser for an arbitrary OpenMP ATOMIC construct.
 //
 // Depending on circumstances, an ATOMIC construct applies to one or more
@@ -1622,16 +1648,6 @@ TYPE_PARSER(
         maybe(nonemptyList(Parser<OpenMPDeclarativeAllocate>{})) / endOmpLine,
         statement(allocateStmt))))
 
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
-TYPE_PARSER(sourced(construct<OpenMPAllocatorsConstruct>(
-    verbatim("ALLOCATORS"_tok), Parser<OmpClauseList>{} / endOmpLine,
-    statement(allocateStmt), maybe(Parser<OmpEndAllocators>{} / endOmpLine))))
-
-TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
-
 // 2.8.2 Declare Simd construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareSimdConstruct>(
     verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index dcd1ac165adc1..4f692f3e9084f 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2571,7 +2571,7 @@ class UnparseVisitor {
     Word(ToUpperCaseLetters(common::EnumToString(x)));
   }
 
-  void Unparse(const OpenMPAtomicConstruct &x) {
+  template <typename Construct> void UnparseBlockConstruct(const Construct &x) {
     BeginOpenMP();
     Word("!$OMP ");
     Walk(std::get<OmpDirectiveSpecification>(x.t));
@@ -2587,6 +2587,10 @@ class UnparseVisitor {
     }
   }
 
+  void Unparse(const OpenMPAtomicConstruct &x) { //
+    UnparseBlockConstruct(x);
+  }
+
   void Unparse(const OpenMPExecutableAllocate &x) {
     const auto &fields =
         std::get<std::optional<std::list<parser::OpenMPDeclarativeAllocate>>>(
@@ -2614,22 +2618,8 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
-  void Unparse(const OmpEndAllocators &x) {
-    BeginOpenMP();
-    Word("!$OMP END ALLOCATE");
-    Put("\n");
-    EndOpenMP();
-  }
-  void Unparse(const OpenMPAllocatorsConstruct &x) {
-    BeginOpenMP();
-    Word("!$OMP ALLOCATE");
-    Walk(std::get<OmpClauseList>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<Statement<AllocateStmt>>(x.t));
-    if (const auto &end = std::get<std::optional<OmpEndAllocators>>(x.t)) {
-      Walk(*end);
-    }
+  void Unparse(const OpenMPAllocatorsConstruct &x) { //
+    UnparseBlockConstruct(x);
   }
   void Unparse(const OmpAssumeDirective &x) {
     BeginOpenMP();
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index 047c604693460..c5ed8796f0c34 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -47,43 +47,12 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
   return !(e == f);
 }
 
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
 struct AnalyzedCondStmt {
   SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
   parser::CharBlock source;
   SourcedActionStmt ift, iff;
 };
 
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
 // Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
 // is that the ActionStmt will be either an assignment or a pointer-assignment,
 // otherwise return std::nullopt.
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 89c1565bf66aa..9b89d19a91f38 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -495,6 +495,12 @@ template <typename Checker> struct DirectiveSpellingVisitor {
   template <typename T> bool Pre(const T &) { return true; }
   template <typename T> void Post(const T &) {}
 
+  template <typename... Ts>
+  static const parser::OmpDirectiveName &GetDirName(
+      const std::tuple<Ts...> &t) {
+    return std::get<parser::OmpDirectiveSpecification>(t).DirName();
+  }
+
   bool Pre(const parser::OmpSectionsDirective &x) {
     checker_(x.source, x.v);
     return false;
@@ -520,8 +526,7 @@ template <typename Checker> struct DirectiveSpellingVisitor {
     return false;
   }
   bool Pre(const parser::OpenMPAllocatorsConstruct &x) {
-    checker_(
-        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocators);
+    checker_(GetDirName(x.t).source, Directive::OMPD_allocators);
     return false;
   }
   bool Pre(const parser::OmpAssumeDirective &x) {
@@ -1657,26 +1662,45 @@ void OmpStructureChecker::Leave(const parser::OpenMPExecutableAllocate &x) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPAllocatorsConstruct &x) {
   isPredefinedAllocator = true;
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+  PushContextAndClauseSets(
+      dirSpec.DirName().source, llvm::omp::Directive::OMPD_allocators);
+
+  if (block.empty()) {
+    context_.Say(dirSpec.source,
+        "The ALLOCATORS construct should contain a single ALLOCATE statement"_err_en_US);
+    return;
+  }
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const auto *allocate{
+      action ? parser::Unwrap<parser::AllocateStmt>(action.stmt) : nullptr};
+
+  if (!allocate) {
+    const parser::CharBlock &source = action ? action.source : x.source;
+    context_.Say(source,
+        "The body of the ALLOCATORS construct should be an ALLOCATE statement"_err_en_US);
+  }
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             parser::Unwrap<parser::OmpClause::Allocate>(clause)}) {
       CheckVarIsNotPartOfAnotherVar(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       CheckPredefinedAllocatorRestriction(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
   dirContext_.pop_back();
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index fd9596a09cd52..f43d2cc75620e 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -38,6 +38,26 @@
 
 namespace Fortran::semantics::omp {
 
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
 std::string ThisVersion(unsigned version) {
   std::string tv{
       std::to_string(version / 10) + "." + std::to_string(version % 10)};
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index dbb0565215357..a96c008fb26e7 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -29,6 +29,19 @@ class Symbol;
 
 // Add this namespace to avoid potential conflicts
 namespace omp {
+// There is no consistent way to get the source of an ActionStmt, but there
+// is "source" in Statement<T>. This structure keeps the ActionStmt with the
+// extracted source for further use.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x);
+SourcedActionStmt GetActionStmt(const parser::Block &block);
+
 std::string ThisVersion(unsigned version);
 std::string TryVersion(unsigned version);
 
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 299bb6ff876e7..bb3914ffa2b1e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -10,6 +10,7 @@
 
 #include "check-acc-structure.h"
 #include "check-omp-structure.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/fold.h"
@@ -353,12 +354,6 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return true;
   }
 
-  bool Pre(const parser::OmpDirectiveSpecification &x) {
-    PushContext(x.source, x.DirId());
-    return true;
-  }
-  void Post(const parser::OmpDirectiveSpecification &) { PopContext(); }
-
   bool Pre(const parser::OmpMetadirectiveDirective &x) {
     PushContext(x.source, llvm::omp::Directive::OMPD_metadirective);
     return true;
@@ -372,6 +367,29 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     GetContext().withinConstruct = true;
   }
 
+  bool Pre(const parser::OpenMPStandaloneConstruct &x) {
+    common::visit(
+        [&](auto &&s) {
+          using TypeS = llvm::remove_cvref_t<decltype(s)>;
+          // These two cases are handled individually.
+          if constexpr ( //
+              !std::is_same_v<TypeS, parser::OpenMPSimpleStandaloneConstruct> &&
+              !std::is_same_v<TypeS, parser::OmpMetadirectiveDirective>) {
+            PushContext(x.source, s.v.DirId());
+          }
+        },
+        x.u);
+    return true;
+  }
+
+  void Post(const parser::OpenMPStandaloneConstruct &x) {
+    // These two cases are handled individually.
+    if (!std::holds_alternative<parser::OpenMPSimpleStandaloneConstruct>(x.u) &&
+        !std::holds_alternative<parser::OmpMetadirectiveDirective>(x.u)) {
+      PopContext();
+    }
+  }
+
   bool Pre(const parser::OpenMPSimpleStandaloneConstruct &);
   void Post(const parser::OpenMPSimpleStandaloneConstruct &) { PopContext(); }
 
@@ -2149,9 +2167,10 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPExecutableAllocate &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPAllocatorsConstruct &x) {
-  PushContext(x.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  PushContext(x.source, dirSpec.DirId());
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       ResolveOmpObjectList(std::get<parser::OmpObjectList>(allocClause->v.t),
@@ -2234,31 +2253,47 @@ void OmpAttributeVisitor::Post(const parser::OpenMPExecutableAllocate &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
-    if (const auto *alloc{
-            std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
-      CheckAllNamesInAllocateStmt(dir.source,
-          std::get<parser::OmpObjectList>(alloc->v.t),
-          std::get<parser::Statement<parser::AllocateStmt>>(x.t).statement);
-
-      auto &modifiers{OmpGetModifiers(alloc->v)};
-      bool hasAllocator{
-          OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(modifiers) ||
-          OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(modifiers)};
-
-      // TODO: As with allocate directive, exclude the case when a requires
-      //       directive with the dynamic_allocators clause is present in
-      //       the same compilation unit (OMP5.0 2.11.3).
-      if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
-          !hasAllocator) {
-        context_.Say(x.source,
-            "ALLOCATORS directives that appear in a TARGET region "
-            "must specify an allocator"_err_en_US);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const parser::AllocateStmt *allocate{[&]() {
+    if (action) {
+      if (auto *alloc{std::get_if<common::Indirection<parser::AllocateStmt>>(
+              &action.stmt->u)}) {
+        return &alloc->value();
+      }
+    }
+    return static_cast<const parser::AllocateStmt *>(nullptr);
+  }()};
+
+  if (allocate) {
+    for (const auto &clause : dirSpec.Clauses().v) {
+      if (auto *alloc{std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
+        CheckAllNamesInAllocateStmt(
+            x.source, std::get<parser::OmpObjectList>(alloc->v.t), *allocate);
+
+        using OmpAllocatorSimpleModifier = parser::OmpAllocatorSimpleModifier;
+        using OmpAllocatorComplexModifier = parser::OmpAllocatorComplexModifier;
+
+        auto &modifiers{OmpGetModifiers(alloc->v)};
+        bool hasAllocator{
+            OmpGetUniqueModifier<OmpAllocatorSimpleModifier>(modifiers) ||
+            OmpGetUniqueModifier<OmpAllocatorComplexModifier>(modifiers)};
+
+        // TODO: As with allocate directive, exclude the case when a requires
+        //       directive with the dynamic_allocators clause is present in
+        //       the same compilation unit (OMP5.0 2.11.3).
+        if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
+            !hasAllocator) {
+          context_.Say(x.source,
+              "ALLOCATORS directives that appear in a TARGET region "
+              "must specify an allocator"_err_en_US);
+        }
       }
     }
   }
+
   PopContext();
 }
 
diff --git a/flang/test/Parser/OpenMP/allocators-unparse.f90 b/flang/test/Parser/OpenMP/allocators-unparse.f90
index 5cd0230471fc4..70feb7a6b527e 100644
--- a/flang/test/Parser/OpenMP/allocators-unparse.f90
+++ b/flang/test/Parser/OpenMP/allocators-unparse.f90
@@ -15,48 +15,62 @@ subroutine allocate()
 
   !$omp allocators allocate(align(32): arr2)
     allocate(arr2(5, 3))
+  !$omp end allocators
 end subroutine allocate
 
 !CHECK: INTEGER, ALLOCATABLE :: arr1(:), arr2(:,:)
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(omp_default_mem_alloc: arr1)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(omp_default_mem_alloc: arr1)
 !CHECK-NEXT: ALLOCATE(arr1(5))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) ALL&
-!CHECK-NEXT:!$OMP&OCATE(omp_default_mem_alloc: arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) A&
+!CHECK-NEXT:!$OMP&LLOCATE(omp_default_mem_alloc: arr2)
 !CHECK-NEXT: ALLOCATE(arr1(10), arr2(3,2))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALIGN(32): arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALIGN(32): arr2)
 !CHECK-NEXT: ALLOCATE(arr2(5,3))
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Thu Jul 10 10:26:06 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 10:26:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <686ff7ae.050a0220.28dd60.7ccf@mx.google.com>


kparzysz wrote:

Stack:
- https://github.com/llvm/llvm-project/pull/148005 (this PR)
- https://github.com/llvm/llvm-project/pull/148008

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Thu Jul 10 10:50:09 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Thu, 10 Jul 2025 10:50:09 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Force lowering to Complex for
	AMDGPU (PR #144927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144927@github.com>
Message-ID: <686ffd51.050a0220.87b31.a9a9@mx.google.com>


https://github.com/mjklemm approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/144927

From flang-commits at lists.llvm.org  Thu Jul 10 10:51:30 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Thu, 10 Jul 2025 10:51:30 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP] Add TargetAMDGPU support for
 Complex argument and return types (PR #144924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144924@github.com>
Message-ID: <686ffda2.050a0220.1248d3.afa3@mx.google.com>


https://github.com/mjklemm approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/144924

From flang-commits at lists.llvm.org  Thu Jul 10 10:59:22 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 10:59:22 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][acc] Create UseDeviceOp for both
 results of hlfir.declare (PR #148017)
Message-ID: <llvm.org/llvm/llvm-project/pull/148017@github.com>


https://github.com/nvptm created https://github.com/llvm/llvm-project/pull/148017

A sample such as 
```
program test
  integer :: N = 100
  real*8 :: b(-1:N)
  !$acc data copy(b)
  !$acc host_data use_device(b)
  call vadd(b)
  !$acc end host_data
  !$acc end data
end

```
is lowered to
```
    %13:2 = hlfir.declare %11(%12) {uniq_name = "_QFEb"} : (!fir.ref<!fir.array<?xf64>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>)
    %14 = acc.copyin var(%13#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
    acc.data dataOperands(%14 : !fir.box<!fir.array<?xf64>>) {
      %15 = acc.use_device var(%13#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
      acc.host_data dataOperands(%15 : !fir.box<!fir.array<?xf64>>) {
        fir.call @_QPvadd(%13#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>) -> ()
        acc.terminator
      }
      acc.terminator
    }
    acc.copyout accVar(%14 : !fir.box<!fir.array<?xf64>>) to var(%13#0 : !fir.box<!fir.array<?xf64>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
```
Note that while the use_device clause is applied to %13#0, the argument passed to vadd is %13#1. To avoid problems later in lowering, this change additionally applies the use_device clause to %13#1, so that the resulting MLIR is
```
   %13:2 = hlfir.declare %11(%12) {uniq_name = "_QFEb"} : (!fir.ref<!fir.array<?xf64>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>)
    %14 = acc.copyin var(%13#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
    acc.data dataOperands(%14 : !fir.box<!fir.array<?xf64>>) {
      %15 = acc.use_device var(%13#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
      %16 = acc.use_device varPtr(%13#1 : !fir.ref<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>> {name = "b"}
      acc.host_data dataOperands(%15, %16 : !fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>) {
        fir.call @_QPvadd(%13#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>) -> ()
        acc.terminator
      }
      acc.terminator
    }
    acc.copyout accVar(%14 : !fir.box<!fir.array<?xf64>>) to var(%13#0 : !fir.box<!fir.array<?xf64>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
  
```

>From 0fb4bc6169d76081b641c78db1c032ab876a6905 Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Wed, 9 Jul 2025 23:13:49 -0700
Subject: [PATCH 1/4] use_device for all other results of hlfir.declare

---
 flang/lib/Lower/OpenACC.cpp | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 42842bcb41a74..4f637b88fd269 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -724,7 +724,7 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
             /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
             /*genDefaultBounds=*/generateDefaultBounds,
             /*strideIncludeLowerExtent=*/strideIncludeLowerExtent);
-    LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
+    LLVM_DEBUG(llvm::dbgs() << __func__ << "Here \n"; info.dump(llvm::dbgs()));
 
     // If the input value is optional and is not a descriptor, we use the
     // rawInput directly.
@@ -738,6 +738,21 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
         implicit, dataClause, baseAddr.getType(), async, asyncDeviceTypes,
         asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
     dataOperands.push_back(op.getAccVar());
+    // If the input value has a descriptor, we need to create a use device op
+    // for the descriptor as well as the base address.
+    if constexpr (std::is_same_v<Op, mlir::acc::UseDeviceOp>) {
+      LLVM_DEBUG(llvm::dbgs() << __func__ << " found usedeviceop \n"; info.dump(llvm::dbgs()));
+      LLVM_DEBUG(llvm::dbgs() << __func__ << " had previously created and added usedeviceop \n"; op.dump());
+      if (mlir::isa<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
+        Op op = createDataEntryOp<Op>(
+            builder, operandLocation, baseAddr.getDefiningOp()->getResult(1), asFortran, bounds, structured,
+            implicit, dataClause, baseAddr.getDefiningOp()->getResult(1).getType(), async, asyncDeviceTypes,
+            asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
+        LLVM_DEBUG(llvm::dbgs() << __func__ << " created usedeviceop \n"; op.dump());
+        dataOperands.push_back(op.getAccVar());
+        LLVM_DEBUG(llvm::dbgs() << __func__ << "added usedeviceop on descriptor\n"; info.dump(llvm::dbgs()));
+      }
+    }
   }
 }
 

>From be91a9ab05a43dff381347f0e6f49dbbe019669f Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Thu, 10 Jul 2025 09:19:58 -0700
Subject: [PATCH 2/4] update tests

---
 .../acc-host-data-unwrap-defaultbounds.f90    | 14 +++++++------
 flang/test/Lower/OpenACC/acc-host-data.f90    | 21 +++++++++++--------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
index 164eb32a8f684..2de7cc5761a2b 100644
--- a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
@@ -15,15 +15,17 @@ subroutine acc_host_data()
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+ ! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>{{.*}}) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
@@ -33,14 +35,14 @@ subroutine acc_host_data()
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.false.)
     a = 1.0
diff --git a/flang/test/Lower/OpenACC/acc-host-data.f90 b/flang/test/Lower/OpenACC/acc-host-data.f90
index 871eabd256ca6..4d09b25b983b9 100644
--- a/flang/test/Lower/OpenACC/acc-host-data.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data.f90
@@ -14,34 +14,37 @@ subroutine acc_host_data()
   !$acc host_data use_device(a)
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 ! CHECK: } attributes {ifPresent}
 
-  !$acc host_data use_device(a) if_present if_present
+  !$acc host_data use_device(a) if_present 
   !$acc end host_data
-! CHECK: acc.host_data dataOperands(%{{.*}} : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: acc.host_data dataOperands(%{{.*}}{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}}) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA0]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.false.)
     a = 1.0

>From ec7ad5b11cc46322c6188bafb23f89cb3767ad70 Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Thu, 10 Jul 2025 09:58:36 -0700
Subject: [PATCH 3/4] Remove debug messages. Format file.

---
 flang/lib/Lower/OpenACC.cpp | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 4f637b88fd269..d43345021063b 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -738,19 +738,17 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
         implicit, dataClause, baseAddr.getType(), async, asyncDeviceTypes,
         asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
     dataOperands.push_back(op.getAccVar());
-    // If the input value has a descriptor, we need to create a use device op
-    // for the descriptor as well as the base address.
+    // For UseDeviceOp, if operand is one of a pair resulting from a
+    // declare operation, create a UseDeviceOp for the other operand as well.
     if constexpr (std::is_same_v<Op, mlir::acc::UseDeviceOp>) {
-      LLVM_DEBUG(llvm::dbgs() << __func__ << " found usedeviceop \n"; info.dump(llvm::dbgs()));
-      LLVM_DEBUG(llvm::dbgs() << __func__ << " had previously created and added usedeviceop \n"; op.dump());
       if (mlir::isa<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
         Op op = createDataEntryOp<Op>(
-            builder, operandLocation, baseAddr.getDefiningOp()->getResult(1), asFortran, bounds, structured,
-            implicit, dataClause, baseAddr.getDefiningOp()->getResult(1).getType(), async, asyncDeviceTypes,
-            asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
-        LLVM_DEBUG(llvm::dbgs() << __func__ << " created usedeviceop \n"; op.dump());
+            builder, operandLocation, baseAddr.getDefiningOp()->getResult(1),
+            asFortran, bounds, structured, implicit, dataClause,
+            baseAddr.getDefiningOp()->getResult(1).getType(), async,
+            asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true,
+            info.isPresent);
         dataOperands.push_back(op.getAccVar());
-        LLVM_DEBUG(llvm::dbgs() << __func__ << "added usedeviceop on descriptor\n"; info.dump(llvm::dbgs()));
       }
     }
   }

>From c385240e757bc2e6c127304a36bae1f1e9d02d60 Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Thu, 10 Jul 2025 09:59:57 -0700
Subject: [PATCH 4/4] Remove debug messages.

---
 flang/lib/Lower/OpenACC.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index d43345021063b..23481d0ef7935 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -724,7 +724,7 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
             /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
             /*genDefaultBounds=*/generateDefaultBounds,
             /*strideIncludeLowerExtent=*/strideIncludeLowerExtent);
-    LLVM_DEBUG(llvm::dbgs() << __func__ << "Here \n"; info.dump(llvm::dbgs()));
+    LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
 
     // If the input value is optional and is not a descriptor, we use the
     // rawInput directly.


From flang-commits at lists.llvm.org  Thu Jul 10 10:59:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 10:59:57 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][acc] Create UseDeviceOp for both
 results of hlfir.declare (PR #148017)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148017@github.com>
Message-ID: <686fff9d.170a0220.176c76.a969@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-openacc

Author: None (nvptm)

<details>
<summary>Changes</summary>

A sample such as 
```
program test
  integer :: N = 100
  real*8 :: b(-1:N)
  !$acc data copy(b)
  !$acc host_data use_device(b)
  call vadd(b)
  !$acc end host_data
  !$acc end data
end

```
is lowered to
```
    %13:2 = hlfir.declare %11(%12) {uniq_name = "_QFEb"} : (!fir.ref&lt;!fir.array&lt;?xf64&gt;&gt;, !fir.shapeshift&lt;1&gt;) -&gt; (!fir.box&lt;!fir.array&lt;?xf64&gt;&gt;, !fir.ref&lt;!fir.array&lt;?xf64&gt;&gt;)
    %14 = acc.copyin var(%13#<!-- -->0 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) -&gt; !fir.box&lt;!fir.array&lt;?xf64&gt;&gt; {dataClause = #acc&lt;data_clause acc_copy&gt;, name = "b"}
    acc.data dataOperands(%14 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) {
      %15 = acc.use_device var(%13#<!-- -->0 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) -&gt; !fir.box&lt;!fir.array&lt;?xf64&gt;&gt; {name = "b"}
      acc.host_data dataOperands(%15 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) {
        fir.call @<!-- -->_QPvadd(%13#<!-- -->1) fastmath&lt;contract&gt; : (!fir.ref&lt;!fir.array&lt;?xf64&gt;&gt;) -&gt; ()
        acc.terminator
      }
      acc.terminator
    }
    acc.copyout accVar(%14 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) to var(%13#<!-- -->0 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) {dataClause = #acc&lt;data_clause acc_copy&gt;, name = "b"}
```
Note that while the use_device clause is applied to %13#<!-- -->0, the argument passed to vadd is %13#<!-- -->1. To avoid problems later in lowering, this change additionally applies the use_device clause to %13#<!-- -->1, so that the resulting MLIR is
```
   %13:2 = hlfir.declare %11(%12) {uniq_name = "_QFEb"} : (!fir.ref&lt;!fir.array&lt;?xf64&gt;&gt;, !fir.shapeshift&lt;1&gt;) -&gt; (!fir.box&lt;!fir.array&lt;?xf64&gt;&gt;, !fir.ref&lt;!fir.array&lt;?xf64&gt;&gt;)
    %14 = acc.copyin var(%13#<!-- -->0 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) -&gt; !fir.box&lt;!fir.array&lt;?xf64&gt;&gt; {dataClause = #acc&lt;data_clause acc_copy&gt;, name = "b"}
    acc.data dataOperands(%14 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) {
      %15 = acc.use_device var(%13#<!-- -->0 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) -&gt; !fir.box&lt;!fir.array&lt;?xf64&gt;&gt; {name = "b"}
      %16 = acc.use_device varPtr(%13#<!-- -->1 : !fir.ref&lt;!fir.array&lt;?xf64&gt;&gt;) -&gt; !fir.ref&lt;!fir.array&lt;?xf64&gt;&gt; {name = "b"}
      acc.host_data dataOperands(%15, %16 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;, !fir.ref&lt;!fir.array&lt;?xf64&gt;&gt;) {
        fir.call @<!-- -->_QPvadd(%13#<!-- -->1) fastmath&lt;contract&gt; : (!fir.ref&lt;!fir.array&lt;?xf64&gt;&gt;) -&gt; ()
        acc.terminator
      }
      acc.terminator
    }
    acc.copyout accVar(%14 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) to var(%13#<!-- -->0 : !fir.box&lt;!fir.array&lt;?xf64&gt;&gt;) {dataClause = #acc&lt;data_clause acc_copy&gt;, name = "b"}
  
```

---
Full diff: https://github.com/llvm/llvm-project/pull/148017.diff


3 Files Affected:

- (modified) flang/lib/Lower/OpenACC.cpp (+13) 
- (modified) flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90 (+8-6) 
- (modified) flang/test/Lower/OpenACC/acc-host-data.f90 (+12-9) 


``````````diff
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 42842bcb41a74..23481d0ef7935 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -738,6 +738,19 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
         implicit, dataClause, baseAddr.getType(), async, asyncDeviceTypes,
         asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
     dataOperands.push_back(op.getAccVar());
+    // For UseDeviceOp, if operand is one of a pair resulting from a
+    // declare operation, create a UseDeviceOp for the other operand as well.
+    if constexpr (std::is_same_v<Op, mlir::acc::UseDeviceOp>) {
+      if (mlir::isa<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
+        Op op = createDataEntryOp<Op>(
+            builder, operandLocation, baseAddr.getDefiningOp()->getResult(1),
+            asFortran, bounds, structured, implicit, dataClause,
+            baseAddr.getDefiningOp()->getResult(1).getType(), async,
+            asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true,
+            info.isPresent);
+        dataOperands.push_back(op.getAccVar());
+      }
+    }
   }
 }
 
diff --git a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
index 164eb32a8f684..2de7cc5761a2b 100644
--- a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
@@ -15,15 +15,17 @@ subroutine acc_host_data()
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+ ! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>{{.*}}) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
@@ -33,14 +35,14 @@ subroutine acc_host_data()
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.false.)
     a = 1.0
diff --git a/flang/test/Lower/OpenACC/acc-host-data.f90 b/flang/test/Lower/OpenACC/acc-host-data.f90
index 871eabd256ca6..4d09b25b983b9 100644
--- a/flang/test/Lower/OpenACC/acc-host-data.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data.f90
@@ -14,34 +14,37 @@ subroutine acc_host_data()
   !$acc host_data use_device(a)
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 ! CHECK: } attributes {ifPresent}
 
-  !$acc host_data use_device(a) if_present if_present
+  !$acc host_data use_device(a) if_present 
   !$acc end host_data
-! CHECK: acc.host_data dataOperands(%{{.*}} : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: acc.host_data dataOperands(%{{.*}}{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}}) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA0]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.false.)
     a = 1.0

``````````

</details>


https://github.com/llvm/llvm-project/pull/148017

From flang-commits at lists.llvm.org  Thu Jul 10 11:06:13 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 10 Jul 2025 11:06:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][docs] Update release notes for
	OpenMP (PR #147988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147988@github.com>
Message-ID: <68700115.170a0220.38fe26.a0cf@mx.google.com>


================
@@ -25,6 +25,8 @@ page](https://llvm.org/releases/).
 ## Major New Features
 
 * Initial support for VOLATILE variables and procedure interface arguments has been added.
+* OpenMP support is no longer considered experimental. All of OpenMP 3 is
----------------
kparzysz wrote:

That's 3.1. We no longer recognize 3.0 as a supported version.

https://github.com/llvm/llvm-project/pull/147988

From flang-commits at lists.llvm.org  Thu Jul 10 13:55:05 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 10 Jul 2025 13:55:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda][NFC] Remove TODO implemented
	in semantic (PR #148058)
Message-ID: <llvm.org/llvm/llvm-project/pull/148058@github.com>


https://github.com/clementval created https://github.com/llvm/llvm-project/pull/148058

This is implemented here: https://github.com/llvm/llvm-project/blob/838701a5403efbaf6e25254377a6f033acee6681/flang/lib/Semantics/check-cuda.cpp#L764

Keep the assert in place as security, 

>From 03426b47510586a5e6da08e2adbb59d6813d7701 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Thu, 10 Jul 2025 13:53:42 -0700
Subject: [PATCH] [flang][cuda][NFC] Remove TODO implemented in semantic

---
 flang/lib/Lower/Bridge.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 987fd3095fdf6..cd55d10314740 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          // TODO: This should probably being checked in semantic and give a
-          // proper error.
           assert(
               nbDeviceResidentObject <= 1 &&
               "Only one reference to the device resident object is supported");


From flang-commits at lists.llvm.org  Thu Jul 10 13:55:38 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 13:55:38 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda][NFC] Remove TODO implemented
	in semantic (PR #148058)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148058@github.com>
Message-ID: <687028ca.630a0220.27f489.c6d2@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Valentin Clement (バレンタイン クレメン) (clementval)

<details>
<summary>Changes</summary>

This is implemented here: https://github.com/llvm/llvm-project/blob/838701a5403efbaf6e25254377a6f033acee6681/flang/lib/Semantics/check-cuda.cpp#L764

Keep the assert in place as security, 

---
Full diff: https://github.com/llvm/llvm-project/pull/148058.diff


1 Files Affected:

- (modified) flang/lib/Lower/Bridge.cpp (-2) 


``````````diff
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 987fd3095fdf6..cd55d10314740 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          // TODO: This should probably being checked in semantic and give a
-          // proper error.
           assert(
               nbDeviceResidentObject <= 1 &&
               "Only one reference to the device resident object is supported");

``````````

</details>


https://github.com/llvm/llvm-project/pull/148058

From flang-commits at lists.llvm.org  Thu Jul 10 20:48:54 2025
From: flang-commits at lists.llvm.org (Zhen Wang via flang-commits)
Date: Thu, 10 Jul 2025 20:48:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda][NFC] Remove TODO implemented
	in semantic (PR #148058)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148058@github.com>
Message-ID: <687089a6.050a0220.7abbb.cd9f@mx.google.com>


================
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          // TODO: This should probably being checked in semantic and give a
----------------
wangzpgi wrote:

This change is to remove this comment? Just to make sure there are not missing files attached :)

https://github.com/llvm/llvm-project/pull/148058

From flang-commits at lists.llvm.org  Thu Jul 10 20:50:22 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 10 Jul 2025 20:50:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda][NFC] Remove TODO implemented
	in semantic (PR #148058)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148058@github.com>
Message-ID: <687089fe.170a0220.306d8e.bee8@mx.google.com>


================
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          // TODO: This should probably being checked in semantic and give a
----------------
clementval wrote:

Yeah the comment is obsolete from some time ago.

https://github.com/llvm/llvm-project/pull/148058

From flang-commits at lists.llvm.org  Thu Jul 10 20:51:38 2025
From: flang-commits at lists.llvm.org (Zhen Wang via flang-commits)
Date: Thu, 10 Jul 2025 20:51:38 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda][NFC] Remove TODO implemented
	in semantic (PR #148058)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148058@github.com>
Message-ID: <68708a4a.630a0220.129ca1.c730@mx.google.com>


================
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          // TODO: This should probably being checked in semantic and give a
----------------
wangzpgi wrote:

Got it. Thanks for the clarification.

https://github.com/llvm/llvm-project/pull/148058

From flang-commits at lists.llvm.org  Thu Jul 10 20:51:51 2025
From: flang-commits at lists.llvm.org (Zhen Wang via flang-commits)
Date: Thu, 10 Jul 2025 20:51:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda][NFC] Remove TODO implemented
	in semantic (PR #148058)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148058@github.com>
Message-ID: <68708a57.170a0220.1dcc01.c173@mx.google.com>


https://github.com/wangzpgi approved this pull request.


https://github.com/llvm/llvm-project/pull/148058

From flang-commits at lists.llvm.org  Thu Jul 10 20:52:59 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 20:52:59 -0700 (PDT)
Subject: [flang-commits] [flang] c919221 - [flang][cuda][NFC] Remove TODO
 implemented in semantic (#148058)
Message-ID: <68708a9b.630a0220.2a28de.c2f0@mx.google.com>


Author: Valentin Clement (バレンタイン クレメン)
Date: 2025-07-10T20:52:55-07:00
New Revision: c919221bbe56fae15b509fcc84c25b0c041eb6b5

URL: https://github.com/llvm/llvm-project/commit/c919221bbe56fae15b509fcc84c25b0c041eb6b5
DIFF: https://github.com/llvm/llvm-project/commit/c919221bbe56fae15b509fcc84c25b0c041eb6b5.diff

LOG: [flang][cuda][NFC] Remove TODO implemented in semantic (#148058)

Added: 
    

Modified: 
    flang/lib/Lower/Bridge.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 987fd3095fdf6..cd55d10314740 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4842,8 +4842,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   .detailsIf<Fortran::semantics::ObjectEntityDetails>()) {
         if (details->cudaDataAttr() &&
             *details->cudaDataAttr() != Fortran::common::CUDADataAttr::Pinned) {
-          // TODO: This should probably being checked in semantic and give a
-          // proper error.
           assert(
               nbDeviceResidentObject <= 1 &&
               "Only one reference to the device resident object is supported");


        

From flang-commits at lists.llvm.org  Thu Jul 10 20:53:01 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 10 Jul 2025 20:53:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda][NFC] Remove TODO implemented
	in semantic (PR #148058)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148058@github.com>
Message-ID: <68708a9d.170a0220.1772cd.bfea@mx.google.com>


https://github.com/clementval closed https://github.com/llvm/llvm-project/pull/148058

From flang-commits at lists.llvm.org  Thu Jul 10 21:39:39 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 10 Jul 2025 21:39:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][do concurrent] Re-model `reduce` to
 match reductions are modelled in OpenMP and OpenACC (PR #145837)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145837@github.com>
Message-ID: <6870958b.170a0220.2cb820.d1ce@mx.google.com>


https://github.com/ergawy closed https://github.com/llvm/llvm-project/pull/145837

From flang-commits at lists.llvm.org  Thu Jul 10 21:39:42 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 10 Jul 2025 21:39:42 -0700 (PDT)
Subject: [flang-commits] [flang] [NFC][flang] Move `ReductionProcessor` to
 `Lower/Support`. (PR #146025)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146025@github.com>
Message-ID: <6870958e.170a0220.20f3b5.c5fb@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/146025

From flang-commits at lists.llvm.org  Thu Jul 10 21:41:03 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 10 Jul 2025 21:41:03 -0700 (PDT)
Subject: [flang-commits] [flang] [NFC][flang] Move `ReductionProcessor` to
 `Lower/Support`. (PR #146025)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146025@github.com>
Message-ID: <687095df.050a0220.2b7974.d16a@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/146025

>From ab6353a0c9e2a874dc159e21f3ed5b76a48bf656 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Thu, 26 Jun 2025 23:30:04 -0500
Subject: [PATCH 1/2] [NFC][flang] Move `ReductionProcessor` to
 `Lower/Support`.

With #145837, the `ReductionProcessor` component is now used by both
OpenMP and `do concurrent`. Therefore, this PR moves it to a shared
location: `flang/Lower/Support`.
---
 flang/{lib => include/flang}/Lower/OpenMP/Clauses.h           | 0
 .../flang/Lower/Support}/ReductionProcessor.h                 | 2 +-
 flang/lib/Lower/Bridge.cpp                                    | 2 +-
 flang/lib/Lower/CMakeLists.txt                                | 2 +-
 flang/lib/Lower/OpenMP/ClauseFinder.h                         | 2 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp                    | 4 ++--
 flang/lib/Lower/OpenMP/ClauseProcessor.h                      | 2 +-
 flang/lib/Lower/OpenMP/Clauses.cpp                            | 2 +-
 flang/lib/Lower/OpenMP/DataSharingProcessor.h                 | 2 +-
 flang/lib/Lower/OpenMP/Decomposer.cpp                         | 2 +-
 flang/lib/Lower/OpenMP/Decomposer.h                           | 2 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp                             | 2 +-
 flang/lib/Lower/OpenMP/Utils.cpp                              | 3 +--
 flang/lib/Lower/OpenMP/Utils.h                                | 2 +-
 flang/lib/Lower/{OpenMP => Support}/ReductionProcessor.cpp    | 4 ++--
 15 files changed, 16 insertions(+), 17 deletions(-)
 rename flang/{lib => include/flang}/Lower/OpenMP/Clauses.h (100%)
 rename flang/{lib/Lower/OpenMP => include/flang/Lower/Support}/ReductionProcessor.h (99%)
 rename flang/lib/Lower/{OpenMP => Support}/ReductionProcessor.cpp (99%)

diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/include/flang/Lower/OpenMP/Clauses.h
similarity index 100%
rename from flang/lib/Lower/OpenMP/Clauses.h
rename to flang/include/flang/Lower/OpenMP/Clauses.h
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/include/flang/Lower/Support/ReductionProcessor.h
similarity index 99%
rename from flang/lib/Lower/OpenMP/ReductionProcessor.h
rename to flang/include/flang/Lower/Support/ReductionProcessor.h
index 95b4b077bdc46..72d8a0096f511 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/include/flang/Lower/Support/ReductionProcessor.h
@@ -13,7 +13,7 @@
 #ifndef FORTRAN_LOWER_REDUCTIONPROCESSOR_H
 #define FORTRAN_LOWER_REDUCTIONPROCESSOR_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Parser/parse-tree.h"
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index e2d3fe964d49b..33c1f1e7a3c3a 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -12,7 +12,6 @@
 
 #include "flang/Lower/Bridge.h"
 
-#include "OpenMP/ReductionProcessor.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -32,6 +31,7 @@
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/Runtime.h"
 #include "flang/Lower/StatementContext.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 #include "flang/Lower/Support/Utils.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/CUFCommon.h"
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 8049cdf333173..cd80aaf553869 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -29,11 +29,11 @@ add_flang_library(FortranLower
   OpenMP/DataSharingProcessor.cpp
   OpenMP/Decomposer.cpp
   OpenMP/OpenMP.cpp
-  OpenMP/ReductionProcessor.cpp
   OpenMP/Utils.cpp
   PFTBuilder.cpp
   Runtime.cpp
   Support/PrivateReductionUtils.cpp
+  Support/ReductionProcessor.cpp
   Support/Utils.cpp
   SymbolMap.cpp
   VectorSubscripts.cpp
diff --git a/flang/lib/Lower/OpenMP/ClauseFinder.h b/flang/lib/Lower/OpenMP/ClauseFinder.h
index 3b77f2ca1d4cb..af52585452833 100644
--- a/flang/lib/Lower/OpenMP/ClauseFinder.h
+++ b/flang/lib/Lower/OpenMP/ClauseFinder.h
@@ -12,7 +12,7 @@
 #ifndef FORTRAN_LOWER_CLAUSEFINDER_H
 #define FORTRAN_LOWER_CLAUSEFINDER_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 
 namespace Fortran {
 namespace lower {
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 5aebfc901e8ac..74087d42a8e6e 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -11,12 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClauseProcessor.h"
-#include "Clauses.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 
 #include "flang/Lower/ConvertExprToHLFIR.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/Frontend/OpenMP/OMP.h.inc"
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 46b749fb66c86..f8a1f7983b79b 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -13,11 +13,11 @@
 #define FORTRAN_LOWER_CLAUSEPROCESSOR_H
 
 #include "ClauseFinder.h"
-#include "Clauses.h"
 #include "Utils.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/Bridge.h"
 #include "flang/Lower/DirectivesCommon.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Parser/dump-parse-tree.h"
 #include "flang/Parser/parse-tree.h"
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index b599d69a36272..22a07219d3a50 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/expression.h"
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index fded04c839fb4..ee2fc70d2e673 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -12,9 +12,9 @@
 #ifndef FORTRAN_LOWER_DATASHARINGPROCESSOR_H
 #define FORTRAN_LOWER_DATASHARINGPROCESSOR_H
 
-#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/OpenMP.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"
diff --git a/flang/lib/Lower/OpenMP/Decomposer.cpp b/flang/lib/Lower/OpenMP/Decomposer.cpp
index 251cba9204adc..9bfbf67bec88c 100644
--- a/flang/lib/Lower/OpenMP/Decomposer.cpp
+++ b/flang/lib/Lower/OpenMP/Decomposer.cpp
@@ -12,8 +12,8 @@
 
 #include "Decomposer.h"
 
-#include "Clauses.h"
 #include "Utils.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Tools/CrossToolHelpers.h"
diff --git a/flang/lib/Lower/OpenMP/Decomposer.h b/flang/lib/Lower/OpenMP/Decomposer.h
index e3291b7c59e21..65492bd76280d 100644
--- a/flang/lib/Lower/OpenMP/Decomposer.h
+++ b/flang/lib/Lower/OpenMP/Decomposer.h
@@ -8,7 +8,7 @@
 #ifndef FORTRAN_LOWER_OPENMP_DECOMPOSER_H
 #define FORTRAN_LOWER_OPENMP_DECOMPOSER_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 65e852cbcc911..4458f62eea95a 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -14,7 +14,6 @@
 
 #include "Atomic.h"
 #include "ClauseProcessor.h"
-#include "Clauses.h"
 #include "DataSharingProcessor.h"
 #include "Decomposer.h"
 #include "Utils.h"
@@ -23,6 +22,7 @@
 #include "flang/Lower/ConvertExpr.h"
 #include "flang/Lower/ConvertVariable.h"
 #include "flang/Lower/DirectivesCommon.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index c226c2558e7aa..2e53f01f1da6a 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -12,9 +12,8 @@
 
 #include "Utils.h"
 
-#include "Clauses.h"
-
 #include "ClauseFinder.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include <flang/Lower/AbstractConverter.h>
 #include <flang/Lower/ConvertType.h>
 #include <flang/Lower/DirectivesCommon.h>
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index a7eb2dc5ee664..1526bd4e90233 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_LOWER_OPENMPUTILS_H
 #define FORTRAN_LOWER_OPENMPUTILS_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Value.h"
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
similarity index 99%
rename from flang/lib/Lower/OpenMP/ReductionProcessor.cpp
rename to flang/lib/Lower/Support/ReductionProcessor.cpp
index d14fc1f7a52da..539d5cd37c2ea 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -10,11 +10,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ReductionProcessor.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 
-#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/ConvertType.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/Support/PrivateReductionUtils.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/Complex.h"

>From e9eb77f38bc2f4fa58d9b138fe66258c795cb69b Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Sun, 29 Jun 2025 23:35:38 -0500
Subject: [PATCH 2/2] fix include after rebase

---
 flang/lib/Lower/OpenMP/Atomic.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 2ab91b239a3cc..6ea331c370640 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -7,13 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "Atomic.h"
-#include "Clauses.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/traverse.h"
 #include "flang/Evaluate/type.h"
 #include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"


From flang-commits at lists.llvm.org  Thu Jul 10 21:39:36 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 21:39:36 -0700 (PDT)
Subject: [flang-commits] [flang] eba35cc - [flang][do concurrent] Re-model
 `reduce` to match reductions are modelled in OpenMP and OpenACC (#145837)
Message-ID: <68709588.170a0220.27cfe6.cb7b@mx.google.com>


Author: Kareem Ergawy
Date: 2025-07-11T06:39:30+02:00
New Revision: eba35cc1c0e4e2c59f9fd1f7a6f3b17cb4d8c765

URL: https://github.com/llvm/llvm-project/commit/eba35cc1c0e4e2c59f9fd1f7a6f3b17cb4d8c765
DIFF: https://github.com/llvm/llvm-project/commit/eba35cc1c0e4e2c59f9fd1f7a6f3b17cb4d8c765.diff

LOG: [flang][do concurrent] Re-model `reduce` to match reductions are modelled in OpenMP and OpenACC (#145837)

This PR proposes re-modelling `reduce` specifiers to match OpenMP and
OpenACC. In particular, this PR includes the following:

* A new `fir` op: `fir.delcare_reduction` which is identical to OpenMP's
`omp.declare_reduction` op.
* Updating the `reduce` clause on `fir.do_concurrent.loop` to use the
new op.
* Re-uses the `ReductionProcessor` component to emit reductions for `do
concurrent` just like we do for OpenMP. To do this, the
`ReductionProcessor` had to be refactored to be more generalized.
* Upates mapping `do concurrent` to `fir.loop ... unordered` nests using
the new reduction model.

Unfortunately, this is a big PR that would be difficult to divide up in
smaller parts because the bottom of the changes are the `fir` table-gen
changes to `do concurrent`. However, doing these MLIR changes cascades
to the other parts that have to be modified to not break things.

This PR goes in the same direction we went for `private/local`
speicifiers. Now the `do concurrent` and OpenMP (and OpenACC) dialects
are modelled in essentially the same way which makes mapping between
them more trivial, hopefully.

PR stack:
- https://github.com/llvm/llvm-project/pull/145837 (this one)
- https://github.com/llvm/llvm-project/pull/146025
- https://github.com/llvm/llvm-project/pull/146028
- https://github.com/llvm/llvm-project/pull/146033

Added: 
    flang/test/Lower/do_concurrent_reduce.f90

Modified: 
    flang/include/flang/Optimizer/Dialect/FIRAttr.td
    flang/include/flang/Optimizer/Dialect/FIROps.td
    flang/lib/Lower/Bridge.cpp
    flang/lib/Lower/OpenMP/ClauseProcessor.cpp
    flang/lib/Lower/OpenMP/ClauseProcessor.h
    flang/lib/Lower/OpenMP/Clauses.h
    flang/lib/Lower/OpenMP/OpenMP.cpp
    flang/lib/Lower/OpenMP/ReductionProcessor.cpp
    flang/lib/Lower/OpenMP/ReductionProcessor.h
    flang/lib/Lower/Support/Utils.cpp
    flang/lib/Optimizer/CodeGen/CodeGen.cpp
    flang/lib/Optimizer/Dialect/FIROps.cpp
    flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
    flang/test/Fir/do_concurrent.fir
    flang/test/Fir/invalid.fir
    flang/test/Lower/loops.f90
    flang/test/Lower/loops3.f90
    flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.td b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
index 2845080030b92..7bd96ac3ea631 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRAttr.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
@@ -112,7 +112,7 @@ def fir_ReduceOperationEnum : I32BitEnumAttr<"ReduceOperationEnum",
       I32BitEnumAttrCaseBit<"MIN", 7, "min">,
       I32BitEnumAttrCaseBit<"IAND", 8, "iand">,
       I32BitEnumAttrCaseBit<"IOR", 9, "ior">,
-      I32BitEnumAttrCaseBit<"EIOR", 10, "eior">
+      I32BitEnumAttrCaseBit<"IEOR", 10, "ieor">
     ]> {
   let separator = ", ";
   let cppNamespace = "::fir";

diff  --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index f440580f0878a..c3d3582a50e7f 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3518,7 +3518,7 @@ def fir_BoxTotalElementsOp
 
 def YieldOp : fir_Op<"yield",
     [Pure, ReturnLike, Terminator,
-     ParentOneOf<["LocalitySpecifierOp"]>]> {
+     ParentOneOf<["LocalitySpecifierOp", "DeclareReductionOp"]>]> {
   let summary = "loop yield and termination operation";
   let description = [{
     "fir.yield" yields SSA values from a fir dialect op region and
@@ -3656,6 +3656,103 @@ def fir_LocalitySpecifierOp : fir_Op<"local", [IsolatedFromAbove]> {
   let hasRegionVerifier = 1;
 }
 
+def fir_DeclareReductionOp : fir_Op<"declare_reduction", [IsolatedFromAbove,
+                                                         Symbol]> {
+  let summary = "declares a reduction kind";
+  let description = [{
+    Note: this operation is adapted from omp::DeclareReductionOp. There is a lot
+    duplication at the moment. TODO Combine both ops into one. See:
+    https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
+
+    Declares a `do concurrent` reduction. This requires two mandatory and three
+    optional regions.
+
+      1. The optional alloc region specifies how to allocate the thread-local
+         reduction value. This region should not contain control flow and all
+         IR should be suitable for inlining straight into an entry block. In
+         the common case this is expected to contain only allocas. It is
+         expected to `fir.yield` the allocated value on all control paths.
+         If allocation is conditional (e.g. only allocate if the mold is
+         allocated), this should be done in the initilizer region and this
+         region not included. The alloc region is not used for by-value
+         reductions (where allocation is implicit).
+      2. The initializer region specifies how to initialize the thread-local
+         reduction value. This is usually the neutral element of the reduction.
+         For convenience, the region has an argument that contains the value
+         of the reduction accumulator at the start of the reduction. If an alloc
+         region is specified, there is a second block argument containing the
+         address of the allocated memory. The initializer region is expected to
+         `fir.yield` the new value on all control flow paths.
+      3. The reduction region specifies how to combine two values into one, i.e.
+         the reduction operator. It accepts the two values as arguments and is
+         expected to `fir.yield` the combined value on all control flow paths.
+      4. The atomic reduction region is optional and specifies how two values
+         can be combined atomically given local accumulator variables. It is
+         expected to store the combined value in the first accumulator variable.
+      5. The cleanup region is optional and specifies how to clean up any memory
+         allocated by the initializer region. The region has an argument that
+         contains the value of the thread-local reduction accumulator. This will
+         be executed after the reduction has completed.
+
+    Note that the MLIR type system does not allow for type-polymorphic
+    reductions. Separate reduction declarations should be created for 
diff erent
+    element and accumulator types.
+
+    For initializer and reduction regions, the operand to `fir.yield` must
+    match the parent operation's results.
+  }];
+
+  let arguments = (ins SymbolNameAttr:$sym_name,
+                       TypeAttr:$type);
+
+  let regions = (region MaxSizedRegion<1>:$allocRegion,
+                        AnyRegion:$initializerRegion,
+                        AnyRegion:$reductionRegion,
+                        AnyRegion:$atomicReductionRegion,
+                        AnyRegion:$cleanupRegion);
+
+  let assemblyFormat = "$sym_name `:` $type attr-dict-with-keyword "
+                       "( `alloc` $allocRegion^ )? "
+                       "`init` $initializerRegion "
+                       "`combiner` $reductionRegion "
+                       "( `atomic` $atomicReductionRegion^ )? "
+                       "( `cleanup` $cleanupRegion^ )? ";
+
+  let extraClassDeclaration = [{
+    mlir::BlockArgument getAllocMoldArg() {
+      auto &region = getAllocRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    mlir::BlockArgument getInitializerMoldArg() {
+      return getInitializerRegion().getArgument(0);
+    }
+    mlir::BlockArgument getInitializerAllocArg() {
+      return getAllocRegion().empty() ?
+          nullptr : getInitializerRegion().getArgument(1);
+    }
+    mlir::BlockArgument getReductionLhsArg() {
+      return getReductionRegion().getArgument(0);
+    }
+    mlir::BlockArgument getReductionRhsArg() {
+      return getReductionRegion().getArgument(1);
+    }
+    mlir::BlockArgument getAtomicReductionLhsArg() {
+      auto &region = getAtomicReductionRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    mlir::BlockArgument getAtomicReductionRhsArg() {
+      auto &region = getAtomicReductionRegion();
+      return region.empty() ? nullptr : region.getArgument(1);
+    }
+    mlir::BlockArgument getCleanupAllocArg() {
+      auto &region = getCleanupRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+  }];
+
+  let hasRegionVerifier = 1;
+}
+
 def fir_DoConcurrentOp : fir_Op<"do_concurrent",
     [SingleBlock, AutomaticAllocationScope]> {
   let summary = "do concurrent loop wrapper";
@@ -3694,6 +3791,25 @@ def fir_LocalSpecifier {
   );
 }
 
+def fir_ReduceSpecifier {
+  dag arguments = (ins
+    Variadic<AnyType>:$reduce_vars,
+    OptionalAttr<DenseBoolArrayAttr>:$reduce_byref,
+
+    // This introduces redundency in how reductions are modelled. In particular,
+    // a single reduction is represented by 2 attributes:
+    //
+    // 1. `$reduce_syms` which is a list of `DeclareReductionOp`s.
+    // 2. `$reduce_attrs` which is an array of `fir::ReduceAttr` values.
+    //
+    // The first makes it easier to map `do concurrent` to parallization models
+    // (e.g. OpenMP and OpenACC) while the second makes it easier to map it to
+    // nests of `fir.do_loop ... unodered` ops.
+    OptionalAttr<SymbolRefArrayAttr>:$reduce_syms,
+    OptionalAttr<ArrayAttr>:$reduce_attrs
+  );
+}
+
 def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     [AttrSizedOperandSegments, DeclareOpInterfaceMethods<LoopLikeOpInterface,
                                                          ["getLoopInductionVars"]>,
@@ -3703,7 +3819,7 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
   let description = [{
     An operation that models a Fortran `do concurrent` loop's header and block.
     This is a single-region single-block terminator op that is expected to
-    terminate the region of a `omp.do_concurrent` wrapper op.
+    terminate the region of a `fir.do_concurrent` wrapper op.
 
     This op borrows from both `scf.parallel` and `fir.do_loop` ops. Similar to
     `scf.parallel`, a loop nest takes 3 groups of SSA values as operands that
@@ -3741,8 +3857,6 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     - `lowerBound`: The group of SSA values for the nest's lower bounds.
     - `upperBound`: The group of SSA values for the nest's upper bounds.
     - `step`: The group of SSA values for the nest's steps.
-    - `reduceOperands`: The reduction SSA values, if any.
-    - `reduceAttrs`: Attributes to store reduction operations, if any.
     - `loopAnnotation`: Loop metadata to be passed down the compiler pipeline to
       LLVM.
   }];
@@ -3751,12 +3865,12 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     Variadic<Index>:$lowerBound,
     Variadic<Index>:$upperBound,
     Variadic<Index>:$step,
-    Variadic<AnyType>:$reduceOperands,
-    OptionalAttr<ArrayAttr>:$reduceAttrs,
     OptionalAttr<LoopAnnotationAttr>:$loopAnnotation
   );
 
-  let arguments = !con(opArgs, fir_LocalSpecifier.arguments);
+  let arguments = !con(opArgs,
+    fir_LocalSpecifier.arguments,
+    fir_ReduceSpecifier.arguments);
 
   let regions = (region SizedRegion<1>:$region);
 
@@ -3777,12 +3891,18 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
                                              getNumLocalOperands());
     }
 
+    mlir::Block::BlockArgListType getRegionReduceArgs() {
+      return getBody()->getArguments().slice(getNumInductionVars()
+                                               + getNumLocalOperands(),
+                                             getNumReduceOperands());
+    }
+
     /// Number of operands controlling the loop
     unsigned getNumControlOperands() { return getLowerBound().size() * 3; }
 
     // Get Number of reduction operands
     unsigned getNumReduceOperands() {
-      return getReduceOperands().size();
+      return getReduceVars().size();
     }
 
     mlir::Operation::operand_range getLocalOperands() {

diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index cd55d10314740..e2d3fe964d49b 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Lower/Bridge.h"
 
+#include "OpenMP/ReductionProcessor.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -127,9 +128,8 @@ struct IncrementLoopInfo {
   bool isConcurrent;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localSymList;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localInitSymList;
-  llvm::SmallVector<
-      std::pair<fir::ReduceOperationEnum, const Fortran::semantics::Symbol *>>
-      reduceSymList;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> reduceSymList;
+  llvm::SmallVector<fir::ReduceOperationEnum> reduceOperatorList;
   llvm::SmallVector<const Fortran::semantics::Symbol *> sharedSymList;
   mlir::Value loopVariable = nullptr;
 
@@ -1993,7 +1993,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     case Fortran::parser::ReductionOperator::Operator::Ior:
       return fir::ReduceOperationEnum::IOR;
     case Fortran::parser::ReductionOperator::Operator::Ieor:
-      return fir::ReduceOperationEnum::EIOR;
+      return fir::ReduceOperationEnum::IEOR;
     }
     llvm_unreachable("illegal reduction operator");
   }
@@ -2027,8 +2027,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
               std::get<Fortran::parser::ReductionOperator>(reduceList->t));
           for (const Fortran::parser::Name &x :
                std::get<std::list<Fortran::parser::Name>>(reduceList->t)) {
-            info.reduceSymList.push_back(
-                std::make_pair(reduce_operation, x.symbol));
+            info.reduceSymList.push_back(x.symbol);
+            info.reduceOperatorList.push_back(reduce_operation);
           }
         }
       }
@@ -2089,6 +2089,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         assign.u = Fortran::evaluate::Assignment::BoundsSpec{};
       genAssignment(assign);
     }
+
     for (const Fortran::semantics::Symbol *sym : info.sharedSymList) {
       const auto *hostDetails =
           sym->detailsIf<Fortran::semantics::HostAssocDetails>();
@@ -2112,6 +2113,45 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       }
     }
 
+    llvm::SmallVector<bool> reduceVarByRef;
+    llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
+    llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
+
+    for (const auto &reduceOp : info.reduceOperatorList)
+      nestReduceAttrs.push_back(
+          fir::ReduceAttr::get(builder->getContext(), reduceOp));
+
+    llvm::SmallVector<mlir::Value> reduceVars;
+    Fortran::lower::omp::ReductionProcessor rp;
+    rp.processReductionArguments<fir::DeclareReductionOp>(
+        toLocation(), *this, info.reduceOperatorList, reduceVars,
+        reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
+
+    doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
+    doConcurrentLoopOp.setReduceSymsAttr(
+        reductionDeclSymbols.empty()
+            ? nullptr
+            : mlir::ArrayAttr::get(builder->getContext(),
+                                   reductionDeclSymbols));
+    doConcurrentLoopOp.setReduceAttrsAttr(
+        nestReduceAttrs.empty()
+            ? nullptr
+            : mlir::ArrayAttr::get(builder->getContext(), nestReduceAttrs));
+    doConcurrentLoopOp.setReduceByrefAttr(
+        reduceVarByRef.empty() ? nullptr
+                               : mlir::DenseBoolArrayAttr::get(
+                                     builder->getContext(), reduceVarByRef));
+
+    for (auto [sym, reduceVar] :
+         llvm::zip_equal(info.reduceSymList, reduceVars)) {
+      auto arg = doConcurrentLoopOp.getRegion().begin()->addArgument(
+          reduceVar.getType(), doConcurrentLoopOp.getLoc());
+      bindSymbol(*sym, hlfir::translateToExtendedValue(
+                           reduceVar.getLoc(), *builder, hlfir::Entity{arg},
+                           /*contiguousHint=*/true)
+                           .first);
+    }
+
     // Note that allocatable, types with ultimate components, and type
     // requiring finalization are forbidden in LOCAL/LOCAL_INIT (F2023 C1130),
     // so no clean-up needs to be generated for these entities.
@@ -2203,6 +2243,12 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       }
     }
 
+    // Introduce a `do concurrent` scope to bind symbols corresponding to local,
+    // local_init, and reduce region arguments.
+    if (!incrementLoopNestInfo.empty() &&
+        incrementLoopNestInfo.back().isConcurrent)
+      localSymbols.pushScope();
+
     // Increment loop begin code. (Infinite/while code was already generated.)
     if (!infiniteLoop && !whileCondition)
       genFIRIncrementLoopBegin(incrementLoopNestInfo, doStmtEval.dirs);
@@ -2226,6 +2272,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     // This call may generate a branch in some contexts.
     genFIR(endDoEval, unstructuredContext);
+
+    if (!incrementLoopNestInfo.empty() &&
+        incrementLoopNestInfo.back().isConcurrent)
+      localSymbols.popScope();
   }
 
   /// Generate FIR to evaluate loop control values (lower, upper and step).
@@ -2408,19 +2458,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         info.stepVariable = builder->createTemporary(loc, stepValue.getType());
         builder->create<fir::StoreOp>(loc, stepValue, info.stepVariable);
       }
-
-      if (genDoConcurrent && nestReduceOperands.empty()) {
-        // Create DO CONCURRENT reduce operands and attributes
-        for (const auto &reduceSym : info.reduceSymList) {
-          const fir::ReduceOperationEnum reduceOperation = reduceSym.first;
-          const Fortran::semantics::Symbol *sym = reduceSym.second;
-          fir::ExtendedValue exv = getSymbolExtendedValue(*sym, nullptr);
-          nestReduceOperands.push_back(fir::getBase(exv));
-          auto reduceAttr =
-              fir::ReduceAttr::get(builder->getContext(), reduceOperation);
-          nestReduceAttrs.push_back(reduceAttr);
-        }
-      }
     }
 
     for (auto [info, lowerValue, upperValue, stepValue] :
@@ -2518,11 +2555,11 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
-          loc, nestLBs, nestUBs, nestSts, nestReduceOperands,
-          nestReduceAttrs.empty()
-              ? nullptr
-              : mlir::ArrayAttr::get(builder->getContext(), nestReduceAttrs),
-          nullptr, /*local_vars=*/std::nullopt, /*local_syms=*/nullptr);
+          loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
+          /*local_vars=*/std::nullopt,
+          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
+          /*reduce_attrs=*/nullptr);
 
       llvm::SmallVector<mlir::Type> loopBlockArgTypes(
           incrementLoopNestInfo.size(), builder->getIndexType());

diff  --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 7bea427099a28..5aebfc901e8ac 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -12,6 +12,7 @@
 
 #include "ClauseProcessor.h"
 #include "Clauses.h"
+#include "ReductionProcessor.h"
 #include "Utils.h"
 
 #include "flang/Lower/ConvertExprToHLFIR.h"
@@ -25,6 +26,21 @@ namespace Fortran {
 namespace lower {
 namespace omp {
 
+using ReductionModifier =
+    Fortran::lower::omp::clause::Reduction::ReductionModifier;
+
+mlir::omp::ReductionModifier translateReductionModifier(ReductionModifier mod) {
+  switch (mod) {
+  case ReductionModifier::Default:
+    return mlir::omp::ReductionModifier::defaultmod;
+  case ReductionModifier::Inscan:
+    return mlir::omp::ReductionModifier::inscan;
+  case ReductionModifier::Task:
+    return mlir::omp::ReductionModifier::task;
+  }
+  return mlir::omp::ReductionModifier::defaultmod;
+}
+
 /// Check for unsupported map operand types.
 static void checkMapType(mlir::Location location, mlir::Type type) {
   if (auto refType = mlir::dyn_cast<fir::ReferenceType>(type))
@@ -1076,6 +1092,18 @@ bool ClauseProcessor::processIf(
   });
   return found;
 }
+
+template <typename T>
+void collectReductionSyms(
+    const T &reduction,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
+  const auto &objectList{std::get<omp::ObjectList>(reduction.t)};
+  for (const Object &object : objectList) {
+    const semantics::Symbol *symbol = object.sym();
+    reductionSyms.push_back(symbol);
+  }
+}
+
 bool ClauseProcessor::processInReduction(
     mlir::Location currentLocation, mlir::omp::InReductionClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &outReductionSyms) const {
@@ -1085,10 +1113,14 @@ bool ClauseProcessor::processInReduction(
         llvm::SmallVector<bool> inReduceVarByRef;
         llvm::SmallVector<mlir::Attribute> inReductionDeclSymbols;
         llvm::SmallVector<const semantics::Symbol *> inReductionSyms;
+        collectReductionSyms(clause, inReductionSyms);
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::InReduction>(
-            currentLocation, converter, clause, inReductionVars,
-            inReduceVarByRef, inReductionDeclSymbols, inReductionSyms);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
+            inReductionSyms);
 
         // Copy local lists into the output.
         llvm::copy(inReductionVars, std::back_inserter(result.inReductionVars));
@@ -1416,10 +1448,23 @@ bool ClauseProcessor::processReduction(
         llvm::SmallVector<bool> reduceVarByRef;
         llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
         llvm::SmallVector<const semantics::Symbol *> reductionSyms;
+        collectReductionSyms(clause, reductionSyms);
+
+        auto mod = std::get<std::optional<ReductionModifier>>(clause.t);
+        if (mod.has_value()) {
+          if (mod.value() == ReductionModifier::Task)
+            TODO(currentLocation, "Reduction modifier `task` is not supported");
+          else
+            result.reductionMod = mlir::omp::ReductionModifierAttr::get(
+                converter.getFirOpBuilder().getContext(),
+                translateReductionModifier(mod.value()));
+        }
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::Reduction>(
-            currentLocation, converter, clause, reductionVars, reduceVarByRef,
-            reductionDeclSymbols, reductionSyms, &result.reductionMod);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            reductionVars, reduceVarByRef, reductionDeclSymbols, reductionSyms);
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
@@ -1435,21 +1480,25 @@ bool ClauseProcessor::processTaskReduction(
   return findRepeatableClause<omp::clause::TaskReduction>(
       [&](const omp::clause::TaskReduction &clause, const parser::CharBlock &) {
         llvm::SmallVector<mlir::Value> taskReductionVars;
-        llvm::SmallVector<bool> TaskReduceVarByRef;
-        llvm::SmallVector<mlir::Attribute> TaskReductionDeclSymbols;
-        llvm::SmallVector<const semantics::Symbol *> TaskReductionSyms;
+        llvm::SmallVector<bool> taskReduceVarByRef;
+        llvm::SmallVector<mlir::Attribute> taskReductionDeclSymbols;
+        llvm::SmallVector<const semantics::Symbol *> taskReductionSyms;
+        collectReductionSyms(clause, taskReductionSyms);
+
         ReductionProcessor rp;
-        rp.processReductionArguments<omp::clause::TaskReduction>(
-            currentLocation, converter, clause, taskReductionVars,
-            TaskReduceVarByRef, TaskReductionDeclSymbols, TaskReductionSyms);
+        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+            currentLocation, converter,
+            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+            taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
+            taskReductionSyms);
         // Copy local lists into the output.
         llvm::copy(taskReductionVars,
                    std::back_inserter(result.taskReductionVars));
-        llvm::copy(TaskReduceVarByRef,
+        llvm::copy(taskReduceVarByRef,
                    std::back_inserter(result.taskReductionByref));
-        llvm::copy(TaskReductionDeclSymbols,
+        llvm::copy(taskReductionDeclSymbols,
                    std::back_inserter(result.taskReductionSyms));
-        llvm::copy(TaskReductionSyms, std::back_inserter(outReductionSyms));
+        llvm::copy(taskReductionSyms, std::back_inserter(outReductionSyms));
       });
 }
 

diff  --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 3d8c4a337a4a4..46b749fb66c86 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -14,7 +14,6 @@
 
 #include "ClauseFinder.h"
 #include "Clauses.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/Bridge.h"

diff  --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index d7ab21d428e32..7f317f05f67b7 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -179,6 +179,7 @@ using IteratorSpecifier = tomp::type::IteratorSpecifierT<TypeTy, IdTy, ExprTy>;
 using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
 using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
 using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
+using ReductionOperatorList = List<ReductionOperator>;
 using DependenceType = tomp::type::DependenceType;
 using Prescriptiveness = tomp::type::Prescriptiveness;
 

diff  --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0a56e888ac44b..65e852cbcc911 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -17,7 +17,6 @@
 #include "Clauses.h"
 #include "DataSharingProcessor.h"
 #include "Decomposer.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Lower/Bridge.h"

diff  --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index 330cef7b54c74..d14fc1f7a52da 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -12,6 +12,7 @@
 
 #include "ReductionProcessor.h"
 
+#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/ConvertType.h"
 #include "flang/Lower/Support/PrivateReductionUtils.h"
@@ -21,8 +22,6 @@
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
-#include "flang/Optimizer/Support/FatalError.h"
-#include "flang/Parser/tools.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "llvm/Support/CommandLine.h"
 #include <type_traits>
@@ -40,35 +39,35 @@ namespace lower {
 namespace omp {
 
 // explicit template declarations
-template void
-ReductionProcessor::processReductionArguments<omp::clause::Reduction>(
+template void ReductionProcessor::processReductionArguments<
+    mlir::omp::DeclareReductionOp, omp::clause::ReductionOperatorList>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::Reduction &reduction,
+    const omp::clause::ReductionOperatorList &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void
-ReductionProcessor::processReductionArguments<omp::clause::TaskReduction>(
+template void ReductionProcessor::processReductionArguments<
+    fir::DeclareReductionOp, llvm::SmallVector<fir::ReduceOperationEnum>>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::TaskReduction &reduction,
+    const llvm::SmallVector<fir::ReduceOperationEnum> &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void
-ReductionProcessor::processReductionArguments<omp::clause::InReduction>(
-    mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const omp::clause::InReduction &reduction,
-    llvm::SmallVectorImpl<mlir::Value> &reductionVars,
-    llvm::SmallVectorImpl<bool> &reduceVarByRef,
-    llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod);
+template mlir::omp::DeclareReductionOp
+ReductionProcessor::createDeclareReduction<mlir::omp::DeclareReductionOp>(
+    AbstractConverter &converter, llvm::StringRef reductionOpName,
+    const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
+    bool isByRef);
+
+template fir::DeclareReductionOp
+ReductionProcessor::createDeclareReduction<fir::DeclareReductionOp>(
+    AbstractConverter &converter, llvm::StringRef reductionOpName,
+    const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
+    bool isByRef);
 
 ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(
     const omp::clause::ProcedureDesignator &pd) {
@@ -106,6 +105,37 @@ ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(
   }
 }
 
+ReductionProcessor::ReductionIdentifier
+ReductionProcessor::getReductionType(const fir::ReduceOperationEnum &redOp) {
+  switch (redOp) {
+  case fir::ReduceOperationEnum::Add:
+    return ReductionIdentifier::ADD;
+  case fir::ReduceOperationEnum::Multiply:
+    return ReductionIdentifier::MULTIPLY;
+
+  case fir::ReduceOperationEnum::AND:
+    return ReductionIdentifier::AND;
+  case fir::ReduceOperationEnum::OR:
+    return ReductionIdentifier::OR;
+
+  case fir::ReduceOperationEnum::EQV:
+    return ReductionIdentifier::EQV;
+  case fir::ReduceOperationEnum::NEQV:
+    return ReductionIdentifier::NEQV;
+
+  case fir::ReduceOperationEnum::IAND:
+    return ReductionIdentifier::IAND;
+  case fir::ReduceOperationEnum::IEOR:
+    return ReductionIdentifier::IEOR;
+  case fir::ReduceOperationEnum::IOR:
+    return ReductionIdentifier::IOR;
+  case fir::ReduceOperationEnum::MAX:
+    return ReductionIdentifier::MAX;
+  case fir::ReduceOperationEnum::MIN:
+    return ReductionIdentifier::MIN;
+  }
+}
+
 bool ReductionProcessor::supportedIntrinsicProcReduction(
     const omp::clause::ProcedureDesignator &pd) {
   semantics::Symbol *sym = pd.v.sym();
@@ -136,28 +166,29 @@ ReductionProcessor::getReductionName(llvm::StringRef name,
   return fir::getTypeAsString(ty, kindMap, (name + byrefAddition).str());
 }
 
-std::string ReductionProcessor::getReductionName(
-    omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp,
-    const fir::KindMapping &kindMap, mlir::Type ty, bool isByRef) {
+std::string
+ReductionProcessor::getReductionName(ReductionIdentifier redId,
+                                     const fir::KindMapping &kindMap,
+                                     mlir::Type ty, bool isByRef) {
   std::string reductionName;
 
-  switch (intrinsicOp) {
-  case omp::clause::DefinedOperator::IntrinsicOperator::Add:
+  switch (redId) {
+  case ReductionIdentifier::ADD:
     reductionName = "add_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::Multiply:
+  case ReductionIdentifier::MULTIPLY:
     reductionName = "multiply_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::AND:
+  case ReductionIdentifier::AND:
     reductionName = "and_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::EQV:
+  case ReductionIdentifier::EQV:
     reductionName = "eqv_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::OR:
+  case ReductionIdentifier::OR:
     reductionName = "or_reduction";
     break;
-  case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:
+  case ReductionIdentifier::NEQV:
     reductionName = "neqv_reduction";
     break;
   default:
@@ -334,8 +365,18 @@ mlir::Value ReductionProcessor::createScalarCombiner(
   return reductionOp;
 }
 
+template <typename ParentDeclOpType>
+static void genYield(fir::FirOpBuilder &builder, mlir::Location loc,
+                     mlir::Value yieldedValue) {
+  if constexpr (std::is_same_v<ParentDeclOpType, mlir::omp::DeclareReductionOp>)
+    builder.create<mlir::omp::YieldOp>(loc, yieldedValue);
+  else
+    builder.create<fir::YieldOp>(loc, yieldedValue);
+}
+
 /// Create reduction combiner region for reduction variables which are boxed
 /// arrays
+template <typename DeclRedOpType>
 static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
                            ReductionProcessor::ReductionIdentifier redId,
                            fir::BaseBoxType boxTy, mlir::Value lhs,
@@ -369,7 +410,7 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value result = ReductionProcessor::createScalarCombiner(
         builder, loc, redId, eleTy, lhs, rhs);
     builder.create<fir::StoreOp>(loc, result, lhsValAddr);
-    builder.create<mlir::omp::YieldOp>(loc, lhsAddr);
+    genYield<DeclRedOpType>(builder, loc, lhsAddr);
     return;
   }
 
@@ -408,10 +449,11 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
   builder.create<fir::StoreOp>(loc, scalarReduction, lhsEleAddr);
 
   builder.setInsertionPointAfter(nest.outerOp);
-  builder.create<mlir::omp::YieldOp>(loc, lhsAddr);
+  genYield<DeclRedOpType>(builder, loc, lhsAddr);
 }
 
 // generate combiner region for reduction operations
+template <typename DeclRedOpType>
 static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
                         ReductionProcessor::ReductionIdentifier redId,
                         mlir::Type ty, mlir::Value lhs, mlir::Value rhs,
@@ -426,15 +468,15 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
         builder, loc, redId, ty, lhsLoaded, rhsLoaded);
     if (isByRef) {
       builder.create<fir::StoreOp>(loc, result, lhs);
-      builder.create<mlir::omp::YieldOp>(loc, lhs);
+      genYield<DeclRedOpType>(builder, loc, lhs);
     } else {
-      builder.create<mlir::omp::YieldOp>(loc, result);
+      genYield<DeclRedOpType>(builder, loc, result);
     }
     return;
   }
   // all arrays should have been boxed
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {
-    genBoxCombiner(builder, loc, redId, boxTy, lhs, rhs);
+    genBoxCombiner<DeclRedOpType>(builder, loc, redId, boxTy, lhs, rhs);
     return;
   }
 
@@ -454,15 +496,13 @@ static mlir::Type unwrapSeqOrBoxedType(mlir::Type ty) {
   return ty;
 }
 
+template <typename OpType>
 static void createReductionAllocAndInitRegions(
-    AbstractConverter &converter, mlir::Location loc,
-    mlir::omp::DeclareReductionOp &reductionDecl,
+    AbstractConverter &converter, mlir::Location loc, OpType &reductionDecl,
     const ReductionProcessor::ReductionIdentifier redId, mlir::Type type,
     bool isByRef) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  auto yield = [&](mlir::Value ret) {
-    builder.create<mlir::omp::YieldOp>(loc, ret);
-  };
+  auto yield = [&](mlir::Value ret) { genYield<OpType>(builder, loc, ret); };
 
   mlir::Block *allocBlock = nullptr;
   mlir::Block *initBlock = nullptr;
@@ -512,7 +552,8 @@ static void createReductionAllocAndInitRegions(
   yield(boxAlloca);
 }
 
-mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
+template <typename OpType>
+OpType ReductionProcessor::createDeclareReduction(
     AbstractConverter &converter, llvm::StringRef reductionOpName,
     const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,
     bool isByRef) {
@@ -522,8 +563,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
 
   assert(!reductionOpName.empty());
 
-  auto decl =
-      module.lookupSymbol<mlir::omp::DeclareReductionOp>(reductionOpName);
+  auto decl = module.lookupSymbol<OpType>(reductionOpName);
   if (decl)
     return decl;
 
@@ -532,8 +572,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
   if (!isByRef)
     type = valTy;
 
-  decl = modBuilder.create<mlir::omp::DeclareReductionOp>(loc, reductionOpName,
-                                                          type);
+  decl = modBuilder.create<OpType>(loc, reductionOpName, type);
   createReductionAllocAndInitRegions(converter, loc, decl, redId, type,
                                      isByRef);
 
@@ -544,7 +583,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
   builder.setInsertionPointToEnd(&decl.getReductionRegion().back());
   mlir::Value op1 = decl.getReductionRegion().front().getArgument(0);
   mlir::Value op2 = decl.getReductionRegion().front().getArgument(1);
-  genCombiner(builder, loc, redId, type, op1, op2, isByRef);
+  genCombiner<OpType>(builder, loc, redId, type, op1, op2, isByRef);
 
   return decl;
 }
@@ -563,64 +602,41 @@ static bool doReductionByRef(mlir::Value reductionVar) {
   return false;
 }
 
-mlir::omp::ReductionModifier translateReductionModifier(ReductionModifier mod) {
-  switch (mod) {
-  case ReductionModifier::Default:
-    return mlir::omp::ReductionModifier::defaultmod;
-  case ReductionModifier::Inscan:
-    return mlir::omp::ReductionModifier::inscan;
-  case ReductionModifier::Task:
-    return mlir::omp::ReductionModifier::task;
-  }
-  return mlir::omp::ReductionModifier::defaultmod;
-}
-
-template <class T>
+template <typename OpType, typename RedOperatorListTy>
 void ReductionProcessor::processReductionArguments(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
-    const T &reduction, llvm::SmallVectorImpl<mlir::Value> &reductionVars,
+    const RedOperatorListTy &redOperatorList,
+    llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-    mlir::omp::ReductionModifierAttr *reductionMod) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
-  if constexpr (std::is_same_v<T, omp::clause::Reduction>) {
-    auto mod = std::get<std::optional<ReductionModifier>>(reduction.t);
-    if (mod.has_value()) {
-      if (mod.value() == ReductionModifier::Task)
-        TODO(currentLocation, "Reduction modifier `task` is not supported");
-      else
-        *reductionMod = mlir::omp::ReductionModifierAttr::get(
-            firOpBuilder.getContext(), translateReductionModifier(mod.value()));
-    }
-  }
-
-  mlir::omp::DeclareReductionOp decl;
-  const auto &redOperatorList{
-      std::get<typename T::ReductionIdentifiers>(reduction.t)};
-  assert(redOperatorList.size() == 1 && "Expecting single operator");
-  const auto &redOperator = redOperatorList.front();
-  const auto &objectList{std::get<omp::ObjectList>(reduction.t)};
-
-  if (!std::holds_alternative<omp::clause::DefinedOperator>(redOperator.u)) {
-    if (const auto *reductionIntrinsic =
-            std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
-      if (!ReductionProcessor::supportedIntrinsicProcReduction(
-              *reductionIntrinsic)) {
+    const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols) {
+  if constexpr (std::is_same_v<RedOperatorListTy,
+                               omp::clause::ReductionOperatorList>) {
+    // For OpenMP reduction clauses, check if the reduction operator is
+    // supported.
+    assert(redOperatorList.size() == 1 && "Expecting single operator");
+    const Fortran::lower::omp::clause::ReductionOperator &redOperator =
+        redOperatorList.front();
+
+    if (!std::holds_alternative<omp::clause::DefinedOperator>(redOperator.u)) {
+      if (const auto *reductionIntrinsic =
+              std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
+        if (!ReductionProcessor::supportedIntrinsicProcReduction(
+                *reductionIntrinsic)) {
+          return;
+        }
+      } else {
         return;
       }
-    } else {
-      return;
     }
   }
 
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
   // Reduction variable processing common to both intrinsic operators and
   // procedure designators
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  for (const Object &object : objectList) {
-    const semantics::Symbol *symbol = object.sym();
-    reductionSymbols.push_back(symbol);
+  for (const semantics::Symbol *symbol : reductionSymbols) {
     mlir::Value symVal = converter.getSymbolAddress(*symbol);
     mlir::Type eleType;
     auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
@@ -672,52 +688,63 @@ void ReductionProcessor::processReductionArguments(
     reduceVarByRef.push_back(doReductionByRef(symVal));
   }
 
+  unsigned idx = 0;
   for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
     auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
     const auto &kindMap = firOpBuilder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
 
-    if (const auto &redDefinedOp =
-            std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
-      const auto &intrinsicOp{
-          std::get<omp::clause::DefinedOperator::IntrinsicOperator>(
-              redDefinedOp->u)};
-      redId = getReductionType(intrinsicOp);
-      switch (redId) {
-      case ReductionIdentifier::ADD:
-      case ReductionIdentifier::MULTIPLY:
-      case ReductionIdentifier::AND:
-      case ReductionIdentifier::EQV:
-      case ReductionIdentifier::OR:
-      case ReductionIdentifier::NEQV:
-        break;
-      default:
-        TODO(currentLocation,
-             "Reduction of some intrinsic operators is not supported");
-        break;
-      }
-
-      reductionName = getReductionName(intrinsicOp, kindMap, redType, isByRef);
-    } else if (const auto *reductionIntrinsic =
-                   std::get_if<omp::clause::ProcedureDesignator>(
-                       &redOperator.u)) {
-      if (!ReductionProcessor::supportedIntrinsicProcReduction(
-              *reductionIntrinsic)) {
-        TODO(currentLocation, "Unsupported intrinsic proc reduction");
+    if constexpr (std::is_same_v<RedOperatorListTy,
+                                 omp::clause::ReductionOperatorList>) {
+      const Fortran::lower::omp::clause::ReductionOperator &redOperator =
+          redOperatorList.front();
+      if (const auto &redDefinedOp =
+              std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
+        const auto &intrinsicOp{
+            std::get<omp::clause::DefinedOperator::IntrinsicOperator>(
+                redDefinedOp->u)};
+        redId = getReductionType(intrinsicOp);
+        switch (redId) {
+        case ReductionIdentifier::ADD:
+        case ReductionIdentifier::MULTIPLY:
+        case ReductionIdentifier::AND:
+        case ReductionIdentifier::EQV:
+        case ReductionIdentifier::OR:
+        case ReductionIdentifier::NEQV:
+          break;
+        default:
+          TODO(currentLocation,
+               "Reduction of some intrinsic operators is not supported");
+          break;
+        }
+
+        reductionName = getReductionName(redId, kindMap, redType, isByRef);
+      } else if (const auto *reductionIntrinsic =
+                     std::get_if<omp::clause::ProcedureDesignator>(
+                         &redOperator.u)) {
+        if (!ReductionProcessor::supportedIntrinsicProcReduction(
+                *reductionIntrinsic)) {
+          TODO(currentLocation, "Unsupported intrinsic proc reduction");
+        }
+        redId = getReductionType(*reductionIntrinsic);
+        reductionName =
+            getReductionName(getRealName(*reductionIntrinsic).ToString(),
+                             kindMap, redType, isByRef);
+      } else {
+        TODO(currentLocation, "Unexpected reduction type");
       }
-      redId = getReductionType(*reductionIntrinsic);
-      reductionName =
-          getReductionName(getRealName(*reductionIntrinsic).ToString(), kindMap,
-                           redType, isByRef);
     } else {
-      TODO(currentLocation, "Unexpected reduction type");
+      // `do concurrent` reductions
+      redId = getReductionType(redOperatorList[idx]);
+      reductionName = getReductionName(redId, kindMap, redType, isByRef);
     }
 
-    decl = createDeclareReduction(converter, reductionName, redId, redType,
-                                  currentLocation, isByRef);
+    OpType decl = createDeclareReduction<OpType>(
+        converter, reductionName, redId, redType, currentLocation, isByRef);
     reductionDeclSymbols.push_back(
         mlir::SymbolRefAttr::get(firOpBuilder.getContext(), decl.getSymName()));
+    ++idx;
   }
 }
 

diff  --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/lib/Lower/OpenMP/ReductionProcessor.h
index a7198b48f6b4e..95b4b077bdc46 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.h
@@ -19,7 +19,6 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/type.h"
-#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Types.h"
 
@@ -65,6 +64,9 @@ class ReductionProcessor {
   static ReductionIdentifier
   getReductionType(omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp);
 
+  static ReductionIdentifier
+  getReductionType(const fir::ReduceOperationEnum &pd);
+
   static bool
   supportedIntrinsicProcReduction(const omp::clause::ProcedureDesignator &pd);
 
@@ -78,10 +80,9 @@ class ReductionProcessor {
                                       const fir::KindMapping &kindMap,
                                       mlir::Type ty, bool isByRef);
 
-  static std::string
-  getReductionName(omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp,
-                   const fir::KindMapping &kindMap, mlir::Type ty,
-                   bool isByRef);
+  static std::string getReductionName(ReductionIdentifier redId,
+                                      const fir::KindMapping &kindMap,
+                                      mlir::Type ty, bool isByRef);
 
   /// This function returns the identity value of the operator \p
   /// reductionOpName. For example:
@@ -113,22 +114,23 @@ class ReductionProcessor {
   /// symbol table. The declaration has a constant initializer with the neutral
   /// value `initValue`, and the reduction combiner carried over from `reduce`.
   /// TODO: add atomic region.
-  static mlir::omp::DeclareReductionOp
-  createDeclareReduction(AbstractConverter &builder,
-                         llvm::StringRef reductionOpName,
-                         const ReductionIdentifier redId, mlir::Type type,
-                         mlir::Location loc, bool isByRef);
+  template <typename OpType>
+  static OpType createDeclareReduction(AbstractConverter &builder,
+                                       llvm::StringRef reductionOpName,
+                                       const ReductionIdentifier redId,
+                                       mlir::Type type, mlir::Location loc,
+                                       bool isByRef);
 
   /// Creates a reduction declaration and associates it with an OpenMP block
   /// directive.
-  template <class T>
+  template <typename OpType, typename RedOperatorListTy>
   static void processReductionArguments(
       mlir::Location currentLocation, lower::AbstractConverter &converter,
-      const T &reduction, llvm::SmallVectorImpl<mlir::Value> &reductionVars,
+      const RedOperatorListTy &redOperatorList,
+      llvm::SmallVectorImpl<mlir::Value> &reductionVars,
       llvm::SmallVectorImpl<bool> &reduceVarByRef,
       llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-      llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
-      mlir::omp::ReductionModifierAttr *reductionMod = nullptr);
+      const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 };
 
 template <typename FloatOp, typename IntegerOp>

diff  --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index c65f51ce6cacd..b9d2574a76ad0 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -668,9 +668,7 @@ void privatizeSymbol(
 
   const semantics::Symbol *sym =
       isDoConcurrent ? &symToPrivatize->GetUltimate() : symToPrivatize;
-  const lower::SymbolBox hsb = isDoConcurrent
-                                   ? converter.shallowLookupSymbol(*sym)
-                                   : converter.lookupOneLevelUpSymbol(*sym);
+  const lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
   assert(hsb && "Host symbol box not found");
 
   mlir::Location symLoc = hsb.getAddr().getLoc();

diff  --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 3bbc32f23bcfa..c35d757be0f09 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -3342,26 +3342,26 @@ struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
   }
 };
 
-struct LocalitySpecifierOpConversion
-    : public fir::FIROpConversion<fir::LocalitySpecifierOp> {
-  using FIROpConversion::FIROpConversion;
+template <typename OpTy>
+struct DoConcurrentSpecifierOpConversion : public fir::FIROpConversion<OpTy> {
+  using fir::FIROpConversion<OpTy>::FIROpConversion;
   llvm::LogicalResult
-  matchAndRewrite(fir::LocalitySpecifierOp localizer, OpAdaptor adaptor,
+  matchAndRewrite(OpTy specifier, typename OpTy::Adaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
 #ifdef EXPENSIVE_CHECKS
     auto uses = mlir::SymbolTable::getSymbolUses(
-        localizer, localizer->getParentOfType<mlir::ModuleOp>());
+        specifier, specifier->getParentOfType<mlir::ModuleOp>());
 
-    // `fir.local` ops are not supposed to have any uses at this point (i.e.
-    // during lowering to LLVM). In case of serialization, the
-    // `fir.do_concurrent` users are expected to have been lowered to
+    // `fir.local|fir.declare_reduction` ops are not supposed to have any uses
+    // at this point (i.e. during lowering to LLVM). In case of serialization,
+    // the `fir.do_concurrent` users are expected to have been lowered to
     // `fir.do_loop` nests. In case of parallelization, the `fir.do_concurrent`
     // users are expected to have been lowered to the target parallel model
     // (e.g. OpenMP).
     assert(uses && uses->empty());
 #endif
 
-    rewriter.eraseOp(localizer);
+    rewriter.eraseOp(specifier);
     return mlir::success();
   }
 };
@@ -4330,20 +4330,22 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, VolatileCastOpConversion, ConvertOpConversion,
       CoordinateOpConversion, CopyOpConversion, DTEntryOpConversion,
-      DeclareOpConversion, DivcOpConversion, EmboxOpConversion,
-      EmboxCharOpConversion, EmboxProcOpConversion, ExtractValueOpConversion,
-      FieldIndexOpConversion, FirEndOpConversion, FreeMemOpConversion,
-      GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
-      IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
-      LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
-      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
-      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
-      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
-      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
-      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
-      XArrayCoorOpConversion, XEmboxOpConversion, XReboxOpConversion,
-      ZeroOpConversion>(converter, options);
+      DeclareOpConversion,
+      DoConcurrentSpecifierOpConversion<fir::LocalitySpecifierOp>,
+      DoConcurrentSpecifierOpConversion<fir::DeclareReductionOp>,
+      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
+      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
+      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
+      GlobalOpConversion, InsertOnRangeOpConversion, IsPresentOpConversion,
+      LenParamIndexOpConversion, LoadOpConversion, MulcOpConversion,
+      NegcOpConversion, NoReassocOpConversion, SelectCaseOpConversion,
+      SelectOpConversion, SelectRankOpConversion, SelectTypeOpConversion,
+      ShapeOpConversion, ShapeShiftOpConversion, ShiftOpConversion,
+      SliceOpConversion, StoreOpConversion, StringLitOpConversion,
+      SubcOpConversion, TypeDescOpConversion, TypeInfoOpConversion,
+      UnboxCharOpConversion, UnboxProcOpConversion, UndefOpConversion,
+      UnreachableOpConversion, XArrayCoorOpConversion, XEmboxOpConversion,
+      XReboxOpConversion, ZeroOpConversion>(converter, options);
 
   // Patterns that are populated without a type converter do not trigger
   // target materializations for the operands of the root op.

diff  --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..6b40e7015fdd8 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -5041,6 +5041,9 @@ void fir::BoxTotalElementsOp::getCanonicalizationPatterns(
 // LocalitySpecifierOp
 //===----------------------------------------------------------------------===//
 
+// TODO This is a copy of omp::PrivateClauseOp::verifiyRegions(). Once we find a
+// solution to merge both ops into one this duplication will not be needed. See:
+// https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
 llvm::LogicalResult fir::LocalitySpecifierOp::verifyRegions() {
   mlir::Type argType = getArgType();
   auto verifyTerminator = [&](mlir::Operation *terminator,
@@ -5136,6 +5139,84 @@ llvm::LogicalResult fir::LocalitySpecifierOp::verifyRegions() {
   return llvm::success();
 }
 
+// TODO This is a copy of omp::DeclareReductionOp::verifiyRegions(). Once we
+// find a solution to merge both ops into one this duplication will not be
+// needed.
+mlir::LogicalResult fir::DeclareReductionOp::verifyRegions() {
+  if (!getAllocRegion().empty()) {
+    for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
+      if (yieldOp.getResults().size() != 1 ||
+          yieldOp.getResults().getTypes()[0] != getType())
+        return emitOpError() << "expects alloc region to yield a value "
+                                "of the reduction type";
+    }
+  }
+
+  if (getInitializerRegion().empty())
+    return emitOpError() << "expects non-empty initializer region";
+  mlir::Block &initializerEntryBlock = getInitializerRegion().front();
+
+  if (initializerEntryBlock.getNumArguments() == 1) {
+    if (!getAllocRegion().empty())
+      return emitOpError() << "expects two arguments to the initializer region "
+                              "when an allocation region is used";
+  } else if (initializerEntryBlock.getNumArguments() == 2) {
+    if (getAllocRegion().empty())
+      return emitOpError() << "expects one argument to the initializer region "
+                              "when no allocation region is used";
+  } else {
+    return emitOpError()
+           << "expects one or two arguments to the initializer region";
+  }
+
+  for (mlir::Value arg : initializerEntryBlock.getArguments())
+    if (arg.getType() != getType())
+      return emitOpError() << "expects initializer region argument to match "
+                              "the reduction type";
+
+  for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
+    if (yieldOp.getResults().size() != 1 ||
+        yieldOp.getResults().getTypes()[0] != getType())
+      return emitOpError() << "expects initializer region to yield a value "
+                              "of the reduction type";
+  }
+
+  if (getReductionRegion().empty())
+    return emitOpError() << "expects non-empty reduction region";
+  mlir::Block &reductionEntryBlock = getReductionRegion().front();
+  if (reductionEntryBlock.getNumArguments() != 2 ||
+      reductionEntryBlock.getArgumentTypes()[0] !=
+          reductionEntryBlock.getArgumentTypes()[1] ||
+      reductionEntryBlock.getArgumentTypes()[0] != getType())
+    return emitOpError() << "expects reduction region with two arguments of "
+                            "the reduction type";
+  for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
+    if (yieldOp.getResults().size() != 1 ||
+        yieldOp.getResults().getTypes()[0] != getType())
+      return emitOpError() << "expects reduction region to yield a value "
+                              "of the reduction type";
+  }
+
+  if (!getAtomicReductionRegion().empty()) {
+    mlir::Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();
+    if (atomicReductionEntryBlock.getNumArguments() != 2 ||
+        atomicReductionEntryBlock.getArgumentTypes()[0] !=
+            atomicReductionEntryBlock.getArgumentTypes()[1])
+      return emitOpError() << "expects atomic reduction region with two "
+                              "arguments of the same type";
+  }
+
+  if (getCleanupRegion().empty())
+    return mlir::success();
+  mlir::Block &cleanupEntryBlock = getCleanupRegion().front();
+  if (cleanupEntryBlock.getNumArguments() != 1 ||
+      cleanupEntryBlock.getArgument(0).getType() != getType())
+    return emitOpError() << "expects cleanup region with one argument "
+                            "of the reduction type";
+
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // DoConcurrentOp
 //===----------------------------------------------------------------------===//
@@ -5157,6 +5238,97 @@ llvm::LogicalResult fir::DoConcurrentOp::verify() {
 // DoConcurrentLoopOp
 //===----------------------------------------------------------------------===//
 
+static mlir::ParseResult parseSpecifierList(
+    mlir::OpAsmParser &parser, mlir::OperationState &result,
+    llvm::StringRef specifierKeyword, llvm::StringRef symsAttrName,
+    llvm::SmallVectorImpl<mlir::OpAsmParser::Argument> &regionArgs,
+    llvm::SmallVectorImpl<mlir::Type> &regionArgTypes,
+    int32_t &numSpecifierOperands, bool isReduce = false) {
+  auto &builder = parser.getBuilder();
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> specifierOperands;
+
+  if (failed(parser.parseOptionalKeyword(specifierKeyword)))
+    return mlir::success();
+
+  std::size_t oldArgTypesSize = regionArgTypes.size();
+  if (failed(parser.parseLParen()))
+    return mlir::failure();
+
+  llvm::SmallVector<bool> isByRefVec;
+  llvm::SmallVector<mlir::SymbolRefAttr> spceifierSymbolVec;
+  llvm::SmallVector<fir::ReduceAttr> attributes;
+
+  if (failed(parser.parseCommaSeparatedList([&]() {
+        if (isReduce)
+          isByRefVec.push_back(
+              parser.parseOptionalKeyword("byref").succeeded());
+
+        if (failed(parser.parseAttribute(spceifierSymbolVec.emplace_back())))
+          return mlir::failure();
+
+        if (isReduce &&
+            failed(parser.parseAttribute(attributes.emplace_back())))
+          return mlir::failure();
+
+        if (parser.parseOperand(specifierOperands.emplace_back()) ||
+            parser.parseArrow() ||
+            parser.parseArgument(regionArgs.emplace_back()))
+          return mlir::failure();
+
+        return mlir::success();
+      })))
+    return mlir::failure();
+
+  if (failed(parser.parseColon()))
+    return mlir::failure();
+
+  if (failed(parser.parseCommaSeparatedList([&]() {
+        if (failed(parser.parseType(regionArgTypes.emplace_back())))
+          return mlir::failure();
+
+        return mlir::success();
+      })))
+    return mlir::failure();
+
+  if (regionArgs.size() != regionArgTypes.size())
+    return parser.emitError(parser.getNameLoc(), "mismatch in number of " +
+                                                     specifierKeyword.str() +
+                                                     " arg and types");
+
+  if (failed(parser.parseRParen()))
+    return mlir::failure();
+
+  for (auto operandType :
+       llvm::zip_equal(specifierOperands,
+                       llvm::drop_begin(regionArgTypes, oldArgTypesSize)))
+    if (parser.resolveOperand(std::get<0>(operandType),
+                              std::get<1>(operandType), result.operands))
+      return mlir::failure();
+
+  if (isReduce)
+    result.addAttribute(
+        fir::DoConcurrentLoopOp::getReduceByrefAttrName(result.name),
+        isByRefVec.empty()
+            ? nullptr
+            : mlir::DenseBoolArrayAttr::get(builder.getContext(), isByRefVec));
+
+  llvm::SmallVector<mlir::Attribute> symbolAttrs(spceifierSymbolVec.begin(),
+                                                 spceifierSymbolVec.end());
+  result.addAttribute(symsAttrName, builder.getArrayAttr(symbolAttrs));
+
+  if (isReduce) {
+    llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
+                                                 attributes.end());
+    result.addAttribute(
+        fir::DoConcurrentLoopOp::getReduceAttrsAttrName(result.name),
+        builder.getArrayAttr(arrayAttr));
+  }
+
+  numSpecifierOperands = specifierOperands.size();
+
+  return mlir::success();
+}
+
 mlir::ParseResult fir::DoConcurrentLoopOp::parse(mlir::OpAsmParser &parser,
                                                  mlir::OperationState &result) {
   auto &builder = parser.getBuilder();
@@ -5192,90 +5364,26 @@ mlir::ParseResult fir::DoConcurrentLoopOp::parse(mlir::OpAsmParser &parser,
       parser.resolveOperands(steps, builder.getIndexType(), result.operands))
     return mlir::failure();
 
-  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> reduceOperands;
-  llvm::SmallVector<mlir::Type> reduceArgTypes;
-  if (succeeded(parser.parseOptionalKeyword("reduce"))) {
-    // Parse reduction attributes and variables.
-    llvm::SmallVector<fir::ReduceAttr> attributes;
-    if (failed(parser.parseCommaSeparatedList(
-            mlir::AsmParser::Delimiter::Paren, [&]() {
-              if (parser.parseAttribute(attributes.emplace_back()) ||
-                  parser.parseArrow() ||
-                  parser.parseOperand(reduceOperands.emplace_back()) ||
-                  parser.parseColonType(reduceArgTypes.emplace_back()))
-                return mlir::failure();
-              return mlir::success();
-            })))
-      return mlir::failure();
-    // Resolve input operands.
-    for (auto operand_type : llvm::zip(reduceOperands, reduceArgTypes))
-      if (parser.resolveOperand(std::get<0>(operand_type),
-                                std::get<1>(operand_type), result.operands))
-        return mlir::failure();
-    llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
-                                                 attributes.end());
-    result.addAttribute(getReduceAttrsAttrName(result.name),
-                        builder.getArrayAttr(arrayAttr));
-  }
-
-  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> localOperands;
-  if (succeeded(parser.parseOptionalKeyword("local"))) {
-    std::size_t oldArgTypesSize = argTypes.size();
-    if (failed(parser.parseLParen()))
-      return mlir::failure();
-
-    llvm::SmallVector<mlir::SymbolRefAttr> localSymbolVec;
-    if (failed(parser.parseCommaSeparatedList([&]() {
-          if (failed(parser.parseAttribute(localSymbolVec.emplace_back())))
-            return mlir::failure();
-
-          if (parser.parseOperand(localOperands.emplace_back()) ||
-              parser.parseArrow() ||
-              parser.parseArgument(regionArgs.emplace_back()))
-            return mlir::failure();
-
-          return mlir::success();
-        })))
-      return mlir::failure();
-
-    if (failed(parser.parseColon()))
-      return mlir::failure();
-
-    if (failed(parser.parseCommaSeparatedList([&]() {
-          if (failed(parser.parseType(argTypes.emplace_back())))
-            return mlir::failure();
-
-          return mlir::success();
-        })))
-      return mlir::failure();
-
-    if (regionArgs.size() != argTypes.size())
-      return parser.emitError(parser.getNameLoc(),
-                              "mismatch in number of local arg and types");
-
-    if (failed(parser.parseRParen()))
-      return mlir::failure();
-
-    for (auto operandType : llvm::zip_equal(
-             localOperands, llvm::drop_begin(argTypes, oldArgTypesSize)))
-      if (parser.resolveOperand(std::get<0>(operandType),
-                                std::get<1>(operandType), result.operands))
-        return mlir::failure();
+  int32_t numLocalOperands = 0;
+  if (failed(parseSpecifierList(parser, result, "local",
+                                getLocalSymsAttrName(result.name), regionArgs,
+                                argTypes, numLocalOperands)))
+    return mlir::failure();
 
-    llvm::SmallVector<mlir::Attribute> symbolAttrs(localSymbolVec.begin(),
-                                                   localSymbolVec.end());
-    result.addAttribute(getLocalSymsAttrName(result.name),
-                        builder.getArrayAttr(symbolAttrs));
-  }
+  int32_t numReduceOperands = 0;
+  if (failed(parseSpecifierList(
+          parser, result, "reduce", getReduceSymsAttrName(result.name),
+          regionArgs, argTypes, numReduceOperands, /*isReduce=*/true)))
+    return mlir::failure();
 
   // Set `operandSegmentSizes` attribute.
-  result.addAttribute(DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
-                      builder.getDenseI32ArrayAttr(
-                          {static_cast<int32_t>(lower.size()),
-                           static_cast<int32_t>(upper.size()),
-                           static_cast<int32_t>(steps.size()),
-                           static_cast<int32_t>(reduceOperands.size()),
-                           static_cast<int32_t>(localOperands.size())}));
+  result.addAttribute(
+      DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
+      builder.getDenseI32ArrayAttr({static_cast<int32_t>(lower.size()),
+                                    static_cast<int32_t>(upper.size()),
+                                    static_cast<int32_t>(steps.size()),
+                                    static_cast<int32_t>(numLocalOperands),
+                                    static_cast<int32_t>(numReduceOperands)}));
 
   // Now parse the body.
   for (auto [arg, type] : llvm::zip_equal(regionArgs, argTypes))
@@ -5297,17 +5405,6 @@ void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
     << ") = (" << getLowerBound() << ") to (" << getUpperBound() << ") step ("
     << getStep() << ")";
 
-  if (!getReduceOperands().empty()) {
-    p << " reduce(";
-    auto attrs = getReduceAttrsAttr();
-    auto operands = getReduceOperands();
-    llvm::interleaveComma(llvm::zip(attrs, operands), p, [&](auto it) {
-      p << std::get<0>(it) << " -> " << std::get<1>(it) << " : "
-        << std::get<1>(it).getType();
-    });
-    p << ')';
-  }
-
   if (!getLocalVars().empty()) {
     p << " local(";
     llvm::interleaveComma(llvm::zip_equal(getLocalSymsAttr(), getLocalVars(),
@@ -5322,13 +5419,34 @@ void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
     p << ")";
   }
 
+  if (!getReduceVars().empty()) {
+    p << " reduce(";
+    llvm::interleaveComma(
+        llvm::zip_equal(getReduceByrefAttr().asArrayRef(), getReduceSymsAttr(),
+                        getReduceAttrsAttr(), getReduceVars(),
+                        getRegionReduceArgs()),
+        p, [&](auto it) {
+          if (std::get<0>(it))
+            p << "byref ";
+
+          p << std::get<1>(it) << " " << std::get<2>(it) << " "
+            << std::get<3>(it) << " -> " << std::get<4>(it);
+        });
+    p << " : ";
+    llvm::interleaveComma(getReduceVars(), p,
+                          [&](auto it) { p << it.getType(); });
+    p << ")";
+  }
+
   p << ' ';
   p.printRegion(getRegion(), /*printEntryBlockArgs=*/false);
   p.printOptionalAttrDict(
       (*this)->getAttrs(),
       /*elidedAttrs=*/{DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
+                       DoConcurrentLoopOp::getLocalSymsAttrName(),
+                       DoConcurrentLoopOp::getReduceSymsAttrName(),
                        DoConcurrentLoopOp::getReduceAttrsAttrName(),
-                       DoConcurrentLoopOp::getLocalSymsAttrName()});
+                       DoConcurrentLoopOp::getReduceByrefAttrName()});
 }
 
 llvm::SmallVector<mlir::Region *> fir::DoConcurrentLoopOp::getLoopRegions() {
@@ -5340,6 +5458,7 @@ llvm::LogicalResult fir::DoConcurrentLoopOp::verify() {
   mlir::Operation::operand_range ubValues = getUpperBound();
   mlir::Operation::operand_range stepValues = getStep();
   mlir::Operation::operand_range localVars = getLocalVars();
+  mlir::Operation::operand_range reduceVars = getReduceVars();
 
   if (lbValues.empty())
     return emitOpError(
@@ -5353,7 +5472,8 @@ llvm::LogicalResult fir::DoConcurrentLoopOp::verify() {
   // Check that the body defines the same number of block arguments as the
   // number of tuple elements in step.
   mlir::Block *body = getBody();
-  unsigned numIndVarArgs = body->getNumArguments() - localVars.size();
+  unsigned numIndVarArgs =
+      body->getNumArguments() - localVars.size() - reduceVars.size();
 
   if (numIndVarArgs != stepValues.size())
     return emitOpError() << "expects the same number of induction variables: "

diff  --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index e440852b3103a..506c8e66dbdfa 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -234,6 +234,10 @@ class DoConcurrentConversion
       loop.setLocalSymsAttr(nullptr);
     }
 
+    for (auto [reduceVar, reduceArg] :
+         llvm::zip_equal(loop.getReduceVars(), loop.getRegionReduceArgs()))
+      rewriter.replaceAllUsesWith(reduceArg, reduceVar);
+
     // Collect iteration variable(s) allocations so that we can move them
     // outside the `fir.do_concurrent` wrapper.
     llvm::SmallVector<mlir::Operation *> opsToMove;
@@ -257,12 +261,16 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceOperands(),
+          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
     }
 
+    loop.getRegion().front().eraseArguments(loop.getNumInductionVars() +
+                                                loop.getNumLocalOperands(),
+                                            loop.getNumReduceOperands());
+
     rewriter.inlineBlockBefore(
         &loopBlock, innermostUnorderdLoop.getBody()->getTerminator(), ivArgs);
     rewriter.eraseOp(doConcurentOp);

diff  --git a/flang/test/Fir/do_concurrent.fir b/flang/test/Fir/do_concurrent.fir
index cc1197ba56bd7..6e2173447855e 100644
--- a/flang/test/Fir/do_concurrent.fir
+++ b/flang/test/Fir/do_concurrent.fir
@@ -63,7 +63,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
     %j = fir.alloca i32
     fir.do_concurrent.loop
       (%i_iv, %j_iv) = (%i_lb, %j_lb) to (%i_ub, %j_ub) step (%i_st, %j_st) 
-      reduce(#fir.reduce_attr<add> -> %sum : !fir.ref<i32>) {
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %sum -> %sum_arg : !fir.ref<i32>) {
       %0 = fir.convert %i_iv : (index) -> i32
       fir.store %0 to %i : !fir.ref<i32>
 
@@ -83,7 +83,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
 // CHECK:           %[[I:.*]] = fir.alloca i32
 // CHECK:           %[[J:.*]] = fir.alloca i32
 // CHECK:           fir.do_concurrent.loop
-// CHECK-SAME:        (%[[I_IV:.*]], %[[J_IV:.*]]) = (%[[I_LB]], %[[J_LB]]) to (%[[I_UB]], %[[J_UB]]) step (%[[I_ST]], %[[J_ST]]) reduce(#fir.reduce_attr<add> -> %[[SUM]] : !fir.ref<i32>) {
+// CHECK-SAME:        (%[[I_IV:.*]], %[[J_IV:.*]]) = (%[[I_LB]], %[[J_LB]]) to (%[[I_UB]], %[[J_UB]]) step (%[[I_ST]], %[[J_ST]]) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[SUM]] -> %{{.*}} : !fir.ref<i32>) {
 // CHECK:             %[[I_IV_CVT:.*]] = fir.convert %[[I_IV]] : (index) -> i32
 // CHECK:             fir.store %[[I_IV_CVT]] to %[[I]] : !fir.ref<i32>
 // CHECK:             %[[J_IV_CVT:.*]] = fir.convert %[[J_IV]] : (index) -> i32
@@ -161,3 +161,62 @@ func.func @do_concurrent_with_locality_specs() {
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @dc_reduce() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c1 = arith.constant 1 : index 
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(byref @add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1 : !fir.ref<i32>) {
+    }
+  }
+  return
+}
+
+// CHECK-LABEL:  func.func @dc_reduce() {
+// CHECK:          %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+// CHECK:          %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:          fir.do_concurrent {
+// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) reduce(byref @add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
+// CHECK:            }
+// CHECK:          }
+// CHECK:          return
+// CHECK:        }
+
+func.func @dc_reduce_2() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  %5 = fir.alloca i32 {bindc_name = "m", uniq_name = "dc_reduce"}
+  %6:2 = hlfir.declare %5 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  %c1 = arith.constant 1 : index 
+
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) 
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %4#0 -> %arg1,
+             @mul_reduction_i32 #fir.reduce_attr<multiply> %6#0 -> %arg2
+             : !fir.ref<i32>, !fir.ref<i32>) {
+    }
+  }
+
+  return
+}
+
+// CHECK-LABEL:  func.func @dc_reduce_2() {
+// CHECK:          %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+// CHECK:          %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+// CHECK:          %[[M_ALLOC:.*]] = fir.alloca i32 {bindc_name = "m", uniq_name = "dc_reduce"}
+// CHECK:          %[[M_DECL:.*]]:2 = hlfir.declare %[[M_ALLOC]] {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:          fir.do_concurrent {
+// CHECK:            fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{[^[:space:]]+}}) 
+// CHECK-SAME:         reduce(
+// CHECK-SAME:           @add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:[^,]+]],
+// CHECK-SAME:           @mul_reduction_i32 #fir.reduce_attr<multiply> %[[M_DECL]]#0 -> %[[M_ARG:[^[:space:]]+]]
+// CHECK-SAME:           : !fir.ref<i32>, !fir.ref<i32>) {
+// CHECK:            }
+// CHECK:          }
+// CHECK:          return
+// CHECK:        }
+

diff  --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index aca0ecc1abdc1..e32ea7ad3c729 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -1256,8 +1256,8 @@ func.func @dc_invalid_reduction(%arg0: index, %arg1: index) {
   %sum = fir.alloca i32
   // expected-error at +2 {{'fir.do_concurrent.loop' op mismatch in number of reduction variables and reduction attributes}}
   fir.do_concurrent {
-    "fir.do_concurrent.loop"(%arg0, %arg1, %arg0, %sum) <{operandSegmentSizes = array<i32: 1, 1, 1, 1, 0>}> ({
-    ^bb0(%arg3: index):
+    "fir.do_concurrent.loop"(%arg0, %arg1, %arg0, %sum) <{operandSegmentSizes = array<i32: 1, 1, 1, 0, 1>}> ({
+    ^bb0(%arg3: index, %sum_arg: i32):
       %tmp = "fir.alloca"() <{in_type = i32, operandSegmentSizes = array<i32: 0, 0>}> : () -> !fir.ref<i32>
     }) : (index, index, index, !fir.ref<i32>) -> ()
   }
@@ -1266,6 +1266,20 @@ func.func @dc_invalid_reduction(%arg0: index, %arg1: index) {
 
 // -----
 
+func.func @dc_reduce_no_attr() {
+  %3 = fir.alloca i32 {bindc_name = "s", uniq_name = "dc_reduce"}
+  %4:2 = hlfir.declare %3 {uniq_name = "dc_reduce"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c1 = arith.constant 1 : index 
+  // expected-error at +2 {{expected attribute value}}
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) reduce(@add_reduction_i32 %4#0 -> %arg1 : !fir.ref<i32>) {
+    }
+  }
+  return
+}
+
+// -----
+
 // Should fail when volatility changes from a fir.convert
 func.func @bad_convert_volatile(%arg0: !fir.ref<i32>) -> !fir.ref<i32, volatile> {
   // expected-error at +1 {{op this conversion does not preserve volatility}}

diff  --git a/flang/test/Lower/do_concurrent_reduce.f90 b/flang/test/Lower/do_concurrent_reduce.f90
new file mode 100644
index 0000000000000..8591a21e2b9e0
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_reduce.f90
@@ -0,0 +1,41 @@
+! RUN: %flang_fc1 -emit-hlfir -mmlir --enable-delayed-privatization-staging=true -o - %s | FileCheck %s
+
+subroutine do_concurrent_reduce
+  implicit none
+  integer :: s, i
+
+  do concurrent (i=1:10) reduce(+:s)
+    s = s + 1
+  end do
+end
+
+! CHECK-LABEL:  fir.declare_reduction @add_reduction_i32 : i32 init {
+! CHECK:        ^bb0(%[[ARG0:.*]]: i32):
+! CHECK:          %[[VAL_0:.*]] = arith.constant 0 : i32
+! CHECK:          fir.yield(%[[VAL_0]] : i32)
+! CHECK:        } combiner {
+! CHECK:          ^bb0(%[[VAL_1:.*]]: i32, %[[VAL_2:.*]]: i32):
+! CHECK:          %[[VAL_3:.*]] = arith.addi %[[VAL_1]], %[[VAL_2]] : i32
+! CHECK:          fir.yield(%[[VAL_3]] : i32)
+! CHECK:        }
+
+! CHECK-LABEL:  func.func @_QPdo_concurrent_reduce() {
+! CHECK:           %[[S_ALLOC:.*]] = fir.alloca i32 {bindc_name = "s", uniq_name = "_QFdo_concurrent_reduceEs"}
+! CHECK:           %[[S_DECL:.*]]:2 = hlfir.declare %[[S_ALLOC]] {uniq_name = "_QFdo_concurrent_reduceEs"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+! CHECK:           fir.do_concurrent {
+! CHECK:             %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i"}
+! CHECK:             %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFdo_concurrent_reduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:             fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{[^[:space:]]+}}) 
+! CHECK-SAME:          reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[S_DECL]]#0 -> %[[S_ARG:.*]] : !fir.ref<i32>) {
+
+! CHECK:               %[[S_ARG_DECL:.*]]:2 = hlfir.declare %[[S_ARG]] {uniq_name = "_QFdo_concurrent_reduceEs"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:               %[[S_ARG_VAL:.*]] = fir.load %[[S_ARG_DECL]]#0 : !fir.ref<i32>
+! CHECK:               %[[C1:.*]] = arith.constant 1 : i32
+! CHECK:               %[[RED_UPDATE:.*]] = arith.addi %[[S_ARG_VAL]], %[[C1]] : i32
+! CHECK:               hlfir.assign %[[RED_UPDATE]] to %[[S_ARG_DECL]]#0 : i32, !fir.ref<i32>
+
+! CHECK:             }
+! CHECK:           }
+! CHECK:           return
+! CHECK:        }

diff  --git a/flang/test/Lower/loops.f90 b/flang/test/Lower/loops.f90
index 60df27a591dc3..64f14ff972272 100644
--- a/flang/test/Lower/loops.f90
+++ b/flang/test/Lower/loops.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
+! RUN: bbc -emit-fir -hlfir=false --enable-delayed-privatization=false -o - %s | FileCheck %s
 
 ! CHECK-LABEL: loop_test
 subroutine loop_test

diff  --git a/flang/test/Lower/loops3.f90 b/flang/test/Lower/loops3.f90
index 84db1972cca16..2965b954b49a8 100644
--- a/flang/test/Lower/loops3.f90
+++ b/flang/test/Lower/loops3.f90
@@ -12,7 +12,7 @@ subroutine loop_test
 
   ! CHECK: %[[VAL_0:.*]] = fir.alloca f32 {bindc_name = "m", uniq_name = "_QFloop_testEm"}
   ! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QFloop_testEsum) : !fir.ref<i32>
-  ! CHECK: fir.do_concurrent.loop ({{.*}}) = ({{.*}}) to ({{.*}}) step ({{.*}}) reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
+  ! CHECK: fir.do_concurrent.loop ({{.*}}) = ({{.*}}) to ({{.*}}) step ({{.*}}) reduce(@add_reduction_i32 #fir.reduce_attr<add> %[[VAL_1]] -> %{{.*}}, @other_reduction_f32 #fir.reduce_attr<max> %[[VAL_0]] -> %{{.*}} : {{.*}}) {
   do concurrent (i=1:5, j=1:5, k=1:5) local(tmp) reduce(+:sum) reduce(max:m)
     tmp = i + j + k
     sum = tmp + sum

diff  --git a/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir b/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
index d9ef36b175598..c550ab8a97d4c 100644
--- a/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
+++ b/flang/test/Transforms/do_concurrent-to-do_loop-unodered.fir
@@ -86,7 +86,7 @@ func.func @dc_2d_reduction(%i_lb: index, %i_ub: index, %i_st: index,
     %j = fir.alloca i32
     fir.do_concurrent.loop
       (%i_iv, %j_iv) = (%i_lb, %j_lb) to (%i_ub, %j_ub) step (%i_st, %j_st) 
-      reduce(#fir.reduce_attr<add> -> %sum : !fir.ref<i32>) {
+      reduce(@add_reduction_i32 #fir.reduce_attr<add> %sum -> %sum_arg : !fir.ref<i32>) {
       %0 = fir.convert %i_iv : (index) -> i32
       fir.store %0 to %i : !fir.ref<i32>
 


        

From flang-commits at lists.llvm.org  Thu Jul 10 22:42:54 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 22:42:54 -0700 (PDT)
Subject: [flang-commits] [flang] 7c8a197 - [NFC][flang] Move
 `ReductionProcessor` to `Lower/Support`. (#146025)
Message-ID: <6870a45e.170a0220.188a83.df75@mx.google.com>


Author: Kareem Ergawy
Date: 2025-07-11T07:42:51+02:00
New Revision: 7c8a197918a0c4044c1be39a26d517eea95a5ec9

URL: https://github.com/llvm/llvm-project/commit/7c8a197918a0c4044c1be39a26d517eea95a5ec9
DIFF: https://github.com/llvm/llvm-project/commit/7c8a197918a0c4044c1be39a26d517eea95a5ec9.diff

LOG: [NFC][flang] Move `ReductionProcessor` to `Lower/Support`. (#146025)

With #145837, the `ReductionProcessor` component is now used by both
OpenMP and `do concurrent`. Therefore, this PR moves it to a shared
location: `flang/Lower/Support`.

PR stack:
- https://github.com/llvm/llvm-project/pull/145837
- https://github.com/llvm/llvm-project/pull/146025 (this one)
- https://github.com/llvm/llvm-project/pull/146028
- https://github.com/llvm/llvm-project/pull/146033

Added: 
    flang/include/flang/Lower/OpenMP/Clauses.h
    flang/include/flang/Lower/Support/ReductionProcessor.h
    flang/lib/Lower/Support/ReductionProcessor.cpp

Modified: 
    flang/lib/Lower/Bridge.cpp
    flang/lib/Lower/CMakeLists.txt
    flang/lib/Lower/OpenMP/Atomic.cpp
    flang/lib/Lower/OpenMP/ClauseFinder.h
    flang/lib/Lower/OpenMP/ClauseProcessor.cpp
    flang/lib/Lower/OpenMP/ClauseProcessor.h
    flang/lib/Lower/OpenMP/Clauses.cpp
    flang/lib/Lower/OpenMP/DataSharingProcessor.h
    flang/lib/Lower/OpenMP/Decomposer.cpp
    flang/lib/Lower/OpenMP/Decomposer.h
    flang/lib/Lower/OpenMP/OpenMP.cpp
    flang/lib/Lower/OpenMP/Utils.cpp
    flang/lib/Lower/OpenMP/Utils.h

Removed: 
    flang/lib/Lower/OpenMP/Clauses.h
    flang/lib/Lower/OpenMP/ReductionProcessor.cpp
    flang/lib/Lower/OpenMP/ReductionProcessor.h


################################################################################
diff  --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/include/flang/Lower/OpenMP/Clauses.h
similarity index 100%
rename from flang/lib/Lower/OpenMP/Clauses.h
rename to flang/include/flang/Lower/OpenMP/Clauses.h

diff  --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/include/flang/Lower/Support/ReductionProcessor.h
similarity index 99%
rename from flang/lib/Lower/OpenMP/ReductionProcessor.h
rename to flang/include/flang/Lower/Support/ReductionProcessor.h
index 95b4b077bdc46..72d8a0096f511 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/include/flang/Lower/Support/ReductionProcessor.h
@@ -13,7 +13,7 @@
 #ifndef FORTRAN_LOWER_REDUCTIONPROCESSOR_H
 #define FORTRAN_LOWER_REDUCTIONPROCESSOR_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Parser/parse-tree.h"

diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index e2d3fe964d49b..33c1f1e7a3c3a 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -12,7 +12,6 @@
 
 #include "flang/Lower/Bridge.h"
 
-#include "OpenMP/ReductionProcessor.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -32,6 +31,7 @@
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/Runtime.h"
 #include "flang/Lower/StatementContext.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 #include "flang/Lower/Support/Utils.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/CUFCommon.h"

diff  --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 8049cdf333173..cd80aaf553869 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -29,11 +29,11 @@ add_flang_library(FortranLower
   OpenMP/DataSharingProcessor.cpp
   OpenMP/Decomposer.cpp
   OpenMP/OpenMP.cpp
-  OpenMP/ReductionProcessor.cpp
   OpenMP/Utils.cpp
   PFTBuilder.cpp
   Runtime.cpp
   Support/PrivateReductionUtils.cpp
+  Support/ReductionProcessor.cpp
   Support/Utils.cpp
   SymbolMap.cpp
   VectorSubscripts.cpp

diff  --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 2ab91b239a3cc..6ea331c370640 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -7,13 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "Atomic.h"
-#include "Clauses.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/traverse.h"
 #include "flang/Evaluate/type.h"
 #include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"

diff  --git a/flang/lib/Lower/OpenMP/ClauseFinder.h b/flang/lib/Lower/OpenMP/ClauseFinder.h
index 3b77f2ca1d4cb..af52585452833 100644
--- a/flang/lib/Lower/OpenMP/ClauseFinder.h
+++ b/flang/lib/Lower/OpenMP/ClauseFinder.h
@@ -12,7 +12,7 @@
 #ifndef FORTRAN_LOWER_CLAUSEFINDER_H
 #define FORTRAN_LOWER_CLAUSEFINDER_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 
 namespace Fortran {
 namespace lower {

diff  --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 5aebfc901e8ac..74087d42a8e6e 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -11,12 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClauseProcessor.h"
-#include "Clauses.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 
 #include "flang/Lower/ConvertExprToHLFIR.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/Frontend/OpenMP/OMP.h.inc"

diff  --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 46b749fb66c86..f8a1f7983b79b 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -13,11 +13,11 @@
 #define FORTRAN_LOWER_CLAUSEPROCESSOR_H
 
 #include "ClauseFinder.h"
-#include "Clauses.h"
 #include "Utils.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/Bridge.h"
 #include "flang/Lower/DirectivesCommon.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Parser/dump-parse-tree.h"
 #include "flang/Parser/parse-tree.h"

diff  --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index b599d69a36272..22a07219d3a50 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/expression.h"

diff  --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index fded04c839fb4..ee2fc70d2e673 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -12,9 +12,9 @@
 #ifndef FORTRAN_LOWER_DATASHARINGPROCESSOR_H
 #define FORTRAN_LOWER_DATASHARINGPROCESSOR_H
 
-#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/OpenMP.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"

diff  --git a/flang/lib/Lower/OpenMP/Decomposer.cpp b/flang/lib/Lower/OpenMP/Decomposer.cpp
index 251cba9204adc..9bfbf67bec88c 100644
--- a/flang/lib/Lower/OpenMP/Decomposer.cpp
+++ b/flang/lib/Lower/OpenMP/Decomposer.cpp
@@ -12,8 +12,8 @@
 
 #include "Decomposer.h"
 
-#include "Clauses.h"
 #include "Utils.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Tools/CrossToolHelpers.h"

diff  --git a/flang/lib/Lower/OpenMP/Decomposer.h b/flang/lib/Lower/OpenMP/Decomposer.h
index e3291b7c59e21..65492bd76280d 100644
--- a/flang/lib/Lower/OpenMP/Decomposer.h
+++ b/flang/lib/Lower/OpenMP/Decomposer.h
@@ -8,7 +8,7 @@
 #ifndef FORTRAN_LOWER_OPENMP_DECOMPOSER_H
 #define FORTRAN_LOWER_OPENMP_DECOMPOSER_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
 #include "llvm/Frontend/OpenMP/OMP.h"

diff  --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 65e852cbcc911..4458f62eea95a 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -14,7 +14,6 @@
 
 #include "Atomic.h"
 #include "ClauseProcessor.h"
-#include "Clauses.h"
 #include "DataSharingProcessor.h"
 #include "Decomposer.h"
 #include "Utils.h"
@@ -23,6 +22,7 @@
 #include "flang/Lower/ConvertExpr.h"
 #include "flang/Lower/ConvertVariable.h"
 #include "flang/Lower/DirectivesCommon.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/BoxValue.h"

diff  --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index c226c2558e7aa..2e53f01f1da6a 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -12,9 +12,8 @@
 
 #include "Utils.h"
 
-#include "Clauses.h"
-
 #include "ClauseFinder.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include <flang/Lower/AbstractConverter.h>
 #include <flang/Lower/ConvertType.h>
 #include <flang/Lower/DirectivesCommon.h>

diff  --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index a7eb2dc5ee664..1526bd4e90233 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_LOWER_OPENMPUTILS_H
 #define FORTRAN_LOWER_OPENMPUTILS_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Value.h"

diff  --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
similarity index 99%
rename from flang/lib/Lower/OpenMP/ReductionProcessor.cpp
rename to flang/lib/Lower/Support/ReductionProcessor.cpp
index d14fc1f7a52da..539d5cd37c2ea 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -10,11 +10,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ReductionProcessor.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 
-#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/ConvertType.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/Support/PrivateReductionUtils.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/Complex.h"


        

From flang-commits at lists.llvm.org  Thu Jul 10 22:42:57 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 10 Jul 2025 22:42:57 -0700 (PDT)
Subject: [flang-commits] [flang] [NFC][flang] Move `ReductionProcessor` to
 `Lower/Support`. (PR #146025)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146025@github.com>
Message-ID: <6870a461.050a0220.73cc3.d4ed@mx.google.com>


https://github.com/ergawy closed https://github.com/llvm/llvm-project/pull/146025

From flang-commits at lists.llvm.org  Thu Jul 10 22:43:00 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 10 Jul 2025 22:43:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir] Small clean-up in
 `fir_DoConcurrentLoopOp`'s defintion (PR #146028)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146028@github.com>
Message-ID: <6870a464.170a0220.1908b2.c4e8@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/146028

From flang-commits at lists.llvm.org  Thu Jul 10 22:43:34 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Thu, 10 Jul 2025 22:43:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir] Small clean-up in
 `fir_DoConcurrentLoopOp`'s defintion (PR #146028)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146028@github.com>
Message-ID: <6870a486.170a0220.1b0558.c013@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/146028

>From ab6353a0c9e2a874dc159e21f3ed5b76a48bf656 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Thu, 26 Jun 2025 23:30:04 -0500
Subject: [PATCH 1/3] [NFC][flang] Move `ReductionProcessor` to
 `Lower/Support`.

With #145837, the `ReductionProcessor` component is now used by both
OpenMP and `do concurrent`. Therefore, this PR moves it to a shared
location: `flang/Lower/Support`.
---
 flang/{lib => include/flang}/Lower/OpenMP/Clauses.h           | 0
 .../flang/Lower/Support}/ReductionProcessor.h                 | 2 +-
 flang/lib/Lower/Bridge.cpp                                    | 2 +-
 flang/lib/Lower/CMakeLists.txt                                | 2 +-
 flang/lib/Lower/OpenMP/ClauseFinder.h                         | 2 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp                    | 4 ++--
 flang/lib/Lower/OpenMP/ClauseProcessor.h                      | 2 +-
 flang/lib/Lower/OpenMP/Clauses.cpp                            | 2 +-
 flang/lib/Lower/OpenMP/DataSharingProcessor.h                 | 2 +-
 flang/lib/Lower/OpenMP/Decomposer.cpp                         | 2 +-
 flang/lib/Lower/OpenMP/Decomposer.h                           | 2 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp                             | 2 +-
 flang/lib/Lower/OpenMP/Utils.cpp                              | 3 +--
 flang/lib/Lower/OpenMP/Utils.h                                | 2 +-
 flang/lib/Lower/{OpenMP => Support}/ReductionProcessor.cpp    | 4 ++--
 15 files changed, 16 insertions(+), 17 deletions(-)
 rename flang/{lib => include/flang}/Lower/OpenMP/Clauses.h (100%)
 rename flang/{lib/Lower/OpenMP => include/flang/Lower/Support}/ReductionProcessor.h (99%)
 rename flang/lib/Lower/{OpenMP => Support}/ReductionProcessor.cpp (99%)

diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/include/flang/Lower/OpenMP/Clauses.h
similarity index 100%
rename from flang/lib/Lower/OpenMP/Clauses.h
rename to flang/include/flang/Lower/OpenMP/Clauses.h
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/include/flang/Lower/Support/ReductionProcessor.h
similarity index 99%
rename from flang/lib/Lower/OpenMP/ReductionProcessor.h
rename to flang/include/flang/Lower/Support/ReductionProcessor.h
index 95b4b077bdc46..72d8a0096f511 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/include/flang/Lower/Support/ReductionProcessor.h
@@ -13,7 +13,7 @@
 #ifndef FORTRAN_LOWER_REDUCTIONPROCESSOR_H
 #define FORTRAN_LOWER_REDUCTIONPROCESSOR_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Parser/parse-tree.h"
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index e2d3fe964d49b..33c1f1e7a3c3a 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -12,7 +12,6 @@
 
 #include "flang/Lower/Bridge.h"
 
-#include "OpenMP/ReductionProcessor.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -32,6 +31,7 @@
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/Runtime.h"
 #include "flang/Lower/StatementContext.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 #include "flang/Lower/Support/Utils.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/CUFCommon.h"
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 8049cdf333173..cd80aaf553869 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -29,11 +29,11 @@ add_flang_library(FortranLower
   OpenMP/DataSharingProcessor.cpp
   OpenMP/Decomposer.cpp
   OpenMP/OpenMP.cpp
-  OpenMP/ReductionProcessor.cpp
   OpenMP/Utils.cpp
   PFTBuilder.cpp
   Runtime.cpp
   Support/PrivateReductionUtils.cpp
+  Support/ReductionProcessor.cpp
   Support/Utils.cpp
   SymbolMap.cpp
   VectorSubscripts.cpp
diff --git a/flang/lib/Lower/OpenMP/ClauseFinder.h b/flang/lib/Lower/OpenMP/ClauseFinder.h
index 3b77f2ca1d4cb..af52585452833 100644
--- a/flang/lib/Lower/OpenMP/ClauseFinder.h
+++ b/flang/lib/Lower/OpenMP/ClauseFinder.h
@@ -12,7 +12,7 @@
 #ifndef FORTRAN_LOWER_CLAUSEFINDER_H
 #define FORTRAN_LOWER_CLAUSEFINDER_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 
 namespace Fortran {
 namespace lower {
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 5aebfc901e8ac..74087d42a8e6e 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -11,12 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClauseProcessor.h"
-#include "Clauses.h"
-#include "ReductionProcessor.h"
 #include "Utils.h"
 
 #include "flang/Lower/ConvertExprToHLFIR.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/Frontend/OpenMP/OMP.h.inc"
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 46b749fb66c86..f8a1f7983b79b 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -13,11 +13,11 @@
 #define FORTRAN_LOWER_CLAUSEPROCESSOR_H
 
 #include "ClauseFinder.h"
-#include "Clauses.h"
 #include "Utils.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/Bridge.h"
 #include "flang/Lower/DirectivesCommon.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Parser/dump-parse-tree.h"
 #include "flang/Parser/parse-tree.h"
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index b599d69a36272..22a07219d3a50 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/expression.h"
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index fded04c839fb4..ee2fc70d2e673 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -12,9 +12,9 @@
 #ifndef FORTRAN_LOWER_DATASHARINGPROCESSOR_H
 #define FORTRAN_LOWER_DATASHARINGPROCESSOR_H
 
-#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/OpenMP.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"
diff --git a/flang/lib/Lower/OpenMP/Decomposer.cpp b/flang/lib/Lower/OpenMP/Decomposer.cpp
index 251cba9204adc..9bfbf67bec88c 100644
--- a/flang/lib/Lower/OpenMP/Decomposer.cpp
+++ b/flang/lib/Lower/OpenMP/Decomposer.cpp
@@ -12,8 +12,8 @@
 
 #include "Decomposer.h"
 
-#include "Clauses.h"
 #include "Utils.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Tools/CrossToolHelpers.h"
diff --git a/flang/lib/Lower/OpenMP/Decomposer.h b/flang/lib/Lower/OpenMP/Decomposer.h
index e3291b7c59e21..65492bd76280d 100644
--- a/flang/lib/Lower/OpenMP/Decomposer.h
+++ b/flang/lib/Lower/OpenMP/Decomposer.h
@@ -8,7 +8,7 @@
 #ifndef FORTRAN_LOWER_OPENMP_DECOMPOSER_H
 #define FORTRAN_LOWER_OPENMP_DECOMPOSER_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 65e852cbcc911..4458f62eea95a 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -14,7 +14,6 @@
 
 #include "Atomic.h"
 #include "ClauseProcessor.h"
-#include "Clauses.h"
 #include "DataSharingProcessor.h"
 #include "Decomposer.h"
 #include "Utils.h"
@@ -23,6 +22,7 @@
 #include "flang/Lower/ConvertExpr.h"
 #include "flang/Lower/ConvertVariable.h"
 #include "flang/Lower/DirectivesCommon.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index c226c2558e7aa..2e53f01f1da6a 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -12,9 +12,8 @@
 
 #include "Utils.h"
 
-#include "Clauses.h"
-
 #include "ClauseFinder.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include <flang/Lower/AbstractConverter.h>
 #include <flang/Lower/ConvertType.h>
 #include <flang/Lower/DirectivesCommon.h>
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index a7eb2dc5ee664..1526bd4e90233 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_LOWER_OPENMPUTILS_H
 #define FORTRAN_LOWER_OPENMPUTILS_H
 
-#include "Clauses.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Value.h"
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
similarity index 99%
rename from flang/lib/Lower/OpenMP/ReductionProcessor.cpp
rename to flang/lib/Lower/Support/ReductionProcessor.cpp
index d14fc1f7a52da..539d5cd37c2ea 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -10,11 +10,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ReductionProcessor.h"
+#include "flang/Lower/Support/ReductionProcessor.h"
 
-#include "Clauses.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/ConvertType.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/Support/PrivateReductionUtils.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/Complex.h"

>From e9eb77f38bc2f4fa58d9b138fe66258c795cb69b Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Sun, 29 Jun 2025 23:35:38 -0500
Subject: [PATCH 2/3] fix include after rebase

---
 flang/lib/Lower/OpenMP/Atomic.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 2ab91b239a3cc..6ea331c370640 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -7,13 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "Atomic.h"
-#include "Clauses.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/traverse.h"
 #include "flang/Evaluate/type.h"
 #include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/OpenMP/Clauses.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"

>From 8e67de7800c02e7d93fadabc92a91618dda0d715 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Fri, 27 Jun 2025 00:05:42 -0500
Subject: [PATCH 3/3] [flang][fir] Small clean-up in `fir_DoConcurrentLoopOp`'s
 defintion

Re-organizes the op definition a little bit and removes a method that
does not add much value to the API.
---
 .../include/flang/Optimizer/Dialect/FIROps.td | 22 +++++++++----------
 .../OpenMP/DoConcurrentConversion.cpp         |  4 ++--
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index c3d3582a50e7f..e3f5c4403002a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3878,9 +3878,17 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
   let hasVerifier = 1;
 
   let extraClassDeclaration = [{
-    unsigned getNumInductionVars() { return getLowerBound().size(); }
+    unsigned getNumInductionVars() {
+      return getLowerBound().size();
+    }
 
-    unsigned getNumLocalOperands() { return getLocalVars().size(); }
+    unsigned getNumLocalOperands() {
+      return getLocalVars().size();
+    }
+
+    unsigned getNumReduceOperands() {
+      return getReduceVars().size();
+    }
 
     mlir::Block::BlockArgListType getInductionVars() {
       return getBody()->getArguments().slice(0, getNumInductionVars());
@@ -3900,16 +3908,6 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     /// Number of operands controlling the loop
     unsigned getNumControlOperands() { return getLowerBound().size() * 3; }
 
-    // Get Number of reduction operands
-    unsigned getNumReduceOperands() {
-      return getReduceVars().size();
-    }
-
-    mlir::Operation::operand_range getLocalOperands() {
-      return getOperands()
-          .slice(getNumControlOperands() + getNumReduceOperands(),
-                 getNumLocalOperands());
-    }
   }];
 }
 
diff --git a/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp b/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
index 28f6c8bf02813..709cf1d0938fa 100644
--- a/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
+++ b/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
@@ -314,9 +314,9 @@ class DoConcurrentConversion
 
     // For `local` (and `local_init`) opernads, emit corresponding `private`
     // clauses and attach these clauses to the workshare loop.
-    if (!loop.getLocalOperands().empty())
+    if (!loop.getLocalVars().empty())
       for (auto [op, sym, arg] : llvm::zip_equal(
-               loop.getLocalOperands(),
+               loop.getLocalVars(),
                loop.getLocalSymsAttr().getAsRange<mlir::SymbolRefAttr>(),
                loop.getRegionLocalArgs())) {
         auto localizer = mlir::SymbolTable::lookupNearestSymbolFrom<


From flang-commits at lists.llvm.org  Thu Jul 10 23:30:40 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 10 Jul 2025 23:30:40 -0700 (PDT)
Subject: [flang-commits] [flang] a510e75 - [flang][fir] Small clean-up in
 `fir_DoConcurrentLoopOp`'s defintion (#146028)
Message-ID: <6870af90.050a0220.fcbd0.d0b6@mx.google.com>


Author: Kareem Ergawy
Date: 2025-07-11T08:30:36+02:00
New Revision: a510e75949a4efa4305ba4c6276c9098693d465e

URL: https://github.com/llvm/llvm-project/commit/a510e75949a4efa4305ba4c6276c9098693d465e
DIFF: https://github.com/llvm/llvm-project/commit/a510e75949a4efa4305ba4c6276c9098693d465e.diff

LOG: [flang][fir] Small clean-up in `fir_DoConcurrentLoopOp`'s defintion (#146028)

Re-organizes the op definition a little bit and removes a method that
does not add much value to the API.

PR stack:
- https://github.com/llvm/llvm-project/pull/145837
- https://github.com/llvm/llvm-project/pull/146025
- https://github.com/llvm/llvm-project/pull/146028 (this one)
- https://github.com/llvm/llvm-project/pull/146033

Added: 
    

Modified: 
    flang/include/flang/Optimizer/Dialect/FIROps.td
    flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index c3d3582a50e7f..e3f5c4403002a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3878,9 +3878,17 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
   let hasVerifier = 1;
 
   let extraClassDeclaration = [{
-    unsigned getNumInductionVars() { return getLowerBound().size(); }
+    unsigned getNumInductionVars() {
+      return getLowerBound().size();
+    }
 
-    unsigned getNumLocalOperands() { return getLocalVars().size(); }
+    unsigned getNumLocalOperands() {
+      return getLocalVars().size();
+    }
+
+    unsigned getNumReduceOperands() {
+      return getReduceVars().size();
+    }
 
     mlir::Block::BlockArgListType getInductionVars() {
       return getBody()->getArguments().slice(0, getNumInductionVars());
@@ -3900,16 +3908,6 @@ def fir_DoConcurrentLoopOp : fir_Op<"do_concurrent.loop",
     /// Number of operands controlling the loop
     unsigned getNumControlOperands() { return getLowerBound().size() * 3; }
 
-    // Get Number of reduction operands
-    unsigned getNumReduceOperands() {
-      return getReduceVars().size();
-    }
-
-    mlir::Operation::operand_range getLocalOperands() {
-      return getOperands()
-          .slice(getNumControlOperands() + getNumReduceOperands(),
-                 getNumLocalOperands());
-    }
   }];
 }
 

diff  --git a/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp b/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
index 28f6c8bf02813..709cf1d0938fa 100644
--- a/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
+++ b/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
@@ -314,9 +314,9 @@ class DoConcurrentConversion
 
     // For `local` (and `local_init`) opernads, emit corresponding `private`
     // clauses and attach these clauses to the workshare loop.
-    if (!loop.getLocalOperands().empty())
+    if (!loop.getLocalVars().empty())
       for (auto [op, sym, arg] : llvm::zip_equal(
-               loop.getLocalOperands(),
+               loop.getLocalVars(),
                loop.getLocalSymsAttr().getAsRange<mlir::SymbolRefAttr>(),
                loop.getRegionLocalArgs())) {
         auto localizer = mlir::SymbolTable::lookupNearestSymbolFrom<


        

From flang-commits at lists.llvm.org  Fri Jul 11 01:05:04 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Fri, 11 Jul 2025 01:05:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <6870c5b0.050a0220.1be058.d47b@mx.google.com>


================
@@ -2234,31 +2253,47 @@ void OmpAttributeVisitor::Post(const parser::OpenMPExecutableAllocate &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
-    if (const auto *alloc{
-            std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
-      CheckAllNamesInAllocateStmt(dir.source,
-          std::get<parser::OmpObjectList>(alloc->v.t),
-          std::get<parser::Statement<parser::AllocateStmt>>(x.t).statement);
-
-      auto &modifiers{OmpGetModifiers(alloc->v)};
-      bool hasAllocator{
-          OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(modifiers) ||
-          OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(modifiers)};
-
-      // TODO: As with allocate directive, exclude the case when a requires
-      //       directive with the dynamic_allocators clause is present in
-      //       the same compilation unit (OMP5.0 2.11.3).
-      if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
-          !hasAllocator) {
-        context_.Say(x.source,
-            "ALLOCATORS directives that appear in a TARGET region "
-            "must specify an allocator"_err_en_US);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const parser::AllocateStmt *allocate{[&]() {
+    if (action) {
+      if (auto *alloc{std::get_if<common::Indirection<parser::AllocateStmt>>(
+              &action.stmt->u)}) {
+        return &alloc->value();
+      }
+    }
+    return static_cast<const parser::AllocateStmt *>(nullptr);
+  }()};
+
+  if (allocate) {
+    for (const auto &clause : dirSpec.Clauses().v) {
+      if (auto *alloc{std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
+        CheckAllNamesInAllocateStmt(
+            x.source, std::get<parser::OmpObjectList>(alloc->v.t), *allocate);
+
+        using OmpAllocatorSimpleModifier = parser::OmpAllocatorSimpleModifier;
+        using OmpAllocatorComplexModifier = parser::OmpAllocatorComplexModifier;
+
+        auto &modifiers{OmpGetModifiers(alloc->v)};
+        bool hasAllocator{
+            OmpGetUniqueModifier<OmpAllocatorSimpleModifier>(modifiers) ||
+            OmpGetUniqueModifier<OmpAllocatorComplexModifier>(modifiers)};
+
+        // TODO: As with allocate directive, exclude the case when a requires
+        //       directive with the dynamic_allocators clause is present in
+        //       the same compilation unit (OMP5.0 2.11.3).
+        if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
+            !hasAllocator) {
+          context_.Say(x.source,
+              "ALLOCATORS directives that appear in a TARGET region "
+              "must specify an allocator"_err_en_US);
+        }
       }
     }
   }
+
----------------
Stylie777 wrote:

extra whitespace

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Fri Jul 11 01:05:04 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Fri, 11 Jul 2025 01:05:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <6870c5b0.170a0220.69bdc.ce3c@mx.google.com>


================
@@ -372,6 +367,29 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     GetContext().withinConstruct = true;
   }
 
+  bool Pre(const parser::OpenMPStandaloneConstruct &x) {
+    common::visit(
+        [&](auto &&s) {
+          using TypeS = llvm::remove_cvref_t<decltype(s)>;
+          // These two cases are handled individually.
+          if constexpr ( //
----------------
Stylie777 wrote:

nit: Same here

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Fri Jul 11 01:05:05 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Fri, 11 Jul 2025 01:05:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <6870c5b1.170a0220.28c1fe.ca7b@mx.google.com>


================
@@ -1276,6 +1276,32 @@ struct OmpEndDirectiveParser {
   llvm::omp::Directive dir_;
 };
 
+struct OmpAllocatorsConstructParser {
+  using resultType = OpenMPAllocatorsConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_allocators) {
+      return std::nullopt;
+    }
+
+    // This should be an allocate-stmt. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_allocators}};
+    return OpenMPAllocatorsConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
----------------
Stylie777 wrote:

nit: Do we need the `\\` here?

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Fri Jul 11 01:12:51 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Fri, 11 Jul 2025 01:12:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <6870c783.170a0220.1ce682.cb06@mx.google.com>


https://github.com/Stylie777 approved this pull request.

LGTM. Few little nit comments but not critical.

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Fri Jul 11 01:17:22 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 11 Jul 2025 01:17:22 -0700 (PDT)
Subject: [flang-commits] [flang] ab1c490 - [flang][do concurrent] Extned
 `getAllocaBlock()` and emit yields correctly (#146853)
Message-ID: <6870c892.170a0220.195517.cb3b@mx.google.com>


Author: Kareem Ergawy
Date: 2025-07-11T10:17:17+02:00
New Revision: ab1c4905f4dc3bdeb05bff1f3de24f73d6d612e4

URL: https://github.com/llvm/llvm-project/commit/ab1c4905f4dc3bdeb05bff1f3de24f73d6d612e4
DIFF: https://github.com/llvm/llvm-project/commit/ab1c4905f4dc3bdeb05bff1f3de24f73d6d612e4.diff

LOG: [flang][do concurrent] Extned `getAllocaBlock()` and emit yields correctly (#146853)

Handles some loose ends in `do concurrent` reduction declarations. This
PR extends `getAllocaBlock` to handle declare ops, and also emit
`fir.yield` in all regions.

Added: 
    flang/test/HLFIR/fir-reduction-alloca-block.fir
    flang/test/Lower/do_concurrent_reduce_allocatable.f90

Modified: 
    flang/lib/Lower/Support/ReductionProcessor.cpp
    flang/lib/Optimizer/Builder/FIRBuilder.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 539d5cd37c2ea..14b2c9836748f 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -529,7 +529,9 @@ static void createReductionAllocAndInitRegions(
         converter, loc, type, initValue, initBlock,
         reductionDecl.getInitializerAllocArg(),
         reductionDecl.getInitializerMoldArg(), reductionDecl.getCleanupRegion(),
-        DeclOperationKind::Reduction);
+        DeclOperationKind::Reduction, /*sym=*/nullptr,
+        /*cannotHaveLowerBounds=*/false,
+        /*isDoConcurrent*/ std::is_same_v<OpType, fir::DeclareReductionOp>);
   }
 
   if (fir::isa_trivial(ty)) {

diff  --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index b5cabdb830e5c..acd5a88a2582d 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -286,6 +286,9 @@ mlir::Block *fir::FirOpBuilder::getAllocaBlock() {
   if (auto firLocalOp = getRegion().getParentOfType<fir::LocalitySpecifierOp>())
     return &getRegion().front();
 
+  if (auto firLocalOp = getRegion().getParentOfType<fir::DeclareReductionOp>())
+    return &getRegion().front();
+
   return getEntryBlock();
 }
 

diff  --git a/flang/test/HLFIR/fir-reduction-alloca-block.fir b/flang/test/HLFIR/fir-reduction-alloca-block.fir
new file mode 100644
index 0000000000000..75857cfbe01d3
--- /dev/null
+++ b/flang/test/HLFIR/fir-reduction-alloca-block.fir
@@ -0,0 +1,31 @@
+// Tests that `fir.local` ops are able to provide an alloca block when required.
+
+// RUN: fir-opt %s -convert-hlfir-to-fir | FileCheck %s
+
+fir.declare_reduction @add_reduction_byref_box_heap_UxUxf32 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> alloc {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+  fir.yield(%0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>)
+} init {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %arg1: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>):
+  %cst = arith.constant 0.000000e+00 : f32
+  %0 = fir.load %arg1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  hlfir.assign %cst to %0 : f32, !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+  fir.yield(%arg1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>)
+} combiner {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %arg1: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>):
+  fir.yield(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>)
+}
+
+// CHECK-LABEL:   fir.declare_reduction @add_reduction_byref_box_heap_UxUxf32 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> alloc {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+// CHECK:           fir.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>)
+
+// CHECK-LABEL:   } init {
+// CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>):
+// CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+// CHECK:           fir.yield(%[[VAL_1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>)
+
+// CHECK-LABEL:   } combiner {
+// CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>):
+// CHECK:           fir.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>)
+// CHECK:         }

diff  --git a/flang/test/Lower/do_concurrent_reduce_allocatable.f90 b/flang/test/Lower/do_concurrent_reduce_allocatable.f90
new file mode 100644
index 0000000000000..873fd10dd1b97
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_reduce_allocatable.f90
@@ -0,0 +1,22 @@
+! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck %s
+
+subroutine do_concurrent_allocatable
+  integer :: i
+  real, allocatable, dimension(:,:) :: x
+
+  do concurrent (i = 1:10) reduce(+: x)
+  end do
+end subroutine
+
+! CHECK: fir.declare_reduction @[[RED_OP:.*]] : ![[RED_TYPE:.*]] alloc {
+! CHECK:   %[[ALLOC:.*]] = fir.alloca
+! CHECK:   fir.yield(%[[ALLOC]] : ![[RED_TYPE]])
+! CHECK: } init {
+! CHECK: ^bb0(%{{.*}}: ![[RED_TYPE]], %[[RED_ARG:.*]]: ![[RED_TYPE]]):
+! CHECK:   fir.yield(%[[RED_ARG]] : !{{.*}})
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[COMB_RES:.*]]: ![[RED_TYPE]], %{{.*}}: ![[RED_TYPE]]):
+! CHECK:   fir.yield(%[[COMB_RES]] : !{{.*}})
+! CHECK: } cleanup {
+! CHECK:   fir.yield
+! CHECK: }


        

From flang-commits at lists.llvm.org  Fri Jul 11 01:17:24 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Fri, 11 Jul 2025 01:17:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][do concurrent] Extned
 `getAllocaBlock()` and emit yields correctly (PR #146853)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146853@github.com>
Message-ID: <6870c894.050a0220.84ed9.e16f@mx.google.com>


https://github.com/ergawy closed https://github.com/llvm/llvm-project/pull/146853

From flang-commits at lists.llvm.org  Fri Jul 11 03:28:07 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 03:28:07 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <6870e737.630a0220.29c5fa.d6d3@mx.google.com>


tblah wrote:

The intention of the rule is to allow people to run fortran programs on systems that do not have C++ standard libraries installed. A header-only dependency on these LLVM functions should be okay because they don't call anything else. Similarly, header-only dependencies on libc++ are okay (like the use of std::numeric_limits here) because that won't create a runtime dependency between flang-rt and the c++ library.

But now that you mention it, I agree this is a less brittle approach so that we don't accidentally end up with runtime dependencies on libc++ because of a change in an llvm header.

However, currently I am getting "multiple definitions of Fortran::common::AddOverflow" when linking flang-rt with your change. I am linking using /usr/bin/ld, building with gcc, and `-DBUILD_SHARED_LIBS=ON`. My build did succeed with clang, lld and no shared libs.

https://github.com/llvm/llvm-project/pull/148169

From flang-commits at lists.llvm.org  Fri Jul 11 03:54:34 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Fri, 11 Jul 2025 03:54:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <6870ed6a.170a0220.12fafe.c818@mx.google.com>


================
@@ -1276,6 +1276,32 @@ struct OmpEndDirectiveParser {
   llvm::omp::Directive dir_;
 };
 
+struct OmpAllocatorsConstructParser {
+  using resultType = OpenMPAllocatorsConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_allocators) {
+      return std::nullopt;
+    }
+
+    // This should be an allocate-stmt. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_allocators}};
+    return OpenMPAllocatorsConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
----------------
kparzysz wrote:

I use those occasionally to keep clang-format from uglifying the code.

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Fri Jul 11 03:54:37 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Fri, 11 Jul 2025 03:54:37 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang] Implement !$omp unroll using
 omp.unroll_heuristic (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <6870ed6d.630a0220.1a4bae.d42d@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Fri Jul 11 04:02:53 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Fri, 11 Jul 2025 04:02:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <6870ef5d.170a0220.367dcf.d167@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/148005

>From 0f4462c45fd80c5417a0970919e600b08ea6e2fa Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Thu, 10 Jul 2025 07:55:32 -0500
Subject: [PATCH 1/2] [flang][OpenMP] Convert AST node for ALLOCATORS to use
 Block as body

The ALLOCATORS construct is one of the few constructs that require a
special form of the associated block.
Convert the AST node to use OmpDirectiveSpecification for the directive
and the optional end directive, and to use parser::Block as the body:
the form of the block is checked in the semantic checks (with a more
meaningful message).
---
 flang/examples/FeatureList/FeatureList.cpp    |  1 -
 flang/include/flang/Parser/dump-parse-tree.h  |  1 -
 flang/include/flang/Parser/parse-tree.h       | 21 ++--
 flang/lib/Parser/openmp-parsers.cpp           | 36 +++++--
 flang/lib/Parser/unparse.cpp                  | 24 ++---
 flang/lib/Semantics/check-omp-atomic.cpp      | 31 ------
 flang/lib/Semantics/check-omp-structure.cpp   | 46 ++++++---
 flang/lib/Semantics/openmp-utils.cpp          | 20 ++++
 flang/lib/Semantics/openmp-utils.h            | 13 +++
 flang/lib/Semantics/resolve-directives.cpp    | 97 +++++++++++++------
 .../test/Parser/OpenMP/allocators-unparse.f90 | 80 ++++++++-------
 flang/test/Semantics/OpenMP/allocators07.f90  | 27 ++++++
 12 files changed, 253 insertions(+), 144 deletions(-)
 create mode 100644 flang/test/Semantics/OpenMP/allocators07.f90

diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index a36b8719e365d..e9aeed18ab0b7 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -472,7 +472,6 @@ struct NodeVisitor {
   READ_FEATURE(OmpIteration)
   READ_FEATURE(OmpIterationOffset)
   READ_FEATURE(OmpIterationVector)
-  READ_FEATURE(OmpEndAllocators)
   READ_FEATURE(OmpEndBlockDirective)
   READ_FEATURE(OmpEndCriticalDirective)
   READ_FEATURE(OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..73c224e3ad235 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -578,7 +578,6 @@ class ParseTreeDumper {
   NODE(parser, OmpDetachClause)
   NODE(parser, OmpDoacrossClause)
   NODE(parser, OmpDestroyClause)
-  NODE(parser, OmpEndAllocators)
   NODE(parser, OmpEndBlockDirective)
   NODE(parser, OmpEndCriticalDirective)
   NODE(parser, OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 43954ff735361..fbc39286a95bf 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4598,8 +4598,11 @@ struct OmpClauseList {
 struct OmpDirectiveSpecification {
   ENUM_CLASS(Flags, None, DeprecatedSyntax);
   TUPLE_CLASS_BOILERPLATE(OmpDirectiveSpecification);
+  const OmpDirectiveName &DirName() const {
+    return std::get<OmpDirectiveName>(t);
+  }
   llvm::omp::Directive DirId() const { //
-    return std::get<OmpDirectiveName>(t).v;
+    return DirName().v;
   }
   const OmpArgumentList &Arguments() const;
   const OmpClauseList &Clauses() const;
@@ -4839,17 +4842,17 @@ struct OpenMPExecutableAllocate {
       t;
 };
 
-EMPTY_CLASS(OmpEndAllocators);
-
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
+// Ref: [5.2:180-181], [6.0:315]
+//
+// allocators-construct ->
+//    ALLOCATORS [allocate-clause...]
+//    block
+//    [END ALLOCATORS]
 struct OpenMPAllocatorsConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPAllocatorsConstruct);
   CharBlock source;
-  std::tuple<Verbatim, OmpClauseList, Statement<AllocateStmt>,
-      std::optional<OmpEndAllocators>>
+  std::tuple<OmpDirectiveSpecification, Block,
+      std::optional<OmpDirectiveSpecification>>
       t;
 };
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 3016ce4ccd2f8..811ca2c855a6e 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1276,6 +1276,32 @@ struct OmpEndDirectiveParser {
   llvm::omp::Directive dir_;
 };
 
+struct OmpAllocatorsConstructParser {
+  using resultType = OpenMPAllocatorsConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_allocators) {
+      return std::nullopt;
+    }
+
+    // This should be an allocate-stmt. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_allocators}};
+    return OpenMPAllocatorsConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
+    construct<OpenMPAllocatorsConstruct>(
+        "ALLOCATORS"_tok >= OmpAllocatorsConstructParser{})))
+
 // Parser for an arbitrary OpenMP ATOMIC construct.
 //
 // Depending on circumstances, an ATOMIC construct applies to one or more
@@ -1622,16 +1648,6 @@ TYPE_PARSER(
         maybe(nonemptyList(Parser<OpenMPDeclarativeAllocate>{})) / endOmpLine,
         statement(allocateStmt))))
 
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
-TYPE_PARSER(sourced(construct<OpenMPAllocatorsConstruct>(
-    verbatim("ALLOCATORS"_tok), Parser<OmpClauseList>{} / endOmpLine,
-    statement(allocateStmt), maybe(Parser<OmpEndAllocators>{} / endOmpLine))))
-
-TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
-
 // 2.8.2 Declare Simd construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareSimdConstruct>(
     verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index dcd1ac165adc1..4f692f3e9084f 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2571,7 +2571,7 @@ class UnparseVisitor {
     Word(ToUpperCaseLetters(common::EnumToString(x)));
   }
 
-  void Unparse(const OpenMPAtomicConstruct &x) {
+  template <typename Construct> void UnparseBlockConstruct(const Construct &x) {
     BeginOpenMP();
     Word("!$OMP ");
     Walk(std::get<OmpDirectiveSpecification>(x.t));
@@ -2587,6 +2587,10 @@ class UnparseVisitor {
     }
   }
 
+  void Unparse(const OpenMPAtomicConstruct &x) { //
+    UnparseBlockConstruct(x);
+  }
+
   void Unparse(const OpenMPExecutableAllocate &x) {
     const auto &fields =
         std::get<std::optional<std::list<parser::OpenMPDeclarativeAllocate>>>(
@@ -2614,22 +2618,8 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
-  void Unparse(const OmpEndAllocators &x) {
-    BeginOpenMP();
-    Word("!$OMP END ALLOCATE");
-    Put("\n");
-    EndOpenMP();
-  }
-  void Unparse(const OpenMPAllocatorsConstruct &x) {
-    BeginOpenMP();
-    Word("!$OMP ALLOCATE");
-    Walk(std::get<OmpClauseList>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<Statement<AllocateStmt>>(x.t));
-    if (const auto &end = std::get<std::optional<OmpEndAllocators>>(x.t)) {
-      Walk(*end);
-    }
+  void Unparse(const OpenMPAllocatorsConstruct &x) { //
+    UnparseBlockConstruct(x);
   }
   void Unparse(const OmpAssumeDirective &x) {
     BeginOpenMP();
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index 047c604693460..c5ed8796f0c34 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -47,43 +47,12 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
   return !(e == f);
 }
 
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
 struct AnalyzedCondStmt {
   SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
   parser::CharBlock source;
   SourcedActionStmt ift, iff;
 };
 
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
 // Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
 // is that the ActionStmt will be either an assignment or a pointer-assignment,
 // otherwise return std::nullopt.
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 89c1565bf66aa..9b89d19a91f38 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -495,6 +495,12 @@ template <typename Checker> struct DirectiveSpellingVisitor {
   template <typename T> bool Pre(const T &) { return true; }
   template <typename T> void Post(const T &) {}
 
+  template <typename... Ts>
+  static const parser::OmpDirectiveName &GetDirName(
+      const std::tuple<Ts...> &t) {
+    return std::get<parser::OmpDirectiveSpecification>(t).DirName();
+  }
+
   bool Pre(const parser::OmpSectionsDirective &x) {
     checker_(x.source, x.v);
     return false;
@@ -520,8 +526,7 @@ template <typename Checker> struct DirectiveSpellingVisitor {
     return false;
   }
   bool Pre(const parser::OpenMPAllocatorsConstruct &x) {
-    checker_(
-        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocators);
+    checker_(GetDirName(x.t).source, Directive::OMPD_allocators);
     return false;
   }
   bool Pre(const parser::OmpAssumeDirective &x) {
@@ -1657,26 +1662,45 @@ void OmpStructureChecker::Leave(const parser::OpenMPExecutableAllocate &x) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPAllocatorsConstruct &x) {
   isPredefinedAllocator = true;
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+  PushContextAndClauseSets(
+      dirSpec.DirName().source, llvm::omp::Directive::OMPD_allocators);
+
+  if (block.empty()) {
+    context_.Say(dirSpec.source,
+        "The ALLOCATORS construct should contain a single ALLOCATE statement"_err_en_US);
+    return;
+  }
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const auto *allocate{
+      action ? parser::Unwrap<parser::AllocateStmt>(action.stmt) : nullptr};
+
+  if (!allocate) {
+    const parser::CharBlock &source = action ? action.source : x.source;
+    context_.Say(source,
+        "The body of the ALLOCATORS construct should be an ALLOCATE statement"_err_en_US);
+  }
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             parser::Unwrap<parser::OmpClause::Allocate>(clause)}) {
       CheckVarIsNotPartOfAnotherVar(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       CheckPredefinedAllocatorRestriction(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
   dirContext_.pop_back();
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index fd9596a09cd52..f43d2cc75620e 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -38,6 +38,26 @@
 
 namespace Fortran::semantics::omp {
 
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
 std::string ThisVersion(unsigned version) {
   std::string tv{
       std::to_string(version / 10) + "." + std::to_string(version % 10)};
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index dbb0565215357..a96c008fb26e7 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -29,6 +29,19 @@ class Symbol;
 
 // Add this namespace to avoid potential conflicts
 namespace omp {
+// There is no consistent way to get the source of an ActionStmt, but there
+// is "source" in Statement<T>. This structure keeps the ActionStmt with the
+// extracted source for further use.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x);
+SourcedActionStmt GetActionStmt(const parser::Block &block);
+
 std::string ThisVersion(unsigned version);
 std::string TryVersion(unsigned version);
 
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 299bb6ff876e7..bb3914ffa2b1e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -10,6 +10,7 @@
 
 #include "check-acc-structure.h"
 #include "check-omp-structure.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/fold.h"
@@ -353,12 +354,6 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return true;
   }
 
-  bool Pre(const parser::OmpDirectiveSpecification &x) {
-    PushContext(x.source, x.DirId());
-    return true;
-  }
-  void Post(const parser::OmpDirectiveSpecification &) { PopContext(); }
-
   bool Pre(const parser::OmpMetadirectiveDirective &x) {
     PushContext(x.source, llvm::omp::Directive::OMPD_metadirective);
     return true;
@@ -372,6 +367,29 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     GetContext().withinConstruct = true;
   }
 
+  bool Pre(const parser::OpenMPStandaloneConstruct &x) {
+    common::visit(
+        [&](auto &&s) {
+          using TypeS = llvm::remove_cvref_t<decltype(s)>;
+          // These two cases are handled individually.
+          if constexpr ( //
+              !std::is_same_v<TypeS, parser::OpenMPSimpleStandaloneConstruct> &&
+              !std::is_same_v<TypeS, parser::OmpMetadirectiveDirective>) {
+            PushContext(x.source, s.v.DirId());
+          }
+        },
+        x.u);
+    return true;
+  }
+
+  void Post(const parser::OpenMPStandaloneConstruct &x) {
+    // These two cases are handled individually.
+    if (!std::holds_alternative<parser::OpenMPSimpleStandaloneConstruct>(x.u) &&
+        !std::holds_alternative<parser::OmpMetadirectiveDirective>(x.u)) {
+      PopContext();
+    }
+  }
+
   bool Pre(const parser::OpenMPSimpleStandaloneConstruct &);
   void Post(const parser::OpenMPSimpleStandaloneConstruct &) { PopContext(); }
 
@@ -2149,9 +2167,10 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPExecutableAllocate &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPAllocatorsConstruct &x) {
-  PushContext(x.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  PushContext(x.source, dirSpec.DirId());
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       ResolveOmpObjectList(std::get<parser::OmpObjectList>(allocClause->v.t),
@@ -2234,31 +2253,47 @@ void OmpAttributeVisitor::Post(const parser::OpenMPExecutableAllocate &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
-    if (const auto *alloc{
-            std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
-      CheckAllNamesInAllocateStmt(dir.source,
-          std::get<parser::OmpObjectList>(alloc->v.t),
-          std::get<parser::Statement<parser::AllocateStmt>>(x.t).statement);
-
-      auto &modifiers{OmpGetModifiers(alloc->v)};
-      bool hasAllocator{
-          OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(modifiers) ||
-          OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(modifiers)};
-
-      // TODO: As with allocate directive, exclude the case when a requires
-      //       directive with the dynamic_allocators clause is present in
-      //       the same compilation unit (OMP5.0 2.11.3).
-      if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
-          !hasAllocator) {
-        context_.Say(x.source,
-            "ALLOCATORS directives that appear in a TARGET region "
-            "must specify an allocator"_err_en_US);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const parser::AllocateStmt *allocate{[&]() {
+    if (action) {
+      if (auto *alloc{std::get_if<common::Indirection<parser::AllocateStmt>>(
+              &action.stmt->u)}) {
+        return &alloc->value();
+      }
+    }
+    return static_cast<const parser::AllocateStmt *>(nullptr);
+  }()};
+
+  if (allocate) {
+    for (const auto &clause : dirSpec.Clauses().v) {
+      if (auto *alloc{std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
+        CheckAllNamesInAllocateStmt(
+            x.source, std::get<parser::OmpObjectList>(alloc->v.t), *allocate);
+
+        using OmpAllocatorSimpleModifier = parser::OmpAllocatorSimpleModifier;
+        using OmpAllocatorComplexModifier = parser::OmpAllocatorComplexModifier;
+
+        auto &modifiers{OmpGetModifiers(alloc->v)};
+        bool hasAllocator{
+            OmpGetUniqueModifier<OmpAllocatorSimpleModifier>(modifiers) ||
+            OmpGetUniqueModifier<OmpAllocatorComplexModifier>(modifiers)};
+
+        // TODO: As with allocate directive, exclude the case when a requires
+        //       directive with the dynamic_allocators clause is present in
+        //       the same compilation unit (OMP5.0 2.11.3).
+        if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
+            !hasAllocator) {
+          context_.Say(x.source,
+              "ALLOCATORS directives that appear in a TARGET region "
+              "must specify an allocator"_err_en_US);
+        }
       }
     }
   }
+
   PopContext();
 }
 
diff --git a/flang/test/Parser/OpenMP/allocators-unparse.f90 b/flang/test/Parser/OpenMP/allocators-unparse.f90
index 5cd0230471fc4..70feb7a6b527e 100644
--- a/flang/test/Parser/OpenMP/allocators-unparse.f90
+++ b/flang/test/Parser/OpenMP/allocators-unparse.f90
@@ -15,48 +15,62 @@ subroutine allocate()
 
   !$omp allocators allocate(align(32): arr2)
     allocate(arr2(5, 3))
+  !$omp end allocators
 end subroutine allocate
 
 !CHECK: INTEGER, ALLOCATABLE :: arr1(:), arr2(:,:)
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(omp_default_mem_alloc: arr1)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(omp_default_mem_alloc: arr1)
 !CHECK-NEXT: ALLOCATE(arr1(5))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) ALL&
-!CHECK-NEXT:!$OMP&OCATE(omp_default_mem_alloc: arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) A&
+!CHECK-NEXT:!$OMP&LLOCATE(omp_default_mem_alloc: arr2)
 !CHECK-NEXT: ALLOCATE(arr1(10), arr2(3,2))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALIGN(32): arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALIGN(32): arr2)
 !CHECK-NEXT: ALLOCATE(arr2(5,3))
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr1'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr1'
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorComplexModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
-!PARSE-TREE-NEXT: OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr1'
-!PARSE-TREE-NEXT: AllocateShapeSpec
-!PARSE-TREE-NEXT: Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '10'
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorComplexModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
+!PARSE-TREE-NEXT: | | OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr1'
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | | | AllocateShapeSpec
+!PARSE-TREE-NEXT: | | | | | Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '5'
+!PARSE-TREE-NEXT: | | | | AllocateShapeSpec
+!PARSE-TREE-NEXT: | | | | | Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '3'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList ->
+!PARSE-TREE-NEXT: | | Flags = None
diff --git a/flang/test/Semantics/OpenMP/allocators07.f90 b/flang/test/Semantics/OpenMP/allocators07.f90
new file mode 100644
index 0000000000000..a28f706965cb1
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/allocators07.f90
@@ -0,0 +1,27 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+subroutine f00
+  implicit none
+  integer, allocatable :: a(:)
+
+  !$omp allocators allocate(a)
+!ERROR: The body of the ALLOCATORS construct should be an ALLOCATE statement
+  continue
+end
+
+subroutine f01
+  implicit none
+  integer, allocatable :: a(:)
+
+!ERROR: The ALLOCATORS construct should contain a single ALLOCATE statement
+  !$omp allocators allocate(a)
+  !$omp end allocators
+end
+
+subroutine f02
+  implicit none
+  integer, allocatable :: a(:)
+
+!ERROR: The ALLOCATORS construct should contain a single ALLOCATE statement
+  !$omp allocators allocate(a)
+end

>From 2fd6d4c21c03252db5a7211b81ef0894a7fb2d46 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 06:02:29 -0500
Subject: [PATCH 2/2] Remove extra whitespace

---
 flang/lib/Semantics/resolve-directives.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index bb3914ffa2b1e..151f4ccae634e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -2293,7 +2293,6 @@ void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
       }
     }
   }
-
   PopContext();
 }
 


From flang-commits at lists.llvm.org  Fri Jul 11 04:03:29 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Fri, 11 Jul 2025 04:03:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <6870ef81.170a0220.1a48ca.d55a@mx.google.com>


================
@@ -2234,31 +2253,47 @@ void OmpAttributeVisitor::Post(const parser::OpenMPExecutableAllocate &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
-    if (const auto *alloc{
-            std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
-      CheckAllNamesInAllocateStmt(dir.source,
-          std::get<parser::OmpObjectList>(alloc->v.t),
-          std::get<parser::Statement<parser::AllocateStmt>>(x.t).statement);
-
-      auto &modifiers{OmpGetModifiers(alloc->v)};
-      bool hasAllocator{
-          OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(modifiers) ||
-          OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(modifiers)};
-
-      // TODO: As with allocate directive, exclude the case when a requires
-      //       directive with the dynamic_allocators clause is present in
-      //       the same compilation unit (OMP5.0 2.11.3).
-      if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
-          !hasAllocator) {
-        context_.Say(x.source,
-            "ALLOCATORS directives that appear in a TARGET region "
-            "must specify an allocator"_err_en_US);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const parser::AllocateStmt *allocate{[&]() {
+    if (action) {
+      if (auto *alloc{std::get_if<common::Indirection<parser::AllocateStmt>>(
+              &action.stmt->u)}) {
+        return &alloc->value();
+      }
+    }
+    return static_cast<const parser::AllocateStmt *>(nullptr);
+  }()};
+
+  if (allocate) {
+    for (const auto &clause : dirSpec.Clauses().v) {
+      if (auto *alloc{std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
+        CheckAllNamesInAllocateStmt(
+            x.source, std::get<parser::OmpObjectList>(alloc->v.t), *allocate);
+
+        using OmpAllocatorSimpleModifier = parser::OmpAllocatorSimpleModifier;
+        using OmpAllocatorComplexModifier = parser::OmpAllocatorComplexModifier;
+
+        auto &modifiers{OmpGetModifiers(alloc->v)};
+        bool hasAllocator{
+            OmpGetUniqueModifier<OmpAllocatorSimpleModifier>(modifiers) ||
+            OmpGetUniqueModifier<OmpAllocatorComplexModifier>(modifiers)};
+
+        // TODO: As with allocate directive, exclude the case when a requires
+        //       directive with the dynamic_allocators clause is present in
+        //       the same compilation unit (OMP5.0 2.11.3).
+        if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
+            !hasAllocator) {
+          context_.Say(x.source,
+              "ALLOCATORS directives that appear in a TARGET region "
+              "must specify an allocator"_err_en_US);
+        }
       }
     }
   }
+
----------------
kparzysz wrote:

Removed

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Fri Jul 11 04:18:38 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 11 Jul 2025 04:18:38 -0700 (PDT)
Subject: [flang-commits] [flang] 6630cde - [flang][docs] Update release
	notes for OpenMP (#147988)
Message-ID: <6870f30e.170a0220.34eebe.e080@mx.google.com>


Author: Tom Eccles
Date: 2025-07-11T12:18:35+01:00
New Revision: 6630cde4e644f3b4147b984f9832c89fb3306698

URL: https://github.com/llvm/llvm-project/commit/6630cde4e644f3b4147b984f9832c89fb3306698
DIFF: https://github.com/llvm/llvm-project/commit/6630cde4e644f3b4147b984f9832c89fb3306698.diff

LOG: [flang][docs] Update release notes for OpenMP (#147988)

Co-authored-by: Kiran Chandramohan <kiranchandramohan at gmail.com>

Added: 
    

Modified: 
    flang/docs/ReleaseNotes.md

Removed: 
    


################################################################################
diff  --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index b2dbbcb5630f4..dfaa2eb6e03fe 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -25,6 +25,8 @@ page](https://llvm.org/releases/).
 ## Major New Features
 
 * Initial support for VOLATILE variables and procedure interface arguments has been added.
+* OpenMP support is stable and no longer considered experimental. All of OpenMP 3.1 is
+  supported, along with much of OpenMP 4.0 and some parts of later standards.
 
 ## Bug Fixes
 


        

From flang-commits at lists.llvm.org  Fri Jul 11 05:28:13 2025
From: flang-commits at lists.llvm.org (Jack Styles via flang-commits)
Date: Fri, 11 Jul 2025 05:28:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <6871035d.170a0220.391753.f9f9@mx.google.com>


================
@@ -1276,6 +1276,32 @@ struct OmpEndDirectiveParser {
   llvm::omp::Directive dir_;
 };
 
+struct OmpAllocatorsConstructParser {
+  using resultType = OpenMPAllocatorsConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_allocators) {
+      return std::nullopt;
+    }
+
+    // This should be an allocate-stmt. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_allocators}};
+    return OpenMPAllocatorsConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
----------------
Stylie777 wrote:

Makes sense

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Fri Jul 11 05:28:58 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 11 Jul 2025 05:28:58 -0700 (PDT)
Subject: [flang-commits] [flang] 61a9d2c - [flang][OpenMP] Use
 OmpDirectiveSpecification in DISPATCH (#148008)
Message-ID: <6871038a.050a0220.2a59c4.e876@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-11T07:28:54-05:00
New Revision: 61a9d2c22dfeaacbef184d8e0bda083bc30c53ae

URL: https://github.com/llvm/llvm-project/commit/61a9d2c22dfeaacbef184d8e0bda083bc30c53ae
DIFF: https://github.com/llvm/llvm-project/commit/61a9d2c22dfeaacbef184d8e0bda083bc30c53ae.diff

LOG: [flang][OpenMP] Use OmpDirectiveSpecification in DISPATCH (#148008)

Dispatch is the last construct (after ATOMIC and ALLOCATORS) where the
associated block requires a specific form.
Using OmpDirectiveSpecification for the begin and the optional end
directives will make the structure of all block directives more uniform.

Added: 
    

Modified: 
    flang/include/flang/Parser/dump-parse-tree.h
    flang/include/flang/Parser/parse-tree.h
    flang/lib/Parser/openmp-parsers.cpp
    flang/lib/Parser/unparse.cpp
    flang/lib/Semantics/check-omp-structure.cpp
    flang/test/Parser/OpenMP/dispatch.f90
    flang/test/Semantics/OpenMP/dispatch.f90

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 73c224e3ad235..32b6ca45609b6 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -710,8 +710,6 @@ class ParseTreeDumper {
   NODE(parser, OpenMPDepobjConstruct)
   NODE(parser, OpenMPUtilityConstruct)
   NODE(parser, OpenMPDispatchConstruct)
-  NODE(parser, OmpDispatchDirective)
-  NODE(parser, OmpEndDispatchDirective)
   NODE(parser, OpenMPFlushConstruct)
   NODE(parser, OpenMPLoopConstruct)
   NODE(parser, OpenMPExecutableAllocate)

diff  --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index fbc39286a95bf..ab2dde7d5dfbe 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4939,19 +4939,11 @@ struct OpenMPDepobjConstruct {
 //                    nocontext-clause |
 //                    novariants-clause |
 //                    nowait-clause
-struct OmpDispatchDirective {
-  TUPLE_CLASS_BOILERPLATE(OmpDispatchDirective);
-  CharBlock source;
-  std::tuple<Verbatim, OmpClauseList> t;
-};
-
-EMPTY_CLASS(OmpEndDispatchDirective);
-
 struct OpenMPDispatchConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPDispatchConstruct);
   CharBlock source;
-  std::tuple<OmpDispatchDirective, Block,
-      std::optional<OmpEndDispatchDirective>>
+  std::tuple<OmpDirectiveSpecification, Block,
+      std::optional<OmpDirectiveSpecification>>
       t;
 };
 

diff  --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 811ca2c855a6e..d70aaab82cbab 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1302,6 +1302,32 @@ TYPE_PARSER(sourced( //
     construct<OpenMPAllocatorsConstruct>(
         "ALLOCATORS"_tok >= OmpAllocatorsConstructParser{})))
 
+struct OmpDispatchConstructParser {
+  using resultType = OpenMPDispatchConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_dispatch) {
+      return std::nullopt;
+    }
+
+    // This should be a function call. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_dispatch}};
+    return OpenMPDispatchConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
+    construct<OpenMPDispatchConstruct>(
+        "DISPATCH"_tok >= OmpDispatchConstructParser{})))
+
 // Parser for an arbitrary OpenMP ATOMIC construct.
 //
 // Depending on circumstances, an ATOMIC construct applies to one or more
@@ -1631,16 +1657,6 @@ TYPE_PARSER(sourced(construct<OmpCriticalDirective>(verbatim("CRITICAL"_tok),
 TYPE_PARSER(construct<OpenMPCriticalConstruct>(
     Parser<OmpCriticalDirective>{}, block, Parser<OmpEndCriticalDirective>{}))
 
-TYPE_PARSER(sourced(construct<OmpDispatchDirective>(
-    verbatim("DISPATCH"_tok), Parser<OmpClauseList>{})))
-
-TYPE_PARSER(
-    construct<OmpEndDispatchDirective>(startOmpLine >> "END DISPATCH"_tok))
-
-TYPE_PARSER(sourced(construct<OpenMPDispatchConstruct>(
-    Parser<OmpDispatchDirective>{} / endOmpLine, block,
-    maybe(Parser<OmpEndDispatchDirective>{} / endOmpLine))))
-
 // 2.11.3 Executable Allocate directive
 TYPE_PARSER(
     sourced(construct<OpenMPExecutableAllocate>(verbatim("ALLOCATE"_tok),

diff  --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 4f692f3e9084f..b66d756bdbf2c 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2758,6 +2758,9 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
+  void Unparse(const OpenMPDispatchConstruct &x) { //
+    UnparseBlockConstruct(x);
+  }
   void Unparse(const OpenMPRequiresConstruct &y) {
     BeginOpenMP();
     Word("!$OMP REQUIRES ");
@@ -2777,15 +2780,6 @@ class UnparseVisitor {
     Walk(x.v);
     return false;
   }
-  void Unparse(const OmpDispatchDirective &x) {
-    Word("!$OMP DISPATCH");
-    Walk(x.t);
-    Put("\n");
-  }
-  void Unparse(const OmpEndDispatchDirective &) {
-    Word("!$OMP END DISPATCH");
-    Put("\n");
-  }
   void Unparse(const OmpErrorDirective &x) {
     Word("!$OMP ERROR ");
     Walk(x.t);

diff  --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 9b89d19a91f38..2425265e196c6 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -509,8 +509,8 @@ template <typename Checker> struct DirectiveSpellingVisitor {
     checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocate);
     return false;
   }
-  bool Pre(const parser::OmpDispatchDirective &x) {
-    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_dispatch);
+  bool Pre(const parser::OpenMPDispatchConstruct &x) {
+    checker_(GetDirName(x.t).source, Directive::OMPD_dispatch);
     return false;
   }
   bool Pre(const parser::OmpErrorDirective &x) {
@@ -1595,28 +1595,31 @@ void OmpStructureChecker::Enter(const parser::OmpErrorDirective &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPDispatchConstruct &x) {
-  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_dispatch);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
   const auto &block{std::get<parser::Block>(x.t)};
-  if (block.empty() || block.size() > 1) {
+  PushContextAndClauseSets(
+      dirSpec.DirName().source, llvm::omp::Directive::OMPD_dispatch);
+
+  if (block.empty()) {
     context_.Say(x.source,
-        "The DISPATCH construct is empty or contains more than one statement"_err_en_US);
+        "The DISPATCH construct should contain a single function or subroutine call"_err_en_US);
     return;
   }
 
-  auto it{block.begin()};
   bool passChecks{false};
-  if (const parser::AssignmentStmt *
-      assignStmt{parser::Unwrap<parser::AssignmentStmt>(*it)}) {
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  if (const auto *assignStmt{
+          parser::Unwrap<parser::AssignmentStmt>(*action.stmt)}) {
     if (parser::Unwrap<parser::FunctionReference>(assignStmt->t)) {
       passChecks = true;
     }
-  } else if (parser::Unwrap<parser::CallStmt>(*it)) {
+  } else if (parser::Unwrap<parser::CallStmt>(*action.stmt)) {
     passChecks = true;
   }
 
   if (!passChecks) {
-    context_.Say(x.source,
-        "The DISPATCH construct does not contain a SUBROUTINE or FUNCTION"_err_en_US);
+    context_.Say(action.source,
+        "The body of the DISPATCH construct should be a function or a subroutine call"_err_en_US);
   }
 }
 

diff  --git a/flang/test/Parser/OpenMP/dispatch.f90 b/flang/test/Parser/OpenMP/dispatch.f90
index 98cd6090334f3..4076c00331225 100644
--- a/flang/test/Parser/OpenMP/dispatch.f90
+++ b/flang/test/Parser/OpenMP/dispatch.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s | FileCheck %s
+! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s | FileCheck %s --check-prefix=PARSE-TREE
 ! RUN: %flang_fc1 -fopenmp -fdebug-unparse %s | FileCheck %s --check-prefix="UNPARSE"
 
 integer function func(a, b, c)
@@ -12,40 +12,57 @@ subroutine sub(x)
   integer :: r
   type(c_ptr) :: x
   integer :: a = 14, b = 7, c = 21
+
 !UNPARSE: !$OMP DISPATCH DEVICE(3_4) NOWAIT NOCONTEXT(.false._4) NOVARIANTS(.true._4)
-!CHECK: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
-!CHECK-NEXT: | | | OmpDispatchDirective
-!CHECK: | | | | OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
-!CHECK-NEXT: | | | | | Scalar -> Integer -> Expr = '3_4'
-!CHECK-NEXT: | | | | | | LiteralConstant -> IntLiteralConstant = '3'
-!CHECK-NEXT: | | | | OmpClause -> Nowait
-!CHECK-NEXT: | | | | OmpClause -> Nocontext -> Scalar -> Logical -> Expr = '.false._4'
-!CHECK-NEXT: | | | | | LiteralConstant -> LogicalLiteralConstant
-!CHECK-NEXT: | | | | | | bool = 'false'
-!CHECK-NEXT: | | | | OmpClause -> Novariants -> Scalar -> Logical -> Expr = '.true._4'
-!CHECK-NEXT: | | | | | EQ
-!CHECK-NEXT: | | | | | | Expr = '1_4'
-!CHECK-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
-!CHECK-NEXT: | | | | | | Expr = '1_4'
-!CHECK-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
-!CHECK-NEXT: | | | Block
- 
+!UNPARSE:   r=func(a,b,c)
+!UNPARSE: !$OMP END DISPATCH
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = dispatch
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
+!PARSE-TREE: | | | Scalar -> Integer -> Expr = '3_4'
+!PARSE-TREE: | | | | LiteralConstant -> IntLiteralConstant = '3'
+!PARSE-TREE: | | OmpClause -> Nowait
+!PARSE-TREE: | | OmpClause -> Nocontext -> Scalar -> Logical -> Expr = '.false._4'
+!PARSE-TREE: | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | bool = 'false'
+!PARSE-TREE: | | OmpClause -> Novariants -> Scalar -> Logical -> Expr = '.true._4'
+!PARSE-TREE: | | | EQ
+!PARSE-TREE: | | | | Expr = '1_4'
+!PARSE-TREE: | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | | | Expr = '1_4'
+!PARSE-TREE: | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt
+![...]
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = dispatch
+!PARSE-TREE: | | OmpClauseList ->
+!PARSE-TREE: | | Flags = None
+
   !$omp dispatch device(3) nowait nocontext(.false.) novariants(1.eq.1)
   r = func(a, b, c)
-!UNPARSE: !$OMP END DISPATCH
-!CHECK: | | | OmpEndDispatchDirective
   !$omp end dispatch
 
 !! Test the "no end dispatch" option.
-!UNPARSE: !$OMP DISPATCH  DEVICE(3_4) IS_DEVICE_PTR(x)
-!CHECK: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
-!CHECK-NEXT: | | | OmpDispatchDirective
-!CHECK: | | | | OmpClause -> IsDevicePtr ->  OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'  
+!UNPARSE: !$OMP DISPATCH DEVICE(3_4) IS_DEVICE_PTR(x)
+!UNPARSE:   r=func(a+1_4,b+2_4,c+3_4)
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = dispatch
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
+!PARSE-TREE: | | | Scalar -> Integer -> Expr = '3_4'
+!PARSE-TREE: | | | | LiteralConstant -> IntLiteralConstant = '3'
+!PARSE-TREE: | | OmpClause -> IsDevicePtr -> OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt
+!PARSE-TREE-NOT: OmpDirectiveSpecification
+
   !$omp dispatch device(3) is_device_ptr(x)
   r = func(a+1, b+2, c+3)
-!CHECK-NOT: | | | OmpEndDispatchDirective
 
 end subroutine sub
-
-
-

diff  --git a/flang/test/Semantics/OpenMP/dispatch.f90 b/flang/test/Semantics/OpenMP/dispatch.f90
index 7dfbeecb2fc1d..af0d6856ab948 100644
--- a/flang/test/Semantics/OpenMP/dispatch.f90
+++ b/flang/test/Semantics/OpenMP/dispatch.f90
@@ -1,24 +1,20 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
 
 subroutine sb1
   integer :: r
   r = 1
-  !ERROR: The DISPATCH construct does not contain a SUBROUTINE or FUNCTION
   !$omp dispatch nowait
+!ERROR: The body of the DISPATCH construct should be a function or a subroutine call
   print *,r
 end subroutine
+
 subroutine sb2
-  integer :: r
-!ERROR: The DISPATCH construct is empty or contains more than one statement
+!ERROR: The DISPATCH construct should contain a single function or subroutine call
   !$omp dispatch
-  call foo()
-  r = bar()
   !$omp end dispatch
-contains
-  subroutine foo
-  end subroutine foo
-  function bar
-    integer :: bar
-    bar = 2
-  end function
+end subroutine
+
+subroutine sb3
+!ERROR: The DISPATCH construct should contain a single function or subroutine call
+  !$omp dispatch
 end subroutine


        

From flang-commits at lists.llvm.org  Fri Jul 11 05:29:02 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Fri, 11 Jul 2025 05:29:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Use
 OmpDirectiveSpecification in DISPATCH (PR #148008)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148008@github.com>
Message-ID: <6871038e.170a0220.127192.fd43@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/148008

From flang-commits at lists.llvm.org  Fri Jul 11 05:52:34 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 05:52:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Convert AST node for
 ALLOCATORS to use Block as body (PR #148005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148005@github.com>
Message-ID: <68710912.630a0220.353839.e2c5@mx.google.com>


https://github.com/tblah commented:

LGTM

https://github.com/llvm/llvm-project/pull/148005

From flang-commits at lists.llvm.org  Fri Jul 11 06:03:19 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 11 Jul 2025 06:03:19 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <68710b97.050a0220.cda97.bdab@mx.google.com>


https://github.com/DavidSpickett updated https://github.com/llvm/llvm-project/pull/148169

>From 1eac80b12fa0c2ca5c68132795b6d2f26e376855 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Wed, 9 Jul 2025 14:53:12 +0000
Subject: [PATCH 1/5] Reland "[flang] Avoid undefined behaviour when parsing
 format expressions (#147539)"

This reverts commit e8e5d07767c444913f837dd35846a92fcf520eab.

This previously failed because the flang-rt build could not find the
llvm header file. It passed on some machines but only because they
had globally installed copies of older llvm.

To fix this, I've copied the required routines from llvm into flang.

With the following justification:
* Flang can, and does, use llvm headers.
* Some Flang headers are also used in Flang-rt.
* Flang-rt cannot use llvm headers.
* Therefore any Flang header using in Flang-rt cannot use llvm headers either.

To support that conclusion, https://flang.llvm.org/docs/IORuntimeInternals.html
states:
"The Fortran I/O runtime support library is written in C++17, and uses some C++17 standard library facilities, but it is intended to not have any link-time dependences on the C++ runtime support library or any LLVM libraries."

This talks about libraries but I assume it applies to llvm in general.

Nothing in flang/include/flang/Common, or flang/include/flang/Common
includes any llvm header, and I see some very similar headers there
that duplicate llvm functionality. Like float128.h.

I can only assume this means these files must remain free of dependencies
on LLVM.

I have copied the two routines literally and put them in the flang::common
namespace, for lack of a better place for them. So they don't clash with something.

I have specialised the function to the 1 type flang needs, as it might
save a bit of compile time.
---
 flang/include/flang/Common/format.h | 75 +++++++++++++++++++++++++++--
 1 file changed, 70 insertions(+), 5 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 1650f56140b4d..78ddb1de2fa26 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -12,6 +12,7 @@
 #include "Fortran-consts.h"
 #include "enum-set.h"
 #include <cstring>
+#include <limits>
 
 // Define a FormatValidator class template to validate a format expression
 // of a given CHAR type.  To enable use in runtime library code as well as
@@ -28,6 +29,68 @@
 
 namespace Fortran::common {
 
+// AddOverflow and MulOverflow are copied from
+// llvm/include/llvm/Support/MathExtras.h. It is ok to include llvm headers in
+// flang, but this flang header is also used in flang-rt, which cannot use llvm
+// headers.
+
+/// Add two signed integers, computing the two's complement truncated result,
+/// returning true if overflow occurred.
+template <typename T>
+std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
+#if __has_builtin(__builtin_add_overflow)
+  return __builtin_add_overflow(X, Y, &Result);
+#else
+  // Perform the unsigned addition.
+  using U = std::make_unsigned_t<T>;
+  const U UX = static_cast<U>(X);
+  const U UY = static_cast<U>(Y);
+  const U UResult = UX + UY;
+
+  // Convert to signed.
+  Result = static_cast<T>(UResult);
+
+  // Adding two positive numbers should result in a positive number.
+  if (X > 0 && Y > 0)
+    return Result <= 0;
+  // Adding two negatives should result in a negative number.
+  if (X < 0 && Y < 0)
+    return Result >= 0;
+  return false;
+#endif
+}
+
+/// Multiply two signed integers, computing the two's complement truncated
+/// result, returning true if an overflow occurred.
+template <typename T>
+std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
+#if __has_builtin(__builtin_mul_overflow)
+  return __builtin_mul_overflow(X, Y, &Result);
+#else
+  // Perform the unsigned multiplication on absolute values.
+  using U = std::make_unsigned_t<T>;
+  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
+  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
+  const U UResult = UX * UY;
+
+  // Convert to signed.
+  const bool IsNegative = (X < 0) ^ (Y < 0);
+  Result = IsNegative ? (0 - UResult) : UResult;
+
+  // If any of the args was 0, result is 0 and no overflow occurs.
+  if (UX == 0 || UY == 0)
+    return false;
+
+  // UX and UY are in [1, 2^n], where n is the number of digits.
+  // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
+  // positive) divided by an argument compares to the other.
+  if (IsNegative)
+    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
+  else
+    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
+#endif
+}
+
 struct FormatMessage {
   const char *text; // message text; may have one %s argument
   const char *arg; // optional %s argument value
@@ -214,16 +277,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (!overflow) {
+        overflow =
+            MulOverflow(static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (!overflow) {
+        overflow = AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
       }
       c = NextChar();
     } while (c >= '0' && c <= '9');

>From aa10d5dfe83482f41aa306741c8c6756fe913753 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 08:20:44 +0000
Subject: [PATCH 2/5] manually specialise template

---
 flang/include/flang/Common/format.h | 30 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 78ddb1de2fa26..3c70aea1ec721 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -30,25 +30,21 @@
 namespace Fortran::common {
 
 // AddOverflow and MulOverflow are copied from
-// llvm/include/llvm/Support/MathExtras.h. It is ok to include llvm headers in
-// flang, but this flang header is also used in flang-rt, which cannot use llvm
-// headers.
+// llvm/include/llvm/Support/MathExtras.h and specialised to int64_t.
 
 /// Add two signed integers, computing the two's complement truncated result,
 /// returning true if overflow occurred.
-template <typename T>
-std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
+bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_add_overflow)
   return __builtin_add_overflow(X, Y, &Result);
 #else
   // Perform the unsigned addition.
-  using U = std::make_unsigned_t<T>;
-  const U UX = static_cast<U>(X);
-  const U UY = static_cast<U>(Y);
-  const U UResult = UX + UY;
+  const uint64_t UX = static_cast<uint64_t>(X);
+  const uint64_t UY = static_cast<uint64_t>(Y);
+  const uint64_t UResult = UX + UY;
 
   // Convert to signed.
-  Result = static_cast<T>(UResult);
+  Result = static_cast<int64_t>(UResult);
 
   // Adding two positive numbers should result in a positive number.
   if (X > 0 && Y > 0)
@@ -62,16 +58,14 @@ std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
 
 /// Multiply two signed integers, computing the two's complement truncated
 /// result, returning true if an overflow occurred.
-template <typename T>
-std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
+int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_mul_overflow)
   return __builtin_mul_overflow(X, Y, &Result);
 #else
   // Perform the unsigned multiplication on absolute values.
-  using U = std::make_unsigned_t<T>;
-  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
-  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
-  const U UResult = UX * UY;
+  const uint64_t UX = X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
+  const uint64_t UY = Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
+  const uint64_t UResult = UX * UY;
 
   // Convert to signed.
   const bool IsNegative = (X < 0) ^ (Y < 0);
@@ -85,9 +79,9 @@ std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) / UY;
   else
-    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
 #endif
 }
 

>From 89009a28fadef67310f0310cc90086ef05afb859 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 09:23:29 +0000
Subject: [PATCH 3/5] format

---
 flang/include/flang/Common/format.h | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 3c70aea1ec721..a853a636bdbe0 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -63,8 +63,10 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   return __builtin_mul_overflow(X, Y, &Result);
 #else
   // Perform the unsigned multiplication on absolute values.
-  const uint64_t UX = X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
-  const uint64_t UY = Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
+  const uint64_t UX =
+      X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
+  const uint64_t UY =
+      Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
   const uint64_t UResult = UX * UY;
 
   // Convert to signed.
@@ -79,7 +81,9 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) / UY;
+    return UX >
+        (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) /
+        UY;
   else
     return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
 #endif

>From b7fc8a5fdbe9a66a775519f25425f9297741042a Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 12:09:01 +0000
Subject: [PATCH 4/5] fix limit template

---
 flang/include/flang/Common/format.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index a853a636bdbe0..91d1db2484557 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -82,10 +82,10 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // positive) divided by an argument compares to the other.
   if (IsNegative)
     return UX >
-        (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) /
+        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + uint64_t(1)) /
         UY;
   else
-    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
 #endif
 }
 

>From 4616aed4c157063dea6a561397a28388ea567804 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 13:01:52 +0000
Subject: [PATCH 5/5] Make static

---
 flang/include/flang/Common/format.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 91d1db2484557..ed12e3a5f6273 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -34,7 +34,7 @@ namespace Fortran::common {
 
 /// Add two signed integers, computing the two's complement truncated result,
 /// returning true if overflow occurred.
-bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
+static inline bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_add_overflow)
   return __builtin_add_overflow(X, Y, &Result);
 #else
@@ -58,7 +58,7 @@ bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 
 /// Multiply two signed integers, computing the two's complement truncated
 /// result, returning true if an overflow occurred.
-int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
+static inline bool MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_mul_overflow)
   return __builtin_mul_overflow(X, Y, &Result);
 #else


From flang-commits at lists.llvm.org  Fri Jul 11 06:05:48 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 11 Jul 2025 06:05:48 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <68710c2c.630a0220.39aeca.e48d@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions h -- flang/include/flang/Common/format.h
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index ed12e3a5f..8b4ffe63c 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -81,11 +81,12 @@ static inline bool MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX >
-        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + uint64_t(1)) /
+    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) +
+                    uint64_t(1)) /
         UY;
   else
-    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
+    return UX >
+        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
 #endif
 }
 

``````````

</details>


https://github.com/llvm/llvm-project/pull/148169

From flang-commits at lists.llvm.org  Fri Jul 11 06:09:04 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 11 Jul 2025 06:09:04 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <68710cf0.050a0220.5c9e6.f19d@mx.google.com>


https://github.com/DavidSpickett updated https://github.com/llvm/llvm-project/pull/148169

>From 1eac80b12fa0c2ca5c68132795b6d2f26e376855 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Wed, 9 Jul 2025 14:53:12 +0000
Subject: [PATCH 1/6] Reland "[flang] Avoid undefined behaviour when parsing
 format expressions (#147539)"

This reverts commit e8e5d07767c444913f837dd35846a92fcf520eab.

This previously failed because the flang-rt build could not find the
llvm header file. It passed on some machines but only because they
had globally installed copies of older llvm.

To fix this, I've copied the required routines from llvm into flang.

With the following justification:
* Flang can, and does, use llvm headers.
* Some Flang headers are also used in Flang-rt.
* Flang-rt cannot use llvm headers.
* Therefore any Flang header using in Flang-rt cannot use llvm headers either.

To support that conclusion, https://flang.llvm.org/docs/IORuntimeInternals.html
states:
"The Fortran I/O runtime support library is written in C++17, and uses some C++17 standard library facilities, but it is intended to not have any link-time dependences on the C++ runtime support library or any LLVM libraries."

This talks about libraries but I assume it applies to llvm in general.

Nothing in flang/include/flang/Common, or flang/include/flang/Common
includes any llvm header, and I see some very similar headers there
that duplicate llvm functionality. Like float128.h.

I can only assume this means these files must remain free of dependencies
on LLVM.

I have copied the two routines literally and put them in the flang::common
namespace, for lack of a better place for them. So they don't clash with something.

I have specialised the function to the 1 type flang needs, as it might
save a bit of compile time.
---
 flang/include/flang/Common/format.h | 75 +++++++++++++++++++++++++++--
 1 file changed, 70 insertions(+), 5 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 1650f56140b4d..78ddb1de2fa26 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -12,6 +12,7 @@
 #include "Fortran-consts.h"
 #include "enum-set.h"
 #include <cstring>
+#include <limits>
 
 // Define a FormatValidator class template to validate a format expression
 // of a given CHAR type.  To enable use in runtime library code as well as
@@ -28,6 +29,68 @@
 
 namespace Fortran::common {
 
+// AddOverflow and MulOverflow are copied from
+// llvm/include/llvm/Support/MathExtras.h. It is ok to include llvm headers in
+// flang, but this flang header is also used in flang-rt, which cannot use llvm
+// headers.
+
+/// Add two signed integers, computing the two's complement truncated result,
+/// returning true if overflow occurred.
+template <typename T>
+std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
+#if __has_builtin(__builtin_add_overflow)
+  return __builtin_add_overflow(X, Y, &Result);
+#else
+  // Perform the unsigned addition.
+  using U = std::make_unsigned_t<T>;
+  const U UX = static_cast<U>(X);
+  const U UY = static_cast<U>(Y);
+  const U UResult = UX + UY;
+
+  // Convert to signed.
+  Result = static_cast<T>(UResult);
+
+  // Adding two positive numbers should result in a positive number.
+  if (X > 0 && Y > 0)
+    return Result <= 0;
+  // Adding two negatives should result in a negative number.
+  if (X < 0 && Y < 0)
+    return Result >= 0;
+  return false;
+#endif
+}
+
+/// Multiply two signed integers, computing the two's complement truncated
+/// result, returning true if an overflow occurred.
+template <typename T>
+std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
+#if __has_builtin(__builtin_mul_overflow)
+  return __builtin_mul_overflow(X, Y, &Result);
+#else
+  // Perform the unsigned multiplication on absolute values.
+  using U = std::make_unsigned_t<T>;
+  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
+  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
+  const U UResult = UX * UY;
+
+  // Convert to signed.
+  const bool IsNegative = (X < 0) ^ (Y < 0);
+  Result = IsNegative ? (0 - UResult) : UResult;
+
+  // If any of the args was 0, result is 0 and no overflow occurs.
+  if (UX == 0 || UY == 0)
+    return false;
+
+  // UX and UY are in [1, 2^n], where n is the number of digits.
+  // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
+  // positive) divided by an argument compares to the other.
+  if (IsNegative)
+    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
+  else
+    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
+#endif
+}
+
 struct FormatMessage {
   const char *text; // message text; may have one %s argument
   const char *arg; // optional %s argument value
@@ -214,16 +277,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (!overflow) {
+        overflow =
+            MulOverflow(static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (!overflow) {
+        overflow = AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
       }
       c = NextChar();
     } while (c >= '0' && c <= '9');

>From aa10d5dfe83482f41aa306741c8c6756fe913753 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 08:20:44 +0000
Subject: [PATCH 2/6] manually specialise template

---
 flang/include/flang/Common/format.h | 30 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 78ddb1de2fa26..3c70aea1ec721 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -30,25 +30,21 @@
 namespace Fortran::common {
 
 // AddOverflow and MulOverflow are copied from
-// llvm/include/llvm/Support/MathExtras.h. It is ok to include llvm headers in
-// flang, but this flang header is also used in flang-rt, which cannot use llvm
-// headers.
+// llvm/include/llvm/Support/MathExtras.h and specialised to int64_t.
 
 /// Add two signed integers, computing the two's complement truncated result,
 /// returning true if overflow occurred.
-template <typename T>
-std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
+bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_add_overflow)
   return __builtin_add_overflow(X, Y, &Result);
 #else
   // Perform the unsigned addition.
-  using U = std::make_unsigned_t<T>;
-  const U UX = static_cast<U>(X);
-  const U UY = static_cast<U>(Y);
-  const U UResult = UX + UY;
+  const uint64_t UX = static_cast<uint64_t>(X);
+  const uint64_t UY = static_cast<uint64_t>(Y);
+  const uint64_t UResult = UX + UY;
 
   // Convert to signed.
-  Result = static_cast<T>(UResult);
+  Result = static_cast<int64_t>(UResult);
 
   // Adding two positive numbers should result in a positive number.
   if (X > 0 && Y > 0)
@@ -62,16 +58,14 @@ std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
 
 /// Multiply two signed integers, computing the two's complement truncated
 /// result, returning true if an overflow occurred.
-template <typename T>
-std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
+int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_mul_overflow)
   return __builtin_mul_overflow(X, Y, &Result);
 #else
   // Perform the unsigned multiplication on absolute values.
-  using U = std::make_unsigned_t<T>;
-  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
-  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
-  const U UResult = UX * UY;
+  const uint64_t UX = X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
+  const uint64_t UY = Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
+  const uint64_t UResult = UX * UY;
 
   // Convert to signed.
   const bool IsNegative = (X < 0) ^ (Y < 0);
@@ -85,9 +79,9 @@ std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) / UY;
   else
-    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
 #endif
 }
 

>From 89009a28fadef67310f0310cc90086ef05afb859 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 09:23:29 +0000
Subject: [PATCH 3/6] format

---
 flang/include/flang/Common/format.h | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 3c70aea1ec721..a853a636bdbe0 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -63,8 +63,10 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   return __builtin_mul_overflow(X, Y, &Result);
 #else
   // Perform the unsigned multiplication on absolute values.
-  const uint64_t UX = X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
-  const uint64_t UY = Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
+  const uint64_t UX =
+      X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
+  const uint64_t UY =
+      Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
   const uint64_t UResult = UX * UY;
 
   // Convert to signed.
@@ -79,7 +81,9 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) / UY;
+    return UX >
+        (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) /
+        UY;
   else
     return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
 #endif

>From b7fc8a5fdbe9a66a775519f25425f9297741042a Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 12:09:01 +0000
Subject: [PATCH 4/6] fix limit template

---
 flang/include/flang/Common/format.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index a853a636bdbe0..91d1db2484557 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -82,10 +82,10 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // positive) divided by an argument compares to the other.
   if (IsNegative)
     return UX >
-        (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) /
+        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + uint64_t(1)) /
         UY;
   else
-    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
 #endif
 }
 

>From 4616aed4c157063dea6a561397a28388ea567804 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 13:01:52 +0000
Subject: [PATCH 5/6] Make static

---
 flang/include/flang/Common/format.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 91d1db2484557..ed12e3a5f6273 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -34,7 +34,7 @@ namespace Fortran::common {
 
 /// Add two signed integers, computing the two's complement truncated result,
 /// returning true if overflow occurred.
-bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
+static inline bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_add_overflow)
   return __builtin_add_overflow(X, Y, &Result);
 #else
@@ -58,7 +58,7 @@ bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 
 /// Multiply two signed integers, computing the two's complement truncated
 /// result, returning true if an overflow occurred.
-int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
+static inline bool MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_mul_overflow)
   return __builtin_mul_overflow(X, Y, &Result);
 #else

>From 3baaad3723767f29b3b434b71e6d2b080b2e38a5 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 13:08:19 +0000
Subject: [PATCH 6/6] format

---
 flang/include/flang/Common/format.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index ed12e3a5f6273..8b4ffe63c4459 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -81,11 +81,12 @@ static inline bool MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX >
-        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + uint64_t(1)) /
+    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) +
+                    uint64_t(1)) /
         UY;
   else
-    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
+    return UX >
+        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
 #endif
 }
 


From flang-commits at lists.llvm.org  Fri Jul 11 06:11:00 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 11 Jul 2025 06:11:00 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <68710d64.050a0220.2abd0.e7ba@mx.google.com>


DavidSpickett wrote:

@tblah try it again, I think the static I added should fix it. It's that or make them members of the using class, which isn't an awful idea either but harder to find later.

Did not fail in my local build but I see why it could, pure chance that it doesn't I expect.

https://github.com/llvm/llvm-project/pull/148169

From flang-commits at lists.llvm.org  Fri Jul 11 06:26:52 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 06:26:52 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Use
 OmpDirectiveSpecification in DISPATCH (PR #148008)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148008@github.com>
Message-ID: <6871111c.630a0220.51312.dfb9@mx.google.com>


https://github.com/tblah commented:

LGTM, thanks

https://github.com/llvm/llvm-project/pull/148008

From flang-commits at lists.llvm.org  Fri Jul 11 06:36:36 2025
From: flang-commits at lists.llvm.org (Christian Ulmann via flang-commits)
Date: Fri, 11 Jul 2025 06:36:36 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][Interfaces] Remove negative
 branch weight verifier (PR #148234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148234@github.com>
Message-ID: <68711364.170a0220.270187.d671@mx.google.com>


https://github.com/Dinistro updated https://github.com/llvm/llvm-project/pull/148234

>From 5d589f1962cfa0e944aa2f035d426acd2978b4a5 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christian.ulmann at nextsilicon.com>
Date: Fri, 11 Jul 2025 11:59:53 +0000
Subject: [PATCH 1/2] [MLIR][Interfaces] Remove negative branch weight verifier

This commit removes the verifier that checked if branch weights are
negative. This check was too strict because weights are interpreted as
unsigned integers.

This showed up when running the verifier on LLVM dialect modules that
were imported from LLVM IR.
---
 .../mlir/Interfaces/ControlFlowInterfaces.td  | 28 +++++++++++--------
 mlir/lib/Interfaces/ControlFlowInterfaces.cpp |  4 ---
 mlir/test/Dialect/ControlFlow/invalid.mlir    | 12 --------
 3 files changed, 16 insertions(+), 28 deletions(-)

diff --git a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
index 46ab0b9ebbc6b..b8d08cc553caa 100644
--- a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
+++ b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
@@ -384,13 +384,15 @@ def WeightedBranchOpInterface : OpInterface<"WeightedBranchOpInterface"> {
     This interface provides weight information for branching terminator
     operations, i.e. terminator operations with successors.
 
-    This interface provides methods for getting/setting integer non-negative
-    weight of each branch. The probability of executing a branch
-    is computed as the ratio between the branch's weight and the total
-    sum of the weights (which cannot be zero).
-    The weights are optional. If they are provided, then their number
+    This interface provides methods for getting/setting integer weights of each
+    branch. The probability of executing a branch is computed as the ratio
+    between the branch's weight and the total sum of the weights (which cannot
+    be zero). The weights are optional. If they are provided, then their number
     must match the number of successors of the operation.
 
+    Note that the branch weight use an i32 representation but they are to be
+    interpreted as unsigned integers.
+
     The default implementations of the methods expect the operation
     to have an attribute of type DenseI32ArrayAttr named branch_weights.
   }];
@@ -440,19 +442,21 @@ def WeightedRegionBranchOpInterface
     This interface provides weight information for region operations
     that exhibit branching behavior between held regions.
 
-    This interface provides methods for getting/setting integer non-negative
-    weight of each branch. The probability of executing a region is computed
-    as the ratio between the region branch's weight and the total sum
-    of the weights (which cannot be zero).
-    The weights are optional. If they are provided, then their number
-    must match the number of regions held by the operation
-    (including empty regions).
+    This interface provides methods for getting/setting integer weights of each
+    branch. The probability of executing a region is computed as the ratio
+    between the region branch's weight and the total sum of the weights (which
+    cannot be zero). The weights are optional. If they are provided, then their
+    number must match the number of regions held by the operation (including
+    empty regions).
 
     The weights specify the probability of branching to a particular
     region when first executing the operation.
     For example, for loop-like operations with a single region
     the weight specifies the probability of entering the loop.
 
+    Note that the branch weight use an i32 representation but they are to be
+    interpreted as unsigned integers.
+
     The default implementations of the methods expect the operation
     to have an attribute of type DenseI32ArrayAttr named branch_weights.
   }];
diff --git a/mlir/lib/Interfaces/ControlFlowInterfaces.cpp b/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
index 3a63db35eec0f..e87bb461b0329 100644
--- a/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
+++ b/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
@@ -99,10 +99,6 @@ static LogicalResult verifyWeights(Operation *op,
                            << ": " << weights.size() << " vs "
                            << expectedWeightsNum;
 
-  for (auto [index, weight] : llvm::enumerate(weights))
-    if (weight < 0)
-      return op->emitError() << "weight #" << index << " must be non-negative";
-
   if (llvm::all_of(weights, [](int32_t value) { return value == 0; }))
     return op->emitError() << "branch weights cannot all be zero";
 
diff --git a/mlir/test/Dialect/ControlFlow/invalid.mlir b/mlir/test/Dialect/ControlFlow/invalid.mlir
index 1b8de22a9ff9f..0a71c62ec31af 100644
--- a/mlir/test/Dialect/ControlFlow/invalid.mlir
+++ b/mlir/test/Dialect/ControlFlow/invalid.mlir
@@ -82,18 +82,6 @@ func.func @wrong_weights_number(%cond: i1) {
 
 // -----
 
-// CHECK-LABEL: func @negative_weight
-func.func @wrong_total_weight(%cond: i1) {
-  // expected-error at +1 {{weight #0 must be non-negative}}
-  cf.cond_br %cond weights([-1, 101]), ^bb1, ^bb2
-  ^bb1:
-    return
-  ^bb2:
-    return
-}
-
-// -----
-
 // CHECK-LABEL: func @zero_weights
 func.func @wrong_total_weight(%cond: i1) {
   // expected-error at +1 {{branch weights cannot all be zero}}

>From ffd648c266b1d50de894d7b3f284dfca14723820 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christian.ulmann at nextsilicon.com>
Date: Fri, 11 Jul 2025 13:36:25 +0000
Subject: [PATCH 2/2] fix Fir test

---
 flang/test/Fir/invalid.fir | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index e32ea7ad3c729..e5dbec44220b7 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -1426,12 +1426,3 @@ func.func @wrong_weights_number_in_if_then_else(%cond: i1) {
   }
   return
 }
-
-// -----
-
-func.func @negative_weight_in_if_then(%cond: i1) {
-// expected-error @below {{weight #0 must be non-negative}}
-  fir.if %cond weights([-1, 101]) {
-  }
-  return
-}


From flang-commits at lists.llvm.org  Fri Jul 11 07:00:30 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 07:00:30 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <687118fe.170a0220.c2c64.dc2b@mx.google.com>


https://github.com/tblah approved this pull request.

Works for me, and I have verified that this doesn't lead to fortran executables having new runtime dependencies. Thanks for sticking with this patch.

https://github.com/llvm/llvm-project/pull/148169

From flang-commits at lists.llvm.org  Fri Jul 11 07:07:02 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 07:07:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed TBAA tags for derived types
 with descriptors. (PR #148093)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148093@github.com>
Message-ID: <68711a86.630a0220.116441.eca1@mx.google.com>


https://github.com/tblah approved this pull request.

Thank you for the fix

https://github.com/llvm/llvm-project/pull/148093

From flang-commits at lists.llvm.org  Fri Jul 11 02:59:40 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Fri, 11 Jul 2025 02:59:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][docs] Update release notes for
	OpenMP (PR #147988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147988@github.com>
Message-ID: <6870e08c.630a0220.25a9e1.d640@mx.google.com>


https://github.com/kiranchandramohan approved this pull request.

LG. Thanks for the update.

https://github.com/llvm/llvm-project/pull/147988

From flang-commits at lists.llvm.org  Fri Jul 11 02:59:40 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Fri, 11 Jul 2025 02:59:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][docs] Update release notes for
	OpenMP (PR #147988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147988@github.com>
Message-ID: <6870e08c.170a0220.3a4a61.cd62@mx.google.com>


https://github.com/kiranchandramohan edited https://github.com/llvm/llvm-project/pull/147988

From flang-commits at lists.llvm.org  Fri Jul 11 02:59:41 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Fri, 11 Jul 2025 02:59:41 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][docs] Update release notes for
	OpenMP (PR #147988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147988@github.com>
Message-ID: <6870e08d.630a0220.2a28de.d4eb@mx.google.com>


================
@@ -25,6 +25,8 @@ page](https://llvm.org/releases/).
 ## Major New Features
 
 * Initial support for VOLATILE variables and procedure interface arguments has been added.
+* OpenMP support is no longer considered experimental. All of OpenMP 3.1 is
----------------
kiranchandramohan wrote:

```suggestion
* OpenMP support is stable and no longer considered experimental. All of OpenMP 3.1 is
```

https://github.com/llvm/llvm-project/pull/147988

From flang-commits at lists.llvm.org  Fri Jul 11 03:28:54 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 03:28:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][docs] Update release notes for
	OpenMP (PR #147988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147988@github.com>
Message-ID: <6870e766.170a0220.26eb38.c588@mx.google.com>


https://github.com/tblah updated https://github.com/llvm/llvm-project/pull/147988

>From b377f15e83bd28a4aacffab83c226612c71a05fd Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Thu, 10 Jul 2025 15:33:46 +0000
Subject: [PATCH 1/3] [flang][docs] Update release notes for OpenMP

---
 flang/docs/ReleaseNotes.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index b2dbbcb5630f4..cbd3f9f4a4e78 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -25,6 +25,8 @@ page](https://llvm.org/releases/).
 ## Major New Features
 
 * Initial support for VOLATILE variables and procedure interface arguments has been added.
+* OpenMP support is no longer considered experimental. All of OpenMP 3 is
+  supported, along with much of OpenMP 4.0 and some parts of later standards.
 
 ## Bug Fixes
 

>From 01f8cc7871be266e26c6b74ef450e8e93052226f Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Fri, 11 Jul 2025 09:33:52 +0000
Subject: [PATCH 2/3] Fix supported standard version

---
 flang/docs/ReleaseNotes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index cbd3f9f4a4e78..8e5867c89ad8f 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -25,7 +25,7 @@ page](https://llvm.org/releases/).
 ## Major New Features
 
 * Initial support for VOLATILE variables and procedure interface arguments has been added.
-* OpenMP support is no longer considered experimental. All of OpenMP 3 is
+* OpenMP support is no longer considered experimental. All of OpenMP 3.1 is
   supported, along with much of OpenMP 4.0 and some parts of later standards.
 
 ## Bug Fixes

>From 8fdca9b1fc4e91bb0ad2af3f2622e61ca458d5ba Mon Sep 17 00:00:00 2001
From: Tom Eccles <t at freedommail.info>
Date: Fri, 11 Jul 2025 11:28:45 +0100
Subject: [PATCH 3/3] Reword OpenMP support desciption

Co-authored-by: Kiran Chandramohan <kiranchandramohan at gmail.com>
---
 flang/docs/ReleaseNotes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index 8e5867c89ad8f..dfaa2eb6e03fe 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -25,7 +25,7 @@ page](https://llvm.org/releases/).
 ## Major New Features
 
 * Initial support for VOLATILE variables and procedure interface arguments has been added.
-* OpenMP support is no longer considered experimental. All of OpenMP 3.1 is
+* OpenMP support is stable and no longer considered experimental. All of OpenMP 3.1 is
   supported, along with much of OpenMP 4.0 and some parts of later standards.
 
 ## Bug Fixes


From flang-commits at lists.llvm.org  Fri Jul 11 04:06:22 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Fri, 11 Jul 2025 04:06:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][docs] Update release notes for
	OpenMP (PR #147988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147988@github.com>
Message-ID: <6870f02e.170a0220.3a4a61.d0c6@mx.google.com>


https://github.com/kparzysz approved this pull request.


https://github.com/llvm/llvm-project/pull/147988

From flang-commits at lists.llvm.org  Fri Jul 11 04:18:42 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 04:18:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][docs] Update release notes for
	OpenMP (PR #147988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147988@github.com>
Message-ID: <6870f312.050a0220.22ff10.e893@mx.google.com>


https://github.com/tblah closed https://github.com/llvm/llvm-project/pull/147988

From flang-commits at lists.llvm.org  Fri Jul 11 04:46:06 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Fri, 11 Jul 2025 04:46:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Use
 OmpDirectiveSpecification in DISPATCH (PR #148008)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148008@github.com>
Message-ID: <6870f97e.170a0220.301918.dba7@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/148008

>From 0f4462c45fd80c5417a0970919e600b08ea6e2fa Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Thu, 10 Jul 2025 07:55:32 -0500
Subject: [PATCH 1/3] [flang][OpenMP] Convert AST node for ALLOCATORS to use
 Block as body

The ALLOCATORS construct is one of the few constructs that require a
special form of the associated block.
Convert the AST node to use OmpDirectiveSpecification for the directive
and the optional end directive, and to use parser::Block as the body:
the form of the block is checked in the semantic checks (with a more
meaningful message).
---
 flang/examples/FeatureList/FeatureList.cpp    |  1 -
 flang/include/flang/Parser/dump-parse-tree.h  |  1 -
 flang/include/flang/Parser/parse-tree.h       | 21 ++--
 flang/lib/Parser/openmp-parsers.cpp           | 36 +++++--
 flang/lib/Parser/unparse.cpp                  | 24 ++---
 flang/lib/Semantics/check-omp-atomic.cpp      | 31 ------
 flang/lib/Semantics/check-omp-structure.cpp   | 46 ++++++---
 flang/lib/Semantics/openmp-utils.cpp          | 20 ++++
 flang/lib/Semantics/openmp-utils.h            | 13 +++
 flang/lib/Semantics/resolve-directives.cpp    | 97 +++++++++++++------
 .../test/Parser/OpenMP/allocators-unparse.f90 | 80 ++++++++-------
 flang/test/Semantics/OpenMP/allocators07.f90  | 27 ++++++
 12 files changed, 253 insertions(+), 144 deletions(-)
 create mode 100644 flang/test/Semantics/OpenMP/allocators07.f90

diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index a36b8719e365d..e9aeed18ab0b7 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -472,7 +472,6 @@ struct NodeVisitor {
   READ_FEATURE(OmpIteration)
   READ_FEATURE(OmpIterationOffset)
   READ_FEATURE(OmpIterationVector)
-  READ_FEATURE(OmpEndAllocators)
   READ_FEATURE(OmpEndBlockDirective)
   READ_FEATURE(OmpEndCriticalDirective)
   READ_FEATURE(OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..73c224e3ad235 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -578,7 +578,6 @@ class ParseTreeDumper {
   NODE(parser, OmpDetachClause)
   NODE(parser, OmpDoacrossClause)
   NODE(parser, OmpDestroyClause)
-  NODE(parser, OmpEndAllocators)
   NODE(parser, OmpEndBlockDirective)
   NODE(parser, OmpEndCriticalDirective)
   NODE(parser, OmpEndLoopDirective)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 43954ff735361..fbc39286a95bf 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4598,8 +4598,11 @@ struct OmpClauseList {
 struct OmpDirectiveSpecification {
   ENUM_CLASS(Flags, None, DeprecatedSyntax);
   TUPLE_CLASS_BOILERPLATE(OmpDirectiveSpecification);
+  const OmpDirectiveName &DirName() const {
+    return std::get<OmpDirectiveName>(t);
+  }
   llvm::omp::Directive DirId() const { //
-    return std::get<OmpDirectiveName>(t).v;
+    return DirName().v;
   }
   const OmpArgumentList &Arguments() const;
   const OmpClauseList &Clauses() const;
@@ -4839,17 +4842,17 @@ struct OpenMPExecutableAllocate {
       t;
 };
 
-EMPTY_CLASS(OmpEndAllocators);
-
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
+// Ref: [5.2:180-181], [6.0:315]
+//
+// allocators-construct ->
+//    ALLOCATORS [allocate-clause...]
+//    block
+//    [END ALLOCATORS]
 struct OpenMPAllocatorsConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPAllocatorsConstruct);
   CharBlock source;
-  std::tuple<Verbatim, OmpClauseList, Statement<AllocateStmt>,
-      std::optional<OmpEndAllocators>>
+  std::tuple<OmpDirectiveSpecification, Block,
+      std::optional<OmpDirectiveSpecification>>
       t;
 };
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 3016ce4ccd2f8..811ca2c855a6e 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1276,6 +1276,32 @@ struct OmpEndDirectiveParser {
   llvm::omp::Directive dir_;
 };
 
+struct OmpAllocatorsConstructParser {
+  using resultType = OpenMPAllocatorsConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_allocators) {
+      return std::nullopt;
+    }
+
+    // This should be an allocate-stmt. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_allocators}};
+    return OpenMPAllocatorsConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
+    construct<OpenMPAllocatorsConstruct>(
+        "ALLOCATORS"_tok >= OmpAllocatorsConstructParser{})))
+
 // Parser for an arbitrary OpenMP ATOMIC construct.
 //
 // Depending on circumstances, an ATOMIC construct applies to one or more
@@ -1622,16 +1648,6 @@ TYPE_PARSER(
         maybe(nonemptyList(Parser<OpenMPDeclarativeAllocate>{})) / endOmpLine,
         statement(allocateStmt))))
 
-// 6.7 Allocators construct [OpenMP 5.2]
-//     allocators-construct -> ALLOCATORS [allocate-clause [,]]
-//                                allocate-stmt
-//                             [omp-end-allocators-construct]
-TYPE_PARSER(sourced(construct<OpenMPAllocatorsConstruct>(
-    verbatim("ALLOCATORS"_tok), Parser<OmpClauseList>{} / endOmpLine,
-    statement(allocateStmt), maybe(Parser<OmpEndAllocators>{} / endOmpLine))))
-
-TYPE_PARSER(construct<OmpEndAllocators>(startOmpLine >> "END ALLOCATORS"_tok))
-
 // 2.8.2 Declare Simd construct
 TYPE_PARSER(sourced(construct<OpenMPDeclareSimdConstruct>(
     verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index dcd1ac165adc1..4f692f3e9084f 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2571,7 +2571,7 @@ class UnparseVisitor {
     Word(ToUpperCaseLetters(common::EnumToString(x)));
   }
 
-  void Unparse(const OpenMPAtomicConstruct &x) {
+  template <typename Construct> void UnparseBlockConstruct(const Construct &x) {
     BeginOpenMP();
     Word("!$OMP ");
     Walk(std::get<OmpDirectiveSpecification>(x.t));
@@ -2587,6 +2587,10 @@ class UnparseVisitor {
     }
   }
 
+  void Unparse(const OpenMPAtomicConstruct &x) { //
+    UnparseBlockConstruct(x);
+  }
+
   void Unparse(const OpenMPExecutableAllocate &x) {
     const auto &fields =
         std::get<std::optional<std::list<parser::OpenMPDeclarativeAllocate>>>(
@@ -2614,22 +2618,8 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
-  void Unparse(const OmpEndAllocators &x) {
-    BeginOpenMP();
-    Word("!$OMP END ALLOCATE");
-    Put("\n");
-    EndOpenMP();
-  }
-  void Unparse(const OpenMPAllocatorsConstruct &x) {
-    BeginOpenMP();
-    Word("!$OMP ALLOCATE");
-    Walk(std::get<OmpClauseList>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<Statement<AllocateStmt>>(x.t));
-    if (const auto &end = std::get<std::optional<OmpEndAllocators>>(x.t)) {
-      Walk(*end);
-    }
+  void Unparse(const OpenMPAllocatorsConstruct &x) { //
+    UnparseBlockConstruct(x);
   }
   void Unparse(const OmpAssumeDirective &x) {
     BeginOpenMP();
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index 047c604693460..c5ed8796f0c34 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -47,43 +47,12 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
   return !(e == f);
 }
 
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
 struct AnalyzedCondStmt {
   SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
   parser::CharBlock source;
   SourcedActionStmt ift, iff;
 };
 
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
 // Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
 // is that the ActionStmt will be either an assignment or a pointer-assignment,
 // otherwise return std::nullopt.
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 89c1565bf66aa..9b89d19a91f38 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -495,6 +495,12 @@ template <typename Checker> struct DirectiveSpellingVisitor {
   template <typename T> bool Pre(const T &) { return true; }
   template <typename T> void Post(const T &) {}
 
+  template <typename... Ts>
+  static const parser::OmpDirectiveName &GetDirName(
+      const std::tuple<Ts...> &t) {
+    return std::get<parser::OmpDirectiveSpecification>(t).DirName();
+  }
+
   bool Pre(const parser::OmpSectionsDirective &x) {
     checker_(x.source, x.v);
     return false;
@@ -520,8 +526,7 @@ template <typename Checker> struct DirectiveSpellingVisitor {
     return false;
   }
   bool Pre(const parser::OpenMPAllocatorsConstruct &x) {
-    checker_(
-        std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocators);
+    checker_(GetDirName(x.t).source, Directive::OMPD_allocators);
     return false;
   }
   bool Pre(const parser::OmpAssumeDirective &x) {
@@ -1657,26 +1662,45 @@ void OmpStructureChecker::Leave(const parser::OpenMPExecutableAllocate &x) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPAllocatorsConstruct &x) {
   isPredefinedAllocator = true;
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+  PushContextAndClauseSets(
+      dirSpec.DirName().source, llvm::omp::Directive::OMPD_allocators);
+
+  if (block.empty()) {
+    context_.Say(dirSpec.source,
+        "The ALLOCATORS construct should contain a single ALLOCATE statement"_err_en_US);
+    return;
+  }
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const auto *allocate{
+      action ? parser::Unwrap<parser::AllocateStmt>(action.stmt) : nullptr};
+
+  if (!allocate) {
+    const parser::CharBlock &source = action ? action.source : x.source;
+    context_.Say(source,
+        "The body of the ALLOCATORS construct should be an ALLOCATE statement"_err_en_US);
+  }
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             parser::Unwrap<parser::OmpClause::Allocate>(clause)}) {
       CheckVarIsNotPartOfAnotherVar(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       CheckPredefinedAllocatorRestriction(
-          dir.source, std::get<parser::OmpObjectList>(allocClause->v.t));
+          dirSpec.source, std::get<parser::OmpObjectList>(allocClause->v.t));
     }
   }
   dirContext_.pop_back();
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index fd9596a09cd52..f43d2cc75620e 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -38,6 +38,26 @@
 
 namespace Fortran::semantics::omp {
 
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
 std::string ThisVersion(unsigned version) {
   std::string tv{
       std::to_string(version / 10) + "." + std::to_string(version % 10)};
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index dbb0565215357..a96c008fb26e7 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -29,6 +29,19 @@ class Symbol;
 
 // Add this namespace to avoid potential conflicts
 namespace omp {
+// There is no consistent way to get the source of an ActionStmt, but there
+// is "source" in Statement<T>. This structure keeps the ActionStmt with the
+// extracted source for further use.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+SourcedActionStmt GetActionStmt(const parser::ExecutionPartConstruct *x);
+SourcedActionStmt GetActionStmt(const parser::Block &block);
+
 std::string ThisVersion(unsigned version);
 std::string TryVersion(unsigned version);
 
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 299bb6ff876e7..bb3914ffa2b1e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -10,6 +10,7 @@
 
 #include "check-acc-structure.h"
 #include "check-omp-structure.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/fold.h"
@@ -353,12 +354,6 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return true;
   }
 
-  bool Pre(const parser::OmpDirectiveSpecification &x) {
-    PushContext(x.source, x.DirId());
-    return true;
-  }
-  void Post(const parser::OmpDirectiveSpecification &) { PopContext(); }
-
   bool Pre(const parser::OmpMetadirectiveDirective &x) {
     PushContext(x.source, llvm::omp::Directive::OMPD_metadirective);
     return true;
@@ -372,6 +367,29 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     GetContext().withinConstruct = true;
   }
 
+  bool Pre(const parser::OpenMPStandaloneConstruct &x) {
+    common::visit(
+        [&](auto &&s) {
+          using TypeS = llvm::remove_cvref_t<decltype(s)>;
+          // These two cases are handled individually.
+          if constexpr ( //
+              !std::is_same_v<TypeS, parser::OpenMPSimpleStandaloneConstruct> &&
+              !std::is_same_v<TypeS, parser::OmpMetadirectiveDirective>) {
+            PushContext(x.source, s.v.DirId());
+          }
+        },
+        x.u);
+    return true;
+  }
+
+  void Post(const parser::OpenMPStandaloneConstruct &x) {
+    // These two cases are handled individually.
+    if (!std::holds_alternative<parser::OpenMPSimpleStandaloneConstruct>(x.u) &&
+        !std::holds_alternative<parser::OmpMetadirectiveDirective>(x.u)) {
+      PopContext();
+    }
+  }
+
   bool Pre(const parser::OpenMPSimpleStandaloneConstruct &);
   void Post(const parser::OpenMPSimpleStandaloneConstruct &) { PopContext(); }
 
@@ -2149,9 +2167,10 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPExecutableAllocate &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPAllocatorsConstruct &x) {
-  PushContext(x.source, llvm::omp::Directive::OMPD_allocators);
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  PushContext(x.source, dirSpec.DirId());
+
+  for (const auto &clause : dirSpec.Clauses().v) {
     if (const auto *allocClause{
             std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
       ResolveOmpObjectList(std::get<parser::OmpObjectList>(allocClause->v.t),
@@ -2234,31 +2253,47 @@ void OmpAttributeVisitor::Post(const parser::OpenMPExecutableAllocate &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
-  const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
-  for (const auto &clause : clauseList.v) {
-    if (const auto *alloc{
-            std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
-      CheckAllNamesInAllocateStmt(dir.source,
-          std::get<parser::OmpObjectList>(alloc->v.t),
-          std::get<parser::Statement<parser::AllocateStmt>>(x.t).statement);
-
-      auto &modifiers{OmpGetModifiers(alloc->v)};
-      bool hasAllocator{
-          OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(modifiers) ||
-          OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(modifiers)};
-
-      // TODO: As with allocate directive, exclude the case when a requires
-      //       directive with the dynamic_allocators clause is present in
-      //       the same compilation unit (OMP5.0 2.11.3).
-      if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
-          !hasAllocator) {
-        context_.Say(x.source,
-            "ALLOCATORS directives that appear in a TARGET region "
-            "must specify an allocator"_err_en_US);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  const parser::AllocateStmt *allocate{[&]() {
+    if (action) {
+      if (auto *alloc{std::get_if<common::Indirection<parser::AllocateStmt>>(
+              &action.stmt->u)}) {
+        return &alloc->value();
+      }
+    }
+    return static_cast<const parser::AllocateStmt *>(nullptr);
+  }()};
+
+  if (allocate) {
+    for (const auto &clause : dirSpec.Clauses().v) {
+      if (auto *alloc{std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
+        CheckAllNamesInAllocateStmt(
+            x.source, std::get<parser::OmpObjectList>(alloc->v.t), *allocate);
+
+        using OmpAllocatorSimpleModifier = parser::OmpAllocatorSimpleModifier;
+        using OmpAllocatorComplexModifier = parser::OmpAllocatorComplexModifier;
+
+        auto &modifiers{OmpGetModifiers(alloc->v)};
+        bool hasAllocator{
+            OmpGetUniqueModifier<OmpAllocatorSimpleModifier>(modifiers) ||
+            OmpGetUniqueModifier<OmpAllocatorComplexModifier>(modifiers)};
+
+        // TODO: As with allocate directive, exclude the case when a requires
+        //       directive with the dynamic_allocators clause is present in
+        //       the same compilation unit (OMP5.0 2.11.3).
+        if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
+            !hasAllocator) {
+          context_.Say(x.source,
+              "ALLOCATORS directives that appear in a TARGET region "
+              "must specify an allocator"_err_en_US);
+        }
       }
     }
   }
+
   PopContext();
 }
 
diff --git a/flang/test/Parser/OpenMP/allocators-unparse.f90 b/flang/test/Parser/OpenMP/allocators-unparse.f90
index 5cd0230471fc4..70feb7a6b527e 100644
--- a/flang/test/Parser/OpenMP/allocators-unparse.f90
+++ b/flang/test/Parser/OpenMP/allocators-unparse.f90
@@ -15,48 +15,62 @@ subroutine allocate()
 
   !$omp allocators allocate(align(32): arr2)
     allocate(arr2(5, 3))
+  !$omp end allocators
 end subroutine allocate
 
 !CHECK: INTEGER, ALLOCATABLE :: arr1(:), arr2(:,:)
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(omp_default_mem_alloc: arr1)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(omp_default_mem_alloc: arr1)
 !CHECK-NEXT: ALLOCATE(arr1(5))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) ALL&
-!CHECK-NEXT:!$OMP&OCATE(omp_default_mem_alloc: arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) A&
+!CHECK-NEXT:!$OMP&LLOCATE(omp_default_mem_alloc: arr2)
 !CHECK-NEXT: ALLOCATE(arr1(10), arr2(3,2))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALIGN(32): arr2)
+!CHECK-NEXT:!$OMP ALLOCATORS ALLOCATE(ALIGN(32): arr2)
 !CHECK-NEXT: ALLOCATE(arr2(5,3))
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr1'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr1'
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorComplexModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
-!PARSE-TREE-NEXT: OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr1'
-!PARSE-TREE-NEXT: AllocateShapeSpec
-!PARSE-TREE-NEXT: Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '10'
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorComplexModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
+!PARSE-TREE-NEXT: | | OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr1'
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
-!PARSE-TREE-NEXT: Verbatim
-!PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
-!PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
-!PARSE-TREE-NEXT: AllocateStmt
-!PARSE-TREE-NEXT: Allocation
-!PARSE-TREE-NEXT: AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
+!PARSE-TREE-NEXT: | | | Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
+!PARSE-TREE-NEXT: | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | Flags = None
+!PARSE-TREE-NEXT: | Block
+!PARSE-TREE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AllocateStmt
+!PARSE-TREE-NEXT: | | | Allocation
+!PARSE-TREE-NEXT: | | | | AllocateObject -> Name = 'arr2'
+!PARSE-TREE-NEXT: | | | | AllocateShapeSpec
+!PARSE-TREE-NEXT: | | | | | Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '5'
+!PARSE-TREE-NEXT: | | | | AllocateShapeSpec
+!PARSE-TREE-NEXT: | | | | | Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '3'
+!PARSE-TREE-NEXT: | OmpDirectiveSpecification
+!PARSE-TREE-NEXT: | | OmpDirectiveName -> llvm::omp::Directive = allocators
+!PARSE-TREE-NEXT: | | OmpClauseList ->
+!PARSE-TREE-NEXT: | | Flags = None
diff --git a/flang/test/Semantics/OpenMP/allocators07.f90 b/flang/test/Semantics/OpenMP/allocators07.f90
new file mode 100644
index 0000000000000..a28f706965cb1
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/allocators07.f90
@@ -0,0 +1,27 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
+
+subroutine f00
+  implicit none
+  integer, allocatable :: a(:)
+
+  !$omp allocators allocate(a)
+!ERROR: The body of the ALLOCATORS construct should be an ALLOCATE statement
+  continue
+end
+
+subroutine f01
+  implicit none
+  integer, allocatable :: a(:)
+
+!ERROR: The ALLOCATORS construct should contain a single ALLOCATE statement
+  !$omp allocators allocate(a)
+  !$omp end allocators
+end
+
+subroutine f02
+  implicit none
+  integer, allocatable :: a(:)
+
+!ERROR: The ALLOCATORS construct should contain a single ALLOCATE statement
+  !$omp allocators allocate(a)
+end

>From 1bcd318939236190a30cfb3259bcb9ca972f1fd3 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Thu, 10 Jul 2025 10:18:32 -0500
Subject: [PATCH 2/3] [flang][OpenMP] Use OmpDirectiveSpecification in DISPATCH

Dispatch is the last construct (after ATOMIC and ALLOCATORS) where the
associated block requires a specific form.
Using OmpDirectiveSpecification for the begin and the optional end
directives will make the structure of all block directives more uniform.
---
 flang/include/flang/Parser/dump-parse-tree.h |  2 -
 flang/include/flang/Parser/parse-tree.h      | 12 +---
 flang/lib/Parser/openmp-parsers.cpp          | 36 +++++++---
 flang/lib/Parser/unparse.cpp                 | 12 +---
 flang/lib/Semantics/check-omp-structure.cpp  | 25 ++++---
 flang/test/Parser/OpenMP/dispatch.f90        | 73 ++++++++++++--------
 flang/test/Semantics/OpenMP/dispatch.f90     | 22 +++---
 7 files changed, 99 insertions(+), 83 deletions(-)

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 73c224e3ad235..32b6ca45609b6 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -710,8 +710,6 @@ class ParseTreeDumper {
   NODE(parser, OpenMPDepobjConstruct)
   NODE(parser, OpenMPUtilityConstruct)
   NODE(parser, OpenMPDispatchConstruct)
-  NODE(parser, OmpDispatchDirective)
-  NODE(parser, OmpEndDispatchDirective)
   NODE(parser, OpenMPFlushConstruct)
   NODE(parser, OpenMPLoopConstruct)
   NODE(parser, OpenMPExecutableAllocate)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index fbc39286a95bf..ab2dde7d5dfbe 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4939,19 +4939,11 @@ struct OpenMPDepobjConstruct {
 //                    nocontext-clause |
 //                    novariants-clause |
 //                    nowait-clause
-struct OmpDispatchDirective {
-  TUPLE_CLASS_BOILERPLATE(OmpDispatchDirective);
-  CharBlock source;
-  std::tuple<Verbatim, OmpClauseList> t;
-};
-
-EMPTY_CLASS(OmpEndDispatchDirective);
-
 struct OpenMPDispatchConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPDispatchConstruct);
   CharBlock source;
-  std::tuple<OmpDispatchDirective, Block,
-      std::optional<OmpEndDispatchDirective>>
+  std::tuple<OmpDirectiveSpecification, Block,
+      std::optional<OmpDirectiveSpecification>>
       t;
 };
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 811ca2c855a6e..d70aaab82cbab 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1302,6 +1302,32 @@ TYPE_PARSER(sourced( //
     construct<OpenMPAllocatorsConstruct>(
         "ALLOCATORS"_tok >= OmpAllocatorsConstructParser{})))
 
+struct OmpDispatchConstructParser {
+  using resultType = OpenMPDispatchConstruct;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_dispatch) {
+      return std::nullopt;
+    }
+
+    // This should be a function call. That will be checked in semantics.
+    Block block;
+    if (auto stmt{attempt(Parser<ExecutionPartConstruct>{}).Parse(state)}) {
+      block.emplace_back(std::move(*stmt));
+    }
+    // Allow empty block. Check for this in semantics.
+
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_dispatch}};
+    return OpenMPDispatchConstruct{
+        std::move(*dirSpec), std::move(block), *maybe(end).Parse(state)};
+  }
+};
+
+TYPE_PARSER(sourced( //
+    construct<OpenMPDispatchConstruct>(
+        "DISPATCH"_tok >= OmpDispatchConstructParser{})))
+
 // Parser for an arbitrary OpenMP ATOMIC construct.
 //
 // Depending on circumstances, an ATOMIC construct applies to one or more
@@ -1631,16 +1657,6 @@ TYPE_PARSER(sourced(construct<OmpCriticalDirective>(verbatim("CRITICAL"_tok),
 TYPE_PARSER(construct<OpenMPCriticalConstruct>(
     Parser<OmpCriticalDirective>{}, block, Parser<OmpEndCriticalDirective>{}))
 
-TYPE_PARSER(sourced(construct<OmpDispatchDirective>(
-    verbatim("DISPATCH"_tok), Parser<OmpClauseList>{})))
-
-TYPE_PARSER(
-    construct<OmpEndDispatchDirective>(startOmpLine >> "END DISPATCH"_tok))
-
-TYPE_PARSER(sourced(construct<OpenMPDispatchConstruct>(
-    Parser<OmpDispatchDirective>{} / endOmpLine, block,
-    maybe(Parser<OmpEndDispatchDirective>{} / endOmpLine))))
-
 // 2.11.3 Executable Allocate directive
 TYPE_PARSER(
     sourced(construct<OpenMPExecutableAllocate>(verbatim("ALLOCATE"_tok),
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 4f692f3e9084f..b66d756bdbf2c 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2758,6 +2758,9 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
+  void Unparse(const OpenMPDispatchConstruct &x) { //
+    UnparseBlockConstruct(x);
+  }
   void Unparse(const OpenMPRequiresConstruct &y) {
     BeginOpenMP();
     Word("!$OMP REQUIRES ");
@@ -2777,15 +2780,6 @@ class UnparseVisitor {
     Walk(x.v);
     return false;
   }
-  void Unparse(const OmpDispatchDirective &x) {
-    Word("!$OMP DISPATCH");
-    Walk(x.t);
-    Put("\n");
-  }
-  void Unparse(const OmpEndDispatchDirective &) {
-    Word("!$OMP END DISPATCH");
-    Put("\n");
-  }
   void Unparse(const OmpErrorDirective &x) {
     Word("!$OMP ERROR ");
     Walk(x.t);
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 9b89d19a91f38..2425265e196c6 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -509,8 +509,8 @@ template <typename Checker> struct DirectiveSpellingVisitor {
     checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_allocate);
     return false;
   }
-  bool Pre(const parser::OmpDispatchDirective &x) {
-    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_dispatch);
+  bool Pre(const parser::OpenMPDispatchConstruct &x) {
+    checker_(GetDirName(x.t).source, Directive::OMPD_dispatch);
     return false;
   }
   bool Pre(const parser::OmpErrorDirective &x) {
@@ -1595,28 +1595,31 @@ void OmpStructureChecker::Enter(const parser::OmpErrorDirective &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPDispatchConstruct &x) {
-  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_dispatch);
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
   const auto &block{std::get<parser::Block>(x.t)};
-  if (block.empty() || block.size() > 1) {
+  PushContextAndClauseSets(
+      dirSpec.DirName().source, llvm::omp::Directive::OMPD_dispatch);
+
+  if (block.empty()) {
     context_.Say(x.source,
-        "The DISPATCH construct is empty or contains more than one statement"_err_en_US);
+        "The DISPATCH construct should contain a single function or subroutine call"_err_en_US);
     return;
   }
 
-  auto it{block.begin()};
   bool passChecks{false};
-  if (const parser::AssignmentStmt *
-      assignStmt{parser::Unwrap<parser::AssignmentStmt>(*it)}) {
+  omp::SourcedActionStmt action{omp::GetActionStmt(block)};
+  if (const auto *assignStmt{
+          parser::Unwrap<parser::AssignmentStmt>(*action.stmt)}) {
     if (parser::Unwrap<parser::FunctionReference>(assignStmt->t)) {
       passChecks = true;
     }
-  } else if (parser::Unwrap<parser::CallStmt>(*it)) {
+  } else if (parser::Unwrap<parser::CallStmt>(*action.stmt)) {
     passChecks = true;
   }
 
   if (!passChecks) {
-    context_.Say(x.source,
-        "The DISPATCH construct does not contain a SUBROUTINE or FUNCTION"_err_en_US);
+    context_.Say(action.source,
+        "The body of the DISPATCH construct should be a function or a subroutine call"_err_en_US);
   }
 }
 
diff --git a/flang/test/Parser/OpenMP/dispatch.f90 b/flang/test/Parser/OpenMP/dispatch.f90
index 98cd6090334f3..4076c00331225 100644
--- a/flang/test/Parser/OpenMP/dispatch.f90
+++ b/flang/test/Parser/OpenMP/dispatch.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s | FileCheck %s
+! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s | FileCheck %s --check-prefix=PARSE-TREE
 ! RUN: %flang_fc1 -fopenmp -fdebug-unparse %s | FileCheck %s --check-prefix="UNPARSE"
 
 integer function func(a, b, c)
@@ -12,40 +12,57 @@ subroutine sub(x)
   integer :: r
   type(c_ptr) :: x
   integer :: a = 14, b = 7, c = 21
+
 !UNPARSE: !$OMP DISPATCH DEVICE(3_4) NOWAIT NOCONTEXT(.false._4) NOVARIANTS(.true._4)
-!CHECK: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
-!CHECK-NEXT: | | | OmpDispatchDirective
-!CHECK: | | | | OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
-!CHECK-NEXT: | | | | | Scalar -> Integer -> Expr = '3_4'
-!CHECK-NEXT: | | | | | | LiteralConstant -> IntLiteralConstant = '3'
-!CHECK-NEXT: | | | | OmpClause -> Nowait
-!CHECK-NEXT: | | | | OmpClause -> Nocontext -> Scalar -> Logical -> Expr = '.false._4'
-!CHECK-NEXT: | | | | | LiteralConstant -> LogicalLiteralConstant
-!CHECK-NEXT: | | | | | | bool = 'false'
-!CHECK-NEXT: | | | | OmpClause -> Novariants -> Scalar -> Logical -> Expr = '.true._4'
-!CHECK-NEXT: | | | | | EQ
-!CHECK-NEXT: | | | | | | Expr = '1_4'
-!CHECK-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
-!CHECK-NEXT: | | | | | | Expr = '1_4'
-!CHECK-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
-!CHECK-NEXT: | | | Block
- 
+!UNPARSE:   r=func(a,b,c)
+!UNPARSE: !$OMP END DISPATCH
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = dispatch
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
+!PARSE-TREE: | | | Scalar -> Integer -> Expr = '3_4'
+!PARSE-TREE: | | | | LiteralConstant -> IntLiteralConstant = '3'
+!PARSE-TREE: | | OmpClause -> Nowait
+!PARSE-TREE: | | OmpClause -> Nocontext -> Scalar -> Logical -> Expr = '.false._4'
+!PARSE-TREE: | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | bool = 'false'
+!PARSE-TREE: | | OmpClause -> Novariants -> Scalar -> Logical -> Expr = '.true._4'
+!PARSE-TREE: | | | EQ
+!PARSE-TREE: | | | | Expr = '1_4'
+!PARSE-TREE: | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | | | Expr = '1_4'
+!PARSE-TREE: | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt
+![...]
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = dispatch
+!PARSE-TREE: | | OmpClauseList ->
+!PARSE-TREE: | | Flags = None
+
   !$omp dispatch device(3) nowait nocontext(.false.) novariants(1.eq.1)
   r = func(a, b, c)
-!UNPARSE: !$OMP END DISPATCH
-!CHECK: | | | OmpEndDispatchDirective
   !$omp end dispatch
 
 !! Test the "no end dispatch" option.
-!UNPARSE: !$OMP DISPATCH  DEVICE(3_4) IS_DEVICE_PTR(x)
-!CHECK: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
-!CHECK-NEXT: | | | OmpDispatchDirective
-!CHECK: | | | | OmpClause -> IsDevicePtr ->  OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'  
+!UNPARSE: !$OMP DISPATCH DEVICE(3_4) IS_DEVICE_PTR(x)
+!UNPARSE:   r=func(a+1_4,b+2_4,c+3_4)
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPDispatchConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = dispatch
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
+!PARSE-TREE: | | | Scalar -> Integer -> Expr = '3_4'
+!PARSE-TREE: | | | | LiteralConstant -> IntLiteralConstant = '3'
+!PARSE-TREE: | | OmpClause -> IsDevicePtr -> OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt
+!PARSE-TREE-NOT: OmpDirectiveSpecification
+
   !$omp dispatch device(3) is_device_ptr(x)
   r = func(a+1, b+2, c+3)
-!CHECK-NOT: | | | OmpEndDispatchDirective
 
 end subroutine sub
-
-
-
diff --git a/flang/test/Semantics/OpenMP/dispatch.f90 b/flang/test/Semantics/OpenMP/dispatch.f90
index 7dfbeecb2fc1d..af0d6856ab948 100644
--- a/flang/test/Semantics/OpenMP/dispatch.f90
+++ b/flang/test/Semantics/OpenMP/dispatch.f90
@@ -1,24 +1,20 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
 
 subroutine sb1
   integer :: r
   r = 1
-  !ERROR: The DISPATCH construct does not contain a SUBROUTINE or FUNCTION
   !$omp dispatch nowait
+!ERROR: The body of the DISPATCH construct should be a function or a subroutine call
   print *,r
 end subroutine
+
 subroutine sb2
-  integer :: r
-!ERROR: The DISPATCH construct is empty or contains more than one statement
+!ERROR: The DISPATCH construct should contain a single function or subroutine call
   !$omp dispatch
-  call foo()
-  r = bar()
   !$omp end dispatch
-contains
-  subroutine foo
-  end subroutine foo
-  function bar
-    integer :: bar
-    bar = 2
-  end function
+end subroutine
+
+subroutine sb3
+!ERROR: The DISPATCH construct should contain a single function or subroutine call
+  !$omp dispatch
 end subroutine

>From 2fd6d4c21c03252db5a7211b81ef0894a7fb2d46 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 06:02:29 -0500
Subject: [PATCH 3/3] Remove extra whitespace

---
 flang/lib/Semantics/resolve-directives.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index bb3914ffa2b1e..151f4ccae634e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -2293,7 +2293,6 @@ void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
       }
     }
   }
-
   PopContext();
 }
 


From flang-commits at lists.llvm.org  Fri Jul 11 04:47:34 2025
From: flang-commits at lists.llvm.org (Kajetan Puchalski via flang-commits)
Date: Fri, 11 Jul 2025 04:47:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add more flags for generating
	codegen tests (PR #148171)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148171@github.com>
Message-ID: <6870f9d6.630a0220.afe42.d9e4@mx.google.com>


https://github.com/mrkajetanp approved this pull request.

Very useful, LGTM, thanks!

https://github.com/llvm/llvm-project/pull/148171

From flang-commits at lists.llvm.org  Fri Jul 11 04:50:20 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 04:50:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add more flags for generating
	codegen tests (PR #148171)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148171@github.com>
Message-ID: <6870fa7c.170a0220.38a1f8.d711@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/148171

From flang-commits at lists.llvm.org  Fri Jul 11 07:17:30 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Fri, 11 Jul 2025 07:17:30 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <68711cfa.170a0220.18f75e.f1e7@mx.google.com>


DavidSpickett wrote:

Thanks, I'll land this Monday when I am here to catch any fallout.

> Works for me, and I have verified that this doesn't lead to fortran executables having new runtime dependencies.

Is this something the build checks? Idk if you can just look at what flang-rt links to, or whether you have to link a fortran program and look at that. If it's not documented how to check that, it would be nice to have written down.

https://github.com/llvm/llvm-project/pull/148169

From flang-commits at lists.llvm.org  Fri Jul 11 07:41:05 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Fri, 11 Jul 2025 07:41:05 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <68712281.050a0220.1520a8.2679@mx.google.com>


tblah wrote:

I'm not sure how to check what a static library links to. If it is dynamic you can use `ldd`. For static libraries I build a Fortran program and see what it links to. I don't want to document that because I wonder if some of the dependencies could get removed from parts of the runtime not being used in that program - it might not be entirely robust.

I dimly remember there being a bot somewhere that kept an eye on these dependencies for flang-rt but whenever I heard about it was a long time ago.

https://github.com/llvm/llvm-project/pull/148169

From flang-commits at lists.llvm.org  Fri Jul 11 08:18:40 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 11 Jul 2025 08:18:40 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <68712b50.050a0220.34ad7c.50d9@mx.google.com>


================
@@ -446,6 +446,16 @@ struct AssociateOpConversion
             ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
               !mlir::isa<fir::BoxCharType>(assocType)))) {
           sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
+        } else if (mlir::isa<fir::ReferenceType>(sourceVar.getType()) &&
+                   mlir::isa<fir::BoxCharType>(assocType)) {
+          auto refTy = mlir::cast<fir::ReferenceType>(sourceVar.getType());
+          auto charTy = mlir::dyn_cast<fir::CharacterType>(refTy.getEleTy());
----------------
vzakhari wrote:

Thanks for the update!

This will only work for the types with constant length.  I do not know if we should be here for non-constant lengths, but I would just take the type parameters from the `associate` operation - this should work for all the cases.

https://github.com/llvm/llvm-project/pull/146851

From flang-commits at lists.llvm.org  Fri Jul 11 08:19:06 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 11 Jul 2025 08:19:06 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Solving issue with adjustr
 intrinsic in where construct (PR #146851)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146851@github.com>
Message-ID: <68712b6a.170a0220.3b849c.41d5@mx.google.com>


================
@@ -0,0 +1,14 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+! Checking no reference to boxchar conversion is in the emitted fir.
+! CHECK-NOT: (!fir.ref<!fir.char<1,4>>) -> !fir.boxchar<1>
----------------
vzakhari wrote:

Please update the checks to show what kind of emboxchar we generate after this change.

https://github.com/llvm/llvm-project/pull/146851

From flang-commits at lists.llvm.org  Fri Jul 11 08:20:16 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 11 Jul 2025 08:20:16 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][Interfaces] Remove negative
 branch weight verifier (PR #148234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148234@github.com>
Message-ID: <68712bb0.170a0220.3bc0df.4e75@mx.google.com>


https://github.com/vzakhari approved this pull request.

Thank you for the fix!

https://github.com/llvm/llvm-project/pull/148234

From flang-commits at lists.llvm.org  Fri Jul 11 09:07:48 2025
From: flang-commits at lists.llvm.org (Scott Manley via flang-commits)
Date: Fri, 11 Jul 2025 09:07:48 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir] Small clean-up in
 `fir_DoConcurrentLoopOp`'s defintion (PR #146028)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146028@github.com>
Message-ID: <687136d4.050a0220.360f73.5898@mx.google.com>


rscottmanley wrote:

Please include @khaki3 on future do concurrent work, thanks!

https://github.com/llvm/llvm-project/pull/146028

From flang-commits at lists.llvm.org  Fri Jul 11 11:29:43 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 11 Jul 2025 11:29:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][acc] Create UseDeviceOp for both
 results of hlfir.declare (PR #148017)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148017@github.com>
Message-ID: <68715817.170a0220.66dfd.6f29@mx.google.com>


https://github.com/nvptm updated https://github.com/llvm/llvm-project/pull/148017

>From 0fb4bc6169d76081b641c78db1c032ab876a6905 Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Wed, 9 Jul 2025 23:13:49 -0700
Subject: [PATCH 1/6] use_device for all other results of hlfir.declare

---
 flang/lib/Lower/OpenACC.cpp | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 42842bcb41a74..4f637b88fd269 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -724,7 +724,7 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
             /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
             /*genDefaultBounds=*/generateDefaultBounds,
             /*strideIncludeLowerExtent=*/strideIncludeLowerExtent);
-    LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
+    LLVM_DEBUG(llvm::dbgs() << __func__ << "Here \n"; info.dump(llvm::dbgs()));
 
     // If the input value is optional and is not a descriptor, we use the
     // rawInput directly.
@@ -738,6 +738,21 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
         implicit, dataClause, baseAddr.getType(), async, asyncDeviceTypes,
         asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
     dataOperands.push_back(op.getAccVar());
+    // If the input value has a descriptor, we need to create a use device op
+    // for the descriptor as well as the base address.
+    if constexpr (std::is_same_v<Op, mlir::acc::UseDeviceOp>) {
+      LLVM_DEBUG(llvm::dbgs() << __func__ << " found usedeviceop \n"; info.dump(llvm::dbgs()));
+      LLVM_DEBUG(llvm::dbgs() << __func__ << " had previously created and added usedeviceop \n"; op.dump());
+      if (mlir::isa<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
+        Op op = createDataEntryOp<Op>(
+            builder, operandLocation, baseAddr.getDefiningOp()->getResult(1), asFortran, bounds, structured,
+            implicit, dataClause, baseAddr.getDefiningOp()->getResult(1).getType(), async, asyncDeviceTypes,
+            asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
+        LLVM_DEBUG(llvm::dbgs() << __func__ << " created usedeviceop \n"; op.dump());
+        dataOperands.push_back(op.getAccVar());
+        LLVM_DEBUG(llvm::dbgs() << __func__ << "added usedeviceop on descriptor\n"; info.dump(llvm::dbgs()));
+      }
+    }
   }
 }
 

>From be91a9ab05a43dff381347f0e6f49dbbe019669f Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Thu, 10 Jul 2025 09:19:58 -0700
Subject: [PATCH 2/6] update tests

---
 .../acc-host-data-unwrap-defaultbounds.f90    | 14 +++++++------
 flang/test/Lower/OpenACC/acc-host-data.f90    | 21 +++++++++++--------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
index 164eb32a8f684..2de7cc5761a2b 100644
--- a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
@@ -15,15 +15,17 @@ subroutine acc_host_data()
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+ ! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>{{.*}}) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
@@ -33,14 +35,14 @@ subroutine acc_host_data()
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.false.)
     a = 1.0
diff --git a/flang/test/Lower/OpenACC/acc-host-data.f90 b/flang/test/Lower/OpenACC/acc-host-data.f90
index 871eabd256ca6..4d09b25b983b9 100644
--- a/flang/test/Lower/OpenACC/acc-host-data.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data.f90
@@ -14,34 +14,37 @@ subroutine acc_host_data()
   !$acc host_data use_device(a)
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 ! CHECK: } attributes {ifPresent}
 
-  !$acc host_data use_device(a) if_present if_present
+  !$acc host_data use_device(a) if_present 
   !$acc end host_data
-! CHECK: acc.host_data dataOperands(%{{.*}} : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: acc.host_data dataOperands(%{{.*}}{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}}) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA0]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.false.)
     a = 1.0

>From ec7ad5b11cc46322c6188bafb23f89cb3767ad70 Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Thu, 10 Jul 2025 09:58:36 -0700
Subject: [PATCH 3/6] Remove debug messages. Format file.

---
 flang/lib/Lower/OpenACC.cpp | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 4f637b88fd269..d43345021063b 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -738,19 +738,17 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
         implicit, dataClause, baseAddr.getType(), async, asyncDeviceTypes,
         asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
     dataOperands.push_back(op.getAccVar());
-    // If the input value has a descriptor, we need to create a use device op
-    // for the descriptor as well as the base address.
+    // For UseDeviceOp, if operand is one of a pair resulting from a
+    // declare operation, create a UseDeviceOp for the other operand as well.
     if constexpr (std::is_same_v<Op, mlir::acc::UseDeviceOp>) {
-      LLVM_DEBUG(llvm::dbgs() << __func__ << " found usedeviceop \n"; info.dump(llvm::dbgs()));
-      LLVM_DEBUG(llvm::dbgs() << __func__ << " had previously created and added usedeviceop \n"; op.dump());
       if (mlir::isa<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
         Op op = createDataEntryOp<Op>(
-            builder, operandLocation, baseAddr.getDefiningOp()->getResult(1), asFortran, bounds, structured,
-            implicit, dataClause, baseAddr.getDefiningOp()->getResult(1).getType(), async, asyncDeviceTypes,
-            asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
-        LLVM_DEBUG(llvm::dbgs() << __func__ << " created usedeviceop \n"; op.dump());
+            builder, operandLocation, baseAddr.getDefiningOp()->getResult(1),
+            asFortran, bounds, structured, implicit, dataClause,
+            baseAddr.getDefiningOp()->getResult(1).getType(), async,
+            asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true,
+            info.isPresent);
         dataOperands.push_back(op.getAccVar());
-        LLVM_DEBUG(llvm::dbgs() << __func__ << "added usedeviceop on descriptor\n"; info.dump(llvm::dbgs()));
       }
     }
   }

>From c385240e757bc2e6c127304a36bae1f1e9d02d60 Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Thu, 10 Jul 2025 09:59:57 -0700
Subject: [PATCH 4/6] Remove debug messages.

---
 flang/lib/Lower/OpenACC.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index d43345021063b..23481d0ef7935 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -724,7 +724,7 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
             /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
             /*genDefaultBounds=*/generateDefaultBounds,
             /*strideIncludeLowerExtent=*/strideIncludeLowerExtent);
-    LLVM_DEBUG(llvm::dbgs() << __func__ << "Here \n"; info.dump(llvm::dbgs()));
+    LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
 
     // If the input value is optional and is not a descriptor, we use the
     // rawInput directly.

>From 675acb2b849edf2519e11b0e3dba38fd3d26603e Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Fri, 11 Jul 2025 11:19:00 -0700
Subject: [PATCH 5/6] Check two operands are different. Refactor code. Add
 motivating test and for assumed shape, pointer and allocatable

---
 flang/lib/Lower/OpenACC.cpp                 | 19 ++++---
 flang/test/Lower/OpenACC/acc-use-device.f90 | 61 +++++++++++++++++++++
 2 files changed, 73 insertions(+), 7 deletions(-)
 create mode 100644 flang/test/Lower/OpenACC/acc-use-device.f90

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 23481d0ef7935..9234dd33c8b54 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -709,6 +709,7 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
                          bool setDeclareAttr = false) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   Fortran::evaluate::ExpressionAnalyzer ea{semanticsContext};
+  const bool unwrapBoxAddr = true;
   for (const auto &accObject : objectList.v) {
     llvm::SmallVector<mlir::Value> bounds;
     std::stringstream asFortran;
@@ -736,19 +737,23 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
     Op op = createDataEntryOp<Op>(
         builder, operandLocation, baseAddr, asFortran, bounds, structured,
         implicit, dataClause, baseAddr.getType(), async, asyncDeviceTypes,
-        asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
+        asyncOnlyDeviceTypes, unwrapBoxAddr, info.isPresent);
     dataOperands.push_back(op.getAccVar());
+
     // For UseDeviceOp, if operand is one of a pair resulting from a
     // declare operation, create a UseDeviceOp for the other operand as well.
     if constexpr (std::is_same_v<Op, mlir::acc::UseDeviceOp>) {
-      if (mlir::isa<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
-        Op op = createDataEntryOp<Op>(
-            builder, operandLocation, baseAddr.getDefiningOp()->getResult(1),
+      if (auto declareOp = mlir::dyn_cast<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
+        mlir::Value otherAddr = declareOp.getResult(1);
+        if (baseAddr != otherAddr) {
+          Op op = createDataEntryOp<Op>(
+            builder, operandLocation, otherAddr,
             asFortran, bounds, structured, implicit, dataClause,
-            baseAddr.getDefiningOp()->getResult(1).getType(), async,
-            asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true,
+            otherAddr.getType(), async,
+            asyncDeviceTypes, asyncOnlyDeviceTypes, unwrapBoxAddr,
             info.isPresent);
-        dataOperands.push_back(op.getAccVar());
+          dataOperands.push_back(op.getAccVar());
+        }
       }
     }
   }
diff --git a/flang/test/Lower/OpenACC/acc-use-device.f90 b/flang/test/Lower/OpenACC/acc-use-device.f90
new file mode 100644
index 0000000000000..081a6e317bfc9
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-use-device.f90
@@ -0,0 +1,61 @@
+! This test checks whether the OpenACC use_device clause is applied on both results of hlfir.declare.
+
+! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
+
+! Test for automatic variable appearing in use_device clause.
+subroutine test()
+  integer :: N = 100
+  real*8 :: b(-1:N)
+! CHECK: %[[A0:.*]] = fir.alloca !fir.array<?xf64>, %{{.*}} {bindc_name = "b", uniq_name = "_QFtestEb"}
+! CHECK: %[[A1:.*]] = fir.shape_shift {{.*}} : (index, index) -> !fir.shapeshift<1>
+! CHECK: %[[A:.*]]:2 = hlfir.declare %[[A0]](%[[A1]]) {uniq_name = "_QFtestEb"} : (!fir.ref<!fir.array<?xf64>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>)
+  
+  !$acc data copy(b)
+! CHECK: %[[B:.*]] = acc.copyin var(%[[A]]#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK: acc.data dataOperands(%[[B]] : !fir.box<!fir.array<?xf64>>) {
+
+  !$acc host_data use_device(b)
+  call vadd(b)
+  !$acc end host_data
+! CHECK: %[[C:.*]] = acc.use_device var(%[[A]]#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
+! CHECK: %[[D:.*]] = acc.use_device varPtr(%[[A]]#1 : !fir.ref<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>> {name = "b"}
+! CHECK: acc.host_data dataOperands(%[[C]], %[[D]] : !fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>) {
+! CHECK: fir.call @_QPvadd(%[[A]]#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>) -> ()
+  !$acc end data
+! CHECK: acc.copyout accVar(%[[B]] : !fir.box<!fir.array<?xf64>>) to var(%[[A]]#0 : !fir.box<!fir.array<?xf64>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+end 
+
+! Test for allocatable, pointer and assumed-shape variables appearing in use_device clause.
+subroutine test2(a, b, c)
+  integer :: N = 100
+  real*8, allocatable :: a(:)
+  real*8, target, allocatable :: d(:)
+  real*8 :: b(:)
+  real*8, pointer :: c(:)
+  call allocate(a(N))
+  call allocate(d(N))
+  c => d
+! CHECK: %[[DS:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[E:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[DS]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest2Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>)
+! CHECK: %[[F:.*]]:2 = hlfir.declare %arg1 dummy_scope %[[DS]] {uniq_name = "_QFtest2Eb"} : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
+! CHECK: %[[G:.*]]:2 = hlfir.declare %arg2 dummy_scope %[[DS]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest2Ec"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>)
+
+  !$acc data copy(a,b,c,d)
+  !$acc host_data use_device(a,b,c)
+  call vadd2(a,b,c)
+  !$acc end host_data
+
+! CHECK: %[[H:.*]] = acc.use_device varPtr(%[[E]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>> {name = "a"}
+! CHECK: %[[I:.*]] = acc.use_device varPtr(%[[E]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>> {name = "a"}
+! CHECK: %[[J:.*]] = acc.use_device var(%[[F]]#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
+! CHECK: %[[K:.*]] = acc.use_device var(%[[F]]#1 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
+! CHECK: %[[L:.*]] = acc.use_device varPtr(%[[G]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>> {name = "c"}
+! CHECK: %[[M:.*]] = acc.use_device varPtr(%[[G]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>> {name = "c"}
+! CHECK: acc.host_data dataOperands(%[[H]], %[[I]], %[[J]], %[[K]], %[[L]], %[[M]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>) {
+
+
+
+
+  !$acc end data
+
+end

>From d5d9b81896cbbb04b175b087d40e4fdfb1129fe0 Mon Sep 17 00:00:00 2001
From: nvpm <pmathew at nvidia.com>
Date: Fri, 11 Jul 2025 11:21:36 -0700
Subject: [PATCH 6/6] Format

---
 flang/lib/Lower/OpenACC.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 9234dd33c8b54..88f510da19ee1 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -743,15 +743,15 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
     // For UseDeviceOp, if operand is one of a pair resulting from a
     // declare operation, create a UseDeviceOp for the other operand as well.
     if constexpr (std::is_same_v<Op, mlir::acc::UseDeviceOp>) {
-      if (auto declareOp = mlir::dyn_cast<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
+      if (auto declareOp =
+              mlir::dyn_cast<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
         mlir::Value otherAddr = declareOp.getResult(1);
         if (baseAddr != otherAddr) {
-          Op op = createDataEntryOp<Op>(
-            builder, operandLocation, otherAddr,
-            asFortran, bounds, structured, implicit, dataClause,
-            otherAddr.getType(), async,
-            asyncDeviceTypes, asyncOnlyDeviceTypes, unwrapBoxAddr,
-            info.isPresent);
+          Op op = createDataEntryOp<Op>(builder, operandLocation, otherAddr,
+                                        asFortran, bounds, structured, implicit,
+                                        dataClause, otherAddr.getType(), async,
+                                        asyncDeviceTypes, asyncOnlyDeviceTypes,
+                                        unwrapBoxAddr, info.isPresent);
           dataOperands.push_back(op.getAccVar());
         }
       }


From flang-commits at lists.llvm.org  Fri Jul 11 11:38:51 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 11:38:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147618@github.com>
Message-ID: <68715a3b.170a0220.1c4b19.7390@mx.google.com>


clementval wrote:

Maybe we can just relax the type of `source_box` to be `BoxOrClassType`. In the end there is no real difference between the two and we can read the information from a fir.box or a fir.class the same way when emboxing. 

https://github.com/llvm/llvm-project/pull/147618

From flang-commits at lists.llvm.org  Fri Jul 11 13:14:51 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 11 Jul 2025 13:14:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147618@github.com>
Message-ID: <687170bb.170a0220.59c05.6cdf@mx.google.com>


vzakhari wrote:

> Maybe we can just relax the type of `source_box` to be `BoxOrClassType`. In the end there is no real difference between the two and we can read the information from a fir.box or a fir.class the same way when emboxing.

I was not sure why the limitation was there, so I did not try to relax it.  Let me test this change..

Thanks for taking a look, Valentin!

https://github.com/llvm/llvm-project/pull/147618

From flang-commits at lists.llvm.org  Fri Jul 11 13:18:20 2025
From: flang-commits at lists.llvm.org (Susan Tan =?utf-8?b?44K5LeOCtuODs+OAgOOCv+ODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 13:18:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add deviation to the spec that
 declaring the same variable is permitted (PR #148288)
Message-ID: <llvm.org/llvm/llvm-project/pull/148288@github.com>


https://github.com/SusanTan created https://github.com/llvm/llvm-project/pull/148288

OpenACC spec says `A var may appear at most once in all the clauses of declare directives for a function, subroutine, program, or module.` but our implementation allows it with a warning generated. Add this to the diviation list for record.



>From 2f8a7fcef4d9911be898ec4afb6b99a80c1941ae Mon Sep 17 00:00:00 2001
From: Susan Tan <zujunt at nvidia.com>
Date: Fri, 11 Jul 2025 13:15:41 -0700
Subject: [PATCH] add deviation of declare same variable

---
 flang/docs/OpenACC.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flang/docs/OpenACC.md b/flang/docs/OpenACC.md
index 87f30ccd953b6..cbb2a91f0963b 100644
--- a/flang/docs/OpenACC.md
+++ b/flang/docs/OpenACC.md
@@ -27,6 +27,9 @@ local:
 * `!$acc cache` directive accepts scalar variable.
 * The `!$acc declare` directive accepts assumed size array arguments for 
   `deviceptr` and `present` clauses.
+* The OpenACC specification disallows a variable appearing multiple times in
+  clauses of `!$acc declare` directives for a function, subroutine, program,
+  or module, but we allow it with a warning.
 
 ## Remarks about incompatibilities with other implementations
 * Array element references in the data clauses are equivalent to array sections


From flang-commits at lists.llvm.org  Fri Jul 11 13:19:02 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 11 Jul 2025 13:19:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add deviation to the spec that
 declaring the same variable is permitted (PR #148288)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148288@github.com>
Message-ID: <687171b6.170a0220.a3385.6b6f@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-openacc

Author: Susan Tan (ス-ザン　タン) (SusanTan)

<details>
<summary>Changes</summary>

OpenACC spec says `A var may appear at most once in all the clauses of declare directives for a function, subroutine, program, or module.` but our implementation allows it with a warning generated. Add this to the diviation list for record.



---
Full diff: https://github.com/llvm/llvm-project/pull/148288.diff


1 Files Affected:

- (modified) flang/docs/OpenACC.md (+3) 


``````````diff
diff --git a/flang/docs/OpenACC.md b/flang/docs/OpenACC.md
index 87f30ccd953b6..cbb2a91f0963b 100644
--- a/flang/docs/OpenACC.md
+++ b/flang/docs/OpenACC.md
@@ -27,6 +27,9 @@ local:
 * `!$acc cache` directive accepts scalar variable.
 * The `!$acc declare` directive accepts assumed size array arguments for 
   `deviceptr` and `present` clauses.
+* The OpenACC specification disallows a variable appearing multiple times in
+  clauses of `!$acc declare` directives for a function, subroutine, program,
+  or module, but we allow it with a warning.
 
 ## Remarks about incompatibilities with other implementations
 * Array element references in the data clauses are equivalent to array sections

``````````

</details>


https://github.com/llvm/llvm-project/pull/148288

From flang-commits at lists.llvm.org  Fri Jul 11 13:21:46 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 13:21:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][openacc] Add deviation to the spec
 that declaring the same variable is permitted (PR #148288)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148288@github.com>
Message-ID: <6871725a.170a0220.27e044.6efe@mx.google.com>


https://github.com/clementval edited https://github.com/llvm/llvm-project/pull/148288

From flang-commits at lists.llvm.org  Fri Jul 11 13:21:53 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 13:21:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][openacc] Add deviation to the spec
 that declaring the same variable is permitted (PR #148288)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148288@github.com>
Message-ID: <68717261.170a0220.37450f.6fc3@mx.google.com>


https://github.com/clementval approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/148288

From flang-commits at lists.llvm.org  Fri Jul 11 13:22:18 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 13:22:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][openacc] Add deviation to the spec
 that declaring the same variable is permitted (PR #148288)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148288@github.com>
Message-ID: <6871727a.630a0220.1a4bae.6b76@mx.google.com>


clementval wrote:

You can add `[NFC]` since it's only doc

https://github.com/llvm/llvm-project/pull/148288

From flang-commits at lists.llvm.org  Fri Jul 11 13:22:53 2025
From: flang-commits at lists.llvm.org (Susan Tan =?utf-8?b?44K5LeOCtuODs+OAgOOCv+ODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 13:22:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][openacc][NFC] Add deviation to the
 spec that declaring the same variable is permitted (PR #148288)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148288@github.com>
Message-ID: <6871729d.170a0220.c9c34.6bf8@mx.google.com>


https://github.com/SusanTan edited https://github.com/llvm/llvm-project/pull/148288

From flang-commits at lists.llvm.org  Fri Jul 11 13:27:48 2025
From: flang-commits at lists.llvm.org (Susan Tan =?utf-8?b?44K5LeOCtuODs+OAgOOCv+ODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 13:27:48 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][acc][NFC] Add deviation to the spec
 that declaring the same variable is permitted (PR #148288)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148288@github.com>
Message-ID: <687173c4.170a0220.2131bc.6ae2@mx.google.com>


https://github.com/SusanTan edited https://github.com/llvm/llvm-project/pull/148288

From flang-commits at lists.llvm.org  Fri Jul 11 13:50:16 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 13:50:16 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Fix false positive on
 unsupported CUDA data transfer (PR #148295)
Message-ID: <llvm.org/llvm/llvm-project/pull/148295@github.com>


https://github.com/clementval created https://github.com/llvm/llvm-project/pull/148295

The switch to `GetSymbolVector` introduced a regression on detecting implicit data transfer when the rhs is a function call.
Make sure the symbol we are looking at are of interest to detect data transfer. 

>From 53de6e50fdc22123fec4516dc597557c9848fc62 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Fri, 11 Jul 2025 13:47:03 -0700
Subject: [PATCH] [flang][cuda] Fix false positive on unsupported CUDA data
 transfer

---
 flang/lib/Evaluate/tools.cpp                 | 25 ++++++++++++--------
 flang/test/Lower/CUDA/cuda-data-transfer.cuf |  8 +++++++
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 3d9f06308d8c1..21e6b3c3dd50d 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1157,23 +1157,28 @@ template semantics::UnorderedSymbolSet CollectCudaSymbols(
 bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
   semantics::UnorderedSymbolSet hostSymbols;
   semantics::UnorderedSymbolSet deviceSymbols;
+  semantics::UnorderedSymbolSet cudaSymbols{CollectCudaSymbols(expr)};
 
   SymbolVector symbols{GetSymbolVector(expr)};
   std::reverse(symbols.begin(), symbols.end());
   bool skipNext{false};
   for (const Symbol &sym : symbols) {
-    bool isComponent{sym.owner().IsDerivedType()};
-    bool skipComponent{false};
-    if (!skipNext) {
-      if (IsCUDADeviceSymbol(sym)) {
-        deviceSymbols.insert(sym);
-      } else if (isComponent) {
-        skipComponent = true; // Component is not device. Look on the base.
-      } else {
-        hostSymbols.insert(sym);
+    if (cudaSymbols.find(sym) != cudaSymbols.end()) {
+      bool isComponent{sym.owner().IsDerivedType()};
+      bool skipComponent{false};
+      if (!skipNext) {
+        if (IsCUDADeviceSymbol(sym)) {
+          deviceSymbols.insert(sym);
+        } else if (isComponent) {
+          skipComponent = true; // Component is not device. Look on the base.
+        } else {
+          hostSymbols.insert(sym);
+        }
       }
+      skipNext = isComponent && !skipComponent;
+    } else {
+      skipNext = false;
     }
-    skipNext = isComponent && !skipComponent;
   }
   bool hasConstant{HasConstant(expr)};
   return (hasConstant || (hostSymbols.size() > 0)) && deviceSymbols.size() > 0;
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index 3a9b55996d9b1..3a4aff977b7a5 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -436,3 +436,11 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub22()
 ! CHECK: cuf.data_transfer
+
+subroutine sub23(n)
+  integer :: n
+  real(8), device :: d(n,n), x(n)
+  x = sum(d,dim=2) ! Was triggering Unsupported CUDA data transfer
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub23


From flang-commits at lists.llvm.org  Fri Jul 11 13:50:47 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 11 Jul 2025 13:50:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Fix false positive on
 unsupported CUDA data transfer (PR #148295)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148295@github.com>
Message-ID: <68717927.050a0220.106949.92a2@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Valentin Clement (バレンタイン クレメン) (clementval)

<details>
<summary>Changes</summary>

The switch to `GetSymbolVector` introduced a regression on detecting implicit data transfer when the rhs is a function call.
Make sure the symbol we are looking at are of interest to detect data transfer. 

---
Full diff: https://github.com/llvm/llvm-project/pull/148295.diff


2 Files Affected:

- (modified) flang/lib/Evaluate/tools.cpp (+15-10) 
- (modified) flang/test/Lower/CUDA/cuda-data-transfer.cuf (+8) 


``````````diff
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 3d9f06308d8c1..21e6b3c3dd50d 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1157,23 +1157,28 @@ template semantics::UnorderedSymbolSet CollectCudaSymbols(
 bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
   semantics::UnorderedSymbolSet hostSymbols;
   semantics::UnorderedSymbolSet deviceSymbols;
+  semantics::UnorderedSymbolSet cudaSymbols{CollectCudaSymbols(expr)};
 
   SymbolVector symbols{GetSymbolVector(expr)};
   std::reverse(symbols.begin(), symbols.end());
   bool skipNext{false};
   for (const Symbol &sym : symbols) {
-    bool isComponent{sym.owner().IsDerivedType()};
-    bool skipComponent{false};
-    if (!skipNext) {
-      if (IsCUDADeviceSymbol(sym)) {
-        deviceSymbols.insert(sym);
-      } else if (isComponent) {
-        skipComponent = true; // Component is not device. Look on the base.
-      } else {
-        hostSymbols.insert(sym);
+    if (cudaSymbols.find(sym) != cudaSymbols.end()) {
+      bool isComponent{sym.owner().IsDerivedType()};
+      bool skipComponent{false};
+      if (!skipNext) {
+        if (IsCUDADeviceSymbol(sym)) {
+          deviceSymbols.insert(sym);
+        } else if (isComponent) {
+          skipComponent = true; // Component is not device. Look on the base.
+        } else {
+          hostSymbols.insert(sym);
+        }
       }
+      skipNext = isComponent && !skipComponent;
+    } else {
+      skipNext = false;
     }
-    skipNext = isComponent && !skipComponent;
   }
   bool hasConstant{HasConstant(expr)};
   return (hasConstant || (hostSymbols.size() > 0)) && deviceSymbols.size() > 0;
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index 3a9b55996d9b1..3a4aff977b7a5 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -436,3 +436,11 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub22()
 ! CHECK: cuf.data_transfer
+
+subroutine sub23(n)
+  integer :: n
+  real(8), device :: d(n,n), x(n)
+  x = sum(d,dim=2) ! Was triggering Unsupported CUDA data transfer
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub23

``````````

</details>


https://github.com/llvm/llvm-project/pull/148295

From flang-commits at lists.llvm.org  Fri Jul 11 13:53:23 2025
From: flang-commits at lists.llvm.org (Zhen Wang via flang-commits)
Date: Fri, 11 Jul 2025 13:53:23 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Fix false positive on
 unsupported CUDA data transfer (PR #148295)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148295@github.com>
Message-ID: <687179c3.170a0220.11bcd.73f4@mx.google.com>


https://github.com/wangzpgi approved this pull request.


https://github.com/llvm/llvm-project/pull/148295

From flang-commits at lists.llvm.org  Fri Jul 11 14:54:07 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 14:54:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][acc] Implement MappableType's
 generatePrivateInit (PR #148302)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148302@github.com>
Message-ID: <687187ff.170a0220.1b0558.6bae@mx.google.com>


https://github.com/clementval approved this pull request.

LGTM. You need to run clang-format I think

https://github.com/llvm/llvm-project/pull/148302

From flang-commits at lists.llvm.org  Fri Jul 11 15:05:25 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Fri, 11 Jul 2025 15:05:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][acc] Implement MappableType's
 generatePrivateInit (PR #148302)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148302@github.com>
Message-ID: <68718aa5.170a0220.2da2a7.757e@mx.google.com>


https://github.com/razvanlupusoru updated https://github.com/llvm/llvm-project/pull/148302

>From bd70322c6a77fff97a2b225ec473c9642a21ab92 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Fri, 11 Jul 2025 14:33:41 -0700
Subject: [PATCH 1/2] [flang][acc] Implement MappableType's generatePrivateInit

The recipe body generation was moved from lowering into FIR's
implementation of MappableType API. And now since all Fortran
variable types implement this type, lowering of OpenACC was updated
to use this API directly. No test changes were needed - all of the
private, firstprivate, and recipe tests get the same body as before.
---
 .../OpenACC/FIROpenACCTypeInterfaces.h        |   8 +
 flang/lib/Lower/CMakeLists.txt                |   1 +
 flang/lib/Lower/OpenACC.cpp                   | 139 +++--------------
 .../OpenACC/FIROpenACCTypeInterfaces.cpp      | 140 ++++++++++++++++++
 4 files changed, 172 insertions(+), 116 deletions(-)

diff --git a/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h b/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
index 3e343f347e4ae..dcbf99c38af73 100644
--- a/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
+++ b/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
@@ -52,6 +52,14 @@ struct OpenACCMappableModel
 
   mlir::acc::VariableTypeCategory getTypeCategory(mlir::Type type,
                                                   mlir::Value var) const;
+
+  mlir::Value
+  generatePrivateInit(mlir::Type type, mlir::OpBuilder &builder,
+                      mlir::Location loc,
+                      mlir::TypedValue<mlir::acc::MappableType> var,
+                      llvm::StringRef varName,
+                      mlir::ValueRange extents,
+                      mlir::Value initVal) const;
 };
 
 } // namespace fir::acc
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index cd80aaf553869..8e20abf0e9f2d 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -51,6 +51,7 @@ add_flang_library(FortranLower
   FIRDialect
   FIRDialectSupport
   FIRBuilder
+  FIROpenACCSupport
   FIRSupport
   FIRTransforms
   HLFIRDialect
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 00c9cbf0d2a8f..39e4444cde4e3 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -959,119 +959,6 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
   llvm::report_fatal_error("Unsupported OpenACC reduction type");
 }
 
-template <typename RecipeOp>
-static void genPrivateLikeInitRegion(fir::FirOpBuilder &builder,
-                                     RecipeOp recipe, mlir::Type argTy,
-                                     mlir::Location loc,
-                                     mlir::Value initValue) {
-  mlir::Value retVal = recipe.getInitRegion().front().getArgument(0);
-  mlir::Type unwrappedTy = fir::unwrapRefType(argTy);
-
-  llvm::StringRef initName;
-  if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>)
-    initName = accReductionInitName;
-  else
-    initName = accPrivateInitName;
-
-  auto getDeclareOpForType = [&](mlir::Type ty) -> hlfir::DeclareOp {
-    auto alloca = builder.create<fir::AllocaOp>(loc, ty);
-    return builder.create<hlfir::DeclareOp>(
-        loc, alloca, initName, /*shape=*/nullptr, llvm::ArrayRef<mlir::Value>{},
-        /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
-  };
-
-  if (fir::isa_trivial(unwrappedTy)) {
-    auto declareOp = getDeclareOpForType(unwrappedTy);
-    if (initValue) {
-      auto convert = builder.createConvert(loc, unwrappedTy, initValue);
-      builder.create<fir::StoreOp>(loc, convert, declareOp.getBase());
-    }
-    retVal = declareOp.getBase();
-  } else if (auto seqTy =
-                 mlir::dyn_cast_or_null<fir::SequenceType>(unwrappedTy)) {
-    if (fir::isa_trivial(seqTy.getEleTy())) {
-      mlir::Value shape;
-      llvm::SmallVector<mlir::Value> extents;
-      if (seqTy.hasDynamicExtents()) {
-        // Extents are passed as block arguments. First argument is the
-        // original value.
-        for (unsigned i = 1; i < recipe.getInitRegion().getArguments().size();
-             ++i)
-          extents.push_back(recipe.getInitRegion().getArgument(i));
-        shape = builder.create<fir::ShapeOp>(loc, extents);
-      } else {
-        shape = genShapeOp(builder, seqTy, loc);
-      }
-      auto alloca = builder.create<fir::AllocaOp>(
-          loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
-      auto declareOp = builder.create<hlfir::DeclareOp>(
-          loc, alloca, initName, shape, llvm::ArrayRef<mlir::Value>{},
-          /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
-
-      if (initValue) {
-        mlir::Type idxTy = builder.getIndexType();
-        mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
-        llvm::SmallVector<fir::DoLoopOp> loops;
-        llvm::SmallVector<mlir::Value> ivs;
-
-        if (seqTy.hasDynamicExtents()) {
-          builder.create<hlfir::AssignOp>(loc, initValue, declareOp.getBase());
-        } else {
-          for (auto ext : seqTy.getShape()) {
-            auto lb = builder.createIntegerConstant(loc, idxTy, 0);
-            auto ub = builder.createIntegerConstant(loc, idxTy, ext - 1);
-            auto step = builder.createIntegerConstant(loc, idxTy, 1);
-            auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
-                                                      /*unordered=*/false);
-            builder.setInsertionPointToStart(loop.getBody());
-            loops.push_back(loop);
-            ivs.push_back(loop.getInductionVar());
-          }
-          auto coord = builder.create<fir::CoordinateOp>(
-              loc, refTy, declareOp.getBase(), ivs);
-          builder.create<fir::StoreOp>(loc, initValue, coord);
-          builder.setInsertionPointAfter(loops[0]);
-        }
-      }
-      retVal = declareOp.getBase();
-    }
-  } else if (auto boxTy =
-                 mlir::dyn_cast_or_null<fir::BaseBoxType>(unwrappedTy)) {
-    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
-    if (fir::isa_trivial(innerTy)) {
-      retVal = getDeclareOpForType(unwrappedTy).getBase();
-    } else if (mlir::isa<fir::SequenceType>(innerTy)) {
-      fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
-      hlfir::Entity source = hlfir::Entity{retVal};
-      auto [temp, cleanup] = hlfir::createTempFromMold(loc, firBuilder, source);
-      if (fir::isa_ref_type(argTy)) {
-        // When the temp is created - it is not a reference - thus we can
-        // end up with a type inconsistency. Therefore ensure storage is created
-        // for it.
-        retVal = getDeclareOpForType(unwrappedTy).getBase();
-        mlir::Value storeDst = retVal;
-        if (fir::unwrapRefType(retVal.getType()) != temp.getType()) {
-          // `createTempFromMold` makes the unfortunate choice to lose the
-          // `fir.heap` and `fir.ptr` types when wrapping with a box. Namely,
-          // when wrapping a `fir.heap<fir.array>`, it will create instead a
-          // `fir.box<fir.array>`. Cast here to deal with this inconsistency.
-          storeDst = firBuilder.createConvert(
-              loc, firBuilder.getRefType(temp.getType()), retVal);
-        }
-        builder.create<fir::StoreOp>(loc, temp, storeDst);
-      } else {
-        retVal = temp;
-      }
-    } else {
-      TODO(loc, "Unsupported boxed type for OpenACC private-like recipe");
-    }
-    if (initValue) {
-      builder.create<hlfir::AssignOp>(loc, initValue, retVal);
-    }
-  }
-  builder.create<mlir::acc::YieldOp>(loc, retVal);
-}
-
 template <typename RecipeOp>
 static RecipeOp genRecipeOp(
     fir::FirOpBuilder &builder, mlir::ModuleOp mod, llvm::StringRef recipeName,
@@ -1100,15 +987,35 @@ static RecipeOp genRecipeOp(
       }
     }
   }
-  builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
-                      argsTy, argsLoc);
+  auto initBlock = builder.createBlock(
+      &recipe.getInitRegion(), recipe.getInitRegion().end(), argsTy, argsLoc);
   builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
   mlir::Value initValue;
   if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>) {
     assert(op != mlir::acc::ReductionOperator::AccNone);
     initValue = getReductionInitValue(builder, loc, fir::unwrapRefType(ty), op);
   }
-  genPrivateLikeInitRegion<RecipeOp>(builder, recipe, ty, loc, initValue);
+
+  // Since we reuse the same recipe for all variables of the same type - we
+  // cannot use the actual variable name. Thus use a temporary name.
+  llvm::StringRef initName;
+  if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>)
+    initName = accReductionInitName;
+  else
+    initName = accPrivateInitName;
+
+  auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(ty);
+  assert(mappableTy &&
+         "Expected that all variable types are considered mappable");
+  auto retVal = mappableTy.generatePrivateInit(
+      builder, loc,
+      mlir::cast<mlir::TypedValue<mlir::acc::MappableType>>(
+          initBlock->getArgument(0)),
+      initName,
+      initBlock->getArguments().take_back(initBlock->getArguments().size() - 1),
+      initValue);
+  builder.create<mlir::acc::YieldOp>(loc, retVal ? retVal
+                                                 : initBlock->getArgument(0));
   return recipe;
 }
 
diff --git a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
index 0767733f53728..db3172b186ddf 100644
--- a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
@@ -25,6 +25,7 @@
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Support/LLVM.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 namespace fir::acc {
@@ -525,4 +526,143 @@ OpenACCPointerLikeModel<fir::LLVMPointerType>::getPointeeTypeCategory(
   return categorizePointee(pointer, varPtr, varType);
 }
 
+static fir::ShapeOp genShapeOp(mlir::OpBuilder &builder,
+                               fir::SequenceType seqTy, mlir::Location loc) {
+  llvm::SmallVector<mlir::Value> extents;
+  mlir::Type idxTy = builder.getIndexType();
+  for (auto extent : seqTy.getShape())
+    extents.push_back(builder.create<mlir::arith::ConstantOp>(
+        loc, idxTy, builder.getIntegerAttr(idxTy, extent)));
+  return builder.create<fir::ShapeOp>(loc, extents);
+}
+
+template <typename Ty>
+mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents,
+    mlir::Value initVal) const {
+  mlir::Value retVal;
+  mlir::Type unwrappedTy = fir::unwrapRefType(type);
+  mlir::ModuleOp mod = builder.getInsertionBlock()
+                           ->getParent()
+                           ->getParentOfType<mlir::ModuleOp>();
+  fir::FirOpBuilder firBuilder(builder, mod);
+
+  auto getDeclareOpForType = [&](mlir::Type ty) -> hlfir::DeclareOp {
+    auto alloca = firBuilder.create<fir::AllocaOp>(loc, ty);
+    return firBuilder.create<hlfir::DeclareOp>(
+        loc, alloca, varName, /*shape=*/nullptr, llvm::ArrayRef<mlir::Value>{},
+        /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
+  };
+
+  if (fir::isa_trivial(unwrappedTy)) {
+    auto declareOp = getDeclareOpForType(unwrappedTy);
+    if (initVal) {
+      auto convert = firBuilder.createConvert(loc, unwrappedTy, initVal);
+      firBuilder.create<fir::StoreOp>(loc, convert, declareOp.getBase());
+    }
+    retVal = declareOp.getBase();
+  } else if (auto seqTy =
+                 mlir::dyn_cast_or_null<fir::SequenceType>(unwrappedTy)) {
+    if (fir::isa_trivial(seqTy.getEleTy())) {
+      mlir::Value shape;
+      if (seqTy.hasDynamicExtents()) {
+        shape = firBuilder.create<fir::ShapeOp>(loc, llvm::to_vector(extents));
+      } else {
+        shape = genShapeOp(firBuilder, seqTy, loc);
+      }
+      auto alloca = firBuilder.create<fir::AllocaOp>(
+          loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
+      auto declareOp = firBuilder.create<hlfir::DeclareOp>(
+          loc, alloca, varName, shape, llvm::ArrayRef<mlir::Value>{},
+          /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
+
+      if (initVal) {
+        mlir::Type idxTy = firBuilder.getIndexType();
+        mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
+        llvm::SmallVector<fir::DoLoopOp> loops;
+        llvm::SmallVector<mlir::Value> ivs;
+
+        if (seqTy.hasDynamicExtents()) {
+          firBuilder.create<hlfir::AssignOp>(loc, initVal, declareOp.getBase());
+        } else {
+          for (auto ext : seqTy.getShape()) {
+            auto lb = firBuilder.createIntegerConstant(loc, idxTy, 0);
+            auto ub = firBuilder.createIntegerConstant(loc, idxTy, ext - 1);
+            auto step = firBuilder.createIntegerConstant(loc, idxTy, 1);
+            auto loop = firBuilder.create<fir::DoLoopOp>(loc, lb, ub, step,
+                                                         /*unordered=*/false);
+            firBuilder.setInsertionPointToStart(loop.getBody());
+            loops.push_back(loop);
+            ivs.push_back(loop.getInductionVar());
+          }
+          auto coord = firBuilder.create<fir::CoordinateOp>(
+              loc, refTy, declareOp.getBase(), ivs);
+          firBuilder.create<fir::StoreOp>(loc, initVal, coord);
+          firBuilder.setInsertionPointAfter(loops[0]);
+        }
+      }
+      retVal = declareOp.getBase();
+    }
+  } else if (auto boxTy =
+                 mlir::dyn_cast_or_null<fir::BaseBoxType>(unwrappedTy)) {
+    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
+    if (fir::isa_trivial(innerTy)) {
+      retVal = getDeclareOpForType(unwrappedTy).getBase();
+    } else if (mlir::isa<fir::SequenceType>(innerTy)) {
+      hlfir::Entity source = hlfir::Entity{var};
+      auto [temp, cleanup] = hlfir::createTempFromMold(loc, firBuilder, source);
+      if (fir::isa_ref_type(type)) {
+        // When the temp is created - it is not a reference - thus we can
+        // end up with a type inconsistency. Therefore ensure storage is created
+        // for it.
+        retVal = getDeclareOpForType(unwrappedTy).getBase();
+        mlir::Value storeDst = retVal;
+        if (fir::unwrapRefType(retVal.getType()) != temp.getType()) {
+          // `createTempFromMold` makes the unfortunate choice to lose the
+          // `fir.heap` and `fir.ptr` types when wrapping with a box. Namely,
+          // when wrapping a `fir.heap<fir.array>`, it will create instead a
+          // `fir.box<fir.array>`. Cast here to deal with this inconsistency.
+          storeDst = firBuilder.createConvert(
+              loc, firBuilder.getRefType(temp.getType()), retVal);
+        }
+        builder.create<fir::StoreOp>(loc, temp, storeDst);
+      } else {
+        retVal = temp;
+      }
+    } else {
+      TODO(loc, "Unsupported boxed type for OpenACC private-like recipe");
+    }
+    if (initVal) {
+      builder.create<hlfir::AssignOp>(loc, initVal, retVal);
+    }
+  }
+  return retVal;
+}
+
+template mlir::Value
+OpenACCMappableModel<fir::BaseBoxType>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const;
+
+template mlir::Value
+OpenACCMappableModel<fir::ReferenceType>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const;
+
+template mlir::Value
+OpenACCMappableModel<fir::HeapType>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const;
+
+template mlir::Value
+OpenACCMappableModel<fir::PointerType>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const;
+
 } // namespace fir::acc

>From 60a1d5603c794823e9e358c99491738e87fe3273 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Fri, 11 Jul 2025 15:05:14 -0700
Subject: [PATCH 2/2] Fix formatting

---
 .../Optimizer/OpenACC/FIROpenACCTypeInterfaces.h    | 13 ++++++-------
 .../Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp  |  6 ++----
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h b/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
index dcbf99c38af73..10853933317f7 100644
--- a/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
+++ b/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
@@ -53,13 +53,12 @@ struct OpenACCMappableModel
   mlir::acc::VariableTypeCategory getTypeCategory(mlir::Type type,
                                                   mlir::Value var) const;
 
-  mlir::Value
-  generatePrivateInit(mlir::Type type, mlir::OpBuilder &builder,
-                      mlir::Location loc,
-                      mlir::TypedValue<mlir::acc::MappableType> var,
-                      llvm::StringRef varName,
-                      mlir::ValueRange extents,
-                      mlir::Value initVal) const;
+  mlir::Value generatePrivateInit(mlir::Type type, mlir::OpBuilder &builder,
+                                  mlir::Location loc,
+                                  mlir::TypedValue<mlir::acc::MappableType> var,
+                                  llvm::StringRef varName,
+                                  mlir::ValueRange extents,
+                                  mlir::Value initVal) const;
 };
 
 } // namespace fir::acc
diff --git a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
index db3172b186ddf..a87e90e8878a3 100644
--- a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
@@ -540,8 +540,7 @@ template <typename Ty>
 mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
     mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
     mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
-    mlir::ValueRange extents,
-    mlir::Value initVal) const {
+    mlir::ValueRange extents, mlir::Value initVal) const {
   mlir::Value retVal;
   mlir::Type unwrappedTy = fir::unwrapRefType(type);
   mlir::ModuleOp mod = builder.getInsertionBlock()
@@ -653,8 +652,7 @@ OpenACCMappableModel<fir::ReferenceType>::generatePrivateInit(
     mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
     mlir::ValueRange extents, mlir::Value initVal) const;
 
-template mlir::Value
-OpenACCMappableModel<fir::HeapType>::generatePrivateInit(
+template mlir::Value OpenACCMappableModel<fir::HeapType>::generatePrivateInit(
     mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
     mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
     mlir::ValueRange extents, mlir::Value initVal) const;


From flang-commits at lists.llvm.org  Fri Jul 11 15:06:56 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 11 Jul 2025 15:06:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Allow embox's source_box to be a
	!fir.box. (PR #148305)
Message-ID: <llvm.org/llvm/llvm-project/pull/148305@github.com>


https://github.com/vzakhari created https://github.com/llvm/llvm-project/pull/148305

In order to create temporary copies of assumed-type arrays
(e.g. for `-frepack-arrays`), we have to allow the source_box
to be a !fir.box.

This patch replaces #147618.


>From d393a96d99d1145e76ffbe870dc048cac690306b Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Fri, 11 Jul 2025 14:50:39 -0700
Subject: [PATCH] [flang] Allow embox's source_box to be a !fir.box.

In order to create temporary copies of assumed-type arrays
(e.g. for `-frepack-arrays`), we have to allow the source_box
to be a !fir.box.

This patch replaces #147618.
---
 .../include/flang/Optimizer/Dialect/FIRCG/CGOps.td |  3 ++-
 flang/include/flang/Optimizer/Dialect/FIROps.td    | 14 +++++---------
 flang/include/flang/Optimizer/Dialect/FIRTypes.td  |  6 ++++--
 flang/test/Fir/fir-ops.fir                         | 10 ++++++++++
 4 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td b/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
index d48caf1a8cb3b..04f839386498c 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
@@ -60,7 +60,8 @@ def fircg_XEmboxOp : fircg_Op<"ext_embox", [AttrSizedOperandSegments]> {
       Variadic<AnyIntegerType>:$slice,
       Variadic<AnyCoordinateType>:$subcomponent,
       Variadic<AnyIntegerType>:$substr, Variadic<AnyIntegerType>:$lenParams,
-      Optional<fir_ClassType>:$sourceBox, OptionalAttr<I32Attr>:$allocator_idx);
+      Optional<BoxOrClassType>:$sourceBox,
+      OptionalAttr<I32Attr>:$allocator_idx);
   let results = (outs BoxOrClassType);
 
   let assemblyFormat = [{
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index e3f5c4403002a..99b5105ab365e 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -834,15 +834,11 @@ def fir_EmboxOp : fir_Op<"embox", [NoMemoryEffect, AttrSizedOperandSegments]> {
         - allocator_idx: specify special allocator to use.
   }];
 
-  let arguments = (ins
-    AnyReferenceLike:$memref,
-    Optional<AnyShapeType>:$shape,
-    Optional<fir_SliceType>:$slice,
-    Variadic<AnyIntegerType>:$typeparams,
-    Optional<fir_ClassType>:$sourceBox,
-    OptionalAttr<AffineMapAttr>:$accessMap,
-    OptionalAttr<I32Attr>:$allocator_idx
-  );
+  let arguments = (ins AnyReferenceLike:$memref, Optional<AnyShapeType>:$shape,
+      Optional<fir_SliceType>:$slice, Variadic<AnyIntegerType>:$typeparams,
+      Optional<BoxOrClassType>:$sourceBox,
+      OptionalAttr<AffineMapAttr>:$accessMap,
+      OptionalAttr<I32Attr>:$allocator_idx);
 
   let results = (outs BoxOrClassType);
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIRTypes.td b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
index 0ead54df3ca97..2fdc9a96804da 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRTypes.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
@@ -626,8 +626,10 @@ def AnyBoxLike : TypeConstraint<Or<[fir_BoxType.predicate,
     fir_BoxCharType.predicate, fir_BoxProcType.predicate,
     fir_ClassType.predicate]>, "any box">;
 
-def BoxOrClassType : TypeConstraint<Or<[fir_BoxType.predicate,
-    fir_ClassType.predicate]>, "box or class">;
+def BoxOrClassLike
+    : TypeConstraint<Or<[fir_BoxType.predicate, fir_ClassType.predicate]>,
+                     "box or class">;
+def BoxOrClassType : Type<BoxOrClassLike.predicate, "box or class">;
 
 def AnyRefOrBoxLike : TypeConstraint<Or<[AnyReferenceLike.predicate,
     AnyBoxLike.predicate, FunctionType.predicate]>,
diff --git a/flang/test/Fir/fir-ops.fir b/flang/test/Fir/fir-ops.fir
index 3585bf9efca3e..0892eb9fa0de8 100644
--- a/flang/test/Fir/fir-ops.fir
+++ b/flang/test/Fir/fir-ops.fir
@@ -843,6 +843,16 @@ func.func @embox_tdesc(%arg0: !fir.class<!fir.array<10x!fir.type<derived_poly{a:
   return
 }
 
+// CHECK-LABEL:   func.func @embox_with_source_box(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.heap<!fir.array<?xnone>>,
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.shape<1>,
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>> {
+func.func @embox_with_source_box(%arg0: !fir.heap<!fir.array<?xnone>>, %arg1: !fir.shape<1>, %arg2: !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>> {
+// CHECK:           %[[VAL_0:.*]] = fir.embox %[[ARG0]](%[[ARG1]]) source_box %[[ARG2]] : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+  %0 = fir.embox %arg0(%arg1) source_box %arg2 : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+  return %0 : !fir.class<!fir.heap<!fir.array<?xnone>>>
+}
+
 func.func @test_fortran_var_attrs() {
   %0 = fir.alloca !fir.heap<f32> {fortran_attrs = #fir.var_attrs<allocatable>}
   %1 = fir.alloca f32 {fortran_attrs = #fir.var_attrs<asynchronous>}


From flang-commits at lists.llvm.org  Fri Jul 11 15:07:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 11 Jul 2025 15:07:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Allow embox's source_box to be a
	!fir.box. (PR #148305)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148305@github.com>
Message-ID: <68718b3d.170a0220.37d962.739e@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Slava Zakharin (vzakhari)

<details>
<summary>Changes</summary>

In order to create temporary copies of assumed-type arrays
(e.g. for `-frepack-arrays`), we have to allow the source_box
to be a !fir.box.

This patch replaces #<!-- -->147618.


---
Full diff: https://github.com/llvm/llvm-project/pull/148305.diff


4 Files Affected:

- (modified) flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td (+2-1) 
- (modified) flang/include/flang/Optimizer/Dialect/FIROps.td (+5-9) 
- (modified) flang/include/flang/Optimizer/Dialect/FIRTypes.td (+4-2) 
- (modified) flang/test/Fir/fir-ops.fir (+10) 


``````````diff
diff --git a/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td b/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
index d48caf1a8cb3b..04f839386498c 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
@@ -60,7 +60,8 @@ def fircg_XEmboxOp : fircg_Op<"ext_embox", [AttrSizedOperandSegments]> {
       Variadic<AnyIntegerType>:$slice,
       Variadic<AnyCoordinateType>:$subcomponent,
       Variadic<AnyIntegerType>:$substr, Variadic<AnyIntegerType>:$lenParams,
-      Optional<fir_ClassType>:$sourceBox, OptionalAttr<I32Attr>:$allocator_idx);
+      Optional<BoxOrClassType>:$sourceBox,
+      OptionalAttr<I32Attr>:$allocator_idx);
   let results = (outs BoxOrClassType);
 
   let assemblyFormat = [{
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index e3f5c4403002a..99b5105ab365e 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -834,15 +834,11 @@ def fir_EmboxOp : fir_Op<"embox", [NoMemoryEffect, AttrSizedOperandSegments]> {
         - allocator_idx: specify special allocator to use.
   }];
 
-  let arguments = (ins
-    AnyReferenceLike:$memref,
-    Optional<AnyShapeType>:$shape,
-    Optional<fir_SliceType>:$slice,
-    Variadic<AnyIntegerType>:$typeparams,
-    Optional<fir_ClassType>:$sourceBox,
-    OptionalAttr<AffineMapAttr>:$accessMap,
-    OptionalAttr<I32Attr>:$allocator_idx
-  );
+  let arguments = (ins AnyReferenceLike:$memref, Optional<AnyShapeType>:$shape,
+      Optional<fir_SliceType>:$slice, Variadic<AnyIntegerType>:$typeparams,
+      Optional<BoxOrClassType>:$sourceBox,
+      OptionalAttr<AffineMapAttr>:$accessMap,
+      OptionalAttr<I32Attr>:$allocator_idx);
 
   let results = (outs BoxOrClassType);
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIRTypes.td b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
index 0ead54df3ca97..2fdc9a96804da 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRTypes.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
@@ -626,8 +626,10 @@ def AnyBoxLike : TypeConstraint<Or<[fir_BoxType.predicate,
     fir_BoxCharType.predicate, fir_BoxProcType.predicate,
     fir_ClassType.predicate]>, "any box">;
 
-def BoxOrClassType : TypeConstraint<Or<[fir_BoxType.predicate,
-    fir_ClassType.predicate]>, "box or class">;
+def BoxOrClassLike
+    : TypeConstraint<Or<[fir_BoxType.predicate, fir_ClassType.predicate]>,
+                     "box or class">;
+def BoxOrClassType : Type<BoxOrClassLike.predicate, "box or class">;
 
 def AnyRefOrBoxLike : TypeConstraint<Or<[AnyReferenceLike.predicate,
     AnyBoxLike.predicate, FunctionType.predicate]>,
diff --git a/flang/test/Fir/fir-ops.fir b/flang/test/Fir/fir-ops.fir
index 3585bf9efca3e..0892eb9fa0de8 100644
--- a/flang/test/Fir/fir-ops.fir
+++ b/flang/test/Fir/fir-ops.fir
@@ -843,6 +843,16 @@ func.func @embox_tdesc(%arg0: !fir.class<!fir.array<10x!fir.type<derived_poly{a:
   return
 }
 
+// CHECK-LABEL:   func.func @embox_with_source_box(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.heap<!fir.array<?xnone>>,
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.shape<1>,
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>> {
+func.func @embox_with_source_box(%arg0: !fir.heap<!fir.array<?xnone>>, %arg1: !fir.shape<1>, %arg2: !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>> {
+// CHECK:           %[[VAL_0:.*]] = fir.embox %[[ARG0]](%[[ARG1]]) source_box %[[ARG2]] : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+  %0 = fir.embox %arg0(%arg1) source_box %arg2 : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+  return %0 : !fir.class<!fir.heap<!fir.array<?xnone>>>
+}
+
 func.func @test_fortran_var_attrs() {
   %0 = fir.alloca !fir.heap<f32> {fortran_attrs = #fir.var_attrs<allocatable>}
   %1 = fir.alloca f32 {fortran_attrs = #fir.var_attrs<asynchronous>}

``````````

</details>


https://github.com/llvm/llvm-project/pull/148305

From flang-commits at lists.llvm.org  Fri Jul 11 15:08:00 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 11 Jul 2025 15:08:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147618@github.com>
Message-ID: <68718b40.170a0220.28b5e3.6de5@mx.google.com>


vzakhari wrote:

I made the suggested change in #148305.  I will run more testing with `-frepack-arrays` to make sure it also resolves the problem.

https://github.com/llvm/llvm-project/pull/147618

From flang-commits at lists.llvm.org  Fri Jul 11 15:08:31 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 15:08:31 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Allow embox's source_box to be a
	!fir.box. (PR #148305)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148305@github.com>
Message-ID: <68718b5f.170a0220.788ca.7277@mx.google.com>


https://github.com/clementval approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/148305

From flang-commits at lists.llvm.org  Fri Jul 11 15:08:56 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 11 Jul 2025 15:08:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147618@github.com>
Message-ID: <68718b78.170a0220.1e5193.7a95@mx.google.com>


clementval wrote:

> I made the suggested change in #148305. I will run more testing with `-frepack-arrays` to make sure it also resolves the problem.

Looks good to me. Thanks for the change. 

https://github.com/llvm/llvm-project/pull/147618

From flang-commits at lists.llvm.org  Fri Jul 11 15:47:39 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 11 Jul 2025 15:47:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147618@github.com>
Message-ID: <6871948b.170a0220.358342.73c3@mx.google.com>


vzakhari wrote:

Replaced by https://github.com/llvm/llvm-project/pull/148305

https://github.com/llvm/llvm-project/pull/147618

From flang-commits at lists.llvm.org  Fri Jul 11 15:47:40 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 11 Jul 2025 15:47:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed assumed-type temporary
 allocation for -frepack-arrays. (PR #147618)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147618@github.com>
Message-ID: <6871948c.170a0220.12a56f.837e@mx.google.com>


https://github.com/vzakhari closed https://github.com/llvm/llvm-project/pull/147618

From flang-commits at lists.llvm.org  Fri Jul 11 09:25:38 2025
From: flang-commits at lists.llvm.org (Pranav Bhandarkar via flang-commits)
Date: Fri, 11 Jul 2025 09:25:38 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add more flags for generating
	codegen tests (PR #148171)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148171@github.com>
Message-ID: <68713b02.170a0220.20f3b5.573a@mx.google.com>


https://github.com/bhandarkar-pranav approved this pull request.

Thank you for this PR! LGTM.

https://github.com/llvm/llvm-project/pull/148171

From flang-commits at lists.llvm.org  Sun Jul 13 22:34:33 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Sun, 13 Jul 2025 22:34:33 -0700 (PDT)
Subject: [flang-commits] [flang] 374d5da - [MLIR][Interfaces] Remove
 negative branch weight verifier (#148234)
Message-ID: <687496e9.170a0220.11083c.e2bf@mx.google.com>


Author: Christian Ulmann
Date: 2025-07-14T07:34:29+02:00
New Revision: 374d5da214ba8017614c350e9fe4688cdc79d5f5

URL: https://github.com/llvm/llvm-project/commit/374d5da214ba8017614c350e9fe4688cdc79d5f5
DIFF: https://github.com/llvm/llvm-project/commit/374d5da214ba8017614c350e9fe4688cdc79d5f5.diff

LOG: [MLIR][Interfaces] Remove negative branch weight verifier (#148234)

This commit removes the verifier that checked if branch weights are
negative. This check was too strict because weights are interpreted as
unsigned integers.

This showed up when running the verifier on LLVM dialect modules that
were imported from LLVM IR.

Added: 
    

Modified: 
    flang/test/Fir/invalid.fir
    mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
    mlir/lib/Interfaces/ControlFlowInterfaces.cpp
    mlir/test/Dialect/ControlFlow/invalid.mlir

Removed: 
    


################################################################################
diff  --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index e32ea7ad3c729..e5dbec44220b7 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -1426,12 +1426,3 @@ func.func @wrong_weights_number_in_if_then_else(%cond: i1) {
   }
   return
 }
-
-// -----
-
-func.func @negative_weight_in_if_then(%cond: i1) {
-// expected-error @below {{weight #0 must be non-negative}}
-  fir.if %cond weights([-1, 101]) {
-  }
-  return
-}

diff  --git a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
index 46ab0b9ebbc6b..b8d08cc553caa 100644
--- a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
+++ b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
@@ -384,13 +384,15 @@ def WeightedBranchOpInterface : OpInterface<"WeightedBranchOpInterface"> {
     This interface provides weight information for branching terminator
     operations, i.e. terminator operations with successors.
 
-    This interface provides methods for getting/setting integer non-negative
-    weight of each branch. The probability of executing a branch
-    is computed as the ratio between the branch's weight and the total
-    sum of the weights (which cannot be zero).
-    The weights are optional. If they are provided, then their number
+    This interface provides methods for getting/setting integer weights of each
+    branch. The probability of executing a branch is computed as the ratio
+    between the branch's weight and the total sum of the weights (which cannot
+    be zero). The weights are optional. If they are provided, then their number
     must match the number of successors of the operation.
 
+    Note that the branch weight use an i32 representation but they are to be
+    interpreted as unsigned integers.
+
     The default implementations of the methods expect the operation
     to have an attribute of type DenseI32ArrayAttr named branch_weights.
   }];
@@ -440,19 +442,21 @@ def WeightedRegionBranchOpInterface
     This interface provides weight information for region operations
     that exhibit branching behavior between held regions.
 
-    This interface provides methods for getting/setting integer non-negative
-    weight of each branch. The probability of executing a region is computed
-    as the ratio between the region branch's weight and the total sum
-    of the weights (which cannot be zero).
-    The weights are optional. If they are provided, then their number
-    must match the number of regions held by the operation
-    (including empty regions).
+    This interface provides methods for getting/setting integer weights of each
+    branch. The probability of executing a region is computed as the ratio
+    between the region branch's weight and the total sum of the weights (which
+    cannot be zero). The weights are optional. If they are provided, then their
+    number must match the number of regions held by the operation (including
+    empty regions).
 
     The weights specify the probability of branching to a particular
     region when first executing the operation.
     For example, for loop-like operations with a single region
     the weight specifies the probability of entering the loop.
 
+    Note that the branch weight use an i32 representation but they are to be
+    interpreted as unsigned integers.
+
     The default implementations of the methods expect the operation
     to have an attribute of type DenseI32ArrayAttr named branch_weights.
   }];

diff  --git a/mlir/lib/Interfaces/ControlFlowInterfaces.cpp b/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
index 3a63db35eec0f..e87bb461b0329 100644
--- a/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
+++ b/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
@@ -99,10 +99,6 @@ static LogicalResult verifyWeights(Operation *op,
                            << ": " << weights.size() << " vs "
                            << expectedWeightsNum;
 
-  for (auto [index, weight] : llvm::enumerate(weights))
-    if (weight < 0)
-      return op->emitError() << "weight #" << index << " must be non-negative";
-
   if (llvm::all_of(weights, [](int32_t value) { return value == 0; }))
     return op->emitError() << "branch weights cannot all be zero";
 

diff  --git a/mlir/test/Dialect/ControlFlow/invalid.mlir b/mlir/test/Dialect/ControlFlow/invalid.mlir
index 1b8de22a9ff9f..0a71c62ec31af 100644
--- a/mlir/test/Dialect/ControlFlow/invalid.mlir
+++ b/mlir/test/Dialect/ControlFlow/invalid.mlir
@@ -82,18 +82,6 @@ func.func @wrong_weights_number(%cond: i1) {
 
 // -----
 
-// CHECK-LABEL: func @negative_weight
-func.func @wrong_total_weight(%cond: i1) {
-  // expected-error at +1 {{weight #0 must be non-negative}}
-  cf.cond_br %cond weights([-1, 101]), ^bb1, ^bb2
-  ^bb1:
-    return
-  ^bb2:
-    return
-}
-
-// -----
-
 // CHECK-LABEL: func @zero_weights
 func.func @wrong_total_weight(%cond: i1) {
   // expected-error at +1 {{branch weights cannot all be zero}}


        

From flang-commits at lists.llvm.org  Sun Jul 13 22:34:35 2025
From: flang-commits at lists.llvm.org (Christian Ulmann via flang-commits)
Date: Sun, 13 Jul 2025 22:34:35 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][Interfaces] Remove negative
 branch weight verifier (PR #148234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148234@github.com>
Message-ID: <687496eb.630a0220.1f27ee.f46e@mx.google.com>


https://github.com/Dinistro closed https://github.com/llvm/llvm-project/pull/148234

From flang-commits at lists.llvm.org  Mon Jul 14 02:13:54 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Mon, 14 Jul 2025 02:13:54 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <6874ca52.170a0220.114e55.e749@mx.google.com>


https://github.com/DavidSpickett updated https://github.com/llvm/llvm-project/pull/148169

>From 1eac80b12fa0c2ca5c68132795b6d2f26e376855 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Wed, 9 Jul 2025 14:53:12 +0000
Subject: [PATCH 1/7] Reland "[flang] Avoid undefined behaviour when parsing
 format expressions (#147539)"

This reverts commit e8e5d07767c444913f837dd35846a92fcf520eab.

This previously failed because the flang-rt build could not find the
llvm header file. It passed on some machines but only because they
had globally installed copies of older llvm.

To fix this, I've copied the required routines from llvm into flang.

With the following justification:
* Flang can, and does, use llvm headers.
* Some Flang headers are also used in Flang-rt.
* Flang-rt cannot use llvm headers.
* Therefore any Flang header using in Flang-rt cannot use llvm headers either.

To support that conclusion, https://flang.llvm.org/docs/IORuntimeInternals.html
states:
"The Fortran I/O runtime support library is written in C++17, and uses some C++17 standard library facilities, but it is intended to not have any link-time dependences on the C++ runtime support library or any LLVM libraries."

This talks about libraries but I assume it applies to llvm in general.

Nothing in flang/include/flang/Common, or flang/include/flang/Common
includes any llvm header, and I see some very similar headers there
that duplicate llvm functionality. Like float128.h.

I can only assume this means these files must remain free of dependencies
on LLVM.

I have copied the two routines literally and put them in the flang::common
namespace, for lack of a better place for them. So they don't clash with something.

I have specialised the function to the 1 type flang needs, as it might
save a bit of compile time.
---
 flang/include/flang/Common/format.h | 75 +++++++++++++++++++++++++++--
 1 file changed, 70 insertions(+), 5 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 1650f56140b4d..78ddb1de2fa26 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -12,6 +12,7 @@
 #include "Fortran-consts.h"
 #include "enum-set.h"
 #include <cstring>
+#include <limits>
 
 // Define a FormatValidator class template to validate a format expression
 // of a given CHAR type.  To enable use in runtime library code as well as
@@ -28,6 +29,68 @@
 
 namespace Fortran::common {
 
+// AddOverflow and MulOverflow are copied from
+// llvm/include/llvm/Support/MathExtras.h. It is ok to include llvm headers in
+// flang, but this flang header is also used in flang-rt, which cannot use llvm
+// headers.
+
+/// Add two signed integers, computing the two's complement truncated result,
+/// returning true if overflow occurred.
+template <typename T>
+std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
+#if __has_builtin(__builtin_add_overflow)
+  return __builtin_add_overflow(X, Y, &Result);
+#else
+  // Perform the unsigned addition.
+  using U = std::make_unsigned_t<T>;
+  const U UX = static_cast<U>(X);
+  const U UY = static_cast<U>(Y);
+  const U UResult = UX + UY;
+
+  // Convert to signed.
+  Result = static_cast<T>(UResult);
+
+  // Adding two positive numbers should result in a positive number.
+  if (X > 0 && Y > 0)
+    return Result <= 0;
+  // Adding two negatives should result in a negative number.
+  if (X < 0 && Y < 0)
+    return Result >= 0;
+  return false;
+#endif
+}
+
+/// Multiply two signed integers, computing the two's complement truncated
+/// result, returning true if an overflow occurred.
+template <typename T>
+std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
+#if __has_builtin(__builtin_mul_overflow)
+  return __builtin_mul_overflow(X, Y, &Result);
+#else
+  // Perform the unsigned multiplication on absolute values.
+  using U = std::make_unsigned_t<T>;
+  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
+  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
+  const U UResult = UX * UY;
+
+  // Convert to signed.
+  const bool IsNegative = (X < 0) ^ (Y < 0);
+  Result = IsNegative ? (0 - UResult) : UResult;
+
+  // If any of the args was 0, result is 0 and no overflow occurs.
+  if (UX == 0 || UY == 0)
+    return false;
+
+  // UX and UY are in [1, 2^n], where n is the number of digits.
+  // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
+  // positive) divided by an argument compares to the other.
+  if (IsNegative)
+    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
+  else
+    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
+#endif
+}
+
 struct FormatMessage {
   const char *text; // message text; may have one %s argument
   const char *arg; // optional %s argument value
@@ -214,16 +277,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (!overflow) {
+        overflow =
+            MulOverflow(static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (!overflow) {
+        overflow = AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
       }
       c = NextChar();
     } while (c >= '0' && c <= '9');

>From aa10d5dfe83482f41aa306741c8c6756fe913753 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 08:20:44 +0000
Subject: [PATCH 2/7] manually specialise template

---
 flang/include/flang/Common/format.h | 30 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 78ddb1de2fa26..3c70aea1ec721 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -30,25 +30,21 @@
 namespace Fortran::common {
 
 // AddOverflow and MulOverflow are copied from
-// llvm/include/llvm/Support/MathExtras.h. It is ok to include llvm headers in
-// flang, but this flang header is also used in flang-rt, which cannot use llvm
-// headers.
+// llvm/include/llvm/Support/MathExtras.h and specialised to int64_t.
 
 /// Add two signed integers, computing the two's complement truncated result,
 /// returning true if overflow occurred.
-template <typename T>
-std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
+bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_add_overflow)
   return __builtin_add_overflow(X, Y, &Result);
 #else
   // Perform the unsigned addition.
-  using U = std::make_unsigned_t<T>;
-  const U UX = static_cast<U>(X);
-  const U UY = static_cast<U>(Y);
-  const U UResult = UX + UY;
+  const uint64_t UX = static_cast<uint64_t>(X);
+  const uint64_t UY = static_cast<uint64_t>(Y);
+  const uint64_t UResult = UX + UY;
 
   // Convert to signed.
-  Result = static_cast<T>(UResult);
+  Result = static_cast<int64_t>(UResult);
 
   // Adding two positive numbers should result in a positive number.
   if (X > 0 && Y > 0)
@@ -62,16 +58,14 @@ std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
 
 /// Multiply two signed integers, computing the two's complement truncated
 /// result, returning true if an overflow occurred.
-template <typename T>
-std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
+int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_mul_overflow)
   return __builtin_mul_overflow(X, Y, &Result);
 #else
   // Perform the unsigned multiplication on absolute values.
-  using U = std::make_unsigned_t<T>;
-  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
-  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
-  const U UResult = UX * UY;
+  const uint64_t UX = X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
+  const uint64_t UY = Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
+  const uint64_t UResult = UX * UY;
 
   // Convert to signed.
   const bool IsNegative = (X < 0) ^ (Y < 0);
@@ -85,9 +79,9 @@ std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) / UY;
   else
-    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
 #endif
 }
 

>From 89009a28fadef67310f0310cc90086ef05afb859 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 09:23:29 +0000
Subject: [PATCH 3/7] format

---
 flang/include/flang/Common/format.h | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 3c70aea1ec721..a853a636bdbe0 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -63,8 +63,10 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   return __builtin_mul_overflow(X, Y, &Result);
 #else
   // Perform the unsigned multiplication on absolute values.
-  const uint64_t UX = X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
-  const uint64_t UY = Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
+  const uint64_t UX =
+      X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
+  const uint64_t UY =
+      Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
   const uint64_t UResult = UX * UY;
 
   // Convert to signed.
@@ -79,7 +81,9 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) / UY;
+    return UX >
+        (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) /
+        UY;
   else
     return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
 #endif

>From b7fc8a5fdbe9a66a775519f25425f9297741042a Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 12:09:01 +0000
Subject: [PATCH 4/7] fix limit template

---
 flang/include/flang/Common/format.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index a853a636bdbe0..91d1db2484557 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -82,10 +82,10 @@ int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // positive) divided by an argument compares to the other.
   if (IsNegative)
     return UX >
-        (static_cast<uint64_t>(std::numeric_limits<T>::max()) + uint64_t(1)) /
+        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + uint64_t(1)) /
         UY;
   else
-    return UX > (static_cast<uint64_t>(std::numeric_limits<T>::max())) / UY;
+    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
 #endif
 }
 

>From 4616aed4c157063dea6a561397a28388ea567804 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 13:01:52 +0000
Subject: [PATCH 5/7] Make static

---
 flang/include/flang/Common/format.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 91d1db2484557..ed12e3a5f6273 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -34,7 +34,7 @@ namespace Fortran::common {
 
 /// Add two signed integers, computing the two's complement truncated result,
 /// returning true if overflow occurred.
-bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
+static inline bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_add_overflow)
   return __builtin_add_overflow(X, Y, &Result);
 #else
@@ -58,7 +58,7 @@ bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
 
 /// Multiply two signed integers, computing the two's complement truncated
 /// result, returning true if an overflow occurred.
-int64_t MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
+static inline bool MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
 #if __has_builtin(__builtin_mul_overflow)
   return __builtin_mul_overflow(X, Y, &Result);
 #else

>From 3baaad3723767f29b3b434b71e6d2b080b2e38a5 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Fri, 11 Jul 2025 13:08:19 +0000
Subject: [PATCH 6/7] format

---
 flang/include/flang/Common/format.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index ed12e3a5f6273..8b4ffe63c4459 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -81,11 +81,12 @@ static inline bool MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
   if (IsNegative)
-    return UX >
-        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + uint64_t(1)) /
+    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) +
+                    uint64_t(1)) /
         UY;
   else
-    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
+    return UX >
+        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
 #endif
 }
 

>From e2d85d266929acc5d9f88691b7bf783399fbf126 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett at linaro.org>
Date: Mon, 14 Jul 2025 09:13:32 +0000
Subject: [PATCH 7/7] update to flang codestyle

---
 flang/include/flang/Common/format.h | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 8b4ffe63c4459..848aa9809e4b7 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -47,11 +47,13 @@ static inline bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
   Result = static_cast<int64_t>(UResult);
 
   // Adding two positive numbers should result in a positive number.
-  if (X > 0 && Y > 0)
+  if (X > 0 && Y > 0) {
     return Result <= 0;
+  }
   // Adding two negatives should result in a negative number.
-  if (X < 0 && Y < 0)
+  if (X < 0 && Y < 0) {
     return Result >= 0;
+  }
   return false;
 #endif
 }
@@ -74,19 +76,21 @@ static inline bool MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
   Result = IsNegative ? (0 - UResult) : UResult;
 
   // If any of the args was 0, result is 0 and no overflow occurs.
-  if (UX == 0 || UY == 0)
+  if (UX == 0 || UY == 0) {
     return false;
+  }
 
   // UX and UY are in [1, 2^n], where n is the number of digits.
   // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
   // positive) divided by an argument compares to the other.
-  if (IsNegative)
+  if (IsNegative) {
     return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) +
                     uint64_t(1)) /
         UY;
-  else
+  } else {
     return UX >
         (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
+  }
 #endif
 }
 


From flang-commits at lists.llvm.org  Mon Jul 14 02:14:08 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 02:14:08 -0700 (PDT)
Subject: [flang-commits] [flang] 29d8c34 - Reland "[flang] Avoid undefined
 behaviour when parsing format expressions (#147539)" (#148169)
Message-ID: <6874ca60.630a0220.39aeca.febc@mx.google.com>


Author: David Spickett
Date: 2025-07-14T10:14:04+01:00
New Revision: 29d8c346c58cc6601e8551c235f957e3cceabd84

URL: https://github.com/llvm/llvm-project/commit/29d8c346c58cc6601e8551c235f957e3cceabd84
DIFF: https://github.com/llvm/llvm-project/commit/29d8c346c58cc6601e8551c235f957e3cceabd84.diff

LOG: Reland "[flang] Avoid undefined behaviour when parsing format expressions (#147539)" (#148169)

This reverts commit e8e5d07767c444913f837dd35846a92fcf520eab.

This previously failed because the flang-rt build could not find the 
llvm header file. It passed on some machines but only because they
had globally installed copies of older llvm.

To fix this, I've copied the required routines from llvm into flang.

With the following justification:
* Flang can, and does, use llvm headers.
* Some Flang headers are also used in Flang-rt.
* Flang-rt cannot use llvm headers.
* Therefore any Flang header using in Flang-rt cannot use llvm headers
either.

To support that conclusion,
https://flang.llvm.org/docs/IORuntimeInternals.html
states:
"The Fortran I/O runtime support library is written in C++17, and uses
some C++17 standard library facilities, but it is intended to not have
any link-time dependences on the C++ runtime support library or any LLVM
libraries."

This talks about libraries but I assume it applies to llvm in general.

Nothing in flang/include/flang/Common, or flang/include/flang/Common
includes any llvm header, and I see some very similar headers there
that duplicate llvm functionality. Like float128.h.

I can only assume this means these files must remain free of
dependencies
on LLVM.

I have copied the two routines literally and put them in the
flang::common
namespace, for lack of a better place for them. So they don't clash with
something.

I have specialised the function to the 1 type flang needs, as it might
save a bit of compile time.

Added: 
    

Modified: 
    flang/include/flang/Common/format.h

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 1650f56140b4d..848aa9809e4b7 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -12,6 +12,7 @@
 #include "Fortran-consts.h"
 #include "enum-set.h"
 #include <cstring>
+#include <limits>
 
 // Define a FormatValidator class template to validate a format expression
 // of a given CHAR type.  To enable use in runtime library code as well as
@@ -28,6 +29,71 @@
 
 namespace Fortran::common {
 
+// AddOverflow and MulOverflow are copied from
+// llvm/include/llvm/Support/MathExtras.h and specialised to int64_t.
+
+/// Add two signed integers, computing the two's complement truncated result,
+/// returning true if overflow occurred.
+static inline bool AddOverflow(int64_t X, int64_t Y, int64_t &Result) {
+#if __has_builtin(__builtin_add_overflow)
+  return __builtin_add_overflow(X, Y, &Result);
+#else
+  // Perform the unsigned addition.
+  const uint64_t UX = static_cast<uint64_t>(X);
+  const uint64_t UY = static_cast<uint64_t>(Y);
+  const uint64_t UResult = UX + UY;
+
+  // Convert to signed.
+  Result = static_cast<int64_t>(UResult);
+
+  // Adding two positive numbers should result in a positive number.
+  if (X > 0 && Y > 0) {
+    return Result <= 0;
+  }
+  // Adding two negatives should result in a negative number.
+  if (X < 0 && Y < 0) {
+    return Result >= 0;
+  }
+  return false;
+#endif
+}
+
+/// Multiply two signed integers, computing the two's complement truncated
+/// result, returning true if an overflow occurred.
+static inline bool MulOverflow(int64_t X, int64_t Y, int64_t &Result) {
+#if __has_builtin(__builtin_mul_overflow)
+  return __builtin_mul_overflow(X, Y, &Result);
+#else
+  // Perform the unsigned multiplication on absolute values.
+  const uint64_t UX =
+      X < 0 ? (0 - static_cast<uint64_t>(X)) : static_cast<uint64_t>(X);
+  const uint64_t UY =
+      Y < 0 ? (0 - static_cast<uint64_t>(Y)) : static_cast<uint64_t>(Y);
+  const uint64_t UResult = UX * UY;
+
+  // Convert to signed.
+  const bool IsNegative = (X < 0) ^ (Y < 0);
+  Result = IsNegative ? (0 - UResult) : UResult;
+
+  // If any of the args was 0, result is 0 and no overflow occurs.
+  if (UX == 0 || UY == 0) {
+    return false;
+  }
+
+  // UX and UY are in [1, 2^n], where n is the number of digits.
+  // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
+  // positive) divided by an argument compares to the other.
+  if (IsNegative) {
+    return UX > (static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) +
+                    uint64_t(1)) /
+        UY;
+  } else {
+    return UX >
+        (static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) / UY;
+  }
+#endif
+}
+
 struct FormatMessage {
   const char *text; // message text; may have one %s argument
   const char *arg; // optional %s argument value
@@ -214,16 +280,18 @@ template <typename CHAR> void FormatValidator<CHAR>::NextToken() {
   case '7':
   case '8':
   case '9': {
-    int64_t lastValue;
     const CHAR *lastCursor;
     integerValue_ = 0;
     bool overflow{false};
     do {
-      lastValue = integerValue_;
       lastCursor = cursor_;
-      integerValue_ = 10 * integerValue_ + c - '0';
-      if (lastValue > integerValue_) {
-        overflow = true;
+      if (!overflow) {
+        overflow =
+            MulOverflow(static_cast<int64_t>(10), integerValue_, integerValue_);
+      }
+      if (!overflow) {
+        overflow = AddOverflow(
+            integerValue_, static_cast<int64_t>(c - '0'), integerValue_);
       }
       c = NextChar();
     } while (c >= '0' && c <= '9');


        

From flang-commits at lists.llvm.org  Mon Jul 14 02:14:14 2025
From: flang-commits at lists.llvm.org (David Spickett via flang-commits)
Date: Mon, 14 Jul 2025 02:14:14 -0700 (PDT)
Subject: [flang-commits] [flang] Reland "[flang] Avoid undefined behaviour
 when parsing format expressions (#147539)" (PR #148169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148169@github.com>
Message-ID: <6874ca66.630a0220.2d2879.f5c9@mx.google.com>


https://github.com/DavidSpickett closed https://github.com/llvm/llvm-project/pull/148169

From flang-commits at lists.llvm.org  Mon Jul 14 02:25:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 02:25:01 -0700 (PDT)
Subject: [flang-commits] [flang] 51d3bc6 - [flang][tco] Add more flags for
 generating codegen tests (#148171)
Message-ID: <6874cced.050a0220.15e9ea.02a3@mx.google.com>


Author: Tom Eccles
Date: 2025-07-14T10:24:58+01:00
New Revision: 51d3bc67d0e11bacf6566690a07779f238845cc7

URL: https://github.com/llvm/llvm-project/commit/51d3bc67d0e11bacf6566690a07779f238845cc7
DIFF: https://github.com/llvm/llvm-project/commit/51d3bc67d0e11bacf6566690a07779f238845cc7.diff

LOG: [flang][tco] Add more flags for generating codegen tests (#148171)

Built on top of https://github.com/llvm/llvm-project/pull/146533

Adding some more options I use to generate minimal testcases for
MLIR->LLVMIR conversion.
Hopefully this will save us all some time when writing these tests.

The following options are added
- `-simplify-mlir` runs CSE and canonicalization at the end of the MLIR
pipeline
- `-enable-aa` allows toggling whether TBAA is enabled
- `-test-gen` is shorthand for `-emit-final-mlir -simplify-mlir
-enable-aa=false`

Added: 
    flang/test/Driver/tco-test-gen.fir

Modified: 
    flang/tools/tco/tco.cpp

Removed: 
    


################################################################################
diff  --git a/flang/test/Driver/tco-test-gen.fir b/flang/test/Driver/tco-test-gen.fir
new file mode 100644
index 0000000000000..38d4e50ecf3aa
--- /dev/null
+++ b/flang/test/Driver/tco-test-gen.fir
@@ -0,0 +1,108 @@
+// RUN: tco -emit-final-mlir %s | FileCheck %s --check-prefixes=CHECK,AA,CMPLX
+// RUN: tco -emit-final-mlir -enable-aa=false %s | FileCheck %s --check-prefixes=CHECK,NOAA,CMPLX
+// RUN: tco -emit-final-mlir -simplify-mlir %s | FileCheck %s --check-prefixes=CHECK,AA,SIMPLE
+// RUN: tco -emit-final-mlir -enable-aa=false -simplify-mlir %s | FileCheck %s --check-prefixes=CHECK,NOAA,SIMPLE
+// RUN: tco -test-gen %s | FileCheck %s --check-prefixes=CHECK,NOAA,SIMPLE
+
+// Just a dummy function that exhibits all of the things we want to turn on and off
+func.func @_QPtest(%arg0: !fir.ref<i32> {fir.bindc_name = "num"}, %arg1: !fir.ref<i32> {fir.bindc_name = "lb"}, %arg2: !fir.ref<i32> {fir.bindc_name = "ub"}, %arg3: !fir.ref<i32> {fir.bindc_name = "step"}) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtestEi"}
+  %2:2 = hlfir.declare %1 {uniq_name = "_QFtestEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3:2 = hlfir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtestElb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEnum"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg3 dummy_scope %0 {uniq_name = "_QFtestEstep"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtestEub"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = fir.load %3#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = fir.load %6#0 : !fir.ref<i32>
+  %10 = fir.convert %9 : (i32) -> index
+  %11 = fir.load %5#0 : !fir.ref<i32>
+  %12 = fir.convert %11 : (i32) -> index
+  %13 = fir.convert %8 : (index) -> i32
+  %14:2 = fir.do_loop %arg4 = %8 to %10 step %12 iter_args(%arg5 = %13) -> (index, i32) {
+    fir.store %arg5 to %2#0 : !fir.ref<i32>
+    %15 = fir.load %4#0 : !fir.ref<i32>
+    %16 = fir.load %2#0 : !fir.ref<i32>
+    %17 = arith.addi %15, %16 : i32
+    hlfir.assign %17 to %4#0 : i32, !fir.ref<i32>
+    %18 = arith.addi %arg4, %12 overflow<nsw> : index
+    %19 = fir.convert %12 : (index) -> i32
+    %20 = fir.load %2#0 : !fir.ref<i32>
+    %21 = arith.addi %20, %19 overflow<nsw> : i32
+    fir.result %18, %21 : index, i32
+  }
+  fir.store %14#1 to %2#0 : !fir.ref<i32>
+  return
+}
+
+// CHECK-LABEL:   llvm.func @_QPtest(
+// CHECK-SAME:      %[[ARG0:.*]]: !llvm.ptr {fir.bindc_name = "num", llvm.nocapture},
+// CHECK-SAME:      %[[ARG1:.*]]: !llvm.ptr {fir.bindc_name = "lb", llvm.nocapture},
+// CHECK-SAME:      %[[ARG2:.*]]: !llvm.ptr {fir.bindc_name = "ub", llvm.nocapture},
+// CHECK-SAME:      %[[ARG3:.*]]: !llvm.ptr {fir.bindc_name = "step", llvm.nocapture}) {
+
+// CMPLX:           %[[VAL_0:.*]] = llvm.mlir.constant(1 : i64) : i64
+// CMPLX:           %[[VAL_1:.*]] = llvm.alloca %[[VAL_0]] x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+// CMPLX:           %[[VAL_2:.*]] = llvm.mlir.constant(1 : index) : i64
+// CMPLX:           %[[VAL_3:.*]] = llvm.mlir.constant(0 : index) : i64
+// CMPLX:           %[[VAL_4:.*]] = llvm.mlir.constant(1 : i64) : i64
+
+// SIMPLE:          %[[VAL_3:.*]] = llvm.mlir.constant(0 : index) : i64
+// SIMPLE:          %[[VAL_2:.*]] = llvm.mlir.constant(1 : index) : i64
+// SIMPLE:          %[[VAL_0:.*]] = llvm.mlir.constant(1 : i64) : i64
+// SIMPLE:          %[[VAL_1:.*]] = llvm.alloca %[[VAL_0]] x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+
+// AA:              %[[VAL_5:.*]] = llvm.load %[[ARG1]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "dummy arg data/_QFtestElb", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "dummy arg data/_QFtestElb", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : !llvm.ptr -> i32
+// NOAA:            %[[VAL_5:.*]] = llvm.load %[[ARG1]] : !llvm.ptr -> i32
+
+// CHECK:           %[[VAL_6:.*]] = llvm.sext %[[VAL_5]] : i32 to i64
+
+// AA:              %[[VAL_7:.*]] = llvm.load %[[ARG2]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "dummy arg data/_QFtestEub", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "dummy arg data/_QFtestEub", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : !llvm.ptr -> i32
+// NOAA:            %[[VAL_7:.*]] = llvm.load %[[ARG2]] : !llvm.ptr -> i32
+
+// CHECK:           %[[VAL_8:.*]] = llvm.sext %[[VAL_7]] : i32 to i64
+
+// AA:              %[[VAL_9:.*]] = llvm.load %[[ARG3]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "dummy arg data/_QFtestEstep", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "dummy arg data/_QFtestEstep", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : !llvm.ptr -> i32
+// NOAA:            %[[VAL_9:.*]] = llvm.load %[[ARG3]] : !llvm.ptr -> i32
+
+// CHECK:           %[[VAL_10:.*]] = llvm.sext %[[VAL_9]] : i32 to i64
+// CHECK:           %[[VAL_11:.*]] = llvm.trunc %[[VAL_6]] : i64 to i32
+// CHECK:           %[[VAL_12:.*]] = llvm.sub %[[VAL_8]], %[[VAL_6]] : i64
+// CHECK:           %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_10]] : i64
+// CHECK:           %[[VAL_14:.*]] = llvm.sdiv %[[VAL_13]], %[[VAL_10]] : i64
+// CHECK:           llvm.br ^bb1(%[[VAL_11]], %[[VAL_14]] : i32, i64)
+// CHECK:         ^bb1(%[[VAL_15:.*]]: i32, %[[VAL_16:.*]]: i64):
+// CHECK:           %[[VAL_17:.*]] = llvm.icmp "sgt" %[[VAL_16]], %{{.*}} : i64
+// CHECK:           llvm.cond_br %[[VAL_17]], ^bb2, ^bb3
+// CHECK:         ^bb2:
+
+// AA:              llvm.store %[[VAL_15]], %[[VAL_1]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "allocated data/_QFtestEi", members = {<#llvm.tbaa_type_desc<id = "allocated data", members = {<#llvm.tbaa_type_desc<id = "target data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "allocated data/_QFtestEi", members = {<#llvm.tbaa_type_desc<id = "allocated data", members = {<#llvm.tbaa_type_desc<id = "target data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : i32, !llvm.ptr
+// NOAA:            llvm.store %[[VAL_15]], %{{.*}} : i32, !llvm.ptr
+
+// AA:              %[[VAL_18:.*]] = llvm.load %[[ARG0]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "dummy arg data/_QFtestEnum", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "dummy arg data/_QFtestEnum", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : !llvm.ptr -> i32
+// NOAA:            %[[VAL_18:.*]] = llvm.load %[[ARG0]] : !llvm.ptr -> i32
+
+// AA:              %[[VAL_19:.*]] = llvm.load %[[VAL_1]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "allocated data/_QFtestEi", members = {<#llvm.tbaa_type_desc<id = "allocated data", members = {<#llvm.tbaa_type_desc<id = "target data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "allocated data/_QFtestEi", members = {<#llvm.tbaa_type_desc<id = "allocated data", members = {<#llvm.tbaa_type_desc<id = "target data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : !llvm.ptr -> i32
+// NOAA:            %[[VAL_19:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32
+
+// CHECK:           %[[VAL_20:.*]] = llvm.add %[[VAL_18]], %[[VAL_19]] : i32
+// AA:              llvm.store %[[VAL_20]], %[[ARG0]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "dummy arg data/_QFtestEnum", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "dummy arg data/_QFtestEnum", members = {<#llvm.tbaa_type_desc<id = "dummy arg data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : i32, !llvm.ptr
+// NOAA:            llvm.store %[[VAL_20]], %[[ARG0]] : i32, !llvm.ptr
+
+// CHECK:           %[[VAL_21:.*]] = llvm.trunc %[[VAL_10]] : i64 to i32
+
+// AA:              %[[VAL_22:.*]] = llvm.load %[[VAL_1]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "allocated data/_QFtestEi", members = {<#llvm.tbaa_type_desc<id = "allocated data", members = {<#llvm.tbaa_type_desc<id = "target data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "allocated data/_QFtestEi", members = {<#llvm.tbaa_type_desc<id = "allocated data", members = {<#llvm.tbaa_type_desc<id = "target data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : !llvm.ptr -> i32
+// NOAA:            %[[VAL_22:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32
+
+// CHECK:           %[[VAL_23:.*]] = llvm.add %[[VAL_22]], %[[VAL_21]] overflow<nsw> : i32
+// CHECK:           %[[VAL_24:.*]] = llvm.sub %[[VAL_16]], %{{.*}} : i64
+// CHECK:           llvm.br ^bb1(%[[VAL_23]], %[[VAL_24]] : i32, i64)
+// CHECK:         ^bb3:
+
+// AA:              llvm.store %[[VAL_15]], %[[VAL_1]] {tbaa = [#llvm.tbaa_tag<base_type = <id = "allocated data/_QFtestEi", members = {<#llvm.tbaa_type_desc<id = "allocated data", members = {<#llvm.tbaa_type_desc<id = "target data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, 0>}>, access_type = <id = "allocated data/_QFtestEi", members = {<#llvm.tbaa_type_desc<id = "allocated data", members = {<#llvm.tbaa_type_desc<id = "target data", members = {<#llvm.tbaa_type_desc<id = "any data access", members = {<#llvm.tbaa_type_desc<id = "any access", members = {<#llvm.tbaa_root<id = "Flang function root _QPtest">, 0>}>, 0>}>, 0>}>, 0>}>, 0>}>, offset = 0>]} : i32, !llvm.ptr
+// NOAA:            llvm.store %[[VAL_15]], %{{.*}} : i32, !llvm.ptr
+
+// CHECK:           llvm.return
+// CHECK:         }
+

diff  --git a/flang/tools/tco/tco.cpp b/flang/tools/tco/tco.cpp
index 806cd3148d09f..d8daf8769cb21 100644
--- a/flang/tools/tco/tco.cpp
+++ b/flang/tools/tco/tco.cpp
@@ -75,6 +75,20 @@ static cl::opt<bool> emitFinalMLIR(
     cl::desc("Only translate FIR to MLIR, do not lower to LLVM IR"),
     cl::init(false));
 
+static cl::opt<bool>
+    simplifyMLIR("simplify-mlir",
+                 cl::desc("Run CSE and canonicalization on MLIR output"),
+                 cl::init(false));
+
+// Enabled by default to accurately reflect -O2
+static cl::opt<bool> enableAliasAnalysis("enable-aa",
+                                         cl::desc("Enable FIR alias analysis"),
+                                         cl::init(true));
+
+static cl::opt<bool> testGeneratorMode(
+    "test-gen", cl::desc("-emit-final-mlir -simplify-mlir -enable-aa=false"),
+    cl::init(false));
+
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
 
@@ -145,7 +159,7 @@ compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
   } else {
     MLIRToLLVMPassPipelineConfig config(llvm::OptimizationLevel::O2);
     config.EnableOpenMP = true;  // assume the input contains OpenMP
-    config.AliasAnalysis = true; // enabled when optimizing for speed
+    config.AliasAnalysis = enableAliasAnalysis && !testGeneratorMode;
     if (codeGenLLVM) {
       // Run only CodeGen passes.
       fir::createDefaultFIRCodeGenPassPipeline(pm, config);
@@ -154,14 +168,19 @@ compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
       fir::registerDefaultInlinerPass(config);
       fir::createMLIRToLLVMPassPipeline(pm, config);
     }
-    if (!emitFinalMLIR)
+    if (simplifyMLIR || testGeneratorMode) {
+      pm.addPass(mlir::createCanonicalizerPass());
+      pm.addPass(mlir::createCSEPass());
+    }
+    if (!emitFinalMLIR && !testGeneratorMode)
       fir::addLLVMDialectToLLVMPass(pm, out.os());
   }
 
   // run the pass manager
   if (mlir::succeeded(pm.run(*owningRef))) {
     // passes ran successfully, so keep the output
-    if ((emitFir || passPipeline.hasAnyOccurrences() || emitFinalMLIR) &&
+    if ((emitFir || passPipeline.hasAnyOccurrences() || emitFinalMLIR ||
+         testGeneratorMode) &&
         !codeGenLLVM)
       printModule(*owningRef, out.os());
     out.keep();


        

From flang-commits at lists.llvm.org  Mon Jul 14 02:27:06 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Mon, 14 Jul 2025 02:27:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Support `do concurrent ... reduce`
 for associating names (PR #148597)
Message-ID: <llvm.org/llvm/llvm-project/pull/148597@github.com>


https://github.com/ergawy created https://github.com/llvm/llvm-project/pull/148597

Extends reduction support for `do concurrent`, in particular, for associating names. Consider the following input:
```fortran
subroutine dc_associate_reduce
  integer :: i
  real, allocatable, dimension(:) :: x

  associate(x_associate => x)
  do concurrent (i = 1:10) reduce(+: x_associate)
  end do
  end associate
end subroutine
```

The declaration of `x_associate` is emitted as follows:
```mlir
%13:2 = hlfir.declare %10(%12) {uniq_name = "...."} : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
```
where the HLFIR base type is an array descriptor (i.e. the allocatable/heap attribute is dropped as stipulated by the spec; section 11.1.3.3).

The problem here is that `declare_reduction` ops accept only reference types. This restriction is alreay partially handed for `fir::BaseBoxType`'s by allocating a stack slot for the descriptor and storing the box in that stack allocation. We have to modify this a littble bit for `associate` since the HLFIR and FIR base types are different (unlike most scenarios).

>From 0aa36a626cf58ae7c8cb80000a5df8ac16811324 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Mon, 14 Jul 2025 04:14:53 -0500
Subject: [PATCH] [flang] Support `do concurrent ... reduce` for associating
 names

Extends reduction support for `do concurrent`, in particular, for
associating names. Consider the following input:
```fortran
subroutine dc_associate_reduce
  integer :: i
  real, allocatable, dimension(:) :: x

  associate(x_associate => x)
  do concurrent (i = 1:10) reduce(+: x_associate)
  end do
  end associate
end subroutine
```

The declaration of `x_associate` is emitted as follows:
```mlir
%13:2 = hlfir.declare %10(%12) {uniq_name = "...."} : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
```
where the HLFIR base type is an array descriptor (i.e. the
allocatable/heap attribute is dropped as stipulated by the spec; section
11.1.3.3).

The problem here is that `declare_reduction` ops accept only reference
types. This restriction is alreay partially handed for
`fir::BaseBoxType`'s by allocating a stack slot for the descriptor and
storing the box in that stack allocation. We have to modify this a
littble bit for `associate` since the HLFIR and FIR base types are
different (unlike most scenarios).
---
 .../lib/Lower/Support/ReductionProcessor.cpp  | 34 +++++++++++++++----
 .../OpenMP/parallel-reduction-array-lb.f90    |  2 +-
 .../OpenMP/reduction-array-intrinsic.f90      |  4 +--
 .../Lower/OpenMP/sections-array-reduction.f90 |  2 +-
 .../OpenMP/taskgroup-task-array-reduction.f90 |  2 +-
 .../wsloop-reduction-array-assumed-shape.f90  |  2 +-
 .../Lower/do_concurrent_reduce_associate.f90  | 20 +++++++++++
 7 files changed, 54 insertions(+), 12 deletions(-)
 create mode 100644 flang/test/Lower/do_concurrent_reduce_associate.f90

diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 14b2c9836748f..ddcecd8c1bb02 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -633,13 +633,25 @@ void ReductionProcessor::processReductionArguments(
     }
   }
 
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
   // Reduction variable processing common to both intrinsic operators and
   // procedure designators
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  mlir::OpBuilder::InsertPoint dcIP;
+  constexpr bool isDoConcurrent =
+      std::is_same_v<OpType, fir::DeclareReductionOp>;
+
+  if (isDoConcurrent) {
+    dcIP = builder.saveInsertionPoint();
+    builder.setInsertionPoint(
+        builder.getRegion().getParentOfType<fir::DoConcurrentOp>());
+  }
+
   for (const semantics::Symbol *symbol : reductionSymbols) {
     mlir::Value symVal = converter.getSymbolAddress(*symbol);
+
+    if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>())
+      symVal = declOp.getBase();
+
     mlir::Type eleType;
     auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
     if (refType)
@@ -667,13 +679,20 @@ void ReductionProcessor::processReductionArguments(
       // boxed arrays are passed as values not by reference. Unfortunately,
       // we can't pass a box by value to omp.redution_declare, so turn it
       // into a reference
+      auto oldIP = builder.saveInsertionPoint();
+      builder.setInsertionPointToStart(builder.getAllocaBlock());
+      auto alloca =
+          builder.create<fir::AllocaOp>(currentLocation, symVal.getType());
+      builder.restoreInsertionPoint(oldIP);
+      builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
+      symVal = alloca;
+    }
 
+    if (mlir::isa<fir::BaseBoxType>(symVal.getType())) {
       auto alloca =
           builder.create<fir::AllocaOp>(currentLocation, symVal.getType());
       builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
       symVal = alloca;
-    } else if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>()) {
-      symVal = declOp.getBase();
     }
 
     // this isn't the same as the by-val and by-ref passing later in the
@@ -693,7 +712,7 @@ void ReductionProcessor::processReductionArguments(
   unsigned idx = 0;
   for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
     auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
-    const auto &kindMap = firOpBuilder.getKindMap();
+    const auto &kindMap = builder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
 
@@ -745,9 +764,12 @@ void ReductionProcessor::processReductionArguments(
     OpType decl = createDeclareReduction<OpType>(
         converter, reductionName, redId, redType, currentLocation, isByRef);
     reductionDeclSymbols.push_back(
-        mlir::SymbolRefAttr::get(firOpBuilder.getContext(), decl.getSymName()));
+        mlir::SymbolRefAttr::get(builder.getContext(), decl.getSymName()));
     ++idx;
   }
+
+  if (isDoConcurrent)
+    builder.restoreInsertionPoint(dcIP);
 }
 
 const semantics::SourceName
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index a5710fcf5352b..ec54294c7104f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -69,6 +69,7 @@ program reduce
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
@@ -76,7 +77,6 @@ program reduce
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape_shift %[[VAL_1]], %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : (index, index, index, index) -> !fir.shapeshift<2>
 ! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3x2xi32>>, !fir.shapeshift<2>) -> (!fir.box<!fir.array<3x2xi32>>, !fir.ref<!fir.array<3x2xi32>>)
-! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           fir.store %[[VAL_6]]#0 to %[[VAL_7]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
 ! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3x2xi32 %[[VAL_7]] -> %[[VAL_8:.*]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>) {
 ! CHECK:             %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3x2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3x2xi32>>>, !fir.ref<!fir.box<!fir.array<3x2xi32>>>)
diff --git a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90 b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
index 0cf88cf889868..104904497745d 100644
--- a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
+++ b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
@@ -64,11 +64,11 @@ subroutine max_array_reduction(l, r)
 ! CHECK-LABEL:   func.func @_QPmax_array_reduction(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "l"},
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
 ! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFmax_array_reductionEl"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFmax_array_reductionEr"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
-! CHECK:           fir.store %[[VAL_3]]#1 to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+! CHECK:           fir.store %[[VAL_3]]#0 to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
 ! CHECK:           omp.parallel reduction(byref @max_byref_box_Uxi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
 ! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmax_array_reductionEl"} : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> (!fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:             %[[VAL_8:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
diff --git a/flang/test/Lower/OpenMP/sections-array-reduction.f90 b/flang/test/Lower/OpenMP/sections-array-reduction.f90
index 91e0680692637..2f2808cebfc0c 100644
--- a/flang/test/Lower/OpenMP/sections-array-reduction.f90
+++ b/flang/test/Lower/OpenMP/sections-array-reduction.f90
@@ -34,7 +34,7 @@ subroutine sectionsReduction(x)
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFsectionsreductionEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
-! CHECK:             fir.store %[[VAL_2]]#1 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
+! CHECK:             fir.store %[[VAL_2]]#0 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:             omp.sections reduction(byref @add_reduction_byref_box_Uxf32 %[[VAL_3]] -> %[[VAL_4:.*]] : !fir.ref<!fir.box<!fir.array<?xf32>>>) {
 ! CHECK:               omp.section {
 ! CHECK:               ^bb0(%[[VAL_5:.*]]: !fir.ref<!fir.box<!fir.array<?xf32>>>):
diff --git a/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90 b/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
index 18d45217272fc..18a4f75b86309 100644
--- a/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
+++ b/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
@@ -22,7 +22,7 @@
 ! CHECK-SAME:      {uniq_name = "_QFtask_reductionEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:          omp.parallel {
 ! CHECK:            %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
-! CHECK:            fir.store %[[VAL_2]]#1 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
+! CHECK:            fir.store %[[VAL_2]]#0 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:            omp.taskgroup task_reduction(byref @add_reduction_byref_box_Uxf32 %[[VAL_3]] -> %[[VAL_4:.*]]: !fir.ref<!fir.box<!fir.array<?xf32>>>) {
 ! CHECK:              %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] 
 ! CHECK-SAME:         {uniq_name = "_QFtask_reductionEx"} : (!fir.ref<!fir.box<!fir.array<?xf32>>>) -> (!fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index 290f9e1981361..d8c5706b912a5 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -81,7 +81,7 @@ subroutine reduce(r)
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_4:.*]] = fir.alloca !fir.box<!fir.array<?xf64>>
-! CHECK:             fir.store %[[VAL_3]]#1 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
+! CHECK:             fir.store %[[VAL_3]]#0 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/do_concurrent_reduce_associate.f90 b/flang/test/Lower/do_concurrent_reduce_associate.f90
new file mode 100644
index 0000000000000..828bd89e75c7b
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_reduce_associate.f90
@@ -0,0 +1,20 @@
+! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck %s
+
+subroutine dc_associate_reduce
+  integer :: i
+  real, allocatable, dimension(:) :: x
+
+  associate(x_associate => x)
+  do concurrent (i = 1:10) reduce(+: x_associate)
+  end do
+  end associate
+end subroutine
+
+! CHECK-LABEL: func.func @_QPdc_associate_reduce() {
+! CHECK:         %[[BOX_ALLOC:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
+! CHECK:         %[[ASSOC_DECL:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "{{.*}}x_associate"}
+! CHECK:         fir.store %[[ASSOC_DECL]]#0 to %[[BOX_ALLOC]]
+! CHECK-NEXT:    fir.do_concurrent {
+! CHECK:           fir.do_concurrent.loop {{.*}} reduce(byref @{{.*}} #fir.reduce_attr<add> %[[BOX_ALLOC]] -> %{{.*}} : !{{.*}}) {
+! CHECK:         }
+! CHECK:       }


From flang-commits at lists.llvm.org  Mon Jul 14 02:27:36 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 02:27:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Support `do concurrent ... reduce`
 for associating names (PR #148597)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148597@github.com>
Message-ID: <6874cd88.170a0220.6bc42.e940@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Kareem Ergawy (ergawy)

<details>
<summary>Changes</summary>

Extends reduction support for `do concurrent`, in particular, for associating names. Consider the following input:
```fortran
subroutine dc_associate_reduce
  integer :: i
  real, allocatable, dimension(:) :: x

  associate(x_associate =&gt; x)
  do concurrent (i = 1:10) reduce(+: x_associate)
  end do
  end associate
end subroutine
```

The declaration of `x_associate` is emitted as follows:
```mlir
%13:2 = hlfir.declare %10(%12) {uniq_name = "...."} : (!fir.heap&lt;!fir.array&lt;?xf32&gt;&gt;, !fir.shapeshift&lt;1&gt;) -&gt; (!fir.box&lt;!fir.array&lt;?xf32&gt;&gt;, !fir.heap&lt;!fir.array&lt;?xf32&gt;&gt;)
```
where the HLFIR base type is an array descriptor (i.e. the allocatable/heap attribute is dropped as stipulated by the spec; section 11.1.3.3).

The problem here is that `declare_reduction` ops accept only reference types. This restriction is alreay partially handed for `fir::BaseBoxType`'s by allocating a stack slot for the descriptor and storing the box in that stack allocation. We have to modify this a littble bit for `associate` since the HLFIR and FIR base types are different (unlike most scenarios).

---
Full diff: https://github.com/llvm/llvm-project/pull/148597.diff


7 Files Affected:

- (modified) flang/lib/Lower/Support/ReductionProcessor.cpp (+28-6) 
- (modified) flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 (+1-1) 
- (modified) flang/test/Lower/OpenMP/reduction-array-intrinsic.f90 (+2-2) 
- (modified) flang/test/Lower/OpenMP/sections-array-reduction.f90 (+1-1) 
- (modified) flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90 (+1-1) 
- (modified) flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 (+1-1) 
- (added) flang/test/Lower/do_concurrent_reduce_associate.f90 (+20) 


``````````diff
diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 14b2c9836748f..ddcecd8c1bb02 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -633,13 +633,25 @@ void ReductionProcessor::processReductionArguments(
     }
   }
 
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
   // Reduction variable processing common to both intrinsic operators and
   // procedure designators
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  mlir::OpBuilder::InsertPoint dcIP;
+  constexpr bool isDoConcurrent =
+      std::is_same_v<OpType, fir::DeclareReductionOp>;
+
+  if (isDoConcurrent) {
+    dcIP = builder.saveInsertionPoint();
+    builder.setInsertionPoint(
+        builder.getRegion().getParentOfType<fir::DoConcurrentOp>());
+  }
+
   for (const semantics::Symbol *symbol : reductionSymbols) {
     mlir::Value symVal = converter.getSymbolAddress(*symbol);
+
+    if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>())
+      symVal = declOp.getBase();
+
     mlir::Type eleType;
     auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
     if (refType)
@@ -667,13 +679,20 @@ void ReductionProcessor::processReductionArguments(
       // boxed arrays are passed as values not by reference. Unfortunately,
       // we can't pass a box by value to omp.redution_declare, so turn it
       // into a reference
+      auto oldIP = builder.saveInsertionPoint();
+      builder.setInsertionPointToStart(builder.getAllocaBlock());
+      auto alloca =
+          builder.create<fir::AllocaOp>(currentLocation, symVal.getType());
+      builder.restoreInsertionPoint(oldIP);
+      builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
+      symVal = alloca;
+    }
 
+    if (mlir::isa<fir::BaseBoxType>(symVal.getType())) {
       auto alloca =
           builder.create<fir::AllocaOp>(currentLocation, symVal.getType());
       builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
       symVal = alloca;
-    } else if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>()) {
-      symVal = declOp.getBase();
     }
 
     // this isn't the same as the by-val and by-ref passing later in the
@@ -693,7 +712,7 @@ void ReductionProcessor::processReductionArguments(
   unsigned idx = 0;
   for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
     auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
-    const auto &kindMap = firOpBuilder.getKindMap();
+    const auto &kindMap = builder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
 
@@ -745,9 +764,12 @@ void ReductionProcessor::processReductionArguments(
     OpType decl = createDeclareReduction<OpType>(
         converter, reductionName, redId, redType, currentLocation, isByRef);
     reductionDeclSymbols.push_back(
-        mlir::SymbolRefAttr::get(firOpBuilder.getContext(), decl.getSymName()));
+        mlir::SymbolRefAttr::get(builder.getContext(), decl.getSymName()));
     ++idx;
   }
+
+  if (isDoConcurrent)
+    builder.restoreInsertionPoint(dcIP);
 }
 
 const semantics::SourceName
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index a5710fcf5352b..ec54294c7104f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -69,6 +69,7 @@ program reduce
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
@@ -76,7 +77,6 @@ program reduce
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape_shift %[[VAL_1]], %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : (index, index, index, index) -> !fir.shapeshift<2>
 ! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3x2xi32>>, !fir.shapeshift<2>) -> (!fir.box<!fir.array<3x2xi32>>, !fir.ref<!fir.array<3x2xi32>>)
-! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           fir.store %[[VAL_6]]#0 to %[[VAL_7]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
 ! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3x2xi32 %[[VAL_7]] -> %[[VAL_8:.*]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>) {
 ! CHECK:             %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3x2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3x2xi32>>>, !fir.ref<!fir.box<!fir.array<3x2xi32>>>)
diff --git a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90 b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
index 0cf88cf889868..104904497745d 100644
--- a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
+++ b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
@@ -64,11 +64,11 @@ subroutine max_array_reduction(l, r)
 ! CHECK-LABEL:   func.func @_QPmax_array_reduction(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "l"},
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
 ! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFmax_array_reductionEl"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFmax_array_reductionEr"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
-! CHECK:           fir.store %[[VAL_3]]#1 to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+! CHECK:           fir.store %[[VAL_3]]#0 to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
 ! CHECK:           omp.parallel reduction(byref @max_byref_box_Uxi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
 ! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmax_array_reductionEl"} : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> (!fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:             %[[VAL_8:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
diff --git a/flang/test/Lower/OpenMP/sections-array-reduction.f90 b/flang/test/Lower/OpenMP/sections-array-reduction.f90
index 91e0680692637..2f2808cebfc0c 100644
--- a/flang/test/Lower/OpenMP/sections-array-reduction.f90
+++ b/flang/test/Lower/OpenMP/sections-array-reduction.f90
@@ -34,7 +34,7 @@ subroutine sectionsReduction(x)
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFsectionsreductionEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
-! CHECK:             fir.store %[[VAL_2]]#1 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
+! CHECK:             fir.store %[[VAL_2]]#0 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:             omp.sections reduction(byref @add_reduction_byref_box_Uxf32 %[[VAL_3]] -> %[[VAL_4:.*]] : !fir.ref<!fir.box<!fir.array<?xf32>>>) {
 ! CHECK:               omp.section {
 ! CHECK:               ^bb0(%[[VAL_5:.*]]: !fir.ref<!fir.box<!fir.array<?xf32>>>):
diff --git a/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90 b/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
index 18d45217272fc..18a4f75b86309 100644
--- a/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
+++ b/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
@@ -22,7 +22,7 @@
 ! CHECK-SAME:      {uniq_name = "_QFtask_reductionEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:          omp.parallel {
 ! CHECK:            %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
-! CHECK:            fir.store %[[VAL_2]]#1 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
+! CHECK:            fir.store %[[VAL_2]]#0 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:            omp.taskgroup task_reduction(byref @add_reduction_byref_box_Uxf32 %[[VAL_3]] -> %[[VAL_4:.*]]: !fir.ref<!fir.box<!fir.array<?xf32>>>) {
 ! CHECK:              %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] 
 ! CHECK-SAME:         {uniq_name = "_QFtask_reductionEx"} : (!fir.ref<!fir.box<!fir.array<?xf32>>>) -> (!fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index 290f9e1981361..d8c5706b912a5 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -81,7 +81,7 @@ subroutine reduce(r)
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_4:.*]] = fir.alloca !fir.box<!fir.array<?xf64>>
-! CHECK:             fir.store %[[VAL_3]]#1 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
+! CHECK:             fir.store %[[VAL_3]]#0 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/do_concurrent_reduce_associate.f90 b/flang/test/Lower/do_concurrent_reduce_associate.f90
new file mode 100644
index 0000000000000..828bd89e75c7b
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_reduce_associate.f90
@@ -0,0 +1,20 @@
+! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck %s
+
+subroutine dc_associate_reduce
+  integer :: i
+  real, allocatable, dimension(:) :: x
+
+  associate(x_associate => x)
+  do concurrent (i = 1:10) reduce(+: x_associate)
+  end do
+  end associate
+end subroutine
+
+! CHECK-LABEL: func.func @_QPdc_associate_reduce() {
+! CHECK:         %[[BOX_ALLOC:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
+! CHECK:         %[[ASSOC_DECL:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "{{.*}}x_associate"}
+! CHECK:         fir.store %[[ASSOC_DECL]]#0 to %[[BOX_ALLOC]]
+! CHECK-NEXT:    fir.do_concurrent {
+! CHECK:           fir.do_concurrent.loop {{.*}} reduce(byref @{{.*}} #fir.reduce_attr<add> %[[BOX_ALLOC]] -> %{{.*}} : !{{.*}}) {
+! CHECK:         }
+! CHECK:       }

``````````

</details>


https://github.com/llvm/llvm-project/pull/148597

From flang-commits at lists.llvm.org  Mon Jul 14 02:34:41 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Mon, 14 Jul 2025 02:34:41 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Support `do concurrent ... reduce`
 for associating names (PR #148597)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148597@github.com>
Message-ID: <6874cf31.050a0220.13b493.17e8@mx.google.com>


https://github.com/ergawy updated https://github.com/llvm/llvm-project/pull/148597

>From 037179d9de52eaf2a0f9958dc082c058229209c5 Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Mon, 14 Jul 2025 04:14:53 -0500
Subject: [PATCH] [flang] Support `do concurrent ... reduce` for associating
 names

Extends reduction support for `do concurrent`, in particular, for
associating names. Consider the following input:
```fortran
subroutine dc_associate_reduce
  integer :: i
  real, allocatable, dimension(:) :: x

  associate(x_associate => x)
  do concurrent (i = 1:10) reduce(+: x_associate)
  end do
  end associate
end subroutine
```

The declaration of `x_associate` is emitted as follows:
```mlir
%13:2 = hlfir.declare %10(%12) {uniq_name = "...."} : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
```
where the HLFIR base type is an array descriptor (i.e. the
allocatable/heap attribute is dropped as stipulated by the spec; section
11.1.3.3).

The problem here is that `declare_reduction` ops accept only reference
types. This restriction is alreay partially handed for
`fir::BaseBoxType`'s by allocating a stack slot for the descriptor and
storing the box in that stack allocation. We have to modify this a
littble bit for `associate` since the HLFIR and FIR base types are
different (unlike most scenarios).
---
 .../lib/Lower/Support/ReductionProcessor.cpp  | 29 ++++++++++++++-----
 .../OpenMP/parallel-reduction-array-lb.f90    |  2 +-
 .../OpenMP/reduction-array-intrinsic.f90      |  4 +--
 .../Lower/OpenMP/sections-array-reduction.f90 |  2 +-
 .../OpenMP/taskgroup-task-array-reduction.f90 |  2 +-
 .../wsloop-reduction-array-assumed-shape.f90  |  2 +-
 .../Lower/do_concurrent_reduce_associate.f90  | 20 +++++++++++++
 7 files changed, 48 insertions(+), 13 deletions(-)
 create mode 100644 flang/test/Lower/do_concurrent_reduce_associate.f90

diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 14b2c9836748f..77cb15a2a77ca 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -633,13 +633,25 @@ void ReductionProcessor::processReductionArguments(
     }
   }
 
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
   // Reduction variable processing common to both intrinsic operators and
   // procedure designators
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  mlir::OpBuilder::InsertPoint dcIP;
+  constexpr bool isDoConcurrent =
+      std::is_same_v<OpType, fir::DeclareReductionOp>;
+
+  if (isDoConcurrent) {
+    dcIP = builder.saveInsertionPoint();
+    builder.setInsertionPoint(
+        builder.getRegion().getParentOfType<fir::DoConcurrentOp>());
+  }
+
   for (const semantics::Symbol *symbol : reductionSymbols) {
     mlir::Value symVal = converter.getSymbolAddress(*symbol);
+
+    if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>())
+      symVal = declOp.getBase();
+
     mlir::Type eleType;
     auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
     if (refType)
@@ -667,13 +679,13 @@ void ReductionProcessor::processReductionArguments(
       // boxed arrays are passed as values not by reference. Unfortunately,
       // we can't pass a box by value to omp.redution_declare, so turn it
       // into a reference
-
+      auto oldIP = builder.saveInsertionPoint();
+      builder.setInsertionPointToStart(builder.getAllocaBlock());
       auto alloca =
           builder.create<fir::AllocaOp>(currentLocation, symVal.getType());
+      builder.restoreInsertionPoint(oldIP);
       builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
       symVal = alloca;
-    } else if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>()) {
-      symVal = declOp.getBase();
     }
 
     // this isn't the same as the by-val and by-ref passing later in the
@@ -693,7 +705,7 @@ void ReductionProcessor::processReductionArguments(
   unsigned idx = 0;
   for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
     auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
-    const auto &kindMap = firOpBuilder.getKindMap();
+    const auto &kindMap = builder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
 
@@ -745,9 +757,12 @@ void ReductionProcessor::processReductionArguments(
     OpType decl = createDeclareReduction<OpType>(
         converter, reductionName, redId, redType, currentLocation, isByRef);
     reductionDeclSymbols.push_back(
-        mlir::SymbolRefAttr::get(firOpBuilder.getContext(), decl.getSymName()));
+        mlir::SymbolRefAttr::get(builder.getContext(), decl.getSymName()));
     ++idx;
   }
+
+  if (isDoConcurrent)
+    builder.restoreInsertionPoint(dcIP);
 }
 
 const semantics::SourceName
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index a5710fcf5352b..ec54294c7104f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -69,6 +69,7 @@ program reduce
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
@@ -76,7 +77,6 @@ program reduce
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape_shift %[[VAL_1]], %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : (index, index, index, index) -> !fir.shapeshift<2>
 ! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3x2xi32>>, !fir.shapeshift<2>) -> (!fir.box<!fir.array<3x2xi32>>, !fir.ref<!fir.array<3x2xi32>>)
-! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           fir.store %[[VAL_6]]#0 to %[[VAL_7]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
 ! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3x2xi32 %[[VAL_7]] -> %[[VAL_8:.*]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>) {
 ! CHECK:             %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3x2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3x2xi32>>>, !fir.ref<!fir.box<!fir.array<3x2xi32>>>)
diff --git a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90 b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
index 0cf88cf889868..104904497745d 100644
--- a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
+++ b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
@@ -64,11 +64,11 @@ subroutine max_array_reduction(l, r)
 ! CHECK-LABEL:   func.func @_QPmax_array_reduction(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "l"},
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
 ! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFmax_array_reductionEl"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFmax_array_reductionEr"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
-! CHECK:           fir.store %[[VAL_3]]#1 to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+! CHECK:           fir.store %[[VAL_3]]#0 to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
 ! CHECK:           omp.parallel reduction(byref @max_byref_box_Uxi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
 ! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmax_array_reductionEl"} : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> (!fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:             %[[VAL_8:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
diff --git a/flang/test/Lower/OpenMP/sections-array-reduction.f90 b/flang/test/Lower/OpenMP/sections-array-reduction.f90
index 91e0680692637..2f2808cebfc0c 100644
--- a/flang/test/Lower/OpenMP/sections-array-reduction.f90
+++ b/flang/test/Lower/OpenMP/sections-array-reduction.f90
@@ -34,7 +34,7 @@ subroutine sectionsReduction(x)
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFsectionsreductionEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
-! CHECK:             fir.store %[[VAL_2]]#1 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
+! CHECK:             fir.store %[[VAL_2]]#0 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:             omp.sections reduction(byref @add_reduction_byref_box_Uxf32 %[[VAL_3]] -> %[[VAL_4:.*]] : !fir.ref<!fir.box<!fir.array<?xf32>>>) {
 ! CHECK:               omp.section {
 ! CHECK:               ^bb0(%[[VAL_5:.*]]: !fir.ref<!fir.box<!fir.array<?xf32>>>):
diff --git a/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90 b/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
index 18d45217272fc..18a4f75b86309 100644
--- a/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
+++ b/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
@@ -22,7 +22,7 @@
 ! CHECK-SAME:      {uniq_name = "_QFtask_reductionEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:          omp.parallel {
 ! CHECK:            %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
-! CHECK:            fir.store %[[VAL_2]]#1 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
+! CHECK:            fir.store %[[VAL_2]]#0 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:            omp.taskgroup task_reduction(byref @add_reduction_byref_box_Uxf32 %[[VAL_3]] -> %[[VAL_4:.*]]: !fir.ref<!fir.box<!fir.array<?xf32>>>) {
 ! CHECK:              %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] 
 ! CHECK-SAME:         {uniq_name = "_QFtask_reductionEx"} : (!fir.ref<!fir.box<!fir.array<?xf32>>>) -> (!fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index 290f9e1981361..d8c5706b912a5 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -81,7 +81,7 @@ subroutine reduce(r)
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_4:.*]] = fir.alloca !fir.box<!fir.array<?xf64>>
-! CHECK:             fir.store %[[VAL_3]]#1 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
+! CHECK:             fir.store %[[VAL_3]]#0 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/do_concurrent_reduce_associate.f90 b/flang/test/Lower/do_concurrent_reduce_associate.f90
new file mode 100644
index 0000000000000..828bd89e75c7b
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_reduce_associate.f90
@@ -0,0 +1,20 @@
+! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck %s
+
+subroutine dc_associate_reduce
+  integer :: i
+  real, allocatable, dimension(:) :: x
+
+  associate(x_associate => x)
+  do concurrent (i = 1:10) reduce(+: x_associate)
+  end do
+  end associate
+end subroutine
+
+! CHECK-LABEL: func.func @_QPdc_associate_reduce() {
+! CHECK:         %[[BOX_ALLOC:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
+! CHECK:         %[[ASSOC_DECL:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "{{.*}}x_associate"}
+! CHECK:         fir.store %[[ASSOC_DECL]]#0 to %[[BOX_ALLOC]]
+! CHECK-NEXT:    fir.do_concurrent {
+! CHECK:           fir.do_concurrent.loop {{.*}} reduce(byref @{{.*}} #fir.reduce_attr<add> %[[BOX_ALLOC]] -> %{{.*}} : !{{.*}}) {
+! CHECK:         }
+! CHECK:       }


From flang-commits at lists.llvm.org  Mon Jul 14 02:37:39 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 14 Jul 2025 02:37:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Support `do concurrent ... reduce`
 for associating names (PR #148597)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148597@github.com>
Message-ID: <6874cfe3.170a0220.ad8e2.f14a@mx.google.com>


https://github.com/tblah approved this pull request.

This looks great. Thanks for fixing the alloca insertion point while you were at it.

https://github.com/llvm/llvm-project/pull/148597

From flang-commits at lists.llvm.org  Mon Jul 14 02:41:38 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Mon, 14 Jul 2025 02:41:38 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Support `do concurrent ... reduce`
 for associating names (PR #148597)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148597@github.com>
Message-ID: <6874d0d2.170a0220.44b6.ef66@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/148597

From flang-commits at lists.llvm.org  Mon Jul 14 02:41:57 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 14 Jul 2025 02:41:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Allow embox's source_box to be a
	!fir.box. (PR #148305)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148305@github.com>
Message-ID: <6874d0e5.170a0220.1b3771.e631@mx.google.com>


https://github.com/tblah approved this pull request.


https://github.com/llvm/llvm-project/pull/148305

From flang-commits at lists.llvm.org  Mon Jul 14 03:18:19 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 03:18:19 -0700 (PDT)
Subject: [flang-commits] [flang] a716cc0 - [flang] Support `do concurrent
 ... reduce` for associating names (#148597)
Message-ID: <6874d96b.170a0220.3a4a61.03c6@mx.google.com>


Author: Kareem Ergawy
Date: 2025-07-14T12:18:16+02:00
New Revision: a716cc0961d1b0fa76aba78a2509253d0646f41d

URL: https://github.com/llvm/llvm-project/commit/a716cc0961d1b0fa76aba78a2509253d0646f41d
DIFF: https://github.com/llvm/llvm-project/commit/a716cc0961d1b0fa76aba78a2509253d0646f41d.diff

LOG: [flang] Support `do concurrent ... reduce` for associating names (#148597)

Extends reduction support for `do concurrent`, in particular, for
associating names. Consider the following input:
```fortran
subroutine dc_associate_reduce
  integer :: i
  real, allocatable, dimension(:) :: x

  associate(x_associate => x)
  do concurrent (i = 1:10) reduce(+: x_associate)
  end do
  end associate
end subroutine
```

The declaration of `x_associate` is emitted as follows:
```mlir
%13:2 = hlfir.declare %10(%12) {uniq_name = "...."} : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
```
where the HLFIR base type is an array descriptor (i.e. the
allocatable/heap attribute is dropped as stipulated by the spec; section
11.1.3.3).

The problem here is that `declare_reduction` ops accept only reference
types. This restriction is already partially handled for
`fir::BaseBoxType`'s by allocating a stack slot for the descriptor and
storing the box in that stack allocation. We have to modify this a
littble bit for `associate` since the HLFIR and FIR base types are
different (unlike most scenarios).

Added: 
    flang/test/Lower/do_concurrent_reduce_associate.f90

Modified: 
    flang/lib/Lower/Support/ReductionProcessor.cpp
    flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
    flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
    flang/test/Lower/OpenMP/sections-array-reduction.f90
    flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
    flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 14b2c9836748f..77cb15a2a77ca 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -633,13 +633,25 @@ void ReductionProcessor::processReductionArguments(
     }
   }
 
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
   // Reduction variable processing common to both intrinsic operators and
   // procedure designators
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  mlir::OpBuilder::InsertPoint dcIP;
+  constexpr bool isDoConcurrent =
+      std::is_same_v<OpType, fir::DeclareReductionOp>;
+
+  if (isDoConcurrent) {
+    dcIP = builder.saveInsertionPoint();
+    builder.setInsertionPoint(
+        builder.getRegion().getParentOfType<fir::DoConcurrentOp>());
+  }
+
   for (const semantics::Symbol *symbol : reductionSymbols) {
     mlir::Value symVal = converter.getSymbolAddress(*symbol);
+
+    if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>())
+      symVal = declOp.getBase();
+
     mlir::Type eleType;
     auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
     if (refType)
@@ -667,13 +679,13 @@ void ReductionProcessor::processReductionArguments(
       // boxed arrays are passed as values not by reference. Unfortunately,
       // we can't pass a box by value to omp.redution_declare, so turn it
       // into a reference
-
+      auto oldIP = builder.saveInsertionPoint();
+      builder.setInsertionPointToStart(builder.getAllocaBlock());
       auto alloca =
           builder.create<fir::AllocaOp>(currentLocation, symVal.getType());
+      builder.restoreInsertionPoint(oldIP);
       builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
       symVal = alloca;
-    } else if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>()) {
-      symVal = declOp.getBase();
     }
 
     // this isn't the same as the by-val and by-ref passing later in the
@@ -693,7 +705,7 @@ void ReductionProcessor::processReductionArguments(
   unsigned idx = 0;
   for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
     auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
-    const auto &kindMap = firOpBuilder.getKindMap();
+    const auto &kindMap = builder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
 
@@ -745,9 +757,12 @@ void ReductionProcessor::processReductionArguments(
     OpType decl = createDeclareReduction<OpType>(
         converter, reductionName, redId, redType, currentLocation, isByRef);
     reductionDeclSymbols.push_back(
-        mlir::SymbolRefAttr::get(firOpBuilder.getContext(), decl.getSymName()));
+        mlir::SymbolRefAttr::get(builder.getContext(), decl.getSymName()));
     ++idx;
   }
+
+  if (isDoConcurrent)
+    builder.restoreInsertionPoint(dcIP);
 }
 
 const semantics::SourceName

diff  --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index a5710fcf5352b..ec54294c7104f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -69,6 +69,7 @@ program reduce
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
@@ -76,7 +77,6 @@ program reduce
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape_shift %[[VAL_1]], %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : (index, index, index, index) -> !fir.shapeshift<2>
 ! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3x2xi32>>, !fir.shapeshift<2>) -> (!fir.box<!fir.array<3x2xi32>>, !fir.ref<!fir.array<3x2xi32>>)
-! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           fir.store %[[VAL_6]]#0 to %[[VAL_7]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
 ! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3x2xi32 %[[VAL_7]] -> %[[VAL_8:.*]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>) {
 ! CHECK:             %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3x2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3x2xi32>>>, !fir.ref<!fir.box<!fir.array<3x2xi32>>>)

diff  --git a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90 b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
index 0cf88cf889868..104904497745d 100644
--- a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
+++ b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
@@ -64,11 +64,11 @@ subroutine max_array_reduction(l, r)
 ! CHECK-LABEL:   func.func @_QPmax_array_reduction(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "l"},
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "r"}) {
+! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
 ! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFmax_array_reductionEl"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFmax_array_reductionEr"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
-! CHECK:           fir.store %[[VAL_3]]#1 to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+! CHECK:           fir.store %[[VAL_3]]#0 to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
 ! CHECK:           omp.parallel reduction(byref @max_byref_box_Uxi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
 ! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmax_array_reductionEl"} : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> (!fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:             %[[VAL_8:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<?xi32>>>

diff  --git a/flang/test/Lower/OpenMP/sections-array-reduction.f90 b/flang/test/Lower/OpenMP/sections-array-reduction.f90
index 91e0680692637..2f2808cebfc0c 100644
--- a/flang/test/Lower/OpenMP/sections-array-reduction.f90
+++ b/flang/test/Lower/OpenMP/sections-array-reduction.f90
@@ -34,7 +34,7 @@ subroutine sectionsReduction(x)
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFsectionsreductionEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
-! CHECK:             fir.store %[[VAL_2]]#1 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
+! CHECK:             fir.store %[[VAL_2]]#0 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:             omp.sections reduction(byref @add_reduction_byref_box_Uxf32 %[[VAL_3]] -> %[[VAL_4:.*]] : !fir.ref<!fir.box<!fir.array<?xf32>>>) {
 ! CHECK:               omp.section {
 ! CHECK:               ^bb0(%[[VAL_5:.*]]: !fir.ref<!fir.box<!fir.array<?xf32>>>):

diff  --git a/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90 b/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
index 18d45217272fc..18a4f75b86309 100644
--- a/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
+++ b/flang/test/Lower/OpenMP/taskgroup-task-array-reduction.f90
@@ -22,7 +22,7 @@
 ! CHECK-SAME:      {uniq_name = "_QFtask_reductionEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:          omp.parallel {
 ! CHECK:            %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
-! CHECK:            fir.store %[[VAL_2]]#1 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
+! CHECK:            fir.store %[[VAL_2]]#0 to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
 ! CHECK:            omp.taskgroup task_reduction(byref @add_reduction_byref_box_Uxf32 %[[VAL_3]] -> %[[VAL_4:.*]]: !fir.ref<!fir.box<!fir.array<?xf32>>>) {
 ! CHECK:              %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] 
 ! CHECK-SAME:         {uniq_name = "_QFtask_reductionEx"} : (!fir.ref<!fir.box<!fir.array<?xf32>>>) -> (!fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index 290f9e1981361..d8c5706b912a5 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -81,7 +81,7 @@ subroutine reduce(r)
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_4:.*]] = fir.alloca !fir.box<!fir.array<?xf64>>
-! CHECK:             fir.store %[[VAL_3]]#1 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
+! CHECK:             fir.store %[[VAL_3]]#0 to %[[VAL_4]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32

diff  --git a/flang/test/Lower/do_concurrent_reduce_associate.f90 b/flang/test/Lower/do_concurrent_reduce_associate.f90
new file mode 100644
index 0000000000000..828bd89e75c7b
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_reduce_associate.f90
@@ -0,0 +1,20 @@
+! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck %s
+
+subroutine dc_associate_reduce
+  integer :: i
+  real, allocatable, dimension(:) :: x
+
+  associate(x_associate => x)
+  do concurrent (i = 1:10) reduce(+: x_associate)
+  end do
+  end associate
+end subroutine
+
+! CHECK-LABEL: func.func @_QPdc_associate_reduce() {
+! CHECK:         %[[BOX_ALLOC:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
+! CHECK:         %[[ASSOC_DECL:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "{{.*}}x_associate"}
+! CHECK:         fir.store %[[ASSOC_DECL]]#0 to %[[BOX_ALLOC]]
+! CHECK-NEXT:    fir.do_concurrent {
+! CHECK:           fir.do_concurrent.loop {{.*}} reduce(byref @{{.*}} #fir.reduce_attr<add> %[[BOX_ALLOC]] -> %{{.*}} : !{{.*}}) {
+! CHECK:         }
+! CHECK:       }


        

From flang-commits at lists.llvm.org  Mon Jul 14 03:18:22 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Mon, 14 Jul 2025 03:18:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Support `do concurrent ... reduce`
 for associating names (PR #148597)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148597@github.com>
Message-ID: <6874d96e.630a0220.282ce6.0c20@mx.google.com>


https://github.com/ergawy closed https://github.com/llvm/llvm-project/pull/148597

From flang-commits at lists.llvm.org  Mon Jul 14 04:37:14 2025
From: flang-commits at lists.llvm.org (=?UTF-8?B?Um9nZXIgRmVycmVyIEliw6HDsWV6?= via flang-commits)
Date: Mon, 14 Jul 2025 04:37:14 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <6874ebea.050a0220.22ff3f.296d@mx.google.com>


rofirrim wrote:

Hi all, @eZWALT is changing jobs and kindly asked me if I could finish on his behalf. I plan to go through all the items of feedback and then rebase against main.

https://github.com/llvm/llvm-project/pull/139293

From flang-commits at lists.llvm.org  Mon Jul 14 05:28:26 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Mon, 14 Jul 2025 05:28:26 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6874f7ea.050a0220.317ebc.0a62@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
----------------
ergawy wrote:

Thanks for looking into this Chaitanya. Here is a proposal on how we can do that:
```diff
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 14cc7bb511f0..36f0265e8995 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -126,22 +126,6 @@ struct PrivateClauseOpConversion
   }
 };
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
 static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
                                     mlir::Type firType) {
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
@@ -259,13 +243,11 @@ struct TargetAllocMemOpConversion
   matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
     mlir::Type heapTy = allocmemOp.getAllocatedType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
     mlir::Location loc = allocmemOp.getLoc();
     auto ity = lowerTy().indexType();
     mlir::Type dataTy = fir::unwrapRefType(heapTy);
     mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
-    mlir::Type llvmPtrTy =
-        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "omp.target_allocmem codegen of derived type with length "
                 "parameters");
@@ -273,6 +255,7 @@ struct TargetAllocMemOpConversion
                                           lowerTy().getDataLayout());
     if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+
     for (mlir::Value opnd : adaptor.getOperands().drop_front())
       size = rewriter.create<mlir::LLVM::MulOp>(
           loc, ity, size, integerCast(lowerTy(), loc, rewriter, ity, opnd));
@@ -281,13 +264,13 @@ struct TargetAllocMemOpConversion
         mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
     if (mallocTyWidth != ity.getIntOrFloatBitWidth())
       size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
-    allocmemOp->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    auto callOp = rewriter.create<mlir::LLVM::CallOp>(
-        loc, llvmPtrTy,
-        mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(
-        allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+
+    rewriter.modifyOpInPlace(allocmemOp, [&]() {
+      allocmemOp.setInType(rewriter.getI8Type());
+      allocmemOp.getTypeparamsMutable().clear();
+      allocmemOp.getTypeparamsMutable().append(size);
+    });
+
     return mlir::success();
   }
 };
diff --git a/flang/test/Fir/omp_target_allocmem_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
index 920220272845..00a4462510a9 100644
--- a/flang/test/Fir/omp_target_allocmem_freemem.fir
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -24,7 +24,8 @@ func.func @omp_target_allocmem_array_of_char() -> () {
 // CHECK-SAME: i32 %[[len:.*]])
 // CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
 // CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
-// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
+// CHECK: %[[mul3:.*]] = mul i64 1, %[[mul2]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul3]], i32 0)
 func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
   %device = arith.constant 0 : i32
   %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index c246d4abbdfe..0744bac98b3f 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5923,7 +5923,12 @@ convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
   mlir::Type heapTy = allocMemOp.getAllocatedType();
   llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
   llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
-  llvm::ConstantInt *allocSize = builder.getInt64(typeSize.getFixedValue());
+  llvm::Value *allocSize = builder.getInt64(typeSize.getFixedValue());
+
+  for (auto typeParam : allocMemOp.getTypeparams())
+    allocSize =
+        builder.CreateMul(allocSize, moduleTranslation.lookupValue(typeParam));
+
   // Create call to "omp_target_alloc" with the args as translated llvm values.
   llvm::CallInst *call =
       builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});
```
With that, after converting `fir`, we alway have a to `omp.target_alloc_mem` op that allocates the correct number of bytes based on the converted `fir` type.

So when `TargetAllocMemOpConversion` is run, you will get something like this:
```mlir
  llvm.func @omp_target_allocmem_array_of_dynchar(%arg0: i32) {
    %0 = llvm.mlir.constant(0 : i32) : i32
    %1 = llvm.mlir.constant(1 : i64) : i64
    %2 = llvm.mlir.constant(9 : i64) : i64
    %3 = llvm.mul %1, %2 : i64
    %4 = llvm.sext %arg0 : i32 to i64
    %5 = llvm.mul %3, %4 : i64
    %6 = omp.target_allocmem %0 : i32, i8(%5 : i64)
    omp.target_freemem %0, %6 : i32, i64
    llvm.return
  }
```
(note that this is the conversion of the 3rd test you added in `omp_target_allocmem_freemem.fir`). You can reproduce the same result using: `./bin/fir-opt --cfg-conversion --fir-to-llvm-ir="target=aarch64-unknown-linux-gnu"` which will test only this part of the pipeline. Then you can use `mlir-translate` to test the final OpenMP to LLVM conversion.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Mon Jul 14 06:27:24 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 14 Jul 2025 06:27:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Avoid unnecessary parsing
 of OpenMP constructs (PR #148629)
Message-ID: <llvm.org/llvm/llvm-project/pull/148629@github.com>


https://github.com/kparzysz created https://github.com/llvm/llvm-project/pull/148629

When parsing a specification part, the parser will look ahead to see if the next construct is an executable construct. In doing so it will invoke OpenMPConstruct parser, whereas the only necessary thing to check would be the directive alone.

>From 53b8d641ed71d7fb837a20293b566463bd623f92 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Thu, 10 Jul 2025 20:08:28 -0500
Subject: [PATCH] [flang][OpenMP] Avoid unnecessary parsing of OpenMP
 constructs

When parsing a specification part, the parser will look ahead to see if
the next construct is an executable construct. In doing so it will invoke
OpenMPConstruct parser, whereas the only necessary thing to check would
be the directive alone.
---
 flang/include/flang/Parser/parse-tree.h      | 10 +++++
 flang/lib/Parser/openmp-parsers.cpp          |  9 +++-
 flang/lib/Parser/parse-tree.cpp              | 47 ++++++++++++++++++++
 flang/lib/Parser/program-parsers.cpp         |  2 +-
 flang/lib/Parser/type-parsers.h              |  1 +
 flang/test/Parser/OpenMP/fail-construct1.f90 |  2 +-
 6 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index ab2dde7d5dfbe..cc1d032f94d4a 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3481,6 +3481,9 @@ struct OmpDirectiveName {
   // This allows "construct<OmpDirectiveName>(Verbatim("<name>"))".
   OmpDirectiveName(const Verbatim &name);
   using WrapperTrait = std::true_type;
+
+  bool IsExecutionPart() const; // Is allowed in the execution part
+
   CharBlock source;
   llvm::omp::Directive v{llvm::omp::Directive::OMPD_unknown};
 };
@@ -5028,6 +5031,13 @@ struct OpenMPLoopConstruct {
       t;
 };
 
+// Lookahead class to identify execution-part OpenMP constructs without
+// parsing the entire OpenMP construct.
+struct OpenMPExecDirective {
+  WRAPPER_CLASS_BOILERPLATE(OpenMPExecDirective, OmpDirectiveName);
+  CharBlock source;
+};
+
 struct OpenMPConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPConstruct);
   std::variant<OpenMPStandaloneConstruct, OpenMPSectionsConstruct,
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index d70aaab82cbab..76c9499410017 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -912,7 +912,7 @@ TYPE_PARSER(construct<OmpNumTasksClause>(
     scalarIntExpr))
 
 TYPE_PARSER(
-    construct<OmpObject>(designator) || construct<OmpObject>("/" >> name / "/"))
+    construct<OmpObject>(designator) || "/" >> construct<OmpObject>(name) / "/")
 
 // OMP 5.0 2.19.4.5 LASTPRIVATE ([lastprivate-modifier :] list)
 TYPE_PARSER(construct<OmpLastprivateClause>(
@@ -1757,6 +1757,13 @@ TYPE_PARSER(construct<OpenMPSectionsConstruct>(
     Parser<OmpBeginSectionsDirective>{} / endOmpLine,
     Parser<OmpSectionBlocks>{}, Parser<OmpEndSectionsDirective>{} / endOmpLine))
 
+static bool IsExecutionPart(const OmpDirectiveName &name) {
+  return name.IsExecutionPart();
+}
+
+TYPE_PARSER(construct<OpenMPExecDirective>(
+    startOmpLine >> predicated(Parser<OmpDirectiveName>{}, IsExecutionPart)))
+
 TYPE_CONTEXT_PARSER("OpenMP construct"_en_US,
     startOmpLine >>
         withMessage("expected OpenMP construct"_err_en_US,
diff --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index 824612e49293f..7b46c9f469b06 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -368,6 +368,53 @@ llvm::omp::Clause OmpClause::Id() const {
   return std::visit([](auto &&s) { return getClauseIdForClass(s); }, u);
 }
 
+bool OmpDirectiveName::IsExecutionPart() const {
+  // Can the directive appear in the execution part of the program.
+  llvm::omp::Directive id{v};
+  switch (llvm::omp::getDirectiveCategory(id)) {
+  case llvm::omp::Category::Executable:
+    return true;
+  case llvm::omp::Category::Declarative:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_allocate:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Informational:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_assume:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Meta:
+    return true;
+  case llvm::omp::Category::Subsidiary:
+    switch (id) {
+    // TODO: case llvm::omp::Directive::OMPD_task_iteration:
+    case llvm::omp::Directive::OMPD_section:
+    case llvm::omp::Directive::OMPD_scan:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Utility:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_error:
+    case llvm::omp::Directive::OMPD_nothing:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  }
+  return false;
+}
+
 const OmpArgumentList &OmpDirectiveSpecification::Arguments() const {
   static OmpArgumentList empty{decltype(OmpArgumentList::v){}};
   if (auto &arguments = std::get<std::optional<OmpArgumentList>>(t)) {
diff --git a/flang/lib/Parser/program-parsers.cpp b/flang/lib/Parser/program-parsers.cpp
index d6e7f11297187..5f4e62ffdbbf2 100644
--- a/flang/lib/Parser/program-parsers.cpp
+++ b/flang/lib/Parser/program-parsers.cpp
@@ -108,7 +108,7 @@ constexpr auto actionStmtLookAhead{first(actionStmt >> ok,
     "ALLOCATE ("_tok, "CALL" >> name >> "("_tok, "GO TO"_tok, "OPEN ("_tok,
     "PRINT"_tok / space / !"("_tok, "READ ("_tok, "WRITE ("_tok)};
 constexpr auto execPartLookAhead{first(actionStmtLookAhead >> ok,
-    openaccConstruct >> ok, openmpConstruct >> ok, "ASSOCIATE ("_tok,
+    openaccConstruct >> ok, openmpExecDirective >> ok, "ASSOCIATE ("_tok,
     "BLOCK"_tok, "SELECT"_tok, "CHANGE TEAM"_sptok, "CRITICAL"_tok, "DO"_tok,
     "IF ("_tok, "WHERE ("_tok, "FORALL ("_tok, "!$CUF"_tok)};
 constexpr auto declErrorRecovery{
diff --git a/flang/lib/Parser/type-parsers.h b/flang/lib/Parser/type-parsers.h
index 8e91ede093fc4..3900c5a86c874 100644
--- a/flang/lib/Parser/type-parsers.h
+++ b/flang/lib/Parser/type-parsers.h
@@ -137,6 +137,7 @@ constexpr Parser<CompilerDirective> compilerDirective;
 constexpr Parser<OpenACCConstruct> openaccConstruct;
 constexpr Parser<OpenACCDeclarativeConstruct> openaccDeclarativeConstruct;
 constexpr Parser<OpenMPConstruct> openmpConstruct;
+constexpr Parser<OpenMPExecDirective> openmpExecDirective;
 constexpr Parser<OpenMPDeclarativeConstruct> openmpDeclarativeConstruct;
 constexpr Parser<OmpEndLoopDirective> ompEndLoopDirective;
 constexpr Parser<IntrinsicVectorTypeSpec> intrinsicVectorTypeSpec; // Extension
diff --git a/flang/test/Parser/OpenMP/fail-construct1.f90 b/flang/test/Parser/OpenMP/fail-construct1.f90
index f0ee22125cee0..f0b3f7438ae58 100644
--- a/flang/test/Parser/OpenMP/fail-construct1.f90
+++ b/flang/test/Parser/OpenMP/fail-construct1.f90
@@ -1,5 +1,5 @@
 ! RUN: not %flang_fc1 -fsyntax-only -fopenmp %s 2>&1 | FileCheck %s
 
-! CHECK: error: expected OpenMP construct
 !$omp  parallel
+! CHECK: error: expected '!$OMP '
 end


From flang-commits at lists.llvm.org  Mon Jul 14 06:27:53 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 06:27:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Avoid unnecessary parsing
 of OpenMP constructs (PR #148629)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148629@github.com>
Message-ID: <687505d9.170a0220.2b9e62.0445@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: Krzysztof Parzyszek (kparzysz)

<details>
<summary>Changes</summary>

When parsing a specification part, the parser will look ahead to see if the next construct is an executable construct. In doing so it will invoke OpenMPConstruct parser, whereas the only necessary thing to check would be the directive alone.

---
Full diff: https://github.com/llvm/llvm-project/pull/148629.diff


6 Files Affected:

- (modified) flang/include/flang/Parser/parse-tree.h (+10) 
- (modified) flang/lib/Parser/openmp-parsers.cpp (+8-1) 
- (modified) flang/lib/Parser/parse-tree.cpp (+47) 
- (modified) flang/lib/Parser/program-parsers.cpp (+1-1) 
- (modified) flang/lib/Parser/type-parsers.h (+1) 
- (modified) flang/test/Parser/OpenMP/fail-construct1.f90 (+1-1) 


``````````diff
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index ab2dde7d5dfbe..cc1d032f94d4a 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3481,6 +3481,9 @@ struct OmpDirectiveName {
   // This allows "construct<OmpDirectiveName>(Verbatim("<name>"))".
   OmpDirectiveName(const Verbatim &name);
   using WrapperTrait = std::true_type;
+
+  bool IsExecutionPart() const; // Is allowed in the execution part
+
   CharBlock source;
   llvm::omp::Directive v{llvm::omp::Directive::OMPD_unknown};
 };
@@ -5028,6 +5031,13 @@ struct OpenMPLoopConstruct {
       t;
 };
 
+// Lookahead class to identify execution-part OpenMP constructs without
+// parsing the entire OpenMP construct.
+struct OpenMPExecDirective {
+  WRAPPER_CLASS_BOILERPLATE(OpenMPExecDirective, OmpDirectiveName);
+  CharBlock source;
+};
+
 struct OpenMPConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPConstruct);
   std::variant<OpenMPStandaloneConstruct, OpenMPSectionsConstruct,
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index d70aaab82cbab..76c9499410017 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -912,7 +912,7 @@ TYPE_PARSER(construct<OmpNumTasksClause>(
     scalarIntExpr))
 
 TYPE_PARSER(
-    construct<OmpObject>(designator) || construct<OmpObject>("/" >> name / "/"))
+    construct<OmpObject>(designator) || "/" >> construct<OmpObject>(name) / "/")
 
 // OMP 5.0 2.19.4.5 LASTPRIVATE ([lastprivate-modifier :] list)
 TYPE_PARSER(construct<OmpLastprivateClause>(
@@ -1757,6 +1757,13 @@ TYPE_PARSER(construct<OpenMPSectionsConstruct>(
     Parser<OmpBeginSectionsDirective>{} / endOmpLine,
     Parser<OmpSectionBlocks>{}, Parser<OmpEndSectionsDirective>{} / endOmpLine))
 
+static bool IsExecutionPart(const OmpDirectiveName &name) {
+  return name.IsExecutionPart();
+}
+
+TYPE_PARSER(construct<OpenMPExecDirective>(
+    startOmpLine >> predicated(Parser<OmpDirectiveName>{}, IsExecutionPart)))
+
 TYPE_CONTEXT_PARSER("OpenMP construct"_en_US,
     startOmpLine >>
         withMessage("expected OpenMP construct"_err_en_US,
diff --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index 824612e49293f..7b46c9f469b06 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -368,6 +368,53 @@ llvm::omp::Clause OmpClause::Id() const {
   return std::visit([](auto &&s) { return getClauseIdForClass(s); }, u);
 }
 
+bool OmpDirectiveName::IsExecutionPart() const {
+  // Can the directive appear in the execution part of the program.
+  llvm::omp::Directive id{v};
+  switch (llvm::omp::getDirectiveCategory(id)) {
+  case llvm::omp::Category::Executable:
+    return true;
+  case llvm::omp::Category::Declarative:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_allocate:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Informational:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_assume:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Meta:
+    return true;
+  case llvm::omp::Category::Subsidiary:
+    switch (id) {
+    // TODO: case llvm::omp::Directive::OMPD_task_iteration:
+    case llvm::omp::Directive::OMPD_section:
+    case llvm::omp::Directive::OMPD_scan:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Utility:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_error:
+    case llvm::omp::Directive::OMPD_nothing:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  }
+  return false;
+}
+
 const OmpArgumentList &OmpDirectiveSpecification::Arguments() const {
   static OmpArgumentList empty{decltype(OmpArgumentList::v){}};
   if (auto &arguments = std::get<std::optional<OmpArgumentList>>(t)) {
diff --git a/flang/lib/Parser/program-parsers.cpp b/flang/lib/Parser/program-parsers.cpp
index d6e7f11297187..5f4e62ffdbbf2 100644
--- a/flang/lib/Parser/program-parsers.cpp
+++ b/flang/lib/Parser/program-parsers.cpp
@@ -108,7 +108,7 @@ constexpr auto actionStmtLookAhead{first(actionStmt >> ok,
     "ALLOCATE ("_tok, "CALL" >> name >> "("_tok, "GO TO"_tok, "OPEN ("_tok,
     "PRINT"_tok / space / !"("_tok, "READ ("_tok, "WRITE ("_tok)};
 constexpr auto execPartLookAhead{first(actionStmtLookAhead >> ok,
-    openaccConstruct >> ok, openmpConstruct >> ok, "ASSOCIATE ("_tok,
+    openaccConstruct >> ok, openmpExecDirective >> ok, "ASSOCIATE ("_tok,
     "BLOCK"_tok, "SELECT"_tok, "CHANGE TEAM"_sptok, "CRITICAL"_tok, "DO"_tok,
     "IF ("_tok, "WHERE ("_tok, "FORALL ("_tok, "!$CUF"_tok)};
 constexpr auto declErrorRecovery{
diff --git a/flang/lib/Parser/type-parsers.h b/flang/lib/Parser/type-parsers.h
index 8e91ede093fc4..3900c5a86c874 100644
--- a/flang/lib/Parser/type-parsers.h
+++ b/flang/lib/Parser/type-parsers.h
@@ -137,6 +137,7 @@ constexpr Parser<CompilerDirective> compilerDirective;
 constexpr Parser<OpenACCConstruct> openaccConstruct;
 constexpr Parser<OpenACCDeclarativeConstruct> openaccDeclarativeConstruct;
 constexpr Parser<OpenMPConstruct> openmpConstruct;
+constexpr Parser<OpenMPExecDirective> openmpExecDirective;
 constexpr Parser<OpenMPDeclarativeConstruct> openmpDeclarativeConstruct;
 constexpr Parser<OmpEndLoopDirective> ompEndLoopDirective;
 constexpr Parser<IntrinsicVectorTypeSpec> intrinsicVectorTypeSpec; // Extension
diff --git a/flang/test/Parser/OpenMP/fail-construct1.f90 b/flang/test/Parser/OpenMP/fail-construct1.f90
index f0ee22125cee0..f0b3f7438ae58 100644
--- a/flang/test/Parser/OpenMP/fail-construct1.f90
+++ b/flang/test/Parser/OpenMP/fail-construct1.f90
@@ -1,5 +1,5 @@
 ! RUN: not %flang_fc1 -fsyntax-only -fopenmp %s 2>&1 | FileCheck %s
 
-! CHECK: error: expected OpenMP construct
 !$omp  parallel
+! CHECK: error: expected '!$OMP '
 end

``````````

</details>


https://github.com/llvm/llvm-project/pull/148629

From flang-commits at lists.llvm.org  Mon Jul 14 02:25:04 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 14 Jul 2025 02:25:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][tco] Add more flags for generating
	codegen tests (PR #148171)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148171@github.com>
Message-ID: <6874ccf0.050a0220.15e9ea.02a5@mx.google.com>


https://github.com/tblah closed https://github.com/llvm/llvm-project/pull/148171

From flang-commits at lists.llvm.org  Mon Jul 14 04:36:04 2025
From: flang-commits at lists.llvm.org (=?UTF-8?B?Um9nZXIgRmVycmVyIEliw6HDsWV6?= via flang-commits)
Date: Mon, 14 Jul 2025 04:36:04 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <6874eba4.170a0220.1c17c3.e98b@mx.google.com>


https://github.com/rofirrim updated https://github.com/llvm/llvm-project/pull/139293

>From fb91129401f61b332fc1147e5a81d553abd7658a Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:25:33 +0000
Subject: [PATCH 01/12] Add fuse directive patch

---
 clang/include/clang-c/Index.h                 |    4 +
 clang/include/clang/AST/RecursiveASTVisitor.h |    3 +
 clang/include/clang/AST/StmtOpenMP.h          |   99 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |    8 +
 clang/include/clang/Basic/StmtNodes.td        |    1 +
 clang/include/clang/Sema/SemaOpenMP.h         |   27 +
 .../include/clang/Serialization/ASTBitCodes.h |    1 +
 clang/lib/AST/StmtOpenMP.cpp                  |   25 +
 clang/lib/AST/StmtPrinter.cpp                 |    5 +
 clang/lib/AST/StmtProfile.cpp                 |    4 +
 clang/lib/Basic/OpenMPKinds.cpp               |    2 +-
 clang/lib/CodeGen/CGStmt.cpp                  |    3 +
 clang/lib/CodeGen/CGStmtOpenMP.cpp            |    8 +
 clang/lib/CodeGen/CodeGenFunction.h           |    1 +
 clang/lib/Sema/SemaExceptionSpec.cpp          |    1 +
 clang/lib/Sema/SemaOpenMP.cpp                 |  600 +++++++
 clang/lib/Sema/TreeTransform.h                |   11 +
 clang/lib/Serialization/ASTReaderStmt.cpp     |   11 +
 clang/lib/Serialization/ASTWriterStmt.cpp     |    6 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp  |    1 +
 clang/test/OpenMP/fuse_ast_print.cpp          |  278 +++
 clang/test/OpenMP/fuse_codegen.cpp            | 1511 +++++++++++++++++
 clang/test/OpenMP/fuse_messages.cpp           |   76 +
 clang/tools/libclang/CIndex.cpp               |    7 +
 clang/tools/libclang/CXCursor.cpp             |    3 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |    4 +
 .../runtime/test/transform/fuse/foreach.cpp   |  192 +++
 openmp/runtime/test/transform/fuse/intfor.c   |   50 +
 .../runtime/test/transform/fuse/iterfor.cpp   |  194 +++
 .../fuse/parallel-wsloop-collapse-foreach.cpp |  208 +++
 .../fuse/parallel-wsloop-collapse-intfor.c    |   45 +
 31 files changed, 3387 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/OpenMP/fuse_ast_print.cpp
 create mode 100644 clang/test/OpenMP/fuse_codegen.cpp
 create mode 100644 clang/test/OpenMP/fuse_messages.cpp
 create mode 100644 openmp/runtime/test/transform/fuse/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/fuse/intfor.c
 create mode 100644 openmp/runtime/test/transform/fuse/iterfor.cpp
 create mode 100644 openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-foreach.cpp
 create mode 100644 openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c

diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index e4cb4327fbaac..148b89ab9cfa4 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2162,6 +2162,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPStripeDirective = 310,
 
+  /** OpenMP fuse directive
+   */
+  CXCursor_OMPFuseDirective = 318,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 5cb2f57edffe4..918216e8df4aa 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3090,6 +3090,9 @@ DEF_TRAVERSE_STMT(OMPUnrollDirective,
 DEF_TRAVERSE_STMT(OMPReverseDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFuseDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index e2fd2114026f7..cb8bb91f4768c 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -962,6 +962,9 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
 
   /// Number of loops generated by this loop transformation.
   unsigned NumGeneratedLoops = 0;
+  /// Number of top level canonical loop nests generated by this loop
+  /// transformation
+  unsigned NumGeneratedLoopNests = 0;
 
 protected:
   explicit OMPLoopTransformationDirective(StmtClass SC,
@@ -973,6 +976,9 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
 
   /// Set the number of loops generated by this loop transformation.
   void setNumGeneratedLoops(unsigned Num) { NumGeneratedLoops = Num; }
+  /// Set the number of top level canonical loop nests generated by this loop
+  /// transformation
+  void setNumGeneratedLoopNests(unsigned Num) { NumGeneratedLoopNests = Num; }
 
 public:
   /// Return the number of associated (consumed) loops.
@@ -981,6 +987,10 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
   /// Return the number of loops generated by this loop transformation.
   unsigned getNumGeneratedLoops() const { return NumGeneratedLoops; }
 
+  /// Return the number of top level canonical loop nests generated by this loop
+  /// transformation
+  unsigned getNumGeneratedLoopNests() const { return NumGeneratedLoopNests; }
+
   /// Get the de-sugared statements after the loop transformation.
   ///
   /// Might be nullptr if either the directive generates no loops and is handled
@@ -995,7 +1005,8 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass ||
+           C == OMPFuseDirectiveClass;
   }
 };
 
@@ -5562,6 +5573,7 @@ class OMPTileDirective final : public OMPLoopTransformationDirective {
                                        llvm::omp::OMPD_tile, StartLoc, EndLoc,
                                        NumLoops) {
     setNumGeneratedLoops(2 * NumLoops);
+    setNumGeneratedLoopNests(1);
   }
 
   void setPreInits(Stmt *PreInits) {
@@ -5793,6 +5805,7 @@ class OMPReverseDirective final : public OMPLoopTransformationDirective {
                                        llvm::omp::OMPD_reverse, StartLoc,
                                        EndLoc, NumLoops) {
     setNumGeneratedLoops(NumLoops);
+    setNumGeneratedLoopNests(1);
   }
 
   void setPreInits(Stmt *PreInits) {
@@ -5865,6 +5878,7 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
                                        llvm::omp::OMPD_interchange, StartLoc,
                                        EndLoc, NumLoops) {
     setNumGeneratedLoops(NumLoops);
+    setNumGeneratedLoopNests(1);
   }
 
   void setPreInits(Stmt *PreInits) {
@@ -5915,6 +5929,89 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
   }
 };
 
+/// Represents the '#pragma omp fuse' loop transformation directive
+///
+/// \code{c}
+/// #pragma omp fuse
+/// {
+///   for(int i = 0; i < m1; ++i) {...}
+///   for(int j = 0; j < m2; ++j) {...}
+///   ...
+/// }
+/// \endcode
+
+class OMPFuseDirective final : public OMPLoopTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  // Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFuseDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                            unsigned NumLoops)
+      : OMPLoopTransformationDirective(OMPFuseDirectiveClass,
+                                       llvm::omp::OMPD_fuse, StartLoc, EndLoc,
+                                       NumLoops) {
+    // This default initialization assumes simple loop fusion.
+    // If a 'looprange' clause is specified, these values must be explicitly set
+    setNumGeneratedLoopNests(1);
+    setNumGeneratedLoops(NumLoops);
+  }
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for #pragma omp fuse'
+  ///
+  /// \param C Context of the AST
+  /// \param StartLoc Location of the introducer (e.g the 'omp' token)
+  /// \param EndLoc Location of the directive's end (e.g the tok::eod)
+  /// \param Clauses The directive's clauses
+  /// \param NumLoops Number of total affected loops
+  /// \param NumLoopNests Number of affected top level canonical loops
+  ///                 (number of items in the 'looprange' clause if present)
+  /// \param AssociatedStmt The outermost associated loop
+  /// \param TransformedStmt The loop nest after fusion, or nullptr in
+  ///                        dependent
+  /// \param PreInits Helper preinits statements for the loop nest
+  static OMPFuseDirective *Create(const ASTContext &C, SourceLocation StartLoc,
+                                  SourceLocation EndLoc,
+                                  ArrayRef<OMPClause *> Clauses,
+                                  unsigned NumLoops, unsigned NumLoopNests,
+                                  Stmt *AssociatedStmt, Stmt *TransformedStmt,
+                                  Stmt *PreInits);
+
+  /// Build an empty '#pragma omp fuse' AST node for deserialization
+  ///
+  /// \param C Context of the AST
+  /// \param NumClauses Number of clauses to allocate
+  /// \param NumLoops Number of associated loops to allocate
+  static OMPFuseDirective *CreateEmpty(const ASTContext &C, unsigned NumClauses,
+                                       unsigned NumLoops);
+
+  /// Gets the associated loops after the transformation. This is the de-sugared
+  /// replacement or nulltpr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFuseDirectiveClass;
+  }
+};
+
 /// This represents '#pragma omp scan' directive.
 ///
 /// \code
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 979ff60b73b75..fe9ca29038a1f 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11612,6 +11612,14 @@ def note_omp_implicit_dsa : Note<
   "implicitly determined as %0">;
 def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
+def warn_omp_different_loop_ind_var_types : Warning <
+  "loop sequence following '#pragma omp %0' contains induction variables of differing types: %1 and %2">;
+def err_omp_not_canonical_loop : Error <
+  "loop after '#pragma omp %0' is not in canonical form">;
+def err_omp_not_a_loop_sequence : Error < 
+  "statement after '#pragma omp %0' must be a loop sequence containing canonical loops or loop-generating constructs">;
+def err_omp_empty_loop_sequence : Error <
+  "loop sequence after '#pragma omp %0' must contain at least 1 canonical loop or loop-generating construct">;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index c9c173f5c7469..45d1a813e4b1f 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -233,6 +233,7 @@ def OMPStripeDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPUnrollDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPReverseDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPInterchangeDirective : StmtNode<OMPLoopTransformationDirective>;
+def OMPFuseDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPForDirective : StmtNode<OMPLoopDirective>;
 def OMPForSimdDirective : StmtNode<OMPLoopDirective>;
 def OMPSectionsDirective : StmtNode<OMPExecutableDirective>;
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 7b169f56b6807..ea21377a8db9c 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -457,6 +457,13 @@ class SemaOpenMP : public SemaBase {
                                              Stmt *AStmt,
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
+
+  /// Called on well-formed '#pragma omp fuse' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
+                                      Stmt *AStmt, SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+
   /// Called on well-formed '\#pragma omp for' after parsing
   /// of the associated statement.
   StmtResult
@@ -1481,6 +1488,26 @@ class SemaOpenMP : public SemaBase {
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits);
 
+  /// Analyzes and checks a loop sequence for use by a loop transformation
+  ///
+  /// \param Kind           The loop transformation directive kind.
+  /// \param NumLoops       [out] Number of total canonical loops
+  /// \param LoopSeqSize    [out] Number of top level canonical loops
+  /// \param LoopHelpers    [out] The multiple loop analyses results.
+  /// \param LoopStmts      [out] The multiple Stmt of each For loop.
+  /// \param OriginalInits  [out] The multiple collection of statements and
+  ///                       declarations that must have been executed/declared
+  ///                       before entering the loop.
+  /// \param Context
+  /// \return Whether there was an absence of errors or not
+  bool checkTransformableLoopSequence(
+      OpenMPDirectiveKind Kind, Stmt *AStmt, unsigned &LoopSeqSize,
+      unsigned &NumLoops,
+      SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+      SmallVectorImpl<Stmt *> &ForStmts,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
+      ASTContext &Context);
+
   /// Helper to keep information about the current `omp begin/end declare
   /// variant` nesting.
   struct OMPDeclareVariantScope {
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 9d265f27b8e31..83b73554d693c 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1948,6 +1948,7 @@ enum StmtCode {
   STMT_OMP_UNROLL_DIRECTIVE,
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
   STMT_OMP_SECTIONS_DIRECTIVE,
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 2eeb5e45ab511..276e43ec9f7d5 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -456,6 +456,8 @@ OMPUnrollDirective::Create(const ASTContext &C, SourceLocation StartLoc,
   auto *Dir = createDirective<OMPUnrollDirective>(
       C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc);
   Dir->setNumGeneratedLoops(NumGeneratedLoops);
+  // The number of generated loops and loop nests during unroll matches
+  Dir->setNumGeneratedLoopNests(NumGeneratedLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
   return Dir;
@@ -508,6 +510,29 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFuseDirective *OMPFuseDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, unsigned NumLoopNests,
+    Stmt *AssociatedStmt, Stmt *TransformedStmt, Stmt *PreInits) {
+
+  OMPFuseDirective *Dir = createDirective<OMPFuseDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  Dir->setNumGeneratedLoopNests(NumLoopNests);
+  Dir->setNumGeneratedLoops(NumLoops);
+  return Dir;
+}
+
+OMPFuseDirective *OMPFuseDirective::CreateEmpty(const ASTContext &C,
+                                                unsigned NumClauses,
+                                                unsigned NumLoops) {
+  return createEmptyDirective<OMPFuseDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPForSimdDirective *
 OMPForSimdDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                             SourceLocation EndLoc, unsigned CollapsedNum,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 28317911d825b..4f57c63154da0 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -790,6 +790,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFuseDirective(OMPFuseDirective *Node) {
+  Indent() << "#pragma omp fuse";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPForDirective(OMPForDirective *Node) {
   Indent() << "#pragma omp for";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index c61450e19f1b6..c5d1d5b48508e 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1026,6 +1026,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPLoopTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFuseDirective(const OMPFuseDirective *S) {
+  VisitOMPLoopTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPForDirective(const OMPForDirective *S) {
   VisitOMPLoopDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index a451fc7c01841..d172450512f13 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -702,7 +702,7 @@ bool clang::isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind) {
 
 bool clang::isOpenMPLoopTransformationDirective(OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
-         DKind == OMPD_interchange || DKind == OMPD_stripe;
+         DKind == OMPD_interchange || DKind == OMPD_stripe || DKind == OMPD_fuse;
 }
 
 bool clang::isOpenMPCombinedParallelADirective(OpenMPDirectiveKind DKind) {
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 8742f8e0fc04a..aa12d62d1b865 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -234,6 +234,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFuseDirectiveClass:
+    EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
+    break;
   case Stmt::OMPForDirectiveClass:
     EmitOMPForDirective(cast<OMPForDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index d9195d749e056..cf03d5d3d88a3 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -198,6 +198,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Fuse = dyn_cast<OMPFuseDirective>(&S)) {
+      PreInits = Fuse->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -2922,6 +2924,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
+  // Emit the de-sugared statement
+  OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {
   bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index a5ab9df01dba9..fe753e5b688b1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3855,6 +3855,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
   void EmitOMPScopeDirective(const OMPScopeDirective &S);
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 0a6cea8869c14..3eb59156c04af 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1493,6 +1493,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
   case Stmt::OMPTargetDataDirectiveClass:
   case Stmt::OMPTargetDirectiveClass:
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 00f4658180807..84ac9587bd54d 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4404,6 +4404,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_unroll:
   case OMPD_reverse:
   case OMPD_interchange:
+  case OMPD_fuse:
   case OMPD_assume:
     break;
   default:
@@ -6221,6 +6222,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_fuse:
+    Res =
+        ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
+    break;
   case OMPD_for:
     Res = ActOnOpenMPForDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc,
                                   VarsWithInheritedDSA);
@@ -14230,6 +14235,8 @@ bool SemaOpenMP::checkTransformableLoopNest(
           DependentPreInits = Dir->getPreInits();
         else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
           DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPFuseDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
         else
           llvm_unreachable("Unhandled loop transformation");
 
@@ -14240,6 +14247,265 @@ bool SemaOpenMP::checkTransformableLoopNest(
   return Result;
 }
 
+class NestedLoopCounterVisitor
+    : public clang::RecursiveASTVisitor<NestedLoopCounterVisitor> {
+public:
+  explicit NestedLoopCounterVisitor() : NestedLoopCount(0) {}
+
+  bool VisitForStmt(clang::ForStmt *FS) {
+    ++NestedLoopCount;
+    return true;
+  }
+
+  bool VisitCXXForRangeStmt(clang::CXXForRangeStmt *FRS) {
+    ++NestedLoopCount;
+    return true;
+  }
+
+  unsigned getNestedLoopCount() const { return NestedLoopCount; }
+
+private:
+  unsigned NestedLoopCount;
+};
+
+bool SemaOpenMP::checkTransformableLoopSequence(
+    OpenMPDirectiveKind Kind, Stmt *AStmt, unsigned &LoopSeqSize,
+    unsigned &NumLoops,
+    SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+    SmallVectorImpl<Stmt *> &ForStmts,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
+    ASTContext &Context) {
+
+  // Checks whether the given statement is a compound statement
+  VarsWithInheritedDSAType TmpDSA;
+  if (!isa<CompoundStmt>(AStmt)) {
+    Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
+        << getOpenMPDirectiveName(Kind);
+    return false;
+  }
+  // Callback for updating pre-inits in case there are even more
+  // loop-sequence-generating-constructs inside of the main compound stmt
+  auto OnTransformationCallback =
+      [&OriginalInits](OMPLoopBasedDirective *Transform) {
+        Stmt *DependentPreInits;
+        if (auto *Dir = dyn_cast<OMPTileDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPUnrollDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPReverseDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPFuseDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else
+          llvm_unreachable("Unhandled loop transformation");
+
+        appendFlattenedStmtList(OriginalInits.back(), DependentPreInits);
+      };
+
+  // Number of top level canonical loop nests observed (And acts as index)
+  LoopSeqSize = 0;
+  // Number of total observed loops
+  NumLoops = 0;
+
+  // Following OpenMP 6.0 API Specification, a Canonical Loop Sequence follows
+  // the grammar:
+  //
+  // canonical-loop-sequence:
+  //  {
+  //    loop-sequence+
+  //  }
+  // where loop-sequence can be any of the following:
+  // 1. canonical-loop-sequence
+  // 2. loop-nest
+  // 3. loop-sequence-generating-construct (i.e OMPLoopTransformationDirective)
+  //
+  // To recognise and traverse this structure the following helper functions
+  // have been defined. handleLoopSequence serves as the recurisve entry point
+  // and tries to match the input AST to the canonical loop sequence grammar
+  // structure
+
+  auto NLCV = NestedLoopCounterVisitor();
+  // Helper functions to validate canonical loop sequence grammar is valid
+  auto isLoopSequenceDerivation = [](auto *Child) {
+    return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
+           isa<OMPLoopTransformationDirective>(Child);
+  };
+  auto isLoopGeneratingStmt = [](auto *Child) {
+    return isa<OMPLoopTransformationDirective>(Child);
+  };
+
+  // Helper Lambda to handle storing initialization and body statements for both
+  // ForStmt and CXXForRangeStmt and checks for any possible mismatch between
+  // induction variables types
+  QualType BaseInductionVarType;
+  auto storeLoopStatements = [&OriginalInits, &ForStmts, &BaseInductionVarType,
+                              this, &Context](Stmt *LoopStmt) {
+    if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
+      OriginalInits.back().push_back(For->getInit());
+      ForStmts.push_back(For);
+      // Extract induction variable
+      if (auto *InitStmt = dyn_cast_or_null<DeclStmt>(For->getInit())) {
+        if (auto *InitDecl = dyn_cast<VarDecl>(InitStmt->getSingleDecl())) {
+          QualType InductionVarType = InitDecl->getType().getCanonicalType();
+
+          // Compare with first loop type
+          if (BaseInductionVarType.isNull()) {
+            BaseInductionVarType = InductionVarType;
+          } else if (!Context.hasSameType(BaseInductionVarType,
+                                          InductionVarType)) {
+            Diag(InitDecl->getBeginLoc(),
+                 diag::warn_omp_different_loop_ind_var_types)
+                << getOpenMPDirectiveName(OMPD_fuse) << BaseInductionVarType
+                << InductionVarType;
+          }
+        }
+      }
+
+    } else {
+      assert(isa<CXXForRangeStmt>(LoopStmt) &&
+             "Expected canonical for or range-based for loops.");
+      auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt);
+      OriginalInits.back().push_back(CXXFor->getBeginStmt());
+      ForStmts.push_back(CXXFor);
+    }
+  };
+  // Helper lambda functions to encapsulate the processing of different
+  // derivations of the canonical loop sequence grammar
+  //
+  // Modularized code for handling loop generation and transformations
+  auto handleLoopGeneration = [&storeLoopStatements, &LoopHelpers,
+                               &OriginalInits, &LoopSeqSize, &NumLoops, Kind,
+                               &TmpDSA, &OnTransformationCallback,
+                               this](Stmt *Child) {
+    auto LoopTransform = dyn_cast<OMPLoopTransformationDirective>(Child);
+    Stmt *TransformedStmt = LoopTransform->getTransformedStmt();
+    unsigned NumGeneratedLoopNests = LoopTransform->getNumGeneratedLoopNests();
+
+    // Handle the case where transformed statement is not available due to
+    // dependent contexts
+    if (!TransformedStmt) {
+      if (NumGeneratedLoopNests > 0)
+        return true;
+      // Unroll full
+      else {
+        Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+            << 0 << getOpenMPDirectiveName(Kind);
+        return false;
+      }
+    }
+    // Handle loop transformations with multiple loop nests
+    // Unroll full
+    if (NumGeneratedLoopNests <= 0) {
+      Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+          << 0 << getOpenMPDirectiveName(Kind);
+      return false;
+      // Future loop transformations that generate multiple canonical loops
+    } else if (NumGeneratedLoopNests > 1) {
+      llvm_unreachable("Multiple canonical loop generating transformations "
+                       "like loop splitting are not yet supported");
+    }
+
+    // Process the transformed loop statement
+    Child = TransformedStmt;
+    OriginalInits.emplace_back();
+    LoopHelpers.emplace_back();
+    OnTransformationCallback(LoopTransform);
+
+    unsigned IsCanonical =
+        checkOpenMPLoop(Kind, nullptr, nullptr, Child, SemaRef, *DSAStack,
+                        TmpDSA, LoopHelpers[LoopSeqSize]);
+
+    if (!IsCanonical) {
+      Diag(Child->getBeginLoc(), diag::err_omp_not_canonical_loop)
+          << getOpenMPDirectiveName(Kind);
+      return false;
+    }
+    storeLoopStatements(TransformedStmt);
+    NumLoops += LoopTransform->getNumGeneratedLoops();
+    return true;
+  };
+
+  // Modularized code for handling regular canonical loops
+  auto handleRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
+                            &LoopSeqSize, &NumLoops, Kind, &TmpDSA, &NLCV,
+                            this](Stmt *Child) {
+    OriginalInits.emplace_back();
+    LoopHelpers.emplace_back();
+    unsigned IsCanonical =
+        checkOpenMPLoop(Kind, nullptr, nullptr, Child, SemaRef, *DSAStack,
+                        TmpDSA, LoopHelpers[LoopSeqSize]);
+
+    if (!IsCanonical) {
+      Diag(Child->getBeginLoc(), diag::err_omp_not_canonical_loop)
+          << getOpenMPDirectiveName(Kind);
+      return false;
+    }
+    storeLoopStatements(Child);
+    NumLoops += NLCV.TraverseStmt(Child);
+    return true;
+  };
+
+  // Helper function to process a Loop Sequence Recursively
+  auto handleLoopSequence = [&](Stmt *LoopSeqStmt,
+                                auto &handleLoopSequenceCallback) -> bool {
+    for (auto *Child : LoopSeqStmt->children()) {
+      if (!Child)
+        continue;
+
+      // Skip over non-loop-sequence statements
+      if (!isLoopSequenceDerivation(Child)) {
+        Child = Child->IgnoreContainers();
+
+        // Ignore empty compound statement
+        if (!Child)
+          continue;
+
+        // In the case of a nested loop sequence ignoring containers would not
+        // be enough, a recurisve transversal of the loop sequence is required
+        if (isa<CompoundStmt>(Child)) {
+          if (!handleLoopSequenceCallback(Child, handleLoopSequenceCallback))
+            return false;
+          // Already been treated, skip this children
+          continue;
+        }
+      }
+      // Regular loop sequence handling
+      if (isLoopSequenceDerivation(Child)) {
+        if (isLoopGeneratingStmt(Child)) {
+          if (!handleLoopGeneration(Child)) {
+            return false;
+          }
+        } else {
+          if (!handleRegularLoop(Child)) {
+            return false;
+          }
+        }
+        ++LoopSeqSize;
+      } else {
+        // Report error for invalid statement inside canonical loop sequence
+        Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+            << 0 << getOpenMPDirectiveName(Kind);
+        return false;
+      }
+    }
+    return true;
+  };
+
+  // Recursive entry point to process the main loop sequence
+  if (!handleLoopSequence(AStmt, handleLoopSequence)) {
+    return false;
+  }
+
+  if (LoopSeqSize <= 0) {
+    Diag(AStmt->getBeginLoc(), diag::err_omp_empty_loop_sequence)
+        << getOpenMPDirectiveName(Kind);
+    return false;
+  }
+  return true;
+}
+
 /// Add preinit statements that need to be propageted from the selected loop.
 static void addLoopPreInits(ASTContext &Context,
                             OMPLoopBasedDirective::HelperExprs &LoopHelper,
@@ -15499,6 +15765,340 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
+                                                Stmt *AStmt,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurrContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+  CaptureVars CopyTransformer(SemaRef);
+
+  // Ensure the structured block is not empty
+  if (!AStmt) {
+    return StmtError();
+  }
+  // Validate that the potential loop sequence is transformable for fusion
+  // Also collect the HelperExprs, Loop Stmts, Inits, and Number of loops
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers;
+  SmallVector<Stmt *> LoopStmts;
+  SmallVector<SmallVector<Stmt *, 0>> OriginalInits;
+
+  unsigned NumLoops;
+  // TODO: Support looprange clause using LoopSeqSize
+  unsigned LoopSeqSize;
+  if (!checkTransformableLoopSequence(OMPD_fuse, AStmt, LoopSeqSize, NumLoops,
+                                      LoopHelpers, LoopStmts, OriginalInits,
+                                      Context)) {
+    return StmtError();
+  }
+
+  // Defer transformation in dependent contexts
+  if (CurrContext->isDependentContext()) {
+    return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                    NumLoops, 1, AStmt, nullptr, nullptr);
+  }
+  assert(LoopHelpers.size() == LoopSeqSize &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == LoopSeqSize &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // PreInits hold a sequence of variable declarations that must be executed
+  // before the fused loop begins. These include bounds, strides, and other
+  // helper variables required for the transformation.
+  SmallVector<Stmt *> PreInits;
+
+  // Select the type with the largest bit width among all induction variables
+  QualType IVType = LoopHelpers[0].IterationVarRef->getType();
+  for (unsigned int I = 1; I < LoopSeqSize; ++I) {
+    QualType CurrentIVType = LoopHelpers[I].IterationVarRef->getType();
+    if (Context.getTypeSize(CurrentIVType) > Context.getTypeSize(IVType)) {
+      IVType = CurrentIVType;
+    }
+  }
+  uint64_t IVBitWidth = Context.getIntWidth(IVType);
+
+  // Create pre-init declarations for all loops lower bounds, upper bounds,
+  // strides and num-iterations
+  SmallVector<VarDecl *, 4> LBVarDecls;
+  SmallVector<VarDecl *, 4> STVarDecls;
+  SmallVector<VarDecl *, 4> NIVarDecls;
+  SmallVector<VarDecl *, 4> UBVarDecls;
+  SmallVector<VarDecl *, 4> IVVarDecls;
+
+  // Helper lambda to create variables for bounds, strides, and other
+  // expressions. Generates both the variable declaration and the corresponding
+  // initialization statement.
+  auto CreateHelperVarAndStmt =
+      [&SemaRef = this->SemaRef, &Context, &CopyTransformer,
+       &IVType](Expr *ExprToCopy, const std::string &BaseName, unsigned I,
+                bool NeedsNewVD = false) {
+        Expr *TransformedExpr =
+            AssertSuccess(CopyTransformer.TransformExpr(ExprToCopy));
+        if (!TransformedExpr)
+          return std::pair<VarDecl *, StmtResult>(nullptr, StmtError());
+
+        auto Name = (Twine(".omp.") + BaseName + std::to_string(I)).str();
+
+        VarDecl *VD;
+        if (NeedsNewVD) {
+          VD = buildVarDecl(SemaRef, SourceLocation(), IVType, Name);
+          SemaRef.AddInitializerToDecl(VD, TransformedExpr, false);
+
+        } else {
+          // Create a unique variable name
+          DeclRefExpr *DRE = cast<DeclRefExpr>(TransformedExpr);
+          VD = cast<VarDecl>(DRE->getDecl());
+          VD->setDeclName(&SemaRef.PP.getIdentifierTable().get(Name));
+        }
+        // Create the corresponding declaration statement
+        StmtResult DeclStmt = new (Context) class DeclStmt(
+            DeclGroupRef(VD), SourceLocation(), SourceLocation());
+        return std::make_pair(VD, DeclStmt);
+      };
+
+  // Process each single loop to generate and collect declarations
+  // and statements for all helper expressions
+  for (unsigned int I = 0; I < LoopSeqSize; ++I) {
+    addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
+                    PreInits);
+
+    auto [UBVD, UBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].UB, "ub", I);
+    auto [LBVD, LBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].LB, "lb", I);
+    auto [STVD, STDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].ST, "st", I);
+    auto [NIVD, NIDStmt] =
+        CreateHelperVarAndStmt(LoopHelpers[I].NumIterations, "ni", I, true);
+    auto [IVVD, IVDStmt] =
+        CreateHelperVarAndStmt(LoopHelpers[I].IterationVarRef, "iv", I);
+
+    if (!LBVD || !STVD || !NIVD || !IVVD)
+      return StmtError();
+
+    UBVarDecls.push_back(UBVD);
+    LBVarDecls.push_back(LBVD);
+    STVarDecls.push_back(STVD);
+    NIVarDecls.push_back(NIVD);
+    IVVarDecls.push_back(IVVD);
+
+    PreInits.push_back(UBDStmt.get());
+    PreInits.push_back(LBDStmt.get());
+    PreInits.push_back(STDStmt.get());
+    PreInits.push_back(NIDStmt.get());
+    PreInits.push_back(IVDStmt.get());
+  }
+
+  auto MakeVarDeclRef = [&SemaRef = this->SemaRef](VarDecl *VD) {
+    return buildDeclRefExpr(SemaRef, VD, VD->getType(), VD->getLocation(),
+                            false);
+  };
+
+  // Following up the creation of the final fused loop will be performed
+  // which has the following shape (considering the selected loops):
+  //
+  // for (fuse.index = 0; fuse.index < max(ni0, ni1..., nik); ++fuse.index) {
+  //    if (fuse.index < ni0){
+  //      iv0 = lb0 + st0 * fuse.index;
+  //      original.index0 = iv0
+  //      body(0);
+  //    }
+  //    if (fuse.index < ni1){
+  //      iv1 = lb1 + st1 * fuse.index;
+  //      original.index1 = iv1
+  //      body(1);
+  //    }
+  //
+  //    ...
+  //
+  //    if (fuse.index < nik){
+  //      ivk = lbk + stk * fuse.index;
+  //      original.indexk = ivk
+  //      body(k);  Expr *InitVal = IntegerLiteral::Create(Context,
+  //      llvm::APInt(IVWidth, 0),
+
+  //    }
+
+  // 1. Create the initialized fuse index
+  const std::string IndexName = Twine(".omp.fuse.index").str();
+  Expr *InitVal = IntegerLiteral::Create(Context, llvm::APInt(IVBitWidth, 0),
+                                         IVType, SourceLocation());
+  VarDecl *IndexDecl =
+      buildVarDecl(SemaRef, {}, IVType, IndexName, nullptr, nullptr);
+  SemaRef.AddInitializerToDecl(IndexDecl, InitVal, false);
+  StmtResult InitStmt = new (Context)
+      DeclStmt(DeclGroupRef(IndexDecl), SourceLocation(), SourceLocation());
+
+  if (!InitStmt.isUsable())
+    return StmtError();
+
+  auto MakeIVRef = [&SemaRef = this->SemaRef, IndexDecl, IVType,
+                    Loc = InitVal->getExprLoc()]() {
+    return buildDeclRefExpr(SemaRef, IndexDecl, IVType, Loc, false);
+  };
+
+  // 2. Iteratively compute the max number of logical iterations Max(NI_1, NI_2,
+  // ..., NI_k)
+  //
+  // This loop accumulates the maximum value across multiple expressions,
+  // ensuring each step constructs a unique AST node for correctness. By using
+  // intermediate temporary variables and conditional operators, we maintain
+  // distinct nodes and avoid duplicating subtrees,  For instance, max(a,b,c):
+  //   omp.temp0 = max(a, b)
+  //   omp.temp1 = max(omp.temp0, c)
+  //   omp.fuse.max = max(omp.temp1, omp.temp0)
+
+  ExprResult MaxExpr;
+  for (unsigned I = 0; I < LoopSeqSize; ++I) {
+    DeclRefExpr *NIRef = MakeVarDeclRef(NIVarDecls[I]);
+    QualType NITy = NIRef->getType();
+
+    if (MaxExpr.isUnset()) {
+      // Initialize MaxExpr with the first NI expression
+      MaxExpr = NIRef;
+    } else {
+      // Create a new acummulator variable t_i = MaxExpr
+      std::string TempName = (Twine(".omp.temp.") + Twine(I)).str();
+      VarDecl *TempDecl =
+          buildVarDecl(SemaRef, {}, NITy, TempName, nullptr, nullptr);
+      TempDecl->setInit(MaxExpr.get());
+      DeclRefExpr *TempRef =
+          buildDeclRefExpr(SemaRef, TempDecl, NITy, SourceLocation(), false);
+      DeclRefExpr *TempRef2 =
+          buildDeclRefExpr(SemaRef, TempDecl, NITy, SourceLocation(), false);
+      // Add a DeclStmt to PreInits to ensure the variable is declared.
+      StmtResult TempStmt = new (Context)
+          DeclStmt(DeclGroupRef(TempDecl), SourceLocation(), SourceLocation());
+
+      if (!TempStmt.isUsable())
+        return StmtError();
+      PreInits.push_back(TempStmt.get());
+
+      // Build MaxExpr <-(MaxExpr > NIRef ? MaxExpr : NIRef)
+      ExprResult Comparison =
+          SemaRef.BuildBinOp(nullptr, SourceLocation(), BO_GT, TempRef, NIRef);
+      // Handle any errors in Comparison creation
+      if (!Comparison.isUsable())
+        return StmtError();
+
+      DeclRefExpr *NIRef2 = MakeVarDeclRef(NIVarDecls[I]);
+      // Update MaxExpr using a conditional expression to hold the max value
+      MaxExpr = new (Context) ConditionalOperator(
+          Comparison.get(), SourceLocation(), TempRef2, SourceLocation(),
+          NIRef2->getExprStmt(), NITy, VK_LValue, OK_Ordinary);
+
+      if (!MaxExpr.isUsable())
+        return StmtError();
+    }
+  }
+  if (!MaxExpr.isUsable())
+    return StmtError();
+
+  // 3. Declare the max variable
+  const std::string MaxName = Twine(".omp.fuse.max").str();
+  VarDecl *MaxDecl =
+      buildVarDecl(SemaRef, {}, IVType, MaxName, nullptr, nullptr);
+  MaxDecl->setInit(MaxExpr.get());
+  DeclRefExpr *MaxRef = buildDeclRefExpr(SemaRef, MaxDecl, IVType, {}, false);
+  StmtResult MaxStmt = new (Context)
+      DeclStmt(DeclGroupRef(MaxDecl), SourceLocation(), SourceLocation());
+
+  if (MaxStmt.isInvalid())
+    return StmtError();
+  PreInits.push_back(MaxStmt.get());
+
+  // 4. Create condition Expr: index < n_max
+  ExprResult CondExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_LT,
+                                           MakeIVRef(), MaxRef);
+  if (!CondExpr.isUsable())
+    return StmtError();
+  // 5. Increment Expr: ++index
+  ExprResult IncrExpr =
+      SemaRef.BuildUnaryOp(CurScope, SourceLocation(), UO_PreInc, MakeIVRef());
+  if (!IncrExpr.isUsable())
+    return StmtError();
+
+  // 6. Build the Fused Loop Body
+  // The final fused loop iterates over the maximum logical range. Inside the
+  // loop, each original loop's index is calculated dynamically, and its body
+  // is executed conditionally.
+  //
+  // Each sub-loop's body is guarded by a conditional statement to ensure
+  // it executes only within its logical iteration range:
+  //
+  //    if (fuse.index < ni_k){
+  //      iv_k = lb_k + st_k * fuse.index;
+  //      original.index = iv_k
+  //      body(k);
+  //    }
+
+  CompoundStmt *FusedBody = nullptr;
+  SmallVector<Stmt *, 4> FusedBodyStmts;
+  for (unsigned I = 0; I < LoopSeqSize; ++I) {
+
+    // Assingment of the original sub-loop index to compute the logical index
+    // IV_k = LB_k + omp.fuse.index * ST_k
+
+    ExprResult IdxExpr =
+        SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Mul,
+                           MakeVarDeclRef(STVarDecls[I]), MakeIVRef());
+    if (!IdxExpr.isUsable())
+      return StmtError();
+    IdxExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Add,
+                                 MakeVarDeclRef(LBVarDecls[I]), IdxExpr.get());
+
+    if (!IdxExpr.isUsable())
+      return StmtError();
+    IdxExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Assign,
+                                 MakeVarDeclRef(IVVarDecls[I]), IdxExpr.get());
+    if (!IdxExpr.isUsable())
+      return StmtError();
+
+    // Update the original i_k = IV_k
+    SmallVector<Stmt *, 4> BodyStmts;
+    BodyStmts.push_back(IdxExpr.get());
+    llvm::append_range(BodyStmts, LoopHelpers[I].Updates);
+
+    if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(SourceCXXFor->getLoopVarStmt());
+
+    Stmt *Body = (isa<ForStmt>(LoopStmts[I]))
+                     ? cast<ForStmt>(LoopStmts[I])->getBody()
+                     : cast<CXXForRangeStmt>(LoopStmts[I])->getBody();
+
+    BodyStmts.push_back(Body);
+
+    CompoundStmt *CombinedBody =
+        CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                             SourceLocation(), SourceLocation());
+    ExprResult Condition =
+        SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_LT, MakeIVRef(),
+                           MakeVarDeclRef(NIVarDecls[I]));
+
+    if (!Condition.isUsable())
+      return StmtError();
+
+    IfStmt *IfStatement = IfStmt::Create(
+        Context, SourceLocation(), IfStatementKind::Ordinary, nullptr, nullptr,
+        Condition.get(), SourceLocation(), SourceLocation(), CombinedBody,
+        SourceLocation(), nullptr);
+
+    FusedBodyStmts.push_back(IfStatement);
+  }
+  FusedBody = CompoundStmt::Create(Context, FusedBodyStmts, FPOptionsOverride(),
+                                   SourceLocation(), SourceLocation());
+
+  // 7. Construct the final fused loop
+  ForStmt *FusedForStmt = new (Context)
+      ForStmt(Context, InitStmt.get(), CondExpr.get(), nullptr, IncrExpr.get(),
+              FusedBody, InitStmt.get()->getBeginLoc(), SourceLocation(),
+              IncrExpr.get()->getEndLoc());
+
+  return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses, NumLoops,
+                                  1, AStmt, FusedForStmt,
+                                  buildPreInits(Context, PreInits));
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
                                                    Expr *Expr,
                                                    SourceLocation StartLoc,
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3e33fb73e01b4..45f556f22c511 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -9675,6 +9675,17 @@ StmtResult TreeTransform<Derived>::TransformOMPInterchangeDirective(
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPForDirective(OMPForDirective *D) {
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 44cfb83ad2db4..291bd8ea4bf18 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2445,6 +2445,7 @@ void ASTStmtReader::VisitOMPLoopTransformationDirective(
     OMPLoopTransformationDirective *D) {
   VisitOMPLoopBasedDirective(D);
   D->setNumGeneratedLoops(Record.readUInt32());
+  D->setNumGeneratedLoopNests(Record.readUInt32());
 }
 
 void ASTStmtReader::VisitOMPTileDirective(OMPTileDirective *D) {
@@ -2467,6 +2468,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPLoopTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFuseDirective(OMPFuseDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPForDirective(OMPForDirective *D) {
   VisitOMPLoopDirective(D);
   D->setHasCancel(Record.readBool());
@@ -3608,6 +3613,12 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       S = OMPReverseDirective::CreateEmpty(Context, NumLoops);
       break;
     }
+    case STMT_OMP_FUSE_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFuseDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
 
     case STMT_OMP_INTERCHANGE_DIRECTIVE: {
       unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index a6e320c7f3eb0..5bf1ecfb968e8 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2449,6 +2449,7 @@ void ASTStmtWriter::VisitOMPLoopTransformationDirective(
     OMPLoopTransformationDirective *D) {
   VisitOMPLoopBasedDirective(D);
   Record.writeUInt32(D->getNumGeneratedLoops());
+  Record.writeUInt32(D->getNumGeneratedLoopNests());
 }
 
 void ASTStmtWriter::VisitOMPTileDirective(OMPTileDirective *D) {
@@ -2476,6 +2477,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFuseDirective(OMPFuseDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+  Code = serialization::STMT_OMP_FUSE_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPForDirective(OMPForDirective *D) {
   VisitOMPLoopDirective(D);
   Record.writeBool(D->hasCancel());
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index c77ef26da568d..7218d7e62acdd 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1814,6 +1814,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:
     case Stmt::OMPDispatchDirectiveClass:
     case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/test/OpenMP/fuse_ast_print.cpp b/clang/test/OpenMP/fuse_ast_print.cpp
new file mode 100644
index 0000000000000..43ce815dab024
--- /dev/null
+++ b/clang/test/OpenMP/fuse_ast_print.cpp
@@ -0,0 +1,278 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing 
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip 
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER 
+
+// placeholder for loop body code
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL: FunctionDecl {{.*}} foo1
+void foo1() {
+    // PRINT: #pragma omp fuse
+    // DUMP:  OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int j = 10; j > 0; --j)
+        // DUMP: ForStmt
+        for (int j = 10; j > 0; --j)
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);
+        // PRINT: for (int k = 0; k <= 10; ++k)
+        // DUMP: ForStmt
+        for (int k = 0; k <= 10; ++k)
+            // PRINT: body(k)
+            // DUMP: CallExpr
+            body(k);
+
+    }
+
+}
+
+// PRINT-LABEL: void foo2(
+// DUMP-LABEL: FunctionDecl {{.*}} foo2
+void foo2() {
+    // PRINT: #pragma omp unroll partial(4)
+    // DUMP: OMPUnrollDirective
+    // DUMP-NEXT: OMPPartialClause
+    // DUMP-NEXT: ConstantExpr
+    // DUMP-NEXT: value: Int 4
+    // DUMP-NEXT: IntegerLiteral {{.*}} 4
+    #pragma omp unroll partial(4)
+    // PRINT: #pragma omp fuse
+    // DUMP-NEXT: OMPFuseDirective 
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int j = 10; j > 0; --j)
+        // DUMP: ForStmt
+        for (int j = 10; j > 0; --j)
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);  
+    }    
+    
+}
+
+//PRINT-LABEL: void foo3(
+//DUMP-LABEL: FunctionTemplateDecl {{.*}} foo3
+template<int Factor1, int Factor2> 
+void foo3() {
+    // PRINT:  #pragma omp fuse
+    // DUMP: OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: #pragma omp unroll partial(Factor1)
+        // DUMP: OMPUnrollDirective
+        #pragma omp unroll partial(Factor1)
+        // PRINT: for (int i = 0; i < 12; i += 1)
+        // DUMP: ForStmt
+        for (int i = 0; i < 12; i += 1)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: #pragma omp unroll partial(Factor2)
+        // DUMP: OMPUnrollDirective
+        #pragma omp unroll partial(Factor2)
+        // PRINT: for (int k = 0; k <= 10; ++k)
+        // DUMP: ForStmt
+        for (int k = 0; k <= 10; ++k)
+            // PRINT: body(k)
+            // DUMP: CallExpr
+            body(k);
+
+    }
+}
+
+// Also test instantiating the template.
+void tfoo3() {
+    foo3<4,2>();
+}
+
+//PRINT-LABEL: void foo4(
+//DUMP-LABEL: FunctionTemplateDecl {{.*}} foo4
+template<typename T, T Step> 
+void foo4(int start, int end) {
+    // PRINT:  #pragma omp fuse
+    // DUMP: OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (T i = start; i < end; i += Step)
+        // DUMP: ForStmt
+        for (T i = start; i < end; i += Step)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+
+        // PRINT: for (T j = end; j > start; j -= Step)
+        // DUMP: ForStmt 
+        for (T j = end; j > start; j -= Step) {
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);
+        }
+
+    }
+}
+
+// Also test instantiating the template.
+void tfoo4() {
+    foo4<int, 4>(0, 64);
+}
+
+
+
+// PRINT-LABEL: void foo5(
+// DUMP-LABEL: FunctionDecl {{.*}} foo5
+void foo5() {
+    double arr[128], arr2[128];
+    // PRINT: #pragma omp fuse
+    // DUMP:  OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT-NEXT: for (auto &&a : arr)
+        // DUMP-NEXT: CXXForRangeStmt
+        for (auto &&a: arr)
+            // PRINT: body(a)
+            // DUMP: CallExpr
+            body(a);
+        // PRINT: for (double v = 42; auto &&b : arr)
+        // DUMP: CXXForRangeStmt
+        for (double v = 42; auto &&b: arr)
+            // PRINT: body(b, v);
+            // DUMP: CallExpr
+            body(b, v);
+        // PRINT: for (auto &&c : arr2)
+        // DUMP: CXXForRangeStmt
+        for (auto &&c: arr2)
+            // PRINT: body(c)
+            // DUMP: CallExpr
+            body(c);
+
+    }
+
+}
+
+// PRINT-LABEL: void foo6(
+// DUMP-LABEL: FunctionDecl {{.*}} foo6
+void foo6() {
+    // PRINT: #pragma omp fuse
+    // DUMP: OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt
+    {
+        // PRINT: #pragma omp fuse
+        // DUMP: OMPFuseDirective
+        #pragma omp fuse 
+        // PRINT: {
+        // DUMP: CompoundStmt
+        {
+            // PRINT: for (int i = 0; i <= 10; ++i)
+            // DUMP: ForStmt
+            for (int i = 0; i <= 10; ++i)
+                body(i);
+            // PRINT: for (int j = 0; j < 100; ++j)
+            // DUMP: ForStmt
+            for(int j = 0; j < 100; ++j)
+                body(j);
+        }
+        // PRINT: #pragma omp unroll partial(4)
+        // DUMP: OMPUnrollDirective
+        #pragma omp unroll partial(4)
+        // PRINT: for (int k = 0; k < 250; ++k)
+        // DUMP: ForStmt
+        for (int k = 0; k < 250; ++k) 
+            body(k);
+    }
+}
+
+// PRINT-LABEL: void foo7(
+// DUMP-LABEL: FunctionDecl {{.*}} foo7
+void foo7() {
+    // PRINT: #pragma omp fuse
+    // DUMP:  OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: {
+        // DUMP: CompoundStmt   
+        {
+            // PRINT: {
+            // DUMP: CompoundStmt   
+            {
+                // PRINT: for (int i = 0; i < 10; i += 2)
+                // DUMP: ForStmt
+                for (int i = 0; i < 10; i += 2)
+                    // PRINT: body(i)
+                    // DUMP: CallExpr
+                    body(i);
+                // PRINT: for (int j = 10; j > 0; --j)
+                // DUMP: ForStmt
+                for (int j = 10; j > 0; --j)
+                    // PRINT: body(j)
+                    // DUMP: CallExpr
+                    body(j);
+            }
+        }
+        // PRINT: {
+        // DUMP: CompoundStmt   
+        {
+            // PRINT: {
+            // DUMP: CompoundStmt   
+            {
+                // PRINT: {
+                // DUMP: CompoundStmt   
+                {
+                    // PRINT: for (int k = 0; k <= 10; ++k)
+                    // DUMP: ForStmt
+                    for (int k = 0; k <= 10; ++k)
+                        // PRINT: body(k)
+                        // DUMP: CallExpr
+                        body(k);
+                }
+            }
+        }
+    }
+
+}
+
+
+
+
+
+#endif
\ No newline at end of file
diff --git a/clang/test/OpenMP/fuse_codegen.cpp b/clang/test/OpenMP/fuse_codegen.cpp
new file mode 100644
index 0000000000000..6c1e21092da43
--- /dev/null
+++ b/clang/test/OpenMP/fuse_codegen.cpp
@@ -0,0 +1,1511 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --replace-value-regex "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --version 5
+// expected-no-diagnostics
+
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -include-pch %t -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK2
+
+#ifndef HEADER
+#define HEADER
+
+//placeholder for loop body code.
+extern "C" void body(...) {}
+
+extern "C" void foo1(int start1, int end1, int step1, int start2, int end2, int step2) {
+    int i,j;
+    #pragma omp fuse
+    {
+        for(i = start1; i < end1; i += step1) body(i);
+        for(j = start2; j < end2; j += step2) body(j);
+    }
+
+}
+
+template <typename T>
+void foo2(T start, T end, T step){
+    T i,j,k;
+    #pragma omp fuse
+    {
+        for(i = start; i < end; i += step) body(i);
+        for(j = end; j > start; j -= step) body(j);
+        for(k = start+step; k < end+step; k += step) body(k);
+    }
+}
+
+extern "C" void tfoo2() {
+    foo2<int>(0, 64, 4);
+}
+
+extern "C" void foo3() {
+    double arr[256];
+    #pragma omp fuse
+    {
+        #pragma omp fuse
+        {
+            for(int i = 0; i < 128; ++i) body(i);
+            for(int j = 0; j < 256; j+=2) body(j);
+        }
+        for(int c = 42; auto &&v: arr) body(c,v);
+        for(int cc = 37; auto &&vv: arr) body(cc, vv);
+    }
+}
+
+
+#endif
+// CHECK1-LABEL: define dso_local void @body(
+// CHECK1-SAME: ...) #[[ATTR0:[0-9]+]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define dso_local void @foo1(
+// CHECK1-SAME: i32 noundef [[START1:%.*]], i32 noundef [[END1:%.*]], i32 noundef [[STEP1:%.*]], i32 noundef [[START2:%.*]], i32 noundef [[END2:%.*]], i32 noundef [[STEP2:%.*]]) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[START1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[END1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[STEP1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[START2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[END2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[STEP2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    store i32 [[START1]], ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[END1]], ptr [[END1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[STEP1]], ptr [[STEP1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[START2]], ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[END2]], ptr [[END2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[STEP2]], ptr [[STEP2_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK1-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
+// CHECK1-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP20]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP21]], [[TMP22]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP23]], %[[COND_TRUE]] ], [ [[TMP24]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
+// CHECK1-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP17]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP30]], [[TMP31]]
+// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP29]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP33]], [[TMP34]]
+// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP32]], [[MUL19]]
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP35]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    br i1 [[CMP21]], label %[[IF_THEN22:.*]], label %[[IF_END27:.*]]
+// CHECK1:       [[IF_THEN22]]:
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP39]], [[TMP40]]
+// CHECK1-NEXT:    [[ADD24:%.*]] = add i32 [[TMP38]], [[MUL23]]
+// CHECK1-NEXT:    store i32 [[ADD24]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[TMP41]], [[MUL25]]
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK1-NEXT:    br label %[[IF_END27]]
+// CHECK1:       [[IF_END27]]:
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP45]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define dso_local void @tfoo2(
+// CHECK1-SAME: ) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    call void @_Z4foo2IiEvT_S0_S0_(i32 noundef 0, i32 noundef 64, i32 noundef 4)
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define linkonce_odr void @_Z4foo2IiEvT_S0_S0_(
+// CHECK1-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0]] comdat {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP21:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK1-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
+// CHECK1-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP29]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP30]]
+// CHECK1-NEXT:    [[SUB27:%.*]] = sub i32 [[DIV26]], 1
+// CHECK1-NEXT:    store i32 [[SUB27]], ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK1-NEXT:    store i32 [[TMP31]], ptr [[DOTOMP_UB2]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB2]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST2]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK1-NEXT:    [[ADD28:%.*]] = add i32 [[TMP32]], 1
+// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP33]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP34]], [[TMP35]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP36]], %[[COND_TRUE]] ], [ [[TMP37]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    br i1 [[CMP29]], label %[[COND_TRUE30:.*]], label %[[COND_FALSE31:.*]]
+// CHECK1:       [[COND_TRUE30]]:
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    br label %[[COND_END32:.*]]
+// CHECK1:       [[COND_FALSE31]]:
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    br label %[[COND_END32]]
+// CHECK1:       [[COND_END32]]:
+// CHECK1-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP40]], %[[COND_TRUE30]] ], [ [[TMP41]], %[[COND_FALSE31]] ]
+// CHECK1-NEXT:    store i32 [[COND33]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    br i1 [[CMP34]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP44]], [[TMP45]]
+// CHECK1-NEXT:    br i1 [[CMP35]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP47]], [[TMP48]]
+// CHECK1-NEXT:    [[ADD36:%.*]] = add i32 [[TMP46]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD36]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP50]], [[TMP51]]
+// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[TMP49]], [[MUL37]]
+// CHECK1-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP53]], [[TMP54]]
+// CHECK1-NEXT:    br i1 [[CMP39]], label %[[IF_THEN40:.*]], label %[[IF_END45:.*]]
+// CHECK1:       [[IF_THEN40]]:
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[ADD42:%.*]] = add i32 [[TMP55]], [[MUL41]]
+// CHECK1-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP58]], [[MUL43]]
+// CHECK1-NEXT:    store i32 [[SUB44]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP61]])
+// CHECK1-NEXT:    br label %[[IF_END45]]
+// CHECK1:       [[IF_END45]]:
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP62]], [[TMP63]]
+// CHECK1-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
+// CHECK1:       [[IF_THEN47]]:
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP65]], [[TMP66]]
+// CHECK1-NEXT:    [[ADD49:%.*]] = add i32 [[TMP64]], [[MUL48]]
+// CHECK1-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV2]], align 4
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP68]], [[TMP69]]
+// CHECK1-NEXT:    [[ADD51:%.*]] = add i32 [[TMP67]], [[MUL50]]
+// CHECK1-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP70]])
+// CHECK1-NEXT:    br label %[[IF_END52]]
+// CHECK1:       [[IF_END52]]:
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define dso_local void @foo3(
+// CHECK1-SAME: ) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB03:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB04:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST05:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI06:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV07:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_UB117:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB118:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_ST119:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_NI120:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV122:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[CC:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE223:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END224:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN227:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_29:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_31:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_32:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_TEMP_142:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX48:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX54:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[VV:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK1-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTOMP_UB03]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB04]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST05]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], 1
+// CHECK1-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
+// CHECK1-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI06]], align 8
+// CHECK1-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY8:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY8]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY10:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP11]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY10]], ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP12]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP14]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK1-NEXT:    [[SUB13:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i64 [[SUB13]], 1
+// CHECK1-NEXT:    [[DIV15:%.*]] = sdiv i64 [[ADD14]], 1
+// CHECK1-NEXT:    [[SUB16:%.*]] = sub nsw i64 [[DIV15]], 1
+// CHECK1-NEXT:    store i64 [[SUB16]], ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_UB117]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB118]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST119]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK1-NEXT:    store i64 [[ADD21]], ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    store i32 37, ptr [[CC]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE223]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY25:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR26:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY25]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR26]], ptr [[__END224]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP18]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[__BEGIN227]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY30:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP19]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY30]], ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[__END224]], align 8
+// CHECK1-NEXT:    store ptr [[TMP20]], ptr [[DOTCAPTURE_EXPR_31]], align 8
+// CHECK1-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_31]], align 8
+// CHECK1-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST33:%.*]] = ptrtoint ptr [[TMP21]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST34:%.*]] = ptrtoint ptr [[TMP22]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB35:%.*]] = sub i64 [[SUB_PTR_LHS_CAST33]], [[SUB_PTR_RHS_CAST34]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV36:%.*]] = sdiv exact i64 [[SUB_PTR_SUB35]], 8
+// CHECK1-NEXT:    [[SUB37:%.*]] = sub nsw i64 [[SUB_PTR_DIV36]], 1
+// CHECK1-NEXT:    [[ADD38:%.*]] = add nsw i64 [[SUB37]], 1
+// CHECK1-NEXT:    [[DIV39:%.*]] = sdiv i64 [[ADD38]], 1
+// CHECK1-NEXT:    [[SUB40:%.*]] = sub nsw i64 [[DIV39]], 1
+// CHECK1-NEXT:    store i64 [[SUB40]], ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK1-NEXT:    store i64 [[TMP23]], ptr [[DOTOMP_UB2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB2]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST2]], align 8
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK1-NEXT:    [[ADD41:%.*]] = add nsw i64 [[TMP24]], 1
+// CHECK1-NEXT:    store i64 [[ADD41]], ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
+// CHECK1-NEXT:    store i64 [[TMP25]], ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP26]], [[TMP27]]
+// CHECK1-NEXT:    br i1 [[CMP43]], label %[[COND_TRUE44:.*]], label %[[COND_FALSE45:.*]]
+// CHECK1:       [[COND_TRUE44]]:
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK1-NEXT:    br label %[[COND_END46:.*]]
+// CHECK1:       [[COND_FALSE45]]:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    br label %[[COND_END46]]
+// CHECK1:       [[COND_END46]]:
+// CHECK1-NEXT:    [[COND47:%.*]] = phi i64 [ [[TMP28]], %[[COND_TRUE44]] ], [ [[TMP29]], %[[COND_FALSE45]] ]
+// CHECK1-NEXT:    store i64 [[COND47]], ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[CMP49:%.*]] = icmp sgt i64 [[TMP30]], [[TMP31]]
+// CHECK1-NEXT:    br i1 [[CMP49]], label %[[COND_TRUE50:.*]], label %[[COND_FALSE51:.*]]
+// CHECK1:       [[COND_TRUE50]]:
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    br label %[[COND_END52:.*]]
+// CHECK1:       [[COND_FALSE51]]:
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    br label %[[COND_END52]]
+// CHECK1:       [[COND_END52]]:
+// CHECK1-NEXT:    [[COND53:%.*]] = phi i64 [ [[TMP32]], %[[COND_TRUE50]] ], [ [[TMP33]], %[[COND_FALSE51]] ]
+// CHECK1-NEXT:    store i64 [[COND53]], ptr [[DOTOMP_FUSE_MAX48]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX48]], align 8
+// CHECK1-NEXT:    [[CMP55:%.*]] = icmp slt i64 [[TMP34]], [[TMP35]]
+// CHECK1-NEXT:    br i1 [[CMP55]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
+// CHECK1-NEXT:    [[CMP56:%.*]] = icmp slt i64 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    br i1 [[CMP56]], label %[[IF_THEN:.*]], label %[[IF_END76:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB04]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP38]] to i64
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST05]], align 4
+// CHECK1-NEXT:    [[CONV58:%.*]] = sext i32 [[TMP39]] to i64
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV58]], [[TMP40]]
+// CHECK1-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV57]], [[MUL]]
+// CHECK1-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK1-NEXT:    store i32 [[CONV60]], ptr [[DOTOMP_IV07]], align 4
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_IV07]], align 4
+// CHECK1-NEXT:    [[MUL61:%.*]] = mul nsw i32 [[TMP41]], 1
+// CHECK1-NEXT:    [[ADD62:%.*]] = add nsw i32 0, [[MUL61]]
+// CHECK1-NEXT:    store i32 [[ADD62]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP63:%.*]] = icmp slt i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    br i1 [[CMP63]], label %[[IF_THEN64:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN64]]:
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP45]], [[TMP46]]
+// CHECK1-NEXT:    [[ADD66:%.*]] = add nsw i32 [[TMP44]], [[MUL65]]
+// CHECK1-NEXT:    store i32 [[ADD66]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[MUL67:%.*]] = mul nsw i32 [[TMP47]], 1
+// CHECK1-NEXT:    [[ADD68:%.*]] = add nsw i32 0, [[MUL67]]
+// CHECK1-NEXT:    store i32 [[ADD68]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP48]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP69:%.*]] = icmp slt i32 [[TMP49]], [[TMP50]]
+// CHECK1-NEXT:    br i1 [[CMP69]], label %[[IF_THEN70:.*]], label %[[IF_END75:.*]]
+// CHECK1:       [[IF_THEN70]]:
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP52]], [[TMP53]]
+// CHECK1-NEXT:    [[ADD72:%.*]] = add nsw i32 [[TMP51]], [[MUL71]]
+// CHECK1-NEXT:    store i32 [[ADD72]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP54]], 2
+// CHECK1-NEXT:    [[ADD74:%.*]] = add nsw i32 0, [[MUL73]]
+// CHECK1-NEXT:    store i32 [[ADD74]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP55]])
+// CHECK1-NEXT:    br label %[[IF_END75]]
+// CHECK1:       [[IF_END75]]:
+// CHECK1-NEXT:    br label %[[IF_END76]]
+// CHECK1:       [[IF_END76]]:
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    [[CMP77:%.*]] = icmp slt i64 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    br i1 [[CMP77]], label %[[IF_THEN78:.*]], label %[[IF_END83:.*]]
+// CHECK1:       [[IF_THEN78]]:
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_LB118]], align 8
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_ST119]], align 8
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    [[ADD80:%.*]] = add nsw i64 [[TMP58]], [[MUL79]]
+// CHECK1-NEXT:    store i64 [[ADD80]], ptr [[DOTOMP_IV122]], align 8
+// CHECK1-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV122]], align 8
+// CHECK1-NEXT:    [[MUL81:%.*]] = mul nsw i64 [[TMP62]], 1
+// CHECK1-NEXT:    [[ADD_PTR82:%.*]] = getelementptr inbounds double, ptr [[TMP61]], i64 [[MUL81]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR82]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP63]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[C]], align 4
+// CHECK1-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP66:%.*]] = load double, ptr [[TMP65]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP64]], double noundef [[TMP66]])
+// CHECK1-NEXT:    br label %[[IF_END83]]
+// CHECK1:       [[IF_END83]]:
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP67]], [[TMP68]]
+// CHECK1-NEXT:    br i1 [[CMP84]], label %[[IF_THEN85:.*]], label %[[IF_END90:.*]]
+// CHECK1:       [[IF_THEN85]]:
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], [[TMP71]]
+// CHECK1-NEXT:    [[ADD87:%.*]] = add nsw i64 [[TMP69]], [[MUL86]]
+// CHECK1-NEXT:    store i64 [[ADD87]], ptr [[DOTOMP_IV2]], align 8
+// CHECK1-NEXT:    [[TMP72:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[TMP73]], 1
+// CHECK1-NEXT:    [[ADD_PTR89:%.*]] = getelementptr inbounds double, ptr [[TMP72]], i64 [[MUL88]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR89]], ptr [[__BEGIN227]], align 8
+// CHECK1-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[__BEGIN227]], align 8
+// CHECK1-NEXT:    store ptr [[TMP74]], ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK1-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP77:%.*]] = load double, ptr [[TMP76]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP75]], double noundef [[TMP77]])
+// CHECK1-NEXT:    br label %[[IF_END90]]
+// CHECK1:       [[IF_END90]]:
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP78]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define dso_local void @body(
+// CHECK2-SAME: ...) #[[ATTR0:[0-9]+]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define dso_local void @foo1(
+// CHECK2-SAME: i32 noundef [[START1:%.*]], i32 noundef [[END1:%.*]], i32 noundef [[STEP1:%.*]], i32 noundef [[START2:%.*]], i32 noundef [[END2:%.*]], i32 noundef [[STEP2:%.*]]) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[START1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[END1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[STEP1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[START2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[END2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[STEP2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    store i32 [[START1]], ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[END1]], ptr [[END1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[STEP1]], ptr [[STEP1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[START2]], ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[END2]], ptr [[END2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[STEP2]], ptr [[STEP2_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK2-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
+// CHECK2-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP20]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP21]], [[TMP22]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP23]], %[[COND_TRUE]] ], [ [[TMP24]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
+// CHECK2-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP17]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP30]], [[TMP31]]
+// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP29]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP33]], [[TMP34]]
+// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP32]], [[MUL19]]
+// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP35]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    br i1 [[CMP21]], label %[[IF_THEN22:.*]], label %[[IF_END27:.*]]
+// CHECK2:       [[IF_THEN22]]:
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP39]], [[TMP40]]
+// CHECK2-NEXT:    [[ADD24:%.*]] = add i32 [[TMP38]], [[MUL23]]
+// CHECK2-NEXT:    store i32 [[ADD24]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[TMP41]], [[MUL25]]
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK2-NEXT:    br label %[[IF_END27]]
+// CHECK2:       [[IF_END27]]:
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP45]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define dso_local void @foo3(
+// CHECK2-SAME: ) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB03:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB04:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST05:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI06:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV07:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_UB117:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB118:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_ST119:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_NI120:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV122:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[CC:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE223:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END224:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN227:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_29:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_31:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_32:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_TEMP_142:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX48:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX54:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[VV:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK2-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTOMP_UB03]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB04]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST05]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], 1
+// CHECK2-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
+// CHECK2-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI06]], align 8
+// CHECK2-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY8:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY8]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY10:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP11]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY10]], ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP12]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP14]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK2-NEXT:    [[SUB13:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i64 [[SUB13]], 1
+// CHECK2-NEXT:    [[DIV15:%.*]] = sdiv i64 [[ADD14]], 1
+// CHECK2-NEXT:    [[SUB16:%.*]] = sub nsw i64 [[DIV15]], 1
+// CHECK2-NEXT:    store i64 [[SUB16]], ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_UB117]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB118]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST119]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK2-NEXT:    store i64 [[ADD21]], ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    store i32 37, ptr [[CC]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE223]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY25:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR26:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY25]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR26]], ptr [[__END224]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP18]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[__BEGIN227]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY30:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP19]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY30]], ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[__END224]], align 8
+// CHECK2-NEXT:    store ptr [[TMP20]], ptr [[DOTCAPTURE_EXPR_31]], align 8
+// CHECK2-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_31]], align 8
+// CHECK2-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST33:%.*]] = ptrtoint ptr [[TMP21]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST34:%.*]] = ptrtoint ptr [[TMP22]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB35:%.*]] = sub i64 [[SUB_PTR_LHS_CAST33]], [[SUB_PTR_RHS_CAST34]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV36:%.*]] = sdiv exact i64 [[SUB_PTR_SUB35]], 8
+// CHECK2-NEXT:    [[SUB37:%.*]] = sub nsw i64 [[SUB_PTR_DIV36]], 1
+// CHECK2-NEXT:    [[ADD38:%.*]] = add nsw i64 [[SUB37]], 1
+// CHECK2-NEXT:    [[DIV39:%.*]] = sdiv i64 [[ADD38]], 1
+// CHECK2-NEXT:    [[SUB40:%.*]] = sub nsw i64 [[DIV39]], 1
+// CHECK2-NEXT:    store i64 [[SUB40]], ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK2-NEXT:    store i64 [[TMP23]], ptr [[DOTOMP_UB2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB2]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST2]], align 8
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK2-NEXT:    [[ADD41:%.*]] = add nsw i64 [[TMP24]], 1
+// CHECK2-NEXT:    store i64 [[ADD41]], ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
+// CHECK2-NEXT:    store i64 [[TMP25]], ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP26]], [[TMP27]]
+// CHECK2-NEXT:    br i1 [[CMP43]], label %[[COND_TRUE44:.*]], label %[[COND_FALSE45:.*]]
+// CHECK2:       [[COND_TRUE44]]:
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK2-NEXT:    br label %[[COND_END46:.*]]
+// CHECK2:       [[COND_FALSE45]]:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    br label %[[COND_END46]]
+// CHECK2:       [[COND_END46]]:
+// CHECK2-NEXT:    [[COND47:%.*]] = phi i64 [ [[TMP28]], %[[COND_TRUE44]] ], [ [[TMP29]], %[[COND_FALSE45]] ]
+// CHECK2-NEXT:    store i64 [[COND47]], ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[CMP49:%.*]] = icmp sgt i64 [[TMP30]], [[TMP31]]
+// CHECK2-NEXT:    br i1 [[CMP49]], label %[[COND_TRUE50:.*]], label %[[COND_FALSE51:.*]]
+// CHECK2:       [[COND_TRUE50]]:
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    br label %[[COND_END52:.*]]
+// CHECK2:       [[COND_FALSE51]]:
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    br label %[[COND_END52]]
+// CHECK2:       [[COND_END52]]:
+// CHECK2-NEXT:    [[COND53:%.*]] = phi i64 [ [[TMP32]], %[[COND_TRUE50]] ], [ [[TMP33]], %[[COND_FALSE51]] ]
+// CHECK2-NEXT:    store i64 [[COND53]], ptr [[DOTOMP_FUSE_MAX48]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX48]], align 8
+// CHECK2-NEXT:    [[CMP55:%.*]] = icmp slt i64 [[TMP34]], [[TMP35]]
+// CHECK2-NEXT:    br i1 [[CMP55]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
+// CHECK2-NEXT:    [[CMP56:%.*]] = icmp slt i64 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    br i1 [[CMP56]], label %[[IF_THEN:.*]], label %[[IF_END76:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB04]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP38]] to i64
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST05]], align 4
+// CHECK2-NEXT:    [[CONV58:%.*]] = sext i32 [[TMP39]] to i64
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV58]], [[TMP40]]
+// CHECK2-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV57]], [[MUL]]
+// CHECK2-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK2-NEXT:    store i32 [[CONV60]], ptr [[DOTOMP_IV07]], align 4
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_IV07]], align 4
+// CHECK2-NEXT:    [[MUL61:%.*]] = mul nsw i32 [[TMP41]], 1
+// CHECK2-NEXT:    [[ADD62:%.*]] = add nsw i32 0, [[MUL61]]
+// CHECK2-NEXT:    store i32 [[ADD62]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP63:%.*]] = icmp slt i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    br i1 [[CMP63]], label %[[IF_THEN64:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN64]]:
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP45]], [[TMP46]]
+// CHECK2-NEXT:    [[ADD66:%.*]] = add nsw i32 [[TMP44]], [[MUL65]]
+// CHECK2-NEXT:    store i32 [[ADD66]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[MUL67:%.*]] = mul nsw i32 [[TMP47]], 1
+// CHECK2-NEXT:    [[ADD68:%.*]] = add nsw i32 0, [[MUL67]]
+// CHECK2-NEXT:    store i32 [[ADD68]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP48]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP69:%.*]] = icmp slt i32 [[TMP49]], [[TMP50]]
+// CHECK2-NEXT:    br i1 [[CMP69]], label %[[IF_THEN70:.*]], label %[[IF_END75:.*]]
+// CHECK2:       [[IF_THEN70]]:
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP52]], [[TMP53]]
+// CHECK2-NEXT:    [[ADD72:%.*]] = add nsw i32 [[TMP51]], [[MUL71]]
+// CHECK2-NEXT:    store i32 [[ADD72]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP54]], 2
+// CHECK2-NEXT:    [[ADD74:%.*]] = add nsw i32 0, [[MUL73]]
+// CHECK2-NEXT:    store i32 [[ADD74]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP55]])
+// CHECK2-NEXT:    br label %[[IF_END75]]
+// CHECK2:       [[IF_END75]]:
+// CHECK2-NEXT:    br label %[[IF_END76]]
+// CHECK2:       [[IF_END76]]:
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    [[CMP77:%.*]] = icmp slt i64 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    br i1 [[CMP77]], label %[[IF_THEN78:.*]], label %[[IF_END83:.*]]
+// CHECK2:       [[IF_THEN78]]:
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_LB118]], align 8
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_ST119]], align 8
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    [[ADD80:%.*]] = add nsw i64 [[TMP58]], [[MUL79]]
+// CHECK2-NEXT:    store i64 [[ADD80]], ptr [[DOTOMP_IV122]], align 8
+// CHECK2-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV122]], align 8
+// CHECK2-NEXT:    [[MUL81:%.*]] = mul nsw i64 [[TMP62]], 1
+// CHECK2-NEXT:    [[ADD_PTR82:%.*]] = getelementptr inbounds double, ptr [[TMP61]], i64 [[MUL81]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR82]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP63]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[C]], align 4
+// CHECK2-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP66:%.*]] = load double, ptr [[TMP65]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP64]], double noundef [[TMP66]])
+// CHECK2-NEXT:    br label %[[IF_END83]]
+// CHECK2:       [[IF_END83]]:
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP67]], [[TMP68]]
+// CHECK2-NEXT:    br i1 [[CMP84]], label %[[IF_THEN85:.*]], label %[[IF_END90:.*]]
+// CHECK2:       [[IF_THEN85]]:
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], [[TMP71]]
+// CHECK2-NEXT:    [[ADD87:%.*]] = add nsw i64 [[TMP69]], [[MUL86]]
+// CHECK2-NEXT:    store i64 [[ADD87]], ptr [[DOTOMP_IV2]], align 8
+// CHECK2-NEXT:    [[TMP72:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[TMP73]], 1
+// CHECK2-NEXT:    [[ADD_PTR89:%.*]] = getelementptr inbounds double, ptr [[TMP72]], i64 [[MUL88]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR89]], ptr [[__BEGIN227]], align 8
+// CHECK2-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[__BEGIN227]], align 8
+// CHECK2-NEXT:    store ptr [[TMP74]], ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK2-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP77:%.*]] = load double, ptr [[TMP76]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP75]], double noundef [[TMP77]])
+// CHECK2-NEXT:    br label %[[IF_END90]]
+// CHECK2:       [[IF_END90]]:
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP78]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define dso_local void @tfoo2(
+// CHECK2-SAME: ) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    call void @_Z4foo2IiEvT_S0_S0_(i32 noundef 0, i32 noundef 64, i32 noundef 4)
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define linkonce_odr void @_Z4foo2IiEvT_S0_S0_(
+// CHECK2-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0]] comdat {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP21:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK2-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
+// CHECK2-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP29]]
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP30]]
+// CHECK2-NEXT:    [[SUB27:%.*]] = sub i32 [[DIV26]], 1
+// CHECK2-NEXT:    store i32 [[SUB27]], ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK2-NEXT:    store i32 [[TMP31]], ptr [[DOTOMP_UB2]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB2]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST2]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK2-NEXT:    [[ADD28:%.*]] = add i32 [[TMP32]], 1
+// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP33]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP34]], [[TMP35]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP36]], %[[COND_TRUE]] ], [ [[TMP37]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    br i1 [[CMP29]], label %[[COND_TRUE30:.*]], label %[[COND_FALSE31:.*]]
+// CHECK2:       [[COND_TRUE30]]:
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    br label %[[COND_END32:.*]]
+// CHECK2:       [[COND_FALSE31]]:
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    br label %[[COND_END32]]
+// CHECK2:       [[COND_END32]]:
+// CHECK2-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP40]], %[[COND_TRUE30]] ], [ [[TMP41]], %[[COND_FALSE31]] ]
+// CHECK2-NEXT:    store i32 [[COND33]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    br i1 [[CMP34]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP44]], [[TMP45]]
+// CHECK2-NEXT:    br i1 [[CMP35]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP47]], [[TMP48]]
+// CHECK2-NEXT:    [[ADD36:%.*]] = add i32 [[TMP46]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD36]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP50]], [[TMP51]]
+// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[TMP49]], [[MUL37]]
+// CHECK2-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP53]], [[TMP54]]
+// CHECK2-NEXT:    br i1 [[CMP39]], label %[[IF_THEN40:.*]], label %[[IF_END45:.*]]
+// CHECK2:       [[IF_THEN40]]:
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[ADD42:%.*]] = add i32 [[TMP55]], [[MUL41]]
+// CHECK2-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP58]], [[MUL43]]
+// CHECK2-NEXT:    store i32 [[SUB44]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP61]])
+// CHECK2-NEXT:    br label %[[IF_END45]]
+// CHECK2:       [[IF_END45]]:
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP62]], [[TMP63]]
+// CHECK2-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
+// CHECK2:       [[IF_THEN47]]:
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP65]], [[TMP66]]
+// CHECK2-NEXT:    [[ADD49:%.*]] = add i32 [[TMP64]], [[MUL48]]
+// CHECK2-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV2]], align 4
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP68]], [[TMP69]]
+// CHECK2-NEXT:    [[ADD51:%.*]] = add i32 [[TMP67]], [[MUL50]]
+// CHECK2-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP70]])
+// CHECK2-NEXT:    br label %[[IF_END52]]
+// CHECK2:       [[IF_END52]]:
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    ret void
+//
+//.
+// CHECK1: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
+// CHECK1: [[META4]] = !{!"llvm.loop.mustprogress"}
+// CHECK1: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
+// CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
+//.
+// CHECK2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
+// CHECK2: [[META4]] = !{!"llvm.loop.mustprogress"}
+// CHECK2: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
+// CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
+//.
diff --git a/clang/test/OpenMP/fuse_messages.cpp b/clang/test/OpenMP/fuse_messages.cpp
new file mode 100644
index 0000000000000..50dedfd2c0dc6
--- /dev/null
+++ b/clang/test/OpenMP/fuse_messages.cpp
@@ -0,0 +1,76 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+void func() {
+
+    // expected-error at +2 {{statement after '#pragma omp fuse' must be a loop sequence containing canonical loops or loop-generating constructs}}
+    #pragma omp fuse 
+    ;
+
+    // expected-error at +2 {{statement after '#pragma omp fuse' must be a for loop}}
+    #pragma omp fuse 
+    {int bar = 0;}
+
+    // expected-error at +4 {{statement after '#pragma omp fuse' must be a for loop}}
+    #pragma omp fuse 
+    {
+        for(int i = 0; i < 10; ++i);
+        int x = 2;
+    }
+
+    // expected-error at +2 {{statement after '#pragma omp fuse' must be a loop sequence containing canonical loops or loop-generating constructs}}
+    #pragma omp fuse 
+    #pragma omp for 
+    for (int i = 0; i < 7; ++i)
+        ;
+
+    {
+        // expected-error at +2 {{expected statement}}
+        #pragma omp fuse
+    }
+
+    // expected-warning at +1 {{extra tokens at the end of '#pragma omp fuse' are ignored}}
+    #pragma omp fuse foo
+    {
+        for (int i = 0; i < 7; ++i)
+            ;
+    }
+
+
+    // expected-error at +1 {{unexpected OpenMP clause 'final' in directive '#pragma omp fuse'}}
+    #pragma omp fuse final(0) 
+    {
+        for (int i = 0; i < 7; ++i)
+            ;
+    }
+
+    //expected-error at +4 {{loop after '#pragma omp fuse' is not in canonical form}}
+    //expected-error at +3 {{increment clause of OpenMP for loop must perform simple addition or subtraction on loop variable 'i'}}
+    #pragma omp fuse 
+    {
+        for(int i = 0; i < 10; i*=2) {
+            ;
+        }
+    }
+
+    //expected-error at +2 {{loop sequence after '#pragma omp fuse' must contain at least 1 canonical loop or loop-generating construct}}
+    #pragma omp fuse 
+    {}
+
+    //expected-error at +3 {{statement after '#pragma omp fuse' must be a for loop}}
+    #pragma omp fuse 
+    {
+        #pragma omp unroll full 
+        for(int i = 0; i < 10; ++i);
+        
+        for(int j = 0; j < 10; ++j);
+    }
+
+    //expected-warning at +5 {{loop sequence following '#pragma omp fuse' contains induction variables of differing types: 'int' and 'unsigned int'}}
+    //expected-warning at +5 {{loop sequence following '#pragma omp fuse' contains induction variables of differing types: 'int' and 'long long'}}
+    #pragma omp fuse 
+    {
+        for(int i = 0; i < 10; ++i);
+        for(unsigned int j = 0; j < 10; ++j);
+        for(long long k = 0; k < 100; ++k);
+    }
+}
\ No newline at end of file
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 3068621d9c004..3afa59b2f2d6c 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2211,6 +2211,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFuseDirective(const OMPFuseDirective *D);
   void VisitOMPForDirective(const OMPForDirective *D);
   void VisitOMPForSimdDirective(const OMPForSimdDirective *D);
   void VisitOMPSectionsDirective(const OMPSectionsDirective *D);
@@ -3369,6 +3370,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPLoopTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFuseDirective(const OMPFuseDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPForDirective(const OMPForDirective *D) {
   VisitOMPLoopDirective(D);
 }
@@ -6323,6 +6328,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFuseDirective:
+    return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPForDirective:
     return cxstring::createRef("OMPForDirective");
   case CXCursor_OMPForSimdDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index a6301daa672c3..a6d032fa302b1 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -687,6 +687,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFuseDirectiveClass:
+    K = CXCursor_OMPFuseDirective;
+    break;
   case Stmt::OMPForDirectiveClass:
     K = CXCursor_OMPForDirective;
     break;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a87111cb5a11d..6352be8069e9e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -855,6 +855,10 @@ def OMP_For : Directive<[Spelling<"for">]> {
   let category = CA_Executable;
   let languages = [L_C];
 }
+def OMP_Fuse : Directive<[Spelling<"fuse">]> {
+  let association = AS_Loop;
+  let category = CA_Executable;
+}
 def OMP_Interchange : Directive<[Spelling<"interchange">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Permutation>,
diff --git a/openmp/runtime/test/transform/fuse/foreach.cpp b/openmp/runtime/test/transform/fuse/foreach.cpp
new file mode 100644
index 0000000000000..cabf4bf8a511d
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/foreach.cpp
@@ -0,0 +1,192 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+
+    void print(const char *msg) const { owner->print(msg); }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+#pragma omp fuse
+  {
+    for (Reporter a{"C"}; auto &&v : Reporter("A"))
+      printf("v=%d\n", v);
+    for (Reporter aa{"D"}; auto &&vv : Reporter("B"))
+      printf("vv=%d\n", vv);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+// CHECK: [C] ctor
+// CHECK-NEXT: [A] ctor
+// CHECK-NEXT: [A] end()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] iterator distance: 3
+// CHECK-NEXT: [D] ctor
+// CHECK-NEXT: [B] ctor
+// CHECK-NEXT: [B] end()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] iterator distance: 3
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: v=0
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: vv=0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: v=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: vv=1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: v=2
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: vv=2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] dtor
+// CHECK-NEXT: [D] dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] dtor
+// CHECK-NEXT: [C] dtor
+// CHECK-NEXT: done
+
+
+#endif
diff --git a/openmp/runtime/test/transform/fuse/intfor.c b/openmp/runtime/test/transform/fuse/intfor.c
new file mode 100644
index 0000000000000..b8171b4df7042
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/intfor.c
@@ -0,0 +1,50 @@
+// RUN: %libomp-compile-and-run  | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp fuse
+  {
+    for (int i = 5; i <= 25; i += 5)
+      printf("i=%d\n", i);
+    for (int j = 10; j < 100; j += 10)
+      printf("j=%d\n", j);
+    for (int k = 10; k > 0; --k)
+      printf("k=%d\n", k);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+#endif /* HEADER */
+
+// CHECK: do
+// CHECK-NEXT: i=5
+// CHECK-NEXT: j=10
+// CHECK-NEXT: k=10
+// CHECK-NEXT: i=10
+// CHECK-NEXT: j=20
+// CHECK-NEXT: k=9
+// CHECK-NEXT: i=15
+// CHECK-NEXT: j=30
+// CHECK-NEXT: k=8
+// CHECK-NEXT: i=20
+// CHECK-NEXT: j=40
+// CHECK-NEXT: k=7
+// CHECK-NEXT: i=25
+// CHECK-NEXT: j=50
+// CHECK-NEXT: k=6
+// CHECK-NEXT: j=60
+// CHECK-NEXT: k=5
+// CHECK-NEXT: j=70
+// CHECK-NEXT: k=4
+// CHECK-NEXT: j=80
+// CHECK-NEXT: k=3
+// CHECK-NEXT: j=90
+// CHECK-NEXT: k=2
+// CHECK-NEXT: k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/fuse/iterfor.cpp b/openmp/runtime/test/transform/fuse/iterfor.cpp
new file mode 100644
index 0000000000000..552484b2981c4
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/iterfor.cpp
@@ -0,0 +1,194 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    bool operator!=(const Iterator &that) const {
+      owner->print("iterator %d != %d", 2 - this->pos, 2 - that.pos);
+      return this->pos != that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+  Reporter C("C");
+  Reporter D("D");
+#pragma omp fuse
+  {
+    for (auto it = C.begin(); it != C.end(); ++it)
+      printf("v=%d\n", *it);
+
+    for (auto it = D.begin(); it != D.end(); ++it)
+      printf("vv=%d\n", *it);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK: do
+// CHECK: [C] ctor
+// CHECK-NEXT: [D] ctor
+// CHECK-NEXT: [C] begin()
+// CHECK-NEXT: [C] begin()
+// CHECK-NEXT: [C] end()
+// CHECK-NEXT: [C] iterator distance: 3
+// CHECK-NEXT: [D] begin()
+// CHECK-NEXT: [D] begin()
+// CHECK-NEXT: [D] end()
+// CHECK-NEXT: [D] iterator distance: 3
+// CHECK-NEXT: [C] iterator advance: 0 += 0
+// CHECK-NEXT: [C] iterator move assign
+// CHECK-NEXT: [C] iterator deref: 0
+// CHECK-NEXT: v=0
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [D] iterator advance: 0 += 0
+// CHECK-NEXT: [D] iterator move assign
+// CHECK-NEXT: [D] iterator deref: 0
+// CHECK-NEXT: vv=0
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [C] iterator advance: 0 += 1
+// CHECK-NEXT: [C] iterator move assign
+// CHECK-NEXT: [C] iterator deref: 1
+// CHECK-NEXT: v=1
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [D] iterator advance: 0 += 1
+// CHECK-NEXT: [D] iterator move assign
+// CHECK-NEXT: [D] iterator deref: 1
+// CHECK-NEXT: vv=1
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [C] iterator advance: 0 += 2
+// CHECK-NEXT: [C] iterator move assign
+// CHECK-NEXT: [C] iterator deref: 2
+// CHECK-NEXT: v=2
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [D] iterator advance: 0 += 2
+// CHECK-NEXT: [D] iterator move assign
+// CHECK-NEXT: [D] iterator deref: 2
+// CHECK-NEXT: vv=2
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: done
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [D] dtor
+// CHECK-NEXT: [C] dtor
diff --git a/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-foreach.cpp
new file mode 100644
index 0000000000000..e9f76713fe3e0
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-foreach.cpp
@@ -0,0 +1,208 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+
+    void print(const char *msg) const { owner->print(msg); }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+  for (int i = 0; i < 3; ++i)
+#pragma omp fuse
+  {
+    for (Reporter c{"init-stmt"}; auto &&v : Reporter("range"))
+      printf("i=%d v=%d\n", i, v);
+    for (int vv = 0; vv < 3; ++vv)
+      printf("i=%d vv=%d\n", i, vv);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK: do
+// CHECK-NEXT: [init-stmt] ctor
+// CHECK-NEXT: [range] ctor
+// CHECK-NEXT: [range] end()
+// CHECK-NEXT: [range] begin()
+// CHECK-NEXT: [range] begin()
+// CHECK-NEXT: [range] iterator distance: 3
+// CHECK-NEXT: [range] iterator advance: 0 += 0
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 0
+// CHECK-NEXT: i=0 v=0
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=0 vv=0
+// CHECK-NEXT: [range] iterator advance: 0 += 1
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 1
+// CHECK-NEXT: i=0 v=1
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=0 vv=1
+// CHECK-NEXT: [range] iterator advance: 0 += 2
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 2
+// CHECK-NEXT: i=0 v=2
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=0 vv=2
+// CHECK-NEXT: [range] iterator advance: 0 += 0
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 0
+// CHECK-NEXT: i=1 v=0
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=1 vv=0
+// CHECK-NEXT: [range] iterator advance: 0 += 1
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 1
+// CHECK-NEXT: i=1 v=1
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=1 vv=1
+// CHECK-NEXT: [range] iterator advance: 0 += 2
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 2
+// CHECK-NEXT: i=1 v=2
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=1 vv=2
+// CHECK-NEXT: [range] iterator advance: 0 += 0
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 0
+// CHECK-NEXT: i=2 v=0
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=2 vv=0
+// CHECK-NEXT: [range] iterator advance: 0 += 1
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 1
+// CHECK-NEXT: i=2 v=1
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=2 vv=1
+// CHECK-NEXT: [range] iterator advance: 0 += 2
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 2
+// CHECK-NEXT: i=2 v=2
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=2 vv=2
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: [range] dtor
+// CHECK-NEXT: [init-stmt] dtor
+// CHECK-NEXT: done
+
diff --git a/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c
new file mode 100644
index 0000000000000..272908e72c429
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c
@@ -0,0 +1,45 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+  for (int i = 0; i < 3; ++i)
+#pragma omp fuse
+  {
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+    for (int k = 0; k < 3; ++k)
+      printf("i=%d k=%d\n", i, k);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK: i=0 j=0
+// CHECK-NEXT: i=0 k=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 k=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=0 k=2
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 k=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 k=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: i=1 k=2
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 k=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=2 k=1
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: i=2 k=2
+// CHECK-NEXT: done

>From 34ac92ada84eeca9573d0b005f24d73738f46626 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:28:04 +0000
Subject: [PATCH 02/12] Add looprange clause

---
 clang/include/clang/AST/OpenMPClause.h        | 100 ++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |   8 +
 clang/include/clang/AST/StmtOpenMP.h          |   9 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   5 +
 clang/include/clang/Parse/Parser.h            |   3 +
 clang/include/clang/Sema/SemaOpenMP.h         |   6 +
 clang/lib/AST/OpenMPClause.cpp                |  35 ++
 clang/lib/AST/StmtOpenMP.cpp                  |   7 +-
 clang/lib/AST/StmtProfile.cpp                 |   7 +
 clang/lib/Basic/OpenMPKinds.cpp               |   2 +
 clang/lib/Parse/ParseOpenMP.cpp               |  36 ++
 clang/lib/Sema/SemaOpenMP.cpp                 | 155 +++++++--
 clang/lib/Sema/TreeTransform.h                |  33 ++
 clang/lib/Serialization/ASTReader.cpp         |  11 +
 clang/lib/Serialization/ASTReaderStmt.cpp     |   4 +-
 clang/lib/Serialization/ASTWriter.cpp         |   8 +
 clang/test/OpenMP/fuse_ast_print.cpp          |  67 ++++
 clang/test/OpenMP/fuse_codegen.cpp            | 320 +++++++++++++++++-
 clang/test/OpenMP/fuse_messages.cpp           | 112 +++++-
 clang/tools/libclang/CIndex.cpp               |   5 +
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |  16 +-
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |   6 +
 22 files changed, 919 insertions(+), 36 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 6fd16bc0f03be..8f937cdef9cd0 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1143,6 +1143,106 @@ class OMPFullClause final : public OMPNoChildClause<llvm::omp::OMPC_full> {
   static OMPFullClause *CreateEmpty(const ASTContext &C);
 };
 
+/// This class represents the 'looprange' clause in the
+/// '#pragma omp fuse' directive
+///
+/// \code {c}
+/// #pragma omp fuse looprange(1,2)
+/// {
+///   for(int i = 0; i < 64; ++i)
+///   for(int j = 0; j < 256; j+=2)
+///   for(int k = 127; k >= 0; --k)
+/// \endcode
+class OMPLoopRangeClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  explicit OMPLoopRangeClause()
+      : OMPClause(llvm::omp::OMPC_looprange, {}, {}) {}
+
+  /// Location of '('
+  SourceLocation LParenLoc;
+
+  /// Location of 'first'
+  SourceLocation FirstLoc;
+
+  /// Location of 'count'
+  SourceLocation CountLoc;
+
+  /// Expr associated with 'first' argument
+  Expr *First = nullptr;
+
+  /// Expr associated with 'count' argument
+  Expr *Count = nullptr;
+
+  /// Set 'first'
+  void setFirst(Expr *First) { this->First = First; }
+
+  /// Set 'count'
+  void setCount(Expr *Count) { this->Count = Count; }
+
+  /// Set location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+  /// Set location of 'first' argument
+  void setFirstLoc(SourceLocation Loc) { FirstLoc = Loc; }
+
+  /// Set location of 'count' argument
+  void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'looprange' clause
+  ///
+  /// \param StartLoc     Starting location of the clause.
+  /// \param LParenLoc    Location of '('.
+  /// \param ModifierLoc  Modifier location.
+  /// \param
+  static OMPLoopRangeClause *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
+         SourceLocation FirstLoc, SourceLocation CountLoc,
+         SourceLocation EndLoc, Expr *First, Expr *Count);
+
+  /// Build an empty 'looprange' node for deserialization
+  ///
+  /// \param C      Context of the AST.
+  static OMPLoopRangeClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the location of 'first'
+  SourceLocation getFirstLoc() const { return FirstLoc; }
+
+  /// Returns the location of 'count'
+  SourceLocation getCountLoc() const { return CountLoc; }
+
+  /// Returns the argument 'first' or nullptr if not set
+  Expr *getFirst() const { return cast_or_null<Expr>(First); }
+
+  /// Returns the argument 'count' or nullptr if not set
+  Expr *getCount() const { return cast_or_null<Expr>(Count); }
+
+  child_range children() {
+    return child_range(reinterpret_cast<Stmt **>(&First),
+                       reinterpret_cast<Stmt **>(&Count) + 1);
+  }
+
+  const_child_range children() const {
+    auto Children = const_cast<OMPLoopRangeClause *>(this)->children();
+    return const_child_range(Children.begin(), Children.end());
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_looprange;
+  }
+};
+
 /// Representation of the 'partial' clause of the '#pragma omp unroll'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 918216e8df4aa..10e44e69dd5da 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3410,6 +3410,14 @@ bool RecursiveASTVisitor<Derived>::VisitOMPFullClause(OMPFullClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPLoopRangeClause(
+    OMPLoopRangeClause *C) {
+  TRY_TO(TraverseStmt(C->getFirst()));
+  TRY_TO(TraverseStmt(C->getCount()));
+  return true;
+}
+
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   TRY_TO(TraverseStmt(C->getFactor()));
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index cb8bb91f4768c..f5115afd0753e 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5572,7 +5572,9 @@ class OMPTileDirective final : public OMPLoopTransformationDirective {
       : OMPLoopTransformationDirective(OMPTileDirectiveClass,
                                        llvm::omp::OMPD_tile, StartLoc, EndLoc,
                                        NumLoops) {
+    // Tiling doubles the original number of loops
     setNumGeneratedLoops(2 * NumLoops);
+    // Produces a single top-level canonical loop nest
     setNumGeneratedLoopNests(1);
   }
 
@@ -5804,6 +5806,7 @@ class OMPReverseDirective final : public OMPLoopTransformationDirective {
       : OMPLoopTransformationDirective(OMPReverseDirectiveClass,
                                        llvm::omp::OMPD_reverse, StartLoc,
                                        EndLoc, NumLoops) {
+    // Reverse produces a single top-level canonical loop nest
     setNumGeneratedLoops(NumLoops);
     setNumGeneratedLoopNests(1);
   }
@@ -5877,6 +5880,8 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
       : OMPLoopTransformationDirective(OMPInterchangeDirectiveClass,
                                        llvm::omp::OMPD_interchange, StartLoc,
                                        EndLoc, NumLoops) {
+    // Interchange produces a single top-level canonical loop
+    // nest, with the exact same amount of total loops
     setNumGeneratedLoops(NumLoops);
     setNumGeneratedLoopNests(1);
   }
@@ -5995,8 +6000,10 @@ class OMPFuseDirective final : public OMPLoopTransformationDirective {
   /// \param C Context of the AST
   /// \param NumClauses Number of clauses to allocate
   /// \param NumLoops Number of associated loops to allocate
+  /// \param NumLoopNests Number of top level loops to allocate
   static OMPFuseDirective *CreateEmpty(const ASTContext &C, unsigned NumClauses,
-                                       unsigned NumLoops);
+                                       unsigned NumLoops,
+                                       unsigned NumLoopNests);
 
   /// Gets the associated loops after the transformation. This is the de-sugared
   /// replacement or nulltpr in dependent contexts.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index fe9ca29038a1f..002aa7a774fbe 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11620,6 +11620,11 @@ def err_omp_not_a_loop_sequence : Error <
   "statement after '#pragma omp %0' must be a loop sequence containing canonical loops or loop-generating constructs">;
 def err_omp_empty_loop_sequence : Error <
   "loop sequence after '#pragma omp %0' must contain at least 1 canonical loop or loop-generating construct">;
+def err_omp_invalid_looprange : Error <
+  "loop range in '#pragma omp %0' exceeds the number of available loops: "
+  "range end '%1' is greater than the total number of loops '%2'">;
+def warn_omp_redundant_fusion : Warning <
+  "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index a47e23ffbd357..08bee0078b5ff 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -6735,6 +6735,9 @@ class Parser : public CodeCompletionHandler {
                                                 OpenMPClauseKind Kind,
                                                 bool ParseOnly);
 
+  /// Parses the 'looprange' clause of a '#pragma omp fuse' directive.
+  OMPClause *ParseOpenMPLoopRangeClause();
+  
   /// Parses the 'sizes' clause of a '#pragma omp tile' directive.
   OMPClause *ParseOpenMPSizesClause();
 
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index ea21377a8db9c..0c28aaf6ab21a 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -922,6 +922,12 @@ class SemaOpenMP : public SemaBase {
                                        SourceLocation StartLoc,
                                        SourceLocation LParenLoc,
                                        SourceLocation EndLoc);
+
+  /// Called on well-form 'looprange' clause after parsing its arguments.
+  OMPClause *
+  ActOnOpenMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
+                             SourceLocation LParenLoc, SourceLocation FirstLoc,
+                             SourceLocation CountLoc, SourceLocation EndLoc);
   /// Called on well-formed 'ordered' clause.
   OMPClause *
   ActOnOpenMPOrderedClause(SourceLocation StartLoc, SourceLocation EndLoc,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 0e5052b944162..0b5808eb100e4 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1024,6 +1024,26 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPLoopRangeClause *
+OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
+                           SourceLocation LParenLoc, SourceLocation EndLoc,
+                           SourceLocation FirstLoc, SourceLocation CountLoc,
+                           Expr *First, Expr *Count) {
+  OMPLoopRangeClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setFirstLoc(FirstLoc);
+  Clause->setCountLoc(CountLoc);
+  Clause->setFirst(First);
+  Clause->setCount(Count);
+  return Clause;
+}
+
+OMPLoopRangeClause *OMPLoopRangeClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPLoopRangeClause();
+}
+
 OMPAllocateClause *OMPAllocateClause::Create(
     const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
     Expr *Allocator, Expr *Alignment, SourceLocation ColonLoc,
@@ -1888,6 +1908,21 @@ void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   }
 }
 
+void OMPClausePrinter::VisitOMPLoopRangeClause(OMPLoopRangeClause *Node) {
+  OS << "looprange";
+
+  Expr *First = Node->getFirst();
+  Expr *Count = Node->getCount();
+
+  if (First && Count) {
+    OS << "(";
+    First->printPretty(OS, nullptr, Policy, 0);
+    OS << ",";
+    Count->printPretty(OS, nullptr, Policy, 0);
+    OS << ")";
+  }
+}
+
 void OMPClausePrinter::VisitOMPAllocatorClause(OMPAllocatorClause *Node) {
   OS << "allocator(";
   Node->getAllocator()->printPretty(OS, nullptr, Policy, 0);
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 276e43ec9f7d5..c5a6732cc2217 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -527,10 +527,13 @@ OMPFuseDirective *OMPFuseDirective::Create(
 
 OMPFuseDirective *OMPFuseDirective::CreateEmpty(const ASTContext &C,
                                                 unsigned NumClauses,
-                                                unsigned NumLoops) {
-  return createEmptyDirective<OMPFuseDirective>(
+                                                unsigned NumLoops,
+                                                unsigned NumLoopNests) {
+  OMPFuseDirective *Dir = createEmptyDirective<OMPFuseDirective>(
       C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
       SourceLocation(), SourceLocation(), NumLoops);
+  Dir->setNumGeneratedLoopNests(NumLoopNests);
+  return Dir;
 }
 
 OMPForSimdDirective *
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index c5d1d5b48508e..34ed3f22f6eb7 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -511,6 +511,13 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
+  if (const Expr *First = C->getFirst())
+    Profiler->VisitExpr(First);
+  if (const Expr *Count = C->getCount())
+    Profiler->VisitExpr(Count);
+}
+
 void OMPClauseProfiler::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   if (C->getAllocator())
     Profiler->VisitStmt(C->getAllocator());
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index d172450512f13..18330181f1509 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -248,6 +248,7 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
   case OMPC_affinity:
   case OMPC_when:
   case OMPC_append_args:
+  case OMPC_looprange:
     break;
   default:
     break;
@@ -583,6 +584,7 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_affinity:
   case OMPC_when:
   case OMPC_append_args:
+  case OMPC_looprange:
     break;
   default:
     break;
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 78d3503d8eb68..2d6d624c1ecc8 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3057,6 +3057,39 @@ OMPClause *Parser::ParseOpenMPSizesClause() {
                                                  OpenLoc, CloseLoc);
 }
 
+OMPClause *Parser::ParseOpenMPLoopRangeClause() {
+  SourceLocation ClauseNameLoc = ConsumeToken();
+  SourceLocation FirstLoc, CountLoc;
+
+  BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
+  if (T.consumeOpen()) {
+    Diag(Tok, diag::err_expected) << tok::l_paren;
+    return nullptr;
+  }
+
+  FirstLoc = Tok.getLocation();
+  ExprResult FirstVal = ParseConstantExpression();
+  if (!FirstVal.isUsable()) {
+    T.skipToEnd();
+    return nullptr;
+  }
+
+  ExpectAndConsume(tok::comma);
+
+  CountLoc = Tok.getLocation();
+  ExprResult CountVal = ParseConstantExpression();
+  if (!CountVal.isUsable()) {
+    T.skipToEnd();
+    return nullptr;
+  }
+
+  T.consumeClose();
+
+  return Actions.OpenMP().ActOnOpenMPLoopRangeClause(
+      FirstVal.get(), CountVal.get(), ClauseNameLoc, T.getOpenLocation(),
+      FirstLoc, CountLoc, T.getCloseLocation());
+}
+
 OMPClause *Parser::ParseOpenMPPermutationClause() {
   SourceLocation ClauseNameLoc, OpenLoc, CloseLoc;
   SmallVector<Expr *> ArgExprs;
@@ -3485,6 +3518,9 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
     }
     Clause = ParseOpenMPClause(CKind, WrongDirective);
     break;
+  case OMPC_looprange:
+    Clause = ParseOpenMPLoopRangeClause();
+    break;  
   default:
     break;
   }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 84ac9587bd54d..3ec3f2ad31e78 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14326,7 +14326,6 @@ bool SemaOpenMP::checkTransformableLoopSequence(
   // and tries to match the input AST to the canonical loop sequence grammar
   // structure
 
-  auto NLCV = NestedLoopCounterVisitor();
   // Helper functions to validate canonical loop sequence grammar is valid
   auto isLoopSequenceDerivation = [](auto *Child) {
     return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
@@ -14429,7 +14428,7 @@ bool SemaOpenMP::checkTransformableLoopSequence(
 
   // Modularized code for handling regular canonical loops
   auto handleRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
-                            &LoopSeqSize, &NumLoops, Kind, &TmpDSA, &NLCV,
+                            &LoopSeqSize, &NumLoops, Kind, &TmpDSA,
                             this](Stmt *Child) {
     OriginalInits.emplace_back();
     LoopHelpers.emplace_back();
@@ -14442,8 +14441,11 @@ bool SemaOpenMP::checkTransformableLoopSequence(
           << getOpenMPDirectiveName(Kind);
       return false;
     }
+
     storeLoopStatements(Child);
-    NumLoops += NLCV.TraverseStmt(Child);
+    auto NLCV = NestedLoopCounterVisitor();
+    NLCV.TraverseStmt(Child);
+    NumLoops += NLCV.getNestedLoopCount();
     return true;
   };
 
@@ -15769,6 +15771,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
                                                 SourceLocation EndLoc) {
+
   ASTContext &Context = getASTContext();
   DeclContext *CurrContext = SemaRef.CurContext;
   Scope *CurScope = SemaRef.getCurScope();
@@ -15785,7 +15788,6 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   SmallVector<SmallVector<Stmt *, 0>> OriginalInits;
 
   unsigned NumLoops;
-  // TODO: Support looprange clause using LoopSeqSize
   unsigned LoopSeqSize;
   if (!checkTransformableLoopSequence(OMPD_fuse, AStmt, LoopSeqSize, NumLoops,
                                       LoopHelpers, LoopStmts, OriginalInits,
@@ -15794,10 +15796,67 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   }
 
   // Defer transformation in dependent contexts
+  // The NumLoopNests argument is set to a placeholder (0)
+  // because a dependent context could prevent determining its true value
   if (CurrContext->isDependentContext()) {
     return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                    NumLoops, 1, AStmt, nullptr, nullptr);
+                                    NumLoops, 0, AStmt, nullptr, nullptr);
   }
+
+  // Handle clauses, which can be any of the following: [looprange, apply]
+  const OMPLoopRangeClause *LRC =
+      OMPExecutableDirective::getSingleClause<OMPLoopRangeClause>(Clauses);
+
+  // The clause arguments are invalidated if any error arises
+  // such as non-constant or non-positive arguments
+  if (LRC && (!LRC->getFirst() || !LRC->getCount()))
+    return StmtError();
+
+  // Delayed semantic check of LoopRange constraint
+  // Evaluates the loop range arguments and returns the first and count values
+  auto EvaluateLoopRangeArguments = [&Context](Expr *First, Expr *Count,
+                                               uint64_t &FirstVal,
+                                               uint64_t &CountVal) {
+    llvm::APSInt FirstInt = First->EvaluateKnownConstInt(Context);
+    llvm::APSInt CountInt = Count->EvaluateKnownConstInt(Context);
+    FirstVal = FirstInt.getZExtValue();
+    CountVal = CountInt.getZExtValue();
+  };
+
+  // Checks if the loop range is valid
+  auto ValidLoopRange = [](uint64_t FirstVal, uint64_t CountVal,
+                           unsigned NumLoops) -> bool {
+    return FirstVal + CountVal - 1 <= NumLoops;
+  };
+  uint64_t FirstVal = 1, CountVal = 0, LastVal = LoopSeqSize;
+
+  if (LRC) {
+    EvaluateLoopRangeArguments(LRC->getFirst(), LRC->getCount(), FirstVal,
+                               CountVal);
+    if (CountVal == 1)
+      SemaRef.Diag(LRC->getCountLoc(), diag::warn_omp_redundant_fusion)
+          << getOpenMPDirectiveName(OMPD_fuse);
+
+    if (!ValidLoopRange(FirstVal, CountVal, LoopSeqSize)) {
+      SemaRef.Diag(LRC->getFirstLoc(), diag::err_omp_invalid_looprange)
+          << getOpenMPDirectiveName(OMPD_fuse) << (FirstVal + CountVal - 1)
+          << LoopSeqSize;
+      return StmtError();
+    }
+
+    LastVal = FirstVal + CountVal - 1;
+  }
+
+  // Complete fusion generates a single canonical loop nest
+  // However looprange clause generates several loop nests
+  unsigned NumLoopNests = LRC ? LoopSeqSize - CountVal + 1 : 1;
+
+  // Emit a warning for redundant loop fusion when the sequence contains only
+  // one loop.
+  if (LoopSeqSize == 1)
+    SemaRef.Diag(AStmt->getBeginLoc(), diag::warn_omp_redundant_fusion)
+        << getOpenMPDirectiveName(OMPD_fuse);
+
   assert(LoopHelpers.size() == LoopSeqSize &&
          "Expecting loop iteration space dimensionality to match number of "
          "affected loops");
@@ -15811,8 +15870,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   SmallVector<Stmt *> PreInits;
 
   // Select the type with the largest bit width among all induction variables
-  QualType IVType = LoopHelpers[0].IterationVarRef->getType();
-  for (unsigned int I = 1; I < LoopSeqSize; ++I) {
+  QualType IVType = LoopHelpers[FirstVal - 1].IterationVarRef->getType();
+  for (unsigned int I = FirstVal; I < LastVal; ++I) {
     QualType CurrentIVType = LoopHelpers[I].IterationVarRef->getType();
     if (Context.getTypeSize(CurrentIVType) > Context.getTypeSize(IVType)) {
       IVType = CurrentIVType;
@@ -15861,20 +15920,21 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
 
   // Process each single loop to generate and collect declarations
   // and statements for all helper expressions
-  for (unsigned int I = 0; I < LoopSeqSize; ++I) {
+  for (unsigned int I = FirstVal - 1, J = 0; I < LastVal; ++I, ++J) {
     addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
                     PreInits);
 
-    auto [UBVD, UBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].UB, "ub", I);
-    auto [LBVD, LBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].LB, "lb", I);
-    auto [STVD, STDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].ST, "st", I);
+    auto [UBVD, UBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].UB, "ub", J);
+    auto [LBVD, LBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].LB, "lb", J);
+    auto [STVD, STDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].ST, "st", J);
     auto [NIVD, NIDStmt] =
-        CreateHelperVarAndStmt(LoopHelpers[I].NumIterations, "ni", I, true);
+        CreateHelperVarAndStmt(LoopHelpers[I].NumIterations, "ni", J, true);
     auto [IVVD, IVDStmt] =
-        CreateHelperVarAndStmt(LoopHelpers[I].IterationVarRef, "iv", I);
+        CreateHelperVarAndStmt(LoopHelpers[I].IterationVarRef, "iv", J);
 
     if (!LBVD || !STVD || !NIVD || !IVVD)
-      return StmtError();
+      assert(LBVD && STVD && NIVD && IVVD &&
+             "OpenMP Fuse Helper variables creation failed");
 
     UBVarDecls.push_back(UBVD);
     LBVarDecls.push_back(LBVD);
@@ -15949,8 +16009,9 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   //   omp.fuse.max = max(omp.temp1, omp.temp0)
 
   ExprResult MaxExpr;
-  for (unsigned I = 0; I < LoopSeqSize; ++I) {
-    DeclRefExpr *NIRef = MakeVarDeclRef(NIVarDecls[I]);
+  // I is the true
+  for (unsigned I = FirstVal - 1, J = 0; I < LastVal; ++I, ++J) {
+    DeclRefExpr *NIRef = MakeVarDeclRef(NIVarDecls[J]);
     QualType NITy = NIRef->getType();
 
     if (MaxExpr.isUnset()) {
@@ -15958,7 +16019,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
       MaxExpr = NIRef;
     } else {
       // Create a new acummulator variable t_i = MaxExpr
-      std::string TempName = (Twine(".omp.temp.") + Twine(I)).str();
+      std::string TempName = (Twine(".omp.temp.") + Twine(J)).str();
       VarDecl *TempDecl =
           buildVarDecl(SemaRef, {}, NITy, TempName, nullptr, nullptr);
       TempDecl->setInit(MaxExpr.get());
@@ -15981,7 +16042,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
       if (!Comparison.isUsable())
         return StmtError();
 
-      DeclRefExpr *NIRef2 = MakeVarDeclRef(NIVarDecls[I]);
+      DeclRefExpr *NIRef2 = MakeVarDeclRef(NIVarDecls[J]);
       // Update MaxExpr using a conditional expression to hold the max value
       MaxExpr = new (Context) ConditionalOperator(
           Comparison.get(), SourceLocation(), TempRef2, SourceLocation(),
@@ -16034,23 +16095,21 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
 
   CompoundStmt *FusedBody = nullptr;
   SmallVector<Stmt *, 4> FusedBodyStmts;
-  for (unsigned I = 0; I < LoopSeqSize; ++I) {
-
+  for (unsigned I = FirstVal - 1, J = 0; I < LastVal; ++I, ++J) {
     // Assingment of the original sub-loop index to compute the logical index
     // IV_k = LB_k + omp.fuse.index * ST_k
-
     ExprResult IdxExpr =
         SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Mul,
-                           MakeVarDeclRef(STVarDecls[I]), MakeIVRef());
+                           MakeVarDeclRef(STVarDecls[J]), MakeIVRef());
     if (!IdxExpr.isUsable())
       return StmtError();
     IdxExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Add,
-                                 MakeVarDeclRef(LBVarDecls[I]), IdxExpr.get());
+                                 MakeVarDeclRef(LBVarDecls[J]), IdxExpr.get());
 
     if (!IdxExpr.isUsable())
       return StmtError();
     IdxExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Assign,
-                                 MakeVarDeclRef(IVVarDecls[I]), IdxExpr.get());
+                                 MakeVarDeclRef(IVVarDecls[J]), IdxExpr.get());
     if (!IdxExpr.isUsable())
       return StmtError();
 
@@ -16065,7 +16124,6 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     Stmt *Body = (isa<ForStmt>(LoopStmts[I]))
                      ? cast<ForStmt>(LoopStmts[I])->getBody()
                      : cast<CXXForRangeStmt>(LoopStmts[I])->getBody();
-
     BodyStmts.push_back(Body);
 
     CompoundStmt *CombinedBody =
@@ -16073,7 +16131,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                              SourceLocation(), SourceLocation());
     ExprResult Condition =
         SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_LT, MakeIVRef(),
-                           MakeVarDeclRef(NIVarDecls[I]));
+                           MakeVarDeclRef(NIVarDecls[J]));
 
     if (!Condition.isUsable())
       return StmtError();
@@ -16094,8 +16152,26 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
               FusedBody, InitStmt.get()->getBeginLoc(), SourceLocation(),
               IncrExpr.get()->getEndLoc());
 
+  // In the case of looprange, the result of fuse won't simply
+  // be a single loop (ForStmt), but rather a loop sequence
+  // (CompoundStmt) of 3 parts: the pre-fusion loops, the fused loop
+  // and the post-fusion loops, preserving its original order.
+  Stmt *FusionStmt = FusedForStmt;
+  if (LRC) {
+    SmallVector<Stmt *, 4> FinalLoops;
+    // Gather all the pre-fusion loops
+    for (unsigned I = 0; I < FirstVal - 1; ++I)
+      FinalLoops.push_back(LoopStmts[I]);
+    // Gather the fused loop
+    FinalLoops.push_back(FusedForStmt);
+    // Gather all the post-fusion loops
+    for (unsigned I = FirstVal + CountVal - 1; I < LoopSeqSize; ++I)
+      FinalLoops.push_back(LoopStmts[I]);
+    FusionStmt = CompoundStmt::Create(Context, FinalLoops, FPOptionsOverride(),
+                                      SourceLocation(), SourceLocation());
+  }
   return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses, NumLoops,
-                                  1, AStmt, FusedForStmt,
+                                  NumLoopNests, AStmt, FusionStmt,
                                   buildPreInits(Context, PreInits));
 }
 
@@ -17218,6 +17294,31 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
+    Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
+    SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {
+
+  // OpenMP [6.0, Restrictions]
+  // First and Count must be integer expressions with positive value
+  ExprResult FirstVal =
+      VerifyPositiveIntegerConstantInClause(First, OMPC_looprange);
+  if (FirstVal.isInvalid())
+    First = nullptr;
+
+  ExprResult CountVal =
+      VerifyPositiveIntegerConstantInClause(Count, OMPC_looprange);
+  if (CountVal.isInvalid())
+    Count = nullptr;
+
+  // OpenMP [6.0, Restrictions]
+  // first + count - 1 must not evaluate to a value greater than the
+  // loop sequence length of the associated canonical loop sequence.
+  // This check must be performed afterwards due to the delayed
+  // parsing and computation of the associated loop sequence
+  return OMPLoopRangeClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                    FirstLoc, CountLoc, EndLoc, First, Count);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPAlignClause(Expr *A, SourceLocation StartLoc,
                                               SourceLocation LParenLoc,
                                               SourceLocation EndLoc) {
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 45f556f22c511..30204faf59b7b 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1775,6 +1775,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  OMPClause *
+  RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
+                            SourceLocation LParenLoc, SourceLocation FirstLoc,
+                            SourceLocation CountLoc, SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPLoopRangeClause(
+        First, Count, StartLoc, LParenLoc, FirstLoc, CountLoc, EndLoc);
+  }
+
   /// Build a new OpenMP 'allocator' clause.
   ///
   /// By default, performs semantic analysis to build the new OpenMP clause.
@@ -10578,6 +10586,31 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {
+  ExprResult F = getDerived().TransformExpr(C->getFirst());
+  if (F.isInvalid())
+    return nullptr;
+
+  ExprResult Cn = getDerived().TransformExpr(C->getCount());
+  if (Cn.isInvalid())
+    return nullptr;
+
+  Expr *First = F.get();
+  Expr *Count = Cn.get();
+
+  bool Changed = (First != C->getFirst()) || (Count != C->getCount());
+
+  // If no changes and AlwaysRebuild() is false, return the original clause
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+
+  return RebuildOMPLoopRangeClause(First, Count, C->getBeginLoc(),
+                                   C->getLParenLoc(), C->getFirstLoc(),
+                                   C->getCountLoc(), C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index a3fbc3d25acab..d5e7c287c23a4 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11089,6 +11089,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_looprange:
+    C = OMPLoopRangeClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_allocator:
     C = new (Context) OMPAllocatorClause();
     break;
@@ -11490,6 +11493,14 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
+  C->setFirst(Record.readSubExpr());
+  C->setCount(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+  C->setFirstLoc(Record.readSourceLocation());
+  C->setCountLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPAllocatorClause(OMPAllocatorClause *C) {
   C->setAllocator(Record.readExpr());
   C->setLParenLoc(Record.readSourceLocation());
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 291bd8ea4bf18..b424b5aa7b0c6 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -3616,7 +3616,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
     case STMT_OMP_FUSE_DIRECTIVE: {
       unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
-      S = OMPFuseDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      unsigned NumLoopNests = Record[ASTStmtReader::NumStmtFields + 2];
+      S = OMPFuseDirective::CreateEmpty(Context, NumClauses, NumLoops,
+                                        NumLoopNests);
       break;
     }
 
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index af7229d748872..c99ffab64c6e6 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7807,6 +7807,14 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
+  Record.AddStmt(C->getFirst());
+  Record.AddStmt(C->getCount());
+  Record.AddSourceLocation(C->getLParenLoc());
+  Record.AddSourceLocation(C->getFirstLoc());
+  Record.AddSourceLocation(C->getCountLoc());
+}
+
 void OMPClauseWriter::VisitOMPAllocatorClause(OMPAllocatorClause *C) {
   Record.AddStmt(C->getAllocator());
   Record.AddSourceLocation(C->getLParenLoc());
diff --git a/clang/test/OpenMP/fuse_ast_print.cpp b/clang/test/OpenMP/fuse_ast_print.cpp
index 43ce815dab024..ac4f0d38a9c68 100644
--- a/clang/test/OpenMP/fuse_ast_print.cpp
+++ b/clang/test/OpenMP/fuse_ast_print.cpp
@@ -271,6 +271,73 @@ void foo7() {
 
 }
 
+// PRINT-LABEL: void foo8(
+// DUMP-LABEL: FunctionDecl {{.*}} foo8
+void foo8() {
+    // PRINT: #pragma omp fuse looprange(2,2)
+    // DUMP:  OMPFuseDirective
+    // DUMP: OMPLooprangeClause
+    #pragma omp fuse looprange(2,2)
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int j = 10; j > 0; --j)
+        // DUMP: ForStmt
+        for (int j = 10; j > 0; --j)
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);
+        // PRINT: for (int k = 0; k <= 10; ++k)
+        // DUMP: ForStmt
+        for (int k = 0; k <= 10; ++k)
+            // PRINT: body(k)
+            // DUMP: CallExpr
+            body(k);
+
+    }
+
+}
+
+//PRINT-LABEL: void foo9(
+//DUMP-LABEL: FunctionTemplateDecl {{.*}} foo9
+//DUMP-LABEL: NonTypeTemplateParmDecl {{.*}} F
+//DUMP-LABEL: NonTypeTemplateParmDecl {{.*}} C
+template<int F, int C> 
+void foo9() {
+    // PRINT:  #pragma omp fuse looprange(F,C)
+    // DUMP: OMPFuseDirective
+    // DUMP: OMPLooprangeClause
+    #pragma omp fuse looprange(F,C)
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int j = 10; j > 0; --j)
+        // DUMP: ForStmt
+        for (int j = 10; j > 0; --j)
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);
+
+    }
+}
+
+// Also test instantiating the template.
+void tfoo9() {
+    foo9<1, 2>();
+}
+
 
 
 
diff --git a/clang/test/OpenMP/fuse_codegen.cpp b/clang/test/OpenMP/fuse_codegen.cpp
index 6c1e21092da43..d9500bed3ce31 100644
--- a/clang/test/OpenMP/fuse_codegen.cpp
+++ b/clang/test/OpenMP/fuse_codegen.cpp
@@ -53,6 +53,18 @@ extern "C" void foo3() {
     }
 }
 
+extern "C" void foo4() {
+    double arr[256];
+
+    #pragma omp fuse looprange(2,2)
+    {
+        for(int i = 0; i < 128; ++i) body(i);
+        for(int j = 0; j < 256; j+=2) body(j);
+        for(int k = 0; k < 64; ++k) body(k);
+        for(int c = 42; auto &&v: arr) body(c,v);
+    }
+}
+
 
 #endif
 // CHECK1-LABEL: define dso_local void @body(
@@ -777,6 +789,157 @@ extern "C" void foo3() {
 // CHECK1-NEXT:    ret void
 //
 //
+// CHECK1-LABEL: define dso_local void @foo4(
+// CHECK1-SAME: ) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[K]], align 4
+// CHECK1-NEXT:    store i32 63, ptr [[DOTOMP_UB1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    store i32 64, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP5]], 128
+// CHECK1-NEXT:    br i1 [[CMP1]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP6]])
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND2:.*]]
+// CHECK1:       [[FOR_COND2]]:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label %[[FOR_BODY4:.*]], label %[[FOR_END17:.*]]
+// CHECK1:       [[FOR_BODY4]]:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP10]], [[TMP11]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP13]], [[TMP14]]
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP12]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP15]], 2
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL6]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP16]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP17]], [[TMP18]]
+// CHECK1-NEXT:    br i1 [[CMP8]], label %[[IF_THEN9:.*]], label %[[IF_END14:.*]]
+// CHECK1:       [[IF_THEN9]]:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP19]], [[MUL10]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP22]], 1
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 0, [[MUL12]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP23]])
+// CHECK1-NEXT:    br label %[[IF_END14]]
+// CHECK1:       [[IF_END14]]:
+// CHECK1-NEXT:    br label %[[FOR_INC15:.*]]
+// CHECK1:       [[FOR_INC15]]:
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC16:%.*]] = add nsw i32 [[TMP24]], 1
+// CHECK1-NEXT:    store i32 [[INC16]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND2]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1:       [[FOR_END17]]:
+// CHECK1-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP25]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY18:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP26]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY18]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND19:.*]]
+// CHECK1:       [[FOR_COND19]]:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP28:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[CMP20:%.*]] = icmp ne ptr [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP20]], label %[[FOR_BODY21:.*]], label %[[FOR_END23:.*]]
+// CHECK1:       [[FOR_BODY21]]:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP29]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[C]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load double, ptr [[TMP31]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP30]], double noundef [[TMP32]])
+// CHECK1-NEXT:    br label %[[FOR_INC22:.*]]
+// CHECK1:       [[FOR_INC22]]:
+// CHECK1-NEXT:    [[TMP33:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw double, ptr [[TMP33]], i32 1
+// CHECK1-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND19]]
+// CHECK1:       [[FOR_END23]]:
+// CHECK1-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define dso_local void @body(
 // CHECK2-SAME: ...) #[[ATTR0:[0-9]+]] {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
@@ -1259,6 +1422,157 @@ extern "C" void foo3() {
 // CHECK2-NEXT:    ret void
 //
 //
+// CHECK2-LABEL: define dso_local void @foo4(
+// CHECK2-SAME: ) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[K]], align 4
+// CHECK2-NEXT:    store i32 63, ptr [[DOTOMP_UB1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    store i32 64, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP5]], 128
+// CHECK2-NEXT:    br i1 [[CMP1]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP6]])
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND2:.*]]
+// CHECK2:       [[FOR_COND2]]:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label %[[FOR_BODY4:.*]], label %[[FOR_END17:.*]]
+// CHECK2:       [[FOR_BODY4]]:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP10]], [[TMP11]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP13]], [[TMP14]]
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP12]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP15]], 2
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL6]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP16]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP17]], [[TMP18]]
+// CHECK2-NEXT:    br i1 [[CMP8]], label %[[IF_THEN9:.*]], label %[[IF_END14:.*]]
+// CHECK2:       [[IF_THEN9]]:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP19]], [[MUL10]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP22]], 1
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 0, [[MUL12]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP23]])
+// CHECK2-NEXT:    br label %[[IF_END14]]
+// CHECK2:       [[IF_END14]]:
+// CHECK2-NEXT:    br label %[[FOR_INC15:.*]]
+// CHECK2:       [[FOR_INC15]]:
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC16:%.*]] = add nsw i32 [[TMP24]], 1
+// CHECK2-NEXT:    store i32 [[INC16]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND2]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK2:       [[FOR_END17]]:
+// CHECK2-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP25]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY18:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP26]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY18]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND19:.*]]
+// CHECK2:       [[FOR_COND19]]:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP28:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[CMP20:%.*]] = icmp ne ptr [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP20]], label %[[FOR_BODY21:.*]], label %[[FOR_END23:.*]]
+// CHECK2:       [[FOR_BODY21]]:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP29]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[C]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load double, ptr [[TMP31]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP30]], double noundef [[TMP32]])
+// CHECK2-NEXT:    br label %[[FOR_INC22:.*]]
+// CHECK2:       [[FOR_INC22]]:
+// CHECK2-NEXT:    [[TMP33:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw double, ptr [[TMP33]], i32 1
+// CHECK2-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND19]]
+// CHECK2:       [[FOR_END23]]:
+// CHECK2-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define dso_local void @tfoo2(
 // CHECK2-SAME: ) #[[ATTR0]] {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
@@ -1494,7 +1808,7 @@ extern "C" void foo3() {
 // CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
 // CHECK2:       [[FOR_END]]:
 // CHECK2-NEXT:    ret void
 //
@@ -1503,9 +1817,13 @@ extern "C" void foo3() {
 // CHECK1: [[META4]] = !{!"llvm.loop.mustprogress"}
 // CHECK1: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
 // CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
+// CHECK1: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
+// CHECK1: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
 //.
 // CHECK2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
 // CHECK2: [[META4]] = !{!"llvm.loop.mustprogress"}
 // CHECK2: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
 // CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
+// CHECK2: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
+// CHECK2: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
 //.
diff --git a/clang/test/OpenMP/fuse_messages.cpp b/clang/test/OpenMP/fuse_messages.cpp
index 50dedfd2c0dc6..2a2491d008a0b 100644
--- a/clang/test/OpenMP/fuse_messages.cpp
+++ b/clang/test/OpenMP/fuse_messages.cpp
@@ -33,6 +33,8 @@ void func() {
     {
         for (int i = 0; i < 7; ++i)
             ;
+        for(int j = 0; j < 100; ++j);
+
     }
 
 
@@ -41,6 +43,8 @@ void func() {
     {
         for (int i = 0; i < 7; ++i)
             ;
+        for(int j = 0; j < 100; ++j);
+
     }
 
     //expected-error at +4 {{loop after '#pragma omp fuse' is not in canonical form}}
@@ -50,6 +54,7 @@ void func() {
         for(int i = 0; i < 10; i*=2) {
             ;
         }
+        for(int j = 0; j < 100; ++j);
     }
 
     //expected-error at +2 {{loop sequence after '#pragma omp fuse' must contain at least 1 canonical loop or loop-generating construct}}
@@ -73,4 +78,109 @@ void func() {
         for(unsigned int j = 0; j < 10; ++j);
         for(long long k = 0; k < 100; ++k);
     }
-}
\ No newline at end of file
+
+    //expected-warning at +2 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    #pragma omp fuse
+    {
+        for(int i = 0; i < 10; ++i);
+    }
+
+    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    #pragma omp fuse looprange(1, 1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+    }
+
+    //expected-error at +1 {{argument to 'looprange' clause must be a strictly positive integer value}}
+    #pragma omp fuse looprange(1, -1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+    }
+
+    //expected-error at +1 {{argument to 'looprange' clause must be a strictly positive integer value}}
+    #pragma omp fuse looprange(1, 0)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+    }
+
+    const int x = 1;
+    constexpr int y = 4;
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '4' is greater than the total number of loops '3'}}
+    #pragma omp fuse looprange(x,y)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '420' is greater than the total number of loops '3'}}
+    #pragma omp fuse looprange(1,420)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+}
+
+// In a template context, but expression itself not instantiation-dependent
+template <typename T>
+static void templated_func() {
+
+    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    #pragma omp fuse looprange(2,1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '5' is greater than the total number of loops '3'}}
+    #pragma omp fuse looprange(3,3)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+
+}
+
+template <int V> 
+static void templated_func_value_dependent() {
+
+    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    #pragma omp fuse looprange(V,1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+}
+
+template <typename T> 
+static void templated_func_type_dependent() {
+    constexpr T s = 1;
+
+    //expected-error at +1 {{argument to 'looprange' clause must be a strictly positive integer value}}
+    #pragma omp fuse looprange(s,s-1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+}
+
+
+void template_inst() {
+    // expected-note at +1 {{in instantiation of function template specialization 'templated_func<int>' requested here}}
+    templated_func<int>();
+    // expected-note at +1 {{in instantiation of function template specialization 'templated_func_value_dependent<1>' requested here}}
+    templated_func_value_dependent<1>();
+    // expected-note at +1 {{in instantiation of function template specialization 'templated_func_type_dependent<int>' requested here}}
+    templated_func_type_dependent<int>();
+
+}
+
+
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 3afa59b2f2d6c..deb6fdd68e6d3 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2417,6 +2417,11 @@ void OMPClauseEnqueue::VisitOMPPartialClause(const OMPPartialClause *C) {
   Visitor->AddStmt(C->getFactor());
 }
 
+void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
+  Visitor->AddStmt(C->getFirst());
+  Visitor->AddStmt(C->getCount());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index de888ff86fe91..a2f258bbf745b 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -1233,6 +1233,15 @@ struct WriteT {
   using EmptyTrait = std::true_type;
 };
 
+// V6: [6.4.7] Looprange clause
+template <typename T, typename I, typename E> struct LoopRangeT {
+  using Begin = E;
+  using End = E;
+
+  using TupleTrait = std::true_type;
+  std::tuple<Begin, End> t;
+};
+
 // ---
 
 template <typename T, typename I, typename E>
@@ -1263,9 +1272,10 @@ using TupleClausesT =
                  DefaultmapT<T, I, E>, DeviceT<T, I, E>, DistScheduleT<T, I, E>,
                  DoacrossT<T, I, E>, FromT<T, I, E>, GrainsizeT<T, I, E>,
                  IfT<T, I, E>, InitT<T, I, E>, InReductionT<T, I, E>,
-                 LastprivateT<T, I, E>, LinearT<T, I, E>, MapT<T, I, E>,
-                 NumTasksT<T, I, E>, OrderT<T, I, E>, ReductionT<T, I, E>,
-                 ScheduleT<T, I, E>, TaskReductionT<T, I, E>, ToT<T, I, E>>;
+                 LastprivateT<T, I, E>, LinearT<T, I, E>, LoopRangeT<T, I, E>,
+                 MapT<T, I, E>, NumTasksT<T, I, E>, OrderT<T, I, E>,
+                 ReductionT<T, I, E>, ScheduleT<T, I, E>,
+                 TaskReductionT<T, I, E>, ToT<T, I, E>>;
 
 template <typename T, typename I, typename E>
 using UnionClausesT = std::variant<DependT<T, I, E>>;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 6352be8069e9e..989b35a7caa2a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -274,6 +274,9 @@ def OMPC_Linear : Clause<[Spelling<"linear">]> {
 def OMPC_Link : Clause<[Spelling<"link">]> {
   let flangClass = "OmpObjectList";
 }
+def OMPC_LoopRange : Clause<[Spelling<"looprange">]> {
+  let clangClass = "OMPLoopRangeClause";
+}
 def OMPC_Map : Clause<[Spelling<"map">]> {
   let clangClass = "OMPMapClause";
   let flangClass = "OmpMapClause";
@@ -856,6 +859,9 @@ def OMP_For : Directive<[Spelling<"for">]> {
   let languages = [L_C];
 }
 def OMP_Fuse : Directive<[Spelling<"fuse">]> {
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_LoopRange, 60>
+  ];
   let association = AS_Loop;
   let category = CA_Executable;
 }

>From c76888b9dd1f516512d2d64afa4766adaad4de1e Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:30:39 +0000
Subject: [PATCH 03/12] Added fuse to documentation

---
 clang/docs/OpenMPSupport.rst | 2 ++
 clang/docs/ReleaseNotes.rst  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 986aaabe1eed4..12325e3294a64 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -376,6 +376,8 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop stripe transformation                                  | :good:`done`              | https://github.com/llvm/llvm-project/pull/119891                                                     |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
+| loop fuse transformation                                    | :good:`done`              | :none:`unclaimed`        |                                                                          |
++-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | work distribute construct                                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | task_iteration                                              | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index dd748ab06873d..c75cb25a4db73 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -1128,6 +1128,7 @@ OpenMP Support
 - Fixed mapping of arrays of structs containing nested structs with user defined
   mappers, by using compiler-generated default mappers for the outer structs for
   such maps.
+- Added support for 'omp fuse' directive.
 
 Improvements
 ^^^^^^^^^^^^

>From 1c054673983282e7e6afef0f11c2a7fbe39181d7 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:43:41 +0000
Subject: [PATCH 04/12] Refactored preinits handling and improved coverage

---
 clang/docs/OpenMPSupport.rst          |    2 +-
 clang/include/clang/AST/StmtOpenMP.h  |    5 +-
 clang/include/clang/Sema/SemaOpenMP.h |   96 +-
 clang/lib/AST/StmtOpenMP.cpp          |   13 +
 clang/lib/Basic/OpenMPKinds.cpp       |    3 +-
 clang/lib/CodeGen/CGExpr.cpp          |    2 +
 clang/lib/CodeGen/CodeGenFunction.h   |    4 +
 clang/lib/Sema/SemaOpenMP.cpp         |  588 ++++---
 clang/test/OpenMP/fuse_ast_print.cpp  |   55 +
 clang/test/OpenMP/fuse_codegen.cpp    | 2117 +++++++++++++++----------
 10 files changed, 1862 insertions(+), 1023 deletions(-)

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 12325e3294a64..8d62208e55f75 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -376,7 +376,7 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop stripe transformation                                  | :good:`done`              | https://github.com/llvm/llvm-project/pull/119891                                                     |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop fuse transformation                                    | :good:`done`              | :none:`unclaimed`        |                                                                          |
+| loop fuse transformation                                    | :good:`prototyped`        | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | work distribute construct                                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index f5115afd0753e..6425f6616a558 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1005,8 +1005,7 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass ||
-           C == OMPFuseDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFuseDirectiveClass;
   }
 };
 
@@ -5653,6 +5652,8 @@ class OMPStripeDirective final : public OMPLoopTransformationDirective {
                                        llvm::omp::OMPD_stripe, StartLoc, EndLoc,
                                        NumLoops) {
     setNumGeneratedLoops(2 * NumLoops);
+    // Similar to Tile, it only generates a single top level loop nest
+    setNumGeneratedLoopNests(1);
   }
 
   void setPreInits(Stmt *PreInits) {
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 0c28aaf6ab21a..547ea95c6cd5d 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1494,16 +1494,96 @@ class SemaOpenMP : public SemaBase {
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits);
 
-  /// Analyzes and checks a loop sequence for use by a loop transformation
+  /// @brief Categories of loops encountered during semantic OpenMP loop
+  /// analysis
+  ///
+  /// This enumeration identifies the structural category of a loop or sequence
+  /// of loops analyzed in the context of OpenMP transformations and directives.
+  /// This categorization helps differentiate between original source loops
+  /// and the structures resulting from applying OpenMP loop transformations.
+  enum class OMPLoopCategory {
+
+    /// @var OMPLoopCategory::RegularLoop
+    /// Represents a standard canonical loop nest found in the
+    /// original source code or an intact loop after transformations
+    /// (i.e Post/Pre loops of a loopranged fusion)
+    RegularLoop,
+
+    /// @var OMPLoopCategory::TransformSingleLoop
+    /// Represents the resulting loop structure when an OpenMP loop
+    //  transformation, generates a single, top-level loop
+    TransformSingleLoop,
+
+    /// @var OMPLoopCategory::TransformLoopSequence
+    /// Represents the resulting loop structure when an OpenMP loop
+    /// transformation
+    /// generates a sequence of two or more canonical loop nests
+    TransformLoopSequence
+  };
+
+  /// The main recursive process of `checkTransformableLoopSequence` that
+  /// performs grammatical parsing of a canonical loop sequence. It extracts
+  /// key information, such as the number of top-level loops, loop statements,
+  /// helper expressions, and other relevant loop-related data, all in a single
+  /// execution to avoid redundant traversals. This analysis flattens inner
+  /// Loop Sequences
+  ///
+  /// \param LoopSeqStmt    The AST of the original statement.
+  /// \param LoopSeqSize    [out] Number of top level canonical loops.
+  /// \param NumLoops       [out] Number of total canonical loops (nested too).
+  /// \param LoopHelpers    [out] The multiple loop analyses results.
+  /// \param ForStmts       [out] The multiple Stmt of each For loop.
+  /// \param OriginalInits  [out] The raw original initialization statements
+  ///                       of each belonging to a loop of the loop sequence
+  /// \param TransformPreInits [out] The multiple collection of statements and
+  ///                       declarations that must have been executed/declared
+  ///                       before entering the loop (each belonging to a
+  ///                       particular loop transformation, nullptr otherwise)
+  /// \param LoopSequencePreInits [out] Additional general collection of loop
+  ///                       transformation related statements and declarations
+  ///                       not bounded to a particular loop that must be
+  ///                       executed before entering the loop transformation
+  /// \param LoopCategories [out] A sequence of OMPLoopCategory values,
+  ///                       one for each loop or loop transformation node
+  ///                       successfully analyzed.
+  /// \param Context
+  /// \param Kind           The loop transformation directive kind.
+  /// \return Whether the original statement is both syntactically and
+  /// semantically correct according to OpenMP 6.0 canonical loop
+  /// sequence definition.
+  bool analyzeLoopSequence(
+      Stmt *LoopSeqStmt, unsigned &LoopSeqSize, unsigned &NumLoops,
+      SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+      SmallVectorImpl<Stmt *> &ForStmts,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+      SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context,
+      OpenMPDirectiveKind Kind);
+
+  /// Validates and checks whether a loop sequence can be transformed according
+  /// to the given directive, providing necessary setup and initialization
+  /// (Driver function) before recursion using `analyzeLoopSequence`.
   ///
   /// \param Kind           The loop transformation directive kind.
-  /// \param NumLoops       [out] Number of total canonical loops
-  /// \param LoopSeqSize    [out] Number of top level canonical loops
+  /// \param AStmt          The AST of the original statement
+  /// \param LoopSeqSize    [out] Number of top level canonical loops.
+  /// \param NumLoops       [out] Number of total canonical loops (nested too)
   /// \param LoopHelpers    [out] The multiple loop analyses results.
-  /// \param LoopStmts      [out] The multiple Stmt of each For loop.
-  /// \param OriginalInits  [out] The multiple collection of statements and
+  /// \param ForStmts       [out] The multiple Stmt of each For loop.
+  /// \param OriginalInits  [out] The raw original initialization statements
+  ///                       of each belonging to a loop of the loop sequence
+  /// \param TransformsPreInits [out] The multiple collection of statements and
   ///                       declarations that must have been executed/declared
-  ///                       before entering the loop.
+  ///                       before entering the loop (each belonging to a
+  ///                       particular loop transformation, nullptr otherwise)
+  /// \param LoopSequencePreInits [out] Additional general collection of loop
+  ///                       transformation related statements and declarations
+  ///                       not bounded to a particular loop that must be
+  ///                       executed before entering the loop transformation
+  /// \param LoopCategories [out] A sequence of OMPLoopCategory values,
+  ///                       one for each loop or loop transformation node
+  ///                       successfully analyzed.
   /// \param Context
   /// \return Whether there was an absence of errors or not
   bool checkTransformableLoopSequence(
@@ -1512,7 +1592,9 @@ class SemaOpenMP : public SemaBase {
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       SmallVectorImpl<Stmt *> &ForStmts,
       SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-      ASTContext &Context);
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+      SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context);
 
   /// Helper to keep information about the current `omp begin/end declare
   /// variant` nesting.
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index c5a6732cc2217..f527e6361b5e5 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -457,6 +457,8 @@ OMPUnrollDirective::Create(const ASTContext &C, SourceLocation StartLoc,
       C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc);
   Dir->setNumGeneratedLoops(NumGeneratedLoops);
   // The number of generated loops and loop nests during unroll matches
+  // given that unroll only generates top level canonical loop nests
+  // so each generated loop is a top level canonical loop nest
   Dir->setNumGeneratedLoopNests(NumGeneratedLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
@@ -520,6 +522,17 @@ OMPFuseDirective *OMPFuseDirective::Create(
       NumLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
+  // The number of top level canonical nests could 
+  // not match the total number of generated loops
+  // Example:
+  // Before fusion:
+  //   for (int i = 0; i < N; ++i)   
+  //     for (int j = 0; j < M; ++j) 
+  //       A[i][j] = i + j;
+  //   
+  //   for (int k = 0; k < P; ++k) 
+  //     B[k] = k * 2;
+  // Here, NumLoopNests = 2, but NumLoops = 3.
   Dir->setNumGeneratedLoopNests(NumLoopNests);
   Dir->setNumGeneratedLoops(NumLoops);
   return Dir;
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 18330181f1509..53a9f80e6d3b7 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -704,7 +704,8 @@ bool clang::isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind) {
 
 bool clang::isOpenMPLoopTransformationDirective(OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
-         DKind == OMPD_interchange || DKind == OMPD_stripe || DKind == OMPD_fuse;
+         DKind == OMPD_interchange || DKind == OMPD_stripe ||
+         DKind == OMPD_fuse;
 }
 
 bool clang::isOpenMPCombinedParallelADirective(OpenMPDirectiveKind DKind) {
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 6cb348ffdf55f..08049d4d4e37d 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3257,6 +3257,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
 
     // No other cases for now.
     } else {
+      llvm::dbgs() << "THE DAMN DECLREFEXPR HASN'T BEEN ENTERED IN LOCALDECLMAP\n";
+      VD->dumpColor();
       llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
     }
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index fe753e5b688b1..bfe24213ed377 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5363,6 +5363,10 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// Set the address of a local variable.
   void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
+    if (LocalDeclMap.count(VD)) {
+      llvm::errs() << "Warning: VarDecl already exists in map: ";
+      VD->dumpColor(); 
+    }
     assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
     LocalDeclMap.insert({VD, Addr});
   }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 3ec3f2ad31e78..3ce256f3ec23b 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DynamicRecursiveASTVisitor.h"
 #include "clang/AST/OpenMPClause.h"
+#include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/StmtVisitor.h"
@@ -47,6 +48,7 @@
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/IR/Assumptions.h"
 #include <optional>
+#include <queue>
 
 using namespace clang;
 using namespace llvm::omp;
@@ -14194,6 +14196,45 @@ StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
       getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
+// Overloaded base case function
+template <typename T, typename F>
+static bool tryHandleAs(T *t, F &&) {
+    return false; 
+}
+
+/**
+ * Tries to recursively cast `t` to one of the given types and invokes `f` if successful.
+ *
+ * @tparam Class The first type to check.
+ * @tparam Rest The remaining types to check.
+ * @tparam T The base type of `t`.
+ * @tparam F The callable type for the function to invoke upon a successful cast.
+ * @param t The object to be checked.
+ * @param f The function to invoke if `t` matches `Class`.
+ * @return `true` if `t` matched any type and `f` was called, otherwise `false`.
+ */
+template <typename Class, typename... Rest, typename T, typename F>
+static bool tryHandleAs(T *t, F &&f) {
+    if (Class *c = dyn_cast<Class>(t)) {
+        f(c); 
+        return true;
+    } else {
+        return tryHandleAs<Rest...>(t, std::forward<F>(f));
+    }
+}
+
+// Updates OriginalInits by checking Transform against loop transformation
+// directives and appending their pre-inits if a match is found.
+static void updatePreInits(OMPLoopBasedDirective *Transform,
+                           SmallVectorImpl<SmallVector<Stmt *, 0>> &PreInits) {
+    if (!tryHandleAs<OMPTileDirective, OMPUnrollDirective, OMPReverseDirective,
+                     OMPInterchangeDirective, OMPFuseDirective>(
+            Transform, [&PreInits](auto *Dir) {
+              appendFlattenedStmtList(PreInits.back(), Dir->getPreInits());
+            }))
+        llvm_unreachable("Unhandled loop transformation");
+}
+
 bool SemaOpenMP::checkTransformableLoopNest(
     OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
@@ -14224,121 +14265,106 @@ bool SemaOpenMP::checkTransformableLoopNest(
         return false;
       },
       [&OriginalInits](OMPLoopBasedDirective *Transform) {
-        Stmt *DependentPreInits;
-        if (auto *Dir = dyn_cast<OMPTileDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPStripeDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPUnrollDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPReverseDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPFuseDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else
-          llvm_unreachable("Unhandled loop transformation");
-
-        appendFlattenedStmtList(OriginalInits.back(), DependentPreInits);
+        updatePreInits(Transform, OriginalInits);
       });
   assert(OriginalInits.back().empty() && "No preinit after innermost loop");
   OriginalInits.pop_back();
   return Result;
 }
 
-class NestedLoopCounterVisitor
-    : public clang::RecursiveASTVisitor<NestedLoopCounterVisitor> {
+// Counts the total number of nested loops, including the outermost loop (the
+// original loop). PRECONDITION of this visitor is that it must be invoked from
+// the original loop to be analyzed. The traversal is stop for Decl's and
+// Expr's given that they may contain inner loops that must not be counted.
+//
+// Example AST structure for the code:
+//
+// int main() {
+//     #pragma omp fuse
+//     {
+//         for (int i = 0; i < 100; i++) {    <-- Outer loop
+//             []() {
+//                 for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
+//             };
+//             for(int j = 0; j < 5; ++j) {}    <-- Inner loop
+//         }
+//         for (int r = 0; i < 100; i++) {    <-- Outer loop
+//             struct LocalClass {
+//                 void bar() {
+//                     for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
+//                 }
+//             };
+//             for(int k = 0; k < 10; ++k) {}    <-- Inner loop
+//             {x = 5; for(k = 0; k < 10; ++k) x += k; x}; <-- NOT A LOOP
+//         }
+//     }
+// }
+// Result: Loop 'i' contains 2 loops, Loop 'r' also contains 2 loops
+class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
+private:
+  unsigned NestedLoopCount = 0;
+
 public:
-  explicit NestedLoopCounterVisitor() : NestedLoopCount(0) {}
+  explicit NestedLoopCounterVisitor() {}
 
-  bool VisitForStmt(clang::ForStmt *FS) {
-    ++NestedLoopCount;
-    return true;
+  unsigned getNestedLoopCount() const { return NestedLoopCount; }
+
+  bool VisitForStmt(ForStmt *FS) override {
+        ++NestedLoopCount;
+        return true;
   }
 
-  bool VisitCXXForRangeStmt(clang::CXXForRangeStmt *FRS) {
-    ++NestedLoopCount;
-    return true;
+  bool VisitCXXForRangeStmt(CXXForRangeStmt *FRS) override {
+        ++NestedLoopCount;
+        return true;
   }
 
-  unsigned getNestedLoopCount() const { return NestedLoopCount; }
+  bool TraverseStmt(Stmt *S) override {
+        if (!S)
+      return true;
 
-private:
-  unsigned NestedLoopCount;
+        // Skip traversal of all expressions, including special cases like
+        // LambdaExpr, StmtExpr, BlockExpr, and RequiresExpr. These expressions
+        // may contain inner statements (and even loops), but they are not part
+        // of the syntactic body of the surrounding loop structure.
+        //  Therefore must not be counted
+        if (isa<Expr>(S))
+      return true;
+
+        // Only recurse into CompoundStmt (block {}) and loop bodies
+        if (isa<CompoundStmt>(S) || isa<ForStmt>(S) ||
+            isa<CXXForRangeStmt>(S)) {
+      return DynamicRecursiveASTVisitor::TraverseStmt(S);
+        }
+
+        // Stop traversal of the rest of statements, that break perfect
+        // loop nesting, such as control flow (IfStmt, SwitchStmt...)
+        return true;
+  }
+
+  bool TraverseDecl(Decl *D) override {
+        // Stop in the case of finding a declaration, it is not important
+        // in order to find nested loops (Possible CXXRecordDecl, RecordDecl,
+        // FunctionDecl...)
+        return true;
+  }
 };
 
-bool SemaOpenMP::checkTransformableLoopSequence(
-    OpenMPDirectiveKind Kind, Stmt *AStmt, unsigned &LoopSeqSize,
-    unsigned &NumLoops,
+bool SemaOpenMP::analyzeLoopSequence(
+    Stmt *LoopSeqStmt, unsigned &LoopSeqSize, unsigned &NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
     SmallVectorImpl<Stmt *> &ForStmts,
     SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-    ASTContext &Context) {
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+    SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context,
+    OpenMPDirectiveKind Kind) {
 
-  // Checks whether the given statement is a compound statement
   VarsWithInheritedDSAType TmpDSA;
-  if (!isa<CompoundStmt>(AStmt)) {
-    Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
-        << getOpenMPDirectiveName(Kind);
-    return false;
-  }
-  // Callback for updating pre-inits in case there are even more
-  // loop-sequence-generating-constructs inside of the main compound stmt
-  auto OnTransformationCallback =
-      [&OriginalInits](OMPLoopBasedDirective *Transform) {
-        Stmt *DependentPreInits;
-        if (auto *Dir = dyn_cast<OMPTileDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPUnrollDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPReverseDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPFuseDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else
-          llvm_unreachable("Unhandled loop transformation");
-
-        appendFlattenedStmtList(OriginalInits.back(), DependentPreInits);
-      };
-
-  // Number of top level canonical loop nests observed (And acts as index)
-  LoopSeqSize = 0;
-  // Number of total observed loops
-  NumLoops = 0;
-
-  // Following OpenMP 6.0 API Specification, a Canonical Loop Sequence follows
-  // the grammar:
-  //
-  // canonical-loop-sequence:
-  //  {
-  //    loop-sequence+
-  //  }
-  // where loop-sequence can be any of the following:
-  // 1. canonical-loop-sequence
-  // 2. loop-nest
-  // 3. loop-sequence-generating-construct (i.e OMPLoopTransformationDirective)
-  //
-  // To recognise and traverse this structure the following helper functions
-  // have been defined. handleLoopSequence serves as the recurisve entry point
-  // and tries to match the input AST to the canonical loop sequence grammar
-  // structure
-
-  // Helper functions to validate canonical loop sequence grammar is valid
-  auto isLoopSequenceDerivation = [](auto *Child) {
-    return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
-           isa<OMPLoopTransformationDirective>(Child);
-  };
-  auto isLoopGeneratingStmt = [](auto *Child) {
-    return isa<OMPLoopTransformationDirective>(Child);
-  };
-
+  QualType BaseInductionVarType;
   // Helper Lambda to handle storing initialization and body statements for both
   // ForStmt and CXXForRangeStmt and checks for any possible mismatch between
   // induction variables types
-  QualType BaseInductionVarType;
   auto storeLoopStatements = [&OriginalInits, &ForStmts, &BaseInductionVarType,
                               this, &Context](Stmt *LoopStmt) {
     if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
@@ -14361,33 +14387,35 @@ bool SemaOpenMP::checkTransformableLoopSequence(
           }
         }
       }
-
     } else {
-      assert(isa<CXXForRangeStmt>(LoopStmt) &&
-             "Expected canonical for or range-based for loops.");
-      auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt);
+      auto *CXXFor = cast<CXXForRangeStmt>(LoopStmt);
       OriginalInits.back().push_back(CXXFor->getBeginStmt());
       ForStmts.push_back(CXXFor);
     }
   };
+
   // Helper lambda functions to encapsulate the processing of different
   // derivations of the canonical loop sequence grammar
   //
   // Modularized code for handling loop generation and transformations
-  auto handleLoopGeneration = [&storeLoopStatements, &LoopHelpers,
-                               &OriginalInits, &LoopSeqSize, &NumLoops, Kind,
-                               &TmpDSA, &OnTransformationCallback,
-                               this](Stmt *Child) {
+  auto analyzeLoopGeneration = [&storeLoopStatements, &LoopHelpers,
+                                &OriginalInits, &TransformsPreInits,
+                                &LoopCategories, &LoopSeqSize, &NumLoops, Kind,
+                                &TmpDSA, &ForStmts, &Context,
+                                &LoopSequencePreInits, this](Stmt *Child) {
     auto LoopTransform = dyn_cast<OMPLoopTransformationDirective>(Child);
     Stmt *TransformedStmt = LoopTransform->getTransformedStmt();
     unsigned NumGeneratedLoopNests = LoopTransform->getNumGeneratedLoopNests();
-
+    unsigned NumGeneratedLoops = LoopTransform->getNumGeneratedLoops();
     // Handle the case where transformed statement is not available due to
     // dependent contexts
     if (!TransformedStmt) {
-      if (NumGeneratedLoopNests > 0)
+      if (NumGeneratedLoopNests > 0) {
+        LoopSeqSize += NumGeneratedLoopNests;
+        NumLoops += NumGeneratedLoops;
         return true;
-      // Unroll full
+      }
+      // Unroll full (0 loops produced)
       else {
         Diag(Child->getBeginLoc(), diag::err_omp_not_for)
             << 0 << getOpenMPDirectiveName(Kind);
@@ -14400,38 +14428,56 @@ bool SemaOpenMP::checkTransformableLoopSequence(
       Diag(Child->getBeginLoc(), diag::err_omp_not_for)
           << 0 << getOpenMPDirectiveName(Kind);
       return false;
-      // Future loop transformations that generate multiple canonical loops
-    } else if (NumGeneratedLoopNests > 1) {
-      llvm_unreachable("Multiple canonical loop generating transformations "
-                       "like loop splitting are not yet supported");
     }
+    // Loop transformatons such as split or loopranged fuse
+    else if (NumGeneratedLoopNests > 1) {
+      // Get the preinits related to this loop sequence generating
+      // loop transformation (i.e loopranged fuse, split...)
+      LoopSequencePreInits.emplace_back();
+      // These preinits differ slightly from regular inits/pre-inits related
+      // to single loop generating loop transformations (interchange, unroll)
+      // given that they are not bounded to a particular loop nest
+      // so they need to be treated independently
+      updatePreInits(LoopTransform, LoopSequencePreInits);
+      return analyzeLoopSequence(TransformedStmt, LoopSeqSize, NumLoops,
+                                 LoopHelpers, ForStmts, OriginalInits,
+                                 TransformsPreInits, LoopSequencePreInits,
+                                 LoopCategories, Context, Kind);
+    }
+    // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
+    else {
+      // Process the transformed loop statement
+      OriginalInits.emplace_back();
+      TransformsPreInits.emplace_back();
+      LoopHelpers.emplace_back();
+      LoopCategories.push_back(OMPLoopCategory::TransformSingleLoop);
+
+      unsigned IsCanonical =
+          checkOpenMPLoop(Kind, nullptr, nullptr, TransformedStmt, SemaRef,
+                          *DSAStack, TmpDSA, LoopHelpers[LoopSeqSize]);
+
+      if (!IsCanonical) {
+        Diag(TransformedStmt->getBeginLoc(), diag::err_omp_not_canonical_loop)
+            << getOpenMPDirectiveName(Kind);
+        return false;
+      }
+      storeLoopStatements(TransformedStmt);
+      updatePreInits(LoopTransform, TransformsPreInits);
 
-    // Process the transformed loop statement
-    Child = TransformedStmt;
-    OriginalInits.emplace_back();
-    LoopHelpers.emplace_back();
-    OnTransformationCallback(LoopTransform);
-
-    unsigned IsCanonical =
-        checkOpenMPLoop(Kind, nullptr, nullptr, Child, SemaRef, *DSAStack,
-                        TmpDSA, LoopHelpers[LoopSeqSize]);
-
-    if (!IsCanonical) {
-      Diag(Child->getBeginLoc(), diag::err_omp_not_canonical_loop)
-          << getOpenMPDirectiveName(Kind);
-      return false;
+      NumLoops += NumGeneratedLoops;
+      ++LoopSeqSize;
+      return true;
     }
-    storeLoopStatements(TransformedStmt);
-    NumLoops += LoopTransform->getNumGeneratedLoops();
-    return true;
   };
 
   // Modularized code for handling regular canonical loops
-  auto handleRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
-                            &LoopSeqSize, &NumLoops, Kind, &TmpDSA,
-                            this](Stmt *Child) {
+  auto analyzeRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
+                             &LoopSeqSize, &NumLoops, Kind, &TmpDSA,
+                             &LoopCategories, this](Stmt *Child) {
     OriginalInits.emplace_back();
     LoopHelpers.emplace_back();
+    LoopCategories.push_back(OMPLoopCategory::RegularLoop);
+
     unsigned IsCanonical =
         checkOpenMPLoop(Kind, nullptr, nullptr, Child, SemaRef, *DSAStack,
                         TmpDSA, LoopHelpers[LoopSeqSize]);
@@ -14449,57 +14495,114 @@ bool SemaOpenMP::checkTransformableLoopSequence(
     return true;
   };
 
-  // Helper function to process a Loop Sequence Recursively
-  auto handleLoopSequence = [&](Stmt *LoopSeqStmt,
-                                auto &handleLoopSequenceCallback) -> bool {
-    for (auto *Child : LoopSeqStmt->children()) {
-      if (!Child)
-        continue;
+  // Helper functions to validate canonical loop sequence grammar is valid
+  auto isLoopSequenceDerivation = [](auto *Child) {
+    return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
+           isa<OMPLoopTransformationDirective>(Child);
+  };
+  auto isLoopGeneratingStmt = [](auto *Child) {
+    return isa<OMPLoopTransformationDirective>(Child);
+  };
+
 
-      // Skip over non-loop-sequence statements
-      if (!isLoopSequenceDerivation(Child)) {
-        Child = Child->IgnoreContainers();
+  // High level grammar validation
+  for (auto *Child : LoopSeqStmt->children()) {
 
-        // Ignore empty compound statement
         if (!Child)
-          continue;
+      continue;
 
-        // In the case of a nested loop sequence ignoring containers would not
-        // be enough, a recurisve transversal of the loop sequence is required
-        if (isa<CompoundStmt>(Child)) {
-          if (!handleLoopSequenceCallback(Child, handleLoopSequenceCallback))
-            return false;
-          // Already been treated, skip this children
-          continue;
+        // Skip over non-loop-sequence statements
+        if (!isLoopSequenceDerivation(Child)) {
+      Child = Child->IgnoreContainers();
+
+      // Ignore empty compound statement
+      if (!Child)
+        continue;
+
+      // In the case of a nested loop sequence ignoring containers would not
+      // be enough, a recurisve transversal of the loop sequence is required
+      if (isa<CompoundStmt>(Child)) {
+        if (!analyzeLoopSequence(Child, LoopSeqSize, NumLoops, LoopHelpers,
+                                 ForStmts, OriginalInits, TransformsPreInits,
+                                 LoopSequencePreInits, LoopCategories, Context,
+                                 Kind))
+          return false;
+        // Already been treated, skip this children
+        continue;
+      }
+        }
+        // Regular loop sequence handling
+        if (isLoopSequenceDerivation(Child)) {
+      if (isLoopGeneratingStmt(Child)) {
+        if (!analyzeLoopGeneration(Child)) {
+          return false;
         }
+        // analyzeLoopGeneration updates Loop Sequence size accordingly
+
+      } else {
+        if (!analyzeRegularLoop(Child)) {
+          return false;
+        }
+        // Update the Loop Sequence size by one
+        ++LoopSeqSize;
       }
-      // Regular loop sequence handling
-      if (isLoopSequenceDerivation(Child)) {
-        if (isLoopGeneratingStmt(Child)) {
-          if (!handleLoopGeneration(Child)) {
-            return false;
-          }
         } else {
-          if (!handleRegularLoop(Child)) {
-            return false;
-          }
+      // Report error for invalid statement inside canonical loop sequence
+      Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+          << 0 << getOpenMPDirectiveName(Kind);
+      return false;
         }
-        ++LoopSeqSize;
-      } else {
-        // Report error for invalid statement inside canonical loop sequence
-        Diag(Child->getBeginLoc(), diag::err_omp_not_for)
-            << 0 << getOpenMPDirectiveName(Kind);
+  }
+  return true;
+}
+
+bool SemaOpenMP::checkTransformableLoopSequence(
+    OpenMPDirectiveKind Kind, Stmt *AStmt, unsigned &LoopSeqSize,
+    unsigned &NumLoops,
+    SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+    SmallVectorImpl<Stmt *> &ForStmts,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+    SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context) {
+
+  // Checks whether the given statement is a compound statement
+  if (!isa<CompoundStmt>(AStmt)) {
+        Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
+            << getOpenMPDirectiveName(Kind);
         return false;
-      }
-    }
-    return true;
-  };
+  }
+  // Number of top level canonical loop nests observed (And acts as index)
+  LoopSeqSize = 0;
+  // Number of total observed loops
+  NumLoops = 0;
+
+  // Following OpenMP 6.0 API Specification, a Canonical Loop Sequence follows
+  // the grammar:
+  //
+  // canonical-loop-sequence:
+  //  {
+  //    loop-sequence+
+  //  }
+  // where loop-sequence can be any of the following:
+  // 1. canonical-loop-sequence
+  // 2. loop-nest
+  // 3. loop-sequence-generating-construct (i.e OMPLoopTransformationDirective)
+  //
+  // To recognise and traverse this structure the following helper functions
+  // have been defined. analyzeLoopSequence serves as the recurisve entry point
+  // and tries to match the input AST to the canonical loop sequence grammar
+  // structure. This function will perform both a semantic and syntactical
+  // analysis of the given statement according to OpenMP 6.0 definition of
+  // the aforementioned canonical loop sequence
 
   // Recursive entry point to process the main loop sequence
-  if (!handleLoopSequence(AStmt, handleLoopSequence)) {
-    return false;
+  if (!analyzeLoopSequence(AStmt, LoopSeqSize, NumLoops, LoopHelpers, ForStmts,
+                           OriginalInits, TransformsPreInits,
+                           LoopSequencePreInits, LoopCategories, Context,
+                           Kind)) {
+        return false;
   }
-
   if (LoopSeqSize <= 0) {
     Diag(AStmt->getBeginLoc(), diag::err_omp_empty_loop_sequence)
         << getOpenMPDirectiveName(Kind);
@@ -14531,9 +14634,7 @@ static void addLoopPreInits(ASTContext &Context,
                                               RangeEnd->getBeginLoc(),
                                               RangeEnd->getEndLoc()));
   }
-
   llvm::append_range(PreInits, OriginalInit);
-
   // List of OMPCapturedExprDecl, for __begin, __end, and NumIterations
   if (auto *PI = cast_or_null<DeclStmt>(LoopHelper.PreInits)) {
     PreInits.push_back(new (Context) DeclStmt(
@@ -15214,7 +15315,7 @@ StmtResult SemaOpenMP::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   Stmt *LoopStmt = nullptr;
   collectLoopStmts(AStmt, {LoopStmt});
 
-  // Determine the PreInit declarations.
+  // Determine the PreInit declarations.e
   SmallVector<Stmt *, 4> PreInits;
   addLoopPreInits(Context, LoopHelper, LoopStmt, OriginalInits[0], PreInits);
 
@@ -15781,28 +15882,35 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   if (!AStmt) {
     return StmtError();
   }
+
+  unsigned NumLoops = 1;
+  unsigned LoopSeqSize = 1;
+
+  // Defer transformation in dependent contexts
+  // The NumLoopNests argument is set to a placeholder 1 (even though
+  // using looprange fuse could yield up to 3 top level loop nests)
+  // because a dependent context could prevent determining its true value
+  if (CurrContext->isDependentContext()) {
+    return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                    NumLoops, LoopSeqSize, AStmt, nullptr,
+                                    nullptr);
+  }
+
   // Validate that the potential loop sequence is transformable for fusion
   // Also collect the HelperExprs, Loop Stmts, Inits, and Number of loops
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers;
   SmallVector<Stmt *> LoopStmts;
   SmallVector<SmallVector<Stmt *, 0>> OriginalInits;
-
-  unsigned NumLoops;
-  unsigned LoopSeqSize;
+  SmallVector<SmallVector<Stmt *, 0>> TransformsPreInits;
+  SmallVector<SmallVector<Stmt *, 0>> LoopSequencePreInits;
+  SmallVector<OMPLoopCategory, 0> LoopCategories;
   if (!checkTransformableLoopSequence(OMPD_fuse, AStmt, LoopSeqSize, NumLoops,
                                       LoopHelpers, LoopStmts, OriginalInits,
-                                      Context)) {
+                                      TransformsPreInits, LoopSequencePreInits,
+                                      LoopCategories, Context)) {
     return StmtError();
   }
 
-  // Defer transformation in dependent contexts
-  // The NumLoopNests argument is set to a placeholder (0)
-  // because a dependent context could prevent determining its true value
-  if (CurrContext->isDependentContext()) {
-    return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                    NumLoops, 0, AStmt, nullptr, nullptr);
-  }
-
   // Handle clauses, which can be any of the following: [looprange, apply]
   const OMPLoopRangeClause *LRC =
       OMPExecutableDirective::getSingleClause<OMPLoopRangeClause>(Clauses);
@@ -15864,11 +15972,6 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
          "Expecting loop iteration space dimensionality to match number of "
          "affected loops");
 
-  // PreInits hold a sequence of variable declarations that must be executed
-  // before the fused loop begins. These include bounds, strides, and other
-  // helper variables required for the transformation.
-  SmallVector<Stmt *> PreInits;
-
   // Select the type with the largest bit width among all induction variables
   QualType IVType = LoopHelpers[FirstVal - 1].IterationVarRef->getType();
   for (unsigned int I = FirstVal; I < LastVal; ++I) {
@@ -15880,7 +15983,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   uint64_t IVBitWidth = Context.getIntWidth(IVType);
 
   // Create pre-init declarations for all loops lower bounds, upper bounds,
-  // strides and num-iterations
+  // strides and num-iterations for every top level loop in the fusion
   SmallVector<VarDecl *, 4> LBVarDecls;
   SmallVector<VarDecl *, 4> STVarDecls;
   SmallVector<VarDecl *, 4> NIVarDecls;
@@ -15918,12 +16021,62 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
         return std::make_pair(VD, DeclStmt);
       };
 
+  // PreInits hold a sequence of variable declarations that must be executed
+  // before the fused loop begins. These include bounds, strides, and other
+  // helper variables required for the transformation. Other loop transforms
+  // also contain their own preinits
+  SmallVector<Stmt *> PreInits;
+  // Iterator to keep track of loop transformations
+  unsigned int TransformIndex = 0;
+
+  //  Update the general preinits using the preinits generated by loop sequence
+  //  generating loop transformations. These preinits differ slightly from
+  //  single-loop transformation preinits, as they can be detached from a
+  //  specific loop inside the multiple generated loop nests. This happens
+  //  because certain helper variables, like '.omp.fuse.max', are introduced to
+  //  handle fused iteration spaces and may not be directly tied to a single
+  //  original loop. the preinit structure must ensure that hidden variables
+  //  like '.omp.fuse.max' are still properly handled.
+  // Transformations that apply this concept: Loopranged Fuse, Split
+  if (!LoopSequencePreInits.empty()) {
+    for (const auto &LTPreInits : LoopSequencePreInits) {
+      if (!LTPreInits.empty()) {
+        llvm::append_range(PreInits, LTPreInits);
+      }
+    }
+  }
+
   // Process each single loop to generate and collect declarations
-  // and statements for all helper expressions
+  // and statements for all helper expressions related to
+  // particular single loop nests
+
+  // Also In the case of the fused loops, we keep track of their original
+  // inits by appending them to their preinits statement, and in the case of
+  // transformations, also append their preinits (which contain the original
+  // loop initialization statement or other statements)
+
+  // Firstly we need to update TransformIndex to match the begining of the
+  // looprange section
+  for (unsigned int I = 0; I < FirstVal - 1; ++I) {
+    if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop)
+      ++TransformIndex;
+  }
   for (unsigned int I = FirstVal - 1, J = 0; I < LastVal; ++I, ++J) {
-    addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
-                    PreInits);
 
+    if (LoopCategories[I] == OMPLoopCategory::RegularLoop) {
+      addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
+                      PreInits);
+    } else if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop) {
+      // For transformed loops, insert both pre-inits and original inits.
+      // Order matters: pre-inits may define variables used in the original
+      // inits such as upper bounds...
+      auto TransformPreInit = TransformsPreInits[TransformIndex++];
+      if (!TransformPreInit.empty()) {
+        llvm::append_range(PreInits, TransformPreInit);
+      }
+      addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
+                      PreInits);
+    }
     auto [UBVD, UBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].UB, "ub", J);
     auto [LBVD, LBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].LB, "lb", J);
     auto [STVD, STDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].ST, "st", J);
@@ -15942,7 +16095,6 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     NIVarDecls.push_back(NIVD);
     IVVarDecls.push_back(IVVD);
 
-    PreInits.push_back(UBDStmt.get());
     PreInits.push_back(LBDStmt.get());
     PreInits.push_back(STDStmt.get());
     PreInits.push_back(NIDStmt.get());
@@ -16118,6 +16270,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     BodyStmts.push_back(IdxExpr.get());
     llvm::append_range(BodyStmts, LoopHelpers[I].Updates);
 
+    // If the loop is a CXXForRangeStmt then the iterator variable is needed
     if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
       BodyStmts.push_back(SourceCXXFor->getLoopVarStmt());
 
@@ -16152,21 +16305,50 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
               FusedBody, InitStmt.get()->getBeginLoc(), SourceLocation(),
               IncrExpr.get()->getEndLoc());
 
-  // In the case of looprange, the result of fuse won't simply
-  // be a single loop (ForStmt), but rather a loop sequence
-  // (CompoundStmt) of 3 parts: the pre-fusion loops, the fused loop
-  // and the post-fusion loops, preserving its original order.
+  //  In the case of looprange, the result of fuse won't simply
+  //  be a single loop (ForStmt), but rather a loop sequence
+  //  (CompoundStmt) of 3 parts: the pre-fusion loops, the fused loop
+  //  and the post-fusion loops, preserving its original order.
+  //
+  //  Note: If looprange clause produces a single fused loop nest then
+  //  this compound statement wrapper is unnecessary (Therefore this
+  //  treatment is skipped)
+
   Stmt *FusionStmt = FusedForStmt;
-  if (LRC) {
+  if (LRC && CountVal != LoopSeqSize) {
     SmallVector<Stmt *, 4> FinalLoops;
-    // Gather all the pre-fusion loops
-    for (unsigned I = 0; I < FirstVal - 1; ++I)
-      FinalLoops.push_back(LoopStmts[I]);
-    // Gather the fused loop
-    FinalLoops.push_back(FusedForStmt);
-    // Gather all the post-fusion loops
-    for (unsigned I = FirstVal + CountVal - 1; I < LoopSeqSize; ++I)
+    // Reset the transform index
+    TransformIndex = 0;
+
+    // Collect all non-fused loops before and after the fused region.
+    // Pre-fusion and post-fusion loops are inserted in order exploiting their
+    // symmetry, along with their corresponding transformation pre-inits if
+    // needed. The fused loop is added between the two regions.
+    for (unsigned I = 0; I < LoopSeqSize; ++I) {
+      if (I >= FirstVal - 1 && I < FirstVal + CountVal - 1) {
+        // Update the Transformation counter to skip already treated
+        // loop transformations
+        if (LoopCategories[I] != OMPLoopCategory::TransformSingleLoop)
+          ++TransformIndex;
+        continue;
+      }
+
+      // No need to handle:
+      // Regular loops: they are kept intact as-is.
+      // Loop-sequence-generating transformations: already handled earlier.
+      // Only TransformSingleLoop requires inserting pre-inits here
+
+      if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop) {
+        auto TransformPreInit = TransformsPreInits[TransformIndex++];
+        if (!TransformPreInit.empty()) {
+          llvm::append_range(PreInits, TransformPreInit);
+        }
+      }
+
       FinalLoops.push_back(LoopStmts[I]);
+    }
+
+    FinalLoops.insert(FinalLoops.begin() + (FirstVal - 1), FusedForStmt);
     FusionStmt = CompoundStmt::Create(Context, FinalLoops, FPOptionsOverride(),
                                       SourceLocation(), SourceLocation());
   }
diff --git a/clang/test/OpenMP/fuse_ast_print.cpp b/clang/test/OpenMP/fuse_ast_print.cpp
index ac4f0d38a9c68..9d85bd1172948 100644
--- a/clang/test/OpenMP/fuse_ast_print.cpp
+++ b/clang/test/OpenMP/fuse_ast_print.cpp
@@ -338,6 +338,61 @@ void tfoo9() {
     foo9<1, 2>();
 }
 
+// PRINT-LABEL: void foo10(
+// DUMP-LABEL: FunctionDecl {{.*}} foo10
+void foo10() {
+    // PRINT: #pragma omp fuse looprange(2,2)
+    // DUMP:  OMPFuseDirective
+    // DUMP: OMPLooprangeClause
+    #pragma omp fuse looprange(2,2)
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int ii = 0; ii < 10; ii += 2)
+        // DUMP: ForStmt
+        for (int ii = 0; ii < 10; ii += 2)
+            // PRINT: body(ii)
+            // DUMP: CallExpr
+            body(ii);
+        // PRINT: #pragma omp fuse looprange(2,2)
+        // DUMP:  OMPFuseDirective
+        // DUMP: OMPLooprangeClause
+        #pragma omp fuse looprange(2,2)
+        {
+            // PRINT: for (int j = 10; j > 0; --j)
+            // DUMP: ForStmt
+            for (int j = 10; j > 0; --j)
+                // PRINT: body(j)
+                // DUMP: CallExpr
+                body(j);
+            // PRINT: for (int jj = 10; jj > 0; --jj)
+            // DUMP: ForStmt
+            for (int jj = 10; jj > 0; --jj)
+                // PRINT: body(jj)
+                // DUMP: CallExpr
+                body(jj);
+            // PRINT: for (int k = 0; k <= 10; ++k)
+            // DUMP: ForStmt
+            for (int k = 0; k <= 10; ++k)
+                // PRINT: body(k)
+                // DUMP: CallExpr
+                body(k);
+            // PRINT: for (int kk = 0; kk <= 10; ++kk)
+            // DUMP: ForStmt
+            for (int kk = 0; kk <= 10; ++kk)
+                // PRINT: body(kk)
+                // DUMP: CallExpr
+                body(kk);
+        }
+    }
+
+}
 
 
 
diff --git a/clang/test/OpenMP/fuse_codegen.cpp b/clang/test/OpenMP/fuse_codegen.cpp
index d9500bed3ce31..742c280ed0172 100644
--- a/clang/test/OpenMP/fuse_codegen.cpp
+++ b/clang/test/OpenMP/fuse_codegen.cpp
@@ -65,6 +65,23 @@ extern "C" void foo4() {
     }
 }
 
+// This exemplifies the usage of loop transformations that generate
+// more than top level canonical loop nests (e.g split, loopranged fuse...)
+extern "C" void foo5() {
+    double arr[256];
+    #pragma omp fuse looprange(2,2)
+    {
+        #pragma omp fuse looprange(2,2)
+        {
+            for(int i = 0; i < 128; ++i) body(i);
+            for(int j = 0; j < 256; j+=2) body(j);
+            for(int k = 0; k < 512; ++k) body(k);
+        }
+        for(int c = 42; auto &&v: arr) body(c,v);
+        for(int cc = 37; auto &&vv: arr) body(cc, vv);
+    }
+}
+
 
 #endif
 // CHECK1-LABEL: define dso_local void @body(
@@ -88,7 +105,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
@@ -97,7 +113,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -129,107 +144,103 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP8]], 1
 // CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP9]], ptr [[J]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[START2_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START2_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[END2_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP13]], [[TMP14]]
 // CHECK1-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP15]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP16]]
 // CHECK1-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
 // CHECK1-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    store i32 [[TMP20]], ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP21]], [[TMP22]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP19]], [[TMP20]]
 // CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // CHECK1:       [[COND_TRUE]]:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
 // CHECK1-NEXT:    br label %[[COND_END:.*]]
 // CHECK1:       [[COND_FALSE]]:
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
 // CHECK1-NEXT:    br label %[[COND_END]]
 // CHECK1:       [[COND_END]]:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP23]], %[[COND_TRUE]] ], [ [[TMP24]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP21]], %[[COND_TRUE]] ], [ [[TMP22]], %[[COND_FALSE]] ]
 // CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK1-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK1:       [[FOR_COND]]:
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
-// CHECK1-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP23]], [[TMP24]]
 // CHECK1-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK1:       [[FOR_BODY]]:
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
 // CHECK1-NEXT:    br i1 [[CMP17]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // CHECK1:       [[IF_THEN]]:
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP30]], [[TMP31]]
-// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP29]], [[MUL]]
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP28]], [[TMP29]]
+// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP27]], [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP33]], [[TMP34]]
-// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP32]], [[MUL19]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP31]], [[TMP32]]
+// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP30]], [[MUL19]]
 // CHECK1-NEXT:    store i32 [[ADD20]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP35]])
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP33]])
 // CHECK1-NEXT:    br label %[[IF_END]]
 // CHECK1:       [[IF_END]]:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP34]], [[TMP35]]
 // CHECK1-NEXT:    br i1 [[CMP21]], label %[[IF_THEN22:.*]], label %[[IF_END27:.*]]
 // CHECK1:       [[IF_THEN22]]:
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP39]], [[TMP40]]
-// CHECK1-NEXT:    [[ADD24:%.*]] = add i32 [[TMP38]], [[MUL23]]
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// CHECK1-NEXT:    [[ADD24:%.*]] = add i32 [[TMP36]], [[MUL23]]
 // CHECK1-NEXT:    store i32 [[ADD24]], ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP42]], [[TMP43]]
-// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[TMP41]], [[MUL25]]
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP40]], [[TMP41]]
+// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[TMP39]], [[MUL25]]
 // CHECK1-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP42]])
 // CHECK1-NEXT:    br label %[[IF_END27]]
 // CHECK1:       [[IF_END27]]:
 // CHECK1-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK1:       [[FOR_INC]]:
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP45]], 1
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
 // CHECK1:       [[FOR_END]]:
@@ -256,7 +267,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
@@ -265,7 +275,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -274,7 +283,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP21:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i32, align 4
@@ -304,172 +312,166 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP8]], 1
 // CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP9]], ptr [[J]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP13]], [[TMP14]]
 // CHECK1-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP15]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP16]]
 // CHECK1-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
 // CHECK1-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
 // CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
+// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
 // CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
 // CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTCAPTURE_EXPR_19]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[DOTNEW_STEP21]], align 4
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP24]], ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP25]], [[TMP26]]
 // CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP29]]
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP30]]
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP27]]
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP28]]
 // CHECK1-NEXT:    [[SUB27:%.*]] = sub i32 [[DIV26]], 1
 // CHECK1-NEXT:    store i32 [[SUB27]], ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK1-NEXT:    store i32 [[TMP31]], ptr [[DOTOMP_UB2]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB2]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST2]], align 4
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK1-NEXT:    [[ADD28:%.*]] = add i32 [[TMP32]], 1
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK1-NEXT:    [[ADD28:%.*]] = add i32 [[TMP29]], 1
 // CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_NI2]], align 4
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    store i32 [[TMP33]], ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP34]], [[TMP35]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP30]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP31]], [[TMP32]]
 // CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // CHECK1:       [[COND_TRUE]]:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
 // CHECK1-NEXT:    br label %[[COND_END:.*]]
 // CHECK1:       [[COND_FALSE]]:
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
 // CHECK1-NEXT:    br label %[[COND_END]]
 // CHECK1:       [[COND_END]]:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP36]], %[[COND_TRUE]] ], [ [[TMP37]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP33]], %[[COND_TRUE]] ], [ [[TMP34]], %[[COND_FALSE]] ]
 // CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_TEMP_2]], align 4
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
-// CHECK1-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP35]], [[TMP36]]
 // CHECK1-NEXT:    br i1 [[CMP29]], label %[[COND_TRUE30:.*]], label %[[COND_FALSE31:.*]]
 // CHECK1:       [[COND_TRUE30]]:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
 // CHECK1-NEXT:    br label %[[COND_END32:.*]]
 // CHECK1:       [[COND_FALSE31]]:
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
 // CHECK1-NEXT:    br label %[[COND_END32]]
 // CHECK1:       [[COND_END32]]:
-// CHECK1-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP40]], %[[COND_TRUE30]] ], [ [[TMP41]], %[[COND_FALSE31]] ]
+// CHECK1-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP37]], %[[COND_TRUE30]] ], [ [[TMP38]], %[[COND_FALSE31]] ]
 // CHECK1-NEXT:    store i32 [[COND33]], ptr [[DOTOMP_FUSE_MAX]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK1-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK1:       [[FOR_COND]]:
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
-// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP39]], [[TMP40]]
 // CHECK1-NEXT:    br i1 [[CMP34]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK1:       [[FOR_BODY]]:
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP44]], [[TMP45]]
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP41]], [[TMP42]]
 // CHECK1-NEXT:    br i1 [[CMP35]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // CHECK1:       [[IF_THEN]]:
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP47]], [[TMP48]]
-// CHECK1-NEXT:    [[ADD36:%.*]] = add i32 [[TMP46]], [[MUL]]
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP44]], [[TMP45]]
+// CHECK1-NEXT:    [[ADD36:%.*]] = add i32 [[TMP43]], [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD36]], ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP50]], [[TMP51]]
-// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[TMP49]], [[MUL37]]
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP47]], [[TMP48]]
+// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[TMP46]], [[MUL37]]
 // CHECK1-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP49]])
 // CHECK1-NEXT:    br label %[[IF_END]]
 // CHECK1:       [[IF_END]]:
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP53]], [[TMP54]]
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP50]], [[TMP51]]
 // CHECK1-NEXT:    br i1 [[CMP39]], label %[[IF_THEN40:.*]], label %[[IF_END45:.*]]
 // CHECK1:       [[IF_THEN40]]:
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP56]], [[TMP57]]
-// CHECK1-NEXT:    [[ADD42:%.*]] = add i32 [[TMP55]], [[MUL41]]
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP53]], [[TMP54]]
+// CHECK1-NEXT:    [[ADD42:%.*]] = add i32 [[TMP52]], [[MUL41]]
 // CHECK1-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP59]], [[TMP60]]
-// CHECK1-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP58]], [[MUL43]]
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP55]], [[MUL43]]
 // CHECK1-NEXT:    store i32 [[SUB44]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP61]])
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP58]])
 // CHECK1-NEXT:    br label %[[IF_END45]]
 // CHECK1:       [[IF_END45]]:
-// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
-// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP62]], [[TMP63]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP59]], [[TMP60]]
 // CHECK1-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
 // CHECK1:       [[IF_THEN47]]:
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP65]], [[TMP66]]
-// CHECK1-NEXT:    [[ADD49:%.*]] = add i32 [[TMP64]], [[MUL48]]
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP62]], [[TMP63]]
+// CHECK1-NEXT:    [[ADD49:%.*]] = add i32 [[TMP61]], [[MUL48]]
 // CHECK1-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV2]], align 4
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP68]], [[TMP69]]
-// CHECK1-NEXT:    [[ADD51:%.*]] = add i32 [[TMP67]], [[MUL50]]
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP65]], [[TMP66]]
+// CHECK1-NEXT:    [[ADD51:%.*]] = add i32 [[TMP64]], [[MUL50]]
 // CHECK1-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP70]])
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP67]])
 // CHECK1-NEXT:    br label %[[IF_END52]]
 // CHECK1:       [[IF_END52]]:
 // CHECK1-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK1:       [[FOR_INC]]:
-// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP68]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
 // CHECK1:       [[FOR_END]]:
@@ -481,13 +483,11 @@ extern "C" void foo4() {
 // CHECK1-NEXT:  [[ENTRY:.*:]]
 // CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -497,48 +497,43 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB03:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_LB04:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_ST05:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_NI06:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_IV07:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB03:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST04:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI05:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV06:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[C:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_UB117:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_LB118:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_ST119:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_NI120:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_IV122:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_10:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB116:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_ST117:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_NI118:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV120:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[CC:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[__RANGE223:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[__END224:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[__BEGIN227:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__RANGE221:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END222:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN225:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_27:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_29:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_31:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_32:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_30:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_TEMP_142:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_TEMP_140:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX48:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX54:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX46:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX52:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[VV:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
 // CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
-// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB1]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
 // CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI1]], align 4
@@ -565,225 +560,219 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
 // CHECK1-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
 // CHECK1-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB03]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST04]], align 4
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTOMP_UB03]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB04]], align 4
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST05]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], 1
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
 // CHECK1-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
-// CHECK1-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI06]], align 8
+// CHECK1-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI05]], align 8
 // CHECK1-NEXT:    store i32 42, ptr [[C]], align 4
 // CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP8]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY7:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY7]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY8:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYDECAY8]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY10:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP11]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYDECAY10]], ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__END2]], align 8
-// CHECK1-NEXT:    store ptr [[TMP12]], ptr [[DOTCAPTURE_EXPR_11]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_11]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
-// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP14]] to i64
+// CHECK1-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY9]], ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP11]], ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
 // CHECK1-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
 // CHECK1-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
-// CHECK1-NEXT:    [[SUB13:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
-// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i64 [[SUB13]], 1
-// CHECK1-NEXT:    [[DIV15:%.*]] = sdiv i64 [[ADD14]], 1
-// CHECK1-NEXT:    [[SUB16:%.*]] = sub nsw i64 [[DIV15]], 1
-// CHECK1-NEXT:    store i64 [[SUB16]], ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK1-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_UB117]], align 8
-// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB118]], align 8
-// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST119]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK1-NEXT:    store i64 [[ADD21]], ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    [[SUB12:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[SUB12]], 1
+// CHECK1-NEXT:    [[DIV14:%.*]] = sdiv i64 [[ADD13]], 1
+// CHECK1-NEXT:    [[SUB15:%.*]] = sub nsw i64 [[DIV14]], 1
+// CHECK1-NEXT:    store i64 [[SUB15]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB116]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST117]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK1-NEXT:    store i64 [[ADD19]], ptr [[DOTOMP_NI118]], align 8
 // CHECK1-NEXT:    store i32 37, ptr [[CC]], align 4
-// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE223]], align 8
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY25:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
-// CHECK1-NEXT:    [[ADD_PTR26:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY25]], i64 256
-// CHECK1-NEXT:    store ptr [[ADD_PTR26]], ptr [[__END224]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP18]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[__BEGIN227]], align 8
-// CHECK1-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY30:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP19]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYDECAY30]], ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK1-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[__END224]], align 8
-// CHECK1-NEXT:    store ptr [[TMP20]], ptr [[DOTCAPTURE_EXPR_31]], align 8
-// CHECK1-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_31]], align 8
-// CHECK1-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST33:%.*]] = ptrtoint ptr [[TMP21]] to i64
-// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST34:%.*]] = ptrtoint ptr [[TMP22]] to i64
-// CHECK1-NEXT:    [[SUB_PTR_SUB35:%.*]] = sub i64 [[SUB_PTR_LHS_CAST33]], [[SUB_PTR_RHS_CAST34]]
-// CHECK1-NEXT:    [[SUB_PTR_DIV36:%.*]] = sdiv exact i64 [[SUB_PTR_SUB35]], 8
-// CHECK1-NEXT:    [[SUB37:%.*]] = sub nsw i64 [[SUB_PTR_DIV36]], 1
-// CHECK1-NEXT:    [[ADD38:%.*]] = add nsw i64 [[SUB37]], 1
-// CHECK1-NEXT:    [[DIV39:%.*]] = sdiv i64 [[ADD38]], 1
-// CHECK1-NEXT:    [[SUB40:%.*]] = sub nsw i64 [[DIV39]], 1
-// CHECK1-NEXT:    store i64 [[SUB40]], ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK1-NEXT:    store i64 [[TMP23]], ptr [[DOTOMP_UB2]], align 8
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE221]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY23:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP15]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR24:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY23]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR24]], ptr [[__END222]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY26:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP16]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY26]], ptr [[__BEGIN225]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__END222]], align 8
+// CHECK1-NEXT:    store ptr [[TMP18]], ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST31:%.*]] = ptrtoint ptr [[TMP19]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST32:%.*]] = ptrtoint ptr [[TMP20]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB33:%.*]] = sub i64 [[SUB_PTR_LHS_CAST31]], [[SUB_PTR_RHS_CAST32]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV34:%.*]] = sdiv exact i64 [[SUB_PTR_SUB33]], 8
+// CHECK1-NEXT:    [[SUB35:%.*]] = sub nsw i64 [[SUB_PTR_DIV34]], 1
+// CHECK1-NEXT:    [[ADD36:%.*]] = add nsw i64 [[SUB35]], 1
+// CHECK1-NEXT:    [[DIV37:%.*]] = sdiv i64 [[ADD36]], 1
+// CHECK1-NEXT:    [[SUB38:%.*]] = sub nsw i64 [[DIV37]], 1
+// CHECK1-NEXT:    store i64 [[SUB38]], ptr [[DOTCAPTURE_EXPR_30]], align 8
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB2]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST2]], align 8
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK1-NEXT:    [[ADD41:%.*]] = add nsw i64 [[TMP24]], 1
-// CHECK1-NEXT:    store i64 [[ADD41]], ptr [[DOTOMP_NI2]], align 8
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
-// CHECK1-NEXT:    store i64 [[TMP25]], ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK1-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP26]], [[TMP27]]
-// CHECK1-NEXT:    br i1 [[CMP43]], label %[[COND_TRUE44:.*]], label %[[COND_FALSE45:.*]]
-// CHECK1:       [[COND_TRUE44]]:
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK1-NEXT:    br label %[[COND_END46:.*]]
-// CHECK1:       [[COND_FALSE45]]:
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK1-NEXT:    br label %[[COND_END46]]
-// CHECK1:       [[COND_END46]]:
-// CHECK1-NEXT:    [[COND47:%.*]] = phi i64 [ [[TMP28]], %[[COND_TRUE44]] ], [ [[TMP29]], %[[COND_FALSE45]] ]
-// CHECK1-NEXT:    store i64 [[COND47]], ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK1-NEXT:    [[CMP49:%.*]] = icmp sgt i64 [[TMP30]], [[TMP31]]
-// CHECK1-NEXT:    br i1 [[CMP49]], label %[[COND_TRUE50:.*]], label %[[COND_FALSE51:.*]]
-// CHECK1:       [[COND_TRUE50]]:
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK1-NEXT:    br label %[[COND_END52:.*]]
-// CHECK1:       [[COND_FALSE51]]:
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK1-NEXT:    br label %[[COND_END52]]
-// CHECK1:       [[COND_END52]]:
-// CHECK1-NEXT:    [[COND53:%.*]] = phi i64 [ [[TMP32]], %[[COND_TRUE50]] ], [ [[TMP33]], %[[COND_FALSE51]] ]
-// CHECK1-NEXT:    store i64 [[COND53]], ptr [[DOTOMP_FUSE_MAX48]], align 8
-// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_30]], align 8
+// CHECK1-NEXT:    [[ADD39:%.*]] = add nsw i64 [[TMP21]], 1
+// CHECK1-NEXT:    store i64 [[ADD39]], ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    store i64 [[TMP22]], ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[CMP41:%.*]] = icmp sgt i64 [[TMP23]], [[TMP24]]
+// CHECK1-NEXT:    br i1 [[CMP41]], label %[[COND_TRUE42:.*]], label %[[COND_FALSE43:.*]]
+// CHECK1:       [[COND_TRUE42]]:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK1-NEXT:    br label %[[COND_END44:.*]]
+// CHECK1:       [[COND_FALSE43]]:
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    br label %[[COND_END44]]
+// CHECK1:       [[COND_END44]]:
+// CHECK1-NEXT:    [[COND45:%.*]] = phi i64 [ [[TMP25]], %[[COND_TRUE42]] ], [ [[TMP26]], %[[COND_FALSE43]] ]
+// CHECK1-NEXT:    store i64 [[COND45]], ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[CMP47:%.*]] = icmp sgt i64 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP47]], label %[[COND_TRUE48:.*]], label %[[COND_FALSE49:.*]]
+// CHECK1:       [[COND_TRUE48]]:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    br label %[[COND_END50:.*]]
+// CHECK1:       [[COND_FALSE49]]:
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    br label %[[COND_END50]]
+// CHECK1:       [[COND_END50]]:
+// CHECK1-NEXT:    [[COND51:%.*]] = phi i64 [ [[TMP29]], %[[COND_TRUE48]] ], [ [[TMP30]], %[[COND_FALSE49]] ]
+// CHECK1-NEXT:    store i64 [[COND51]], ptr [[DOTOMP_FUSE_MAX46]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX52]], align 8
 // CHECK1-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK1:       [[FOR_COND]]:
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX48]], align 8
-// CHECK1-NEXT:    [[CMP55:%.*]] = icmp slt i64 [[TMP34]], [[TMP35]]
-// CHECK1-NEXT:    br i1 [[CMP55]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX46]], align 8
+// CHECK1-NEXT:    [[CMP53:%.*]] = icmp slt i64 [[TMP31]], [[TMP32]]
+// CHECK1-NEXT:    br i1 [[CMP53]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK1:       [[FOR_BODY]]:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
-// CHECK1-NEXT:    [[CMP56:%.*]] = icmp slt i64 [[TMP36]], [[TMP37]]
-// CHECK1-NEXT:    br i1 [[CMP56]], label %[[IF_THEN:.*]], label %[[IF_END76:.*]]
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    [[CMP54:%.*]] = icmp slt i64 [[TMP33]], [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP54]], label %[[IF_THEN:.*]], label %[[IF_END74:.*]]
 // CHECK1:       [[IF_THEN]]:
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB04]], align 4
-// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP38]] to i64
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST05]], align 4
-// CHECK1-NEXT:    [[CONV58:%.*]] = sext i32 [[TMP39]] to i64
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV58]], [[TMP40]]
-// CHECK1-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV57]], [[MUL]]
-// CHECK1-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
-// CHECK1-NEXT:    store i32 [[CONV60]], ptr [[DOTOMP_IV07]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_IV07]], align 4
-// CHECK1-NEXT:    [[MUL61:%.*]] = mul nsw i32 [[TMP41]], 1
-// CHECK1-NEXT:    [[ADD62:%.*]] = add nsw i32 0, [[MUL61]]
-// CHECK1-NEXT:    store i32 [[ADD62]], ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    [[CMP63:%.*]] = icmp slt i32 [[TMP42]], [[TMP43]]
-// CHECK1-NEXT:    br i1 [[CMP63]], label %[[IF_THEN64:.*]], label %[[IF_END:.*]]
-// CHECK1:       [[IF_THEN64]]:
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP45]], [[TMP46]]
-// CHECK1-NEXT:    [[ADD66:%.*]] = add nsw i32 [[TMP44]], [[MUL65]]
-// CHECK1-NEXT:    store i32 [[ADD66]], ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[MUL67:%.*]] = mul nsw i32 [[TMP47]], 1
-// CHECK1-NEXT:    [[ADD68:%.*]] = add nsw i32 0, [[MUL67]]
-// CHECK1-NEXT:    store i32 [[ADD68]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP48]])
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_LB03]], align 4
+// CHECK1-NEXT:    [[CONV55:%.*]] = sext i32 [[TMP35]] to i64
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_ST04]], align 4
+// CHECK1-NEXT:    [[CONV56:%.*]] = sext i32 [[TMP36]] to i64
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV56]], [[TMP37]]
+// CHECK1-NEXT:    [[ADD57:%.*]] = add nsw i64 [[CONV55]], [[MUL]]
+// CHECK1-NEXT:    [[CONV58:%.*]] = trunc i64 [[ADD57]] to i32
+// CHECK1-NEXT:    store i32 [[CONV58]], ptr [[DOTOMP_IV06]], align 4
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV06]], align 4
+// CHECK1-NEXT:    [[MUL59:%.*]] = mul nsw i32 [[TMP38]], 1
+// CHECK1-NEXT:    [[ADD60:%.*]] = add nsw i32 0, [[MUL59]]
+// CHECK1-NEXT:    store i32 [[ADD60]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP61:%.*]] = icmp slt i32 [[TMP39]], [[TMP40]]
+// CHECK1-NEXT:    br i1 [[CMP61]], label %[[IF_THEN62:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN62]]:
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL63:%.*]] = mul nsw i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[ADD64:%.*]] = add nsw i32 [[TMP41]], [[MUL63]]
+// CHECK1-NEXT:    store i32 [[ADD64]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP44]], 1
+// CHECK1-NEXT:    [[ADD66:%.*]] = add nsw i32 0, [[MUL65]]
+// CHECK1-NEXT:    store i32 [[ADD66]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP45]])
 // CHECK1-NEXT:    br label %[[IF_END]]
 // CHECK1:       [[IF_END]]:
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP69:%.*]] = icmp slt i32 [[TMP49]], [[TMP50]]
-// CHECK1-NEXT:    br i1 [[CMP69]], label %[[IF_THEN70:.*]], label %[[IF_END75:.*]]
-// CHECK1:       [[IF_THEN70]]:
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP52]], [[TMP53]]
-// CHECK1-NEXT:    [[ADD72:%.*]] = add nsw i32 [[TMP51]], [[MUL71]]
-// CHECK1-NEXT:    store i32 [[ADD72]], ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP54]], 2
-// CHECK1-NEXT:    [[ADD74:%.*]] = add nsw i32 0, [[MUL73]]
-// CHECK1-NEXT:    store i32 [[ADD74]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP55]])
-// CHECK1-NEXT:    br label %[[IF_END75]]
-// CHECK1:       [[IF_END75]]:
-// CHECK1-NEXT:    br label %[[IF_END76]]
-// CHECK1:       [[IF_END76]]:
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK1-NEXT:    [[CMP77:%.*]] = icmp slt i64 [[TMP56]], [[TMP57]]
-// CHECK1-NEXT:    br i1 [[CMP77]], label %[[IF_THEN78:.*]], label %[[IF_END83:.*]]
-// CHECK1:       [[IF_THEN78]]:
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_LB118]], align 8
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_ST119]], align 8
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], [[TMP60]]
-// CHECK1-NEXT:    [[ADD80:%.*]] = add nsw i64 [[TMP58]], [[MUL79]]
-// CHECK1-NEXT:    store i64 [[ADD80]], ptr [[DOTOMP_IV122]], align 8
-// CHECK1-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV122]], align 8
-// CHECK1-NEXT:    [[MUL81:%.*]] = mul nsw i64 [[TMP62]], 1
-// CHECK1-NEXT:    [[ADD_PTR82:%.*]] = getelementptr inbounds double, ptr [[TMP61]], i64 [[MUL81]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR82]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    store ptr [[TMP63]], ptr [[V]], align 8
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[C]], align 4
-// CHECK1-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[V]], align 8
-// CHECK1-NEXT:    [[TMP66:%.*]] = load double, ptr [[TMP65]], align 8
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP64]], double noundef [[TMP66]])
-// CHECK1-NEXT:    br label %[[IF_END83]]
-// CHECK1:       [[IF_END83]]:
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK1-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP67]], [[TMP68]]
-// CHECK1-NEXT:    br i1 [[CMP84]], label %[[IF_THEN85:.*]], label %[[IF_END90:.*]]
-// CHECK1:       [[IF_THEN85]]:
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
-// CHECK1-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
-// CHECK1-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[ADD87:%.*]] = add nsw i64 [[TMP69]], [[MUL86]]
-// CHECK1-NEXT:    store i64 [[ADD87]], ptr [[DOTOMP_IV2]], align 8
-// CHECK1-NEXT:    [[TMP72:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK1-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
-// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[TMP73]], 1
-// CHECK1-NEXT:    [[ADD_PTR89:%.*]] = getelementptr inbounds double, ptr [[TMP72]], i64 [[MUL88]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR89]], ptr [[__BEGIN227]], align 8
-// CHECK1-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[__BEGIN227]], align 8
-// CHECK1-NEXT:    store ptr [[TMP74]], ptr [[VV]], align 8
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[CC]], align 4
-// CHECK1-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[VV]], align 8
-// CHECK1-NEXT:    [[TMP77:%.*]] = load double, ptr [[TMP76]], align 8
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP75]], double noundef [[TMP77]])
-// CHECK1-NEXT:    br label %[[IF_END90]]
-// CHECK1:       [[IF_END90]]:
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP67:%.*]] = icmp slt i32 [[TMP46]], [[TMP47]]
+// CHECK1-NEXT:    br i1 [[CMP67]], label %[[IF_THEN68:.*]], label %[[IF_END73:.*]]
+// CHECK1:       [[IF_THEN68]]:
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL69:%.*]] = mul nsw i32 [[TMP49]], [[TMP50]]
+// CHECK1-NEXT:    [[ADD70:%.*]] = add nsw i32 [[TMP48]], [[MUL69]]
+// CHECK1-NEXT:    store i32 [[ADD70]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP51]], 2
+// CHECK1-NEXT:    [[ADD72:%.*]] = add nsw i32 0, [[MUL71]]
+// CHECK1-NEXT:    store i32 [[ADD72]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK1-NEXT:    br label %[[IF_END73]]
+// CHECK1:       [[IF_END73]]:
+// CHECK1-NEXT:    br label %[[IF_END74]]
+// CHECK1:       [[IF_END74]]:
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[CMP75:%.*]] = icmp slt i64 [[TMP53]], [[TMP54]]
+// CHECK1-NEXT:    br i1 [[CMP75]], label %[[IF_THEN76:.*]], label %[[IF_END81:.*]]
+// CHECK1:       [[IF_THEN76]]:
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i64, ptr [[DOTOMP_LB116]], align 8
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_ST117]], align 8
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul nsw i64 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[ADD78:%.*]] = add nsw i64 [[TMP55]], [[MUL77]]
+// CHECK1-NEXT:    store i64 [[ADD78]], ptr [[DOTOMP_IV120]], align 8
+// CHECK1-NEXT:    [[TMP58:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_IV120]], align 8
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], 1
+// CHECK1-NEXT:    [[ADD_PTR80:%.*]] = getelementptr inbounds double, ptr [[TMP58]], i64 [[MUL79]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR80]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP60:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP60]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[C]], align 4
+// CHECK1-NEXT:    [[TMP62:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP63:%.*]] = load double, ptr [[TMP62]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP61]], double noundef [[TMP63]])
+// CHECK1-NEXT:    br label %[[IF_END81]]
+// CHECK1:       [[IF_END81]]:
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[CMP82:%.*]] = icmp slt i64 [[TMP64]], [[TMP65]]
+// CHECK1-NEXT:    br i1 [[CMP82]], label %[[IF_THEN83:.*]], label %[[IF_END88:.*]]
+// CHECK1:       [[IF_THEN83]]:
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[MUL84:%.*]] = mul nsw i64 [[TMP67]], [[TMP68]]
+// CHECK1-NEXT:    [[ADD85:%.*]] = add nsw i64 [[TMP66]], [[MUL84]]
+// CHECK1-NEXT:    store i64 [[ADD85]], ptr [[DOTOMP_IV2]], align 8
+// CHECK1-NEXT:    [[TMP69:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
+// CHECK1-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], 1
+// CHECK1-NEXT:    [[ADD_PTR87:%.*]] = getelementptr inbounds double, ptr [[TMP69]], i64 [[MUL86]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR87]], ptr [[__BEGIN225]], align 8
+// CHECK1-NEXT:    [[TMP71:%.*]] = load ptr, ptr [[__BEGIN225]], align 8
+// CHECK1-NEXT:    store ptr [[TMP71]], ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK1-NEXT:    [[TMP73:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP74:%.*]] = load double, ptr [[TMP73]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP72]], double noundef [[TMP74]])
+// CHECK1-NEXT:    br label %[[IF_END88]]
+// CHECK1:       [[IF_END88]]:
 // CHECK1-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK1:       [[FOR_INC]]:
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP78]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP75]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX52]], align 8
 // CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK1:       [[FOR_END]]:
 // CHECK1-NEXT:    ret void
@@ -794,13 +783,11 @@ extern "C" void foo4() {
 // CHECK1-NEXT:  [[ENTRY:.*:]]
 // CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -815,12 +802,10 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
-// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
 // CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[K]], align 4
-// CHECK1-NEXT:    store i32 63, ptr [[DOTOMP_UB1]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
 // CHECK1-NEXT:    store i32 64, ptr [[DOTOMP_NI1]], align 4
@@ -940,6 +925,277 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    ret void
 //
 //
+// CHECK1-LABEL: define dso_local void @foo5(
+// CHECK1-SAME: ) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB03:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST04:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI05:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV06:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_10:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB116:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_ST117:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_NI118:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV120:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_TEMP_121:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX22:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX29:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[CC:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE264:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN265:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END267:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[VV:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[K]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    store i32 512, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK1-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB03]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST04]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
+// CHECK1-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
+// CHECK1-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP8]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY7:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY7]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY9]], ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP11]], ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK1-NEXT:    [[SUB12:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[SUB12]], 1
+// CHECK1-NEXT:    [[DIV14:%.*]] = sdiv i64 [[ADD13]], 1
+// CHECK1-NEXT:    [[SUB15:%.*]] = sub nsw i64 [[DIV14]], 1
+// CHECK1-NEXT:    store i64 [[SUB15]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB116]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST117]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK1-NEXT:    store i64 [[ADD19]], ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[CMP23:%.*]] = icmp sgt i64 [[TMP16]], [[TMP17]]
+// CHECK1-NEXT:    br i1 [[CMP23]], label %[[COND_TRUE24:.*]], label %[[COND_FALSE25:.*]]
+// CHECK1:       [[COND_TRUE24]]:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK1-NEXT:    br label %[[COND_END26:.*]]
+// CHECK1:       [[COND_FALSE25]]:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    br label %[[COND_END26]]
+// CHECK1:       [[COND_END26]]:
+// CHECK1-NEXT:    [[COND27:%.*]] = phi i64 [ [[TMP18]], %[[COND_TRUE24]] ], [ [[TMP19]], %[[COND_FALSE25]] ]
+// CHECK1-NEXT:    store i64 [[COND27]], ptr [[DOTOMP_FUSE_MAX22]], align 8
+// CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP20]], 128
+// CHECK1-NEXT:    br i1 [[CMP28]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP21]])
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND30:.*]]
+// CHECK1:       [[FOR_COND30]]:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX22]], align 8
+// CHECK1-NEXT:    [[CMP31:%.*]] = icmp slt i64 [[TMP23]], [[TMP24]]
+// CHECK1-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY32:.*]], label %[[FOR_END63:.*]]
+// CHECK1:       [[FOR_BODY32]]:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    [[CMP33:%.*]] = icmp slt i64 [[TMP25]], [[TMP26]]
+// CHECK1-NEXT:    br i1 [[CMP33]], label %[[IF_THEN:.*]], label %[[IF_END53:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB03]], align 4
+// CHECK1-NEXT:    [[CONV34:%.*]] = sext i32 [[TMP27]] to i64
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_ST04]], align 4
+// CHECK1-NEXT:    [[CONV35:%.*]] = sext i32 [[TMP28]] to i64
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV35]], [[TMP29]]
+// CHECK1-NEXT:    [[ADD36:%.*]] = add nsw i64 [[CONV34]], [[MUL]]
+// CHECK1-NEXT:    [[CONV37:%.*]] = trunc i64 [[ADD36]] to i32
+// CHECK1-NEXT:    store i32 [[CONV37]], ptr [[DOTOMP_IV06]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_IV06]], align 4
+// CHECK1-NEXT:    [[MUL38:%.*]] = mul nsw i32 [[TMP30]], 1
+// CHECK1-NEXT:    [[ADD39:%.*]] = add nsw i32 0, [[MUL38]]
+// CHECK1-NEXT:    store i32 [[ADD39]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP40:%.*]] = icmp slt i32 [[TMP31]], [[TMP32]]
+// CHECK1-NEXT:    br i1 [[CMP40]], label %[[IF_THEN41:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN41]]:
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL42:%.*]] = mul nsw i32 [[TMP34]], [[TMP35]]
+// CHECK1-NEXT:    [[ADD43:%.*]] = add nsw i32 [[TMP33]], [[MUL42]]
+// CHECK1-NEXT:    store i32 [[ADD43]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[MUL44:%.*]] = mul nsw i32 [[TMP36]], 2
+// CHECK1-NEXT:    [[ADD45:%.*]] = add nsw i32 0, [[MUL44]]
+// CHECK1-NEXT:    store i32 [[ADD45]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP37]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP46:%.*]] = icmp slt i32 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
+// CHECK1:       [[IF_THEN47]]:
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL48:%.*]] = mul nsw i32 [[TMP41]], [[TMP42]]
+// CHECK1-NEXT:    [[ADD49:%.*]] = add nsw i32 [[TMP40]], [[MUL48]]
+// CHECK1-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul nsw i32 [[TMP43]], 1
+// CHECK1-NEXT:    [[ADD51:%.*]] = add nsw i32 0, [[MUL50]]
+// CHECK1-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK1-NEXT:    br label %[[IF_END52]]
+// CHECK1:       [[IF_END52]]:
+// CHECK1-NEXT:    br label %[[IF_END53]]
+// CHECK1:       [[IF_END53]]:
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[CMP54:%.*]] = icmp slt i64 [[TMP45]], [[TMP46]]
+// CHECK1-NEXT:    br i1 [[CMP54]], label %[[IF_THEN55:.*]], label %[[IF_END60:.*]]
+// CHECK1:       [[IF_THEN55]]:
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB116]], align 8
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_ST117]], align 8
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[MUL56:%.*]] = mul nsw i64 [[TMP48]], [[TMP49]]
+// CHECK1-NEXT:    [[ADD57:%.*]] = add nsw i64 [[TMP47]], [[MUL56]]
+// CHECK1-NEXT:    store i64 [[ADD57]], ptr [[DOTOMP_IV120]], align 8
+// CHECK1-NEXT:    [[TMP50:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV120]], align 8
+// CHECK1-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[TMP51]], 1
+// CHECK1-NEXT:    [[ADD_PTR59:%.*]] = getelementptr inbounds double, ptr [[TMP50]], i64 [[MUL58]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR59]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP52:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP52]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[C]], align 4
+// CHECK1-NEXT:    [[TMP54:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP55:%.*]] = load double, ptr [[TMP54]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP53]], double noundef [[TMP55]])
+// CHECK1-NEXT:    br label %[[IF_END60]]
+// CHECK1:       [[IF_END60]]:
+// CHECK1-NEXT:    br label %[[FOR_INC61:.*]]
+// CHECK1:       [[FOR_INC61]]:
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[INC62:%.*]] = add nsw i64 [[TMP56]], 1
+// CHECK1-NEXT:    store i64 [[INC62]], ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND30]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK1:       [[FOR_END63]]:
+// CHECK1-NEXT:    store i32 37, ptr [[CC]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE264]], align 8
+// CHECK1-NEXT:    [[TMP57:%.*]] = load ptr, ptr [[__RANGE264]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY66:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP57]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY66]], ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    [[TMP58:%.*]] = load ptr, ptr [[__RANGE264]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY68:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP58]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR69:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY68]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR69]], ptr [[__END267]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND70:.*]]
+// CHECK1:       [[FOR_COND70]]:
+// CHECK1-NEXT:    [[TMP59:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    [[TMP60:%.*]] = load ptr, ptr [[__END267]], align 8
+// CHECK1-NEXT:    [[CMP71:%.*]] = icmp ne ptr [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    br i1 [[CMP71]], label %[[FOR_BODY72:.*]], label %[[FOR_END74:.*]]
+// CHECK1:       [[FOR_BODY72]]:
+// CHECK1-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    store ptr [[TMP61]], ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP64:%.*]] = load double, ptr [[TMP63]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP62]], double noundef [[TMP64]])
+// CHECK1-NEXT:    br label %[[FOR_INC73:.*]]
+// CHECK1:       [[FOR_INC73]]:
+// CHECK1-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw double, ptr [[TMP65]], i32 1
+// CHECK1-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND70]]
+// CHECK1:       [[FOR_END74]]:
+// CHECK1-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define dso_local void @body(
 // CHECK2-SAME: ...) #[[ATTR0:[0-9]+]] {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
@@ -961,7 +1217,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
@@ -970,7 +1225,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -1002,107 +1256,103 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
 // CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP8]], 1
 // CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP9]], ptr [[J]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[START2_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START2_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[END2_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP13]], [[TMP14]]
 // CHECK2-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP15]]
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP16]]
 // CHECK2-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
 // CHECK2-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    store i32 [[TMP20]], ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP21]], [[TMP22]]
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP19]], [[TMP20]]
 // CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // CHECK2:       [[COND_TRUE]]:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
 // CHECK2-NEXT:    br label %[[COND_END:.*]]
 // CHECK2:       [[COND_FALSE]]:
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
 // CHECK2-NEXT:    br label %[[COND_END]]
 // CHECK2:       [[COND_END]]:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP23]], %[[COND_TRUE]] ], [ [[TMP24]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP21]], %[[COND_TRUE]] ], [ [[TMP22]], %[[COND_FALSE]] ]
 // CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK2-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK2:       [[FOR_COND]]:
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
-// CHECK2-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP23]], [[TMP24]]
 // CHECK2-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK2:       [[FOR_BODY]]:
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
 // CHECK2-NEXT:    br i1 [[CMP17]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // CHECK2:       [[IF_THEN]]:
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP30]], [[TMP31]]
-// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP29]], [[MUL]]
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP28]], [[TMP29]]
+// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP27]], [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP33]], [[TMP34]]
-// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP32]], [[MUL19]]
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP31]], [[TMP32]]
+// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP30]], [[MUL19]]
 // CHECK2-NEXT:    store i32 [[ADD20]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP35]])
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP33]])
 // CHECK2-NEXT:    br label %[[IF_END]]
 // CHECK2:       [[IF_END]]:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP34]], [[TMP35]]
 // CHECK2-NEXT:    br i1 [[CMP21]], label %[[IF_THEN22:.*]], label %[[IF_END27:.*]]
 // CHECK2:       [[IF_THEN22]]:
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP39]], [[TMP40]]
-// CHECK2-NEXT:    [[ADD24:%.*]] = add i32 [[TMP38]], [[MUL23]]
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// CHECK2-NEXT:    [[ADD24:%.*]] = add i32 [[TMP36]], [[MUL23]]
 // CHECK2-NEXT:    store i32 [[ADD24]], ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP42]], [[TMP43]]
-// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[TMP41]], [[MUL25]]
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP40]], [[TMP41]]
+// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[TMP39]], [[MUL25]]
 // CHECK2-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP42]])
 // CHECK2-NEXT:    br label %[[IF_END27]]
 // CHECK2:       [[IF_END27]]:
 // CHECK2-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK2:       [[FOR_INC]]:
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP45]], 1
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
 // CHECK2:       [[FOR_END]]:
@@ -1114,13 +1364,11 @@ extern "C" void foo4() {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
 // CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -1130,48 +1378,43 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB03:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_LB04:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_ST05:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_NI06:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_IV07:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB03:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST04:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI05:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV06:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[C:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_UB117:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_LB118:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_ST119:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_NI120:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_IV122:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_10:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB116:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_ST117:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_NI118:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV120:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[CC:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[__RANGE223:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[__END224:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[__BEGIN227:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__RANGE221:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END222:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN225:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_27:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_29:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_31:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_32:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_30:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_TEMP_142:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_TEMP_140:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX48:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX54:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX46:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX52:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[VV:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
 // CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
-// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB1]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
 // CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI1]], align 4
@@ -1198,225 +1441,219 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
 // CHECK2-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
 // CHECK2-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB03]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST04]], align 4
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTOMP_UB03]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB04]], align 4
-// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST05]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], 1
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
 // CHECK2-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
-// CHECK2-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI06]], align 8
+// CHECK2-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI05]], align 8
 // CHECK2-NEXT:    store i32 42, ptr [[C]], align 4
 // CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP8]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY7:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY7]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY8:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYDECAY8]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY10:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP11]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYDECAY10]], ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__END2]], align 8
-// CHECK2-NEXT:    store ptr [[TMP12]], ptr [[DOTCAPTURE_EXPR_11]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_11]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
-// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP14]] to i64
+// CHECK2-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY9]], ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP11]], ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
 // CHECK2-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
 // CHECK2-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
-// CHECK2-NEXT:    [[SUB13:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
-// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i64 [[SUB13]], 1
-// CHECK2-NEXT:    [[DIV15:%.*]] = sdiv i64 [[ADD14]], 1
-// CHECK2-NEXT:    [[SUB16:%.*]] = sub nsw i64 [[DIV15]], 1
-// CHECK2-NEXT:    store i64 [[SUB16]], ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK2-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_UB117]], align 8
-// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB118]], align 8
-// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST119]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK2-NEXT:    store i64 [[ADD21]], ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    [[SUB12:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[SUB12]], 1
+// CHECK2-NEXT:    [[DIV14:%.*]] = sdiv i64 [[ADD13]], 1
+// CHECK2-NEXT:    [[SUB15:%.*]] = sub nsw i64 [[DIV14]], 1
+// CHECK2-NEXT:    store i64 [[SUB15]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB116]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST117]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK2-NEXT:    store i64 [[ADD19]], ptr [[DOTOMP_NI118]], align 8
 // CHECK2-NEXT:    store i32 37, ptr [[CC]], align 4
-// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE223]], align 8
-// CHECK2-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY25:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
-// CHECK2-NEXT:    [[ADD_PTR26:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY25]], i64 256
-// CHECK2-NEXT:    store ptr [[ADD_PTR26]], ptr [[__END224]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP18]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[__BEGIN227]], align 8
-// CHECK2-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY30:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP19]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYDECAY30]], ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK2-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[__END224]], align 8
-// CHECK2-NEXT:    store ptr [[TMP20]], ptr [[DOTCAPTURE_EXPR_31]], align 8
-// CHECK2-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_31]], align 8
-// CHECK2-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST33:%.*]] = ptrtoint ptr [[TMP21]] to i64
-// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST34:%.*]] = ptrtoint ptr [[TMP22]] to i64
-// CHECK2-NEXT:    [[SUB_PTR_SUB35:%.*]] = sub i64 [[SUB_PTR_LHS_CAST33]], [[SUB_PTR_RHS_CAST34]]
-// CHECK2-NEXT:    [[SUB_PTR_DIV36:%.*]] = sdiv exact i64 [[SUB_PTR_SUB35]], 8
-// CHECK2-NEXT:    [[SUB37:%.*]] = sub nsw i64 [[SUB_PTR_DIV36]], 1
-// CHECK2-NEXT:    [[ADD38:%.*]] = add nsw i64 [[SUB37]], 1
-// CHECK2-NEXT:    [[DIV39:%.*]] = sdiv i64 [[ADD38]], 1
-// CHECK2-NEXT:    [[SUB40:%.*]] = sub nsw i64 [[DIV39]], 1
-// CHECK2-NEXT:    store i64 [[SUB40]], ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK2-NEXT:    store i64 [[TMP23]], ptr [[DOTOMP_UB2]], align 8
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE221]], align 8
+// CHECK2-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY23:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP15]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR24:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY23]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR24]], ptr [[__END222]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY26:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP16]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY26]], ptr [[__BEGIN225]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__END222]], align 8
+// CHECK2-NEXT:    store ptr [[TMP18]], ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST31:%.*]] = ptrtoint ptr [[TMP19]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST32:%.*]] = ptrtoint ptr [[TMP20]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB33:%.*]] = sub i64 [[SUB_PTR_LHS_CAST31]], [[SUB_PTR_RHS_CAST32]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV34:%.*]] = sdiv exact i64 [[SUB_PTR_SUB33]], 8
+// CHECK2-NEXT:    [[SUB35:%.*]] = sub nsw i64 [[SUB_PTR_DIV34]], 1
+// CHECK2-NEXT:    [[ADD36:%.*]] = add nsw i64 [[SUB35]], 1
+// CHECK2-NEXT:    [[DIV37:%.*]] = sdiv i64 [[ADD36]], 1
+// CHECK2-NEXT:    [[SUB38:%.*]] = sub nsw i64 [[DIV37]], 1
+// CHECK2-NEXT:    store i64 [[SUB38]], ptr [[DOTCAPTURE_EXPR_30]], align 8
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB2]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST2]], align 8
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK2-NEXT:    [[ADD41:%.*]] = add nsw i64 [[TMP24]], 1
-// CHECK2-NEXT:    store i64 [[ADD41]], ptr [[DOTOMP_NI2]], align 8
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
-// CHECK2-NEXT:    store i64 [[TMP25]], ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK2-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP26]], [[TMP27]]
-// CHECK2-NEXT:    br i1 [[CMP43]], label %[[COND_TRUE44:.*]], label %[[COND_FALSE45:.*]]
-// CHECK2:       [[COND_TRUE44]]:
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK2-NEXT:    br label %[[COND_END46:.*]]
-// CHECK2:       [[COND_FALSE45]]:
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK2-NEXT:    br label %[[COND_END46]]
-// CHECK2:       [[COND_END46]]:
-// CHECK2-NEXT:    [[COND47:%.*]] = phi i64 [ [[TMP28]], %[[COND_TRUE44]] ], [ [[TMP29]], %[[COND_FALSE45]] ]
-// CHECK2-NEXT:    store i64 [[COND47]], ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK2-NEXT:    [[CMP49:%.*]] = icmp sgt i64 [[TMP30]], [[TMP31]]
-// CHECK2-NEXT:    br i1 [[CMP49]], label %[[COND_TRUE50:.*]], label %[[COND_FALSE51:.*]]
-// CHECK2:       [[COND_TRUE50]]:
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK2-NEXT:    br label %[[COND_END52:.*]]
-// CHECK2:       [[COND_FALSE51]]:
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK2-NEXT:    br label %[[COND_END52]]
-// CHECK2:       [[COND_END52]]:
-// CHECK2-NEXT:    [[COND53:%.*]] = phi i64 [ [[TMP32]], %[[COND_TRUE50]] ], [ [[TMP33]], %[[COND_FALSE51]] ]
-// CHECK2-NEXT:    store i64 [[COND53]], ptr [[DOTOMP_FUSE_MAX48]], align 8
-// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_30]], align 8
+// CHECK2-NEXT:    [[ADD39:%.*]] = add nsw i64 [[TMP21]], 1
+// CHECK2-NEXT:    store i64 [[ADD39]], ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    store i64 [[TMP22]], ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[CMP41:%.*]] = icmp sgt i64 [[TMP23]], [[TMP24]]
+// CHECK2-NEXT:    br i1 [[CMP41]], label %[[COND_TRUE42:.*]], label %[[COND_FALSE43:.*]]
+// CHECK2:       [[COND_TRUE42]]:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK2-NEXT:    br label %[[COND_END44:.*]]
+// CHECK2:       [[COND_FALSE43]]:
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    br label %[[COND_END44]]
+// CHECK2:       [[COND_END44]]:
+// CHECK2-NEXT:    [[COND45:%.*]] = phi i64 [ [[TMP25]], %[[COND_TRUE42]] ], [ [[TMP26]], %[[COND_FALSE43]] ]
+// CHECK2-NEXT:    store i64 [[COND45]], ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[CMP47:%.*]] = icmp sgt i64 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP47]], label %[[COND_TRUE48:.*]], label %[[COND_FALSE49:.*]]
+// CHECK2:       [[COND_TRUE48]]:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    br label %[[COND_END50:.*]]
+// CHECK2:       [[COND_FALSE49]]:
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    br label %[[COND_END50]]
+// CHECK2:       [[COND_END50]]:
+// CHECK2-NEXT:    [[COND51:%.*]] = phi i64 [ [[TMP29]], %[[COND_TRUE48]] ], [ [[TMP30]], %[[COND_FALSE49]] ]
+// CHECK2-NEXT:    store i64 [[COND51]], ptr [[DOTOMP_FUSE_MAX46]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX52]], align 8
 // CHECK2-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK2:       [[FOR_COND]]:
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX48]], align 8
-// CHECK2-NEXT:    [[CMP55:%.*]] = icmp slt i64 [[TMP34]], [[TMP35]]
-// CHECK2-NEXT:    br i1 [[CMP55]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX46]], align 8
+// CHECK2-NEXT:    [[CMP53:%.*]] = icmp slt i64 [[TMP31]], [[TMP32]]
+// CHECK2-NEXT:    br i1 [[CMP53]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK2:       [[FOR_BODY]]:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
-// CHECK2-NEXT:    [[CMP56:%.*]] = icmp slt i64 [[TMP36]], [[TMP37]]
-// CHECK2-NEXT:    br i1 [[CMP56]], label %[[IF_THEN:.*]], label %[[IF_END76:.*]]
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    [[CMP54:%.*]] = icmp slt i64 [[TMP33]], [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP54]], label %[[IF_THEN:.*]], label %[[IF_END74:.*]]
 // CHECK2:       [[IF_THEN]]:
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB04]], align 4
-// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP38]] to i64
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST05]], align 4
-// CHECK2-NEXT:    [[CONV58:%.*]] = sext i32 [[TMP39]] to i64
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV58]], [[TMP40]]
-// CHECK2-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV57]], [[MUL]]
-// CHECK2-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
-// CHECK2-NEXT:    store i32 [[CONV60]], ptr [[DOTOMP_IV07]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_IV07]], align 4
-// CHECK2-NEXT:    [[MUL61:%.*]] = mul nsw i32 [[TMP41]], 1
-// CHECK2-NEXT:    [[ADD62:%.*]] = add nsw i32 0, [[MUL61]]
-// CHECK2-NEXT:    store i32 [[ADD62]], ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    [[CMP63:%.*]] = icmp slt i32 [[TMP42]], [[TMP43]]
-// CHECK2-NEXT:    br i1 [[CMP63]], label %[[IF_THEN64:.*]], label %[[IF_END:.*]]
-// CHECK2:       [[IF_THEN64]]:
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP45]], [[TMP46]]
-// CHECK2-NEXT:    [[ADD66:%.*]] = add nsw i32 [[TMP44]], [[MUL65]]
-// CHECK2-NEXT:    store i32 [[ADD66]], ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[MUL67:%.*]] = mul nsw i32 [[TMP47]], 1
-// CHECK2-NEXT:    [[ADD68:%.*]] = add nsw i32 0, [[MUL67]]
-// CHECK2-NEXT:    store i32 [[ADD68]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP48]])
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_LB03]], align 4
+// CHECK2-NEXT:    [[CONV55:%.*]] = sext i32 [[TMP35]] to i64
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_ST04]], align 4
+// CHECK2-NEXT:    [[CONV56:%.*]] = sext i32 [[TMP36]] to i64
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV56]], [[TMP37]]
+// CHECK2-NEXT:    [[ADD57:%.*]] = add nsw i64 [[CONV55]], [[MUL]]
+// CHECK2-NEXT:    [[CONV58:%.*]] = trunc i64 [[ADD57]] to i32
+// CHECK2-NEXT:    store i32 [[CONV58]], ptr [[DOTOMP_IV06]], align 4
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV06]], align 4
+// CHECK2-NEXT:    [[MUL59:%.*]] = mul nsw i32 [[TMP38]], 1
+// CHECK2-NEXT:    [[ADD60:%.*]] = add nsw i32 0, [[MUL59]]
+// CHECK2-NEXT:    store i32 [[ADD60]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP61:%.*]] = icmp slt i32 [[TMP39]], [[TMP40]]
+// CHECK2-NEXT:    br i1 [[CMP61]], label %[[IF_THEN62:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN62]]:
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL63:%.*]] = mul nsw i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[ADD64:%.*]] = add nsw i32 [[TMP41]], [[MUL63]]
+// CHECK2-NEXT:    store i32 [[ADD64]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP44]], 1
+// CHECK2-NEXT:    [[ADD66:%.*]] = add nsw i32 0, [[MUL65]]
+// CHECK2-NEXT:    store i32 [[ADD66]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP45]])
 // CHECK2-NEXT:    br label %[[IF_END]]
 // CHECK2:       [[IF_END]]:
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP69:%.*]] = icmp slt i32 [[TMP49]], [[TMP50]]
-// CHECK2-NEXT:    br i1 [[CMP69]], label %[[IF_THEN70:.*]], label %[[IF_END75:.*]]
-// CHECK2:       [[IF_THEN70]]:
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP52]], [[TMP53]]
-// CHECK2-NEXT:    [[ADD72:%.*]] = add nsw i32 [[TMP51]], [[MUL71]]
-// CHECK2-NEXT:    store i32 [[ADD72]], ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP54]], 2
-// CHECK2-NEXT:    [[ADD74:%.*]] = add nsw i32 0, [[MUL73]]
-// CHECK2-NEXT:    store i32 [[ADD74]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP55]])
-// CHECK2-NEXT:    br label %[[IF_END75]]
-// CHECK2:       [[IF_END75]]:
-// CHECK2-NEXT:    br label %[[IF_END76]]
-// CHECK2:       [[IF_END76]]:
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK2-NEXT:    [[CMP77:%.*]] = icmp slt i64 [[TMP56]], [[TMP57]]
-// CHECK2-NEXT:    br i1 [[CMP77]], label %[[IF_THEN78:.*]], label %[[IF_END83:.*]]
-// CHECK2:       [[IF_THEN78]]:
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_LB118]], align 8
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_ST119]], align 8
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], [[TMP60]]
-// CHECK2-NEXT:    [[ADD80:%.*]] = add nsw i64 [[TMP58]], [[MUL79]]
-// CHECK2-NEXT:    store i64 [[ADD80]], ptr [[DOTOMP_IV122]], align 8
-// CHECK2-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV122]], align 8
-// CHECK2-NEXT:    [[MUL81:%.*]] = mul nsw i64 [[TMP62]], 1
-// CHECK2-NEXT:    [[ADD_PTR82:%.*]] = getelementptr inbounds double, ptr [[TMP61]], i64 [[MUL81]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR82]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    store ptr [[TMP63]], ptr [[V]], align 8
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[C]], align 4
-// CHECK2-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[V]], align 8
-// CHECK2-NEXT:    [[TMP66:%.*]] = load double, ptr [[TMP65]], align 8
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP64]], double noundef [[TMP66]])
-// CHECK2-NEXT:    br label %[[IF_END83]]
-// CHECK2:       [[IF_END83]]:
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK2-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP67]], [[TMP68]]
-// CHECK2-NEXT:    br i1 [[CMP84]], label %[[IF_THEN85:.*]], label %[[IF_END90:.*]]
-// CHECK2:       [[IF_THEN85]]:
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
-// CHECK2-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
-// CHECK2-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[ADD87:%.*]] = add nsw i64 [[TMP69]], [[MUL86]]
-// CHECK2-NEXT:    store i64 [[ADD87]], ptr [[DOTOMP_IV2]], align 8
-// CHECK2-NEXT:    [[TMP72:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK2-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
-// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[TMP73]], 1
-// CHECK2-NEXT:    [[ADD_PTR89:%.*]] = getelementptr inbounds double, ptr [[TMP72]], i64 [[MUL88]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR89]], ptr [[__BEGIN227]], align 8
-// CHECK2-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[__BEGIN227]], align 8
-// CHECK2-NEXT:    store ptr [[TMP74]], ptr [[VV]], align 8
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[CC]], align 4
-// CHECK2-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[VV]], align 8
-// CHECK2-NEXT:    [[TMP77:%.*]] = load double, ptr [[TMP76]], align 8
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP75]], double noundef [[TMP77]])
-// CHECK2-NEXT:    br label %[[IF_END90]]
-// CHECK2:       [[IF_END90]]:
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP67:%.*]] = icmp slt i32 [[TMP46]], [[TMP47]]
+// CHECK2-NEXT:    br i1 [[CMP67]], label %[[IF_THEN68:.*]], label %[[IF_END73:.*]]
+// CHECK2:       [[IF_THEN68]]:
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL69:%.*]] = mul nsw i32 [[TMP49]], [[TMP50]]
+// CHECK2-NEXT:    [[ADD70:%.*]] = add nsw i32 [[TMP48]], [[MUL69]]
+// CHECK2-NEXT:    store i32 [[ADD70]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP51]], 2
+// CHECK2-NEXT:    [[ADD72:%.*]] = add nsw i32 0, [[MUL71]]
+// CHECK2-NEXT:    store i32 [[ADD72]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK2-NEXT:    br label %[[IF_END73]]
+// CHECK2:       [[IF_END73]]:
+// CHECK2-NEXT:    br label %[[IF_END74]]
+// CHECK2:       [[IF_END74]]:
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[CMP75:%.*]] = icmp slt i64 [[TMP53]], [[TMP54]]
+// CHECK2-NEXT:    br i1 [[CMP75]], label %[[IF_THEN76:.*]], label %[[IF_END81:.*]]
+// CHECK2:       [[IF_THEN76]]:
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i64, ptr [[DOTOMP_LB116]], align 8
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_ST117]], align 8
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul nsw i64 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[ADD78:%.*]] = add nsw i64 [[TMP55]], [[MUL77]]
+// CHECK2-NEXT:    store i64 [[ADD78]], ptr [[DOTOMP_IV120]], align 8
+// CHECK2-NEXT:    [[TMP58:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_IV120]], align 8
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], 1
+// CHECK2-NEXT:    [[ADD_PTR80:%.*]] = getelementptr inbounds double, ptr [[TMP58]], i64 [[MUL79]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR80]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP60:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP60]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[C]], align 4
+// CHECK2-NEXT:    [[TMP62:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP63:%.*]] = load double, ptr [[TMP62]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP61]], double noundef [[TMP63]])
+// CHECK2-NEXT:    br label %[[IF_END81]]
+// CHECK2:       [[IF_END81]]:
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[CMP82:%.*]] = icmp slt i64 [[TMP64]], [[TMP65]]
+// CHECK2-NEXT:    br i1 [[CMP82]], label %[[IF_THEN83:.*]], label %[[IF_END88:.*]]
+// CHECK2:       [[IF_THEN83]]:
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[MUL84:%.*]] = mul nsw i64 [[TMP67]], [[TMP68]]
+// CHECK2-NEXT:    [[ADD85:%.*]] = add nsw i64 [[TMP66]], [[MUL84]]
+// CHECK2-NEXT:    store i64 [[ADD85]], ptr [[DOTOMP_IV2]], align 8
+// CHECK2-NEXT:    [[TMP69:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
+// CHECK2-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], 1
+// CHECK2-NEXT:    [[ADD_PTR87:%.*]] = getelementptr inbounds double, ptr [[TMP69]], i64 [[MUL86]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR87]], ptr [[__BEGIN225]], align 8
+// CHECK2-NEXT:    [[TMP71:%.*]] = load ptr, ptr [[__BEGIN225]], align 8
+// CHECK2-NEXT:    store ptr [[TMP71]], ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK2-NEXT:    [[TMP73:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP74:%.*]] = load double, ptr [[TMP73]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP72]], double noundef [[TMP74]])
+// CHECK2-NEXT:    br label %[[IF_END88]]
+// CHECK2:       [[IF_END88]]:
 // CHECK2-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK2:       [[FOR_INC]]:
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP78]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP75]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX52]], align 8
 // CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
 // CHECK2:       [[FOR_END]]:
 // CHECK2-NEXT:    ret void
@@ -1427,13 +1664,11 @@ extern "C" void foo4() {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
 // CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -1448,12 +1683,10 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
-// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
 // CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[K]], align 4
-// CHECK2-NEXT:    store i32 63, ptr [[DOTOMP_UB1]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
 // CHECK2-NEXT:    store i32 64, ptr [[DOTOMP_NI1]], align 4
@@ -1573,6 +1806,277 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    ret void
 //
 //
+// CHECK2-LABEL: define dso_local void @foo5(
+// CHECK2-SAME: ) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB03:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST04:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI05:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV06:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_10:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB116:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_ST117:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_NI118:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV120:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_TEMP_121:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX22:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX29:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[CC:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE264:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN265:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END267:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[VV:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[K]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    store i32 512, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK2-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB03]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST04]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
+// CHECK2-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
+// CHECK2-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP8]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY7:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY7]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY9]], ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP11]], ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK2-NEXT:    [[SUB12:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[SUB12]], 1
+// CHECK2-NEXT:    [[DIV14:%.*]] = sdiv i64 [[ADD13]], 1
+// CHECK2-NEXT:    [[SUB15:%.*]] = sub nsw i64 [[DIV14]], 1
+// CHECK2-NEXT:    store i64 [[SUB15]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB116]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST117]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK2-NEXT:    store i64 [[ADD19]], ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[CMP23:%.*]] = icmp sgt i64 [[TMP16]], [[TMP17]]
+// CHECK2-NEXT:    br i1 [[CMP23]], label %[[COND_TRUE24:.*]], label %[[COND_FALSE25:.*]]
+// CHECK2:       [[COND_TRUE24]]:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK2-NEXT:    br label %[[COND_END26:.*]]
+// CHECK2:       [[COND_FALSE25]]:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    br label %[[COND_END26]]
+// CHECK2:       [[COND_END26]]:
+// CHECK2-NEXT:    [[COND27:%.*]] = phi i64 [ [[TMP18]], %[[COND_TRUE24]] ], [ [[TMP19]], %[[COND_FALSE25]] ]
+// CHECK2-NEXT:    store i64 [[COND27]], ptr [[DOTOMP_FUSE_MAX22]], align 8
+// CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP20]], 128
+// CHECK2-NEXT:    br i1 [[CMP28]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP21]])
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND30:.*]]
+// CHECK2:       [[FOR_COND30]]:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX22]], align 8
+// CHECK2-NEXT:    [[CMP31:%.*]] = icmp slt i64 [[TMP23]], [[TMP24]]
+// CHECK2-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY32:.*]], label %[[FOR_END63:.*]]
+// CHECK2:       [[FOR_BODY32]]:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    [[CMP33:%.*]] = icmp slt i64 [[TMP25]], [[TMP26]]
+// CHECK2-NEXT:    br i1 [[CMP33]], label %[[IF_THEN:.*]], label %[[IF_END53:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB03]], align 4
+// CHECK2-NEXT:    [[CONV34:%.*]] = sext i32 [[TMP27]] to i64
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_ST04]], align 4
+// CHECK2-NEXT:    [[CONV35:%.*]] = sext i32 [[TMP28]] to i64
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV35]], [[TMP29]]
+// CHECK2-NEXT:    [[ADD36:%.*]] = add nsw i64 [[CONV34]], [[MUL]]
+// CHECK2-NEXT:    [[CONV37:%.*]] = trunc i64 [[ADD36]] to i32
+// CHECK2-NEXT:    store i32 [[CONV37]], ptr [[DOTOMP_IV06]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_IV06]], align 4
+// CHECK2-NEXT:    [[MUL38:%.*]] = mul nsw i32 [[TMP30]], 1
+// CHECK2-NEXT:    [[ADD39:%.*]] = add nsw i32 0, [[MUL38]]
+// CHECK2-NEXT:    store i32 [[ADD39]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP40:%.*]] = icmp slt i32 [[TMP31]], [[TMP32]]
+// CHECK2-NEXT:    br i1 [[CMP40]], label %[[IF_THEN41:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN41]]:
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL42:%.*]] = mul nsw i32 [[TMP34]], [[TMP35]]
+// CHECK2-NEXT:    [[ADD43:%.*]] = add nsw i32 [[TMP33]], [[MUL42]]
+// CHECK2-NEXT:    store i32 [[ADD43]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[MUL44:%.*]] = mul nsw i32 [[TMP36]], 2
+// CHECK2-NEXT:    [[ADD45:%.*]] = add nsw i32 0, [[MUL44]]
+// CHECK2-NEXT:    store i32 [[ADD45]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP37]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP46:%.*]] = icmp slt i32 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
+// CHECK2:       [[IF_THEN47]]:
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL48:%.*]] = mul nsw i32 [[TMP41]], [[TMP42]]
+// CHECK2-NEXT:    [[ADD49:%.*]] = add nsw i32 [[TMP40]], [[MUL48]]
+// CHECK2-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul nsw i32 [[TMP43]], 1
+// CHECK2-NEXT:    [[ADD51:%.*]] = add nsw i32 0, [[MUL50]]
+// CHECK2-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK2-NEXT:    br label %[[IF_END52]]
+// CHECK2:       [[IF_END52]]:
+// CHECK2-NEXT:    br label %[[IF_END53]]
+// CHECK2:       [[IF_END53]]:
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[CMP54:%.*]] = icmp slt i64 [[TMP45]], [[TMP46]]
+// CHECK2-NEXT:    br i1 [[CMP54]], label %[[IF_THEN55:.*]], label %[[IF_END60:.*]]
+// CHECK2:       [[IF_THEN55]]:
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB116]], align 8
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_ST117]], align 8
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[MUL56:%.*]] = mul nsw i64 [[TMP48]], [[TMP49]]
+// CHECK2-NEXT:    [[ADD57:%.*]] = add nsw i64 [[TMP47]], [[MUL56]]
+// CHECK2-NEXT:    store i64 [[ADD57]], ptr [[DOTOMP_IV120]], align 8
+// CHECK2-NEXT:    [[TMP50:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV120]], align 8
+// CHECK2-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[TMP51]], 1
+// CHECK2-NEXT:    [[ADD_PTR59:%.*]] = getelementptr inbounds double, ptr [[TMP50]], i64 [[MUL58]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR59]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP52:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP52]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[C]], align 4
+// CHECK2-NEXT:    [[TMP54:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP55:%.*]] = load double, ptr [[TMP54]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP53]], double noundef [[TMP55]])
+// CHECK2-NEXT:    br label %[[IF_END60]]
+// CHECK2:       [[IF_END60]]:
+// CHECK2-NEXT:    br label %[[FOR_INC61:.*]]
+// CHECK2:       [[FOR_INC61]]:
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[INC62:%.*]] = add nsw i64 [[TMP56]], 1
+// CHECK2-NEXT:    store i64 [[INC62]], ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND30]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK2:       [[FOR_END63]]:
+// CHECK2-NEXT:    store i32 37, ptr [[CC]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE264]], align 8
+// CHECK2-NEXT:    [[TMP57:%.*]] = load ptr, ptr [[__RANGE264]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY66:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP57]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY66]], ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    [[TMP58:%.*]] = load ptr, ptr [[__RANGE264]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY68:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP58]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR69:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY68]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR69]], ptr [[__END267]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND70:.*]]
+// CHECK2:       [[FOR_COND70]]:
+// CHECK2-NEXT:    [[TMP59:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    [[TMP60:%.*]] = load ptr, ptr [[__END267]], align 8
+// CHECK2-NEXT:    [[CMP71:%.*]] = icmp ne ptr [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    br i1 [[CMP71]], label %[[FOR_BODY72:.*]], label %[[FOR_END74:.*]]
+// CHECK2:       [[FOR_BODY72]]:
+// CHECK2-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    store ptr [[TMP61]], ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP64:%.*]] = load double, ptr [[TMP63]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP62]], double noundef [[TMP64]])
+// CHECK2-NEXT:    br label %[[FOR_INC73:.*]]
+// CHECK2:       [[FOR_INC73]]:
+// CHECK2-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw double, ptr [[TMP65]], i32 1
+// CHECK2-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND70]]
+// CHECK2:       [[FOR_END74]]:
+// CHECK2-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define dso_local void @tfoo2(
 // CHECK2-SAME: ) #[[ATTR0]] {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
@@ -1593,7 +2097,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
@@ -1602,7 +2105,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -1611,7 +2113,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP21:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i32, align 4
@@ -1641,174 +2142,168 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
 // CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP8]], 1
 // CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP9]], ptr [[J]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP13]], [[TMP14]]
 // CHECK2-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP15]]
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP16]]
 // CHECK2-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
 // CHECK2-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
 // CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
+// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
 // CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
 // CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTCAPTURE_EXPR_19]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[DOTNEW_STEP21]], align 4
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP24]], ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP25]], [[TMP26]]
 // CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP29]]
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP30]]
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP27]]
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP28]]
 // CHECK2-NEXT:    [[SUB27:%.*]] = sub i32 [[DIV26]], 1
 // CHECK2-NEXT:    store i32 [[SUB27]], ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK2-NEXT:    store i32 [[TMP31]], ptr [[DOTOMP_UB2]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB2]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST2]], align 4
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK2-NEXT:    [[ADD28:%.*]] = add i32 [[TMP32]], 1
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK2-NEXT:    [[ADD28:%.*]] = add i32 [[TMP29]], 1
 // CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_NI2]], align 4
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    store i32 [[TMP33]], ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP34]], [[TMP35]]
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP30]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP31]], [[TMP32]]
 // CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // CHECK2:       [[COND_TRUE]]:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
 // CHECK2-NEXT:    br label %[[COND_END:.*]]
 // CHECK2:       [[COND_FALSE]]:
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
 // CHECK2-NEXT:    br label %[[COND_END]]
 // CHECK2:       [[COND_END]]:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP36]], %[[COND_TRUE]] ], [ [[TMP37]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP33]], %[[COND_TRUE]] ], [ [[TMP34]], %[[COND_FALSE]] ]
 // CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_TEMP_2]], align 4
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
-// CHECK2-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP35]], [[TMP36]]
 // CHECK2-NEXT:    br i1 [[CMP29]], label %[[COND_TRUE30:.*]], label %[[COND_FALSE31:.*]]
 // CHECK2:       [[COND_TRUE30]]:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
 // CHECK2-NEXT:    br label %[[COND_END32:.*]]
 // CHECK2:       [[COND_FALSE31]]:
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
 // CHECK2-NEXT:    br label %[[COND_END32]]
 // CHECK2:       [[COND_END32]]:
-// CHECK2-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP40]], %[[COND_TRUE30]] ], [ [[TMP41]], %[[COND_FALSE31]] ]
+// CHECK2-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP37]], %[[COND_TRUE30]] ], [ [[TMP38]], %[[COND_FALSE31]] ]
 // CHECK2-NEXT:    store i32 [[COND33]], ptr [[DOTOMP_FUSE_MAX]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK2-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK2:       [[FOR_COND]]:
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
-// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP39]], [[TMP40]]
 // CHECK2-NEXT:    br i1 [[CMP34]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK2:       [[FOR_BODY]]:
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP44]], [[TMP45]]
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP41]], [[TMP42]]
 // CHECK2-NEXT:    br i1 [[CMP35]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // CHECK2:       [[IF_THEN]]:
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP47]], [[TMP48]]
-// CHECK2-NEXT:    [[ADD36:%.*]] = add i32 [[TMP46]], [[MUL]]
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP44]], [[TMP45]]
+// CHECK2-NEXT:    [[ADD36:%.*]] = add i32 [[TMP43]], [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD36]], ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP50]], [[TMP51]]
-// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[TMP49]], [[MUL37]]
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP47]], [[TMP48]]
+// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[TMP46]], [[MUL37]]
 // CHECK2-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP49]])
 // CHECK2-NEXT:    br label %[[IF_END]]
 // CHECK2:       [[IF_END]]:
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP53]], [[TMP54]]
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP50]], [[TMP51]]
 // CHECK2-NEXT:    br i1 [[CMP39]], label %[[IF_THEN40:.*]], label %[[IF_END45:.*]]
 // CHECK2:       [[IF_THEN40]]:
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP56]], [[TMP57]]
-// CHECK2-NEXT:    [[ADD42:%.*]] = add i32 [[TMP55]], [[MUL41]]
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP53]], [[TMP54]]
+// CHECK2-NEXT:    [[ADD42:%.*]] = add i32 [[TMP52]], [[MUL41]]
 // CHECK2-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP59]], [[TMP60]]
-// CHECK2-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP58]], [[MUL43]]
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP55]], [[MUL43]]
 // CHECK2-NEXT:    store i32 [[SUB44]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP61]])
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP58]])
 // CHECK2-NEXT:    br label %[[IF_END45]]
 // CHECK2:       [[IF_END45]]:
-// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
-// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP62]], [[TMP63]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP59]], [[TMP60]]
 // CHECK2-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
 // CHECK2:       [[IF_THEN47]]:
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP65]], [[TMP66]]
-// CHECK2-NEXT:    [[ADD49:%.*]] = add i32 [[TMP64]], [[MUL48]]
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP62]], [[TMP63]]
+// CHECK2-NEXT:    [[ADD49:%.*]] = add i32 [[TMP61]], [[MUL48]]
 // CHECK2-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV2]], align 4
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP68]], [[TMP69]]
-// CHECK2-NEXT:    [[ADD51:%.*]] = add i32 [[TMP67]], [[MUL50]]
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP65]], [[TMP66]]
+// CHECK2-NEXT:    [[ADD51:%.*]] = add i32 [[TMP64]], [[MUL50]]
 // CHECK2-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP70]])
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP67]])
 // CHECK2-NEXT:    br label %[[IF_END52]]
 // CHECK2:       [[IF_END52]]:
 // CHECK2-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK2:       [[FOR_INC]]:
-// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP68]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       [[FOR_END]]:
 // CHECK2-NEXT:    ret void
 //
@@ -1819,6 +2314,8 @@ extern "C" void foo4() {
 // CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
 // CHECK1: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
 // CHECK1: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
+// CHECK1: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
+// CHECK1: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
 //.
 // CHECK2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
 // CHECK2: [[META4]] = !{!"llvm.loop.mustprogress"}
@@ -1826,4 +2323,6 @@ extern "C" void foo4() {
 // CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
 // CHECK2: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
 // CHECK2: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
+// CHECK2: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
+// CHECK2: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
 //.

>From 860fcd94d930c9644b4d0427471f2873e7afcf8b Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:44:48 +0000
Subject: [PATCH 05/12] Fixed missing diagnostic groups in warnings

---
 clang/include/clang/Basic/DiagnosticSemaKinds.td | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 002aa7a774fbe..e85cd32d78b5c 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11613,7 +11613,8 @@ def note_omp_implicit_dsa : Note<
 def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
 def warn_omp_different_loop_ind_var_types : Warning <
-  "loop sequence following '#pragma omp %0' contains induction variables of differing types: %1 and %2">;
+  "loop sequence following '#pragma omp %0' contains induction variables of differing types: %1 and %2">,
+  InGroup<OpenMPLoopForm>;
 def err_omp_not_canonical_loop : Error <
   "loop after '#pragma omp %0' is not in canonical form">;
 def err_omp_not_a_loop_sequence : Error < 
@@ -11624,7 +11625,8 @@ def err_omp_invalid_looprange : Error <
   "loop range in '#pragma omp %0' exceeds the number of available loops: "
   "range end '%1' is greater than the total number of loops '%2'">;
 def warn_omp_redundant_fusion : Warning <
-  "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">;
+  "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">,
+  InGroup<OpenMPClauses>;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;

>From 65cbfeb945e6b8016696906db43dd590adb285b2 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:49:50 +0000
Subject: [PATCH 06/12] Fixed formatting and comments

---
 clang/lib/Sema/SemaOpenMP.cpp | 112 ++++++++++++++++++----------------
 1 file changed, 58 insertions(+), 54 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 3ce256f3ec23b..2985b256cf153 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14197,42 +14197,43 @@ StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
 }
 
 // Overloaded base case function
-template <typename T, typename F>
-static bool tryHandleAs(T *t, F &&) {
-    return false; 
+template <typename T, typename F> static bool tryHandleAs(T *t, F &&) {
+  return false;
 }
 
 /**
- * Tries to recursively cast `t` to one of the given types and invokes `f` if successful.
+ * Tries to recursively cast `t` to one of the given types and invokes `f` if
+ * successful.
  *
  * @tparam Class The first type to check.
  * @tparam Rest The remaining types to check.
  * @tparam T The base type of `t`.
- * @tparam F The callable type for the function to invoke upon a successful cast.
+ * @tparam F The callable type for the function to invoke upon a successful
+ * cast.
  * @param t The object to be checked.
  * @param f The function to invoke if `t` matches `Class`.
  * @return `true` if `t` matched any type and `f` was called, otherwise `false`.
  */
 template <typename Class, typename... Rest, typename T, typename F>
 static bool tryHandleAs(T *t, F &&f) {
-    if (Class *c = dyn_cast<Class>(t)) {
-        f(c); 
-        return true;
-    } else {
-        return tryHandleAs<Rest...>(t, std::forward<F>(f));
-    }
+  if (Class *c = dyn_cast<Class>(t)) {
+    f(c);
+    return true;
+  } else {
+    return tryHandleAs<Rest...>(t, std::forward<F>(f));
+  }
 }
 
 // Updates OriginalInits by checking Transform against loop transformation
 // directives and appending their pre-inits if a match is found.
 static void updatePreInits(OMPLoopBasedDirective *Transform,
                            SmallVectorImpl<SmallVector<Stmt *, 0>> &PreInits) {
-    if (!tryHandleAs<OMPTileDirective, OMPUnrollDirective, OMPReverseDirective,
-                     OMPInterchangeDirective, OMPFuseDirective>(
-            Transform, [&PreInits](auto *Dir) {
-              appendFlattenedStmtList(PreInits.back(), Dir->getPreInits());
-            }))
-        llvm_unreachable("Unhandled loop transformation");
+  if (!tryHandleAs<OMPTileDirective, OMPUnrollDirective, OMPReverseDirective,
+                   OMPInterchangeDirective, OMPFuseDirective>(
+          Transform, [&PreInits](auto *Dir) {
+            appendFlattenedStmtList(PreInits.back(), Dir->getPreInits());
+          }))
+    llvm_unreachable("Unhandled loop transformation");
 }
 
 bool SemaOpenMP::checkTransformableLoopNest(
@@ -14310,43 +14311,42 @@ class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
   unsigned getNestedLoopCount() const { return NestedLoopCount; }
 
   bool VisitForStmt(ForStmt *FS) override {
-        ++NestedLoopCount;
-        return true;
+    ++NestedLoopCount;
+    return true;
   }
 
   bool VisitCXXForRangeStmt(CXXForRangeStmt *FRS) override {
-        ++NestedLoopCount;
-        return true;
+    ++NestedLoopCount;
+    return true;
   }
 
   bool TraverseStmt(Stmt *S) override {
-        if (!S)
+    if (!S)
       return true;
 
-        // Skip traversal of all expressions, including special cases like
-        // LambdaExpr, StmtExpr, BlockExpr, and RequiresExpr. These expressions
-        // may contain inner statements (and even loops), but they are not part
-        // of the syntactic body of the surrounding loop structure.
-        //  Therefore must not be counted
-        if (isa<Expr>(S))
+    // Skip traversal of all expressions, including special cases like
+    // LambdaExpr, StmtExpr, BlockExpr, and RequiresExpr. These expressions
+    // may contain inner statements (and even loops), but they are not part
+    // of the syntactic body of the surrounding loop structure.
+    //  Therefore must not be counted
+    if (isa<Expr>(S))
       return true;
 
-        // Only recurse into CompoundStmt (block {}) and loop bodies
-        if (isa<CompoundStmt>(S) || isa<ForStmt>(S) ||
-            isa<CXXForRangeStmt>(S)) {
+    // Only recurse into CompoundStmt (block {}) and loop bodies
+    if (isa<CompoundStmt>(S) || isa<ForStmt>(S) || isa<CXXForRangeStmt>(S)) {
       return DynamicRecursiveASTVisitor::TraverseStmt(S);
-        }
+    }
 
-        // Stop traversal of the rest of statements, that break perfect
-        // loop nesting, such as control flow (IfStmt, SwitchStmt...)
-        return true;
+    // Stop traversal of the rest of statements, that break perfect
+    // loop nesting, such as control flow (IfStmt, SwitchStmt...)
+    return true;
   }
 
   bool TraverseDecl(Decl *D) override {
-        // Stop in the case of finding a declaration, it is not important
-        // in order to find nested loops (Possible CXXRecordDecl, RecordDecl,
-        // FunctionDecl...)
-        return true;
+    // Stop in the case of finding a declaration, it is not important
+    // in order to find nested loops (Possible CXXRecordDecl, RecordDecl,
+    // FunctionDecl...)
+    return true;
   }
 };
 
@@ -14504,15 +14504,14 @@ bool SemaOpenMP::analyzeLoopSequence(
     return isa<OMPLoopTransformationDirective>(Child);
   };
 
-
   // High level grammar validation
   for (auto *Child : LoopSeqStmt->children()) {
 
-        if (!Child)
+    if (!Child)
       continue;
 
-        // Skip over non-loop-sequence statements
-        if (!isLoopSequenceDerivation(Child)) {
+    // Skip over non-loop-sequence statements
+    if (!isLoopSequenceDerivation(Child)) {
       Child = Child->IgnoreContainers();
 
       // Ignore empty compound statement
@@ -14530,9 +14529,9 @@ bool SemaOpenMP::analyzeLoopSequence(
         // Already been treated, skip this children
         continue;
       }
-        }
-        // Regular loop sequence handling
-        if (isLoopSequenceDerivation(Child)) {
+    }
+    // Regular loop sequence handling
+    if (isLoopSequenceDerivation(Child)) {
       if (isLoopGeneratingStmt(Child)) {
         if (!analyzeLoopGeneration(Child)) {
           return false;
@@ -14546,12 +14545,12 @@ bool SemaOpenMP::analyzeLoopSequence(
         // Update the Loop Sequence size by one
         ++LoopSeqSize;
       }
-        } else {
+    } else {
       // Report error for invalid statement inside canonical loop sequence
       Diag(Child->getBeginLoc(), diag::err_omp_not_for)
           << 0 << getOpenMPDirectiveName(Kind);
       return false;
-        }
+    }
   }
   return true;
 }
@@ -14568,9 +14567,9 @@ bool SemaOpenMP::checkTransformableLoopSequence(
 
   // Checks whether the given statement is a compound statement
   if (!isa<CompoundStmt>(AStmt)) {
-        Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
-            << getOpenMPDirectiveName(Kind);
-        return false;
+    Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
+        << getOpenMPDirectiveName(Kind);
+    return false;
   }
   // Number of top level canonical loop nests observed (And acts as index)
   LoopSeqSize = 0;
@@ -14601,7 +14600,7 @@ bool SemaOpenMP::checkTransformableLoopSequence(
                            OriginalInits, TransformsPreInits,
                            LoopSequencePreInits, LoopCategories, Context,
                            Kind)) {
-        return false;
+    return false;
   }
   if (LoopSeqSize <= 0) {
     Diag(AStmt->getBeginLoc(), diag::err_omp_empty_loop_sequence)
@@ -15315,7 +15314,7 @@ StmtResult SemaOpenMP::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   Stmt *LoopStmt = nullptr;
   collectLoopStmts(AStmt, {LoopStmt});
 
-  // Determine the PreInit declarations.e
+  // Determine the PreInit declarations.
   SmallVector<Stmt *, 4> PreInits;
   addLoopPreInits(Context, LoopHelper, LoopStmt, OriginalInits[0], PreInits);
 
@@ -15931,13 +15930,18 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     CountVal = CountInt.getZExtValue();
   };
 
-  // Checks if the loop range is valid
+  // OpenMP [6.0, Restrictions]
+  // first + count - 1 must not evaluate to a value greater than the
+  // loop sequence length of the associated canonical loop sequence.
   auto ValidLoopRange = [](uint64_t FirstVal, uint64_t CountVal,
                            unsigned NumLoops) -> bool {
     return FirstVal + CountVal - 1 <= NumLoops;
   };
   uint64_t FirstVal = 1, CountVal = 0, LastVal = LoopSeqSize;
 
+  // Validates the loop range after evaluating the semantic information
+  // and ensures that the range is valid for the given loop sequence size.
+  // Expressions are evaluated at compile time to obtain constant values.
   if (LRC) {
     EvaluateLoopRangeArguments(LRC->getFirst(), LRC->getCount(), FirstVal,
                                CountVal);

>From b0fb1b3e26f1d9ceaac4495dcfad84f54f96d2a2 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:58:54 +0000
Subject: [PATCH 07/12] Added minimal changes to enable flang future
 implementation

---
 flang/include/flang/Parser/dump-parse-tree.h | 1 +
 flang/include/flang/Parser/parse-tree.h      | 9 +++++++++
 flang/lib/Lower/OpenMP/Clauses.cpp           | 5 +++++
 flang/lib/Lower/OpenMP/Clauses.h             | 1 +
 flang/lib/Parser/openmp-parsers.cpp          | 7 +++++++
 flang/lib/Parser/unparse.cpp                 | 7 +++++++
 flang/lib/Semantics/check-omp-structure.cpp  | 9 +++++++++
 llvm/include/llvm/Frontend/OpenMP/OMP.td     | 1 +
 8 files changed, 40 insertions(+)

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..76aa3f7b90156 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -600,6 +600,7 @@ class ParseTreeDumper {
   NODE(OmpLinearClause, Modifier)
   NODE(parser, OmpLinearModifier)
   NODE_ENUM(OmpLinearModifier, Value)
+  NODE(parser, OmpLoopRangeClause)
   NODE(parser, OmpStepComplexModifier)
   NODE(parser, OmpStepSimpleModifier)
   NODE(parser, OmpLoopDirective)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 61f97b855b0e5..d32db62db2628 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4367,6 +4367,15 @@ struct OmpLinearClause {
   std::tuple<OmpObjectList, MODIFIERS(), /*PostModified=*/bool> t;
 };
 
+// Ref: [6.0:207-208]
+//
+// loop-range-clause ->
+//    LOOPRANGE(first, count)                       // since 6.0
+struct OmpLoopRangeClause {
+  TUPLE_CLASS_BOILERPLATE(OmpLoopRangeClause);
+  std::tuple<ScalarIntConstantExpr, ScalarIntConstantExpr> t;
+};
+
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-clause ->
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index b599d69a36272..a38249bf2b588 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -997,6 +997,11 @@ Link make(const parser::OmpClause::Link &inp,
   return Link{/*List=*/makeObjects(inp.v, semaCtx)};
 }
 
+LoopRange make(const parser::OmpClause::Looprange &inp,
+            semantics::SemanticsContext &semaCtx) {
+  llvm_unreachable("Unimplemented: looprange");
+}
+
 Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index d7ab21d428e32..bda8571e65f23 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -239,6 +239,7 @@ using Initializer = tomp::clause::InitializerT<TypeTy, IdTy, ExprTy>;
 using InReduction = tomp::clause::InReductionT<TypeTy, IdTy, ExprTy>;
 using IsDevicePtr = tomp::clause::IsDevicePtrT<TypeTy, IdTy, ExprTy>;
 using Lastprivate = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
+using LoopRange = tomp::clause::LoopRangeT<TypeTy, IdTy, ExprTy>;
 using Linear = tomp::clause::LinearT<TypeTy, IdTy, ExprTy>;
 using Link = tomp::clause::LinkT<TypeTy, IdTy, ExprTy>;
 using Map = tomp::clause::MapT<TypeTy, IdTy, ExprTy>;
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..d53389746dbec 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -847,6 +847,11 @@ TYPE_PARSER(
         maybe(":"_tok >> nonemptyList(Parser<OmpLinearClause::Modifier>{})),
         /*PostModified=*/pure(true)))
 
+TYPE_PARSER(
+  construct<OmpLoopRangeClause>(scalarIntConstantExpr,
+                                "," >> scalarIntConstantExpr)
+)
+
 // OpenMPv5.2 12.5.2 detach-clause -> DETACH (event-handle)
 TYPE_PARSER(construct<OmpDetachClause>(Parser<OmpObject>{}))
 
@@ -1020,6 +1025,8 @@ TYPE_PARSER( //
                     parenthesized(Parser<OmpLinearClause>{}))) ||
     "LINK" >> construct<OmpClause>(construct<OmpClause::Link>(
                   parenthesized(Parser<OmpObjectList>{}))) ||
+    "LOOPRANGE" >> construct<OmpClause>(construct<OmpClause::Looprange>(
+                  parenthesized(Parser<OmpLoopRangeClause>{}))) ||
     "MAP" >> construct<OmpClause>(construct<OmpClause::Map>(
                  parenthesized(Parser<OmpMapClause>{}))) ||
     "MATCH" >> construct<OmpClause>(construct<OmpClause::Match>(
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ed0f227fd5b98..18e8a63ca68aa 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2318,6 +2318,13 @@ class UnparseVisitor {
       }
     }
   }
+  void Unparse(const OmpLoopRangeClause &x) {
+    Word("LOOPRANGE(");
+    Walk(std::get<0>(x.t));
+    Put(", ");
+    Walk(std::get<1>(x.t));
+    Put(")");
+  }
   void Unparse(const OmpReductionClause &x) {
     using Modifier = OmpReductionClause::Modifier;
     Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 68cea6739830d..2a03c6a1fd0e4 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -4446,6 +4446,15 @@ CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(Collapse, OMPC_collapse)
 CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(Safelen, OMPC_safelen)
 CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(Simdlen, OMPC_simdlen)
 
+void OmpStructureChecker::Enter(const parser::OmpClause::Looprange &x) {
+  context_.Say(GetContext().clauseSource,
+      "LOOPRANGE clause is not implemented yet"_err_en_US,
+      ContextDirectiveAsFortran());
+}
+
+void OmpStructureChecker::Enter(const parser::OmpClause::FreeAgent &x) {
+  context_.Say(GetContext().clauseSource,
+      "FREE_AGENT clause is not implemented yet"_err_en_US,
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 989b35a7caa2a..f8acdc62aba3d 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -276,6 +276,7 @@ def OMPC_Link : Clause<[Spelling<"link">]> {
 }
 def OMPC_LoopRange : Clause<[Spelling<"looprange">]> {
   let clangClass = "OMPLoopRangeClause";
+  let flangClass = "OmpLoopRangeClause";
 }
 def OMPC_Map : Clause<[Spelling<"map">]> {
   let clangClass = "OMPMapClause";

>From b252aa910ef7c5c278a86bd7195bbf3bb18dd18d Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Wed, 21 May 2025 13:14:22 +0000
Subject: [PATCH 08/12] Address basic PR feedback

---
 clang/include/clang/AST/OpenMPClause.h      |  93 ++++----
 clang/include/clang/AST/StmtOpenMP.h        |   2 +-
 clang/include/clang/Sema/SemaOpenMP.h       |  14 +-
 clang/lib/AST/OpenMPClause.cpp              |  17 +-
 clang/lib/CodeGen/CGExpr.cpp                |   5 +-
 clang/lib/CodeGen/CodeGenFunction.h         |   4 -
 clang/lib/Sema/SemaOpenMP.cpp               | 224 +++++++++-----------
 flang/lib/Semantics/check-omp-structure.cpp |   3 -
 8 files changed, 166 insertions(+), 196 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8f937cdef9cd0..3df5133a17fb4 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1153,82 +1153,73 @@ class OMPFullClause final : public OMPNoChildClause<llvm::omp::OMPC_full> {
 ///   for(int j = 0; j < 256; j+=2)
 ///   for(int k = 127; k >= 0; --k)
 /// \endcode
-class OMPLoopRangeClause final : public OMPClause {
+class OMPLoopRangeClause final
+    : public OMPClause,
+      private llvm::TrailingObjects<OMPLoopRangeClause, Expr *> {
   friend class OMPClauseReader;
-
-  explicit OMPLoopRangeClause()
-      : OMPClause(llvm::omp::OMPC_looprange, {}, {}) {}
+  friend class llvm::TrailingObjects<OMPLoopRangeClause, Expr *>;
 
   /// Location of '('
   SourceLocation LParenLoc;
 
-  /// Location of 'first'
-  SourceLocation FirstLoc;
-
-  /// Location of 'count'
-  SourceLocation CountLoc;
-
-  /// Expr associated with 'first' argument
-  Expr *First = nullptr;
-
-  /// Expr associated with 'count' argument
-  Expr *Count = nullptr;
-
-  /// Set 'first'
-  void setFirst(Expr *First) { this->First = First; }
+  /// Location of first and count expressions
+  SourceLocation FirstLoc, CountLoc;
 
-  /// Set 'count'
-  void setCount(Expr *Count) { this->Count = Count; }
+  /// Number of looprange arguments (always 2: first, count)
+  unsigned NumArgs = 2;
 
-  /// Set location of '('.
-  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
-
-  /// Set location of 'first' argument
-  void setFirstLoc(SourceLocation Loc) { FirstLoc = Loc; }
+  /// Set the argument expressions.
+  void setArgs(ArrayRef<Expr *> Args) {
+    assert(Args.size() == NumArgs && "Expected exactly 2 looprange arguments");
+    std::copy(Args.begin(), Args.end(), getTrailingObjects<Expr *>());
+  }
 
-  /// Set location of 'count' argument
-  void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
+  /// Build an empty clause for deserialization.
+  explicit OMPLoopRangeClause()
+      : OMPClause(llvm::omp::OMPC_looprange, {}, {}), NumArgs(2) {}
 
 public:
-  /// Build an AST node for a 'looprange' clause
-  ///
-  /// \param StartLoc     Starting location of the clause.
-  /// \param LParenLoc    Location of '('.
-  /// \param ModifierLoc  Modifier location.
-  /// \param
+  /// Build a 'looprange' clause AST node.
   static OMPLoopRangeClause *
   Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
          SourceLocation FirstLoc, SourceLocation CountLoc,
-         SourceLocation EndLoc, Expr *First, Expr *Count);
+         SourceLocation EndLoc, ArrayRef<Expr *> Args);
 
-  /// Build an empty 'looprange' node for deserialization
-  ///
-  /// \param C      Context of the AST.
+  /// Build an empty 'looprange' clause node.
   static OMPLoopRangeClause *CreateEmpty(const ASTContext &C);
 
-  /// Returns the location of '('
+  // Location getters/setters
   SourceLocation getLParenLoc() const { return LParenLoc; }
-
-  /// Returns the location of 'first'
   SourceLocation getFirstLoc() const { return FirstLoc; }
-
-  /// Returns the location of 'count'
   SourceLocation getCountLoc() const { return CountLoc; }
 
-  /// Returns the argument 'first' or nullptr if not set
-  Expr *getFirst() const { return cast_or_null<Expr>(First); }
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+  void setFirstLoc(SourceLocation Loc) { FirstLoc = Loc; }
+  void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
 
-  /// Returns the argument 'count' or nullptr if not set
-  Expr *getCount() const { return cast_or_null<Expr>(Count); }
+  /// Get looprange arguments: first and count
+  Expr *getFirst() const { return getArgs()[0]; }
+  Expr *getCount() const { return getArgs()[1]; }
 
-  child_range children() {
-    return child_range(reinterpret_cast<Stmt **>(&First),
-                       reinterpret_cast<Stmt **>(&Count) + 1);
+  /// Set looprange arguments: first and count
+  void setFirst(Expr *E) { getArgs()[0] = E; }
+  void setCount(Expr *E) { getArgs()[1] = E; }
+
+  MutableArrayRef<Expr *> getArgs() {
+    return MutableArrayRef<Expr *>(getTrailingObjects<Expr *>(), NumArgs);
+  }
+  ArrayRef<Expr *> getArgs() const {
+    return ArrayRef<Expr *>(getTrailingObjects<Expr *>(), NumArgs);
   }
 
+  child_range children() {
+    return child_range(reinterpret_cast<Stmt **>(getArgs().begin()),
+                       reinterpret_cast<Stmt **>(getArgs().end()));
+  }
   const_child_range children() const {
-    auto Children = const_cast<OMPLoopRangeClause *>(this)->children();
-    return const_child_range(Children.begin(), Children.end());
+    auto AR = getArgs();
+    return const_child_range(reinterpret_cast<Stmt *const *>(AR.begin()),
+                             reinterpret_cast<Stmt *const *>(AR.end()));
   }
 
   child_range used_children() {
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 6425f6616a558..0421c06245cac 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5883,7 +5883,7 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
                                        EndLoc, NumLoops) {
     // Interchange produces a single top-level canonical loop
     // nest, with the exact same amount of total loops
-    setNumGeneratedLoops(NumLoops);
+    setNumGeneratedLoops(3 * NumLoops);
     setNumGeneratedLoopNests(1);
   }
 
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 547ea95c6cd5d..f848c4a7d715e 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1492,7 +1492,7 @@ class SemaOpenMP : public SemaBase {
   bool checkTransformableLoopNest(
       OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
-      Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits);
+      Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits);
 
   /// @brief Categories of loops encountered during semantic OpenMP loop
   /// analysis
@@ -1555,9 +1555,9 @@ class SemaOpenMP : public SemaBase {
       Stmt *LoopSeqStmt, unsigned &LoopSeqSize, unsigned &NumLoops,
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       SmallVectorImpl<Stmt *> &ForStmts,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &TransformsPreInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &LoopSequencePreInits,
       SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context,
       OpenMPDirectiveKind Kind);
 
@@ -1591,9 +1591,9 @@ class SemaOpenMP : public SemaBase {
       unsigned &NumLoops,
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       SmallVectorImpl<Stmt *> &ForStmts,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &TransformsPreInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &LoopSequencePreInits,
       SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context);
 
   /// Helper to keep information about the current `omp begin/end declare
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 0b5808eb100e4..e0570262b2a05 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1026,22 +1026,25 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
 
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
-                           SourceLocation LParenLoc, SourceLocation EndLoc,
-                           SourceLocation FirstLoc, SourceLocation CountLoc,
-                           Expr *First, Expr *Count) {
+                           SourceLocation LParenLoc, SourceLocation FirstLoc,
+                           SourceLocation CountLoc, SourceLocation EndLoc,
+                           ArrayRef<Expr *> Args) {
+
+  assert(Args.size() == 2 &&
+         "looprange clause must have exactly two arguments");
   OMPLoopRangeClause *Clause = CreateEmpty(C);
   Clause->setLocStart(StartLoc);
   Clause->setLParenLoc(LParenLoc);
-  Clause->setLocEnd(EndLoc);
   Clause->setFirstLoc(FirstLoc);
   Clause->setCountLoc(CountLoc);
-  Clause->setFirst(First);
-  Clause->setCount(Count);
+  Clause->setLocEnd(EndLoc);
+  Clause->setArgs(Args);
   return Clause;
 }
 
 OMPLoopRangeClause *OMPLoopRangeClause::CreateEmpty(const ASTContext &C) {
-  return new (C) OMPLoopRangeClause();
+  void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(2));
+  return new (Mem) OMPLoopRangeClause();
 }
 
 OMPAllocateClause *OMPAllocateClause::Create(
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 08049d4d4e37d..f983b88eb61ec 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3256,11 +3256,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
           var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
 
     // No other cases for now.
-    } else {
-      llvm::dbgs() << "THE DAMN DECLREFEXPR HASN'T BEEN ENTERED IN LOCALDECLMAP\n";
-      VD->dumpColor();
+    } else
       llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
-    }
 
     // Handle threadlocal function locals.
     if (VD->getTLSKind() != VarDecl::TLS_None)
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index bfe24213ed377..fe753e5b688b1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5363,10 +5363,6 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// Set the address of a local variable.
   void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
-    if (LocalDeclMap.count(VD)) {
-      llvm::errs() << "Warning: VarDecl already exists in map: ";
-      VD->dumpColor(); 
-    }
     assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
     LocalDeclMap.insert({VD, Addr});
   }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 2985b256cf153..9819dcfe60360 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14196,38 +14196,37 @@ StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
       getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-// Overloaded base case function
+/// Overloaded base case function
 template <typename T, typename F> static bool tryHandleAs(T *t, F &&) {
   return false;
 }
 
-/**
- * Tries to recursively cast `t` to one of the given types and invokes `f` if
- * successful.
- *
- * @tparam Class The first type to check.
- * @tparam Rest The remaining types to check.
- * @tparam T The base type of `t`.
- * @tparam F The callable type for the function to invoke upon a successful
- * cast.
- * @param t The object to be checked.
- * @param f The function to invoke if `t` matches `Class`.
- * @return `true` if `t` matched any type and `f` was called, otherwise `false`.
- */
+///
+/// Tries to recursively cast `t` to one of the given types and invokes `f` if
+/// successful.
+///
+/// @tparam Class The first type to check.
+/// @tparam Rest The remaining types to check.
+/// @tparam T The base type of `t`.
+/// @tparam F The callable type for the function to invoke upon a successful
+/// cast.
+/// @param t The object to be checked.
+/// @param f The function to invoke if `t` matches `Class`.
+/// @return `true` if `t` matched any type and `f` was called, otherwise
+/// `false`.
 template <typename Class, typename... Rest, typename T, typename F>
 static bool tryHandleAs(T *t, F &&f) {
   if (Class *c = dyn_cast<Class>(t)) {
     f(c);
     return true;
-  } else {
-    return tryHandleAs<Rest...>(t, std::forward<F>(f));
   }
+  return tryHandleAs<Rest...>(t, std::forward<F>(f));
 }
 
-// Updates OriginalInits by checking Transform against loop transformation
-// directives and appending their pre-inits if a match is found.
+/// Updates OriginalInits by checking Transform against loop transformation
+/// directives and appending their pre-inits if a match is found.
 static void updatePreInits(OMPLoopBasedDirective *Transform,
-                           SmallVectorImpl<SmallVector<Stmt *, 0>> &PreInits) {
+                           SmallVectorImpl<SmallVector<Stmt *>> &PreInits) {
   if (!tryHandleAs<OMPTileDirective, OMPUnrollDirective, OMPReverseDirective,
                    OMPInterchangeDirective, OMPFuseDirective>(
           Transform, [&PreInits](auto *Dir) {
@@ -14239,7 +14238,7 @@ static void updatePreInits(OMPLoopBasedDirective *Transform,
 bool SemaOpenMP::checkTransformableLoopNest(
     OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
-    Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits) {
+    Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits) {
   OriginalInits.emplace_back();
   bool Result = OMPLoopBasedDirective::doForAllLoops(
       AStmt->IgnoreContainers(), /*TryImperfectlyNestedLoops=*/false, NumLoops,
@@ -14273,40 +14272,40 @@ bool SemaOpenMP::checkTransformableLoopNest(
   return Result;
 }
 
-// Counts the total number of nested loops, including the outermost loop (the
-// original loop). PRECONDITION of this visitor is that it must be invoked from
-// the original loop to be analyzed. The traversal is stop for Decl's and
-// Expr's given that they may contain inner loops that must not be counted.
-//
-// Example AST structure for the code:
-//
-// int main() {
-//     #pragma omp fuse
-//     {
-//         for (int i = 0; i < 100; i++) {    <-- Outer loop
-//             []() {
-//                 for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
-//             };
-//             for(int j = 0; j < 5; ++j) {}    <-- Inner loop
-//         }
-//         for (int r = 0; i < 100; i++) {    <-- Outer loop
-//             struct LocalClass {
-//                 void bar() {
-//                     for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
-//                 }
-//             };
-//             for(int k = 0; k < 10; ++k) {}    <-- Inner loop
-//             {x = 5; for(k = 0; k < 10; ++k) x += k; x}; <-- NOT A LOOP
-//         }
-//     }
-// }
-// Result: Loop 'i' contains 2 loops, Loop 'r' also contains 2 loops
+/// Counts the total number of nested loops, including the outermost loop (the
+/// original loop). PRECONDITION of this visitor is that it must be invoked from
+/// the original loop to be analyzed. The traversal is stop for Decl's and
+/// Expr's given that they may contain inner loops that must not be counted.
+///
+/// Example AST structure for the code:
+///
+/// int main() {
+///     #pragma omp fuse
+///     {
+///         for (int i = 0; i < 100; i++) {    <-- Outer loop
+///             []() {
+///                 for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
+///             };
+///             for(int j = 0; j < 5; ++j) {}    <-- Inner loop
+///         }
+///         for (int r = 0; i < 100; i++) {    <-- Outer loop
+///             struct LocalClass {
+///                 void bar() {
+///                     for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
+///                 }
+///             };
+///             for(int k = 0; k < 10; ++k) {}    <-- Inner loop
+///             {x = 5; for(k = 0; k < 10; ++k) x += k; x}; <-- NOT A LOOP
+///         }
+///     }
+/// }
+/// Result: Loop 'i' contains 2 loops, Loop 'r' also contains 2 loops
 class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
 private:
   unsigned NestedLoopCount = 0;
 
 public:
-  explicit NestedLoopCounterVisitor() {}
+  explicit NestedLoopCounterVisitor() = default;
 
   unsigned getNestedLoopCount() const { return NestedLoopCount; }
 
@@ -14333,7 +14332,7 @@ class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
       return true;
 
     // Only recurse into CompoundStmt (block {}) and loop bodies
-    if (isa<CompoundStmt>(S) || isa<ForStmt>(S) || isa<CXXForRangeStmt>(S)) {
+    if (isa<CompoundStmt, ForStmt, CXXForRangeStmt>(S)) {
       return DynamicRecursiveASTVisitor::TraverseStmt(S);
     }
 
@@ -14354,19 +14353,18 @@ bool SemaOpenMP::analyzeLoopSequence(
     Stmt *LoopSeqStmt, unsigned &LoopSeqSize, unsigned &NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
     SmallVectorImpl<Stmt *> &ForStmts,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &TransformsPreInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &LoopSequencePreInits,
     SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context,
     OpenMPDirectiveKind Kind) {
 
   VarsWithInheritedDSAType TmpDSA;
   QualType BaseInductionVarType;
-  // Helper Lambda to handle storing initialization and body statements for both
-  // ForStmt and CXXForRangeStmt and checks for any possible mismatch between
-  // induction variables types
-  auto storeLoopStatements = [&OriginalInits, &ForStmts, &BaseInductionVarType,
-                              this, &Context](Stmt *LoopStmt) {
+  /// Helper Lambda to handle storing initialization and body statements for
+  /// both ForStmt and CXXForRangeStmt and checks for any possible mismatch
+  /// between induction variables types
+  auto StoreLoopStatements = [&](Stmt *LoopStmt) {
     if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
       OriginalInits.back().push_back(For->getInit());
       ForStmts.push_back(For);
@@ -14394,16 +14392,11 @@ bool SemaOpenMP::analyzeLoopSequence(
     }
   };
 
-  // Helper lambda functions to encapsulate the processing of different
-  // derivations of the canonical loop sequence grammar
-  //
-  // Modularized code for handling loop generation and transformations
-  auto analyzeLoopGeneration = [&storeLoopStatements, &LoopHelpers,
-                                &OriginalInits, &TransformsPreInits,
-                                &LoopCategories, &LoopSeqSize, &NumLoops, Kind,
-                                &TmpDSA, &ForStmts, &Context,
-                                &LoopSequencePreInits, this](Stmt *Child) {
-    auto LoopTransform = dyn_cast<OMPLoopTransformationDirective>(Child);
+  /// Helper lambda functions to encapsulate the processing of different
+  /// derivations of the canonical loop sequence grammar
+  /// Modularized code for handling loop generation and transformations
+  auto AnalyzeLoopGeneration = [&](Stmt *Child) {
+    auto *LoopTransform = dyn_cast<OMPLoopTransformationDirective>(Child);
     Stmt *TransformedStmt = LoopTransform->getTransformedStmt();
     unsigned NumGeneratedLoopNests = LoopTransform->getNumGeneratedLoopNests();
     unsigned NumGeneratedLoops = LoopTransform->getNumGeneratedLoops();
@@ -14414,9 +14407,8 @@ bool SemaOpenMP::analyzeLoopSequence(
         LoopSeqSize += NumGeneratedLoopNests;
         NumLoops += NumGeneratedLoops;
         return true;
-      }
-      // Unroll full (0 loops produced)
-      else {
+      } else {
+        // Unroll full (0 loops produced)
         Diag(Child->getBeginLoc(), diag::err_omp_not_for)
             << 0 << getOpenMPDirectiveName(Kind);
         return false;
@@ -14443,9 +14435,8 @@ bool SemaOpenMP::analyzeLoopSequence(
                                  LoopHelpers, ForStmts, OriginalInits,
                                  TransformsPreInits, LoopSequencePreInits,
                                  LoopCategories, Context, Kind);
-    }
-    // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
-    else {
+    } else {
+      // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
       // Process the transformed loop statement
       OriginalInits.emplace_back();
       TransformsPreInits.emplace_back();
@@ -14461,7 +14452,7 @@ bool SemaOpenMP::analyzeLoopSequence(
             << getOpenMPDirectiveName(Kind);
         return false;
       }
-      storeLoopStatements(TransformedStmt);
+      StoreLoopStatements(TransformedStmt);
       updatePreInits(LoopTransform, TransformsPreInits);
 
       NumLoops += NumGeneratedLoops;
@@ -14470,10 +14461,8 @@ bool SemaOpenMP::analyzeLoopSequence(
     }
   };
 
-  // Modularized code for handling regular canonical loops
-  auto analyzeRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
-                             &LoopSeqSize, &NumLoops, Kind, &TmpDSA,
-                             &LoopCategories, this](Stmt *Child) {
+  /// Modularized code for handling regular canonical loops
+  auto AnalyzeRegularLoop = [&](Stmt *Child) {
     OriginalInits.emplace_back();
     LoopHelpers.emplace_back();
     LoopCategories.push_back(OMPLoopCategory::RegularLoop);
@@ -14488,19 +14477,19 @@ bool SemaOpenMP::analyzeLoopSequence(
       return false;
     }
 
-    storeLoopStatements(Child);
+    StoreLoopStatements(Child);
     auto NLCV = NestedLoopCounterVisitor();
     NLCV.TraverseStmt(Child);
     NumLoops += NLCV.getNestedLoopCount();
     return true;
   };
 
-  // Helper functions to validate canonical loop sequence grammar is valid
-  auto isLoopSequenceDerivation = [](auto *Child) {
-    return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
-           isa<OMPLoopTransformationDirective>(Child);
+  /// Helper functions to validate loop sequence grammar derivations
+  auto IsLoopSequenceDerivation = [](auto *Child) {
+    return isa<ForStmt, CXXForRangeStmt, OMPLoopTransformationDirective>(Child);
   };
-  auto isLoopGeneratingStmt = [](auto *Child) {
+  /// Helper functions to validate loop generating grammar derivations
+  auto IsLoopGeneratingStmt = [](auto *Child) {
     return isa<OMPLoopTransformationDirective>(Child);
   };
 
@@ -14511,7 +14500,7 @@ bool SemaOpenMP::analyzeLoopSequence(
       continue;
 
     // Skip over non-loop-sequence statements
-    if (!isLoopSequenceDerivation(Child)) {
+    if (!IsLoopSequenceDerivation(Child)) {
       Child = Child->IgnoreContainers();
 
       // Ignore empty compound statement
@@ -14531,17 +14520,17 @@ bool SemaOpenMP::analyzeLoopSequence(
       }
     }
     // Regular loop sequence handling
-    if (isLoopSequenceDerivation(Child)) {
-      if (isLoopGeneratingStmt(Child)) {
-        if (!analyzeLoopGeneration(Child)) {
+    if (IsLoopSequenceDerivation(Child)) {
+      if (IsLoopGeneratingStmt(Child)) {
+        if (!AnalyzeLoopGeneration(Child))
           return false;
-        }
-        // analyzeLoopGeneration updates Loop Sequence size accordingly
+
+        // AnalyzeLoopGeneration updates Loop Sequence size accordingly
 
       } else {
-        if (!analyzeRegularLoop(Child)) {
+        if (!AnalyzeRegularLoop(Child))
           return false;
-        }
+
         // Update the Loop Sequence size by one
         ++LoopSeqSize;
       }
@@ -14560,9 +14549,9 @@ bool SemaOpenMP::checkTransformableLoopSequence(
     unsigned &NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
     SmallVectorImpl<Stmt *> &ForStmts,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &TransformsPreInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &LoopSequencePreInits,
     SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context) {
 
   // Checks whether the given statement is a compound statement
@@ -14598,10 +14587,9 @@ bool SemaOpenMP::checkTransformableLoopSequence(
   // Recursive entry point to process the main loop sequence
   if (!analyzeLoopSequence(AStmt, LoopSeqSize, NumLoops, LoopHelpers, ForStmts,
                            OriginalInits, TransformsPreInits,
-                           LoopSequencePreInits, LoopCategories, Context,
-                           Kind)) {
+                           LoopSequencePreInits, LoopCategories, Context, Kind))
     return false;
-  }
+
   if (LoopSeqSize <= 0) {
     Diag(AStmt->getBeginLoc(), diag::err_omp_empty_loop_sequence)
         << getOpenMPDirectiveName(Kind);
@@ -14693,7 +14681,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Verify and diagnose loop nest.
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
   Stmt *Body = nullptr;
-  SmallVector<SmallVector<Stmt *, 0>, 4> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_tile, AStmt, NumLoops, LoopHelpers, Body,
                                   OriginalInits))
     return StmtError();
@@ -14970,7 +14958,7 @@ StmtResult SemaOpenMP::ActOnOpenMPStripeDirective(ArrayRef<OMPClause *> Clauses,
   // Verify and diagnose loop nest.
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
   Stmt *Body = nullptr;
-  SmallVector<SmallVector<Stmt *, 0>, 4> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_stripe, AStmt, NumLoops, LoopHelpers,
                                   Body, OriginalInits))
     return StmtError();
@@ -15231,7 +15219,7 @@ StmtResult SemaOpenMP::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   Stmt *Body = nullptr;
   SmallVector<OMPLoopBasedDirective::HelperExprs, NumLoops> LoopHelpers(
       NumLoops);
-  SmallVector<SmallVector<Stmt *, 0>, NumLoops + 1> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, NumLoops + 1> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_unroll, AStmt, NumLoops, LoopHelpers,
                                   Body, OriginalInits))
     return StmtError();
@@ -15499,7 +15487,7 @@ StmtResult SemaOpenMP::ActOnOpenMPReverseDirective(Stmt *AStmt,
   Stmt *Body = nullptr;
   SmallVector<OMPLoopBasedDirective::HelperExprs, NumLoops> LoopHelpers(
       NumLoops);
-  SmallVector<SmallVector<Stmt *, 0>, NumLoops + 1> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, NumLoops + 1> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_reverse, AStmt, NumLoops, LoopHelpers,
                                   Body, OriginalInits))
     return StmtError();
@@ -15691,7 +15679,7 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
   // Verify and diagnose loop nest.
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
   Stmt *Body = nullptr;
-  SmallVector<SmallVector<Stmt *, 0>, 2> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, 2> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_interchange, AStmt, NumLoops,
                                   LoopHelpers, Body, OriginalInits))
     return StmtError();
@@ -15878,9 +15866,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   CaptureVars CopyTransformer(SemaRef);
 
   // Ensure the structured block is not empty
-  if (!AStmt) {
+  if (!AStmt)
     return StmtError();
-  }
 
   unsigned NumLoops = 1;
   unsigned LoopSeqSize = 1;
@@ -15899,16 +15886,15 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   // Also collect the HelperExprs, Loop Stmts, Inits, and Number of loops
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers;
   SmallVector<Stmt *> LoopStmts;
-  SmallVector<SmallVector<Stmt *, 0>> OriginalInits;
-  SmallVector<SmallVector<Stmt *, 0>> TransformsPreInits;
-  SmallVector<SmallVector<Stmt *, 0>> LoopSequencePreInits;
+  SmallVector<SmallVector<Stmt *>> OriginalInits;
+  SmallVector<SmallVector<Stmt *>> TransformsPreInits;
+  SmallVector<SmallVector<Stmt *>> LoopSequencePreInits;
   SmallVector<OMPLoopCategory, 0> LoopCategories;
   if (!checkTransformableLoopSequence(OMPD_fuse, AStmt, LoopSeqSize, NumLoops,
                                       LoopHelpers, LoopStmts, OriginalInits,
                                       TransformsPreInits, LoopSequencePreInits,
-                                      LoopCategories, Context)) {
+                                      LoopCategories, Context))
     return StmtError();
-  }
 
   // Handle clauses, which can be any of the following: [looprange, apply]
   const OMPLoopRangeClause *LRC =
@@ -15998,9 +15984,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   // expressions. Generates both the variable declaration and the corresponding
   // initialization statement.
   auto CreateHelperVarAndStmt =
-      [&SemaRef = this->SemaRef, &Context, &CopyTransformer,
-       &IVType](Expr *ExprToCopy, const std::string &BaseName, unsigned I,
-                bool NeedsNewVD = false) {
+      [&, &SemaRef = SemaRef](Expr *ExprToCopy, const std::string &BaseName,
+                              unsigned I, bool NeedsNewVD = false) {
         Expr *TransformedExpr =
             AssertSuccess(CopyTransformer.TransformExpr(ExprToCopy));
         if (!TransformedExpr)
@@ -16044,9 +16029,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   // Transformations that apply this concept: Loopranged Fuse, Split
   if (!LoopSequencePreInits.empty()) {
     for (const auto &LTPreInits : LoopSequencePreInits) {
-      if (!LTPreInits.empty()) {
+      if (!LTPreInits.empty())
         llvm::append_range(PreInits, LTPreInits);
-      }
     }
   }
 
@@ -16075,9 +16059,9 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
       // Order matters: pre-inits may define variables used in the original
       // inits such as upper bounds...
       auto TransformPreInit = TransformsPreInits[TransformIndex++];
-      if (!TransformPreInit.empty()) {
+      if (!TransformPreInit.empty())
         llvm::append_range(PreInits, TransformPreInit);
-      }
+
       addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
                       PreInits);
     }
@@ -17496,13 +17480,15 @@ OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
   if (CountVal.isInvalid())
     Count = nullptr;
 
+  SmallVector<Expr *, 2> ArgsVec = {First, Count};
+
   // OpenMP [6.0, Restrictions]
   // first + count - 1 must not evaluate to a value greater than the
   // loop sequence length of the associated canonical loop sequence.
   // This check must be performed afterwards due to the delayed
   // parsing and computation of the associated loop sequence
   return OMPLoopRangeClause::Create(getASTContext(), StartLoc, LParenLoc,
-                                    FirstLoc, CountLoc, EndLoc, First, Count);
+                                    FirstLoc, CountLoc, EndLoc, ArgsVec);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPAlignClause(Expr *A, SourceLocation StartLoc,
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2a03c6a1fd0e4..ac4883c4f2a18 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -4452,9 +4452,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Looprange &x) {
       ContextDirectiveAsFortran());
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::FreeAgent &x) {
-  context_.Say(GetContext().clauseSource,
-      "FREE_AGENT clause is not implemented yet"_err_en_US,
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
 

>From e294777879dd46c1859a03c307e70dd03abe11b7 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Thu, 22 May 2025 10:39:39 +0000
Subject: [PATCH 09/12] Removed unncessary warning and updated tests
 accordingly

---
 .../clang/Basic/DiagnosticSemaKinds.td        |  3 --
 clang/lib/Sema/SemaOpenMP.cpp                 | 21 +--------
 clang/test/OpenMP/fuse_messages.cpp           | 43 +++++++++++++++----
 3 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index e85cd32d78b5c..2bd0f895204c9 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11612,9 +11612,6 @@ def note_omp_implicit_dsa : Note<
   "implicitly determined as %0">;
 def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
-def warn_omp_different_loop_ind_var_types : Warning <
-  "loop sequence following '#pragma omp %0' contains induction variables of differing types: %1 and %2">,
-  InGroup<OpenMPLoopForm>;
 def err_omp_not_canonical_loop : Error <
   "loop after '#pragma omp %0' is not in canonical form">;
 def err_omp_not_a_loop_sequence : Error < 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 9819dcfe60360..5f36d968c68fa 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14360,31 +14360,12 @@ bool SemaOpenMP::analyzeLoopSequence(
     OpenMPDirectiveKind Kind) {
 
   VarsWithInheritedDSAType TmpDSA;
-  QualType BaseInductionVarType;
   /// Helper Lambda to handle storing initialization and body statements for
-  /// both ForStmt and CXXForRangeStmt and checks for any possible mismatch
-  /// between induction variables types
+  /// both ForStmt and CXXForRangeStmt
   auto StoreLoopStatements = [&](Stmt *LoopStmt) {
     if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
       OriginalInits.back().push_back(For->getInit());
       ForStmts.push_back(For);
-      // Extract induction variable
-      if (auto *InitStmt = dyn_cast_or_null<DeclStmt>(For->getInit())) {
-        if (auto *InitDecl = dyn_cast<VarDecl>(InitStmt->getSingleDecl())) {
-          QualType InductionVarType = InitDecl->getType().getCanonicalType();
-
-          // Compare with first loop type
-          if (BaseInductionVarType.isNull()) {
-            BaseInductionVarType = InductionVarType;
-          } else if (!Context.hasSameType(BaseInductionVarType,
-                                          InductionVarType)) {
-            Diag(InitDecl->getBeginLoc(),
-                 diag::warn_omp_different_loop_ind_var_types)
-                << getOpenMPDirectiveName(OMPD_fuse) << BaseInductionVarType
-                << InductionVarType;
-          }
-        }
-      }
     } else {
       auto *CXXFor = cast<CXXForRangeStmt>(LoopStmt);
       OriginalInits.back().push_back(CXXFor->getBeginStmt());
diff --git a/clang/test/OpenMP/fuse_messages.cpp b/clang/test/OpenMP/fuse_messages.cpp
index 2a2491d008a0b..4902d424373e5 100644
--- a/clang/test/OpenMP/fuse_messages.cpp
+++ b/clang/test/OpenMP/fuse_messages.cpp
@@ -70,15 +70,6 @@ void func() {
         for(int j = 0; j < 10; ++j);
     }
 
-    //expected-warning at +5 {{loop sequence following '#pragma omp fuse' contains induction variables of differing types: 'int' and 'unsigned int'}}
-    //expected-warning at +5 {{loop sequence following '#pragma omp fuse' contains induction variables of differing types: 'int' and 'long long'}}
-    #pragma omp fuse 
-    {
-        for(int i = 0; i < 10; ++i);
-        for(unsigned int j = 0; j < 10; ++j);
-        for(long long k = 0; k < 100; ++k);
-    }
-
     //expected-warning at +2 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
     #pragma omp fuse
     {
@@ -123,6 +114,40 @@ void func() {
         for(int j = 0; j < 100; ++j);
         for(int k = 0; k < 50; ++k);
     }
+
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '6' is greater than the total number of loops '5'}}
+    #pragma omp fuse looprange(1,6)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+        // This fusion results in  2 loops
+        #pragma omp fuse looprange(1,2)
+        {
+            for(int i = 0; i < 10; ++i);
+            for(int j = 0; j < 100; ++j);
+            for(int k = 0; k < 50; ++k);
+        }
+    }
+
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '4' is greater than the total number of loops '3'}}
+    #pragma omp fuse looprange(2,3)
+    {
+        #pragma omp unroll partial(2)
+        for(int i = 0; i < 10; ++i);
+        
+        #pragma omp reverse
+        for(int j = 0; j < 10; ++j);
+
+        #pragma omp fuse 
+        {
+            {
+                #pragma omp reverse
+                for(int j = 0; j < 10; ++j);
+            }            
+            for(int k = 0; k < 50; ++k);
+        }
+    }
 }
 
 // In a template context, but expression itself not instantiation-dependent

>From 1c8f0fe23fdd78de1512505f128fa9e8bff655f9 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 20 Jun 2025 14:17:29 +0000
Subject: [PATCH 10/12] Address formatting issues

---
 clang/include/clang/Parse/Parser.h  |  2 +-
 clang/lib/AST/StmtOpenMP.cpp        | 10 +++++-----
 clang/lib/Parse/ParseOpenMP.cpp     |  2 +-
 flang/lib/Lower/OpenMP/Clauses.cpp  |  2 +-
 flang/lib/Parser/openmp-parsers.cpp |  8 +++-----
 5 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 08bee0078b5ff..9364007f3cf41 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -6737,7 +6737,7 @@ class Parser : public CodeCompletionHandler {
 
   /// Parses the 'looprange' clause of a '#pragma omp fuse' directive.
   OMPClause *ParseOpenMPLoopRangeClause();
-  
+
   /// Parses the 'sizes' clause of a '#pragma omp tile' directive.
   OMPClause *ParseOpenMPSizesClause();
 
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index f527e6361b5e5..1f49e9f2a0640 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -522,15 +522,15 @@ OMPFuseDirective *OMPFuseDirective::Create(
       NumLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
-  // The number of top level canonical nests could 
+  // The number of top level canonical nests could
   // not match the total number of generated loops
   // Example:
   // Before fusion:
-  //   for (int i = 0; i < N; ++i)   
-  //     for (int j = 0; j < M; ++j) 
+  //   for (int i = 0; i < N; ++i)
+  //     for (int j = 0; j < M; ++j)
   //       A[i][j] = i + j;
-  //   
-  //   for (int k = 0; k < P; ++k) 
+  //
+  //   for (int k = 0; k < P; ++k)
   //     B[k] = k * 2;
   // Here, NumLoopNests = 2, but NumLoops = 3.
   Dir->setNumGeneratedLoopNests(NumLoopNests);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 2d6d624c1ecc8..48d9c184131cd 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3520,7 +3520,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
     break;
   case OMPC_looprange:
     Clause = ParseOpenMPLoopRangeClause();
-    break;  
+    break;
   default:
     break;
   }
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index a38249bf2b588..c94d56cb57756 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -998,7 +998,7 @@ Link make(const parser::OmpClause::Link &inp,
 }
 
 LoopRange make(const parser::OmpClause::Looprange &inp,
-            semantics::SemanticsContext &semaCtx) {
+               semantics::SemanticsContext &semaCtx) {
   llvm_unreachable("Unimplemented: looprange");
 }
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index d53389746dbec..39978e402e63b 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -847,10 +847,8 @@ TYPE_PARSER(
         maybe(":"_tok >> nonemptyList(Parser<OmpLinearClause::Modifier>{})),
         /*PostModified=*/pure(true)))
 
-TYPE_PARSER(
-  construct<OmpLoopRangeClause>(scalarIntConstantExpr,
-                                "," >> scalarIntConstantExpr)
-)
+TYPE_PARSER(construct<OmpLoopRangeClause>(
+    scalarIntConstantExpr, "," >> scalarIntConstantExpr))
 
 // OpenMPv5.2 12.5.2 detach-clause -> DETACH (event-handle)
 TYPE_PARSER(construct<OmpDetachClause>(Parser<OmpObject>{}))
@@ -1026,7 +1024,7 @@ TYPE_PARSER( //
     "LINK" >> construct<OmpClause>(construct<OmpClause::Link>(
                   parenthesized(Parser<OmpObjectList>{}))) ||
     "LOOPRANGE" >> construct<OmpClause>(construct<OmpClause::Looprange>(
-                  parenthesized(Parser<OmpLoopRangeClause>{}))) ||
+                       parenthesized(Parser<OmpLoopRangeClause>{}))) ||
     "MAP" >> construct<OmpClause>(construct<OmpClause::Map>(
                  parenthesized(Parser<OmpMapClause>{}))) ||
     "MATCH" >> construct<OmpClause>(construct<OmpClause::Match>(

>From 009d8630c7ff97dedc543df04d0b18ab4579a503 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 20 Jun 2025 14:44:31 +0000
Subject: [PATCH 11/12] Address minor feedback part 2

---
 clang/include/clang/AST/OpenMPClause.h           |  8 ++++++--
 clang/include/clang/AST/StmtOpenMP.h             |  1 +
 clang/include/clang/Basic/DiagnosticSemaKinds.td | 10 +++++-----
 clang/lib/Sema/SemaOpenMP.cpp                    | 15 +++++----------
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 3df5133a17fb4..478c41322f34a 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1197,12 +1197,16 @@ class OMPLoopRangeClause final
   void setFirstLoc(SourceLocation Loc) { FirstLoc = Loc; }
   void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
 
-  /// Get looprange arguments: first and count
+  /// Get looprange 'first' expression
   Expr *getFirst() const { return getArgs()[0]; }
+
+  /// Get looprange 'count' expression
   Expr *getCount() const { return getArgs()[1]; }
 
-  /// Set looprange arguments: first and count
+  /// Set looprange 'first' expression
   void setFirst(Expr *E) { getArgs()[0] = E; }
+
+  /// Set looprange 'count' expression
   void setCount(Expr *E) { getArgs()[1] = E; }
 
   MutableArrayRef<Expr *> getArgs() {
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 0421c06245cac..5ec3677fc7507 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -976,6 +976,7 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
 
   /// Set the number of loops generated by this loop transformation.
   void setNumGeneratedLoops(unsigned Num) { NumGeneratedLoops = Num; }
+
   /// Set the number of top level canonical loop nests generated by this loop
   /// transformation
   void setNumGeneratedLoopNests(unsigned Num) { NumGeneratedLoopNests = Num; }
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 2bd0f895204c9..d807b6b076724 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11612,16 +11612,16 @@ def note_omp_implicit_dsa : Note<
   "implicitly determined as %0">;
 def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
-def err_omp_not_canonical_loop : Error <
+def err_omp_not_canonical_loop : Error<
   "loop after '#pragma omp %0' is not in canonical form">;
-def err_omp_not_a_loop_sequence : Error < 
+def err_omp_not_a_loop_sequence : Error< 
   "statement after '#pragma omp %0' must be a loop sequence containing canonical loops or loop-generating constructs">;
-def err_omp_empty_loop_sequence : Error <
+def err_omp_empty_loop_sequence : Error<
   "loop sequence after '#pragma omp %0' must contain at least 1 canonical loop or loop-generating construct">;
-def err_omp_invalid_looprange : Error <
+def err_omp_invalid_looprange : Error<
   "loop range in '#pragma omp %0' exceeds the number of available loops: "
   "range end '%1' is greater than the total number of loops '%2'">;
-def warn_omp_redundant_fusion : Warning <
+def warn_omp_redundant_fusion : Warning<
   "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">,
   InGroup<OpenMPClauses>;
 def err_omp_not_for : Error<
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 5f36d968c68fa..8aa21c5c01220 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -22,7 +22,6 @@
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DynamicRecursiveASTVisitor.h"
 #include "clang/AST/OpenMPClause.h"
-#include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/StmtVisitor.h"
@@ -48,7 +47,6 @@
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/IR/Assumptions.h"
 #include <optional>
-#include <queue>
 
 using namespace clang;
 using namespace llvm::omp;
@@ -14201,7 +14199,6 @@ template <typename T, typename F> static bool tryHandleAs(T *t, F &&) {
   return false;
 }
 
-///
 /// Tries to recursively cast `t` to one of the given types and invokes `f` if
 /// successful.
 ///
@@ -14274,7 +14271,7 @@ bool SemaOpenMP::checkTransformableLoopNest(
 
 /// Counts the total number of nested loops, including the outermost loop (the
 /// original loop). PRECONDITION of this visitor is that it must be invoked from
-/// the original loop to be analyzed. The traversal is stop for Decl's and
+/// the original loop to be analyzed. The traversal stops for Decl's and
 /// Expr's given that they may contain inner loops that must not be counted.
 ///
 /// Example AST structure for the code:
@@ -15945,7 +15942,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
 
   // Select the type with the largest bit width among all induction variables
   QualType IVType = LoopHelpers[FirstVal - 1].IterationVarRef->getType();
-  for (unsigned int I = FirstVal; I < LastVal; ++I) {
+  for (unsigned I = FirstVal; I < LastVal; ++I) {
     QualType CurrentIVType = LoopHelpers[I].IterationVarRef->getType();
     if (Context.getTypeSize(CurrentIVType) > Context.getTypeSize(IVType)) {
       IVType = CurrentIVType;
@@ -16054,9 +16051,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     auto [IVVD, IVDStmt] =
         CreateHelperVarAndStmt(LoopHelpers[I].IterationVarRef, "iv", J);
 
-    if (!LBVD || !STVD || !NIVD || !IVVD)
-      assert(LBVD && STVD && NIVD && IVVD &&
-             "OpenMP Fuse Helper variables creation failed");
+    assert(LBVD && STVD && NIVD && IVVD &&
+           "OpenMP Fuse Helper variables creation failed");
 
     UBVarDecls.push_back(UBVD);
     LBVarDecls.push_back(LBVD);
@@ -16097,11 +16093,10 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   //      original.indexk = ivk
   //      body(k);  Expr *InitVal = IntegerLiteral::Create(Context,
   //      llvm::APInt(IVWidth, 0),
-
   //    }
 
   // 1. Create the initialized fuse index
-  const std::string IndexName = Twine(".omp.fuse.index").str();
+  StringRef IndexName = ".omp.fuse.index";
   Expr *InitVal = IntegerLiteral::Create(Context, llvm::APInt(IVBitWidth, 0),
                                          IVType, SourceLocation());
   VarDecl *IndexDecl =

>From eb464451c30c19745591738d71e3496d4d11514a Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Mon, 14 Jul 2025 11:34:36 +0000
Subject: [PATCH 12/12] Address some of the feedback

---
 clang/include/clang/AST/OpenMPClause.h | 46 ++++---------
 clang/include/clang/Sema/SemaOpenMP.h  |  2 +-
 clang/lib/AST/OpenMPClause.cpp         | 11 ++-
 clang/lib/Sema/SemaOpenMP.cpp          | 92 ++++++++++++--------------
 4 files changed, 63 insertions(+), 88 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 478c41322f34a..5034ff9bacbfc 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1154,11 +1154,8 @@ class OMPFullClause final : public OMPNoChildClause<llvm::omp::OMPC_full> {
 ///   for(int k = 127; k >= 0; --k)
 /// \endcode
 class OMPLoopRangeClause final
-    : public OMPClause,
-      private llvm::TrailingObjects<OMPLoopRangeClause, Expr *> {
+    : public OMPClause {
   friend class OMPClauseReader;
-  friend class llvm::TrailingObjects<OMPLoopRangeClause, Expr *>;
-
   /// Location of '('
   SourceLocation LParenLoc;
 
@@ -1166,24 +1163,25 @@ class OMPLoopRangeClause final
   SourceLocation FirstLoc, CountLoc;
 
   /// Number of looprange arguments (always 2: first, count)
-  unsigned NumArgs = 2;
+  static constexpr unsigned NumArgs = 2;
+  Stmt *Args[NumArgs] = {nullptr, nullptr};
 
-  /// Set the argument expressions.
-  void setArgs(ArrayRef<Expr *> Args) {
-    assert(Args.size() == NumArgs && "Expected exactly 2 looprange arguments");
-    std::copy(Args.begin(), Args.end(), getTrailingObjects<Expr *>());
-  }
+  /// Set looprange 'first' expression
+  void setFirst(Expr *E) { Args[0] = E; }
+
+  /// Set looprange 'count' expression
+  void setCount(Expr *E) { Args[1] = E; }
 
   /// Build an empty clause for deserialization.
   explicit OMPLoopRangeClause()
-      : OMPClause(llvm::omp::OMPC_looprange, {}, {}), NumArgs(2) {}
+      : OMPClause(llvm::omp::OMPC_looprange, {}, {}) {}
 
 public:
   /// Build a 'looprange' clause AST node.
   static OMPLoopRangeClause *
   Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
          SourceLocation FirstLoc, SourceLocation CountLoc,
-         SourceLocation EndLoc, ArrayRef<Expr *> Args);
+         SourceLocation EndLoc, Expr* First, Expr* Count);
 
   /// Build an empty 'looprange' clause node.
   static OMPLoopRangeClause *CreateEmpty(const ASTContext &C);
@@ -1198,32 +1196,16 @@ class OMPLoopRangeClause final
   void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
 
   /// Get looprange 'first' expression
-  Expr *getFirst() const { return getArgs()[0]; }
+  Expr *getFirst() const { return cast_or_null<Expr>(Args[0]); }
 
   /// Get looprange 'count' expression
-  Expr *getCount() const { return getArgs()[1]; }
-
-  /// Set looprange 'first' expression
-  void setFirst(Expr *E) { getArgs()[0] = E; }
-
-  /// Set looprange 'count' expression
-  void setCount(Expr *E) { getArgs()[1] = E; }
-
-  MutableArrayRef<Expr *> getArgs() {
-    return MutableArrayRef<Expr *>(getTrailingObjects<Expr *>(), NumArgs);
-  }
-  ArrayRef<Expr *> getArgs() const {
-    return ArrayRef<Expr *>(getTrailingObjects<Expr *>(), NumArgs);
-  }
+  Expr *getCount() const { return cast_or_null<Expr>(Args[1]); }
 
   child_range children() {
-    return child_range(reinterpret_cast<Stmt **>(getArgs().begin()),
-                       reinterpret_cast<Stmt **>(getArgs().end()));
+    return child_range(Args, Args + NumArgs);
   }
   const_child_range children() const {
-    auto AR = getArgs();
-    return const_child_range(reinterpret_cast<Stmt *const *>(AR.begin()),
-                             reinterpret_cast<Stmt *const *>(AR.end()));
+    return const_child_range(Args, Args + NumArgs);
   }
 
   child_range used_children() {
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index f848c4a7d715e..09c97dc2e119b 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1494,7 +1494,7 @@ class SemaOpenMP : public SemaBase {
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits);
 
-  /// @brief Categories of loops encountered during semantic OpenMP loop
+  /// Categories of loops encountered during semantic OpenMP loop
   /// analysis
   ///
   /// This enumeration identifies the structural category of a loop or sequence
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index e0570262b2a05..4eb249948932e 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1028,23 +1028,20 @@ OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
                            SourceLocation CountLoc, SourceLocation EndLoc,
-                           ArrayRef<Expr *> Args) {
-
-  assert(Args.size() == 2 &&
-         "looprange clause must have exactly two arguments");
+                           Expr *First, Expr* Count) {
   OMPLoopRangeClause *Clause = CreateEmpty(C);
   Clause->setLocStart(StartLoc);
   Clause->setLParenLoc(LParenLoc);
   Clause->setFirstLoc(FirstLoc);
   Clause->setCountLoc(CountLoc);
   Clause->setLocEnd(EndLoc);
-  Clause->setArgs(Args);
+  Clause->setFirst(First);
+  Clause->setCount(Count);
   return Clause;
 }
 
 OMPLoopRangeClause *OMPLoopRangeClause::CreateEmpty(const ASTContext &C) {
-  void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(2));
-  return new (Mem) OMPLoopRangeClause();
+  return new (C) OMPLoopRangeClause();
 }
 
 OMPAllocateClause *OMPAllocateClause::Create(
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 8aa21c5c01220..edffb041b3664 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14195,29 +14195,29 @@ StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
 }
 
 /// Overloaded base case function
-template <typename T, typename F> static bool tryHandleAs(T *t, F &&) {
+template <typename T, typename F> static bool tryHandleAs(T *, F &&) {
   return false;
 }
 
-/// Tries to recursively cast `t` to one of the given types and invokes `f` if
-/// successful.
+/// Tries to recursively cast `Type` to one of the given types and invokes
+/// `Func` if successful.
 ///
-/// @tparam Class The first type to check.
-/// @tparam Rest The remaining types to check.
-/// @tparam T The base type of `t`.
-/// @tparam F The callable type for the function to invoke upon a successful
+/// \tparam Class The first type to check.
+/// \tparam Rest The remaining types to check.
+/// \tparam T The base type of `Type`.
+/// \tparam F The callable type for the function to invoke upon a successful
 /// cast.
-/// @param t The object to be checked.
-/// @param f The function to invoke if `t` matches `Class`.
-/// @return `true` if `t` matched any type and `f` was called, otherwise
+/// \param Type The object to be checked.
+/// \param Func The function to invoke if `Type` matches `Class`.
+/// \return `true` if `Type` matched any type and `Func` was called, otherwise
 /// `false`.
 template <typename Class, typename... Rest, typename T, typename F>
-static bool tryHandleAs(T *t, F &&f) {
-  if (Class *c = dyn_cast<Class>(t)) {
-    f(c);
+static bool tryHandleAs(T *Type, F &&Func) {
+  if (Class *C = dyn_cast<Class>(Type)) {
+    Func(C);
     return true;
   }
-  return tryHandleAs<Rest...>(t, std::forward<F>(f));
+  return tryHandleAs<Rest...>(Type, std::forward<F>(Func));
 }
 
 /// Updates OriginalInits by checking Transform against loop transformation
@@ -14297,7 +14297,7 @@ bool SemaOpenMP::checkTransformableLoopNest(
 ///     }
 /// }
 /// Result: Loop 'i' contains 2 loops, Loop 'r' also contains 2 loops
-class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
+class NestedLoopCounterVisitor final : public DynamicRecursiveASTVisitor {
 private:
   unsigned NestedLoopCount = 0;
 
@@ -14385,22 +14385,21 @@ bool SemaOpenMP::analyzeLoopSequence(
         LoopSeqSize += NumGeneratedLoopNests;
         NumLoops += NumGeneratedLoops;
         return true;
-      } else {
-        // Unroll full (0 loops produced)
-        Diag(Child->getBeginLoc(), diag::err_omp_not_for)
-            << 0 << getOpenMPDirectiveName(Kind);
-        return false;
       }
+      // Unroll full (0 loops produced)
+      Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+          << 0 << getOpenMPDirectiveName(Kind);
+      return false;
     }
     // Handle loop transformations with multiple loop nests
     // Unroll full
-    if (NumGeneratedLoopNests <= 0) {
+    if (!NumGeneratedLoopNests) {
       Diag(Child->getBeginLoc(), diag::err_omp_not_for)
           << 0 << getOpenMPDirectiveName(Kind);
       return false;
     }
     // Loop transformatons such as split or loopranged fuse
-    else if (NumGeneratedLoopNests > 1) {
+    if (NumGeneratedLoopNests > 1) {
       // Get the preinits related to this loop sequence generating
       // loop transformation (i.e loopranged fuse, split...)
       LoopSequencePreInits.emplace_back();
@@ -14413,30 +14412,29 @@ bool SemaOpenMP::analyzeLoopSequence(
                                  LoopHelpers, ForStmts, OriginalInits,
                                  TransformsPreInits, LoopSequencePreInits,
                                  LoopCategories, Context, Kind);
-    } else {
-      // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
-      // Process the transformed loop statement
-      OriginalInits.emplace_back();
-      TransformsPreInits.emplace_back();
-      LoopHelpers.emplace_back();
-      LoopCategories.push_back(OMPLoopCategory::TransformSingleLoop);
-
-      unsigned IsCanonical =
-          checkOpenMPLoop(Kind, nullptr, nullptr, TransformedStmt, SemaRef,
-                          *DSAStack, TmpDSA, LoopHelpers[LoopSeqSize]);
-
-      if (!IsCanonical) {
-        Diag(TransformedStmt->getBeginLoc(), diag::err_omp_not_canonical_loop)
-            << getOpenMPDirectiveName(Kind);
-        return false;
-      }
-      StoreLoopStatements(TransformedStmt);
-      updatePreInits(LoopTransform, TransformsPreInits);
+    }
+    // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
+    // Process the transformed loop statement
+    OriginalInits.emplace_back();
+    TransformsPreInits.emplace_back();
+    LoopHelpers.emplace_back();
+    LoopCategories.push_back(OMPLoopCategory::TransformSingleLoop);
 
-      NumLoops += NumGeneratedLoops;
-      ++LoopSeqSize;
-      return true;
+    unsigned IsCanonical =
+        checkOpenMPLoop(Kind, nullptr, nullptr, TransformedStmt, SemaRef,
+                        *DSAStack, TmpDSA, LoopHelpers[LoopSeqSize]);
+
+    if (!IsCanonical) {
+      Diag(TransformedStmt->getBeginLoc(), diag::err_omp_not_canonical_loop)
+          << getOpenMPDirectiveName(Kind);
+      return false;
     }
+    StoreLoopStatements(TransformedStmt);
+    updatePreInits(LoopTransform, TransformsPreInits);
+
+    NumLoops += NumGeneratedLoops;
+    ++LoopSeqSize;
+    return true;
   };
 
   /// Modularized code for handling regular canonical loops
@@ -16303,7 +16301,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
       // Only TransformSingleLoop requires inserting pre-inits here
 
       if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop) {
-        auto TransformPreInit = TransformsPreInits[TransformIndex++];
+        const auto &TransformPreInit = TransformsPreInits[TransformIndex++];
         if (!TransformPreInit.empty()) {
           llvm::append_range(PreInits, TransformPreInit);
         }
@@ -17456,15 +17454,13 @@ OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
   if (CountVal.isInvalid())
     Count = nullptr;
 
-  SmallVector<Expr *, 2> ArgsVec = {First, Count};
-
   // OpenMP [6.0, Restrictions]
   // first + count - 1 must not evaluate to a value greater than the
   // loop sequence length of the associated canonical loop sequence.
   // This check must be performed afterwards due to the delayed
   // parsing and computation of the associated loop sequence
   return OMPLoopRangeClause::Create(getASTContext(), StartLoc, LParenLoc,
-                                    FirstLoc, CountLoc, EndLoc, ArgsVec);
+                                    FirstLoc, CountLoc, EndLoc, First, Count);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPAlignClause(Expr *A, SourceLocation StartLoc,


From flang-commits at lists.llvm.org  Mon Jul 14 06:22:37 2025
From: flang-commits at lists.llvm.org (=?UTF-8?B?Um9nZXIgRmVycmVyIEliw6HDsWV6?= via flang-commits)
Date: Mon, 14 Jul 2025 06:22:37 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <6875049d.170a0220.1768e9.fc15@mx.google.com>


https://github.com/rofirrim updated https://github.com/llvm/llvm-project/pull/139293

>From fb91129401f61b332fc1147e5a81d553abd7658a Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:25:33 +0000
Subject: [PATCH 01/13] Add fuse directive patch

---
 clang/include/clang-c/Index.h                 |    4 +
 clang/include/clang/AST/RecursiveASTVisitor.h |    3 +
 clang/include/clang/AST/StmtOpenMP.h          |   99 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |    8 +
 clang/include/clang/Basic/StmtNodes.td        |    1 +
 clang/include/clang/Sema/SemaOpenMP.h         |   27 +
 .../include/clang/Serialization/ASTBitCodes.h |    1 +
 clang/lib/AST/StmtOpenMP.cpp                  |   25 +
 clang/lib/AST/StmtPrinter.cpp                 |    5 +
 clang/lib/AST/StmtProfile.cpp                 |    4 +
 clang/lib/Basic/OpenMPKinds.cpp               |    2 +-
 clang/lib/CodeGen/CGStmt.cpp                  |    3 +
 clang/lib/CodeGen/CGStmtOpenMP.cpp            |    8 +
 clang/lib/CodeGen/CodeGenFunction.h           |    1 +
 clang/lib/Sema/SemaExceptionSpec.cpp          |    1 +
 clang/lib/Sema/SemaOpenMP.cpp                 |  600 +++++++
 clang/lib/Sema/TreeTransform.h                |   11 +
 clang/lib/Serialization/ASTReaderStmt.cpp     |   11 +
 clang/lib/Serialization/ASTWriterStmt.cpp     |    6 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp  |    1 +
 clang/test/OpenMP/fuse_ast_print.cpp          |  278 +++
 clang/test/OpenMP/fuse_codegen.cpp            | 1511 +++++++++++++++++
 clang/test/OpenMP/fuse_messages.cpp           |   76 +
 clang/tools/libclang/CIndex.cpp               |    7 +
 clang/tools/libclang/CXCursor.cpp             |    3 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |    4 +
 .../runtime/test/transform/fuse/foreach.cpp   |  192 +++
 openmp/runtime/test/transform/fuse/intfor.c   |   50 +
 .../runtime/test/transform/fuse/iterfor.cpp   |  194 +++
 .../fuse/parallel-wsloop-collapse-foreach.cpp |  208 +++
 .../fuse/parallel-wsloop-collapse-intfor.c    |   45 +
 31 files changed, 3387 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/OpenMP/fuse_ast_print.cpp
 create mode 100644 clang/test/OpenMP/fuse_codegen.cpp
 create mode 100644 clang/test/OpenMP/fuse_messages.cpp
 create mode 100644 openmp/runtime/test/transform/fuse/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/fuse/intfor.c
 create mode 100644 openmp/runtime/test/transform/fuse/iterfor.cpp
 create mode 100644 openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-foreach.cpp
 create mode 100644 openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c

diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index e4cb4327fbaac..148b89ab9cfa4 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2162,6 +2162,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPStripeDirective = 310,
 
+  /** OpenMP fuse directive
+   */
+  CXCursor_OMPFuseDirective = 318,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 5cb2f57edffe4..918216e8df4aa 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3090,6 +3090,9 @@ DEF_TRAVERSE_STMT(OMPUnrollDirective,
 DEF_TRAVERSE_STMT(OMPReverseDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFuseDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index e2fd2114026f7..cb8bb91f4768c 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -962,6 +962,9 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
 
   /// Number of loops generated by this loop transformation.
   unsigned NumGeneratedLoops = 0;
+  /// Number of top level canonical loop nests generated by this loop
+  /// transformation
+  unsigned NumGeneratedLoopNests = 0;
 
 protected:
   explicit OMPLoopTransformationDirective(StmtClass SC,
@@ -973,6 +976,9 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
 
   /// Set the number of loops generated by this loop transformation.
   void setNumGeneratedLoops(unsigned Num) { NumGeneratedLoops = Num; }
+  /// Set the number of top level canonical loop nests generated by this loop
+  /// transformation
+  void setNumGeneratedLoopNests(unsigned Num) { NumGeneratedLoopNests = Num; }
 
 public:
   /// Return the number of associated (consumed) loops.
@@ -981,6 +987,10 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
   /// Return the number of loops generated by this loop transformation.
   unsigned getNumGeneratedLoops() const { return NumGeneratedLoops; }
 
+  /// Return the number of top level canonical loop nests generated by this loop
+  /// transformation
+  unsigned getNumGeneratedLoopNests() const { return NumGeneratedLoopNests; }
+
   /// Get the de-sugared statements after the loop transformation.
   ///
   /// Might be nullptr if either the directive generates no loops and is handled
@@ -995,7 +1005,8 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass ||
+           C == OMPFuseDirectiveClass;
   }
 };
 
@@ -5562,6 +5573,7 @@ class OMPTileDirective final : public OMPLoopTransformationDirective {
                                        llvm::omp::OMPD_tile, StartLoc, EndLoc,
                                        NumLoops) {
     setNumGeneratedLoops(2 * NumLoops);
+    setNumGeneratedLoopNests(1);
   }
 
   void setPreInits(Stmt *PreInits) {
@@ -5793,6 +5805,7 @@ class OMPReverseDirective final : public OMPLoopTransformationDirective {
                                        llvm::omp::OMPD_reverse, StartLoc,
                                        EndLoc, NumLoops) {
     setNumGeneratedLoops(NumLoops);
+    setNumGeneratedLoopNests(1);
   }
 
   void setPreInits(Stmt *PreInits) {
@@ -5865,6 +5878,7 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
                                        llvm::omp::OMPD_interchange, StartLoc,
                                        EndLoc, NumLoops) {
     setNumGeneratedLoops(NumLoops);
+    setNumGeneratedLoopNests(1);
   }
 
   void setPreInits(Stmt *PreInits) {
@@ -5915,6 +5929,89 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
   }
 };
 
+/// Represents the '#pragma omp fuse' loop transformation directive
+///
+/// \code{c}
+/// #pragma omp fuse
+/// {
+///   for(int i = 0; i < m1; ++i) {...}
+///   for(int j = 0; j < m2; ++j) {...}
+///   ...
+/// }
+/// \endcode
+
+class OMPFuseDirective final : public OMPLoopTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  // Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFuseDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                            unsigned NumLoops)
+      : OMPLoopTransformationDirective(OMPFuseDirectiveClass,
+                                       llvm::omp::OMPD_fuse, StartLoc, EndLoc,
+                                       NumLoops) {
+    // This default initialization assumes simple loop fusion.
+    // If a 'looprange' clause is specified, these values must be explicitly set
+    setNumGeneratedLoopNests(1);
+    setNumGeneratedLoops(NumLoops);
+  }
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for #pragma omp fuse'
+  ///
+  /// \param C Context of the AST
+  /// \param StartLoc Location of the introducer (e.g the 'omp' token)
+  /// \param EndLoc Location of the directive's end (e.g the tok::eod)
+  /// \param Clauses The directive's clauses
+  /// \param NumLoops Number of total affected loops
+  /// \param NumLoopNests Number of affected top level canonical loops
+  ///                 (number of items in the 'looprange' clause if present)
+  /// \param AssociatedStmt The outermost associated loop
+  /// \param TransformedStmt The loop nest after fusion, or nullptr in
+  ///                        dependent
+  /// \param PreInits Helper preinits statements for the loop nest
+  static OMPFuseDirective *Create(const ASTContext &C, SourceLocation StartLoc,
+                                  SourceLocation EndLoc,
+                                  ArrayRef<OMPClause *> Clauses,
+                                  unsigned NumLoops, unsigned NumLoopNests,
+                                  Stmt *AssociatedStmt, Stmt *TransformedStmt,
+                                  Stmt *PreInits);
+
+  /// Build an empty '#pragma omp fuse' AST node for deserialization
+  ///
+  /// \param C Context of the AST
+  /// \param NumClauses Number of clauses to allocate
+  /// \param NumLoops Number of associated loops to allocate
+  static OMPFuseDirective *CreateEmpty(const ASTContext &C, unsigned NumClauses,
+                                       unsigned NumLoops);
+
+  /// Gets the associated loops after the transformation. This is the de-sugared
+  /// replacement or nulltpr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFuseDirectiveClass;
+  }
+};
+
 /// This represents '#pragma omp scan' directive.
 ///
 /// \code
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 979ff60b73b75..fe9ca29038a1f 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11612,6 +11612,14 @@ def note_omp_implicit_dsa : Note<
   "implicitly determined as %0">;
 def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
+def warn_omp_different_loop_ind_var_types : Warning <
+  "loop sequence following '#pragma omp %0' contains induction variables of differing types: %1 and %2">;
+def err_omp_not_canonical_loop : Error <
+  "loop after '#pragma omp %0' is not in canonical form">;
+def err_omp_not_a_loop_sequence : Error < 
+  "statement after '#pragma omp %0' must be a loop sequence containing canonical loops or loop-generating constructs">;
+def err_omp_empty_loop_sequence : Error <
+  "loop sequence after '#pragma omp %0' must contain at least 1 canonical loop or loop-generating construct">;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index c9c173f5c7469..45d1a813e4b1f 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -233,6 +233,7 @@ def OMPStripeDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPUnrollDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPReverseDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPInterchangeDirective : StmtNode<OMPLoopTransformationDirective>;
+def OMPFuseDirective : StmtNode<OMPLoopTransformationDirective>;
 def OMPForDirective : StmtNode<OMPLoopDirective>;
 def OMPForSimdDirective : StmtNode<OMPLoopDirective>;
 def OMPSectionsDirective : StmtNode<OMPExecutableDirective>;
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 7b169f56b6807..ea21377a8db9c 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -457,6 +457,13 @@ class SemaOpenMP : public SemaBase {
                                              Stmt *AStmt,
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
+
+  /// Called on well-formed '#pragma omp fuse' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
+                                      Stmt *AStmt, SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+
   /// Called on well-formed '\#pragma omp for' after parsing
   /// of the associated statement.
   StmtResult
@@ -1481,6 +1488,26 @@ class SemaOpenMP : public SemaBase {
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits);
 
+  /// Analyzes and checks a loop sequence for use by a loop transformation
+  ///
+  /// \param Kind           The loop transformation directive kind.
+  /// \param NumLoops       [out] Number of total canonical loops
+  /// \param LoopSeqSize    [out] Number of top level canonical loops
+  /// \param LoopHelpers    [out] The multiple loop analyses results.
+  /// \param LoopStmts      [out] The multiple Stmt of each For loop.
+  /// \param OriginalInits  [out] The multiple collection of statements and
+  ///                       declarations that must have been executed/declared
+  ///                       before entering the loop.
+  /// \param Context
+  /// \return Whether there was an absence of errors or not
+  bool checkTransformableLoopSequence(
+      OpenMPDirectiveKind Kind, Stmt *AStmt, unsigned &LoopSeqSize,
+      unsigned &NumLoops,
+      SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+      SmallVectorImpl<Stmt *> &ForStmts,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
+      ASTContext &Context);
+
   /// Helper to keep information about the current `omp begin/end declare
   /// variant` nesting.
   struct OMPDeclareVariantScope {
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 9d265f27b8e31..83b73554d693c 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1948,6 +1948,7 @@ enum StmtCode {
   STMT_OMP_UNROLL_DIRECTIVE,
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
   STMT_OMP_SECTIONS_DIRECTIVE,
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 2eeb5e45ab511..276e43ec9f7d5 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -456,6 +456,8 @@ OMPUnrollDirective::Create(const ASTContext &C, SourceLocation StartLoc,
   auto *Dir = createDirective<OMPUnrollDirective>(
       C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc);
   Dir->setNumGeneratedLoops(NumGeneratedLoops);
+  // The number of generated loops and loop nests during unroll matches
+  Dir->setNumGeneratedLoopNests(NumGeneratedLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
   return Dir;
@@ -508,6 +510,29 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFuseDirective *OMPFuseDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, unsigned NumLoopNests,
+    Stmt *AssociatedStmt, Stmt *TransformedStmt, Stmt *PreInits) {
+
+  OMPFuseDirective *Dir = createDirective<OMPFuseDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  Dir->setNumGeneratedLoopNests(NumLoopNests);
+  Dir->setNumGeneratedLoops(NumLoops);
+  return Dir;
+}
+
+OMPFuseDirective *OMPFuseDirective::CreateEmpty(const ASTContext &C,
+                                                unsigned NumClauses,
+                                                unsigned NumLoops) {
+  return createEmptyDirective<OMPFuseDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPForSimdDirective *
 OMPForSimdDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                             SourceLocation EndLoc, unsigned CollapsedNum,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 28317911d825b..4f57c63154da0 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -790,6 +790,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFuseDirective(OMPFuseDirective *Node) {
+  Indent() << "#pragma omp fuse";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPForDirective(OMPForDirective *Node) {
   Indent() << "#pragma omp for";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index c61450e19f1b6..c5d1d5b48508e 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1026,6 +1026,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPLoopTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFuseDirective(const OMPFuseDirective *S) {
+  VisitOMPLoopTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPForDirective(const OMPForDirective *S) {
   VisitOMPLoopDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index a451fc7c01841..d172450512f13 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -702,7 +702,7 @@ bool clang::isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind) {
 
 bool clang::isOpenMPLoopTransformationDirective(OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
-         DKind == OMPD_interchange || DKind == OMPD_stripe;
+         DKind == OMPD_interchange || DKind == OMPD_stripe || DKind == OMPD_fuse;
 }
 
 bool clang::isOpenMPCombinedParallelADirective(OpenMPDirectiveKind DKind) {
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 8742f8e0fc04a..aa12d62d1b865 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -234,6 +234,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFuseDirectiveClass:
+    EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
+    break;
   case Stmt::OMPForDirectiveClass:
     EmitOMPForDirective(cast<OMPForDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index d9195d749e056..cf03d5d3d88a3 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -198,6 +198,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Fuse = dyn_cast<OMPFuseDirective>(&S)) {
+      PreInits = Fuse->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -2922,6 +2924,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
+  // Emit the de-sugared statement
+  OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {
   bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index a5ab9df01dba9..fe753e5b688b1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3855,6 +3855,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
   void EmitOMPScopeDirective(const OMPScopeDirective &S);
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 0a6cea8869c14..3eb59156c04af 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1493,6 +1493,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
   case Stmt::OMPTargetDataDirectiveClass:
   case Stmt::OMPTargetDirectiveClass:
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 00f4658180807..84ac9587bd54d 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4404,6 +4404,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_unroll:
   case OMPD_reverse:
   case OMPD_interchange:
+  case OMPD_fuse:
   case OMPD_assume:
     break;
   default:
@@ -6221,6 +6222,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_fuse:
+    Res =
+        ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
+    break;
   case OMPD_for:
     Res = ActOnOpenMPForDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc,
                                   VarsWithInheritedDSA);
@@ -14230,6 +14235,8 @@ bool SemaOpenMP::checkTransformableLoopNest(
           DependentPreInits = Dir->getPreInits();
         else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
           DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPFuseDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
         else
           llvm_unreachable("Unhandled loop transformation");
 
@@ -14240,6 +14247,265 @@ bool SemaOpenMP::checkTransformableLoopNest(
   return Result;
 }
 
+class NestedLoopCounterVisitor
+    : public clang::RecursiveASTVisitor<NestedLoopCounterVisitor> {
+public:
+  explicit NestedLoopCounterVisitor() : NestedLoopCount(0) {}
+
+  bool VisitForStmt(clang::ForStmt *FS) {
+    ++NestedLoopCount;
+    return true;
+  }
+
+  bool VisitCXXForRangeStmt(clang::CXXForRangeStmt *FRS) {
+    ++NestedLoopCount;
+    return true;
+  }
+
+  unsigned getNestedLoopCount() const { return NestedLoopCount; }
+
+private:
+  unsigned NestedLoopCount;
+};
+
+bool SemaOpenMP::checkTransformableLoopSequence(
+    OpenMPDirectiveKind Kind, Stmt *AStmt, unsigned &LoopSeqSize,
+    unsigned &NumLoops,
+    SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+    SmallVectorImpl<Stmt *> &ForStmts,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
+    ASTContext &Context) {
+
+  // Checks whether the given statement is a compound statement
+  VarsWithInheritedDSAType TmpDSA;
+  if (!isa<CompoundStmt>(AStmt)) {
+    Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
+        << getOpenMPDirectiveName(Kind);
+    return false;
+  }
+  // Callback for updating pre-inits in case there are even more
+  // loop-sequence-generating-constructs inside of the main compound stmt
+  auto OnTransformationCallback =
+      [&OriginalInits](OMPLoopBasedDirective *Transform) {
+        Stmt *DependentPreInits;
+        if (auto *Dir = dyn_cast<OMPTileDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPUnrollDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPReverseDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else if (auto *Dir = dyn_cast<OMPFuseDirective>(Transform))
+          DependentPreInits = Dir->getPreInits();
+        else
+          llvm_unreachable("Unhandled loop transformation");
+
+        appendFlattenedStmtList(OriginalInits.back(), DependentPreInits);
+      };
+
+  // Number of top level canonical loop nests observed (And acts as index)
+  LoopSeqSize = 0;
+  // Number of total observed loops
+  NumLoops = 0;
+
+  // Following OpenMP 6.0 API Specification, a Canonical Loop Sequence follows
+  // the grammar:
+  //
+  // canonical-loop-sequence:
+  //  {
+  //    loop-sequence+
+  //  }
+  // where loop-sequence can be any of the following:
+  // 1. canonical-loop-sequence
+  // 2. loop-nest
+  // 3. loop-sequence-generating-construct (i.e OMPLoopTransformationDirective)
+  //
+  // To recognise and traverse this structure the following helper functions
+  // have been defined. handleLoopSequence serves as the recurisve entry point
+  // and tries to match the input AST to the canonical loop sequence grammar
+  // structure
+
+  auto NLCV = NestedLoopCounterVisitor();
+  // Helper functions to validate canonical loop sequence grammar is valid
+  auto isLoopSequenceDerivation = [](auto *Child) {
+    return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
+           isa<OMPLoopTransformationDirective>(Child);
+  };
+  auto isLoopGeneratingStmt = [](auto *Child) {
+    return isa<OMPLoopTransformationDirective>(Child);
+  };
+
+  // Helper Lambda to handle storing initialization and body statements for both
+  // ForStmt and CXXForRangeStmt and checks for any possible mismatch between
+  // induction variables types
+  QualType BaseInductionVarType;
+  auto storeLoopStatements = [&OriginalInits, &ForStmts, &BaseInductionVarType,
+                              this, &Context](Stmt *LoopStmt) {
+    if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
+      OriginalInits.back().push_back(For->getInit());
+      ForStmts.push_back(For);
+      // Extract induction variable
+      if (auto *InitStmt = dyn_cast_or_null<DeclStmt>(For->getInit())) {
+        if (auto *InitDecl = dyn_cast<VarDecl>(InitStmt->getSingleDecl())) {
+          QualType InductionVarType = InitDecl->getType().getCanonicalType();
+
+          // Compare with first loop type
+          if (BaseInductionVarType.isNull()) {
+            BaseInductionVarType = InductionVarType;
+          } else if (!Context.hasSameType(BaseInductionVarType,
+                                          InductionVarType)) {
+            Diag(InitDecl->getBeginLoc(),
+                 diag::warn_omp_different_loop_ind_var_types)
+                << getOpenMPDirectiveName(OMPD_fuse) << BaseInductionVarType
+                << InductionVarType;
+          }
+        }
+      }
+
+    } else {
+      assert(isa<CXXForRangeStmt>(LoopStmt) &&
+             "Expected canonical for or range-based for loops.");
+      auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt);
+      OriginalInits.back().push_back(CXXFor->getBeginStmt());
+      ForStmts.push_back(CXXFor);
+    }
+  };
+  // Helper lambda functions to encapsulate the processing of different
+  // derivations of the canonical loop sequence grammar
+  //
+  // Modularized code for handling loop generation and transformations
+  auto handleLoopGeneration = [&storeLoopStatements, &LoopHelpers,
+                               &OriginalInits, &LoopSeqSize, &NumLoops, Kind,
+                               &TmpDSA, &OnTransformationCallback,
+                               this](Stmt *Child) {
+    auto LoopTransform = dyn_cast<OMPLoopTransformationDirective>(Child);
+    Stmt *TransformedStmt = LoopTransform->getTransformedStmt();
+    unsigned NumGeneratedLoopNests = LoopTransform->getNumGeneratedLoopNests();
+
+    // Handle the case where transformed statement is not available due to
+    // dependent contexts
+    if (!TransformedStmt) {
+      if (NumGeneratedLoopNests > 0)
+        return true;
+      // Unroll full
+      else {
+        Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+            << 0 << getOpenMPDirectiveName(Kind);
+        return false;
+      }
+    }
+    // Handle loop transformations with multiple loop nests
+    // Unroll full
+    if (NumGeneratedLoopNests <= 0) {
+      Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+          << 0 << getOpenMPDirectiveName(Kind);
+      return false;
+      // Future loop transformations that generate multiple canonical loops
+    } else if (NumGeneratedLoopNests > 1) {
+      llvm_unreachable("Multiple canonical loop generating transformations "
+                       "like loop splitting are not yet supported");
+    }
+
+    // Process the transformed loop statement
+    Child = TransformedStmt;
+    OriginalInits.emplace_back();
+    LoopHelpers.emplace_back();
+    OnTransformationCallback(LoopTransform);
+
+    unsigned IsCanonical =
+        checkOpenMPLoop(Kind, nullptr, nullptr, Child, SemaRef, *DSAStack,
+                        TmpDSA, LoopHelpers[LoopSeqSize]);
+
+    if (!IsCanonical) {
+      Diag(Child->getBeginLoc(), diag::err_omp_not_canonical_loop)
+          << getOpenMPDirectiveName(Kind);
+      return false;
+    }
+    storeLoopStatements(TransformedStmt);
+    NumLoops += LoopTransform->getNumGeneratedLoops();
+    return true;
+  };
+
+  // Modularized code for handling regular canonical loops
+  auto handleRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
+                            &LoopSeqSize, &NumLoops, Kind, &TmpDSA, &NLCV,
+                            this](Stmt *Child) {
+    OriginalInits.emplace_back();
+    LoopHelpers.emplace_back();
+    unsigned IsCanonical =
+        checkOpenMPLoop(Kind, nullptr, nullptr, Child, SemaRef, *DSAStack,
+                        TmpDSA, LoopHelpers[LoopSeqSize]);
+
+    if (!IsCanonical) {
+      Diag(Child->getBeginLoc(), diag::err_omp_not_canonical_loop)
+          << getOpenMPDirectiveName(Kind);
+      return false;
+    }
+    storeLoopStatements(Child);
+    NumLoops += NLCV.TraverseStmt(Child);
+    return true;
+  };
+
+  // Helper function to process a Loop Sequence Recursively
+  auto handleLoopSequence = [&](Stmt *LoopSeqStmt,
+                                auto &handleLoopSequenceCallback) -> bool {
+    for (auto *Child : LoopSeqStmt->children()) {
+      if (!Child)
+        continue;
+
+      // Skip over non-loop-sequence statements
+      if (!isLoopSequenceDerivation(Child)) {
+        Child = Child->IgnoreContainers();
+
+        // Ignore empty compound statement
+        if (!Child)
+          continue;
+
+        // In the case of a nested loop sequence ignoring containers would not
+        // be enough, a recurisve transversal of the loop sequence is required
+        if (isa<CompoundStmt>(Child)) {
+          if (!handleLoopSequenceCallback(Child, handleLoopSequenceCallback))
+            return false;
+          // Already been treated, skip this children
+          continue;
+        }
+      }
+      // Regular loop sequence handling
+      if (isLoopSequenceDerivation(Child)) {
+        if (isLoopGeneratingStmt(Child)) {
+          if (!handleLoopGeneration(Child)) {
+            return false;
+          }
+        } else {
+          if (!handleRegularLoop(Child)) {
+            return false;
+          }
+        }
+        ++LoopSeqSize;
+      } else {
+        // Report error for invalid statement inside canonical loop sequence
+        Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+            << 0 << getOpenMPDirectiveName(Kind);
+        return false;
+      }
+    }
+    return true;
+  };
+
+  // Recursive entry point to process the main loop sequence
+  if (!handleLoopSequence(AStmt, handleLoopSequence)) {
+    return false;
+  }
+
+  if (LoopSeqSize <= 0) {
+    Diag(AStmt->getBeginLoc(), diag::err_omp_empty_loop_sequence)
+        << getOpenMPDirectiveName(Kind);
+    return false;
+  }
+  return true;
+}
+
 /// Add preinit statements that need to be propageted from the selected loop.
 static void addLoopPreInits(ASTContext &Context,
                             OMPLoopBasedDirective::HelperExprs &LoopHelper,
@@ -15499,6 +15765,340 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
+                                                Stmt *AStmt,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurrContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+  CaptureVars CopyTransformer(SemaRef);
+
+  // Ensure the structured block is not empty
+  if (!AStmt) {
+    return StmtError();
+  }
+  // Validate that the potential loop sequence is transformable for fusion
+  // Also collect the HelperExprs, Loop Stmts, Inits, and Number of loops
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers;
+  SmallVector<Stmt *> LoopStmts;
+  SmallVector<SmallVector<Stmt *, 0>> OriginalInits;
+
+  unsigned NumLoops;
+  // TODO: Support looprange clause using LoopSeqSize
+  unsigned LoopSeqSize;
+  if (!checkTransformableLoopSequence(OMPD_fuse, AStmt, LoopSeqSize, NumLoops,
+                                      LoopHelpers, LoopStmts, OriginalInits,
+                                      Context)) {
+    return StmtError();
+  }
+
+  // Defer transformation in dependent contexts
+  if (CurrContext->isDependentContext()) {
+    return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                    NumLoops, 1, AStmt, nullptr, nullptr);
+  }
+  assert(LoopHelpers.size() == LoopSeqSize &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == LoopSeqSize &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // PreInits hold a sequence of variable declarations that must be executed
+  // before the fused loop begins. These include bounds, strides, and other
+  // helper variables required for the transformation.
+  SmallVector<Stmt *> PreInits;
+
+  // Select the type with the largest bit width among all induction variables
+  QualType IVType = LoopHelpers[0].IterationVarRef->getType();
+  for (unsigned int I = 1; I < LoopSeqSize; ++I) {
+    QualType CurrentIVType = LoopHelpers[I].IterationVarRef->getType();
+    if (Context.getTypeSize(CurrentIVType) > Context.getTypeSize(IVType)) {
+      IVType = CurrentIVType;
+    }
+  }
+  uint64_t IVBitWidth = Context.getIntWidth(IVType);
+
+  // Create pre-init declarations for all loops lower bounds, upper bounds,
+  // strides and num-iterations
+  SmallVector<VarDecl *, 4> LBVarDecls;
+  SmallVector<VarDecl *, 4> STVarDecls;
+  SmallVector<VarDecl *, 4> NIVarDecls;
+  SmallVector<VarDecl *, 4> UBVarDecls;
+  SmallVector<VarDecl *, 4> IVVarDecls;
+
+  // Helper lambda to create variables for bounds, strides, and other
+  // expressions. Generates both the variable declaration and the corresponding
+  // initialization statement.
+  auto CreateHelperVarAndStmt =
+      [&SemaRef = this->SemaRef, &Context, &CopyTransformer,
+       &IVType](Expr *ExprToCopy, const std::string &BaseName, unsigned I,
+                bool NeedsNewVD = false) {
+        Expr *TransformedExpr =
+            AssertSuccess(CopyTransformer.TransformExpr(ExprToCopy));
+        if (!TransformedExpr)
+          return std::pair<VarDecl *, StmtResult>(nullptr, StmtError());
+
+        auto Name = (Twine(".omp.") + BaseName + std::to_string(I)).str();
+
+        VarDecl *VD;
+        if (NeedsNewVD) {
+          VD = buildVarDecl(SemaRef, SourceLocation(), IVType, Name);
+          SemaRef.AddInitializerToDecl(VD, TransformedExpr, false);
+
+        } else {
+          // Create a unique variable name
+          DeclRefExpr *DRE = cast<DeclRefExpr>(TransformedExpr);
+          VD = cast<VarDecl>(DRE->getDecl());
+          VD->setDeclName(&SemaRef.PP.getIdentifierTable().get(Name));
+        }
+        // Create the corresponding declaration statement
+        StmtResult DeclStmt = new (Context) class DeclStmt(
+            DeclGroupRef(VD), SourceLocation(), SourceLocation());
+        return std::make_pair(VD, DeclStmt);
+      };
+
+  // Process each single loop to generate and collect declarations
+  // and statements for all helper expressions
+  for (unsigned int I = 0; I < LoopSeqSize; ++I) {
+    addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
+                    PreInits);
+
+    auto [UBVD, UBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].UB, "ub", I);
+    auto [LBVD, LBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].LB, "lb", I);
+    auto [STVD, STDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].ST, "st", I);
+    auto [NIVD, NIDStmt] =
+        CreateHelperVarAndStmt(LoopHelpers[I].NumIterations, "ni", I, true);
+    auto [IVVD, IVDStmt] =
+        CreateHelperVarAndStmt(LoopHelpers[I].IterationVarRef, "iv", I);
+
+    if (!LBVD || !STVD || !NIVD || !IVVD)
+      return StmtError();
+
+    UBVarDecls.push_back(UBVD);
+    LBVarDecls.push_back(LBVD);
+    STVarDecls.push_back(STVD);
+    NIVarDecls.push_back(NIVD);
+    IVVarDecls.push_back(IVVD);
+
+    PreInits.push_back(UBDStmt.get());
+    PreInits.push_back(LBDStmt.get());
+    PreInits.push_back(STDStmt.get());
+    PreInits.push_back(NIDStmt.get());
+    PreInits.push_back(IVDStmt.get());
+  }
+
+  auto MakeVarDeclRef = [&SemaRef = this->SemaRef](VarDecl *VD) {
+    return buildDeclRefExpr(SemaRef, VD, VD->getType(), VD->getLocation(),
+                            false);
+  };
+
+  // Following up the creation of the final fused loop will be performed
+  // which has the following shape (considering the selected loops):
+  //
+  // for (fuse.index = 0; fuse.index < max(ni0, ni1..., nik); ++fuse.index) {
+  //    if (fuse.index < ni0){
+  //      iv0 = lb0 + st0 * fuse.index;
+  //      original.index0 = iv0
+  //      body(0);
+  //    }
+  //    if (fuse.index < ni1){
+  //      iv1 = lb1 + st1 * fuse.index;
+  //      original.index1 = iv1
+  //      body(1);
+  //    }
+  //
+  //    ...
+  //
+  //    if (fuse.index < nik){
+  //      ivk = lbk + stk * fuse.index;
+  //      original.indexk = ivk
+  //      body(k);  Expr *InitVal = IntegerLiteral::Create(Context,
+  //      llvm::APInt(IVWidth, 0),
+
+  //    }
+
+  // 1. Create the initialized fuse index
+  const std::string IndexName = Twine(".omp.fuse.index").str();
+  Expr *InitVal = IntegerLiteral::Create(Context, llvm::APInt(IVBitWidth, 0),
+                                         IVType, SourceLocation());
+  VarDecl *IndexDecl =
+      buildVarDecl(SemaRef, {}, IVType, IndexName, nullptr, nullptr);
+  SemaRef.AddInitializerToDecl(IndexDecl, InitVal, false);
+  StmtResult InitStmt = new (Context)
+      DeclStmt(DeclGroupRef(IndexDecl), SourceLocation(), SourceLocation());
+
+  if (!InitStmt.isUsable())
+    return StmtError();
+
+  auto MakeIVRef = [&SemaRef = this->SemaRef, IndexDecl, IVType,
+                    Loc = InitVal->getExprLoc()]() {
+    return buildDeclRefExpr(SemaRef, IndexDecl, IVType, Loc, false);
+  };
+
+  // 2. Iteratively compute the max number of logical iterations Max(NI_1, NI_2,
+  // ..., NI_k)
+  //
+  // This loop accumulates the maximum value across multiple expressions,
+  // ensuring each step constructs a unique AST node for correctness. By using
+  // intermediate temporary variables and conditional operators, we maintain
+  // distinct nodes and avoid duplicating subtrees,  For instance, max(a,b,c):
+  //   omp.temp0 = max(a, b)
+  //   omp.temp1 = max(omp.temp0, c)
+  //   omp.fuse.max = max(omp.temp1, omp.temp0)
+
+  ExprResult MaxExpr;
+  for (unsigned I = 0; I < LoopSeqSize; ++I) {
+    DeclRefExpr *NIRef = MakeVarDeclRef(NIVarDecls[I]);
+    QualType NITy = NIRef->getType();
+
+    if (MaxExpr.isUnset()) {
+      // Initialize MaxExpr with the first NI expression
+      MaxExpr = NIRef;
+    } else {
+      // Create a new acummulator variable t_i = MaxExpr
+      std::string TempName = (Twine(".omp.temp.") + Twine(I)).str();
+      VarDecl *TempDecl =
+          buildVarDecl(SemaRef, {}, NITy, TempName, nullptr, nullptr);
+      TempDecl->setInit(MaxExpr.get());
+      DeclRefExpr *TempRef =
+          buildDeclRefExpr(SemaRef, TempDecl, NITy, SourceLocation(), false);
+      DeclRefExpr *TempRef2 =
+          buildDeclRefExpr(SemaRef, TempDecl, NITy, SourceLocation(), false);
+      // Add a DeclStmt to PreInits to ensure the variable is declared.
+      StmtResult TempStmt = new (Context)
+          DeclStmt(DeclGroupRef(TempDecl), SourceLocation(), SourceLocation());
+
+      if (!TempStmt.isUsable())
+        return StmtError();
+      PreInits.push_back(TempStmt.get());
+
+      // Build MaxExpr <-(MaxExpr > NIRef ? MaxExpr : NIRef)
+      ExprResult Comparison =
+          SemaRef.BuildBinOp(nullptr, SourceLocation(), BO_GT, TempRef, NIRef);
+      // Handle any errors in Comparison creation
+      if (!Comparison.isUsable())
+        return StmtError();
+
+      DeclRefExpr *NIRef2 = MakeVarDeclRef(NIVarDecls[I]);
+      // Update MaxExpr using a conditional expression to hold the max value
+      MaxExpr = new (Context) ConditionalOperator(
+          Comparison.get(), SourceLocation(), TempRef2, SourceLocation(),
+          NIRef2->getExprStmt(), NITy, VK_LValue, OK_Ordinary);
+
+      if (!MaxExpr.isUsable())
+        return StmtError();
+    }
+  }
+  if (!MaxExpr.isUsable())
+    return StmtError();
+
+  // 3. Declare the max variable
+  const std::string MaxName = Twine(".omp.fuse.max").str();
+  VarDecl *MaxDecl =
+      buildVarDecl(SemaRef, {}, IVType, MaxName, nullptr, nullptr);
+  MaxDecl->setInit(MaxExpr.get());
+  DeclRefExpr *MaxRef = buildDeclRefExpr(SemaRef, MaxDecl, IVType, {}, false);
+  StmtResult MaxStmt = new (Context)
+      DeclStmt(DeclGroupRef(MaxDecl), SourceLocation(), SourceLocation());
+
+  if (MaxStmt.isInvalid())
+    return StmtError();
+  PreInits.push_back(MaxStmt.get());
+
+  // 4. Create condition Expr: index < n_max
+  ExprResult CondExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_LT,
+                                           MakeIVRef(), MaxRef);
+  if (!CondExpr.isUsable())
+    return StmtError();
+  // 5. Increment Expr: ++index
+  ExprResult IncrExpr =
+      SemaRef.BuildUnaryOp(CurScope, SourceLocation(), UO_PreInc, MakeIVRef());
+  if (!IncrExpr.isUsable())
+    return StmtError();
+
+  // 6. Build the Fused Loop Body
+  // The final fused loop iterates over the maximum logical range. Inside the
+  // loop, each original loop's index is calculated dynamically, and its body
+  // is executed conditionally.
+  //
+  // Each sub-loop's body is guarded by a conditional statement to ensure
+  // it executes only within its logical iteration range:
+  //
+  //    if (fuse.index < ni_k){
+  //      iv_k = lb_k + st_k * fuse.index;
+  //      original.index = iv_k
+  //      body(k);
+  //    }
+
+  CompoundStmt *FusedBody = nullptr;
+  SmallVector<Stmt *, 4> FusedBodyStmts;
+  for (unsigned I = 0; I < LoopSeqSize; ++I) {
+
+    // Assingment of the original sub-loop index to compute the logical index
+    // IV_k = LB_k + omp.fuse.index * ST_k
+
+    ExprResult IdxExpr =
+        SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Mul,
+                           MakeVarDeclRef(STVarDecls[I]), MakeIVRef());
+    if (!IdxExpr.isUsable())
+      return StmtError();
+    IdxExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Add,
+                                 MakeVarDeclRef(LBVarDecls[I]), IdxExpr.get());
+
+    if (!IdxExpr.isUsable())
+      return StmtError();
+    IdxExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Assign,
+                                 MakeVarDeclRef(IVVarDecls[I]), IdxExpr.get());
+    if (!IdxExpr.isUsable())
+      return StmtError();
+
+    // Update the original i_k = IV_k
+    SmallVector<Stmt *, 4> BodyStmts;
+    BodyStmts.push_back(IdxExpr.get());
+    llvm::append_range(BodyStmts, LoopHelpers[I].Updates);
+
+    if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(SourceCXXFor->getLoopVarStmt());
+
+    Stmt *Body = (isa<ForStmt>(LoopStmts[I]))
+                     ? cast<ForStmt>(LoopStmts[I])->getBody()
+                     : cast<CXXForRangeStmt>(LoopStmts[I])->getBody();
+
+    BodyStmts.push_back(Body);
+
+    CompoundStmt *CombinedBody =
+        CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                             SourceLocation(), SourceLocation());
+    ExprResult Condition =
+        SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_LT, MakeIVRef(),
+                           MakeVarDeclRef(NIVarDecls[I]));
+
+    if (!Condition.isUsable())
+      return StmtError();
+
+    IfStmt *IfStatement = IfStmt::Create(
+        Context, SourceLocation(), IfStatementKind::Ordinary, nullptr, nullptr,
+        Condition.get(), SourceLocation(), SourceLocation(), CombinedBody,
+        SourceLocation(), nullptr);
+
+    FusedBodyStmts.push_back(IfStatement);
+  }
+  FusedBody = CompoundStmt::Create(Context, FusedBodyStmts, FPOptionsOverride(),
+                                   SourceLocation(), SourceLocation());
+
+  // 7. Construct the final fused loop
+  ForStmt *FusedForStmt = new (Context)
+      ForStmt(Context, InitStmt.get(), CondExpr.get(), nullptr, IncrExpr.get(),
+              FusedBody, InitStmt.get()->getBeginLoc(), SourceLocation(),
+              IncrExpr.get()->getEndLoc());
+
+  return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses, NumLoops,
+                                  1, AStmt, FusedForStmt,
+                                  buildPreInits(Context, PreInits));
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
                                                    Expr *Expr,
                                                    SourceLocation StartLoc,
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3e33fb73e01b4..45f556f22c511 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -9675,6 +9675,17 @@ StmtResult TreeTransform<Derived>::TransformOMPInterchangeDirective(
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPForDirective(OMPForDirective *D) {
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 44cfb83ad2db4..291bd8ea4bf18 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2445,6 +2445,7 @@ void ASTStmtReader::VisitOMPLoopTransformationDirective(
     OMPLoopTransformationDirective *D) {
   VisitOMPLoopBasedDirective(D);
   D->setNumGeneratedLoops(Record.readUInt32());
+  D->setNumGeneratedLoopNests(Record.readUInt32());
 }
 
 void ASTStmtReader::VisitOMPTileDirective(OMPTileDirective *D) {
@@ -2467,6 +2468,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPLoopTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFuseDirective(OMPFuseDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPForDirective(OMPForDirective *D) {
   VisitOMPLoopDirective(D);
   D->setHasCancel(Record.readBool());
@@ -3608,6 +3613,12 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       S = OMPReverseDirective::CreateEmpty(Context, NumLoops);
       break;
     }
+    case STMT_OMP_FUSE_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFuseDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
 
     case STMT_OMP_INTERCHANGE_DIRECTIVE: {
       unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index a6e320c7f3eb0..5bf1ecfb968e8 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2449,6 +2449,7 @@ void ASTStmtWriter::VisitOMPLoopTransformationDirective(
     OMPLoopTransformationDirective *D) {
   VisitOMPLoopBasedDirective(D);
   Record.writeUInt32(D->getNumGeneratedLoops());
+  Record.writeUInt32(D->getNumGeneratedLoopNests());
 }
 
 void ASTStmtWriter::VisitOMPTileDirective(OMPTileDirective *D) {
@@ -2476,6 +2477,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFuseDirective(OMPFuseDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+  Code = serialization::STMT_OMP_FUSE_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPForDirective(OMPForDirective *D) {
   VisitOMPLoopDirective(D);
   Record.writeBool(D->hasCancel());
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index c77ef26da568d..7218d7e62acdd 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1814,6 +1814,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:
     case Stmt::OMPDispatchDirectiveClass:
     case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/test/OpenMP/fuse_ast_print.cpp b/clang/test/OpenMP/fuse_ast_print.cpp
new file mode 100644
index 0000000000000..43ce815dab024
--- /dev/null
+++ b/clang/test/OpenMP/fuse_ast_print.cpp
@@ -0,0 +1,278 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing 
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip 
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -std=c++20 -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER 
+
+// placeholder for loop body code
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL: FunctionDecl {{.*}} foo1
+void foo1() {
+    // PRINT: #pragma omp fuse
+    // DUMP:  OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int j = 10; j > 0; --j)
+        // DUMP: ForStmt
+        for (int j = 10; j > 0; --j)
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);
+        // PRINT: for (int k = 0; k <= 10; ++k)
+        // DUMP: ForStmt
+        for (int k = 0; k <= 10; ++k)
+            // PRINT: body(k)
+            // DUMP: CallExpr
+            body(k);
+
+    }
+
+}
+
+// PRINT-LABEL: void foo2(
+// DUMP-LABEL: FunctionDecl {{.*}} foo2
+void foo2() {
+    // PRINT: #pragma omp unroll partial(4)
+    // DUMP: OMPUnrollDirective
+    // DUMP-NEXT: OMPPartialClause
+    // DUMP-NEXT: ConstantExpr
+    // DUMP-NEXT: value: Int 4
+    // DUMP-NEXT: IntegerLiteral {{.*}} 4
+    #pragma omp unroll partial(4)
+    // PRINT: #pragma omp fuse
+    // DUMP-NEXT: OMPFuseDirective 
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int j = 10; j > 0; --j)
+        // DUMP: ForStmt
+        for (int j = 10; j > 0; --j)
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);  
+    }    
+    
+}
+
+//PRINT-LABEL: void foo3(
+//DUMP-LABEL: FunctionTemplateDecl {{.*}} foo3
+template<int Factor1, int Factor2> 
+void foo3() {
+    // PRINT:  #pragma omp fuse
+    // DUMP: OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: #pragma omp unroll partial(Factor1)
+        // DUMP: OMPUnrollDirective
+        #pragma omp unroll partial(Factor1)
+        // PRINT: for (int i = 0; i < 12; i += 1)
+        // DUMP: ForStmt
+        for (int i = 0; i < 12; i += 1)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: #pragma omp unroll partial(Factor2)
+        // DUMP: OMPUnrollDirective
+        #pragma omp unroll partial(Factor2)
+        // PRINT: for (int k = 0; k <= 10; ++k)
+        // DUMP: ForStmt
+        for (int k = 0; k <= 10; ++k)
+            // PRINT: body(k)
+            // DUMP: CallExpr
+            body(k);
+
+    }
+}
+
+// Also test instantiating the template.
+void tfoo3() {
+    foo3<4,2>();
+}
+
+//PRINT-LABEL: void foo4(
+//DUMP-LABEL: FunctionTemplateDecl {{.*}} foo4
+template<typename T, T Step> 
+void foo4(int start, int end) {
+    // PRINT:  #pragma omp fuse
+    // DUMP: OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (T i = start; i < end; i += Step)
+        // DUMP: ForStmt
+        for (T i = start; i < end; i += Step)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+
+        // PRINT: for (T j = end; j > start; j -= Step)
+        // DUMP: ForStmt 
+        for (T j = end; j > start; j -= Step) {
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);
+        }
+
+    }
+}
+
+// Also test instantiating the template.
+void tfoo4() {
+    foo4<int, 4>(0, 64);
+}
+
+
+
+// PRINT-LABEL: void foo5(
+// DUMP-LABEL: FunctionDecl {{.*}} foo5
+void foo5() {
+    double arr[128], arr2[128];
+    // PRINT: #pragma omp fuse
+    // DUMP:  OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT-NEXT: for (auto &&a : arr)
+        // DUMP-NEXT: CXXForRangeStmt
+        for (auto &&a: arr)
+            // PRINT: body(a)
+            // DUMP: CallExpr
+            body(a);
+        // PRINT: for (double v = 42; auto &&b : arr)
+        // DUMP: CXXForRangeStmt
+        for (double v = 42; auto &&b: arr)
+            // PRINT: body(b, v);
+            // DUMP: CallExpr
+            body(b, v);
+        // PRINT: for (auto &&c : arr2)
+        // DUMP: CXXForRangeStmt
+        for (auto &&c: arr2)
+            // PRINT: body(c)
+            // DUMP: CallExpr
+            body(c);
+
+    }
+
+}
+
+// PRINT-LABEL: void foo6(
+// DUMP-LABEL: FunctionDecl {{.*}} foo6
+void foo6() {
+    // PRINT: #pragma omp fuse
+    // DUMP: OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt
+    {
+        // PRINT: #pragma omp fuse
+        // DUMP: OMPFuseDirective
+        #pragma omp fuse 
+        // PRINT: {
+        // DUMP: CompoundStmt
+        {
+            // PRINT: for (int i = 0; i <= 10; ++i)
+            // DUMP: ForStmt
+            for (int i = 0; i <= 10; ++i)
+                body(i);
+            // PRINT: for (int j = 0; j < 100; ++j)
+            // DUMP: ForStmt
+            for(int j = 0; j < 100; ++j)
+                body(j);
+        }
+        // PRINT: #pragma omp unroll partial(4)
+        // DUMP: OMPUnrollDirective
+        #pragma omp unroll partial(4)
+        // PRINT: for (int k = 0; k < 250; ++k)
+        // DUMP: ForStmt
+        for (int k = 0; k < 250; ++k) 
+            body(k);
+    }
+}
+
+// PRINT-LABEL: void foo7(
+// DUMP-LABEL: FunctionDecl {{.*}} foo7
+void foo7() {
+    // PRINT: #pragma omp fuse
+    // DUMP:  OMPFuseDirective
+    #pragma omp fuse 
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: {
+        // DUMP: CompoundStmt   
+        {
+            // PRINT: {
+            // DUMP: CompoundStmt   
+            {
+                // PRINT: for (int i = 0; i < 10; i += 2)
+                // DUMP: ForStmt
+                for (int i = 0; i < 10; i += 2)
+                    // PRINT: body(i)
+                    // DUMP: CallExpr
+                    body(i);
+                // PRINT: for (int j = 10; j > 0; --j)
+                // DUMP: ForStmt
+                for (int j = 10; j > 0; --j)
+                    // PRINT: body(j)
+                    // DUMP: CallExpr
+                    body(j);
+            }
+        }
+        // PRINT: {
+        // DUMP: CompoundStmt   
+        {
+            // PRINT: {
+            // DUMP: CompoundStmt   
+            {
+                // PRINT: {
+                // DUMP: CompoundStmt   
+                {
+                    // PRINT: for (int k = 0; k <= 10; ++k)
+                    // DUMP: ForStmt
+                    for (int k = 0; k <= 10; ++k)
+                        // PRINT: body(k)
+                        // DUMP: CallExpr
+                        body(k);
+                }
+            }
+        }
+    }
+
+}
+
+
+
+
+
+#endif
\ No newline at end of file
diff --git a/clang/test/OpenMP/fuse_codegen.cpp b/clang/test/OpenMP/fuse_codegen.cpp
new file mode 100644
index 0000000000000..6c1e21092da43
--- /dev/null
+++ b/clang/test/OpenMP/fuse_codegen.cpp
@@ -0,0 +1,1511 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --replace-value-regex "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --version 5
+// expected-no-diagnostics
+
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fclang-abi-compat=latest -fopenmp -fopenmp-version=60 -include-pch %t -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK2
+
+#ifndef HEADER
+#define HEADER
+
+//placeholder for loop body code.
+extern "C" void body(...) {}
+
+extern "C" void foo1(int start1, int end1, int step1, int start2, int end2, int step2) {
+    int i,j;
+    #pragma omp fuse
+    {
+        for(i = start1; i < end1; i += step1) body(i);
+        for(j = start2; j < end2; j += step2) body(j);
+    }
+
+}
+
+template <typename T>
+void foo2(T start, T end, T step){
+    T i,j,k;
+    #pragma omp fuse
+    {
+        for(i = start; i < end; i += step) body(i);
+        for(j = end; j > start; j -= step) body(j);
+        for(k = start+step; k < end+step; k += step) body(k);
+    }
+}
+
+extern "C" void tfoo2() {
+    foo2<int>(0, 64, 4);
+}
+
+extern "C" void foo3() {
+    double arr[256];
+    #pragma omp fuse
+    {
+        #pragma omp fuse
+        {
+            for(int i = 0; i < 128; ++i) body(i);
+            for(int j = 0; j < 256; j+=2) body(j);
+        }
+        for(int c = 42; auto &&v: arr) body(c,v);
+        for(int cc = 37; auto &&vv: arr) body(cc, vv);
+    }
+}
+
+
+#endif
+// CHECK1-LABEL: define dso_local void @body(
+// CHECK1-SAME: ...) #[[ATTR0:[0-9]+]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define dso_local void @foo1(
+// CHECK1-SAME: i32 noundef [[START1:%.*]], i32 noundef [[END1:%.*]], i32 noundef [[STEP1:%.*]], i32 noundef [[START2:%.*]], i32 noundef [[END2:%.*]], i32 noundef [[STEP2:%.*]]) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[START1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[END1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[STEP1_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[START2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[END2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[STEP2_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    store i32 [[START1]], ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[END1]], ptr [[END1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[STEP1]], ptr [[STEP1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[START2]], ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[END2]], ptr [[END2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[STEP2]], ptr [[STEP2_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP1_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK1-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
+// CHECK1-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP20]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP21]], [[TMP22]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP23]], %[[COND_TRUE]] ], [ [[TMP24]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
+// CHECK1-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP17]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP30]], [[TMP31]]
+// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP29]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP33]], [[TMP34]]
+// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP32]], [[MUL19]]
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP35]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    br i1 [[CMP21]], label %[[IF_THEN22:.*]], label %[[IF_END27:.*]]
+// CHECK1:       [[IF_THEN22]]:
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP39]], [[TMP40]]
+// CHECK1-NEXT:    [[ADD24:%.*]] = add i32 [[TMP38]], [[MUL23]]
+// CHECK1-NEXT:    store i32 [[ADD24]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[TMP41]], [[MUL25]]
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK1-NEXT:    br label %[[IF_END27]]
+// CHECK1:       [[IF_END27]]:
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP45]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define dso_local void @tfoo2(
+// CHECK1-SAME: ) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    call void @_Z4foo2IiEvT_S0_S0_(i32 noundef 0, i32 noundef 64, i32 noundef 4)
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define linkonce_odr void @_Z4foo2IiEvT_S0_S0_(
+// CHECK1-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0]] comdat {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTNEW_STEP21:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK1-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
+// CHECK1-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP29]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP30]]
+// CHECK1-NEXT:    [[SUB27:%.*]] = sub i32 [[DIV26]], 1
+// CHECK1-NEXT:    store i32 [[SUB27]], ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK1-NEXT:    store i32 [[TMP31]], ptr [[DOTOMP_UB2]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB2]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST2]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK1-NEXT:    [[ADD28:%.*]] = add i32 [[TMP32]], 1
+// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP33]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP34]], [[TMP35]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP36]], %[[COND_TRUE]] ], [ [[TMP37]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    br i1 [[CMP29]], label %[[COND_TRUE30:.*]], label %[[COND_FALSE31:.*]]
+// CHECK1:       [[COND_TRUE30]]:
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    br label %[[COND_END32:.*]]
+// CHECK1:       [[COND_FALSE31]]:
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    br label %[[COND_END32]]
+// CHECK1:       [[COND_END32]]:
+// CHECK1-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP40]], %[[COND_TRUE30]] ], [ [[TMP41]], %[[COND_FALSE31]] ]
+// CHECK1-NEXT:    store i32 [[COND33]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    br i1 [[CMP34]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP44]], [[TMP45]]
+// CHECK1-NEXT:    br i1 [[CMP35]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP47]], [[TMP48]]
+// CHECK1-NEXT:    [[ADD36:%.*]] = add i32 [[TMP46]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD36]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP50]], [[TMP51]]
+// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[TMP49]], [[MUL37]]
+// CHECK1-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP53]], [[TMP54]]
+// CHECK1-NEXT:    br i1 [[CMP39]], label %[[IF_THEN40:.*]], label %[[IF_END45:.*]]
+// CHECK1:       [[IF_THEN40]]:
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[ADD42:%.*]] = add i32 [[TMP55]], [[MUL41]]
+// CHECK1-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP58]], [[MUL43]]
+// CHECK1-NEXT:    store i32 [[SUB44]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP61]])
+// CHECK1-NEXT:    br label %[[IF_END45]]
+// CHECK1:       [[IF_END45]]:
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP62]], [[TMP63]]
+// CHECK1-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
+// CHECK1:       [[IF_THEN47]]:
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP65]], [[TMP66]]
+// CHECK1-NEXT:    [[ADD49:%.*]] = add i32 [[TMP64]], [[MUL48]]
+// CHECK1-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV2]], align 4
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP68]], [[TMP69]]
+// CHECK1-NEXT:    [[ADD51:%.*]] = add i32 [[TMP67]], [[MUL50]]
+// CHECK1-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP70]])
+// CHECK1-NEXT:    br label %[[IF_END52]]
+// CHECK1:       [[IF_END52]]:
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK1-LABEL: define dso_local void @foo3(
+// CHECK1-SAME: ) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB03:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB04:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST05:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI06:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV07:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_UB117:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB118:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_ST119:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_NI120:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV122:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[CC:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE223:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END224:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN227:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_29:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_31:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_32:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_TEMP_142:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX48:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX54:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[VV:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK1-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTOMP_UB03]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB04]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST05]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], 1
+// CHECK1-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
+// CHECK1-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI06]], align 8
+// CHECK1-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY8:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY8]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY10:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP11]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY10]], ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP12]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP14]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK1-NEXT:    [[SUB13:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i64 [[SUB13]], 1
+// CHECK1-NEXT:    [[DIV15:%.*]] = sdiv i64 [[ADD14]], 1
+// CHECK1-NEXT:    [[SUB16:%.*]] = sub nsw i64 [[DIV15]], 1
+// CHECK1-NEXT:    store i64 [[SUB16]], ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_UB117]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB118]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST119]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK1-NEXT:    store i64 [[ADD21]], ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    store i32 37, ptr [[CC]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE223]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY25:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR26:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY25]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR26]], ptr [[__END224]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP18]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[__BEGIN227]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY30:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP19]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY30]], ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[__END224]], align 8
+// CHECK1-NEXT:    store ptr [[TMP20]], ptr [[DOTCAPTURE_EXPR_31]], align 8
+// CHECK1-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_31]], align 8
+// CHECK1-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST33:%.*]] = ptrtoint ptr [[TMP21]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST34:%.*]] = ptrtoint ptr [[TMP22]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB35:%.*]] = sub i64 [[SUB_PTR_LHS_CAST33]], [[SUB_PTR_RHS_CAST34]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV36:%.*]] = sdiv exact i64 [[SUB_PTR_SUB35]], 8
+// CHECK1-NEXT:    [[SUB37:%.*]] = sub nsw i64 [[SUB_PTR_DIV36]], 1
+// CHECK1-NEXT:    [[ADD38:%.*]] = add nsw i64 [[SUB37]], 1
+// CHECK1-NEXT:    [[DIV39:%.*]] = sdiv i64 [[ADD38]], 1
+// CHECK1-NEXT:    [[SUB40:%.*]] = sub nsw i64 [[DIV39]], 1
+// CHECK1-NEXT:    store i64 [[SUB40]], ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK1-NEXT:    store i64 [[TMP23]], ptr [[DOTOMP_UB2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB2]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST2]], align 8
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK1-NEXT:    [[ADD41:%.*]] = add nsw i64 [[TMP24]], 1
+// CHECK1-NEXT:    store i64 [[ADD41]], ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
+// CHECK1-NEXT:    store i64 [[TMP25]], ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP26]], [[TMP27]]
+// CHECK1-NEXT:    br i1 [[CMP43]], label %[[COND_TRUE44:.*]], label %[[COND_FALSE45:.*]]
+// CHECK1:       [[COND_TRUE44]]:
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK1-NEXT:    br label %[[COND_END46:.*]]
+// CHECK1:       [[COND_FALSE45]]:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    br label %[[COND_END46]]
+// CHECK1:       [[COND_END46]]:
+// CHECK1-NEXT:    [[COND47:%.*]] = phi i64 [ [[TMP28]], %[[COND_TRUE44]] ], [ [[TMP29]], %[[COND_FALSE45]] ]
+// CHECK1-NEXT:    store i64 [[COND47]], ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[CMP49:%.*]] = icmp sgt i64 [[TMP30]], [[TMP31]]
+// CHECK1-NEXT:    br i1 [[CMP49]], label %[[COND_TRUE50:.*]], label %[[COND_FALSE51:.*]]
+// CHECK1:       [[COND_TRUE50]]:
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    br label %[[COND_END52:.*]]
+// CHECK1:       [[COND_FALSE51]]:
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    br label %[[COND_END52]]
+// CHECK1:       [[COND_END52]]:
+// CHECK1-NEXT:    [[COND53:%.*]] = phi i64 [ [[TMP32]], %[[COND_TRUE50]] ], [ [[TMP33]], %[[COND_FALSE51]] ]
+// CHECK1-NEXT:    store i64 [[COND53]], ptr [[DOTOMP_FUSE_MAX48]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX48]], align 8
+// CHECK1-NEXT:    [[CMP55:%.*]] = icmp slt i64 [[TMP34]], [[TMP35]]
+// CHECK1-NEXT:    br i1 [[CMP55]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
+// CHECK1-NEXT:    [[CMP56:%.*]] = icmp slt i64 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    br i1 [[CMP56]], label %[[IF_THEN:.*]], label %[[IF_END76:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB04]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP38]] to i64
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST05]], align 4
+// CHECK1-NEXT:    [[CONV58:%.*]] = sext i32 [[TMP39]] to i64
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV58]], [[TMP40]]
+// CHECK1-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV57]], [[MUL]]
+// CHECK1-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK1-NEXT:    store i32 [[CONV60]], ptr [[DOTOMP_IV07]], align 4
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_IV07]], align 4
+// CHECK1-NEXT:    [[MUL61:%.*]] = mul nsw i32 [[TMP41]], 1
+// CHECK1-NEXT:    [[ADD62:%.*]] = add nsw i32 0, [[MUL61]]
+// CHECK1-NEXT:    store i32 [[ADD62]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP63:%.*]] = icmp slt i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    br i1 [[CMP63]], label %[[IF_THEN64:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN64]]:
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP45]], [[TMP46]]
+// CHECK1-NEXT:    [[ADD66:%.*]] = add nsw i32 [[TMP44]], [[MUL65]]
+// CHECK1-NEXT:    store i32 [[ADD66]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[MUL67:%.*]] = mul nsw i32 [[TMP47]], 1
+// CHECK1-NEXT:    [[ADD68:%.*]] = add nsw i32 0, [[MUL67]]
+// CHECK1-NEXT:    store i32 [[ADD68]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP48]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP69:%.*]] = icmp slt i32 [[TMP49]], [[TMP50]]
+// CHECK1-NEXT:    br i1 [[CMP69]], label %[[IF_THEN70:.*]], label %[[IF_END75:.*]]
+// CHECK1:       [[IF_THEN70]]:
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP52]], [[TMP53]]
+// CHECK1-NEXT:    [[ADD72:%.*]] = add nsw i32 [[TMP51]], [[MUL71]]
+// CHECK1-NEXT:    store i32 [[ADD72]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP54]], 2
+// CHECK1-NEXT:    [[ADD74:%.*]] = add nsw i32 0, [[MUL73]]
+// CHECK1-NEXT:    store i32 [[ADD74]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP55]])
+// CHECK1-NEXT:    br label %[[IF_END75]]
+// CHECK1:       [[IF_END75]]:
+// CHECK1-NEXT:    br label %[[IF_END76]]
+// CHECK1:       [[IF_END76]]:
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    [[CMP77:%.*]] = icmp slt i64 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    br i1 [[CMP77]], label %[[IF_THEN78:.*]], label %[[IF_END83:.*]]
+// CHECK1:       [[IF_THEN78]]:
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_LB118]], align 8
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_ST119]], align 8
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    [[ADD80:%.*]] = add nsw i64 [[TMP58]], [[MUL79]]
+// CHECK1-NEXT:    store i64 [[ADD80]], ptr [[DOTOMP_IV122]], align 8
+// CHECK1-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV122]], align 8
+// CHECK1-NEXT:    [[MUL81:%.*]] = mul nsw i64 [[TMP62]], 1
+// CHECK1-NEXT:    [[ADD_PTR82:%.*]] = getelementptr inbounds double, ptr [[TMP61]], i64 [[MUL81]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR82]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP63]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[C]], align 4
+// CHECK1-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP66:%.*]] = load double, ptr [[TMP65]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP64]], double noundef [[TMP66]])
+// CHECK1-NEXT:    br label %[[IF_END83]]
+// CHECK1:       [[IF_END83]]:
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP67]], [[TMP68]]
+// CHECK1-NEXT:    br i1 [[CMP84]], label %[[IF_THEN85:.*]], label %[[IF_END90:.*]]
+// CHECK1:       [[IF_THEN85]]:
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], [[TMP71]]
+// CHECK1-NEXT:    [[ADD87:%.*]] = add nsw i64 [[TMP69]], [[MUL86]]
+// CHECK1-NEXT:    store i64 [[ADD87]], ptr [[DOTOMP_IV2]], align 8
+// CHECK1-NEXT:    [[TMP72:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[TMP73]], 1
+// CHECK1-NEXT:    [[ADD_PTR89:%.*]] = getelementptr inbounds double, ptr [[TMP72]], i64 [[MUL88]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR89]], ptr [[__BEGIN227]], align 8
+// CHECK1-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[__BEGIN227]], align 8
+// CHECK1-NEXT:    store ptr [[TMP74]], ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK1-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP77:%.*]] = load double, ptr [[TMP76]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP75]], double noundef [[TMP77]])
+// CHECK1-NEXT:    br label %[[IF_END90]]
+// CHECK1:       [[IF_END90]]:
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP78]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define dso_local void @body(
+// CHECK2-SAME: ...) #[[ATTR0:[0-9]+]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define dso_local void @foo1(
+// CHECK2-SAME: i32 noundef [[START1:%.*]], i32 noundef [[END1:%.*]], i32 noundef [[STEP1:%.*]], i32 noundef [[START2:%.*]], i32 noundef [[END2:%.*]], i32 noundef [[STEP2:%.*]]) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[START1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[END1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[STEP1_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[START2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[END2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[STEP2_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    store i32 [[START1]], ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[END1]], ptr [[END1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[STEP1]], ptr [[STEP1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[START2]], ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[END2]], ptr [[END2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[STEP2]], ptr [[STEP2_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP1_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK2-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
+// CHECK2-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP20]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP21]], [[TMP22]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP23]], %[[COND_TRUE]] ], [ [[TMP24]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
+// CHECK2-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP17]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP30]], [[TMP31]]
+// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP29]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP33]], [[TMP34]]
+// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP32]], [[MUL19]]
+// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP35]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    br i1 [[CMP21]], label %[[IF_THEN22:.*]], label %[[IF_END27:.*]]
+// CHECK2:       [[IF_THEN22]]:
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP39]], [[TMP40]]
+// CHECK2-NEXT:    [[ADD24:%.*]] = add i32 [[TMP38]], [[MUL23]]
+// CHECK2-NEXT:    store i32 [[ADD24]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[TMP41]], [[MUL25]]
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK2-NEXT:    br label %[[IF_END27]]
+// CHECK2:       [[IF_END27]]:
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP45]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define dso_local void @foo3(
+// CHECK2-SAME: ) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB03:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB04:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST05:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI06:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV07:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_UB117:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB118:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_ST119:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_NI120:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV122:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[CC:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE223:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END224:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN227:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_29:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_31:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_32:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_TEMP_142:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX48:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX54:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[VV:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK2-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTOMP_UB03]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB04]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST05]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], 1
+// CHECK2-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
+// CHECK2-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI06]], align 8
+// CHECK2-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY8:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY8]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY10:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP11]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY10]], ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP12]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP14]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK2-NEXT:    [[SUB13:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i64 [[SUB13]], 1
+// CHECK2-NEXT:    [[DIV15:%.*]] = sdiv i64 [[ADD14]], 1
+// CHECK2-NEXT:    [[SUB16:%.*]] = sub nsw i64 [[DIV15]], 1
+// CHECK2-NEXT:    store i64 [[SUB16]], ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_UB117]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB118]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST119]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK2-NEXT:    store i64 [[ADD21]], ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    store i32 37, ptr [[CC]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE223]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY25:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR26:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY25]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR26]], ptr [[__END224]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP18]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[__BEGIN227]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[__RANGE223]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY30:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP19]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY30]], ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[__END224]], align 8
+// CHECK2-NEXT:    store ptr [[TMP20]], ptr [[DOTCAPTURE_EXPR_31]], align 8
+// CHECK2-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_31]], align 8
+// CHECK2-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST33:%.*]] = ptrtoint ptr [[TMP21]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST34:%.*]] = ptrtoint ptr [[TMP22]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB35:%.*]] = sub i64 [[SUB_PTR_LHS_CAST33]], [[SUB_PTR_RHS_CAST34]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV36:%.*]] = sdiv exact i64 [[SUB_PTR_SUB35]], 8
+// CHECK2-NEXT:    [[SUB37:%.*]] = sub nsw i64 [[SUB_PTR_DIV36]], 1
+// CHECK2-NEXT:    [[ADD38:%.*]] = add nsw i64 [[SUB37]], 1
+// CHECK2-NEXT:    [[DIV39:%.*]] = sdiv i64 [[ADD38]], 1
+// CHECK2-NEXT:    [[SUB40:%.*]] = sub nsw i64 [[DIV39]], 1
+// CHECK2-NEXT:    store i64 [[SUB40]], ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK2-NEXT:    store i64 [[TMP23]], ptr [[DOTOMP_UB2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB2]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST2]], align 8
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
+// CHECK2-NEXT:    [[ADD41:%.*]] = add nsw i64 [[TMP24]], 1
+// CHECK2-NEXT:    store i64 [[ADD41]], ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
+// CHECK2-NEXT:    store i64 [[TMP25]], ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP26]], [[TMP27]]
+// CHECK2-NEXT:    br i1 [[CMP43]], label %[[COND_TRUE44:.*]], label %[[COND_FALSE45:.*]]
+// CHECK2:       [[COND_TRUE44]]:
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
+// CHECK2-NEXT:    br label %[[COND_END46:.*]]
+// CHECK2:       [[COND_FALSE45]]:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    br label %[[COND_END46]]
+// CHECK2:       [[COND_END46]]:
+// CHECK2-NEXT:    [[COND47:%.*]] = phi i64 [ [[TMP28]], %[[COND_TRUE44]] ], [ [[TMP29]], %[[COND_FALSE45]] ]
+// CHECK2-NEXT:    store i64 [[COND47]], ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[CMP49:%.*]] = icmp sgt i64 [[TMP30]], [[TMP31]]
+// CHECK2-NEXT:    br i1 [[CMP49]], label %[[COND_TRUE50:.*]], label %[[COND_FALSE51:.*]]
+// CHECK2:       [[COND_TRUE50]]:
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    br label %[[COND_END52:.*]]
+// CHECK2:       [[COND_FALSE51]]:
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    br label %[[COND_END52]]
+// CHECK2:       [[COND_END52]]:
+// CHECK2-NEXT:    [[COND53:%.*]] = phi i64 [ [[TMP32]], %[[COND_TRUE50]] ], [ [[TMP33]], %[[COND_FALSE51]] ]
+// CHECK2-NEXT:    store i64 [[COND53]], ptr [[DOTOMP_FUSE_MAX48]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX48]], align 8
+// CHECK2-NEXT:    [[CMP55:%.*]] = icmp slt i64 [[TMP34]], [[TMP35]]
+// CHECK2-NEXT:    br i1 [[CMP55]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
+// CHECK2-NEXT:    [[CMP56:%.*]] = icmp slt i64 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    br i1 [[CMP56]], label %[[IF_THEN:.*]], label %[[IF_END76:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB04]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP38]] to i64
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST05]], align 4
+// CHECK2-NEXT:    [[CONV58:%.*]] = sext i32 [[TMP39]] to i64
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV58]], [[TMP40]]
+// CHECK2-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV57]], [[MUL]]
+// CHECK2-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK2-NEXT:    store i32 [[CONV60]], ptr [[DOTOMP_IV07]], align 4
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_IV07]], align 4
+// CHECK2-NEXT:    [[MUL61:%.*]] = mul nsw i32 [[TMP41]], 1
+// CHECK2-NEXT:    [[ADD62:%.*]] = add nsw i32 0, [[MUL61]]
+// CHECK2-NEXT:    store i32 [[ADD62]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP63:%.*]] = icmp slt i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    br i1 [[CMP63]], label %[[IF_THEN64:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN64]]:
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP45]], [[TMP46]]
+// CHECK2-NEXT:    [[ADD66:%.*]] = add nsw i32 [[TMP44]], [[MUL65]]
+// CHECK2-NEXT:    store i32 [[ADD66]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[MUL67:%.*]] = mul nsw i32 [[TMP47]], 1
+// CHECK2-NEXT:    [[ADD68:%.*]] = add nsw i32 0, [[MUL67]]
+// CHECK2-NEXT:    store i32 [[ADD68]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP48]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP69:%.*]] = icmp slt i32 [[TMP49]], [[TMP50]]
+// CHECK2-NEXT:    br i1 [[CMP69]], label %[[IF_THEN70:.*]], label %[[IF_END75:.*]]
+// CHECK2:       [[IF_THEN70]]:
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP52]], [[TMP53]]
+// CHECK2-NEXT:    [[ADD72:%.*]] = add nsw i32 [[TMP51]], [[MUL71]]
+// CHECK2-NEXT:    store i32 [[ADD72]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP54]], 2
+// CHECK2-NEXT:    [[ADD74:%.*]] = add nsw i32 0, [[MUL73]]
+// CHECK2-NEXT:    store i32 [[ADD74]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP55]])
+// CHECK2-NEXT:    br label %[[IF_END75]]
+// CHECK2:       [[IF_END75]]:
+// CHECK2-NEXT:    br label %[[IF_END76]]
+// CHECK2:       [[IF_END76]]:
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    [[CMP77:%.*]] = icmp slt i64 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    br i1 [[CMP77]], label %[[IF_THEN78:.*]], label %[[IF_END83:.*]]
+// CHECK2:       [[IF_THEN78]]:
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_LB118]], align 8
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_ST119]], align 8
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    [[ADD80:%.*]] = add nsw i64 [[TMP58]], [[MUL79]]
+// CHECK2-NEXT:    store i64 [[ADD80]], ptr [[DOTOMP_IV122]], align 8
+// CHECK2-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV122]], align 8
+// CHECK2-NEXT:    [[MUL81:%.*]] = mul nsw i64 [[TMP62]], 1
+// CHECK2-NEXT:    [[ADD_PTR82:%.*]] = getelementptr inbounds double, ptr [[TMP61]], i64 [[MUL81]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR82]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP63]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[C]], align 4
+// CHECK2-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP66:%.*]] = load double, ptr [[TMP65]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP64]], double noundef [[TMP66]])
+// CHECK2-NEXT:    br label %[[IF_END83]]
+// CHECK2:       [[IF_END83]]:
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP67]], [[TMP68]]
+// CHECK2-NEXT:    br i1 [[CMP84]], label %[[IF_THEN85:.*]], label %[[IF_END90:.*]]
+// CHECK2:       [[IF_THEN85]]:
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], [[TMP71]]
+// CHECK2-NEXT:    [[ADD87:%.*]] = add nsw i64 [[TMP69]], [[MUL86]]
+// CHECK2-NEXT:    store i64 [[ADD87]], ptr [[DOTOMP_IV2]], align 8
+// CHECK2-NEXT:    [[TMP72:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[TMP73]], 1
+// CHECK2-NEXT:    [[ADD_PTR89:%.*]] = getelementptr inbounds double, ptr [[TMP72]], i64 [[MUL88]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR89]], ptr [[__BEGIN227]], align 8
+// CHECK2-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[__BEGIN227]], align 8
+// CHECK2-NEXT:    store ptr [[TMP74]], ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK2-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP77:%.*]] = load double, ptr [[TMP76]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP75]], double noundef [[TMP77]])
+// CHECK2-NEXT:    br label %[[IF_END90]]
+// CHECK2:       [[IF_END90]]:
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP78]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define dso_local void @tfoo2(
+// CHECK2-SAME: ) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    call void @_Z4foo2IiEvT_S0_S0_(i32 noundef 0, i32 noundef 64, i32 noundef 4)
+// CHECK2-NEXT:    ret void
+//
+//
+// CHECK2-LABEL: define linkonce_odr void @_Z4foo2IiEvT_S0_S0_(
+// CHECK2-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0]] comdat {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_6:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTNEW_STEP21:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP6]]
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
+// CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK2-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
+// CHECK2-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP29]]
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP30]]
+// CHECK2-NEXT:    [[SUB27:%.*]] = sub i32 [[DIV26]], 1
+// CHECK2-NEXT:    store i32 [[SUB27]], ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK2-NEXT:    store i32 [[TMP31]], ptr [[DOTOMP_UB2]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB2]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST2]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK2-NEXT:    [[ADD28:%.*]] = add i32 [[TMP32]], 1
+// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP33]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP34]], [[TMP35]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP36]], %[[COND_TRUE]] ], [ [[TMP37]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    br i1 [[CMP29]], label %[[COND_TRUE30:.*]], label %[[COND_FALSE31:.*]]
+// CHECK2:       [[COND_TRUE30]]:
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    br label %[[COND_END32:.*]]
+// CHECK2:       [[COND_FALSE31]]:
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    br label %[[COND_END32]]
+// CHECK2:       [[COND_END32]]:
+// CHECK2-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP40]], %[[COND_TRUE30]] ], [ [[TMP41]], %[[COND_FALSE31]] ]
+// CHECK2-NEXT:    store i32 [[COND33]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    br i1 [[CMP34]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP44]], [[TMP45]]
+// CHECK2-NEXT:    br i1 [[CMP35]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP47]], [[TMP48]]
+// CHECK2-NEXT:    [[ADD36:%.*]] = add i32 [[TMP46]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD36]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP50]], [[TMP51]]
+// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[TMP49]], [[MUL37]]
+// CHECK2-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP53]], [[TMP54]]
+// CHECK2-NEXT:    br i1 [[CMP39]], label %[[IF_THEN40:.*]], label %[[IF_END45:.*]]
+// CHECK2:       [[IF_THEN40]]:
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[ADD42:%.*]] = add i32 [[TMP55]], [[MUL41]]
+// CHECK2-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP58]], [[MUL43]]
+// CHECK2-NEXT:    store i32 [[SUB44]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP61]])
+// CHECK2-NEXT:    br label %[[IF_END45]]
+// CHECK2:       [[IF_END45]]:
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP62]], [[TMP63]]
+// CHECK2-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
+// CHECK2:       [[IF_THEN47]]:
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP65]], [[TMP66]]
+// CHECK2-NEXT:    [[ADD49:%.*]] = add i32 [[TMP64]], [[MUL48]]
+// CHECK2-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV2]], align 4
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP68]], [[TMP69]]
+// CHECK2-NEXT:    [[ADD51:%.*]] = add i32 [[TMP67]], [[MUL50]]
+// CHECK2-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP70]])
+// CHECK2-NEXT:    br label %[[IF_END52]]
+// CHECK2:       [[IF_END52]]:
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    ret void
+//
+//.
+// CHECK1: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
+// CHECK1: [[META4]] = !{!"llvm.loop.mustprogress"}
+// CHECK1: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
+// CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
+//.
+// CHECK2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
+// CHECK2: [[META4]] = !{!"llvm.loop.mustprogress"}
+// CHECK2: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
+// CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
+//.
diff --git a/clang/test/OpenMP/fuse_messages.cpp b/clang/test/OpenMP/fuse_messages.cpp
new file mode 100644
index 0000000000000..50dedfd2c0dc6
--- /dev/null
+++ b/clang/test/OpenMP/fuse_messages.cpp
@@ -0,0 +1,76 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+void func() {
+
+    // expected-error at +2 {{statement after '#pragma omp fuse' must be a loop sequence containing canonical loops or loop-generating constructs}}
+    #pragma omp fuse 
+    ;
+
+    // expected-error at +2 {{statement after '#pragma omp fuse' must be a for loop}}
+    #pragma omp fuse 
+    {int bar = 0;}
+
+    // expected-error at +4 {{statement after '#pragma omp fuse' must be a for loop}}
+    #pragma omp fuse 
+    {
+        for(int i = 0; i < 10; ++i);
+        int x = 2;
+    }
+
+    // expected-error at +2 {{statement after '#pragma omp fuse' must be a loop sequence containing canonical loops or loop-generating constructs}}
+    #pragma omp fuse 
+    #pragma omp for 
+    for (int i = 0; i < 7; ++i)
+        ;
+
+    {
+        // expected-error at +2 {{expected statement}}
+        #pragma omp fuse
+    }
+
+    // expected-warning at +1 {{extra tokens at the end of '#pragma omp fuse' are ignored}}
+    #pragma omp fuse foo
+    {
+        for (int i = 0; i < 7; ++i)
+            ;
+    }
+
+
+    // expected-error at +1 {{unexpected OpenMP clause 'final' in directive '#pragma omp fuse'}}
+    #pragma omp fuse final(0) 
+    {
+        for (int i = 0; i < 7; ++i)
+            ;
+    }
+
+    //expected-error at +4 {{loop after '#pragma omp fuse' is not in canonical form}}
+    //expected-error at +3 {{increment clause of OpenMP for loop must perform simple addition or subtraction on loop variable 'i'}}
+    #pragma omp fuse 
+    {
+        for(int i = 0; i < 10; i*=2) {
+            ;
+        }
+    }
+
+    //expected-error at +2 {{loop sequence after '#pragma omp fuse' must contain at least 1 canonical loop or loop-generating construct}}
+    #pragma omp fuse 
+    {}
+
+    //expected-error at +3 {{statement after '#pragma omp fuse' must be a for loop}}
+    #pragma omp fuse 
+    {
+        #pragma omp unroll full 
+        for(int i = 0; i < 10; ++i);
+        
+        for(int j = 0; j < 10; ++j);
+    }
+
+    //expected-warning at +5 {{loop sequence following '#pragma omp fuse' contains induction variables of differing types: 'int' and 'unsigned int'}}
+    //expected-warning at +5 {{loop sequence following '#pragma omp fuse' contains induction variables of differing types: 'int' and 'long long'}}
+    #pragma omp fuse 
+    {
+        for(int i = 0; i < 10; ++i);
+        for(unsigned int j = 0; j < 10; ++j);
+        for(long long k = 0; k < 100; ++k);
+    }
+}
\ No newline at end of file
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 3068621d9c004..3afa59b2f2d6c 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2211,6 +2211,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFuseDirective(const OMPFuseDirective *D);
   void VisitOMPForDirective(const OMPForDirective *D);
   void VisitOMPForSimdDirective(const OMPForSimdDirective *D);
   void VisitOMPSectionsDirective(const OMPSectionsDirective *D);
@@ -3369,6 +3370,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPLoopTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFuseDirective(const OMPFuseDirective *D) {
+  VisitOMPLoopTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPForDirective(const OMPForDirective *D) {
   VisitOMPLoopDirective(D);
 }
@@ -6323,6 +6328,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFuseDirective:
+    return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPForDirective:
     return cxstring::createRef("OMPForDirective");
   case CXCursor_OMPForSimdDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index a6301daa672c3..a6d032fa302b1 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -687,6 +687,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFuseDirectiveClass:
+    K = CXCursor_OMPFuseDirective;
+    break;
   case Stmt::OMPForDirectiveClass:
     K = CXCursor_OMPForDirective;
     break;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a87111cb5a11d..6352be8069e9e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -855,6 +855,10 @@ def OMP_For : Directive<[Spelling<"for">]> {
   let category = CA_Executable;
   let languages = [L_C];
 }
+def OMP_Fuse : Directive<[Spelling<"fuse">]> {
+  let association = AS_Loop;
+  let category = CA_Executable;
+}
 def OMP_Interchange : Directive<[Spelling<"interchange">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Permutation>,
diff --git a/openmp/runtime/test/transform/fuse/foreach.cpp b/openmp/runtime/test/transform/fuse/foreach.cpp
new file mode 100644
index 0000000000000..cabf4bf8a511d
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/foreach.cpp
@@ -0,0 +1,192 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+
+    void print(const char *msg) const { owner->print(msg); }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+#pragma omp fuse
+  {
+    for (Reporter a{"C"}; auto &&v : Reporter("A"))
+      printf("v=%d\n", v);
+    for (Reporter aa{"D"}; auto &&vv : Reporter("B"))
+      printf("vv=%d\n", vv);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+// CHECK: [C] ctor
+// CHECK-NEXT: [A] ctor
+// CHECK-NEXT: [A] end()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] begin()
+// CHECK-NEXT: [A] iterator distance: 3
+// CHECK-NEXT: [D] ctor
+// CHECK-NEXT: [B] ctor
+// CHECK-NEXT: [B] end()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] begin()
+// CHECK-NEXT: [B] iterator distance: 3
+// CHECK-NEXT: [A] iterator advance: 0 += 0
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 0
+// CHECK-NEXT: v=0
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: vv=0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 1
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 1
+// CHECK-NEXT: v=1
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: vv=1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 2
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 2
+// CHECK-NEXT: v=2
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: vv=2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] dtor
+// CHECK-NEXT: [D] dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] dtor
+// CHECK-NEXT: [C] dtor
+// CHECK-NEXT: done
+
+
+#endif
diff --git a/openmp/runtime/test/transform/fuse/intfor.c b/openmp/runtime/test/transform/fuse/intfor.c
new file mode 100644
index 0000000000000..b8171b4df7042
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/intfor.c
@@ -0,0 +1,50 @@
+// RUN: %libomp-compile-and-run  | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp fuse
+  {
+    for (int i = 5; i <= 25; i += 5)
+      printf("i=%d\n", i);
+    for (int j = 10; j < 100; j += 10)
+      printf("j=%d\n", j);
+    for (int k = 10; k > 0; --k)
+      printf("k=%d\n", k);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+#endif /* HEADER */
+
+// CHECK: do
+// CHECK-NEXT: i=5
+// CHECK-NEXT: j=10
+// CHECK-NEXT: k=10
+// CHECK-NEXT: i=10
+// CHECK-NEXT: j=20
+// CHECK-NEXT: k=9
+// CHECK-NEXT: i=15
+// CHECK-NEXT: j=30
+// CHECK-NEXT: k=8
+// CHECK-NEXT: i=20
+// CHECK-NEXT: j=40
+// CHECK-NEXT: k=7
+// CHECK-NEXT: i=25
+// CHECK-NEXT: j=50
+// CHECK-NEXT: k=6
+// CHECK-NEXT: j=60
+// CHECK-NEXT: k=5
+// CHECK-NEXT: j=70
+// CHECK-NEXT: k=4
+// CHECK-NEXT: j=80
+// CHECK-NEXT: k=3
+// CHECK-NEXT: j=90
+// CHECK-NEXT: k=2
+// CHECK-NEXT: k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/fuse/iterfor.cpp b/openmp/runtime/test/transform/fuse/iterfor.cpp
new file mode 100644
index 0000000000000..552484b2981c4
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/iterfor.cpp
@@ -0,0 +1,194 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    bool operator!=(const Iterator &that) const {
+      owner->print("iterator %d != %d", 2 - this->pos, 2 - that.pos);
+      return this->pos != that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+  Reporter C("C");
+  Reporter D("D");
+#pragma omp fuse
+  {
+    for (auto it = C.begin(); it != C.end(); ++it)
+      printf("v=%d\n", *it);
+
+    for (auto it = D.begin(); it != D.end(); ++it)
+      printf("vv=%d\n", *it);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK: do
+// CHECK: [C] ctor
+// CHECK-NEXT: [D] ctor
+// CHECK-NEXT: [C] begin()
+// CHECK-NEXT: [C] begin()
+// CHECK-NEXT: [C] end()
+// CHECK-NEXT: [C] iterator distance: 3
+// CHECK-NEXT: [D] begin()
+// CHECK-NEXT: [D] begin()
+// CHECK-NEXT: [D] end()
+// CHECK-NEXT: [D] iterator distance: 3
+// CHECK-NEXT: [C] iterator advance: 0 += 0
+// CHECK-NEXT: [C] iterator move assign
+// CHECK-NEXT: [C] iterator deref: 0
+// CHECK-NEXT: v=0
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [D] iterator advance: 0 += 0
+// CHECK-NEXT: [D] iterator move assign
+// CHECK-NEXT: [D] iterator deref: 0
+// CHECK-NEXT: vv=0
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [C] iterator advance: 0 += 1
+// CHECK-NEXT: [C] iterator move assign
+// CHECK-NEXT: [C] iterator deref: 1
+// CHECK-NEXT: v=1
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [D] iterator advance: 0 += 1
+// CHECK-NEXT: [D] iterator move assign
+// CHECK-NEXT: [D] iterator deref: 1
+// CHECK-NEXT: vv=1
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [C] iterator advance: 0 += 2
+// CHECK-NEXT: [C] iterator move assign
+// CHECK-NEXT: [C] iterator deref: 2
+// CHECK-NEXT: v=2
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [D] iterator advance: 0 += 2
+// CHECK-NEXT: [D] iterator move assign
+// CHECK-NEXT: [D] iterator deref: 2
+// CHECK-NEXT: vv=2
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: done
+// CHECK-NEXT: [D] iterator dtor
+// CHECK-NEXT: [C] iterator dtor
+// CHECK-NEXT: [D] dtor
+// CHECK-NEXT: [C] dtor
diff --git a/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-foreach.cpp
new file mode 100644
index 0000000000000..e9f76713fe3e0
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-foreach.cpp
@@ -0,0 +1,208 @@
+// RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdarg>
+#include <cstdio>
+#include <vector>
+
+struct Reporter {
+  const char *name;
+
+  Reporter(const char *name) : name(name) { print("ctor"); }
+
+  Reporter() : name("<anon>") { print("ctor"); }
+
+  Reporter(const Reporter &that) : name(that.name) { print("copy ctor"); }
+
+  Reporter(Reporter &&that) : name(that.name) { print("move ctor"); }
+
+  ~Reporter() { print("dtor"); }
+
+  const Reporter &operator=(const Reporter &that) {
+    print("copy assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  const Reporter &operator=(Reporter &&that) {
+    print("move assign");
+    this->name = that.name;
+    return *this;
+  }
+
+  struct Iterator {
+    const Reporter *owner;
+    int pos;
+
+    Iterator(const Reporter *owner, int pos) : owner(owner), pos(pos) {}
+
+    Iterator(const Iterator &that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator copy ctor");
+    }
+
+    Iterator(Iterator &&that) : owner(that.owner), pos(that.pos) {
+      owner->print("iterator move ctor");
+    }
+
+    ~Iterator() { owner->print("iterator dtor"); }
+
+    const Iterator &operator=(const Iterator &that) {
+      owner->print("iterator copy assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    const Iterator &operator=(Iterator &&that) {
+      owner->print("iterator move assign");
+      this->owner = that.owner;
+      this->pos = that.pos;
+      return *this;
+    }
+
+    bool operator==(const Iterator &that) const {
+      owner->print("iterator %d == %d", 2 - this->pos, 2 - that.pos);
+      return this->pos == that.pos;
+    }
+
+    Iterator &operator++() {
+      owner->print("iterator prefix ++");
+      pos -= 1;
+      return *this;
+    }
+
+    Iterator operator++(int) {
+      owner->print("iterator postfix ++");
+      auto result = *this;
+      pos -= 1;
+      return result;
+    }
+
+    int operator*() const {
+      int result = 2 - pos;
+      owner->print("iterator deref: %i", result);
+      return result;
+    }
+
+    size_t operator-(const Iterator &that) const {
+      int result = (2 - this->pos) - (2 - that.pos);
+      owner->print("iterator distance: %d", result);
+      return result;
+    }
+
+    Iterator operator+(int steps) const {
+      owner->print("iterator advance: %i += %i", 2 - this->pos, steps);
+      return Iterator(owner, pos - steps);
+    }
+
+    void print(const char *msg) const { owner->print(msg); }
+  };
+
+  Iterator begin() const {
+    print("begin()");
+    return Iterator(this, 2);
+  }
+
+  Iterator end() const {
+    print("end()");
+    return Iterator(this, -1);
+  }
+
+  void print(const char *msg, ...) const {
+    va_list args;
+    va_start(args, msg);
+    printf("[%s] ", name);
+    vprintf(msg, args);
+    printf("\n");
+    va_end(args);
+  }
+};
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+  for (int i = 0; i < 3; ++i)
+#pragma omp fuse
+  {
+    for (Reporter c{"init-stmt"}; auto &&v : Reporter("range"))
+      printf("i=%d v=%d\n", i, v);
+    for (int vv = 0; vv < 3; ++vv)
+      printf("i=%d vv=%d\n", i, vv);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK: do
+// CHECK-NEXT: [init-stmt] ctor
+// CHECK-NEXT: [range] ctor
+// CHECK-NEXT: [range] end()
+// CHECK-NEXT: [range] begin()
+// CHECK-NEXT: [range] begin()
+// CHECK-NEXT: [range] iterator distance: 3
+// CHECK-NEXT: [range] iterator advance: 0 += 0
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 0
+// CHECK-NEXT: i=0 v=0
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=0 vv=0
+// CHECK-NEXT: [range] iterator advance: 0 += 1
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 1
+// CHECK-NEXT: i=0 v=1
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=0 vv=1
+// CHECK-NEXT: [range] iterator advance: 0 += 2
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 2
+// CHECK-NEXT: i=0 v=2
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=0 vv=2
+// CHECK-NEXT: [range] iterator advance: 0 += 0
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 0
+// CHECK-NEXT: i=1 v=0
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=1 vv=0
+// CHECK-NEXT: [range] iterator advance: 0 += 1
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 1
+// CHECK-NEXT: i=1 v=1
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=1 vv=1
+// CHECK-NEXT: [range] iterator advance: 0 += 2
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 2
+// CHECK-NEXT: i=1 v=2
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=1 vv=2
+// CHECK-NEXT: [range] iterator advance: 0 += 0
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 0
+// CHECK-NEXT: i=2 v=0
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=2 vv=0
+// CHECK-NEXT: [range] iterator advance: 0 += 1
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 1
+// CHECK-NEXT: i=2 v=1
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=2 vv=1
+// CHECK-NEXT: [range] iterator advance: 0 += 2
+// CHECK-NEXT: [range] iterator move assign
+// CHECK-NEXT: [range] iterator deref: 2
+// CHECK-NEXT: i=2 v=2
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: i=2 vv=2
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: [range] iterator dtor
+// CHECK-NEXT: [range] dtor
+// CHECK-NEXT: [init-stmt] dtor
+// CHECK-NEXT: done
+
diff --git a/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c
new file mode 100644
index 0000000000000..272908e72c429
--- /dev/null
+++ b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c
@@ -0,0 +1,45 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+  for (int i = 0; i < 3; ++i)
+#pragma omp fuse
+  {
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+    for (int k = 0; k < 3; ++k)
+      printf("i=%d k=%d\n", i, k);
+  }
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK: i=0 j=0
+// CHECK-NEXT: i=0 k=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 k=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=0 k=2
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 k=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 k=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: i=1 k=2
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 k=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=2 k=1
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: i=2 k=2
+// CHECK-NEXT: done

>From 34ac92ada84eeca9573d0b005f24d73738f46626 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:28:04 +0000
Subject: [PATCH 02/13] Add looprange clause

---
 clang/include/clang/AST/OpenMPClause.h        | 100 ++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |   8 +
 clang/include/clang/AST/StmtOpenMP.h          |   9 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   5 +
 clang/include/clang/Parse/Parser.h            |   3 +
 clang/include/clang/Sema/SemaOpenMP.h         |   6 +
 clang/lib/AST/OpenMPClause.cpp                |  35 ++
 clang/lib/AST/StmtOpenMP.cpp                  |   7 +-
 clang/lib/AST/StmtProfile.cpp                 |   7 +
 clang/lib/Basic/OpenMPKinds.cpp               |   2 +
 clang/lib/Parse/ParseOpenMP.cpp               |  36 ++
 clang/lib/Sema/SemaOpenMP.cpp                 | 155 +++++++--
 clang/lib/Sema/TreeTransform.h                |  33 ++
 clang/lib/Serialization/ASTReader.cpp         |  11 +
 clang/lib/Serialization/ASTReaderStmt.cpp     |   4 +-
 clang/lib/Serialization/ASTWriter.cpp         |   8 +
 clang/test/OpenMP/fuse_ast_print.cpp          |  67 ++++
 clang/test/OpenMP/fuse_codegen.cpp            | 320 +++++++++++++++++-
 clang/test/OpenMP/fuse_messages.cpp           | 112 +++++-
 clang/tools/libclang/CIndex.cpp               |   5 +
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |  16 +-
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |   6 +
 22 files changed, 919 insertions(+), 36 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 6fd16bc0f03be..8f937cdef9cd0 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1143,6 +1143,106 @@ class OMPFullClause final : public OMPNoChildClause<llvm::omp::OMPC_full> {
   static OMPFullClause *CreateEmpty(const ASTContext &C);
 };
 
+/// This class represents the 'looprange' clause in the
+/// '#pragma omp fuse' directive
+///
+/// \code {c}
+/// #pragma omp fuse looprange(1,2)
+/// {
+///   for(int i = 0; i < 64; ++i)
+///   for(int j = 0; j < 256; j+=2)
+///   for(int k = 127; k >= 0; --k)
+/// \endcode
+class OMPLoopRangeClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  explicit OMPLoopRangeClause()
+      : OMPClause(llvm::omp::OMPC_looprange, {}, {}) {}
+
+  /// Location of '('
+  SourceLocation LParenLoc;
+
+  /// Location of 'first'
+  SourceLocation FirstLoc;
+
+  /// Location of 'count'
+  SourceLocation CountLoc;
+
+  /// Expr associated with 'first' argument
+  Expr *First = nullptr;
+
+  /// Expr associated with 'count' argument
+  Expr *Count = nullptr;
+
+  /// Set 'first'
+  void setFirst(Expr *First) { this->First = First; }
+
+  /// Set 'count'
+  void setCount(Expr *Count) { this->Count = Count; }
+
+  /// Set location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+  /// Set location of 'first' argument
+  void setFirstLoc(SourceLocation Loc) { FirstLoc = Loc; }
+
+  /// Set location of 'count' argument
+  void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'looprange' clause
+  ///
+  /// \param StartLoc     Starting location of the clause.
+  /// \param LParenLoc    Location of '('.
+  /// \param ModifierLoc  Modifier location.
+  /// \param
+  static OMPLoopRangeClause *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
+         SourceLocation FirstLoc, SourceLocation CountLoc,
+         SourceLocation EndLoc, Expr *First, Expr *Count);
+
+  /// Build an empty 'looprange' node for deserialization
+  ///
+  /// \param C      Context of the AST.
+  static OMPLoopRangeClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the location of 'first'
+  SourceLocation getFirstLoc() const { return FirstLoc; }
+
+  /// Returns the location of 'count'
+  SourceLocation getCountLoc() const { return CountLoc; }
+
+  /// Returns the argument 'first' or nullptr if not set
+  Expr *getFirst() const { return cast_or_null<Expr>(First); }
+
+  /// Returns the argument 'count' or nullptr if not set
+  Expr *getCount() const { return cast_or_null<Expr>(Count); }
+
+  child_range children() {
+    return child_range(reinterpret_cast<Stmt **>(&First),
+                       reinterpret_cast<Stmt **>(&Count) + 1);
+  }
+
+  const_child_range children() const {
+    auto Children = const_cast<OMPLoopRangeClause *>(this)->children();
+    return const_child_range(Children.begin(), Children.end());
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_looprange;
+  }
+};
+
 /// Representation of the 'partial' clause of the '#pragma omp unroll'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 918216e8df4aa..10e44e69dd5da 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3410,6 +3410,14 @@ bool RecursiveASTVisitor<Derived>::VisitOMPFullClause(OMPFullClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPLoopRangeClause(
+    OMPLoopRangeClause *C) {
+  TRY_TO(TraverseStmt(C->getFirst()));
+  TRY_TO(TraverseStmt(C->getCount()));
+  return true;
+}
+
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   TRY_TO(TraverseStmt(C->getFactor()));
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index cb8bb91f4768c..f5115afd0753e 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5572,7 +5572,9 @@ class OMPTileDirective final : public OMPLoopTransformationDirective {
       : OMPLoopTransformationDirective(OMPTileDirectiveClass,
                                        llvm::omp::OMPD_tile, StartLoc, EndLoc,
                                        NumLoops) {
+    // Tiling doubles the original number of loops
     setNumGeneratedLoops(2 * NumLoops);
+    // Produces a single top-level canonical loop nest
     setNumGeneratedLoopNests(1);
   }
 
@@ -5804,6 +5806,7 @@ class OMPReverseDirective final : public OMPLoopTransformationDirective {
       : OMPLoopTransformationDirective(OMPReverseDirectiveClass,
                                        llvm::omp::OMPD_reverse, StartLoc,
                                        EndLoc, NumLoops) {
+    // Reverse produces a single top-level canonical loop nest
     setNumGeneratedLoops(NumLoops);
     setNumGeneratedLoopNests(1);
   }
@@ -5877,6 +5880,8 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
       : OMPLoopTransformationDirective(OMPInterchangeDirectiveClass,
                                        llvm::omp::OMPD_interchange, StartLoc,
                                        EndLoc, NumLoops) {
+    // Interchange produces a single top-level canonical loop
+    // nest, with the exact same amount of total loops
     setNumGeneratedLoops(NumLoops);
     setNumGeneratedLoopNests(1);
   }
@@ -5995,8 +6000,10 @@ class OMPFuseDirective final : public OMPLoopTransformationDirective {
   /// \param C Context of the AST
   /// \param NumClauses Number of clauses to allocate
   /// \param NumLoops Number of associated loops to allocate
+  /// \param NumLoopNests Number of top level loops to allocate
   static OMPFuseDirective *CreateEmpty(const ASTContext &C, unsigned NumClauses,
-                                       unsigned NumLoops);
+                                       unsigned NumLoops,
+                                       unsigned NumLoopNests);
 
   /// Gets the associated loops after the transformation. This is the de-sugared
   /// replacement or nulltpr in dependent contexts.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index fe9ca29038a1f..002aa7a774fbe 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11620,6 +11620,11 @@ def err_omp_not_a_loop_sequence : Error <
   "statement after '#pragma omp %0' must be a loop sequence containing canonical loops or loop-generating constructs">;
 def err_omp_empty_loop_sequence : Error <
   "loop sequence after '#pragma omp %0' must contain at least 1 canonical loop or loop-generating construct">;
+def err_omp_invalid_looprange : Error <
+  "loop range in '#pragma omp %0' exceeds the number of available loops: "
+  "range end '%1' is greater than the total number of loops '%2'">;
+def warn_omp_redundant_fusion : Warning <
+  "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index a47e23ffbd357..08bee0078b5ff 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -6735,6 +6735,9 @@ class Parser : public CodeCompletionHandler {
                                                 OpenMPClauseKind Kind,
                                                 bool ParseOnly);
 
+  /// Parses the 'looprange' clause of a '#pragma omp fuse' directive.
+  OMPClause *ParseOpenMPLoopRangeClause();
+  
   /// Parses the 'sizes' clause of a '#pragma omp tile' directive.
   OMPClause *ParseOpenMPSizesClause();
 
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index ea21377a8db9c..0c28aaf6ab21a 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -922,6 +922,12 @@ class SemaOpenMP : public SemaBase {
                                        SourceLocation StartLoc,
                                        SourceLocation LParenLoc,
                                        SourceLocation EndLoc);
+
+  /// Called on well-form 'looprange' clause after parsing its arguments.
+  OMPClause *
+  ActOnOpenMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
+                             SourceLocation LParenLoc, SourceLocation FirstLoc,
+                             SourceLocation CountLoc, SourceLocation EndLoc);
   /// Called on well-formed 'ordered' clause.
   OMPClause *
   ActOnOpenMPOrderedClause(SourceLocation StartLoc, SourceLocation EndLoc,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 0e5052b944162..0b5808eb100e4 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1024,6 +1024,26 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPLoopRangeClause *
+OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
+                           SourceLocation LParenLoc, SourceLocation EndLoc,
+                           SourceLocation FirstLoc, SourceLocation CountLoc,
+                           Expr *First, Expr *Count) {
+  OMPLoopRangeClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setFirstLoc(FirstLoc);
+  Clause->setCountLoc(CountLoc);
+  Clause->setFirst(First);
+  Clause->setCount(Count);
+  return Clause;
+}
+
+OMPLoopRangeClause *OMPLoopRangeClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPLoopRangeClause();
+}
+
 OMPAllocateClause *OMPAllocateClause::Create(
     const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
     Expr *Allocator, Expr *Alignment, SourceLocation ColonLoc,
@@ -1888,6 +1908,21 @@ void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   }
 }
 
+void OMPClausePrinter::VisitOMPLoopRangeClause(OMPLoopRangeClause *Node) {
+  OS << "looprange";
+
+  Expr *First = Node->getFirst();
+  Expr *Count = Node->getCount();
+
+  if (First && Count) {
+    OS << "(";
+    First->printPretty(OS, nullptr, Policy, 0);
+    OS << ",";
+    Count->printPretty(OS, nullptr, Policy, 0);
+    OS << ")";
+  }
+}
+
 void OMPClausePrinter::VisitOMPAllocatorClause(OMPAllocatorClause *Node) {
   OS << "allocator(";
   Node->getAllocator()->printPretty(OS, nullptr, Policy, 0);
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 276e43ec9f7d5..c5a6732cc2217 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -527,10 +527,13 @@ OMPFuseDirective *OMPFuseDirective::Create(
 
 OMPFuseDirective *OMPFuseDirective::CreateEmpty(const ASTContext &C,
                                                 unsigned NumClauses,
-                                                unsigned NumLoops) {
-  return createEmptyDirective<OMPFuseDirective>(
+                                                unsigned NumLoops,
+                                                unsigned NumLoopNests) {
+  OMPFuseDirective *Dir = createEmptyDirective<OMPFuseDirective>(
       C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
       SourceLocation(), SourceLocation(), NumLoops);
+  Dir->setNumGeneratedLoopNests(NumLoopNests);
+  return Dir;
 }
 
 OMPForSimdDirective *
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index c5d1d5b48508e..34ed3f22f6eb7 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -511,6 +511,13 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
+  if (const Expr *First = C->getFirst())
+    Profiler->VisitExpr(First);
+  if (const Expr *Count = C->getCount())
+    Profiler->VisitExpr(Count);
+}
+
 void OMPClauseProfiler::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   if (C->getAllocator())
     Profiler->VisitStmt(C->getAllocator());
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index d172450512f13..18330181f1509 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -248,6 +248,7 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
   case OMPC_affinity:
   case OMPC_when:
   case OMPC_append_args:
+  case OMPC_looprange:
     break;
   default:
     break;
@@ -583,6 +584,7 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_affinity:
   case OMPC_when:
   case OMPC_append_args:
+  case OMPC_looprange:
     break;
   default:
     break;
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 78d3503d8eb68..2d6d624c1ecc8 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3057,6 +3057,39 @@ OMPClause *Parser::ParseOpenMPSizesClause() {
                                                  OpenLoc, CloseLoc);
 }
 
+OMPClause *Parser::ParseOpenMPLoopRangeClause() {
+  SourceLocation ClauseNameLoc = ConsumeToken();
+  SourceLocation FirstLoc, CountLoc;
+
+  BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
+  if (T.consumeOpen()) {
+    Diag(Tok, diag::err_expected) << tok::l_paren;
+    return nullptr;
+  }
+
+  FirstLoc = Tok.getLocation();
+  ExprResult FirstVal = ParseConstantExpression();
+  if (!FirstVal.isUsable()) {
+    T.skipToEnd();
+    return nullptr;
+  }
+
+  ExpectAndConsume(tok::comma);
+
+  CountLoc = Tok.getLocation();
+  ExprResult CountVal = ParseConstantExpression();
+  if (!CountVal.isUsable()) {
+    T.skipToEnd();
+    return nullptr;
+  }
+
+  T.consumeClose();
+
+  return Actions.OpenMP().ActOnOpenMPLoopRangeClause(
+      FirstVal.get(), CountVal.get(), ClauseNameLoc, T.getOpenLocation(),
+      FirstLoc, CountLoc, T.getCloseLocation());
+}
+
 OMPClause *Parser::ParseOpenMPPermutationClause() {
   SourceLocation ClauseNameLoc, OpenLoc, CloseLoc;
   SmallVector<Expr *> ArgExprs;
@@ -3485,6 +3518,9 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
     }
     Clause = ParseOpenMPClause(CKind, WrongDirective);
     break;
+  case OMPC_looprange:
+    Clause = ParseOpenMPLoopRangeClause();
+    break;  
   default:
     break;
   }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 84ac9587bd54d..3ec3f2ad31e78 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14326,7 +14326,6 @@ bool SemaOpenMP::checkTransformableLoopSequence(
   // and tries to match the input AST to the canonical loop sequence grammar
   // structure
 
-  auto NLCV = NestedLoopCounterVisitor();
   // Helper functions to validate canonical loop sequence grammar is valid
   auto isLoopSequenceDerivation = [](auto *Child) {
     return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
@@ -14429,7 +14428,7 @@ bool SemaOpenMP::checkTransformableLoopSequence(
 
   // Modularized code for handling regular canonical loops
   auto handleRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
-                            &LoopSeqSize, &NumLoops, Kind, &TmpDSA, &NLCV,
+                            &LoopSeqSize, &NumLoops, Kind, &TmpDSA,
                             this](Stmt *Child) {
     OriginalInits.emplace_back();
     LoopHelpers.emplace_back();
@@ -14442,8 +14441,11 @@ bool SemaOpenMP::checkTransformableLoopSequence(
           << getOpenMPDirectiveName(Kind);
       return false;
     }
+
     storeLoopStatements(Child);
-    NumLoops += NLCV.TraverseStmt(Child);
+    auto NLCV = NestedLoopCounterVisitor();
+    NLCV.TraverseStmt(Child);
+    NumLoops += NLCV.getNestedLoopCount();
     return true;
   };
 
@@ -15769,6 +15771,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
                                                 SourceLocation EndLoc) {
+
   ASTContext &Context = getASTContext();
   DeclContext *CurrContext = SemaRef.CurContext;
   Scope *CurScope = SemaRef.getCurScope();
@@ -15785,7 +15788,6 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   SmallVector<SmallVector<Stmt *, 0>> OriginalInits;
 
   unsigned NumLoops;
-  // TODO: Support looprange clause using LoopSeqSize
   unsigned LoopSeqSize;
   if (!checkTransformableLoopSequence(OMPD_fuse, AStmt, LoopSeqSize, NumLoops,
                                       LoopHelpers, LoopStmts, OriginalInits,
@@ -15794,10 +15796,67 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   }
 
   // Defer transformation in dependent contexts
+  // The NumLoopNests argument is set to a placeholder (0)
+  // because a dependent context could prevent determining its true value
   if (CurrContext->isDependentContext()) {
     return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                    NumLoops, 1, AStmt, nullptr, nullptr);
+                                    NumLoops, 0, AStmt, nullptr, nullptr);
   }
+
+  // Handle clauses, which can be any of the following: [looprange, apply]
+  const OMPLoopRangeClause *LRC =
+      OMPExecutableDirective::getSingleClause<OMPLoopRangeClause>(Clauses);
+
+  // The clause arguments are invalidated if any error arises
+  // such as non-constant or non-positive arguments
+  if (LRC && (!LRC->getFirst() || !LRC->getCount()))
+    return StmtError();
+
+  // Delayed semantic check of LoopRange constraint
+  // Evaluates the loop range arguments and returns the first and count values
+  auto EvaluateLoopRangeArguments = [&Context](Expr *First, Expr *Count,
+                                               uint64_t &FirstVal,
+                                               uint64_t &CountVal) {
+    llvm::APSInt FirstInt = First->EvaluateKnownConstInt(Context);
+    llvm::APSInt CountInt = Count->EvaluateKnownConstInt(Context);
+    FirstVal = FirstInt.getZExtValue();
+    CountVal = CountInt.getZExtValue();
+  };
+
+  // Checks if the loop range is valid
+  auto ValidLoopRange = [](uint64_t FirstVal, uint64_t CountVal,
+                           unsigned NumLoops) -> bool {
+    return FirstVal + CountVal - 1 <= NumLoops;
+  };
+  uint64_t FirstVal = 1, CountVal = 0, LastVal = LoopSeqSize;
+
+  if (LRC) {
+    EvaluateLoopRangeArguments(LRC->getFirst(), LRC->getCount(), FirstVal,
+                               CountVal);
+    if (CountVal == 1)
+      SemaRef.Diag(LRC->getCountLoc(), diag::warn_omp_redundant_fusion)
+          << getOpenMPDirectiveName(OMPD_fuse);
+
+    if (!ValidLoopRange(FirstVal, CountVal, LoopSeqSize)) {
+      SemaRef.Diag(LRC->getFirstLoc(), diag::err_omp_invalid_looprange)
+          << getOpenMPDirectiveName(OMPD_fuse) << (FirstVal + CountVal - 1)
+          << LoopSeqSize;
+      return StmtError();
+    }
+
+    LastVal = FirstVal + CountVal - 1;
+  }
+
+  // Complete fusion generates a single canonical loop nest
+  // However looprange clause generates several loop nests
+  unsigned NumLoopNests = LRC ? LoopSeqSize - CountVal + 1 : 1;
+
+  // Emit a warning for redundant loop fusion when the sequence contains only
+  // one loop.
+  if (LoopSeqSize == 1)
+    SemaRef.Diag(AStmt->getBeginLoc(), diag::warn_omp_redundant_fusion)
+        << getOpenMPDirectiveName(OMPD_fuse);
+
   assert(LoopHelpers.size() == LoopSeqSize &&
          "Expecting loop iteration space dimensionality to match number of "
          "affected loops");
@@ -15811,8 +15870,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   SmallVector<Stmt *> PreInits;
 
   // Select the type with the largest bit width among all induction variables
-  QualType IVType = LoopHelpers[0].IterationVarRef->getType();
-  for (unsigned int I = 1; I < LoopSeqSize; ++I) {
+  QualType IVType = LoopHelpers[FirstVal - 1].IterationVarRef->getType();
+  for (unsigned int I = FirstVal; I < LastVal; ++I) {
     QualType CurrentIVType = LoopHelpers[I].IterationVarRef->getType();
     if (Context.getTypeSize(CurrentIVType) > Context.getTypeSize(IVType)) {
       IVType = CurrentIVType;
@@ -15861,20 +15920,21 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
 
   // Process each single loop to generate and collect declarations
   // and statements for all helper expressions
-  for (unsigned int I = 0; I < LoopSeqSize; ++I) {
+  for (unsigned int I = FirstVal - 1, J = 0; I < LastVal; ++I, ++J) {
     addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
                     PreInits);
 
-    auto [UBVD, UBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].UB, "ub", I);
-    auto [LBVD, LBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].LB, "lb", I);
-    auto [STVD, STDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].ST, "st", I);
+    auto [UBVD, UBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].UB, "ub", J);
+    auto [LBVD, LBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].LB, "lb", J);
+    auto [STVD, STDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].ST, "st", J);
     auto [NIVD, NIDStmt] =
-        CreateHelperVarAndStmt(LoopHelpers[I].NumIterations, "ni", I, true);
+        CreateHelperVarAndStmt(LoopHelpers[I].NumIterations, "ni", J, true);
     auto [IVVD, IVDStmt] =
-        CreateHelperVarAndStmt(LoopHelpers[I].IterationVarRef, "iv", I);
+        CreateHelperVarAndStmt(LoopHelpers[I].IterationVarRef, "iv", J);
 
     if (!LBVD || !STVD || !NIVD || !IVVD)
-      return StmtError();
+      assert(LBVD && STVD && NIVD && IVVD &&
+             "OpenMP Fuse Helper variables creation failed");
 
     UBVarDecls.push_back(UBVD);
     LBVarDecls.push_back(LBVD);
@@ -15949,8 +16009,9 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   //   omp.fuse.max = max(omp.temp1, omp.temp0)
 
   ExprResult MaxExpr;
-  for (unsigned I = 0; I < LoopSeqSize; ++I) {
-    DeclRefExpr *NIRef = MakeVarDeclRef(NIVarDecls[I]);
+  // I is the true
+  for (unsigned I = FirstVal - 1, J = 0; I < LastVal; ++I, ++J) {
+    DeclRefExpr *NIRef = MakeVarDeclRef(NIVarDecls[J]);
     QualType NITy = NIRef->getType();
 
     if (MaxExpr.isUnset()) {
@@ -15958,7 +16019,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
       MaxExpr = NIRef;
     } else {
       // Create a new acummulator variable t_i = MaxExpr
-      std::string TempName = (Twine(".omp.temp.") + Twine(I)).str();
+      std::string TempName = (Twine(".omp.temp.") + Twine(J)).str();
       VarDecl *TempDecl =
           buildVarDecl(SemaRef, {}, NITy, TempName, nullptr, nullptr);
       TempDecl->setInit(MaxExpr.get());
@@ -15981,7 +16042,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
       if (!Comparison.isUsable())
         return StmtError();
 
-      DeclRefExpr *NIRef2 = MakeVarDeclRef(NIVarDecls[I]);
+      DeclRefExpr *NIRef2 = MakeVarDeclRef(NIVarDecls[J]);
       // Update MaxExpr using a conditional expression to hold the max value
       MaxExpr = new (Context) ConditionalOperator(
           Comparison.get(), SourceLocation(), TempRef2, SourceLocation(),
@@ -16034,23 +16095,21 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
 
   CompoundStmt *FusedBody = nullptr;
   SmallVector<Stmt *, 4> FusedBodyStmts;
-  for (unsigned I = 0; I < LoopSeqSize; ++I) {
-
+  for (unsigned I = FirstVal - 1, J = 0; I < LastVal; ++I, ++J) {
     // Assingment of the original sub-loop index to compute the logical index
     // IV_k = LB_k + omp.fuse.index * ST_k
-
     ExprResult IdxExpr =
         SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Mul,
-                           MakeVarDeclRef(STVarDecls[I]), MakeIVRef());
+                           MakeVarDeclRef(STVarDecls[J]), MakeIVRef());
     if (!IdxExpr.isUsable())
       return StmtError();
     IdxExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Add,
-                                 MakeVarDeclRef(LBVarDecls[I]), IdxExpr.get());
+                                 MakeVarDeclRef(LBVarDecls[J]), IdxExpr.get());
 
     if (!IdxExpr.isUsable())
       return StmtError();
     IdxExpr = SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_Assign,
-                                 MakeVarDeclRef(IVVarDecls[I]), IdxExpr.get());
+                                 MakeVarDeclRef(IVVarDecls[J]), IdxExpr.get());
     if (!IdxExpr.isUsable())
       return StmtError();
 
@@ -16065,7 +16124,6 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     Stmt *Body = (isa<ForStmt>(LoopStmts[I]))
                      ? cast<ForStmt>(LoopStmts[I])->getBody()
                      : cast<CXXForRangeStmt>(LoopStmts[I])->getBody();
-
     BodyStmts.push_back(Body);
 
     CompoundStmt *CombinedBody =
@@ -16073,7 +16131,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                              SourceLocation(), SourceLocation());
     ExprResult Condition =
         SemaRef.BuildBinOp(CurScope, SourceLocation(), BO_LT, MakeIVRef(),
-                           MakeVarDeclRef(NIVarDecls[I]));
+                           MakeVarDeclRef(NIVarDecls[J]));
 
     if (!Condition.isUsable())
       return StmtError();
@@ -16094,8 +16152,26 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
               FusedBody, InitStmt.get()->getBeginLoc(), SourceLocation(),
               IncrExpr.get()->getEndLoc());
 
+  // In the case of looprange, the result of fuse won't simply
+  // be a single loop (ForStmt), but rather a loop sequence
+  // (CompoundStmt) of 3 parts: the pre-fusion loops, the fused loop
+  // and the post-fusion loops, preserving its original order.
+  Stmt *FusionStmt = FusedForStmt;
+  if (LRC) {
+    SmallVector<Stmt *, 4> FinalLoops;
+    // Gather all the pre-fusion loops
+    for (unsigned I = 0; I < FirstVal - 1; ++I)
+      FinalLoops.push_back(LoopStmts[I]);
+    // Gather the fused loop
+    FinalLoops.push_back(FusedForStmt);
+    // Gather all the post-fusion loops
+    for (unsigned I = FirstVal + CountVal - 1; I < LoopSeqSize; ++I)
+      FinalLoops.push_back(LoopStmts[I]);
+    FusionStmt = CompoundStmt::Create(Context, FinalLoops, FPOptionsOverride(),
+                                      SourceLocation(), SourceLocation());
+  }
   return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses, NumLoops,
-                                  1, AStmt, FusedForStmt,
+                                  NumLoopNests, AStmt, FusionStmt,
                                   buildPreInits(Context, PreInits));
 }
 
@@ -17218,6 +17294,31 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
+    Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
+    SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {
+
+  // OpenMP [6.0, Restrictions]
+  // First and Count must be integer expressions with positive value
+  ExprResult FirstVal =
+      VerifyPositiveIntegerConstantInClause(First, OMPC_looprange);
+  if (FirstVal.isInvalid())
+    First = nullptr;
+
+  ExprResult CountVal =
+      VerifyPositiveIntegerConstantInClause(Count, OMPC_looprange);
+  if (CountVal.isInvalid())
+    Count = nullptr;
+
+  // OpenMP [6.0, Restrictions]
+  // first + count - 1 must not evaluate to a value greater than the
+  // loop sequence length of the associated canonical loop sequence.
+  // This check must be performed afterwards due to the delayed
+  // parsing and computation of the associated loop sequence
+  return OMPLoopRangeClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                    FirstLoc, CountLoc, EndLoc, First, Count);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPAlignClause(Expr *A, SourceLocation StartLoc,
                                               SourceLocation LParenLoc,
                                               SourceLocation EndLoc) {
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 45f556f22c511..30204faf59b7b 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1775,6 +1775,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  OMPClause *
+  RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
+                            SourceLocation LParenLoc, SourceLocation FirstLoc,
+                            SourceLocation CountLoc, SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPLoopRangeClause(
+        First, Count, StartLoc, LParenLoc, FirstLoc, CountLoc, EndLoc);
+  }
+
   /// Build a new OpenMP 'allocator' clause.
   ///
   /// By default, performs semantic analysis to build the new OpenMP clause.
@@ -10578,6 +10586,31 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {
+  ExprResult F = getDerived().TransformExpr(C->getFirst());
+  if (F.isInvalid())
+    return nullptr;
+
+  ExprResult Cn = getDerived().TransformExpr(C->getCount());
+  if (Cn.isInvalid())
+    return nullptr;
+
+  Expr *First = F.get();
+  Expr *Count = Cn.get();
+
+  bool Changed = (First != C->getFirst()) || (Count != C->getCount());
+
+  // If no changes and AlwaysRebuild() is false, return the original clause
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+
+  return RebuildOMPLoopRangeClause(First, Count, C->getBeginLoc(),
+                                   C->getLParenLoc(), C->getFirstLoc(),
+                                   C->getCountLoc(), C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index a3fbc3d25acab..d5e7c287c23a4 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11089,6 +11089,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_looprange:
+    C = OMPLoopRangeClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_allocator:
     C = new (Context) OMPAllocatorClause();
     break;
@@ -11490,6 +11493,14 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
+  C->setFirst(Record.readSubExpr());
+  C->setCount(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+  C->setFirstLoc(Record.readSourceLocation());
+  C->setCountLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPAllocatorClause(OMPAllocatorClause *C) {
   C->setAllocator(Record.readExpr());
   C->setLParenLoc(Record.readSourceLocation());
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 291bd8ea4bf18..b424b5aa7b0c6 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -3616,7 +3616,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
     case STMT_OMP_FUSE_DIRECTIVE: {
       unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
-      S = OMPFuseDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      unsigned NumLoopNests = Record[ASTStmtReader::NumStmtFields + 2];
+      S = OMPFuseDirective::CreateEmpty(Context, NumClauses, NumLoops,
+                                        NumLoopNests);
       break;
     }
 
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index af7229d748872..c99ffab64c6e6 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7807,6 +7807,14 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
+  Record.AddStmt(C->getFirst());
+  Record.AddStmt(C->getCount());
+  Record.AddSourceLocation(C->getLParenLoc());
+  Record.AddSourceLocation(C->getFirstLoc());
+  Record.AddSourceLocation(C->getCountLoc());
+}
+
 void OMPClauseWriter::VisitOMPAllocatorClause(OMPAllocatorClause *C) {
   Record.AddStmt(C->getAllocator());
   Record.AddSourceLocation(C->getLParenLoc());
diff --git a/clang/test/OpenMP/fuse_ast_print.cpp b/clang/test/OpenMP/fuse_ast_print.cpp
index 43ce815dab024..ac4f0d38a9c68 100644
--- a/clang/test/OpenMP/fuse_ast_print.cpp
+++ b/clang/test/OpenMP/fuse_ast_print.cpp
@@ -271,6 +271,73 @@ void foo7() {
 
 }
 
+// PRINT-LABEL: void foo8(
+// DUMP-LABEL: FunctionDecl {{.*}} foo8
+void foo8() {
+    // PRINT: #pragma omp fuse looprange(2,2)
+    // DUMP:  OMPFuseDirective
+    // DUMP: OMPLooprangeClause
+    #pragma omp fuse looprange(2,2)
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int j = 10; j > 0; --j)
+        // DUMP: ForStmt
+        for (int j = 10; j > 0; --j)
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);
+        // PRINT: for (int k = 0; k <= 10; ++k)
+        // DUMP: ForStmt
+        for (int k = 0; k <= 10; ++k)
+            // PRINT: body(k)
+            // DUMP: CallExpr
+            body(k);
+
+    }
+
+}
+
+//PRINT-LABEL: void foo9(
+//DUMP-LABEL: FunctionTemplateDecl {{.*}} foo9
+//DUMP-LABEL: NonTypeTemplateParmDecl {{.*}} F
+//DUMP-LABEL: NonTypeTemplateParmDecl {{.*}} C
+template<int F, int C> 
+void foo9() {
+    // PRINT:  #pragma omp fuse looprange(F,C)
+    // DUMP: OMPFuseDirective
+    // DUMP: OMPLooprangeClause
+    #pragma omp fuse looprange(F,C)
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int j = 10; j > 0; --j)
+        // DUMP: ForStmt
+        for (int j = 10; j > 0; --j)
+            // PRINT: body(j)
+            // DUMP: CallExpr
+            body(j);
+
+    }
+}
+
+// Also test instantiating the template.
+void tfoo9() {
+    foo9<1, 2>();
+}
+
 
 
 
diff --git a/clang/test/OpenMP/fuse_codegen.cpp b/clang/test/OpenMP/fuse_codegen.cpp
index 6c1e21092da43..d9500bed3ce31 100644
--- a/clang/test/OpenMP/fuse_codegen.cpp
+++ b/clang/test/OpenMP/fuse_codegen.cpp
@@ -53,6 +53,18 @@ extern "C" void foo3() {
     }
 }
 
+extern "C" void foo4() {
+    double arr[256];
+
+    #pragma omp fuse looprange(2,2)
+    {
+        for(int i = 0; i < 128; ++i) body(i);
+        for(int j = 0; j < 256; j+=2) body(j);
+        for(int k = 0; k < 64; ++k) body(k);
+        for(int c = 42; auto &&v: arr) body(c,v);
+    }
+}
+
 
 #endif
 // CHECK1-LABEL: define dso_local void @body(
@@ -777,6 +789,157 @@ extern "C" void foo3() {
 // CHECK1-NEXT:    ret void
 //
 //
+// CHECK1-LABEL: define dso_local void @foo4(
+// CHECK1-SAME: ) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[K]], align 4
+// CHECK1-NEXT:    store i32 63, ptr [[DOTOMP_UB1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    store i32 64, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP5]], 128
+// CHECK1-NEXT:    br i1 [[CMP1]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP6]])
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND2:.*]]
+// CHECK1:       [[FOR_COND2]]:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label %[[FOR_BODY4:.*]], label %[[FOR_END17:.*]]
+// CHECK1:       [[FOR_BODY4]]:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP10]], [[TMP11]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP13]], [[TMP14]]
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP12]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP15]], 2
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL6]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP16]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP17]], [[TMP18]]
+// CHECK1-NEXT:    br i1 [[CMP8]], label %[[IF_THEN9:.*]], label %[[IF_END14:.*]]
+// CHECK1:       [[IF_THEN9]]:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP19]], [[MUL10]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP22]], 1
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 0, [[MUL12]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP23]])
+// CHECK1-NEXT:    br label %[[IF_END14]]
+// CHECK1:       [[IF_END14]]:
+// CHECK1-NEXT:    br label %[[FOR_INC15:.*]]
+// CHECK1:       [[FOR_INC15]]:
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC16:%.*]] = add nsw i32 [[TMP24]], 1
+// CHECK1-NEXT:    store i32 [[INC16]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND2]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1:       [[FOR_END17]]:
+// CHECK1-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP25]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY18:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP26]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY18]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND19:.*]]
+// CHECK1:       [[FOR_COND19]]:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP28:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[CMP20:%.*]] = icmp ne ptr [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP20]], label %[[FOR_BODY21:.*]], label %[[FOR_END23:.*]]
+// CHECK1:       [[FOR_BODY21]]:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP29]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[C]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load double, ptr [[TMP31]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP30]], double noundef [[TMP32]])
+// CHECK1-NEXT:    br label %[[FOR_INC22:.*]]
+// CHECK1:       [[FOR_INC22]]:
+// CHECK1-NEXT:    [[TMP33:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw double, ptr [[TMP33]], i32 1
+// CHECK1-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND19]]
+// CHECK1:       [[FOR_END23]]:
+// CHECK1-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define dso_local void @body(
 // CHECK2-SAME: ...) #[[ATTR0:[0-9]+]] {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
@@ -1259,6 +1422,157 @@ extern "C" void foo3() {
 // CHECK2-NEXT:    ret void
 //
 //
+// CHECK2-LABEL: define dso_local void @foo4(
+// CHECK2-SAME: ) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[K]], align 4
+// CHECK2-NEXT:    store i32 63, ptr [[DOTOMP_UB1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    store i32 64, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP5]], 128
+// CHECK2-NEXT:    br i1 [[CMP1]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP6]])
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND2:.*]]
+// CHECK2:       [[FOR_COND2]]:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label %[[FOR_BODY4:.*]], label %[[FOR_END17:.*]]
+// CHECK2:       [[FOR_BODY4]]:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP10]], [[TMP11]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP13]], [[TMP14]]
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP12]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP15]], 2
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL6]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP16]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP17]], [[TMP18]]
+// CHECK2-NEXT:    br i1 [[CMP8]], label %[[IF_THEN9:.*]], label %[[IF_END14:.*]]
+// CHECK2:       [[IF_THEN9]]:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP19]], [[MUL10]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP22]], 1
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 0, [[MUL12]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP23]])
+// CHECK2-NEXT:    br label %[[IF_END14]]
+// CHECK2:       [[IF_END14]]:
+// CHECK2-NEXT:    br label %[[FOR_INC15:.*]]
+// CHECK2:       [[FOR_INC15]]:
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC16:%.*]] = add nsw i32 [[TMP24]], 1
+// CHECK2-NEXT:    store i32 [[INC16]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND2]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK2:       [[FOR_END17]]:
+// CHECK2-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP25]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY18:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP26]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY18]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND19:.*]]
+// CHECK2:       [[FOR_COND19]]:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP28:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[CMP20:%.*]] = icmp ne ptr [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP20]], label %[[FOR_BODY21:.*]], label %[[FOR_END23:.*]]
+// CHECK2:       [[FOR_BODY21]]:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP29]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[C]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load double, ptr [[TMP31]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP30]], double noundef [[TMP32]])
+// CHECK2-NEXT:    br label %[[FOR_INC22:.*]]
+// CHECK2:       [[FOR_INC22]]:
+// CHECK2-NEXT:    [[TMP33:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw double, ptr [[TMP33]], i32 1
+// CHECK2-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND19]]
+// CHECK2:       [[FOR_END23]]:
+// CHECK2-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define dso_local void @tfoo2(
 // CHECK2-SAME: ) #[[ATTR0]] {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
@@ -1494,7 +1808,7 @@ extern "C" void foo3() {
 // CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
 // CHECK2:       [[FOR_END]]:
 // CHECK2-NEXT:    ret void
 //
@@ -1503,9 +1817,13 @@ extern "C" void foo3() {
 // CHECK1: [[META4]] = !{!"llvm.loop.mustprogress"}
 // CHECK1: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
 // CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
+// CHECK1: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
+// CHECK1: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
 //.
 // CHECK2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
 // CHECK2: [[META4]] = !{!"llvm.loop.mustprogress"}
 // CHECK2: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
 // CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
+// CHECK2: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
+// CHECK2: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
 //.
diff --git a/clang/test/OpenMP/fuse_messages.cpp b/clang/test/OpenMP/fuse_messages.cpp
index 50dedfd2c0dc6..2a2491d008a0b 100644
--- a/clang/test/OpenMP/fuse_messages.cpp
+++ b/clang/test/OpenMP/fuse_messages.cpp
@@ -33,6 +33,8 @@ void func() {
     {
         for (int i = 0; i < 7; ++i)
             ;
+        for(int j = 0; j < 100; ++j);
+
     }
 
 
@@ -41,6 +43,8 @@ void func() {
     {
         for (int i = 0; i < 7; ++i)
             ;
+        for(int j = 0; j < 100; ++j);
+
     }
 
     //expected-error at +4 {{loop after '#pragma omp fuse' is not in canonical form}}
@@ -50,6 +54,7 @@ void func() {
         for(int i = 0; i < 10; i*=2) {
             ;
         }
+        for(int j = 0; j < 100; ++j);
     }
 
     //expected-error at +2 {{loop sequence after '#pragma omp fuse' must contain at least 1 canonical loop or loop-generating construct}}
@@ -73,4 +78,109 @@ void func() {
         for(unsigned int j = 0; j < 10; ++j);
         for(long long k = 0; k < 100; ++k);
     }
-}
\ No newline at end of file
+
+    //expected-warning at +2 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    #pragma omp fuse
+    {
+        for(int i = 0; i < 10; ++i);
+    }
+
+    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    #pragma omp fuse looprange(1, 1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+    }
+
+    //expected-error at +1 {{argument to 'looprange' clause must be a strictly positive integer value}}
+    #pragma omp fuse looprange(1, -1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+    }
+
+    //expected-error at +1 {{argument to 'looprange' clause must be a strictly positive integer value}}
+    #pragma omp fuse looprange(1, 0)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+    }
+
+    const int x = 1;
+    constexpr int y = 4;
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '4' is greater than the total number of loops '3'}}
+    #pragma omp fuse looprange(x,y)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '420' is greater than the total number of loops '3'}}
+    #pragma omp fuse looprange(1,420)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+}
+
+// In a template context, but expression itself not instantiation-dependent
+template <typename T>
+static void templated_func() {
+
+    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    #pragma omp fuse looprange(2,1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '5' is greater than the total number of loops '3'}}
+    #pragma omp fuse looprange(3,3)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+
+}
+
+template <int V> 
+static void templated_func_value_dependent() {
+
+    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    #pragma omp fuse looprange(V,1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+}
+
+template <typename T> 
+static void templated_func_type_dependent() {
+    constexpr T s = 1;
+
+    //expected-error at +1 {{argument to 'looprange' clause must be a strictly positive integer value}}
+    #pragma omp fuse looprange(s,s-1)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+    }
+}
+
+
+void template_inst() {
+    // expected-note at +1 {{in instantiation of function template specialization 'templated_func<int>' requested here}}
+    templated_func<int>();
+    // expected-note at +1 {{in instantiation of function template specialization 'templated_func_value_dependent<1>' requested here}}
+    templated_func_value_dependent<1>();
+    // expected-note at +1 {{in instantiation of function template specialization 'templated_func_type_dependent<int>' requested here}}
+    templated_func_type_dependent<int>();
+
+}
+
+
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 3afa59b2f2d6c..deb6fdd68e6d3 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2417,6 +2417,11 @@ void OMPClauseEnqueue::VisitOMPPartialClause(const OMPPartialClause *C) {
   Visitor->AddStmt(C->getFactor());
 }
 
+void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
+  Visitor->AddStmt(C->getFirst());
+  Visitor->AddStmt(C->getCount());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index de888ff86fe91..a2f258bbf745b 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -1233,6 +1233,15 @@ struct WriteT {
   using EmptyTrait = std::true_type;
 };
 
+// V6: [6.4.7] Looprange clause
+template <typename T, typename I, typename E> struct LoopRangeT {
+  using Begin = E;
+  using End = E;
+
+  using TupleTrait = std::true_type;
+  std::tuple<Begin, End> t;
+};
+
 // ---
 
 template <typename T, typename I, typename E>
@@ -1263,9 +1272,10 @@ using TupleClausesT =
                  DefaultmapT<T, I, E>, DeviceT<T, I, E>, DistScheduleT<T, I, E>,
                  DoacrossT<T, I, E>, FromT<T, I, E>, GrainsizeT<T, I, E>,
                  IfT<T, I, E>, InitT<T, I, E>, InReductionT<T, I, E>,
-                 LastprivateT<T, I, E>, LinearT<T, I, E>, MapT<T, I, E>,
-                 NumTasksT<T, I, E>, OrderT<T, I, E>, ReductionT<T, I, E>,
-                 ScheduleT<T, I, E>, TaskReductionT<T, I, E>, ToT<T, I, E>>;
+                 LastprivateT<T, I, E>, LinearT<T, I, E>, LoopRangeT<T, I, E>,
+                 MapT<T, I, E>, NumTasksT<T, I, E>, OrderT<T, I, E>,
+                 ReductionT<T, I, E>, ScheduleT<T, I, E>,
+                 TaskReductionT<T, I, E>, ToT<T, I, E>>;
 
 template <typename T, typename I, typename E>
 using UnionClausesT = std::variant<DependT<T, I, E>>;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 6352be8069e9e..989b35a7caa2a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -274,6 +274,9 @@ def OMPC_Linear : Clause<[Spelling<"linear">]> {
 def OMPC_Link : Clause<[Spelling<"link">]> {
   let flangClass = "OmpObjectList";
 }
+def OMPC_LoopRange : Clause<[Spelling<"looprange">]> {
+  let clangClass = "OMPLoopRangeClause";
+}
 def OMPC_Map : Clause<[Spelling<"map">]> {
   let clangClass = "OMPMapClause";
   let flangClass = "OmpMapClause";
@@ -856,6 +859,9 @@ def OMP_For : Directive<[Spelling<"for">]> {
   let languages = [L_C];
 }
 def OMP_Fuse : Directive<[Spelling<"fuse">]> {
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_LoopRange, 60>
+  ];
   let association = AS_Loop;
   let category = CA_Executable;
 }

>From c76888b9dd1f516512d2d64afa4766adaad4de1e Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:30:39 +0000
Subject: [PATCH 03/13] Added fuse to documentation

---
 clang/docs/OpenMPSupport.rst | 2 ++
 clang/docs/ReleaseNotes.rst  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 986aaabe1eed4..12325e3294a64 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -376,6 +376,8 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop stripe transformation                                  | :good:`done`              | https://github.com/llvm/llvm-project/pull/119891                                                     |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
+| loop fuse transformation                                    | :good:`done`              | :none:`unclaimed`        |                                                                          |
++-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | work distribute construct                                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | task_iteration                                              | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index dd748ab06873d..c75cb25a4db73 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -1128,6 +1128,7 @@ OpenMP Support
 - Fixed mapping of arrays of structs containing nested structs with user defined
   mappers, by using compiler-generated default mappers for the outer structs for
   such maps.
+- Added support for 'omp fuse' directive.
 
 Improvements
 ^^^^^^^^^^^^

>From 1c054673983282e7e6afef0f11c2a7fbe39181d7 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:43:41 +0000
Subject: [PATCH 04/13] Refactored preinits handling and improved coverage

---
 clang/docs/OpenMPSupport.rst          |    2 +-
 clang/include/clang/AST/StmtOpenMP.h  |    5 +-
 clang/include/clang/Sema/SemaOpenMP.h |   96 +-
 clang/lib/AST/StmtOpenMP.cpp          |   13 +
 clang/lib/Basic/OpenMPKinds.cpp       |    3 +-
 clang/lib/CodeGen/CGExpr.cpp          |    2 +
 clang/lib/CodeGen/CodeGenFunction.h   |    4 +
 clang/lib/Sema/SemaOpenMP.cpp         |  588 ++++---
 clang/test/OpenMP/fuse_ast_print.cpp  |   55 +
 clang/test/OpenMP/fuse_codegen.cpp    | 2117 +++++++++++++++----------
 10 files changed, 1862 insertions(+), 1023 deletions(-)

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 12325e3294a64..8d62208e55f75 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -376,7 +376,7 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop stripe transformation                                  | :good:`done`              | https://github.com/llvm/llvm-project/pull/119891                                                     |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop fuse transformation                                    | :good:`done`              | :none:`unclaimed`        |                                                                          |
+| loop fuse transformation                                    | :good:`prototyped`        | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | work distribute construct                                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index f5115afd0753e..6425f6616a558 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1005,8 +1005,7 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass ||
-           C == OMPFuseDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFuseDirectiveClass;
   }
 };
 
@@ -5653,6 +5652,8 @@ class OMPStripeDirective final : public OMPLoopTransformationDirective {
                                        llvm::omp::OMPD_stripe, StartLoc, EndLoc,
                                        NumLoops) {
     setNumGeneratedLoops(2 * NumLoops);
+    // Similar to Tile, it only generates a single top level loop nest
+    setNumGeneratedLoopNests(1);
   }
 
   void setPreInits(Stmt *PreInits) {
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 0c28aaf6ab21a..547ea95c6cd5d 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1494,16 +1494,96 @@ class SemaOpenMP : public SemaBase {
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits);
 
-  /// Analyzes and checks a loop sequence for use by a loop transformation
+  /// @brief Categories of loops encountered during semantic OpenMP loop
+  /// analysis
+  ///
+  /// This enumeration identifies the structural category of a loop or sequence
+  /// of loops analyzed in the context of OpenMP transformations and directives.
+  /// This categorization helps differentiate between original source loops
+  /// and the structures resulting from applying OpenMP loop transformations.
+  enum class OMPLoopCategory {
+
+    /// @var OMPLoopCategory::RegularLoop
+    /// Represents a standard canonical loop nest found in the
+    /// original source code or an intact loop after transformations
+    /// (i.e Post/Pre loops of a loopranged fusion)
+    RegularLoop,
+
+    /// @var OMPLoopCategory::TransformSingleLoop
+    /// Represents the resulting loop structure when an OpenMP loop
+    //  transformation, generates a single, top-level loop
+    TransformSingleLoop,
+
+    /// @var OMPLoopCategory::TransformLoopSequence
+    /// Represents the resulting loop structure when an OpenMP loop
+    /// transformation
+    /// generates a sequence of two or more canonical loop nests
+    TransformLoopSequence
+  };
+
+  /// The main recursive process of `checkTransformableLoopSequence` that
+  /// performs grammatical parsing of a canonical loop sequence. It extracts
+  /// key information, such as the number of top-level loops, loop statements,
+  /// helper expressions, and other relevant loop-related data, all in a single
+  /// execution to avoid redundant traversals. This analysis flattens inner
+  /// Loop Sequences
+  ///
+  /// \param LoopSeqStmt    The AST of the original statement.
+  /// \param LoopSeqSize    [out] Number of top level canonical loops.
+  /// \param NumLoops       [out] Number of total canonical loops (nested too).
+  /// \param LoopHelpers    [out] The multiple loop analyses results.
+  /// \param ForStmts       [out] The multiple Stmt of each For loop.
+  /// \param OriginalInits  [out] The raw original initialization statements
+  ///                       of each belonging to a loop of the loop sequence
+  /// \param TransformPreInits [out] The multiple collection of statements and
+  ///                       declarations that must have been executed/declared
+  ///                       before entering the loop (each belonging to a
+  ///                       particular loop transformation, nullptr otherwise)
+  /// \param LoopSequencePreInits [out] Additional general collection of loop
+  ///                       transformation related statements and declarations
+  ///                       not bounded to a particular loop that must be
+  ///                       executed before entering the loop transformation
+  /// \param LoopCategories [out] A sequence of OMPLoopCategory values,
+  ///                       one for each loop or loop transformation node
+  ///                       successfully analyzed.
+  /// \param Context
+  /// \param Kind           The loop transformation directive kind.
+  /// \return Whether the original statement is both syntactically and
+  /// semantically correct according to OpenMP 6.0 canonical loop
+  /// sequence definition.
+  bool analyzeLoopSequence(
+      Stmt *LoopSeqStmt, unsigned &LoopSeqSize, unsigned &NumLoops,
+      SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+      SmallVectorImpl<Stmt *> &ForStmts,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+      SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context,
+      OpenMPDirectiveKind Kind);
+
+  /// Validates and checks whether a loop sequence can be transformed according
+  /// to the given directive, providing necessary setup and initialization
+  /// (Driver function) before recursion using `analyzeLoopSequence`.
   ///
   /// \param Kind           The loop transformation directive kind.
-  /// \param NumLoops       [out] Number of total canonical loops
-  /// \param LoopSeqSize    [out] Number of top level canonical loops
+  /// \param AStmt          The AST of the original statement
+  /// \param LoopSeqSize    [out] Number of top level canonical loops.
+  /// \param NumLoops       [out] Number of total canonical loops (nested too)
   /// \param LoopHelpers    [out] The multiple loop analyses results.
-  /// \param LoopStmts      [out] The multiple Stmt of each For loop.
-  /// \param OriginalInits  [out] The multiple collection of statements and
+  /// \param ForStmts       [out] The multiple Stmt of each For loop.
+  /// \param OriginalInits  [out] The raw original initialization statements
+  ///                       of each belonging to a loop of the loop sequence
+  /// \param TransformsPreInits [out] The multiple collection of statements and
   ///                       declarations that must have been executed/declared
-  ///                       before entering the loop.
+  ///                       before entering the loop (each belonging to a
+  ///                       particular loop transformation, nullptr otherwise)
+  /// \param LoopSequencePreInits [out] Additional general collection of loop
+  ///                       transformation related statements and declarations
+  ///                       not bounded to a particular loop that must be
+  ///                       executed before entering the loop transformation
+  /// \param LoopCategories [out] A sequence of OMPLoopCategory values,
+  ///                       one for each loop or loop transformation node
+  ///                       successfully analyzed.
   /// \param Context
   /// \return Whether there was an absence of errors or not
   bool checkTransformableLoopSequence(
@@ -1512,7 +1592,9 @@ class SemaOpenMP : public SemaBase {
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       SmallVectorImpl<Stmt *> &ForStmts,
       SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-      ASTContext &Context);
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
+      SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+      SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context);
 
   /// Helper to keep information about the current `omp begin/end declare
   /// variant` nesting.
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index c5a6732cc2217..f527e6361b5e5 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -457,6 +457,8 @@ OMPUnrollDirective::Create(const ASTContext &C, SourceLocation StartLoc,
       C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc);
   Dir->setNumGeneratedLoops(NumGeneratedLoops);
   // The number of generated loops and loop nests during unroll matches
+  // given that unroll only generates top level canonical loop nests
+  // so each generated loop is a top level canonical loop nest
   Dir->setNumGeneratedLoopNests(NumGeneratedLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
@@ -520,6 +522,17 @@ OMPFuseDirective *OMPFuseDirective::Create(
       NumLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
+  // The number of top level canonical nests could 
+  // not match the total number of generated loops
+  // Example:
+  // Before fusion:
+  //   for (int i = 0; i < N; ++i)   
+  //     for (int j = 0; j < M; ++j) 
+  //       A[i][j] = i + j;
+  //   
+  //   for (int k = 0; k < P; ++k) 
+  //     B[k] = k * 2;
+  // Here, NumLoopNests = 2, but NumLoops = 3.
   Dir->setNumGeneratedLoopNests(NumLoopNests);
   Dir->setNumGeneratedLoops(NumLoops);
   return Dir;
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 18330181f1509..53a9f80e6d3b7 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -704,7 +704,8 @@ bool clang::isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind) {
 
 bool clang::isOpenMPLoopTransformationDirective(OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
-         DKind == OMPD_interchange || DKind == OMPD_stripe || DKind == OMPD_fuse;
+         DKind == OMPD_interchange || DKind == OMPD_stripe ||
+         DKind == OMPD_fuse;
 }
 
 bool clang::isOpenMPCombinedParallelADirective(OpenMPDirectiveKind DKind) {
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 6cb348ffdf55f..08049d4d4e37d 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3257,6 +3257,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
 
     // No other cases for now.
     } else {
+      llvm::dbgs() << "THE DAMN DECLREFEXPR HASN'T BEEN ENTERED IN LOCALDECLMAP\n";
+      VD->dumpColor();
       llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
     }
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index fe753e5b688b1..bfe24213ed377 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5363,6 +5363,10 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// Set the address of a local variable.
   void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
+    if (LocalDeclMap.count(VD)) {
+      llvm::errs() << "Warning: VarDecl already exists in map: ";
+      VD->dumpColor(); 
+    }
     assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
     LocalDeclMap.insert({VD, Addr});
   }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 3ec3f2ad31e78..3ce256f3ec23b 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DynamicRecursiveASTVisitor.h"
 #include "clang/AST/OpenMPClause.h"
+#include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/StmtVisitor.h"
@@ -47,6 +48,7 @@
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/IR/Assumptions.h"
 #include <optional>
+#include <queue>
 
 using namespace clang;
 using namespace llvm::omp;
@@ -14194,6 +14196,45 @@ StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
       getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
+// Overloaded base case function
+template <typename T, typename F>
+static bool tryHandleAs(T *t, F &&) {
+    return false; 
+}
+
+/**
+ * Tries to recursively cast `t` to one of the given types and invokes `f` if successful.
+ *
+ * @tparam Class The first type to check.
+ * @tparam Rest The remaining types to check.
+ * @tparam T The base type of `t`.
+ * @tparam F The callable type for the function to invoke upon a successful cast.
+ * @param t The object to be checked.
+ * @param f The function to invoke if `t` matches `Class`.
+ * @return `true` if `t` matched any type and `f` was called, otherwise `false`.
+ */
+template <typename Class, typename... Rest, typename T, typename F>
+static bool tryHandleAs(T *t, F &&f) {
+    if (Class *c = dyn_cast<Class>(t)) {
+        f(c); 
+        return true;
+    } else {
+        return tryHandleAs<Rest...>(t, std::forward<F>(f));
+    }
+}
+
+// Updates OriginalInits by checking Transform against loop transformation
+// directives and appending their pre-inits if a match is found.
+static void updatePreInits(OMPLoopBasedDirective *Transform,
+                           SmallVectorImpl<SmallVector<Stmt *, 0>> &PreInits) {
+    if (!tryHandleAs<OMPTileDirective, OMPUnrollDirective, OMPReverseDirective,
+                     OMPInterchangeDirective, OMPFuseDirective>(
+            Transform, [&PreInits](auto *Dir) {
+              appendFlattenedStmtList(PreInits.back(), Dir->getPreInits());
+            }))
+        llvm_unreachable("Unhandled loop transformation");
+}
+
 bool SemaOpenMP::checkTransformableLoopNest(
     OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
@@ -14224,121 +14265,106 @@ bool SemaOpenMP::checkTransformableLoopNest(
         return false;
       },
       [&OriginalInits](OMPLoopBasedDirective *Transform) {
-        Stmt *DependentPreInits;
-        if (auto *Dir = dyn_cast<OMPTileDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPStripeDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPUnrollDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPReverseDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPFuseDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else
-          llvm_unreachable("Unhandled loop transformation");
-
-        appendFlattenedStmtList(OriginalInits.back(), DependentPreInits);
+        updatePreInits(Transform, OriginalInits);
       });
   assert(OriginalInits.back().empty() && "No preinit after innermost loop");
   OriginalInits.pop_back();
   return Result;
 }
 
-class NestedLoopCounterVisitor
-    : public clang::RecursiveASTVisitor<NestedLoopCounterVisitor> {
+// Counts the total number of nested loops, including the outermost loop (the
+// original loop). PRECONDITION of this visitor is that it must be invoked from
+// the original loop to be analyzed. The traversal is stop for Decl's and
+// Expr's given that they may contain inner loops that must not be counted.
+//
+// Example AST structure for the code:
+//
+// int main() {
+//     #pragma omp fuse
+//     {
+//         for (int i = 0; i < 100; i++) {    <-- Outer loop
+//             []() {
+//                 for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
+//             };
+//             for(int j = 0; j < 5; ++j) {}    <-- Inner loop
+//         }
+//         for (int r = 0; i < 100; i++) {    <-- Outer loop
+//             struct LocalClass {
+//                 void bar() {
+//                     for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
+//                 }
+//             };
+//             for(int k = 0; k < 10; ++k) {}    <-- Inner loop
+//             {x = 5; for(k = 0; k < 10; ++k) x += k; x}; <-- NOT A LOOP
+//         }
+//     }
+// }
+// Result: Loop 'i' contains 2 loops, Loop 'r' also contains 2 loops
+class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
+private:
+  unsigned NestedLoopCount = 0;
+
 public:
-  explicit NestedLoopCounterVisitor() : NestedLoopCount(0) {}
+  explicit NestedLoopCounterVisitor() {}
 
-  bool VisitForStmt(clang::ForStmt *FS) {
-    ++NestedLoopCount;
-    return true;
+  unsigned getNestedLoopCount() const { return NestedLoopCount; }
+
+  bool VisitForStmt(ForStmt *FS) override {
+        ++NestedLoopCount;
+        return true;
   }
 
-  bool VisitCXXForRangeStmt(clang::CXXForRangeStmt *FRS) {
-    ++NestedLoopCount;
-    return true;
+  bool VisitCXXForRangeStmt(CXXForRangeStmt *FRS) override {
+        ++NestedLoopCount;
+        return true;
   }
 
-  unsigned getNestedLoopCount() const { return NestedLoopCount; }
+  bool TraverseStmt(Stmt *S) override {
+        if (!S)
+      return true;
 
-private:
-  unsigned NestedLoopCount;
+        // Skip traversal of all expressions, including special cases like
+        // LambdaExpr, StmtExpr, BlockExpr, and RequiresExpr. These expressions
+        // may contain inner statements (and even loops), but they are not part
+        // of the syntactic body of the surrounding loop structure.
+        //  Therefore must not be counted
+        if (isa<Expr>(S))
+      return true;
+
+        // Only recurse into CompoundStmt (block {}) and loop bodies
+        if (isa<CompoundStmt>(S) || isa<ForStmt>(S) ||
+            isa<CXXForRangeStmt>(S)) {
+      return DynamicRecursiveASTVisitor::TraverseStmt(S);
+        }
+
+        // Stop traversal of the rest of statements, that break perfect
+        // loop nesting, such as control flow (IfStmt, SwitchStmt...)
+        return true;
+  }
+
+  bool TraverseDecl(Decl *D) override {
+        // Stop in the case of finding a declaration, it is not important
+        // in order to find nested loops (Possible CXXRecordDecl, RecordDecl,
+        // FunctionDecl...)
+        return true;
+  }
 };
 
-bool SemaOpenMP::checkTransformableLoopSequence(
-    OpenMPDirectiveKind Kind, Stmt *AStmt, unsigned &LoopSeqSize,
-    unsigned &NumLoops,
+bool SemaOpenMP::analyzeLoopSequence(
+    Stmt *LoopSeqStmt, unsigned &LoopSeqSize, unsigned &NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
     SmallVectorImpl<Stmt *> &ForStmts,
     SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-    ASTContext &Context) {
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+    SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context,
+    OpenMPDirectiveKind Kind) {
 
-  // Checks whether the given statement is a compound statement
   VarsWithInheritedDSAType TmpDSA;
-  if (!isa<CompoundStmt>(AStmt)) {
-    Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
-        << getOpenMPDirectiveName(Kind);
-    return false;
-  }
-  // Callback for updating pre-inits in case there are even more
-  // loop-sequence-generating-constructs inside of the main compound stmt
-  auto OnTransformationCallback =
-      [&OriginalInits](OMPLoopBasedDirective *Transform) {
-        Stmt *DependentPreInits;
-        if (auto *Dir = dyn_cast<OMPTileDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPUnrollDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPReverseDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPInterchangeDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else if (auto *Dir = dyn_cast<OMPFuseDirective>(Transform))
-          DependentPreInits = Dir->getPreInits();
-        else
-          llvm_unreachable("Unhandled loop transformation");
-
-        appendFlattenedStmtList(OriginalInits.back(), DependentPreInits);
-      };
-
-  // Number of top level canonical loop nests observed (And acts as index)
-  LoopSeqSize = 0;
-  // Number of total observed loops
-  NumLoops = 0;
-
-  // Following OpenMP 6.0 API Specification, a Canonical Loop Sequence follows
-  // the grammar:
-  //
-  // canonical-loop-sequence:
-  //  {
-  //    loop-sequence+
-  //  }
-  // where loop-sequence can be any of the following:
-  // 1. canonical-loop-sequence
-  // 2. loop-nest
-  // 3. loop-sequence-generating-construct (i.e OMPLoopTransformationDirective)
-  //
-  // To recognise and traverse this structure the following helper functions
-  // have been defined. handleLoopSequence serves as the recurisve entry point
-  // and tries to match the input AST to the canonical loop sequence grammar
-  // structure
-
-  // Helper functions to validate canonical loop sequence grammar is valid
-  auto isLoopSequenceDerivation = [](auto *Child) {
-    return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
-           isa<OMPLoopTransformationDirective>(Child);
-  };
-  auto isLoopGeneratingStmt = [](auto *Child) {
-    return isa<OMPLoopTransformationDirective>(Child);
-  };
-
+  QualType BaseInductionVarType;
   // Helper Lambda to handle storing initialization and body statements for both
   // ForStmt and CXXForRangeStmt and checks for any possible mismatch between
   // induction variables types
-  QualType BaseInductionVarType;
   auto storeLoopStatements = [&OriginalInits, &ForStmts, &BaseInductionVarType,
                               this, &Context](Stmt *LoopStmt) {
     if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
@@ -14361,33 +14387,35 @@ bool SemaOpenMP::checkTransformableLoopSequence(
           }
         }
       }
-
     } else {
-      assert(isa<CXXForRangeStmt>(LoopStmt) &&
-             "Expected canonical for or range-based for loops.");
-      auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt);
+      auto *CXXFor = cast<CXXForRangeStmt>(LoopStmt);
       OriginalInits.back().push_back(CXXFor->getBeginStmt());
       ForStmts.push_back(CXXFor);
     }
   };
+
   // Helper lambda functions to encapsulate the processing of different
   // derivations of the canonical loop sequence grammar
   //
   // Modularized code for handling loop generation and transformations
-  auto handleLoopGeneration = [&storeLoopStatements, &LoopHelpers,
-                               &OriginalInits, &LoopSeqSize, &NumLoops, Kind,
-                               &TmpDSA, &OnTransformationCallback,
-                               this](Stmt *Child) {
+  auto analyzeLoopGeneration = [&storeLoopStatements, &LoopHelpers,
+                                &OriginalInits, &TransformsPreInits,
+                                &LoopCategories, &LoopSeqSize, &NumLoops, Kind,
+                                &TmpDSA, &ForStmts, &Context,
+                                &LoopSequencePreInits, this](Stmt *Child) {
     auto LoopTransform = dyn_cast<OMPLoopTransformationDirective>(Child);
     Stmt *TransformedStmt = LoopTransform->getTransformedStmt();
     unsigned NumGeneratedLoopNests = LoopTransform->getNumGeneratedLoopNests();
-
+    unsigned NumGeneratedLoops = LoopTransform->getNumGeneratedLoops();
     // Handle the case where transformed statement is not available due to
     // dependent contexts
     if (!TransformedStmt) {
-      if (NumGeneratedLoopNests > 0)
+      if (NumGeneratedLoopNests > 0) {
+        LoopSeqSize += NumGeneratedLoopNests;
+        NumLoops += NumGeneratedLoops;
         return true;
-      // Unroll full
+      }
+      // Unroll full (0 loops produced)
       else {
         Diag(Child->getBeginLoc(), diag::err_omp_not_for)
             << 0 << getOpenMPDirectiveName(Kind);
@@ -14400,38 +14428,56 @@ bool SemaOpenMP::checkTransformableLoopSequence(
       Diag(Child->getBeginLoc(), diag::err_omp_not_for)
           << 0 << getOpenMPDirectiveName(Kind);
       return false;
-      // Future loop transformations that generate multiple canonical loops
-    } else if (NumGeneratedLoopNests > 1) {
-      llvm_unreachable("Multiple canonical loop generating transformations "
-                       "like loop splitting are not yet supported");
     }
+    // Loop transformatons such as split or loopranged fuse
+    else if (NumGeneratedLoopNests > 1) {
+      // Get the preinits related to this loop sequence generating
+      // loop transformation (i.e loopranged fuse, split...)
+      LoopSequencePreInits.emplace_back();
+      // These preinits differ slightly from regular inits/pre-inits related
+      // to single loop generating loop transformations (interchange, unroll)
+      // given that they are not bounded to a particular loop nest
+      // so they need to be treated independently
+      updatePreInits(LoopTransform, LoopSequencePreInits);
+      return analyzeLoopSequence(TransformedStmt, LoopSeqSize, NumLoops,
+                                 LoopHelpers, ForStmts, OriginalInits,
+                                 TransformsPreInits, LoopSequencePreInits,
+                                 LoopCategories, Context, Kind);
+    }
+    // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
+    else {
+      // Process the transformed loop statement
+      OriginalInits.emplace_back();
+      TransformsPreInits.emplace_back();
+      LoopHelpers.emplace_back();
+      LoopCategories.push_back(OMPLoopCategory::TransformSingleLoop);
+
+      unsigned IsCanonical =
+          checkOpenMPLoop(Kind, nullptr, nullptr, TransformedStmt, SemaRef,
+                          *DSAStack, TmpDSA, LoopHelpers[LoopSeqSize]);
+
+      if (!IsCanonical) {
+        Diag(TransformedStmt->getBeginLoc(), diag::err_omp_not_canonical_loop)
+            << getOpenMPDirectiveName(Kind);
+        return false;
+      }
+      storeLoopStatements(TransformedStmt);
+      updatePreInits(LoopTransform, TransformsPreInits);
 
-    // Process the transformed loop statement
-    Child = TransformedStmt;
-    OriginalInits.emplace_back();
-    LoopHelpers.emplace_back();
-    OnTransformationCallback(LoopTransform);
-
-    unsigned IsCanonical =
-        checkOpenMPLoop(Kind, nullptr, nullptr, Child, SemaRef, *DSAStack,
-                        TmpDSA, LoopHelpers[LoopSeqSize]);
-
-    if (!IsCanonical) {
-      Diag(Child->getBeginLoc(), diag::err_omp_not_canonical_loop)
-          << getOpenMPDirectiveName(Kind);
-      return false;
+      NumLoops += NumGeneratedLoops;
+      ++LoopSeqSize;
+      return true;
     }
-    storeLoopStatements(TransformedStmt);
-    NumLoops += LoopTransform->getNumGeneratedLoops();
-    return true;
   };
 
   // Modularized code for handling regular canonical loops
-  auto handleRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
-                            &LoopSeqSize, &NumLoops, Kind, &TmpDSA,
-                            this](Stmt *Child) {
+  auto analyzeRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
+                             &LoopSeqSize, &NumLoops, Kind, &TmpDSA,
+                             &LoopCategories, this](Stmt *Child) {
     OriginalInits.emplace_back();
     LoopHelpers.emplace_back();
+    LoopCategories.push_back(OMPLoopCategory::RegularLoop);
+
     unsigned IsCanonical =
         checkOpenMPLoop(Kind, nullptr, nullptr, Child, SemaRef, *DSAStack,
                         TmpDSA, LoopHelpers[LoopSeqSize]);
@@ -14449,57 +14495,114 @@ bool SemaOpenMP::checkTransformableLoopSequence(
     return true;
   };
 
-  // Helper function to process a Loop Sequence Recursively
-  auto handleLoopSequence = [&](Stmt *LoopSeqStmt,
-                                auto &handleLoopSequenceCallback) -> bool {
-    for (auto *Child : LoopSeqStmt->children()) {
-      if (!Child)
-        continue;
+  // Helper functions to validate canonical loop sequence grammar is valid
+  auto isLoopSequenceDerivation = [](auto *Child) {
+    return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
+           isa<OMPLoopTransformationDirective>(Child);
+  };
+  auto isLoopGeneratingStmt = [](auto *Child) {
+    return isa<OMPLoopTransformationDirective>(Child);
+  };
+
 
-      // Skip over non-loop-sequence statements
-      if (!isLoopSequenceDerivation(Child)) {
-        Child = Child->IgnoreContainers();
+  // High level grammar validation
+  for (auto *Child : LoopSeqStmt->children()) {
 
-        // Ignore empty compound statement
         if (!Child)
-          continue;
+      continue;
 
-        // In the case of a nested loop sequence ignoring containers would not
-        // be enough, a recurisve transversal of the loop sequence is required
-        if (isa<CompoundStmt>(Child)) {
-          if (!handleLoopSequenceCallback(Child, handleLoopSequenceCallback))
-            return false;
-          // Already been treated, skip this children
-          continue;
+        // Skip over non-loop-sequence statements
+        if (!isLoopSequenceDerivation(Child)) {
+      Child = Child->IgnoreContainers();
+
+      // Ignore empty compound statement
+      if (!Child)
+        continue;
+
+      // In the case of a nested loop sequence ignoring containers would not
+      // be enough, a recurisve transversal of the loop sequence is required
+      if (isa<CompoundStmt>(Child)) {
+        if (!analyzeLoopSequence(Child, LoopSeqSize, NumLoops, LoopHelpers,
+                                 ForStmts, OriginalInits, TransformsPreInits,
+                                 LoopSequencePreInits, LoopCategories, Context,
+                                 Kind))
+          return false;
+        // Already been treated, skip this children
+        continue;
+      }
+        }
+        // Regular loop sequence handling
+        if (isLoopSequenceDerivation(Child)) {
+      if (isLoopGeneratingStmt(Child)) {
+        if (!analyzeLoopGeneration(Child)) {
+          return false;
         }
+        // analyzeLoopGeneration updates Loop Sequence size accordingly
+
+      } else {
+        if (!analyzeRegularLoop(Child)) {
+          return false;
+        }
+        // Update the Loop Sequence size by one
+        ++LoopSeqSize;
       }
-      // Regular loop sequence handling
-      if (isLoopSequenceDerivation(Child)) {
-        if (isLoopGeneratingStmt(Child)) {
-          if (!handleLoopGeneration(Child)) {
-            return false;
-          }
         } else {
-          if (!handleRegularLoop(Child)) {
-            return false;
-          }
+      // Report error for invalid statement inside canonical loop sequence
+      Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+          << 0 << getOpenMPDirectiveName(Kind);
+      return false;
         }
-        ++LoopSeqSize;
-      } else {
-        // Report error for invalid statement inside canonical loop sequence
-        Diag(Child->getBeginLoc(), diag::err_omp_not_for)
-            << 0 << getOpenMPDirectiveName(Kind);
+  }
+  return true;
+}
+
+bool SemaOpenMP::checkTransformableLoopSequence(
+    OpenMPDirectiveKind Kind, Stmt *AStmt, unsigned &LoopSeqSize,
+    unsigned &NumLoops,
+    SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+    SmallVectorImpl<Stmt *> &ForStmts,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
+    SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+    SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context) {
+
+  // Checks whether the given statement is a compound statement
+  if (!isa<CompoundStmt>(AStmt)) {
+        Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
+            << getOpenMPDirectiveName(Kind);
         return false;
-      }
-    }
-    return true;
-  };
+  }
+  // Number of top level canonical loop nests observed (And acts as index)
+  LoopSeqSize = 0;
+  // Number of total observed loops
+  NumLoops = 0;
+
+  // Following OpenMP 6.0 API Specification, a Canonical Loop Sequence follows
+  // the grammar:
+  //
+  // canonical-loop-sequence:
+  //  {
+  //    loop-sequence+
+  //  }
+  // where loop-sequence can be any of the following:
+  // 1. canonical-loop-sequence
+  // 2. loop-nest
+  // 3. loop-sequence-generating-construct (i.e OMPLoopTransformationDirective)
+  //
+  // To recognise and traverse this structure the following helper functions
+  // have been defined. analyzeLoopSequence serves as the recurisve entry point
+  // and tries to match the input AST to the canonical loop sequence grammar
+  // structure. This function will perform both a semantic and syntactical
+  // analysis of the given statement according to OpenMP 6.0 definition of
+  // the aforementioned canonical loop sequence
 
   // Recursive entry point to process the main loop sequence
-  if (!handleLoopSequence(AStmt, handleLoopSequence)) {
-    return false;
+  if (!analyzeLoopSequence(AStmt, LoopSeqSize, NumLoops, LoopHelpers, ForStmts,
+                           OriginalInits, TransformsPreInits,
+                           LoopSequencePreInits, LoopCategories, Context,
+                           Kind)) {
+        return false;
   }
-
   if (LoopSeqSize <= 0) {
     Diag(AStmt->getBeginLoc(), diag::err_omp_empty_loop_sequence)
         << getOpenMPDirectiveName(Kind);
@@ -14531,9 +14634,7 @@ static void addLoopPreInits(ASTContext &Context,
                                               RangeEnd->getBeginLoc(),
                                               RangeEnd->getEndLoc()));
   }
-
   llvm::append_range(PreInits, OriginalInit);
-
   // List of OMPCapturedExprDecl, for __begin, __end, and NumIterations
   if (auto *PI = cast_or_null<DeclStmt>(LoopHelper.PreInits)) {
     PreInits.push_back(new (Context) DeclStmt(
@@ -15214,7 +15315,7 @@ StmtResult SemaOpenMP::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   Stmt *LoopStmt = nullptr;
   collectLoopStmts(AStmt, {LoopStmt});
 
-  // Determine the PreInit declarations.
+  // Determine the PreInit declarations.e
   SmallVector<Stmt *, 4> PreInits;
   addLoopPreInits(Context, LoopHelper, LoopStmt, OriginalInits[0], PreInits);
 
@@ -15781,28 +15882,35 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   if (!AStmt) {
     return StmtError();
   }
+
+  unsigned NumLoops = 1;
+  unsigned LoopSeqSize = 1;
+
+  // Defer transformation in dependent contexts
+  // The NumLoopNests argument is set to a placeholder 1 (even though
+  // using looprange fuse could yield up to 3 top level loop nests)
+  // because a dependent context could prevent determining its true value
+  if (CurrContext->isDependentContext()) {
+    return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                    NumLoops, LoopSeqSize, AStmt, nullptr,
+                                    nullptr);
+  }
+
   // Validate that the potential loop sequence is transformable for fusion
   // Also collect the HelperExprs, Loop Stmts, Inits, and Number of loops
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers;
   SmallVector<Stmt *> LoopStmts;
   SmallVector<SmallVector<Stmt *, 0>> OriginalInits;
-
-  unsigned NumLoops;
-  unsigned LoopSeqSize;
+  SmallVector<SmallVector<Stmt *, 0>> TransformsPreInits;
+  SmallVector<SmallVector<Stmt *, 0>> LoopSequencePreInits;
+  SmallVector<OMPLoopCategory, 0> LoopCategories;
   if (!checkTransformableLoopSequence(OMPD_fuse, AStmt, LoopSeqSize, NumLoops,
                                       LoopHelpers, LoopStmts, OriginalInits,
-                                      Context)) {
+                                      TransformsPreInits, LoopSequencePreInits,
+                                      LoopCategories, Context)) {
     return StmtError();
   }
 
-  // Defer transformation in dependent contexts
-  // The NumLoopNests argument is set to a placeholder (0)
-  // because a dependent context could prevent determining its true value
-  if (CurrContext->isDependentContext()) {
-    return OMPFuseDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                    NumLoops, 0, AStmt, nullptr, nullptr);
-  }
-
   // Handle clauses, which can be any of the following: [looprange, apply]
   const OMPLoopRangeClause *LRC =
       OMPExecutableDirective::getSingleClause<OMPLoopRangeClause>(Clauses);
@@ -15864,11 +15972,6 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
          "Expecting loop iteration space dimensionality to match number of "
          "affected loops");
 
-  // PreInits hold a sequence of variable declarations that must be executed
-  // before the fused loop begins. These include bounds, strides, and other
-  // helper variables required for the transformation.
-  SmallVector<Stmt *> PreInits;
-
   // Select the type with the largest bit width among all induction variables
   QualType IVType = LoopHelpers[FirstVal - 1].IterationVarRef->getType();
   for (unsigned int I = FirstVal; I < LastVal; ++I) {
@@ -15880,7 +15983,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   uint64_t IVBitWidth = Context.getIntWidth(IVType);
 
   // Create pre-init declarations for all loops lower bounds, upper bounds,
-  // strides and num-iterations
+  // strides and num-iterations for every top level loop in the fusion
   SmallVector<VarDecl *, 4> LBVarDecls;
   SmallVector<VarDecl *, 4> STVarDecls;
   SmallVector<VarDecl *, 4> NIVarDecls;
@@ -15918,12 +16021,62 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
         return std::make_pair(VD, DeclStmt);
       };
 
+  // PreInits hold a sequence of variable declarations that must be executed
+  // before the fused loop begins. These include bounds, strides, and other
+  // helper variables required for the transformation. Other loop transforms
+  // also contain their own preinits
+  SmallVector<Stmt *> PreInits;
+  // Iterator to keep track of loop transformations
+  unsigned int TransformIndex = 0;
+
+  //  Update the general preinits using the preinits generated by loop sequence
+  //  generating loop transformations. These preinits differ slightly from
+  //  single-loop transformation preinits, as they can be detached from a
+  //  specific loop inside the multiple generated loop nests. This happens
+  //  because certain helper variables, like '.omp.fuse.max', are introduced to
+  //  handle fused iteration spaces and may not be directly tied to a single
+  //  original loop. the preinit structure must ensure that hidden variables
+  //  like '.omp.fuse.max' are still properly handled.
+  // Transformations that apply this concept: Loopranged Fuse, Split
+  if (!LoopSequencePreInits.empty()) {
+    for (const auto &LTPreInits : LoopSequencePreInits) {
+      if (!LTPreInits.empty()) {
+        llvm::append_range(PreInits, LTPreInits);
+      }
+    }
+  }
+
   // Process each single loop to generate and collect declarations
-  // and statements for all helper expressions
+  // and statements for all helper expressions related to
+  // particular single loop nests
+
+  // Also In the case of the fused loops, we keep track of their original
+  // inits by appending them to their preinits statement, and in the case of
+  // transformations, also append their preinits (which contain the original
+  // loop initialization statement or other statements)
+
+  // Firstly we need to update TransformIndex to match the begining of the
+  // looprange section
+  for (unsigned int I = 0; I < FirstVal - 1; ++I) {
+    if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop)
+      ++TransformIndex;
+  }
   for (unsigned int I = FirstVal - 1, J = 0; I < LastVal; ++I, ++J) {
-    addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
-                    PreInits);
 
+    if (LoopCategories[I] == OMPLoopCategory::RegularLoop) {
+      addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
+                      PreInits);
+    } else if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop) {
+      // For transformed loops, insert both pre-inits and original inits.
+      // Order matters: pre-inits may define variables used in the original
+      // inits such as upper bounds...
+      auto TransformPreInit = TransformsPreInits[TransformIndex++];
+      if (!TransformPreInit.empty()) {
+        llvm::append_range(PreInits, TransformPreInit);
+      }
+      addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
+                      PreInits);
+    }
     auto [UBVD, UBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].UB, "ub", J);
     auto [LBVD, LBDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].LB, "lb", J);
     auto [STVD, STDStmt] = CreateHelperVarAndStmt(LoopHelpers[I].ST, "st", J);
@@ -15942,7 +16095,6 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     NIVarDecls.push_back(NIVD);
     IVVarDecls.push_back(IVVD);
 
-    PreInits.push_back(UBDStmt.get());
     PreInits.push_back(LBDStmt.get());
     PreInits.push_back(STDStmt.get());
     PreInits.push_back(NIDStmt.get());
@@ -16118,6 +16270,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     BodyStmts.push_back(IdxExpr.get());
     llvm::append_range(BodyStmts, LoopHelpers[I].Updates);
 
+    // If the loop is a CXXForRangeStmt then the iterator variable is needed
     if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
       BodyStmts.push_back(SourceCXXFor->getLoopVarStmt());
 
@@ -16152,21 +16305,50 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
               FusedBody, InitStmt.get()->getBeginLoc(), SourceLocation(),
               IncrExpr.get()->getEndLoc());
 
-  // In the case of looprange, the result of fuse won't simply
-  // be a single loop (ForStmt), but rather a loop sequence
-  // (CompoundStmt) of 3 parts: the pre-fusion loops, the fused loop
-  // and the post-fusion loops, preserving its original order.
+  //  In the case of looprange, the result of fuse won't simply
+  //  be a single loop (ForStmt), but rather a loop sequence
+  //  (CompoundStmt) of 3 parts: the pre-fusion loops, the fused loop
+  //  and the post-fusion loops, preserving its original order.
+  //
+  //  Note: If looprange clause produces a single fused loop nest then
+  //  this compound statement wrapper is unnecessary (Therefore this
+  //  treatment is skipped)
+
   Stmt *FusionStmt = FusedForStmt;
-  if (LRC) {
+  if (LRC && CountVal != LoopSeqSize) {
     SmallVector<Stmt *, 4> FinalLoops;
-    // Gather all the pre-fusion loops
-    for (unsigned I = 0; I < FirstVal - 1; ++I)
-      FinalLoops.push_back(LoopStmts[I]);
-    // Gather the fused loop
-    FinalLoops.push_back(FusedForStmt);
-    // Gather all the post-fusion loops
-    for (unsigned I = FirstVal + CountVal - 1; I < LoopSeqSize; ++I)
+    // Reset the transform index
+    TransformIndex = 0;
+
+    // Collect all non-fused loops before and after the fused region.
+    // Pre-fusion and post-fusion loops are inserted in order exploiting their
+    // symmetry, along with their corresponding transformation pre-inits if
+    // needed. The fused loop is added between the two regions.
+    for (unsigned I = 0; I < LoopSeqSize; ++I) {
+      if (I >= FirstVal - 1 && I < FirstVal + CountVal - 1) {
+        // Update the Transformation counter to skip already treated
+        // loop transformations
+        if (LoopCategories[I] != OMPLoopCategory::TransformSingleLoop)
+          ++TransformIndex;
+        continue;
+      }
+
+      // No need to handle:
+      // Regular loops: they are kept intact as-is.
+      // Loop-sequence-generating transformations: already handled earlier.
+      // Only TransformSingleLoop requires inserting pre-inits here
+
+      if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop) {
+        auto TransformPreInit = TransformsPreInits[TransformIndex++];
+        if (!TransformPreInit.empty()) {
+          llvm::append_range(PreInits, TransformPreInit);
+        }
+      }
+
       FinalLoops.push_back(LoopStmts[I]);
+    }
+
+    FinalLoops.insert(FinalLoops.begin() + (FirstVal - 1), FusedForStmt);
     FusionStmt = CompoundStmt::Create(Context, FinalLoops, FPOptionsOverride(),
                                       SourceLocation(), SourceLocation());
   }
diff --git a/clang/test/OpenMP/fuse_ast_print.cpp b/clang/test/OpenMP/fuse_ast_print.cpp
index ac4f0d38a9c68..9d85bd1172948 100644
--- a/clang/test/OpenMP/fuse_ast_print.cpp
+++ b/clang/test/OpenMP/fuse_ast_print.cpp
@@ -338,6 +338,61 @@ void tfoo9() {
     foo9<1, 2>();
 }
 
+// PRINT-LABEL: void foo10(
+// DUMP-LABEL: FunctionDecl {{.*}} foo10
+void foo10() {
+    // PRINT: #pragma omp fuse looprange(2,2)
+    // DUMP:  OMPFuseDirective
+    // DUMP: OMPLooprangeClause
+    #pragma omp fuse looprange(2,2)
+    // PRINT: {
+    // DUMP: CompoundStmt       
+    {
+        // PRINT: for (int i = 0; i < 10; i += 2)
+        // DUMP: ForStmt
+        for (int i = 0; i < 10; i += 2)
+            // PRINT: body(i)
+            // DUMP: CallExpr
+            body(i);
+        // PRINT: for (int ii = 0; ii < 10; ii += 2)
+        // DUMP: ForStmt
+        for (int ii = 0; ii < 10; ii += 2)
+            // PRINT: body(ii)
+            // DUMP: CallExpr
+            body(ii);
+        // PRINT: #pragma omp fuse looprange(2,2)
+        // DUMP:  OMPFuseDirective
+        // DUMP: OMPLooprangeClause
+        #pragma omp fuse looprange(2,2)
+        {
+            // PRINT: for (int j = 10; j > 0; --j)
+            // DUMP: ForStmt
+            for (int j = 10; j > 0; --j)
+                // PRINT: body(j)
+                // DUMP: CallExpr
+                body(j);
+            // PRINT: for (int jj = 10; jj > 0; --jj)
+            // DUMP: ForStmt
+            for (int jj = 10; jj > 0; --jj)
+                // PRINT: body(jj)
+                // DUMP: CallExpr
+                body(jj);
+            // PRINT: for (int k = 0; k <= 10; ++k)
+            // DUMP: ForStmt
+            for (int k = 0; k <= 10; ++k)
+                // PRINT: body(k)
+                // DUMP: CallExpr
+                body(k);
+            // PRINT: for (int kk = 0; kk <= 10; ++kk)
+            // DUMP: ForStmt
+            for (int kk = 0; kk <= 10; ++kk)
+                // PRINT: body(kk)
+                // DUMP: CallExpr
+                body(kk);
+        }
+    }
+
+}
 
 
 
diff --git a/clang/test/OpenMP/fuse_codegen.cpp b/clang/test/OpenMP/fuse_codegen.cpp
index d9500bed3ce31..742c280ed0172 100644
--- a/clang/test/OpenMP/fuse_codegen.cpp
+++ b/clang/test/OpenMP/fuse_codegen.cpp
@@ -65,6 +65,23 @@ extern "C" void foo4() {
     }
 }
 
+// This exemplifies the usage of loop transformations that generate
+// more than top level canonical loop nests (e.g split, loopranged fuse...)
+extern "C" void foo5() {
+    double arr[256];
+    #pragma omp fuse looprange(2,2)
+    {
+        #pragma omp fuse looprange(2,2)
+        {
+            for(int i = 0; i < 128; ++i) body(i);
+            for(int j = 0; j < 256; j+=2) body(j);
+            for(int k = 0; k < 512; ++k) body(k);
+        }
+        for(int c = 42; auto &&v: arr) body(c,v);
+        for(int cc = 37; auto &&vv: arr) body(cc, vv);
+    }
+}
+
 
 #endif
 // CHECK1-LABEL: define dso_local void @body(
@@ -88,7 +105,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
@@ -97,7 +113,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -129,107 +144,103 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP8]], 1
 // CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP9]], ptr [[J]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[START2_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START2_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[END2_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP13]], [[TMP14]]
 // CHECK1-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP15]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP16]]
 // CHECK1-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
 // CHECK1-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    store i32 [[TMP20]], ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP21]], [[TMP22]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP19]], [[TMP20]]
 // CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // CHECK1:       [[COND_TRUE]]:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
 // CHECK1-NEXT:    br label %[[COND_END:.*]]
 // CHECK1:       [[COND_FALSE]]:
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
 // CHECK1-NEXT:    br label %[[COND_END]]
 // CHECK1:       [[COND_END]]:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP23]], %[[COND_TRUE]] ], [ [[TMP24]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP21]], %[[COND_TRUE]] ], [ [[TMP22]], %[[COND_FALSE]] ]
 // CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK1-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK1:       [[FOR_COND]]:
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
-// CHECK1-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP23]], [[TMP24]]
 // CHECK1-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK1:       [[FOR_BODY]]:
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
 // CHECK1-NEXT:    br i1 [[CMP17]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // CHECK1:       [[IF_THEN]]:
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP30]], [[TMP31]]
-// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP29]], [[MUL]]
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP28]], [[TMP29]]
+// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP27]], [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP33]], [[TMP34]]
-// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP32]], [[MUL19]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP31]], [[TMP32]]
+// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP30]], [[MUL19]]
 // CHECK1-NEXT:    store i32 [[ADD20]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP35]])
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP33]])
 // CHECK1-NEXT:    br label %[[IF_END]]
 // CHECK1:       [[IF_END]]:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP34]], [[TMP35]]
 // CHECK1-NEXT:    br i1 [[CMP21]], label %[[IF_THEN22:.*]], label %[[IF_END27:.*]]
 // CHECK1:       [[IF_THEN22]]:
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP39]], [[TMP40]]
-// CHECK1-NEXT:    [[ADD24:%.*]] = add i32 [[TMP38]], [[MUL23]]
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// CHECK1-NEXT:    [[ADD24:%.*]] = add i32 [[TMP36]], [[MUL23]]
 // CHECK1-NEXT:    store i32 [[ADD24]], ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP42]], [[TMP43]]
-// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[TMP41]], [[MUL25]]
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP40]], [[TMP41]]
+// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[TMP39]], [[MUL25]]
 // CHECK1-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP42]])
 // CHECK1-NEXT:    br label %[[IF_END27]]
 // CHECK1:       [[IF_END27]]:
 // CHECK1-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK1:       [[FOR_INC]]:
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP45]], 1
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
 // CHECK1:       [[FOR_END]]:
@@ -256,7 +267,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
@@ -265,7 +275,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -274,7 +283,6 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP21:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i32, align 4
@@ -304,172 +312,166 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP8]], 1
 // CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP9]], ptr [[J]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP12]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK1-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP13]], [[TMP14]]
 // CHECK1-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP15]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK1-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP16]]
 // CHECK1-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
 // CHECK1-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
 // CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
+// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
 // CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
 // CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTCAPTURE_EXPR_19]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[DOTNEW_STEP21]], align 4
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK1-NEXT:    store i32 [[TMP24]], ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP25]], [[TMP26]]
 // CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP29]]
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP30]]
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP27]]
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP28]]
 // CHECK1-NEXT:    [[SUB27:%.*]] = sub i32 [[DIV26]], 1
 // CHECK1-NEXT:    store i32 [[SUB27]], ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK1-NEXT:    store i32 [[TMP31]], ptr [[DOTOMP_UB2]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB2]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST2]], align 4
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK1-NEXT:    [[ADD28:%.*]] = add i32 [[TMP32]], 1
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK1-NEXT:    [[ADD28:%.*]] = add i32 [[TMP29]], 1
 // CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_NI2]], align 4
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    store i32 [[TMP33]], ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP34]], [[TMP35]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP30]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP31]], [[TMP32]]
 // CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // CHECK1:       [[COND_TRUE]]:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
 // CHECK1-NEXT:    br label %[[COND_END:.*]]
 // CHECK1:       [[COND_FALSE]]:
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
 // CHECK1-NEXT:    br label %[[COND_END]]
 // CHECK1:       [[COND_END]]:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP36]], %[[COND_TRUE]] ], [ [[TMP37]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP33]], %[[COND_TRUE]] ], [ [[TMP34]], %[[COND_FALSE]] ]
 // CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_TEMP_2]], align 4
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
-// CHECK1-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP35]], [[TMP36]]
 // CHECK1-NEXT:    br i1 [[CMP29]], label %[[COND_TRUE30:.*]], label %[[COND_FALSE31:.*]]
 // CHECK1:       [[COND_TRUE30]]:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
 // CHECK1-NEXT:    br label %[[COND_END32:.*]]
 // CHECK1:       [[COND_FALSE31]]:
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
 // CHECK1-NEXT:    br label %[[COND_END32]]
 // CHECK1:       [[COND_END32]]:
-// CHECK1-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP40]], %[[COND_TRUE30]] ], [ [[TMP41]], %[[COND_FALSE31]] ]
+// CHECK1-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP37]], %[[COND_TRUE30]] ], [ [[TMP38]], %[[COND_FALSE31]] ]
 // CHECK1-NEXT:    store i32 [[COND33]], ptr [[DOTOMP_FUSE_MAX]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK1-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK1:       [[FOR_COND]]:
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
-// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP39]], [[TMP40]]
 // CHECK1-NEXT:    br i1 [[CMP34]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK1:       [[FOR_BODY]]:
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP44]], [[TMP45]]
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP41]], [[TMP42]]
 // CHECK1-NEXT:    br i1 [[CMP35]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // CHECK1:       [[IF_THEN]]:
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP47]], [[TMP48]]
-// CHECK1-NEXT:    [[ADD36:%.*]] = add i32 [[TMP46]], [[MUL]]
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP44]], [[TMP45]]
+// CHECK1-NEXT:    [[ADD36:%.*]] = add i32 [[TMP43]], [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD36]], ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP50]], [[TMP51]]
-// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[TMP49]], [[MUL37]]
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP47]], [[TMP48]]
+// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[TMP46]], [[MUL37]]
 // CHECK1-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP49]])
 // CHECK1-NEXT:    br label %[[IF_END]]
 // CHECK1:       [[IF_END]]:
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP53]], [[TMP54]]
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP50]], [[TMP51]]
 // CHECK1-NEXT:    br i1 [[CMP39]], label %[[IF_THEN40:.*]], label %[[IF_END45:.*]]
 // CHECK1:       [[IF_THEN40]]:
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP56]], [[TMP57]]
-// CHECK1-NEXT:    [[ADD42:%.*]] = add i32 [[TMP55]], [[MUL41]]
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP53]], [[TMP54]]
+// CHECK1-NEXT:    [[ADD42:%.*]] = add i32 [[TMP52]], [[MUL41]]
 // CHECK1-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK1-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP59]], [[TMP60]]
-// CHECK1-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP58]], [[MUL43]]
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK1-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP55]], [[MUL43]]
 // CHECK1-NEXT:    store i32 [[SUB44]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP61]])
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP58]])
 // CHECK1-NEXT:    br label %[[IF_END45]]
 // CHECK1:       [[IF_END45]]:
-// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
-// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP62]], [[TMP63]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP59]], [[TMP60]]
 // CHECK1-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
 // CHECK1:       [[IF_THEN47]]:
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP65]], [[TMP66]]
-// CHECK1-NEXT:    [[ADD49:%.*]] = add i32 [[TMP64]], [[MUL48]]
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP62]], [[TMP63]]
+// CHECK1-NEXT:    [[ADD49:%.*]] = add i32 [[TMP61]], [[MUL48]]
 // CHECK1-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV2]], align 4
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP68]], [[TMP69]]
-// CHECK1-NEXT:    [[ADD51:%.*]] = add i32 [[TMP67]], [[MUL50]]
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP65]], [[TMP66]]
+// CHECK1-NEXT:    [[ADD51:%.*]] = add i32 [[TMP64]], [[MUL50]]
 // CHECK1-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP70]])
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP67]])
 // CHECK1-NEXT:    br label %[[IF_END52]]
 // CHECK1:       [[IF_END52]]:
 // CHECK1-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK1:       [[FOR_INC]]:
-// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP68]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
 // CHECK1:       [[FOR_END]]:
@@ -481,13 +483,11 @@ extern "C" void foo4() {
 // CHECK1-NEXT:  [[ENTRY:.*:]]
 // CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -497,48 +497,43 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB03:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_LB04:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_ST05:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_NI06:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_IV07:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB03:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST04:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI05:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV06:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[C:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_UB117:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_LB118:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_ST119:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_NI120:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_IV122:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_10:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB116:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_ST117:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_NI118:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV120:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[CC:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[__RANGE223:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[__END224:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[__BEGIN227:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__RANGE221:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END222:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN225:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_27:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_29:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_31:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_32:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_30:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_TEMP_142:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_TEMP_140:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX48:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX54:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX46:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX52:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[VV:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
 // CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
-// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB1]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
 // CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI1]], align 4
@@ -565,225 +560,219 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
 // CHECK1-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
 // CHECK1-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB03]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST04]], align 4
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTOMP_UB03]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB04]], align 4
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST05]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], 1
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
 // CHECK1-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
-// CHECK1-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI06]], align 8
+// CHECK1-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI05]], align 8
 // CHECK1-NEXT:    store i32 42, ptr [[C]], align 4
 // CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP8]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY7:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY7]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY8:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYDECAY8]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY10:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP11]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYDECAY10]], ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__END2]], align 8
-// CHECK1-NEXT:    store ptr [[TMP12]], ptr [[DOTCAPTURE_EXPR_11]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_11]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
-// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP14]] to i64
+// CHECK1-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY9]], ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP11]], ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
 // CHECK1-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
 // CHECK1-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
-// CHECK1-NEXT:    [[SUB13:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
-// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i64 [[SUB13]], 1
-// CHECK1-NEXT:    [[DIV15:%.*]] = sdiv i64 [[ADD14]], 1
-// CHECK1-NEXT:    [[SUB16:%.*]] = sub nsw i64 [[DIV15]], 1
-// CHECK1-NEXT:    store i64 [[SUB16]], ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK1-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_UB117]], align 8
-// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB118]], align 8
-// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST119]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK1-NEXT:    store i64 [[ADD21]], ptr [[DOTOMP_NI120]], align 8
+// CHECK1-NEXT:    [[SUB12:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[SUB12]], 1
+// CHECK1-NEXT:    [[DIV14:%.*]] = sdiv i64 [[ADD13]], 1
+// CHECK1-NEXT:    [[SUB15:%.*]] = sub nsw i64 [[DIV14]], 1
+// CHECK1-NEXT:    store i64 [[SUB15]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB116]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST117]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK1-NEXT:    store i64 [[ADD19]], ptr [[DOTOMP_NI118]], align 8
 // CHECK1-NEXT:    store i32 37, ptr [[CC]], align 4
-// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE223]], align 8
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY25:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
-// CHECK1-NEXT:    [[ADD_PTR26:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY25]], i64 256
-// CHECK1-NEXT:    store ptr [[ADD_PTR26]], ptr [[__END224]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP18]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[__BEGIN227]], align 8
-// CHECK1-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK1-NEXT:    [[ARRAYDECAY30:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP19]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYDECAY30]], ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK1-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[__END224]], align 8
-// CHECK1-NEXT:    store ptr [[TMP20]], ptr [[DOTCAPTURE_EXPR_31]], align 8
-// CHECK1-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_31]], align 8
-// CHECK1-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST33:%.*]] = ptrtoint ptr [[TMP21]] to i64
-// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST34:%.*]] = ptrtoint ptr [[TMP22]] to i64
-// CHECK1-NEXT:    [[SUB_PTR_SUB35:%.*]] = sub i64 [[SUB_PTR_LHS_CAST33]], [[SUB_PTR_RHS_CAST34]]
-// CHECK1-NEXT:    [[SUB_PTR_DIV36:%.*]] = sdiv exact i64 [[SUB_PTR_SUB35]], 8
-// CHECK1-NEXT:    [[SUB37:%.*]] = sub nsw i64 [[SUB_PTR_DIV36]], 1
-// CHECK1-NEXT:    [[ADD38:%.*]] = add nsw i64 [[SUB37]], 1
-// CHECK1-NEXT:    [[DIV39:%.*]] = sdiv i64 [[ADD38]], 1
-// CHECK1-NEXT:    [[SUB40:%.*]] = sub nsw i64 [[DIV39]], 1
-// CHECK1-NEXT:    store i64 [[SUB40]], ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK1-NEXT:    store i64 [[TMP23]], ptr [[DOTOMP_UB2]], align 8
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE221]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY23:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP15]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR24:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY23]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR24]], ptr [[__END222]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY26:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP16]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY26]], ptr [[__BEGIN225]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__END222]], align 8
+// CHECK1-NEXT:    store ptr [[TMP18]], ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST31:%.*]] = ptrtoint ptr [[TMP19]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST32:%.*]] = ptrtoint ptr [[TMP20]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB33:%.*]] = sub i64 [[SUB_PTR_LHS_CAST31]], [[SUB_PTR_RHS_CAST32]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV34:%.*]] = sdiv exact i64 [[SUB_PTR_SUB33]], 8
+// CHECK1-NEXT:    [[SUB35:%.*]] = sub nsw i64 [[SUB_PTR_DIV34]], 1
+// CHECK1-NEXT:    [[ADD36:%.*]] = add nsw i64 [[SUB35]], 1
+// CHECK1-NEXT:    [[DIV37:%.*]] = sdiv i64 [[ADD36]], 1
+// CHECK1-NEXT:    [[SUB38:%.*]] = sub nsw i64 [[DIV37]], 1
+// CHECK1-NEXT:    store i64 [[SUB38]], ptr [[DOTCAPTURE_EXPR_30]], align 8
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB2]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST2]], align 8
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK1-NEXT:    [[ADD41:%.*]] = add nsw i64 [[TMP24]], 1
-// CHECK1-NEXT:    store i64 [[ADD41]], ptr [[DOTOMP_NI2]], align 8
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
-// CHECK1-NEXT:    store i64 [[TMP25]], ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK1-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP26]], [[TMP27]]
-// CHECK1-NEXT:    br i1 [[CMP43]], label %[[COND_TRUE44:.*]], label %[[COND_FALSE45:.*]]
-// CHECK1:       [[COND_TRUE44]]:
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK1-NEXT:    br label %[[COND_END46:.*]]
-// CHECK1:       [[COND_FALSE45]]:
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK1-NEXT:    br label %[[COND_END46]]
-// CHECK1:       [[COND_END46]]:
-// CHECK1-NEXT:    [[COND47:%.*]] = phi i64 [ [[TMP28]], %[[COND_TRUE44]] ], [ [[TMP29]], %[[COND_FALSE45]] ]
-// CHECK1-NEXT:    store i64 [[COND47]], ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK1-NEXT:    [[CMP49:%.*]] = icmp sgt i64 [[TMP30]], [[TMP31]]
-// CHECK1-NEXT:    br i1 [[CMP49]], label %[[COND_TRUE50:.*]], label %[[COND_FALSE51:.*]]
-// CHECK1:       [[COND_TRUE50]]:
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK1-NEXT:    br label %[[COND_END52:.*]]
-// CHECK1:       [[COND_FALSE51]]:
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK1-NEXT:    br label %[[COND_END52]]
-// CHECK1:       [[COND_END52]]:
-// CHECK1-NEXT:    [[COND53:%.*]] = phi i64 [ [[TMP32]], %[[COND_TRUE50]] ], [ [[TMP33]], %[[COND_FALSE51]] ]
-// CHECK1-NEXT:    store i64 [[COND53]], ptr [[DOTOMP_FUSE_MAX48]], align 8
-// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_30]], align 8
+// CHECK1-NEXT:    [[ADD39:%.*]] = add nsw i64 [[TMP21]], 1
+// CHECK1-NEXT:    store i64 [[ADD39]], ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    store i64 [[TMP22]], ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[CMP41:%.*]] = icmp sgt i64 [[TMP23]], [[TMP24]]
+// CHECK1-NEXT:    br i1 [[CMP41]], label %[[COND_TRUE42:.*]], label %[[COND_FALSE43:.*]]
+// CHECK1:       [[COND_TRUE42]]:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK1-NEXT:    br label %[[COND_END44:.*]]
+// CHECK1:       [[COND_FALSE43]]:
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    br label %[[COND_END44]]
+// CHECK1:       [[COND_END44]]:
+// CHECK1-NEXT:    [[COND45:%.*]] = phi i64 [ [[TMP25]], %[[COND_TRUE42]] ], [ [[TMP26]], %[[COND_FALSE43]] ]
+// CHECK1-NEXT:    store i64 [[COND45]], ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[CMP47:%.*]] = icmp sgt i64 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP47]], label %[[COND_TRUE48:.*]], label %[[COND_FALSE49:.*]]
+// CHECK1:       [[COND_TRUE48]]:
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK1-NEXT:    br label %[[COND_END50:.*]]
+// CHECK1:       [[COND_FALSE49]]:
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    br label %[[COND_END50]]
+// CHECK1:       [[COND_END50]]:
+// CHECK1-NEXT:    [[COND51:%.*]] = phi i64 [ [[TMP29]], %[[COND_TRUE48]] ], [ [[TMP30]], %[[COND_FALSE49]] ]
+// CHECK1-NEXT:    store i64 [[COND51]], ptr [[DOTOMP_FUSE_MAX46]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX52]], align 8
 // CHECK1-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK1:       [[FOR_COND]]:
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX48]], align 8
-// CHECK1-NEXT:    [[CMP55:%.*]] = icmp slt i64 [[TMP34]], [[TMP35]]
-// CHECK1-NEXT:    br i1 [[CMP55]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX46]], align 8
+// CHECK1-NEXT:    [[CMP53:%.*]] = icmp slt i64 [[TMP31]], [[TMP32]]
+// CHECK1-NEXT:    br i1 [[CMP53]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK1:       [[FOR_BODY]]:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
-// CHECK1-NEXT:    [[CMP56:%.*]] = icmp slt i64 [[TMP36]], [[TMP37]]
-// CHECK1-NEXT:    br i1 [[CMP56]], label %[[IF_THEN:.*]], label %[[IF_END76:.*]]
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    [[CMP54:%.*]] = icmp slt i64 [[TMP33]], [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP54]], label %[[IF_THEN:.*]], label %[[IF_END74:.*]]
 // CHECK1:       [[IF_THEN]]:
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB04]], align 4
-// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP38]] to i64
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST05]], align 4
-// CHECK1-NEXT:    [[CONV58:%.*]] = sext i32 [[TMP39]] to i64
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV58]], [[TMP40]]
-// CHECK1-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV57]], [[MUL]]
-// CHECK1-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
-// CHECK1-NEXT:    store i32 [[CONV60]], ptr [[DOTOMP_IV07]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_IV07]], align 4
-// CHECK1-NEXT:    [[MUL61:%.*]] = mul nsw i32 [[TMP41]], 1
-// CHECK1-NEXT:    [[ADD62:%.*]] = add nsw i32 0, [[MUL61]]
-// CHECK1-NEXT:    store i32 [[ADD62]], ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK1-NEXT:    [[CMP63:%.*]] = icmp slt i32 [[TMP42]], [[TMP43]]
-// CHECK1-NEXT:    br i1 [[CMP63]], label %[[IF_THEN64:.*]], label %[[IF_END:.*]]
-// CHECK1:       [[IF_THEN64]]:
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP45]], [[TMP46]]
-// CHECK1-NEXT:    [[ADD66:%.*]] = add nsw i32 [[TMP44]], [[MUL65]]
-// CHECK1-NEXT:    store i32 [[ADD66]], ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK1-NEXT:    [[MUL67:%.*]] = mul nsw i32 [[TMP47]], 1
-// CHECK1-NEXT:    [[ADD68:%.*]] = add nsw i32 0, [[MUL67]]
-// CHECK1-NEXT:    store i32 [[ADD68]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP48]])
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_LB03]], align 4
+// CHECK1-NEXT:    [[CONV55:%.*]] = sext i32 [[TMP35]] to i64
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_ST04]], align 4
+// CHECK1-NEXT:    [[CONV56:%.*]] = sext i32 [[TMP36]] to i64
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV56]], [[TMP37]]
+// CHECK1-NEXT:    [[ADD57:%.*]] = add nsw i64 [[CONV55]], [[MUL]]
+// CHECK1-NEXT:    [[CONV58:%.*]] = trunc i64 [[ADD57]] to i32
+// CHECK1-NEXT:    store i32 [[CONV58]], ptr [[DOTOMP_IV06]], align 4
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV06]], align 4
+// CHECK1-NEXT:    [[MUL59:%.*]] = mul nsw i32 [[TMP38]], 1
+// CHECK1-NEXT:    [[ADD60:%.*]] = add nsw i32 0, [[MUL59]]
+// CHECK1-NEXT:    store i32 [[ADD60]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP61:%.*]] = icmp slt i32 [[TMP39]], [[TMP40]]
+// CHECK1-NEXT:    br i1 [[CMP61]], label %[[IF_THEN62:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN62]]:
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL63:%.*]] = mul nsw i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[ADD64:%.*]] = add nsw i32 [[TMP41]], [[MUL63]]
+// CHECK1-NEXT:    store i32 [[ADD64]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP44]], 1
+// CHECK1-NEXT:    [[ADD66:%.*]] = add nsw i32 0, [[MUL65]]
+// CHECK1-NEXT:    store i32 [[ADD66]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP45]])
 // CHECK1-NEXT:    br label %[[IF_END]]
 // CHECK1:       [[IF_END]]:
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK1-NEXT:    [[CMP69:%.*]] = icmp slt i32 [[TMP49]], [[TMP50]]
-// CHECK1-NEXT:    br i1 [[CMP69]], label %[[IF_THEN70:.*]], label %[[IF_END75:.*]]
-// CHECK1:       [[IF_THEN70]]:
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK1-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP52]], [[TMP53]]
-// CHECK1-NEXT:    [[ADD72:%.*]] = add nsw i32 [[TMP51]], [[MUL71]]
-// CHECK1-NEXT:    store i32 [[ADD72]], ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK1-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP54]], 2
-// CHECK1-NEXT:    [[ADD74:%.*]] = add nsw i32 0, [[MUL73]]
-// CHECK1-NEXT:    store i32 [[ADD74]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP55]])
-// CHECK1-NEXT:    br label %[[IF_END75]]
-// CHECK1:       [[IF_END75]]:
-// CHECK1-NEXT:    br label %[[IF_END76]]
-// CHECK1:       [[IF_END76]]:
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK1-NEXT:    [[CMP77:%.*]] = icmp slt i64 [[TMP56]], [[TMP57]]
-// CHECK1-NEXT:    br i1 [[CMP77]], label %[[IF_THEN78:.*]], label %[[IF_END83:.*]]
-// CHECK1:       [[IF_THEN78]]:
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_LB118]], align 8
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_ST119]], align 8
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], [[TMP60]]
-// CHECK1-NEXT:    [[ADD80:%.*]] = add nsw i64 [[TMP58]], [[MUL79]]
-// CHECK1-NEXT:    store i64 [[ADD80]], ptr [[DOTOMP_IV122]], align 8
-// CHECK1-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV122]], align 8
-// CHECK1-NEXT:    [[MUL81:%.*]] = mul nsw i64 [[TMP62]], 1
-// CHECK1-NEXT:    [[ADD_PTR82:%.*]] = getelementptr inbounds double, ptr [[TMP61]], i64 [[MUL81]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR82]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    store ptr [[TMP63]], ptr [[V]], align 8
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[C]], align 4
-// CHECK1-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[V]], align 8
-// CHECK1-NEXT:    [[TMP66:%.*]] = load double, ptr [[TMP65]], align 8
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP64]], double noundef [[TMP66]])
-// CHECK1-NEXT:    br label %[[IF_END83]]
-// CHECK1:       [[IF_END83]]:
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK1-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP67]], [[TMP68]]
-// CHECK1-NEXT:    br i1 [[CMP84]], label %[[IF_THEN85:.*]], label %[[IF_END90:.*]]
-// CHECK1:       [[IF_THEN85]]:
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
-// CHECK1-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
-// CHECK1-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[ADD87:%.*]] = add nsw i64 [[TMP69]], [[MUL86]]
-// CHECK1-NEXT:    store i64 [[ADD87]], ptr [[DOTOMP_IV2]], align 8
-// CHECK1-NEXT:    [[TMP72:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK1-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
-// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[TMP73]], 1
-// CHECK1-NEXT:    [[ADD_PTR89:%.*]] = getelementptr inbounds double, ptr [[TMP72]], i64 [[MUL88]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR89]], ptr [[__BEGIN227]], align 8
-// CHECK1-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[__BEGIN227]], align 8
-// CHECK1-NEXT:    store ptr [[TMP74]], ptr [[VV]], align 8
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[CC]], align 4
-// CHECK1-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[VV]], align 8
-// CHECK1-NEXT:    [[TMP77:%.*]] = load double, ptr [[TMP76]], align 8
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP75]], double noundef [[TMP77]])
-// CHECK1-NEXT:    br label %[[IF_END90]]
-// CHECK1:       [[IF_END90]]:
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP67:%.*]] = icmp slt i32 [[TMP46]], [[TMP47]]
+// CHECK1-NEXT:    br i1 [[CMP67]], label %[[IF_THEN68:.*]], label %[[IF_END73:.*]]
+// CHECK1:       [[IF_THEN68]]:
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL69:%.*]] = mul nsw i32 [[TMP49]], [[TMP50]]
+// CHECK1-NEXT:    [[ADD70:%.*]] = add nsw i32 [[TMP48]], [[MUL69]]
+// CHECK1-NEXT:    store i32 [[ADD70]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP51]], 2
+// CHECK1-NEXT:    [[ADD72:%.*]] = add nsw i32 0, [[MUL71]]
+// CHECK1-NEXT:    store i32 [[ADD72]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK1-NEXT:    br label %[[IF_END73]]
+// CHECK1:       [[IF_END73]]:
+// CHECK1-NEXT:    br label %[[IF_END74]]
+// CHECK1:       [[IF_END74]]:
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[CMP75:%.*]] = icmp slt i64 [[TMP53]], [[TMP54]]
+// CHECK1-NEXT:    br i1 [[CMP75]], label %[[IF_THEN76:.*]], label %[[IF_END81:.*]]
+// CHECK1:       [[IF_THEN76]]:
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i64, ptr [[DOTOMP_LB116]], align 8
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_ST117]], align 8
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul nsw i64 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[ADD78:%.*]] = add nsw i64 [[TMP55]], [[MUL77]]
+// CHECK1-NEXT:    store i64 [[ADD78]], ptr [[DOTOMP_IV120]], align 8
+// CHECK1-NEXT:    [[TMP58:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_IV120]], align 8
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], 1
+// CHECK1-NEXT:    [[ADD_PTR80:%.*]] = getelementptr inbounds double, ptr [[TMP58]], i64 [[MUL79]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR80]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP60:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP60]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[C]], align 4
+// CHECK1-NEXT:    [[TMP62:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP63:%.*]] = load double, ptr [[TMP62]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP61]], double noundef [[TMP63]])
+// CHECK1-NEXT:    br label %[[IF_END81]]
+// CHECK1:       [[IF_END81]]:
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK1-NEXT:    [[CMP82:%.*]] = icmp slt i64 [[TMP64]], [[TMP65]]
+// CHECK1-NEXT:    br i1 [[CMP82]], label %[[IF_THEN83:.*]], label %[[IF_END88:.*]]
+// CHECK1:       [[IF_THEN83]]:
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[MUL84:%.*]] = mul nsw i64 [[TMP67]], [[TMP68]]
+// CHECK1-NEXT:    [[ADD85:%.*]] = add nsw i64 [[TMP66]], [[MUL84]]
+// CHECK1-NEXT:    store i64 [[ADD85]], ptr [[DOTOMP_IV2]], align 8
+// CHECK1-NEXT:    [[TMP69:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
+// CHECK1-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], 1
+// CHECK1-NEXT:    [[ADD_PTR87:%.*]] = getelementptr inbounds double, ptr [[TMP69]], i64 [[MUL86]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR87]], ptr [[__BEGIN225]], align 8
+// CHECK1-NEXT:    [[TMP71:%.*]] = load ptr, ptr [[__BEGIN225]], align 8
+// CHECK1-NEXT:    store ptr [[TMP71]], ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK1-NEXT:    [[TMP73:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP74:%.*]] = load double, ptr [[TMP73]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP72]], double noundef [[TMP74]])
+// CHECK1-NEXT:    br label %[[IF_END88]]
+// CHECK1:       [[IF_END88]]:
 // CHECK1-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK1:       [[FOR_INC]]:
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP78]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP75]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX52]], align 8
 // CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK1:       [[FOR_END]]:
 // CHECK1-NEXT:    ret void
@@ -794,13 +783,11 @@ extern "C" void foo4() {
 // CHECK1-NEXT:  [[ENTRY:.*:]]
 // CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -815,12 +802,10 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
-// CHECK1-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
 // CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[K]], align 4
-// CHECK1-NEXT:    store i32 63, ptr [[DOTOMP_UB1]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
 // CHECK1-NEXT:    store i32 64, ptr [[DOTOMP_NI1]], align 4
@@ -940,6 +925,277 @@ extern "C" void foo4() {
 // CHECK1-NEXT:    ret void
 //
 //
+// CHECK1-LABEL: define dso_local void @foo5(
+// CHECK1-SAME: ) #[[ATTR0]] {
+// CHECK1-NEXT:  [[ENTRY:.*:]]
+// CHECK1-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_LB03:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_ST04:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_NI05:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV06:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_10:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB116:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_ST117:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_NI118:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV120:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_TEMP_121:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_FUSE_MAX22:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_FUSE_INDEX29:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[CC:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[__RANGE264:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__BEGIN265:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[__END267:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[VV:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[K]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    store i32 512, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK1-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK1:       [[COND_TRUE]]:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END:.*]]
+// CHECK1:       [[COND_FALSE]]:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    br label %[[COND_END]]
+// CHECK1:       [[COND_END]]:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK1-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK1-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK1-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB03]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_ST04]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
+// CHECK1-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
+// CHECK1-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP8]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY7:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY7]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY9]], ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP11]], ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
+// CHECK1-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK1-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK1-NEXT:    [[SUB12:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[SUB12]], 1
+// CHECK1-NEXT:    [[DIV14:%.*]] = sdiv i64 [[ADD13]], 1
+// CHECK1-NEXT:    [[SUB15:%.*]] = sub nsw i64 [[DIV14]], 1
+// CHECK1-NEXT:    store i64 [[SUB15]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB116]], align 8
+// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_ST117]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK1-NEXT:    store i64 [[ADD19]], ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[CMP23:%.*]] = icmp sgt i64 [[TMP16]], [[TMP17]]
+// CHECK1-NEXT:    br i1 [[CMP23]], label %[[COND_TRUE24:.*]], label %[[COND_FALSE25:.*]]
+// CHECK1:       [[COND_TRUE24]]:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK1-NEXT:    br label %[[COND_END26:.*]]
+// CHECK1:       [[COND_FALSE25]]:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    br label %[[COND_END26]]
+// CHECK1:       [[COND_END26]]:
+// CHECK1-NEXT:    [[COND27:%.*]] = phi i64 [ [[TMP18]], %[[COND_TRUE24]] ], [ [[TMP19]], %[[COND_FALSE25]] ]
+// CHECK1-NEXT:    store i64 [[COND27]], ptr [[DOTOMP_FUSE_MAX22]], align 8
+// CHECK1-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK1:       [[FOR_COND]]:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP20]], 128
+// CHECK1-NEXT:    br i1 [[CMP28]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK1:       [[FOR_BODY]]:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP21]])
+// CHECK1-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK1:       [[FOR_INC]]:
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK1-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1:       [[FOR_END]]:
+// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND30:.*]]
+// CHECK1:       [[FOR_COND30]]:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX22]], align 8
+// CHECK1-NEXT:    [[CMP31:%.*]] = icmp slt i64 [[TMP23]], [[TMP24]]
+// CHECK1-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY32:.*]], label %[[FOR_END63:.*]]
+// CHECK1:       [[FOR_BODY32]]:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK1-NEXT:    [[CMP33:%.*]] = icmp slt i64 [[TMP25]], [[TMP26]]
+// CHECK1-NEXT:    br i1 [[CMP33]], label %[[IF_THEN:.*]], label %[[IF_END53:.*]]
+// CHECK1:       [[IF_THEN]]:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB03]], align 4
+// CHECK1-NEXT:    [[CONV34:%.*]] = sext i32 [[TMP27]] to i64
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_ST04]], align 4
+// CHECK1-NEXT:    [[CONV35:%.*]] = sext i32 [[TMP28]] to i64
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV35]], [[TMP29]]
+// CHECK1-NEXT:    [[ADD36:%.*]] = add nsw i64 [[CONV34]], [[MUL]]
+// CHECK1-NEXT:    [[CONV37:%.*]] = trunc i64 [[ADD36]] to i32
+// CHECK1-NEXT:    store i32 [[CONV37]], ptr [[DOTOMP_IV06]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_IV06]], align 4
+// CHECK1-NEXT:    [[MUL38:%.*]] = mul nsw i32 [[TMP30]], 1
+// CHECK1-NEXT:    [[ADD39:%.*]] = add nsw i32 0, [[MUL38]]
+// CHECK1-NEXT:    store i32 [[ADD39]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK1-NEXT:    [[CMP40:%.*]] = icmp slt i32 [[TMP31]], [[TMP32]]
+// CHECK1-NEXT:    br i1 [[CMP40]], label %[[IF_THEN41:.*]], label %[[IF_END:.*]]
+// CHECK1:       [[IF_THEN41]]:
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL42:%.*]] = mul nsw i32 [[TMP34]], [[TMP35]]
+// CHECK1-NEXT:    [[ADD43:%.*]] = add nsw i32 [[TMP33]], [[MUL42]]
+// CHECK1-NEXT:    store i32 [[ADD43]], ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK1-NEXT:    [[MUL44:%.*]] = mul nsw i32 [[TMP36]], 2
+// CHECK1-NEXT:    [[ADD45:%.*]] = add nsw i32 0, [[MUL44]]
+// CHECK1-NEXT:    store i32 [[ADD45]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP37]])
+// CHECK1-NEXT:    br label %[[IF_END]]
+// CHECK1:       [[IF_END]]:
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK1-NEXT:    [[CMP46:%.*]] = icmp slt i32 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
+// CHECK1:       [[IF_THEN47]]:
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK1-NEXT:    [[MUL48:%.*]] = mul nsw i32 [[TMP41]], [[TMP42]]
+// CHECK1-NEXT:    [[ADD49:%.*]] = add nsw i32 [[TMP40]], [[MUL48]]
+// CHECK1-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul nsw i32 [[TMP43]], 1
+// CHECK1-NEXT:    [[ADD51:%.*]] = add nsw i32 0, [[MUL50]]
+// CHECK1-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK1-NEXT:    br label %[[IF_END52]]
+// CHECK1:       [[IF_END52]]:
+// CHECK1-NEXT:    br label %[[IF_END53]]
+// CHECK1:       [[IF_END53]]:
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK1-NEXT:    [[CMP54:%.*]] = icmp slt i64 [[TMP45]], [[TMP46]]
+// CHECK1-NEXT:    br i1 [[CMP54]], label %[[IF_THEN55:.*]], label %[[IF_END60:.*]]
+// CHECK1:       [[IF_THEN55]]:
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB116]], align 8
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_ST117]], align 8
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[MUL56:%.*]] = mul nsw i64 [[TMP48]], [[TMP49]]
+// CHECK1-NEXT:    [[ADD57:%.*]] = add nsw i64 [[TMP47]], [[MUL56]]
+// CHECK1-NEXT:    store i64 [[ADD57]], ptr [[DOTOMP_IV120]], align 8
+// CHECK1-NEXT:    [[TMP50:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV120]], align 8
+// CHECK1-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[TMP51]], 1
+// CHECK1-NEXT:    [[ADD_PTR59:%.*]] = getelementptr inbounds double, ptr [[TMP50]], i64 [[MUL58]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR59]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP52:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    store ptr [[TMP52]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[C]], align 4
+// CHECK1-NEXT:    [[TMP54:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP55:%.*]] = load double, ptr [[TMP54]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP53]], double noundef [[TMP55]])
+// CHECK1-NEXT:    br label %[[IF_END60]]
+// CHECK1:       [[IF_END60]]:
+// CHECK1-NEXT:    br label %[[FOR_INC61:.*]]
+// CHECK1:       [[FOR_INC61]]:
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    [[INC62:%.*]] = add nsw i64 [[TMP56]], 1
+// CHECK1-NEXT:    store i64 [[INC62]], ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND30]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK1:       [[FOR_END63]]:
+// CHECK1-NEXT:    store i32 37, ptr [[CC]], align 4
+// CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE264]], align 8
+// CHECK1-NEXT:    [[TMP57:%.*]] = load ptr, ptr [[__RANGE264]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY66:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP57]], i64 0, i64 0
+// CHECK1-NEXT:    store ptr [[ARRAYDECAY66]], ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    [[TMP58:%.*]] = load ptr, ptr [[__RANGE264]], align 8
+// CHECK1-NEXT:    [[ARRAYDECAY68:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP58]], i64 0, i64 0
+// CHECK1-NEXT:    [[ADD_PTR69:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY68]], i64 256
+// CHECK1-NEXT:    store ptr [[ADD_PTR69]], ptr [[__END267]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND70:.*]]
+// CHECK1:       [[FOR_COND70]]:
+// CHECK1-NEXT:    [[TMP59:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    [[TMP60:%.*]] = load ptr, ptr [[__END267]], align 8
+// CHECK1-NEXT:    [[CMP71:%.*]] = icmp ne ptr [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    br i1 [[CMP71]], label %[[FOR_BODY72:.*]], label %[[FOR_END74:.*]]
+// CHECK1:       [[FOR_BODY72]]:
+// CHECK1-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    store ptr [[TMP61]], ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK1-NEXT:    [[TMP64:%.*]] = load double, ptr [[TMP63]], align 8
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP62]], double noundef [[TMP64]])
+// CHECK1-NEXT:    br label %[[FOR_INC73:.*]]
+// CHECK1:       [[FOR_INC73]]:
+// CHECK1-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw double, ptr [[TMP65]], i32 1
+// CHECK1-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN265]], align 8
+// CHECK1-NEXT:    br label %[[FOR_COND70]]
+// CHECK1:       [[FOR_END74]]:
+// CHECK1-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define dso_local void @body(
 // CHECK2-SAME: ...) #[[ATTR0:[0-9]+]] {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
@@ -961,7 +1217,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
@@ -970,7 +1225,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -1002,107 +1256,103 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
 // CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP8]], 1
 // CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[START2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP9]], ptr [[J]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[START2_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START2_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[END2_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[STEP2_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP13]], [[TMP14]]
 // CHECK2-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP15]]
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP16]]
 // CHECK2-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
 // CHECK2-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    store i32 [[TMP20]], ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP21]], [[TMP22]]
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP19]], [[TMP20]]
 // CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // CHECK2:       [[COND_TRUE]]:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
 // CHECK2-NEXT:    br label %[[COND_END:.*]]
 // CHECK2:       [[COND_FALSE]]:
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
 // CHECK2-NEXT:    br label %[[COND_END]]
 // CHECK2:       [[COND_END]]:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP23]], %[[COND_TRUE]] ], [ [[TMP24]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP21]], %[[COND_TRUE]] ], [ [[TMP22]], %[[COND_FALSE]] ]
 // CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK2-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK2:       [[FOR_COND]]:
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
-// CHECK2-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP23]], [[TMP24]]
 // CHECK2-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK2:       [[FOR_BODY]]:
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP17:%.*]] = icmp ult i32 [[TMP25]], [[TMP26]]
 // CHECK2-NEXT:    br i1 [[CMP17]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // CHECK2:       [[IF_THEN]]:
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP30]], [[TMP31]]
-// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP29]], [[MUL]]
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP28]], [[TMP29]]
+// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP27]], [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP33]], [[TMP34]]
-// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP32]], [[MUL19]]
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL19:%.*]] = mul i32 [[TMP31]], [[TMP32]]
+// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP30]], [[MUL19]]
 // CHECK2-NEXT:    store i32 [[ADD20]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP35]])
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP33]])
 // CHECK2-NEXT:    br label %[[IF_END]]
 // CHECK2:       [[IF_END]]:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP34]], [[TMP35]]
 // CHECK2-NEXT:    br i1 [[CMP21]], label %[[IF_THEN22:.*]], label %[[IF_END27:.*]]
 // CHECK2:       [[IF_THEN22]]:
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP39]], [[TMP40]]
-// CHECK2-NEXT:    [[ADD24:%.*]] = add i32 [[TMP38]], [[MUL23]]
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL23:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// CHECK2-NEXT:    [[ADD24:%.*]] = add i32 [[TMP36]], [[MUL23]]
 // CHECK2-NEXT:    store i32 [[ADD24]], ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP42]], [[TMP43]]
-// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[TMP41]], [[MUL25]]
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[MUL25:%.*]] = mul i32 [[TMP40]], [[TMP41]]
+// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[TMP39]], [[MUL25]]
 // CHECK2-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP42]])
 // CHECK2-NEXT:    br label %[[IF_END27]]
 // CHECK2:       [[IF_END27]]:
 // CHECK2-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK2:       [[FOR_INC]]:
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP45]], 1
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
 // CHECK2:       [[FOR_END]]:
@@ -1114,13 +1364,11 @@ extern "C" void foo4() {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
 // CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -1130,48 +1378,43 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB03:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_LB04:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_ST05:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_NI06:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_IV07:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB03:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST04:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI05:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV06:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[C:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_12:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_UB117:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_LB118:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_ST119:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_NI120:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_IV122:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_10:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB116:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_ST117:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_NI118:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV120:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[CC:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[__RANGE223:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[__END224:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[__BEGIN227:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__RANGE221:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END222:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN225:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_27:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_29:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_31:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_32:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_30:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IV2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_TEMP_142:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_TEMP_140:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_TEMP_2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX48:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX54:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX46:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX52:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[VV:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
 // CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
-// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB1]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
 // CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI1]], align 4
@@ -1198,225 +1441,219 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
 // CHECK2-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
 // CHECK2-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB03]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST04]], align 4
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTOMP_UB03]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB04]], align 4
-// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST05]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], 1
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
 // CHECK2-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
-// CHECK2-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI06]], align 8
+// CHECK2-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI05]], align 8
 // CHECK2-NEXT:    store i32 42, ptr [[C]], align 4
 // CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP8]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY7:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY7]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY8:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYDECAY8]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY10:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP11]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYDECAY10]], ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__END2]], align 8
-// CHECK2-NEXT:    store ptr [[TMP12]], ptr [[DOTCAPTURE_EXPR_11]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_11]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
-// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP14]] to i64
+// CHECK2-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY9]], ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP11]], ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
 // CHECK2-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
 // CHECK2-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
-// CHECK2-NEXT:    [[SUB13:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
-// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i64 [[SUB13]], 1
-// CHECK2-NEXT:    [[DIV15:%.*]] = sdiv i64 [[ADD14]], 1
-// CHECK2-NEXT:    [[SUB16:%.*]] = sub nsw i64 [[DIV15]], 1
-// CHECK2-NEXT:    store i64 [[SUB16]], ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK2-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_UB117]], align 8
-// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB118]], align 8
-// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST119]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_12]], align 8
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK2-NEXT:    store i64 [[ADD21]], ptr [[DOTOMP_NI120]], align 8
+// CHECK2-NEXT:    [[SUB12:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[SUB12]], 1
+// CHECK2-NEXT:    [[DIV14:%.*]] = sdiv i64 [[ADD13]], 1
+// CHECK2-NEXT:    [[SUB15:%.*]] = sub nsw i64 [[DIV14]], 1
+// CHECK2-NEXT:    store i64 [[SUB15]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB116]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST117]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK2-NEXT:    store i64 [[ADD19]], ptr [[DOTOMP_NI118]], align 8
 // CHECK2-NEXT:    store i32 37, ptr [[CC]], align 4
-// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE223]], align 8
-// CHECK2-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY25:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
-// CHECK2-NEXT:    [[ADD_PTR26:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY25]], i64 256
-// CHECK2-NEXT:    store ptr [[ADD_PTR26]], ptr [[__END224]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP18]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[__BEGIN227]], align 8
-// CHECK2-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[__RANGE223]], align 8
-// CHECK2-NEXT:    [[ARRAYDECAY30:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP19]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYDECAY30]], ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK2-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[__END224]], align 8
-// CHECK2-NEXT:    store ptr [[TMP20]], ptr [[DOTCAPTURE_EXPR_31]], align 8
-// CHECK2-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_31]], align 8
-// CHECK2-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST33:%.*]] = ptrtoint ptr [[TMP21]] to i64
-// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST34:%.*]] = ptrtoint ptr [[TMP22]] to i64
-// CHECK2-NEXT:    [[SUB_PTR_SUB35:%.*]] = sub i64 [[SUB_PTR_LHS_CAST33]], [[SUB_PTR_RHS_CAST34]]
-// CHECK2-NEXT:    [[SUB_PTR_DIV36:%.*]] = sdiv exact i64 [[SUB_PTR_SUB35]], 8
-// CHECK2-NEXT:    [[SUB37:%.*]] = sub nsw i64 [[SUB_PTR_DIV36]], 1
-// CHECK2-NEXT:    [[ADD38:%.*]] = add nsw i64 [[SUB37]], 1
-// CHECK2-NEXT:    [[DIV39:%.*]] = sdiv i64 [[ADD38]], 1
-// CHECK2-NEXT:    [[SUB40:%.*]] = sub nsw i64 [[DIV39]], 1
-// CHECK2-NEXT:    store i64 [[SUB40]], ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK2-NEXT:    store i64 [[TMP23]], ptr [[DOTOMP_UB2]], align 8
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE221]], align 8
+// CHECK2-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY23:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP15]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR24:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY23]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR24]], ptr [[__END222]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY26:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP16]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY26]], ptr [[__BEGIN225]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[__RANGE221]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY28:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP17]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY28]], ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[__END222]], align 8
+// CHECK2-NEXT:    store ptr [[TMP18]], ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST31:%.*]] = ptrtoint ptr [[TMP19]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST32:%.*]] = ptrtoint ptr [[TMP20]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB33:%.*]] = sub i64 [[SUB_PTR_LHS_CAST31]], [[SUB_PTR_RHS_CAST32]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV34:%.*]] = sdiv exact i64 [[SUB_PTR_SUB33]], 8
+// CHECK2-NEXT:    [[SUB35:%.*]] = sub nsw i64 [[SUB_PTR_DIV34]], 1
+// CHECK2-NEXT:    [[ADD36:%.*]] = add nsw i64 [[SUB35]], 1
+// CHECK2-NEXT:    [[DIV37:%.*]] = sdiv i64 [[ADD36]], 1
+// CHECK2-NEXT:    [[SUB38:%.*]] = sub nsw i64 [[DIV37]], 1
+// CHECK2-NEXT:    store i64 [[SUB38]], ptr [[DOTCAPTURE_EXPR_30]], align 8
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB2]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST2]], align 8
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_32]], align 8
-// CHECK2-NEXT:    [[ADD41:%.*]] = add nsw i64 [[TMP24]], 1
-// CHECK2-NEXT:    store i64 [[ADD41]], ptr [[DOTOMP_NI2]], align 8
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
-// CHECK2-NEXT:    store i64 [[TMP25]], ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK2-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP26]], [[TMP27]]
-// CHECK2-NEXT:    br i1 [[CMP43]], label %[[COND_TRUE44:.*]], label %[[COND_FALSE45:.*]]
-// CHECK2:       [[COND_TRUE44]]:
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_TEMP_142]], align 8
-// CHECK2-NEXT:    br label %[[COND_END46:.*]]
-// CHECK2:       [[COND_FALSE45]]:
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK2-NEXT:    br label %[[COND_END46]]
-// CHECK2:       [[COND_END46]]:
-// CHECK2-NEXT:    [[COND47:%.*]] = phi i64 [ [[TMP28]], %[[COND_TRUE44]] ], [ [[TMP29]], %[[COND_FALSE45]] ]
-// CHECK2-NEXT:    store i64 [[COND47]], ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK2-NEXT:    [[CMP49:%.*]] = icmp sgt i64 [[TMP30]], [[TMP31]]
-// CHECK2-NEXT:    br i1 [[CMP49]], label %[[COND_TRUE50:.*]], label %[[COND_FALSE51:.*]]
-// CHECK2:       [[COND_TRUE50]]:
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
-// CHECK2-NEXT:    br label %[[COND_END52:.*]]
-// CHECK2:       [[COND_FALSE51]]:
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK2-NEXT:    br label %[[COND_END52]]
-// CHECK2:       [[COND_END52]]:
-// CHECK2-NEXT:    [[COND53:%.*]] = phi i64 [ [[TMP32]], %[[COND_TRUE50]] ], [ [[TMP33]], %[[COND_FALSE51]] ]
-// CHECK2-NEXT:    store i64 [[COND53]], ptr [[DOTOMP_FUSE_MAX48]], align 8
-// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_30]], align 8
+// CHECK2-NEXT:    [[ADD39:%.*]] = add nsw i64 [[TMP21]], 1
+// CHECK2-NEXT:    store i64 [[ADD39]], ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    store i64 [[TMP22]], ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[CMP41:%.*]] = icmp sgt i64 [[TMP23]], [[TMP24]]
+// CHECK2-NEXT:    br i1 [[CMP41]], label %[[COND_TRUE42:.*]], label %[[COND_FALSE43:.*]]
+// CHECK2:       [[COND_TRUE42]]:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_TEMP_140]], align 8
+// CHECK2-NEXT:    br label %[[COND_END44:.*]]
+// CHECK2:       [[COND_FALSE43]]:
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    br label %[[COND_END44]]
+// CHECK2:       [[COND_END44]]:
+// CHECK2-NEXT:    [[COND45:%.*]] = phi i64 [ [[TMP25]], %[[COND_TRUE42]] ], [ [[TMP26]], %[[COND_FALSE43]] ]
+// CHECK2-NEXT:    store i64 [[COND45]], ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[CMP47:%.*]] = icmp sgt i64 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP47]], label %[[COND_TRUE48:.*]], label %[[COND_FALSE49:.*]]
+// CHECK2:       [[COND_TRUE48]]:
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_TEMP_2]], align 8
+// CHECK2-NEXT:    br label %[[COND_END50:.*]]
+// CHECK2:       [[COND_FALSE49]]:
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    br label %[[COND_END50]]
+// CHECK2:       [[COND_END50]]:
+// CHECK2-NEXT:    [[COND51:%.*]] = phi i64 [ [[TMP29]], %[[COND_TRUE48]] ], [ [[TMP30]], %[[COND_FALSE49]] ]
+// CHECK2-NEXT:    store i64 [[COND51]], ptr [[DOTOMP_FUSE_MAX46]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX52]], align 8
 // CHECK2-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK2:       [[FOR_COND]]:
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX48]], align 8
-// CHECK2-NEXT:    [[CMP55:%.*]] = icmp slt i64 [[TMP34]], [[TMP35]]
-// CHECK2-NEXT:    br i1 [[CMP55]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX46]], align 8
+// CHECK2-NEXT:    [[CMP53:%.*]] = icmp slt i64 [[TMP31]], [[TMP32]]
+// CHECK2-NEXT:    br i1 [[CMP53]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK2:       [[FOR_BODY]]:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_NI06]], align 8
-// CHECK2-NEXT:    [[CMP56:%.*]] = icmp slt i64 [[TMP36]], [[TMP37]]
-// CHECK2-NEXT:    br i1 [[CMP56]], label %[[IF_THEN:.*]], label %[[IF_END76:.*]]
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    [[CMP54:%.*]] = icmp slt i64 [[TMP33]], [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP54]], label %[[IF_THEN:.*]], label %[[IF_END74:.*]]
 // CHECK2:       [[IF_THEN]]:
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_LB04]], align 4
-// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP38]] to i64
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_ST05]], align 4
-// CHECK2-NEXT:    [[CONV58:%.*]] = sext i32 [[TMP39]] to i64
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV58]], [[TMP40]]
-// CHECK2-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV57]], [[MUL]]
-// CHECK2-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
-// CHECK2-NEXT:    store i32 [[CONV60]], ptr [[DOTOMP_IV07]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_IV07]], align 4
-// CHECK2-NEXT:    [[MUL61:%.*]] = mul nsw i32 [[TMP41]], 1
-// CHECK2-NEXT:    [[ADD62:%.*]] = add nsw i32 0, [[MUL61]]
-// CHECK2-NEXT:    store i32 [[ADD62]], ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    [[CMP63:%.*]] = icmp slt i32 [[TMP42]], [[TMP43]]
-// CHECK2-NEXT:    br i1 [[CMP63]], label %[[IF_THEN64:.*]], label %[[IF_END:.*]]
-// CHECK2:       [[IF_THEN64]]:
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP45]], [[TMP46]]
-// CHECK2-NEXT:    [[ADD66:%.*]] = add nsw i32 [[TMP44]], [[MUL65]]
-// CHECK2-NEXT:    store i32 [[ADD66]], ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[MUL67:%.*]] = mul nsw i32 [[TMP47]], 1
-// CHECK2-NEXT:    [[ADD68:%.*]] = add nsw i32 0, [[MUL67]]
-// CHECK2-NEXT:    store i32 [[ADD68]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP48]])
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_LB03]], align 4
+// CHECK2-NEXT:    [[CONV55:%.*]] = sext i32 [[TMP35]] to i64
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_ST04]], align 4
+// CHECK2-NEXT:    [[CONV56:%.*]] = sext i32 [[TMP36]] to i64
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV56]], [[TMP37]]
+// CHECK2-NEXT:    [[ADD57:%.*]] = add nsw i64 [[CONV55]], [[MUL]]
+// CHECK2-NEXT:    [[CONV58:%.*]] = trunc i64 [[ADD57]] to i32
+// CHECK2-NEXT:    store i32 [[CONV58]], ptr [[DOTOMP_IV06]], align 4
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV06]], align 4
+// CHECK2-NEXT:    [[MUL59:%.*]] = mul nsw i32 [[TMP38]], 1
+// CHECK2-NEXT:    [[ADD60:%.*]] = add nsw i32 0, [[MUL59]]
+// CHECK2-NEXT:    store i32 [[ADD60]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP61:%.*]] = icmp slt i32 [[TMP39]], [[TMP40]]
+// CHECK2-NEXT:    br i1 [[CMP61]], label %[[IF_THEN62:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN62]]:
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL63:%.*]] = mul nsw i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[ADD64:%.*]] = add nsw i32 [[TMP41]], [[MUL63]]
+// CHECK2-NEXT:    store i32 [[ADD64]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[MUL65:%.*]] = mul nsw i32 [[TMP44]], 1
+// CHECK2-NEXT:    [[ADD66:%.*]] = add nsw i32 0, [[MUL65]]
+// CHECK2-NEXT:    store i32 [[ADD66]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP45]])
 // CHECK2-NEXT:    br label %[[IF_END]]
 // CHECK2:       [[IF_END]]:
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP69:%.*]] = icmp slt i32 [[TMP49]], [[TMP50]]
-// CHECK2-NEXT:    br i1 [[CMP69]], label %[[IF_THEN70:.*]], label %[[IF_END75:.*]]
-// CHECK2:       [[IF_THEN70]]:
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP52]], [[TMP53]]
-// CHECK2-NEXT:    [[ADD72:%.*]] = add nsw i32 [[TMP51]], [[MUL71]]
-// CHECK2-NEXT:    store i32 [[ADD72]], ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP54]], 2
-// CHECK2-NEXT:    [[ADD74:%.*]] = add nsw i32 0, [[MUL73]]
-// CHECK2-NEXT:    store i32 [[ADD74]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP55]])
-// CHECK2-NEXT:    br label %[[IF_END75]]
-// CHECK2:       [[IF_END75]]:
-// CHECK2-NEXT:    br label %[[IF_END76]]
-// CHECK2:       [[IF_END76]]:
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_NI120]], align 8
-// CHECK2-NEXT:    [[CMP77:%.*]] = icmp slt i64 [[TMP56]], [[TMP57]]
-// CHECK2-NEXT:    br i1 [[CMP77]], label %[[IF_THEN78:.*]], label %[[IF_END83:.*]]
-// CHECK2:       [[IF_THEN78]]:
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i64, ptr [[DOTOMP_LB118]], align 8
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_ST119]], align 8
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], [[TMP60]]
-// CHECK2-NEXT:    [[ADD80:%.*]] = add nsw i64 [[TMP58]], [[MUL79]]
-// CHECK2-NEXT:    store i64 [[ADD80]], ptr [[DOTOMP_IV122]], align 8
-// CHECK2-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_9]], align 8
-// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV122]], align 8
-// CHECK2-NEXT:    [[MUL81:%.*]] = mul nsw i64 [[TMP62]], 1
-// CHECK2-NEXT:    [[ADD_PTR82:%.*]] = getelementptr inbounds double, ptr [[TMP61]], i64 [[MUL81]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR82]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    store ptr [[TMP63]], ptr [[V]], align 8
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[C]], align 4
-// CHECK2-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[V]], align 8
-// CHECK2-NEXT:    [[TMP66:%.*]] = load double, ptr [[TMP65]], align 8
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP64]], double noundef [[TMP66]])
-// CHECK2-NEXT:    br label %[[IF_END83]]
-// CHECK2:       [[IF_END83]]:
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
-// CHECK2-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP67]], [[TMP68]]
-// CHECK2-NEXT:    br i1 [[CMP84]], label %[[IF_THEN85:.*]], label %[[IF_END90:.*]]
-// CHECK2:       [[IF_THEN85]]:
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
-// CHECK2-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
-// CHECK2-NEXT:    [[TMP71:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[ADD87:%.*]] = add nsw i64 [[TMP69]], [[MUL86]]
-// CHECK2-NEXT:    store i64 [[ADD87]], ptr [[DOTOMP_IV2]], align 8
-// CHECK2-NEXT:    [[TMP72:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_29]], align 8
-// CHECK2-NEXT:    [[TMP73:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
-// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[TMP73]], 1
-// CHECK2-NEXT:    [[ADD_PTR89:%.*]] = getelementptr inbounds double, ptr [[TMP72]], i64 [[MUL88]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR89]], ptr [[__BEGIN227]], align 8
-// CHECK2-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[__BEGIN227]], align 8
-// CHECK2-NEXT:    store ptr [[TMP74]], ptr [[VV]], align 8
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[CC]], align 4
-// CHECK2-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[VV]], align 8
-// CHECK2-NEXT:    [[TMP77:%.*]] = load double, ptr [[TMP76]], align 8
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP75]], double noundef [[TMP77]])
-// CHECK2-NEXT:    br label %[[IF_END90]]
-// CHECK2:       [[IF_END90]]:
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP67:%.*]] = icmp slt i32 [[TMP46]], [[TMP47]]
+// CHECK2-NEXT:    br i1 [[CMP67]], label %[[IF_THEN68:.*]], label %[[IF_END73:.*]]
+// CHECK2:       [[IF_THEN68]]:
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL69:%.*]] = mul nsw i32 [[TMP49]], [[TMP50]]
+// CHECK2-NEXT:    [[ADD70:%.*]] = add nsw i32 [[TMP48]], [[MUL69]]
+// CHECK2-NEXT:    store i32 [[ADD70]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[MUL71:%.*]] = mul nsw i32 [[TMP51]], 2
+// CHECK2-NEXT:    [[ADD72:%.*]] = add nsw i32 0, [[MUL71]]
+// CHECK2-NEXT:    store i32 [[ADD72]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK2-NEXT:    br label %[[IF_END73]]
+// CHECK2:       [[IF_END73]]:
+// CHECK2-NEXT:    br label %[[IF_END74]]
+// CHECK2:       [[IF_END74]]:
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[CMP75:%.*]] = icmp slt i64 [[TMP53]], [[TMP54]]
+// CHECK2-NEXT:    br i1 [[CMP75]], label %[[IF_THEN76:.*]], label %[[IF_END81:.*]]
+// CHECK2:       [[IF_THEN76]]:
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i64, ptr [[DOTOMP_LB116]], align 8
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_ST117]], align 8
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul nsw i64 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[ADD78:%.*]] = add nsw i64 [[TMP55]], [[MUL77]]
+// CHECK2-NEXT:    store i64 [[ADD78]], ptr [[DOTOMP_IV120]], align 8
+// CHECK2-NEXT:    [[TMP58:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i64, ptr [[DOTOMP_IV120]], align 8
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[TMP59]], 1
+// CHECK2-NEXT:    [[ADD_PTR80:%.*]] = getelementptr inbounds double, ptr [[TMP58]], i64 [[MUL79]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR80]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP60:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP60]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[C]], align 4
+// CHECK2-NEXT:    [[TMP62:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP63:%.*]] = load double, ptr [[TMP62]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP61]], double noundef [[TMP63]])
+// CHECK2-NEXT:    br label %[[IF_END81]]
+// CHECK2:       [[IF_END81]]:
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i64, ptr [[DOTOMP_NI2]], align 8
+// CHECK2-NEXT:    [[CMP82:%.*]] = icmp slt i64 [[TMP64]], [[TMP65]]
+// CHECK2-NEXT:    br i1 [[CMP82]], label %[[IF_THEN83:.*]], label %[[IF_END88:.*]]
+// CHECK2:       [[IF_THEN83]]:
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_LB2]], align 8
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_ST2]], align 8
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[MUL84:%.*]] = mul nsw i64 [[TMP67]], [[TMP68]]
+// CHECK2-NEXT:    [[ADD85:%.*]] = add nsw i64 [[TMP66]], [[MUL84]]
+// CHECK2-NEXT:    store i64 [[ADD85]], ptr [[DOTOMP_IV2]], align 8
+// CHECK2-NEXT:    [[TMP69:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_27]], align 8
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i64, ptr [[DOTOMP_IV2]], align 8
+// CHECK2-NEXT:    [[MUL86:%.*]] = mul nsw i64 [[TMP70]], 1
+// CHECK2-NEXT:    [[ADD_PTR87:%.*]] = getelementptr inbounds double, ptr [[TMP69]], i64 [[MUL86]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR87]], ptr [[__BEGIN225]], align 8
+// CHECK2-NEXT:    [[TMP71:%.*]] = load ptr, ptr [[__BEGIN225]], align 8
+// CHECK2-NEXT:    store ptr [[TMP71]], ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK2-NEXT:    [[TMP73:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP74:%.*]] = load double, ptr [[TMP73]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP72]], double noundef [[TMP74]])
+// CHECK2-NEXT:    br label %[[IF_END88]]
+// CHECK2:       [[IF_END88]]:
 // CHECK2-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK2:       [[FOR_INC]]:
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX54]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP78]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX54]], align 8
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX52]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP75]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTOMP_FUSE_INDEX52]], align 8
 // CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
 // CHECK2:       [[FOR_END]]:
 // CHECK2-NEXT:    ret void
@@ -1427,13 +1664,11 @@ extern "C" void foo4() {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
 // CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -1448,12 +1683,10 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
-// CHECK2-NEXT:    store i32 127, ptr [[DOTOMP_UB0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
 // CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[K]], align 4
-// CHECK2-NEXT:    store i32 63, ptr [[DOTOMP_UB1]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
 // CHECK2-NEXT:    store i32 64, ptr [[DOTOMP_NI1]], align 4
@@ -1573,6 +1806,277 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    ret void
 //
 //
+// CHECK2-LABEL: define dso_local void @foo5(
+// CHECK2-SAME: ) #[[ATTR0]] {
+// CHECK2-NEXT:  [[ENTRY:.*:]]
+// CHECK2-NEXT:    [[ARR:%.*]] = alloca [256 x double], align 16
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV0:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_IV1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_TEMP_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_LB03:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_ST04:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_NI05:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV06:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[C:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_8:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_10:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB116:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_ST117:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_NI118:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV120:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_TEMP_121:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_FUSE_MAX22:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_FUSE_INDEX29:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[CC:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[__RANGE264:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__BEGIN265:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[__END267:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[VV:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    store i32 0, ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    store i32 128, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[K]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    store i32 512, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP0]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], [[TMP2]]
+// CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// CHECK2:       [[COND_TRUE]]:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END:.*]]
+// CHECK2:       [[COND_FALSE]]:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    br label %[[COND_END]]
+// CHECK2:       [[COND_END]]:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP3]], %[[COND_TRUE]] ], [ [[TMP4]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK2-NEXT:    [[SUB2:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK2-NEXT:    store i32 [[SUB2]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB03]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST04]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP7]], 1
+// CHECK2-NEXT:    [[CONV:%.*]] = sext i32 [[ADD]] to i64
+// CHECK2-NEXT:    store i64 [[CONV]], ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    store i32 42, ptr [[C]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP8]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY7:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP9]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY7]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP10]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY9]], ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[__END2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP11]], ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_10]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP12]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[TMP13]] to i64
+// CHECK2-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i64 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
+// CHECK2-NEXT:    [[SUB_PTR_DIV:%.*]] = sdiv exact i64 [[SUB_PTR_SUB]], 8
+// CHECK2-NEXT:    [[SUB12:%.*]] = sub nsw i64 [[SUB_PTR_DIV]], 1
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[SUB12]], 1
+// CHECK2-NEXT:    [[DIV14:%.*]] = sdiv i64 [[ADD13]], 1
+// CHECK2-NEXT:    [[SUB15:%.*]] = sub nsw i64 [[DIV14]], 1
+// CHECK2-NEXT:    store i64 [[SUB15]], ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB116]], align 8
+// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_ST117]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_11]], align 8
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i64 [[TMP14]], 1
+// CHECK2-NEXT:    store i64 [[ADD19]], ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    store i64 [[TMP15]], ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[CMP23:%.*]] = icmp sgt i64 [[TMP16]], [[TMP17]]
+// CHECK2-NEXT:    br i1 [[CMP23]], label %[[COND_TRUE24:.*]], label %[[COND_FALSE25:.*]]
+// CHECK2:       [[COND_TRUE24]]:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTOMP_TEMP_121]], align 8
+// CHECK2-NEXT:    br label %[[COND_END26:.*]]
+// CHECK2:       [[COND_FALSE25]]:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    br label %[[COND_END26]]
+// CHECK2:       [[COND_END26]]:
+// CHECK2-NEXT:    [[COND27:%.*]] = phi i64 [ [[TMP18]], %[[COND_TRUE24]] ], [ [[TMP19]], %[[COND_FALSE25]] ]
+// CHECK2-NEXT:    store i64 [[COND27]], ptr [[DOTOMP_FUSE_MAX22]], align 8
+// CHECK2-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND:.*]]
+// CHECK2:       [[FOR_COND]]:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP20]], 128
+// CHECK2-NEXT:    br i1 [[CMP28]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// CHECK2:       [[FOR_BODY]]:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP21]])
+// CHECK2-NEXT:    br label %[[FOR_INC:.*]]
+// CHECK2:       [[FOR_INC]]:
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2:       [[FOR_END]]:
+// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND30:.*]]
+// CHECK2:       [[FOR_COND30]]:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i64, ptr [[DOTOMP_FUSE_MAX22]], align 8
+// CHECK2-NEXT:    [[CMP31:%.*]] = icmp slt i64 [[TMP23]], [[TMP24]]
+// CHECK2-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY32:.*]], label %[[FOR_END63:.*]]
+// CHECK2:       [[FOR_BODY32]]:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i64, ptr [[DOTOMP_NI05]], align 8
+// CHECK2-NEXT:    [[CMP33:%.*]] = icmp slt i64 [[TMP25]], [[TMP26]]
+// CHECK2-NEXT:    br i1 [[CMP33]], label %[[IF_THEN:.*]], label %[[IF_END53:.*]]
+// CHECK2:       [[IF_THEN]]:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB03]], align 4
+// CHECK2-NEXT:    [[CONV34:%.*]] = sext i32 [[TMP27]] to i64
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_ST04]], align 4
+// CHECK2-NEXT:    [[CONV35:%.*]] = sext i32 [[TMP28]] to i64
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV35]], [[TMP29]]
+// CHECK2-NEXT:    [[ADD36:%.*]] = add nsw i64 [[CONV34]], [[MUL]]
+// CHECK2-NEXT:    [[CONV37:%.*]] = trunc i64 [[ADD36]] to i32
+// CHECK2-NEXT:    store i32 [[CONV37]], ptr [[DOTOMP_IV06]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_IV06]], align 4
+// CHECK2-NEXT:    [[MUL38:%.*]] = mul nsw i32 [[TMP30]], 1
+// CHECK2-NEXT:    [[ADD39:%.*]] = add nsw i32 0, [[MUL38]]
+// CHECK2-NEXT:    store i32 [[ADD39]], ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP40:%.*]] = icmp slt i32 [[TMP31]], [[TMP32]]
+// CHECK2-NEXT:    br i1 [[CMP40]], label %[[IF_THEN41:.*]], label %[[IF_END:.*]]
+// CHECK2:       [[IF_THEN41]]:
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL42:%.*]] = mul nsw i32 [[TMP34]], [[TMP35]]
+// CHECK2-NEXT:    [[ADD43:%.*]] = add nsw i32 [[TMP33]], [[MUL42]]
+// CHECK2-NEXT:    store i32 [[ADD43]], ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[MUL44:%.*]] = mul nsw i32 [[TMP36]], 2
+// CHECK2-NEXT:    [[ADD45:%.*]] = add nsw i32 0, [[MUL44]]
+// CHECK2-NEXT:    store i32 [[ADD45]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP37]])
+// CHECK2-NEXT:    br label %[[IF_END]]
+// CHECK2:       [[IF_END]]:
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP46:%.*]] = icmp slt i32 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
+// CHECK2:       [[IF_THEN47]]:
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL48:%.*]] = mul nsw i32 [[TMP41]], [[TMP42]]
+// CHECK2-NEXT:    [[ADD49:%.*]] = add nsw i32 [[TMP40]], [[MUL48]]
+// CHECK2-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul nsw i32 [[TMP43]], 1
+// CHECK2-NEXT:    [[ADD51:%.*]] = add nsw i32 0, [[MUL50]]
+// CHECK2-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP44]])
+// CHECK2-NEXT:    br label %[[IF_END52]]
+// CHECK2:       [[IF_END52]]:
+// CHECK2-NEXT:    br label %[[IF_END53]]
+// CHECK2:       [[IF_END53]]:
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_NI118]], align 8
+// CHECK2-NEXT:    [[CMP54:%.*]] = icmp slt i64 [[TMP45]], [[TMP46]]
+// CHECK2-NEXT:    br i1 [[CMP54]], label %[[IF_THEN55:.*]], label %[[IF_END60:.*]]
+// CHECK2:       [[IF_THEN55]]:
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB116]], align 8
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_ST117]], align 8
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[MUL56:%.*]] = mul nsw i64 [[TMP48]], [[TMP49]]
+// CHECK2-NEXT:    [[ADD57:%.*]] = add nsw i64 [[TMP47]], [[MUL56]]
+// CHECK2-NEXT:    store i64 [[ADD57]], ptr [[DOTOMP_IV120]], align 8
+// CHECK2-NEXT:    [[TMP50:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_8]], align 8
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV120]], align 8
+// CHECK2-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[TMP51]], 1
+// CHECK2-NEXT:    [[ADD_PTR59:%.*]] = getelementptr inbounds double, ptr [[TMP50]], i64 [[MUL58]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR59]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP52:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    store ptr [[TMP52]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[C]], align 4
+// CHECK2-NEXT:    [[TMP54:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP55:%.*]] = load double, ptr [[TMP54]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP53]], double noundef [[TMP55]])
+// CHECK2-NEXT:    br label %[[IF_END60]]
+// CHECK2:       [[IF_END60]]:
+// CHECK2-NEXT:    br label %[[FOR_INC61:.*]]
+// CHECK2:       [[FOR_INC61]]:
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i64, ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    [[INC62:%.*]] = add nsw i64 [[TMP56]], 1
+// CHECK2-NEXT:    store i64 [[INC62]], ptr [[DOTOMP_FUSE_INDEX29]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND30]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK2:       [[FOR_END63]]:
+// CHECK2-NEXT:    store i32 37, ptr [[CC]], align 4
+// CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE264]], align 8
+// CHECK2-NEXT:    [[TMP57:%.*]] = load ptr, ptr [[__RANGE264]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY66:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP57]], i64 0, i64 0
+// CHECK2-NEXT:    store ptr [[ARRAYDECAY66]], ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    [[TMP58:%.*]] = load ptr, ptr [[__RANGE264]], align 8
+// CHECK2-NEXT:    [[ARRAYDECAY68:%.*]] = getelementptr inbounds [256 x double], ptr [[TMP58]], i64 0, i64 0
+// CHECK2-NEXT:    [[ADD_PTR69:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY68]], i64 256
+// CHECK2-NEXT:    store ptr [[ADD_PTR69]], ptr [[__END267]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND70:.*]]
+// CHECK2:       [[FOR_COND70]]:
+// CHECK2-NEXT:    [[TMP59:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    [[TMP60:%.*]] = load ptr, ptr [[__END267]], align 8
+// CHECK2-NEXT:    [[CMP71:%.*]] = icmp ne ptr [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    br i1 [[CMP71]], label %[[FOR_BODY72:.*]], label %[[FOR_END74:.*]]
+// CHECK2:       [[FOR_BODY72]]:
+// CHECK2-NEXT:    [[TMP61:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    store ptr [[TMP61]], ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[CC]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load ptr, ptr [[VV]], align 8
+// CHECK2-NEXT:    [[TMP64:%.*]] = load double, ptr [[TMP63]], align 8
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP62]], double noundef [[TMP64]])
+// CHECK2-NEXT:    br label %[[FOR_INC73:.*]]
+// CHECK2:       [[FOR_INC73]]:
+// CHECK2-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw double, ptr [[TMP65]], i32 1
+// CHECK2-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN265]], align 8
+// CHECK2-NEXT:    br label %[[FOR_COND70]]
+// CHECK2:       [[FOR_END74]]:
+// CHECK2-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define dso_local void @tfoo2(
 // CHECK2-SAME: ) #[[ATTR0]] {
 // CHECK2-NEXT:  [[ENTRY:.*:]]
@@ -1593,7 +2097,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST0:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI0:%.*]] = alloca i32, align 4
@@ -1602,7 +2105,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP8:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI1:%.*]] = alloca i32, align 4
@@ -1611,7 +2113,6 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP21:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_UB2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_ST2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_NI2:%.*]] = alloca i32, align 4
@@ -1641,174 +2142,168 @@ extern "C" void foo4() {
 // CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP7]]
 // CHECK2-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK2-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB0]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB0]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP8]], 1
 // CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP9]], ptr [[J]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP11]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP12]], ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// CHECK2-NEXT:    [[SUB10:%.*]] = sub i32 [[TMP13]], [[TMP14]]
 // CHECK2-NEXT:    [[SUB11:%.*]] = sub i32 [[SUB10]], 1
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP15]]
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[SUB11]], [[TMP16]]
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP17]]
+// CHECK2-NEXT:    [[DIV13:%.*]] = udiv i32 [[ADD12]], [[TMP16]]
 // CHECK2-NEXT:    [[SUB14:%.*]] = sub i32 [[DIV13]], 1
 // CHECK2-NEXT:    store i32 [[SUB14]], ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    store i32 [[TMP18]], ptr [[DOTOMP_UB1]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB1]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_9]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
 // CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
+// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
 // CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP22]], [[TMP23]]
 // CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTCAPTURE_EXPR_19]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[DOTNEW_STEP21]], align 4
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// CHECK2-NEXT:    store i32 [[TMP24]], ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_19]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP25]], [[TMP26]]
 // CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP29]]
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP30]]
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP27]]
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP28]]
 // CHECK2-NEXT:    [[SUB27:%.*]] = sub i32 [[DIV26]], 1
 // CHECK2-NEXT:    store i32 [[SUB27]], ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK2-NEXT:    store i32 [[TMP31]], ptr [[DOTOMP_UB2]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB2]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_ST2]], align 4
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
-// CHECK2-NEXT:    [[ADD28:%.*]] = add i32 [[TMP32]], 1
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_22]], align 4
+// CHECK2-NEXT:    [[ADD28:%.*]] = add i32 [[TMP29]], 1
 // CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_NI2]], align 4
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    store i32 [[TMP33]], ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP34]], [[TMP35]]
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    store i32 [[TMP30]], ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[TMP31]], [[TMP32]]
 // CHECK2-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // CHECK2:       [[COND_TRUE]]:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTOMP_TEMP_1]], align 4
 // CHECK2-NEXT:    br label %[[COND_END:.*]]
 // CHECK2:       [[COND_FALSE]]:
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
 // CHECK2-NEXT:    br label %[[COND_END]]
 // CHECK2:       [[COND_END]]:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP36]], %[[COND_TRUE]] ], [ [[TMP37]], %[[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP33]], %[[COND_TRUE]] ], [ [[TMP34]], %[[COND_FALSE]] ]
 // CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_TEMP_2]], align 4
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
-// CHECK2-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[CMP29:%.*]] = icmp ugt i32 [[TMP35]], [[TMP36]]
 // CHECK2-NEXT:    br i1 [[CMP29]], label %[[COND_TRUE30:.*]], label %[[COND_FALSE31:.*]]
 // CHECK2:       [[COND_TRUE30]]:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTOMP_TEMP_2]], align 4
 // CHECK2-NEXT:    br label %[[COND_END32:.*]]
 // CHECK2:       [[COND_FALSE31]]:
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
 // CHECK2-NEXT:    br label %[[COND_END32]]
 // CHECK2:       [[COND_END32]]:
-// CHECK2-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP40]], %[[COND_TRUE30]] ], [ [[TMP41]], %[[COND_FALSE31]] ]
+// CHECK2-NEXT:    [[COND33:%.*]] = phi i32 [ [[TMP37]], %[[COND_TRUE30]] ], [ [[TMP38]], %[[COND_FALSE31]] ]
 // CHECK2-NEXT:    store i32 [[COND33]], ptr [[DOTOMP_FUSE_MAX]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_FUSE_INDEX]], align 4
 // CHECK2-NEXT:    br label %[[FOR_COND:.*]]
 // CHECK2:       [[FOR_COND]]:
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
-// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTOMP_FUSE_MAX]], align 4
+// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 [[TMP39]], [[TMP40]]
 // CHECK2-NEXT:    br i1 [[CMP34]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // CHECK2:       [[FOR_BODY]]:
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
-// CHECK2-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP44]], [[TMP45]]
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTOMP_NI0]], align 4
+// CHECK2-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP41]], [[TMP42]]
 // CHECK2-NEXT:    br i1 [[CMP35]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // CHECK2:       [[IF_THEN]]:
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP47]], [[TMP48]]
-// CHECK2-NEXT:    [[ADD36:%.*]] = add i32 [[TMP46]], [[MUL]]
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_LB0]], align 4
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_ST0]], align 4
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP44]], [[TMP45]]
+// CHECK2-NEXT:    [[ADD36:%.*]] = add i32 [[TMP43]], [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD36]], ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP50]], [[TMP51]]
-// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[TMP49]], [[MUL37]]
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTOMP_IV0]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP47]], [[TMP48]]
+// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[TMP46]], [[MUL37]]
 // CHECK2-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP52]])
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP49]])
 // CHECK2-NEXT:    br label %[[IF_END]]
 // CHECK2:       [[IF_END]]:
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
-// CHECK2-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP53]], [[TMP54]]
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTOMP_NI1]], align 4
+// CHECK2-NEXT:    [[CMP39:%.*]] = icmp ult i32 [[TMP50]], [[TMP51]]
 // CHECK2-NEXT:    br i1 [[CMP39]], label %[[IF_THEN40:.*]], label %[[IF_END45:.*]]
 // CHECK2:       [[IF_THEN40]]:
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP56]], [[TMP57]]
-// CHECK2-NEXT:    [[ADD42:%.*]] = add i32 [[TMP55]], [[MUL41]]
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTOMP_LB1]], align 4
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTOMP_ST1]], align 4
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL41:%.*]] = mul i32 [[TMP53]], [[TMP54]]
+// CHECK2-NEXT:    [[ADD42:%.*]] = add i32 [[TMP52]], [[MUL41]]
 // CHECK2-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
-// CHECK2-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP59]], [[TMP60]]
-// CHECK2-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP58]], [[MUL43]]
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTOMP_IV1]], align 4
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP8]], align 4
+// CHECK2-NEXT:    [[MUL43:%.*]] = mul i32 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[SUB44:%.*]] = sub i32 [[TMP55]], [[MUL43]]
 // CHECK2-NEXT:    store i32 [[SUB44]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP61]])
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP58]])
 // CHECK2-NEXT:    br label %[[IF_END45]]
 // CHECK2:       [[IF_END45]]:
-// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
-// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP62]], [[TMP63]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTOMP_NI2]], align 4
+// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 [[TMP59]], [[TMP60]]
 // CHECK2-NEXT:    br i1 [[CMP46]], label %[[IF_THEN47:.*]], label %[[IF_END52:.*]]
 // CHECK2:       [[IF_THEN47]]:
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP65]], [[TMP66]]
-// CHECK2-NEXT:    [[ADD49:%.*]] = add i32 [[TMP64]], [[MUL48]]
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTOMP_LB2]], align 4
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTOMP_ST2]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[MUL48:%.*]] = mul i32 [[TMP62]], [[TMP63]]
+// CHECK2-NEXT:    [[ADD49:%.*]] = add i32 [[TMP61]], [[MUL48]]
 // CHECK2-NEXT:    store i32 [[ADD49]], ptr [[DOTOMP_IV2]], align 4
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
-// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP68]], [[TMP69]]
-// CHECK2-NEXT:    [[ADD51:%.*]] = add i32 [[TMP67]], [[MUL50]]
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTOMP_IV2]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP21]], align 4
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[TMP65]], [[TMP66]]
+// CHECK2-NEXT:    [[ADD51:%.*]] = add i32 [[TMP64]], [[MUL50]]
 // CHECK2-NEXT:    store i32 [[ADD51]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP70]])
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP67]])
 // CHECK2-NEXT:    br label %[[IF_END52]]
 // CHECK2:       [[IF_END52]]:
 // CHECK2-NEXT:    br label %[[FOR_INC:.*]]
 // CHECK2:       [[FOR_INC]]:
-// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP71]], 1
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTOMP_FUSE_INDEX]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP68]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTOMP_FUSE_INDEX]], align 4
-// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       [[FOR_END]]:
 // CHECK2-NEXT:    ret void
 //
@@ -1819,6 +2314,8 @@ extern "C" void foo4() {
 // CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
 // CHECK1: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
 // CHECK1: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
+// CHECK1: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
+// CHECK1: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
 //.
 // CHECK2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
 // CHECK2: [[META4]] = !{!"llvm.loop.mustprogress"}
@@ -1826,4 +2323,6 @@ extern "C" void foo4() {
 // CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
 // CHECK2: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
 // CHECK2: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
+// CHECK2: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
+// CHECK2: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
 //.

>From 860fcd94d930c9644b4d0427471f2873e7afcf8b Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:44:48 +0000
Subject: [PATCH 05/13] Fixed missing diagnostic groups in warnings

---
 clang/include/clang/Basic/DiagnosticSemaKinds.td | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 002aa7a774fbe..e85cd32d78b5c 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11613,7 +11613,8 @@ def note_omp_implicit_dsa : Note<
 def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
 def warn_omp_different_loop_ind_var_types : Warning <
-  "loop sequence following '#pragma omp %0' contains induction variables of differing types: %1 and %2">;
+  "loop sequence following '#pragma omp %0' contains induction variables of differing types: %1 and %2">,
+  InGroup<OpenMPLoopForm>;
 def err_omp_not_canonical_loop : Error <
   "loop after '#pragma omp %0' is not in canonical form">;
 def err_omp_not_a_loop_sequence : Error < 
@@ -11624,7 +11625,8 @@ def err_omp_invalid_looprange : Error <
   "loop range in '#pragma omp %0' exceeds the number of available loops: "
   "range end '%1' is greater than the total number of loops '%2'">;
 def warn_omp_redundant_fusion : Warning <
-  "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">;
+  "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">,
+  InGroup<OpenMPClauses>;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;

>From 65cbfeb945e6b8016696906db43dd590adb285b2 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:49:50 +0000
Subject: [PATCH 06/13] Fixed formatting and comments

---
 clang/lib/Sema/SemaOpenMP.cpp | 112 ++++++++++++++++++----------------
 1 file changed, 58 insertions(+), 54 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 3ce256f3ec23b..2985b256cf153 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14197,42 +14197,43 @@ StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
 }
 
 // Overloaded base case function
-template <typename T, typename F>
-static bool tryHandleAs(T *t, F &&) {
-    return false; 
+template <typename T, typename F> static bool tryHandleAs(T *t, F &&) {
+  return false;
 }
 
 /**
- * Tries to recursively cast `t` to one of the given types and invokes `f` if successful.
+ * Tries to recursively cast `t` to one of the given types and invokes `f` if
+ * successful.
  *
  * @tparam Class The first type to check.
  * @tparam Rest The remaining types to check.
  * @tparam T The base type of `t`.
- * @tparam F The callable type for the function to invoke upon a successful cast.
+ * @tparam F The callable type for the function to invoke upon a successful
+ * cast.
  * @param t The object to be checked.
  * @param f The function to invoke if `t` matches `Class`.
  * @return `true` if `t` matched any type and `f` was called, otherwise `false`.
  */
 template <typename Class, typename... Rest, typename T, typename F>
 static bool tryHandleAs(T *t, F &&f) {
-    if (Class *c = dyn_cast<Class>(t)) {
-        f(c); 
-        return true;
-    } else {
-        return tryHandleAs<Rest...>(t, std::forward<F>(f));
-    }
+  if (Class *c = dyn_cast<Class>(t)) {
+    f(c);
+    return true;
+  } else {
+    return tryHandleAs<Rest...>(t, std::forward<F>(f));
+  }
 }
 
 // Updates OriginalInits by checking Transform against loop transformation
 // directives and appending their pre-inits if a match is found.
 static void updatePreInits(OMPLoopBasedDirective *Transform,
                            SmallVectorImpl<SmallVector<Stmt *, 0>> &PreInits) {
-    if (!tryHandleAs<OMPTileDirective, OMPUnrollDirective, OMPReverseDirective,
-                     OMPInterchangeDirective, OMPFuseDirective>(
-            Transform, [&PreInits](auto *Dir) {
-              appendFlattenedStmtList(PreInits.back(), Dir->getPreInits());
-            }))
-        llvm_unreachable("Unhandled loop transformation");
+  if (!tryHandleAs<OMPTileDirective, OMPUnrollDirective, OMPReverseDirective,
+                   OMPInterchangeDirective, OMPFuseDirective>(
+          Transform, [&PreInits](auto *Dir) {
+            appendFlattenedStmtList(PreInits.back(), Dir->getPreInits());
+          }))
+    llvm_unreachable("Unhandled loop transformation");
 }
 
 bool SemaOpenMP::checkTransformableLoopNest(
@@ -14310,43 +14311,42 @@ class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
   unsigned getNestedLoopCount() const { return NestedLoopCount; }
 
   bool VisitForStmt(ForStmt *FS) override {
-        ++NestedLoopCount;
-        return true;
+    ++NestedLoopCount;
+    return true;
   }
 
   bool VisitCXXForRangeStmt(CXXForRangeStmt *FRS) override {
-        ++NestedLoopCount;
-        return true;
+    ++NestedLoopCount;
+    return true;
   }
 
   bool TraverseStmt(Stmt *S) override {
-        if (!S)
+    if (!S)
       return true;
 
-        // Skip traversal of all expressions, including special cases like
-        // LambdaExpr, StmtExpr, BlockExpr, and RequiresExpr. These expressions
-        // may contain inner statements (and even loops), but they are not part
-        // of the syntactic body of the surrounding loop structure.
-        //  Therefore must not be counted
-        if (isa<Expr>(S))
+    // Skip traversal of all expressions, including special cases like
+    // LambdaExpr, StmtExpr, BlockExpr, and RequiresExpr. These expressions
+    // may contain inner statements (and even loops), but they are not part
+    // of the syntactic body of the surrounding loop structure.
+    //  Therefore must not be counted
+    if (isa<Expr>(S))
       return true;
 
-        // Only recurse into CompoundStmt (block {}) and loop bodies
-        if (isa<CompoundStmt>(S) || isa<ForStmt>(S) ||
-            isa<CXXForRangeStmt>(S)) {
+    // Only recurse into CompoundStmt (block {}) and loop bodies
+    if (isa<CompoundStmt>(S) || isa<ForStmt>(S) || isa<CXXForRangeStmt>(S)) {
       return DynamicRecursiveASTVisitor::TraverseStmt(S);
-        }
+    }
 
-        // Stop traversal of the rest of statements, that break perfect
-        // loop nesting, such as control flow (IfStmt, SwitchStmt...)
-        return true;
+    // Stop traversal of the rest of statements, that break perfect
+    // loop nesting, such as control flow (IfStmt, SwitchStmt...)
+    return true;
   }
 
   bool TraverseDecl(Decl *D) override {
-        // Stop in the case of finding a declaration, it is not important
-        // in order to find nested loops (Possible CXXRecordDecl, RecordDecl,
-        // FunctionDecl...)
-        return true;
+    // Stop in the case of finding a declaration, it is not important
+    // in order to find nested loops (Possible CXXRecordDecl, RecordDecl,
+    // FunctionDecl...)
+    return true;
   }
 };
 
@@ -14504,15 +14504,14 @@ bool SemaOpenMP::analyzeLoopSequence(
     return isa<OMPLoopTransformationDirective>(Child);
   };
 
-
   // High level grammar validation
   for (auto *Child : LoopSeqStmt->children()) {
 
-        if (!Child)
+    if (!Child)
       continue;
 
-        // Skip over non-loop-sequence statements
-        if (!isLoopSequenceDerivation(Child)) {
+    // Skip over non-loop-sequence statements
+    if (!isLoopSequenceDerivation(Child)) {
       Child = Child->IgnoreContainers();
 
       // Ignore empty compound statement
@@ -14530,9 +14529,9 @@ bool SemaOpenMP::analyzeLoopSequence(
         // Already been treated, skip this children
         continue;
       }
-        }
-        // Regular loop sequence handling
-        if (isLoopSequenceDerivation(Child)) {
+    }
+    // Regular loop sequence handling
+    if (isLoopSequenceDerivation(Child)) {
       if (isLoopGeneratingStmt(Child)) {
         if (!analyzeLoopGeneration(Child)) {
           return false;
@@ -14546,12 +14545,12 @@ bool SemaOpenMP::analyzeLoopSequence(
         // Update the Loop Sequence size by one
         ++LoopSeqSize;
       }
-        } else {
+    } else {
       // Report error for invalid statement inside canonical loop sequence
       Diag(Child->getBeginLoc(), diag::err_omp_not_for)
           << 0 << getOpenMPDirectiveName(Kind);
       return false;
-        }
+    }
   }
   return true;
 }
@@ -14568,9 +14567,9 @@ bool SemaOpenMP::checkTransformableLoopSequence(
 
   // Checks whether the given statement is a compound statement
   if (!isa<CompoundStmt>(AStmt)) {
-        Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
-            << getOpenMPDirectiveName(Kind);
-        return false;
+    Diag(AStmt->getBeginLoc(), diag::err_omp_not_a_loop_sequence)
+        << getOpenMPDirectiveName(Kind);
+    return false;
   }
   // Number of top level canonical loop nests observed (And acts as index)
   LoopSeqSize = 0;
@@ -14601,7 +14600,7 @@ bool SemaOpenMP::checkTransformableLoopSequence(
                            OriginalInits, TransformsPreInits,
                            LoopSequencePreInits, LoopCategories, Context,
                            Kind)) {
-        return false;
+    return false;
   }
   if (LoopSeqSize <= 0) {
     Diag(AStmt->getBeginLoc(), diag::err_omp_empty_loop_sequence)
@@ -15315,7 +15314,7 @@ StmtResult SemaOpenMP::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   Stmt *LoopStmt = nullptr;
   collectLoopStmts(AStmt, {LoopStmt});
 
-  // Determine the PreInit declarations.e
+  // Determine the PreInit declarations.
   SmallVector<Stmt *, 4> PreInits;
   addLoopPreInits(Context, LoopHelper, LoopStmt, OriginalInits[0], PreInits);
 
@@ -15931,13 +15930,18 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     CountVal = CountInt.getZExtValue();
   };
 
-  // Checks if the loop range is valid
+  // OpenMP [6.0, Restrictions]
+  // first + count - 1 must not evaluate to a value greater than the
+  // loop sequence length of the associated canonical loop sequence.
   auto ValidLoopRange = [](uint64_t FirstVal, uint64_t CountVal,
                            unsigned NumLoops) -> bool {
     return FirstVal + CountVal - 1 <= NumLoops;
   };
   uint64_t FirstVal = 1, CountVal = 0, LastVal = LoopSeqSize;
 
+  // Validates the loop range after evaluating the semantic information
+  // and ensures that the range is valid for the given loop sequence size.
+  // Expressions are evaluated at compile time to obtain constant values.
   if (LRC) {
     EvaluateLoopRangeArguments(LRC->getFirst(), LRC->getCount(), FirstVal,
                                CountVal);

>From b0fb1b3e26f1d9ceaac4495dcfad84f54f96d2a2 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 9 May 2025 10:58:54 +0000
Subject: [PATCH 07/13] Added minimal changes to enable flang future
 implementation

---
 flang/include/flang/Parser/dump-parse-tree.h | 1 +
 flang/include/flang/Parser/parse-tree.h      | 9 +++++++++
 flang/lib/Lower/OpenMP/Clauses.cpp           | 5 +++++
 flang/lib/Lower/OpenMP/Clauses.h             | 1 +
 flang/lib/Parser/openmp-parsers.cpp          | 7 +++++++
 flang/lib/Parser/unparse.cpp                 | 7 +++++++
 flang/lib/Semantics/check-omp-structure.cpp  | 9 +++++++++
 llvm/include/llvm/Frontend/OpenMP/OMP.td     | 1 +
 8 files changed, 40 insertions(+)

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..76aa3f7b90156 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -600,6 +600,7 @@ class ParseTreeDumper {
   NODE(OmpLinearClause, Modifier)
   NODE(parser, OmpLinearModifier)
   NODE_ENUM(OmpLinearModifier, Value)
+  NODE(parser, OmpLoopRangeClause)
   NODE(parser, OmpStepComplexModifier)
   NODE(parser, OmpStepSimpleModifier)
   NODE(parser, OmpLoopDirective)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 61f97b855b0e5..d32db62db2628 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4367,6 +4367,15 @@ struct OmpLinearClause {
   std::tuple<OmpObjectList, MODIFIERS(), /*PostModified=*/bool> t;
 };
 
+// Ref: [6.0:207-208]
+//
+// loop-range-clause ->
+//    LOOPRANGE(first, count)                       // since 6.0
+struct OmpLoopRangeClause {
+  TUPLE_CLASS_BOILERPLATE(OmpLoopRangeClause);
+  std::tuple<ScalarIntConstantExpr, ScalarIntConstantExpr> t;
+};
+
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-clause ->
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index b599d69a36272..a38249bf2b588 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -997,6 +997,11 @@ Link make(const parser::OmpClause::Link &inp,
   return Link{/*List=*/makeObjects(inp.v, semaCtx)};
 }
 
+LoopRange make(const parser::OmpClause::Looprange &inp,
+            semantics::SemanticsContext &semaCtx) {
+  llvm_unreachable("Unimplemented: looprange");
+}
+
 Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index d7ab21d428e32..bda8571e65f23 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -239,6 +239,7 @@ using Initializer = tomp::clause::InitializerT<TypeTy, IdTy, ExprTy>;
 using InReduction = tomp::clause::InReductionT<TypeTy, IdTy, ExprTy>;
 using IsDevicePtr = tomp::clause::IsDevicePtrT<TypeTy, IdTy, ExprTy>;
 using Lastprivate = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
+using LoopRange = tomp::clause::LoopRangeT<TypeTy, IdTy, ExprTy>;
 using Linear = tomp::clause::LinearT<TypeTy, IdTy, ExprTy>;
 using Link = tomp::clause::LinkT<TypeTy, IdTy, ExprTy>;
 using Map = tomp::clause::MapT<TypeTy, IdTy, ExprTy>;
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..d53389746dbec 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -847,6 +847,11 @@ TYPE_PARSER(
         maybe(":"_tok >> nonemptyList(Parser<OmpLinearClause::Modifier>{})),
         /*PostModified=*/pure(true)))
 
+TYPE_PARSER(
+  construct<OmpLoopRangeClause>(scalarIntConstantExpr,
+                                "," >> scalarIntConstantExpr)
+)
+
 // OpenMPv5.2 12.5.2 detach-clause -> DETACH (event-handle)
 TYPE_PARSER(construct<OmpDetachClause>(Parser<OmpObject>{}))
 
@@ -1020,6 +1025,8 @@ TYPE_PARSER( //
                     parenthesized(Parser<OmpLinearClause>{}))) ||
     "LINK" >> construct<OmpClause>(construct<OmpClause::Link>(
                   parenthesized(Parser<OmpObjectList>{}))) ||
+    "LOOPRANGE" >> construct<OmpClause>(construct<OmpClause::Looprange>(
+                  parenthesized(Parser<OmpLoopRangeClause>{}))) ||
     "MAP" >> construct<OmpClause>(construct<OmpClause::Map>(
                  parenthesized(Parser<OmpMapClause>{}))) ||
     "MATCH" >> construct<OmpClause>(construct<OmpClause::Match>(
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ed0f227fd5b98..18e8a63ca68aa 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2318,6 +2318,13 @@ class UnparseVisitor {
       }
     }
   }
+  void Unparse(const OmpLoopRangeClause &x) {
+    Word("LOOPRANGE(");
+    Walk(std::get<0>(x.t));
+    Put(", ");
+    Walk(std::get<1>(x.t));
+    Put(")");
+  }
   void Unparse(const OmpReductionClause &x) {
     using Modifier = OmpReductionClause::Modifier;
     Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 68cea6739830d..2a03c6a1fd0e4 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -4446,6 +4446,15 @@ CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(Collapse, OMPC_collapse)
 CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(Safelen, OMPC_safelen)
 CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(Simdlen, OMPC_simdlen)
 
+void OmpStructureChecker::Enter(const parser::OmpClause::Looprange &x) {
+  context_.Say(GetContext().clauseSource,
+      "LOOPRANGE clause is not implemented yet"_err_en_US,
+      ContextDirectiveAsFortran());
+}
+
+void OmpStructureChecker::Enter(const parser::OmpClause::FreeAgent &x) {
+  context_.Say(GetContext().clauseSource,
+      "FREE_AGENT clause is not implemented yet"_err_en_US,
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 989b35a7caa2a..f8acdc62aba3d 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -276,6 +276,7 @@ def OMPC_Link : Clause<[Spelling<"link">]> {
 }
 def OMPC_LoopRange : Clause<[Spelling<"looprange">]> {
   let clangClass = "OMPLoopRangeClause";
+  let flangClass = "OmpLoopRangeClause";
 }
 def OMPC_Map : Clause<[Spelling<"map">]> {
   let clangClass = "OMPMapClause";

>From b252aa910ef7c5c278a86bd7195bbf3bb18dd18d Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Wed, 21 May 2025 13:14:22 +0000
Subject: [PATCH 08/13] Address basic PR feedback

---
 clang/include/clang/AST/OpenMPClause.h      |  93 ++++----
 clang/include/clang/AST/StmtOpenMP.h        |   2 +-
 clang/include/clang/Sema/SemaOpenMP.h       |  14 +-
 clang/lib/AST/OpenMPClause.cpp              |  17 +-
 clang/lib/CodeGen/CGExpr.cpp                |   5 +-
 clang/lib/CodeGen/CodeGenFunction.h         |   4 -
 clang/lib/Sema/SemaOpenMP.cpp               | 224 +++++++++-----------
 flang/lib/Semantics/check-omp-structure.cpp |   3 -
 8 files changed, 166 insertions(+), 196 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8f937cdef9cd0..3df5133a17fb4 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1153,82 +1153,73 @@ class OMPFullClause final : public OMPNoChildClause<llvm::omp::OMPC_full> {
 ///   for(int j = 0; j < 256; j+=2)
 ///   for(int k = 127; k >= 0; --k)
 /// \endcode
-class OMPLoopRangeClause final : public OMPClause {
+class OMPLoopRangeClause final
+    : public OMPClause,
+      private llvm::TrailingObjects<OMPLoopRangeClause, Expr *> {
   friend class OMPClauseReader;
-
-  explicit OMPLoopRangeClause()
-      : OMPClause(llvm::omp::OMPC_looprange, {}, {}) {}
+  friend class llvm::TrailingObjects<OMPLoopRangeClause, Expr *>;
 
   /// Location of '('
   SourceLocation LParenLoc;
 
-  /// Location of 'first'
-  SourceLocation FirstLoc;
-
-  /// Location of 'count'
-  SourceLocation CountLoc;
-
-  /// Expr associated with 'first' argument
-  Expr *First = nullptr;
-
-  /// Expr associated with 'count' argument
-  Expr *Count = nullptr;
-
-  /// Set 'first'
-  void setFirst(Expr *First) { this->First = First; }
+  /// Location of first and count expressions
+  SourceLocation FirstLoc, CountLoc;
 
-  /// Set 'count'
-  void setCount(Expr *Count) { this->Count = Count; }
+  /// Number of looprange arguments (always 2: first, count)
+  unsigned NumArgs = 2;
 
-  /// Set location of '('.
-  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
-
-  /// Set location of 'first' argument
-  void setFirstLoc(SourceLocation Loc) { FirstLoc = Loc; }
+  /// Set the argument expressions.
+  void setArgs(ArrayRef<Expr *> Args) {
+    assert(Args.size() == NumArgs && "Expected exactly 2 looprange arguments");
+    std::copy(Args.begin(), Args.end(), getTrailingObjects<Expr *>());
+  }
 
-  /// Set location of 'count' argument
-  void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
+  /// Build an empty clause for deserialization.
+  explicit OMPLoopRangeClause()
+      : OMPClause(llvm::omp::OMPC_looprange, {}, {}), NumArgs(2) {}
 
 public:
-  /// Build an AST node for a 'looprange' clause
-  ///
-  /// \param StartLoc     Starting location of the clause.
-  /// \param LParenLoc    Location of '('.
-  /// \param ModifierLoc  Modifier location.
-  /// \param
+  /// Build a 'looprange' clause AST node.
   static OMPLoopRangeClause *
   Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
          SourceLocation FirstLoc, SourceLocation CountLoc,
-         SourceLocation EndLoc, Expr *First, Expr *Count);
+         SourceLocation EndLoc, ArrayRef<Expr *> Args);
 
-  /// Build an empty 'looprange' node for deserialization
-  ///
-  /// \param C      Context of the AST.
+  /// Build an empty 'looprange' clause node.
   static OMPLoopRangeClause *CreateEmpty(const ASTContext &C);
 
-  /// Returns the location of '('
+  // Location getters/setters
   SourceLocation getLParenLoc() const { return LParenLoc; }
-
-  /// Returns the location of 'first'
   SourceLocation getFirstLoc() const { return FirstLoc; }
-
-  /// Returns the location of 'count'
   SourceLocation getCountLoc() const { return CountLoc; }
 
-  /// Returns the argument 'first' or nullptr if not set
-  Expr *getFirst() const { return cast_or_null<Expr>(First); }
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+  void setFirstLoc(SourceLocation Loc) { FirstLoc = Loc; }
+  void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
 
-  /// Returns the argument 'count' or nullptr if not set
-  Expr *getCount() const { return cast_or_null<Expr>(Count); }
+  /// Get looprange arguments: first and count
+  Expr *getFirst() const { return getArgs()[0]; }
+  Expr *getCount() const { return getArgs()[1]; }
 
-  child_range children() {
-    return child_range(reinterpret_cast<Stmt **>(&First),
-                       reinterpret_cast<Stmt **>(&Count) + 1);
+  /// Set looprange arguments: first and count
+  void setFirst(Expr *E) { getArgs()[0] = E; }
+  void setCount(Expr *E) { getArgs()[1] = E; }
+
+  MutableArrayRef<Expr *> getArgs() {
+    return MutableArrayRef<Expr *>(getTrailingObjects<Expr *>(), NumArgs);
+  }
+  ArrayRef<Expr *> getArgs() const {
+    return ArrayRef<Expr *>(getTrailingObjects<Expr *>(), NumArgs);
   }
 
+  child_range children() {
+    return child_range(reinterpret_cast<Stmt **>(getArgs().begin()),
+                       reinterpret_cast<Stmt **>(getArgs().end()));
+  }
   const_child_range children() const {
-    auto Children = const_cast<OMPLoopRangeClause *>(this)->children();
-    return const_child_range(Children.begin(), Children.end());
+    auto AR = getArgs();
+    return const_child_range(reinterpret_cast<Stmt *const *>(AR.begin()),
+                             reinterpret_cast<Stmt *const *>(AR.end()));
   }
 
   child_range used_children() {
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 6425f6616a558..0421c06245cac 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5883,7 +5883,7 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
                                        EndLoc, NumLoops) {
     // Interchange produces a single top-level canonical loop
     // nest, with the exact same amount of total loops
-    setNumGeneratedLoops(NumLoops);
+    setNumGeneratedLoops(3 * NumLoops);
     setNumGeneratedLoopNests(1);
   }
 
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 547ea95c6cd5d..f848c4a7d715e 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1492,7 +1492,7 @@ class SemaOpenMP : public SemaBase {
   bool checkTransformableLoopNest(
       OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
-      Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits);
+      Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits);
 
   /// @brief Categories of loops encountered during semantic OpenMP loop
   /// analysis
@@ -1555,9 +1555,9 @@ class SemaOpenMP : public SemaBase {
       Stmt *LoopSeqStmt, unsigned &LoopSeqSize, unsigned &NumLoops,
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       SmallVectorImpl<Stmt *> &ForStmts,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &TransformsPreInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &LoopSequencePreInits,
       SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context,
       OpenMPDirectiveKind Kind);
 
@@ -1591,9 +1591,9 @@ class SemaOpenMP : public SemaBase {
       unsigned &NumLoops,
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       SmallVectorImpl<Stmt *> &ForStmts,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
-      SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &TransformsPreInits,
+      SmallVectorImpl<SmallVector<Stmt *>> &LoopSequencePreInits,
       SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context);
 
   /// Helper to keep information about the current `omp begin/end declare
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 0b5808eb100e4..e0570262b2a05 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1026,22 +1026,25 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
 
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
-                           SourceLocation LParenLoc, SourceLocation EndLoc,
-                           SourceLocation FirstLoc, SourceLocation CountLoc,
-                           Expr *First, Expr *Count) {
+                           SourceLocation LParenLoc, SourceLocation FirstLoc,
+                           SourceLocation CountLoc, SourceLocation EndLoc,
+                           ArrayRef<Expr *> Args) {
+
+  assert(Args.size() == 2 &&
+         "looprange clause must have exactly two arguments");
   OMPLoopRangeClause *Clause = CreateEmpty(C);
   Clause->setLocStart(StartLoc);
   Clause->setLParenLoc(LParenLoc);
-  Clause->setLocEnd(EndLoc);
   Clause->setFirstLoc(FirstLoc);
   Clause->setCountLoc(CountLoc);
-  Clause->setFirst(First);
-  Clause->setCount(Count);
+  Clause->setLocEnd(EndLoc);
+  Clause->setArgs(Args);
   return Clause;
 }
 
 OMPLoopRangeClause *OMPLoopRangeClause::CreateEmpty(const ASTContext &C) {
-  return new (C) OMPLoopRangeClause();
+  void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(2));
+  return new (Mem) OMPLoopRangeClause();
 }
 
 OMPAllocateClause *OMPAllocateClause::Create(
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 08049d4d4e37d..f983b88eb61ec 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3256,11 +3256,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
           var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
 
     // No other cases for now.
-    } else {
-      llvm::dbgs() << "THE DAMN DECLREFEXPR HASN'T BEEN ENTERED IN LOCALDECLMAP\n";
-      VD->dumpColor();
+    } else
       llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
-    }
 
     // Handle threadlocal function locals.
     if (VD->getTLSKind() != VarDecl::TLS_None)
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index bfe24213ed377..fe753e5b688b1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5363,10 +5363,6 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// Set the address of a local variable.
   void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
-    if (LocalDeclMap.count(VD)) {
-      llvm::errs() << "Warning: VarDecl already exists in map: ";
-      VD->dumpColor(); 
-    }
     assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
     LocalDeclMap.insert({VD, Addr});
   }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 2985b256cf153..9819dcfe60360 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14196,38 +14196,37 @@ StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
       getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-// Overloaded base case function
+/// Overloaded base case function
 template <typename T, typename F> static bool tryHandleAs(T *t, F &&) {
   return false;
 }
 
-/**
- * Tries to recursively cast `t` to one of the given types and invokes `f` if
- * successful.
- *
- * @tparam Class The first type to check.
- * @tparam Rest The remaining types to check.
- * @tparam T The base type of `t`.
- * @tparam F The callable type for the function to invoke upon a successful
- * cast.
- * @param t The object to be checked.
- * @param f The function to invoke if `t` matches `Class`.
- * @return `true` if `t` matched any type and `f` was called, otherwise `false`.
- */
+///
+/// Tries to recursively cast `t` to one of the given types and invokes `f` if
+/// successful.
+///
+/// @tparam Class The first type to check.
+/// @tparam Rest The remaining types to check.
+/// @tparam T The base type of `t`.
+/// @tparam F The callable type for the function to invoke upon a successful
+/// cast.
+/// @param t The object to be checked.
+/// @param f The function to invoke if `t` matches `Class`.
+/// @return `true` if `t` matched any type and `f` was called, otherwise
+/// `false`.
 template <typename Class, typename... Rest, typename T, typename F>
 static bool tryHandleAs(T *t, F &&f) {
   if (Class *c = dyn_cast<Class>(t)) {
     f(c);
     return true;
-  } else {
-    return tryHandleAs<Rest...>(t, std::forward<F>(f));
   }
+  return tryHandleAs<Rest...>(t, std::forward<F>(f));
 }
 
-// Updates OriginalInits by checking Transform against loop transformation
-// directives and appending their pre-inits if a match is found.
+/// Updates OriginalInits by checking Transform against loop transformation
+/// directives and appending their pre-inits if a match is found.
 static void updatePreInits(OMPLoopBasedDirective *Transform,
-                           SmallVectorImpl<SmallVector<Stmt *, 0>> &PreInits) {
+                           SmallVectorImpl<SmallVector<Stmt *>> &PreInits) {
   if (!tryHandleAs<OMPTileDirective, OMPUnrollDirective, OMPReverseDirective,
                    OMPInterchangeDirective, OMPFuseDirective>(
           Transform, [&PreInits](auto *Dir) {
@@ -14239,7 +14238,7 @@ static void updatePreInits(OMPLoopBasedDirective *Transform,
 bool SemaOpenMP::checkTransformableLoopNest(
     OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
-    Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits) {
+    Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits) {
   OriginalInits.emplace_back();
   bool Result = OMPLoopBasedDirective::doForAllLoops(
       AStmt->IgnoreContainers(), /*TryImperfectlyNestedLoops=*/false, NumLoops,
@@ -14273,40 +14272,40 @@ bool SemaOpenMP::checkTransformableLoopNest(
   return Result;
 }
 
-// Counts the total number of nested loops, including the outermost loop (the
-// original loop). PRECONDITION of this visitor is that it must be invoked from
-// the original loop to be analyzed. The traversal is stop for Decl's and
-// Expr's given that they may contain inner loops that must not be counted.
-//
-// Example AST structure for the code:
-//
-// int main() {
-//     #pragma omp fuse
-//     {
-//         for (int i = 0; i < 100; i++) {    <-- Outer loop
-//             []() {
-//                 for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
-//             };
-//             for(int j = 0; j < 5; ++j) {}    <-- Inner loop
-//         }
-//         for (int r = 0; i < 100; i++) {    <-- Outer loop
-//             struct LocalClass {
-//                 void bar() {
-//                     for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
-//                 }
-//             };
-//             for(int k = 0; k < 10; ++k) {}    <-- Inner loop
-//             {x = 5; for(k = 0; k < 10; ++k) x += k; x}; <-- NOT A LOOP
-//         }
-//     }
-// }
-// Result: Loop 'i' contains 2 loops, Loop 'r' also contains 2 loops
+/// Counts the total number of nested loops, including the outermost loop (the
+/// original loop). PRECONDITION of this visitor is that it must be invoked from
+/// the original loop to be analyzed. The traversal is stop for Decl's and
+/// Expr's given that they may contain inner loops that must not be counted.
+///
+/// Example AST structure for the code:
+///
+/// int main() {
+///     #pragma omp fuse
+///     {
+///         for (int i = 0; i < 100; i++) {    <-- Outer loop
+///             []() {
+///                 for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
+///             };
+///             for(int j = 0; j < 5; ++j) {}    <-- Inner loop
+///         }
+///         for (int r = 0; i < 100; i++) {    <-- Outer loop
+///             struct LocalClass {
+///                 void bar() {
+///                     for(int j = 0; j < 100; j++) {}  <-- NOT A LOOP
+///                 }
+///             };
+///             for(int k = 0; k < 10; ++k) {}    <-- Inner loop
+///             {x = 5; for(k = 0; k < 10; ++k) x += k; x}; <-- NOT A LOOP
+///         }
+///     }
+/// }
+/// Result: Loop 'i' contains 2 loops, Loop 'r' also contains 2 loops
 class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
 private:
   unsigned NestedLoopCount = 0;
 
 public:
-  explicit NestedLoopCounterVisitor() {}
+  explicit NestedLoopCounterVisitor() = default;
 
   unsigned getNestedLoopCount() const { return NestedLoopCount; }
 
@@ -14333,7 +14332,7 @@ class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
       return true;
 
     // Only recurse into CompoundStmt (block {}) and loop bodies
-    if (isa<CompoundStmt>(S) || isa<ForStmt>(S) || isa<CXXForRangeStmt>(S)) {
+    if (isa<CompoundStmt, ForStmt, CXXForRangeStmt>(S)) {
       return DynamicRecursiveASTVisitor::TraverseStmt(S);
     }
 
@@ -14354,19 +14353,18 @@ bool SemaOpenMP::analyzeLoopSequence(
     Stmt *LoopSeqStmt, unsigned &LoopSeqSize, unsigned &NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
     SmallVectorImpl<Stmt *> &ForStmts,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &TransformsPreInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &LoopSequencePreInits,
     SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context,
     OpenMPDirectiveKind Kind) {
 
   VarsWithInheritedDSAType TmpDSA;
   QualType BaseInductionVarType;
-  // Helper Lambda to handle storing initialization and body statements for both
-  // ForStmt and CXXForRangeStmt and checks for any possible mismatch between
-  // induction variables types
-  auto storeLoopStatements = [&OriginalInits, &ForStmts, &BaseInductionVarType,
-                              this, &Context](Stmt *LoopStmt) {
+  /// Helper Lambda to handle storing initialization and body statements for
+  /// both ForStmt and CXXForRangeStmt and checks for any possible mismatch
+  /// between induction variables types
+  auto StoreLoopStatements = [&](Stmt *LoopStmt) {
     if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
       OriginalInits.back().push_back(For->getInit());
       ForStmts.push_back(For);
@@ -14394,16 +14392,11 @@ bool SemaOpenMP::analyzeLoopSequence(
     }
   };
 
-  // Helper lambda functions to encapsulate the processing of different
-  // derivations of the canonical loop sequence grammar
-  //
-  // Modularized code for handling loop generation and transformations
-  auto analyzeLoopGeneration = [&storeLoopStatements, &LoopHelpers,
-                                &OriginalInits, &TransformsPreInits,
-                                &LoopCategories, &LoopSeqSize, &NumLoops, Kind,
-                                &TmpDSA, &ForStmts, &Context,
-                                &LoopSequencePreInits, this](Stmt *Child) {
-    auto LoopTransform = dyn_cast<OMPLoopTransformationDirective>(Child);
+  /// Helper lambda functions to encapsulate the processing of different
+  /// derivations of the canonical loop sequence grammar
+  /// Modularized code for handling loop generation and transformations
+  auto AnalyzeLoopGeneration = [&](Stmt *Child) {
+    auto *LoopTransform = dyn_cast<OMPLoopTransformationDirective>(Child);
     Stmt *TransformedStmt = LoopTransform->getTransformedStmt();
     unsigned NumGeneratedLoopNests = LoopTransform->getNumGeneratedLoopNests();
     unsigned NumGeneratedLoops = LoopTransform->getNumGeneratedLoops();
@@ -14414,9 +14407,8 @@ bool SemaOpenMP::analyzeLoopSequence(
         LoopSeqSize += NumGeneratedLoopNests;
         NumLoops += NumGeneratedLoops;
         return true;
-      }
-      // Unroll full (0 loops produced)
-      else {
+      } else {
+        // Unroll full (0 loops produced)
         Diag(Child->getBeginLoc(), diag::err_omp_not_for)
             << 0 << getOpenMPDirectiveName(Kind);
         return false;
@@ -14443,9 +14435,8 @@ bool SemaOpenMP::analyzeLoopSequence(
                                  LoopHelpers, ForStmts, OriginalInits,
                                  TransformsPreInits, LoopSequencePreInits,
                                  LoopCategories, Context, Kind);
-    }
-    // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
-    else {
+    } else {
+      // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
       // Process the transformed loop statement
       OriginalInits.emplace_back();
       TransformsPreInits.emplace_back();
@@ -14461,7 +14452,7 @@ bool SemaOpenMP::analyzeLoopSequence(
             << getOpenMPDirectiveName(Kind);
         return false;
       }
-      storeLoopStatements(TransformedStmt);
+      StoreLoopStatements(TransformedStmt);
       updatePreInits(LoopTransform, TransformsPreInits);
 
       NumLoops += NumGeneratedLoops;
@@ -14470,10 +14461,8 @@ bool SemaOpenMP::analyzeLoopSequence(
     }
   };
 
-  // Modularized code for handling regular canonical loops
-  auto analyzeRegularLoop = [&storeLoopStatements, &LoopHelpers, &OriginalInits,
-                             &LoopSeqSize, &NumLoops, Kind, &TmpDSA,
-                             &LoopCategories, this](Stmt *Child) {
+  /// Modularized code for handling regular canonical loops
+  auto AnalyzeRegularLoop = [&](Stmt *Child) {
     OriginalInits.emplace_back();
     LoopHelpers.emplace_back();
     LoopCategories.push_back(OMPLoopCategory::RegularLoop);
@@ -14488,19 +14477,19 @@ bool SemaOpenMP::analyzeLoopSequence(
       return false;
     }
 
-    storeLoopStatements(Child);
+    StoreLoopStatements(Child);
     auto NLCV = NestedLoopCounterVisitor();
     NLCV.TraverseStmt(Child);
     NumLoops += NLCV.getNestedLoopCount();
     return true;
   };
 
-  // Helper functions to validate canonical loop sequence grammar is valid
-  auto isLoopSequenceDerivation = [](auto *Child) {
-    return isa<ForStmt>(Child) || isa<CXXForRangeStmt>(Child) ||
-           isa<OMPLoopTransformationDirective>(Child);
+  /// Helper functions to validate loop sequence grammar derivations
+  auto IsLoopSequenceDerivation = [](auto *Child) {
+    return isa<ForStmt, CXXForRangeStmt, OMPLoopTransformationDirective>(Child);
   };
-  auto isLoopGeneratingStmt = [](auto *Child) {
+  /// Helper functions to validate loop generating grammar derivations
+  auto IsLoopGeneratingStmt = [](auto *Child) {
     return isa<OMPLoopTransformationDirective>(Child);
   };
 
@@ -14511,7 +14500,7 @@ bool SemaOpenMP::analyzeLoopSequence(
       continue;
 
     // Skip over non-loop-sequence statements
-    if (!isLoopSequenceDerivation(Child)) {
+    if (!IsLoopSequenceDerivation(Child)) {
       Child = Child->IgnoreContainers();
 
       // Ignore empty compound statement
@@ -14531,17 +14520,17 @@ bool SemaOpenMP::analyzeLoopSequence(
       }
     }
     // Regular loop sequence handling
-    if (isLoopSequenceDerivation(Child)) {
-      if (isLoopGeneratingStmt(Child)) {
-        if (!analyzeLoopGeneration(Child)) {
+    if (IsLoopSequenceDerivation(Child)) {
+      if (IsLoopGeneratingStmt(Child)) {
+        if (!AnalyzeLoopGeneration(Child))
           return false;
-        }
-        // analyzeLoopGeneration updates Loop Sequence size accordingly
+
+        // AnalyzeLoopGeneration updates Loop Sequence size accordingly
 
       } else {
-        if (!analyzeRegularLoop(Child)) {
+        if (!AnalyzeRegularLoop(Child))
           return false;
-        }
+
         // Update the Loop Sequence size by one
         ++LoopSeqSize;
       }
@@ -14560,9 +14549,9 @@ bool SemaOpenMP::checkTransformableLoopSequence(
     unsigned &NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
     SmallVectorImpl<Stmt *> &ForStmts,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &OriginalInits,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &TransformsPreInits,
-    SmallVectorImpl<SmallVector<Stmt *, 0>> &LoopSequencePreInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &TransformsPreInits,
+    SmallVectorImpl<SmallVector<Stmt *>> &LoopSequencePreInits,
     SmallVectorImpl<OMPLoopCategory> &LoopCategories, ASTContext &Context) {
 
   // Checks whether the given statement is a compound statement
@@ -14598,10 +14587,9 @@ bool SemaOpenMP::checkTransformableLoopSequence(
   // Recursive entry point to process the main loop sequence
   if (!analyzeLoopSequence(AStmt, LoopSeqSize, NumLoops, LoopHelpers, ForStmts,
                            OriginalInits, TransformsPreInits,
-                           LoopSequencePreInits, LoopCategories, Context,
-                           Kind)) {
+                           LoopSequencePreInits, LoopCategories, Context, Kind))
     return false;
-  }
+
   if (LoopSeqSize <= 0) {
     Diag(AStmt->getBeginLoc(), diag::err_omp_empty_loop_sequence)
         << getOpenMPDirectiveName(Kind);
@@ -14693,7 +14681,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Verify and diagnose loop nest.
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
   Stmt *Body = nullptr;
-  SmallVector<SmallVector<Stmt *, 0>, 4> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_tile, AStmt, NumLoops, LoopHelpers, Body,
                                   OriginalInits))
     return StmtError();
@@ -14970,7 +14958,7 @@ StmtResult SemaOpenMP::ActOnOpenMPStripeDirective(ArrayRef<OMPClause *> Clauses,
   // Verify and diagnose loop nest.
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
   Stmt *Body = nullptr;
-  SmallVector<SmallVector<Stmt *, 0>, 4> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_stripe, AStmt, NumLoops, LoopHelpers,
                                   Body, OriginalInits))
     return StmtError();
@@ -15231,7 +15219,7 @@ StmtResult SemaOpenMP::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   Stmt *Body = nullptr;
   SmallVector<OMPLoopBasedDirective::HelperExprs, NumLoops> LoopHelpers(
       NumLoops);
-  SmallVector<SmallVector<Stmt *, 0>, NumLoops + 1> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, NumLoops + 1> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_unroll, AStmt, NumLoops, LoopHelpers,
                                   Body, OriginalInits))
     return StmtError();
@@ -15499,7 +15487,7 @@ StmtResult SemaOpenMP::ActOnOpenMPReverseDirective(Stmt *AStmt,
   Stmt *Body = nullptr;
   SmallVector<OMPLoopBasedDirective::HelperExprs, NumLoops> LoopHelpers(
       NumLoops);
-  SmallVector<SmallVector<Stmt *, 0>, NumLoops + 1> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, NumLoops + 1> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_reverse, AStmt, NumLoops, LoopHelpers,
                                   Body, OriginalInits))
     return StmtError();
@@ -15691,7 +15679,7 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
   // Verify and diagnose loop nest.
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
   Stmt *Body = nullptr;
-  SmallVector<SmallVector<Stmt *, 0>, 2> OriginalInits;
+  SmallVector<SmallVector<Stmt *>, 2> OriginalInits;
   if (!checkTransformableLoopNest(OMPD_interchange, AStmt, NumLoops,
                                   LoopHelpers, Body, OriginalInits))
     return StmtError();
@@ -15878,9 +15866,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   CaptureVars CopyTransformer(SemaRef);
 
   // Ensure the structured block is not empty
-  if (!AStmt) {
+  if (!AStmt)
     return StmtError();
-  }
 
   unsigned NumLoops = 1;
   unsigned LoopSeqSize = 1;
@@ -15899,16 +15886,15 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   // Also collect the HelperExprs, Loop Stmts, Inits, and Number of loops
   SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers;
   SmallVector<Stmt *> LoopStmts;
-  SmallVector<SmallVector<Stmt *, 0>> OriginalInits;
-  SmallVector<SmallVector<Stmt *, 0>> TransformsPreInits;
-  SmallVector<SmallVector<Stmt *, 0>> LoopSequencePreInits;
+  SmallVector<SmallVector<Stmt *>> OriginalInits;
+  SmallVector<SmallVector<Stmt *>> TransformsPreInits;
+  SmallVector<SmallVector<Stmt *>> LoopSequencePreInits;
   SmallVector<OMPLoopCategory, 0> LoopCategories;
   if (!checkTransformableLoopSequence(OMPD_fuse, AStmt, LoopSeqSize, NumLoops,
                                       LoopHelpers, LoopStmts, OriginalInits,
                                       TransformsPreInits, LoopSequencePreInits,
-                                      LoopCategories, Context)) {
+                                      LoopCategories, Context))
     return StmtError();
-  }
 
   // Handle clauses, which can be any of the following: [looprange, apply]
   const OMPLoopRangeClause *LRC =
@@ -15998,9 +15984,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   // expressions. Generates both the variable declaration and the corresponding
   // initialization statement.
   auto CreateHelperVarAndStmt =
-      [&SemaRef = this->SemaRef, &Context, &CopyTransformer,
-       &IVType](Expr *ExprToCopy, const std::string &BaseName, unsigned I,
-                bool NeedsNewVD = false) {
+      [&, &SemaRef = SemaRef](Expr *ExprToCopy, const std::string &BaseName,
+                              unsigned I, bool NeedsNewVD = false) {
         Expr *TransformedExpr =
             AssertSuccess(CopyTransformer.TransformExpr(ExprToCopy));
         if (!TransformedExpr)
@@ -16044,9 +16029,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   // Transformations that apply this concept: Loopranged Fuse, Split
   if (!LoopSequencePreInits.empty()) {
     for (const auto &LTPreInits : LoopSequencePreInits) {
-      if (!LTPreInits.empty()) {
+      if (!LTPreInits.empty())
         llvm::append_range(PreInits, LTPreInits);
-      }
     }
   }
 
@@ -16075,9 +16059,9 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
       // Order matters: pre-inits may define variables used in the original
       // inits such as upper bounds...
       auto TransformPreInit = TransformsPreInits[TransformIndex++];
-      if (!TransformPreInit.empty()) {
+      if (!TransformPreInit.empty())
         llvm::append_range(PreInits, TransformPreInit);
-      }
+
       addLoopPreInits(Context, LoopHelpers[I], LoopStmts[I], OriginalInits[I],
                       PreInits);
     }
@@ -17496,13 +17480,15 @@ OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
   if (CountVal.isInvalid())
     Count = nullptr;
 
+  SmallVector<Expr *, 2> ArgsVec = {First, Count};
+
   // OpenMP [6.0, Restrictions]
   // first + count - 1 must not evaluate to a value greater than the
   // loop sequence length of the associated canonical loop sequence.
   // This check must be performed afterwards due to the delayed
   // parsing and computation of the associated loop sequence
   return OMPLoopRangeClause::Create(getASTContext(), StartLoc, LParenLoc,
-                                    FirstLoc, CountLoc, EndLoc, First, Count);
+                                    FirstLoc, CountLoc, EndLoc, ArgsVec);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPAlignClause(Expr *A, SourceLocation StartLoc,
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2a03c6a1fd0e4..ac4883c4f2a18 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -4452,9 +4452,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Looprange &x) {
       ContextDirectiveAsFortran());
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::FreeAgent &x) {
-  context_.Say(GetContext().clauseSource,
-      "FREE_AGENT clause is not implemented yet"_err_en_US,
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
 

>From e294777879dd46c1859a03c307e70dd03abe11b7 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Thu, 22 May 2025 10:39:39 +0000
Subject: [PATCH 09/13] Removed unncessary warning and updated tests
 accordingly

---
 .../clang/Basic/DiagnosticSemaKinds.td        |  3 --
 clang/lib/Sema/SemaOpenMP.cpp                 | 21 +--------
 clang/test/OpenMP/fuse_messages.cpp           | 43 +++++++++++++++----
 3 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index e85cd32d78b5c..2bd0f895204c9 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11612,9 +11612,6 @@ def note_omp_implicit_dsa : Note<
   "implicitly determined as %0">;
 def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
-def warn_omp_different_loop_ind_var_types : Warning <
-  "loop sequence following '#pragma omp %0' contains induction variables of differing types: %1 and %2">,
-  InGroup<OpenMPLoopForm>;
 def err_omp_not_canonical_loop : Error <
   "loop after '#pragma omp %0' is not in canonical form">;
 def err_omp_not_a_loop_sequence : Error < 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 9819dcfe60360..5f36d968c68fa 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14360,31 +14360,12 @@ bool SemaOpenMP::analyzeLoopSequence(
     OpenMPDirectiveKind Kind) {
 
   VarsWithInheritedDSAType TmpDSA;
-  QualType BaseInductionVarType;
   /// Helper Lambda to handle storing initialization and body statements for
-  /// both ForStmt and CXXForRangeStmt and checks for any possible mismatch
-  /// between induction variables types
+  /// both ForStmt and CXXForRangeStmt
   auto StoreLoopStatements = [&](Stmt *LoopStmt) {
     if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
       OriginalInits.back().push_back(For->getInit());
       ForStmts.push_back(For);
-      // Extract induction variable
-      if (auto *InitStmt = dyn_cast_or_null<DeclStmt>(For->getInit())) {
-        if (auto *InitDecl = dyn_cast<VarDecl>(InitStmt->getSingleDecl())) {
-          QualType InductionVarType = InitDecl->getType().getCanonicalType();
-
-          // Compare with first loop type
-          if (BaseInductionVarType.isNull()) {
-            BaseInductionVarType = InductionVarType;
-          } else if (!Context.hasSameType(BaseInductionVarType,
-                                          InductionVarType)) {
-            Diag(InitDecl->getBeginLoc(),
-                 diag::warn_omp_different_loop_ind_var_types)
-                << getOpenMPDirectiveName(OMPD_fuse) << BaseInductionVarType
-                << InductionVarType;
-          }
-        }
-      }
     } else {
       auto *CXXFor = cast<CXXForRangeStmt>(LoopStmt);
       OriginalInits.back().push_back(CXXFor->getBeginStmt());
diff --git a/clang/test/OpenMP/fuse_messages.cpp b/clang/test/OpenMP/fuse_messages.cpp
index 2a2491d008a0b..4902d424373e5 100644
--- a/clang/test/OpenMP/fuse_messages.cpp
+++ b/clang/test/OpenMP/fuse_messages.cpp
@@ -70,15 +70,6 @@ void func() {
         for(int j = 0; j < 10; ++j);
     }
 
-    //expected-warning at +5 {{loop sequence following '#pragma omp fuse' contains induction variables of differing types: 'int' and 'unsigned int'}}
-    //expected-warning at +5 {{loop sequence following '#pragma omp fuse' contains induction variables of differing types: 'int' and 'long long'}}
-    #pragma omp fuse 
-    {
-        for(int i = 0; i < 10; ++i);
-        for(unsigned int j = 0; j < 10; ++j);
-        for(long long k = 0; k < 100; ++k);
-    }
-
     //expected-warning at +2 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
     #pragma omp fuse
     {
@@ -123,6 +114,40 @@ void func() {
         for(int j = 0; j < 100; ++j);
         for(int k = 0; k < 50; ++k);
     }
+
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '6' is greater than the total number of loops '5'}}
+    #pragma omp fuse looprange(1,6)
+    {
+        for(int i = 0; i < 10; ++i);
+        for(int j = 0; j < 100; ++j);
+        for(int k = 0; k < 50; ++k);
+        // This fusion results in  2 loops
+        #pragma omp fuse looprange(1,2)
+        {
+            for(int i = 0; i < 10; ++i);
+            for(int j = 0; j < 100; ++j);
+            for(int k = 0; k < 50; ++k);
+        }
+    }
+
+    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '4' is greater than the total number of loops '3'}}
+    #pragma omp fuse looprange(2,3)
+    {
+        #pragma omp unroll partial(2)
+        for(int i = 0; i < 10; ++i);
+        
+        #pragma omp reverse
+        for(int j = 0; j < 10; ++j);
+
+        #pragma omp fuse 
+        {
+            {
+                #pragma omp reverse
+                for(int j = 0; j < 10; ++j);
+            }            
+            for(int k = 0; k < 50; ++k);
+        }
+    }
 }
 
 // In a template context, but expression itself not instantiation-dependent

>From 1c8f0fe23fdd78de1512505f128fa9e8bff655f9 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 20 Jun 2025 14:17:29 +0000
Subject: [PATCH 10/13] Address formatting issues

---
 clang/include/clang/Parse/Parser.h  |  2 +-
 clang/lib/AST/StmtOpenMP.cpp        | 10 +++++-----
 clang/lib/Parse/ParseOpenMP.cpp     |  2 +-
 flang/lib/Lower/OpenMP/Clauses.cpp  |  2 +-
 flang/lib/Parser/openmp-parsers.cpp |  8 +++-----
 5 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 08bee0078b5ff..9364007f3cf41 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -6737,7 +6737,7 @@ class Parser : public CodeCompletionHandler {
 
   /// Parses the 'looprange' clause of a '#pragma omp fuse' directive.
   OMPClause *ParseOpenMPLoopRangeClause();
-  
+
   /// Parses the 'sizes' clause of a '#pragma omp tile' directive.
   OMPClause *ParseOpenMPSizesClause();
 
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index f527e6361b5e5..1f49e9f2a0640 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -522,15 +522,15 @@ OMPFuseDirective *OMPFuseDirective::Create(
       NumLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
-  // The number of top level canonical nests could 
+  // The number of top level canonical nests could
   // not match the total number of generated loops
   // Example:
   // Before fusion:
-  //   for (int i = 0; i < N; ++i)   
-  //     for (int j = 0; j < M; ++j) 
+  //   for (int i = 0; i < N; ++i)
+  //     for (int j = 0; j < M; ++j)
   //       A[i][j] = i + j;
-  //   
-  //   for (int k = 0; k < P; ++k) 
+  //
+  //   for (int k = 0; k < P; ++k)
   //     B[k] = k * 2;
   // Here, NumLoopNests = 2, but NumLoops = 3.
   Dir->setNumGeneratedLoopNests(NumLoopNests);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 2d6d624c1ecc8..48d9c184131cd 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3520,7 +3520,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
     break;
   case OMPC_looprange:
     Clause = ParseOpenMPLoopRangeClause();
-    break;  
+    break;
   default:
     break;
   }
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index a38249bf2b588..c94d56cb57756 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -998,7 +998,7 @@ Link make(const parser::OmpClause::Link &inp,
 }
 
 LoopRange make(const parser::OmpClause::Looprange &inp,
-            semantics::SemanticsContext &semaCtx) {
+               semantics::SemanticsContext &semaCtx) {
   llvm_unreachable("Unimplemented: looprange");
 }
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index d53389746dbec..39978e402e63b 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -847,10 +847,8 @@ TYPE_PARSER(
         maybe(":"_tok >> nonemptyList(Parser<OmpLinearClause::Modifier>{})),
         /*PostModified=*/pure(true)))
 
-TYPE_PARSER(
-  construct<OmpLoopRangeClause>(scalarIntConstantExpr,
-                                "," >> scalarIntConstantExpr)
-)
+TYPE_PARSER(construct<OmpLoopRangeClause>(
+    scalarIntConstantExpr, "," >> scalarIntConstantExpr))
 
 // OpenMPv5.2 12.5.2 detach-clause -> DETACH (event-handle)
 TYPE_PARSER(construct<OmpDetachClause>(Parser<OmpObject>{}))
@@ -1026,7 +1024,7 @@ TYPE_PARSER( //
     "LINK" >> construct<OmpClause>(construct<OmpClause::Link>(
                   parenthesized(Parser<OmpObjectList>{}))) ||
     "LOOPRANGE" >> construct<OmpClause>(construct<OmpClause::Looprange>(
-                  parenthesized(Parser<OmpLoopRangeClause>{}))) ||
+                       parenthesized(Parser<OmpLoopRangeClause>{}))) ||
     "MAP" >> construct<OmpClause>(construct<OmpClause::Map>(
                  parenthesized(Parser<OmpMapClause>{}))) ||
     "MATCH" >> construct<OmpClause>(construct<OmpClause::Match>(

>From 009d8630c7ff97dedc543df04d0b18ab4579a503 Mon Sep 17 00:00:00 2001
From: eZWALT <waltertheshadow333 at gmail.com>
Date: Fri, 20 Jun 2025 14:44:31 +0000
Subject: [PATCH 11/13] Address minor feedback part 2

---
 clang/include/clang/AST/OpenMPClause.h           |  8 ++++++--
 clang/include/clang/AST/StmtOpenMP.h             |  1 +
 clang/include/clang/Basic/DiagnosticSemaKinds.td | 10 +++++-----
 clang/lib/Sema/SemaOpenMP.cpp                    | 15 +++++----------
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 3df5133a17fb4..478c41322f34a 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1197,12 +1197,16 @@ class OMPLoopRangeClause final
   void setFirstLoc(SourceLocation Loc) { FirstLoc = Loc; }
   void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
 
-  /// Get looprange arguments: first and count
+  /// Get looprange 'first' expression
   Expr *getFirst() const { return getArgs()[0]; }
+
+  /// Get looprange 'count' expression
   Expr *getCount() const { return getArgs()[1]; }
 
-  /// Set looprange arguments: first and count
+  /// Set looprange 'first' expression
   void setFirst(Expr *E) { getArgs()[0] = E; }
+
+  /// Set looprange 'count' expression
   void setCount(Expr *E) { getArgs()[1] = E; }
 
   MutableArrayRef<Expr *> getArgs() {
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 0421c06245cac..5ec3677fc7507 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -976,6 +976,7 @@ class OMPLoopTransformationDirective : public OMPLoopBasedDirective {
 
   /// Set the number of loops generated by this loop transformation.
   void setNumGeneratedLoops(unsigned Num) { NumGeneratedLoops = Num; }
+
   /// Set the number of top level canonical loop nests generated by this loop
   /// transformation
   void setNumGeneratedLoopNests(unsigned Num) { NumGeneratedLoopNests = Num; }
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 2bd0f895204c9..d807b6b076724 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11612,16 +11612,16 @@ def note_omp_implicit_dsa : Note<
   "implicitly determined as %0">;
 def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
-def err_omp_not_canonical_loop : Error <
+def err_omp_not_canonical_loop : Error<
   "loop after '#pragma omp %0' is not in canonical form">;
-def err_omp_not_a_loop_sequence : Error < 
+def err_omp_not_a_loop_sequence : Error< 
   "statement after '#pragma omp %0' must be a loop sequence containing canonical loops or loop-generating constructs">;
-def err_omp_empty_loop_sequence : Error <
+def err_omp_empty_loop_sequence : Error<
   "loop sequence after '#pragma omp %0' must contain at least 1 canonical loop or loop-generating construct">;
-def err_omp_invalid_looprange : Error <
+def err_omp_invalid_looprange : Error<
   "loop range in '#pragma omp %0' exceeds the number of available loops: "
   "range end '%1' is greater than the total number of loops '%2'">;
-def warn_omp_redundant_fusion : Warning <
+def warn_omp_redundant_fusion : Warning<
   "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">,
   InGroup<OpenMPClauses>;
 def err_omp_not_for : Error<
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 5f36d968c68fa..8aa21c5c01220 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -22,7 +22,6 @@
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DynamicRecursiveASTVisitor.h"
 #include "clang/AST/OpenMPClause.h"
-#include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/StmtVisitor.h"
@@ -48,7 +47,6 @@
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/IR/Assumptions.h"
 #include <optional>
-#include <queue>
 
 using namespace clang;
 using namespace llvm::omp;
@@ -14201,7 +14199,6 @@ template <typename T, typename F> static bool tryHandleAs(T *t, F &&) {
   return false;
 }
 
-///
 /// Tries to recursively cast `t` to one of the given types and invokes `f` if
 /// successful.
 ///
@@ -14274,7 +14271,7 @@ bool SemaOpenMP::checkTransformableLoopNest(
 
 /// Counts the total number of nested loops, including the outermost loop (the
 /// original loop). PRECONDITION of this visitor is that it must be invoked from
-/// the original loop to be analyzed. The traversal is stop for Decl's and
+/// the original loop to be analyzed. The traversal stops for Decl's and
 /// Expr's given that they may contain inner loops that must not be counted.
 ///
 /// Example AST structure for the code:
@@ -15945,7 +15942,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
 
   // Select the type with the largest bit width among all induction variables
   QualType IVType = LoopHelpers[FirstVal - 1].IterationVarRef->getType();
-  for (unsigned int I = FirstVal; I < LastVal; ++I) {
+  for (unsigned I = FirstVal; I < LastVal; ++I) {
     QualType CurrentIVType = LoopHelpers[I].IterationVarRef->getType();
     if (Context.getTypeSize(CurrentIVType) > Context.getTypeSize(IVType)) {
       IVType = CurrentIVType;
@@ -16054,9 +16051,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
     auto [IVVD, IVDStmt] =
         CreateHelperVarAndStmt(LoopHelpers[I].IterationVarRef, "iv", J);
 
-    if (!LBVD || !STVD || !NIVD || !IVVD)
-      assert(LBVD && STVD && NIVD && IVVD &&
-             "OpenMP Fuse Helper variables creation failed");
+    assert(LBVD && STVD && NIVD && IVVD &&
+           "OpenMP Fuse Helper variables creation failed");
 
     UBVarDecls.push_back(UBVD);
     LBVarDecls.push_back(LBVD);
@@ -16097,11 +16093,10 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   //      original.indexk = ivk
   //      body(k);  Expr *InitVal = IntegerLiteral::Create(Context,
   //      llvm::APInt(IVWidth, 0),
-
   //    }
 
   // 1. Create the initialized fuse index
-  const std::string IndexName = Twine(".omp.fuse.index").str();
+  StringRef IndexName = ".omp.fuse.index";
   Expr *InitVal = IntegerLiteral::Create(Context, llvm::APInt(IVBitWidth, 0),
                                          IVType, SourceLocation());
   VarDecl *IndexDecl =

>From eb464451c30c19745591738d71e3496d4d11514a Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Mon, 14 Jul 2025 11:34:36 +0000
Subject: [PATCH 12/13] Address some of the feedback

---
 clang/include/clang/AST/OpenMPClause.h | 46 ++++---------
 clang/include/clang/Sema/SemaOpenMP.h  |  2 +-
 clang/lib/AST/OpenMPClause.cpp         | 11 ++-
 clang/lib/Sema/SemaOpenMP.cpp          | 92 ++++++++++++--------------
 4 files changed, 63 insertions(+), 88 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 478c41322f34a..5034ff9bacbfc 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1154,11 +1154,8 @@ class OMPFullClause final : public OMPNoChildClause<llvm::omp::OMPC_full> {
 ///   for(int k = 127; k >= 0; --k)
 /// \endcode
 class OMPLoopRangeClause final
-    : public OMPClause,
-      private llvm::TrailingObjects<OMPLoopRangeClause, Expr *> {
+    : public OMPClause {
   friend class OMPClauseReader;
-  friend class llvm::TrailingObjects<OMPLoopRangeClause, Expr *>;
-
   /// Location of '('
   SourceLocation LParenLoc;
 
@@ -1166,24 +1163,25 @@ class OMPLoopRangeClause final
   SourceLocation FirstLoc, CountLoc;
 
   /// Number of looprange arguments (always 2: first, count)
-  unsigned NumArgs = 2;
+  static constexpr unsigned NumArgs = 2;
+  Stmt *Args[NumArgs] = {nullptr, nullptr};
 
-  /// Set the argument expressions.
-  void setArgs(ArrayRef<Expr *> Args) {
-    assert(Args.size() == NumArgs && "Expected exactly 2 looprange arguments");
-    std::copy(Args.begin(), Args.end(), getTrailingObjects<Expr *>());
-  }
+  /// Set looprange 'first' expression
+  void setFirst(Expr *E) { Args[0] = E; }
+
+  /// Set looprange 'count' expression
+  void setCount(Expr *E) { Args[1] = E; }
 
   /// Build an empty clause for deserialization.
   explicit OMPLoopRangeClause()
-      : OMPClause(llvm::omp::OMPC_looprange, {}, {}), NumArgs(2) {}
+      : OMPClause(llvm::omp::OMPC_looprange, {}, {}) {}
 
 public:
   /// Build a 'looprange' clause AST node.
   static OMPLoopRangeClause *
   Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
          SourceLocation FirstLoc, SourceLocation CountLoc,
-         SourceLocation EndLoc, ArrayRef<Expr *> Args);
+         SourceLocation EndLoc, Expr* First, Expr* Count);
 
   /// Build an empty 'looprange' clause node.
   static OMPLoopRangeClause *CreateEmpty(const ASTContext &C);
@@ -1198,32 +1196,16 @@ class OMPLoopRangeClause final
   void setCountLoc(SourceLocation Loc) { CountLoc = Loc; }
 
   /// Get looprange 'first' expression
-  Expr *getFirst() const { return getArgs()[0]; }
+  Expr *getFirst() const { return cast_or_null<Expr>(Args[0]); }
 
   /// Get looprange 'count' expression
-  Expr *getCount() const { return getArgs()[1]; }
-
-  /// Set looprange 'first' expression
-  void setFirst(Expr *E) { getArgs()[0] = E; }
-
-  /// Set looprange 'count' expression
-  void setCount(Expr *E) { getArgs()[1] = E; }
-
-  MutableArrayRef<Expr *> getArgs() {
-    return MutableArrayRef<Expr *>(getTrailingObjects<Expr *>(), NumArgs);
-  }
-  ArrayRef<Expr *> getArgs() const {
-    return ArrayRef<Expr *>(getTrailingObjects<Expr *>(), NumArgs);
-  }
+  Expr *getCount() const { return cast_or_null<Expr>(Args[1]); }
 
   child_range children() {
-    return child_range(reinterpret_cast<Stmt **>(getArgs().begin()),
-                       reinterpret_cast<Stmt **>(getArgs().end()));
+    return child_range(Args, Args + NumArgs);
   }
   const_child_range children() const {
-    auto AR = getArgs();
-    return const_child_range(reinterpret_cast<Stmt *const *>(AR.begin()),
-                             reinterpret_cast<Stmt *const *>(AR.end()));
+    return const_child_range(Args, Args + NumArgs);
   }
 
   child_range used_children() {
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index f848c4a7d715e..09c97dc2e119b 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1494,7 +1494,7 @@ class SemaOpenMP : public SemaBase {
       SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
       Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits);
 
-  /// @brief Categories of loops encountered during semantic OpenMP loop
+  /// Categories of loops encountered during semantic OpenMP loop
   /// analysis
   ///
   /// This enumeration identifies the structural category of a loop or sequence
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index e0570262b2a05..4eb249948932e 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1028,23 +1028,20 @@ OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
                            SourceLocation CountLoc, SourceLocation EndLoc,
-                           ArrayRef<Expr *> Args) {
-
-  assert(Args.size() == 2 &&
-         "looprange clause must have exactly two arguments");
+                           Expr *First, Expr* Count) {
   OMPLoopRangeClause *Clause = CreateEmpty(C);
   Clause->setLocStart(StartLoc);
   Clause->setLParenLoc(LParenLoc);
   Clause->setFirstLoc(FirstLoc);
   Clause->setCountLoc(CountLoc);
   Clause->setLocEnd(EndLoc);
-  Clause->setArgs(Args);
+  Clause->setFirst(First);
+  Clause->setCount(Count);
   return Clause;
 }
 
 OMPLoopRangeClause *OMPLoopRangeClause::CreateEmpty(const ASTContext &C) {
-  void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(2));
-  return new (Mem) OMPLoopRangeClause();
+  return new (C) OMPLoopRangeClause();
 }
 
 OMPAllocateClause *OMPAllocateClause::Create(
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 8aa21c5c01220..edffb041b3664 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -14195,29 +14195,29 @@ StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
 }
 
 /// Overloaded base case function
-template <typename T, typename F> static bool tryHandleAs(T *t, F &&) {
+template <typename T, typename F> static bool tryHandleAs(T *, F &&) {
   return false;
 }
 
-/// Tries to recursively cast `t` to one of the given types and invokes `f` if
-/// successful.
+/// Tries to recursively cast `Type` to one of the given types and invokes
+/// `Func` if successful.
 ///
-/// @tparam Class The first type to check.
-/// @tparam Rest The remaining types to check.
-/// @tparam T The base type of `t`.
-/// @tparam F The callable type for the function to invoke upon a successful
+/// \tparam Class The first type to check.
+/// \tparam Rest The remaining types to check.
+/// \tparam T The base type of `Type`.
+/// \tparam F The callable type for the function to invoke upon a successful
 /// cast.
-/// @param t The object to be checked.
-/// @param f The function to invoke if `t` matches `Class`.
-/// @return `true` if `t` matched any type and `f` was called, otherwise
+/// \param Type The object to be checked.
+/// \param Func The function to invoke if `Type` matches `Class`.
+/// \return `true` if `Type` matched any type and `Func` was called, otherwise
 /// `false`.
 template <typename Class, typename... Rest, typename T, typename F>
-static bool tryHandleAs(T *t, F &&f) {
-  if (Class *c = dyn_cast<Class>(t)) {
-    f(c);
+static bool tryHandleAs(T *Type, F &&Func) {
+  if (Class *C = dyn_cast<Class>(Type)) {
+    Func(C);
     return true;
   }
-  return tryHandleAs<Rest...>(t, std::forward<F>(f));
+  return tryHandleAs<Rest...>(Type, std::forward<F>(Func));
 }
 
 /// Updates OriginalInits by checking Transform against loop transformation
@@ -14297,7 +14297,7 @@ bool SemaOpenMP::checkTransformableLoopNest(
 ///     }
 /// }
 /// Result: Loop 'i' contains 2 loops, Loop 'r' also contains 2 loops
-class NestedLoopCounterVisitor : public DynamicRecursiveASTVisitor {
+class NestedLoopCounterVisitor final : public DynamicRecursiveASTVisitor {
 private:
   unsigned NestedLoopCount = 0;
 
@@ -14385,22 +14385,21 @@ bool SemaOpenMP::analyzeLoopSequence(
         LoopSeqSize += NumGeneratedLoopNests;
         NumLoops += NumGeneratedLoops;
         return true;
-      } else {
-        // Unroll full (0 loops produced)
-        Diag(Child->getBeginLoc(), diag::err_omp_not_for)
-            << 0 << getOpenMPDirectiveName(Kind);
-        return false;
       }
+      // Unroll full (0 loops produced)
+      Diag(Child->getBeginLoc(), diag::err_omp_not_for)
+          << 0 << getOpenMPDirectiveName(Kind);
+      return false;
     }
     // Handle loop transformations with multiple loop nests
     // Unroll full
-    if (NumGeneratedLoopNests <= 0) {
+    if (!NumGeneratedLoopNests) {
       Diag(Child->getBeginLoc(), diag::err_omp_not_for)
           << 0 << getOpenMPDirectiveName(Kind);
       return false;
     }
     // Loop transformatons such as split or loopranged fuse
-    else if (NumGeneratedLoopNests > 1) {
+    if (NumGeneratedLoopNests > 1) {
       // Get the preinits related to this loop sequence generating
       // loop transformation (i.e loopranged fuse, split...)
       LoopSequencePreInits.emplace_back();
@@ -14413,30 +14412,29 @@ bool SemaOpenMP::analyzeLoopSequence(
                                  LoopHelpers, ForStmts, OriginalInits,
                                  TransformsPreInits, LoopSequencePreInits,
                                  LoopCategories, Context, Kind);
-    } else {
-      // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
-      // Process the transformed loop statement
-      OriginalInits.emplace_back();
-      TransformsPreInits.emplace_back();
-      LoopHelpers.emplace_back();
-      LoopCategories.push_back(OMPLoopCategory::TransformSingleLoop);
-
-      unsigned IsCanonical =
-          checkOpenMPLoop(Kind, nullptr, nullptr, TransformedStmt, SemaRef,
-                          *DSAStack, TmpDSA, LoopHelpers[LoopSeqSize]);
-
-      if (!IsCanonical) {
-        Diag(TransformedStmt->getBeginLoc(), diag::err_omp_not_canonical_loop)
-            << getOpenMPDirectiveName(Kind);
-        return false;
-      }
-      StoreLoopStatements(TransformedStmt);
-      updatePreInits(LoopTransform, TransformsPreInits);
+    }
+    // Vast majority: (Tile, Unroll, Stripe, Reverse, Interchange, Fuse all)
+    // Process the transformed loop statement
+    OriginalInits.emplace_back();
+    TransformsPreInits.emplace_back();
+    LoopHelpers.emplace_back();
+    LoopCategories.push_back(OMPLoopCategory::TransformSingleLoop);
 
-      NumLoops += NumGeneratedLoops;
-      ++LoopSeqSize;
-      return true;
+    unsigned IsCanonical =
+        checkOpenMPLoop(Kind, nullptr, nullptr, TransformedStmt, SemaRef,
+                        *DSAStack, TmpDSA, LoopHelpers[LoopSeqSize]);
+
+    if (!IsCanonical) {
+      Diag(TransformedStmt->getBeginLoc(), diag::err_omp_not_canonical_loop)
+          << getOpenMPDirectiveName(Kind);
+      return false;
     }
+    StoreLoopStatements(TransformedStmt);
+    updatePreInits(LoopTransform, TransformsPreInits);
+
+    NumLoops += NumGeneratedLoops;
+    ++LoopSeqSize;
+    return true;
   };
 
   /// Modularized code for handling regular canonical loops
@@ -16303,7 +16301,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
       // Only TransformSingleLoop requires inserting pre-inits here
 
       if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop) {
-        auto TransformPreInit = TransformsPreInits[TransformIndex++];
+        const auto &TransformPreInit = TransformsPreInits[TransformIndex++];
         if (!TransformPreInit.empty()) {
           llvm::append_range(PreInits, TransformPreInit);
         }
@@ -17456,15 +17454,13 @@ OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
   if (CountVal.isInvalid())
     Count = nullptr;
 
-  SmallVector<Expr *, 2> ArgsVec = {First, Count};
-
   // OpenMP [6.0, Restrictions]
   // first + count - 1 must not evaluate to a value greater than the
   // loop sequence length of the associated canonical loop sequence.
   // This check must be performed afterwards due to the delayed
   // parsing and computation of the associated loop sequence
   return OMPLoopRangeClause::Create(getASTContext(), StartLoc, LParenLoc,
-                                    FirstLoc, CountLoc, EndLoc, ArgsVec);
+                                    FirstLoc, CountLoc, EndLoc, First, Count);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPAlignClause(Expr *A, SourceLocation StartLoc,

>From 010a57f0e86d2b989e746d286a93b18add2e50ef Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Mon, 14 Jul 2025 13:21:39 +0000
Subject: [PATCH 13/13] More changes addressing feedback

---
 .../clang/Basic/DiagnosticSemaKinds.td        |  7 ++-
 clang/lib/Sema/SemaOpenMP.cpp                 |  6 +--
 clang/test/OpenMP/fuse_messages.cpp           | 51 +++++++++----------
 .../fuse/parallel-wsloop-collapse-intfor.c    |  2 +-
 4 files changed, 32 insertions(+), 34 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index d807b6b076724..10b5349fa4d43 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11614,15 +11614,14 @@ def err_omp_loop_var_dsa : Error<
   "loop iteration variable in the associated loop of 'omp %1' directive may not be %0, predetermined as %2">;
 def err_omp_not_canonical_loop : Error<
   "loop after '#pragma omp %0' is not in canonical form">;
-def err_omp_not_a_loop_sequence : Error< 
+def err_omp_not_a_loop_sequence : Error<
   "statement after '#pragma omp %0' must be a loop sequence containing canonical loops or loop-generating constructs">;
 def err_omp_empty_loop_sequence : Error<
   "loop sequence after '#pragma omp %0' must contain at least 1 canonical loop or loop-generating construct">;
 def err_omp_invalid_looprange : Error<
-  "loop range in '#pragma omp %0' exceeds the number of available loops: "
-  "range end '%1' is greater than the total number of loops '%2'">;
+  "looprange clause selects loops from %1 to %2 but this exceeds the number of loops (%3) in the loop sequence">;
 def warn_omp_redundant_fusion : Warning<
-  "loop range in '#pragma omp %0' contains only a single loop, resulting in redundant fusion">,
+  "looprange clause selects a single loop, resulting in redundant fusion">,
   InGroup<OpenMPClauses>;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index edffb041b3664..2f5a1b39294b6 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15913,8 +15913,8 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
 
     if (!ValidLoopRange(FirstVal, CountVal, LoopSeqSize)) {
       SemaRef.Diag(LRC->getFirstLoc(), diag::err_omp_invalid_looprange)
-          << getOpenMPDirectiveName(OMPD_fuse) << (FirstVal + CountVal - 1)
-          << LoopSeqSize;
+          << getOpenMPDirectiveName(OMPD_fuse) << FirstVal
+          << (FirstVal + CountVal - 1) << LoopSeqSize;
       return StmtError();
     }
 
@@ -16021,7 +16021,7 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
 
   // Firstly we need to update TransformIndex to match the begining of the
   // looprange section
-  for (unsigned int I = 0; I < FirstVal - 1; ++I) {
+  for (unsigned I : llvm::seq<unsigned>(FirstVal - 1)) {
     if (LoopCategories[I] == OMPLoopCategory::TransformSingleLoop)
       ++TransformIndex;
   }
diff --git a/clang/test/OpenMP/fuse_messages.cpp b/clang/test/OpenMP/fuse_messages.cpp
index 4902d424373e5..5df904fc2b15e 100644
--- a/clang/test/OpenMP/fuse_messages.cpp
+++ b/clang/test/OpenMP/fuse_messages.cpp
@@ -3,23 +3,23 @@
 void func() {
 
     // expected-error at +2 {{statement after '#pragma omp fuse' must be a loop sequence containing canonical loops or loop-generating constructs}}
-    #pragma omp fuse 
+    #pragma omp fuse
     ;
 
     // expected-error at +2 {{statement after '#pragma omp fuse' must be a for loop}}
-    #pragma omp fuse 
+    #pragma omp fuse
     {int bar = 0;}
 
     // expected-error at +4 {{statement after '#pragma omp fuse' must be a for loop}}
-    #pragma omp fuse 
+    #pragma omp fuse
     {
         for(int i = 0; i < 10; ++i);
         int x = 2;
     }
 
     // expected-error at +2 {{statement after '#pragma omp fuse' must be a loop sequence containing canonical loops or loop-generating constructs}}
-    #pragma omp fuse 
-    #pragma omp for 
+    #pragma omp fuse
+    #pragma omp for
     for (int i = 0; i < 7; ++i)
         ;
 
@@ -39,7 +39,7 @@ void func() {
 
 
     // expected-error at +1 {{unexpected OpenMP clause 'final' in directive '#pragma omp fuse'}}
-    #pragma omp fuse final(0) 
+    #pragma omp fuse final(0)
     {
         for (int i = 0; i < 7; ++i)
             ;
@@ -49,7 +49,7 @@ void func() {
 
     //expected-error at +4 {{loop after '#pragma omp fuse' is not in canonical form}}
     //expected-error at +3 {{increment clause of OpenMP for loop must perform simple addition or subtraction on loop variable 'i'}}
-    #pragma omp fuse 
+    #pragma omp fuse
     {
         for(int i = 0; i < 10; i*=2) {
             ;
@@ -58,25 +58,25 @@ void func() {
     }
 
     //expected-error at +2 {{loop sequence after '#pragma omp fuse' must contain at least 1 canonical loop or loop-generating construct}}
-    #pragma omp fuse 
+    #pragma omp fuse
     {}
 
     //expected-error at +3 {{statement after '#pragma omp fuse' must be a for loop}}
-    #pragma omp fuse 
+    #pragma omp fuse
     {
-        #pragma omp unroll full 
+        #pragma omp unroll full
         for(int i = 0; i < 10; ++i);
-        
+
         for(int j = 0; j < 10; ++j);
     }
 
-    //expected-warning at +2 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    //expected-warning at +2 {{looprange clause selects a single loop, resulting in redundant fusion}}
     #pragma omp fuse
     {
         for(int i = 0; i < 10; ++i);
     }
 
-    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    //expected-warning at +1 {{looprange clause selects a single loop, resulting in redundant fusion}}
     #pragma omp fuse looprange(1, 1)
     {
         for(int i = 0; i < 10; ++i);
@@ -99,7 +99,7 @@ void func() {
 
     const int x = 1;
     constexpr int y = 4;
-    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '4' is greater than the total number of loops '3'}}
+    //expected-error at +1 {{looprange clause selects loops from 1 to 4 but this exceeds the number of loops (3) in the loop sequence}}
     #pragma omp fuse looprange(x,y)
     {
         for(int i = 0; i < 10; ++i);
@@ -107,7 +107,7 @@ void func() {
         for(int k = 0; k < 50; ++k);
     }
 
-    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '420' is greater than the total number of loops '3'}}
+    //expected-error at +1 {{looprange clause selects loops from 1 to 420 but this exceeds the number of loops (3) in the loop sequence}}
     #pragma omp fuse looprange(1,420)
     {
         for(int i = 0; i < 10; ++i);
@@ -115,7 +115,7 @@ void func() {
         for(int k = 0; k < 50; ++k);
     }
 
-    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '6' is greater than the total number of loops '5'}}
+    //expected-error at +1 {{looprange clause selects loops from 1 to 6 but this exceeds the number of loops (5) in the loop sequence}}
     #pragma omp fuse looprange(1,6)
     {
         for(int i = 0; i < 10; ++i);
@@ -130,21 +130,21 @@ void func() {
         }
     }
 
-    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '4' is greater than the total number of loops '3'}}
+    //expected-error at +1 {{looprange clause selects loops from 2 to 4 but this exceeds the number of loops (3) in the loop sequence}}
     #pragma omp fuse looprange(2,3)
     {
         #pragma omp unroll partial(2)
         for(int i = 0; i < 10; ++i);
-        
+
         #pragma omp reverse
         for(int j = 0; j < 10; ++j);
 
-        #pragma omp fuse 
+        #pragma omp fuse
         {
             {
                 #pragma omp reverse
                 for(int j = 0; j < 10; ++j);
-            }            
+            }
             for(int k = 0; k < 50; ++k);
         }
     }
@@ -154,7 +154,7 @@ void func() {
 template <typename T>
 static void templated_func() {
 
-    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    //expected-warning at +1 {{looprange clause selects a single loop, resulting in redundant fusion}}
     #pragma omp fuse looprange(2,1)
     {
         for(int i = 0; i < 10; ++i);
@@ -162,7 +162,7 @@ static void templated_func() {
         for(int k = 0; k < 50; ++k);
     }
 
-    //expected-error at +1 {{loop range in '#pragma omp fuse' exceeds the number of available loops: range end '5' is greater than the total number of loops '3'}}
+    //expected-error at +1 {{looprange clause selects loops from 3 to 5 but this exceeds the number of loops (3) in the loop sequence}}
     #pragma omp fuse looprange(3,3)
     {
         for(int i = 0; i < 10; ++i);
@@ -172,10 +172,10 @@ static void templated_func() {
 
 }
 
-template <int V> 
+template <int V>
 static void templated_func_value_dependent() {
 
-    //expected-warning at +1 {{loop range in '#pragma omp fuse' contains only a single loop, resulting in redundant fusion}}
+    //expected-warning at +1 {{looprange clause selects a single loop, resulting in redundant fusion}}
     #pragma omp fuse looprange(V,1)
     {
         for(int i = 0; i < 10; ++i);
@@ -184,7 +184,7 @@ static void templated_func_value_dependent() {
     }
 }
 
-template <typename T> 
+template <typename T>
 static void templated_func_type_dependent() {
     constexpr T s = 1;
 
@@ -205,7 +205,6 @@ void template_inst() {
     templated_func_value_dependent<1>();
     // expected-note at +1 {{in instantiation of function template specialization 'templated_func_type_dependent<int>' requested here}}
     templated_func_type_dependent<int>();
-
 }
 
 
diff --git a/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c
index 272908e72c429..9630fec50bc20 100644
--- a/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c
+++ b/openmp/runtime/test/transform/fuse/parallel-wsloop-collapse-intfor.c
@@ -24,7 +24,7 @@ int main() {
 #endif /* HEADER */
 
 // CHECK:      do
-// CHECK: i=0 j=0
+// CHECK-NEXT: i=0 j=0
 // CHECK-NEXT: i=0 k=0
 // CHECK-NEXT: i=0 j=1
 // CHECK-NEXT: i=0 k=1


From flang-commits at lists.llvm.org  Mon Jul 14 07:39:07 2025
From: flang-commits at lists.llvm.org (Anchu Rajendran S via flang-commits)
Date: Mon, 14 Jul 2025 07:39:07 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [mlir]
 [flang][flang-driver][mlir][OpenMP] atomic control support (PR #143441)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143441@github.com>
Message-ID: <6875168b.170a0220.3b508e.5dfa@mx.google.com>


https://github.com/anchuraj updated https://github.com/llvm/llvm-project/pull/143441

>From 4816fb9bc0761d8761798300bb419db0072c9d04 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 4 Jun 2025 15:12:49 -0500
Subject: [PATCH 1/5] [flang][flang-driver] atomic control support

---
 clang/include/clang/Driver/Options.td         | 22 +++++-----
 flang/include/flang/Frontend/TargetOptions.h  |  5 +++
 .../Optimizer/Dialect/Support/FIRContext.h    | 14 +++++++
 flang/lib/Frontend/CompilerInvocation.cpp     | 12 ++++++
 flang/lib/Lower/Bridge.cpp                    |  6 +++
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 40 +++++++++++++++++++
 .../Lower/OpenMP/atomic-control-options.f90   | 37 +++++++++++++++++
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     | 14 +++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  8 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  9 +++++
 10 files changed, 153 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/atomic-control-options.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8e38f0062140e..c1c7929b0c15d 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2303,21 +2303,21 @@ def fsymbol_partition_EQ : Joined<["-"], "fsymbol-partition=">, Group<f_Group>,
 
 defm atomic_remote_memory : BoolFOption<"atomic-remote-memory",
   LangOpts<"AtomicRemoteMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on remote memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on remote memory">>;
 
 defm atomic_fine_grained_memory : BoolFOption<"atomic-fine-grained-memory",
   LangOpts<"AtomicFineGrainedMemory">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "May have">,
-  NegFlag<SetFalse, [], [ClangOption], "Assume no">,
-  BothFlags<[], [ClangOption], " atomic operations on fine-grained memory">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "May have">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Assume no">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations on fine-grained memory">>;
 
 defm atomic_ignore_denormal_mode : BoolFOption<"atomic-ignore-denormal-mode",
   LangOpts<"AtomicIgnoreDenormalMode">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option], "Allow">,
-  NegFlag<SetFalse, [], [ClangOption], "Disallow">,
-  BothFlags<[], [ClangOption], " atomic operations to ignore denormal mode">>;
+  PosFlag<SetTrue, [], [ClangOption, CC1Option, FlangOption, FC1Option], "Allow">,
+  NegFlag<SetFalse, [], [ClangOption, FlangOption], "Disallow">,
+  BothFlags<[], [ClangOption, FlangOption], " atomic operations to ignore denormal mode">>;
 
 defm memory_profile : OptInCC1FFlag<"memory-profile", "Enable", "Disable", " heap memory profiling">;
 def fmemory_profile_EQ : Joined<["-"], "fmemory-profile=">,
@@ -5314,9 +5314,9 @@ defm amdgpu_precise_memory_op
                   " precise memory mode (AMDGPU only)">;
 
 def munsafe_fp_atomics : Flag<["-"], "munsafe-fp-atomics">,
-  Visibility<[ClangOption, CC1Option]>, Alias<fatomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>, Alias<fatomic_ignore_denormal_mode>;
 def mno_unsafe_fp_atomics : Flag<["-"], "mno-unsafe-fp-atomics">,
-  Visibility<[ClangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
+  Visibility<[ClangOption, FlangOption]>, Alias<fno_atomic_ignore_denormal_mode>;
 
 def faltivec : Flag<["-"], "faltivec">, Group<f_Group>;
 def fno_altivec : Flag<["-"], "fno-altivec">, Group<f_Group>;
diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 002d8d158abd4..26256fd775f11 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -53,6 +53,11 @@ class TargetOptions {
 
   /// Print verbose assembly
   bool asmVerbose = false;
+
+  /// Atomic Control Options for AMD GPU
+  bool amdgpuIgnoreDenormalMode = false;
+  bool amdgpuRemoteMemory = false;
+  bool amdgpuFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 2df14f83c11e1..ac563bcc402c7 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,10 +58,24 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
+// Setters and Getters for atomic control options
+void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
+void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
+void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
+bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
+// set Atomic control options for amd gpu.
+void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
+                                   bool amdgpuIgnoreDenormalMode,
+                                   bool amdgpuNoFineGrainedMemory,
+                                   bool amdgpuNoRemoteMemory);
+
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 30d81f3daa969..00f854b905c72 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,6 +496,18 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
+  if (llvm::Triple(opts.triple).isAMDGPU()) {
+    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
+        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+    opts.amdgpuFineGrainedMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_fine_grained_memory,
+        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+    opts.amdgpuRemoteMemory = args.hasFlag(
+        clang::driver::options::OPT_fatomic_remote_memory,
+        clang::driver::options::OPT_fno_atomic_remote_memory, false);
+  }
+
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
     opts.cpu = a->getValue();
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ff35840a6668c..acbfad589dfbb 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,6 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
+  if (targetOpts.amdgpuIgnoreDenormalMode)
+    fir::setAmdgpuIgnoreDenormalMode(*module);
+  if (targetOpts.amdgpuFineGrainedMemory)
+    fir::setAmdgpuFineGrainedMemory(*module);
+  if (targetOpts.amdgpuRemoteMemory)
+    fir::setAmdgpuRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 01c0be66d1ecc..b961793dbdfd5 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,6 +88,46 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
+static constexpr const char *amdgpuIgnoreDenormalModeName =
+    "fir.amdgpu.ignore.denormal.mode";
+void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
+    return true;
+  return false;
+}
+
+static constexpr const char *amdgpuFineGrainedMemoryName =
+    "fir.amdgpu.fine.grained.memory";
+void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+  if (auto attr =
+          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
+    return true;
+  return false;
+}
+static constexpr const char *amdgpuRemoteMemoryName =
+    "fir.amdgpu.remote.memory";
+void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  auto *ctx = mod.getContext();
+  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+}
+
+bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
+    return true;
+  return false;
+}
+
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
   if (auto attr = mod->getAttrOfType<mlir::StringAttr>(tuneCpuName))
     return attr.getValue();
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
new file mode 100644
index 0000000000000..1eb0f617f365e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o - | FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o - | FileCheck -check-prefix=IGNORE-DENORMAL %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o - | FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-hlfir -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o - | FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, B, threads
+    threads = 128
+    A = 0
+    B = 0
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+    !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !$omp target parallel num_threads(threads)
+    !$omp atomic capture
+        A = A + B
+        B = A
+    !$omp end atomic
+    !$omp end target parallel
+end program test
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 704d0b2220e8a..84887dfd2c6f9 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -54,6 +54,20 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Runtime library flags attribute that holds information for lowering to LLVM.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1543,9 +1543,11 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
     operations.
   }] # clausesDescription;
 
-  let arguments = !con((ins Arg<OpenMP_PointerLikeType,
-                                "Address of variable to be updated",
-                                [MemRead, MemWrite]>:$x), clausesArgs);
+  let arguments = !con(
+      (ins Arg<OpenMP_PointerLikeType,
+               "Address of variable to be updated", [MemRead, MemWrite]>:$x,
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
+      clausesArgs);
 
   // Override region definition.
   let regions = (region SizedRegion<1>:$region);
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 47cfc5278a5d0..e377b7aab3e43 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1562,6 +1562,15 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
     omp.yield(%newval : i32)
   }
 
+  // CHECK: omp.atomic.update %[[X]] : memref<i32> {
+  // CHECK-NEXT: (%[[XVAL:.*]]: i32):
+  // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
+  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  omp.atomic.update %x : memref<i32> {
+  ^bb0(%xval:i32):
+    omp.yield(%const:i32)
+  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+
   return
 }
 

>From 63d3b489ce5d1210172f8bc1d3b18d81ff52886d Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 13:58:07 -0500
Subject: [PATCH 2/5] R2: Making Atomic control non-optional

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   2 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |   2 +-
 mlir/test/Dialect/OpenMP/canonicalize.mlir    |  10 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  46 +++----
 mlir/test/Dialect/OpenMP/ops.mlir             | 127 +++++++++---------
 .../Target/LLVMIR/openmp-llvm-invalid.mlir    |   8 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  98 +++++++-------
 .../openmp-target-nesting-in-host-ops.mlir    |   2 +-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |   2 +-
 9 files changed, 148 insertions(+), 149 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 375e35a146f78..399f7fa6e78f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          OptionalAttr<AtomicControlAttr>:$atomic_control),
+          AtomicControlAttr:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index d69de998346b5..0b2445d0a0d18 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -133,7 +133,7 @@ func.func @atomic_update() {
     %1 = arith.constant 1 : i32
     %2 = arith.addi %arg0, %1  : i32
     omp.yield(%2 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 llvm.mlir.global internal @_QFsEc() : i32 {
diff --git a/mlir/test/Dialect/OpenMP/canonicalize.mlir b/mlir/test/Dialect/OpenMP/canonicalize.mlir
index de6c931ecc5fd..2492010c4d3c7 100644
--- a/mlir/test/Dialect/OpenMP/canonicalize.mlir
+++ b/mlir/test/Dialect/OpenMP/canonicalize.mlir
@@ -4,7 +4,7 @@ func.func @update_no_op(%x : memref<i32>) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -17,7 +17,7 @@ func.func @update_write_op(%x : memref<i32>, %value: i32) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%value : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -33,7 +33,7 @@ func.func @update_normal(%x : memref<i32>, %value: i32) {
   ^bb0(%xval : i32):
     %newval = arith.addi %xval, %value : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -50,7 +50,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.addi %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -65,7 +65,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.muli %xval, %c0 : i32
     omp.yield(%newval: i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..afc3830286f30 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1007,7 +1007,7 @@ func.func @omp_atomic_update1(%x: memref<i32>, %expr: f32) {
   ^bb0(%xval: f32):
     %newval = llvm.fadd %xval, %expr : f32
     omp.yield (%newval : f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1020,7 +1020,7 @@ func.func @omp_atomic_update2(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.terminator
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1032,7 +1032,7 @@ func.func @omp_atomic_update3(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1044,7 +1044,7 @@ func.func @omp_atomic_update4(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1056,7 +1056,7 @@ func.func @omp_atomic_update5(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1068,7 +1068,7 @@ func.func @omp_atomic_update6(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval, %expr : i32, i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1080,7 +1080,7 @@ func.func @omp_atomic_update7(%x: memref<i32>, %expr: i32, %y: f32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%y: f32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1092,7 +1092,7 @@ func.func @omp_atomic_update8(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32, %tmp: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1104,7 +1104,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1116,7 +1116,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1128,7 +1128,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
@@ -1164,12 +1164,12 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.update %x : memref<i32> {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1197,7 +1197,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
   return
@@ -1212,7 +1212,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.write %x = %expr : memref<i32>, i32
     omp.terminator
   }
@@ -1240,7 +1240,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %y : memref<i32>, memref<i32>, i32
     omp.terminator
   }
@@ -1256,7 +1256,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.terminator
   }
 }
@@ -1281,7 +1281,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1296,7 +1296,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1311,7 +1311,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1326,7 +1326,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1341,7 +1341,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1356,7 +1356,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x memory_order(seq_cst) : memref<i32>, memref<i32>, i32
   }
   return
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index e377b7aab3e43..5aa7befb2dc26 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
@@ -1380,57 +1380,57 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i1):
     %newval = llvm.and %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.shl %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.intr.smax(%[[XVAL]], %[[EXPR]]) : (i32, i32) -> i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.intr.smax(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.icmp "eq" %[[XVAL]], %[[EXPRBOOL]] : i1
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i1)
-  // }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %xBool : memref<i1> {
   ^bb0(%xval: i1):
     %newval = llvm.icmp "eq" %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%xval:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   %const = arith.constant 42 : i32
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1440,7 +1440,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1450,7 +1450,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1460,7 +1460,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1470,7 +1470,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1480,7 +1480,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1490,7 +1490,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1500,7 +1500,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1510,7 +1510,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(contended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1520,7 +1520,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1530,7 +1530,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(release) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1540,7 +1540,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update memory_order(relaxed) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1550,7 +1550,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1560,17 +1560,16 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control_attr = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
-
+  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
   return
 }
 
@@ -1582,7 +1581,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture{
@@ -1590,7 +1589,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   // CHECK: omp.atomic.capture {
@@ -1599,7 +1598,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: }
   omp.atomic.capture{
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
@@ -1607,7 +1606,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
@@ -1623,7 +1622,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(none) {
@@ -1631,7 +1630,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1640,7 +1639,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended) {
@@ -1648,7 +1647,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1657,7 +1656,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended) {
@@ -1665,7 +1664,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1674,7 +1673,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(nonspeculative) {
@@ -1682,7 +1681,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1691,7 +1690,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(speculative) {
@@ -1699,7 +1698,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1708,7 +1707,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, nonspeculative) {
@@ -1716,7 +1715,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1725,7 +1724,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, nonspeculative) {
@@ -1733,7 +1732,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1742,7 +1741,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, speculative) {
@@ -1750,7 +1749,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1759,7 +1758,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) {
@@ -1767,7 +1766,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1776,7 +1775,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(seq_cst) {
@@ -1784,7 +1783,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1793,7 +1792,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acq_rel) {
@@ -1801,7 +1800,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1810,7 +1809,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acquire) {
@@ -1818,7 +1817,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1827,7 +1826,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(release) {
@@ -1835,7 +1834,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1844,7 +1843,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(relaxed) {
@@ -1852,7 +1851,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1861,7 +1860,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) memory_order(seq_cst) {
@@ -1869,7 +1868,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -2653,14 +2652,14 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
   // CHECK: omp.atomic.update %[[x]] : !llvm.ptr {
   // CHECK-NEXT: ^{{[[:alnum:]]+}}(%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: }
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
   omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   %val = llvm.load %x : !llvm.ptr -> i32
   omp.atomic.write %v = %val : !llvm.ptr, i32
   omp.atomic.update %x : !llvm.ptr {
     ^bb0(%xval: i32):
       omp.yield(%xval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index 41bc5c4ba525f..00f53864b5fa7 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -9,7 +9,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
     %t2 = llvm.sdiv %t1, %expr : i32
     %newval = llvm.add %xval, %t2 : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -24,7 +24,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.mul %expr, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -41,7 +41,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
     ^bb0(%xval: i32):
       %newval = llvm.mul %expr, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
@@ -59,7 +59,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
       %t2 = llvm.sdiv %t1, %expr : i32
       %newval = llvm.add %xval, %t2 : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 32f0ba5b105ff..ab7f756bf95f9 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,13 +1462,13 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1563,7 +1563,7 @@ llvm.func @_QPomp_atomic_update_complex() {
       %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
       %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
       omp.yield(%16 : !llvm.struct<(f32, f32)>)
-    }
+    } {atomic_control = #omp.atomic_control<>}
    llvm.return
 }
 
@@ -1623,7 +1623,7 @@ llvm.func @_QPomp_atomic_capture_complex() {
         %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
         %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
         omp.yield(%20 : !llvm.struct<(f32, f32)>)
-      }
+      } {atomic_control = #omp.atomic_control<>}
       omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.ptr, !llvm.struct<(f32, f32)>
     }
     llvm.return
@@ -1665,7 +1665,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %expr, %xval : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1683,7 +1683,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1701,7 +1701,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
   // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
   // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
@@ -1710,7 +1710,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1732,7 +1732,7 @@ llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
     %2 = llvm.fadd %1, %0 : f32
     %3 = llvm.fptosi %2 : f32 to i32
     omp.yield(%3 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   llvm.return
 }
 
@@ -1751,7 +1751,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1763,7 +1763,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1775,7 +1775,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1787,7 +1787,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1799,7 +1799,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1814,7 +1814,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1829,7 +1829,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1844,7 +1844,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1859,7 +1859,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1874,7 +1874,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1889,7 +1889,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1904,7 +1904,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1919,7 +1919,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1934,7 +1934,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1949,7 +1949,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1964,7 +1964,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -1979,7 +1979,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -2001,7 +2001,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw sub ptr %[[x]], i32 %[[expr]] monotonic
@@ -2012,7 +2012,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw and ptr %[[x]], i32 %[[expr]] monotonic
@@ -2023,7 +2023,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw or ptr %[[x]], i32 %[[expr]] monotonic
@@ -2034,7 +2034,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw xor ptr %[[x]], i32 %[[expr]] monotonic
@@ -2045,7 +2045,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2060,7 +2060,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2075,7 +2075,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2090,7 +2090,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2105,7 +2105,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2120,7 +2120,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2135,7 +2135,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2150,7 +2150,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2165,7 +2165,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2180,7 +2180,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2195,7 +2195,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2211,7 +2211,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2227,7 +2227,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -2265,7 +2265,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acquire
@@ -2276,7 +2276,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] release
@@ -2287,7 +2287,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
@@ -2298,7 +2298,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acq_rel
@@ -2309,7 +2309,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
 
   llvm.return
@@ -3055,7 +3055,7 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
index cbf273b887bc7..a9a4ffd533136 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
@@ -110,7 +110,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    }
+    } {atomic_control = #omp.atomic_control<>}
 
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 29725a02c075a..0ec244bf28f33 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -19,7 +19,7 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   ^bb0(%arg0: i32):
     %result = llvm.add %arg0, %expr : i32
     omp.yield(%result : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
 
   llvm.return
 }

>From 0b1b23ac98e9486d79b69092698623782ee96921 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 11 Jun 2025 14:11:02 -0500
Subject: [PATCH 3/5] R3: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h       |  2 +-
 .../flang/Optimizer/Dialect/Support/FIRContext.h   |  4 ++--
 flang/lib/Optimizer/Dialect/Support/FIRContext.cpp | 14 +++-----------
 flang/test/Fir/convert-to-llvm-openmp-and-fir.fir  |  4 ++--
 mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td |  3 ++-
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 26256fd775f11..dd29b4e3e8b08 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,7 +54,7 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic Control Options for AMD GPU
+  /// Atomic control options for AMD gpu
   bool amdgpuIgnoreDenormalMode = false;
   bool amdgpuRemoteMemory = false;
   bool amdgpuFineGrainedMemory = false;
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac563bcc402c7..ac5fd33874db8 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,7 +58,7 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and Getters for atomic control options
+// Setters and getters for atomic control options.
 void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
 void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
@@ -70,7 +70,7 @@ bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set Atomic control options for amd gpu.
+// set atomic control options for AMD gpu.
 void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
                                    bool amdgpuIgnoreDenormalMode,
                                    bool amdgpuNoFineGrainedMemory,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index b961793dbdfd5..297a4c7a39e44 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -96,10 +96,7 @@ void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
 }
 
 static constexpr const char *amdgpuFineGrainedMemoryName =
@@ -110,10 +107,7 @@ void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  if (auto attr =
-          mod->getAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
 }
 static constexpr const char *amdgpuRemoteMemoryName =
     "fir.amdgpu.remote.memory";
@@ -123,9 +117,7 @@ void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
 }
 
 bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  if (auto attr = mod->getAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName))
-    return true;
-  return false;
+  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 24e5cad84b709..47c00d489c200 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -619,7 +619,7 @@ func.func @_QPs() {
     %c1_i32 = arith.constant 1 : i32
     %1 = arith.addi %arg0, %c1_i32 : i32
     omp.yield(%1 : i32)
-  }
+  } {atomic_control = #omp.atomic_control<>}
   return
 }
 fir.global internal @_QFsEc : i32 {
@@ -634,7 +634,7 @@ fir.global internal @_QFsEc : i32 {
 // CHECK:      %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:      %[[OUT_VAL:.*]] = llvm.add %[[IN_VAL]], %[[CONST_1]]  : i32
 // CHECK:      omp.yield(%[[OUT_VAL]] : i32)
-// CHECK:    }
+// CHECK:    } {atomic_control = #omp.atomic_control<>}
 // CHECK:    llvm.return
 // CHECK:  }
 // CHECK:  llvm.mlir.global internal @[[GLOBAL]]() {{.*}} : i32 {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 84887dfd2c6f9..61861173ae875 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -58,7 +58,8 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // AtomicControlAttr
 //===----------------------------------------------------------------------===//
 
-// Runtime library flags attribute that holds information for lowering to LLVM.
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
       (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,

>From 565d104ba013acd7f7e64cc989e774f0f1f34e97 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Mon, 23 Jun 2025 12:14:19 -0500
Subject: [PATCH 4/5] R4: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |  6 ++--
 .../Optimizer/Dialect/Support/FIRContext.h    | 18 ++++------
 flang/lib/Frontend/CompilerInvocation.cpp     | 20 +++++------
 flang/lib/Lower/Bridge.cpp                    | 12 +++----
 flang/lib/Lower/OpenMP/Atomic.cpp             |  9 ++++-
 .../Optimizer/Dialect/Support/FIRContext.cpp  | 34 +++++++++----------
 .../Lower/OpenMP/atomic-control-options.f90   | 16 ++++-----
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     |  6 ++--
 mlir/test/Dialect/OpenMP/ops.mlir             |  4 +--
 9 files changed, 61 insertions(+), 64 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index dd29b4e3e8b08..6ef5338acdc6e 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -55,9 +55,9 @@ class TargetOptions {
   bool asmVerbose = false;
 
   /// Atomic control options for AMD gpu
-  bool amdgpuIgnoreDenormalMode = false;
-  bool amdgpuRemoteMemory = false;
-  bool amdgpuFineGrainedMemory = false;
+  bool ignoreDenormalMode = false;
+  bool remoteMemory = false;
+  bool fineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index ac5fd33874db8..273b4683b6d9c 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -59,23 +59,17 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
 // Setters and getters for atomic control options.
-void setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod);
-void setAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-bool getAmdgpuFineGrainedMemory(mlir::ModuleOp mod);
-void setAmdgpuRemoteMemory(mlir::ModuleOp mod);
-bool getAmdgpuRemoteMemory(mlir::ModuleOp mod);
+void setIgnoreDenormalMode(mlir::ModuleOp mod);
+bool getIgnoreDenormalMode(mlir::ModuleOp mod);
+void setFineGrainedMemory(mlir::ModuleOp mod);
+bool getFineGrainedMemory(mlir::ModuleOp mod);
+void setRemoteMemory(mlir::ModuleOp mod);
+bool getRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
 void setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 
-// set atomic control options for AMD gpu.
-void setAmdgpuAtomicControlOptions(mlir::ModuleOp mod,
-                                   bool amdgpuIgnoreDenormalMode,
-                                   bool amdgpuNoFineGrainedMemory,
-                                   bool amdgpuNoRemoteMemory);
-
 /// Get the tune CPU string from the Module or return a null reference.
 llvm::StringRef getTuneCPU(mlir::ModuleOp mod);
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 00f854b905c72..63170a6df6960 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,17 +496,15 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  if (llvm::Triple(opts.triple).isAMDGPU()) {
-    opts.amdgpuIgnoreDenormalMode = args.hasFlag(
-        clang::driver::options::OPT_fatomic_ignore_denormal_mode,
-        clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-    opts.amdgpuFineGrainedMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_fine_grained_memory,
-        clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-    opts.amdgpuRemoteMemory = args.hasFlag(
-        clang::driver::options::OPT_fatomic_remote_memory,
-        clang::driver::options::OPT_fno_atomic_remote_memory, false);
-  }
+  opts.ignoreDenormalMode = args.hasFlag(
+      clang::driver::options::OPT_fatomic_ignore_denormal_mode,
+      clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
+  opts.fineGrainedMemory = args.hasFlag(
+      clang::driver::options::OPT_fatomic_fine_grained_memory,
+      clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
+  opts.remoteMemory =
+      args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
+                   clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_target_cpu))
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index acbfad589dfbb..af6e72aa8cd10 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,12 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.amdgpuIgnoreDenormalMode)
-    fir::setAmdgpuIgnoreDenormalMode(*module);
-  if (targetOpts.amdgpuFineGrainedMemory)
-    fir::setAmdgpuFineGrainedMemory(*module);
-  if (targetOpts.amdgpuRemoteMemory)
-    fir::setAmdgpuRemoteMemory(*module);
+  if (targetOpts.ignoreDenormalMode)
+    fir::setIgnoreDenormalMode(*module);
+  if (targetOpts.fineGrainedMemory)
+    fir::setFineGrainedMemory(*module);
+  if (targetOpts.remoteMemory)
+    fir::setRemoteMemory(*module);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 2ab91b239a3cc..cbc850d67cc4d 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -634,9 +634,16 @@ genAtomicUpdate(lower::AbstractConverter &converter,
     }
   }
 
+  mlir::ModuleOp module = builder.getModule();
+  mlir::omp::AtomicControlAttr atomicControlAttr =
+      mlir::omp::AtomicControlAttr::get(
+          builder.getContext(), fir::getAtomicIgnoreDenormalMode(module),
+          fir::getAtomicFineGrainedMemory(module),
+          fir::getAtomicRemoteMemory(module));
   builder.restoreInsertionPoint(atomicAt);
   auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+      loc, atomAddr, atomicControlAttr, hint,
+      makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 297a4c7a39e44..e3599204f5e2c 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,36 +88,34 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *amdgpuIgnoreDenormalModeName =
-    "fir.amdgpu.ignore.denormal.mode";
-void fir::setAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
+static constexpr const char *ignoreDenormalModeName =
+    "fir.ignore.denormal.mode";
+void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuIgnoreDenormalModeName);
+bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
 }
 
-static constexpr const char *amdgpuFineGrainedMemoryName =
-    "fir.amdgpu.fine.grained.memory";
-void fir::setAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
+static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
+void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuFineGrainedMemoryName);
+bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
 }
-static constexpr const char *amdgpuRemoteMemoryName =
-    "fir.amdgpu.remote.memory";
-void fir::setAmdgpuRemoteMemory(mlir::ModuleOp mod) {
+static constexpr const char *remoteMemoryName = "fir.remote.memory";
+void fir::setRemoteMemory(mlir::ModuleOp mod) {
   auto *ctx = mod.getContext();
-  mod->setAttr(amdgpuRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
 }
 
-bool fir::getAmdgpuRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(amdgpuRemoteMemoryName);
+bool fir::getRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Lower/OpenMP/atomic-control-options.f90 b/flang/test/Lower/OpenMP/atomic-control-options.f90
index 1eb0f617f365e..8f86a151b3713 100644
--- a/flang/test/Lower/OpenMP/atomic-control-options.f90
+++ b/flang/test/Lower/OpenMP/atomic-control-options.f90
@@ -9,25 +9,25 @@ program test
     A = 0
     B = 0
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic
     A =  A + 1
     !$omp end target parallel
     !UNSAFE-FP-ATOMICS: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !UNSAFE-FP-ATOMICS: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !IGNORE-DENORMAL: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true>}
+    !IGNORE-DENORMAL: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
     !FINE-GRAINED-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_fine_grained_memory = true>}
+    !FINE-GRAINED-MEMORY: } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
     !REMOTE-MEMORY: omp.atomic.update %{{.*}} : !fir.ref<i32> {
-    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<amdgpu_remote_memory = true>}
+    !REMOTE-MEMORY: } {atomic_control = #omp.atomic_control<remote_memory = true>}
     !$omp target parallel num_threads(threads)
     !$omp atomic capture
         A = A + B
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 61861173ae875..0fefe7e017e94 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -62,9 +62,9 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
 // characteristics which are required for lowering atomic operations.
 def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
   let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$amdgpu_ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$amdgpu_remote_memory);
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 5aa7befb2dc26..9a72ddfc2989d 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1565,11 +1565,11 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control = #omp.atomic_control<amdgpu_ignore_denormal_mode = true, amdgpu_fine_grained_memory = true, amdgpu_remote_memory = true>}
+  } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true, fine_grained_memory = true, remote_memory = true>}
   return
 }
 

>From 352ae37ba5560cf132f46811d2b6ffb5d9a0c6ce Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Wed, 2 Jul 2025 11:26:03 -0500
Subject: [PATCH 5/5] R5: Addressing review comments

---
 flang/include/flang/Frontend/TargetOptions.h  |   8 +-
 .../Optimizer/Dialect/Support/FIRContext.h    |  25 +++-
 flang/lib/Frontend/CompilerInvocation.cpp     |   6 +-
 flang/lib/Lower/Bridge.cpp                    |  10 +-
 .../Optimizer/Dialect/Support/FIRContext.cpp  |  55 +++++---
 .../Fir/convert-to-llvm-openmp-and-fir.fir    |   4 +-
 .../mlir/Dialect/OpenMP/OpenMPAttrDefs.td     |  30 ++---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   2 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |   2 +-
 mlir/test/Dialect/OpenMP/canonicalize.mlir    |  10 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  46 +++----
 mlir/test/Dialect/OpenMP/ops.mlir             | 122 +++++++++---------
 .../Target/LLVMIR/openmp-llvm-invalid.mlir    |   8 +-
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      |  98 +++++++-------
 .../openmp-target-nesting-in-host-ops.mlir    |   2 +-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |   2 +-
 16 files changed, 228 insertions(+), 202 deletions(-)

diff --git a/flang/include/flang/Frontend/TargetOptions.h b/flang/include/flang/Frontend/TargetOptions.h
index 6ef5338acdc6e..f6e5634d5a995 100644
--- a/flang/include/flang/Frontend/TargetOptions.h
+++ b/flang/include/flang/Frontend/TargetOptions.h
@@ -54,10 +54,10 @@ class TargetOptions {
   /// Print verbose assembly
   bool asmVerbose = false;
 
-  /// Atomic control options for AMD gpu
-  bool ignoreDenormalMode = false;
-  bool remoteMemory = false;
-  bool fineGrainedMemory = false;
+  /// Atomic control options
+  bool atomicIgnoreDenormalMode = false;
+  bool atomicRemoteMemory = false;
+  bool atomicFineGrainedMemory = false;
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index 273b4683b6d9c..c0c0b744206cd 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -58,13 +58,24 @@ void setTargetCPU(mlir::ModuleOp mod, llvm::StringRef cpu);
 /// Get the target CPU string from the Module or return a null reference.
 llvm::StringRef getTargetCPU(mlir::ModuleOp mod);
 
-// Setters and getters for atomic control options.
-void setIgnoreDenormalMode(mlir::ModuleOp mod);
-bool getIgnoreDenormalMode(mlir::ModuleOp mod);
-void setFineGrainedMemory(mlir::ModuleOp mod);
-bool getFineGrainedMemory(mlir::ModuleOp mod);
-void setRemoteMemory(mlir::ModuleOp mod);
-bool getRemoteMemory(mlir::ModuleOp mod);
+/// Sets whether Denormal Mode can be ignored or not for lowering of floating
+/// point atomic operations.
+void setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value);
+/// Gets whether Denormal Mode can be ignored or not for lowering of floating
+/// point atomic operations.
+bool getAtomicIgnoreDenormalMode(mlir::ModuleOp mod);
+/// Sets whether fine grained memory can be used or not for lowering of atomic
+/// operations.
+void setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether fine grained memory can be used or not for lowering of atomic
+/// operations.
+bool getAtomicFineGrainedMemory(mlir::ModuleOp mod);
+/// Sets whether remote memory can be used or not for lowering of atomic
+/// operations.
+void setAtomicRemoteMemory(mlir::ModuleOp mod, bool value);
+/// Gets whether remote memory can be used or not for lowering of atomic
+/// operations.
+bool getAtomicRemoteMemory(mlir::ModuleOp mod);
 
 /// Set the tune CPU for the module. `cpu` must not be deallocated while
 /// module `mod` is still live.
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 63170a6df6960..39021207e897e 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -496,13 +496,13 @@ static void parseTargetArgs(TargetOptions &opts, llvm::opt::ArgList &args) {
           args.getLastArg(clang::driver::options::OPT_triple))
     opts.triple = a->getValue();
 
-  opts.ignoreDenormalMode = args.hasFlag(
+  opts.atomicIgnoreDenormalMode = args.hasFlag(
       clang::driver::options::OPT_fatomic_ignore_denormal_mode,
       clang::driver::options::OPT_fno_atomic_ignore_denormal_mode, false);
-  opts.fineGrainedMemory = args.hasFlag(
+  opts.atomicFineGrainedMemory = args.hasFlag(
       clang::driver::options::OPT_fatomic_fine_grained_memory,
       clang::driver::options::OPT_fno_atomic_fine_grained_memory, false);
-  opts.remoteMemory =
+  opts.atomicRemoteMemory =
       args.hasFlag(clang::driver::options::OPT_fatomic_remote_memory,
                    clang::driver::options::OPT_fno_atomic_remote_memory, false);
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index af6e72aa8cd10..3795ce2e048b6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6666,12 +6666,10 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setKindMapping(*module, kindMap);
   fir::setTargetCPU(*module, targetMachine.getTargetCPU());
   fir::setTuneCPU(*module, targetOpts.cpuToTuneFor);
-  if (targetOpts.ignoreDenormalMode)
-    fir::setIgnoreDenormalMode(*module);
-  if (targetOpts.fineGrainedMemory)
-    fir::setFineGrainedMemory(*module);
-  if (targetOpts.remoteMemory)
-    fir::setRemoteMemory(*module);
+  fir::setAtomicIgnoreDenormalMode(*module,
+                                   targetOpts.atomicIgnoreDenormalMode);
+  fir::setAtomicFineGrainedMemory(*module, targetOpts.atomicFineGrainedMemory);
+  fir::setAtomicRemoteMemory(*module, targetOpts.atomicRemoteMemory);
   fir::setTargetFeatures(*module, targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module, targetMachine.createDataLayout());
   fir::setIdent(*module, Fortran::common::getFlangFullVersion());
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index e3599204f5e2c..c9c921008597d 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -88,34 +88,51 @@ void fir::setTuneCPU(mlir::ModuleOp mod, llvm::StringRef cpu) {
   mod->setAttr(tuneCpuName, mlir::StringAttr::get(ctx, cpu));
 }
 
-static constexpr const char *ignoreDenormalModeName =
-    "fir.ignore.denormal.mode";
-void fir::setIgnoreDenormalMode(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(ignoreDenormalModeName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicIgnoreDenormalModeName =
+    "fir.atomic_ignore_denormal_mode";
+void fir::setAtomicIgnoreDenormalMode(mlir::ModuleOp mod, bool value) {
+  if (value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicIgnoreDenormalModeName, mlir::UnitAttr::get(ctx));
+  } else {
+    if (mod->hasAttr(atomicIgnoreDenormalModeName))
+      mod->removeAttr(atomicIgnoreDenormalModeName);
+  }
 }
 
-bool fir::getIgnoreDenormalMode(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(ignoreDenormalModeName);
+bool fir::getAtomicIgnoreDenormalMode(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicIgnoreDenormalModeName);
 }
 
-static constexpr const char *fineGrainedMemoryName = "fir.fine.grained.memory";
-void fir::setFineGrainedMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(fineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicFineGrainedMemoryName =
+    "fir.atomic_fine_grained_memory";
+void fir::setAtomicFineGrainedMemory(mlir::ModuleOp mod, bool value) {
+  if (value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicFineGrainedMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if (mod->hasAttr(atomicFineGrainedMemoryName))
+      mod->removeAttr(atomicFineGrainedMemoryName);
+  }
 }
 
-bool fir::getFineGrainedMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(fineGrainedMemoryName);
+bool fir::getAtomicFineGrainedMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicFineGrainedMemoryName);
 }
-static constexpr const char *remoteMemoryName = "fir.remote.memory";
-void fir::setRemoteMemory(mlir::ModuleOp mod) {
-  auto *ctx = mod.getContext();
-  mod->setAttr(remoteMemoryName, mlir::UnitAttr::get(ctx));
+static constexpr const char *atomicRemoteMemoryName =
+    "fir.atomic_remote_memory";
+void fir::setAtomicRemoteMemory(mlir::ModuleOp mod, bool value) {
+  if (value) {
+    auto *ctx = mod.getContext();
+    mod->setAttr(atomicRemoteMemoryName, mlir::UnitAttr::get(ctx));
+  } else {
+    if (mod->hasAttr(atomicRemoteMemoryName))
+      mod->removeAttr(atomicRemoteMemoryName);
+  }
 }
 
-bool fir::getRemoteMemory(mlir::ModuleOp mod) {
-  return mod->hasAttrOfType<mlir::UnitAttr>(remoteMemoryName);
+bool fir::getAtomicRemoteMemory(mlir::ModuleOp mod) {
+  return mod->hasAttrOfType<mlir::UnitAttr>(atomicRemoteMemoryName);
 }
 
 llvm::StringRef fir::getTuneCPU(mlir::ModuleOp mod) {
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 47c00d489c200..24e5cad84b709 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -619,7 +619,7 @@ func.func @_QPs() {
     %c1_i32 = arith.constant 1 : i32
     %1 = arith.addi %arg0, %c1_i32 : i32
     omp.yield(%1 : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 fir.global internal @_QFsEc : i32 {
@@ -634,7 +634,7 @@ fir.global internal @_QFsEc : i32 {
 // CHECK:      %[[CONST_1:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:      %[[OUT_VAL:.*]] = llvm.add %[[IN_VAL]], %[[CONST_1]]  : i32
 // CHECK:      omp.yield(%[[OUT_VAL]] : i32)
-// CHECK:    } {atomic_control = #omp.atomic_control<>}
+// CHECK:    }
 // CHECK:    llvm.return
 // CHECK:  }
 // CHECK:  llvm.mlir.global internal @[[GLOBAL]]() {{.*}} : i32 {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 0fefe7e017e94..72ce4c6a21cb3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -22,6 +22,21 @@ class OpenMP_Attr<string name, string attrMnemonic, list<Trait> traits = [],
   let mnemonic = attrMnemonic;
 }
 
+//===----------------------------------------------------------------------===//
+// AtomicControlAttr
+//===----------------------------------------------------------------------===//
+
+// Atomic control attributes hold information about architectural
+// characteristics which are required for lowering atomic operations.
+def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
+  let parameters =
+      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
+          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
+          DefaultValuedParameter<"bool", "false">:$remote_memory);
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // DeclareTargetAttr
 //===----------------------------------------------------------------------===//
@@ -54,21 +69,6 @@ def FlagsAttr : OpenMP_Attr<"Flags", "flags"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicControlAttr
-//===----------------------------------------------------------------------===//
-
-// Atomic control attributes hold information about architectural
-// characteristics which are required for lowering atomic operations.
-def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
-  let parameters =
-      (ins DefaultValuedParameter<"bool", "false">:$ignore_denormal_mode,
-          DefaultValuedParameter<"bool", "false">:$fine_grained_memory,
-          DefaultValuedParameter<"bool", "false">:$remote_memory);
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
 //===----------------------------------------------------------------------===//
 // TaskDependArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 399f7fa6e78f0..375e35a146f78 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1546,7 +1546,7 @@ def AtomicUpdateOp : OpenMP_Op<"atomic.update", traits = [
   let arguments = !con(
       (ins Arg<OpenMP_PointerLikeType,
                "Address of variable to be updated", [MemRead, MemWrite]>:$x,
-          AtomicControlAttr:$atomic_control),
+          OptionalAttr<AtomicControlAttr>:$atomic_control),
       clausesArgs);
 
   // Override region definition.
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index 0b2445d0a0d18..d69de998346b5 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -133,7 +133,7 @@ func.func @atomic_update() {
     %1 = arith.constant 1 : i32
     %2 = arith.addi %arg0, %1  : i32
     omp.yield(%2 : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 llvm.mlir.global internal @_QFsEc() : i32 {
diff --git a/mlir/test/Dialect/OpenMP/canonicalize.mlir b/mlir/test/Dialect/OpenMP/canonicalize.mlir
index 2492010c4d3c7..de6c931ecc5fd 100644
--- a/mlir/test/Dialect/OpenMP/canonicalize.mlir
+++ b/mlir/test/Dialect/OpenMP/canonicalize.mlir
@@ -4,7 +4,7 @@ func.func @update_no_op(%x : memref<i32>) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%xval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -17,7 +17,7 @@ func.func @update_write_op(%x : memref<i32>, %value: i32) {
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval : i32):
     omp.yield(%value : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -33,7 +33,7 @@ func.func @update_normal(%x : memref<i32>, %value: i32) {
   ^bb0(%xval : i32):
     %newval = arith.addi %xval, %value : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -50,7 +50,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.addi %xval, %c0 : i32
     omp.yield(%newval: i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -65,7 +65,7 @@ func.func @update_unnecessary_computations(%x: memref<i32>) {
   ^bb0(%xval: i32):
     %newval = arith.muli %xval, %c0 : i32
     omp.yield(%newval: i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index afc3830286f30..060b3cd2455a0 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1007,7 +1007,7 @@ func.func @omp_atomic_update1(%x: memref<i32>, %expr: f32) {
   ^bb0(%xval: f32):
     %newval = llvm.fadd %xval, %expr : f32
     omp.yield (%newval : f32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1020,7 +1020,7 @@ func.func @omp_atomic_update2(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.terminator
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1032,7 +1032,7 @@ func.func @omp_atomic_update3(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1044,7 +1044,7 @@ func.func @omp_atomic_update4(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1056,7 +1056,7 @@ func.func @omp_atomic_update5(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1068,7 +1068,7 @@ func.func @omp_atomic_update6(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval, %expr : i32, i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1080,7 +1080,7 @@ func.func @omp_atomic_update7(%x: memref<i32>, %expr: i32, %y: f32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%y: f32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1092,7 +1092,7 @@ func.func @omp_atomic_update8(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32, %tmp: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1104,7 +1104,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1116,7 +1116,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1128,7 +1128,7 @@ func.func @omp_atomic_update(%x: memref<i32>, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield (%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
@@ -1164,12 +1164,12 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.update %x : memref<i32> {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.terminator
   }
   return
@@ -1197,7 +1197,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.terminator
   }
   return
@@ -1212,7 +1212,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.write %x = %expr : memref<i32>, i32
     omp.terminator
   }
@@ -1240,7 +1240,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %y : memref<i32>, memref<i32>, i32
     omp.terminator
   }
@@ -1256,7 +1256,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %y: memref<i32>, %v: memref<i32>,
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield (%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.terminator
   }
 }
@@ -1281,7 +1281,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1296,7 +1296,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1311,7 +1311,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1326,7 +1326,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1341,7 +1341,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   return
@@ -1356,7 +1356,7 @@ func.func @omp_atomic_capture(%x: memref<i32>, %v: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x memory_order(seq_cst) : memref<i32>, memref<i32>, i32
   }
   return
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 9a72ddfc2989d..744803e0761d9 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1371,7 +1371,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.and %[[XVAL]], %[[EXPRBOOL]] : i1
@@ -1380,57 +1380,57 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i1):
     %newval = llvm.and %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.shl %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.intr.smax(%[[XVAL]], %[[EXPR]]) : (i32, i32) -> i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval: i32):
     %newval = llvm.intr.smax(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update %[[XBOOL]] : memref<i1>
   // CHECK-NEXT: (%[[XVAL:.*]]: i1):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.icmp "eq" %[[XVAL]], %[[EXPRBOOL]] : i1
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i1)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   omp.atomic.update %xBool : memref<i1> {
   ^bb0(%xval: i1):
     %newval = llvm.icmp "eq" %xval, %exprBool : i1
     omp.yield(%newval : i1)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%xval:i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%{{.+}} : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   %const = arith.constant 42 : i32
   omp.atomic.update %x : memref<i32> {
   ^bb0(%xval:i32):
     omp.yield(%const:i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1440,7 +1440,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(uncontended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1450,7 +1450,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(contended) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1460,7 +1460,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1470,7 +1470,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1480,7 +1480,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(uncontended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1490,7 +1490,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(contended, nonspeculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1500,7 +1500,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1510,7 +1510,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(contended, speculative) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1520,7 +1520,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1530,7 +1530,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update memory_order(release) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1540,7 +1540,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update memory_order(relaxed) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1550,7 +1550,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update hint(uncontended, speculative) memory_order(seq_cst) %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1560,7 +1560,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   // CHECK: omp.atomic.update %[[X]] : memref<i32> {
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
@@ -1581,7 +1581,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture{
@@ -1589,7 +1589,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
   // CHECK: omp.atomic.capture {
@@ -1598,7 +1598,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: }
   omp.atomic.capture{
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
@@ -1606,7 +1606,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
   // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
@@ -1622,7 +1622,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(none) {
@@ -1630,7 +1630,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1639,7 +1639,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended) {
@@ -1647,7 +1647,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1656,7 +1656,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended) {
@@ -1664,7 +1664,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1673,7 +1673,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(nonspeculative) {
@@ -1681,7 +1681,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1690,7 +1690,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(speculative) {
@@ -1698,7 +1698,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1707,7 +1707,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, nonspeculative) {
@@ -1715,7 +1715,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1724,7 +1724,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, nonspeculative) {
@@ -1732,7 +1732,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1741,7 +1741,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(uncontended, speculative) {
@@ -1749,7 +1749,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1758,7 +1758,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) {
@@ -1766,7 +1766,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1775,7 +1775,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(seq_cst) {
@@ -1783,7 +1783,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1792,7 +1792,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acq_rel) {
@@ -1800,7 +1800,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1809,7 +1809,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(acquire) {
@@ -1817,7 +1817,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1826,7 +1826,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(release) {
@@ -1834,7 +1834,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1843,7 +1843,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture memory_order(relaxed) {
@@ -1851,7 +1851,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -1860,7 +1860,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
   // CHECK-NEXT:   omp.yield(%[[newval]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   // CHECK-NEXT: omp.atomic.read %[[v]] = %[[x]] : memref<i32>, memref<i32>, i32
   // CHECK-NEXT: }
   omp.atomic.capture hint(contended, speculative) memory_order(seq_cst) {
@@ -1868,7 +1868,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : memref<i32>, memref<i32>, i32
   }
 
@@ -2652,14 +2652,14 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
   // CHECK: omp.atomic.update %[[x]] : !llvm.ptr {
   // CHECK-NEXT: ^{{[[:alnum:]]+}}(%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   omp.yield(%[[XVAL]] : i32)
-  // CHECK-NEXT: } {atomic_control = #omp.atomic_control<>}
+  // CHECK-NEXT: }
   omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   %val = llvm.load %x : !llvm.ptr -> i32
   omp.atomic.write %v = %val : !llvm.ptr, i32
   omp.atomic.update %x : !llvm.ptr {
     ^bb0(%xval: i32):
       omp.yield(%xval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index 00f53864b5fa7..41bc5c4ba525f 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -9,7 +9,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
     %t2 = llvm.sdiv %t1, %expr : i32
     %newval = llvm.add %xval, %t2 : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   llvm.return
 }
 
@@ -24,7 +24,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.mul %expr, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   llvm.return
 }
 
@@ -41,7 +41,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
     ^bb0(%xval: i32):
       %newval = llvm.mul %expr, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
   llvm.return
 }
@@ -59,7 +59,7 @@ llvm.func @omp_atomic_update_multiple_step_update(%x: !llvm.ptr, %v: !llvm.ptr,
       %t2 = llvm.sdiv %t1, %expr : i32
       %newval = llvm.add %xval, %t2 : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index ab7f756bf95f9..32f0ba5b105ff 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1462,13 +1462,13 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
   ^bb0(%xval: i32):
     %newval = llvm.mul %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   // CHECK: atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
   omp.atomic.update %x : !llvm.ptr {
   ^bb0(%xval: i32):
     %newval = llvm.add %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   llvm.return
 }
 
@@ -1563,7 +1563,7 @@ llvm.func @_QPomp_atomic_update_complex() {
       %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
       %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
       omp.yield(%16 : !llvm.struct<(f32, f32)>)
-    } {atomic_control = #omp.atomic_control<>}
+    }
    llvm.return
 }
 
@@ -1623,7 +1623,7 @@ llvm.func @_QPomp_atomic_capture_complex() {
         %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
         %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
         omp.yield(%20 : !llvm.struct<(f32, f32)>)
-      } {atomic_control = #omp.atomic_control<>}
+      }
       omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.ptr, !llvm.struct<(f32, f32)>
     }
     llvm.return
@@ -1665,7 +1665,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %expr, %xval : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   llvm.return
 }
 
@@ -1683,7 +1683,7 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = llvm.shl %xval, %expr : i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   llvm.return
 }
 
@@ -1701,7 +1701,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
   // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
   // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
@@ -1710,7 +1710,7 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
   ^bb0(%xval: i32):
     %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
     omp.yield(%newval : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   llvm.return
 }
 
@@ -1732,7 +1732,7 @@ llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
     %2 = llvm.fadd %1, %0 : f32
     %3 = llvm.fptosi %2 : f32 to i32
     omp.yield(%3 : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
   llvm.return
 }
 
@@ -1751,7 +1751,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1763,7 +1763,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1775,7 +1775,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1787,7 +1787,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1799,7 +1799,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1814,7 +1814,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1829,7 +1829,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1844,7 +1844,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1859,7 +1859,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1874,7 +1874,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1889,7 +1889,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1904,7 +1904,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1919,7 +1919,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1934,7 +1934,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1949,7 +1949,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
   }
 
@@ -1964,7 +1964,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -1979,7 +1979,7 @@ llvm.func @omp_atomic_capture_prefix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
     omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
   }
 
@@ -2001,7 +2001,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[res:.*]] = atomicrmw sub ptr %[[x]], i32 %[[expr]] monotonic
@@ -2012,7 +2012,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sub %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[res:.*]] = atomicrmw and ptr %[[x]], i32 %[[expr]] monotonic
@@ -2023,7 +2023,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.and %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[res:.*]] = atomicrmw or ptr %[[x]], i32 %[[expr]] monotonic
@@ -2034,7 +2034,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.or %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[res:.*]] = atomicrmw xor ptr %[[x]], i32 %[[expr]] monotonic
@@ -2045,7 +2045,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.xor %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2060,7 +2060,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.mul %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2075,7 +2075,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.sdiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2090,7 +2090,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.udiv %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2105,7 +2105,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.shl %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2120,7 +2120,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.lshr %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2135,7 +2135,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = llvm.ashr %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2150,7 +2150,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2165,7 +2165,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.smin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2180,7 +2180,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2195,7 +2195,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: i32):
       %newval = "llvm.intr.umin"(%xval, %expr) : (i32, i32) -> i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2211,7 +2211,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fadd %xval, %exprf : f32
       omp.yield(%newval : f32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[xval:.*]] = phi i32
@@ -2227,7 +2227,7 @@ llvm.func @omp_atomic_capture_postfix_update(
     ^bb0(%xval: f32):
       %newval = llvm.fsub %xval, %exprf : f32
       omp.yield(%newval : f32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   llvm.return
@@ -2265,7 +2265,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acquire
@@ -2276,7 +2276,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] release
@@ -2287,7 +2287,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] monotonic
@@ -2298,7 +2298,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   // CHECK: %[[res:.*]] = atomicrmw add ptr %[[x]], i32 %[[expr]] acq_rel
@@ -2309,7 +2309,7 @@ llvm.func @omp_atomic_capture_misc(
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
 
   llvm.return
@@ -3055,7 +3055,7 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
index a9a4ffd533136..cbf273b887bc7 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-nesting-in-host-ops.mlir
@@ -110,7 +110,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
     ^bb0(%xval: i32):
       %newval = llvm.add %xval, %expr : i32
       omp.yield(%newval : i32)
-    } {atomic_control = #omp.atomic_control<>}
+    }
 
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 0ec244bf28f33..29725a02c075a 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -19,7 +19,7 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   ^bb0(%arg0: i32):
     %result = llvm.add %arg0, %expr : i32
     omp.yield(%result : i32)
-  } {atomic_control = #omp.atomic_control<>}
+  }
 
   llvm.return
 }


From flang-commits at lists.llvm.org  Mon Jul 14 08:24:54 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 14 Jul 2025 08:24:54 -0700 (PDT)
Subject: [flang-commits]  [flang] [flang] Extension: TRANSFER(boz,
	integer or real scalar) (PR #147604)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147604@github.com>
Message-ID: <68752146.630a0220.1a4bae.82ba@mx.google.com>


https://github.com/klausler updated https://github.com/llvm/llvm-project/pull/147604

>From 9dca92c88e565b007e125414086c7c8a8dec8d5b Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 8 Jul 2025 15:17:16 -0700
Subject: [PATCH] [flang] Extension: TRANSFER(boz, integer or real scalar)

Interpret TRANSFER(SOURCE=BOZ literal, MOLD=integer or real scalar)
as if it had been a reference to the standard INT or REAL intrinsic
functions, for which a BOZ literal is an acceptable argument, with
a warning about non-conformance.  It's a needless extension that
has somehow crept into some other compilers and real applications.
---
 flang/docs/Extensions.md                      |  2 ++
 .../include/flang/Support/Fortran-features.h  |  3 ++-
 flang/lib/Evaluate/fold.cpp                   | 10 ++++++++++
 flang/lib/Evaluate/intrinsics.cpp             |  9 +++++++++
 flang/lib/Support/Fortran-features.cpp        |  1 +
 flang/test/Evaluate/transfer-boz.f90          | 19 +++++++++++++++++++
 6 files changed, 43 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Evaluate/transfer-boz.f90

diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 37a49f12f9177..72d12cd92600d 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -252,6 +252,8 @@ end
   the type resolution of such BOZ literals usages is highly non portable).
 * BOZ literals can also be used as REAL values in some contexts where the
   type is unambiguous, such as initializations of REAL parameters.
+* `TRANSFER(boz, MOLD=integer or real scalar)` is accepted as an alternate
+  spelling of `INT(boz, KIND=kind(mold))` or `REAL(boz, KIND=kind(mold))`.
 * EQUIVALENCE of numeric and character sequences (a ubiquitous extension),
   as well as of sequences of non-default kinds of numeric types
   with each other.
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 857de9479e4e5..743abf606ab5f 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,8 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload,
+    TransferBOZ)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
diff --git a/flang/lib/Evaluate/fold.cpp b/flang/lib/Evaluate/fold.cpp
index 45e842abf589f..71ead1b3afa91 100644
--- a/flang/lib/Evaluate/fold.cpp
+++ b/flang/lib/Evaluate/fold.cpp
@@ -287,6 +287,16 @@ std::optional<Expr<SomeType>> FoldTransfer(
         CHECK(status == InitialImage::NotAConstant);
       }
     }
+  } else if (source && moldType) {
+    if (const auto *boz{std::get_if<BOZLiteralConstant>(&source->u)}) {
+      // TRANSFER(BOZ, MOLD=integer or real) extension
+      if (context.languageFeatures().ShouldWarn(
+              common::LanguageFeature::TransferBOZ)) {
+        context.messages().Say(common::LanguageFeature::TransferBOZ,
+            "TRANSFER(BOZ literal) is not standard"_port_en_US);
+      }
+      return Fold(context, ConvertToType(*moldType, Expr<SomeType>{*boz}));
+    }
   }
   return std::nullopt;
 }
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index e802915945e26..4773e136c41cb 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1015,6 +1015,15 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
             {"mold", SameType, Rank::anyOrAssumedRank},
             {"size", AnyInt, Rank::scalar}},
         SameType, Rank::vector, IntrinsicClass::transformationalFunction},
+    // TRANSFER(BOZ, MOLD=integer or real scalar) extension
+    {"transfer",
+        {{"source", AnyNumeric, Rank::elementalOrBOZ},
+            {"mold", SameInt, Rank::scalar}},
+        SameInt, Rank::scalar, IntrinsicClass::transformationalFunction},
+    {"transfer",
+        {{"source", AnyNumeric, Rank::elementalOrBOZ},
+            {"mold", SameReal, Rank::scalar}},
+        SameReal, Rank::scalar, IntrinsicClass::transformationalFunction},
     {"transpose", {{"matrix", SameType, Rank::matrix}}, SameType, Rank::matrix,
         IntrinsicClass::transformationalFunction},
     {"trim", {{"string", SameCharNoLen, Rank::scalar}}, SameCharNoLen,
diff --git a/flang/lib/Support/Fortran-features.cpp b/flang/lib/Support/Fortran-features.cpp
index fc69fc638eda1..df51b3c577125 100644
--- a/flang/lib/Support/Fortran-features.cpp
+++ b/flang/lib/Support/Fortran-features.cpp
@@ -103,6 +103,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
   warnLanguage_.set(LanguageFeature::ListDirectedSize);
   warnLanguage_.set(LanguageFeature::IgnoreIrrelevantAttributes);
   warnLanguage_.set(LanguageFeature::AmbiguousStructureConstructor);
+  warnLanguage_.set(LanguageFeature::TransferBOZ);
   warnUsage_.set(UsageWarning::ShortArrayActual);
   warnUsage_.set(UsageWarning::FoldingException);
   warnUsage_.set(UsageWarning::FoldingAvoidsRuntimeCrash);
diff --git a/flang/test/Evaluate/transfer-boz.f90 b/flang/test/Evaluate/transfer-boz.f90
new file mode 100644
index 0000000000000..584cf318a2818
--- /dev/null
+++ b/flang/test/Evaluate/transfer-boz.f90
@@ -0,0 +1,19 @@
+!RUN: %flang_fc1 -fdebug-unparse %s | FileCheck %s
+
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: PRINT *, -17_1
+!CHECK: PRINT *, -16657_2
+!CHECK: PRINT *, -559038737_4
+!CHECK: PRINT *, -6.259853398707798016e18_4
+!CHECK: PRINT *, 1.8457939563190925445492984919045437485528002903297587499184927815557801644025850059853181797614625558939883606558067941052215743386077465406508281130260792904117042434297523782789810991060595088402505831985608687099349932384478542748214418349497866218005870896526832696740683567118136284392505567470522678176873737470272665943917715636680452977840047387616237048162886116765806372690690621665407240253312632381089695794498069551496558727944808752442655077077609749137334421734380446229972733723471196114034171513283476953482396960122117078210644171413215132537631249069125441199719426016512325803207900875869312966829207468856964517085119314095898691593840972164432952065765882286023532502465793032567186884340163939544586513274992967126308940351009368896484375e-314_8
+
+print *, transfer(z'deadbeef', 1_1)
+print *, transfer(z'deadbeef', 1_2)
+print *, transfer(z'deadbeef', 1_4)
+print *, transfer(z'deadbeef', 1._4)
+print *, transfer(z'deadbeef', 1._8)
+end


From flang-commits at lists.llvm.org  Mon Jul 14 08:47:17 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 08:47:17 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <68752685.630a0220.e2e5a.9752@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
----------------
skc7 wrote:

Moved these functions to utils.h in latest patch.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Mon Jul 14 08:47:30 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 08:47:30 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <68752692.170a0220.13a34b.756b@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    mlir::Type llvmPtrTy =
+        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
----------------
skc7 wrote:

Thanks. Updated.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Mon Jul 14 08:48:17 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 08:48:17 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <687526c1.050a0220.37c087.9e54@mx.google.com>


================
@@ -125,10 +125,177 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
----------------
skc7 wrote:

Thanks @ergawy. This does fix the issue I was facing. Updated the patch.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Mon Jul 14 08:29:40 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 14 Jul 2025 08:29:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't emit needless symbols to
 hermetic module files (PR #144765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144765@github.com>
Message-ID: <68752264.170a0220.384fe9.94e8@mx.google.com>


https://github.com/klausler updated https://github.com/llvm/llvm-project/pull/144765

>From de069ae733d7b002a7317f6288bdf49bd6b86267 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 17 Jun 2025 11:51:02 -0700
Subject: [PATCH 1/2] [flang] Don't emit needless symbols to hermetic module
 files

When emitting the dependent modules for a hermetic module file,
omit symbols that are not necessary (directly or otherwise) for
the declarations and definitions in the main module.
---
 .../flang/Semantics/symbol-dependence.h       |  36 ++
 flang/include/flang/Semantics/tools.h         |   2 +-
 flang/lib/Semantics/CMakeLists.txt            |   1 +
 flang/lib/Semantics/check-declarations.cpp    |  12 +-
 flang/lib/Semantics/expression.cpp            |   3 +-
 flang/lib/Semantics/mod-file.cpp              | 377 +++++++++++-------
 flang/lib/Semantics/mod-file.h                |  11 +-
 flang/lib/Semantics/resolve-names.cpp         |  22 +-
 flang/lib/Semantics/semantics.cpp             |  12 +-
 flang/lib/Semantics/symbol-dependence.cpp     | 356 +++++++++++++++++
 flang/lib/Semantics/symbol.cpp                |  28 +-
 flang/lib/Semantics/tools.cpp                 |  33 +-
 flang/lib/Semantics/type.cpp                  |  26 +-
 flang/test/Semantics/modfile03.f90            |   8 +-
 flang/test/Semantics/modfile65.f90            |   6 +-
 flang/test/Semantics/modfile78.F90            |   1 -
 16 files changed, 718 insertions(+), 216 deletions(-)
 create mode 100644 flang/include/flang/Semantics/symbol-dependence.h
 create mode 100644 flang/lib/Semantics/symbol-dependence.cpp

diff --git a/flang/include/flang/Semantics/symbol-dependence.h b/flang/include/flang/Semantics/symbol-dependence.h
new file mode 100644
index 0000000000000..9bcff564a4c04
--- /dev/null
+++ b/flang/include/flang/Semantics/symbol-dependence.h
@@ -0,0 +1,36 @@
+//===-- include/flang/Semantics/symbol-dependence.h -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
+#define FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
+
+#include "flang/Semantics/symbol.h"
+
+namespace Fortran::semantics {
+
+// For a set or scope of symbols, computes the transitive closure of their
+// dependences due to their types, bounds, specific procedures, interfaces,
+// initialization, storage association, &c. Includes the original symbol
+// or members of the original set.  Does not include dependences from
+// subprogram definitions, only their interfaces.
+enum DependenceCollectionFlags {
+  NoDependenceCollectionFlags = 0,
+  IncludeOriginalSymbols = 1 << 0,
+  FollowUseAssociations = 1 << 1,
+  IncludeSpecificsOfGenerics = 1 << 2,
+  IncludeUsesOfGenerics = 1 << 3,
+  NotJustForOneModule = 1 << 4,
+};
+
+SymbolVector CollectAllDependences(const SymbolVector &,
+    int = NoDependenceCollectionFlags, const Scope * = nullptr);
+SymbolVector CollectAllDependences(
+    const Scope &, int = NoDependenceCollectionFlags);
+
+} // namespace Fortran::semantics
+#endif // FORTRAN_SEMANTICS_SYMBOL_DEPENDENCE_H_
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index fb670528f3ce4..2050d23870657 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -43,7 +43,7 @@ const Scope &GetProgramUnitOrBlockConstructContaining(const Symbol &);
 
 const Scope *FindModuleContaining(const Scope &);
 const Scope *FindModuleOrSubmoduleContaining(const Scope &);
-const Scope *FindModuleFileContaining(const Scope &);
+bool IsInModuleFile(const Scope &);
 const Scope *FindPureProcedureContaining(const Scope &);
 const Scope *FindOpenACCConstructContaining(const Scope *);
 
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 109bc2dbb8569..c711103e793d8 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -48,6 +48,7 @@ add_flang_library(FortranSemantics
   runtime-type-info.cpp
   scope.cpp
   semantics.cpp
+  symbol-dependence.cpp
   symbol.cpp
   tools.cpp
   type.cpp
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index f9d64485f1407..36e47f7999538 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -124,10 +124,7 @@ class CheckHelper {
     }
     return msg;
   }
-  bool InModuleFile() const {
-    return FindModuleFileContaining(context_.FindScope(messages_.at())) !=
-        nullptr;
-  }
+  bool InModuleFile() const { return context_.IsInModuleFile(messages_.at()); }
   template <typename FeatureOrUsageWarning, typename... A>
   parser::Message *Warn(FeatureOrUsageWarning warning, A &&...x) {
     if (!context_.ShouldWarn(warning) || InModuleFile()) {
@@ -139,8 +136,7 @@ class CheckHelper {
   template <typename FeatureOrUsageWarning, typename... A>
   parser::Message *Warn(
       FeatureOrUsageWarning warning, parser::CharBlock source, A &&...x) {
-    if (!context_.ShouldWarn(warning) ||
-        FindModuleFileContaining(context_.FindScope(source))) {
+    if (!context_.ShouldWarn(warning) || context_.IsInModuleFile(source)) {
       return nullptr;
     } else {
       return messages_.Say(warning, source, std::forward<A>(x)...);
@@ -4050,7 +4046,7 @@ void DistinguishabilityHelper::Add(const Symbol &generic, GenericKind kind,
 }
 
 void DistinguishabilityHelper::Check(const Scope &scope) {
-  if (FindModuleFileContaining(scope)) {
+  if (IsInModuleFile(scope)) {
     // Distinguishability was checked when the module was created;
     // don't let optional warnings then become errors now.
     return;
@@ -4109,7 +4105,7 @@ void DistinguishabilityHelper::SayNotDistinguishable(const Scope &scope,
   if (isWarning &&
       (!context_.ShouldWarn(
            common::LanguageFeature::IndistinguishableSpecifics) ||
-          FindModuleFileContaining(scope))) {
+          IsInModuleFile(scope))) {
     return;
   }
   std::string name1{proc1.name().ToString()};
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index f4af738284ed7..50c6d51abebdf 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -1852,8 +1852,7 @@ void ArrayConstructorContext::Push(MaybeExpr &&x) {
     } else {
       if (!(messageDisplayedSet_ & 2)) {
         exprAnalyzer_.Say(
-            "Values in array constructor must have the same declared type "
-            "when no explicit type appears"_err_en_US); // C7110
+            "Values in array constructor must have the same declared type when no explicit type appears"_err_en_US); // C7110
         messageDisplayedSet_ |= 2;
       }
     }
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index 82c8536902eb2..1a27449e5a786 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -15,6 +15,7 @@
 #include "flang/Parser/unparse.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol-dependence.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/Support/FileSystem.h"
@@ -130,7 +131,7 @@ static std::string ModFileName(const SourceName &name,
   return ancestorName.empty() ? result : ancestorName + '-' + result;
 }
 
-// Write the module file for symbol, which must be a module or submodule.
+// Writes the module file for symbol, which must be a module or submodule.
 void ModFileWriter::Write(const Symbol &symbol) {
   const auto &module{symbol.get<ModuleDetails>()};
   if (symbol.test(Symbol::Flag::ModFile) || module.moduleFileHash()) {
@@ -142,26 +143,15 @@ void ModFileWriter::Write(const Symbol &symbol) {
   std::string path{context_.moduleDirectory() + '/' +
       ModFileName(symbol.name(), ancestorName, context_.moduleFileSuffix())};
 
-  std::set<std::string> hermeticModuleNames;
-  hermeticModuleNames.insert(symbol.name().ToString());
-  UnorderedSymbolSet additionalModules;
-  PutSymbols(DEREF(symbol.scope()),
-      hermeticModuleFileOutput_ ? &additionalModules : nullptr);
-  auto asStr{GetAsString(symbol)};
-  while (!additionalModules.empty()) {
-    UnorderedSymbolSet nextPass{std::move(additionalModules)};
-    additionalModules.clear();
-    for (const Symbol &modSym : nextPass) {
-      if (!modSym.owner().IsIntrinsicModules() &&
-          hermeticModuleNames.find(modSym.name().ToString()) ==
-              hermeticModuleNames.end()) {
-        hermeticModuleNames.insert(modSym.name().ToString());
-        PutSymbols(DEREF(modSym.scope()), &additionalModules);
-        asStr += GetAsString(modSym);
-      }
-    }
+  SymbolVector dependenceClosure;
+  if (hermeticModuleFileOutput_ && !isSubmodule_) {
+    dependenceClosure = CollectAllDependences(DEREF(symbol.scope()),
+        FollowUseAssociations | IncludeUsesOfGenerics |
+            IncludeSpecificsOfGenerics | NotJustForOneModule);
   }
-
+  PutSymbols(DEREF(symbol.scope()), hermeticModuleFileOutput_);
+  auto asStr{GetAsString(&symbol, symbol.name().ToString())};
+  asStr += PutDependencyModules(symbol.name().ToString(), dependenceClosure);
   ModuleCheckSumType checkSum;
   if (std::error_code error{
           WriteFile(path, asStr, checkSum, context_.debugModuleWriter())}) {
@@ -177,9 +167,9 @@ void ModFileWriter::WriteClosure(llvm::raw_ostream &out, const Symbol &symbol,
       !nonIntrinsicModulesWritten.insert(symbol).second) {
     return;
   }
-  PutSymbols(DEREF(symbol.scope()), /*hermeticModules=*/nullptr);
+  PutSymbols(DEREF(symbol.scope()), /*omitModules=*/false);
   needsBuf_.clear(); // omit module checksums
-  auto str{GetAsString(symbol)};
+  auto str{GetAsString(&symbol, symbol.name().ToString())};
   for (auto depRef : std::move(usedNonIntrinsicModules_)) {
     WriteClosure(out, *depRef, nonIntrinsicModulesWritten);
   }
@@ -188,22 +178,23 @@ void ModFileWriter::WriteClosure(llvm::raw_ostream &out, const Symbol &symbol,
 
 // Return the entire body of the module file
 // and clear saved uses, decls, and contains.
-std::string ModFileWriter::GetAsString(const Symbol &symbol) {
+std::string ModFileWriter::GetAsString(const Symbol *symbol, std::string name) {
   std::string buf;
   llvm::raw_string_ostream all{buf};
   all << needs_.str();
   needs_.str().clear();
-  auto &details{symbol.get<ModuleDetails>()};
-  if (!details.isSubmodule()) {
-    all << "module " << symbol.name();
+  const ModuleDetails *details{
+      symbol ? &symbol->get<ModuleDetails>() : nullptr};
+  if (!details || !details->isSubmodule()) {
+    all << "module " << name;
   } else {
-    auto *parent{details.parent()->symbol()};
-    auto *ancestor{details.ancestor()->symbol()};
+    auto *parent{details->parent()->symbol()};
+    auto *ancestor{details->ancestor()->symbol()};
     all << "submodule(" << ancestor->name();
     if (parent != ancestor) {
       all << ':' << parent->name();
     }
-    all << ") " << symbol.name();
+    all << ") " << name;
   }
   all << '\n' << uses_.str();
   uses_.str().clear();
@@ -223,79 +214,22 @@ std::string ModFileWriter::GetAsString(const Symbol &symbol) {
 // Collect symbols from constant and specification expressions that are being
 // referenced directly from other modules; they may require new USE
 // associations.
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &, const Scope &);
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &set, const Symbol &symbol, const Scope &scope) {
-  auto HarvestBound{[&](const Bound &bound) {
-    if (const auto &expr{bound.GetExplicit()}) {
-      for (SymbolRef ref : evaluate::CollectSymbols(*expr)) {
-        set.emplace(*ref);
-      }
-    }
-  }};
-  auto HarvestShapeSpec{[&](const ShapeSpec &shapeSpec) {
-    HarvestBound(shapeSpec.lbound());
-    HarvestBound(shapeSpec.ubound());
-  }};
-  auto HarvestArraySpec{[&](const ArraySpec &arraySpec) {
-    for (const auto &shapeSpec : arraySpec) {
-      HarvestShapeSpec(shapeSpec);
-    }
-  }};
-
-  if (symbol.has<DerivedTypeDetails>()) {
-    if (symbol.scope()) {
-      HarvestSymbolsNeededFromOtherModules(set, *symbol.scope());
-    }
-  } else if (const auto &generic{symbol.detailsIf<GenericDetails>()};
-             generic && generic->derivedType()) {
-    const Symbol &dtSym{*generic->derivedType()};
-    if (dtSym.has<DerivedTypeDetails>()) {
-      if (dtSym.scope()) {
-        HarvestSymbolsNeededFromOtherModules(set, *dtSym.scope());
-      }
-    } else {
-      CHECK(dtSym.has<UseDetails>() || dtSym.has<UseErrorDetails>());
-    }
-  } else if (const auto *object{symbol.detailsIf<ObjectEntityDetails>()}) {
-    HarvestArraySpec(object->shape());
-    HarvestArraySpec(object->coshape());
-    if (IsNamedConstant(symbol) || scope.IsDerivedType()) {
-      if (object->init()) {
-        for (SymbolRef ref : evaluate::CollectSymbols(*object->init())) {
-          set.emplace(*ref);
-        }
-      }
-    }
-  } else if (const auto *proc{symbol.detailsIf<ProcEntityDetails>()}) {
-    if (proc->init() && *proc->init() && scope.IsDerivedType()) {
-      set.emplace(**proc->init());
-    }
-  } else if (const auto *subp{symbol.detailsIf<SubprogramDetails>()}) {
-    for (const Symbol *dummy : subp->dummyArgs()) {
-      if (dummy) {
-        HarvestSymbolsNeededFromOtherModules(set, *dummy, scope);
-      }
-    }
-    if (subp->isFunction()) {
-      HarvestSymbolsNeededFromOtherModules(set, subp->result(), scope);
-    }
-  }
-}
-
-static void HarvestSymbolsNeededFromOtherModules(
-    SourceOrderedSymbolSet &set, const Scope &scope) {
-  for (const auto &[_, symbol] : scope) {
-    HarvestSymbolsNeededFromOtherModules(set, *symbol, scope);
+static SourceOrderedSymbolSet HarvestSymbolsNeededFromOtherModules(
+    const Scope &scope) {
+  SourceOrderedSymbolSet set;
+  for (const Symbol &symbol : CollectAllDependences(scope)) {
+    set.insert(symbol);
   }
+  return set;
 }
 
-void ModFileWriter::PrepareRenamings(const Scope &scope) {
-  // Identify use-associated symbols already in scope under some name
-  std::map<const Symbol *, const Symbol *> useMap;
-  for (const auto &[name, symbolRef] : scope) {
-    const Symbol *symbol{&*symbolRef};
+// Identifies use-associated symbols already in scope under some name
+using UseRenameMap = std::map<const Symbol *, const Symbol *>;
+template <typename SYMBOLS>
+UseRenameMap GetUseRenamings(const SYMBOLS &symbols) {
+  UseRenameMap useMap;
+  for (const Symbol &sym : symbols) {
+    const Symbol *symbol{&sym};
     while (const auto *hostAssoc{symbol->detailsIf<HostAssocDetails>()}) {
       symbol = &hostAssoc->symbol();
     }
@@ -303,65 +237,80 @@ void ModFileWriter::PrepareRenamings(const Scope &scope) {
       useMap.emplace(&use->symbol(), symbol);
     }
   }
+  return useMap;
+}
+
+void ModFileWriter::PrepareRenamings(const Scope &scope) {
   // Collect symbols needed from other modules
-  SourceOrderedSymbolSet symbolsNeeded;
-  HarvestSymbolsNeededFromOtherModules(symbolsNeeded, scope);
+  SourceOrderedSymbolSet symbolsNeeded{
+      HarvestSymbolsNeededFromOtherModules(scope)};
   // Establish any necessary renamings of symbols in other modules
   // to their names in this scope, creating those new names when needed.
+  UseRenameMap useMap{GetUseRenamings(symbolsNeeded)};
   auto &renamings{context_.moduleFileOutputRenamings()};
-  for (SymbolRef s : symbolsNeeded) {
-    if (s->owner().kind() != Scope::Kind::Module) {
-      // Not a USE'able name from a module's top scope;
-      // component, binding, dummy argument, &c.
-      continue;
-    }
-    const Scope *sMod{FindModuleContaining(s->owner())};
-    if (!sMod || sMod == &scope) {
-      continue;
-    }
-    if (auto iter{useMap.find(&*s)}; iter != useMap.end()) {
-      renamings.emplace(&*s, iter->second->name());
-      continue;
-    }
-    SourceName rename{s->name()};
-    if (const Symbol * found{scope.FindSymbol(s->name())}) {
-      if (found == &*s) {
-        continue; // available in scope
-      }
-      if (const auto *generic{found->detailsIf<GenericDetails>()}) {
-        if (generic->derivedType() == &*s || generic->specific() == &*s) {
-          continue;
-        }
-      } else if (found->has<UseDetails>()) {
-        if (&found->GetUltimate() == &*s) {
-          continue; // already use-associated with same name
-        }
+  for (const Symbol &sym : symbolsNeeded) {
+    if (auto iter{useMap.find(&sym)}; iter != useMap.end()) {
+      renamings.emplace(&sym, iter->second->name());
+    } else {
+      PutRenamedSymbolUse(scope, sym);
+    }
+  }
+}
+
+std::optional<SourceName> ModFileWriter::GetUseName(
+    const Scope &scope, const Symbol &sym, const Scope &symMod) {
+  if (sym.owner().kind() != Scope::Kind::Module) {
+    // Not a USE'able name from a module's top scope;
+    // component, binding, dummy argument, &c.
+    return std::nullopt;
+  }
+  if (&symMod == &scope) {
+    return std::nullopt; // already here
+  }
+  if (const Symbol *found{scope.FindSymbol(sym.name())}) {
+    if (found == &sym) {
+      return std::nullopt; // available in scope
+    }
+    if (const auto *generic{found->detailsIf<GenericDetails>()}) {
+      if (generic->derivedType() == &sym || generic->specific() == &sym) {
+        return std::nullopt;
       }
-      if (&s->owner() != &found->owner()) { // Symbol needs renaming
-        rename = scope.context().SaveTempName(
-            DEREF(sMod->symbol()).name().ToString() + "$" +
-            s->name().ToString());
+    } else if (found->has<UseDetails>()) {
+      if (&found->GetUltimate() == &sym) {
+        return std::nullopt; // already use-associated with same name
       }
     }
-    // Symbol is used in this scope but not visible under its name
-    if (sMod->parent().IsIntrinsicModules()) {
-      uses_ << "use,intrinsic::";
-    } else {
-      uses_ << "use ";
+    if (&sym.owner() != &found->owner()) { // Symbol needs renaming
+      return context_.SaveTempName(DEREF(symMod.symbol()).name().ToString() +
+          "$" + sym.name().ToString());
     }
-    uses_ << DEREF(sMod->symbol()).name() << ",only:";
-    if (rename != s->name()) {
-      uses_ << rename << "=>";
-      renamings.emplace(&s->GetUltimate(), rename);
+  }
+  return sym.name(); // needs USE, no renaming
+}
+
+void ModFileWriter::PutRenamedSymbolUse(const Scope &scope, const Symbol &sym) {
+  if (const Scope *symMod{FindModuleContaining(sym.owner())}) {
+    if (auto rename{GetUseName(scope, sym, *symMod)}) {
+      // Symbol is used in this scope but not visible under its name
+      if (symMod->parent().IsIntrinsicModules()) {
+        uses_ << "use,intrinsic::";
+      } else {
+        uses_ << "use ";
+      }
+      uses_ << DEREF(symMod->symbol()).name() << ",only:";
+      if (*rename != sym.name()) {
+        uses_ << *rename << "=>";
+        context_.moduleFileOutputRenamings().emplace(
+            &sym.GetUltimate(), *rename);
+      }
+      uses_ << sym.name() << '\n';
+      useExtraAttrs_ << "private::" << *rename << '\n';
     }
-    uses_ << s->name() << '\n';
-    useExtraAttrs_ << "private::" << rename << '\n';
   }
 }
 
 // Put out the visible symbols from scope.
-void ModFileWriter::PutSymbols(
-    const Scope &scope, UnorderedSymbolSet *hermeticModules) {
+void ModFileWriter::PutSymbols(const Scope &scope, bool omitModules) {
   SymbolVector sorted;
   SymbolVector uses;
   auto &renamings{context_.moduleFileOutputRenamings()};
@@ -369,12 +318,10 @@ void ModFileWriter::PutSymbols(
   PrepareRenamings(scope);
   SourceOrderedSymbolSet modules;
   CollectSymbols(scope, sorted, uses, modules);
-  // Write module files for dependencies first so that their
+  // Write module files for compiled dependency modules first so that their
   // hashes are known.
-  for (const Symbol &mod : modules) {
-    if (hermeticModules) {
-      hermeticModules->insert(mod);
-    } else {
+  if (!omitModules) {
+    for (const Symbol &mod : modules) {
       Write(mod);
       // It's possible that the module's file already existed and
       // without its own hash due to being embedded in a hermetic
@@ -412,6 +359,127 @@ void ModFileWriter::PutSymbols(
   renamings = std::move(previousRenamings);
 }
 
+// Combines duplicate top-level symbols, which can arise when
+// merging dependency modules from hermetic module files.
+static SymbolVector CombineDuplicateSymbols(const SymbolVector &originals) {
+  std::map<SourceName, const Symbol *> distinct;
+  SymbolVector symbols;
+  symbols.reserve(originals.size());
+  for (const Symbol &symbol : originals) {
+    if (symbol.owner().IsModule()) {
+      if (auto pair{distinct.emplace(symbol.name(), &symbol)}; !pair.second) {
+        // name already present
+        if (symbol.has<GenericDetails>()) {
+          pair.first->second = &symbol;
+        } else {
+          continue;
+        }
+      }
+    }
+    symbols.emplace_back(symbol);
+  }
+  return symbols;
+}
+
+std::string ModFileWriter::PutDependencyModules(
+    std::string originalModuleName, const SymbolVector &order) {
+  // Partition symbols by module name.
+  // Ignore symbols from intrinsic modules and the original module.
+  std::map<std::string, SymbolVector> perModuleName;
+  for (const Symbol &symbol : order) {
+    if (const Scope *module{FindModuleContaining(symbol.owner())}) {
+      if (!module->parent().IsIntrinsicModules()) {
+        if (auto name{module->GetName()}) {
+          if (*name != originalModuleName) {
+            perModuleName[name->ToString()].emplace_back(symbol);
+          }
+        }
+      }
+    }
+  }
+  std::string result;
+  for (const auto &[moduleName, symbols] : perModuleName) {
+    SymbolVector distinct{CombineDuplicateSymbols(symbols)};
+    result += PutDependencyModule(moduleName, distinct);
+  }
+  return result;
+}
+
+std::string ModFileWriter::PutDependencyModule(
+    const std::string &moduleName, const SymbolVector &needed) {
+  const Scope &scope{DEREF(FindModuleContaining(needed.front()->owner()))};
+  CHECK(scope.IsModule());
+  // The needed symbols for this module may depend on use-associated
+  // symbols from other dependency modules, so collect them.
+  SymbolVector symbols{CollectAllDependences(
+      needed, IncludeOriginalSymbols | IncludeUsesOfGenerics, &scope)};
+  SymbolVector order, namelists, generics;
+  std::set<std::string> names, commonNames, genericNames;
+  order.reserve(symbols.size());
+  for (const Symbol &symbol : symbols) {
+    std::string symbolName{symbol.name().ToString()};
+    if (symbol.test(Symbol::Flag::ParentComp) ||
+        symbol.test(Symbol::Flag::CompilerCreated) ||
+        !symbol.owner().IsModule()) {
+    } else if (symbol.has<CommonBlockDetails>()) {
+      if (commonNames.find(symbolName) == commonNames.end()) {
+        order.push_back(symbol);
+        commonNames.insert(symbolName);
+      }
+    } else if (const auto *generic{symbol.detailsIf<GenericDetails>()}) {
+      if (names.find(symbolName) == names.end()) {
+        if (generic->specific() &&
+            &generic->specific()->owner() == &symbol.owner()) {
+          order.push_back(*generic->specific());
+          names.insert(symbolName);
+        } else if (generic->derivedType() &&
+            &generic->derivedType()->owner() == &symbol.owner()) {
+          order.push_back(*generic->derivedType());
+          names.insert(symbolName);
+        }
+      }
+      if (genericNames.find(symbolName) == genericNames.end()) {
+        generics.push_back(symbol);
+        genericNames.insert(symbolName);
+      }
+    } else if (names.find(symbolName) != names.end()) {
+      if (const auto *use{symbol.detailsIf<UseDetails>()}) {
+        if (use->symbol().GetUltimate().has<GenericDetails>()) {
+          order.push_back(symbol); // pmk duplicates?
+        }
+      }
+    } else if (symbol.has<NamelistDetails>()) {
+      namelists.push_back(symbol);
+      names.insert(symbolName);
+    } else {
+      order.push_back(symbol);
+      names.insert(symbolName);
+    }
+  }
+  order.insert(order.end(), generics.begin(), generics.end());
+  order.insert(order.end(), namelists.begin(), namelists.end());
+  // Emit the symbols
+  std::string buf;
+  llvm::raw_string_ostream typeBindings{buf};
+  auto &renamings{context_.moduleFileOutputRenamings()};
+  auto previousRenamings{std::move(renamings)};
+  UseRenameMap useMap{GetUseRenamings(order)};
+  for (const Symbol &sym : order) {
+    if (auto iter{useMap.find(&sym)}; iter != useMap.end()) {
+      renamings.emplace(&sym, iter->second->name());
+    } else if (const Scope *from{FindModuleContaining(sym.owner())};
+        from && from->GetName().value().ToString() != moduleName) {
+      PutRenamedSymbolUse(scope, sym);
+    } else {
+      PutSymbol(typeBindings, sym);
+    }
+  }
+  // pmk TODO: equivalence sets
+  CHECK(typeBindings.str().empty());
+  renamings = std::move(previousRenamings);
+  return GetAsString(nullptr, moduleName);
+}
+
 // Emit components in order
 bool ModFileWriter::PutComponents(const Symbol &typeSymbol) {
   const auto &scope{DEREF(typeSymbol.scope())};
@@ -1578,14 +1646,15 @@ Scope *ModFileReader::Read(SourceName name, std::optional<bool> isIntrinsic,
   // their own nested scope that will be visible only to USE statements
   // within the module file.
   Scope *previousHermetic{context_.currentHermeticModuleFileScope()};
+  Scope *hermeticScope{nullptr};
   if (parseTree.v.size() > 1) {
     parser::Program hermeticModules{std::move(parseTree.v)};
     parseTree.v.emplace_back(std::move(hermeticModules.v.front()));
     hermeticModules.v.pop_front();
-    Scope &hermeticScope{topScope.MakeScope(Scope::Kind::Global)};
-    context_.set_currentHermeticModuleFileScope(&hermeticScope);
-    ResolveNames(context_, hermeticModules, hermeticScope);
-    for (auto &[_, ref] : hermeticScope) {
+    hermeticScope = &topScope.MakeScope(Scope::Kind::Global);
+    context_.set_currentHermeticModuleFileScope(hermeticScope);
+    ResolveNames(context_, hermeticModules, *hermeticScope);
+    for (auto &[_, ref] : *hermeticScope) {
       CHECK(ref->has<ModuleDetails>());
       ref->set(Symbol::Flag::ModFile);
     }
diff --git a/flang/lib/Semantics/mod-file.h b/flang/lib/Semantics/mod-file.h
index 9e5724089b3c5..309c23f78f44e 100644
--- a/flang/lib/Semantics/mod-file.h
+++ b/flang/lib/Semantics/mod-file.h
@@ -66,9 +66,16 @@ class ModFileWriter {
   void WriteAll(const Scope &);
   void WriteOne(const Scope &);
   void Write(const Symbol &);
-  std::string GetAsString(const Symbol &);
+  std::string GetAsString(const Symbol *, std::string);
   void PrepareRenamings(const Scope &);
-  void PutSymbols(const Scope &, UnorderedSymbolSet *hermetic);
+  std::optional<SourceName> GetUseName(
+      const Scope &, const Symbol &, const Scope &symMod);
+  void PutRenamedSymbolUse(const Scope &, const Symbol &);
+  void PutSymbols(const Scope &, bool omitModules);
+  std::string PutDependencyModules(
+      std::string originalModuleName, const SymbolVector &);
+  std::string PutDependencyModule(
+      const std::string &modName, const SymbolVector &);
   // Returns true if a derived type with bindings and "contains" was emitted
   bool PutComponents(const Symbol &);
   void PutSymbol(llvm::raw_ostream &, const Symbol &);
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index d0336c9cb661d..b3bbd682bde60 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -3504,8 +3504,7 @@ ModuleVisitor::SymbolRename ModuleVisitor::AddUse(
         useModuleScope_->GetName().value());
     return {};
   }
-  if (useSymbol->attrs().test(Attr::PRIVATE) &&
-      !FindModuleFileContaining(currScope())) {
+  if (useSymbol->attrs().test(Attr::PRIVATE) && !IsInModuleFile(currScope())) {
     // Privacy is not enforced in module files so that generic interfaces
     // can be resolved to private specific procedures in specification
     // expressions.
@@ -3617,10 +3616,11 @@ static bool CheckCompatibleDistinctUltimates(SemanticsContext &context,
     if (const auto *useObject{useUltimate.detailsIf<ObjectEntityDetails>()}) {
       auto localType{evaluate::DynamicType::From(localUltimate)};
       auto useType{evaluate::DynamicType::From(useUltimate)};
-      if (localUltimate.size() != useUltimate.size() ||
-          (localType &&
-              (!useType || !localType->IsTkLenCompatibleWith(*useType) ||
-                  !useType->IsTkLenCompatibleWith(*localType))) ||
+      if (localUltimate.size() != useUltimate.size()) {
+        isError = true;
+      } else if ((localType &&
+                     (!useType || !localType->IsTkLenCompatibleWith(*useType) ||
+                         !useType->IsTkLenCompatibleWith(*localType))) ||
           (!localType && useType)) {
         isError = true;
       } else if (IsNamedConstant(localUltimate)) {
@@ -7424,7 +7424,8 @@ void DeclarationVisitor::SetType(
 std::optional<DerivedTypeSpec> DeclarationVisitor::ResolveDerivedType(
     const parser::Name &name) {
   Scope &outer{NonDerivedTypeScope()};
-  Symbol *symbol{FindSymbol(outer, name)};
+  Symbol *original{FindSymbol(outer, name)};
+  Symbol *symbol{original};
   Symbol *ultimate{symbol ? &symbol->GetUltimate() : nullptr};
   auto *generic{ultimate ? ultimate->detailsIf<GenericDetails>() : nullptr};
   if (generic) {
@@ -7437,11 +7438,12 @@ std::optional<DerivedTypeSpec> DeclarationVisitor::ResolveDerivedType(
       (generic && &ultimate->owner() == &outer)) {
     if (allowForwardReferenceToDerivedType()) {
       if (!symbol) {
-        symbol = &MakeSymbol(outer, name.source, Attrs{});
+        symbol = original = &MakeSymbol(outer, name.source, Attrs{});
         Resolve(name, *symbol);
       } else if (generic) {
         // forward ref to type with later homonymous generic
-        symbol = &outer.MakeSymbol(name.source, Attrs{}, UnknownDetails{});
+        symbol = original =
+            &outer.MakeSymbol(name.source, Attrs{}, UnknownDetails{});
         generic->set_derivedType(*symbol);
         name.symbol = symbol;
       }
@@ -7461,7 +7463,7 @@ std::optional<DerivedTypeSpec> DeclarationVisitor::ResolveDerivedType(
   if (CheckUseError(name)) {
     return std::nullopt;
   } else if (symbol->GetUltimate().has<DerivedTypeDetails>()) {
-    return DerivedTypeSpec{name.source, *symbol};
+    return DerivedTypeSpec{name.source, *original};
   } else {
     Say(name, "'%s' is not a derived type"_err_en_US);
     return std::nullopt;
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..ad901e75820f4 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -42,6 +42,7 @@
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
 #include "flang/Support/default-kinds.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Host.h"
@@ -456,13 +457,7 @@ void SemanticsContext::UpdateScopeIndex(
 }
 
 bool SemanticsContext::IsInModuleFile(parser::CharBlock source) const {
-  for (const Scope *scope{&FindScope(source)}; !scope->IsGlobal();
-       scope = &scope->parent()) {
-    if (scope->IsModuleFile()) {
-      return true;
-    }
-  }
-  return false;
+  return semantics::IsInModuleFile(FindScope(source));
 }
 
 void SemanticsContext::PopConstruct() {
@@ -648,7 +643,8 @@ bool Semantics::Perform() {
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}
-          .set_hermeticModuleFileOutput(hermeticModuleFileOutput_)
+          .set_hermeticModuleFileOutput(
+              hermeticModuleFileOutput_ || getenv("PMK_HERMETIC"))
           .WriteAll();
 }
 
diff --git a/flang/lib/Semantics/symbol-dependence.cpp b/flang/lib/Semantics/symbol-dependence.cpp
new file mode 100644
index 0000000000000..2591f609f3d00
--- /dev/null
+++ b/flang/lib/Semantics/symbol-dependence.cpp
@@ -0,0 +1,356 @@
+//===-- lib/Semantics/symbol-dependence.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Semantics/symbol-dependence.h"
+#include "flang/Common/idioms.h"
+#include "flang/Common/restorer.h"
+#include "flang/Common/visit.h"
+#include <queue>
+
+static constexpr bool EnableDebugging{false};
+
+namespace Fortran::semantics {
+
+// Helper class that collects all of the symbol dependences for a
+// given symbol.
+class Collector {
+public:
+  explicit Collector(int flags) : flags_{flags} {}
+
+  void CollectSymbolDependences(const Symbol &);
+  UnorderedSymbolSet MustFollowDependences() { return std::move(dependences_); }
+  SymbolVector AllDependences() { return std::move(mentions_); }
+
+private:
+  // This symbol is depended upon and its declaration must precede
+  // the symbol of interest.
+  void MustFollow(const Symbol &x) {
+    if (!possibleImports_ || !DoesScopeContain(possibleImports_, x)) {
+      dependences_.insert(x);
+    }
+  }
+  // This symbol is depended upon, but is not necessarily a dependence
+  // that must precede the symbol of interest in the output of the
+  // topological sort.
+  void Need(const Symbol &x) {
+    if (mentioned_.insert(x).second) {
+      mentions_.emplace_back(x);
+    }
+  }
+  void Need(const Symbol *x) {
+    if (x) {
+      Need(*x);
+    }
+  }
+
+  // These overloads of Collect() are mutally recursive, so they're
+  // packaged as member functions of a class.
+  void Collect(const Symbol &x) {
+    Need(x);
+    const auto *subp{x.detailsIf<SubprogramDetails>()};
+    if ((subp && subp->isInterface()) || IsDummy(x) ||
+        x.has<CommonBlockDetails>() || x.has<NamelistDetails>()) {
+      // can be forward-referenced
+    } else {
+      MustFollow(x);
+    }
+  }
+  void Collect(SymbolRef x) { Collect(*x); }
+  template <typename A> void Collect(const std::optional<A> &x) {
+    if (x) {
+      Collect(*x);
+    }
+  }
+  template <typename A> void Collect(const A *x) {
+    if (x) {
+      Collect(*x);
+    }
+  }
+  void Collect(const UnorderedSymbolSet &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const SourceOrderedSymbolSet &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const SymbolVector &x) {
+    for (const Symbol &symbol : x) {
+      Collect(symbol);
+    }
+  }
+  void Collect(const Scope &x) { Collect(x.GetSymbols()); }
+  template <typename T> void Collect(const evaluate::Expr<T> &x) {
+    UnorderedSymbolSet exprSyms{evaluate::CollectSymbols(x)};
+    for (const Symbol &sym : exprSyms) {
+      if (!sym.owner().IsDerivedType()) {
+        Collect(sym);
+      }
+    }
+  }
+  void Collect(const DeclTypeSpec &type) {
+    if (type.category() == DeclTypeSpec::Category::Character) {
+      Collect(type.characterTypeSpec().length());
+    } else {
+      Collect(type.AsDerived());
+    }
+  }
+  void Collect(const DerivedTypeSpec &type) {
+    const Symbol &typeSym{type.originalTypeSymbol()};
+    if (!derivedTypeReferenceCanBeForward_ || !type.parameters().empty()) {
+      MustFollow(typeSym);
+    }
+    Need(typeSym);
+    for (const auto &[_, value] : type.parameters()) {
+      Collect(value);
+    }
+  }
+  void Collect(const ParamValue &x) { Collect(x.GetExplicit()); }
+  void Collect(const Bound &x) { Collect(x.GetExplicit()); }
+  void Collect(const ShapeSpec &x) {
+    Collect(x.lbound());
+    Collect(x.ubound());
+  }
+  void Collect(const ArraySpec &x) {
+    for (const ShapeSpec &shapeSpec : x) {
+      Collect(shapeSpec);
+    }
+  }
+
+  UnorderedSymbolSet mentioned_, dependences_;
+  SymbolVector mentions_;
+  int flags_{NoDependenceCollectionFlags};
+  bool derivedTypeReferenceCanBeForward_{false};
+  const Scope *possibleImports_{nullptr};
+};
+
+void Collector::CollectSymbolDependences(const Symbol &symbol) {
+  if (symbol.has<ProcBindingDetails>() || symbol.has<SubprogramDetails>()) {
+    // type will be picked up later for the function result, if any
+  } else if (symbol.has<UseDetails>() || symbol.has<UseErrorDetails>() ||
+      symbol.has<HostAssocDetails>()) {
+  } else if (IsAllocatableOrPointer(symbol) && symbol.owner().IsDerivedType()) {
+    bool saveCanBeForward{derivedTypeReferenceCanBeForward_};
+    derivedTypeReferenceCanBeForward_ = true;
+    Collect(symbol.GetType());
+    derivedTypeReferenceCanBeForward_ = saveCanBeForward;
+  } else {
+    Collect(symbol.GetType());
+  }
+  common::visit(
+      common::visitors{
+          [this, &symbol](const ObjectEntityDetails &x) {
+            Collect(x.shape());
+            Collect(x.coshape());
+            if (IsNamedConstant(symbol) || symbol.owner().IsDerivedType()) {
+              Collect(x.init());
+            }
+            Need(x.commonBlock());
+            if (const auto *set{FindEquivalenceSet(symbol)}) {
+              for (const EquivalenceObject &equivObject : *set) {
+                Need(equivObject.symbol);
+              }
+            }
+          },
+          [this, &symbol](const ProcEntityDetails &x) {
+            Collect(x.rawProcInterface());
+            if (symbol.owner().IsDerivedType()) {
+              Collect(x.init());
+            }
+          },
+          [this](const ProcBindingDetails &x) { Need(x.symbol()); },
+          [this, &symbol](const SubprogramDetails &x) {
+            // Note dummy arguments & result symbol without dependence, unless
+            // the subprogram is an interface block that might need to IMPORT
+            // a type.
+            bool needImports{x.isInterface()};
+            auto restorer{common::ScopedSet(
+                possibleImports_, needImports ? symbol.scope() : nullptr)};
+            for (const Symbol *dummy : x.dummyArgs()) {
+              if (dummy) {
+                Need(*dummy);
+                if (needImports) {
+                  CollectSymbolDependences(*dummy);
+                }
+              }
+            }
+            if (x.isFunction()) {
+              Need(x.result());
+              if (needImports) {
+                CollectSymbolDependences(x.result());
+              }
+            }
+          },
+          [this, &symbol](const DerivedTypeDetails &x) {
+            Collect(symbol.scope());
+            for (const auto &[_, symbolRef] : x.finals()) {
+              Need(*symbolRef);
+            }
+          },
+          [this](const GenericDetails &x) {
+            Collect(x.derivedType());
+            Collect(x.specific());
+            if (flags_ & IncludeUsesOfGenerics) {
+              for (const Symbol &use : x.uses()) {
+                Collect(use);
+              }
+            }
+            if (flags_ & IncludeSpecificsOfGenerics) {
+              for (const Symbol &specific : x.specificProcs()) {
+                Collect(specific);
+              }
+            }
+          },
+          [this](const NamelistDetails &x) {
+            for (const Symbol &symbol : x.objects()) {
+              Collect(symbol);
+            }
+          },
+          [this](const CommonBlockDetails &x) {
+            for (auto ref : x.objects()) {
+              Collect(*ref);
+            }
+          },
+          [this](const UseDetails &x) {
+            if (flags_ & FollowUseAssociations) {
+              Need(x.symbol());
+            }
+          },
+          [this](const HostAssocDetails &x) { Need(x.symbol()); },
+          [](const auto &) {},
+      },
+      symbol.details());
+}
+
+SymbolVector CollectAllDependences(const Scope &scope, int flags) {
+  SymbolVector basis{scope.GetSymbols()};
+  return CollectAllDependences(basis, flags, &scope);
+}
+
+// Returns a vector of symbols, topologically sorted by dependence
+SymbolVector CollectAllDependences(
+    const SymbolVector &original, int flags, const Scope *forScope) {
+  std::queue<const Symbol *> work;
+  UnorderedSymbolSet enqueued;
+  for (const Symbol &symbol : original) {
+    if (!symbol.test(Symbol::Flag::CompilerCreated)) {
+      work.push(&symbol);
+      enqueued.insert(symbol);
+    }
+  }
+  // For each symbol, collect its dependences into "topology".
+  // The "visited" vector and "enqueued" set hold all of the
+  // symbols considered.
+  std::map<const Symbol *, UnorderedSymbolSet> topology;
+  std::vector<const Symbol *> visited;
+  visited.reserve(2 * original.size());
+  std::optional<SourceName> forModuleName;
+  if (forScope && !(flags & NotJustForOneModule)) {
+    if (const Scope *forModule{FindModuleContaining(*forScope)}) {
+      forModuleName = forModule->GetName();
+    }
+  }
+  while (!work.empty()) {
+    const Symbol &symbol{*work.front()};
+    work.pop();
+    visited.push_back(&symbol);
+    Collector collector{flags};
+    bool doCollection{true};
+    if (forModuleName) {
+      if (const Scope *symModule{FindModuleContaining(symbol.owner())}) {
+        if (auto symModName{symModule->GetName()}) {
+          doCollection = *forModuleName == *symModName;
+        }
+      }
+    }
+    if (doCollection) {
+      collector.CollectSymbolDependences(symbol);
+    }
+    auto dependences{collector.MustFollowDependences()};
+    auto mentions{collector.AllDependences()};
+    if constexpr (EnableDebugging) {
+      for (const Symbol &need : dependences) {
+        llvm::errs() << "symbol " << symbol << " must follow " << need << '\n';
+      }
+      for (const Symbol &need : mentions) {
+        llvm::errs() << "symbol " << symbol << " needs " << need << '\n';
+      }
+    }
+    CHECK(topology.find(&symbol) == topology.end());
+    topology.emplace(&symbol, std::move(dependences));
+    for (const Symbol &symbol : mentions) {
+      if (!symbol.test(Symbol::Flag::CompilerCreated)) {
+        if (enqueued.insert(symbol).second) {
+          work.push(&symbol);
+        }
+      }
+    }
+  }
+  CHECK(enqueued.size() == visited.size());
+  // Topological sorting
+  // Subtle: This inverted topology map uses a SymbolVector, not a set
+  // of symbols, so that the order of symbols in the final output remains
+  // deterministic.
+  std::map<const Symbol *, SymbolVector> invertedTopology;
+  for (const Symbol *symbol : visited) {
+    invertedTopology[symbol] = SymbolVector{};
+  }
+  std::map<const Symbol *, std::size_t> numWaitingFor;
+  for (const Symbol *symbol : visited) {
+    auto topoIter{topology.find(symbol)};
+    CHECK(topoIter != topology.end());
+    const auto &needs{topoIter->second};
+    if (needs.empty()) {
+      work.push(symbol);
+    } else {
+      numWaitingFor[symbol] = needs.size();
+      for (const Symbol &need : needs) {
+        invertedTopology[&need].push_back(*symbol);
+      }
+    }
+  }
+  CHECK(visited.size() == work.size() + numWaitingFor.size());
+  SymbolVector resultVector;
+  while (!work.empty()) {
+    const Symbol &symbol{*work.front()};
+    work.pop();
+    resultVector.push_back(symbol);
+    auto enqueuedIter{enqueued.find(symbol)};
+    CHECK(enqueuedIter != enqueued.end());
+    enqueued.erase(enqueuedIter);
+    if (auto invertedIter{invertedTopology.find(&symbol)};
+        invertedIter != invertedTopology.end()) {
+      for (const Symbol &neededBy : invertedIter->second) {
+        std::size_t stillAwaiting{numWaitingFor[&neededBy] - 1};
+        if (stillAwaiting == 0) {
+          work.push(&neededBy);
+        } else {
+          numWaitingFor[&neededBy] = stillAwaiting;
+        }
+      }
+    }
+  }
+  if constexpr (EnableDebugging) {
+    llvm::errs() << "Topological sort failed in CollectAllDependences\n";
+    for (const Symbol &remnant : enqueued) {
+      auto topoIter{topology.find(&remnant)};
+      CHECK(topoIter != topology.end());
+      llvm::errs() << "  remnant symbol " << remnant << " needs:\n";
+      for (const Symbol &n : topoIter->second) {
+        llvm::errs() << "   " << n << '\n';
+      }
+    }
+  }
+  CHECK(enqueued.empty());
+  CHECK(resultVector.size() == visited.size());
+  return resultVector;
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index 0380207927ad3..23bb16ad30106 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -277,13 +277,27 @@ void GenericDetails::CopyFrom(const GenericDetails &from) {
     CHECK(!derivedType_ || derivedType_ == from.derivedType_);
     derivedType_ = from.derivedType_;
   }
-  for (std::size_t i{0}; i < from.specificProcs_.size(); ++i) {
-    if (llvm::none_of(specificProcs_, [&](const Symbol &mySymbol) {
-          return &mySymbol.GetUltimate() ==
-              &from.specificProcs_[i]->GetUltimate();
-        })) {
-      specificProcs_.push_back(from.specificProcs_[i]);
-      bindingNames_.push_back(from.bindingNames_[i]);
+  for (std::size_t j{0}; j < from.specificProcs_.size(); ++j) {
+    auto fromSpecific{from.specificProcs_[j]};
+    SourceName fromBinding{from.bindingNames_[j]};
+    const Symbol &fromUltimate{fromSpecific->GetUltimate()};
+    const Scope *fromModule{FindModuleContaining(fromUltimate.owner())};
+    auto fromModuleName{fromModule ? fromModule->GetName() : std::nullopt};
+    bool addit{true};
+    for (std::size_t k{0}; addit && k < specificProcs_.size(); ++k) {
+      const Symbol &ultimate{specificProcs_[k]->GetUltimate()};
+      if (&fromUltimate == &ultimate) {
+        addit = false;
+      } else if (fromBinding == bindingNames_[k]) {
+        const Scope *module{FindModuleContaining(ultimate.owner())};
+        auto moduleName{module ? module->GetName() : std::nullopt};
+        addit =
+            !(fromModuleName && moduleName && *fromModuleName == *moduleName);
+      }
+    }
+    if (addit) {
+      specificProcs_.push_back(fromSpecific);
+      bindingNames_.push_back(fromBinding);
     }
   }
 }
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index d27d250b3f11e..17a76becc1306 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -6,15 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Parser/tools.h"
+#include "flang/Semantics/tools.h"
 #include "flang/Common/indirection.h"
 #include "flang/Parser/dump-parse-tree.h"
 #include "flang/Parser/message.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
-#include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
 #include "flang/Support/Fortran.h"
 #include "llvm/Support/raw_ostream.h"
@@ -58,9 +58,16 @@ const Scope *FindModuleOrSubmoduleContaining(const Scope &start) {
   });
 }
 
-const Scope *FindModuleFileContaining(const Scope &start) {
-  return FindScopeContaining(
-      start, [](const Scope &scope) { return scope.IsModuleFile(); });
+bool IsInModuleFile(const Scope &start) {
+  for (const Scope *scope{&start};; scope = &scope->parent()) {
+    if (scope->IsModuleFile() ||
+        scope == scope->context().currentHermeticModuleFileScope()) {
+      return true;
+    } else if (scope->IsTopLevel()) {
+      break;
+    }
+  }
+  return false;
 }
 
 const Scope &GetProgramUnitContaining(const Scope &start) {
@@ -1158,7 +1165,7 @@ std::optional<parser::MessageFormattedText> CheckAccessibleSymbol(
     const Scope &scope, const Symbol &symbol) {
   if (IsAccessible(symbol, scope)) {
     return std::nullopt;
-  } else if (FindModuleFileContaining(scope)) {
+  } else if (IsInModuleFile(scope)) {
     // Don't enforce component accessibility checks in module files;
     // there may be forward-substituted named constants of derived type
     // whose structure constructors reference private components.
@@ -1845,4 +1852,18 @@ bool HadUseError(
   }
 }
 
+bool CheckForSymbolMatch(const SomeExpr *lhs, const SomeExpr *rhs) {
+  if (lhs && rhs) {
+    if (SymbolVector lhsSymbols{evaluate::GetSymbolVector(*lhs)};
+        !lhsSymbols.empty()) {
+      const Symbol &first{*lhsSymbols.front()};
+      for (const Symbol &symbol : evaluate::GetSymbolVector(*rhs)) {
+        if (first == symbol) {
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/type.cpp b/flang/lib/Semantics/type.cpp
index 964a37e1c822b..b8f35c9bf4507 100644
--- a/flang/lib/Semantics/type.cpp
+++ b/flang/lib/Semantics/type.cpp
@@ -22,9 +22,19 @@
 
 namespace Fortran::semantics {
 
+static const Symbol &ResolveOriginalTypeSymbol(const Symbol *symbol) {
+  symbol = &symbol->GetUltimate();
+  if (const auto *generic{symbol->detailsIf<GenericDetails>()}) {
+    CHECK(generic->derivedType() != nullptr);
+    return generic->derivedType()->GetUltimate();
+  } else {
+    return *symbol;
+  }
+}
+
 DerivedTypeSpec::DerivedTypeSpec(SourceName name, const Symbol &typeSymbol)
     : name_{name}, originalTypeSymbol_{typeSymbol},
-      typeSymbol_{typeSymbol.GetUltimate()} {
+      typeSymbol_{ResolveOriginalTypeSymbol(&typeSymbol)} {
   CHECK(typeSymbol_.has<DerivedTypeDetails>());
 }
 DerivedTypeSpec::DerivedTypeSpec(const DerivedTypeSpec &that) = default;
@@ -252,16 +262,12 @@ static bool MatchKindParams(const Symbol &typeSymbol,
 }
 
 bool DerivedTypeSpec::MatchesOrExtends(const DerivedTypeSpec &that) const {
-  const Symbol *typeSymbol{&typeSymbol_};
-  while (typeSymbol != &that.typeSymbol_) {
-    if (const DerivedTypeSpec *
-        parent{typeSymbol->GetParentTypeSpec(typeSymbol->scope())}) {
-      typeSymbol = &parent->typeSymbol_;
-    } else {
-      return false;
-    }
+  const DerivedTypeSpec *type{this};
+  while (type &&
+      !evaluate::AreSameDerivedTypeIgnoringTypeParameters(*type, that)) {
+    type = type->typeSymbol_.GetParentTypeSpec(type->typeSymbol_.scope());
   }
-  return MatchKindParams(*typeSymbol, *this, that);
+  return type && MatchKindParams(type->typeSymbol_, *this, that);
 }
 
 class InstantiateHelper {
diff --git a/flang/test/Semantics/modfile03.f90 b/flang/test/Semantics/modfile03.f90
index eb3136f0aa8bc..399547ca41958 100644
--- a/flang/test/Semantics/modfile03.f90
+++ b/flang/test/Semantics/modfile03.f90
@@ -69,7 +69,7 @@ pure integer function f1(i)
 module m5b
   use m5a, only: k2 => k1, l2 => l1, f2 => f1
   interface
-    subroutine s(x, y)
+    subroutine s5b(x, y)
       import f2, l2
       character(l2, k2) :: x
       character(f2(l2)) :: y
@@ -82,7 +82,7 @@ subroutine s(x, y)
 ! use m5a,only:l2=>l1
 ! use m5a,only:f2=>f1
 ! interface
-!  subroutine s(x,y)
+!  subroutine s5b(x,y)
 !   import::f2
 !   import::l2
 !   character(l2,4)::x
@@ -142,7 +142,7 @@ module m6d
 module m6e
   use m6a, only: t2 => t1
   interface
-    subroutine s(x)
+    subroutine s6e(x)
       import t2
       type(t2) :: x
     end subroutine
@@ -152,7 +152,7 @@ subroutine s(x)
 !module m6e
 ! use m6a,only:t2=>t1
 ! interface
-!  subroutine s(x)
+!  subroutine s6e(x)
 !   import::t2
 !   type(t2)::x
 !  end
diff --git a/flang/test/Semantics/modfile65.f90 b/flang/test/Semantics/modfile65.f90
index 249255e02129f..3eabf47d4d55e 100644
--- a/flang/test/Semantics/modfile65.f90
+++ b/flang/test/Semantics/modfile65.f90
@@ -42,12 +42,12 @@ module m4
 !use m2,only:n
 !use m3,only:m
 !end
+!module m1
+!integer(4),parameter::n=123_4
+!end
 !module m2
 !use m1,only:n
 !end
 !module m3
 !use m1,only:m=>n
 !end
-!module m1
-!integer(4),parameter::n=123_4
-!end
diff --git a/flang/test/Semantics/modfile78.F90 b/flang/test/Semantics/modfile78.F90
index 19b9ac39de934..f612c2fa30d64 100644
--- a/flang/test/Semantics/modfile78.F90
+++ b/flang/test/Semantics/modfile78.F90
@@ -27,7 +27,6 @@ module modfile78c
 !CHECK: integer(4)::global_variable
 !CHECK: end
 !CHECK: module modfile78b
-!CHECK: use modfile78a,only:global_variable
 !CHECK: contains
 !CHECK: subroutine test()
 !CHECK: end

>From 28d4df0ce012af443c05a7e9b754123dc99db58f Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Mon, 7 Jul 2025 08:29:55 -0700
Subject: [PATCH 2/2] more

---
 flang/lib/Semantics/mod-file.cpp          | 49 +++++++++++++++++++----
 flang/lib/Semantics/mod-file.h            |  3 +-
 flang/lib/Semantics/symbol-dependence.cpp |  7 ++++
 3 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index 1a27449e5a786..f9b0c61fe0696 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -28,6 +28,8 @@
 #include <variant>
 #include <vector>
 
+static constexpr bool pruneHermeticModuleFiles{true};
+
 namespace Fortran::semantics {
 
 using namespace parser::literals;
@@ -144,14 +146,40 @@ void ModFileWriter::Write(const Symbol &symbol) {
       ModFileName(symbol.name(), ancestorName, context_.moduleFileSuffix())};
 
   SymbolVector dependenceClosure;
-  if (hermeticModuleFileOutput_ && !isSubmodule_) {
+  if (hermeticModuleFileOutput_ && !isSubmodule_ && pruneHermeticModuleFiles) {
     dependenceClosure = CollectAllDependences(DEREF(symbol.scope()),
         FollowUseAssociations | IncludeUsesOfGenerics |
             IncludeSpecificsOfGenerics | NotJustForOneModule);
   }
-  PutSymbols(DEREF(symbol.scope()), hermeticModuleFileOutput_);
+  UnorderedSymbolSet fullHermeticModules;
+  PutSymbols(DEREF(symbol.scope()), hermeticModuleFileOutput_,
+      hermeticModuleFileOutput_ && !pruneHermeticModuleFiles
+          ? &fullHermeticModules
+          : nullptr);
   auto asStr{GetAsString(&symbol, symbol.name().ToString())};
-  asStr += PutDependencyModules(symbol.name().ToString(), dependenceClosure);
+  if (!dependenceClosure.empty()) {
+    // Emit minimal modules on which this module depends, if emitting a
+    // hermetic module file
+    asStr += PutDependencyModules(symbol.name().ToString(), dependenceClosure);
+  } else if (!fullHermeticModules.empty()) {
+    // Emit full (complete) modules on which this module depends
+    std::set<std::string> hermeticModuleNames;
+    hermeticModuleNames.insert(symbol.name().ToString());
+    while (!fullHermeticModules.empty()) {
+      UnorderedSymbolSet nextPass{std::move(fullHermeticModules)};
+      fullHermeticModules.clear();
+      for (const Symbol &modSym : nextPass) {
+        if (!modSym.owner().IsIntrinsicModules() &&
+            hermeticModuleNames.find(modSym.name().ToString()) ==
+                hermeticModuleNames.end()) {
+          hermeticModuleNames.insert(modSym.name().ToString());
+          PutSymbols(DEREF(modSym.scope()), /*omitModules=*/false,
+              &fullHermeticModules);
+          asStr += GetAsString(&modSym, modSym.name().ToString());
+        }
+      }
+    }
+  }
   ModuleCheckSumType checkSum;
   if (std::error_code error{
           WriteFile(path, asStr, checkSum, context_.debugModuleWriter())}) {
@@ -167,7 +195,8 @@ void ModFileWriter::WriteClosure(llvm::raw_ostream &out, const Symbol &symbol,
       !nonIntrinsicModulesWritten.insert(symbol).second) {
     return;
   }
-  PutSymbols(DEREF(symbol.scope()), /*omitModules=*/false);
+  PutSymbols(DEREF(symbol.scope()), /*omitModules=*/false,
+      /*fullHermeticModules=*/nullptr);
   needsBuf_.clear(); // omit module checksums
   auto str{GetAsString(&symbol, symbol.name().ToString())};
   for (auto depRef : std::move(usedNonIntrinsicModules_)) {
@@ -310,7 +339,8 @@ void ModFileWriter::PutRenamedSymbolUse(const Scope &scope, const Symbol &sym) {
 }
 
 // Put out the visible symbols from scope.
-void ModFileWriter::PutSymbols(const Scope &scope, bool omitModules) {
+void ModFileWriter::PutSymbols(const Scope &scope, bool omitModules,
+    UnorderedSymbolSet *fullHermeticModules) {
   SymbolVector sorted;
   SymbolVector uses;
   auto &renamings{context_.moduleFileOutputRenamings()};
@@ -320,7 +350,11 @@ void ModFileWriter::PutSymbols(const Scope &scope, bool omitModules) {
   CollectSymbols(scope, sorted, uses, modules);
   // Write module files for compiled dependency modules first so that their
   // hashes are known.
-  if (!omitModules) {
+  if (fullHermeticModules) {
+    for (const Symbol &mod : modules) {
+      fullHermeticModules->insert(mod);
+    }
+  } else if (!omitModules) {
     for (const Symbol &mod : modules) {
       Write(mod);
       // It's possible that the module's file already existed and
@@ -445,7 +479,7 @@ std::string ModFileWriter::PutDependencyModule(
     } else if (names.find(symbolName) != names.end()) {
       if (const auto *use{symbol.detailsIf<UseDetails>()}) {
         if (use->symbol().GetUltimate().has<GenericDetails>()) {
-          order.push_back(symbol); // pmk duplicates?
+          order.push_back(symbol);
         }
       }
     } else if (symbol.has<NamelistDetails>()) {
@@ -474,7 +508,6 @@ std::string ModFileWriter::PutDependencyModule(
       PutSymbol(typeBindings, sym);
     }
   }
-  // pmk TODO: equivalence sets
   CHECK(typeBindings.str().empty());
   renamings = std::move(previousRenamings);
   return GetAsString(nullptr, moduleName);
diff --git a/flang/lib/Semantics/mod-file.h b/flang/lib/Semantics/mod-file.h
index 309c23f78f44e..6b25542c138ca 100644
--- a/flang/lib/Semantics/mod-file.h
+++ b/flang/lib/Semantics/mod-file.h
@@ -71,7 +71,8 @@ class ModFileWriter {
   std::optional<SourceName> GetUseName(
       const Scope &, const Symbol &, const Scope &symMod);
   void PutRenamedSymbolUse(const Scope &, const Symbol &);
-  void PutSymbols(const Scope &, bool omitModules);
+  void PutSymbols(
+      const Scope &, bool omitModules, UnorderedSymbolSet *fullHermeticModules);
   std::string PutDependencyModules(
       std::string originalModuleName, const SymbolVector &);
   std::string PutDependencyModule(
diff --git a/flang/lib/Semantics/symbol-dependence.cpp b/flang/lib/Semantics/symbol-dependence.cpp
index 2591f609f3d00..83ed2991203ea 100644
--- a/flang/lib/Semantics/symbol-dependence.cpp
+++ b/flang/lib/Semantics/symbol-dependence.cpp
@@ -158,6 +158,13 @@ void Collector::CollectSymbolDependences(const Symbol &symbol) {
                 Need(equivObject.symbol);
               }
             }
+            if (symbol.owner().IsModule()) {
+              if (const EquivalenceSet *equiv{FindEquivalenceSet(symbol)}) {
+                for (const EquivalenceObject &eqObj : *equiv) {
+                  Need(eqObj.symbol);
+                }
+              }
+            }
           },
           [this, &symbol](const ProcEntityDetails &x) {
             Collect(x.rawProcInterface());


From flang-commits at lists.llvm.org  Mon Jul 14 08:43:26 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 08:43:26 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6875259e.170a0220.23ff9d.826b@mx.google.com>


https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/145464

>From 4915fb8f1d27847dc3d36899e233d5ac988f96c5 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 23 Jun 2025 16:39:55 +0530
Subject: [PATCH 1/7] [flang] Introduce omp_target_allocmem and
 omp_target_freemem fir ops.

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  58 ++++++++++
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +++++++++++++++++-
 2 files changed, 159 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 8ac847dd7dd0a..2dff0f05fade7 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,6 +517,64 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
+def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Creates a heap memory reference suitable for storing a value of the
+    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
+    The memory object is in an undefined state.  `omp_target_allocmem` operations must
+    be paired with `omp_target_freemem` operations to avoid memory leaks.
+
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyIntegerType>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyIntegerType>:$typeparams,
+    Variadic<AnyIntegerType>:$shape
+  );
+  let results = (outs fir_HeapType);
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+    bool hasLenParams() { return !getTypeparams().empty(); }
+    bool hasShapeOperands() { return !getShape().empty(); }
+    unsigned numLenParams() { return getTypeparams().size(); }
+    operand_range getLenParams() { return getTypeparams(); }
+    unsigned numShapeOperands() { return getShape().size(); }
+    operand_range getShapeOperands() { return getShape(); }
+    static mlir::Type getRefTy(mlir::Type ty);
+  }];
+}
+
+def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free a heap object";
+
+  let description = [{
+    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
+    The memory object that is deallocated is placed in an undefined state
+    after `fir.omp_target_freemem`.
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      ...
+      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyIntegerType, "", [MemFree]>:$device,
+  Arg<fir_HeapType, "", [MemFree]>:$heapref
+  );
+}
+
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..042ade6b1e0a1 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,6 +1168,105 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetAllocMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = heap.getType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
+    mlir::Location loc = heap.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(dataTy);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
+    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(loc, rewriter, mallocTy, size);
+    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+        heap, ::getLlvmPtrType(heap.getContext()),
+        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
+    return mlir::success();
+  }
+
+  /// Compute the allocation size in bytes of the element type of
+  /// \p llTy pointer type. The result is returned as a value of \p idxTy
+  /// integer type.
+  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy) const {
+    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+  }
+};
+} // namespace
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp freeFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
+    return freeFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_free",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMVoidType::get(module->getContext()),
+          {getLlvmPtrType(module->getContext()), i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetFreeMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
+    mlir::Location loc = freemem.getLoc();
+    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
+    rewriter.create<mlir::LLVM::CallOp>(
+        loc, mlir::TypeRange{},
+        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
+        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
+    rewriter.eraseOp(freemem);
+    return mlir::success();
+  }
+};
+} // namespace
+
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4274,7 +4373,8 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
+      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,

>From e41a2c76786538fc411e104542a0282cfebef4f7 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:20:36 +0530
Subject: [PATCH 2/7] [flang] Fix parsing and printing.

---
 .../include/flang/Optimizer/Dialect/FIROps.td | 13 ++-
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 90 ++++++++++++++++---
 flang/test/Fir/omp_target_allocmem.fir        | 28 ++++++
 flang/test/Fir/omp_target_freemem.fir         | 28 ++++++
 4 files changed, 145 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Fir/omp_target_allocmem.fir
 create mode 100644 flang/test/Fir/omp_target_freemem.fir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 2dff0f05fade7..666b66a8670d6 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -528,7 +528,8 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
     be paired with `omp_target_freemem` operations to avoid memory leaks.
 
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
     ```
   }];
 
@@ -542,6 +543,9 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
   );
   let results = (outs fir_HeapType);
 
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
   let extraClassDeclaration = [{
     mlir::Type getAllocatedType();
     bool hasLenParams() { return !getTypeparams().empty(); }
@@ -563,9 +567,9 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     The memory object that is deallocated is placed in an undefined state
     after `fir.omp_target_freemem`.
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
-      ...
-      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
     ```
   }];
 
@@ -573,6 +577,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   Arg<AnyIntegerType, "", [MemFree]>:$device,
   Arg<fir_HeapType, "", [MemFree]>:$heapref
   );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index ecfa2939e96a6..9335a4b041ac8 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,24 +106,38 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca and Allocmem
-///
+/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
+/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
-                                            mlir::OpAsmParser &parser,
-                                            mlir::OperationState &result) {
+static mlir::ParseResult
+parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
+                   mlir::OperationState &result, bool isTargetOp = false) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+  // Parse device number as a new operand
+  if (isTargetOp) {
+    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+    mlir::Type deviceType;
+    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+      return mlir::failure();
+    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+      return mlir::failure();
+    if (parser.parseComma())
+      return mlir::failure();
+  }
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
-  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -147,13 +161,19 @@ static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
+
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
-                                                 {typeparamsSize, shapeSize}));
+  llvm::SmallVector<std::int32_t> segmentSizes;
+  if (isTargetOp)
+    segmentSizes.push_back(1);
+  segmentSizes.push_back(typeparamsSize);
+  segmentSizes.push_back(shapeSize);
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -385,6 +405,56 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// OmpTargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
+  return mlir::cast<fir::HeapType>(getType()).getEleTy();
+}
+
+mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
+  return fir::HeapType::get(ty);
+}
+
+mlir::ParseResult
+fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                mlir::OperationState &result) {
+  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
+}
+
+void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
+  llvm::SmallVector<llvm::StringRef> visited;
+  if (verifyInType(getInType(), visited, numShapeOperands()))
+    return emitOpError("invalid type for allocation");
+  if (verifyTypeParamCount(getInType(), numLenParams()))
+    return emitOpError("LEN params do not correspond to type");
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<fir::HeapType>(outType))
+    return emitOpError("must be a !fir.heap type");
+  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
+    return emitOpError("cannot allocate !fir.box of unknown rank or type");
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
new file mode 100644
index 0000000000000..5140c91c9510c
--- /dev/null
+++ b/flang/test/Fir/omp_target_allocmem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  return %1 : !fir.heap<!fir.array<3x3xi32>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_freemem.fir
new file mode 100644
index 0000000000000..02e136076a9cf
--- /dev/null
+++ b/flang/test/Fir/omp_target_freemem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  return
+}

>From a864094e5bd482499f82ef913c92c1f5b6bd070f Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:32:21 +0530
Subject: [PATCH 3/7] [flang] Fix doc in td

---
 flang/include/flang/Optimizer/Dialect/FIROps.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 666b66a8670d6..93d617027e30b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -560,7 +560,7 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
 
 def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object";
+  let summary = "free a heap object on an openmp device";
 
   let description = [{
     Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
@@ -569,7 +569,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     ```
       %device = arith.constant 0 : i32
       %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
     ```
   }];
 

>From 2f5b289464787d89ea00ab295ccb2ebee6b57329 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 12:22:55 +0530
Subject: [PATCH 4/7] [omp][mlir] Introduce TargetAllocMem and TargetFreeMem
 ops in openMP mlir dialect

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  63 -------
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +----------
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 161 ++++++++++++++++++
 flang/lib/Optimizer/Dialect/FIROps.cpp        |  88 +---------
 flang/test/Fir/omp_target_allocmem.fir        |  28 ---
 ...em.fir => omp_target_allocmem_freemem.fir} |  19 ++-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  64 +++++++
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 102 +++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  82 +++++++++
 .../ompenmp-target-allocmem-freemem.mlir      |  42 +++++
 10 files changed, 473 insertions(+), 278 deletions(-)
 delete mode 100644 flang/test/Fir/omp_target_allocmem.fir
 rename flang/test/Fir/{omp_target_freemem.fir => omp_target_allocmem_freemem.fir} (51%)
 create mode 100644 mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 93d617027e30b..8ac847dd7dd0a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,69 +517,6 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
-def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
-    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
-  let summary = "allocate storage on an openmp device for an object of a given type";
-
-  let description = [{
-    Creates a heap memory reference suitable for storing a value of the
-    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
-    The memory object is in an undefined state.  `omp_target_allocmem` operations must
-    be paired with `omp_target_freemem` operations to avoid memory leaks.
-
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-    ```
-  }];
-
-  let arguments = (ins
-    Arg<AnyIntegerType>:$device,
-    TypeAttr:$in_type,
-    OptionalAttr<StrAttr>:$uniq_name,
-    OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyIntegerType>:$typeparams,
-    Variadic<AnyIntegerType>:$shape
-  );
-  let results = (outs fir_HeapType);
-
-  let hasCustomAssemblyFormat = 1;
-  let hasVerifier = 1;
-
-  let extraClassDeclaration = [{
-    mlir::Type getAllocatedType();
-    bool hasLenParams() { return !getTypeparams().empty(); }
-    bool hasShapeOperands() { return !getShape().empty(); }
-    unsigned numLenParams() { return getTypeparams().size(); }
-    operand_range getLenParams() { return getTypeparams(); }
-    unsigned numShapeOperands() { return getShape().size(); }
-    operand_range getShapeOperands() { return getShape(); }
-    static mlir::Type getRefTy(mlir::Type ty);
-  }];
-}
-
-def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
-  [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object on an openmp device";
-
-  let description = [{
-    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
-    The memory object that is deallocated is placed in an undefined state
-    after `fir.omp_target_freemem`.
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
-    ```
-  }];
-
-  let arguments = (ins
-  Arg<AnyIntegerType, "", [MemFree]>:$device,
-  Arg<fir_HeapType, "", [MemFree]>:$heapref
-  );
-  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
-}
-
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 042ade6b1e0a1..a3de3ae9d116a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1168,105 +1168,6 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetAllocMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::Type heapTy = heap.getType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
-    mlir::Location loc = heap.getLoc();
-    auto ity = lowerTy().indexType();
-    mlir::Type dataTy = fir::unwrapRefType(heapTy);
-    mlir::Type llvmObjectTy = convertObjectType(dataTy);
-    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
-      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
-                "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
-      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands().drop_front())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
-    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
-    auto mallocTy =
-        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
-    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
-      size = integerCast(loc, rewriter, mallocTy, size);
-    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        heap, ::getLlvmPtrType(heap.getContext()),
-        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
-    return mlir::success();
-  }
-
-  /// Compute the allocation size in bytes of the element type of
-  /// \p llTy pointer type. The result is returned as a value of \p idxTy
-  /// integer type.
-  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
-  }
-};
-} // namespace
-
-static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp freeFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
-    return freeFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_free",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMVoidType::get(module->getContext()),
-          {getLlvmPtrType(module->getContext()), i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetFreeMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
-    mlir::Location loc = freemem.getLoc();
-    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
-    rewriter.create<mlir::LLVM::CallOp>(
-        loc, mlir::TypeRange{},
-        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
-        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
-    rewriter.eraseOp(freemem);
-    return mlir::success();
-  }
-};
-} // namespace
-
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4373,8 +4274,7 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
-      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 37f1c9f97e1ce..a04c5d7eb7ee7 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -125,10 +125,171 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type
+convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
+    mlir::Type ty, mlir::Value val, bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "omp.target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(lowerTy(), loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
+    allocmemOp->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    auto callOp = rewriter.create<mlir::LLVM::CallOp>(
+        loc, llvmPtrTy,
+        mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    return mlir::success();
+  }
+};
 } // namespace
 
 void fir::populateOpenMPFIRToLLVMConversionPatterns(
     const LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
   patterns.add<MapInfoOpConversion>(converter);
   patterns.add<PrivateClauseOpConversion>(converter);
+  patterns.add<TargetAllocMemOpConversion>(converter);
 }
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 9335a4b041ac8..f6c794b8fe9ae 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,38 +106,24 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// Parser shared by Alloca and Allocmem
 /// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
-/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
-///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
-///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult
-parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
-                   mlir::OperationState &result, bool isTargetOp = false) {
-  auto &builder = parser.getBuilder();
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
-  // Parse device number as a new operand
-  if (isTargetOp) {
-    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
-    mlir::Type deviceType;
-    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
-      return mlir::failure();
-    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
-      return mlir::failure();
-    if (parser.parseComma())
-      return mlir::failure();
-  }
+static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
+                                            mlir::OpAsmParser &parser,
+                                            mlir::OperationState &result) {
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
+  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -161,19 +147,13 @@ parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
-
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  llvm::SmallVector<std::int32_t> segmentSizes;
-  if (isTargetOp)
-    segmentSizes.push_back(1);
-  segmentSizes.push_back(typeparamsSize);
-  segmentSizes.push_back(shapeSize);
-  result.addAttribute("operandSegmentSizes",
-                      builder.getDenseI32ArrayAttr(segmentSizes));
+  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
+                                                 {typeparamsSize, shapeSize}));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -405,56 +385,6 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
-//===----------------------------------------------------------------------===//
-// OmpTargetAllocMemOp
-//===----------------------------------------------------------------------===//
-
-mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
-  return mlir::cast<fir::HeapType>(getType()).getEleTy();
-}
-
-mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
-  return fir::HeapType::get(ty);
-}
-
-mlir::ParseResult
-fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
-                                mlir::OperationState &result) {
-  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
-}
-
-void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
-  p << " ";
-  p.printOperand(getDevice());
-  p << " : ";
-  p << getDevice().getType();
-  p << ", ";
-  p << getInType();
-  if (!getTypeparams().empty()) {
-    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
-  }
-  for (auto sh : getShape()) {
-    p << ", ";
-    p.printOperand(sh);
-  }
-  p.printOptionalAttrDict((*this)->getAttrs(),
-                          {"in_type", "operandSegmentSizes"});
-}
-
-llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
-  llvm::SmallVector<llvm::StringRef> visited;
-  if (verifyInType(getInType(), visited, numShapeOperands()))
-    return emitOpError("invalid type for allocation");
-  if (verifyTypeParamCount(getInType(), numLenParams()))
-    return emitOpError("LEN params do not correspond to type");
-  mlir::Type outType = getType();
-  if (!mlir::dyn_cast<fir::HeapType>(outType))
-    return emitOpError("must be a !fir.heap type");
-  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
-    return emitOpError("cannot allocate !fir.box of unknown rank or type");
-  return mlir::success();
-}
-
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
deleted file mode 100644
index 5140c91c9510c..0000000000000
--- a/flang/test/Fir/omp_target_allocmem.fir
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
-// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
-func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  return %1 : !fir.heap<!fir.array<3x3xi32>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
-// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
-func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
-// CHECK-SAME: i32 %[[len:.*]])
-// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
-// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
-// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
-func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
-}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
similarity index 51%
rename from flang/test/Fir/omp_target_freemem.fir
rename to flang/test/Fir/omp_target_allocmem_freemem.fir
index 02e136076a9cf..9202202728454 100644
--- a/flang/test/Fir/omp_target_freemem.fir
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -1,28 +1,33 @@
 // RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_nonchar() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3xi32>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_char() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
 func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index ac80926053a2d..bcc7c906041ac 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1887,4 +1887,68 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Allocates memory on the specified OpenMP device for an object of the given type.
+    Returns an integer value representing the device pointer to the allocated memory.
+    The memory is uninitialized after allocation. Operations must be paired with 
+    `omp.target_freemem` to avoid memory leaks.
+
+    ```mlir
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyInteger>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyInteger>:$typeparams,
+    Variadic<AnyInteger>:$shape
+  );
+  let results = (outs I64);
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// TargetFreeMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetFreeMemOp : OpenMP_Op<"target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free memory on an openmp device";
+
+  let description = [{
+    Deallocates memory on the specified OpenMP device that was previously 
+    allocated by an `omp.target_allocmem` operation. The memory is placed 
+    in an undefined state after deallocation.
+    ```
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+      omp.target_freemem %device, %ptr : i32, i64
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyInteger, "", [MemFree]>:$device,
+  Arg<I64, "", [MemFree]>:$heapref
+  );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..5e3ad6ccf3ffe 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3493,6 +3493,108 @@ LogicalResult ScanOp::verify() {
                    "reduction modifier");
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type omp::TargetAllocMemOp::getAllocatedType() {
+  return getInTypeAttr().getValue();
+}
+
+/// operation ::= %res = (`omp.target_alloc_mem`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
+static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+
+  // Parse device number as a new operand
+  mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+  mlir::Type deviceType;
+  if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+    return mlir::failure();
+  if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+    return mlir::failure();
+  if (parser.parseComma())
+    return mlir::failure();
+
+  mlir::Type intype;
+  if (parser.parseType(intype))
+    return mlir::failure();
+  result.addAttribute("in_type", mlir::TypeAttr::get(intype));
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
+  llvm::SmallVector<mlir::Type> typeVec;
+  if (!parser.parseOptionalLParen()) {
+    // parse the LEN params of the derived type. (<params> : <types>)
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
+        parser.parseColonTypeList(typeVec) || parser.parseRParen())
+      return mlir::failure();
+    typeparamsSize = operands.size();
+    hasOperands = true;
+  }
+  std::int32_t shapeSize = 0;
+  if (!parser.parseOptionalComma()) {
+    // parse size to scale by, vector of n dimensions of type index
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None))
+      return mlir::failure();
+    shapeSize = operands.size() - typeparamsSize;
+    auto idxTy = builder.getIndexType();
+    for (std::int32_t i = typeparamsSize, end = operands.size(); i != end; ++i)
+      typeVec.push_back(idxTy);
+    hasOperands = true;
+  }
+  if (hasOperands &&
+      parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
+                             result.operands))
+    return mlir::failure();
+
+  mlir::Type restype = builder.getIntegerType(64);
+  ;
+  if (!restype) {
+    parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
+    return mlir::failure();
+  }
+  llvm::SmallVector<std::int32_t> segmentSizes{1, typeparamsSize, shapeSize};
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
+  if (parser.parseOptionalAttrDict(result.attributes) ||
+      parser.addTypeToList(restype, result.types))
+    return mlir::failure();
+  return mlir::success();
+}
+
+mlir::ParseResult omp::TargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  return parseTargetAllocMemOp(parser, result);
+}
+
+void omp::TargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult omp::TargetAllocMemOp::verify() {
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<IntegerType>(outType))
+    return emitOpError("must be a integer type");
+  return mlir::success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index eece8573f00ec..63b41f3b13363 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5714,6 +5714,82 @@ static bool isTargetDeviceOp(Operation *op) {
   return false;
 }
 
+static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder,
+                                         llvm::Module *llvmModule) {
+  llvm::Type *i64Ty = builder.getInt64Ty();
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *returnType = builder.getPtrTy(0);
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(returnType, {i64Ty, i32Ty}, false);
+  llvm::Function *func = cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_alloc", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                        LLVM::ModuleTranslation &moduleTranslation) {
+  auto allocMemOp = cast<omp::TargetAllocMemOp>(opInst);
+  if (!allocMemOp)
+    return failure();
+
+  // Get "omp_target_alloc" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = allocMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the allocation size.
+  llvm::DataLayout dataLayout = llvmModule->getDataLayout();
+  mlir::Type heapTy = allocMemOp.getAllocatedType();
+  llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
+  llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
+  llvm::ConstantInt *allocSize = builder.getInt64(typeSize.getFixedValue());
+  // Create call to "omp_target_alloc" with the args as translated llvm values.
+  llvm::CallInst *call =
+      builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});
+  llvm::Value *resultI64 = builder.CreatePtrToInt(call, builder.getInt64Ty());
+
+  // Map the result
+  moduleTranslation.mapValue(allocMemOp.getResult(), resultI64);
+  return success();
+}
+
+static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder,
+                                        llvm::Module *llvmModule) {
+  llvm::Type *ptrTy = builder.getPtrTy(0);
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *voidTy = builder.getVoidTy();
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(voidTy, {ptrTy, i32Ty}, false);
+  llvm::Function *func = dyn_cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_free", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation) {
+  auto freeMemOp = cast<omp::TargetFreeMemOp>(opInst);
+  if (!freeMemOp)
+    return failure();
+
+  // Get "omp_target_free" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = freeMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the corresponding heapref value in llvm
+  mlir::Value heapref = freeMemOp.getHeapref();
+  llvm::Value *llvmHeapref = moduleTranslation.lookupValue(heapref);
+  // Convert heapref int to ptr and call "omp_target_free"
+  llvm::Value *intToPtr =
+      builder.CreateIntToPtr(llvmHeapref, builder.getPtrTy(0));
+  builder.CreateCall(ompTragetFreeFunc, {intToPtr, llvmDeviceNum});
+  return success();
+}
+
 /// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including
 /// OpenMP runtime calls).
 static LogicalResult
@@ -5871,6 +5947,12 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
                 // etc. and then discarded
                 return success();
               })
+          .Case([&](omp::TargetAllocMemOp) {
+            return convertTargetAllocMemOp(*op, builder, moduleTranslation);
+          })
+          .Case([&](omp::TargetFreeMemOp) {
+            return convertTargetFreeMemOp(*op, builder, moduleTranslation);
+          })
           .Default([&](Operation *inst) {
             return inst->emitError()
                    << "not yet implemented: " << inst->getName();
diff --git a/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
new file mode 100644
index 0000000000000..1bc97609ccff4
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt %s -convert-openmp-to-llvm | mlir-translate -mlir-to-llvmir | FileCheck %s
+
+// This file contains MLIR test cases for omp.target_allocmem and omp.target_freemem
+
+// CHECK-LABEL: test_alloc_free_i64
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 8, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_i64() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, i64
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_1d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 64, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_1d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_2d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 1024, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_2d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16x16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}

>From 1e7a216d3d9e07146ebb364025983e0f5000eefa Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 21:05:35 +0530
Subject: [PATCH 5/7] Fix comments

---
 flang/lib/Optimizer/Dialect/FIROps.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index f6c794b8fe9ae..ba5c81d826e5e 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -107,7 +107,6 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
 }
 
 /// Parser shared by Alloca and Allocmem
-/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword

>From 3879eb7271098d52281aca44e77fc82af9987f02 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 9 Jul 2025 15:49:06 +0530
Subject: [PATCH 6/7] clang format

---
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 24 ++++++++++++-------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index a04c5d7eb7ee7..14cc7bb511f0f 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -142,8 +142,8 @@ static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
           /*isVarArg=*/false));
 }
 
-static mlir::Type
-convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
     return converter.convertBoxTypeAsStruct(boxTy);
   return converter.convertType(firType);
@@ -189,8 +189,9 @@ computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
 }
 
 static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
   return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
 }
 
@@ -224,8 +225,10 @@ genAllocationScaleSize(OP op, mlir::Type ity,
 }
 
 static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
-    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
-    mlir::Type ty, mlir::Value val, bool fold = false) {
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
   auto valTy = val.getType();
   // If the value was not yet lowered, lower its type so that it can
   // be used in getPrimitiveTypeSizeInBits.
@@ -261,11 +264,13 @@ struct TargetAllocMemOpConversion
     auto ity = lowerTy().indexType();
     mlir::Type dataTy = fir::unwrapRefType(heapTy);
     mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
-    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    mlir::Type llvmPtrTy =
+        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "omp.target_allocmem codegen of derived type with length "
                 "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy,
+                                          lowerTy().getDataLayout());
     if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands().drop_front())
@@ -281,7 +286,8 @@ struct TargetAllocMemOpConversion
         loc, llvmPtrTy,
         mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
         addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(
+        allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
     return mlir::success();
   }
 };

>From 4314bfb881dca9f23f912b672be4cff9aade84ca Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 14 Jul 2025 19:12:23 +0530
Subject: [PATCH 7/7] Update TargetAllocMemOpConversion. Move utility functions
 to utils

Co-authored by @ergawy
---
 flang/include/flang/Optimizer/Support/Utils.h |  63 ++++
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 106 ++-----
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 148 +--------
 flang/lib/Optimizer/Support/Utils.cpp         |  51 ++++
 .../test/Fir/omp_target_allocmem_freemem.fir  | 281 +++++++++++++++++-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   4 +-
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |   1 -
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |   5 +-
 8 files changed, 427 insertions(+), 232 deletions(-)

diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index ec73af6ec72e9..967224610db72 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -27,6 +27,8 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
 
+#include "flang/Optimizer/CodeGen/TypeConverter.h"
+
 namespace fir {
 /// Return the integer value of a arith::ConstantOp.
 inline std::int64_t toInt(mlir::arith::ConstantOp cop) {
@@ -224,6 +226,67 @@ std::optional<llvm::ArrayRef<int64_t>> getComponentLowerBoundsIfNonDefault(
     fir::RecordType recordType, llvm::StringRef component,
     mlir::ModuleOp module, const mlir::SymbolTable *symbolTable = nullptr);
 
+// Convert FIR type to LLVM without turning fir.box<T> into memory
+// reference.
+mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                             mlir::Type firType);
+
+/// Generate a LLVM constant value of type `ity`, using the provided offset.
+mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset);
+
+/// Helper function for generating the LLVM IR that computes the distance
+/// in bytes between adjacent elements pointed to by a pointer
+/// of type \p ptrTy. The result is returned as a value of \p idxTy integer
+/// type.
+mlir::Value computeElementDistance(mlir::Location loc,
+                                   mlir::Type llvmObjectType, mlir::Type idxTy,
+                                   mlir::ConversionPatternRewriter &rewriter,
+                                   const mlir::DataLayout &dataLayout);
+
+// Compute the alloc scale size (constant factors encoded in the array type).
+// We do this for arrays without a constant interior or arrays of character with
+// dynamic length arrays, since those are the only ones that get decayed to a
+// pointer to the element type.
+template <typename OP>
+inline mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        fir::genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+/// Perform an extension or truncation as needed on an integer value. Lowering
+/// to the specific target may involve some sign-extending or truncation of
+/// values, particularly to fit them from abstract box types to the
+/// appropriate reified structures.
+mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                        mlir::Location loc,
+                        mlir::ConversionPatternRewriter &rewriter,
+                        mlir::Type ty, mlir::Value val, bool fold = false);
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_SUPPORT_UTILS_H
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..6706660504d4e 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -85,14 +85,6 @@ static inline mlir::Type getI8Type(mlir::MLIRContext *context) {
   return mlir::IntegerType::get(context, 8);
 }
 
-static mlir::LLVM::ConstantOp
-genConstantIndex(mlir::Location loc, mlir::Type ity,
-                 mlir::ConversionPatternRewriter &rewriter,
-                 std::int64_t offset) {
-  auto cattr = rewriter.getI64IntegerAttr(offset);
-  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
-}
-
 static mlir::Block *createBlock(mlir::ConversionPatternRewriter &rewriter,
                                 mlir::Block *insertBefore) {
   assert(insertBefore && "expected valid insertion block");
@@ -165,39 +157,6 @@ getDependentTypeMemSizeFn(fir::RecordType recTy, fir::AllocaOp op,
   TODO(op.getLoc(), "did not find allocation function");
 }
 
-// Compute the alloc scale size (constant factors encoded in the array type).
-// We do this for arrays without a constant interior or arrays of character with
-// dynamic length arrays, since those are the only ones that get decayed to a
-// pointer to the element type.
-template <typename OP>
-static mlir::Value
-genAllocationScaleSize(OP op, mlir::Type ity,
-                       mlir::ConversionPatternRewriter &rewriter) {
-  mlir::Location loc = op.getLoc();
-  mlir::Type dataTy = op.getInType();
-  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
-  fir::SequenceType::Extent constSize = 1;
-  if (seqTy) {
-    int constRows = seqTy.getConstantRows();
-    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
-    if (constRows != static_cast<int>(shape.size())) {
-      for (auto extent : shape) {
-        if (constRows-- > 0)
-          continue;
-        if (extent != fir::SequenceType::getUnknownExtent())
-          constSize *= extent;
-      }
-    }
-  }
-
-  if (constSize != 1) {
-    mlir::Value constVal{
-        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
-    return constVal;
-  }
-  return nullptr;
-}
-
 namespace {
 struct DeclareOpConversion : public fir::FIROpConversion<fir::cg::XDeclareOp> {
 public:
@@ -232,7 +191,7 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
     auto loc = alloc.getLoc();
     mlir::Type ity = lowerTy().indexType();
     unsigned i = 0;
-    mlir::Value size = genConstantIndex(loc, ity, rewriter, 1).getResult();
+    mlir::Value size = fir::genConstantIndex(loc, ity, rewriter, 1).getResult();
     mlir::Type firObjType = fir::unwrapRefType(alloc.getType());
     mlir::Type llvmObjectType = convertObjectType(firObjType);
     if (alloc.hasLenParams()) {
@@ -264,7 +223,7 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
                << scalarType << " with type parameters";
       }
     }
-    if (auto scaleSize = genAllocationScaleSize(alloc, ity, rewriter))
+    if (auto scaleSize = fir::genAllocationScaleSize(alloc, ity, rewriter))
       size =
           rewriter.createOrFold<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     if (alloc.hasShapeOperands()) {
@@ -441,7 +400,7 @@ struct BoxIsArrayOpConversion : public fir::FIROpConversion<fir::BoxIsArrayOp> {
     auto loc = boxisarray.getLoc();
     TypePair boxTyPair = getBoxTypePair(boxisarray.getVal().getType());
     mlir::Value rank = getRankFromBox(loc, boxTyPair, a, rewriter);
-    mlir::Value c0 = genConstantIndex(loc, rank.getType(), rewriter, 0);
+    mlir::Value c0 = fir::genConstantIndex(loc, rank.getType(), rewriter, 0);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         boxisarray, mlir::LLVM::ICmpPredicate::ne, rank, c0);
     return mlir::success();
@@ -777,7 +736,7 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
       // Do folding for constant inputs.
       if (auto constVal = fir::getIntIfConstant(op0)) {
         mlir::Value normVal =
-            genConstantIndex(loc, toTy, rewriter, *constVal ? 1 : 0);
+            fir::genConstantIndex(loc, toTy, rewriter, *constVal ? 1 : 0);
         rewriter.replaceOp(convert, normVal);
         return mlir::success();
       }
@@ -790,7 +749,7 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
       }
 
       // Compare the input with zero.
-      mlir::Value zero = genConstantIndex(loc, fromTy, rewriter, 0);
+      mlir::Value zero = fir::genConstantIndex(loc, fromTy, rewriter, 0);
       auto isTrue = rewriter.create<mlir::LLVM::ICmpOp>(
           loc, mlir::LLVM::ICmpPredicate::ne, op0, zero);
 
@@ -1037,21 +996,6 @@ static mlir::SymbolRefAttr getMalloc(fir::AllocMemOp op,
   return getMallocInModule(mod, op, rewriter, indexType);
 }
 
-/// Helper function for generating the LLVM IR that computes the distance
-/// in bytes between adjacent elements pointed to by a pointer
-/// of type \p ptrTy. The result is returned as a value of \p idxTy integer
-/// type.
-static mlir::Value
-computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
-                       mlir::Type idxTy,
-                       mlir::ConversionPatternRewriter &rewriter,
-                       const mlir::DataLayout &dataLayout) {
-  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
-  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
-  std::int64_t distance = llvm::alignTo(size, alignment);
-  return genConstantIndex(loc, idxTy, rewriter, distance);
-}
-
 /// Return value of the stride in bytes between adjacent elements
 /// of LLVM type \p llTy. The result is returned as a value of
 /// \p idxTy integer type.
@@ -1060,7 +1004,7 @@ genTypeStrideInBytes(mlir::Location loc, mlir::Type idxTy,
                      mlir::ConversionPatternRewriter &rewriter, mlir::Type llTy,
                      const mlir::DataLayout &dataLayout) {
   // Create a pointer type and use computeElementDistance().
-  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+  return fir::computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
 }
 
 namespace {
@@ -1079,7 +1023,7 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "fir.allocmem codegen of derived type with length parameters");
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
+    if (auto scaleSize = fir::genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands())
       size = rewriter.create<mlir::LLVM::MulOp>(
@@ -1102,7 +1046,8 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
   mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
                                  mlir::ConversionPatternRewriter &rewriter,
                                  mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+    return fir::computeElementDistance(loc, llTy, idxTy, rewriter,
+                                       getDataLayout());
   }
 };
 } // namespace
@@ -1286,7 +1231,7 @@ genCUFAllocDescriptor(mlir::Location loc,
   mlir::Type structTy = typeConverter.convertBoxTypeAsStruct(boxTy);
   std::size_t boxSize = dl->getTypeSizeInBits(structTy) / 8;
   mlir::Value sizeInBytes =
-      genConstantIndex(loc, llvmIntPtrType, rewriter, boxSize);
+      fir::genConstantIndex(loc, llvmIntPtrType, rewriter, boxSize);
   llvm::SmallVector args = {sizeInBytes, sourceFile, sourceLine};
   return rewriter
       .create<mlir::LLVM::CallOp>(loc, fctTy, RTNAME_STRING(CUFAllocDescriptor),
@@ -1521,7 +1466,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
       // representation of derived types with pointer/allocatable components.
       // This has been seen in hashing algorithms using TRANSFER.
       mlir::Value zero =
-          genConstantIndex(loc, rewriter.getI64Type(), rewriter, 0);
+          fir::genConstantIndex(loc, rewriter.getI64Type(), rewriter, 0);
       descriptor = insertField(rewriter, loc, descriptor,
                                {getLenParamFieldId(boxTy), 0}, zero);
     }
@@ -1864,8 +1809,8 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
     bool hasSlice = !xbox.getSlice().empty();
     unsigned sliceOffset = xbox.getSliceOperandIndex();
     mlir::Location loc = xbox.getLoc();
-    mlir::Value zero = genConstantIndex(loc, i64Ty, rewriter, 0);
-    mlir::Value one = genConstantIndex(loc, i64Ty, rewriter, 1);
+    mlir::Value zero = fir::genConstantIndex(loc, i64Ty, rewriter, 0);
+    mlir::Value one = fir::genConstantIndex(loc, i64Ty, rewriter, 1);
     mlir::Value prevPtrOff = one;
     mlir::Type eleTy = boxTy.getEleTy();
     const unsigned rank = xbox.getRank();
@@ -1914,7 +1859,7 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
         prevDimByteStride =
             getCharacterByteSize(loc, rewriter, charTy, adaptor.getLenParams());
       } else {
-        prevDimByteStride = genConstantIndex(
+        prevDimByteStride = fir::genConstantIndex(
             loc, i64Ty, rewriter,
             charTy.getLen() * lowerTy().characterBitsize(charTy) / 8);
       }
@@ -2072,7 +2017,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     if (auto charTy = mlir::dyn_cast<fir::CharacterType>(inputEleTy)) {
       if (charTy.hasConstantLen()) {
         mlir::Value len =
-            genConstantIndex(loc, idxTy, rewriter, charTy.getLen());
+            fir::genConstantIndex(loc, idxTy, rewriter, charTy.getLen());
         lenParams.emplace_back(len);
       } else {
         mlir::Value len = getElementSizeFromBox(loc, idxTy, inputBoxTyPair,
@@ -2081,7 +2026,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
           assert(!isInGlobalOp(rewriter) &&
                  "character target in global op must have constant length");
           mlir::Value width =
-              genConstantIndex(loc, idxTy, rewriter, charTy.getFKind());
+              fir::genConstantIndex(loc, idxTy, rewriter, charTy.getFKind());
           len = rewriter.create<mlir::LLVM::SDivOp>(loc, idxTy, len, width);
         }
         lenParams.emplace_back(len);
@@ -2135,8 +2080,9 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
                 mlir::ConversionPatternRewriter &rewriter) const {
     mlir::Location loc = rebox.getLoc();
     mlir::Value zero =
-        genConstantIndex(loc, lowerTy().indexType(), rewriter, 0);
-    mlir::Value one = genConstantIndex(loc, lowerTy().indexType(), rewriter, 1);
+        fir::genConstantIndex(loc, lowerTy().indexType(), rewriter, 0);
+    mlir::Value one =
+        fir::genConstantIndex(loc, lowerTy().indexType(), rewriter, 1);
     for (auto iter : llvm::enumerate(llvm::zip(extents, strides))) {
       mlir::Value extent = std::get<0>(iter.value());
       unsigned dim = iter.index();
@@ -2168,7 +2114,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     mlir::Location loc = rebox.getLoc();
     mlir::Type byteTy = ::getI8Type(rebox.getContext());
     mlir::Type idxTy = lowerTy().indexType();
-    mlir::Value zero = genConstantIndex(loc, idxTy, rewriter, 0);
+    mlir::Value zero = fir::genConstantIndex(loc, idxTy, rewriter, 0);
     // Apply subcomponent and substring shift on base address.
     if (!rebox.getSubcomponent().empty() || !rebox.getSubstr().empty()) {
       // Cast to inputEleTy* so that a GEP can be used.
@@ -2196,7 +2142,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     // and strides.
     llvm::SmallVector<mlir::Value> slicedExtents;
     llvm::SmallVector<mlir::Value> slicedStrides;
-    mlir::Value one = genConstantIndex(loc, idxTy, rewriter, 1);
+    mlir::Value one = fir::genConstantIndex(loc, idxTy, rewriter, 1);
     const bool sliceHasOrigins = !rebox.getShift().empty();
     unsigned sliceOps = rebox.getSliceOperandIndex();
     unsigned shiftOps = rebox.getShiftOperandIndex();
@@ -2269,7 +2215,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     // which may be OK if all new extents are ones, the stride does not
     // matter, use one.
     mlir::Value stride = inputStrides.empty()
-                             ? genConstantIndex(loc, idxTy, rewriter, 1)
+                             ? fir::genConstantIndex(loc, idxTy, rewriter, 1)
                              : inputStrides[0];
     for (unsigned i = 0; i < rebox.getShape().size(); ++i) {
       mlir::Value rawExtent = operands[rebox.getShapeOperandIndex() + i];
@@ -2504,9 +2450,9 @@ struct XArrayCoorOpConversion
     unsigned shiftOffset = coor.getShiftOperandIndex();
     unsigned sliceOffset = coor.getSliceOperandIndex();
     auto sliceOps = coor.getSlice().begin();
-    mlir::Value one = genConstantIndex(loc, idxTy, rewriter, 1);
+    mlir::Value one = fir::genConstantIndex(loc, idxTy, rewriter, 1);
     mlir::Value prevExt = one;
-    mlir::Value offset = genConstantIndex(loc, idxTy, rewriter, 0);
+    mlir::Value offset = fir::genConstantIndex(loc, idxTy, rewriter, 0);
     const bool isShifted = !coor.getShift().empty();
     const bool isSliced = !coor.getSlice().empty();
     const bool baseIsBoxed =
@@ -2836,7 +2782,7 @@ struct CoordinateOpConversion
         // of lower bound aspects. This both accounts for dynamically sized
         // types and non contiguous arrays.
         auto idxTy = lowerTy().indexType();
-        mlir::Value off = genConstantIndex(loc, idxTy, rewriter, 0);
+        mlir::Value off = fir::genConstantIndex(loc, idxTy, rewriter, 0);
         unsigned arrayDim = arrTy.getDimension();
         for (unsigned dim = 0; dim < arrayDim && it != end; ++dim, ++it) {
           mlir::Value stride =
@@ -3761,7 +3707,7 @@ struct IsPresentOpConversion : public fir::FIROpConversion<fir::IsPresentOp> {
       ptr = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, ptr, 0);
     }
     mlir::LLVM::ConstantOp c0 =
-        genConstantIndex(isPresent.getLoc(), idxTy, rewriter, 0);
+        fir::genConstantIndex(isPresent.getLoc(), idxTy, rewriter, 0);
     auto addr = rewriter.create<mlir::LLVM::PtrToIntOp>(loc, idxTy, ptr);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         isPresent, mlir::LLVM::ICmpPredicate::ne, addr, c0);
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 14cc7bb511f0f..c8f9c2c154b95 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -21,6 +21,7 @@
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/FatalError.h"
 #include "flang/Optimizer/Support/InternalNames.h"
+#include "flang/Optimizer/Support/Utils.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -126,130 +127,6 @@ struct PrivateClauseOpConversion
   }
 };
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
-static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
-                                    mlir::Type firType) {
-  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
-    return converter.convertBoxTypeAsStruct(boxTy);
-  return converter.convertType(firType);
-}
-
-static llvm::SmallVector<mlir::NamedAttribute>
-addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
-                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
-                     int32_t numCallOperands) {
-  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
-  newAttrs.reserve(attrs.size() + 2);
-
-  for (mlir::NamedAttribute attr : attrs) {
-    if (attr.getName() != "operandSegmentSizes")
-      newAttrs.push_back(attr);
-  }
-
-  newAttrs.push_back(rewriter.getNamedAttr(
-      "operandSegmentSizes",
-      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
-  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
-                                           rewriter.getDenseI32ArrayAttr({})));
-  return newAttrs;
-}
-
-static mlir::LLVM::ConstantOp
-genConstantIndex(mlir::Location loc, mlir::Type ity,
-                 mlir::ConversionPatternRewriter &rewriter,
-                 std::int64_t offset) {
-  auto cattr = rewriter.getI64IntegerAttr(offset);
-  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
-}
-
-static mlir::Value
-computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
-                       mlir::Type idxTy,
-                       mlir::ConversionPatternRewriter &rewriter,
-                       const mlir::DataLayout &dataLayout) {
-  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
-  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
-  std::int64_t distance = llvm::alignTo(size, alignment);
-  return genConstantIndex(loc, idxTy, rewriter, distance);
-}
-
-static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                      mlir::ConversionPatternRewriter &rewriter,
-                                      mlir::Type llTy,
-                                      const mlir::DataLayout &dataLayout) {
-  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
-}
-
-template <typename OP>
-static mlir::Value
-genAllocationScaleSize(OP op, mlir::Type ity,
-                       mlir::ConversionPatternRewriter &rewriter) {
-  mlir::Location loc = op.getLoc();
-  mlir::Type dataTy = op.getInType();
-  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
-  fir::SequenceType::Extent constSize = 1;
-  if (seqTy) {
-    int constRows = seqTy.getConstantRows();
-    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
-    if (constRows != static_cast<int>(shape.size())) {
-      for (auto extent : shape) {
-        if (constRows-- > 0)
-          continue;
-        if (extent != fir::SequenceType::getUnknownExtent())
-          constSize *= extent;
-      }
-    }
-  }
-
-  if (constSize != 1) {
-    mlir::Value constVal{
-        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
-    return constVal;
-  }
-  return nullptr;
-}
-
-static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
-                               mlir::Location loc,
-                               mlir::ConversionPatternRewriter &rewriter,
-                               mlir::Type ty, mlir::Value val,
-                               bool fold = false) {
-  auto valTy = val.getType();
-  // If the value was not yet lowered, lower its type so that it can
-  // be used in getPrimitiveTypeSizeInBits.
-  if (!mlir::isa<mlir::IntegerType>(valTy))
-    valTy = converter.convertType(valTy);
-  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
-  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
-  if (fold) {
-    if (toSize < fromSize)
-      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
-    if (toSize > fromSize)
-      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
-  } else {
-    if (toSize < fromSize)
-      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
-    if (toSize > fromSize)
-      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
-  }
-  return val;
-}
-
 // FIR Op specific conversion for TargetAllocMemOp
 struct TargetAllocMemOpConversion
     : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
@@ -259,19 +136,16 @@ struct TargetAllocMemOpConversion
   matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
     mlir::Type heapTy = allocmemOp.getAllocatedType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
     mlir::Location loc = allocmemOp.getLoc();
     auto ity = lowerTy().indexType();
     mlir::Type dataTy = fir::unwrapRefType(heapTy);
-    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
-    mlir::Type llvmPtrTy =
-        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    mlir::Type llvmObjectTy = fir::convertObjectType(lowerTy(), dataTy);
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "omp.target_allocmem codegen of derived type with length "
                 "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy,
-                                          lowerTy().getDataLayout());
-    if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
+    mlir::Value size = fir::computeElementDistance(
+        loc, llvmObjectTy, ity, rewriter, lowerTy().getDataLayout());
+    if (auto scaleSize = fir::genAllocationScaleSize(allocmemOp, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands().drop_front())
       size = rewriter.create<mlir::LLVM::MulOp>(
@@ -281,13 +155,11 @@ struct TargetAllocMemOpConversion
         mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
     if (mallocTyWidth != ity.getIntOrFloatBitWidth())
       size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
-    allocmemOp->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    auto callOp = rewriter.create<mlir::LLVM::CallOp>(
-        loc, llvmPtrTy,
-        mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(
-        allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    rewriter.modifyOpInPlace(allocmemOp, [&]() {
+      allocmemOp.setInType(rewriter.getI8Type());
+      allocmemOp.getTypeparamsMutable().clear();
+      allocmemOp.getTypeparamsMutable().append(size);
+    });
     return mlir::success();
   }
 };
diff --git a/flang/lib/Optimizer/Support/Utils.cpp b/flang/lib/Optimizer/Support/Utils.cpp
index 5d663e28336c0..dbd42285e8fae 100644
--- a/flang/lib/Optimizer/Support/Utils.cpp
+++ b/flang/lib/Optimizer/Support/Utils.cpp
@@ -50,3 +50,54 @@ std::optional<llvm::ArrayRef<int64_t>> fir::getComponentLowerBoundsIfNonDefault(
       return componentInfo.getLowerBounds();
   return std::nullopt;
 }
+
+mlir::Type fir::convertObjectType(const fir::LLVMTypeConverter &converter,
+                                  mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+mlir::LLVM::ConstantOp
+fir::genConstantIndex(mlir::Location loc, mlir::Type ity,
+                      mlir::ConversionPatternRewriter &rewriter,
+                      std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+mlir::Value
+fir::computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                            mlir::Type idxTy,
+                            mlir::ConversionPatternRewriter &rewriter,
+                            const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return fir::genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+mlir::Value fir::integerCast(const fir::LLVMTypeConverter &converter,
+                             mlir::Location loc,
+                             mlir::ConversionPatternRewriter &rewriter,
+                             mlir::Type ty, mlir::Value val, bool fold) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
diff --git a/flang/test/Fir/omp_target_allocmem_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
index 9202202728454..03eb94acb1ac7 100644
--- a/flang/test/Fir/omp_target_allocmem_freemem.fir
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -1,8 +1,106 @@
 // RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
 
-// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
-// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// UNSUPPORTED: system-windows
+// Disabled on 32-bit targets due to the additional `trunc` opcodes required
+// UNSUPPORTED: target-x86
+// UNSUPPORTED: target=sparc-{{.*}}
+// UNSUPPORTED: target=sparcel-{{.*}}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 4, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, i32
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalars_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 400, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalars_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %0 = arith.constant 100 : index
+  %1 = omp.target_allocmem %device : i32, i32, %0
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_char() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 10, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<1,10>
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_char_kind() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 20, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_char_kind() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<2,10>
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_dynchar(
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_dynchar(%l : i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<1,?>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_dynchar_kind(
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 2, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_dynchar_kind(%l : i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<2,?>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 36, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
 func.func @omp_target_allocmem_array_of_nonchar() -> () {
   %device = arith.constant 0 : i32
   %1 = omp.target_allocmem %device : i32, !fir.array<3x3xi32>
@@ -10,9 +108,12 @@ func.func @omp_target_allocmem_array_of_nonchar() -> () {
   return
 }
 
-// CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
-// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_char() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 90, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
 func.func @omp_target_allocmem_array_of_char() -> () {
   %device = arith.constant 0 : i32
   %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
@@ -21,13 +122,173 @@ func.func @omp_target_allocmem_array_of_char() -> () {
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
-// CHECK-SAME: i32 %[[len:.*]])
-// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
-// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
-// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 9, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
 func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
   %device = arith.constant 0 : i32
   %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
   omp.target_freemem %device, %1 : i32, i64
   return
 }
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_nonchar(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 12, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_nonchar(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?xi32>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_nonchar2(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 4, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_nonchar2(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?xi32>, %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_char(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 60, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_char(%e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x!fir.char<2,10>>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_char2(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 20, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_char2(%e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?x!fir.char<2,10>>, %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_dynchar(
+// CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 6, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP1]]
+// CHECK-NEXT:    [[TMP6:%.*]] = mul i64 1, [[TMP5]]
+// CHECK-NEXT:    [[TMP7:%.*]] = call ptr @omp_target_alloc(i64 [[TMP6]], i32 0)
+// CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[TMP7]] to i64
+// CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP9]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_dynchar(%l: i32, %e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x!fir.char<2,?>>(%l : i32), %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_dynchar2(
+// CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 2, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP1]]
+// CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP1]]
+// CHECK-NEXT:    [[TMP7:%.*]] = mul i64 1, [[TMP6]]
+// CHECK-NEXT:    [[TMP8:%.*]] = call ptr @omp_target_alloc(i64 [[TMP7]], i32 0)
+// CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[TMP8]] to i64
+// CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP10]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_dynchar2(%l: i32, %e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?x!fir.char<2,?>>(%l : i32), %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_nonchar(
+// CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 240, [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 1, [[TMP4]]
+// CHECK-NEXT:    [[TMP6:%.*]] = call ptr @omp_target_alloc(i64 [[TMP5]], i32 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+// CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP8]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_nonchar(%0 : index, %1 : index) -> () {
+  %device = arith.constant 0 : i32
+  %2 = omp.target_allocmem %device : i32, !fir.array<4x?x3x?x5xi32>, %0, %1
+  omp.target_freemem %device, %2 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_char(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 240, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_char(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x4x!fir.char<2,10>>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_dynchar(
+// CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 24, [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 1, [[TMP4]]
+// CHECK-NEXT:    [[TMP6:%.*]] = call ptr @omp_target_alloc(i64 [[TMP5]], i32 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+// CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP8]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_dynchar(%arg0: index, %arg1: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x4x!fir.char<2,?>>(%arg0 : index), %arg1
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index bcc7c906041ac..93891b843ba56 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1912,8 +1912,8 @@ def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
     TypeAttr:$in_type,
     OptionalAttr<StrAttr>:$uniq_name,
     OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyInteger>:$typeparams,
-    Variadic<AnyInteger>:$shape
+    Variadic<IntLikeType>:$typeparams,
+    Variadic<IntLikeType>:$shape
   );
   let results = (outs I64);
 
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 5e3ad6ccf3ffe..bba796dc326c9 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3551,7 +3551,6 @@ static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,
     return mlir::failure();
 
   mlir::Type restype = builder.getIntegerType(64);
-  ;
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 63b41f3b13363..77e43fe4cb1b9 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5744,7 +5744,10 @@ convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
   mlir::Type heapTy = allocMemOp.getAllocatedType();
   llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
   llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
-  llvm::ConstantInt *allocSize = builder.getInt64(typeSize.getFixedValue());
+  llvm::Value *allocSize = builder.getInt64(typeSize.getFixedValue());
+  for (auto typeParam : allocMemOp.getTypeparams())
+    allocSize =
+        builder.CreateMul(allocSize, moduleTranslation.lookupValue(typeParam));
   // Create call to "omp_target_alloc" with the args as translated llvm values.
   llvm::CallInst *call =
       builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});


From flang-commits at lists.llvm.org  Mon Jul 14 09:40:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 09:40:45 -0700 (PDT)
Subject: [flang-commits] [flang] fc99ef7 - [flang] Allow embox's source_box
 to be a !fir.box. (#148305)
Message-ID: <6875330d.050a0220.3d43cb.39bc@mx.google.com>


Author: Slava Zakharin
Date: 2025-07-14T09:40:42-07:00
New Revision: fc99ef7411e7f33e03e639a22f83a271ea8aaaa8

URL: https://github.com/llvm/llvm-project/commit/fc99ef7411e7f33e03e639a22f83a271ea8aaaa8
DIFF: https://github.com/llvm/llvm-project/commit/fc99ef7411e7f33e03e639a22f83a271ea8aaaa8.diff

LOG: [flang] Allow embox's source_box to be a !fir.box. (#148305)

In order to create temporary copies of assumed-type arrays
(e.g. for `-frepack-arrays`), we have to allow the source_box
to be a !fir.box.

This patch replaces #147618.

Added: 
    

Modified: 
    flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
    flang/include/flang/Optimizer/Dialect/FIROps.td
    flang/include/flang/Optimizer/Dialect/FIRTypes.td
    flang/test/Fir/fir-ops.fir

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td b/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
index d48caf1a8cb3b..04f839386498c 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRCG/CGOps.td
@@ -60,7 +60,8 @@ def fircg_XEmboxOp : fircg_Op<"ext_embox", [AttrSizedOperandSegments]> {
       Variadic<AnyIntegerType>:$slice,
       Variadic<AnyCoordinateType>:$subcomponent,
       Variadic<AnyIntegerType>:$substr, Variadic<AnyIntegerType>:$lenParams,
-      Optional<fir_ClassType>:$sourceBox, OptionalAttr<I32Attr>:$allocator_idx);
+      Optional<BoxOrClassType>:$sourceBox,
+      OptionalAttr<I32Attr>:$allocator_idx);
   let results = (outs BoxOrClassType);
 
   let assemblyFormat = [{

diff  --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index e3f5c4403002a..99b5105ab365e 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -834,15 +834,11 @@ def fir_EmboxOp : fir_Op<"embox", [NoMemoryEffect, AttrSizedOperandSegments]> {
         - allocator_idx: specify special allocator to use.
   }];
 
-  let arguments = (ins
-    AnyReferenceLike:$memref,
-    Optional<AnyShapeType>:$shape,
-    Optional<fir_SliceType>:$slice,
-    Variadic<AnyIntegerType>:$typeparams,
-    Optional<fir_ClassType>:$sourceBox,
-    OptionalAttr<AffineMapAttr>:$accessMap,
-    OptionalAttr<I32Attr>:$allocator_idx
-  );
+  let arguments = (ins AnyReferenceLike:$memref, Optional<AnyShapeType>:$shape,
+      Optional<fir_SliceType>:$slice, Variadic<AnyIntegerType>:$typeparams,
+      Optional<BoxOrClassType>:$sourceBox,
+      OptionalAttr<AffineMapAttr>:$accessMap,
+      OptionalAttr<I32Attr>:$allocator_idx);
 
   let results = (outs BoxOrClassType);
 

diff  --git a/flang/include/flang/Optimizer/Dialect/FIRTypes.td b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
index 0ead54df3ca97..2fdc9a96804da 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRTypes.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
@@ -626,8 +626,10 @@ def AnyBoxLike : TypeConstraint<Or<[fir_BoxType.predicate,
     fir_BoxCharType.predicate, fir_BoxProcType.predicate,
     fir_ClassType.predicate]>, "any box">;
 
-def BoxOrClassType : TypeConstraint<Or<[fir_BoxType.predicate,
-    fir_ClassType.predicate]>, "box or class">;
+def BoxOrClassLike
+    : TypeConstraint<Or<[fir_BoxType.predicate, fir_ClassType.predicate]>,
+                     "box or class">;
+def BoxOrClassType : Type<BoxOrClassLike.predicate, "box or class">;
 
 def AnyRefOrBoxLike : TypeConstraint<Or<[AnyReferenceLike.predicate,
     AnyBoxLike.predicate, FunctionType.predicate]>,

diff  --git a/flang/test/Fir/fir-ops.fir b/flang/test/Fir/fir-ops.fir
index 3585bf9efca3e..0892eb9fa0de8 100644
--- a/flang/test/Fir/fir-ops.fir
+++ b/flang/test/Fir/fir-ops.fir
@@ -843,6 +843,16 @@ func.func @embox_tdesc(%arg0: !fir.class<!fir.array<10x!fir.type<derived_poly{a:
   return
 }
 
+// CHECK-LABEL:   func.func @embox_with_source_box(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.heap<!fir.array<?xnone>>,
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.shape<1>,
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>> {
+func.func @embox_with_source_box(%arg0: !fir.heap<!fir.array<?xnone>>, %arg1: !fir.shape<1>, %arg2: !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>> {
+// CHECK:           %[[VAL_0:.*]] = fir.embox %[[ARG0]](%[[ARG1]]) source_box %[[ARG2]] : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+  %0 = fir.embox %arg0(%arg1) source_box %arg2 : (!fir.heap<!fir.array<?xnone>>, !fir.shape<1>, !fir.box<!fir.array<?xnone>>) -> !fir.class<!fir.heap<!fir.array<?xnone>>>
+  return %0 : !fir.class<!fir.heap<!fir.array<?xnone>>>
+}
+
 func.func @test_fortran_var_attrs() {
   %0 = fir.alloca !fir.heap<f32> {fortran_attrs = #fir.var_attrs<allocatable>}
   %1 = fir.alloca f32 {fortran_attrs = #fir.var_attrs<asynchronous>}


        

From flang-commits at lists.llvm.org  Mon Jul 14 09:40:48 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Mon, 14 Jul 2025 09:40:48 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Allow embox's source_box to be a
	!fir.box. (PR #148305)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148305@github.com>
Message-ID: <68753310.170a0220.28055f.7ff2@mx.google.com>


https://github.com/vzakhari closed https://github.com/llvm/llvm-project/pull/148305

From flang-commits at lists.llvm.org  Mon Jul 14 09:41:18 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 09:41:18 -0700 (PDT)
Subject: [flang-commits] [flang] 9e56d0d - [flang] Fixed TBAA tags for
 derived types with descriptors. (#148093)
Message-ID: <6875332e.630a0220.b84e5.0319@mx.google.com>


Author: Slava Zakharin
Date: 2025-07-14T09:41:15-07:00
New Revision: 9e56d0d2ab125377740245599e3cc950fcf406bc

URL: https://github.com/llvm/llvm-project/commit/9e56d0d2ab125377740245599e3cc950fcf406bc
DIFF: https://github.com/llvm/llvm-project/commit/9e56d0d2ab125377740245599e3cc950fcf406bc.diff

LOG: [flang] Fixed TBAA tags for derived types with descriptors. (#148093)

We cannot attach any "data" or "descriptor" tag to accesses
of derived types that contain descriptors, because this
will make them non-aliasing with any generic "data" or "descriptor"
accesses, which is not correct. We have to skip TBAA tags attachment
for such accesses same way we do it for boxes.

Added: 
    flang/test/Transforms/tbaa-derived-with-descriptor.fir

Modified: 
    flang/lib/Optimizer/Transforms/AddAliasTags.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index 4e788f75d9d77..b27c1b26dedb3 100644
--- a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -79,6 +79,10 @@ class PassState {
   // For the given fir.declare returns the outermost fir.dummy_scope
   // in the current function.
   fir::DummyScopeOp getOutermostScope(fir::DeclareOp declareOp) const;
+  // Returns true, if the given type of a memref of a FirAliasTagOpInterface
+  // operation is a descriptor or contains a descriptor
+  // (e.g. !fir.ref<!fir.type<Derived{f:!fir.box<!fir.heap<f32>>}>>).
+  bool typeReferencesDescriptor(mlir::Type type);
 
 private:
   mlir::DominanceInfo &domInfo;
@@ -95,6 +99,10 @@ class PassState {
   // to the dominance information.
   llvm::DenseMap<mlir::func::FuncOp, llvm::SmallVector<fir::DummyScopeOp, 16>>
       sortedScopeOperations;
+
+  // Local pass cache for derived types that contain descriptor
+  // member(s), to avoid the cost of isRecordWithDescriptorMember().
+  llvm::DenseSet<mlir::Type> typesContainingDescriptors;
 };
 
 // Process fir.dummy_scope operations in the given func:
@@ -145,6 +153,22 @@ fir::DummyScopeOp PassState::getOutermostScope(fir::DeclareOp declareOp) const {
   return nullptr;
 }
 
+bool PassState::typeReferencesDescriptor(mlir::Type type) {
+  type = fir::unwrapAllRefAndSeqType(type);
+  if (mlir::isa<fir::BaseBoxType>(type))
+    return true;
+
+  if (mlir::isa<fir::RecordType>(type)) {
+    if (typesContainingDescriptors.contains(type))
+      return true;
+    if (fir::isRecordWithDescriptorMember(type)) {
+      typesContainingDescriptors.insert(type);
+      return true;
+    }
+  }
+  return false;
+}
+
 class AddAliasTagsPass : public fir::impl::AddAliasTagsBase<AddAliasTagsPass> {
 public:
   void runOnOperation() override;
@@ -200,11 +224,17 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
          "load and store only access one address");
   mlir::Value memref = accessedOperands.front();
 
-  // skip boxes. These get an "any descriptor access" tag in TBAABuilder
-  // (CodeGen). I didn't see any speedup from giving each box a separate TBAA
-  // type.
-  if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(memref.getType())))
+  // Skip boxes and derived types that contain descriptors.
+  // The box accesses get an "any descriptor access" tag in TBAABuilder
+  // (CodeGen). The derived types accesses get "any access" tag
+  // (because they access both the data and the descriptor(s)).
+  // Note that it would be incorrect to attach any "data" access
+  // tag to the derived type accesses here, because the tags
+  // attached to the descriptor accesses in CodeGen will make
+  // them non-conflicting with any descriptor accesses.
+  if (state.typeReferencesDescriptor(memref.getType()))
     return;
+
   LLVM_DEBUG(llvm::dbgs() << "Analysing " << op << "\n");
 
   const fir::AliasAnalysis::Source &source = state.getSource(memref);

diff  --git a/flang/test/Transforms/tbaa-derived-with-descriptor.fir b/flang/test/Transforms/tbaa-derived-with-descriptor.fir
new file mode 100644
index 0000000000000..18b9a801911f7
--- /dev/null
+++ b/flang/test/Transforms/tbaa-derived-with-descriptor.fir
@@ -0,0 +1,64 @@
+// Test TBAA tags attachment for accesses of derived types
+// that contain descriptors. They cannot be attached
+// with any reasonable "data" tag by fir-add-alias-tags pass,
+// because this will conflict with "any descriptor"
+// tags attached to the descriptor accesses that may be part
+// of these derived type values.
+// Representative Fortran example:
+// subroutine test
+//   type t
+//      real, allocatable :: f
+//   end type t
+//   associate(a => callee([1.0]))
+//     call ext(a(1)%f)
+//   end associate
+// contains
+//   elemental type(t) function callee(x)
+//     real, intent(in) :: x
+//     callee = t(x)
+//   end function callee
+// end subroutine test
+// RUN: fir-opt --fir-add-alias-tags %s | FileCheck %s
+
+func.func @_QPtest() {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.alloca !fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>> {bindc_name = ".tmp.array"}
+  %1 = fir.convert %0 : (!fir.ref<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>) -> !fir.heap<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>
+  %2 = fir.alloca f32 {bindc_name = ".tmp"}
+  %3 = fir.alloca !fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}> {bindc_name = ".result"}
+  %4 = fir.dummy_scope : !fir.dscope
+  %14 = fir.address_of(@_QQro.1xr4.0) : !fir.ref<!fir.array<1xf32>>
+  %15 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %16 = fir.declare %14(%15) {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQro.1xr4.0"} : (!fir.ref<!fir.array<1xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<1xf32>>
+  %17 = fir.declare %1(%15) {uniq_name = ".tmp.array"} : (!fir.heap<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>, !fir.shape<1>) -> !fir.heap<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>
+  fir.do_loop %arg0 = %c1 to %c1 step %c1 unordered {
+    %27 = fir.array_coor %16(%15) %arg0 : (!fir.ref<!fir.array<1xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+    %28 = fir.declare %2 {uniq_name = ".tmp"} : (!fir.ref<f32>) -> !fir.ref<f32>
+// CHECK: fir.load %{{.*}} {tbaa = {{.*}}} : !fir.ref<f32>
+    %29 = fir.load %27 : !fir.ref<f32>
+// CHECK: fir.store %{{.*}} {tbaa = {{.*}}} : !fir.ref<f32>
+    fir.store %29 to %28 : !fir.ref<f32>
+    %30 = fir.call @_QFtestPcallee(%28) proc_attrs<elemental, pure> fastmath<fast> : (!fir.ref<f32>) -> !fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>
+    fir.save_result %30 to %3 : !fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>, !fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>
+    %31 = fir.declare %3 {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>) -> !fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>
+    %32 = fir.array_coor %17(%15) %arg0 : (!fir.heap<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>, !fir.shape<1>, index) -> !fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>
+// CHECK: fir.load %{{[0-9]+}} : !fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>
+    %33 = fir.load %31 : !fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>
+// CHECK: fir.store %{{.*}} to %{{[0-9]+}} : !fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>
+    fir.store %33 to %32 : !fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>
+  }
+  %18 = fir.convert %17 : (!fir.heap<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>) -> !fir.ref<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>
+  %19 = fir.declare %18(%15) {uniq_name = "_QFtestEa"} : (!fir.ref<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>, !fir.shape<1>) -> !fir.ref<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>
+  %20 = fir.array_coor %19(%15) %c1 : (!fir.ref<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>, !fir.shape<1>, index) -> !fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>
+  %21 = fir.coordinate_of %20, f : (!fir.ref<!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>) -> !fir.ref<!fir.box<!fir.heap<f32>>>
+// CHECK: fir.load %{{[0-9]+}} : !fir.ref<!fir.box<!fir.heap<f32>>>
+  %22 = fir.load %21 : !fir.ref<!fir.box<!fir.heap<f32>>>
+  %23 = fir.box_addr %22 : (!fir.box<!fir.heap<f32>>) -> !fir.heap<f32>
+  %24 = fir.convert %23 : (!fir.heap<f32>) -> !fir.ref<f32>
+  fir.call @_QPext(%24) fastmath<fast> : (!fir.ref<f32>) -> ()
+  %25 = fir.embox %17(%15) : (!fir.heap<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>>
+  %26 = fir.convert %25 : (!fir.box<!fir.heap<!fir.array<1x!fir.type<_QFtestTt{f:!fir.box<!fir.heap<f32>>}>>>>) -> !fir.box<none>
+  fir.call @_FortranADestroyWithoutFinalization(%26) : (!fir.box<none>) -> ()
+  return
+}


        

From flang-commits at lists.llvm.org  Mon Jul 14 09:41:21 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Mon, 14 Jul 2025 09:41:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed TBAA tags for derived types
 with descriptors. (PR #148093)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148093@github.com>
Message-ID: <68753331.170a0220.11083c.8f2c@mx.google.com>


https://github.com/vzakhari closed https://github.com/llvm/llvm-project/pull/148093

From flang-commits at lists.llvm.org  Mon Jul 14 09:41:49 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Mon, 14 Jul 2025 09:41:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147881@github.com>
Message-ID: <6875334d.170a0220.2a6e5e.9741@mx.google.com>


https://github.com/vzakhari closed https://github.com/llvm/llvm-project/pull/147881

From flang-commits at lists.llvm.org  Mon Jul 14 09:41:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 09:41:46 -0700 (PDT)
Subject: [flang-commits] [flang] 4775b96 - [flang] Optimize redundant array
	repacking. (#147881)
Message-ID: <6875334a.170a0220.3c6aa8.9099@mx.google.com>


Author: Slava Zakharin
Date: 2025-07-14T09:41:42-07:00
New Revision: 4775b9689844e1ec7b425e039aff4fcbb758b4d4

URL: https://github.com/llvm/llvm-project/commit/4775b9689844e1ec7b425e039aff4fcbb758b4d4
DIFF: https://github.com/llvm/llvm-project/commit/4775b9689844e1ec7b425e039aff4fcbb758b4d4.diff

LOG: [flang] Optimize redundant array repacking. (#147881)

This patch allows optimizing redundant array repacking, when
the source array is statically known to be contiguous.
This is part of the implementation plan for the array repacking
feature, though, it does not affect any real life use case
as long as FIR inlining is not a thing. I experimented with
simple cases of FIR inling using `-inline-all`, and I recorded
these cases in optimize-array-repacking.fir tests.

Added: 
    flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
    flang/test/Transforms/optimize-array-repacking.fir

Modified: 
    flang/include/flang/Optimizer/Builder/HLFIRTools.h
    flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
    flang/include/flang/Optimizer/Transforms/Passes.h
    flang/include/flang/Optimizer/Transforms/Passes.td
    flang/lib/Optimizer/Builder/HLFIRTools.cpp
    flang/lib/Optimizer/Dialect/FIROps.cpp
    flang/lib/Optimizer/Passes/Pipelines.cpp
    flang/lib/Optimizer/Transforms/CMakeLists.txt
    flang/lib/Optimizer/Transforms/LoopVersioning.cpp
    flang/test/Driver/bbc-mlir-pass-pipeline.f90
    flang/test/Driver/mlir-pass-pipeline.f90
    flang/test/Fir/basic-program.fir
    flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
    flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 2cbad6e268a38..49dfc85dc76e6 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -542,6 +542,14 @@ Entity gen1DSection(mlir::Location loc, fir::FirOpBuilder &builder,
 /// contiguous.
 bool designatePreservesContinuity(hlfir::DesignateOp op);
 
+/// Return true iff the given \p base desribes an object
+/// that is contiguous. If \p checkWhole is true, then
+/// the object must be contiguous in all dimensions,
+/// otherwise, it must be contiguous in the innermost dimension.
+/// This function is an extension of hlfir::Entity::isSimplyContiguous(),
+/// and it can be used on pure FIR representation as well as on HLFIR.
+bool isSimplyContiguous(mlir::Value base, bool checkWhole = true);
+
 } // namespace hlfir
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_HLFIRTOOLS_H

diff  --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index 0a2337be7455e..ae471eb0d2e04 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -238,7 +238,19 @@ std::optional<int64_t> getAllocaByteSize(fir::AllocaOp alloca,
 /// When \p checkWhole is false, then the checking is only done
 /// for continuity in the innermost dimension, otherwise,
 /// the checking is done for continuity of the whole result of rebox.
-bool reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole = true);
+/// The caller may specify \p mayHaveNonDefaultLowerBounds, if it is known,
+/// to allow better handling of the rebox operations representing
+/// full array slices.
+bool reboxPreservesContinuity(fir::ReboxOp rebox,
+                              bool mayHaveNonDefaultLowerBounds = true,
+                              bool checkWhole = true);
+
+/// Return true, if \p embox operation produces a contiguous
+/// entity.
+/// When \p checkWhole is false, then the checking is only done
+/// for continuity in the innermost dimension, otherwise,
+/// the checking is done for continuity of the whole result of embox
+bool isContiguousEmbox(fir::EmboxOp embox, bool checkWhole = true);
 
 } // namespace fir
 

diff  --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index dc8a5b9141ad2..6f5dff4687cbb 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -31,39 +31,7 @@ namespace fir {
 // Passes defined in Passes.td
 //===----------------------------------------------------------------------===//
 
-#define GEN_PASS_DECL_ABSTRACTRESULTOPT
-#define GEN_PASS_DECL_AFFINEDIALECTPROMOTION
-#define GEN_PASS_DECL_AFFINEDIALECTDEMOTION
-#define GEN_PASS_DECL_ANNOTATECONSTANTOPERANDS
-#define GEN_PASS_DECL_ARRAYVALUECOPY
-#define GEN_PASS_DECL_ASSUMEDRANKOPCONVERSION
-#define GEN_PASS_DECL_CHARACTERCONVERSION
-#define GEN_PASS_DECL_CFGCONVERSION
-#define GEN_PASS_DECL_CUFADDCONSTRUCTOR
-#define GEN_PASS_DECL_CUFDEVICEGLOBAL
-#define GEN_PASS_DECL_CUFGPUTOLLVMCONVERSION
-#define GEN_PASS_DECL_CUFOPCONVERSION
-#define GEN_PASS_DECL_CUFCOMPUTESHAREDMEMORYOFFSETSANDSIZE
-#define GEN_PASS_DECL_EXTERNALNAMECONVERSION
-#define GEN_PASS_DECL_MEMREFDATAFLOWOPT
-#define GEN_PASS_DECL_SIMPLIFYINTRINSICS
-#define GEN_PASS_DECL_MEMORYALLOCATIONOPT
-#define GEN_PASS_DECL_SIMPLIFYREGIONLITE
-#define GEN_PASS_DECL_ALGEBRAICSIMPLIFICATION
-#define GEN_PASS_DECL_POLYMORPHICOPCONVERSION
-#define GEN_PASS_DECL_OPENACCDATAOPERANDCONVERSION
-#define GEN_PASS_DECL_ADDDEBUGINFO
-#define GEN_PASS_DECL_STACKARRAYS
-#define GEN_PASS_DECL_STACKRECLAIM
-#define GEN_PASS_DECL_LOOPVERSIONING
-#define GEN_PASS_DECL_ADDALIASTAGS
-#define GEN_PASS_DECL_VSCALEATTR
-#define GEN_PASS_DECL_FUNCTIONATTR
-#define GEN_PASS_DECL_CONSTANTARGUMENTGLOBALISATIONOPT
-#define GEN_PASS_DECL_COMPILERGENERATEDNAMESCONVERSION
-#define GEN_PASS_DECL_SETRUNTIMECALLATTRIBUTES
-#define GEN_PASS_DECL_GENRUNTIMECALLSFORTEST
-#define GEN_PASS_DECL_SIMPLIFYFIROPERATIONS
+#define GEN_PASS_DECL
 
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 

diff  --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 34842f9785942..b230f60b4c59e 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -551,4 +551,14 @@ def SimplifyFIROperations : Pass<"simplify-fir-operations", "mlir::ModuleOp"> {
       "Prefer expanding without using Fortran runtime calls.">];
 }
 
+def OptimizeArrayRepacking
+    : Pass<"optimize-array-repacking", "mlir::func::FuncOp"> {
+  let summary = "Optimizes redundant array repacking operations";
+  let description = [{
+    If the source of fir.pack_array is known to be contiguous,
+    then this pass erases such operations. The corresponding
+    fir.unpack_array operations are also removed.
+  }];
+}
+
 #endif // FLANG_OPTIMIZER_TRANSFORMS_PASSES

diff  --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index b54cccbef14fa..e59a6bf2bf224 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -212,10 +212,17 @@ bool hlfir::Entity::mayHaveNonDefaultLowerBounds() const {
   if (auto varIface = getIfVariableInterface())
     return isShapeWithLowerBounds(varIface.getShape());
   // Go through chain of fir.box converts.
-  if (auto convert = getDefiningOp<fir::ConvertOp>())
+  if (auto convert = getDefiningOp<fir::ConvertOp>()) {
     return hlfir::Entity{convert.getValue()}.mayHaveNonDefaultLowerBounds();
-  // TODO: Embox and Rebox do not have hlfir variable interface, but are
-  // easy to reason about.
+  } else if (auto rebox = getDefiningOp<fir::ReboxOp>()) {
+    // If slicing is involved, then the resulting box has
+    // default lower bounds. If there is no slicing,
+    // then the result depends on the shape operand
+    // (whether it has non default lower bounds or not).
+    return !rebox.getSlice() && isShapeWithLowerBounds(rebox.getShape());
+  } else if (auto embox = getDefiningOp<fir::EmboxOp>()) {
+    return !embox.getSlice() && isShapeWithLowerBounds(embox.getShape());
+  }
   return true;
 }
 
@@ -1646,3 +1653,30 @@ bool hlfir::designatePreservesContinuity(hlfir::DesignateOp op) {
   }
   return true;
 }
+
+bool hlfir::isSimplyContiguous(mlir::Value base, bool checkWhole) {
+  hlfir::Entity entity{base};
+  if (entity.isSimplyContiguous())
+    return true;
+
+  // Look at the definition.
+  mlir::Operation *def = base.getDefiningOp();
+  if (!def)
+    return false;
+
+  return mlir::TypeSwitch<mlir::Operation *, bool>(def)
+      .Case<fir::EmboxOp>(
+          [&](auto op) { return fir::isContiguousEmbox(op, checkWhole); })
+      .Case<fir::ReboxOp>([&](auto op) {
+        hlfir::Entity box{op.getBox()};
+        return fir::reboxPreservesContinuity(
+                   op, box.mayHaveNonDefaultLowerBounds(), checkWhole) &&
+               isSimplyContiguous(box, checkWhole);
+      })
+      .Case<fir::DeclareOp, hlfir::DeclareOp>([&](auto op) {
+        return isSimplyContiguous(op.getMemref(), checkWhole);
+      })
+      .Case<fir::ConvertOp>(
+          [&](auto op) { return isSimplyContiguous(op.getValue()); })
+      .Default([](auto &&) { return false; });
+}

diff  --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 6b40e7015fdd8..b6bf2753b80ce 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -1944,6 +1944,128 @@ llvm::LogicalResult fir::EmboxOp::verify() {
   return mlir::success();
 }
 
+/// Returns true if \p extent matches the extent of the \p box's
+/// dimension \p dim.
+static bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
+  if (auto op = extent.getDefiningOp<fir::BoxDimsOp>())
+    if (op.getVal() == box && op.getExtent() == extent)
+      if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
+        return *dimOperand == dim;
+  return false;
+}
+
+/// Returns true if \p lb matches the lower bound of the \p box's
+/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
+/// then \p lb may be an integer constant 1.
+static bool isBoxLb(mlir::Value box, std::int64_t dim, mlir::Value lb,
+                    bool mayHaveNonDefaultLowerBounds = true) {
+  if (auto op = lb.getDefiningOp<fir::BoxDimsOp>()) {
+    if (op.getVal() == box && op.getLowerBound() == lb)
+      if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
+        return *dimOperand == dim;
+  } else if (!mayHaveNonDefaultLowerBounds) {
+    if (auto constantLb = fir::getIntIfConstant(lb))
+      return *constantLb == 1;
+  }
+  return false;
+}
+
+/// Returns true if \p ub matches the upper bound of the \p box's
+/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
+/// then the dimension's lower bound may be an integer constant 1.
+/// Note that the upper bound is usually a result of computation
+/// involving the lower bound and the extent, and the function
+/// tries its best to recognize the computation pattern.
+/// The conservative result 'false' does not necessarily mean
+/// that \p ub is not an actual upper bound value.
+static bool isBoxUb(mlir::Value box, std::int64_t dim, mlir::Value ub,
+                    bool mayHaveNonDefaultLowerBounds = true) {
+  if (auto sub1 = ub.getDefiningOp<mlir::arith::SubIOp>()) {
+    auto one = fir::getIntIfConstant(sub1.getOperand(1));
+    if (!one || *one != 1)
+      return false;
+    if (auto add = sub1.getOperand(0).getDefiningOp<mlir::arith::AddIOp>())
+      if ((isBoxLb(box, dim, add.getOperand(0)) &&
+           isBoxExtent(box, dim, add.getOperand(1))) ||
+          (isBoxLb(box, dim, add.getOperand(1)) &&
+           isBoxExtent(box, dim, add.getOperand(0))))
+        return true;
+  } else if (!mayHaveNonDefaultLowerBounds) {
+    return isBoxExtent(box, dim, ub);
+  }
+  return false;
+}
+
+/// Checks if the given \p sliceOp specifies a contiguous
+/// array slice. If \p checkWhole is true, then the check
+/// is done for all dimensions, otherwise, only for the innermost
+/// dimension.
+/// The simplest way to prove that this is an contiguous slice
+/// is to check whether the slice stride(s) is 1.
+/// For more complex cases, extra information must be provided
+/// by the caller:
+///   * \p origBox - if not null, then the source array is represented
+///     with this !fir.box value. The box is used to recognize
+///     the full dimension slices, which are specified by the triplets
+///     computed from the dimensions' lower bounds and extents.
+///   * \p mayHaveNonDefaultLowerBounds may be set to false to indicate
+///     that the source entity has default lower bounds, so the full
+///     dimension slices computations may use 1 for the lower bound.
+static bool isContiguousArraySlice(fir::SliceOp sliceOp, bool checkWhole = true,
+                                   mlir::Value origBox = nullptr,
+                                   bool mayHaveNonDefaultLowerBounds = true) {
+  if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
+    // TODO: generalize code for the triples analysis with
+    // hlfir::designatePreservesContinuity, especially when
+    // recognition of the whole dimension slices is added.
+    auto triples = sliceOp.getTriples();
+    assert((triples.size() % 3) == 0 && "invalid triples size");
+
+    // A slice with step=1 in the innermost dimension preserves
+    // the continuity of the array in the innermost dimension.
+    // If checkWhole is false, then check only the innermost slice triples.
+    std::size_t checkUpTo = checkWhole ? triples.size() : 3;
+    checkUpTo = std::min(checkUpTo, triples.size());
+    for (std::size_t i = 0; i < checkUpTo; i += 3) {
+      if (triples[i] != triples[i + 1]) {
+        // This is a section of the dimension. Only allow it
+        // to be the first triple, if the source of the slice
+        // is a boxed array. If it is a raw pointer, then
+        // the result will still be contiguous, as long as
+        // the strides are all ones.
+        // When origBox is not null, we must prove that the triple
+        // covers the whole dimension and the stride is one,
+        // before claiming contiguity for this dimension.
+        if (i != 0 && origBox) {
+          std::int64_t dim = i / 3;
+          if (!isBoxLb(origBox, dim, triples[i],
+                       mayHaveNonDefaultLowerBounds) ||
+              !isBoxUb(origBox, dim, triples[i + 1],
+                       mayHaveNonDefaultLowerBounds))
+            return false;
+        }
+        auto constantStep = fir::getIntIfConstant(triples[i + 2]);
+        if (!constantStep || *constantStep != 1)
+          return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+bool fir::isContiguousEmbox(fir::EmboxOp embox, bool checkWhole) {
+  auto sliceArg = embox.getSlice();
+  if (!sliceArg)
+    return true;
+
+  if (auto sliceOp =
+          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
+    return isContiguousArraySlice(sliceOp, checkWhole);
+
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // EmboxCharOp
 //===----------------------------------------------------------------------===//
@@ -4794,7 +4916,9 @@ mlir::Type fir::applyPathToType(mlir::Type eleTy, mlir::ValueRange path) {
   return eleTy;
 }
 
-bool fir::reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole) {
+bool fir::reboxPreservesContinuity(fir::ReboxOp rebox,
+                                   bool mayHaveNonDefaultLowerBounds,
+                                   bool checkWhole) {
   // If slicing is not involved, then the rebox does not affect
   // the continuity of the array.
   auto sliceArg = rebox.getSlice();
@@ -4802,33 +4926,10 @@ bool fir::reboxPreservesContinuity(fir::ReboxOp rebox, bool checkWhole) {
     return true;
 
   if (auto sliceOp =
-          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp())) {
-    if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
-      // TODO: generalize code for the triples analysis with
-      // hlfir::designatePreservesContinuity, especially when
-      // recognition of the whole dimension slices is added.
-      auto triples = sliceOp.getTriples();
-      assert((triples.size() % 3) == 0 && "invalid triples size");
-
-      // A slice with step=1 in the innermost dimension preserves
-      // the continuity of the array in the innermost dimension.
-      // If checkWhole is false, then check only the innermost slice triples.
-      std::size_t checkUpTo = checkWhole ? triples.size() : 3;
-      checkUpTo = std::min(checkUpTo, triples.size());
-      for (std::size_t i = 0; i < checkUpTo; i += 3) {
-        if (triples[i] != triples[i + 1]) {
-          // This is a section of the dimension. Only allow it
-          // to be the first triple.
-          if (i != 0)
-            return false;
-          auto constantStep = fir::getIntIfConstant(triples[i + 2]);
-          if (!constantStep || *constantStep != 1)
-            return false;
-        }
-      }
-      return true;
-    }
-  }
+          mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
+    return isContiguousArraySlice(sliceOp, checkWhole, rebox.getBox(),
+                                  mayHaveNonDefaultLowerBounds);
+
   return false;
 }
 

diff  --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 333bfc8b7aed8..ca8e820608688 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -207,6 +207,10 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
   pm.addPass(fir::createPolymorphicOpConversion());
   pm.addPass(fir::createAssumedRankOpConversion());
 
+  // Optimize redundant array repacking operations,
+  // if the source is known to be contiguous.
+  if (pc.OptLevel.isOptimizingForSpeed())
+    pm.addPass(fir::createOptimizeArrayRepacking());
   pm.addPass(fir::createLowerRepackArraysPass());
   // Expand FIR operations that may use SCF dialect for their
   // implementation. This is a mandatory pass.

diff  --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index 846d6c64dbd04..a8812e08c1ccd 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -34,6 +34,7 @@ add_flang_library(FIRTransforms
   SetRuntimeCallAttributes.cpp
   GenRuntimeCallsForTest.cpp
   SimplifyFIROperations.cpp
+  OptimizeArrayRepacking.cpp
 
   DEPENDS
   CUFAttrs

diff  --git a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
index 50e7ee5599ab1..056bdf63d914f 100644
--- a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
@@ -214,7 +214,9 @@ static mlir::Value unwrapPassThroughOps(mlir::Value val) {
 /// of the value, otherwise return the value
 static mlir::Value unwrapReboxOp(mlir::Value val) {
   while (fir::ReboxOp rebox = val.getDefiningOp<fir::ReboxOp>()) {
-    if (!fir::reboxPreservesContinuity(rebox, /*checkWhole=*/false)) {
+    if (!fir::reboxPreservesContinuity(rebox,
+                                       /*mayHaveNonDefaultLowerBounds=*/true,
+                                       /*checkWhole=*/false)) {
       LLVM_DEBUG(llvm::dbgs() << "REBOX may produce non-contiguous array: "
                               << rebox << '\n');
       break;

diff  --git a/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
new file mode 100644
index 0000000000000..1688f2887a57a
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
@@ -0,0 +1,90 @@
+//===- OptimizeArrayRepacking.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+/// \file
+/// This pass removes redundant fir.pack_array operations, if it can prove
+/// that the source array is contiguous. In this case, it relink all uses
+/// of fir.pack_array result to the source. If such a rewrite happens,
+/// it may turn the using fir.unpack_array operation into one with the same
+/// temp and original operands - these are also removed as redundant.
+//===----------------------------------------------------------------------===//
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Support/Utils.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+namespace fir {
+#define GEN_PASS_DEF_OPTIMIZEARRAYREPACKING
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+} // namespace fir
+
+namespace {
+class OptimizeArrayRepackingPass
+    : public fir::impl::OptimizeArrayRepackingBase<OptimizeArrayRepackingPass> {
+public:
+  void runOnOperation() override;
+};
+
+/// Relinks all uses of redundant fir.pack_array to the source.
+class PackingOfContiguous : public mlir::OpRewritePattern<fir::PackArrayOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  mlir::LogicalResult matchAndRewrite(fir::PackArrayOp,
+                                      mlir::PatternRewriter &) const override;
+};
+
+/// Erases fir.unpack_array with have the matching temp and original
+/// operands.
+class NoopUnpacking : public mlir::OpRewritePattern<fir::UnpackArrayOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  mlir::LogicalResult matchAndRewrite(fir::UnpackArrayOp,
+                                      mlir::PatternRewriter &) const override;
+};
+} // namespace
+
+mlir::LogicalResult
+PackingOfContiguous::matchAndRewrite(fir::PackArrayOp op,
+                                     mlir::PatternRewriter &rewriter) const {
+  mlir::Value box = op.getArray();
+  if (hlfir::isSimplyContiguous(box, !op.getInnermost())) {
+    rewriter.replaceAllUsesWith(op, box);
+    rewriter.eraseOp(op);
+    return mlir::success();
+  }
+  return mlir::failure();
+}
+
+mlir::LogicalResult
+NoopUnpacking::matchAndRewrite(fir::UnpackArrayOp op,
+                               mlir::PatternRewriter &rewriter) const {
+  if (op.getTemp() == op.getOriginal()) {
+    rewriter.eraseOp(op);
+    return mlir::success();
+  }
+  return mlir::failure();
+}
+
+void OptimizeArrayRepackingPass::runOnOperation() {
+  mlir::func::FuncOp funcOp = getOperation();
+  mlir::MLIRContext *context = &getContext();
+  mlir::RewritePatternSet patterns(context);
+  mlir::GreedyRewriteConfig config;
+  config.setRegionSimplificationLevel(
+      mlir::GreedySimplifyRegionLevel::Disabled);
+  patterns.insert<PackingOfContiguous>(context);
+  patterns.insert<NoopUnpacking>(context);
+  if (mlir::failed(
+          mlir::applyPatternsGreedily(funcOp, std::move(patterns), config))) {
+    mlir::emitError(funcOp.getLoc(), "failure in array repacking optimization");
+    signalPassFailure();
+  }
+}

diff  --git a/flang/test/Driver/bbc-mlir-pass-pipeline.f90 b/flang/test/Driver/bbc-mlir-pass-pipeline.f90
index 137c19608c38f..f3791fe9f8dc3 100644
--- a/flang/test/Driver/bbc-mlir-pass-pipeline.f90
+++ b/flang/test/Driver/bbc-mlir-pass-pipeline.f90
@@ -47,6 +47,8 @@
 
 ! CHECK-NEXT: PolymorphicOpConversion
 ! CHECK-NEXT: AssumedRankOpConversion
+! CHECK-NEXT: 'func.func' Pipeline
+! CHECK-NEXT:   OptimizeArrayRepacking
 ! CHECK-NEXT: LowerRepackArraysPass
 ! CHECK-NEXT: SimplifyFIROperations
 

diff  --git a/flang/test/Driver/mlir-pass-pipeline.f90 b/flang/test/Driver/mlir-pass-pipeline.f90
index 99f192ce7aec2..0bcd055a84b87 100644
--- a/flang/test/Driver/mlir-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-pass-pipeline.f90
@@ -105,6 +105,8 @@
 
 ! ALL-NEXT: PolymorphicOpConversion
 ! ALL-NEXT: AssumedRankOpConversion
+! O2-NEXT:  'func.func' Pipeline
+! O2-NEXT:    OptimizeArrayRepacking
 ! ALL-NEXT: LowerRepackArraysPass
 ! ALL-NEXT: SimplifyFIROperations
 ! O2-NEXT:  AddAliasTags

diff  --git a/flang/test/Fir/basic-program.fir b/flang/test/Fir/basic-program.fir
index 7ac8b92f48953..c9fe53bf093a1 100644
--- a/flang/test/Fir/basic-program.fir
+++ b/flang/test/Fir/basic-program.fir
@@ -103,6 +103,8 @@ func.func @_QQmain() {
 
 // PASSES-NEXT: PolymorphicOpConversion
 // PASSES-NEXT: AssumedRankOpConversion
+// PASSES-NEXT: 'func.func' Pipeline
+// PASSES-NEXT:   OptimizeArrayRepacking
 // PASSES-NEXT: LowerRepackArraysPass
 // PASSES-NEXT: SimplifyFIROperations
 // PASSES-NEXT: AddAliasTags

diff  --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
index 35530c66f4038..8684a429ea5b4 100644
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
@@ -41,19 +41,11 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_17]] : (i64) -> index
 // CHECK:               fir.do_loop %[[VAL_28:.*]] = %[[VAL_2]] to %[[VAL_27]] step %[[VAL_2]] unordered {
 // CHECK:                 %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (index) -> i64
-// CHECK:                 %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_31:.*]] = fir.convert %[[VAL_29]] : (i64) -> index
-// CHECK:                 %[[VAL_32:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_33:.*]] = arith.addi %[[VAL_31]], %[[VAL_32]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_34:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_33]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_34:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_29]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_35:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
 // CHECK:                 %[[VAL_36:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_37:.*]] = arith.addi %[[VAL_29]], %[[VAL_36]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (i64) -> index
-// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_39]], %[[VAL_40]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_42:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_41]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_42:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_37]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_35]] to %[[VAL_42]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:               %[[VAL_43:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
@@ -61,17 +53,9 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:               fir.do_loop %[[VAL_45:.*]] = %[[VAL_2]] to %[[VAL_44]] step %[[VAL_2]] unordered {
 // CHECK:                 %[[VAL_46:.*]] = fir.convert %[[VAL_45]] : (index) -> i64
 // CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_17]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_48:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_49:.*]] = fir.convert %[[VAL_47]] : (i64) -> index
-// CHECK:                 %[[VAL_50:.*]] = arith.subi %[[VAL_48]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_51:.*]] = arith.addi %[[VAL_49]], %[[VAL_50]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_52:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_51]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_52:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_47]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_53:.*]] = fir.load %[[VAL_52]] : !fir.ref<i32>
-// CHECK:                 %[[VAL_54:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_55:.*]] = fir.convert %[[VAL_46]] : (i64) -> index
-// CHECK:                 %[[VAL_56:.*]] = arith.subi %[[VAL_54]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_57:.*]] = arith.addi %[[VAL_55]], %[[VAL_56]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_58:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_57]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_58:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_46]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_53]] to %[[VAL_58]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:             } else {
@@ -86,11 +70,7 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:                 %[[VAL_67:.*]] = fir.load %[[VAL_66]] : !fir.ref<i32>
 // CHECK:                 %[[VAL_68:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_69:.*]] = arith.addi %[[VAL_61]], %[[VAL_68]] overflow<nsw, nuw> : i64
-// CHECK:                 %[[VAL_70:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_71:.*]] = fir.convert %[[VAL_69]] : (i64) -> index
-// CHECK:                 %[[VAL_72:.*]] = arith.subi %[[VAL_70]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_73:.*]] = arith.addi %[[VAL_71]], %[[VAL_72]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_74:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_73]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_74:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_69]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_67]] to %[[VAL_74]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:               %[[VAL_75:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
@@ -104,11 +84,7 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:                 %[[VAL_83:.*]] = arith.addi %[[VAL_81]], %[[VAL_82]] overflow<nsw, nuw> : index
 // CHECK:                 %[[VAL_84:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_83]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_85:.*]] = fir.load %[[VAL_84]] : !fir.ref<i32>
-// CHECK:                 %[[VAL_86:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_87:.*]] = fir.convert %[[VAL_78]] : (i64) -> index
-// CHECK:                 %[[VAL_88:.*]] = arith.subi %[[VAL_86]]#0, %[[VAL_2]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_89:.*]] = arith.addi %[[VAL_87]], %[[VAL_88]] overflow<nsw, nuw> : index
-// CHECK:                 %[[VAL_90:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_89]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:                 %[[VAL_90:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_78]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_85]] to %[[VAL_90]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:             }

diff  --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
index d29e9a26c20ba..73e6e5c2421ff 100644
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-matmul.fir
@@ -13,7 +13,6 @@ func.func @matmul_matrix_matrix_integer(%arg0: !hlfir.expr<?x?xi16>, %arg1: !hlf
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0 : i32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi16>) -> !fir.shape<2>
@@ -27,26 +26,14 @@ func.func @matmul_matrix_matrix_integer(%arg0: !hlfir.expr<?x?xi16>, %arg1: !hlf
 // ANSE:             %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xi32>>
 // ANSE:             fir.do_loop %[[VAL_14:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_19:.*]] = arith.addi %[[VAL_15]], %[[VAL_18]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_14]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_15]], %[[VAL_14]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
 // ANSE:                 hlfir.assign %[[VAL_4]] to %[[VAL_22]] : i32, !fir.ref<i32>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_23:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:                 fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                   %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]] = arith.subi %[[VAL_26]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_29:.*]] = arith.addi %[[VAL_25]], %[[VAL_28]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.addi %[[VAL_24]], %[[VAL_30]] : index
-// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_25]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
 // ANSE:                   %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
 // ANSE:                   %[[VAL_34:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_25]], %[[VAL_23]] : (!hlfir.expr<?x?xi16>, index, index) -> i16
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_23]], %[[VAL_24]] : (!hlfir.expr<?x?xi32>, index, index) -> i32
@@ -94,7 +81,6 @@ func.func @matmul_matrix_matrix_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
@@ -108,26 +94,14 @@ func.func @matmul_matrix_matrix_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 // ANSE:             %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
 // ANSE:             fir.do_loop %[[VAL_14:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_19:.*]] = arith.addi %[[VAL_15]], %[[VAL_18]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_14]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_15]], %[[VAL_14]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
 // ANSE:                 hlfir.assign %[[VAL_4]] to %[[VAL_22]] : f32, !fir.ref<f32>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_23:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]] = arith.subi %[[VAL_26]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_29:.*]] = arith.addi %[[VAL_25]], %[[VAL_28]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.addi %[[VAL_24]], %[[VAL_30]] : index
-// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+// ANSE:                   %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_25]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
 // ANSE:                   %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<f32>
 // ANSE:                   %[[VAL_34:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_25]], %[[VAL_23]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_23]], %[[VAL_24]] : (!hlfir.expr<?x?xf16>, index, index) -> f16
@@ -175,7 +149,6 @@ func.func @matmul_matrix_matrix_complex(%arg0: !hlfir.expr<?x?xcomplex<f32>>, %a
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>
@@ -192,26 +165,14 @@ func.func @matmul_matrix_matrix_complex(%arg0: !hlfir.expr<?x?xcomplex<f32>>, %a
 // ANSE:             %[[VAL_16:.*]] = fir.insert_value %[[VAL_15]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_18]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_24:.*]] = arith.addi %[[VAL_17]], %[[VAL_23]] : index
-// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_18]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                 hlfir.assign %[[VAL_16]] to %[[VAL_25]] : complex<f32>, !fir.ref<complex<f32>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_27:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_28:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_29:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_29]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_28]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_34:.*]] = arith.addi %[[VAL_27]], %[[VAL_33]] : index
-// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_32]], %[[VAL_34]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_28]], %[[VAL_27]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_37:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_28]], %[[VAL_26]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>
 // ANSE:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_26]], %[[VAL_27]] : (!hlfir.expr<?x?xcomplex<f16>>, index, index) -> complex<f16>
@@ -262,7 +223,6 @@ func.func @matmul_matrix_matrix_complex_real(%arg0: !hlfir.expr<?x?xcomplex<f32>
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>
@@ -279,26 +239,14 @@ func.func @matmul_matrix_matrix_complex_real(%arg0: !hlfir.expr<?x?xcomplex<f32>
 // ANSE:             %[[VAL_16:.*]] = fir.insert_value %[[VAL_15]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_18]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_24:.*]] = arith.addi %[[VAL_17]], %[[VAL_23]] : index
-// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_18]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                 hlfir.assign %[[VAL_16]] to %[[VAL_25]] : complex<f32>, !fir.ref<complex<f32>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_27:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {
 // ANSE:                 fir.do_loop %[[VAL_28:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                   %[[VAL_29:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_30:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_29]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_28]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = arith.subi %[[VAL_30]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_34:.*]] = arith.addi %[[VAL_27]], %[[VAL_33]] : index
-// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_32]], %[[VAL_34]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
+// ANSE:                   %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_28]], %[[VAL_27]])  : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f32>>
 // ANSE:                   %[[VAL_37:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_28]], %[[VAL_26]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>
 // ANSE:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_26]], %[[VAL_27]] : (!hlfir.expr<?x?xf16>, index, index) -> f16
@@ -357,7 +305,6 @@ func.func @matmul_matrix_matrix_logical(%arg0: !hlfir.expr<?x?x!fir.logical<1>>,
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant false
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<1>>) -> !fir.shape<2>
@@ -372,26 +319,14 @@ func.func @matmul_matrix_matrix_logical(%arg0: !hlfir.expr<?x?x!fir.logical<1>>,
 // ANSE:             %[[VAL_14:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>
 // ANSE:             fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_16:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_20:.*]] = arith.addi %[[VAL_16]], %[[VAL_19]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_15]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_16]], %[[VAL_15]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
 // ANSE:                 hlfir.assign %[[VAL_14]] to %[[VAL_23]] : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 // ANSE:               }
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] unordered {
 // ANSE:               fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {
 // ANSE:                 fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:                   %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_13]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// ANSE:                   %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_30:.*]] = arith.addi %[[VAL_26]], %[[VAL_29]] : index
-// ANSE:                   %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_3]] : index
-// ANSE:                   %[[VAL_32:.*]] = arith.addi %[[VAL_25]], %[[VAL_31]] : index
-// ANSE:                   %[[VAL_33:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// ANSE:                   %[[VAL_33:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_26]], %[[VAL_25]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
 // ANSE:                   %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<!fir.logical<4>>
 // ANSE:                   %[[VAL_35:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_26]], %[[VAL_24]] : (!hlfir.expr<?x?x!fir.logical<1>>, index, index) -> !fir.logical<1>
 // ANSE:                   %[[VAL_36:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_24]], %[[VAL_25]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
@@ -446,7 +381,6 @@ func.func @matmul_matrix_vector_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
@@ -457,18 +391,12 @@ func.func @matmul_matrix_vector_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.
 // ANSE:           ^bb0(%[[VAL_10:.*]]: !fir.ref<!fir.array<?xf32>>):
 // ANSE:             %[[VAL_11:.*]] = fir.embox %[[VAL_10]](%[[VAL_8]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 // ANSE:             fir.do_loop %[[VAL_12:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {
-// ANSE:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_11]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:               %[[VAL_14:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_3]] : index
-// ANSE:               %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_14]] : index
-// ANSE:               %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_15]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:               %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_12]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:               hlfir.assign %[[VAL_4]] to %[[VAL_16]] : f32, !fir.ref<f32>
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
-// ANSE:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_11]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_20:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_21:.*]] = arith.addi %[[VAL_18]], %[[VAL_20]] : index
-// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_21]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_18]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<f32>
 // ANSE:                 %[[VAL_24:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]], %[[VAL_17]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
 // ANSE:                 %[[VAL_25:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_17]] : (!hlfir.expr<?xf16>, index) -> f16
@@ -513,7 +441,6 @@ func.func @matmul_vector_matrix_real(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.ex
 
 // NOANSE: hlfir.matmul
 
-// ANSE:           %[[VAL_2:.*]] = arith.constant 0 : index
 // ANSE:           %[[VAL_3:.*]] = arith.constant 1 : index
 // ANSE:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 // ANSE:           %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
@@ -525,18 +452,12 @@ func.func @matmul_vector_matrix_real(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.ex
 // ANSE:           ^bb0(%[[VAL_11:.*]]: !fir.ref<!fir.array<?xf32>>):
 // ANSE:             %[[VAL_12:.*]] = fir.embox %[[VAL_11]](%[[VAL_9]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 // ANSE:             fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] unordered {
-// ANSE:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_12]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:               %[[VAL_15:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_3]] : index
-// ANSE:               %[[VAL_16:.*]] = arith.addi %[[VAL_13]], %[[VAL_15]] : index
-// ANSE:               %[[VAL_17:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_16]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:               %[[VAL_17:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_13]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:               hlfir.assign %[[VAL_4]] to %[[VAL_17]] : f32, !fir.ref<f32>
 // ANSE:             }
 // ANSE:             fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {
 // ANSE:               fir.do_loop %[[VAL_19:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] {
-// ANSE:                 %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_12]], %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// ANSE:                 %[[VAL_21:.*]] = arith.subi %[[VAL_20]]#0, %[[VAL_3]] : index
-// ANSE:                 %[[VAL_22:.*]] = arith.addi %[[VAL_19]], %[[VAL_21]] : index
-// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_22]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// ANSE:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
 // ANSE:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f32>
 // ANSE:                 %[[VAL_25:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]] : (!hlfir.expr<?xf32>, index) -> f32
 // ANSE:                 %[[VAL_26:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_18]], %[[VAL_19]] : (!hlfir.expr<?x?xf16>, index, index) -> f16

diff  --git a/flang/test/Transforms/optimize-array-repacking.fir b/flang/test/Transforms/optimize-array-repacking.fir
new file mode 100644
index 0000000000000..6269fa441fe44
--- /dev/null
+++ b/flang/test/Transforms/optimize-array-repacking.fir
@@ -0,0 +1,660 @@
+// Test that the redundant fir.[un]pack_array operations
+// are optimized away, when the source is statically known
+// to be contiguous.
+// RUN: fir-opt --optimize-array-repacking %s | FileCheck %s
+
+// FIR is produced by compiling the sources with -mllvm -inline-all.
+// module inner
+// contains
+//   subroutine inner_repack1(x)
+//     real :: x(:)
+//   end subroutine inner_repack1
+//   subroutine inner_repack2(x)
+//     real :: x(:,:)
+//   end subroutine inner_repack2
+// end module inner
+
+// subroutine test_explicit_shape_cst(x)
+//   real :: x(100)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_explicit_shape_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_cst(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_cstEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_explicit_shape_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_var(x, n, l, u)
+//   integer :: n, l, u
+//   real :: x(n)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_explicit_shape_var
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_var(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}, %arg2: !fir.ref<i32> {fir.bindc_name = "l"}, %arg3: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg3 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %4 = fir.load %2 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9 = fir.declare %arg0(%8) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_varEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %10 = fir.load %1 : !fir.ref<i32>
+  %11 = fir.load %3 : !fir.ref<i32>
+  %12 = fir.convert %10 : (i32) -> index
+  %13 = fir.convert %11 : (i32) -> index
+  %14 = fir.slice %12, %13, %c1 : (index, index, index) -> !fir.slice<1>
+  %15 = fir.embox %9(%8) [%14] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.dummy_scope : !fir.dscope
+  %17 = fir.pack_array %15 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.declare %17 dummy_scope %16 {uniq_name = "_QFtest_explicit_shape_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %17 to %15 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_cst(x)
+//   real :: x(*)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_assumed_size_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_cst(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_cstEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_assumed_size_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_var(x, l, u)
+//   integer :: l, u
+//   real :: x(*)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_assumed_size_var
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_var(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %4 = fir.declare %arg0(%3) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_varEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %10 = fir.embox %4(%3) [%9] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_assumed_size_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_cst(x)
+//   real, allocatable :: x(:)
+//   call repack(x(10:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_allocatable_cst
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_cst(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_cst(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c41 = arith.constant 41 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_cstEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %3 = fir.shape %c41 : (index) -> !fir.shape<1>
+  %4 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %5:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %6 = fir.shape_shift %5#0, %5#1 : (index, index) -> !fir.shapeshift<1>
+  %7 = fir.array_coor %4(%6) %c10 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, index) -> !fir.ref<f32>
+  %8 = fir.convert %7 : (!fir.ref<f32>) -> !fir.ref<!fir.array<41xf32>>
+  %9 = fir.embox %8(%3) : (!fir.ref<!fir.array<41xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<41xf32>>
+  %10 = fir.convert %9 : (!fir.box<!fir.array<41xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_allocatable_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_var(x, l, u)
+//   integer :: l, u
+//   real, allocatable :: x(:)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine test_allocatable_var
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_var(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_var(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_allocatable_varEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_allocatable_varEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_varEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.box_addr %4 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %10:3 = fir.box_dims %4, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %11 = fir.shape_shift %10#0, %10#1 : (index, index) -> !fir.shapeshift<1>
+  %12 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %13 = fir.embox %9(%11) [%12] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.dummy_scope : !fir.dscope
+  %15 = fir.pack_array %13 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.declare %15 dummy_scope %14 {uniq_name = "_QFtest_allocatable_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %15 to %13 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_full(x)
+//   real, allocatable :: x(:,:)
+//   call repack(x(:,:))
+// contains
+//   subroutine repack(x)
+//     real :: x(:,:)
+//   end subroutine repack
+// end subroutine test_allocatable_full
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_full(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_full(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_fullEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %4:3 = fir.box_dims %2, %c1 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %5 = arith.addi %3#0, %3#1 : index
+  %6 = arith.subi %5, %c1 : index
+  %7 = arith.addi %4#0, %4#1 : index
+  %8 = arith.subi %7, %c1 : index
+  %9 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.heap<!fir.array<?x?xf32>>
+  %10 = fir.shape_shift %3#0, %3#1, %4#0, %4#1 : (index, index, index, index) -> !fir.shapeshift<2>
+  %11 = fir.slice %3#0, %6, %c1, %4#0, %8, %c1 : (index, index, index, index, index, index) -> !fir.slice<2>
+  %12 = fir.embox %9(%10) [%11] : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>, !fir.slice<2>) -> !fir.box<!fir.array<?x?xf32>>
+  %13 = fir.dummy_scope : !fir.dscope
+  %14 = fir.pack_array %12 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %15 = fir.declare %14 dummy_scope %13 {uniq_name = "_QFtest_allocatable_fullFrepackEx"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %14 to %12 heap : !fir.box<!fir.array<?x?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_cst_chain(x)
+//   real :: x(100)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_explicit_shape_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_cst_chain(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_cst_chainEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_explicit_shape_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.rebox %10 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %12 = fir.dummy_scope : !fir.dscope
+  %13 = fir.pack_array %11 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.declare %13 dummy_scope %12 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %13 to %11 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_explicit_shape_var_chain(x, n, l, u)
+//   integer :: n, l, u
+//   real :: x(n)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_explicit_shape_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_explicit_shape_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_explicit_shape_var_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}, %arg2: !fir.ref<i32> {fir.bindc_name = "l"}, %arg3: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg3 dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %4 = fir.load %2 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9 = fir.declare %arg0(%8) dummy_scope %0 {uniq_name = "_QFtest_explicit_shape_var_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %10 = fir.load %1 : !fir.ref<i32>
+  %11 = fir.load %3 : !fir.ref<i32>
+  %12 = fir.convert %10 : (i32) -> index
+  %13 = fir.convert %11 : (i32) -> index
+  %14 = fir.slice %12, %13, %c1 : (index, index, index) -> !fir.slice<1>
+  %15 = fir.embox %9(%8) [%14] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.dummy_scope : !fir.dscope
+  %17 = fir.pack_array %15 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.declare %17 dummy_scope %16 {uniq_name = "_QFtest_explicit_shape_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %19 = fir.rebox %18 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %20 = fir.dummy_scope : !fir.dscope
+  %21 = fir.pack_array %19 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %22 = fir.declare %21 dummy_scope %20 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %21 to %19 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %17 to %15 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_cst_chain(x)
+//   real :: x(*)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_assumed_size_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_cst_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_cst_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %3 = fir.shape %c50 : (index) -> !fir.shape<1>
+  %4 = fir.array_coor %2(%1) %c1 : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
+  %5 = fir.convert %4 : (!fir.ref<f32>) -> !fir.ref<!fir.array<50xf32>>
+  %6 = fir.embox %5(%3) : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFtest_assumed_size_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.rebox %10 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %12 = fir.dummy_scope : !fir.dscope
+  %13 = fir.pack_array %11 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.declare %13 dummy_scope %12 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %13 to %11 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_assumed_size_var_chain(x, l, u)
+//   integer :: l, u
+//   real :: x(*)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_assumed_size_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_assumed_size_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_assumed_size_var_chain(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c1 = arith.constant 1 : index
+  %c-1 = arith.constant -1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.shape %c-1 : (index) -> !fir.shape<1>
+  %4 = fir.declare %arg0(%3) dummy_scope %0 {uniq_name = "_QFtest_assumed_size_var_chainEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<?xf32>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %10 = fir.embox %4(%3) [%9] : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_assumed_size_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.rebox %13 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %15 = fir.dummy_scope : !fir.dscope
+  %16 = fir.pack_array %14 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.declare %16 dummy_scope %15 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %16 to %14 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_cst_chain(x)
+//   real, allocatable :: x(:)
+//   call repack(x(10:50))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_allocatable_cst_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_cst_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_cst_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c41 = arith.constant 41 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_cst_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %3 = fir.shape %c41 : (index) -> !fir.shape<1>
+  %4 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %5:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %6 = fir.shape_shift %5#0, %5#1 : (index, index) -> !fir.shapeshift<1>
+  %7 = fir.array_coor %4(%6) %c10 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, index) -> !fir.ref<f32>
+  %8 = fir.convert %7 : (!fir.ref<f32>) -> !fir.ref<!fir.array<41xf32>>
+  %9 = fir.embox %8(%3) : (!fir.ref<!fir.array<41xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<41xf32>>
+  %10 = fir.convert %9 : (!fir.box<!fir.array<41xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "_QFtest_allocatable_cst_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.rebox %13 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %15 = fir.dummy_scope : !fir.dscope
+  %16 = fir.pack_array %14 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.declare %16 dummy_scope %15 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %16 to %14 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_var_chain(x, l, u)
+//   integer :: l, u
+//   real, allocatable :: x(:)
+//   call repack(x(l:u))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:)
+//     call inner_repack1(x)
+//   end subroutine repack
+// end subroutine test_allocatable_var_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_var_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_var_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "l"}, %arg2: !fir.ref<i32> {fir.bindc_name = "u"}) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtest_allocatable_var_chainEl"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFtest_allocatable_var_chainEu"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_var_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %5 = fir.load %1 : !fir.ref<i32>
+  %6 = fir.load %2 : !fir.ref<i32>
+  %7 = fir.convert %5 : (i32) -> index
+  %8 = fir.convert %6 : (i32) -> index
+  %9 = fir.box_addr %4 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+  %10:3 = fir.box_dims %4, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %11 = fir.shape_shift %10#0, %10#1 : (index, index) -> !fir.shapeshift<1>
+  %12 = fir.slice %7, %8, %c1 : (index, index, index) -> !fir.slice<1>
+  %13 = fir.embox %9(%11) [%12] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %14 = fir.dummy_scope : !fir.dscope
+  %15 = fir.pack_array %13 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %16 = fir.declare %15 dummy_scope %14 {uniq_name = "_QFtest_allocatable_var_chainFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %17 = fir.rebox %16 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %18 = fir.dummy_scope : !fir.dscope
+  %19 = fir.pack_array %17 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %20 = fir.declare %19 dummy_scope %18 {uniq_name = "_QMinnerFinner_repack1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %19 to %17 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %15 to %13 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine test_allocatable_full_chain(x)
+//   real, allocatable :: x(:,:)
+//   call repack(x(:,:))
+// contains
+//   subroutine repack(x)
+//     use inner
+//     real :: x(:,:)
+//     call inner_repack2(x)
+//   end subroutine repack
+// end subroutine test_allocatable_full_chain
+//
+// CHECK-LABEL:   func.func @_QPtest_allocatable_full_chain(
+// CHECK-NOT: fir.{{.*}}pack_array
+func.func @_QPtest_allocatable_full_chain(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatable_full_chainEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %4:3 = fir.box_dims %2, %c1 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %5 = arith.addi %3#0, %3#1 : index
+  %6 = arith.subi %5, %c1 : index
+  %7 = arith.addi %4#0, %4#1 : index
+  %8 = arith.subi %7, %c1 : index
+  %9 = fir.box_addr %2 : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.heap<!fir.array<?x?xf32>>
+  %10 = fir.shape_shift %3#0, %3#1, %4#0, %4#1 : (index, index, index, index) -> !fir.shapeshift<2>
+  %11 = fir.slice %3#0, %6, %c1, %4#0, %8, %c1 : (index, index, index, index, index, index) -> !fir.slice<2>
+  %12 = fir.embox %9(%10) [%11] : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>, !fir.slice<2>) -> !fir.box<!fir.array<?x?xf32>>
+  %13 = fir.dummy_scope : !fir.dscope
+  %14 = fir.pack_array %12 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %15 = fir.declare %14 dummy_scope %13 {uniq_name = "_QFtest_allocatable_full_chainFrepackEx"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  %16 = fir.rebox %15 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %17 = fir.dummy_scope : !fir.dscope
+  %18 = fir.pack_array %16 heap innermost : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>>
+  %19 = fir.declare %18 dummy_scope %17 {uniq_name = "_QMinnerFinner_repack2Ex"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %18 to %16 heap : !fir.box<!fir.array<?x?xf32>>
+  fir.unpack_array %14 to %12 heap : !fir.box<!fir.array<?x?xf32>>
+  return
+}
+
+// TODO: if both fir.pack_array have the same property,
+// then the second one is redundant, because the first
+// repack makes 'x' contiguous.
+// subroutine neg_test_assumed_shape(x)
+//   real :: x(:)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_assumed_shape
+//
+// CHECK-LABEL:   func.func @_QPneg_test_assumed_shape(
+// CHECK: fir.pack_array
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_assumed_shape(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.pack_array %arg0 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %2 = fir.declare %1 dummy_scope %0 {uniq_name = "_QFneg_test_assumed_shapeEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  %3 = fir.rebox %2 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %4 = fir.slice %c1, %c50, %c1 : (index, index, index) -> !fir.slice<1>
+  %5 = fir.rebox %3 [%4] : (!fir.box<!fir.array<?xf32>>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
+  %6 = fir.convert %5 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %7 = fir.dummy_scope : !fir.dscope
+  %8 = fir.pack_array %6 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %9 = fir.declare %8 dummy_scope %7 {uniq_name = "_QFneg_test_assumed_shapeFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %8 to %6 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %1 to %arg0 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_non_contig_slice_cst(x)
+//   real :: x(100)
+//   call repack(x(1:50:2))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_non_contig_slice_cst
+//
+// CHECK-LABEL:   func.func @_QPneg_test_non_contig_slice_cst(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_non_contig_slice_cst(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}) {
+  %c2 = arith.constant 2 : index
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %2 = fir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_cstEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %3 = fir.slice %c1, %c50, %c2 : (index, index, index) -> !fir.slice<1>
+  %4 = fir.embox %2(%1) [%3] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<25xf32>>
+  %5 = fir.convert %4 : (!fir.box<!fir.array<25xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %6 = fir.dummy_scope : !fir.dscope
+  %7 = fir.pack_array %5 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.declare %7 dummy_scope %6 {uniq_name = "_QFneg_test_non_contig_slice_cstFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %7 to %5 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_non_contig_slice_var(x, s)
+//   integer :: s
+//   real :: x(100)
+//   call repack(x(1:50:s))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_non_contig_slice_var
+//
+// CHECK-LABEL:   func.func @_QPneg_test_non_contig_slice_var(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_non_contig_slice_var(%arg0: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "s"}) {
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %c100 = arith.constant 100 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_varEs"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %2 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %3 = fir.declare %arg0(%2) dummy_scope %0 {uniq_name = "_QFneg_test_non_contig_slice_varEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<100xf32>>
+  %4 = fir.load %1 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = fir.slice %c1, %c50, %5 : (index, index, index) -> !fir.slice<1>
+  %7 = fir.embox %3(%2) [%6] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFneg_test_non_contig_slice_varFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// subroutine neg_test_pointer(x)
+//   real, pointer :: x(:)
+//   call repack(x(1:50))
+// contains
+//   subroutine repack(x)
+//     real :: x(:)
+//   end subroutine repack
+// end subroutine neg_test_pointer
+//
+// CHECK-LABEL:   func.func @_QPneg_test_pointer(
+// CHECK: fir.pack_array
+// CHECK: fir.unpack_array
+func.func @_QPneg_test_pointer(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {fir.bindc_name = "x"}) {
+  %c0 = arith.constant 0 : index
+  %c50 = arith.constant 50 : index
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFneg_test_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+  %4 = fir.shift %3#0 : (index) -> !fir.shift<1>
+  %5 = fir.slice %c1, %c50, %c1 : (index, index, index) -> !fir.slice<1>
+  %6 = fir.rebox %2(%4) [%5] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, !fir.shift<1>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<50xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %8 = fir.dummy_scope : !fir.dscope
+  %9 = fir.pack_array %7 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %10 = fir.declare %9 dummy_scope %8 {uniq_name = "_QFneg_test_pointerFrepackEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
+  return
+}


        

From flang-commits at lists.llvm.org  Mon Jul 14 09:45:36 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 14 Jul 2025 09:45:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Lower] fix warning (PR #148667)
Message-ID: <llvm.org/llvm/llvm-project/pull/148667@github.com>


https://github.com/tblah created https://github.com/llvm/llvm-project/pull/148667

GCC 9.3.0
```
.../flang/lib/Lower/Support/ReductionProcessor.cpp:137:1: error: control reaches end of non-void function [-Werror=return-type]
```

>From d7efe3a8d91de159f1ded451d63fec5d7e466991 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Mon, 14 Jul 2025 16:44:03 +0000
Subject: [PATCH] [flang][Lower] fix warning

GCC 9.3.0
```
.../flang/lib/Lower/Support/ReductionProcessor.cpp:137:1: error: control reaches end of non-void function [-Werror=return-type]
```
---
 flang/lib/Lower/Support/ReductionProcessor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 77cb15a2a77ca..c0be1e229f825 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -134,6 +134,7 @@ ReductionProcessor::getReductionType(const fir::ReduceOperationEnum &redOp) {
   case fir::ReduceOperationEnum::MIN:
     return ReductionIdentifier::MIN;
   }
+  llvm_unreachable("Unhandled ReductionIdentifier case");
 }
 
 bool ReductionProcessor::supportedIntrinsicProcReduction(


From flang-commits at lists.llvm.org  Mon Jul 14 09:57:27 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 14 Jul 2025 09:57:27 -0700 (PDT)
Subject: [flang-commits]  [flang] [flang] Extension: TRANSFER(boz,
	integer or real scalar) (PR #147604)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147604@github.com>
Message-ID: <687536f7.050a0220.cabbc.a628@mx.google.com>


https://github.com/klausler updated https://github.com/llvm/llvm-project/pull/147604

>From 5c036ab2f1b2cf70769c23effb8e05b92be93b50 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 8 Jul 2025 15:17:16 -0700
Subject: [PATCH] [flang] Extension: TRANSFER(boz, integer or real scalar)

Interpret TRANSFER(SOURCE=BOZ literal, MOLD=integer or real scalar)
as if it had been a reference to the standard INT or REAL intrinsic
functions, for which a BOZ literal is an acceptable argument, with
a warning about non-conformance.  It's a needless extension that
has somehow crept into some other compilers and real applications.
---
 flang/docs/Extensions.md                      |  2 ++
 .../include/flang/Support/Fortran-features.h  |  3 ++-
 flang/lib/Evaluate/fold.cpp                   | 10 ++++++++++
 flang/lib/Evaluate/intrinsics.cpp             |  9 +++++++++
 flang/lib/Support/Fortran-features.cpp        |  1 +
 flang/test/Evaluate/transfer-boz.f90          | 19 +++++++++++++++++++
 6 files changed, 43 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Evaluate/transfer-boz.f90

diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 37a49f12f9177..72d12cd92600d 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -252,6 +252,8 @@ end
   the type resolution of such BOZ literals usages is highly non portable).
 * BOZ literals can also be used as REAL values in some contexts where the
   type is unambiguous, such as initializations of REAL parameters.
+* `TRANSFER(boz, MOLD=integer or real scalar)` is accepted as an alternate
+  spelling of `INT(boz, KIND=kind(mold))` or `REAL(boz, KIND=kind(mold))`.
 * EQUIVALENCE of numeric and character sequences (a ubiquitous extension),
   as well as of sequences of non-default kinds of numeric types
   with each other.
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 857de9479e4e5..743abf606ab5f 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,8 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload,
+    TransferBOZ)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
diff --git a/flang/lib/Evaluate/fold.cpp b/flang/lib/Evaluate/fold.cpp
index 45e842abf589f..71ead1b3afa91 100644
--- a/flang/lib/Evaluate/fold.cpp
+++ b/flang/lib/Evaluate/fold.cpp
@@ -287,6 +287,16 @@ std::optional<Expr<SomeType>> FoldTransfer(
         CHECK(status == InitialImage::NotAConstant);
       }
     }
+  } else if (source && moldType) {
+    if (const auto *boz{std::get_if<BOZLiteralConstant>(&source->u)}) {
+      // TRANSFER(BOZ, MOLD=integer or real) extension
+      if (context.languageFeatures().ShouldWarn(
+              common::LanguageFeature::TransferBOZ)) {
+        context.messages().Say(common::LanguageFeature::TransferBOZ,
+            "TRANSFER(BOZ literal) is not standard"_port_en_US);
+      }
+      return Fold(context, ConvertToType(*moldType, Expr<SomeType>{*boz}));
+    }
   }
   return std::nullopt;
 }
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index e802915945e26..4773e136c41cb 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1015,6 +1015,15 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
             {"mold", SameType, Rank::anyOrAssumedRank},
             {"size", AnyInt, Rank::scalar}},
         SameType, Rank::vector, IntrinsicClass::transformationalFunction},
+    // TRANSFER(BOZ, MOLD=integer or real scalar) extension
+    {"transfer",
+        {{"source", AnyNumeric, Rank::elementalOrBOZ},
+            {"mold", SameInt, Rank::scalar}},
+        SameInt, Rank::scalar, IntrinsicClass::transformationalFunction},
+    {"transfer",
+        {{"source", AnyNumeric, Rank::elementalOrBOZ},
+            {"mold", SameReal, Rank::scalar}},
+        SameReal, Rank::scalar, IntrinsicClass::transformationalFunction},
     {"transpose", {{"matrix", SameType, Rank::matrix}}, SameType, Rank::matrix,
         IntrinsicClass::transformationalFunction},
     {"trim", {{"string", SameCharNoLen, Rank::scalar}}, SameCharNoLen,
diff --git a/flang/lib/Support/Fortran-features.cpp b/flang/lib/Support/Fortran-features.cpp
index fc69fc638eda1..df51b3c577125 100644
--- a/flang/lib/Support/Fortran-features.cpp
+++ b/flang/lib/Support/Fortran-features.cpp
@@ -103,6 +103,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
   warnLanguage_.set(LanguageFeature::ListDirectedSize);
   warnLanguage_.set(LanguageFeature::IgnoreIrrelevantAttributes);
   warnLanguage_.set(LanguageFeature::AmbiguousStructureConstructor);
+  warnLanguage_.set(LanguageFeature::TransferBOZ);
   warnUsage_.set(UsageWarning::ShortArrayActual);
   warnUsage_.set(UsageWarning::FoldingException);
   warnUsage_.set(UsageWarning::FoldingAvoidsRuntimeCrash);
diff --git a/flang/test/Evaluate/transfer-boz.f90 b/flang/test/Evaluate/transfer-boz.f90
new file mode 100644
index 0000000000000..cb02ce45cb9ef
--- /dev/null
+++ b/flang/test/Evaluate/transfer-boz.f90
@@ -0,0 +1,19 @@
+!RUN: %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
+
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: PRINT *, -17_1
+!CHECK: PRINT *, -16657_2
+!CHECK: PRINT *, -559038737_4
+!CHECK: PRINT *, -6.259853398707798016e18_4
+!CHECK: PRINT *, 1.8457939563190925445492984919045437485528002903297587499184927815557801644025850059853181797614625558939883606558067941052215743386077465406508281130260792904117042434297523782789810991060595088402505831985608687099349932384478542748214418349497866218005870896526832696740683567118136284392505567470522678176873737470272665943917715636680452977840047387616237048162886116765806372690690621665407240253312632381089695794498069551496558727944808752442655077077609749137334421734380446229972733723471196114034171513283476953482396960122117078210644171413215132537631249069125441199719426016512325803207900875869312966829207468856964517085119314095898691593840972164432952065765882286023532502465793032567186884340163939544586513274992967126308940351009368896484375e-314_8
+
+print *, transfer(z'deadbeef', 1_1)
+print *, transfer(z'deadbeef', 1_2)
+print *, transfer(z'deadbeef', 1_4)
+print *, transfer(z'deadbeef', 1._4)
+print *, transfer(z'deadbeef', 1._8)
+end


From flang-commits at lists.llvm.org  Mon Jul 14 09:59:21 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 09:59:21 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][core] NFC update builder
	create API usage (PR #147311)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147311@github.com>
Message-ID: <68753769.170a0220.14b484.9895@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions h,cpp -- flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp flang/lib/Optimizer/Dialect/FIROps.cpp flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp mlir/examples/toy/Ch2/mlir/Dialect.cpp mlir/examples/toy/Ch3/mlir/Dialect.cpp mlir/examples/toy/Ch4/mlir/Dialect.cpp mlir/examples/toy/Ch5/mlir/Dialect.cpp mlir/examples/toy/Ch6/mlir/Dialect.cpp mlir/examples/toy/Ch7/mlir/Dialect.cpp mlir/examples/transform/Ch2/lib/MyExtension.cpp mlir/examples/transform/Ch3/lib/MyExtension.cpp mlir/examples/transform/Ch4/lib/MyExtension.cpp mlir/include/mlir/Dialect/Affine/IR/AffineOps.h mlir/include/mlir/Dialect/Arith/IR/Arith.h mlir/lib/CAPI/Dialect/PDL.cpp mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp mlir/lib/Conversion/LLVMCommon/Pattern.cpp mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp mlir/lib/Conversion/MathToLibm/MathToLibm.cpp mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp mlir/lib/Conversion/TosaToArith/TosaToArith.cpp mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp mlir/lib/Dialect/AMX/IR/AMXDialect.cpp mlir/lib/Dialect/Affine/IR/AffineOps.cpp mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp mlir/lib/Dialect/Affine/Utils/Utils.cpp mlir/lib/Dialect/Arith/IR/ArithDialect.cpp mlir/lib/Dialect/Arith/IR/ArithOps.cpp mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp mlir/lib/Dialect/Arith/Utils/Utils.cpp mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp mlir/lib/Dialect/ArmSME/IR/Utils.cpp mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp mlir/lib/Dialect/Async/IR/Async.cpp mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp mlir/lib/Dialect/Complex/IR/ComplexOps.cpp mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp mlir/lib/Dialect/EmitC/IR/EmitC.cpp mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp mlir/lib/Dialect/Func/IR/FuncOps.cpp mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp mlir/lib/Dialect/Func/Utils/Utils.cpp mlir/lib/Dialect/GPU/IR/GPUDialect.cpp mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp mlir/lib/Dialect/GPU/TransformOps/Utils.cpp mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp mlir/lib/Dialect/IRDL/IR/IRDL.cpp mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp mlir/lib/Dialect/Index/IR/IndexOps.cpp mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp mlir/lib/Dialect/Linalg/Transforms/Loops.cpp mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp mlir/lib/Dialect/Linalg/Transforms/Padding.cpp mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp mlir/lib/Dialect/Linalg/Transforms/Split.cpp mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp mlir/lib/Dialect/Linalg/Utils/Utils.cpp mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp mlir/lib/Dialect/MPI/IR/MPIOps.cpp mlir/lib/Dialect/Math/IR/MathOps.cpp mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp mlir/lib/Dialect/Mesh/IR/MeshOps.cpp mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp mlir/lib/Dialect/PDL/IR/PDL.cpp mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp mlir/lib/Dialect/Quant/IR/QuantOps.cpp mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp mlir/lib/Dialect/SCF/IR/SCF.cpp mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp mlir/lib/Dialect/SCF/Utils/Utils.cpp mlir/lib/Dialect/SMT/IR/SMTDialect.cpp mlir/lib/Dialect/SMT/IR/SMTOps.cpp mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp mlir/lib/Dialect/Shape/IR/Shape.cpp mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp mlir/lib/Dialect/Tensor/IR/TensorOps.cpp mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp mlir/lib/Dialect/Tensor/Utils/Utils.cpp mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp mlir/lib/Dialect/Tosa/IR/TosaOps.cpp mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp mlir/lib/Dialect/Transform/IR/TransformOps.cpp mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp mlir/lib/Dialect/UB/IR/UBOps.cpp mlir/lib/Dialect/Vector/IR/VectorOps.cpp mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp mlir/lib/IR/BuiltinDialect.cpp mlir/lib/Query/Query.cpp mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp mlir/lib/Target/LLVMIR/ModuleImport.cpp mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp mlir/lib/Transforms/Utils/DialectConversion.cpp mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp mlir/test/lib/Transforms/TestTransformsOps.cpp mlir/test/python/lib/PythonTestDialect.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index 5d66c3900..3a205246d 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -825,9 +825,10 @@ public:
     // it to include the appropriate checks. If the current value is NaN the
     // old value of pool will be taken otherwise we use the result.
     if (nanMode == "IGNORE") {
-      auto genericOp = linalg::GenericOp::create(rewriter,
-          loc, resultOp.getType(0), resultOp.getInputs(), resultOp.getOutputs(),
-          resultOp.getIndexingMapsArray(), resultOp.getIteratorTypesArray(),
+      auto genericOp = linalg::GenericOp::create(
+          rewriter, loc, resultOp.getType(0), resultOp.getInputs(),
+          resultOp.getOutputs(), resultOp.getIndexingMapsArray(),
+          resultOp.getIteratorTypesArray(),
           [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
             IRMapping map;
             auto oldBlock = resultOp.getRegion().begin();
@@ -835,11 +836,11 @@ public:
             auto &oldMaxOp = *resultOp.getBlock()->begin();
             map.map(oldArgs, blockArgs);
             auto *newOp = opBuilder.clone(oldMaxOp, map);
-            Value isNaN = arith::CmpFOp::create(opBuilder,
-                loc, arith::CmpFPredicate::UNO, blockArgs.front(),
-                blockArgs.front());
-            auto selectOp = arith::SelectOp::create(opBuilder,
-                loc, isNaN, blockArgs.back(), newOp->getResult(0));
+            Value isNaN =
+                arith::CmpFOp::create(opBuilder, loc, arith::CmpFPredicate::UNO,
+                                      blockArgs.front(), blockArgs.front());
+            auto selectOp = arith::SelectOp::create(
+                opBuilder, loc, isNaN, blockArgs.back(), newOp->getResult(0));
             linalg::YieldOp::create(opBuilder, loc, selectOp.getResult());
           });
       rewriter.replaceOp(resultOp, genericOp);
diff --git a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
index 12018912f..dcac0ab69 100644
--- a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
@@ -12,8 +12,8 @@
 
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index 7dd1b856f..28ea795c1 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -8,9 +8,9 @@
 
 #include "mlir/Dialect/Async/IR/Async.h"
 
+#include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
-#include "mlir/IR/Builders.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
index b8923bb50..04d949d5a 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
@@ -433,8 +433,9 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
           IntegerType::get(rewriter.getContext(),
                            input.getType().getIntOrFloatBitWidth()),
           input);
-    }    Value extendedInput = arith::ExtUIOp::create(rewriter, loc, dstIntegerType,
-                                                 input);
+    }
+    Value extendedInput =
+        arith::ExtUIOp::create(rewriter, loc, dstIntegerType, input);
 
     // Special case 0-rank memref stores. No need for masking.
     if (convertedType.getRank() == 0) {
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 0a8178fe1..be728f7b6 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -1286,8 +1286,9 @@ struct ConvertVectorTransferRead final
           IntegerType::get(rewriter.getContext(),
                            padding.getType().getIntOrFloatBitWidth()),
           padding);
-    }auto newPadding = arith::ExtUIOp::create(rewriter, loc, containerElemTy,
-                                             padding);
+    }
+    auto newPadding =
+        arith::ExtUIOp::create(rewriter, loc, containerElemTy, padding);
 
     auto stridedMetadata =
         memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());

``````````

</details>


https://github.com/llvm/llvm-project/pull/147311

From flang-commits at lists.llvm.org  Mon Jul 14 09:59:53 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 09:59:53 -0700 (PDT)
Subject: [flang-commits] [flang] 3d6aac3 - [flang][Lower] fix warning
	(#148667)
Message-ID: <68753789.050a0220.2b6cc.c225@mx.google.com>


Author: Tom Eccles
Date: 2025-07-14T17:59:50+01:00
New Revision: 3d6aac31fcb5f7a988be5e484e44bdc9033265c5

URL: https://github.com/llvm/llvm-project/commit/3d6aac31fcb5f7a988be5e484e44bdc9033265c5
DIFF: https://github.com/llvm/llvm-project/commit/3d6aac31fcb5f7a988be5e484e44bdc9033265c5.diff

LOG: [flang][Lower] fix warning (#148667)

GCC 9.3.0
```
.../flang/lib/Lower/Support/ReductionProcessor.cpp:137:1: error: control reaches end of non-void function [-Werror=return-type]
```

Added: 
    

Modified: 
    flang/lib/Lower/Support/ReductionProcessor.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 77cb15a2a77ca..c0be1e229f825 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -134,6 +134,7 @@ ReductionProcessor::getReductionType(const fir::ReduceOperationEnum &redOp) {
   case fir::ReduceOperationEnum::MIN:
     return ReductionIdentifier::MIN;
   }
+  llvm_unreachable("Unhandled ReductionIdentifier case");
 }
 
 bool ReductionProcessor::supportedIntrinsicProcReduction(


        

From flang-commits at lists.llvm.org  Mon Jul 14 09:50:13 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 09:50:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [TargetVerifier][AMDGPU] Add
 TargetVerifier. (PR #123609)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/123609@github.com>
Message-ID: <68753545.170a0220.788ca.95e8@mx.google.com>


https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/123609

>From d9b0a5df811451d4d4a2e37fc4933c8071022317 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 20 Jan 2025 04:51:26 -0800
Subject: [PATCH 01/33] [TargetVerifier][AMDGPU] Add TargetVerifier.

This pass verifies the IR for an individual backend. This is
different than Lint because it consolidates all checks for a
given backend in a single pass. A check for Lint may be
undefined behavior across all targets, whereas a check in
TargetVerifier would only pertain to the specified target
but can check more than just undefined behavior such are IR
validity. A use case of this would be to reject programs
with invalid IR while fuzzing.
---
 llvm/include/llvm/IR/Module.h                 |   9 +
 llvm/include/llvm/Target/TargetVerifier.h     |  82 +++++++
 .../TargetVerify/AMDGPUTargetVerifier.h       |  36 +++
 llvm/lib/IR/Verifier.cpp                      |  18 +-
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 213 ++++++++++++++++++
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        |  62 +++++
 llvm/tools/llvm-tgt-verify/CMakeLists.txt     |  34 +++
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 172 ++++++++++++++
 9 files changed, 623 insertions(+), 4 deletions(-)
 create mode 100644 llvm/include/llvm/Target/TargetVerifier.h
 create mode 100644 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify.ll
 create mode 100644 llvm/tools/llvm-tgt-verify/CMakeLists.txt
 create mode 100644 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp

diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index a99937a90cbb7..3037b2c1644d9 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,6 +215,15 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
+  /// Is this Module valid as determined by one of the verification passes
+  /// i.e. Lint, Verifier, TargetVerifier.
+  bool IsValid = true;
+
+  /// Is this Module using intrinsics to record the position of debugging
+  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
+  /// \ref BasicBlock.
+  bool IsNewDbgInfoFormat;
+
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
new file mode 100644
index 0000000000000..e00c6a7b260c9
--- /dev/null
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -0,0 +1,82 @@
+//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at TargetVerifier.cpp or an
+// individual backend's TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_VERIFIER_H
+#define LLVM_TARGET_VERIFIER_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+
+class Function;
+
+class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {}
+};
+
+class TargetVerify {
+protected:
+  void WriteValues(ArrayRef<const Value *> Vs) {
+    for (const Value *V : Vs) {
+      if (!V)
+        continue;
+      if (isa<Instruction>(V)) {
+        MessagesStr << *V << '\n';
+      } else {
+        V->printAsOperand(MessagesStr, true, Mod);
+        MessagesStr << '\n';
+      }
+    }
+  }
+
+  /// A check failed, so printout out the condition and the message.
+  ///
+  /// This provides a nice place to put a breakpoint if you want to see why
+  /// something is not correct.
+  void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
+
+  /// A check failed (with values to print).
+  ///
+  /// This calls the Message-only version so that the above is easier to set
+  /// a breakpoint on.
+  template <typename T1, typename... Ts>
+  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) {
+    CheckFailed(Message);
+    WriteValues({V1, Vs...});
+  }
+public:
+  Module *Mod;
+  Triple TT;
+
+  std::string Messages;
+  raw_string_ostream MessagesStr;
+
+  TargetVerify(Module *Mod)
+      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
+        MessagesStr(Messages) {}
+
+  void run(Function &F) {};
+};
+
+} // namespace llvm
+
+#endif // LLVM_TARGET_VERIFIER_H
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
new file mode 100644
index 0000000000000..e6ff57629b141
--- /dev/null
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -0,0 +1,36 @@
+//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU ---*- C++ -*-===//
+////
+//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//// See https://llvm.org/LICENSE.txt for license information.
+//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+////
+////===----------------------------------------------------------------------===//
+////
+//// This file defines target verifier interfaces that can be used for some
+//// validation of input to the system, and for checking that transformations
+//// haven't done something bad. In contrast to the Verifier or Lint, the
+//// TargetVerifier looks for constructions invalid to a particular target
+//// machine.
+////
+//// To see what specifically is checked, look at an individual backend's
+//// TargetVerifier.
+////
+////===----------------------------------------------------------------------===//
+
+#ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
+#define LLVM_AMDGPU_TARGET_VERIFIER_H
+
+#include "llvm/Target/TargetVerifier.h"
+
+namespace llvm {
+
+class Function;
+
+class AMDGPUTargetVerifierPass : public TargetVerifierPass {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1f1041b259736..18e9760f9c617 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -135,6 +135,10 @@ static cl::opt<bool> VerifyNoAliasScopeDomination(
     cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
              "scopes are not dominating"));
 
+static cl::opt<bool>
+    VerifyAbortOnError("verifier-abort-on-error", cl::init(false),
+                       cl::desc("In the Verifier pass, abort on errors."));
+
 namespace llvm {
 
 struct VerifierSupport {
@@ -7865,16 +7869,22 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
-  if (FatalErrors && (Res.IRBroken || Res.DebugInfoBroken))
-    report_fatal_error("Broken module found, compilation aborted!");
+  if (Res.IRBroken || Res.DebugInfoBroken) {
+    M.IsValid = false;
+    if (VerifyAbortOnError && FatalErrors)
+      report_fatal_error("Broken module found, compilation aborted!");
+  }
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
-  if (res.IRBroken && FatalErrors)
-    report_fatal_error("Broken function found, compilation aborted!");
+  if (res.IRBroken) {
+    F.getParent()->IsValid = false;
+    if (VerifyAbortOnError && FatalErrors)
+      report_fatal_error("Broken function found, compilation aborted!");
+  }
 
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
new file mode 100644
index 0000000000000..585b19065c142
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -0,0 +1,213 @@
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<bool>
+MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
+
+// Check - We know that cond should be true, if not print an error message.
+#define Check(C, ...)                                                          \
+  do {                                                                         \
+    if (!(C)) {                                                                \
+      TargetVerify::CheckFailed(__VA_ARGS__);                                  \
+      return;                                                                  \
+    }                                                                          \
+  } while (false)
+
+static bool isMFMA(unsigned IID) {
+  switch (IID) {
+    case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
+    case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
+    case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
+    case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
+    case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
+    case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
+    case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
+    case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
+    case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
+    case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
+    case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
+
+    case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
+    case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
+
+    case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
+    case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
+
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
+
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
+      return true;
+    default:
+      return false;
+  }
+}
+
+namespace llvm {
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  Module *Mod;
+
+  DominatorTree *DT;
+  PostDominatorTree *PDT;
+  UniformityInfo *UA;
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  void run(Function &F);
+};
+
+static bool IsValidInt(const Type *Ty) {
+  return Ty->isIntegerTy(1) ||
+         Ty->isIntegerTy(8) ||
+         Ty->isIntegerTy(16) ||
+         Ty->isIntegerTy(32) ||
+         Ty->isIntegerTy(64) ||
+         Ty->isIntegerTy(128);
+}
+
+static bool isShader(CallingConv::ID CC) {
+  switch(CC) {
+    case CallingConv::AMDGPU_VS:
+    case CallingConv::AMDGPU_LS:
+    case CallingConv::AMDGPU_HS:
+    case CallingConv::AMDGPU_ES:
+    case CallingConv::AMDGPU_GS:
+    case CallingConv::AMDGPU_PS:
+    case CallingConv::AMDGPU_CS_Chain:
+    case CallingConv::AMDGPU_CS_ChainPreserve:
+    case CallingConv::AMDGPU_CS:
+      return true;
+    default:
+      return false;
+  }
+}
+
+void AMDGPUTargetVerify::run(Function &F) {
+  // Ensure shader calling convention returns void
+  if (isShader(F.getCallingConv()))
+    Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
+
+  for (auto &BB : F) {
+
+    for (auto &I : BB) {
+      if (MarkUniform)
+        outs() << UA->isUniform(&I) << ' ' << I << '\n';
+
+      // Ensure integral types are valid: i8, i16, i32, i64, i128
+      if (I.getType()->isIntegerTy())
+        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
+      for (unsigned i = 0; i < I.getNumOperands(); ++i)
+        if (I.getOperand(i)->getType()->isIntegerTy())
+          Check(IsValidInt(I.getOperand(i)->getType()),
+                "Int type is invalid.", I.getOperand(i));
+
+      // Ensure no store to const memory
+      if (auto *SI = dyn_cast<StoreInst>(&I))
+      {
+        unsigned AS = SI->getPointerAddressSpace();
+        Check(AS != 4, "Write to const memory", SI);
+      }
+
+      // Ensure no kernel to kernel calls.
+      if (auto *CI = dyn_cast<CallInst>(&I))
+      {
+        CallingConv::ID CalleeCC = CI->getCallingConv();
+        if (CalleeCC == CallingConv::AMDGPU_KERNEL)
+        {
+          CallingConv::ID CallerCC = CI->getParent()->getParent()->getCallingConv();
+          Check(CallerCC != CallingConv::AMDGPU_KERNEL,
+            "A kernel may not call a kernel", CI->getParent()->getParent());
+        }
+      }
+
+      // Ensure MFMA is not in control flow with diverging operands
+      if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+        if (isMFMA(II->getIntrinsicID())) {
+          bool InControlFlow = false;
+          for (const auto &P : predecessors(&BB))
+            if (!PDT->dominates(&BB, P)) {
+              InControlFlow = true;
+              break;
+            }
+          for (const auto &S : successors(&BB))
+            if (!DT->dominates(&BB, S)) {
+              InControlFlow = true;
+              break;
+            }
+          if (InControlFlow) {
+            // If operands to MFMA are not uniform, MFMA cannot be in control flow
+            bool hasUniformOperands = true;
+            for (unsigned i = 0; i < II->getNumOperands(); i++) {
+              if (!UA->isUniform(II->getOperand(i))) {
+                dbgs() << "Not uniform: " << *II->getOperand(i) << '\n';
+                hasUniformOperands = false;
+              }
+            }
+            if (!hasUniformOperands) Check(false, "MFMA in control flow", II);
+            //else Check(false, "MFMA in control flow (uniform operands)", II);
+          }
+          //else Check(false, "MFMA not in control flow", II);
+        }
+      }
+    }
+  }
+}
+
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+
+  auto *Mod = F.getParent();
+
+  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  TV.run(F);
+
+  dbgs() << TV.MessagesStr.str();
+  if (!TV.MessagesStr.str().empty()) {
+    F.getParent()->IsValid = false;
+  }
+
+  return PreservedAnalyses::all();
+}
+} // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index c6d70ee39202e..a47b5f2732f91 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -111,6 +111,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUTargetMachine.cpp
   AMDGPUTargetObjectFile.cpp
   AMDGPUTargetTransformInfo.cpp
+  AMDGPUTargetVerifier.cpp
   AMDGPUWaitSGPRHazards.cpp
   AMDGPUUnifyDivergentExitNodes.cpp
   AMDGPUUnifyMetadata.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
new file mode 100644
index 0000000000000..f56ff992a56c2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -0,0 +1,62 @@
+; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
+
+define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg) #0 {
+; CHECK: Not uniform: %in.f32 = load <32 x float>, ptr addrspace(1) %gep, align 128
+; CHECK-NEXT: MFMA in control flow
+; CHECK-NEXT:   %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
+s:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
+  %in.i32 = load <32 x i32>, ptr addrspace(1) %gep
+  %in.f32 = load <32 x float>, ptr addrspace(1) %gep
+
+  %0 = icmp eq <32 x i32> %in.i32, zeroinitializer
+  %div.br = extractelement <32 x i1> %0, i32 0
+  br i1 %div.br, label %if.3, label %else.0
+
+if.3:
+  br label %join
+
+else.0:
+  %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
+  br label %join
+
+join:
+  ret void
+}
+
+define amdgpu_cs i32 @shader() {
+; CHECK: Shaders must return void
+  ret i32 0
+}
+
+define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
+; CHECK: Undefined behavior: Write to memory in const addrspace
+; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
+; CHECK-NEXT: Write to const memory
+; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
+  %r = add i32 %a, %b
+  store i32 %r, ptr addrspace(4) %out
+  ret void
+}
+
+define amdgpu_kernel void @kernel_callee(ptr %x) {
+  ret void
+}
+
+define amdgpu_kernel void @kernel_caller(ptr %x) {
+; CHECK: A kernel may not call a kernel
+; CHECK-NEXT: ptr @kernel_caller
+  call amdgpu_kernel void @kernel_callee(ptr %x)
+  ret void
+}
+
+
+; Function Attrs: nounwind
+define i65 @invalid_type(i65 %x) #0 {
+; CHECK: Int type is invalid.
+; CHECK-NEXT: %tmp2 = ashr i65 %x, 64
+entry:
+  %tmp2 = ashr i65 %x, 64
+  ret i65 %tmp2
+}
diff --git a/llvm/tools/llvm-tgt-verify/CMakeLists.txt b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
new file mode 100644
index 0000000000000..fe47c85e6cdce
--- /dev/null
+++ b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
@@ -0,0 +1,34 @@
+set(LLVM_LINK_COMPONENTS
+  AllTargetsAsmParsers
+  AllTargetsCodeGens
+  AllTargetsDescs
+  AllTargetsInfos
+  Analysis
+  AsmPrinter
+  CodeGen
+  CodeGenTypes
+  Core
+  IRPrinter
+  IRReader
+  MC
+  MIRParser
+  Passes
+  Remarks
+  ScalarOpts
+  SelectionDAG
+  Support
+  Target
+  TargetParser
+  TransformUtils
+  Vectorize
+  )
+
+add_llvm_tool(llvm-tgt-verify
+  llvm-tgt-verify.cpp
+
+  DEPENDS
+  intrinsics_gen
+  SUPPORT_PLUGINS
+  )
+
+export_executable_symbols_for_plugins(llc)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
new file mode 100644
index 0000000000000..68422abd6f4cc
--- /dev/null
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -0,0 +1,172 @@
+//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tool to fuzz instruction selection using libFuzzer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/InitializePasses.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/Lint.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/CodeGen/CommandFlags.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetVerifier.h"
+
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#define DEBUG_TYPE "isel-fuzzer"
+
+using namespace llvm;
+
+static codegen::RegisterCodeGenFlags CGF;
+
+static cl::opt<std::string>
+InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
+
+static cl::opt<bool>
+    StacktraceAbort("stacktrace-abort",
+        cl::desc("Turn on stacktrace"), cl::init(false));
+
+static cl::opt<bool>
+    NoLint("no-lint",
+        cl::desc("Turn off Lint"), cl::init(false));
+
+static cl::opt<bool>
+    NoVerify("no-verifier",
+        cl::desc("Turn off Verifier"), cl::init(false));
+
+static cl::opt<char>
+    OptLevel("O",
+             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
+                      "(default = '-O2')"),
+             cl::Prefix, cl::init('2'));
+
+static cl::opt<std::string>
+    TargetTriple("mtriple", cl::desc("Override target triple for module"));
+
+static std::unique_ptr<TargetMachine> TM;
+
+static void handleLLVMFatalError(void *, const char *Message, bool) {
+  if (StacktraceAbort) {
+    dbgs() << "LLVM ERROR: " << Message << "\n"
+           << "Aborting.\n";
+    abort();
+  }
+}
+
+int main(int argc, char **argv) {
+  StringRef ExecName = argv[0];
+  InitLLVM X(argc, argv);
+
+  InitializeAllTargets();
+  InitializeAllTargetMCs();
+  InitializeAllAsmPrinters();
+  InitializeAllAsmParsers();
+
+  PassRegistry *Registry = PassRegistry::getPassRegistry();
+  initializeCore(*Registry);
+  initializeCodeGen(*Registry);
+  initializeAnalysis(*Registry);
+  initializeTarget(*Registry);
+
+  cl::ParseCommandLineOptions(argc, argv);
+
+  if (TargetTriple.empty()) {
+    errs() << ExecName << ": -mtriple must be specified\n";
+    exit(1);
+  }
+
+  CodeGenOptLevel OLvl;
+  if (auto Level = CodeGenOpt::parseLevel(OptLevel)) {
+    OLvl = *Level;
+  } else {
+    errs() << ExecName << ": invalid optimization level.\n";
+    return 1;
+  }
+  ExitOnError ExitOnErr(std::string(ExecName) + ": error:");
+  TM = ExitOnErr(codegen::createTargetMachineForTriple(
+      Triple::normalize(TargetTriple), OLvl));
+  assert(TM && "Could not allocate target machine!");
+
+  // Make sure we print the summary and the current unit when LLVM errors out.
+  install_fatal_error_handler(handleLLVMFatalError, nullptr);
+
+  LLVMContext Context;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
+  if (!M) {
+    errs() << "Invalid mod\n";
+    return 1;
+  }
+  auto S = Triple::normalize(TargetTriple);
+  M->setTargetTriple(S);
+
+  PassInstrumentationCallbacks PIC;
+  StandardInstrumentations SI(Context, false/*debug PM*/,
+                              false);
+  registerCodeGenCallback(PIC, *TM);
+
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+  //TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
+
+  MachineFunctionAnalysisManager MFAM;
+  LoopAnalysisManager LAM;
+  FunctionAnalysisManager FAM;
+  CGSCCAnalysisManager CGAM;
+  ModuleAnalysisManager MAM;
+  PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.registerMachineFunctionAnalyses(MFAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
+
+  SI.registerCallbacks(PIC, &MAM);
+
+  //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
+
+  Triple TT(M->getTargetTriple());
+  if (!NoLint)
+    FPM.addPass(LintPass());
+  if (!NoVerify)
+    MPM.addPass(VerifierPass());
+  if (TT.isAMDGPU())
+    FPM.addPass(AMDGPUTargetVerifierPass());
+  else if (false) {} // ...
+  else
+    FPM.addPass(TargetVerifierPass());
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
+  MPM.run(*M, MAM);
+
+  if (!M->IsValid)
+    return 1;
+
+  return 0;
+}

>From 3b69ade6d8880cadc883698b64a06bd9207d084b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 3 Feb 2025 07:15:12 -0800
Subject: [PATCH 02/33] Add hook for target verifier in llc,opt

---
 .../llvm/Passes/StandardInstrumentations.h    |  6 ++++--
 llvm/include/llvm/Target/TargetVerifier.h     |  1 +
 .../TargetVerify/AMDGPUTargetVerifier.h       | 18 ++++++++++++++++++
 llvm/lib/LTO/LTOBackend.cpp                   |  2 +-
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp         |  2 +-
 llvm/lib/Passes/CMakeLists.txt                |  1 +
 llvm/lib/Passes/PassBuilderBindings.cpp       |  2 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  | 19 +++++++++++++++----
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 12 ++++++------
 llvm/lib/Target/CMakeLists.txt                |  2 ++
 .../CodeGen/AMDGPU/tgt-verify-llc-fail.ll     |  6 ++++++
 .../CodeGen/AMDGPU/tgt-verify-llc-pass.ll     |  6 ++++++
 llvm/tools/llc/NewPMDriver.cpp                |  2 +-
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 +-
 llvm/tools/opt/NewPMDriver.cpp                |  2 +-
 llvm/unittests/IR/PassManagerTest.cpp         |  6 +++---
 16 files changed, 68 insertions(+), 21 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll

diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 4ee5ab2554868..7d7b858b447cc 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,7 +478,8 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM);
+                         ModuleAnalysisManager *MAM,
+			 FunctionAnalysisManager *FAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -624,7 +625,8 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM = nullptr);
+                         ModuleAnalysisManager *MAM,
+			 FunctionAnalysisManager *FAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index e00c6a7b260c9..ad5aeb895953d 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -75,6 +75,7 @@ class TargetVerify {
         MessagesStr(Messages) {}
 
   void run(Function &F) {};
+  void run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index e6ff57629b141..d8a3fda4f87dc 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -22,6 +22,10 @@
 
 #include "llvm/Target/TargetVerifier.h"
 
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/IR/Dominators.h"
+
 namespace llvm {
 
 class Function;
@@ -31,6 +35,20 @@ class AMDGPUTargetVerifierPass : public TargetVerifierPass {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  Module *Mod;
+
+  DominatorTree *DT;
+  PostDominatorTree *PDT;
+  UniformityInfo *UA;
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  void run(Function &F);
+};
+
 } // namespace llvm
 
 #endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 5e8cd12fe040b..7ae23ef4e7df4 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -273,7 +273,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
                               Conf.VerifyEach);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 
   RegisterPassPlugins(Conf.PassPlugins, PB);
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 5b333cda31fb8..8db92e70d5a96 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -241,7 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PipelineTuningOptions PTO;
   PTO.LoopVectorization = true;
   PTO.SLPVectorization = true;
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 91c8c4f67074d..e5912f23903b6 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,6 +30,7 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
+  TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index 933fe89e53a94..f0e1abb8cebc4 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -76,7 +76,7 @@ static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   // Run the pipeline.
   if (Fun) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 4e8abc43572d7..e2764b162fddb 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,6 +40,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
+#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -1449,9 +1450,10 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 }
 
 void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                              ModuleAnalysisManager *MAM) {
+                                              ModuleAnalysisManager *MAM,
+					      FunctionAnalysisManager *FAM) {
   PIC.registerAfterPassCallback(
-      [this, MAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+      [this, MAM, FAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
         const auto *F = unwrapIR<Function>(IR);
@@ -1468,6 +1470,15 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
             report_fatal_error(formatv("Broken function found after pass "
                                        "\"{0}\", compilation aborted!",
                                        P));
+
+          if (FAM) {
+            TargetVerify TV(const_cast<Module*>(F->getParent()));
+            TV.run(*const_cast<Function*>(F), *FAM);
+	    if (!F->getParent()->IsValid)
+              report_fatal_error(formatv("Broken function found after pass "
+                                         "\"{0}\", compilation aborted!",
+                                         P));
+          }
         } else {
           const auto *M = unwrapIR<Module>(IR);
           if (!M) {
@@ -2507,7 +2518,7 @@ void PrintCrashIRInstrumentation::registerCallbacks(
 }
 
 void StandardInstrumentations::registerCallbacks(
-    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM) {
+    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM, FunctionAnalysisManager *FAM) {
   PrintIR.registerCallbacks(PIC);
   PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
@@ -2516,7 +2527,7 @@ void StandardInstrumentations::registerCallbacks(
   PrintChangedIR.registerCallbacks(PIC);
   PseudoProbeVerification.registerCallbacks(PIC);
   if (VerifyEach)
-    Verify.registerCallbacks(PIC, MAM);
+    Verify.registerCallbacks(PIC, MAM, FAM);
   PrintChangedDiff.registerCallbacks(PIC);
   WebsiteChangeReporter.registerCallbacks(PIC);
   ChangeTester.registerCallbacks(PIC);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 585b19065c142..e6cdec7160229 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -14,8 +14,8 @@
 
 using namespace llvm;
 
-static cl::opt<bool>
-MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
+//static cl::opt<bool>
+//MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
 
 // Check - We know that cond should be true, if not print an error message.
 #define Check(C, ...)                                                          \
@@ -81,7 +81,7 @@ static bool isMFMA(unsigned IID) {
 }
 
 namespace llvm {
-class AMDGPUTargetVerify : public TargetVerify {
+/*class AMDGPUTargetVerify : public TargetVerify {
 public:
   Module *Mod;
 
@@ -93,7 +93,7 @@ class AMDGPUTargetVerify : public TargetVerify {
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   void run(Function &F);
-};
+};*/
 
 static bool IsValidInt(const Type *Ty) {
   return Ty->isIntegerTy(1) ||
@@ -129,8 +129,8 @@ void AMDGPUTargetVerify::run(Function &F) {
   for (auto &BB : F) {
 
     for (auto &I : BB) {
-      if (MarkUniform)
-        outs() << UA->isUniform(&I) << ' ' << I << '\n';
+      //if (MarkUniform)
+        //outs() << UA->isUniform(&I) << ' ' << I << '\n';
 
       // Ensure integral types are valid: i8, i16, i32, i64, i128
       if (I.getType()->isIntegerTy())
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index 9472288229cac..f2a5d545ce84f 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -7,6 +7,8 @@ add_llvm_component_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
+  TargetVerifier.cpp
+  AMDGPU/AMDGPUTargetVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Target
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
new file mode 100644
index 0000000000000..584097d7bc134
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -0,0 +1,6 @@
+; RUN: not not llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each -o - < %s 2>&1 | FileCheck %s
+
+define amdgpu_cs i32 @nonvoid_shader() {
+; CHECK: LLVM ERROR
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
new file mode 100644
index 0000000000000..0c3a5fe5ac4a5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -0,0 +1,6 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each %s -o - 2>&1 | FileCheck %s
+
+define amdgpu_cs void @void_shader() {
+; CHECK: ModuleToFunctionPassAdaptor
+  ret void
+}
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index fa82689ecf9ae..a060d16e74958 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -126,7 +126,7 @@ int llvm::compileModuleWithNewPM(
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 68422abd6f4cc..3352d07deff2f 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -147,7 +147,7 @@ int main(int argc, char **argv) {
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
 
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index 7d168a6ceb17c..a8977d80bdf44 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -423,7 +423,7 @@ bool llvm::runPassPipeline(
   PrintPassOpts.SkipAnalyses = DebugPM == DebugLogging::Quiet;
   StandardInstrumentations SI(M.getContext(), DebugPM != DebugLogging::None,
                               VK == VerifierKind::EachPass, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   DebugifyEachInstrumentation Debugify;
   DebugifyStatsMap DIStatsMap;
   DebugInfoPerPass DebugInfoBeforePass;
diff --git a/llvm/unittests/IR/PassManagerTest.cpp b/llvm/unittests/IR/PassManagerTest.cpp
index a6487169224c2..bb4db6120035f 100644
--- a/llvm/unittests/IR/PassManagerTest.cpp
+++ b/llvm/unittests/IR/PassManagerTest.cpp
@@ -828,7 +828,7 @@ TEST_F(PassManagerTest, FunctionPassCFGChecker) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -877,7 +877,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerInvalidateAnalysis) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -945,7 +945,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerWrapped) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });

>From a89ff83fbb4a39d6e91ae3bccbb6244c32433a4b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 16 Apr 2025 10:19:00 -0400
Subject: [PATCH 03/33] Run AMDGPUTargetVerifier within AMDGPU pipeline. Move
 IsValid from Module to TargetVerify.

---
 clang/lib/CodeGen/BackendUtil.cpp               | 2 +-
 llvm/include/llvm/IR/Module.h                   | 4 ----
 llvm/include/llvm/Target/TargetVerifier.h       | 2 ++
 llvm/lib/IR/Verifier.cpp                        | 4 ++--
 llvm/lib/Passes/StandardInstrumentations.cpp    | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  | 5 +++++
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 3 ++-
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp  | 6 +++---
 8 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 7e0a3cf5591ce..560129f871026 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -917,7 +917,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       TheModule->getContext(),
       (CodeGenOpts.DebugPassManager || DebugPassStructure),
       CodeGenOpts.VerifyEach, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
   // Handle the assignment tracking feature options.
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 3037b2c1644d9..34ac47a2b8be4 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,10 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module valid as determined by one of the verification passes
-  /// i.e. Lint, Verifier, TargetVerifier.
-  bool IsValid = true;
-
   /// Is this Module using intrinsics to record the position of debugging
   /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
   /// \ref BasicBlock.
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index ad5aeb895953d..2d0c039132c35 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -70,6 +70,8 @@ class TargetVerify {
   std::string Messages;
   raw_string_ostream MessagesStr;
 
+  bool IsValid = true;
+
   TargetVerify(Module *Mod)
       : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
         MessagesStr(Messages) {}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 18e9760f9c617..0054dd26327ae 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7870,7 +7870,7 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
   if (Res.IRBroken || Res.DebugInfoBroken) {
-    M.IsValid = false;
+    //M.IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken module found, compilation aborted!");
   }
@@ -7881,7 +7881,7 @@ PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
   if (res.IRBroken) {
-    F.getParent()->IsValid = false;
+    //F.getParent()->IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken function found, compilation aborted!");
   }
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index e2764b162fddb..2bc54cf7d4891 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -1474,7 +1474,7 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
           if (FAM) {
             TargetVerify TV(const_cast<Module*>(F->getParent()));
             TV.run(*const_cast<Function*>(F), *FAM);
-	    if (!F->getParent()->IsValid)
+	    if (!TV.IsValid)
               report_fatal_error(formatv("Broken function found after pass "
                                          "\"{0}\", compilation aborted!",
                                          P));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index d59087839b0e1..dc88042a66df9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,6 +90,7 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
@@ -1328,6 +1329,8 @@ void AMDGPUPassConfig::addIRPasses() {
       addPass(createLICMPass());
   }
 
+  //addPass(AMDGPUTargetVerifierPass());
+
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
@@ -2071,6 +2074,8 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   // but EarlyCSE can do neither of them.
   if (isPassEnabled(EnableScalarIRPasses))
     addEarlyCSEOrGVNPass(addPass);
+
+  addPass(AMDGPUTargetVerifierPass());
 }
 
 void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index e6cdec7160229..c70a6d1b6fa66 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -205,7 +205,8 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
 
   dbgs() << TV.MessagesStr.str();
   if (!TV.MessagesStr.str().empty()) {
-    F.getParent()->IsValid = false;
+    TV.IsValid = false;
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 3352d07deff2f..fbe7f6089ff18 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -163,9 +163,9 @@ int main(int argc, char **argv) {
     FPM.addPass(TargetVerifierPass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
-  MPM.run(*M, MAM);
-
-  if (!M->IsValid)
+  auto PA = MPM.run(*M, MAM);
+  auto PAC = PA.getChecker<VerifierAnalysis>();
+  if (!PAC.preserved())
     return 1;
 
   return 0;

>From df507f9bfc4caa5578b48be68fb6c362808ec4af Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 16 Apr 2025 11:08:20 -0400
Subject: [PATCH 04/33] Remove cmd line options that aren't required. Make
 error message explicit.

---
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll | 4 ++--
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index 584097d7bc134..c5e59d4a2369e 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,6 +1,6 @@
-; RUN: not not llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
-; CHECK: LLVM ERROR
+; CHECK: Shaders must return void
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index 0c3a5fe5ac4a5..8a503b7624a73 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each %s -o - 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
-; CHECK: ModuleToFunctionPassAdaptor
+; CHECK-NOT: Shaders must return void
   ret void
 }

>From 942312d57ea09d3dc9224d5cb3753390db13b2ba Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 19:36:34 -0400
Subject: [PATCH 05/33] Return Verifier none status through PreservedAnalyses
 on fail.

---
 llvm/lib/Analysis/Lint.cpp                     | 4 +++-
 llvm/lib/IR/Verifier.cpp                       | 2 ++
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 8 +++++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 1168005f48c0e..c8627e6db0692 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -735,9 +735,11 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) {
   Lint L(Mod, DL, AA, AC, DT, TLI);
   L.visit(F);
   dbgs() << L.MessagesStr.str();
-  if (AbortOnError && !L.MessagesStr.str().empty())
+  if (AbortOnError && !L.MessagesStr.str().empty()) {
     report_fatal_error(
         "linter found errors, aborting. (enabled by abort-on-error)", false);
+    return PreservedAnalyses::none();
+  }
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 0054dd26327ae..439823ba8b682 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7873,6 +7873,7 @@ PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
     //M.IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken module found, compilation aborted!");
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
@@ -7884,6 +7885,7 @@ PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
     //F.getParent()->IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken function found, compilation aborted!");
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index fbe7f6089ff18..042824ac37fea 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -164,9 +164,11 @@ int main(int argc, char **argv) {
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
   auto PA = MPM.run(*M, MAM);
-  auto PAC = PA.getChecker<VerifierAnalysis>();
-  if (!PAC.preserved())
-    return 1;
+  {
+    auto PAC = PA.getChecker<VerifierAnalysis>();
+    if (!PAC.preserved())
+      return 1;
+  }
 
   return 0;
 }

>From 6216a104469edfa1da128ce64be2cba2c2421b00 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 21:16:02 -0400
Subject: [PATCH 06/33] Rebase update.

---
 llvm/include/llvm/Target/TargetVerifier.h      | 2 +-
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 2d0c039132c35..fe683311b901c 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -73,7 +73,7 @@ class TargetVerify {
   bool IsValid = true;
 
   TargetVerify(Module *Mod)
-      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
+      : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
   void run(Function &F) {};
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 042824ac37fea..627bc51ef3a43 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -123,7 +123,7 @@ int main(int argc, char **argv) {
     return 1;
   }
   auto S = Triple::normalize(TargetTriple);
-  M->setTargetTriple(S);
+  M->setTargetTriple(Triple(S));
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Context, false/*debug PM*/,
@@ -153,7 +153,7 @@ int main(int argc, char **argv) {
 
   Triple TT(M->getTargetTriple());
   if (!NoLint)
-    FPM.addPass(LintPass());
+    FPM.addPass(LintPass(false));
   if (!NoVerify)
     MPM.addPass(VerifierPass());
   if (TT.isAMDGPU())

>From 4b7068f49c24f28d4fac762ea6f1f8420028d525 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 22:49:52 -0400
Subject: [PATCH 07/33] Add generic TargetVerifier.

---
 llvm/lib/Target/TargetVerifier.cpp | 32 ++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 llvm/lib/Target/TargetVerifier.cpp

diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
new file mode 100644
index 0000000000000..de3ff749e7c3c
--- /dev/null
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -0,0 +1,32 @@
+#include "llvm/Target/TargetVerifier.h"
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+
+namespace llvm {
+
+void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
+  if (TT.isAMDGPU()) {
+    auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
+    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+    }
+  }
+}
+
+} // namespace llvm

>From 86e806635cdf1f1606af654817aefa8d797b8cf5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:13:14 -0400
Subject: [PATCH 08/33] Remove store to const check since it is in Lint already

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 8 --------
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll          | 2 --
 2 files changed, 10 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index c70a6d1b6fa66..1cf2b277bee26 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -140,14 +140,6 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(IsValidInt(I.getOperand(i)->getType()),
                 "Int type is invalid.", I.getOperand(i));
 
-      // Ensure no store to const memory
-      if (auto *SI = dyn_cast<StoreInst>(&I))
-      {
-        unsigned AS = SI->getPointerAddressSpace();
-        Check(AS != 4, "Write to const memory", SI);
-      }
-
-      // Ensure no kernel to kernel calls.
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
         CallingConv::ID CalleeCC = CI->getCallingConv();
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index f56ff992a56c2..c628abbde11d1 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -32,8 +32,6 @@ define amdgpu_cs i32 @shader() {
 
 define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
 ; CHECK: Undefined behavior: Write to memory in const addrspace
-; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
-; CHECK-NEXT: Write to const memory
 ; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
   %r = add i32 %a, %b
   store i32 %r, ptr addrspace(4) %out

>From 17fc5c6be3406d4360e25b2de595d31afe7f77bc Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:14:38 -0400
Subject: [PATCH 09/33] Add chain followed by unreachable check

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp |  6 ++++++
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll          | 10 ++++++++++
 2 files changed, 16 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 1cf2b277bee26..8ea773bc0e66f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -142,6 +142,7 @@ void AMDGPUTargetVerify::run(Function &F) {
 
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
+        // Ensure no kernel to kernel calls.
         CallingConv::ID CalleeCC = CI->getCallingConv();
         if (CalleeCC == CallingConv::AMDGPU_KERNEL)
         {
@@ -149,6 +150,11 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(CallerCC != CallingConv::AMDGPU_KERNEL,
             "A kernel may not call a kernel", CI->getParent()->getParent());
         }
+
+        // Ensure chain intrinsics are followed by unreachables.
+        if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
+          Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
+            "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
 
       // Ensure MFMA is not in control flow with diverging operands
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index c628abbde11d1..e620df94ccde4 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -58,3 +58,13 @@ entry:
   %tmp2 = ashr i65 %x, 64
   ret i65 %tmp2
 }
+
+declare void @llvm.amdgcn.cs.chain.v3i32(ptr, i32, <3 x i32>, <3 x i32>, i32, ...)
+declare amdgpu_cs_chain void @chain_callee(<3 x i32> inreg, <3 x i32>)
+
+define amdgpu_cs void @no_unreachable(<3 x i32> inreg %a, <3 x i32> %b) {
+; CHECK: llvm.amdgcn.cs.chain must be followed by unreachable
+; CHECK-NEXT: call void (ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.v3i32.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  ret void
+}

>From 46e662cce2d1f3716fcf5c23be39fdb27a2d7ecb Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:26:19 -0400
Subject: [PATCH 10/33] Remove mfma check

---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 89 -------------------
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        | 25 ------
 2 files changed, 114 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 8ea773bc0e66f..684ced5bba574 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -14,9 +14,6 @@
 
 using namespace llvm;
 
-//static cl::opt<bool>
-//MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
-
 // Check - We know that cond should be true, if not print an error message.
 #define Check(C, ...)                                                          \
   do {                                                                         \
@@ -26,60 +23,6 @@ using namespace llvm;
     }                                                                          \
   } while (false)
 
-static bool isMFMA(unsigned IID) {
-  switch (IID) {
-    case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
-    case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
-    case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
-    case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
-    case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
-    case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
-    case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
-    case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
-    case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
-    case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
-    case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
-
-    case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
-    case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
-
-    case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
-    case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
-
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
-
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
-      return true;
-    default:
-      return false;
-  }
-}
-
 namespace llvm {
 /*class AMDGPUTargetVerify : public TargetVerify {
 public:
@@ -129,8 +72,6 @@ void AMDGPUTargetVerify::run(Function &F) {
   for (auto &BB : F) {
 
     for (auto &I : BB) {
-      //if (MarkUniform)
-        //outs() << UA->isUniform(&I) << ' ' << I << '\n';
 
       // Ensure integral types are valid: i8, i16, i32, i64, i128
       if (I.getType()->isIntegerTy())
@@ -156,36 +97,6 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
             "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
-
-      // Ensure MFMA is not in control flow with diverging operands
-      if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
-        if (isMFMA(II->getIntrinsicID())) {
-          bool InControlFlow = false;
-          for (const auto &P : predecessors(&BB))
-            if (!PDT->dominates(&BB, P)) {
-              InControlFlow = true;
-              break;
-            }
-          for (const auto &S : successors(&BB))
-            if (!DT->dominates(&BB, S)) {
-              InControlFlow = true;
-              break;
-            }
-          if (InControlFlow) {
-            // If operands to MFMA are not uniform, MFMA cannot be in control flow
-            bool hasUniformOperands = true;
-            for (unsigned i = 0; i < II->getNumOperands(); i++) {
-              if (!UA->isUniform(II->getOperand(i))) {
-                dbgs() << "Not uniform: " << *II->getOperand(i) << '\n';
-                hasUniformOperands = false;
-              }
-            }
-            if (!hasUniformOperands) Check(false, "MFMA in control flow", II);
-            //else Check(false, "MFMA in control flow (uniform operands)", II);
-          }
-          //else Check(false, "MFMA not in control flow", II);
-        }
-      }
     }
   }
 }
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index e620df94ccde4..62b220d7d9f49 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -1,30 +1,5 @@
 ; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
 
-define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg) #0 {
-; CHECK: Not uniform: %in.f32 = load <32 x float>, ptr addrspace(1) %gep, align 128
-; CHECK-NEXT: MFMA in control flow
-; CHECK-NEXT:   %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
-s:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
-  %in.i32 = load <32 x i32>, ptr addrspace(1) %gep
-  %in.f32 = load <32 x float>, ptr addrspace(1) %gep
-
-  %0 = icmp eq <32 x i32> %in.i32, zeroinitializer
-  %div.br = extractelement <32 x i1> %0, i32 0
-  br i1 %div.br, label %if.3, label %else.0
-
-if.3:
-  br label %join
-
-else.0:
-  %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
-  br label %join
-
-join:
-  ret void
-}
-
 define amdgpu_cs i32 @shader() {
 ; CHECK: Shaders must return void
   ret i32 0

>From 7e471c9111d6a3a33f0de6bac883353669a69b0a Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 21 Apr 2025 20:54:10 -0400
Subject: [PATCH 11/33] Add registerVerifierPasses to PassBuilder and add the
 verifier passes to PassRegistry.

---
 llvm/include/llvm/InitializePasses.h          |  1 +
 llvm/include/llvm/Passes/PassBuilder.h        | 21 +++++++
 .../llvm/Passes/TargetPassRegistry.inc        | 12 ++++
 .../TargetVerify/AMDGPUTargetVerifier.h       | 11 ++--
 llvm/lib/Passes/PassBuilder.cpp               |  7 +++
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 11 ++++
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 56 ++++++++++++++++++-
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  1 +
 8 files changed, 114 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 1b5b1d5888824..662f6b71aa46a 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,6 +333,7 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
+LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index f13b5c678a894..c134f2d0c1112 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -173,6 +173,13 @@ class PassBuilder {
   /// additional analyses.
   LLVM_ABI void registerLoopAnalyses(LoopAnalysisManager &LAM);
 
+  /// Registers all available verifier passes.
+  ///
+  /// This is an interface that can be used to populate a
+  /// \c ModuleAnalysisManager with all registered loop analyses. Callers can
+  /// still manually register any additional analyses.
+  void registerVerifierPasses(ModulePassManager &PM, FunctionPassManager &);
+
   /// Registers all available machine function analysis passes.
   ///
   /// This is an interface that can be used to populate a \c
@@ -576,6 +583,15 @@ class PassBuilder {
   }
   /// @}}
 
+  /// Register a callback for parsing an Verifier Name to populate
+  /// the given managers.
+  void registerVerifierCallback(
+      const std::function<bool(ModulePassManager &MPM)> &C,
+      const std::function<bool(FunctionPassManager &MPM)> &CF) {
+    VerifierCallbacks.push_back(C);
+    FnVerifierCallbacks.push_back(CF);
+  }
+
   /// {{@ Register pipeline parsing callbacks with this pass builder instance.
   /// Using these callbacks, callers can parse both a single pass name, as well
   /// as entire sub-pipelines, and populate the PassManager instance
@@ -851,6 +867,11 @@ class PassBuilder {
   // Callbacks to parse `filter` parameter in register allocation passes
   SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
+  // Verifier callbacks
+  SmallVector<std::function<bool(ModulePassManager &)>, 2>
+      VerifierCallbacks;
+  SmallVector<std::function<bool(FunctionPassManager &)>, 2>
+      FnVerifierCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc
index 521913cb25a4a..2d04b874cf360 100644
--- a/llvm/include/llvm/Passes/TargetPassRegistry.inc
+++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc
@@ -151,6 +151,18 @@ PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM,
   return false;
 });
 
+PB.registerVerifierCallback([](ModulePassManager &PM) {
+#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS) PM.addPass(CREATE_PASS)
+#include GET_PASS_REGISTRY
+#undef VERIFIER_MODULE_ANALYSIS
+  return false;
+}, [](FunctionPassManager &FPM) {
+#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS) FPM.addPass(CREATE_PASS)
+#include GET_PASS_REGISTRY
+#undef VERIFIER_FUNCTION_ANALYSIS
+  return false;
+});
+
 #undef ADD_PASS
 #undef ADD_PASS_WITH_PARAMS
 
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index d8a3fda4f87dc..b6a7412e8c1ef 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -39,14 +39,17 @@ class AMDGPUTargetVerify : public TargetVerify {
 public:
   Module *Mod;
 
-  DominatorTree *DT;
-  PostDominatorTree *PDT;
-  UniformityInfo *UA;
+  DominatorTree *DT = nullptr;
+  PostDominatorTree *PDT = nullptr;
+  UniformityInfo *UA = nullptr;
+
+  AMDGPUTargetVerify(Module *Mod)
+    : TargetVerify(Mod), Mod(Mod) {}
 
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
-  void run(Function &F);
+  bool run(Function &F);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 4603eaff8ade9..561996d0c8980 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -576,6 +576,13 @@ void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     C(LAM);
 }
 
+void PassBuilder::registerVerifierPasses(ModulePassManager &MPM, FunctionPassManager &FPM) {
+  for (auto &C : VerifierCallbacks)
+    C(MPM);
+  for (auto &C : FnVerifierCallbacks)
+    C(FPM);
+}
+
 static std::optional<std::pair<bool, bool>>
 parseFunctionPipelineName(StringRef Name) {
   std::pair<bool, bool> Params;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 13453963eec6d..652cd23f62443 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -82,6 +82,17 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
+#ifndef VERIFIER_MODULE_ANALYSIS
+#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS)
+#endif
+#ifndef VERIFIER_FUNCTION_ANALYSIS
+#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
+#endif
+VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
+#undef VERIFIER_MODULE_ANALYSIS
+#undef VERIFIER_FUNCTION_ANALYSIS
+
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 684ced5bba574..63a7526b9abdc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -5,6 +5,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -19,7 +20,7 @@ using namespace llvm;
   do {                                                                         \
     if (!(C)) {                                                                \
       TargetVerify::CheckFailed(__VA_ARGS__);                                  \
-      return;                                                                  \
+      return false;                                                                  \
     }                                                                          \
   } while (false)
 
@@ -64,7 +65,7 @@ static bool isShader(CallingConv::ID CC) {
   }
 }
 
-void AMDGPUTargetVerify::run(Function &F) {
+bool AMDGPUTargetVerify::run(Function &F) {
   // Ensure shader calling convention returns void
   if (isShader(F.getCallingConv()))
     Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
@@ -99,6 +100,10 @@ void AMDGPUTargetVerify::run(Function &F) {
       }
     }
   }
+
+  if (!MessagesStr.str().empty())
+    return false;
+  return true;
 }
 
 PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
@@ -120,4 +125,51 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
 
   return PreservedAnalyses::all();
 }
+
+struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<AMDGPUTargetVerify> TV;
+  bool FatalErrors = true;
+
+  AMDGPUTargetVerifierLegacyPass() : FunctionPass(ID) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
+      : FunctionPass(ID),
+        FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<AMDGPUTargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (TV->run(F) && FatalErrors) {
+      errs() << "in function " << F.getName() << '\n';
+      report_fatal_error("Broken function found, compilation aborted!");
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    //IsValid &= TV->run();
+    if (FatalErrors && !IsValid)
+      report_fatal_error("Broken module found, compilation aborted!");
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char AMDGPUTargetVerifierLegacyPass::ID = 0;
 } // namespace llvm
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverify", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 627bc51ef3a43..503db7b1f8d18 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -144,6 +144,7 @@ int main(int argc, char **argv) {
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
+  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 

>From 1538985ecdd840e68207f83ca7b159bccef98ff5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Tue, 22 Apr 2025 14:57:31 -0400
Subject: [PATCH 12/33] Remove leftovers. Add titles. Add call to
 registerVerifierCallbacks in llc.

---
 llvm/lib/Passes/CMakeLists.txt                |  2 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  4 ---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 35 +++++++++++--------
 llvm/lib/Target/TargetVerifier.cpp            | 19 ++++++++++
 llvm/tools/llc/NewPMDriver.cpp                |  6 ++--
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  7 ++--
 6 files changed, 45 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index e5912f23903b6..b8d25b15186e8 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,7 +30,7 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
-  TargetParser
+  #TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index dc88042a66df9..0c41c633dcb3d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1329,8 +1329,6 @@ void AMDGPUPassConfig::addIRPasses() {
       addPass(createLICMPass());
   }
 
-  //addPass(AMDGPUTargetVerifierPass());
-
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
@@ -2074,8 +2072,6 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   // but EarlyCSE can do neither of them.
   if (isPassEnabled(EnableScalarIRPasses))
     addEarlyCSEOrGVNPass(addPass);
-
-  addPass(AMDGPUTargetVerifierPass());
 }
 
 void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 63a7526b9abdc..0eecedaebc7ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -1,3 +1,22 @@
+//===-- AMDGPUTargetVerifier.cpp - AMDGPU -------------------------*- C++ -*-===//
+////
+//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//// See https://llvm.org/LICENSE.txt for license information.
+//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+////
+////===----------------------------------------------------------------------===//
+////
+//// This file defines target verifier interfaces that can be used for some
+//// validation of input to the system, and for checking that transformations
+//// haven't done something bad. In contrast to the Verifier or Lint, the
+//// TargetVerifier looks for constructions invalid to a particular target
+//// machine.
+////
+//// To see what specifically is checked, look at an individual backend's
+//// TargetVerifier.
+////
+////===----------------------------------------------------------------------===//
+
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
 #include "llvm/Analysis/UniformityAnalysis.h"
@@ -25,19 +44,6 @@ using namespace llvm;
   } while (false)
 
 namespace llvm {
-/*class AMDGPUTargetVerify : public TargetVerify {
-public:
-  Module *Mod;
-
-  DominatorTree *DT;
-  PostDominatorTree *PDT;
-  UniformityInfo *UA;
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
-    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
-  void run(Function &F);
-};*/
 
 static bool IsValidInt(const Type *Ty) {
   return Ty->isIntegerTy(1) ||
@@ -147,7 +153,7 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   }
 
   bool runOnFunction(Function &F) override {
-    if (TV->run(F) && FatalErrors) {
+    if (!TV->run(F) && FatalErrors) {
       errs() << "in function " << F.getName() << '\n';
       report_fatal_error("Broken function found, compilation aborted!");
     }
@@ -160,7 +166,6 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
       if (F.isDeclaration())
         IsValid &= TV->run(F);
 
-    //IsValid &= TV->run();
     if (FatalErrors && !IsValid)
       report_fatal_error("Broken module found, compilation aborted!");
     return false;
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index de3ff749e7c3c..992a0c91d93b1 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -1,3 +1,22 @@
+//===-- TargetVerifier.cpp - LLVM IR Target Verifier ----------------*- C++ -*-===//
+////
+///// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+///// See https://llvm.org/LICENSE.txt for license information.
+///// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+/////
+/////===----------------------------------------------------------------------===//
+/////
+///// This file defines target verifier interfaces that can be used for some
+///// validation of input to the system, and for checking that transformations
+///// haven't done something bad. In contrast to the Verifier or Lint, the
+///// TargetVerifier looks for constructions invalid to a particular target
+///// machine.
+/////
+///// To see what specifically is checked, look at TargetVerifier.cpp or an
+///// individual backend's TargetVerifier.
+/////
+/////===----------------------------------------------------------------------===//
+
 #include "llvm/Target/TargetVerifier.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index a060d16e74958..a8f6b999af06e 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -114,6 +114,8 @@ int llvm::compileModuleWithNewPM(
                               VK == VerifierKind::EachPass);
   registerCodeGenCallback(PIC, *Target);
 
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
   MachineFunctionAnalysisManager MFAM;
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
@@ -125,15 +127,13 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
+  PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
 
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
-
   if (!PassPipeline.empty()) {
     // Construct a custom pass pipeline that starts after instruction
     // selection.
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 503db7b1f8d18..b00bab66c6c3e 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -1,4 +1,4 @@
-//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
+//===--- llvm-tgt-verify.cpp - Target Verifier ----------------- ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Tool to fuzz instruction selection using libFuzzer.
+// Tool to verify a target.
 //
 //===----------------------------------------------------------------------===//
 
@@ -144,14 +144,11 @@ int main(int argc, char **argv) {
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
-  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
-  //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
-
   Triple TT(M->getTargetTriple());
   if (!NoLint)
     FPM.addPass(LintPass(false));

>From 32b0c0d5d74b3b78699b78736604e87369ca4217 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 24 Apr 2025 16:02:21 -0400
Subject: [PATCH 13/33] Add pass to legacy PM.

---
 llvm/include/llvm/CodeGen/Passes.h            |  2 +
 llvm/include/llvm/Target/TargetVerifier.h     |  6 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  |  4 +-
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  2 +-
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 45 ----------
 llvm/lib/Target/TargetVerifier.cpp            | 87 ++++++++++++++++++-
 llvm/tools/llc/NewPMDriver.cpp                |  6 +-
 llvm/tools/llc/llc.cpp                        |  4 +
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  1 +
 9 files changed, 105 insertions(+), 52 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 990452fa11fec..53bd70183e476 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -620,6 +620,8 @@ LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
+
+LLVM_ABI FunctionPass *createTargetVerifierLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index fe683311b901c..23ef2e0b8d4ef 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -30,7 +30,7 @@ class Function;
 
 class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 class TargetVerify {
@@ -76,8 +76,8 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  void run(Function &F) {};
-  void run(Function &F, FunctionAnalysisManager &AM);
+  bool run(Function &F);
+  bool run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 2bc54cf7d4891..539f4b24b5110 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -57,6 +57,8 @@ static cl::opt<bool> VerifyAnalysisInvalidation("verify-analysis-invalidation",
 #endif
 );
 
+static cl::opt<bool> VerifyTargetEach("verify-tgt-each");
+
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
 // of this option.  Note that this option has no effect without -print-changed.
@@ -1471,7 +1473,7 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
                                        "\"{0}\", compilation aborted!",
                                        P));
 
-          if (FAM) {
+          if (VerifyTargetEach && FAM) {
             TargetVerify TV(const_cast<Module*>(F->getParent()));
             TV.run(*const_cast<Function*>(F), *FAM);
 	    if (!TV.IsValid)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 652cd23f62443..ad4fa93d09e0c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -89,7 +89,7 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("tgtverifier", TargetVerifierPass())
 #undef VERIFIER_MODULE_ANALYSIS
 #undef VERIFIER_FUNCTION_ANALYSIS
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 0eecedaebc7ce..96bcaaf6f2ac9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -132,49 +132,4 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
   return PreservedAnalyses::all();
 }
 
-struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
-  static char ID;
-
-  std::unique_ptr<AMDGPUTargetVerify> TV;
-  bool FatalErrors = true;
-
-  AMDGPUTargetVerifierLegacyPass() : FunctionPass(ID) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
-      : FunctionPass(ID),
-        FatalErrors(FatalErrors) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool doInitialization(Module &M) override {
-    TV = std::make_unique<AMDGPUTargetVerify>(&M);
-    return false;
-  }
-
-  bool runOnFunction(Function &F) override {
-    if (!TV->run(F) && FatalErrors) {
-      errs() << "in function " << F.getName() << '\n';
-      report_fatal_error("Broken function found, compilation aborted!");
-    }
-    return false;
-  }
-
-  bool doFinalization(Module &M) override {
-    bool IsValid = true;
-    for (Function &F : M)
-      if (F.isDeclaration())
-        IsValid &= TV->run(F);
-
-    if (FatalErrors && !IsValid)
-      report_fatal_error("Broken module found, compilation aborted!");
-    return false;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-  }
-};
-char AMDGPUTargetVerifierLegacyPass::ID = 0;
 } // namespace llvm
-INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverify", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 992a0c91d93b1..170fc4769c1d8 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Target/TargetVerifier.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
+#include "llvm/InitializePasses.h"
 #include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Support/Debug.h"
@@ -32,7 +33,22 @@
 
 namespace llvm {
 
-void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
+bool TargetVerify::run(Function &F) {
+  if (TT.isAMDGPU()) {
+    AMDGPUTargetVerify TV(Mod);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+      return false;
+    }
+    return true;
+  }
+  report_fatal_error("Target has no verification method\n");
+}
+
+bool TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
   if (TT.isAMDGPU()) {
     auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
     auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
@@ -44,8 +60,77 @@ void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
     dbgs() << TV.MessagesStr.str();
     if (!TV.MessagesStr.str().empty()) {
       TV.IsValid = false;
+      return false;
+    }
+    return true;
+  }
+  report_fatal_error("Target has no verification method\n");
+}
+
+PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto TT = F.getParent()->getTargetTriple();
+
+  if (TT.isAMDGPU()) {
+    auto *Mod = F.getParent();
+
+    auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+      return PreservedAnalyses::none();
     }
+    return PreservedAnalyses::all();
   }
+  report_fatal_error("Target has no verification method\n");
 }
 
+struct TargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<TargetVerify> TV;
+
+  TargetVerifierLegacyPass() : FunctionPass(ID) {
+    initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<TargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (!TV->run(F)) {
+      errs() << "in function " << F.getName() << '\n';
+      report_fatal_error("broken function found, compilation aborted!");
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    if (!IsValid)
+      report_fatal_error("broken module found, compilation aborted!");
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char TargetVerifierLegacyPass::ID = 0;
+FunctionPass *createTargetVerifierLegacyPass() {
+  return new TargetVerifierLegacyPass();
+}
 } // namespace llvm
+using namespace llvm;
+INITIALIZE_PASS(TargetVerifierLegacyPass, "tgtverifier", "Target Verifier", false, false)
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index a8f6b999af06e..4b95977a10c5f 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -57,6 +57,9 @@ static cl::opt<bool>
     DebugPM("debug-pass-manager", cl::Hidden,
             cl::desc("Print pass management debugging information"));
 
+static cl::opt<bool> VerifyTarget("verify-tgt-new-pm",
+		                  cl::desc("Verify the target"));
+
 bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
   DiagnosticHandler::handleDiagnostics(DI);
   if (DI.getKind() == llvm::DK_SrcMgr) {
@@ -127,7 +130,8 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
-  PB.registerVerifierPasses(MPM, FPM);
+  if (VerifyTarget)
+    PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 140459ba2de21..1fd8a9f9cd9f8 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -209,6 +209,8 @@ static cl::opt<std::string> PassPipeline(
 static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
                                cl::desc("Alias for -passes"));
 
+static cl::opt<bool> VerifyTarget("verify-tgt", cl::desc("Verify the target"));
+
 namespace {
 
 std::vector<std::string> &getRunPassNames() {
@@ -658,6 +660,8 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
+  if (VerifyTarget)
+    PM.add(createTargetVerifierLegacyPass());
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index b00bab66c6c3e..b86c2318b45b7 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -141,6 +141,7 @@ int main(int argc, char **argv) {
   ModuleAnalysisManager MAM;
   PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
   PB.registerModuleAnalyses(MAM);
+  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);

>From 387b2d5936a699238ac775726b0b30860f68507f Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 00:13:42 -0400
Subject: [PATCH 14/33] Add fam in other projects.

---
 flang/lib/Frontend/FrontendActions.cpp      | 2 +-
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index d684eeb696755..c15151fc5c1b8 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -947,7 +947,7 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
 
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
-  si.registerCallbacks(pic, &mam);
+  si.registerCallbacks(pic, &mam, &fam);
   if (ci.isTimingEnabled())
     si.getTimePasses().setOutStream(ci.getTimingStreamLLVM());
   pto.LoopUnrolling = opts.UnrollLoops;
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index 0f58391c50667..f9664025f61f1 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -577,7 +577,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index 7117eaf4982b0..eae06d9f57467 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -851,7 +851,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index cb7b6cc8651c1..30ad79ef2fc58 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -970,7 +970,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 91b7191a07c6f..4a39bc33c5591 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1139,7 +1139,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Promote allocas to registers.

>From d2c9425f9e951768842ca9771148611bdba5e1bb Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 02:30:40 -0400
Subject: [PATCH 15/33] Avoid fatal errors in llc.

---
 llvm/lib/Target/TargetVerifier.cpp             | 18 +++++++++++++-----
 .../test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll |  2 +-
 .../test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll |  2 +-
 llvm/tools/llc/llc.cpp                         |  2 +-
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 170fc4769c1d8..3be50f4ef6da3 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -94,8 +94,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   static char ID;
 
   std::unique_ptr<TargetVerify> TV;
+  bool FatalErrors = false;
 
-  TargetVerifierLegacyPass() : FunctionPass(ID) {
+  TargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
+    FatalErrors(FatalErrors) {
     initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
@@ -107,7 +109,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   bool runOnFunction(Function &F) override {
     if (!TV->run(F)) {
       errs() << "in function " << F.getName() << '\n';
-      report_fatal_error("broken function found, compilation aborted!");
+      if (FatalErrors)
+        report_fatal_error("broken function found, compilation aborted!");
+      else
+        errs() << "broken function found, compilation aborted!\n";
     }
     return false;
   }
@@ -119,7 +124,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
         IsValid &= TV->run(F);
 
     if (!IsValid)
-      report_fatal_error("broken module found, compilation aborted!");
+      if (FatalErrors)
+        report_fatal_error("broken module found, compilation aborted!");
+      else
+        errs() << "broken module found, compilation aborted!\n";
     return false;
   }
 
@@ -128,8 +136,8 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   }
 };
 char TargetVerifierLegacyPass::ID = 0;
-FunctionPass *createTargetVerifierLegacyPass() {
-  return new TargetVerifierLegacyPass();
+FunctionPass *createTargetVerifierLegacyPass(bool FatalErrors) {
+  return new TargetVerifierLegacyPass(FatalErrors);
 }
 } // namespace llvm
 using namespace llvm;
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index c5e59d4a2369e..e2d9edda5d008 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
 ; CHECK: Shaders must return void
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index 8a503b7624a73..a2dab0ff47924 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm %s -o - 2>&1 | FileCheck %s --allow-empty
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
 ; CHECK-NOT: Shaders must return void
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 1fd8a9f9cd9f8..329d95826551f 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -661,7 +661,7 @@ static int compileModule(char **argv, LLVMContext &Context) {
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
   if (VerifyTarget)
-    PM.add(createTargetVerifierLegacyPass());
+    PM.add(createTargetVerifierLegacyPass(false));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {

>From 31a79a57d9d41365ffcd7b263b2a05153463b2f5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 03:40:47 -0400
Subject: [PATCH 16/33] Add tool to build/test.

---
 llvm/test/CMakeLists.txt                      |  1 +
 llvm/test/lit.cfg.py                          |  1 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |  1 +
 .../llvm/tools/llvm-tgt-verify/BUILD.gn       | 25 +++++++++++++++++++
 4 files changed, 28 insertions(+)
 create mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn

diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 2a6135da9a61e..a2bb112db8746 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -136,6 +136,7 @@ set(LLVM_TEST_DEPENDS
           llvm-strip
           llvm-symbolizer
           llvm-tblgen
+          llvm-tgt-verify
           llvm-readtapi
           llvm-tli-checker
           llvm-undname
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index bd6e37c848d8c..bf06e80dc7ae5 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -228,6 +228,7 @@ def get_asan_rtlib():
         "llvm-strings",
         "llvm-strip",
         "llvm-tblgen",
+        "llvm-tgt-verify",
         "llvm-readtapi",
         "llvm-undname",
         "llvm-windres",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 946b63f8a54fb..36cf956545caf 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -320,6 +320,7 @@ group("test") {
     "//llvm/tools/llvm-strings",
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-tli-checker",
+    "//llvm/tools/llvm-tgt-verify",
     "//llvm/tools/llvm-undname",
     "//llvm/tools/llvm-xray",
     "//llvm/tools/lto",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
new file mode 100644
index 0000000000000..b751bafc5052c
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tgtverifier("llvm-tgt-verify") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmPrinter",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGenTypes",
+    "//llvm/lib/Core",
+    "//llvm/lib/IRPrinter",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/MC",
+    "//llvm/lib/MIRParser",
+    "//llvm/lib/Passes",
+    "//llvm/lib/Remarks",
+    "//llvm/lib/ScalarOpts",
+    "//llvm/lib/SelectionDAG",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/TargetParser",
+    "//llvm/lib/TransformUtils",
+    "//llvm/lib/Vectorize",
+  ]
+  sources = [ "llvm-tgt-verify.cpp" ]
+}

>From b51b530c5672945e06a20b93b1f652d097386e61 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 28 Apr 2025 10:42:24 -0400
Subject: [PATCH 17/33] Cleanup of unrequired functions.

---
 llvm/include/llvm/Target/TargetVerifier.h     |  1 -
 .../TargetVerify/AMDGPUTargetVerifier.h       |  1 -
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 25 +++----------------
 llvm/lib/Target/TargetVerifier.cpp            | 22 ++--------------
 4 files changed, 6 insertions(+), 43 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 23ef2e0b8d4ef..427a05b2648a9 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -77,7 +77,6 @@ class TargetVerify {
         MessagesStr(Messages) {}
 
   bool run(Function &F);
-  bool run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index b6a7412e8c1ef..74e5b5f7a1efd 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -32,7 +32,6 @@ class Function;
 
 class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 class AMDGPUTargetVerify : public TargetVerify {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 96bcaaf6f2ac9..bda412f723242 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -107,29 +107,12 @@ bool AMDGPUTargetVerify::run(Function &F) {
     }
   }
 
-  if (!MessagesStr.str().empty())
+  //dbgs() << MessagesStr.str();
+  if (!MessagesStr.str().empty()) {
+    //IsValid = false;
     return false;
-  return true;
-}
-
-PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
-
-  auto *Mod = F.getParent();
-
-  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-  TV.run(F);
-
-  dbgs() << TV.MessagesStr.str();
-  if (!TV.MessagesStr.str().empty()) {
-    TV.IsValid = false;
-    return PreservedAnalyses::none();
   }
-
-  return PreservedAnalyses::all();
+  return true;
 }
 
 } // namespace llvm
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 3be50f4ef6da3..6b57c18ff9316 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -48,25 +48,6 @@ bool TargetVerify::run(Function &F) {
   report_fatal_error("Target has no verification method\n");
 }
 
-bool TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
-  if (TT.isAMDGPU()) {
-    auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
-    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return false;
-    }
-    return true;
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
 PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto TT = F.getParent()->getTargetTriple();
 
@@ -123,11 +104,12 @@ struct TargetVerifierLegacyPass : public FunctionPass {
       if (F.isDeclaration())
         IsValid &= TV->run(F);
 
-    if (!IsValid)
+    if (!IsValid) {
       if (FatalErrors)
         report_fatal_error("broken module found, compilation aborted!");
       else
         errs() << "broken module found, compilation aborted!\n";
+    }
     return false;
   }
 

>From d41d5f9e6a33b70f71a20d58642fd73c9327e1e5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 28 Apr 2025 10:43:32 -0400
Subject: [PATCH 18/33] Make virtual.

---
 llvm/include/llvm/Target/TargetVerifier.h                    | 2 +-
 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 427a05b2648a9..ade2676a64325 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -76,7 +76,7 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  bool run(Function &F);
+  virtual bool run(Function &F);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 74e5b5f7a1efd..b97fbc046e391 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -48,7 +48,7 @@ class AMDGPUTargetVerify : public TargetVerify {
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
-  bool run(Function &F);
+  bool run(Function &F) override;
 };
 
 } // namespace llvm

>From 7ece5e262a4f4aff350a511af5d33496ac2d5604 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 10:56:43 -0400
Subject: [PATCH 19/33] Remove from legacy PM. Add to target dependent
 pipeline.

---
 llvm/include/llvm/InitializePasses.h          |  2 +-
 llvm/include/llvm/Target/TargetVerifier.h     |  4 +-
 .../TargetVerify/AMDGPUTargetVerifier.h       |  1 +
 llvm/lib/Passes/StandardInstrumentations.cpp  | 10 +--
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  2 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  2 +
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 74 ++++++++++++++-
 llvm/lib/Target/TargetVerifier.cpp            | 90 -------------------
 llvm/tools/llc/llc.cpp                        |  2 -
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 -
 10 files changed, 84 insertions(+), 105 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 662f6b71aa46a..d718f92877276 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,7 +333,7 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
-LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+//LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index ade2676a64325..1d12eb55bbf0a 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -30,7 +30,7 @@ class Function;
 
 class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  virtual PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) = 0;
 };
 
 class TargetVerify {
@@ -76,7 +76,7 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  virtual bool run(Function &F);
+  virtual bool run(Function &F) = 0;
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index b97fbc046e391..49bcbc8849e3c 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -32,6 +32,7 @@ class Function;
 
 class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
 };
 
 class AMDGPUTargetVerify : public TargetVerify {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 539f4b24b5110..905ba667fac7a 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,7 +40,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
-#include "llvm/Target/TargetVerifier.h"
+//#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -1474,12 +1474,12 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
                                        P));
 
           if (VerifyTargetEach && FAM) {
-            TargetVerify TV(const_cast<Module*>(F->getParent()));
-            TV.run(*const_cast<Function*>(F), *FAM);
-	    if (!TV.IsValid)
+            //TargetVerify TV(const_cast<Module*>(F->getParent()));
+            //TV.run(*const_cast<Function*>(F), *FAM);
+	    /*if (!TV.IsValid)
               report_fatal_error(formatv("Broken function found after pass "
                                          "\"{0}\", compilation aborted!",
-                                         P));
+                                         P));*/
           }
         } else {
           const auto *M = unwrapIR<Module>(IR);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index ad4fa93d09e0c..652cd23f62443 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -89,7 +89,7 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("tgtverifier", TargetVerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
 #undef VERIFIER_MODULE_ANALYSIS
 #undef VERIFIER_FUNCTION_ANALYSIS
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 0c41c633dcb3d..718655b7ebf00 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -2010,6 +2010,8 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
 }
 
 void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
+  addPass(AMDGPUTargetVerifierPass());
+
   if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
     addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index bda412f723242..cedd9ddc78011 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -107,12 +107,82 @@ bool AMDGPUTargetVerify::run(Function &F) {
     }
   }
 
-  //dbgs() << MessagesStr.str();
+  dbgs() << MessagesStr.str();
   if (!MessagesStr.str().empty()) {
-    //IsValid = false;
+    IsValid = false;
     return false;
   }
   return true;
 }
 
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto *Mod = F.getParent();
+
+  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  TV.run(F);
+
+  dbgs() << TV.MessagesStr.str();
+  if (!TV.MessagesStr.str().empty()) {
+    TV.IsValid = false;
+    return PreservedAnalyses::none();
+  }
+  return PreservedAnalyses::all();
+}
+
+/*
+struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<AMDGPUTargetVerify> TV;
+  bool FatalErrors = false;
+
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
+    FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<AMDGPUTargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (!TV->run(F)) {
+      errs() << "in function " << F.getName() << '\n';
+      if (FatalErrors)
+        report_fatal_error("broken function found, compilation aborted!");
+      else
+        errs() << "broken function found, compilation aborted!\n";
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    if (!IsValid) {
+      if (FatalErrors)
+        report_fatal_error("broken module found, compilation aborted!");
+      else
+        errs() << "broken module found, compilation aborted!\n";
+    }
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char AMDGPUTargetVerifierLegacyPass::ID = 0;
+FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
+  return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
+}*/
 } // namespace llvm
+//INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 6b57c18ff9316..c63ae2a2c5daf 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -33,94 +33,4 @@
 
 namespace llvm {
 
-bool TargetVerify::run(Function &F) {
-  if (TT.isAMDGPU()) {
-    AMDGPUTargetVerify TV(Mod);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return false;
-    }
-    return true;
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
-PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
-  auto TT = F.getParent()->getTargetTriple();
-
-  if (TT.isAMDGPU()) {
-    auto *Mod = F.getParent();
-
-    auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return PreservedAnalyses::none();
-    }
-    return PreservedAnalyses::all();
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
-struct TargetVerifierLegacyPass : public FunctionPass {
-  static char ID;
-
-  std::unique_ptr<TargetVerify> TV;
-  bool FatalErrors = false;
-
-  TargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
-    FatalErrors(FatalErrors) {
-    initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool doInitialization(Module &M) override {
-    TV = std::make_unique<TargetVerify>(&M);
-    return false;
-  }
-
-  bool runOnFunction(Function &F) override {
-    if (!TV->run(F)) {
-      errs() << "in function " << F.getName() << '\n';
-      if (FatalErrors)
-        report_fatal_error("broken function found, compilation aborted!");
-      else
-        errs() << "broken function found, compilation aborted!\n";
-    }
-    return false;
-  }
-
-  bool doFinalization(Module &M) override {
-    bool IsValid = true;
-    for (Function &F : M)
-      if (F.isDeclaration())
-        IsValid &= TV->run(F);
-
-    if (!IsValid) {
-      if (FatalErrors)
-        report_fatal_error("broken module found, compilation aborted!");
-      else
-        errs() << "broken module found, compilation aborted!\n";
-    }
-    return false;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-  }
-};
-char TargetVerifierLegacyPass::ID = 0;
-FunctionPass *createTargetVerifierLegacyPass(bool FatalErrors) {
-  return new TargetVerifierLegacyPass(FatalErrors);
-}
 } // namespace llvm
-using namespace llvm;
-INITIALIZE_PASS(TargetVerifierLegacyPass, "tgtverifier", "Target Verifier", false, false)
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 329d95826551f..2e9e4837fe467 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -660,8 +660,6 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
-  if (VerifyTarget)
-    PM.add(createTargetVerifierLegacyPass(false));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index b86c2318b45b7..d832dcdff4ad0 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -158,8 +158,6 @@ int main(int argc, char **argv) {
   if (TT.isAMDGPU())
     FPM.addPass(AMDGPUTargetVerifierPass());
   else if (false) {} // ...
-  else
-    FPM.addPass(TargetVerifierPass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
   auto PA = MPM.run(*M, MAM);

>From 42b89826fa18b4796c9fdee25864e929a2351661 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 11:18:50 -0400
Subject: [PATCH 20/33] Add back to legacy PM.

---
 llvm/include/llvm/InitializePasses.h            | 1 -
 llvm/lib/Target/AMDGPU/AMDGPU.h                 | 3 +++
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  | 1 +
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 8 ++++----
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index d718f92877276..1b5b1d5888824 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,7 +333,6 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
-//LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 5a917734e9c74..26fb0a967fd76 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -549,6 +549,9 @@ extern char &GCNRewritePartialRegUsesID;
 void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
 extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
+FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
+void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 718655b7ebf00..8b0b8f6df740e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1411,6 +1411,7 @@ bool AMDGPUPassConfig::addGCPasses() {
 //===----------------------------------------------------------------------===//
 
 bool GCNPassConfig::addPreISel() {
+  addPass(createAMDGPUTargetVerifierLegacyPass(false));
   AMDGPUPassConfig::addPreISel();
 
   if (TM->getOptLevel() > CodeGenOptLevel::None)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index cedd9ddc78011..c4d303bee6ef8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -17,6 +17,7 @@
 ////
 ////===----------------------------------------------------------------------===//
 
+#include "AMDGPU.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
 #include "llvm/Analysis/UniformityAnalysis.h"
@@ -24,7 +25,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-#include "llvm/InitializePasses.h"
+//#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -133,7 +134,6 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
   return PreservedAnalyses::all();
 }
 
-/*
 struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   static char ID;
 
@@ -183,6 +183,6 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
 char AMDGPUTargetVerifierLegacyPass::ID = 0;
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
   return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
-}*/
+}
 } // namespace llvm
-//INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)

>From 230f33b8db0650a59c5656d2caf5c7be31f0aa49 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 14:04:10 -0400
Subject: [PATCH 21/33] Remove reference to FAM in registerCallbacks and
 VerifyEach for TargetVerify in instrumentation

---
 clang/lib/CodeGen/BackendUtil.cpp             |  2 +-
 flang/lib/Frontend/FrontendActions.cpp        |  2 +-
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp   |  2 +-
 .../llvm/Passes/StandardInstrumentations.h    |  6 ++----
 llvm/lib/LTO/LTOBackend.cpp                   |  2 +-
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp         |  2 +-
 llvm/lib/Passes/PassBuilderBindings.cpp       |  2 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  | 21 ++++---------------
 llvm/tools/llc/NewPMDriver.cpp                |  7 ++++---
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 +-
 llvm/tools/opt/NewPMDriver.cpp                |  2 +-
 llvm/unittests/IR/PassManagerTest.cpp         |  6 +++---
 15 files changed, 24 insertions(+), 38 deletions(-)

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 560129f871026..7e0a3cf5591ce 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -917,7 +917,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       TheModule->getContext(),
       (CodeGenOpts.DebugPassManager || DebugPassStructure),
       CodeGenOpts.VerifyEach, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
   // Handle the assignment tracking feature options.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index c15151fc5c1b8..d684eeb696755 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -947,7 +947,7 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
 
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
-  si.registerCallbacks(pic, &mam, &fam);
+  si.registerCallbacks(pic, &mam);
   if (ci.isTimingEnabled())
     si.getTimePasses().setOutStream(ci.getTimingStreamLLVM());
   pto.LoopUnrolling = opts.UnrollLoops;
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index f9664025f61f1..0f58391c50667 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -577,7 +577,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index eae06d9f57467..7117eaf4982b0 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -851,7 +851,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index 30ad79ef2fc58..cb7b6cc8651c1 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -970,7 +970,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 4a39bc33c5591..91b7191a07c6f 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1139,7 +1139,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Promote allocas to registers.
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 7d7b858b447cc..df9d8de8a025a 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,8 +478,7 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM,
-			 FunctionAnalysisManager *FAM);
+                         ModuleAnalysisManager *MAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -625,8 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM,
-			 FunctionAnalysisManager *FAM);
+                         ModuleAnalysisManager *MAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 7ae23ef4e7df4..5e8cd12fe040b 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -273,7 +273,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
                               Conf.VerifyEach);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 
   RegisterPassPlugins(Conf.PassPlugins, PB);
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 8db92e70d5a96..5b333cda31fb8 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -241,7 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PipelineTuningOptions PTO;
   PTO.LoopVectorization = true;
   PTO.SLPVectorization = true;
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index f0e1abb8cebc4..933fe89e53a94 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -76,7 +76,7 @@ static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   // Run the pipeline.
   if (Fun) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 905ba667fac7a..4e8abc43572d7 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,7 +40,6 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
-//#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -57,8 +56,6 @@ static cl::opt<bool> VerifyAnalysisInvalidation("verify-analysis-invalidation",
 #endif
 );
 
-static cl::opt<bool> VerifyTargetEach("verify-tgt-each");
-
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
 // of this option.  Note that this option has no effect without -print-changed.
@@ -1452,10 +1449,9 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 }
 
 void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                              ModuleAnalysisManager *MAM,
-					      FunctionAnalysisManager *FAM) {
+                                              ModuleAnalysisManager *MAM) {
   PIC.registerAfterPassCallback(
-      [this, MAM, FAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+      [this, MAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
         const auto *F = unwrapIR<Function>(IR);
@@ -1472,15 +1468,6 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
             report_fatal_error(formatv("Broken function found after pass "
                                        "\"{0}\", compilation aborted!",
                                        P));
-
-          if (VerifyTargetEach && FAM) {
-            //TargetVerify TV(const_cast<Module*>(F->getParent()));
-            //TV.run(*const_cast<Function*>(F), *FAM);
-	    /*if (!TV.IsValid)
-              report_fatal_error(formatv("Broken function found after pass "
-                                         "\"{0}\", compilation aborted!",
-                                         P));*/
-          }
         } else {
           const auto *M = unwrapIR<Module>(IR);
           if (!M) {
@@ -2520,7 +2507,7 @@ void PrintCrashIRInstrumentation::registerCallbacks(
 }
 
 void StandardInstrumentations::registerCallbacks(
-    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM, FunctionAnalysisManager *FAM) {
+    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM) {
   PrintIR.registerCallbacks(PIC);
   PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
@@ -2529,7 +2516,7 @@ void StandardInstrumentations::registerCallbacks(
   PrintChangedIR.registerCallbacks(PIC);
   PseudoProbeVerification.registerCallbacks(PIC);
   if (VerifyEach)
-    Verify.registerCallbacks(PIC, MAM, FAM);
+    Verify.registerCallbacks(PIC, MAM);
   PrintChangedDiff.registerCallbacks(PIC);
   WebsiteChangeReporter.registerCallbacks(PIC);
   ChangeTester.registerCallbacks(PIC);
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 4b95977a10c5f..863a555798dab 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -117,8 +117,6 @@ int llvm::compileModuleWithNewPM(
                               VK == VerifierKind::EachPass);
   registerCodeGenCallback(PIC, *Target);
 
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
   MachineFunctionAnalysisManager MFAM;
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
@@ -133,11 +131,14 @@ int llvm::compileModuleWithNewPM(
   if (VerifyTarget)
     PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
 
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+
   if (!PassPipeline.empty()) {
     // Construct a custom pass pipeline that starts after instruction
     // selection.
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index d832dcdff4ad0..50f4e56bb6af6 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -148,7 +148,7 @@ int main(int argc, char **argv) {
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   Triple TT(M->getTargetTriple());
   if (!NoLint)
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index a8977d80bdf44..7d168a6ceb17c 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -423,7 +423,7 @@ bool llvm::runPassPipeline(
   PrintPassOpts.SkipAnalyses = DebugPM == DebugLogging::Quiet;
   StandardInstrumentations SI(M.getContext(), DebugPM != DebugLogging::None,
                               VK == VerifierKind::EachPass, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   DebugifyEachInstrumentation Debugify;
   DebugifyStatsMap DIStatsMap;
   DebugInfoPerPass DebugInfoBeforePass;
diff --git a/llvm/unittests/IR/PassManagerTest.cpp b/llvm/unittests/IR/PassManagerTest.cpp
index bb4db6120035f..a6487169224c2 100644
--- a/llvm/unittests/IR/PassManagerTest.cpp
+++ b/llvm/unittests/IR/PassManagerTest.cpp
@@ -828,7 +828,7 @@ TEST_F(PassManagerTest, FunctionPassCFGChecker) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -877,7 +877,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerInvalidateAnalysis) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -945,7 +945,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerWrapped) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });

>From c680041d5134d6edd47e8dec65d4986feb63fb7d Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 14:44:28 -0400
Subject: [PATCH 22/33] Remove references to registry

---
 llvm/include/llvm/Passes/PassBuilder.h        | 21 -------------------
 .../llvm/Passes/StandardInstrumentations.h    |  2 +-
 .../llvm/Passes/TargetPassRegistry.inc        | 12 -----------
 llvm/lib/Passes/PassBuilder.cpp               |  7 -------
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 11 ----------
 llvm/tools/llc/NewPMDriver.cpp                |  5 -----
 llvm/tools/llc/llc.cpp                        |  2 --
 7 files changed, 1 insertion(+), 59 deletions(-)

diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index c134f2d0c1112..f13b5c678a894 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -173,13 +173,6 @@ class PassBuilder {
   /// additional analyses.
   LLVM_ABI void registerLoopAnalyses(LoopAnalysisManager &LAM);
 
-  /// Registers all available verifier passes.
-  ///
-  /// This is an interface that can be used to populate a
-  /// \c ModuleAnalysisManager with all registered loop analyses. Callers can
-  /// still manually register any additional analyses.
-  void registerVerifierPasses(ModulePassManager &PM, FunctionPassManager &);
-
   /// Registers all available machine function analysis passes.
   ///
   /// This is an interface that can be used to populate a \c
@@ -583,15 +576,6 @@ class PassBuilder {
   }
   /// @}}
 
-  /// Register a callback for parsing an Verifier Name to populate
-  /// the given managers.
-  void registerVerifierCallback(
-      const std::function<bool(ModulePassManager &MPM)> &C,
-      const std::function<bool(FunctionPassManager &MPM)> &CF) {
-    VerifierCallbacks.push_back(C);
-    FnVerifierCallbacks.push_back(CF);
-  }
-
   /// {{@ Register pipeline parsing callbacks with this pass builder instance.
   /// Using these callbacks, callers can parse both a single pass name, as well
   /// as entire sub-pipelines, and populate the PassManager instance
@@ -867,11 +851,6 @@ class PassBuilder {
   // Callbacks to parse `filter` parameter in register allocation passes
   SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
-  // Verifier callbacks
-  SmallVector<std::function<bool(ModulePassManager &)>, 2>
-      VerifierCallbacks;
-  SmallVector<std::function<bool(FunctionPassManager &)>, 2>
-      FnVerifierCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index df9d8de8a025a..d7c1bbf04c328 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -624,7 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc
index 2d04b874cf360..521913cb25a4a 100644
--- a/llvm/include/llvm/Passes/TargetPassRegistry.inc
+++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc
@@ -151,18 +151,6 @@ PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM,
   return false;
 });
 
-PB.registerVerifierCallback([](ModulePassManager &PM) {
-#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS) PM.addPass(CREATE_PASS)
-#include GET_PASS_REGISTRY
-#undef VERIFIER_MODULE_ANALYSIS
-  return false;
-}, [](FunctionPassManager &FPM) {
-#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS) FPM.addPass(CREATE_PASS)
-#include GET_PASS_REGISTRY
-#undef VERIFIER_FUNCTION_ANALYSIS
-  return false;
-});
-
 #undef ADD_PASS
 #undef ADD_PASS_WITH_PARAMS
 
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 561996d0c8980..4603eaff8ade9 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -576,13 +576,6 @@ void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     C(LAM);
 }
 
-void PassBuilder::registerVerifierPasses(ModulePassManager &MPM, FunctionPassManager &FPM) {
-  for (auto &C : VerifierCallbacks)
-    C(MPM);
-  for (auto &C : FnVerifierCallbacks)
-    C(FPM);
-}
-
 static std::optional<std::pair<bool, bool>>
 parseFunctionPipelineName(StringRef Name) {
   std::pair<bool, bool> Params;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 652cd23f62443..13453963eec6d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -82,17 +82,6 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
-#ifndef VERIFIER_MODULE_ANALYSIS
-#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS)
-#endif
-#ifndef VERIFIER_FUNCTION_ANALYSIS
-#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
-#endif
-VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
-#undef VERIFIER_MODULE_ANALYSIS
-#undef VERIFIER_FUNCTION_ANALYSIS
-
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 863a555798dab..fa82689ecf9ae 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -57,9 +57,6 @@ static cl::opt<bool>
     DebugPM("debug-pass-manager", cl::Hidden,
             cl::desc("Print pass management debugging information"));
 
-static cl::opt<bool> VerifyTarget("verify-tgt-new-pm",
-		                  cl::desc("Verify the target"));
-
 bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
   DiagnosticHandler::handleDiagnostics(DI);
   if (DI.getKind() == llvm::DK_SrcMgr) {
@@ -128,8 +125,6 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
-  if (VerifyTarget)
-    PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM);
 
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 2e9e4837fe467..140459ba2de21 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -209,8 +209,6 @@ static cl::opt<std::string> PassPipeline(
 static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
                                cl::desc("Alias for -passes"));
 
-static cl::opt<bool> VerifyTarget("verify-tgt", cl::desc("Verify the target"));
-
 namespace {
 
 std::vector<std::string> &getRunPassNames() {

>From 3f5634a532856668a42f44ea04d0ffcc862c96d4 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 19:05:53 -0400
Subject: [PATCH 23/33] Remove int check

---
 .../lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index c4d303bee6ef8..2ca0bbeb57653 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -25,7 +25,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-//#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -46,15 +45,6 @@ using namespace llvm;
 
 namespace llvm {
 
-static bool IsValidInt(const Type *Ty) {
-  return Ty->isIntegerTy(1) ||
-         Ty->isIntegerTy(8) ||
-         Ty->isIntegerTy(16) ||
-         Ty->isIntegerTy(32) ||
-         Ty->isIntegerTy(64) ||
-         Ty->isIntegerTy(128);
-}
-
 static bool isShader(CallingConv::ID CC) {
   switch(CC) {
     case CallingConv::AMDGPU_VS:
@@ -81,14 +71,6 @@ bool AMDGPUTargetVerify::run(Function &F) {
 
     for (auto &I : BB) {
 
-      // Ensure integral types are valid: i8, i16, i32, i64, i128
-      if (I.getType()->isIntegerTy())
-        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
-      for (unsigned i = 0; i < I.getNumOperands(); ++i)
-        if (I.getOperand(i)->getType()->isIntegerTy())
-          Check(IsValidInt(I.getOperand(i)->getType()),
-                "Int type is invalid.", I.getOperand(i));
-
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
         // Ensure no kernel to kernel calls.

>From 6007895007af8925e32b709009c0d6131141c282 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 21:56:28 -0400
Subject: [PATCH 24/33] Remove modifications to Lint/Verifier.

---
 llvm/lib/Analysis/Lint.cpp |  4 +---
 llvm/lib/IR/Verifier.cpp   | 20 ++++----------------
 2 files changed, 5 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index c8627e6db0692..1168005f48c0e 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -735,11 +735,9 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) {
   Lint L(Mod, DL, AA, AC, DT, TLI);
   L.visit(F);
   dbgs() << L.MessagesStr.str();
-  if (AbortOnError && !L.MessagesStr.str().empty()) {
+  if (AbortOnError && !L.MessagesStr.str().empty())
     report_fatal_error(
         "linter found errors, aborting. (enabled by abort-on-error)", false);
-    return PreservedAnalyses::none();
-  }
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 439823ba8b682..1f1041b259736 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -135,10 +135,6 @@ static cl::opt<bool> VerifyNoAliasScopeDomination(
     cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
              "scopes are not dominating"));
 
-static cl::opt<bool>
-    VerifyAbortOnError("verifier-abort-on-error", cl::init(false),
-                       cl::desc("In the Verifier pass, abort on errors."));
-
 namespace llvm {
 
 struct VerifierSupport {
@@ -7869,24 +7865,16 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
-  if (Res.IRBroken || Res.DebugInfoBroken) {
-    //M.IsValid = false;
-    if (VerifyAbortOnError && FatalErrors)
-      report_fatal_error("Broken module found, compilation aborted!");
-    return PreservedAnalyses::none();
-  }
+  if (FatalErrors && (Res.IRBroken || Res.DebugInfoBroken))
+    report_fatal_error("Broken module found, compilation aborted!");
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
-  if (res.IRBroken) {
-    //F.getParent()->IsValid = false;
-    if (VerifyAbortOnError && FatalErrors)
-      report_fatal_error("Broken function found, compilation aborted!");
-    return PreservedAnalyses::none();
-  }
+  if (res.IRBroken && FatalErrors)
+    report_fatal_error("Broken function found, compilation aborted!");
 
   return PreservedAnalyses::all();
 }

>From 756c3ebee47f59f69a267da6079edafce4f595ca Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 22:21:47 -0400
Subject: [PATCH 25/33] Remove llvm-tgt-verify tool.

---
 llvm/test/CMakeLists.txt                      |   1 -
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        |  45 -----
 llvm/test/lit.cfg.py                          |   1 -
 llvm/tools/llvm-tgt-verify/CMakeLists.txt     |  34 ----
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 171 ------------------
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |   1 -
 .../llvm/tools/llvm-tgt-verify/BUILD.gn       |  25 ---
 7 files changed, 278 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify.ll
 delete mode 100644 llvm/tools/llvm-tgt-verify/CMakeLists.txt
 delete mode 100644 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
 delete mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn

diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index a2bb112db8746..2a6135da9a61e 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -136,7 +136,6 @@ set(LLVM_TEST_DEPENDS
           llvm-strip
           llvm-symbolizer
           llvm-tblgen
-          llvm-tgt-verify
           llvm-readtapi
           llvm-tli-checker
           llvm-undname
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
deleted file mode 100644
index 62b220d7d9f49..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
-
-define amdgpu_cs i32 @shader() {
-; CHECK: Shaders must return void
-  ret i32 0
-}
-
-define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
-; CHECK: Undefined behavior: Write to memory in const addrspace
-; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
-  %r = add i32 %a, %b
-  store i32 %r, ptr addrspace(4) %out
-  ret void
-}
-
-define amdgpu_kernel void @kernel_callee(ptr %x) {
-  ret void
-}
-
-define amdgpu_kernel void @kernel_caller(ptr %x) {
-; CHECK: A kernel may not call a kernel
-; CHECK-NEXT: ptr @kernel_caller
-  call amdgpu_kernel void @kernel_callee(ptr %x)
-  ret void
-}
-
-
-; Function Attrs: nounwind
-define i65 @invalid_type(i65 %x) #0 {
-; CHECK: Int type is invalid.
-; CHECK-NEXT: %tmp2 = ashr i65 %x, 64
-entry:
-  %tmp2 = ashr i65 %x, 64
-  ret i65 %tmp2
-}
-
-declare void @llvm.amdgcn.cs.chain.v3i32(ptr, i32, <3 x i32>, <3 x i32>, i32, ...)
-declare amdgpu_cs_chain void @chain_callee(<3 x i32> inreg, <3 x i32>)
-
-define amdgpu_cs void @no_unreachable(<3 x i32> inreg %a, <3 x i32> %b) {
-; CHECK: llvm.amdgcn.cs.chain must be followed by unreachable
-; CHECK-NEXT: call void (ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.v3i32.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
-  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
-  ret void
-}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index bf06e80dc7ae5..bd6e37c848d8c 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -228,7 +228,6 @@ def get_asan_rtlib():
         "llvm-strings",
         "llvm-strip",
         "llvm-tblgen",
-        "llvm-tgt-verify",
         "llvm-readtapi",
         "llvm-undname",
         "llvm-windres",
diff --git a/llvm/tools/llvm-tgt-verify/CMakeLists.txt b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
deleted file mode 100644
index fe47c85e6cdce..0000000000000
--- a/llvm/tools/llvm-tgt-verify/CMakeLists.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  AllTargetsAsmParsers
-  AllTargetsCodeGens
-  AllTargetsDescs
-  AllTargetsInfos
-  Analysis
-  AsmPrinter
-  CodeGen
-  CodeGenTypes
-  Core
-  IRPrinter
-  IRReader
-  MC
-  MIRParser
-  Passes
-  Remarks
-  ScalarOpts
-  SelectionDAG
-  Support
-  Target
-  TargetParser
-  TransformUtils
-  Vectorize
-  )
-
-add_llvm_tool(llvm-tgt-verify
-  llvm-tgt-verify.cpp
-
-  DEPENDS
-  intrinsics_gen
-  SUPPORT_PLUGINS
-  )
-
-export_executable_symbols_for_plugins(llc)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
deleted file mode 100644
index 50f4e56bb6af6..0000000000000
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ /dev/null
@@ -1,171 +0,0 @@
-//===--- llvm-tgt-verify.cpp - Target Verifier ----------------- ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Tool to verify a target.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/InitializePasses.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/Lint.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Bitcode/BitcodeReader.h"
-#include "llvm/Bitcode/BitcodeWriter.h"
-#include "llvm/CodeGen/CommandFlags.h"
-#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Verifier.h"
-#include "llvm/IRReader/IRReader.h"
-#include "llvm/Passes/PassBuilder.h"
-#include "llvm/Passes/StandardInstrumentations.h"
-#include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/InitLLVM.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/TargetSelect.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetVerifier.h"
-
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
-
-#define DEBUG_TYPE "isel-fuzzer"
-
-using namespace llvm;
-
-static codegen::RegisterCodeGenFlags CGF;
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-static cl::opt<bool>
-    StacktraceAbort("stacktrace-abort",
-        cl::desc("Turn on stacktrace"), cl::init(false));
-
-static cl::opt<bool>
-    NoLint("no-lint",
-        cl::desc("Turn off Lint"), cl::init(false));
-
-static cl::opt<bool>
-    NoVerify("no-verifier",
-        cl::desc("Turn off Verifier"), cl::init(false));
-
-static cl::opt<char>
-    OptLevel("O",
-             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
-                      "(default = '-O2')"),
-             cl::Prefix, cl::init('2'));
-
-static cl::opt<std::string>
-    TargetTriple("mtriple", cl::desc("Override target triple for module"));
-
-static std::unique_ptr<TargetMachine> TM;
-
-static void handleLLVMFatalError(void *, const char *Message, bool) {
-  if (StacktraceAbort) {
-    dbgs() << "LLVM ERROR: " << Message << "\n"
-           << "Aborting.\n";
-    abort();
-  }
-}
-
-int main(int argc, char **argv) {
-  StringRef ExecName = argv[0];
-  InitLLVM X(argc, argv);
-
-  InitializeAllTargets();
-  InitializeAllTargetMCs();
-  InitializeAllAsmPrinters();
-  InitializeAllAsmParsers();
-
-  PassRegistry *Registry = PassRegistry::getPassRegistry();
-  initializeCore(*Registry);
-  initializeCodeGen(*Registry);
-  initializeAnalysis(*Registry);
-  initializeTarget(*Registry);
-
-  cl::ParseCommandLineOptions(argc, argv);
-
-  if (TargetTriple.empty()) {
-    errs() << ExecName << ": -mtriple must be specified\n";
-    exit(1);
-  }
-
-  CodeGenOptLevel OLvl;
-  if (auto Level = CodeGenOpt::parseLevel(OptLevel)) {
-    OLvl = *Level;
-  } else {
-    errs() << ExecName << ": invalid optimization level.\n";
-    return 1;
-  }
-  ExitOnError ExitOnErr(std::string(ExecName) + ": error:");
-  TM = ExitOnErr(codegen::createTargetMachineForTriple(
-      Triple::normalize(TargetTriple), OLvl));
-  assert(TM && "Could not allocate target machine!");
-
-  // Make sure we print the summary and the current unit when LLVM errors out.
-  install_fatal_error_handler(handleLLVMFatalError, nullptr);
-
-  LLVMContext Context;
-  SMDiagnostic Err;
-  std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
-  if (!M) {
-    errs() << "Invalid mod\n";
-    return 1;
-  }
-  auto S = Triple::normalize(TargetTriple);
-  M->setTargetTriple(Triple(S));
-
-  PassInstrumentationCallbacks PIC;
-  StandardInstrumentations SI(Context, false/*debug PM*/,
-                              false);
-  registerCodeGenCallback(PIC, *TM);
-
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
-  //TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
-
-  MachineFunctionAnalysisManager MFAM;
-  LoopAnalysisManager LAM;
-  FunctionAnalysisManager FAM;
-  CGSCCAnalysisManager CGAM;
-  ModuleAnalysisManager MAM;
-  PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
-  PB.registerModuleAnalyses(MAM);
-  //PB.registerVerifierPasses(MPM, FPM);
-  PB.registerCGSCCAnalyses(CGAM);
-  PB.registerFunctionAnalyses(FAM);
-  PB.registerLoopAnalyses(LAM);
-  PB.registerMachineFunctionAnalyses(MFAM);
-  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-
-  SI.registerCallbacks(PIC, &MAM);
-
-  Triple TT(M->getTargetTriple());
-  if (!NoLint)
-    FPM.addPass(LintPass(false));
-  if (!NoVerify)
-    MPM.addPass(VerifierPass());
-  if (TT.isAMDGPU())
-    FPM.addPass(AMDGPUTargetVerifierPass());
-  else if (false) {} // ...
-  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
-
-  auto PA = MPM.run(*M, MAM);
-  {
-    auto PAC = PA.getChecker<VerifierAnalysis>();
-    if (!PAC.preserved())
-      return 1;
-  }
-
-  return 0;
-}
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 36cf956545caf..946b63f8a54fb 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -320,7 +320,6 @@ group("test") {
     "//llvm/tools/llvm-strings",
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-tli-checker",
-    "//llvm/tools/llvm-tgt-verify",
     "//llvm/tools/llvm-undname",
     "//llvm/tools/llvm-xray",
     "//llvm/tools/lto",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
deleted file mode 100644
index b751bafc5052c..0000000000000
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
+++ /dev/null
@@ -1,25 +0,0 @@
-import("//llvm/utils/TableGen/tablegen.gni")
-
-tgtverifier("llvm-tgt-verify") {
-  deps = [
-    "//llvm/lib/Analysis",
-    "//llvm/lib/AsmPrinter",
-    "//llvm/lib/CodeGen",
-    "//llvm/lib/CodeGenTypes",
-    "//llvm/lib/Core",
-    "//llvm/lib/IRPrinter",
-    "//llvm/lib/IRReader",
-    "//llvm/lib/MC",
-    "//llvm/lib/MIRParser",
-    "//llvm/lib/Passes",
-    "//llvm/lib/Remarks",
-    "//llvm/lib/ScalarOpts",
-    "//llvm/lib/SelectionDAG",
-    "//llvm/lib/Support",
-    "//llvm/lib/Target",
-    "//llvm/lib/TargetParser",
-    "//llvm/lib/TransformUtils",
-    "//llvm/lib/Vectorize",
-  ]
-  sources = [ "llvm-tgt-verify.cpp" ]
-}

>From 3f1d72ff01d0303c0fc17d9a6c40e46661bf1fb0 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 22:23:15 -0400
Subject: [PATCH 26/33] Remove TargetVerifier.cpp

---
 llvm/lib/Passes/CMakeLists.txt     |  1 -
 llvm/lib/Target/CMakeLists.txt     |  2 --
 llvm/lib/Target/TargetVerifier.cpp | 36 ------------------------------
 3 files changed, 39 deletions(-)
 delete mode 100644 llvm/lib/Target/TargetVerifier.cpp

diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index b8d25b15186e8..91c8c4f67074d 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,7 +30,6 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
-  #TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index f2a5d545ce84f..9472288229cac 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -7,8 +7,6 @@ add_llvm_component_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
-  TargetVerifier.cpp
-  AMDGPU/AMDGPUTargetVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Target
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
deleted file mode 100644
index c63ae2a2c5daf..0000000000000
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-//===-- TargetVerifier.cpp - LLVM IR Target Verifier ----------------*- C++ -*-===//
-////
-///// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-///// See https://llvm.org/LICENSE.txt for license information.
-///// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-/////
-/////===----------------------------------------------------------------------===//
-/////
-///// This file defines target verifier interfaces that can be used for some
-///// validation of input to the system, and for checking that transformations
-///// haven't done something bad. In contrast to the Verifier or Lint, the
-///// TargetVerifier looks for constructions invalid to a particular target
-///// machine.
-/////
-///// To see what specifically is checked, look at TargetVerifier.cpp or an
-///// individual backend's TargetVerifier.
-/////
-/////===----------------------------------------------------------------------===//
-
-#include "llvm/Target/TargetVerifier.h"
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
-
-#include "llvm/InitializePasses.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/IR/Dominators.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Value.h"
-
-namespace llvm {
-
-} // namespace llvm

>From 1c721cf05616091cf3c42e4c244400a393abe755 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 1 May 2025 03:02:30 -0400
Subject: [PATCH 27/33] clang-format

---
 llvm/include/llvm/Target/TargetVerifier.h     | 10 +-
 .../TargetVerify/AMDGPUTargetVerifier.h       | 44 ++++-----
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 98 ++++++++++---------
 3 files changed, 77 insertions(+), 75 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 1d12eb55bbf0a..3f8c710a88768 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -1,4 +1,4 @@
-//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier ---*- C++ -*-===//
+//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier --*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -20,8 +20,8 @@
 #ifndef LLVM_TARGET_VERIFIER_H
 #define LLVM_TARGET_VERIFIER_H
 
-#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
@@ -59,10 +59,11 @@ class TargetVerify {
   /// This calls the Message-only version so that the above is easier to set
   /// a breakpoint on.
   template <typename T1, typename... Ts>
-  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) {
+  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
     CheckFailed(Message);
     WriteValues({V1, Vs...});
   }
+
 public:
   Module *Mod;
   Triple TT;
@@ -73,8 +74,7 @@ class TargetVerify {
   bool IsValid = true;
 
   TargetVerify(Module *Mod)
-      : Mod(Mod), TT(Mod->getTargetTriple()),
-        MessagesStr(Messages) {}
+      : Mod(Mod), TT(Mod->getTargetTriple()), MessagesStr(Messages) {}
 
   virtual bool run(Function &F) = 0;
 };
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 49bcbc8849e3c..5b8d9ec259b63 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -1,29 +1,29 @@
-//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU ---*- C++ -*-===//
-////
-//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//// See https://llvm.org/LICENSE.txt for license information.
-//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-////
-////===----------------------------------------------------------------------===//
-////
-//// This file defines target verifier interfaces that can be used for some
-//// validation of input to the system, and for checking that transformations
-//// haven't done something bad. In contrast to the Verifier or Lint, the
-//// TargetVerifier looks for constructions invalid to a particular target
-//// machine.
-////
-//// To see what specifically is checked, look at an individual backend's
-//// TargetVerifier.
-////
-////===----------------------------------------------------------------------===//
+//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU -- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
 
 #ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
 #define LLVM_AMDGPU_TARGET_VERIFIER_H
 
 #include "llvm/Target/TargetVerifier.h"
 
-#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/IR/Dominators.h"
 
 namespace llvm {
@@ -43,10 +43,10 @@ class AMDGPUTargetVerify : public TargetVerify {
   PostDominatorTree *PDT = nullptr;
   UniformityInfo *UA = nullptr;
 
-  AMDGPUTargetVerify(Module *Mod)
-    : TargetVerify(Mod), Mod(Mod) {}
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod), Mod(Mod) {}
 
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
+                     UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   bool run(Function &F) override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 2ca0bbeb57653..eb22eb2177f7f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -1,34 +1,34 @@
-//===-- AMDGPUTargetVerifier.cpp - AMDGPU -------------------------*- C++ -*-===//
-////
-//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//// See https://llvm.org/LICENSE.txt for license information.
-//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-////
-////===----------------------------------------------------------------------===//
-////
-//// This file defines target verifier interfaces that can be used for some
-//// validation of input to the system, and for checking that transformations
-//// haven't done something bad. In contrast to the Verifier or Lint, the
-//// TargetVerifier looks for constructions invalid to a particular target
-//// machine.
-////
-//// To see what specifically is checked, look at an individual backend's
-//// TargetVerifier.
-////
-////===----------------------------------------------------------------------===//
+//===-- AMDGPUTargetVerifier.cpp - AMDGPU -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
 
-#include "AMDGPU.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+#include "AMDGPU.h"
 
-#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Debug.h"
 
 #include "llvm/Support/raw_ostream.h"
 
@@ -39,53 +39,52 @@ using namespace llvm;
   do {                                                                         \
     if (!(C)) {                                                                \
       TargetVerify::CheckFailed(__VA_ARGS__);                                  \
-      return false;                                                                  \
     }                                                                          \
   } while (false)
 
 namespace llvm {
 
 static bool isShader(CallingConv::ID CC) {
-  switch(CC) {
-    case CallingConv::AMDGPU_VS:
-    case CallingConv::AMDGPU_LS:
-    case CallingConv::AMDGPU_HS:
-    case CallingConv::AMDGPU_ES:
-    case CallingConv::AMDGPU_GS:
-    case CallingConv::AMDGPU_PS:
-    case CallingConv::AMDGPU_CS_Chain:
-    case CallingConv::AMDGPU_CS_ChainPreserve:
-    case CallingConv::AMDGPU_CS:
-      return true;
-    default:
-      return false;
+  switch (CC) {
+  case CallingConv::AMDGPU_VS:
+  case CallingConv::AMDGPU_LS:
+  case CallingConv::AMDGPU_HS:
+  case CallingConv::AMDGPU_ES:
+  case CallingConv::AMDGPU_GS:
+  case CallingConv::AMDGPU_PS:
+  case CallingConv::AMDGPU_CS_Chain:
+  case CallingConv::AMDGPU_CS_ChainPreserve:
+  case CallingConv::AMDGPU_CS:
+    return true;
+  default:
+    return false;
   }
 }
 
 bool AMDGPUTargetVerify::run(Function &F) {
   // Ensure shader calling convention returns void
   if (isShader(F.getCallingConv()))
-    Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
+    Check(F.getReturnType() == Type::getVoidTy(F.getContext()),
+          "Shaders must return void");
 
   for (auto &BB : F) {
 
     for (auto &I : BB) {
 
-      if (auto *CI = dyn_cast<CallInst>(&I))
-      {
+      if (auto *CI = dyn_cast<CallInst>(&I)) {
         // Ensure no kernel to kernel calls.
         CallingConv::ID CalleeCC = CI->getCallingConv();
-        if (CalleeCC == CallingConv::AMDGPU_KERNEL)
-        {
-          CallingConv::ID CallerCC = CI->getParent()->getParent()->getCallingConv();
+        if (CalleeCC == CallingConv::AMDGPU_KERNEL) {
+          CallingConv::ID CallerCC =
+              CI->getParent()->getParent()->getCallingConv();
           Check(CallerCC != CallingConv::AMDGPU_KERNEL,
-            "A kernel may not call a kernel", CI->getParent()->getParent());
+                "A kernel may not call a kernel", CI->getParent()->getParent());
         }
 
         // Ensure chain intrinsics are followed by unreachables.
         if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
           Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
-            "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
+                "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
     }
   }
@@ -98,7 +97,8 @@ bool AMDGPUTargetVerify::run(Function &F) {
   return true;
 }
 
-PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F,
+                                                FunctionAnalysisManager &AM) {
   auto *Mod = F.getParent();
 
   auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
@@ -122,9 +122,10 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   std::unique_ptr<AMDGPUTargetVerify> TV;
   bool FatalErrors = false;
 
-  AMDGPUTargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
-    FatalErrors(FatalErrors) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
+      : FunctionPass(ID), FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(
+        *PassRegistry::getPassRegistry());
   }
 
   bool doInitialization(Module &M) override {
@@ -167,4 +168,5 @@ FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
   return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
 }
 } // namespace llvm
-INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier",
+                "AMDGPU Target Verifier", false, false)

>From f59b0f27beb766516186dcab8bde8463734ee64b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 1 May 2025 03:49:20 -0400
Subject: [PATCH 28/33] Add VerifyTarget option

---
 .../llvm/Target/TargetVerify/AMDGPUTargetVerifier.h      | 6 ++----
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp           | 9 +++++++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 5b8d9ec259b63..8ed7dd7ea2f69 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -37,17 +37,15 @@ class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 
 class AMDGPUTargetVerify : public TargetVerify {
 public:
-  Module *Mod;
-
   DominatorTree *DT = nullptr;
   PostDominatorTree *PDT = nullptr;
   UniformityInfo *UA = nullptr;
 
-  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod), Mod(Mod) {}
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
 
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
                      UniformityInfo *UA)
-    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   bool run(Function &F) override;
 };
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8b0b8f6df740e..be8bde1c1c0e0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -482,6 +482,9 @@ static cl::opt<bool> HasClosedWorldAssumption(
     cl::desc("Whether has closed-world assumption at link time"),
     cl::init(false), cl::Hidden);
 
+static cl::opt<bool> VerifyTarget("verify-tgt",
+                                  cl::desc("Enable the target verifier"));
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
@@ -1411,7 +1414,8 @@ bool AMDGPUPassConfig::addGCPasses() {
 //===----------------------------------------------------------------------===//
 
 bool GCNPassConfig::addPreISel() {
-  addPass(createAMDGPUTargetVerifierLegacyPass(false));
+  if (VerifyTarget)
+    addPass(createAMDGPUTargetVerifierLegacyPass(false));
   AMDGPUPassConfig::addPreISel();
 
   if (TM->getOptLevel() > CodeGenOptLevel::None)
@@ -2011,7 +2015,8 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
 }
 
 void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
-  addPass(AMDGPUTargetVerifierPass());
+  if (VerifyTarget)
+    addPass(AMDGPUTargetVerifierPass());
 
   if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
     addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));

>From f0f12f552a0a48eda3e1dc0b7db88e605067f945 Mon Sep 17 00:00:00 2001
From: Joey F <jofernau at amd.com>
Date: Wed, 7 May 2025 21:59:43 -0400
Subject: [PATCH 29/33] Remove AMDGPUTargetVerifier.h

---
 .../TargetVerify/AMDGPUTargetVerifier.h       | 55 -------------------
 llvm/lib/Target/AMDGPU/AMDGPU.h               |  5 ++
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  1 -
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 16 +++++-
 4 files changed, 20 insertions(+), 57 deletions(-)
 delete mode 100644 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h

diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
deleted file mode 100644
index 8ed7dd7ea2f69..0000000000000
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU -- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines target verifier interfaces that can be used for some
-// validation of input to the system, and for checking that transformations
-// haven't done something bad. In contrast to the Verifier or Lint, the
-// TargetVerifier looks for constructions invalid to a particular target
-// machine.
-//
-// To see what specifically is checked, look at an individual backend's
-// TargetVerifier.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
-#define LLVM_AMDGPU_TARGET_VERIFIER_H
-
-#include "llvm/Target/TargetVerifier.h"
-
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/IR/Dominators.h"
-
-namespace llvm {
-
-class Function;
-
-class AMDGPUTargetVerifierPass : public TargetVerifierPass {
-public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
-};
-
-class AMDGPUTargetVerify : public TargetVerify {
-public:
-  DominatorTree *DT = nullptr;
-  PostDominatorTree *PDT = nullptr;
-  UniformityInfo *UA = nullptr;
-
-  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
-                     UniformityInfo *UA)
-      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
-  bool run(Function &F) override;
-};
-
-} // namespace llvm
-
-#endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 26fb0a967fd76..f5f083ca066a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -15,6 +15,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Target/TargetVerifier.h"
 
 namespace llvm {
 
@@ -551,6 +552,10 @@ extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
 void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+class AMDGPUTargetVerifierPass : public TargetVerifierPass {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
+};
 
 namespace AMDGPU {
 enum TargetIndex {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index be8bde1c1c0e0..2009459aac329 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,7 +90,6 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/FormatVariadic.h"
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index eb22eb2177f7f..cc96164a1a558 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -17,7 +17,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "AMDGPU.h"
 
 #include "llvm/Analysis/PostDominators.h"
@@ -44,6 +43,21 @@ using namespace llvm;
 
 namespace llvm {
 
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  DominatorTree *DT = nullptr;
+  PostDominatorTree *PDT = nullptr;
+  UniformityInfo *UA = nullptr;
+
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
+                     UniformityInfo *UA)
+      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  bool run(Function &F) override;
+};
+
 static bool isShader(CallingConv::ID CC) {
   switch (CC) {
   case CallingConv::AMDGPU_VS:

>From 88a6d6b0faeff37cd98be19fd88632f9546720be Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 7 May 2025 22:47:55 -0400
Subject: [PATCH 30/33] Remove analyses.

---
 .../lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index cc96164a1a558..d07b5fb668c14 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -19,9 +19,6 @@
 
 #include "AMDGPU.h"
 
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -45,16 +42,7 @@ namespace llvm {
 
 class AMDGPUTargetVerify : public TargetVerify {
 public:
-  DominatorTree *DT = nullptr;
-  PostDominatorTree *PDT = nullptr;
-  UniformityInfo *UA = nullptr;
-
   AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
-                     UniformityInfo *UA)
-      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
   bool run(Function &F) override;
 };
 
@@ -115,11 +103,7 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F,
                                                 FunctionAnalysisManager &AM) {
   auto *Mod = F.getParent();
 
-  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  AMDGPUTargetVerify TV(Mod);
   TV.run(F);
 
   dbgs() << TV.MessagesStr.str();

>From 0d99a659957a043ec7dfa4762a1f1c5bd34e50b0 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 8 May 2025 12:59:01 -0400
Subject: [PATCH 31/33] function name style, TargetVerifier comment, option
 name generality

---
 llvm/include/llvm/Target/TargetVerifier.h       | 14 +++++++-------
 llvm/lib/Target/AMDGPU/AMDGPU.h                 |  3 +--
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp |  2 +-
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll |  2 +-
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll |  2 +-
 6 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 3f8c710a88768..eff846694ac10 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -12,8 +12,8 @@
 // TargetVerifier looks for constructions invalid to a particular target
 // machine.
 //
-// To see what specifically is checked, look at TargetVerifier.cpp or an
-// individual backend's TargetVerifier.
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
 //
 //===----------------------------------------------------------------------===//
 
@@ -35,7 +35,7 @@ class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 
 class TargetVerify {
 protected:
-  void WriteValues(ArrayRef<const Value *> Vs) {
+  void writeValues(ArrayRef<const Value *> Vs) {
     for (const Value *V : Vs) {
       if (!V)
         continue;
@@ -52,16 +52,16 @@ class TargetVerify {
   ///
   /// This provides a nice place to put a breakpoint if you want to see why
   /// something is not correct.
-  void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
+  void checkFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
 
   /// A check failed (with values to print).
   ///
   /// This calls the Message-only version so that the above is easier to set
   /// a breakpoint on.
   template <typename T1, typename... Ts>
-  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
-    CheckFailed(Message);
-    WriteValues({V1, Vs...});
+  void checkFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
+    checkFailed(Message);
+    writeValues({V1, Vs...});
   }
 
 public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index f5f083ca066a2..af09a7e4547f3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -552,8 +552,7 @@ extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
 void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
-class AMDGPUTargetVerifierPass : public TargetVerifierPass {
-public:
+struct AMDGPUTargetVerifierPass : public TargetVerifierPass {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
 };
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 2009459aac329..370fde6cce6ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -481,7 +481,7 @@ static cl::opt<bool> HasClosedWorldAssumption(
     cl::desc("Whether has closed-world assumption at link time"),
     cl::init(false), cl::Hidden);
 
-static cl::opt<bool> VerifyTarget("verify-tgt",
+static cl::opt<bool> VerifyTarget("amdgpu-verify-tgt",
                                   cl::desc("Enable the target verifier"));
 
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index d07b5fb668c14..3246b0ac0c624 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -34,7 +34,7 @@ using namespace llvm;
 #define Check(C, ...)                                                          \
   do {                                                                         \
     if (!(C)) {                                                                \
-      TargetVerify::CheckFailed(__VA_ARGS__);                                  \
+      TargetVerify::checkFailed(__VA_ARGS__);                                  \
     }                                                                          \
   } while (false)
 
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index e2d9edda5d008..819e66bb46b0a 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
 ; CHECK: Shaders must return void
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index a2dab0ff47924..08854df01acb2 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
 ; CHECK-NOT: Shaders must return void

>From 79215beef3f1330292ecee15b422f5cbfdeea9b5 Mon Sep 17 00:00:00 2001
From: Joey F <jofernau at amd.com>
Date: Mon, 16 Jun 2025 19:09:24 -0400
Subject: [PATCH 32/33] Fix after rebase

---
 llvm/include/llvm/CodeGen/Passes.h                  | 2 --
 llvm/include/llvm/IR/Module.h                       | 5 -----
 llvm/include/llvm/Passes/StandardInstrumentations.h | 4 ++--
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 53bd70183e476..990452fa11fec 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -620,8 +620,6 @@ LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
-
-LLVM_ABI FunctionPass *createTargetVerifierLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 34ac47a2b8be4..a99937a90cbb7 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,11 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module using intrinsics to record the position of debugging
-  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
-  /// \ref BasicBlock.
-  bool IsNewDbgInfoFormat;
-
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index d7c1bbf04c328..4ee5ab2554868 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,7 +478,7 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -624,7 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM = nullptr);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };

>From f25ecf1e14d99b797105a5d42386c9019be8d4d4 Mon Sep 17 00:00:00 2001
From: jofrn <jofernau at amd.com>
Date: Tue, 8 Jul 2025 08:02:10 -0400
Subject: [PATCH 33/33] Add back int check. These are invalid for AMDGPU.

---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 49 ++++++-------------
 .../Verifier/AMDGPU/tgt-verify-int-type.ll    |  6 +++
 2 files changed, 21 insertions(+), 34 deletions(-)
 create mode 100644 llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 3246b0ac0c624..ae35322c905f1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -46,48 +46,29 @@ class AMDGPUTargetVerify : public TargetVerify {
   bool run(Function &F) override;
 };
 
-static bool isShader(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::AMDGPU_VS:
-  case CallingConv::AMDGPU_LS:
-  case CallingConv::AMDGPU_HS:
-  case CallingConv::AMDGPU_ES:
-  case CallingConv::AMDGPU_GS:
-  case CallingConv::AMDGPU_PS:
-  case CallingConv::AMDGPU_CS_Chain:
-  case CallingConv::AMDGPU_CS_ChainPreserve:
-  case CallingConv::AMDGPU_CS:
-    return true;
-  default:
-    return false;
-  }
+static bool IsValidInt(const Type *Ty) {
+  return Ty->isIntegerTy(1) ||
+         Ty->isIntegerTy(8) ||
+         Ty->isIntegerTy(16) ||
+         Ty->isIntegerTy(32) ||
+         Ty->isIntegerTy(64) ||
+         Ty->isIntegerTy(128);
 }
 
 bool AMDGPUTargetVerify::run(Function &F) {
-  // Ensure shader calling convention returns void
-  if (isShader(F.getCallingConv()))
-    Check(F.getReturnType() == Type::getVoidTy(F.getContext()),
-          "Shaders must return void");
 
   for (auto &BB : F) {
 
     for (auto &I : BB) {
 
-      if (auto *CI = dyn_cast<CallInst>(&I)) {
-        // Ensure no kernel to kernel calls.
-        CallingConv::ID CalleeCC = CI->getCallingConv();
-        if (CalleeCC == CallingConv::AMDGPU_KERNEL) {
-          CallingConv::ID CallerCC =
-              CI->getParent()->getParent()->getCallingConv();
-          Check(CallerCC != CallingConv::AMDGPU_KERNEL,
-                "A kernel may not call a kernel", CI->getParent()->getParent());
-        }
-
-        // Ensure chain intrinsics are followed by unreachables.
-        if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
-          Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
-                "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
-      }
+      // Ensure integral types are valid: i8, i16, i32, i64, i128
+      if (I.getType()->isIntegerTy())
+        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
+      for (unsigned i = 0; i < I.getNumOperands(); ++i)
+        if (I.getOperand(i)->getType()->isIntegerTy())
+          Check(IsValidInt(I.getOperand(i)->getType()),
+                "Int type is invalid.", I.getOperand(i));
+
     }
   }
 
diff --git a/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll b/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll
new file mode 100644
index 0000000000000..bd3a7106756cd
--- /dev/null
+++ b/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll
@@ -0,0 +1,6 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt -o - < %s 2>&1 | FileCheck %s
+
+define amdgpu_cs i65 @invalid_int() {
+; CHECK: Int type is invalid.
+  ret i65 0
+}


From flang-commits at lists.llvm.org  Mon Jul 14 09:50:20 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 14 Jul 2025 09:50:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Lower] fix warning (PR #148667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148667@github.com>
Message-ID: <6875354c.170a0220.a642e.8a5b@mx.google.com>


https://github.com/klausler approved this pull request.


https://github.com/llvm/llvm-project/pull/148667

From flang-commits at lists.llvm.org  Mon Jul 14 09:50:14 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 14 Jul 2025 09:50:14 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][core] NFC update builder
	create API usage (PR #147311)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147311@github.com>
Message-ID: <68753546.170a0220.22ae60.85cf@mx.google.com>


https://github.com/makslevental updated https://github.com/llvm/llvm-project/pull/147311

>From 2f00b126ca57868f4fe83bf5694d586f891aa891 Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 10:35:48 -0400
Subject: [PATCH 1/5] [mlir][core] update builder create API

---
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           | 176 ++---
 .../AffineToStandard/AffineToStandard.cpp     |  26 +-
 .../ArithToAMDGPU/ArithToAMDGPU.cpp           |  90 +--
 .../ArithToArmSME/ArithToArmSME.cpp           |   6 +-
 .../Conversion/ArithToEmitC/ArithToEmitC.cpp  |  44 +-
 .../Conversion/ArithToLLVM/ArithToLLVM.cpp    |  28 +-
 .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp  |  74 +-
 .../ArmNeon2dToIntr/ArmNeon2dToIntr.cpp       |   6 +-
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  | 122 +--
 .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp    |  68 +-
 .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp    |  66 +-
 .../BufferizationToMemRef.cpp                 |  26 +-
 .../ComplexCommon/DivisionConverter.cpp       | 392 ++++-----
 .../ComplexToLLVM/ComplexToLLVM.cpp           |  28 +-
 .../ComplexToLibm/ComplexToLibm.cpp           |   2 +-
 .../ComplexToStandard/ComplexToStandard.cpp   | 598 +++++++-------
 .../ControlFlowToLLVM/ControlFlowToLLVM.cpp   |   8 +-
 .../ControlFlowToSCF/ControlFlowToSCF.cpp     |  22 +-
 .../Conversion/FuncToEmitC/FuncToEmitC.cpp    |   2 +-
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp |  48 +-
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |  82 +-
 .../GPUCommon/GPUToLLVMConversion.cpp         |  78 +-
 .../GPUCommon/IndexIntrinsicsOpLowering.h     |  10 +-
 .../GPUCommon/OpToFuncCallLowering.h          |  12 +-
 .../Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp  |  20 +-
 .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp        |  40 +-
 .../Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp    |  40 +-
 .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp      |  50 +-
 mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp |  54 +-
 .../Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp  |   4 +-
 .../Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp  |   4 +-
 .../Conversion/IndexToLLVM/IndexToLLVM.cpp    |  68 +-
 .../Conversion/IndexToSPIRV/IndexToSPIRV.cpp  |  68 +-
 .../Conversion/LLVMCommon/MemRefBuilder.cpp   |  78 +-
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    |  62 +-
 .../Conversion/LLVMCommon/PrintCallHelper.cpp |   8 +-
 .../Conversion/LLVMCommon/StructBuilder.cpp   |   4 +-
 .../Conversion/LLVMCommon/TypeConverter.cpp   |  14 +-
 .../Conversion/LLVMCommon/VectorPattern.cpp   |   6 +-
 .../LinalgToStandard/LinalgToStandard.cpp     |   4 +-
 mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp   |  64 +-
 .../Conversion/MathToFuncs/MathToFuncs.cpp    | 180 ++---
 mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp |  38 +-
 mlir/lib/Conversion/MathToLibm/MathToLibm.cpp |  14 +-
 .../Conversion/MathToSPIRV/MathToSPIRV.cpp    |  82 +-
 .../MemRefToEmitC/MemRefToEmitC.cpp           |   6 +-
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  | 138 ++--
 .../MemRefToSPIRV/MemRefToSPIRV.cpp           |  18 +-
 mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp   | 204 ++---
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    | 158 ++--
 .../Conversion/OpenACCToSCF/OpenACCToSCF.cpp  |   2 +-
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |   2 +-
 .../PDLToPDLInterp/PDLToPDLInterp.cpp         | 112 +--
 .../SCFToControlFlow/SCFToControlFlow.cpp     |  36 +-
 mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp |  14 +-
 mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp     |  40 +-
 .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp    |  40 +-
 mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp |  40 +-
 .../ConvertLaunchFuncToLLVMCalls.cpp          |   8 +-
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    | 124 +--
 .../ConvertShapeConstraints.cpp               |   2 +-
 .../ShapeToStandard/ShapeToStandard.cpp       | 150 ++--
 .../TensorToSPIRV/TensorToSPIRV.cpp           |   6 +-
 .../Conversion/TosaToArith/TosaToArith.cpp    | 100 +--
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  | 742 +++++++++---------
 .../TosaToLinalg/TosaToLinalgNamed.cpp        | 218 ++---
 .../TosaToMLProgram/TosaToMLProgram.cpp       |   6 +-
 mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp   |  24 +-
 .../Conversion/TosaToTensor/TosaToTensor.cpp  |  20 +-
 .../VectorToArmSME/VectorToArmSME.cpp         |  56 +-
 .../Conversion/VectorToGPU/VectorToGPU.cpp    |  76 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      | 158 ++--
 .../Conversion/VectorToSCF/VectorToSCF.cpp    | 166 ++--
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  48 +-
 .../VectorToXeGPU/VectorToXeGPU.cpp           |  20 +-
 .../AMDGPU/Transforms/EmulateAtomics.cpp      |  20 +-
 .../AMDGPU/Transforms/MaskedloadToLoad.cpp    |  28 +-
 .../Transforms/ResolveStridedMetadata.cpp     |   6 +-
 mlir/lib/Dialect/AMX/IR/AMXDialect.cpp        |   8 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  34 +-
 .../Transforms/AffineDataCopyGeneration.cpp   |   2 +-
 .../Transforms/AffineExpandIndexOps.cpp       |  20 +-
 .../Affine/Transforms/DecomposeAffineOps.cpp  |   4 +-
 .../Dialect/Affine/Transforms/LoopFusion.cpp  |   2 +-
 .../Transforms/PipelineDataTransfer.cpp       |   6 +-
 .../Affine/Transforms/ReifyValueBounds.cpp    |   6 +-
 .../Affine/Transforms/SuperVectorize.cpp      |  24 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |  86 +-
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |  88 +--
 mlir/lib/Dialect/Arith/IR/ArithDialect.cpp    |   2 +-
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |  40 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |   2 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/EmulateUnsupportedFloats.cpp   |   4 +-
 .../Arith/Transforms/EmulateWideInt.cpp       | 176 ++---
 .../Dialect/Arith/Transforms/ExpandOps.cpp    | 228 +++---
 .../Transforms/IntRangeOptimizations.cpp      |  10 +-
 .../Arith/Transforms/ReifyValueBounds.cpp     |  16 +-
 .../Transforms/ShardingInterfaceImpl.cpp      |   2 +-
 mlir/lib/Dialect/Arith/Utils/Utils.cpp        |  90 +--
 .../LowerContractionToNeonI8MMPattern.cpp     |   8 +-
 mlir/lib/Dialect/ArmSME/IR/Utils.cpp          |  14 +-
 .../ArmSME/Transforms/OuterProductFusion.cpp  |   8 +-
 .../ArmSME/Transforms/TileAllocation.cpp      |   4 +-
 .../ArmSME/Transforms/VectorLegalization.cpp  | 110 +--
 .../Transforms/LegalizeForLLVMExport.cpp      |  16 +-
 .../Transforms/LegalizeVectorStorage.cpp      |  12 +-
 .../LowerContractionToSVEI8MMPattern.cpp      |  60 +-
 mlir/lib/Dialect/Async/IR/Async.cpp           |   2 +-
 .../Async/Transforms/AsyncParallelFor.cpp     | 194 ++---
 .../Transforms/AsyncRuntimeRefCounting.cpp    |  14 +-
 .../Async/Transforms/AsyncToAsyncRuntime.cpp  |  78 +-
 .../IR/BufferDeallocationOpInterface.cpp      |  10 +-
 .../IR/BufferizableOpInterface.cpp            |  10 +-
 .../Bufferization/IR/BufferizationOps.cpp     |  30 +-
 .../BufferizationTransformOps.cpp             |   4 +-
 .../BufferDeallocationSimplification.cpp      |  12 +-
 .../Transforms/BufferResultsToOutParams.cpp   |   6 +-
 .../Bufferization/Transforms/BufferUtils.cpp  |   2 +-
 .../Bufferization/Transforms/Bufferize.cpp    |   6 +-
 .../DropEquivalentBufferResults.cpp           |   4 +-
 .../Transforms/EmptyTensorElimination.cpp     |   2 +-
 .../FuncBufferizableOpInterfaceImpl.cpp       |   4 +-
 .../Transforms/LowerDeallocations.cpp         | 142 ++--
 .../OwnershipBasedBufferDeallocation.cpp      |  14 +-
 .../lib/Dialect/Complex/IR/ComplexDialect.cpp |   2 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.cpp |   6 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |  12 +-
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp           |   4 +-
 .../Dialect/EmitC/Transforms/Transforms.cpp   |   4 +-
 .../EmitC/Transforms/TypeConversions.cpp      |   2 +-
 .../EmitC/Transforms/WrapFuncInClass.cpp      |   8 +-
 .../Func/Extensions/InlinerExtension.cpp      |   2 +-
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |   2 +-
 .../Func/TransformOps/FuncTransformOps.cpp    |   2 +-
 .../Func/Transforms/FuncConversions.cpp       |   2 +-
 mlir/lib/Dialect/Func/Utils/Utils.cpp         |   6 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |  10 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |  10 +-
 mlir/lib/Dialect/GPU/TransformOps/Utils.cpp   |  24 +-
 .../GPU/Transforms/AllReduceLowering.cpp      |   2 +-
 .../GPU/Transforms/AsyncRegionRewriter.cpp    |   8 +-
 .../GPU/Transforms/DecomposeMemRefs.cpp       |   4 +-
 .../GPU/Transforms/GlobalIdRewriter.cpp       |   8 +-
 .../GPU/Transforms/KernelOutlining.cpp        |  10 +-
 .../GPU/Transforms/MemoryPromotion.cpp        |  16 +-
 .../Dialect/GPU/Transforms/ModuleToBinary.cpp |   2 +-
 .../GPU/Transforms/PromoteShuffleToAMDGPU.cpp |   4 +-
 .../GPU/Transforms/ShuffleRewriter.cpp        |  22 +-
 .../GPU/Transforms/SubgroupIdRewriter.cpp     |  22 +-
 .../GPU/Transforms/SubgroupReduceLowering.cpp |  70 +-
 .../Dialect/GPU/Utils/DistributionUtils.cpp   |   4 +-
 mlir/lib/Dialect/Index/IR/IndexOps.cpp        |   8 +-
 .../LLVMIR/IR/BasicPtxBuilderInterface.cpp    |   4 +-
 .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp   |   2 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |  16 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp |  46 +-
 .../Dialect/LLVMIR/Transforms/AddComdats.cpp  |   4 +-
 .../Transforms/InlinerInterfaceImpl.cpp       |  22 +-
 .../LLVMIR/Transforms/LegalizeForExport.cpp   |   2 +-
 .../LLVMIR/Transforms/OptimizeForNVVM.cpp     |  30 +-
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 202 ++---
 .../TransformOps/LinalgTransformOps.cpp       |  14 +-
 .../BufferizableOpInterfaceImpl.cpp           |   2 +-
 .../Transforms/ConvertConv2DToImg2Col.cpp     | 108 +--
 .../Transforms/ConvertToDestinationStyle.cpp  |  52 +-
 .../Transforms/DataLayoutPropagation.cpp      |  30 +-
 ...DecomposeGenericByUnfoldingPermutation.cpp |  10 +-
 .../Linalg/Transforms/DecomposeLinalgOps.cpp  |  14 +-
 .../Dialect/Linalg/Transforms/Detensorize.cpp |   6 +-
 .../Linalg/Transforms/DropUnitDims.cpp        |  22 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  40 +-
 .../Linalg/Transforms/ElementwiseToLinalg.cpp |   4 +-
 .../EraseUnusedOperandsAndResults.cpp         |   2 +-
 .../FusePadOpWithLinalgProducer.cpp           |   8 +-
 mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp |   2 +-
 .../Linalg/Transforms/Generalization.cpp      |   2 +-
 .../Linalg/Transforms/HoistPadding.cpp        |  20 +-
 .../Dialect/Linalg/Transforms/Hoisting.cpp    |   2 +-
 .../Transforms/InlineScalarOperands.cpp       |   6 +-
 .../Dialect/Linalg/Transforms/Interchange.cpp |   2 +-
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |   8 +-
 .../Transforms/MeshShardingInterfaceImpl.cpp  |  18 +-
 .../Linalg/Transforms/NamedOpConversions.cpp  |   8 +-
 .../Transforms/PackAndUnpackPatterns.cpp      |   6 +-
 .../Linalg/Transforms/PadTilingInterface.cpp  |   6 +-
 .../lib/Dialect/Linalg/Transforms/Padding.cpp |   6 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |  44 +-
 .../Transforms/RuntimeOpVerification.cpp      |   4 +-
 mlir/lib/Dialect/Linalg/Transforms/Split.cpp  |   2 +-
 .../Linalg/Transforms/SplitReduction.cpp      |  32 +-
 .../SwapExtractSliceWithFillPatterns.cpp      |   2 +-
 mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp |  16 +-
 .../Linalg/Transforms/TilingInterfaceImpl.cpp |  48 +-
 .../Dialect/Linalg/Transforms/Transforms.cpp  |  66 +-
 .../Linalg/Transforms/TransposeConv2D.cpp     |   6 +-
 .../Linalg/Transforms/TransposeMatmul.cpp     |  26 +-
 .../Linalg/Transforms/Vectorization.cpp       | 176 ++---
 .../Linalg/Transforms/WinogradConv2D.cpp      | 114 +--
 mlir/lib/Dialect/Linalg/Utils/Utils.cpp       |  18 +-
 .../BufferizableOpInterfaceImpl.cpp           |   2 +-
 mlir/lib/Dialect/MPI/IR/MPIOps.cpp            |   2 +-
 mlir/lib/Dialect/Math/IR/MathOps.cpp          |   2 +-
 .../Transforms/AlgebraicSimplification.cpp    |  20 +-
 .../Math/Transforms/ExpandPatterns.cpp        | 226 +++---
 .../Transforms/ExtendToSupportedTypes.cpp     |   4 +-
 .../Transforms/PolynomialApproximation.cpp    | 536 ++++++-------
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    |   6 +-
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |  22 +-
 .../TransformOps/MemRefTransformOps.cpp       |   2 +-
 .../Transforms/AllocationOpInterfaceImpl.cpp  |   8 +-
 .../MemRef/Transforms/ComposeSubView.cpp      |   2 +-
 .../MemRef/Transforms/EmulateNarrowType.cpp   |  37 +-
 .../Dialect/MemRef/Transforms/ExpandOps.cpp   |  14 +-
 .../MemRef/Transforms/ExpandRealloc.cpp       |  20 +-
 .../Transforms/ExpandStridedMetadata.cpp      |  24 +-
 .../Transforms/ExtractAddressComputations.cpp |  16 +-
 .../MemRef/Transforms/FlattenMemRefs.cpp      |  28 +-
 .../MemRef/Transforms/FoldMemRefAliasOps.cpp  |   6 +-
 .../Transforms/IndependenceTransforms.cpp     |   8 +-
 .../Dialect/MemRef/Transforms/MultiBuffer.cpp |   8 +-
 .../MemRef/Transforms/NormalizeMemRefs.cpp    |   2 +-
 .../MemRef/Transforms/ReifyResultShapes.cpp   |   4 +-
 .../ResolveShapedTypeResultDims.cpp           |   2 +-
 .../Transforms/RuntimeOpVerification.cpp      |  88 +--
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |  14 +-
 .../Mesh/Transforms/Simplifications.cpp       |   4 +-
 .../Dialect/Mesh/Transforms/Spmdization.cpp   |  14 +-
 .../Dialect/Mesh/Transforms/Transforms.cpp    |  36 +-
 .../NVGPU/TransformOps/NVGPUTransformOps.cpp  |  62 +-
 .../NVGPU/Transforms/CreateAsyncGroups.cpp    |  16 +-
 .../NVGPU/Transforms/OptimizeSharedMemory.cpp |  12 +-
 .../Quant/Transforms/LowerQuantOps.cpp        |  78 +-
 .../Quant/Transforms/StripFuncQuantTypes.cpp  |   2 +-
 mlir/lib/Dialect/SCF/IR/SCF.cpp               |  78 +-
 .../SCF/TransformOps/SCFTransformOps.cpp      |   4 +-
 .../BufferizableOpInterfaceImpl.cpp           |  18 +-
 .../lib/Dialect/SCF/Transforms/ForToWhile.cpp |   8 +-
 .../SCF/Transforms/ForallToParallel.cpp       |   2 +-
 .../Dialect/SCF/Transforms/LoopPipelining.cpp |  70 +-
 .../SCF/Transforms/LoopSpecialization.cpp     |  14 +-
 .../SCF/Transforms/ParallelLoopFusion.cpp     |   4 +-
 .../SCF/Transforms/ParallelLoopTiling.cpp     |  32 +-
 .../Transforms/StructuralTypeConversions.cpp  |   6 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  32 +-
 .../SCF/Transforms/UpliftWhileToFor.cpp       |  20 +-
 .../SCF/Transforms/WrapInZeroTripCheck.cpp    |  10 +-
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          |  94 +--
 mlir/lib/Dialect/SMT/IR/SMTDialect.cpp        |   4 +-
 mlir/lib/Dialect/SMT/IR/SMTOps.cpp            |   4 +-
 mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp  |  10 +-
 .../SPIRV/IR/SPIRVCanonicalization.cpp        |  20 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp    |   4 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp        |  22 +-
 .../Linking/ModuleCombiner/ModuleCombiner.cpp |   2 +-
 .../Transforms/LowerABIAttributesPass.cpp     |  18 +-
 .../SPIRV/Transforms/RewriteInsertsPass.cpp   |   2 +-
 .../SPIRV/Transforms/SPIRVConversion.cpp      |  62 +-
 .../Transforms/SPIRVWebGPUTransforms.cpp      |  32 +-
 .../Transforms/UnifyAliasedResourcePass.cpp   |  28 +-
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  30 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/OutlineShapeComputation.cpp    |   6 +-
 .../Shape/Transforms/ShapeToShapeLowering.cpp |   6 +-
 .../SparseTensor/IR/SparseTensorDialect.cpp   |   4 +-
 .../IR/SparseTensorInterfaces.cpp             |   2 +-
 .../Transforms/SparseAssembler.cpp            |  16 +-
 .../Transforms/SparseBufferRewriting.cpp      | 366 ++++-----
 .../Transforms/SparseGPUCodegen.cpp           | 178 ++---
 .../Transforms/SparseIterationToScf.cpp       |  26 +-
 .../Transforms/SparseReinterpretMap.cpp       |  20 +-
 .../Transforms/SparseSpaceCollapse.cpp        |   2 +-
 .../SparseStorageSpecifierToLLVM.cpp          |  10 +-
 .../Transforms/SparseTensorCodegen.cpp        | 138 ++--
 .../Transforms/SparseTensorConversion.cpp     |  44 +-
 .../Transforms/SparseTensorRewriting.cpp      | 212 ++---
 .../Transforms/SparseVectorization.cpp        |  50 +-
 .../Transforms/Sparsification.cpp             |  68 +-
 .../Transforms/StageSparseOperations.cpp      |   4 +-
 .../Transforms/Utils/CodegenUtils.cpp         |  74 +-
 .../Transforms/Utils/CodegenUtils.h           |  30 +-
 .../Transforms/Utils/LoopEmitter.cpp          |  54 +-
 .../Utils/SparseTensorDescriptor.cpp          |  10 +-
 .../Transforms/Utils/SparseTensorDescriptor.h |   2 +-
 .../Transforms/Utils/SparseTensorIterator.cpp |  76 +-
 .../lib/Dialect/SparseTensor/Utils/Merger.cpp | 116 +--
 .../Extensions/MeshShardingExtensions.cpp     |   8 +-
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      |  66 +-
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp   |  20 +-
 .../TransformOps/TensorTransformOps.cpp       |   4 +-
 .../BufferizableOpInterfaceImpl.cpp           |  44 +-
 .../Tensor/Transforms/EmptyOpPatterns.cpp     |   2 +-
 .../ExtractSliceFromReshapeUtils.cpp          |   8 +-
 .../Tensor/Transforms/FoldTensorSubsetOps.cpp |   2 +-
 .../Transforms/IndependenceTransforms.cpp     |   8 +-
 .../Tensor/Transforms/ReshapePatterns.cpp     |  10 +-
 .../Tensor/Transforms/RewriteAsConstant.cpp   |   2 +-
 .../Transforms/RuntimeOpVerification.cpp      |  40 +-
 .../SubsetInsertionOpInterfaceImpl.cpp        |   2 +-
 mlir/lib/Dialect/Tensor/Utils/Utils.cpp       |   6 +-
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |   4 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |  12 +-
 .../Transforms/TosaDecomposeDepthwise.cpp     |  16 +-
 .../Transforms/TosaDecomposeTransposeConv.cpp |   8 +-
 .../Tosa/Transforms/TosaInferShapes.cpp       |   2 +-
 .../Tosa/Transforms/TosaReduceTransposes.cpp  |   4 +-
 .../Tosa/Transforms/TosaTypeConverters.cpp    |   4 +-
 .../Dialect/Tosa/Utils/ConversionUtils.cpp    |  16 +-
 mlir/lib/Dialect/UB/IR/UBOps.cpp              |   2 +-
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  60 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Vector/Transforms/LowerVectorBitCast.cpp  |   8 +-
 .../Transforms/LowerVectorBroadcast.cpp       |  22 +-
 .../Vector/Transforms/LowerVectorContract.cpp | 128 +--
 .../Vector/Transforms/LowerVectorGather.cpp   |  36 +-
 .../Transforms/LowerVectorInterleave.cpp      |  20 +-
 .../Vector/Transforms/LowerVectorMask.cpp     |  22 +-
 .../Transforms/LowerVectorMultiReduction.cpp  |  38 +-
 .../Vector/Transforms/LowerVectorScan.cpp     |  16 +-
 .../Transforms/LowerVectorShapeCast.cpp       |  36 +-
 .../Vector/Transforms/LowerVectorTransfer.cpp |  26 +-
 .../Transforms/LowerVectorTranspose.cpp       |  26 +-
 .../Vector/Transforms/VectorDistribute.cpp    |  76 +-
 .../Transforms/VectorDropLeadUnitDim.cpp      |  34 +-
 .../VectorEmulateMaskedLoadStore.cpp          |  28 +-
 .../Transforms/VectorEmulateNarrowType.cpp    | 177 ++---
 ...sertExtractStridedSliceRewritePatterns.cpp |  30 +-
 .../Vector/Transforms/VectorLinearize.cpp     |   6 +-
 .../Transforms/VectorMaskElimination.cpp      |   2 +-
 .../Transforms/VectorTransferOpTransforms.cpp |  26 +-
 .../VectorTransferSplitRewritePatterns.cpp    |  72 +-
 .../Vector/Transforms/VectorTransforms.cpp    |  82 +-
 .../Vector/Transforms/VectorUnroll.cpp        |  32 +-
 mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp |   6 +-
 .../Dialect/X86Vector/IR/X86VectorDialect.cpp |   6 +-
 .../X86Vector/Transforms/AVXTranspose.cpp     |  26 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   4 +-
 .../Transforms/XeGPUSubgroupDistribute.cpp    |  28 +-
 .../Dialect/XeGPU/Transforms/XeGPUUnroll.cpp  |  38 +-
 .../Transforms/XeGPUWgToSgDistribute.cpp      |  22 +-
 mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp   |  12 +-
 mlir/lib/IR/BuiltinDialect.cpp                |   2 +-
 mlir/lib/Query/Query.cpp                      |   2 +-
 .../lib/Target/LLVMIR/LLVMImportInterface.cpp |   2 +-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |  76 +-
 .../SPIRV/Deserialization/DeserializeOps.cpp  |  20 +-
 .../SPIRV/Deserialization/Deserializer.cpp    |  36 +-
 mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp       |  44 +-
 .../Transforms/Utils/DialectConversion.cpp    |   2 +-
 349 files changed, 6991 insertions(+), 6987 deletions(-)

diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index ef35ee208f002..e7e0a4ebd2092 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -50,20 +50,20 @@ static Value convertUnsignedToI32(ConversionPatternRewriter &rewriter,
   if (i32 == valTy)
     return val;
   return valTy.getWidth() > 32
-             ? Value(rewriter.create<LLVM::TruncOp>(loc, i32, val))
-             : Value(rewriter.create<LLVM::ZExtOp>(loc, i32, val));
+             ? Value(LLVM::TruncOp::create(rewriter, loc, i32, val))
+             : Value(LLVM::ZExtOp::create(rewriter, loc, i32, val));
 }
 
 static Value createI32Constant(ConversionPatternRewriter &rewriter,
                                Location loc, int32_t value) {
   Type i32 = rewriter.getI32Type();
-  return rewriter.create<LLVM::ConstantOp>(loc, i32, value);
+  return LLVM::ConstantOp::create(rewriter, loc, i32, value);
 }
 
 static Value createI1Constant(ConversionPatternRewriter &rewriter, Location loc,
                               bool value) {
   Type llvmI1 = rewriter.getI1Type();
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmI1, value);
+  return LLVM::ConstantOp::create(rewriter, loc, llvmI1, value);
 }
 
 /// Returns the linear index used to access an element in the memref.
@@ -78,11 +78,11 @@ static Value getLinearIndexI32(ConversionPatternRewriter &rewriter,
           ShapedType::isDynamic(stride)
               ? convertUnsignedToI32(rewriter, loc,
                                      memRefDescriptor.stride(rewriter, loc, i))
-              : rewriter.create<LLVM::ConstantOp>(loc, i32, stride);
-      increment = rewriter.create<LLVM::MulOp>(loc, increment, strideValue);
+              : LLVM::ConstantOp::create(rewriter, loc, i32, stride);
+      increment = LLVM::MulOp::create(rewriter, loc, increment, strideValue);
     }
     index =
-        index ? rewriter.create<LLVM::AddOp>(loc, index, increment) : increment;
+        index ? LLVM::AddOp::create(rewriter, loc, index, increment) : increment;
   }
   return index ? index : createI32Constant(rewriter, loc, 0);
 }
@@ -110,14 +110,14 @@ static Value getNumRecords(ConversionPatternRewriter &rewriter, Location loc,
   for (uint32_t i = 0, e = memrefType.getRank(); i < e; ++i) {
     Value size = memrefDescriptor.size(rewriter, loc, i);
     Value stride = memrefDescriptor.stride(rewriter, loc, i);
-    Value maxThisDim = rewriter.create<LLVM::MulOp>(loc, size, stride);
+    Value maxThisDim = LLVM::MulOp::create(rewriter, loc, size, stride);
     maxIndex = maxIndex
-                   ? rewriter.create<LLVM::UMaxOp>(loc, maxIndex, maxThisDim)
+                   ? LLVM::UMaxOp::create(rewriter, loc, maxIndex, maxThisDim)
                    : maxThisDim;
   }
   Value maxIndexI32 = convertUnsignedToI32(rewriter, loc, maxIndex);
   Value byteWidthConst = createI32Constant(rewriter, loc, elementByteWidth);
-  return rewriter.create<LLVM::MulOp>(loc, maxIndexI32, byteWidthConst);
+  return LLVM::MulOp::create(rewriter, loc, maxIndexI32, byteWidthConst);
 }
 
 static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
@@ -132,13 +132,13 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
   Value stride;
   if (chipset.majorVersion == 9 && chipset >= kGfx942 && cacheSwizzleStride) {
     Value cacheStrideZext =
-        rewriter.create<LLVM::ZExtOp>(loc, i16, cacheSwizzleStride);
-    Value swizzleBit = rewriter.create<LLVM::ConstantOp>(
+        LLVM::ZExtOp::create(rewriter, loc, i16, cacheSwizzleStride);
+    Value swizzleBit = LLVM::ConstantOp::create(rewriter,
         loc, i16, rewriter.getI16IntegerAttr(1 << 14));
-    stride = rewriter.create<LLVM::OrOp>(loc, cacheStrideZext, swizzleBit,
+    stride = LLVM::OrOp::create(rewriter, loc, cacheStrideZext, swizzleBit,
                                          /*isDisjoint=*/true);
   } else {
-    stride = rewriter.create<LLVM::ConstantOp>(loc, i16,
+    stride = LLVM::ConstantOp::create(rewriter, loc, i16,
                                                rewriter.getI16IntegerAttr(0));
   }
   // Get the number of elements.
@@ -209,18 +209,18 @@ struct FatRawBufferCastLowering
             : descriptor.alignedPtr(rewriter, loc);
 
     Value offset = adaptor.getResetOffset()
-                       ? rewriter.create<LLVM::ConstantOp>(
+                       ? LLVM::ConstantOp::create(rewriter,
                              loc, getIndexType(), rewriter.getIndexAttr(0))
                        : descriptor.offset(rewriter, loc);
 
     bool hasSizes = memrefType.getRank() > 0;
     // No need to unpack() and pack() all the individual sizes and strides,
     // so we'll just extract the arrays.
-    Value sizes = hasSizes ? rewriter.create<LLVM::ExtractValueOp>(
+    Value sizes = hasSizes ? LLVM::ExtractValueOp::create(rewriter,
                                  loc, descriptor, kSizePosInMemRefDescriptor)
                            : Value{};
     Value strides = hasSizes
-                        ? rewriter.create<LLVM::ExtractValueOp>(
+                        ? LLVM::ExtractValueOp::create(rewriter,
                               loc, descriptor, kStridePosInMemRefDescriptor)
                         : Value{};
 
@@ -231,16 +231,16 @@ struct FatRawBufferCastLowering
     Value result = MemRefDescriptor::poison(
         rewriter, loc,
         getTypeConverter()->convertType(op.getResult().getType()));
-    result = rewriter.create<LLVM::InsertValueOp>(
+    result = LLVM::InsertValueOp::create(rewriter,
         loc, result, fatPtr, kAllocatedPtrPosInMemRefDescriptor);
-    result = rewriter.create<LLVM::InsertValueOp>(
+    result = LLVM::InsertValueOp::create(rewriter,
         loc, result, fatPtr, kAlignedPtrPosInMemRefDescriptor);
-    result = rewriter.create<LLVM::InsertValueOp>(loc, result, offset,
+    result = LLVM::InsertValueOp::create(rewriter, loc, result, offset,
                                                   kOffsetPosInMemRefDescriptor);
     if (hasSizes) {
-      result = rewriter.create<LLVM::InsertValueOp>(loc, result, sizes,
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, sizes,
                                                     kSizePosInMemRefDescriptor);
-      result = rewriter.create<LLVM::InsertValueOp>(
+      result = LLVM::InsertValueOp::create(rewriter,
           loc, result, strides, kStridePosInMemRefDescriptor);
     }
     rewriter.replaceOp(op, result);
@@ -343,7 +343,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     if (storeData) {
       if (llvmBufferValType != llvmWantedDataType) {
         Value castForStore =
-            rewriter.create<LLVM::BitcastOp>(loc, llvmBufferValType, storeData);
+            LLVM::BitcastOp::create(rewriter, loc, llvmBufferValType, storeData);
         args.push_back(castForStore);
       } else {
         args.push_back(storeData);
@@ -352,7 +352,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     if (atomicCmpData) {
       if (llvmBufferValType != llvmWantedDataType) {
-        Value castForCmp = rewriter.create<LLVM::BitcastOp>(
+        Value castForCmp = LLVM::BitcastOp::create(rewriter,
             loc, llvmBufferValType, atomicCmpData);
         args.push_back(castForCmp);
       } else {
@@ -383,17 +383,17 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
         indexOffset && *indexOffset > 0) {
       Value extraOffsetConst = createI32Constant(rewriter, loc, *indexOffset);
       voffset =
-          voffset ? rewriter.create<LLVM::AddOp>(loc, voffset, extraOffsetConst)
+          voffset ? LLVM::AddOp::create(rewriter, loc, voffset, extraOffsetConst)
                   : extraOffsetConst;
     }
-    voffset = rewriter.create<LLVM::MulOp>(loc, voffset, byteWidthConst);
+    voffset = LLVM::MulOp::create(rewriter, loc, voffset, byteWidthConst);
     args.push_back(voffset);
 
     // SGPR offset.
     Value sgprOffset = adaptor.getSgprOffset();
     if (!sgprOffset)
       sgprOffset = createI32Constant(rewriter, loc, 0);
-    sgprOffset = rewriter.create<LLVM::MulOp>(loc, sgprOffset, byteWidthConst);
+    sgprOffset = LLVM::MulOp::create(rewriter, loc, sgprOffset, byteWidthConst);
     args.push_back(sgprOffset);
 
     // bit 0: GLC = 0 (atomics drop value, less coherency)
@@ -403,12 +403,12 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
                                            llvmBufferValType);
-    Operation *lowered = rewriter.create<Intrinsic>(loc, resultTypes, args,
+    Operation *lowered = Intrinsic::create(rewriter, loc, resultTypes, args,
                                                     ArrayRef<NamedAttribute>());
     if (lowered->getNumResults() == 1) {
       Value replacement = lowered->getResult(0);
       if (llvmBufferValType != llvmWantedDataType) {
-        replacement = rewriter.create<LLVM::BitcastOp>(loc, llvmWantedDataType,
+        replacement = LLVM::BitcastOp::create(rewriter, loc, llvmWantedDataType,
                                                        replacement);
       }
       rewriter.replaceOp(gpuOp, replacement);
@@ -465,12 +465,12 @@ struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
                << chipset.majorVersion;
 
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::SWaitcntOp>(loc, ldsOnlyBits);
+      ROCDL::SWaitcntOp::create(rewriter, loc, ldsOnlyBits);
       rewriter.replaceOpWithNewOp<ROCDL::SBarrierOp>(op);
     } else {
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::WaitDscntOp>(loc, 0);
-      rewriter.create<ROCDL::BarrierSignalOp>(loc, -1);
+      ROCDL::WaitDscntOp::create(rewriter, loc, 0);
+      ROCDL::BarrierSignalOp::create(rewriter, loc, -1);
       rewriter.replaceOpWithNewOp<ROCDL::BarrierWaitOp>(op, -1);
     }
 
@@ -516,18 +516,18 @@ static Value convertMFMAVectorOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   if (auto vectorType = dyn_cast<VectorType>(inputType)) {
     if (vectorType.getElementType().isBF16() && !allowBf16)
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, vectorType.clone(rewriter.getI16Type()), input);
     if (vectorType.getElementType().isInteger(8) &&
         vectorType.getNumElements() <= 8)
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, rewriter.getIntegerType(vectorType.getNumElements() * 8), input);
     if (isa<IntegerType>(vectorType.getElementType()) &&
         vectorType.getElementTypeBitWidth() <= 8) {
       int64_t numWords = llvm::divideCeil(
           vectorType.getNumElements() * vectorType.getElementTypeBitWidth(),
           32);
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, VectorType::get(numWords, rewriter.getI32Type()), input);
     }
   }
@@ -549,8 +549,8 @@ static Value castMFMAScaleOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   Type outputType = rewriter.getI32Type();
   if (auto intType = dyn_cast<IntegerType>(inputType))
-    return rewriter.create<LLVM::ZExtOp>(loc, outputType, input);
-  return rewriter.create<LLVM::BitcastOp>(loc, outputType, input);
+    return LLVM::ZExtOp::create(rewriter, loc, outputType, input);
+  return LLVM::BitcastOp::create(rewriter, loc, outputType, input);
 }
 
 /// Push an input operand. If it is a float type, nothing to do. If it is
@@ -576,7 +576,7 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   Type elemType = vectorType.getElementType();
 
   if (elemType.isBF16())
-    llvmInput = rewriter.create<LLVM::BitcastOp>(
+    llvmInput = LLVM::BitcastOp::create(rewriter,
         loc, vectorType.clone(rewriter.getI16Type()), llvmInput);
   if (elemType.getIntOrFloatBitWidth() > 8) {
     operands.push_back(llvmInput);
@@ -613,7 +613,7 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   // (256 / 64) * 4 = 16 bits of input (on gfx12+) but take i32 arguments.
   // Add in the zeros here.
   if (numBits < 32)
-    castInput = rewriter.create<LLVM::ZExtOp>(loc, i32, castInput);
+    castInput = LLVM::ZExtOp::create(rewriter, loc, i32, castInput);
   operands.push_back(castInput);
 }
 
@@ -633,7 +633,7 @@ static void wmmaPushOutputOperand(ConversionPatternRewriter &rewriter,
   auto vectorType = dyn_cast<VectorType>(inputType);
   Type elemType = vectorType.getElementType();
   if (elemType.isBF16())
-    output = rewriter.create<LLVM::BitcastOp>(
+    output = LLVM::BitcastOp::create(rewriter,
         loc, vectorType.clone(rewriter.getI16Type()), output);
   operands.push_back(output);
   if (elemType.isF16() || elemType.isBF16() || elemType.isInteger(16)) {
@@ -992,7 +992,7 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
     };
     Value lowered = rewriter.create(loweredOp)->getResult(0);
     if (outType != intrinsicOutType)
-      lowered = rewriter.create<LLVM::BitcastOp>(loc, outType, lowered);
+      lowered = LLVM::BitcastOp::create(rewriter, loc, outType, lowered);
     rewriter.replaceOp(op, lowered);
     return success();
   }
@@ -1093,7 +1093,7 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
     Operation *maybeCastBack = lowered;
     if (rawOutType != outType)
       maybeCastBack =
-          rewriter.create<LLVM::BitcastOp>(loc, outType, lowered->getResult(0));
+          LLVM::BitcastOp::create(rewriter, loc, outType, lowered->getResult(0));
     rewriter.replaceOp(op, maybeCastBack->getResults());
 
     return success();
@@ -1144,21 +1144,21 @@ struct TransposeLoadOpLowering
     case 4: {
       assert(numElements == 16);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr4_b64>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr4_b64::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 6: {
       assert(numElements == 16);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr6_b96>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr6_b96::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 8: {
       assert(numElements == 8);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr8_b64>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr8_b64::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
@@ -1316,21 +1316,21 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
   Type sourceElemType = getElementTypeOrSelf(op.getSource());
   // Extend to a v4i8
   if (!sourceVecType || sourceVecType.getNumElements() < 4) {
-    Value longVec = rewriter.create<LLVM::UndefOp>(loc, v4i8);
+    Value longVec = LLVM::UndefOp::create(rewriter, loc, v4i8);
     if (!sourceVecType) {
-      longVec = rewriter.create<LLVM::InsertElementOp>(
+      longVec = LLVM::InsertElementOp::create(rewriter,
           loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
-        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        Value elem = LLVM::ExtractElementOp::create(rewriter, loc, source, idx);
         longVec =
-            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+            LLVM::InsertElementOp::create(rewriter, loc, longVec, elem, idx);
       }
     }
     source = longVec;
   }
-  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+  Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
   if (resultVecType) {
     if (typeIsExpectedBf8ForChipset(chipset, sourceElemType)) {
       rewriter.replaceOpWithNewOp<ROCDL::CvtPkF32Bf8Op>(op, f32, i32Source,
@@ -1382,21 +1382,21 @@ LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
 
   // Extend to a packedVectorType
   if (sourceVecType.getNumElements() < packedVecType.getNumElements()) {
-    Value longVec = rewriter.create<LLVM::ZeroOp>(loc, packedVecType);
+    Value longVec = LLVM::ZeroOp::create(rewriter, loc, packedVecType);
     if (!sourceVecType) {
-      longVec = rewriter.create<LLVM::InsertElementOp>(
+      longVec = LLVM::InsertElementOp::create(rewriter,
           loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
-        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        Value elem = LLVM::ExtractElementOp::create(rewriter, loc, source, idx);
         longVec =
-            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+            LLVM::InsertElementOp::create(rewriter, loc, longVec, elem, idx);
       }
     }
     source = longVec;
   }
-  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+  Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
 
   if (isa<Float8E5M2Type>(sourceElemType) && destElemType.isF32())
     rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Bf8Op>(
@@ -1454,53 +1454,53 @@ LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
   Value scale = adaptor.getScale();
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, intResultType, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, intResultType, existing);
   else
-    existing = rewriter.create<LLVM::ZeroOp>(loc, intResultType);
+    existing = LLVM::ZeroOp::create(rewriter, loc, intResultType);
 
   if (sourceVecType.getNumElements() < 2) {
     Value c0 = createI32Constant(rewriter, loc, 0);
-    Value elem0 = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
+    Value elem0 = LLVM::ExtractElementOp::create(rewriter, loc, source, c0);
     VectorType v2 = VectorType::get(2, sourceElemType);
-    source = rewriter.create<LLVM::ZeroOp>(loc, v2);
-    source = rewriter.create<LLVM::InsertElementOp>(loc, source, elem0, c0);
+    source = LLVM::ZeroOp::create(rewriter, loc, v2);
+    source = LLVM::InsertElementOp::create(rewriter, loc, source, elem0, c0);
   }
 
   Value sourceA, sourceB;
   if (sourceElemType.isF32()) {
     Value c0 = createI32Constant(rewriter, loc, 0);
     Value c1 = createI32Constant(rewriter, loc, 1);
-    sourceA = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
-    sourceB = rewriter.create<LLVM::ExtractElementOp>(loc, source, c1);
+    sourceA = LLVM::ExtractElementOp::create(rewriter, loc, source, c0);
+    sourceB = LLVM::ExtractElementOp::create(rewriter, loc, source, c1);
   }
 
   Value result;
   if (sourceElemType.isF32() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F32Op>(
+    result = ROCDL::CvtScaleF32PkBf8F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F16Op>(
+    result = ROCDL::CvtScaleF32PkBf8F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8Bf16Op>(
+    result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F32Op>(
+    result = ROCDL::CvtScaleF32PkFp8F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F16Op>(
+    result = ROCDL::CvtScaleF32PkFp8F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8Bf16Op>(
+    result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F32Op>(
+    result = ROCDL::CvtScaleF32PkFp4F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F16Op>(
+    result = ROCDL::CvtScaleF32PkFp4F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4Bf16Op>(
+    result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else
     return failure();
@@ -1526,19 +1526,19 @@ LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
   Value sourceA = adaptor.getSourceA();
   Value sourceB = adaptor.getSourceB();
   if (!sourceB)
-    sourceB = rewriter.create<LLVM::UndefOp>(loc, sourceA.getType());
+    sourceB = LLVM::UndefOp::create(rewriter, loc, sourceA.getType());
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, i32, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, i32, existing);
   else
-    existing = rewriter.create<LLVM::UndefOp>(loc, i32);
+    existing = LLVM::UndefOp::create(rewriter, loc, i32);
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtPkBf8F32Op>(loc, i32, sourceA, sourceB,
+    result = ROCDL::CvtPkBf8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
                                                    existing, op.getWordIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtPkFp8F32Op>(loc, i32, sourceA, sourceB,
+    result = ROCDL::CvtPkFp8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
                                                    existing, op.getWordIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
@@ -1563,16 +1563,16 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
   Value stoch = adaptor.getStochiasticParam();
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, i32, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, i32, existing);
   else
-    existing = rewriter.create<LLVM::UndefOp>(loc, i32);
+    existing = LLVM::UndefOp::create(rewriter, loc, i32);
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtSrBf8F32Op>(
+    result = ROCDL::CvtSrBf8F32Op::create(rewriter,
         loc, i32, source, stoch, existing, op.getStoreIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtSrFp8F32Op>(
+    result = ROCDL::CvtSrFp8F32Op::create(rewriter,
         loc, i32, source, stoch, existing, op.getStoreIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
@@ -1617,14 +1617,14 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
       if (operandType.getIntOrFloatBitWidth() <= 16) {
         if (llvm::isa<FloatType>(operandType)) {
           operand =
-              rewriter.create<LLVM::BitcastOp>(loc, llvmSrcIntType, operand);
+              LLVM::BitcastOp::create(rewriter, loc, llvmSrcIntType, operand);
         }
         auto llvmVecType = typeConverter->convertType(mlir::VectorType::get(
             32 / operandType.getIntOrFloatBitWidth(), llvmSrcIntType));
-        Value undefVec = rewriter.create<LLVM::UndefOp>(loc, llvmVecType);
-        operand = rewriter.create<LLVM::InsertElementOp>(
+        Value undefVec = LLVM::UndefOp::create(rewriter, loc, llvmVecType);
+        operand = LLVM::InsertElementOp::create(rewriter,
             loc, undefVec, operand, createI32Constant(rewriter, loc, 0));
-        operand = rewriter.create<LLVM::BitcastOp>(loc, llvmType, operand);
+        operand = LLVM::BitcastOp::create(rewriter, loc, llvmType, operand);
       }
       return operand;
     };
@@ -1711,14 +1711,14 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
     bool boundCtrl = DppOp->getAttrOfType<BoolAttr>("bound_ctrl").getValue();
 
     // create a ROCDL_DPPMovOp instruction with the appropriate attributes
-    auto dppMovOp = rewriter.create<ROCDL::DPPUpdateOp>(
+    auto dppMovOp = ROCDL::DPPUpdateOp::create(rewriter,
         loc, llvmType, old, src, DppCtrl, rowMask, bankMask, boundCtrl);
 
     Value result = dppMovOp.getRes();
     if (srcType.getIntOrFloatBitWidth() < 32) {
-      result = rewriter.create<LLVM::TruncOp>(loc, llvmSrcIntType, result);
+      result = LLVM::TruncOp::create(rewriter, loc, llvmSrcIntType, result);
       if (!llvm::isa<IntegerType>(srcType)) {
-        result = rewriter.create<LLVM::BitcastOp>(loc, srcType, result);
+        result = LLVM::BitcastOp::create(rewriter, loc, srcType, result);
       }
     }
 
@@ -1752,7 +1752,7 @@ struct AMDGPUSwizzleBitModeLowering
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
       Value res =
-          rewriter.create<ROCDL::DsSwizzleOp>(loc, v.getType(), v, maskValue);
+          ROCDL::DsSwizzleOp::create(rewriter, loc, v.getType(), v, maskValue);
       swizzled.emplace_back(res);
     }
 
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 3b143ca1ef9ce..5a9d946c2fe4e 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -50,9 +50,9 @@ static Value buildMinMaxReductionSeq(Location loc,
   Value value = *valueIt++;
   for (; valueIt != values.end(); ++valueIt) {
     if (predicate == arith::CmpIPredicate::sgt)
-      value = builder.create<arith::MaxSIOp>(loc, value, *valueIt);
+      value = arith::MaxSIOp::create(builder, loc, value, *valueIt);
     else
-      value = builder.create<arith::MinSIOp>(loc, value, *valueIt);
+      value = arith::MinSIOp::create(builder, loc, value, *valueIt);
   }
 
   return value;
@@ -154,8 +154,8 @@ class AffineForLowering : public OpRewritePattern<AffineForOp> {
     Value lowerBound = lowerAffineLowerBound(op, rewriter);
     Value upperBound = lowerAffineUpperBound(op, rewriter);
     Value step =
-        rewriter.create<arith::ConstantIndexOp>(loc, op.getStepAsInt());
-    auto scfForOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound,
+        arith::ConstantIndexOp::create(rewriter, loc, op.getStepAsInt());
+    auto scfForOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound,
                                                 step, op.getInits());
     rewriter.eraseBlock(scfForOp.getBody());
     rewriter.inlineRegionBefore(op.getRegion(), scfForOp.getRegion(),
@@ -197,7 +197,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     }
     steps.reserve(op.getSteps().size());
     for (int64_t step : op.getSteps())
-      steps.push_back(rewriter.create<arith::ConstantIndexOp>(loc, step));
+      steps.push_back(arith::ConstantIndexOp::create(rewriter, loc, step));
 
     // Get the terminator op.
     auto affineParOpTerminator =
@@ -205,7 +205,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     scf::ParallelOp parOp;
     if (op.getResults().empty()) {
       // Case with no reduction operations/return values.
-      parOp = rewriter.create<scf::ParallelOp>(loc, lowerBoundTuple,
+      parOp = scf::ParallelOp::create(rewriter, loc, lowerBoundTuple,
                                                upperBoundTuple, steps,
                                                /*bodyBuilderFn=*/nullptr);
       rewriter.eraseBlock(parOp.getBody());
@@ -233,7 +233,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       identityVals.push_back(
           arith::getIdentityValue(reductionOpValue, resultType, rewriter, loc));
     }
-    parOp = rewriter.create<scf::ParallelOp>(
+    parOp = scf::ParallelOp::create(rewriter,
         loc, lowerBoundTuple, upperBoundTuple, steps, identityVals,
         /*bodyBuilderFn=*/nullptr);
 
@@ -261,7 +261,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       Value reductionResult = arith::getReductionOp(
           reductionOpValue, rewriter, loc, reductionBody.getArgument(0),
           reductionBody.getArgument(1));
-      rewriter.create<scf::ReduceReturnOp>(loc, reductionResult);
+      scf::ReduceReturnOp::create(rewriter, loc, reductionResult);
     }
     rewriter.replaceOp(op, parOp.getResults());
     return success();
@@ -278,7 +278,7 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
 
     // Now we just have to handle the condition logic.
     auto integerSet = op.getIntegerSet();
-    Value zeroConstant = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zeroConstant = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value, 8> operands(op.getOperands());
     auto operandsRef = llvm::ArrayRef(operands);
 
@@ -298,17 +298,17 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
       auto pred =
           isEquality ? arith::CmpIPredicate::eq : arith::CmpIPredicate::sge;
       Value cmpVal =
-          rewriter.create<arith::CmpIOp>(loc, pred, affResult, zeroConstant);
+          arith::CmpIOp::create(rewriter, loc, pred, affResult, zeroConstant);
       cond = cond
-                 ? rewriter.create<arith::AndIOp>(loc, cond, cmpVal).getResult()
+                 ? arith::AndIOp::create(rewriter, loc, cond, cmpVal).getResult()
                  : cmpVal;
     }
     cond = cond ? cond
-                : rewriter.create<arith::ConstantIntOp>(loc, /*value=*/1,
+                : arith::ConstantIntOp::create(rewriter, loc, /*value=*/1,
                                                         /*width=*/1);
 
     bool hasElseRegion = !op.getElseRegion().empty();
-    auto ifOp = rewriter.create<scf::IfOp>(loc, op.getResultTypes(), cond,
+    auto ifOp = scf::IfOp::create(rewriter, loc, op.getResultTypes(), cond,
                                            hasElseRegion);
     rewriter.inlineRegionBefore(op.getThenRegion(),
                                 &ifOp.getThenRegion().back());
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index cf9bb3a000050..df3f2c4926614 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -115,9 +115,9 @@ static Value castF32To(Type desType, Value f32, Location loc,
   if (elementType.isF32())
     return f32;
   if (elementType.getIntOrFloatBitWidth() < 32)
-    return rewriter.create<arith::TruncFOp>(loc, desType, f32);
+    return arith::TruncFOp::create(rewriter, loc, desType, f32);
   if (elementType.getIntOrFloatBitWidth() > 32)
-    return rewriter.create<arith::ExtFOp>(loc, desType, f32);
+    return arith::ExtFOp::create(rewriter, loc, desType, f32);
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
@@ -139,7 +139,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
   VectorType extResType = VectorType::get(2, rewriter.getF32Type());
   if (!inVecType) {
-    Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
+    Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
         loc, rewriter.getF32Type(), in, 0);
     Value result = castF32To(outElemType, asFloat, loc, rewriter);
     rewriter.replaceOp(op, result);
@@ -147,7 +147,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   }
   int64_t numElements = inVecType.getNumElements();
 
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   VectorType outType = cast<VectorType>(op.getOut().getType());
 
@@ -155,10 +155,10 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
     Value zerodSplat =
         rewriter.createOrFold<vector::SplatOp>(loc, outType, zero);
     Value scalarIn =
-        rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
+        vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     Value scalarExt =
-        rewriter.create<arith::ExtFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarExt, zerodSplat,
+        arith::ExtFOp::create(rewriter, loc, outElemType, scalarIn);
+    Value result = vector::InsertOp::create(rewriter, loc, scalarExt, zerodSplat,
                                                      ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
@@ -171,32 +171,32 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   if (inVecType.getRank() > 1) {
     inVecType = VectorType::get(SmallVector<int64_t>{numElements},
                                 inVecType.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVecType, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVecType, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
-    Value inSlice = rewriter.create<vector::ExtractStridedSliceOp>(
+    Value inSlice = vector::ExtractStridedSliceOp::create(rewriter,
         loc, in, i, elemsThisOp, 1);
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
       if (i + j + 1 < numElements) { // Convert two 8-bit elements
-        Value asFloats = rewriter.create<amdgpu::ExtPackedFp8Op>(
+        Value asFloats = amdgpu::ExtPackedFp8Op::create(rewriter,
             loc, extResType, inSlice, j / 2);
         Type desType = VectorType::get(2, outElemType);
         Value asType = castF32To(desType, asFloats, loc, rewriter);
-        result = rewriter.create<vector::InsertStridedSliceOp>(
+        result = vector::InsertStridedSliceOp::create(rewriter,
             loc, asType, result, i + j, 1);
       } else { // Convert a 8-bit element
-        Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
+        Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
             loc, rewriter.getF32Type(), inSlice, j / 2 * 2);
         Value asType = castF32To(outElemType, asFloat, loc, rewriter);
-        result = rewriter.create<vector::InsertOp>(loc, asType, result, i + j);
+        result = vector::InsertOp::create(rewriter, loc, asType, result, i + j);
       }
     }
   }
 
   if (inVecType.getRank() != outType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outType, result);
   }
 
   rewriter.replaceOp(op, result);
@@ -208,9 +208,9 @@ static Value castToF32(Value value, Location loc, PatternRewriter &rewriter) {
   if (type.isF32())
     return value;
   if (type.getIntOrFloatBitWidth() < 32)
-    return rewriter.create<arith::ExtFOp>(loc, rewriter.getF32Type(), value);
+    return arith::ExtFOp::create(rewriter, loc, rewriter.getF32Type(), value);
   if (type.getIntOrFloatBitWidth() > 32)
-    return rewriter.create<arith::TruncFOp>(loc, rewriter.getF32Type(), value);
+    return arith::TruncFOp::create(rewriter, loc, rewriter.getF32Type(), value);
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
@@ -250,13 +250,13 @@ static Value clampInput(PatternRewriter &rewriter, Location loc,
       loc, arith::CmpFPredicate::OEQ, source, negInf);
   Value isNan = rewriter.createOrFold<arith::CmpFOp>(
       loc, arith::CmpFPredicate::UNO, source, source);
-  Value isNonFinite = rewriter.create<arith::OrIOp>(
-      loc, rewriter.create<arith::OrIOp>(loc, isInf, isNegInf), isNan);
+  Value isNonFinite = arith::OrIOp::create(rewriter,
+      loc, arith::OrIOp::create(rewriter, loc, isInf, isNegInf), isNan);
 
-  Value clampedBelow = rewriter.create<arith::MaximumFOp>(loc, source, minCst);
-  Value clamped = rewriter.create<arith::MinimumFOp>(loc, clampedBelow, maxCst);
+  Value clampedBelow = arith::MaximumFOp::create(rewriter, loc, source, minCst);
+  Value clamped = arith::MinimumFOp::create(rewriter, loc, clampedBelow, maxCst);
   Value res =
-      rewriter.create<arith::SelectOp>(loc, isNonFinite, source, clamped);
+      arith::SelectOp::create(rewriter, loc, isNonFinite, source, clamped);
   return res;
 }
 
@@ -290,24 +290,24 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   VectorType truncResType = VectorType::get(4, outElemType);
   if (!inVectorTy) {
     Value asFloat = castToF32(in, loc, rewriter);
-    Value asF8s = rewriter.create<amdgpu::PackedTrunc2xFp8Op>(
+    Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
         loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
         /*existing=*/nullptr);
-    Value result = rewriter.create<vector::ExtractOp>(loc, asF8s, 0);
+    Value result = vector::ExtractOp::create(rewriter, loc, asF8s, 0);
     rewriter.replaceOp(op, result);
     return success();
   }
 
   int64_t numElements = outVecType.getNumElements();
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   if (outVecType.getShape().empty()) {
     Value scalarIn =
-        rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
+        vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     // Recurse to send the 0-D vector case to the 1-D vector case
     Value scalarTrunc =
-        rewriter.create<arith::TruncFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarTrunc, zero,
+        arith::TruncFOp::create(rewriter, loc, outElemType, scalarIn);
+    Value result = vector::InsertOp::create(rewriter, loc, scalarTrunc, zero,
                                                      ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
@@ -320,32 +320,32 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
                                  inVectorTy.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVectorTy, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVectorTy, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
     Value thisResult = nullptr;
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
-      Value elemA = rewriter.create<vector::ExtractOp>(loc, in, i + j);
+      Value elemA = vector::ExtractOp::create(rewriter, loc, in, i + j);
       Value asFloatA = castToF32(elemA, loc, rewriter);
       Value asFloatB = nullptr;
       if (j + 1 < elemsThisOp) {
-        Value elemB = rewriter.create<vector::ExtractOp>(loc, in, i + j + 1);
+        Value elemB = vector::ExtractOp::create(rewriter, loc, in, i + j + 1);
         asFloatB = castToF32(elemB, loc, rewriter);
       }
-      thisResult = rewriter.create<amdgpu::PackedTrunc2xFp8Op>(
+      thisResult = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
           loc, truncResType, asFloatA, asFloatB, j / 2, thisResult);
     }
     if (elemsThisOp < 4)
-      thisResult = rewriter.create<vector::ExtractStridedSliceOp>(
+      thisResult = vector::ExtractStridedSliceOp::create(rewriter,
           loc, thisResult, 0, elemsThisOp, 1);
-    result = rewriter.create<vector::InsertStridedSliceOp>(loc, thisResult,
+    result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
                                                            result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
@@ -373,10 +373,10 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
 
   // Handle the case where input type is not a vector type
   if (!inVectorTy) {
-    auto sourceB = rewriter.create<LLVM::PoisonOp>(loc, rewriter.getF32Type());
+    auto sourceB = LLVM::PoisonOp::create(rewriter, loc, rewriter.getF32Type());
     Value asF16s =
-        rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, in, sourceB);
-    Value result = rewriter.create<vector::ExtractOp>(loc, asF16s, 0);
+        ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, in, sourceB);
+    Value result = vector::ExtractOp::create(rewriter, loc, asF16s, 0);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -388,7 +388,7 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
                                  inVectorTy.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVectorTy, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVectorTy, in);
   }
 
   // Handle the vector case. We also handle the (uncommon) case where the vector
@@ -396,25 +396,25 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   for (int64_t i = 0; i < numElements; i += 2) {
     int64_t elemsThisOp = std::min(numElements, i + 2) - i;
     Value thisResult = nullptr;
-    Value elemA = rewriter.create<vector::ExtractOp>(loc, in, i);
-    Value elemB = rewriter.create<LLVM::PoisonOp>(loc, rewriter.getF32Type());
+    Value elemA = vector::ExtractOp::create(rewriter, loc, in, i);
+    Value elemB = LLVM::PoisonOp::create(rewriter, loc, rewriter.getF32Type());
 
     if (elemsThisOp == 2) {
-      elemB = rewriter.create<vector::ExtractOp>(loc, in, i + 1);
+      elemB = vector::ExtractOp::create(rewriter, loc, in, i + 1);
     }
 
     thisResult =
-        rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, elemA, elemB);
+        ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, elemA, elemB);
     // Place back the truncated result into the possibly larger vector. If we
     // are operating on a size 2 vector, these operations should be folded away
-    thisResult = rewriter.create<vector::ExtractStridedSliceOp>(
+    thisResult = vector::ExtractStridedSliceOp::create(rewriter,
         loc, thisResult, 0, elemsThisOp, 1);
-    result = rewriter.create<vector::InsertStridedSliceOp>(loc, thisResult,
+    result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
                                                            result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
index cbe0b3fda3410..40bf8f1cffb04 100644
--- a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
+++ b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
@@ -74,14 +74,14 @@ struct ConstantOpToArmSMELowering : public OpRewritePattern<arith::ConstantOp> {
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
     auto denseAttr1D = DenseElementsAttr::get(
         tileSliceType, denseAttr.getSplatValue<Attribute>());
-    auto constantOp1D = rewriter.create<arith::ConstantOp>(loc, denseAttr1D);
+    auto constantOp1D = arith::ConstantOp::create(rewriter, loc, denseAttr1D);
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to write vector to tile
       // slice.
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, constantOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
index a5c08a6378021..c5b3fdc536f72 100644
--- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
+++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
@@ -110,7 +110,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     emitc::CmpPredicate predicate;
     switch (op.getPredicate()) {
     case arith::CmpFPredicate::AlwaysFalse: {
-      auto constant = rewriter.create<emitc::ConstantOp>(
+      auto constant = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getI1Type(),
           rewriter.getBoolAttr(/*value=*/false));
       rewriter.replaceOp(op, constant);
@@ -179,7 +179,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
       return success();
     }
     case arith::CmpFPredicate::AlwaysTrue: {
-      auto constant = rewriter.create<emitc::ConstantOp>(
+      auto constant = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getI1Type(),
           rewriter.getBoolAttr(/*value=*/true));
       rewriter.replaceOp(op, constant);
@@ -189,7 +189,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
 
     // Compare the values naively
     auto cmpResult =
-        rewriter.create<emitc::CmpOp>(op.getLoc(), op.getType(), predicate,
+        emitc::CmpOp::create(rewriter, op.getLoc(), op.getType(), predicate,
                                       adaptor.getLhs(), adaptor.getRhs());
 
     // Adjust the results for unordered/ordered semantics
@@ -213,7 +213,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNaN(ConversionPatternRewriter &rewriter, Location loc,
               Value operand) const {
     // A value is NaN exactly when it compares unequal to itself.
-    return rewriter.create<emitc::CmpOp>(
+    return emitc::CmpOp::create(rewriter,
         loc, rewriter.getI1Type(), emitc::CmpPredicate::ne, operand, operand);
   }
 
@@ -221,7 +221,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNotNaN(ConversionPatternRewriter &rewriter, Location loc,
                  Value operand) const {
     // A value is not NaN exactly when it compares equal to itself.
-    return rewriter.create<emitc::CmpOp>(
+    return emitc::CmpOp::create(rewriter,
         loc, rewriter.getI1Type(), emitc::CmpPredicate::eq, operand, operand);
   }
 
@@ -231,7 +231,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
                                Location loc, Value first, Value second) const {
     auto firstIsNaN = isNaN(rewriter, loc, first);
     auto secondIsNaN = isNaN(rewriter, loc, second);
-    return rewriter.create<emitc::LogicalOrOp>(loc, rewriter.getI1Type(),
+    return emitc::LogicalOrOp::create(rewriter, loc, rewriter.getI1Type(),
                                                firstIsNaN, secondIsNaN);
   }
 
@@ -241,7 +241,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
                              Value first, Value second) const {
     auto firstIsNotNaN = isNotNaN(rewriter, loc, first);
     auto secondIsNotNaN = isNotNaN(rewriter, loc, second);
-    return rewriter.create<emitc::LogicalAndOp>(loc, rewriter.getI1Type(),
+    return emitc::LogicalAndOp::create(rewriter, loc, rewriter.getI1Type(),
                                                 firstIsNotNaN, secondIsNotNaN);
   }
 };
@@ -378,9 +378,9 @@ class CastConversion : public OpConversionPattern<ArithOp> {
       Type attrType = (emitc::isPointerWideType(operandType))
                           ? rewriter.getIndexType()
                           : operandType;
-      auto constOne = rewriter.create<emitc::ConstantOp>(
+      auto constOne = emitc::ConstantOp::create(rewriter,
           op.getLoc(), operandType, rewriter.getOneAttr(attrType));
-      auto oneAndOperand = rewriter.create<emitc::BitwiseAndOp>(
+      auto oneAndOperand = emitc::BitwiseAndOp::create(rewriter,
           op.getLoc(), operandType, adaptor.getIn(), constOne);
       rewriter.replaceOpWithNewOp<emitc::CastOp>(op, opReturnType,
                                                  oneAndOperand);
@@ -467,7 +467,7 @@ class BinaryUIOpConversion final : public OpConversionPattern<ArithOp> {
     Value rhsAdapted = adaptValueType(uiBinOp.getRhs(), rewriter, unsignedType);
 
     auto newDivOp =
-        rewriter.create<EmitCOp>(uiBinOp.getLoc(), unsignedType,
+        EmitCOp::create(rewriter, uiBinOp.getLoc(), unsignedType,
                                  ArrayRef<Value>{lhsAdapted, rhsAdapted});
     Value resultAdapted = adaptValueType(newDivOp, rewriter, newRetTy);
     rewriter.replaceOp(uiBinOp, resultAdapted);
@@ -588,38 +588,38 @@ class ShiftOpConversion : public OpConversionPattern<ArithOp> {
     // Add a runtime check for overflow
     Value width;
     if (emitc::isPointerWideType(type)) {
-      Value eight = rewriter.create<emitc::ConstantOp>(
+      Value eight = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rhsType, rewriter.getIndexAttr(8));
-      emitc::CallOpaqueOp sizeOfCall = rewriter.create<emitc::CallOpaqueOp>(
+      emitc::CallOpaqueOp sizeOfCall = emitc::CallOpaqueOp::create(rewriter,
           op.getLoc(), rhsType, "sizeof", ArrayRef<Value>{eight});
-      width = rewriter.create<emitc::MulOp>(op.getLoc(), rhsType, eight,
+      width = emitc::MulOp::create(rewriter, op.getLoc(), rhsType, eight,
                                             sizeOfCall.getResult(0));
     } else {
-      width = rewriter.create<emitc::ConstantOp>(
+      width = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rhsType,
           rewriter.getIntegerAttr(rhsType, type.getIntOrFloatBitWidth()));
     }
 
-    Value excessCheck = rewriter.create<emitc::CmpOp>(
+    Value excessCheck = emitc::CmpOp::create(rewriter,
         op.getLoc(), rewriter.getI1Type(), emitc::CmpPredicate::lt, rhs, width);
 
     // Any concrete value is a valid refinement of poison.
-    Value poison = rewriter.create<emitc::ConstantOp>(
+    Value poison = emitc::ConstantOp::create(rewriter,
         op.getLoc(), arithmeticType,
         (isa<IntegerType>(arithmeticType)
              ? rewriter.getIntegerAttr(arithmeticType, 0)
              : rewriter.getIndexAttr(0)));
 
-    emitc::ExpressionOp ternary = rewriter.create<emitc::ExpressionOp>(
+    emitc::ExpressionOp ternary = emitc::ExpressionOp::create(rewriter,
         op.getLoc(), arithmeticType, /*do_not_inline=*/false);
     Block &bodyBlock = ternary.getBodyRegion().emplaceBlock();
     auto currentPoint = rewriter.getInsertionPoint();
     rewriter.setInsertionPointToStart(&bodyBlock);
     Value arithmeticResult =
-        rewriter.create<EmitCOp>(op.getLoc(), arithmeticType, lhs, rhs);
-    Value resultOrPoison = rewriter.create<emitc::ConditionalOp>(
+        EmitCOp::create(rewriter, op.getLoc(), arithmeticType, lhs, rhs);
+    Value resultOrPoison = emitc::ConditionalOp::create(rewriter,
         op.getLoc(), arithmeticType, excessCheck, arithmeticResult, poison);
-    rewriter.create<emitc::YieldOp>(op.getLoc(), resultOrPoison);
+    emitc::YieldOp::create(rewriter, op.getLoc(), resultOrPoison);
     rewriter.setInsertionPoint(op->getBlock(), currentPoint);
 
     Value result = adaptValueType(ternary, rewriter, type);
@@ -700,11 +700,11 @@ class FtoICastOpConversion : public OpConversionPattern<CastOp> {
                                   /*isSigned=*/false);
     }
 
-    Value result = rewriter.create<emitc::CastOp>(
+    Value result = emitc::CastOp::create(rewriter,
         castOp.getLoc(), actualResultType, adaptor.getOperands());
 
     if (isa<arith::FPToUIOp>(castOp)) {
-      result = rewriter.create<emitc::CastOp>(castOp.getLoc(), dstType, result);
+      result = emitc::CastOp::create(rewriter, castOp.getLoc(), dstType, result);
     }
     rewriter.replaceOp(castOp, result);
 
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index f7bf581adc9e3..f7d4963b5e293 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -293,10 +293,10 @@ LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
       [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
         typename OpTy::Adaptor adaptor(operands);
         if (targetBits < sourceBits) {
-          return rewriter.create<LLVM::TruncOp>(op.getLoc(), llvm1DVectorTy,
+          return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
                                                 adaptor.getIn());
         }
-        return rewriter.create<ExtCastTy>(op.getLoc(), llvm1DVectorTy,
+        return ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
                                           adaptor.getIn());
       },
       rewriter);
@@ -324,12 +324,12 @@ LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
     Type newOverflowType = typeConverter->convertType(overflowResultType);
     Type structType =
         LLVM::LLVMStructType::getLiteral(ctx, {sumResultType, newOverflowType});
-    Value addOverflow = rewriter.create<LLVM::UAddWithOverflowOp>(
+    Value addOverflow = LLVM::UAddWithOverflowOp::create(rewriter,
         loc, structType, adaptor.getLhs(), adaptor.getRhs());
     Value sumExtracted =
-        rewriter.create<LLVM::ExtractValueOp>(loc, addOverflow, 0);
+        LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 0);
     Value overflowExtracted =
-        rewriter.create<LLVM::ExtractValueOp>(loc, addOverflow, 1);
+        LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 1);
     rewriter.replaceOp(op, {sumExtracted, overflowExtracted});
     return success();
   }
@@ -381,15 +381,15 @@ LogicalResult MulIExtendedOpLowering<ArithMulOp, IsSigned>::matchAndRewrite(
            "LLVM dialect should support all signless integer types");
 
     using LLVMExtOp = std::conditional_t<IsSigned, LLVM::SExtOp, LLVM::ZExtOp>;
-    Value lhsExt = rewriter.create<LLVMExtOp>(loc, wideType, adaptor.getLhs());
-    Value rhsExt = rewriter.create<LLVMExtOp>(loc, wideType, adaptor.getRhs());
-    Value mulExt = rewriter.create<LLVM::MulOp>(loc, wideType, lhsExt, rhsExt);
+    Value lhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getLhs());
+    Value rhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getRhs());
+    Value mulExt = LLVM::MulOp::create(rewriter, loc, wideType, lhsExt, rhsExt);
 
     // Split the 2*N-bit wide result into two N-bit values.
-    Value low = rewriter.create<LLVM::TruncOp>(loc, resultType, mulExt);
-    Value shiftVal = rewriter.create<LLVM::ConstantOp>(loc, shiftValAttr);
-    Value highExt = rewriter.create<LLVM::LShrOp>(loc, mulExt, shiftVal);
-    Value high = rewriter.create<LLVM::TruncOp>(loc, resultType, highExt);
+    Value low = LLVM::TruncOp::create(rewriter, loc, resultType, mulExt);
+    Value shiftVal = LLVM::ConstantOp::create(rewriter, loc, shiftValAttr);
+    Value highExt = LLVM::LShrOp::create(rewriter, loc, mulExt, shiftVal);
+    Value high = LLVM::TruncOp::create(rewriter, loc, resultType, highExt);
 
     rewriter.replaceOp(op, {low, high});
     return success();
@@ -435,7 +435,7 @@ CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return rewriter.create<LLVM::ICmpOp>(
+        return LLVM::ICmpOp::create(rewriter,
             op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs());
@@ -471,7 +471,7 @@ CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return rewriter.create<LLVM::FCmpOp>(
+        return LLVM::FCmpOp::create(rewriter,
             op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs(), fmf);
diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index 434d7df853a5e..c93cf13f831ef 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -117,11 +117,11 @@ static Value getScalarOrVectorConstInt(Type type, uint64_t value,
   if (auto vectorType = dyn_cast<VectorType>(type)) {
     Attribute element = IntegerAttr::get(vectorType.getElementType(), value);
     auto attr = SplatElementsAttr::get(vectorType, element);
-    return builder.create<spirv::ConstantOp>(loc, vectorType, attr);
+    return spirv::ConstantOp::create(builder, loc, vectorType, attr);
   }
 
   if (auto intType = dyn_cast<IntegerType>(type))
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, value));
 
   return nullptr;
@@ -418,18 +418,18 @@ static Value emulateSignedRemainder(Location loc, Value lhs, Value rhs,
   Type type = lhs.getType();
 
   // Calculate the remainder with spirv.UMod.
-  Value lhsAbs = builder.create<SignedAbsOp>(loc, type, lhs);
-  Value rhsAbs = builder.create<SignedAbsOp>(loc, type, rhs);
-  Value abs = builder.create<spirv::UModOp>(loc, lhsAbs, rhsAbs);
+  Value lhsAbs = SignedAbsOp::create(builder, loc, type, lhs);
+  Value rhsAbs = SignedAbsOp::create(builder, loc, type, rhs);
+  Value abs = spirv::UModOp::create(builder, loc, lhsAbs, rhsAbs);
 
   // Fix the sign.
   Value isPositive;
   if (lhs == signOperand)
-    isPositive = builder.create<spirv::IEqualOp>(loc, lhs, lhsAbs);
+    isPositive = spirv::IEqualOp::create(builder, loc, lhs, lhsAbs);
   else
-    isPositive = builder.create<spirv::IEqualOp>(loc, rhs, rhsAbs);
-  Value absNegate = builder.create<spirv::SNegateOp>(loc, type, abs);
-  return builder.create<spirv::SelectOp>(loc, type, isPositive, abs, absNegate);
+    isPositive = spirv::IEqualOp::create(builder, loc, rhs, rhsAbs);
+  Value absNegate = spirv::SNegateOp::create(builder, loc, type, abs);
+  return spirv::SelectOp::create(builder, loc, type, isPositive, abs, absNegate);
 }
 
 /// Converts arith.remsi to GLSL SPIR-V ops.
@@ -601,12 +601,12 @@ struct ExtSII1Pattern final : public OpConversionPattern<arith::ExtSIOp> {
     Value allOnes;
     if (auto intTy = dyn_cast<IntegerType>(dstType)) {
       unsigned componentBitwidth = intTy.getWidth();
-      allOnes = rewriter.create<spirv::ConstantOp>(
+      allOnes = spirv::ConstantOp::create(rewriter,
           loc, intTy,
           rewriter.getIntegerAttr(intTy, APInt::getAllOnes(componentBitwidth)));
     } else if (auto vectorTy = dyn_cast<VectorType>(dstType)) {
       unsigned componentBitwidth = vectorTy.getElementTypeBitWidth();
-      allOnes = rewriter.create<spirv::ConstantOp>(
+      allOnes = spirv::ConstantOp::create(rewriter,
           loc, vectorTy,
           SplatElementsAttr::get(vectorTy,
                                  APInt::getAllOnes(componentBitwidth)));
@@ -653,7 +653,7 @@ struct ExtSIPattern final : public OpConversionPattern<arith::ExtSIOp> {
       // First shift left to sequeeze out all leading bits beyond the original
       // bitwidth. Here we need to use the original source and result type's
       // bitwidth.
-      auto shiftLOp = rewriter.create<spirv::ShiftLeftLogicalOp>(
+      auto shiftLOp = spirv::ShiftLeftLogicalOp::create(rewriter,
           op.getLoc(), dstType, adaptor.getIn(), shiftSize);
 
       // Then we perform arithmetic right shift to make sure we have the right
@@ -757,9 +757,9 @@ struct TruncII1Pattern final : public OpConversionPattern<arith::TruncIOp> {
     auto srcType = adaptor.getOperands().front().getType();
     // Check if (x & 1) == 1.
     Value mask = spirv::ConstantOp::getOne(srcType, loc, rewriter);
-    Value maskedSrc = rewriter.create<spirv::BitwiseAndOp>(
+    Value maskedSrc = spirv::BitwiseAndOp::create(rewriter,
         loc, srcType, adaptor.getOperands()[0], mask);
-    Value isOne = rewriter.create<spirv::IEqualOp>(loc, maskedSrc, mask);
+    Value isOne = spirv::IEqualOp::create(rewriter, loc, maskedSrc, mask);
 
     Value zero = spirv::ConstantOp::getZero(dstType, loc, rewriter);
     Value one = spirv::ConstantOp::getOne(dstType, loc, rewriter);
@@ -914,9 +914,9 @@ class CmpIOpBooleanPattern final : public OpConversionPattern<arith::CmpIOp> {
       if (auto vectorType = dyn_cast<VectorType>(dstType))
         type = VectorType::get(vectorType.getShape(), type);
       Value extLhs =
-          rewriter.create<arith::ExtUIOp>(op.getLoc(), type, adaptor.getLhs());
+          arith::ExtUIOp::create(rewriter, op.getLoc(), type, adaptor.getLhs());
       Value extRhs =
-          rewriter.create<arith::ExtUIOp>(op.getLoc(), type, adaptor.getRhs());
+          arith::ExtUIOp::create(rewriter, op.getLoc(), type, adaptor.getRhs());
 
       rewriter.replaceOpWithNewOp<arith::CmpIOp>(op, op.getPredicate(), extLhs,
                                                  extRhs);
@@ -1067,12 +1067,12 @@ class CmpFOpNanNonePattern final : public OpConversionPattern<arith::CmpFOp> {
         replace = spirv::ConstantOp::getZero(op.getType(), loc, rewriter);
       }
     } else {
-      Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-      Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+      Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+      Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-      replace = rewriter.create<spirv::LogicalOrOp>(loc, lhsIsNan, rhsIsNan);
+      replace = spirv::LogicalOrOp::create(rewriter, loc, lhsIsNan, rhsIsNan);
       if (op.getPredicate() == arith::CmpFPredicate::ORD)
-        replace = rewriter.create<spirv::LogicalNotOp>(loc, replace);
+        replace = spirv::LogicalNotOp::create(rewriter, loc, replace);
     }
 
     rewriter.replaceOp(op, replace);
@@ -1094,17 +1094,17 @@ class AddUIExtendedOpPattern final
                   ConversionPatternRewriter &rewriter) const override {
     Type dstElemTy = adaptor.getLhs().getType();
     Location loc = op->getLoc();
-    Value result = rewriter.create<spirv::IAddCarryOp>(loc, adaptor.getLhs(),
+    Value result = spirv::IAddCarryOp::create(rewriter, loc, adaptor.getLhs(),
                                                        adaptor.getRhs());
 
-    Value sumResult = rewriter.create<spirv::CompositeExtractOp>(
+    Value sumResult = spirv::CompositeExtractOp::create(rewriter,
         loc, result, llvm::ArrayRef(0));
-    Value carryValue = rewriter.create<spirv::CompositeExtractOp>(
+    Value carryValue = spirv::CompositeExtractOp::create(rewriter,
         loc, result, llvm::ArrayRef(1));
 
     // Convert the carry value to boolean.
     Value one = spirv::ConstantOp::getOne(dstElemTy, loc, rewriter);
-    Value carryResult = rewriter.create<spirv::IEqualOp>(loc, carryValue, one);
+    Value carryResult = spirv::IEqualOp::create(rewriter, loc, carryValue, one);
 
     rewriter.replaceOp(op, {sumResult, carryResult});
     return success();
@@ -1125,11 +1125,11 @@ class MulIExtendedOpPattern final : public OpConversionPattern<ArithMulOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op->getLoc();
     Value result =
-        rewriter.create<SPIRVMulOp>(loc, adaptor.getLhs(), adaptor.getRhs());
+        SPIRVMulOp::create(rewriter, loc, adaptor.getLhs(), adaptor.getRhs());
 
-    Value low = rewriter.create<spirv::CompositeExtractOp>(loc, result,
+    Value low = spirv::CompositeExtractOp::create(rewriter, loc, result,
                                                            llvm::ArrayRef(0));
-    Value high = rewriter.create<spirv::CompositeExtractOp>(loc, result,
+    Value high = spirv::CompositeExtractOp::create(rewriter, loc, result,
                                                             llvm::ArrayRef(1));
 
     rewriter.replaceOp(op, {low, high});
@@ -1183,19 +1183,19 @@ class MinimumMaximumFOpPattern final : public OpConversionPattern<Op> {
 
     Location loc = op.getLoc();
     Value spirvOp =
-        rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
+        SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
 
     if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
       rewriter.replaceOp(op, spirvOp);
       return success();
     }
 
-    Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-    Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+    Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+    Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-    Value select1 = rewriter.create<spirv::SelectOp>(loc, dstType, lhsIsNan,
+    Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
                                                      adaptor.getLhs(), spirvOp);
-    Value select2 = rewriter.create<spirv::SelectOp>(loc, dstType, rhsIsNan,
+    Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
                                                      adaptor.getRhs(), select1);
 
     rewriter.replaceOp(op, select2);
@@ -1237,7 +1237,7 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
 
     Location loc = op.getLoc();
     Value spirvOp =
-        rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
+        SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
 
     if (!shouldInsertNanGuards<SPIRVOp>() ||
         bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
@@ -1245,12 +1245,12 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
       return success();
     }
 
-    Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-    Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+    Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+    Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-    Value select1 = rewriter.create<spirv::SelectOp>(loc, dstType, lhsIsNan,
+    Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
                                                      adaptor.getRhs(), spirvOp);
-    Value select2 = rewriter.create<spirv::SelectOp>(loc, dstType, rhsIsNan,
+    Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
                                                      adaptor.getLhs(), select1);
 
     rewriter.replaceOp(op, select2);
diff --git a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
index 9c6de938a7108..2295d661d44a5 100644
--- a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
+++ b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
@@ -41,10 +41,10 @@ class Sdot2dLoweringPattern : public OpRewritePattern<Sdot2dOp> {
     Value c2d = op.getC();
     Location loc = op.getLoc();
     Value b1d =
-        rewriter.create<vector::ShapeCastOp>(loc, flattenedVectorType, b2d);
+        vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, b2d);
     Value c1d =
-        rewriter.create<vector::ShapeCastOp>(loc, flattenedVectorType, c2d);
-    Value newOp = rewriter.create<SdotOp>(loc, op.getRes().getType(), op.getA(),
+        vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, c2d);
+    Value newOp = SdotOp::create(rewriter, loc, op.getRes().getType(), op.getA(),
                                           b1d, c1d);
     rewriter.replaceOp(op, {newOp});
     return success();
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 21ea444e31821..cf46e0c10df09 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -45,37 +45,37 @@ static Operation *createLoadTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_ld1b_horiz>(
+      return arm_sme::aarch64_sme_ld1b_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_ld1h_horiz>(
+      return arm_sme::aarch64_sme_ld1h_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_ld1w_horiz>(
+      return arm_sme::aarch64_sme_ld1w_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_ld1d_horiz>(
+      return arm_sme::aarch64_sme_ld1d_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_ld1q_horiz>(
+      return arm_sme::aarch64_sme_ld1q_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_ld1b_vert>(
+      return arm_sme::aarch64_sme_ld1b_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_ld1h_vert>(
+      return arm_sme::aarch64_sme_ld1h_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_ld1w_vert>(
+      return arm_sme::aarch64_sme_ld1w_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_ld1d_vert>(
+      return arm_sme::aarch64_sme_ld1d_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_ld1q_vert>(
+      return arm_sme::aarch64_sme_ld1q_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
       break;
     }
@@ -91,37 +91,37 @@ static Operation *createStoreTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_st1b_horiz>(
+      return arm_sme::aarch64_sme_st1b_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_st1h_horiz>(
+      return arm_sme::aarch64_sme_st1h_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_st1w_horiz>(
+      return arm_sme::aarch64_sme_st1w_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_st1d_horiz>(
+      return arm_sme::aarch64_sme_st1d_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_st1q_horiz>(
+      return arm_sme::aarch64_sme_st1q_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_st1b_vert>(
+      return arm_sme::aarch64_sme_st1b_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_st1h_vert>(
+      return arm_sme::aarch64_sme_st1h_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_st1w_vert>(
+      return arm_sme::aarch64_sme_st1w_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_st1d_vert>(
+      return arm_sme::aarch64_sme_st1d_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_st1q_vert>(
+      return arm_sme::aarch64_sme_st1q_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   }
@@ -146,15 +146,15 @@ createAllocaForTile(RewriterBase &rewriter, Location loc,
   // Move to the first operation in the function.
   rewriter.setInsertionPointToStart(&func.getBlocks().front());
   // Create an alloca matching the tile size of the `tileOp`.
-  auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
+  auto vscale = vector::VectorScaleOp::create(rewriter, loc);
   auto tileElementType = tileOp.getTileType().getElementType();
   auto memrefType = MemRefType::get(
       {ShapedType::kDynamic, ShapedType::kDynamic}, tileElementType);
   unsigned minElements = arm_sme::getSMETileSliceMinNumElts(tileElementType);
   auto minElementsOp =
-      rewriter.create<arith::ConstantIndexOp>(loc, minElements);
-  auto vectorLen = rewriter.create<arith::MulIOp>(loc, vscale, minElementsOp);
-  auto alloca = rewriter.create<memref::AllocaOp>(
+      arith::ConstantIndexOp::create(rewriter, loc, minElements);
+  auto vectorLen = arith::MulIOp::create(rewriter, loc, vscale, minElementsOp);
+  auto alloca = memref::AllocaOp::create(rewriter,
       loc, memrefType, ValueRange{vectorLen, vectorLen});
   return alloca;
 }
@@ -293,9 +293,9 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                                 Value tileMemory, Value sliceIndex) const {
     auto llvmType = getTypeConverter()->convertType(tileMemory.getType());
     auto descriptor =
-        rewriter.create<UnrealizedConversionCastOp>(loc, llvmType, tileMemory);
-    auto zero = rewriter.create<arith::ConstantIntOp>(loc, 0, /*width=*/64);
-    auto sliceIndexI64 = rewriter.create<arith::IndexCastOp>(
+        UnrealizedConversionCastOp::create(rewriter, loc, llvmType, tileMemory);
+    auto zero = arith::ConstantIntOp::create(rewriter, loc, 0, /*width=*/64);
+    auto sliceIndexI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), sliceIndex);
     return getStridedElementPtr(
         static_cast<ConversionPatternRewriter &>(rewriter), loc,
@@ -309,27 +309,27 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                      arm_sme::ArmSMETileType tileType, VectorType sliceType,
                      IntegerAttr tileId, Value sliceIndex) const {
     // Cast the slice index to an i32.
-    auto sliceIndexI32 = rewriter.create<arith::IndexCastOp>(
+    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), sliceIndex);
     // Create an all-true predicate for the slice.
     auto predicateType = sliceType.clone(rewriter.getI1Type());
-    auto allTruePredicate = rewriter.create<arith::ConstantOp>(
+    auto allTruePredicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
     // Create padding vector (never used due to all-true predicate).
-    auto padVector = rewriter.create<LLVM::PoisonOp>(loc, sliceType);
+    auto padVector = LLVM::PoisonOp::create(rewriter, loc, sliceType);
     // Get a pointer to the current slice.
     auto slicePtr =
         getInMemoryTileSlicePtr(rewriter, loc, tileAlloca, sliceIndex);
     // Read the value of the current slice from ZA.
-    auto currentTileSlice = rewriter.create<arm_sme::aarch64_sme_read_horiz>(
+    auto currentTileSlice = arm_sme::aarch64_sme_read_horiz::create(rewriter,
         loc, sliceType, padVector, allTruePredicate, tileId, sliceIndexI32);
     // Load the new tile slice back from memory into ZA.
     createLoadTileSliceIntrinsic(
         rewriter, loc, tileType, arm_sme::TileSliceLayout::Horizontal,
         allTruePredicate, slicePtr, tileId, sliceIndexI32);
     // Store the current tile slice to memory.
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    rewriter.create<vector::StoreOp>(loc, currentTileSlice, tileAlloca,
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    vector::StoreOp::create(rewriter, loc, currentTileSlice, tileAlloca,
                                      ValueRange{sliceIndex, zero});
   }
 
@@ -341,12 +341,12 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     RewriterBase::InsertionGuard guard(rewriter);
     // Create an scf.for over all tile slices.
     auto minNumElts =
-        rewriter.create<arith::ConstantIndexOp>(loc, sliceType.getDimSize(0));
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto upperBound = rewriter.create<arith::MulIOp>(
-        loc, minNumElts, rewriter.create<vector::VectorScaleOp>(loc));
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        arith::ConstantIndexOp::create(rewriter, loc, sliceType.getDimSize(0));
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto upperBound = arith::MulIOp::create(rewriter,
+        loc, minNumElts, vector::VectorScaleOp::create(rewriter, loc));
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     // Emit a swap for each tile slice.
     rewriter.setInsertionPointToStart(forOp.getBody());
     auto sliceIndex = forOp.getInductionVar();
@@ -479,7 +479,7 @@ struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
     //
     // This holds for all tile sizes.
     int32_t zeroMask = baseMaskForSize << int32_t(tileId.getInt());
-    rewriter.create<arm_sme::aarch64_sme_zero>(
+    arm_sme::aarch64_sme_zero::create(rewriter,
         loc, rewriter.getI32IntegerAttr(zeroMask));
 
     // Create a placeholder op to preserve dataflow.
@@ -513,7 +513,7 @@ struct LoadTileSliceConversion
     auto tileSlice = loadTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
@@ -559,7 +559,7 @@ struct StoreTileSliceConversion
     auto tileSlice = storeTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     auto maskOp = storeTileSliceOp.getMask();
@@ -595,26 +595,26 @@ struct InsertTileSliceConversion
     auto tileSlice = insertTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
-    auto one = rewriter.create<arith::ConstantOp>(
+    auto one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI1Type(),
         rewriter.getIntegerAttr(rewriter.getI1Type(), 1));
     auto predTy = VectorType::get(tileType.getShape()[0], rewriter.getI1Type(),
                                   /*scalableDims=*/{true});
-    auto allActiveMask = rewriter.create<vector::SplatOp>(loc, predTy, one);
+    auto allActiveMask = vector::SplatOp::create(rewriter, loc, predTy, one);
 
     // Create 'arm_sme.intr.write.(horiz|vert)' to write vector to tile slice.
     switch (insertTileSliceOp.getLayout()) {
     case arm_sme::TileSliceLayout::Horizontal:
-      rewriter.create<arm_sme::aarch64_sme_write_horiz>(
+      arm_sme::aarch64_sme_write_horiz::create(rewriter,
           loc, tileId, tileSliceI32, allActiveMask,
           insertTileSliceOp.getVector());
       break;
     case arm_sme::TileSliceLayout::Vertical:
-      rewriter.create<arm_sme::aarch64_sme_write_vert>(
+      arm_sme::aarch64_sme_write_vert::create(rewriter,
           loc, tileId, tileSliceI32, allActiveMask,
           insertTileSliceOp.getVector());
       break;
@@ -646,15 +646,15 @@ struct ExtractTileSliceConversion
 
     // Create an 'all true' predicate for the tile slice.
     auto predicateType = sliceType.cloneWith({}, rewriter.getI1Type());
-    auto allTruePredicate = rewriter.create<arith::ConstantOp>(
+    auto allTruePredicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
 
     // Zero destination/fallback for tile slice extraction.
-    auto zeroVector = rewriter.create<arith::ConstantOp>(
+    auto zeroVector = arith::ConstantOp::create(rewriter,
         loc, sliceType, rewriter.getZeroAttr(sliceType));
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto sliceIndexI32 = rewriter.create<arith::IndexCastOp>(
+    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), sliceIndex);
 
     // Create 'arm_sme.intr.read.(horiz|vert)' to extract the tile slice.
@@ -743,7 +743,7 @@ struct OuterProductOpConversion
     Value acc = outerProductOp.getAcc();
     if (!acc) {
       // Initalize accumulator with zero.
-      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, resultVectorType);
+      auto zero = arm_sme::ZeroOp::create(rewriter, loc, resultVectorType);
       zero.setTileId(tileId);
       acc = zero;
     }
@@ -754,14 +754,14 @@ struct OuterProductOpConversion
     if (!lhsMask || !rhsMask) {
       auto predTy =
           outerProductOp.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = rewriter.create<arith::ConstantOp>(
+      Value allActiveMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
     // Create 'arm_sme.intr.mopa' outer product intrinsic.
-    rewriter.create<arm_sme::aarch64_sme_mopa>(loc, tileId, lhsMask, rhsMask,
+    arm_sme::aarch64_sme_mopa::create(rewriter, loc, tileId, lhsMask, rhsMask,
                                                outerProductOp.getLhs(),
                                                outerProductOp.getRhs());
 
@@ -792,7 +792,7 @@ struct OuterProductWideningOpConversion
     Value acc = op.getAcc();
     if (!acc) {
       // Initalize accumulator with zero.
-      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, op.getResultType());
+      auto zero = arm_sme::ZeroOp::create(rewriter, loc, op.getResultType());
       zero.setTileId(tileId);
       acc = zero;
     }
@@ -801,13 +801,13 @@ struct OuterProductWideningOpConversion
     Value rhsMask = op.getRhsMask();
     if (!lhsMask || !rhsMask) {
       auto predTy = op.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = rewriter.create<arith::ConstantOp>(
+      Value allActiveMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
-    rewriter.create<OuterProductWideningIntrOp>(
+    OuterProductWideningIntrOp::create(rewriter,
         loc, tileId, lhsMask, rhsMask, adaptor.getLhs(), adaptor.getRhs());
 
     // The outerproduct intrinsics have no result, replace
@@ -843,13 +843,13 @@ struct StreamingVLOpConversion
     auto *intrOp = [&]() -> Operation * {
       switch (streamingVlOp.getTypeSize()) {
       case arm_sme::TypeSize::Byte:
-        return rewriter.create<arm_sme::aarch64_sme_cntsb>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsb::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Half:
-        return rewriter.create<arm_sme::aarch64_sme_cntsh>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsh::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Word:
-        return rewriter.create<arm_sme::aarch64_sme_cntsw>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsw::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Double:
-        return rewriter.create<arm_sme::aarch64_sme_cntsd>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsd::create(rewriter, loc, i64Type);
       }
       llvm_unreachable("unknown type size in StreamingVLOpConversion");
     }();
@@ -872,7 +872,7 @@ static void mergeConsecutiveTileZerosInBlock(Block *block) {
     if (zeroOpsToMerge.size() <= 1)
       return;
     IRRewriter rewriter(zeroOpsToMerge.front());
-    rewriter.create<arm_sme::aarch64_sme_zero>(
+    arm_sme::aarch64_sme_zero::create(rewriter,
         zeroOpsToMerge.front().getLoc(),
         rewriter.getI32IntegerAttr(mergedZeroMask));
     for (auto zeroOp : zeroOpsToMerge)
diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 458628c29c6ac..3591ed65f8d84 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -39,7 +39,7 @@ SmallVector<Value, 2> getMemrefIndices(ValueRange indices, unsigned rank,
   auto tileSliceOffset = tileSliceIndex;
 
   auto baseIndexPlusTileSliceOffset =
-      rewriter.create<arith::AddIOp>(loc, indices[0], tileSliceOffset);
+      arith::AddIOp::create(rewriter, loc, indices[0], tileSliceOffset);
   outIndices.push_back(baseIndexPlusTileSliceOffset);
   outIndices.push_back(indices[1]);
 
@@ -59,10 +59,10 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   if (memrefIndices.size() != 2)
     return rewriter.notifyMatchFailure(loc, "invalid number of indices");
 
-  auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
       loc, arm_sme::getSMETileSliceMinNumElts(tileType.getElementType()));
   auto vscale =
-      rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
+      vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
   auto predicateType =
       VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
 
@@ -70,7 +70,7 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   // elements in a vector of SVL bits for a given element type (SVL_B,
   // SVL_H, ..., SVL_Q).
   auto numTileSlices =
-      rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
+      arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
 
   Value predicate;
   Value upperBound;
@@ -82,28 +82,28 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
     // The upper bound of the loop must be clamped at `numTileSlices` as
     // `vector.create_mask` allows operands to be greater than the size of a
     // dimension.
-    auto numRowI64 = rewriter.create<arith::IndexCastOp>(
+    auto numRowI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), maskDim0);
-    auto numTileSlicesI64 = rewriter.create<arith::IndexCastOp>(
+    auto numTileSlicesI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), numTileSlices);
     auto upperBoundI64 =
-        rewriter.create<arith::MinSIOp>(loc, numRowI64, numTileSlicesI64);
-    upperBound = rewriter.create<arith::IndexCastOp>(
+        arith::MinSIOp::create(rewriter, loc, numRowI64, numTileSlicesI64);
+    upperBound = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getIndexType(), upperBoundI64);
 
     predicate =
-        rewriter.create<vector::CreateMaskOp>(loc, predicateType, maskDim1);
+        vector::CreateMaskOp::create(rewriter, loc, predicateType, maskDim1);
   } else {
     upperBound = numTileSlices;
     // No mask. Create an 'all true' predicate for the tile slice.
-    predicate = rewriter.create<arith::ConstantOp>(
+    predicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
   }
 
   bool hasCarriedArgs = bool(initTile);
-  auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-  auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+  auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
                                            hasCarriedArgs ? ValueRange{initTile}
                                                           : ValueRange{});
 
@@ -118,7 +118,7 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
 
   assert(bool(nextTile) == hasCarriedArgs);
   if (nextTile)
-    rewriter.create<scf::YieldOp>(loc, nextTile);
+    scf::YieldOp::create(rewriter, loc, nextTile);
 
   return forOp;
 }
@@ -194,9 +194,9 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
       // Initialize tile with zero to satisfy padding. Inactive cols will be
       // zeroed anyway since the loads use zeroing predication. For inactive
       // rows however, no load will occur so these need to be zeroed.
-      initTile = rewriter.create<arm_sme::ZeroOp>(loc, tileType);
+      initTile = arm_sme::ZeroOp::create(rewriter, loc, tileType);
     } else {
-      initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+      initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
     }
 
     // Create a loop to load the active tile slices from memory.
@@ -207,7 +207,7 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
             Value currentTile) -> Value {
           // Create 'arm_sme.load_tile_slice' to load tile slice from memory
           // into tile.
-          return rewriter.create<arm_sme::LoadTileSliceOp>(
+          return arm_sme::LoadTileSliceOp::create(rewriter,
               loc, tileType, tileLoadOp.getBase(), predicate, currentTile,
               memrefIndices, tileSliceIndex, tileLoadOp.getLayout());
         });
@@ -283,21 +283,21 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto numRows = createMaskOp.getOperands()[0];
     auto numCols = createMaskOp.getOperands()[1];
 
-    auto numColsI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto numColsI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), numCols);
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     // Create a loop that loads each ZA tile slice from memory.
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
         loc, arm_sme::getSMETileSliceMinNumElts(tileElementType));
     auto vscale =
-        rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+        vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto numTileSlices =
-        rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, numTileSlices,
+        arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices,
                                              step, ValueRange{initTile});
 
     rewriter.setInsertionPointToStart(forOp.getBody());
@@ -306,16 +306,16 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto currentTile = forOp.getRegionIterArg(0);
 
     // Combine masks.
-    auto rowIsActive = rewriter.create<arith::CmpIOp>(
+    auto rowIsActive = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, tileSliceIndex, numRows);
-    auto rowIsActiveI32 = rewriter.create<arith::ExtSIOp>(
+    auto rowIsActiveI32 = arith::ExtSIOp::create(rewriter,
         loc, rewriter.getI32Type(), rowIsActive);
-    auto mask = rewriter.create<arith::AndIOp>(loc, rowIsActiveI32, numColsI32);
+    auto mask = arith::AndIOp::create(rewriter, loc, rowIsActiveI32, numColsI32);
     auto maskIndex =
-        rewriter.create<arith::IndexCastOp>(loc, rewriter.getIndexType(), mask);
+        arith::IndexCastOp::create(rewriter, loc, rewriter.getIndexType(), mask);
     auto predicateType =
         VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
-    auto maskOp1D = rewriter.create<vector::CreateMaskOp>(
+    auto maskOp1D = vector::CreateMaskOp::create(rewriter,
         loc, predicateType, maskIndex.getResult());
 
     auto memrefIndices = getMemrefIndices(
@@ -324,17 +324,17 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
 
     // Splat pad into 1-D vector matching type of tile slice.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    auto pad1DOp = rewriter.create<vector::SplatOp>(loc, tileSliceType, padOp);
+    auto pad1DOp = vector::SplatOp::create(rewriter, loc, tileSliceType, padOp);
 
-    auto loadSlice = rewriter.create<vector::MaskedLoadOp>(
+    auto loadSlice = vector::MaskedLoadOp::create(rewriter,
         loc, tileSliceType, tileLoadOp.getBase(), memrefIndices, maskOp1D,
         /*passthru=*/pad1DOp);
 
     // Create 'arm_sme.insert_tile_slice' to insert slice into tile.
-    auto insertSlice = rewriter.create<arm_sme::InsertTileSliceOp>(
+    auto insertSlice = arm_sme::InsertTileSliceOp::create(rewriter,
         loc, tileType, loadSlice->getResult(0), currentTile, tileSliceIndex,
         tileLoadOp.getLayout());
-    rewriter.create<scf::YieldOp>(loc, insertSlice.getResult());
+    scf::YieldOp::create(rewriter, loc, insertSlice.getResult());
 
     rewriter.setInsertionPointAfter(forOp);
 
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index 94f7caa315cf7..e1d28f19ee236 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -203,7 +203,7 @@ static void addAsyncRuntimeApiDeclarations(ModuleOp module) {
   auto addFuncDecl = [&](StringRef name, FunctionType type) {
     if (module.lookupSymbol(name))
       return;
-    builder.create<func::FuncOp>(name, type).setPrivate();
+    func::FuncOp::create(builder, name, type).setPrivate();
   };
 
   MLIRContext *ctx = module.getContext();
@@ -254,15 +254,15 @@ static void addResumeFunction(ModuleOp module) {
   auto voidTy = LLVM::LLVMVoidType::get(ctx);
   Type ptrType = AsyncAPI::opaquePointerType(ctx);
 
-  auto resumeOp = moduleBuilder.create<LLVM::LLVMFuncOp>(
+  auto resumeOp = LLVM::LLVMFuncOp::create(moduleBuilder,
       kResume, LLVM::LLVMFunctionType::get(voidTy, {ptrType}));
   resumeOp.setPrivate();
 
   auto *block = resumeOp.addEntryBlock(moduleBuilder);
   auto blockBuilder = ImplicitLocOpBuilder::atBlockEnd(loc, block);
 
-  blockBuilder.create<LLVM::CoroResumeOp>(resumeOp.getArgument(0));
-  blockBuilder.create<LLVM::ReturnOp>(ValueRange());
+  LLVM::CoroResumeOp::create(blockBuilder, resumeOp.getArgument(0));
+  LLVM::ReturnOp::create(blockBuilder, ValueRange());
 }
 
 //===----------------------------------------------------------------------===//
@@ -282,7 +282,7 @@ class AsyncRuntimeTypeConverter : public TypeConverter {
     // in patterns for other dialects.
     auto addUnrealizedCast = [](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+      auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
       return cast.getResult(0);
     };
 
@@ -343,8 +343,8 @@ class CoroIdOpConversion : public AsyncOpConversionPattern<CoroIdOp> {
 
     // Constants for initializing coroutine frame.
     auto constZero =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), 0);
-    auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, ptrType);
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), 0);
+    auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, ptrType);
 
     // Get coroutine id: @llvm.coro.id.
     rewriter.replaceOpWithNewOp<LLVM::CoroIdOp>(
@@ -372,32 +372,32 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
 
     // Get coroutine frame size: @llvm.coro.size.i64.
     Value coroSize =
-        rewriter.create<LLVM::CoroSizeOp>(loc, rewriter.getI64Type());
+        LLVM::CoroSizeOp::create(rewriter, loc, rewriter.getI64Type());
     // Get coroutine frame alignment: @llvm.coro.align.i64.
     Value coroAlign =
-        rewriter.create<LLVM::CoroAlignOp>(loc, rewriter.getI64Type());
+        LLVM::CoroAlignOp::create(rewriter, loc, rewriter.getI64Type());
 
     // Round up the size to be multiple of the alignment. Since aligned_alloc
     // requires the size parameter be an integral multiple of the alignment
     // parameter.
     auto makeConstant = [&](uint64_t c) {
-      return rewriter.create<LLVM::ConstantOp>(op->getLoc(),
+      return LLVM::ConstantOp::create(rewriter, op->getLoc(),
                                                rewriter.getI64Type(), c);
     };
-    coroSize = rewriter.create<LLVM::AddOp>(op->getLoc(), coroSize, coroAlign);
+    coroSize = LLVM::AddOp::create(rewriter, op->getLoc(), coroSize, coroAlign);
     coroSize =
-        rewriter.create<LLVM::SubOp>(op->getLoc(), coroSize, makeConstant(1));
+        LLVM::SubOp::create(rewriter, op->getLoc(), coroSize, makeConstant(1));
     Value negCoroAlign =
-        rewriter.create<LLVM::SubOp>(op->getLoc(), makeConstant(0), coroAlign);
+        LLVM::SubOp::create(rewriter, op->getLoc(), makeConstant(0), coroAlign);
     coroSize =
-        rewriter.create<LLVM::AndOp>(op->getLoc(), coroSize, negCoroAlign);
+        LLVM::AndOp::create(rewriter, op->getLoc(), coroSize, negCoroAlign);
 
     // Allocate memory for the coroutine frame.
     auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn(
         rewriter, op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
     if (failed(allocFuncOp))
       return failure();
-    auto coroAlloc = rewriter.create<LLVM::CallOp>(
+    auto coroAlloc = LLVM::CallOp::create(rewriter,
         loc, allocFuncOp.value(), ValueRange{coroAlign, coroSize});
 
     // Begin a coroutine: @llvm.coro.begin.
@@ -427,7 +427,7 @@ class CoroFreeOpConversion : public AsyncOpConversionPattern<CoroFreeOp> {
 
     // Get a pointer to the coroutine frame memory: @llvm.coro.free.
     auto coroMem =
-        rewriter.create<LLVM::CoroFreeOp>(loc, ptrType, adaptor.getOperands());
+        LLVM::CoroFreeOp::create(rewriter, loc, ptrType, adaptor.getOperands());
 
     // Free the memory.
     auto freeFuncOp =
@@ -455,13 +455,13 @@ class CoroEndOpConversion : public OpConversionPattern<CoroEndOp> {
   matchAndRewrite(CoroEndOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // We are not in the block that is part of the unwind sequence.
-    auto constFalse = rewriter.create<LLVM::ConstantOp>(
+    auto constFalse = LLVM::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getI1Type(), rewriter.getBoolAttr(false));
-    auto noneToken = rewriter.create<LLVM::NoneTokenOp>(op->getLoc());
+    auto noneToken = LLVM::NoneTokenOp::create(rewriter, op->getLoc());
 
     // Mark the end of a coroutine: @llvm.coro.end.
     auto coroHdl = adaptor.getHandle();
-    rewriter.create<LLVM::CoroEndOp>(
+    LLVM::CoroEndOp::create(rewriter,
         op->getLoc(), rewriter.getI1Type(),
         ValueRange({coroHdl, constFalse, noneToken}));
     rewriter.eraseOp(op);
@@ -534,12 +534,12 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     auto loc = op->getLoc();
 
     // This is not a final suspension point.
-    auto constFalse = rewriter.create<LLVM::ConstantOp>(
+    auto constFalse = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI1Type(), rewriter.getBoolAttr(false));
 
     // Suspend a coroutine: @llvm.coro.suspend
     auto coroState = adaptor.getState();
-    auto coroSuspend = rewriter.create<LLVM::CoroSuspendOp>(
+    auto coroSuspend = LLVM::CoroSuspendOp::create(rewriter,
         loc, i8, ValueRange({coroState, constFalse}));
 
     // Cast return code to i32.
@@ -551,7 +551,7 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     llvm::SmallVector<Block *, 2> caseDest = {op.getResumeDest(),
                                               op.getCleanupDest()};
     rewriter.replaceOpWithNewOp<LLVM::SwitchOp>(
-        op, rewriter.create<LLVM::SExtOp>(loc, i32, coroSuspend.getResult()),
+        op, LLVM::SExtOp::create(rewriter, loc, i32, coroSuspend.getResult()),
         /*defaultDestination=*/op.getSuspendDest(),
         /*defaultOperands=*/ValueRange(),
         /*caseValues=*/caseValues,
@@ -602,11 +602,11 @@ class RuntimeCreateOpLowering : public ConvertOpToLLVMPattern<RuntimeCreateOp> {
 
         // %Size = getelementptr %T* null, int 1
         // %SizeI = ptrtoint %T* %Size to i64
-        auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, storagePtrType);
+        auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, storagePtrType);
         auto gep =
-            rewriter.create<LLVM::GEPOp>(loc, storagePtrType, storedType,
+            LLVM::GEPOp::create(rewriter, loc, storagePtrType, storedType,
                                          nullPtr, ArrayRef<LLVM::GEPArg>{1});
-        return rewriter.create<LLVM::PtrToIntOp>(loc, i64, gep);
+        return LLVM::PtrToIntOp::create(rewriter, loc, i64, gep);
       };
 
       rewriter.replaceOpWithNewOp<func::CallOp>(op, kCreateValue, resultType,
@@ -739,7 +739,7 @@ class RuntimeAwaitOpLowering : public OpConversionPattern<RuntimeAwaitOp> {
             .Case<ValueType>([](Type) { return kAwaitValue; })
             .Case<GroupType>([](Type) { return kAwaitGroup; });
 
-    rewriter.create<func::CallOp>(op->getLoc(), apiFuncName, TypeRange(),
+    func::CallOp::create(rewriter, op->getLoc(), apiFuncName, TypeRange(),
                                   adaptor.getOperands());
     rewriter.eraseOp(op);
 
@@ -772,11 +772,11 @@ class RuntimeAwaitAndResumeOpLowering
 
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = rewriter.create<LLVM::AddressOfOp>(
+    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
         op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
         kResume);
 
-    rewriter.create<func::CallOp>(
+    func::CallOp::create(rewriter,
         op->getLoc(), apiFuncName, TypeRange(),
         ValueRange({operand, handle, resumePtr.getRes()}));
     rewriter.eraseOp(op);
@@ -801,7 +801,7 @@ class RuntimeResumeOpLowering
                   ConversionPatternRewriter &rewriter) const override {
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = rewriter.create<LLVM::AddressOfOp>(
+    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
         op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
         kResume);
 
@@ -832,7 +832,7 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = rewriter.create<func::CallOp>(
+    auto storagePtr = func::CallOp::create(rewriter,
         loc, kGetValueStorage, TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
@@ -845,7 +845,7 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     Value castedStoragePtr = storagePtr.getResult(0);
     // Store the yielded value into the async value storage.
     auto value = adaptor.getValue();
-    rewriter.create<LLVM::StoreOp>(loc, value, castedStoragePtr);
+    LLVM::StoreOp::create(rewriter, loc, value, castedStoragePtr);
 
     // Erase the original runtime store operation.
     rewriter.eraseOp(op);
@@ -872,7 +872,7 @@ class RuntimeLoadOpLowering : public ConvertOpToLLVMPattern<RuntimeLoadOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = rewriter.create<func::CallOp>(
+    auto storagePtr = func::CallOp::create(rewriter,
         loc, kGetValueStorage, TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
@@ -960,7 +960,7 @@ class RefCountingOpLowering : public OpConversionPattern<RefCountingOp> {
   LogicalResult
   matchAndRewrite(RefCountingOp op, typename RefCountingOp::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto count = rewriter.create<arith::ConstantOp>(
+    auto count = arith::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getI64Type(),
         rewriter.getI64IntegerAttr(op.getCount()));
 
diff --git a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
index b9991f36cdaaf..a63a1ceb00ebb 100644
--- a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
+++ b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
@@ -47,26 +47,26 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
 
     if (auto unrankedType = dyn_cast<UnrankedMemRefType>(type)) {
       // Constants
-      Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-      Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+      Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+      Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
       // Dynamically evaluate the size and shape of the unranked memref
-      Value rank = rewriter.create<memref::RankOp>(loc, op.getInput());
+      Value rank = memref::RankOp::create(rewriter, loc, op.getInput());
       MemRefType allocType =
           MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType());
-      Value shape = rewriter.create<memref::AllocaOp>(loc, allocType, rank);
+      Value shape = memref::AllocaOp::create(rewriter, loc, allocType, rank);
 
       // Create a loop to query dimension sizes, store them as a shape, and
       // compute the total size of the memref
       auto loopBody = [&](OpBuilder &builder, Location loc, Value i,
                           ValueRange args) {
         auto acc = args.front();
-        auto dim = rewriter.create<memref::DimOp>(loc, op.getInput(), i);
+        auto dim = memref::DimOp::create(rewriter, loc, op.getInput(), i);
 
-        rewriter.create<memref::StoreOp>(loc, dim, shape, i);
-        acc = rewriter.create<arith::MulIOp>(loc, acc, dim);
+        memref::StoreOp::create(rewriter, loc, dim, shape, i);
+        acc = arith::MulIOp::create(rewriter, loc, acc, dim);
 
-        rewriter.create<scf::YieldOp>(loc, acc);
+        scf::YieldOp::create(rewriter, loc, acc);
       };
       auto size = rewriter
                       .create<scf::ForOp>(loc, zero, rank, one, ValueRange(one),
@@ -78,9 +78,9 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
 
       // Allocate new memref with 1D dynamic shape, then reshape into the
       // shape of the original unranked memref
-      alloc = rewriter.create<memref::AllocOp>(loc, memrefType, size);
+      alloc = memref::AllocOp::create(rewriter, loc, memrefType, size);
       alloc =
-          rewriter.create<memref::ReshapeOp>(loc, unrankedType, alloc, shape);
+          memref::ReshapeOp::create(rewriter, loc, unrankedType, alloc, shape);
     } else {
       MemRefType memrefType = cast<MemRefType>(type);
       MemRefLayoutAttrInterface layout;
@@ -103,14 +103,14 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
       }
 
       // Allocate a memref with identity layout.
-      alloc = rewriter.create<memref::AllocOp>(loc, allocType, dynamicOperands);
+      alloc = memref::AllocOp::create(rewriter, loc, allocType, dynamicOperands);
       // Cast the allocation to the specified type if needed.
       if (memrefType != allocType)
         alloc =
-            rewriter.create<memref::CastOp>(op->getLoc(), memrefType, alloc);
+            memref::CastOp::create(rewriter, op->getLoc(), memrefType, alloc);
     }
 
-    rewriter.create<memref::CopyOp>(loc, op.getInput(), alloc);
+    memref::CopyOp::create(rewriter, loc, op.getInput(), alloc);
     rewriter.replaceOp(op, alloc);
     return success();
   }
diff --git a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
index 70b22386f1eea..1961e1004ba8e 100644
--- a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
+++ b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
@@ -23,41 +23,41 @@ void mlir::complex::convertDivToLLVMUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, LLVM::FastmathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, rhsRe, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, rhsIm, rhsIm, fmf), fmf);
+  Value rhsSqNorm = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsIm, fmf), fmf);
+  Value realNumerator = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
 
-  Value imagNumerator = rewriter.create<LLVM::FSubOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsIm, fmf), fmf);
+  Value imagNumerator = LLVM::FSubOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
-  *resultRe = rewriter.create<LLVM::FDivOp>(loc, realNumerator, rhsSqNorm, fmf);
-  *resultIm = rewriter.create<LLVM::FDivOp>(loc, imagNumerator, rhsSqNorm, fmf);
+  *resultRe = LLVM::FDivOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
+  *resultIm = LLVM::FDivOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 void mlir::complex::convertDivToStandardUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, arith::FastMathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, rhsRe, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, rhsIm, rhsIm, fmf), fmf);
+  Value rhsSqNorm = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsIm, fmf), fmf);
-  Value imagNumerator = rewriter.create<arith::SubFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsIm, fmf), fmf);
+  Value realNumerator = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
+  Value imagNumerator = arith::SubFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
   *resultRe =
-      rewriter.create<arith::DivFOp>(loc, realNumerator, rhsSqNorm, fmf);
+      arith::DivFOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
   *resultIm =
-      rewriter.create<arith::DivFOp>(loc, imagNumerator, rhsSqNorm, fmf);
+      arith::DivFOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 // Smith's algorithm to divide complex numbers. It is just a bit smarter
@@ -94,180 +94,180 @@ void mlir::complex::convertDivToLLVMUsingRangeReduction(
   auto elementType = cast<FloatType>(rhsRe.getType());
 
   Value rhsRealImagRatio =
-      rewriter.create<LLVM::FDivOp>(loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = rewriter.create<LLVM::FAddOp>(
+      LLVM::FDivOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
+  Value rhsRealImagDenom = LLVM::FAddOp::create(rewriter,
       loc, rhsIm,
-      rewriter.create<LLVM::FMulOp>(loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRealImagRatio, fmf),
+      LLVM::FMulOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
+  Value realNumerator1 = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
       lhsIm, fmf);
   Value resultReal1 =
-      rewriter.create<LLVM::FDivOp>(loc, realNumerator1, rhsRealImagDenom, fmf);
-  Value imagNumerator1 = rewriter.create<LLVM::FSubOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRealImagRatio, fmf),
+      LLVM::FDivOp::create(rewriter, loc, realNumerator1, rhsRealImagDenom, fmf);
+  Value imagNumerator1 = LLVM::FSubOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
       lhsRe, fmf);
   Value resultImag1 =
-      rewriter.create<LLVM::FDivOp>(loc, imagNumerator1, rhsRealImagDenom, fmf);
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator1, rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
-      rewriter.create<LLVM::FDivOp>(loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = rewriter.create<LLVM::FAddOp>(
+      LLVM::FDivOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
+  Value rhsImagRealDenom = LLVM::FAddOp::create(rewriter,
       loc, rhsRe,
-      rewriter.create<LLVM::FMulOp>(loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = rewriter.create<LLVM::FAddOp>(
+      LLVM::FMulOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
+  Value realNumerator2 = LLVM::FAddOp::create(rewriter,
       loc, lhsRe,
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsImagRealRatio, fmf), fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
   Value resultReal2 =
-      rewriter.create<LLVM::FDivOp>(loc, realNumerator2, rhsImagRealDenom, fmf);
-  Value imagNumerator2 = rewriter.create<LLVM::FSubOp>(
+      LLVM::FDivOp::create(rewriter, loc, realNumerator2, rhsImagRealDenom, fmf);
+  Value imagNumerator2 = LLVM::FSubOp::create(rewriter,
       loc, lhsIm,
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsImagRealRatio, fmf), fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
   Value resultImag2 =
-      rewriter.create<LLVM::FDivOp>(loc, imagNumerator2, rhsImagRealDenom, fmf);
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator2, rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = rewriter.create<LLVM::ConstantOp>(
+  Value zero = LLVM::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
-  Value rhsRealAbs = rewriter.create<LLVM::FAbsOp>(loc, rhsRe, fmf);
-  Value rhsRealIsZero = rewriter.create<LLVM::FCmpOp>(
+  Value rhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, rhsRe, fmf);
+  Value rhsRealIsZero = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, zero);
-  Value rhsImagAbs = rewriter.create<LLVM::FAbsOp>(loc, rhsIm, fmf);
-  Value rhsImagIsZero = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, rhsIm, fmf);
+  Value rhsImagIsZero = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, zero);
   Value lhsRealIsNotNaN =
-      rewriter.create<LLVM::FCmpOp>(loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
+      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
   Value lhsImagIsNotNaN =
-      rewriter.create<LLVM::FCmpOp>(loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
+      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
   Value lhsContainsNotNaNValue =
-      rewriter.create<LLVM::OrOp>(loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = rewriter.create<LLVM::AndOp>(
+      LLVM::OrOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
+  Value resultIsInfinity = LLVM::AndOp::create(rewriter,
       loc, lhsContainsNotNaNValue,
-      rewriter.create<LLVM::AndOp>(loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = rewriter.create<LLVM::ConstantOp>(
+      LLVM::AndOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
+  Value inf = LLVM::ConstantOp::create(rewriter,
       loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfrhsReal =
-      rewriter.create<LLVM::CopySignOp>(loc, inf, rhsRe);
+      LLVM::CopySignOp::create(rewriter, loc, inf, rhsRe);
   Value infinityResultReal =
-      rewriter.create<LLVM::FMulOp>(loc, infWithSignOfrhsReal, lhsRe, fmf);
+      LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsRe, fmf);
   Value infinityResultImag =
-      rewriter.create<LLVM::FMulOp>(loc, infWithSignOfrhsReal, lhsIm, fmf);
+      LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsRealFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, rhsRealAbs, inf);
-  Value rhsImagFinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, rhsImagAbs, inf);
   Value rhsFinite =
-      rewriter.create<LLVM::AndOp>(loc, rhsRealFinite, rhsImagFinite);
-  Value lhsRealAbs = rewriter.create<LLVM::FAbsOp>(loc, lhsRe, fmf);
-  Value lhsRealInfinite = rewriter.create<LLVM::FCmpOp>(
+      LLVM::AndOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
+  Value lhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, lhsRe, fmf);
+  Value lhsRealInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, lhsRealAbs, inf);
-  Value lhsImagAbs = rewriter.create<LLVM::FAbsOp>(loc, lhsIm, fmf);
-  Value lhsImagInfinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, lhsIm, fmf);
+  Value lhsImagInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, lhsImagAbs, inf);
   Value lhsInfinite =
-      rewriter.create<LLVM::OrOp>(loc, lhsRealInfinite, lhsImagInfinite);
+      LLVM::OrOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
-      rewriter.create<LLVM::AndOp>(loc, lhsInfinite, rhsFinite);
-  Value one = rewriter.create<LLVM::ConstantOp>(
+      LLVM::AndOp::create(rewriter, loc, lhsInfinite, rhsFinite);
+  Value one = LLVM::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, lhsRealInfinite, one, zero),
+  Value lhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesrhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRealIsInfWithSign, rhsRe, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesrhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
+  Value resultReal3 = LLVM::FMulOp::create(rewriter,
       loc, inf,
-      rewriter.create<LLVM::FAddOp>(loc, lhsRealIsInfWithSignTimesrhsReal,
+      LLVM::FAddOp::create(rewriter, loc, lhsRealIsInfWithSignTimesrhsReal,
                                     lhsImagIsInfWithSignTimesrhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesrhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRealIsInfWithSign, rhsIm, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesrhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
+  Value resultImag3 = LLVM::FMulOp::create(rewriter,
       loc, inf,
-      rewriter.create<LLVM::FSubOp>(loc, lhsImagIsInfWithSignTimesrhsReal,
+      LLVM::FSubOp::create(rewriter, loc, lhsImagIsInfWithSignTimesrhsReal,
                                     lhsRealIsInfWithSignTimesrhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsRealFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, lhsRealAbs, inf);
-  Value lhsImagFinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsImagFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, lhsImagAbs, inf);
   Value lhsFinite =
-      rewriter.create<LLVM::AndOp>(loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = rewriter.create<LLVM::FCmpOp>(
+      LLVM::AndOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
+  Value rhsRealInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, inf);
-  Value rhsImagInfinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, inf);
   Value rhsInfinite =
-      rewriter.create<LLVM::OrOp>(loc, rhsRealInfinite, rhsImagInfinite);
+      LLVM::OrOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
-      rewriter.create<LLVM::AndOp>(loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, rhsRealInfinite, one, zero),
+      LLVM::AndOp::create(rewriter, loc, lhsFinite, rhsInfinite);
+  Value rhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimeslhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRealIsInfWithSign, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
+  Value resultReal4 = LLVM::FMulOp::create(rewriter,
       loc, zero,
-      rewriter.create<LLVM::FAddOp>(loc, rhsRealIsInfWithSignTimeslhsReal,
+      LLVM::FAddOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsReal,
                                     rhsImagIsInfWithSignTimeslhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimeslhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRealIsInfWithSign, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
+  Value resultImag4 = LLVM::FMulOp::create(rewriter,
       loc, zero,
-      rewriter.create<LLVM::FSubOp>(loc, rhsRealIsInfWithSignTimeslhsImag,
+      LLVM::FSubOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsImag,
                                     rhsImagIsInfWithSignTimeslhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = rewriter.create<LLVM::FCmpOp>(
+  Value realAbsSmallerThanImagAbs = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::olt, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = rewriter.create<LLVM::SelectOp>(
+  Value resultReal5 = LLVM::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = rewriter.create<LLVM::SelectOp>(
+  Value resultImag5 = LLVM::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase3 = LLVM::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase3 = LLVM::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase2 = LLVM::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase2 = LLVM::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase1 = LLVM::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase1 = LLVM::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = rewriter.create<LLVM::FCmpOp>(
+  Value resultRealIsNaN = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::uno, resultReal5, zero);
-  Value resultImagIsNaN = rewriter.create<LLVM::FCmpOp>(
+  Value resultImagIsNaN = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::uno, resultImag5, zero);
   Value resultIsNaN =
-      rewriter.create<LLVM::AndOp>(loc, resultRealIsNaN, resultImagIsNaN);
+      LLVM::AndOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = rewriter.create<LLVM::SelectOp>(
+  *resultRe = LLVM::SelectOp::create(rewriter,
       loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = rewriter.create<LLVM::SelectOp>(
+  *resultIm = LLVM::SelectOp::create(rewriter,
       loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
 }
 
@@ -278,179 +278,179 @@ void mlir::complex::convertDivToStandardUsingRangeReduction(
   auto elementType = cast<FloatType>(rhsRe.getType());
 
   Value rhsRealImagRatio =
-      rewriter.create<arith::DivFOp>(loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = rewriter.create<arith::AddFOp>(
+      arith::DivFOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
+  Value rhsRealImagDenom = arith::AddFOp::create(rewriter,
       loc, rhsIm,
-      rewriter.create<arith::MulFOp>(loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRealImagRatio, fmf),
+      arith::MulFOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
+  Value realNumerator1 = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
       lhsIm, fmf);
-  Value resultReal1 = rewriter.create<arith::DivFOp>(loc, realNumerator1,
+  Value resultReal1 = arith::DivFOp::create(rewriter, loc, realNumerator1,
                                                      rhsRealImagDenom, fmf);
-  Value imagNumerator1 = rewriter.create<arith::SubFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRealImagRatio, fmf),
+  Value imagNumerator1 = arith::SubFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
       lhsRe, fmf);
-  Value resultImag1 = rewriter.create<arith::DivFOp>(loc, imagNumerator1,
+  Value resultImag1 = arith::DivFOp::create(rewriter, loc, imagNumerator1,
                                                      rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
-      rewriter.create<arith::DivFOp>(loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = rewriter.create<arith::AddFOp>(
+      arith::DivFOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
+  Value rhsImagRealDenom = arith::AddFOp::create(rewriter,
       loc, rhsRe,
-      rewriter.create<arith::MulFOp>(loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = rewriter.create<arith::AddFOp>(
+      arith::MulFOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
+  Value realNumerator2 = arith::AddFOp::create(rewriter,
       loc, lhsRe,
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsImagRealRatio, fmf), fmf);
-  Value resultReal2 = rewriter.create<arith::DivFOp>(loc, realNumerator2,
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
+  Value resultReal2 = arith::DivFOp::create(rewriter, loc, realNumerator2,
                                                      rhsImagRealDenom, fmf);
-  Value imagNumerator2 = rewriter.create<arith::SubFOp>(
+  Value imagNumerator2 = arith::SubFOp::create(rewriter,
       loc, lhsIm,
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsImagRealRatio, fmf), fmf);
-  Value resultImag2 = rewriter.create<arith::DivFOp>(loc, imagNumerator2,
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
+  Value resultImag2 = arith::DivFOp::create(rewriter, loc, imagNumerator2,
                                                      rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
-  Value rhsRealAbs = rewriter.create<math::AbsFOp>(loc, rhsRe, fmf);
-  Value rhsRealIsZero = rewriter.create<arith::CmpFOp>(
+  Value rhsRealAbs = math::AbsFOp::create(rewriter, loc, rhsRe, fmf);
+  Value rhsRealIsZero = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsRealAbs, zero);
-  Value rhsImagAbs = rewriter.create<math::AbsFOp>(loc, rhsIm, fmf);
-  Value rhsImagIsZero = rewriter.create<arith::CmpFOp>(
+  Value rhsImagAbs = math::AbsFOp::create(rewriter, loc, rhsIm, fmf);
+  Value rhsImagIsZero = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsImagAbs, zero);
-  Value lhsRealIsNotNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsRealIsNotNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ORD, lhsRe, zero);
-  Value lhsImagIsNotNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsImagIsNotNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ORD, lhsIm, zero);
   Value lhsContainsNotNaNValue =
-      rewriter.create<arith::OrIOp>(loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = rewriter.create<arith::AndIOp>(
+      arith::OrIOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
+  Value resultIsInfinity = arith::AndIOp::create(rewriter,
       loc, lhsContainsNotNaNValue,
-      rewriter.create<arith::AndIOp>(loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = rewriter.create<arith::ConstantOp>(
+      arith::AndIOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
+  Value inf = arith::ConstantOp::create(rewriter,
       loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfRhsReal =
-      rewriter.create<math::CopySignOp>(loc, inf, rhsRe);
+      math::CopySignOp::create(rewriter, loc, inf, rhsRe);
   Value infinityResultReal =
-      rewriter.create<arith::MulFOp>(loc, infWithSignOfRhsReal, lhsRe, fmf);
+      arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsRe, fmf);
   Value infinityResultImag =
-      rewriter.create<arith::MulFOp>(loc, infWithSignOfRhsReal, lhsIm, fmf);
+      arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = rewriter.create<arith::CmpFOp>(
+  Value rhsRealFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, rhsRealAbs, inf);
-  Value rhsImagFinite = rewriter.create<arith::CmpFOp>(
+  Value rhsImagFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, rhsImagAbs, inf);
   Value rhsFinite =
-      rewriter.create<arith::AndIOp>(loc, rhsRealFinite, rhsImagFinite);
-  Value lhsRealAbs = rewriter.create<math::AbsFOp>(loc, lhsRe, fmf);
-  Value lhsRealInfinite = rewriter.create<arith::CmpFOp>(
+      arith::AndIOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
+  Value lhsRealAbs = math::AbsFOp::create(rewriter, loc, lhsRe, fmf);
+  Value lhsRealInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, lhsRealAbs, inf);
-  Value lhsImagAbs = rewriter.create<math::AbsFOp>(loc, lhsIm, fmf);
-  Value lhsImagInfinite = rewriter.create<arith::CmpFOp>(
+  Value lhsImagAbs = math::AbsFOp::create(rewriter, loc, lhsIm, fmf);
+  Value lhsImagInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, lhsImagAbs, inf);
   Value lhsInfinite =
-      rewriter.create<arith::OrIOp>(loc, lhsRealInfinite, lhsImagInfinite);
+      arith::OrIOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
-      rewriter.create<arith::AndIOp>(loc, lhsInfinite, rhsFinite);
-  Value one = rewriter.create<arith::ConstantOp>(
+      arith::AndIOp::create(rewriter, loc, lhsInfinite, rhsFinite);
+  Value one = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, lhsRealInfinite, one, zero),
+  Value lhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesRhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRealIsInfWithSign, rhsRe, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesRhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
+  Value resultReal3 = arith::MulFOp::create(rewriter,
       loc, inf,
-      rewriter.create<arith::AddFOp>(loc, lhsRealIsInfWithSignTimesRhsReal,
+      arith::AddFOp::create(rewriter, loc, lhsRealIsInfWithSignTimesRhsReal,
                                      lhsImagIsInfWithSignTimesRhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesRhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsRealIsInfWithSign, rhsIm, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesRhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
+  Value resultImag3 = arith::MulFOp::create(rewriter,
       loc, inf,
-      rewriter.create<arith::SubFOp>(loc, lhsImagIsInfWithSignTimesRhsReal,
+      arith::SubFOp::create(rewriter, loc, lhsImagIsInfWithSignTimesRhsReal,
                                      lhsRealIsInfWithSignTimesRhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = rewriter.create<arith::CmpFOp>(
+  Value lhsRealFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, lhsRealAbs, inf);
-  Value lhsImagFinite = rewriter.create<arith::CmpFOp>(
+  Value lhsImagFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, lhsImagAbs, inf);
   Value lhsFinite =
-      rewriter.create<arith::AndIOp>(loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = rewriter.create<arith::CmpFOp>(
+      arith::AndIOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
+  Value rhsRealInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsRealAbs, inf);
-  Value rhsImagInfinite = rewriter.create<arith::CmpFOp>(
+  Value rhsImagInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsImagAbs, inf);
   Value rhsInfinite =
-      rewriter.create<arith::OrIOp>(loc, rhsRealInfinite, rhsImagInfinite);
+      arith::OrIOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
-      rewriter.create<arith::AndIOp>(loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, rhsRealInfinite, one, zero),
+      arith::AndIOp::create(rewriter, loc, lhsFinite, rhsInfinite);
+  Value rhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimesLhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRealIsInfWithSign, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
+  Value resultReal4 = arith::MulFOp::create(rewriter,
       loc, zero,
-      rewriter.create<arith::AddFOp>(loc, rhsRealIsInfWithSignTimesLhsReal,
+      arith::AddFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsReal,
                                      rhsImagIsInfWithSignTimesLhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimesLhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRealIsInfWithSign, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
+  Value resultImag4 = arith::MulFOp::create(rewriter,
       loc, zero,
-      rewriter.create<arith::SubFOp>(loc, rhsRealIsInfWithSignTimesLhsImag,
+      arith::SubFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsImag,
                                      rhsImagIsInfWithSignTimesLhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = rewriter.create<arith::CmpFOp>(
+  Value realAbsSmallerThanImagAbs = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OLT, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = rewriter.create<arith::SelectOp>(
+  Value resultReal5 = arith::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = rewriter.create<arith::SelectOp>(
+  Value resultImag5 = arith::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase3 = arith::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase3 = arith::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase2 = arith::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase2 = arith::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase1 = arith::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase1 = arith::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = rewriter.create<arith::CmpFOp>(
+  Value resultRealIsNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::UNO, resultReal5, zero);
-  Value resultImagIsNaN = rewriter.create<arith::CmpFOp>(
+  Value resultImagIsNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::UNO, resultImag5, zero);
   Value resultIsNaN =
-      rewriter.create<arith::AndIOp>(loc, resultRealIsNaN, resultImagIsNaN);
+      arith::AndIOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = rewriter.create<arith::SelectOp>(
+  *resultRe = arith::SelectOp::create(rewriter,
       loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = rewriter.create<arith::SelectOp>(
+  *resultIm = arith::SelectOp::create(rewriter,
       loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
 }
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index e5e862315941d..a06935c4387c6 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -35,7 +35,7 @@ static constexpr unsigned kImaginaryPosInComplexNumberStruct = 1;
 
 ComplexStructBuilder ComplexStructBuilder::poison(OpBuilder &builder,
                                                   Location loc, Type type) {
-  Value val = builder.create<LLVM::PoisonOp>(loc, type);
+  Value val = LLVM::PoisonOp::create(builder, loc, type);
   return ComplexStructBuilder(val);
 }
 
@@ -79,9 +79,9 @@ struct AbsOpConversion : public ConvertOpToLLVMPattern<complex::AbsOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value sqNorm = rewriter.create<LLVM::FAddOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, real, real, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, imag, imag, fmf), fmf);
+    Value sqNorm = LLVM::FAddOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, real, real, fmf),
+        LLVM::FMulOp::create(rewriter, loc, imag, imag, fmf), fmf);
 
     rewriter.replaceOpWithNewOp<LLVM::SqrtOp>(op, sqNorm);
     return success();
@@ -192,9 +192,9 @@ struct AddOpConversion : public ConvertOpToLLVMPattern<complex::AddOp> {
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
     Value real =
-        rewriter.create<LLVM::FAddOp>(loc, arg.lhs.real(), arg.rhs.real(), fmf);
+        LLVM::FAddOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
     Value imag =
-        rewriter.create<LLVM::FAddOp>(loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+        LLVM::FAddOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
@@ -278,13 +278,13 @@ struct MulOpConversion : public ConvertOpToLLVMPattern<complex::MulOp> {
     Value lhsRe = arg.lhs.real();
     Value lhsIm = arg.lhs.imag();
 
-    Value real = rewriter.create<LLVM::FSubOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, rhsRe, lhsRe, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, rhsIm, lhsIm, fmf), fmf);
+    Value real = LLVM::FSubOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, lhsRe, fmf),
+        LLVM::FMulOp::create(rewriter, loc, rhsIm, lhsIm, fmf), fmf);
 
-    Value imag = rewriter.create<LLVM::FAddOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRe, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsIm, fmf), fmf);
+    Value imag = LLVM::FAddOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+        LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
@@ -314,9 +314,9 @@ struct SubOpConversion : public ConvertOpToLLVMPattern<complex::SubOp> {
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
     Value real =
-        rewriter.create<LLVM::FSubOp>(loc, arg.lhs.real(), arg.rhs.real(), fmf);
+        LLVM::FSubOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
     Value imag =
-        rewriter.create<LLVM::FSubOp>(loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+        LLVM::FSubOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
diff --git a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
index 56269d189873a..4800f65c675e6 100644
--- a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
+++ b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
@@ -84,7 +84,7 @@ LogicalResult ScalarOpToLibmCall<Op, TypeResolver>::matchAndRewrite(
     rewriter.setInsertionPointToStart(&module->getRegion(0).front());
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-    opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name,
+    opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
                                            opFunctionTy);
     opFunc.setPrivate();
   }
diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 0c832c452718b..d14e3888b118c 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -31,44 +31,44 @@ enum class AbsFn { abs, sqrt, rsqrt };
 // Returns the absolute value, its square root or its reciprocal square root.
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
                  ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
-  Value one = b.create<arith::ConstantOp>(real.getType(),
+  Value one = arith::ConstantOp::create(b, real.getType(),
                                           b.getFloatAttr(real.getType(), 1.0));
 
-  Value absReal = b.create<math::AbsFOp>(real, fmf);
-  Value absImag = b.create<math::AbsFOp>(imag, fmf);
+  Value absReal = math::AbsFOp::create(b, real, fmf);
+  Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-  Value max = b.create<arith::MaximumFOp>(absReal, absImag, fmf);
-  Value min = b.create<arith::MinimumFOp>(absReal, absImag, fmf);
+  Value max = arith::MaximumFOp::create(b, absReal, absImag, fmf);
+  Value min = arith::MinimumFOp::create(b, absReal, absImag, fmf);
 
   // The lowering below requires NaNs and infinities to work correctly.
   arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear(
       fmf, arith::FastMathFlags::nnan | arith::FastMathFlags::ninf);
-  Value ratio = b.create<arith::DivFOp>(min, max, fmfWithNaNInf);
-  Value ratioSq = b.create<arith::MulFOp>(ratio, ratio, fmfWithNaNInf);
-  Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmfWithNaNInf);
+  Value ratio = arith::DivFOp::create(b, min, max, fmfWithNaNInf);
+  Value ratioSq = arith::MulFOp::create(b, ratio, ratio, fmfWithNaNInf);
+  Value ratioSqPlusOne = arith::AddFOp::create(b, ratioSq, one, fmfWithNaNInf);
   Value result;
 
   if (fn == AbsFn::rsqrt) {
-    ratioSqPlusOne = b.create<math::RsqrtOp>(ratioSqPlusOne, fmfWithNaNInf);
-    min = b.create<math::RsqrtOp>(min, fmfWithNaNInf);
-    max = b.create<math::RsqrtOp>(max, fmfWithNaNInf);
+    ratioSqPlusOne = math::RsqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
+    min = math::RsqrtOp::create(b, min, fmfWithNaNInf);
+    max = math::RsqrtOp::create(b, max, fmfWithNaNInf);
   }
 
   if (fn == AbsFn::sqrt) {
-    Value quarter = b.create<arith::ConstantOp>(
+    Value quarter = arith::ConstantOp::create(b,
         real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
-    Value sqrt = b.create<math::SqrtOp>(max, fmfWithNaNInf);
-    Value p025 = b.create<math::PowFOp>(ratioSqPlusOne, quarter, fmfWithNaNInf);
-    result = b.create<arith::MulFOp>(sqrt, p025, fmfWithNaNInf);
+    Value sqrt = math::SqrtOp::create(b, max, fmfWithNaNInf);
+    Value p025 = math::PowFOp::create(b, ratioSqPlusOne, quarter, fmfWithNaNInf);
+    result = arith::MulFOp::create(b, sqrt, p025, fmfWithNaNInf);
   } else {
-    Value sqrt = b.create<math::SqrtOp>(ratioSqPlusOne, fmfWithNaNInf);
-    result = b.create<arith::MulFOp>(max, sqrt, fmfWithNaNInf);
+    Value sqrt = math::SqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
+    result = arith::MulFOp::create(b, max, sqrt, fmfWithNaNInf);
   }
 
-  Value isNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, result,
+  Value isNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, result,
                                         result, fmfWithNaNInf);
-  return b.create<arith::SelectOp>(isNaN, min, result);
+  return arith::SelectOp::create(b, isNaN, min, result);
 }
 
 struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
@@ -81,8 +81,8 @@ struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
 
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
     rewriter.replaceOp(op, computeAbs(real, imag, fmf, b));
 
     return success();
@@ -105,28 +105,28 @@ struct Atan2OpConversion : public OpConversionPattern<complex::Atan2Op> {
     Value lhs = adaptor.getLhs();
     Value rhs = adaptor.getRhs();
 
-    Value rhsSquared = b.create<complex::MulOp>(type, rhs, rhs, fmf);
-    Value lhsSquared = b.create<complex::MulOp>(type, lhs, lhs, fmf);
+    Value rhsSquared = complex::MulOp::create(b, type, rhs, rhs, fmf);
+    Value lhsSquared = complex::MulOp::create(b, type, lhs, lhs, fmf);
     Value rhsSquaredPlusLhsSquared =
-        b.create<complex::AddOp>(type, rhsSquared, lhsSquared, fmf);
+        complex::AddOp::create(b, type, rhsSquared, lhsSquared, fmf);
     Value sqrtOfRhsSquaredPlusLhsSquared =
-        b.create<complex::SqrtOp>(type, rhsSquaredPlusLhsSquared, fmf);
+        complex::SqrtOp::create(b, type, rhsSquaredPlusLhsSquared, fmf);
 
     Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
-    Value one = b.create<arith::ConstantOp>(elementType,
+        arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
+    Value one = arith::ConstantOp::create(b, elementType,
                                             b.getFloatAttr(elementType, 1));
-    Value i = b.create<complex::CreateOp>(type, zero, one);
-    Value iTimesLhs = b.create<complex::MulOp>(i, lhs, fmf);
-    Value rhsPlusILhs = b.create<complex::AddOp>(rhs, iTimesLhs, fmf);
+    Value i = complex::CreateOp::create(b, type, zero, one);
+    Value iTimesLhs = complex::MulOp::create(b, i, lhs, fmf);
+    Value rhsPlusILhs = complex::AddOp::create(b, rhs, iTimesLhs, fmf);
 
-    Value divResult = b.create<complex::DivOp>(
+    Value divResult = complex::DivOp::create(b,
         rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf);
-    Value logResult = b.create<complex::LogOp>(divResult, fmf);
+    Value logResult = complex::LogOp::create(b, divResult, fmf);
 
-    Value negativeOne = b.create<arith::ConstantOp>(
+    Value negativeOne = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -1));
-    Value negativeI = b.create<complex::CreateOp>(type, zero, negativeOne);
+    Value negativeI = complex::CreateOp::create(b, type, zero, negativeOne);
 
     rewriter.replaceOpWithNewOp<complex::MulOp>(op, negativeI, logResult, fmf);
     return success();
@@ -146,14 +146,14 @@ struct ComparisonOpConversion : public OpConversionPattern<ComparisonOp> {
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getLhs().getType()).getElementType();
 
-    Value realLhs = rewriter.create<complex::ReOp>(loc, type, adaptor.getLhs());
-    Value imagLhs = rewriter.create<complex::ImOp>(loc, type, adaptor.getLhs());
-    Value realRhs = rewriter.create<complex::ReOp>(loc, type, adaptor.getRhs());
-    Value imagRhs = rewriter.create<complex::ImOp>(loc, type, adaptor.getRhs());
+    Value realLhs = complex::ReOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value imagLhs = complex::ImOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value realRhs = complex::ReOp::create(rewriter, loc, type, adaptor.getRhs());
+    Value imagRhs = complex::ImOp::create(rewriter, loc, type, adaptor.getRhs());
     Value realComparison =
-        rewriter.create<arith::CmpFOp>(loc, p, realLhs, realRhs);
+        arith::CmpFOp::create(rewriter, loc, p, realLhs, realRhs);
     Value imagComparison =
-        rewriter.create<arith::CmpFOp>(loc, p, imagLhs, imagRhs);
+        arith::CmpFOp::create(rewriter, loc, p, imagLhs, imagRhs);
 
     rewriter.replaceOpWithNewOp<ResultCombiner>(op, realComparison,
                                                 imagComparison);
@@ -176,13 +176,13 @@ struct BinaryComplexOpConversion : public OpConversionPattern<BinaryComplexOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
-    Value realLhs = b.create<complex::ReOp>(elementType, adaptor.getLhs());
-    Value realRhs = b.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value resultReal = b.create<BinaryStandardOp>(elementType, realLhs, realRhs,
+    Value realLhs = complex::ReOp::create(b, elementType, adaptor.getLhs());
+    Value realRhs = complex::ReOp::create(b, elementType, adaptor.getRhs());
+    Value resultReal = BinaryStandardOp::create(b, elementType, realLhs, realRhs,
                                                   fmf.getValue());
-    Value imagLhs = b.create<complex::ImOp>(elementType, adaptor.getLhs());
-    Value imagRhs = b.create<complex::ImOp>(elementType, adaptor.getRhs());
-    Value resultImag = b.create<BinaryStandardOp>(elementType, imagLhs, imagRhs,
+    Value imagLhs = complex::ImOp::create(b, elementType, adaptor.getLhs());
+    Value imagRhs = complex::ImOp::create(b, elementType, adaptor.getRhs());
+    Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs, imagRhs,
                                                   fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -205,20 +205,20 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
 
     // Trigonometric ops use a set of common building blocks to convert to real
     // ops. Here we create these building blocks and call into an op-specific
     // implementation in the subclass to combine them.
-    Value half = rewriter.create<arith::ConstantOp>(
+    Value half = arith::ConstantOp::create(rewriter,
         loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
-    Value exp = rewriter.create<math::ExpOp>(loc, imag, fmf);
-    Value scaledExp = rewriter.create<arith::MulFOp>(loc, half, exp, fmf);
-    Value reciprocalExp = rewriter.create<arith::DivFOp>(loc, half, exp, fmf);
-    Value sin = rewriter.create<math::SinOp>(loc, real, fmf);
-    Value cos = rewriter.create<math::CosOp>(loc, real, fmf);
+    Value exp = math::ExpOp::create(rewriter, loc, imag, fmf);
+    Value scaledExp = arith::MulFOp::create(rewriter, loc, half, exp, fmf);
+    Value reciprocalExp = arith::DivFOp::create(rewriter, loc, half, exp, fmf);
+    Value sin = math::SinOp::create(rewriter, loc, real, fmf);
+    Value cos = math::CosOp::create(rewriter, loc, real, fmf);
 
     auto resultPair =
         combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf);
@@ -251,11 +251,11 @@ struct CosOpConversion : public TrigonometricOpConversion<complex::CosOp> {
     //   Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x
     //   Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x
     Value sum =
-        rewriter.create<arith::AddFOp>(loc, reciprocalExp, scaledExp, fmf);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, cos, fmf);
+        arith::AddFOp::create(rewriter, loc, reciprocalExp, scaledExp, fmf);
+    Value resultReal = arith::MulFOp::create(rewriter, loc, sum, cos, fmf);
     Value diff =
-        rewriter.create<arith::SubFOp>(loc, reciprocalExp, scaledExp, fmf);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, sin, fmf);
+        arith::SubFOp::create(rewriter, loc, reciprocalExp, scaledExp, fmf);
+    Value resultImag = arith::MulFOp::create(rewriter, loc, diff, sin, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -275,13 +275,13 @@ struct DivOpConversion : public OpConversionPattern<complex::DivOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value lhsReal =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getLhs());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getLhs());
     Value lhsImag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getLhs());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getLhs());
     Value rhsReal =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getRhs());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getRhs());
     Value rhsImag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getRhs());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getRhs());
 
     Value resultReal, resultImag;
 
@@ -318,16 +318,16 @@ struct ExpOpConversion : public OpConversionPattern<complex::ExpOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value expReal = rewriter.create<math::ExpOp>(loc, real, fmf.getValue());
-    Value cosImag = rewriter.create<math::CosOp>(loc, imag, fmf.getValue());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value expReal = math::ExpOp::create(rewriter, loc, real, fmf.getValue());
+    Value cosImag = math::CosOp::create(rewriter, loc, imag, fmf.getValue());
     Value resultReal =
-        rewriter.create<arith::MulFOp>(loc, expReal, cosImag, fmf.getValue());
-    Value sinImag = rewriter.create<math::SinOp>(loc, imag, fmf.getValue());
+        arith::MulFOp::create(rewriter, loc, expReal, cosImag, fmf.getValue());
+    Value sinImag = math::SinOp::create(rewriter, loc, imag, fmf.getValue());
     Value resultImag =
-        rewriter.create<arith::MulFOp>(loc, expReal, sinImag, fmf.getValue());
+        arith::MulFOp::create(rewriter, loc, expReal, sinImag, fmf.getValue());
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -340,11 +340,11 @@ Value evaluatePolynomial(ImplicitLocOpBuilder &b, Value arg,
                          arith::FastMathFlagsAttr fmf) {
   auto argType = mlir::cast<FloatType>(arg.getType());
   Value poly =
-      b.create<arith::ConstantOp>(b.getFloatAttr(argType, coefficients[0]));
+      arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[0]));
   for (unsigned i = 1; i < coefficients.size(); ++i) {
-    poly = b.create<math::FmaOp>(
+    poly = math::FmaOp::create(b,
         poly, arg,
-        b.create<arith::ConstantOp>(b.getFloatAttr(argType, coefficients[i])),
+        arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[i])),
         fmf);
   }
   return poly;
@@ -365,26 +365,26 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
 
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     ImplicitLocOpBuilder b(op.getLoc(), rewriter);
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
-    Value zero = b.create<arith::ConstantOp>(b.getFloatAttr(elemType, 0.0));
-    Value one = b.create<arith::ConstantOp>(b.getFloatAttr(elemType, 1.0));
+    Value zero = arith::ConstantOp::create(b, b.getFloatAttr(elemType, 0.0));
+    Value one = arith::ConstantOp::create(b, b.getFloatAttr(elemType, 1.0));
 
-    Value expm1Real = b.create<math::ExpM1Op>(real, fmf);
-    Value expReal = b.create<arith::AddFOp>(expm1Real, one, fmf);
+    Value expm1Real = math::ExpM1Op::create(b, real, fmf);
+    Value expReal = arith::AddFOp::create(b, expm1Real, one, fmf);
 
-    Value sinImag = b.create<math::SinOp>(imag, fmf);
+    Value sinImag = math::SinOp::create(b, imag, fmf);
     Value cosm1Imag = emitCosm1(imag, fmf, b);
-    Value cosImag = b.create<arith::AddFOp>(cosm1Imag, one, fmf);
+    Value cosImag = arith::AddFOp::create(b, cosm1Imag, one, fmf);
 
-    Value realResult = b.create<arith::AddFOp>(
-        b.create<arith::MulFOp>(expm1Real, cosImag, fmf), cosm1Imag, fmf);
+    Value realResult = arith::AddFOp::create(b,
+        arith::MulFOp::create(b, expm1Real, cosImag, fmf), cosm1Imag, fmf);
 
-    Value imagIsZero = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag,
+    Value imagIsZero = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag,
                                                zero, fmf.getValue());
-    Value imagResult = b.create<arith::SelectOp>(
-        imagIsZero, zero, b.create<arith::MulFOp>(expReal, sinImag, fmf));
+    Value imagResult = arith::SelectOp::create(b,
+        imagIsZero, zero, arith::MulFOp::create(b, expReal, sinImag, fmf));
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, realResult,
                                                    imagResult);
@@ -395,8 +395,8 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
   Value emitCosm1(Value arg, arith::FastMathFlagsAttr fmf,
                   ImplicitLocOpBuilder &b) const {
     auto argType = mlir::cast<FloatType>(arg.getType());
-    auto negHalf = b.create<arith::ConstantOp>(b.getFloatAttr(argType, -0.5));
-    auto negOne = b.create<arith::ConstantOp>(b.getFloatAttr(argType, -1.0));
+    auto negHalf = arith::ConstantOp::create(b, b.getFloatAttr(argType, -0.5));
+    auto negOne = arith::ConstantOp::create(b, b.getFloatAttr(argType, -1.0));
 
     // Algorithm copied from cephes cosm1.
     SmallVector<double, 7> kCoeffs{
@@ -405,23 +405,23 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
         2.4801587301570552304991E-5,  -1.3888888888888872993737E-3,
         4.1666666666666666609054E-2,
     };
-    Value cos = b.create<math::CosOp>(arg, fmf);
-    Value forLargeArg = b.create<arith::AddFOp>(cos, negOne, fmf);
+    Value cos = math::CosOp::create(b, arg, fmf);
+    Value forLargeArg = arith::AddFOp::create(b, cos, negOne, fmf);
 
-    Value argPow2 = b.create<arith::MulFOp>(arg, arg, fmf);
-    Value argPow4 = b.create<arith::MulFOp>(argPow2, argPow2, fmf);
+    Value argPow2 = arith::MulFOp::create(b, arg, arg, fmf);
+    Value argPow4 = arith::MulFOp::create(b, argPow2, argPow2, fmf);
     Value poly = evaluatePolynomial(b, argPow2, kCoeffs, fmf);
 
     auto forSmallArg =
-        b.create<arith::AddFOp>(b.create<arith::MulFOp>(argPow4, poly, fmf),
-                                b.create<arith::MulFOp>(negHalf, argPow2, fmf));
+        arith::AddFOp::create(b, arith::MulFOp::create(b, argPow4, poly, fmf),
+                                arith::MulFOp::create(b, negHalf, argPow2, fmf));
 
     // (pi/4)^2 is approximately 0.61685
     Value piOver4Pow2 =
-        b.create<arith::ConstantOp>(b.getFloatAttr(argType, 0.61685));
-    Value cond = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, argPow2,
+        arith::ConstantOp::create(b, b.getFloatAttr(argType, 0.61685));
+    Value cond = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, argPow2,
                                          piOver4Pow2, fmf.getValue());
-    return b.create<arith::SelectOp>(cond, forLargeArg, forSmallArg);
+    return arith::SelectOp::create(b, cond, forLargeArg, forSmallArg);
   }
 };
 
@@ -436,13 +436,13 @@ struct LogOpConversion : public OpConversionPattern<complex::LogOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
-    Value abs = b.create<complex::AbsOp>(elementType, adaptor.getComplex(),
+    Value abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(),
                                          fmf.getValue());
-    Value resultReal = b.create<math::LogOp>(elementType, abs, fmf.getValue());
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value resultReal = math::LogOp::create(b, elementType, abs, fmf.getValue());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value resultImag =
-        b.create<math::Atan2Op>(elementType, imag, real, fmf.getValue());
+        math::Atan2Op::create(b, elementType, imag, real, fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -460,40 +460,40 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
-    Value half = b.create<arith::ConstantOp>(elementType,
+    Value half = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 0.5));
-    Value one = b.create<arith::ConstantOp>(elementType,
+    Value one = arith::ConstantOp::create(b, elementType,
                                             b.getFloatAttr(elementType, 1));
-    Value realPlusOne = b.create<arith::AddFOp>(real, one, fmf);
-    Value absRealPlusOne = b.create<math::AbsFOp>(realPlusOne, fmf);
-    Value absImag = b.create<math::AbsFOp>(imag, fmf);
+    Value realPlusOne = arith::AddFOp::create(b, real, one, fmf);
+    Value absRealPlusOne = math::AbsFOp::create(b, realPlusOne, fmf);
+    Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-    Value maxAbs = b.create<arith::MaximumFOp>(absRealPlusOne, absImag, fmf);
-    Value minAbs = b.create<arith::MinimumFOp>(absRealPlusOne, absImag, fmf);
+    Value maxAbs = arith::MaximumFOp::create(b, absRealPlusOne, absImag, fmf);
+    Value minAbs = arith::MinimumFOp::create(b, absRealPlusOne, absImag, fmf);
 
-    Value useReal = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGT,
+    Value useReal = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT,
                                             realPlusOne, absImag, fmf);
-    Value maxMinusOne = b.create<arith::SubFOp>(maxAbs, one, fmf);
+    Value maxMinusOne = arith::SubFOp::create(b, maxAbs, one, fmf);
     Value maxAbsOfRealPlusOneAndImagMinusOne =
-        b.create<arith::SelectOp>(useReal, real, maxMinusOne);
+        arith::SelectOp::create(b, useReal, real, maxMinusOne);
     arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear(
         fmf, arith::FastMathFlags::nnan | arith::FastMathFlags::ninf);
-    Value minMaxRatio = b.create<arith::DivFOp>(minAbs, maxAbs, fmfWithNaNInf);
+    Value minMaxRatio = arith::DivFOp::create(b, minAbs, maxAbs, fmfWithNaNInf);
     Value logOfMaxAbsOfRealPlusOneAndImag =
-        b.create<math::Log1pOp>(maxAbsOfRealPlusOneAndImagMinusOne, fmf);
-    Value logOfSqrtPart = b.create<math::Log1pOp>(
-        b.create<arith::MulFOp>(minMaxRatio, minMaxRatio, fmfWithNaNInf),
+        math::Log1pOp::create(b, maxAbsOfRealPlusOneAndImagMinusOne, fmf);
+    Value logOfSqrtPart = math::Log1pOp::create(b,
+        arith::MulFOp::create(b, minMaxRatio, minMaxRatio, fmfWithNaNInf),
         fmfWithNaNInf);
-    Value r = b.create<arith::AddFOp>(
-        b.create<arith::MulFOp>(half, logOfSqrtPart, fmfWithNaNInf),
+    Value r = arith::AddFOp::create(b,
+        arith::MulFOp::create(b, half, logOfSqrtPart, fmfWithNaNInf),
         logOfMaxAbsOfRealPlusOneAndImag, fmfWithNaNInf);
-    Value resultReal = b.create<arith::SelectOp>(
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
+    Value resultReal = arith::SelectOp::create(b,
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
         minAbs, r);
-    Value resultImag = b.create<math::Atan2Op>(imag, realPlusOne, fmf);
+    Value resultImag = math::Atan2Op::create(b, imag, realPlusOne, fmf);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -511,21 +511,21 @@ struct MulOpConversion : public OpConversionPattern<complex::MulOp> {
     auto elementType = cast<FloatType>(type.getElementType());
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     auto fmfValue = fmf.getValue();
-    Value lhsReal = b.create<complex::ReOp>(elementType, adaptor.getLhs());
-    Value lhsImag = b.create<complex::ImOp>(elementType, adaptor.getLhs());
-    Value rhsReal = b.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value rhsImag = b.create<complex::ImOp>(elementType, adaptor.getRhs());
+    Value lhsReal = complex::ReOp::create(b, elementType, adaptor.getLhs());
+    Value lhsImag = complex::ImOp::create(b, elementType, adaptor.getLhs());
+    Value rhsReal = complex::ReOp::create(b, elementType, adaptor.getRhs());
+    Value rhsImag = complex::ImOp::create(b, elementType, adaptor.getRhs());
     Value lhsRealTimesRhsReal =
-        b.create<arith::MulFOp>(lhsReal, rhsReal, fmfValue);
+        arith::MulFOp::create(b, lhsReal, rhsReal, fmfValue);
     Value lhsImagTimesRhsImag =
-        b.create<arith::MulFOp>(lhsImag, rhsImag, fmfValue);
-    Value real = b.create<arith::SubFOp>(lhsRealTimesRhsReal,
+        arith::MulFOp::create(b, lhsImag, rhsImag, fmfValue);
+    Value real = arith::SubFOp::create(b, lhsRealTimesRhsReal,
                                          lhsImagTimesRhsImag, fmfValue);
     Value lhsImagTimesRhsReal =
-        b.create<arith::MulFOp>(lhsImag, rhsReal, fmfValue);
+        arith::MulFOp::create(b, lhsImag, rhsReal, fmfValue);
     Value lhsRealTimesRhsImag =
-        b.create<arith::MulFOp>(lhsReal, rhsImag, fmfValue);
-    Value imag = b.create<arith::AddFOp>(lhsImagTimesRhsReal,
+        arith::MulFOp::create(b, lhsReal, rhsImag, fmfValue);
+    Value imag = arith::AddFOp::create(b, lhsImagTimesRhsReal,
                                          lhsRealTimesRhsImag, fmfValue);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, imag);
     return success();
@@ -543,11 +543,11 @@ struct NegOpConversion : public OpConversionPattern<complex::NegOp> {
     auto elementType = cast<FloatType>(type.getElementType());
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negReal = rewriter.create<arith::NegFOp>(loc, real);
-    Value negImag = rewriter.create<arith::NegFOp>(loc, imag);
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value negReal = arith::NegFOp::create(rewriter, loc, real);
+    Value negImag = arith::NegFOp::create(rewriter, loc, imag);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, negReal, negImag);
     return success();
   }
@@ -570,11 +570,11 @@ struct SinOpConversion : public TrigonometricOpConversion<complex::SinOp> {
     //   Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x
     //   Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x
     Value sum =
-        rewriter.create<arith::AddFOp>(loc, scaledExp, reciprocalExp, fmf);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, sin, fmf);
+        arith::AddFOp::create(rewriter, loc, scaledExp, reciprocalExp, fmf);
+    Value resultReal = arith::MulFOp::create(rewriter, loc, sum, sin, fmf);
     Value diff =
-        rewriter.create<arith::SubFOp>(loc, scaledExp, reciprocalExp, fmf);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, cos, fmf);
+        arith::SubFOp::create(rewriter, loc, scaledExp, reciprocalExp, fmf);
+    Value resultImag = arith::MulFOp::create(rewriter, loc, diff, cos, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -593,64 +593,64 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
-    Value half = b.create<arith::ConstantOp>(elementType,
+    Value half = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 0.5));
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value absSqrt = computeAbs(real, imag, fmf, b, AbsFn::sqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value sqrtArg = b.create<arith::MulFOp>(argArg, half, fmf);
-    Value cos = b.create<math::CosOp>(sqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(sqrtArg, fmf);
+    Value argArg = math::Atan2Op::create(b, imag, real, fmf);
+    Value sqrtArg = arith::MulFOp::create(b, argArg, half, fmf);
+    Value cos = math::CosOp::create(b, sqrtArg, fmf);
+    Value sin = math::SinOp::create(b, sqrtArg, fmf);
     // sin(atan2(0, inf)) = 0, sqrt(abs(inf)) = inf, but we can't multiply
     // 0 * inf.
     Value sinIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, sin, zero, fmf);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, sin, zero, fmf);
 
-    Value resultReal = b.create<arith::MulFOp>(absSqrt, cos, fmf);
-    Value resultImag = b.create<arith::SelectOp>(
-        sinIsZero, zero, b.create<arith::MulFOp>(absSqrt, sin, fmf));
+    Value resultReal = arith::MulFOp::create(b, absSqrt, cos, fmf);
+    Value resultImag = arith::SelectOp::create(b,
+        sinIsZero, zero, arith::MulFOp::create(b, absSqrt, sin, fmf));
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
       Value inf = cst(APFloat::getInf(floatSemantics));
       Value negInf = cst(APFloat::getInf(floatSemantics, true));
       Value nan = cst(APFloat::getNaN(floatSemantics));
-      Value absImag = b.create<math::AbsFOp>(elementType, imag, fmf);
+      Value absImag = math::AbsFOp::create(b, elementType, imag, fmf);
 
       Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
       Value absImagIsNotInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::ONE, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::ONE, absImag, inf, fmf);
       Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, inf, fmf);
       Value realIsNegInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, negInf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, negInf, fmf);
 
-      resultReal = b.create<arith::SelectOp>(
-          b.create<arith::AndIOp>(realIsNegInf, absImagIsNotInf), zero,
+      resultReal = arith::SelectOp::create(b,
+          arith::AndIOp::create(b, realIsNegInf, absImagIsNotInf), zero,
           resultReal);
-      resultReal = b.create<arith::SelectOp>(
-          b.create<arith::OrIOp>(absImagIsInf, realIsInf), inf, resultReal);
+      resultReal = arith::SelectOp::create(b,
+          arith::OrIOp::create(b, absImagIsInf, realIsInf), inf, resultReal);
 
-      Value imagSignInf = b.create<math::CopySignOp>(inf, imag, fmf);
-      resultImag = b.create<arith::SelectOp>(
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, absSqrt, absSqrt),
+      Value imagSignInf = math::CopySignOp::create(b, inf, imag, fmf);
+      resultImag = arith::SelectOp::create(b,
+          arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, absSqrt, absSqrt),
           nan, resultImag);
-      resultImag = b.create<arith::SelectOp>(
-          b.create<arith::OrIOp>(absImagIsInf, realIsNegInf), imagSignInf,
+      resultImag = arith::SelectOp::create(b,
+          arith::OrIOp::create(b, absImagIsInf, realIsNegInf), imagSignInf,
           resultImag);
     }
 
     Value resultIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
-    resultReal = b.create<arith::SelectOp>(resultIsZero, zero, resultReal);
-    resultImag = b.create<arith::SelectOp>(resultIsZero, zero, resultImag);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
+    resultReal = arith::SelectOp::create(b, resultIsZero, zero, resultReal);
+    resultImag = arith::SelectOp::create(b, resultIsZero, zero, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -669,19 +669,19 @@ struct SignOpConversion : public OpConversionPattern<complex::SignOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
+        arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
     Value realIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, zero);
     Value imagIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero);
-    Value isZero = b.create<arith::AndIOp>(realIsZero, imagIsZero);
-    auto abs = b.create<complex::AbsOp>(elementType, adaptor.getComplex(), fmf);
-    Value realSign = b.create<arith::DivFOp>(real, abs, fmf);
-    Value imagSign = b.create<arith::DivFOp>(imag, abs, fmf);
-    Value sign = b.create<complex::CreateOp>(type, realSign, imagSign);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero);
+    Value isZero = arith::AndIOp::create(b, realIsZero, imagIsZero);
+    auto abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(), fmf);
+    Value realSign = arith::DivFOp::create(b, real, abs, fmf);
+    Value imagSign = arith::DivFOp::create(b, imag, abs, fmf);
+    Value sign = complex::CreateOp::create(b, type, realSign, imagSign);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isZero,
                                                  adaptor.getComplex(), sign);
     return success();
@@ -703,84 +703,84 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
     const auto &floatSemantics = elementType.getFloatSemantics();
 
     Value real =
-        b.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(b, loc, elementType, adaptor.getComplex());
     Value imag =
-        b.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negOne = b.create<arith::ConstantOp>(
+        complex::ImOp::create(b, loc, elementType, adaptor.getComplex());
+    Value negOne = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -1.0));
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
       std::swap(real, imag);
-      real = b.create<arith::MulFOp>(real, negOne, fmf);
+      real = arith::MulFOp::create(b, real, negOne, fmf);
     }
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value four = b.create<arith::ConstantOp>(elementType,
+    Value four = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 4.0));
-    Value twoReal = b.create<arith::AddFOp>(real, real, fmf);
-    Value negTwoReal = b.create<arith::MulFOp>(negOne, twoReal, fmf);
+    Value twoReal = arith::AddFOp::create(b, real, real, fmf);
+    Value negTwoReal = arith::MulFOp::create(b, negOne, twoReal, fmf);
 
-    Value expTwoRealMinusOne = b.create<math::ExpM1Op>(twoReal, fmf);
-    Value expNegTwoRealMinusOne = b.create<math::ExpM1Op>(negTwoReal, fmf);
+    Value expTwoRealMinusOne = math::ExpM1Op::create(b, twoReal, fmf);
+    Value expNegTwoRealMinusOne = math::ExpM1Op::create(b, negTwoReal, fmf);
     Value realNum =
-        b.create<arith::SubFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+        arith::SubFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
 
-    Value cosImag = b.create<math::CosOp>(imag, fmf);
-    Value cosImagSq = b.create<arith::MulFOp>(cosImag, cosImag, fmf);
-    Value twoCosTwoImagPlusOne = b.create<arith::MulFOp>(cosImagSq, four, fmf);
-    Value sinImag = b.create<math::SinOp>(imag, fmf);
+    Value cosImag = math::CosOp::create(b, imag, fmf);
+    Value cosImagSq = arith::MulFOp::create(b, cosImag, cosImag, fmf);
+    Value twoCosTwoImagPlusOne = arith::MulFOp::create(b, cosImagSq, four, fmf);
+    Value sinImag = math::SinOp::create(b, imag, fmf);
 
-    Value imagNum = b.create<arith::MulFOp>(
-        four, b.create<arith::MulFOp>(cosImag, sinImag, fmf), fmf);
+    Value imagNum = arith::MulFOp::create(b,
+        four, arith::MulFOp::create(b, cosImag, sinImag, fmf), fmf);
 
     Value expSumMinusTwo =
-        b.create<arith::AddFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+        arith::AddFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
     Value denom =
-        b.create<arith::AddFOp>(expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
+        arith::AddFOp::create(b, expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
 
-    Value isInf = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+    Value isInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
                                           expSumMinusTwo, inf, fmf);
-    Value realLimit = b.create<math::CopySignOp>(negOne, real, fmf);
+    Value realLimit = math::CopySignOp::create(b, negOne, real, fmf);
 
-    Value resultReal = b.create<arith::SelectOp>(
-        isInf, realLimit, b.create<arith::DivFOp>(realNum, denom, fmf));
-    Value resultImag = b.create<arith::DivFOp>(imagNum, denom, fmf);
+    Value resultReal = arith::SelectOp::create(b,
+        isInf, realLimit, arith::DivFOp::create(b, realNum, denom, fmf));
+    Value resultImag = arith::DivFOp::create(b, imagNum, denom, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value zero = b.create<arith::ConstantOp>(
+      Value absReal = math::AbsFOp::create(b, real, fmf);
+      Value zero = arith::ConstantOp::create(b,
           elementType, b.getFloatAttr(elementType, 0.0));
       Value nan = cst(APFloat::getNaN(floatSemantics));
 
       Value absRealIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
       Value imagIsZero =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-      Value absRealIsNotInf = b.create<arith::XOrIOp>(
-          absRealIsInf, b.create<arith::ConstantIntOp>(true, /*width=*/1));
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
+      Value absRealIsNotInf = arith::XOrIOp::create(b,
+          absRealIsInf, arith::ConstantIntOp::create(b, true, /*width=*/1));
 
-      Value imagNumIsNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO,
+      Value imagNumIsNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO,
                                                    imagNum, imagNum, fmf);
       Value resultRealIsNaN =
-          b.create<arith::AndIOp>(imagNumIsNaN, absRealIsNotInf);
-      Value resultImagIsZero = b.create<arith::OrIOp>(
-          imagIsZero, b.create<arith::AndIOp>(absRealIsInf, imagNumIsNaN));
+          arith::AndIOp::create(b, imagNumIsNaN, absRealIsNotInf);
+      Value resultImagIsZero = arith::OrIOp::create(b,
+          imagIsZero, arith::AndIOp::create(b, absRealIsInf, imagNumIsNaN));
 
-      resultReal = b.create<arith::SelectOp>(resultRealIsNaN, nan, resultReal);
+      resultReal = arith::SelectOp::create(b, resultRealIsNaN, nan, resultReal);
       resultImag =
-          b.create<arith::SelectOp>(resultImagIsZero, zero, resultImag);
+          arith::SelectOp::create(b, resultImagIsZero, zero, resultImag);
     }
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
       std::swap(resultReal, resultImag);
-      resultImag = b.create<arith::MulFOp>(resultImag, negOne, fmf);
+      resultImag = arith::MulFOp::create(b, resultImag, negOne, fmf);
     }
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
@@ -799,10 +799,10 @@ struct ConjOpConversion : public OpConversionPattern<complex::ConjOp> {
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negImag = rewriter.create<arith::NegFOp>(loc, elementType, imag);
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value negImag = arith::NegFOp::create(rewriter, loc, elementType, imag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, negImag);
 
@@ -818,97 +818,97 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
                                  arith::FastMathFlags fmf) {
   auto elementType = cast<FloatType>(type.getElementType());
 
-  Value a = builder.create<complex::ReOp>(lhs);
-  Value b = builder.create<complex::ImOp>(lhs);
+  Value a = complex::ReOp::create(builder, lhs);
+  Value b = complex::ImOp::create(builder, lhs);
 
-  Value abs = builder.create<complex::AbsOp>(lhs, fmf);
-  Value absToC = builder.create<math::PowFOp>(abs, c, fmf);
+  Value abs = complex::AbsOp::create(builder, lhs, fmf);
+  Value absToC = math::PowFOp::create(builder, abs, c, fmf);
 
-  Value negD = builder.create<arith::NegFOp>(d, fmf);
-  Value argLhs = builder.create<math::Atan2Op>(b, a, fmf);
-  Value negDArgLhs = builder.create<arith::MulFOp>(negD, argLhs, fmf);
-  Value expNegDArgLhs = builder.create<math::ExpOp>(negDArgLhs, fmf);
+  Value negD = arith::NegFOp::create(builder, d, fmf);
+  Value argLhs = math::Atan2Op::create(builder, b, a, fmf);
+  Value negDArgLhs = arith::MulFOp::create(builder, negD, argLhs, fmf);
+  Value expNegDArgLhs = math::ExpOp::create(builder, negDArgLhs, fmf);
 
-  Value coeff = builder.create<arith::MulFOp>(absToC, expNegDArgLhs, fmf);
-  Value lnAbs = builder.create<math::LogOp>(abs, fmf);
-  Value cArgLhs = builder.create<arith::MulFOp>(c, argLhs, fmf);
-  Value dLnAbs = builder.create<arith::MulFOp>(d, lnAbs, fmf);
-  Value q = builder.create<arith::AddFOp>(cArgLhs, dLnAbs, fmf);
-  Value cosQ = builder.create<math::CosOp>(q, fmf);
-  Value sinQ = builder.create<math::SinOp>(q, fmf);
+  Value coeff = arith::MulFOp::create(builder, absToC, expNegDArgLhs, fmf);
+  Value lnAbs = math::LogOp::create(builder, abs, fmf);
+  Value cArgLhs = arith::MulFOp::create(builder, c, argLhs, fmf);
+  Value dLnAbs = arith::MulFOp::create(builder, d, lnAbs, fmf);
+  Value q = arith::AddFOp::create(builder, cArgLhs, dLnAbs, fmf);
+  Value cosQ = math::CosOp::create(builder, q, fmf);
+  Value sinQ = math::SinOp::create(builder, q, fmf);
 
-  Value inf = builder.create<arith::ConstantOp>(
+  Value inf = arith::ConstantOp::create(builder,
       elementType,
       builder.getFloatAttr(elementType,
                            APFloat::getInf(elementType.getFloatSemantics())));
-  Value zero = builder.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(builder,
       elementType, builder.getFloatAttr(elementType, 0.0));
-  Value one = builder.create<arith::ConstantOp>(
+  Value one = arith::ConstantOp::create(builder,
       elementType, builder.getFloatAttr(elementType, 1.0));
-  Value complexOne = builder.create<complex::CreateOp>(type, one, zero);
-  Value complexZero = builder.create<complex::CreateOp>(type, zero, zero);
-  Value complexInf = builder.create<complex::CreateOp>(type, inf, zero);
+  Value complexOne = complex::CreateOp::create(builder, type, one, zero);
+  Value complexZero = complex::CreateOp::create(builder, type, zero, zero);
+  Value complexInf = complex::CreateOp::create(builder, type, inf, zero);
 
   // Case 0:
   // d^c is 0 if d is 0 and c > 0. 0^0 is defined to be 1.0, see
   // Branch Cuts for Complex Elementary Functions or Much Ado About
   // Nothing's Sign Bit, W. Kahan, Section 10.
   Value absEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, abs, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, abs, zero, fmf);
   Value dEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, d, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, d, zero, fmf);
   Value cEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, c, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, c, zero, fmf);
   Value bEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, b, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, b, zero, fmf);
 
   Value zeroLeC =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLE, zero, c, fmf);
-  Value coeffCosQ = builder.create<arith::MulFOp>(coeff, cosQ, fmf);
-  Value coeffSinQ = builder.create<arith::MulFOp>(coeff, sinQ, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLE, zero, c, fmf);
+  Value coeffCosQ = arith::MulFOp::create(builder, coeff, cosQ, fmf);
+  Value coeffSinQ = arith::MulFOp::create(builder, coeff, sinQ, fmf);
   Value complexOneOrZero =
-      builder.create<arith::SelectOp>(cEqZero, complexOne, complexZero);
+      arith::SelectOp::create(builder, cEqZero, complexOne, complexZero);
   Value coeffCosSin =
-      builder.create<complex::CreateOp>(type, coeffCosQ, coeffSinQ);
-  Value cutoff0 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(
-          builder.create<arith::AndIOp>(absEqZero, dEqZero), zeroLeC),
+      complex::CreateOp::create(builder, type, coeffCosQ, coeffSinQ);
+  Value cutoff0 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder,
+          arith::AndIOp::create(builder, absEqZero, dEqZero), zeroLeC),
       complexOneOrZero, coeffCosSin);
 
   // Case 1:
   // x^0 is defined to be 1 for any x, see
   // Branch Cuts for Complex Elementary Functions or Much Ado About
   // Nothing's Sign Bit, W. Kahan, Section 10.
-  Value rhsEqZero = builder.create<arith::AndIOp>(cEqZero, dEqZero);
+  Value rhsEqZero = arith::AndIOp::create(builder, cEqZero, dEqZero);
   Value cutoff1 =
-      builder.create<arith::SelectOp>(rhsEqZero, complexOne, cutoff0);
+      arith::SelectOp::create(builder, rhsEqZero, complexOne, cutoff0);
 
   // Case 2:
   // 1^(c + d*i) = 1 + 0*i
-  Value lhsEqOne = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, one, fmf),
+  Value lhsEqOne = arith::AndIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, one, fmf),
       bEqZero);
   Value cutoff2 =
-      builder.create<arith::SelectOp>(lhsEqOne, complexOne, cutoff1);
+      arith::SelectOp::create(builder, lhsEqOne, complexOne, cutoff1);
 
   // Case 3:
   // inf^(c + 0*i) = inf + 0*i, c > 0
-  Value lhsEqInf = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, inf, fmf),
+  Value lhsEqInf = arith::AndIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, inf, fmf),
       bEqZero);
-  Value rhsGt0 = builder.create<arith::AndIOp>(
+  Value rhsGt0 = arith::AndIOp::create(builder,
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, c, zero, fmf));
-  Value cutoff3 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(lhsEqInf, rhsGt0), complexInf, cutoff2);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, c, zero, fmf));
+  Value cutoff3 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder, lhsEqInf, rhsGt0), complexInf, cutoff2);
 
   // Case 4:
   // inf^(c + 0*i) = 0 + 0*i, c < 0
-  Value rhsLt0 = builder.create<arith::AndIOp>(
+  Value rhsLt0 = arith::AndIOp::create(builder,
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, c, zero, fmf));
-  Value cutoff4 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(lhsEqInf, rhsLt0), complexZero, cutoff3);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, c, zero, fmf));
+  Value cutoff4 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder, lhsEqInf, rhsLt0), complexZero, cutoff3);
 
   return cutoff4;
 }
@@ -923,8 +923,8 @@ struct PowOpConversion : public OpConversionPattern<complex::PowOp> {
     auto type = cast<ComplexType>(adaptor.getLhs().getType());
     auto elementType = cast<FloatType>(type.getElementType());
 
-    Value c = builder.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value d = builder.create<complex::ImOp>(elementType, adaptor.getRhs());
+    Value c = complex::ReOp::create(builder, elementType, adaptor.getRhs());
+    Value d = complex::ImOp::create(builder, elementType, adaptor.getRhs());
 
     rewriter.replaceOp(op, {powOpConversionImpl(builder, type, adaptor.getLhs(),
                                                 c, d, op.getFastmath())});
@@ -945,64 +945,64 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value negHalf = b.create<arith::ConstantOp>(
+    Value negHalf = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -0.5));
     Value nan = cst(APFloat::getNaN(floatSemantics));
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value absRsqrt = computeAbs(real, imag, fmf, b, AbsFn::rsqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value rsqrtArg = b.create<arith::MulFOp>(argArg, negHalf, fmf);
-    Value cos = b.create<math::CosOp>(rsqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(rsqrtArg, fmf);
+    Value argArg = math::Atan2Op::create(b, imag, real, fmf);
+    Value rsqrtArg = arith::MulFOp::create(b, argArg, negHalf, fmf);
+    Value cos = math::CosOp::create(b, rsqrtArg, fmf);
+    Value sin = math::SinOp::create(b, rsqrtArg, fmf);
 
-    Value resultReal = b.create<arith::MulFOp>(absRsqrt, cos, fmf);
-    Value resultImag = b.create<arith::MulFOp>(absRsqrt, sin, fmf);
+    Value resultReal = arith::MulFOp::create(b, absRsqrt, cos, fmf);
+    Value resultImag = arith::MulFOp::create(b, absRsqrt, sin, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value negOne = b.create<arith::ConstantOp>(
+      Value negOne = arith::ConstantOp::create(b,
           elementType, b.getFloatAttr(elementType, -1));
 
-      Value realSignedZero = b.create<math::CopySignOp>(zero, real, fmf);
-      Value imagSignedZero = b.create<math::CopySignOp>(zero, imag, fmf);
+      Value realSignedZero = math::CopySignOp::create(b, zero, real, fmf);
+      Value imagSignedZero = math::CopySignOp::create(b, zero, imag, fmf);
       Value negImagSignedZero =
-          b.create<arith::MulFOp>(negOne, imagSignedZero, fmf);
+          arith::MulFOp::create(b, negOne, imagSignedZero, fmf);
 
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value absImag = b.create<math::AbsFOp>(imag, fmf);
+      Value absReal = math::AbsFOp::create(b, real, fmf);
+      Value absImag = math::AbsFOp::create(b, imag, fmf);
 
       Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
       Value realIsNan =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, real, real, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, real, real, fmf);
       Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
-      Value inIsNanInf = b.create<arith::AndIOp>(absImagIsInf, realIsNan);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value inIsNanInf = arith::AndIOp::create(b, absImagIsInf, realIsNan);
 
-      Value resultIsZero = b.create<arith::OrIOp>(inIsNanInf, realIsInf);
+      Value resultIsZero = arith::OrIOp::create(b, inIsNanInf, realIsInf);
 
       resultReal =
-          b.create<arith::SelectOp>(resultIsZero, realSignedZero, resultReal);
-      resultImag = b.create<arith::SelectOp>(resultIsZero, negImagSignedZero,
+          arith::SelectOp::create(b, resultIsZero, realSignedZero, resultReal);
+      resultImag = arith::SelectOp::create(b, resultIsZero, negImagSignedZero,
                                              resultImag);
     }
 
     Value isRealZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero, fmf);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, zero, fmf);
     Value isImagZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-    Value isZero = b.create<arith::AndIOp>(isRealZero, isImagZero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
+    Value isZero = arith::AndIOp::create(b, isRealZero, isImagZero);
 
-    resultReal = b.create<arith::SelectOp>(isZero, inf, resultReal);
-    resultImag = b.create<arith::SelectOp>(isZero, nan, resultImag);
+    resultReal = arith::SelectOp::create(b, isZero, inf, resultReal);
+    resultImag = arith::SelectOp::create(b, isZero, nan, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -1021,9 +1021,9 @@ struct AngleOpConversion : public OpConversionPattern<complex::AngleOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, type, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, type, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, type, adaptor.getComplex());
+        complex::ImOp::create(rewriter, loc, type, adaptor.getComplex());
 
     rewriter.replaceOpWithNewOp<math::Atan2Op>(op, imag, real, fmf);
 
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index 13a084407e53f..5c3d26874faed 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -73,13 +73,13 @@ struct AssertOpLowering : public ConvertOpToLLVMPattern<cf::AssertOp> {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
         auto abortFuncTy = LLVM::LLVMFunctionType::get(getVoidType(), {});
-        abortFunc = rewriter.create<LLVM::LLVMFuncOp>(rewriter.getUnknownLoc(),
+        abortFunc = LLVM::LLVMFuncOp::create(rewriter, rewriter.getUnknownLoc(),
                                                       "abort", abortFuncTy);
       }
-      rewriter.create<LLVM::CallOp>(loc, abortFunc, ValueRange());
-      rewriter.create<LLVM::UnreachableOp>(loc);
+      LLVM::CallOp::create(rewriter, loc, abortFunc, ValueRange());
+      LLVM::UnreachableOp::create(rewriter, loc);
     } else {
-      rewriter.create<LLVM::BrOp>(loc, ValueRange(), continuationBlock);
+      LLVM::BrOp::create(rewriter, loc, ValueRange(), continuationBlock);
     }
 
     // Generate assertion test.
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index 9831dcaaaccc8..aa468c29536c7 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -33,7 +33,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
     MutableArrayRef<Region> regions) {
   if (auto condBrOp = dyn_cast<cf::CondBranchOp>(controlFlowCondOp)) {
     assert(regions.size() == 2);
-    auto ifOp = builder.create<scf::IfOp>(controlFlowCondOp->getLoc(),
+    auto ifOp = scf::IfOp::create(builder, controlFlowCondOp->getLoc(),
                                           resultTypes, condBrOp.getCondition());
     ifOp.getThenRegion().takeBody(regions[0]);
     ifOp.getElseRegion().takeBody(regions[1]);
@@ -43,7 +43,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
   if (auto switchOp = dyn_cast<cf::SwitchOp>(controlFlowCondOp)) {
     // `getCFGSwitchValue` returns an i32 that we need to convert to index
     // fist.
-    auto cast = builder.create<arith::IndexCastUIOp>(
+    auto cast = arith::IndexCastUIOp::create(builder,
         controlFlowCondOp->getLoc(), builder.getIndexType(),
         switchOp.getFlag());
     SmallVector<int64_t> cases;
@@ -55,7 +55,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
 
     assert(regions.size() == cases.size() + 1);
 
-    auto indexSwitchOp = builder.create<scf::IndexSwitchOp>(
+    auto indexSwitchOp = scf::IndexSwitchOp::create(builder,
         controlFlowCondOp->getLoc(), resultTypes, cast, cases, cases.size());
 
     indexSwitchOp.getDefaultRegion().takeBody(regions[0]);
@@ -75,7 +75,7 @@ LogicalResult
 ControlFlowToSCFTransformation::createStructuredBranchRegionTerminatorOp(
     Location loc, OpBuilder &builder, Operation *branchRegionOp,
     Operation *replacedControlFlowOp, ValueRange results) {
-  builder.create<scf::YieldOp>(loc, results);
+  scf::YieldOp::create(builder, loc, results);
   return success();
 }
 
@@ -84,7 +84,7 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
     OpBuilder &builder, Operation *replacedOp, ValueRange loopVariablesInit,
     Value condition, ValueRange loopVariablesNextIter, Region &&loopBody) {
   Location loc = replacedOp->getLoc();
-  auto whileOp = builder.create<scf::WhileOp>(loc, loopVariablesInit.getTypes(),
+  auto whileOp = scf::WhileOp::create(builder, loc, loopVariablesInit.getTypes(),
                                               loopVariablesInit);
 
   whileOp.getBefore().takeBody(loopBody);
@@ -92,15 +92,15 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
   builder.setInsertionPointToEnd(&whileOp.getBefore().back());
   // `getCFGSwitchValue` returns a i32. We therefore need to truncate the
   // condition to i1 first. It is guaranteed to be either 0 or 1 already.
-  builder.create<scf::ConditionOp>(
-      loc, builder.create<arith::TruncIOp>(loc, builder.getI1Type(), condition),
+  scf::ConditionOp::create(builder,
+      loc, arith::TruncIOp::create(builder, loc, builder.getI1Type(), condition),
       loopVariablesNextIter);
 
   Block *afterBlock = builder.createBlock(&whileOp.getAfter());
   afterBlock->addArguments(
       loopVariablesInit.getTypes(),
       SmallVector<Location>(loopVariablesInit.size(), loc));
-  builder.create<scf::YieldOp>(loc, afterBlock->getArguments());
+  scf::YieldOp::create(builder, loc, afterBlock->getArguments());
 
   return whileOp.getOperation();
 }
@@ -108,7 +108,7 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
 Value ControlFlowToSCFTransformation::getCFGSwitchValue(Location loc,
                                                         OpBuilder &builder,
                                                         unsigned int value) {
-  return builder.create<arith::ConstantOp>(loc,
+  return arith::ConstantOp::create(builder, loc,
                                            builder.getI32IntegerAttr(value));
 }
 
@@ -117,7 +117,7 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
     ArrayRef<unsigned int> caseValues, BlockRange caseDestinations,
     ArrayRef<ValueRange> caseArguments, Block *defaultDest,
     ValueRange defaultArgs) {
-  builder.create<cf::SwitchOp>(loc, flag, defaultDest, defaultArgs,
+  cf::SwitchOp::create(builder, loc, flag, defaultDest, defaultArgs,
                                llvm::to_vector_of<int32_t>(caseValues),
                                caseDestinations, caseArguments);
 }
@@ -125,7 +125,7 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
 Value ControlFlowToSCFTransformation::getUndefValue(Location loc,
                                                     OpBuilder &builder,
                                                     Type type) {
-  return builder.create<ub::PoisonOp>(loc, type, nullptr);
+  return ub::PoisonOp::create(builder, loc, type, nullptr);
 }
 
 FailureOr<Operation *>
diff --git a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
index f8dc06f41ab87..20167f168185a 100644
--- a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
+++ b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
@@ -99,7 +99,7 @@ class FuncOpConversion final : public OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted `emitc.func` op.
-    emitc::FuncOp newFuncOp = rewriter.create<emitc::FuncOp>(
+    emitc::FuncOp newFuncOp = emitc::FuncOp::create(rewriter,
         funcOp.getLoc(), funcOp.getName(),
         FunctionType::get(rewriter.getContext(),
                           signatureConverter.getConvertedTypes(),
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 36235636d6ba2..51dbe6cbb7de3 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -115,7 +115,7 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   SmallVector<NamedAttribute> attributes;
   filterFuncAttributes(funcOp, attributes);
 
-  auto wrapperFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto wrapperFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperFuncType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
@@ -129,13 +129,13 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   for (auto [index, argType] : llvm::enumerate(type.getInputs())) {
     Value arg = wrapperFuncOp.getArgument(index + argOffset);
     if (auto memrefType = dyn_cast<MemRefType>(argType)) {
-      Value loaded = rewriter.create<LLVM::LoadOp>(
+      Value loaded = LLVM::LoadOp::create(rewriter,
           loc, typeConverter.convertType(memrefType), arg);
       MemRefDescriptor::unpack(rewriter, loc, loaded, memrefType, args);
       continue;
     }
     if (isa<UnrankedMemRefType>(argType)) {
-      Value loaded = rewriter.create<LLVM::LoadOp>(
+      Value loaded = LLVM::LoadOp::create(rewriter,
           loc, typeConverter.convertType(argType), arg);
       UnrankedMemRefDescriptor::unpack(rewriter, loc, loaded, args);
       continue;
@@ -144,14 +144,14 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
     args.push_back(arg);
   }
 
-  auto call = rewriter.create<LLVM::CallOp>(loc, newFuncOp, args);
+  auto call = LLVM::CallOp::create(rewriter, loc, newFuncOp, args);
 
   if (resultStructType) {
-    rewriter.create<LLVM::StoreOp>(loc, call.getResult(),
+    LLVM::StoreOp::create(rewriter, loc, call.getResult(),
                                    wrapperFuncOp.getArgument(0));
-    rewriter.create<LLVM::ReturnOp>(loc, ValueRange{});
+    LLVM::ReturnOp::create(rewriter, loc, ValueRange{});
   } else {
-    rewriter.create<LLVM::ReturnOp>(loc, call.getResults());
+    LLVM::ReturnOp::create(rewriter, loc, call.getResults());
   }
 }
 
@@ -182,7 +182,7 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   filterFuncAttributes(funcOp, attributes);
 
   // Create the auxiliary function.
-  auto wrapperFunc = builder.create<LLVM::LLVMFuncOp>(
+  auto wrapperFunc = LLVM::LLVMFuncOp::create(builder,
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
@@ -201,11 +201,11 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   if (resultStructType) {
     // Allocate the struct on the stack and pass the pointer.
     Type resultType = cast<LLVM::LLVMFunctionType>(wrapperType).getParamType(0);
-    Value one = builder.create<LLVM::ConstantOp>(
+    Value one = LLVM::ConstantOp::create(builder,
         loc, typeConverter.convertType(builder.getIndexType()),
         builder.getIntegerAttr(builder.getIndexType(), 1));
     Value result =
-        builder.create<LLVM::AllocaOp>(loc, resultType, resultStructType, one);
+        LLVM::AllocaOp::create(builder, loc, resultType, resultStructType, one);
     args.push_back(result);
   }
 
@@ -229,12 +229,12 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
                     wrapperArgsRange.take_front(numToDrop));
 
       auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext());
-      Value one = builder.create<LLVM::ConstantOp>(
+      Value one = LLVM::ConstantOp::create(builder,
           loc, typeConverter.convertType(builder.getIndexType()),
           builder.getIntegerAttr(builder.getIndexType(), 1));
-      Value allocated = builder.create<LLVM::AllocaOp>(
+      Value allocated = LLVM::AllocaOp::create(builder,
           loc, ptrTy, packed.getType(), one, /*alignment=*/0);
-      builder.create<LLVM::StoreOp>(loc, packed, allocated);
+      LLVM::StoreOp::create(builder, loc, packed, allocated);
       arg = allocated;
     } else {
       arg = wrapperArgsRange[0];
@@ -245,14 +245,14 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   }
   assert(wrapperArgsRange.empty() && "did not map some of the arguments");
 
-  auto call = builder.create<LLVM::CallOp>(loc, wrapperFunc, args);
+  auto call = LLVM::CallOp::create(builder, loc, wrapperFunc, args);
 
   if (resultStructType) {
     Value result =
-        builder.create<LLVM::LoadOp>(loc, resultStructType, args.front());
-    builder.create<LLVM::ReturnOp>(loc, result);
+        LLVM::LoadOp::create(builder, loc, resultStructType, args.front());
+    LLVM::ReturnOp::create(builder, loc, result);
   } else {
-    builder.create<LLVM::ReturnOp>(loc, call.getResults());
+    LLVM::ReturnOp::create(builder, loc, call.getResults());
   }
 }
 
@@ -283,7 +283,7 @@ static void restoreByValRefArgumentType(
     Type resTy = typeConverter.convertType(
         cast<TypeAttr>(byValRefAttr->getValue()).getValue());
 
-    Value valueArg = rewriter.create<LLVM::LoadOp>(arg.getLoc(), resTy, arg);
+    Value valueArg = LLVM::LoadOp::create(rewriter, arg.getLoc(), resTy, arg);
     rewriter.replaceUsesOfBlockArgument(arg, valueArg);
   }
 }
@@ -357,7 +357,7 @@ FailureOr<LLVM::LLVMFuncOp> mlir::convertFuncOpToLLVMFuncOp(
     symbolTable.remove(funcOp);
   }
 
-  auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
       /*dsoLocal=*/false, /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr,
       attributes);
@@ -509,7 +509,7 @@ struct ConstantOpLowering : public ConvertOpToLLVMPattern<func::ConstantOp> {
       return rewriter.notifyMatchFailure(op, "failed to convert result type");
 
     auto newOp =
-        rewriter.create<LLVM::AddressOfOp>(op.getLoc(), type, op.getValue());
+        LLVM::AddressOfOp::create(rewriter, op.getLoc(), type, op.getValue());
     for (const NamedAttribute &attr : op->getAttrs()) {
       if (attr.getName().strref() == "value")
         continue;
@@ -556,7 +556,7 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
     auto promoted = this->getTypeConverter()->promoteOperands(
         callOp.getLoc(), /*opOperands=*/callOp->getOperands(),
         adaptor.getOperands(), rewriter, useBarePtrCallConv);
-    auto newOp = rewriter.create<LLVM::CallOp>(
+    auto newOp = LLVM::CallOp::create(rewriter,
         callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
         promoted, callOp->getAttrs());
 
@@ -573,7 +573,7 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
       // Extract individual results from the structure and return them as list.
       results.reserve(numResults);
       for (unsigned i = 0; i < numResults; ++i) {
-        results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        results.push_back(LLVM::ExtractValueOp::create(rewriter,
             callOp.getLoc(), newOp->getResult(0), i));
       }
     }
@@ -726,9 +726,9 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<func::ReturnOp> {
       return rewriter.notifyMatchFailure(op, "could not convert result types");
     }
 
-    Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
+    Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
     for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
-      packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
+      packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
     }
     rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, TypeRange(), packed,
                                                 op->getAttrs());
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index 01ca5e99a9aff..f5d5460412da2 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -28,7 +28,7 @@ LLVM::LLVMFuncOp mlir::getOrDefineFunction(gpu::GPUModuleOp moduleOp,
   if (!(ret = moduleOp.template lookupSymbol<LLVM::LLVMFuncOp>(name))) {
     OpBuilder::InsertionGuard guard(b);
     b.setInsertionPointToStart(moduleOp.getBody());
-    ret = b.create<LLVM::LLVMFuncOp>(loc, name, type, LLVM::Linkage::External);
+    ret = LLVM::LLVMFuncOp::create(b, loc, name, type, LLVM::Linkage::External);
   }
   return ret;
 }
@@ -68,7 +68,7 @@ mlir::getOrCreateStringConstant(OpBuilder &b, Location loc,
   OpBuilder::InsertionGuard guard(b);
   b.setInsertionPointToStart(moduleOp.getBody());
   SmallString<16> name = getUniqueSymbolName(moduleOp, namePrefix);
-  return b.create<LLVM::GlobalOp>(loc, globalType,
+  return LLVM::GlobalOp::create(b, loc, globalType,
                                   /*isConstant=*/true, LLVM::Linkage::Internal,
                                   name, attr, alignment, addrSpace);
 }
@@ -151,7 +151,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
               gpuFuncOp.getWorkgroupAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      auto globalOp = rewriter.create<LLVM::GlobalOp>(
+      auto globalOp = LLVM::GlobalOp::create(rewriter,
           gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
           LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment,
           workgroupAddrSpace);
@@ -220,7 +220,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
   LLVM::CConv callingConvention = gpuFuncOp.isKernel()
                                       ? kernelCallingConvention
                                       : nonKernelCallingConvention;
-  auto llvmFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto llvmFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
       LLVM::Linkage::External, /*dsoLocal=*/false, callingConvention,
       /*comdat=*/nullptr, attributes);
@@ -266,10 +266,10 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       for (const auto [idx, global] : llvm::enumerate(workgroupBuffers)) {
         auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext(),
                                                   global.getAddrSpace());
-        Value address = rewriter.create<LLVM::AddressOfOp>(
+        Value address = LLVM::AddressOfOp::create(rewriter,
             loc, ptrType, global.getSymNameAttr());
         Value memory =
-            rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getType(),
+            LLVM::GEPOp::create(rewriter, loc, ptrType, global.getType(),
                                          address, ArrayRef<LLVM::GEPArg>{0, 0});
 
         // Build a memref descriptor pointing to the buffer to plug with the
@@ -298,14 +298,14 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       Type elementType = typeConverter->convertType(type.getElementType());
       auto ptrType =
           LLVM::LLVMPointerType::get(rewriter.getContext(), allocaAddrSpace);
-      Value numElements = rewriter.create<LLVM::ConstantOp>(
+      Value numElements = LLVM::ConstantOp::create(rewriter,
           gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
       uint64_t alignment = 0;
       if (auto alignAttr =
               dyn_cast_or_null<IntegerAttr>(gpuFuncOp.getPrivateAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      Value allocated = rewriter.create<LLVM::AllocaOp>(
+      Value allocated = LLVM::AllocaOp::create(rewriter,
           gpuFuncOp.getLoc(), ptrType, elementType, numElements, alignment);
       Value descr = MemRefDescriptor::fromStaticShape(
           rewriter, loc, *getTypeConverter(), type, allocated);
@@ -418,8 +418,8 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
           {llvmI64, ptrType, /*length (bytes)*/ llvmI64, /*isLast*/ llvmI32}));
 
   /// Start the printf hostcall
-  Value zeroI64 = rewriter.create<LLVM::ConstantOp>(loc, llvmI64, 0);
-  auto printfBeginCall = rewriter.create<LLVM::CallOp>(loc, ocklBegin, zeroI64);
+  Value zeroI64 = LLVM::ConstantOp::create(rewriter, loc, llvmI64, 0);
+  auto printfBeginCall = LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
   Value printfDesc = printfBeginCall.getResult();
 
   // Create the global op or find an existing one.
@@ -427,20 +427,20 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
       rewriter, loc, moduleOp, llvmI8, "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element and pass it to printf()
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
       loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
-  Value stringLen = rewriter.create<LLVM::ConstantOp>(
+  Value stringLen = LLVM::ConstantOp::create(rewriter,
       loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
 
-  Value oneI32 = rewriter.create<LLVM::ConstantOp>(loc, llvmI32, 1);
-  Value zeroI32 = rewriter.create<LLVM::ConstantOp>(loc, llvmI32, 0);
+  Value oneI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 1);
+  Value zeroI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 0);
 
-  auto appendFormatCall = rewriter.create<LLVM::CallOp>(
+  auto appendFormatCall = LLVM::CallOp::create(rewriter,
       loc, ocklAppendStringN,
       ValueRange{printfDesc, stringStart, stringLen,
                  adaptor.getArgs().empty() ? oneI32 : zeroI32});
@@ -456,17 +456,17 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
     SmallVector<mlir::Value, 2 + argsPerAppend + 1> arguments;
     arguments.push_back(printfDesc);
     arguments.push_back(
-        rewriter.create<LLVM::ConstantOp>(loc, llvmI32, numArgsThisCall));
+        LLVM::ConstantOp::create(rewriter, loc, llvmI32, numArgsThisCall));
     for (size_t i = group; i < bound; ++i) {
       Value arg = adaptor.getArgs()[i];
       if (auto floatType = dyn_cast<FloatType>(arg.getType())) {
         if (!floatType.isF64())
-          arg = rewriter.create<LLVM::FPExtOp>(
+          arg = LLVM::FPExtOp::create(rewriter,
               loc, typeConverter->convertType(rewriter.getF64Type()), arg);
-        arg = rewriter.create<LLVM::BitcastOp>(loc, llvmI64, arg);
+        arg = LLVM::BitcastOp::create(rewriter, loc, llvmI64, arg);
       }
       if (arg.getType().getIntOrFloatBitWidth() != 64)
-        arg = rewriter.create<LLVM::ZExtOp>(loc, llvmI64, arg);
+        arg = LLVM::ZExtOp::create(rewriter, loc, llvmI64, arg);
 
       arguments.push_back(arg);
     }
@@ -477,7 +477,7 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
 
     auto isLast = (bound == nArgs) ? oneI32 : zeroI32;
     arguments.push_back(isLast);
-    auto call = rewriter.create<LLVM::CallOp>(loc, ocklAppendArgs, arguments);
+    auto call = LLVM::CallOp::create(rewriter, loc, ocklAppendArgs, arguments);
     printfDesc = call.getResult();
   }
   rewriter.eraseOp(gpuPrintfOp);
@@ -510,12 +510,12 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
       /*alignment=*/0, addressSpace);
 
   // Get a pointer to the format string's first element
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
       loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
 
   // Construct arguments and function call
@@ -525,7 +525,7 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
   printfArgs.push_back(stringStart);
   printfArgs.append(argsRange.begin(), argsRange.end());
 
-  rewriter.create<LLVM::CallOp>(loc, printfDecl, printfArgs);
+  LLVM::CallOp::create(rewriter, loc, printfDecl, printfArgs);
   rewriter.eraseOp(gpuPrintfOp);
   return success();
 }
@@ -559,9 +559,9 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
                                 "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(loc, global);
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter, loc, global);
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
   SmallVector<Type> types;
   SmallVector<Value> args;
@@ -572,27 +572,27 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
     assert(type.isIntOrFloat());
     if (isa<FloatType>(type)) {
       type = rewriter.getF64Type();
-      promotedArg = rewriter.create<LLVM::FPExtOp>(loc, type, arg);
+      promotedArg = LLVM::FPExtOp::create(rewriter, loc, type, arg);
     }
     types.push_back(type);
     args.push_back(promotedArg);
   }
   Type structType =
       LLVM::LLVMStructType::getLiteral(gpuPrintfOp.getContext(), types);
-  Value one = rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
+  Value one = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
                                                 rewriter.getIndexAttr(1));
   Value tempAlloc =
-      rewriter.create<LLVM::AllocaOp>(loc, ptrType, structType, one,
+      LLVM::AllocaOp::create(rewriter, loc, ptrType, structType, one,
                                       /*alignment=*/0);
   for (auto [index, arg] : llvm::enumerate(args)) {
-    Value ptr = rewriter.create<LLVM::GEPOp>(
+    Value ptr = LLVM::GEPOp::create(rewriter,
         loc, ptrType, structType, tempAlloc,
         ArrayRef<LLVM::GEPArg>{0, static_cast<int32_t>(index)});
-    rewriter.create<LLVM::StoreOp>(loc, arg, ptr);
+    LLVM::StoreOp::create(rewriter, loc, arg, ptr);
   }
   std::array<Value, 2> printfArgs = {stringStart, tempAlloc};
 
-  rewriter.create<LLVM::CallOp>(loc, vprintfDecl, printfArgs);
+  LLVM::CallOp::create(rewriter, loc, vprintfDecl, printfArgs);
   rewriter.eraseOp(gpuPrintfOp);
   return success();
 }
@@ -607,22 +607,22 @@ static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands,
   TypeRange operandTypes(operands);
   VectorType vectorType = cast<VectorType>(llvm1DVectorTy);
   Location loc = op->getLoc();
-  Value result = rewriter.create<LLVM::PoisonOp>(loc, vectorType);
+  Value result = LLVM::PoisonOp::create(rewriter, loc, vectorType);
   Type indexType = converter.convertType(rewriter.getIndexType());
   StringAttr name = op->getName().getIdentifier();
   Type elementType = vectorType.getElementType();
 
   for (int64_t i = 0; i < vectorType.getNumElements(); ++i) {
-    Value index = rewriter.create<LLVM::ConstantOp>(loc, indexType, i);
+    Value index = LLVM::ConstantOp::create(rewriter, loc, indexType, i);
     auto extractElement = [&](Value operand) -> Value {
       if (!isa<VectorType>(operand.getType()))
         return operand;
-      return rewriter.create<LLVM::ExtractElementOp>(loc, operand, index);
+      return LLVM::ExtractElementOp::create(rewriter, loc, operand, index);
     };
     auto scalarOperands = llvm::map_to_vector(operands, extractElement);
     Operation *scalarOp =
         rewriter.create(loc, name, scalarOperands, elementType, op->getAttrs());
-    result = rewriter.create<LLVM::InsertElementOp>(
+    result = LLVM::InsertElementOp::create(rewriter,
         loc, result, scalarOp->getResult(0), index);
   }
   return result;
@@ -705,7 +705,7 @@ LLVM::GlobalOp getDynamicSharedMemorySymbol(
   auto zeroSizedArrayType = LLVM::LLVMArrayType::get(
       typeConverter->convertType(memrefType.getElementType()), 0);
 
-  return rewriter.create<LLVM::GlobalOp>(
+  return LLVM::GlobalOp::create(rewriter,
       op->getLoc(), zeroSizedArrayType, /*isConstant=*/false,
       LLVM::Linkage::Internal, symName, /*value=*/Attribute(), alignmentByte,
       addressSpace.value());
@@ -732,12 +732,12 @@ LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
   // Step 3. Get address of the global symbol
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(op);
-  auto basePtr = rewriter.create<LLVM::AddressOfOp>(loc, shmemOp);
+  auto basePtr = LLVM::AddressOfOp::create(rewriter, loc, shmemOp);
   Type baseType = basePtr->getResultTypes().front();
 
   // Step 4. Generate GEP using offsets
   SmallVector<LLVM::GEPArg> gepArgs = {0};
-  Value shmemPtr = rewriter.create<LLVM::GEPOp>(loc, baseType, elementType,
+  Value shmemPtr = LLVM::GEPOp::create(rewriter, loc, baseType, elementType,
                                                 basePtr, gepArgs);
   // Step 5. Create a memref descriptor
   SmallVector<Value> shape, strides;
@@ -799,9 +799,9 @@ LogicalResult GPUReturnOpLowering::matchAndRewrite(
     return rewriter.notifyMatchFailure(op, "could not convert result types");
   }
 
-  Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
+  Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
   for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
-    packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
+    packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
   }
   rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, TypeRange(), packed,
                                               op->getAttrs());
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 167cabbc57db9..5e8254a0b6035 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -79,7 +79,7 @@ class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern<OpTy> {
     uint64_t rank = type.getRank();
     Value numElements = desc.size(rewriter, loc, /*pos=*/0);
     for (unsigned i = 1; i < rank; i++)
-      numElements = rewriter.create<LLVM::MulOp>(
+      numElements = LLVM::MulOp::create(rewriter,
           loc, numElements, desc.size(rewriter, loc, /*pos=*/i));
     return numElements;
   }
@@ -582,7 +582,7 @@ LLVM::CallOp FunctionCallBuilder::create(Location loc, OpBuilder &builder,
     return OpBuilder::atBlockEnd(module.getBody())
         .create<LLVM::LLVMFuncOp>(loc, functionName, functionType);
   }();
-  return builder.create<LLVM::CallOp>(loc, function, arguments);
+  return LLVM::CallOp::create(builder, loc, function, arguments);
 }
 
 // Corresponding to cusparseIndexType_t defined in cusparse.h.
@@ -780,12 +780,12 @@ LogicalResult ConvertAllocOpToGpuRuntimeCallPattern::matchAndRewrite(
 
   // Allocate the underlying buffer and store a pointer to it in the MemRef
   // descriptor.
-  auto nullPtr = rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPointerType);
+  auto nullPtr = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPointerType);
   Value stream = adaptor.getAsyncDependencies().empty()
                      ? nullPtr
                      : adaptor.getAsyncDependencies().front();
 
-  auto isHostShared = rewriter.create<mlir::LLVM::ConstantOp>(
+  auto isHostShared = mlir::LLVM::ConstantOp::create(rewriter,
       loc, llvmInt8Type, rewriter.getI8IntegerAttr(isShared));
 
   Value allocatedPtr =
@@ -1012,7 +1012,7 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
       uint64_t staticSize = static_cast<uint64_t>(bitwidth / 8) *
                             static_cast<uint64_t>(memrefTy.getNumElements());
 
-      Value sizeArg = rewriter.create<LLVM::ConstantOp>(
+      Value sizeArg = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(staticSize));
       llvmArgumentsWithSizes.push_back(llvmArg); // Presumably a bare pointer.
       llvmArgumentsWithSizes.push_back(sizeArg);
@@ -1025,7 +1025,7 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
         gpu::KernelDim3{adaptor.getClusterSizeX(), adaptor.getClusterSizeY(),
                         adaptor.getClusterSizeZ()};
   }
-  rewriter.create<gpu::LaunchFuncOp>(
+  gpu::LaunchFuncOp::create(rewriter,
       launchOp.getLoc(), launchOp.getKernelAttr(),
       gpu::KernelDim3{adaptor.getGridSizeX(), adaptor.getGridSizeY(),
                       adaptor.getGridSizeZ()},
@@ -1048,7 +1048,7 @@ static Value bitAndAddrspaceCast(Location loc,
                                  const LLVMTypeConverter &typeConverter) {
   auto sourceTy = cast<LLVM::LLVMPointerType>(sourcePtr.getType());
   if (destinationType.getAddressSpace() != sourceTy.getAddressSpace())
-    sourcePtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+    sourcePtr = LLVM::AddrSpaceCastOp::create(rewriter,
         loc,
         LLVM::LLVMPointerType::get(rewriter.getContext(),
                                    destinationType.getAddressSpace()),
@@ -1072,13 +1072,13 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite(
   Value numElements = getNumElements(rewriter, loc, memRefType, srcDesc);
 
   Type elementPtrType = getElementPtrType(memRefType);
-  Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, elementPtrType);
-  Value gepPtr = rewriter.create<LLVM::GEPOp>(
+  Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
+  Value gepPtr = LLVM::GEPOp::create(rewriter,
       loc, elementPtrType,
       typeConverter->convertType(memRefType.getElementType()), nullPtr,
       numElements);
   auto sizeBytes =
-      rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);
+      LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
 
   auto src = bitAndAddrspaceCast(loc, rewriter, llvmPointerType,
                                  srcDesc.alignedPtr(rewriter, loc),
@@ -1123,7 +1123,7 @@ LogicalResult ConvertMemsetOpToGpuRuntimeCallPattern::matchAndRewrite(
   Value numElements = getNumElements(rewriter, loc, memRefType, dstDesc);
 
   auto value =
-      rewriter.create<LLVM::BitcastOp>(loc, bitCastType, adaptor.getValue());
+      LLVM::BitcastOp::create(rewriter, loc, bitCastType, adaptor.getValue());
   auto dst = bitAndAddrspaceCast(loc, rewriter, llvmPointerType,
                                  dstDesc.alignedPtr(rewriter, loc),
                                  *getTypeConverter());
@@ -1150,14 +1150,14 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite(
 template <typename T>
 static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmInt32Type = builder.getIntegerType(32);
-  return builder.create<LLVM::ConstantOp>(loc, llvmInt32Type,
+  return LLVM::ConstantOp::create(builder, loc, llvmInt32Type,
                                           static_cast<int32_t>(tValue));
 }
 
 template <typename T>
 static Value genConstFloat32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmFloat32Type = builder.getF32Type();
-  return builder.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(builder,
       loc, llvmFloat32Type,
       builder.getF32FloatAttr(static_cast<float>(tValue)));
 }
@@ -1189,11 +1189,11 @@ LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite(
   // the dnmat is used with spmat with 2:4 sparsity
   if (dims.size() == 2) {
     if (isSpMMCusparseLtOp(op.getDnTensor())) {
-      auto handleSz = rewriter.create<LLVM::ConstantOp>(
+      auto handleSz = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(11032));
-      handle = rewriter.create<LLVM::AllocaOp>(
+      handle = LLVM::AllocaOp::create(rewriter,
           loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
-      handle = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, handle);
+      handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
       createLtDnMatCallBuilder
           .create(loc, rewriter,
@@ -1351,11 +1351,11 @@ LogicalResult ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto dtp = genConstInt32From(rewriter, loc, getCuSparseDataTypeFrom(dType));
 
   // CUDA runner asserts the size is 44104 bytes.
-  auto handleSz = rewriter.create<LLVM::ConstantOp>(
+  auto handleSz = LLVM::ConstantOp::create(rewriter,
       loc, getIndexType(), rewriter.getIndexAttr(44104));
-  Value handle = rewriter.create<LLVM::AllocaOp>(
+  Value handle = LLVM::AllocaOp::create(rewriter,
       loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
-  handle = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, handle);
+  handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
   create2To4SpMatCallBuilder
       .create(loc, rewriter,
@@ -1441,9 +1441,9 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
         genConstInt32From(rewriter, loc, get2To4PruneFlag(op.getSpmatA()));
     auto computeType = genConstInt32From(
         rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType()));
-    auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+    auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(3));
-    auto bufferSize = rewriter.create<LLVM::AllocaOp>(
+    auto bufferSize = LLVM::AllocaOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, three, /*alignment=*/16);
     createCuSparseLtSpMMBufferSizeBuilder
         .create(loc, rewriter,
@@ -1452,20 +1452,20 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
                  pruneFlag, stream})
         .getResult();
 
-    auto bufferSizePtr1 = rewriter.create<LLVM::GEPOp>(
+    auto bufferSizePtr1 = LLVM::GEPOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{rewriter.create<LLVM::ConstantOp>(
+        ValueRange{LLVM::ConstantOp::create(rewriter,
             loc, getIndexType(), rewriter.getIndexAttr(1))});
-    auto bufferSizePtr2 = rewriter.create<LLVM::GEPOp>(
+    auto bufferSizePtr2 = LLVM::GEPOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{rewriter.create<LLVM::ConstantOp>(
+        ValueRange{LLVM::ConstantOp::create(rewriter,
             loc, getIndexType(), rewriter.getIndexAttr(2))});
     auto bufferSize0 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSize);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSize);
     auto bufferSize1 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr1);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSizePtr1);
     auto bufferSize2 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr2);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSizePtr2);
 
     rewriter.replaceOp(op, {bufferSize0, bufferSize1, bufferSize2, stream});
   } else {
@@ -1669,28 +1669,28 @@ LogicalResult ConvertSpMatGetSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
   Location loc = op.getLoc();
   auto stream = adaptor.getAsyncDependencies().front();
 
-  auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+  auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                  rewriter.getIndexAttr(3));
-  auto buffer = rewriter.create<LLVM::AllocaOp>(
+  auto buffer = LLVM::AllocaOp::create(rewriter,
       loc, llvmPointerType, llvmInt64Type, three, /*alignment=*/16);
 
-  auto rowsPtr = rewriter.create<LLVM::GEPOp>(
+  auto rowsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(0))});
-  auto colsPtr = rewriter.create<LLVM::GEPOp>(
+  auto colsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(1))});
-  auto nnzsPtr = rewriter.create<LLVM::GEPOp>(
+  auto nnzsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(2))});
   createSpMatGetSizeBuilder.create(
       loc, rewriter, {adaptor.getSpmat(), rowsPtr, colsPtr, nnzsPtr, stream});
-  auto rows = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, rowsPtr);
-  auto cols = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, colsPtr);
-  auto nnzs = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, nnzsPtr);
+  auto rows = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, rowsPtr);
+  auto cols = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, colsPtr);
+  auto nnzs = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, nnzsPtr);
 
   rewriter.replaceOp(op, {rows, cols, nnzs, stream});
   return success();
diff --git a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
index aab2409ed6328..70964727351a9 100644
--- a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
@@ -59,13 +59,13 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
     Operation *newOp;
     switch (op.getDimension()) {
     case gpu::Dimension::x:
-      newOp = rewriter.create<XOp>(loc, IntegerType::get(context, 32));
+      newOp = XOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     case gpu::Dimension::y:
-      newOp = rewriter.create<YOp>(loc, IntegerType::get(context, 32));
+      newOp = YOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     case gpu::Dimension::z:
-      newOp = rewriter.create<ZOp>(loc, IntegerType::get(context, 32));
+      newOp = ZOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     }
 
@@ -124,10 +124,10 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
                                   rewriter.getContext(), 32, min, max));
     }
     if (indexBitwidth > 32) {
-      newOp = rewriter.create<LLVM::SExtOp>(
+      newOp = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
     } else if (indexBitwidth < 32) {
-      newOp = rewriter.create<LLVM::TruncOp>(
+      newOp = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
     }
 
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index 64cf09e600b88..33b09b06649b9 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -103,7 +103,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
     LLVMFuncOp funcOp = appendOrGetFuncOp(funcName, funcType, op);
     auto callOp =
-        rewriter.create<LLVM::CallOp>(op->getLoc(), funcOp, castedOperands);
+        LLVM::CallOp::create(rewriter, op->getLoc(), funcOp, castedOperands);
 
     if (resultType == adaptor.getOperands().front().getType()) {
       rewriter.replaceOp(op, {callOp.getResult()});
@@ -115,10 +115,10 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // there is no guarantee of a specific value being used to indicate true,
     // compare for inequality with zero (rather than truncate or shift).
     if (isResultBool) {
-      Value zero = rewriter.create<LLVM::ConstantOp>(
+      Value zero = LLVM::ConstantOp::create(rewriter,
           op->getLoc(), rewriter.getIntegerType(32),
           rewriter.getI32IntegerAttr(0));
-      Value truncated = rewriter.create<LLVM::ICmpOp>(
+      Value truncated = LLVM::ICmpOp::create(rewriter,
           op->getLoc(), LLVM::ICmpPredicate::ne, callOp.getResult(), zero);
       rewriter.replaceOp(op, {truncated});
       return success();
@@ -126,7 +126,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
     assert(callOp.getResult().getType().isF32() &&
            "only f32 types are supposed to be truncated back");
-    Value truncated = rewriter.create<LLVM::FPTruncOp>(
+    Value truncated = LLVM::FPTruncOp::create(rewriter,
         op->getLoc(), adaptor.getOperands().front().getType(),
         callOp.getResult());
     rewriter.replaceOp(op, {truncated});
@@ -142,7 +142,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     if (!f16Func.empty() && isa<Float16Type>(type))
       return operand;
 
-    return rewriter.create<LLVM::FPExtOp>(
+    return LLVM::FPExtOp::create(rewriter,
         operand.getLoc(), Float32Type::get(rewriter.getContext()), operand);
   }
 
@@ -169,7 +169,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // location as debug info metadata inside of a function cannot be used
     // outside of that function.
     auto globalloc = op->getLoc()->findInstanceOfOrUnknown<FileLineColLoc>();
-    return b.create<LLVMFuncOp>(globalloc, funcName, funcType);
+    return LLVMFuncOp::create(b, globalloc, funcName, funcType);
   }
 
   StringRef getFunctionName(Type type, SourceOp op) const {
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index 8b6b553f6eed0..fc469102a54d2 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -54,7 +54,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
       SymbolTable::lookupSymbolIn(symbolTable, name));
   if (!func) {
     OpBuilder b(symbolTable->getRegion(0));
-    func = b.create<LLVM::LLVMFuncOp>(
+    func = LLVM::LLVMFuncOp::create(b,
         symbolTable->getLoc(), name,
         LLVM::LLVMFunctionType::get(resultType, paramTypes));
     func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
@@ -79,7 +79,7 @@ static LLVM::CallOp createSPIRVBuiltinCall(Location loc,
                                            ConversionPatternRewriter &rewriter,
                                            LLVM::LLVMFuncOp func,
                                            ValueRange args) {
-  auto call = rewriter.create<LLVM::CallOp>(loc, func, args);
+  auto call = LLVM::CallOp::create(rewriter, loc, func, args);
   call.setCConv(func.getCConv());
   call.setConvergentAttr(func.getConvergentAttr());
   call.setNoUnwindAttr(func.getNoUnwindAttr());
@@ -121,7 +121,7 @@ struct GPUBarrierConversion final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
     constexpr int64_t localMemFenceFlag = 1;
     Location loc = op->getLoc();
     Value flag =
-        rewriter.create<LLVM::ConstantOp>(loc, flagTy, localMemFenceFlag);
+        LLVM::ConstantOp::create(rewriter, loc, flagTy, localMemFenceFlag);
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, flag));
     return success();
   }
@@ -162,7 +162,7 @@ struct LaunchConfigConversion : ConvertToLLVMPattern {
 
     Location loc = op->getLoc();
     gpu::Dimension dim = getDimension(op);
-    Value dimVal = rewriter.create<LLVM::ConstantOp>(loc, dimTy,
+    Value dimVal = LLVM::ConstantOp::create(rewriter, loc, dimTy,
                                                      static_cast<int64_t>(dim));
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, dimVal));
     return success();
@@ -291,12 +291,12 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                                          ConversionPatternRewriter &rewriter) {
     return TypeSwitch<Type, Value>(oldVal.getType())
         .Case([&](BFloat16Type) {
-          return rewriter.create<LLVM::BitcastOp>(loc, rewriter.getI16Type(),
+          return LLVM::BitcastOp::create(rewriter, loc, rewriter.getI16Type(),
                                                   oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
-            return rewriter.create<LLVM::ZExtOp>(loc, rewriter.getI8Type(),
+            return LLVM::ZExtOp::create(rewriter, loc, rewriter.getI8Type(),
                                                  oldVal);
           return oldVal;
         })
@@ -308,11 +308,11 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                                           ConversionPatternRewriter &rewriter) {
     return TypeSwitch<Type, Value>(newTy)
         .Case([&](BFloat16Type) {
-          return rewriter.create<LLVM::BitcastOp>(loc, newTy, oldVal);
+          return LLVM::BitcastOp::create(rewriter, loc, newTy, oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
-            return rewriter.create<LLVM::TruncOp>(loc, newTy, oldVal);
+            return LLVM::TruncOp::create(rewriter, loc, newTy, oldVal);
           return oldVal;
         })
         .Default(oldVal);
@@ -349,7 +349,7 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
         bitcastOrTruncAfterShuffle(result, op.getType(0), loc, rewriter);
 
     Value trueVal =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI1Type(), true);
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getI1Type(), true);
     rewriter.replaceOp(op, {resultOrConversion, trueVal});
     return success();
   }
@@ -426,7 +426,7 @@ struct GPUSubgroupOpConversion final : ConvertOpToLLVMPattern<SubgroupOp> {
       if (indexTy.getIntOrFloatBitWidth() < resultTy.getIntOrFloatBitWidth()) {
         return failure();
       }
-      result = rewriter.create<LLVM::ZExtOp>(loc, indexTy, result);
+      result = LLVM::ZExtOp::create(rewriter, loc, indexTy, result);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 1ef6edea93c58..7a6601ea754e9 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -118,9 +118,9 @@ struct GPUSubgroupReduceOpLowering
 
     Location loc = op->getLoc();
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
-    Value offset = rewriter.create<LLVM::ConstantOp>(loc, int32Type, -1);
+    Value offset = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
 
-    auto reduxOp = rewriter.create<NVVM::ReduxOp>(loc, int32Type, op.getValue(),
+    auto reduxOp = NVVM::ReduxOp::create(rewriter, loc, int32Type, op.getValue(),
                                                   mode.value(), offset);
 
     rewriter.replaceOp(op, reduxOp->getResult(0));
@@ -158,13 +158,13 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     auto predTy = IntegerType::get(rewriter.getContext(), 1);
 
-    Value one = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 1);
-    Value minusOne = rewriter.create<LLVM::ConstantOp>(loc, int32Type, -1);
-    Value thirtyTwo = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 32);
-    Value numLeadInactiveLane = rewriter.create<LLVM::SubOp>(
+    Value one = LLVM::ConstantOp::create(rewriter, loc, int32Type, 1);
+    Value minusOne = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
+    Value thirtyTwo = LLVM::ConstantOp::create(rewriter, loc, int32Type, 32);
+    Value numLeadInactiveLane = LLVM::SubOp::create(rewriter,
         loc, int32Type, thirtyTwo, adaptor.getWidth());
     // Bit mask of active lanes: `(-1) >> (32 - activeWidth)`.
-    Value activeMask = rewriter.create<LLVM::LShrOp>(loc, int32Type, minusOne,
+    Value activeMask = LLVM::LShrOp::create(rewriter, loc, int32Type, minusOne,
                                                      numLeadInactiveLane);
     Value maskAndClamp;
     if (op.getMode() == gpu::ShuffleMode::UP) {
@@ -173,7 +173,7 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     } else {
       // Clamp lane: `activeWidth - 1`
       maskAndClamp =
-          rewriter.create<LLVM::SubOp>(loc, int32Type, adaptor.getWidth(), one);
+          LLVM::SubOp::create(rewriter, loc, int32Type, adaptor.getWidth(), one);
     }
 
     bool predIsUsed = !op->getResult(1).use_empty();
@@ -184,13 +184,13 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
       resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(),
                                                   {valueTy, predTy});
     }
-    Value shfl = rewriter.create<NVVM::ShflOp>(
+    Value shfl = NVVM::ShflOp::create(rewriter,
         loc, resultTy, activeMask, adaptor.getValue(), adaptor.getOffset(),
         maskAndClamp, convertShflKind(op.getMode()), returnValueAndIsValidAttr);
     if (predIsUsed) {
-      Value shflValue = rewriter.create<LLVM::ExtractValueOp>(loc, shfl, 0);
+      Value shflValue = LLVM::ExtractValueOp::create(rewriter, loc, shfl, 0);
       Value isActiveSrcLane =
-          rewriter.create<LLVM::ExtractValueOp>(loc, shfl, 1);
+          LLVM::ExtractValueOp::create(rewriter, loc, shfl, 1);
       rewriter.replaceOp(op, {shflValue, isActiveSrcLane});
     } else {
       rewriter.replaceOp(op, {shfl, nullptr});
@@ -215,15 +215,15 @@ struct GPULaneIdOpToNVVM : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
       bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
           /*bitWidth=*/32, /*lower=*/0, /*upper=*/kWarpSize);
     Value newOp =
-        rewriter.create<NVVM::LaneIdOp>(loc, rewriter.getI32Type(), bounds);
+        NVVM::LaneIdOp::create(rewriter, loc, rewriter.getI32Type(), bounds);
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      newOp = rewriter.create<LLVM::SExtOp>(
+      newOp = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp);
     } else if (indexBitwidth < 32) {
-      newOp = rewriter.create<LLVM::TruncOp>(
+      newOp = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp);
     }
     rewriter.replaceOp(op, {newOp});
@@ -271,10 +271,10 @@ struct AssertOpToAssertfailLowering
     Block *afterBlock =
         rewriter.splitBlock(assertBlock, ++assertOp->getIterator());
     rewriter.setInsertionPointToEnd(beforeBlock);
-    rewriter.create<cf::CondBranchOp>(loc, adaptor.getArg(), afterBlock,
+    cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(), afterBlock,
                                       assertBlock);
     rewriter.setInsertionPointToEnd(assertBlock);
-    rewriter.create<cf::BranchOp>(loc, afterBlock);
+    cf::BranchOp::create(rewriter, loc, afterBlock);
 
     // Continue cf.assert lowering.
     rewriter.setInsertionPoint(assertOp);
@@ -301,11 +301,11 @@ struct AssertOpToAssertfailLowering
     // Create constants.
     auto getGlobal = [&](LLVM::GlobalOp global) {
       // Get a pointer to the format string's first element.
-      Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+      Value globalPtr = LLVM::AddressOfOp::create(rewriter,
           loc, LLVM::LLVMPointerType::get(ctx, global.getAddrSpace()),
           global.getSymNameAttr());
       Value start =
-          rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+          LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                        globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
       return start;
     };
@@ -316,8 +316,8 @@ struct AssertOpToAssertfailLowering
     Value assertFunc = getGlobal(getOrCreateStringConstant(
         rewriter, loc, moduleOp, i8Type, "assert_func_", funcName));
     Value assertLine =
-        rewriter.create<LLVM::ConstantOp>(loc, i32Type, fileLine);
-    Value c1 = rewriter.create<LLVM::ConstantOp>(loc, i64Type, 1);
+        LLVM::ConstantOp::create(rewriter, loc, i32Type, fileLine);
+    Value c1 = LLVM::ConstantOp::create(rewriter, loc, i64Type, 1);
 
     // Insert function call to __assertfail.
     SmallVector<Value> arguments{assertMessage, assertFile, assertLine,
diff --git a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
index 45fd933d58857..7d5f65c470386 100644
--- a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
@@ -126,7 +126,7 @@ struct WmmaLoadOpToNVVMLowering
         cast<MemRefType>(subgroupMmaLoadMatrixOp.getSrcMemref().getType()),
         adaptor.getSrcMemref(), adaptor.getIndices());
 
-    Value leadingDim = rewriter.create<LLVM::ConstantOp>(
+    Value leadingDim = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI32Type(),
         subgroupMmaLoadMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMALoadOp>(
@@ -173,7 +173,7 @@ struct WmmaStoreOpToNVVMLowering
     auto matrixType = cast<LLVM::LLVMStructType>(adaptor.getSrc().getType());
     for (unsigned i = 0, e = matrixType.getBody().size(); i < e; ++i) {
       Value toUse =
-          rewriter.create<LLVM::ExtractValueOp>(loc, adaptor.getSrc(), i);
+          LLVM::ExtractValueOp::create(rewriter, loc, adaptor.getSrc(), i);
       storeOpOperands.push_back(toUse);
     }
 
@@ -181,7 +181,7 @@ struct WmmaStoreOpToNVVMLowering
         rewriter, loc,
         cast<MemRefType>(subgroupMmaStoreMatrixOp.getDstMemref().getType()),
         adaptor.getDstMemref(), adaptor.getIndices());
-    Value leadingDim = rewriter.create<LLVM::ConstantOp>(
+    Value leadingDim = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI32Type(),
         subgroupMmaStoreMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMAStoreOp>(
@@ -216,7 +216,7 @@ struct WmmaMmaOpToNVVMLowering
     auto unpackOp = [&](Value operand) {
       auto structType = cast<LLVM::LLVMStructType>(operand.getType());
       for (size_t i = 0, e = structType.getBody().size(); i < e; ++i) {
-        Value toUse = rewriter.create<LLVM::ExtractValueOp>(loc, operand, i);
+        Value toUse = LLVM::ExtractValueOp::create(rewriter, loc, operand, i);
         unpackedOps.push_back(toUse);
       }
     };
@@ -280,19 +280,19 @@ struct WmmaConstantOpToNVVMLowering
         cast<gpu::MMAMatrixType>(subgroupMmaConstantOp.getType()));
     // If the element type is a vector create a vector from the operand.
     if (auto vecType = dyn_cast<VectorType>(type.getBody()[0])) {
-      Value vecCst = rewriter.create<LLVM::PoisonOp>(loc, vecType);
+      Value vecCst = LLVM::PoisonOp::create(rewriter, loc, vecType);
       for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) {
-        Value idx = rewriter.create<LLVM::ConstantOp>(
+        Value idx = LLVM::ConstantOp::create(rewriter,
             loc, rewriter.getI32Type(), vecEl);
-        vecCst = rewriter.create<LLVM::InsertElementOp>(loc, vecType, vecCst,
+        vecCst = LLVM::InsertElementOp::create(rewriter, loc, vecType, vecCst,
                                                         cst, idx);
       }
       cst = vecCst;
     }
-    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, type);
+    Value matrixStruct = LLVM::PoisonOp::create(rewriter, loc, type);
     for (size_t i : llvm::seq(size_t(0), type.getBody().size())) {
       matrixStruct =
-          rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, cst, i);
+          LLVM::InsertValueOp::create(rewriter, loc, matrixStruct, cst, i);
     }
     rewriter.replaceOp(subgroupMmaConstantOp, matrixStruct);
     return success();
@@ -305,17 +305,17 @@ static Value createMinMaxF(OpBuilder &builder, Location loc, Value lhs,
   Type i1Type = builder.getI1Type();
   if (auto vecType = dyn_cast<VectorType>(lhs.getType()))
     i1Type = VectorType::get(vecType.getShape(), i1Type);
-  Value cmp = builder.create<LLVM::FCmpOp>(
+  Value cmp = LLVM::FCmpOp::create(builder,
       loc, i1Type, isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt,
       lhs, rhs);
-  Value sel = builder.create<LLVM::SelectOp>(loc, cmp, lhs, rhs);
-  Value isNan = builder.create<LLVM::FCmpOp>(
+  Value sel = LLVM::SelectOp::create(builder, loc, cmp, lhs, rhs);
+  Value isNan = LLVM::FCmpOp::create(builder,
       loc, i1Type, LLVM::FCmpPredicate::uno, lhs, rhs);
-  Value nan = builder.create<LLVM::ConstantOp>(
+  Value nan = LLVM::ConstantOp::create(builder,
       loc, lhs.getType(),
       builder.getFloatAttr(floatType,
                            APFloat::getQNaN(floatType.getFloatSemantics())));
-  return builder.create<LLVM::SelectOp>(loc, isNan, nan, sel);
+  return LLVM::SelectOp::create(builder, loc, isNan, nan, sel);
 }
 
 static Value createScalarOp(OpBuilder &builder, Location loc,
@@ -323,11 +323,11 @@ static Value createScalarOp(OpBuilder &builder, Location loc,
                             ArrayRef<Value> operands) {
   switch (op) {
   case gpu::MMAElementwiseOp::ADDF:
-    return builder.create<LLVM::FAddOp>(loc, operands[0].getType(), operands);
+    return LLVM::FAddOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::MULF:
-    return builder.create<LLVM::FMulOp>(loc, operands[0].getType(), operands);
+    return LLVM::FMulOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::DIVF:
-    return builder.create<LLVM::FDivOp>(loc, operands[0].getType(), operands);
+    return LLVM::FDivOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::MAXF:
     return createMinMaxF(builder, loc, operands[0], operands[1],
                          /*isMin=*/false);
@@ -356,18 +356,18 @@ struct WmmaElementwiseOpToNVVMLowering
     size_t numOperands = adaptor.getOperands().size();
     LLVM::LLVMStructType destType = convertMMAToLLVMType(
         cast<gpu::MMAMatrixType>(subgroupMmaElementwiseOp.getType()));
-    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, destType);
+    Value matrixStruct = LLVM::PoisonOp::create(rewriter, loc, destType);
     for (size_t i = 0, e = destType.getBody().size(); i < e; ++i) {
       SmallVector<Value> extractedOperands;
       for (size_t opIdx = 0; opIdx < numOperands; opIdx++) {
-        extractedOperands.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
             loc, adaptor.getOperands()[opIdx], i));
       }
       Value element =
           createScalarOp(rewriter, loc, subgroupMmaElementwiseOp.getOpType(),
                          extractedOperands);
       matrixStruct =
-          rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, element, i);
+          LLVM::InsertValueOp::create(rewriter, loc, matrixStruct, element, i);
     }
     rewriter.replaceOp(subgroupMmaElementwiseOp, matrixStruct);
     return success();
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 456bfaba980ca..9b43d3ef602d1 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -61,10 +61,10 @@ static Value truncOrExtToLLVMType(ConversionPatternRewriter &rewriter,
       IntegerType::get(rewriter.getContext(), converter.getIndexTypeBitwidth());
   // TODO: use <=> in C++20.
   if (indexBitwidth > intWidth) {
-    return rewriter.create<LLVM::SExtOp>(loc, indexBitwidthType, value);
+    return LLVM::SExtOp::create(rewriter, loc, indexBitwidthType, value);
   }
   if (indexBitwidth < intWidth) {
-    return rewriter.create<LLVM::TruncOp>(loc, indexBitwidthType, value);
+    return LLVM::TruncOp::create(rewriter, loc, indexBitwidthType, value);
   }
   return value;
 }
@@ -82,11 +82,11 @@ static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func) {
 static Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
                        const unsigned indexBitwidth) {
   auto int32Type = IntegerType::get(rewriter.getContext(), 32);
-  Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
-  Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
-  Value mbcntLo = rewriter.create<ROCDL::MbcntLoOp>(loc, int32Type,
+  Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
+  Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
+  Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, int32Type,
                                                     ValueRange{minus1, zero});
-  Value laneId = rewriter.create<ROCDL::MbcntHiOp>(loc, int32Type,
+  Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, int32Type,
                                                    ValueRange{minus1, mbcntLo});
   return laneId;
 }
@@ -110,20 +110,20 @@ struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
     // followed by: %lid = call @llvm.amdgcn.mbcnt.hi(-1, %mlo)
 
     Type intTy = IntegerType::get(context, 32);
-    Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
-    Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
+    Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
+    Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
     Value mbcntLo =
-        rewriter.create<ROCDL::MbcntLoOp>(loc, intTy, ValueRange{minus1, zero});
-    Value laneId = rewriter.create<ROCDL::MbcntHiOp>(
+        ROCDL::MbcntLoOp::create(rewriter, loc, intTy, ValueRange{minus1, zero});
+    Value laneId = ROCDL::MbcntHiOp::create(rewriter,
         loc, intTy, ValueRange{minus1, mbcntLo});
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      laneId = rewriter.create<LLVM::SExtOp>(
+      laneId = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), laneId);
     } else if (indexBitwidth < 32) {
-      laneId = rewriter.create<LLVM::TruncOp>(
+      laneId = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), laneId);
     }
     rewriter.replaceOp(op, {laneId});
@@ -149,7 +149,7 @@ struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
           /*bitWidth=*/32, /*lower=*/isBeforeGfx10 ? 64 : 32,
           /*upper=*/op.getUpperBoundAttr().getInt() + 1);
     }
-    Value wavefrontOp = rewriter.create<ROCDL::WavefrontSizeOp>(
+    Value wavefrontOp = ROCDL::WavefrontSizeOp::create(rewriter,
         op.getLoc(), rewriter.getI32Type(), bounds);
     wavefrontOp = truncOrExtToLLVMType(rewriter, op.getLoc(), wavefrontOp,
                                        *getTypeConverter());
@@ -190,43 +190,43 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
 
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     Value width = adaptor.getWidth();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 0);
-    Value negwidth = rewriter.create<LLVM::SubOp>(loc, int32Type, zero, width);
-    Value add = rewriter.create<LLVM::AddOp>(loc, int32Type, srcLaneId, width);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, int32Type, 0);
+    Value negwidth = LLVM::SubOp::create(rewriter, loc, int32Type, zero, width);
+    Value add = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId, width);
     Value widthOrZeroIfOutside =
-        rewriter.create<LLVM::AndOp>(loc, int32Type, add, negwidth);
+        LLVM::AndOp::create(rewriter, loc, int32Type, add, negwidth);
     Value dstLane;
 
     switch (op.getMode()) {
     case gpu::ShuffleMode::UP:
-      dstLane = rewriter.create<LLVM::SubOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::SubOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::DOWN:
-      dstLane = rewriter.create<LLVM::AddOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::XOR:
-      dstLane = rewriter.create<LLVM::XOrOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::XOrOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::IDX:
       dstLane = adaptor.getOffset();
       break;
     }
-    Value isActiveSrcLane = rewriter.create<LLVM::ICmpOp>(
+    Value isActiveSrcLane = LLVM::ICmpOp::create(rewriter,
         loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
-    Value selectDstLane = rewriter.create<LLVM::SelectOp>(loc, isActiveSrcLane,
+    Value selectDstLane = LLVM::SelectOp::create(rewriter, loc, isActiveSrcLane,
                                                           dstLane, srcLaneId);
-    Value two = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 2);
+    Value two = LLVM::ConstantOp::create(rewriter, loc, int32Type, 2);
     Value dwordAlignedDstLane =
-        rewriter.create<LLVM::ShlOp>(loc, int32Type, selectDstLane, two);
+        LLVM::ShlOp::create(rewriter, loc, int32Type, selectDstLane, two);
 
     SmallVector<Value> decomposed =
         LLVM::decomposeValue(rewriter, loc, initShflValue, int32Type);
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
-      Value res = rewriter.create<ROCDL::DsBpermuteOp>(loc, int32Type,
+      Value res = ROCDL::DsBpermuteOp::create(rewriter, loc, int32Type,
                                                        dwordAlignedDstLane, v);
       swizzled.emplace_back(res);
     }
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index b99ed261ecfa3..71618cbf8a2f2 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -169,11 +169,11 @@ LogicalResult LaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
 
   Value vector =
       spirv::getBuiltinVariableValue(op, builtin, builtinType, rewriter);
-  Value dim = rewriter.create<spirv::CompositeExtractOp>(
+  Value dim = spirv::CompositeExtractOp::create(rewriter,
       op.getLoc(), builtinType, vector,
       rewriter.getI32ArrayAttr({static_cast<int32_t>(op.getDimension())}));
   if (forShader && builtinType != indexType)
-    dim = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType, dim);
+    dim = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType, dim);
   rewriter.replaceOp(op, dim);
   return success();
 }
@@ -198,7 +198,7 @@ SingleDimLaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
   Value builtinValue =
       spirv::getBuiltinVariableValue(op, builtin, i32Type, rewriter);
   if (i32Type != indexType)
-    builtinValue = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType,
+    builtinValue = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType,
                                                       builtinValue);
   rewriter.replaceOp(op, builtinValue);
   return success();
@@ -257,7 +257,7 @@ lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter,
       signatureConverter.addInputs(argType.index(), convertedType);
     }
   }
-  auto newFuncOp = rewriter.create<spirv::FuncOp>(
+  auto newFuncOp = spirv::FuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(signatureConverter.getConvertedTypes(), {}));
   for (const auto &namedAttr : funcOp->getAttrs()) {
@@ -367,7 +367,7 @@ LogicalResult GPUModuleConversion::matchAndRewrite(
 
   // Add a keyword to the module name to avoid symbolic conflict.
   std::string spvModuleName = (kSPIRVModule + moduleOp.getName()).str();
-  auto spvModule = rewriter.create<spirv::ModuleOp>(
+  auto spvModule = spirv::ModuleOp::create(rewriter,
       moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
       StringRef(spvModuleName));
 
@@ -452,41 +452,41 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
 
   switch (shuffleOp.getMode()) {
   case gpu::ShuffleMode::XOR: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleXorOp>(
+    result = spirv::GroupNonUniformShuffleXorOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::IDX: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleOp>(
+    result = spirv::GroupNonUniformShuffleOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::DOWN: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleDownOp>(
+    result = spirv::GroupNonUniformShuffleDownOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
 
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
-        rewriter.create<arith::AddIOp>(loc, laneId, adaptor.getOffset());
-    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+        arith::AddIOp::create(rewriter, loc, laneId, adaptor.getOffset());
+    validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                               resultLaneId, adaptor.getWidth());
     break;
   }
   case gpu::ShuffleMode::UP: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleUpOp>(
+    result = spirv::GroupNonUniformShuffleUpOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
 
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
-        rewriter.create<arith::SubIOp>(loc, laneId, adaptor.getOffset());
+        arith::SubIOp::create(rewriter, loc, laneId, adaptor.getOffset());
     auto i32Type = rewriter.getIntegerType(32);
-    validVal = rewriter.create<arith::CmpIOp>(
+    validVal = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, resultLaneId,
-        rewriter.create<arith::ConstantOp>(
+        arith::ConstantOp::create(rewriter,
             loc, i32Type, rewriter.getIntegerAttr(i32Type, 0)));
     break;
   }
@@ -516,14 +516,14 @@ LogicalResult GPURotateConversion::matchAndRewrite(
 
   Location loc = rotateOp.getLoc();
   auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
-  Value rotateResult = rewriter.create<spirv::GroupNonUniformRotateKHROp>(
+  Value rotateResult = spirv::GroupNonUniformRotateKHROp::create(rewriter,
       loc, scope, adaptor.getValue(), adaptor.getOffset(), adaptor.getWidth());
   Value validVal;
   if (widthAttr.getValue().getZExtValue() == subgroupSize) {
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(), loc, rewriter);
   } else {
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
-    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
+    validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                               laneId, adaptor.getWidth());
   }
 
@@ -548,13 +548,13 @@ static Value createGroupReduceOpImpl(OpBuilder &builder, Location loc,
                                 ? spirv::GroupOperation::ClusteredReduce
                                 : spirv::GroupOperation::Reduce);
   if (isUniform) {
-    return builder.create<UniformOp>(loc, type, scope, groupOp, arg)
+    return UniformOp::create(builder, loc, type, scope, groupOp, arg)
         .getResult();
   }
 
   Value clusterSizeValue;
   if (clusterSize.has_value())
-    clusterSizeValue = builder.create<spirv::ConstantOp>(
+    clusterSizeValue = spirv::ConstantOp::create(builder,
         loc, builder.getI32Type(),
         builder.getIntegerAttr(builder.getI32Type(), *clusterSize));
 
@@ -740,7 +740,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstName =
         makeVarName(moduleOp, llvm::Twine(globalVarName) + "_sc");
 
-    return rewriter.create<spirv::SpecConstantOp>(
+    return spirv::SpecConstantOp::create(rewriter,
         loc, rewriter.getStringAttr(specCstName), attr);
   };
   {
@@ -774,7 +774,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstCompositeName =
         (llvm::Twine(globalVarName) + "_scc").str();
 
-    specCstComposite = rewriter.create<spirv::SpecConstantCompositeOp>(
+    specCstComposite = spirv::SpecConstantCompositeOp::create(rewriter,
         loc, TypeAttr::get(globalType),
         rewriter.getStringAttr(specCstCompositeName),
         rewriter.getArrayAttr(constituents));
@@ -785,15 +785,15 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     // Define a GlobalVarOp initialized using specialized constants
     // that is used to specify the printf format string
     // to be passed to the SPIRV CLPrintfOp.
-    globalVar = rewriter.create<spirv::GlobalVariableOp>(
+    globalVar = spirv::GlobalVariableOp::create(rewriter,
         loc, ptrType, globalVarName, FlatSymbolRefAttr::get(specCstComposite));
 
     globalVar->setAttr("Constant", rewriter.getUnitAttr());
   }
   // Get SSA value of Global variable and create pointer to i8 to point to
   // the format string.
-  Value globalPtr = rewriter.create<spirv::AddressOfOp>(loc, globalVar);
-  Value fmtStr = rewriter.create<spirv::BitcastOp>(
+  Value globalPtr = spirv::AddressOfOp::create(rewriter, loc, globalVar);
+  Value fmtStr = spirv::BitcastOp::create(rewriter,
       loc,
       spirv::PointerType::get(i8Type, spirv::StorageClass::UniformConstant),
       globalPtr);
@@ -801,7 +801,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
   // Get printf arguments.
   auto printfArgs = llvm::to_vector_of<Value, 4>(adaptor.getArgs());
 
-  rewriter.create<spirv::CLPrintfOp>(loc, i32Type, fmtStr, printfArgs);
+  spirv::CLPrintfOp::create(rewriter, loc, i32Type, fmtStr, printfArgs);
 
   // Need to erase the gpu.printf op as gpu.printf does not use result vs
   // spirv::CLPrintfOp has i32 resultType so cannot replace with new SPIR-V
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index 0b2c06a08db2d..da8ffe47a7d33 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -144,11 +144,11 @@ void GPUToSPIRVPass::runOnOperation() {
     if (targetEnvSupportsKernelCapability(moduleOp)) {
       moduleOp.walk([&](gpu::GPUFuncOp funcOp) {
         builder.setInsertionPoint(funcOp);
-        auto newFuncOp = builder.create<func::FuncOp>(
+        auto newFuncOp = func::FuncOp::create(builder,
             funcOp.getLoc(), funcOp.getName(), funcOp.getFunctionType());
         auto entryBlock = newFuncOp.addEntryBlock();
         builder.setInsertionPointToEnd(entryBlock);
-        builder.create<func::ReturnOp>(funcOp.getLoc());
+        func::ReturnOp::create(builder, funcOp.getLoc());
         newFuncOp->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                            builder.getUnitAttr());
         funcOp.erase();
diff --git a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
index 7bb86b5ce1ddd..70646aa5f2e74 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
@@ -283,7 +283,7 @@ struct WmmaLoadOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = rewriter.create<spirv::ConstantOp>(
+    auto strideValue = spirv::ConstantOp::create(rewriter,
         loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
@@ -315,7 +315,7 @@ struct WmmaStoreOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = rewriter.create<spirv::ConstantOp>(
+    auto strideValue = spirv::ConstantOp::create(rewriter,
         loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
diff --git a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
index 0473bb59fa6aa..0d5b318f739cc 100644
--- a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
+++ b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
@@ -36,34 +36,34 @@ struct ConvertIndexCeilDivS : mlir::ConvertOpToLLVMPattern<CeilDivSOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value posOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
-    Value negOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), -1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value posOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
+    Value negOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), -1);
 
     // Compute `x`.
     Value mPos =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::sgt, m, zero);
-    Value x = rewriter.create<LLVM::SelectOp>(loc, mPos, negOne, posOne);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, m, zero);
+    Value x = LLVM::SelectOp::create(rewriter, loc, mPos, negOne, posOne);
 
     // Compute the positive result.
-    Value nPlusX = rewriter.create<LLVM::AddOp>(loc, n, x);
-    Value nPlusXDivM = rewriter.create<LLVM::SDivOp>(loc, nPlusX, m);
-    Value posRes = rewriter.create<LLVM::AddOp>(loc, nPlusXDivM, posOne);
+    Value nPlusX = LLVM::AddOp::create(rewriter, loc, n, x);
+    Value nPlusXDivM = LLVM::SDivOp::create(rewriter, loc, nPlusX, m);
+    Value posRes = LLVM::AddOp::create(rewriter, loc, nPlusXDivM, posOne);
 
     // Compute the negative result.
-    Value negN = rewriter.create<LLVM::SubOp>(loc, zero, n);
-    Value negNDivM = rewriter.create<LLVM::SDivOp>(loc, negN, m);
-    Value negRes = rewriter.create<LLVM::SubOp>(loc, zero, negNDivM);
+    Value negN = LLVM::SubOp::create(rewriter, loc, zero, n);
+    Value negNDivM = LLVM::SDivOp::create(rewriter, loc, negN, m);
+    Value negRes = LLVM::SubOp::create(rewriter, loc, zero, negNDivM);
 
     // Pick the positive result if `n` and `m` have the same sign and `n` is
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
     Value nPos =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::sgt, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, n, zero);
     Value sameSign =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, nPos, mPos);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, nPos, mPos);
     Value nNonZero =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, n, zero);
-    Value cmp = rewriter.create<LLVM::AndOp>(loc, sameSign, nNonZero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
+    Value cmp = LLVM::AndOp::create(rewriter, loc, sameSign, nNonZero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
@@ -83,17 +83,17 @@ struct ConvertIndexCeilDivU : mlir::ConvertOpToLLVMPattern<CeilDivUOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value one = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value one = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
 
     // Compute the non-zero result.
-    Value minusOne = rewriter.create<LLVM::SubOp>(loc, n, one);
-    Value quotient = rewriter.create<LLVM::UDivOp>(loc, minusOne, m);
-    Value plusOne = rewriter.create<LLVM::AddOp>(loc, quotient, one);
+    Value minusOne = LLVM::SubOp::create(rewriter, loc, n, one);
+    Value quotient = LLVM::UDivOp::create(rewriter, loc, minusOne, m);
+    Value plusOne = LLVM::AddOp::create(rewriter, loc, quotient, one);
 
     // Pick the result.
     Value cmp =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, n, zero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, zero, plusOne);
     return success();
   }
@@ -114,32 +114,32 @@ struct ConvertIndexFloorDivS : mlir::ConvertOpToLLVMPattern<FloorDivSOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value posOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
-    Value negOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), -1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value posOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
+    Value negOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), -1);
 
     // Compute `x`.
     Value mNeg =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::slt, m, zero);
-    Value x = rewriter.create<LLVM::SelectOp>(loc, mNeg, posOne, negOne);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, m, zero);
+    Value x = LLVM::SelectOp::create(rewriter, loc, mNeg, posOne, negOne);
 
     // Compute the negative result.
-    Value xMinusN = rewriter.create<LLVM::SubOp>(loc, x, n);
-    Value xMinusNDivM = rewriter.create<LLVM::SDivOp>(loc, xMinusN, m);
-    Value negRes = rewriter.create<LLVM::SubOp>(loc, negOne, xMinusNDivM);
+    Value xMinusN = LLVM::SubOp::create(rewriter, loc, x, n);
+    Value xMinusNDivM = LLVM::SDivOp::create(rewriter, loc, xMinusN, m);
+    Value negRes = LLVM::SubOp::create(rewriter, loc, negOne, xMinusNDivM);
 
     // Compute the positive result.
-    Value posRes = rewriter.create<LLVM::SDivOp>(loc, n, m);
+    Value posRes = LLVM::SDivOp::create(rewriter, loc, n, m);
 
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
     Value nNeg =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::slt, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, n, zero);
     Value diffSign =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
     Value nNonZero =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, n, zero);
-    Value cmp = rewriter.create<LLVM::AndOp>(loc, diffSign, nNonZero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
+    Value cmp = LLVM::AndOp::create(rewriter, loc, diffSign, nNonZero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, negRes, posRes);
     return success();
   }
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index 4821962f989e6..6fc097059c5e5 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -111,33 +111,33 @@ struct ConvertIndexCeilDivSPattern final : OpConversionPattern<CeilDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = rewriter.create<spirv::ConstantOp>(
+    Value posOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = rewriter.create<spirv::ConstantOp>(
+    Value negOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
-    Value mPos = rewriter.create<spirv::SGreaterThanOp>(loc, m, zero);
-    Value x = rewriter.create<spirv::SelectOp>(loc, mPos, negOne, posOne);
+    Value mPos = spirv::SGreaterThanOp::create(rewriter, loc, m, zero);
+    Value x = spirv::SelectOp::create(rewriter, loc, mPos, negOne, posOne);
 
     // Compute the positive result.
-    Value nPlusX = rewriter.create<spirv::IAddOp>(loc, n, x);
-    Value nPlusXDivM = rewriter.create<spirv::SDivOp>(loc, nPlusX, m);
-    Value posRes = rewriter.create<spirv::IAddOp>(loc, nPlusXDivM, posOne);
+    Value nPlusX = spirv::IAddOp::create(rewriter, loc, n, x);
+    Value nPlusXDivM = spirv::SDivOp::create(rewriter, loc, nPlusX, m);
+    Value posRes = spirv::IAddOp::create(rewriter, loc, nPlusXDivM, posOne);
 
     // Compute the negative result.
-    Value negN = rewriter.create<spirv::ISubOp>(loc, zero, n);
-    Value negNDivM = rewriter.create<spirv::SDivOp>(loc, negN, m);
-    Value negRes = rewriter.create<spirv::ISubOp>(loc, zero, negNDivM);
+    Value negN = spirv::ISubOp::create(rewriter, loc, zero, n);
+    Value negNDivM = spirv::SDivOp::create(rewriter, loc, negN, m);
+    Value negRes = spirv::ISubOp::create(rewriter, loc, zero, negNDivM);
 
     // Pick the positive result if `n` and `m` have the same sign and `n` is
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
-    Value nPos = rewriter.create<spirv::SGreaterThanOp>(loc, n, zero);
-    Value sameSign = rewriter.create<spirv::LogicalEqualOp>(loc, nPos, mPos);
-    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
-    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, sameSign, nNonZero);
+    Value nPos = spirv::SGreaterThanOp::create(rewriter, loc, n, zero);
+    Value sameSign = spirv::LogicalEqualOp::create(rewriter, loc, nPos, mPos);
+    Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
+    Value cmp = spirv::LogicalAndOp::create(rewriter, loc, sameSign, nNonZero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
@@ -161,18 +161,18 @@ struct ConvertIndexCeilDivUPattern final : OpConversionPattern<CeilDivUOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value one = rewriter.create<spirv::ConstantOp>(loc, n_type,
+    Value one = spirv::ConstantOp::create(rewriter, loc, n_type,
                                                    IntegerAttr::get(n_type, 1));
 
     // Compute the non-zero result.
-    Value minusOne = rewriter.create<spirv::ISubOp>(loc, n, one);
-    Value quotient = rewriter.create<spirv::UDivOp>(loc, minusOne, m);
-    Value plusOne = rewriter.create<spirv::IAddOp>(loc, quotient, one);
+    Value minusOne = spirv::ISubOp::create(rewriter, loc, n, one);
+    Value quotient = spirv::UDivOp::create(rewriter, loc, minusOne, m);
+    Value plusOne = spirv::IAddOp::create(rewriter, loc, quotient, one);
 
     // Pick the result
-    Value cmp = rewriter.create<spirv::IEqualOp>(loc, n, zero);
+    Value cmp = spirv::IEqualOp::create(rewriter, loc, n, zero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, zero, plusOne);
     return success();
   }
@@ -197,32 +197,32 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = rewriter.create<spirv::ConstantOp>(
+    Value posOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = rewriter.create<spirv::ConstantOp>(
+    Value negOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
-    Value mNeg = rewriter.create<spirv::SLessThanOp>(loc, m, zero);
-    Value x = rewriter.create<spirv::SelectOp>(loc, mNeg, posOne, negOne);
+    Value mNeg = spirv::SLessThanOp::create(rewriter, loc, m, zero);
+    Value x = spirv::SelectOp::create(rewriter, loc, mNeg, posOne, negOne);
 
     // Compute the negative result
-    Value xMinusN = rewriter.create<spirv::ISubOp>(loc, x, n);
-    Value xMinusNDivM = rewriter.create<spirv::SDivOp>(loc, xMinusN, m);
-    Value negRes = rewriter.create<spirv::ISubOp>(loc, negOne, xMinusNDivM);
+    Value xMinusN = spirv::ISubOp::create(rewriter, loc, x, n);
+    Value xMinusNDivM = spirv::SDivOp::create(rewriter, loc, xMinusN, m);
+    Value negRes = spirv::ISubOp::create(rewriter, loc, negOne, xMinusNDivM);
 
     // Compute the positive result.
-    Value posRes = rewriter.create<spirv::SDivOp>(loc, n, m);
+    Value posRes = spirv::SDivOp::create(rewriter, loc, n, m);
 
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
-    Value nNeg = rewriter.create<spirv::SLessThanOp>(loc, n, zero);
-    Value diffSign = rewriter.create<spirv::LogicalNotEqualOp>(loc, nNeg, mNeg);
-    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
+    Value nNeg = spirv::SLessThanOp::create(rewriter, loc, n, zero);
+    Value diffSign = spirv::LogicalNotEqualOp::create(rewriter, loc, nNeg, mNeg);
+    Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
 
-    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, diffSign, nNonZero);
+    Value cmp = spirv::LogicalAndOp::create(rewriter, loc, diffSign, nNonZero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index e34d5f74d232f..e8ad0d4644389 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -32,7 +32,7 @@ MemRefDescriptor::MemRefDescriptor(Value descriptor)
 MemRefDescriptor MemRefDescriptor::poison(OpBuilder &builder, Location loc,
                                           Type descriptorType) {
 
-  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
+  Value descriptor = LLVM::PoisonOp::create(builder, loc, descriptorType);
   return MemRefDescriptor(descriptor);
 }
 
@@ -99,20 +99,20 @@ void MemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc,
 // integer attribute.
 static Value createIndexAttrConstant(OpBuilder &builder, Location loc,
                                      Type resultType, int64_t value) {
-  return builder.create<LLVM::ConstantOp>(loc, resultType,
+  return LLVM::ConstantOp::create(builder, loc, resultType,
                                           builder.getIndexAttr(value));
 }
 
 /// Builds IR extracting the offset from the descriptor.
 Value MemRefDescriptor::offset(OpBuilder &builder, Location loc) {
-  return builder.create<LLVM::ExtractValueOp>(loc, value,
+  return LLVM::ExtractValueOp::create(builder, loc, value,
                                               kOffsetPosInMemRefDescriptor);
 }
 
 /// Builds IR inserting the offset into the descriptor.
 void MemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                  Value offset) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, offset,
+  value = LLVM::InsertValueOp::create(builder, loc, value, offset,
                                               kOffsetPosInMemRefDescriptor);
 }
 
@@ -125,7 +125,7 @@ void MemRefDescriptor::setConstantOffset(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th size from the descriptor.
 Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) {
-  return builder.create<LLVM::ExtractValueOp>(
+  return LLVM::ExtractValueOp::create(builder,
       loc, value, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
@@ -137,22 +137,22 @@ Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
 
   // Copy size values to stack-allocated memory.
   auto one = createIndexAttrConstant(builder, loc, indexType, 1);
-  auto sizes = builder.create<LLVM::ExtractValueOp>(
+  auto sizes = LLVM::ExtractValueOp::create(builder,
       loc, value, llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
-  auto sizesPtr = builder.create<LLVM::AllocaOp>(loc, ptrTy, arrayTy, one,
+  auto sizesPtr = LLVM::AllocaOp::create(builder, loc, ptrTy, arrayTy, one,
                                                  /*alignment=*/0);
-  builder.create<LLVM::StoreOp>(loc, sizes, sizesPtr);
+  LLVM::StoreOp::create(builder, loc, sizes, sizesPtr);
 
   // Load an return size value of interest.
-  auto resultPtr = builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, sizesPtr,
+  auto resultPtr = LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, sizesPtr,
                                                ArrayRef<LLVM::GEPArg>{0, pos});
-  return builder.create<LLVM::LoadOp>(loc, indexType, resultPtr);
+  return LLVM::LoadOp::create(builder, loc, indexType, resultPtr);
 }
 
 /// Builds IR inserting the pos-th size into the descriptor
 void MemRefDescriptor::setSize(OpBuilder &builder, Location loc, unsigned pos,
                                Value size) {
-  value = builder.create<LLVM::InsertValueOp>(
+  value = LLVM::InsertValueOp::create(builder,
       loc, value, size, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
@@ -164,14 +164,14 @@ void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th stride from the descriptor.
 Value MemRefDescriptor::stride(OpBuilder &builder, Location loc, unsigned pos) {
-  return builder.create<LLVM::ExtractValueOp>(
+  return LLVM::ExtractValueOp::create(builder,
       loc, value, ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
 
 /// Builds IR inserting the pos-th stride into the descriptor
 void MemRefDescriptor::setStride(OpBuilder &builder, Location loc, unsigned pos,
                                  Value stride) {
-  value = builder.create<LLVM::InsertValueOp>(
+  value = LLVM::InsertValueOp::create(builder,
       loc, value, stride,
       ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
@@ -207,7 +207,7 @@ Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
           ? offset(builder, loc)
           : createIndexAttrConstant(builder, loc, indexType, offsetCst);
   Type elementType = converter.convertType(type.getElementType());
-  ptr = builder.create<LLVM::GEPOp>(loc, ptr.getType(), elementType, ptr,
+  ptr = LLVM::GEPOp::create(builder, loc, ptr.getType(), elementType, ptr,
                                     offsetVal);
   return ptr;
 }
@@ -303,7 +303,7 @@ UnrankedMemRefDescriptor::UnrankedMemRefDescriptor(Value descriptor)
 UnrankedMemRefDescriptor UnrankedMemRefDescriptor::poison(OpBuilder &builder,
                                                           Location loc,
                                                           Type descriptorType) {
-  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
+  Value descriptor = LLVM::PoisonOp::create(builder, loc, descriptorType);
   return UnrankedMemRefDescriptor(descriptor);
 }
 Value UnrankedMemRefDescriptor::rank(OpBuilder &builder, Location loc) const {
@@ -380,18 +380,18 @@ void UnrankedMemRefDescriptor::computeSizes(
         builder, loc, indexType,
         llvm::divideCeil(typeConverter.getPointerBitwidth(addressSpace), 8));
     Value doublePointerSize =
-        builder.create<LLVM::MulOp>(loc, indexType, two, pointerSize);
+        LLVM::MulOp::create(builder, loc, indexType, two, pointerSize);
 
     // (1 + 2 * rank) * sizeof(index)
     Value rank = desc.rank(builder, loc);
-    Value doubleRank = builder.create<LLVM::MulOp>(loc, indexType, two, rank);
+    Value doubleRank = LLVM::MulOp::create(builder, loc, indexType, two, rank);
     Value doubleRankIncremented =
-        builder.create<LLVM::AddOp>(loc, indexType, doubleRank, one);
-    Value rankIndexSize = builder.create<LLVM::MulOp>(
+        LLVM::AddOp::create(builder, loc, indexType, doubleRank, one);
+    Value rankIndexSize = LLVM::MulOp::create(builder,
         loc, indexType, doubleRankIncremented, indexSize);
 
     // Total allocation size.
-    Value allocationSize = builder.create<LLVM::AddOp>(
+    Value allocationSize = LLVM::AddOp::create(builder,
         loc, indexType, doublePointerSize, rankIndexSize);
     sizes.push_back(allocationSize);
   }
@@ -400,13 +400,13 @@ void UnrankedMemRefDescriptor::computeSizes(
 Value UnrankedMemRefDescriptor::allocatedPtr(
     OpBuilder &builder, Location loc, Value memRefDescPtr,
     LLVM::LLVMPointerType elemPtrType) {
-  return builder.create<LLVM::LoadOp>(loc, elemPtrType, memRefDescPtr);
+  return LLVM::LoadOp::create(builder, loc, elemPtrType, memRefDescPtr);
 }
 
 void UnrankedMemRefDescriptor::setAllocatedPtr(
     OpBuilder &builder, Location loc, Value memRefDescPtr,
     LLVM::LLVMPointerType elemPtrType, Value allocatedPtr) {
-  builder.create<LLVM::StoreOp>(loc, allocatedPtr, memRefDescPtr);
+  LLVM::StoreOp::create(builder, loc, allocatedPtr, memRefDescPtr);
 }
 
 static std::pair<Value, Type>
@@ -423,9 +423,9 @@ Value UnrankedMemRefDescriptor::alignedPtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   Value alignedGep =
-      builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+      LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                   elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
-  return builder.create<LLVM::LoadOp>(loc, elemPtrType, alignedGep);
+  return LLVM::LoadOp::create(builder, loc, elemPtrType, alignedGep);
 }
 
 void UnrankedMemRefDescriptor::setAlignedPtr(
@@ -435,9 +435,9 @@ void UnrankedMemRefDescriptor::setAlignedPtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   Value alignedGep =
-      builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+      LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                   elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
-  builder.create<LLVM::StoreOp>(loc, alignedPtr, alignedGep);
+  LLVM::StoreOp::create(builder, loc, alignedPtr, alignedGep);
 }
 
 Value UnrankedMemRefDescriptor::offsetBasePtr(
@@ -446,7 +446,7 @@ Value UnrankedMemRefDescriptor::offsetBasePtr(
   auto [elementPtrPtr, elemPtrPtrType] =
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
-  return builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+  return LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                      elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
 }
 
@@ -456,7 +456,7 @@ Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
                                        LLVM::LLVMPointerType elemPtrType) {
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
-  return builder.create<LLVM::LoadOp>(loc, typeConverter.getIndexType(),
+  return LLVM::LoadOp::create(builder, loc, typeConverter.getIndexType(),
                                       offsetPtr);
 }
 
@@ -467,7 +467,7 @@ void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                          Value offset) {
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
-  builder.create<LLVM::StoreOp>(loc, offset, offsetPtr);
+  LLVM::StoreOp::create(builder, loc, offset, offsetPtr);
 }
 
 Value UnrankedMemRefDescriptor::sizeBasePtr(
@@ -477,7 +477,7 @@ Value UnrankedMemRefDescriptor::sizeBasePtr(
   Type structTy = LLVM::LLVMStructType::getLiteral(
       indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy});
   auto resultType = LLVM::LLVMPointerType::get(builder.getContext());
-  return builder.create<LLVM::GEPOp>(loc, resultType, structTy, memRefDescPtr,
+  return LLVM::GEPOp::create(builder, loc, resultType, structTy, memRefDescPtr,
                                      ArrayRef<LLVM::GEPArg>{0, 3});
 }
 
@@ -489,8 +489,8 @@ Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value sizeStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, index);
-  return builder.create<LLVM::LoadOp>(loc, indexTy, sizeStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, index);
+  return LLVM::LoadOp::create(builder, loc, indexTy, sizeStoreGep);
 }
 
 void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
@@ -501,8 +501,8 @@ void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value sizeStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, index);
-  builder.create<LLVM::StoreOp>(loc, size, sizeStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, index);
+  LLVM::StoreOp::create(builder, loc, size, sizeStoreGep);
 }
 
 Value UnrankedMemRefDescriptor::strideBasePtr(
@@ -511,7 +511,7 @@ Value UnrankedMemRefDescriptor::strideBasePtr(
   Type indexTy = typeConverter.getIndexType();
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
-  return builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, rank);
+  return LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, rank);
 }
 
 Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
@@ -522,8 +522,8 @@ Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value strideStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, strideBasePtr, index);
-  return builder.create<LLVM::LoadOp>(loc, indexTy, strideStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, strideBasePtr, index);
+  return LLVM::LoadOp::create(builder, loc, indexTy, strideStoreGep);
 }
 
 void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
@@ -534,6 +534,6 @@ void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value strideStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, strideBasePtr, index);
-  builder.create<LLVM::StoreOp>(loc, stride, strideStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, strideBasePtr, index);
+  LLVM::StoreOp::create(builder, loc, stride, strideStoreGep);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index c5f72f7e10b8c..57aed79874ad0 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -57,7 +57,7 @@ Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder,
                                                     Location loc,
                                                     Type resultType,
                                                     int64_t value) {
-  return builder.create<LLVM::ConstantOp>(loc, resultType,
+  return LLVM::ConstantOp::create(builder, loc, resultType,
                                           builder.getIndexAttr(value));
 }
 
@@ -123,7 +123,7 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
       runningStride = sizes[i];
     else if (stride == ShapedType::kDynamic)
       runningStride =
-          rewriter.create<LLVM::MulOp>(loc, runningStride, sizes[i]);
+          LLVM::MulOp::create(rewriter, loc, runningStride, sizes[i]);
     else
       runningStride = createIndexAttrConstant(rewriter, loc, indexType, stride);
   }
@@ -131,10 +131,10 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
     // Buffer size in bytes.
     Type elementType = typeConverter->convertType(memRefType.getElementType());
     auto elementPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, elementPtrType);
-    Value gepPtr = rewriter.create<LLVM::GEPOp>(
+    Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
+    Value gepPtr = LLVM::GEPOp::create(rewriter,
         loc, elementPtrType, elementType, nullPtr, runningStride);
-    size = rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);
+    size = LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
   } else {
     size = runningStride;
   }
@@ -149,10 +149,10 @@ Value ConvertToLLVMPattern::getSizeInBytes(
   // which is a common pattern of getting the size of a type in bytes.
   Type llvmType = typeConverter->convertType(type);
   auto convertedPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-  auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, convertedPtrType);
-  auto gep = rewriter.create<LLVM::GEPOp>(loc, convertedPtrType, llvmType,
+  auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, convertedPtrType);
+  auto gep = LLVM::GEPOp::create(rewriter, loc, convertedPtrType, llvmType,
                                           nullPtr, ArrayRef<LLVM::GEPArg>{1});
-  return rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gep);
+  return LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gep);
 }
 
 Value ConvertToLLVMPattern::getNumElements(
@@ -175,7 +175,7 @@ Value ConvertToLLVMPattern::getNumElements(
           staticSize == ShapedType::kDynamic
               ? dynamicSizes[dynamicIndex++]
               : createIndexAttrConstant(rewriter, loc, indexType, staticSize);
-      numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
     } else {
       numElements =
           staticSize == ShapedType::kDynamic
@@ -276,14 +276,14 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
             ? builder
                   .create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
                   .getResult()
-            : builder.create<LLVM::AllocaOp>(loc, getPtrType(),
+            : LLVM::AllocaOp::create(builder, loc, getPtrType(),
                                              IntegerType::get(getContext(), 8),
                                              allocationSize,
                                              /*alignment=*/0);
     Value source = desc.memRefDescPtr(builder, loc);
-    builder.create<LLVM::MemcpyOp>(loc, memory, source, allocationSize, false);
+    LLVM::MemcpyOp::create(builder, loc, memory, source, allocationSize, false);
     if (!toDynamic)
-      builder.create<LLVM::CallOp>(loc, freeFunc.value(), source);
+      LLVM::CallOp::create(builder, loc, freeFunc.value(), source);
 
     // Create a new descriptor. The same descriptor can be returned multiple
     // times, attempting to modify its pointer can lead to memory leaks
@@ -349,7 +349,7 @@ LogicalResult LLVM::detail::oneToOneRewrite(
   SmallVector<Value, 4> results;
   results.reserve(numResults);
   for (unsigned i = 0; i < numResults; ++i) {
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+    results.push_back(LLVM::ExtractValueOp::create(rewriter,
         op->getLoc(), newOp->getResult(0), i));
   }
   rewriter.replaceOp(op, results);
@@ -371,7 +371,7 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   if (numResults != 0)
     resType = typeConverter.packOperationResults(op->getResultTypes());
 
-  auto callIntrOp = rewriter.create<LLVM::CallIntrinsicOp>(
+  auto callIntrOp = LLVM::CallIntrinsicOp::create(rewriter,
       loc, resType, rewriter.getStringAttr(intrinsic), operands);
   // Propagate attributes.
   callIntrOp->setAttrs(op->getAttrDictionary());
@@ -388,7 +388,7 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   results.reserve(numResults);
   Value intrRes = callIntrOp.getResults();
   for (unsigned i = 0; i < numResults; ++i)
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(loc, intrRes, i));
+    results.push_back(LLVM::ExtractValueOp::create(rewriter, loc, intrRes, i));
   rewriter.replaceOp(op, results);
 
   return success();
@@ -406,7 +406,7 @@ static unsigned getBitWidth(Type type) {
 static Value createI32Constant(OpBuilder &builder, Location loc,
                                int32_t value) {
   Type i32 = builder.getI32Type();
-  return builder.create<LLVM::ConstantOp>(loc, i32, value);
+  return LLVM::ConstantOp::create(builder, loc, i32, value);
 }
 
 SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
@@ -418,17 +418,17 @@ SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
   unsigned srcBitWidth = getBitWidth(srcType);
   unsigned dstBitWidth = getBitWidth(dstType);
   if (srcBitWidth == dstBitWidth) {
-    Value cast = builder.create<LLVM::BitcastOp>(loc, dstType, src);
+    Value cast = LLVM::BitcastOp::create(builder, loc, dstType, src);
     return {cast};
   }
 
   if (dstBitWidth > srcBitWidth) {
     auto smallerInt = builder.getIntegerType(srcBitWidth);
     if (srcType != smallerInt)
-      src = builder.create<LLVM::BitcastOp>(loc, smallerInt, src);
+      src = LLVM::BitcastOp::create(builder, loc, smallerInt, src);
 
     auto largerInt = builder.getIntegerType(dstBitWidth);
-    Value res = builder.create<LLVM::ZExtOp>(loc, largerInt, src);
+    Value res = LLVM::ZExtOp::create(builder, loc, largerInt, src);
     return {res};
   }
   assert(srcBitWidth % dstBitWidth == 0 &&
@@ -436,12 +436,12 @@ SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
   int64_t numElements = srcBitWidth / dstBitWidth;
   auto vecType = VectorType::get(numElements, dstType);
 
-  src = builder.create<LLVM::BitcastOp>(loc, vecType, src);
+  src = LLVM::BitcastOp::create(builder, loc, vecType, src);
 
   SmallVector<Value> res;
   for (auto i : llvm::seq(numElements)) {
     Value idx = createI32Constant(builder, loc, i);
-    Value elem = builder.create<LLVM::ExtractElementOp>(loc, src, idx);
+    Value elem = LLVM::ExtractElementOp::create(builder, loc, src, idx);
     res.emplace_back(elem);
   }
 
@@ -461,28 +461,28 @@ Value mlir::LLVM::composeValue(OpBuilder &builder, Location loc, ValueRange src,
     if (dstBitWidth < srcBitWidth) {
       auto largerInt = builder.getIntegerType(srcBitWidth);
       if (res.getType() != largerInt)
-        res = builder.create<LLVM::BitcastOp>(loc, largerInt, res);
+        res = LLVM::BitcastOp::create(builder, loc, largerInt, res);
 
       auto smallerInt = builder.getIntegerType(dstBitWidth);
-      res = builder.create<LLVM::TruncOp>(loc, smallerInt, res);
+      res = LLVM::TruncOp::create(builder, loc, smallerInt, res);
     }
 
     if (res.getType() != dstType)
-      res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+      res = LLVM::BitcastOp::create(builder, loc, dstType, res);
 
     return res;
   }
 
   int64_t numElements = src.size();
   auto srcType = VectorType::get(numElements, src.front().getType());
-  Value res = builder.create<LLVM::PoisonOp>(loc, srcType);
+  Value res = LLVM::PoisonOp::create(builder, loc, srcType);
   for (auto &&[i, elem] : llvm::enumerate(src)) {
     Value idx = createI32Constant(builder, loc, i);
-    res = builder.create<LLVM::InsertElementOp>(loc, srcType, res, elem, idx);
+    res = LLVM::InsertElementOp::create(builder, loc, srcType, res, elem, idx);
   }
 
   if (res.getType() != dstType)
-    res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+    res = LLVM::BitcastOp::create(builder, loc, dstType, res);
 
   return res;
 }
@@ -518,18 +518,18 @@ Value mlir::LLVM::getStridedElementPtr(OpBuilder &builder, Location loc,
       Value stride =
           ShapedType::isDynamic(strides[i])
               ? memRefDescriptor.stride(builder, loc, i)
-              : builder.create<LLVM::ConstantOp>(
+              : LLVM::ConstantOp::create(builder,
                     loc, indexType, builder.getIndexAttr(strides[i]));
       increment =
-          builder.create<LLVM::MulOp>(loc, increment, stride, intOverflowFlags);
+          LLVM::MulOp::create(builder, loc, increment, stride, intOverflowFlags);
     }
-    index = index ? builder.create<LLVM::AddOp>(loc, index, increment,
+    index = index ? LLVM::AddOp::create(builder, loc, index, increment,
                                                 intOverflowFlags)
                   : increment;
   }
 
   Type elementPtrType = memRefDescriptor.getElementPtrType();
-  return index ? builder.create<LLVM::GEPOp>(
+  return index ? LLVM::GEPOp::create(builder,
                      loc, elementPtrType,
                      converter.convertType(type.getElementType()), base, index,
                      noWrapFlags)
diff --git a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
index 49c73fbc9dd79..091fb7e4c2afb 100644
--- a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
@@ -66,7 +66,7 @@ LogicalResult mlir::LLVM::createPrintStrCall(
       DenseElementsAttr::get(dataAttrType, llvm::ArrayRef(elementVals));
   auto arrayTy =
       LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), elementVals.size());
-  auto globalOp = builder.create<LLVM::GlobalOp>(
+  auto globalOp = LLVM::GlobalOp::create(builder,
       loc, arrayTy, /*constant=*/true, LLVM::Linkage::Private,
       ensureSymbolNameIsUnique(moduleOp, symbolName, symbolTables), dataAttr);
 
@@ -74,15 +74,15 @@ LogicalResult mlir::LLVM::createPrintStrCall(
   // Emit call to `printStr` in runtime library.
   builder.restoreInsertionPoint(ip);
   auto msgAddr =
-      builder.create<LLVM::AddressOfOp>(loc, ptrTy, globalOp.getName());
+      LLVM::AddressOfOp::create(builder, loc, ptrTy, globalOp.getName());
   SmallVector<LLVM::GEPArg> indices(1, 0);
   Value gep =
-      builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, msgAddr, indices);
+      LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, msgAddr, indices);
   FailureOr<LLVM::LLVMFuncOp> printer =
       LLVM::lookupOrCreatePrintStringFn(builder, moduleOp, runtimeFunctionName);
   if (failed(printer))
     return failure();
-  builder.create<LLVM::CallOp>(loc, TypeRange(),
+  LLVM::CallOp::create(builder, loc, TypeRange(),
                                SymbolRefAttr::get(printer.value()), gep);
   return success();
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
index 1cd0bd85f9894..13ed4628c3c9e 100644
--- a/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
@@ -24,10 +24,10 @@ StructBuilder::StructBuilder(Value v) : value(v), structType(v.getType()) {
 
 Value StructBuilder::extractPtr(OpBuilder &builder, Location loc,
                                 unsigned pos) const {
-  return builder.create<LLVM::ExtractValueOp>(loc, value, pos);
+  return LLVM::ExtractValueOp::create(builder, loc, value, pos);
 }
 
 void StructBuilder::setPtr(OpBuilder &builder, Location loc, unsigned pos,
                            Value ptr) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, ptr, pos);
+  value = LLVM::InsertValueOp::create(builder, loc, value, ptr, pos);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 7312594c761f7..77135188dabcf 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -91,7 +91,7 @@ static Value unrankedMemRefMaterialization(OpBuilder &builder,
       packUnrankedMemRefDesc(builder, resultType, inputs, loc, converter);
   if (!packed)
     return Value();
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, packed)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, packed)
       .getResult(0);
 }
 
@@ -107,7 +107,7 @@ static Value rankedMemRefMaterialization(OpBuilder &builder,
       packRankedMemRefDesc(builder, resultType, inputs, loc, converter);
   if (!packed)
     return Value();
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, packed)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, packed)
       .getResult(0);
 }
 
@@ -224,12 +224,12 @@ LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
   // non-LLVM types persist after an LLVM conversion.
   addSourceMaterialization([&](OpBuilder &builder, Type resultType,
                                ValueRange inputs, Location loc) {
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
   addTargetMaterialization([&](OpBuilder &builder, Type resultType,
                                ValueRange inputs, Location loc) {
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
 
@@ -731,12 +731,12 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand,
   // Alloca with proper alignment. We do not expect optimizations of this
   // alloca op and so we omit allocating at the entry block.
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
-  Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
+  Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
                                                builder.getIndexAttr(1));
   Value allocated =
-      builder.create<LLVM::AllocaOp>(loc, ptrType, operand.getType(), one);
+      LLVM::AllocaOp::create(builder, loc, ptrType, operand.getType(), one);
   // Store into the alloca'ed descriptor.
-  builder.create<LLVM::StoreOp>(loc, operand, allocated);
+  LLVM::StoreOp::create(builder, loc, operand, allocated);
   return allocated;
 }
 
diff --git a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
index bf3f31729c3da..4fc208e05eb9d 100644
--- a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
@@ -87,17 +87,17 @@ LogicalResult LLVM::detail::handleMultidimensionalVectors(
   auto result1DVectorTy = resultTypeInfo.llvm1DVectorTy;
   auto resultNDVectoryTy = resultTypeInfo.llvmNDVectorTy;
   auto loc = op->getLoc();
-  Value desc = rewriter.create<LLVM::PoisonOp>(loc, resultNDVectoryTy);
+  Value desc = LLVM::PoisonOp::create(rewriter, loc, resultNDVectoryTy);
   nDVectorIterate(resultTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
     // For this unrolled `position` corresponding to the `linearIndex`^th
     // element, extract operand vectors
     SmallVector<Value, 4> extractedOperands;
     for (const auto &operand : llvm::enumerate(operands)) {
-      extractedOperands.push_back(rewriter.create<LLVM::ExtractValueOp>(
+      extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
           loc, operand.value(), position));
     }
     Value newVal = createOperand(result1DVectorTy, extractedOperands);
-    desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, newVal, position);
+    desc = LLVM::InsertValueOp::create(rewriter, loc, desc, newVal, position);
   });
   rewriter.replaceOp(op, desc);
   return success();
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index c3f213147b7a7..498a5cfc40204 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -78,7 +78,7 @@ getLibraryCallSymbolRef(Operation *op, PatternRewriter &rewriter) {
   // Insert before module terminator.
   rewriter.setInsertionPoint(module.getBody(),
                              std::prev(module.getBody()->end()));
-  func::FuncOp funcOp = rewriter.create<func::FuncOp>(
+  func::FuncOp funcOp = func::FuncOp::create(rewriter,
       op->getLoc(), fnNameAttr.getValue(), libFnType);
   // Insert a function attribute that will trigger the emission of the
   // corresponding `_mlir_ciface_xxx` interface so that external libraries see
@@ -101,7 +101,7 @@ createTypeCanonicalizedMemRefOperands(OpBuilder &b, Location loc,
       continue;
     }
     Value cast =
-        b.create<memref::CastOp>(loc, makeStridedLayoutDynamic(memrefType), op);
+        memref::CastOp::create(b, loc, makeStridedLayoutDynamic(memrefType), op);
     res.push_back(cast);
   }
   return res;
diff --git a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
index d4deff5b88070..30521d66a2e4c 100644
--- a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
+++ b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
@@ -54,18 +54,18 @@ std::pair<Value, Value> getRawPtrAndSize(const Location loc,
                                          Value memRef, Type elType) {
   Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
   Value dataPtr =
-      rewriter.create<LLVM::ExtractValueOp>(loc, ptrType, memRef, 1);
-  Value offset = rewriter.create<LLVM::ExtractValueOp>(
+      LLVM::ExtractValueOp::create(rewriter, loc, ptrType, memRef, 1);
+  Value offset = LLVM::ExtractValueOp::create(rewriter,
       loc, rewriter.getI64Type(), memRef, 2);
   Value resPtr =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, elType, dataPtr, offset);
+      LLVM::GEPOp::create(rewriter, loc, ptrType, elType, dataPtr, offset);
   Value size;
   if (cast<LLVM::LLVMStructType>(memRef.getType()).getBody().size() > 3) {
-    size = rewriter.create<LLVM::ExtractValueOp>(loc, memRef,
+    size = LLVM::ExtractValueOp::create(rewriter, loc, memRef,
                                                  ArrayRef<int64_t>{3, 0});
-    size = rewriter.create<LLVM::TruncOp>(loc, rewriter.getI32Type(), size);
+    size = LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), size);
   } else {
-    size = rewriter.create<arith::ConstantIntOp>(loc, 1, 32);
+    size = arith::ConstantIntOp::create(rewriter, loc, 1, 32);
   }
   return {resPtr, size};
 }
@@ -157,13 +157,13 @@ class MPICHImplTraits : public MPIImplTraits {
   Value getCommWorld(const Location loc,
                      ConversionPatternRewriter &rewriter) override {
     static constexpr int MPI_COMM_WORLD = 0x44000000;
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
                                              MPI_COMM_WORLD);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return rewriter.create<LLVM::TruncOp>(loc, rewriter.getI32Type(), comm);
+    return LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), comm);
   }
 
   intptr_t getStatusIgnore() override { return 1; }
@@ -195,7 +195,7 @@ class MPICHImplTraits : public MPIImplTraits {
       mtype = MPI_UINT8_T;
     else
       assert(false && "unsupported type");
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), mtype);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), mtype);
   }
 
   Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter,
@@ -245,7 +245,7 @@ class MPICHImplTraits : public MPIImplTraits {
       op = MPI_REPLACE;
       break;
     }
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), op);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), op);
   }
 };
 
@@ -281,15 +281,15 @@ class OMPIImplTraits : public MPIImplTraits {
     getOrDefineExternalStruct(loc, rewriter, name, commStructT);
 
     // get address of symbol
-    auto comm = rewriter.create<LLVM::AddressOfOp>(
+    auto comm = LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, name));
-    return rewriter.create<LLVM::PtrToIntOp>(loc, rewriter.getI64Type(), comm);
+    return LLVM::PtrToIntOp::create(rewriter, loc, rewriter.getI64Type(), comm);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return rewriter.create<LLVM::IntToPtrOp>(
+    return LLVM::IntToPtrOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(rewriter.getContext()), comm);
   }
 
@@ -330,7 +330,7 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, mtype, typeStructT);
     // get address of symbol
-    return rewriter.create<LLVM::AddressOfOp>(
+    return LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, mtype));
   }
@@ -389,7 +389,7 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, op, opStructT);
     // get address of symbol
-    return rewriter.create<LLVM::AddressOfOp>(
+    return LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, op));
   }
@@ -424,7 +424,7 @@ struct InitOpLowering : public ConvertOpToLLVMPattern<mpi::InitOp> {
     Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
 
     // instantiate nullptr `%nullptr = llvm.mlir.zero : !llvm.ptr`
-    auto nullPtrOp = rewriter.create<LLVM::ZeroOp>(loc, ptrType);
+    auto nullPtrOp = LLVM::ZeroOp::create(rewriter, loc, ptrType);
     Value llvmnull = nullPtrOp.getRes();
 
     // grab a reference to the global module op:
@@ -513,9 +513,9 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
 
     // get communicator
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, i32, 1);
+    auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
     auto outPtr =
-        rewriter.create<LLVM::AllocaOp>(loc, ptrType, comm.getType(), one);
+        LLVM::AllocaOp::create(rewriter, loc, ptrType, comm.getType(), one);
 
     // int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm * newcomm)
     auto funcType =
@@ -524,14 +524,14 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
     LLVM::LLVMFuncOp funcDecl = getOrDefineFunction(moduleOp, loc, rewriter,
                                                     "MPI_Comm_split", funcType);
 
-    auto callOp = rewriter.create<LLVM::CallOp>(
+    auto callOp = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{comm, adaptor.getColor(), adaptor.getKey(),
                    outPtr.getRes()});
 
     // load the communicator into a register
-    Value res = rewriter.create<LLVM::LoadOp>(loc, i32, outPtr.getResult());
-    res = rewriter.create<LLVM::SExtOp>(loc, rewriter.getI64Type(), res);
+    Value res = LLVM::LoadOp::create(rewriter, loc, i32, outPtr.getResult());
+    res = LLVM::SExtOp::create(rewriter, loc, rewriter.getI64Type(), res);
 
     // if retval is checked, replace uses of retval with the results from the
     // call op
@@ -580,14 +580,14 @@ struct CommRankOpLowering : public ConvertOpToLLVMPattern<mpi::CommRankOp> {
         moduleOp, loc, rewriter, "MPI_Comm_rank", rankFuncType);
 
     // replace with function call
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, i32, 1);
-    auto rankptr = rewriter.create<LLVM::AllocaOp>(loc, ptrType, i32, one);
-    auto callOp = rewriter.create<LLVM::CallOp>(
+    auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
+    auto rankptr = LLVM::AllocaOp::create(rewriter, loc, ptrType, i32, one);
+    auto callOp = LLVM::CallOp::create(rewriter,
         loc, initDecl, ValueRange{comm, rankptr.getRes()});
 
     // load the rank into a register
     auto loadedRank =
-        rewriter.create<LLVM::LoadOp>(loc, i32, rankptr.getResult());
+        LLVM::LoadOp::create(rewriter, loc, i32, rankptr.getResult());
 
     // if retval is checked, replace uses of retval with the results from the
     // call op
@@ -641,7 +641,7 @@ struct SendOpLowering : public ConvertOpToLLVMPattern<mpi::SendOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Send", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getDest(), adaptor.getTag(),
                    comm});
@@ -683,10 +683,10 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
     auto mpiTraits = MPIImplTraits::get(moduleOp);
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    Value statusIgnore = rewriter.create<LLVM::ConstantOp>(
+    Value statusIgnore = LLVM::ConstantOp::create(rewriter,
         loc, i64, mpiTraits->getStatusIgnore());
     statusIgnore =
-        rewriter.create<LLVM::IntToPtrOp>(loc, ptrType, statusIgnore);
+        LLVM::IntToPtrOp::create(rewriter, loc, ptrType, statusIgnore);
 
     // LLVM Function type representing `i32 MPI_Recv(data, count, datatype, dst,
     // tag, comm)`
@@ -698,7 +698,7 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Recv", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getSource(),
                    adaptor.getTag(), comm, statusIgnore});
@@ -738,9 +738,9 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
 
     // If input and output are the same, request in-place operation.
     if (adaptor.getSendbuf() == adaptor.getRecvbuf()) {
-      sendPtr = rewriter.create<LLVM::ConstantOp>(
+      sendPtr = LLVM::ConstantOp::create(rewriter,
           loc, i64, reinterpret_cast<int64_t>(mpiTraits->getInPlace()));
-      sendPtr = rewriter.create<LLVM::IntToPtrOp>(loc, ptrType, sendPtr);
+      sendPtr = LLVM::IntToPtrOp::create(rewriter, loc, ptrType, sendPtr);
     }
 
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
@@ -757,7 +757,7 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Allreduce", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{sendPtr, recvPtr, sendSize, dataType, mpiOp, commWorld});
 
diff --git a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
index 7f4655e53609e..bb3511440e041 100644
--- a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
+++ b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
@@ -121,7 +121,7 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     initValueAttr = FloatAttr::get(resultElementType, 0.0);
   else
     initValueAttr = IntegerAttr::get(resultElementType, 0);
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(vecType, initValueAttr));
   SmallVector<int64_t> strides = computeStrides(shape);
   for (int64_t linearIndex = 0; linearIndex < numElements; ++linearIndex) {
@@ -129,11 +129,11 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     SmallVector<Value> operands;
     for (Value input : op->getOperands())
       operands.push_back(
-          rewriter.create<vector::ExtractOp>(loc, input, positions));
+          vector::ExtractOp::create(rewriter, loc, input, positions));
     Value scalarOp =
-        rewriter.create<Op>(loc, vecType.getElementType(), operands);
+        Op::create(rewriter, loc, vecType.getElementType(), operands);
     result =
-        rewriter.create<vector::InsertOp>(loc, scalarOp, result, positions);
+        vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
   }
   rewriter.replaceOp(op, result);
   return success();
@@ -195,7 +195,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
 
   FunctionType funcType = FunctionType::get(
       builder.getContext(), {elementType, elementType}, elementType);
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
   LLVM::linkage::Linkage inlineLinkage = LLVM::linkage::Linkage::LinkonceODR;
   Attribute linkage =
       LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
@@ -208,11 +208,11 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value zeroValue = builder.create<arith::ConstantOp>(
+  Value zeroValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 0));
-  Value oneValue = builder.create<arith::ConstantOp>(
+  Value oneValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 1));
-  Value minusOneValue = builder.create<arith::ConstantOp>(
+  Value minusOneValue = arith::ConstantOp::create(builder,
       elementType,
       builder.getIntegerAttr(elementType,
                              APInt(elementType.getIntOrFloatBitWidth(), -1ULL,
@@ -221,81 +221,81 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   // if (p == T(0))
   //   return T(1);
   auto pIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroValue);
   Block *thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(pIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(pIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsZero, thenBlock, fallthroughBlock);
 
   // if (p < T(0)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto pIsNeg =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, pArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg, zeroValue);
   //   if (b == T(0))
   builder.createBlock(funcBody);
   auto bIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, zeroValue);
   //     return T(1) / T(0);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(
-      builder.create<arith::DivSIOp>(oneValue, zeroValue).getResult());
+  func::ReturnOp::create(builder,
+      arith::DivSIOp::create(builder, oneValue, zeroValue).getResult());
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(0)).
   builder.setInsertionPointToEnd(bIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(bIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, bIsZero, thenBlock, fallthroughBlock);
 
   //   if (b == T(1))
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto bIsOne =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bArg, oneValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, oneValue);
   //    return T(1);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(1)).
   builder.setInsertionPointToEnd(bIsOne->getBlock());
-  builder.create<cf::CondBranchOp>(bIsOne, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, bIsOne, thenBlock, fallthroughBlock);
 
   //   if (b == T(-1)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
-  auto bIsMinusOne = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto bIsMinusOne = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                    bArg, minusOneValue);
   //     if (p & T(1))
   builder.createBlock(funcBody);
-  auto pIsOdd = builder.create<arith::CmpIOp>(
-      arith::CmpIPredicate::ne, builder.create<arith::AndIOp>(pArg, oneValue),
+  auto pIsOdd = arith::CmpIOp::create(builder,
+      arith::CmpIPredicate::ne, arith::AndIOp::create(builder, pArg, oneValue),
       zeroValue);
   //       return T(-1);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(minusOneValue);
+  func::ReturnOp::create(builder, minusOneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(pIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(pIsOdd, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsOdd, thenBlock, fallthroughBlock);
 
   //     return T(1);
   //   } // b == T(-1)
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(-1)).
   builder.setInsertionPointToEnd(bIsMinusOne->getBlock());
-  builder.create<cf::CondBranchOp>(bIsMinusOne, pIsOdd->getBlock(),
+  cf::CondBranchOp::create(builder, bIsMinusOne, pIsOdd->getBlock(),
                                    fallthroughBlock);
 
   //   return T(0);
   // } // (p < T(0))
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<func::ReturnOp>(zeroValue);
+  func::ReturnOp::create(builder, zeroValue);
   Block *loopHeader = builder.createBlock(
       funcBody, funcBody->end(), {elementType, elementType, elementType},
       {builder.getLoc(), builder.getLoc(), builder.getLoc()});
   // Set up conditional branch for (p < T(0)).
   builder.setInsertionPointToEnd(pIsNeg->getBlock());
   // Set initial values of 'result', 'b' and 'p' for the loop.
-  builder.create<cf::CondBranchOp>(pIsNeg, bIsZero->getBlock(), loopHeader,
+  cf::CondBranchOp::create(builder, pIsNeg, bIsZero->getBlock(), loopHeader,
                                    ValueRange{oneValue, bArg, pArg});
 
   // T result = T(1);
@@ -313,44 +313,44 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & T(1))
-  auto powerTmpIsOdd = builder.create<arith::CmpIOp>(
+  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
       arith::CmpIPredicate::ne,
-      builder.create<arith::AndIOp>(powerTmp, oneValue), zeroValue);
+      arith::AndIOp::create(builder, powerTmp, oneValue), zeroValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
-  Value newResultTmp = builder.create<arith::MulIOp>(resultTmp, baseTmp);
+  Value newResultTmp = arith::MulIOp::create(builder, resultTmp, baseTmp);
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), elementType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(thenBlock);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(powerTmpIsOdd, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
                                    resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
   //   p >>= T(1);
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newPowerTmp = builder.create<arith::ShRUIOp>(powerTmp, oneValue);
+  Value newPowerTmp = arith::ShRUIOp::create(builder, powerTmp, oneValue);
 
   //   if (p == T(0))
-  auto newPowerIsZero = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                       newPowerTmp, zeroValue);
   //     return result;
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(newResultTmp);
+  func::ReturnOp::create(builder, newResultTmp);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(newPowerIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, newPowerIsZero, thenBlock, fallthroughBlock);
 
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newBaseTmp = builder.create<arith::MulIOp>(baseTmp, baseTmp);
+  Value newBaseTmp = arith::MulIOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  builder.create<cf::BranchOp>(
+  cf::BranchOp::create(builder,
       ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
   return funcOp;
 }
@@ -420,7 +420,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   llvm::raw_string_ostream nameOS(funcName);
   nameOS << '_' << baseType;
   nameOS << '_' << powType;
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
   LLVM::linkage::Linkage inlineLinkage = LLVM::linkage::Linkage::LinkonceODR;
   Attribute linkage =
       LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
@@ -433,46 +433,46 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value oneBValue = builder.create<arith::ConstantOp>(
+  Value oneBValue = arith::ConstantOp::create(builder,
       baseType, builder.getFloatAttr(baseType, 1.0));
-  Value zeroPValue = builder.create<arith::ConstantOp>(
+  Value zeroPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, 0));
-  Value onePValue = builder.create<arith::ConstantOp>(
+  Value onePValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, 1));
-  Value minPValue = builder.create<arith::ConstantOp>(
+  Value minPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMinValue(
                                                    powType.getWidth())));
-  Value maxPValue = builder.create<arith::ConstantOp>(
+  Value maxPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMaxValue(
                                                    powType.getWidth())));
 
   // if (p == Tp{0})
   //   return Tb{1};
   auto pIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, zeroPValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroPValue);
   Block *thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneBValue);
+  func::ReturnOp::create(builder, oneBValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == Tp{0}).
   builder.setInsertionPointToEnd(pIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(pIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsZero, thenBlock, fallthroughBlock);
 
   builder.setInsertionPointToEnd(fallthroughBlock);
   // bool isNegativePower{p < Tp{0}}
-  auto pIsNeg = builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, pArg,
+  auto pIsNeg = arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg,
                                               zeroPValue);
   // bool isMin{p == std::numeric_limits<Tp>::min()};
   auto pIsMin =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, minPValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, minPValue);
 
   // if (isMin) {
   //   p = std::numeric_limits<Tp>::max();
   // } else if (isNegativePower) {
   //   p = -p;
   // }
-  Value negP = builder.create<arith::SubIOp>(zeroPValue, pArg);
-  auto pInit = builder.create<arith::SelectOp>(pIsNeg, negP, pArg);
-  pInit = builder.create<arith::SelectOp>(pIsMin, maxPValue, pInit);
+  Value negP = arith::SubIOp::create(builder, zeroPValue, pArg);
+  auto pInit = arith::SelectOp::create(builder, pIsNeg, negP, pArg);
+  pInit = arith::SelectOp::create(builder, pIsMin, maxPValue, pInit);
 
   // Tb result = Tb{1};
   // Tb origBase = Tb{b};
@@ -489,7 +489,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
       {builder.getLoc(), builder.getLoc(), builder.getLoc()});
   // Set initial values of 'result', 'b' and 'p' for the loop.
   builder.setInsertionPointToEnd(pInit->getBlock());
-  builder.create<cf::BranchOp>(loopHeader, ValueRange{oneBValue, bArg, pInit});
+  cf::BranchOp::create(builder, loopHeader, ValueRange{oneBValue, bArg, pInit});
 
   // Create loop body.
   Value resultTmp = loopHeader->getArgument(0);
@@ -498,29 +498,29 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & Tp{1})
-  auto powerTmpIsOdd = builder.create<arith::CmpIOp>(
+  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
       arith::CmpIPredicate::ne,
-      builder.create<arith::AndIOp>(powerTmp, onePValue), zeroPValue);
+      arith::AndIOp::create(builder, powerTmp, onePValue), zeroPValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
-  Value newResultTmp = builder.create<arith::MulFOp>(resultTmp, baseTmp);
+  Value newResultTmp = arith::MulFOp::create(builder, resultTmp, baseTmp);
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(thenBlock);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
   // Set up conditional branch for (p & Tp{1}).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(powerTmpIsOdd, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
                                    resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
   //   p >>= Tp{1};
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newPowerTmp = builder.create<arith::ShRUIOp>(powerTmp, onePValue);
+  Value newPowerTmp = arith::ShRUIOp::create(builder, powerTmp, onePValue);
 
   //   if (p == Tp{0})
-  auto newPowerIsZero = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                       newPowerTmp, zeroPValue);
   //     break;
   //
@@ -531,9 +531,9 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newBaseTmp = builder.create<arith::MulFOp>(baseTmp, baseTmp);
+  Value newBaseTmp = arith::MulFOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  builder.create<cf::BranchOp>(
+  cf::BranchOp::create(builder,
       ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
 
   // Set up conditional branch for early loop exit:
@@ -542,7 +542,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Block *loopExit = builder.createBlock(funcBody, funcBody->end(), baseType,
                                         builder.getLoc());
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(newPowerIsZero, loopExit, newResultTmp,
+  cf::CondBranchOp::create(builder, newPowerIsZero, loopExit, newResultTmp,
                                    fallthroughBlock, ValueRange{});
 
   // if (isMin) {
@@ -553,11 +553,11 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(loopExit);
-  builder.create<cf::CondBranchOp>(pIsMin, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, pIsMin, thenBlock, fallthroughBlock,
                                    newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
-  newResultTmp = builder.create<arith::MulFOp>(newResultTmp, bArg);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  newResultTmp = arith::MulFOp::create(builder, newResultTmp, bArg);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
 
   /// if (isNegativePower) {
   ///   result = Tb{1} / result;
@@ -567,15 +567,15 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Block *returnBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                            builder.getLoc());
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<cf::CondBranchOp>(pIsNeg, thenBlock, returnBlock,
+  cf::CondBranchOp::create(builder, pIsNeg, thenBlock, returnBlock,
                                    newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
-  newResultTmp = builder.create<arith::DivFOp>(oneBValue, newResultTmp);
-  builder.create<cf::BranchOp>(newResultTmp, returnBlock);
+  newResultTmp = arith::DivFOp::create(builder, oneBValue, newResultTmp);
+  cf::BranchOp::create(builder, newResultTmp, returnBlock);
 
   // return result;
   builder.setInsertionPointToEnd(returnBlock);
-  builder.create<func::ReturnOp>(returnBlock->getArgument(0));
+  func::ReturnOp::create(builder, returnBlock->getArgument(0));
 
   return funcOp;
 }
@@ -667,7 +667,7 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
   nameOS << '_' << elementType;
   FunctionType funcType =
       FunctionType::get(builder.getContext(), {elementType}, elementType);
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
 
   // LinkonceODR ensures that there is only one implementation of this function
   // across all math.ctlz functions that are lowered in this way.
@@ -683,32 +683,32 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
 
   Value arg = funcOp.getArgument(0);
   Type indexType = builder.getIndexType();
-  Value bitWidthValue = builder.create<arith::ConstantOp>(
+  Value bitWidthValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, bitWidth));
-  Value zeroValue = builder.create<arith::ConstantOp>(
+  Value zeroValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 0));
 
   Value inputEqZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, arg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, arg, zeroValue);
 
   // if input == 0, return bit width, else enter loop.
-  scf::IfOp ifOp = builder.create<scf::IfOp>(
+  scf::IfOp ifOp = scf::IfOp::create(builder,
       elementType, inputEqZero, /*addThenBlock=*/true, /*addElseBlock=*/true);
   ifOp.getThenBodyBuilder().create<scf::YieldOp>(loc, bitWidthValue);
 
   auto elseBuilder =
       ImplicitLocOpBuilder::atBlockEnd(loc, &ifOp.getElseRegion().front());
 
-  Value oneIndex = elseBuilder.create<arith::ConstantOp>(
+  Value oneIndex = arith::ConstantOp::create(elseBuilder,
       indexType, elseBuilder.getIndexAttr(1));
-  Value oneValue = elseBuilder.create<arith::ConstantOp>(
+  Value oneValue = arith::ConstantOp::create(elseBuilder,
       elementType, elseBuilder.getIntegerAttr(elementType, 1));
-  Value bitWidthIndex = elseBuilder.create<arith::ConstantOp>(
+  Value bitWidthIndex = arith::ConstantOp::create(elseBuilder,
       indexType, elseBuilder.getIndexAttr(bitWidth));
-  Value nValue = elseBuilder.create<arith::ConstantOp>(
+  Value nValue = arith::ConstantOp::create(elseBuilder,
       elementType, elseBuilder.getIntegerAttr(elementType, 0));
 
-  auto loop = elseBuilder.create<scf::ForOp>(
+  auto loop = scf::ForOp::create(elseBuilder,
       oneIndex, bitWidthIndex, oneIndex,
       // Initial values for two loop induction variables, the arg which is being
       // shifted left in each iteration, and the n value which tracks the count
@@ -725,25 +725,25 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
         Value argIter = args[0];
         Value nIter = args[1];
 
-        Value argIsNonNegative = b.create<arith::CmpIOp>(
+        Value argIsNonNegative = arith::CmpIOp::create(b,
             loc, arith::CmpIPredicate::slt, argIter, zeroValue);
-        scf::IfOp ifOp = b.create<scf::IfOp>(
+        scf::IfOp ifOp = scf::IfOp::create(b,
             loc, argIsNonNegative,
             [&](OpBuilder &b, Location loc) {
               // If arg is negative, continue (effectively, break)
-              b.create<scf::YieldOp>(loc, ValueRange{argIter, nIter});
+              scf::YieldOp::create(b, loc, ValueRange{argIter, nIter});
             },
             [&](OpBuilder &b, Location loc) {
               // Otherwise, increment n and shift arg left.
-              Value nNext = b.create<arith::AddIOp>(loc, nIter, oneValue);
-              Value argNext = b.create<arith::ShLIOp>(loc, argIter, oneValue);
-              b.create<scf::YieldOp>(loc, ValueRange{argNext, nNext});
+              Value nNext = arith::AddIOp::create(b, loc, nIter, oneValue);
+              Value argNext = arith::ShLIOp::create(b, loc, argIter, oneValue);
+              scf::YieldOp::create(b, loc, ValueRange{argNext, nNext});
             });
-        b.create<scf::YieldOp>(loc, ifOp.getResults());
+        scf::YieldOp::create(b, loc, ifOp.getResults());
       });
-  elseBuilder.create<scf::YieldOp>(loop.getResult(1));
+  scf::YieldOp::create(elseBuilder, loop.getResult(1));
 
-  builder.create<func::ReturnOp>(ifOp.getResult(0));
+  func::ReturnOp::create(builder, ifOp.getResult(0));
   return funcOp;
 }
 
diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
index f4d69ce8235bb..26559f394c221 100644
--- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
+++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
@@ -107,7 +107,7 @@ struct IntOpWithFlagLowering : public ConvertOpToLLVMPattern<MathOp> {
     return LLVM::detail::handleMultidimensionalVectors(
         op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
-          return rewriter.create<LLVMOp>(loc, llvm1DVectorTy, operands[0],
+          return LLVMOp::create(rewriter, loc, llvm1DVectorTy, operands[0],
                                          false);
         },
         rewriter);
@@ -145,14 +145,14 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (LLVM::isCompatibleVectorType(llvmOperandType)) {
-        one = rewriter.create<LLVM::ConstantOp>(
+        one = LLVM::ConstantOp::create(rewriter,
             loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
+        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
-      auto exp = rewriter.create<LLVM::ExpOp>(loc, adaptor.getOperand(),
+      auto exp = LLVM::ExpOp::create(rewriter, loc, adaptor.getOperand(),
                                               expAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FSubOp>(
           op, llvmOperandType, ValueRange{exp, one}, subAttrs.getAttrs());
@@ -171,10 +171,10 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto exp = rewriter.create<LLVM::ExpOp>(
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto exp = LLVM::ExpOp::create(rewriter,
               loc, llvm1DVectorTy, operands[0], expAttrs.getAttrs());
-          return rewriter.create<LLVM::FSubOp>(
+          return LLVM::FSubOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{exp, one}, subAttrs.getAttrs());
         },
         rewriter);
@@ -205,14 +205,14 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one =
           isa<VectorType>(llvmOperandType)
-              ? rewriter.create<LLVM::ConstantOp>(
+              ? LLVM::ConstantOp::create(rewriter,
                     loc, llvmOperandType,
                     SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                            floatOne))
-              : rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType,
+              : LLVM::ConstantOp::create(rewriter, loc, llvmOperandType,
                                                   floatOne);
 
-      auto add = rewriter.create<LLVM::FAddOp>(
+      auto add = LLVM::FAddOp::create(rewriter,
           loc, llvmOperandType, ValueRange{one, adaptor.getOperand()},
           addAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::LogOp>(
@@ -232,11 +232,11 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto add = rewriter.create<LLVM::FAddOp>(loc, llvm1DVectorTy,
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto add = LLVM::FAddOp::create(rewriter, loc, llvm1DVectorTy,
                                                    ValueRange{one, operands[0]},
                                                    addAttrs.getAttrs());
-          return rewriter.create<LLVM::LogOp>(
+          return LLVM::LogOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{add}, logAttrs.getAttrs());
         },
         rewriter);
@@ -267,14 +267,14 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (isa<VectorType>(llvmOperandType)) {
-        one = rewriter.create<LLVM::ConstantOp>(
+        one = LLVM::ConstantOp::create(rewriter,
             loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
+        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
-      auto sqrt = rewriter.create<LLVM::SqrtOp>(loc, adaptor.getOperand(),
+      auto sqrt = LLVM::SqrtOp::create(rewriter, loc, adaptor.getOperand(),
                                                 sqrtAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FDivOp>(
           op, llvmOperandType, ValueRange{one, sqrt}, divAttrs.getAttrs());
@@ -293,10 +293,10 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto sqrt = rewriter.create<LLVM::SqrtOp>(
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto sqrt = LLVM::SqrtOp::create(rewriter,
               loc, llvm1DVectorTy, operands[0], sqrtAttrs.getAttrs());
-          return rewriter.create<LLVM::FDivOp>(
+          return LLVM::FDivOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{one, sqrt}, divAttrs.getAttrs());
         },
         rewriter);
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index a0ce7d3b75fc2..19303dc3e8b55 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -84,7 +84,7 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto shape = vecType.getShape();
   int64_t numElements = vecType.getNumElements();
 
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(
                vecType, FloatAttr::get(vecType.getElementType(), 0.0)));
   SmallVector<int64_t> strides = computeStrides(shape);
@@ -93,11 +93,11 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     SmallVector<Value> operands;
     for (auto input : op->getOperands())
       operands.push_back(
-          rewriter.create<vector::ExtractOp>(loc, input, positions));
+          vector::ExtractOp::create(rewriter, loc, input, positions));
     Value scalarOp =
-        rewriter.create<Op>(loc, vecType.getElementType(), operands);
+        Op::create(rewriter, loc, vecType.getElementType(), operands);
     result =
-        rewriter.create<vector::InsertOp>(loc, scalarOp, result, positions);
+        vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
   }
   rewriter.replaceOp(op, {result});
   return success();
@@ -114,9 +114,9 @@ PromoteOpToF32<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto f32 = rewriter.getF32Type();
   auto extendedOperands = llvm::to_vector(
       llvm::map_range(op->getOperands(), [&](Value operand) -> Value {
-        return rewriter.create<arith::ExtFOp>(loc, f32, operand);
+        return arith::ExtFOp::create(rewriter, loc, f32, operand);
       }));
-  auto newOp = rewriter.create<Op>(loc, f32, extendedOperands);
+  auto newOp = Op::create(rewriter, loc, f32, extendedOperands);
   rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, opType, newOp);
   return success();
 }
@@ -139,7 +139,7 @@ ScalarOpToLibmCall<Op>::matchAndRewrite(Op op,
     rewriter.setInsertionPointToStart(&module->getRegion(0).front());
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-    opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name,
+    opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
                                            opFunctionTy);
     opFunc.setPrivate();
 
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 59db14ed816be..26891aa7c2025 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -36,11 +36,11 @@ static Value getScalarOrVectorI32Constant(Type type, int value,
     if (!vectorType.getElementType().isInteger(32))
       return nullptr;
     SmallVector<int> values(vectorType.getNumElements(), value);
-    return builder.create<spirv::ConstantOp>(loc, type,
+    return spirv::ConstantOp::create(builder, loc, type,
                                              builder.getI32VectorAttr(values));
   }
   if (type.isInteger(32))
-    return builder.create<spirv::ConstantOp>(loc, type,
+    return spirv::ConstantOp::create(builder, loc, type,
                                              builder.getI32IntegerAttr(value));
 
   return nullptr;
@@ -144,9 +144,9 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
     Type intType = rewriter.getIntegerType(bitwidth);
     uint64_t intValue = uint64_t(1) << (bitwidth - 1);
 
-    Value signMask = rewriter.create<spirv::ConstantOp>(
+    Value signMask = spirv::ConstantOp::create(rewriter,
         loc, intType, rewriter.getIntegerAttr(intType, intValue));
-    Value valueMask = rewriter.create<spirv::ConstantOp>(
+    Value valueMask = spirv::ConstantOp::create(rewriter,
         loc, intType, rewriter.getIntegerAttr(intType, intValue - 1u));
 
     if (auto vectorType = dyn_cast<VectorType>(type)) {
@@ -156,24 +156,24 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
 
       SmallVector<Value> signSplat(count, signMask);
       signMask =
-          rewriter.create<spirv::CompositeConstructOp>(loc, intType, signSplat);
+          spirv::CompositeConstructOp::create(rewriter, loc, intType, signSplat);
 
       SmallVector<Value> valueSplat(count, valueMask);
-      valueMask = rewriter.create<spirv::CompositeConstructOp>(loc, intType,
+      valueMask = spirv::CompositeConstructOp::create(rewriter, loc, intType,
                                                                valueSplat);
     }
 
     Value lhsCast =
-        rewriter.create<spirv::BitcastOp>(loc, intType, adaptor.getLhs());
+        spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getLhs());
     Value rhsCast =
-        rewriter.create<spirv::BitcastOp>(loc, intType, adaptor.getRhs());
+        spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getRhs());
 
-    Value value = rewriter.create<spirv::BitwiseAndOp>(
+    Value value = spirv::BitwiseAndOp::create(rewriter,
         loc, intType, ValueRange{lhsCast, valueMask});
-    Value sign = rewriter.create<spirv::BitwiseAndOp>(
+    Value sign = spirv::BitwiseAndOp::create(rewriter,
         loc, intType, ValueRange{rhsCast, signMask});
 
-    Value result = rewriter.create<spirv::BitwiseOrOp>(loc, intType,
+    Value result = spirv::BitwiseOrOp::create(rewriter, loc, intType,
                                                        ValueRange{value, sign});
     rewriter.replaceOpWithNewOp<spirv::BitcastOp>(copySignOp, type, result);
     return success();
@@ -214,18 +214,18 @@ struct CountLeadingZerosPattern final
     Value val31 = getScalarOrVectorI32Constant(type, 31, rewriter, loc);
     Value val32 = getScalarOrVectorI32Constant(type, 32, rewriter, loc);
 
-    Value msb = rewriter.create<spirv::GLFindUMsbOp>(loc, input);
+    Value msb = spirv::GLFindUMsbOp::create(rewriter, loc, input);
     // We need to subtract from 31 given that the index returned by GLSL
     // FindUMsb is counted from the least significant bit. Theoretically this
     // also gives the correct result even if the integer has all zero bits, in
     // which case GL FindUMsb would return -1.
-    Value subMsb = rewriter.create<spirv::ISubOp>(loc, val31, msb);
+    Value subMsb = spirv::ISubOp::create(rewriter, loc, val31, msb);
     // However, certain Vulkan implementations have driver bugs for the corner
     // case where the input is zero. And.. it can be smart to optimize a select
     // only involving the corner case. So separately compute the result when the
     // input is either zero or one.
-    Value subInput = rewriter.create<spirv::ISubOp>(loc, val32, input);
-    Value cmp = rewriter.create<spirv::ULessThanEqualOp>(loc, input, val1);
+    Value subInput = spirv::ISubOp::create(rewriter, loc, val32, input);
+    Value cmp = spirv::ULessThanEqualOp::create(rewriter, loc, input, val1);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(countOp, cmp, subInput,
                                                  subMsb);
     return success();
@@ -253,7 +253,7 @@ struct ExpM1OpPattern final : public OpConversionPattern<math::ExpM1Op> {
     if (!type)
       return failure();
 
-    Value exp = rewriter.create<ExpOp>(loc, type, adaptor.getOperand());
+    Value exp = ExpOp::create(rewriter, loc, type, adaptor.getOperand());
     auto one = spirv::ConstantOp::getOne(type, loc, rewriter);
     rewriter.replaceOpWithNewOp<spirv::FSubOp>(operation, exp, one);
     return success();
@@ -283,7 +283,7 @@ struct Log1pOpPattern final : public OpConversionPattern<math::Log1pOp> {
 
     auto one = spirv::ConstantOp::getOne(type, operation.getLoc(), rewriter);
     Value onePlus =
-        rewriter.create<spirv::FAddOp>(loc, one, adaptor.getOperand());
+        spirv::FAddOp::create(rewriter, loc, one, adaptor.getOperand());
     rewriter.replaceOpWithNewOp<LogOp>(operation, type, onePlus);
     return success();
   }
@@ -321,14 +321,14 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
 
     auto getConstantValue = [&](double value) {
       if (auto floatType = dyn_cast<FloatType>(type)) {
-        return rewriter.create<spirv::ConstantOp>(
+        return spirv::ConstantOp::create(rewriter,
             loc, type, rewriter.getFloatAttr(floatType, value));
       }
       if (auto vectorType = dyn_cast<VectorType>(type)) {
         Type elemType = vectorType.getElementType();
 
         if (isa<FloatType>(elemType)) {
-          return rewriter.create<spirv::ConstantOp>(
+          return spirv::ConstantOp::create(rewriter,
               loc, type,
               DenseFPElementsAttr::get(
                   vectorType, FloatAttr::get(elemType, value).getValue()));
@@ -341,7 +341,7 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
     Value constantValue = getConstantValue(
         std::is_same<MathLogOp, math::Log2Op>() ? log2Reciprocal
                                                 : log10Reciprocal);
-    Value log = rewriter.create<SpirvLogOp>(loc, adaptor.getOperand());
+    Value log = SpirvLogOp::create(rewriter, loc, adaptor.getOperand());
     rewriter.replaceOpWithNewOp<spirv::FMulOp>(operation, type, log,
                                                constantValue);
     return success();
@@ -386,7 +386,7 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
     Location loc = powfOp.getLoc();
     Value zero = spirv::ConstantOp::getZero(operandType, loc, rewriter);
     Value lessThan =
-        rewriter.create<spirv::FOrdLessThanOp>(loc, adaptor.getLhs(), zero);
+        spirv::FOrdLessThanOp::create(rewriter, loc, adaptor.getLhs(), zero);
 
     // Per C/C++ spec:
     // > pow(base, exponent) returns NaN (and raises FE_INVALID) if base is
@@ -394,11 +394,11 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
     // Calculate the reminder from the exponent and check whether it is zero.
     Value floatOne = spirv::ConstantOp::getOne(operandType, loc, rewriter);
     Value expRem =
-        rewriter.create<spirv::FRemOp>(loc, adaptor.getRhs(), floatOne);
+        spirv::FRemOp::create(rewriter, loc, adaptor.getRhs(), floatOne);
     Value expRemNonZero =
-        rewriter.create<spirv::FOrdNotEqualOp>(loc, expRem, zero);
+        spirv::FOrdNotEqualOp::create(rewriter, loc, expRem, zero);
     Value cmpNegativeWithFractionalExp =
-        rewriter.create<spirv::LogicalAndOp>(loc, expRemNonZero, lessThan);
+        spirv::LogicalAndOp::create(rewriter, loc, expRemNonZero, lessThan);
     // Create NaN result and replace base value if conditions are met.
     const auto &floatSemantics = scalarFloatType.getFloatSemantics();
     const auto nan = APFloat::getNaN(floatSemantics);
@@ -407,10 +407,10 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
       nanAttr = DenseElementsAttr::get(vectorType, nan);
 
     Value NanValue =
-        rewriter.create<spirv::ConstantOp>(loc, operandType, nanAttr);
-    Value lhs = rewriter.create<spirv::SelectOp>(
+        spirv::ConstantOp::create(rewriter, loc, operandType, nanAttr);
+    Value lhs = spirv::SelectOp::create(rewriter,
         loc, cmpNegativeWithFractionalExp, NanValue, adaptor.getLhs());
-    Value abs = rewriter.create<spirv::GLFAbsOp>(loc, lhs);
+    Value abs = spirv::GLFAbsOp::create(rewriter, loc, lhs);
 
     // TODO: The following just forcefully casts y into an integer value in
     // order to properly propagate the sign, assuming integer y cases. It
@@ -418,18 +418,18 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
 
     // Cast exponent to integer and calculate exponent % 2 != 0.
     Value intRhs =
-        rewriter.create<spirv::ConvertFToSOp>(loc, intType, adaptor.getRhs());
+        spirv::ConvertFToSOp::create(rewriter, loc, intType, adaptor.getRhs());
     Value intOne = spirv::ConstantOp::getOne(intType, loc, rewriter);
     Value bitwiseAndOne =
-        rewriter.create<spirv::BitwiseAndOp>(loc, intRhs, intOne);
-    Value isOdd = rewriter.create<spirv::IEqualOp>(loc, bitwiseAndOne, intOne);
+        spirv::BitwiseAndOp::create(rewriter, loc, intRhs, intOne);
+    Value isOdd = spirv::IEqualOp::create(rewriter, loc, bitwiseAndOne, intOne);
 
     // calculate pow based on abs(lhs)^rhs.
-    Value pow = rewriter.create<spirv::GLPowOp>(loc, abs, adaptor.getRhs());
-    Value negate = rewriter.create<spirv::FNegateOp>(loc, pow);
+    Value pow = spirv::GLPowOp::create(rewriter, loc, abs, adaptor.getRhs());
+    Value negate = spirv::FNegateOp::create(rewriter, loc, pow);
     // if the exponent is odd and lhs < 0, negate the result.
     Value shouldNegate =
-        rewriter.create<spirv::LogicalAndOp>(loc, lessThan, isOdd);
+        spirv::LogicalAndOp::create(rewriter, loc, lessThan, isOdd);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(powfOp, shouldNegate, negate,
                                                  pow);
     return success();
@@ -455,22 +455,22 @@ struct RoundOpPattern final : public OpConversionPattern<math::RoundOp> {
     auto one = spirv::ConstantOp::getOne(ty, loc, rewriter);
     Value half;
     if (VectorType vty = dyn_cast<VectorType>(ty)) {
-      half = rewriter.create<spirv::ConstantOp>(
+      half = spirv::ConstantOp::create(rewriter,
           loc, vty,
           DenseElementsAttr::get(vty,
                                  rewriter.getFloatAttr(ety, 0.5).getValue()));
     } else {
-      half = rewriter.create<spirv::ConstantOp>(
+      half = spirv::ConstantOp::create(rewriter,
           loc, ty, rewriter.getFloatAttr(ety, 0.5));
     }
 
-    auto abs = rewriter.create<spirv::GLFAbsOp>(loc, operand);
-    auto floor = rewriter.create<spirv::GLFloorOp>(loc, abs);
-    auto sub = rewriter.create<spirv::FSubOp>(loc, abs, floor);
+    auto abs = spirv::GLFAbsOp::create(rewriter, loc, operand);
+    auto floor = spirv::GLFloorOp::create(rewriter, loc, abs);
+    auto sub = spirv::FSubOp::create(rewriter, loc, abs, floor);
     auto greater =
-        rewriter.create<spirv::FOrdGreaterThanEqualOp>(loc, sub, half);
-    auto select = rewriter.create<spirv::SelectOp>(loc, greater, one, zero);
-    auto add = rewriter.create<spirv::FAddOp>(loc, floor, select);
+        spirv::FOrdGreaterThanEqualOp::create(rewriter, loc, sub, half);
+    auto select = spirv::SelectOp::create(rewriter, loc, greater, one, zero);
+    auto add = spirv::FAddOp::create(rewriter, loc, floor, select);
     rewriter.replaceOpWithNewOp<math::CopySignOp>(roundOp, add, operand);
     return success();
   }
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
index db244d1d1cac8..db6ce600af76a 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
@@ -161,7 +161,7 @@ struct ConvertLoad final : public OpConversionPattern<memref::LoadOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = rewriter.create<emitc::SubscriptOp>(
+    auto subscript = emitc::SubscriptOp::create(rewriter,
         op.getLoc(), arrayValue, operands.getIndices());
 
     rewriter.replaceOpWithNewOp<emitc::LoadOp>(op, resultTy, subscript);
@@ -181,7 +181,7 @@ struct ConvertStore final : public OpConversionPattern<memref::StoreOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = rewriter.create<emitc::SubscriptOp>(
+    auto subscript = emitc::SubscriptOp::create(rewriter,
         op.getLoc(), arrayValue, operands.getIndices());
     rewriter.replaceOpWithNewOp<emitc::AssignOp>(op, subscript,
                                                  operands.getValue());
@@ -212,7 +212,7 @@ void mlir::populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   };
 
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 83681b2d5fd87..4ef2168380620 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -87,12 +87,12 @@ getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
 ///   aligned = bumped - bumped % alignment
 static Value createAligned(ConversionPatternRewriter &rewriter, Location loc,
                            Value input, Value alignment) {
-  Value one = rewriter.create<LLVM::ConstantOp>(loc, alignment.getType(),
+  Value one = LLVM::ConstantOp::create(rewriter, loc, alignment.getType(),
                                                 rewriter.getIndexAttr(1));
-  Value bump = rewriter.create<LLVM::SubOp>(loc, alignment, one);
-  Value bumped = rewriter.create<LLVM::AddOp>(loc, input, bump);
-  Value mod = rewriter.create<LLVM::URemOp>(loc, bumped, alignment);
-  return rewriter.create<LLVM::SubOp>(loc, bumped, mod);
+  Value bump = LLVM::SubOp::create(rewriter, loc, alignment, one);
+  Value bumped = LLVM::AddOp::create(rewriter, loc, input, bump);
+  Value mod = LLVM::URemOp::create(rewriter, loc, bumped, alignment);
+  return LLVM::SubOp::create(rewriter, loc, bumped, mod);
 }
 
 /// Computes the byte size for the MemRef element type.
@@ -123,7 +123,7 @@ static Value castAllocFuncResult(ConversionPatternRewriter &rewriter,
   assert(succeeded(maybeMemrefAddrSpace) && "unsupported address space");
   unsigned memrefAddrSpace = *maybeMemrefAddrSpace;
   if (allocatedPtrTy.getAddressSpace() != memrefAddrSpace)
-    allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+    allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
         allocatedPtr);
   return allocatedPtr;
@@ -168,14 +168,14 @@ class AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
     Value alignment = getAlignment(rewriter, loc, op);
     if (alignment) {
       // Adjust the allocation size to consider alignment.
-      sizeBytes = rewriter.create<LLVM::AddOp>(loc, sizeBytes, alignment);
+      sizeBytes = LLVM::AddOp::create(rewriter, loc, sizeBytes, alignment);
     }
 
     // Allocate the underlying buffer.
     Type elementPtrType = this->getElementPtrType(memRefType);
     assert(elementPtrType && "could not compute element ptr type");
     auto results =
-        rewriter.create<LLVM::CallOp>(loc, allocFuncOp.value(), sizeBytes);
+        LLVM::CallOp::create(rewriter, loc, allocFuncOp.value(), sizeBytes);
 
     Value allocatedPtr =
         castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
@@ -184,11 +184,11 @@ class AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
     if (alignment) {
       // Compute the aligned pointer.
       Value allocatedInt =
-          rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), allocatedPtr);
+          LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), allocatedPtr);
       Value alignmentInt =
           createAligned(rewriter, loc, allocatedInt, alignment);
       alignedPtr =
-          rewriter.create<LLVM::IntToPtrOp>(loc, elementPtrType, alignmentInt);
+          LLVM::IntToPtrOp::create(rewriter, loc, elementPtrType, alignmentInt);
     }
 
     // Create the MemRef descriptor.
@@ -268,7 +268,7 @@ class AlignedAllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
       sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
 
     Type elementPtrType = this->getElementPtrType(memRefType);
-    auto results = rewriter.create<LLVM::CallOp>(
+    auto results = LLVM::CallOp::create(rewriter,
         loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
 
     Value ptr =
@@ -360,7 +360,7 @@ struct AllocaOpLowering : public ConvertOpToLLVMPattern<memref::AllocaOp> {
     auto elementPtrType =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addrSpace);
 
-    auto allocatedElementPtr = rewriter.create<LLVM::AllocaOp>(
+    auto allocatedElementPtr = LLVM::AllocaOp::create(rewriter,
         loc, elementPtrType, elementType, size, op.getAlignment().value_or(0));
 
     // Create the MemRef descriptor.
@@ -397,7 +397,7 @@ struct AllocaScopeOpLowering
           remainingOpsBlock, allocaScopeOp.getResultTypes(),
           SmallVector<Location>(allocaScopeOp->getNumResults(),
                                 allocaScopeOp.getLoc()));
-      rewriter.create<LLVM::BrOp>(loc, ValueRange(), remainingOpsBlock);
+      LLVM::BrOp::create(rewriter, loc, ValueRange(), remainingOpsBlock);
     }
 
     // Inline body region.
@@ -407,8 +407,8 @@ struct AllocaScopeOpLowering
 
     // Save stack and then branch into the body of the region.
     rewriter.setInsertionPointToEnd(currentBlock);
-    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
-    rewriter.create<LLVM::BrOp>(loc, ValueRange(), beforeBody);
+    auto stackSaveOp = LLVM::StackSaveOp::create(rewriter, loc, getPtrType());
+    LLVM::BrOp::create(rewriter, loc, ValueRange(), beforeBody);
 
     // Replace the alloca_scope return with a branch that jumps out of the body.
     // Stack restore before leaving the body region.
@@ -420,7 +420,7 @@ struct AllocaScopeOpLowering
 
     // Insert stack restore before jumping out the body of the region.
     rewriter.setInsertionPoint(branchOp);
-    rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);
+    LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
 
     // Replace the op with values return from the body region.
     rewriter.replaceOp(allocaScopeOp, continueBlock->getArguments());
@@ -451,10 +451,10 @@ struct AssumeAlignmentOpLowering
     // This is more direct than ptrtoint-based checks, is explicitly supported,
     // and works with non-integral address spaces.
     Value trueCond =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getBoolAttr(true));
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getBoolAttr(true));
     Value alignmentConst =
         createIndexAttrConstant(rewriter, loc, getIndexType(), alignment);
-    rewriter.create<LLVM::AssumeOp>(loc, trueCond, LLVM::AssumeAlignTag(), ptr,
+    LLVM::AssumeOp::create(rewriter, loc, trueCond, LLVM::AssumeAlignTag(), ptr,
                                     alignmentConst);
     rewriter.replaceOp(op, memref);
     return success();
@@ -559,16 +559,16 @@ struct DimOpLowering : public ConvertOpToLLVMPattern<memref::DimOp> {
     // Get pointer to offset field of memref<element_type> descriptor.
     auto indexPtrTy =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addressSpace);
-    Value offsetPtr = rewriter.create<LLVM::GEPOp>(
+    Value offsetPtr = LLVM::GEPOp::create(rewriter,
         loc, indexPtrTy, elementType, underlyingRankedDesc,
         ArrayRef<LLVM::GEPArg>{0, 2});
 
     // The size value that we have to extract can be obtained using GEPop with
     // `dimOp.index() + 1` index argument.
-    Value idxPlusOne = rewriter.create<LLVM::AddOp>(
+    Value idxPlusOne = LLVM::AddOp::create(rewriter,
         loc, createIndexAttrConstant(rewriter, loc, getIndexType(), 1),
         adaptor.getIndex());
-    Value sizePtr = rewriter.create<LLVM::GEPOp>(
+    Value sizePtr = LLVM::GEPOp::create(rewriter,
         loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr,
         idxPlusOne);
     return rewriter
@@ -674,9 +674,9 @@ struct GenericAtomicRMWOpLowering
     auto memRefType = cast<MemRefType>(atomicOp.getMemref().getType());
     auto dataPtr = getStridedElementPtr(
         rewriter, loc, memRefType, adaptor.getMemref(), adaptor.getIndices());
-    Value init = rewriter.create<LLVM::LoadOp>(
+    Value init = LLVM::LoadOp::create(rewriter,
         loc, typeConverter->convertType(memRefType.getElementType()), dataPtr);
-    rewriter.create<LLVM::BrOp>(loc, init, loopBlock);
+    LLVM::BrOp::create(rewriter, loc, init, loopBlock);
 
     // Prepare the body of the loop block.
     rewriter.setInsertionPointToStart(loopBlock);
@@ -696,14 +696,14 @@ struct GenericAtomicRMWOpLowering
     // Append the cmpxchg op to the end of the loop block.
     auto successOrdering = LLVM::AtomicOrdering::acq_rel;
     auto failureOrdering = LLVM::AtomicOrdering::monotonic;
-    auto cmpxchg = rewriter.create<LLVM::AtomicCmpXchgOp>(
+    auto cmpxchg = LLVM::AtomicCmpXchgOp::create(rewriter,
         loc, dataPtr, loopArgument, result, successOrdering, failureOrdering);
     // Extract the %new_loaded and %ok values from the pair.
-    Value newLoaded = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 0);
-    Value ok = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 1);
+    Value newLoaded = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 0);
+    Value ok = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 1);
 
     // Conditionally branch to the end or back to the loop depending on %ok.
-    rewriter.create<LLVM::CondBrOp>(loc, ok, endBlock, ArrayRef<Value>(),
+    LLVM::CondBrOp::create(rewriter, loc, ok, endBlock, ArrayRef<Value>(),
                                     loopBlock, newLoaded);
 
     rewriter.setInsertionPointToEnd(endBlock);
@@ -796,8 +796,8 @@ class GlobalMemrefOpLowering : public ConvertOpToLLVMPattern<memref::GlobalOp> {
     if (!isExternal && isUninitialized) {
       rewriter.createBlock(&newGlobal.getInitializerRegion());
       Value undef[] = {
-          rewriter.create<LLVM::UndefOp>(newGlobal.getLoc(), arrayTy)};
-      rewriter.create<LLVM::ReturnOp>(newGlobal.getLoc(), undef);
+          LLVM::UndefOp::create(rewriter, newGlobal.getLoc(), arrayTy)};
+      LLVM::ReturnOp::create(rewriter, newGlobal.getLoc(), undef);
     }
     return success();
   }
@@ -842,11 +842,11 @@ struct GetGlobalMemrefOpLowering
     Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter());
     auto ptrTy = LLVM::LLVMPointerType::get(rewriter.getContext(), memSpace);
     auto addressOf =
-        rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, op.getName());
+        LLVM::AddressOfOp::create(rewriter, loc, ptrTy, op.getName());
 
     // Get the address of the first element in the array by creating a GEP with
     // the address of the GV as the base, and (rank + 1) number of 0 indices.
-    auto gep = rewriter.create<LLVM::GEPOp>(
+    auto gep = LLVM::GEPOp::create(rewriter,
         loc, ptrTy, arrayTy, addressOf,
         SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
 
@@ -857,7 +857,7 @@ struct GetGlobalMemrefOpLowering
     Value deadBeefConst =
         createIndexAttrConstant(rewriter, op->getLoc(), intPtrType, 0xdeadbeef);
     auto deadBeefPtr =
-        rewriter.create<LLVM::IntToPtrOp>(loc, ptrTy, deadBeefConst);
+        LLVM::IntToPtrOp::create(rewriter, loc, ptrTy, deadBeefConst);
 
     // Both allocated and aligned pointers are same. We could potentially stash
     // a nullptr for the allocated pointer since we do not expect any dealloc.
@@ -1009,7 +1009,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
           loc, adaptor.getSource(), rewriter);
 
       // rank = ConstantOp srcRank
-      auto rankVal = rewriter.create<LLVM::ConstantOp>(
+      auto rankVal = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(rank));
       // poison = PoisonOp
       UnrankedMemRefDescriptor memRefDesc =
@@ -1029,7 +1029,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
       auto ptr = memRefDesc.memRefDescPtr(rewriter, loc);
 
       // struct = LoadOp ptr
-      auto loadOp = rewriter.create<LLVM::LoadOp>(loc, targetStructType, ptr);
+      auto loadOp = LLVM::LoadOp::create(rewriter, loc, targetStructType, ptr);
       rewriter.replaceOp(memRefCastOp, loadOp.getResult());
     } else {
       llvm_unreachable("Unsupported unranked memref to unranked memref cast");
@@ -1063,31 +1063,31 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     MemRefDescriptor srcDesc(adaptor.getSource());
 
     // Compute number of elements.
-    Value numElements = rewriter.create<LLVM::ConstantOp>(
+    Value numElements = LLVM::ConstantOp::create(rewriter,
         loc, getIndexType(), rewriter.getIndexAttr(1));
     for (int pos = 0; pos < srcType.getRank(); ++pos) {
       auto size = srcDesc.size(rewriter, loc, pos);
-      numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
     }
 
     // Get element size.
     auto sizeInBytes = getSizeInBytes(loc, srcType.getElementType(), rewriter);
     // Compute total.
     Value totalSize =
-        rewriter.create<LLVM::MulOp>(loc, numElements, sizeInBytes);
+        LLVM::MulOp::create(rewriter, loc, numElements, sizeInBytes);
 
     Type elementType = typeConverter->convertType(srcType.getElementType());
 
     Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc);
     Value srcOffset = srcDesc.offset(rewriter, loc);
-    Value srcPtr = rewriter.create<LLVM::GEPOp>(
+    Value srcPtr = LLVM::GEPOp::create(rewriter,
         loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset);
     MemRefDescriptor targetDesc(adaptor.getTarget());
     Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc);
     Value targetOffset = targetDesc.offset(rewriter, loc);
-    Value targetPtr = rewriter.create<LLVM::GEPOp>(
+    Value targetPtr = LLVM::GEPOp::create(rewriter,
         loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset);
-    rewriter.create<LLVM::MemcpyOp>(loc, targetPtr, srcPtr, totalSize,
+    LLVM::MemcpyOp::create(rewriter, loc, targetPtr, srcPtr, totalSize,
                                     /*isVolatile=*/false);
     rewriter.eraseOp(op);
 
@@ -1103,7 +1103,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
 
     // First make sure we have an unranked memref descriptor representation.
     auto makeUnranked = [&, this](Value ranked, MemRefType type) {
-      auto rank = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      auto rank = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                     type.getRank());
       auto *typeConverter = getTypeConverter();
       auto ptr =
@@ -1116,7 +1116,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     };
 
     // Save stack position before promoting descriptors
-    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
+    auto stackSaveOp = LLVM::StackSaveOp::create(rewriter, loc, getPtrType());
 
     auto srcMemRefType = dyn_cast<MemRefType>(srcType);
     Value unrankedSource =
@@ -1128,13 +1128,13 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
                          : adaptor.getTarget();
 
     // Now promote the unranked descriptors to the stack.
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+    auto one = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                  rewriter.getIndexAttr(1));
     auto promote = [&](Value desc) {
       auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
       auto allocated =
-          rewriter.create<LLVM::AllocaOp>(loc, ptrType, desc.getType(), one);
-      rewriter.create<LLVM::StoreOp>(loc, desc, allocated);
+          LLVM::AllocaOp::create(rewriter, loc, ptrType, desc.getType(), one);
+      LLVM::StoreOp::create(rewriter, loc, desc, allocated);
       return allocated;
     };
 
@@ -1149,11 +1149,11 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
         sourcePtr.getType(), symbolTables);
     if (failed(copyFn))
       return failure();
-    rewriter.create<LLVM::CallOp>(loc, copyFn.value(),
+    LLVM::CallOp::create(rewriter, loc, copyFn.value(),
                                   ValueRange{elemSize, sourcePtr, targetPtr});
 
     // Restore stack used for descriptors
-    rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);
+    LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
 
     rewriter.eraseOp(op);
 
@@ -1204,9 +1204,9 @@ struct MemorySpaceCastOpLowering
       MemRefDescriptor::unpack(rewriter, loc, adaptor.getSource(), resultTypeR,
                                descVals);
       descVals[0] =
-          rewriter.create<LLVM::AddrSpaceCastOp>(loc, newPtrType, descVals[0]);
+          LLVM::AddrSpaceCastOp::create(rewriter, loc, newPtrType, descVals[0]);
       descVals[1] =
-          rewriter.create<LLVM::AddrSpaceCastOp>(loc, newPtrType, descVals[1]);
+          LLVM::AddrSpaceCastOp::create(rewriter, loc, newPtrType, descVals[1]);
       Value result = MemRefDescriptor::pack(rewriter, loc, *getTypeConverter(),
                                             resultTypeR, descVals);
       rewriter.replaceOp(op, result);
@@ -1241,7 +1241,7 @@ struct MemorySpaceCastOpLowering
       UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                              result, resultAddrSpace, sizes);
       Value resultUnderlyingSize = sizes.front();
-      Value resultUnderlyingDesc = rewriter.create<LLVM::AllocaOp>(
+      Value resultUnderlyingDesc = LLVM::AllocaOp::create(rewriter,
           loc, getPtrType(), rewriter.getI8Type(), resultUnderlyingSize);
       result.setMemRefDescPtr(rewriter, loc, resultUnderlyingDesc);
 
@@ -1256,9 +1256,9 @@ struct MemorySpaceCastOpLowering
       Value alignedPtr =
           sourceDesc.alignedPtr(rewriter, loc, *getTypeConverter(),
                                 sourceUnderlyingDesc, sourceElemPtrType);
-      allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+      allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
           loc, resultElemPtrType, allocatedPtr);
-      alignedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+      alignedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
           loc, resultElemPtrType, alignedPtr);
 
       result.setAllocatedPtr(rewriter, loc, resultUnderlyingDesc,
@@ -1277,11 +1277,11 @@ struct MemorySpaceCastOpLowering
       int64_t bytesToSkip =
           2 * llvm::divideCeil(
                   getTypeConverter()->getPointerBitwidth(resultAddrSpace), 8);
-      Value bytesToSkipConst = rewriter.create<LLVM::ConstantOp>(
+      Value bytesToSkipConst = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(bytesToSkip));
-      Value copySize = rewriter.create<LLVM::SubOp>(
+      Value copySize = LLVM::SubOp::create(rewriter,
           loc, getIndexType(), resultUnderlyingSize, bytesToSkipConst);
-      rewriter.create<LLVM::MemcpyOp>(loc, resultIndexVals, sourceIndexVals,
+      LLVM::MemcpyOp::create(rewriter, loc, resultIndexVals, sourceIndexVals,
                                       copySize, /*isVolatile=*/false);
 
       rewriter.replaceOp(op, ValueRange{result});
@@ -1485,7 +1485,7 @@ struct MemRefReshapeOpLowering
         } else {
           Value shapeOp = reshapeOp.getShape();
           Value index = createIndexAttrConstant(rewriter, loc, indexType, i);
-          dimSize = rewriter.create<memref::LoadOp>(loc, shapeOp, index);
+          dimSize = memref::LoadOp::create(rewriter, loc, shapeOp, index);
           Type indexType = getIndexType();
           if (dimSize.getType() != indexType)
             dimSize = typeConverter->materializeTargetConversion(
@@ -1497,7 +1497,7 @@ struct MemRefReshapeOpLowering
         desc.setStride(rewriter, loc, i, stride);
 
         // Prepare the stride value for the next dimension.
-        stride = rewriter.create<LLVM::MulOp>(loc, stride, dimSize);
+        stride = LLVM::MulOp::create(rewriter, loc, stride, dimSize);
       }
 
       *descriptor = desc;
@@ -1522,7 +1522,7 @@ struct MemRefReshapeOpLowering
     SmallVector<Value, 4> sizes;
     UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                            targetDesc, addressSpace, sizes);
-    Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>(
+    Value underlyingDescPtr = LLVM::AllocaOp::create(rewriter,
         loc, getPtrType(), IntegerType::get(getContext(), 8), sizes.front());
     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
 
@@ -1554,7 +1554,7 @@ struct MemRefReshapeOpLowering
     Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc);
     Value oneIndex = createIndexAttrConstant(rewriter, loc, getIndexType(), 1);
     Value resultRankMinusOne =
-        rewriter.create<LLVM::SubOp>(loc, resultRank, oneIndex);
+        LLVM::SubOp::create(rewriter, loc, resultRank, oneIndex);
 
     Block *initBlock = rewriter.getInsertionBlock();
     Type indexType = getTypeConverter()->getIndexType();
@@ -1568,14 +1568,14 @@ struct MemRefReshapeOpLowering
     rewriter.mergeBlocks(remainingBlock, condBlock, ValueRange());
 
     rewriter.setInsertionPointToEnd(initBlock);
-    rewriter.create<LLVM::BrOp>(loc, ValueRange({resultRankMinusOne, oneIndex}),
+    LLVM::BrOp::create(rewriter, loc, ValueRange({resultRankMinusOne, oneIndex}),
                                 condBlock);
     rewriter.setInsertionPointToStart(condBlock);
     Value indexArg = condBlock->getArgument(0);
     Value strideArg = condBlock->getArgument(1);
 
     Value zeroIndex = createIndexAttrConstant(rewriter, loc, indexType, 0);
-    Value pred = rewriter.create<LLVM::ICmpOp>(
+    Value pred = LLVM::ICmpOp::create(rewriter,
         loc, IntegerType::get(rewriter.getContext(), 1),
         LLVM::ICmpPredicate::sge, indexArg, zeroIndex);
 
@@ -1585,22 +1585,22 @@ struct MemRefReshapeOpLowering
 
     // Copy size from shape to descriptor.
     auto llvmIndexPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value sizeLoadGep = rewriter.create<LLVM::GEPOp>(
+    Value sizeLoadGep = LLVM::GEPOp::create(rewriter,
         loc, llvmIndexPtrType,
         typeConverter->convertType(shapeMemRefType.getElementType()),
         shapeOperandPtr, indexArg);
-    Value size = rewriter.create<LLVM::LoadOp>(loc, indexType, sizeLoadGep);
+    Value size = LLVM::LoadOp::create(rewriter, loc, indexType, sizeLoadGep);
     UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(),
                                       targetSizesBase, indexArg, size);
 
     // Write stride value and compute next one.
     UnrankedMemRefDescriptor::setStride(rewriter, loc, *getTypeConverter(),
                                         targetStridesBase, indexArg, strideArg);
-    Value nextStride = rewriter.create<LLVM::MulOp>(loc, strideArg, size);
+    Value nextStride = LLVM::MulOp::create(rewriter, loc, strideArg, size);
 
     // Decrement loop counter and branch back.
-    Value decrement = rewriter.create<LLVM::SubOp>(loc, indexArg, oneIndex);
-    rewriter.create<LLVM::BrOp>(loc, ValueRange({decrement, nextStride}),
+    Value decrement = LLVM::SubOp::create(rewriter, loc, indexArg, oneIndex);
+    LLVM::BrOp::create(rewriter, loc, ValueRange({decrement, nextStride}),
                                 condBlock);
 
     Block *remainder =
@@ -1608,7 +1608,7 @@ struct MemRefReshapeOpLowering
 
     // Hook up the cond exit to the remainder.
     rewriter.setInsertionPointToEnd(condBlock);
-    rewriter.create<LLVM::CondBrOp>(loc, pred, bodyBlock, ValueRange(),
+    LLVM::CondBrOp::create(rewriter, loc, pred, bodyBlock, ValueRange(),
                                     remainder, ValueRange());
 
     // Reset position to beginning of new remainder block.
@@ -1738,7 +1738,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
       return createIndexAttrConstant(rewriter, loc, indexType, strides[idx]);
     if (nextSize)
       return runningStride
-                 ? rewriter.create<LLVM::MulOp>(loc, runningStride, nextSize)
+                 ? LLVM::MulOp::create(rewriter, loc, runningStride, nextSize)
                  : nextSize;
     assert(!runningStride);
     return createIndexAttrConstant(rewriter, loc, indexType, 1);
@@ -1783,7 +1783,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
 
     // Field 2: Copy the actual aligned pointer to payload.
     Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc);
-    alignedPtr = rewriter.create<LLVM::GEPOp>(
+    alignedPtr = LLVM::GEPOp::create(rewriter,
         loc, alignedPtr.getType(),
         typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr,
         adaptor.getByteShift());
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index b866afbce98b0..7b7ad9b95b0ec 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -79,7 +79,7 @@ adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter,
   assert(indices.size() == 2);
   indices.back() = builder.createOrFold<spirv::SDivOp>(loc, lastDim, idx);
   Type t = typeConverter.convertType(op.getComponentPtr().getType());
-  return builder.create<spirv::AccessChainOp>(loc, t, op.getBasePtr(), indices);
+  return spirv::AccessChainOp::create(builder, loc, t, op.getBasePtr(), indices);
 }
 
 /// Casts the given `srcBool` into an integer of `dstType`.
@@ -107,7 +107,7 @@ static Value shiftValue(Location loc, Value value, Value offset, Value mask,
     value = castBoolToIntN(loc, value, dstType, builder);
   } else {
     if (valueBits < targetBits) {
-      value = builder.create<spirv::UConvertOp>(
+      value = spirv::UConvertOp::create(builder,
           loc, builder.getIntegerType(targetBits), value);
     }
 
@@ -372,7 +372,7 @@ AllocOpPattern::matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor,
     std::string varName =
         std::string("__workgroup_mem__") +
         std::to_string(std::distance(varOps.begin(), varOps.end()));
-    varOp = rewriter.create<spirv::GlobalVariableOp>(loc, spirvType, varName,
+    varOp = spirv::GlobalVariableOp::create(rewriter, loc, spirvType, varName,
                                                      /*initializer=*/nullptr);
   }
 
@@ -572,7 +572,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
           loadOp, "failed to determine memory requirements");
 
     auto [memoryAccess, alignment] = *memoryRequirements;
-    Value loadVal = rewriter.create<spirv::LoadOp>(loc, accessChain,
+    Value loadVal = spirv::LoadOp::create(rewriter, loc, accessChain,
                                                    memoryAccess, alignment);
     if (isBool)
       loadVal = castIntNToBool(loc, loadVal, rewriter);
@@ -601,7 +601,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
         loadOp, "failed to determine memory requirements");
 
   auto [memoryAccess, alignment] = *memoryRequirements;
-  Value spvLoadOp = rewriter.create<spirv::LoadOp>(loc, dstType, adjustedPtr,
+  Value spvLoadOp = spirv::LoadOp::create(rewriter, loc, dstType, adjustedPtr,
                                                    memoryAccess, alignment);
 
   // Shift the bits to the rightmost.
@@ -770,10 +770,10 @@ IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor,
   if (!scope)
     return rewriter.notifyMatchFailure(storeOp, "atomic scope not available");
 
-  Value result = rewriter.create<spirv::AtomicAndOp>(
+  Value result = spirv::AtomicAndOp::create(rewriter,
       loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
       clearBitsMask);
-  result = rewriter.create<spirv::AtomicOrOp>(
+  result = spirv::AtomicOrOp::create(rewriter,
       loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
       storeVal);
 
@@ -851,11 +851,11 @@ LogicalResult MemorySpaceCastOpPattern::matchAndRewrite(
   }
   if (sourceSc != spirv::StorageClass::Generic) {
     result =
-        rewriter.create<spirv::PtrCastToGenericOp>(loc, genericPtrType, result);
+        spirv::PtrCastToGenericOp::create(rewriter, loc, genericPtrType, result);
   }
   if (resultSc != spirv::StorageClass::Generic) {
     result =
-        rewriter.create<spirv::GenericCastToPtrOp>(loc, resultPtrType, result);
+        spirv::GenericCastToPtrOp::create(rewriter, loc, resultPtrType, result);
   }
   rewriter.replaceOp(addrCastOp, result);
   return success();
diff --git a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
index b93128441f2b5..f16b92a287d0a 100644
--- a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
+++ b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
@@ -65,7 +65,7 @@ static SmallVector<Value> getMixedAsValues(OpBuilder b, const Location &loc,
       values.emplace_back(*(dyn++));
     } else {
       TypedAttr val = type == i64 ? b.getI64IntegerAttr(s) : b.getIndexAttr(s);
-      values.emplace_back(b.create<arith::ConstantOp>(loc, type, val));
+      values.emplace_back(arith::ConstantOp::create(b, loc, type, val));
     }
   }
   return values;
@@ -79,9 +79,9 @@ static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
   SmallVector<Value> multiIndex(n);
 
   for (int i = n - 1; i >= 0; --i) {
-    multiIndex[i] = b.create<arith::RemSIOp>(loc, linearIndex, dimensions[i]);
+    multiIndex[i] = arith::RemSIOp::create(b, loc, linearIndex, dimensions[i]);
     if (i > 0)
-      linearIndex = b.create<arith::DivSIOp>(loc, linearIndex, dimensions[i]);
+      linearIndex = arith::DivSIOp::create(b, loc, linearIndex, dimensions[i]);
   }
 
   return multiIndex;
@@ -91,13 +91,13 @@ static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
 Value multiToLinearIndex(Location loc, OpBuilder b, ValueRange multiIndex,
                          ValueRange dimensions) {
 
-  Value linearIndex = b.create<arith::ConstantIndexOp>(loc, 0);
-  Value stride = b.create<arith::ConstantIndexOp>(loc, 1);
+  Value linearIndex = arith::ConstantIndexOp::create(b, loc, 0);
+  Value stride = arith::ConstantIndexOp::create(b, loc, 1);
 
   for (int i = multiIndex.size() - 1; i >= 0; --i) {
-    Value off = b.create<arith::MulIOp>(loc, multiIndex[i], stride);
-    linearIndex = b.create<arith::AddIOp>(loc, linearIndex, off);
-    stride = b.create<arith::MulIOp>(loc, stride, dimensions[i]);
+    Value off = arith::MulIOp::create(b, loc, multiIndex[i], stride);
+    linearIndex = arith::AddIOp::create(b, loc, linearIndex, off);
+    stride = arith::MulIOp::create(b, loc, stride, dimensions[i]);
   }
 
   return linearIndex;
@@ -144,10 +144,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     auto i64 = rewriter.getI64Type();
     std::array<int64_t, 2> shape = {static_cast<int64_t>(splitAxes.size()),
                                     maxNAxes};
-    Value resSplitAxes = rewriter.create<tensor::EmptyOp>(loc, shape, i16);
+    Value resSplitAxes = tensor::EmptyOp::create(rewriter, loc, shape, i16);
     auto attr = IntegerAttr::get(i16, -1);
-    Value fillValue = rewriter.create<arith::ConstantOp>(loc, i16, attr);
-    resSplitAxes = rewriter.create<linalg::FillOp>(loc, fillValue, resSplitAxes)
+    Value fillValue = arith::ConstantOp::create(rewriter, loc, i16, attr);
+    resSplitAxes = linalg::FillOp::create(rewriter, loc, fillValue, resSplitAxes)
                        .getResult(0);
 
     // explicitly write values into tensor row by row
@@ -162,8 +162,8 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       std::array<int64_t, 2> sizes = {1, size};
       auto tensorType = RankedTensorType::get({size}, i16);
       auto attrs = DenseIntElementsAttr::get(tensorType, axes.asArrayRef());
-      auto vals = rewriter.create<arith::ConstantOp>(loc, tensorType, attrs);
-      resSplitAxes = rewriter.create<tensor::InsertSliceOp>(
+      auto vals = arith::ConstantOp::create(rewriter, loc, tensorType, attrs);
+      resSplitAxes = tensor::InsertSliceOp::create(rewriter,
           loc, vals, resSplitAxes, empty, empty, empty, offs, sizes, strides);
     }
 
@@ -179,7 +179,7 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
                   .create<tensor::EmptyOp>(loc, std::array<int64_t, 2>{0, 0},
                                            i64)
                   .getResult()
-            : rewriter.create<tensor::FromElementsOp>(loc, type, haloSizes)
+            : tensor::FromElementsOp::create(rewriter, loc, type, haloSizes)
                   .getResult();
 
     // To hold sharded dims offsets, create Tensor with shape {nSplits,
@@ -189,7 +189,7 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     // MeshOp)
     Value resOffsets;
     if (adaptor.getStaticShardedDimsOffsets().empty()) {
-      resOffsets = rewriter.create<tensor::EmptyOp>(
+      resOffsets = tensor::EmptyOp::create(rewriter,
           loc, std::array<int64_t, 2>{0, 0}, i64);
     } else {
       SymbolTableCollection symbolTableCollection;
@@ -204,12 +204,12 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       assert(maxSplitSize);
       ++maxSplitSize; // add one for the total size
 
-      resOffsets = rewriter.create<tensor::EmptyOp>(
+      resOffsets = tensor::EmptyOp::create(rewriter,
           loc, std::array<int64_t, 2>{nSplits, maxSplitSize}, i64);
-      Value zero = rewriter.create<arith::ConstantOp>(
+      Value zero = arith::ConstantOp::create(rewriter,
           loc, i64, rewriter.getI64IntegerAttr(ShapedType::kDynamic));
       resOffsets =
-          rewriter.create<linalg::FillOp>(loc, zero, resOffsets).getResult(0);
+          linalg::FillOp::create(rewriter, loc, zero, resOffsets).getResult(0);
       SmallVector<Value> offsets =
           getMixedAsValues(rewriter, loc, adaptor.getStaticShardedDimsOffsets(),
                            adaptor.getDynamicShardedDimsOffsets());
@@ -220,10 +220,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
         assert(splitSize != ShapedType::kDynamic && splitSize < maxSplitSize);
         ++splitSize; // add one for the total size
         ArrayRef<Value> values(&offsets[curr], splitSize);
-        Value vals = rewriter.create<tensor::FromElementsOp>(loc, values);
+        Value vals = tensor::FromElementsOp::create(rewriter, loc, values);
         std::array<int64_t, 2> offs = {static_cast<int64_t>(i), 0};
         std::array<int64_t, 2> sizes = {1, splitSize};
-        resOffsets = rewriter.create<tensor::InsertSliceOp>(
+        resOffsets = tensor::InsertSliceOp::create(rewriter,
             loc, vals, resOffsets, empty, empty, empty, offs, sizes, strides);
         curr += splitSize;
       }
@@ -236,10 +236,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       return failure();
 
     resSplitAxes =
-        rewriter.create<tensor::CastOp>(loc, resTypes[0], resSplitAxes);
+        tensor::CastOp::create(rewriter, loc, resTypes[0], resSplitAxes);
     resHaloSizes =
-        rewriter.create<tensor::CastOp>(loc, resTypes[1], resHaloSizes);
-    resOffsets = rewriter.create<tensor::CastOp>(loc, resTypes[2], resOffsets);
+        tensor::CastOp::create(rewriter, loc, resTypes[1], resHaloSizes);
+    resOffsets = tensor::CastOp::create(rewriter, loc, resTypes[2], resOffsets);
 
     rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
         op, TupleType::get(op.getContext(), resTypes),
@@ -269,9 +269,9 @@ struct ConvertProcessMultiIndexOp
     SmallVector<Value> dims;
     llvm::transform(
         meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
-          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+          return arith::ConstantIndexOp::create(rewriter, loc, i).getResult();
         });
-    Value rank = rewriter.create<ProcessLinearIndexOp>(op.getLoc(), meshOp);
+    Value rank = ProcessLinearIndexOp::create(rewriter, op.getLoc(), meshOp);
     auto mIdx = linearToMultiIndex(loc, rewriter, rank, dims);
 
     // optionally extract subset of mesh axes
@@ -302,7 +302,7 @@ class ConvertProcessLinearIndexOp
     Location loc = op.getLoc();
     auto ctx = op.getContext();
     Value commWorld =
-        rewriter.create<mpi::CommWorldOp>(loc, mpi::CommType::get(ctx));
+        mpi::CommWorldOp::create(rewriter, loc, mpi::CommType::get(ctx));
     auto rank =
         rewriter
             .create<mpi::CommRankOp>(
@@ -341,40 +341,40 @@ struct ConvertNeighborsLinearIndicesOp
     SmallVector<Value> dims;
     llvm::transform(
         meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
-          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+          return arith::ConstantIndexOp::create(rewriter, loc, i).getResult();
         });
     Value dimSz = dims[axes[0]];
-    Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    Value minus1 = rewriter.create<arith::ConstantIndexOp>(loc, -1);
-    Value atBorder = rewriter.create<arith::CmpIOp>(
+    Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    Value minus1 = arith::ConstantIndexOp::create(rewriter, loc, -1);
+    Value atBorder = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sle, orgIdx,
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
-    auto down = rewriter.create<scf::IfOp>(
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
+    auto down = scf::IfOp::create(rewriter,
         loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
-          builder.create<scf::YieldOp>(loc, minus1);
+          scf::YieldOp::create(builder, loc, minus1);
         },
         [&](OpBuilder &builder, Location loc) {
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
-              rewriter.create<arith::SubIOp>(op.getLoc(), orgIdx, one)
+              arith::SubIOp::create(rewriter, op.getLoc(), orgIdx, one)
                   .getResult();
-          builder.create<scf::YieldOp>(
+          scf::YieldOp::create(builder,
               loc, multiToLinearIndex(loc, rewriter, tmp, dims));
         });
-    atBorder = rewriter.create<arith::CmpIOp>(
+    atBorder = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, orgIdx,
-        rewriter.create<arith::SubIOp>(loc, dimSz, one).getResult());
-    auto up = rewriter.create<scf::IfOp>(
+        arith::SubIOp::create(rewriter, loc, dimSz, one).getResult());
+    auto up = scf::IfOp::create(rewriter,
         loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
-          builder.create<scf::YieldOp>(loc, minus1);
+          scf::YieldOp::create(builder, loc, minus1);
         },
         [&](OpBuilder &builder, Location loc) {
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
-              rewriter.create<arith::AddIOp>(op.getLoc(), orgIdx, one);
-          builder.create<scf::YieldOp>(
+              arith::AddIOp::create(rewriter, op.getLoc(), orgIdx, one);
+          scf::YieldOp::create(builder,
               loc, multiToLinearIndex(loc, rewriter, tmp, dims));
         });
     rewriter.replaceOp(op, ValueRange{down.getResult(0), up.getResult(0)});
@@ -447,7 +447,7 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           rewriter, loc, sharding.getStaticShardedDimsOffsets(),
           sharding.getDynamicShardedDimsOffsets(), index);
       if (!tmp.empty())
-        shardedDimsOffs = rewriter.create<tensor::FromElementsOp>(
+        shardedDimsOffs = tensor::FromElementsOp::create(rewriter,
             loc, RankedTensorType::get({(int64_t)tmp.size()}, index), tmp);
     }
 
@@ -457,9 +457,9 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
     int64_t pos = 0;
     SmallVector<Value> shardShape;
     Value zero =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(index));
     Value one =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getOneAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getOneAttr(index));
 
     // Iterate over the dimensions of the tensor shape, get their split Axes,
     // and compute the sharded shape.
@@ -470,7 +470,7 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
         // The current dimension might not be sharded.
         // Create a value from the static position in shardDimsOffsets.
         Value posVal =
-            rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(pos));
+            arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(pos));
         // Get the index of the local shard in the mesh axis.
         Value idx = multiIdx[axes[0]];
         auto numShards =
@@ -482,29 +482,29 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
             return op->emitError() << "Only single axis sharding is "
                                    << "supported for each dimension.";
           }
-          idx = rewriter.create<arith::AddIOp>(loc, posVal, idx);
+          idx = arith::AddIOp::create(rewriter, loc, posVal, idx);
           // Compute size = shardedDimsOffs[idx+1] - shardedDimsOffs[idx].
           Value off =
-              rewriter.create<tensor::ExtractOp>(loc, shardedDimsOffs, idx);
-          idx = rewriter.create<arith::AddIOp>(loc, idx, one);
+              tensor::ExtractOp::create(rewriter, loc, shardedDimsOffs, idx);
+          idx = arith::AddIOp::create(rewriter, loc, idx, one);
           Value nextOff =
-              rewriter.create<tensor::ExtractOp>(loc, shardedDimsOffs, idx);
-          Value sz = rewriter.create<arith::SubIOp>(loc, nextOff, off);
+              tensor::ExtractOp::create(rewriter, loc, shardedDimsOffs, idx);
+          Value sz = arith::SubIOp::create(rewriter, loc, nextOff, off);
           shardShape.emplace_back(sz);
         } else {
-          Value numShardsVal = rewriter.create<arith::ConstantOp>(
+          Value numShardsVal = arith::ConstantOp::create(rewriter,
               loc, rewriter.getIndexAttr(numShards));
           // Compute shard dim size by distributing odd elements to trailing
           // shards:
           // sz = dim / numShards
           //      + (idx >= (numShards - (dim % numShards)) ? 1 : 0)
-          Value sz = rewriter.create<arith::DivSIOp>(loc, dim, numShardsVal);
-          Value sz1 = rewriter.create<arith::RemSIOp>(loc, dim, numShardsVal);
-          sz1 = rewriter.create<arith::SubIOp>(loc, numShardsVal, sz1);
-          auto cond = rewriter.create<arith::CmpIOp>(
+          Value sz = arith::DivSIOp::create(rewriter, loc, dim, numShardsVal);
+          Value sz1 = arith::RemSIOp::create(rewriter, loc, dim, numShardsVal);
+          sz1 = arith::SubIOp::create(rewriter, loc, numShardsVal, sz1);
+          auto cond = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::sge, idx, sz1);
-          Value odd = rewriter.create<arith::SelectOp>(loc, cond, one, zero);
-          sz = rewriter.create<arith::AddIOp>(loc, sz, odd);
+          Value odd = arith::SelectOp::create(rewriter, loc, cond, one, zero);
+          sz = arith::AddIOp::create(rewriter, loc, sz, odd);
           shardShape.emplace_back(sz);
         }
         pos += numShards + 1; // add one for the total size.
@@ -568,7 +568,7 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (isa<RankedTensorType>(input.getType())) {
       auto memrefType = MemRefType::get(
           inputShape, cast<ShapedType>(input.getType()).getElementType());
-      input = iBuilder.create<bufferization::ToBufferOp>(memrefType, input);
+      input = bufferization::ToBufferOp::create(iBuilder, memrefType, input);
     }
     MemRefType inType = cast<MemRefType>(input.getType());
 
@@ -577,15 +577,15 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     for (auto i = 0; i < inType.getRank(); ++i) {
       auto s = inputShape[i];
       if (ShapedType::isDynamic(s))
-        shape[i] = iBuilder.create<memref::DimOp>(input, s).getResult();
+        shape[i] = memref::DimOp::create(iBuilder, input, s).getResult();
       else
         shape[i] = iBuilder.getIndexAttr(s);
     }
 
     // Allocate buffer and copy input to buffer.
-    Value buffer = iBuilder.create<memref::AllocOp>(
+    Value buffer = memref::AllocOp::create(iBuilder,
         shape, cast<ShapedType>(op.getType()).getElementType());
-    iBuilder.create<linalg::CopyOp>(input, buffer);
+    linalg::CopyOp::create(iBuilder, input, buffer);
 
     // Get an MPI_Comm_split for the AllReduce operation.
     // The color is the linear index of the process in the mesh along the
@@ -594,9 +594,9 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
                                        iBuilder.getIndexType());
     SmallVector<Value> myMultiIndex =
-        iBuilder.create<ProcessMultiIndexOp>(indexResultTypes, mesh)
+        ProcessMultiIndexOp::create(iBuilder, indexResultTypes, mesh)
             .getResult();
-    Value zero = iBuilder.create<arith::ConstantIndexOp>(0);
+    Value zero = arith::ConstantIndexOp::create(iBuilder, 0);
     SmallVector<Value> multiKey(myMultiIndex.size(), zero);
 
     auto redAxes = adaptor.getMeshAxes();
@@ -607,15 +607,15 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
 
     Value color =
         createProcessLinearIndex(mesh, myMultiIndex, redAxes, iBuilder);
-    color = iBuilder.create<arith::IndexCastOp>(iBuilder.getI32Type(), color);
+    color = arith::IndexCastOp::create(iBuilder, iBuilder.getI32Type(), color);
     Value key = createProcessLinearIndex(mesh, multiKey, redAxes, iBuilder);
-    key = iBuilder.create<arith::IndexCastOp>(iBuilder.getI32Type(), key);
+    key = arith::IndexCastOp::create(iBuilder, iBuilder.getI32Type(), key);
 
     // Finally split the communicator
     auto commType = mpi::CommType::get(op->getContext());
-    Value commWorld = iBuilder.create<mpi::CommWorldOp>(commType);
+    Value commWorld = mpi::CommWorldOp::create(iBuilder, commType);
     auto comm =
-        iBuilder.create<mpi::CommSplitOp>(commType, commWorld, color, key)
+        mpi::CommSplitOp::create(iBuilder, commType, commWorld, color, key)
             .getNewcomm();
 
     Value buffer1d = buffer;
@@ -623,18 +623,18 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (inType.getRank() > 1) {
       ReassociationIndices reassociation(inType.getRank());
       std::iota(reassociation.begin(), reassociation.end(), 0);
-      buffer1d = iBuilder.create<memref::CollapseShapeOp>(
+      buffer1d = memref::CollapseShapeOp::create(iBuilder,
           buffer, ArrayRef<ReassociationIndices>(reassociation));
     }
 
     // Create the MPI AllReduce operation.
-    iBuilder.create<mpi::AllReduceOp>(
+    mpi::AllReduceOp::create(iBuilder,
         TypeRange(), buffer1d, buffer1d,
         getMPIReductionOp(adaptor.getReductionAttr()), comm);
 
     // If the destination is a memref, cast it to a tensor
     if (isa<RankedTensorType>(op.getType()))
-      buffer = iBuilder.create<bufferization::ToTensorOp>(op.getType(), buffer,
+      buffer = bufferization::ToTensorOp::create(iBuilder, op.getType(), buffer,
                                                           true);
 
     rewriter.replaceOp(op, buffer);
@@ -676,7 +676,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     auto toValue = [&rewriter, &loc](OpFoldResult &v) -> Value {
       if (auto value = dyn_cast<Value>(v))
         return value;
-      return rewriter.create<arith::ConstantOp>(
+      return arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(
                    cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
     };
@@ -689,7 +689,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       auto mmemrefType = MemRefType::get(
           dstShape, cast<ShapedType>(array.getType()).getElementType());
       array =
-          rewriter.create<bufferization::ToBufferOp>(loc, mmemrefType, array);
+          bufferization::ToBufferOp::create(rewriter, loc, mmemrefType, array);
     }
     auto rank = cast<ShapedType>(array.getType()).getRank();
     auto opSplitAxes = adaptor.getSplitAxes().getAxes();
@@ -713,7 +713,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     for (auto i = 0; i < rank; ++i) {
       auto s = dstShape[i];
       if (ShapedType::isDynamic(s))
-        shape[i] = rewriter.create<memref::DimOp>(loc, array, s).getResult();
+        shape[i] = memref::DimOp::create(rewriter, loc, array, s).getResult();
       else
         shape[i] = rewriter.getIndexAttr(s);
 
@@ -723,12 +723,12 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         offsets[i] = haloSizes[currHaloDim * 2];
 
         // prepare shape and offsets of highest dim's halo exchange
-        Value _haloSz = rewriter.create<arith::AddIOp>(
+        Value _haloSz = arith::AddIOp::create(rewriter,
             loc, toValue(haloSizes[currHaloDim * 2]),
             toValue(haloSizes[currHaloDim * 2 + 1]));
         // the halo shape of lower dims exlude the halos
         dimSizes[i] =
-            rewriter.create<arith::SubIOp>(loc, toValue(shape[i]), _haloSz)
+            arith::SubIOp::create(rewriter, loc, toValue(shape[i]), _haloSz)
                 .getResult();
       } else {
         dimSizes[i] = shape[i];
@@ -736,14 +736,14 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     }
 
     auto tagAttr = rewriter.getI32IntegerAttr(91); // we just pick something
-    auto tag = rewriter.create<arith::ConstantOp>(loc, tagAttr);
+    auto tag = arith::ConstantOp::create(rewriter, loc, tagAttr);
     auto zeroAttr = rewriter.getI32IntegerAttr(0); // for detecting v<0
-    auto zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
+    auto zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
 
     SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
                                        rewriter.getIndexType());
     auto myMultiIndex =
-        rewriter.create<ProcessMultiIndexOp>(loc, indexResultTypes, mesh)
+        ProcessMultiIndexOp::create(rewriter, loc, indexResultTypes, mesh)
             .getResult();
     // traverse all split axes from high to low dim
     for (ssize_t dim = opSplitAxes.size() - 1; dim >= 0; --dim) {
@@ -758,19 +758,19 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
                                                        splitAxes)
                      .getResults();
       // MPI operates on i32...
-      Value neighbourIDs[2] = {rewriter.create<arith::IndexCastOp>(
+      Value neighbourIDs[2] = {arith::IndexCastOp::create(rewriter,
                                    loc, rewriter.getI32Type(), tmp[0]),
-                               rewriter.create<arith::IndexCastOp>(
+                               arith::IndexCastOp::create(rewriter,
                                    loc, rewriter.getI32Type(), tmp[1])};
 
       auto lowerRecvOffset = rewriter.getIndexAttr(0);
       auto lowerSendOffset = toValue(haloSizes[currHaloDim * 2]);
-      auto upperRecvOffset = rewriter.create<arith::SubIOp>(
+      auto upperRecvOffset = arith::SubIOp::create(rewriter,
           loc, toValue(shape[dim]), toValue(haloSizes[currHaloDim * 2 + 1]));
-      auto upperSendOffset = rewriter.create<arith::SubIOp>(
+      auto upperSendOffset = arith::SubIOp::create(rewriter,
           loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
 
-      Value commWorld = rewriter.create<mpi::CommWorldOp>(
+      Value commWorld = mpi::CommWorldOp::create(rewriter,
           loc, mpi::CommType::get(op->getContext()));
 
       // Make sure we send/recv in a way that does not lead to a dead-lock.
@@ -787,37 +787,37 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         // Processes on the mesh borders have only one neighbor
         auto to = upperHalo ? neighbourIDs[0] : neighbourIDs[1];
         auto from = upperHalo ? neighbourIDs[1] : neighbourIDs[0];
-        auto hasFrom = rewriter.create<arith::CmpIOp>(
+        auto hasFrom = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sge, from, zero);
-        auto hasTo = rewriter.create<arith::CmpIOp>(
+        auto hasTo = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sge, to, zero);
-        auto buffer = rewriter.create<memref::AllocOp>(
+        auto buffer = memref::AllocOp::create(rewriter,
             loc, dimSizes, cast<ShapedType>(array.getType()).getElementType());
         // if has neighbor: copy halo data from array to buffer and send
-        rewriter.create<scf::IfOp>(
+        scf::IfOp::create(rewriter,
             loc, hasTo, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(lowerSendOffset)
                                        : OpFoldResult(upperSendOffset);
-              auto subview = builder.create<memref::SubViewOp>(
+              auto subview = memref::SubViewOp::create(builder,
                   loc, array, offsets, dimSizes, strides);
-              builder.create<memref::CopyOp>(loc, subview, buffer);
-              builder.create<mpi::SendOp>(loc, TypeRange{}, buffer, tag, to,
+              memref::CopyOp::create(builder, loc, subview, buffer);
+              mpi::SendOp::create(builder, loc, TypeRange{}, buffer, tag, to,
                                           commWorld);
-              builder.create<scf::YieldOp>(loc);
+              scf::YieldOp::create(builder, loc);
             });
         // if has neighbor: receive halo data into buffer and copy to array
-        rewriter.create<scf::IfOp>(
+        scf::IfOp::create(rewriter,
             loc, hasFrom, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(upperRecvOffset)
                                        : OpFoldResult(lowerRecvOffset);
-              builder.create<mpi::RecvOp>(loc, TypeRange{}, buffer, tag, from,
+              mpi::RecvOp::create(builder, loc, TypeRange{}, buffer, tag, from,
                                           commWorld);
-              auto subview = builder.create<memref::SubViewOp>(
+              auto subview = memref::SubViewOp::create(builder,
                   loc, array, offsets, dimSizes, strides);
-              builder.create<memref::CopyOp>(loc, buffer, subview);
-              builder.create<scf::YieldOp>(loc);
+              memref::CopyOp::create(builder, loc, buffer, subview);
+              scf::YieldOp::create(builder, loc);
             });
-        rewriter.create<memref::DeallocOp>(loc, buffer);
+        memref::DeallocOp::create(rewriter, loc, buffer);
         offsets[dim] = orgOffset;
       };
 
@@ -825,15 +825,15 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         OpFoldResult &v = haloSizes[currHaloDim * 2 + upOrDown];
         Value haloSz = dyn_cast<Value>(v);
         if (!haloSz)
-          haloSz = rewriter.create<arith::ConstantOp>(
+          haloSz = arith::ConstantOp::create(rewriter,
               loc, rewriter.getI32IntegerAttr(
                        cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
-        auto hasSize = rewriter.create<arith::CmpIOp>(
+        auto hasSize = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sgt, haloSz, zero);
-        rewriter.create<scf::IfOp>(loc, hasSize,
+        scf::IfOp::create(rewriter, loc, hasSize,
                                    [&](OpBuilder &builder, Location loc) {
                                      genSendRecv(upOrDown > 0);
-                                     builder.create<scf::YieldOp>(loc);
+                                     scf::YieldOp::create(builder, loc);
                                    });
       };
 
@@ -852,7 +852,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       rewriter.replaceOp(op, array);
     } else {
       assert(isa<RankedTensorType>(op.getResult().getType()));
-      rewriter.replaceOp(op, rewriter.create<bufferization::ToTensorOp>(
+      rewriter.replaceOp(op, bufferization::ToTensorOp::create(rewriter,
                                  loc, op.getResult().getType(), array,
                                  /*restrict=*/true, /*writable=*/true));
     }
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 80b3d85488495..3f82342001ea5 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -53,7 +53,7 @@ static Value truncToI32(ImplicitLocOpBuilder &b, Value value) {
   assert(llvm::isa<IntegerType>(type) && "expected an integer Value");
   if (type.getIntOrFloatBitWidth() <= 32)
     return value;
-  return b.create<LLVM::TruncOp>(b.getI32Type(), value);
+  return LLVM::TruncOp::create(b, b.getI32Type(), value);
 }
 
 /// Returns the type for the intrinsic given the vectorResultType of the
@@ -113,7 +113,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
   Type f32x1Ty = VectorType::get(1, f32Ty);
 
   auto makeConst = [&](int32_t index) -> Value {
-    return rewriter.create<LLVM::ConstantOp>(loc, IntegerType::get(ctx, 32),
+    return LLVM::ConstantOp::create(rewriter, loc, IntegerType::get(ctx, 32),
                                              rewriter.getI32IntegerAttr(index));
   };
 
@@ -126,7 +126,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
         arrayType.getElementType() == f32x1Ty) {
       for (unsigned i = 0; i < structType.getBody().size(); i++) {
         Value el =
-            rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i);
+            LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i);
         el = rewriter.createOrFold<LLVM::BitcastOp>(
             loc, arrayType.getElementType(), el);
         elements.push_back(el);
@@ -143,23 +143,23 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
       for (unsigned i = 0, e = structType.getBody().size() / 2; i < e; i++) {
         Value vec =
-            rewriter.create<LLVM::PoisonOp>(loc, arrayType.getElementType());
+            LLVM::PoisonOp::create(rewriter, loc, arrayType.getElementType());
         Value x1 =
-            rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i * 2);
-        Value x2 = rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult,
+            LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i * 2);
+        Value x2 = LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult,
                                                          i * 2 + 1);
-        vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec,
+        vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
                                                      x1, makeConst(0));
-        vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec,
+        vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
                                                      x2, makeConst(1));
         elements.push_back(vec);
       }
     }
 
     // Create the final vectorized result.
-    Value result = rewriter.create<LLVM::PoisonOp>(loc, arrayType);
+    Value result = LLVM::PoisonOp::create(rewriter, loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
-      result = rewriter.create<LLVM::InsertValueOp>(loc, result, el.value(),
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, el.value(),
                                                     el.index());
     }
     return result;
@@ -187,7 +187,7 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
   auto arrayTy = cast<LLVM::LLVMArrayType>(operand.getType());
 
   for (unsigned i = 0, e = arrayTy.getNumElements(); i < e; ++i) {
-    Value toUse = b.create<LLVM::ExtractValueOp>(operand, i);
+    Value toUse = LLVM::ExtractValueOp::create(b, operand, i);
 
     // For 4xi8 vectors, the intrinsic expects these to be provided as i32
     // scalar types.
@@ -195,7 +195,7 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
         arrayTy.getElementType() == i4x8Ty ||
         (arrayTy.getElementType() == f32x1Ty &&
          operandPtxType == NVVM::MMATypes::tf32)) {
-      result.push_back(b.create<LLVM::BitcastOp>(i32Ty, toUse));
+      result.push_back(LLVM::BitcastOp::create(b, i32Ty, toUse));
       continue;
     }
 
@@ -208,9 +208,9 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
                          innerArrayTy.getElementType() == f32Ty)) {
       for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
            idx < innerSize; idx++) {
-        result.push_back(b.create<LLVM::ExtractElementOp>(
+        result.push_back(LLVM::ExtractElementOp::create(b,
             toUse,
-            b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(idx))));
+            LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(idx))));
       }
       continue;
     }
@@ -285,7 +285,7 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
-    Value ldMatrixResult = b.create<NVVM::LdMatrixOp>(
+    Value ldMatrixResult = NVVM::LdMatrixOp::create(b,
         ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
@@ -296,13 +296,13 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     // actual vector type (still of width 32b) and repack them into a result
     // struct.
     Type finalResultType = typeConverter->convertType(vectorResultType);
-    Value result = b.create<LLVM::PoisonOp>(finalResultType);
+    Value result = LLVM::PoisonOp::create(b, finalResultType);
     for (int64_t i = 0, e = vectorResultType.getDimSize(0); i < e; i++) {
       Value i32Register =
-          num32BitRegs > 1 ? b.create<LLVM::ExtractValueOp>(ldMatrixResult, i)
+          num32BitRegs > 1 ? LLVM::ExtractValueOp::create(b, ldMatrixResult, i)
                            : ldMatrixResult;
-      Value casted = b.create<LLVM::BitcastOp>(innerVectorType, i32Register);
-      result = b.create<LLVM::InsertValueOp>(result, casted, i);
+      Value casted = LLVM::BitcastOp::create(b, innerVectorType, i32Register);
+      result = LLVM::InsertValueOp::create(b, result, casted, i);
     }
 
     rewriter.replaceOp(op, result);
@@ -375,7 +375,7 @@ struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
     Type desiredRetTy = typeConverter->convertType(op->getResultTypes()[0]);
     Type intrinsicResTy = inferIntrinsicResultType(
         typeConverter->convertType(op->getResultTypes()[0]));
-    Value intrinsicResult = b.create<NVVM::MmaOp>(
+    Value intrinsicResult = NVVM::MmaOp::create(b,
         intrinsicResTy, matA, matB, matC,
         /*shape=*/gemmShape,
         /*b1Op=*/std::nullopt,
@@ -565,7 +565,7 @@ static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
     llvm::append_range(asmVals, args);
   asmVals.push_back(indexData);
 
-  return b.create<LLVM::InlineAsmOp>(
+  return LLVM::InlineAsmOp::create(b,
       /*resultTypes=*/intrinsicResultType,
       /*operands=*/asmVals,
       /*asm_string=*/asmStr,
@@ -631,7 +631,7 @@ struct NVGPUMmaSparseSyncLowering
       return op->emitOpError() << "Expected metadata type to be LLVM "
                                   "VectorType of 2 i16 elements";
     sparseMetadata =
-        b.create<LLVM::BitcastOp>(rewriter.getI32Type(), sparseMetadata);
+        LLVM::BitcastOp::create(b, rewriter.getI32Type(), sparseMetadata);
 
     FailureOr<LLVM::InlineAsmOp> intrinsicResult = emitMmaSparseSyncOpAsm(
         b, *ptxTypeA, *ptxTypeB, *ptxTypeC, *ptxTypeC, overflow, matA, matB,
@@ -682,7 +682,7 @@ struct NVGPUAsyncCopyLowering
     // Intrinsics takes a global pointer so we need an address space cast.
     auto srcPointerGlobalType = LLVM::LLVMPointerType::get(
         op->getContext(), NVVM::NVVMMemorySpace::kGlobalMemorySpace);
-    scrPtr = b.create<LLVM::AddrSpaceCastOp>(srcPointerGlobalType, scrPtr);
+    scrPtr = LLVM::AddrSpaceCastOp::create(b, srcPointerGlobalType, scrPtr);
     int64_t dstElements = adaptor.getDstElements().getZExtValue();
     int64_t sizeInBytes =
         (dstMemrefType.getElementTypeBitWidth() * dstElements) / 8;
@@ -697,13 +697,13 @@ struct NVGPUAsyncCopyLowering
       // The rest of the DstElements in the destination (shared memory) are
       // filled with zeros.
       Value c3I32 =
-          b.create<LLVM::ConstantOp>(b.getI32Type(), b.getI32IntegerAttr(3));
-      Value bitwidth = b.create<LLVM::ConstantOp>(
+          LLVM::ConstantOp::create(b, b.getI32Type(), b.getI32IntegerAttr(3));
+      Value bitwidth = LLVM::ConstantOp::create(b,
           b.getI32Type(),
           b.getI32IntegerAttr(srcMemrefType.getElementTypeBitWidth()));
-      Value srcElementsI32 = b.create<LLVM::TruncOp>(b.getI32Type(), srcBytes);
-      srcBytes = b.create<LLVM::LShrOp>(
-          b.create<LLVM::MulOp>(bitwidth, srcElementsI32), c3I32);
+      Value srcElementsI32 = LLVM::TruncOp::create(b, b.getI32Type(), srcBytes);
+      srcBytes = LLVM::LShrOp::create(b,
+          LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
     }
     // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
     // 16 dst bytes.
@@ -712,13 +712,13 @@ struct NVGPUAsyncCopyLowering
             ? NVVM::LoadCacheModifierKind::CG
             : NVVM::LoadCacheModifierKind::CA;
 
-    b.create<NVVM::CpAsyncOp>(
+    NVVM::CpAsyncOp::create(b,
         dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
         NVVM::LoadCacheModifierKindAttr::get(op->getContext(), cacheModifier),
         srcBytes);
 
     // Drop the result token.
-    Value zero = b.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(b,
         IntegerType::get(op.getContext(), 32), rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
@@ -733,9 +733,9 @@ struct NVGPUAsyncCreateGroupLowering
   LogicalResult
   matchAndRewrite(nvgpu::DeviceAsyncCreateGroupOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    rewriter.create<NVVM::CpAsyncCommitGroupOp>(op.getLoc());
+    NVVM::CpAsyncCommitGroupOp::create(rewriter, op.getLoc());
     // Drop the result token.
-    Value zero = rewriter.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(rewriter,
         op->getLoc(), IntegerType::get(op.getContext(), 32),
         rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
@@ -753,7 +753,7 @@ struct NVGPUAsyncWaitLowering
                   ConversionPatternRewriter &rewriter) const override {
     // If numGroup is not present pick 0 as a conservative correct value.
     int32_t numGroups = adaptor.getNumGroups().value_or(0);
-    rewriter.create<NVVM::CpAsyncWaitGroupOp>(op.getLoc(), numGroups);
+    NVVM::CpAsyncWaitGroupOp::create(rewriter, op.getLoc(), numGroups);
     rewriter.eraseOp(op);
     return success();
   }
@@ -771,7 +771,7 @@ struct NVGPUMBarrierCreateLowering
     SymbolTable symbolTable(moduleOp);
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(&moduleOp.front());
-    auto global = rewriter.create<memref::GlobalOp>(
+    auto global = memref::GlobalOp::create(rewriter,
         funcOp->getLoc(), "__mbarrier",
         /*sym_visibility=*/rewriter.getStringAttr("private"),
         /*type=*/barrierType,
@@ -974,7 +974,7 @@ struct NVGPUMBarrierTryWaitParityLowering
                        adaptor.getMbarId(), rewriter);
     Value ticks = truncToI32(b, adaptor.getTicks());
     Value phase =
-        b.create<LLVM::ZExtOp>(b.getI32Type(), adaptor.getPhaseParity());
+        LLVM::ZExtOp::create(b, b.getI32Type(), adaptor.getPhaseParity());
 
     if (isMbarrierShared(op.getBarriers().getType())) {
       rewriter.replaceOpWithNewOp<NVVM::MBarrierTryWaitParitySharedOp>(
@@ -1063,16 +1063,16 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 
     auto ti64 = b.getIntegerType(64);
     auto makeConst = [&](uint64_t index) -> Value {
-      return b.create<LLVM::ConstantOp>(ti64, b.getI64IntegerAttr(index));
+      return LLVM::ConstantOp::create(b, ti64, b.getI64IntegerAttr(index));
     };
     auto shiftLeft = [&](Value value, unsigned shift) -> Value {
-      return b.create<LLVM::ShlOp>(ti64, value, makeConst(shift));
+      return LLVM::ShlOp::create(b, ti64, value, makeConst(shift));
     };
     auto shiftRight = [&](Value value, unsigned shift) -> Value {
-      return b.create<LLVM::LShrOp>(ti64, value, makeConst(shift));
+      return LLVM::LShrOp::create(b, ti64, value, makeConst(shift));
     };
     auto insertBit = [&](Value desc, Value val, int startBit) {
-      return b.create<LLVM::OrOp>(ti64, desc, shiftLeft(val, startBit));
+      return LLVM::OrOp::create(b, ti64, desc, shiftLeft(val, startBit));
     };
 
     int64_t sizeN = op.getTensorMap().getType().getTensor().getDimSize(0);
@@ -1086,7 +1086,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
     Value baseAddr = getStridedElementPtr(
         rewriter, op->getLoc(), cast<MemRefType>(op.getTensor().getType()),
         adaptor.getTensor(), {});
-    Value basePtr = b.create<LLVM::PtrToIntOp>(ti64, baseAddr);
+    Value basePtr = LLVM::PtrToIntOp::create(b, ti64, baseAddr);
     // Just use 14 bits for base address
     Value basePtr14bit = shiftRight(shiftLeft(basePtr, 46), 50);
 
@@ -1118,7 +1118,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 };
 
 static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
-  return b.create<LLVM::ConstantOp>(b.getIntegerType(64),
+  return LLVM::ConstantOp::create(b, b.getIntegerType(64),
                                     b.getI32IntegerAttr(index));
 }
 
@@ -1182,12 +1182,12 @@ struct NVGPUTmaCreateDescriptorOpLowering
     auto promotedOperands = getTypeConverter()->promoteOperands(
         b.getLoc(), op->getOperands(), adaptor.getOperands(), b);
 
-    Value boxArrayPtr = b.create<LLVM::AllocaOp>(llvmPointerType, llvmInt64Type,
+    Value boxArrayPtr = LLVM::AllocaOp::create(b, llvmPointerType, llvmInt64Type,
                                                  makeI64Const(b, 5));
     for (auto [index, value] : llvm::enumerate(adaptor.getBoxDimensions())) {
-      Value gep = b.create<LLVM::GEPOp>(llvmPointerType, llvmPointerType,
+      Value gep = LLVM::GEPOp::create(b, llvmPointerType, llvmPointerType,
                                         boxArrayPtr, makeI64Const(b, index));
-      b.create<LLVM::StoreOp>(value, gep);
+      LLVM::StoreOp::create(b, value, gep);
     }
 
     nvgpu::TensorMapDescriptorType desc = op.getTensorMap().getType();
@@ -1337,7 +1337,7 @@ struct NVGPUWarpgroupMmaOpLowering
 
     /// Basic function to generate Add
     Value makeAdd(Value lhs, Value rhs) {
-      return b.create<LLVM::AddOp>(lhs.getType(), lhs, rhs);
+      return LLVM::AddOp::create(b, lhs.getType(), lhs, rhs);
     };
 
     /// Moves the descriptor pointer of matrix-A for the next wgmma instruction.
@@ -1430,7 +1430,7 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      return b.create<NVVM::WgmmaMmaAsyncOp>(
+      return NVVM::WgmmaMmaAsyncOp::create(b,
           matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
           itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
           overflow);
@@ -1439,19 +1439,19 @@ struct NVGPUWarpgroupMmaOpLowering
     /// Generates multiple wgmma instructions to complete the given GEMM shape
     Value generateWgmmaGroup() {
       Value wgmmaResult =
-          b.create<LLVM::PoisonOp>(adaptor.getMatrixC().getType());
+          LLVM::PoisonOp::create(b, adaptor.getMatrixC().getType());
 
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = b.create<LLVM::ExtractValueOp>(adaptor.getMatrixC(), i);
+        Value matrixC = LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
             matrixC = generateWgmma(i, j, k, matrixC);
         wgmmaResults.push_back(matrixC);
       }
       for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
-        wgmmaResult = b.create<LLVM::InsertValueOp>(wgmmaResult.getType(),
+        wgmmaResult = LLVM::InsertValueOp::create(b, wgmmaResult.getType(),
                                                     wgmmaResult, matrix, idx);
       }
       return wgmmaResult;
@@ -1486,10 +1486,10 @@ struct NVGPUWarpgroupMmaOpLowering
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
     Value generateWarpgroupMma() {
-      b.create<NVVM::WgmmaFenceAlignedOp>();
+      NVVM::WgmmaFenceAlignedOp::create(b, );
       Value wgmmaResult = generateWgmmaGroup();
-      b.create<NVVM::WgmmaGroupSyncAlignedOp>();
-      b.create<NVVM::WgmmaWaitGroupSyncOp>(op.getWaitGroup());
+      NVVM::WgmmaGroupSyncAlignedOp::create(b, );
+      NVVM::WgmmaWaitGroupSyncOp::create(b, op.getWaitGroup());
       return wgmmaResult;
     }
   };
@@ -1557,7 +1557,7 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     Type i32 = b.getI32Type();
 
     auto makeConst = [&](int32_t index) -> Value {
-      return b.create<LLVM::ConstantOp>(i32, b.getI32IntegerAttr(index));
+      return LLVM::ConstantOp::create(b, i32, b.getI32IntegerAttr(index));
     };
     Value c1 = makeConst(1);
     Value c2 = makeConst(2);
@@ -1567,29 +1567,29 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     Value warpSize = makeConst(kWarpSize);
 
     auto makeMul = [&](Value lhs, Value rhs) -> Value {
-      return b.create<LLVM::MulOp>(lhs.getType(), lhs, rhs);
+      return LLVM::MulOp::create(b, lhs.getType(), lhs, rhs);
     };
     auto makeAdd = [&](Value lhs, Value rhs) -> Value {
-      return b.create<LLVM::AddOp>(lhs.getType(), lhs, rhs);
+      return LLVM::AddOp::create(b, lhs.getType(), lhs, rhs);
     };
 
     auto makeExtractAndStore = [&](int i, Value wgmmaResult, Value x, Value y,
                                    TypedValue<::mlir::MemRefType> memref) {
       Type it = b.getIndexType();
-      Value idx = b.create<arith::IndexCastOp>(it, x);
-      Value idy0 = b.create<arith::IndexCastOp>(it, y);
-      Value idy1 = b.create<arith::IndexCastOp>(it, makeAdd(y, c1));
-      Value d0 = b.create<LLVM::ExtractValueOp>(wgmmaResult, i);
-      Value d1 = b.create<LLVM::ExtractValueOp>(wgmmaResult, i + 1);
-      b.create<memref::StoreOp>(d0, memref, ValueRange{idx, idy0});
-      b.create<memref::StoreOp>(d1, memref, ValueRange{idx, idy1});
+      Value idx = arith::IndexCastOp::create(b, it, x);
+      Value idy0 = arith::IndexCastOp::create(b, it, y);
+      Value idy1 = arith::IndexCastOp::create(b, it, makeAdd(y, c1));
+      Value d0 = LLVM::ExtractValueOp::create(b, wgmmaResult, i);
+      Value d1 = LLVM::ExtractValueOp::create(b, wgmmaResult, i + 1);
+      memref::StoreOp::create(b, d0, memref, ValueRange{idx, idy0});
+      memref::StoreOp::create(b, d1, memref, ValueRange{idx, idy1});
     };
 
-    Value tidx = b.create<NVVM::ThreadIdXOp>(i32);
-    Value laneId = b.create<LLVM::URemOp>(i32, tidx, warpSize);
-    Value warpId = b.create<LLVM::UDivOp>(i32, tidx, warpSize);
-    Value lane4Id = b.create<LLVM::UDivOp>(i32, laneId, c4);
-    Value lane4modId = b.create<LLVM::URemOp>(i32, laneId, c4);
+    Value tidx = NVVM::ThreadIdXOp::create(b, i32);
+    Value laneId = LLVM::URemOp::create(b, i32, tidx, warpSize);
+    Value warpId = LLVM::UDivOp::create(b, i32, tidx, warpSize);
+    Value lane4Id = LLVM::UDivOp::create(b, i32, laneId, c4);
+    Value lane4modId = LLVM::URemOp::create(b, i32, laneId, c4);
 
     Value tj = makeMul(lane4modId, c2);
     Value ti = makeAdd(lane4Id, makeMul(warpId, c16));
@@ -1626,7 +1626,7 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     auto stype = cast<LLVM::LLVMStructType>(matriDValue.getType());
     for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(matrixD);
-      Value innerStructValue = b.create<LLVM::ExtractValueOp>(matriDValue, idx);
+      Value innerStructValue = LLVM::ExtractValueOp::create(b, matriDValue, idx);
       storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
@@ -1648,22 +1648,22 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
     Type elemType = cast<LLVM::LLVMStructType>(packStructType.getBody().front())
                         .getBody()
                         .front();
-    Value zero = b.create<LLVM::ConstantOp>(elemType, b.getZeroAttr(elemType));
-    Value packStruct = b.create<LLVM::PoisonOp>(packStructType);
+    Value zero = LLVM::ConstantOp::create(b, elemType, b.getZeroAttr(elemType));
+    Value packStruct = LLVM::PoisonOp::create(b, packStructType);
     SmallVector<Value> innerStructs;
     // Unpack the structs and set all values to zero
     for (auto [idx, s] : llvm::enumerate(packStructType.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(s);
-      Value structValue = b.create<LLVM::ExtractValueOp>(packStruct, idx);
+      Value structValue = LLVM::ExtractValueOp::create(b, packStruct, idx);
       for (unsigned i = 0; i < structType.getBody().size(); ++i) {
-        structValue = b.create<LLVM::InsertValueOp>(
+        structValue = LLVM::InsertValueOp::create(b,
             structType, structValue, zero, ArrayRef<int64_t>({i}));
       }
       innerStructs.push_back(structValue);
     }
     // Pack the inner structs into a single struct
     for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
-      packStruct = b.create<LLVM::InsertValueOp>(packStruct.getType(),
+      packStruct = LLVM::InsertValueOp::create(b, packStruct.getType(),
                                                  packStruct, matrix, idx);
     }
     rewriter.replaceOp(op, packStruct);
@@ -1681,7 +1681,7 @@ struct NVGPUTmaFenceOpLowering
     ImplicitLocOpBuilder b(op->getLoc(), rewriter);
     auto i32Ty = b.getI32Type();
     Value tensormapSize =
-        b.create<LLVM::ConstantOp>(i32Ty, rewriter.getI32IntegerAttr(128));
+        LLVM::ConstantOp::create(b, i32Ty, rewriter.getI32IntegerAttr(128));
 
     auto memscope =
         NVVM::MemScopeKindAttr::get(ctx, ::mlir::NVVM::MemScopeKind::SYS);
@@ -1716,13 +1716,13 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
     VectorType inTy = op.getIn().getType();
     // apply rcp.approx.ftz.f on each element in vector.
     auto convert1DVec = [&](Type llvm1DVectorTy, Value inVec) {
-      Value ret1DVec = b.create<LLVM::PoisonOp>(llvm1DVectorTy);
+      Value ret1DVec = LLVM::PoisonOp::create(b, llvm1DVectorTy);
       int numElems = llvm::cast<VectorType>(llvm1DVectorTy).getNumElements();
       for (int i = 0; i < numElems; i++) {
-        Value idx = b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(i));
-        Value elem = b.create<LLVM::ExtractElementOp>(inVec, idx);
-        Value dst = b.create<NVVM::RcpApproxFtzF32Op>(f32Ty, elem);
-        ret1DVec = b.create<LLVM::InsertElementOp>(ret1DVec, dst, idx);
+        Value idx = LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(i));
+        Value elem = LLVM::ExtractElementOp::create(b, inVec, idx);
+        Value dst = NVVM::RcpApproxFtzF32Op::create(b, f32Ty, elem);
+        ret1DVec = LLVM::InsertElementOp::create(b, ret1DVec, dst, idx);
       }
       return ret1DVec;
     };
diff --git a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
index 479725aae8afd..a6125fc07dcd5 100644
--- a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
+++ b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
@@ -39,7 +39,7 @@ class ExpandIfCondition : public OpRewritePattern<OpTy> {
 
     IntegerAttr constAttr;
     if (!matchPattern(op.getIfCond(), m_Constant(&constAttr))) {
-      auto ifOp = rewriter.create<scf::IfOp>(op.getLoc(), TypeRange(),
+      auto ifOp = scf::IfOp::create(rewriter, op.getLoc(), TypeRange(),
                                              op.getIfCond(), false);
       rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });
       auto thenBodyBuilder = ifOp.getThenBodyBuilder(rewriter.getListener());
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 7ac9687c4eeda..4bf59124a9bdb 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -95,7 +95,7 @@ struct OpenMPOpConversion : public ConvertOpToLLVMPattern<T> {
     }
 
     // Create new operation.
-    auto newOp = rewriter.create<T>(op.getLoc(), resTypes, convertedOperands,
+    auto newOp = T::create(rewriter, op.getLoc(), resTypes, convertedOperands,
                                     convertedAttrs);
 
     // Translate regions.
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 7d20109b3db59..75cd35c0d7876 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -196,7 +196,7 @@ Block *PatternLowering::generateMatcher(MatcherNode &node, Region &region,
   // finalize.
   if (isa<ExitNode>(node)) {
     builder.setInsertionPointToEnd(block);
-    builder.create<pdl_interp::FinalizeOp>(matcherFunc.getLoc());
+    pdl_interp::FinalizeOp::create(builder, matcherFunc.getLoc());
     return block;
   }
 
@@ -272,7 +272,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     auto *operationPos = cast<OperationPosition>(pos);
     if (operationPos->isOperandDefiningOp())
       // Standard (downward) traversal which directly follows the defining op.
-      value = builder.create<pdl_interp::GetDefiningOpOp>(
+      value = pdl_interp::GetDefiningOpOp::create(builder,
           loc, builder.getType<pdl::OperationType>(), parentVal);
     else
       // A passthrough operation position.
@@ -287,23 +287,23 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     // requested to use a representative value (e.g., upward traversal).
     if (isa<pdl::RangeType>(parentVal.getType()) &&
         usersPos->useRepresentative())
-      value = builder.create<pdl_interp::ExtractOp>(loc, parentVal, 0);
+      value = pdl_interp::ExtractOp::create(builder, loc, parentVal, 0);
     else
       value = parentVal;
 
     // The second operation retrieves the users.
-    value = builder.create<pdl_interp::GetUsersOp>(loc, value);
+    value = pdl_interp::GetUsersOp::create(builder, loc, value);
     break;
   }
   case Predicates::ForEachPos: {
     assert(!failureBlockStack.empty() && "expected valid failure block");
-    auto foreach = builder.create<pdl_interp::ForEachOp>(
+    auto foreach = pdl_interp::ForEachOp::create(builder,
         loc, parentVal, failureBlockStack.back(), /*initLoop=*/true);
     value = foreach.getLoopVariable();
 
     // Create the continuation block.
     Block *continueBlock = builder.createBlock(&foreach.getRegion());
-    builder.create<pdl_interp::ContinueOp>(loc);
+    pdl_interp::ContinueOp::create(builder, loc);
     failureBlockStack.push_back(continueBlock);
 
     currentBlock = &foreach.getRegion().front();
@@ -311,7 +311,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::OperandPos: {
     auto *operandPos = cast<OperandPosition>(pos);
-    value = builder.create<pdl_interp::GetOperandOp>(
+    value = pdl_interp::GetOperandOp::create(builder,
         loc, builder.getType<pdl::ValueType>(), parentVal,
         operandPos->getOperandNumber());
     break;
@@ -319,28 +319,28 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::OperandGroupPos: {
     auto *operandPos = cast<OperandGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = builder.create<pdl_interp::GetOperandsOp>(
+    value = pdl_interp::GetOperandsOp::create(builder,
         loc, operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, operandPos->getOperandGroupNumber());
     break;
   }
   case Predicates::AttributePos: {
     auto *attrPos = cast<AttributePosition>(pos);
-    value = builder.create<pdl_interp::GetAttributeOp>(
+    value = pdl_interp::GetAttributeOp::create(builder,
         loc, builder.getType<pdl::AttributeType>(), parentVal,
         attrPos->getName().strref());
     break;
   }
   case Predicates::TypePos: {
     if (isa<pdl::AttributeType>(parentVal.getType()))
-      value = builder.create<pdl_interp::GetAttributeTypeOp>(loc, parentVal);
+      value = pdl_interp::GetAttributeTypeOp::create(builder, loc, parentVal);
     else
-      value = builder.create<pdl_interp::GetValueTypeOp>(loc, parentVal);
+      value = pdl_interp::GetValueTypeOp::create(builder, loc, parentVal);
     break;
   }
   case Predicates::ResultPos: {
     auto *resPos = cast<ResultPosition>(pos);
-    value = builder.create<pdl_interp::GetResultOp>(
+    value = pdl_interp::GetResultOp::create(builder,
         loc, builder.getType<pdl::ValueType>(), parentVal,
         resPos->getResultNumber());
     break;
@@ -348,7 +348,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::ResultGroupPos: {
     auto *resPos = cast<ResultGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = builder.create<pdl_interp::GetResultsOp>(
+    value = pdl_interp::GetResultsOp::create(builder,
         loc, resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, resPos->getResultGroupNumber());
     break;
@@ -356,16 +356,16 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::AttributeLiteralPos: {
     auto *attrPos = cast<AttributeLiteralPosition>(pos);
     value =
-        builder.create<pdl_interp::CreateAttributeOp>(loc, attrPos->getValue());
+        pdl_interp::CreateAttributeOp::create(builder, loc, attrPos->getValue());
     break;
   }
   case Predicates::TypeLiteralPos: {
     auto *typePos = cast<TypeLiteralPosition>(pos);
     Attribute rawTypeAttr = typePos->getValue();
     if (TypeAttr typeAttr = dyn_cast<TypeAttr>(rawTypeAttr))
-      value = builder.create<pdl_interp::CreateTypeOp>(loc, typeAttr);
+      value = pdl_interp::CreateTypeOp::create(builder, loc, typeAttr);
     else
-      value = builder.create<pdl_interp::CreateTypesOp>(
+      value = pdl_interp::CreateTypesOp::create(builder,
           loc, cast<ArrayAttr>(rawTypeAttr));
     break;
   }
@@ -413,54 +413,54 @@ void PatternLowering::generate(BoolNode *boolNode, Block *&currentBlock,
   Predicates::Kind kind = question->getKind();
   switch (kind) {
   case Predicates::IsNotNullQuestion:
-    builder.create<pdl_interp::IsNotNullOp>(loc, val, success, failure);
+    pdl_interp::IsNotNullOp::create(builder, loc, val, success, failure);
     break;
   case Predicates::OperationNameQuestion: {
     auto *opNameAnswer = cast<OperationNameAnswer>(answer);
-    builder.create<pdl_interp::CheckOperationNameOp>(
+    pdl_interp::CheckOperationNameOp::create(builder,
         loc, val, opNameAnswer->getValue().getStringRef(), success, failure);
     break;
   }
   case Predicates::TypeQuestion: {
     auto *ans = cast<TypeAnswer>(answer);
     if (isa<pdl::RangeType>(val.getType()))
-      builder.create<pdl_interp::CheckTypesOp>(
+      pdl_interp::CheckTypesOp::create(builder,
           loc, val, llvm::cast<ArrayAttr>(ans->getValue()), success, failure);
     else
-      builder.create<pdl_interp::CheckTypeOp>(
+      pdl_interp::CheckTypeOp::create(builder,
           loc, val, llvm::cast<TypeAttr>(ans->getValue()), success, failure);
     break;
   }
   case Predicates::AttributeQuestion: {
     auto *ans = cast<AttributeAnswer>(answer);
-    builder.create<pdl_interp::CheckAttributeOp>(loc, val, ans->getValue(),
+    pdl_interp::CheckAttributeOp::create(builder, loc, val, ans->getValue(),
                                                  success, failure);
     break;
   }
   case Predicates::OperandCountAtLeastQuestion:
   case Predicates::OperandCountQuestion:
-    builder.create<pdl_interp::CheckOperandCountOp>(
+    pdl_interp::CheckOperandCountOp::create(builder,
         loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::OperandCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::ResultCountAtLeastQuestion:
   case Predicates::ResultCountQuestion:
-    builder.create<pdl_interp::CheckResultCountOp>(
+    pdl_interp::CheckResultCountOp::create(builder,
         loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::ResultCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::EqualToQuestion: {
     bool trueAnswer = isa<TrueAnswer>(answer);
-    builder.create<pdl_interp::AreEqualOp>(loc, val, args.front(),
+    pdl_interp::AreEqualOp::create(builder, loc, val, args.front(),
                                            trueAnswer ? success : failure,
                                            trueAnswer ? failure : success);
     break;
   }
   case Predicates::ConstraintQuestion: {
     auto *cstQuestion = cast<ConstraintQuestion>(question);
-    auto applyConstraintOp = builder.create<pdl_interp::ApplyConstraintOp>(
+    auto applyConstraintOp = pdl_interp::ApplyConstraintOp::create(builder,
         loc, cstQuestion->getResultTypes(), cstQuestion->getName(), args,
         cstQuestion->getIsNegated(), success, failure);
 
@@ -487,7 +487,7 @@ static void createSwitchOp(Value val, Block *defaultDest, OpBuilder &builder,
     blocks.push_back(it.second);
     values.push_back(cast<PredT>(it.first)->getValue());
   }
-  builder.create<OpT>(val.getLoc(), val, values, defaultDest, blocks);
+  OpT::create(builder, val.getLoc(), val, values, defaultDest, blocks);
 }
 
 void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
@@ -536,11 +536,11 @@ void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
       unsigned ans = cast<UnsignedAnswer>(child.first)->getValue();
       switch (kind) {
       case Predicates::OperandCountAtLeastQuestion:
-        builder.create<pdl_interp::CheckOperandCountOp>(
+        pdl_interp::CheckOperandCountOp::create(builder,
             loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
         break;
       case Predicates::ResultCountAtLeastQuestion:
-        builder.create<pdl_interp::CheckResultCountOp>(
+        pdl_interp::CheckResultCountOp::create(builder,
             loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
         break;
       default:
@@ -619,7 +619,7 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
       rootKindAttr = builder.getStringAttr(*rootKind);
 
   builder.setInsertionPointToEnd(currentBlock);
-  auto matchOp = builder.create<pdl_interp::RecordMatchOp>(
+  auto matchOp = pdl_interp::RecordMatchOp::create(builder,
       pattern.getLoc(), mappedMatchValues, locOps.getArrayRef(),
       rewriterFuncRef, rootKindAttr, generatedOpsAttr, pattern.getBenefitAttr(),
       failureBlockStack.back());
@@ -632,7 +632,7 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
 SymbolRefAttr PatternLowering::generateRewriter(
     pdl::PatternOp pattern, SmallVectorImpl<Position *> &usedMatchValues) {
   builder.setInsertionPointToEnd(rewriterModule.getBody());
-  auto rewriterFunc = builder.create<pdl_interp::FuncOp>(
+  auto rewriterFunc = pdl_interp::FuncOp::create(builder,
       pattern.getLoc(), "pdl_generated_rewriter",
       builder.getFunctionType({}, {}));
   rewriterSymbolTable.insert(rewriterFunc);
@@ -651,17 +651,17 @@ SymbolRefAttr PatternLowering::generateRewriter(
     Operation *oldOp = oldValue.getDefiningOp();
     if (pdl::AttributeOp attrOp = dyn_cast<pdl::AttributeOp>(oldOp)) {
       if (Attribute value = attrOp.getValueAttr()) {
-        return newValue = builder.create<pdl_interp::CreateAttributeOp>(
+        return newValue = pdl_interp::CreateAttributeOp::create(builder,
                    attrOp.getLoc(), value);
       }
     } else if (pdl::TypeOp typeOp = dyn_cast<pdl::TypeOp>(oldOp)) {
       if (TypeAttr type = typeOp.getConstantTypeAttr()) {
-        return newValue = builder.create<pdl_interp::CreateTypeOp>(
+        return newValue = pdl_interp::CreateTypeOp::create(builder,
                    typeOp.getLoc(), type);
       }
     } else if (pdl::TypesOp typeOp = dyn_cast<pdl::TypesOp>(oldOp)) {
       if (ArrayAttr type = typeOp.getConstantTypesAttr()) {
-        return newValue = builder.create<pdl_interp::CreateTypesOp>(
+        return newValue = pdl_interp::CreateTypesOp::create(builder,
                    typeOp.getLoc(), typeOp.getType(), type);
       }
     }
@@ -684,7 +684,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
     auto mappedArgs =
         llvm::map_range(rewriter.getExternalArgs(), mapRewriteValue);
     args.append(mappedArgs.begin(), mappedArgs.end());
-    builder.create<pdl_interp::ApplyRewriteOp>(
+    pdl_interp::ApplyRewriteOp::create(builder,
         rewriter.getLoc(), /*resultTypes=*/TypeRange(), rewriteName, args);
   } else {
     // Otherwise this is a dag rewriter defined using PDL operations.
@@ -703,7 +703,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
       llvm::to_vector<8>(rewriterFunc.front().getArgumentTypes()),
       /*results=*/{}));
 
-  builder.create<pdl_interp::FinalizeOp>(rewriter.getLoc());
+  pdl_interp::FinalizeOp::create(builder, rewriter.getLoc());
   return SymbolRefAttr::get(
       builder.getContext(),
       pdl_interp::PDLInterpDialect::getRewriterModuleName(),
@@ -716,7 +716,7 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 2> arguments;
   for (Value argument : rewriteOp.getArgs())
     arguments.push_back(mapRewriteValue(argument));
-  auto interpOp = builder.create<pdl_interp::ApplyRewriteOp>(
+  auto interpOp = pdl_interp::ApplyRewriteOp::create(builder,
       rewriteOp.getLoc(), rewriteOp.getResultTypes(), rewriteOp.getNameAttr(),
       arguments);
   for (auto it : llvm::zip(rewriteOp.getResults(), interpOp.getResults()))
@@ -726,7 +726,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::AttributeOp attrOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  Value newAttr = builder.create<pdl_interp::CreateAttributeOp>(
+  Value newAttr = pdl_interp::CreateAttributeOp::create(builder,
       attrOp.getLoc(), attrOp.getValueAttr());
   rewriteValues[attrOp] = newAttr;
 }
@@ -734,7 +734,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::EraseOp eraseOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  builder.create<pdl_interp::EraseOp>(eraseOp.getLoc(),
+  pdl_interp::EraseOp::create(builder, eraseOp.getLoc(),
                                       mapRewriteValue(eraseOp.getOpValue()));
 }
 
@@ -756,7 +756,7 @@ void PatternLowering::generateRewriter(
 
   // Create the new operation.
   Location loc = operationOp.getLoc();
-  Value createdOp = builder.create<pdl_interp::CreateOperationOp>(
+  Value createdOp = pdl_interp::CreateOperationOp::create(builder,
       loc, *operationOp.getOpName(), types, hasInferredResultTypes, operands,
       attributes, operationOp.getAttributeValueNames());
   rewriteValues[operationOp.getOp()] = createdOp;
@@ -768,8 +768,8 @@ void PatternLowering::generateRewriter(
   if (resultTys.size() == 1 && isa<pdl::RangeType>(resultTys[0].getType())) {
     Value &type = rewriteValues[resultTys[0]];
     if (!type) {
-      auto results = builder.create<pdl_interp::GetResultsOp>(loc, createdOp);
-      type = builder.create<pdl_interp::GetValueTypeOp>(loc, results);
+      auto results = pdl_interp::GetResultsOp::create(builder, loc, createdOp);
+      type = pdl_interp::GetValueTypeOp::create(builder, loc, results);
     }
     return;
   }
@@ -789,12 +789,12 @@ void PatternLowering::generateRewriter(
     // groups because the exact index of the result is not statically known.
     Value resultVal;
     if (seenVariableLength)
-      resultVal = builder.create<pdl_interp::GetResultsOp>(
+      resultVal = pdl_interp::GetResultsOp::create(builder,
           loc, isVariadic ? valueRangeTy : valueTy, createdOp, it.index());
     else
-      resultVal = builder.create<pdl_interp::GetResultOp>(
+      resultVal = pdl_interp::GetResultOp::create(builder,
           loc, valueTy, createdOp, it.index());
-    type = builder.create<pdl_interp::GetValueTypeOp>(loc, resultVal);
+    type = pdl_interp::GetValueTypeOp::create(builder, loc, resultVal);
   }
 }
 
@@ -804,7 +804,7 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 4> replOperands;
   for (Value operand : rangeOp.getArguments())
     replOperands.push_back(mapRewriteValue(operand));
-  rewriteValues[rangeOp] = builder.create<pdl_interp::CreateRangeOp>(
+  rewriteValues[rangeOp] = pdl_interp::CreateRangeOp::create(builder,
       rangeOp.getLoc(), rangeOp.getType(), replOperands);
 }
 
@@ -820,7 +820,7 @@ void PatternLowering::generateRewriter(
     // Don't use replace if we know the replaced operation has no results.
     auto opOp = replaceOp.getOpValue().getDefiningOp<pdl::OperationOp>();
     if (!opOp || !opOp.getTypeValues().empty()) {
-      replOperands.push_back(builder.create<pdl_interp::GetResultsOp>(
+      replOperands.push_back(pdl_interp::GetResultsOp::create(builder,
           replOp.getLoc(), mapRewriteValue(replOp)));
     }
   } else {
@@ -830,12 +830,12 @@ void PatternLowering::generateRewriter(
 
   // If there are no replacement values, just create an erase instead.
   if (replOperands.empty()) {
-    builder.create<pdl_interp::EraseOp>(
+    pdl_interp::EraseOp::create(builder,
         replaceOp.getLoc(), mapRewriteValue(replaceOp.getOpValue()));
     return;
   }
 
-  builder.create<pdl_interp::ReplaceOp>(replaceOp.getLoc(),
+  pdl_interp::ReplaceOp::create(builder, replaceOp.getLoc(),
                                         mapRewriteValue(replaceOp.getOpValue()),
                                         replOperands);
 }
@@ -843,7 +843,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::ResultOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultOp>(
+  rewriteValues[resultOp] = pdl_interp::GetResultOp::create(builder,
       resultOp.getLoc(), builder.getType<pdl::ValueType>(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
@@ -851,7 +851,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::ResultsOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultsOp>(
+  rewriteValues[resultOp] = pdl_interp::GetResultsOp::create(builder,
       resultOp.getLoc(), resultOp.getType(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
@@ -863,7 +863,7 @@ void PatternLowering::generateRewriter(
   // type.
   if (TypeAttr typeAttr = typeOp.getConstantTypeAttr()) {
     rewriteValues[typeOp] =
-        builder.create<pdl_interp::CreateTypeOp>(typeOp.getLoc(), typeAttr);
+        pdl_interp::CreateTypeOp::create(builder, typeOp.getLoc(), typeAttr);
   }
 }
 
@@ -873,7 +873,7 @@ void PatternLowering::generateRewriter(
   // If the type isn't constant, the users (e.g. OperationOp) will resolve this
   // type.
   if (ArrayAttr typeAttr = typeOp.getConstantTypesAttr()) {
-    rewriteValues[typeOp] = builder.create<pdl_interp::CreateTypesOp>(
+    rewriteValues[typeOp] = pdl_interp::CreateTypesOp::create(builder,
         typeOp.getLoc(), typeOp.getType(), typeAttr);
   }
 }
@@ -939,9 +939,9 @@ void PatternLowering::generateOperationResultTypeRewriter(
         !replacedOp->isBeforeInBlock(op))
       continue;
 
-    Value replacedOpResults = builder.create<pdl_interp::GetResultsOp>(
+    Value replacedOpResults = pdl_interp::GetResultsOp::create(builder,
         replacedOp->getLoc(), mapRewriteValue(replOpVal));
-    types.push_back(builder.create<pdl_interp::GetValueTypeOp>(
+    types.push_back(pdl_interp::GetValueTypeOp::create(builder,
         replacedOp->getLoc(), replacedOpResults));
     return;
   }
@@ -985,7 +985,7 @@ void PDLToPDLInterpPass::runOnOperation() {
   // Create the main matcher function This function contains all of the match
   // related functionality from patterns in the module.
   OpBuilder builder = OpBuilder::atBlockBegin(module.getBody());
-  auto matcherFunc = builder.create<pdl_interp::FuncOp>(
+  auto matcherFunc = pdl_interp::FuncOp::create(builder,
       module.getLoc(), pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
       builder.getFunctionType(builder.getType<pdl::OperationType>(),
                               /*results=*/{}),
@@ -993,7 +993,7 @@ void PDLToPDLInterpPass::runOnOperation() {
 
   // Create a nested module to hold the functions invoked for rewriting the IR
   // after a successful match.
-  ModuleOp rewriterModule = builder.create<ModuleOp>(
+  ModuleOp rewriterModule = ModuleOp::create(builder,
       module.getLoc(), pdl_interp::PDLInterpDialect::getRewriterModuleName());
 
   // Generate the code for the patterns within the module.
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index 0df91a243d07a..68527da73e1c7 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -340,7 +340,7 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   Operation *terminator = lastBodyBlock->getTerminator();
   rewriter.setInsertionPointToEnd(lastBodyBlock);
   auto step = forOp.getStep();
-  auto stepped = rewriter.create<arith::AddIOp>(loc, iv, step).getResult();
+  auto stepped = arith::AddIOp::create(rewriter, loc, iv, step).getResult();
   if (!stepped)
     return failure();
 
@@ -348,7 +348,7 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   loopCarried.push_back(stepped);
   loopCarried.append(terminator->operand_begin(), terminator->operand_end());
   auto branchOp =
-      rewriter.create<cf::BranchOp>(loc, conditionBlock, loopCarried);
+      cf::BranchOp::create(rewriter, loc, conditionBlock, loopCarried);
 
   // Let the CondBranchOp carry the LLVM attributes from the ForOp, such as the
   // llvm.loop_annotation attribute.
@@ -375,14 +375,14 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   SmallVector<Value, 8> destOperands;
   destOperands.push_back(lowerBound);
   llvm::append_range(destOperands, forOp.getInitArgs());
-  rewriter.create<cf::BranchOp>(loc, conditionBlock, destOperands);
+  cf::BranchOp::create(rewriter, loc, conditionBlock, destOperands);
 
   // With the body block done, we can fill in the condition block.
   rewriter.setInsertionPointToEnd(conditionBlock);
-  auto comparison = rewriter.create<arith::CmpIOp>(
+  auto comparison = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::slt, iv, upperBound);
 
-  rewriter.create<cf::CondBranchOp>(loc, comparison, firstBodyBlock,
+  cf::CondBranchOp::create(rewriter, loc, comparison, firstBodyBlock,
                                     ArrayRef<Value>(), endBlock,
                                     ArrayRef<Value>());
 
@@ -409,7 +409,7 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
     continueBlock =
         rewriter.createBlock(remainingOpsBlock, ifOp.getResultTypes(),
                              SmallVector<Location>(ifOp.getNumResults(), loc));
-    rewriter.create<cf::BranchOp>(loc, remainingOpsBlock);
+    cf::BranchOp::create(rewriter, loc, remainingOpsBlock);
   }
 
   // Move blocks from the "then" region to the region containing 'scf.if',
@@ -419,7 +419,7 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
   Operation *thenTerminator = thenRegion.back().getTerminator();
   ValueRange thenTerminatorOperands = thenTerminator->getOperands();
   rewriter.setInsertionPointToEnd(&thenRegion.back());
-  rewriter.create<cf::BranchOp>(loc, continueBlock, thenTerminatorOperands);
+  cf::BranchOp::create(rewriter, loc, continueBlock, thenTerminatorOperands);
   rewriter.eraseOp(thenTerminator);
   rewriter.inlineRegionBefore(thenRegion, continueBlock);
 
@@ -433,13 +433,13 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
     Operation *elseTerminator = elseRegion.back().getTerminator();
     ValueRange elseTerminatorOperands = elseTerminator->getOperands();
     rewriter.setInsertionPointToEnd(&elseRegion.back());
-    rewriter.create<cf::BranchOp>(loc, continueBlock, elseTerminatorOperands);
+    cf::BranchOp::create(rewriter, loc, continueBlock, elseTerminatorOperands);
     rewriter.eraseOp(elseTerminator);
     rewriter.inlineRegionBefore(elseRegion, continueBlock);
   }
 
   rewriter.setInsertionPointToEnd(condBlock);
-  rewriter.create<cf::CondBranchOp>(loc, ifOp.getCondition(), thenBlock,
+  cf::CondBranchOp::create(rewriter, loc, ifOp.getCondition(), thenBlock,
                                     /*trueArgs=*/ArrayRef<Value>(), elseBlock,
                                     /*falseArgs=*/ArrayRef<Value>());
 
@@ -459,13 +459,13 @@ ExecuteRegionLowering::matchAndRewrite(ExecuteRegionOp op,
 
   auto &region = op.getRegion();
   rewriter.setInsertionPointToEnd(condBlock);
-  rewriter.create<cf::BranchOp>(loc, &region.front());
+  cf::BranchOp::create(rewriter, loc, &region.front());
 
   for (Block &block : region) {
     if (auto terminator = dyn_cast<scf::YieldOp>(block.getTerminator())) {
       ValueRange terminatorOperands = terminator->getOperands();
       rewriter.setInsertionPointToEnd(&block);
-      rewriter.create<cf::BranchOp>(loc, remainingOpsBlock, terminatorOperands);
+      cf::BranchOp::create(rewriter, loc, remainingOpsBlock, terminatorOperands);
       rewriter.eraseOp(terminator);
     }
   }
@@ -503,7 +503,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
   for (auto [iv, lower, upper, step] :
        llvm::zip(parallelOp.getInductionVars(), parallelOp.getLowerBound(),
                  parallelOp.getUpperBound(), parallelOp.getStep())) {
-    ForOp forOp = rewriter.create<ForOp>(loc, lower, upper, step, iterArgs);
+    ForOp forOp = ForOp::create(rewriter, loc, lower, upper, step, iterArgs);
     ivs.push_back(forOp.getInductionVar());
     auto iterRange = forOp.getRegionIterArgs();
     iterArgs.assign(iterRange.begin(), iterRange.end());
@@ -517,7 +517,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
       // A loop is constructed with an empty "yield" terminator if there are
       // no results.
       rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
-      rewriter.create<scf::YieldOp>(loc, forOp.getResults());
+      scf::YieldOp::create(rewriter, loc, forOp.getResults());
     }
 
     rewriter.setInsertionPointToStart(forOp.getBody());
@@ -549,7 +549,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
   // has been already created in loop construction).
   if (!yieldOperands.empty()) {
     rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
-    rewriter.create<scf::YieldOp>(loc, yieldOperands);
+    scf::YieldOp::create(rewriter, loc, yieldOperands);
   }
 
   rewriter.replaceOp(parallelOp, loopResults);
@@ -575,7 +575,7 @@ LogicalResult WhileLowering::matchAndRewrite(WhileOp whileOp,
 
   // Branch to the "before" region.
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(loc, before, whileOp.getInits());
+  cf::BranchOp::create(rewriter, loc, before, whileOp.getInits());
 
   // Replace terminators with branches. Assuming bodies are SESE, which holds
   // given only the patterns from this file, we only need to look at the last
@@ -625,7 +625,7 @@ DoWhileLowering::matchAndRewrite(WhileOp whileOp,
 
   // Branch to the "before" region.
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(whileOp.getLoc(), before, whileOp.getInits());
+  cf::BranchOp::create(rewriter, whileOp.getLoc(), before, whileOp.getInits());
 
   // Loop around the "before" region based on condition.
   rewriter.setInsertionPointToEnd(before);
@@ -695,10 +695,10 @@ IndexSwitchLowering::matchAndRewrite(IndexSwitchOp op,
   SmallVector<ValueRange> caseOperands(caseSuccessors.size(), {});
 
   // Cast switch index to integer case value.
-  Value caseValue = rewriter.create<arith::IndexCastOp>(
+  Value caseValue = arith::IndexCastOp::create(rewriter,
       op.getLoc(), rewriter.getI32Type(), op.getArg());
 
-  rewriter.create<cf::SwitchOp>(
+  cf::SwitchOp::create(rewriter,
       op.getLoc(), caseValue, *defaultBlock, ValueRange(),
       rewriter.getDenseI32ArrayAttr(caseValues), caseSuccessors, caseOperands);
   rewriter.replaceOp(op, continueBlock->getArguments());
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index dcb48529a74e6..37134fe9d992a 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -91,7 +91,7 @@ createVariablesForResults(T op, const TypeConverter *typeConverter,
     Type varType = emitc::LValueType::get(resultType);
     emitc::OpaqueAttr noInit = emitc::OpaqueAttr::get(context, "");
     emitc::VariableOp var =
-        rewriter.create<emitc::VariableOp>(loc, varType, noInit);
+        emitc::VariableOp::create(rewriter, loc, varType, noInit);
     resultVariables.push_back(var);
   }
 
@@ -103,14 +103,14 @@ createVariablesForResults(T op, const TypeConverter *typeConverter,
 static void assignValues(ValueRange values, ValueRange variables,
                          ConversionPatternRewriter &rewriter, Location loc) {
   for (auto [value, var] : llvm::zip(values, variables))
-    rewriter.create<emitc::AssignOp>(loc, var, value);
+    emitc::AssignOp::create(rewriter, loc, var, value);
 }
 
 SmallVector<Value> loadValues(const SmallVector<Value> &variables,
                               PatternRewriter &rewriter, Location loc) {
   return llvm::map_to_vector<>(variables, [&](Value var) {
     Type type = cast<emitc::LValueType>(var.getType()).getValueType();
-    return rewriter.create<emitc::LoadOp>(loc, type, var).getResult();
+    return emitc::LoadOp::create(rewriter, loc, type, var).getResult();
   });
 }
 
@@ -129,7 +129,7 @@ static LogicalResult lowerYield(Operation *op, ValueRange resultVariables,
 
   assignValues(yieldOperands, resultVariables, rewriter, loc);
 
-  rewriter.create<emitc::YieldOp>(loc);
+  emitc::YieldOp::create(rewriter, loc);
   rewriter.eraseOp(yield);
 
   return success();
@@ -164,7 +164,7 @@ ForLowering::matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
 
   assignValues(adaptor.getInitArgs(), resultVariables, rewriter, loc);
 
-  emitc::ForOp loweredFor = rewriter.create<emitc::ForOp>(
+  emitc::ForOp loweredFor = emitc::ForOp::create(rewriter,
       loc, adaptor.getLowerBound(), adaptor.getUpperBound(), adaptor.getStep());
 
   Block *loweredBody = loweredFor.getBody();
@@ -257,7 +257,7 @@ IfLowering::matchAndRewrite(IfOp ifOp, OpAdaptor adaptor,
   bool hasElseBlock = !elseRegion.empty();
 
   auto loweredIf =
-      rewriter.create<emitc::IfOp>(loc, adaptor.getCondition(), false, false);
+      emitc::IfOp::create(rewriter, loc, adaptor.getCondition(), false, false);
 
   Region &loweredThenRegion = loweredIf.getThenRegion();
   auto result = lowerRegion(thenRegion, loweredThenRegion);
@@ -304,7 +304,7 @@ LogicalResult IndexSwitchOpLowering::matchAndRewrite(
                                        "create variables for results failed");
   }
 
-  auto loweredSwitch = rewriter.create<emitc::SwitchOp>(
+  auto loweredSwitch = emitc::SwitchOp::create(rewriter,
       loc, adaptor.getArg(), adaptor.getCases(), indexSwitchOp.getNumCases());
 
   // Lowering all case regions.
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index 844e66e927c4d..bfdb5a688b286 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -84,7 +84,7 @@ static Operation::operand_range getUpperBoundOperands(AffineForOp forOp) {
 // Get a Value that corresponds to the loop step.  If the step is an attribute,
 // materialize a corresponding constant using builder.
 static Value getOrCreateStep(AffineForOp forOp, OpBuilder &builder) {
-  return builder.create<arith::ConstantIndexOp>(forOp.getLoc(),
+  return arith::ConstantIndexOp::create(builder, forOp.getLoc(),
                                                 forOp.getStepAsInt());
 }
 
@@ -190,12 +190,12 @@ AffineLoopToGpuConverter::collectBounds(AffineForOp forOp, unsigned numLoops) {
       return std::nullopt;
     }
 
-    Value range = builder.create<arith::SubIOp>(currentLoop.getLoc(),
+    Value range = arith::SubIOp::create(builder, currentLoop.getLoc(),
                                                 upperBound, lowerBound);
     Value step = getOrCreateStep(currentLoop, builder);
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
       range =
-          builder.create<arith::CeilDivSIOp>(currentLoop.getLoc(), range, step);
+          arith::CeilDivSIOp::create(builder, currentLoop.getLoc(), range, step);
     dims.push_back(range);
 
     lbs.push_back(lowerBound);
@@ -221,7 +221,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
   // no loop mapped to a specific dimension, use constant "1" as its size.
   Value constOne =
       (numBlockDims < 3 || numThreadDims < 3)
-          ? builder.create<arith::ConstantIndexOp>(rootForOp.getLoc(), 1)
+          ? arith::ConstantIndexOp::create(builder, rootForOp.getLoc(), 1)
           : nullptr;
   Value gridSizeX = numBlockDims > 0 ? dims[0] : constOne;
   Value gridSizeY = numBlockDims > 1 ? dims[1] : constOne;
@@ -232,7 +232,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
 
   // Create a launch op and move the body region of the innermost loop to the
   // launch op.
-  auto launchOp = builder.create<gpu::LaunchOp>(
+  auto launchOp = gpu::LaunchOp::create(builder,
       rootForOp.getLoc(), gridSizeX, gridSizeY, gridSizeZ, blockSizeX,
       blockSizeY, blockSizeZ);
 
@@ -244,7 +244,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
   Location terminatorLoc = terminator.getLoc();
   terminator.erase();
   builder.setInsertionPointToEnd(innermostForOp.getBody());
-  builder.create<gpu::TerminatorOp>(terminatorLoc, TypeRange());
+  gpu::TerminatorOp::create(builder, terminatorLoc, TypeRange());
   launchOp.getBody().front().getOperations().splice(
       launchOp.getBody().front().begin(),
       innermostForOp.getBody()->getOperations());
@@ -263,10 +263,10 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
             : getDim3Value(launchOp.getThreadIds(), en.index() - numBlockDims);
     Value step = steps[en.index()];
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
-      id = builder.create<arith::MulIOp>(rootForOp.getLoc(), step, id);
+      id = arith::MulIOp::create(builder, rootForOp.getLoc(), step, id);
 
     Value ivReplacement =
-        builder.create<arith::AddIOp>(rootForOp.getLoc(), *lbArgumentIt, id);
+        arith::AddIOp::create(builder, rootForOp.getLoc(), *lbArgumentIt, id);
     en.value().replaceAllUsesWith(ivReplacement);
     std::advance(lbArgumentIt, 1);
     std::advance(stepArgumentIt, 1);
@@ -319,7 +319,7 @@ static Value deriveStaticUpperBound(Value upperBound,
   if (auto minOp = upperBound.getDefiningOp<AffineMinOp>()) {
     for (const AffineExpr &result : minOp.getMap().getResults()) {
       if (auto constExpr = dyn_cast<AffineConstantExpr>(result)) {
-        return rewriter.create<arith::ConstantIndexOp>(minOp.getLoc(),
+        return arith::ConstantIndexOp::create(rewriter, minOp.getLoc(),
                                                        constExpr.getValue());
       }
     }
@@ -344,7 +344,7 @@ static Value deriveStaticUpperBound(Value upperBound,
         if ((lhs.value() < 0) != (rhs.value() < 0))
           return {};
 
-        return rewriter.create<arith::ConstantIndexOp>(
+        return arith::ConstantIndexOp::create(rewriter,
             multiplyOp.getLoc(), lhs.value() * rhs.value());
       }
   }
@@ -422,7 +422,7 @@ static LogicalResult processParallelLoop(
     if (launchIndependent(val))
       return val;
     if (auto constOp = val.getDefiningOp<arith::ConstantOp>())
-      return rewriter.create<arith::ConstantOp>(constOp.getLoc(),
+      return arith::ConstantOp::create(rewriter, constOp.getLoc(),
                                                 constOp.getValue());
     return {};
   };
@@ -453,7 +453,7 @@ static LogicalResult processParallelLoop(
           1, 2,
           rewriter.getAffineDimExpr(0) * rewriter.getAffineSymbolExpr(0) +
               rewriter.getAffineSymbolExpr(1));
-      newIndex = rewriter.create<AffineApplyOp>(
+      newIndex = AffineApplyOp::create(rewriter,
           loc, annotation.getMap().compose(lowerAndStep),
           ValueRange{operand, ensureLaunchIndependent(step),
                      ensureLaunchIndependent(lowerBound)});
@@ -498,7 +498,7 @@ static LogicalResult processParallelLoop(
               1, 2,
               ((rewriter.getAffineDimExpr(0) - rewriter.getAffineSymbolExpr(0))
                    .ceilDiv(rewriter.getAffineSymbolExpr(1))));
-          Value launchBound = rewriter.create<AffineApplyOp>(
+          Value launchBound = AffineApplyOp::create(rewriter,
               loc, annotation.getBound().compose(stepMap),
               ValueRange{
                   ensureLaunchIndependent(
@@ -517,10 +517,10 @@ static LogicalResult processParallelLoop(
         if (!boundIsPrecise) {
           // We are using an approximation, create a surrounding conditional.
           Value originalBound = std::get<3>(config);
-          arith::CmpIOp pred = rewriter.create<arith::CmpIOp>(
+          arith::CmpIOp pred = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::slt, newIndex,
               cloningMap.lookupOrDefault(originalBound));
-          scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, pred, false);
+          scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, pred, false);
           rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
           // Put a sentinel into the worklist so we know when to pop out of the
           // if body again. We use the launchOp here, as that cannot be part of
@@ -530,7 +530,7 @@ static LogicalResult processParallelLoop(
       }
     } else {
       // Create a sequential for loop.
-      auto loopOp = rewriter.create<scf::ForOp>(
+      auto loopOp = scf::ForOp::create(rewriter,
           loc, cloningMap.lookupOrDefault(lowerBound),
           cloningMap.lookupOrDefault(upperBound),
           cloningMap.lookupOrDefault(step));
@@ -608,12 +608,12 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
   // sizes. Those will be refined later as we discover them from mappings.
   Location loc = parallelOp.getLoc();
   Value constantOne =
-      rewriter.create<arith::ConstantIndexOp>(parallelOp.getLoc(), 1);
-  gpu::LaunchOp launchOp = rewriter.create<gpu::LaunchOp>(
+      arith::ConstantIndexOp::create(rewriter, parallelOp.getLoc(), 1);
+  gpu::LaunchOp launchOp = gpu::LaunchOp::create(rewriter,
       parallelOp.getLoc(), constantOne, constantOne, constantOne, constantOne,
       constantOne, constantOne);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
-  rewriter.create<gpu::TerminatorOp>(loc);
+  gpu::TerminatorOp::create(rewriter, loc);
   rewriter.setInsertionPointToStart(&launchOp.getBody().front());
 
   IRMapping cloningMap;
@@ -667,7 +667,7 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
       if (externalValues.size())
         return failure();
       // Replace by gpu.all_reduce.
-      auto gpuRedOp = rewriter.create<gpu::AllReduceOp>(loc, newValue);
+      auto gpuRedOp = gpu::AllReduceOp::create(rewriter, loc, newValue);
       cloningMap.map(parentLoop->getResult(0), gpuRedOp.getResult());
       // Copy region.
       rewriter.inlineRegionBefore(reduceOp.getRegion(0), gpuRedOp.getRegion(),
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 584ac2f11b670..7b4c37d44824d 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -187,7 +187,7 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
            scf::ReduceOp reduce, int64_t reductionIndex, Attribute initValue) {
   OpBuilder::InsertionGuard guard(builder);
   Type type = reduce.getOperands()[reductionIndex].getType();
-  auto decl = builder.create<omp::DeclareReductionOp>(reduce.getLoc(),
+  auto decl = omp::DeclareReductionOp::create(builder, reduce.getLoc(),
                                                       "__scf_reduction", type);
   symbolTable.insert(decl);
 
@@ -196,8 +196,8 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
                       {reduce.getOperands()[reductionIndex].getLoc()});
   builder.setInsertionPointToEnd(&decl.getInitializerRegion().back());
   Value init =
-      builder.create<LLVM::ConstantOp>(reduce.getLoc(), type, initValue);
-  builder.create<omp::YieldOp>(reduce.getLoc(), init);
+      LLVM::ConstantOp::create(builder, reduce.getLoc(), type, initValue);
+  omp::YieldOp::create(builder, reduce.getLoc(), init);
 
   Operation *terminator =
       &reduce.getReductions()[reductionIndex].front().back();
@@ -227,12 +227,12 @@ static omp::DeclareReductionOp addAtomicRMW(OpBuilder &builder,
                       {reduceOperandLoc, reduceOperandLoc});
   Block *atomicBlock = &decl.getAtomicReductionRegion().back();
   builder.setInsertionPointToEnd(atomicBlock);
-  Value loaded = builder.create<LLVM::LoadOp>(reduce.getLoc(), decl.getType(),
+  Value loaded = LLVM::LoadOp::create(builder, reduce.getLoc(), decl.getType(),
                                               atomicBlock->getArgument(1));
-  builder.create<LLVM::AtomicRMWOp>(reduce.getLoc(), atomicKind,
+  LLVM::AtomicRMWOp::create(builder, reduce.getLoc(), atomicKind,
                                     atomicBlock->getArgument(0), loaded,
                                     LLVM::AtomicOrdering::monotonic);
-  builder.create<omp::YieldOp>(reduce.getLoc(), ArrayRef<Value>());
+  omp::YieldOp::create(builder, reduce.getLoc(), ArrayRef<Value>());
   return decl;
 }
 
@@ -380,7 +380,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     // Allocate reduction variables. Make sure the we don't overflow the stack
     // with local `alloca`s by saving and restoring the stack pointer.
     Location loc = parallelOp.getLoc();
-    Value one = rewriter.create<LLVM::ConstantOp>(
+    Value one = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerType(64), rewriter.getI64IntegerAttr(1));
     SmallVector<Value> reductionVariables;
     reductionVariables.reserve(parallelOp.getNumReductions());
@@ -391,8 +391,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
              "cannot create a reduction variable if the type is not an LLVM "
              "pointer element");
       Value storage =
-          rewriter.create<LLVM::AllocaOp>(loc, ptrType, init.getType(), one, 0);
-      rewriter.create<LLVM::StoreOp>(loc, init, storage);
+          LLVM::AllocaOp::create(rewriter, loc, ptrType, init.getType(), one, 0);
+      LLVM::StoreOp::create(rewriter, loc, init, storage);
       reductionVariables.push_back(storage);
     }
 
@@ -411,7 +411,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
       assert(redRegion.hasOneBlock() &&
              "expect reduction region to have one block");
       Value pvtRedVar = parallelOp.getRegion().addArgument(x.getType(), loc);
-      Value pvtRedVal = rewriter.create<LLVM::LoadOp>(reduce.getLoc(),
+      Value pvtRedVal = LLVM::LoadOp::create(rewriter, reduce.getLoc(),
                                                       rD.getType(), pvtRedVar);
       // Make a copy of the reduction combiner region in the body
       mlir::OpBuilder builder(rewriter.getContext());
@@ -427,7 +427,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           assert(yieldOp && yieldOp.getResults().size() == 1 &&
                  "expect YieldOp in reduction region to return one result");
           Value redVal = yieldOp.getResults()[0];
-          rewriter.create<LLVM::StoreOp>(loc, redVal, pvtRedVar);
+          LLVM::StoreOp::create(rewriter, loc, redVal, pvtRedVar);
           rewriter.eraseOp(yieldOp);
           break;
         }
@@ -437,11 +437,11 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
 
     Value numThreadsVar;
     if (numThreads > 0) {
-      numThreadsVar = rewriter.create<LLVM::ConstantOp>(
+      numThreadsVar = LLVM::ConstantOp::create(rewriter,
           loc, rewriter.getI32IntegerAttr(numThreads));
     }
     // Create the parallel wrapper.
-    auto ompParallel = rewriter.create<omp::ParallelOp>(
+    auto ompParallel = omp::ParallelOp::create(rewriter,
         loc,
         /* allocate_vars = */ llvm::SmallVector<Value>{},
         /* allocator_vars = */ llvm::SmallVector<Value>{},
@@ -464,7 +464,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
       {
         OpBuilder::InsertionGuard allocaGuard(rewriter);
         // Create worksharing loop wrapper.
-        auto wsloopOp = rewriter.create<omp::WsloopOp>(parallelOp.getLoc());
+        auto wsloopOp = omp::WsloopOp::create(rewriter, parallelOp.getLoc());
         if (!reductionVariables.empty()) {
           wsloopOp.setReductionSymsAttr(
               ArrayAttr::get(rewriter.getContext(), reductionSyms));
@@ -476,7 +476,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           wsloopOp.setReductionByref(
               DenseBoolArrayAttr::get(rewriter.getContext(), reductionByRef));
         }
-        rewriter.create<omp::TerminatorOp>(loc); // omp.parallel terminator.
+        omp::TerminatorOp::create(rewriter, loc); // omp.parallel terminator.
 
         // The wrapper's entry block arguments will define the reduction
         // variables.
@@ -490,7 +490,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
                                               parallelOp.getLoc()));
 
         // Create loop nest and populate region with contents of scf.parallel.
-        auto loopOp = rewriter.create<omp::LoopNestOp>(
+        auto loopOp = omp::LoopNestOp::create(rewriter,
             parallelOp.getLoc(), parallelOp.getLowerBound(),
             parallelOp.getUpperBound(), parallelOp.getStep());
 
@@ -511,13 +511,13 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
             rewriter.splitBlock(&loopOpEntryBlock, loopOpEntryBlock.begin());
         rewriter.setInsertionPointToStart(&loopOpEntryBlock);
 
-        auto scope = rewriter.create<memref::AllocaScopeOp>(parallelOp.getLoc(),
+        auto scope = memref::AllocaScopeOp::create(rewriter, parallelOp.getLoc(),
                                                             TypeRange());
-        rewriter.create<omp::YieldOp>(loc, ValueRange());
+        omp::YieldOp::create(rewriter, loc, ValueRange());
         Block *scopeBlock = rewriter.createBlock(&scope.getBodyRegion());
         rewriter.mergeBlocks(ops, scopeBlock);
         rewriter.setInsertionPointToEnd(&*scope.getBodyRegion().begin());
-        rewriter.create<memref::AllocaScopeReturnOp>(loc, ValueRange());
+        memref::AllocaScopeReturnOp::create(rewriter, loc, ValueRange());
       }
     }
 
@@ -526,7 +526,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     results.reserve(reductionVariables.size());
     for (auto [variable, type] :
          llvm::zip(reductionVariables, parallelOp.getResultTypes())) {
-      Value res = rewriter.create<LLVM::LoadOp>(loc, type, variable);
+      Value res = LLVM::LoadOp::create(rewriter, loc, type, variable);
       results.push_back(res);
     }
     rewriter.replaceOp(parallelOp, results);
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index 78d13278fef53..b494fa98f1362 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -71,12 +71,12 @@ void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp,
     auto pointerType =
         spirv::PointerType::get(convertedType, spirv::StorageClass::Function);
     rewriter.setInsertionPoint(newOp);
-    auto alloc = rewriter.create<spirv::VariableOp>(
+    auto alloc = spirv::VariableOp::create(rewriter,
         loc, pointerType, spirv::StorageClass::Function,
         /*initializer=*/nullptr);
     allocas.push_back(alloc);
     rewriter.setInsertionPointAfter(newOp);
-    Value loadResult = rewriter.create<spirv::LoadOp>(loc, alloc);
+    Value loadResult = spirv::LoadOp::create(rewriter, loc, alloc);
     resultValue.push_back(loadResult);
   }
   rewriter.replaceOp(scfOp, resultValue);
@@ -135,7 +135,7 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // a single back edge from the continue to header block, and a single exit
     // from header to merge.
     auto loc = forOp.getLoc();
-    auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
+    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     OpBuilder::InsertionGuard guard(rewriter);
@@ -172,15 +172,15 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     args.append(adaptor.getInitArgs().begin(), adaptor.getInitArgs().end());
     // Branch into it from the entry.
     rewriter.setInsertionPointToEnd(&(loopOp.getBody().front()));
-    rewriter.create<spirv::BranchOp>(loc, header, args);
+    spirv::BranchOp::create(rewriter, loc, header, args);
 
     // Generate the rest of the loop header.
     rewriter.setInsertionPointToEnd(header);
     auto *mergeBlock = loopOp.getMergeBlock();
-    auto cmpOp = rewriter.create<spirv::SLessThanOp>(
+    auto cmpOp = spirv::SLessThanOp::create(rewriter,
         loc, rewriter.getI1Type(), newIndVar, adaptor.getUpperBound());
 
-    rewriter.create<spirv::BranchConditionalOp>(
+    spirv::BranchConditionalOp::create(rewriter,
         loc, cmpOp, body, ArrayRef<Value>(), mergeBlock, ArrayRef<Value>());
 
     // Generate instructions to increment the step of the induction variable and
@@ -189,9 +189,9 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     rewriter.setInsertionPointToEnd(continueBlock);
 
     // Add the step to the induction variable and branch to the header.
-    Value updatedIndVar = rewriter.create<spirv::IAddOp>(
+    Value updatedIndVar = spirv::IAddOp::create(rewriter,
         loc, newIndVar.getType(), newIndVar, adaptor.getStep());
-    rewriter.create<spirv::BranchOp>(loc, header, updatedIndVar);
+    spirv::BranchOp::create(rewriter, loc, header, updatedIndVar);
 
     // Infer the return types from the init operands. Vector type may get
     // converted to CooperativeMatrix or to Vector type, to avoid having complex
@@ -238,10 +238,10 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     // Create `spirv.selection` operation, selection header block and merge
     // block.
     auto selectionOp =
-        rewriter.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
+        spirv::SelectionOp::create(rewriter, loc, spirv::SelectionControl::None);
     auto *mergeBlock = rewriter.createBlock(&selectionOp.getBody(),
                                             selectionOp.getBody().end());
-    rewriter.create<spirv::MergeOp>(loc);
+    spirv::MergeOp::create(rewriter, loc);
 
     OpBuilder::InsertionGuard guard(rewriter);
     auto *selectionHeaderBlock =
@@ -251,7 +251,7 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     auto &thenRegion = ifOp.getThenRegion();
     auto *thenBlock = &thenRegion.front();
     rewriter.setInsertionPointToEnd(&thenRegion.back());
-    rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+    spirv::BranchOp::create(rewriter, loc, mergeBlock);
     rewriter.inlineRegionBefore(thenRegion, mergeBlock);
 
     auto *elseBlock = mergeBlock;
@@ -261,13 +261,13 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
       auto &elseRegion = ifOp.getElseRegion();
       elseBlock = &elseRegion.front();
       rewriter.setInsertionPointToEnd(&elseRegion.back());
-      rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+      spirv::BranchOp::create(rewriter, loc, mergeBlock);
       rewriter.inlineRegionBefore(elseRegion, mergeBlock);
     }
 
     // Create a `spirv.BranchConditional` operation for selection header block.
     rewriter.setInsertionPointToEnd(selectionHeaderBlock);
-    rewriter.create<spirv::BranchConditionalOp>(loc, adaptor.getCondition(),
+    spirv::BranchConditionalOp::create(rewriter, loc, adaptor.getCondition(),
                                                 thenBlock, ArrayRef<Value>(),
                                                 elseBlock, ArrayRef<Value>());
 
@@ -310,7 +310,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
 
       auto loc = terminatorOp.getLoc();
       for (unsigned i = 0, e = operands.size(); i < e; i++)
-        rewriter.create<spirv::StoreOp>(loc, allocas[i], operands[i]);
+        spirv::StoreOp::create(rewriter, loc, allocas[i], operands[i]);
       if (isa<spirv::LoopOp>(parent)) {
         // For loops we also need to update the branch jumping back to the
         // header.
@@ -319,7 +319,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
         SmallVector<Value, 8> args(br.getBlockArguments());
         args.append(operands.begin(), operands.end());
         rewriter.setInsertionPoint(br);
-        rewriter.create<spirv::BranchOp>(terminatorOp.getLoc(), br.getTarget(),
+        spirv::BranchOp::create(rewriter, terminatorOp.getLoc(), br.getTarget(),
                                          args);
         rewriter.eraseOp(br);
       }
@@ -340,7 +340,7 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
   matchAndRewrite(scf::WhileOp whileOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto loc = whileOp.getLoc();
-    auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
+    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     Region &beforeRegion = whileOp.getBefore();
@@ -382,7 +382,7 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
     // Jump from the loop entry block to the loop header block.
     rewriter.setInsertionPointToEnd(&entryBlock);
-    rewriter.create<spirv::BranchOp>(loc, &beforeBlock, adaptor.getInits());
+    spirv::BranchOp::create(rewriter, loc, &beforeBlock, adaptor.getInits());
 
     auto condLoc = cond.getLoc();
 
@@ -403,18 +403,18 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
       // Create local variables before the scf.while op.
       rewriter.setInsertionPoint(loopOp);
-      auto alloc = rewriter.create<spirv::VariableOp>(
+      auto alloc = spirv::VariableOp::create(rewriter,
           condLoc, pointerType, spirv::StorageClass::Function,
           /*initializer=*/nullptr);
 
       // Load the final result values after the scf.while op.
       rewriter.setInsertionPointAfter(loopOp);
-      auto loadResult = rewriter.create<spirv::LoadOp>(condLoc, alloc);
+      auto loadResult = spirv::LoadOp::create(rewriter, condLoc, alloc);
       resultValues[i] = loadResult;
 
       // Store the current iteration's result value.
       rewriter.setInsertionPointToEnd(&beforeBlock);
-      rewriter.create<spirv::StoreOp>(condLoc, alloc, res);
+      spirv::StoreOp::create(rewriter, condLoc, alloc, res);
     }
 
     rewriter.setInsertionPointToEnd(&beforeBlock);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index d7ae9f0e94fe8..934bb0d1c37fd 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -68,7 +68,7 @@ static unsigned calculateGlobalIndex(spirv::GlobalVariableOp op) {
 /// Copies the given number of bytes from src to dst pointers.
 static void copy(Location loc, Value dst, Value src, Value size,
                  OpBuilder &builder) {
-  builder.create<LLVM::MemcpyOp>(loc, dst, src, size, /*isVolatile=*/false);
+  LLVM::MemcpyOp::create(builder, loc, dst, src, size, /*isVolatile=*/false);
 }
 
 /// Encodes the binding and descriptor set numbers into a new symbolic name.
@@ -194,7 +194,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
     if (!kernelFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(module.getBody());
-      kernelFunc = rewriter.create<LLVM::LLVMFuncOp>(
+      kernelFunc = LLVM::LLVMFuncOp::create(rewriter,
           rewriter.getUnknownLoc(), newKernelFuncName,
           LLVM::LLVMFunctionType::get(LLVM::LLVMVoidType::get(context),
                                       ArrayRef<Type>()));
@@ -245,7 +245,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       if (!dstGlobal) {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
-        dstGlobal = rewriter.create<LLVM::GlobalOp>(
+        dstGlobal = LLVM::GlobalOp::create(rewriter,
             loc, dstGlobalType,
             /*isConstant=*/false, LLVM::Linkage::Linkonce, name, Attribute(),
             /*alignment=*/0);
@@ -255,7 +255,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       // Copy the data from src operand pointer to dst global variable. Save
       // src, dst and size so that we can copy data back after emulating the
       // kernel call.
-      Value dst = rewriter.create<LLVM::AddressOfOp>(
+      Value dst = LLVM::AddressOfOp::create(rewriter,
           loc, typeConverter->convertType(spirvGlobal.getType()),
           dstGlobal.getSymName());
       copy(loc, dst, src, sizeBytes, rewriter);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 1d92b5d5562b5..3eee444725f74 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -94,12 +94,12 @@ static IntegerAttr minusOneIntegerAttribute(Type type, Builder builder) {
 static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
                                       PatternRewriter &rewriter) {
   if (isa<VectorType>(srcType)) {
-    return rewriter.create<LLVM::ConstantOp>(
+    return LLVM::ConstantOp::create(rewriter,
         loc, dstType,
         SplatElementsAttr::get(cast<ShapedType>(srcType),
                                minusOneIntegerAttribute(srcType, rewriter)));
   }
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, dstType, minusOneIntegerAttribute(srcType, rewriter));
 }
 
@@ -108,13 +108,13 @@ static Value createFPConstant(Location loc, Type srcType, Type dstType,
                               PatternRewriter &rewriter, double value) {
   if (auto vecType = dyn_cast<VectorType>(srcType)) {
     auto floatType = cast<FloatType>(vecType.getElementType());
-    return rewriter.create<LLVM::ConstantOp>(
+    return LLVM::ConstantOp::create(rewriter,
         loc, dstType,
         SplatElementsAttr::get(vecType,
                                rewriter.getFloatAttr(floatType, value)));
   }
   auto floatType = cast<FloatType>(srcType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, dstType, rewriter.getFloatAttr(floatType, value));
 }
 
@@ -134,13 +134,13 @@ static Value optionallyTruncateOrExtend(Location loc, Value value,
                                : getBitWidth(srcType);
 
   if (valueBitWidth < targetBitWidth)
-    return rewriter.create<LLVM::ZExtOp>(loc, llvmType, value);
+    return LLVM::ZExtOp::create(rewriter, loc, llvmType, value);
   // If the bit widths of `Count` and `Offset` are greater than the bit width
   // of the target type, they are truncated. Truncation is safe since `Count`
   // and `Offset` must be no more than 64 for op behaviour to be defined. Hence,
   // both values can be expressed in 8 bits.
   if (valueBitWidth > targetBitWidth)
-    return rewriter.create<LLVM::TruncOp>(loc, llvmType, value);
+    return LLVM::TruncOp::create(rewriter, loc, llvmType, value);
   return value;
 }
 
@@ -151,11 +151,11 @@ static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
   auto vectorType = VectorType::get(numElements, toBroadcast.getType());
   auto llvmVectorType = typeConverter.convertType(vectorType);
   auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
-  Value broadcasted = rewriter.create<LLVM::PoisonOp>(loc, llvmVectorType);
+  Value broadcasted = LLVM::PoisonOp::create(rewriter, loc, llvmVectorType);
   for (unsigned i = 0; i < numElements; ++i) {
-    auto index = rewriter.create<LLVM::ConstantOp>(
+    auto index = LLVM::ConstantOp::create(rewriter,
         loc, llvmI32Type, rewriter.getI32IntegerAttr(i));
-    broadcasted = rewriter.create<LLVM::InsertElementOp>(
+    broadcasted = LLVM::InsertElementOp::create(rewriter,
         loc, llvmVectorType, broadcasted, toBroadcast, index);
   }
   return broadcasted;
@@ -217,7 +217,7 @@ static Type convertStructTypePacked(spirv::StructType type,
 /// Creates LLVM dialect constant with the given value.
 static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
                                  unsigned value) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, IntegerType::get(rewriter.getContext(), 32),
       rewriter.getIntegerAttr(rewriter.getI32Type(), value));
 }
@@ -322,7 +322,7 @@ class AccessChainPattern : public SPIRVToLLVMConversion<spirv::AccessChainOp> {
     auto llvmIndexType = getTypeConverter()->convertType(indexType);
     if (!llvmIndexType)
       return rewriter.notifyMatchFailure(op, "type conversion failed");
-    Value zero = rewriter.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(rewriter,
         op.getLoc(), llvmIndexType, rewriter.getIntegerAttr(indexType, 0));
     indices.insert(indices.begin(), zero);
 
@@ -375,20 +375,20 @@ class BitFieldInsertPattern
     // Create a mask with bits set outside [Offset, Offset + Count - 1].
     Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
     Value maskShiftedByCount =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
-    Value negated = rewriter.create<LLVM::XOrOp>(loc, dstType,
+        LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
+    Value negated = LLVM::XOrOp::create(rewriter, loc, dstType,
                                                  maskShiftedByCount, minusOne);
     Value maskShiftedByCountAndOffset =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, negated, offset);
-    Value mask = rewriter.create<LLVM::XOrOp>(
+        LLVM::ShlOp::create(rewriter, loc, dstType, negated, offset);
+    Value mask = LLVM::XOrOp::create(rewriter,
         loc, dstType, maskShiftedByCountAndOffset, minusOne);
 
     // Extract unchanged bits from the `Base`  that are outside of
     // [Offset, Offset + Count - 1]. Then `or` with shifted `Insert`.
     Value baseAndMask =
-        rewriter.create<LLVM::AndOp>(loc, dstType, op.getBase(), mask);
+        LLVM::AndOp::create(rewriter, loc, dstType, op.getBase(), mask);
     Value insertShiftedByOffset =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, op.getInsert(), offset);
+        LLVM::ShlOp::create(rewriter, loc, dstType, op.getInsert(), offset);
     rewriter.replaceOpWithNewOp<LLVM::OrOp>(op, dstType, baseAndMask,
                                             insertShiftedByOffset);
     return success();
@@ -470,23 +470,23 @@ class BitFieldSExtractPattern
     auto baseSize = rewriter.getIntegerAttr(integerType, getBitWidth(srcType));
     Value size =
         isa<VectorType>(srcType)
-            ? rewriter.create<LLVM::ConstantOp>(
+            ? LLVM::ConstantOp::create(rewriter,
                   loc, dstType,
                   SplatElementsAttr::get(cast<ShapedType>(srcType), baseSize))
-            : rewriter.create<LLVM::ConstantOp>(loc, dstType, baseSize);
+            : LLVM::ConstantOp::create(rewriter, loc, dstType, baseSize);
 
     // Shift `Base` left by [sizeof(Base) - (Count + Offset)], so that the bit
     // at Offset + Count - 1 is the most significant bit now.
     Value countPlusOffset =
-        rewriter.create<LLVM::AddOp>(loc, dstType, count, offset);
+        LLVM::AddOp::create(rewriter, loc, dstType, count, offset);
     Value amountToShiftLeft =
-        rewriter.create<LLVM::SubOp>(loc, dstType, size, countPlusOffset);
-    Value baseShiftedLeft = rewriter.create<LLVM::ShlOp>(
+        LLVM::SubOp::create(rewriter, loc, dstType, size, countPlusOffset);
+    Value baseShiftedLeft = LLVM::ShlOp::create(rewriter,
         loc, dstType, op.getBase(), amountToShiftLeft);
 
     // Shift the result right, filling the bits with the sign bit.
     Value amountToShiftRight =
-        rewriter.create<LLVM::AddOp>(loc, dstType, offset, amountToShiftLeft);
+        LLVM::AddOp::create(rewriter, loc, dstType, offset, amountToShiftLeft);
     rewriter.replaceOpWithNewOp<LLVM::AShrOp>(op, dstType, baseShiftedLeft,
                                               amountToShiftRight);
     return success();
@@ -516,13 +516,13 @@ class BitFieldUExtractPattern
     // Create a mask with bits set at [0, Count - 1].
     Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
     Value maskShiftedByCount =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
-    Value mask = rewriter.create<LLVM::XOrOp>(loc, dstType, maskShiftedByCount,
+        LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
+    Value mask = LLVM::XOrOp::create(rewriter, loc, dstType, maskShiftedByCount,
                                               minusOne);
 
     // Shift `Base` by `Offset` and apply the mask on it.
     Value shiftedBase =
-        rewriter.create<LLVM::LShrOp>(loc, dstType, op.getBase(), offset);
+        LLVM::LShrOp::create(rewriter, loc, dstType, op.getBase(), offset);
     rewriter.replaceOpWithNewOp<LLVM::AndOp>(op, dstType, shiftedBase, mask);
     return success();
   }
@@ -694,7 +694,7 @@ class ExecutionModePattern
     auto structType = LLVM::LLVMStructType::getLiteral(context, fields);
 
     // Create `llvm.mlir.global` with initializer region containing one block.
-    auto global = rewriter.create<LLVM::GlobalOp>(
+    auto global = LLVM::GlobalOp::create(rewriter,
         UnknownLoc::get(context), structType, /*isConstant=*/true,
         LLVM::Linkage::External, executionModeInfoName, Attribute(),
         /*alignment=*/0);
@@ -704,22 +704,22 @@ class ExecutionModePattern
 
     // Initialize the struct and set the execution mode value.
     rewriter.setInsertionPointToStart(block);
-    Value structValue = rewriter.create<LLVM::PoisonOp>(loc, structType);
-    Value executionMode = rewriter.create<LLVM::ConstantOp>(
+    Value structValue = LLVM::PoisonOp::create(rewriter, loc, structType);
+    Value executionMode = LLVM::ConstantOp::create(rewriter,
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
-    structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
+    structValue = LLVM::InsertValueOp::create(rewriter, loc, structValue,
                                                        executionMode, 0);
 
     // Insert extra operands if they exist into execution mode info struct.
     for (unsigned i = 0, e = values.size(); i < e; ++i) {
       auto attr = values.getValue()[i];
-      Value entry = rewriter.create<LLVM::ConstantOp>(loc, llvmI32Type, attr);
-      structValue = rewriter.create<LLVM::InsertValueOp>(
+      Value entry = LLVM::ConstantOp::create(rewriter, loc, llvmI32Type, attr);
+      structValue = LLVM::InsertValueOp::create(rewriter,
           loc, structValue, entry, ArrayRef<int64_t>({1, i}));
     }
-    rewriter.create<LLVM::ReturnOp>(loc, ArrayRef<Value>({structValue}));
+    LLVM::ReturnOp::create(rewriter, loc, ArrayRef<Value>({structValue}));
     rewriter.eraseOp(op);
     return success();
   }
@@ -913,7 +913,7 @@ class InverseSqrtPattern
 
     Location loc = op.getLoc();
     Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
-    Value sqrt = rewriter.create<LLVM::SqrtOp>(loc, dstType, op.getOperand());
+    Value sqrt = LLVM::SqrtOp::create(rewriter, loc, dstType, op.getOperand());
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(op, dstType, one, sqrt);
     return success();
   }
@@ -973,10 +973,10 @@ class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
     IntegerAttr minusOne = minusOneIntegerAttribute(srcType, rewriter);
     auto mask =
         isa<VectorType>(srcType)
-            ? rewriter.create<LLVM::ConstantOp>(
+            ? LLVM::ConstantOp::create(rewriter,
                   loc, dstType,
                   SplatElementsAttr::get(cast<VectorType>(srcType), minusOne))
-            : rewriter.create<LLVM::ConstantOp>(loc, dstType, minusOne);
+            : LLVM::ConstantOp::create(rewriter, loc, dstType, minusOne);
     rewriter.template replaceOpWithNewOp<LLVM::XOrOp>(notOp, dstType,
                                                       notOp.getOperand(), mask);
     return success();
@@ -1034,7 +1034,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
     return func;
 
   OpBuilder b(symbolTable->getRegion(0));
-  func = b.create<LLVM::LLVMFuncOp>(
+  func = LLVM::LLVMFuncOp::create(b,
       symbolTable->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes));
   func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
@@ -1047,7 +1047,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
 static LLVM::CallOp createSPIRVBuiltinCall(Location loc, OpBuilder &builder,
                                            LLVM::LLVMFuncOp func,
                                            ValueRange args) {
-  auto call = builder.create<LLVM::CallOp>(loc, func, args);
+  auto call = LLVM::CallOp::create(builder, loc, func, args);
   call.setCConv(func.getCConv());
   call.setConvergentAttr(func.getConvergentAttr());
   call.setNoUnwindAttr(func.getNoUnwindAttr());
@@ -1078,11 +1078,11 @@ class ControlBarrierPattern : public SPIRVToLLVMConversion<BarrierOpTy> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, {i32, i32, i32}, voidTy);
 
     Location loc = controlBarrierOp->getLoc();
-    Value execution = rewriter.create<LLVM::ConstantOp>(
+    Value execution = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value memory = rewriter.create<LLVM::ConstantOp>(
+    Value memory = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getMemoryScope()));
-    Value semantics = rewriter.create<LLVM::ConstantOp>(
+    Value semantics = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getMemorySemantics()));
 
     auto call = createSPIRVBuiltinCall(loc, rewriter, func,
@@ -1255,9 +1255,9 @@ class GroupReducePattern : public SPIRVToLLVMConversion<ReduceOp> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, paramTypes, retTy);
 
     Location loc = op.getLoc();
-    Value scope = rewriter.create<LLVM::ConstantOp>(
+    Value scope = LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value groupOp = rewriter.create<LLVM::ConstantOp>(
+    Value groupOp = LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, static_cast<int32_t>(adaptor.getGroupOperation()));
     SmallVector<Value> operands{scope, groupOp};
     operands.append(adaptor.getOperands().begin(), adaptor.getOperands().end());
@@ -1368,7 +1368,7 @@ class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
       return failure();
     Block *headerBlock = loopOp.getHeaderBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
-    rewriter.create<LLVM::BrOp>(loc, brOp.getBlockArguments(), headerBlock);
+    LLVM::BrOp::create(rewriter, loc, brOp.getBlockArguments(), headerBlock);
     rewriter.eraseBlock(entryBlock);
 
     // Branch from merge block to end block.
@@ -1376,7 +1376,7 @@ class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
     Operation *terminator = mergeBlock->getTerminator();
     ValueRange terminatorOperands = terminator->getOperands();
     rewriter.setInsertionPointToEnd(mergeBlock);
-    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, endBlock);
+    LLVM::BrOp::create(rewriter, loc, terminatorOperands, endBlock);
 
     rewriter.inlineRegionBefore(loopOp.getBody(), endBlock);
     rewriter.replaceOp(loopOp, endBlock->getArguments());
@@ -1434,13 +1434,13 @@ class SelectionPattern : public SPIRVToLLVMConversion<spirv::SelectionOp> {
     Operation *terminator = mergeBlock->getTerminator();
     ValueRange terminatorOperands = terminator->getOperands();
     rewriter.setInsertionPointToEnd(mergeBlock);
-    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, continueBlock);
+    LLVM::BrOp::create(rewriter, loc, terminatorOperands, continueBlock);
 
     // Link current block to `true` and `false` blocks within the selection.
     Block *trueBlock = condBrOp.getTrueBlock();
     Block *falseBlock = condBrOp.getFalseBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
-    rewriter.create<LLVM::CondBrOp>(loc, condBrOp.getCondition(), trueBlock,
+    LLVM::CondBrOp::create(rewriter, loc, condBrOp.getCondition(), trueBlock,
                                     condBrOp.getTrueTargetOperands(),
                                     falseBlock,
                                     condBrOp.getFalseTargetOperands());
@@ -1521,8 +1521,8 @@ class TanPattern : public SPIRVToLLVMConversion<spirv::GLTanOp> {
       return rewriter.notifyMatchFailure(tanOp, "type conversion failed");
 
     Location loc = tanOp.getLoc();
-    Value sin = rewriter.create<LLVM::SinOp>(loc, dstType, tanOp.getOperand());
-    Value cos = rewriter.create<LLVM::CosOp>(loc, dstType, tanOp.getOperand());
+    Value sin = LLVM::SinOp::create(rewriter, loc, dstType, tanOp.getOperand());
+    Value cos = LLVM::CosOp::create(rewriter, loc, dstType, tanOp.getOperand());
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanOp, dstType, sin, cos);
     return success();
   }
@@ -1549,13 +1549,13 @@ class TanhPattern : public SPIRVToLLVMConversion<spirv::GLTanhOp> {
     Location loc = tanhOp.getLoc();
     Value two = createFPConstant(loc, srcType, dstType, rewriter, 2.0);
     Value multiplied =
-        rewriter.create<LLVM::FMulOp>(loc, dstType, two, tanhOp.getOperand());
-    Value exponential = rewriter.create<LLVM::ExpOp>(loc, dstType, multiplied);
+        LLVM::FMulOp::create(rewriter, loc, dstType, two, tanhOp.getOperand());
+    Value exponential = LLVM::ExpOp::create(rewriter, loc, dstType, multiplied);
     Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
     Value numerator =
-        rewriter.create<LLVM::FSubOp>(loc, dstType, exponential, one);
+        LLVM::FSubOp::create(rewriter, loc, dstType, exponential, one);
     Value denominator =
-        rewriter.create<LLVM::FAddOp>(loc, dstType, exponential, one);
+        LLVM::FAddOp::create(rewriter, loc, dstType, exponential, one);
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanhOp, dstType, numerator,
                                               denominator);
     return success();
@@ -1594,8 +1594,8 @@ class VariablePattern : public SPIRVToLLVMConversion<spirv::VariableOp> {
     if (!elementType)
       return rewriter.notifyMatchFailure(varOp, "type conversion failed");
     Value allocated =
-        rewriter.create<LLVM::AllocaOp>(loc, dstType, elementType, size);
-    rewriter.create<LLVM::StoreOp>(loc, adaptor.getInitializer(), allocated);
+        LLVM::AllocaOp::create(rewriter, loc, dstType, elementType, size);
+    LLVM::StoreOp::create(rewriter, loc, adaptor.getInitializer(), allocated);
     rewriter.replaceOp(varOp, allocated);
     return success();
   }
@@ -1656,7 +1656,7 @@ class FuncConversionPattern : public SPIRVToLLVMConversion<spirv::FuncOp> {
     // Create a new `LLVMFuncOp`
     Location loc = funcOp.getLoc();
     StringRef name = funcOp.getName();
-    auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(loc, name, llvmType);
+    auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter, loc, name, llvmType);
 
     // Convert SPIR-V Function Control to equivalent LLVM function attribute
     MLIRContext *context = funcOp.getContext();
@@ -1710,7 +1710,7 @@ class ModuleConversionPattern : public SPIRVToLLVMConversion<spirv::ModuleOp> {
                   ConversionPatternRewriter &rewriter) const override {
 
     auto newModuleOp =
-        rewriter.create<ModuleOp>(spvModuleOp.getLoc(), spvModuleOp.getName());
+        ModuleOp::create(rewriter, spvModuleOp.getLoc(), spvModuleOp.getName());
     rewriter.inlineRegionBefore(spvModuleOp.getRegion(), newModuleOp.getBody());
 
     // Remove the terminator block that was automatically added by builder
@@ -1751,7 +1751,7 @@ class VectorShufflePattern
     auto componentsArray = components.getValue();
     auto *context = rewriter.getContext();
     auto llvmI32Type = IntegerType::get(context, 32);
-    Value targetOp = rewriter.create<LLVM::PoisonOp>(loc, dstType);
+    Value targetOp = LLVM::PoisonOp::create(rewriter, loc, dstType);
     for (unsigned i = 0; i < componentsArray.size(); i++) {
       if (!isa<IntegerAttr>(componentsArray[i]))
         return op.emitError("unable to support non-constant component");
@@ -1767,15 +1767,15 @@ class VectorShufflePattern
         baseVector = vector2;
       }
 
-      Value dstIndex = rewriter.create<LLVM::ConstantOp>(
+      Value dstIndex = LLVM::ConstantOp::create(rewriter,
           loc, llvmI32Type, rewriter.getIntegerAttr(rewriter.getI32Type(), i));
-      Value index = rewriter.create<LLVM::ConstantOp>(
+      Value index = LLVM::ConstantOp::create(rewriter,
           loc, llvmI32Type,
           rewriter.getIntegerAttr(rewriter.getI32Type(), indexVal - offsetVal));
 
-      auto extractOp = rewriter.create<LLVM::ExtractElementOp>(
+      auto extractOp = LLVM::ExtractElementOp::create(rewriter,
           loc, scalarType, baseVector, index);
-      targetOp = rewriter.create<LLVM::InsertElementOp>(loc, dstType, targetOp,
+      targetOp = LLVM::InsertElementOp::create(rewriter, loc, dstType, targetOp,
                                                         extractOp, dstIndex);
     }
     rewriter.replaceOp(op, targetOp);
diff --git a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
index da9ad3dd67328..245e60b04ec31 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
@@ -32,7 +32,7 @@ class ConvertCstrRequireOp : public OpRewritePattern<shape::CstrRequireOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(shape::CstrRequireOp op,
                                 PatternRewriter &rewriter) const override {
-    rewriter.create<cf::AssertOp>(op.getLoc(), op.getPred(), op.getMsgAttr());
+    cf::AssertOp::create(rewriter, op.getLoc(), op.getPred(), op.getMsgAttr());
     rewriter.replaceOpWithNewOp<shape::ConstWitnessOp>(op, true);
     return success();
   }
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index bbe1490137bf8..58cffa6119d93 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -82,19 +82,19 @@ struct BroadcastOpConverter : public OpConversionPattern<BroadcastOp> {
 // number of extent tensors and shifted offsets into them.
 Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
                         ValueRange rankDiffs, Value outputDimension) {
-  Value one = lb.create<arith::ConstantIndexOp>(1);
+  Value one = arith::ConstantIndexOp::create(lb, 1);
   Value broadcastedDim = one;
   for (auto tup : llvm::zip(extentTensors, rankDiffs)) {
     Value shape = std::get<0>(tup);
     Value rankDiff = std::get<1>(tup);
-    Value outOfBounds = lb.create<arith::CmpIOp>(arith::CmpIPredicate::ult,
+    Value outOfBounds = arith::CmpIOp::create(lb, arith::CmpIPredicate::ult,
                                                  outputDimension, rankDiff);
     Type indexTy = lb.getIndexType();
     broadcastedDim =
-        lb.create<IfOp>(
+        IfOp::create(lb,
               outOfBounds,
               [&](OpBuilder &b, Location loc) {
-                b.create<scf::YieldOp>(loc, broadcastedDim);
+                scf::YieldOp::create(b, loc, broadcastedDim);
               },
               [&](OpBuilder &b, Location loc) {
                 // The broadcasting logic is:
@@ -104,17 +104,17 @@ Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
                 // - otherwise, take the extent as-is.
                 // Note that this logic remains correct in the presence
                 // of dimensions of zero extent.
-                Value lesserRankOperandDimension = b.create<arith::SubIOp>(
+                Value lesserRankOperandDimension = arith::SubIOp::create(b,
                     loc, indexTy, outputDimension, rankDiff);
-                Value lesserRankOperandExtent = b.create<tensor::ExtractOp>(
+                Value lesserRankOperandExtent = tensor::ExtractOp::create(b,
                     loc, shape, ValueRange{lesserRankOperandDimension});
 
                 Value dimIsOne =
-                    b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+                    arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
                                             lesserRankOperandExtent, one);
-                Value dim = b.create<arith::SelectOp>(
+                Value dim = arith::SelectOp::create(b,
                     loc, dimIsOne, broadcastedDim, lesserRankOperandExtent);
-                b.create<scf::YieldOp>(loc, dim);
+                scf::YieldOp::create(b, loc, dim);
               })
             .getResult(0);
   }
@@ -133,7 +133,7 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
   auto loc = op.getLoc();
   ImplicitLocOpBuilder lb(loc, rewriter);
 
-  Value zero = lb.create<arith::ConstantIndexOp>(0);
+  Value zero = arith::ConstantIndexOp::create(lb, 0);
   Type indexTy = lb.getIndexType();
 
   // Save all the ranks for bounds checking. Because this is a tensor
@@ -141,31 +141,31 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
   // dimension in the tensor.
   SmallVector<Value> ranks, rankDiffs;
   llvm::append_range(ranks, llvm::map_range(adaptor.getShapes(), [&](Value v) {
-                       return lb.create<tensor::DimOp>(v, zero);
+                       return tensor::DimOp::create(lb, v, zero);
                      }));
 
   // Find the maximum rank
   Value maxRank = ranks.front();
   for (Value v : llvm::drop_begin(ranks, 1)) {
-    maxRank = lb.create<arith::MaxUIOp>(v, maxRank);
+    maxRank = arith::MaxUIOp::create(lb, v, maxRank);
   }
 
   // Calculate the difference of ranks and the maximum rank for later offsets.
   llvm::append_range(rankDiffs, llvm::map_range(ranks, [&](Value v) {
-                       return lb.create<arith::SubIOp>(indexTy, maxRank, v);
+                       return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
-  Value replacement = lb.create<tensor::GenerateOp>(
+  Value replacement = tensor::GenerateOp::create(lb,
       getExtentTensorType(lb.getContext()), ValueRange{maxRank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value broadcastedDim =
             getBroadcastedDim(ImplicitLocOpBuilder(loc, b), adaptor.getShapes(),
                               rankDiffs, args[0]);
 
-        b.create<tensor::YieldOp>(loc, broadcastedDim);
+        tensor::YieldOp::create(b, loc, broadcastedDim);
       });
   if (replacement.getType() != op.getType())
-    replacement = lb.create<tensor::CastOp>(op.getType(), replacement);
+    replacement = tensor::CastOp::create(lb, op.getType(), replacement);
   rewriter.replaceOp(op, replacement);
   return success();
 }
@@ -194,12 +194,12 @@ LogicalResult ConstShapeOpConverter::matchAndRewrite(
   SmallVector<Value, 4> extentOperands;
   for (auto extent : op.getShape()) {
     extentOperands.push_back(
-        rewriter.create<arith::ConstantIndexOp>(loc, extent.getLimitedValue()));
+        arith::ConstantIndexOp::create(rewriter, loc, extent.getLimitedValue()));
   }
   Type resultTy =
       RankedTensorType::get({op.getShape().size()}, rewriter.getIndexType());
   Value tensor =
-      rewriter.create<tensor::FromElementsOp>(loc, resultTy, extentOperands);
+      tensor::FromElementsOp::create(rewriter, loc, resultTy, extentOperands);
   rewriter.replaceOpWithNewOp<tensor::CastOp>(op, resultTy, tensor);
   return success();
 }
@@ -245,8 +245,8 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
 
   auto loc = op.getLoc();
   ImplicitLocOpBuilder lb(loc, rewriter);
-  Value zero = lb.create<arith::ConstantIndexOp>(0);
-  Value one = lb.create<arith::ConstantIndexOp>(1);
+  Value zero = arith::ConstantIndexOp::create(lb, 0);
+  Value one = arith::ConstantIndexOp::create(lb, 1);
   Type indexTy = lb.getIndexType();
 
   // Save all the ranks for bounds checking. Because this is a tensor
@@ -254,25 +254,25 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
   // dimension in the tensor.
   SmallVector<Value> ranks, rankDiffs;
   llvm::append_range(ranks, llvm::map_range(adaptor.getShapes(), [&](Value v) {
-                       return lb.create<tensor::DimOp>(v, zero);
+                       return tensor::DimOp::create(lb, v, zero);
                      }));
 
   // Find the maximum rank
   Value maxRank = ranks.front();
   for (Value v : llvm::drop_begin(ranks, 1)) {
-    maxRank = lb.create<arith::MaxUIOp>(v, maxRank);
+    maxRank = arith::MaxUIOp::create(lb, v, maxRank);
   }
 
   // Calculate the difference of ranks and the maximum rank for later offsets.
   llvm::append_range(rankDiffs, llvm::map_range(ranks, [&](Value v) {
-                       return lb.create<arith::SubIOp>(indexTy, maxRank, v);
+                       return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
   Type i1Ty = rewriter.getI1Type();
   Value trueVal =
-      rewriter.create<arith::ConstantOp>(loc, i1Ty, rewriter.getBoolAttr(true));
+      arith::ConstantOp::create(rewriter, loc, i1Ty, rewriter.getBoolAttr(true));
 
-  auto reduceResult = lb.create<ForOp>(
+  auto reduceResult = ForOp::create(lb,
       loc, zero, maxRank, one, ValueRange{trueVal},
       [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
         // Find a non-1 dim, if it exists. Note that the first part of this
@@ -285,38 +285,38 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
         for (auto tup : llvm::zip(adaptor.getShapes(), rankDiffs)) {
           Value shape, rankDiff;
           std::tie(shape, rankDiff) = tup;
-          Value outOfBounds = b.create<arith::CmpIOp>(
+          Value outOfBounds = arith::CmpIOp::create(b,
               loc, arith::CmpIPredicate::ult, iv, rankDiff);
           broadcastable =
-              b.create<IfOp>(
+              IfOp::create(b,
                    loc, outOfBounds,
                    [&](OpBuilder &b, Location loc) {
                      // Non existent dimensions are always broadcastable
-                     b.create<scf::YieldOp>(loc, broadcastable);
+                     scf::YieldOp::create(b, loc, broadcastable);
                    },
                    [&](OpBuilder &b, Location loc) {
                      // Every value needs to be either 1, or the same non-1
                      // value to be broadcastable in this dim.
                      Value operandDimension =
-                         b.create<arith::SubIOp>(loc, indexTy, iv, rankDiff);
-                     Value dimensionExtent = b.create<tensor::ExtractOp>(
+                         arith::SubIOp::create(b, loc, indexTy, iv, rankDiff);
+                     Value dimensionExtent = tensor::ExtractOp::create(b,
                          loc, shape, ValueRange{operandDimension});
 
-                     Value equalOne = b.create<arith::CmpIOp>(
+                     Value equalOne = arith::CmpIOp::create(b,
                          loc, arith::CmpIPredicate::eq, dimensionExtent, one);
-                     Value equalBroadcasted = b.create<arith::CmpIOp>(
+                     Value equalBroadcasted = arith::CmpIOp::create(b,
                          loc, arith::CmpIPredicate::eq, dimensionExtent,
                          broadcastedDim);
-                     Value result = b.create<arith::AndIOp>(
+                     Value result = arith::AndIOp::create(b,
                          loc, broadcastable,
-                         b.create<arith::OrIOp>(loc, equalOne,
+                         arith::OrIOp::create(b, loc, equalOne,
                                                 equalBroadcasted));
-                     b.create<scf::YieldOp>(loc, result);
+                     scf::YieldOp::create(b, loc, result);
                    })
                   .getResult(0);
         }
 
-        b.create<scf::YieldOp>(loc, broadcastable);
+        scf::YieldOp::create(b, loc, broadcastable);
       });
 
   rewriter.replaceOp(op, reduceResult.getResults().front());
@@ -339,7 +339,7 @@ DimOpConverter::matchAndRewrite(DimOp op, OpAdaptor adaptor,
   // Lower to dim(X, i) to get_extent(shape_of(X), i) and rely on further
   // lowerings. This can be further optimized if needed to avoid intermediate
   // steps.
-  auto shapeOf = rewriter.create<shape::ShapeOfOp>(op.getLoc(), op.getValue());
+  auto shapeOf = shape::ShapeOfOp::create(rewriter, op.getLoc(), op.getValue());
   rewriter.replaceOpWithNewOp<shape::GetExtentOp>(op, op.getType(), shapeOf,
                                                   op.getIndex());
   return success();
@@ -421,16 +421,16 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
 
   auto loc = op.getLoc();
 
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
   Type indexTy = rewriter.getIndexType();
   Value rank =
-      rewriter.create<tensor::DimOp>(loc, indexTy, adaptor.getShape(), zero);
+      tensor::DimOp::create(rewriter, loc, indexTy, adaptor.getShape(), zero);
 
-  auto loop = rewriter.create<scf::ForOp>(
+  auto loop = scf::ForOp::create(rewriter,
       loc, zero, rank, one, op.getInitVals(),
       [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
-        Value extent = b.create<tensor::ExtractOp>(loc, adaptor.getShape(), iv);
+        Value extent = tensor::ExtractOp::create(b, loc, adaptor.getShape(), iv);
 
         SmallVector<Value, 2> mappedValues{iv, extent};
         mappedValues.append(args.begin(), args.end());
@@ -444,7 +444,7 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
         SmallVector<Value, 2> mappedResults;
         for (auto result : reduceBody->getTerminator()->getOperands())
           mappedResults.push_back(mapping.lookup(result));
-        b.create<scf::YieldOp>(loc, mappedResults);
+        scf::YieldOp::create(b, loc, mappedResults);
       });
 
   rewriter.replaceOp(op, loop.getResults());
@@ -507,43 +507,43 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, OpAdaptor adaptor,
 
   auto loc = op.getLoc();
   Type indexTy = rewriter.getIndexType();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   Value firstShape = adaptor.getShapes().front();
   Value firstRank =
-      rewriter.create<tensor::DimOp>(loc, indexTy, firstShape, zero);
+      tensor::DimOp::create(rewriter, loc, indexTy, firstShape, zero);
   Value result = nullptr;
   // Generate a linear sequence of compares, all with firstShape as lhs.
   for (Value shape : adaptor.getShapes().drop_front(1)) {
-    Value rank = rewriter.create<tensor::DimOp>(loc, indexTy, shape, zero);
-    Value eqRank = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value rank = tensor::DimOp::create(rewriter, loc, indexTy, shape, zero);
+    Value eqRank = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                   firstRank, rank);
-    auto same = rewriter.create<IfOp>(
+    auto same = IfOp::create(rewriter,
         loc, eqRank,
         [&](OpBuilder &b, Location loc) {
-          Value one = b.create<arith::ConstantIndexOp>(loc, 1);
+          Value one = arith::ConstantIndexOp::create(b, loc, 1);
           Value init =
-              b.create<arith::ConstantOp>(loc, i1Ty, b.getBoolAttr(true));
-          auto loop = b.create<scf::ForOp>(
+              arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(true));
+          auto loop = scf::ForOp::create(b,
               loc, zero, firstRank, one, ValueRange{init},
               [&](OpBuilder &b, Location nestedLoc, Value iv, ValueRange args) {
                 Value conj = args[0];
                 Value lhsExtent =
-                    b.create<tensor::ExtractOp>(loc, firstShape, iv);
-                Value rhsExtent = b.create<tensor::ExtractOp>(loc, shape, iv);
-                Value eqExtent = b.create<arith::CmpIOp>(
+                    tensor::ExtractOp::create(b, loc, firstShape, iv);
+                Value rhsExtent = tensor::ExtractOp::create(b, loc, shape, iv);
+                Value eqExtent = arith::CmpIOp::create(b,
                     loc, arith::CmpIPredicate::eq, lhsExtent, rhsExtent);
-                Value conjNext = b.create<arith::AndIOp>(loc, conj, eqExtent);
-                b.create<scf::YieldOp>(loc, ValueRange({conjNext}));
+                Value conjNext = arith::AndIOp::create(b, loc, conj, eqExtent);
+                scf::YieldOp::create(b, loc, ValueRange({conjNext}));
               });
-          b.create<scf::YieldOp>(loc, loop.getResults());
+          scf::YieldOp::create(b, loc, loop.getResults());
         },
         [&](OpBuilder &b, Location loc) {
           Value result =
-              b.create<arith::ConstantOp>(loc, i1Ty, b.getBoolAttr(false));
-          b.create<scf::YieldOp>(loc, result);
+              arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(false));
+          scf::YieldOp::create(b, loc, result);
         });
     result = !result ? same.getResult(0)
-                     : rewriter.create<arith::AndIOp>(loc, result,
+                     : arith::AndIOp::create(rewriter, loc, result,
                                                       same.getResult(0));
   }
   rewriter.replaceOp(op, result);
@@ -581,17 +581,17 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
     int64_t rank = rankedTensorTy.getRank();
     for (int64_t i = 0; i < rank; i++) {
       if (rankedTensorTy.isDynamicDim(i)) {
-        Value extent = rewriter.create<tensor::DimOp>(loc, tensor, i);
+        Value extent = tensor::DimOp::create(rewriter, loc, tensor, i);
         extentValues.push_back(extent);
       } else {
-        Value extent = rewriter.create<arith::ConstantIndexOp>(
+        Value extent = arith::ConstantIndexOp::create(rewriter,
             loc, rankedTensorTy.getDimSize(i));
         extentValues.push_back(extent);
       }
     }
 
     // Materialize extent tensor.
-    Value staticExtentTensor = rewriter.create<tensor::FromElementsOp>(
+    Value staticExtentTensor = tensor::FromElementsOp::create(rewriter,
         loc, RankedTensorType::get({rank}, rewriter.getIndexType()),
         extentValues);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(),
@@ -601,13 +601,13 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
 
   // Lower to `tensor.generate` otherwise.
   auto *ctx = rewriter.getContext();
-  Value rank = rewriter.create<tensor::RankOp>(loc, tensor);
+  Value rank = tensor::RankOp::create(rewriter, loc, tensor);
   rewriter.replaceOpWithNewOp<tensor::GenerateOp>(
       op, getExtentTensorType(ctx), ValueRange{rank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value dim = args.front();
-        Value extent = b.create<tensor::DimOp>(loc, tensor, dim);
-        b.create<tensor::YieldOp>(loc, extent);
+        Value extent = tensor::DimOp::create(b, loc, tensor, dim);
+        tensor::YieldOp::create(b, loc, extent);
       });
 
   return success();
@@ -634,21 +634,21 @@ LogicalResult SplitAtOpConversion::matchAndRewrite(
     return failure();
 
   ImplicitLocOpBuilder b(op.getLoc(), rewriter);
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value rank = b.create<tensor::DimOp>(adaptor.getOperand(), zero);
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value rank = tensor::DimOp::create(b, adaptor.getOperand(), zero);
 
   // index < 0 ? index + rank : index
   Value originalIndex = adaptor.getIndex();
-  Value add = b.create<arith::AddIOp>(originalIndex, rank);
+  Value add = arith::AddIOp::create(b, originalIndex, rank);
   Value indexIsNegative =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::slt, originalIndex, zero);
-  Value index = b.create<arith::SelectOp>(indexIsNegative, add, originalIndex);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, originalIndex, zero);
+  Value index = arith::SelectOp::create(b, indexIsNegative, add, originalIndex);
 
-  Value one = b.create<arith::ConstantIndexOp>(1);
+  Value one = arith::ConstantIndexOp::create(b, 1);
   Value head =
-      b.create<tensor::ExtractSliceOp>(adaptor.getOperand(), zero, index, one);
-  Value tailSize = b.create<arith::SubIOp>(rank, index);
-  Value tail = b.create<tensor::ExtractSliceOp>(adaptor.getOperand(), index,
+      tensor::ExtractSliceOp::create(b, adaptor.getOperand(), zero, index, one);
+  Value tailSize = arith::SubIOp::create(b, rank, index);
+  Value tail = tensor::ExtractSliceOp::create(b, adaptor.getOperand(), index,
                                                 tailSize, one);
   rewriter.replaceOp(op, {head, tail});
   return success();
diff --git a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
index 2c4d27502a521..f06c3e1f7cb18 100644
--- a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
+++ b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
@@ -68,10 +68,10 @@ class TensorExtractPattern final
       // We could use the initializer directly; but certain driver compilers
       // have bugs dealing with that. So for now, use spirv.Store for
       // initialization.
-      varOp = rewriter.create<spirv::VariableOp>(loc, varType,
+      varOp = spirv::VariableOp::create(rewriter, loc, varType,
                                                  spirv::StorageClass::Function,
                                                  /*initializer=*/nullptr);
-      rewriter.create<spirv::StoreOp>(loc, varOp, adaptor.getTensor());
+      spirv::StoreOp::create(rewriter, loc, varOp, adaptor.getTensor());
     } else {
       // Need to store the value to the local variable. It's questionable
       // whether we want to support such case though.
@@ -83,7 +83,7 @@ class TensorExtractPattern final
 
     Value index = spirv::linearizeIndex(adaptor.getIndices(), strides,
                                         /*offset=*/0, indexType, loc, rewriter);
-    auto acOp = rewriter.create<spirv::AccessChainOp>(loc, varOp, index);
+    auto acOp = spirv::AccessChainOp::create(rewriter, loc, varOp, index);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(extractOp, acOp);
 
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 40ad63610e23f..44872c8d7540d 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -51,7 +51,7 @@ TypedAttr getConstantAttr(Type type, int64_t value, PatternRewriter &rewriter) {
 
 Value getConstantValue(Location loc, Type type, int64_t value,
                        PatternRewriter &rewriter) {
-  return rewriter.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(rewriter,
       loc, getConstantAttr(type, value, rewriter));
 }
 
@@ -82,41 +82,41 @@ class ApplyScaleGenericOpConverter
     Value one64 = getConstantValue(loc, i64Ty, 1, rewriter);
     Value thirtyOne32 = getConstantValue(loc, i32Ty, 31, rewriter);
 
-    Value shift32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, op.getShift());
+    Value shift32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, op.getShift());
 
     // Compute the multiplication in 64-bits then select the high / low parts.
     Value value64 = value;
     if (getElementTypeOrSelf(valueTy) != rewriter.getI64Type())
-      value64 = rewriter.create<arith::ExtSIOp>(loc, i64Ty, value);
+      value64 = arith::ExtSIOp::create(rewriter, loc, i64Ty, value);
     Value multiplier64 =
-        rewriter.create<arith::ExtSIOp>(loc, i64Ty, multiplier32);
+        arith::ExtSIOp::create(rewriter, loc, i64Ty, multiplier32);
     Value multiply64 =
-        rewriter.create<arith::MulIOp>(loc, value64, multiplier64);
+        arith::MulIOp::create(rewriter, loc, value64, multiplier64);
 
     // Apply normal rounding.
-    Value shift64 = rewriter.create<arith::ExtUIOp>(loc, i64Ty, shift32);
-    Value round = rewriter.create<arith::ShLIOp>(loc, one64, shift64);
-    round = rewriter.create<arith::ShRUIOp>(loc, round, one64);
-    multiply64 = rewriter.create<arith::AddIOp>(loc, multiply64, round);
+    Value shift64 = arith::ExtUIOp::create(rewriter, loc, i64Ty, shift32);
+    Value round = arith::ShLIOp::create(rewriter, loc, one64, shift64);
+    round = arith::ShRUIOp::create(rewriter, loc, round, one64);
+    multiply64 = arith::AddIOp::create(rewriter, loc, multiply64, round);
 
     // Apply double rounding if necessary.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       int64_t roundInt = 1 << 30;
       Value roundUp = getConstantValue(loc, i64Ty, roundInt, rewriter);
       Value roundDown = getConstantValue(loc, i64Ty, -roundInt, rewriter);
-      Value positive = rewriter.create<arith::CmpIOp>(
+      Value positive = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, value, zero);
       Value dir =
-          rewriter.create<arith::SelectOp>(loc, positive, roundUp, roundDown);
-      Value val = rewriter.create<arith::AddIOp>(loc, dir, multiply64);
-      Value valid = rewriter.create<arith::CmpIOp>(
+          arith::SelectOp::create(rewriter, loc, positive, roundUp, roundDown);
+      Value val = arith::AddIOp::create(rewriter, loc, dir, multiply64);
+      Value valid = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sgt, shift32, thirtyOne32);
       multiply64 =
-          rewriter.create<arith::SelectOp>(loc, valid, val, multiply64);
+          arith::SelectOp::create(rewriter, loc, valid, val, multiply64);
     }
 
-    Value result64 = rewriter.create<arith::ShRSIOp>(loc, multiply64, shift64);
-    Value result32 = rewriter.create<arith::TruncIOp>(loc, i32Ty, result64);
+    Value result64 = arith::ShRSIOp::create(rewriter, loc, multiply64, shift64);
+    Value result32 = arith::TruncIOp::create(rewriter, loc, i32Ty, result64);
 
     rewriter.replaceOp(op, result32);
     return success();
@@ -146,7 +146,7 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
 
     Value value32 = op.getValue();
     Value multiplier32 = op.getMultiplier();
-    Value shift32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, op.getShift());
+    Value shift32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, op.getShift());
 
     // Constants used during the scaling operation.
     Value zero32 = getConstantValue(loc, i32Ty, 0, rewriter);
@@ -158,86 +158,86 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
     // Compute the multiplication in 64-bits then select the high / low parts.
     // Grab out the high/low of the computation
     auto value64 =
-        rewriter.create<arith::MulSIExtendedOp>(loc, value32, multiplier32);
+        arith::MulSIExtendedOp::create(rewriter, loc, value32, multiplier32);
     Value low32 = value64.getLow();
     Value high32 = value64.getHigh();
 
     // Determine the direction and amount to shift the high bits.
-    Value shiftOver32 = rewriter.create<arith::CmpIOp>(
+    Value shiftOver32 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, shift32, thirtyTwo32);
-    Value roundHighBits = rewriter.create<arith::CmpIOp>(
+    Value roundHighBits = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, shift32, thirtyTwo32);
 
     Value shiftHighL =
-        rewriter.create<arith::SubIOp>(loc, thirtyTwo32, shift32);
+        arith::SubIOp::create(rewriter, loc, thirtyTwo32, shift32);
     Value shiftHighR =
-        rewriter.create<arith::SubIOp>(loc, shift32, thirtyTwo32);
+        arith::SubIOp::create(rewriter, loc, shift32, thirtyTwo32);
 
     shiftHighL =
-        rewriter.create<arith::SelectOp>(loc, shiftOver32, zero32, shiftHighL);
+        arith::SelectOp::create(rewriter, loc, shiftOver32, zero32, shiftHighL);
     shiftHighR =
-        rewriter.create<arith::SelectOp>(loc, shiftOver32, shiftHighR, zero32);
+        arith::SelectOp::create(rewriter, loc, shiftOver32, shiftHighR, zero32);
 
     // Conditionally perform our double round.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       Value negOne32 = getConstantValue(loc, i32Ty, -1, rewriter);
-      Value valuePositive = rewriter.create<arith::CmpIOp>(
+      Value valuePositive = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, value32, zero32);
 
       Value roundDir =
-          rewriter.create<arith::SelectOp>(loc, valuePositive, one32, negOne32);
+          arith::SelectOp::create(rewriter, loc, valuePositive, one32, negOne32);
       roundDir =
-          rewriter.create<arith::SelectOp>(loc, shiftOver32, roundDir, zero32);
+          arith::SelectOp::create(rewriter, loc, shiftOver32, roundDir, zero32);
 
-      Value shiftLow = rewriter.create<arith::ShRUIOp>(loc, low32, thirty32);
-      Value rounded = rewriter.create<arith::AddIOp>(loc, shiftLow, roundDir);
-      Value carry = rewriter.create<arith::ShRSIOp>(loc, rounded, two32);
+      Value shiftLow = arith::ShRUIOp::create(rewriter, loc, low32, thirty32);
+      Value rounded = arith::AddIOp::create(rewriter, loc, shiftLow, roundDir);
+      Value carry = arith::ShRSIOp::create(rewriter, loc, rounded, two32);
 
       Value shiftRound =
-          rewriter.create<arith::ShLIOp>(loc, roundDir, thirty32);
+          arith::ShLIOp::create(rewriter, loc, roundDir, thirty32);
 
-      low32 = rewriter.create<arith::AddIOp>(loc, low32, shiftRound);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, carry);
+      low32 = arith::AddIOp::create(rewriter, loc, low32, shiftRound);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, carry);
     }
 
     // Conditionally apply rounding in the low bits.
     {
-      Value shiftSubOne = rewriter.create<arith::SubIOp>(loc, shift32, one32);
-      Value roundBit = rewriter.create<arith::ShLIOp>(loc, one32, shiftSubOne);
-      roundBit = rewriter.create<arith::SelectOp>(loc, roundHighBits, zero32,
+      Value shiftSubOne = arith::SubIOp::create(rewriter, loc, shift32, one32);
+      Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
+      roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, zero32,
                                                   roundBit);
 
-      Value newLow32 = rewriter.create<arith::AddIOp>(loc, low32, roundBit);
-      Value wasRounded = rewriter.create<arith::CmpIOp>(
+      Value newLow32 = arith::AddIOp::create(rewriter, loc, low32, roundBit);
+      Value wasRounded = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ugt, low32, newLow32);
       low32 = newLow32;
 
-      Value rounded32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, wasRounded);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, rounded32);
+      Value rounded32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, wasRounded);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, rounded32);
     }
 
     // Conditionally apply rounding in the high bits.
     {
       Value shiftSubOne =
-          rewriter.create<arith::SubIOp>(loc, shiftHighR, one32);
-      Value roundBit = rewriter.create<arith::ShLIOp>(loc, one32, shiftSubOne);
-      roundBit = rewriter.create<arith::SelectOp>(loc, roundHighBits, roundBit,
+          arith::SubIOp::create(rewriter, loc, shiftHighR, one32);
+      Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
+      roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, roundBit,
                                                   zero32);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, roundBit);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, roundBit);
     }
 
     // Combine the correct high/low bits into the final rescale result.
-    high32 = rewriter.create<arith::ShLIOp>(loc, high32, shiftHighL);
-    high32 = rewriter.create<arith::ShRSIOp>(loc, high32, shiftHighR);
-    low32 = rewriter.create<arith::ShRUIOp>(loc, low32, shift32);
-    low32 = rewriter.create<arith::SelectOp>(loc, shiftOver32, zero32, low32);
+    high32 = arith::ShLIOp::create(rewriter, loc, high32, shiftHighL);
+    high32 = arith::ShRSIOp::create(rewriter, loc, high32, shiftHighR);
+    low32 = arith::ShRUIOp::create(rewriter, loc, low32, shift32);
+    low32 = arith::SelectOp::create(rewriter, loc, shiftOver32, zero32, low32);
 
     // Apply the rounding behavior and shift to the final alignment.
-    Value result = rewriter.create<arith::AddIOp>(loc, low32, high32);
+    Value result = arith::AddIOp::create(rewriter, loc, low32, high32);
 
     // Truncate if necessary.
     if (!getElementTypeOrSelf(resultTy).isInteger(32)) {
-      result = rewriter.create<arith::TruncIOp>(loc, resultTy, result);
+      result = arith::TruncIOp::create(rewriter, loc, resultTy, result);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 2f608bbd637b4..2aec35ed9d8ca 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -70,13 +70,13 @@ materializeBinaryNanCheckIfRequired(OpTy op, PatternRewriter &rewriter,
     return result;
 
   // Unordered comparison of NaN against itself will always return true.
-  Value lhsIsNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsIsNaN = arith::CmpFOp::create(rewriter,
       op.getLoc(), arith::CmpFPredicate::UNO, lhs, lhs);
-  Value rhsIsNaN = rewriter.create<arith::CmpFOp>(
+  Value rhsIsNaN = arith::CmpFOp::create(rewriter,
       op.getLoc(), arith::CmpFPredicate::UNO, rhs, rhs);
   Value rhsOrResult =
-      rewriter.create<arith::SelectOp>(op.getLoc(), lhsIsNaN, rhs, result);
-  return rewriter.create<arith::SelectOp>(op.getLoc(), rhsIsNaN, lhs,
+      arith::SelectOp::create(rewriter, op.getLoc(), lhsIsNaN, rhs, result);
+  return arith::SelectOp::create(rewriter, op.getLoc(), rhsIsNaN, lhs,
                                           rhsOrResult);
 }
 
@@ -89,38 +89,38 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
   // tosa::AbsOp
   if (isa<tosa::AbsOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::AbsFOp>(loc, resultTypes, args);
+    return math::AbsFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AbsOp>(op) && isa<IntegerType>(elementTy)) {
-    auto zero = rewriter.create<arith::ConstantOp>(
+    auto zero = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(elementTy));
-    auto neg = rewriter.create<arith::SubIOp>(loc, zero, args[0]);
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], neg);
+    auto neg = arith::SubIOp::create(rewriter, loc, zero, args[0]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], neg);
   }
 
   // tosa::AddOp
   if (isa<tosa::AddOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::AddFOp>(loc, resultTypes, args);
+    return arith::AddFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AddOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::AddIOp>(loc, resultTypes, args);
+    return arith::AddIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::SubOp
   if (isa<tosa::SubOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::SubFOp>(loc, resultTypes, args);
+    return arith::SubFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::SubOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::SubIOp>(loc, resultTypes, args);
+    return arith::SubIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::IntDivOp
   if (isa<tosa::IntDivOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::DivSIOp>(loc, resultTypes, args);
+    return arith::DivSIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ReciprocalOp
   if (isa<tosa::ReciprocalOp>(op) && isa<FloatType>(elementTy)) {
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, FloatAttr::get(elementTy, 1));
-    return rewriter.create<arith::DivFOp>(loc, resultTypes, one, args[0]);
+        arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
+    return arith::DivFOp::create(rewriter, loc, resultTypes, one, args[0]);
   }
 
   // tosa::MulOp
@@ -140,7 +140,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                                           "Cannot have shift value for float");
         return nullptr;
       }
-      return rewriter.create<arith::MulFOp>(loc, resultTypes, args[0], args[1]);
+      return arith::MulFOp::create(rewriter, loc, resultTypes, args[0], args[1]);
     }
 
     if (isa<IntegerType>(elementTy)) {
@@ -149,21 +149,21 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
       if (shift > 0) {
         auto shiftConst =
-            rewriter.create<arith::ConstantIntOp>(loc, shift, /*bitwidth=*/8);
+            arith::ConstantIntOp::create(rewriter, loc, shift, /*bitwidth=*/8);
         if (!a.getType().isInteger(32))
-          a = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), a);
+          a = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), a);
 
         if (!b.getType().isInteger(32))
-          b = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), b);
+          b = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), b);
 
-        auto result = rewriter.create<tosa::ApplyScaleOp>(
+        auto result = tosa::ApplyScaleOp::create(rewriter,
             loc, rewriter.getI32Type(), a, b, shiftConst,
             rewriter.getStringAttr("SINGLE_ROUND"));
 
         if (elementTy.isInteger(32))
           return result;
 
-        return rewriter.create<arith::TruncIOp>(loc, elementTy, result);
+        return arith::TruncIOp::create(rewriter, loc, elementTy, result);
       }
 
       int aWidth = a.getType().getIntOrFloatBitWidth();
@@ -171,11 +171,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       int cWidth = resultTypes[0].getIntOrFloatBitWidth();
 
       if (aWidth < cWidth)
-        a = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], a);
+        a = arith::ExtSIOp::create(rewriter, loc, resultTypes[0], a);
       if (bWidth < cWidth)
-        b = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], b);
+        b = arith::ExtSIOp::create(rewriter, loc, resultTypes[0], b);
 
-      return rewriter.create<arith::MulIOp>(loc, resultTypes, a, b);
+      return arith::MulIOp::create(rewriter, loc, resultTypes, a, b);
     }
   }
 
@@ -201,13 +201,13 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     int64_t outZp = *maybeOutZp;
 
     if (isa<FloatType>(elementTy))
-      return rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
+      return arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
 
     if (isa<IntegerType>(elementTy)) {
       if (!inZp && !outZp) {
-        auto constant = rewriter.create<arith::ConstantOp>(
+        auto constant = arith::ConstantOp::create(rewriter,
             loc, IntegerAttr::get(elementTy, 0));
-        return rewriter.create<arith::SubIOp>(loc, resultTypes, constant,
+        return arith::SubIOp::create(rewriter, loc, resultTypes, constant,
                                               args[0]);
       }
 
@@ -231,60 +231,60 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       }
 
       Type intermediateType = rewriter.getIntegerType(intermediateBitWidth);
-      Value zpAddValue = rewriter.create<arith::ConstantOp>(
+      Value zpAddValue = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
 
       // The negation can be applied by doing:
       //  outputValue = inZp + outZp - inputValue
       auto ext =
-          rewriter.create<arith::ExtSIOp>(loc, intermediateType, args[0]);
-      auto sub = rewriter.create<arith::SubIOp>(loc, zpAddValue, ext);
+          arith::ExtSIOp::create(rewriter, loc, intermediateType, args[0]);
+      auto sub = arith::SubIOp::create(rewriter, loc, zpAddValue, ext);
 
       // Clamp to the negation range.
-      Value min = rewriter.create<arith::ConstantIntOp>(
+      Value min = arith::ConstantIntOp::create(rewriter,
           loc, intermediateType,
           APInt::getSignedMinValue(inputBitWidth).getSExtValue());
-      Value max = rewriter.create<arith::ConstantIntOp>(
+      Value max = arith::ConstantIntOp::create(rewriter,
           loc, intermediateType,
           APInt::getSignedMaxValue(inputBitWidth).getSExtValue());
       auto clamp = clampIntHelper(loc, sub, min, max, rewriter, false);
 
       // Truncate to the final value.
-      return rewriter.create<arith::TruncIOp>(loc, elementTy, clamp);
+      return arith::TruncIOp::create(rewriter, loc, elementTy, clamp);
     }
   }
 
   // tosa::BitwiseAndOp
   if (isa<tosa::BitwiseAndOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::AndIOp>(loc, resultTypes, args);
+    return arith::AndIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::BitwiseOrOp
   if (isa<tosa::BitwiseOrOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::OrIOp>(loc, resultTypes, args);
+    return arith::OrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::BitwiseNotOp
   if (isa<tosa::BitwiseNotOp>(op) && isa<IntegerType>(elementTy)) {
     auto allOnesAttr = rewriter.getIntegerAttr(
         elementTy, APInt::getAllOnes(elementTy.getIntOrFloatBitWidth()));
-    auto allOnes = rewriter.create<arith::ConstantOp>(loc, allOnesAttr);
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args[0], allOnes);
+    auto allOnes = arith::ConstantOp::create(rewriter, loc, allOnesAttr);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], allOnes);
   }
 
   // tosa::BitwiseXOrOp
   if (isa<tosa::BitwiseXorOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalLeftShiftOp
   if (isa<tosa::LogicalLeftShiftOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::ShLIOp>(loc, resultTypes, args);
+    return arith::ShLIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalRightShiftOp
   if (isa<tosa::LogicalRightShiftOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::ShRUIOp>(loc, resultTypes, args);
+    return arith::ShRUIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ArithmeticRightShiftOp
   if (isa<tosa::ArithmeticRightShiftOp>(op) && isa<IntegerType>(elementTy)) {
-    auto result = rewriter.create<arith::ShRSIOp>(loc, resultTypes, args);
+    auto result = arith::ShRSIOp::create(rewriter, loc, resultTypes, args);
     auto round = cast<BoolAttr>(op->getAttr("round")).getValue();
     if (!round) {
       return result;
@@ -292,153 +292,153 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     Type i1Ty = IntegerType::get(rewriter.getContext(), /*width=*/1);
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(elementTy, 1));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 1));
     auto zero =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(elementTy, 0));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 0));
     auto i1one =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(i1Ty, 1));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(i1Ty, 1));
 
     // Checking that input2 != 0
-    auto shiftValueGreaterThanZero = rewriter.create<arith::CmpIOp>(
+    auto shiftValueGreaterThanZero = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, args[1], zero);
 
     // Checking for the last bit of input1 to be 1
     auto subtract =
-        rewriter.create<arith::SubIOp>(loc, resultTypes, args[1], one);
+        arith::SubIOp::create(rewriter, loc, resultTypes, args[1], one);
     auto shifted =
-        rewriter.create<arith::ShRSIOp>(loc, resultTypes, args[0], subtract)
+        arith::ShRSIOp::create(rewriter, loc, resultTypes, args[0], subtract)
             ->getResults();
-    auto truncated = rewriter.create<arith::TruncIOp>(
+    auto truncated = arith::TruncIOp::create(rewriter,
         loc, i1Ty, shifted, ArrayRef<NamedAttribute>());
     auto isInputOdd =
-        rewriter.create<arith::AndIOp>(loc, i1Ty, truncated, i1one);
+        arith::AndIOp::create(rewriter, loc, i1Ty, truncated, i1one);
 
-    auto shouldRound = rewriter.create<arith::AndIOp>(
+    auto shouldRound = arith::AndIOp::create(rewriter,
         loc, i1Ty, shiftValueGreaterThanZero, isInputOdd);
     auto extended =
-        rewriter.create<arith::ExtUIOp>(loc, resultTypes, shouldRound);
-    return rewriter.create<arith::AddIOp>(loc, resultTypes, result, extended);
+        arith::ExtUIOp::create(rewriter, loc, resultTypes, shouldRound);
+    return arith::AddIOp::create(rewriter, loc, resultTypes, result, extended);
   }
 
   // tosa::ClzOp
   if (isa<tosa::ClzOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<math::CountLeadingZerosOp>(loc, elementTy, args[0]);
+    return math::CountLeadingZerosOp::create(rewriter, loc, elementTy, args[0]);
   }
 
   // tosa::LogicalAnd
   if (isa<tosa::LogicalAndOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::AndIOp>(loc, resultTypes, args);
+    return arith::AndIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalNot
   if (isa<tosa::LogicalNotOp>(op) && elementTy.isInteger(1)) {
-    auto one = rewriter.create<arith::ConstantOp>(
+    auto one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(elementTy, 1));
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args[0], one);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], one);
   }
 
   // tosa::LogicalOr
   if (isa<tosa::LogicalOrOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::OrIOp>(loc, resultTypes, args);
+    return arith::OrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalXor
   if (isa<tosa::LogicalXorOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::PowOp
   if (isa<tosa::PowOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::PowFOp>(loc, resultTypes, args);
+    return mlir::math::PowFOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::RsqrtOp
   if (isa<tosa::RsqrtOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::RsqrtOp>(loc, resultTypes, args);
+    return mlir::math::RsqrtOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogOp
   if (isa<tosa::LogOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::LogOp>(loc, resultTypes, args);
+    return mlir::math::LogOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ExpOp
   if (isa<tosa::ExpOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::ExpOp>(loc, resultTypes, args);
+    return mlir::math::ExpOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::SinOp
   if (isa<tosa::SinOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::SinOp>(loc, resultTypes, args);
+    return mlir::math::SinOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::CosOp
   if (isa<tosa::CosOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::CosOp>(loc, resultTypes, args);
+    return mlir::math::CosOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::TanhOp
   if (isa<tosa::TanhOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::TanhOp>(loc, resultTypes, args);
+    return mlir::math::TanhOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ErfOp
   if (isa<tosa::ErfOp>(op) && llvm::isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::ErfOp>(loc, resultTypes, args);
+    return mlir::math::ErfOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::GreaterOp
   if (isa<tosa::GreaterOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGT,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGT,
                                           args[0], args[1]);
 
   if (isa<tosa::GreaterOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sgt,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sgt,
                                           args[0], args[1]);
 
   // tosa::GreaterEqualOp
   if (isa<tosa::GreaterEqualOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGE,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGE,
                                           args[0], args[1]);
 
   if (isa<tosa::GreaterEqualOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sge,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sge,
                                           args[0], args[1]);
 
   // tosa::EqualOp
   if (isa<tosa::EqualOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OEQ,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OEQ,
                                           args[0], args[1]);
 
   if (isa<tosa::EqualOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                           args[0], args[1]);
 
   // tosa::SelectOp
   if (isa<tosa::SelectOp>(op)) {
     elementTy = cast<ShapedType>(op->getOperand(1).getType()).getElementType();
     if (isa<FloatType>(elementTy) || isa<IntegerType>(elementTy))
-      return rewriter.create<arith::SelectOp>(loc, args[0], args[1], args[2]);
+      return arith::SelectOp::create(rewriter, loc, args[0], args[1], args[2]);
   }
 
   // tosa::MaximumOp
   if (isa<tosa::MaximumOp>(op) && isa<FloatType>(elementTy)) {
-    auto max = rewriter.create<arith::MaximumFOp>(loc, args[0], args[1]);
+    auto max = arith::MaximumFOp::create(rewriter, loc, args[0], args[1]);
     return materializeBinaryNanCheckIfRequired(llvm::cast<tosa::MaximumOp>(op),
                                                rewriter, args[0], args[1], max);
   }
 
   if (isa<tosa::MaximumOp>(op) && elementTy.isSignlessInteger()) {
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], args[1]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   // tosa::MinimumOp
   if (isa<tosa::MinimumOp>(op) && isa<FloatType>(elementTy)) {
-    auto min = rewriter.create<arith::MinimumFOp>(loc, args[0], args[1]);
+    auto min = arith::MinimumFOp::create(rewriter, loc, args[0], args[1]);
     return materializeBinaryNanCheckIfRequired(llvm::cast<tosa::MinimumOp>(op),
                                                rewriter, args[0], args[1], min);
   }
 
   if (isa<tosa::MinimumOp>(op) && elementTy.isSignlessInteger()) {
-    return rewriter.create<arith::MinSIOp>(loc, args[0], args[1]);
+    return arith::MinSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   // tosa::CeilOp
   if (isa<tosa::CeilOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::CeilOp>(loc, resultTypes, args);
+    return math::CeilOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::FloorOp
   if (isa<tosa::FloorOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::FloorOp>(loc, resultTypes, args);
+    return math::FloorOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ClampOp
   if (isa<tosa::ClampOp>(op) && isa<FloatType>(elementTy)) {
@@ -449,9 +449,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                    APFloat::rmNearestTiesToEven, &losesInfo);
     maxApf.convert(cast<FloatType>(elementTy).getFloatSemantics(),
                    APFloat::rmNearestTiesToEven, &losesInfo);
-    auto min = rewriter.create<arith::ConstantOp>(
+    auto min = arith::ConstantOp::create(rewriter,
         loc, elementTy, rewriter.getFloatAttr(elementTy, minApf));
-    auto max = rewriter.create<arith::ConstantOp>(
+    auto max = arith::ConstantOp::create(rewriter,
         loc, elementTy, rewriter.getFloatAttr(elementTy, maxApf));
     auto result = clampFloatHelper(loc, args[0], min, max, rewriter);
 
@@ -478,11 +478,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     //   return init if x == NaN else result
 
     // Unordered comparison of NaN against itself will always return true.
-    Value isNaN = rewriter.create<arith::CmpFOp>(
+    Value isNaN = arith::CmpFOp::create(rewriter,
         op->getLoc(), arith::CmpFPredicate::UNO, args[0], args[0]);
     // TOSA specifies that in "ignore" NaN mode the result is "min" if the input
     // is NaN.
-    return rewriter.create<arith::SelectOp>(op->getLoc(), isNaN, min, result);
+    return arith::SelectOp::create(rewriter, op->getLoc(), isNaN, min, result);
   }
 
   if (isa<tosa::ClampOp>(op) && isa<IntegerType>(elementTy)) {
@@ -515,9 +515,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     min = std::min(min, maxRepresentable);
     max = std::min(max, maxRepresentable);
 
-    auto minVal = rewriter.create<arith::ConstantIntOp>(
+    auto minVal = arith::ConstantIntOp::create(rewriter,
         loc, min, intTy.getIntOrFloatBitWidth());
-    auto maxVal = rewriter.create<arith::ConstantIntOp>(
+    auto maxVal = arith::ConstantIntOp::create(rewriter,
         loc, max, intTy.getIntOrFloatBitWidth());
     return clampIntHelper(loc, args[0], minVal, maxVal, rewriter,
                           intTy.isUnsignedInteger());
@@ -526,11 +526,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   // tosa::SigmoidOp
   if (isa<tosa::SigmoidOp>(op) && isa<FloatType>(elementTy)) {
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, FloatAttr::get(elementTy, 1));
-    auto negate = rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
-    auto exp = rewriter.create<mlir::math::ExpOp>(loc, resultTypes, negate);
-    auto added = rewriter.create<arith::AddFOp>(loc, resultTypes, exp, one);
-    return rewriter.create<arith::DivFOp>(loc, resultTypes, one, added);
+        arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
+    auto negate = arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
+    auto exp = mlir::math::ExpOp::create(rewriter, loc, resultTypes, negate);
+    auto added = arith::AddFOp::create(rewriter, loc, resultTypes, exp, one);
+    return arith::DivFOp::create(rewriter, loc, resultTypes, one, added);
   }
 
   // tosa::CastOp
@@ -549,20 +549,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       return args.front();
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtFOp>(loc, resultTypes, args,
+      return arith::ExtFOp::create(rewriter, loc, resultTypes, args,
                                             ArrayRef<NamedAttribute>());
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && !bitExtend)
-      return rewriter.create<arith::TruncFOp>(loc, resultTypes, args,
+      return arith::TruncFOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     // 1-bit integers need to be treated as signless.
     if (srcTy.isInteger(1) && arith::UIToFPOp::areCastCompatible(srcTy, dstTy))
-      return rewriter.create<arith::UIToFPOp>(loc, resultTypes, args,
+      return arith::UIToFPOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     if (srcTy.isInteger(1) && isa<IntegerType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtUIOp>(loc, resultTypes, args,
+      return arith::ExtUIOp::create(rewriter, loc, resultTypes, args,
                                              ArrayRef<NamedAttribute>());
 
     // Unsigned integers need an unrealized cast so that they can be passed
@@ -574,25 +574,25 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                   loc, rewriter.getIntegerType(srcTy.getIntOrFloatBitWidth()),
                   args[0])
               .getResult(0);
-      return rewriter.create<arith::UIToFPOp>(loc, resultTypes[0],
+      return arith::UIToFPOp::create(rewriter, loc, resultTypes[0],
                                               unrealizedCast);
     }
 
     // All other si-to-fp conversions should be handled by SIToFP.
     if (arith::SIToFPOp::areCastCompatible(srcTy, dstTy))
-      return rewriter.create<arith::SIToFPOp>(loc, resultTypes, args,
+      return arith::SIToFPOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     // Casting to boolean, floats need to only be checked as not-equal to zero.
     if (isa<FloatType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = rewriter.create<arith::ConstantOp>(
+      Value zero = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(srcTy, 0.0));
-      return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNE,
+      return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNE,
                                             args.front(), zero);
     }
 
     if (arith::FPToSIOp::areCastCompatible(srcTy, dstTy)) {
-      auto rounded = rewriter.create<math::RoundEvenOp>(loc, args[0]);
+      auto rounded = math::RoundEvenOp::create(rewriter, loc, args[0]);
 
       const auto &fltSemantics = cast<FloatType>(srcTy).getFloatSemantics();
       // Check whether neither int min nor int max can be represented in the
@@ -601,33 +601,33 @@ static Value createLinalgBodyCalculationForElementwiseOp(
           APFloat::semanticsMaxExponent(fltSemantics)) {
         // Use cmp + select to replace infinites by int min / int max. Other
         // integral values can be represented in the integer space.
-        auto conv = rewriter.create<arith::FPToSIOp>(loc, dstTy, rounded);
-        auto posInf = rewriter.create<arith::ConstantOp>(
+        auto conv = arith::FPToSIOp::create(rewriter, loc, dstTy, rounded);
+        auto posInf = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(getElementTypeOrSelf(srcTy),
                                        APFloat::getInf(fltSemantics)));
-        auto negInf = rewriter.create<arith::ConstantOp>(
+        auto negInf = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(
                      getElementTypeOrSelf(srcTy),
                      APFloat::getInf(fltSemantics, /*Negative=*/true)));
-        auto overflow = rewriter.create<arith::CmpFOp>(
+        auto overflow = arith::CmpFOp::create(rewriter,
             loc, arith::CmpFPredicate::UEQ, rounded, posInf);
-        auto underflow = rewriter.create<arith::CmpFOp>(
+        auto underflow = arith::CmpFOp::create(rewriter,
             loc, arith::CmpFPredicate::UEQ, rounded, negInf);
-        auto intMin = rewriter.create<arith::ConstantOp>(
+        auto intMin = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIntegerAttr(
                      getElementTypeOrSelf(dstTy),
                      APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())));
-        auto intMax = rewriter.create<arith::ConstantOp>(
+        auto intMax = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIntegerAttr(
                      getElementTypeOrSelf(dstTy),
                      APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
         auto maxClamped =
-            rewriter.create<arith::SelectOp>(loc, overflow, intMax, conv);
-        return rewriter.create<arith::SelectOp>(loc, underflow, intMin,
+            arith::SelectOp::create(rewriter, loc, overflow, intMax, conv);
+        return arith::SelectOp::create(rewriter, loc, underflow, intMin,
                                                 maxClamped);
       }
 
-      auto intMinFP = rewriter.create<arith::ConstantOp>(
+      auto intMinFP = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
                    APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())
@@ -640,7 +640,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         // consists of a single leading bit. Therefore we can clamp the input
         // in the floating-point domain.
 
-        auto intMaxFP = rewriter.create<arith::ConstantOp>(
+        auto intMaxFP = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(
                      getElementTypeOrSelf(srcTy),
                      APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
@@ -648,14 +648,14 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
         Value clamped =
             clampFloatHelper(loc, rounded, intMinFP, intMaxFP, rewriter);
-        return rewriter.create<arith::FPToSIOp>(loc, dstTy, clamped);
+        return arith::FPToSIOp::create(rewriter, loc, dstTy, clamped);
       }
 
       // Due to earlier check we know exponant range is big enough to represent
       // int min. We can therefore rely on int max + 1 being representable as
       // well because it's just int min with a positive sign. So clamp the min
       // value and compare against that to select the max int value if needed.
-      auto intMaxPlusOneFP = rewriter.create<arith::ConstantOp>(
+      auto intMaxPlusOneFP = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
                    static_cast<double>(
@@ -663,35 +663,35 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                            .getSExtValue()) +
                        1.0f));
 
-      auto intMax = rewriter.create<arith::ConstantOp>(
+      auto intMax = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIntegerAttr(
                    getElementTypeOrSelf(dstTy),
                    APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
       auto minClampedFP =
-          rewriter.create<arith::MaximumFOp>(loc, rounded, intMinFP);
+          arith::MaximumFOp::create(rewriter, loc, rounded, intMinFP);
       auto minClamped =
-          rewriter.create<arith::FPToSIOp>(loc, dstTy, minClampedFP);
-      auto overflow = rewriter.create<arith::CmpFOp>(
+          arith::FPToSIOp::create(rewriter, loc, dstTy, minClampedFP);
+      auto overflow = arith::CmpFOp::create(rewriter,
           loc, arith::CmpFPredicate::UGE, rounded, intMaxPlusOneFP);
-      return rewriter.create<arith::SelectOp>(loc, overflow, intMax,
+      return arith::SelectOp::create(rewriter, loc, overflow, intMax,
                                               minClamped);
     }
 
     // Casting to boolean, integers need to only be checked as not-equal to
     // zero.
     if (isa<IntegerType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = rewriter.create<arith::ConstantIntOp>(
+      Value zero = arith::ConstantIntOp::create(rewriter,
           loc, 0, srcTy.getIntOrFloatBitWidth());
-      return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+      return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
                                             args.front(), zero);
     }
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtSIOp>(loc, resultTypes, args,
+      return arith::ExtSIOp::create(rewriter, loc, resultTypes, args,
                                              ArrayRef<NamedAttribute>());
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && !bitExtend) {
-      return rewriter.create<arith::TruncIOp>(loc, dstTy, args[0]);
+      return arith::TruncIOp::create(rewriter, loc, dstTy, args[0]);
     }
   }
 
@@ -710,14 +710,14 @@ static Value createIndex(PatternRewriter &rewriter, Location loc,
   auto [it, inserted] = indexPool.try_emplace(index);
   if (inserted)
     it->second =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(index));
   return it->second;
 }
 
 static Value getTensorDim(PatternRewriter &rewriter, Location loc,
                           IndexPool &indexPool, Value tensor, int64_t index) {
   auto indexValue = createIndex(rewriter, loc, indexPool, index);
-  return rewriter.create<tensor::DimOp>(loc, tensor, indexValue).getResult();
+  return tensor::DimOp::create(rewriter, loc, tensor, indexValue).getResult();
 }
 
 static OpFoldResult getOrFoldTensorDim(PatternRewriter &rewriter, Location loc,
@@ -783,7 +783,7 @@ computeTargetSize(PatternRewriter &rewriter, Location loc, IndexPool &indexPool,
   for (size_t i = 1; i < operandsWithDynamicDim.size(); i++) {
     auto nextSize =
         getTensorDim(rewriter, loc, indexPool, operandsWithDynamicDim[i], dim);
-    targetSize = rewriter.create<arith::MaxUIOp>(loc, targetSize, nextSize);
+    targetSize = arith::MaxUIOp::create(rewriter, loc, targetSize, nextSize);
   }
   return {targetSize, nullptr};
 }
@@ -838,7 +838,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
   // Check if broadcast is necessary
   auto one = createIndex(rewriter, loc, indexPool, 1);
   auto runtimeSize = getTensorDim(rewriter, loc, indexPool, operand, dim);
-  auto broadcastNecessary = rewriter.create<arith::CmpIOp>(
+  auto broadcastNecessary = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::eq, runtimeSize, one);
 
   // Emit 'then' region of 'scf.if'
@@ -855,7 +855,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                                                     operand, index);
       outputTensorShape.push_back(size);
     }
-    Value outputTensor = opBuilder.create<tensor::EmptyOp>(
+    Value outputTensor = tensor::EmptyOp::create(opBuilder,
         loc, outputTensorShape, rankedTensorType.getElementType());
 
     // Emit 'linalg.generic' op
@@ -866,7 +866,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                 getNParallelLoopsAttrs(rank),
                 [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
                   // Emit 'linalg.yield' op
-                  opBuilder.create<linalg::YieldOp>(loc, blockArgs.front());
+                  linalg::YieldOp::create(opBuilder, loc, blockArgs.front());
                 })
             .getResult(0);
 
@@ -875,16 +875,16 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
         loc, operand.getType(), resultTensor);
 
     // Emit 'scf.yield' op
-    opBuilder.create<scf::YieldOp>(loc, castResultTensor);
+    scf::YieldOp::create(opBuilder, loc, castResultTensor);
   };
 
   // Emit 'else' region of 'scf.if'
   auto emitElseRegion = [&](OpBuilder &opBuilder, Location loc) {
-    opBuilder.create<scf::YieldOp>(loc, operand);
+    scf::YieldOp::create(opBuilder, loc, operand);
   };
 
   // Emit 'scf.if' op
-  auto ifOp = rewriter.create<scf::IfOp>(loc, broadcastNecessary,
+  auto ifOp = scf::IfOp::create(rewriter, loc, broadcastNecessary,
                                          emitThenRegion, emitElseRegion);
   return ifOp.getResult(0);
 }
@@ -930,7 +930,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
   if (!resultType) {
     return rewriter.notifyMatchFailure(operation, "failed to convert type");
   }
-  Value outputTensor = rewriter.create<tensor::EmptyOp>(
+  Value outputTensor = tensor::EmptyOp::create(rewriter,
       loc, targetShape, resultType.getElementType());
 
   // Create affine maps. Input affine maps broadcast static dimensions of size
@@ -957,7 +957,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
 
   // Emit 'linalg.generic' op
   bool encounteredError = false;
-  auto linalgOp = rewriter.create<linalg::GenericOp>(
+  auto linalgOp = linalg::GenericOp::create(rewriter,
       loc, outputTensor.getType(), operands, outputTensor, affineMaps,
       getNParallelLoopsAttrs(rank),
       [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
@@ -968,7 +968,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
           encounteredError = true;
           return;
         }
-        opBuilder.create<linalg::YieldOp>(loc, opResult);
+        linalg::YieldOp::create(opBuilder, loc, opResult);
       });
   if (encounteredError)
     return rewriter.notifyMatchFailure(
@@ -1078,42 +1078,42 @@ static Value createLinalgBodyCalculationForReduceOp(Operation *op,
                                                     PatternRewriter &rewriter) {
   Location loc = op->getLoc();
   if (isa<tosa::ReduceSumOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::AddFOp>(loc, args);
+    return arith::AddFOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceSumOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::AddIOp>(loc, args);
+    return arith::AddIOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceProductOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MulFOp>(loc, args);
+    return arith::MulFOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceProductOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MulIOp>(loc, args);
+    return arith::MulIOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceMinOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MinimumFOp>(loc, args[0], args[1]);
+    return arith::MinimumFOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMinOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MinSIOp>(loc, args[0], args[1]);
+    return arith::MinSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMaxOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MaximumFOp>(loc, args[0], args[1]);
+    return arith::MaximumFOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMaxOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], args[1]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceAllOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::AndIOp>(loc, args);
+    return arith::AndIOp::create(rewriter, loc, args);
 
   if (isa<tosa::ReduceAnyOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::OrIOp>(loc, args);
+    return arith::OrIOp::create(rewriter, loc, args);
 
   return {};
 }
@@ -1139,7 +1139,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     if (axis != i) {
       reduceShape.push_back(inputTy.getDimSize(i));
       if (inputTy.isDynamicDim(i))
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
     }
   }
 
@@ -1158,7 +1158,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     return rewriter.notifyMatchFailure(
         op, "No initial value found for reduction operation");
 
-  auto fillValue = rewriter.create<arith::ConstantOp>(loc, fillValueAttr);
+  auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
   auto filledTensor = rewriter
                           .create<linalg::FillOp>(loc, ValueRange{fillValue},
                                                   ValueRange{emptyTensor})
@@ -1176,7 +1176,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
       // Additionally we have to keep track of whether we've seen any non-NaN
       // values and then do a final select based on this predicate.
       auto trueAttr = rewriter.getBoolAttr(true);
-      auto trueValue = rewriter.create<arith::ConstantOp>(loc, trueAttr);
+      auto trueValue = arith::ConstantOp::create(rewriter, loc, trueAttr);
       auto emptyBoolTensor =
           rewriter
               .create<tensor::EmptyOp>(loc, reduceShape, trueValue.getType(),
@@ -1202,7 +1202,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
   }
 
   bool didEncounterError = false;
-  linalg::LinalgOp linalgOp = rewriter.create<linalg::ReduceOp>(
+  linalg::LinalgOp linalgOp = linalg::ReduceOp::create(rewriter,
       loc, inputs, outputs, axis,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange blockArgs) {
         std::array<Value, 2> binaryArgs{
@@ -1219,21 +1219,21 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
           auto oldAllResultsNanFlagValue = blockArgs[3];
 
           // Unordered comparison of NaN against itself will always return true.
-          Value isNaN = nestedBuilder.create<arith::CmpFOp>(
+          Value isNaN = arith::CmpFOp::create(nestedBuilder,
               op->getLoc(), arith::CmpFPredicate::UNO, inputValue, inputValue);
           // If we've encountered a NaN, take the non-NaN value.
-          auto selectOp = nestedBuilder.create<arith::SelectOp>(
+          auto selectOp = arith::SelectOp::create(nestedBuilder,
               op->getLoc(), isNaN, initialValue, result);
           // Update the flag which keeps track of whether we have seen a non-NaN
           // value.
-          auto newAllResultsNanFlagValue = nestedBuilder.create<arith::AndIOp>(
+          auto newAllResultsNanFlagValue = arith::AndIOp::create(nestedBuilder,
               op->getLoc(), oldAllResultsNanFlagValue, isNaN);
           resultsToYield.push_back(selectOp);
           resultsToYield.push_back(newAllResultsNanFlagValue);
         } else {
           resultsToYield.push_back(result);
         }
-        nestedBuilder.create<linalg::YieldOp>(loc, resultsToYield);
+        linalg::YieldOp::create(nestedBuilder, loc, resultsToYield);
       });
 
   if (!didEncounterError)
@@ -1250,7 +1250,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     auto nanValueAttr = rewriter.getFloatAttr(
         elementTy,
         APFloat::getNaN(cast<FloatType>(elementTy).getFloatSemantics(), false));
-    auto nanValue = rewriter.create<arith::ConstantOp>(loc, nanValueAttr);
+    auto nanValue = arith::ConstantOp::create(rewriter, loc, nanValueAttr);
     auto emptyNanTensor =
         rewriter
             .create<tensor::EmptyOp>(loc, reduceShape,
@@ -1278,7 +1278,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     ins.push_back(linalgOp->getResult(0));
     outs.push_back(finalEmptyTensor);
     auto linalgSelect =
-        rewriter.create<linalg::SelectOp>(op->getLoc(), ins, outs);
+        linalg::SelectOp::create(rewriter, op->getLoc(), ins, outs);
     linalgOp = linalgSelect;
   }
 
@@ -1350,7 +1350,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < outputTy.getRank(); i++) {
       if (outputTy.isDynamicDim(i)) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
@@ -1401,7 +1401,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value multiplierConstant;
     int64_t multiplierArg = 0;
     if (multiplierValues.size() == 1) {
-      multiplierConstant = rewriter.create<arith::ConstantOp>(
+      multiplierConstant = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32IntegerAttr(multiplierValues.front()));
     } else {
       SmallVector<AffineExpr, 2> multiplierExprs{
@@ -1409,7 +1409,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
       auto multiplierType =
           RankedTensorType::get({static_cast<int64_t>(multiplierValues.size())},
                                 rewriter.getI32Type());
-      genericInputs.push_back(rewriter.create<arith::ConstantOp>(
+      genericInputs.push_back(arith::ConstantOp::create(rewriter,
           loc, DenseIntElementsAttr::get(multiplierType, multiplierValues)));
 
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
@@ -1424,7 +1424,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value shiftConstant;
     int64_t shiftArg = 0;
     if (shiftValues.size() == 1) {
-      shiftConstant = rewriter.create<arith::ConstantOp>(
+      shiftConstant = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI8IntegerAttr(shiftValues.front()));
     } else {
       SmallVector<AffineExpr, 2> shiftExprs = {
@@ -1432,7 +1432,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
       auto shiftType =
           RankedTensorType::get({static_cast<int64_t>(shiftValues.size())},
                                 rewriter.getIntegerType(8));
-      genericInputs.push_back(rewriter.create<arith::ConstantOp>(
+      genericInputs.push_back(arith::ConstantOp::create(rewriter,
           loc, DenseIntElementsAttr::get(shiftType, shiftValues)));
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
                                             /*symbolCount=*/0, shiftExprs,
@@ -1444,11 +1444,11 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(rank));
 
     // Construct the indexing maps needed for linalg.generic ops.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, outputTy.getShape(), outputTy.getElementType(),
         ArrayRef<Value>({dynDims}));
 
-    auto linalgOp = rewriter.create<linalg::GenericOp>(
+    auto linalgOp = linalg::GenericOp::create(rewriter,
         loc, outputTy, genericInputs, ValueRange{emptyTensor}, indexingMaps,
         getNParallelLoopsAttrs(rank),
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
@@ -1466,7 +1466,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           const int32_t inBitwidth = valueTy.getIntOrFloatBitWidth();
           // Extend zeropoint for sub-32bits widths.
           const int32_t inAttrBitwidth = inBitwidth > 32 ? inBitwidth : 32;
-          auto inputZp = nestedBuilder.create<arith::ConstantOp>(
+          auto inputZp = arith::ConstantOp::create(nestedBuilder,
               loc, IntegerAttr::get(rewriter.getIntegerType(inAttrBitwidth),
                                     *maybeIZp));
 
@@ -1482,7 +1482,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           unsigned outBitWidth = outIntType.getWidth();
           const int32_t outAttrBitwidth = 32;
           assert(outBitWidth <= 32 && "Unexpected output zeropoint bitwidth");
-          auto outputZp = nestedBuilder.create<arith::ConstantOp>(
+          auto outputZp = arith::ConstantOp::create(nestedBuilder,
               loc, IntegerAttr::get(rewriter.getIntegerType(outAttrBitwidth),
                                     *maybeOZp));
 
@@ -1501,24 +1501,24 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           }
           if (valueTy.getIntOrFloatBitWidth() < 32) {
             if (op.getInputUnsigned()) {
-              value = nestedBuilder.create<arith::ExtUIOp>(
+              value = arith::ExtUIOp::create(nestedBuilder,
                   nestedLoc, nestedBuilder.getI32Type(), value);
             } else {
-              value = nestedBuilder.create<arith::ExtSIOp>(
+              value = arith::ExtSIOp::create(nestedBuilder,
                   nestedLoc, nestedBuilder.getI32Type(), value);
             }
           }
 
           value =
-              nestedBuilder.create<arith::SubIOp>(nestedLoc, value, inputZp);
+              arith::SubIOp::create(nestedBuilder, nestedLoc, value, inputZp);
 
-          value = nestedBuilder.create<tosa::ApplyScaleOp>(
+          value = tosa::ApplyScaleOp::create(nestedBuilder,
               loc, nestedBuilder.getI32Type(), value, multiplier, shift,
               roundingMode);
 
           // Move to the new zero-point.
           value =
-              nestedBuilder.create<arith::AddIOp>(nestedLoc, value, outputZp);
+              arith::AddIOp::create(nestedBuilder, nestedLoc, value, outputZp);
 
           // Saturate to the output size.
           int32_t intMin = APInt::getSignedMinValue(outBitWidth).getSExtValue();
@@ -1530,16 +1530,16 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
             intMax = APInt::getMaxValue(outBitWidth).getZExtValue();
           }
 
-          auto intMinVal = nestedBuilder.create<arith::ConstantOp>(
+          auto intMinVal = arith::ConstantOp::create(nestedBuilder,
               loc, nestedBuilder.getI32IntegerAttr(intMin));
-          auto intMaxVal = nestedBuilder.create<arith::ConstantOp>(
+          auto intMaxVal = arith::ConstantOp::create(nestedBuilder,
               loc, nestedBuilder.getI32IntegerAttr(intMax));
 
           value = clampIntHelper(nestedLoc, value, intMinVal, intMaxVal,
                                  nestedBuilder, /*isUnsigned=*/false);
 
           if (outIntType.getWidth() < 32) {
-            value = nestedBuilder.create<arith::TruncIOp>(
+            value = arith::TruncIOp::create(nestedBuilder,
                 nestedLoc, rewriter.getIntegerType(outIntType.getWidth()),
                 value);
           }
@@ -1550,7 +1550,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
                                                             outIntType, value)
                         .getResult(0);
           }
-          nestedBuilder.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(nestedBuilder, loc, value);
         });
 
     rewriter.replaceOp(op, linalgOp->getResults());
@@ -1608,25 +1608,25 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
     auto collapseTy =
         RankedTensorType::get({inputTy.getDimSize(0), inputTy.getDimSize(3)},
                               inputTy.getElementType());
-    Value collapse = builder.create<tensor::CollapseShapeOp>(collapseTy, input,
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, input,
                                                              reassociationMap);
 
     // Get any dynamic shapes that appear in the input format.
     llvm::SmallVector<Value> outputDynSize;
     if (inputTy.isDynamicDim(0))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 0));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 0));
     if (inputTy.isDynamicDim(3))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 3));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 3));
 
     // Generate the elementwise operation for casting scaling the input value.
     auto genericTy = collapseTy.clone(resultTy.getElementType());
-    Value empty = builder.create<tensor::EmptyOp>(
+    Value empty = tensor::EmptyOp::create(builder,
         genericTy.getShape(), resultTy.getElementType(), outputDynSize);
     auto genericMap = rewriter.getMultiDimIdentityMap(genericTy.getRank());
     SmallVector<utils::IteratorType> iterators(genericTy.getRank(),
                                                utils::IteratorType::parallel);
 
-    auto generic = builder.create<linalg::GenericOp>(
+    auto generic = linalg::GenericOp::create(builder,
         genericTy, ValueRange{collapse}, ValueRange{empty},
         ArrayRef<AffineMap>{genericMap, genericMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
@@ -1634,22 +1634,22 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
           // This is the quantized case.
           if (inputTy.getElementType() != resultTy.getElementType()) {
             value =
-                b.create<arith::ExtSIOp>(loc, resultTy.getElementType(), value);
+                arith::ExtSIOp::create(b, loc, resultTy.getElementType(), value);
 
             if (isBilinear && scale[0] != 0) {
-              Value scaleY = b.create<arith::ConstantOp>(
+              Value scaleY = arith::ConstantOp::create(b,
                   loc, b.getI32IntegerAttr(scale[0]));
-              value = b.create<arith::MulIOp>(loc, value, scaleY);
+              value = arith::MulIOp::create(b, loc, value, scaleY);
             }
 
             if (isBilinear && scale[2] != 0) {
-              Value scaleX = b.create<arith::ConstantOp>(
+              Value scaleX = arith::ConstantOp::create(b,
                   loc, b.getI32IntegerAttr(scale[2]));
-              value = b.create<arith::MulIOp>(loc, value, scaleX);
+              value = arith::MulIOp::create(b, loc, value, scaleX);
             }
           }
 
-          b.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(b, loc, value);
         });
 
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -1697,7 +1697,7 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     resizeShape.push_back(channels);
 
     auto resizeTy = resultTy.clone(resizeShape);
-    auto resize = builder.create<tosa::ResizeOp>(resizeTy, input, op.getScale(),
+    auto resize = tosa::ResizeOp::create(builder, resizeTy, input, op.getScale(),
                                                  op.getOffset(), op.getBorder(),
                                                  op.getMode());
 
@@ -1720,19 +1720,19 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     collapseShape.push_back(channels);
 
     auto collapseTy = resultTy.clone(collapseShape);
-    Value collapse = builder.create<tensor::CollapseShapeOp>(collapseTy, resize,
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, resize,
                                                              reassociationMap);
 
     // Broadcast the collapsed shape to the output result.
     llvm::SmallVector<Value> outputDynSize;
     if (inputTy.isDynamicDim(0))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 0));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 0));
     if (inputTy.isDynamicDim(3))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 3));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 3));
 
     SmallVector<utils::IteratorType> iterators(resultTy.getRank(),
                                                utils::IteratorType::parallel);
-    Value empty = builder.create<tensor::EmptyOp>(
+    Value empty = tensor::EmptyOp::create(builder,
         resultTy.getShape(), resultTy.getElementType(), outputDynSize);
 
     SmallVector<AffineExpr, 4> inputExprs{rewriter.getAffineDimExpr(0)};
@@ -1751,7 +1751,7 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
         ArrayRef<AffineMap>{inputMap, outputMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
           Value value = args[0];
-          b.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(b, loc, value);
         });
 
     return success();
@@ -1789,9 +1789,9 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
 
     SmallVector<AffineMap, 2> affineMaps = {
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
-    auto emptyTensor = b.create<tensor::EmptyOp>(resultTy.getShape(), resultETy,
+    auto emptyTensor = tensor::EmptyOp::create(b, resultTy.getShape(), resultETy,
                                                  *dynamicDimsOr);
-    auto genericOp = b.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(b,
         resultTy, ValueRange({}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     Value resize = genericOp.getResult(0);
@@ -1800,19 +1800,19 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       OpBuilder::InsertionGuard regionGuard(b);
       b.createBlock(&genericOp.getRegion(), genericOp.getRegion().end(),
                     TypeRange({resultETy}), loc);
-      Value batch = b.create<linalg::IndexOp>(0);
-      Value y = b.create<linalg::IndexOp>(1);
-      Value x = b.create<linalg::IndexOp>(2);
-      Value channel = b.create<linalg::IndexOp>(3);
+      Value batch = linalg::IndexOp::create(b, 0);
+      Value y = linalg::IndexOp::create(b, 1);
+      Value x = linalg::IndexOp::create(b, 2);
+      Value channel = linalg::IndexOp::create(b, 3);
 
       Value zeroI32 =
-          b.create<arith::ConstantOp>(b.getZeroAttr(b.getI32Type()));
-      Value zeroFp = b.create<arith::ConstantOp>(b.getZeroAttr(floatTy));
-      Value hMax = b.create<arith::ConstantOp>(b.getI32IntegerAttr(imageH - 1));
-      Value wMax = b.create<arith::ConstantOp>(b.getI32IntegerAttr(imageW - 1));
+          arith::ConstantOp::create(b, b.getZeroAttr(b.getI32Type()));
+      Value zeroFp = arith::ConstantOp::create(b, b.getZeroAttr(floatTy));
+      Value hMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageH - 1));
+      Value wMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageW - 1));
 
-      Value inY = b.create<arith::IndexCastOp>(b.getI32Type(), y);
-      Value inX = b.create<arith::IndexCastOp>(b.getI32Type(), x);
+      Value inY = arith::IndexCastOp::create(b, b.getI32Type(), y);
+      Value inX = arith::IndexCastOp::create(b, b.getI32Type(), x);
 
       SmallVector<int64_t> scale, offset, border;
       if (!tosa::getConstShapeValues(op.getScale().getDefiningOp(), scale) ||
@@ -1824,16 +1824,16 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       }
 
       Value yScaleN, yScaleD, xScaleN, xScaleD;
-      yScaleN = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[0]));
-      yScaleD = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[1]));
-      xScaleN = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[2]));
-      xScaleD = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[3]));
+      yScaleN = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[0]));
+      yScaleD = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[1]));
+      xScaleN = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[2]));
+      xScaleD = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[3]));
 
       Value yOffset, xOffset, yBorder, xBorder;
-      yOffset = b.create<arith::ConstantOp>(b.getI32IntegerAttr(offset[0]));
-      xOffset = b.create<arith::ConstantOp>(b.getI32IntegerAttr(offset[1]));
-      yBorder = b.create<arith::ConstantOp>(b.getI32IntegerAttr(border[0]));
-      xBorder = b.create<arith::ConstantOp>(b.getI32IntegerAttr(border[1]));
+      yOffset = arith::ConstantOp::create(b, b.getI32IntegerAttr(offset[0]));
+      xOffset = arith::ConstantOp::create(b, b.getI32IntegerAttr(offset[1]));
+      yBorder = arith::ConstantOp::create(b, b.getI32IntegerAttr(border[0]));
+      xBorder = arith::ConstantOp::create(b, b.getI32IntegerAttr(border[1]));
 
       // Compute the ix and dx values for both the X and Y dimensions.
       auto getIndexAndDeltaFp = [&](Value &index, Value &delta, Value in,
@@ -1846,16 +1846,16 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         }
         // x = x * scale_d + offset;
         // ix = floor(x / scale_n)
-        Value val = b.create<arith::MulIOp>(in, scaleD);
-        val = b.create<arith::AddIOp>(val, offset);
-        index = b.create<arith::FloorDivSIOp>(val, scaleN);
+        Value val = arith::MulIOp::create(b, in, scaleD);
+        val = arith::AddIOp::create(b, val, offset);
+        index = arith::FloorDivSIOp::create(b, val, scaleN);
 
         // rx = x % scale_n
         // dx = rx / scale_n
-        Value r = b.create<arith::RemSIOp>(val, scaleN);
-        Value rFp = b.create<arith::SIToFPOp>(floatTy, r);
-        Value scaleNfp = b.create<arith::UIToFPOp>(floatTy, scaleN);
-        delta = b.create<arith::DivFOp>(rFp, scaleNfp);
+        Value r = arith::RemSIOp::create(b, val, scaleN);
+        Value rFp = arith::SIToFPOp::create(b, floatTy, r);
+        Value scaleNfp = arith::UIToFPOp::create(b, floatTy, scaleN);
+        delta = arith::DivFOp::create(b, rFp, scaleNfp);
       };
 
       // Compute the ix and dx values for the X and Y dimensions - int case.
@@ -1870,11 +1870,11 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         // x = x * scale_d + offset;
         // ix = floor(x / scale_n)
         //  dx = x - ix * scale_n;
-        Value val = b.create<arith::MulIOp>(in, scaleD);
-        val = b.create<arith::AddIOp>(val, offset);
-        index = b.create<arith::DivSIOp>(val, scaleN);
-        delta = b.create<arith::MulIOp>(index, scaleN);
-        delta = b.create<arith::SubIOp>(val, delta);
+        Value val = arith::MulIOp::create(b, in, scaleD);
+        val = arith::AddIOp::create(b, val, offset);
+        index = arith::DivSIOp::create(b, val, scaleN);
+        delta = arith::MulIOp::create(b, index, scaleN);
+        delta = arith::SubIOp::create(b, val, delta);
       };
 
       Value ix, iy, dx, dy;
@@ -1887,54 +1887,54 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       }
 
       if (op.getMode() == "NEAREST_NEIGHBOR") {
-        auto one = b.create<arith::ConstantOp>(b.getI32IntegerAttr(1));
+        auto one = arith::ConstantOp::create(b, b.getI32IntegerAttr(1));
 
         auto getNearestIndexAndClamp = [&](Value val, Value dval, Value scale,
                                            Value max, int size,
                                            ImplicitLocOpBuilder &b) -> Value {
           if (size == 1) {
-            return b.create<arith::ConstantIndexOp>(0);
+            return arith::ConstantIndexOp::create(b, 0);
           }
 
           Value pred;
           if (floatingPointMode) {
-            auto h = b.create<arith::ConstantOp>(b.getFloatAttr(floatTy, 0.5f));
-            pred = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, dval, h);
+            auto h = arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 0.5f));
+            pred = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, dval, h);
           } else {
-            Value dvalDouble = b.create<arith::ShLIOp>(dval, one);
-            pred = b.create<arith::CmpIOp>(arith::CmpIPredicate::sge,
+            Value dvalDouble = arith::ShLIOp::create(b, dval, one);
+            pred = arith::CmpIOp::create(b, arith::CmpIPredicate::sge,
                                            dvalDouble, scale);
           }
 
-          auto offset = b.create<arith::SelectOp>(pred, one, zeroI32);
-          val = b.create<arith::AddIOp>(val, offset);
+          auto offset = arith::SelectOp::create(b, pred, one, zeroI32);
+          val = arith::AddIOp::create(b, val, offset);
           val = clampIntHelper(loc, val, zeroI32, max, b, /*isUnsigned=*/false);
-          return b.create<arith::IndexCastOp>(b.getIndexType(), val);
+          return arith::IndexCastOp::create(b, b.getIndexType(), val);
         };
 
         iy = getNearestIndexAndClamp(iy, dy, yScaleN, hMax, imageH, b);
         ix = getNearestIndexAndClamp(ix, dx, xScaleN, wMax, imageW, b);
 
-        Value result = b.create<tensor::ExtractOp>(
+        Value result = tensor::ExtractOp::create(b,
             input, ValueRange{batch, iy, ix, channel});
 
-        b.create<linalg::YieldOp>(result);
+        linalg::YieldOp::create(b, result);
       } else {
         // The mode here must be BILINEAR.
         assert(op.getMode() == "BILINEAR");
 
-        auto oneVal = b.create<arith::ConstantOp>(b.getI32IntegerAttr(1));
+        auto oneVal = arith::ConstantOp::create(b, b.getI32IntegerAttr(1));
 
         auto getClampedIdxs = [&](Value &val0, Value &val1, int size, Value in,
                                   Value max, ImplicitLocOpBuilder &b) {
           val0 = in;
-          val1 = b.create<arith::AddIOp>(val0, oneVal);
+          val1 = arith::AddIOp::create(b, val0, oneVal);
           val0 =
               clampIntHelper(loc, val0, zeroI32, max, b, /*isUnsigned=*/false);
           val1 =
               clampIntHelper(loc, val1, zeroI32, max, b, /*isUnsigned=*/false);
-          val0 = b.create<arith::IndexCastOp>(b.getIndexType(), val0);
-          val1 = b.create<arith::IndexCastOp>(b.getIndexType(), val1);
+          val0 = arith::IndexCastOp::create(b, b.getIndexType(), val0);
+          val1 = arith::IndexCastOp::create(b, b.getIndexType(), val1);
         };
 
         // Linalg equivalent to the section below:
@@ -1946,27 +1946,27 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         getClampedIdxs(y0, y1, imageH, iy, hMax, b);
         getClampedIdxs(x0, x1, imageW, ix, wMax, b);
 
-        Value y0x0 = b.create<tensor::ExtractOp>(
+        Value y0x0 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y0, x0, channel});
-        Value y0x1 = b.create<tensor::ExtractOp>(
+        Value y0x1 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y0, x1, channel});
-        Value y1x0 = b.create<tensor::ExtractOp>(
+        Value y1x0 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y1, x0, channel});
-        Value y1x1 = b.create<tensor::ExtractOp>(
+        Value y1x1 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y1, x1, channel});
 
         if (floatingPointMode) {
           auto oneVal =
-              b.create<arith::ConstantOp>(b.getFloatAttr(floatTy, 1.0f));
+              arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 1.0f));
           auto interpolate = [&](Value val0, Value val1, Value delta,
                                  int inputSize,
                                  ImplicitLocOpBuilder &b) -> Value {
             if (inputSize == 1)
               return val0;
-            Value oneMinusDelta = b.create<arith::SubFOp>(oneVal, delta);
-            Value mul0 = b.create<arith::MulFOp>(val0, oneMinusDelta);
-            Value mul1 = b.create<arith::MulFOp>(val1, delta);
-            return b.create<arith::AddFOp>(mul0, mul1);
+            Value oneMinusDelta = arith::SubFOp::create(b, oneVal, delta);
+            Value mul0 = arith::MulFOp::create(b, val0, oneMinusDelta);
+            Value mul1 = arith::MulFOp::create(b, val1, delta);
+            return arith::AddFOp::create(b, mul0, mul1);
           };
 
           // Linalg equivalent to the section below:
@@ -1982,18 +1982,18 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
           // Linalg equivalent to the section below:
           //   result = topAcc * (unit_y - dy) + bottomAcc * dy
           Value result = interpolate(topAcc, bottomAcc, dy, imageH, b);
-          b.create<linalg::YieldOp>(result);
+          linalg::YieldOp::create(b, result);
         } else {
           // Perform in quantized space.
-          y0x0 = b.create<arith::ExtSIOp>(resultETy, y0x0);
-          y0x1 = b.create<arith::ExtSIOp>(resultETy, y0x1);
-          y1x0 = b.create<arith::ExtSIOp>(resultETy, y1x0);
-          y1x1 = b.create<arith::ExtSIOp>(resultETy, y1x1);
+          y0x0 = arith::ExtSIOp::create(b, resultETy, y0x0);
+          y0x1 = arith::ExtSIOp::create(b, resultETy, y0x1);
+          y1x0 = arith::ExtSIOp::create(b, resultETy, y1x0);
+          y1x1 = arith::ExtSIOp::create(b, resultETy, y1x1);
 
           const int64_t deltaBitwidth = dx.getType().getIntOrFloatBitWidth();
           if (resultETy.getIntOrFloatBitWidth() > deltaBitwidth) {
-            dx = b.create<arith::ExtSIOp>(resultETy, dx);
-            dy = b.create<arith::ExtSIOp>(resultETy, dy);
+            dx = arith::ExtSIOp::create(b, resultETy, dx);
+            dy = arith::ExtSIOp::create(b, resultETy, dy);
           }
 
           Value yScaleNExt = yScaleN;
@@ -2002,26 +2002,26 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
           const int64_t scaleBitwidth =
               xScaleN.getType().getIntOrFloatBitWidth();
           if (resultETy.getIntOrFloatBitWidth() > scaleBitwidth) {
-            yScaleNExt = b.create<arith::ExtSIOp>(resultETy, yScaleN);
-            xScaleNExt = b.create<arith::ExtSIOp>(resultETy, xScaleN);
+            yScaleNExt = arith::ExtSIOp::create(b, resultETy, yScaleN);
+            xScaleNExt = arith::ExtSIOp::create(b, resultETy, xScaleN);
           }
 
           auto interpolate = [](Value val0, Value val1, Value weight1,
                                 Value scale, int inputSize,
                                 ImplicitLocOpBuilder &b) -> Value {
             if (inputSize == 1)
-              return b.create<arith::MulIOp>(val0, scale);
-            Value weight0 = b.create<arith::SubIOp>(scale, weight1);
-            Value mul0 = b.create<arith::MulIOp>(val0, weight0);
-            Value mul1 = b.create<arith::MulIOp>(val1, weight1);
-            return b.create<arith::AddIOp>(mul0, mul1);
+              return arith::MulIOp::create(b, val0, scale);
+            Value weight0 = arith::SubIOp::create(b, scale, weight1);
+            Value mul0 = arith::MulIOp::create(b, val0, weight0);
+            Value mul1 = arith::MulIOp::create(b, val1, weight1);
+            return arith::AddIOp::create(b, mul0, mul1);
           };
 
           Value topAcc = interpolate(y0x0, y0x1, dx, xScaleNExt, imageW, b);
           Value bottomAcc = interpolate(y1x0, y1x1, dx, xScaleNExt, imageW, b);
           Value result =
               interpolate(topAcc, bottomAcc, dy, yScaleNExt, imageH, b);
-          b.create<linalg::YieldOp>(result);
+          linalg::YieldOp::create(b, result);
         }
       }
     }
@@ -2072,11 +2072,11 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i)) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
-    Value axisDimSize = rewriter.create<tensor::DimOp>(loc, input, axis);
+    Value axisDimSize = tensor::DimOp::create(rewriter, loc, input, axis);
 
     // First fill the output buffer with the init value.
     auto emptyTensor = rewriter
@@ -2094,21 +2094,21 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
           llvm::SmallVector<Value> indices;
           for (unsigned int i = 0; i < inputTy.getRank(); i++) {
             Value index =
-                rewriter.create<linalg::IndexOp>(nestedLoc, i).getResult();
+                linalg::IndexOp::create(rewriter, nestedLoc, i).getResult();
             if (i == axis) {
-              auto one = rewriter.create<arith::ConstantIndexOp>(nestedLoc, 1);
+              auto one = arith::ConstantIndexOp::create(rewriter, nestedLoc, 1);
               auto sizeMinusOne =
-                  rewriter.create<arith::SubIOp>(nestedLoc, axisDimSize, one);
-              index = rewriter.create<arith::SubIOp>(nestedLoc, sizeMinusOne,
+                  arith::SubIOp::create(rewriter, nestedLoc, axisDimSize, one);
+              index = arith::SubIOp::create(rewriter, nestedLoc, sizeMinusOne,
                                                      index);
             }
 
             indices.push_back(index);
           }
 
-          auto extract = nestedBuilder.create<tensor::ExtractOp>(
+          auto extract = tensor::ExtractOp::create(nestedBuilder,
               nestedLoc, input, indices);
-          nestedBuilder.create<linalg::YieldOp>(op.getLoc(),
+          linalg::YieldOp::create(nestedBuilder, op.getLoc(),
                                                 extract.getResult());
         });
     return success();
@@ -2148,11 +2148,11 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i) || multiples[i] == -1) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         op.getLoc(), genericShape, elementTy, dynDims);
 
     // We needs to map the input shape to the non-broadcasted dimensions.
@@ -2168,12 +2168,12 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<AffineMap, 2> affineMaps = {
         readAffineMap, rewriter.getMultiDimIdentityMap(genericShape.size())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, RankedTensorType::get(genericShape, elementTy), input,
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(genericShape.size()),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-          nestedBuilder.create<linalg::YieldOp>(op.getLoc(), *args.begin());
+          linalg::YieldOp::create(nestedBuilder, op.getLoc(), *args.begin());
         });
 
     auto shapeValue = getTosaConstShape(
@@ -2220,7 +2220,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i) && i != axis) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
@@ -2229,7 +2229,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
                               .create<tensor::EmptyOp>(loc, resultTy.getShape(),
                                                        outElementTy, dynDims)
                               .getResult();
-    auto fillValueIdx = rewriter.create<arith::ConstantOp>(
+    auto fillValueIdx = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(outElementTy, 0));
     auto filledTensorIdx =
         rewriter
@@ -2250,7 +2250,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           argmaxOp, "unsupported tosa.argmax element type");
 
     auto fillValueMax =
-        rewriter.create<arith::ConstantOp>(loc, fillValueMaxAttr);
+        arith::ConstantOp::create(rewriter, loc, fillValueMaxAttr);
     auto filledTensorMax =
         rewriter
             .create<linalg::FillOp>(loc, ValueRange{fillValueMax},
@@ -2274,7 +2274,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     bool didEncounterError = false;
     auto maps = AffineMap::inferFromExprList({srcExprs, dstExprs, dstExprs},
                                              rewriter.getContext());
-    auto linalgOp = rewriter.create<linalg::GenericOp>(
+    auto linalgOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
         ValueRange({filledTensorIdx, filledTensorMax}), maps, iteratorTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
@@ -2283,41 +2283,41 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           auto oldIndex = blockArgs[1];
           auto oldValue = blockArgs[2];
 
-          Value newIndex = rewriter.create<arith::IndexCastOp>(
+          Value newIndex = arith::IndexCastOp::create(rewriter,
               nestedLoc, oldIndex.getType(),
-              rewriter.create<linalg::IndexOp>(loc, axis));
+              linalg::IndexOp::create(rewriter, loc, axis));
 
           Value predicate;
           if (isa<FloatType>(inElementTy)) {
             if (argmaxOp.getNanMode() == "IGNORE") {
               // Only update index & max value for non NaN values. If all
               // values are NaNs, the initial index will be return which is 0.
-              predicate = rewriter.create<arith::CmpFOp>(
+              predicate = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::OGT, newValue, oldValue);
             } else {
               // Update max value if either of the following is true:
               // - new value is bigger
               // - cur max is not NaN and new value is NaN
-              Value gt = rewriter.create<arith::CmpFOp>(
+              Value gt = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::UGT, newValue, oldValue);
-              Value oldNonNaN = rewriter.create<arith::CmpFOp>(
+              Value oldNonNaN = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::ORD, oldValue, oldValue);
-              predicate = rewriter.create<arith::AndIOp>(
+              predicate = arith::AndIOp::create(rewriter,
                   nestedLoc, rewriter.getI1Type(), gt, oldNonNaN);
             }
           } else if (isa<IntegerType>(inElementTy)) {
-            predicate = rewriter.create<arith::CmpIOp>(
+            predicate = arith::CmpIOp::create(rewriter,
                 nestedLoc, arith::CmpIPredicate::sgt, newValue, oldValue);
           } else {
             didEncounterError = true;
             return;
           }
 
-          auto resultMax = rewriter.create<arith::SelectOp>(
+          auto resultMax = arith::SelectOp::create(rewriter,
               nestedLoc, predicate, newValue, oldValue);
-          auto resultIndex = rewriter.create<arith::SelectOp>(
+          auto resultIndex = arith::SelectOp::create(rewriter,
               nestedLoc, predicate, newIndex, oldIndex);
-          nestedBuilder.create<linalg::YieldOp>(
+          linalg::YieldOp::create(nestedBuilder,
               nestedLoc, ValueRange({resultIndex, resultMax}));
         });
 
@@ -2363,19 +2363,19 @@ class GatherConverter : public OpConversionPattern<tosa::GatherOp> {
             rewriter.getContext()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy}), ValueRange{indices},
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
           auto indexValue = args[0];
-          auto index0 = rewriter.create<linalg::IndexOp>(loc, 0);
-          Value index1 = rewriter.create<arith::IndexCastOp>(
+          auto index0 = linalg::IndexOp::create(rewriter, loc, 0);
+          Value index1 = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getIndexType(), indexValue);
-          auto index2 = rewriter.create<linalg::IndexOp>(loc, 2);
-          Value extract = rewriter.create<tensor::ExtractOp>(
+          auto index2 = linalg::IndexOp::create(rewriter, loc, 2);
+          Value extract = tensor::ExtractOp::create(rewriter,
               loc, input, ValueRange{index0, index1, index2});
-          rewriter.create<linalg::YieldOp>(loc, extract);
+          linalg::YieldOp::create(rewriter, loc, extract);
         });
     rewriter.replaceOp(op, genericOp.getResult(0));
     return success();
@@ -2424,7 +2424,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
     for (int i = 0; i < resultTy.getRank(); ++i) {
       if (inputTy.isDynamicDim(i)) {
         dynDims.push_back(
-            rewriter.create<tensor::DimOp>(loc, op.getOperand(0), i));
+            tensor::DimOp::create(rewriter, loc, op.getOperand(0), i));
       }
     }
 
@@ -2437,7 +2437,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
         rewriter.getMultiDimIdentityMap(resultTy.getRank()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, resultTy, ValueRange({input}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -2452,69 +2452,69 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
       rewriter.setInsertionPointToStart(block);
       if (inputElementTy.isInteger(8) && tableElementTy.isInteger(8) &&
           resultElementTy.isInteger(8)) {
-        Value index = rewriter.create<arith::IndexCastOp>(
+        Value index = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), inputValue);
-        Value offset = rewriter.create<arith::ConstantIndexOp>(loc, 128);
-        index = rewriter.create<arith::AddIOp>(loc, rewriter.getIndexType(),
+        Value offset = arith::ConstantIndexOp::create(rewriter, loc, 128);
+        index = arith::AddIOp::create(rewriter, loc, rewriter.getIndexType(),
                                                index, offset);
         Value extract =
-            rewriter.create<tensor::ExtractOp>(loc, table, ValueRange{index});
-        rewriter.create<linalg::YieldOp>(loc, extract);
+            tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
+        linalg::YieldOp::create(rewriter, loc, extract);
         return success();
       }
 
       if (inputElementTy.isInteger(16) && tableElementTy.isInteger(16) &&
           resultElementTy.isInteger(32)) {
-        Value extend = rewriter.create<arith::ExtSIOp>(
+        Value extend = arith::ExtSIOp::create(rewriter,
             loc, rewriter.getI32Type(), inputValue);
 
-        auto offset = rewriter.create<arith::ConstantOp>(
+        auto offset = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(32768));
-        auto seven = rewriter.create<arith::ConstantOp>(
+        auto seven = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(7));
-        auto one = rewriter.create<arith::ConstantOp>(
+        auto one = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(1));
-        auto b1111111 = rewriter.create<arith::ConstantOp>(
+        auto b1111111 = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(127));
 
         // Compute the index and fractional part from the input value:
         // value = value + 32768
         // index = value >> 7;
         // fraction = 0x01111111 & value
-        auto extendAdd = rewriter.create<arith::AddIOp>(loc, extend, offset);
-        Value index = rewriter.create<arith::ShRUIOp>(loc, extendAdd, seven);
+        auto extendAdd = arith::AddIOp::create(rewriter, loc, extend, offset);
+        Value index = arith::ShRUIOp::create(rewriter, loc, extendAdd, seven);
         Value fraction =
-            rewriter.create<arith::AndIOp>(loc, extendAdd, b1111111);
+            arith::AndIOp::create(rewriter, loc, extendAdd, b1111111);
 
         // Extract the base and next values from the table.
         // base = (int32_t) table[index];
         // next = (int32_t) table[index + 1];
-        Value indexPlusOne = rewriter.create<arith::AddIOp>(loc, index, one);
+        Value indexPlusOne = arith::AddIOp::create(rewriter, loc, index, one);
 
-        index = rewriter.create<arith::IndexCastOp>(
+        index = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), index);
-        indexPlusOne = rewriter.create<arith::IndexCastOp>(
+        indexPlusOne = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), indexPlusOne);
 
         Value base =
-            rewriter.create<tensor::ExtractOp>(loc, table, ValueRange{index});
-        Value next = rewriter.create<tensor::ExtractOp>(
+            tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
+        Value next = tensor::ExtractOp::create(rewriter,
             loc, table, ValueRange{indexPlusOne});
 
         base =
-            rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), base);
+            arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), base);
         next =
-            rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), next);
+            arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), next);
 
         // Use the fractional part to interpolate between the input values:
         // result = (base << 7) + (next - base) * fraction
-        Value baseScaled = rewriter.create<arith::ShLIOp>(loc, base, seven);
-        Value diff = rewriter.create<arith::SubIOp>(loc, next, base);
-        Value diffScaled = rewriter.create<arith::MulIOp>(loc, diff, fraction);
+        Value baseScaled = arith::ShLIOp::create(rewriter, loc, base, seven);
+        Value diff = arith::SubIOp::create(rewriter, loc, next, base);
+        Value diffScaled = arith::MulIOp::create(rewriter, loc, diff, fraction);
         Value result =
-            rewriter.create<arith::AddIOp>(loc, baseScaled, diffScaled);
+            arith::AddIOp::create(rewriter, loc, baseScaled, diffScaled);
 
-        rewriter.create<linalg::YieldOp>(loc, result);
+        linalg::YieldOp::create(rewriter, loc, result);
 
         return success();
       }
@@ -2532,8 +2532,8 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static OpFoldResult halfPlusOne(OpBuilder &builder, Location loc,
                                   OpFoldResult ofr) {
-    auto one = builder.create<arith::ConstantIndexOp>(loc, 1);
-    auto two = builder.create<arith::ConstantIndexOp>(loc, 2);
+    auto one = arith::ConstantIndexOp::create(builder, loc, 1);
+    auto two = arith::ConstantIndexOp::create(builder, loc, 2);
 
     auto value = getValueOrCreateConstantIndexOp(builder, loc, ofr);
     auto divBy2 = builder.createOrFold<arith::DivUIOp>(loc, value, two);
@@ -2562,9 +2562,9 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
                                 RankedTensorType type,
                                 llvm::ArrayRef<Value> dynamicSizes) {
     auto emptyTensor =
-        rewriter.create<tensor::EmptyOp>(loc, type, dynamicSizes);
+        tensor::EmptyOp::create(rewriter, loc, type, dynamicSizes);
     auto fillValueAttr = rewriter.getZeroAttr(type.getElementType());
-    auto fillValue = rewriter.create<arith::ConstantOp>(loc, fillValueAttr);
+    auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
     auto filledTensor = rewriter
                             .create<linalg::FillOp>(loc, ValueRange{fillValue},
                                                     ValueRange{emptyTensor})
@@ -2574,18 +2574,18 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static Value castIndexToFloat(OpBuilder &builder, Location loc,
                                 FloatType type, Value value) {
-    auto integerVal = builder.create<arith::IndexCastUIOp>(
+    auto integerVal = arith::IndexCastUIOp::create(builder,
         loc,
         type.getIntOrFloatBitWidth() > 32 ? builder.getI64Type()
                                           : builder.getI32Type(),
         value);
 
-    return builder.create<arith::UIToFPOp>(loc, type, integerVal);
+    return arith::UIToFPOp::create(builder, loc, type, integerVal);
   }
 
   static Value createLinalgIndex(OpBuilder &builder, Location loc,
                                  FloatType type, int64_t index) {
-    auto indexVal = builder.create<linalg::IndexOp>(loc, index);
+    auto indexVal = linalg::IndexOp::create(builder, loc, index);
     return castIndexToFloat(builder, loc, type, indexVal);
   }
 
@@ -2640,7 +2640,7 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
     // Constants and dimension sizes
     auto twoPiAttr = rewriter.getFloatAttr(elementType, 6.283185307179586);
-    auto twoPi = rewriter.create<arith::ConstantOp>(loc, twoPiAttr);
+    auto twoPi = arith::ConstantOp::create(rewriter, loc, twoPiAttr);
     auto constH = castIndexToFloat(rewriter, loc, elementType, dimH);
     auto constW = castIndexToFloat(rewriter, loc, elementType, dimW);
 
@@ -2650,43 +2650,43 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
       Value sumImag = args[2];
 
       // Indices for angle computation
-      Value oy = builder.create<linalg::IndexOp>(loc, 1);
-      Value ox = builder.create<linalg::IndexOp>(loc, 2);
-      Value iy = builder.create<linalg::IndexOp>(loc, 3);
-      Value ix = builder.create<linalg::IndexOp>(loc, 4);
+      Value oy = linalg::IndexOp::create(builder, loc, 1);
+      Value ox = linalg::IndexOp::create(builder, loc, 2);
+      Value iy = linalg::IndexOp::create(builder, loc, 3);
+      Value ix = linalg::IndexOp::create(builder, loc, 4);
 
       // Calculating angle without integer parts of components as sin/cos are
       // periodic: angle = 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix * ox) % W )
       // / W);
-      auto iyXoy = builder.create<index::MulOp>(loc, iy, oy);
-      auto ixXox = builder.create<index::MulOp>(loc, ix, ox);
+      auto iyXoy = index::MulOp::create(builder, loc, iy, oy);
+      auto ixXox = index::MulOp::create(builder, loc, ix, ox);
 
-      auto iyRem = builder.create<index::RemUOp>(loc, iyXoy, dimH);
-      auto ixRem = builder.create<index::RemUOp>(loc, ixXox, dimW);
+      auto iyRem = index::RemUOp::create(builder, loc, iyXoy, dimH);
+      auto ixRem = index::RemUOp::create(builder, loc, ixXox, dimW);
 
       auto iyRemFloat = castIndexToFloat(builder, loc, elementType, iyRem);
       auto ixRemFloat = castIndexToFloat(builder, loc, elementType, ixRem);
 
-      auto yComponent = builder.create<arith::DivFOp>(loc, iyRemFloat, constH);
-      auto xComponent = builder.create<arith::DivFOp>(loc, ixRemFloat, constW);
-      auto sumXY = builder.create<arith::AddFOp>(loc, yComponent, xComponent);
-      auto angle = builder.create<arith::MulFOp>(loc, twoPi, sumXY);
+      auto yComponent = arith::DivFOp::create(builder, loc, iyRemFloat, constH);
+      auto xComponent = arith::DivFOp::create(builder, loc, ixRemFloat, constW);
+      auto sumXY = arith::AddFOp::create(builder, loc, yComponent, xComponent);
+      auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       // realComponent = valReal * cos(angle)
       // imagComponent = valReal * sin(angle)
-      auto cosAngle = builder.create<math::CosOp>(loc, angle);
-      auto sinAngle = builder.create<math::SinOp>(loc, angle);
+      auto cosAngle = math::CosOp::create(builder, loc, angle);
+      auto sinAngle = math::SinOp::create(builder, loc, angle);
       auto realComponent =
-          builder.create<arith::MulFOp>(loc, valReal, cosAngle);
+          arith::MulFOp::create(builder, loc, valReal, cosAngle);
       auto imagComponent =
-          builder.create<arith::MulFOp>(loc, valReal, sinAngle);
+          arith::MulFOp::create(builder, loc, valReal, sinAngle);
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = builder.create<arith::AddFOp>(loc, sumReal, realComponent);
-      auto outImag = builder.create<arith::SubFOp>(loc, sumImag, imagComponent);
+      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag = arith::SubFOp::create(builder, loc, sumImag, imagComponent);
 
-      builder.create<linalg::YieldOp>(loc, ValueRange{outReal, outImag});
+      linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
 
     rewriter.replaceOpWithNewOp<linalg::GenericOp>(
@@ -2760,7 +2760,7 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
 
     // Constants and dimension sizes
     auto twoPiAttr = rewriter.getFloatAttr(real_el_ty, 6.283185307179586);
-    auto twoPi = rewriter.create<arith::ConstantOp>(loc, twoPiAttr);
+    auto twoPi = arith::ConstantOp::create(rewriter, loc, twoPiAttr);
     Value constH =
         RFFT2dConverter::castIndexToFloat(rewriter, loc, real_el_ty, dimH);
     Value constW =
@@ -2773,57 +2773,57 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
       Value sumImag = args[3];
 
       // Indices for angle computation
-      Value oy = builder.create<linalg::IndexOp>(loc, 1);
-      Value ox = builder.create<linalg::IndexOp>(loc, 2);
-      Value iy = builder.create<linalg::IndexOp>(loc, 3);
-      Value ix = builder.create<linalg::IndexOp>(loc, 4);
+      Value oy = linalg::IndexOp::create(builder, loc, 1);
+      Value ox = linalg::IndexOp::create(builder, loc, 2);
+      Value iy = linalg::IndexOp::create(builder, loc, 3);
+      Value ix = linalg::IndexOp::create(builder, loc, 4);
 
       // float_t angle = sign_val * 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix *
       // ox) % W ) / W);
-      auto iyXoy = builder.create<index::MulOp>(loc, iy, oy);
-      auto ixXox = builder.create<index::MulOp>(loc, ix, ox);
+      auto iyXoy = index::MulOp::create(builder, loc, iy, oy);
+      auto ixXox = index::MulOp::create(builder, loc, ix, ox);
 
-      auto iyRem = builder.create<index::RemUOp>(loc, iyXoy, dimH);
-      auto ixRem = builder.create<index::RemUOp>(loc, ixXox, dimW);
+      auto iyRem = index::RemUOp::create(builder, loc, iyXoy, dimH);
+      auto ixRem = index::RemUOp::create(builder, loc, ixXox, dimW);
 
       auto iyRemFloat =
           RFFT2dConverter::castIndexToFloat(builder, loc, real_el_ty, iyRem);
       auto ixRemFloat =
           RFFT2dConverter::castIndexToFloat(builder, loc, real_el_ty, ixRem);
 
-      auto yComponent = builder.create<arith::DivFOp>(loc, iyRemFloat, constH);
-      auto xComponent = builder.create<arith::DivFOp>(loc, ixRemFloat, constW);
+      auto yComponent = arith::DivFOp::create(builder, loc, iyRemFloat, constH);
+      auto xComponent = arith::DivFOp::create(builder, loc, ixRemFloat, constW);
 
-      auto sumXY = builder.create<arith::AddFOp>(loc, yComponent, xComponent);
-      auto angle = builder.create<arith::MulFOp>(loc, twoPi, sumXY);
+      auto sumXY = arith::AddFOp::create(builder, loc, yComponent, xComponent);
+      auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       if (inverse.getValue()) {
-        angle = builder.create<arith::MulFOp>(
+        angle = arith::MulFOp::create(builder,
             loc, angle,
-            rewriter.create<arith::ConstantOp>(
+            arith::ConstantOp::create(rewriter,
                 loc, rewriter.getFloatAttr(real_el_ty, -1.0)));
       }
 
       // realComponent = val_real * cos(a) + val_imag * sin(a);
       // imagComponent = -val_real * sin(a) + val_imag * cos(a);
-      auto cosAngle = builder.create<math::CosOp>(loc, angle);
-      auto sinAngle = builder.create<math::SinOp>(loc, angle);
+      auto cosAngle = math::CosOp::create(builder, loc, angle);
+      auto sinAngle = math::SinOp::create(builder, loc, angle);
 
-      auto rcos = builder.create<arith::MulFOp>(loc, valReal, cosAngle);
-      auto rsin = builder.create<arith::MulFOp>(loc, valImag, sinAngle);
-      auto realComponent = builder.create<arith::AddFOp>(loc, rcos, rsin);
+      auto rcos = arith::MulFOp::create(builder, loc, valReal, cosAngle);
+      auto rsin = arith::MulFOp::create(builder, loc, valImag, sinAngle);
+      auto realComponent = arith::AddFOp::create(builder, loc, rcos, rsin);
 
-      auto icos = builder.create<arith::MulFOp>(loc, valImag, cosAngle);
-      auto isin = builder.create<arith::MulFOp>(loc, valReal, sinAngle);
+      auto icos = arith::MulFOp::create(builder, loc, valImag, cosAngle);
+      auto isin = arith::MulFOp::create(builder, loc, valReal, sinAngle);
 
-      auto imagComponent = builder.create<arith::SubFOp>(loc, icos, isin);
+      auto imagComponent = arith::SubFOp::create(builder, loc, icos, isin);
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = builder.create<arith::AddFOp>(loc, sumReal, realComponent);
-      auto outImag = builder.create<arith::AddFOp>(loc, sumImag, imagComponent);
+      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag = arith::AddFOp::create(builder, loc, sumImag, imagComponent);
 
-      builder.create<linalg::YieldOp>(loc, ValueRange{outReal, outImag});
+      linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
 
     rewriter.replaceOpWithNewOp<linalg::GenericOp>(
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index 00b9a065dfb3d..bfe43e8dcc274 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -52,9 +52,9 @@ static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
     highIndices.push_back(rewriter.getIndexAttr(highPad));
   }
 
-  Value padValue = rewriter.create<arith::ConstantOp>(loc, padAttr);
+  Value padValue = arith::ConstantOp::create(rewriter, loc, padAttr);
 
-  return rewriter.create<tensor::PadOp>(
+  return tensor::PadOp::create(rewriter,
       loc, RankedTensorType::get(paddedShape, inputETy), input, lowIndices,
       highIndices, padValue);
 }
@@ -72,10 +72,10 @@ linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias,
             Value biasVal = args[0];
             Type resType = args[1].getType();
             if (resType != biasVal.getType()) {
-              biasVal = builder.create<arith::ExtSIOp>(loc, resType, biasVal);
+              biasVal = arith::ExtSIOp::create(builder, loc, resType, biasVal);
             }
-            Value added = builder.create<arith::AddIOp>(loc, biasVal, args[1]);
-            builder.create<linalg::YieldOp>(loc, added);
+            Value added = arith::AddIOp::create(builder, loc, biasVal, args[1]);
+            linalg::YieldOp::create(builder, loc, added);
           })
       .getResult(0);
 }
@@ -134,19 +134,19 @@ static mlir::Value linalgBroadcastAndMaybeExt(PatternRewriter &rewriter,
             if (resType != biasVal.getType()) {
               biasVal =
                   resultTy.getElementType().isFloat()
-                      ? builder.create<arith::ExtFOp>(loc, resType, biasVal)
+                      ? arith::ExtFOp::create(builder, loc, resType, biasVal)
                             .getResult()
-                      : builder.create<arith::ExtSIOp>(loc, resType, biasVal)
+                      : arith::ExtSIOp::create(builder, loc, resType, biasVal)
                             .getResult();
             }
-            builder.create<linalg::YieldOp>(loc, biasVal);
+            linalg::YieldOp::create(builder, loc, biasVal);
           })
       .getResult(0);
 }
 
 static mlir::Value reifyConstantDim(int64_t attr,
                                     ImplicitLocOpBuilder &builder) {
-  return builder.create<arith::ConstantIndexOp>(attr);
+  return arith::ConstantIndexOp::create(builder, attr);
 }
 
 // Calculating the output width/height using the formula:
@@ -160,22 +160,22 @@ static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim,
                                           int64_t dilationAttr,
                                           OpBuilder &rewriter) {
   ImplicitLocOpBuilder builder(loc, rewriter);
-  auto one = rewriter.create<arith::ConstantOp>(
+  auto one = arith::ConstantOp::create(rewriter,
       loc, IntegerAttr::get(inputDim.getType(), 1));
   Value padBefore = reifyConstantDim(padBeforeAttr, builder);
-  Value paddedBefore = builder.create<arith::AddIOp>(inputDim, padBefore);
+  Value paddedBefore = arith::AddIOp::create(builder, inputDim, padBefore);
   Value padAfter = reifyConstantDim(padAfterAttr, builder);
-  Value paddedAfter = builder.create<arith::AddIOp>(paddedBefore, padAfter);
+  Value paddedAfter = arith::AddIOp::create(builder, paddedBefore, padAfter);
 
-  Value subOne = builder.create<arith::SubIOp>(kernelDim, one);
+  Value subOne = arith::SubIOp::create(builder, kernelDim, one);
   Value dilation = reifyConstantDim(dilationAttr, builder);
-  Value dilated = builder.create<arith::MulIOp>(dilation, subOne);
-  Value addOne = builder.create<arith::AddIOp>(dilated, one);
+  Value dilated = arith::MulIOp::create(builder, dilation, subOne);
+  Value addOne = arith::AddIOp::create(builder, dilated, one);
 
-  Value subtract = builder.create<arith::SubIOp>(paddedAfter, addOne);
+  Value subtract = arith::SubIOp::create(builder, paddedAfter, addOne);
   Value stride = reifyConstantDim(strideAttr, builder);
-  Value divide = builder.create<arith::DivUIOp>(subtract, stride);
-  return builder.create<arith::AddIOp>(divide, one);
+  Value divide = arith::DivUIOp::create(builder, subtract, stride);
+  return arith::AddIOp::create(builder, divide, one);
 }
 
 // Creates a vector of the dynamic output dims for Conv2D and Depthwise_Conv2D
@@ -198,9 +198,9 @@ static SmallVector<Value> inferDynamicDimsForConv(
       auto padBottom = padAttr[i * 2 + 1];
       auto stride = strideAttr[i];
       auto dilation = dilationAttr[i];
-      Value initDynDim = rewriter.create<tensor::DimOp>(loc, input, inputDim);
+      Value initDynDim = tensor::DimOp::create(rewriter, loc, input, inputDim);
       Value kernelDynDim =
-          rewriter.create<tensor::DimOp>(loc, weight, kernelDim);
+          tensor::DimOp::create(rewriter, loc, weight, kernelDim);
       // H = F(IH, pad_top, pad_bottom, dilation_y, KH, stride_y)
       dynDims[inputDim] =
           getConvOrPoolOutputDim(loc, initDynDim, padTop, padBottom,
@@ -211,7 +211,7 @@ static SmallVector<Value> inferDynamicDimsForConv(
   // Get the batch/channels dimensions.
   for (int i = 0; i < inputRank; i++) {
     if (resultTy.isDynamicDim(i) && !dynDims[i])
-      dynDims[i] = rewriter.create<tensor::DimOp>(loc, input, i);
+      dynDims[i] = tensor::DimOp::create(rewriter, loc, input, i);
   }
 
   SmallVector<Value> filteredDims = condenseValues(dynDims);
@@ -350,7 +350,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
         auto weightPermAttr = rewriter.getDenseI32ArrayAttr(weightPerm);
         Type newWeightTy =
             RankedTensorType::get(newWeightShape, weightTy.getElementType());
-        weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
+        weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
                                                     weightPermAttr);
       }
     }
@@ -372,7 +372,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       auto weightPermAttr = rewriter.getDenseI32ArrayAttr(weightPerm);
       Type newWeightTy =
           RankedTensorType::get(newWeightShape, weightTy.getElementType());
-      weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
+      weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
                                                   weightPermAttr);
     }
 
@@ -384,7 +384,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     auto strideAttr = rewriter.getI64TensorAttr(stride);
     auto dilationAttr = rewriter.getI64TensorAttr(dilation);
 
-    Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), accETy, filteredDims);
 
     Value broadcastBias =
@@ -394,8 +394,8 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       auto iZp = rewriter.getI32IntegerAttr(inputZpVal);
       auto kZp = rewriter.getI32IntegerAttr(weightZpVal);
 
-      auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
+      auto iZpVal = arith::ConstantOp::create(rewriter, loc, iZp);
+      auto kZpVal = arith::ConstantOp::create(rewriter, loc, kZp);
 
       Value conv =
           rewriter
@@ -417,7 +417,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     // We may need to truncate back to the result type if the accumulator was
     // wider than the result.
     if (resultTy != accTy)
-      conv = rewriter.create<tosa::CastOp>(loc, resultTy, conv);
+      conv = tosa::CastOp::create(rewriter, loc, resultTy, conv);
 
     rewriter.replaceOp(op, conv);
     return success();
@@ -526,15 +526,15 @@ class DepthwiseConvConverter
                               accETy);
 
     auto resultZeroAttr = rewriter.getZeroAttr(accETy);
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, linalgConvTy.getShape(), accETy, filteredDims);
-    Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
+    Value zero = arith::ConstantOp::create(rewriter, loc, resultZeroAttr);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
                            .result();
 
-    Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultETy, filteredDims);
 
     // Broadcast the initial value to the output tensor before convolving.
@@ -553,7 +553,7 @@ class DepthwiseConvConverter
       // We may need to truncate back to the result type if the accumulator was
       // wider than the result.
       if (accETy != resultETy)
-        conv = rewriter.create<tosa::CastOp>(
+        conv = tosa::CastOp::create(rewriter,
             loc,
             RankedTensorType::get(cast<ShapedType>(conv.getType()).getShape(),
                                   resultETy),
@@ -561,7 +561,7 @@ class DepthwiseConvConverter
 
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
+      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
           loc, resultTy, conv, reassociationMap);
 
       Value result =
@@ -574,20 +574,20 @@ class DepthwiseConvConverter
                       ValueRange args) {
                     Value added;
                     if (llvm::isa<FloatType>(inputETy))
-                      added = nestedBuilder.create<arith::AddFOp>(loc, args[0],
+                      added = arith::AddFOp::create(nestedBuilder, loc, args[0],
                                                                   args[1]);
                     else
-                      added = nestedBuilder.create<arith::AddIOp>(loc, args[0],
+                      added = arith::AddIOp::create(nestedBuilder, loc, args[0],
                                                                   args[1]);
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
+                    linalg::YieldOp::create(nestedBuilder, nestedLoc, added);
                   })
               .getResult(0);
       rewriter.replaceOp(op, result);
     } else {
       IntegerAttr iZp = rewriter.getI32IntegerAttr(inputZpVal);
       IntegerAttr wZp = rewriter.getI32IntegerAttr(weightZpVal);
-      auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, wZp);
+      auto iZpVal = arith::ConstantOp::create(rewriter, loc, iZp);
+      auto kZpVal = arith::ConstantOp::create(rewriter, loc, wZp);
       Value conv =
           rewriter
               .create<linalg::DepthwiseConv2DNhwcHwcmQOp>(
@@ -596,7 +596,7 @@ class DepthwiseConvConverter
               .getResult(0);
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
+      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
           loc, resultTy, conv, reassociationMap);
       Value result = linalgIntBroadcastExtSIAdd(
           rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
@@ -621,22 +621,22 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
     dynDims.resize(cast<ShapedType>(op->getResult(0).getType()).getRank());
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(0)) {
-      dynDims[0] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 0);
+      dynDims[0] = tensor::DimOp::create(rewriter, loc, op->getOperand(0), 0);
     }
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(1)) {
-      dynDims[1] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 1);
+      dynDims[1] = tensor::DimOp::create(rewriter, loc, op->getOperand(0), 1);
     }
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(2)) {
-      dynDims[2] = rewriter.create<tensor::DimOp>(loc, op->getOperand(1), 2);
+      dynDims[2] = tensor::DimOp::create(rewriter, loc, op->getOperand(1), 2);
     }
 
     SmallVector<Value> filteredDims = condenseValues(dynDims);
 
     auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
-    Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
@@ -670,9 +670,9 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
       return success();
     }
 
-    auto aZp = rewriter.create<arith::ConstantOp>(
+    auto aZp = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI32IntegerAttr(aZpVal));
-    auto bZp = rewriter.create<arith::ConstantOp>(
+    auto bZp = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI32IntegerAttr(bZpVal));
     rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
         op, TypeRange{op.getType()},
@@ -702,7 +702,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     // Batch dimension
     if (resultTy.isDynamicDim(0))
-      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+      dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
     // Height/width dimensions
     for (int64_t dim : {1, 2}) {
@@ -713,10 +713,10 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       int64_t index = dim - 1;
 
       // Input height/width
-      Value ihw = rewriter.create<tensor::DimOp>(loc, input, dim);
+      Value ihw = tensor::DimOp::create(rewriter, loc, input, dim);
 
       // Kernel height/width
-      Value khw = rewriter.create<arith::ConstantIndexOp>(loc, kernel[index]);
+      Value khw = arith::ConstantIndexOp::create(rewriter, loc, kernel[index]);
 
       // Output height/width
       Value ohw = getConvOrPoolOutputDim(loc, ihw, pad[index * 2],
@@ -727,7 +727,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     // Channel dimension
     if (resultTy.isDynamicDim(3))
-      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 3));
+      dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 3));
 
     return dynamicDims;
   }
@@ -776,7 +776,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     Value paddedInput = applyPad(loc, input, pad, initialAttr, rewriter);
 
-    Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
+    Value initialValue = arith::ConstantOp::create(rewriter, loc, initialAttr);
 
     ArrayRef<int64_t> kernel = op.getKernel();
     ArrayRef<int64_t> stride = op.getStride();
@@ -785,15 +785,15 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
 
     Value filledEmptyTensor =
-        rewriter.create<linalg::FillOp>(loc, initialValue, emptyTensor)
+        linalg::FillOp::create(rewriter, loc, initialValue, emptyTensor)
             .result();
 
     Value fakeWindowDims =
-        rewriter.create<tensor::EmptyOp>(loc, kernel, resultETy);
+        tensor::EmptyOp::create(rewriter, loc, kernel, resultETy);
 
     if (isUnsigned) {
       rewriter.replaceOpWithNewOp<linalg::PoolingNhwcMaxUnsignedOp>(
@@ -802,7 +802,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       return llvm::success();
     }
 
-    auto resultOp = rewriter.create<linalg::PoolingNhwcMaxOp>(
+    auto resultOp = linalg::PoolingNhwcMaxOp::create(rewriter,
         op->getLoc(), ArrayRef<Type>{resultTy},
         ValueRange{paddedInput, fakeWindowDims}, filledEmptyTensor, strideAttr,
         dilationAttr);
@@ -823,7 +823,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     // it to include the appropriate checks. If the current value is NaN the
     // old value of pool will be taken otherwise we use the result.
     if (nanMode == "IGNORE") {
-      auto genericOp = rewriter.create<linalg::GenericOp>(
+      auto genericOp = linalg::GenericOp::create(rewriter,
           loc, resultOp.getType(0), resultOp.getInputs(), resultOp.getOutputs(),
           resultOp.getIndexingMapsArray(), resultOp.getIteratorTypesArray(),
           [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
@@ -833,12 +833,12 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
             auto &oldMaxOp = *resultOp.getBlock()->begin();
             map.map(oldArgs, blockArgs);
             auto *newOp = opBuilder.clone(oldMaxOp, map);
-            Value isNaN = opBuilder.create<arith::CmpFOp>(
+            Value isNaN = arith::CmpFOp::create(opBuilder,
                 loc, arith::CmpFPredicate::UNO, blockArgs.front(),
                 blockArgs.front());
-            auto selectOp = opBuilder.create<arith::SelectOp>(
+            auto selectOp = arith::SelectOp::create(opBuilder,
                 loc, isNaN, blockArgs.back(), newOp->getResult(0));
-            opBuilder.create<linalg::YieldOp>(loc, selectOp.getResult());
+            linalg::YieldOp::create(opBuilder, loc, selectOp.getResult());
           });
       rewriter.replaceOp(resultOp, genericOp);
     }
@@ -894,7 +894,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Value paddedInput = applyPad(loc, input, pad, padAttr, rewriter);
 
     auto initialAttr = rewriter.getZeroAttr(accETy);
-    Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
+    Value initialValue = arith::ConstantOp::create(rewriter, loc, initialAttr);
 
     ArrayRef<int64_t> kernel = op.getKernel();
     ArrayRef<int64_t> stride = op.getStride();
@@ -903,7 +903,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value poolEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value poolEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, accTy.getShape(), accETy, dynamicDims);
 
     Value filledEmptyTensor =
@@ -913,7 +913,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             .result();
 
     Value fakeWindowDims =
-        rewriter.create<tensor::EmptyOp>(loc, kernel, accETy);
+        tensor::EmptyOp::create(rewriter, loc, kernel, accETy);
 
     // Sum across the pooled region.
     Value poolingOp = rewriter
@@ -925,24 +925,24 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
 
     // Normalize the summed value by the number of elements grouped in each
     // pool.
-    Value iH = rewriter.create<tensor::DimOp>(loc, poolingOp, 1);
-    Value iW = rewriter.create<tensor::DimOp>(loc, poolingOp, 2);
+    Value iH = tensor::DimOp::create(rewriter, loc, poolingOp, 1);
+    Value iW = tensor::DimOp::create(rewriter, loc, poolingOp, 2);
 
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    iH = rewriter.create<arith::SubIOp>(loc, iH, one);
-    iW = rewriter.create<arith::SubIOp>(loc, iW, one);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    iH = arith::SubIOp::create(rewriter, loc, iH, one);
+    iW = arith::SubIOp::create(rewriter, loc, iW, one);
 
-    Value genericEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value genericEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultETy, dynamicDims);
 
     auto affineMap = rewriter.getMultiDimIdentityMap(resultTy.getRank());
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
         ValueRange{genericEmptyTensor},
         ArrayRef<AffineMap>({affineMap, affineMap}),
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
-          auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+          auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
           // Determines what the portion of valid input is covered by the
           // kernel.
@@ -950,30 +950,30 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             if (pad == 0)
               return valid;
 
-            auto padVal = rewriter.create<arith::ConstantIndexOp>(loc, pad);
-            Value dpos = rewriter.create<arith::SubIOp>(loc, pos, padVal);
+            auto padVal = arith::ConstantIndexOp::create(rewriter, loc, pad);
+            Value dpos = arith::SubIOp::create(rewriter, loc, pos, padVal);
 
-            Value offset = rewriter.create<arith::MinSIOp>(loc, dpos, zero);
-            return rewriter.create<arith::AddIOp>(loc, valid, offset)
+            Value offset = arith::MinSIOp::create(rewriter, loc, dpos, zero);
+            return arith::AddIOp::create(rewriter, loc, valid, offset)
                 ->getResult(0);
           };
 
           auto coverageFn = [&](int64_t i, Value isize) -> Value {
             Value strideVal =
-                rewriter.create<arith::ConstantIndexOp>(loc, stride[i - 1]);
+                arith::ConstantIndexOp::create(rewriter, loc, stride[i - 1]);
             Value val =
-                rewriter.create<arith::ConstantIndexOp>(loc, kernel[i - 1]);
+                arith::ConstantIndexOp::create(rewriter, loc, kernel[i - 1]);
 
             // Find the position relative to the input tensor's ends.
-            Value left = rewriter.create<linalg::IndexOp>(loc, i);
-            Value right = rewriter.create<arith::SubIOp>(loc, isize, left);
-            left = rewriter.create<arith::MulIOp>(loc, left, strideVal);
-            right = rewriter.create<arith::MulIOp>(loc, right, strideVal);
+            Value left = linalg::IndexOp::create(rewriter, loc, i);
+            Value right = arith::SubIOp::create(rewriter, loc, isize, left);
+            left = arith::MulIOp::create(rewriter, loc, left, strideVal);
+            right = arith::MulIOp::create(rewriter, loc, right, strideVal);
 
             // Determine how much padding was included.
             val = padFn(val, left, pad[i * 2]);
             val = padFn(val, right, pad[i * 2 + 1]);
-            return rewriter.create<arith::MaxSIOp>(loc, one, val);
+            return arith::MaxSIOp::create(rewriter, loc, one, val);
           };
 
           // Compute the indices from either end.
@@ -981,70 +981,70 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
           Value kW3 = coverageFn(2, iW);
 
           // Compute the total number of elements and normalize.
-          auto count = rewriter.create<arith::IndexCastOp>(
+          auto count = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getI32Type(),
-              rewriter.create<arith::MulIOp>(loc, kH3, kW3));
+              arith::MulIOp::create(rewriter, loc, kH3, kW3));
 
           // Divide by the number of summed values. For floats this is just
           // a div however for quantized values input normalization had
           // to be applied.
           Value poolVal = args[0];
           if (isa<FloatType>(accETy)) {
-            auto countF = rewriter.create<arith::SIToFPOp>(loc, accETy, count);
-            poolVal = rewriter.create<arith::DivFOp>(loc, poolVal, countF)
+            auto countF = arith::SIToFPOp::create(rewriter, loc, accETy, count);
+            poolVal = arith::DivFOp::create(rewriter, loc, poolVal, countF)
                           ->getResult(0);
             if (accETy.getIntOrFloatBitWidth() >
                 resultETy.getIntOrFloatBitWidth())
               poolVal =
-                  rewriter.create<arith::TruncFOp>(loc, resultETy, poolVal);
+                  arith::TruncFOp::create(rewriter, loc, resultETy, poolVal);
           } else {
 
             // If we have quantization information we need to apply an offset
             // for the input zp value.
             if (inputZpVal != 0) {
-              auto inputZp = rewriter.create<arith::ConstantOp>(
+              auto inputZp = arith::ConstantOp::create(rewriter,
                   loc, b.getIntegerAttr(accETy, inputZpVal));
               Value offset =
-                  rewriter.create<arith::MulIOp>(loc, accETy, count, inputZp);
+                  arith::MulIOp::create(rewriter, loc, accETy, count, inputZp);
               poolVal =
-                  rewriter.create<arith::SubIOp>(loc, accETy, poolVal, offset);
+                  arith::SubIOp::create(rewriter, loc, accETy, poolVal, offset);
             }
 
             // Compute: k = 32 - count_leading_zeros(value - 1)
-            Value one32 = rewriter.create<arith::ConstantOp>(
+            Value one32 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI32IntegerAttr(1));
-            Value thirtyTwo32 = rewriter.create<arith::ConstantOp>(
+            Value thirtyTwo32 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI32IntegerAttr(32));
 
             Value countSubOne =
-                rewriter.create<arith::SubIOp>(loc, count, one32);
+                arith::SubIOp::create(rewriter, loc, count, one32);
             Value leadingZeros =
-                rewriter.create<math::CountLeadingZerosOp>(loc, countSubOne);
+                math::CountLeadingZerosOp::create(rewriter, loc, countSubOne);
             Value k =
-                rewriter.create<arith::SubIOp>(loc, thirtyTwo32, leadingZeros);
+                arith::SubIOp::create(rewriter, loc, thirtyTwo32, leadingZeros);
 
             // Compute: numerator = ((1 << 30) + 1) << k
             Value k64 =
-                rewriter.create<arith::ExtUIOp>(loc, rewriter.getI64Type(), k);
-            Value thirtyShiftPlusOne = rewriter.create<arith::ConstantOp>(
+                arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), k);
+            Value thirtyShiftPlusOne = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
             Value numerator =
-                rewriter.create<arith::ShLIOp>(loc, thirtyShiftPlusOne, k64);
+                arith::ShLIOp::create(rewriter, loc, thirtyShiftPlusOne, k64);
 
             // Compute: scale.multiplier = numerator / value;
-            Value count64 = rewriter.create<arith::ExtUIOp>(
+            Value count64 = arith::ExtUIOp::create(rewriter,
                 loc, rewriter.getI64Type(), count);
             Value multiplier =
-                rewriter.create<arith::DivUIOp>(loc, numerator, count64);
-            multiplier = rewriter.create<arith::TruncIOp>(
+                arith::DivUIOp::create(rewriter, loc, numerator, count64);
+            multiplier = arith::TruncIOp::create(rewriter,
                 loc, rewriter.getI32Type(), multiplier);
 
             // Compute: scale.shift = 30 + k
             Value k8 =
-                rewriter.create<arith::TruncIOp>(loc, rewriter.getI8Type(), k);
-            Value thirty8 = rewriter.create<arith::ConstantOp>(
+                arith::TruncIOp::create(rewriter, loc, rewriter.getI8Type(), k);
+            Value thirty8 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI8IntegerAttr(30));
-            Value shift = rewriter.create<arith::AddIOp>(loc, k8, thirty8);
+            Value shift = arith::AddIOp::create(rewriter, loc, k8, thirty8);
 
             auto scaled =
                 rewriter
@@ -1056,19 +1056,19 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // If we have quantization information we need to apply output
             // zeropoint.
             if (outputZpVal != 0) {
-              auto outputZp = rewriter.create<arith::ConstantOp>(
+              auto outputZp = arith::ConstantOp::create(rewriter,
                   loc, b.getIntegerAttr(scaled.getType(), outputZpVal));
-              scaled = rewriter.create<arith::AddIOp>(loc, scaled, outputZp)
+              scaled = arith::AddIOp::create(rewriter, loc, scaled, outputZp)
                            .getResult();
             }
 
             // Apply Clip.
             int64_t outBitwidth = resultETy.getIntOrFloatBitWidth();
 
-            auto min = rewriter.create<arith::ConstantIntOp>(
+            auto min = arith::ConstantIntOp::create(rewriter,
                 loc, accETy,
                 APInt::getSignedMinValue(outBitwidth).getSExtValue());
-            auto max = rewriter.create<arith::ConstantIntOp>(
+            auto max = arith::ConstantIntOp::create(rewriter,
                 loc, accETy,
                 APInt::getSignedMaxValue(outBitwidth).getSExtValue());
             auto clamp = clampIntHelper(loc, scaled, min, max, rewriter,
@@ -1078,11 +1078,11 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // Convert type.
             if (resultETy != clamp.getType()) {
               poolVal =
-                  rewriter.create<arith::TruncIOp>(loc, resultETy, poolVal);
+                  arith::TruncIOp::create(rewriter, loc, resultETy, poolVal);
             }
           }
 
-          rewriter.create<linalg::YieldOp>(loc, poolVal);
+          linalg::YieldOp::create(rewriter, loc, poolVal);
         });
 
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -1107,7 +1107,7 @@ class TransposeConverter : public OpRewritePattern<tosa::TransposeOp> {
     auto permutedSizes =
         applyTOSAPermutation<OpFoldResult>(inputSizes, constantPerms);
 
-    auto permutedInit = rewriter.create<tensor::EmptyOp>(
+    auto permutedInit = tensor::EmptyOp::create(rewriter,
         loc, permutedSizes, op.getInput1().getType().getElementType());
     rewriter.replaceOpWithNewOp<linalg::TransposeOp>(
         op, op.getInput1(), permutedInit,
diff --git a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
index 7dbccd19a0518..cd36aadce410a 100644
--- a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
+++ b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
@@ -27,7 +27,7 @@ class VariableOpConverter : public OpRewritePattern<tosa::VariableOp> {
   LogicalResult matchAndRewrite(tosa::VariableOp op,
                                 PatternRewriter &rewriter) const final {
     auto variableType = tosa::getVariableType(op);
-    auto newVariable = rewriter.create<mlir::ml_program::GlobalOp>(
+    auto newVariable = mlir::ml_program::GlobalOp::create(rewriter,
         op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
         op.getInitialValueAttr(), /*sym_visibility=*/nullptr);
     newVariable.setPrivate();
@@ -45,7 +45,7 @@ class VariableWriteOpConverter
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableWrite = rewriter.create<ml_program::GlobalStoreOp>(
+    auto newVariableWrite = ml_program::GlobalStoreOp::create(rewriter,
         op.getLoc(), globalSymbolRef, op.getInput1());
     rewriter.replaceOp(op, newVariableWrite);
     return success();
@@ -60,7 +60,7 @@ class VariableReadOpConverter : public OpRewritePattern<tosa::VariableReadOp> {
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableRead = rewriter.create<ml_program::GlobalLoadOp>(
+    auto newVariableRead = ml_program::GlobalLoadOp::create(rewriter,
         op.getLoc(), op.getType(), globalSymbolRef);
     rewriter.replaceOp(op, newVariableRead);
 
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index 03f9d20ad69de..9188b8d1368da 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -30,7 +30,7 @@ static void inlineIfCase(Region &srcRegion, Region &dstRegion,
 
   auto yield = cast<YieldOp>(headBlock->getTerminator());
   rewriter.setInsertionPoint(yield);
-  rewriter.create<scf::YieldOp>(yield.getLoc(), yield.getInputs());
+  scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
   rewriter.eraseOp(yield);
 
   headBlock->eraseArguments(0, headBlock->getNumArguments());
@@ -47,12 +47,12 @@ static void inlineWhileCase(Region &srcRegion, Region &dstRegion,
   rewriter.setInsertionPoint(yield);
   if (isCond) {
     auto condition =
-        rewriter.create<tensor::ExtractOp>(yield.getLoc(), yield.getOperand(0));
-    rewriter.create<scf::ConditionOp>(yield.getLoc(), condition,
+        tensor::ExtractOp::create(rewriter, yield.getLoc(), yield.getOperand(0));
+    scf::ConditionOp::create(rewriter, yield.getLoc(), condition,
                                       headBlock->getArguments());
   } else {
     rewriter.setInsertionPoint(yield);
-    rewriter.create<scf::YieldOp>(yield.getLoc(), yield.getInputs());
+    scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
   }
   rewriter.eraseOp(yield);
 }
@@ -66,8 +66,8 @@ class IfOpConverter : public OpRewritePattern<tosa::IfOp> {
   LogicalResult matchAndRewrite(tosa::IfOp op,
                                 PatternRewriter &rewriter) const final {
     auto condition =
-        rewriter.create<tensor::ExtractOp>(op.getLoc(), op.getCondition());
-    auto newIf = rewriter.create<scf::IfOp>(op.getLoc(), op.getResultTypes(),
+        tensor::ExtractOp::create(rewriter, op.getLoc(), op.getCondition());
+    auto newIf = scf::IfOp::create(rewriter, op.getLoc(), op.getResultTypes(),
                                             condition, true);
 
     inlineIfCase(op.getThenGraph(), newIf.getThenRegion(), op.getInputList(),
@@ -88,7 +88,7 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
 
   static Value createIndexConst(OpBuilder &builder, Location loc,
                                 int64_t value) {
-    return builder.create<arith::ConstantIndexOp>(loc, value);
+    return arith::ConstantIndexOp::create(builder, loc, value);
   }
 
 public:
@@ -119,8 +119,8 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       auto n = ivs[0];
 
       // Read the index and cast it to index type
-      auto index = builder.create<tensor::ExtractOp>(loc, indices, ivs);
-      auto castIndex = builder.create<arith::IndexCastOp>(
+      auto index = tensor::ExtractOp::create(builder, loc, indices, ivs);
+      auto castIndex = arith::IndexCastOp::create(builder,
           loc, builder.getIndexType(), index);
 
       // Offset, sizes, and strides for the input tensor
@@ -130,12 +130,12 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       llvm::SmallVector<Value> sizes = {one, one, dimC};
       llvm::SmallVector<Value> strides = {one, one, one};
 
-      auto slice = builder.create<tensor::ExtractSliceOp>(
+      auto slice = tensor::ExtractSliceOp::create(builder,
           loc, input, inputOffset, sizes, strides);
 
       // Insert the slice into the output accumulator tensor.
       llvm::SmallVector<Value> outputOffset = {n, castIndex, zero};
-      auto updated = builder.create<tensor::InsertSliceOp>(
+      auto updated = tensor::InsertSliceOp::create(builder,
           loc, slice, args[0], outputOffset, sizes, strides);
 
       return {updated};
@@ -155,7 +155,7 @@ class WhileOpConverter : public OpRewritePattern<tosa::WhileOp> {
 
   LogicalResult matchAndRewrite(tosa::WhileOp op,
                                 PatternRewriter &rewriter) const final {
-    auto newWhile = rewriter.create<scf::WhileOp>(
+    auto newWhile = scf::WhileOp::create(rewriter,
         op.getLoc(), op.getResultTypes(), op.getInputList());
     rewriter.createBlock(&newWhile.getBefore());
     rewriter.createBlock(&newWhile.getAfter());
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index c6cbcb0f8ab2b..a7219d423a77a 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -308,13 +308,13 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
       if (ShapedType::isStatic(sizes.back()))
         continue;
 
-      auto dim = rewriter.create<tensor::DimOp>(loc, input, index);
-      auto offset = rewriter.create<arith::ConstantOp>(
+      auto dim = tensor::DimOp::create(rewriter, loc, input, index);
+      auto offset = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(sliceStarts[index]));
-      dynSizes.push_back(rewriter.create<arith::SubIOp>(loc, dim, offset));
+      dynSizes.push_back(arith::SubIOp::create(rewriter, loc, dim, offset));
     }
 
-    auto newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}), dynSizes,
         ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
         rewriter.getDenseI64ArrayAttr(sizes),
@@ -361,7 +361,7 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
 
     Value padConstant = rewriter.createOrFold<tensor::ExtractOp>(
         loc, padOp.getPadConst(),
-        ValueRange({rewriter.create<arith::ConstantIndexOp>(loc, 0)}));
+        ValueRange({arith::ConstantIndexOp::create(rewriter, loc, 0)}));
 
     if (!padConstant) {
       return rewriter.notifyMatchFailure(
@@ -375,15 +375,15 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
     highValues.reserve(rank);
 
     for (int i = 0; i < rank; i++) {
-      Value lowVal = rewriter.create<arith::ConstantOp>(
+      Value lowVal = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(paddingVals[2 * i]));
-      Value highVal = rewriter.create<arith::ConstantOp>(
+      Value highVal = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(paddingVals[2 * i + 1]));
       lowValues.push_back(lowVal);
       highValues.push_back(highVal);
     }
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, padOp.getType(), input, lowValues, highValues, padConstant);
 
     rewriter.replaceOp(padOp, newPadOp.getResult());
@@ -402,7 +402,7 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
     Location loc = op.getLoc();
     int axis = op.getAxis();
     Value axisValue =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(axis));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(axis));
     int64_t rank = resultType.getRank();
 
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
@@ -439,7 +439,7 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
       }
     }
 
-    Value result = rewriter.create<tensor::EmptyOp>(
+    Value result = tensor::EmptyOp::create(rewriter,
         loc, resultType.getShape(), resultType.getElementType(), dynDims);
 
     for (auto [arg, offset] : llvm::zip(adaptor.getOperands(), axisOffsets)) {
diff --git a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
index d6f9495b2567c..9e4eb347b3689 100644
--- a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
+++ b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
@@ -226,7 +226,7 @@ struct BroadcastOpToArmSMELowering
         (srcVectorType && (srcVectorType.getRank() == 0))) {
       // Broadcast scalar or 0-d vector to 1-d vector.
       VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-      broadcastOp1D = rewriter.create<vector::BroadcastOp>(
+      broadcastOp1D = vector::BroadcastOp::create(rewriter,
           loc, tileSliceType, broadcastOp.getSource());
     } else if (srcVectorType && (srcVectorType.getRank() == 1))
       // Value to broadcast is already a 1-d vector, nothing to do.
@@ -234,13 +234,13 @@ struct BroadcastOpToArmSMELowering
     else
       return failure();
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to broadcast the value
       // to each tile slice.
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
@@ -292,14 +292,14 @@ struct SplatOpToArmSMELowering : public OpRewritePattern<vector::SplatOp> {
 
     // First, broadcast the scalar to a 1-d vector.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    Value broadcastOp1D = rewriter.create<vector::BroadcastOp>(
+    Value broadcastOp1D = vector::BroadcastOp::create(rewriter,
         loc, tileSliceType, splatOp.getInput());
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
@@ -370,21 +370,21 @@ struct TransposeOpToArmSMELowering
 
     // Allocate buffer to store input tile to.
     Value vscale =
-        rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-    Value minTileSlices = rewriter.create<arith::ConstantOp>(
+        vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+    Value minTileSlices = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexAttr(tileType.getDimSize(0)));
     Value c0 =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(0));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(0));
     Value numTileSlices =
-        rewriter.create<arith::MulIOp>(loc, vscale, minTileSlices);
+        arith::MulIOp::create(rewriter, loc, vscale, minTileSlices);
     auto bufferType =
         MemRefType::get({ShapedType::kDynamic, ShapedType::kDynamic},
                         tileType.getElementType());
-    auto buffer = rewriter.create<memref::AllocaOp>(
+    auto buffer = memref::AllocaOp::create(rewriter,
         loc, bufferType, ValueRange{numTileSlices, numTileSlices});
 
     // Store input tile.
-    auto tileStoreOp = rewriter.create<arm_sme::TileStoreOp>(
+    auto tileStoreOp = arm_sme::TileStoreOp::create(rewriter,
         loc, input, buffer, ValueRange{c0, c0});
 
     // Reload input tile vertically.
@@ -489,9 +489,9 @@ struct VectorOuterProductToArmSMELowering
 
     VectorType operandMaskType = VectorType::Builder(maskType).dropDim(0);
     Value lhsMask =
-        rewriter.create<vector::CreateMaskOp>(loc, operandMaskType, lhsMaskDim);
+        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, lhsMaskDim);
     Value rhsMask =
-        rewriter.create<vector::CreateMaskOp>(loc, operandMaskType, rhsMaskDim);
+        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, rhsMaskDim);
 
     return std::make_pair(lhsMask, rhsMask);
   }
@@ -531,7 +531,7 @@ struct VectorExtractToArmSMELowering
     }
 
     Value sliceIndex = vector::getAsValues(rewriter, loc, position[0]).front();
-    auto extractTileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+    auto extractTileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
         loc, sourceVector, sliceIndex);
 
     if (position.size() == 1) {
@@ -593,9 +593,9 @@ struct VectorInsertToArmSMELowering
     if (position.size() == 2) {
       // Two indices case: Insert single element into tile.
       // We need to first extract the existing slice and update the element.
-      tileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+      tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
           loc, insertOp.getDest(), sliceIndex);
-      tileSlice = rewriter.create<vector::InsertOp>(loc, source, tileSlice,
+      tileSlice = vector::InsertOp::create(rewriter, loc, source, tileSlice,
                                                     position[1]);
     }
 
@@ -642,22 +642,22 @@ struct VectorPrintToArmSMELowering : public OpRewritePattern<vector::PrintOp> {
     auto loc = printOp.getLoc();
 
     // Create a loop over the rows of the tile.
-    auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
+    auto vscale = vector::VectorScaleOp::create(rewriter, loc);
     auto minTileRows =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorType.getDimSize(0));
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto upperBound = rewriter.create<arith::MulIOp>(loc, minTileRows, vscale);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        arith::ConstantIndexOp::create(rewriter, loc, vectorType.getDimSize(0));
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto upperBound = arith::MulIOp::create(rewriter, loc, minTileRows, vscale);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     {
       // Loop body.
       rewriter.setInsertionPointToStart(forOp.getBody());
       // Extract the current row from the tile.
       Value rowIndex = forOp.getInductionVar();
-      auto tileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+      auto tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
           loc, printOp.getSource(), rowIndex);
       // Print the row with a 1D vector.print.
-      rewriter.create<vector::PrintOp>(loc, tileSlice,
+      vector::PrintOp::create(rewriter, loc, tileSlice,
                                        printOp.getPunctuation());
     }
 
@@ -707,7 +707,7 @@ struct FoldTransferWriteOfExtractTileSlice
     Value mask = writeOp.getMask();
     if (!mask) {
       auto maskType = writeOp.getVectorType().clone(rewriter.getI1Type());
-      mask = rewriter.create<arith::ConstantOp>(
+      mask = arith::ConstantOp::create(rewriter,
           writeOp.getLoc(), maskType, DenseElementsAttr::get(maskType, true));
     }
 
@@ -776,9 +776,9 @@ struct ExtractFromCreateMaskToPselLowering
     // Create the two 1-D masks at the location of the 2-D create_mask (which is
     // usually outside a loop). This prevents the need for later hoisting.
     rewriter.setInsertionPoint(createMaskOp);
-    auto rowMask = rewriter.create<vector::CreateMaskOp>(
+    auto rowMask = vector::CreateMaskOp::create(rewriter,
         loc, rowMaskType, createMaskOp.getOperand(0));
-    auto colMask = rewriter.create<vector::CreateMaskOp>(
+    auto colMask = vector::CreateMaskOp::create(rewriter,
         loc, colMaskType, createMaskOp.getOperand(1));
 
     rewriter.setInsertionPoint(extractOp);
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 18adaa793787c..5b3e67509672a 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -412,22 +412,22 @@ struct PrepareContractToGPUMMA
     if (maps == infer({{m, k}, {k, n}, {m, n}}))
       return rewriter.notifyMatchFailure(op, "contraction already prepared");
     if (maps == infer({{m, k}, {n, k}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
       std::swap(rhs, lhs);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
       std::swap(rhs, lhs);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
       std::swap(lhs, rhs);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
       std::swap(lhs, rhs);
     } else {
@@ -494,13 +494,13 @@ struct CombineTransferReadOpTranspose final
     // Fuse through the integer extend op.
     if (extOp) {
       if (isa<arith::ExtSIOp>(extOp))
-        result = rewriter.create<arith::ExtSIOp>(loc, op.getType(), result)
+        result = arith::ExtSIOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
       else if (isa<arith::ExtUIOp>(extOp))
-        result = rewriter.create<arith::ExtUIOp>(loc, op.getType(), result)
+        result = arith::ExtUIOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
       else
-        result = rewriter.create<arith::ExtFOp>(loc, op.getType(), result)
+        result = arith::ExtFOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
     }
 
@@ -579,7 +579,7 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op,
   }
   gpu::MMAMatrixType type =
       gpu::MMAMatrixType::get(op.getVectorType().getShape(), elType, fragType);
-  Value load = rewriter.create<gpu::SubgroupMmaLoadMatrixOp>(
+  Value load = gpu::SubgroupMmaLoadMatrixOp::create(rewriter,
       op.getLoc(), type, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride),
       isTranspose ? rewriter.getUnitAttr() : UnitAttr());
@@ -610,7 +610,7 @@ convertTransferWriteOp(RewriterBase &rewriter, vector::TransferWriteOp op,
   }
 
   Value matrix = it->second;
-  auto store = rewriter.create<gpu::SubgroupMmaStoreMatrixOp>(
+  auto store = gpu::SubgroupMmaStoreMatrixOp::create(rewriter,
       op.getLoc(), matrix, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride), /*transpose=*/UnitAttr());
   (void)store;
@@ -661,7 +661,7 @@ convertConstantOpMmaSync(RewriterBase &rewriter, arith::ConstantOp op,
     return rewriter.notifyMatchFailure(op, "not a splat");
   }
 
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       op.getLoc(), vectorType,
       DenseElementsAttr::get(vectorType, dense.getSplatValue<Attribute>()));
   valueMapping[op.getResult()] = result;
@@ -743,7 +743,7 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   }
 
   // Adjust the load offset.
-  auto laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  auto laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
   FailureOr<AffineMap> offsets =
       nvgpu::getLaneIdToLdMatrixMatrixCoord(rewriter, loc, *params);
   if (failed(offsets)) {
@@ -757,7 +757,7 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   getXferIndices<vector::TransferReadOp>(rewriter, op, *offsets, {laneId},
                                          indices);
 
-  nvgpu::LdMatrixOp newOp = rewriter.create<nvgpu::LdMatrixOp>(
+  nvgpu::LdMatrixOp newOp = nvgpu::LdMatrixOp::create(rewriter,
       loc, vectorType, op.getBase(), indices, *transpose, params->numTiles);
   valueMapping[op] = newOp->getResult(0);
   return success();
@@ -782,17 +782,17 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
             "conversion to distributed non-ldmatrix compatible load");
   }
 
-  Value laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   // This is the individual element type.
   Type loadedElType = regInfo->registerLLVMType;
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
 
-  Value fill = rewriter.create<arith::ConstantOp>(
+  Value fill = arith::ConstantOp::create(rewriter,
       op.getLoc(), vectorType.getElementType(),
       rewriter.getZeroAttr(vectorType.getElementType()));
   Value result =
-      rewriter.create<vector::SplatOp>(op.getLoc(), fill, vectorType);
+      vector::SplatOp::create(rewriter, op.getLoc(), fill, vectorType);
 
   bool isTransposeLoad = !op.getPermutationMap().isMinorIdentity();
 
@@ -809,16 +809,16 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       if (failed(coords))
         return rewriter.notifyMatchFailure(op, "no coords");
 
-      Value logicalValueId = rewriter.create<arith::ConstantOp>(
+      Value logicalValueId = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexType(),
           rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
       SmallVector<Value, 4> newIndices;
       getXferIndices<vector::TransferReadOp>(
           rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
 
-      Value el = rewriter.create<vector::LoadOp>(loc, loadedElType,
+      Value el = vector::LoadOp::create(rewriter, loc, loadedElType,
                                                  op.getBase(), newIndices);
-      result = rewriter.create<vector::InsertOp>(loc, el, result, i);
+      result = vector::InsertOp::create(rewriter, loc, el, result, i);
     }
   } else {
     if (auto vecType = dyn_cast<VectorType>(loadedElType)) {
@@ -828,7 +828,7 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       for (unsigned innerIdx = 0; innerIdx < vectorType.getShape()[1];
            innerIdx++) {
 
-        Value logicalValueId = rewriter.create<arith::ConstantOp>(
+        Value logicalValueId = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIndexType(),
             rewriter.getIndexAttr(i * regInfo->elementsPerRegister + innerIdx));
         FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
@@ -839,9 +839,9 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
         SmallVector<Value, 4> newIndices;
         getXferIndices<vector::TransferReadOp>(
             rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
-        Value el = rewriter.create<memref::LoadOp>(op.getLoc(), loadedElType,
+        Value el = memref::LoadOp::create(rewriter, op.getLoc(), loadedElType,
                                                    op.getBase(), newIndices);
-        result = rewriter.create<vector::InsertOp>(
+        result = vector::InsertOp::create(rewriter,
             op.getLoc(), el, result, ArrayRef<int64_t>{i, innerIdx});
       }
     }
@@ -916,10 +916,10 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
     return rewriter.notifyMatchFailure(op, "not mma sync reg info");
 
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
-  Value laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   for (unsigned i = 0; i < vectorType.getShape()[0]; i++) {
-    Value logicalValueId = rewriter.create<arith::ConstantOp>(
+    Value logicalValueId = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexType(),
         rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
     FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
@@ -928,11 +928,11 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
       return rewriter.notifyMatchFailure(op, "no coords");
 
     Value el =
-        rewriter.create<vector::ExtractOp>(loc, matrix, ArrayRef<int64_t>{i});
+        vector::ExtractOp::create(rewriter, loc, matrix, ArrayRef<int64_t>{i});
     SmallVector<Value, 4> newIndices;
     getXferIndices<vector::TransferWriteOp>(
         rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
-    rewriter.create<vector::StoreOp>(loc, el, op.getBase(), newIndices);
+    vector::StoreOp::create(rewriter, loc, el, op.getBase(), newIndices);
   }
 
   LLVM_DEBUG(DBGS() << "erase: " << op << "\n");
@@ -1015,7 +1015,7 @@ convertExtractStridedSlice(RewriterBase &rewriter,
   else if (offsets[1])
     sliceOffset[0] = (warpVectorShape[1] / offsets[1]);
 
-  Value newOp = rewriter.create<vector::ExtractStridedSliceOp>(
+  Value newOp = vector::ExtractStridedSliceOp::create(rewriter,
       loc, sourceVector, sliceOffset, sliceShape, strides);
 
   valueMapping[op] = newOp;
@@ -1035,7 +1035,7 @@ convertContractOp(RewriterBase &rewriter, vector::ContractionOp op,
       itC == valueMapping.end())
     return rewriter.notifyMatchFailure(op, "no mapping");
   Value opA = itA->second, opB = itB->second, opC = itC->second;
-  Value matmul = rewriter.create<gpu::SubgroupMmaComputeOp>(
+  Value matmul = gpu::SubgroupMmaComputeOp::create(rewriter,
       op.getLoc(), opC.getType(), opA, opB, opC, /*a_transpose=*/UnitAttr(),
       /*b_transpose=*/UnitAttr());
   valueMapping[op.getResult()] = matmul;
@@ -1058,7 +1058,7 @@ convertContractOpToMmaSync(RewriterBase &rewriter, vector::ContractionOp op,
   int64_t m = cast<VectorType>(op.getLhs().getType()).getShape()[0];
   int64_t n = cast<VectorType>(op.getRhs().getType()).getShape()[0];
   int64_t k = cast<VectorType>(op.getLhs().getType()).getShape()[1];
-  Value matmul = rewriter.create<nvgpu::MmaSyncOp>(
+  Value matmul = nvgpu::MmaSyncOp::create(rewriter,
       op.getLoc(), opA, opB, opC, rewriter.getI64ArrayAttr({m, n, k}));
   valueMapping[op.getResult()] = matmul;
   return success();
@@ -1076,12 +1076,12 @@ convertConstantOp(RewriterBase &rewriter, arith::ConstantOp op,
   auto splat =
       cast<SplatElementsAttr>(op.getValue()).getSplatValue<TypedAttr>();
   auto scalarConstant =
-      rewriter.create<arith::ConstantOp>(op.getLoc(), splat.getType(), splat);
+      arith::ConstantOp::create(rewriter, op.getLoc(), splat.getType(), splat);
   const char *fragType = inferFragType(op);
   auto vecType = cast<VectorType>(op.getType());
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = rewriter.create<gpu::SubgroupMmaConstantMatrixOp>(
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
       op.getLoc(), type, scalarConstant);
   valueMapping[op.getResult()] = matrix;
   return success();
@@ -1100,7 +1100,7 @@ convertBroadcastOp(RewriterBase &rewriter, vector::BroadcastOp op,
   auto vecType = op.getResultVectorType();
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = rewriter.create<gpu::SubgroupMmaConstantMatrixOp>(
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
       op.getLoc(), type, op.getSource());
   valueMapping[op.getResult()] = matrix;
   return success();
@@ -1118,7 +1118,7 @@ static scf::ForOp replaceForOpWithNewSignature(RewriterBase &rewriter,
   rewriter.setInsertionPoint(loop);
   auto operands = llvm::to_vector<4>(loop.getInitArgs());
   llvm::append_range(operands, newInitArgs);
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
       operands);
   rewriter.eraseBlock(newLoop.getBody());
@@ -1189,7 +1189,7 @@ convertYieldOp(RewriterBase &rewriter, scf::YieldOp op,
     yieldOperands[operand.index()] = loop.getInitArgs()[operand.index()];
     yieldOperands.push_back(it->second);
   }
-  rewriter.create<scf::YieldOp>(op.getLoc(), yieldOperands);
+  scf::YieldOp::create(rewriter, op.getLoc(), yieldOperands);
 
   LLVM_DEBUG(DBGS() << "erase: " << op << "\n");
   rewriter.eraseOp(op);
@@ -1220,7 +1220,7 @@ convertElementwiseOp(RewriterBase &rewriter, Operation *op,
                                          resultType.getOperand());
   }
 
-  Value newOp = rewriter.create<gpu::SubgroupMmaElementwiseOp>(
+  Value newOp = gpu::SubgroupMmaElementwiseOp::create(rewriter,
       op->getLoc(), resultType, matrixOperands, opType);
   valueMapping[op->getResult(0)] = newOp;
   return success();
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 501d98862672d..1b31d391cb0bc 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -43,13 +43,13 @@ static Value insertOne(ConversionPatternRewriter &rewriter,
   assert(rank > 0 && "0-D vector corner case should have been handled already");
   if (rank == 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = rewriter.create<LLVM::ConstantOp>(
+    auto constant = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
-    return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2,
+    return LLVM::InsertElementOp::create(rewriter, loc, llvmType, val1, val2,
                                                   constant);
   }
-  return rewriter.create<LLVM::InsertValueOp>(loc, val1, val2, pos);
+  return LLVM::InsertValueOp::create(rewriter, loc, val1, val2, pos);
 }
 
 // Helper that picks the proper sequence for extracting.
@@ -58,13 +58,13 @@ static Value extractOne(ConversionPatternRewriter &rewriter,
                         Value val, Type llvmType, int64_t rank, int64_t pos) {
   if (rank <= 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = rewriter.create<LLVM::ConstantOp>(
+    auto constant = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
-    return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val,
+    return LLVM::ExtractElementOp::create(rewriter, loc, llvmType, val,
                                                    constant);
   }
-  return rewriter.create<LLVM::ExtractValueOp>(loc, val, pos);
+  return LLVM::ExtractValueOp::create(rewriter, loc, val, pos);
 }
 
 // Helper that returns data layout alignment of a vector.
@@ -141,7 +141,7 @@ static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc,
   auto ptrsType =
       LLVM::getVectorType(pType, vectorType.getDimSize(0),
                           /*isScalable=*/vectorType.getScalableDims()[0]);
-  return rewriter.create<LLVM::GEPOp>(
+  return LLVM::GEPOp::create(rewriter,
       loc, ptrsType, typeConverter.convertType(memRefType.getElementType()),
       base, index);
 }
@@ -152,7 +152,7 @@ static Value getAsLLVMValue(OpBuilder &builder, Location loc,
                             OpFoldResult foldResult) {
   if (auto attr = dyn_cast<Attribute>(foldResult)) {
     auto intAttr = cast<IntegerAttr>(attr);
-    return builder.create<LLVM::ConstantOp>(loc, intAttr).getResult();
+    return LLVM::ConstantOp::create(builder, loc, intAttr).getResult();
   }
 
   return cast<Value>(foldResult);
@@ -475,7 +475,7 @@ class ReductionNeutralFPMax {};
 static Value createReductionNeutralValue(ReductionNeutralZero neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmType,
+  return LLVM::ConstantOp::create(rewriter, loc, llvmType,
                                            rewriter.getZeroAttr(llvmType));
 }
 
@@ -483,7 +483,7 @@ static Value createReductionNeutralValue(ReductionNeutralZero neutral,
 static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType, rewriter.getIntegerAttr(llvmType, 1));
 }
 
@@ -491,7 +491,7 @@ static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
 static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
 }
 
@@ -499,7 +499,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
 static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(
           llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth())));
@@ -509,7 +509,7 @@ static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -519,7 +519,7 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -529,7 +529,7 @@ static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -539,7 +539,7 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -550,7 +550,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
@@ -562,7 +562,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
@@ -591,7 +591,7 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
   auto vShape = vType.getShape();
   assert(vShape.size() == 1 && "Unexpected multi-dim vector type");
 
-  Value baseVecLength = rewriter.create<LLVM::ConstantOp>(
+  Value baseVecLength = LLVM::ConstantOp::create(rewriter,
       loc, rewriter.getI32Type(),
       rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0]));
 
@@ -599,11 +599,11 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
     return baseVecLength;
 
   // For a scalable vector type, create and return `vScale * baseVecLength`.
-  Value vScale = rewriter.create<vector::VectorScaleOp>(loc);
+  Value vScale = vector::VectorScaleOp::create(rewriter, loc);
   vScale =
-      rewriter.create<arith::IndexCastOp>(loc, rewriter.getI32Type(), vScale);
+      arith::IndexCastOp::create(rewriter, loc, rewriter.getI32Type(), vScale);
   Value scalableVecLength =
-      rewriter.create<arith::MulIOp>(loc, baseVecLength, vScale);
+      arith::MulIOp::create(rewriter, loc, baseVecLength, vScale);
   return scalableVecLength;
 }
 
@@ -616,10 +616,10 @@ static Value createIntegerReductionArithmeticOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator) {
 
-  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
 
   if (accumulator)
-    result = rewriter.create<ScalarOp>(loc, accumulator, result);
+    result = ScalarOp::create(rewriter, loc, accumulator, result);
   return result;
 }
 
@@ -631,11 +631,11 @@ template <class LLVMRedIntrinOp>
 static Value createIntegerReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) {
-  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
   if (accumulator) {
     Value cmp =
-        rewriter.create<LLVM::ICmpOp>(loc, predicate, accumulator, result);
-    result = rewriter.create<LLVM::SelectOp>(loc, cmp, accumulator, result);
+        LLVM::ICmpOp::create(rewriter, loc, predicate, accumulator, result);
+    result = LLVM::SelectOp::create(rewriter, loc, cmp, accumulator, result);
   }
   return result;
 }
@@ -666,11 +666,11 @@ static Value createFPReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::FastmathFlagsAttr fmf) {
   Value result =
-      rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand, fmf);
+      LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand, fmf);
 
   if (accumulator) {
     result =
-        rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
+        typename VectorToScalarMapper<LLVMRedIntrinOp>::Type::create(rewriter,
             loc, result, accumulator);
   }
 
@@ -702,7 +702,7 @@ static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
   const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
   auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
   auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
-  return rewriter.create<LLVM::ConstantOp>(loc, vectorType, denseValue);
+  return LLVM::ConstantOp::create(rewriter, loc, vectorType, denseValue);
 }
 
 /// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
@@ -717,7 +717,7 @@ lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
                                 Value mask, LLVM::FastmathFlagsAttr fmf) {
   const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
       rewriter, loc, llvmType, vectorOperand.getType());
-  const Value selectedVectorByMask = rewriter.create<LLVM::SelectOp>(
+  const Value selectedVectorByMask = LLVM::SelectOp::create(rewriter,
       loc, mask, vectorOperand, vectorMaskNeutral);
   return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
       rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
@@ -730,7 +730,7 @@ lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
                              Value accumulator, LLVM::FastmathFlagsAttr fmf) {
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
-  return rewriter.create<LLVMRedIntrinOp>(loc, llvmType,
+  return LLVMRedIntrinOp::create(rewriter, loc, llvmType,
                                           /*startValue=*/accumulator,
                                           vectorOperand, fmf);
 }
@@ -745,7 +745,7 @@ lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter,
                                        Value vectorOperand, Value accumulator) {
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
-  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
+  return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
                                             /*startValue=*/accumulator,
                                             vectorOperand);
 }
@@ -758,7 +758,7 @@ static Value lowerPredicatedReductionWithStartValue(
                                                          llvmType, accumulator);
   Value vectorLength =
       createVectorLengthValue(rewriter, loc, vectorOperand.getType());
-  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
+  return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
                                             /*startValue=*/accumulator,
                                             vectorOperand, mask, vectorLength);
 }
@@ -1071,7 +1071,7 @@ class VectorShuffleOpConversion
     // For rank 0 and 1, where both operands have *exactly* the same vector
     // type, there is direct shuffle support in LLVM. Use it!
     if (rank <= 1 && v1Type == v2Type) {
-      Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>(
+      Value llvmShuffleOp = LLVM::ShuffleVectorOp::create(rewriter,
           loc, adaptor.getV1(), adaptor.getV2(),
           llvm::to_vector_of<int32_t>(mask));
       rewriter.replaceOp(shuffleOp, llvmShuffleOp);
@@ -1085,7 +1085,7 @@ class VectorShuffleOpConversion
       eltType = arrayType.getElementType();
     else
       eltType = cast<VectorType>(llvmType).getElementType();
-    Value insert = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
+    Value insert = LLVM::PoisonOp::create(rewriter, loc, llvmType);
     int64_t insPos = 0;
     for (int64_t extPos : mask) {
       Value value = adaptor.getV1();
@@ -1122,7 +1122,7 @@ class VectorExtractElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = extractEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = rewriter.create<LLVM::ConstantOp>(
+      auto zero = LLVM::ConstantOp::create(rewriter,
           loc, typeConverter->convertType(idxType),
           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
@@ -1193,12 +1193,12 @@ class VectorExtractOpConversion
       if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) {
         return failure();
       }
-      extracted = rewriter.create<LLVM::ExtractValueOp>(
+      extracted = LLVM::ExtractValueOp::create(rewriter,
           loc, extracted, getAsIntegers(position));
     }
 
     if (extractsScalar) {
-      extracted = rewriter.create<LLVM::ExtractElementOp>(
+      extracted = LLVM::ExtractElementOp::create(rewriter,
           loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back()));
     }
 
@@ -1256,7 +1256,7 @@ class VectorInsertElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = insertEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = rewriter.create<LLVM::ConstantOp>(
+      auto zero = LLVM::ConstantOp::create(rewriter,
           loc, typeConverter->convertType(idxType),
           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
@@ -1342,7 +1342,7 @@ class VectorInsertOpConversion
           // llvm.extractvalue does not support dynamic dimensions.
           return failure();
         }
-        sourceAggregate = rewriter.create<LLVM::ExtractValueOp>(
+        sourceAggregate = LLVM::ExtractValueOp::create(rewriter,
             loc, adaptor.getDest(),
             getAsIntegers(positionOf1DVectorWithinAggregate));
       } else {
@@ -1351,7 +1351,7 @@ class VectorInsertOpConversion
         sourceAggregate = adaptor.getDest();
       }
       // Insert the scalar into the 1D vector.
-      sourceAggregate = rewriter.create<LLVM::InsertElementOp>(
+      sourceAggregate = LLVM::InsertElementOp::create(rewriter,
           loc, sourceAggregate.getType(), sourceAggregate,
           adaptor.getValueToStore(),
           getAsLLVMValue(rewriter, loc, positionOfScalarWithin1DVector));
@@ -1359,7 +1359,7 @@ class VectorInsertOpConversion
 
     Value result = sourceAggregate;
     if (isNestedAggregate) {
-      result = rewriter.create<LLVM::InsertValueOp>(
+      result = LLVM::InsertValueOp::create(rewriter,
           loc, adaptor.getDest(), sourceAggregate,
           getAsIntegers(positionOf1DVectorWithinAggregate));
     }
@@ -1439,15 +1439,15 @@ class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
 
     auto loc = op.getLoc();
     auto elemType = vType.getElementType();
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero);
+    Value desc = vector::SplatOp::create(rewriter, loc, vType, zero);
     for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
-      Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i);
-      Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i);
-      Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i);
-      Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC);
-      desc = rewriter.create<InsertOp>(loc, fma, desc, i);
+      Value extrLHS = ExtractOp::create(rewriter, loc, op.getLhs(), i);
+      Value extrRHS = ExtractOp::create(rewriter, loc, op.getRhs(), i);
+      Value extrACC = ExtractOp::create(rewriter, loc, op.getAcc(), i);
+      Value fma = FMAOp::create(rewriter, loc, extrLHS, extrRHS, extrACC);
+      desc = InsertOp::create(rewriter, loc, fma, desc, i);
     }
     rewriter.replaceOp(op, desc);
     return success();
@@ -1537,7 +1537,7 @@ class VectorTypeCastOpConversion
     desc.setAlignedPtr(rewriter, loc, ptr);
     // Fill offset 0.
     auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0);
-    auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr);
+    auto zero = LLVM::ConstantOp::create(rewriter, loc, int64Ty, attr);
     desc.setOffset(rewriter, loc, zero);
 
     // Fill size and stride descriptors in memref.
@@ -1546,11 +1546,11 @@ class VectorTypeCastOpConversion
       int64_t index = indexedSize.index();
       auto sizeAttr =
           rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value());
-      auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr);
+      auto size = LLVM::ConstantOp::create(rewriter, loc, int64Ty, sizeAttr);
       desc.setSize(rewriter, loc, index, size);
       auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(),
                                                 (*targetStrides)[index]);
-      auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr);
+      auto stride = LLVM::ConstantOp::create(rewriter, loc, int64Ty, strideAttr);
       desc.setStride(rewriter, loc, index, stride);
     }
 
@@ -1578,13 +1578,13 @@ class VectorCreateMaskOpConversion
     IntegerType idxType =
         force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
     auto loc = op->getLoc();
-    Value indices = rewriter.create<LLVM::StepVectorOp>(
+    Value indices = LLVM::StepVectorOp::create(rewriter,
         loc, LLVM::getVectorType(idxType, dstType.getShape()[0],
                                  /*isScalable=*/true));
     auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType,
                                                  adaptor.getOperands()[0]);
-    Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound);
-    Value comp = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+    Value bounds = SplatOp::create(rewriter, loc, indices.getType(), bound);
+    Value comp = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                 indices, bounds);
     rewriter.replaceOp(op, comp);
     return success();
@@ -1741,15 +1741,15 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 
     switch (conversion) {
     case PrintConversion::ZeroExt64:
-      value = rewriter.create<arith::ExtUIOp>(
+      value = arith::ExtUIOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::SignExt64:
-      value = rewriter.create<arith::ExtSIOp>(
+      value = arith::ExtSIOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::Bitcast16:
-      value = rewriter.create<LLVM::BitcastOp>(
+      value = LLVM::BitcastOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 16), value);
       break;
     case PrintConversion::None:
@@ -1762,7 +1762,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
   // Helper to emit a call.
   static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
                        Operation *ref, ValueRange params = ValueRange()) {
-    rewriter.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(ref),
+    LLVM::CallOp::create(rewriter, loc, TypeRange(), SymbolRefAttr::get(ref),
                                   params);
   }
 };
@@ -1782,8 +1782,8 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     // First insert it into a poison vector so we can shuffle it.
     auto vectorType = typeConverter->convertType(splatOp.getType());
     Value poison =
-        rewriter.create<LLVM::PoisonOp>(splatOp.getLoc(), vectorType);
-    auto zero = rewriter.create<LLVM::ConstantOp>(
+        LLVM::PoisonOp::create(rewriter, splatOp.getLoc(), vectorType);
+    auto zero = LLVM::ConstantOp::create(rewriter,
         splatOp.getLoc(),
         typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
@@ -1796,7 +1796,7 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     }
 
     // For 1-d vector, we additionally do a `vectorshuffle`.
-    auto v = rewriter.create<LLVM::InsertElementOp>(
+    auto v = LLVM::InsertElementOp::create(rewriter,
         splatOp.getLoc(), vectorType, poison, adaptor.getInput(), zero);
 
     int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
@@ -1832,26 +1832,26 @@ struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
       return failure();
 
     // Construct returned value.
-    Value desc = rewriter.create<LLVM::PoisonOp>(loc, llvmNDVectorTy);
+    Value desc = LLVM::PoisonOp::create(rewriter, loc, llvmNDVectorTy);
 
     // Construct a 1-D vector with the splatted value that we insert in all the
     // places within the returned descriptor.
-    Value vdesc = rewriter.create<LLVM::PoisonOp>(loc, llvm1DVectorTy);
-    auto zero = rewriter.create<LLVM::ConstantOp>(
+    Value vdesc = LLVM::PoisonOp::create(rewriter, loc, llvm1DVectorTy);
+    auto zero = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
-    Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc,
+    Value v = LLVM::InsertElementOp::create(rewriter, loc, llvm1DVectorTy, vdesc,
                                                      adaptor.getInput(), zero);
 
     // Shuffle the value across the desired number of elements.
     int64_t width = resultType.getDimSize(resultType.getRank() - 1);
     SmallVector<int32_t> zeroValues(width, 0);
-    v = rewriter.create<LLVM::ShuffleVectorOp>(loc, v, v, zeroValues);
+    v = LLVM::ShuffleVectorOp::create(rewriter, loc, v, v, zeroValues);
 
     // Iterate of linear index, convert to coords space and insert splatted 1-D
     // vector in each position.
     nDVectorIterate(vectorTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
-      desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, v, position);
+      desc = LLVM::InsertValueOp::create(rewriter, loc, desc, v, position);
     });
     rewriter.replaceOp(splatOp, desc);
     return success();
@@ -1921,12 +1921,12 @@ struct VectorDeinterleaveOpLowering
       auto deinterleaveResults = deinterleaveOp.getResultTypes();
       auto packedOpResults =
           llvmTypeConverter->packOperationResults(deinterleaveResults);
-      auto intrinsic = rewriter.create<LLVM::vector_deinterleave2>(
+      auto intrinsic = LLVM::vector_deinterleave2::create(rewriter,
           loc, packedOpResults, adaptor.getSource());
 
-      auto evenResult = rewriter.create<LLVM::ExtractValueOp>(
+      auto evenResult = LLVM::ExtractValueOp::create(rewriter,
           loc, intrinsic->getResult(0), 0);
-      auto oddResult = rewriter.create<LLVM::ExtractValueOp>(
+      auto oddResult = LLVM::ExtractValueOp::create(rewriter,
           loc, intrinsic->getResult(0), 1);
 
       rewriter.replaceOp(deinterleaveOp, ValueRange{evenResult, oddResult});
@@ -1950,10 +1950,10 @@ struct VectorDeinterleaveOpLowering
         oddShuffleMask.push_back(i);
     }
 
-    auto poison = rewriter.create<LLVM::PoisonOp>(loc, sourceType);
-    auto evenShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+    auto poison = LLVM::PoisonOp::create(rewriter, loc, sourceType);
+    auto evenShuffle = LLVM::ShuffleVectorOp::create(rewriter,
         loc, adaptor.getSource(), poison, evenShuffleMask);
-    auto oddShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+    auto oddShuffle = LLVM::ShuffleVectorOp::create(rewriter,
         loc, adaptor.getSource(), poison, oddShuffleMask);
 
     rewriter.replaceOp(deinterleaveOp, ValueRange{evenShuffle, oddShuffle});
@@ -1977,9 +1977,9 @@ struct VectorFromElementsLowering
       return rewriter.notifyMatchFailure(fromElementsOp,
                                          "rank > 1 vectors are not supported");
     Type llvmType = typeConverter->convertType(vectorType);
-    Value result = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
+    Value result = LLVM::PoisonOp::create(rewriter, loc, llvmType);
     for (auto [idx, val] : llvm::enumerate(adaptor.getElements()))
-      result = rewriter.create<vector::InsertOp>(loc, val, result, idx);
+      result = vector::InsertOp::create(rewriter, loc, val, result, idx);
     rewriter.replaceOp(fromElementsOp, result);
     return success();
   }
@@ -2003,11 +2003,11 @@ struct VectorToElementsLowering
       if (element.use_empty())
         continue;
 
-      auto constIdx = rewriter.create<LLVM::ConstantOp>(
+      auto constIdx = LLVM::ConstantOp::create(rewriter,
           loc, idxType, rewriter.getIntegerAttr(idxType, idx));
       auto llvmType = typeConverter->convertType(element.getType());
 
-      Value result = rewriter.create<LLVM::ExtractElementOp>(loc, llvmType,
+      Value result = LLVM::ExtractElementOp::create(rewriter, loc, llvmType,
                                                              source, constIdx);
       results[idx] = result;
     }
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index ed51b2126dcdd..7385e28aeb04a 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -132,9 +132,9 @@ static void maybeYieldValue(OpBuilder &b, Location loc, bool hasRetVal,
                             Value value) {
   if (hasRetVal) {
     assert(value && "Expected non-empty value");
-    b.create<scf::YieldOp>(loc, value);
+    scf::YieldOp::create(b, loc, value);
   } else {
-    b.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(b, loc);
   }
 }
 
@@ -154,7 +154,7 @@ static Value generateMaskCheck(OpBuilder &b, OpTy xferOp, Value iv) {
     return Value();
 
   Location loc = xferOp.getLoc();
-  return b.create<vector::ExtractOp>(loc, xferOp.getMask(), iv);
+  return vector::ExtractOp::create(b, loc, xferOp.getMask(), iv);
 }
 
 /// Helper function TransferOpConversion and TransferOp1dConversion.
@@ -201,21 +201,21 @@ static Value generateInBoundsCheck(
     Value base = xferOp.getIndices()[*dim];
     Value memrefIdx =
         affine::makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
-    cond = lb.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, memrefDim,
+    cond = arith::CmpIOp::create(lb, arith::CmpIPredicate::sgt, memrefDim,
                                     memrefIdx);
   }
 
   // Condition check 2: Masked in?
   if (auto maskCond = generateMaskCheck(b, xferOp, iv)) {
     if (cond)
-      cond = lb.create<arith::AndIOp>(cond, maskCond);
+      cond = arith::AndIOp::create(lb, cond, maskCond);
     else
       cond = maskCond;
   }
 
   // If the condition is non-empty, generate an SCF::IfOp.
   if (cond) {
-    auto check = lb.create<scf::IfOp>(
+    auto check = scf::IfOp::create(lb,
         cond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
@@ -226,7 +226,7 @@ static Value generateInBoundsCheck(
           if (outOfBoundsCase) {
             maybeYieldValue(b, loc, hasRetVal, outOfBoundsCase(b, loc));
           } else {
-            b.create<scf::YieldOp>(loc);
+            scf::YieldOp::create(b, loc);
           }
         });
 
@@ -303,14 +303,14 @@ static BufferAllocs allocBuffers(OpBuilder &b, OpTy xferOp) {
 
   BufferAllocs result;
   auto bufferType = MemRefType::get({}, xferOp.getVectorType());
-  result.dataBuffer = b.create<memref::AllocaOp>(loc, bufferType);
+  result.dataBuffer = memref::AllocaOp::create(b, loc, bufferType);
 
   if (xferOp.getMask()) {
     auto maskType = MemRefType::get({}, xferOp.getMask().getType());
-    auto maskBuffer = b.create<memref::AllocaOp>(loc, maskType);
+    auto maskBuffer = memref::AllocaOp::create(b, loc, maskType);
     b.setInsertionPoint(xferOp);
-    b.create<memref::StoreOp>(loc, xferOp.getMask(), maskBuffer);
-    result.maskBuffer = b.create<memref::LoadOp>(loc, maskBuffer, ValueRange());
+    memref::StoreOp::create(b, loc, xferOp.getMask(), maskBuffer);
+    result.maskBuffer = memref::LoadOp::create(b, loc, maskBuffer, ValueRange());
   }
 
   return result;
@@ -421,14 +421,14 @@ struct Strategy<TransferReadOp> {
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
-    auto newXferOp = b.create<vector::TransferReadOp>(
+    auto newXferOp = vector::TransferReadOp::create(b,
         loc, vecType, xferOp.getBase(), xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
         xferOp.getPadding(), Value(), inBoundsAttr);
 
     maybeApplyPassLabel(b, newXferOp, options.targetRank);
 
-    b.create<memref::StoreOp>(loc, newXferOp.getVector(), buffer, storeIndices);
+    memref::StoreOp::create(b, loc, newXferOp.getVector(), buffer, storeIndices);
     return newXferOp;
   }
 
@@ -444,8 +444,8 @@ struct Strategy<TransferReadOp> {
     Location loc = xferOp.getLoc();
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
-    auto vec = b.create<vector::SplatOp>(loc, vecType, xferOp.getPadding());
-    b.create<memref::StoreOp>(loc, vec, buffer, storeIndices);
+    auto vec = vector::SplatOp::create(b, loc, vecType, xferOp.getPadding());
+    memref::StoreOp::create(b, loc, vec, buffer, storeIndices);
 
     return Value();
   }
@@ -506,11 +506,11 @@ struct Strategy<TransferWriteOp> {
     getXferIndices(b, xferOp, iv, xferIndices);
 
     Location loc = xferOp.getLoc();
-    auto vec = b.create<memref::LoadOp>(loc, buffer, loadIndices);
+    auto vec = memref::LoadOp::create(b, loc, buffer, loadIndices);
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
     auto source = loopState.empty() ? xferOp.getBase() : loopState[0];
     Type type = isTensorOp(xferOp) ? xferOp.getShapedType() : Type();
-    auto newXferOp = b.create<vector::TransferWriteOp>(
+    auto newXferOp = vector::TransferWriteOp::create(b,
         loc, type, vec, source, xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
         inBoundsAttr);
@@ -610,7 +610,7 @@ struct PrepareTransferReadConversion
     }
 
     Location loc = xferOp.getLoc();
-    rewriter.create<memref::StoreOp>(loc, newXfer->getResult(0),
+    memref::StoreOp::create(rewriter, loc, newXfer->getResult(0),
                                      buffers.dataBuffer);
     rewriter.replaceOpWithNewOp<memref::LoadOp>(xferOp, buffers.dataBuffer);
 
@@ -653,9 +653,9 @@ struct PrepareTransferWriteConversion
 
     Location loc = xferOp.getLoc();
     auto buffers = allocBuffers(rewriter, xferOp);
-    rewriter.create<memref::StoreOp>(loc, xferOp.getVector(),
+    memref::StoreOp::create(rewriter, loc, xferOp.getVector(),
                                      buffers.dataBuffer);
-    auto loadedVec = rewriter.create<memref::LoadOp>(loc, buffers.dataBuffer);
+    auto loadedVec = memref::LoadOp::create(rewriter, loc, buffers.dataBuffer);
     rewriter.modifyOpInPlace(xferOp, [&]() {
       xferOp.getValueToStoreMutable().assign(loadedVec);
       xferOp->setAttr(kPassLabel, rewriter.getUnitAttr());
@@ -735,17 +735,17 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
       auto signlessTargetVectorType =
           vectorType.cloneWith({}, getIntTypeWithSignlessSemantics(legalIntTy));
       auto targetVectorType = vectorType.cloneWith({}, legalIntTy);
-      value = rewriter.create<vector::BitCastOp>(loc, signlessSourceVectorType,
+      value = vector::BitCastOp::create(rewriter, loc, signlessSourceVectorType,
                                                  value);
       if (value.getType() != signlessTargetVectorType) {
         if (width == 1 || intTy.isUnsigned())
-          value = rewriter.create<arith::ExtUIOp>(loc, signlessTargetVectorType,
+          value = arith::ExtUIOp::create(rewriter, loc, signlessTargetVectorType,
                                                   value);
         else
-          value = rewriter.create<arith::ExtSIOp>(loc, signlessTargetVectorType,
+          value = arith::ExtSIOp::create(rewriter, loc, signlessTargetVectorType,
                                                   value);
       }
-      value = rewriter.create<vector::BitCastOp>(loc, targetVectorType, value);
+      value = vector::BitCastOp::create(rewriter, loc, targetVectorType, value);
       vectorType = targetVectorType;
     }
 
@@ -762,28 +762,28 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
                                         std::multiplies<int64_t>());
       auto flatVectorType =
           VectorType::get({flatLength}, vectorType.getElementType());
-      value = rewriter.create<vector::ShapeCastOp>(loc, flatVectorType, value);
+      value = vector::ShapeCastOp::create(rewriter, loc, flatVectorType, value);
     }
 
     vector::PrintOp firstClose;
     SmallVector<Value, 8> loopIndices;
     for (unsigned d = 0; d < shape.size(); d++) {
       // Setup loop bounds and step.
-      Value lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-      Value upperBound = rewriter.create<arith::ConstantIndexOp>(loc, shape[d]);
-      Value step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+      Value lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+      Value upperBound = arith::ConstantIndexOp::create(rewriter, loc, shape[d]);
+      Value step = arith::ConstantIndexOp::create(rewriter, loc, 1);
       if (!scalableDimensions.empty() && scalableDimensions[d]) {
-        auto vscale = rewriter.create<vector::VectorScaleOp>(
+        auto vscale = vector::VectorScaleOp::create(rewriter,
             loc, rewriter.getIndexType());
-        upperBound = rewriter.create<arith::MulIOp>(loc, upperBound, vscale);
+        upperBound = arith::MulIOp::create(rewriter, loc, upperBound, vscale);
       }
-      auto lastIndex = rewriter.create<arith::SubIOp>(loc, upperBound, step);
+      auto lastIndex = arith::SubIOp::create(rewriter, loc, upperBound, step);
 
       // Create a loop to print the elements surrounded by parentheses.
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+      vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
       auto loop =
-          rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
-      auto printClose = rewriter.create<vector::PrintOp>(
+          scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
+      auto printClose = vector::PrintOp::create(rewriter,
           loc, vector::PrintPunctuation::Close);
       if (!firstClose)
         firstClose = printClose;
@@ -793,13 +793,13 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
 
       // Print a comma after all but the last element.
       rewriter.setInsertionPointToStart(loop.getBody());
-      auto notLastIndex = rewriter.create<arith::CmpIOp>(
+      auto notLastIndex = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ult, loopIdx, lastIndex);
-      rewriter.create<scf::IfOp>(loc, notLastIndex,
+      scf::IfOp::create(rewriter, loc, notLastIndex,
                                  [&](OpBuilder &builder, Location loc) {
-                                   builder.create<vector::PrintOp>(
+                                   vector::PrintOp::create(builder,
                                        loc, vector::PrintPunctuation::Comma);
-                                   builder.create<scf::YieldOp>(loc);
+                                   scf::YieldOp::create(builder, loc);
                                  });
 
       rewriter.setInsertionPointToStart(loop.getBody());
@@ -810,22 +810,23 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     Value flatIndex;
     auto currentStride = 1;
     for (int d = shape.size() - 1; d >= 0; d--) {
-      auto stride = rewriter.create<arith::ConstantIndexOp>(loc, currentStride);
-      auto index = rewriter.create<arith::MulIOp>(loc, stride, loopIndices[d]);
+      auto stride = arith::ConstantIndexOp::create(rewriter, loc, currentStride);
+      auto index = arith::MulIOp::create(rewriter, loc, stride, loopIndices[d]);
       if (flatIndex)
-        flatIndex = rewriter.create<arith::AddIOp>(loc, flatIndex, index);
+        flatIndex = arith::AddIOp::create(rewriter, loc, flatIndex, index);
       else
         flatIndex = index;
       currentStride *= shape[d];
     }
 
     // Print the scalar elements in the inner most loop.
-    auto element = rewriter.create<vector::ExtractOp>(loc, value, flatIndex);
-    rewriter.create<vector::PrintOp>(loc, element,
+    auto element =
+        vector::ExtractOp::create(rewriter, loc, value, flatIndex);
+    vector::PrintOp::create(rewriter, loc, element,
                                      vector::PrintPunctuation::NoPunctuation);
 
     rewriter.setInsertionPointAfter(firstClose);
-    rewriter.create<vector::PrintOp>(loc, printOp.getPunctuation());
+    vector::PrintOp::create(rewriter, loc, printOp.getPunctuation());
     rewriter.eraseOp(printOp);
     return success();
   }
@@ -916,7 +917,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                                          "Failed to unpack one vector dim.");
 
     auto castedDataBuffer =
-        locB.create<vector::TypeCastOp>(*castedDataType, dataBuffer);
+        vector::TypeCastOp::create(locB, *castedDataType, dataBuffer);
 
     // If the xferOp has a mask: Find and cast mask buffer.
     Value castedMaskBuffer;
@@ -935,21 +936,21 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
         auto maskBufferType = cast<MemRefType>(maskBuffer.getType());
         MemRefType castedMaskType = *unpackOneDim(maskBufferType);
         castedMaskBuffer =
-            locB.create<vector::TypeCastOp>(castedMaskType, maskBuffer);
+            vector::TypeCastOp::create(locB, castedMaskType, maskBuffer);
       }
     }
 
     // Loop bounds and step.
-    auto lb = locB.create<arith::ConstantIndexOp>(0);
-    auto ub = locB.create<arith::ConstantIndexOp>(
+    auto lb = arith::ConstantIndexOp::create(locB, 0);
+    auto ub = arith::ConstantIndexOp::create(locB,
         castedDataType->getDimSize(castedDataType->getRank() - 1));
-    auto step = locB.create<arith::ConstantIndexOp>(1);
+    auto step = arith::ConstantIndexOp::create(locB, 1);
     // TransferWriteOps that operate on tensors return the modified tensor and
     // require a loop state.
     auto loopState = Strategy<OpTy>::initialLoopState(xferOp);
 
     // Generate for loop.
-    auto result = locB.create<scf::ForOp>(
+    auto result = scf::ForOp::create(locB,
         lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(),
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopState) {
           Type stateType = loopState.empty() ? Type() : loopState[0].getType();
@@ -975,7 +976,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                   SmallVector<Value, 8> loadIndices;
                   getMaskBufferLoadIndices(xferOp, castedMaskBuffer,
                                            loadIndices, iv);
-                  auto mask = b.create<memref::LoadOp>(loc, castedMaskBuffer,
+                  auto mask = memref::LoadOp::create(b, loc, castedMaskBuffer,
                                                        loadIndices);
                   rewriter.modifyOpInPlace(newXfer, [&]() {
                     newXfer.getMaskMutable().assign(mask);
@@ -1119,29 +1120,29 @@ struct ScalableTransposeTransferWriteConversion
     auto transposeSource = transposeOp.getVector();
     SmallVector<Value> transposeSourceSlices =
         llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-          return rewriter.create<vector::ExtractOp>(loc, transposeSource, idx);
+          return vector::ExtractOp::create(rewriter, loc, transposeSource, idx);
         });
 
     // Loop bounds and step.
-    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lb = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto ub =
         maskDims->empty()
             ? Value(createVscaleMultiple(vectorType.getDimSize(0)))
             : vector::getAsValues(rewriter, loc, maskDims->front()).front();
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
     // Generate a new mask for the slice.
     VectorType sliceType = VectorType::Builder(vectorType).dropDim(0);
     Value sliceMask = nullptr;
     if (!maskDims->empty()) {
-      sliceMask = rewriter.create<vector::CreateMaskOp>(
+      sliceMask = vector::CreateMaskOp::create(rewriter,
           loc, sliceType.clone(rewriter.getI1Type()),
           ArrayRef<OpFoldResult>(*maskDims).drop_front());
     }
 
     Value initDest = isTensorOp(writeOp) ? writeOp.getBase() : Value{};
     ValueRange initLoopArgs = initDest ? initDest : ValueRange{};
-    auto result = rewriter.create<scf::ForOp>(
+    auto result = scf::ForOp::create(rewriter,
         loc, lb, ub, step, initLoopArgs,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopIterArgs) {
           // Indices for the new transfer op.
@@ -1151,23 +1152,23 @@ struct ScalableTransposeTransferWriteConversion
           // Extract a transposed slice from the source vector.
           SmallVector<Value> transposeElements =
               llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-                return b.create<vector::ExtractOp>(
+                return vector::ExtractOp::create(b,
                     loc, transposeSourceSlices[idx], iv);
               });
-          auto sliceVec = b.create<vector::FromElementsOp>(loc, sliceType,
+          auto sliceVec = vector::FromElementsOp::create(b, loc, sliceType,
                                                            transposeElements);
 
           // Create the transfer_write for the slice.
           Value dest =
               loopIterArgs.empty() ? writeOp.getBase() : loopIterArgs.front();
-          auto newWriteOp = b.create<vector::TransferWriteOp>(
+          auto newWriteOp = vector::TransferWriteOp::create(b,
               loc, sliceVec, dest, xferIndices,
               ArrayRef<bool>(writeOp.getInBoundsValues()).drop_front());
           if (sliceMask)
             newWriteOp.getMaskMutable().assign(sliceMask);
 
           // Yield from the loop.
-          b.create<scf::YieldOp>(loc, loopIterArgs.empty()
+          scf::YieldOp::create(b, loc, loopIterArgs.empty()
                                           ? ValueRange{}
                                           : newWriteOp.getResult());
         });
@@ -1207,7 +1208,7 @@ static void maybeAssignMask(OpBuilder &b, OpTy xferOp, OpTy newXferOp,
 
     llvm::SmallVector<int64_t, 1> indices({i});
     Location loc = xferOp.getLoc();
-    auto newMask = b.create<vector::ExtractOp>(loc, xferOp.getMask(), indices);
+    auto newMask = vector::ExtractOp::create(b, loc, xferOp.getMask(), indices);
     newXferOp.getMaskMutable().assign(newMask);
   }
 
@@ -1261,7 +1262,7 @@ struct UnrollTransferReadConversion
     if (auto insertOp = getInsertOp(xferOp))
       return insertOp.getDest();
     Location loc = xferOp.getLoc();
-    return rewriter.create<vector::SplatOp>(loc, xferOp.getVectorType(),
+    return vector::SplatOp::create(rewriter, loc, xferOp.getVectorType(),
                                             xferOp.getPadding());
   }
 
@@ -1317,7 +1318,7 @@ struct UnrollTransferReadConversion
     // Generate fully unrolled loop of transfer ops.
     Location loc = xferOp.getLoc();
     for (int64_t i = 0; i < dimSize; ++i) {
-      Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
+      Value iv = arith::ConstantIndexOp::create(rewriter, loc, i);
 
       // FIXME: Rename this lambda - it does much more than just
       // in-bounds-check generation.
@@ -1336,7 +1337,7 @@ struct UnrollTransferReadConversion
 
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
 
-            auto newXferOp = b.create<vector::TransferReadOp>(
+            auto newXferOp = vector::TransferReadOp::create(b,
                 loc, newXferVecType, xferOp.getBase(), xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
                 xferOp.getPadding(), Value(), inBoundsAttr);
@@ -1346,10 +1347,10 @@ struct UnrollTransferReadConversion
             if (newXferVecType.getRank() == 0) {
               // vector.insert does not accept rank-0 as the non-indexed
               // argument. Extract the scalar before inserting.
-              valToInser = b.create<vector::ExtractOp>(loc, valToInser,
+              valToInser = vector::ExtractOp::create(b, loc, valToInser,
                                                        SmallVector<int64_t>());
             }
-            return b.create<vector::InsertOp>(loc, valToInser, vec,
+            return vector::InsertOp::create(b, loc, valToInser, vec,
                                               insertionIndices);
           },
           /*outOfBoundsCase=*/
@@ -1460,7 +1461,7 @@ struct UnrollTransferWriteConversion
     // Generate fully unrolled loop of transfer ops.
     Location loc = xferOp.getLoc();
     for (int64_t i = 0; i < dimSize; ++i) {
-      Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
+      Value iv = arith::ConstantIndexOp::create(rewriter, loc, i);
 
       auto updatedSource = generateInBoundsCheck(
           rewriter, xferOp, iv, unpackedDim(xferOp),
@@ -1477,19 +1478,19 @@ struct UnrollTransferWriteConversion
             extractionIndices.push_back(b.getI64IntegerAttr(i));
 
             auto extracted =
-                b.create<vector::ExtractOp>(loc, vec, extractionIndices);
+                vector::ExtractOp::create(b, loc, vec, extractionIndices);
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
             Value xferVec;
             if (inputVectorTy.getRank() == 1) {
               // When target-rank=0, unrolling would causes the vector input
               // argument into `transfer_write` to become a scalar. We solve
               // this by broadcasting the scalar to a 0D vector.
-              xferVec = b.create<vector::BroadcastOp>(
+              xferVec = vector::BroadcastOp::create(b,
                   loc, VectorType::get({}, extracted.getType()), extracted);
             } else {
               xferVec = extracted;
             }
-            auto newXferOp = b.create<vector::TransferWriteOp>(
+            auto newXferOp = vector::TransferWriteOp::create(b,
                 loc, sourceType, xferVec, source, xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
                 inBoundsAttr);
@@ -1572,18 +1573,18 @@ struct Strategy1d<TransferReadOp> {
         b, xferOp, iv, dim, TypeRange(xferOp.getVectorType()),
         /*inBoundsCase=*/
         [&](OpBuilder &b, Location loc) {
-          Value val = b.create<memref::LoadOp>(loc, xferOp.getBase(), indices);
-          return b.create<vector::InsertOp>(loc, val, vec, iv);
+          Value val = memref::LoadOp::create(b, loc, xferOp.getBase(), indices);
+          return vector::InsertOp::create(b, loc, val, vec, iv);
         },
         /*outOfBoundsCase=*/
         [&](OpBuilder & /*b*/, Location loc) { return vec; });
-    b.create<scf::YieldOp>(loc, nextVec);
+    scf::YieldOp::create(b, loc, nextVec);
   }
 
   static Value initialLoopState(OpBuilder &b, TransferReadOp xferOp) {
     // Inititalize vector with padding value.
     Location loc = xferOp.getLoc();
-    return b.create<vector::SplatOp>(loc, xferOp.getVectorType(),
+    return vector::SplatOp::create(b, loc, xferOp.getVectorType(),
                                      xferOp.getPadding());
   }
 };
@@ -1601,10 +1602,11 @@ struct Strategy1d<TransferWriteOp> {
     generateInBoundsCheck(
         b, xferOp, iv, dim,
         /*inBoundsCase=*/[&](OpBuilder &b, Location loc) {
-          auto val = b.create<vector::ExtractOp>(loc, xferOp.getVector(), iv);
-          b.create<memref::StoreOp>(loc, val, xferOp.getBase(), indices);
+          auto val =
+              vector::ExtractOp::create(b, loc, xferOp.getVector(), iv);
+          memref::StoreOp::create(b, loc, val, xferOp.getBase(), indices);
         });
-    b.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(b, loc);
   }
 
   static Value initialLoopState(OpBuilder &b, TransferWriteOp xferOp) {
@@ -1665,15 +1667,15 @@ struct TransferOp1dConversion : public VectorToSCFPattern<OpTy> {
     // Loop bounds, step, state...
     Location loc = xferOp.getLoc();
     auto vecType = xferOp.getVectorType();
-    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lb = arith::ConstantIndexOp::create(rewriter, loc, 0);
     Value ub =
-        rewriter.create<arith::ConstantIndexOp>(loc, vecType.getDimSize(0));
+        arith::ConstantIndexOp::create(rewriter, loc, vecType.getDimSize(0));
     if (vecType.isScalable()) {
       Value vscale =
-          rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-      ub = rewriter.create<arith::MulIOp>(loc, ub, vscale);
+          vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+      ub = arith::MulIOp::create(rewriter, loc, ub, vscale);
     }
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto loopState = Strategy1d<OpTy>::initialLoopState(rewriter, xferOp);
 
     // Generate for loop.
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 21d8e1d9f1156..2466e72277441 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -147,18 +147,18 @@ static Value sanitizeDynamicIndex(ConversionPatternRewriter &rewriter,
                                   Location loc, Value dynamicIndex,
                                   int64_t kPoisonIndex, unsigned vectorSize) {
   if (llvm::isPowerOf2_32(vectorSize)) {
-    Value inBoundsMask = rewriter.create<spirv::ConstantOp>(
+    Value inBoundsMask = spirv::ConstantOp::create(rewriter,
         loc, dynamicIndex.getType(),
         rewriter.getIntegerAttr(dynamicIndex.getType(), vectorSize - 1));
-    return rewriter.create<spirv::BitwiseAndOp>(loc, dynamicIndex,
+    return spirv::BitwiseAndOp::create(rewriter, loc, dynamicIndex,
                                                 inBoundsMask);
   }
-  Value poisonIndex = rewriter.create<spirv::ConstantOp>(
+  Value poisonIndex = spirv::ConstantOp::create(rewriter,
       loc, dynamicIndex.getType(),
       rewriter.getIntegerAttr(dynamicIndex.getType(), kPoisonIndex));
   Value cmpResult =
-      rewriter.create<spirv::IEqualOp>(loc, dynamicIndex, poisonIndex);
-  return rewriter.create<spirv::SelectOp>(
+      spirv::IEqualOp::create(rewriter, loc, dynamicIndex, poisonIndex);
+  return spirv::SelectOp::create(rewriter,
       loc, cmpResult,
       spirv::ConstantOp::getZero(dynamicIndex.getType(), loc, rewriter),
       dynamicIndex);
@@ -427,7 +427,7 @@ static SmallVector<Value> extractAllElements(
   Location loc = reduceOp.getLoc();
 
   for (int i = 0; i < numElements; ++i) {
-    values.push_back(rewriter.create<spirv::CompositeExtractOp>(
+    values.push_back(spirv::CompositeExtractOp::create(rewriter,
         loc, srcVectorType.getElementType(), adaptor.getVector(),
         rewriter.getI32ArrayAttr({i})));
   }
@@ -481,16 +481,16 @@ struct VectorReductionPattern final : OpConversionPattern<vector::ReductionOp> {
 #define INT_AND_FLOAT_CASE(kind, iop, fop)                                     \
   case vector::CombiningKind::kind:                                            \
     if (llvm::isa<IntegerType>(resultType)) {                                  \
-      result = rewriter.create<spirv::iop>(loc, resultType, result, next);     \
+      result = spirv::iop::create(rewriter, loc, resultType, result, next);     \
     } else {                                                                   \
       assert(llvm::isa<FloatType>(resultType));                                \
-      result = rewriter.create<spirv::fop>(loc, resultType, result, next);     \
+      result = spirv::fop::create(rewriter, loc, resultType, result, next);     \
     }                                                                          \
     break
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = rewriter.create<fop>(loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);              \
     break
 
         INT_AND_FLOAT_CASE(ADD, IAddOp, FAddOp);
@@ -537,7 +537,7 @@ struct VectorReductionFloatMinMax final
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = rewriter.create<fop>(loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);              \
     break
 
         INT_OR_FLOAT_CASE(MAXIMUMF, SPIRVFMaxOp);
@@ -613,7 +613,7 @@ struct VectorShuffleOpConvert final
     auto getElementAtIdx = [&rewriter, loc = shuffleOp.getLoc()](
                                Value scalarOrVec, int32_t idx) -> Value {
       if (auto vecTy = dyn_cast<VectorType>(scalarOrVec.getType()))
-        return rewriter.create<spirv::CompositeExtractOp>(loc, scalarOrVec,
+        return spirv::CompositeExtractOp::create(rewriter, loc, scalarOrVec,
                                                           idx);
 
       assert(idx == 0 && "Invalid scalar element index");
@@ -712,10 +712,10 @@ struct VectorDeinterleaveOpConvert final
     // We cannot use `spirv::VectorShuffleOp` directly in this case, and need to
     // use `spirv::CompositeExtractOp`.
     if (n == 2) {
-      auto elem0 = rewriter.create<spirv::CompositeExtractOp>(
+      auto elem0 = spirv::CompositeExtractOp::create(rewriter,
           loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({0}));
 
-      auto elem1 = rewriter.create<spirv::CompositeExtractOp>(
+      auto elem1 = spirv::CompositeExtractOp::create(rewriter,
           loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({1}));
 
       rewriter.replaceOp(deinterleaveOp, {elem0, elem1});
@@ -733,11 +733,11 @@ struct VectorDeinterleaveOpConvert final
         llvm::map_to_vector(seqOdd, [](int i) { return i * 2 + 1; });
 
     // Create two SPIR-V shuffles.
-    auto shuffleEven = rewriter.create<spirv::VectorShuffleOp>(
+    auto shuffleEven = spirv::VectorShuffleOp::create(rewriter,
         loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesEven));
 
-    auto shuffleOdd = rewriter.create<spirv::VectorShuffleOp>(
+    auto shuffleOdd = spirv::VectorShuffleOp::create(rewriter,
         loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesOdd));
 
@@ -781,7 +781,7 @@ struct VectorLoadOpConverter final
     // to a scalar.
     Value castedAccessChain = (vectorType.getNumElements() == 1)
                                   ? accessChain
-                                  : rewriter.create<spirv::BitcastOp>(
+                                  : spirv::BitcastOp::create(rewriter,
                                         loc, vectorPtrType, accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, spirvVectorType,
@@ -823,7 +823,7 @@ struct VectorStoreOpConverter final
     // to a scalar.
     Value castedAccessChain = (vectorType.getNumElements() == 1)
                                   ? accessChain
-                                  : rewriter.create<spirv::BitcastOp>(
+                                  : spirv::BitcastOp::create(rewriter,
                                         loc, vectorPtrType, accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, castedAccessChain,
@@ -905,9 +905,9 @@ struct VectorReductionToIntDotProd final
       auto v4i8Type = VectorType::get({4}, i8Type);
       Location loc = op.getLoc();
       Value zero = spirv::ConstantOp::getZero(i8Type, loc, rewriter);
-      lhsIn = rewriter.create<spirv::CompositeConstructOp>(
+      lhsIn = spirv::CompositeConstructOp::create(rewriter,
           loc, v4i8Type, ValueRange{lhsIn, zero});
-      rhsIn = rewriter.create<spirv::CompositeConstructOp>(
+      rhsIn = spirv::CompositeConstructOp::create(rewriter,
           loc, v4i8Type, ValueRange{rhsIn, zero});
     }
 
@@ -971,14 +971,14 @@ struct VectorReductionToFPDotProd final
       Attribute oneAttr =
           rewriter.getFloatAttr(vectorType.getElementType(), 1.0);
       oneAttr = SplatElementsAttr::get(vectorType, oneAttr);
-      rhs = rewriter.create<spirv::ConstantOp>(loc, vectorType, oneAttr);
+      rhs = spirv::ConstantOp::create(rewriter, loc, vectorType, oneAttr);
     }
     assert(lhs);
     assert(rhs);
 
-    Value res = rewriter.create<spirv::DotOp>(loc, resultType, lhs, rhs);
+    Value res = spirv::DotOp::create(rewriter, loc, resultType, lhs, rhs);
     if (acc)
-      res = rewriter.create<spirv::FAddOp>(loc, acc, res);
+      res = spirv::FAddOp::create(rewriter, loc, acc, res);
 
     rewriter.replaceOp(op, res);
     return success();
@@ -1013,7 +1013,7 @@ struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
     source.reserve(numElements);
     for (int64_t i = 0; i < numElements; ++i) {
       Attribute intAttr = rewriter.getIntegerAttr(intType, i);
-      Value constOp = rewriter.create<spirv::ConstantOp>(loc, intType, intAttr);
+      Value constOp = spirv::ConstantOp::create(rewriter, loc, intType, intAttr);
       source.push_back(constOp);
     }
     rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(stepOp, dstType,
@@ -1056,7 +1056,7 @@ struct VectorToElementOpConvert final
       if (element.use_empty())
         continue;
 
-      Value result = rewriter.create<spirv::CompositeExtractOp>(
+      Value result = spirv::CompositeExtractOp::create(rewriter,
           loc, elementType, adaptor.getSource(),
           rewriter.getI32ArrayAttr({static_cast<int32_t>(idx)}));
       results[idx] = result;
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index 2e6a16ddbfdaa..8e074f91744e8 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -108,7 +108,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
 
   xegpu::CreateNdDescOp ndDesc;
   if (srcTy.hasStaticShape()) {
-    ndDesc = rewriter.create<xegpu::CreateNdDescOp>(loc, descType, src,
+    ndDesc = xegpu::CreateNdDescOp::create(rewriter, loc, descType, src,
                                                     getAsOpFoldResult(offsets));
   } else {
     // In case of any dynamic shapes, source's shape and strides have to be
@@ -116,7 +116,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
     SmallVector<Value> sourceDims;
     unsigned srcRank = srcTy.getRank();
     for (unsigned i = 0; i < srcRank; ++i)
-      sourceDims.push_back(rewriter.create<memref::DimOp>(loc, src, i));
+      sourceDims.push_back(memref::DimOp::create(rewriter, loc, src, i));
 
     SmallVector<int64_t> constOffsets;
     SmallVector<Value> dynOffsets;
@@ -135,17 +135,17 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
 
     // Compute strides in reverse order.
     SmallVector<Value> dynStrides;
-    Value accStride = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value accStride = arith::ConstantIndexOp::create(rewriter, loc, 1);
     // Last stride is guaranteed to be static and unit.
     for (int i = static_cast<int>(strides.size()) - 2; i >= 0; --i) {
       accStride =
-          rewriter.create<arith::MulIOp>(loc, accStride, sourceDims[i + 1]);
+          arith::MulIOp::create(rewriter, loc, accStride, sourceDims[i + 1]);
       if (strides[i] == ShapedType::kDynamic)
         dynStrides.push_back(accStride);
     }
     std::reverse(dynStrides.begin(), dynStrides.end());
 
-    ndDesc = rewriter.create<xegpu::CreateNdDescOp>(
+    ndDesc = xegpu::CreateNdDescOp::create(rewriter,
         loc, descType, src, dynOffsets, dynShapes, dynStrides,
         DenseI64ArrayAttr::get(rewriter.getContext(), constOffsets),
         DenseI64ArrayAttr::get(rewriter.getContext(), srcTy.getShape()),
@@ -200,7 +200,7 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
                                                   ArrayRef<int64_t>{1, 0});
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadOp = rewriter.create<xegpu::LoadNdOp>(
+    auto loadOp = xegpu::LoadNdOp::create(rewriter,
         loc, vecTy, ndDesc, /*packed=*/nullptr, transposeAttr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
@@ -238,7 +238,7 @@ struct TransferWriteLowering
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeOp =
-        rewriter.create<xegpu::StoreNdOp>(loc, writeOp.getVector(), ndDesc,
+        xegpu::StoreNdOp::create(rewriter, loc, writeOp.getVector(), ndDesc,
                                           /*l1_hint=*/hint,
                                           /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(writeOp, storeOp);
@@ -269,7 +269,7 @@ struct LoadLowering : public OpRewritePattern<vector::LoadOp> {
 
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadNdOp = rewriter.create<xegpu::LoadNdOp>(
+    auto loadNdOp = xegpu::LoadNdOp::create(rewriter,
         loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
@@ -303,7 +303,7 @@ struct StoreLowering : public OpRewritePattern<vector::StoreOp> {
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeNdOp =
-        rewriter.create<xegpu::StoreNdOp>(loc, vector, ndDesc,
+        xegpu::StoreNdOp::create(rewriter, loc, vector, ndDesc,
                                           /*l1_hint=*/hint,
                                           /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(storeOp, storeNdOp);
@@ -339,7 +339,7 @@ struct ContractionLowering : public OpRewritePattern<vector::ContractionOp> {
     if (!isRowMajorMatmul(contractOp.getIndexingMapsAttr()))
       return rewriter.notifyMatchFailure(contractOp, "Invalid indexing maps");
 
-    auto dpasOp = rewriter.create<xegpu::DpasOp>(
+    auto dpasOp = xegpu::DpasOp::create(rewriter,
         loc, TypeRange{contractOp.getResultType()}, ValueRange{lhs, rhs, acc});
     rewriter.replaceOp(contractOp, dpasOp);
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
index 37e0d2af55fe1..0230810f260ca 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -99,8 +99,8 @@ static Value flattenVecToBits(ConversionPatternRewriter &rewriter, Location loc,
       vectorType.getElementTypeBitWidth() * vectorType.getNumElements();
   Type allBitsType = rewriter.getIntegerType(bitwidth);
   auto allBitsVecType = VectorType::get({1}, allBitsType);
-  Value bitcast = rewriter.create<vector::BitCastOp>(loc, allBitsVecType, val);
-  Value scalar = rewriter.create<vector::ExtractOp>(loc, bitcast, 0);
+  Value bitcast = vector::BitCastOp::create(rewriter, loc, allBitsVecType, val);
+  Value scalar = vector::ExtractOp::create(rewriter, loc, bitcast, 0);
   return scalar;
 }
 
@@ -119,25 +119,25 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   SmallVector<NamedAttribute> loadAttrs;
   patchOperandSegmentSizes(origAttrs, loadAttrs, DataArgAction::Drop);
   Value initialLoad =
-      rewriter.create<RawBufferLoadOp>(loc, dataType, invariantArgs, loadAttrs);
+      RawBufferLoadOp::create(rewriter, loc, dataType, invariantArgs, loadAttrs);
   Block *currentBlock = rewriter.getInsertionBlock();
   Block *afterAtomic =
       rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
   Block *loopBlock = rewriter.createBlock(afterAtomic, {dataType}, {loc});
 
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(loc, loopBlock, initialLoad);
+  cf::BranchOp::create(rewriter, loc, loopBlock, initialLoad);
 
   rewriter.setInsertionPointToEnd(loopBlock);
   Value prevLoad = loopBlock->getArgument(0);
-  Value operated = rewriter.create<ArithOp>(loc, data, prevLoad);
+  Value operated = ArithOp::create(rewriter, loc, data, prevLoad);
   dataType = operated.getType();
 
   SmallVector<NamedAttribute> cmpswapAttrs;
   patchOperandSegmentSizes(origAttrs, cmpswapAttrs, DataArgAction::Duplicate);
   SmallVector<Value> cmpswapArgs = {operated, prevLoad};
   cmpswapArgs.append(invariantArgs.begin(), invariantArgs.end());
-  Value atomicRes = rewriter.create<RawBufferAtomicCmpswapOp>(
+  Value atomicRes = RawBufferAtomicCmpswapOp::create(rewriter,
       loc, dataType, cmpswapArgs, cmpswapAttrs);
 
   // We care about exact bitwise equality here, so do some bitcasts.
@@ -150,13 +150,13 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   if (auto floatDataTy = dyn_cast<FloatType>(dataType)) {
     Type equivInt = rewriter.getIntegerType(floatDataTy.getWidth());
     prevLoadForCompare =
-        rewriter.create<arith::BitcastOp>(loc, equivInt, prevLoad);
+        arith::BitcastOp::create(rewriter, loc, equivInt, prevLoad);
     atomicResForCompare =
-        rewriter.create<arith::BitcastOp>(loc, equivInt, atomicRes);
+        arith::BitcastOp::create(rewriter, loc, equivInt, atomicRes);
   }
-  Value canLeave = rewriter.create<arith::CmpIOp>(
+  Value canLeave = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::eq, atomicResForCompare, prevLoadForCompare);
-  rewriter.create<cf::CondBranchOp>(loc, canLeave, afterAtomic, ValueRange{},
+  cf::CondBranchOp::create(rewriter, loc, canLeave, afterAtomic, ValueRange{},
                                     loopBlock, atomicRes);
   rewriter.eraseOp(atomicOp);
   return success();
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
index 60c8660658a95..a11804c1d7741 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -56,10 +56,10 @@ static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
                                            vector::MaskedLoadOp maskedOp,
                                            bool passthru) {
   VectorType vectorType = maskedOp.getVectorType();
-  Value load = builder.create<vector::LoadOp>(
+  Value load = vector::LoadOp::create(builder,
       loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
   if (passthru)
-    load = builder.create<arith::SelectOp>(loc, vectorType, maskedOp.getMask(),
+    load = arith::SelectOp::create(builder,loc, vectorType, maskedOp.getMask(),
                                            load, maskedOp.getPassThru());
   return load;
 }
@@ -110,7 +110,7 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
     SmallVector<OpFoldResult> indices = maskedOp.getIndices();
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, src);
     SmallVector<OpFoldResult> strides =
         stridedMetadata.getConstifiedMixedStrides();
     SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
@@ -124,47 +124,47 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
 
     // delta = bufferSize - linearizedOffset
     Value vectorSizeOffset =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
+        arith::ConstantIndexOp::create(rewriter, loc, vectorSize);
     Value linearIndex =
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
     Value totalSize = getValueOrCreateConstantIndexOp(
         rewriter, loc, linearizedInfo.linearizedSize);
-    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
+    Value delta = arith::SubIOp::create(rewriter, loc, totalSize, linearIndex);
 
     // 1) check if delta < vectorSize
-    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
+    Value isOutofBounds = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
 
     // 2) check if (detla % elements_per_word != 0)
-    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
+    Value elementsPerWord = arith::ConstantIndexOp::create(rewriter,
         loc, llvm::divideCeil(32, elementBitWidth));
-    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
+    Value isNotWordAligned = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne,
-        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
+        arith::RemUIOp::create(rewriter, loc, delta, elementsPerWord),
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
 
     // We take the fallback of maskedload default lowering only it is both
     // out-of-bounds and not word aligned. The fallback ensures correct results
     // when loading at the boundary of the buffer since buffer load returns
     // inconsistent zeros for the whole word when boundary is crossed.
     Value ifCondition =
-        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
+        arith::AndIOp::create(rewriter, loc, isOutofBounds, isNotWordAligned);
 
     auto thenBuilder = [&](OpBuilder &builder, Location loc) {
       Operation *read = builder.clone(*maskedOp.getOperation());
       read->setAttr(kMaskedloadNeedsMask, builder.getUnitAttr());
       Value readResult = read->getResult(0);
-      builder.create<scf::YieldOp>(loc, readResult);
+      scf::YieldOp::create(builder, loc, readResult);
     };
 
     auto elseBuilder = [&](OpBuilder &builder, Location loc) {
       Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp,
                                                 /*passthru=*/true);
-      rewriter.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(rewriter, loc, res);
     };
 
     auto ifOp =
-        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
+        scf::IfOp::create(rewriter, loc, ifCondition, thenBuilder, elseBuilder);
 
     rewriter.replaceOp(maskedOp, ifOp);
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
index 195f59d625554..4771d655e36c2 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
@@ -37,7 +37,7 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       return rewriter.notifyMatchFailure(metadataOp,
                                          "not a fat raw buffer cast");
     Location loc = castOp.getLoc();
-    auto sourceMetadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+    auto sourceMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
         loc, castOp.getSource());
     SmallVector<Value> results;
     if (metadataOp.getBaseBuffer().use_empty()) {
@@ -48,13 +48,13 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       if (baseBufferType == castOp.getResult().getType()) {
         results.push_back(castOp.getResult());
       } else {
-        results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+        results.push_back(memref::ReinterpretCastOp::create(rewriter,
             loc, baseBufferType, castOp.getResult(), /*offset=*/0,
             /*sizes=*/ArrayRef<int64_t>{}, /*strides=*/ArrayRef<int64_t>{}));
       }
     }
     if (castOp.getResetOffset())
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, 0));
     else
       results.push_back(sourceMetadata.getOffset());
     llvm::append_range(results, sourceMetadata.getSizes());
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 12b375b373fa9..748ff1edbfeb2 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -76,8 +76,8 @@ static SmallVector<Value> getTileSizes(Location loc, amx::TileType tType,
   auto mattr = rewriter.getI16IntegerAttr(tType.getDimSize(0));
   auto nattr = rewriter.getI16IntegerAttr(tType.getDimSize(1) * bytes);
   return SmallVector<Value>{
-      rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, mattr),
-      rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, nattr)};
+      LLVM::ConstantOp::create(rewriter, loc, llvmInt16Type, mattr),
+      LLVM::ConstantOp::create(rewriter, loc, llvmInt16Type, nattr)};
 }
 
 /// Maps the 2-dim memref shape to the 64-bit stride. Note that the buffer
@@ -95,7 +95,7 @@ static Value getStride(Location loc, MemRefType mType, Value base,
     // Dynamic stride needs code to compute the stride at runtime.
     MemRefDescriptor memrefDescriptor(base);
     auto attr = rewriter.getI64IntegerAttr(bytes);
-    Value scale = rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr);
+    Value scale = LLVM::ConstantOp::create(rewriter, loc, llvmInt64Type, attr);
     return rewriter
         .create<LLVM::MulOp>(loc, llvmInt64Type, scale,
                              memrefDescriptor.stride(rewriter, loc, preLast))
@@ -103,7 +103,7 @@ static Value getStride(Location loc, MemRefType mType, Value base,
   }
   // Use direct constant for static stride.
   auto attr = rewriter.getI64IntegerAttr(strides[preLast] * bytes);
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr)
+  return LLVM::ConstantOp::create(rewriter, loc, llvmInt64Type, attr)
       .getResult();
 }
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index e8e8f624d806e..3113d5896741b 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -240,7 +240,7 @@ Operation *AffineDialect::materializeConstant(OpBuilder &builder,
                                               Attribute value, Type type,
                                               Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
 
@@ -1282,7 +1282,7 @@ mlir::affine::makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map,
   map = foldAttributesIntoMap(b, map, operands, valueOperands);
   composeAffineMapAndOperands(&map, &valueOperands, composeAffineMin);
   assert(map);
-  return b.create<AffineApplyOp>(loc, map, valueOperands);
+  return AffineApplyOp::create(b, loc, map, valueOperands);
 }
 
 AffineApplyOp
@@ -1389,7 +1389,7 @@ static OpTy makeComposedMinMax(OpBuilder &b, Location loc, AffineMap map,
   SmallVector<Value> valueOperands;
   map = foldAttributesIntoMap(b, map, operands, valueOperands);
   composeMultiResultAffineMap(map, valueOperands);
-  return b.create<OpTy>(loc, b.getIndexType(), map, valueOperands);
+  return OpTy::create(b, loc, b.getIndexType(), map, valueOperands);
 }
 
 AffineMinOp
@@ -2688,7 +2688,7 @@ FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInits());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  AffineForOp newLoop = rewriter.create<AffineForOp>(
+  AffineForOp newLoop = AffineForOp::create(rewriter,
       getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
       getUpperBoundOperands(), getUpperBoundMap(), getStepAsInt(), inits);
 
@@ -2831,7 +2831,7 @@ static void buildAffineLoopNestImpl(
         OpBuilder::InsertionGuard nestedGuard(nestedBuilder);
         bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
       }
-      nestedBuilder.create<AffineYieldOp>(nestedLoc);
+      AffineYieldOp::create(nestedBuilder, nestedLoc);
     };
 
     // Delegate actual loop creation to the callback in order to dispatch
@@ -2846,7 +2846,7 @@ static AffineForOp
 buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb,
                              int64_t ub, int64_t step,
                              AffineForOp::BodyBuilderFn bodyBuilderFn) {
-  return builder.create<AffineForOp>(loc, lb, ub, step,
+  return AffineForOp::create(builder, loc, lb, ub, step,
                                      /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
@@ -2860,7 +2860,7 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
   if (lbConst && ubConst)
     return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
                                         ubConst.value(), step, bodyBuilderFn);
-  return builder.create<AffineForOp>(loc, lb, builder.getDimIdentityMap(), ub,
+  return AffineForOp::create(builder, loc, lb, builder.getDimIdentityMap(), ub,
                                      builder.getDimIdentityMap(), step,
                                      /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
@@ -4883,7 +4883,7 @@ struct DropUnitExtentBasis
     Location loc = delinearizeOp->getLoc();
     auto getZero = [&]() -> Value {
       if (!zero)
-        zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+        zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
       return zero.value();
     };
 
@@ -4906,7 +4906,7 @@ struct DropUnitExtentBasis
 
     if (!newBasis.empty()) {
       // Will drop the leading nullptr from `basis` if there was no outer bound.
-      auto newDelinearizeOp = rewriter.create<affine::AffineDelinearizeIndexOp>(
+      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(rewriter,
           loc, delinearizeOp.getLinearIndex(), newBasis);
       int newIndex = 0;
       // Map back the new delinearized indices to the values they replace.
@@ -4971,11 +4971,11 @@ struct CancelDelinearizeOfLinearizeDisjointExactTail
       return success();
     }
 
-    Value newLinearize = rewriter.create<affine::AffineLinearizeIndexOp>(
+    Value newLinearize = affine::AffineLinearizeIndexOp::create(rewriter,
         linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
         ArrayRef<OpFoldResult>{linearizeBasis}.drop_back(numMatches),
         linearizeOp.getDisjoint());
-    auto newDelinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
         delinearizeOp.getLoc(), newLinearize,
         ArrayRef<OpFoldResult>{delinearizeBasis}.drop_back(numMatches),
         delinearizeOp.hasOuterBound());
@@ -5049,17 +5049,17 @@ struct SplitDelinearizeSpanningLastLinearizeArg final
           "need at least two elements to form the basis product");
 
     Value linearizeWithoutBack =
-        rewriter.create<affine::AffineLinearizeIndexOp>(
+        affine::AffineLinearizeIndexOp::create(rewriter,
             linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
             linearizeOp.getDynamicBasis(),
             linearizeOp.getStaticBasis().drop_back(),
             linearizeOp.getDisjoint());
     auto delinearizeWithoutSplitPart =
-        rewriter.create<affine::AffineDelinearizeIndexOp>(
+        affine::AffineDelinearizeIndexOp::create(rewriter,
             delinearizeOp.getLoc(), linearizeWithoutBack,
             delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
             delinearizeOp.hasOuterBound());
-    auto delinearizeBack = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(rewriter,
         delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
         basis.take_back(elemsToSplit), /*hasOuterBound=*/true);
     SmallVector<Value> results = llvm::to_vector(
@@ -5272,7 +5272,7 @@ OpFoldResult computeProduct(Location loc, OpBuilder &builder,
   }
   if (auto constant = dyn_cast<AffineConstantExpr>(result))
     return getAsIndexOpFoldResult(builder.getContext(), constant.getValue());
-  return builder.create<AffineApplyOp>(loc, result, dynamicPart).getResult();
+  return AffineApplyOp::create(builder, loc, result, dynamicPart).getResult();
 }
 
 /// If conseceutive outputs of a delinearize_index are linearized with the same
@@ -5437,7 +5437,7 @@ struct CancelLinearizeOfDelinearizePortion final
       newDelinBasis.erase(newDelinBasis.begin() + m.delinStart,
                           newDelinBasis.begin() + m.delinStart + m.length);
       newDelinBasis.insert(newDelinBasis.begin() + m.delinStart, newSize);
-      auto newDelinearize = rewriter.create<AffineDelinearizeIndexOp>(
+      auto newDelinearize = AffineDelinearizeIndexOp::create(rewriter,
           m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
           newDelinBasis);
 
@@ -5445,7 +5445,7 @@ struct CancelLinearizeOfDelinearizePortion final
       // create a residual affine.delinearize_index that delinearizes the
       // merged output into its component parts.
       Value combinedElem = newDelinearize.getResult(m.delinStart);
-      auto residualDelinearize = rewriter.create<AffineDelinearizeIndexOp>(
+      auto residualDelinearize = AffineDelinearizeIndexOp::create(rewriter,
           m.delinearize.getLoc(), combinedElem, basisToMerge);
 
       // Swap all the uses of the unaffected delinearize outputs to the new
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index f18cec5a14fae..df39544aeaa09 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -202,7 +202,7 @@ void AffineDataCopyGeneration::runOnBlock(Block *block,
 void AffineDataCopyGeneration::runOnOperation() {
   func::FuncOp f = getOperation();
   OpBuilder topBuilder(f.getBody());
-  zeroIndex = topBuilder.create<arith::ConstantIndexOp>(f.getLoc(), 0);
+  zeroIndex = arith::ConstantIndexOp::create(topBuilder, f.getLoc(), 0);
 
   // Nests that are copy-in's or copy-out's; the root AffineForOps of those
   // nests are stored herein.
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
index 5430bdc4ff858..fb0d0d5027085 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
@@ -58,7 +58,7 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
       // Note: basis elements and their products are, definitionally,
       // non-negative, so `nuw` is justified.
       if (dynamicPart)
-        dynamicPart = rewriter.create<arith::MulIOp>(
+        dynamicPart = arith::MulIOp::create(rewriter,
             loc, dynamicPart, dynamicBasis[dynamicIndex - 1], ovflags);
       else
         dynamicPart = dynamicBasis[dynamicIndex - 1];
@@ -74,7 +74,7 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
           rewriter.createOrFold<arith::ConstantIndexOp>(loc, staticPart);
       if (dynamicPart)
         stride =
-            rewriter.create<arith::MulIOp>(loc, dynamicPart, stride, ovflags);
+            arith::MulIOp::create(rewriter, loc, dynamicPart, stride, ovflags);
       result.push_back(stride);
     }
   }
@@ -106,19 +106,19 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
   Value zero = rewriter.createOrFold<arith::ConstantIndexOp>(loc, 0);
 
   Value initialPart =
-      rewriter.create<arith::FloorDivSIOp>(loc, linearIdx, strides.front());
+      arith::FloorDivSIOp::create(rewriter, loc, linearIdx, strides.front());
   results.push_back(initialPart);
 
   auto emitModTerm = [&](Value stride) -> Value {
-    Value remainder = rewriter.create<arith::RemSIOp>(loc, linearIdx, stride);
-    Value remainderNegative = rewriter.create<arith::CmpIOp>(
+    Value remainder = arith::RemSIOp::create(rewriter, loc, linearIdx, stride);
+    Value remainderNegative = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, remainder, zero);
     // If the correction is relevant, this term is <= stride, which is known
     // to be positive in `index`. Otherwise, while 2 * stride might overflow,
     // this branch won't be taken, so the risk of `poison` is fine.
-    Value corrected = rewriter.create<arith::AddIOp>(
+    Value corrected = arith::AddIOp::create(rewriter,
         loc, remainder, stride, arith::IntegerOverflowFlags::nsw);
-    Value mod = rewriter.create<arith::SelectOp>(loc, remainderNegative,
+    Value mod = arith::SelectOp::create(rewriter, loc, remainderNegative,
                                                  corrected, remainder);
     return mod;
   };
@@ -131,7 +131,7 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
     // We know both inputs are positive, so floorDiv == div.
     // This could potentially be a divui, but it's not clear if that would
     // cause issues.
-    Value divided = rewriter.create<arith::DivSIOp>(loc, modulus, nextStride);
+    Value divided = arith::DivSIOp::create(rewriter, loc, modulus, nextStride);
     results.push_back(divided);
   }
 
@@ -167,7 +167,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   // our hands on an `OpOperand&` for the loop invariant counting function.
   for (auto [stride, idxOp] :
        llvm::zip_equal(strides, llvm::drop_end(op.getMultiIndexMutable()))) {
-    Value scaledIdx = rewriter.create<arith::MulIOp>(
+    Value scaledIdx = arith::MulIOp::create(rewriter,
         loc, idxOp.get(), stride, arith::IntegerOverflowFlags::nsw);
     int64_t numHoistableLoops = numEnclosingInvariantLoops(idxOp);
     scaledValues.emplace_back(scaledIdx, numHoistableLoops);
@@ -184,7 +184,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   Value result = scaledValues.front().first;
   for (auto [scaledValue, numHoistableLoops] : llvm::drop_begin(scaledValues)) {
     std::ignore = numHoistableLoops;
-    result = rewriter.create<arith::AddIOp>(loc, result, scaledValue,
+    result = arith::AddIOp::create(rewriter, loc, result, scaledValue,
                                             arith::IntegerOverflowFlags::nsw);
   }
   rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
index 4fd0cf9b3cd25..a78cea5bff6dd 100644
--- a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
@@ -88,7 +88,7 @@ static AffineApplyOp createSubApply(RewriterBase &rewriter,
   auto rhsMap = AffineMap::get(m.getNumDims(), m.getNumSymbols(), expr, ctx);
   SmallVector<Value> rhsOperands = originalOp->getOperands();
   canonicalizeMapAndOperands(&rhsMap, &rhsOperands);
-  return rewriter.create<AffineApplyOp>(originalOp.getLoc(), rhsMap,
+  return AffineApplyOp::create(rewriter, originalOp.getLoc(), rhsMap,
                                         rhsOperands);
 }
 
@@ -160,7 +160,7 @@ FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
   auto current = createSubApply(rewriter, op, subExpressions[0]);
   for (int64_t i = 1, e = subExpressions.size(); i < e; ++i) {
     Value tmp = createSubApply(rewriter, op, subExpressions[i]);
-    current = rewriter.create<AffineApplyOp>(op.getLoc(), binMap,
+    current = AffineApplyOp::create(rewriter, op.getLoc(), binMap,
                                              ValueRange{current, tmp});
     LLVM_DEBUG(DBGS() << "--reassociate into: " << current << "\n");
   }
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 95848d0b67547..b19206c7afb92 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -424,7 +424,7 @@ static Value createPrivateMemRef(AffineForOp forOp,
   // consumer loop nests to reduce their live range. Currently they are added
   // at the beginning of the block, because loop nests can be reordered
   // during the fusion pass.
-  Value newMemRef = top.create<memref::AllocOp>(forOp.getLoc(), newMemRefType);
+  Value newMemRef = memref::AllocOp::create(top, forOp.getLoc(), newMemRefType);
 
   // Build an AffineMap to remap access functions based on lower bound offsets.
   SmallVector<AffineExpr, 4> remapExprs;
diff --git a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
index 05a352f39a93c..17384654586c4 100644
--- a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
@@ -100,7 +100,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
 
   // Create and place the alloc right before the 'affine.for' operation.
-  Value newMemRef = bOuter.create<memref::AllocOp>(
+  Value newMemRef = memref::AllocOp::create(bOuter,
       forOp.getLoc(), newMemRefType, allocOperands);
 
   // Create 'iv mod 2' value to index the leading dimension.
@@ -108,7 +108,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   int64_t step = forOp.getStepAsInt();
   auto modTwoMap =
       AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, d0.floorDiv(step) % 2);
-  auto ivModTwoOp = bInner.create<AffineApplyOp>(forOp.getLoc(), modTwoMap,
+  auto ivModTwoOp = AffineApplyOp::create(bInner, forOp.getLoc(), modTwoMap,
                                                  forOp.getInductionVar());
 
   // replaceAllMemRefUsesWith will succeed unless the forOp body has
@@ -130,7 +130,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
   // Insert the dealloc op right after the for loop.
   bOuter.setInsertionPointAfter(forOp);
-  bOuter.create<memref::DeallocOp>(forOp.getLoc(), newMemRef);
+  memref::DeallocOp::create(bOuter, forOp.getLoc(), newMemRef);
 
   return true;
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
index 1a266b72d1f8d..9537d3e75c26a 100644
--- a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
@@ -51,10 +51,10 @@ OpFoldResult affine::materializeComputedBound(
            "expected dynamic dim");
     if (isa<RankedTensorType>(value.getType())) {
       // A tensor dimension is used: generate a tensor.dim.
-      operands.push_back(b.create<tensor::DimOp>(loc, value, *dim));
+      operands.push_back(tensor::DimOp::create(b, loc, value, *dim));
     } else if (isa<MemRefType>(value.getType())) {
       // A memref dimension is used: generate a memref.dim.
-      operands.push_back(b.create<memref::DimOp>(loc, value, *dim));
+      operands.push_back(memref::DimOp::create(b, loc, value, *dim));
     } else {
       llvm_unreachable("cannot generate DimOp for unsupported shaped type");
     }
@@ -76,7 +76,7 @@ OpFoldResult affine::materializeComputedBound(
         operands[expr.getPosition() + boundMap.getNumDims()]);
   // General case: build affine.apply op.
   return static_cast<OpFoldResult>(
-      b.create<affine::AffineApplyOp>(loc, boundMap, operands).getResult());
+      affine::AffineApplyOp::create(b, loc, boundMap, operands).getResult());
 }
 
 FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound(
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index 7fae260767e0a..b078514cbf978 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -905,7 +905,7 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   for (auto resultExpr : map.getResults()) {
     auto singleResMap =
         AffineMap::get(map.getNumDims(), map.getNumSymbols(), resultExpr);
-    auto afOp = state.builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+    auto afOp = state.AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                     mapOperands);
     results.push_back(afOp);
   }
@@ -961,7 +961,7 @@ static arith::ConstantOp vectorizeConstant(arith::ConstantOp constOp,
   auto vecForOp = cast<AffineForOp>(parentOp);
   state.builder.setInsertionPointToStart(vecForOp.getBody());
   auto newConstOp =
-      state.builder.create<arith::ConstantOp>(constOp.getLoc(), vecAttr);
+      state.arith::ConstantOp::create(builder, constOp.getLoc(), vecAttr);
 
   // Register vector replacement for future uses in the scope.
   state.registerOpVectorReplacement(constOp, newConstOp);
@@ -986,7 +986,7 @@ static Operation *vectorizeAffineApplyOp(AffineApplyOp applyOp,
     }
   }
 
-  auto newApplyOp = state.builder.create<AffineApplyOp>(
+  auto newApplyOp = state.AffineApplyOp::create(builder,
       applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
 
   // Register the new affine.apply result.
@@ -1010,7 +1010,7 @@ static arith::ConstantOp createInitialVector(arith::AtomicRMWKind reductionKind,
   auto vecTy = getVectorType(scalarTy, state.strategy);
   auto vecAttr = DenseElementsAttr::get(vecTy, valueAttr);
   auto newConstOp =
-      state.builder.create<arith::ConstantOp>(oldOperand.getLoc(), vecAttr);
+      state.arith::ConstantOp::create(builder, oldOperand.getLoc(), vecAttr);
 
   return newConstOp;
 }
@@ -1062,10 +1062,10 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   AffineMap ubMap = vecForOp.getUpperBoundMap();
   Value ub;
   if (ubMap.getNumResults() == 1)
-    ub = state.builder.create<AffineApplyOp>(loc, vecForOp.getUpperBoundMap(),
+    ub = state.AffineApplyOp::create(builder, loc, vecForOp.getUpperBoundMap(),
                                              vecForOp.getUpperBoundOperands());
   else
-    ub = state.builder.create<AffineMinOp>(loc, vecForOp.getUpperBoundMap(),
+    ub = state.AffineMinOp::create(builder, loc, vecForOp.getUpperBoundMap(),
                                            vecForOp.getUpperBoundOperands());
   // Then we compute the number of (original) iterations left in the loop.
   AffineExpr subExpr =
@@ -1080,7 +1080,7 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   Type maskTy = VectorType::get(state.strategy->vectorSizes,
                                 state.builder.getIntegerType(1));
   Value mask =
-      state.builder.create<vector::CreateMaskOp>(loc, maskTy, itersLeft);
+      state.vector::CreateMaskOp::create(builder, loc, maskTy, itersLeft);
 
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ creating a mask:\n"
                     << itersLeft << "\n"
@@ -1123,7 +1123,7 @@ static Operation *vectorizeUniform(Value uniformVal,
   state.builder.setInsertionPointAfterValue(uniformScalarRepl);
 
   auto vectorTy = getVectorType(uniformVal.getType(), state.strategy);
-  auto bcastOp = state.builder.create<BroadcastOp>(uniformVal.getLoc(),
+  auto bcastOp = state.BroadcastOp::create(builder, uniformVal.getLoc(),
                                                    vectorTy, uniformScalarRepl);
   state.registerValueVectorReplacement(uniformVal, bcastOp);
   return bcastOp;
@@ -1256,7 +1256,7 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.builder.create<vector::TransferReadOp>(
+  auto transfer = state.vector::TransferReadOp::create(builder,
       loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
       /*padding=*/std::nullopt, permutationMap);
 
@@ -1303,7 +1303,7 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.builder.create<vector::TransferWriteOp>(
+  auto transfer = state.vector::TransferWriteOp::create(builder,
       storeOp.getLoc(), vectorValue, storeOp.getMemRef(), indices,
       permutationMap);
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
@@ -1387,7 +1387,7 @@ static Operation *vectorizeAffineForOp(AffineForOp forOp,
     }
   }
 
-  auto vecForOp = state.builder.create<AffineForOp>(
+  auto vecForOp = state.AffineForOp::create(builder,
       forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(),
       forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep,
       vecIterOperands,
@@ -1512,7 +1512,7 @@ static Operation *vectorizeAffineYieldOp(AffineYieldOp yieldOp,
       // IterOperands are neutral element vectors.
       Value neutralVal = cast<AffineForOp>(newParentOp).getInits()[i];
       state.builder.setInsertionPoint(combinerOps.back());
-      Value maskedReducedVal = state.builder.create<arith::SelectOp>(
+      Value maskedReducedVal = state.arith::SelectOp::create(builder,
           reducedVal.getLoc(), mask, reducedVal, neutralVal);
       LLVM_DEBUG(
           dbgs() << "\n[early-vect]+++++ masking an input to a binary op that"
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 0501616ad912c..0f16e84263ec1 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -55,7 +55,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
 
   OpBuilder b(forOp);
   auto lbMap = forOp.getLowerBoundMap();
-  auto lb = b.create<AffineApplyOp>(forOp.getLoc(), lbMap,
+  auto lb = AffineApplyOp::create(b, forOp.getLoc(), lbMap,
                                     forOp.getLowerBoundOperands());
 
   // For each upper bound expr, get the range.
@@ -72,7 +72,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
     auto bumpMap = AffineMap::get(tripCountMap.getNumDims(),
                                   tripCountMap.getNumSymbols(), bumpExprs[i]);
     bumpValues[i] =
-        b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, tripCountOperands);
+        AffineApplyOp::create(b, forOp.getLoc(), bumpMap, tripCountOperands);
   }
 
   SmallVector<AffineExpr, 4> newUbExprs(tripCountMap.getNumResults());
@@ -135,7 +135,7 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         builder.setInsertionPointToStart(&func.getFunctionBody().front());
       else
         builder.setInsertionPoint(forOp);
-      auto constOp = builder.create<arith::ConstantIndexOp>(
+      auto constOp = arith::ConstantIndexOp::create(builder,
           forOp.getLoc(), forOp.getConstantLowerBound());
       iv.replaceAllUsesWith(constOp);
     } else {
@@ -147,7 +147,7 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         iv.replaceAllUsesWith(lbOperands[0]);
       } else {
         auto affineApplyOp =
-            builder.create<AffineApplyOp>(forOp.getLoc(), lbMap, lbOperands);
+            AffineApplyOp::create(builder, forOp.getLoc(), lbMap, lbOperands);
         iv.replaceAllUsesWith(affineApplyOp);
       }
     }
@@ -182,7 +182,7 @@ static AffineForOp generateShiftedLoop(
   assert(ubMap.getNumInputs() == ubOperands.size());
 
   auto loopChunk =
-      b.create<AffineForOp>(srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
+      AffineForOp::create(b, srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
                             ubMap, srcForOp.getStepAsInt());
   auto loopChunkIV = loopChunk.getInductionVar();
   auto srcIV = srcForOp.getInductionVar();
@@ -198,7 +198,7 @@ static AffineForOp generateShiftedLoop(
     // Generate the remapping if the shift is not zero: remappedIV = newIV -
     // shift.
     if (!srcIV.use_empty() && shift != 0) {
-      auto ivRemap = bodyBuilder.create<AffineApplyOp>(
+      auto ivRemap = AffineApplyOp::create(bodyBuilder,
           srcForOp.getLoc(),
           bodyBuilder.getSingleDimShiftAffineMap(
               -static_cast<int64_t>(srcForOp.getStepAsInt() * shift)),
@@ -434,7 +434,7 @@ static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
   for (unsigned i = 0; i < width; i++) {
     OpBuilder b(topLoop);
     // Loop bounds will be set later.
-    AffineForOp pointLoop = b.create<AffineForOp>(loc, 0, 0);
+    AffineForOp pointLoop = AffineForOp::create(b, loc, 0, 0);
     pointLoop.getBody()->getOperations().splice(
         pointLoop.getBody()->begin(), topLoop->getBlock()->getOperations(),
         topLoop);
@@ -448,7 +448,7 @@ static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
   for (unsigned i = width; i < 2 * width; i++) {
     OpBuilder b(topLoop);
     // Loop bounds will be set later.
-    AffineForOp tileSpaceLoop = b.create<AffineForOp>(loc, 0, 0);
+    AffineForOp tileSpaceLoop = AffineForOp::create(b, loc, 0, 0);
     tileSpaceLoop.getBody()->getOperations().splice(
         tileSpaceLoop.getBody()->begin(), topLoop->getBlock()->getOperations(),
         topLoop);
@@ -1063,7 +1063,7 @@ LogicalResult mlir::affine::loopUnrollByFactor(
         // iv' = iv + i * step
         auto d0 = b.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, d0 + i * step);
-        return b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, iv);
+        return AffineApplyOp::create(b, forOp.getLoc(), bumpMap, iv);
       },
       /*annotateFn=*/annotateFn,
       /*iterArgs=*/iterArgs, /*yieldedValues=*/yieldedValues);
@@ -1227,7 +1227,7 @@ LogicalResult mlir::affine::loopUnrollJamByFactor(AffineForOp forOp,
         auto d0 = builder.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, d0 + i * step);
         auto ivUnroll =
-            builder.create<AffineApplyOp>(forOp.getLoc(), bumpMap, forOpIV);
+            AffineApplyOp::create(builder, forOp.getLoc(), bumpMap, forOpIV);
         operandMaps[i - 1].map(forOpIV, ivUnroll);
       }
       // Clone the sub-block being unroll-jammed.
@@ -1556,7 +1556,7 @@ stripmineSink(AffineForOp forOp, uint64_t factor,
   for (auto t : targets) {
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator(t.getBody());
-    auto newForOp = b.create<AffineForOp>(t.getLoc(), lbOperands, lbMap,
+    auto newForOp = AffineForOp::create(b, t.getLoc(), lbOperands, lbMap,
                                           ubOperands, ubMap, originalStep);
     auto begin = t.getBody()->begin();
     // Skip terminator and `newForOp` which is just before the terminator.
@@ -1631,9 +1631,9 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
   // 1. Store the upper bound of the outermost loop in a variable.
   Value prev;
   if (!llvm::hasSingleElement(origUbMap.getResults()))
-    prev = builder.create<AffineMinOp>(loc, origUbMap, ubOperands);
+    prev = AffineMinOp::create(builder, loc, origUbMap, ubOperands);
   else
-    prev = builder.create<AffineApplyOp>(loc, origUbMap, ubOperands);
+    prev = AffineApplyOp::create(builder, loc, origUbMap, ubOperands);
   upperBoundSymbols.push_back(prev);
 
   // 2. Emit code computing the upper bound of the coalesced loop as product of
@@ -1645,15 +1645,15 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
     Value upperBound;
     // If upper bound map has more than one result, take their minimum.
     if (!llvm::hasSingleElement(origUbMap.getResults()))
-      upperBound = builder.create<AffineMinOp>(loc, origUbMap, ubOperands);
+      upperBound = AffineMinOp::create(builder, loc, origUbMap, ubOperands);
     else
-      upperBound = builder.create<AffineApplyOp>(loc, origUbMap, ubOperands);
+      upperBound = AffineApplyOp::create(builder, loc, origUbMap, ubOperands);
     upperBoundSymbols.push_back(upperBound);
     SmallVector<Value, 4> operands;
     operands.push_back(prev);
     operands.push_back(upperBound);
     // Maintain running product of loop upper bounds.
-    prev = builder.create<AffineApplyOp>(
+    prev = AffineApplyOp::create(builder,
         loc,
         AffineMap::get(/*dimCount=*/1,
                        /*symbolCount=*/1,
@@ -1683,7 +1683,7 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> operands;
       operands.push_back(previous);
       operands.push_back(upperBoundSymbols[idx]);
-      previous = builder.create<AffineApplyOp>(
+      previous = AffineApplyOp::create(builder,
           loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
@@ -1700,7 +1700,7 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> applyOperands;
       applyOperands.push_back(previous);
       applyOperands.push_back(upperBoundSymbols[idx - 1]);
-      inductionVariable = builder.create<AffineApplyOp>(
+      inductionVariable = AffineApplyOp::create(builder,
           loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
@@ -1738,21 +1738,21 @@ void mlir::affine::mapLoopToProcessorIds(scf::ForOp forOp,
 
   Value linearIndex = processorId.front();
   for (unsigned i = 1, e = processorId.size(); i < e; ++i) {
-    auto mulApplyOp = b.create<AffineApplyOp>(
+    auto mulApplyOp = AffineApplyOp::create(b,
         loc, mulMap, ValueRange{linearIndex, numProcessors[i]});
-    linearIndex = b.create<AffineApplyOp>(
+    linearIndex = AffineApplyOp::create(b,
         loc, addMap, ValueRange{mulApplyOp, processorId[i]});
   }
 
-  auto mulApplyOp = b.create<AffineApplyOp>(
+  auto mulApplyOp = AffineApplyOp::create(b,
       loc, mulMap, ValueRange{linearIndex, forOp.getStep()});
-  Value lb = b.create<AffineApplyOp>(
+  Value lb = AffineApplyOp::create(b,
       loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()});
   forOp.setLowerBound(lb);
 
   Value step = forOp.getStep();
   for (auto numProcs : numProcessors)
-    step = b.create<AffineApplyOp>(loc, mulMap, ValueRange{numProcs, step});
+    step = AffineApplyOp::create(b, loc, mulMap, ValueRange{numProcs, step});
   forOp.setStep(step);
 }
 
@@ -1889,7 +1889,7 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
 
     auto fastBufOffsetMap =
         AffineMap::get(lbOperands.size(), 0, fastBufOffsets[d]);
-    auto offset = b.create<AffineApplyOp>(loc, fastBufOffsetMap, lbOperands);
+    auto offset = AffineApplyOp::create(b, loc, fastBufOffsetMap, lbOperands);
 
     // Construct the subscript for the fast memref being copied into/from:
     // x - offset_x.
@@ -1916,16 +1916,16 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
 
   if (!isCopyOut) {
     // Copy in.
-    auto load = b.create<AffineLoadOp>(loc, memref, memIndices);
-    b.create<AffineStoreOp>(loc, load, fastMemRef, fastBufMap,
+    auto load = AffineLoadOp::create(b, loc, memref, memIndices);
+    AffineStoreOp::create(b, loc, load, fastMemRef, fastBufMap,
                             fastBufMapOperands);
     return copyNestRoot;
   }
 
   // Copy out.
   auto load =
-      b.create<AffineLoadOp>(loc, fastMemRef, fastBufMap, fastBufMapOperands);
-  b.create<AffineStoreOp>(loc, load, memref, memIndices);
+      AffineLoadOp::create(b, loc, fastMemRef, fastBufMap, fastBufMapOperands);
+  AffineStoreOp::create(b, loc, load, memref, memIndices);
   return copyNestRoot;
 }
 
@@ -1960,7 +1960,7 @@ static LogicalResult generateCopy(
 
   auto f = begin->getParentOfType<FunctionOpInterface>();
   OpBuilder topBuilder(f.getFunctionBody());
-  Value zeroIndex = topBuilder.create<arith::ConstantIndexOp>(f.getLoc(), 0);
+  Value zeroIndex = arith::ConstantIndexOp::create(topBuilder, f.getLoc(), 0);
 
   *sizeInBytes = 0;
 
@@ -2071,7 +2071,7 @@ static LogicalResult generateCopy(
         memIndices.push_back(zeroIndex);
       } else {
         memIndices.push_back(
-            top.create<arith::ConstantIndexOp>(loc, indexVal).getResult());
+            arith::ConstantIndexOp::create(top, loc, indexVal).getResult());
       }
     } else {
       // The coordinate for the start location is just the lower bound along the
@@ -2085,7 +2085,7 @@ static LogicalResult generateCopy(
       lbs[d] = lbs[d].replaceDimsAndSymbols(
           /*dimReplacements=*/{}, symReplacements, lbs[d].getNumSymbols(),
           /*numResultSyms=*/0);
-      memIndices.push_back(b.create<AffineApplyOp>(loc, lbs[d], regionSymbols));
+      memIndices.push_back(AffineApplyOp::create(b, loc, lbs[d], regionSymbols));
     }
     // The fast buffer is copied into at location zero; addressing is relative.
     bufIndices.push_back(zeroIndex);
@@ -2109,7 +2109,7 @@ static LogicalResult generateCopy(
     // Create the fast memory space buffer just before the 'affine.for'
     // operation.
     fastMemRef =
-        prologue.create<memref::AllocOp>(loc, fastMemRefType).getResult();
+        memref::AllocOp::create(prologue, loc, fastMemRefType).getResult();
     // Record it.
     fastBufferMap[memref] = fastMemRef;
     // fastMemRefType is a constant shaped memref.
@@ -2126,7 +2126,7 @@ static LogicalResult generateCopy(
     fastMemRef = fastBufferMap[memref];
   }
 
-  auto numElementsSSA = top.create<arith::ConstantIndexOp>(loc, *numElements);
+  auto numElementsSSA = arith::ConstantIndexOp::create(top, loc, *numElements);
 
   Value dmaStride;
   Value numEltPerDmaStride;
@@ -2143,8 +2143,8 @@ static LogicalResult generateCopy(
 
     if (!dmaStrideInfos.empty()) {
       dmaStride =
-          top.create<arith::ConstantIndexOp>(loc, dmaStrideInfos[0].stride);
-      numEltPerDmaStride = top.create<arith::ConstantIndexOp>(
+          arith::ConstantIndexOp::create(top, loc, dmaStrideInfos[0].stride);
+      numEltPerDmaStride = arith::ConstantIndexOp::create(top,
           loc, dmaStrideInfos[0].numEltPerStride);
     }
   }
@@ -2175,20 +2175,20 @@ static LogicalResult generateCopy(
     // Create a tag (single element 1-d memref) for the DMA.
     auto tagMemRefType = MemRefType::get({1}, top.getIntegerType(32), {},
                                          copyOptions.tagMemorySpace);
-    auto tagMemRef = prologue.create<memref::AllocOp>(loc, tagMemRefType);
+    auto tagMemRef = memref::AllocOp::create(prologue, loc, tagMemRefType);
 
     SmallVector<Value, 4> tagIndices({zeroIndex});
     auto tagAffineMap = b.getMultiDimIdentityMap(tagIndices.size());
     fullyComposeAffineMapAndOperands(&tagAffineMap, &tagIndices);
     if (!region.isWrite()) {
       // DMA non-blocking read from original buffer to fast buffer.
-      b.create<AffineDmaStartOp>(loc, memref, memAffineMap, memIndices,
+      AffineDmaStartOp::create(b, loc, memref, memAffineMap, memIndices,
                                  fastMemRef, bufAffineMap, bufIndices,
                                  tagMemRef, tagAffineMap, tagIndices,
                                  numElementsSSA, dmaStride, numEltPerDmaStride);
     } else {
       // DMA non-blocking write from fast buffer to the original memref.
-      auto op = b.create<AffineDmaStartOp>(
+      auto op = AffineDmaStartOp::create(b,
           loc, fastMemRef, bufAffineMap, bufIndices, memref, memAffineMap,
           memIndices, tagMemRef, tagAffineMap, tagIndices, numElementsSSA,
           dmaStride, numEltPerDmaStride);
@@ -2199,11 +2199,11 @@ static LogicalResult generateCopy(
     }
 
     // Matching DMA wait to block on completion; tag always has a 0 index.
-    b.create<AffineDmaWaitOp>(loc, tagMemRef, tagAffineMap, zeroIndex,
+    AffineDmaWaitOp::create(b, loc, tagMemRef, tagAffineMap, zeroIndex,
                               numElementsSSA);
 
     // Generate dealloc for the tag.
-    auto tagDeallocOp = epilogue.create<memref::DeallocOp>(loc, tagMemRef);
+    auto tagDeallocOp = memref::DeallocOp::create(epilogue, loc, tagMemRef);
     if (*nEnd == end && isCopyOutAtEndOfBlock)
       // Since new ops are being appended (for outgoing DMAs), adjust the end to
       // mark end of range of the original.
@@ -2212,7 +2212,7 @@ static LogicalResult generateCopy(
 
   // Generate dealloc for the buffer.
   if (!existingBuf) {
-    auto bufDeallocOp = epilogue.create<memref::DeallocOp>(loc, fastMemRef);
+    auto bufDeallocOp = memref::DeallocOp::create(epilogue, loc, fastMemRef);
     // When generating pointwise copies, `nEnd' has to be set to deallocOp on
     // the fast buffer (since it marks the new end insertion point).
     if (!copyOptions.generateDma && *nEnd == end && isCopyOutAtEndOfBlock)
@@ -2582,7 +2582,7 @@ AffineForOp mlir::affine::createCanonicalizedAffineForOp(
   canonicalizeMapAndOperands(&ubMap, &upperOperands);
   ubMap = removeDuplicateExprs(ubMap);
 
-  return b.create<AffineForOp>(loc, lowerOperands, lbMap, upperOperands, ubMap,
+  return AffineForOp::create(b, loc, lowerOperands, lbMap, upperOperands, ubMap,
                                step);
 }
 
@@ -2666,7 +2666,7 @@ static AffineIfOp createSeparationCondition(MutableArrayRef<AffineForOp> loops,
   SmallVector<Value, 4> setOperands;
   cst.getValues(0, cst.getNumDimAndSymbolVars(), &setOperands);
   canonicalizeSetAndOperands(&ifCondSet, &setOperands);
-  return b.create<AffineIfOp>(loops[0].getLoc(), ifCondSet, setOperands,
+  return AffineIfOp::create(b, loops[0].getLoc(), ifCondSet, setOperands,
                               /*withElseRegion=*/true);
 }
 
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 83816a1543708..0b0950968ecba 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -57,7 +57,7 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     if (!lhs || !rhs)
       return nullptr;
-    auto op = builder.create<OpTy>(loc, lhs, rhs, overflowFlags);
+    auto op = OpTy::create(builder, loc, lhs, rhs, overflowFlags);
     return op.getResult();
   }
 
@@ -91,13 +91,13 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value remainder = builder.create<arith::RemSIOp>(loc, lhs, rhs);
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value isRemainderNegative = builder.create<arith::CmpIOp>(
+    Value remainder = arith::RemSIOp::create(builder, loc, lhs, rhs);
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value isRemainderNegative = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::slt, remainder, zeroCst);
     Value correctedRemainder =
-        builder.create<arith::AddIOp>(loc, remainder, rhs);
-    Value result = builder.create<arith::SelectOp>(
+        arith::AddIOp::create(builder, loc, remainder, rhs);
+    Value result = arith::SelectOp::create(builder,
         loc, isRemainderNegative, correctedRemainder, remainder);
     return result;
   }
@@ -130,17 +130,17 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value noneCst = builder.create<arith::ConstantIndexOp>(loc, -1);
-    Value negative = builder.create<arith::CmpIOp>(
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value noneCst = arith::ConstantIndexOp::create(builder, loc, -1);
+    Value negative = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::slt, lhs, zeroCst);
-    Value negatedDecremented = builder.create<arith::SubIOp>(loc, noneCst, lhs);
+    Value negatedDecremented = arith::SubIOp::create(builder, loc, noneCst, lhs);
     Value dividend =
-        builder.create<arith::SelectOp>(loc, negative, negatedDecremented, lhs);
-    Value quotient = builder.create<arith::DivSIOp>(loc, dividend, rhs);
+        arith::SelectOp::create(builder, loc, negative, negatedDecremented, lhs);
+    Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value correctedQuotient =
-        builder.create<arith::SubIOp>(loc, noneCst, quotient);
-    Value result = builder.create<arith::SelectOp>(loc, negative,
+        arith::SubIOp::create(builder, loc, noneCst, quotient);
+    Value result = arith::SelectOp::create(builder, loc, negative,
                                                    correctedQuotient, quotient);
     return result;
   }
@@ -169,26 +169,26 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value oneCst = builder.create<arith::ConstantIndexOp>(loc, 1);
-    Value nonPositive = builder.create<arith::CmpIOp>(
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value oneCst = arith::ConstantIndexOp::create(builder, loc, 1);
+    Value nonPositive = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::sle, lhs, zeroCst);
-    Value negated = builder.create<arith::SubIOp>(loc, zeroCst, lhs);
-    Value decremented = builder.create<arith::SubIOp>(loc, lhs, oneCst);
+    Value negated = arith::SubIOp::create(builder, loc, zeroCst, lhs);
+    Value decremented = arith::SubIOp::create(builder, loc, lhs, oneCst);
     Value dividend =
-        builder.create<arith::SelectOp>(loc, nonPositive, negated, decremented);
-    Value quotient = builder.create<arith::DivSIOp>(loc, dividend, rhs);
+        arith::SelectOp::create(builder, loc, nonPositive, negated, decremented);
+    Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value negatedQuotient =
-        builder.create<arith::SubIOp>(loc, zeroCst, quotient);
+        arith::SubIOp::create(builder, loc, zeroCst, quotient);
     Value incrementedQuotient =
-        builder.create<arith::AddIOp>(loc, quotient, oneCst);
-    Value result = builder.create<arith::SelectOp>(
+        arith::AddIOp::create(builder, loc, quotient, oneCst);
+    Value result = arith::SelectOp::create(builder,
         loc, nonPositive, negatedQuotient, incrementedQuotient);
     return result;
   }
 
   Value visitConstantExpr(AffineConstantExpr expr) {
-    auto op = builder.create<arith::ConstantIndexOp>(loc, expr.getValue());
+    auto op = arith::ConstantIndexOp::create(builder, loc, expr.getValue());
     return op.getResult();
   }
 
@@ -298,7 +298,7 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   // block.
   IRMapping operandMap;
   OpBuilder b(hoistOverOp);
-  auto hoistedIfOp = b.create<AffineIfOp>(ifOp.getLoc(), ifOp.getIntegerSet(),
+  auto hoistedIfOp = AffineIfOp::create(b, ifOp.getLoc(), ifOp.getIntegerSet(),
                                           ifOp.getOperands(),
                                           /*elseBlock=*/true);
 
@@ -369,7 +369,7 @@ mlir::affine::affineParallelize(AffineForOp forOp,
       parallelReductions, [](const LoopReduction &red) { return red.value; }));
   auto reductionKinds = llvm::to_vector<4>(llvm::map_range(
       parallelReductions, [](const LoopReduction &red) { return red.kind; }));
-  AffineParallelOp newPloop = outsideBuilder.create<AffineParallelOp>(
+  AffineParallelOp newPloop = AffineParallelOp::create(outsideBuilder,
       loc, ValueRange(reducedValues).getTypes(), reductionKinds,
       llvm::ArrayRef(lowerBoundMap), lowerBoundOperands,
       llvm::ArrayRef(upperBoundMap), upperBoundOperands,
@@ -541,7 +541,7 @@ void mlir::affine::normalizeAffineParallel(AffineParallelOp op) {
     SmallVector<Value, 8> applyOperands{dimOperands};
     applyOperands.push_back(iv);
     applyOperands.append(symbolOperands.begin(), symbolOperands.end());
-    auto apply = builder.create<AffineApplyOp>(op.getLoc(), map, applyOperands);
+    auto apply = AffineApplyOp::create(builder, op.getLoc(), map, applyOperands);
     iv.replaceAllUsesExcept(apply, apply);
   }
 
@@ -622,7 +622,7 @@ LogicalResult mlir::affine::normalizeAffineFor(AffineForOp op,
   AffineValueMap newIvToOldIvMap;
   AffineValueMap::difference(lbMap, scaleIvValueMap, &newIvToOldIvMap);
   (void)newIvToOldIvMap.canonicalize();
-  auto newIV = opBuilder.create<AffineApplyOp>(
+  auto newIV = AffineApplyOp::create(opBuilder,
       loc, newIvToOldIvMap.getAffineMap(), newIvToOldIvMap.getOperands());
   op.getInductionVar().replaceAllUsesExcept(newIV->getResult(0), newIV);
   return success();
@@ -1187,7 +1187,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     for (auto resultExpr : oldMap.getResults()) {
       auto singleResMap = AffineMap::get(oldMap.getNumDims(),
                                          oldMap.getNumSymbols(), resultExpr);
-      auto afOp = builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+      auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                 oldMapOperands);
       oldMemRefOperands.push_back(afOp);
       affineApplyOps.push_back(afOp);
@@ -1214,7 +1214,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     for (auto resultExpr : indexRemap.getResults()) {
       auto singleResMap = AffineMap::get(
           indexRemap.getNumDims(), indexRemap.getNumSymbols(), resultExpr);
-      auto afOp = builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+      auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                 remapOperands);
       remapOutputs.push_back(afOp);
       affineApplyOps.push_back(afOp);
@@ -1264,7 +1264,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     // AffineMapAccessInterface, we need to apply the values of `newMapOperands`
     // to the `newMap` to get the correct indices.
     for (unsigned i = 0; i < newMemRefRank; i++) {
-      state.operands.push_back(builder.create<AffineApplyOp>(
+      state.operands.push_back(AffineApplyOp::create(builder,
           op->getLoc(),
           AffineMap::get(newMap.getNumDims(), newMap.getNumSymbols(),
                          newMap.getResult(i)),
@@ -1450,7 +1450,7 @@ void mlir::affine::createAffineComputationSlice(
   for (auto resultExpr : composedMap.getResults()) {
     auto singleResMap = AffineMap::get(composedMap.getNumDims(),
                                        composedMap.getNumSymbols(), resultExpr);
-    sliceOps->push_back(builder.create<AffineApplyOp>(
+    sliceOps->push_back(AffineApplyOp::create(builder,
         opInst->getLoc(), singleResMap, composedOpOperands));
   }
 
@@ -1681,7 +1681,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
       // Create ConstantOp for static dimension.
       auto constantAttr = b.getIntegerAttr(b.getIndexType(), oldMemRefShape[d]);
       inAffineApply.emplace_back(
-          b.create<arith::ConstantOp>(allocOp.getLoc(), constantAttr));
+          arith::ConstantOp::create(b, allocOp.getLoc(), constantAttr));
     }
   }
 
@@ -1705,7 +1705,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
       AffineMap newMap =
           AffineMap::get(map.getNumInputs(), map.getNumSymbols(), newMapOutput);
       Value affineApp =
-          b.create<AffineApplyOp>(allocOp.getLoc(), newMap, inAffineApply);
+          AffineApplyOp::create(b, allocOp.getLoc(), newMap, inAffineApply);
       newDynamicSizes.emplace_back(affineApp);
     }
     newDimIdx++;
@@ -1741,10 +1741,10 @@ LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp allocOp) {
                           newDynamicSizes);
     // Add the new dynamic sizes in new AllocOp.
     newAlloc =
-        b.create<AllocLikeOp>(allocOp.getLoc(), newMemRefType, newDynamicSizes,
+        AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType, newDynamicSizes,
                               allocOp.getAlignmentAttr());
   } else {
-    newAlloc = b.create<AllocLikeOp>(allocOp.getLoc(), newMemRefType,
+    newAlloc = AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType,
                                      allocOp.getAlignmentAttr());
   }
   // Replace all uses of the old memref.
@@ -1803,10 +1803,10 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0, e = memrefType.getRank(); i < e; i++) {
     if (memrefType.isDynamicDim(i))
       mapOperands[i] =
-          b.create<arith::SubIOp>(loc, oldSizes[0].getType(), oldSizes[idx++],
-                                  b.create<arith::ConstantIndexOp>(loc, 1));
+          arith::SubIOp::create(b, loc, oldSizes[0].getType(), oldSizes[idx++],
+                                  arith::ConstantIndexOp::create(b, loc, 1));
     else
-      mapOperands[i] = b.create<arith::ConstantIndexOp>(loc, oldShape[i] - 1);
+      mapOperands[i] = arith::ConstantIndexOp::create(b, loc, oldShape[i] - 1);
   }
   for (unsigned i = 0, e = oldStrides.size(); i < e; i++)
     mapOperands[memrefType.getRank() + i] = oldStrides[i];
@@ -1816,7 +1816,7 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0; i < newRank; i++) {
     if (!newMemRefType.isDynamicDim(i))
       continue;
-    newSizes.push_back(b.create<AffineApplyOp>(
+    newSizes.push_back(AffineApplyOp::create(b,
         loc,
         AffineMap::get(oldLayoutMap.getNumDims(), oldLayoutMap.getNumSymbols(),
                        oldLayoutMap.getResult(i)),
@@ -1824,11 +1824,11 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   }
   for (unsigned i = 0, e = newSizes.size(); i < e; i++) {
     newSizes[i] =
-        b.create<arith::AddIOp>(loc, newSizes[i].getType(), newSizes[i],
-                                b.create<arith::ConstantIndexOp>(loc, 1));
+        arith::AddIOp::create(b, loc, newSizes[i].getType(), newSizes[i],
+                                arith::ConstantIndexOp::create(b, loc, 1));
   }
   // Create the new reinterpret_cast op.
-  auto newReinterpretCast = b.create<memref::ReinterpretCastOp>(
+  auto newReinterpretCast = memref::ReinterpretCastOp::create(b,
       loc, newMemRefType, reinterpretCastOp.getSource(),
       /*offsets=*/ValueRange(), newSizes,
       /*strides=*/ValueRange(),
diff --git a/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp b/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
index ebcb951cf3518..e7cbee6b06c45 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
@@ -64,7 +64,7 @@ Operation *arith::ArithDialect::materializeConstant(OpBuilder &builder,
                                                     Attribute value, Type type,
                                                     Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   return ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 4e40d4ebda004..60c1fc6adacff 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -242,7 +242,7 @@ bool arith::ConstantOp::isBuildableWith(Attribute value, Type type) {
 ConstantOp arith::ConstantOp::materialize(OpBuilder &builder, Attribute value,
                                           Type type, Location loc) {
   if (isBuildableWith(value, type))
-    return builder.create<arith::ConstantOp>(loc, cast<TypedAttr>(value));
+    return arith::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
   return nullptr;
 }
 
@@ -304,7 +304,7 @@ Value mlir::arith::getZeroConstant(OpBuilder &builder, Location loc,
          "type doesn't have a zero representation");
   TypedAttr zeroAttr = builder.getZeroAttr(type);
   assert(zeroAttr && "unsupported type for zero attribute");
-  return builder.create<arith::ConstantOp>(loc, zeroAttr);
+  return arith::ConstantOp::create(builder, loc, zeroAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2334,7 +2334,7 @@ class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
     // comparison.
     rewriter.replaceOpWithNewOp<CmpIOp>(
         op, pred, intVal,
-        rewriter.create<ConstantOp>(
+        ConstantOp::create(rewriter,
             op.getLoc(), intVal.getType(),
             rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
     return success();
@@ -2373,9 +2373,9 @@ struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
         matchPattern(op.getFalseValue(), m_One())) {
       rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
           op, op.getType(),
-          rewriter.create<arith::XOrIOp>(
+          arith::XOrIOp::create(rewriter,
               op.getLoc(), op.getCondition(),
-              rewriter.create<arith::ConstantIntOp>(
+              arith::ConstantIntOp::create(rewriter,
                   op.getLoc(), op.getCondition().getType(), 1)));
       return success();
     }
@@ -2692,7 +2692,7 @@ Value mlir::arith::getIdentityValue(AtomicRMWKind op, Type resultType,
                                     bool useOnlyFiniteValue) {
   auto attr =
       getIdentityValueAttr(op, resultType, builder, loc, useOnlyFiniteValue);
-  return builder.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(builder, loc, attr);
 }
 
 /// Return the value obtained by applying the reduction operation kind
@@ -2701,33 +2701,33 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
                                   Location loc, Value lhs, Value rhs) {
   switch (op) {
   case AtomicRMWKind::addf:
-    return builder.create<arith::AddFOp>(loc, lhs, rhs);
+    return arith::AddFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::addi:
-    return builder.create<arith::AddIOp>(loc, lhs, rhs);
+    return arith::AddIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::mulf:
-    return builder.create<arith::MulFOp>(loc, lhs, rhs);
+    return arith::MulFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::muli:
-    return builder.create<arith::MulIOp>(loc, lhs, rhs);
+    return arith::MulIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maximumf:
-    return builder.create<arith::MaximumFOp>(loc, lhs, rhs);
+    return arith::MaximumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minimumf:
-    return builder.create<arith::MinimumFOp>(loc, lhs, rhs);
+    return arith::MinimumFOp::create(builder, loc, lhs, rhs);
    case AtomicRMWKind::maxnumf:
-    return builder.create<arith::MaxNumFOp>(loc, lhs, rhs);
+    return arith::MaxNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minnumf:
-    return builder.create<arith::MinNumFOp>(loc, lhs, rhs);
+    return arith::MinNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maxs:
-    return builder.create<arith::MaxSIOp>(loc, lhs, rhs);
+    return arith::MaxSIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::mins:
-    return builder.create<arith::MinSIOp>(loc, lhs, rhs);
+    return arith::MinSIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maxu:
-    return builder.create<arith::MaxUIOp>(loc, lhs, rhs);
+    return arith::MaxUIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minu:
-    return builder.create<arith::MinUIOp>(loc, lhs, rhs);
+    return arith::MinUIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::ori:
-    return builder.create<arith::OrIOp>(loc, lhs, rhs);
+    return arith::OrIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::andi:
-    return builder.create<arith::AndIOp>(loc, lhs, rhs);
+    return arith::AndIOp::create(builder, loc, lhs, rhs);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index f2e7732e8ea4a..d21da23903544 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -67,7 +67,7 @@ struct SelectOpInterface
       return state.getMemrefWithUniqueOwnership(builder, value,
                                                 value.getParentBlock());
 
-    Value ownership = builder.create<arith::SelectOp>(
+    Value ownership = arith::SelectOp::create(builder,
         op->getLoc(), selectOp.getCondition(),
         state.getOwnership(selectOp.getTrueValue(), block).getIndicator(),
         state.getOwnership(selectOp.getFalseValue(), block).getIndicator());
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
index afee162053bea..b073a31850678 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -170,10 +170,10 @@ struct SelectOpInterface
         return failure();
       if (trueBuffer.getType() != *targetType)
         trueBuffer =
-            rewriter.create<memref::CastOp>(loc, *targetType, trueBuffer);
+            memref::CastOp::create(rewriter, loc, *targetType, trueBuffer);
       if (falseBuffer.getType() != *targetType)
         falseBuffer =
-            rewriter.create<memref::CastOp>(loc, *targetType, falseBuffer);
+            memref::CastOp::create(rewriter, loc, *targetType, falseBuffer);
     }
 
     replaceOpWithNewBufferizedOp<arith::SelectOp>(
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
index 62022bfb7df1e..674bd7b8ee201 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
@@ -75,7 +75,7 @@ LogicalResult EmulateFloatPattern::matchAndRewrite(
   for (auto [res, oldType, newType] : llvm::zip_equal(
            MutableArrayRef{newResults}, op->getResultTypes(), resultTypes)) {
     if (oldType != newType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, oldType, res);
+      auto truncFOp = arith::TruncFOp::create(rewriter, loc, oldType, res);
       truncFOp.setFastmath(arith::FastMathFlags::contract);
       res = truncFOp.getResult();
     }
@@ -98,7 +98,7 @@ void mlir::arith::populateEmulateUnsupportedFloatsConversions(
   });
   converter.addTargetMaterialization(
       [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        auto extFOp = arith::ExtFOp::create(b, loc, target, input);
         extFOp.setFastmath(arith::FastMathFlags::contract);
         return extFOp;
       });
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
index d5d1559c658ff..44e896a97cb4b 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
@@ -72,7 +72,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
 
   // Scalarize the result in case of 1D vectors.
   if (shape.size() == 1)
-    return rewriter.create<vector::ExtractOp>(loc, input, lastOffset);
+    return vector::ExtractOp::create(rewriter, loc, input, lastOffset);
 
   SmallVector<int64_t> offsets(shape.size(), 0);
   offsets.back() = lastOffset;
@@ -80,7 +80,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
   sizes.back() = 1;
   SmallVector<int64_t> strides(shape.size(), 1);
 
-  return rewriter.create<vector::ExtractStridedSliceOp>(loc, input, offsets,
+  return vector::ExtractStridedSliceOp::create(rewriter, loc, input, offsets,
                                                         sizes, strides);
 }
 
@@ -107,7 +107,7 @@ static Value dropTrailingX1Dim(ConversionPatternRewriter &rewriter,
   assert(shape.back() == 1 && "Expected the last vector dim to be x1");
 
   auto newVecTy = VectorType::get(shape.drop_back(), vecTy.getElementType());
-  return rewriter.create<vector::ShapeCastOp>(loc, newVecTy, input);
+  return vector::ShapeCastOp::create(rewriter, loc, newVecTy, input);
 }
 
 /// Performs a vector shape cast to append an x1 dimension. If the
@@ -122,7 +122,7 @@ static Value appendX1Dim(ConversionPatternRewriter &rewriter, Location loc,
   auto newShape = llvm::to_vector(vecTy.getShape());
   newShape.push_back(1);
   auto newTy = VectorType::get(newShape, vecTy.getElementType());
-  return rewriter.create<vector::ShapeCastOp>(loc, newTy, input);
+  return vector::ShapeCastOp::create(rewriter, loc, newTy, input);
 }
 
 /// Inserts the `source` vector slice into the `dest` vector at offset
@@ -136,12 +136,12 @@ static Value insertLastDimSlice(ConversionPatternRewriter &rewriter,
 
   // Handle scalar source.
   if (isa<IntegerType>(source.getType()))
-    return rewriter.create<vector::InsertOp>(loc, source, dest, lastOffset);
+    return vector::InsertOp::create(rewriter, loc, source, dest, lastOffset);
 
   SmallVector<int64_t> offsets(shape.size(), 0);
   offsets.back() = lastOffset;
   SmallVector<int64_t> strides(shape.size(), 1);
-  return rewriter.create<vector::InsertStridedSliceOp>(loc, source, dest,
+  return vector::InsertStridedSliceOp::create(rewriter, loc, source, dest,
                                                        offsets, strides);
 }
 
@@ -254,12 +254,12 @@ struct ConvertAddI final : OpConversionPattern<arith::AddIOp> {
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
     auto lowSum =
-        rewriter.create<arith::AddUIExtendedOp>(loc, lhsElem0, rhsElem0);
+        arith::AddUIExtendedOp::create(rewriter, loc, lhsElem0, rhsElem0);
     Value overflowVal =
-        rewriter.create<arith::ExtUIOp>(loc, newElemTy, lowSum.getOverflow());
+        arith::ExtUIOp::create(rewriter, loc, newElemTy, lowSum.getOverflow());
 
-    Value high0 = rewriter.create<arith::AddIOp>(loc, overflowVal, lhsElem1);
-    Value high = rewriter.create<arith::AddIOp>(loc, high0, rhsElem1);
+    Value high0 = arith::AddIOp::create(rewriter, loc, overflowVal, lhsElem1);
+    Value high = arith::AddIOp::create(rewriter, loc, high0, rhsElem1);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {lowSum.getSum(), high});
@@ -293,8 +293,8 @@ struct ConvertBitwiseBinary final : OpConversionPattern<BinaryOp> {
     auto [rhsElem0, rhsElem1] =
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
-    Value resElem0 = rewriter.create<BinaryOp>(loc, lhsElem0, rhsElem0);
-    Value resElem1 = rewriter.create<BinaryOp>(loc, lhsElem1, rhsElem1);
+    Value resElem0 = BinaryOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem1 = BinaryOp::create(rewriter, loc, lhsElem1, rhsElem1);
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
     rewriter.replaceOp(op, resultVec);
@@ -346,26 +346,26 @@ struct ConvertCmpI final : OpConversionPattern<arith::CmpIOp> {
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
     Value lowCmp =
-        rewriter.create<arith::CmpIOp>(loc, lowPred, lhsElem0, rhsElem0);
+        arith::CmpIOp::create(rewriter, loc, lowPred, lhsElem0, rhsElem0);
     Value highCmp =
-        rewriter.create<arith::CmpIOp>(loc, highPred, lhsElem1, rhsElem1);
+        arith::CmpIOp::create(rewriter, loc, highPred, lhsElem1, rhsElem1);
 
     Value cmpResult{};
     switch (highPred) {
     case arith::CmpIPredicate::eq: {
-      cmpResult = rewriter.create<arith::AndIOp>(loc, lowCmp, highCmp);
+      cmpResult = arith::AndIOp::create(rewriter, loc, lowCmp, highCmp);
       break;
     }
     case arith::CmpIPredicate::ne: {
-      cmpResult = rewriter.create<arith::OrIOp>(loc, lowCmp, highCmp);
+      cmpResult = arith::OrIOp::create(rewriter, loc, lowCmp, highCmp);
       break;
     }
     default: {
       // Handle inequality checks.
-      Value highEq = rewriter.create<arith::CmpIOp>(
+      Value highEq = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, lhsElem1, rhsElem1);
       cmpResult =
-          rewriter.create<arith::SelectOp>(loc, highEq, lowCmp, highCmp);
+          arith::SelectOp::create(rewriter, loc, highEq, lowCmp, highCmp);
       break;
     }
     }
@@ -401,14 +401,14 @@ struct ConvertMulI final : OpConversionPattern<arith::MulIOp> {
     // Multiplying two i2N integers produces (at most) an i4N result, but
     // because the calculation of top i2N is not necessary, we omit it.
     auto mulLowLow =
-        rewriter.create<arith::MulUIExtendedOp>(loc, lhsElem0, rhsElem0);
-    Value mulLowHi = rewriter.create<arith::MulIOp>(loc, lhsElem0, rhsElem1);
-    Value mulHiLow = rewriter.create<arith::MulIOp>(loc, lhsElem1, rhsElem0);
+        arith::MulUIExtendedOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value mulLowHi = arith::MulIOp::create(rewriter, loc, lhsElem0, rhsElem1);
+    Value mulHiLow = arith::MulIOp::create(rewriter, loc, lhsElem1, rhsElem0);
 
     Value resLow = mulLowLow.getLow();
     Value resHi =
-        rewriter.create<arith::AddIOp>(loc, mulLowLow.getHigh(), mulLowHi);
-    resHi = rewriter.create<arith::AddIOp>(loc, resHi, mulHiLow);
+        arith::AddIOp::create(rewriter, loc, mulLowLow.getHigh(), mulLowHi);
+    resHi = arith::AddIOp::create(rewriter, loc, resHi, mulHiLow);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resLow, resHi});
@@ -443,10 +443,10 @@ struct ConvertExtSI final : OpConversionPattern<arith::ExtSIOp> {
         loc, newResultComponentTy, newOperand);
     Value operandZeroCst =
         createScalarOrSplatConstant(rewriter, loc, newResultComponentTy, 0);
-    Value signBit = rewriter.create<arith::CmpIOp>(
+    Value signBit = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, extended, operandZeroCst);
     Value signValue =
-        rewriter.create<arith::ExtSIOp>(loc, newResultComponentTy, signBit);
+        arith::ExtSIOp::create(rewriter, loc, newResultComponentTy, signBit);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {extended, signValue});
@@ -508,7 +508,7 @@ struct ConvertMaxMin final : OpConversionPattern<SourceOp> {
     // Rewrite Max*I/Min*I as compare and select over original operands. Let
     // the CmpI and Select emulation patterns handle the final legalization.
     Value cmp =
-        rewriter.create<arith::CmpIOp>(loc, CmpPred, op.getLhs(), op.getRhs());
+        arith::CmpIOp::create(rewriter, loc, CmpPred, op.getLhs(), op.getRhs());
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, op.getLhs(),
                                                  op.getRhs());
     return success();
@@ -587,7 +587,7 @@ struct ConvertIndexCastIndexToInt final : OpConversionPattern<CastOp> {
     // Sign or zero-extend the result. Let the matching conversion pattern
     // legalize the extension op.
     Value underlyingVal =
-        rewriter.create<CastOp>(loc, narrowTy, adaptor.getIn());
+        CastOp::create(rewriter, loc, narrowTy, adaptor.getIn());
     rewriter.replaceOpWithNewOp<ExtensionOp>(op, resultType, underlyingVal);
     return success();
   }
@@ -616,9 +616,9 @@ struct ConvertSelect final : OpConversionPattern<arith::SelectOp> {
     Value cond = appendX1Dim(rewriter, loc, adaptor.getCondition());
 
     Value resElem0 =
-        rewriter.create<arith::SelectOp>(loc, cond, trueElem0, falseElem0);
+        arith::SelectOp::create(rewriter, loc, cond, trueElem0, falseElem0);
     Value resElem1 =
-        rewriter.create<arith::SelectOp>(loc, cond, trueElem1, falseElem1);
+        arith::SelectOp::create(rewriter, loc, cond, trueElem1, falseElem1);
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
     rewriter.replaceOp(op, resultVec);
@@ -680,33 +680,33 @@ struct ConvertShLI final : OpConversionPattern<arith::ShLIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = rewriter.create<arith::CmpIOp>(
+    Value illegalElemShift = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem0, rhsElem0);
-    Value resElem0 = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShLIOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem0 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                       zeroCst, shiftedElem0);
 
-    Value cappedShiftAmount = rewriter.create<arith::SelectOp>(
+    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value rightShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, elemBitWidth, cappedShiftAmount);
+        arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedRight =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem0, rightShiftAmount);
+        arith::ShRUIOp::create(rewriter, loc, lhsElem0, rightShiftAmount);
     Value overshotShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, rhsElem0, elemBitWidth);
+        arith::SubIOp::create(rewriter, loc, rhsElem0, elemBitWidth);
     Value shiftedLeft =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem0, overshotShiftAmount);
+        arith::ShLIOp::create(rewriter, loc, lhsElem0, overshotShiftAmount);
 
     Value shiftedElem1 =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem1, rhsElem0);
-    Value resElem1High = rewriter.create<arith::SelectOp>(
+        arith::ShLIOp::create(rewriter, loc, lhsElem1, rhsElem0);
+    Value resElem1High = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, zeroCst, shiftedElem1);
-    Value resElem1Low = rewriter.create<arith::SelectOp>(
+    Value resElem1Low = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, shiftedLeft, shiftedRight);
     Value resElem1 =
-        rewriter.create<arith::OrIOp>(loc, resElem1Low, resElem1High);
+        arith::OrIOp::create(rewriter, loc, resElem1Low, resElem1High);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
@@ -769,33 +769,33 @@ struct ConvertShRUI final : OpConversionPattern<arith::ShRUIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = rewriter.create<arith::CmpIOp>(
+    Value illegalElemShift = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem0, rhsElem0);
-    Value resElem0Low = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShRUIOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem0Low = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                          zeroCst, shiftedElem0);
     Value shiftedElem1 =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem1, rhsElem0);
-    Value resElem1 = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShRUIOp::create(rewriter, loc, lhsElem1, rhsElem0);
+    Value resElem1 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                       zeroCst, shiftedElem1);
 
-    Value cappedShiftAmount = rewriter.create<arith::SelectOp>(
+    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value leftShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, elemBitWidth, cappedShiftAmount);
+        arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedLeft =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem1, leftShiftAmount);
+        arith::ShLIOp::create(rewriter, loc, lhsElem1, leftShiftAmount);
     Value overshotShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, rhsElem0, elemBitWidth);
+        arith::SubIOp::create(rewriter, loc, rhsElem0, elemBitWidth);
     Value shiftedRight =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem1, overshotShiftAmount);
+        arith::ShRUIOp::create(rewriter, loc, lhsElem1, overshotShiftAmount);
 
-    Value resElem0High = rewriter.create<arith::SelectOp>(
+    Value resElem0High = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, shiftedRight, shiftedLeft);
     Value resElem0 =
-        rewriter.create<arith::OrIOp>(loc, resElem0Low, resElem0High);
+        arith::OrIOp::create(rewriter, loc, resElem0Low, resElem0High);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
@@ -832,32 +832,32 @@ struct ConvertShRSI final : OpConversionPattern<arith::ShRSIOp> {
     // Perform as many ops over the narrow integer type as possible and let the
     // other emulation patterns convert the rest.
     Value elemZero = createScalarOrSplatConstant(rewriter, loc, narrowTy, 0);
-    Value signBit = rewriter.create<arith::CmpIOp>(
+    Value signBit = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, lhsElem1, elemZero);
     signBit = dropTrailingX1Dim(rewriter, loc, signBit);
 
     // Create a bit pattern of either all ones or all zeros. Then shift it left
     // to calculate the sign extension bits created by shifting the original
     // sign bit right.
-    Value allSign = rewriter.create<arith::ExtSIOp>(loc, oldTy, signBit);
+    Value allSign = arith::ExtSIOp::create(rewriter, loc, oldTy, signBit);
     Value maxShift =
         createScalarOrSplatConstant(rewriter, loc, narrowTy, origBitwidth);
     Value numNonSignExtBits =
-        rewriter.create<arith::SubIOp>(loc, maxShift, rhsElem0);
+        arith::SubIOp::create(rewriter, loc, maxShift, rhsElem0);
     numNonSignExtBits = dropTrailingX1Dim(rewriter, loc, numNonSignExtBits);
     numNonSignExtBits =
-        rewriter.create<arith::ExtUIOp>(loc, oldTy, numNonSignExtBits);
+        arith::ExtUIOp::create(rewriter, loc, oldTy, numNonSignExtBits);
     Value signBits =
-        rewriter.create<arith::ShLIOp>(loc, allSign, numNonSignExtBits);
+        arith::ShLIOp::create(rewriter, loc, allSign, numNonSignExtBits);
 
     // Use original arguments to create the right shift.
     Value shrui =
-        rewriter.create<arith::ShRUIOp>(loc, op.getLhs(), op.getRhs());
-    Value shrsi = rewriter.create<arith::OrIOp>(loc, shrui, signBits);
+        arith::ShRUIOp::create(rewriter, loc, op.getLhs(), op.getRhs());
+    Value shrsi = arith::OrIOp::create(rewriter, loc, shrui, signBits);
 
     // Handle shifting by zero. This is necessary when the `signBits` shift is
     // invalid.
-    Value isNoop = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value isNoop = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                   rhsElem0, elemZero);
     isNoop = dropTrailingX1Dim(rewriter, loc, isNoop);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNoop, op.getLhs(),
@@ -892,14 +892,14 @@ struct ConvertSubI final : OpConversionPattern<arith::SubIOp> {
 
     // Emulates LHS - RHS by [LHS0 - RHS0, LHS1 - RHS1 - CARRY] where
     // CARRY is 1 or 0.
-    Value low = rewriter.create<arith::SubIOp>(loc, lhsElem0, rhsElem0);
+    Value low = arith::SubIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     // We have a carry if lhsElem0 < rhsElem0.
-    Value carry0 = rewriter.create<arith::CmpIOp>(
+    Value carry0 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, lhsElem0, rhsElem0);
-    Value carryVal = rewriter.create<arith::ExtUIOp>(loc, newElemTy, carry0);
+    Value carryVal = arith::ExtUIOp::create(rewriter, loc, newElemTy, carry0);
 
-    Value high0 = rewriter.create<arith::SubIOp>(loc, lhsElem1, carryVal);
-    Value high = rewriter.create<arith::SubIOp>(loc, high0, rhsElem1);
+    Value high0 = arith::SubIOp::create(rewriter, loc, lhsElem1, carryVal);
+    Value high = arith::SubIOp::create(rewriter, loc, high0, rhsElem1);
 
     Value resultVec = constructResultVector(rewriter, loc, newTy, {low, high});
     rewriter.replaceOp(op, resultVec);
@@ -933,13 +933,13 @@ struct ConvertSIToFP final : OpConversionPattern<arith::SIToFPOp> {
     // result or not based on that sign bit. We implement negation by
     // subtracting from zero. Note that this relies on the the other conversion
     // patterns to legalize created ops and narrow the bit widths.
-    Value isNeg = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+    Value isNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                  in, zeroCst);
-    Value neg = rewriter.create<arith::SubIOp>(loc, zeroCst, in);
-    Value abs = rewriter.create<arith::SelectOp>(loc, isNeg, neg, in);
+    Value neg = arith::SubIOp::create(rewriter, loc, zeroCst, in);
+    Value abs = arith::SelectOp::create(rewriter, loc, isNeg, neg, in);
 
-    Value absResult = rewriter.create<arith::UIToFPOp>(loc, op.getType(), abs);
-    Value negResult = rewriter.create<arith::NegFOp>(loc, absResult);
+    Value absResult = arith::UIToFPOp::create(rewriter, loc, op.getType(), abs);
+    Value negResult = arith::NegFOp::create(rewriter, loc, absResult);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNeg, negResult,
                                                  absResult);
     return success();
@@ -985,13 +985,13 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     //
     // Note 2: We do not strictly need the `hi == 0`, case, but it makes
     // constant folding easier.
-    Value hiEqZero = rewriter.create<arith::CmpIOp>(
+    Value hiEqZero = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, hiInt, zeroCst);
 
     Type resultTy = op.getType();
     Type resultElemTy = getElementTypeOrSelf(resultTy);
-    Value lowFp = rewriter.create<arith::UIToFPOp>(loc, resultTy, lowInt);
-    Value hiFp = rewriter.create<arith::UIToFPOp>(loc, resultTy, hiInt);
+    Value lowFp = arith::UIToFPOp::create(rewriter, loc, resultTy, lowInt);
+    Value hiFp = arith::UIToFPOp::create(rewriter, loc, resultTy, hiInt);
 
     int64_t pow2Int = int64_t(1) << newBitWidth;
     TypedAttr pow2Attr =
@@ -999,10 +999,10 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     if (auto vecTy = dyn_cast<VectorType>(resultTy))
       pow2Attr = SplatElementsAttr::get(vecTy, pow2Attr);
 
-    Value pow2Val = rewriter.create<arith::ConstantOp>(loc, resultTy, pow2Attr);
+    Value pow2Val = arith::ConstantOp::create(rewriter, loc, resultTy, pow2Attr);
 
-    Value hiVal = rewriter.create<arith::MulFOp>(loc, hiFp, pow2Val);
-    Value result = rewriter.create<arith::AddFOp>(loc, lowFp, hiVal);
+    Value hiVal = arith::MulFOp::create(rewriter, loc, hiFp, pow2Val);
+    Value result = arith::AddFOp::create(rewriter, loc, lowFp, hiVal);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, hiEqZero, lowFp, result);
     return success();
@@ -1037,22 +1037,22 @@ struct ConvertFPToSI final : OpConversionPattern<arith::FPToSIOp> {
     // result is UB.
 
     TypedAttr zeroAttr = rewriter.getZeroAttr(fpTy);
-    Value zeroCst = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
+    Value zeroCst = arith::ConstantOp::create(rewriter, loc, zeroAttr);
     Value zeroCstInt = createScalarOrSplatConstant(rewriter, loc, intTy, 0);
 
     // Get the absolute value. One could have used math.absf here, but that
     // introduces an extra dependency.
-    Value isNeg = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OLT,
+    Value isNeg = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OLT,
                                                  inFp, zeroCst);
-    Value negInFp = rewriter.create<arith::NegFOp>(loc, inFp);
+    Value negInFp = arith::NegFOp::create(rewriter, loc, inFp);
 
-    Value absVal = rewriter.create<arith::SelectOp>(loc, isNeg, negInFp, inFp);
+    Value absVal = arith::SelectOp::create(rewriter, loc, isNeg, negInFp, inFp);
 
     // Defer the absolute value to fptoui.
-    Value res = rewriter.create<arith::FPToUIOp>(loc, intTy, absVal);
+    Value res = arith::FPToUIOp::create(rewriter, loc, intTy, absVal);
 
     // Negate the value if < 0 .
-    Value neg = rewriter.create<arith::SubIOp>(loc, zeroCstInt, res);
+    Value neg = arith::SubIOp::create(rewriter, loc, zeroCstInt, res);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNeg, neg, res);
     return success();
@@ -1109,17 +1109,17 @@ struct ConvertFPToUI final : OpConversionPattern<arith::FPToUIOp> {
     if (auto vecType = dyn_cast<VectorType>(fpTy))
       powBitwidthAttr = SplatElementsAttr::get(vecType, powBitwidthAttr);
     Value powBitwidthFloatCst =
-        rewriter.create<arith::ConstantOp>(loc, powBitwidthAttr);
+        arith::ConstantOp::create(rewriter, loc, powBitwidthAttr);
 
     Value fpDivPowBitwidth =
-        rewriter.create<arith::DivFOp>(loc, inFp, powBitwidthFloatCst);
+        arith::DivFOp::create(rewriter, loc, inFp, powBitwidthFloatCst);
     Value resHigh =
-        rewriter.create<arith::FPToUIOp>(loc, newHalfType, fpDivPowBitwidth);
+        arith::FPToUIOp::create(rewriter, loc, newHalfType, fpDivPowBitwidth);
     // Calculate fp - resHigh * 2^N by getting the remainder of the division
     Value remainder =
-        rewriter.create<arith::RemFOp>(loc, inFp, powBitwidthFloatCst);
+        arith::RemFOp::create(rewriter, loc, inFp, powBitwidthFloatCst);
     Value resLow =
-        rewriter.create<arith::FPToUIOp>(loc, newHalfType, remainder);
+        arith::FPToUIOp::create(rewriter, loc, newHalfType, remainder);
 
     Value high = appendX1Dim(rewriter, loc, resHigh);
     Value low = appendX1Dim(rewriter, loc, resLow);
diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
index f497d2db3bf7c..dcdb0d1687504 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
@@ -29,10 +29,10 @@ static Value createConst(Location loc, Type type, int value,
                          PatternRewriter &rewriter) {
   auto attr = rewriter.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return rewriter.create<arith::ConstantOp>(
+    return arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
-  return rewriter.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(rewriter, loc, attr);
 }
 
 /// Create a float constant.
@@ -40,11 +40,11 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                               PatternRewriter &rewriter) {
   auto attr = rewriter.getFloatAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return rewriter.create<arith::ConstantOp>(
+    return arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return rewriter.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(rewriter, loc, attr);
 }
 
 /// Creates shapedType using shape from cloneFrom and base type from cloneTo
@@ -68,11 +68,11 @@ struct CeilDivUIOpConverter : public OpRewritePattern<arith::CeilDivUIOp> {
     Value b = op.getRhs();
     Value zero = createConst(loc, a.getType(), 0, rewriter);
     Value compare =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, a, zero);
     Value one = createConst(loc, a.getType(), 1, rewriter);
-    Value minusOne = rewriter.create<arith::SubIOp>(loc, a, one);
-    Value quotient = rewriter.create<arith::DivUIOp>(loc, minusOne, b);
-    Value plusOne = rewriter.create<arith::AddIOp>(loc, quotient, one);
+    Value minusOne = arith::SubIOp::create(rewriter, loc, a, one);
+    Value quotient = arith::DivUIOp::create(rewriter, loc, minusOne, b);
+    Value plusOne = arith::AddIOp::create(rewriter, loc, quotient, one);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, compare, zero, plusOne);
     return success();
   }
@@ -97,22 +97,22 @@ struct CeilDivSIOpConverter : public OpRewritePattern<arith::CeilDivSIOp> {
     Value zero = createConst(loc, type, 0, rewriter);
     Value one = createConst(loc, type, 1, rewriter);
 
-    Value quotient = rewriter.create<arith::DivSIOp>(loc, a, b);
-    Value product = rewriter.create<arith::MulIOp>(loc, quotient, b);
-    Value notEqualDivisor = rewriter.create<arith::CmpIOp>(
+    Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
+    Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
+    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, a, product);
 
     Value aNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
     Value bNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, b, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
 
-    Value signEqual = rewriter.create<arith::CmpIOp>(
+    Value signEqual = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, aNeg, bNeg);
     Value cond =
-        rewriter.create<arith::AndIOp>(loc, notEqualDivisor, signEqual);
+        arith::AndIOp::create(rewriter, loc, notEqualDivisor, signEqual);
 
-    Value quotientPlusOne = rewriter.create<arith::AddIOp>(loc, quotient, one);
+    Value quotientPlusOne = arith::AddIOp::create(rewriter, loc, quotient, one);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cond, quotientPlusOne,
                                                  quotient);
@@ -136,25 +136,25 @@ struct FloorDivSIOpConverter : public OpRewritePattern<arith::FloorDivSIOp> {
     Value a = op.getLhs();
     Value b = op.getRhs();
 
-    Value quotient = rewriter.create<arith::DivSIOp>(loc, a, b);
-    Value product = rewriter.create<arith::MulIOp>(loc, quotient, b);
-    Value notEqualDivisor = rewriter.create<arith::CmpIOp>(
+    Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
+    Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
+    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, a, product);
     Value zero = createConst(loc, type, 0, rewriter);
 
     Value aNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
     Value bNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, b, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
 
-    Value signOpposite = rewriter.create<arith::CmpIOp>(
+    Value signOpposite = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, aNeg, bNeg);
     Value cond =
-        rewriter.create<arith::AndIOp>(loc, notEqualDivisor, signOpposite);
+        arith::AndIOp::create(rewriter, loc, notEqualDivisor, signOpposite);
 
     Value minusOne = createConst(loc, type, -1, rewriter);
     Value quotientMinusOne =
-        rewriter.create<arith::AddIOp>(loc, quotient, minusOne);
+        arith::AddIOp::create(rewriter, loc, quotient, minusOne);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cond, quotientMinusOne,
                                                  quotient);
@@ -172,7 +172,7 @@ struct MaxMinIOpConverter : public OpRewritePattern<OpTy> {
     Value lhs = op.getLhs();
     Value rhs = op.getRhs();
 
-    Value cmp = rewriter.create<arith::CmpIOp>(op.getLoc(), pred, lhs, rhs);
+    Value cmp = arith::CmpIOp::create(rewriter, op.getLoc(), pred, lhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, lhs, rhs);
     return success();
   }
@@ -193,11 +193,11 @@ struct MaximumMinimumFOpConverter : public OpRewritePattern<OpTy> {
     static_assert(pred == arith::CmpFPredicate::UGT ||
                       pred == arith::CmpFPredicate::ULT,
                   "pred must be either UGT or ULT");
-    Value cmp = rewriter.create<arith::CmpFOp>(loc, pred, lhs, rhs);
-    Value select = rewriter.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+    Value cmp = arith::CmpFOp::create(rewriter, loc, pred, lhs, rhs);
+    Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'.
-    Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
+    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
                                                  rhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
@@ -219,11 +219,11 @@ struct MaxNumMinNumFOpConverter : public OpRewritePattern<OpTy> {
     static_assert(pred == arith::CmpFPredicate::UGT ||
                       pred == arith::CmpFPredicate::ULT,
                   "pred must be either UGT or ULT");
-    Value cmp = rewriter.create<arith::CmpFOp>(loc, pred, lhs, rhs);
-    Value select = rewriter.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+    Value cmp = arith::CmpFOp::create(rewriter, loc, pred, lhs, rhs);
+    Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where lhs is NaN: 'isNaN(lhs) ? rhs : select'.
-    Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
+    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
                                                  lhs, lhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
@@ -248,12 +248,12 @@ struct BFloat16ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type i16Ty = cloneToShapedType(operandTy, b.getI16Type());
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
 
-    Value bitcast = b.create<arith::BitcastOp>(i16Ty, operand);
-    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
+    Value bitcast = arith::BitcastOp::create(b, i16Ty, operand);
+    Value exti = arith::ExtUIOp::create(b, i32Ty, bitcast);
 
     Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
-    Value shl = b.create<arith::ShLIOp>(exti, c16);
-    Value result = b.create<arith::BitcastOp>(resultTy, shl);
+    Value shl = arith::ShLIOp::create(b, exti, c16);
+    Value result = arith::BitcastOp::create(b, resultTy, shl);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -297,7 +297,7 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     // exponent bits, that simple truncation is the desired outcome for
     // infinities.
     Value isNan =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNE, operand, operand);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNE, operand, operand);
     // Constant used to make the rounding bias.
     Value c7FFF = createConst(op.getLoc(), i32Ty, 0x7fff, rewriter);
     // Constant used to generate a quiet NaN.
@@ -306,30 +306,30 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
     Value c1 = createConst(op.getLoc(), i32Ty, 1, rewriter);
     // Reinterpret the input f32 value as bits.
-    Value bitcast = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value bitcast = arith::BitcastOp::create(b, i32Ty, operand);
     // Read bit 16 as a value in {0,1}.
     Value bit16 =
-        b.create<arith::AndIOp>(b.create<arith::ShRUIOp>(bitcast, c16), c1);
+        arith::AndIOp::create(b, arith::ShRUIOp::create(b, bitcast, c16), c1);
     // Determine the rounding bias to add as either 0x7fff or 0x8000 depending
     // on bit 16, implementing the tie-breaking "to nearest even".
-    Value roundingBias = b.create<arith::AddIOp>(bit16, c7FFF);
+    Value roundingBias = arith::AddIOp::create(b, bit16, c7FFF);
     // Add the rounding bias. Generally we want this to be added to the
     // mantissa, but nothing prevents this to from carrying into the exponent
     // bits, which would feel like a bug, but this is the magic trick here:
     // when that happens, the mantissa gets reset to zero and the exponent
     // gets incremented by the carry... which is actually exactly what we
     // want.
-    Value biased = b.create<arith::AddIOp>(bitcast, roundingBias);
+    Value biased = arith::AddIOp::create(b, bitcast, roundingBias);
     // Now that the rounding-bias has been added, truncating the low bits
     // yields the correctly rounded result.
-    Value biasedAndShifted = b.create<arith::ShRUIOp>(biased, c16);
+    Value biasedAndShifted = arith::ShRUIOp::create(b, biased, c16);
     Value normalCaseResultI16 =
-        b.create<arith::TruncIOp>(i16Ty, biasedAndShifted);
+        arith::TruncIOp::create(b, i16Ty, biasedAndShifted);
     // Select either the above-computed result, or a quiet NaN constant
     // if the input was NaN.
     Value select =
-        b.create<arith::SelectOp>(isNan, c7FC0I16, normalCaseResultI16);
-    Value result = b.create<arith::BitcastOp>(resultTy, select);
+        arith::SelectOp::create(b, isNan, c7FC0I16, normalCaseResultI16);
+    Value result = arith::BitcastOp::create(b, resultTy, select);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -382,7 +382,7 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
     Type i4Ty = cloneToShapedType(operandTy, b.getI4Type());
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
-    Value i4Bits = b.create<arith::BitcastOp>(i4Ty, operand);
+    Value i4Bits = arith::BitcastOp::create(b, i4Ty, operand);
 
     Value c0x0 = createConst(loc, i4Ty, 0x0, rewriter);
     Value c0x1 = createConst(loc, i4Ty, 0x1, rewriter);
@@ -391,38 +391,38 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
 
     // Set last Exponent bit and Mantissa.
     Value c0x00000014 = createConst(loc, i32Ty, 0x14, rewriter);
-    Value bits1To24 = b.create<arith::ShLIOp>(i4Bits, c0x2);
+    Value bits1To24 = arith::ShLIOp::create(b, i4Bits, c0x2);
     Value isHalf =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x1);
-    bits1To24 = b.create<arith::SelectOp>(isHalf, c0x0, bits1To24);
-    bits1To24 = b.create<arith::ExtUIOp>(i32Ty, bits1To24);
-    bits1To24 = b.create<arith::ShLIOp>(bits1To24, c0x00000014);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, i4Bits, c0x1);
+    bits1To24 = arith::SelectOp::create(b, isHalf, c0x0, bits1To24);
+    bits1To24 = arith::ExtUIOp::create(b, i32Ty, bits1To24);
+    bits1To24 = arith::ShLIOp::create(b, bits1To24, c0x00000014);
 
     // Set first 7 bits of Exponent.
     Value zeroExpBits = createConst(loc, i32Ty, 0x00000000, rewriter);
     Value highExpBits = createConst(loc, i32Ty, 0x40000000, rewriter);
     Value lowExpBits = createConst(loc, i32Ty, 0x3f000000, rewriter);
     Value useLargerExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x4);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x4);
     Value bits25To31 =
-        b.create<arith::SelectOp>(useLargerExp, highExpBits, lowExpBits);
+        arith::SelectOp::create(b, useLargerExp, highExpBits, lowExpBits);
     Value zeroExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x0);
-    bits25To31 = b.create<arith::SelectOp>(zeroExp, zeroExpBits, bits25To31);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, i4Bits, c0x0);
+    bits25To31 = arith::SelectOp::create(b, zeroExp, zeroExpBits, bits25To31);
 
     // Set sign.
     Value c0x80000000 = createConst(loc, i32Ty, 0x80000000, rewriter);
     Value c0x8 = createConst(loc, i4Ty, 0x8, rewriter);
     Value negative =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x8);
-    Value bit32 = b.create<arith::SelectOp>(negative, c0x80000000, zeroExpBits);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x8);
+    Value bit32 = arith::SelectOp::create(b, negative, c0x80000000, zeroExpBits);
 
     // Add segments together.
-    Value bits1To31 = b.create<arith::AddIOp>(bits1To24, bits25To31);
-    Value bits1To32 = b.create<arith::AddIOp>(bits1To31, bit32);
-    Value result = b.create<arith::BitcastOp>(f32Ty, bits1To32);
+    Value bits1To31 = arith::AddIOp::create(b, bits1To24, bits25To31);
+    Value bits1To32 = arith::AddIOp::create(b, bits1To31, bit32);
+    Value result = arith::BitcastOp::create(b, f32Ty, bits1To32);
     if (!isa<Float32Type>(resultETy))
-      result = b.create<arith::TruncFOp>(resultTy, result);
+      result = arith::TruncFOp::create(b, resultTy, result);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -448,25 +448,25 @@ struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
-    Value bitcast = b.create<arith::BitcastOp>(i8Ty, operand);
+    Value bitcast = arith::BitcastOp::create(b, i8Ty, operand);
     // create constants for NaNs
     Value cF8NaN = createConst(op.getLoc(), i8Ty, 0xff, rewriter);
     Value cF32NaN = createConst(op.getLoc(), i32Ty, 0xffffffff, rewriter);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
 
-    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
-    Value f32Bits = b.create<arith::ShLIOp>(exti, cF32MantissaWidth);
+    Value exti = arith::ExtUIOp::create(b, i32Ty, bitcast);
+    Value f32Bits = arith::ShLIOp::create(b, exti, cF32MantissaWidth);
 
     Value isNan =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bitcast, cF8NaN);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, bitcast, cF8NaN);
     // select for NaNs
-    f32Bits = b.create<arith::SelectOp>(isNan, cF32NaN, f32Bits);
-    Value result = b.create<arith::BitcastOp>(f32Ty, f32Bits);
+    f32Bits = arith::SelectOp::create(b, isNan, cF32NaN, f32Bits);
+    Value result = arith::BitcastOp::create(b, f32Ty, f32Bits);
     if (resultETy.getIntOrFloatBitWidth() < 32) {
-      result = b.create<arith::TruncFOp>(resultTy, result, nullptr,
+      result = arith::TruncFOp::create(b, resultTy, result, nullptr,
                                          op.getFastmathAttr());
     } else if (resultETy.getIntOrFloatBitWidth() > 32) {
-      result = b.create<arith::ExtFOp>(resultTy, result, op.getFastmathAttr());
+      result = arith::ExtFOp::create(b, resultTy, result, op.getFastmathAttr());
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -519,7 +519,7 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (!isa<Float32Type>(operandETy))
-      operand = b.create<arith::ExtFOp>(f32Ty, operand);
+      operand = arith::ExtFOp::create(b, f32Ty, operand);
     if (!isa<Float4E2M1FNType>(resultETy))
       return rewriter.notifyMatchFailure(op, "not a trunc of F4E2M1FN");
 
@@ -533,65 +533,65 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     // Step 0: Clamp to bounds.
     Value cHigherBound = createFloatConst(loc, f32Ty, APFloat(6.0f), rewriter);
     Value cLowerBound = createFloatConst(loc, f32Ty, APFloat(-6.0f), rewriter);
-    Value operandClamped = b.create<arith::MinNumFOp>(cHigherBound, operand);
-    operandClamped = b.create<arith::MaxNumFOp>(cLowerBound, operandClamped);
-    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operandClamped);
+    Value operandClamped = arith::MinNumFOp::create(b, cHigherBound, operand);
+    operandClamped = arith::MaxNumFOp::create(b, cLowerBound, operandClamped);
+    Value f32Bits = arith::BitcastOp::create(b, i32Ty, operandClamped);
 
     // Step 1: Set sign bit.
     Value cF32ExpManWidth = createConst(loc, i32Ty, 31, rewriter); // 23
-    Value f32Sign = b.create<arith::ShRUIOp>(f32Bits, cF32ExpManWidth);
-    Value f4Sign = b.create<arith::TruncIOp>(i4Ty, f32Sign);
-    Value f4Bits = b.create<arith::ShLIOp>(f4Sign, c0x3);
+    Value f32Sign = arith::ShRUIOp::create(b, f32Bits, cF32ExpManWidth);
+    Value f4Sign = arith::TruncIOp::create(b, i4Ty, f32Sign);
+    Value f4Bits = arith::ShLIOp::create(b, f4Sign, c0x3);
 
     // Step 2: Convert exponent by adjusting bias.
     Value biasAdjustment = createConst(loc, i32Ty, 0x7e, rewriter);
     Value cF4MantissaWidth = c0x1;                                   // 1
     Value cF32MantissaWidth = createConst(loc, i32Ty, 23, rewriter); // 23
-    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
+    Value f32SignExp = arith::ShRUIOp::create(b, f32Bits, cF32MantissaWidth);
     Value biasAdjustedSignExp =
-        b.create<arith::SubIOp>(f32SignExp, biasAdjustment);
-    Value f4Exp = b.create<arith::TruncIOp>(i4Ty, biasAdjustedSignExp);
-    f4Exp = b.create<arith::ShLIOp>(f4Exp, cF4MantissaWidth);
-    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Exp);
+        arith::SubIOp::create(b, f32SignExp, biasAdjustment);
+    Value f4Exp = arith::TruncIOp::create(b, i4Ty, biasAdjustedSignExp);
+    f4Exp = arith::ShLIOp::create(b, f4Exp, cF4MantissaWidth);
+    f4Bits = arith::AddIOp::create(b, f4Bits, f4Exp);
 
     // Step 3: Set mantissa to first bit.
     Value cF32FirstBitMask = createConst(loc, i32Ty, 0x400000, rewriter);
-    Value man1Bit = b.create<arith::AndIOp>(f32Bits, cF32FirstBitMask);
-    man1Bit = b.create<arith::ShRUIOp>(man1Bit, c0x00000016);
-    Value f4Man = b.create<arith::TruncIOp>(i4Ty, man1Bit);
-    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Man);
+    Value man1Bit = arith::AndIOp::create(b, f32Bits, cF32FirstBitMask);
+    man1Bit = arith::ShRUIOp::create(b, man1Bit, c0x00000016);
+    Value f4Man = arith::TruncIOp::create(b, i4Ty, man1Bit);
+    f4Bits = arith::AddIOp::create(b, f4Bits, f4Man);
 
     // Step 4: Special consideration for conversion to 0.5.
     Value cF32MantissaMask = createConst(loc, i32Ty, 0x7fffff, rewriter);
-    Value f8Exp = b.create<arith::TruncIOp>(i8Ty, biasAdjustedSignExp);
+    Value f8Exp = arith::TruncIOp::create(b, i8Ty, biasAdjustedSignExp);
     Value isSubnormal =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::sle, f8Exp, c0x00);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::sle, f8Exp, c0x00);
     Value isNegOneExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0xff);
-    Value man23Bits = b.create<arith::AndIOp>(f32Bits, cF32MantissaMask);
-    Value isNonZeroMan = b.create<arith::CmpIOp>(arith::CmpIPredicate::ugt,
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0xff);
+    Value man23Bits = arith::AndIOp::create(b, f32Bits, cF32MantissaMask);
+    Value isNonZeroMan = arith::CmpIOp::create(b, arith::CmpIPredicate::ugt,
                                                  man23Bits, zeroExpBits);
-    Value roundToHalf = b.create<arith::AndIOp>(isNegOneExp, isNonZeroMan);
+    Value roundToHalf = arith::AndIOp::create(b, isNegOneExp, isNonZeroMan);
     Value isZeroExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0x00);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0x00);
     Value subnormalF4Bits = createConst(loc, i4Ty, 0xf, rewriter);
     Value halfF4Bits = createConst(loc, i4Ty, 0x0, rewriter);
     Value subResult =
-        b.create<arith::SelectOp>(isSubnormal, subnormalF4Bits, f4Bits);
-    subResult = b.create<arith::SelectOp>(roundToHalf, halfF4Bits, subResult);
-    f4Bits = b.create<arith::SelectOp>(isZeroExp, f4Bits, subResult);
+        arith::SelectOp::create(b, isSubnormal, subnormalF4Bits, f4Bits);
+    subResult = arith::SelectOp::create(b, roundToHalf, halfF4Bits, subResult);
+    f4Bits = arith::SelectOp::create(b, isZeroExp, f4Bits, subResult);
 
     // Step 5: Round up if necessary.
     Value cF32Last22BitMask = createConst(loc, i32Ty, 0x3fffff, rewriter);
     Value cRound = createConst(loc, i32Ty, 0x200000, rewriter); // 010 0000...
-    Value man22Bits = b.create<arith::AndIOp>(f32Bits, cF32Last22BitMask);
+    Value man22Bits = arith::AndIOp::create(b, f32Bits, cF32Last22BitMask);
     Value shouldRound =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, man22Bits, cRound);
-    shouldRound = b.create<arith::OrIOp>(shouldRound, isSubnormal);
-    Value roundedF4Bits = b.create<arith::AddIOp>(f4Bits, c0x1);
-    f4Bits = b.create<arith::SelectOp>(shouldRound, roundedF4Bits, f4Bits);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, man22Bits, cRound);
+    shouldRound = arith::OrIOp::create(b, shouldRound, isSubnormal);
+    Value roundedF4Bits = arith::AddIOp::create(b, f4Bits, c0x1);
+    f4Bits = arith::SelectOp::create(b, shouldRound, roundedF4Bits, f4Bits);
 
-    Value result = b.create<arith::BitcastOp>(resultTy, f4Bits);
+    Value result = arith::BitcastOp::create(b, resultTy, f4Bits);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -626,16 +626,16 @@ struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (operandETy.getIntOrFloatBitWidth() < 32) {
-      operand = b.create<arith::ExtFOp>(f32Ty, operand, op.getFastmathAttr());
+      operand = arith::ExtFOp::create(b, f32Ty, operand, op.getFastmathAttr());
     } else if (operandETy.getIntOrFloatBitWidth() > 32) {
-      operand = b.create<arith::TruncFOp>(
+      operand = arith::TruncFOp::create(b,
           f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
     }
-    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value f32Bits = arith::BitcastOp::create(b, i32Ty, operand);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
-    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
-    Value exp8Bits = b.create<arith::TruncIOp>(i8Ty, f32SignExp);
-    Value result = b.create<arith::BitcastOp>(resultTy, exp8Bits);
+    Value f32SignExp = arith::ShRUIOp::create(b, f32Bits, cF32MantissaWidth);
+    Value exp8Bits = arith::TruncIOp::create(b, i8Ty, f32SignExp);
+    Value result = arith::BitcastOp::create(b, resultTy, exp8Bits);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -654,7 +654,7 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
     if (scaleETy.getIntOrFloatBitWidth() >= 16) {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
-      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+      scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
                                                op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
@@ -666,11 +666,11 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
     // extf on scale will essentially create floating point number
     // of type resulTy that is 2^scale and will also propagate NaNs
     Value scaleExt =
-        b.create<arith::ExtFOp>(resultTy, scaleOperand, op.getFastmathAttr());
+        arith::ExtFOp::create(b, resultTy, scaleOperand, op.getFastmathAttr());
     Value inputExt =
-        b.create<arith::ExtFOp>(resultTy, inputOperand, op.getFastmathAttr());
+        arith::ExtFOp::create(b, resultTy, inputOperand, op.getFastmathAttr());
     Value result =
-        b.create<arith::MulFOp>(inputExt, scaleExt, op.getFastmathAttr());
+        arith::MulFOp::create(b, inputExt, scaleExt, op.getFastmathAttr());
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -695,7 +695,7 @@ struct ScalingTruncFOpConverter
     if (scaleETy.getIntOrFloatBitWidth() >= 16) {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
-      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+      scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
                                                op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
@@ -708,10 +708,10 @@ struct ScalingTruncFOpConverter
     // this will create a floating point number of type
     // inputTy that is 2^scale and will also propagate NaNs
     scaleOperand =
-        b.create<arith::ExtFOp>(inputTy, scaleOperand, op.getFastmathAttr());
-    Value result = b.create<arith::DivFOp>(inputOperand, scaleOperand,
+        arith::ExtFOp::create(b, inputTy, scaleOperand, op.getFastmathAttr());
+    Value result = arith::DivFOp::create(b, inputOperand, scaleOperand,
                                            op.getFastmathAttr());
-    Value resultCast = b.create<arith::TruncFOp>(
+    Value resultCast = arith::TruncFOp::create(b,
         resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
     rewriter.replaceOp(op, resultCast);
     return success();
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index f2f93883eb2b7..777ff0ecaa314 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -305,18 +305,18 @@ static Value doCast(OpBuilder &builder, Location loc, Value src, Type dstType,
 
   if (isa<IndexType>(srcElemType) || isa<IndexType>(dstElemType)) {
     if (castKind == CastKind::Signed)
-      return builder.create<arith::IndexCastOp>(loc, dstType, src);
-    return builder.create<arith::IndexCastUIOp>(loc, dstType, src);
+      return arith::IndexCastOp::create(builder, loc, dstType, src);
+    return arith::IndexCastUIOp::create(builder, loc, dstType, src);
   }
 
   auto srcInt = cast<IntegerType>(srcElemType);
   auto dstInt = cast<IntegerType>(dstElemType);
   if (dstInt.getWidth() < srcInt.getWidth())
-    return builder.create<arith::TruncIOp>(loc, dstType, src);
+    return arith::TruncIOp::create(builder, loc, dstType, src);
 
   if (castKind == CastKind::Signed)
-    return builder.create<arith::ExtSIOp>(loc, dstType, src);
-  return builder.create<arith::ExtUIOp>(loc, dstType, src);
+    return arith::ExtSIOp::create(builder, loc, dstType, src);
+  return arith::ExtUIOp::create(builder, loc, dstType, src);
 }
 
 struct NarrowElementwise final : OpTraitRewritePattern<OpTrait::Elementwise> {
diff --git a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
index 5fb7953f93700..904aab33345f0 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
@@ -23,7 +23,7 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
   std::function<Value(AffineExpr)> buildExpr = [&](AffineExpr e) -> Value {
     switch (e.getKind()) {
     case AffineExprKind::Constant:
-      return b.create<ConstantIndexOp>(loc,
+      return ConstantIndexOp::create(b, loc,
                                        cast<AffineConstantExpr>(e).getValue());
     case AffineExprKind::DimId:
       return operands[cast<AffineDimExpr>(e).getPosition()];
@@ -32,27 +32,27 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
                       map.getNumDims()];
     case AffineExprKind::Add: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<AddIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return AddIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                               buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mul: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<MulIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return MulIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                               buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::FloorDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<DivSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return DivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::CeilDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<CeilDivSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return CeilDivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                    buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mod: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<RemSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return RemSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                buildExpr(binaryExpr.getRHS()));
     }
     }
@@ -89,10 +89,10 @@ FailureOr<OpFoldResult> mlir::arith::reifyValueBound(
            "expected dynamic dim");
     if (isa<RankedTensorType>(value.getType())) {
       // A tensor dimension is used: generate a tensor.dim.
-      operands.push_back(b.create<tensor::DimOp>(loc, value, *dim));
+      operands.push_back(tensor::DimOp::create(b, loc, value, *dim));
     } else if (isa<MemRefType>(value.getType())) {
       // A memref dimension is used: generate a memref.dim.
-      operands.push_back(b.create<memref::DimOp>(loc, value, *dim));
+      operands.push_back(memref::DimOp::create(b, loc, value, *dim));
     } else {
       llvm_unreachable("cannot generate DimOp for unsupported shaped type");
     }
diff --git a/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
index 3478adcb4a128..dd6efe6d6bc31 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
@@ -83,7 +83,7 @@ struct ConstantShardingInterface
           cOp.getType(), getMesh(op, sharding.getMeshAttr(), symbolTable),
           sharding));
       auto newValue = value.resizeSplat(newType);
-      auto newOp = builder.create<ConstantOp>(op->getLoc(), newType, newValue);
+      auto newOp = ConstantOp::create(builder, op->getLoc(), newType, newValue);
       spmdizationMap.map(op->getResult(0), newOp.getResult());
       spmdizationMap.map(op, newOp.getOperation());
     } else {
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index 3cd8684878a11..6e9cc14c6ada5 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -68,7 +68,7 @@ mlir::inferExpandShapeOutputShape(OpBuilder &b, Location loc,
     // dynamism.
     Value indexGroupSize = cast<Value>(inputShape[inputIndex]);
     Value indexGroupStaticSizesProduct =
-        b.create<arith::ConstantIndexOp>(loc, indexGroupStaticSizesProductInt);
+        arith::ConstantIndexOp::create(b, loc, indexGroupStaticSizesProductInt);
     Value dynamicDimSize = b.createOrFold<arith::DivSIOp>(
         loc, indexGroupSize, indexGroupStaticSizesProduct);
     outputShapeValues.push_back(dynamicDimSize);
@@ -105,7 +105,7 @@ Value mlir::getValueOrCreateConstantIntOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return b.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(b,
       loc, b.getIntegerAttr(attr.getType(), attr.getValue().getSExtValue()));
 }
 
@@ -114,7 +114,7 @@ Value mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return b.create<arith::ConstantIndexOp>(loc, attr.getValue().getSExtValue());
+  return arith::ConstantIndexOp::create(b, loc, attr.getValue().getSExtValue());
 }
 
 Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
@@ -125,7 +125,7 @@ Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
   bool targetIsIndex = targetType.isIndex();
   bool valueIsIndex = value.getType().isIndex();
   if (targetIsIndex ^ valueIsIndex)
-    return b.create<arith::IndexCastOp>(loc, targetType, value);
+    return arith::IndexCastOp::create(b, loc, targetType, value);
 
   auto targetIntegerType = dyn_cast<IntegerType>(targetType);
   auto valueIntegerType = dyn_cast<IntegerType>(value.getType());
@@ -134,8 +134,8 @@ Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
   assert(targetIntegerType.getSignedness() == valueIntegerType.getSignedness());
 
   if (targetIntegerType.getWidth() > valueIntegerType.getWidth())
-    return b.create<arith::ExtSIOp>(loc, targetIntegerType, value);
-  return b.create<arith::TruncIOp>(loc, targetIntegerType, value);
+    return arith::ExtSIOp::create(b, loc, targetIntegerType, value);
+  return arith::TruncIOp::create(b, loc, targetIntegerType, value);
 }
 
 static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
@@ -143,21 +143,21 @@ static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
   // If operand is floating point, cast directly to the int type.
   if (isa<FloatType>(operand.getType())) {
     if (isUnsigned)
-      return b.create<arith::FPToUIOp>(toType, operand);
-    return b.create<arith::FPToSIOp>(toType, operand);
+      return arith::FPToUIOp::create(b, toType, operand);
+    return arith::FPToSIOp::create(b, toType, operand);
   }
   // Cast index operands directly to the int type.
   if (operand.getType().isIndex())
-    return b.create<arith::IndexCastOp>(toType, operand);
+    return arith::IndexCastOp::create(b, toType, operand);
   if (auto fromIntType = dyn_cast<IntegerType>(operand.getType())) {
     // Either extend or truncate.
     if (toType.getWidth() > fromIntType.getWidth()) {
       if (isUnsigned)
-        return b.create<arith::ExtUIOp>(toType, operand);
-      return b.create<arith::ExtSIOp>(toType, operand);
+        return arith::ExtUIOp::create(b, toType, operand);
+      return arith::ExtSIOp::create(b, toType, operand);
     }
     if (toType.getWidth() < fromIntType.getWidth())
-      return b.create<arith::TruncIOp>(toType, operand);
+      return arith::TruncIOp::create(b, toType, operand);
     return operand;
   }
 
@@ -170,14 +170,14 @@ static Value convertScalarToFpDtype(ImplicitLocOpBuilder &b, Value operand,
   // Note that it is unclear how to cast from BF16<->FP16.
   if (isa<IntegerType>(operand.getType())) {
     if (isUnsigned)
-      return b.create<arith::UIToFPOp>(toType, operand);
-    return b.create<arith::SIToFPOp>(toType, operand);
+      return arith::UIToFPOp::create(b, toType, operand);
+    return arith::SIToFPOp::create(b, toType, operand);
   }
   if (auto fromFpTy = dyn_cast<FloatType>(operand.getType())) {
     if (toType.getWidth() > fromFpTy.getWidth())
-      return b.create<arith::ExtFOp>(toType, operand);
+      return arith::ExtFOp::create(b, toType, operand);
     if (toType.getWidth() < fromFpTy.getWidth())
-      return b.create<arith::TruncFOp>(toType, operand);
+      return arith::TruncFOp::create(b, toType, operand);
     return operand;
   }
 
@@ -190,18 +190,18 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
   if (auto fromComplexType = dyn_cast<ComplexType>(operand.getType())) {
     if (isa<FloatType>(targetType.getElementType()) &&
         isa<FloatType>(fromComplexType.getElementType())) {
-      Value real = b.create<complex::ReOp>(operand);
-      Value imag = b.create<complex::ImOp>(operand);
+      Value real = complex::ReOp::create(b, operand);
+      Value imag = complex::ImOp::create(b, operand);
       Type targetETy = targetType.getElementType();
       if (targetType.getElementType().getIntOrFloatBitWidth() <
           fromComplexType.getElementType().getIntOrFloatBitWidth()) {
-        real = b.create<arith::TruncFOp>(targetETy, real);
-        imag = b.create<arith::TruncFOp>(targetETy, imag);
+        real = arith::TruncFOp::create(b, targetETy, real);
+        imag = arith::TruncFOp::create(b, targetETy, imag);
       } else {
-        real = b.create<arith::ExtFOp>(targetETy, real);
-        imag = b.create<arith::ExtFOp>(targetETy, imag);
+        real = arith::ExtFOp::create(b, targetETy, real);
+        imag = arith::ExtFOp::create(b, targetETy, imag);
       }
-      return b.create<complex::CreateOp>(targetType, real, imag);
+      return complex::CreateOp::create(b, targetType, real, imag);
     }
   }
 
@@ -210,27 +210,27 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
     auto toBitwidth = toFpTy.getIntOrFloatBitWidth();
     Value from = operand;
     if (from.getType().getIntOrFloatBitWidth() < toBitwidth) {
-      from = b.create<arith::ExtFOp>(toFpTy, from);
+      from = arith::ExtFOp::create(b, toFpTy, from);
     }
     if (from.getType().getIntOrFloatBitWidth() > toBitwidth) {
-      from = b.create<arith::TruncFOp>(toFpTy, from);
+      from = arith::TruncFOp::create(b, toFpTy, from);
     }
-    Value zero = b.create<mlir::arith::ConstantFloatOp>(
+    Value zero = mlir::arith::ConstantFloatOp::create(b,
         toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
-    return b.create<complex::CreateOp>(targetType, from, zero);
+    return complex::CreateOp::create(b, targetType, from, zero);
   }
 
   if (isa<IntegerType>(operand.getType())) {
     FloatType toFpTy = cast<FloatType>(targetType.getElementType());
     Value from = operand;
     if (isUnsigned) {
-      from = b.create<arith::UIToFPOp>(toFpTy, from);
+      from = arith::UIToFPOp::create(b, toFpTy, from);
     } else {
-      from = b.create<arith::SIToFPOp>(toFpTy, from);
+      from = arith::SIToFPOp::create(b, toFpTy, from);
     }
-    Value zero = b.create<mlir::arith::ConstantFloatOp>(
+    Value zero = mlir::arith::ConstantFloatOp::create(b,
         toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
-    return b.create<complex::CreateOp>(targetType, from, zero);
+    return complex::CreateOp::create(b, targetType, from, zero);
   }
 
   return {};
@@ -278,7 +278,7 @@ Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
     attr = SplatElementsAttr::get(vecTy, value);
   }
 
-  return builder.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(builder, loc, attr);
 }
 
 Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
@@ -310,35 +310,35 @@ Type mlir::getType(OpFoldResult ofr) {
 }
 
 Value ArithBuilder::_and(Value lhs, Value rhs) {
-  return b.create<arith::AndIOp>(loc, lhs, rhs);
+  return arith::AndIOp::create(b, loc, lhs, rhs);
 }
 Value ArithBuilder::add(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::AddFOp>(loc, lhs, rhs);
-  return b.create<arith::AddIOp>(loc, lhs, rhs, ovf);
+    return arith::AddFOp::create(b, loc, lhs, rhs);
+  return arith::AddIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::sub(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::SubFOp>(loc, lhs, rhs);
-  return b.create<arith::SubIOp>(loc, lhs, rhs, ovf);
+    return arith::SubFOp::create(b, loc, lhs, rhs);
+  return arith::SubIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::mul(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::MulFOp>(loc, lhs, rhs);
-  return b.create<arith::MulIOp>(loc, lhs, rhs, ovf);
+    return arith::MulFOp::create(b, loc, lhs, rhs);
+  return arith::MulIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::sgt(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGT, lhs, rhs);
-  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sgt, lhs, rhs);
+    return arith::CmpFOp::create(b, loc, arith::CmpFPredicate::OGT, lhs, rhs);
+  return arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sgt, lhs, rhs);
 }
 Value ArithBuilder::slt(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OLT, lhs, rhs);
-  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, lhs, rhs);
+    return arith::CmpFOp::create(b, loc, arith::CmpFPredicate::OLT, lhs, rhs);
+  return arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, lhs, rhs);
 }
 Value ArithBuilder::select(Value cmp, Value lhs, Value rhs) {
-  return b.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+  return arith::SelectOp::create(b, loc, cmp, lhs, rhs);
 }
 
 namespace mlir::arith {
@@ -349,7 +349,7 @@ Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values) {
 
 Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
                     Type resultType) {
-  Value one = builder.create<ConstantOp>(loc, resultType,
+  Value one = ConstantOp::create(builder, loc, resultType,
                                          builder.getOneAttr(resultType));
   ArithBuilder arithBuilder(builder, loc);
   return std::accumulate(
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
index a07be7801869f..b9a81b71fa301 100644
--- a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
@@ -229,7 +229,7 @@ class LowerContractionToNeonI8MMPattern
 
     // Initial accumulator for the final result. This is the un-tiled result if
     // tiling is done.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, op.getResultType(), rewriter.getZeroAttr(op.getResultType()));
 
     SmallVector<int64_t> unrolledSize = *op.getShapeForUnroll();
@@ -281,7 +281,7 @@ class LowerContractionToNeonI8MMPattern
       if (isVecmat) {
         auto expandForSMMLA = [&](Value tiledOperand,
                                   VectorType expandedTypeType) {
-          auto emptyOperand = rewriter.create<arith::ConstantOp>(
+          auto emptyOperand = arith::ConstantOp::create(rewriter,
               loc, expandedTypeType, rewriter.getZeroAttr(expandedTypeType));
           SmallVector<int64_t> offsets(
               cast<ShapedType>(emptyOperand.getType()).getRank(), 0);
@@ -298,7 +298,7 @@ class LowerContractionToNeonI8MMPattern
       // using the instruction for unsigned by signed multiplication with
       // reversed operands.
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledAcc = rewriter.create<vector::TransposeOp>(
+        tiledAcc = vector::TransposeOp::create(rewriter,
             loc, tiledAcc, ArrayRef<int64_t>({1, 0}));
 
       // Collapse tiled operands to 1D vectors required by smmla intrinsic
@@ -331,7 +331,7 @@ class LowerContractionToNeonI8MMPattern
       // Because of the reversed operands the result is obtained transposed.
       // Transpose it back,
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledRes = rewriter.create<vector::TransposeOp>(
+        tiledRes = vector::TransposeOp::create(rewriter,
             loc, tiledRes, ArrayRef<int64_t>({1, 0}));
 
       // With vecmat, only one row of tiled ACC can be inserted into the final
diff --git a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
index 5f00cef90159e..23ec00196f714 100644
--- a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
@@ -75,21 +75,21 @@ scf::ForOp createLoopOverTileSlices(
     PatternRewriter &rewriter, Location loc, Value initTile,
     std::function<Value(OpBuilder &, Location, Value, Value)> makeLoopBody) {
   OpBuilder::InsertionGuard g(rewriter);
-  auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-  auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+  auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
       loc, llvm::cast<VectorType>(initTile.getType()).getDimSize(0));
   auto vscale =
-      rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-  auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+      vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+  auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto numTileSlices =
-      rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
-  auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, numTileSlices, step,
+      arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices, step,
                                            ValueRange{initTile});
   rewriter.setInsertionPointToStart(forOp.getBody());
   Value nextTile =
       makeLoopBody(rewriter, loc, /*tileSliceIndex=*/forOp.getInductionVar(),
                    /*currentTile=*/forOp.getRegionIterArg(0));
-  rewriter.create<scf::YieldOp>(loc, nextTile);
+  scf::YieldOp::create(rewriter, loc, nextTile);
   return forOp;
 }
 
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
index 23f2c2bf65e47..4d03bbdabdf09 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
@@ -136,7 +136,7 @@ class OuterProductFusion2Way
 
     auto loc = op.getLoc();
     auto packInputs = [&](Value lhs, Value rhs) {
-      return rewriter.create<vector::InterleaveOp>(loc, lhs, rhs);
+      return vector::InterleaveOp::create(rewriter, loc, lhs, rhs);
     };
 
     auto lhs = packInputs(op1.getLhs().getDefiningOp()->getOperand(0),
@@ -284,7 +284,7 @@ class OuterProductFusion4Way
 
     auto loc = op.getLoc();
     auto packInputs = [&](Value lhs, Value rhs) {
-      return rewriter.create<vector::InterleaveOp>(loc, lhs, rhs);
+      return vector::InterleaveOp::create(rewriter, loc, lhs, rhs);
     };
 
     auto lhs0 = packInputs(op1.getLhs().getDefiningOp()->getOperand(0),
@@ -456,7 +456,7 @@ struct SwapVectorExtractOfArithExtend
     Value extendSource = extendOp->getOperand(0);
 
     // Create new extract from source of extend.
-    Value newExtract = rewriter.create<vector::ExtractOp>(
+    Value newExtract = vector::ExtractOp::create(rewriter,
         loc, extendSource, extractOp.getMixedPosition());
 
     // Extend new extract to original result type.
@@ -503,7 +503,7 @@ struct SwapVectorScalableExtractOfArithExtend
     // Create new extract from source of extend.
     VectorType extractResultVectorType =
         resultType.clone(extendSourceVectorType.getElementType());
-    Value newExtract = rewriter.create<vector::ScalableExtractOp>(
+    Value newExtract = vector::ScalableExtractOp::create(rewriter,
         loc, extractResultVectorType, extendSource, extractOp.getPos());
 
     // Extend new extract to original result type.
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index b3c988d455420..d925c19852679 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -210,7 +210,7 @@ void splitCondBranches(IRRewriter &rewriter, FunctionOpInterface function) {
 
   auto insertJump = [&](Location loc, Block *source, Block *dest, auto args) {
     rewriter.setInsertionPointToEnd(source);
-    rewriter.create<cf::BranchOp>(loc, dest, args);
+    cf::BranchOp::create(rewriter, loc, dest, args);
   };
 
   for (auto condBranch : worklist) {
@@ -253,7 +253,7 @@ void insertCopiesAtBranches(IRRewriter &rewriter,
     for (OpOperand &operand : terminator->getOpOperands()) {
       if (isValidSMETileVectorType(operand.get().getType())) {
         auto copy =
-            rewriter.create<CopyTileOp>(terminator->getLoc(), operand.get());
+            CopyTileOp::create(rewriter, terminator->getLoc(), operand.get());
         rewriter.modifyOpInPlace(terminator, [&] { operand.assign(copy); });
       }
     }
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 1e8e1265affa0..3362d0ce13533 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -82,13 +82,13 @@ SmallVector<Value, 2> addConstantScalableOffset(OpBuilder &builder,
                                                 Location loc,
                                                 ValueRange indices,
                                                 ArrayRef<int> scalableOffsets) {
-  auto vscale = builder.create<vector::VectorScaleOp>(loc);
+  auto vscale = vector::VectorScaleOp::create(builder, loc);
   return llvm::map_to_vector(
       llvm::zip_equal(indices, scalableOffsets), [&](auto pair) -> Value {
         auto [index, base] = pair;
-        auto offset = builder.create<arith::MulIOp>(
-            loc, builder.create<arith::ConstantIndexOp>(loc, base), vscale);
-        return builder.create<arith::AddIOp>(loc, index, offset);
+        auto offset = arith::MulIOp::create(builder,
+            loc, arith::ConstantIndexOp::create(builder, loc, base), vscale);
+        return arith::AddIOp::create(builder, loc, index, offset);
       });
 }
 
@@ -132,7 +132,7 @@ Value extractSMEMask(OpBuilder &builder, Location loc, Value mask,
   // from the mask operands to get the parameters for this sub-tile.
   auto smeTileMaskDims = addConstantScalableOffset(
       builder, loc, createMask.getOperands(), {-smeTile.row, -smeTile.col});
-  auto smeTileCreateMask = builder.create<vector::CreateMaskOp>(
+  auto smeTileCreateMask = vector::CreateMaskOp::create(builder,
       loc, smeTile.type.clone(builder.getI1Type()), smeTileMaskDims);
   return smeTileCreateMask.getResult();
 }
@@ -190,7 +190,7 @@ struct LegalizeArithConstantOpsByDecomposition
 
     auto smeTileType = getSMETileTypeForElement(vectorType.getElementType());
     auto tileCount = getNumberOfSMETilesForVectorType(vectorType);
-    auto tileSplat = rewriter.create<arith::ConstantOp>(
+    auto tileSplat = arith::ConstantOp::create(rewriter,
         constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
     SmallVector<Value> repl(tileCount, tileSplat);
     rewriter.replaceOpWithMultiple(constantOp, {repl});
@@ -237,11 +237,11 @@ struct LegalizeVectorOuterProductOpsByDecomposition
              decomposeToSMETiles(rewriter, vectorType, smeTileType))) {
 
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto lhs = rewriter.create<vector::ScalableExtractOp>(
+      auto lhs = vector::ScalableExtractOp::create(rewriter,
           loc, sliceType, outerProductOp.getLhs(), smeTile.row);
-      auto rhs = rewriter.create<vector::ScalableExtractOp>(
+      auto rhs = vector::ScalableExtractOp::create(rewriter,
           loc, sliceType, outerProductOp.getRhs(), smeTile.col);
-      auto smeOuterProduct = rewriter.create<vector::OuterProductOp>(
+      auto smeOuterProduct = vector::OuterProductOp::create(rewriter,
           loc, smeTileType, lhs, rhs,
           !accSMETiles.empty() ? accSMETiles[index] : Value{},
           outerProductOp.getKind());
@@ -314,7 +314,7 @@ struct LegalizeTransferReadOpsByDecomposition
     for (SMESubTile smeTile :
          decomposeToSMETiles(rewriter, vectorType, smeTileType, transposed)) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeRead = rewriter.create<vector::TransferReadOp>(
+      auto smeRead = vector::TransferReadOp::create(rewriter,
           loc, smeTileType, readOp.getBase(),
           getSMESubTileIndices(rewriter, loc, readOp.getIndices(), smeTile),
           readOp.getPermutationMapAttr(), readOp.getPadding(), smeMask,
@@ -363,7 +363,7 @@ struct LegalizeTransferWriteOpsByDecomposition
     for (auto [index, smeTile] : llvm::enumerate(decomposeToSMETiles(
              rewriter, vectorType, smeTileType, transposed))) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeWrite = rewriter.create<vector::TransferWriteOp>(
+      auto smeWrite = vector::TransferWriteOp::create(rewriter,
           loc, inputSMETiles[index], destTensorOrMemref,
           getSMESubTileIndices(rewriter, loc, writeOp.getIndices(), smeTile),
           writeOp.getPermutationMapAttr(), smeMask, writeOp.getInBoundsAttr());
@@ -456,11 +456,11 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
         VectorType::get(minTileSlices, rewriter.getI1Type(), true);
 
     // Create loop over all tile slices.
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto upperBound = createVscaleMultiple(minTileSlices);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto storeLoop =
-        rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     rewriter.setInsertionPointToStart(storeLoop.getBody());
 
     // For each sub-tile of the multi-tile `vectorType`.
@@ -474,29 +474,29 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
 
       // The current slice of `vectorType` we are processing.
       auto sliceIndex =
-          rewriter.create<arith::AddIOp>(loc, tileRow, tileSliceIndex);
+          arith::AddIOp::create(rewriter, loc, tileRow, tileSliceIndex);
 
       // Where in the destination memref the current slice will be stored.
-      auto storeRow = rewriter.create<arith::AddIOp>(loc, sliceIndex,
+      auto storeRow = arith::AddIOp::create(rewriter, loc, sliceIndex,
                                                      writeOp.getIndices()[0]);
       auto storeCol =
-          rewriter.create<arith::AddIOp>(loc, tileCol, writeOp.getIndices()[1]);
+          arith::AddIOp::create(rewriter, loc, tileCol, writeOp.getIndices()[1]);
 
       // Extract the mask for the current slice.
       Value sliceMask = nullptr;
       if (mask) {
-        sliceMask = rewriter.create<vector::ExtractOp>(
+        sliceMask = vector::ExtractOp::create(rewriter,
             loc, mask, OpFoldResult(sliceIndex));
         if (sliceMaskType != sliceMask.getType())
-          sliceMask = rewriter.create<vector::ScalableExtractOp>(
+          sliceMask = vector::ScalableExtractOp::create(rewriter,
               loc, sliceMaskType, sliceMask, smeTile.col);
       }
 
       // Extract and store the current slice.
       Value tile = inputSMETiles[index];
       auto slice =
-          rewriter.create<vector::ExtractOp>(loc, tile, tileSliceIndex);
-      rewriter.create<vector::TransferWriteOp>(
+          vector::ExtractOp::create(rewriter, loc, tile, tileSliceIndex);
+      vector::TransferWriteOp::create(rewriter,
           loc, slice, writeOp.getBase(), ValueRange{storeRow, storeCol},
           AffineMapAttr::get(writeOp.getPermutationMap().dropResult(0)),
           sliceMask,
@@ -567,13 +567,13 @@ struct FoldExtractFromVectorOfSMELikeCreateMasks
           extractOp,
           "constant vector.create_masks dims should be folded elsewhere");
 
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto extractionIndex = getValueOrCreateConstantIndexOp(
         rewriter, loc, extractOp.getMixedPosition()[0]);
-    auto extractionInTrueRegion = rewriter.create<arith::CmpIOp>(
+    auto extractionInTrueRegion = arith::CmpIOp::create(rewriter,
         loc, rewriter.getI1Type(), arith::CmpIPredicate::slt, extractionIndex,
         frontMaskDim);
-    auto newMaskFrontDim = rewriter.create<arith::SelectOp>(
+    auto newMaskFrontDim = arith::SelectOp::create(rewriter,
         loc, extractionInTrueRegion, createMaskOp.getOperand(1), zero);
 
     rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(
@@ -660,8 +660,8 @@ struct LiftIllegalVectorTransposeToMemory
           illegalRead, "expected read to have identity permutation map");
 
     auto loc = transposeOp.getLoc();
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
     // Create a subview that matches the size of the illegal read vector type.
     auto readType = illegalRead.getVectorType();
@@ -669,14 +669,14 @@ struct LiftIllegalVectorTransposeToMemory
         llvm::zip_equal(readType.getShape(), readType.getScalableDims()),
         [&](auto dim) -> Value {
           auto [size, isScalable] = dim;
-          auto dimSize = rewriter.create<arith::ConstantIndexOp>(loc, size);
+          auto dimSize = arith::ConstantIndexOp::create(rewriter, loc, size);
           if (!isScalable)
             return dimSize;
-          auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
-          return rewriter.create<arith::MulIOp>(loc, vscale, dimSize);
+          auto vscale = vector::VectorScaleOp::create(rewriter, loc);
+          return arith::MulIOp::create(rewriter, loc, vscale, dimSize);
         });
     SmallVector<Value> strides(readType.getRank(), Value(one));
-    auto readSubview = rewriter.create<memref::SubViewOp>(
+    auto readSubview = memref::SubViewOp::create(rewriter,
         loc, illegalRead.getBase(), illegalRead.getIndices(), readSizes,
         strides);
 
@@ -686,13 +686,13 @@ struct LiftIllegalVectorTransposeToMemory
     if (mask) {
       // Note: The transpose for the mask should fold into the
       // vector.create_mask/constant_mask op, which will then become legal.
-      mask = rewriter.create<vector::TransposeOp>(loc, mask,
+      mask = vector::TransposeOp::create(rewriter, loc, mask,
                                                   transposeOp.getPermutation());
     }
     // - The source memref
     mlir::AffineMap transposeMap = AffineMap::getPermutationMap(
         transposeOp.getPermutation(), getContext());
-    auto transposedSubview = rewriter.create<memref::TransposeOp>(
+    auto transposedSubview = memref::TransposeOp::create(rewriter,
         loc, readSubview, AffineMapAttr::get(transposeMap));
     ArrayAttr inBoundsAttr = illegalRead.getInBoundsAttr();
     // - The `in_bounds` attribute
@@ -706,7 +706,7 @@ struct LiftIllegalVectorTransposeToMemory
     VectorType legalReadType = resultType.clone(readType.getElementType());
     // Note: The indices are all zero as the subview is already offset.
     SmallVector<Value> readIndices(illegalRead.getIndices().size(), zero);
-    auto legalRead = rewriter.create<vector::TransferReadOp>(
+    auto legalRead = vector::TransferReadOp::create(rewriter,
         loc, legalReadType, transposedSubview, readIndices,
         illegalRead.getPermutationMapAttr(), illegalRead.getPadding(), mask,
         inBoundsAttr);
@@ -797,12 +797,12 @@ struct LowerIllegalTransposeStoreViaZA
         AffineMap::getPermutationMap(ArrayRef<int64_t>{1, 0}, getContext()));
 
     // Note: We need to use `get_tile` as there's no vector-level `undef`.
-    Value undefTile = rewriter.create<arm_sme::GetTileOp>(loc, smeTileType);
+    Value undefTile = arm_sme::GetTileOp::create(rewriter, loc, smeTileType);
     Value destTensorOrMemref = writeOp.getBase();
     auto numSlicesPerTile =
         std::min(sourceType.getDimSize(0), smeTileType.getDimSize(0));
     auto numSlices =
-        rewriter.create<arith::ConstantIndexOp>(loc, numSlicesPerTile);
+        arith::ConstantIndexOp::create(rewriter, loc, numSlicesPerTile);
     for (auto [index, smeTile] : llvm::enumerate(
              decomposeToSMETiles(rewriter, sourceType, smeTileType))) {
       // 1. _Deliberately_ drop a scalable dimension and insert a fixed number
@@ -811,46 +811,46 @@ struct LowerIllegalTransposeStoreViaZA
       // rows of the tile after 1*vscale rows.
       Value tile = undefTile;
       for (int d = 0; d < numSlicesPerTile; ++d) {
-        Value vector = rewriter.create<vector::ExtractOp>(
+        Value vector = vector::ExtractOp::create(rewriter,
             loc, transposeOp.getVector(),
             rewriter.getIndexAttr(d + smeTile.row));
         if (vector.getType() != smeSliceType) {
-          vector = rewriter.create<vector::ScalableExtractOp>(
+          vector = vector::ScalableExtractOp::create(rewriter,
               loc, smeSliceType, vector, smeTile.col);
         }
-        tile = rewriter.create<vector::InsertOp>(loc, vector, tile, d);
+        tile = vector::InsertOp::create(rewriter, loc, vector, tile, d);
       }
 
       // 2. Transpose the tile position.
       auto transposedRow = createVscaleMultiple(smeTile.col);
       auto transposedCol =
-          rewriter.create<arith::ConstantIndexOp>(loc, smeTile.row);
+          arith::ConstantIndexOp::create(rewriter, loc, smeTile.row);
 
       // 3. Compute mask for tile store.
       Value maskRows;
       Value maskCols;
       if (auto mask = writeOp.getMask()) {
         auto createMask = mask.getDefiningOp<vector::CreateMaskOp>();
-        maskRows = rewriter.create<arith::SubIOp>(loc, createMask.getOperand(0),
+        maskRows = arith::SubIOp::create(rewriter, loc, createMask.getOperand(0),
                                                   transposedRow);
-        maskCols = rewriter.create<arith::SubIOp>(loc, createMask.getOperand(1),
+        maskCols = arith::SubIOp::create(rewriter, loc, createMask.getOperand(1),
                                                   transposedCol);
-        maskCols = rewriter.create<index::MinSOp>(loc, maskCols, numSlices);
+        maskCols = index::MinSOp::create(rewriter, loc, maskCols, numSlices);
       } else {
         maskRows = createVscaleMultiple(smeTileType.getDimSize(0));
         maskCols = numSlices;
       }
-      auto subMask = rewriter.create<vector::CreateMaskOp>(
+      auto subMask = vector::CreateMaskOp::create(rewriter,
           loc, smeTileType.clone(rewriter.getI1Type()),
           ValueRange{maskRows, maskCols});
 
       // 4. Emit a transposed tile write.
       auto writeIndices = writeOp.getIndices();
       Value destRow =
-          rewriter.create<arith::AddIOp>(loc, transposedRow, writeIndices[0]);
+          arith::AddIOp::create(rewriter, loc, transposedRow, writeIndices[0]);
       Value destCol =
-          rewriter.create<arith::AddIOp>(loc, transposedCol, writeIndices[1]);
-      auto smeWrite = rewriter.create<vector::TransferWriteOp>(
+          arith::AddIOp::create(rewriter, loc, transposedCol, writeIndices[1]);
+      auto smeWrite = vector::TransferWriteOp::create(rewriter,
           loc, tile, destTensorOrMemref, ValueRange{destRow, destCol},
           transposeMap, subMask, writeOp.getInBounds());
 
@@ -934,41 +934,41 @@ struct LowerColumnTransferReadToLoops
 
     // Create a loop over all rows and load one element at a time.
     auto loc = readOp.getLoc();
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto createVscaleMultiple =
         vector::makeVscaleConstantBuilder(rewriter, loc);
     auto upperBound = createVscaleMultiple(numRows);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    Value init = rewriter.create<arith::ConstantOp>(
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    Value init = arith::ConstantOp::create(rewriter,
         loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
 
     scf::ForOp loadLoop;
     {
       OpBuilder::InsertionGuard g(rewriter);
-      loadLoop = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+      loadLoop = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
                                              ValueRange{init});
       rewriter.setInsertionPointToStart(loadLoop.getBody());
 
       auto tileSliceIndex = loadLoop.getInductionVar();
 
-      auto idx0 = rewriter.create<arith::AddIOp>(loc, tileSliceIndex,
+      auto idx0 = arith::AddIOp::create(rewriter, loc, tileSliceIndex,
                                                  readOp.getIndices()[0]);
       auto idx1 = readOp.getIndices()[1];
 
-      Value scalar = rewriter.create<memref::LoadOp>(
+      Value scalar = memref::LoadOp::create(rewriter,
           loc, readOp.getBase(), SmallVector<Value>({idx0, idx1}));
 
-      Operation *updateInit = rewriter.create<vector::InsertOp>(
+      Operation *updateInit = vector::InsertOp::create(rewriter,
           loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
 
-      rewriter.create<scf::YieldOp>(loc, updateInit->getResult(0));
+      scf::YieldOp::create(rewriter, loc, updateInit->getResult(0));
     }
 
     // The read operation has been "legalized", but since the original result
     // type was a 2D vector, we need to cast before returning the result. This
     // ShapeCast should cancel-out with some other ShapeCast (i.e. it's a
     // no-op).
-    auto sc = rewriter.create<vector::ShapeCastOp>(
+    auto sc = vector::ShapeCastOp::create(rewriter,
         loc, readOp.getResult().getType(), loadLoop.getResult(0));
 
     rewriter.replaceOp(readOp, sc);
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index 7b64e5745fcef..3ee211fc09bee 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -87,7 +87,7 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     VectorType sourceType = source.getType();
     VectorType resultType = convertOp.getResult().getType();
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
 
     // We want to iterate over the input vector in steps of the trailing
@@ -100,14 +100,14 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     for (SmallVector<int64_t> index :
          StaticTileOffsetRange(sourceType.getShape(), tileShape)) {
       auto extractOrInsertPosition = ArrayRef(index).drop_back();
-      auto sourceVector = rewriter.create<vector::ExtractOp>(
+      auto sourceVector = vector::ExtractOp::create(rewriter,
           loc, source, extractOrInsertPosition);
       VectorType convertedType =
           VectorType::Builder(llvm::cast<VectorType>(sourceVector.getType()))
               .setDim(0, resultType.getShape().back());
       auto convertedVector =
-          rewriter.create<IntrOp>(loc, TypeRange{convertedType}, sourceVector);
-      result = rewriter.create<vector::InsertOp>(loc, convertedVector, result,
+          IntrOp::create(rewriter, loc, TypeRange{convertedType}, sourceVector);
+      result = vector::InsertOp::create(rewriter, loc, convertedVector, result,
                                                  extractOrInsertPosition);
     }
 
@@ -135,11 +135,11 @@ struct PselOpLowering : public ConvertOpToLLVMPattern<PselOp> {
                   ConversionPatternRewriter &rewriter) const override {
     auto svboolType = VectorType::get(16, rewriter.getI1Type(), true);
     auto loc = pselOp.getLoc();
-    auto svboolP1 = rewriter.create<ConvertToSvboolIntrOp>(loc, svboolType,
+    auto svboolP1 = ConvertToSvboolIntrOp::create(rewriter, loc, svboolType,
                                                            adaptor.getP1());
-    auto indexI32 = rewriter.create<arith::IndexCastOp>(
+    auto indexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), pselOp.getIndex());
-    auto pselIntr = rewriter.create<PselIntrOp>(loc, svboolType, svboolP1,
+    auto pselIntr = PselIntrOp::create(rewriter, loc, svboolType, svboolP1,
                                                 pselOp.getP2(), indexI32);
     rewriter.replaceOpWithNewOp<ConvertFromSvboolIntrOp>(
         pselOp, adaptor.getP1().getType(), pselIntr);
@@ -174,7 +174,7 @@ struct CreateMaskOpLowering
                                          "not SVE predicate-sized");
 
     auto loc = createMaskOp.getLoc();
-    auto zero = rewriter.create<LLVM::ZeroOp>(loc, rewriter.getI64Type());
+    auto zero = LLVM::ZeroOp::create(rewriter, loc, rewriter.getI64Type());
     rewriter.replaceOpWithNewOp<WhileLTIntrOp>(createMaskOp, maskType, zero,
                                                adaptor.getOperands()[0]);
     return success();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
index 3dbb93b8a0669..33de12ca17cfc 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
@@ -71,7 +71,7 @@ void replaceOpWithUnrealizedConversion(PatternRewriter &rewriter, TOp op,
                                        TLegalizerCallback callback) {
   replaceOpWithLegalizedOp(rewriter, op, [&](TOp newOp) {
     // Mark our `unrealized_conversion_casts` with a pass label.
-    return rewriter.create<UnrealizedConversionCastOp>(
+    return UnrealizedConversionCastOp::create(rewriter,
         op.getLoc(), TypeRange{op.getResult().getType()},
         ValueRange{callback(newOp)},
         NamedAttribute(rewriter.getStringAttr(kSVELegalizerTag),
@@ -239,7 +239,7 @@ struct LegalizeSVEMaskStoreConversion
 
     auto legalMaskType = widenScalableMaskTypeToSvbool(
         llvm::cast<VectorType>(valueToStore.getType()));
-    auto convertToSvbool = rewriter.create<arm_sve::ConvertToSvboolOp>(
+    auto convertToSvbool = arm_sve::ConvertToSvboolOp::create(rewriter,
         loc, legalMaskType, valueToStore);
     // Replace this store with a conversion to a storable svbool mask [1],
     // followed by a wider store.
@@ -290,7 +290,7 @@ struct LegalizeSVEMaskLoadConversion : public OpRewritePattern<memref::LoadOp> {
     replaceOpWithLegalizedOp(rewriter, loadOp, [&](memref::LoadOp newLoadOp) {
       newLoadOp.setMemRef(*legalMemref);
       newLoadOp.getResult().setType(legalMaskType);
-      return rewriter.create<arm_sve::ConvertFromSvboolOp>(
+      return arm_sve::ConvertFromSvboolOp::create(rewriter,
           loc, loadedMask.getType(), newLoadOp);
     });
 
@@ -408,7 +408,7 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
       reassoc.back().push_back(i);
     if (!memref::CollapseShapeOp::isGuaranteedCollapsible(memTy, reassoc))
       return failure();
-    Value collapsedMem = rewriter.create<memref::CollapseShapeOp>(
+    Value collapsedMem = memref::CollapseShapeOp::create(rewriter,
         readOp.getLoc(), readOp.getBase(), reassoc);
 
     // Get a vector type with collapsed trailing dimensions.
@@ -424,13 +424,13 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
     auto indices = readOp.getIndices().drop_back(numCollapseDims - 1);
 
     // Create the new `transfer_read`.
-    auto newReadOp = rewriter.create<vector::TransferReadOp>(
+    auto newReadOp = vector::TransferReadOp::create(rewriter,
         readOp.getLoc(), collapsedVT, collapsedMem, indices,
         readOp.getPadding(),
         ArrayRef<bool>(origInBounds).drop_back(numCollapseDims - 1));
 
     // Cast back to the original vector type.
-    auto toOrigShape = rewriter.create<vector::ShapeCastOp>(readOp.getLoc(),
+    auto toOrigShape = vector::ShapeCastOp::create(rewriter, readOp.getLoc(),
                                                             origVT, newReadOp);
 
     rewriter.replaceOp(readOp, toOrigShape);
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
index bd051b100a91b..0dfd58c582dfd 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
@@ -87,15 +87,15 @@ Value createMMLA(PatternRewriter &rewriter, MMLA op, Location loc,
                  mlir::VectorType accType, Value acc, Value lhs, Value rhs) {
   switch (op) {
   case MMLA::Signed:
-    return rewriter.create<arm_sve::SmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::SmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::Unsigned:
-    return rewriter.create<arm_sve::UmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::UmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::Mixed:
-    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::UsmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::MixedSwapped:
     // The accumulator comes transposed and the result will be transposed
     // later, so all we have to do here is swap the operands.
-    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, rhs, lhs);
+    return arm_sve::UsmmlaOp::create(rewriter, loc, accType, acc, rhs, lhs);
   }
 }
 
@@ -233,25 +233,25 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> lhsTile;
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the LHS tile.
-      auto r0 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+      auto r0 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
                                                    ArrayRef<int64_t>{i});
-      auto r1 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+      auto r1 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
                                                    ArrayRef<int64_t>{i + 1});
       // Concatenate to obtain a 16 x i8 flattened sub-tile.
-      auto t = rewriter.create<vector::ShuffleOp>(
+      auto t = vector::ShuffleOp::create(rewriter,
           loc, r0, r1,
           llvm::ArrayRef<int64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
                                   14, 15});
       // Turn it into a scalable vector.
-      auto s = rewriter.create<vector::ScalableInsertOp>(
-          loc, t, rewriter.create<ub::PoisonOp>(loc, nxv16i8), 0);
+      auto s = vector::ScalableInsertOp::create(rewriter,
+          loc, t, ub::PoisonOp::create(rewriter, loc, nxv16i8), 0);
       // Replicate the sub-tile VSCALE times to fill the entire vector.
-      auto r = rewriter.create<arm_sve::DupQLaneOp>(loc, s, 0);
+      auto r = arm_sve::DupQLaneOp::create(rewriter, loc, s, 0);
       lhsTile.push_back(r);
     }
 
     // "Flatten" the RHS tile from <[N]x8> to <[8*N]>.
-    auto rhs = rewriter.create<vector::ShapeCastOp>(
+    auto rhs = vector::ShapeCastOp::create(rewriter,
         maybeRhs->getLoc(),
         VectorType::get(/*shape=*/8 * N, rewriter.getI8Type(),
                         /*scalableDims=*/{true}),
@@ -261,7 +261,7 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> rhsTile;
     for (int64_t j = 0; j < N; j += 2)
       rhsTile.push_back(
-          rewriter.create<vector::ScalableExtractOp>(loc, nxv16i8, rhs, j * 8));
+          vector::ScalableExtractOp::create(rewriter, loc, nxv16i8, rhs, j * 8));
 
     // Handy types for packing/unpacking of the accumulator tile.
     auto accRowTy = VectorType::get(/*shape=*/N, rewriter.getI32Type(),
@@ -277,33 +277,33 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> accTile;
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the accumulator tile.
-      auto r0 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+      auto r0 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
                                                    ArrayRef<int64_t>{i});
-      auto r1 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+      auto r1 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
                                                    ArrayRef<int64_t>{i + 1});
       Value accTileVec;
       if (mmlaOp == MMLA::MixedSwapped) {
         // We need to swap the positions of the LHS and RHS (since we don't have
         // a signed * unsigned operation), but then each individual 2x2 tile of
         // the acumulator and (later) the result need to be transposed.
-        accTileVec = rewriter.create<vector::InterleaveOp>(loc, r0, r1);
+        accTileVec = vector::InterleaveOp::create(rewriter, loc, r0, r1);
       } else {
         // Bitcast them to 64-bit elements, so subsequent
         // interleave/deinterleave work on pairs of 32-bit numbers.
-        auto r0I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r0);
-        auto r1I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r1);
+        auto r0I64 = vector::BitCastOp::create(rewriter, loc, accRow64Ty, r0);
+        auto r1I64 = vector::BitCastOp::create(rewriter, loc, accRow64Ty, r1);
 
         // Interleave the rows, effectively flattening each 2x2 tile into 4
         // consecutive elements.
-        auto intrI64 = rewriter.create<vector::InterleaveOp>(loc, r0I64, r1I64);
+        auto intrI64 = vector::InterleaveOp::create(rewriter, loc, r0I64, r1I64);
 
         // Bitcast back to 32-bit elements.
         accTileVec =
-            rewriter.create<vector::BitCastOp>(loc, accRowX2Ty, intrI64);
+            vector::BitCastOp::create(rewriter, loc, accRowX2Ty, intrI64);
       }
       // Extract ACC sub-tiles.
       for (int64_t j = 0; j < N; j += 2)
-        accTile.push_back(rewriter.create<vector::ScalableExtractOp>(
+        accTile.push_back(vector::ScalableExtractOp::create(rewriter,
             loc, nxv4i32, accTileVec, j * 2));
     }
 
@@ -317,12 +317,12 @@ class LowerContractionToSVEI8MMPattern
       }
 
     // Unpack the OUT sub-tiles and insert into the result.
-    Value result = rewriter.create<ub::PoisonOp>(loc, op.getResultType());
+    Value result = ub::PoisonOp::create(rewriter, loc, op.getResultType());
     for (int64_t i = 0; i < M / 2; ++i) {
       // Collect a number of sub-tiles in a row.
-      Value row = rewriter.create<ub::PoisonOp>(loc, accRowX2Ty);
+      Value row = ub::PoisonOp::create(rewriter, loc, accRowX2Ty);
       for (int64_t j = 0; j < N / 2; ++j)
-        row = rewriter.create<vector::ScalableInsertOp>(
+        row = vector::ScalableInsertOp::create(rewriter,
             loc, outTile[i * N / 2 + j], row, j * 4);
 
       // Unpack the row to obtain two rows of the output. If we have the out
@@ -331,22 +331,22 @@ class LowerContractionToSVEI8MMPattern
       // Otherwise, the interleave is by pairs.
       Value out0, out1;
       if (mmlaOp == MMLA::MixedSwapped) {
-        auto tmp = rewriter.create<vector::DeinterleaveOp>(loc, row);
+        auto tmp = vector::DeinterleaveOp::create(rewriter, loc, row);
         out0 = tmp.getRes1();
         out1 = tmp.getRes2();
       } else {
         // Deinterleave by pairs.
-        auto row64 = rewriter.create<vector::BitCastOp>(loc, accRowX264Ty, row);
-        auto deintr64 = rewriter.create<vector::DeinterleaveOp>(loc, row64);
+        auto row64 = vector::BitCastOp::create(rewriter, loc, accRowX264Ty, row);
+        auto deintr64 = vector::DeinterleaveOp::create(rewriter, loc, row64);
 
         // Bitcast back into 32-bit elements and insert into the result.
-        out0 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+        out0 = vector::BitCastOp::create(rewriter, loc, accRowTy,
                                                   deintr64.getRes1());
-        out1 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+        out1 = vector::BitCastOp::create(rewriter, loc, accRowTy,
                                                   deintr64.getRes2());
       }
-      result = rewriter.create<vector::InsertOp>(loc, out0, result, i * 2);
-      result = rewriter.create<vector::InsertOp>(loc, out1, result, i * 2 + 1);
+      result = vector::InsertOp::create(rewriter, loc, out0, result, i * 2);
+      result = vector::InsertOp::create(rewriter, loc, out1, result, i * 2 + 1);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index 08a57db6853a0..dc7b07d911c17 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -97,7 +97,7 @@ void ExecuteOp::build(OpBuilder &builder, OperationState &result,
   // expected result is empty. Otherwise, leave this to the caller
   // because we don't know which values to return from the execute op.
   if (resultTypes.empty() && !bodyBuilder) {
-    builder.create<async::YieldOp>(result.location, ValueRange());
+    async::YieldOp::create(builder, result.location, ValueRange());
   } else if (bodyBuilder) {
     bodyBuilder(builder, result.location, bodyBlock->getArguments());
   }
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index 27fa92cee79c2..447411ad989ba 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -191,8 +191,8 @@ static SmallVector<Value> delinearize(ImplicitLocOpBuilder &b, Value index,
   assert(!tripCounts.empty() && "tripCounts must be not empty");
 
   for (ssize_t i = tripCounts.size() - 1; i >= 0; --i) {
-    coords[i] = b.create<arith::RemSIOp>(index, tripCounts[i]);
-    index = b.create<arith::DivSIOp>(index, tripCounts[i]);
+    coords[i] = arith::RemSIOp::create(b, index, tripCounts[i]);
+    index = arith::DivSIOp::create(b, index, tripCounts[i]);
   }
 
   return coords;
@@ -276,15 +276,15 @@ static ParallelComputeFunction createParallelComputeFunction(
   BlockArgument blockSize = args.blockSize();
 
   // Constants used below.
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Materialize known constants as constant operation in the function body.
   auto values = [&](ArrayRef<BlockArgument> args, ArrayRef<IntegerAttr> attrs) {
     return llvm::to_vector(
         llvm::map_range(llvm::zip(args, attrs), [&](auto tuple) -> Value {
           if (IntegerAttr attr = std::get<1>(tuple))
-            return b.create<arith::ConstantOp>(attr);
+            return arith::ConstantOp::create(b, attr);
           return std::get<0>(tuple);
         }));
   };
@@ -303,17 +303,17 @@ static ParallelComputeFunction createParallelComputeFunction(
   // one-dimensional iteration space.
   Value tripCount = tripCounts[0];
   for (unsigned i = 1; i < tripCounts.size(); ++i)
-    tripCount = b.create<arith::MulIOp>(tripCount, tripCounts[i]);
+    tripCount = arith::MulIOp::create(b, tripCount, tripCounts[i]);
 
   // Find one-dimensional iteration bounds: [blockFirstIndex, blockLastIndex]:
   //   blockFirstIndex = blockIndex * blockSize
-  Value blockFirstIndex = b.create<arith::MulIOp>(blockIndex, blockSize);
+  Value blockFirstIndex = arith::MulIOp::create(b, blockIndex, blockSize);
 
   // The last one-dimensional index in the block defined by the `blockIndex`:
   //   blockLastIndex = min(blockFirstIndex + blockSize, tripCount) - 1
-  Value blockEnd0 = b.create<arith::AddIOp>(blockFirstIndex, blockSize);
-  Value blockEnd1 = b.create<arith::MinSIOp>(blockEnd0, tripCount);
-  Value blockLastIndex = b.create<arith::SubIOp>(blockEnd1, c1);
+  Value blockEnd0 = arith::AddIOp::create(b, blockFirstIndex, blockSize);
+  Value blockEnd1 = arith::MinSIOp::create(b, blockEnd0, tripCount);
+  Value blockLastIndex = arith::SubIOp::create(b, blockEnd1, c1);
 
   // Convert one-dimensional indices to multi-dimensional coordinates.
   auto blockFirstCoord = delinearize(b, blockFirstIndex, tripCounts);
@@ -326,7 +326,7 @@ static ParallelComputeFunction createParallelComputeFunction(
   // dimension when inner compute dimension contains multiple blocks.
   SmallVector<Value> blockEndCoord(op.getNumLoops());
   for (size_t i = 0; i < blockLastCoord.size(); ++i)
-    blockEndCoord[i] = b.create<arith::AddIOp>(blockLastCoord[i], c1);
+    blockEndCoord[i] = arith::AddIOp::create(b, blockLastCoord[i], c1);
 
   // Construct a loop nest out of scf.for operations that will iterate over
   // all coordinates in [blockFirstCoord, blockLastCoord] range.
@@ -369,20 +369,20 @@ static ParallelComputeFunction createParallelComputeFunction(
       ImplicitLocOpBuilder b(loc, nestedBuilder);
 
       // Compute induction variable for `loopIdx`.
-      computeBlockInductionVars[loopIdx] = b.create<arith::AddIOp>(
-          lowerBounds[loopIdx], b.create<arith::MulIOp>(iv, steps[loopIdx]));
+      computeBlockInductionVars[loopIdx] = arith::AddIOp::create(b,
+          lowerBounds[loopIdx], arith::MulIOp::create(b, iv, steps[loopIdx]));
 
       // Check if we are inside first or last iteration of the loop.
-      isBlockFirstCoord[loopIdx] = b.create<arith::CmpIOp>(
+      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(b,
           arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
-      isBlockLastCoord[loopIdx] = b.create<arith::CmpIOp>(
+      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(b,
           arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
 
       // Check if the previous loop is in its first or last iteration.
       if (loopIdx > 0) {
-        isBlockFirstCoord[loopIdx] = b.create<arith::AndIOp>(
+        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(b,
             isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
-        isBlockLastCoord[loopIdx] = b.create<arith::AndIOp>(
+        isBlockLastCoord[loopIdx] = arith::AndIOp::create(b,
             isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
       }
 
@@ -391,24 +391,24 @@ static ParallelComputeFunction createParallelComputeFunction(
         if (loopIdx + 1 >= op.getNumLoops() - numBlockAlignedInnerLoops) {
           // For block aligned loops we always iterate starting from 0 up to
           // the loop trip counts.
-          b.create<scf::ForOp>(c0, tripCounts[loopIdx + 1], c1, ValueRange(),
+          scf::ForOp::create(b, c0, tripCounts[loopIdx + 1], c1, ValueRange(),
                                workLoopBuilder(loopIdx + 1));
 
         } else {
           // Select nested loop lower/upper bounds depending on our position in
           // the multi-dimensional iteration space.
-          auto lb = b.create<arith::SelectOp>(isBlockFirstCoord[loopIdx],
+          auto lb = arith::SelectOp::create(b, isBlockFirstCoord[loopIdx],
                                               blockFirstCoord[loopIdx + 1], c0);
 
-          auto ub = b.create<arith::SelectOp>(isBlockLastCoord[loopIdx],
+          auto ub = arith::SelectOp::create(b, isBlockLastCoord[loopIdx],
                                               blockEndCoord[loopIdx + 1],
                                               tripCounts[loopIdx + 1]);
 
-          b.create<scf::ForOp>(lb, ub, c1, ValueRange(),
+          scf::ForOp::create(b, lb, ub, c1, ValueRange(),
                                workLoopBuilder(loopIdx + 1));
         }
 
-        b.create<scf::YieldOp>(loc);
+        scf::YieldOp::create(b, loc);
         return;
       }
 
@@ -419,13 +419,13 @@ static ParallelComputeFunction createParallelComputeFunction(
 
       for (auto &bodyOp : op.getRegion().front().without_terminator())
         b.clone(bodyOp, mapping);
-      b.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(b, loc);
     };
   };
 
-  b.create<scf::ForOp>(blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
+  scf::ForOp::create(b, blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
                        workLoopBuilder(0));
-  b.create<func::ReturnOp>(ValueRange());
+  func::ReturnOp::create(b, ValueRange());
 
   return {op.getNumLoops(), func, std::move(computeFuncType.captures)};
 }
@@ -485,8 +485,8 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   b.setInsertionPointToEnd(block);
 
   Type indexTy = b.getIndexType();
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
-  Value c2 = b.create<arith::ConstantIndexOp>(2);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
+  Value c2 = arith::ConstantIndexOp::create(b, 2);
 
   // Get the async group that will track async dispatch completion.
   Value group = block->getArgument(0);
@@ -501,7 +501,7 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   SmallVector<Location> locations = {loc, loc};
 
   // Create a recursive dispatch loop.
-  scf::WhileOp whileOp = b.create<scf::WhileOp>(types, operands);
+  scf::WhileOp whileOp = scf::WhileOp::create(b, types, operands);
   Block *before = b.createBlock(&whileOp.getBefore(), {}, types, locations);
   Block *after = b.createBlock(&whileOp.getAfter(), {}, types, locations);
 
@@ -511,10 +511,10 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
     b.setInsertionPointToEnd(before);
     Value start = before->getArgument(0);
     Value end = before->getArgument(1);
-    Value distance = b.create<arith::SubIOp>(end, start);
+    Value distance = arith::SubIOp::create(b, end, start);
     Value dispatch =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, distance, c1);
-    b.create<scf::ConditionOp>(dispatch, before->getArguments());
+        arith::CmpIOp::create(b, arith::CmpIPredicate::sgt, distance, c1);
+    scf::ConditionOp::create(b, dispatch, before->getArguments());
   }
 
   // Setup the async dispatch loop body: recursively call dispatch function
@@ -523,9 +523,9 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
     b.setInsertionPointToEnd(after);
     Value start = after->getArgument(0);
     Value end = after->getArgument(1);
-    Value distance = b.create<arith::SubIOp>(end, start);
-    Value halfDistance = b.create<arith::DivSIOp>(distance, c2);
-    Value midIndex = b.create<arith::AddIOp>(start, halfDistance);
+    Value distance = arith::SubIOp::create(b, end, start);
+    Value halfDistance = arith::DivSIOp::create(b, distance, c2);
+    Value midIndex = arith::AddIOp::create(b, start, halfDistance);
 
     // Call parallel compute function inside the async.execute region.
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
@@ -536,16 +536,16 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
       operands[1] = midIndex;
       operands[2] = end;
 
-      executeBuilder.create<func::CallOp>(executeLoc, func.getSymName(),
+      func::CallOp::create(executeBuilder, executeLoc, func.getSymName(),
                                           func.getResultTypes(), operands);
-      executeBuilder.create<async::YieldOp>(executeLoc, ValueRange());
+      async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to dispatch half of the block range.
-    auto execute = b.create<ExecuteOp>(TypeRange(), ValueRange(), ValueRange(),
+    auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
                                        executeBodyBuilder);
-    b.create<AddToGroupOp>(indexTy, execute.getToken(), group);
-    b.create<scf::YieldOp>(ValueRange({start, midIndex}));
+    AddToGroupOp::create(b, indexTy, execute.getToken(), group);
+    scf::YieldOp::create(b, ValueRange({start, midIndex}));
   }
 
   // After dispatching async operations to process the tail of the block range
@@ -557,10 +557,10 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   SmallVector<Value> computeFuncOperands = {blockStart};
   computeFuncOperands.append(forwardedInputs.begin(), forwardedInputs.end());
 
-  b.create<func::CallOp>(computeFunc.func.getSymName(),
+  func::CallOp::create(b, computeFunc.func.getSymName(),
                          computeFunc.func.getResultTypes(),
                          computeFuncOperands);
-  b.create<func::ReturnOp>(ValueRange());
+  func::ReturnOp::create(b, ValueRange());
 
   return func;
 }
@@ -578,8 +578,8 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
   func::FuncOp asyncDispatchFunction =
       createAsyncDispatchFunction(parallelComputeFunction, rewriter);
 
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Appends operands shared by async dispatch and parallel compute functions to
   // the given operands vector.
@@ -595,7 +595,7 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
   // completely. If this will be known statically, then canonicalization will
   // erase async group operations.
   Value isSingleBlock =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, blockCount, c1);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, blockCount, c1);
 
   auto syncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
     ImplicitLocOpBuilder b(loc, nestedBuilder);
@@ -604,10 +604,10 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     SmallVector<Value> operands = {c0, blockSize};
     appendBlockComputeOperands(operands);
 
-    b.create<func::CallOp>(parallelComputeFunction.func.getSymName(),
+    func::CallOp::create(b, parallelComputeFunction.func.getSymName(),
                            parallelComputeFunction.func.getResultTypes(),
                            operands);
-    b.create<scf::YieldOp>();
+    scf::YieldOp::create(b, );
   };
 
   auto asyncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -616,24 +616,24 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     // Create an async.group to wait on all async tokens from the concurrent
     // execution of multiple parallel compute function. First block will be
     // executed synchronously in the caller thread.
-    Value groupSize = b.create<arith::SubIOp>(blockCount, c1);
-    Value group = b.create<CreateGroupOp>(GroupType::get(ctx), groupSize);
+    Value groupSize = arith::SubIOp::create(b, blockCount, c1);
+    Value group = CreateGroupOp::create(b, GroupType::get(ctx), groupSize);
 
     // Launch async dispatch function for [0, blockCount) range.
     SmallVector<Value> operands = {group, c0, blockCount, blockSize};
     appendBlockComputeOperands(operands);
 
-    b.create<func::CallOp>(asyncDispatchFunction.getSymName(),
+    func::CallOp::create(b, asyncDispatchFunction.getSymName(),
                            asyncDispatchFunction.getResultTypes(), operands);
 
     // Wait for the completion of all parallel compute operations.
-    b.create<AwaitAllOp>(group);
+    AwaitAllOp::create(b, group);
 
-    b.create<scf::YieldOp>();
+    scf::YieldOp::create(b, );
   };
 
   // Dispatch either single block compute function, or launch async dispatch.
-  b.create<scf::IfOp>(isSingleBlock, syncDispatch, asyncDispatch);
+  scf::IfOp::create(b, isSingleBlock, syncDispatch, asyncDispatch);
 }
 
 // Dispatch parallel compute functions by submitting all async compute tasks
@@ -647,14 +647,14 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
 
   func::FuncOp compute = parallelComputeFunction.func;
 
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Create an async.group to wait on all async tokens from the concurrent
   // execution of multiple parallel compute function. First block will be
   // executed synchronously in the caller thread.
-  Value groupSize = b.create<arith::SubIOp>(blockCount, c1);
-  Value group = b.create<CreateGroupOp>(GroupType::get(ctx), groupSize);
+  Value groupSize = arith::SubIOp::create(b, blockCount, c1);
+  Value group = CreateGroupOp::create(b, GroupType::get(ctx), groupSize);
 
   // Call parallel compute function for all blocks.
   using LoopBodyBuilder =
@@ -681,28 +681,28 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     // Call parallel compute function inside the async.execute region.
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
                                   Location executeLoc, ValueRange executeArgs) {
-      executeBuilder.create<func::CallOp>(executeLoc, compute.getSymName(),
+      func::CallOp::create(executeBuilder, executeLoc, compute.getSymName(),
                                           compute.getResultTypes(),
                                           computeFuncOperands(iv));
-      executeBuilder.create<async::YieldOp>(executeLoc, ValueRange());
+      async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to launch parallel computate function.
-    auto execute = b.create<ExecuteOp>(TypeRange(), ValueRange(), ValueRange(),
+    auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
                                        executeBodyBuilder);
-    b.create<AddToGroupOp>(rewriter.getIndexType(), execute.getToken(), group);
-    b.create<scf::YieldOp>();
+    AddToGroupOp::create(b, rewriter.getIndexType(), execute.getToken(), group);
+    scf::YieldOp::create(b, );
   };
 
   // Iterate over all compute blocks and launch parallel compute operations.
-  b.create<scf::ForOp>(c1, blockCount, c1, ValueRange(), loopBuilder);
+  scf::ForOp::create(b, c1, blockCount, c1, ValueRange(), loopBuilder);
 
   // Call parallel compute function for the first block in the caller thread.
-  b.create<func::CallOp>(compute.getSymName(), compute.getResultTypes(),
+  func::CallOp::create(b, compute.getSymName(), compute.getResultTypes(),
                          computeFuncOperands(c0));
 
   // Wait for the completion of all async compute operations.
-  b.create<AwaitAllOp>(group);
+  AwaitAllOp::create(b, group);
 }
 
 LogicalResult
@@ -738,17 +738,17 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
   // for the scf.parallel operation.
   Value tripCount = tripCounts[0];
   for (size_t i = 1; i < tripCounts.size(); ++i)
-    tripCount = b.create<arith::MulIOp>(tripCount, tripCounts[i]);
+    tripCount = arith::MulIOp::create(b, tripCount, tripCounts[i]);
 
   // Short circuit no-op parallel loops (zero iterations) that can arise from
   // the memrefs with dynamic dimension(s) equal to zero.
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
   Value isZeroIterations =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, tripCount, c0);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, tripCount, c0);
 
   // Do absolutely nothing if the trip count is zero.
   auto noOp = [&](OpBuilder &nestedBuilder, Location loc) {
-    nestedBuilder.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(nestedBuilder, loc);
   };
 
   // Compute the parallel block size and dispatch concurrent tasks computing
@@ -798,9 +798,9 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
     Value numWorkerThreadsVal;
     if (numWorkerThreads >= 0)
-      numWorkerThreadsVal = b.create<arith::ConstantIndexOp>(numWorkerThreads);
+      numWorkerThreadsVal = arith::ConstantIndexOp::create(b, numWorkerThreads);
     else
-      numWorkerThreadsVal = b.create<async::RuntimeNumWorkerThreadsOp>();
+      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b, );
 
     // With large number of threads the value of creating many compute blocks
     // is reduced because the problem typically becomes memory bound. For this
@@ -819,38 +819,38 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
         {4, 4.0f}, {8, 2.0f}, {16, 1.0f}, {32, 0.8f}, {64, 0.6f}};
     const float initialOvershardingFactor = 8.0f;
 
-    Value scalingFactor = b.create<arith::ConstantFloatOp>(
+    Value scalingFactor = arith::ConstantFloatOp::create(b,
         b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
     for (const std::pair<int, float> &p : overshardingBrackets) {
-      Value bracketBegin = b.create<arith::ConstantIndexOp>(p.first);
-      Value inBracket = b.create<arith::CmpIOp>(
+      Value bracketBegin = arith::ConstantIndexOp::create(b, p.first);
+      Value inBracket = arith::CmpIOp::create(b,
           arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
-      Value bracketScalingFactor = b.create<arith::ConstantFloatOp>(
+      Value bracketScalingFactor = arith::ConstantFloatOp::create(b,
           b.getF32Type(), llvm::APFloat(p.second));
-      scalingFactor = b.create<arith::SelectOp>(inBracket, bracketScalingFactor,
+      scalingFactor = arith::SelectOp::create(b, inBracket, bracketScalingFactor,
                                                 scalingFactor);
     }
     Value numWorkersIndex =
-        b.create<arith::IndexCastOp>(b.getI32Type(), numWorkerThreadsVal);
+        arith::IndexCastOp::create(b, b.getI32Type(), numWorkerThreadsVal);
     Value numWorkersFloat =
-        b.create<arith::SIToFPOp>(b.getF32Type(), numWorkersIndex);
+        arith::SIToFPOp::create(b, b.getF32Type(), numWorkersIndex);
     Value scaledNumWorkers =
-        b.create<arith::MulFOp>(scalingFactor, numWorkersFloat);
+        arith::MulFOp::create(b, scalingFactor, numWorkersFloat);
     Value scaledNumInt =
-        b.create<arith::FPToSIOp>(b.getI32Type(), scaledNumWorkers);
+        arith::FPToSIOp::create(b, b.getI32Type(), scaledNumWorkers);
     Value scaledWorkers =
-        b.create<arith::IndexCastOp>(b.getIndexType(), scaledNumInt);
+        arith::IndexCastOp::create(b, b.getIndexType(), scaledNumInt);
 
-    Value maxComputeBlocks = b.create<arith::MaxSIOp>(
-        b.create<arith::ConstantIndexOp>(1), scaledWorkers);
+    Value maxComputeBlocks = arith::MaxSIOp::create(b,
+        arith::ConstantIndexOp::create(b, 1), scaledWorkers);
 
     // Compute parallel block size from the parallel problem size:
     //   blockSize = min(tripCount,
     //                   max(ceil_div(tripCount, maxComputeBlocks),
     //                       minTaskSize))
-    Value bs0 = b.create<arith::CeilDivSIOp>(tripCount, maxComputeBlocks);
-    Value bs1 = b.create<arith::MaxSIOp>(bs0, minTaskSize);
-    Value blockSize = b.create<arith::MinSIOp>(tripCount, bs1);
+    Value bs0 = arith::CeilDivSIOp::create(b, tripCount, maxComputeBlocks);
+    Value bs1 = arith::MaxSIOp::create(b, bs0, minTaskSize);
+    Value blockSize = arith::MinSIOp::create(b, tripCount, bs1);
 
     // Dispatch parallel compute function using async recursive work splitting,
     // or by submitting compute task sequentially from a caller thread.
@@ -860,7 +860,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     // the parallel operation body for a subset of iteration space.
 
     // Compute the number of parallel compute blocks.
-    Value blockCount = b.create<arith::CeilDivSIOp>(tripCount, blockSize);
+    Value blockCount = arith::CeilDivSIOp::create(b, tripCount, blockSize);
 
     // Dispatch parallel compute function without hints to unroll inner loops.
     auto dispatchDefault = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -869,7 +869,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       doDispatch(b, rewriter, compute, op, blockSize, blockCount, tripCounts);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     };
 
     // Dispatch parallel compute function with hints for unrolling inner loops.
@@ -880,34 +880,34 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       // Align the block size to be a multiple of the statically known
       // number of iterations in the inner loops.
-      Value numIters = b.create<arith::ConstantIndexOp>(
+      Value numIters = arith::ConstantIndexOp::create(b,
           numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value alignedBlockSize = b.create<arith::MulIOp>(
-          b.create<arith::CeilDivSIOp>(blockSize, numIters), numIters);
+      Value alignedBlockSize = arith::MulIOp::create(b,
+          arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
       doDispatch(b, rewriter, compute, op, alignedBlockSize, blockCount,
                  tripCounts);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     };
 
     // Dispatch to block aligned compute function only if the computed block
     // size is larger than the number of iterations in the unrollable inner
     // loops, because otherwise it can reduce the available parallelism.
     if (numUnrollableLoops > 0) {
-      Value numIters = b.create<arith::ConstantIndexOp>(
+      Value numIters = arith::ConstantIndexOp::create(b,
           numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value useBlockAlignedComputeFn = b.create<arith::CmpIOp>(
+      Value useBlockAlignedComputeFn = arith::CmpIOp::create(b,
           arith::CmpIPredicate::sge, blockSize, numIters);
 
-      b.create<scf::IfOp>(useBlockAlignedComputeFn, dispatchBlockAligned,
+      scf::IfOp::create(b, useBlockAlignedComputeFn, dispatchBlockAligned,
                           dispatchDefault);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     } else {
       dispatchDefault(b, loc);
     }
   };
 
   // Replace the `scf.parallel` operation with the parallel compute function.
-  b.create<scf::IfOp>(isZeroIterations, noOp, dispatch);
+  scf::IfOp::create(b, isZeroIterations, noOp, dispatch);
 
   // Parallel operation was replaced with a block iteration loop.
   rewriter.eraseOp(op);
@@ -922,7 +922,7 @@ void AsyncParallelForPass::runOnOperation() {
   populateAsyncParallelForPatterns(
       patterns, asyncDispatch, numWorkerThreads,
       [&](ImplicitLocOpBuilder builder, scf::ParallelOp op) {
-        return builder.create<arith::ConstantIndexOp>(minTaskSize);
+        return arith::ConstantIndexOp::create(builder, minTaskSize);
       });
   if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
     signalPassFailure();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
index 0da9b3a730f85..25ce0fea50581 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
@@ -48,7 +48,7 @@ static LogicalResult dropRefIfNoUses(Value value, unsigned count = 1) {
   else
     b.setInsertionPointToStart(value.getParentBlock());
 
-  b.create<RuntimeDropRefOp>(value.getLoc(), value, b.getI64IntegerAttr(1));
+  RuntimeDropRefOp::create(b, value.getLoc(), value, b.getI64IntegerAttr(1));
   return success();
 }
 
@@ -309,7 +309,7 @@ LogicalResult AsyncRuntimeRefCountingPass::addDropRefAfterLastUse(Value value) {
 
     // Add a drop_ref immediately after the last user.
     builder.setInsertionPointAfter(lastUser);
-    builder.create<RuntimeDropRefOp>(loc, value, builder.getI64IntegerAttr(1));
+    RuntimeDropRefOp::create(builder, loc, value, builder.getI64IntegerAttr(1));
   }
 
   return success();
@@ -327,7 +327,7 @@ AsyncRuntimeRefCountingPass::addAddRefBeforeFunctionCall(Value value) {
     // Add a reference before the function call to pass the value at `+1`
     // reference to the function entry block.
     builder.setInsertionPoint(user);
-    builder.create<RuntimeAddRefOp>(loc, value, builder.getI64IntegerAttr(1));
+    RuntimeAddRefOp::create(builder, loc, value, builder.getI64IntegerAttr(1));
   }
 
   return success();
@@ -411,11 +411,11 @@ AsyncRuntimeRefCountingPass::addDropRefInDivergentLivenessSuccessor(
         refCountingBlock = &successor->getParent()->emplaceBlock();
         refCountingBlock->moveBefore(successor);
         OpBuilder builder = OpBuilder::atBlockEnd(refCountingBlock);
-        builder.create<cf::BranchOp>(value.getLoc(), successor);
+        cf::BranchOp::create(builder, value.getLoc(), successor);
       }
 
       OpBuilder builder = OpBuilder::atBlockBegin(refCountingBlock);
-      builder.create<RuntimeDropRefOp>(value.getLoc(), value,
+      RuntimeDropRefOp::create(builder, value.getLoc(), value,
                                        builder.getI64IntegerAttr(1));
 
       // No need to update the terminator operation.
@@ -507,13 +507,13 @@ AsyncRuntimePolicyBasedRefCountingPass::addRefCounting(Value value) {
       // Create `add_ref` operation before the operand owner.
       if (cnt > 0) {
         b.setInsertionPoint(operand.getOwner());
-        b.create<RuntimeAddRefOp>(loc, value, b.getI64IntegerAttr(cnt));
+        RuntimeAddRefOp::create(b, loc, value, b.getI64IntegerAttr(cnt));
       }
 
       // Create `drop_ref` operation after the operand owner.
       if (cnt < 0) {
         b.setInsertionPointAfter(operand.getOwner());
-        b.create<RuntimeDropRefOp>(loc, value, b.getI64IntegerAttr(-cnt));
+        RuntimeDropRefOp::create(b, loc, value, b.getI64IntegerAttr(-cnt));
       }
     }
   }
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index 832d2ec65cf7e..0485ad99cf7d2 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -187,22 +187,22 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
 
   std::optional<Value> retToken;
   if (isStateful)
-    retToken.emplace(builder.create<RuntimeCreateOp>(TokenType::get(ctx)));
+    retToken.emplace(RuntimeCreateOp::create(builder, TokenType::get(ctx)));
 
   llvm::SmallVector<Value, 4> retValues;
   ArrayRef<Type> resValueTypes =
       isStateful ? func.getResultTypes().drop_front() : func.getResultTypes();
   for (auto resType : resValueTypes)
     retValues.emplace_back(
-        builder.create<RuntimeCreateOp>(resType).getResult());
+        RuntimeCreateOp::create(builder, resType).getResult());
 
   // ------------------------------------------------------------------------ //
   // Initialize coroutine: get coroutine id and coroutine handle.
   // ------------------------------------------------------------------------ //
-  auto coroIdOp = builder.create<CoroIdOp>(CoroIdType::get(ctx));
+  auto coroIdOp = CoroIdOp::create(builder, CoroIdType::get(ctx));
   auto coroHdlOp =
-      builder.create<CoroBeginOp>(CoroHandleType::get(ctx), coroIdOp.getId());
-  builder.create<cf::BranchOp>(originalEntryBlock);
+      CoroBeginOp::create(builder, CoroHandleType::get(ctx), coroIdOp.getId());
+  cf::BranchOp::create(builder, originalEntryBlock);
 
   Block *cleanupBlock = func.addBlock();
   Block *cleanupBlockForDestroy = func.addBlock();
@@ -213,10 +213,10 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   // ------------------------------------------------------------------------ //
   auto buildCleanupBlock = [&](Block *cb) {
     builder.setInsertionPointToStart(cb);
-    builder.create<CoroFreeOp>(coroIdOp.getId(), coroHdlOp.getHandle());
+    CoroFreeOp::create(builder, coroIdOp.getId(), coroHdlOp.getHandle());
 
     // Branch into the suspend block.
-    builder.create<cf::BranchOp>(suspendBlock);
+    cf::BranchOp::create(builder, suspendBlock);
   };
   buildCleanupBlock(cleanupBlock);
   buildCleanupBlock(cleanupBlockForDestroy);
@@ -228,7 +228,7 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   builder.setInsertionPointToStart(suspendBlock);
 
   // Mark the end of a coroutine: async.coro.end
-  builder.create<CoroEndOp>(coroHdlOp.getHandle());
+  CoroEndOp::create(builder, coroHdlOp.getHandle());
 
   // Return created optional `async.token` and `async.values` from the suspend
   // block. This will be the return value of a coroutine ramp function.
@@ -236,7 +236,7 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   if (retToken)
     ret.push_back(*retToken);
   llvm::append_range(ret, retValues);
-  builder.create<func::ReturnOp>(ret);
+  func::ReturnOp::create(builder, ret);
 
   // `async.await` op lowering will create resume blocks for async
   // continuations, and will conditionally branch to cleanup or suspend blocks.
@@ -273,13 +273,13 @@ static Block *setupSetErrorBlock(CoroMachinery &coro) {
 
   // Coroutine set_error block: set error on token and all returned values.
   if (coro.asyncToken)
-    builder.create<RuntimeSetErrorOp>(*coro.asyncToken);
+    RuntimeSetErrorOp::create(builder, *coro.asyncToken);
 
   for (Value retValue : coro.returnValues)
-    builder.create<RuntimeSetErrorOp>(retValue);
+    RuntimeSetErrorOp::create(builder, retValue);
 
   // Branch into the cleanup block.
-  builder.create<cf::BranchOp>(coro.cleanup);
+  cf::BranchOp::create(builder, coro.cleanup);
 
   return *coro.setError;
 }
@@ -334,13 +334,13 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Await on all dependencies before starting to execute the body region.
     for (size_t i = 0; i < numDependencies; ++i)
-      builder.create<AwaitOp>(func.getArgument(i));
+      AwaitOp::create(builder, func.getArgument(i));
 
     // Await on all async value operands and unwrap the payload.
     SmallVector<Value, 4> unwrappedOperands(numOperands);
     for (size_t i = 0; i < numOperands; ++i) {
       Value operand = func.getArgument(numDependencies + i);
-      unwrappedOperands[i] = builder.create<AwaitOp>(loc, operand).getResult();
+      unwrappedOperands[i] = AwaitOp::create(builder, loc, operand).getResult();
     }
 
     // Map from function inputs defined above the execute op to the function
@@ -367,14 +367,14 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Save the coroutine state: async.coro.save
     auto coroSaveOp =
-        builder.create<CoroSaveOp>(CoroStateType::get(ctx), coro.coroHandle);
+        CoroSaveOp::create(builder, CoroStateType::get(ctx), coro.coroHandle);
 
     // Pass coroutine to the runtime to be resumed on a runtime managed
     // thread.
-    builder.create<RuntimeResumeOp>(coro.coroHandle);
+    RuntimeResumeOp::create(builder, coro.coroHandle);
 
     // Add async.coro.suspend as a suspended block terminator.
-    builder.create<CoroSuspendOp>(coroSaveOp.getState(), coro.suspend,
+    CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend,
                                   branch.getDest(), coro.cleanupForDestroy);
 
     branch.erase();
@@ -383,7 +383,7 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
   // Replace the original `async.execute` with a call to outlined function.
   {
     ImplicitLocOpBuilder callBuilder(loc, execute);
-    auto callOutlinedFunc = callBuilder.create<func::CallOp>(
+    auto callOutlinedFunc = func::CallOp::create(callBuilder,
         func.getName(), execute.getResultTypes(), functionInputs.getArrayRef());
     execute.replaceAllUsesWith(callOutlinedFunc.getResults());
     execute.erase();
@@ -452,7 +452,7 @@ class AsyncFuncOpLowering : public OpConversionPattern<async::FuncOp> {
     Location loc = op->getLoc();
 
     auto newFuncOp =
-        rewriter.create<func::FuncOp>(loc, op.getName(), op.getFunctionType());
+        func::FuncOp::create(rewriter, loc, op.getName(), op.getFunctionType());
 
     SymbolTable::setSymbolVisibility(newFuncOp,
                                      SymbolTable::getSymbolVisibility(op));
@@ -522,16 +522,16 @@ class AsyncReturnOpLowering : public OpConversionPattern<async::ReturnOp> {
     for (auto tuple : llvm::zip(adaptor.getOperands(), coro.returnValues)) {
       Value returnValue = std::get<0>(tuple);
       Value asyncValue = std::get<1>(tuple);
-      rewriter.create<RuntimeStoreOp>(loc, returnValue, asyncValue);
-      rewriter.create<RuntimeSetAvailableOp>(loc, asyncValue);
+      RuntimeStoreOp::create(rewriter, loc, returnValue, asyncValue);
+      RuntimeSetAvailableOp::create(rewriter, loc, asyncValue);
     }
 
     if (coro.asyncToken)
       // Switch the coroutine completion token to available state.
-      rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
+      RuntimeSetAvailableOp::create(rewriter, loc, *coro.asyncToken);
 
     rewriter.eraseOp(op);
-    rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    cf::BranchOp::create(rewriter, loc, coro.cleanup);
     return success();
   }
 
@@ -582,15 +582,15 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
     // the async object (token, value or group) to become available.
     if (!isInCoroutine) {
       ImplicitLocOpBuilder builder(loc, rewriter);
-      builder.create<RuntimeAwaitOp>(loc, operand);
+      RuntimeAwaitOp::create(builder, loc, operand);
 
       // Assert that the awaited operands is not in the error state.
-      Value isError = builder.create<RuntimeIsErrorOp>(i1, operand);
-      Value notError = builder.create<arith::XOrIOp>(
-          isError, builder.create<arith::ConstantOp>(
+      Value isError = RuntimeIsErrorOp::create(builder, i1, operand);
+      Value notError = arith::XOrIOp::create(builder,
+          isError, arith::ConstantOp::create(builder,
                        loc, i1, builder.getIntegerAttr(i1, 1)));
 
-      builder.create<cf::AssertOp>(notError,
+      cf::AssertOp::create(builder, notError,
                                    "Awaited async operand is in error state");
     }
 
@@ -606,15 +606,15 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
       // Save the coroutine state and resume on a runtime managed thread when
       // the operand becomes available.
       auto coroSaveOp =
-          builder.create<CoroSaveOp>(CoroStateType::get(ctx), coro.coroHandle);
-      builder.create<RuntimeAwaitAndResumeOp>(operand, coro.coroHandle);
+          CoroSaveOp::create(builder, CoroStateType::get(ctx), coro.coroHandle);
+      RuntimeAwaitAndResumeOp::create(builder, operand, coro.coroHandle);
 
       // Split the entry block before the await operation.
       Block *resume = rewriter.splitBlock(suspended, Block::iterator(op));
 
       // Add async.coro.suspend as a suspended block terminator.
       builder.setInsertionPointToEnd(suspended);
-      builder.create<CoroSuspendOp>(coroSaveOp.getState(), coro.suspend, resume,
+      CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend, resume,
                                     coro.cleanupForDestroy);
 
       // Split the resume block into error checking and continuation.
@@ -622,8 +622,8 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
 
       // Check if the awaited value is in the error state.
       builder.setInsertionPointToStart(resume);
-      auto isError = builder.create<RuntimeIsErrorOp>(loc, i1, operand);
-      builder.create<cf::CondBranchOp>(isError,
+      auto isError = RuntimeIsErrorOp::create(builder, loc, i1, operand);
+      cf::CondBranchOp::create(builder, isError,
                                        /*trueDest=*/setupSetErrorBlock(coro),
                                        /*trueArgs=*/ArrayRef<Value>(),
                                        /*falseDest=*/continuation,
@@ -673,7 +673,7 @@ class AwaitValueOpLowering : public AwaitOpLoweringBase<AwaitOp, ValueType> {
                       ConversionPatternRewriter &rewriter) const override {
     // Load from the async value storage.
     auto valueType = cast<ValueType>(operand.getType()).getValueType();
-    return rewriter.create<RuntimeLoadOp>(op->getLoc(), valueType, operand);
+    return RuntimeLoadOp::create(rewriter, op->getLoc(), valueType, operand);
   }
 };
 
@@ -714,15 +714,15 @@ class YieldOpLowering : public OpConversionPattern<async::YieldOp> {
     for (auto tuple : llvm::zip(adaptor.getOperands(), coro.returnValues)) {
       Value yieldValue = std::get<0>(tuple);
       Value asyncValue = std::get<1>(tuple);
-      rewriter.create<RuntimeStoreOp>(loc, yieldValue, asyncValue);
-      rewriter.create<RuntimeSetAvailableOp>(loc, asyncValue);
+      RuntimeStoreOp::create(rewriter, loc, yieldValue, asyncValue);
+      RuntimeSetAvailableOp::create(rewriter, loc, asyncValue);
     }
 
     if (coro.asyncToken)
       // Switch the coroutine completion token to available state.
-      rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
+      RuntimeSetAvailableOp::create(rewriter, loc, *coro.asyncToken);
 
-    rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    cf::BranchOp::create(rewriter, loc, coro.cleanup);
     rewriter.eraseOp(op);
 
     return success();
@@ -756,7 +756,7 @@ class AssertOpLowering : public OpConversionPattern<cf::AssertOp> {
 
     Block *cont = rewriter.splitBlock(op->getBlock(), Block::iterator(op));
     rewriter.setInsertionPointToEnd(cont->getPrevNode());
-    rewriter.create<cf::CondBranchOp>(loc, adaptor.getArg(),
+    cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(),
                                       /*trueDest=*/cont,
                                       /*trueArgs=*/ArrayRef<Value>(),
                                       /*falseDest=*/setupSetErrorBlock(coro),
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index 2bf326a07783e..a1deab67f6bfa 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -35,7 +35,7 @@ using namespace bufferization;
 //===----------------------------------------------------------------------===//
 
 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
-  return builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
 }
 
 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
@@ -150,7 +150,7 @@ DeallocationState::getMemrefWithUniqueOwnership(OpBuilder &builder,
   // ownerships more intelligently to not end up with an 'Unknown' ownership in
   // the first place.
   auto cloneOp =
-      builder.create<bufferization::CloneOp>(memref.getLoc(), memref);
+      bufferization::CloneOp::create(builder, memref.getLoc(), memref);
   Value condition = buildBoolValue(builder, memref.getLoc(), true);
   Value newMemref = cloneOp.getResult();
   updateOwnership(newMemref, condition);
@@ -196,7 +196,7 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // Simply cast unranked MemRefs to ranked memrefs with 0 dimensions such
     // that we can call extract_strided_metadata on it.
     if (auto unrankedMemRefTy = dyn_cast<UnrankedMemRefType>(memref.getType()))
-      memref = builder.create<memref::ReinterpretCastOp>(
+      memref = memref::ReinterpretCastOp::create(builder,
           loc, memref,
           /*offset=*/builder.getIndexAttr(0),
           /*sizes=*/ArrayRef<OpFoldResult>{},
@@ -207,7 +207,7 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // alloc operation has to be passed to the dealloc operation. Passing
     // subviews, etc. to a dealloc operation is not allowed.
     memrefs.push_back(
-        builder.create<memref::ExtractStridedMetadataOp>(loc, memref)
+        memref::ExtractStridedMetadataOp::create(builder, loc, memref)
             .getResult(0));
     conditions.push_back(ownership.getIndicator());
   }
@@ -296,7 +296,7 @@ FailureOr<Operation *> deallocation_impl::insertDeallocOpForReturnLike(
   if (memrefs.empty() && toRetain.empty())
     return op;
 
-  auto deallocOp = builder.create<bufferization::DeallocOp>(
+  auto deallocOp = bufferization::DeallocOp::create(builder,
       op->getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 8f17a82fabe03..05a1367a6b126 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -170,7 +170,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   if (llvm::isa<RankedTensorType>(shapedValue.getType())) {
     tensor = shapedValue;
   } else if (llvm::isa<MemRefType>(shapedValue.getType())) {
-    tensor = b.create<ToTensorOp>(
+    tensor = ToTensorOp::create(b,
         loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
         shapedValue);
   } else if (llvm::isa<UnrankedTensorType>(shapedValue.getType()) ||
@@ -209,7 +209,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   }
 
   // Create AllocTensorOp.
-  auto allocTensorOp = b.create<AllocTensorOp>(loc, tensorType, dynamicSizes,
+  auto allocTensorOp = AllocTensorOp::create(b, loc, tensorType, dynamicSizes,
                                                copy ? tensor : Value());
 
   // Add 'memory_space' attribute. Not needed if 'copy' operand is specified.
@@ -753,7 +753,7 @@ void bufferization::replaceOpWithBufferizedValues(RewriterBase &rewriter,
       // ToTensorOp. Throughout bufferization, this ToTensorOp will gradually
       // loose all of its users and eventually DCE away.
       rewriter.setInsertionPointAfter(op);
-      replacement = rewriter.create<bufferization::ToTensorOp>(
+      replacement = bufferization::ToTensorOp::create(rewriter,
           replacement.getLoc(), opResult.getType(), replacement);
     }
     replacements.push_back(replacement);
@@ -779,7 +779,7 @@ FailureOr<Value> BufferizationOptions::createAlloc(OpBuilder &b, Location loc,
         .create<memref::AllocOp>(loc, type, dynShape,
                                  b.getI64IntegerAttr(bufferAlignment))
         .getResult();
-  return b.create<memref::AllocOp>(loc, type, dynShape).getResult();
+  return memref::AllocOp::create(b, loc, type, dynShape).getResult();
 }
 
 /// Create a memory copy between two memref buffers.
@@ -788,7 +788,7 @@ LogicalResult BufferizationOptions::createMemCpy(OpBuilder &b, Location loc,
   if (memCpyFn)
     return (*memCpyFn)(b, loc, from, to);
 
-  b.create<memref::CopyOp>(loc, from, to);
+  memref::CopyOp::create(b, loc, from, to);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 875a06546c9f0..e64fee235e50b 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -58,7 +58,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
   // a fix extra conditions in `isGuaranteedCastCompatible`.
   if (memref::CastOp::areCastCompatible(srcType, destType) &&
       isGuaranteedCastCompatible(srcType, destType)) {
-    Value casted = b.create<memref::CastOp>(value.getLoc(), destType, value);
+    Value casted = memref::CastOp::create(b, value.getLoc(), destType, value);
     return casted;
   }
 
@@ -67,7 +67,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
   for (int i = 0; i < destType.getRank(); ++i) {
     if (destType.getShape()[i] != ShapedType::kDynamic)
       continue;
-    Value size = b.create<memref::DimOp>(loc, value, i);
+    Value size = memref::DimOp::create(b, loc, value, i);
     dynamicOperands.push_back(size);
   }
 
@@ -134,10 +134,10 @@ void mlir::bufferization::populateDynamicDimSizes(
   for (int64_t i = 0; i < shapedType.getRank(); ++i) {
     if (shapedType.isDynamicDim(i)) {
       if (llvm::isa<MemRefType>(shapedType)) {
-        dynamicDims.push_back(b.create<memref::DimOp>(loc, shapedValue, i));
+        dynamicDims.push_back(memref::DimOp::create(b, loc, shapedValue, i));
       } else {
         assert(llvm::isa<RankedTensorType>(shapedType) && "expected tensor");
-        dynamicDims.push_back(b.create<tensor::DimOp>(loc, shapedValue, i));
+        dynamicDims.push_back(tensor::DimOp::create(b, loc, shapedValue, i));
       }
     }
   }
@@ -321,7 +321,7 @@ struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
         newShape, op.getType().getElementType(), op.getType().getEncoding());
     if (newType == op.getType())
       return failure();
-    auto newOp = rewriter.create<AllocTensorOp>(
+    auto newOp = AllocTensorOp::create(rewriter,
         op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
@@ -427,7 +427,7 @@ void AllocTensorOp::print(OpAsmPrinter &p) {
 Value AllocTensorOp::getDynamicSize(OpBuilder &b, unsigned idx) {
   assert(isDynamicDim(idx) && "expected dynamic dim");
   if (getCopy())
-    return b.create<tensor::DimOp>(getLoc(), getCopy(), idx);
+    return tensor::DimOp::create(b, getLoc(), getCopy(), idx);
   return getOperand(getIndexOfDynamicSize(idx));
 }
 
@@ -513,7 +513,7 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
     }
 
     if (source.getType() != cloneOp.getType())
-      source = rewriter.create<memref::CastOp>(cloneOp.getLoc(),
+      source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
                                                cloneOp.getType(), source);
     rewriter.replaceOp(cloneOp, source);
     rewriter.eraseOp(redundantDealloc);
@@ -538,7 +538,7 @@ LogicalResult DeallocTensorOp::bufferize(RewriterBase &rewriter,
   FailureOr<Value> buffer = getBuffer(rewriter, getTensor(), options, state);
   if (failed(buffer))
     return failure();
-  rewriter.create<memref::DeallocOp>(getLoc(), *buffer);
+  memref::DeallocOp::create(rewriter, getLoc(), *buffer);
   rewriter.eraseOp(getOperation());
   return success();
 }
@@ -643,7 +643,7 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return builder.create<ToTensorOp>(
+  return ToTensorOp::create(builder,
       loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
       /*restrict=*/true, getWritable());
 }
@@ -806,7 +806,7 @@ struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
       return failure();
     auto memrefType = MemRefType::get(srcTensorType.getShape(),
                                       srcTensorType.getElementType());
-    Value memref = rewriter.create<ToBufferOp>(toBuffer.getLoc(), memrefType,
+    Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
                                                tensorCastOperand.getOperand());
     rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                 memref);
@@ -880,12 +880,12 @@ LogicalResult ToBufferOp::bufferize(RewriterBase &rewriter,
 
 std::optional<Operation *> CloneOp::buildDealloc(OpBuilder &builder,
                                                  Value alloc) {
-  return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
+  return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
       .getOperation();
 }
 
 std::optional<Value> CloneOp::buildClone(OpBuilder &builder, Value alloc) {
-  return builder.create<CloneOp>(alloc.getLoc(), alloc).getResult();
+  return CloneOp::create(builder, alloc.getLoc(), alloc).getResult();
 }
 
 //===----------------------------------------------------------------------===//
@@ -959,7 +959,7 @@ struct DeallocRemoveDuplicateDeallocMemrefs
         Value &newCond = newConditions[memrefToCondition[memref]];
         if (newCond != cond)
           newCond =
-              rewriter.create<arith::OrIOp>(deallocOp.getLoc(), newCond, cond);
+              arith::OrIOp::create(rewriter, deallocOp.getLoc(), newCond, cond);
       } else {
         memrefToCondition.insert({memref, newConditions.size()});
         newMemrefs.push_back(memref);
@@ -1014,7 +1014,7 @@ struct DeallocRemoveDuplicateRetainedMemrefs
     // We need to create a new op because the number of results is always the
     // same as the number of condition operands.
     auto newDeallocOp =
-        rewriter.create<DeallocOp>(deallocOp.getLoc(), deallocOp.getMemrefs(),
+        DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
                                    deallocOp.getConditions(), newRetained);
     SmallVector<Value> replacements(
         llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
@@ -1036,7 +1036,7 @@ struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     if (deallocOp.getMemrefs().empty()) {
-      Value constFalse = rewriter.create<arith::ConstantOp>(
+      Value constFalse = arith::ConstantOp::create(rewriter,
           deallocOp.getLoc(), rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
diff --git a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
index db1eb20512033..7f495b0ac164c 100644
--- a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
@@ -70,12 +70,12 @@ transform::OneShotBufferizeOp::apply(transform::TransformRewriter &rewriter,
         *getFunctionBoundaryTypeConversion());
   if (getMemcpyOp() == "memref.copy") {
     options.memCpyFn = [](OpBuilder &b, Location loc, Value from, Value to) {
-      b.create<memref::CopyOp>(loc, from, to);
+      memref::CopyOp::create(b, loc, from, to);
       return success();
     };
   } else if (getMemcpyOp() == "linalg.copy") {
     options.memCpyFn = [](OpBuilder &b, Location loc, Value from, Value to) {
-      b.create<linalg::CopyOp>(loc, from, to);
+      linalg::CopyOp::create(b, loc, from, to);
       return success();
     };
   } else {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index c5fab80ecaa08..835ad61f54da2 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -167,7 +167,7 @@ struct RemoveDeallocMemrefsContainedInRetained
       std::optional<bool> analysisResult =
           analysis.isSameAllocation(retained, memref);
       if (analysisResult == true) {
-        auto disjunction = rewriter.create<arith::OrIOp>(
+        auto disjunction = arith::OrIOp::create(rewriter,
             deallocOp.getLoc(), updatedCondition, cond);
         rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(),
                                       disjunction);
@@ -247,14 +247,14 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias
         continue;
       }
 
-      replacements.push_back(rewriter.create<arith::ConstantOp>(
+      replacements.push_back(arith::ConstantOp::create(rewriter,
           deallocOp.getLoc(), rewriter.getBoolAttr(false)));
     }
 
     if (newRetainedMemrefs.size() == deallocOp.getRetained().size())
       return failure();
 
-    auto newDeallocOp = rewriter.create<DeallocOp>(
+    auto newDeallocOp = DeallocOp::create(rewriter,
         deallocOp.getLoc(), deallocOp.getMemrefs(), deallocOp.getConditions(),
         newRetainedMemrefs);
     int i = 0;
@@ -326,7 +326,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       }
 
       // Create new bufferization.dealloc op for `memref`.
-      auto newDeallocOp = rewriter.create<DeallocOp>(loc, memref, cond,
+      auto newDeallocOp = DeallocOp::create(rewriter, loc, memref, cond,
                                                      deallocOp.getRetained());
       updatedConditions.push_back(
           llvm::to_vector(ValueRange(newDeallocOp.getUpdatedConditions())));
@@ -337,7 +337,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       return failure();
 
     // Create bufferization.dealloc op for all remaining memrefs.
-    auto newDeallocOp = rewriter.create<DeallocOp>(
+    auto newDeallocOp = DeallocOp::create(rewriter,
         loc, remainingMemrefs, remainingConditions, deallocOp.getRetained());
 
     // Bit-or all conditions.
@@ -347,7 +347,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       assert(replacements.size() == additionalConditions.size() &&
              "expected same number of updated conditions");
       for (int64_t i = 0, e = replacements.size(); i < e; ++i) {
-        replacements[i] = rewriter.create<arith::OrIOp>(
+        replacements[i] = arith::OrIOp::create(rewriter,
             loc, replacements[i], additionalConditions[i]);
       }
     }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index 6924e88c50e1b..52490a0b2293b 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -132,7 +132,7 @@ updateReturnOps(func::FuncOp func, ArrayRef<BlockArgument> appendedEntryArgs,
           return WalkResult::interrupt();
       }
     }
-    builder.create<func::ReturnOp>(op.getLoc(), keepAsReturnOperands);
+    func::ReturnOp::create(builder, op.getLoc(), keepAsReturnOperands);
     op.erase();
     return WalkResult::advance();
   });
@@ -190,7 +190,7 @@ updateCalls(ModuleOp module,
         assert(hasFullyDynamicLayoutMap(memrefType) &&
                "layout map not supported");
         outParam =
-            builder.create<memref::CastOp>(op.getLoc(), memrefType, outParam);
+            memref::CastOp::create(builder, op.getLoc(), memrefType, outParam);
       }
       memref.replaceAllUsesWith(outParam);
       outParams.push_back(outParam);
@@ -200,7 +200,7 @@ updateCalls(ModuleOp module,
     newOperands.append(outParams.begin(), outParams.end());
     auto newResultTypes = llvm::to_vector<6>(llvm::map_range(
         replaceWithNewCallResults, [](Value v) { return v.getType(); }));
-    auto newCall = builder.create<func::CallOp>(op.getLoc(), op.getCalleeAttr(),
+    auto newCall = func::CallOp::create(builder, op.getLoc(), op.getCalleeAttr(),
                                                 newResultTypes, newOperands);
     for (auto t : llvm::zip(replaceWithNewCallResults, newCall.getResults()))
       std::get<0>(t).replaceAllUsesWith(std::get<1>(t));
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
index a66be7d3b2f52..80984f21df551 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
@@ -141,7 +141,7 @@ bufferization::getGlobalFor(arith::ConstantOp constantOp,
       cast<MemRefType>(getMemRefTypeWithStaticIdentityLayout(type));
   if (memorySpace)
     memrefType = MemRefType::Builder(memrefType).setMemorySpace(memorySpace);
-  auto global = globalBuilder.create<memref::GlobalOp>(
+  auto global = memref::GlobalOp::create(globalBuilder,
       constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
       /*sym_visibility=*/globalBuilder.getStringAttr("private"),
       /*type=*/memrefType,
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 246555dc8c699..4a68ecba79fb7 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -434,7 +434,7 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
     // Replace all uses of the original tensor bbArg.
     rewriter.setInsertionPointToStart(block);
     if (!bbArgUses.empty()) {
-      Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+      Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
           bbArg.getLoc(), tensorType, bbArg);
       for (OpOperand *use : bbArgUses)
         use->set(toTensorOp);
@@ -466,12 +466,12 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
       if (failed(operandBufferType))
         return failure();
       rewriter.setInsertionPointAfterValue(operand);
-      Value bufferizedOperand = rewriter.create<bufferization::ToBufferOp>(
+      Value bufferizedOperand = bufferization::ToBufferOp::create(rewriter,
           operand.getLoc(), *operandBufferType, operand);
       // A cast is needed if the operand and the block argument have different
       // bufferized types.
       if (type != *operandBufferType)
-        bufferizedOperand = rewriter.create<memref::CastOp>(
+        bufferizedOperand = memref::CastOp::create(rewriter,
             operand.getLoc(), type, bufferizedOperand);
       newOperands.push_back(bufferizedOperand);
     }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index c10d290f5d0ed..42d63b66485d5 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -118,7 +118,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
     // Update function calls.
     for (func::CallOp callOp : callerMap[funcOp]) {
       rewriter.setInsertionPoint(callOp);
-      auto newCallOp = rewriter.create<func::CallOp>(callOp.getLoc(), funcOp,
+      auto newCallOp = func::CallOp::create(rewriter, callOp.getLoc(), funcOp,
                                                      callOp.getOperands());
       SmallVector<Value> newResults;
       int64_t nextResult = 0;
@@ -134,7 +134,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
         Type expectedType = callOp.getResult(i).getType();
         if (replacement.getType() != expectedType) {
           // A cast must be inserted at the call site.
-          replacement = rewriter.create<memref::CastOp>(
+          replacement = memref::CastOp::create(rewriter,
               callOp.getLoc(), expectedType, replacement);
         }
         newResults.push_back(replacement);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
index b7db2e847a335..923cbc0e2259d 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
@@ -168,7 +168,7 @@ LogicalResult mlir::bufferization::eliminateEmptyTensors(
             cast<ShapedType>(v.getType()).getElementType())
           continue;
         rewriter.setInsertionPointAfterValue(replacement);
-        replacement = rewriter.create<tensor::CastOp>(v.getLoc(), v.getType(),
+        replacement = tensor::CastOp::create(rewriter, v.getLoc(), v.getType(),
                                                       replacement);
       }
       // Replace the specific use of the tensor::EmptyOp.
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index 2a98203da9d7d..025d5764c6d9e 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -319,7 +319,7 @@ struct CallOpInterface
     }
 
     // 3. Create the new CallOp.
-    Operation *newCallOp = rewriter.create<func::CallOp>(
+    Operation *newCallOp = func::CallOp::create(rewriter,
         callOp.getLoc(), funcOp.getSymName(), resultTypes, newOperands);
     newCallOp->setAttrs(callOp->getAttrs());
 
@@ -483,7 +483,7 @@ struct FuncOpInterface
 
         // Note: If `inferFunctionResultLayout = true`, casts are later folded
         // away.
-        Value toBufferOp = rewriter.create<bufferization::ToBufferOp>(
+        Value toBufferOp = bufferization::ToBufferOp::create(rewriter,
             returnOp.getLoc(), bufferizedType, returnVal);
         returnValues.push_back(toBufferOp);
       }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
index a611126e320ad..83ccd573a2734 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
@@ -64,8 +64,8 @@ class DeallocOpConversion
 
     rewriter.replaceOpWithNewOp<scf::IfOp>(
         op, adaptor.getConditions()[0], [&](OpBuilder &builder, Location loc) {
-          builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[0]);
-          builder.create<scf::YieldOp>(loc);
+          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
+          scf::YieldOp::create(builder, loc);
         });
     return success();
   }
@@ -108,13 +108,13 @@ class DeallocOpConversion
     // Compute the base pointer indices, compare all retained indices to the
     // memref index to check if they alias.
     SmallVector<Value> doesNotAliasList;
-    Value memrefAsIdx = rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(
+    Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(rewriter,
         op->getLoc(), adaptor.getMemrefs()[0]);
     for (Value retained : adaptor.getRetained()) {
       Value retainedAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op->getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op->getLoc(),
                                                                   retained);
-      Value doesNotAlias = rewriter.create<arith::CmpIOp>(
+      Value doesNotAlias = arith::CmpIOp::create(rewriter,
           op->getLoc(), arith::CmpIPredicate::ne, memrefAsIdx, retainedAsIdx);
       doesNotAliasList.push_back(doesNotAlias);
     }
@@ -122,17 +122,17 @@ class DeallocOpConversion
     // AND-reduce the list of booleans from above.
     Value prev = doesNotAliasList.front();
     for (Value doesNotAlias : ArrayRef(doesNotAliasList).drop_front())
-      prev = rewriter.create<arith::AndIOp>(op->getLoc(), prev, doesNotAlias);
+      prev = arith::AndIOp::create(rewriter, op->getLoc(), prev, doesNotAlias);
 
     // Also consider the condition given by the dealloc operation and perform a
     // conditional deallocation guarded by that value.
-    Value shouldDealloc = rewriter.create<arith::AndIOp>(
+    Value shouldDealloc = arith::AndIOp::create(rewriter,
         op->getLoc(), prev, adaptor.getConditions()[0]);
 
-    rewriter.create<scf::IfOp>(
+    scf::IfOp::create(rewriter,
         op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-          builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[0]);
-          builder.create<scf::YieldOp>(loc);
+          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
+          scf::YieldOp::create(builder, loc);
         });
 
     // Compute the replacement values for the dealloc operation results. This
@@ -140,12 +140,12 @@ class DeallocOpConversion
     // `select(does_alias_with_memref(r), memref_cond, false)` for each retained
     // value r.
     SmallVector<Value> replacements;
-    Value trueVal = rewriter.create<arith::ConstantOp>(
+    Value trueVal = arith::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getBoolAttr(true));
     for (Value doesNotAlias : doesNotAliasList) {
       Value aliases =
-          rewriter.create<arith::XOrIOp>(op->getLoc(), doesNotAlias, trueVal);
-      Value result = rewriter.create<arith::AndIOp>(op->getLoc(), aliases,
+          arith::XOrIOp::create(rewriter, op->getLoc(), doesNotAlias, trueVal);
+      Value result = arith::AndIOp::create(rewriter, op->getLoc(), aliases,
                                                     adaptor.getConditions()[0]);
       replacements.push_back(result);
     }
@@ -230,18 +230,18 @@ class DeallocOpConversion
     // Without storing them to memrefs, we could not use for-loops but only a
     // completely unrolled version of it, potentially leading to code-size
     // blow-up.
-    Value toDeallocMemref = rewriter.create<memref::AllocOp>(
+    Value toDeallocMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
                                      rewriter.getIndexType()));
-    Value conditionMemref = rewriter.create<memref::AllocOp>(
+    Value conditionMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getConditions().size()},
                                      rewriter.getI1Type()));
-    Value toRetainMemref = rewriter.create<memref::AllocOp>(
+    Value toRetainMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
                                      rewriter.getIndexType()));
 
     auto getConstValue = [&](uint64_t value) -> Value {
-      return rewriter.create<arith::ConstantOp>(op.getLoc(),
+      return arith::ConstantOp::create(rewriter, op.getLoc(),
                                                 rewriter.getIndexAttr(value));
     };
 
@@ -249,58 +249,58 @@ class DeallocOpConversion
     // at runtime.
     for (auto [i, toDealloc] : llvm::enumerate(adaptor.getMemrefs())) {
       Value memrefAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op.getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
                                                                   toDealloc);
-      rewriter.create<memref::StoreOp>(op.getLoc(), memrefAsIdx,
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx,
                                        toDeallocMemref, getConstValue(i));
     }
 
     for (auto [i, cond] : llvm::enumerate(adaptor.getConditions()))
-      rewriter.create<memref::StoreOp>(op.getLoc(), cond, conditionMemref,
+      memref::StoreOp::create(rewriter, op.getLoc(), cond, conditionMemref,
                                        getConstValue(i));
 
     for (auto [i, toRetain] : llvm::enumerate(adaptor.getRetained())) {
       Value memrefAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op.getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
                                                                   toRetain);
-      rewriter.create<memref::StoreOp>(op.getLoc(), memrefAsIdx, toRetainMemref,
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx, toRetainMemref,
                                        getConstValue(i));
     }
 
     // Cast the allocated memrefs to dynamic shape because we want only one
     // helper function no matter how many operands the bufferization.dealloc
     // has.
-    Value castedDeallocMemref = rewriter.create<memref::CastOp>(
+    Value castedDeallocMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toDeallocMemref);
-    Value castedCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         conditionMemref);
-    Value castedRetainMemref = rewriter.create<memref::CastOp>(
+    Value castedRetainMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toRetainMemref);
 
-    Value deallocCondsMemref = rewriter.create<memref::AllocOp>(
+    Value deallocCondsMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
                                      rewriter.getI1Type()));
-    Value retainCondsMemref = rewriter.create<memref::AllocOp>(
+    Value retainCondsMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
                                      rewriter.getI1Type()));
 
-    Value castedDeallocCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedDeallocCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         deallocCondsMemref);
-    Value castedRetainCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedRetainCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         retainCondsMemref);
 
     Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>();
-    rewriter.create<func::CallOp>(
+    func::CallOp::create(rewriter,
         op.getLoc(), deallocHelperFuncMap.lookup(symtableOp),
         SmallVector<Value>{castedDeallocMemref, castedRetainMemref,
                            castedCondsMemref, castedDeallocCondsMemref,
@@ -308,30 +308,30 @@ class DeallocOpConversion
 
     for (unsigned i = 0, e = adaptor.getMemrefs().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value shouldDealloc = rewriter.create<memref::LoadOp>(
+      Value shouldDealloc = memref::LoadOp::create(rewriter,
           op.getLoc(), deallocCondsMemref, idxValue);
-      rewriter.create<scf::IfOp>(
+      scf::IfOp::create(rewriter,
           op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-            builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[i]);
-            builder.create<scf::YieldOp>(loc);
+            memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[i]);
+            scf::YieldOp::create(builder, loc);
           });
     }
 
     SmallVector<Value> replacements;
     for (unsigned i = 0, e = adaptor.getRetained().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value ownership = rewriter.create<memref::LoadOp>(
+      Value ownership = memref::LoadOp::create(rewriter,
           op.getLoc(), retainCondsMemref, idxValue);
       replacements.push_back(ownership);
     }
 
     // Deallocate above allocated memrefs again to avoid memory leaks.
     // Deallocation will not be run on code after this stage.
-    rewriter.create<memref::DeallocOp>(op.getLoc(), toDeallocMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), toRetainMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), conditionMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), deallocCondsMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), retainCondsMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), toDeallocMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), toRetainMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), conditionMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), deallocCondsMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), retainCondsMemref);
 
     rewriter.replaceOp(op, replacements);
     return success();
@@ -349,7 +349,7 @@ class DeallocOpConversion
                   ConversionPatternRewriter &rewriter) const override {
     // Lower the trivial case.
     if (adaptor.getMemrefs().empty()) {
-      Value falseVal = rewriter.create<arith::ConstantOp>(
+      Value falseVal = arith::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           op, SmallVector<Value>(adaptor.getRetained().size(), falseVal));
@@ -449,30 +449,30 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
   Value retainCondsMemref = helperFuncOp.getArguments()[4];
 
   // Insert some prerequisites.
-  Value c0 = builder.create<arith::ConstantOp>(loc, builder.getIndexAttr(0));
-  Value c1 = builder.create<arith::ConstantOp>(loc, builder.getIndexAttr(1));
+  Value c0 = arith::ConstantOp::create(builder, loc, builder.getIndexAttr(0));
+  Value c1 = arith::ConstantOp::create(builder, loc, builder.getIndexAttr(1));
   Value trueValue =
-      builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(true));
+      arith::ConstantOp::create(builder, loc, builder.getBoolAttr(true));
   Value falseValue =
-      builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(false));
-  Value toDeallocSize = builder.create<memref::DimOp>(loc, toDeallocMemref, c0);
-  Value toRetainSize = builder.create<memref::DimOp>(loc, toRetainMemref, c0);
+      arith::ConstantOp::create(builder, loc, builder.getBoolAttr(false));
+  Value toDeallocSize = memref::DimOp::create(builder, loc, toDeallocMemref, c0);
+  Value toRetainSize = memref::DimOp::create(builder, loc, toRetainMemref, c0);
 
-  builder.create<scf::ForOp>(
+  scf::ForOp::create(builder,
       loc, c0, toRetainSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value i, ValueRange iterArgs) {
-        builder.create<memref::StoreOp>(loc, falseValue, retainCondsMemref, i);
-        builder.create<scf::YieldOp>(loc);
+        memref::StoreOp::create(builder, loc, falseValue, retainCondsMemref, i);
+        scf::YieldOp::create(builder, loc);
       });
 
-  builder.create<scf::ForOp>(
+  scf::ForOp::create(builder,
       loc, c0, toDeallocSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value outerIter,
           ValueRange iterArgs) {
         Value toDealloc =
-            builder.create<memref::LoadOp>(loc, toDeallocMemref, outerIter);
+            memref::LoadOp::create(builder, loc, toDeallocMemref, outerIter);
         Value cond =
-            builder.create<memref::LoadOp>(loc, conditionMemref, outerIter);
+            memref::LoadOp::create(builder, loc, conditionMemref, outerIter);
 
         // Build the first for loop that computes aliasing with retained
         // memrefs.
@@ -482,31 +482,31 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, toRetainSize, c1, trueValue,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value retainValue = builder.create<memref::LoadOp>(
+                      Value retainValue = memref::LoadOp::create(builder,
                           loc, toRetainMemref, i);
-                      Value doesAlias = builder.create<arith::CmpIOp>(
+                      Value doesAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::eq, retainValue,
                           toDealloc);
-                      builder.create<scf::IfOp>(
+                      scf::IfOp::create(builder,
                           loc, doesAlias,
                           [&](OpBuilder &builder, Location loc) {
                             Value retainCondValue =
-                                builder.create<memref::LoadOp>(
+                                memref::LoadOp::create(builder,
                                     loc, retainCondsMemref, i);
                             Value aggregatedRetainCond =
-                                builder.create<arith::OrIOp>(
+                                arith::OrIOp::create(builder,
                                     loc, retainCondValue, cond);
-                            builder.create<memref::StoreOp>(
+                            memref::StoreOp::create(builder,
                                 loc, aggregatedRetainCond, retainCondsMemref,
                                 i);
-                            builder.create<scf::YieldOp>(loc);
+                            scf::YieldOp::create(builder, loc);
                           });
-                      Value doesntAlias = builder.create<arith::CmpIOp>(
+                      Value doesntAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::ne, retainValue,
                           toDealloc);
-                      Value yieldValue = builder.create<arith::AndIOp>(
+                      Value yieldValue = arith::AndIOp::create(builder,
                           loc, iterArgs[0], doesntAlias);
-                      builder.create<scf::YieldOp>(loc, yieldValue);
+                      scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
@@ -518,24 +518,24 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, outerIter, c1, noRetainAlias,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value prevDeallocValue = builder.create<memref::LoadOp>(
+                      Value prevDeallocValue = memref::LoadOp::create(builder,
                           loc, toDeallocMemref, i);
-                      Value doesntAlias = builder.create<arith::CmpIOp>(
+                      Value doesntAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::ne, prevDeallocValue,
                           toDealloc);
-                      Value yieldValue = builder.create<arith::AndIOp>(
+                      Value yieldValue = arith::AndIOp::create(builder,
                           loc, iterArgs[0], doesntAlias);
-                      builder.create<scf::YieldOp>(loc, yieldValue);
+                      scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
-        Value shouldDealoc = builder.create<arith::AndIOp>(loc, noAlias, cond);
-        builder.create<memref::StoreOp>(loc, shouldDealoc, deallocCondsMemref,
+        Value shouldDealoc = arith::AndIOp::create(builder, loc, noAlias, cond);
+        memref::StoreOp::create(builder, loc, shouldDealoc, deallocCondsMemref,
                                         outerIter);
-        builder.create<scf::YieldOp>(loc);
+        scf::YieldOp::create(builder, loc);
       });
 
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
   return helperFuncOp;
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index 1eeafc4df8cf1..71890bfea4e5d 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -43,7 +43,7 @@ using namespace mlir::bufferization;
 //===----------------------------------------------------------------------===//
 
 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
-  return builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
 }
 
 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
@@ -755,12 +755,12 @@ Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
           .create<scf::IfOp>(
               memref.getLoc(), condition,
               [&](OpBuilder &builder, Location loc) {
-                builder.create<scf::YieldOp>(loc, newMemref);
+                scf::YieldOp::create(builder, loc, newMemref);
               },
               [&](OpBuilder &builder, Location loc) {
                 Value clone =
-                    builder.create<bufferization::CloneOp>(loc, newMemref);
-                builder.create<scf::YieldOp>(loc, clone);
+                    bufferization::CloneOp::create(builder, loc, newMemref);
+                scf::YieldOp::create(builder, loc, clone);
               })
           .getResult(0);
   Value trueVal = buildBoolValue(builder, memref.getLoc(), true);
@@ -797,7 +797,7 @@ BufferDeallocation::handleInterface(BranchOpInterface op) {
   state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
                            toRetain);
 
-  auto deallocOp = builder.create<bufferization::DeallocOp>(
+  auto deallocOp = bufferization::DeallocOp::create(builder,
       op.getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
@@ -885,10 +885,10 @@ BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
       builder.setInsertionPoint(op);
       Ownership ownership = state.getOwnership(operand, block);
       if (ownership.isUnique()) {
-        Value ownershipInverted = builder.create<arith::XOrIOp>(
+        Value ownershipInverted = arith::XOrIOp::create(builder,
             op.getLoc(), ownership.getIndicator(),
             buildBoolValue(builder, op.getLoc(), true));
-        builder.create<cf::AssertOp>(
+        cf::AssertOp::create(builder,
             op.getLoc(), ownershipInverted,
             "expected that the block does not have ownership");
       }
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
index f5a42c572ff96..925c72959bd3a 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
@@ -48,7 +48,7 @@ Operation *complex::ComplexDialect::materializeConstant(OpBuilder &builder,
                                                         Type type,
                                                         Location loc) {
   if (complex::ConstantOp::isBuildableWith(value, type)) {
-    return builder.create<complex::ConstantOp>(loc, type,
+    return complex::ConstantOp::create(builder, loc, type,
                                                llvm::cast<ArrayAttr>(value));
   }
   return arith::ConstantOp::materialize(builder, value, type, loc);
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 0c11c76cf1f71..38e47cc5e3714 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -312,7 +312,7 @@ struct SimplifyCondBranchIdenticalSuccessors
       if (std::get<0>(it) == std::get<1>(it))
         mergedOperands.push_back(std::get<0>(it));
       else
-        mergedOperands.push_back(rewriter.create<arith::SelectOp>(
+        mergedOperands.push_back(arith::SelectOp::create(rewriter,
             condbr.getLoc(), condition, std::get<0>(it), std::get<1>(it)));
     }
 
@@ -412,7 +412,7 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantTrue)
-            constantTrue = rewriter.create<arith::ConstantOp>(
+            constantTrue = arith::ConstantOp::create(rewriter,
                 condbr.getLoc(), ty, rewriter.getBoolAttr(true));
 
           rewriter.modifyOpInPlace(use.getOwner(),
@@ -427,7 +427,7 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantFalse)
-            constantFalse = rewriter.create<arith::ConstantOp>(
+            constantFalse = arith::ConstantOp::create(rewriter,
                 condbr.getLoc(), ty, rewriter.getBoolAttr(false));
 
           rewriter.modifyOpInPlace(use.getOwner(),
diff --git a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index a077f56f4f472..23f50930cb3b3 100644
--- a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -87,7 +87,7 @@ struct CondBranchOpInterface
                                destOperands.getAsOperandRange(), toRetain);
       SmallVector<Value> adaptedConditions(
           llvm::map_range(conditions, conditionModifier));
-      auto deallocOp = builder.create<bufferization::DeallocOp>(
+      auto deallocOp = bufferization::DeallocOp::create(builder,
           condBr.getLoc(), memrefs, adaptedConditions, toRetain);
       state.resetOwnerships(deallocOp.getRetained(), condBr->getBlock());
       for (auto [retained, ownership] : llvm::zip(
@@ -115,18 +115,18 @@ struct CondBranchOpInterface
     DeallocOp thenTakenDeallocOp = insertDeallocForBranch(
         condBr.getTrueDest(), condBr.getTrueDestOperandsMutable(),
         [&](Value cond) {
-          return builder.create<arith::AndIOp>(condBr.getLoc(), cond,
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond,
                                                condBr.getCondition());
         },
         thenMapping);
     DeallocOp elseTakenDeallocOp = insertDeallocForBranch(
         condBr.getFalseDest(), condBr.getFalseDestOperandsMutable(),
         [&](Value cond) {
-          Value trueVal = builder.create<arith::ConstantOp>(
+          Value trueVal = arith::ConstantOp::create(builder,
               condBr.getLoc(), builder.getBoolAttr(true));
-          Value negation = builder.create<arith::XOrIOp>(
+          Value negation = arith::XOrIOp::create(builder,
               condBr.getLoc(), trueVal, condBr.getCondition());
-          return builder.create<arith::AndIOp>(condBr.getLoc(), cond, negation);
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond, negation);
         },
         elseMapping);
 
@@ -143,7 +143,7 @@ struct CondBranchOpInterface
 
     for (Value retained : commonValues) {
       state.resetOwnerships(retained, condBr->getBlock());
-      Value combinedOwnership = builder.create<arith::SelectOp>(
+      Value combinedOwnership = arith::SelectOp::create(builder,
           condBr.getLoc(), condBr.getCondition(), thenMapping[retained],
           elseMapping[retained]);
       state.updateOwnership(retained, combinedOwnership, condBr->getBlock());
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index fccbca6ed05dd..568da8905cbc8 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -47,13 +47,13 @@ void EmitCDialect::initialize() {
 Operation *EmitCDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
-  return builder.create<emitc::ConstantOp>(loc, type, value);
+  return emitc::ConstantOp::create(builder, loc, type, value);
 }
 
 /// Default callback for builders of ops carrying a region. Inserts a yield
 /// without arguments.
 void mlir::emitc::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<emitc::YieldOp>(loc);
+  emitc::YieldOp::create(builder, loc);
 }
 
 bool mlir::emitc::isSupportedEmitCType(Type type) {
diff --git a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
index 12218f5072982..d5fe3b4ae1e7f 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
@@ -24,7 +24,7 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
   Location loc = op->getLoc();
 
   builder.setInsertionPointAfter(op);
-  auto expressionOp = builder.create<emitc::ExpressionOp>(loc, resultType);
+  auto expressionOp = emitc::ExpressionOp::create(builder, loc, resultType);
 
   // Replace all op's uses with the new expression's result.
   result.replaceAllUsesWith(expressionOp.getResult());
@@ -33,7 +33,7 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
   Region &region = expressionOp.getRegion();
   Block &block = region.emplaceBlock();
   builder.setInsertionPointToEnd(&block);
-  auto yieldOp = builder.create<emitc::YieldOp>(loc, result);
+  auto yieldOp = emitc::YieldOp::create(builder, loc, result);
 
   // Move op into the new expression.
   op->moveBefore(yieldOp);
diff --git a/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp b/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
index 72c8fd0f32485..ab7be8d6cedd9 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
@@ -21,7 +21,7 @@ Value materializeAsUnrealizedCast(OpBuilder &builder, Type resultType,
   if (inputs.size() != 1)
     return Value();
 
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
       .getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
index 17d436f6df028..72c331ad96607 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
@@ -50,7 +50,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
                                 PatternRewriter &rewriter) const override {
 
     auto className = funcOp.getSymNameAttr().str() + "Class";
-    ClassOp newClassOp = rewriter.create<ClassOp>(funcOp.getLoc(), className);
+    ClassOp newClassOp = ClassOp::create(rewriter, funcOp.getLoc(), className);
 
     SmallVector<std::pair<StringAttr, TypeAttr>> fields;
     rewriter.createBlock(&newClassOp.getBody());
@@ -67,7 +67,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
 
       TypeAttr typeAttr = TypeAttr::get(val.getType());
       fields.push_back({fieldName, typeAttr});
-      rewriter.create<emitc::FieldOp>(funcOp.getLoc(), fieldName, typeAttr,
+      emitc::FieldOp::create(rewriter, funcOp.getLoc(), fieldName, typeAttr,
                                       argAttr);
     }
 
@@ -75,7 +75,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
     FunctionType funcType = funcOp.getFunctionType();
     Location loc = funcOp.getLoc();
     FuncOp newFuncOp =
-        rewriter.create<emitc::FuncOp>(loc, ("execute"), funcType);
+        emitc::FuncOp::create(rewriter, loc, ("execute"), funcType);
 
     rewriter.createBlock(&newFuncOp.getBody());
     newFuncOp.getBody().takeBody(funcOp.getBody());
@@ -85,7 +85,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
     newArguments.reserve(fields.size());
     for (auto &[fieldName, attr] : fields) {
       GetFieldOp arg =
-          rewriter.create<emitc::GetFieldOp>(loc, attr.getValue(), fieldName);
+          emitc::GetFieldOp::create(rewriter, loc, attr.getValue(), fieldName);
       newArguments.push_back(arg);
     }
 
diff --git a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
index 3328d58551bff..7485ba40b4344 100644
--- a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
+++ b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
@@ -61,7 +61,7 @@ struct FuncInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    builder.create<cf::BranchOp>(op->getLoc(), newDest, returnOp.getOperands());
+    cf::BranchOp::create(builder, op->getLoc(), newDest, returnOp.getOperands());
     op->erase();
   }
 
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index d8309d81f4a3f..9a33fc8e07dde 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -50,7 +50,7 @@ void FuncDialect::initialize() {
 Operation *FuncDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                             Type type, Location loc) {
   if (ConstantOp::isBuildableWith(value, type))
-    return builder.create<ConstantOp>(loc, type,
+    return ConstantOp::create(builder, loc, type,
                                       llvm::cast<FlatSymbolRefAttr>(value));
   return nullptr;
 }
diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
index 11fc696a258c0..10515251fef7d 100644
--- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
+++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
@@ -170,7 +170,7 @@ transform::CastAndCallOp::apply(transform::TransformRewriter &rewriter,
     }
   }
 
-  auto callOp = rewriter.create<func::CallOp>(insertionPoint->getLoc(),
+  auto callOp = func::CallOp::create(rewriter, insertionPoint->getLoc(),
                                               targetFunction, inputs);
 
   // Cast the call results back to the expected types. If any conversions fail
diff --git a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
index a3638c8766a5c..4b70cf172b407 100644
--- a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
@@ -46,7 +46,7 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 
     // Substitute with the new result types from the corresponding FuncType
     // conversion.
-    auto newCallOp = rewriter.create<CallOp>(
+    auto newCallOp = CallOp::create(rewriter,
         callOp.getLoc(), callOp.getCallee(), convertedResults,
         flattenValues(adaptor.getOperands()));
     SmallVector<ValueRange> replacements;
diff --git a/mlir/lib/Dialect/Func/Utils/Utils.cpp b/mlir/lib/Dialect/Func/Utils/Utils.cpp
index 0e9662689ef78..1eba777921828 100644
--- a/mlir/lib/Dialect/Func/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Func/Utils/Utils.cpp
@@ -44,7 +44,7 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
   for (unsigned int idx : newResultsOrder)
     newOutputTypes.push_back(origOutputTypes[idx]);
   rewriter.setInsertionPoint(funcOp);
-  auto newFuncOp = rewriter.create<func::FuncOp>(
+  auto newFuncOp = func::FuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(newInputTypes, newOutputTypes));
 
@@ -80,7 +80,7 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
     newReturnValues.push_back(returnOp.getOperand(idx));
   rewriter.setInsertionPoint(returnOp);
   auto newReturnOp =
-      rewriter.create<func::ReturnOp>(newFuncOp.getLoc(), newReturnValues);
+      func::ReturnOp::create(rewriter, newFuncOp.getLoc(), newReturnValues);
   newReturnOp->setDiscardableAttrs(returnOp->getDiscardableAttrDictionary());
   rewriter.eraseOp(returnOp);
 
@@ -109,7 +109,7 @@ func::replaceCallOpWithNewOrder(RewriterBase &rewriter, func::CallOp callOp,
   // Replace the kernel call operation with a new one that has the
   // reordered arguments.
   rewriter.setInsertionPoint(callOp);
-  auto newCallOp = rewriter.create<func::CallOp>(
+  auto newCallOp = func::CallOp::create(rewriter,
       callOp.getLoc(), callOp.getCallee(), newResultTypes, newArgsOrderValues);
   newCallOp.setNoInlineAttr(callOp.getNoInlineAttr());
   for (auto &&[newIndex, origIndex] : llvm::enumerate(newResultsOrder))
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 30b5ac9809139..f7eb41be287e4 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -1137,7 +1137,7 @@ struct FoldLaunchArguments : public OpRewritePattern<LaunchOp> {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(&op.getBody().front());
         zero =
-            rewriter.create<arith::ConstantIndexOp>(op.getLoc(), /*value=*/0);
+            arith::ConstantIndexOp::create(rewriter, op.getLoc(), /*value=*/0);
       }
       rewriter.replaceAllUsesWith(id, zero);
       simplified = true;
@@ -1381,9 +1381,9 @@ static void printLaunchFuncOperands(OpAsmPrinter &printer, Operation *,
 void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
                       int32_t offset, int32_t width, ShuffleMode mode) {
   build(builder, result, value,
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(offset)),
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(width)),
         mode);
 }
@@ -1395,9 +1395,9 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
 void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
                      int32_t offset, int32_t width) {
   build(builder, result, value,
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(offset)),
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(width)));
 }
 
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index c9e91535df946..2035f371694a7 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -560,7 +560,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   Value predicate;
   if (originalBasisWasProvided) {
     for (Value tmpPredicate : builderResult.predicateOps) {
-      predicate = predicate ? rewriter.create<arith::AndIOp>(loc, predicate,
+      predicate = predicate ? arith::AndIOp::create(rewriter, loc, predicate,
                                                              tmpPredicate)
                             : tmpPredicate;
     }
@@ -573,7 +573,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   Block::iterator insertionPoint;
   if (predicate) {
     // Step 6.a. If predicated, move at the beginning.
-    auto ifOp = rewriter.create<scf::IfOp>(loc, predicate,
+    auto ifOp = scf::IfOp::create(rewriter, loc, predicate,
                                            /*withElseRegion=*/false);
     targetBlock = ifOp.thenBlock();
     insertionPoint = ifOp.thenBlock()->begin();
@@ -632,7 +632,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapForallToBlocksImpl(
     // the insertion point.
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPointToStart(parentBlock);
-    zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   }
 
   ForallRewriteResult rewriteResult;
@@ -884,7 +884,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapOneForallToThreadsImpl(
     return diag;
   // Add a syncthreads if needed. TODO: warpsync
   if (syncAfterDistribute)
-    rewriter.create<BarrierOp>(loc);
+    BarrierOp::create(rewriter, loc);
 
   return DiagnosedSilenceableFailure::success();
 }
@@ -901,7 +901,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapNestedForallToThreadsImpl(
 
   // Create an early zero index value for replacements.
   Location loc = target->getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   DiagnosedSilenceableFailure diag = DiagnosedSilenceableFailure::success();
   WalkResult walkResult = target->walk([&](scf::ForallOp forallOp) {
     diag = mlir::transform::gpu::mapOneForallToThreadsImpl(
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index f6bdbe384c08f..6f0510138da20 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -76,8 +76,8 @@ buildPredicates(RewriterBase &rewriter, Location loc, ArrayRef<Value> activeIds,
     }
     if (activeMappingSize == availableMappingSize)
       continue;
-    Value idx = rewriter.create<arith::ConstantIndexOp>(loc, activeMappingSize);
-    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value idx = arith::ConstantIndexOp::create(rewriter, loc, activeMappingSize);
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                                 activeId, idx);
     predicateOps.push_back(pred);
   }
@@ -98,11 +98,11 @@ static Value buildLinearId(RewriterBase &rewriter, Location loc,
   bindDims(rewriter.getContext(), tx, ty, tz);
   bindSymbols(rewriter.getContext(), bdx, bdy);
   SmallVector<OpFoldResult> vals{
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::x)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::x)
           .getResult(),
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::y)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::y)
           .getResult(),
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::z)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::z)
           .getResult(),
       originalBasisOfr[0], originalBasisOfr[1]};
   OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
@@ -209,9 +209,9 @@ static GpuIdBuilderFnType common3DIdBuilderFn(int64_t multiplicity = 1) {
                             ArrayRef<int64_t> originalBasis) {
     IndexType indexType = rewriter.getIndexType();
     SmallVector<Value> ids{
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::x),
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::y),
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::z)};
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::x),
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::y),
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::z)};
     // In the 3-D mapping case, scale the first dimension by the multiplicity.
     SmallVector<Value> scaledIds = ids;
     AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
@@ -411,7 +411,7 @@ DiagnosedSilenceableFailure createGpuLaunch(
     return diag;
 
   auto createConst = [&](int dim) {
-    return rewriter.create<arith::ConstantIndexOp>(loc, dim);
+    return arith::ConstantIndexOp::create(rewriter, loc, dim);
   };
   OpBuilder::InsertionGuard guard(rewriter);
   Value one = createConst(1);
@@ -421,10 +421,10 @@ DiagnosedSilenceableFailure createGpuLaunch(
   Value blkSizeX = blockDimX.has_value() ? createConst(blockDimX.value()) : one;
   Value blkSizeY = blockDimY.has_value() ? createConst(blockDimY.value()) : one;
   Value blkSizeZ = blockDimZ.has_value() ? createConst(blockDimZ.value()) : one;
-  launchOp = rewriter.create<LaunchOp>(loc, gridSizeX, gridSizeY, gridSizeZ,
+  launchOp = LaunchOp::create(rewriter, loc, gridSizeX, gridSizeY, gridSizeZ,
                                        blkSizeX, blkSizeY, blkSizeZ);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
-  rewriter.create<TerminatorOp>(loc);
+  TerminatorOp::create(rewriter, loc);
   return DiagnosedSilenceableFailure::success();
 }
 
@@ -445,7 +445,7 @@ DiagnosedSilenceableFailure alterGpuLaunch(
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPointAfterValue(currentBlockdim.x);
   auto createConstValue = [&](int dim) {
-    return rewriter.create<arith::ConstantIndexOp>(currentBlockdim.x.getLoc(),
+    return arith::ConstantIndexOp::create(rewriter, currentBlockdim.x.getLoc(),
                                                    dim);
   };
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
index 98dc8ad3aa416..8c449144af3a9 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
@@ -145,7 +145,7 @@ struct GpuAllReduceRewriter {
   // Shortcut to create an op from rewriter using loc as the first argument.
   template <typename T, typename... Args>
   T create(Args... args) {
-    return rewriter.create<T>(loc, std::forward<Args>(args)...);
+    return T::create(rewriter, loc, std::forward<Args>(args)...);
   }
 
   // Creates dimension op of type T, with the result casted to int32.
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index c39ba4a41898d..ccb5245442206 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -129,7 +129,7 @@ struct GpuAsyncRegionPass::ThreadTokenCallback {
   }
 
   Value createWaitOp(Location loc, Type resultType, ValueRange operands) {
-    return builder.create<gpu::WaitOp>(loc, resultType, operands)
+    return gpu::WaitOp::create(builder, loc, resultType, operands)
         .getAsyncToken();
   }
 
@@ -165,7 +165,7 @@ async::ExecuteOp addExecuteResults(async::ExecuteOp executeOp,
 
   // Clone executeOp with the extra results.
   OpBuilder builder(executeOp);
-  auto newOp = builder.create<async::ExecuteOp>(
+  auto newOp = async::ExecuteOp::create(builder,
       executeOp.getLoc(), TypeRange{resultTypes}.drop_front() /*drop token*/,
       executeOp.getDependencies(), executeOp.getBodyOperands());
   IRMapping mapper;
@@ -247,7 +247,7 @@ struct GpuAsyncRegionPass::DeferWaitCallback {
           builder.setInsertionPointAfter(op);
           for (auto asyncToken : asyncTokens)
             tokens.push_back(
-                builder.create<async::AwaitOp>(loc, asyncToken).getResult());
+                async::AwaitOp::create(builder, loc, asyncToken).getResult());
           // Set `it` after the inserted async.await ops.
           it = builder.getInsertionPoint();
         })
@@ -279,7 +279,7 @@ struct GpuAsyncRegionPass::DeferWaitCallback {
 
     // Otherwise, insert a gpu.wait before 'it'.
     builder.setInsertionPoint(it->getBlock(), it);
-    auto waitOp = builder.create<gpu::WaitOp>(loc, Type{}, tokens);
+    auto waitOp = gpu::WaitOp::create(builder, loc, Type{}, tokens);
 
     // If the new waitOp is at the end of an async.execute region, add it to the
     // worklist. 'operator()(executeOp)' would do the same, but this is faster.
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index 65b9407a7efba..5c84a340f0e08 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -62,7 +62,7 @@ getFlatOffsetAndStrides(OpBuilder &rewriter, Location loc, Value source,
     OpBuilder::InsertionGuard g(rewriter);
     setInsertionPointToStart(rewriter, source);
     newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, source);
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, source);
   }
 
   auto &&[sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
@@ -108,7 +108,7 @@ static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
   auto &&[base, offset, ignore] =
       getFlatOffsetAndStrides(rewriter, loc, source, offsetsTemp);
   MemRefType retType = inferCastResultType(base, offset);
-  return rewriter.create<memref::ReinterpretCastOp>(loc, retType, base, offset,
+  return memref::ReinterpretCastOp::create(rewriter, loc, retType, base, offset,
                                                     ArrayRef<OpFoldResult>(),
                                                     ArrayRef<OpFoldResult>());
 }
diff --git a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
index 153ceb23a6ecd..6519b65cec465 100644
--- a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
@@ -26,11 +26,11 @@ struct GpuGlobalIdRewriter : public OpRewritePattern<gpu::GlobalIdOp> {
                                 PatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     auto dim = op.getDimension();
-    auto blockId = rewriter.create<gpu::BlockIdOp>(loc, dim);
-    auto blockDim = rewriter.create<gpu::BlockDimOp>(loc, dim);
+    auto blockId = gpu::BlockIdOp::create(rewriter, loc, dim);
+    auto blockDim = gpu::BlockDimOp::create(rewriter, loc, dim);
     // Compute blockId.x * blockDim.x
-    auto tmp = rewriter.create<index::MulOp>(op.getLoc(), blockId, blockDim);
-    auto threadId = rewriter.create<gpu::ThreadIdOp>(loc, dim);
+    auto tmp = index::MulOp::create(rewriter, op.getLoc(), blockId, blockDim);
+    auto threadId = gpu::ThreadIdOp::create(rewriter, loc, dim);
     // Compute threadId.x + blockId.x * blockDim.x
     rewriter.replaceOpWithNewOp<index::AddOp>(op, threadId, tmp);
     return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index 34ea9fcab4188..b98c060ac1da5 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -40,7 +40,7 @@ template <typename OpTy>
 static void createForAllDimensions(OpBuilder &builder, Location loc,
                                    SmallVectorImpl<Value> &values) {
   for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z})
-    values.push_back(builder.create<OpTy>(loc, builder.getIndexType(), dim));
+    values.push_back(OpTy::create(builder, loc, builder.getIndexType(), dim));
 }
 
 /// Adds operations generating block/thread ids and grid/block dimensions at the
@@ -195,7 +195,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   }
   FunctionType type =
       FunctionType::get(launchOp.getContext(), kernelOperandTypes, {});
-  auto outlinedFunc = builder.create<gpu::GPUFuncOp>(
+  auto outlinedFunc = gpu::GPUFuncOp::create(builder,
       loc, kernelFnName, type,
       TypeRange(ValueRange(launchOp.getWorkgroupAttributions())),
       TypeRange(ValueRange(launchOp.getPrivateAttributions())));
@@ -247,7 +247,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
     if (!terminator)
       continue;
     OpBuilder replacer(terminator);
-    replacer.create<gpu::ReturnOp>(terminator->getLoc());
+    gpu::ReturnOp::create(replacer, terminator->getLoc());
     terminator->erase();
   }
 
@@ -287,7 +287,7 @@ static void convertToLaunchFuncOp(gpu::LaunchOp launchOp,
   Value asyncToken = launchOp.getAsyncToken();
   std::optional<gpu::KernelDim3> clusterSize =
       launchOp.getClusterSizeOperandValues();
-  auto launchFunc = builder.create<gpu::LaunchFuncOp>(
+  auto launchFunc = gpu::LaunchFuncOp::create(builder,
       launchOp.getLoc(), kernelFunc, launchOp.getGridSizeOperandValues(),
       launchOp.getBlockSizeOperandValues(),
       launchOp.getDynamicSharedMemorySize(), operands,
@@ -415,7 +415,7 @@ class GpuKernelOutliningPass
     // Check if the module already exists in the symbol table
     if (!kernelModule) {
       // If not found, create a new GPU module
-      kernelModule = builder.create<gpu::GPUModuleOp>(kernelFunc.getLoc(),
+      kernelModule = gpu::GPUModuleOp::create(builder, kernelFunc.getLoc(),
                                                       kernelModuleName);
     }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 14c44f27a6249..2bee1a1619665 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -34,8 +34,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
   auto rank = memRefType.getRank();
 
   SmallVector<Value, 4> lbs, ubs, steps;
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value one = b.create<arith::ConstantIndexOp>(1);
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value one = arith::ConstantIndexOp::create(b, 1);
 
   // Make sure we have enough loops to use all thread dimensions, these trivial
   // loops should be outermost and therefore inserted first.
@@ -59,8 +59,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
   auto indexType = b.getIndexType();
   SmallVector<Value, 3> threadIds, blockDims;
   for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z}) {
-    threadIds.push_back(b.create<gpu::ThreadIdOp>(indexType, dim));
-    blockDims.push_back(b.create<gpu::BlockDimOp>(indexType, dim));
+    threadIds.push_back(gpu::ThreadIdOp::create(b, indexType, dim));
+    blockDims.push_back(gpu::BlockDimOp::create(b, indexType, dim));
   }
 
   // Produce the loop nest with copies.
@@ -70,8 +70,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
       [&](OpBuilder &b, Location loc, ValueRange loopIvs) {
         ivs.assign(loopIvs.begin(), loopIvs.end());
         auto activeIvs = llvm::ArrayRef(ivs).take_back(rank);
-        Value loaded = b.create<memref::LoadOp>(loc, from, activeIvs);
-        b.create<memref::StoreOp>(loc, loaded, to, activeIvs);
+        Value loaded = memref::LoadOp::create(b, loc, from, activeIvs);
+        memref::StoreOp::create(b, loc, loaded, to, activeIvs);
       });
 
   // Map the innermost loops to threads in reverse order.
@@ -131,10 +131,10 @@ static void insertCopies(Region &region, Location loc, Value from, Value to) {
 
   auto b = ImplicitLocOpBuilder::atBlockBegin(loc, &region.front());
   insertCopyLoops(b, from, to);
-  b.create<gpu::BarrierOp>();
+  gpu::BarrierOp::create(b, );
 
   b.setInsertionPoint(&region.front().back());
-  b.create<gpu::BarrierOp>();
+  gpu::BarrierOp::create(b, );
   insertCopyLoops(b, to, from);
 }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
index 9a69e6dde4274..dd164e5aaa7c4 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
@@ -108,7 +108,7 @@ LogicalResult moduleSerializer(GPUModuleOp op,
       !handler && moduleHandler)
     handler = moduleHandler;
   builder.setInsertionPointAfter(op);
-  builder.create<gpu::BinaryOp>(op.getLoc(), op.getName(), handler,
+  gpu::BinaryOp::create(builder, op.getLoc(), op.getName(), handler,
                                 builder.getArrayAttr(objects));
   op->erase();
   return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
index 171e64346f155..5a55a39a7748e 100644
--- a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
@@ -48,10 +48,10 @@ struct PromoteShuffleToSwizzlePattern
                                          "offset must be in the range [0, 31]");
 
     Location loc = op.getLoc();
-    Value res = rewriter.create<amdgpu::SwizzleBitModeOp>(
+    Value res = amdgpu::SwizzleBitModeOp::create(rewriter,
         loc, op.getResult(0).getType(), op.getValue(), /*andMask=*/31,
         /*orMask=*/0, /*xorMask=*/offsetValue);
-    Value valid = rewriter.create<arith::ConstantIntOp>(loc, 1, /*width*/ 1);
+    Value valid = arith::ConstantIntOp::create(rewriter, loc, 1, /*width*/ 1);
     rewriter.replaceOp(op, {res, valid});
     return success();
   }
diff --git a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
index 2d6df0ff6d02d..7aa5288d8260d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
@@ -47,16 +47,16 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
 
     // Float types must be converted to i64 to extract the bits.
     if (isa<FloatType>(valueType))
-      value = rewriter.create<arith::BitcastOp>(valueLoc, i64, value);
+      value = arith::BitcastOp::create(rewriter, valueLoc, i64, value);
 
     // Get the low bits by trunc(value).
-    lo = rewriter.create<arith::TruncIOp>(valueLoc, i32, value);
+    lo = arith::TruncIOp::create(rewriter, valueLoc, i32, value);
 
     // Get the high bits by trunc(value >> 32).
-    auto c32 = rewriter.create<arith::ConstantOp>(
+    auto c32 = arith::ConstantOp::create(rewriter,
         valueLoc, rewriter.getIntegerAttr(i64, 32));
-    hi = rewriter.create<arith::ShRUIOp>(valueLoc, value, c32);
-    hi = rewriter.create<arith::TruncIOp>(valueLoc, i32, hi);
+    hi = arith::ShRUIOp::create(rewriter, valueLoc, value, c32);
+    hi = arith::TruncIOp::create(rewriter, valueLoc, i32, hi);
 
     // Shuffle the values.
     ValueRange loRes =
@@ -71,21 +71,21 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
             .getResults();
 
     // Convert lo back to i64.
-    lo = rewriter.create<arith::ExtUIOp>(valueLoc, i64, loRes[0]);
+    lo = arith::ExtUIOp::create(rewriter, valueLoc, i64, loRes[0]);
 
     // Convert hi back to i64.
-    hi = rewriter.create<arith::ExtUIOp>(valueLoc, i64, hiRes[0]);
-    hi = rewriter.create<arith::ShLIOp>(valueLoc, hi, c32);
+    hi = arith::ExtUIOp::create(rewriter, valueLoc, i64, hiRes[0]);
+    hi = arith::ShLIOp::create(rewriter, valueLoc, hi, c32);
 
     // Obtain the shuffled bits hi | lo.
-    value = rewriter.create<arith::OrIOp>(loc, hi, lo);
+    value = arith::OrIOp::create(rewriter, loc, hi, lo);
 
     // Convert the value back to float.
     if (isa<FloatType>(valueType))
-      value = rewriter.create<arith::BitcastOp>(valueLoc, valueType, value);
+      value = arith::BitcastOp::create(rewriter, valueLoc, valueType, value);
 
     // Obtain the shuffle validity by combining both validities.
-    auto validity = rewriter.create<arith::AndIOp>(loc, loRes[1], hiRes[1]);
+    auto validity = arith::AndIOp::create(rewriter, loc, loRes[1], hiRes[1]);
 
     // Replace the op.
     rewriter.replaceOp(op, {value, validity});
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
index 05631ad87dd71..f401e6a267eef 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
@@ -54,22 +54,22 @@ struct GpuSubgroupIdRewriter final : OpRewritePattern<gpu::SubgroupIdOp> {
     Location loc = op->getLoc();
     Type indexType = rewriter.getIndexType();
 
-    Value dimX = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::x);
-    Value dimY = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::y);
-    Value tidX = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
-    Value tidY = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::y);
-    Value tidZ = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::z);
+    Value dimX = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::x);
+    Value dimY = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::y);
+    Value tidX = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
+    Value tidY = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::y);
+    Value tidZ = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::z);
 
-    Value dimYxIdZ = rewriter.create<arith::MulIOp>(loc, indexType, dimY, tidZ);
+    Value dimYxIdZ = arith::MulIOp::create(rewriter, loc, indexType, dimY, tidZ);
     Value dimYxIdZPlusIdY =
-        rewriter.create<arith::AddIOp>(loc, indexType, dimYxIdZ, tidY);
+        arith::AddIOp::create(rewriter, loc, indexType, dimYxIdZ, tidY);
     Value dimYxIdZPlusIdYTimesDimX =
-        rewriter.create<arith::MulIOp>(loc, indexType, dimX, dimYxIdZPlusIdY);
-    Value IdXPlusDimYxIdZPlusIdYTimesDimX = rewriter.create<arith::AddIOp>(
+        arith::MulIOp::create(rewriter, loc, indexType, dimX, dimYxIdZPlusIdY);
+    Value IdXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(rewriter,
         loc, indexType, tidX, dimYxIdZPlusIdYTimesDimX);
-    Value subgroupSize = rewriter.create<gpu::SubgroupSizeOp>(
+    Value subgroupSize = gpu::SubgroupSizeOp::create(rewriter,
         loc, rewriter.getIndexType(), /*upper_bound = */ nullptr);
-    Value subgroupIdOp = rewriter.create<arith::DivUIOp>(
+    Value subgroupIdOp = arith::DivUIOp::create(rewriter,
         loc, indexType, IdXPlusDimYxIdZPlusIdYTimesDimX, subgroupSize);
     rewriter.replaceOp(op, {subgroupIdOp});
     return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index 1b3d13623c548..7290f92f6fe08 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -79,7 +79,7 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
 
     Location loc = op.getLoc();
     Value res =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(vecTy));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(vecTy));
 
     for (unsigned i = 0; i != numNewReductions; ++i) {
       int64_t startIdx = i * elementsPerShuffle;
@@ -90,22 +90,22 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
       Value extracted;
       if (numElems == 1) {
         extracted =
-            rewriter.create<vector::ExtractOp>(loc, op.getValue(), startIdx);
+            vector::ExtractOp::create(rewriter, loc, op.getValue(), startIdx);
       } else {
-        extracted = rewriter.create<vector::ExtractStridedSliceOp>(
+        extracted = vector::ExtractStridedSliceOp::create(rewriter,
             loc, op.getValue(), /*offsets=*/startIdx, /*sizes=*/numElems,
             /*strides=*/1);
       }
 
-      Value reduce = rewriter.create<gpu::SubgroupReduceOp>(
+      Value reduce = gpu::SubgroupReduceOp::create(rewriter,
           loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
           op.getClusterStride());
       if (numElems == 1) {
-        res = rewriter.create<vector::InsertOp>(loc, reduce, res, startIdx);
+        res = vector::InsertOp::create(rewriter, loc, reduce, res, startIdx);
         continue;
       }
 
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, reduce, res, /*offsets=*/startIdx, /*strides=*/1);
     }
 
@@ -138,8 +138,8 @@ struct ScalarizeSingleElementReduce final
     assert(vecTy.getRank() == 1 && "Unexpected vector type");
     assert(!vecTy.isScalable() && "Unexpected vector type");
     Location loc = op.getLoc();
-    Value extracted = rewriter.create<vector::ExtractOp>(loc, op.getValue(), 0);
-    Value reduce = rewriter.create<gpu::SubgroupReduceOp>(
+    Value extracted = vector::ExtractOp::create(rewriter, loc, op.getValue(), 0);
+    Value reduce = gpu::SubgroupReduceOp::create(rewriter,
         loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
         op.getClusterStride());
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, vecTy, reduce);
@@ -254,14 +254,14 @@ struct ScalarSubgroupReduceToShuffles final
     auto packFn = [loc, &rewriter, equivIntType,
                    shuffleIntType](Value unpackedVal) -> Value {
       auto asInt =
-          rewriter.create<arith::BitcastOp>(loc, equivIntType, unpackedVal);
-      return rewriter.create<arith::ExtUIOp>(loc, shuffleIntType, asInt);
+          arith::BitcastOp::create(rewriter, loc, equivIntType, unpackedVal);
+      return arith::ExtUIOp::create(rewriter, loc, shuffleIntType, asInt);
     };
     auto unpackFn = [loc, &rewriter, equivIntType,
                      valueTy](Value packedVal) -> Value {
       auto asInt =
-          rewriter.create<arith::TruncIOp>(loc, equivIntType, packedVal);
-      return rewriter.create<arith::BitcastOp>(loc, valueTy, asInt);
+          arith::TruncIOp::create(rewriter, loc, equivIntType, packedVal);
+      return arith::BitcastOp::create(rewriter, loc, valueTy, asInt);
     };
 
     rewriter.replaceOp(
@@ -326,9 +326,9 @@ struct VectorSubgroupReduceToShuffles final
         static_cast<int64_t>(elementsPerShuffle), vecTy.getElementType());
     Value extendedInput = op.getValue();
     if (vecBitwidth < shuffleBitwidth) {
-      auto zero = rewriter.create<arith::ConstantOp>(
+      auto zero = arith::ConstantOp::create(rewriter,
           loc, rewriter.getZeroAttr(extendedVecTy));
-      extendedInput = rewriter.create<vector::InsertStridedSliceOp>(
+      extendedInput = vector::InsertStridedSliceOp::create(rewriter,
           loc, extendedInput, zero, /*offsets=*/0, /*strides=*/1);
     }
 
@@ -337,21 +337,21 @@ struct VectorSubgroupReduceToShuffles final
 
     auto packFn = [loc, &rewriter, shuffleVecType](Value unpackedVal) -> Value {
       auto asIntVec =
-          rewriter.create<vector::BitCastOp>(loc, shuffleVecType, unpackedVal);
-      return rewriter.create<vector::ExtractOp>(loc, asIntVec, 0);
+          vector::BitCastOp::create(rewriter, loc, shuffleVecType, unpackedVal);
+      return vector::ExtractOp::create(rewriter, loc, asIntVec, 0);
     };
     auto unpackFn = [loc, &rewriter, shuffleVecType,
                      extendedVecTy](Value packedVal) -> Value {
       auto asIntVec =
-          rewriter.create<vector::BroadcastOp>(loc, shuffleVecType, packedVal);
-      return rewriter.create<vector::BitCastOp>(loc, extendedVecTy, asIntVec);
+          vector::BroadcastOp::create(rewriter, loc, shuffleVecType, packedVal);
+      return vector::BitCastOp::create(rewriter, loc, extendedVecTy, asIntVec);
     };
 
     Value res = createSubgroupShuffleReduction(
         rewriter, loc, extendedInput, op.getOp(), *ci, packFn, unpackFn);
 
     if (vecBitwidth < shuffleBitwidth) {
-      res = rewriter.create<vector::ExtractStridedSliceOp>(
+      res = vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/0, /*sizes=*/vecTy.getNumElements(),
           /*strides=*/1);
     }
@@ -378,7 +378,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   const bool boundCtrl = true;
   if (ci.clusterSize >= 2) {
     // Perform reduction between all lanes N <-> N+1.
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({1, 0, 3, 2}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -387,7 +387,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
 
   if (ci.clusterSize >= 4) {
     // Perform reduction between all lanes N <-> N+2.
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({2, 3, 0, 1}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -396,7 +396,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 8) {
     // Perform reduction between all lanes N <-> 7-N,
     // e.g lane[0] <-> lane[7], lane[1] <-> lane[6]..., lane[3] <-> lane[4].
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::row_half_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -405,7 +405,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 16) {
     // Perform reduction between all lanes N <-> 15-N,
     // e.g lane[0] <-> lane[15], lane[1] <-> lane[14]..., lane[7] <-> lane[8].
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::row_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -415,7 +415,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
     if (chipset.majorVersion <= 9) {
       // Broadcast last value from each row to next row.
       // Use row mask to avoid polluting rows 1 and 3.
-      dpp = rewriter.create<amdgpu::DPPOp>(
+      dpp = amdgpu::DPPOp::create(rewriter,
           loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_15,
           rewriter.getUnitAttr(), 0xa, allBanks,
           /*bound_ctrl*/ false);
@@ -423,9 +423,9 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
           rewriter, loc, gpu::convertReductionKind(mode), res, dpp);
     } else if (chipset.majorVersion <= 12) {
       // Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2).
-      Value uint32Max = rewriter.create<arith::ConstantOp>(
+      Value uint32Max = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
-      dpp = rewriter.create<ROCDL::PermlaneX16Op>(loc, res.getType(), res, res,
+      dpp = ROCDL::PermlaneX16Op::create(rewriter, loc, res.getType(), res, res,
                                                   uint32Max, uint32Max,
                                                   /*fi=*/true,
                                                   /*bound_ctrl=*/false);
@@ -437,37 +437,37 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
               "this device.");
     }
     if (ci.subgroupSize == 32) {
-      Value lane31 = rewriter.create<arith::ConstantOp>(
+      Value lane31 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      res = rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane31);
+      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
     }
   }
   if (ci.clusterSize >= 64) {
     if (chipset.majorVersion <= 9) {
       // Broadcast 31st lane value to rows 2 and 3.
-      dpp = rewriter.create<amdgpu::DPPOp>(
+      dpp = amdgpu::DPPOp::create(rewriter,
           loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_31,
           rewriter.getUnitAttr(), 0xf, allBanks,
           /*bound_ctrl*/ true);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), dpp, res);
       // Obtain reduction from last rows, the previous rows are polluted.
-      Value lane63 = rewriter.create<arith::ConstantOp>(
+      Value lane63 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
-      res = rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane63);
+      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
 
     } else if (chipset.majorVersion <= 12) {
       // Assume reduction across 32 lanes has been done.
       // Perform final reduction manually by summing values in lane 0 and
       // lane 32.
-      Value lane31 = rewriter.create<arith::ConstantOp>(
+      Value lane31 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      Value lane63 = rewriter.create<arith::ConstantOp>(
+      Value lane63 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
       lane31 =
-          rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane31);
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
       lane63 =
-          rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane63);
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), lane31, lane63);
     } else {
diff --git a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
index 29f6f32892f72..473a2e515c12b 100644
--- a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
+++ b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
@@ -27,7 +27,7 @@ WarpDistributionPattern::moveRegionToNewWarpOpAndReplaceReturns(
   // Create a new op before the existing one, with the extra operands.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(warpOp);
-  auto newWarpOp = rewriter.create<WarpExecuteOnLane0Op>(
+  auto newWarpOp = WarpExecuteOnLane0Op::create(rewriter,
       warpOp.getLoc(), newReturnTypes, warpOp.getLaneid(), warpOp.getWarpSize(),
       warpOp.getArgs(), warpOp.getBody()->getArgumentTypes());
 
@@ -124,7 +124,7 @@ bool WarpDistributionPattern::delinearizeLaneId(
 
   int64_t usedThreads = 1;
 
-  Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
   delinearizedIds.assign(sizes.size(), zero);
 
   for (int i = sizes.size() - 1; i >= 0; --i) {
diff --git a/mlir/lib/Dialect/Index/IR/IndexOps.cpp b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
index bab9e2852a460..7cab93709b9c6 100644
--- a/mlir/lib/Dialect/Index/IR/IndexOps.cpp
+++ b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
@@ -36,7 +36,7 @@ Operation *IndexDialect::materializeConstant(OpBuilder &b, Attribute value,
   if (auto boolValue = dyn_cast<BoolAttr>(value)) {
     if (!type.isSignlessInteger(1))
       return nullptr;
-    return b.create<BoolConstantOp>(loc, type, boolValue);
+    return BoolConstantOp::create(b, loc, type, boolValue);
   }
 
   // Materialize integer attributes as `index`.
@@ -46,7 +46,7 @@ Operation *IndexDialect::materializeConstant(OpBuilder &b, Attribute value,
       return nullptr;
     assert(indexValue.getValue().getBitWidth() ==
            IndexType::kInternalStorageBitWidth);
-    return b.create<ConstantOp>(loc, indexValue);
+    return ConstantOp::create(b, loc, indexValue);
   }
 
   return nullptr;
@@ -715,10 +715,10 @@ LogicalResult CmpOp::canonicalize(CmpOp op, PatternRewriter &rewriter) {
 
   index::CmpOp newCmp;
   if (rhsIsZero)
-    newCmp = rewriter.create<index::CmpOp>(op.getLoc(), op.getPred(),
+    newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
                                            subOp.getLhs(), subOp.getRhs());
   else
-    newCmp = rewriter.create<index::CmpOp>(op.getLoc(), op.getPred(),
+    newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
                                            subOp.getRhs(), subOp.getLhs());
   rewriter.replaceOp(op, newCmp);
   return success();
diff --git a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
index c17ef1029faf6..b1d376c7c03db 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
@@ -90,7 +90,7 @@ void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
     }
     for (auto [idx, t] : llvm::enumerate(stype.getBody())) {
       if (itype != PTXRegisterMod::Write) {
-        Value extractValue = rewriter.create<LLVM::ExtractValueOp>(
+        Value extractValue = LLVM::ExtractValueOp::create(rewriter,
             interfaceOp->getLoc(), v, idx);
         addValue(extractValue);
       }
@@ -132,7 +132,7 @@ LLVM::InlineAsmOp PtxBuilder::build() {
   // Replace all % with $
   llvm::replace(ptxInstruction, '%', '$');
 
-  return rewriter.create<LLVM::InlineAsmOp>(
+  return LLVM::InlineAsmOp::create(rewriter,
       interfaceOp->getLoc(),
       /*result types=*/resultTypes,
       /*operands=*/ptxOperands,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 89f765dacda35..7d3f59d128367 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -89,7 +89,7 @@ mlir::LLVM::lookupOrCreateFn(OpBuilder &b, Operation *moduleOp, StringRef name,
   OpBuilder::InsertionGuard g(b);
   assert(!moduleOp->getRegion(0).empty() && "expected non-empty region");
   b.setInsertionPointToStart(&moduleOp->getRegion(0).front());
-  auto funcOp = b.create<LLVM::LLVMFuncOp>(
+  auto funcOp = LLVM::LLVMFuncOp::create(b,
       moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 62dce32bc4531..884ae4a2eac64 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3309,7 +3309,7 @@ bool LLVM::ConstantOp::isBuildableWith(Attribute value, Type type) {
 ConstantOp LLVM::ConstantOp::materialize(OpBuilder &builder, Attribute value,
                                          Type type, Location loc) {
   if (isBuildableWith(value, type))
-    return builder.create<LLVM::ConstantOp>(loc, cast<TypedAttr>(value));
+    return LLVM::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
   return nullptr;
 }
 
@@ -4275,13 +4275,13 @@ Operation *LLVMDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // a builtin zero attribute and thus will materialize as a llvm.mlir.constant.
   if (auto symbol = dyn_cast<FlatSymbolRefAttr>(value))
     if (isa<LLVM::LLVMPointerType>(type))
-      return builder.create<LLVM::AddressOfOp>(loc, type, symbol);
+      return LLVM::AddressOfOp::create(builder, loc, type, symbol);
   if (isa<LLVM::UndefAttr>(value))
-    return builder.create<LLVM::UndefOp>(loc, type);
+    return LLVM::UndefOp::create(builder, loc, type);
   if (isa<LLVM::PoisonAttr>(value))
-    return builder.create<LLVM::PoisonOp>(loc, type);
+    return LLVM::PoisonOp::create(builder, loc, type);
   if (isa<LLVM::ZeroAttr>(value))
-    return builder.create<LLVM::ZeroOp>(loc, type);
+    return LLVM::ZeroOp::create(builder, loc, type);
   // Otherwise try materializing it as a regular llvm.mlir.constant op.
   return LLVM::ConstantOp::materialize(builder, value, type, loc);
 }
@@ -4304,15 +4304,15 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
   MLIRContext *ctx = builder.getContext();
   auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
-  auto global = moduleBuilder.create<LLVM::GlobalOp>(
+  auto global = LLVM::GlobalOp::create(moduleBuilder,
       loc, type, /*isConstant=*/true, linkage, name,
       builder.getStringAttr(value), /*alignment=*/0);
 
   LLVMPointerType ptrType = LLVMPointerType::get(ctx);
   // Get the pointer to the first character in the global string.
   Value globalPtr =
-      builder.create<LLVM::AddressOfOp>(loc, ptrType, global.getSymNameAttr());
-  return builder.create<LLVM::GEPOp>(loc, ptrType, type, globalPtr,
+      LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
+  return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
                                      ArrayRef<GEPArg>{0, 0});
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index bc451f8b028fc..98fb9b09f0cd5 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -37,7 +37,7 @@ llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() {
 
 Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot,
                                       OpBuilder &builder) {
-  return builder.create<LLVM::UndefOp>(getLoc(), slot.elemType);
+  return LLVM::UndefOp::create(builder, getLoc(), slot.elemType);
 }
 
 void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
@@ -45,7 +45,7 @@ void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
                                          OpBuilder &builder) {
   for (Operation *user : getOperation()->getUsers())
     if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
-      builder.create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
+      LLVM::DbgValueOp::create(builder, declareOp.getLoc(), argument,
                                        declareOp.getVarInfo(),
                                        declareOp.getLocationExpr());
 }
@@ -89,7 +89,7 @@ DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
   for (Attribute index : usedIndices) {
     Type elemType = destructurableType.getTypeAtIndex(index);
     assert(elemType && "used index must exist");
-    auto subAlloca = builder.create<LLVM::AllocaOp>(
+    auto subAlloca = LLVM::AllocaOp::create(builder,
         getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
         getArraySize());
     newAllocators.push_back(subAlloca);
@@ -260,13 +260,13 @@ static Value createExtractAndCast(OpBuilder &builder, Location loc,
   // Truncate the integer if the size of the target is less than the value.
   if (isBigEndian(dataLayout)) {
     uint64_t shiftAmount = srcTypeSize - targetTypeSize;
-    auto shiftConstant = builder.create<LLVM::ConstantOp>(
+    auto shiftConstant = LLVM::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(srcType, shiftAmount));
     replacement =
         builder.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
   }
 
-  replacement = builder.create<LLVM::TruncOp>(
+  replacement = LLVM::TruncOp::create(builder,
       loc, builder.getIntegerType(targetTypeSize), replacement);
 
   // Now cast the integer to the actual target type if required.
@@ -304,7 +304,7 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
     // On big endian systems, a store to the base pointer overwrites the most
     // significant bits. To accomodate for this, the stored value needs to be
     // shifted into the according position.
-    Value bigEndianShift = builder.create<LLVM::ConstantOp>(
+    Value bigEndianShift = LLVM::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
     valueAsInt =
         builder.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
@@ -325,7 +325,7 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
   }
 
   // Mask out the affected bits ...
-  Value mask = builder.create<LLVM::ConstantOp>(
+  Value mask = LLVM::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
   Value masked = builder.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
 
@@ -644,7 +644,7 @@ DeletionKind LLVM::DbgValueOp::removeBlockingUses(
   // debug local variable info. This allows the debugger to inform the user that
   // the variable has been optimized out.
   auto undef =
-      builder.create<UndefOp>(getValue().getLoc(), getValue().getType());
+      UndefOp::create(builder, getValue().getLoc(), getValue().getType());
   getValueMutable().assign(undef);
   return DeletionKind::Keep;
 }
@@ -655,7 +655,7 @@ void LLVM::DbgDeclareOp::visitReplacedValues(
     ArrayRef<std::pair<Operation *, Value>> definitions, OpBuilder &builder) {
   for (auto [op, value] : definitions) {
     builder.setInsertionPointAfter(op);
-    builder.create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(),
+    LLVM::DbgValueOp::create(builder, getLoc(), value, getVarInfo(),
                                      getLocationExpr());
   }
 }
@@ -978,7 +978,7 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetOp toReplace,
               IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize))
           .getResult();
 
-  builder.create<LLVM::MemsetOp>(toReplace.getLoc(), subslots.at(index).ptr,
+  LLVM::MemsetOp::create(builder, toReplace.getLoc(), subslots.at(index).ptr,
                                  toReplace.getVal(), newMemsetSizeValue,
                                  toReplace.getIsVolatile());
 }
@@ -991,7 +991,7 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
   auto newMemsetSizeValue =
       IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize);
 
-  builder.create<LLVM::MemsetInlineOp>(
+  LLVM::MemsetInlineOp::create(builder,
       toReplace.getLoc(), subslots.at(index).ptr, toReplace.getVal(),
       newMemsetSizeValue, toReplace.getIsVolatile());
 }
@@ -1063,7 +1063,7 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       APInt memsetVal(/*numBits=*/width, /*val=*/0);
       for (unsigned loBit = 0; loBit < width; loBit += 8)
         memsetVal.insertBits(constantPattern.getValue(), loBit);
-      return builder.create<LLVM::ConstantOp>(
+      return LLVM::ConstantOp::create(builder,
           op.getLoc(), IntegerAttr::get(intType, memsetVal));
     }
 
@@ -1075,14 +1075,14 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
     // value and or-ing it with the previous value.
     uint64_t coveredBits = 8;
     Value currentValue =
-        builder.create<LLVM::ZExtOp>(op.getLoc(), intType, op.getVal());
+        LLVM::ZExtOp::create(builder, op.getLoc(), intType, op.getVal());
     while (coveredBits < width) {
       Value shiftBy =
-          builder.create<LLVM::ConstantOp>(op.getLoc(), intType, coveredBits);
+          LLVM::ConstantOp::create(builder, op.getLoc(), intType, coveredBits);
       Value shifted =
-          builder.create<LLVM::ShlOp>(op.getLoc(), currentValue, shiftBy);
+          LLVM::ShlOp::create(builder, op.getLoc(), currentValue, shiftBy);
       currentValue =
-          builder.create<LLVM::OrOp>(op.getLoc(), currentValue, shifted);
+          LLVM::OrOp::create(builder, op.getLoc(), currentValue, shifted);
       coveredBits *= 2;
     }
 
@@ -1094,7 +1094,7 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       })
       .Case([&](FloatType type) -> Value {
         Value intVal = buildMemsetValue(type.getWidth());
-        return builder.create<LLVM::BitcastOp>(op.getLoc(), type, intVal);
+        return LLVM::BitcastOp::create(builder, op.getLoc(), type, intVal);
       })
       .Default([](Type) -> Value {
         llvm_unreachable(
@@ -1282,7 +1282,7 @@ static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot) {
 template <class MemcpyLike>
 static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot,
                              OpBuilder &builder) {
-  return builder.create<LLVM::LoadOp>(op.getLoc(), slot.elemType, op.getSrc());
+  return LLVM::LoadOp::create(builder, op.getLoc(), slot.elemType, op.getSrc());
 }
 
 template <class MemcpyLike>
@@ -1309,7 +1309,7 @@ memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot,
                          const SmallPtrSetImpl<OpOperand *> &blockingUses,
                          OpBuilder &builder, Value reachingDefinition) {
   if (op.loadsFrom(slot))
-    builder.create<LLVM::StoreOp>(op.getLoc(), reachingDefinition, op.getDst());
+    LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition, op.getDst());
   return DeletionKind::Delete;
 }
 
@@ -1354,10 +1354,10 @@ template <class MemcpyLike>
 void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                MemcpyLike toReplace, Value dst, Value src,
                                Type toCpy, bool isVolatile) {
-  Value memcpySize = builder.create<LLVM::ConstantOp>(
+  Value memcpySize = LLVM::ConstantOp::create(builder,
       toReplace.getLoc(), IntegerAttr::get(toReplace.getLen().getType(),
                                            layout.getTypeSize(toCpy)));
-  builder.create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize,
+  MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize,
                              isVolatile);
 }
 
@@ -1367,7 +1367,7 @@ void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                Value src, Type toCpy, bool isVolatile) {
   Type lenType = IntegerType::get(toReplace->getContext(),
                                   toReplace.getLen().getBitWidth());
-  builder.create<LLVM::MemcpyInlineOp>(
+  LLVM::MemcpyInlineOp::create(builder,
       toReplace.getLoc(), dst, src,
       IntegerAttr::get(lenType, layout.getTypeSize(toCpy)), isVolatile);
 }
@@ -1409,7 +1409,7 @@ memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
     SmallVector<LLVM::GEPArg> gepIndices{
         0, static_cast<int32_t>(
                cast<IntegerAttr>(index).getValue().getZExtValue())};
-    Value subslotPtrInOther = builder.create<LLVM::GEPOp>(
+    Value subslotPtrInOther = LLVM::GEPOp::create(builder,
         op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()), slot.elemType,
         isDst ? op.getSrc() : op.getDst(), gepIndices);
 
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
index 6fbb0d24826d0..1b1268da15646 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
@@ -28,13 +28,13 @@ static void addComdat(LLVM::LLVMFuncOp &op, OpBuilder &builder,
     PatternRewriter::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(module.getBody());
     comdatOp =
-        builder.create<mlir::LLVM::ComdatOp>(module.getLoc(), comdatName);
+        mlir::LLVM::ComdatOp::create(builder, module.getLoc(), comdatName);
     symbolTable.insert(comdatOp);
   }
 
   PatternRewriter::InsertionGuard guard(builder);
   builder.setInsertionPointToStart(&comdatOp.getBody().back());
-  auto selectorOp = builder.create<mlir::LLVM::ComdatSelectorOp>(
+  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(builder,
       comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
   op.setComdatAttr(mlir::SymbolRefAttr::get(
       builder.getContext(), comdatName,
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index 7f3afffc9645e..2ecef4b4c8a51 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -113,18 +113,18 @@ handleInlinedAllocas(Operation *call,
     // scope if some are already present in the body of the caller. This is not
     // invalid IR, but LLVM cleans these up in InstCombineCalls.cpp, along with
     // other cases where the stacksave/stackrestore is redundant.
-    stackPtr = builder.create<LLVM::StackSaveOp>(
+    stackPtr = LLVM::StackSaveOp::create(builder,
         call->getLoc(), LLVM::LLVMPointerType::get(call->getContext()));
   }
   builder.setInsertionPointToStart(callerEntryBlock);
   for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
-    auto newConstant = builder.create<LLVM::ConstantOp>(
+    auto newConstant = LLVM::ConstantOp::create(builder,
         allocaOp->getLoc(), allocaOp.getArraySize().getType(), arraySize);
     // Insert a lifetime start intrinsic where the alloca was before moving it.
     if (shouldInsertLifetime) {
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(allocaOp);
-      builder.create<LLVM::LifetimeStartOp>(
+      LLVM::LifetimeStartOp::create(builder,
           allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
           allocaOp.getResult());
     }
@@ -139,10 +139,10 @@ handleInlinedAllocas(Operation *call,
       continue;
     builder.setInsertionPoint(block.getTerminator());
     if (hasDynamicAlloca)
-      builder.create<LLVM::StackRestoreOp>(call->getLoc(), stackPtr);
+      LLVM::StackRestoreOp::create(builder, call->getLoc(), stackPtr);
     for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
       if (shouldInsertLifetime)
-        builder.create<LLVM::LifetimeEndOp>(
+        LLVM::LifetimeEndOp::create(builder,
             allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
             allocaOp.getResult());
     }
@@ -603,15 +603,15 @@ static Value handleByValArgumentInit(OpBuilder &builder, Location loc,
     OpBuilder::InsertionGuard insertionGuard(builder);
     Block *entryBlock = &(*argument.getParentRegion()->begin());
     builder.setInsertionPointToStart(entryBlock);
-    Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
+    Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
                                                  builder.getI64IntegerAttr(1));
-    allocaOp = builder.create<LLVM::AllocaOp>(
+    allocaOp = LLVM::AllocaOp::create(builder,
         loc, argument.getType(), elementType, one, targetAlignment);
   }
   // Copy the pointee to the newly allocated value.
-  Value copySize = builder.create<LLVM::ConstantOp>(
+  Value copySize = LLVM::ConstantOp::create(builder,
       loc, builder.getI64Type(), builder.getI64IntegerAttr(elementTypeSize));
-  builder.create<LLVM::MemcpyOp>(loc, allocaOp, argument, copySize,
+  LLVM::MemcpyOp::create(builder, loc, allocaOp, argument, copySize,
                                  /*isVolatile=*/false);
   return allocaOp;
 }
@@ -747,7 +747,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    builder.create<LLVM::BrOp>(op->getLoc(), returnOp.getOperands(), newDest);
+    LLVM::BrOp::create(builder, op->getLoc(), returnOp.getOperands(), newDest);
     op->erase();
   }
 
@@ -801,7 +801,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
     // and is extremely unlikely to exist in the code prior to inlining, using
     // this to communicate between this method and `processInlinedCallBlocks`.
     // TODO: Fix this by refactoring the inliner interface.
-    auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
+    auto copyOp = LLVM::SSACopyOp::create(builder, call->getLoc(), argument);
     if (argumentAttrs.contains(LLVM::LLVMDialect::getNoAliasAttrName()))
       copyOp->setDiscardableAttr(
           builder.getStringAttr(LLVM::LLVMDialect::getNoAliasAttrName()),
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
index 1a5a6e4ea1a3c..6fa4f6935f11c 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
@@ -58,7 +58,7 @@ static void ensureDistinctSuccessors(Block &bb) {
       terminator->setSuccessor(dummyBlock, position);
       for (BlockArgument arg : successor.first->getArguments())
         dummyBlock->addArgument(arg.getType(), arg.getLoc());
-      builder.create<LLVM::BrOp>(terminator->getLoc(),
+      LLVM::BrOp::create(builder, terminator->getLoc(),
                                  dummyBlock->getArguments(), successor.first);
     }
   }
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
index 8db32ec1526c4..de6c3043bbcdd 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
@@ -59,32 +59,32 @@ LogicalResult ExpandDivF16::matchAndRewrite(LLVM::FDivOp op,
   Type i32Type = rewriter.getI32Type();
 
   // Extend lhs and rhs to fp32.
-  Value lhs = rewriter.create<LLVM::FPExtOp>(loc, f32Type, op.getLhs());
-  Value rhs = rewriter.create<LLVM::FPExtOp>(loc, f32Type, op.getRhs());
+  Value lhs = LLVM::FPExtOp::create(rewriter, loc, f32Type, op.getLhs());
+  Value rhs = LLVM::FPExtOp::create(rewriter, loc, f32Type, op.getRhs());
 
   // float rcp = rcp.approx.ftz.f32(rhs), approx = lhs * rcp.
-  Value rcp = rewriter.create<NVVM::RcpApproxFtzF32Op>(loc, f32Type, rhs);
-  Value approx = rewriter.create<LLVM::FMulOp>(loc, lhs, rcp);
+  Value rcp = NVVM::RcpApproxFtzF32Op::create(rewriter, loc, f32Type, rhs);
+  Value approx = LLVM::FMulOp::create(rewriter, loc, lhs, rcp);
 
   // Refine the approximation with one Newton iteration:
   // float refined = approx + (lhs - approx * rhs) * rcp;
-  Value err = rewriter.create<LLVM::FMAOp>(
-      loc, approx, rewriter.create<LLVM::FNegOp>(loc, rhs), lhs);
-  Value refined = rewriter.create<LLVM::FMAOp>(loc, err, rcp, approx);
+  Value err = LLVM::FMAOp::create(rewriter,
+      loc, approx, LLVM::FNegOp::create(rewriter, loc, rhs), lhs);
+  Value refined = LLVM::FMAOp::create(rewriter, loc, err, rcp, approx);
 
   // Use refined value if approx is normal (exponent neither all 0 or all 1).
-  Value mask = rewriter.create<LLVM::ConstantOp>(
+  Value mask = LLVM::ConstantOp::create(rewriter,
       loc, i32Type, rewriter.getUI32IntegerAttr(0x7f800000));
-  Value cast = rewriter.create<LLVM::BitcastOp>(loc, i32Type, approx);
-  Value exp = rewriter.create<LLVM::AndOp>(loc, i32Type, cast, mask);
-  Value zero = rewriter.create<LLVM::ConstantOp>(
+  Value cast = LLVM::BitcastOp::create(rewriter, loc, i32Type, approx);
+  Value exp = LLVM::AndOp::create(rewriter, loc, i32Type, cast, mask);
+  Value zero = LLVM::ConstantOp::create(rewriter,
       loc, i32Type, rewriter.getUI32IntegerAttr(0));
-  Value pred = rewriter.create<LLVM::OrOp>(
+  Value pred = LLVM::OrOp::create(rewriter,
       loc,
-      rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, exp, zero),
-      rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, exp, mask));
+      LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, zero),
+      LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, mask));
   Value result =
-      rewriter.create<LLVM::SelectOp>(loc, f32Type, pred, approx, refined);
+      LLVM::SelectOp::create(rewriter, loc, f32Type, pred, approx, refined);
 
   // Replace with trucation back to fp16.
   rewriter.replaceOpWithNewOp<LLVM::FPTruncOp>(op, op.getType(), result);
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 3aa6ac3ea0918..51cc1232df891 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -62,10 +62,10 @@ static OpFoldResult getDimValue(OpBuilder &builder, Location loc, Value v,
   return getAsOpFoldResult(
       TypeSwitch<Type, Value>(v.getType())
           .Case<RankedTensorType>([&](RankedTensorType t) -> Value {
-            return builder.create<tensor::DimOp>(loc, v, dim);
+            return tensor::DimOp::create(builder, loc, v, dim);
           })
           .Case<MemRefType>([&](MemRefType t) -> Value {
-            return builder.create<memref::DimOp>(loc, v, dim);
+            return memref::DimOp::create(builder, loc, v, dim);
           }));
 }
 
@@ -77,11 +77,11 @@ static Operation *getSlice(OpBuilder &b, Location loc, Value source,
                            ArrayRef<OpFoldResult> strides) {
   return TypeSwitch<Type, Operation *>(source.getType())
       .Case<RankedTensorType>([&](RankedTensorType t) -> Operation * {
-        return b.create<tensor::ExtractSliceOp>(loc, source, offsets, sizes,
+        return tensor::ExtractSliceOp::create(b, loc, source, offsets, sizes,
                                                 strides);
       })
       .Case<MemRefType>([&](MemRefType type) -> Operation * {
-        return b.create<memref::SubViewOp>(loc, source, offsets, sizes,
+        return memref::SubViewOp::create(b, loc, source, offsets, sizes,
                                            strides);
       })
       .Default([&](Type t) -> Operation * { return nullptr; });
@@ -453,35 +453,35 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     switch (unaryFn) {
     case UnaryFn::exp:
-      return builder.create<math::ExpOp>(arg.getLoc(), arg);
+      return math::ExpOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::log:
-      return builder.create<math::LogOp>(arg.getLoc(), arg);
+      return math::LogOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::abs:
-      return builder.create<math::AbsFOp>(arg.getLoc(), arg);
+      return math::AbsFOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::ceil:
-      return builder.create<math::CeilOp>(arg.getLoc(), arg);
+      return math::CeilOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::floor:
-      return builder.create<math::FloorOp>(arg.getLoc(), arg);
+      return math::FloorOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::negf:
-      return builder.create<arith::NegFOp>(arg.getLoc(), arg);
+      return arith::NegFOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::reciprocal: {
       Attribute oneAttr = builder.getOneAttr(arg.getType());
-      auto one = builder.create<arith::ConstantOp>(arg.getLoc(),
+      auto one = arith::ConstantOp::create(builder, arg.getLoc(),
                                                    ::cast<TypedAttr>(oneAttr));
-      return builder.create<arith::DivFOp>(arg.getLoc(), one, arg);
+      return arith::DivFOp::create(builder, arg.getLoc(), one, arg);
     }
     case UnaryFn::round:
-      return builder.create<math::RoundOp>(arg.getLoc(), arg);
+      return math::RoundOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::sqrt:
-      return builder.create<math::SqrtOp>(arg.getLoc(), arg);
+      return math::SqrtOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::rsqrt:
-      return builder.create<math::RsqrtOp>(arg.getLoc(), arg);
+      return math::RsqrtOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::square:
-      return builder.create<arith::MulFOp>(arg.getLoc(), arg, arg);
+      return arith::MulFOp::create(builder, arg.getLoc(), arg, arg);
     case UnaryFn::tanh:
-      return builder.create<math::TanhOp>(arg.getLoc(), arg);
+      return math::TanhOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::erf:
-      return builder.create<math::ErfOp>(arg.getLoc(), arg);
+      return math::ErfOp::create(builder, arg.getLoc(), arg);
     }
     if (emitError) {
       emitError() << "unsupported unary function";
@@ -516,17 +516,17 @@ class RegionBuilderHelper {
     switch (binaryFn) {
     case BinaryFn::add:
       if (allComplex)
-        return builder.create<complex::AddOp>(arg0.getLoc(), arg0, arg1);
+        return complex::AddOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::AddFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::AddFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool)
-        return builder.create<arith::OrIOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::AddIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::OrIOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::AddIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::sub:
       if (allComplex)
-        return builder.create<complex::SubOp>(arg0.getLoc(), arg0, arg1);
+        return complex::SubOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::SubFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::SubFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool) {
         if (emitError) {
           emitError() << "unsupported operation: sub with bools";
@@ -534,20 +534,20 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: sub with bools");
       }
-      return builder.create<arith::SubIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::SubIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::mul:
       if (allComplex)
-        return builder.create<complex::MulOp>(arg0.getLoc(), arg0, arg1);
+        return complex::MulOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::MulFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MulFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool)
-        return builder.create<arith::AndIOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MulIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::AndIOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MulIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::div:
       if (allComplex)
-        return builder.create<complex::DivOp>(arg0.getLoc(), arg0, arg1);
+        return complex::DivOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::DivFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::DivFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool) {
         if (emitError) {
           emitError() << "unsupported operation: div with bools";
@@ -555,7 +555,7 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: div with bools");
       }
-      return builder.create<arith::DivSIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::DivSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::div_unsigned:
       if (!allInteger || allBool) {
         if (emitError) {
@@ -564,30 +564,30 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: unsigned div not on uint");
       }
-      return builder.create<arith::DivUIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::DivUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_signed:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MaximumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MaxSIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MaximumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MaxSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::min_signed:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MinimumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MinSIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MinimumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MinSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_unsigned:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MaximumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MaxUIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MaximumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MaxUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::min_unsigned:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MinimumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MinUIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MinimumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MinUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::powf:
       assert(allFloatingPoint);
-      return builder.create<math::PowFOp>(arg0.getLoc(), arg0, arg1);
+      return math::PowFOp::create(builder, arg0.getLoc(), arg0, arg1);
     }
     if (emitError) {
       emitError() << "unsupported binary function";
@@ -610,7 +610,7 @@ class RegionBuilderHelper {
     case TernaryFn::select:
       if (!headBool && !(tailFloatingPoint || tailInteger))
         llvm_unreachable("unsupported non numeric type");
-      return builder.create<arith::SelectOp>(arg0.getLoc(), arg0, arg1, arg2);
+      return arith::SelectOp::create(builder, arg0.getLoc(), arg0, arg1, arg2);
     }
     if (emitError) {
       emitError() << "unsupported ternary function";
@@ -639,7 +639,7 @@ class RegionBuilderHelper {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
-    builder.create<YieldOp>(loc, values);
+    YieldOp::create(builder, loc, values);
   }
 
   Value constant(const std::string &value) {
@@ -647,13 +647,13 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
     Attribute valueAttr = parseAttribute(value, builder.getContext());
-    return builder.create<arith::ConstantOp>(loc, ::cast<TypedAttr>(valueAttr));
+    return arith::ConstantOp::create(builder, loc, ::cast<TypedAttr>(valueAttr));
   }
 
   Value index(int64_t dim) {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
-    return builder.create<IndexOp>(builder.getUnknownLoc(), dim);
+    return IndexOp::create(builder, builder.getUnknownLoc(), dim);
   }
 
   Type getIntegerType(unsigned width) {
@@ -749,12 +749,12 @@ struct FoldFillWithTensorReshape : OpRewritePattern<TensorReshapeOp> {
     TensorReshapeOp newInit;
     if constexpr (std::is_same<TensorReshapeOp, tensor::ExpandShapeOp>::value) {
 
-      newInit = rewriter.create<TensorReshapeOp>(
+      newInit = TensorReshapeOp::create(rewriter,
           loc, reshapeOp.getResultType(), oldFill.output(),
           reshapeOp.getReassociation(), reshapeOp.getOutputShape(),
           reshapeOp.getStaticOutputShape());
     } else {
-      newInit = rewriter.create<TensorReshapeOp>(loc, reshapeOp.getResultType(),
+      newInit = TensorReshapeOp::create(rewriter, loc, reshapeOp.getResultType(),
                                                  oldFill.output(),
                                                  reshapeOp.getReassociation());
     }
@@ -786,7 +786,7 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
       return rewriter.notifyMatchFailure(
           padOp, "failed to reify tensor.pad op result shape");
 
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         padOp.getLoc(), reifiedShape.front(),
         padOp.getResultType().getElementType());
     Value replacement =
@@ -795,7 +795,7 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
                             ValueRange{emptyTensor})
             .getResult(0);
     if (replacement.getType() != padOp.getResultType()) {
-      replacement = rewriter.create<tensor::CastOp>(
+      replacement = tensor::CastOp::create(rewriter,
           fillOp.getLoc(), padOp.getResultType(), replacement);
     }
     rewriter.replaceOp(padOp, replacement);
@@ -889,7 +889,7 @@ struct FoldInsertPadIntoFill : public OpRewritePattern<tensor::InsertSliceOp> {
     for (int i = 0, e = srcPadType.getRank(); i < e; ++i) {
       if (srcPadType.isDynamicDim(i)) {
         newSizes.push_back(
-            rewriter.create<tensor::DimOp>(loc, srcPadOp.getSource(), i)
+            tensor::DimOp::create(rewriter, loc, srcPadOp.getSource(), i)
                 .getResult());
       } else {
         newSizes.push_back(rewriter.getIndexAttr(srcPadType.getDimSize(i)));
@@ -942,7 +942,7 @@ static FailureOr<FillOp> foldFillPackIntoFillOp(RewriterBase &rewriter,
   if (!packOpDest.hasOneUse())
     return failure();
 
-  return rewriter.create<linalg::FillOp>(packOp.getLoc(), fillOp.getInputs(),
+  return linalg::FillOp::create(rewriter, packOp.getLoc(), fillOp.getInputs(),
                                          packOp.getDest());
 }
 
@@ -1042,7 +1042,7 @@ struct FoldConcatsOfFill : public OpRewritePattern<tensor::ConcatOp> {
           concatOp, "not all operands are defined by a compatible fill op");
     }
 
-    Value outsConcat = rewriter.create<tensor::ConcatOp>(
+    Value outsConcat = tensor::ConcatOp::create(rewriter,
         concatOp.getLoc(), concatOp.getDim(), allOuts);
     rewriter.replaceOpWithNewOp<linalg::FillOp>(
         concatOp, firstFillOp.getDpsInputOperand(0)->get(), outsConcat);
@@ -1407,13 +1407,13 @@ struct EraseIdentityLinalgOp : public OpRewritePattern<OpTy> {
         // TODO: unify the two ops?
         if (sparse_tensor::getSparseTensorEncoding(returnType) ||
             sparse_tensor::getSparseTensorEncoding(resultType))
-          returnedArg = rewriter.create<sparse_tensor::ConvertOp>(
+          returnedArg = sparse_tensor::ConvertOp::create(rewriter,
               linalgOp.getLoc(), resultType, returnedArg);
         else {
           if (!tensor::CastOp::areCastCompatible(returnedArg.getType(),
                                                  resultType))
             return failure();
-          returnedArg = rewriter.create<tensor::CastOp>(
+          returnedArg = tensor::CastOp::create(rewriter,
               linalgOp.getLoc(), resultType, returnedArg);
         }
       }
@@ -1528,7 +1528,7 @@ static void addBodyWithPayloadOp(OpAsmParser &parser, OperationState &result,
       TypeRange{llvm::cast<ShapedType>(result.operands.back().getType())
                     .getElementType()},
       payloadOpAttrs);
-  b.create<YieldOp>(result.location, payloadOp->getResults());
+  YieldOp::create(b, result.location, payloadOp->getResults());
 }
 
 ParseResult MapOp::parse(OpAsmParser &parser, OperationState &result) {
@@ -1945,7 +1945,7 @@ static void buildIdentityRegion(OpBuilder &builder, Location loc,
   buildGenericRegion(builder, loc, region, inputs, outputs,
                      [](OpBuilder &b, Location loc, ValueRange args) {
                        if (!args.empty())
-                         b.create<linalg::YieldOp>(loc, args[0]);
+                         linalg::YieldOp::create(b, loc, args[0]);
                      });
 }
 
@@ -2138,7 +2138,7 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     unsigned inputRank = broadcastInputTy.getRank();
     for (unsigned i = 0; i < inputRank; ++i) {
       if (broadcastInputTy.isDynamicDim(i)) {
-        dims.push_back(rewriter.create<tensor::DimOp>(loc, broadcastInput, i)
+        dims.push_back(tensor::DimOp::create(rewriter, loc, broadcastInput, i)
                            ->getResult(0));
       } else {
         dims.push_back(IntegerAttr::get(IndexType::get(ctx),
@@ -2147,7 +2147,7 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     }
     SmallVector<OpFoldResult> transposeResultShapes =
         applyPermutation(dims, resultPerms);
-    Value transposeInit = rewriter.create<tensor::EmptyOp>(
+    Value transposeInit = tensor::EmptyOp::create(rewriter,
         transposeOp.getLoc(), transposeResultShapes,
         broadcastInputTy.getElementType());
 
@@ -2547,7 +2547,7 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     // continue to propagate as far up the stack as it can go.
     OpOperand *outOperand = linalgOp.getDpsInitOperand(resultNumber);
     Value newOperand =
-        rewriter.create<tensor::CastOp>(loc, resultType, outOperand->get());
+        tensor::CastOp::create(rewriter, loc, resultType, outOperand->get());
     SmallVector<Value> newOperands = linalgOp.getDpsInputs();
     SmallVector<Value> outputOperands(linalgOp.getDpsInits().begin(),
                                       linalgOp.getDpsInits().end());
@@ -2560,7 +2560,7 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     Operation *newOp = clone(rewriter, linalgOp, resultTypes, newOperands);
 
     // Create a tensor.cast operation back to the original type.
-    Value castBack = rewriter.create<tensor::CastOp>(
+    Value castBack = tensor::CastOp::create(rewriter,
         loc, resultValue.getType(), newOp->getResult(resultNumber));
 
     SmallVector<Value> results(newOp->result_begin(), newOp->result_end());
@@ -2653,7 +2653,7 @@ static void createNewOperandWithStaticSizes(
     changeNeeded = true;
     // Get the new operand value given its size and element type by
     // casting it.
-    Value newOperand = rewriter.create<tensor::CastOp>(loc, resultType, src);
+    Value newOperand = tensor::CastOp::create(rewriter, loc, resultType, src);
     unsigned index = opOperand->getOperandNumber();
     newOperands[index] = newOperand;
   }
@@ -2718,7 +2718,7 @@ struct InferStaticShapeOfOperands : public OpInterfaceRewritePattern<LinalgOp> {
       Type oldType = oldResult.getType();
       replacements.push_back(
           (newType != oldType)
-              ? rewriter.create<tensor::CastOp>(loc, oldType, newResult)
+              ? tensor::CastOp::create(rewriter, loc, oldType, newResult)
               : newResult);
     }
     rewriter.replaceOp(linalgOp, replacements);
@@ -2756,8 +2756,8 @@ SmallVector<Range> SoftmaxOp::getIterationDomain(OpBuilder &builder) {
   int64_t operandRank = getInputOperandRank();
   SmallVector<Range> loopBounds(operandRank);
   Location loc = getLoc();
-  Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-  Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+  Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+  Value one = arith::ConstantIndexOp::create(builder, loc, 1);
   Value source = getInput();
   for (auto dim : llvm::seq<int64_t>(0, operandRank)) {
     loopBounds[dim].offset = zero;
@@ -2924,11 +2924,11 @@ static Value reduce(OpBuilder &builder, Location loc, Value input, Value output,
          "We should have two maps: 1 for the input, 1 for the output");
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
 
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, output.getType(), input, output, indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value result = b.create<T>(loc, args[0], args[1]);
-        b.create<linalg::YieldOp>(loc, result);
+        Value result = T::create(b, loc, args[0], args[1]);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -2947,12 +2947,12 @@ static Value buildSubAndExpOp(OpBuilder &builder, Location loc, Value input,
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
   // Add the affine map for the output argument.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, input.getType(), ValueRange{input, max}, output, indexingMaps,
       iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value diff = b.create<arith::SubFOp>(loc, args[0], args[1]);
-        Value result = b.create<math::ExpOp>(loc, diff);
-        b.create<linalg::YieldOp>(loc, result);
+        Value diff = arith::SubFOp::create(b, loc, args[0], args[1]);
+        Value result = math::ExpOp::create(b, loc, diff);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -2974,12 +2974,12 @@ static Value buildDivOp(OpBuilder &builder, Location loc, Value numerator,
   assert(indexingMaps[0].isIdentity() && "Numerator map should be identity");
   // Add the affine map for the output tensor.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, numerator.getType(), ValueRange{numerator, denominator}, output,
       indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value result = b.create<arith::DivFOp>(loc, args[0], args[1]);
-        b.create<linalg::YieldOp>(loc, result);
+        Value result = arith::DivFOp::create(b, loc, args[0], args[1]);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -3015,12 +3015,12 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
   Value output = getOutput();
   dims.erase(dims.begin() + reductionDim);
   // Step 1: Compute max along dim.
-  Value outputReduce = b.create<tensor::EmptyOp>(loc, dims, elementType);
+  Value outputReduce = tensor::EmptyOp::create(b, loc, dims, elementType);
   Value neutralForMaxF = arith::getIdentityValue(arith::AtomicRMWKind::maxnumf,
                                                  elementType, b, loc,
                                                  /*useOnlyFiniteValue=*/true);
   Value neutralForMaxFInit =
-      b.create<linalg::FillOp>(loc, Value{neutralForMaxF}, outputReduce)
+      linalg::FillOp::create(b, loc, Value{neutralForMaxF}, outputReduce)
           .result();
   Value max =
       reduce<arith::MaxNumFOp>(b, loc, input, neutralForMaxFInit, reductionDim);
@@ -3032,7 +3032,7 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
   Value zero = arith::getIdentityValue(arith::AtomicRMWKind::addf, elementType,
                                        b, loc, /*useOnlyFiniteValue=*/true);
   Value zeroInit =
-      b.create<linalg::FillOp>(loc, Value{zero}, outputReduce).result();
+      linalg::FillOp::create(b, loc, Value{zero}, outputReduce).result();
   Value denominator =
       reduce<arith::AddFOp>(b, loc, numerator, zeroInit, reductionDim);
 
@@ -3153,7 +3153,7 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
   int64_t filterRank = getFilterOperandRank();
   SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
   Location loc = getLoc();
-  auto filterSlice = builder.create<tensor::ExtractSliceOp>(
+  auto filterSlice = tensor::ExtractSliceOp::create(builder,
       loc, getFilter(), sliceOffsets, sliceSizes, filterStrides);
   tiledOperands.emplace_back(filterSlice);
 
@@ -3164,7 +3164,7 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -3333,7 +3333,7 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
       {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
   int64_t inputRank = getInputOperandRank();
   SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
-  auto inputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto inputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getInput(), sliceOffsets, sliceSizes, inputStrides);
   tiledOperands.emplace_back(inputSlice);
 
@@ -3344,7 +3344,7 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -3504,7 +3504,7 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
                      sizes[getValueFDim()]});
   int64_t valueRank = getValueOperandRank();
   SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
-  auto valueSlice = builder.create<tensor::ExtractSliceOp>(
+  auto valueSlice = tensor::ExtractSliceOp::create(builder,
       loc, getValue(), sliceOffsets, sliceSizes, sliceStrides);
   tiledOperands.emplace_back(valueSlice);
 
@@ -3515,7 +3515,7 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> strides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, strides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -4971,7 +4971,7 @@ Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
            llvm::cast<RankedTensorType>(source.getType()).getShape())) {
     if (ShapedType::isDynamic(value))
       mixedSizes.push_back(
-          b.create<tensor::DimOp>(loc, source, index).getResult());
+          tensor::DimOp::create(b, loc, source, index).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(value));
   }
@@ -4985,7 +4985,7 @@ Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
 
   mixedSizes.append(innerTileSizes.begin(), innerTileSizes.end());
   auto elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
-  return b.create<tensor::EmptyOp>(loc, mixedSizes, elemType);
+  return tensor::EmptyOp::create(b, loc, mixedSizes, elemType);
 }
 
 PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
@@ -4996,7 +4996,7 @@ PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
   Value transposedDest =
       createDestinationTensor(b, loc, getSource(), metadata.innerTiles,
                               metadata.innerDimsPos, metadata.outerDimsPerm);
-  return b.create<PackOp>(loc, getSource(), transposedDest,
+  return PackOp::create(b, loc, getSource(), transposedDest,
                           metadata.innerDimsPos, metadata.innerTiles,
                           getPaddingValue(), metadata.outerDimsPerm);
 }
@@ -5138,7 +5138,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (srcShape != packOp.getSourceType().getShape()) {
       auto newSrcType = packOp.getSourceType().clone(srcShape);
       source =
-          rewriter.create<tensor::CastOp>(loc, newSrcType, packOp.getSource());
+          tensor::CastOp::create(rewriter, loc, newSrcType, packOp.getSource());
     }
     Value dest = packOp.getDest();
     RankedTensorType originalResultType = packOp.getDestType();
@@ -5146,7 +5146,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (needUpdateDestType) {
       auto newDestType = packOp.getDestType().clone(destShape);
       dest =
-          rewriter.create<tensor::CastOp>(loc, newDestType, packOp.getDest());
+          tensor::CastOp::create(rewriter, loc, newDestType, packOp.getDest());
     }
     rewriter.modifyOpInPlace(packOp, [&] {
       packOp.getSourceMutable().assign(source);
@@ -5157,7 +5157,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (needUpdateDestType) {
       rewriter.setInsertionPointAfter(packOp);
       auto castOp =
-          rewriter.create<tensor::CastOp>(loc, originalResultType, packOp);
+          tensor::CastOp::create(rewriter, loc, originalResultType, packOp);
       rewriter.replaceAllUsesExcept(packOp, castOp, castOp);
     }
     return success();
@@ -5250,7 +5250,7 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    PackOp newOp = rewriter.create<PackOp>(
+    PackOp newOp = PackOp::create(rewriter,
         op.getLoc(), newOperands[0], newOperands[1], op.getInnerDimsPos(),
         newMixedTileSizes, op.getPaddingValue(), op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
@@ -5259,7 +5259,7 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
     Value replacement = (newResult.getType() != oldResult.getType())
-                            ? rewriter.create<tensor::CastOp>(
+                            ? tensor::CastOp::create(rewriter,
                                   op->getLoc(), oldResult.getType(), newResult)
                             : newResult;
 
@@ -5358,7 +5358,7 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
   for (auto i :
        llvm::seq<unsigned>(0, srcType.getRank() - innerTileSizes.size())) {
     if (srcType.isDynamicDim(i))
-      mixedSizes.push_back(b.create<tensor::DimOp>(loc, source, i).getResult());
+      mixedSizes.push_back(tensor::DimOp::create(b, loc, source, i).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(srcType.getDimSize(i)));
   }
@@ -5371,7 +5371,7 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
     mixedSizes[dimPos] = dimMul(mixedSizes[dimPos], tileSize);
 
   auto elemType = srcType.getElementType();
-  return b.create<tensor::EmptyOp>(loc, mixedSizes, elemType);
+  return tensor::EmptyOp::create(b, loc, mixedSizes, elemType);
 }
 
 UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
@@ -5380,7 +5380,7 @@ UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
                                          ArrayRef<int64_t> outerPermutation) {
   PackOrUnPackTransposeResult metadata = commonPermutationOfPackAndUnPackOp(
       *this, innerPermutation, outerPermutation);
-  return b.create<UnPackOp>(loc, transposedSource, getDest(),
+  return UnPackOp::create(b, loc, transposedSource, getDest(),
                             metadata.innerDimsPos, metadata.innerTiles,
                             metadata.outerDimsPerm);
 }
@@ -5454,7 +5454,7 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
             extractSliceUser.getResultType().getRank()) {
       OpBuilder::InsertionGuard g(rewriter);
       rewriter.setInsertionPoint(unPackOp);
-      auto newDest = rewriter.create<tensor::ExtractSliceOp>(
+      auto newDest = tensor::ExtractSliceOp::create(rewriter,
           unPackOp->getLoc(), unPackOp.getDest(),
           extractSliceUser.getMixedOffsets(), extractSliceUser.getMixedSizes(),
           extractSliceUser.getMixedStrides());
@@ -5474,16 +5474,16 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
     Value source = unPackOp.getSource();
     if (srcShape != unPackOp.getSourceType().getShape()) {
       auto newSrcType = unPackOp.getSourceType().clone(srcShape);
-      source = rewriter.create<tensor::CastOp>(loc, newSrcType,
+      source = tensor::CastOp::create(rewriter, loc, newSrcType,
                                                unPackOp.getSource());
     }
     Value dest = unPackOp.getDest();
     if (destShape != unPackOp.getDestType().getShape()) {
       auto newDestType = unPackOp.getDestType().clone(destShape);
       dest =
-          rewriter.create<tensor::CastOp>(loc, newDestType, unPackOp.getDest());
+          tensor::CastOp::create(rewriter, loc, newDestType, unPackOp.getDest());
     }
-    Value newOp = rewriter.create<UnPackOp>(
+    Value newOp = UnPackOp::create(rewriter,
         loc, source, dest, unPackOp.getInnerDimsPos(), unPackOp.getMixedTiles(),
         unPackOp.getOuterDimsPerm());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(
@@ -5542,7 +5542,7 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    UnPackOp newOp = rewriter.create<UnPackOp>(
+    UnPackOp newOp = UnPackOp::create(rewriter,
         op.getLoc(), sourceTensor, newOperands[1], op.getInnerDimsPos(),
         newMixedTileSizes, op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
@@ -5551,7 +5551,7 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
     Value replacement = (newResult.getType() != oldResult.getType())
-                            ? rewriter.create<tensor::CastOp>(
+                            ? tensor::CastOp::create(rewriter,
                                   op->getLoc(), oldResult.getType(), newResult)
                             : newResult;
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 5d5f9de465561..ba03d09e5072e 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -672,7 +672,7 @@ static Operation *replaceForAllWithNewSignature(
   newOuts.push_back(outputs[resultNumber]);
 
   // Create new scf.forall op
-  auto newforallOp = rewriter.create<scf::ForallOp>(
+  auto newforallOp = scf::ForallOp::create(rewriter,
       loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
       forallOp.getMixedStep(), newOuts, forallOp.getMapping());
   rewriter.eraseBlock(newforallOp.getBody());
@@ -699,7 +699,7 @@ static Operation *replaceForAllWithNewSignature(
   Value src = tileAndFuseResult.tiledValues[0];
   Value dst = newforallOp.getRegionIterArgs().back();
   SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-  rewriter.create<tensor::ParallelInsertSliceOp>(firstYieldOp->getLoc(), src,
+  tensor::ParallelInsertSliceOp::create(rewriter, firstYieldOp->getLoc(), src,
                                                  dst, offsets, sizes, strides);
 
   for (auto result : llvm::enumerate(forallOp.getResults())) {
@@ -3410,12 +3410,12 @@ transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
         for (auto [ofrIdx, ofr] : llvm::enumerate(getMixedSizes())) {
           if (auto attr = llvm::dyn_cast_if_present<Attribute>(ofr)) {
             if (scalableSizes[ofrIdx]) {
-              auto val = b.create<arith::ConstantIndexOp>(
+              auto val = arith::ConstantIndexOp::create(b,
                   getLoc(), cast<IntegerAttr>(attr).getInt());
               Value vscale =
-                  b.create<vector::VectorScaleOp>(getLoc(), b.getIndexType());
+                  vector::VectorScaleOp::create(b, getLoc(), b.getIndexType());
               sizes.push_back(
-                  b.create<arith::MulIOp>(getLoc(), val, vscale).getResult());
+                  arith::MulIOp::create(b, getLoc(), val, vscale).getResult());
             } else {
               sizes.push_back(attr);
             }
@@ -3626,7 +3626,7 @@ static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter,
   SmallVector<OpFoldResult> normalizedSteps(normalizedUbs.size(),
                                             rewriter.getIndexAttr(1));
 
-  auto normalizedForallOp = rewriter.create<scf::ForallOp>(
+  auto normalizedForallOp = scf::ForallOp::create(rewriter,
       loc, normalizedLbs, normalizedUbs, normalizedSteps, loop.getOutputs(),
       loop.getMapping(), [](OpBuilder &, Location, ValueRange) {});
 
@@ -4128,7 +4128,7 @@ DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target,
         target->template getParentOfType<scf::InParallelOp>());
   }
 
-  Value extracted = rewriter.create<tensor::ExtractSliceOp>(
+  Value extracted = tensor::ExtractSliceOp::create(rewriter,
       target.getLoc(), target.getDest(), target.getMixedOffsets(),
       target.getMixedSizes(), target.getMixedStrides());
   Value copied = rewriter
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
index 1f6d96ca0f81f..2cb8ad367f93c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -184,7 +184,7 @@ struct SoftmaxOpInterface
         getBuffer(rewriter, softmaxOp.getOutput(), options, state);
     if (failed(outputBuffer))
       return failure();
-    rewriter.create<linalg::SoftmaxOp>(softmaxOp.getLoc(),
+    linalg::SoftmaxOp::create(rewriter, softmaxOp.getLoc(),
                                        /*result=*/TypeRange(), *inputBuffer,
                                        *outputBuffer, softmaxOp.getDimension());
     replaceOpWithBufferizedValues(rewriter, op, *outputBuffer);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index a7732b939e70d..61c779850366f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -30,10 +30,10 @@ static bool hasAllOneValues(DenseIntElementsAttr attr) {
 
 static Value createAdd(Location loc, Value x, Value y, OpBuilder &builder) {
   if (isa<IntegerType>(x.getType()))
-    return builder.create<arith::AddIOp>(loc, x, y);
+    return arith::AddIOp::create(builder, loc, x, y);
   if (isa<ComplexType>(x.getType()))
-    return builder.create<complex::AddOp>(loc, x, y);
-  return builder.create<arith::AddFOp>(loc, x, y);
+    return complex::AddOp::create(builder, loc, x, y);
+  return arith::AddFOp::create(builder, loc, x, y);
 }
 
 static Value createMul(Location loc, Value x, Value y, Type accType,
@@ -44,10 +44,10 @@ static Value createMul(Location loc, Value x, Value y, Type accType,
   Value yConvert =
       convertScalarToDtype(builder, loc, y, accType, /*isUnsignedCast=*/false);
   if (isa<ComplexType>(accType))
-    return builder.create<complex::MulOp>(loc, xConvert, yConvert);
+    return complex::MulOp::create(builder, loc, xConvert, yConvert);
   if (isa<IntegerType>(accType))
-    return builder.create<arith::MulIOp>(loc, xConvert, yConvert);
-  return builder.create<arith::MulFOp>(loc, xConvert, yConvert);
+    return arith::MulIOp::create(builder, loc, xConvert, yConvert);
+  return arith::MulFOp::create(builder, loc, xConvert, yConvert);
 }
 
 // Delinearizes the given composite `index` by the basis specified in `factors`.
@@ -56,7 +56,7 @@ static SmallVector<Value> unrollIndex(OpBuilder &b, Location loc, Value index,
   assert(!factors.empty() && "empty factor list");
   SmallVector<Value> basis;
   for (int64_t f : factors)
-    basis.push_back(b.create<arith::ConstantOp>(loc, b.getIndexAttr(f)));
+    basis.push_back(arith::ConstantOp::create(b, loc, b.getIndexAttr(f)));
   FailureOr<SmallVector<Value>> multiIndex =
       affine::delinearizeIndex(b, loc, index, basis);
   assert(!failed(multiIndex) && "Failed to linearize img2col index");
@@ -115,17 +115,17 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0, 1, 2}, {3}};
   auto reshapedFilterType =
       RankedTensorType::get({fh * fw * ic, oc}, filterType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
@@ -138,15 +138,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value mIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> mIndices = unrollIndex(
@@ -170,9 +170,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because the filter does not share the same batch dimension,
@@ -187,7 +187,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -196,11 +196,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
@@ -244,7 +244,7 @@ rewriteInIm2Col(RewriterBase &rewriter,
     SmallVector<int64_t> targetShape = llvm::to_vector<4>(llvm::map_range(
         indices, [&](int64_t index) -> int64_t { return inputShape[index]; }));
 
-    Value outputTensor = rewriter.create<tensor::EmptyOp>(
+    Value outputTensor = tensor::EmptyOp::create(rewriter,
         loc, targetShape, operandTensorType.getElementType());
 
     SmallVector<utils::IteratorType> loopAttributeTypes(
@@ -255,12 +255,12 @@ rewriteInIm2Col(RewriterBase &rewriter,
             AffineMap::get(nloops, 0, exprs, rewriter.getContext())),
         AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-    auto transposedOp = rewriter.create<linalg::GenericOp>(
+    auto transposedOp = linalg::GenericOp::create(rewriter,
         loc, outputTensor.getType(),
         /*inputs=*/operand, /*outputs=*/outputTensor, indexingMaps,
         loopAttributeTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-          nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+          linalg::YieldOp::create(nestedBuilder, nestedLoc, args[0]);
         });
 
     return transposedOp.getResult(0);
@@ -307,15 +307,15 @@ rewriteInIm2Col(RewriterBase &rewriter,
       AffineMap::get(nloops, 0, inputExprs, rewriter.getContext()),
       AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/inputT, /*outputs=*/colTensor, indexingMaps,
       loopAttributeTypes,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, args[0]);
       });
 
   SmallVector<ReassociationIndices> img2ColTensorReassocIndices = {
@@ -331,16 +331,16 @@ rewriteInIm2Col(RewriterBase &rewriter,
   auto reshapedOutputTensorType =
       RankedTensorType::get({n * c, oh * ow}, outputType.getElementType());
 
-  Value reshapedImg2ColTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedImg2ColTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedImg2ColTensorType, img2ColTensor.getResult(0),
       img2ColTensorReassocIndices);
-  Value reshapedFilterTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilterTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterTensorType, filterT, filterReassociationIndice);
-  Value reshapedoutputTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedoutputTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputTensorType, transposedOutputTensor,
       outputReassociationIndice);
 
-  auto batchMatVecResult = rewriter.create<linalg::BatchMatvecOp>(
+  auto batchMatVecResult = linalg::BatchMatvecOp::create(rewriter,
       loc, TypeRange{reshapedoutputTensor.getType()},
       ValueRange{reshapedImg2ColTensor, reshapedFilterTensor},
       ValueRange{reshapedoutputTensor});
@@ -348,7 +348,7 @@ rewriteInIm2Col(RewriterBase &rewriter,
   SmallVector<ReassociationIndices> batchMatVecReassociationIndice = {{0, 1},
                                                                       {2, 3}};
 
-  auto batchMatVecResultReshaped = rewriter.create<tensor::ExpandShapeOp>(
+  auto batchMatVecResultReshaped = tensor::ExpandShapeOp::create(rewriter,
       loc, transposedOutputTensor.getType(), batchMatVecResult.getResult(0),
       batchMatVecReassociationIndice);
 
@@ -400,18 +400,18 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, ic * fh * fw}, inputType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1}, {2, 3}};
   auto reshapedOutputType =
       RankedTensorType::get({n, oc, oh * ow}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   // Convert the input to a (BKN) tensor.
   SmallVector<int64_t, 4> colTensorShape = {n, ic * fh * fw, oh * ow};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   auto nloops = colTensorShape.size();
@@ -423,15 +423,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<AffineMap, 4> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value nIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value nIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> kIndices = unrollIndex(
@@ -455,9 +455,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
 
         // im2col[n, ic*fh*fw, oh*ow] = input[n, ic, sh*oh + fh, sw*ow + fw]
         SmallVector<Value> extractionIndices{bIndex, icIndex, hIndex, wIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because the filter does not share the same batch dimension,
@@ -471,7 +471,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   auto resultMap = AffineMap::get(4, 0, {bDim, mDim, nDim}, context);
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{reshapedFilter, img2ColTensor.getResult(0)},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -480,11 +480,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
@@ -535,17 +535,17 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, fh * fw * ic}, filterType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
@@ -558,15 +558,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value mIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> mIndices = unrollIndex(
@@ -590,9 +590,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because we didn't transpose the filters we don't actually have a batched
@@ -606,7 +606,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -615,11 +615,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index 39e2aac27e213..f21563c323c1b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -37,7 +37,7 @@ static Value createInserts(RewriterBase &rewriter, Location loc, int dim,
   if (dim == static_cast<int>(shape.size()) - 1) {
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
-      destination = rewriter.create<tensor::InsertOp>(loc, *elementIt,
+      destination = tensor::InsertOp::create(rewriter, loc, *elementIt,
                                                       destination, indices);
       ++elementIt;
     }
@@ -65,7 +65,7 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
       MaterializeInDestination: {
     // Note: This is the preferred way of memcpy'ing because no layout map
     // and/or memory space must be specified for the source.
-    auto materializeOp = b.create<bufferization::MaterializeInDestinationOp>(
+    auto materializeOp = bufferization::MaterializeInDestinationOp::create(b,
         loc, tensorSource, memrefDest);
     materializeOp.setWritable(true);
   } break;
@@ -73,19 +73,19 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = b.create<bufferization::ToBufferOp>(
+    Value toBuffer = bufferization::ToBufferOp::create(b,
         loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
-    b.create<memref::CopyOp>(loc, toBuffer, memrefDest);
+    memref::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
   case linalg::BufferizeToAllocationOptions::MemcpyOp::LinalgCopy: {
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = b.create<bufferization::ToBufferOp>(
+    Value toBuffer = bufferization::ToBufferOp::create(b,
         loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
-    b.create<linalg::CopyOp>(loc, toBuffer, memrefDest);
+    linalg::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
   };
 }
@@ -120,14 +120,14 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
             ->materializeConstant(rewriter, constYieldedValue,
                                   yieldedValue.getType(), yieldedValue.getLoc())
             ->getResult(0);
-    auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(fillValue),
+    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(fillValue),
                                                   ValueRange(dest));
     return fillOp;
   }
 
   if (invariantYieldedValue) {
     // Padding with an invariant value.
-    auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(yieldedValue),
+    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(yieldedValue),
                                                   ValueRange(dest));
     return fillOp;
   }
@@ -137,7 +137,7 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(resultType.getRank()));
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, resultType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{dest}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
@@ -146,7 +146,7 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
   rewriter.setInsertionPointToStart(body);
   SmallVector<Value> bbArgReplacements;
   for (int64_t i = 0; i < resultType.getRank(); ++i)
-    bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
+    bbArgReplacements.push_back(linalg::IndexOp::create(rewriter, loc, i));
   rewriter.mergeBlocks(padOp.getBody(), body, bbArgReplacements);
 
   // Update terminator.
@@ -179,8 +179,8 @@ static SmallVector<Value> reifyOrComputeDynamicSizes(OpBuilder &b,
   for (int64_t i = 0; i < tensorType.getRank(); ++i) {
     if (tensorType.isDynamicDim(i))
       dynSizes.push_back(
-          b.create<DimOp>(value.getLoc(), value,
-                          b.create<arith::ConstantIndexOp>(value.getLoc(), i)));
+          DimOp::create(b, value.getLoc(), value,
+                          arith::ConstantIndexOp::create(b, value.getLoc(), i)));
   }
   return dynSizes;
 }
@@ -201,15 +201,15 @@ createAllocationForTensor(RewriterBase &rewriter, Location loc, Value value,
   Value alloc;
   if (options.allocOp ==
       linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloc) {
-    alloc = rewriter.create<memref::AllocOp>(loc, memrefType, dynamicSizes);
+    alloc = memref::AllocOp::create(rewriter, loc, memrefType, dynamicSizes);
     if (options.emitDealloc) {
       // Place deallocation at the end of the block.
       rewriter.setInsertionPoint(rewriter.getInsertionBlock()->getTerminator());
-      rewriter.create<memref::DeallocOp>(loc, alloc);
+      memref::DeallocOp::create(rewriter, loc, alloc);
     }
   } else if (options.allocOp ==
              linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloca) {
-    alloc = rewriter.create<memref::AllocaOp>(loc, memrefType, dynamicSizes);
+    alloc = memref::AllocaOp::create(rewriter, loc, memrefType, dynamicSizes);
     // No dealloc is needed.
   }
 
@@ -243,13 +243,13 @@ Value linalg::bufferizeToAllocation(
       getMixedSizes(rewriter, loc, padOp.getSource());
   SmallVector<OpFoldResult> strides(padOp.getResultType().getRank(),
                                     rewriter.getIndexAttr(1));
-  Value subview = rewriter.create<memref::SubViewOp>(
+  Value subview = memref::SubViewOp::create(rewriter,
       loc, alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
   createMemcpy(rewriter, loc, padOp.getSource(), subview, options);
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
       loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(padOp, toTensorOp);
@@ -338,7 +338,7 @@ Value linalg::bufferizeToAllocation(
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
       loc, allocTensorOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(allocTensorOp, toTensorOp);
@@ -354,7 +354,7 @@ FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
   auto shape = tensorType.getShape();
 
   // Create tensor.empty.
-  auto emptyOp = rewriter.create<EmptyOp>(loc, tensorType, ValueRange());
+  auto emptyOp = EmptyOp::create(rewriter, loc, tensorType, ValueRange());
 
   // Case: tensor<elem_type>.
   if (shape.empty()) {
@@ -369,7 +369,7 @@ FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
   SmallVector<Value, 2> constants;
   constants.reserve(maxDim);
   for (int i = 0; i < maxDim; ++i)
-    constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
+    constants.push_back(arith::ConstantIndexOp::create(rewriter, loc, i));
 
   // Traverse all elements and create tensor.insert ops.
   auto elementIt = fromElementsOp.getElements().begin();
@@ -395,14 +395,14 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
 
   // Create tensor.empty.
   auto emptyOp =
-      rewriter.create<EmptyOp>(loc, tensorType, generateOp.getDynamicExtents());
+      EmptyOp::create(rewriter, loc, tensorType, generateOp.getDynamicExtents());
 
   // Create linalg.generic.
   SmallVector<utils::IteratorType> iteratorTypes(tensorType.getRank(),
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(tensorType.getRank()));
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, tensorType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{emptyOp.getResult()}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
@@ -411,7 +411,7 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
   rewriter.setInsertionPointToStart(body);
   SmallVector<Value> bbArgReplacements;
   for (int64_t i = 0; i < tensorType.getRank(); ++i)
-    bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
+    bbArgReplacements.push_back(linalg::IndexOp::create(rewriter, loc, i));
   rewriter.mergeBlocks(&generateOp.getBody().front(), body, bbArgReplacements);
 
   // Update terminator.
@@ -450,13 +450,13 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
       llvm::all_of(padOp.getMixedHighPad(), isZeroInteger)) {
     using bufferization::AllocTensorOp;
     Value allocated =
-        rewriter.create<AllocTensorOp>(loc, resultType, dynamicSizes);
+        AllocTensorOp::create(rewriter, loc, resultType, dynamicSizes);
     auto copyOp = rewriter.replaceOpWithNewOp<linalg::CopyOp>(
         padOp, padOp.getSource(), allocated);
     return copyOp.getOperation();
   }
 
-  Value empty = rewriter.create<EmptyOp>(loc, resultType, dynamicSizes);
+  Value empty = EmptyOp::create(rewriter, loc, resultType, dynamicSizes);
   // Create linalg.fill or linalg.generic.
   Operation *fillOp = movePaddingToFillOrGenericOp(rewriter, loc, padOp, empty);
   rewriter.setInsertionPointAfter(fillOp);
@@ -567,7 +567,7 @@ Value linalg::bufferizeToAllocation(
       createMemcpy(rewriter, op->getLoc(), operand->get(), alloc, options);
     }
     rewriter.modifyOpInPlace(op, [&]() {
-      auto toTensorOp = rewriter.create<ToTensorOp>(
+      auto toTensorOp = ToTensorOp::create(rewriter,
           op->getLoc(), operand->get().getType(), alloc);
       operand->set(toTensorOp);
       if (options.bufferizeDestinationOnly) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index 70574903f7111..eb2c7756dc6ae 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -287,7 +287,7 @@ getOrCreatePackedViewOfOperand(OpBuilder &b, Location loc, PackInfo packInfo,
 
   auto empty = linalg::PackOp::createDestinationTensor(
       b, loc, opOperand->get(), innerTileSizes, innerDimsPos, outerDimsPerm);
-  auto packedOperand = b.create<linalg::PackOp>(
+  auto packedOperand = linalg::PackOp::create(b,
       loc, opOperand->get(), empty, innerDimsPos, innerTileSizes,
       /*padding=*/std::nullopt, outerDimsPerm);
   return std::make_tuple(packedOperand, indexingMap);
@@ -345,7 +345,7 @@ static GenericOp packGenericOp(RewriterBase &rewriter, GenericOp genericOp,
 
   indexingMaps.push_back(packedOutIndexingMap);
 
-  auto newGenericOp = rewriter.create<linalg::GenericOp>(
+  auto newGenericOp = linalg::GenericOp::create(rewriter,
       loc, dest.getType(), inputOperands, dest, indexingMaps, iterTypes,
       /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
   rewriter.cloneRegionBefore(genericOp.getRegion(), newGenericOp.getRegion(),
@@ -457,7 +457,7 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, linalg::PackOp packOp,
   if (!packOpDest.hasOneUse())
     return failure();
   if (auto emptyOp = packOpDest.getDefiningOp<tensor::EmptyOp>()) {
-    packOpDest = rewriter.create<tensor::EmptyOp>(
+    packOpDest = tensor::EmptyOp::create(rewriter,
         genericOp->getLoc(), emptyOp.getMixedSizes(),
         emptyOp.getType().getElementType());
   } else {
@@ -562,7 +562,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     auto empty = linalg::PackOp::createDestinationTensor(
         rewriter, loc, padOp.getSource(), mixedTiles, innerDimsPos,
         outerDimsPerm);
-    auto sourcePack = rewriter.create<linalg::PackOp>(
+    auto sourcePack = linalg::PackOp::create(rewriter,
         loc, padOp.getSource(), empty, innerDimsPos, mixedTiles,
         /*padding=*/std::nullopt, outerDimsPerm);
 
@@ -579,7 +579,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, /*result=*/Type(), sourcePack, lowPad, highPad, paddingVal,
         padOp.getNofold());
 
@@ -588,7 +588,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     if (!padOp->hasOneUse()) {
       auto unpackEmpty = linalg::UnPackOp::createDestinationTensor(
           rewriter, loc, newPadOp, mixedTiles, innerDimsPos, outerDimsPerm);
-      Value unpackedPad = rewriter.create<linalg::UnPackOp>(
+      Value unpackedPad = linalg::UnPackOp::create(rewriter,
           loc, newPadOp, unpackEmpty, innerDimsPos, mixedTiles, outerDimsPerm);
       rewriter.replaceAllUsesExcept(padOp, unpackedPad, sourcePack);
     }
@@ -719,7 +719,7 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
   auto emptyOp = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), collapseOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newPackOp = rewriter.create<linalg::PackOp>(
+  auto newPackOp = linalg::PackOp::create(rewriter,
       packOp.getLoc(), collapseOp.getSrc(), emptyOp, projectedInnerDimsPos,
       packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPerm);
 
@@ -735,7 +735,7 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
     nextPos += 1;
   }
 
-  auto newCollapseOp = rewriter.create<tensor::CollapseShapeOp>(
+  auto newCollapseOp = tensor::CollapseShapeOp::create(rewriter,
       collapseOp.getLoc(), packOp.getType(), newPackOp, newReassocIndices);
   rewriter.replaceOp(packOp, newCollapseOp);
 
@@ -853,12 +853,12 @@ bubbleUpPackOpThroughExpandShape(tensor::ExpandShapeOp expandOp,
   Value destTensor = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), expandOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, /*outerDimsPerm=*/SmallVector<int64_t>{});
-  Value packedVal = rewriter.create<linalg::PackOp>(
+  Value packedVal = linalg::PackOp::create(rewriter,
       packOp.getLoc(), expandOp.getSrc(), destTensor, projectedInnerDimsPos,
       packOp.getMixedTiles(), packOp.getPaddingValue(),
       /*outerDimsPerm=*/SmallVector<int64_t>{});
 
-  Value newExpandOp = rewriter.create<tensor::ExpandShapeOp>(
+  Value newExpandOp = tensor::ExpandShapeOp::create(rewriter,
       packOp.getLoc(), packOp.getDestType(), packedVal, *reassocExpand);
   rewriter.replaceOp(packOp, newExpandOp);
 
@@ -972,14 +972,14 @@ static LogicalResult pushDownUnPackOpThroughExpandShape(
 
   RankedTensorType newExpandType = linalg::PackOp::inferPackedType(
       expandTy, innerTileSizes, projectedInnerDimsPos, newOuterDimsPerm);
-  auto newExpandOp = rewriter.create<tensor::ExpandShapeOp>(
+  auto newExpandOp = tensor::ExpandShapeOp::create(rewriter,
       expandOp.getLoc(), newExpandType, unPackOp.getSource(),
       newReassocIndices);
 
   auto emptyOp = linalg::UnPackOp::createDestinationTensor(
       rewriter, unPackOp.getLoc(), newExpandOp, unPackOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newUnPackOp = rewriter.create<linalg::UnPackOp>(
+  auto newUnPackOp = linalg::UnPackOp::create(rewriter,
       unPackOp.getLoc(), newExpandOp.getResult(), emptyOp,
       projectedInnerDimsPos, unPackOp.getMixedTiles(), newOuterDimsPerm);
   rewriter.replaceOp(expandOp, newUnPackOp);
@@ -1212,16 +1212,16 @@ struct PushDownUnPackThroughPadOp : public OpRewritePattern<tensor::PadOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, /*result=*/Type(), unpackOp.getSource(), lowPad, highPad,
         paddingVal, padOp.getNofold());
 
     // Inject the linalg.unpack right after the packed padOp.
-    Value outputUnPack = rewriter.create<tensor::EmptyOp>(
+    Value outputUnPack = tensor::EmptyOp::create(rewriter,
         loc, padOp.getResultType().getShape(),
         padOp.getResultType().getElementType());
 
-    Value replacement = rewriter.create<linalg::UnPackOp>(
+    Value replacement = linalg::UnPackOp::create(rewriter,
         loc, newPadOp.getResult(), outputUnPack, innerDimsPos,
         unpackOp.getMixedTiles(), outerDimsPerm);
     rewriter.replaceOp(padOp, replacement);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
index 692bf595267d4..82d386b3b9099 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
@@ -198,9 +198,9 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
         transposedShape[i] = inputRTType.getShape()[permutation[i]];
 
       Value emptyTensor =
-          rewriter.create<tensor::EmptyOp>(loc, transposedShape, elType);
+          tensor::EmptyOp::create(rewriter, loc, transposedShape, elType);
 
-      auto transposeOp = rewriter.create<TransposeOp>(loc, newInitValues[i],
+      auto transposeOp = TransposeOp::create(rewriter, loc, newInitValues[i],
                                                       emptyTensor, permutation);
       newInitValues[i] = transposeOp->getResult(0);
       isChanged = true;
@@ -209,10 +209,10 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
     // Does it require broadcast?
     if (!broadcastedDims.empty()) {
       assert(broadcastedDims.size() && "should have non size broadcast");
-      Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+      Value emptyTensor = tensor::EmptyOp::create(rewriter,
           loc, outputShape, inputRTType.getElementType());
 
-      auto broadcastOp = rewriter.create<linalg::BroadcastOp>(
+      auto broadcastOp = linalg::BroadcastOp::create(rewriter,
           loc, newInitValues[i], emptyTensor, broadcastedDims);
 
       newInitValues[i] = broadcastOp->getResult(0);
@@ -227,7 +227,7 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
   SmallVector<Value> operands = op->getOperands();
   ValueRange operandsRef(operands);
 
-  auto newOp = rewriter.create<linalg::GenericOp>(
+  auto newOp = linalg::GenericOp::create(rewriter,
       /*location=*/op.getLoc(),
       /*resultTensorTypes=*/op->getResultTypes(),
       /*inputs=*/newInitValues,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
index 1419175304899..bd3fad01aa48b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
@@ -133,12 +133,12 @@ static Value getZero(OpBuilder &b, Location loc, Type elementType) {
   assert(elementType.isIntOrIndexOrFloat() &&
          "expected scalar type while computing zero value");
   if (isa<IntegerType>(elementType))
-    return b.create<arith::ConstantIntOp>(loc, elementType, 0);
+    return arith::ConstantIntOp::create(b, loc, elementType, 0);
   if (elementType.isIndex())
-    return b.create<arith::ConstantIndexOp>(loc, 0);
+    return arith::ConstantIndexOp::create(b, loc, 0);
   // Assume float.
   auto floatType = cast<FloatType>(elementType);
-  return b.create<arith::ConstantFloatOp>(
+  return arith::ConstantFloatOp::create(b,
       loc, floatType, APFloat::getZero(floatType.getFloatSemantics()));
 }
 
@@ -189,7 +189,7 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
     // Fall back path, use an `init_tensor` and identity indexing map.
     AffineMap indexingMap = rewriter.getMultiDimIdentityMap(domain.size());
     Value emptyTensor =
-        rewriter.create<tensor::EmptyOp>(loc, domain, scalarOpResult.getType());
+        tensor::EmptyOp::create(rewriter, loc, domain, scalarOpResult.getType());
     newInitValues.push_back(emptyTensor);
     newResultTypes.push_back(emptyTensor.getType());
     peeledGenericOpIndexingMaps.push_back(indexingMap);
@@ -202,7 +202,7 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
   resultTypes.append(newResultTypes.begin(), newResultTypes.end());
   auto indexingMapAttr =
       rewriter.getAffineMapArrayAttr(peeledGenericOpIndexingMaps);
-  return rewriter.create<GenericOp>(
+  return GenericOp::create(rewriter,
       loc, resultTypes, genericOp.getInputs(), outsOperands, indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
       [](OpBuilder, Location, ValueRange) {});
@@ -239,7 +239,7 @@ DecomposeLinalgOp::createResidualGenericOp(GenericOp genericOp,
     indexingMaps.push_back(genericOp.getMatchingIndexingMap(&outOperand));
 
   auto indexingMapAttr = rewriter.getAffineMapArrayAttr(indexingMaps);
-  return rewriter.create<GenericOp>(
+  return GenericOp::create(rewriter,
       genericOp->getLoc(), genericOp->getResultTypes(),
       residualGenericOpOperands, genericOp.getOutputs(), indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
@@ -324,7 +324,7 @@ DecomposeLinalgOp::matchAndRewrite(GenericOp genericOp,
     yieldedVals.append(llvm::to_vector(
         llvm::map_range(peeledScalarOperation->getResults(),
                         [](OpResult opr) -> Value { return opr; })));
-    rewriter.create<YieldOp>(genericOp.getLoc(), yieldedVals);
+    YieldOp::create(rewriter, genericOp.getLoc(), yieldedVals);
   }
 
   /// In the split operations, replace block arguments uses that refer to
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index ef24eb881d68b..b8e73bd5929dd 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -34,7 +34,7 @@ static Value sourceMaterializationCallback(OpBuilder &builder, Type type,
 
   // A detensored value is converted back by creating a new tensor from its
   // element(s).
-  return builder.create<tensor::FromElementsOp>(
+  return tensor::FromElementsOp::create(builder,
       loc, RankedTensorType::get({}, inputType), inputs[0]);
 }
 
@@ -147,7 +147,7 @@ class DetensorizeTypeConverter : public TypeConverter {
     // A tensor value is detensoried by extracting its element(s).
     addTargetMaterialization([](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      return builder.create<tensor::ExtractOp>(loc, inputs[0], ValueRange{});
+      return tensor::ExtractOp::create(builder, loc, inputs[0], ValueRange{});
     });
 
     addSourceMaterialization(sourceMaterializationCallback);
@@ -481,7 +481,7 @@ struct LinalgDetensorize
         rewriter.splitBlock(entryBlock, entryBlock->begin());
     rewriter.setInsertionPointToStart(entryBlock);
     auto branch =
-        rewriter.create<cf::BranchOp>(rewriter.getUnknownLoc(), postEntryBlock);
+        cf::BranchOp::create(rewriter, rewriter.getUnknownLoc(), postEntryBlock);
 
     if (aggressiveMode.getValue()) {
       AggressiveDetensoringModel costModel;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index e0062d15e61ca..fd8c177cc7247 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -118,14 +118,14 @@ struct MoveInitOperandsToInput : public OpRewritePattern<GenericOp> {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointAfterValue(op->get());
       auto elemType = cast<ShapedType>(op->get().getType()).getElementType();
-      auto empty = rewriter.create<tensor::EmptyOp>(
+      auto empty = tensor::EmptyOp::create(rewriter,
           loc, tensor::getMixedSizes(rewriter, loc, op->get()), elemType);
 
       unsigned start = genericOp.getDpsInits().getBeginOperandIndex();
       newOutputOperands[op->getOperandNumber() - start] = empty.getResult();
     }
 
-    auto newOp = rewriter.create<GenericOp>(
+    auto newOp = GenericOp::create(rewriter,
         loc, genericOp.getResultTypes(), newInputOperands, newOutputOperands,
         newIndexingMaps, genericOp.getIteratorTypesArray(),
         /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
@@ -295,7 +295,7 @@ static Value collapseValue(
     MemRefLayoutAttrInterface layout;
     auto targetType = MemRefType::get(targetShape, memrefType.getElementType(),
                                       layout, memrefType.getMemorySpace());
-    return rewriter.create<memref::CollapseShapeOp>(loc, targetType, operand,
+    return memref::CollapseShapeOp::create(rewriter, loc, targetType, operand,
                                                     reassociation);
   }
   if (auto tensorType = dyn_cast<RankedTensorType>(operand.getType())) {
@@ -314,7 +314,7 @@ static Value collapseValue(
         "unknown rank reduction strategy");
     auto targetType =
         RankedTensorType::get(targetShape, tensorType.getElementType());
-    return rewriter.create<tensor::CollapseShapeOp>(loc, targetType, operand,
+    return tensor::CollapseShapeOp::create(rewriter, loc, targetType, operand,
                                                     reassociation);
   }
   llvm_unreachable("unsupported operand type");
@@ -519,7 +519,7 @@ linalg::dropUnitDims(RewriterBase &rewriter, GenericOp genericOp,
   for (unsigned i : llvm::seq<unsigned>(0, genericOp.getNumResults()))
     resultTypes.push_back(newOutputs[i].getType());
   GenericOp replacementOp =
-      rewriter.create<GenericOp>(loc, resultTypes, newInputs, newOutputs,
+      GenericOp::create(rewriter, loc, resultTypes, newInputs, newOutputs,
                                  newIndexingMaps, newIteratorTypes);
   rewriter.inlineRegionBefore(genericOp.getRegion(), replacementOp.getRegion(),
                               replacementOp.getRegion().begin());
@@ -652,7 +652,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         collapseValue(rewriter, padOp.getLoc(), padOp.getSource(), newShape,
                       reassociationMap, options.rankReductionStrategy);
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         padOp.getLoc(), /*result=*/Type(), collapsedSource, newLowPad,
         newHighPad, paddingVal, padOp.getNofold());
 
@@ -670,7 +670,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         expandedSizes.push_back(tensor::getMixedSize(
             rewriter, padOp.getLoc(), newPadOp, dim - numUnitDims));
       }
-      dest = rewriter.create<tensor::EmptyOp>(
+      dest = tensor::EmptyOp::create(rewriter,
           padOp.getLoc(), expandedSizes,
           padOp.getResultType().getElementType());
     }
@@ -713,7 +713,7 @@ struct RankReducedExtractSliceOp
             strides));
 
     Location loc = sliceOp.getLoc();
-    Value newSlice = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSlice = tensor::ExtractSliceOp::create(rewriter,
         loc, rankReducedType, sliceOp.getSource(), offsets, sizes, strides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSlice, *reassociation);
@@ -747,7 +747,7 @@ struct RankReducedInsertSliceOp : public OpRewritePattern<InsertOpTy> {
       // parallel case.
       if (std::is_same<InsertOpTy, tensor::ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      reshapedSource = rewriter.create<tensor::CollapseShapeOp>(
+      reshapedSource = tensor::CollapseShapeOp::create(rewriter,
           loc, insertSliceOp.getSource(), *reassociation);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
@@ -898,7 +898,7 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
   /// Expand result tensor.
   Value expandResult(PatternRewriter &rewriter, Value result,
                      RankedTensorType expandedType, int64_t dim) const {
-    return rewriter.create<tensor::ExpandShapeOp>(
+    return tensor::ExpandShapeOp::create(rewriter,
         result.getLoc(), expandedType, result,
         getReassociationForReshapeAtDim(expandedType.getRank(), dim));
   }
@@ -934,7 +934,7 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
     SmallVector<Type, 1> collapsedResultTy;
     if (isa<RankedTensorType>(collapsedInit.getType()))
       collapsedResultTy.push_back(collapsedInit.getType());
-    auto collapsedOp = rewriter.create<ToOpTy>(
+    auto collapsedOp = ToOpTy::create(rewriter,
         loc, collapsedResultTy, ValueRange{collapsedLhs, collapsedRhs},
         ValueRange{collapsedInit});
     for (auto attr : contractionOp->getAttrs()) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index 9c0f6e5d6469e..754f7652a776b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -236,11 +236,11 @@ static void generateFusedElementwiseOpRegion(
     fusedIndices.reserve(numFusedOpLoops);
     llvm::transform(llvm::seq<uint64_t>(0, numFusedOpLoops),
                     std::back_inserter(fusedIndices), [&](uint64_t dim) {
-                      return rewriter.create<IndexOp>(producer.getLoc(), dim);
+                      return IndexOp::create(rewriter, producer.getLoc(), dim);
                     });
     for (IndexOp indexOp :
          llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
-      Value newIndex = rewriter.create<affine::AffineApplyOp>(
+      Value newIndex = affine::AffineApplyOp::create(rewriter,
           producer.getLoc(),
           consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
       mapper.map(indexOp.getResult(), newIndex);
@@ -327,7 +327,7 @@ static void generateFusedElementwiseOpRegion(
   }
   for (auto consumerYieldVal : consumerYieldOp.getOperands())
     fusedYieldValues.push_back(mapper.lookupOrDefault(consumerYieldVal));
-  rewriter.create<YieldOp>(fusedOp.getLoc(), fusedYieldValues);
+  YieldOp::create(rewriter, fusedOp.getLoc(), fusedYieldValues);
 
   // Sanity checks.
   assert(fusedBlock->getNumArguments() == fusedOp.getNumOperands() &&
@@ -416,7 +416,7 @@ mlir::linalg::fuseElementwiseOps(RewriterBase &rewriter,
   }
 
   // Generate the fused op.
-  auto fusedOp = rewriter.create<GenericOp>(
+  auto fusedOp = GenericOp::create(rewriter,
       consumer.getLoc(), fusedResultTypes, fusedInputOperands,
       fusedOutputOperands, rewriter.getAffineMapArrayAttr(fusedIndexMaps),
       consumer.getIteratorTypes(),
@@ -750,9 +750,9 @@ static void updateExpandedGenericOpRegion(PatternRewriter &rewriter,
     expandedIndices.reserve(expandedDims.size() - 1);
     llvm::transform(
         expandedDims.drop_front(), std::back_inserter(expandedIndices),
-        [&](int64_t dim) { return rewriter.create<IndexOp>(loc, dim); });
+        [&](int64_t dim) { return IndexOp::create(rewriter, loc, dim); });
     OpFoldResult newIndex =
-        rewriter.create<IndexOp>(loc, expandedDims.front()).getResult();
+        IndexOp::create(rewriter, loc, expandedDims.front()).getResult();
     for (auto [expandedShape, expandedIndex] :
          llvm::zip(expandedDimsShape, expandedIndices)) {
       AffineExpr idx, acc, shape;
@@ -796,7 +796,7 @@ static Operation *createExpandedTransposeOp(PatternRewriter &rewriter,
       newPerm.push_back(dim);
     }
   }
-  return rewriter.create<TransposeOp>(transposeOp.getLoc(), expandedInput,
+  return TransposeOp::create(rewriter, transposeOp.getLoc(), expandedInput,
                                       output, invertPermutationVector(newPerm));
 }
 
@@ -813,7 +813,7 @@ static Operation *createExpandedGenericOp(
     for (auto j : expansionInfo.getExpandedDims(i))
       iteratorTypes[j] = type;
 
-  Operation *fused = rewriter.create<GenericOp>(
+  Operation *fused = GenericOp::create(rewriter,
       linalgOp.getLoc(), resultTypes, expandedOpOperands, outputs,
       expandedOpIndexingMaps, iteratorTypes);
 
@@ -933,7 +933,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
                 reassociation,
                 /*isExpandingReshape=*/true)))
           return std::nullopt;
-        expandedOpOperands.push_back(rewriter.create<tensor::ExpandShapeOp>(
+        expandedOpOperands.push_back(tensor::ExpandShapeOp::create(rewriter,
             loc, expandedOperandType, opOperand->get(), reassociation,
             expandedOperandShape));
         continue;
@@ -961,7 +961,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               reassociation,
               /*isExpandingReshape=*/true)))
         return std::nullopt;
-      outputs.push_back(rewriter.create<tensor::ExpandShapeOp>(
+      outputs.push_back(tensor::ExpandShapeOp::create(rewriter,
           loc, expandedOutputType, opOperand.get(), reassociation,
           expandedOutputShape));
     } else {
@@ -984,7 +984,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               linalgOp.getMatchingIndexingMap(
                   linalgOp.getDpsInitOperand(resultNumber)),
               expansionInfo);
-      resultVals.push_back(rewriter.create<tensor::CollapseShapeOp>(
+      resultVals.push_back(tensor::CollapseShapeOp::create(rewriter,
           linalgOp.getLoc(), opResult.getType(),
           fusedOp->getResult(resultNumber), reassociation));
     } else {
@@ -1086,7 +1086,7 @@ class FoldPadWithProducerReshapeOpByExpansion
 
     Location loc = padOp->getLoc();
     RankedTensorType expandedPaddedType = paddedType.clone(expandedPaddedShape);
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
@@ -1603,7 +1603,7 @@ static void generateCollapsedIndexingRegion(
        enumerate(collapsingInfo.getCollapsedOpToOrigOpMapping())) {
     ReassociationIndicesRef foldedDimsRef(foldedDims.value());
     Value newIndexVal =
-        rewriter.create<linalg::IndexOp>(loc, foldedDims.index());
+        linalg::IndexOp::create(rewriter, loc, foldedDims.index());
     for (auto dim : llvm::reverse(foldedDimsRef.drop_front())) {
       Value loopDim =
           getValueOrCreateConstantIndexOp(rewriter, loc, loopRange[dim]);
@@ -1687,7 +1687,7 @@ GenericOp cloneToCollapsedOp<GenericOp>(RewriterBase &rewriter,
   SmallVector<utils::IteratorType> iteratorTypes(getCollapsedOpIteratorTypes(
       origOp.getIteratorTypesArray(), collapsingInfo));
 
-  GenericOp collapsedOp = rewriter.create<linalg::GenericOp>(
+  GenericOp collapsedOp = linalg::GenericOp::create(rewriter,
       origOp.getLoc(), resultTypes, inputOperands, outputOperands, indexingMaps,
       iteratorTypes, [](OpBuilder &builder, Location loc, ValueRange args) {});
   Block *origOpBlock = &origOp->getRegion(0).front();
@@ -1794,11 +1794,11 @@ FailureOr<CollapseResult> mlir::linalg::collapseOpIterationDims(
       if (isa<MemRefType>(collapsedOpResult.getType())) {
         MemRefType expandShapeResultType = MemRefType::get(
             originalResultType.getShape(), originalResultType.getElementType());
-        result = rewriter.create<memref::ExpandShapeOp>(
+        result = memref::ExpandShapeOp::create(rewriter,
             loc, expandShapeResultType, collapsedOpResult, reassociation,
             resultShape);
       } else {
-        result = rewriter.create<tensor::ExpandShapeOp>(
+        result = tensor::ExpandShapeOp::create(rewriter,
             loc, originalResultType, collapsedOpResult, reassociation,
             resultShape);
       }
@@ -1982,7 +1982,7 @@ class FoldPadWithProducerReshapeOpByCollapsing
 
     RankedTensorType collapsedPaddedType =
         paddedType.clone(collapsedPaddedShape);
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
@@ -2117,10 +2117,10 @@ class FoldScalarOrSplatConstant : public OpRewritePattern<GenericOp> {
 
       // Create a constant scalar value from the splat constant.
       Value scalarConstant =
-          rewriter.create<arith::ConstantOp>(def->getLoc(), constantAttr);
+          arith::ConstantOp::create(rewriter, def->getLoc(), constantAttr);
 
       SmallVector<Value> outputOperands = genericOp.getOutputs();
-      auto fusedOp = rewriter.create<GenericOp>(
+      auto fusedOp = GenericOp::create(rewriter,
           rewriter.getFusedLoc(fusedLocs), genericOp->getResultTypes(),
           /*inputs=*/fusedOperands,
           /*outputs=*/outputOperands,
@@ -2183,7 +2183,7 @@ struct RemoveOutsDependency : public OpRewritePattern<GenericOp> {
         modifiedOutput = true;
         SmallVector<OpFoldResult> mixedSizes =
             tensor::getMixedSizes(rewriter, loc, operandVal);
-        Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+        Value emptyTensor = tensor::EmptyOp::create(rewriter,
             loc, mixedSizes, operandType.getElementType());
         op->setOperand(opOperand.getOperandNumber(), emptyTensor);
       }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index c4af09ca01421..d93f8c26a9f0a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -64,7 +64,7 @@ getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) {
       continue;
 
     // Extract static / dynamic shape mix from the first operand.
-    res.push_back(b.create<tensor::EmptyOp>(
+    res.push_back(tensor::EmptyOp::create(b,
         loc, tensor::getMixedSizes(b, loc, operands.front()),
         cast<RankedTensorType>(t).getElementType()));
   }
@@ -104,7 +104,7 @@ struct ConvertAnyElementwiseMappableOpOnRankedTensors : public RewritePattern {
               builder.create(loc, op->getName().getIdentifier(),
                              regionArgs.take_front(op->getNumOperands()),
                              resultTypes, op->getAttrs());
-          builder.create<linalg::YieldOp>(loc, scalarOp->getResults());
+          linalg::YieldOp::create(builder, loc, scalarOp->getResults());
         });
     return success();
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
index d375878fb2c91..4e95aae86e670 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
@@ -259,7 +259,7 @@ mlir::linalg::deduplicateOperandsAndRemoveDeadResults(
   for (Value v : newOutputOperands)
     if (isa<TensorType>(v.getType()))
       newResultTypes.push_back(v.getType());
-  auto newOp = rewriter.create<GenericOp>(
+  auto newOp = GenericOp::create(rewriter,
       loc, newResultTypes, newInputOperands, newOutputOperands,
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       genericOp.getIteratorTypes(), genericOp.getDocAttr(),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
index 44469bc404a7c..e19e224c25028 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
@@ -72,14 +72,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
     // Create the tensor of same size as output of the pad op.
     RankedTensorType padResultType = padOp.getResultType();
     auto resultSizes = resultShape[0];
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultSizes, padResultType.getElementType());
 
     // Fill the tensor with the pad value.
     // TODO: There is an option to fill only the boundaries. For now just
     // filling the whole tensor.
     auto fillTensor =
-        rewriter.create<linalg::FillOp>(loc, padValue, emptyTensor.getResult());
+        linalg::FillOp::create(rewriter, loc, padValue, emptyTensor.getResult());
 
     // Construct a slice of the fill result that is to be replaced with the
     // result of the generic op. The low pad values are the offsets, the size of
@@ -93,14 +93,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
          llvm::enumerate(cast<RankedTensorType>(source.getType()).getShape())) {
       if (ShapedType::isDynamic(shape.value())) {
         sizes.push_back(
-            rewriter.create<tensor::DimOp>(loc, source, shape.index())
+            tensor::DimOp::create(rewriter, loc, source, shape.index())
                 .getResult());
       } else {
         sizes.push_back(rewriter.getIndexAttr(shape.value()));
       }
     }
     SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-    auto slice = rewriter.create<tensor::ExtractSliceOp>(
+    auto slice = tensor::ExtractSliceOp::create(rewriter,
         loc, fillTensor.getResult(0), offsets, sizes, strides);
 
     // Clone the generic op.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 9bc7be2623849..41252c68ffda9 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -277,7 +277,7 @@ mlir::linalg::fuseProducerOfTensor(OpBuilder &b, OpResult producerOpResult,
   // mismatches. Insert a `tensor.cast` op to propagate the transformation
   // invariant that types are compatible.
   if (consumerType != def.getType())
-    def = b.create<tensor::CastOp>(fusedProducer.getLoc(), consumerType, def);
+    def = tensor::CastOp::create(b, fusedProducer.getLoc(), consumerType, def);
   consumerOpOperand.set(def);
   return FusionInfo{cast<LinalgOp>(producerOpResult.getOwner()), fusedProducer};
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 05f2157b77aeb..7994af8834990 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -61,7 +61,7 @@ FailureOr<GenericOp> mlir::linalg::generalizeNamedOp(RewriterBase &rewriter,
   // All named ops have a region attached that can be inlined.
   assert(linalgOp->getNumRegions() == 1 &&
          "expect named op to have one region attached");
-  GenericOp genericOp = rewriter.create<GenericOp>(
+  GenericOp genericOp = GenericOp::create(rewriter,
       linalgOp.getLoc(), resultTypes, inputs, outputs, indexingMaps, iterators);
   rewriter.inlineRegionBefore(linalgOp->getRegion(0), genericOp.getRegion(),
                               genericOp.getRegion().begin());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index 94ed46442180c..86f24578376c1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -591,7 +591,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
 
     // Create a packing loop that takes `hoistedPackedTensor` as iteration
     // argument.
-    auto clonedForOp = rewriter.create<scf::ForOp>(
+    auto clonedForOp = scf::ForOp::create(rewriter,
         loc, bvm.lookupOrDefault(forOp.getLowerBound()),
         bvm.lookupOrDefault(forOp.getUpperBound()),
         bvm.lookupOrDefault(forOp.getStep()), hoistedPackedTensor);
@@ -640,10 +640,10 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   TransposeOp maybeTransposeOp;
   Value paddedTensor = bvm.lookup(opToHoist.getResult());
   if (!transposeVector.empty()) {
-    Value outputTensor = rewriter.create<tensor::ExtractSliceOp>(
+    Value outputTensor = tensor::ExtractSliceOp::create(rewriter,
         loc, transposedTensorType, hoistedPackedTensor, offsets, sizes,
         strides);
-    maybeTransposeOp = rewriter.create<linalg::TransposeOp>(
+    maybeTransposeOp = linalg::TransposeOp::create(rewriter,
         loc, paddedTensor, outputTensor, transposeVector);
     paddedTensor = maybeTransposeOp.getResult()[0];
   }
@@ -652,7 +652,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   if (nPackedLoops > 0) {
     // Step 4. Create InsertSliceOp at the innermost loop level, inserting an
     // optionally transposed padded slice into the packed tensor.
-    Value inserted = rewriter.create<tensor::InsertSliceOp>(
+    Value inserted = tensor::InsertSliceOp::create(rewriter,
         loc, paddedTensor, hoistedPackedTensor, offsets, sizes, strides);
 
     // Step 5. Iteratively pop the stack and propagate the yield.
@@ -660,7 +660,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
     for (Value iv : llvm::reverse(clonedLoopIvs)) {
       auto forOp = scf::getForInductionVarOwner(iv);
       rewriter.setInsertionPointToEnd(&forOp.getRegion().front());
-      rewriter.create<scf::YieldOp>(loc, valueToYield);
+      scf::YieldOp::create(rewriter, loc, valueToYield);
       valueToYield = forOp.getResult(0);
     }
   }
@@ -712,7 +712,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   rewriter.setInsertionPoint(outerLoop);
   SmallVector<Value> dynamicTensorSizes =
       analysis.getHoistedPackedTensorSizes(rewriter, loc);
-  auto emptyOp = rewriter.create<tensor::EmptyOp>(
+  auto emptyOp = tensor::EmptyOp::create(rewriter,
       loc, hoistedPackedTensorType.getShape(),
       hoistedPackedTensorType.getElementType(), dynamicTensorSizes);
 
@@ -840,7 +840,7 @@ padThroughLoopIterArg(RewriterBase &rewriter, Value paddedValueBeforeHoisting,
   {
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(forOp);
-    extracted = rewriter.create<tensor::ExtractSliceOp>(
+    extracted = tensor::ExtractSliceOp::create(rewriter,
         hoistedPackedTensor.getLoc(), hoistedPackedTensor,
         outerSliceOp.getMixedOffsets(), outerSliceOp.getMixedSizes(),
         outerSliceOp.getMixedStrides());
@@ -934,7 +934,7 @@ static Value replaceByPackingResult(RewriterBase &rewriter,
   // offsets = [maybe_leading_ivs, 0 .. 0].
   // sizes = [1 .. 1, transposedShape] (defined above).
   // strides = [1 .. 1] (defined above)
-  return rewriter.create<tensor::ExtractSliceOp>(
+  return tensor::ExtractSliceOp::create(rewriter,
       loc, transposedTensorType, hoistedPackedTensor, offsets,
       packingResult.sizes, packingResult.strides);
 }
@@ -982,9 +982,9 @@ FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(newResult.getDefiningOp());
     // Transpose the packed tensor back to the original storage order.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, paddedTensorType.getShape(), paddedTensorType.getElementType());
-    TransposeOp unTransposeOp = rewriter.create<linalg::TransposeOp>(
+    TransposeOp unTransposeOp = linalg::TransposeOp::create(rewriter,
         loc, newResult, emptyTensor, transposeVector);
     newResult = unTransposeOp.getResult()[0];
     transposeOps.push_back(unTransposeOp);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index f2e51c29f3241..5397fc572e558 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -53,7 +53,7 @@ static scf::ForOp replaceWithDifferentYield(RewriterBase &rewriter,
   assert(index < inits.size());
   inits[index] = newInitOperand;
 
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
       inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
index 1f3336d2bfbb9..a900215ccf014 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
@@ -60,7 +60,7 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
 
     Location loc = genericOp->getLoc();
     SmallVector<Value> outputOperands = genericOp.getOutputs();
-    auto newOp = rewriter.create<GenericOp>(
+    auto newOp = GenericOp::create(rewriter,
         loc, genericOp->getResultTypes(), newOperands, outputOperands,
         newIndexingMaps, genericOp.getIteratorTypesArray());
     rewriter.cloneRegionBefore(genericOp.getRegion(), newOp.getRegion(),
@@ -77,11 +77,11 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
       SmallVector<Value> indicesValues;
       for (auto idx : indices)
         indicesValues.emplace_back(
-            rewriter.create<arith::ConstantIndexOp>(loc, idx));
+            arith::ConstantIndexOp::create(rewriter, loc, idx));
       Value scalarValue = opOperand->get();
       if (isa<RankedTensorType>(scalarValue.getType())) {
         scalarValue =
-            rewriter.create<tensor::ExtractOp>(loc, scalarValue, indicesValues);
+            tensor::ExtractOp::create(rewriter, loc, scalarValue, indicesValues);
       }
       body->getArgument(idx).replaceAllUsesWith(scalarValue);
       body->eraseArgument(idx);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index a92a0c83e0316..2372313e29c58 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -88,7 +88,7 @@ mlir::linalg::interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp,
       allIndices.reserve(genericOp.getNumLoops());
       llvm::transform(llvm::seq<uint64_t>(0, genericOp.getNumLoops()),
                       std::back_inserter(allIndices), [&](uint64_t dim) {
-                        return rewriter.create<IndexOp>(indexOp->getLoc(), dim);
+                        return IndexOp::create(rewriter, indexOp->getLoc(), dim);
                       });
       rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
           indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 488041a43a2ef..0c08c1d8b997d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -49,7 +49,7 @@ static SmallVector<Value> makeCanonicalAffineApplies(OpBuilder &b, Location loc,
     auto exprMap = AffineMap::get(dims, map.getNumSymbols(), e);
     SmallVector<Value> operands(vals);
     affine::canonicalizeMapAndOperands(&exprMap, &operands);
-    res.push_back(b.create<affine::AffineApplyOp>(loc, exprMap, operands));
+    res.push_back(affine::AffineApplyOp::create(b, loc, exprMap, operands));
   }
   return res;
 }
@@ -70,7 +70,7 @@ static void inlineRegionAndEmitStore(OpBuilder &b, Location loc, OpType op,
   Operation *terminator = block.getTerminator();
   for (OpOperand &operand : terminator->getOpOperands()) {
     Value toStore = map.lookupOrDefault(operand.get());
-    b.create<StoreOpTy>(loc, toStore, outputBuffers[operand.getOperandNumber()],
+    StoreOpTy::create(b, loc, toStore, outputBuffers[operand.getOperandNumber()],
                         indexing[operand.getOperandNumber()]);
   }
 }
@@ -145,7 +145,7 @@ static void emitScalarImplementation(OpBuilder &b, Location loc,
     auto indexing = makeCanonicalAffineApplies(
         b, loc, linalgOp.getMatchingIndexingMap(inputOperand), allIvsPlusDims);
     indexedValues.push_back(
-        b.create<LoadOpTy>(loc, inputOperand->get(), indexing));
+        LoadOpTy::create(b, loc, inputOperand->get(), indexing));
   }
   // 1.b. Emit load from output views.
   for (OpOperand &outputOperand : linalgOp.getDpsInitsMutable()) {
@@ -153,7 +153,7 @@ static void emitScalarImplementation(OpBuilder &b, Location loc,
         b, loc, linalgOp.getMatchingIndexingMap(&outputOperand),
         allIvsPlusDims);
     indexedValues.push_back(
-        b.create<LoadOpTy>(loc, outputOperand.get(), indexing));
+        LoadOpTy::create(b, loc, outputOperand.get(), indexing));
   }
 
   // TODO: When a region inliner exists, use it.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
index ee1957aaa6a53..28b246714df53 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
@@ -131,17 +131,17 @@ static Value createDestinationPassingStyleInitOperand(
     ImplicitLocOpBuilder &builder) {
   Value processLinearIndexInReductionGroup = mesh::createProcessLinearIndex(
       meshOp.getSymName(), reductionMeshAxes, builder);
-  Value zero = builder.create<arith::ConstantIndexOp>(0);
-  Value isLeadProcess = builder.create<arith::CmpIOp>(
+  Value zero = arith::ConstantIndexOp::create(builder, 0);
+  Value isLeadProcess = arith::CmpIOp::create(builder,
       builder.getI1Type(), arith::CmpIPredicate::eq,
       processLinearIndexInReductionGroup, zero);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(spmdizedOperand.getType(),
+  scf::IfOp ifOp = scf::IfOp::create(builder, spmdizedOperand.getType(),
                                              isLeadProcess, true, true);
   // Then block.
   {
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPointToEnd(&ifOp.getThenRegion().front());
-    builder.create<scf::YieldOp>(spmdizedOperand);
+    scf::YieldOp::create(builder, spmdizedOperand);
   }
 
   // Else block.
@@ -157,14 +157,14 @@ static Value createDestinationPassingStyleInitOperand(
     std::optional<TypedAttr> neutralEl =
         arith::getNeutralElement(combinerOps[0]);
 
-    Value init = builder.create<tensor::EmptyOp>(op.getLoc(), shape,
+    Value init = tensor::EmptyOp::create(builder, op.getLoc(), shape,
                                                  neutralEl.value().getType());
     Value constant =
-        builder.create<arith::ConstantOp>(op.getLoc(), neutralEl.value());
-    Value fill = builder.create<linalg::FillOp>(op.getLoc(), constant, init)
+        arith::ConstantOp::create(builder, op.getLoc(), neutralEl.value());
+    Value fill = linalg::FillOp::create(builder, op.getLoc(), constant, init)
                      .getResult(0);
 
-    builder.create<scf::YieldOp>(fill);
+    scf::YieldOp::create(builder, fill);
   }
   return ifOp.getResult(0);
 }
@@ -202,7 +202,7 @@ static void createAllReduceForResultWithoutPartialSharding(
   }
 
   Value spmdizedLinalgOpResult = spmdizationMap.lookup(unshardedLinalgOpResult);
-  Value reducedValue = builder.create<mesh::AllReduceOp>(
+  Value reducedValue = mesh::AllReduceOp::create(builder,
       spmdizedLinalgOpResult, resultSharding.getMesh(), allReduceMeshAxes,
       reductionKind);
   spmdizationMap.map(unshardedLinalgOpResult, reducedValue);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
index bb1e974391878..019c845b4e0e7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
@@ -59,7 +59,7 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
   auto newKernelTy = RankedTensorType::get(
       {kernelTy.getDimSize(0), kernelTy.getDimSize(1), kernelTy.getDimSize(2)},
       kernelTy.getElementType());
-  auto collapsedKernel = rewriter.create<tensor::CollapseShapeOp>(
+  auto collapsedKernel = tensor::CollapseShapeOp::create(rewriter,
       loc, newKernelTy, kernel, collapsedKernelDims);
 
   // Collapse init dims.
@@ -70,7 +70,7 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       RankedTensorType::get({initTy.getDimSize(0), initTy.getDimSize(1),
                              initTy.getDimSize(2), initTy.getDimSize(3)},
                             initTy.getElementType());
-  auto collapsedInit = rewriter.create<tensor::CollapseShapeOp>(
+  auto collapsedInit = tensor::CollapseShapeOp::create(rewriter,
       loc, newInitTy, init, collapsedInitDims);
 
   SmallVector<NamedAttribute> preservedAttrs;
@@ -78,13 +78,13 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       TypeSwitch<Operation *, Operation *>(operation)
           .Case<DepthwiseConv2DNhwcHwcmOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return rewriter.create<DepthwiseConv2DNhwcHwcOp>(
+            return DepthwiseConv2DNhwcHwcOp::create(rewriter,
                 loc, newInitTy, ValueRange{input, collapsedKernel},
                 ValueRange{collapsedInit}, stride, dilation);
           })
           .Case<DepthwiseConv2DNhwcHwcmQOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return rewriter.create<DepthwiseConv2DNhwcHwcQOp>(
+            return DepthwiseConv2DNhwcHwcQOp::create(rewriter,
                 loc, newInitTy, ValueRange{input, collapsedKernel, iZp, kZp},
                 ValueRange{collapsedInit}, stride, dilation);
           })
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
index 2afa2f9b71c2a..1368bb7e77db0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
@@ -143,7 +143,7 @@ struct SimplifyUnPackToCollapseShape : public OpRewritePattern<UnPackOp> {
                        Type newOperandType, ArrayAttr reassociation) const {
     if (operand.getType() == newOperandType)
       return operand;
-    return rewriter.create<tensor::CollapseShapeOp>(loc, newOperandType,
+    return tensor::CollapseShapeOp::create(rewriter, loc, newOperandType,
                                                     operand, reassociation);
   }
 
@@ -265,7 +265,7 @@ struct FoldUnpackWithExtractSliceOp
 
     // Create a new empty output tensor.
     Type elementType = unpackOp.getDestType().getElementType();
-    Value output = rewriter.create<tensor::EmptyOp>(
+    Value output = tensor::EmptyOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getMixedSizes(), elementType);
     rewriter.replaceOpWithNewOp<UnPackOp>(
         sliceOp, unpackOp.getSource(), output, unpackOp.getInnerDimsPos(),
@@ -529,7 +529,7 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
     auto elemType =
         cast<ShapedType>(unPackOp->getResultTypes()[0]).getElementType();
-    Value output = rewriter.create<tensor::EmptyOp>(
+    Value output = tensor::EmptyOp::create(rewriter,
         unPackOp->getLoc(), unpackOpResultDims[0], elemType);
 
     rewriter.replaceOpWithNewOp<UnPackOp>(
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index 5eb3761f7aca1..fb43c1f92e5b3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -192,10 +192,10 @@ static Value padOperand(RewriterBase &rewriter, TilingInterface opToPad,
   if (auto complexTy =
           dyn_cast<ComplexType>(getElementTypeOrSelf(v.getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingValueAttr);
-    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
                                                         complexTy, complexAttr);
   } else {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         opToPad.getLoc(), cast<TypedAttr>(paddingValueAttr));
   }
 
@@ -323,7 +323,7 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
         loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index dc9e11eccac4d..af614e88fb3d5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -219,10 +219,10 @@ static FailureOr<Value> padOperandToSmallestStaticBoundingBox(
   if (auto complexTy = dyn_cast<ComplexType>(
           getElementTypeOrSelf(opOperand->get().getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingAttr);
-    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
                                                         complexTy, complexAttr);
   } else {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         opToPad.getLoc(), cast<TypedAttr>(paddingAttr));
   }
 
@@ -313,7 +313,7 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
         loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index b274502e16903..4042a43b04091 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -63,10 +63,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
     staticBufferType =
         MemRefType::Builder(staticBufferType).setMemorySpace(memorySpaceAttr);
     if (options.useAlloca) {
-      return b.create<memref::AllocaOp>(staticBufferType, ValueRange{},
+      return memref::AllocaOp::create(b, staticBufferType, ValueRange{},
                                         alignmentAttr);
     }
-    return b.create<memref::AllocOp>(staticBufferType, ValueRange{},
+    return memref::AllocOp::create(b, staticBufferType, ValueRange{},
                                      alignmentAttr);
   }
 
@@ -76,10 +76,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
   dynamicBufferType =
       MemRefType::Builder(dynamicBufferType).setMemorySpace(memorySpaceAttr);
   Value mul = b.createOrFold<arith::MulIOp>(
-      b.create<arith::ConstantIndexOp>(width), allocSize);
+      arith::ConstantIndexOp::create(b, width), allocSize);
   if (options.useAlloca)
-    return b.create<memref::AllocaOp>(dynamicBufferType, mul, alignmentAttr);
-  return b.create<memref::AllocOp>(dynamicBufferType, mul, alignmentAttr);
+    return memref::AllocaOp::create(b, dynamicBufferType, mul, alignmentAttr);
+  return memref::AllocOp::create(b, dynamicBufferType, mul, alignmentAttr);
 }
 
 /// Default allocation callback function. This allocates a promoted buffer when
@@ -92,8 +92,8 @@ static std::optional<Value> defaultAllocBufferCallBack(
     std::optional<unsigned> alignment, DataLayout &layout) {
   ShapedType viewType = subView.getType();
   ImplicitLocOpBuilder b(subView.getLoc(), builder);
-  auto zero = b.create<arith::ConstantIndexOp>(0);
-  auto one = b.create<arith::ConstantIndexOp>(1);
+  auto zero = arith::ConstantIndexOp::create(b, 0);
+  auto one = arith::ConstantIndexOp::create(b, 1);
 
   Attribute memorySpaceAttr;
   if (options.memorySpace.has_value())
@@ -123,7 +123,7 @@ defaultDeallocBufferCallBack(const LinalgPromotionOptions &options,
                              OpBuilder &b, Value fullLocalView) {
   if (!options.useAlloca) {
     auto viewOp = cast<memref::ViewOp>(fullLocalView.getDefiningOp());
-    b.create<memref::DeallocOp>(viewOp.getSource().getLoc(),
+    memref::DeallocOp::create(b, viewOp.getSource().getLoc(),
                                 viewOp.getSource());
   }
   return success();
@@ -211,7 +211,7 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
   Location loc = linalgOp.getLoc();
   auto defaultCopyCallBack = [loc](OpBuilder &b, Value src,
                                    Value dst) -> LogicalResult {
-    b.create<linalg::CopyOp>(loc, src, dst);
+    linalg::CopyOp::create(b, loc, src, dst);
     return success();
   };
   copyInFn = (options.copyInFn ? *(options.copyInFn) : defaultCopyCallBack);
@@ -265,7 +265,7 @@ FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
               /*stopCondition=*/nullptr, /*closedUB=*/true);
       size = failed(upperBound)
                  ? getValueOrCreateConstantIndexOp(b, loc, rangeValue.size)
-                 : b.create<arith::ConstantIndexOp>(loc, *upperBound);
+                 : arith::ConstantIndexOp::create(b, loc, *upperBound);
     }
     LLVM_DEBUG(llvm::dbgs() << "Extracted tightest: " << size << "\n");
     fullSizes.push_back(size);
@@ -310,23 +310,23 @@ promoteSubViews(ImplicitLocOpBuilder &b,
     Value fillVal =
         llvm::TypeSwitch<Type, Value>(subviewEltType)
             .Case([&](FloatType t) {
-              return b.create<arith::ConstantOp>(FloatAttr::get(t, 0.0));
+              return arith::ConstantOp::create(b, FloatAttr::get(t, 0.0));
             })
             .Case([&](IntegerType t) {
-              return b.create<arith::ConstantOp>(IntegerAttr::get(t, 0));
+              return arith::ConstantOp::create(b, IntegerAttr::get(t, 0));
             })
             .Case([&](ComplexType t) {
               Value tmp;
               if (auto et = dyn_cast<FloatType>(t.getElementType()))
-                tmp = b.create<arith::ConstantOp>(FloatAttr::get(et, 0.0));
+                tmp = arith::ConstantOp::create(b, FloatAttr::get(et, 0.0));
               else if (auto et = cast<IntegerType>(t.getElementType()))
-                tmp = b.create<arith::ConstantOp>(IntegerAttr::get(et, 0));
-              return b.create<complex::CreateOp>(t, tmp, tmp);
+                tmp = arith::ConstantOp::create(b, IntegerAttr::get(et, 0));
+              return complex::CreateOp::create(b, t, tmp, tmp);
             })
             .Default([](auto) { return Value(); });
     if (!fillVal)
       return failure();
-    b.create<linalg::FillOp>(fillVal, promotionInfo->fullLocalView);
+    linalg::FillOp::create(b, fillVal, promotionInfo->fullLocalView);
   }
 
   // Copy data into the promoted buffers. Use callback if provided.
@@ -459,9 +459,9 @@ static std::optional<Value> allocateSubviewGPUMemoryInAddressSpace(
       gpu::AddressSpaceAttr::get(builder.getContext(), addressSpace));
   Value buffer;
   if (addressSpace == gpu::GPUDialect::getWorkgroupAddressSpace()) {
-    buffer = builder.create<memref::AllocOp>(funcOp.getLoc(), type);
+    buffer = memref::AllocOp::create(builder, funcOp.getLoc(), type);
   } else if (addressSpace == gpu::GPUDialect::getPrivateAddressSpace()) {
-    buffer = builder.create<memref::AllocaOp>(funcOp.getLoc(), type);
+    buffer = memref::AllocaOp::create(builder, funcOp.getLoc(), type);
   } else {
     return std::nullopt;
   }
@@ -487,9 +487,9 @@ LogicalResult mlir::linalg::deallocateWorkgroupMemory(OpBuilder &,
 /// the copy operation to ensure data integrity.
 LogicalResult mlir::linalg::copyToWorkgroupMemory(OpBuilder &b, Value src,
                                                   Value dst) {
-  b.create<gpu::BarrierOp>(src.getLoc());
-  Operation *copyOp = b.create<memref::CopyOp>(src.getLoc(), src, dst);
-  b.create<gpu::BarrierOp>(copyOp->getLoc());
+  gpu::BarrierOp::create(b, src.getLoc());
+  Operation *copyOp = memref::CopyOp::create(b, src.getLoc(), src, dst);
+  gpu::BarrierOp::create(b, copyOp->getLoc());
   return success();
 }
 
@@ -504,7 +504,7 @@ std::optional<Value> mlir::linalg::allocateGPUPrivateMemory(
 /// Normal copy to between src and dst.
 LogicalResult mlir::linalg::copyToGPUPrivateMemory(OpBuilder &b, Value src,
                                                    Value dst) {
-  b.create<memref::CopyOp>(src.getLoc(), src, dst);
+  memref::CopyOp::create(b, src.getLoc(), src, dst);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
index b30182dc84079..eac0e47b18a7d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
@@ -38,8 +38,8 @@ struct StructuredOpInterface
     SmallVector<Range> loopRanges = linalgOp.createLoopRanges(builder, loc);
     auto [starts, ends, _] = getOffsetsSizesAndStrides(loopRanges);
 
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    auto one = arith::ConstantIndexOp::create(builder, loc, 1);
 
     // Subtract one from the loop ends before composing with the indexing map
     transform(ends, ends.begin(), [&](OpFoldResult end) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
index 671dea8bb415f..1fac80142dddf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
@@ -52,7 +52,7 @@ createSplitPart(RewriterBase &b, Location loc, TilingInterface op,
       return nullptr;
     SmallVector<OpFoldResult> resultStrides(resultOffsets.size(),
                                             b.getIndexAttr(1));
-    Value inserted = b.create<tensor::InsertSliceOp>(
+    Value inserted = tensor::InsertSliceOp::create(b,
         loc, result, resultOperands[index], resultOffsets, resultSizes,
         resultStrides);
     results.push_back(inserted);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
index 5bfdbc6d0bb59..1b0bfbbec8624 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
@@ -115,7 +115,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
         newShape,
         cast<RankedTensorType>(operand->get().getType()).getElementType());
 
-    Value newInput = b.create<tensor::ExpandShapeOp>(
+    Value newInput = tensor::ExpandShapeOp::create(b,
         loc, newType, operand->get(), reassociation);
     newInputs.push_back(newInput);
   }
@@ -140,18 +140,18 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   Value emptyOrAllocTensor;
   if (useAlloc) {
-    emptyOrAllocTensor = b.create<bufferization::AllocTensorOp>(
+    emptyOrAllocTensor = bufferization::AllocTensorOp::create(b,
         loc,
         RankedTensorType::get(newOutputShape,
                               op.getRegionOutputArgs()[0].getType()),
         ValueRange{});
   } else {
-    emptyOrAllocTensor = b.create<tensor::EmptyOp>(
+    emptyOrAllocTensor = tensor::EmptyOp::create(b,
         loc, newOutputShape, op.getRegionOutputArgs()[0].getType());
   }
-  Value constantOp = b.create<arith::ConstantOp>(loc, *identity);
+  Value constantOp = arith::ConstantOp::create(b, loc, *identity);
   Value identityTensor =
-      b.create<linalg::FillOp>(op->getLoc(), constantOp, emptyOrAllocTensor)
+      linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor)
           .getResult(0);
 
   newMaps.push_back(AffineMap::get(oldOutputMap.getNumDims() + 1, 0, outputExpr,
@@ -168,7 +168,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   // Create the new op matching the original op with an extra parallel
   // dimension.
-  GenericOp genericOp = b.create<GenericOp>(
+  GenericOp genericOp = GenericOp::create(b,
       loc, TypeRange({emptyOrAllocTensor.getType()}), newInputs,
       ValueRange({identityTensor}), newMaps, newIteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
@@ -191,14 +191,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   AffineMap outputMap = AffineMap::get(intermRank, 0, exprs, op.getContext());
   SmallVector<AffineMap> reductionMaps = {inputMap, outputMap};
 
-  auto reduction = b.create<GenericOp>(
+  auto reduction = GenericOp::create(b,
       loc, op->getResultTypes(), ValueRange({genericOp.getResult(0)}),
       op.getDpsInits(), reductionMaps, reductionIteratorTypes,
       [reductionOp](OpBuilder &b, Location loc, ValueRange inputs) {
         Operation *clonedReductionOp = b.clone(*reductionOp);
         clonedReductionOp->setOperand(0, inputs[0]);
         clonedReductionOp->setOperand(1, inputs[1]);
-        b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+        linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
       });
   b.replaceOp(op, reduction.getResults());
 
@@ -318,14 +318,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
     Value emptyOrAllocTensor;
     if (useAlloc) {
       emptyOrAllocTensor =
-          b.create<bufferization::AllocTensorOp>(loc, newT, dims);
+          bufferization::AllocTensorOp::create(b, loc, newT, dims);
     } else {
-      emptyOrAllocTensor = b.create<tensor::EmptyOp>(loc, newT.getShape(),
+      emptyOrAllocTensor = tensor::EmptyOp::create(b, loc, newT.getShape(),
                                                      t.getElementType(), dims);
     }
-    Value constantOp = b.create<arith::ConstantOp>(loc, std::get<1>(it));
+    Value constantOp = arith::ConstantOp::create(b, loc, std::get<1>(it));
     fillOps.push_back(
-        b.create<linalg::FillOp>(op->getLoc(), constantOp, emptyOrAllocTensor));
+        linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor));
     newOutputs.push_back(fillOps.back().getResult(0));
     emptyOrAllocTensorOps.push_back(emptyOrAllocTensor.getDefiningOp());
   }
@@ -354,7 +354,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   SmallVector<Value> newInputs = op.getDpsInputs();
   // Add a single shape-only tensor to carry the dimensions without resorting to
   // more complex inversions.
-  newInputs.push_back(b.create<tensor::EmptyOp>(
+  newInputs.push_back(tensor::EmptyOp::create(b,
       loc, ArrayRef<int64_t>{reductionDimSize / splitFactor, splitFactor},
       b.getIntegerType(1)));
   // Output tensors are already good to go.
@@ -365,7 +365,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   iteratorTypes.insert(iteratorTypes.begin() + reductionDimPos,
                        utils::IteratorType::parallel);
   GenericOp genericOp =
-      b.create<GenericOp>(loc, ValueRange(newOutputs).getTypes(), newInputs,
+      GenericOp::create(b, loc, ValueRange(newOutputs).getTypes(), newInputs,
                           newOutputs, newMaps, iteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
                        genericOp.getRegion().begin());
@@ -396,7 +396,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
         utils::IteratorType::reduction;
 
     // clang-format off
-    auto reductionOp = b.create<GenericOp>(
+    auto reductionOp = GenericOp::create(b,
         loc,
         originalOutputType,
         reindexedOutput,
@@ -407,7 +407,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
           Operation *clonedReductionOp = b.clone(*combinerOp);
           clonedReductionOp->setOperand(0, bbArgs[0]);
           clonedReductionOp->setOperand(1, bbArgs[1]);
-          b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+          linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
         });
     // clang-format on
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
index d35aad514e884..854d3855d654d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
@@ -29,7 +29,7 @@ struct SwapExtractSliceOfFill final
     if (!fillOp || !fillOp->hasOneUse())
       return failure();
 
-    auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
         extractOp.getLoc(), extractOp.getType(), fillOp.getOutputs()[0],
         extractOp.getMixedOffsets(), extractOp.getMixedSizes(),
         extractOp.getMixedStrides());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 4741afe8a417d..c2803423fafd7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -94,10 +94,10 @@ static void emitIsPositiveIndexAssertion(ImplicitLocOpBuilder &b,
     return;
   }
 
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value condition = b.create<arith::CmpIOp>(arith::CmpIPredicate::sgt,
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value condition = arith::CmpIOp::create(b, arith::CmpIPredicate::sgt,
                                             cast<Value>(value), zero);
-  b.create<cf::AssertOp>(
+  cf::AssertOp::create(b,
       condition,
       b.getStringAttr("expected strictly positive tile size and divisor"));
 }
@@ -317,9 +317,9 @@ mlir::linalg::computeMultiTileSizes(OpBuilder &builder, LinalgOp op,
     Value coveredSize =
         apply(s0 * s1 + s2 * s3, {spec.lowTileSize, spec.lowTripCount,
                                   spec.highTileSize, spec.highTripCount});
-    Value equals = b.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+    Value equals = arith::CmpIOp::create(b, arith::CmpIPredicate::eq,
                                            coveredSize, tripCount);
-    b.create<cf::AssertOp>(
+    cf::AssertOp::create(b,
         equals, builder.getStringAttr(
                     "could not compute dynamic multi-size tile shapes"));
   }
@@ -656,7 +656,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       getValueOrCreateConstantIndexOp(b, loc, nonZeroNumThreads);
 
   // 2. Create the ForallOp with an empty region.
-  scf::ForallOp forallOp = b.create<scf::ForallOp>(
+  scf::ForallOp forallOp = scf::ForallOp::create(b,
       loc, getAsOpFoldResult(materializedNonZeroNumThreads), initTensors,
       mapping);
 
@@ -689,7 +689,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       sizes[reductionDim] = b.getIndexAttr(1);
       outOffsets[reductionDim] = forallOp.getInductionVars()[0];
       // TODO: use SubsetExtractOpInterface once it is available.
-      tiledDpsInitOperands.push_back(b.create<tensor::ExtractSliceOp>(
+      tiledDpsInitOperands.push_back(tensor::ExtractSliceOp::create(b,
           loc, cast<RankedTensorType>(initOperand.getType()),
           destBbArgs[destNum], outOffsets, sizes, strides));
     }
@@ -768,7 +768,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
     // 6.b. Parallel insertions are inserted at the end of the combining
     // terminator.
     b.setInsertionPointToEnd(forallOp.getTerminator().getBody());
-    b.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(b,
         loc, result, bbArg, resultOffsetsRank, resultSizesRank, strides);
   }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
index 513cecef29b61..94449fae260ee 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
@@ -44,7 +44,7 @@ static SmallVector<Value> getIndicesForAccess(OpBuilder &b, Location loc,
   for (auto result : indexingMap.getResults()) {
     AffineMap m = AffineMap::get(indexingMap.getNumDims(),
                                  indexingMap.getNumSymbols(), result);
-    Value v = b.create<affine::AffineApplyOp>(loc, m, ivs);
+    Value v = affine::AffineApplyOp::create(b, loc, m, ivs);
     indices.push_back(v);
   }
   return indices;
@@ -72,7 +72,7 @@ static LogicalResult inlinePayload(OpBuilder &b, LinalgOp linalgOp,
     OpOperand *storeInto = linalgOp.getDpsInitOperand(operand.index());
     auto indices = getIndicesForAccess(
         b, loc, linalgOp.getMatchingIndexingMap(storeInto), ivs);
-    b.create<memref::StoreOp>(
+    memref::StoreOp::create(b,
         loc, toStore, linalgOp.getDpsInitOperand(operand.index())->get(),
         indices);
   }
@@ -351,7 +351,7 @@ struct LinalgOpTilingInterface
       SmallVector<Value> indices = getIndicesForAccess(
           builder, linalgOpLoc, linalgOp.getMatchingIndexingMap(&operand), ivs);
       Value load =
-          builder.create<memref::LoadOp>(linalgOpLoc, operand.get(), indices);
+          memref::LoadOp::create(builder, linalgOpLoc, operand.get(), indices);
       indexedValues.push_back(load);
     }
 
@@ -519,10 +519,10 @@ struct LinalgOpPartialReductionInterface
 
       Type elType = getElementTypeOrSelf(result.getType());
       Value emptyTensor =
-          b.create<tensor::EmptyOp>(loc, partialResultShape, elType);
-      Value constantOp = b.create<arith::ConstantOp>(loc, *identity);
+          tensor::EmptyOp::create(b, loc, partialResultShape, elType);
+      Value constantOp = arith::ConstantOp::create(b, loc, *identity);
       auto identityTensor =
-          b.create<linalg::FillOp>(loc, constantOp, emptyTensor);
+          linalg::FillOp::create(b, loc, constantOp, emptyTensor);
       inits.push_back(identityTensor.getResult(0));
     }
 
@@ -574,7 +574,7 @@ struct LinalgOpPartialReductionInterface
       RankedTensorType sliceResultType = RankedTensorType::get(
           sliceInfo.resultShape, valueToTileType.getElementType(),
           valueToTileType.getEncoding());
-      auto sliceOp = b.create<tensor::ExtractSliceOp>(
+      auto sliceOp = tensor::ExtractSliceOp::create(b,
           loc, sliceResultType, valueToTile, sliceInfo.offsets, sliceInfo.sizes,
           sliceInfo.strides);
       tiledInits.push_back(sliceOp.getResult());
@@ -603,7 +603,7 @@ struct LinalgOpPartialReductionInterface
     auto resultTypes = ValueRange(tiledInits).getTypes();
     if (tilingStrategy ==
         ReductionTilingStrategy::PartialReductionOuterReduction) {
-      auto genericOp = b.create<GenericOp>(
+      auto genericOp = GenericOp::create(b,
           loc, resultTypes, tiledInputs, tiledInits, newMaps, newIteratorTypes);
       IRMapping mapping;
       op->getRegion(0).cloneInto(&genericOp.getRegion(),
@@ -648,7 +648,7 @@ struct LinalgOpPartialReductionInterface
         }
       }
 
-      auto reduction = b.create<linalg::ReduceOp>(
+      auto reduction = linalg::ReduceOp::create(b,
           loc, partialResult, init, partialReductionDims,
           [&linalgOp, &initIdx](OpBuilder &b, Location loc, ValueRange inputs) {
             // Get the combiner op.
@@ -659,7 +659,7 @@ struct LinalgOpPartialReductionInterface
             // Combine the input at idx and output at numInits + idx.
             clonedReductionOp->setOperand(0, inputs[0]);
             clonedReductionOp->setOperand(1, inputs[1]);
-            b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+            linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
           });
 
       mergeOperations.push_back(reduction);
@@ -790,7 +790,7 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto sourceSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getSource(), inputIndices, inputSizes, strides);
     tiledOperands.push_back(sourceSlice);
 
@@ -800,7 +800,7 @@ struct PackOpTiling
       return {};
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = b.create<tensor::ExtractSliceOp>(
+    auto outSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
@@ -809,7 +809,7 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = b.create<PackOp>(
+    Operation *tiledPackOp = PackOp::create(b,
         loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
@@ -968,7 +968,7 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto sourceSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getSource(), offsets, sizes, strides);
     tiledOperands.push_back(sourceSlice);
 
@@ -984,7 +984,7 @@ struct PackOpTiling
       return failure();
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = b.create<tensor::ExtractSliceOp>(
+    auto outSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
@@ -992,7 +992,7 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = b.create<PackOp>(
+    Operation *tiledPackOp = PackOp::create(b,
         loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
@@ -1172,7 +1172,7 @@ struct UnPackOpTiling
     sliceSrcSizes.append(unpackOp.getMixedTiles());
     sliceSrcStrides.append(numInnerTiles, oneAttr);
     SmallVector<Operation *> generatedSlices;
-    tensor::ExtractSliceOp sliceSource = b.create<tensor::ExtractSliceOp>(
+    tensor::ExtractSliceOp sliceSource = tensor::ExtractSliceOp::create(b,
         loc, unpackOp.getSource(), sliceSrcIndices, sliceSrcSizes,
         sliceSrcStrides);
     generatedSlices.push_back(sliceSource);
@@ -1180,12 +1180,12 @@ struct UnPackOpTiling
     SmallVector<OpFoldResult> destStrides(destRank, oneAttr);
     Value sliceDest;
     if (isPerfectTilingCase) {
-      auto destSliceOp = b.create<tensor::ExtractSliceOp>(
+      auto destSliceOp = tensor::ExtractSliceOp::create(b,
           loc, unpackOp.getDest(), offsets, sizes, destStrides);
       sliceDest = destSliceOp;
       generatedSlices.push_back(destSliceOp);
     } else {
-      sliceDest = b.create<tensor::EmptyOp>(
+      sliceDest = tensor::EmptyOp::create(b,
           loc, destExpandedSizes, unpackOp.getDestType().getElementType());
     }
 
@@ -1193,7 +1193,7 @@ struct UnPackOpTiling
     for (auto tile : unpackOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledUnpackOp = b.create<UnPackOp>(
+    Operation *tiledUnpackOp = UnPackOp::create(b,
         loc, TypeRange{sliceDest.getType()}, tiledOperands, op->getAttrs());
 
     if (isPerfectTilingCase)
@@ -1201,7 +1201,7 @@ struct UnPackOpTiling
                           SmallVector<Value>(tiledUnpackOp->getResults()),
                           generatedSlices};
 
-    auto extractSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractSlice = tensor::ExtractSliceOp::create(b,
         loc, tiledUnpackOp->getResult(0), resultOffsetsFromDest, sizes,
         destStrides);
     return TilingResult{
@@ -1337,13 +1337,13 @@ struct UnPackOpTiling
 
     SmallVector<Value> tiledOperands;
     // Create slice of the dest operand.
-    auto extractDestSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractDestSlice = tensor::ExtractSliceOp::create(b,
         loc, unPackOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(extractDestSlice);
 
     strides.append(unPackOp.getSourceRank() - outputRank, oneAttr);
     // Create slice of the source operand.
-    auto extractSourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractSourceSlice = tensor::ExtractSliceOp::create(b,
         loc, unPackOp.getSource(), offsets, sizes, strides);
     tiledOperands.insert(tiledOperands.begin(), extractSourceSlice);
     for (auto tile : unPackOp.getInnerTiles())
@@ -1351,7 +1351,7 @@ struct UnPackOpTiling
 
     // Create tiled unpack op.
     Operation *tiledUnPackOp =
-        b.create<UnPackOp>(loc, TypeRange{extractDestSlice.getType()},
+        UnPackOp::create(b, loc, TypeRange{extractDestSlice.getType()},
                            tiledOperands, op->getAttrs());
 
     return TilingResult{{tiledUnPackOp},
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index eab74dab4eb75..3526f62f770dc 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -269,11 +269,11 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       packingMetadata.reassociations);
   Value paddingValue = packOp.getPaddingValue();
   if (!paddingValue) {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(getElementTypeOrSelf(collapsed)));
   }
   auto padOp =
-      rewriter.create<tensor::PadOp>(loc, collapsed, packOp.getSource(), lows,
+      tensor::PadOp::create(rewriter, loc, collapsed, packOp.getSource(), lows,
                                      highs, paddingValue, /*nofold=*/false);
 
   LLVM_DEBUG(
@@ -313,7 +313,7 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       SmallVector<OpFoldResult> sizes =
           tensor::getMixedSizes(rewriter, loc, packOp.getDest());
 
-      auto insertSliceOp = rewriter.create<tensor::InsertSliceOp>(
+      auto insertSliceOp = tensor::InsertSliceOp::create(rewriter,
           loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
           /*offsets=*/zeros, sizes, /*strides=*/ones);
 
@@ -329,14 +329,14 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   // 5. Expand from the padded result to the stripMinedShape.
   auto expandShapeResultType =
       RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape);
-  auto reshapeOp = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapeOp = tensor::ExpandShapeOp::create(rewriter,
       loc, expandShapeResultType, padOp.getResult(),
       packingMetadata.reassociations);
 
   // 6. Transpose stripMinedShape to packedShape.
   SmallVector<int64_t> transpPerm =
       invertPermutationVector(packedToStripMinedShapePerm);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
 
   LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
@@ -371,7 +371,7 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
     SmallVector<OpFoldResult> sizes(packedRank - destShape.size(), one);
     sizes.append(tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()));
 
-    auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
         loc, destTensorType, unPackOp.getSource(),
         SmallVector<OpFoldResult>(packedRank, zero), sizes,
         SmallVector<OpFoldResult>(packedRank, one));
@@ -404,9 +404,9 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
   SmallVector<OpFoldResult, 4> dims =
       tensor::getMixedSizes(rewriter, loc, unPackOp.getSource());
   applyPermutationToVector(dims, packedToStripMinedShapePerm);
-  auto emptyOp = rewriter.create<tensor::EmptyOp>(
+  auto emptyOp = tensor::EmptyOp::create(rewriter,
       loc, dims, stripMinedTensorType.getElementType());
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm);
 
   LLVM_DEBUG(
@@ -426,20 +426,20 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
       DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
 
   // 4. Collapse from the stripMinedShape to the padded result.
-  auto reshapeOp = rewriter.create<tensor::CollapseShapeOp>(
+  auto reshapeOp = tensor::CollapseShapeOp::create(rewriter,
       loc, collapsedType, transposeOp->getResult(0),
       packingMetadata.reassociations);
 
   // 5. ExtractSlice.
   int64_t destRank = destTensorType.getRank();
-  auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+  auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
       loc, destTensorType, reshapeOp->getResult(0),
       SmallVector<OpFoldResult>(destRank, zero),
       tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()),
       SmallVector<OpFoldResult>(destRank, one));
 
   // 6. Inject a copy to preserve DPS.
-  auto copyOp = rewriter.create<linalg::CopyOp>(
+  auto copyOp = linalg::CopyOp::create(rewriter,
       loc, extractSliceOp->getResult(0), unPackOp.getDest());
 
   // 7. Replace unPackOp by copyOp.
@@ -554,15 +554,15 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
               operandType.getShape(), innerPos,
               cast<ShapedType>(dest.getType()).getShape(), {},
               innerPackSizes)) {
-        packOps.push_back(rewriter.create<linalg::PackOp>(
+        packOps.push_back(linalg::PackOp::create(rewriter,
             loc, operand, dest, innerPos, innerPackSizes));
       } else {
         // TODO: value of the padding attribute should be determined by
         // consumers.
         auto zeroAttr =
             rewriter.getZeroAttr(getElementTypeOrSelf(dest.getType()));
-        Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
-        packOps.push_back(rewriter.create<linalg::PackOp>(
+        Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
+        packOps.push_back(linalg::PackOp::create(rewriter,
             loc, operand, dest, innerPos, innerPackSizes, zero));
       }
       inputsAndInits.push_back(packOps.back());
@@ -574,7 +574,7 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       ValueRange{inputsAndInits}.take_front(linalgOp.getNumDpsInputs());
   ValueRange inits =
       ValueRange{inputsAndInits}.take_back(linalgOp.getNumDpsInits());
-  auto packedLinalgOp = rewriter.create<linalg::GenericOp>(
+  auto packedLinalgOp = linalg::GenericOp::create(rewriter,
       linalgOp.getLoc(), inits.getTypes(), inputs, inits, indexingMaps,
       iteratorTypes);
   packedLinalgOp.getRegion().takeBody(linalgOp->getRegion(0));
@@ -589,7 +589,7 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       continue;
     }
     // Build the symmetrical UnPackOp to the existing PackOp.
-    unPackOps.push_back(rewriter.create<linalg::UnPackOp>(
+    unPackOps.push_back(linalg::UnPackOp::create(rewriter,
         packedLinalgOp->getLoc(), result, maybePackedInit.getSource(),
         maybePackedInit.getInnerDimsPos(), maybePackedInit.getMixedTiles()));
     results.push_back(unPackOps.back());
@@ -655,7 +655,7 @@ static LinalgOp transposeOneLinalgOperandAndReplace(
   operands[opOperand.getOperandNumber()] = transposedValue;
 
   ValueRange operandsRef(operands);
-  auto transposedGenericOp = rewriter.create<linalg::GenericOp>(
+  auto transposedGenericOp = linalg::GenericOp::create(rewriter,
       /*location=*/linalgOp->getLoc(),
       /*resultTensorTypes=*/
       operandsRef.drop_front(linalgOp.getNumDpsInputs()).getTypes(),
@@ -904,7 +904,7 @@ mlir::linalg::LinalgTilingOptions::setTileSizes(ArrayRef<int64_t> ts) {
     b.setInsertionPointToStart(
         &op->getParentOfType<func::FuncOp>().getBody().front());
     return llvm::to_vector<4>(map_range(tileSizes, [&](int64_t s) {
-      Value v = b.create<arith::ConstantIndexOp>(op->getLoc(), s);
+      Value v = arith::ConstantIndexOp::create(b, op->getLoc(), s);
       return v;
     }));
   };
@@ -926,11 +926,11 @@ Value DecomposePadOpPattern::createFillOrGenerateOp(
     // Move the padding value defined inside the PadOp block to outside.
     if (padValue.getParentBlock() == &padOp.getRegion().front())
       rewriter.moveOpBefore(padValue.getDefiningOp(), padOp);
-    return rewriter.create<FillOp>(padOp.getLoc(), padValue, dest).result();
+    return FillOp::create(rewriter, padOp.getLoc(), padValue, dest).result();
   }
 
   // Fill could not be optimized: Lower to tensor::GenerateOp with region.
-  auto generateOp = rewriter.create<tensor::GenerateOp>(
+  auto generateOp = tensor::GenerateOp::create(rewriter,
       padOp.getLoc(), padOp.getResultType(), dynSizes);
   // Copy region to new op.
   IRMapping bvm;
@@ -970,7 +970,7 @@ DecomposePadOpPattern::matchAndRewrite(tensor::PadOp padOp,
   }
 
   // Init tensor and fill it with padding.
-  Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+  Value emptyTensor = tensor::EmptyOp::create(rewriter,
       padOp.getLoc(), staticSizes, resultType.getElementType(), dynSizes);
   Value fill = createFillOrGenerateOp(rewriter, padOp, emptyTensor, dynSizes);
 
@@ -1222,11 +1222,11 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
 
   applyPermutationToVector<OpFoldResult>(transShapeForEmptyOp,
                                          srcPermForTranspose);
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, transShapeForEmptyOp, packOp.getSourceType().getElementType());
 
   // 2.2 Create linalg.transpose
-  auto transposedOp = rewriter.create<linalg::TransposeOp>(loc, input, empty,
+  auto transposedOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
                                                            srcPermForTranspose);
 
   // 3. Insert the inner tile to the destination:
@@ -1246,7 +1246,7 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   }
 
   // 4. Replace tensor.packOp with tensor.insert_slice created above
-  auto insert = rewriter.create<tensor::InsertSliceOp>(
+  auto insert = tensor::InsertSliceOp::create(rewriter,
       loc, transposedOp.getResult()[0], packOp.getDest(), writeOffsets,
       writeSizes, writeStrides);
   rewriter.replaceOp(packOp, insert.getResult());
@@ -1313,7 +1313,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     // outer-untiled-dims
     if (ShapedType::isDynamic(srcShape[i])) {
       OpFoldResult dynamicDim =
-          rewriter.create<tensor::DimOp>(loc, source, i).getResult();
+          tensor::DimOp::create(rewriter, loc, source, i).getResult();
       extractSliceSizes.push_back(dynamicDim);
       shapeForEmptyOp.push_back(dynamicDim);
     } else {
@@ -1340,7 +1340,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   readShapeForExtractSlice.append(tileShape.begin(), tileShape.end());
   Type elemType = unpackOp.getSourceType().getElementType();
   auto readType = RankedTensorType::get(readShapeForExtractSlice, elemType);
-  Value innerTile = rewriter.create<tensor::ExtractSliceOp>(
+  Value innerTile = tensor::ExtractSliceOp::create(rewriter,
       loc, readType, unpackOp.getSource(), extractSliceOffsets,
       extractSliceSizes, extractSliceStrides);
 
@@ -1352,9 +1352,9 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   applyPermutationToVector<OpFoldResult>(shapeForEmptyOp, perm);
 
   Value empty =
-      rewriter.create<tensor::EmptyOp>(loc, shapeForEmptyOp, elemType);
+      tensor::EmptyOp::create(rewriter, loc, shapeForEmptyOp, elemType);
   auto transposedOp =
-      rewriter.create<linalg::TransposeOp>(loc, innerTile, empty, perm);
+      linalg::TransposeOp::create(rewriter, loc, innerTile, empty, perm);
 
   // 3. Handle in-complete tiles if needed. It truncates trailing data from the
   // transposed tile.
@@ -1369,7 +1369,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
           tensor::getMixedSize(rewriter, loc, unpackOp.getDest(), i));
   }
 
-  auto partialTile = rewriter.create<tensor::ExtractSliceOp>(
+  auto partialTile = tensor::ExtractSliceOp::create(rewriter,
       loc, transposedOp.getResult()[0], tileOffsets, tileSizes, tileStrides);
 
   // 4. Insert the result to the destination tensor.
@@ -1382,7 +1382,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     else
       writeSizes.push_back(oneIdxAttr);
   }
-  auto insert = rewriter.create<tensor::InsertSliceOp>(
+  auto insert = tensor::InsertSliceOp::create(rewriter,
       loc, partialTile, unpackOp.getDest(), writeOffsets, writeSizes,
       writeStrides);
   rewriter.replaceOp(unpackOp, insert.getResult());
@@ -1491,7 +1491,7 @@ FailureOr<Conv1DOp> DownscaleSizeOneWindowed2DConvolution<Conv2DOp, Conv1DOp>::
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = rewriter.create<Conv1DOp>(
+  auto conv1DOp = Conv1DOp::create(rewriter,
       loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
@@ -1578,7 +1578,7 @@ DownscaleDepthwiseConv2DNhwcHwcOp::returningMatchAndRewrite(
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = rewriter.create<DepthwiseConv1DNwcWcOp>(
+  auto conv1DOp = DepthwiseConv1DNwcWcOp::create(rewriter,
       loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
@@ -1635,7 +1635,7 @@ DownscaleConv2DOp::returningMatchAndRewrite(Conv2DOp convOp,
   Value newOutput = tensor::createCanonicalRankReducingExtractSliceOp(
       rewriter, loc, output, newOutputType);
 
-  auto conv1DOp = rewriter.create<Conv1DOp>(loc, newOutputType,
+  auto conv1DOp = Conv1DOp::create(rewriter, loc, newOutputType,
                                             ValueRange{newInput, newKernel},
                                             ValueRange{newOutput});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index 092aecceef6b3..b933a2d143925 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -67,7 +67,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
   Value input;
   if (isTensorOp) {
 
-    input = rewriter.create<tensor::EmptyOp>(loc, newFilterShape, elementTy)
+    input = tensor::EmptyOp::create(rewriter, loc, newFilterShape, elementTy)
                 .getResult();
   } else {
     input = rewriter
@@ -78,7 +78,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
 
   // We can then construct the transposition on our filter.
   auto transpose =
-      rewriter.create<linalg::TransposeOp>(loc, filter, input, filterPerm);
+      linalg::TransposeOp::create(rewriter, loc, filter, input, filterPerm);
 
   Value newFilter;
   if (isTensorOp) {
@@ -98,7 +98,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
     resultTy.push_back(op->getResult(0).getType());
   }
   auto newConv =
-      rewriter.create<HWCFConvOp>(loc, resultTy, newInputs, op.getOutputs(),
+      HWCFConvOp::create(rewriter, loc, resultTy, newInputs, op.getOutputs(),
                                   op.getStrides(), op.getDilations());
   rewriter.replaceOp(op, newConv);
   return newConv.getOperation();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index 934781d1cab75..6b1e89c1594d1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -47,24 +47,24 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
 
   SmallVector<Value> dynamicDims;
   if (type.isDynamicDim(1))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 1));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
   if (type.isDynamicDim(0))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, ArrayRef<int64_t>{shape[1], shape[0]}, type.getElementType(),
       dynamicDims);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, input, empty, ArrayRef<int64_t>{1, 0});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = rewriter.create<linalg::MatmulTransposeAOp>(
+    newMatmulOp = linalg::MatmulTransposeAOp::create(rewriter,
         loc, matmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), matmulOp.getInputs()[1]},
         matmulOp.getOutputs());
   } else {
-    newMatmulOp = rewriter.create<linalg::MatmulTransposeBOp>(
+    newMatmulOp = linalg::MatmulTransposeBOp::create(rewriter,
         loc, matmulOp.getResultTypes(),
         ValueRange{matmulOp.getInputs()[0], transposeOp->getResult(0)},
         matmulOp.getOutputs());
@@ -102,26 +102,26 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
 
   SmallVector<Value> dynamicDims;
   if (type.isDynamicDim(0))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
   if (type.isDynamicDim(2))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 2));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 2));
   if (type.isDynamicDim(1))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 1));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, ArrayRef<int64_t>{shape[0], shape[2], shape[1]},
       type.getElementType(), dynamicDims);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, input, empty, ArrayRef<int64_t>{0, 2, 1});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = rewriter.create<linalg::BatchMatmulTransposeAOp>(
+    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(rewriter,
         loc, batchMatmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), batchMatmulOp.getInputs()[1]},
         batchMatmulOp.getOutputs());
   } else {
-    newMatmulOp = rewriter.create<linalg::BatchMatmulTransposeBOp>(
+    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(rewriter,
         loc, batchMatmulOp.getResultTypes(),
         ValueRange{batchMatmulOp.getInputs()[0], transposeOp->getResult(0)},
         batchMatmulOp.getOutputs());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 5a8c5eab3f444..2fff28ffc7474 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -119,7 +119,7 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, input, /*offsets=*/ArrayRef<int64_t>{w + kw}, sizes, strides));
       }
     }
@@ -130,7 +130,7 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, input,
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             sizes, strides));
@@ -149,7 +149,7 @@ static SmallVector<Value> extractConvFilterSlices(RewriterBase &rewriter,
   // Extract rhs slice of size [{c, f} for channeled convolutions and {1} for
   // non-chanelled convolution] @ [kw].
   for (int64_t kw = 0; kw < kwSize; ++kw) {
-    result.push_back(rewriter.create<vector::ExtractOp>(
+    result.push_back(vector::ExtractOp::create(rewriter,
         loc, filter, /*offsets=*/ArrayRef<int64_t>{kw}));
   }
   return result;
@@ -167,7 +167,7 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {wSizeStep};
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/ArrayRef<int64_t>{w}, sizes, strides));
     }
   } else {
@@ -176,7 +176,7 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {nSize, wSizeStep, fSize};
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/ArrayRef<int64_t>{0, w, 0}, sizes, strides));
     }
   }
@@ -194,7 +194,7 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // This does not depend on kw.
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{w}, strides);
     }
   } else {
@@ -202,7 +202,7 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // convolution. This does not depend on kw.
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           strides);
     }
@@ -337,7 +337,7 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
   for (int vecDim = 0, end = canonicalVecShape.size(); vecDim < end; ++vecDim) {
     if (ShapedType::isStatic(iterSpaceStaticSizes[vecDim])) {
       // Create constant index op for static dimensions.
-      iterSpaceValueSizes.push_back(rewriter.create<arith::ConstantIndexOp>(
+      iterSpaceValueSizes.push_back(arith::ConstantIndexOp::create(rewriter,
           linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
       continue;
     }
@@ -351,9 +351,9 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
       return failure();
 
     Value dynamicDim = linalgOp.hasPureTensorSemantics()
-                           ? (Value)rewriter.create<tensor::DimOp>(
+                           ? (Value)tensor::DimOp::create(rewriter,
                                  linalgOp.getLoc(), operand, operandDimPos)
-                           : (Value)rewriter.create<memref::DimOp>(
+                           : (Value)memref::DimOp::create(rewriter,
                                  linalgOp.getLoc(), operand, operandDimPos);
     iterSpaceValueSizes.push_back(dynamicDim);
   }
@@ -474,7 +474,7 @@ Value VectorizationState::getOrCreateMaskFor(
          "Masked 0-d vectors are not supported yet");
 
   // Create the mask based on the dimension values.
-  Value mask = rewriter.create<vector::CreateMaskOp>(linalgOp.getLoc(),
+  Value mask = vector::CreateMaskOp::create(rewriter, linalgOp.getLoc(),
                                                      maskType, upperBounds);
   LDBG("Creating new mask: " << mask << "\n");
   activeMaskCache[maskingMap] = mask;
@@ -643,7 +643,7 @@ static Operation *buildMultiDimReduce(OpBuilder &b, Operation *reduceOp,
                                       ArrayRef<bool> dimsToMask) {
   auto maybeKind = getCombinerOpKind(reduceOp);
   assert(maybeKind && "Failed precondition: could not get reduction kind");
-  return b.create<vector::MultiDimReductionOp>(
+  return vector::MultiDimReductionOp::create(b,
       reduceOp->getLoc(), valueToReduce, acc, dimsToMask, *maybeKind);
 }
 
@@ -689,17 +689,17 @@ static Value buildVectorWrite(RewriterBase &rewriter, Value value,
   if (vectorType.getRank() > 0) {
     AffineMap writeMap = inversePermutation(reindexIndexingMap(opOperandMap));
     SmallVector<Value> indices(linalgOp.getRank(outputOperand),
-                               rewriter.create<arith::ConstantIndexOp>(loc, 0));
+                               arith::ConstantIndexOp::create(rewriter, loc, 0));
     value = broadcastIfNeeded(rewriter, value, vectorType);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = rewriter.create<vector::TransferWriteOp>(
+    write = vector::TransferWriteOp::create(rewriter,
         loc, value, outputOperand->get(), indices, writeMap);
   } else {
     // 0-d case is still special: do not invert the reindexing writeMap.
     if (!isa<VectorType>(value.getType()))
-      value = rewriter.create<vector::BroadcastOp>(loc, vectorType, value);
+      value = vector::BroadcastOp::create(rewriter, loc, vectorType, value);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = rewriter.create<vector::TransferWriteOp>(
+    write = vector::TransferWriteOp::create(rewriter,
         loc, value, outputOperand->get(), ValueRange{});
   }
 
@@ -778,7 +778,7 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto indexVectorType =
       VectorType::get({targetShape[dim]}, rewriter.getIndexType(),
                       state.getScalableVecDims()[dim]);
-  auto indexSteps = rewriter.create<vector::StepOp>(loc, indexVectorType);
+  auto indexSteps = vector::StepOp::create(rewriter, loc, indexVectorType);
   // Return the one-dimensional index vector if it lives in the trailing
   // dimension of the iteration space since the vectorization algorithm in this
   // case can handle the broadcast.
@@ -793,14 +793,14 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto permMap =
       AffineMap::getPermutationMap(permPattern, linalgOp.getContext());
 
-  auto broadCastOp = rewriter.create<vector::BroadcastOp>(
+  auto broadCastOp = vector::BroadcastOp::create(rewriter,
       loc, state.getCanonicalVecType(rewriter.getIndexType(), permMap),
       indexSteps);
   SmallVector<int64_t> transposition =
       llvm::to_vector<16>(llvm::seq<int64_t>(0, linalgOp.getNumLoops()));
   std::swap(transposition.back(), transposition[dim]);
   auto transposeOp =
-      rewriter.create<vector::TransposeOp>(loc, broadCastOp, transposition);
+      vector::TransposeOp::create(rewriter, loc, broadCastOp, transposition);
   return VectorizationHookResult{VectorizationHookStatus::NewOp, transposeOp};
 }
 
@@ -853,19 +853,19 @@ static Value calculateGatherOffset(RewriterBase &rewriter,
 
   const size_t numIndices = extractOp.getIndices().size();
   for (size_t i = 1; i < numIndices; i++) {
-    Value dimIdx = rewriter.create<arith::ConstantIndexOp>(loc, i);
+    Value dimIdx = arith::ConstantIndexOp::create(rewriter, loc, i);
 
     auto dimSize = broadcastIfNeeded(
         rewriter,
-        rewriter.create<tensor::DimOp>(loc, extractOp.getTensor(), dimIdx),
+        tensor::DimOp::create(rewriter, loc, extractOp.getTensor(), dimIdx),
         indexVecType);
 
-    offset = rewriter.create<arith::MulIOp>(loc, offset, dimSize);
+    offset = arith::MulIOp::create(rewriter, loc, offset, dimSize);
 
     auto extractOpIndex = broadcastIfNeeded(
         rewriter, bvm.lookup(extractOp.getIndices()[i]), indexVecType);
 
-    offset = rewriter.create<arith::AddIOp>(loc, extractOpIndex, offset);
+    offset = arith::AddIOp::create(rewriter, loc, extractOpIndex, offset);
   }
 
   return offset;
@@ -1110,18 +1110,18 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
 
   // Compute the static loop sizes of the extract op.
   auto resultType = state.getCanonicalVecType(extractOp.getResult().getType());
-  auto maskConstantOp = rewriter.create<arith::ConstantOp>(
+  auto maskConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseIntElementsAttr::get(state.getCanonicalVecType(rewriter.getI1Type()),
                                 /*value=*/true));
   auto passThruConstantOp =
-      rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(resultType));
+      arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(resultType));
 
   // Base indices are currently set to 0. We will need to re-visit if more
   // generic scenarios are to be supported.
   SmallVector<Value> baseIndices(
       extractOp.getIndices().size(),
-      rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      arith::ConstantIndexOp::create(rewriter, loc, 0));
 
   VectorMemoryAccessKind memAccessKind =
       getTensorExtractMemoryAccessPattern(extractOp, linalgOp, resultType);
@@ -1131,7 +1131,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     Value offset = calculateGatherOffset(rewriter, state, extractOp, bvm);
 
     // Generate the gather load
-    Operation *gatherOp = rewriter.create<vector::GatherOp>(
+    Operation *gatherOp = vector::GatherOp::create(rewriter,
         loc, resultType, extractOp.getTensor(), baseIndices, offset,
         maskConstantOp, passThruConstantOp);
     gatherOp = state.maskOperation(rewriter, gatherOp, linalgOp);
@@ -1166,13 +1166,13 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
       continue;
     }
 
-    auto indexAs1dVector = rewriter.create<vector::ShapeCastOp>(
+    auto indexAs1dVector = vector::ShapeCastOp::create(rewriter,
         loc,
         VectorType::get(resultType.getShape().back(), rewriter.getIndexType(),
                         resultType.getScalableDims().back()),
         idx);
     transferReadIdxs.push_back(
-        rewriter.create<vector::ExtractOp>(loc, indexAs1dVector, 0));
+        vector::ExtractOp::create(rewriter, loc, indexAs1dVector, 0));
   }
 
   // `tensor.extract_element` is always in-bounds, hence the following holds.
@@ -1186,7 +1186,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     SmallVector<AffineExpr> exprs(dstRank, getAffineConstantExpr(0, ctx));
     auto permutationMap = AffineMap::get(srcRank, 0, exprs, ctx);
 
-    auto transferReadOp = rewriter.create<vector::TransferReadOp>(
+    auto transferReadOp = vector::TransferReadOp::create(rewriter,
         loc, resultType, extractOp.getTensor(), transferReadIdxs,
         /*padding=*/std::nullopt, permutationMap, inBounds);
 
@@ -1195,7 +1195,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     // valid here).
     SmallVector<int64_t> readMaskShape = {1};
     auto readMaskType = VectorType::get(readMaskShape, rewriter.getI1Type());
-    auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+    auto allTrue = vector::ConstantMaskOp::create(rewriter,
         loc, readMaskType, vector::ConstantMaskKind::AllTrue);
     auto *maskedReadOp =
         mlir::vector::maskOperation(rewriter, transferReadOp, allTrue);
@@ -1223,7 +1223,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     rankDiff--;
   }
 
-  auto transferReadOp = rewriter.create<vector::TransferReadOp>(
+  auto transferReadOp = vector::TransferReadOp::create(rewriter,
       loc, resultType, extractOp.getTensor(), transferReadIdxs,
       /*padding=*/std::nullopt, permutationMap, inBounds);
 
@@ -1405,7 +1405,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
   // 3. Turn all BBArgs into vector.transfer_read / load.
   Location loc = linalgOp.getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   for (OpOperand *opOperand : linalgOp.getOpOperandsMatchingBBargs()) {
     BlockArgument bbarg = linalgOp.getMatchingBlockArgument(opOperand);
     if (linalgOp.isScalar(opOperand)) {
@@ -1435,7 +1435,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
-    Operation *read = rewriter.create<vector::TransferReadOp>(
+    Operation *read = vector::TransferReadOp::create(rewriter,
         loc, readType, opOperand->get(), indices,
         /*padding=*/std::nullopt, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
@@ -1452,7 +1452,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     // 3.c. Not all ops support 0-d vectors, extract the scalar for now.
     // TODO: remove this.
     if (readType.getRank() == 0)
-      readValue = rewriter.create<vector::ExtractOp>(loc, readValue,
+      readValue = vector::ExtractOp::create(rewriter, loc, readValue,
                                                      ArrayRef<int64_t>());
 
     LDBG("New vectorized bbarg(" << bbarg.getArgNumber() << "): " << readValue
@@ -1660,13 +1660,13 @@ createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
           writeIndices.size() == static_cast<size_t>(destRank)) &&
          "Invalid number of write indices!");
   if (writeIndices.empty()) {
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     writeIndices.assign(destRank, zero);
   }
 
   // Generate the xfer_write Op
   Operation *write =
-      builder.create<vector::TransferWriteOp>(loc,
+      vector::TransferWriteOp::create(builder, loc,
                                               /*vector=*/vecToStore,
                                               /*source=*/dest,
                                               /*indices=*/writeIndices,
@@ -1742,7 +1742,7 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   Location loc = packOp.getLoc();
   auto padValue = packOp.getPaddingValue();
   if (!padValue) {
-    padValue = rewriter.create<arith::ConstantOp>(
+    padValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(packOp.getSourceType().getElementType()));
   }
   ReifiedRankedShapedTypeDims reifiedReturnShapes;
@@ -1782,16 +1782,16 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   auto tiledPackType = VectorType::get(getTiledPackShape(packOp, destShape),
                                        packOp.getDestType().getElementType());
   auto shapeCastOp =
-      rewriter.create<vector::ShapeCastOp>(loc, tiledPackType, maskedRead);
+      vector::ShapeCastOp::create(rewriter, loc, tiledPackType, maskedRead);
 
   // Create TransposeOp.
   auto destPermutation =
       invertPermutationVector(getPackInverseDestPerm(packOp));
-  auto transposeOp = rewriter.create<vector::TransposeOp>(
+  auto transposeOp = vector::TransposeOp::create(rewriter,
       loc, shapeCastOp.getResult(), destPermutation);
 
   // Create TransferWriteOp.
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedReturnShapes[0],
       transposeOp.getResult().getType().getElementType());
   Operation *write =
@@ -1892,7 +1892,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   }
   Location loc = unpackOp->getLoc();
 
-  auto padValue = rewriter.create<arith::ConstantOp>(
+  auto padValue = arith::ConstantOp::create(rewriter,
       loc, rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
 
   // Read result, mask if necessary. If transferReadOp shape is not equal
@@ -1911,7 +1911,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   RankedTensorType stripMineTensorType =
       RankedTensorType::get(stripMineShape, stripMineElemType);
   // Transpose the appropriate rows to match output.
-  vector::TransposeOp transposeOp = rewriter.create<vector::TransposeOp>(
+  vector::TransposeOp transposeOp = vector::TransposeOp::create(rewriter,
       loc, readResult, lastDimToInsertPosPerm);
 
   // Collapse the vector to the size required by result.
@@ -1919,7 +1919,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       stripMineTensorType, packMetadata.reassociations);
   mlir::VectorType vecCollapsedType =
       VectorType::get(collapsedType.getShape(), collapsedType.getElementType());
-  vector::ShapeCastOp shapeCastOp = rewriter.create<vector::ShapeCastOp>(
+  vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(rewriter,
       loc, vecCollapsedType, transposeOp->getResult(0));
 
   // writeVectorSizes had to match the shapecast shape for dynamic sizes,
@@ -1928,7 +1928,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       unpackOp.getDestType().hasStaticShape()
           ? vectorSizes
           : shapeCastOp.getResultVectorType().getShape());
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedRetShapes[0],
       shapeCastOp.getResult().getType().getElementType());
   Operation *write = createWriteOrMaskedWrite(
@@ -1963,7 +1963,7 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
       /*useInBoundsInsteadOfMasking=*/false);
 
   // Create Xfer write Op
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedReturnShapes[0], padOp.getResultType().getElementType());
   Operation *write = createWriteOrMaskedWrite(rewriter, loc, maskedRead, dest);
   newResults.push_back(write->getResult(0));
@@ -2634,19 +2634,19 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
   auto writeType = VectorType::get(dstType.getShape(), dstElementType);
 
   Location loc = copyOp->getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   SmallVector<Value> indices(srcType.getRank(), zero);
 
-  Value readValue = rewriter.create<vector::TransferReadOp>(
+  Value readValue = vector::TransferReadOp::create(rewriter,
       loc, readType, copyOp.getSource(), indices,
       /*padding=*/std::nullopt,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   if (cast<VectorType>(readValue.getType()).getRank() == 0) {
     readValue =
-        rewriter.create<vector::ExtractOp>(loc, readValue, ArrayRef<int64_t>());
-    readValue = rewriter.create<vector::BroadcastOp>(loc, writeType, readValue);
+        vector::ExtractOp::create(rewriter, loc, readValue, ArrayRef<int64_t>());
+    readValue = vector::BroadcastOp::create(rewriter, loc, writeType, readValue);
   }
-  Operation *writeValue = rewriter.create<vector::TransferWriteOp>(
+  Operation *writeValue = vector::TransferWriteOp::create(rewriter,
       loc, readValue, copyOp.getTarget(), indices,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   rewriter.replaceOp(copyOp, writeValue->getResults());
@@ -2957,7 +2957,7 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   if (!padValue) {
     auto elemType = sourceType.getElementType();
-    padValue = rewriter.create<arith::ConstantOp>(
+    padValue = arith::ConstantOp::create(rewriter,
         sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
   }
 
@@ -2989,7 +2989,7 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   // Create read
   SmallVector<Value> readIndices(
-      vecType.getRank(), rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      vecType.getRank(), arith::ConstantIndexOp::create(rewriter, loc, 0));
   Value read = mlir::vector::createReadOrMaskedRead(
       rewriter, loc, source, vecType.getShape(), padValue,
       /*useInBoundsInsteadOfMasking=*/inputVectorSizes.empty());
@@ -3076,8 +3076,8 @@ struct PadOpVectorizationWithInsertSlicePattern
     // Generate TransferReadOp: Read entire source tensor and add high
     // padding.
     SmallVector<Value> readIndices(
-        vecRank, rewriter.create<arith::ConstantIndexOp>(padOp.getLoc(), 0));
-    auto read = rewriter.create<vector::TransferReadOp>(
+        vecRank, arith::ConstantIndexOp::create(rewriter, padOp.getLoc(), 0));
+    auto read = vector::TransferReadOp::create(rewriter,
         padOp.getLoc(), vecType, padOp.getSource(), readIndices, padValue);
 
     // Generate TransferWriteOp: Write to InsertSliceOp's dest tensor at
@@ -3212,7 +3212,7 @@ LogicalResult LinalgCopyVTRForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_read, the attribute must be reset
   // conservatively.
   auto vectorType = xferOp.getVectorType();
-  Value res = rewriter.create<vector::TransferReadOp>(
+  Value res = vector::TransferReadOp::create(rewriter,
       xferOp.getLoc(), vectorType, in, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getPadding(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(
@@ -3271,7 +3271,7 @@ LogicalResult LinalgCopyVTWForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_write, the attribute must be reset
   // conservatively.
   auto vector = xferOp.getVector();
-  rewriter.create<vector::TransferWriteOp>(
+  vector::TransferWriteOp::create(rewriter,
       xferOp.getLoc(), vector, out, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(SmallVector<bool>(
@@ -3467,7 +3467,7 @@ struct Conv1DGenerator
     }
 
     vector::TransferWriteOp write;
-    Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
     // w is unrolled (i.e. wSizeStep == 1) iff strideW > 1.
     // When strideW == 1, we can batch the contiguous loads and avoid
@@ -3486,16 +3486,16 @@ struct Conv1DGenerator
     SmallVector<Value> resPadding(resShape.size(), zero);
 
     // Read the whole lhs, rhs and res in one shot (with zero padding).
-    Value lhs = rewriter.create<vector::TransferReadOp>(
+    Value lhs = vector::TransferReadOp::create(rewriter,
         loc, lhsType, lhsShaped, lhsPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     // This is needed only for Conv.
     Value rhs = nullptr;
     if (oper == ConvOperationKind::Conv)
-      rhs = rewriter.create<vector::TransferReadOp>(
+      rhs = vector::TransferReadOp::create(rewriter,
           loc, rhsType, rhsShaped, rhsPadding,
           /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
-    Value res = rewriter.create<vector::TransferReadOp>(
+    Value res = vector::TransferReadOp::create(rewriter,
         loc, resType, resShaped, resPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
 
@@ -3511,16 +3511,16 @@ struct Conv1DGenerator
       // To match base vectorization case, we pre-transpose current case.
       // ncw -> nwc
       static constexpr std::array<int64_t, 3> permLhs = {0, 2, 1};
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, permLhs);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, permLhs);
       // fcw -> wcf
       static constexpr std::array<int64_t, 3> permRhs = {2, 1, 0};
 
       // This is needed only for Conv.
       if (oper == ConvOperationKind::Conv)
-        rhs = rewriter.create<vector::TransposeOp>(loc, rhs, permRhs);
+        rhs = vector::TransposeOp::create(rewriter, loc, rhs, permRhs);
       // nfw -> nwf
       static constexpr std::array<int64_t, 3> permRes = {0, 2, 1};
-      res = rewriter.create<vector::TransposeOp>(loc, res, permRes);
+      res = vector::TransposeOp::create(rewriter, loc, res, permRes);
       break;
     }
     }
@@ -3585,7 +3585,7 @@ struct Conv1DGenerator
     case Conv1DOpOrder::Ncw: {
       // nwf -> nfw
       static constexpr std::array<int64_t, 3> perm = {0, 2, 1};
-      res = rewriter.create<vector::TransposeOp>(loc, res, perm);
+      res = vector::TransposeOp::create(rewriter, loc, res, perm);
       break;
     }
     }
@@ -3609,16 +3609,16 @@ struct Conv1DGenerator
         cast<ShapedType>(val.getType()).cloneWith(std::nullopt, dstElementType);
 
     if (isa<IntegerType>(srcElementType) && isa<FloatType>(dstElementType)) {
-      return rewriter.create<arith::SIToFPOp>(loc, dstType, val);
+      return arith::SIToFPOp::create(rewriter, loc, dstType, val);
     }
 
     if (isa<FloatType>(srcElementType) && isa<FloatType>(dstElementType) &&
         srcWidth < dstWidth)
-      return rewriter.create<arith::ExtFOp>(loc, dstType, val);
+      return arith::ExtFOp::create(rewriter, loc, dstType, val);
 
     if (isa<IntegerType>(srcElementType) && isa<IntegerType>(dstElementType) &&
         srcWidth < dstWidth)
-      return rewriter.create<arith::ExtSIOp>(loc, dstType, val);
+      return arith::ExtSIOp::create(rewriter, loc, dstType, val);
 
     assert(false && "unhandled promotion case");
     return nullptr;
@@ -3633,7 +3633,7 @@ struct Conv1DGenerator
     bindDims(ctx, n, w, f, c);
     lhs = promote(rewriter, loc, lhs, res.getType());
     rhs = promote(rewriter, loc, rhs, res.getType());
-    auto contrationOp = rewriter.create<vector::ContractionOp>(
+    auto contrationOp = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, res,
         /*indexingMaps=*/MapList{{n, w, c}, {c, f}, {n, w, f}},
         /*iteratorTypes=*/ArrayRef<vector::IteratorType>{par, par, par, red});
@@ -3645,7 +3645,7 @@ struct Conv1DGenerator
   // convolution.
   Value conv1dSliceAsOuterProduct(RewriterBase &rewriter, Location loc,
                                   Value lhs, Value rhs, Value res) {
-    return rewriter.create<vector::OuterProductOp>(
+    return vector::OuterProductOp::create(rewriter,
         loc, res.getType(), lhs, rhs, res, vector::CombiningKind::ADD);
   }
 
@@ -3693,7 +3693,7 @@ struct Conv1DGenerator
     bindShapeDims(resShapedType, nSize, wSize);
 
     vector::TransferWriteOp write;
-    Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
     // w is unrolled (i.e. wSizeStep == 1) iff strideW > 1.
     // When strideW == 1, we can batch the contiguous loads and avoid
@@ -3736,28 +3736,28 @@ struct Conv1DGenerator
           cast<LinalgOp>(op).hasPureTensorSemantics(), opToMask, rewriter);
 
       Value maskOp =
-          rewriter.create<vector::CreateMaskOp>(loc, maskType, mixedDims);
+          vector::CreateMaskOp::create(rewriter, loc, maskType, mixedDims);
 
       return mlir::vector::maskOperation(rewriter, opToMask, maskOp);
     };
 
     // Read lhs slice of size {n, w * strideW + kw * dilationW, c} @ [0, 0,
     // 0].
-    Value lhs = rewriter.create<vector::TransferReadOp>(
+    Value lhs = vector::TransferReadOp::create(rewriter,
         loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     auto maybeMaskedLhs = maybeMaskXferOp(
         lhsType.getShape(), lhsType.getScalableDims(), lhs.getDefiningOp());
 
     // Read rhs slice of size {kw, c} @ [0, 0].
-    Value rhs = rewriter.create<vector::TransferReadOp>(
+    Value rhs = vector::TransferReadOp::create(rewriter,
         loc, rhsType, rhsShaped, ValueRange{zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
     auto maybeMaskedRhs = maybeMaskXferOp(
         rhsType.getShape(), rhsType.getScalableDims(), rhs.getDefiningOp());
 
     // Read res slice of size {n, w, c} @ [0, 0, 0].
-    Value res = rewriter.create<vector::TransferReadOp>(
+    Value res = vector::TransferReadOp::create(rewriter,
         loc, resType, resShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
     auto maybeMaskedRes = maybeMaskXferOp(
@@ -3775,7 +3775,7 @@ struct Conv1DGenerator
     //   @ [0, sw * w + dw * kw, 0].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        lhsVals.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        lhsVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, maybeMaskedLhs->getResult(0),
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             inOutSliceSizes, inOutStrides));
@@ -3783,13 +3783,13 @@ struct Conv1DGenerator
     }
     // Extract rhs slice of size {c} @ [kw].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
-      rhsVals.push_back(rewriter.create<vector::ExtractOp>(
+      rhsVals.push_back(vector::ExtractOp::create(rewriter,
           loc, maybeMaskedRhs->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{kw}));
     }
     // Extract res slice: {n, wSizeStep, c} @ [0, w, 0].
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      resVals.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      resVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0}, inOutSliceSizes,
           inOutStrides));
@@ -3815,16 +3815,16 @@ struct Conv1DGenerator
         if (flatten) {
           // Flatten the input and output vectors (collapse the channel
           // dimension)
-          lhsVal = rewriter.create<vector::ShapeCastOp>(
+          lhsVal = vector::ShapeCastOp::create(rewriter,
               loc, lhsTypeAfterFlattening, lhsVals[linearIndex(kw, w)]);
-          resVal = rewriter.create<vector::ShapeCastOp>(
+          resVal = vector::ShapeCastOp::create(rewriter,
               loc, resTypeAfterFlattening, resVals[w]);
         }
         resVals[w] = depthwiseConv1dSliceAsMulAcc(rewriter, loc, lhsVal,
                                                   rhsVals[kw], resVal, flatten);
         if (flatten) {
           // Un-flatten the output vector (restore the channel dimension)
-          resVals[w] = rewriter.create<vector::ShapeCastOp>(
+          resVals[w] = vector::ShapeCastOp::create(rewriter,
               loc, VectorType::get(inOutSliceSizes, resEltType), resVals[w]);
         }
       }
@@ -3843,7 +3843,7 @@ struct Conv1DGenerator
     // Write back res slice: {n, wSizeStep, c} @ [0, w, 0].
     // This does not depend on kw.
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      maybeMaskedRes = rewriter.create<vector::InsertStridedSliceOp>(
+      maybeMaskedRes = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           /*strides=*/ArrayRef<int64_t>{1, 1, 1});
@@ -3853,7 +3853,7 @@ struct Conv1DGenerator
     //===------------------------------------------------------------------===//
 
     // Write back res slice of size {n, w, c} @ [0, 0, 0].
-    Operation *resOut = rewriter.create<vector::TransferWriteOp>(
+    Operation *resOut = vector::TransferWriteOp::create(rewriter,
         loc, maybeMaskedRes->getResult(0), resShaped,
         ValueRange{zero, zero, zero});
     return maybeMaskXferOp(resType.getShape(), resType.getScalableDims(),
@@ -3891,10 +3891,10 @@ struct Conv1DGenerator
           indices.push_back(j);
       }
 
-      rhs = rewriter.create<vector::ShuffleOp>(loc, rhs, rhs, indices);
+      rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, indices);
     }
     // Broadcast the filter to match the output vector
-    rhs = rewriter.create<vector::BroadcastOp>(
+    rhs = vector::BroadcastOp::create(rewriter,
         loc, resTy.clone(rhsTy.getElementType()), rhs);
 
     rhs = promote(rewriter, loc, rhs, resTy);
@@ -3903,10 +3903,10 @@ struct Conv1DGenerator
       return nullptr;
 
     if (isa<FloatType>(resTy.getElementType()))
-      return rewriter.create<vector::FMAOp>(loc, lhs, rhs, res);
+      return vector::FMAOp::create(rewriter, loc, lhs, rhs, res);
 
-    auto mul = rewriter.create<arith::MulIOp>(loc, lhs, rhs);
-    return rewriter.create<arith::AddIOp>(loc, mul, res);
+    auto mul = arith::MulIOp::create(rewriter, loc, lhs, rhs);
+    return arith::AddIOp::create(rewriter, loc, mul, res);
   }
 
   /// Entry point for non-channeled convolution:
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index 9fd084487e3fd..f9c6aae9374a8 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -201,7 +201,7 @@ Value create2DTransformMatrix(OpBuilder &builder, Location loc,
                               TransformMatrix transform, Type type) {
   ArrayRef<float> constVec(transform.table, transform.rows * transform.cols);
 
-  return builder.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(builder,
       loc, DenseFPElementsAttr::get(
                RankedTensorType::get(
                    SmallVector<int64_t>{transform.rows, transform.cols}, type),
@@ -233,7 +233,7 @@ Value extract2DDataFrom4D(OpBuilder &builder, Location loc, Value source,
 
   auto extractFilterType =
       RankedTensorType::get({extractHeight, extractWidth}, elementType);
-  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
+  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
       loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
@@ -267,7 +267,7 @@ Value extract2DDataFrom6D(OpBuilder &builder, Location loc, Value source,
   SmallVector<OpFoldResult> strides(srcSize, oneIndex);
 
   auto extractFilterType = RankedTensorType::get({height, width}, elementType);
-  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
+  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
       loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
@@ -293,7 +293,7 @@ Value insert2DDataTo4D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
+  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
       loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
@@ -321,7 +321,7 @@ Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
+  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
       loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
@@ -372,7 +372,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
   if (filterW != r && filterW != 1)
     return Value();
 
-  Value zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
                        ValueRange args) -> scf::ValueVector {
     Value FIter = ivs[0];
@@ -386,7 +386,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
 
     int64_t retRows = 1;
     Value matmulRetValue = extractFilter;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix G.
@@ -401,11 +401,11 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value G = create2DTransformMatrix(builder, loc, GMatrix, elementType);
       // Multiply G x g.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{G, extractFilter}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -423,11 +423,11 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value GT = create2DTransformMatrix(builder, loc, GTMatrix, elementType);
       // Multiply u = (G x g) x GT.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, GT}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -445,9 +445,9 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
     return {insertSliceOp};
   };
 
-  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterF);
-  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterC);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterF);
+  auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterC);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx}, {fUpperBound, cUpperBound},
       {oneStep, oneStep}, {retValue}, buildBody);
@@ -516,9 +516,9 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = builder.create<affine::AffineApplyOp>(
+    Value heightOffset = affine::AffineApplyOp::create(builder,
         loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = builder.create<affine::AffineApplyOp>(
+    Value widthOffset = affine::AffineApplyOp::create(builder,
         loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     // Extract (H, W) from (N, H, W, C).
@@ -530,7 +530,7 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     int64_t retRows = 1;
     int64_t retCols = 1;
     Value matmulRetValue = extractInput;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix BT.
@@ -545,12 +545,12 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value BT =
           create2DTransformMatrix(builder, loc, BTMatrix, builder.getF32Type());
       // Multiply BT x d.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{BT, matmulRetValue}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -568,11 +568,11 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       Value B =
           create2DTransformMatrix(builder, loc, BMatrix, builder.getF32Type());
       // Multiply v = (BT x d) x B.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, B}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -586,12 +586,12 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     return {combinedVal};
   };
 
-  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tileH);
-  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
-  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputN);
-  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputC);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tileH);
+  auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
+  auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputN);
+  auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputC);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
       {tileHBound, tileWBound, nUpperBound, cUpperBound},
@@ -629,7 +629,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       {filterShape[0] * filterShape[1], filterShape[2], filterShape[3]},
       filterElementType);
   SmallVector<ReassociationIndices> filterReassoc = {{0, 1}, {2}, {3}};
-  Value collapseFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value collapseFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, filterReassocType, transformedFilter, filterReassoc);
 
   // Convert (alphaH, alphaW, tileH, tileW, N, C) to
@@ -643,7 +643,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
        inputShape[2] * inputShape[3] * inputShape[4], inputShape[5]},
       inputElementType);
   SmallVector<ReassociationIndices> inputReassoc = {{0, 1}, {2, 3, 4}, {5}};
-  Value collapseInput = rewriter.create<tensor::CollapseShapeOp>(
+  Value collapseInput = tensor::CollapseShapeOp::create(rewriter,
       loc, inputReassocType, transformedInput, inputReassoc);
 
   // Batched matrix multiply.
@@ -655,11 +655,11 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
                     .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                              outputElementType)
                     .getResult();
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, rewriter.getZeroAttr(outputElementType));
-  Value init = rewriter.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+  Value init = linalg::FillOp::create(rewriter, loc, zero, empty).getResult(0);
 
-  auto matmulOp = rewriter.create<linalg::BatchMatmulOp>(
+  auto matmulOp = linalg::BatchMatmulOp::create(rewriter,
       loc, matmulType, ValueRange({collapseInput, collapseFilter}),
       ValueRange{init});
 
@@ -670,7 +670,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       RankedTensorType::get({inputShape[0], inputShape[1], inputShape[2],
                              inputShape[3], inputShape[4], filterShape[3]},
                             outputElementType);
-  auto expandOutput = rewriter.create<tensor::ExpandShapeOp>(
+  auto expandOutput = tensor::ExpandShapeOp::create(rewriter,
       loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
   return expandOutput;
 }
@@ -750,15 +750,15 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
     int64_t retRows = leftTransform ? ATMatrix.rows : 1;
 
     Value matmulRetValue = extractValue;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
 
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = builder.create<affine::AffineApplyOp>(
+    Value heightOffset = affine::AffineApplyOp::create(builder,
         loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = builder.create<affine::AffineApplyOp>(
+    Value widthOffset = affine::AffineApplyOp::create(builder,
         loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     Value outInitVal =
@@ -775,12 +775,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                          .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                                   elementType)
                          .getResult();
-        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+        init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       }
 
       Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
       // Multiply AT x m.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{AT, matmulRetValue}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -794,19 +794,19 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                          .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                                   elementType)
                          .getResult();
-        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+        init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       }
 
       Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
       // Multiply y = (AT x m) x A.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, A}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
     if (scalarFactor != 1) {
       // Multiply by scalar factor and add outInitVal.
-      Value scalarFactorValue = builder.create<arith::ConstantOp>(
+      Value scalarFactorValue = arith::ConstantOp::create(builder,
           loc, FloatAttr::get(elementType, scalarFactor));
       auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
       auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
@@ -824,11 +824,11 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                       utils::IteratorType::parallel},
                   [&](OpBuilder &nestedBuilder, Location nestedLoc,
                       ValueRange args) {
-                    auto mulf = nestedBuilder.create<arith::MulFOp>(
+                    auto mulf = arith::MulFOp::create(nestedBuilder,
                         nestedLoc, args[0], args[1]);
-                    auto addf = nestedBuilder.create<arith::AddFOp>(
+                    auto addf = arith::AddFOp::create(nestedBuilder,
                         nestedLoc, mulf.getResult(), args[2]);
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc,
+                    linalg::YieldOp::create(nestedBuilder, nestedLoc,
                                                           addf.getResult());
                   })
               .getResult(0);
@@ -847,12 +847,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
   int64_t tilwH = valueShape[2];
   int64_t tileW = valueShape[3];
-  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tilwH);
-  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
-  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueN);
-  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueF);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tilwH);
+  auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
+  auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueN);
+  auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueF);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
       {tileHBound, tileWBound, nUpperBound, fUpperBound},
@@ -867,7 +867,7 @@ static Value padToAlignedTensor(RewriterBase &rewriter, Location loc,
   auto valueType = cast<ShapedType>(value.getType());
   Type elementType = valueType.getElementType();
   auto alignedType = RankedTensorType::get(alignedShape, elementType);
-  Value padValue = rewriter.create<arith::ConstantOp>(
+  Value padValue = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
 
   return linalg::makeComposedPadHighOp(rewriter, loc, alignedType, value,
@@ -887,7 +887,7 @@ static Value extractFromAlignedTensor(RewriterBase &rewriter, Location loc,
   SmallVector<OpFoldResult> sizes =
       getAsOpFoldResult(rewriter.getI64ArrayAttr(extractedShape));
 
-  return rewriter.create<tensor::ExtractSliceOp>(loc, extractedType, value,
+  return tensor::ExtractSliceOp::create(rewriter, loc, extractedType, value,
                                                  offsets, sizes, strides);
 }
 
@@ -979,9 +979,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   int64_t tileW = llvm::divideCeilSigned(outputW, widthM);
   auto retType = RankedTensorType::get({alphaH, alphaW, filterC, filterF},
                                        filterElementType);
-  Value retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
+  Value retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
                                                     filterElementType);
-  auto transformedFilter = rewriter.create<linalg::WinogradFilterTransformOp>(
+  auto transformedFilter = linalg::WinogradFilterTransformOp::create(rewriter,
       loc, retType, filter, retValue, fmr);
 
   // --- Create operation for input transform ---
@@ -998,9 +998,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
 
   retType = RankedTensorType::get(
       {alphaH, alphaW, tileH, tileW, inputN, inputC}, inputElementType);
-  retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
+  retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
                                               inputElementType);
-  auto transformedInput = rewriter.create<linalg::WinogradInputTransformOp>(
+  auto transformedInput = linalg::WinogradInputTransformOp::create(rewriter,
       loc, retType, input, retValue, fmr);
 
   Type outputElementType = outputType.getElementType();
@@ -1023,7 +1023,7 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
     outputType = alignedOutputType;
   }
 
-  Value transformedOutput = rewriter.create<linalg::WinogradOutputTransformOp>(
+  Value transformedOutput = linalg::WinogradOutputTransformOp::create(rewriter,
       loc, outputType, matmulRet, output, fmr);
 
   // When output size is not aligned with output tile size, extract the
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index e89e80b07d20d..d964aa5cb4e81 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -320,14 +320,14 @@ GenericOp makeMemRefCopyOp(OpBuilder &b, Location loc, Value from, Value to) {
       AffineMap::getMultiDimIdentityMap(memrefTypeTo.getRank(), b.getContext());
   SmallVector<utils::IteratorType> iteratorTypes(memrefTypeTo.getRank(),
                                                  utils::IteratorType::parallel);
-  return b.create<linalg::GenericOp>(
+  return linalg::GenericOp::create(b,
       loc,
       /*inputs=*/from,
       /*outputs=*/to,
       /*indexingMaps=*/llvm::ArrayRef({id, id}),
       /*iteratorTypes=*/iteratorTypes,
       [](OpBuilder &b, Location loc, ValueRange args) {
-        b.create<linalg::YieldOp>(loc, args.front());
+        linalg::YieldOp::create(b, loc, args.front());
       });
 }
 
@@ -483,7 +483,7 @@ static void generateParallelLoopNest(
   case DistributionMethod::None: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    b.create<scf::ParallelOp>(
+    scf::ParallelOp::create(b,
         loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
@@ -499,7 +499,7 @@ static void generateParallelLoopNest(
   case DistributionMethod::Cyclic: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    b.create<scf::ParallelOp>(
+    scf::ParallelOp::create(b,
         loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
@@ -519,13 +519,13 @@ static void generateParallelLoopNest(
     for (unsigned i = 1; i < numProcessed; ++i)
       cond = ab._and(cond, ab.slt(lbs[i], ubs[i]));
     ivStorage.append(lbs.begin(), std::next(lbs.begin(), numProcessed));
-    b.create<scf::IfOp>(loc, cond, [&](OpBuilder &b, Location loc) {
+    scf::IfOp::create(b, loc, cond, [&](OpBuilder &b, Location loc) {
       generateParallelLoopNest(b, loc, lbs.drop_front(numProcessed),
                                ubs.drop_front(numProcessed),
                                steps.drop_front(numProcessed),
                                iteratorTypes.drop_front(numProcessed),
                                remainderProcInfo, bodyBuilderFn, ivStorage);
-      b.create<scf::YieldOp>(loc, ValueRange{});
+      scf::YieldOp::create(b, loc, ValueRange{});
     });
     return;
   }
@@ -595,12 +595,12 @@ static Operation *materializeTiledShape(OpBuilder &builder, Location loc,
   auto shapedType = dyn_cast<ShapedType>(valueToTile.getType());
   auto *sliceOp = TypeSwitch<ShapedType, Operation *>(shapedType)
                       .Case([&](MemRefType) {
-                        return builder.create<memref::SubViewOp>(
+                        return memref::SubViewOp::create(builder,
                             loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
                       .Case([&](RankedTensorType) {
-                        return builder.create<tensor::ExtractSliceOp>(
+                        return tensor::ExtractSliceOp::create(builder,
                             loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
@@ -793,7 +793,7 @@ SmallVector<Value> insertSlicesBack(OpBuilder &builder, Location loc,
     // `tiledOperands`.
     Value outputTensor = operands[opOperand.getOperandNumber()];
     if (auto sliceOp = outputTensor.getDefiningOp<tensor::ExtractSliceOp>()) {
-      Value inserted = builder.create<tensor::InsertSliceOp>(
+      Value inserted = tensor::InsertSliceOp::create(builder,
           loc, sliceOp.getSource().getType(), results[resultIdx],
           sliceOp.getSource(), sliceOp.getOffsets(), sliceOp.getSizes(),
           sliceOp.getStrides(), sliceOp.getStaticOffsets(),
diff --git a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
index ff6af63eee531..9dadd9bea2f67 100644
--- a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -135,7 +135,7 @@ struct GlobalStoreOpInterface
     auto memrefType = getMemRefTypeWithStaticIdentityLayout(tensorType);
 
     auto loc = globalStoreOp.getLoc();
-    auto targetMemref = rewriter.create<memref::GetGlobalOp>(
+    auto targetMemref = memref::GetGlobalOp::create(rewriter,
         loc, memrefType, globalStoreOp.getGlobalAttr().getLeafReference());
 
     auto sourceMemref =
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index 7940ff60a48e7..749cca0575581 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -60,7 +60,7 @@ struct FoldRank final : public mlir::OpRewritePattern<mlir::mpi::CommRankOp> {
     if (!isa<IntegerAttr>(dltiAttr.value()))
       return op->emitError()
              << "Expected an integer attribute for MPI:comm_world_rank";
-    Value res = b.create<arith::ConstantIndexOp>(
+    Value res = arith::ConstantIndexOp::create(b,
         op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
     if (Value retVal = op.getRetval())
       b.replaceOp(op, {retVal, res});
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index 26441a9d78658..a21631cbf8510 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -746,7 +746,7 @@ Operation *math::MathDialect::materializeConstant(OpBuilder &builder,
                                                   Attribute value, Type type,
                                                   Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
index 13e2a4b5541b2..770f32841c0f6 100644
--- a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
@@ -65,7 +65,7 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
   // Maybe broadcasts scalar value into vector type compatible with `op`.
   auto bcast = [&](Value value) -> Value {
     if (auto vec = dyn_cast<VectorType>(op.getType()))
-      return rewriter.create<vector::BroadcastOp>(op.getLoc(), vec, value);
+      return vector::BroadcastOp::create(rewriter, op.getLoc(), vec, value);
     return value;
   };
 
@@ -84,14 +84,14 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
   // Replace `pow(x, 3.0)` with `x * x * x`.
   if (isExponentValue(3.0)) {
     Value square =
-        rewriter.create<arith::MulFOp>(op.getLoc(), ValueRange({x, x}));
+        arith::MulFOp::create(rewriter, op.getLoc(), ValueRange({x, x}));
     rewriter.replaceOpWithNewOp<arith::MulFOp>(op, ValueRange({x, square}));
     return success();
   }
 
   // Replace `pow(x, -1.0)` with `1.0 / x`.
   if (isExponentValue(-1.0)) {
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getFloatAttr(getElementTypeOrSelf(op.getType()), 1.0));
     rewriter.replaceOpWithNewOp<arith::DivFOp>(op, ValueRange({bcast(one), x}));
     return success();
@@ -111,8 +111,8 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
 
   // Replace `pow(x, 0.75)` with `sqrt(sqrt(x)) * sqrt(x)`.
   if (isExponentValue(0.75)) {
-    Value powHalf = rewriter.create<math::SqrtOp>(op.getLoc(), x);
-    Value powQuarter = rewriter.create<math::SqrtOp>(op.getLoc(), powHalf);
+    Value powHalf = math::SqrtOp::create(rewriter, op.getLoc(), x);
+    Value powQuarter = math::SqrtOp::create(rewriter, op.getLoc(), powHalf);
     rewriter.replaceOpWithNewOp<arith::MulFOp>(op,
                                                ValueRange{powHalf, powQuarter});
     return success();
@@ -168,17 +168,17 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   // Maybe broadcasts scalar value into vector type compatible with `op`.
   auto bcast = [&loc, &op, &rewriter](Value value) -> Value {
     if (auto vec = dyn_cast<VectorType>(op.getType()))
-      return rewriter.create<vector::BroadcastOp>(loc, vec, value);
+      return vector::BroadcastOp::create(rewriter, loc, vec, value);
     return value;
   };
 
   Value one;
   Type opType = getElementTypeOrSelf(op.getType());
   if constexpr (std::is_same_v<PowIOpTy, math::FPowIOp>)
-    one = rewriter.create<arith::ConstantOp>(
+    one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getFloatAttr(opType, 1.0));
   else
-    one = rewriter.create<arith::ConstantOp>(
+    one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(opType, 1));
 
   // Replace `[fi]powi(x, 0)` with `1`.
@@ -208,12 +208,12 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   //     with:
   //       (1 / x) * (1 / x) * (1 / x) * ...
   for (unsigned i = 1; i < exponentValue; ++i)
-    result = rewriter.create<MulOpTy>(loc, result, base);
+    result = MulOpTy::create(rewriter, loc, result, base);
 
   // Inverse the base for negative exponent, i.e. for
   // `[fi]powi(x, negative_exponent)` set `x` to `1 / x`.
   if (exponentIsNegative)
-    result = rewriter.create<DivOpTy>(loc, bcast(one), result);
+    result = DivOpTy::create(rewriter, loc, bcast(one), result);
 
   rewriter.replaceOp(op, result);
   return success();
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index bccd486def4bf..b5c0ddd617805 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -32,11 +32,11 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                 APFloat::rmNearestTiesToEven, &losesInfo);
   auto attr = b.getFloatAttr(eltType, value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return b.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(b, loc,
                                        DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return b.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(b, loc, attr);
 }
 
 static Value createFloatConst(Location loc, Type type, double value,
@@ -49,11 +49,11 @@ static Value createIntConst(Location loc, Type type, int64_t value,
                             OpBuilder &b) {
   auto attr = b.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return b.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(b, loc,
                                        DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return b.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(b, loc, attr);
 }
 
 static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
@@ -61,11 +61,11 @@ static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
   Type i64Ty = b.getI64Type();
   if (auto shapedTy = dyn_cast<ShapedType>(opType))
     i64Ty = shapedTy.clone(i64Ty);
-  Value fixedConvert = b.create<arith::FPToSIOp>(i64Ty, operand);
-  Value fpFixedConvert = b.create<arith::SIToFPOp>(opType, fixedConvert);
+  Value fixedConvert = arith::FPToSIOp::create(b, i64Ty, operand);
+  Value fpFixedConvert = arith::SIToFPOp::create(b, opType, fixedConvert);
   // The truncation does not preserve the sign when the truncated
   // value is -0. So here the sign is copied again.
-  return b.create<math::CopySignOp>(fpFixedConvert, operand);
+  return math::CopySignOp::create(b, fpFixedConvert, operand);
 }
 
 // sinhf(float x) -> (exp(x) - exp(-x)) / 2
@@ -74,12 +74,12 @@ static LogicalResult convertSinhOp(math::SinhOp op, PatternRewriter &rewriter) {
   Value operand = op.getOperand();
   Type opType = operand.getType();
 
-  Value exp = b.create<math::ExpOp>(operand);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value nexp = b.create<math::ExpOp>(neg);
-  Value sub = b.create<arith::SubFOp>(exp, nexp);
+  Value exp = math::ExpOp::create(b, operand);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value nexp = math::ExpOp::create(b, neg);
+  Value sub = arith::SubFOp::create(b, exp, nexp);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(sub, half);
+  Value res = arith::MulFOp::create(b, sub, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -90,12 +90,12 @@ static LogicalResult convertCoshOp(math::CoshOp op, PatternRewriter &rewriter) {
   Value operand = op.getOperand();
   Type opType = operand.getType();
 
-  Value exp = b.create<math::ExpOp>(operand);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value nexp = b.create<math::ExpOp>(neg);
-  Value add = b.create<arith::AddFOp>(exp, nexp);
+  Value exp = math::ExpOp::create(b, operand);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value nexp = math::ExpOp::create(b, neg);
+  Value add = arith::AddFOp::create(b, exp, nexp);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(add, half);
+  Value res = arith::MulFOp::create(b, add, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -116,23 +116,23 @@ static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) {
   Value negTwo = createFloatConst(loc, floatType, -2.0, rewriter);
 
   // Compute sign(x) = cast<float_type>(x < 0) * (-2) + 1
-  Value isNegative = rewriter.create<arith::CmpFOp>(
+  Value isNegative = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OLT, op.getOperand(), zero);
   Value isNegativeFloat =
-      rewriter.create<arith::UIToFPOp>(loc, floatType, isNegative);
+      arith::UIToFPOp::create(rewriter, loc, floatType, isNegative);
   Value isNegativeTimesNegTwo =
-      rewriter.create<arith::MulFOp>(loc, isNegativeFloat, negTwo);
-  Value sign = rewriter.create<arith::AddFOp>(loc, isNegativeTimesNegTwo, one);
+      arith::MulFOp::create(rewriter, loc, isNegativeFloat, negTwo);
+  Value sign = arith::AddFOp::create(rewriter, loc, isNegativeTimesNegTwo, one);
 
   // Normalize input to positive value: y = sign(x) * x
-  Value positiveX = rewriter.create<arith::MulFOp>(loc, sign, op.getOperand());
+  Value positiveX = arith::MulFOp::create(rewriter, loc, sign, op.getOperand());
 
   // Decompose on normalized input
-  Value negDoubledX = rewriter.create<arith::MulFOp>(loc, negTwo, positiveX);
-  Value exp2x = rewriter.create<math::ExpOp>(loc, negDoubledX);
-  Value dividend = rewriter.create<arith::SubFOp>(loc, one, exp2x);
-  Value divisor = rewriter.create<arith::AddFOp>(loc, one, exp2x);
-  Value positiveRes = rewriter.create<arith::DivFOp>(loc, dividend, divisor);
+  Value negDoubledX = arith::MulFOp::create(rewriter, loc, negTwo, positiveX);
+  Value exp2x = math::ExpOp::create(rewriter, loc, negDoubledX);
+  Value dividend = arith::SubFOp::create(rewriter, loc, one, exp2x);
+  Value divisor = arith::AddFOp::create(rewriter, loc, one, exp2x);
+  Value positiveRes = arith::DivFOp::create(rewriter, loc, dividend, divisor);
 
   // Multiply result by sign(x) to retain signs from negative inputs
   rewriter.replaceOpWithNewOp<arith::MulFOp>(op, sign, positiveRes);
@@ -145,9 +145,9 @@ static LogicalResult convertTanOp(math::TanOp op, PatternRewriter &rewriter) {
   ImplicitLocOpBuilder b(op->getLoc(), rewriter);
   Value operand = op.getOperand();
   Type type = operand.getType();
-  Value sin = b.create<math::SinOp>(type, operand);
-  Value cos = b.create<math::CosOp>(type, operand);
-  Value div = b.create<arith::DivFOp>(type, sin, cos);
+  Value sin = math::SinOp::create(b, type, operand);
+  Value cos = math::CosOp::create(b, type, operand);
+  Value div = arith::DivFOp::create(b, type, sin, cos);
   rewriter.replaceOp(op, div);
   return success();
 }
@@ -160,10 +160,10 @@ static LogicalResult convertAsinhOp(math::AsinhOp op,
   Type opType = operand.getType();
 
   Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value fma = b.create<math::FmaOp>(operand, operand, one);
-  Value sqrt = b.create<math::SqrtOp>(fma);
-  Value add = b.create<arith::AddFOp>(operand, sqrt);
-  Value res = b.create<math::LogOp>(add);
+  Value fma = math::FmaOp::create(b, operand, operand, one);
+  Value sqrt = math::SqrtOp::create(b, fma);
+  Value add = arith::AddFOp::create(b, operand, sqrt);
+  Value res = math::LogOp::create(b, add);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -176,10 +176,10 @@ static LogicalResult convertAcoshOp(math::AcoshOp op,
   Type opType = operand.getType();
 
   Value negOne = createFloatConst(op->getLoc(), opType, -1.0, rewriter);
-  Value fma = b.create<math::FmaOp>(operand, operand, negOne);
-  Value sqrt = b.create<math::SqrtOp>(fma);
-  Value add = b.create<arith::AddFOp>(operand, sqrt);
-  Value res = b.create<math::LogOp>(add);
+  Value fma = math::FmaOp::create(b, operand, operand, negOne);
+  Value sqrt = math::SqrtOp::create(b, fma);
+  Value add = arith::AddFOp::create(b, operand, sqrt);
+  Value res = math::LogOp::create(b, add);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -192,13 +192,13 @@ static LogicalResult convertAtanhOp(math::AtanhOp op,
   Type opType = operand.getType();
 
   Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value add = b.create<arith::AddFOp>(operand, one);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value sub = b.create<arith::AddFOp>(neg, one);
-  Value div = b.create<arith::DivFOp>(add, sub);
-  Value log = b.create<math::LogOp>(div);
+  Value add = arith::AddFOp::create(b, operand, one);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value sub = arith::AddFOp::create(b, neg, one);
+  Value div = arith::DivFOp::create(b, add, sub);
+  Value log = math::LogOp::create(b, div);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(log, half);
+  Value res = arith::MulFOp::create(b, log, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -209,8 +209,8 @@ static LogicalResult convertFmaFOp(math::FmaOp op, PatternRewriter &rewriter) {
   Value operandB = op.getOperand(1);
   Value operandC = op.getOperand(2);
   Type type = op.getType();
-  Value mult = b.create<arith::MulFOp>(type, operandA, operandB);
-  Value add = b.create<arith::AddFOp>(type, mult, operandC);
+  Value mult = arith::MulFOp::create(b, type, operandA, operandB);
+  Value add = arith::AddFOp::create(b, type, mult, operandC);
   rewriter.replaceOp(op, add);
   return success();
 }
@@ -235,11 +235,11 @@ static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) {
   Value zero = createFloatConst(op->getLoc(), opType, 0.00, rewriter);
   Value one = createFloatConst(op->getLoc(), opType, 1.00, rewriter);
 
-  Value gtCheck = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand,
+  Value gtCheck = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT, operand,
                                           fpFixedConvert);
-  Value incrValue = b.create<arith::SelectOp>(op->getLoc(), gtCheck, one, zero);
+  Value incrValue = arith::SelectOp::create(b, op->getLoc(), gtCheck, one, zero);
 
-  Value ret = b.create<arith::AddFOp>(opType, fpFixedConvert, incrValue);
+  Value ret = arith::AddFOp::create(b, opType, fpFixedConvert, incrValue);
   rewriter.replaceOp(op, ret);
   return success();
 }
@@ -257,8 +257,8 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
 
   auto convertFPowItoPowf = [&]() -> LogicalResult {
     Value castPowerToFp =
-        rewriter.create<arith::SIToFPOp>(op.getLoc(), baseType, power);
-    Value res = rewriter.create<math::PowFOp>(op.getLoc(), baseType, base,
+        arith::SIToFPOp::create(rewriter, op.getLoc(), baseType, power);
+    Value res = math::PowFOp::create(rewriter, op.getLoc(), baseType, base,
                                               castPowerToFp);
     rewriter.replaceOp(op, res);
     return success();
@@ -280,9 +280,9 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
 
   while (absPower > 0) {
     if (absPower & 1)
-      res = b.create<arith::MulFOp>(baseType, base, res);
+      res = arith::MulFOp::create(b, baseType, base, res);
     absPower >>= 1;
-    base = b.create<arith::MulFOp>(baseType, base, base);
+    base = arith::MulFOp::create(b, baseType, base, base);
   }
 
   // Make sure not to introduce UB in case of negative power.
@@ -302,13 +302,13 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
         createFloatConst(op->getLoc(), baseType,
                          APFloat::getInf(sem, /*Negative=*/true), rewriter);
     Value zeroEqCheck =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, res, zero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, res, zero);
     Value negZeroEqCheck =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, res, negZero);
-    res = b.create<arith::DivFOp>(baseType, one, res);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, res, negZero);
+    res = arith::DivFOp::create(b, baseType, one, res);
     res =
-        b.create<arith::SelectOp>(op->getLoc(), zeroEqCheck, posInfinity, res);
-    res = b.create<arith::SelectOp>(op->getLoc(), negZeroEqCheck, negInfinity,
+        arith::SelectOp::create(b, op->getLoc(), zeroEqCheck, posInfinity, res);
+    res = arith::SelectOp::create(b, op->getLoc(), negZeroEqCheck, negInfinity,
                                     res);
   }
 
@@ -330,7 +330,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       cast<mlir::FloatType>(getElementTypeOrSelf(typeB)).getFloatSemantics();
   APFloat valueB(sem);
   auto mulf = [&](Value x, Value y) -> Value {
-    return b.create<arith::MulFOp>(x, y);
+    return arith::MulFOp::create(b, x, y);
   };
   if (matchPattern(operandB, m_ConstantFloat(&valueB))) {
     if (valueB.isZero()) {
@@ -347,19 +347,19 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
     if (valueB.isExactlyValue(-1.0)) {
       // a^(-1) -> 1 / a
       Value one = createFloatConst(op->getLoc(), typeA, 1.0, rewriter);
-      Value div = b.create<arith::DivFOp>(one, operandA);
+      Value div = arith::DivFOp::create(b, one, operandA);
       rewriter.replaceOp(op, div);
       return success();
     }
     if (valueB.isExactlyValue(0.5)) {
       // a^(1/2) -> sqrt(a)
-      Value sqrt = b.create<math::SqrtOp>(operandA);
+      Value sqrt = math::SqrtOp::create(b, operandA);
       rewriter.replaceOp(op, sqrt);
       return success();
     }
     if (valueB.isExactlyValue(-0.5)) {
       // a^(-1/2) -> 1 / sqrt(a)
-      Value rsqrt = b.create<math::RsqrtOp>(operandA);
+      Value rsqrt = math::RsqrtOp::create(b, operandA);
       rewriter.replaceOp(op, rsqrt);
       return success();
     }
@@ -372,7 +372,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       // a^(-2) -> 1 / (a * a)
       Value one =
           createFloatConst(op->getLoc(), operandA.getType(), 1.0, rewriter);
-      Value div = b.create<arith::DivFOp>(one, mulf(operandA, operandA));
+      Value div = arith::DivFOp::create(b, one, mulf(operandA, operandA));
       rewriter.replaceOp(op, div);
       return success();
     }
@@ -382,9 +382,9 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
     }
   }
 
-  Value logA = b.create<math::LogOp>(operandA);
-  Value mult = b.create<arith::MulFOp>(operandB, logA);
-  Value expResult = b.create<math::ExpOp>(mult);
+  Value logA = math::LogOp::create(b, operandA);
+  Value mult = arith::MulFOp::create(b, operandB, logA);
+  Value expResult = math::ExpOp::create(b, mult);
   rewriter.replaceOp(op, expResult);
   return success();
 }
@@ -399,8 +399,8 @@ static LogicalResult convertExp2fOp(math::Exp2Op op,
   Value operand = op.getOperand();
   Type opType = operand.getType();
   Value ln2 = createFloatConst(op->getLoc(), opType, llvm::numbers::ln2, b);
-  Value mult = b.create<arith::MulFOp>(opType, operand, ln2);
-  Value exp = b.create<math::ExpOp>(op->getLoc(), mult);
+  Value mult = arith::MulFOp::create(b, opType, operand, ln2);
+  Value exp = math::ExpOp::create(b, op->getLoc(), mult);
   rewriter.replaceOp(op, exp);
   return success();
 }
@@ -426,8 +426,8 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   Value c127 = createIntConst(loc, i32Ty, 127, b);
   Value expMask = createIntConst(loc, i32Ty, (1 << 8) - 1, b);
 
-  Value incrValue = b.create<math::CopySignOp>(half, operand);
-  Value add = b.create<arith::AddFOp>(opType, operand, incrValue);
+  Value incrValue = math::CopySignOp::create(b, half, operand);
+  Value add = arith::AddFOp::create(b, opType, operand, incrValue);
   Value fpFixedConvert = createTruncatedFPValue(add, b);
 
   // There are three cases where adding 0.5 to the value and truncating by
@@ -450,14 +450,14 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   //     i64 leading to wrong outputs.
   //
   // All three cases satisfy the property `biasedExp >= 23`.
-  Value operandBitcast = b.create<arith::BitcastOp>(i32Ty, operand);
-  Value operandExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(operandBitcast, c23), expMask);
-  Value operandBiasedExp = b.create<arith::SubIOp>(operandExp, c127);
+  Value operandBitcast = arith::BitcastOp::create(b, i32Ty, operand);
+  Value operandExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
   Value isSpecialValOrLargeVal =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::sge, operandBiasedExp, c23);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::sge, operandBiasedExp, c23);
 
-  Value result = b.create<arith::SelectOp>(isSpecialValOrLargeVal, operand,
+  Value result = arith::SelectOp::create(b, isSpecialValOrLargeVal, operand,
                                            fpFixedConvert);
   rewriter.replaceOp(op, result);
   return success();
@@ -489,20 +489,20 @@ static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
     auto mask = createIntConst(loc, operandTy, allbits >> half, rewriter);
 
     Value pred =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule, x, mask);
-    Value add = rewriter.create<arith::AddIOp>(loc, count, bits);
-    Value shift = rewriter.create<arith::ShLIOp>(loc, x, bits);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ule, x, mask);
+    Value add = arith::AddIOp::create(rewriter, loc, count, bits);
+    Value shift = arith::ShLIOp::create(rewriter, loc, x, bits);
 
-    x = rewriter.create<arith::SelectOp>(loc, pred, shift, x);
-    count = rewriter.create<arith::SelectOp>(loc, pred, add, count);
+    x = arith::SelectOp::create(rewriter, loc, pred, shift, x);
+    count = arith::SelectOp::create(rewriter, loc, pred, add, count);
   }
 
   Value zero = createIntConst(loc, operandTy, 0, rewriter);
-  Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+  Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                               operand, zero);
 
   Value bwval = createIntConst(loc, operandTy, bitwidth, rewriter);
-  Value sel = rewriter.create<arith::SelectOp>(loc, pred, bwval, count);
+  Value sel = arith::SelectOp::create(rewriter, loc, pred, bwval, count);
   rewriter.replaceOp(op, sel);
   return success();
 }
@@ -549,29 +549,29 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   Value c23Mask = createIntConst(loc, iTy, (1ull << mantissaWidth) - 1, b);
   Value expMask = createIntConst(loc, iTy, (1ull << exponentWidth) - 1, b);
 
-  Value operandBitcast = b.create<arith::BitcastOp>(iTy, operand);
-  Value round = b.create<math::RoundOp>(operand);
-  Value roundBitcast = b.create<arith::BitcastOp>(iTy, round);
+  Value operandBitcast = arith::BitcastOp::create(b, iTy, operand);
+  Value round = math::RoundOp::create(b, operand);
+  Value roundBitcast = arith::BitcastOp::create(b, iTy, round);
 
   // Get biased exponents for operand and round(operand)
-  Value operandExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(operandBitcast, c23), expMask);
-  Value operandBiasedExp = b.create<arith::SubIOp>(operandExp, c127);
-  Value roundExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(roundBitcast, c23), expMask);
-  Value roundBiasedExp = b.create<arith::SubIOp>(roundExp, c127);
+  Value operandExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
+  Value roundExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, roundBitcast, c23), expMask);
+  Value roundBiasedExp = arith::SubIOp::create(b, roundExp, c127);
 
   auto safeShiftRight = [&](Value x, Value shift) -> Value {
     // Clamp shift to valid range [0, bitwidth - 1] to avoid undefined behavior
-    Value clampedShift = b.create<arith::MaxSIOp>(shift, c0);
-    clampedShift = b.create<arith::MinSIOp>(clampedShift, c31);
-    return b.create<arith::ShRUIOp>(x, clampedShift);
+    Value clampedShift = arith::MaxSIOp::create(b, shift, c0);
+    clampedShift = arith::MinSIOp::create(b, clampedShift, c31);
+    return arith::ShRUIOp::create(b, x, clampedShift);
   };
 
   auto maskMantissa = [&](Value mantissa,
                           Value mantissaMaskRightShift) -> Value {
     Value shiftedMantissaMask = safeShiftRight(c23Mask, mantissaMaskRightShift);
-    return b.create<arith::AndIOp>(mantissa, shiftedMantissaMask);
+    return arith::AndIOp::create(b, mantissa, shiftedMantissaMask);
   };
 
   // A whole number `x`, such that `|x| != 1`, is even if the mantissa, ignoring
@@ -589,13 +589,13 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // `biasedExp > 23`, so they get treated as large numbers with no room for
   // decimals, which are always even.
   Value roundBiasedExpEq0 =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, roundBiasedExp, c0);
-  Value roundBiasedExpMinus1 = b.create<arith::SubIOp>(roundBiasedExp, c1);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, roundBiasedExp, c0);
+  Value roundBiasedExpMinus1 = arith::SubIOp::create(b, roundBiasedExp, c1);
   Value roundMaskedMantissa = maskMantissa(roundBitcast, roundBiasedExpMinus1);
-  Value roundIsNotEvenOrSpecialVal = b.create<arith::CmpIOp>(
+  Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(b,
       arith::CmpIPredicate::ne, roundMaskedMantissa, c0);
   roundIsNotEvenOrSpecialVal =
-      b.create<arith::OrIOp>(roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
+      arith::OrIOp::create(b, roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
 
   // A value `x` with `0 <= biasedExp < 23`, is halfway between two consecutive
   // integers if the bit at index `biasedExp` starting from the left in the
@@ -604,37 +604,37 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // values +-0.5 are the only halfway values that have `biasedExp == -1 < 0`,
   // so these are handled separately. In particular, if `biasedExp == -1`, the
   // value is halfway if the entire mantissa is zero.
-  Value operandBiasedExpEqNeg1 = b.create<arith::CmpIOp>(
+  Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(b,
       arith::CmpIPredicate::eq, operandBiasedExp, cNeg1);
-  Value expectedOperandMaskedMantissa = b.create<arith::SelectOp>(
+  Value expectedOperandMaskedMantissa = arith::SelectOp::create(b,
       operandBiasedExpEqNeg1, c0, safeShiftRight(c2To22, operandBiasedExp));
   Value operandMaskedMantissa = maskMantissa(operandBitcast, operandBiasedExp);
   Value operandIsHalfway =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, operandMaskedMantissa,
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, operandMaskedMantissa,
                               expectedOperandMaskedMantissa);
   // Ensure `biasedExp` is in the valid range for half values.
-  Value operandBiasedExpGeNeg1 = b.create<arith::CmpIOp>(
+  Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(b,
       arith::CmpIPredicate::sge, operandBiasedExp, cNeg1);
   Value operandBiasedExpLt23 =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::slt, operandBiasedExp, c23);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, operandBiasedExp, c23);
   operandIsHalfway =
-      b.create<arith::AndIOp>(operandIsHalfway, operandBiasedExpLt23);
+      arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpLt23);
   operandIsHalfway =
-      b.create<arith::AndIOp>(operandIsHalfway, operandBiasedExpGeNeg1);
+      arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpGeNeg1);
 
   // Adjust rounded operand with `round(operand) - sign(operand)` to correct the
   // case where `round` rounded in the opposite direction of `roundeven`.
-  Value sign = b.create<math::CopySignOp>(c1Float, operand);
-  Value roundShifted = b.create<arith::SubFOp>(round, sign);
+  Value sign = math::CopySignOp::create(b, c1Float, operand);
+  Value roundShifted = arith::SubFOp::create(b, round, sign);
   // If the rounded value is even or a special value, we default to the behavior
   // of `math.round`.
   Value needsShift =
-      b.create<arith::AndIOp>(roundIsNotEvenOrSpecialVal, operandIsHalfway);
-  Value result = b.create<arith::SelectOp>(needsShift, roundShifted, round);
+      arith::AndIOp::create(b, roundIsNotEvenOrSpecialVal, operandIsHalfway);
+  Value result = arith::SelectOp::create(b, needsShift, roundShifted, round);
   // The `x - sign` adjustment does not preserve the sign when we are adjusting
   // the value -1 to -0. So here the sign is copied again to ensure that -0.5 is
   // rounded to -0.0.
-  result = b.create<math::CopySignOp>(result, operand);
+  result = math::CopySignOp::create(b, result, operand);
   rewriter.replaceOp(op, result);
   return success();
 }
@@ -656,7 +656,7 @@ static LogicalResult convertRsqrtOp(math::RsqrtOp op,
 
   Location loc = op->getLoc();
   auto constOneFloat = createFloatConst(loc, operandTy, 1.0, rewriter);
-  auto sqrtOp = rewriter.create<math::SqrtOp>(loc, operand);
+  auto sqrtOp = math::SqrtOp::create(rewriter, loc, operand);
   rewriter.replaceOpWithNewOp<arith::DivFOp>(op, constOneFloat, sqrtOp);
   return success();
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
index a570ed5118ef0..9d6ad613fc945 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
@@ -73,7 +73,7 @@ void mlir::math::populateExtendToSupportedTypesTypeConverter(
       });
   typeConverter.addTargetMaterialization(
       [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        auto extFOp = arith::ExtFOp::create(b, loc, target, input);
         extFOp.setFastmath(arith::FastMathFlags::contract);
         return extFOp;
       });
@@ -104,7 +104,7 @@ LogicalResult ExtendToSupportedTypesRewritePattern::matchAndRewrite(
   for (auto [result, newType, origType] : llvm::zip_equal(
            results, (*legalized)->getResultTypes(), op->getResultTypes())) {
     if (newType != origType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, origType, result);
+      auto truncFOp = arith::TruncFOp::create(rewriter, loc, origType, result);
       truncFOp.setFastmath(arith::FastMathFlags::contract);
       result = truncFOp.getResult();
     }
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index dd2dfe372b683..baa35fb5a2c5d 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -72,7 +72,7 @@ static Value broadcast(ImplicitLocOpBuilder &builder, Value value,
                        std::optional<VectorShape> shape) {
   assert(!isa<VectorType>(value.getType()) && "must be scalar value");
   auto type = broadcast(value.getType(), shape);
-  return shape ? builder.create<BroadcastOp>(type, value) : value;
+  return shape ? BroadcastOp::create(builder, type, value) : value;
 }
 
 //----------------------------------------------------------------------------//
@@ -130,7 +130,7 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
       auto eltType = cast<VectorType>(operand.getType()).getElementType();
       auto expandedType = VectorType::get(expandedShape, eltType);
       expandedOperands[i] =
-          builder.create<vector::ShapeCastOp>(expandedType, operand);
+          vector::ShapeCastOp::create(builder, expandedType, operand);
     }
   }
 
@@ -148,7 +148,7 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
     SmallVector<Value> extracted(expandedOperands.size());
     for (const auto &tuple : llvm::enumerate(expandedOperands))
       extracted[tuple.index()] =
-          builder.create<vector::ExtractOp>(tuple.value(), offsets);
+          vector::ExtractOp::create(builder, tuple.value(), offsets);
 
     results[i] = compute(extracted);
   }
@@ -156,15 +156,15 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
   // Stitch results together into one large vector.
   Type resultEltType = cast<VectorType>(results[0].getType()).getElementType();
   Type resultExpandedType = VectorType::get(expandedShape, resultEltType);
-  Value result = builder.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(builder,
       resultExpandedType, builder.getZeroAttr(resultExpandedType));
 
   for (int64_t i = 0; i < maxIndex; ++i)
-    result = builder.create<vector::InsertOp>(results[i], result,
+    result = vector::InsertOp::create(builder, results[i], result,
                                               delinearize(i, strides));
 
   // Reshape back to the original vector shape.
-  return builder.create<vector::ShapeCastOp>(
+  return vector::ShapeCastOp::create(builder,
       VectorType::get(inputShape, resultEltType), result);
 }
 
@@ -173,28 +173,28 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
 //----------------------------------------------------------------------------//
 
 static Value boolCst(ImplicitLocOpBuilder &builder, bool value) {
-  return builder.create<arith::ConstantOp>(builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, builder.getBoolAttr(value));
 }
 
 static Value floatCst(ImplicitLocOpBuilder &builder, float value,
                       Type elementType) {
   assert((elementType.isF16() || elementType.isF32()) &&
          "x must be f16 or f32 type.");
-  return builder.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(builder,
       builder.getFloatAttr(elementType, value));
 }
 
 static Value f32Cst(ImplicitLocOpBuilder &builder, double value) {
-  return builder.create<arith::ConstantOp>(builder.getF32FloatAttr(value));
+  return arith::ConstantOp::create(builder, builder.getF32FloatAttr(value));
 }
 
 static Value i32Cst(ImplicitLocOpBuilder &builder, int32_t value) {
-  return builder.create<arith::ConstantOp>(builder.getI32IntegerAttr(value));
+  return arith::ConstantOp::create(builder, builder.getI32IntegerAttr(value));
 }
 
 static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
   Value i32Value = i32Cst(builder, static_cast<int32_t>(bits));
-  return builder.create<arith::BitcastOp>(builder.getF32Type(), i32Value);
+  return arith::BitcastOp::create(builder, builder.getF32Type(), i32Value);
 }
 
 //----------------------------------------------------------------------------//
@@ -203,15 +203,15 @@ static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
 
 // Return the minimum of the two values or NaN if value is NaN
 static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return builder.create<arith::SelectOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT, value, bound),
+  return arith::SelectOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT, value, bound),
       value, bound);
 }
 
 // Return the maximum of the two values or NaN if value is NaN
 static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return builder.create<arith::SelectOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::UGT, value, bound),
+  return arith::SelectOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::UGT, value, bound),
       value, bound);
 }
 
@@ -241,24 +241,24 @@ static std::pair<Value, Value> frexp(ImplicitLocOpBuilder &builder, Value arg,
   Value cstInvMantMask = f32FromBits(builder, ~0x7f800000u);
 
   // Bitcast to i32 for bitwise operations.
-  Value i32Half = builder.create<arith::BitcastOp>(i32, cstHalf);
-  Value i32InvMantMask = builder.create<arith::BitcastOp>(i32, cstInvMantMask);
-  Value i32Arg = builder.create<arith::BitcastOp>(i32Vec, arg);
+  Value i32Half = arith::BitcastOp::create(builder, i32, cstHalf);
+  Value i32InvMantMask = arith::BitcastOp::create(builder, i32, cstInvMantMask);
+  Value i32Arg = arith::BitcastOp::create(builder, i32Vec, arg);
 
   // Compute normalized fraction.
-  Value tmp0 = builder.create<arith::AndIOp>(i32Arg, bcast(i32InvMantMask));
-  Value tmp1 = builder.create<arith::OrIOp>(tmp0, bcast(i32Half));
-  Value normalizedFraction = builder.create<arith::BitcastOp>(f32Vec, tmp1);
+  Value tmp0 = arith::AndIOp::create(builder, i32Arg, bcast(i32InvMantMask));
+  Value tmp1 = arith::OrIOp::create(builder, tmp0, bcast(i32Half));
+  Value normalizedFraction = arith::BitcastOp::create(builder, f32Vec, tmp1);
 
   // Compute exponent.
-  Value arg0 = isPositive ? arg : builder.create<math::AbsFOp>(arg);
-  Value biasedExponentBits = builder.create<arith::ShRUIOp>(
-      builder.create<arith::BitcastOp>(i32Vec, arg0),
+  Value arg0 = isPositive ? arg : math::AbsFOp::create(builder, arg);
+  Value biasedExponentBits = arith::ShRUIOp::create(builder,
+      arith::BitcastOp::create(builder, i32Vec, arg0),
       bcast(i32Cst(builder, 23)));
   Value biasedExponent =
-      builder.create<arith::SIToFPOp>(f32Vec, biasedExponentBits);
+      arith::SIToFPOp::create(builder, f32Vec, biasedExponentBits);
   Value exponent =
-      builder.create<arith::SubFOp>(biasedExponent, bcast(cst126f));
+      arith::SubFOp::create(builder, biasedExponent, bcast(cst126f));
 
   return {normalizedFraction, exponent};
 }
@@ -278,10 +278,10 @@ static Value exp2I32(ImplicitLocOpBuilder &builder, Value arg) {
   // Set the exponent bias to zero.
   auto bias = bcast(i32Cst(builder, 127));
 
-  Value biasedArg = builder.create<arith::AddIOp>(arg, bias);
+  Value biasedArg = arith::AddIOp::create(builder, arg, bias);
   Value exp2ValueInt =
-      builder.create<arith::ShLIOp>(biasedArg, exponetBitLocation);
-  Value exp2ValueF32 = builder.create<arith::BitcastOp>(f32Vec, exp2ValueInt);
+      arith::ShLIOp::create(builder, biasedArg, exponetBitLocation);
+  Value exp2ValueF32 = arith::BitcastOp::create(builder, f32Vec, exp2ValueInt);
 
   return exp2ValueF32;
 }
@@ -300,10 +300,10 @@ Value makePolynomialCalculation(ImplicitLocOpBuilder &builder,
   if (coeffs.size() == 1)
     return coeffs[0];
 
-  Value res = builder.create<math::FmaOp>(x, coeffs[coeffs.size() - 1],
+  Value res = math::FmaOp::create(builder, x, coeffs[coeffs.size() - 1],
                                           coeffs[coeffs.size() - 2]);
   for (auto i = ptrdiff_t(coeffs.size()) - 3; i >= 0; --i) {
-    res = builder.create<math::FmaOp>(x, res, coeffs[i]);
+    res = math::FmaOp::create(builder, x, res, coeffs[i]);
   }
   return res;
 }
@@ -343,9 +343,9 @@ LogicalResult insertCasts(Operation *op, PatternRewriter &rewriter) {
   Location loc = op->getLoc();
   SmallVector<Value> operands;
   for (auto operand : op->getOperands())
-    operands.push_back(rewriter.create<arith::ExtFOp>(loc, newType, operand));
+    operands.push_back(arith::ExtFOp::create(rewriter, loc, newType, operand));
   auto result =
-      rewriter.create<T>(loc, TypeRange{newType}, operands, op->getAttrs());
+      T::create(rewriter, loc, TypeRange{newType}, operands, op->getAttrs());
   rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, origType, result);
   return success();
 }
@@ -393,18 +393,18 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
   std::optional<VectorShape> shape = vectorShape(op.getOperand());
 
   ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
-  Value abs = builder.create<math::AbsFOp>(operand);
+  Value abs = math::AbsFOp::create(builder, operand);
 
   auto one = broadcast(builder, f32Cst(builder, 1.0), shape);
 
   // When 0.66 < x <= 2.41 we do (x-1) / (x+1):
   auto twoThirds = broadcast(builder, f32Cst(builder, 0.66), shape);
   Value cmp2 =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, abs, twoThirds);
-  Value addone = builder.create<arith::AddFOp>(abs, one);
-  Value subone = builder.create<arith::SubFOp>(abs, one);
-  Value xnum = builder.create<arith::SelectOp>(cmp2, subone, abs);
-  Value xden = builder.create<arith::SelectOp>(cmp2, addone, one);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, abs, twoThirds);
+  Value addone = arith::AddFOp::create(builder, abs, one);
+  Value subone = arith::SubFOp::create(builder, abs, one);
+  Value xnum = arith::SelectOp::create(builder, cmp2, subone, abs);
+  Value xden = arith::SelectOp::create(builder, cmp2, addone, one);
 
   auto bcast = [&](Value value) -> Value {
     return broadcast(builder, value, shape);
@@ -413,12 +413,12 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
   // Break into the <= 0.66 or > 2.41 we do x or 1/x:
   auto tan3pio8 = bcast(f32Cst(builder, 2.41421356237309504880));
   Value cmp1 =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, abs, tan3pio8);
-  xnum = builder.create<arith::SelectOp>(cmp1, one, xnum);
-  xden = builder.create<arith::SelectOp>(cmp1, abs, xden);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, abs, tan3pio8);
+  xnum = arith::SelectOp::create(builder, cmp1, one, xnum);
+  xden = arith::SelectOp::create(builder, cmp1, abs, xden);
 
-  Value x = builder.create<arith::DivFOp>(xnum, xden);
-  Value xx = builder.create<arith::MulFOp>(x, x);
+  Value x = arith::DivFOp::create(builder, xnum, xden);
+  Value xx = arith::MulFOp::create(builder, x, x);
 
   // Perform the Taylor series approximation for atan over the range
   // [0.0, 0.66].
@@ -435,31 +435,31 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
 
   // Apply the polynomial approximation for the numerator:
   Value n = p0;
-  n = builder.create<math::FmaOp>(xx, n, p1);
-  n = builder.create<math::FmaOp>(xx, n, p2);
-  n = builder.create<math::FmaOp>(xx, n, p3);
-  n = builder.create<math::FmaOp>(xx, n, p4);
-  n = builder.create<arith::MulFOp>(n, xx);
+  n = math::FmaOp::create(builder, xx, n, p1);
+  n = math::FmaOp::create(builder, xx, n, p2);
+  n = math::FmaOp::create(builder, xx, n, p3);
+  n = math::FmaOp::create(builder, xx, n, p4);
+  n = arith::MulFOp::create(builder, n, xx);
 
   // Apply the polynomial approximation for the denominator:
   Value d = q0;
-  d = builder.create<math::FmaOp>(xx, d, q1);
-  d = builder.create<math::FmaOp>(xx, d, q2);
-  d = builder.create<math::FmaOp>(xx, d, q3);
-  d = builder.create<math::FmaOp>(xx, d, q4);
+  d = math::FmaOp::create(builder, xx, d, q1);
+  d = math::FmaOp::create(builder, xx, d, q2);
+  d = math::FmaOp::create(builder, xx, d, q3);
+  d = math::FmaOp::create(builder, xx, d, q4);
 
   // Compute approximation of theta:
-  Value ans0 = builder.create<arith::DivFOp>(n, d);
-  ans0 = builder.create<math::FmaOp>(ans0, x, x);
+  Value ans0 = arith::DivFOp::create(builder, n, d);
+  ans0 = math::FmaOp::create(builder, ans0, x, x);
 
   // Correct for the input mapping's angles:
   Value mpi4 = bcast(f32Cst(builder, llvm::numbers::pi / 4));
-  Value ans2 = builder.create<arith::AddFOp>(mpi4, ans0);
-  Value ans = builder.create<arith::SelectOp>(cmp2, ans2, ans0);
+  Value ans2 = arith::AddFOp::create(builder, mpi4, ans0);
+  Value ans = arith::SelectOp::create(builder, cmp2, ans2, ans0);
 
   Value mpi2 = bcast(f32Cst(builder, llvm::numbers::pi / 2));
-  Value ans1 = builder.create<arith::SubFOp>(mpi2, ans0);
-  ans = builder.create<arith::SelectOp>(cmp1, ans1, ans);
+  Value ans1 = arith::SubFOp::create(builder, mpi2, ans0);
+  ans = arith::SelectOp::create(builder, cmp1, ans1, ans);
 
   // Correct for signing of the input.
   rewriter.replaceOpWithNewOp<math::CopySignOp>(op, ans, operand);
@@ -492,44 +492,44 @@ Atan2Approximation::matchAndRewrite(math::Atan2Op op,
   std::optional<VectorShape> shape = vectorShape(op.getResult());
 
   // Compute atan in the valid range.
-  auto div = builder.create<arith::DivFOp>(y, x);
-  auto atan = builder.create<math::AtanOp>(div);
+  auto div = arith::DivFOp::create(builder, y, x);
+  auto atan = math::AtanOp::create(builder, div);
 
   // Determine what the atan would be for a 180 degree rotation.
   auto zero = broadcast(builder, f32Cst(builder, 0.0f), shape);
   auto pi = broadcast(builder, f32Cst(builder, 3.14159265359f), shape);
-  auto addPi = builder.create<arith::AddFOp>(atan, pi);
-  auto subPi = builder.create<arith::SubFOp>(atan, pi);
+  auto addPi = arith::AddFOp::create(builder, atan, pi);
+  auto subPi = arith::SubFOp::create(builder, atan, pi);
   auto atanGt =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, atan, zero);
-  auto flippedAtan = builder.create<arith::SelectOp>(atanGt, subPi, addPi);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, atan, zero);
+  auto flippedAtan = arith::SelectOp::create(builder, atanGt, subPi, addPi);
 
   // Determine whether to directly use atan or use the 180 degree flip
-  auto xGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, x, zero);
-  Value result = builder.create<arith::SelectOp>(xGt, atan, flippedAtan);
+  auto xGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, x, zero);
+  Value result = arith::SelectOp::create(builder, xGt, atan, flippedAtan);
 
   // Handle x = 0, y > 0
   Value xZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, x, zero);
-  Value yGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, y, zero);
-  Value isHalfPi = builder.create<arith::AndIOp>(xZero, yGt);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, x, zero);
+  Value yGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, y, zero);
+  Value isHalfPi = arith::AndIOp::create(builder, xZero, yGt);
   auto halfPi = broadcast(builder, f32Cst(builder, 1.57079632679f), shape);
-  result = builder.create<arith::SelectOp>(isHalfPi, halfPi, result);
+  result = arith::SelectOp::create(builder, isHalfPi, halfPi, result);
 
   // Handle x = 0, y < 0
-  Value yLt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, y, zero);
-  Value isNegativeHalfPiPi = builder.create<arith::AndIOp>(xZero, yLt);
+  Value yLt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, y, zero);
+  Value isNegativeHalfPiPi = arith::AndIOp::create(builder, xZero, yLt);
   auto negativeHalfPiPi =
       broadcast(builder, f32Cst(builder, -1.57079632679f), shape);
-  result = builder.create<arith::SelectOp>(isNegativeHalfPiPi, negativeHalfPiPi,
+  result = arith::SelectOp::create(builder, isNegativeHalfPiPi, negativeHalfPiPi,
                                            result);
 
   // Handle x = 0, y = 0;
   Value yZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, y, zero);
-  Value isNan = builder.create<arith::AndIOp>(xZero, yZero);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, y, zero);
+  Value isNan = arith::AndIOp::create(builder, xZero, yZero);
   Value cstNan = broadcast(builder, f32FromBits(builder, 0x7fc00000), shape);
-  result = builder.create<arith::SelectOp>(isNan, cstNan, result);
+  result = arith::SelectOp::create(builder, isNan, cstNan, result);
 
   rewriter.replaceOp(op, result);
   return success();
@@ -569,8 +569,8 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
 
   // Mask for tiny values that are approximated with `operand`.
   Value tiny = bcast(f32Cst(builder, 0.0004f));
-  Value tinyMask = builder.create<arith::CmpFOp>(
-      arith::CmpFPredicate::OLT, builder.create<math::AbsFOp>(op.getOperand()),
+  Value tinyMask = arith::CmpFOp::create(builder,
+      arith::CmpFPredicate::OLT, math::AbsFOp::create(builder, op.getOperand()),
       tiny);
 
   // The monomial coefficients of the numerator polynomial (odd).
@@ -589,25 +589,25 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
   Value beta6 = bcast(f32Cst(builder, 1.19825839466702e-06f));
 
   // Since the polynomials are odd/even, we need x^2.
-  Value x2 = builder.create<arith::MulFOp>(x, x);
+  Value x2 = arith::MulFOp::create(builder, x, x);
 
   // Evaluate the numerator polynomial p.
-  Value p = builder.create<math::FmaOp>(x2, alpha13, alpha11);
-  p = builder.create<math::FmaOp>(x2, p, alpha9);
-  p = builder.create<math::FmaOp>(x2, p, alpha7);
-  p = builder.create<math::FmaOp>(x2, p, alpha5);
-  p = builder.create<math::FmaOp>(x2, p, alpha3);
-  p = builder.create<math::FmaOp>(x2, p, alpha1);
-  p = builder.create<arith::MulFOp>(x, p);
+  Value p = math::FmaOp::create(builder, x2, alpha13, alpha11);
+  p = math::FmaOp::create(builder, x2, p, alpha9);
+  p = math::FmaOp::create(builder, x2, p, alpha7);
+  p = math::FmaOp::create(builder, x2, p, alpha5);
+  p = math::FmaOp::create(builder, x2, p, alpha3);
+  p = math::FmaOp::create(builder, x2, p, alpha1);
+  p = arith::MulFOp::create(builder, x, p);
 
   // Evaluate the denominator polynomial q.
-  Value q = builder.create<math::FmaOp>(x2, beta6, beta4);
-  q = builder.create<math::FmaOp>(x2, q, beta2);
-  q = builder.create<math::FmaOp>(x2, q, beta0);
+  Value q = math::FmaOp::create(builder, x2, beta6, beta4);
+  q = math::FmaOp::create(builder, x2, q, beta2);
+  q = math::FmaOp::create(builder, x2, q, beta0);
 
   // Divide the numerator by the denominator.
-  Value res = builder.create<arith::SelectOp>(
-      tinyMask, x, builder.create<arith::DivFOp>(p, q));
+  Value res = arith::SelectOp::create(builder,
+      tinyMask, x, arith::DivFOp::create(builder, p, q));
 
   rewriter.replaceOp(op, res);
 
@@ -690,57 +690,57 @@ LogApproximationBase<Op>::logMatchAndRewrite(Op op, PatternRewriter &rewriter,
   //     e -= 1;
   //     x = x + x - 1.0;
   //   } else { x = x - 1.0; }
-  Value mask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x,
+  Value mask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x,
                                              cstCephesSQRTHF);
-  Value tmp = builder.create<arith::SelectOp>(mask, x, cstZero);
+  Value tmp = arith::SelectOp::create(builder, mask, x, cstZero);
 
-  x = builder.create<arith::SubFOp>(x, cstOne);
-  e = builder.create<arith::SubFOp>(
-      e, builder.create<arith::SelectOp>(mask, cstOne, cstZero));
-  x = builder.create<arith::AddFOp>(x, tmp);
+  x = arith::SubFOp::create(builder, x, cstOne);
+  e = arith::SubFOp::create(builder,
+      e, arith::SelectOp::create(builder, mask, cstOne, cstZero));
+  x = arith::AddFOp::create(builder, x, tmp);
 
-  Value x2 = builder.create<arith::MulFOp>(x, x);
-  Value x3 = builder.create<arith::MulFOp>(x2, x);
+  Value x2 = arith::MulFOp::create(builder, x, x);
+  Value x3 = arith::MulFOp::create(builder, x2, x);
 
   // Evaluate the polynomial approximant of degree 8 in three parts.
   Value y0, y1, y2;
-  y0 = builder.create<math::FmaOp>(cstCephesLogP0, x, cstCephesLogP1);
-  y1 = builder.create<math::FmaOp>(cstCephesLogP3, x, cstCephesLogP4);
-  y2 = builder.create<math::FmaOp>(cstCephesLogP6, x, cstCephesLogP7);
-  y0 = builder.create<math::FmaOp>(y0, x, cstCephesLogP2);
-  y1 = builder.create<math::FmaOp>(y1, x, cstCephesLogP5);
-  y2 = builder.create<math::FmaOp>(y2, x, cstCephesLogP8);
-  y0 = builder.create<math::FmaOp>(y0, x3, y1);
-  y0 = builder.create<math::FmaOp>(y0, x3, y2);
-  y0 = builder.create<arith::MulFOp>(y0, x3);
-
-  y0 = builder.create<math::FmaOp>(cstNegHalf, x2, y0);
-  x = builder.create<arith::AddFOp>(x, y0);
+  y0 = math::FmaOp::create(builder, cstCephesLogP0, x, cstCephesLogP1);
+  y1 = math::FmaOp::create(builder, cstCephesLogP3, x, cstCephesLogP4);
+  y2 = math::FmaOp::create(builder, cstCephesLogP6, x, cstCephesLogP7);
+  y0 = math::FmaOp::create(builder, y0, x, cstCephesLogP2);
+  y1 = math::FmaOp::create(builder, y1, x, cstCephesLogP5);
+  y2 = math::FmaOp::create(builder, y2, x, cstCephesLogP8);
+  y0 = math::FmaOp::create(builder, y0, x3, y1);
+  y0 = math::FmaOp::create(builder, y0, x3, y2);
+  y0 = arith::MulFOp::create(builder, y0, x3);
+
+  y0 = math::FmaOp::create(builder, cstNegHalf, x2, y0);
+  x = arith::AddFOp::create(builder, x, y0);
 
   if (base2) {
     Value cstLog2e = bcast(f32Cst(builder, static_cast<float>(LOG2E_VALUE)));
-    x = builder.create<math::FmaOp>(x, cstLog2e, e);
+    x = math::FmaOp::create(builder, x, cstLog2e, e);
   } else {
     Value cstLn2 = bcast(f32Cst(builder, static_cast<float>(LN2_VALUE)));
-    x = builder.create<math::FmaOp>(e, cstLn2, x);
+    x = math::FmaOp::create(builder, e, cstLn2, x);
   }
 
-  Value invalidMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT,
+  Value invalidMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT,
                                                     op.getOperand(), cstZero);
-  Value zeroMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value zeroMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                  op.getOperand(), cstZero);
-  Value posInfMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value posInfMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                    op.getOperand(), cstPosInf);
 
   // Filter out invalid values:
   //  • x == 0     -> -INF
   //  • x < 0      ->  NAN
   //  • x == +INF  -> +INF
-  Value aproximation = builder.create<arith::SelectOp>(
+  Value aproximation = arith::SelectOp::create(builder,
       zeroMask, cstMinusInf,
-      builder.create<arith::SelectOp>(
+      arith::SelectOp::create(builder,
           invalidMask, cstNan,
-          builder.create<arith::SelectOp>(posInfMask, cstPosInf, x)));
+          arith::SelectOp::create(builder, posInfMask, cstPosInf, x)));
 
   rewriter.replaceOp(op, aproximation);
 
@@ -805,17 +805,17 @@ Log1pApproximation::matchAndRewrite(math::Log1pOp op,
   //             "logLarge" below.
   Value cstOne = bcast(f32Cst(builder, 1.0f));
   Value x = op.getOperand();
-  Value u = builder.create<arith::AddFOp>(x, cstOne);
+  Value u = arith::AddFOp::create(builder, x, cstOne);
   Value uSmall =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, cstOne);
-  Value logU = builder.create<math::LogOp>(u);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, cstOne);
+  Value logU = math::LogOp::create(builder, u);
   Value uInf =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, logU);
-  Value logLarge = builder.create<arith::MulFOp>(
-      x, builder.create<arith::DivFOp>(
-             logU, builder.create<arith::SubFOp>(u, cstOne)));
-  Value approximation = builder.create<arith::SelectOp>(
-      builder.create<arith::OrIOp>(uSmall, uInf), x, logLarge);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, logU);
+  Value logLarge = arith::MulFOp::create(builder,
+      x, arith::DivFOp::create(builder,
+             logU, arith::SubFOp::create(builder, u, cstOne)));
+  Value approximation = arith::SelectOp::create(builder,
+      arith::OrIOp::create(builder, uSmall, uInf), x, logLarge);
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -853,27 +853,27 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   };
 
   auto fma = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
   auto sub = [&](Value a, Value b) -> Value {
-    return builder.create<arith::SubFOp>(a, b);
+    return arith::SubFOp::create(builder, a, b);
   };
 
-  auto abs = [&](Value a) -> Value { return builder.create<math::AbsFOp>(a); };
+  auto abs = [&](Value a) -> Value { return math::AbsFOp::create(builder, a); };
 
-  auto sqrt = [&](Value a) -> Value { return builder.create<math::SqrtOp>(a); };
+  auto sqrt = [&](Value a) -> Value { return math::SqrtOp::create(builder, a); };
 
   auto scopy = [&](Value a, Value b) -> Value {
-    return builder.create<math::CopySignOp>(a, b);
+    return math::CopySignOp::create(builder, a, b);
   };
 
   auto sel = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<arith::SelectOp>(a, b, c);
+    return arith::SelectOp::create(builder, a, b, c);
   };
 
   Value abso = abs(operand);
@@ -881,7 +881,7 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   Value opp = sqrt(sub(bcast(floatCst(builder, 1.0, elementType)), aa));
 
   Value gt =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, aa,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, aa,
                                     bcast(floatCst(builder, 0.5, elementType)));
 
   Value x = sel(gt, opp, abso);
@@ -948,51 +948,51 @@ AcosPolynomialApproximation::matchAndRewrite(math::AcosOp op,
   };
 
   auto fma = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
-  Value negOperand = builder.create<arith::NegFOp>(operand);
+  Value negOperand = arith::NegFOp::create(builder, operand);
   Value zero = bcast(floatCst(builder, 0.0, elementType));
   Value half = bcast(floatCst(builder, 0.5, elementType));
   Value negOne = bcast(floatCst(builder, -1.0, elementType));
   Value selR =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand, zero);
-  Value r = builder.create<arith::SelectOp>(selR, negOperand, operand);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, operand, zero);
+  Value r = arith::SelectOp::create(builder, selR, negOperand, operand);
   Value chkConst = bcast(floatCst(builder, -0.5625, elementType));
   Value firstPred =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, r, chkConst);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, r, chkConst);
 
   Value trueVal =
       fma(bcast(floatCst(builder, 9.3282184640716537e-1, elementType)),
           bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
-          builder.create<math::AsinOp>(r));
+          math::AsinOp::create(builder, r));
 
-  Value falseVal = builder.create<math::SqrtOp>(fma(half, r, half));
-  falseVal = builder.create<math::AsinOp>(falseVal);
+  Value falseVal = math::SqrtOp::create(builder, fma(half, r, half));
+  falseVal = math::AsinOp::create(builder, falseVal);
   falseVal = mul(bcast(floatCst(builder, 2.0, elementType)), falseVal);
 
-  r = builder.create<arith::SelectOp>(firstPred, trueVal, falseVal);
+  r = arith::SelectOp::create(builder, firstPred, trueVal, falseVal);
 
   // Check whether the operand lies in between [-1.0, 0.0).
   Value greaterThanNegOne =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, operand, negOne);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGE, operand, negOne);
 
   Value lessThanZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
 
   Value betweenNegOneZero =
-      builder.create<arith::AndIOp>(greaterThanNegOne, lessThanZero);
+      arith::AndIOp::create(builder, greaterThanNegOne, lessThanZero);
 
   trueVal = fma(bcast(floatCst(builder, 1.8656436928143307e+0, elementType)),
                 bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
-                builder.create<arith::NegFOp>(r));
+                arith::NegFOp::create(builder, r));
 
   Value finalVal =
-      builder.create<arith::SelectOp>(betweenNegOneZero, trueVal, r);
+      arith::SelectOp::create(builder, betweenNegOneZero, trueVal, r);
 
   rewriter.replaceOp(op, finalVal);
   return success();
@@ -1075,9 +1075,9 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
   bounds[2] = bcast(floatCst(builder, 3.75f, elementType));
 
   Value isNegativeArg =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
-  Value negArg = builder.create<arith::NegFOp>(operand);
-  Value x = builder.create<arith::SelectOp>(isNegativeArg, negArg, operand);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
+  Value negArg = arith::NegFOp::create(builder, operand);
+  Value x = arith::SelectOp::create(builder, isNegativeArg, negArg, operand);
 
   Value offset = offsets[0];
   Value p[polyDegree + 1];
@@ -1091,30 +1091,30 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
   Value isLessThanBound[intervalsCount];
   for (int j = 0; j < intervalsCount - 1; ++j) {
     isLessThanBound[j] =
-        builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, bounds[j]);
+        arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, bounds[j]);
     for (int i = 0; i <= polyDegree; ++i) {
-      p[i] = builder.create<arith::SelectOp>(isLessThanBound[j], p[i],
+      p[i] = arith::SelectOp::create(builder, isLessThanBound[j], p[i],
                                              pp[j + 1][i]);
-      q[i] = builder.create<arith::SelectOp>(isLessThanBound[j], q[i],
+      q[i] = arith::SelectOp::create(builder, isLessThanBound[j], q[i],
                                              qq[j + 1][i]);
     }
-    offset = builder.create<arith::SelectOp>(isLessThanBound[j], offset,
+    offset = arith::SelectOp::create(builder, isLessThanBound[j], offset,
                                              offsets[j + 1]);
   }
-  isLessThanBound[intervalsCount - 1] = builder.create<arith::CmpFOp>(
+  isLessThanBound[intervalsCount - 1] = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::ULT, x, bounds[intervalsCount - 1]);
 
   Value pPoly = makePolynomialCalculation(builder, p, x);
   Value qPoly = makePolynomialCalculation(builder, q, x);
-  Value rationalPoly = builder.create<arith::DivFOp>(pPoly, qPoly);
-  Value formula = builder.create<arith::AddFOp>(offset, rationalPoly);
-  formula = builder.create<arith::SelectOp>(isLessThanBound[intervalsCount - 1],
+  Value rationalPoly = arith::DivFOp::create(builder, pPoly, qPoly);
+  Value formula = arith::AddFOp::create(builder, offset, rationalPoly);
+  formula = arith::SelectOp::create(builder, isLessThanBound[intervalsCount - 1],
                                             formula, one);
 
   // erf is odd function: erf(x) = -erf(-x).
-  Value negFormula = builder.create<arith::NegFOp>(formula);
+  Value negFormula = arith::NegFOp::create(builder, formula);
   Value res =
-      builder.create<arith::SelectOp>(isNegativeArg, negFormula, formula);
+      arith::SelectOp::create(builder, isNegativeArg, negFormula, formula);
 
   rewriter.replaceOp(op, res);
 
@@ -1155,65 +1155,65 @@ ErfcPolynomialApproximation::matchAndRewrite(math::ErfcOp op,
   Value posInf = bcast(floatCst(builder, INFINITY, et));
   Value clampVal = bcast(floatCst(builder, 10.0546875f, et));
 
-  Value a = builder.create<math::AbsFOp>(x);
-  Value p = builder.create<arith::AddFOp>(a, pos2);
-  Value r = builder.create<arith::DivFOp>(one, p);
-  Value q = builder.create<math::FmaOp>(neg4, r, one);
-  Value t = builder.create<math::FmaOp>(builder.create<arith::AddFOp>(q, one),
+  Value a = math::AbsFOp::create(builder, x);
+  Value p = arith::AddFOp::create(builder, a, pos2);
+  Value r = arith::DivFOp::create(builder, one, p);
+  Value q = math::FmaOp::create(builder, neg4, r, one);
+  Value t = math::FmaOp::create(builder, arith::AddFOp::create(builder, q, one),
                                         neg2, a);
-  Value e = builder.create<math::FmaOp>(builder.create<arith::NegFOp>(a), q, t);
-  q = builder.create<math::FmaOp>(r, e, q);
+  Value e = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), q, t);
+  q = math::FmaOp::create(builder, r, e, q);
 
   p = bcast(floatCst(builder, -0x1.a4a000p-12f, et));        // -4.01139259e-4
   Value c1 = bcast(floatCst(builder, -0x1.42a260p-10f, et)); // -1.23075210e-3
-  p = builder.create<math::FmaOp>(p, q, c1);
+  p = math::FmaOp::create(builder, p, q, c1);
   Value c2 = bcast(floatCst(builder, 0x1.585714p-10f, et)); //  1.31355342e-3
-  p = builder.create<math::FmaOp>(p, q, c2);
+  p = math::FmaOp::create(builder, p, q, c2);
   Value c3 = bcast(floatCst(builder, 0x1.1adcc4p-07f, et)); // 8.63227434e-3
-  p = builder.create<math::FmaOp>(p, q, c3);
+  p = math::FmaOp::create(builder, p, q, c3);
   Value c4 = bcast(floatCst(builder, -0x1.081b82p-07f, et)); // -8.05991981e-3
-  p = builder.create<math::FmaOp>(p, q, c4);
+  p = math::FmaOp::create(builder, p, q, c4);
   Value c5 = bcast(floatCst(builder, -0x1.bc0b6ap-05f, et)); // -5.42046614e-2
-  p = builder.create<math::FmaOp>(p, q, c5);
+  p = math::FmaOp::create(builder, p, q, c5);
   Value c6 = bcast(floatCst(builder, 0x1.4ffc46p-03f, et)); //  1.64055392e-1
-  p = builder.create<math::FmaOp>(p, q, c6);
+  p = math::FmaOp::create(builder, p, q, c6);
   Value c7 = bcast(floatCst(builder, -0x1.540840p-03f, et)); // -1.66031361e-1
-  p = builder.create<math::FmaOp>(p, q, c7);
+  p = math::FmaOp::create(builder, p, q, c7);
   Value c8 = bcast(floatCst(builder, -0x1.7bf616p-04f, et)); // -9.27639827e-2
-  p = builder.create<math::FmaOp>(p, q, c8);
+  p = math::FmaOp::create(builder, p, q, c8);
   Value c9 = bcast(floatCst(builder, 0x1.1ba03ap-02f, et)); // 2.76978403e-1
-  p = builder.create<math::FmaOp>(p, q, c9);
-
-  Value d = builder.create<math::FmaOp>(pos2, a, one);
-  r = builder.create<arith::DivFOp>(one, d);
-  q = builder.create<math::FmaOp>(p, r, r);
-  Value negfa = builder.create<arith::NegFOp>(a);
-  Value fmaqah = builder.create<math::FmaOp>(q, negfa, onehalf);
-  Value psubq = builder.create<arith::SubFOp>(p, q);
-  e = builder.create<math::FmaOp>(fmaqah, pos2, psubq);
-  r = builder.create<math::FmaOp>(e, r, q);
-
-  Value s = builder.create<arith::MulFOp>(a, a);
-  e = builder.create<math::ExpOp>(builder.create<arith::NegFOp>(s));
-
-  t = builder.create<math::FmaOp>(builder.create<arith::NegFOp>(a), a, s);
-  r = builder.create<math::FmaOp>(
+  p = math::FmaOp::create(builder, p, q, c9);
+
+  Value d = math::FmaOp::create(builder, pos2, a, one);
+  r = arith::DivFOp::create(builder, one, d);
+  q = math::FmaOp::create(builder, p, r, r);
+  Value negfa = arith::NegFOp::create(builder, a);
+  Value fmaqah = math::FmaOp::create(builder, q, negfa, onehalf);
+  Value psubq = arith::SubFOp::create(builder, p, q);
+  e = math::FmaOp::create(builder, fmaqah, pos2, psubq);
+  r = math::FmaOp::create(builder, e, r, q);
+
+  Value s = arith::MulFOp::create(builder, a, a);
+  e = math::ExpOp::create(builder, arith::NegFOp::create(builder, s));
+
+  t = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), a, s);
+  r = math::FmaOp::create(builder,
       r, e,
-      builder.create<arith::MulFOp>(builder.create<arith::MulFOp>(r, e), t));
+      arith::MulFOp::create(builder, arith::MulFOp::create(builder, r, e), t));
 
-  Value isNotLessThanInf = builder.create<arith::XOrIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, a, posInf),
+  Value isNotLessThanInf = arith::XOrIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, a, posInf),
       trueValue);
-  r = builder.create<arith::SelectOp>(isNotLessThanInf,
-                                      builder.create<arith::AddFOp>(x, x), r);
+  r = arith::SelectOp::create(builder, isNotLessThanInf,
+                                      arith::AddFOp::create(builder, x, x), r);
   Value isGreaterThanClamp =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, a, clampVal);
-  r = builder.create<arith::SelectOp>(isGreaterThanClamp, zero, r);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, a, clampVal);
+  r = arith::SelectOp::create(builder, isGreaterThanClamp, zero, r);
 
   Value isNegative =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, zero);
-  r = builder.create<arith::SelectOp>(
-      isNegative, builder.create<arith::SubFOp>(pos2, r), r);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, zero);
+  r = arith::SelectOp::create(builder,
+      isNegative, arith::SubFOp::create(builder, pos2, r), r);
 
   rewriter.replaceOp(op, r);
   return success();
@@ -1235,8 +1235,8 @@ Value clampWithNormals(ImplicitLocOpBuilder &builder,
   };
 
   auto selectCmp = [&builder](auto pred, Value value, Value bound) {
-    return builder.create<arith::SelectOp>(
-        builder.create<arith::CmpFOp>(pred, value, bound), value, bound);
+    return arith::SelectOp::create(builder,
+        arith::CmpFOp::create(builder, pred, value, bound), value, bound);
   };
 
   // Note: prefer UGE/ULE vs. UGT/ULT, since they generate vmaxps/vminps vs.
@@ -1268,17 +1268,17 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
   ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
 
   auto add = [&](Value a, Value b) -> Value {
-    return builder.create<arith::AddFOp>(a, b);
+    return arith::AddFOp::create(builder, a, b);
   };
   auto bcast = [&](Value value) -> Value {
     return broadcast(builder, value, shape);
   };
-  auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); };
+  auto floor = [&](Value a) { return math::FloorOp::create(builder, a); };
   auto fmla = [&](Value a, Value b, Value c) {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
   // Polynomial approximation from Cephes.
@@ -1382,7 +1382,7 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
 
   // Convert n' to an i32.  This is safe because we clamped it above.
   auto i32Vec = broadcast(builder.getI32Type(), shape);
-  Value nI32 = builder.create<arith::FPToSIOp>(i32Vec, n);
+  Value nI32 = arith::FPToSIOp::create(builder, i32Vec, n);
 
   // Creates the value 2^n' if -126 <= n' <= 127 and 0 if n' = -127.
   Value pow2 = exp2I32(builder, nI32);
@@ -1430,26 +1430,26 @@ ExpM1Approximation::matchAndRewrite(math::ExpM1Op op,
   Value cstOne = bcast(f32Cst(builder, 1.0f));
   Value cstNegOne = bcast(f32Cst(builder, -1.0f));
   Value x = op.getOperand();
-  Value u = builder.create<math::ExpOp>(x);
+  Value u = math::ExpOp::create(builder, x);
   Value uEqOneOrNaN =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::UEQ, u, cstOne);
-  Value uMinusOne = builder.create<arith::SubFOp>(u, cstOne);
-  Value uMinusOneEqNegOne = builder.create<arith::CmpFOp>(
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::UEQ, u, cstOne);
+  Value uMinusOne = arith::SubFOp::create(builder, u, cstOne);
+  Value uMinusOneEqNegOne = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::OEQ, uMinusOne, cstNegOne);
   // logU = log(u) ~= x
-  Value logU = builder.create<math::LogOp>(u);
+  Value logU = math::LogOp::create(builder, u);
 
   // Detect exp(x) = +inf; written this way to avoid having to form +inf.
   Value isInf =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, logU, u);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, logU, u);
 
   // (u - 1) * (x / ~x)
-  Value expm1 = builder.create<arith::MulFOp>(
-      uMinusOne, builder.create<arith::DivFOp>(x, logU));
-  expm1 = builder.create<arith::SelectOp>(isInf, u, expm1);
-  Value approximation = builder.create<arith::SelectOp>(
+  Value expm1 = arith::MulFOp::create(builder,
+      uMinusOne, arith::DivFOp::create(builder, x, logU));
+  expm1 = arith::SelectOp::create(builder, isInf, u, expm1);
+  Value approximation = arith::SelectOp::create(builder,
       uEqOneOrNaN, x,
-      builder.create<arith::SelectOp>(uMinusOneEqNegOne, cstNegOne, expm1));
+      arith::SelectOp::create(builder, uMinusOneEqNegOne, cstNegOne, expm1));
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -1494,40 +1494,40 @@ LogicalResult SinAndCosApproximation<isSine, OpTy>::matchAndRewrite(
     return broadcast(builder, value, shape);
   };
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
   auto sub = [&](Value a, Value b) -> Value {
-    return builder.create<arith::SubFOp>(a, b);
+    return arith::SubFOp::create(builder, a, b);
   };
-  auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); };
+  auto floor = [&](Value a) { return math::FloorOp::create(builder, a); };
 
   auto i32Vec = broadcast(builder.getI32Type(), shape);
   auto fPToSingedInteger = [&](Value a) -> Value {
-    return builder.create<arith::FPToSIOp>(i32Vec, a);
+    return arith::FPToSIOp::create(builder, i32Vec, a);
   };
 
   auto modulo4 = [&](Value a) -> Value {
-    return builder.create<arith::AndIOp>(a, bcast(i32Cst(builder, 3)));
+    return arith::AndIOp::create(builder, a, bcast(i32Cst(builder, 3)));
   };
 
   auto isEqualTo = [&](Value a, Value b) -> Value {
-    return builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, a, b);
+    return arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, a, b);
   };
 
   auto isGreaterThan = [&](Value a, Value b) -> Value {
-    return builder.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, a, b);
+    return arith::CmpIOp::create(builder, arith::CmpIPredicate::sgt, a, b);
   };
 
   auto select = [&](Value cond, Value t, Value f) -> Value {
-    return builder.create<arith::SelectOp>(cond, t, f);
+    return arith::SelectOp::create(builder, cond, t, f);
   };
 
   auto fmla = [&](Value a, Value b, Value c) {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto bitwiseOr = [&](Value a, Value b) {
-    return builder.create<arith::OrIOp>(a, b);
+    return arith::OrIOp::create(builder, a, b);
   };
 
   Value twoOverPi = bcast(f32Cst(builder, (float)TWO_OVER_PI));
@@ -1624,7 +1624,7 @@ CbrtApproximation::matchAndRewrite(math::CbrtOp op,
   intTy = broadcast(intTy, shape);
 
   auto bconst = [&](TypedAttr attr) -> Value {
-    Value value = b.create<arith::ConstantOp>(attr);
+    Value value = arith::ConstantOp::create(b, attr);
     return broadcast(b, value, shape);
   };
 
@@ -1641,44 +1641,44 @@ CbrtApproximation::matchAndRewrite(math::CbrtOp op,
   // union {int ix; float x;};
   // x = x0;
   // ix = ix/4 + ix/16;
-  Value absValue = b.create<math::AbsFOp>(operand);
-  Value intValue = b.create<arith::BitcastOp>(intTy, absValue);
-  Value divideBy4 = b.create<arith::ShRSIOp>(intValue, intTwo);
-  Value divideBy16 = b.create<arith::ShRSIOp>(intValue, intFour);
-  intValue = b.create<arith::AddIOp>(divideBy4, divideBy16);
+  Value absValue = math::AbsFOp::create(b, operand);
+  Value intValue = arith::BitcastOp::create(b, intTy, absValue);
+  Value divideBy4 = arith::ShRSIOp::create(b, intValue, intTwo);
+  Value divideBy16 = arith::ShRSIOp::create(b, intValue, intFour);
+  intValue = arith::AddIOp::create(b, divideBy4, divideBy16);
 
   // ix = ix + ix/16;
-  divideBy16 = b.create<arith::ShRSIOp>(intValue, intFour);
-  intValue = b.create<arith::AddIOp>(intValue, divideBy16);
+  divideBy16 = arith::ShRSIOp::create(b, intValue, intFour);
+  intValue = arith::AddIOp::create(b, intValue, divideBy16);
 
   // ix = ix + ix/256;
-  Value divideBy256 = b.create<arith::ShRSIOp>(intValue, intEight);
-  intValue = b.create<arith::AddIOp>(intValue, divideBy256);
+  Value divideBy256 = arith::ShRSIOp::create(b, intValue, intEight);
+  intValue = arith::AddIOp::create(b, intValue, divideBy256);
 
   // ix = 0x2a5137a0 + ix;
-  intValue = b.create<arith::AddIOp>(intValue, intMagic);
+  intValue = arith::AddIOp::create(b, intValue, intMagic);
 
   // Perform one newtons step:
   // x = 0.33333333f*(2.0f*x + x0/(x*x));
-  Value floatValue = b.create<arith::BitcastOp>(floatTy, intValue);
-  Value squared = b.create<arith::MulFOp>(floatValue, floatValue);
-  Value mulTwo = b.create<arith::MulFOp>(floatValue, fpTwo);
-  Value divSquared = b.create<arith::DivFOp>(absValue, squared);
-  floatValue = b.create<arith::AddFOp>(mulTwo, divSquared);
-  floatValue = b.create<arith::MulFOp>(floatValue, fpThird);
+  Value floatValue = arith::BitcastOp::create(b, floatTy, intValue);
+  Value squared = arith::MulFOp::create(b, floatValue, floatValue);
+  Value mulTwo = arith::MulFOp::create(b, floatValue, fpTwo);
+  Value divSquared = arith::DivFOp::create(b, absValue, squared);
+  floatValue = arith::AddFOp::create(b, mulTwo, divSquared);
+  floatValue = arith::MulFOp::create(b, floatValue, fpThird);
 
   // x = 0.33333333f*(2.0f*x + x0/(x*x));
-  squared = b.create<arith::MulFOp>(floatValue, floatValue);
-  mulTwo = b.create<arith::MulFOp>(floatValue, fpTwo);
-  divSquared = b.create<arith::DivFOp>(absValue, squared);
-  floatValue = b.create<arith::AddFOp>(mulTwo, divSquared);
-  floatValue = b.create<arith::MulFOp>(floatValue, fpThird);
+  squared = arith::MulFOp::create(b, floatValue, floatValue);
+  mulTwo = arith::MulFOp::create(b, floatValue, fpTwo);
+  divSquared = arith::DivFOp::create(b, absValue, squared);
+  floatValue = arith::AddFOp::create(b, mulTwo, divSquared);
+  floatValue = arith::MulFOp::create(b, floatValue, fpThird);
 
   // Check for zero and restore sign.
   Value isZero =
-      b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absValue, fpZero);
-  floatValue = b.create<arith::SelectOp>(isZero, fpZero, floatValue);
-  floatValue = b.create<math::CopySignOp>(floatValue, operand);
+      arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absValue, fpZero);
+  floatValue = arith::SelectOp::create(b, isZero, fpZero, floatValue);
+  floatValue = math::CopySignOp::create(b, floatValue, operand);
 
   rewriter.replaceOp(op, floatValue);
   return success();
@@ -1719,29 +1719,29 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
   Value cstNegHalf = bcast(f32Cst(builder, -0.5f));
   Value cstMinNormPos = bcast(f32FromBits(builder, 0x00800000u));
 
-  Value negHalf = builder.create<arith::MulFOp>(op.getOperand(), cstNegHalf);
+  Value negHalf = arith::MulFOp::create(builder, op.getOperand(), cstNegHalf);
 
   // Select only the inverse sqrt of positive normals (denormals are
   // flushed to zero).
-  Value ltMinMask = builder.create<arith::CmpFOp>(
+  Value ltMinMask = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::OLT, op.getOperand(), cstMinNormPos);
-  Value infMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value infMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                 op.getOperand(), cstPosInf);
-  Value notNormalFiniteMask = builder.create<arith::OrIOp>(ltMinMask, infMask);
+  Value notNormalFiniteMask = arith::OrIOp::create(builder, ltMinMask, infMask);
 
   // Compute an approximate result.
   Value yApprox = handleMultidimensionalVectors(
       builder, op->getOperands(), 8, [&builder](ValueRange operands) -> Value {
-        return builder.create<x86vector::RsqrtOp>(operands);
+        return x86vector::RsqrtOp::create(builder, operands);
       });
 
   // Do a single step of Newton-Raphson iteration to improve the approximation.
   // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n).
   // It is essential to evaluate the inner term like this because forming
   // y_n^2 may over- or underflow.
-  Value inner = builder.create<arith::MulFOp>(negHalf, yApprox);
-  Value fma = builder.create<math::FmaOp>(yApprox, inner, cstOnePointFive);
-  Value yNewton = builder.create<arith::MulFOp>(yApprox, fma);
+  Value inner = arith::MulFOp::create(builder, negHalf, yApprox);
+  Value fma = math::FmaOp::create(builder, yApprox, inner, cstOnePointFive);
+  Value yNewton = arith::MulFOp::create(builder, yApprox, fma);
 
   // Select the result of the Newton-Raphson step for positive normal arguments.
   // For other arguments, choose the output of the intrinsic. This will
@@ -1749,7 +1749,7 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
   // x is zero or a positive denormalized float (equivalent to flushing positive
   // denormalized inputs to zero).
   Value res =
-      builder.create<arith::SelectOp>(notNormalFiniteMask, yApprox, yNewton);
+      arith::SelectOp::create(builder, notNormalFiniteMask, yApprox, yNewton);
   rewriter.replaceOp(op, res);
 
   return success();
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index c5643f6e2f830..46298fb189e41 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -85,10 +85,10 @@ Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
   // TODO: support more types.
   return TypeSwitch<Type, Value>(slot.elemType)
       .Case([&](MemRefType t) {
-        return builder.create<memref::AllocaOp>(getLoc(), t);
+        return memref::AllocaOp::create(builder, getLoc(), t);
       })
       .Default([&](Type t) {
-        return builder.create<arith::ConstantOp>(getLoc(), t,
+        return arith::ConstantOp::create(builder, getLoc(), t,
                                                  builder.getZeroAttr(t));
       });
 }
@@ -135,7 +135,7 @@ DenseMap<Attribute, MemorySlot> memref::AllocaOp::destructure(
   for (Attribute usedIndex : usedIndices) {
     Type elemType = memrefType.getTypeAtIndex(usedIndex);
     MemRefType elemPtr = MemRefType::get({}, elemType);
-    auto subAlloca = builder.create<memref::AllocaOp>(getLoc(), elemPtr);
+    auto subAlloca = memref::AllocaOp::create(builder, getLoc(), elemPtr);
     newAllocators.push_back(subAlloca);
     slotMap.try_emplace<MemorySlot>(usedIndex,
                                     {subAlloca.getResult(), elemType});
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index d1a9920aa66c5..706ac4b4867f4 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -213,7 +213,7 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
 
     // Create and insert the alloc op for the new memref.
-    auto newAlloc = rewriter.create<AllocLikeOp>(
+    auto newAlloc = AllocLikeOp::create(rewriter,
         alloc.getLoc(), newMemRefType, dynamicSizes, alloc.getSymbolOperands(),
         alloc.getAlignmentAttr());
     // Insert a cast so we have the same type as the old alloc.
@@ -836,7 +836,7 @@ void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
 void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
                   int64_t index) {
   auto loc = result.location;
-  Value indexValue = builder.create<arith::ConstantIndexOp>(loc, index);
+  Value indexValue = arith::ConstantIndexOp::create(builder, loc, index);
   build(builder, result, source, indexValue);
 }
 
@@ -1083,9 +1083,9 @@ struct DimOfMemRefReshape : public OpRewritePattern<DimOp> {
     rewriter.setInsertionPointAfter(reshape);
     Location loc = dim.getLoc();
     Value load =
-        rewriter.create<LoadOp>(loc, reshape.getShape(), dim.getIndex());
+        LoadOp::create(rewriter, loc, reshape.getShape(), dim.getIndex());
     if (load.getType() != dim.getType())
-      load = rewriter.create<arith::IndexCastOp>(loc, dim.getType(), load);
+      load = arith::IndexCastOp::create(rewriter, loc, dim.getType(), load);
     rewriter.replaceOp(dim, load);
     return success();
   }
@@ -1358,7 +1358,7 @@ static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
     assert(isa<Attribute>(maybeConstant) &&
            "The constified value should be either unchanged (i.e., == result) "
            "or a constant");
-    Value constantVal = rewriter.create<arith::ConstantIndexOp>(
+    Value constantVal = arith::ConstantIndexOp::create(rewriter,
         loc, llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
     for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
       // modifyOpInPlace: lambda cannot capture structured bindings in C++17
@@ -2587,7 +2587,7 @@ struct CollapseShapeOpMemRefCastFolder
       rewriter.modifyOpInPlace(
           op, [&]() { op.getSrcMutable().assign(cast.getSource()); });
     } else {
-      Value newOp = rewriter.create<CollapseShapeOp>(
+      Value newOp = CollapseShapeOp::create(rewriter,
           op->getLoc(), cast.getSource(), op.getReassociationIndices());
       rewriter.replaceOpWithNewOp<CastOp>(op, op.getType(), newOp);
     }
@@ -3045,15 +3045,15 @@ SmallVector<Range, 8> mlir::getOrCreateRanges(OffsetSizeAndStrideOpInterface op,
     Value offset =
         op.isDynamicOffset(idx)
             ? op.getDynamicOffset(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticOffset(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticOffset(idx));
     Value size =
         op.isDynamicSize(idx)
             ? op.getDynamicSize(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticSize(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticSize(idx));
     Value stride =
         op.isDynamicStride(idx)
             ? op.getDynamicStride(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticStride(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticStride(idx));
     res.emplace_back(Range{offset, size, stride});
   }
   return res;
@@ -3212,7 +3212,7 @@ class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
     if (!resultType)
       return failure();
 
-    Value newSubView = rewriter.create<SubViewOp>(
+    Value newSubView = SubViewOp::create(rewriter,
         subViewOp.getLoc(), resultType, castOp.getSource(),
         subViewOp.getOffsets(), subViewOp.getSizes(), subViewOp.getStrides(),
         subViewOp.getStaticOffsets(), subViewOp.getStaticSizes(),
@@ -3534,7 +3534,7 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
       return failure();
 
     // Create new ViewOp.
-    auto newViewOp = rewriter.create<ViewOp>(
+    auto newViewOp = ViewOp::create(rewriter,
         viewOp.getLoc(), newMemRefType, viewOp.getOperand(0),
         viewOp.getByteShift(), newOperands);
     // Insert a cast so we have the same type as the old memref type.
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 0c03670b4535f..5c556308b1e9f 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -155,7 +155,7 @@ transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
       Type resultType = alloca.getResult().getType();
       OpBuilder builder(rewriter.getContext());
       // TODO: Add a better builder for this.
-      globalOp = builder.create<memref::GlobalOp>(
+      globalOp = memref::GlobalOp::create(builder,
           loc, StringAttr::get(ctx, "alloca"), StringAttr::get(ctx, "private"),
           TypeAttr::get(resultType), Attribute{}, UnitAttr{}, IntegerAttr{});
       symbolTable.insert(globalOp);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
index c433415944323..a2b689ed13cbf 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
@@ -22,11 +22,11 @@ struct DefaultAllocationInterface
           DefaultAllocationInterface, memref::AllocOp> {
   static std::optional<Operation *> buildDealloc(OpBuilder &builder,
                                                  Value alloc) {
-    return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
+    return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
         .getOperation();
   }
   static std::optional<Value> buildClone(OpBuilder &builder, Value alloc) {
-    return builder.create<bufferization::CloneOp>(alloc.getLoc(), alloc)
+    return bufferization::CloneOp::create(builder, alloc.getLoc(), alloc)
         .getResult();
   }
   static ::mlir::HoistingKind getHoistingKind() {
@@ -35,7 +35,7 @@ struct DefaultAllocationInterface
   static ::std::optional<::mlir::Operation *>
   buildPromotedAlloc(OpBuilder &builder, Value alloc) {
     Operation *definingOp = alloc.getDefiningOp();
-    return builder.create<memref::AllocaOp>(
+    return memref::AllocaOp::create(builder,
         definingOp->getLoc(), cast<MemRefType>(definingOp->getResultTypes()[0]),
         definingOp->getOperands(), definingOp->getAttrs());
   }
@@ -52,7 +52,7 @@ struct DefaultReallocationInterface
           DefaultAllocationInterface, memref::ReallocOp> {
   static std::optional<Operation *> buildDealloc(OpBuilder &builder,
                                                  Value realloc) {
-    return builder.create<memref::DeallocOp>(realloc.getLoc(), realloc)
+    return memref::DeallocOp::create(builder, realloc.getLoc(), realloc)
         .getOperation();
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
index 7c777e807f08c..9d9923bfd0541 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
@@ -124,7 +124,7 @@ struct ComposeSubViewOpPattern : public OpRewritePattern<memref::SubViewOp> {
         }
 
         AffineMap map = AffineMap::get(0, affineApplyOperands.size(), expr);
-        Value result = rewriter.create<affine::AffineApplyOp>(
+        Value result = affine::AffineApplyOp::create(rewriter,
             op.getLoc(), map, affineApplyOperands);
         offsets.push_back(result);
       }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
index ec2bc95291455..c35036d7631c8 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
@@ -99,7 +99,7 @@ static Value getOffsetForBitwidth(Location loc, OpFoldResult srcIdx,
       affine::makeComposedFoldedAffineApply(builder, loc, offsetExpr, {srcIdx});
   Value bitOffset = getValueOrCreateConstantIndexOp(builder, loc, offsetVal);
   IntegerType dstType = builder.getIntegerType(targetBits);
-  return builder.create<arith::IndexCastOp>(loc, dstType, bitOffset);
+  return arith::IndexCastOp::create(builder, loc, dstType, bitOffset);
 }
 
 /// When writing a subbyte size, masked bitwise operations are used to only
@@ -112,14 +112,14 @@ static Value getSubByteWriteMask(Location loc, OpFoldResult linearizedIndices,
   auto dstIntegerType = builder.getIntegerType(dstBits);
   auto maskRightAlignedAttr =
       builder.getIntegerAttr(dstIntegerType, (1 << srcBits) - 1);
-  Value maskRightAligned = builder.create<arith::ConstantOp>(
+  Value maskRightAligned = arith::ConstantOp::create(builder,
       loc, dstIntegerType, maskRightAlignedAttr);
   Value writeMaskInverse =
-      builder.create<arith::ShLIOp>(loc, maskRightAligned, bitwidthOffset);
+      arith::ShLIOp::create(builder, loc, maskRightAligned, bitwidthOffset);
   auto flipValAttr = builder.getIntegerAttr(dstIntegerType, -1);
   Value flipVal =
-      builder.create<arith::ConstantOp>(loc, dstIntegerType, flipValAttr);
-  return builder.create<arith::XOrIOp>(loc, writeMaskInverse, flipVal);
+      arith::ConstantOp::create(builder, loc, dstIntegerType, flipValAttr);
+  return arith::XOrIOp::create(builder, loc, writeMaskInverse, flipVal);
 }
 
 /// Returns the scaled linearized index based on the `srcBits` and `dstBits`
@@ -141,7 +141,7 @@ getLinearizedSrcIndices(OpBuilder &builder, Location loc, int64_t srcBits,
                         const SmallVector<OpFoldResult> &indices,
                         Value memref) {
   auto stridedMetadata =
-      builder.create<memref::ExtractStridedMetadataOp>(loc, memref);
+      memref::ExtractStridedMetadataOp::create(builder, loc, memref);
   OpFoldResult linearizedIndices;
   std::tie(std::ignore, linearizedIndices) =
       memref::getLinearizedMemRefOffsetAndSize(
@@ -298,7 +298,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
     // Special case 0-rank memref loads.
     Value bitsLoad;
     if (convertedType.getRank() == 0) {
-      bitsLoad = rewriter.create<memref::LoadOp>(loc, adaptor.getMemref(),
+      bitsLoad = memref::LoadOp::create(rewriter, loc, adaptor.getMemref(),
                                                  ValueRange{});
     } else {
       // Linearize the indices of the original load instruction. Do not account
@@ -306,7 +306,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
       OpFoldResult linearizedIndices = getLinearizedSrcIndices(
           rewriter, loc, srcBits, adaptor.getIndices(), op.getMemRef());
 
-      Value newLoad = rewriter.create<memref::LoadOp>(
+      Value newLoad = memref::LoadOp::create(rewriter,
           loc, adaptor.getMemref(),
           getIndicesForLoadOrStore(rewriter, loc, linearizedIndices, srcBits,
                                    dstBits));
@@ -315,7 +315,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
       // Note, currently only the big-endian is supported.
       Value bitwidthOffset = getOffsetForBitwidth(loc, linearizedIndices,
                                                   srcBits, dstBits, rewriter);
-      bitsLoad = rewriter.create<arith::ShRSIOp>(loc, newLoad, bitwidthOffset);
+      bitsLoad = arith::ShRSIOp::create(rewriter, loc, newLoad, bitwidthOffset);
     }
 
     // Get the corresponding bits. If the arith computation bitwidth equals
@@ -331,13 +331,13 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
             : IntegerType::get(rewriter.getContext(),
                                resultTy.getIntOrFloatBitWidth());
     if (conversionTy == convertedElementType) {
-      auto mask = rewriter.create<arith::ConstantOp>(
+      auto mask = arith::ConstantOp::create(rewriter,
           loc, convertedElementType,
           rewriter.getIntegerAttr(convertedElementType, (1 << srcBits) - 1));
 
-      result = rewriter.create<arith::AndIOp>(loc, bitsLoad, mask);
+      result = arith::AndIOp::create(rewriter, loc, bitsLoad, mask);
     } else {
-      result = rewriter.create<arith::TruncIOp>(loc, conversionTy, bitsLoad);
+      result = arith::TruncIOp::create(rewriter, loc, conversionTy, bitsLoad);
     }
 
     if (conversionTy != resultTy) {
@@ -435,11 +435,12 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
           input);
     }
     Value extendedInput =
-        rewriter.create<arith::ExtUIOp>(loc, dstIntegerType, input);
+        arith::ExtUIOp::create(rewriter, loc, dstIntegerType,
+                                                          input);
 
     // Special case 0-rank memref stores. No need for masking.
     if (convertedType.getRank() == 0) {
-      rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::assign,
+      memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::assign,
                                            extendedInput, adaptor.getMemref(),
                                            ValueRange{});
       rewriter.eraseOp(op);
@@ -456,14 +457,14 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
                                           dstBits, bitwidthOffset, rewriter);
     // Align the value to write with the destination bits
     Value alignedVal =
-        rewriter.create<arith::ShLIOp>(loc, extendedInput, bitwidthOffset);
+        arith::ShLIOp::create(rewriter, loc, extendedInput, bitwidthOffset);
 
     // Clear destination bits
-    rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::andi,
+    memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::andi,
                                          writeMask, adaptor.getMemref(),
                                          storeIndices);
     // Write srcs bits to destination
-    rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::ori,
+    memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::ori,
                                          alignedVal, adaptor.getMemref(),
                                          storeIndices);
     rewriter.eraseOp(op);
@@ -525,7 +526,7 @@ struct ConvertMemRefSubview final : OpConversionPattern<memref::SubViewOp> {
     }
 
     // Transform the offsets, sizes and strides according to the emulation.
-    auto stridedMetadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+    auto stridedMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
         loc, subViewOp.getViewSource());
 
     OpFoldResult linearizedIndices;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
index e6e4c3b07ecb8..34f9afae9d2ad 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
@@ -48,15 +48,15 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
       Value size;
       // Load dynamic sizes from the shape input, use constants for static dims.
       if (op.getType().isDynamicDim(i)) {
-        Value index = rewriter.create<arith::ConstantIndexOp>(loc, i);
-        size = rewriter.create<memref::LoadOp>(loc, op.getShape(), index);
+        Value index = arith::ConstantIndexOp::create(rewriter, loc, i);
+        size = memref::LoadOp::create(rewriter, loc, op.getShape(), index);
         if (!isa<IndexType>(size.getType()))
-          size = rewriter.create<arith::IndexCastOp>(
+          size = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getIndexType(), size);
         sizes[i] = size;
       } else {
         auto sizeAttr = rewriter.getIndexAttr(op.getType().getDimSize(i));
-        size = rewriter.create<arith::ConstantOp>(loc, sizeAttr);
+        size = arith::ConstantOp::create(rewriter, loc, sizeAttr);
         sizes[i] = sizeAttr;
       }
       if (stride)
@@ -66,10 +66,10 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
 
       if (i > 0) {
         if (stride) {
-          stride = rewriter.create<arith::MulIOp>(loc, stride, size);
+          stride = arith::MulIOp::create(rewriter, loc, stride, size);
         } else if (op.getType().isDynamicDim(i)) {
-          stride = rewriter.create<arith::MulIOp>(
-              loc, rewriter.create<arith::ConstantIndexOp>(loc, staticStride),
+          stride = arith::MulIOp::create(rewriter,
+              loc, arith::ConstantIndexOp::create(rewriter, loc, staticStride),
               size);
         } else {
           staticStride *= op.getType().getDimSize(i);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
index 7475d442b7b9a..b54fa61714323 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
@@ -73,7 +73,7 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     if (ShapedType::isDynamic(inputSize)) {
       Value dimZero = getValueOrCreateConstantIndexOp(rewriter, loc,
                                                       rewriter.getIndexAttr(0));
-      currSize = rewriter.create<memref::DimOp>(loc, op.getSource(), dimZero)
+      currSize = memref::DimOp::create(rewriter, loc, op.getSource(), dimZero)
                      .getResult();
     }
 
@@ -88,9 +88,9 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     // the old buffer is smaller than the requested size.
     Value lhs = getValueOrCreateConstantIndexOp(rewriter, loc, currSize);
     Value rhs = getValueOrCreateConstantIndexOp(rewriter, loc, targetSize);
-    Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                                 lhs, rhs);
-    auto ifOp = rewriter.create<scf::IfOp>(
+    auto ifOp = scf::IfOp::create(rewriter,
         loc, cond,
         [&](OpBuilder &builder, Location loc) {
           // Allocate the new buffer. If it is a dynamic memref we need to pass
@@ -100,25 +100,25 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           if (op.getDynamicResultSize())
             dynamicSizeOperands.push_back(op.getDynamicResultSize());
 
-          Value newAlloc = builder.create<memref::AllocOp>(
+          Value newAlloc = memref::AllocOp::create(builder,
               loc, op.getResult().getType(), dynamicSizeOperands,
               op.getAlignmentAttr());
 
           // Take a subview of the new (bigger) buffer such that we can copy the
           // old values over (the copy operation requires both operands to have
           // the same shape).
-          Value subview = builder.create<memref::SubViewOp>(
+          Value subview = memref::SubViewOp::create(builder,
               loc, newAlloc, ArrayRef<OpFoldResult>{rewriter.getIndexAttr(0)},
               ArrayRef<OpFoldResult>{currSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
-          builder.create<memref::CopyOp>(loc, op.getSource(), subview);
+          memref::CopyOp::create(builder, loc, op.getSource(), subview);
 
           // Insert the deallocation of the old buffer only if requested
           // (enabled by default).
           if (emitDeallocs)
-            builder.create<memref::DeallocOp>(loc, op.getSource());
+            memref::DeallocOp::create(builder, loc, op.getSource());
 
-          builder.create<scf::YieldOp>(loc, newAlloc);
+          scf::YieldOp::create(builder, loc, newAlloc);
         },
         [&](OpBuilder &builder, Location loc) {
           // We need to reinterpret-cast here because either the input or output
@@ -126,11 +126,11 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           // dynamic or vice-versa. If both are static and the original buffer
           // is already bigger than the requested size, the cast represents a
           // subview operation.
-          Value casted = builder.create<memref::ReinterpretCastOp>(
+          Value casted = memref::ReinterpretCastOp::create(builder,
               loc, cast<MemRefType>(op.getResult().getType()), op.getSource(),
               rewriter.getIndexAttr(0), ArrayRef<OpFoldResult>{targetSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
-          builder.create<scf::YieldOp>(loc, casted);
+          scf::YieldOp::create(builder, loc, casted);
         });
 
     rewriter.replaceOp(op, ifOp.getResult(0));
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index 2ba798f48ac7c..f581b6490fadb 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -66,7 +66,7 @@ resolveSubviewStridedMetadata(RewriterBase &rewriter,
   unsigned sourceRank = sourceType.getRank();
 
   auto newExtractStridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(origLoc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, origLoc, source);
 
   auto [sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
 #ifndef NDEBUG
@@ -577,7 +577,7 @@ static FailureOr<StridedMetadata> resolveReshapeStridedMetadata(
   unsigned sourceRank = sourceType.getRank();
 
   auto newExtractStridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(origLoc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, origLoc, source);
 
   // Collect statically known information.
   auto [strides, offset] = sourceType.getStridesAndOffset();
@@ -828,14 +828,14 @@ struct ExtractStridedMetadataOpAllocFolder
       if (allocLikeOp.getType() == baseBufferType)
         results.push_back(allocLikeOp);
       else
-        results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+        results.push_back(memref::ReinterpretCastOp::create(rewriter,
             loc, baseBufferType, allocLikeOp, offset,
             /*sizes=*/ArrayRef<int64_t>(),
             /*strides=*/ArrayRef<int64_t>()));
     }
 
     // Offset.
-    results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    results.push_back(arith::ConstantIndexOp::create(rewriter, loc, offset));
 
     for (OpFoldResult size : sizes)
       results.push_back(getValueOrCreateConstantIndexOp(rewriter, loc, size));
@@ -900,19 +900,19 @@ struct ExtractStridedMetadataOpGetGlobalFolder
     if (getGlobalOp.getType() == baseBufferType)
       results.push_back(getGlobalOp);
     else
-      results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+      results.push_back(memref::ReinterpretCastOp::create(rewriter,
           loc, baseBufferType, getGlobalOp, offset,
           /*sizes=*/ArrayRef<int64_t>(),
           /*strides=*/ArrayRef<int64_t>()));
 
     // Offset.
-    results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    results.push_back(arith::ConstantIndexOp::create(rewriter, loc, offset));
 
     for (auto size : sizes)
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, size));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, size));
 
     for (auto stride : strides)
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, stride));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, stride));
 
     rewriter.replaceOp(op, results);
     return success();
@@ -1009,7 +1009,7 @@ class ExtractStridedMetadataOpReinterpretCastFolder
     results.resize_for_overwrite(rank * 2 + 2);
 
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(
+        memref::ExtractStridedMetadataOp::create(rewriter,
             loc, reinterpretCastOp.getSource());
 
     // Register the base_buffer.
@@ -1083,7 +1083,7 @@ class ExtractStridedMetadataOpCastFolder
     results.resize_for_overwrite(rank * 2 + 2);
 
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc,
+        memref::ExtractStridedMetadataOp::create(rewriter, loc,
                                                           castOp.getSource());
 
     // Register the base_buffer.
@@ -1143,7 +1143,7 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
     if (!memSpaceCastOp)
       return failure();
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(
+        memref::ExtractStridedMetadataOp::create(rewriter,
             loc, memSpaceCastOp.getSource());
     SmallVector<Value> results(newExtractStridedMetadata.getResults());
     // As with most other strided metadata rewrite patterns, don't introduce
@@ -1158,7 +1158,7 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
       MemRefType::Builder newTypeBuilder(baseBufferType);
       newTypeBuilder.setMemorySpace(
           memSpaceCastOp.getResult().getType().getMemorySpace());
-      results[0] = rewriter.create<memref::MemorySpaceCastOp>(
+      results[0] = memref::MemorySpaceCastOp::create(rewriter,
           loc, Type{newTypeBuilder}, baseBuffer);
     } else {
       results[0] = nullptr;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
index 2f5c9436fb8c7..65f398f6dc5a8 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
@@ -42,7 +42,7 @@ static memref::LoadOp rebuildLoadOp(RewriterBase &rewriter,
                                     memref::LoadOp loadOp, Value srcMemRef,
                                     ArrayRef<Value> indices) {
   Location loc = loadOp.getLoc();
-  return rewriter.create<memref::LoadOp>(loc, srcMemRef, indices,
+  return memref::LoadOp::create(rewriter, loc, srcMemRef, indices,
                                          loadOp.getNontemporal());
 }
 
@@ -72,7 +72,7 @@ static memref::StoreOp rebuildStoreOp(RewriterBase &rewriter,
                                       memref::StoreOp storeOp, Value srcMemRef,
                                       ArrayRef<Value> indices) {
   Location loc = storeOp.getLoc();
-  return rewriter.create<memref::StoreOp>(loc, storeOp.getValueToStore(),
+  return memref::StoreOp::create(rewriter, loc, storeOp.getValueToStore(),
                                           srcMemRef, indices,
                                           storeOp.getNontemporal());
 }
@@ -104,7 +104,7 @@ static nvgpu::LdMatrixOp rebuildLdMatrixOp(RewriterBase &rewriter,
                                            Value srcMemRef,
                                            ArrayRef<Value> indices) {
   Location loc = ldMatrixOp.getLoc();
-  return rewriter.create<nvgpu::LdMatrixOp>(
+  return nvgpu::LdMatrixOp::create(rewriter,
       loc, ldMatrixOp.getResult().getType(), srcMemRef, indices,
       ldMatrixOp.getTranspose(), ldMatrixOp.getNumTiles());
 }
@@ -132,7 +132,7 @@ rebuildTransferReadOp(RewriterBase &rewriter,
                       vector::TransferReadOp transferReadOp, Value srcMemRef,
                       ArrayRef<Value> indices) {
   Location loc = transferReadOp.getLoc();
-  return rewriter.create<vector::TransferReadOp>(
+  return vector::TransferReadOp::create(rewriter,
       loc, transferReadOp.getResult().getType(), srcMemRef, indices,
       transferReadOp.getPermutationMap(), transferReadOp.getPadding(),
       transferReadOp.getMask(), transferReadOp.getInBoundsAttr());
@@ -150,7 +150,7 @@ rebuildTransferWriteOp(RewriterBase &rewriter,
                        vector::TransferWriteOp transferWriteOp, Value srcMemRef,
                        ArrayRef<Value> indices) {
   Location loc = transferWriteOp.getLoc();
-  return rewriter.create<vector::TransferWriteOp>(
+  return vector::TransferWriteOp::create(rewriter,
       loc, transferWriteOp.getValue(), srcMemRef, indices,
       transferWriteOp.getPermutationMapAttr(), transferWriteOp.getMask(),
       transferWriteOp.getInBoundsAttr());
@@ -183,7 +183,7 @@ getGenericOpViewSizeForEachDim(RewriterBase &rewriter,
                                LoadStoreLikeOp loadStoreLikeOp) {
   Location loc = loadStoreLikeOp.getLoc();
   auto extractStridedMetadataOp =
-      rewriter.create<memref::ExtractStridedMetadataOp>(
+      memref::ExtractStridedMetadataOp::create(rewriter,
           loc, getSrcMemRef(loadStoreLikeOp));
   SmallVector<OpFoldResult> srcSizes =
       extractStridedMetadataOp.getConstifiedMixedSizes();
@@ -267,12 +267,12 @@ struct LoadStoreLikeOpRewriter : public OpRewritePattern<LoadStoreLikeOp> {
     // apply them properly to the input indices.
     // Therefore the strides multipliers are simply ones.
     auto subview =
-        rewriter.create<memref::SubViewOp>(loc, /*source=*/srcMemRef,
+        memref::SubViewOp::create(rewriter, loc, /*source=*/srcMemRef,
                                            /*offsets=*/indices,
                                            /*sizes=*/sizes, /*strides=*/ones);
     // Rewrite the load/store with the subview as the base pointer.
     SmallVector<Value> zeros(loadStoreRank,
-                             rewriter.create<arith::ConstantIndexOp>(loc, 0));
+                             arith::ConstantIndexOp::create(rewriter, loc, 0));
     LoadStoreLikeOp newLoadStore = rebuildOpFromAddressAndIndices(
         rewriter, loadStoreLikeOp, subview.getResult(), zeros);
     rewriter.replaceOp(loadStoreLikeOp, newLoadStore->getResults());
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
index 76f7788c4dcc5..48dbabfdf890a 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
@@ -40,7 +40,7 @@ using namespace mlir;
 static Value getValueFromOpFoldResult(OpBuilder &rewriter, Location loc,
                                       OpFoldResult in) {
   if (Attribute offsetAttr = dyn_cast<Attribute>(in)) {
-    return rewriter.create<arith::ConstantIndexOp>(
+    return arith::ConstantIndexOp::create(rewriter,
         loc, cast<IntegerAttr>(offsetAttr).getInt());
   }
   return cast<Value>(in);
@@ -60,7 +60,7 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
   }
 
   memref::ExtractStridedMetadataOp stridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(loc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, loc, source);
 
   auto typeBit = sourceType.getElementType().getIntOrFloatBitWidth();
   OpFoldResult linearizedIndices;
@@ -74,7 +74,7 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
           getAsOpFoldResult(indices));
 
   return std::make_pair(
-      rewriter.create<memref::ReinterpretCastOp>(
+      memref::ReinterpretCastOp::create(rewriter,
           loc, source,
           /* offset = */ linearizedInfo.linearizedOffset,
           /* shapes = */
@@ -111,7 +111,7 @@ template <typename T>
 static void castAllocResult(T oper, T newOper, Location loc,
                             PatternRewriter &rewriter) {
   memref::ExtractStridedMetadataOp stridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(loc, oper);
+      memref::ExtractStridedMetadataOp::create(rewriter, loc, oper);
   rewriter.replaceOpWithNewOp<memref::ReinterpretCastOp>(
       oper, cast<MemRefType>(oper.getType()), newOper,
       /*offset=*/rewriter.getIndexAttr(0),
@@ -125,62 +125,62 @@ static void replaceOp(T op, PatternRewriter &rewriter, Value flatMemref,
   Location loc = op->getLoc();
   llvm::TypeSwitch<Operation *>(op.getOperation())
       .template Case<memref::AllocOp>([&](auto oper) {
-        auto newAlloc = rewriter.create<memref::AllocOp>(
+        auto newAlloc = memref::AllocOp::create(rewriter,
             loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloc, loc, rewriter);
       })
       .template Case<memref::AllocaOp>([&](auto oper) {
-        auto newAlloca = rewriter.create<memref::AllocaOp>(
+        auto newAlloca = memref::AllocaOp::create(rewriter,
             loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloca, loc, rewriter);
       })
       .template Case<memref::LoadOp>([&](auto op) {
-        auto newLoad = rewriter.create<memref::LoadOp>(
+        auto newLoad = memref::LoadOp::create(rewriter,
             loc, op->getResultTypes(), flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<memref::StoreOp>([&](auto op) {
-        auto newStore = rewriter.create<memref::StoreOp>(
+        auto newStore = memref::StoreOp::create(rewriter,
             loc, op->getOperands().front(), flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::LoadOp>([&](auto op) {
-        auto newLoad = rewriter.create<vector::LoadOp>(
+        auto newLoad = vector::LoadOp::create(rewriter,
             loc, op->getResultTypes(), flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<vector::StoreOp>([&](auto op) {
-        auto newStore = rewriter.create<vector::StoreOp>(
+        auto newStore = vector::StoreOp::create(rewriter,
             loc, op->getOperands().front(), flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::MaskedLoadOp>([&](auto op) {
-        auto newMaskedLoad = rewriter.create<vector::MaskedLoadOp>(
+        auto newMaskedLoad = vector::MaskedLoadOp::create(rewriter,
             loc, op.getType(), flatMemref, ValueRange{offset}, op.getMask(),
             op.getPassThru());
         newMaskedLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedLoad.getResult());
       })
       .template Case<vector::MaskedStoreOp>([&](auto op) {
-        auto newMaskedStore = rewriter.create<vector::MaskedStoreOp>(
+        auto newMaskedStore = vector::MaskedStoreOp::create(rewriter,
             loc, flatMemref, ValueRange{offset}, op.getMask(),
             op.getValueToStore());
         newMaskedStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedStore);
       })
       .template Case<vector::TransferReadOp>([&](auto op) {
-        auto newTransferRead = rewriter.create<vector::TransferReadOp>(
+        auto newTransferRead = vector::TransferReadOp::create(rewriter,
             loc, op.getType(), flatMemref, ValueRange{offset}, op.getPadding());
         rewriter.replaceOp(op, newTransferRead.getResult());
       })
       .template Case<vector::TransferWriteOp>([&](auto op) {
-        auto newTransferWrite = rewriter.create<vector::TransferWriteOp>(
+        auto newTransferWrite = vector::TransferWriteOp::create(rewriter,
             loc, op.getVector(), flatMemref, ValueRange{offset});
         rewriter.replaceOp(op, newTransferWrite);
       })
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index 89be188af9129..8e99a101f28a6 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -76,7 +76,7 @@ static LogicalResult resolveSourceIndicesExpandShape(
         llvm::map_to_vector(group, [&](int64_t d) { return destShape[d]; });
     SmallVector<Value> groupIndices =
         llvm::map_to_vector(group, [&](int64_t d) { return indices[d]; });
-    Value collapsedIndex = rewriter.create<affine::AffineLinearizeIndexOp>(
+    Value collapsedIndex = affine::AffineLinearizeIndexOp::create(rewriter,
         loc, groupIndices, groupBasis, /*disjoint=*/startsInbounds);
     sourceIndices.push_back(collapsedIndex);
   }
@@ -102,7 +102,7 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
                                   ValueRange indices,
                                   SmallVectorImpl<Value> &sourceIndices) {
   // Note: collapse_shape requires a strided memref, we can do this.
-  auto metadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+  auto metadata = memref::ExtractStridedMetadataOp::create(rewriter,
       loc, collapseShapeOp.getSrc());
   SmallVector<OpFoldResult> sourceSizes = metadata.getConstifiedMixedSizes();
   for (auto [index, group] :
@@ -117,7 +117,7 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
 
     SmallVector<OpFoldResult> basis =
         llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-    auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
         loc, index, basis, /*hasOuterBound=*/true);
     llvm::append_range(sourceIndices, delinearize.getResults());
   }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
index 35c661ecb886d..953024d70d0aa 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
@@ -51,7 +51,7 @@ FailureOr<Value> memref::buildIndependentOp(OpBuilder &b,
 
   // Create a new memref::AllocaOp.
   Value newAllocaOp =
-      b.create<AllocaOp>(loc, newSizes, allocaOp.getType().getElementType());
+      AllocaOp::create(b, loc, newSizes, allocaOp.getType().getElementType());
 
   // Create a memref::SubViewOp.
   SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
@@ -71,10 +71,10 @@ propagateSubViewOp(RewriterBase &rewriter,
   MemRefType newResultType = SubViewOp::inferRankReducedResultType(
       op.getType().getShape(), op.getSourceType(), op.getMixedOffsets(),
       op.getMixedSizes(), op.getMixedStrides());
-  Value newSubview = rewriter.create<SubViewOp>(
+  Value newSubview = SubViewOp::create(rewriter,
       op.getLoc(), newResultType, conversionOp.getOperand(0),
       op.getMixedOffsets(), op.getMixedSizes(), op.getMixedStrides());
-  auto newConversionOp = rewriter.create<UnrealizedConversionCastOp>(
+  auto newConversionOp = UnrealizedConversionCastOp::create(rewriter,
       op.getLoc(), op.getType(), newSubview);
   rewriter.replaceAllUsesWith(op.getResult(), newConversionOp->getResult(0));
   return newConversionOp;
@@ -106,7 +106,7 @@ static void replaceAndPropagateMemRefType(RewriterBase &rewriter,
   SmallVector<UnrealizedConversionCastOp> unrealizedConversions;
   for (const auto &it :
        llvm::enumerate(llvm::zip(from->getResults(), to->getResults()))) {
-    unrealizedConversions.push_back(rewriter.create<UnrealizedConversionCastOp>(
+    unrealizedConversions.push_back(UnrealizedConversionCastOp::create(rewriter,
         to->getLoc(), std::get<0>(it.value()).getType(),
         std::get<1>(it.value())));
     rewriter.replaceAllUsesWith(from->getResult(it.index()),
diff --git a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
index 0a84962150ead..b7e4388193a06 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
@@ -63,7 +63,7 @@ static void replaceUsesAndPropagateType(RewriterBase &rewriter,
         subviewUse.getType().getShape(), cast<MemRefType>(val.getType()),
         subviewUse.getStaticOffsets(), subviewUse.getStaticSizes(),
         subviewUse.getStaticStrides());
-    Value newSubview = rewriter.create<memref::SubViewOp>(
+    Value newSubview = memref::SubViewOp::create(rewriter,
         subviewUse->getLoc(), newType, val, subviewUse.getMixedOffsets(),
         subviewUse.getMixedSizes(), subviewUse.getMixedStrides());
 
@@ -177,7 +177,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   Location loc = allocOp->getLoc();
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(allocOp);
-  auto mbAlloc = rewriter.create<memref::AllocOp>(
+  auto mbAlloc = memref::AllocOp::create(rewriter,
       loc, mbMemRefType, ValueRange{}, allocOp->getAttrs());
   LLVM_DEBUG(DBGS() << "--multi-buffered alloc: " << mbAlloc << "\n");
 
@@ -211,7 +211,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   // Strides is [1, 1 ... 1 ].
   MemRefType dstMemref = memref::SubViewOp::inferRankReducedResultType(
       originalShape, mbMemRefType, offsets, sizes, strides);
-  Value subview = rewriter.create<memref::SubViewOp>(loc, dstMemref, mbAlloc,
+  Value subview = memref::SubViewOp::create(rewriter, loc, dstMemref, mbAlloc,
                                                      offsets, sizes, strides);
   LLVM_DEBUG(DBGS() << "--multi-buffered slice: " << subview << "\n");
 
@@ -224,7 +224,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPoint(deallocOp);
     auto newDeallocOp =
-        rewriter.create<memref::DeallocOp>(deallocOp->getLoc(), mbAlloc);
+        memref::DeallocOp::create(rewriter, deallocOp->getLoc(), mbAlloc);
     (void)newDeallocOp;
     LLVM_DEBUG(DBGS() << "----Created dealloc: " << newDeallocOp << "\n");
     rewriter.eraseOp(deallocOp);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index 4ec04321dd3e2..c522b2bd5ddec 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -276,7 +276,7 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
     if (!callOp)
       continue;
     Operation *newCallOp =
-        builder.create<func::CallOp>(userOp->getLoc(), callOp.getCalleeAttr(),
+        func::CallOp::create(builder, userOp->getLoc(), callOp.getCalleeAttr(),
                                      resultTypes, userOp->getOperands());
     bool replacingMemRefUsesFailed = false;
     bool returnTypeChanged = false;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
index e6b9e2f7e8213..ea7f394f48ba4 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
@@ -116,10 +116,10 @@ static LogicalResult reifyOpResultShapes(RewriterBase &rewriter,
     // Update the type.
     newRes.setType(reifiedTy);
     if (isa<RankedTensorType>(reifiedTy)) {
-      newResults.push_back(rewriter.create<tensor::CastOp>(loc, oldTy, newRes));
+      newResults.push_back(tensor::CastOp::create(rewriter, loc, oldTy, newRes));
     } else {
       assert(isa<MemRefType>(reifiedTy) && "expected a memref type");
-      newResults.push_back(rewriter.create<memref::CastOp>(loc, oldTy, newRes));
+      newResults.push_back(memref::CastOp::create(rewriter, loc, oldTy, newRes));
     }
   }
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
index 89a3895d06ba5..6a81a15f30e47 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
@@ -69,7 +69,7 @@ struct DimOfShapedTypeOpInterface : public OpRewritePattern<OpTy> {
     Location loc = dimOp->getLoc();
     rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
         dimOp, resultShape,
-        rewriter.create<arith::ConstantIndexOp>(loc, *dimIndex).getResult());
+        arith::ConstantIndexOp::create(rewriter, loc, *dimIndex).getResult());
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
index d231516884c7d..fe9555a7a16fd 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
@@ -40,15 +40,15 @@ struct AssumeAlignmentOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto assumeOp = cast<AssumeAlignmentOp>(op);
-    Value ptr = builder.create<ExtractAlignedPointerAsIndexOp>(
+    Value ptr = ExtractAlignedPointerAsIndexOp::create(builder,
         loc, assumeOp.getMemref());
-    Value rest = builder.create<arith::RemUIOp>(
+    Value rest = arith::RemUIOp::create(builder,
         loc, ptr,
-        builder.create<arith::ConstantIndexOp>(loc, assumeOp.getAlignment()));
-    Value isAligned = builder.create<arith::CmpIOp>(
+        arith::ConstantIndexOp::create(builder, loc, assumeOp.getAlignment()));
+    Value isAligned = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::eq, rest,
-        builder.create<arith::ConstantIndexOp>(loc, 0));
-    builder.create<cf::AssertOp>(
+        arith::ConstantIndexOp::create(builder, loc, 0));
+    cf::AssertOp::create(builder,
         loc, isAligned,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "memref is not aligned to " +
@@ -71,12 +71,12 @@ struct CastOpInterface
 
     if (isa<UnrankedMemRefType>(srcType)) {
       // Check rank.
-      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
-          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
-      Value isSameRank = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
+      Value isSameRank = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameRank,
           RuntimeVerifiableOpInterface::generateErrorMessage(op,
                                                              "rank mismatch"));
@@ -95,8 +95,8 @@ struct CastOpInterface
         MemRefType::get(dynamicShape, resultType.getElementType(),
                         stridedLayout, resultType.getMemorySpace());
     Value helperCast =
-        builder.create<CastOp>(loc, dynStridesType, castOp.getSource());
-    auto metadataOp = builder.create<ExtractStridedMetadataOp>(loc, helperCast);
+        CastOp::create(builder, loc, dynStridesType, castOp.getSource());
+    auto metadataOp = ExtractStridedMetadataOp::create(builder, loc, helperCast);
 
     // Check dimension sizes.
     for (const auto &it : llvm::enumerate(resultType.getShape())) {
@@ -110,12 +110,12 @@ struct CastOpInterface
         continue;
 
       Value srcDimSz =
-          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+          DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameSz = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameSz = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
@@ -132,10 +132,10 @@ struct CastOpInterface
       // Static/dynamic offset -> dynamic offset does not need verification.
       Value srcOffset = metadataOp.getResult(1);
       Value resultOffsetVal =
-          builder.create<arith::ConstantIndexOp>(loc, resultOffset);
-      Value isSameOffset = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultOffset);
+      Value isSameOffset = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcOffset, resultOffsetVal);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameOffset,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset mismatch"));
@@ -150,10 +150,10 @@ struct CastOpInterface
       Value srcStride =
           metadataOp.getResult(2 + resultType.getRank() + it.index());
       Value resultStrideVal =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameStride = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameStride = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcStride, resultStrideVal);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameStride,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "stride mismatch of dim " + std::to_string(it.index())));
@@ -186,7 +186,7 @@ struct CopyOpInterface
       auto getDimSize = [&](Value memRef, MemRefType type,
                             int64_t dim) -> Value {
         return type.isDynamicDim(dim)
-                   ? builder.create<DimOp>(loc, memRef, dim).getResult()
+                   ? DimOp::create(builder, loc, memRef, dim).getResult()
                    : builder
                          .create<arith::ConstantIndexOp>(loc,
                                                          type.getDimSize(dim))
@@ -194,9 +194,9 @@ struct CopyOpInterface
       };
       Value sourceDim = getDimSize(copyOp.getSource(), rankedSourceType, i);
       Value targetDim = getDimSize(copyOp.getTarget(), rankedTargetType, i);
-      Value sameDimSize = builder.create<arith::CmpIOp>(
+      Value sameDimSize = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, sameDimSize,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size of " + std::to_string(i) +
@@ -211,9 +211,9 @@ struct DimOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto dimOp = cast<DimOp>(op);
-    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    builder.create<cf::AssertOp>(
+    Value rank = RankOp::create(builder, loc, dimOp.getSource());
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    cf::AssertOp::create(builder,
         loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
@@ -237,7 +237,7 @@ struct LoadStoreOpInterface
     }
     auto indices = loadStoreOp.getIndices();
 
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     Value assertCond;
     for (auto i : llvm::seq<int64_t>(0, rank)) {
       Value dimOp = builder.createOrFold<memref::DimOp>(loc, memref, i);
@@ -247,7 +247,7 @@ struct LoadStoreOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    builder.create<cf::AssertOp>(
+    cf::AssertOp::create(builder,
         loc, assertCond,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "out-of-bounds access"));
@@ -265,10 +265,10 @@ struct SubViewOpInterface
     // For each dimension, assert that:
     // 0 <= offset < dim_size
     // 0 <= offset + (size - 1) * stride < dim_size
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value one = arith::ConstantIndexOp::create(builder, loc, 1);
     auto metadataOp =
-        builder.create<ExtractStridedMetadataOp>(loc, subView.getSource());
+        ExtractStridedMetadataOp::create(builder, loc, subView.getSource());
     for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
       Value offset = getValueOrCreateConstantIndexOp(
           builder, loc, subView.getMixedOffsets()[i]);
@@ -281,20 +281,20 @@ struct SubViewOpInterface
       Value dimSize = metadataOp.getSizes()[i];
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
       // Verify that slice does not run out-of-bounds.
-      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOne = arith::SubIOp::create(builder, loc, size, one);
       Value sizeMinusOneTimesStride =
-          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+          arith::MulIOp::create(builder, loc, sizeMinusOne, stride);
       Value lastPos =
-          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+          arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "subview runs out-of-bounds along dimension " +
@@ -315,7 +315,7 @@ struct ExpandShapeOpInterface
     for (const auto &it :
          llvm::enumerate(expandShapeOp.getReassociationIndices())) {
       Value srcDimSz =
-          builder.create<DimOp>(loc, expandShapeOp.getSrc(), it.index());
+          DimOp::create(builder, loc, expandShapeOp.getSrc(), it.index());
       int64_t groupSz = 1;
       bool foundDynamicDim = false;
       for (int64_t resultDim : it.value()) {
@@ -330,14 +330,14 @@ struct ExpandShapeOpInterface
         groupSz *= expandShapeOp.getResultType().getDimSize(resultDim);
       }
       Value staticResultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, groupSz);
+          arith::ConstantIndexOp::create(builder, loc, groupSz);
       // staticResultDimSz must divide srcDimSz evenly.
       Value mod =
-          builder.create<arith::RemSIOp>(loc, srcDimSz, staticResultDimSz);
-      Value isModZero = builder.create<arith::CmpIOp>(
+          arith::RemSIOp::create(builder, loc, srcDimSz, staticResultDimSz);
+      Value isModZero = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, mod,
-          builder.create<arith::ConstantIndexOp>(loc, 0));
-      builder.create<cf::AssertOp>(
+          arith::ConstantIndexOp::create(builder, loc, 0));
+      cf::AssertOp::create(builder,
           loc, isModZero,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "static result dims in reassoc group do not "
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index cf506d1e7812b..a440749dbd703 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -91,7 +91,7 @@ SmallVector<Value> mlir::mesh::getMixedAsValues(OpBuilder b,
       values.emplace_back(*(dyn++));
     } else {
       TypedAttr val = type == i64 ? b.getI64IntegerAttr(s) : b.getIndexAttr(s);
-      values.emplace_back(b.create<arith::ConstantOp>(loc, type, val));
+      values.emplace_back(arith::ConstantOp::create(b, loc, type, val));
     }
   }
   return values;
@@ -316,9 +316,9 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
 
   if (!newShardOp) {
     auto shardingOp =
-        builder.create<ShardingOp>(operandValue.getLoc(), sharding);
+        ShardingOp::create(builder, operandValue.getLoc(), sharding);
     newShardOp =
-        builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+        ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                                 /*annotate_for_users*/ false);
   }
   operandValue.replaceUsesWithIf(
@@ -330,7 +330,7 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
     return;
   }
 
-  auto newShardOp2 = builder.create<ShardOp>(operandValue.getLoc(), newShardOp,
+  auto newShardOp2 = ShardOp::create(builder, operandValue.getLoc(), newShardOp,
                                              newShardOp.getSharding(),
                                              /*annotate_for_users*/ true);
   newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
@@ -378,9 +378,9 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
 
   builder.setInsertionPoint(operandOp);
   auto shardingOp =
-      builder.create<ShardingOp>(operand.get().getLoc(), sharding);
+      ShardingOp::create(builder, operand.get().getLoc(), sharding);
   auto newShardOp =
-      builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+      ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                               /*annotate_for_users*/ true);
   IRRewriter rewriter(builder);
   rewriter.replaceUsesWithIf(
@@ -395,7 +395,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
 
   builder.setInsertionPoint(newShardOp);
   auto newPreceedingShardOp =
-      builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+      ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                               /*annotate_for_users*/ false);
   rewriter.replaceUsesWithIf(
       newShardOp.getSrc(), newPreceedingShardOp, [&newShardOp](OpOperand &use) {
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
index 9da3c9a3dd160..65446142aabcc 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
@@ -91,7 +91,7 @@ struct MeshShapeFolder
         newShapeOpMeshAxes.push_back(opMeshAxes[i]);
       } else {
         // Fold static mesh axes.
-        newResults[i] = builder.create<arith::ConstantOp>(
+        newResults[i] = arith::ConstantOp::create(builder,
             builder.getIndexAttr(meshAxisSize));
       }
     }
@@ -99,7 +99,7 @@ struct MeshShapeFolder
     // Leave only the dynamic mesh axes to be queried.
     if (!newShapeOpMeshAxes.empty()) {
       MeshShapeOp newShapeOp =
-          builder.create<MeshShapeOp>(mesh.getSymName(), newShapeOpMeshAxes);
+          MeshShapeOp::create(builder, mesh.getSymName(), newShapeOpMeshAxes);
       for (size_t i = 0; i < newShapeOp->getResults().size(); ++i) {
         newResults[newToOldResultsIndexMap[i]] = newShapeOp->getResults()[i];
       }
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index d7b7234f69347..9229bfaa1dffb 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -265,7 +265,7 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
       targetShardingInUnsplitLastAxis(ctx, sourceSharding, splitTensorAxis);
   ShapedType allGatherResultShape = allGatherResultShapeInUnsplitLastAxis(
       sourceShard.getType(), mesh.getShape()[splitMeshAxis], splitTensorAxis);
-  Value allGatherResult = builder.create<AllGatherOp>(
+  Value allGatherResult = AllGatherOp::create(builder,
       RankedTensorType::get(allGatherResultShape.getShape(),
                             allGatherResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({splitMeshAxis}), sourceShard,
@@ -273,7 +273,7 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      builder.create<tensor::CastOp>(targetShape, allGatherResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allGatherResult).getResult());
   return {targetShard, targetSharding};
 }
 
@@ -398,7 +398,7 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType allToAllResultShape = allToAllResultShapeInMoveLastAxis(
       sourceShard.getType(), mesh.getShape()[meshAxis], sourceTensorAxis,
       targetTensorAxis);
-  Value allToAllResult = builder.create<AllToAllOp>(
+  Value allToAllResult = AllToAllOp::create(builder,
       RankedTensorType::get(allToAllResultShape.getShape(),
                             allToAllResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({meshAxis}), sourceShard,
@@ -406,7 +406,7 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      builder.create<tensor::CastOp>(targetShape, allToAllResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allToAllResult).getResult());
   return {targetShard, targetSharding};
 }
 
@@ -477,13 +477,13 @@ tryUpdateHaloInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
 
   // Extract core from source and copy into destination core.
   auto noVals = ValueRange{};
-  auto initVal = builder.create<tensor::EmptyOp>(
+  auto initVal = tensor::EmptyOp::create(builder,
       sourceShard.getLoc(), outShape, sourceShard.getType().getElementType());
-  auto core = builder.create<tensor::ExtractSliceOp>(
+  auto core = tensor::ExtractSliceOp::create(builder,
       sourceShard.getLoc(),
       RankedTensorType::get(coreShape, sourceShard.getType().getElementType()),
       sourceShard, noVals, noVals, noVals, srcCoreOffs, coreShape, strides);
-  auto initOprnd = builder.create<tensor::InsertSliceOp>(
+  auto initOprnd = tensor::InsertSliceOp::create(builder,
       sourceShard.getLoc(), core, initVal, noVals, noVals, noVals, tgtCoreOffs,
       coreShape, strides);
 
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
index f08ef75d8a004..c85394924608d 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
@@ -49,10 +49,10 @@ struct ProcessMultiIndexOpLowering
 
     ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     builder.setInsertionPointAfter(op.getOperation());
-    Value linearIndex = builder.create<ProcessLinearIndexOp>(mesh);
-    ValueRange meshShape = builder.create<MeshShapeOp>(mesh).getResults();
+    Value linearIndex = ProcessLinearIndexOp::create(builder, mesh);
+    ValueRange meshShape = MeshShapeOp::create(builder, mesh).getResults();
     SmallVector<Value> completeMultiIndex =
-        builder.create<affine::AffineDelinearizeIndexOp>(linearIndex, meshShape)
+        affine::AffineDelinearizeIndexOp::create(builder, linearIndex, meshShape)
             .getMultiIndex();
     SmallVector<Value> multiIndex;
     ArrayRef<MeshAxis> opMeshAxes = op.getAxes();
@@ -101,32 +101,32 @@ struct AllSliceOpLowering
     ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     builder.setInsertionPointAfter(op.getOperation());
 
-    Value zero = builder.create<arith::ConstantOp>(builder.getIndexAttr(0));
+    Value zero = arith::ConstantOp::create(builder, builder.getIndexAttr(0));
 
     Operation::result_range processInGroupMultiIndex =
-        builder.create<ProcessMultiIndexOp>(mesh.getSymName(), op.getMeshAxes())
+        ProcessMultiIndexOp::create(builder, mesh.getSymName(), op.getMeshAxes())
             .getResults();
 
     Operation::result_range processGroupShape =
-        builder.create<MeshShapeOp>(mesh.getSymName(), op.getMeshAxes())
+        MeshShapeOp::create(builder, mesh.getSymName(), op.getMeshAxes())
             .getResult();
     Value processGroupSize =
         createCollectiveProcessGroupSize(mesh, op.getMeshAxes(), builder);
 
     int64_t sliceAxis = op.getSliceAxis().getSExtValue();
     Value operandSliceAxisSize =
-        builder.create<tensor::DimOp>(op.getOperand(), sliceAxis);
+        tensor::DimOp::create(builder, op.getOperand(), sliceAxis);
     Value operandSliceAxisSizeModProcessGroupSize =
-        builder.create<arith::RemUIOp>(operandSliceAxisSize, processGroupSize);
-    Value isTargetShapeExactlyDivisible = builder.create<arith::CmpIOp>(
+        arith::RemUIOp::create(builder, operandSliceAxisSize, processGroupSize);
+    Value isTargetShapeExactlyDivisible = arith::CmpIOp::create(builder,
         arith::CmpIPredicate::eq, operandSliceAxisSizeModProcessGroupSize,
         zero);
-    builder.create<cf::AssertOp>(isTargetShapeExactlyDivisible,
+    cf::AssertOp::create(builder, isTargetShapeExactlyDivisible,
                                  "Slicing a tensor with axis size that is "
                                  "not exactly divisible by the "
                                  "mesh process group size is not supported.");
     Value resultSliceAxisSize =
-        builder.create<arith::DivUIOp>(operandSliceAxisSize, processGroupSize);
+        arith::DivUIOp::create(builder, operandSliceAxisSize, processGroupSize);
     OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
         llvm::to_vector_of<OpFoldResult>(processInGroupMultiIndex),
         llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
@@ -139,7 +139,7 @@ struct AllSliceOpLowering
       if (i == sliceAxis) {
         sizes.emplace_back(resultSliceAxisSize);
       } else {
-        Value dimSize = builder.create<tensor::DimOp>(op.getOperand(), i);
+        Value dimSize = tensor::DimOp::create(builder, op.getOperand(), i);
         sizes.emplace_back(dimSize);
       }
     }
@@ -152,10 +152,10 @@ struct AllSliceOpLowering
                  resultSliceAxisSize);
     SmallVector<OpFoldResult> strides(
         operandType.getRank(), getAsIndexOpFoldResult(builder.getContext(), 1));
-    Value slice = builder.create<tensor::ExtractSliceOp>(
+    Value slice = tensor::ExtractSliceOp::create(builder,
         op.getOperand(), offsets, sizes, strides);
     Value newResult =
-        builder.create<tensor::CastOp>(op.getResult().getType(), slice);
+        tensor::CastOp::create(builder, op.getResult().getType(), slice);
     rewriter.replaceAllUsesWith(op.getResult(), newResult);
 
     return success();
@@ -201,7 +201,7 @@ TypedValue<IndexType>
 createCollectiveProcessGroupSize(MeshOp mesh, ArrayRef<MeshAxis> axes,
                                  ImplicitLocOpBuilder &builder) {
   Operation::result_range meshShape =
-      builder.create<mesh::MeshShapeOp>(mesh, axes).getResults();
+      mesh::MeshShapeOp::create(builder, mesh, axes).getResults();
   return cast<TypedValue<IndexType>>(arith::createProduct(
       builder, builder.getLoc(), llvm::to_vector_of<Value>(meshShape),
       builder.getIndexType()));
@@ -212,13 +212,13 @@ createProcessLinearIndex(StringRef mesh, ValueRange processInGroupMultiIndex,
                          ArrayRef<MeshAxis> meshAxes,
                          ImplicitLocOpBuilder &builder) {
   Operation::result_range processGroupShape =
-      builder.create<MeshShapeOp>(mesh, meshAxes).getResult();
+      MeshShapeOp::create(builder, mesh, meshAxes).getResult();
   OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
       llvm::to_vector_of<OpFoldResult>(processInGroupMultiIndex),
       llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
   auto res = dyn_cast<Value>(processInGroupLinearIndex);
   if (!res)
-    res = builder.create<arith::ConstantIndexOp>(
+    res = arith::ConstantIndexOp::create(builder,
         cast<IntegerAttr>(cast<Attribute>(processInGroupLinearIndex)).getInt());
   return cast<TypedValue<IndexType>>(res);
 }
@@ -227,7 +227,7 @@ TypedValue<IndexType> createProcessLinearIndex(StringRef mesh,
                                                ArrayRef<MeshAxis> meshAxes,
                                                ImplicitLocOpBuilder &builder) {
   return createProcessLinearIndex(
-      mesh, builder.create<ProcessMultiIndexOp>(mesh, meshAxes).getResults(),
+      mesh, ProcessMultiIndexOp::create(builder, mesh, meshAxes).getResults(),
       meshAxes, builder);
 }
 } // namespace mlir::mesh
diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
index d2c94b124cdfb..85317a68c9f00 100644
--- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
+++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
@@ -334,13 +334,13 @@ static Operation *replaceOpWithPredicatedOp(RewriterBase &rewriter,
   //
   Location loc = asyncCopyOp->getLoc();
   Value dstElements =
-      rewriter.create<arith::ConstantOp>(loc, asyncCopyOp.getDstElementsAttr());
+      arith::ConstantOp::create(rewriter, loc, asyncCopyOp.getDstElementsAttr());
   Value originalSrcElement =
       asyncCopyOp.getSrcElements() ? asyncCopyOp.getSrcElements() : dstElements;
-  Value c0Index = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto srcElements = rewriter.create<arith::SelectOp>(
+  Value c0Index = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto srcElements = arith::SelectOp::create(rewriter,
       loc, predicate, originalSrcElement, c0Index);
-  auto asyncCopyZeroFillOp = rewriter.create<nvgpu::DeviceAsyncCopyOp>(
+  auto asyncCopyZeroFillOp = nvgpu::DeviceAsyncCopyOp::create(rewriter,
       loc, nvgpu::DeviceAsyncTokenType::get(asyncCopyOp.getContext()),
       asyncCopyOp.getDst(), asyncCopyOp.getDstIndices(), asyncCopyOp.getSrc(),
       asyncCopyOp.getSrcIndices(), asyncCopyOp.getDstElements(), srcElements,
@@ -675,7 +675,7 @@ MmaSyncBuilder::buildMemRefLoads(OpBuilder &b, Location loc,
   for (auto indexing : indexings) {
     Value row = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.row()));
     Value col = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.col()));
-    auto load = b.create<memref::LoadOp>(loc, memref, ValueRange{row, col});
+    auto load = memref::LoadOp::create(b, loc, memref, ValueRange{row, col});
     res.push_back(load);
   }
   return res;
@@ -688,7 +688,7 @@ Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
 
   Type elementType = getElementTypeOrSelf(memref.getType());
   auto vt = VectorType::get(vectorShape, elementType);
-  Value res = b.create<vector::SplatOp>(loc, vt, loads[0]);
+  Value res = vector::SplatOp::create(b, loc, vt, loads[0]);
   foreachIndividualVectorElement(
       res,
       /*applyFn=*/
@@ -697,7 +697,7 @@ Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
       },
       /*reduceFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
-        res = b.create<vector::InsertOp>(loc, v, res, indices);
+        res = vector::InsertOp::create(b, loc, v, res, indices);
       });
 
   return res;
@@ -715,7 +715,7 @@ SmallVector<Operation *> MmaSyncBuilder::buildMemRefStores(
     Value row = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.row()));
     Value col = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.col()));
     Operation *store =
-        b.create<memref::StoreOp>(loc, val, memref, ValueRange{row, col});
+        memref::StoreOp::create(b, loc, val, memref, ValueRange{row, col});
     res.push_back(store);
   }
   return res;
@@ -730,7 +730,7 @@ SmallVector<Operation *> MmaSyncBuilder::buildMmaSyncMemRefStoreOperand(
       vectorToStore,
       /*applyFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
-        return b.create<vector::ExtractOp>(loc, vectorToStore, indices);
+        return vector::ExtractOp::create(b, loc, vectorToStore, indices);
       },
       /*reduceFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
@@ -810,7 +810,7 @@ FailureOr<Operation *> MmaSyncBuilder::buildMmaSync(LinalgOp linalgOp) {
                                             rhsIndexFn, rhsShape);
   Value res = buildMmaSyncMemRefLoadOperand(b, loc, laneId, resMemRef,
                                             resIndexFn, resShape);
-  res = b.create<nvgpu::MmaSyncOp>(loc, lhs, rhs, res, info.mmaShape,
+  res = nvgpu::MmaSyncOp::create(b, loc, lhs, rhs, res, info.mmaShape,
                                    info.tf32Enabled);
   buildMmaSyncMemRefStoreOperand(b, loc, res, laneId, resMemRef, resIndexFn,
                                  resShape);
@@ -832,7 +832,7 @@ DiagnosedSilenceableFailure transform::RewriteMatmulAsMmaSyncOp::applyToOne(
     }
     Location loc = linalgOp.getLoc();
     // TODO: more robust computation of laneId, for now assume a single warp.
-    Value laneId = rewriter.create<gpu::ThreadIdOp>(
+    Value laneId = gpu::ThreadIdOp::create(rewriter,
         loc, rewriter.getIndexType(), gpu::Dimension::x);
     if (succeeded(MmaSyncBuilder(rewriter, loc, laneId).buildMmaSync(linalgOp)))
       fail = false;
@@ -897,12 +897,12 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
     ArrayRef<TypedValue<MemRefType>> sharedMemBuffers,
     TypedValue<nvgpu::MBarrierGroupType> barrier) {
   SmallVector<Operation *> loadOps;
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Value tidx = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Value tidx = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
   Value cond =
-      rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, tidx, zero);
+      arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, tidx, zero);
   // clang-format off
-  rewriter.create<scf::IfOp>(
+  scf::IfOp::create(rewriter,
     /*location=*/loc,
     /*conditional=*/cond,
     /*thenBuilder=*/
@@ -917,14 +917,14 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
       // TODO: Note that cutlass predeclares the barrier arrive tx before the tma.async.load.
       // This may or may not have perf implications.
       buildBarrierArriveTx(barrier, sizes);
-      rewriter.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(rewriter, loc);
     },
     /*elseBuilder=*/
     [&](OpBuilder &lb, Location loc) {
       // TODO: is this for no-thread divergence?
       // Should we just yield the size and hoist?
       buildBarrierArriveTx(barrier, getAsIndexOpFoldResult(rewriter.getContext(), 0));
-      rewriter.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(rewriter, loc);
     });
   // clang-format on
   return loadOps;
@@ -939,14 +939,14 @@ static Attribute getSharedAddressSpaceAttribute(OpBuilder &b) {
 TypedValue<nvgpu::MBarrierGroupType>
 HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) {
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value barrier = rewriter.create<nvgpu::MBarrierCreateOp>(
+  Value barrier = nvgpu::MBarrierCreateOp::create(rewriter,
       loc,
       nvgpu::MBarrierGroupType::get(rewriter.getContext(), sharedMemorySpace));
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierInitOp>(
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierInitOp::create(rewriter,
       loc, barrier, getValueOrCreateConstantIndexOp(rewriter, loc, numThreads),
       zero, Value());
-  rewriter.create<gpu::BarrierOp>(loc);
+  gpu::BarrierOp::create(rewriter, loc);
   return cast<TypedValue<nvgpu::MBarrierGroupType>>(barrier);
 }
 
@@ -955,7 +955,7 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
                                            gpu::LaunchOp launchOp) {
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(launchOp);
-  Value unrankedMemRef = rewriter.create<memref::CastOp>(
+  Value unrankedMemRef = memref::CastOp::create(rewriter,
       loc,
       UnrankedMemRefType::get(memref.getType().getElementType(),
                               memref.getType().getMemorySpace()),
@@ -966,7 +966,7 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
       getValueOrCreateConstantIndexOp(rewriter, loc, mixedSizes);
 
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value desc = rewriter.create<nvgpu::TmaCreateDescriptorOp>(
+  Value desc = nvgpu::TmaCreateDescriptorOp::create(rewriter,
       loc,
       nvgpu::TensorMapDescriptorType::get(
           rewriter.getContext(),
@@ -985,8 +985,8 @@ OpFoldResult HopperBuilder::buildTmaAsyncLoad(
     TypedValue<nvgpu::MBarrierGroupType> barrier,
     SmallVectorImpl<Operation *> &loadOps) {
   MLIRContext *ctx = rewriter.getContext();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Operation *loadOp = rewriter.create<nvgpu::TmaAsyncLoadOp>(
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Operation *loadOp = nvgpu::TmaAsyncLoadOp::create(rewriter,
       loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero,
       Value(), Value());
   loadOps.push_back(loadOp);
@@ -1012,22 +1012,22 @@ void HopperBuilder::buildBarrierArriveTx(
   OpFoldResult size =
       affine::makeComposedFoldedAffineApply(rewriter, loc, sumExpr, mixedSizes);
   Value sizeVal = getValueOrCreateConstantIndexOp(rewriter, loc, size);
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierArriveExpectTxOp>(loc, barrier, sizeVal, zero,
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierArriveExpectTxOp::create(rewriter, loc, barrier, sizeVal, zero,
                                                    Value());
 }
 
 void HopperBuilder::buildTryWaitParity(
     TypedValue<nvgpu::MBarrierGroupType> barrier) {
   Type i1 = rewriter.getI1Type();
-  Value parity = rewriter.create<LLVM::ConstantOp>(loc, i1, 0);
+  Value parity = LLVM::ConstantOp::create(rewriter, loc, i1, 0);
   // 10M is an arbitrary, not too small or too big number to specify the number
   // of ticks before retry.
   // TODO: hoist this in a default dialect constant.
   Value ticksBeforeRetry =
-      rewriter.create<arith::ConstantIndexOp>(loc, 10000000);
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierTryWaitParityOp>(loc, barrier, parity,
+      arith::ConstantIndexOp::create(rewriter, loc, 10000000);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierTryWaitParityOp::create(rewriter, loc, barrier, parity,
                                                   ticksBeforeRetry, zero);
 }
 
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
index 47a0c7096de95..b8baadb4ef241 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
@@ -109,17 +109,17 @@ static Value buildNumReadElements(OpBuilder &b, Location loc,
   for (auto [pos, sz] : llvm::zip(transferMask->extractPosition,
                                   transferMask->createMaskOp->getOperands())) {
     Value cmp =
-        b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
-                                b.create<arith::ConstantIndexOp>(loc, pos), sz);
+        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt,
+                                arith::ConstantIndexOp::create(b, loc, pos), sz);
     if (!cond) {
       cond = cmp;
       continue;
     }
-    cond = b.create<arith::AndIOp>(loc, cmp, cond);
+    cond = arith::AndIOp::create(b, loc, cmp, cond);
   }
-  return b.create<arith::SelectOp>(
+  return arith::SelectOp::create(b,
       loc, cond, transferMask->createMaskOp->getOperands().back(),
-      b.create<arith::ConstantIndexOp>(loc, 0));
+      arith::ConstantIndexOp::create(b, loc, 0));
 }
 
 /// Return "true" if the conversion to async copy is supported by "async copy".
@@ -251,7 +251,7 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
       int64_t sizeInBytes =
           (dstMemref.getElementTypeBitWidth() * numElements) / 8;
       // bypass_l1 only possible with 16 byte transfer.
-      Value token = rewriter.create<nvgpu::DeviceAsyncCopyOp>(
+      Value token = nvgpu::DeviceAsyncCopyOp::create(rewriter,
           writeOp->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
           /*dst=*/storeBase, /*dstIndices=*/nvgpu::getIndices(writeOp),
           /*src=*/loadBase,
@@ -264,10 +264,10 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
     }
 
     // Create the group and wait for it right after.
-    Value groupToken = rewriter.create<nvgpu::DeviceAsyncCreateGroupOp>(
+    Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(rewriter,
         op->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
         tokens);
-    rewriter.create<nvgpu::DeviceAsyncWaitOp>(op->getLoc(), groupToken,
+    nvgpu::DeviceAsyncWaitOp::create(rewriter, op->getLoc(), groupToken,
                                               nullptr);
     // Clean up old stores.
     for (Operation *writeOp : group)
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
index 44e7fa961da12..2d2ff94a25f92 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
@@ -74,27 +74,27 @@ static Value permuteVectorOffset(OpBuilder &b, Location loc,
   int64_t mask = (1LL << (m - n)) - 1;
   if (permuteEveryN > 1)
     mask = mask << llvm::Log2_64(permuteEveryN);
-  Value srcBits = b.create<arith::ConstantIndexOp>(loc, mask);
-  srcBits = b.create<arith::AndIOp>(loc, src, srcBits);
+  Value srcBits = arith::ConstantIndexOp::create(b, loc, mask);
+  srcBits = arith::AndIOp::create(b, loc, src, srcBits);
 
   // Use the src bits to permute the target bits b[N:M] containing the
   // vector offset.
   if (permuteEveryN > 1) {
     int64_t shlBits = n - llvm::Log2_64(permuteEveryN);
     if (shlBits > 0) {
-      Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, shlBits);
+      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, shlBits);
       srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
     } else if (shlBits < 0) {
-      Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, -1 * shlBits);
+      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, -1 * shlBits);
       srcBits = b.createOrFold<arith::ShRUIOp>(loc, srcBits, finalShiftVal);
     }
   } else {
-    Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, n);
+    Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, n);
     srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
   }
 
   Value permutedVectorIdx =
-      b.create<arith::XOrIOp>(loc, indices[tgtDim], srcBits);
+      arith::XOrIOp::create(b, loc, indices[tgtDim], srcBits);
   return permutedVectorIdx;
 }
 
diff --git a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
index 793db73575b4f..a378b09c27c41 100644
--- a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
@@ -72,7 +72,7 @@ Value getScalarOrTensorConstant(OpBuilder &builder, Location loc, Value scalar,
 
   // Create tensor splat
   auto tensorConstant =
-      builder.create<tensor::SplatOp>(loc, scalar, referenceShape);
+      tensor::SplatOp::create(builder, loc, scalar, referenceShape);
   return tensorConstant;
 }
 
@@ -94,21 +94,21 @@ std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
   // Get unranked input shape and total size
   auto *context = builder.getContext();
   auto shapeType = shape::getExtentTensorType(context);
-  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
-  Value inputSize = builder.create<shape::NumElementsOp>(
+  auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
+  Value inputSize = shape::NumElementsOp::create(builder,
       loc, builder.getIndexType(), inputShape);
 
   // Turn input size into 1D tensor
   auto flatShapeType = shape::getExtentTensorType(context, 1);
   auto flatInputShape =
-      builder.create<tensor::FromElementsOp>(loc, flatShapeType, inputSize);
+      tensor::FromElementsOp::create(builder, loc, flatShapeType, inputSize);
 
   // Reshape input tensor into 1D
   auto inputType = cast<UnrankedTensorType>(input.getType());
   auto elementType = inputType.getElementType();
   auto flatInputType =
       RankedTensorType::get({ShapedType::kDynamic}, elementType);
-  auto flatInput = builder.create<tensor::ReshapeOp>(loc, flatInputType, input,
+  auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
                                                      flatInputShape);
   return std::make_pair(flatInput, inputShape);
 }
@@ -142,30 +142,30 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto *context = builder.getContext();
   auto indexType = builder.getIndexType();
   auto shapeType = shape::getExtentTensorType(context);
-  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
+  auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
 
   // Get shape and sizes on left and right of axis
-  auto axisValue = builder.create<arith::ConstantIndexOp>(loc, axis);
-  auto axisNextValue = builder.create<arith::ConstantIndexOp>(loc, axis + 1);
+  auto axisValue = arith::ConstantIndexOp::create(builder, loc, axis);
+  auto axisNextValue = arith::ConstantIndexOp::create(builder, loc, axis + 1);
   auto shapeLeft =
       builder
           .create<shape::SplitAtOp>(loc, TypeRange{shapeType, shapeType},
                                     inputShape, axisValue)
           .getResult(0);
   auto sizeLeft =
-      builder.create<shape::NumElementsOp>(loc, indexType, shapeLeft);
+      shape::NumElementsOp::create(builder, loc, indexType, shapeLeft);
   auto shapeRight =
       builder
           .create<shape::SplitAtOp>(loc, TypeRange{shapeType, shapeType},
                                     inputShape, axisNextValue)
           .getResult(1);
   auto sizeRight =
-      builder.create<shape::NumElementsOp>(loc, indexType, shapeRight);
+      shape::NumElementsOp::create(builder, loc, indexType, shapeRight);
 
   // Compute flat input shape as a 3-element 1D tensor
-  auto axisSizeValue = builder.create<arith::ConstantIndexOp>(loc, axisSize);
+  auto axisSizeValue = arith::ConstantIndexOp::create(builder, loc, axisSize);
   auto flatShapeType = shape::getExtentTensorType(context, 3);
-  auto flatInputShape = builder.create<tensor::FromElementsOp>(
+  auto flatInputShape = tensor::FromElementsOp::create(builder,
       loc, flatShapeType, ValueRange{sizeLeft, axisSizeValue, sizeRight});
 
   // Reshape input to 3D tensor
@@ -173,7 +173,7 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto elementType = inputType.getElementType();
   auto flatInputType = RankedTensorType::get(
       {ShapedType::kDynamic, axisSize, ShapedType::kDynamic}, elementType);
-  auto flatInput = builder.create<tensor::ReshapeOp>(loc, flatInputType, input,
+  auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
                                                      flatInputShape);
 
   return std::make_pair(flatInput, inputShape);
@@ -192,7 +192,7 @@ Value restoreUnrankedTensorShape(OpBuilder &builder, Location loc, Value input,
   auto inputType = cast<RankedTensorType>(input.getType());
   auto elementType = inputType.getElementType();
   auto unrankedType = UnrankedTensorType::get(elementType);
-  return builder.create<tensor::ReshapeOp>(loc, unrankedType, input,
+  return tensor::ReshapeOp::create(builder, loc, unrankedType, input,
                                            inputShape);
 }
 
@@ -215,7 +215,7 @@ Value materializePerChannelScales(OpBuilder &builder, Location loc,
   auto tensorType =
       RankedTensorType::get({(int64_t)scales.size()}, expressedType);
   auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, scalesAttr);
 }
 
 // Create a tensor constant containing all zero points in a per-channel
@@ -239,7 +239,7 @@ Value materializePerChannelZeroPoints(
   auto tensorType =
       RankedTensorType::get({(int64_t)zeroPoints.size()}, storageType);
   auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, zeroPointsAttr);
 }
 
 // Create a tensor constant containing all scales in a sub-channel quantized
@@ -263,7 +263,7 @@ Value materializeSubChannelScales(
   auto tensorType =
       RankedTensorType::get(scales.getType().getShape(), expressedType);
   auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, scalesAttr);
 }
 
 // Create a tensor constant containing all zero points in a sub-channel
@@ -287,7 +287,7 @@ Value materializeSubChannelZeroPoints(
   auto tensorType =
       RankedTensorType::get(zeroPoints.getType().getShape(), storageType);
   auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, zeroPointsAttr);
 }
 
 // Clamp the given scalar or tensor input using the storage bounds encoded in
@@ -314,9 +314,9 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
   // Materialize bounds
   auto inputType = input.getType();
   auto storageType = quantizedType.getStorageType();
-  auto storageMinScalar = builder.create<arith::ConstantIntOp>(
+  auto storageMinScalar = arith::ConstantIntOp::create(builder,
       loc, storageType, quantizedType.getStorageTypeMin());
-  auto storageMaxScalar = builder.create<arith::ConstantIntOp>(
+  auto storageMaxScalar = arith::ConstantIntOp::create(builder,
       loc, storageType, quantizedType.getStorageTypeMax());
   auto storageMin = getScalarOrTensorConstant(builder, loc, storageMinScalar,
                                               inputType, inputShape);
@@ -325,11 +325,11 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
 
   // Clamp
   if (quantizedType.isSigned()) {
-    input = builder.create<arith::MaxSIOp>(loc, input, storageMin);
-    input = builder.create<arith::MinSIOp>(loc, input, storageMax);
+    input = arith::MaxSIOp::create(builder, loc, input, storageMin);
+    input = arith::MinSIOp::create(builder, loc, input, storageMax);
   } else {
-    input = builder.create<arith::MaxUIOp>(loc, input, storageMin);
-    input = builder.create<arith::MinUIOp>(loc, input, storageMax);
+    input = arith::MaxUIOp::create(builder, loc, input, storageMin);
+    input = arith::MinUIOp::create(builder, loc, input, storageMax);
   }
   return input;
 }
@@ -338,16 +338,16 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
 Value convertFloatToInteger(OpBuilder &builder, Location loc, Value input,
                             Type resultType, bool isSigned) {
   if (isSigned)
-    return builder.create<arith::FPToSIOp>(loc, resultType, input);
-  return builder.create<arith::FPToUIOp>(loc, resultType, input);
+    return arith::FPToSIOp::create(builder, loc, resultType, input);
+  return arith::FPToUIOp::create(builder, loc, resultType, input);
 }
 
 // Emit op 'arith.sitofp' or 'arith.uitofp'.
 Value convertIntegerToFloat(OpBuilder &builder, Location loc, Value input,
                             Type resultType, bool isSigned) {
   if (isSigned)
-    return builder.create<arith::SIToFPOp>(loc, resultType, input);
-  return builder.create<arith::UIToFPOp>(loc, resultType, input);
+    return arith::SIToFPOp::create(builder, loc, resultType, input);
+  return arith::UIToFPOp::create(builder, loc, resultType, input);
 }
 
 // Quantize a scalar or ranked tensor value. The stored value is clamped using
@@ -362,7 +362,7 @@ Value quantizeValue(OpBuilder &builder, Location loc, Value input,
   scale = getScalarOrTensorConstant(builder, loc, scale, inputType, inputShape);
 
   // Scale input
-  auto scaledValue = builder.create<arith::DivFOp>(loc, input, scale);
+  auto scaledValue = arith::DivFOp::create(builder, loc, input, scale);
 
   // Skip unnecessary computations if no zero point is given
   Value storedValueFloat = scaledValue;
@@ -377,7 +377,7 @@ Value quantizeValue(OpBuilder &builder, Location loc, Value input,
 
     // Add zero point to stored value
     storedValueFloat =
-        builder.create<arith::AddFOp>(loc, scaledValue, zeroPoint);
+        arith::AddFOp::create(builder, loc, scaledValue, zeroPoint);
   }
 
   // Convert stored value to storage type
@@ -418,11 +418,11 @@ Value dequantizeValue(OpBuilder &builder, Location loc, Value input,
                                       quantizedType.isSigned());
 
     // Subtract zero point to stored value
-    result = builder.create<arith::SubFOp>(loc, result, zeroPoint);
+    result = arith::SubFOp::create(builder, loc, result, zeroPoint);
   }
 
   // Multiply by scale
-  result = builder.create<arith::MulFOp>(loc, result, scale);
+  result = arith::MulFOp::create(builder, loc, result, scale);
   return result;
 }
 
@@ -477,11 +477,11 @@ Value convertPerLayerRanked(OpBuilder &builder, Location loc, Operation *op,
   auto storageType = quantizedType.getStorageType();
   auto scaleAttr =
       builder.getFloatAttr(expressedType, quantizedType.getScale());
-  auto scale = builder.create<arith::ConstantOp>(loc, expressedType, scaleAttr);
+  auto scale = arith::ConstantOp::create(builder, loc, expressedType, scaleAttr);
   auto zeroPointAttr =
       builder.getIntegerAttr(storageType, quantizedType.getZeroPoint());
   auto zeroPoint =
-      builder.create<arith::ConstantOp>(loc, storageType, zeroPointAttr);
+      arith::ConstantOp::create(builder, loc, storageType, zeroPointAttr);
 
   auto inputShape = getScalarOrTensorShape(builder, loc, input);
   return convertRanked(builder, loc, op, input, inputShape, scale, zeroPoint,
@@ -546,7 +546,7 @@ Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
                          ? quantizedType.getStorageType()
                          : quantizedType.getExpressedType();
   auto initShape = tensor::getMixedSizes(builder, loc, input);
-  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+  Value init = tensor::EmptyOp::create(builder, loc, initShape, elementType);
 
   SmallVector<utils::IteratorType> iteratorTypes(inputRank,
                                                  utils::IteratorType::parallel);
@@ -572,7 +572,7 @@ Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
                               convertRanked(builder, loc, op, input, {}, scale,
                                             zeroPoint, quantizedType);
 
-                          builder.create<linalg::YieldOp>(loc, result);
+                          linalg::YieldOp::create(builder, loc, result);
                         })
                     .getResult(0);
 
@@ -642,7 +642,7 @@ Value convertSubChannel(OpBuilder &builder, Location loc, Operation *op,
                          ? quantizedType.getStorageType()
                          : quantizedType.getExpressedType();
   auto initShape = tensor::getMixedSizes(builder, loc, input);
-  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+  Value init = tensor::EmptyOp::create(builder, loc, initShape, elementType);
 
   SmallVector<utils::IteratorType> iteratorTypes(inputRank,
                                                  utils::IteratorType::parallel);
@@ -675,7 +675,7 @@ Value convertSubChannel(OpBuilder &builder, Location loc, Operation *op,
                               convertRanked(builder, loc, op, input, {}, scale,
                                             zeroPoint, quantizedType);
 
-                          builder.create<linalg::YieldOp>(loc, result);
+                          linalg::YieldOp::create(builder, loc, result);
                         })
                     .getResult(0);
 
@@ -729,7 +729,7 @@ struct DequantizeCastOpConversion
     // Convert quantized input to storage type
     auto storageScalarOrTensorType =
         getScalarOrTensorType(quantizedType.getStorageType(), input.getType());
-    input = rewriter.create<quant::StorageCastOp>(
+    input = quant::StorageCastOp::create(rewriter,
         loc, storageScalarOrTensorType, input);
 
     auto result = convertQuantized(rewriter, loc, op, input, quantizedType);
diff --git a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
index 4009faa21576d..f5e0aef365ff7 100644
--- a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
@@ -46,7 +46,7 @@ class QuantizedTypeConverter : public TypeConverter {
 
   static Value materializeConversion(OpBuilder &builder, Type type,
                                      ValueRange inputs, Location loc) {
-    return builder.create<quant::StorageCastOp>(loc, type,
+    return quant::StorageCastOp::create(builder, loc, type,
                                                 llvm::getSingleElement(inputs));
   }
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 5a3bd984530db..82fa50a6ac6e0 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -85,7 +85,7 @@ void SCFDialect::initialize() {
 
 /// Default callback for IfOp builders. Inserts a yield without arguments.
 void mlir::scf::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<scf::YieldOp>(loc);
+  scf::YieldOp::create(builder, loc);
 }
 
 /// Verifies that the first block of the given `region` is terminated by a
@@ -241,12 +241,12 @@ struct MultiBlockExecuteInliner : public OpRewritePattern<ExecuteRegionOp> {
     Block *postBlock = rewriter.splitBlock(prevBlock, op->getIterator());
     rewriter.setInsertionPointToEnd(prevBlock);
 
-    rewriter.create<cf::BranchOp>(op.getLoc(), &op.getRegion().front());
+    cf::BranchOp::create(rewriter, op.getLoc(), &op.getRegion().front());
 
     for (Block &blk : op.getRegion()) {
       if (YieldOp yieldOp = dyn_cast<YieldOp>(blk.getTerminator())) {
         rewriter.setInsertionPoint(yieldOp);
-        rewriter.create<cf::BranchOp>(yieldOp.getLoc(), postBlock,
+        cf::BranchOp::create(rewriter, yieldOp.getLoc(), postBlock,
                                       yieldOp.getResults());
         rewriter.eraseOp(yieldOp);
       }
@@ -557,7 +557,7 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
       [](OpBuilder &, Location, Value, ValueRange) {});
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
@@ -673,7 +673,7 @@ void mlir::scf::promote(RewriterBase &rewriter, scf::ForallOp forallOp) {
     Value dst = parallelInsertSliceOp.getDest();
     Value src = parallelInsertSliceOp.getSource();
     if (llvm::isa<TensorType>(src.getType())) {
-      results.push_back(rewriter.create<tensor::InsertSliceOp>(
+      results.push_back(tensor::InsertSliceOp::create(rewriter,
           forallOp.getLoc(), dst.getType(), src, dst,
           parallelInsertSliceOp.getOffsets(), parallelInsertSliceOp.getSizes(),
           parallelInsertSliceOp.getStrides(),
@@ -722,7 +722,7 @@ LoopNest mlir::scf::buildLoopNest(
   ValueRange currentIterArgs = iterArgs;
   Location currentLoc = loc;
   for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
-    auto loop = builder.create<scf::ForOp>(
+    auto loop = scf::ForOp::create(builder,
         currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
             ValueRange args) {
@@ -742,7 +742,7 @@ LoopNest mlir::scf::buildLoopNest(
   // For all loops but the innermost, yield the results of the nested loop.
   for (unsigned i = 0, e = loops.size() - 1; i < e; ++i) {
     builder.setInsertionPointToEnd(loops[i].getBody());
-    builder.create<scf::YieldOp>(loc, loops[i + 1].getResults());
+    scf::YieldOp::create(builder, loc, loops[i + 1].getResults());
   }
 
   // In the body of the innermost loop, call the body building function if any
@@ -756,7 +756,7 @@ LoopNest mlir::scf::buildLoopNest(
          "loop nest body must return as many values as loop has iteration "
          "arguments");
   builder.setInsertionPointToEnd(loops.back().getBody());
-  builder.create<scf::YieldOp>(loc, results);
+  scf::YieldOp::create(builder, loc, results);
 
   // Return the loops.
   ValueVector nestResults;
@@ -801,7 +801,7 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   }
 
   // 2. Create the new forOp shell.
-  scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+  scf::ForOp newForOp = scf::ForOp::create(rewriter,
       forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
       forOp.getStep(), newIterOperands);
   newForOp->setAttrs(forOp->getAttrs());
@@ -831,7 +831,7 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
                          clonedYieldOp.getOperand(yieldIdx));
   SmallVector<Value> newYieldOperands = clonedYieldOp.getOperands();
   newYieldOperands[yieldIdx] = castOut;
-  rewriter.create<scf::YieldOp>(newForOp.getLoc(), newYieldOperands);
+  scf::YieldOp::create(rewriter, newForOp.getLoc(), newYieldOperands);
   rewriter.eraseOp(clonedYieldOp);
 
   // 6. Inject an outgoing cast op after the forOp.
@@ -926,7 +926,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     if (!canonicalize)
       return failure();
 
-    scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+    scf::ForOp newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), newIterArgs);
     newForOp->setAttrs(forOp->getAttrs());
@@ -970,7 +970,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
       for (unsigned idx = 0, e = keepMask.size(); idx < e; ++idx)
         if (keepMask[idx])
           filteredOperands.push_back(mergedTerminator.getOperand(idx));
-      rewriter.create<scf::YieldOp>(mergedTerminator.getLoc(),
+      scf::YieldOp::create(rewriter, mergedTerminator.getLoc(),
                                     filteredOperands);
     };
 
@@ -1111,7 +1111,7 @@ struct ForOpTensorCastFolder : public OpRewritePattern<ForOp> {
           op, replaceAndCastForOpIterArg(
                   rewriter, op, iterOpOperand, incomingCast.getSource(),
                   [](OpBuilder &b, Location loc, Type type, Value source) {
-                    return b.create<tensor::CastOp>(loc, type, source);
+                    return tensor::CastOp::create(b, loc, type, source);
                   }));
       return success();
     }
@@ -1685,7 +1685,7 @@ struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
 
     // Step 3. Create a new scf.forall op with the new shared_outs' operands
     //         fetched earlier
-    auto newForallOp = rewriter.create<scf::ForallOp>(
+    auto newForallOp = scf::ForallOp::create(rewriter,
         forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
         forallOp.getMapping(),
@@ -1782,7 +1782,7 @@ struct ForallOpSingleOrZeroIterationDimsFolder
 
     // Replace the loop by a lower-dimensional loop.
     ForallOp newOp;
-    newOp = rewriter.create<ForallOp>(loc, newMixedLowerBounds,
+    newOp = ForallOp::create(rewriter, loc, newMixedLowerBounds,
                                       newMixedUpperBounds, newMixedSteps,
                                       op.getOutputs(), std::nullopt, nullptr);
     newOp.getBodyRegion().getBlocks().clear();
@@ -1867,7 +1867,7 @@ struct FoldTensorCastOfOutputIntoForallOp
 
     // Create new loop.
     Location loc = forallOp.getLoc();
-    auto newForallOp = rewriter.create<ForallOp>(
+    auto newForallOp = ForallOp::create(rewriter,
         loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
         forallOp.getMixedStep(), newOutputTensors, forallOp.getMapping(),
         [&](OpBuilder nestedBuilder, Location nestedLoc, ValueRange bbArgs) {
@@ -1875,7 +1875,7 @@ struct FoldTensorCastOfOutputIntoForallOp
               llvm::to_vector(bbArgs.take_back(forallOp->getNumResults()));
           for (auto [index, cast] : tensorCastProducers) {
             Value &oldTypeBBArg = castBlockArgs[index];
-            oldTypeBBArg = nestedBuilder.create<tensor::CastOp>(
+            oldTypeBBArg = tensor::CastOp::create(nestedBuilder,
                 nestedLoc, cast.dstType, oldTypeBBArg);
           }
 
@@ -1902,7 +1902,7 @@ struct FoldTensorCastOfOutputIntoForallOp
     SmallVector<Value> castResults = newForallOp.getResults();
     for (auto &item : tensorCastProducers) {
       Value &oldTypeResult = castResults[item.first];
-      oldTypeResult = rewriter.create<tensor::CastOp>(loc, item.second.dstType,
+      oldTypeResult = tensor::CastOp::create(rewriter, loc, item.second.dstType,
                                                       oldTypeResult);
     }
     rewriter.replaceOp(forallOp, castResults);
@@ -2309,7 +2309,7 @@ struct RemoveUnusedResults : public OpRewritePattern<IfOp> {
 
     // Create a replacement operation with empty then and else regions.
     auto newOp =
-        rewriter.create<IfOp>(op.getLoc(), newTypes, op.getCondition());
+        IfOp::create(rewriter, op.getLoc(), newTypes, op.getCondition());
     rewriter.createBlock(&newOp.getThenRegion());
     rewriter.createBlock(&newOp.getElseRegion());
 
@@ -2372,7 +2372,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
     if (nonHoistable.size() == op->getNumResults())
       return failure();
 
-    IfOp replacement = rewriter.create<IfOp>(op.getLoc(), nonHoistable, cond,
+    IfOp replacement = IfOp::create(rewriter, op.getLoc(), nonHoistable, cond,
                                              /*withElseRegion=*/false);
     if (replacement.thenBlock())
       rewriter.eraseBlock(replacement.thenBlock());
@@ -2398,7 +2398,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       } else if (trueVal == falseVal)
         results[it.index()] = trueVal;
       else
-        results[it.index()] = rewriter.create<arith::SelectOp>(
+        results[it.index()] = arith::SelectOp::create(rewriter,
             op.getLoc(), cond, trueVal, falseVal);
     }
 
@@ -2450,7 +2450,7 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantTrue)
-          constantTrue = rewriter.create<arith::ConstantOp>(
+          constantTrue = arith::ConstantOp::create(rewriter,
               op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
 
         rewriter.modifyOpInPlace(use.getOwner(),
@@ -2460,7 +2460,7 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantFalse)
-          constantFalse = rewriter.create<arith::ConstantOp>(
+          constantFalse = arith::ConstantOp::create(rewriter,
               op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
 
         rewriter.modifyOpInPlace(use.getOwner(),
@@ -2547,7 +2547,7 @@ struct ReplaceIfYieldWithConditionOrValue : public OpRewritePattern<IfOp> {
       if (!trueVal && falseVal) {
         if (!opResult.use_empty()) {
           Dialect *constDialect = trueResult.getDefiningOp()->getDialect();
-          Value notCond = rewriter.create<arith::XOrIOp>(
+          Value notCond = arith::XOrIOp::create(rewriter,
               op.getLoc(), op.getCondition(),
               constDialect
                   ->materializeConstant(rewriter,
@@ -2661,7 +2661,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
     SmallVector<Type> mergedTypes(prevIf.getResultTypes());
     llvm::append_range(mergedTypes, nextIf.getResultTypes());
 
-    IfOp combinedIf = rewriter.create<IfOp>(
+    IfOp combinedIf = IfOp::create(rewriter,
         nextIf.getLoc(), mergedTypes, prevIf.getCondition(), /*hasElse=*/false);
     rewriter.eraseBlock(&combinedIf.getThenRegion().back());
 
@@ -2677,7 +2677,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
 
       SmallVector<Value> mergedYields(thenYield.getOperands());
       llvm::append_range(mergedYields, thenYield2.getOperands());
-      rewriter.create<YieldOp>(thenYield2.getLoc(), mergedYields);
+      YieldOp::create(rewriter, thenYield2.getLoc(), mergedYields);
       rewriter.eraseOp(thenYield);
       rewriter.eraseOp(thenYield2);
     }
@@ -2701,7 +2701,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
         SmallVector<Value> mergedElseYields(elseYield.getOperands());
         llvm::append_range(mergedElseYields, elseYield2.getOperands());
 
-        rewriter.create<YieldOp>(elseYield2.getLoc(), mergedElseYields);
+        YieldOp::create(rewriter, elseYield2.getLoc(), mergedElseYields);
         rewriter.eraseOp(elseYield);
         rewriter.eraseOp(elseYield2);
       }
@@ -2823,9 +2823,9 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     }
 
     Location loc = op.getLoc();
-    Value newCondition = rewriter.create<arith::AndIOp>(
+    Value newCondition = arith::AndIOp::create(rewriter,
         loc, op.getCondition(), nestedIf.getCondition());
-    auto newIf = rewriter.create<IfOp>(loc, op.getResultTypes(), newCondition);
+    auto newIf = IfOp::create(rewriter, loc, op.getResultTypes(), newCondition);
     Block *newIfBlock = rewriter.createBlock(&newIf.getThenRegion());
 
     SmallVector<Value> results;
@@ -2833,7 +2833,7 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     rewriter.setInsertionPoint(newIf);
 
     for (auto idx : elseYieldsToUpgradeToSelect)
-      results[idx] = rewriter.create<arith::SelectOp>(
+      results[idx] = arith::SelectOp::create(rewriter,
           op.getLoc(), op.getCondition(), thenYield[idx], elseYield[idx]);
 
     rewriter.mergeBlocks(nestedIf.thenBlock(), newIfBlock);
@@ -2842,7 +2842,7 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     if (!elseYield.empty()) {
       rewriter.createBlock(&newIf.getElseRegion());
       rewriter.setInsertionPointToEnd(newIf.elseBlock());
-      rewriter.create<YieldOp>(loc, elseYield);
+      YieldOp::create(rewriter, loc, elseYield);
     }
     rewriter.replaceOp(op, results);
     return success();
@@ -3160,7 +3160,7 @@ struct ParallelOpSingleOrZeroIterationDimsFolder
     }
     // Replace the parallel loop by lower-dimensional parallel loop.
     auto newOp =
-        rewriter.create<ParallelOp>(op.getLoc(), newLowerBounds, newUpperBounds,
+        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds, newUpperBounds,
                                     newSteps, op.getInitVals(), nullptr);
     // Erase the empty block that was inserted by the builder.
     rewriter.eraseBlock(newOp.getBody());
@@ -3541,7 +3541,7 @@ struct WhileConditionTruth : public OpRewritePattern<WhileOp> {
       if (std::get<0>(yieldedAndBlockArgs) == term.getCondition()) {
         if (!std::get<1>(yieldedAndBlockArgs).use_empty()) {
           if (!constantTrue)
-            constantTrue = rewriter.create<arith::ConstantOp>(
+            constantTrue = arith::ConstantOp::create(rewriter,
                 op.getLoc(), term.getCondition().getType(),
                 rewriter.getBoolAttr(true));
 
@@ -3685,7 +3685,7 @@ struct RemoveLoopInvariantArgsFromBeforeBlock
     }
 
     auto newWhile =
-        rewriter.create<WhileOp>(op.getLoc(), op.getResultTypes(), newInitArgs);
+        WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(), newInitArgs);
 
     Block &newBeforeBlock = *rewriter.createBlock(
         &newWhile.getBefore(), /*insertPt*/ {},
@@ -3807,7 +3807,7 @@ struct RemoveLoopInvariantValueYielded : public OpRewritePattern<WhileOp> {
                                                newCondOpArgs);
     }
 
-    auto newWhile = rewriter.create<WhileOp>(op.getLoc(), newAfterBlockType,
+    auto newWhile = WhileOp::create(rewriter, op.getLoc(), newAfterBlockType,
                                              op.getOperands());
 
     Block &newAfterBlock =
@@ -3914,7 +3914,7 @@ struct WhileUnusedResult : public OpRewritePattern<WhileOp> {
     }
 
     auto newWhile =
-        rewriter.create<WhileOp>(op.getLoc(), newResultTypes, op.getInits());
+        WhileOp::create(rewriter, op.getLoc(), newResultTypes, op.getInits());
 
     Block &newAfterBlock = *rewriter.createBlock(
         &newWhile.getAfter(), /*insertPt*/ {}, newResultTypes, newArgLocs);
@@ -4043,7 +4043,7 @@ struct WhileRemoveUnusedArgs : public OpRewritePattern<WhileOp> {
 
     Location loc = op.getLoc();
     auto newWhileOp =
-        rewriter.create<WhileOp>(loc, op.getResultTypes(), newInits,
+        WhileOp::create(rewriter, loc, op.getResultTypes(), newInits,
                                  /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
@@ -4091,7 +4091,7 @@ struct WhileRemoveDuplicatedResults : public OpRewritePattern<WhileOp> {
     ValueRange argsRange(newArgs);
 
     Location loc = op.getLoc();
-    auto newWhileOp = rewriter.create<scf::WhileOp>(
+    auto newWhileOp = scf::WhileOp::create(rewriter,
         loc, argsRange.getTypes(), op.getInits(), /*beforeBody*/ nullptr,
         /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
@@ -4187,7 +4187,7 @@ struct WhileOpAlignBeforeArgs : public OpRewritePattern<WhileOp> {
     for (auto &&[i, j] : llvm::enumerate(*mapping))
       newResultTypes[j] = loop.getResult(i).getType();
 
-    auto newLoop = rewriter.create<WhileOp>(
+    auto newLoop = WhileOp::create(rewriter,
         loop.getLoc(), newResultTypes, loop.getInits(),
         /*beforeBuilder=*/nullptr, /*afterBuilder=*/nullptr);
     auto newBefore = newLoop.getBeforeBody();
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index 57c27231f2144..e9943a3489d46 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -163,7 +163,7 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
   OpBuilder::InsertionGuard g(b);
   b.setInsertionPoint(op);
   scf::ExecuteRegionOp executeRegionOp =
-      b.create<scf::ExecuteRegionOp>(op->getLoc(), op->getResultTypes());
+      scf::ExecuteRegionOp::create(b, op->getLoc(), op->getResultTypes());
   {
     OpBuilder::InsertionGuard g(b);
     b.setInsertionPointToStart(&executeRegionOp.getRegion().emplaceBlock());
@@ -172,7 +172,7 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
     assert(clonedRegion.empty() && "expected empty region");
     b.inlineRegionBefore(op->getRegions().front(), clonedRegion,
                          clonedRegion.end());
-    b.create<scf::YieldOp>(op->getLoc(), clonedOp->getResults());
+    scf::YieldOp::create(b, op->getLoc(), clonedOp->getResults());
   }
   b.replaceOp(op, executeRegionOp.getResults());
   return executeRegionOp;
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index d36d91249ed36..0214ce3090656 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -42,7 +42,7 @@ static Value castBuffer(OpBuilder &b, Value buffer, Type type) {
   // iter_arg's layout map must be changed (see uses of `castBuffer`).
   assert(memref::CastOp::areCastCompatible(buffer.getType(), type) &&
          "scf.while op bufferization: cast incompatible");
-  return b.create<memref::CastOp>(buffer.getLoc(), type, buffer).getResult();
+  return memref::CastOp::create(b, buffer.getLoc(), type, buffer).getResult();
 }
 
 /// Helper function for loop bufferization. Return "true" if the given value
@@ -191,7 +191,7 @@ struct ExecuteRegionOpInterface
 
     // Create new op and move over region.
     auto newOp =
-        rewriter.create<scf::ExecuteRegionOp>(op->getLoc(), newResultTypes);
+        scf::ExecuteRegionOp::create(rewriter, op->getLoc(), newResultTypes);
     newOp.getRegion().takeBody(executeRegionOp.getRegion());
 
     // Bufferize every block.
@@ -205,7 +205,7 @@ struct ExecuteRegionOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(executeRegionOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
+        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
             executeRegionOp.getLoc(), it.value(),
             newOp->getResult(it.index())));
       } else {
@@ -261,7 +261,7 @@ struct IfOpInterface
     // Create new op.
     rewriter.setInsertionPoint(ifOp);
     auto newIfOp =
-        rewriter.create<scf::IfOp>(ifOp.getLoc(), newTypes, ifOp.getCondition(),
+        scf::IfOp::create(rewriter, ifOp.getLoc(), newTypes, ifOp.getCondition(),
                                    /*withElseRegion=*/true);
 
     // Move over then/else blocks.
@@ -374,7 +374,7 @@ struct IndexSwitchOpInterface
 
     // Create new op.
     rewriter.setInsertionPoint(switchOp);
-    auto newSwitchOp = rewriter.create<scf::IndexSwitchOp>(
+    auto newSwitchOp = scf::IndexSwitchOp::create(rewriter,
         switchOp.getLoc(), newTypes, switchOp.getArg(), switchOp.getCases(),
         switchOp.getCases().size());
 
@@ -769,7 +769,7 @@ struct ForOpInterface
     }
 
     // Construct a new scf.for op with memref instead of tensor values.
-    auto newForOp = rewriter.create<scf::ForOp>(
+    auto newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), castedInitArgs);
     newForOp->setAttrs(forOp->getAttrs());
@@ -1005,7 +1005,7 @@ struct WhileOpInterface
     // Construct a new scf.while op with memref instead of tensor values.
     ValueRange argsRangeBefore(castedInitArgs);
     TypeRange argsTypesBefore(argsRangeBefore);
-    auto newWhileOp = rewriter.create<scf::WhileOp>(
+    auto newWhileOp = scf::WhileOp::create(rewriter,
         whileOp.getLoc(), argsTypesAfter, castedInitArgs);
 
     // Add before/after regions to the new op.
@@ -1265,7 +1265,7 @@ struct ForallOpInterface
              forallOp.getBody()->getArguments().drop_front(rank), buffers)) {
       BlockArgument bbArg = std::get<0>(it);
       Value buffer = std::get<1>(it);
-      Value bufferAsTensor = rewriter.create<ToTensorOp>(
+      Value bufferAsTensor = ToTensorOp::create(rewriter,
           forallOp.getLoc(), bbArg.getType(), buffer);
       bbArg.replaceAllUsesWith(bufferAsTensor);
     }
@@ -1274,7 +1274,7 @@ struct ForallOpInterface
     // introduced terminator.
     rewriter.setInsertionPoint(forallOp);
     ForallOp newForallOp;
-    newForallOp = rewriter.create<ForallOp>(
+    newForallOp = ForallOp::create(rewriter,
         forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
         /*outputs=*/ValueRange(), forallOp.getMapping());
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
index 3e93dc80b18ec..bf4eba3319a4d 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
@@ -50,7 +50,7 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     SmallVector<Value> initArgs;
     initArgs.push_back(forOp.getLowerBound());
     llvm::append_range(initArgs, forOp.getInitArgs());
-    auto whileOp = rewriter.create<WhileOp>(forOp.getLoc(), lcvTypes, initArgs,
+    auto whileOp = WhileOp::create(rewriter, forOp.getLoc(), lcvTypes, initArgs,
                                             forOp->getAttrs());
 
     // 'before' region contains the loop condition and forwarding of iteration
@@ -58,10 +58,10 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     auto *beforeBlock = rewriter.createBlock(
         &whileOp.getBefore(), whileOp.getBefore().begin(), lcvTypes, lcvLocs);
     rewriter.setInsertionPointToStart(whileOp.getBeforeBody());
-    auto cmpOp = rewriter.create<arith::CmpIOp>(
+    auto cmpOp = arith::CmpIOp::create(rewriter,
         whileOp.getLoc(), arith::CmpIPredicate::slt,
         beforeBlock->getArgument(0), forOp.getUpperBound());
-    rewriter.create<scf::ConditionOp>(whileOp.getLoc(), cmpOp.getResult(),
+    scf::ConditionOp::create(rewriter, whileOp.getLoc(), cmpOp.getResult(),
                                       beforeBlock->getArguments());
 
     // Inline for-loop body into an executeRegion operation in the "after"
@@ -72,7 +72,7 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
 
     // Add induction variable incrementation
     rewriter.setInsertionPointToEnd(afterBlock);
-    auto ivIncOp = rewriter.create<arith::AddIOp>(
+    auto ivIncOp = arith::AddIOp::create(rewriter,
         whileOp.getLoc(), afterBlock->getArgument(0), forOp.getStep());
 
     // Rewrite uses of the for-loop block arguments to the new while-loop
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
index 44e6840b03a3d..b95604fa44cb9 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
@@ -40,7 +40,7 @@ LogicalResult mlir::scf::forallToParallelLoop(RewriterBase &rewriter,
   SmallVector<Value> steps = forallOp.getStep(rewriter);
 
   // Create empty scf.parallel op.
-  auto parallelOp = rewriter.create<scf::ParallelOp>(loc, lbs, ubs, steps);
+  auto parallelOp = scf::ParallelOp::create(rewriter, loc, lbs, ubs, steps);
   rewriter.eraseBlock(&parallelOp.getRegion().front());
   rewriter.inlineRegionBefore(forallOp.getRegion(), parallelOp.getRegion(),
                               parallelOp.getRegion().begin());
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index bcecef5e6e0a9..f1ede64d01e0f 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -279,24 +279,24 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
     if (dynamicLoop) {
       Type t = ub.getType();
       // pred = ub > lb + (i * step)
-      Value iv = rewriter.create<arith::AddIOp>(
+      Value iv = arith::AddIOp::create(rewriter,
           loc, lb,
-          rewriter.create<arith::MulIOp>(
+          arith::MulIOp::create(rewriter,
               loc, step,
-              rewriter.create<arith::ConstantOp>(
+              arith::ConstantOp::create(rewriter,
                   loc, rewriter.getIntegerAttr(t, i))));
-      predicates[i] = rewriter.create<arith::CmpIOp>(
+      predicates[i] = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::slt, iv, ub);
     }
 
     // special handling for induction variable as the increment is implicit.
     // iv = lb + i * step
     Type t = lb.getType();
-    Value iv = rewriter.create<arith::AddIOp>(
+    Value iv = arith::AddIOp::create(rewriter,
         loc, lb,
-        rewriter.create<arith::MulIOp>(
+        arith::MulIOp::create(rewriter,
             loc, step,
-            rewriter.create<arith::ConstantOp>(loc,
+            arith::ConstantOp::create(rewriter, loc,
                                                rewriter.getIntegerAttr(t, i))));
     setValueMapping(forOp.getInductionVar(), iv, i);
     for (Operation *op : opOrder) {
@@ -332,7 +332,7 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
             Value prevValue = valueMapping
                 [forOp.getRegionIterArgs()[operand.getOperandNumber()]]
                 [i - stages[op]];
-            source = rewriter.create<arith::SelectOp>(
+            source = arith::SelectOp::create(rewriter,
                 loc, predicates[predicateIdx], source, prevValue);
           }
           setValueMapping(forOp.getRegionIterArgs()[operand.getOperandNumber()],
@@ -444,14 +444,14 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop(
     Type t = ub.getType();
     Location loc = forOp.getLoc();
     // newUb = ub - maxStage * step
-    Value maxStageValue = rewriter.create<arith::ConstantOp>(
+    Value maxStageValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(t, maxStage));
     Value maxStageByStep =
-        rewriter.create<arith::MulIOp>(loc, step, maxStageValue);
-    newUb = rewriter.create<arith::SubIOp>(loc, ub, maxStageByStep);
+        arith::MulIOp::create(rewriter, loc, step, maxStageValue);
+    newUb = arith::SubIOp::create(rewriter, loc, ub, maxStageByStep);
   }
   auto newForOp =
-      rewriter.create<scf::ForOp>(forOp.getLoc(), forOp.getLowerBound(), newUb,
+      scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(), newUb,
                                   forOp.getStep(), newLoopArg);
   // When there are no iter args, the loop body terminator will be created.
   // Since we always create it below, remove the terminator if it was created.
@@ -483,14 +483,14 @@ LogicalResult LoopPipelinerInternal::createKernel(
     Type t = ub.getType();
     for (unsigned i = 0; i < maxStage; i++) {
       // c = ub - (maxStage - i) * step
-      Value c = rewriter.create<arith::SubIOp>(
+      Value c = arith::SubIOp::create(rewriter,
           loc, ub,
-          rewriter.create<arith::MulIOp>(
+          arith::MulIOp::create(rewriter,
               loc, step,
-              rewriter.create<arith::ConstantOp>(
+              arith::ConstantOp::create(rewriter,
                   loc, rewriter.getIntegerAttr(t, int64_t(maxStage - i)))));
 
-      Value pred = rewriter.create<arith::CmpIOp>(
+      Value pred = arith::CmpIOp::create(rewriter,
           newForOp.getLoc(), arith::CmpIPredicate::slt,
           newForOp.getInductionVar(), c);
       predicates[i] = pred;
@@ -515,12 +515,12 @@ LogicalResult LoopPipelinerInternal::createKernel(
 
         // offset = (maxStage - stages[op]) * step
         Type t = step.getType();
-        Value offset = rewriter.create<arith::MulIOp>(
+        Value offset = arith::MulIOp::create(rewriter,
             forOp.getLoc(), step,
-            rewriter.create<arith::ConstantOp>(
+            arith::ConstantOp::create(rewriter,
                 forOp.getLoc(),
                 rewriter.getIntegerAttr(t, maxStage - stages[op])));
-        Value iv = rewriter.create<arith::AddIOp>(
+        Value iv = arith::AddIOp::create(rewriter,
             forOp.getLoc(), newForOp.getInductionVar(), offset);
         nestedNewOp->setOperand(operand->getOperandNumber(), iv);
         rewriter.setInsertionPointAfter(newOp);
@@ -594,7 +594,7 @@ LogicalResult LoopPipelinerInternal::createKernel(
         auto defStage = stages.find(def);
         if (defStage != stages.end() && defStage->second < maxStage) {
           Value pred = predicates[defStage->second];
-          source = rewriter.create<arith::SelectOp>(
+          source = arith::SelectOp::create(rewriter,
               pred.getLoc(), pred, source,
               newForOp.getBody()
                   ->getArguments()[yieldOperand.getOperandNumber() + 1]);
@@ -638,7 +638,7 @@ LogicalResult LoopPipelinerInternal::createKernel(
                       maxStage - defStage->second + 1);
     }
   }
-  rewriter.create<scf::YieldOp>(forOp.getLoc(), yieldOperands);
+  scf::YieldOp::create(rewriter, forOp.getLoc(), yieldOperands);
   return success();
 }
 
@@ -652,7 +652,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // removed by dead code if not used.
 
   auto createConst = [&](int v) {
-    return rewriter.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(rewriter, loc,
                                               rewriter.getIntegerAttr(t, v));
   };
 
@@ -661,41 +661,41 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // - total_iterations = (range_diff + step + (step < 0 ? 1 : -1)) / step
   Value zero = createConst(0);
   Value one = createConst(1);
-  Value stepLessZero = rewriter.create<arith::CmpIOp>(
+  Value stepLessZero = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::slt, step, zero);
   Value stepDecr =
-      rewriter.create<arith::SelectOp>(loc, stepLessZero, one, createConst(-1));
+      arith::SelectOp::create(rewriter, loc, stepLessZero, one, createConst(-1));
 
-  Value rangeDiff = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  Value rangeIncrStep = rewriter.create<arith::AddIOp>(loc, rangeDiff, step);
+  Value rangeDiff = arith::SubIOp::create(rewriter, loc, ub, lb);
+  Value rangeIncrStep = arith::AddIOp::create(rewriter, loc, rangeDiff, step);
   Value rangeDecr =
-      rewriter.create<arith::AddIOp>(loc, rangeIncrStep, stepDecr);
-  Value totalIterations = rewriter.create<arith::DivSIOp>(loc, rangeDecr, step);
+      arith::AddIOp::create(rewriter, loc, rangeIncrStep, stepDecr);
+  Value totalIterations = arith::DivSIOp::create(rewriter, loc, rangeDecr, step);
 
   // If total_iters < max_stage, start the epilogue at zero to match the
   // ramp-up in the prologue.
   // start_iter = max(0, total_iters - max_stage)
-  Value iterI = rewriter.create<arith::SubIOp>(loc, totalIterations,
+  Value iterI = arith::SubIOp::create(rewriter, loc, totalIterations,
                                                createConst(maxStage));
-  iterI = rewriter.create<arith::MaxSIOp>(loc, zero, iterI);
+  iterI = arith::MaxSIOp::create(rewriter, loc, zero, iterI);
 
   // Capture predicates for dynamic loops.
   SmallVector<Value> predicates(maxStage + 1);
 
   for (int64_t i = 1; i <= maxStage; i++) {
     // newLastIter = lb + step * iterI
-    Value newlastIter = rewriter.create<arith::AddIOp>(
-        loc, lb, rewriter.create<arith::MulIOp>(loc, step, iterI));
+    Value newlastIter = arith::AddIOp::create(rewriter,
+        loc, lb, arith::MulIOp::create(rewriter, loc, step, iterI));
 
     setValueMapping(forOp.getInductionVar(), newlastIter, i);
 
     // increment to next iterI
-    iterI = rewriter.create<arith::AddIOp>(loc, iterI, one);
+    iterI = arith::AddIOp::create(rewriter, loc, iterI, one);
 
     if (dynamicLoop) {
       // Disable stages when `i` is greater than total_iters.
       // pred = total_iters >= i
-      predicates[i] = rewriter.create<arith::CmpIOp>(
+      predicates[i] = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, totalIterations, createConst(i));
     }
   }
@@ -758,7 +758,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
           unsigned nextVersion = currentVersion + 1;
           Value pred = predicates[currentVersion];
           Value prevValue = valueMapping[mapVal][currentVersion];
-          auto selOp = rewriter.create<arith::SelectOp>(loc, pred, pair.value(),
+          auto selOp = arith::SelectOp::create(rewriter, loc, pred, pair.value(),
                                                         prevValue);
           returnValues[ri] = selOp;
           if (nextVersion <= maxStage)
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index b71ec985fa6a1..b24e276272c2c 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -64,13 +64,13 @@ static void specializeParallelLoopForUnrolling(ParallelOp op) {
   Value cond;
   for (auto bound : llvm::zip(op.getUpperBound(), constantIndices)) {
     Value constant =
-        b.create<arith::ConstantIndexOp>(op.getLoc(), std::get<1>(bound));
-    Value cmp = b.create<arith::CmpIOp>(op.getLoc(), arith::CmpIPredicate::eq,
+        arith::ConstantIndexOp::create(b, op.getLoc(), std::get<1>(bound));
+    Value cmp = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
                                         std::get<0>(bound), constant);
-    cond = cond ? b.create<arith::AndIOp>(op.getLoc(), cond, cmp) : cmp;
+    cond = cond ? arith::AndIOp::create(b, op.getLoc(), cond, cmp) : cmp;
     map.map(std::get<0>(bound), constant);
   }
-  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
   ifOp.getElseBodyBuilder().clone(*op.getOperation());
   op.erase();
@@ -95,11 +95,11 @@ static void specializeForLoopForUnrolling(ForOp op) {
 
   OpBuilder b(op);
   IRMapping map;
-  Value constant = b.create<arith::ConstantIndexOp>(op.getLoc(), minConstant);
-  Value cond = b.create<arith::CmpIOp>(op.getLoc(), arith::CmpIPredicate::eq,
+  Value constant = arith::ConstantIndexOp::create(b, op.getLoc(), minConstant);
+  Value cond = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
                                        bound, constant);
   map.map(bound, constant);
-  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
   ifOp.getElseBodyBuilder().clone(*op.getOperation());
   op.erase();
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index ad1267381c4f2..013c3b69620e2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -190,7 +190,7 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
 
   IRRewriter b(builder);
   b.setInsertionPoint(secondPloop);
-  auto newSecondPloop = b.create<ParallelOp>(
+  auto newSecondPloop = ParallelOp::create(b,
       secondPloop.getLoc(), secondPloop.getLowerBound(),
       secondPloop.getUpperBound(), secondPloop.getStep(), newInitVars);
 
@@ -212,7 +212,7 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
     SmallVector<Value> newReduceArgs(reduceArgs1.begin(), reduceArgs1.end());
     newReduceArgs.append(reduceArgs2.begin(), reduceArgs2.end());
 
-    auto newReduceOp = b.create<scf::ReduceOp>(term2.getLoc(), newReduceArgs);
+    auto newReduceOp = scf::ReduceOp::create(b, term2.getLoc(), newReduceArgs);
 
     for (auto &&[i, reg] : llvm::enumerate(llvm::concat<Region>(
              term1.getReductions(), term2.getReductions()))) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
index 66f7bc27f82ff..2ca3b1e366bdf 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
@@ -58,27 +58,27 @@ std::pair<ParallelOp, ParallelOp>
 mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
                             bool noMinMaxBounds) {
   OpBuilder b(op);
-  auto zero = b.create<arith::ConstantIndexOp>(op.getLoc(), 0);
+  auto zero = arith::ConstantIndexOp::create(b, op.getLoc(), 0);
   SmallVector<Value, 2> tileSizeConstants;
   tileSizeConstants.reserve(op.getUpperBound().size());
   for (size_t i = 0, end = op.getUpperBound().size(); i != end; ++i) {
     if (i < tileSizes.size())
       tileSizeConstants.push_back(
-          b.create<arith::ConstantIndexOp>(op.getLoc(), tileSizes[i]));
+          arith::ConstantIndexOp::create(b, op.getLoc(), tileSizes[i]));
     else
       // Just pick 1 for the remaining dimensions.
       tileSizeConstants.push_back(
-          b.create<arith::ConstantIndexOp>(op.getLoc(), 1));
+          arith::ConstantIndexOp::create(b, op.getLoc(), 1));
   }
 
   // Create the outer loop with adjusted steps.
   SmallVector<Value, 2> newSteps;
   newSteps.reserve(op.getStep().size());
   for (auto step : llvm::zip(op.getStep(), tileSizeConstants)) {
-    newSteps.push_back(b.create<arith::MulIOp>(op.getLoc(), std::get<0>(step),
+    newSteps.push_back(arith::MulIOp::create(b, op.getLoc(), std::get<0>(step),
                                                std::get<1>(step)));
   }
-  auto outerLoop = b.create<ParallelOp>(op.getLoc(), op.getLowerBound(),
+  auto outerLoop = ParallelOp::create(b, op.getLoc(), op.getLowerBound(),
                                         op.getUpperBound(), newSteps);
   b.setInsertionPointToStart(outerLoop.getBody());
 
@@ -130,10 +130,10 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // Otherwise, we dynamically compute the bound for
     // each iteration of the outer loop.
     newBounds.push_back(
-        b.create<affine::AffineMinOp>(op.getLoc(), b.getIndexType(), minMap,
+        affine::AffineMinOp::create(b, op.getLoc(), b.getIndexType(), minMap,
                                       ValueRange{newStep, upperBound, iv}));
   }
-  auto innerLoop = b.create<ParallelOp>(
+  auto innerLoop = ParallelOp::create(b,
       op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
       op.getStep());
 
@@ -141,20 +141,20 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     b.setInsertionPointToStart(innerLoop.getBody());
     // Insert in-bound check
     Value inbound =
-        b.create<arith::ConstantIntOp>(op.getLoc(), b.getIntegerType(1), 1);
+        arith::ConstantIntOp::create(b, op.getLoc(), b.getIntegerType(1), 1);
     for (auto [outerUpperBound, outerIV, innerIV, innerStep] :
          llvm::zip(outerLoop.getUpperBound(), outerLoop.getInductionVars(),
                    innerLoop.getInductionVars(), innerLoop.getStep())) {
       // %in_bound = %in_bound &&
       //             (%inner_iv * %inner_step + %outer_iv < %outer_upper_bound)
-      Value index = b.create<arith::AddIOp>(
-          op.getLoc(), b.create<arith::MulIOp>(op.getLoc(), innerIV, innerStep),
+      Value index = arith::AddIOp::create(b,
+          op.getLoc(), arith::MulIOp::create(b, op.getLoc(), innerIV, innerStep),
           outerIV);
-      Value dimInbound = b.create<arith::CmpIOp>(
+      Value dimInbound = arith::CmpIOp::create(b,
           op.getLoc(), arith::CmpIPredicate::ult, index, outerUpperBound);
-      inbound = b.create<arith::AndIOp>(op.getLoc(), inbound, dimInbound);
+      inbound = arith::AndIOp::create(b, op.getLoc(), inbound, dimInbound);
     }
-    auto ifInbound = b.create<IfOp>(op.getLoc(),
+    auto ifInbound = IfOp::create(b, op.getLoc(),
                                     /*resultTypes*/ ArrayRef<Type>{}, inbound,
                                     /*hasElseRegion*/ false);
     ifInbound.getThenRegion().takeBody(op.getRegion());
@@ -162,12 +162,12 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // Replace the scf.reduce terminator with an scf.yield terminator.
     Operation *reduceOp = thenBlock.getTerminator();
     b.setInsertionPointToEnd(&thenBlock);
-    b.create<scf::YieldOp>(reduceOp->getLoc());
+    scf::YieldOp::create(b, reduceOp->getLoc());
     reduceOp->erase();
     b.setInsertionPointToStart(innerLoop.getBody());
     for (const auto &ivs : llvm::enumerate(llvm::zip(
              innerLoop.getInductionVars(), outerLoop.getInductionVars()))) {
-      auto newIndex = b.create<arith::AddIOp>(
+      auto newIndex = arith::AddIOp::create(b,
           op.getLoc(), std::get<0>(ivs.value()), std::get<1>(ivs.value()));
       thenBlock.getArgument(ivs.index())
           .replaceAllUsesExcept(newIndex, newIndex);
@@ -179,7 +179,7 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     for (auto ivs : llvm::zip(innerLoop.getInductionVars(),
                               outerLoop.getInductionVars())) {
       Value innerIndex = std::get<0>(ivs);
-      auto newIndex = b.create<arith::AddIOp>(op.getLoc(), std::get<0>(ivs),
+      auto newIndex = arith::AddIOp::create(b, op.getLoc(), std::get<0>(ivs),
                                               std::get<1>(ivs));
       innerIndex.replaceAllUsesExcept(newIndex, newIndex);
     }
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 09326242eec2a..41895e46d953d 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -112,7 +112,7 @@ class ConvertForOpTypes
 
     // We can not do clone as the number of result types after conversion
     // might be different.
-    ForOp newOp = rewriter.create<ForOp>(
+    ForOp newOp = ForOp::create(rewriter,
         op.getLoc(), llvm::getSingleElement(adaptor.getLowerBound()),
         llvm::getSingleElement(adaptor.getUpperBound()),
         llvm::getSingleElement(adaptor.getStep()),
@@ -142,7 +142,7 @@ class ConvertIfOpTypes
                                       ConversionPatternRewriter &rewriter,
                                       TypeRange dstTypes) const {
 
-    IfOp newOp = rewriter.create<IfOp>(
+    IfOp newOp = IfOp::create(rewriter,
         op.getLoc(), dstTypes, llvm::getSingleElement(adaptor.getCondition()),
         true);
     newOp->setAttrs(op->getAttrs());
@@ -171,7 +171,7 @@ class ConvertWhileOpTypes
   std::optional<WhileOp> convertSourceOp(WhileOp op, OneToNOpAdaptor adaptor,
                                          ConversionPatternRewriter &rewriter,
                                          TypeRange dstTypes) const {
-    auto newOp = rewriter.create<WhileOp>(op.getLoc(), dstTypes,
+    auto newOp = WhileOp::create(rewriter, op.getLoc(), dstTypes,
                                           flattenValues(adaptor.getOperands()));
 
     for (auto i : {0u, 1u}) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index c7d634283fd4e..3171b96d46b8a 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -450,7 +450,7 @@ static LogicalResult generateLoopNestUsingForOp(
   SmallVector<Value> ivs;
   for (auto [lb, ub, step] : llvm::zip_equal(lbVals, ubVals, stepVals)) {
     auto loop =
-        rewriter.create<scf::ForOp>(loc, lb, ub, step, destinationTensors,
+        scf::ForOp::create(rewriter, loc, lb, ub, step, destinationTensors,
                                     [](OpBuilder &bodyBuilder, Location bodyLoc,
                                        Value iv, ValueRange /*iterArgs*/) {});
     loops.push_back(loop);
@@ -479,12 +479,12 @@ static LogicalResult generateLoopNestUsingForOp(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    auto insertSlice = rewriter.create<tensor::InsertSliceOp>(
+    auto insertSlice = tensor::InsertSliceOp::create(rewriter,
         loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
     yieldedValues.push_back(insertSlice);
   }
-  rewriter.create<scf::YieldOp>(loc, yieldedValues);
+  scf::YieldOp::create(rewriter, loc, yieldedValues);
 
   // Add the scf.yield operations for all the outer loops.
   for (auto [outerLoop, innerLoop] :
@@ -492,7 +492,7 @@ static LogicalResult generateLoopNestUsingForOp(
                        MutableArrayRef(loops).drop_front())) {
     rewriter.setInsertionPointToEnd(
         cast<scf::ForOp>(outerLoop.getOperation()).getBody());
-    rewriter.create<scf::YieldOp>(outerLoop.getLoc(), innerLoop->getResults());
+    scf::YieldOp::create(rewriter, outerLoop.getLoc(), innerLoop->getResults());
   }
   return success();
 }
@@ -533,13 +533,13 @@ static LogicalResult generateLoopNestUsingForallOp(
         continue;
       nonZeroNumThreads.push_back(nt);
     }
-    forallOp = rewriter.create<scf::ForallOp>(loc, nonZeroNumThreads,
+    forallOp = scf::ForallOp::create(rewriter, loc, nonZeroNumThreads,
                                               destinationTensors, mappingAttr);
   } else {
     SmallVector<OpFoldResult> lbs, ubs, steps;
     std::tie(lbs, ubs, steps) =
         getLoopBounds(rewriter, loc, loopRanges, tileSizes);
-    forallOp = rewriter.create<scf::ForallOp>(loc, lbs, ubs, steps,
+    forallOp = scf::ForallOp::create(rewriter, loc, lbs, ubs, steps,
                                               destinationTensors, mappingAttr);
   }
   loops.push_back(forallOp);
@@ -561,7 +561,7 @@ static LogicalResult generateLoopNestUsingForallOp(
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
 
-    rewriter.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(rewriter,
         loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
   }
@@ -798,7 +798,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
 
   auto inits = llvm::to_vector(loopOp.getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = rewriter.create<scf::ForOp>(
+  auto newLoop = scf::ForOp::create(rewriter,
       loc, loopOp.getLowerBound(), loopOp.getUpperBound(), loopOp.getStep(),
       inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
@@ -829,7 +829,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    Value insert = rewriter.create<tensor::InsertSliceOp>(
+    Value insert = tensor::InsertSliceOp::create(rewriter,
         yieldOp->getLoc(), tiledValue, regionIterArg, resultOffset, resultSize,
         resultStride);
     newYieldValues.push_back(insert);
@@ -851,7 +851,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
   rewriter.setInsertionPoint(loopOp);
   auto inits = llvm::to_vector(loopOp.getOutputs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = rewriter.create<scf::ForallOp>(
+  auto newLoop = scf::ForallOp::create(rewriter,
       loc, loopOp.getMixedLowerBound(), loopOp.getMixedUpperBound(),
       loopOp.getMixedStep(), inits, loopOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
@@ -884,7 +884,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
            tiledValues, regionIterArgs, resultOffsets, resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    rewriter.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(rewriter,
         terminator.getLoc(), tiledValue, iterArg, resultOffset, resultSize,
         resultStride);
   }
@@ -935,7 +935,7 @@ static LogicalResult addInitOperandsToLoopNest(
     // Create a new loop with the new init values for this loop.
     SmallVector<Value> newInits = llvm::to_vector(forLoop.getInitArgs());
     newInits.append(newInitValues.begin(), newInitValues.end());
-    auto newLoop = rewriter.create<scf::ForOp>(
+    auto newLoop = scf::ForOp::create(rewriter,
         forLoop.getLoc(), forLoop.getLowerBound(), forLoop.getUpperBound(),
         forLoop.getStep(), newInits,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {});
@@ -1419,7 +1419,7 @@ FailureOr<SmallVector<Operation *>> mlir::scf::yieldReplacementForFusedProducer(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = rewriter.create<tensor::ExtractSliceOp>(
+        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
             loc, newRegionArg, offsetList[index], sizesList[index],
             SmallVector<OpFoldResult>(offsetList[index].size(),
                                       rewriter.getIndexAttr(1)));
@@ -2092,7 +2092,7 @@ cloneAsInsertSlice<tensor::InsertSliceOp>(RewriterBase &rewriter,
 template <>
 tensor::InsertSliceOp cloneAsInsertSlice<tensor::ParallelInsertSliceOp>(
     RewriterBase &rewriter, tensor::ParallelInsertSliceOp insertSliceOp) {
-  return rewriter.create<tensor::InsertSliceOp>(
+  return tensor::InsertSliceOp::create(rewriter,
       insertSliceOp->getLoc(), insertSliceOp.getSource(),
       insertSliceOp.getDest(), insertSliceOp.getMixedOffsets(),
       insertSliceOp.getMixedSizes(), insertSliceOp.getMixedStrides());
@@ -2314,7 +2314,7 @@ mlir::scf::tileAndFuseConsumerOfSlices(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = rewriter.create<tensor::ExtractSliceOp>(
+        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
             loc, newRegionArg, resultOffsets[index], resultSizes[index],
             SmallVector<OpFoldResult>(resultOffsets[index].size(),
                                       rewriter.getIndexAttr(1)));
@@ -2391,7 +2391,7 @@ mlir::scf::lowerToLoopsUsingSCFForOp(RewriterBase &rewriter,
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.size);
     Value strideVal =
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.stride);
-    auto loop = rewriter.create<scf::ForOp>(op.getLoc(), offsetVal, sizeVal,
+    auto loop = scf::ForOp::create(rewriter, op.getLoc(), offsetVal, sizeVal,
                                             strideVal, ValueRange{});
     loops.push_back(loop);
     ivs.push_back(loop.getInductionVar());
diff --git a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
index 29d6d2574a2be..14ded96516eab 100644
--- a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
@@ -191,7 +191,7 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
   // dummy builder instead.
   auto emptyBuilder = [](OpBuilder &, Location, Value, ValueRange) {};
   auto newLoop =
-      rewriter.create<scf::ForOp>(loc, lb, ub, step, newArgs, emptyBuilder);
+      scf::ForOp::create(rewriter, loc, lb, ub, step, newArgs, emptyBuilder);
 
   Block *newBody = newLoop.getBody();
 
@@ -238,18 +238,18 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
   rewriter.setInsertionPointAfter(newLoop);
   Value one;
   if (isa<IndexType>(step.getType())) {
-    one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    one = arith::ConstantIndexOp::create(rewriter, loc, 1);
   } else {
-    one = rewriter.create<arith::ConstantIntOp>(loc, step.getType(), 1);
+    one = arith::ConstantIntOp::create(rewriter, loc, step.getType(), 1);
   }
 
-  Value stepDec = rewriter.create<arith::SubIOp>(loc, step, one);
-  Value len = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  len = rewriter.create<arith::AddIOp>(loc, len, stepDec);
-  len = rewriter.create<arith::DivSIOp>(loc, len, step);
-  len = rewriter.create<arith::SubIOp>(loc, len, one);
-  Value res = rewriter.create<arith::MulIOp>(loc, len, step);
-  res = rewriter.create<arith::AddIOp>(loc, lb, res);
+  Value stepDec = arith::SubIOp::create(rewriter, loc, step, one);
+  Value len = arith::SubIOp::create(rewriter, loc, ub, lb);
+  len = arith::AddIOp::create(rewriter, loc, len, stepDec);
+  len = arith::DivSIOp::create(rewriter, loc, len, step);
+  len = arith::SubIOp::create(rewriter, loc, len, one);
+  Value res = arith::MulIOp::create(rewriter, loc, len, step);
+  res = arith::AddIOp::create(rewriter, loc, lb, res);
 
   // Reconstruct `scf.while` results, inserting final induction var value
   // into proper place.
diff --git a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
index f829208ce8798..707ed836cf63e 100644
--- a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
@@ -96,7 +96,7 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       condOp.getArgs(), [&](Value arg) { return mapper.lookupOrDefault(arg); });
 
   // Create rotated while loop.
-  auto newLoopOp = rewriter.create<scf::WhileOp>(
+  auto newLoopOp = scf::WhileOp::create(rewriter,
       whileOp.getLoc(), whileOp.getResultTypes(), clonedCondArgs,
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Rotate and move the loop body into before block.
@@ -109,21 +109,21 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       },
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Pass through values.
-        builder.create<scf::YieldOp>(loc, args);
+        scf::YieldOp::create(builder, loc, args);
       });
 
   // Create zero-trip-check and move the while loop in.
-  auto ifOp = rewriter.create<scf::IfOp>(
+  auto ifOp = scf::IfOp::create(rewriter,
       whileOp.getLoc(), clonedCondition,
       [&](OpBuilder &builder, Location loc) {
         // Then runs the while loop.
         rewriter.moveOpBefore(newLoopOp, builder.getInsertionBlock(),
                               builder.getInsertionPoint());
-        builder.create<scf::YieldOp>(loc, newLoopOp.getResults());
+        scf::YieldOp::create(builder, loc, newLoopOp.getResults());
       },
       [&](OpBuilder &builder, Location loc) {
         // Else returns the results from precondition.
-        builder.create<scf::YieldOp>(loc, clonedCondArgs);
+        scf::YieldOp::create(builder, loc, clonedCondArgs);
       });
 
   rewriter.replaceOp(whileOp, ifOp);
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index f4047be68ccf2..6a387ddf8908a 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -153,7 +153,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
       FunctionType::get(rewriter.getContext(), outlinedFuncArgTypes,
                         originalTerminator->getOperandTypes());
   auto outlinedFunc =
-      rewriter.create<func::FuncOp>(loc, funcName, outlinedFuncType);
+      func::FuncOp::create(rewriter, loc, funcName, outlinedFuncType);
   Block *outlinedFuncBody = outlinedFunc.addEntryBlock();
 
   // Merge blocks while replacing the original block operands.
@@ -168,7 +168,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
         outlinedFuncBlockArgs.take_front(numOriginalBlockArguments));
     // Explicitly set up a new ReturnOp terminator.
     rewriter.setInsertionPointToEnd(outlinedFuncBody);
-    rewriter.create<func::ReturnOp>(loc, originalTerminator->getResultTypes(),
+    func::ReturnOp::create(rewriter, loc, originalTerminator->getResultTypes(),
                                     originalTerminator->getOperands());
   }
 
@@ -185,7 +185,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
     SmallVector<Value> callValues;
     llvm::append_range(callValues, newBlock->getArguments());
     llvm::append_range(callValues, outlinedValues);
-    auto call = rewriter.create<func::CallOp>(loc, outlinedFunc, callValues);
+    auto call = func::CallOp::create(rewriter, loc, outlinedFunc, callValues);
     if (callOp)
       *callOp = call;
 
@@ -274,12 +274,12 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
 
-  Value divisorMinusOneCst = builder.create<arith::ConstantOp>(
+  Value divisorMinusOneCst = arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(dividend.getType(), divisor - 1));
-  Value divisorCst = builder.create<arith::ConstantOp>(
+  Value divisorCst = arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(dividend.getType(), divisor));
-  Value sum = builder.create<arith::AddIOp>(loc, dividend, divisorMinusOneCst);
-  return builder.create<arith::DivUIOp>(loc, sum, divisorCst);
+  Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOneCst);
+  return arith::DivUIOp::create(builder, loc, sum, divisorCst);
 }
 
 // Build the IR that performs ceil division of a positive value by another
@@ -290,11 +290,11 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
                              Value divisor) {
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
-  Value cstOne = builder.create<arith::ConstantOp>(
+  Value cstOne = arith::ConstantOp::create(builder,
       loc, builder.getOneAttr(dividend.getType()));
-  Value divisorMinusOne = builder.create<arith::SubIOp>(loc, divisor, cstOne);
-  Value sum = builder.create<arith::AddIOp>(loc, dividend, divisorMinusOne);
-  return builder.create<arith::DivUIOp>(loc, sum, divisor);
+  Value divisorMinusOne = arith::SubIOp::create(builder, loc, divisor, cstOne);
+  Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOne);
+  return arith::DivUIOp::create(builder, loc, sum, divisor);
 }
 
 /// Returns the trip count of `forOp` if its' low bound, high bound and step are
@@ -404,7 +404,7 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'upperBoundUnrolled' and set epilogue loop flag.
     generateEpilogueLoop = upperBoundUnrolledCst < ubCst;
     if (generateEpilogueLoop)
-      upperBoundUnrolled = boundsBuilder.create<arith::ConstantOp>(
+      upperBoundUnrolled = arith::ConstantOp::create(boundsBuilder,
           loc, boundsBuilder.getIntegerAttr(forOp.getUpperBound().getType(),
                                             upperBoundUnrolledCst));
     else
@@ -413,7 +413,7 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'stepUnrolled'.
     stepUnrolled = stepCst == stepUnrolledCst
                        ? step
-                       : boundsBuilder.create<arith::ConstantOp>(
+                       : arith::ConstantOp::create(boundsBuilder,
                              loc, boundsBuilder.getIntegerAttr(
                                       step.getType(), stepUnrolledCst));
   } else {
@@ -423,22 +423,22 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     auto lowerBound = forOp.getLowerBound();
     auto upperBound = forOp.getUpperBound();
     Value diff =
-        boundsBuilder.create<arith::SubIOp>(loc, upperBound, lowerBound);
+        arith::SubIOp::create(boundsBuilder, loc, upperBound, lowerBound);
     Value tripCount = ceilDivPositive(boundsBuilder, loc, diff, step);
-    Value unrollFactorCst = boundsBuilder.create<arith::ConstantOp>(
+    Value unrollFactorCst = arith::ConstantOp::create(boundsBuilder,
         loc, boundsBuilder.getIntegerAttr(tripCount.getType(), unrollFactor));
     Value tripCountRem =
-        boundsBuilder.create<arith::RemSIOp>(loc, tripCount, unrollFactorCst);
+        arith::RemSIOp::create(boundsBuilder, loc, tripCount, unrollFactorCst);
     // Compute tripCountEvenMultiple = tripCount - (tripCount % unrollFactor)
     Value tripCountEvenMultiple =
-        boundsBuilder.create<arith::SubIOp>(loc, tripCount, tripCountRem);
+        arith::SubIOp::create(boundsBuilder, loc, tripCount, tripCountRem);
     // Compute upperBoundUnrolled = lowerBound + tripCountEvenMultiple * step
-    upperBoundUnrolled = boundsBuilder.create<arith::AddIOp>(
+    upperBoundUnrolled = arith::AddIOp::create(boundsBuilder,
         loc, lowerBound,
-        boundsBuilder.create<arith::MulIOp>(loc, tripCountEvenMultiple, step));
+        arith::MulIOp::create(boundsBuilder, loc, tripCountEvenMultiple, step));
     // Scale 'step' by 'unrollFactor'.
     stepUnrolled =
-        boundsBuilder.create<arith::MulIOp>(loc, step, unrollFactorCst);
+        arith::MulIOp::create(boundsBuilder, loc, step, unrollFactorCst);
   }
 
   UnrolledLoopInfo resultLoops;
@@ -474,11 +474,11 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
       forOp.getBody(), forOp.getInductionVar(), unrollFactor,
       [&](unsigned i, Value iv, OpBuilder b) {
         // iv' = iv + step * i;
-        auto stride = b.create<arith::MulIOp>(
+        auto stride = arith::MulIOp::create(b,
             loc, step,
-            b.create<arith::ConstantOp>(loc,
+            arith::ConstantOp::create(b, loc,
                                         b.getIntegerAttr(iv.getType(), i)));
-        return b.create<arith::AddIOp>(loc, iv, stride);
+        return arith::AddIOp::create(b, loc, iv, stride);
       },
       annotateFn, iterArgs, yieldedValues);
   // Promote the loop body up if this has turned into a single iteration loop.
@@ -781,13 +781,13 @@ void mlir::denormalizeInductionVariable(RewriterBase &rewriter, Location loc,
   if (!isStepOne) {
     Value origStepValue =
         getValueOrCreateConstantIntOp(rewriter, loc, origStep);
-    scaled = rewriter.create<arith::MulIOp>(loc, normalizedIv, origStepValue);
+    scaled = arith::MulIOp::create(rewriter, loc, normalizedIv, origStepValue);
     preserve.insert(scaled.getDefiningOp());
   }
   denormalizedIv = scaled;
   if (!isZeroBased) {
     Value origLbValue = getValueOrCreateConstantIntOp(rewriter, loc, origLb);
-    denormalizedIv = rewriter.create<arith::AddIOp>(loc, scaled, origLbValue);
+    denormalizedIv = arith::AddIOp::create(rewriter, loc, scaled, origLbValue);
     preserve.insert(denormalizedIv.getDefiningOp());
   }
 
@@ -824,7 +824,7 @@ static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc,
       continue;
     if (productOf)
       productOf =
-          rewriter.create<arith::MulIOp>(loc, productOf.value(), v).getResult();
+          arith::MulIOp::create(rewriter, loc, productOf.value(), v).getResult();
     else
       productOf = v;
   }
@@ -851,7 +851,7 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
 
   if (linearizedIv.getType().isIndex()) {
     Operation *delinearizedOp =
-        rewriter.create<affine::AffineDelinearizeIndexOp>(loc, linearizedIv,
+        affine::AffineDelinearizeIndexOp::create(rewriter, loc, linearizedIv,
                                                           ubs);
     auto resultVals = llvm::map_to_vector(
         delinearizedOp->getResults(), [](OpResult r) -> Value { return r; });
@@ -874,7 +874,7 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
     if (!isUbOne.test(index)) {
       break;
     }
-    delinearizedIvs[index] = rewriter.create<arith::ConstantOp>(
+    delinearizedIvs[index] = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(ub.getType()));
     numLeadingOneUbs++;
   }
@@ -883,16 +883,16 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
   for (unsigned i = numLeadingOneUbs, e = ubs.size(); i < e; ++i) {
     unsigned idx = ubs.size() - (i - numLeadingOneUbs) - 1;
     if (i != numLeadingOneUbs && !isUbOne.test(idx + 1)) {
-      previous = rewriter.create<arith::DivSIOp>(loc, previous, ubs[idx + 1]);
+      previous = arith::DivSIOp::create(rewriter, loc, previous, ubs[idx + 1]);
       preservedUsers.insert(previous.getDefiningOp());
     }
     Value iv = previous;
     if (i != e - 1) {
       if (!isUbOne.test(idx)) {
-        iv = rewriter.create<arith::RemSIOp>(loc, previous, ubs[idx]);
+        iv = arith::RemSIOp::create(rewriter, loc, previous, ubs[idx]);
         preservedUsers.insert(iv.getDefiningOp());
       } else {
-        iv = rewriter.create<arith::ConstantOp>(
+        iv = arith::ConstantOp::create(rewriter,
             loc, rewriter.getZeroAttr(ubs[idx].getType()));
       }
     }
@@ -1093,12 +1093,12 @@ void mlir::collapseParallelLoops(
 
   // Combine iteration spaces.
   SmallVector<Value, 3> lowerBounds, upperBounds, steps;
-  auto cst0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto cst1 = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto cst0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto cst1 = arith::ConstantIndexOp::create(rewriter, loc, 1);
   for (auto &sortedDimension : sortedDimensions) {
-    Value newUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value newUpperBound = arith::ConstantIndexOp::create(rewriter, loc, 1);
     for (auto idx : sortedDimension) {
-      newUpperBound = rewriter.create<arith::MulIOp>(
+      newUpperBound = arith::MulIOp::create(rewriter,
           loc, newUpperBound, normalizedUpperBounds[idx]);
     }
     lowerBounds.push_back(cst0);
@@ -1112,7 +1112,7 @@ void mlir::collapseParallelLoops(
   // value. The remainders then determine based on that range, which iteration
   // of the original induction value this represents. This is a normalized value
   // that is un-normalized already by the previous logic.
-  auto newPloop = rewriter.create<scf::ParallelOp>(
+  auto newPloop = scf::ParallelOp::create(rewriter,
       loc, lowerBounds, upperBounds, steps,
       [&](OpBuilder &insideBuilder, Location, ValueRange ploopIVs) {
         for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) {
@@ -1123,14 +1123,14 @@ void mlir::collapseParallelLoops(
             unsigned idx = combinedDimensions[i][j];
 
             // Determine the current induction value's current loop iteration
-            Value iv = insideBuilder.create<arith::RemSIOp>(
+            Value iv = arith::RemSIOp::create(insideBuilder,
                 loc, previous, normalizedUpperBounds[idx]);
             replaceAllUsesInRegionWith(loops.getBody()->getArgument(idx), iv,
                                        loops.getRegion());
 
             // Remove the effect of the current induction value to prepare for
             // the next value.
-            previous = insideBuilder.create<arith::DivSIOp>(
+            previous = arith::DivSIOp::create(insideBuilder,
                 loc, previous, normalizedUpperBounds[idx]);
           }
 
@@ -1241,7 +1241,7 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
   auto iv = forOp.getInductionVar();
 
   OpBuilder b(forOp);
-  forOp.setStep(b.create<arith::MulIOp>(forOp.getLoc(), originalStep, factor));
+  forOp.setStep(arith::MulIOp::create(b, forOp.getLoc(), originalStep, factor));
 
   Loops innerLoops;
   for (auto t : targets) {
@@ -1251,12 +1251,12 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
 
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator((t.getBody()));
-    Value stepped = b.create<arith::AddIOp>(t.getLoc(), iv, forOp.getStep());
+    Value stepped = arith::AddIOp::create(b, t.getLoc(), iv, forOp.getStep());
     Value ub =
-        b.create<arith::MinSIOp>(t.getLoc(), forOp.getUpperBound(), stepped);
+        arith::MinSIOp::create(b, t.getLoc(), forOp.getUpperBound(), stepped);
 
     // Splice [begin, begin + nOps - 1) into `newForOp` and replace uses.
-    auto newForOp = b.create<scf::ForOp>(t.getLoc(), iv, ub, originalStep);
+    auto newForOp = scf::ForOp::create(b, t.getLoc(), iv, ub, originalStep);
     newForOp.getBody()->getOperations().splice(
         newForOp.getBody()->getOperations().begin(),
         t.getBody()->getOperations(), begin, std::next(begin, nOps - 1));
@@ -1343,7 +1343,7 @@ TileLoops mlir::extractFixedOuterLoops(scf::ForOp rootForOp,
     auto forOp = forOps[i];
     OpBuilder builder(forOp);
     auto loc = forOp.getLoc();
-    Value diff = builder.create<arith::SubIOp>(loc, forOp.getUpperBound(),
+    Value diff = arith::SubIOp::create(builder, loc, forOp.getUpperBound(),
                                                forOp.getLowerBound());
     Value numIterations = ceilDivPositive(builder, loc, diff, forOp.getStep());
     Value iterationsPerBlock =
@@ -1376,7 +1376,7 @@ scf::ForallOp mlir::fuseIndependentSiblingForallLoops(scf::ForallOp target,
 
   // Create a new scf.forall op after the source loop.
   rewriter.setInsertionPointAfter(source);
-  scf::ForallOp fusedLoop = rewriter.create<scf::ForallOp>(
+  scf::ForallOp fusedLoop = scf::ForallOp::create(rewriter,
       source.getLoc(), source.getMixedLowerBound(), source.getMixedUpperBound(),
       source.getMixedStep(), fusedOuts, source.getMapping());
 
@@ -1429,7 +1429,7 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   // Create a new scf.for op after the source loop (with scf.yield terminator
   // (without arguments) only in case its init_args is empty).
   rewriter.setInsertionPointAfter(source);
-  scf::ForOp fusedLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp fusedLoop = scf::ForOp::create(rewriter,
       source.getLoc(), source.getLowerBound(), source.getUpperBound(),
       source.getStep(), fusedInitArgs);
 
@@ -1456,7 +1456,7 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   for (Value operand : source.getBody()->getTerminator()->getOperands())
     yieldResults.push_back(mapping.lookupOrDefault(operand));
   if (!yieldResults.empty())
-    rewriter.create<scf::YieldOp>(source.getLoc(), yieldResults);
+    scf::YieldOp::create(rewriter, source.getLoc(), yieldResults);
 
   // Replace old loops by substituting their uses by results of the fused loop.
   rewriter.replaceOp(target, fusedLoop.getResults().take_front(numTargetOuts));
@@ -1487,7 +1487,7 @@ FailureOr<scf::ForallOp> mlir::normalizeForallOp(RewriterBase &rewriter,
 
   // Use the normalized builder since the lower bounds are always 0 and the
   // steps are always 1.
-  auto normalizedForallOp = rewriter.create<scf::ForallOp>(
+  auto normalizedForallOp = scf::ForallOp::create(rewriter,
       loc, newUbs, forallOp.getOutputs(), forallOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
 
diff --git a/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp b/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
index 66eed861b2bb7..48c0b1ed8e14f 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
@@ -30,14 +30,14 @@ Operation *SMTDialect::materializeConstant(OpBuilder &builder, Attribute value,
     if (auto attrValue = dyn_cast<BitVectorAttr>(value)) {
       assert(bvType == attrValue.getType() &&
              "attribute and desired result types have to match");
-      return builder.create<BVConstantOp>(loc, attrValue);
+      return BVConstantOp::create(builder, loc, attrValue);
     }
   }
 
   // BoolType constants can materialize into smt.constant
   if (auto boolType = dyn_cast<BoolType>(type)) {
     if (auto attrValue = dyn_cast<BoolAttr>(value))
-      return builder.create<BoolConstantOp>(loc, attrValue);
+      return BoolConstantOp::create(builder, loc, attrValue);
   }
 
   return nullptr;
diff --git a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
index 8977a3abc125d..c517ef2b18f6f 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
@@ -405,7 +405,7 @@ static void buildQuantifier(
         SmallVector<Location>(boundVarTypes.size(), odsState.location));
     Value returnVal =
         bodyBuilder(odsBuilder, odsState.location, block->getArguments());
-    odsBuilder.create<smt::YieldOp>(odsState.location, returnVal);
+    smt::YieldOp::create(odsBuilder, odsState.location, returnVal);
   }
   if (patternBuilder) {
     Region *region = odsState.addRegion();
@@ -416,7 +416,7 @@ static void buildQuantifier(
         SmallVector<Location>(boundVarTypes.size(), odsState.location));
     ValueRange returnVals =
         patternBuilder(odsBuilder, odsState.location, block->getArguments());
-    odsBuilder.create<smt::YieldOp>(odsState.location, returnVals);
+    smt::YieldOp::create(odsBuilder, odsState.location, returnVals);
   }
 }
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
index 371456552b5b5..cacc52d0feaf0 100644
--- a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
@@ -391,7 +391,7 @@ void LoopOp::addEntryAndMergeBlock(OpBuilder &builder) {
   builder.createBlock(&getBody());
 
   // Add a spirv.mlir.merge op into the merge block.
-  builder.create<spirv::MergeOp>(getLoc());
+  spirv::MergeOp::create(builder, getLoc());
 }
 
 //===----------------------------------------------------------------------===//
@@ -543,7 +543,7 @@ void SelectionOp::addMergeBlock(OpBuilder &builder) {
   builder.createBlock(&getBody());
 
   // Add a spirv.mlir.merge op into the merge block.
-  builder.create<spirv::MergeOp>(getLoc());
+  spirv::MergeOp::create(builder, getLoc());
 }
 
 SelectionOp
@@ -551,7 +551,7 @@ SelectionOp::createIfThen(Location loc, Value condition,
                           function_ref<void(OpBuilder &builder)> thenBody,
                           OpBuilder &builder) {
   auto selectionOp =
-      builder.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
+      spirv::SelectionOp::create(builder, loc, spirv::SelectionControl::None);
 
   selectionOp.addMergeBlock(builder);
   Block *mergeBlock = selectionOp.getMergeBlock();
@@ -562,14 +562,14 @@ SelectionOp::createIfThen(Location loc, Value condition,
     OpBuilder::InsertionGuard guard(builder);
     thenBlock = builder.createBlock(mergeBlock);
     thenBody(builder);
-    builder.create<spirv::BranchOp>(loc, mergeBlock);
+    spirv::BranchOp::create(builder, loc, mergeBlock);
   }
 
   // Build the header block.
   {
     OpBuilder::InsertionGuard guard(builder);
     builder.createBlock(thenBlock);
-    builder.create<spirv::BranchConditionalOp>(
+    spirv::BranchConditionalOp::create(builder,
         loc, condition, thenBlock,
         /*trueArguments=*/ArrayRef<Value>(), mergeBlock,
         /*falseArguments=*/ArrayRef<Value>());
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index 047f8da0cc003..53c07fc88e2ab 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -178,16 +178,16 @@ struct IAddCarryFold final : OpRewritePattern<spirv::IAddCarryOp> {
       return failure();
 
     Value addsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, adds);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, adds);
 
     Value carrysVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, carrys);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, carrys);
 
     // Create empty struct
-    Value undef = rewriter.create<spirv::UndefOp>(loc, op.getType());
+    Value undef = spirv::UndefOp::create(rewriter, loc, op.getType());
     // Fill in adds at id 0
     Value intermediate =
-        rewriter.create<spirv::CompositeInsertOp>(loc, addsVal, undef, 0);
+        spirv::CompositeInsertOp::create(rewriter, loc, addsVal, undef, 0);
     // Fill in carrys at id 1
     rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(op, carrysVal,
                                                           intermediate, 1);
@@ -260,16 +260,16 @@ struct MulExtendedFold final : OpRewritePattern<MulOp> {
       return failure();
 
     Value lowBitsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, lowBits);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, lowBits);
 
     Value highBitsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, highBits);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, highBits);
 
     // Create empty struct
-    Value undef = rewriter.create<spirv::UndefOp>(loc, op.getType());
+    Value undef = spirv::UndefOp::create(rewriter, loc, op.getType());
     // Fill in lowBits at id 0
     Value intermediate =
-        rewriter.create<spirv::CompositeInsertOp>(loc, lowBitsVal, undef, 0);
+        spirv::CompositeInsertOp::create(rewriter, loc, lowBitsVal, undef, 0);
     // Fill in highBits at id 1
     rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(op, highBitsVal,
                                                           intermediate, 1);
@@ -1309,10 +1309,10 @@ struct ConvertSelectionOpToSelect final : OpRewritePattern<spirv::SelectionOp> {
     auto storeOpAttributes =
         cast<spirv::StoreOp>(trueBlock->front())->getAttrs();
 
-    auto selectOp = rewriter.create<spirv::SelectOp>(
+    auto selectOp = spirv::SelectOp::create(rewriter,
         selectionOp.getLoc(), trueValue.getType(),
         brConditionalOp.getCondition(), trueValue, falseValue);
-    rewriter.create<spirv::StoreOp>(selectOp.getLoc(), ptrValue,
+    spirv::StoreOp::create(rewriter, selectOp.getLoc(), ptrValue,
                                     selectOp.getResult(), storeOpAttributes);
 
     // `spirv.mlir.selection` is not needed anymore.
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
index f32c53b8f0b9e..c9a8e97bd3296 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
@@ -940,12 +940,12 @@ Operation *SPIRVDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (!spirv::ConstantOp::isBuildableWith(type))
     return nullptr;
 
-  return builder.create<spirv::ConstantOp>(loc, type, value);
+  return spirv::ConstantOp::create(builder, loc, type, value);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 7042f12ba00f8..945f95b0f1dc6 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -651,25 +651,25 @@ spirv::ConstantOp spirv::ConstantOp::getZero(Type type, Location loc,
   if (auto intType = llvm::dyn_cast<IntegerType>(type)) {
     unsigned width = intType.getWidth();
     if (width == 1)
-      return builder.create<spirv::ConstantOp>(loc, type,
+      return spirv::ConstantOp::create(builder, loc, type,
                                                builder.getBoolAttr(false));
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getFloatAttr(floatType, 0.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 0).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 0.0).getValue()));
@@ -684,25 +684,25 @@ spirv::ConstantOp spirv::ConstantOp::getOne(Type type, Location loc,
   if (auto intType = llvm::dyn_cast<IntegerType>(type)) {
     unsigned width = intType.getWidth();
     if (width == 1)
-      return builder.create<spirv::ConstantOp>(loc, type,
+      return spirv::ConstantOp::create(builder, loc, type,
                                                builder.getBoolAttr(true));
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getFloatAttr(floatType, 1.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 1).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 1.0).getValue()));
@@ -1884,7 +1884,7 @@ ParseResult spirv::SpecConstantOperationOp::parse(OpAsmParser &parser,
 
   OpBuilder builder(parser.getContext());
   builder.setInsertionPointToEnd(&block);
-  builder.create<spirv::YieldOp>(wrappedOp->getLoc(), wrappedOp->getResult(0));
+  spirv::YieldOp::create(builder, wrappedOp->getLoc(), wrappedOp->getResult(0));
   result.location = wrappedOp->getLoc();
 
   result.addTypes(wrappedOp->getResult(0).getType());
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 8da688806bade..0795678a9c66b 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -105,7 +105,7 @@ OwningOpRef<spirv::ModuleOp> combine(ArrayRef<spirv::ModuleOp> inputModules,
     }
   }
 
-  auto combinedModule = combinedModuleBuilder.create<spirv::ModuleOp>(
+  auto combinedModule = spirv::ModuleOp::create(combinedModuleBuilder,
       firstModule.getLoc(), addressingModel, memoryModel, vceTriple);
   combinedModuleBuilder.setInsertionPointToStart(combinedModule.getBody());
 
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 85525a5a02fa2..574f9a7c24b12 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -70,7 +70,7 @@ createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
   varType =
       spirv::PointerType::get(varPointeeType, varPtrType.getStorageClass());
 
-  return builder.create<spirv::GlobalVariableOp>(
+  return spirv::GlobalVariableOp::create(builder,
       funcOp.getLoc(), varType, varName, abiInfo.getDescriptorSet(),
       abiInfo.getBinding());
 }
@@ -146,7 +146,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     return funcOp.emitRemark("lower entry point failure: could not select "
                              "execution model based on 'spirv.target_env'");
 
-  builder.create<spirv::EntryPointOp>(funcOp.getLoc(), *executionModel, funcOp,
+  spirv::EntryPointOp::create(builder, funcOp.getLoc(), *executionModel, funcOp,
                                       interfaceVars);
 
   // Specifies the spirv.ExecutionModeOp.
@@ -154,7 +154,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::LocalSize);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
                                              spirv::ExecutionMode::LocalSize,
                                              workgroupSizeAttr.asArrayRef());
       // Erase workgroup size.
@@ -167,7 +167,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SubgroupSize);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
                                              spirv::ExecutionMode::SubgroupSize,
                                              *subgroupSize);
       // Erase subgroup size.
@@ -180,7 +180,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SignedZeroInfNanPreserve);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(
+      spirv::ExecutionModeOp::create(builder,
           funcOp.getLoc(), funcOp,
           spirv::ExecutionMode::SignedZeroInfNanPreserve, *targetWidth);
       // Erase target width.
@@ -259,7 +259,7 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
 
     // Insert spirv::AddressOf and spirv::AccessChain operations.
     Value replacement =
-        rewriter.create<spirv::AddressOfOp>(funcOp.getLoc(), var);
+        spirv::AddressOfOp::create(rewriter, funcOp.getLoc(), var);
     // Check if the arg is a scalar or vector type. In that case, the value
     // needs to be loaded into registers.
     // TODO: This is loading value of the scalar into registers
@@ -269,9 +269,9 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     if (cast<spirv::SPIRVType>(argType.value()).isScalarOrVector()) {
       auto zero =
           spirv::ConstantOp::getZero(indexType, funcOp.getLoc(), rewriter);
-      auto loadPtr = rewriter.create<spirv::AccessChainOp>(
+      auto loadPtr = spirv::AccessChainOp::create(rewriter,
           funcOp.getLoc(), replacement, zero.getConstant());
-      replacement = rewriter.create<spirv::LoadOp>(funcOp.getLoc(), loadPtr);
+      replacement = spirv::LoadOp::create(rewriter, funcOp.getLoc(), loadPtr);
     }
     signatureConverter.remapInput(argType.index(), replacement);
   }
@@ -308,7 +308,7 @@ void LowerABIAttributesPass::runOnOperation() {
                                             ValueRange inputs, Location loc) {
     if (inputs.size() != 1 || !isa<spirv::PointerType>(inputs[0].getType()))
       return Value();
-    return builder.create<spirv::BitcastOp>(loc, type, inputs[0]).getResult();
+    return spirv::BitcastOp::create(builder, loc, type, inputs[0]).getResult();
   });
 
   RewritePatternSet patterns(context);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
index ab5898d0e3925..8bb34de8988b8 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
@@ -65,7 +65,7 @@ void RewriteInsertsPass::runOnOperation() {
       operands.push_back(insertionOp.getObject());
 
     OpBuilder builder(lastCompositeInsertOp);
-    auto compositeConstructOp = builder.create<spirv::CompositeConstructOp>(
+    auto compositeConstructOp = spirv::CompositeConstructOp::create(builder,
         location, compositeType, operands);
 
     lastCompositeInsertOp.replaceAllUsesWith(
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index f70b3325f8725..d1fa4bbc208d7 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -669,21 +669,21 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
                               Location loc) {
   // We can only cast one value in SPIR-V.
   if (inputs.size() != 1) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   Value input = inputs.front();
 
   // Only support integer types for now. Floating point types to be implemented.
   if (!isa<IntegerType>(type)) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   auto inputType = cast<IntegerType>(input.getType());
 
   auto scalarType = dyn_cast<spirv::ScalarType>(type);
   if (!scalarType) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -691,14 +691,14 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // truncating to go back so we don't need to worry about the signedness.
   // For extension, we cannot have enough signal here to decide which op to use.
   if (inputType.getIntOrFloatBitWidth() < scalarType.getIntOrFloatBitWidth()) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
   // Boolean values would need to use different ops than normal integer values.
   if (type.isInteger(1)) {
     Value one = spirv::ConstantOp::getOne(inputType, loc, builder);
-    return builder.create<spirv::IEqualOp>(loc, input, one);
+    return spirv::IEqualOp::create(builder, loc, input, one);
   }
 
   // Check that the source integer type is supported by the environment.
@@ -708,7 +708,7 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   scalarType.getCapabilities(caps);
   if (failed(checkCapabilityRequirements(type, targetEnv, caps)) ||
       failed(checkExtensionRequirements(type, targetEnv, exts))) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -716,9 +716,9 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // care about signedness here. Still try to use a corresponding op for better
   // consistency though.
   if (type.isSignedInteger()) {
-    return builder.create<spirv::SConvertOp>(loc, type, input);
+    return spirv::SConvertOp::create(builder, loc, type, input);
   }
-  return builder.create<spirv::UConvertOp>(loc, type, input);
+  return spirv::UConvertOp::create(builder, loc, type, input);
 }
 
 //===----------------------------------------------------------------------===//
@@ -770,7 +770,7 @@ getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
                                            spirv::StorageClass::Input);
     std::string name = getBuiltinVarName(builtin, prefix, suffix);
     newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+        spirv::GlobalVariableOp::create(builder, loc, ptrType, name, builtin);
     break;
   }
   case spirv::BuiltIn::SubgroupId:
@@ -781,7 +781,7 @@ getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
         spirv::PointerType::get(integerType, spirv::StorageClass::Input);
     std::string name = getBuiltinVarName(builtin, prefix, suffix);
     newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+        spirv::GlobalVariableOp::create(builder, loc, ptrType, name, builtin);
     break;
   }
   default:
@@ -842,7 +842,7 @@ getOrInsertPushConstantVariable(Location loc, Block &block,
   auto builder = OpBuilder::atBlockBegin(&block, b.getListener());
   auto type = getPushConstantStorageType(elementCount, builder, indexType);
   const char *name = "__push_constant_var__";
-  return builder.create<spirv::GlobalVariableOp>(loc, type, name,
+  return spirv::GlobalVariableOp::create(builder, loc, type, name,
                                                  /*initializer=*/nullptr);
 }
 
@@ -879,7 +879,7 @@ struct FuncOpConversion final : OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted spirv.func op.
-    auto newFuncOp = rewriter.create<spirv::FuncOp>(
+    auto newFuncOp = spirv::FuncOp::create(rewriter,
         funcOp.getLoc(), funcOp.getName(),
         rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
                                  resultType ? TypeRange(resultType)
@@ -919,7 +919,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
     }
 
     // Create a new func op with the original type and copy the function body.
-    auto newFuncOp = rewriter.create<func::FuncOp>(funcOp.getLoc(),
+    auto newFuncOp = func::FuncOp::create(rewriter, funcOp.getLoc(),
                                                    funcOp.getName(), fnType);
     rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
                                 newFuncOp.end());
@@ -954,7 +954,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto origVecType = dyn_cast<VectorType>(origType);
       if (!origVecType) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = rewriter.create<arith::ConstantOp>(
+        Value result = arith::ConstantOp::create(rewriter,
             loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
@@ -967,7 +967,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto targetShape = getTargetShape(origVecType);
       if (!targetShape) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = rewriter.create<arith::ConstantOp>(
+        Value result = arith::ConstantOp::create(rewriter,
             loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
@@ -982,11 +982,11 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
           llvm::to_vector_of<int64_t, 4>(origVecType.getShape());
 
       // Prepare the result vector.
-      Value result = rewriter.create<arith::ConstantOp>(
+      Value result = arith::ConstantOp::create(rewriter,
           loc, origVecType, rewriter.getZeroAttr(origVecType));
       ++newOpCount;
       // Prepare the placeholder for the new arguments that will be added later.
-      Value dummy = rewriter.create<arith::ConstantOp>(
+      Value dummy = arith::ConstantOp::create(rewriter,
           loc, unrolledType, rewriter.getZeroAttr(unrolledType));
       ++newOpCount;
 
@@ -995,7 +995,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       SmallVector<Type> newTypes;
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        result = rewriter.create<vector::InsertStridedSliceOp>(
+        result = vector::InsertStridedSliceOp::create(rewriter,
             loc, dummy, result, offsets, strides);
         newTypes.push_back(unrolledType);
         unrolledInputNums.push_back(newInputNo);
@@ -1109,12 +1109,12 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
       Value returnValue = returnOp.getOperand(origResultNo);
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        Value result = rewriter.create<vector::ExtractStridedSliceOp>(
+        Value result = vector::ExtractStridedSliceOp::create(rewriter,
             loc, returnValue, offsets, extractShape, strides);
         if (originalShape.size() > 1) {
           SmallVector<int64_t> extractIndices(originalShape.size() - 1, 0);
           result =
-              rewriter.create<vector::ExtractOp>(loc, result, extractIndices);
+              vector::ExtractOp::create(rewriter, loc, result, extractIndices);
         }
         newOperands.push_back(result);
         newTypes.push_back(unrolledType);
@@ -1132,7 +1132,7 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
     // Replace the return op using the new operands. This will automatically
     // update the entry block as well.
     rewriter.replaceOp(returnOp,
-                       rewriter.create<func::ReturnOp>(loc, newOperands));
+                       func::ReturnOp::create(rewriter, loc, newOperands));
 
     return success();
   }
@@ -1157,8 +1157,8 @@ Value mlir::spirv::getBuiltinVariableValue(Operation *op,
   spirv::GlobalVariableOp varOp =
       getOrInsertBuiltinVariable(*parent->getRegion(0).begin(), op->getLoc(),
                                  builtin, integerType, builder, prefix, suffix);
-  Value ptr = builder.create<spirv::AddressOfOp>(op->getLoc(), varOp);
-  return builder.create<spirv::LoadOp>(op->getLoc(), ptr);
+  Value ptr = spirv::AddressOfOp::create(builder, op->getLoc(), varOp);
+  return spirv::LoadOp::create(builder, op->getLoc(), ptr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1179,12 +1179,12 @@ Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
       loc, parent->getRegion(0).front(), elementCount, builder, integerType);
 
   Value zeroOp = spirv::ConstantOp::getZero(integerType, loc, builder);
-  Value offsetOp = builder.create<spirv::ConstantOp>(
+  Value offsetOp = spirv::ConstantOp::create(builder,
       loc, integerType, builder.getI32IntegerAttr(offset));
-  auto addrOp = builder.create<spirv::AddressOfOp>(loc, varOp);
-  auto acOp = builder.create<spirv::AccessChainOp>(
+  auto addrOp = spirv::AddressOfOp::create(builder, loc, varOp);
+  auto acOp = spirv::AccessChainOp::create(builder,
       loc, addrOp, llvm::ArrayRef({zeroOp, offsetOp}));
-  return builder.create<spirv::LoadOp>(loc, acOp);
+  return spirv::LoadOp::create(builder, loc, acOp);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1244,7 +1244,7 @@ Value mlir::spirv::getVulkanElementPtr(const SPIRVTypeConverter &typeConverter,
     linearizedIndices.push_back(
         linearizeIndex(indices, strides, offset, indexType, loc, builder));
   }
-  return builder.create<spirv::AccessChainOp>(loc, basePtr, linearizedIndices);
+  return spirv::AccessChainOp::create(builder, loc, basePtr, linearizedIndices);
 }
 
 Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
@@ -1275,10 +1275,10 @@ Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
       cast<spirv::PointerType>(basePtr.getType()).getPointeeType();
   if (isa<spirv::ArrayType>(pointeeType)) {
     linearizedIndices.push_back(linearIndex);
-    return builder.create<spirv::AccessChainOp>(loc, basePtr,
+    return spirv::AccessChainOp::create(builder, loc, basePtr,
                                                 linearizedIndices);
   }
-  return builder.create<spirv::PtrAccessChainOp>(loc, basePtr, linearIndex,
+  return spirv::PtrAccessChainOp::create(builder, loc, basePtr, linearIndex,
                                                  linearizedIndices);
 }
 
@@ -1465,7 +1465,7 @@ SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr,
       });
   addTargetMaterialization([](OpBuilder &builder, Type type, ValueRange inputs,
                               Location loc) {
-    auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return cast.getResult(0);
   });
 }
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
index af1cf2a1373e3..17f79780afa20 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
@@ -64,16 +64,16 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   //     and 4 additions after constant folding.
   //   - With sign-extended arguments, we end up emitting 8 multiplications and
   //     and 12 additions after CSE.
-  Value cstLowMask = rewriter.create<ConstantOp>(
+  Value cstLowMask = ConstantOp::create(rewriter,
       loc, lhs.getType(), getScalarOrSplatAttr(argTy, (1 << 16) - 1));
   auto getLowDigit = [&rewriter, loc, cstLowMask](Value val) {
-    return rewriter.create<BitwiseAndOp>(loc, val, cstLowMask);
+    return BitwiseAndOp::create(rewriter, loc, val, cstLowMask);
   };
 
-  Value cst16 = rewriter.create<ConstantOp>(loc, lhs.getType(),
+  Value cst16 = ConstantOp::create(rewriter, loc, lhs.getType(),
                                             getScalarOrSplatAttr(argTy, 16));
   auto getHighDigit = [&rewriter, loc, cst16](Value val) {
-    return rewriter.create<ShiftRightLogicalOp>(loc, val, cst16);
+    return ShiftRightLogicalOp::create(rewriter, loc, val, cst16);
   };
 
   auto getSignDigit = [&rewriter, loc, cst16, &getHighDigit](Value val) {
@@ -82,10 +82,10 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
     // fine. We do not have to introduce an extra constant since any
     // value in [15, 32) would do.
     return getHighDigit(
-        rewriter.create<ShiftRightArithmeticOp>(loc, val, cst16));
+        ShiftRightArithmeticOp::create(rewriter, loc, val, cst16));
   };
 
-  Value cst0 = rewriter.create<ConstantOp>(loc, lhs.getType(),
+  Value cst0 = ConstantOp::create(rewriter, loc, lhs.getType(),
                                            getScalarOrSplatAttr(argTy, 0));
 
   Value lhsLow = getLowDigit(lhs);
@@ -108,7 +108,7 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
         continue;
 
       Value &thisResDigit = resultDigits[i + j];
-      Value mul = rewriter.create<IMulOp>(loc, lhsDigit, rhsDigit);
+      Value mul = IMulOp::create(rewriter, loc, lhsDigit, rhsDigit);
       Value current = rewriter.createOrFold<IAddOp>(loc, thisResDigit, mul);
       thisResDigit = getLowDigit(current);
 
@@ -122,13 +122,13 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   }
 
   auto combineDigits = [loc, cst16, &rewriter](Value low, Value high) {
-    Value highBits = rewriter.create<ShiftLeftLogicalOp>(loc, high, cst16);
-    return rewriter.create<BitwiseOrOp>(loc, low, highBits);
+    Value highBits = ShiftLeftLogicalOp::create(rewriter, loc, high, cst16);
+    return BitwiseOrOp::create(rewriter, loc, low, highBits);
   };
   Value low = combineDigits(resultDigits[0], resultDigits[1]);
   Value high = combineDigits(resultDigits[2], resultDigits[3]);
 
-  return rewriter.create<CompositeConstructOp>(
+  return CompositeConstructOp::create(rewriter,
       loc, mulOp->getResultTypes().front(), llvm::ArrayRef({low, high}));
 }
 
@@ -185,16 +185,16 @@ struct ExpandAddCarryPattern final : OpRewritePattern<IAddCarryOp> {
           llvm::formatv("Unexpected integer type for WebGPU: '{0}'", elemTy));
 
     Value one =
-        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 1));
+        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 1));
     Value zero =
-        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 0));
+        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 0));
 
     // Calculate the carry by checking if the addition resulted in an overflow.
-    Value out = rewriter.create<IAddOp>(loc, lhs, rhs);
-    Value cmp = rewriter.create<ULessThanOp>(loc, out, lhs);
-    Value carry = rewriter.create<SelectOp>(loc, cmp, one, zero);
+    Value out = IAddOp::create(rewriter, loc, lhs, rhs);
+    Value cmp = ULessThanOp::create(rewriter, loc, out, lhs);
+    Value carry = SelectOp::create(rewriter, loc, cmp, one, zero);
 
-    Value add = rewriter.create<CompositeConstructOp>(
+    Value add = CompositeConstructOp::create(rewriter,
         loc, op->getResultTypes().front(), llvm::ArrayRef({out, carry}));
 
     rewriter.replaceOp(op, add);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
index 527d92634c196..e7bb2d76666fb 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
@@ -380,13 +380,13 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = dstNumBytes / srcNumBytes;
-      auto ratioValue = rewriter.create<spirv::ConstantOp>(
+      auto ratioValue = spirv::ConstantOp::create(rewriter,
           loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
-          rewriter.create<spirv::SDivOp>(loc, indexType, oldIndex, ratioValue);
+          spirv::SDivOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
       indices.push_back(
-          rewriter.create<spirv::SModOp>(loc, indexType, oldIndex, ratioValue));
+          spirv::SModOp::create(rewriter, loc, indexType, oldIndex, ratioValue));
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -407,11 +407,11 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = srcNumBytes / dstNumBytes;
-      auto ratioValue = rewriter.create<spirv::ConstantOp>(
+      auto ratioValue = spirv::ConstantOp::create(rewriter,
           loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
-          rewriter.create<spirv::IMulOp>(loc, indexType, oldIndex, ratioValue);
+          spirv::IMulOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -435,14 +435,14 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
     auto dstElemType = cast<spirv::SPIRVType>(dstPtrType.getPointeeType());
 
     Location loc = loadOp.getLoc();
-    auto newLoadOp = rewriter.create<spirv::LoadOp>(loc, adaptor.getPtr());
+    auto newLoadOp = spirv::LoadOp::create(rewriter, loc, adaptor.getPtr());
     if (srcElemType == dstElemType) {
       rewriter.replaceOp(loadOp, newLoadOp->getResults());
       return success();
     }
 
     if (areSameBitwidthScalarType(srcElemType, dstElemType)) {
-      auto castOp = rewriter.create<spirv::BitcastOp>(loc, srcElemType,
+      auto castOp = spirv::BitcastOp::create(rewriter, loc, srcElemType,
                                                       newLoadOp.getValue());
       rewriter.replaceOp(loadOp, castOp->getResults());
 
@@ -475,14 +475,14 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
       auto indices = llvm::to_vector<4>(acOp.getIndices());
       for (int i = 1; i < ratio; ++i) {
         // Load all subsequent components belonging to this element.
-        indices.back() = rewriter.create<spirv::IAddOp>(
+        indices.back() = spirv::IAddOp::create(rewriter,
             loc, i32Type, indices.back(), oneValue);
-        auto componentAcOp = rewriter.create<spirv::AccessChainOp>(
+        auto componentAcOp = spirv::AccessChainOp::create(rewriter,
             loc, acOp.getBasePtr(), indices);
         // Assuming little endian, this reads lower-ordered bits of the number
         // to lower-numbered components of the vector.
         components.push_back(
-            rewriter.create<spirv::LoadOp>(loc, componentAcOp));
+            spirv::LoadOp::create(rewriter, loc, componentAcOp));
       }
 
       // Create a vector of the components and then cast back to the larger
@@ -510,15 +510,15 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
               castType = VectorType::get({count}, castType);
 
             for (Value &c : components)
-              c = rewriter.create<spirv::BitcastOp>(loc, castType, c);
+              c = spirv::BitcastOp::create(rewriter, loc, castType, c);
           }
         }
-      Value vectorValue = rewriter.create<spirv::CompositeConstructOp>(
+      Value vectorValue = spirv::CompositeConstructOp::create(rewriter,
           loc, vectorType, components);
 
       if (!isa<VectorType>(srcElemType))
         vectorValue =
-            rewriter.create<spirv::BitcastOp>(loc, srcElemType, vectorValue);
+            spirv::BitcastOp::create(rewriter, loc, srcElemType, vectorValue);
       rewriter.replaceOp(loadOp, vectorValue);
       return success();
     }
@@ -546,7 +546,7 @@ struct ConvertStore : public ConvertAliasResource<spirv::StoreOp> {
     Location loc = storeOp.getLoc();
     Value value = adaptor.getValue();
     if (srcElemType != dstElemType)
-      value = rewriter.create<spirv::BitcastOp>(loc, dstElemType, value);
+      value = spirv::BitcastOp::create(rewriter, loc, dstElemType, value);
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, adaptor.getPtr(),
                                                 value, storeOp->getAttrs());
     return success();
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 5f395eef9d601..7339ba5c2379c 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -151,16 +151,16 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (llvm::isa<ShapeType>(type) || isExtentTensorType(type))
-    return builder.create<ConstShapeOp>(
+    return ConstShapeOp::create(builder,
         loc, type, llvm::cast<DenseIntElementsAttr>(value));
   if (llvm::isa<SizeType>(type))
-    return builder.create<ConstSizeOp>(loc, type,
+    return ConstSizeOp::create(builder, loc, type,
                                        llvm::cast<IntegerAttr>(value));
   if (llvm::isa<WitnessType>(type))
-    return builder.create<ConstWitnessOp>(loc, type,
+    return ConstWitnessOp::create(builder, loc, type,
                                           llvm::cast<BoolAttr>(value));
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
@@ -316,7 +316,7 @@ struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
     auto newYieldOp =
         rewriter.replaceOpWithNewOp<AssumingYieldOp>(yieldOp, newYieldOperands);
     rewriter.setInsertionPoint(op);
-    auto newOp = rewriter.create<AssumingOp>(
+    auto newOp = AssumingOp::create(rewriter,
         op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
     newOp.getDoRegion().takeBody(op.getDoRegion());
 
@@ -385,7 +385,7 @@ void AssumingOp::build(
 
   // Build body.
   SmallVector<Value, 2> yieldValues = bodyBuilder(builder, result.location);
-  builder.create<AssumingYieldOp>(result.location, yieldValues);
+  AssumingYieldOp::create(builder, result.location, yieldValues);
 
   SmallVector<Type, 2> assumingTypes;
   for (Value v : yieldValues)
@@ -736,13 +736,13 @@ struct BroadcastForwardSingleOperandPattern
     if (replacement.getType() != op.getType()) {
       auto loc = op.getLoc();
       if (llvm::isa<ShapeType>(op.getType())) {
-        replacement = rewriter.create<FromExtentTensorOp>(loc, replacement);
+        replacement = FromExtentTensorOp::create(rewriter, loc, replacement);
       } else {
         assert(!llvm::isa<ShapeType>(op.getType()) &&
                !llvm::isa<ShapeType>(replacement.getType()) &&
                "expect extent tensor cast");
         replacement =
-            rewriter.create<tensor::CastOp>(loc, op.getType(), replacement);
+            tensor::CastOp::create(rewriter, loc, op.getType(), replacement);
       }
     }
 
@@ -780,7 +780,7 @@ struct BroadcastFoldConstantOperandsPattern
     auto foldedConstantOperandsTy = RankedTensorType::get(
         {static_cast<int64_t>(foldedConstantShape.size())},
         rewriter.getIndexType());
-    newShapeOperands.push_back(rewriter.create<ConstShapeOp>(
+    newShapeOperands.push_back(ConstShapeOp::create(rewriter,
         op.getLoc(), foldedConstantOperandsTy,
         rewriter.getIndexTensorAttr(foldedConstantShape)));
     rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(),
@@ -845,7 +845,7 @@ struct BroadcastConcretizeResultTypePattern
       }
     }
 
-    auto newOp = rewriter.create<BroadcastOp>(
+    auto newOp = BroadcastOp::create(rewriter,
         op.getLoc(), getExtentTensorType(getContext(), maxRank),
         op.getShapes());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
@@ -1354,11 +1354,11 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
   auto loc = result.location;
   auto dimAttr = builder.getIndexAttr(dim);
   if (llvm::isa<ShapeType>(shape.getType())) {
-    Value dim = builder.create<ConstSizeOp>(loc, dimAttr);
+    Value dim = ConstSizeOp::create(builder, loc, dimAttr);
     build(builder, result, builder.getType<SizeType>(), shape, dim);
   } else {
     Value dim =
-        builder.create<arith::ConstantOp>(loc, builder.getIndexType(), dimAttr);
+        arith::ConstantOp::create(builder, loc, builder.getIndexType(), dimAttr);
     build(builder, result, builder.getIndexType(), shape, dim);
   }
 }
@@ -1708,7 +1708,7 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
                                   rewriter.getIndexTensorAttr(type.getShape()))
             .getResult();
     if (constShape.getType() != op.getResult().getType())
-      constShape = rewriter.create<tensor::CastOp>(
+      constShape = tensor::CastOp::create(rewriter,
           loc, op.getResult().getType(), constShape);
     rewriter.replaceOp(op, constShape);
     return success();
@@ -1751,10 +1751,10 @@ struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
 
     if (opTensorTy != shapeTensorTy) {
       if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
-        shape = rewriter.create<tensor::CastOp>(op.getLoc(), opTensorTy, shape);
+        shape = tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
       else if (!isExtentTensorType(shapeTensorTy))
         shape =
-            rewriter.create<arith::IndexCastOp>(op.getLoc(), opTensorTy, shape);
+            arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
     }
 
     rewriter.replaceOp(op, shape);
diff --git a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
index 8a471c12d21e4..5533a7e914bf0 100644
--- a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -57,7 +57,7 @@ struct AssumingOpInterface
 
     // Create new op and move over region.
     TypeRange newResultTypes(yieldOp.getOperands());
-    auto newOp = rewriter.create<shape::AssumingOp>(
+    auto newOp = shape::AssumingOp::create(rewriter,
         op->getLoc(), newResultTypes, assumingOp.getWitness());
     newOp.getDoRegion().takeBody(assumingOp.getRegion());
 
@@ -66,7 +66,7 @@ struct AssumingOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(assumingOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
+        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
             assumingOp.getLoc(), it.value(), newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
diff --git a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
index ae06a34b65709..522347ff2b993 100644
--- a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
@@ -69,7 +69,7 @@ createFuncFromCluster(OpBuilder &b, const SmallVector<Operation *, 8> &cluster,
       cluster.empty()
           ? b.getFunctionType(shape.getType(), shape.getType())
           : b.getFunctionType(ValueRange(inputs).getTypes(), shape.getType());
-  shape::FuncOp fnOp = b.create<shape::FuncOp>(loc, fnName, fnType);
+  shape::FuncOp fnOp = shape::FuncOp::create(b, loc, fnName, fnType);
   Block *block = fnOp.addEntryBlock();
   b.setInsertionPointToEnd(block);
   IRMapping bvm;
@@ -85,7 +85,7 @@ createFuncFromCluster(OpBuilder &b, const SmallVector<Operation *, 8> &cluster,
   llvm::SmallVector<Value, 4> fnReturns;
   fnReturns.push_back(bvm.lookupOrDefault(shape));
 
-  b.create<shape::ReturnOp>(loc, fnReturns);
+  shape::ReturnOp::create(b, loc, fnReturns);
   fnOp.setPrivate();
   return std::make_pair(fnOp, inputs);
 }
@@ -187,7 +187,7 @@ class TensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
   LogicalResult matchAndRewrite(tensor::DimOp op,
                                 PatternRewriter &rewriter) const override {
     auto shapeOf =
-        rewriter.create<shape::ShapeOfOp>(op.getLoc(), op.getSource());
+        shape::ShapeOfOp::create(rewriter, op.getLoc(), op.getSource());
     rewriter.replaceOpWithNewOp<shape::GetExtentOp>(op, op.getType(), shapeOf,
                                                     op.getIndex());
     return success();
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index 121e0cc133e19..b5f6230bc2362 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -44,14 +44,14 @@ NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
                    ->materializeConstant(rewriter, rewriter.getIndexAttr(1),
                                          valueType, loc)
                    ->getResult(0);
-  ReduceOp reduce = rewriter.create<ReduceOp>(loc, op.getShape(), init);
+  ReduceOp reduce = ReduceOp::create(rewriter, loc, op.getShape(), init);
 
   // Generate reduce operator.
   Block *body = reduce.getBody();
   OpBuilder b = OpBuilder::atBlockEnd(body);
-  Value product = b.create<MulOp>(loc, valueType, body->getArgument(1),
+  Value product = MulOp::create(b, loc, valueType, body->getArgument(1),
                                   body->getArgument(2));
-  b.create<shape::YieldOp>(loc, product);
+  shape::YieldOp::create(b, loc, product);
 
   rewriter.replaceOp(op, reduce.getResult());
   return success();
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index 38246b96977c8..e2a6a59636a84 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -559,7 +559,7 @@ SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
   SmallVector<Type> retType(
       dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
       builder.getIndexType());
-  auto transOp = builder.create<CrdTranslateOp>(loc, retType, crds, dir, *this);
+  auto transOp = CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
   return transOp.getOutCrds();
 }
 
@@ -1481,7 +1481,7 @@ LogicalResult CrdTranslateOp::fold(FoldAdaptor adaptor,
 
 void LvlOp::build(OpBuilder &builder, OperationState &state, Value source,
                   int64_t index) {
-  Value val = builder.create<arith::ConstantIndexOp>(state.location, index);
+  Value val = arith::ConstantIndexOp::create(builder, state.location, index);
   return build(builder, state, source, val);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
index 9c84f4c25866f..74842cc1f310b 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
@@ -41,7 +41,7 @@ LogicalResult sparse_tensor::detail::stageWithSortImpl(
 
   // -> sort
   Type dstCOOTp = dstStt.getCOOType(/*ordered=*/true);
-  Value dstCOO = rewriter.create<ReorderCOOOp>(
+  Value dstCOO = ReorderCOOOp::create(rewriter,
       loc, dstCOOTp, srcCOO, SparseTensorSortKind::HybridQuickSort);
 
   // -> dest.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
index 8ee801ba46349..6545bfbcac5c7 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
@@ -88,13 +88,13 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
         } else if (directOut) {
           Value mem;
           if (kind == SparseTensorFieldKind::PosMemRef)
-            mem = builder.create<sparse_tensor::ToPositionsOp>(loc, inputs[0],
+            mem = sparse_tensor::ToPositionsOp::create(builder, loc, inputs[0],
                                                                lv);
           else if (kind == SparseTensorFieldKind::CrdMemRef)
-            mem = builder.create<sparse_tensor::ToCoordinatesOp>(loc, inputs[0],
+            mem = sparse_tensor::ToCoordinatesOp::create(builder, loc, inputs[0],
                                                                  lv);
           else
-            mem = builder.create<sparse_tensor::ToValuesOp>(loc, inputs[0]);
+            mem = sparse_tensor::ToValuesOp::create(builder, loc, inputs[0]);
           toVals.push_back(mem);
         } else {
           ShapedType rtp = cast<ShapedType>(t);
@@ -109,7 +109,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
 
     if (isIn) {
       // Assemble multiple inputs into a single sparse tensor.
-      auto a = builder.create<sparse_tensor::AssembleOp>(loc, rtp, inputs);
+      auto a = sparse_tensor::AssembleOp::create(builder, loc, rtp, inputs);
       toVals.push_back(a.getResult());
     } else if (!directOut) {
       // Disassemble a single sparse input into multiple outputs.
@@ -117,7 +117,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
       unsigned len = retTypes.size();
       retTypes.append(cntTypes);
       auto d =
-          builder.create<sparse_tensor::DisassembleOp>(loc, retTypes, inputs);
+          sparse_tensor::DisassembleOp::create(builder, loc, retTypes, inputs);
       for (unsigned i = 0; i < len; i++)
         toVals.push_back(d.getResult(i));
     }
@@ -199,7 +199,7 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     OpBuilder moduleBuilder(modOp.getBodyRegion());
     unsigned extra = inputTypes.size();
     inputTypes.append(extraTypes);
-    auto func = moduleBuilder.create<func::FuncOp>(
+    auto func = func::FuncOp::create(moduleBuilder,
         loc, orgName, FunctionType::get(context, inputTypes, outputTypes));
     func.setPublic();
 
@@ -216,14 +216,14 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     // Call the original, now private method. A subsequent inlining pass can
     // determine whether cloning the method body in place is worthwhile.
     auto org = SymbolRefAttr::get(context, wrapper);
-    auto call = rewriter.create<func::CallOp>(loc, funcOp.getResultTypes(), org,
+    auto call = func::CallOp::create(rewriter, loc, funcOp.getResultTypes(), org,
                                               inputs);
 
     // Convert outputs and return.
     SmallVector<Value> outputs;
     convVals(rewriter, loc, funcOp.getResultTypes(), call.getResults(),
              body->getArguments(), outputs, extra, /*isIn=*/false, directOut);
-    rewriter.create<func::ReturnOp>(loc, outputs);
+    func::ReturnOp::create(rewriter, loc, outputs);
 
     // Finally, migrate a potential c-interface property.
     if (funcOp->getAttrOfType<UnitAttr>(
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
index 0c5912bb73772..12507ff1c5251 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
@@ -94,7 +94,7 @@ static FlatSymbolRefAttr getMangledSortHelperFunc(
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPoint(insertPoint);
     Location loc = insertPoint.getLoc();
-    func = builder.create<func::FuncOp>(
+    func = func::FuncOp::create(builder,
         loc, nameOstream.str(),
         FunctionType::get(context, operands.getTypes(), resultTypes));
     func.setPrivate();
@@ -111,13 +111,13 @@ static void forEachIJPairInXs(
     uint64_t ny,
     function_ref<void(uint64_t, Value, Value, Value)> bodyBuilder) {
   Value cstep = constantIndex(builder, loc, xPerm.getNumResults() + ny);
-  Value iOffset = builder.create<arith::MulIOp>(loc, args[0], cstep);
-  Value jOffset = builder.create<arith::MulIOp>(loc, args[1], cstep);
+  Value iOffset = arith::MulIOp::create(builder, loc, args[0], cstep);
+  Value jOffset = arith::MulIOp::create(builder, loc, args[1], cstep);
   for (unsigned k = 0, e = xPerm.getNumResults(); k < e; k++) {
     unsigned actualK = cast<AffineDimExpr>(xPerm.getResult(k)).getPosition();
     Value ak = constantIndex(builder, loc, actualK);
-    Value i = builder.create<arith::AddIOp>(loc, ak, iOffset);
-    Value j = builder.create<arith::AddIOp>(loc, ak, jOffset);
+    Value i = arith::AddIOp::create(builder, loc, ak, iOffset);
+    Value j = arith::AddIOp::create(builder, loc, ak, jOffset);
     Value buffer = args[xStartIdx];
 
     bodyBuilder(k, i, j, buffer);
@@ -165,10 +165,10 @@ static void forEachIJPairInAllBuffers(
 static void createSwap(OpBuilder &builder, Location loc, ValueRange args,
                        AffineMap xPerm, uint64_t ny) {
   auto swapOnePair = [&](uint64_t unused, Value i, Value j, Value buffer) {
-    Value vi = builder.create<memref::LoadOp>(loc, buffer, i);
-    Value vj = builder.create<memref::LoadOp>(loc, buffer, j);
-    builder.create<memref::StoreOp>(loc, vj, buffer, i);
-    builder.create<memref::StoreOp>(loc, vi, buffer, j);
+    Value vi = memref::LoadOp::create(builder, loc, buffer, i);
+    Value vj = memref::LoadOp::create(builder, loc, buffer, j);
+    memref::StoreOp::create(builder, loc, vj, buffer, i);
+    memref::StoreOp::create(builder, loc, vi, buffer, j);
   };
 
   forEachIJPairInAllBuffers(builder, loc, args, xPerm, ny, swapOnePair);
@@ -193,7 +193,7 @@ static Value createInlinedCompareImplementation(
       OpBuilder::InsertionGuard insertionGuard(builder);
       auto ifOp = cast<scf::IfOp>(val.getDefiningOp());
       builder.setInsertionPointAfter(ifOp);
-      builder.create<scf::YieldOp>(loc, ifOp.getResult(0));
+      scf::YieldOp::create(builder, loc, ifOp.getResult(0));
     }
   };
 
@@ -207,25 +207,25 @@ static Value createInlinedCompareImplementation(
 /// result of the comparison.
 static Value createEqCompare(OpBuilder &builder, Location loc, Value i, Value j,
                              Value x, bool isFirstDim, bool isLastDim) {
-  Value vi = builder.create<memref::LoadOp>(loc, x, i);
-  Value vj = builder.create<memref::LoadOp>(loc, x, j);
+  Value vi = memref::LoadOp::create(builder, loc, x, i);
+  Value vj = memref::LoadOp::create(builder, loc, x, j);
 
   Value res;
   if (isLastDim) {
-    res = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, vi, vj);
+    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
-      builder.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(builder, loc, res);
   } else {
     Value ne =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, vi, vj);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIntegerType(1),
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
                                                ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value f = constantI1(builder, loc, false);
-    builder.create<scf::YieldOp>(loc, f);
+    scf::YieldOp::create(builder, loc, f);
 
     // If (x[i] == x[j]). Set up the insertion point for the nested if-stmt that
     // checks the remaining dimensions.
@@ -261,26 +261,26 @@ static Value createInlinedEqCompare(OpBuilder &builder, Location loc,
 static Value createLessThanCompare(OpBuilder &builder, Location loc, Value i,
                                    Value j, Value x, bool isFirstDim,
                                    bool isLastDim) {
-  Value vi = builder.create<memref::LoadOp>(loc, x, i);
-  Value vj = builder.create<memref::LoadOp>(loc, x, j);
+  Value vi = memref::LoadOp::create(builder, loc, x, i);
+  Value vj = memref::LoadOp::create(builder, loc, x, j);
 
   Value res;
   if (isLastDim) {
-    res = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, vi, vj);
+    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
-      builder.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(builder, loc, res);
   } else {
     Value ne =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, vi, vj);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIntegerType(1),
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
                                                ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value lt =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, vi, vj);
-    builder.create<scf::YieldOp>(loc, lt);
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
+    scf::YieldOp::create(builder, loc, lt);
 
     // If (x[i] == x[j]). Set up the insertion point for the nested if-stmt that
     // checks the remaining dimensions.
@@ -337,17 +337,17 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value p = args[hiIdx];
   SmallVector<Type, 2> types(2, p.getType()); // Only two types.
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(
+  scf::WhileOp whileOp = scf::WhileOp::create(builder,
       loc, types, SmallVector<Value, 2>{args[loIdx], args[hiIdx]});
 
   // The before-region of the WhileOp.
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, types, {loc, loc});
   builder.setInsertionPointToEnd(before);
-  Value cond1 = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                               before->getArgument(0),
                                               before->getArgument(1));
-  builder.create<scf::ConditionOp>(loc, cond1, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond1, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -357,9 +357,9 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   Value hi = after->getArgument(1);
   // Compute mid = (lo + hi) >> 1.
   Value c1 = constantIndex(builder, loc, 1);
-  Value mid = builder.create<arith::ShRUIOp>(
-      loc, builder.create<arith::AddIOp>(loc, lo, hi), c1);
-  Value midp1 = builder.create<arith::AddIOp>(loc, mid, c1);
+  Value mid = arith::ShRUIOp::create(builder,
+      loc, arith::AddIOp::create(builder, loc, lo, hi), c1);
+  Value midp1 = arith::AddIOp::create(builder, loc, mid, c1);
 
   // Compare xs[p] < xs[mid].
   SmallVector<Value> compareOperands{p, mid};
@@ -372,12 +372,12 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   //     hi = mid;
   //   else
   //     lo = mid + 1;
-  Value newLo = builder.create<arith::SelectOp>(loc, cond2, lo, midp1);
-  Value newHi = builder.create<arith::SelectOp>(loc, cond2, mid, hi);
-  builder.create<scf::YieldOp>(loc, ValueRange{newLo, newHi});
+  Value newLo = arith::SelectOp::create(builder, loc, cond2, lo, midp1);
+  Value newHi = arith::SelectOp::create(builder, loc, cond2, mid, hi);
+  scf::YieldOp::create(builder, loc, ValueRange{newLo, newHi});
 
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc, whileOp.getResult(0));
+  func::ReturnOp::create(builder, loc, whileOp.getResult(0));
 }
 
 /// Creates code to advance i in a loop based on xs[p] as follows:
@@ -393,7 +393,7 @@ static std::pair<Value, Value> createScanLoop(OpBuilder &builder,
                                               uint64_t ny, int step) {
   Location loc = func.getLoc();
   scf::WhileOp whileOp =
-      builder.create<scf::WhileOp>(loc, TypeRange{i.getType()}, ValueRange{i});
+      scf::WhileOp::create(builder, loc, TypeRange{i.getType()}, ValueRange{i});
 
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, {i.getType()}, {loc});
@@ -409,14 +409,14 @@ static std::pair<Value, Value> createScanLoop(OpBuilder &builder,
   }
   compareOperands.append(xs.begin(), xs.end());
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  builder.create<scf::ConditionOp>(loc, cond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond, before->getArguments());
 
   Block *after =
       builder.createBlock(&whileOp.getAfter(), {}, {i.getType()}, {loc});
   builder.setInsertionPointToEnd(after);
   Value cs = constantIndex(builder, loc, step);
-  i = builder.create<arith::AddIOp>(loc, after->getArgument(0), cs);
-  builder.create<scf::YieldOp>(loc, ValueRange{i});
+  i = arith::AddIOp::create(builder, loc, after->getArgument(0), cs);
+  scf::YieldOp::create(builder, loc, ValueRange{i});
   i = whileOp.getResult(0);
 
   builder.setInsertionPointAfter(whileOp);
@@ -440,7 +440,7 @@ static scf::IfOp createCompareThenSwap(OpBuilder &builder, Location loc,
   compareOperands[0] = b;
   compareOperands[1] = a;
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else=*/false);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, cond, /*else=*/false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   swapOperands[0] = b;
   swapOperands[1] = a;
@@ -517,12 +517,12 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
   swapOperands.append(args.begin() + xStartIdx, args.end());
   Location loc = func.getLoc();
   Value c1 = constantIndex(builder, loc, 1);
-  Value hiP1 = builder.create<arith::AddIOp>(loc, hi, c1);
-  Value len = builder.create<arith::SubIOp>(loc, hiP1, lo);
+  Value hiP1 = arith::AddIOp::create(builder, loc, hi, c1);
+  Value len = arith::SubIOp::create(builder, loc, hiP1, lo);
   Value lenThreshold = constantIndex(builder, loc, 1000);
-  Value lenCond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value lenCond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                                 len, lenThreshold);
-  scf::IfOp lenIf = builder.create<scf::IfOp>(loc, lenCond, /*else=*/true);
+  scf::IfOp lenIf = scf::IfOp::create(builder, loc, lenCond, /*else=*/true);
 
   // When len < 1000, choose pivot from median of 3 values.
   builder.setInsertionPointToStart(&lenIf.getThenRegion().front());
@@ -531,13 +531,13 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
 
   // When len >= 1000, choose pivot from median of 5 values.
   builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
-  Value miP1 = builder.create<arith::AddIOp>(loc, hi, c1);
-  Value a = builder.create<arith::AddIOp>(loc, lo, miP1);
+  Value miP1 = arith::AddIOp::create(builder, loc, hi, c1);
+  Value a = arith::AddIOp::create(builder, loc, lo, miP1);
   // Value a is the middle between [loc, mi].
-  a = builder.create<arith::ShRUIOp>(loc, a, c1);
-  Value b = builder.create<arith::AddIOp>(loc, mi, hiP1);
+  a = arith::ShRUIOp::create(builder, loc, a, c1);
+  Value b = arith::AddIOp::create(builder, loc, mi, hiP1);
   // Value b is the middle between [mi, hi].
-  b = builder.create<arith::ShRUIOp>(loc, b, c1);
+  b = arith::ShRUIOp::create(builder, loc, b, c1);
   createSort5(builder, loc, xPerm, ny, swapOperands, compareOperands, lo, a, mi,
               b, hi);
 
@@ -589,24 +589,24 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value sum = builder.create<arith::AddIOp>(loc, lo, hi);
+  Value sum = arith::AddIOp::create(builder, loc, lo, hi);
   Value c1 = constantIndex(builder, loc, 1);
-  Value p = builder.create<arith::ShRUIOp>(loc, sum, c1);
+  Value p = arith::ShRUIOp::create(builder, loc, sum, c1);
 
   Value i = lo;
-  Value j = builder.create<arith::SubIOp>(loc, hi, c1);
+  Value j = arith::SubIOp::create(builder, loc, hi, c1);
   createChoosePivot(builder, module, func, xPerm, ny, i, j, p, args);
   Value trueVal = constantI1(builder, loc, true); // The value for while (true)
   SmallVector<Value, 4> operands{i, j, p, trueVal}; // Exactly four values.
   SmallVector<Type, 4> types{i.getType(), j.getType(), p.getType(),
                              trueVal.getType()};
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(loc, types, operands);
+  scf::WhileOp whileOp = scf::WhileOp::create(builder, loc, types, operands);
 
   // The before-region of the WhileOp.
   Block *before = builder.createBlock(&whileOp.getBefore(), {}, types,
                                       {loc, loc, loc, loc});
   builder.setInsertionPointToEnd(before);
-  builder.create<scf::ConditionOp>(loc, before->getArgument(3),
+  scf::ConditionOp::create(builder, loc, before->getArgument(3),
                                    before->getArguments());
 
   // The after-region of the WhileOp.
@@ -629,70 +629,70 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
 
   // If i < j:
   Value cond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, i, j);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, i, j);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   SmallVector<Value> swapOperands{i, j};
   swapOperands.append(args.begin() + xStartIdx, args.end());
   createSwap(builder, loc, swapOperands, xPerm, ny);
   // If the pivot is moved, update p with the new pivot.
   Value icond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, i, p);
-  scf::IfOp ifOpI = builder.create<scf::IfOp>(loc, TypeRange{p.getType()},
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, i, p);
+  scf::IfOp ifOpI = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
                                               icond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpI.getThenRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{j});
+  scf::YieldOp::create(builder, loc, ValueRange{j});
   builder.setInsertionPointToStart(&ifOpI.getElseRegion().front());
   Value jcond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, j, p);
-  scf::IfOp ifOpJ = builder.create<scf::IfOp>(loc, TypeRange{p.getType()},
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, j, p);
+  scf::IfOp ifOpJ = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
                                               jcond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpJ.getThenRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{i});
+  scf::YieldOp::create(builder, loc, ValueRange{i});
   builder.setInsertionPointToStart(&ifOpJ.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{p});
+  scf::YieldOp::create(builder, loc, ValueRange{p});
   builder.setInsertionPointAfter(ifOpJ);
-  builder.create<scf::YieldOp>(loc, ifOpJ.getResults());
+  scf::YieldOp::create(builder, loc, ifOpJ.getResults());
   builder.setInsertionPointAfter(ifOpI);
   Value compareEqIJ =
-      builder.create<arith::AndIOp>(loc, iCompareEq, jCompareEq);
-  scf::IfOp ifOp2 = builder.create<scf::IfOp>(
+      arith::AndIOp::create(builder, loc, iCompareEq, jCompareEq);
+  scf::IfOp ifOp2 = scf::IfOp::create(builder,
       loc, TypeRange{i.getType(), j.getType()}, compareEqIJ, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
-  Value i2 = builder.create<arith::AddIOp>(loc, i, c1);
-  Value j2 = builder.create<arith::SubIOp>(loc, j, c1);
-  builder.create<scf::YieldOp>(loc, ValueRange{i2, j2});
+  Value i2 = arith::AddIOp::create(builder, loc, i, c1);
+  Value j2 = arith::SubIOp::create(builder, loc, j, c1);
+  scf::YieldOp::create(builder, loc, ValueRange{i2, j2});
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{i, j});
+  scf::YieldOp::create(builder, loc, ValueRange{i, j});
   builder.setInsertionPointAfter(ifOp2);
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc,
       ValueRange{ifOp2.getResult(0), ifOp2.getResult(1), ifOpI.getResult(0),
                  /*cont=*/constantI1(builder, loc, true)});
 
   // False branch for if i < j (i.e., i >= j):
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  p = builder.create<arith::AddIOp>(loc, j,
+  p = arith::AddIOp::create(builder, loc, j,
                                     constantOne(builder, loc, j.getType()));
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc, ValueRange{i, j, p, /*cont=*/constantI1(builder, loc, false)});
 
   // Return for the whileOp.
   builder.setInsertionPointAfter(ifOp);
-  builder.create<scf::YieldOp>(loc, ifOp.getResults());
+  scf::YieldOp::create(builder, loc, ifOp.getResults());
 
   // Return for the function.
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc, whileOp.getResult(2));
+  func::ReturnOp::create(builder, loc, whileOp.getResult(2));
 }
 
 /// Computes (n-2)/n, assuming n has index type.
 static Value createSubTwoDividedByTwo(OpBuilder &builder, Location loc,
                                       Value n) {
   Value i2 = constantIndex(builder, loc, 2);
-  Value res = builder.create<arith::SubIOp>(loc, n, i2);
+  Value res = arith::SubIOp::create(builder, loc, n, i2);
   Value i1 = constantIndex(builder, loc, 1);
-  return builder.create<arith::ShRUIOp>(loc, res, i1);
+  return arith::ShRUIOp::create(builder, loc, res, i1);
 }
 
 /// Creates a function to heapify the subtree with root `start` within the full
@@ -743,16 +743,16 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   // If (n >= 2).
   Value c2 = constantIndex(builder, loc, 2);
   Value condN =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, n, c2);
-  scf::IfOp ifN = builder.create<scf::IfOp>(loc, condN, /*else=*/false);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, n, c2);
+  scf::IfOp ifN = scf::IfOp::create(builder, loc, condN, /*else=*/false);
   builder.setInsertionPointToStart(&ifN.getThenRegion().front());
-  Value child = builder.create<arith::SubIOp>(loc, start, first);
+  Value child = arith::SubIOp::create(builder, loc, start, first);
 
   // If ((n-2)/2 >= child).
   Value t = createSubTwoDividedByTwo(builder, loc, n);
   Value condNc =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp ifNc = builder.create<scf::IfOp>(loc, condNc, /*else=*/false);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
+  scf::IfOp ifNc = scf::IfOp::create(builder, loc, condNc, /*else=*/false);
 
   builder.setInsertionPointToStart(&ifNc.getThenRegion().front());
   Value c1 = constantIndex(builder, loc, 1);
@@ -768,32 +768,32 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   //   if (child+1 < n && data[childIndex] < data[childIndex+1])
   //     childIndex ++; child ++ // Right child is bigger.
   auto getLargerChild = [&](Value r) -> std::pair<Value, Value> {
-    Value lChild = builder.create<arith::ShLIOp>(loc, r, c1);
-    lChild = builder.create<arith::AddIOp>(loc, lChild, c1);
-    Value lChildIdx = builder.create<arith::AddIOp>(loc, lChild, first);
-    Value rChild = builder.create<arith::AddIOp>(loc, lChild, c1);
-    Value cond1 = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value lChild = arith::ShLIOp::create(builder, loc, r, c1);
+    lChild = arith::AddIOp::create(builder, loc, lChild, c1);
+    Value lChildIdx = arith::AddIOp::create(builder, loc, lChild, first);
+    Value rChild = arith::AddIOp::create(builder, loc, lChild, c1);
+    Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                                 rChild, n);
     SmallVector<Type, 2> ifTypes(2, r.getType());
     scf::IfOp if1 =
-        builder.create<scf::IfOp>(loc, ifTypes, cond1, /*else=*/true);
+        scf::IfOp::create(builder, loc, ifTypes, cond1, /*else=*/true);
     builder.setInsertionPointToStart(&if1.getThenRegion().front());
-    Value rChildIdx = builder.create<arith::AddIOp>(loc, rChild, first);
+    Value rChildIdx = arith::AddIOp::create(builder, loc, rChild, first);
     // Compare data[left] < data[right].
     compareOperands[0] = lChildIdx;
     compareOperands[1] = rChildIdx;
     Value cond2 =
         createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
     scf::IfOp if2 =
-        builder.create<scf::IfOp>(loc, ifTypes, cond2, /*else=*/true);
+        scf::IfOp::create(builder, loc, ifTypes, cond2, /*else=*/true);
     builder.setInsertionPointToStart(&if2.getThenRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{rChild, rChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{rChild, rChildIdx});
     builder.setInsertionPointToStart(&if2.getElseRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{lChild, lChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{lChild, lChildIdx});
     builder.setInsertionPointAfter(if2);
-    builder.create<scf::YieldOp>(loc, if2.getResults());
+    scf::YieldOp::create(builder, loc, if2.getResults());
     builder.setInsertionPointToStart(&if1.getElseRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{lChild, lChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{lChild, lChildIdx});
     builder.setInsertionPointAfter(if1);
     return std::make_pair(if1.getResult(0), if1.getResult(1));
   };
@@ -803,7 +803,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
 
   // While (data[start] < data[childIndex]).
   SmallVector<Type, 3> types(3, child.getType());
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(
+  scf::WhileOp whileOp = scf::WhileOp::create(builder,
       loc, types, SmallVector<Value, 2>{start, child, childIdx});
 
   // The before-region of the WhileOp.
@@ -815,7 +815,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   compareOperands[0] = start;
   compareOperands[1] = childIdx;
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  builder.create<scf::ConditionOp>(loc, cond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after = builder.createBlock(&whileOp.getAfter(), {}, types, locs);
@@ -827,20 +827,20 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   createSwap(builder, loc, swapOperands, xPerm, ny);
   start = childIdx;
   Value cond2 =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp if2 = builder.create<scf::IfOp>(
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
+  scf::IfOp if2 = scf::IfOp::create(builder,
       loc, TypeRange{child.getType(), child.getType()}, cond2, /*else=*/true);
   builder.setInsertionPointToStart(&if2.getThenRegion().front());
   auto [newChild, newChildIdx] = getLargerChild(child);
-  builder.create<scf::YieldOp>(loc, ValueRange{newChild, newChildIdx});
+  scf::YieldOp::create(builder, loc, ValueRange{newChild, newChildIdx});
   builder.setInsertionPointToStart(&if2.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{child, childIdx});
+  scf::YieldOp::create(builder, loc, ValueRange{child, childIdx});
   builder.setInsertionPointAfter(if2);
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc, ValueRange{start, if2.getResult(0), if2.getResult(1)});
 
   builder.setInsertionPointAfter(ifN);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Creates a function to perform heap sort on the values in the range of index
@@ -870,45 +870,45 @@ static void createHeapSortFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value n = builder.create<arith::SubIOp>(loc, hi, lo);
+  Value n = arith::SubIOp::create(builder, loc, hi, lo);
 
   // For i = (n-2)/2 downto 0.
   Value c0 = constantIndex(builder, loc, 0);
   Value c1 = constantIndex(builder, loc, 1);
   Value s = createSubTwoDividedByTwo(builder, loc, n);
-  Value up = builder.create<arith::AddIOp>(loc, s, c1);
-  scf::ForOp forI = builder.create<scf::ForOp>(loc, c0, up, c1);
+  Value up = arith::AddIOp::create(builder, loc, s, c1);
+  scf::ForOp forI = scf::ForOp::create(builder, loc, c0, up, c1);
   builder.setInsertionPointToStart(forI.getBody());
-  Value i = builder.create<arith::SubIOp>(loc, s, forI.getInductionVar());
-  Value lopi = builder.create<arith::AddIOp>(loc, lo, i);
+  Value i = arith::SubIOp::create(builder, loc, s, forI.getInductionVar());
+  Value lopi = arith::AddIOp::create(builder, loc, lo, i);
   SmallVector<Value> shiftDownOperands = {lo, lopi};
   shiftDownOperands.append(args.begin() + xStartIdx, args.end());
   shiftDownOperands.push_back(n);
   FlatSymbolRefAttr shiftDownFunc = getMangledSortHelperFunc(
       builder, func, TypeRange(), kShiftDownFuncNamePrefix, xPerm, ny,
       shiftDownOperands, createShiftDownFunc, /*nTrailingP=*/1);
-  builder.create<func::CallOp>(loc, shiftDownFunc, TypeRange(),
+  func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
                                shiftDownOperands);
 
   builder.setInsertionPointAfter(forI);
   // For l = n downto 2.
-  up = builder.create<arith::SubIOp>(loc, n, c1);
-  scf::ForOp forL = builder.create<scf::ForOp>(loc, c0, up, c1);
+  up = arith::SubIOp::create(builder, loc, n, c1);
+  scf::ForOp forL = scf::ForOp::create(builder, loc, c0, up, c1);
   builder.setInsertionPointToStart(forL.getBody());
-  Value l = builder.create<arith::SubIOp>(loc, n, forL.getInductionVar());
-  Value loplm1 = builder.create<arith::AddIOp>(loc, lo, l);
-  loplm1 = builder.create<arith::SubIOp>(loc, loplm1, c1);
+  Value l = arith::SubIOp::create(builder, loc, n, forL.getInductionVar());
+  Value loplm1 = arith::AddIOp::create(builder, loc, lo, l);
+  loplm1 = arith::SubIOp::create(builder, loc, loplm1, c1);
   SmallVector<Value> swapOperands{lo, loplm1};
   swapOperands.append(args.begin() + xStartIdx, args.end());
   createSwap(builder, loc, swapOperands, xPerm, ny);
   shiftDownOperands[1] = lo;
   shiftDownOperands[shiftDownOperands.size() - 1] =
-      builder.create<arith::SubIOp>(loc, l, c1);
-  builder.create<func::CallOp>(loc, shiftDownFunc, TypeRange(),
+      arith::SubIOp::create(builder, loc, l, c1);
+  func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
                                shiftDownOperands);
 
   builder.setInsertionPointAfter(forL);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// A helper for generating code to perform quick sort. It partitions [lo, hi),
@@ -933,35 +933,35 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
                                       args.drop_back(nTrailingP))
                 .getResult(0);
 
-  Value lenLow = builder.create<arith::SubIOp>(loc, p, lo);
-  Value lenHigh = builder.create<arith::SubIOp>(loc, hi, p);
+  Value lenLow = arith::SubIOp::create(builder, loc, p, lo);
+  Value lenHigh = arith::SubIOp::create(builder, loc, hi, p);
   // Partition already sorts array with len <= 2
   Value c2 = constantIndex(builder, loc, 2);
-  Value len = builder.create<arith::SubIOp>(loc, hi, lo);
+  Value len = arith::SubIOp::create(builder, loc, hi, lo);
   Value lenGtTwo =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ugt, len, c2);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ugt, len, c2);
   scf::IfOp ifLenGtTwo =
-      builder.create<scf::IfOp>(loc, types, lenGtTwo, /*else=*/true);
+      scf::IfOp::create(builder, loc, types, lenGtTwo, /*else=*/true);
   builder.setInsertionPointToStart(&ifLenGtTwo.getElseRegion().front());
   // Returns an empty range to mark the entire region is fully sorted.
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
   // Else len > 2, need recursion.
   builder.setInsertionPointToStart(&ifLenGtTwo.getThenRegion().front());
-  Value cond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule,
+  Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
                                              lenLow, lenHigh);
 
   Value c0 = constantIndex(builder, loc, 0);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
 
   auto mayRecursion = [&](Value low, Value high, Value len) {
     Value cond =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, len, c0);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else=*/false);
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, len, c0);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, cond, /*else=*/false);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     SmallVector<Value> operands{low, high};
     operands.append(args.begin() + xStartIdx, args.end());
-    builder.create<func::CallOp>(loc, func, operands);
+    func::CallOp::create(builder, loc, func, operands);
     builder.setInsertionPointAfter(ifOp);
   };
 
@@ -969,14 +969,14 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
   // the bigger partition to be processed by the enclosed while-loop.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   mayRecursion(lo, p, lenLow);
-  builder.create<scf::YieldOp>(loc, ValueRange{p, hi});
+  scf::YieldOp::create(builder, loc, ValueRange{p, hi});
 
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
   mayRecursion(p, hi, lenHigh);
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, p});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, p});
 
   builder.setInsertionPointAfter(ifOp);
-  builder.create<scf::YieldOp>(loc, ifOp.getResults());
+  scf::YieldOp::create(builder, loc, ifOp.getResults());
 
   builder.setInsertionPointAfter(ifLenGtTwo);
   return std::make_pair(ifLenGtTwo.getResult(0), ifLenGtTwo.getResult(1));
@@ -1011,10 +1011,10 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   Value c1 = constantIndex(builder, loc, 1);
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value lop1 = builder.create<arith::AddIOp>(loc, lo, c1);
+  Value lop1 = arith::AddIOp::create(builder, loc, lo, c1);
 
   // Start the outer for-stmt with induction variable i.
-  scf::ForOp forOpI = builder.create<scf::ForOp>(loc, lop1, hi, c1);
+  scf::ForOp forOpI = scf::ForOp::create(builder, loc, lop1, hi, c1);
   builder.setInsertionPointToStart(forOpI.getBody());
   Value i = forOpI.getInductionVar();
 
@@ -1035,24 +1035,24 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t unused, Value i, Value unused2, Value buffer) {
-        d.push_back(builder.create<memref::LoadOp>(loc, buffer, i));
+        d.push_back(memref::LoadOp::create(builder, loc, buffer, i));
       });
 
   // Start the inner for-stmt with induction variable j, for moving data[p..i)
   // to data[p+1..i+1).
-  Value imp = builder.create<arith::SubIOp>(loc, i, p);
+  Value imp = arith::SubIOp::create(builder, loc, i, p);
   Value c0 = constantIndex(builder, loc, 0);
-  scf::ForOp forOpJ = builder.create<scf::ForOp>(loc, c0, imp, c1);
+  scf::ForOp forOpJ = scf::ForOp::create(builder, loc, c0, imp, c1);
   builder.setInsertionPointToStart(forOpJ.getBody());
   Value j = forOpJ.getInductionVar();
-  Value imj = builder.create<arith::SubIOp>(loc, i, j);
+  Value imj = arith::SubIOp::create(builder, loc, i, j);
   operands[1] = imj;
-  operands[0] = builder.create<arith::SubIOp>(loc, imj, c1);
+  operands[0] = arith::SubIOp::create(builder, loc, imj, c1);
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t unused, Value imjm1, Value imj, Value buffer) {
-        Value t = builder.create<memref::LoadOp>(loc, buffer, imjm1);
-        builder.create<memref::StoreOp>(loc, t, buffer, imj);
+        Value t = memref::LoadOp::create(builder, loc, buffer, imjm1);
+        memref::StoreOp::create(builder, loc, t, buffer, imj);
       });
 
   // Store the value at data[i] to data[p].
@@ -1061,11 +1061,11 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t k, Value p, Value usused, Value buffer) {
-        builder.create<memref::StoreOp>(loc, d[k], buffer, p);
+        memref::StoreOp::create(builder, loc, d[k], buffer, p);
       });
 
   builder.setInsertionPointAfter(forOpI);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Creates a function to perform quick sort or a hybrid quick sort on the
@@ -1127,7 +1127,7 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   Value hi = args[hiIdx];
   SmallVector<Type, 2> types(2, lo.getType()); // Only two types.
   scf::WhileOp whileOp =
-      builder.create<scf::WhileOp>(loc, types, SmallVector<Value, 2>{lo, hi});
+      scf::WhileOp::create(builder, loc, types, SmallVector<Value, 2>{lo, hi});
 
   // The before-region of the WhileOp.
   Block *before =
@@ -1136,10 +1136,10 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   lo = before->getArgument(0);
   hi = before->getArgument(1);
   Value loP1 =
-      builder.create<arith::AddIOp>(loc, lo, constantIndex(builder, loc, 1));
+      arith::AddIOp::create(builder, loc, lo, constantIndex(builder, loc, 1));
   Value needSort =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, loP1, hi);
-  builder.create<scf::ConditionOp>(loc, needSort, before->getArguments());
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, loP1, hi);
+  scf::ConditionOp::create(builder, loc, needSort, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -1151,53 +1151,53 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   args[1] = hi;
 
   if (isHybrid) {
-    Value len = builder.create<arith::SubIOp>(loc, hi, lo);
+    Value len = arith::SubIOp::create(builder, loc, hi, lo);
     Value lenLimit = constantIndex(builder, loc, 30);
-    Value lenCond = builder.create<arith::CmpIOp>(
+    Value lenCond = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::ule, len, lenLimit);
     scf::IfOp lenIf =
-        builder.create<scf::IfOp>(loc, types, lenCond, /*else=*/true);
+        scf::IfOp::create(builder, loc, types, lenCond, /*else=*/true);
 
     // When len <= limit.
     builder.setInsertionPointToStart(&lenIf.getThenRegion().front());
     FlatSymbolRefAttr insertionSortFunc = getMangledSortHelperFunc(
         builder, func, TypeRange(), kSortStableFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createSortStableFunc);
-    builder.create<func::CallOp>(loc, insertionSortFunc, TypeRange(),
+    func::CallOp::create(builder, loc, insertionSortFunc, TypeRange(),
                                  ValueRange(args).drop_back(nTrailingP));
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When len > limit.
     builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
     Value depthLimit = args.back();
-    depthLimit = builder.create<arith::SubIOp>(loc, depthLimit,
+    depthLimit = arith::SubIOp::create(builder, loc, depthLimit,
                                                constantI64(builder, loc, 1));
     Value depthCond =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule,
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
                                       depthLimit, constantI64(builder, loc, 0));
     scf::IfOp depthIf =
-        builder.create<scf::IfOp>(loc, types, depthCond, /*else=*/true);
+        scf::IfOp::create(builder, loc, types, depthCond, /*else=*/true);
 
     // When depth exceeds limit.
     builder.setInsertionPointToStart(&depthIf.getThenRegion().front());
     FlatSymbolRefAttr heapSortFunc = getMangledSortHelperFunc(
         builder, func, TypeRange(), kHeapSortFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createHeapSortFunc);
-    builder.create<func::CallOp>(loc, heapSortFunc, TypeRange(),
+    func::CallOp::create(builder, loc, heapSortFunc, TypeRange(),
                                  ValueRange(args).drop_back(nTrailingP));
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When depth doesn't exceed limit.
     builder.setInsertionPointToStart(&depthIf.getElseRegion().front());
     args.back() = depthLimit;
     std::tie(lo, hi) =
         createQuickSort(builder, module, func, args, xPerm, ny, nTrailingP);
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
     builder.setInsertionPointAfter(depthIf);
     lo = depthIf.getResult(0);
     hi = depthIf.getResult(1);
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
     builder.setInsertionPointAfter(lenIf);
     lo = lenIf.getResult(0);
@@ -1208,11 +1208,11 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   }
 
   // New [lo, hi) for the next while-loop iteration.
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
   // After the while-loop.
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Implements the rewriting for operator sort and sort_coo.
@@ -1228,7 +1228,7 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     if (!mtp.isDynamicDim(0)) {
       auto newMtp =
           MemRefType::get({ShapedType::kDynamic}, mtp.getElementType());
-      v = rewriter.create<memref::CastOp>(loc, newMtp, v);
+      v = memref::CastOp::create(rewriter, loc, newMtp, v);
     }
     operands.push_back(v);
   }
@@ -1248,12 +1248,12 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     // As a heuristics, set depthLimit = 2 * log2(n).
     Value lo = operands[loIdx];
     Value hi = operands[hiIdx];
-    Value len = rewriter.create<arith::IndexCastOp>(
+    Value len = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(),
-        rewriter.create<arith::SubIOp>(loc, hi, lo));
-    Value depthLimit = rewriter.create<arith::SubIOp>(
+        arith::SubIOp::create(rewriter, loc, hi, lo));
+    Value depthLimit = arith::SubIOp::create(rewriter,
         loc, constantI64(rewriter, loc, 64),
-        rewriter.create<math::CountLeadingZerosOp>(loc, len));
+        math::CountLeadingZerosOp::create(rewriter, loc, len));
     operands.push_back(depthLimit);
     break;
   }
@@ -1307,33 +1307,33 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
     Location loc = op->getLoc();
     Value c0 = constantIndex(rewriter, loc, 0);
     Value buffer = op.getInBuffer();
-    Value capacity = rewriter.create<memref::DimOp>(loc, buffer, c0);
+    Value capacity = memref::DimOp::create(rewriter, loc, buffer, c0);
     Value size = op.getCurSize();
     Value value = op.getValue();
 
     Value n = op.getN() ? op.getN() : constantIndex(rewriter, loc, 1);
-    Value newSize = rewriter.create<arith::AddIOp>(loc, size, n);
+    Value newSize = arith::AddIOp::create(rewriter, loc, size, n);
     auto nValue = dyn_cast_or_null<arith::ConstantIndexOp>(n.getDefiningOp());
     bool nIsOne = (nValue && nValue.value() == 1);
 
     if (!op.getInbounds()) {
-      Value cond = rewriter.create<arith::CmpIOp>(
+      Value cond = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ugt, newSize, capacity);
 
       Value c2 = constantIndex(rewriter, loc, 2);
       auto bufferType =
           MemRefType::get({ShapedType::kDynamic}, value.getType());
-      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, bufferType, cond,
+      scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, bufferType, cond,
                                                   /*else=*/true);
       // True branch.
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       if (nIsOne) {
-        capacity = rewriter.create<arith::MulIOp>(loc, capacity, c2);
+        capacity = arith::MulIOp::create(rewriter, loc, capacity, c2);
       } else {
         // Use a do-while loop to calculate the new capacity as follows:
         //   do { new_capacity *= 2 } while (size > new_capacity)
         scf::WhileOp whileOp =
-            rewriter.create<scf::WhileOp>(loc, capacity.getType(), capacity);
+            scf::WhileOp::create(rewriter, loc, capacity.getType(), capacity);
 
         // The before-region of the WhileOp.
         Block *before = rewriter.createBlock(&whileOp.getBefore(), {},
@@ -1341,36 +1341,36 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
         rewriter.setInsertionPointToEnd(before);
 
         capacity =
-            rewriter.create<arith::MulIOp>(loc, before->getArgument(0), c2);
-        cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ugt,
+            arith::MulIOp::create(rewriter, loc, before->getArgument(0), c2);
+        cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ugt,
                                               newSize, capacity);
-        rewriter.create<scf::ConditionOp>(loc, cond, ValueRange{capacity});
+        scf::ConditionOp::create(rewriter, loc, cond, ValueRange{capacity});
         // The after-region of the WhileOp.
         Block *after = rewriter.createBlock(&whileOp.getAfter(), {},
                                             {capacity.getType()}, {loc});
         rewriter.setInsertionPointToEnd(after);
-        rewriter.create<scf::YieldOp>(loc, after->getArguments());
+        scf::YieldOp::create(rewriter, loc, after->getArguments());
 
         rewriter.setInsertionPointAfter(whileOp);
         capacity = whileOp.getResult(0);
       }
 
       Value newBuffer =
-          rewriter.create<memref::ReallocOp>(loc, bufferType, buffer, capacity);
+          memref::ReallocOp::create(rewriter, loc, bufferType, buffer, capacity);
       if (enableBufferInitialization) {
-        Value fillSize = rewriter.create<arith::SubIOp>(loc, capacity, newSize);
+        Value fillSize = arith::SubIOp::create(rewriter, loc, capacity, newSize);
         Value fillValue = constantZero(rewriter, loc, value.getType());
-        Value subBuffer = rewriter.create<memref::SubViewOp>(
+        Value subBuffer = memref::SubViewOp::create(rewriter,
             loc, newBuffer, /*offset=*/ValueRange{newSize},
             /*size=*/ValueRange{fillSize},
             /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
-        rewriter.create<linalg::FillOp>(loc, fillValue, subBuffer);
+        linalg::FillOp::create(rewriter, loc, fillValue, subBuffer);
       }
-      rewriter.create<scf::YieldOp>(loc, newBuffer);
+      scf::YieldOp::create(rewriter, loc, newBuffer);
 
       // False branch.
       rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      rewriter.create<scf::YieldOp>(loc, buffer);
+      scf::YieldOp::create(rewriter, loc, buffer);
 
       // Prepare for adding the value to the end of the buffer.
       rewriter.setInsertionPointAfter(ifOp);
@@ -1379,12 +1379,12 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
 
     // Add the value to the end of the buffer.
     if (nIsOne) {
-      rewriter.create<memref::StoreOp>(loc, value, buffer, size);
+      memref::StoreOp::create(rewriter, loc, value, buffer, size);
     } else {
-      Value subBuffer = rewriter.create<memref::SubViewOp>(
+      Value subBuffer = memref::SubViewOp::create(rewriter,
           loc, buffer, /*offset=*/ValueRange{size}, /*size=*/ValueRange{n},
           /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
-      rewriter.create<linalg::FillOp>(loc, value, subBuffer);
+      linalg::FillOp::create(rewriter, loc, value, subBuffer);
     }
 
     // Update the buffer size.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index e89b34d457ff8..001754b5531be 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -59,7 +59,7 @@ static gpu::GPUModuleOp genGPUModule(OpBuilder &builder, ModuleOp topModule) {
     return op; // existing
   markAsGPUContainer(topModule);
   builder.setInsertionPointToStart(topModule.getBody());
-  return builder.create<gpu::GPUModuleOp>(topModule->getLoc(),
+  return gpu::GPUModuleOp::create(builder, topModule->getLoc(),
                                           "sparse_kernels");
 }
 
@@ -81,7 +81,7 @@ static gpu::GPUFuncOp genGPUFunc(OpBuilder &builder, gpu::GPUModuleOp gpuModule,
     argsTp.push_back(arg.getType());
   FunctionType type = FunctionType::get(gpuModule->getContext(), argsTp, {});
   auto gpuFunc =
-      builder.create<gpu::GPUFuncOp>(gpuModule->getLoc(), kernelName, type);
+      gpu::GPUFuncOp::create(builder, gpuModule->getLoc(), kernelName, type);
   gpuFunc->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                    builder.getUnitAttr());
   return gpuFunc;
@@ -115,28 +115,28 @@ static Value genHostRegisterMemref(OpBuilder &builder, Location loc,
   MemRefType memTp = cast<MemRefType>(mem.getType());
   UnrankedMemRefType resTp =
       UnrankedMemRefType::get(memTp.getElementType(), /*memorySpace=*/0);
-  Value cast = builder.create<memref::CastOp>(loc, resTp, mem);
-  builder.create<gpu::HostRegisterOp>(loc, cast);
+  Value cast = memref::CastOp::create(builder, loc, resTp, mem);
+  gpu::HostRegisterOp::create(builder, loc, cast);
   return cast;
 }
 
 /// Unmaps the provided buffer, expecting the casted buffer.
 static void genHostUnregisterMemref(OpBuilder &builder, Location loc,
                                     Value cast) {
-  builder.create<gpu::HostUnregisterOp>(loc, cast);
+  gpu::HostUnregisterOp::create(builder, loc, cast);
 }
 
 /// Generates first wait in an asynchronous chain.
 static Value genFirstWait(OpBuilder &builder, Location loc) {
   Type tokenType = builder.getType<gpu::AsyncTokenType>();
-  return builder.create<gpu::WaitOp>(loc, tokenType, ValueRange())
+  return gpu::WaitOp::create(builder, loc, tokenType, ValueRange())
       .getAsyncToken();
 }
 
 /// Generates last, blocking wait in an asynchronous chain.
 static void genBlockingWait(OpBuilder &builder, Location loc,
                             ValueRange operands) {
-  builder.create<gpu::WaitOp>(loc, Type(), operands);
+  gpu::WaitOp::create(builder, loc, Type(), operands);
 }
 
 /// Allocates memory on the device.
@@ -156,7 +156,7 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
       dynamicSizes.push_back(dimOp);
     }
   }
-  return builder.create<gpu::AllocOp>(loc, TypeRange({memTp, token.getType()}),
+  return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
                                       token, dynamicSizes, ValueRange());
 }
 
@@ -164,14 +164,14 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
 static Value genHostBuffer(OpBuilder &builder, Location loc, Type type,
                            Value size) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
-  return builder.create<memref::AllocOp>(loc, memTp, size).getResult();
+  return memref::AllocOp::create(builder, loc, memTp, size).getResult();
 }
 
 // Allocates a typed buffer on the device with given size.
 static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Type type,
                                    Value size, Value token) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
-  return builder.create<gpu::AllocOp>(loc, TypeRange({memTp, token.getType()}),
+  return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
                                       token, size, ValueRange());
 }
 
@@ -184,14 +184,14 @@ static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Value size,
 /// Deallocates memory from the device.
 static Value genDeallocMemRef(OpBuilder &builder, Location loc, Value mem,
                               Value token) {
-  return builder.create<gpu::DeallocOp>(loc, token.getType(), token, mem)
+  return gpu::DeallocOp::create(builder, loc, token.getType(), token, mem)
       .getAsyncToken();
 }
 
 /// Copies memory between host and device (direction is implicit).
 static Value genCopyMemRef(OpBuilder &builder, Location loc, Value dst,
                            Value src, Value token) {
-  return builder.create<gpu::MemcpyOp>(loc, token.getType(), token, dst, src)
+  return gpu::MemcpyOp::create(builder, loc, token.getType(), token, dst, src)
       .getAsyncToken();
 }
 
@@ -212,7 +212,7 @@ static Value genTensorToMemref(PatternRewriter &rewriter, Location loc,
   auto tensorType = llvm::cast<ShapedType>(tensor.getType());
   auto memrefType =
       MemRefType::get(tensorType.getShape(), tensorType.getElementType());
-  return rewriter.create<bufferization::ToBufferOp>(loc, memrefType, tensor);
+  return bufferization::ToBufferOp::create(rewriter, loc, memrefType, tensor);
 }
 
 /// Prepares the outlined arguments, passing scalars and buffers in. Here we
@@ -293,13 +293,13 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
   // so that:
   //   row = blockIdx.x * blockDim.x + threadIdx.x
   //   inc = blockDim.x * gridDim.x
-  Value bid = rewriter.create<gpu::BlockIdOp>(loc, gpu::Dimension::x);
-  Value bsz = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::x);
-  Value tid = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
-  Value gsz = rewriter.create<gpu::GridDimOp>(loc, gpu::Dimension::x);
-  Value mul = rewriter.create<arith::MulIOp>(loc, bid, bsz);
-  Value row = rewriter.create<arith::AddIOp>(loc, mul, tid);
-  Value inc = rewriter.create<arith::MulIOp>(loc, bsz, gsz);
+  Value bid = gpu::BlockIdOp::create(rewriter, loc, gpu::Dimension::x);
+  Value bsz = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::x);
+  Value tid = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
+  Value gsz = gpu::GridDimOp::create(rewriter, loc, gpu::Dimension::x);
+  Value mul = arith::MulIOp::create(rewriter, loc, bid, bsz);
+  Value row = arith::AddIOp::create(rewriter, loc, mul, tid);
+  Value inc = arith::MulIOp::create(rewriter, loc, bsz, gsz);
 
   // Construct the iteration over the computational space that
   // accounts for the fact that the total number of threads and
@@ -308,7 +308,7 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
   //     <loop-body>
   //   }
   Value upper = irMap.lookup(forallOp.getUpperBound()[0]);
-  scf::ForOp forOp = rewriter.create<scf::ForOp>(loc, row, upper, inc);
+  scf::ForOp forOp = scf::ForOp::create(rewriter, loc, row, upper, inc);
   // The scf.for builder creates an empty block. scf.for does not allow multiple
   // blocks in its region, so delete the block before `cloneRegionBefore` adds
   // an additional block.
@@ -321,7 +321,7 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
 
   // Done.
   rewriter.setInsertionPointAfter(forOp);
-  rewriter.create<gpu::ReturnOp>(gpuFunc->getLoc());
+  gpu::ReturnOp::create(rewriter, gpuFunc->getLoc());
 }
 
 //===----------------------------------------------------------------------===//
@@ -496,11 +496,11 @@ static Value genFirstPosOrCrds(OpBuilder &builder, Location loc, Value a,
   if (format == CuSparseFormat::kCOO) {
     // Library uses SoA COO, direct IR uses AoS COO.
     if (enableRT)
-      return builder.create<ToCoordinatesOp>(loc, a, 0);
-    return builder.create<ToCoordinatesBufferOp>(loc, a);
+      return ToCoordinatesOp::create(builder, loc, a, 0);
+    return ToCoordinatesBufferOp::create(builder, loc, a);
   }
   // Formats CSR/CSC and BSR use positions at 1.
-  return builder.create<ToPositionsOp>(loc, a, 1);
+  return ToPositionsOp::create(builder, loc, a, 1);
 }
 
 /// Generates the second coordinates of a sparse matrix.
@@ -510,7 +510,7 @@ static Value genSecondCrds(OpBuilder &builder, Location loc, Value a,
   if (isCOO && !enableRT)
     return Value(); // nothing needed
   // Formats CSR/CSC and BSR use coordinates at 1.
-  return builder.create<ToCoordinatesOp>(loc, a, 1);
+  return ToCoordinatesOp::create(builder, loc, a, 1);
 }
 
 /// Generates the sparse matrix handle.
@@ -523,12 +523,12 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
     // Library uses SoA COO, direct IR uses AoS COO.
     if (enableRT) {
       assert(colA);
-      return builder.create<gpu::CreateCooOp>(loc, handleTp, tokenTp, token,
+      return gpu::CreateCooOp::create(builder, loc, handleTp, tokenTp, token,
                                               sz1, sz2, nseA, rowA, colA, valA);
     }
 #ifdef CUSPARSE_COO_AOS
     assert(!colA);
-    return builder.create<gpu::CreateCooAoSOp>(loc, handleTp, tokenTp, token,
+    return gpu::CreateCooAoSOp::create(builder, loc, handleTp, tokenTp, token,
                                                sz1, sz2, nseA, rowA, valA);
 #else
     llvm_unreachable("gpu::CreateCooAoSOp is deprecated");
@@ -536,10 +536,10 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   }
   assert(colA);
   if (format == CuSparseFormat::kCSR)
-    return builder.create<gpu::CreateCsrOp>(loc, handleTp, tokenTp, token, sz1,
+    return gpu::CreateCsrOp::create(builder, loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
   if (format == CuSparseFormat::kCSC)
-    return builder.create<gpu::CreateCscOp>(loc, handleTp, tokenTp, token, sz1,
+    return gpu::CreateCscOp::create(builder, loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
   // BSR requires a bit more work since we need to pass in the block size
   // and all others sizes in terms of blocks (#block-rows, #block-cols,
@@ -549,11 +549,11 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   assert(dims.size() == 2 && dims[0] == dims[1]);
   uint64_t b = dims[0];
   Value bSz = constantIndex(builder, loc, b);
-  Value bRows = builder.create<arith::DivUIOp>(loc, sz1, bSz);
-  Value bCols = builder.create<arith::DivUIOp>(loc, sz2, bSz);
-  Value bNum = builder.create<arith::DivUIOp>(
+  Value bRows = arith::DivUIOp::create(builder, loc, sz1, bSz);
+  Value bCols = arith::DivUIOp::create(builder, loc, sz2, bSz);
+  Value bNum = arith::DivUIOp::create(builder,
       loc, nseA, constantIndex(builder, loc, b * b));
-  return builder.create<gpu::CreateBsrOp>(loc, handleTp, tokenTp, token, bRows,
+  return gpu::CreateBsrOp::create(builder, loc, handleTp, tokenTp, token, bRows,
                                           bCols, bNum, bSz, bSz, rowA, colA,
                                           valA);
 }
@@ -579,12 +579,12 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   //   a : memR/memC/memV -> rowA,colA,valA
   //   x : memX           -> vecX
   //   y : memY           -> vecY
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
   Value szY = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szX = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value memR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, a, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, a);
+  Value memV = ToValuesOp::create(rewriter, loc, a);
   Value rowA = genAllocCopy(rewriter, loc, memR, tokens);
   Value colA = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valA = genAllocCopy(rewriter, loc, memV, tokens);
@@ -606,18 +606,18 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dvecX = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dvecX = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, vecX, szX);
   Value dnX = dvecX.getResult(0);
   token = dvecX.getAsyncToken();
-  auto dvecY = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dvecY = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, vecY, szY);
   Value dnY = dvecY.getResult(0);
   token = dvecY.getAsyncToken();
   auto dnYType = llvm::cast<ShapedType>(y.getType()).getElementType();
 
   // Precompute buffersize for SpMV.
-  auto bufferComp = rewriter.create<gpu::SpMVBufferSizeOp>(
+  auto bufferComp = gpu::SpMVBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, spMatA, dnX, dnY,
       /*computeType=*/dnYType);
   Value bufferSz = bufferComp.getResult(0);
@@ -627,16 +627,16 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SpMV.
-  auto spmvComp = rewriter.create<gpu::SpMVOp>(
+  auto spmvComp = gpu::SpMVOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnX, dnY, /*computeType=*/dnYType, buffer);
   token = spmvComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnX)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnX)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnY)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnY)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, rowA, token);
   if (colA)
@@ -676,13 +676,13 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   //   a : memR/memC/memV -> rowA,colA,valA
   //   b : bufB           -> matB
   //   c : bufC           -> matC
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
   Value memR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, a, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, a);
+  Value memV = ToValuesOp::create(rewriter, loc, a);
   Value rowA = genAllocCopy(rewriter, loc, memR, tokens);
   Value colA = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valA = genAllocCopy(rewriter, loc, memV, tokens);
@@ -704,12 +704,12 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matB,
       SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matC,
       SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
@@ -717,7 +717,7 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dmatCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SpMM.
-  auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
   Value bufferSz = bufferComp.getResult(0);
@@ -728,16 +728,16 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Perform the SpMM.
-  auto spmmComp = rewriter.create<gpu::SpMMOp>(
+  auto spmmComp = gpu::SpMMOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType, buffer);
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnC)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, rowA, token);
   if (colA)
@@ -778,17 +778,17 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   //   b : bmemR/bmemC/bmemV -> rowB,colB,valB
   //   c : materializes
   auto dnCType = cTp.getElementType();
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
-  Value nseB = rewriter.create<NumberOfEntriesOp>(loc, b);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
+  Value nseB = NumberOfEntriesOp::create(rewriter, loc, b);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
   Value amemR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value amemC = genSecondCrds(rewriter, loc, a, format, enableRT); // not empty
-  Value amemV = rewriter.create<ToValuesOp>(loc, a);
+  Value amemV = ToValuesOp::create(rewriter, loc, a);
   Value bmemR = genFirstPosOrCrds(rewriter, loc, b, format, enableRT);
   Value bmemC = genSecondCrds(rewriter, loc, b, format, enableRT); // not empty
-  Value bmemV = rewriter.create<ToValuesOp>(loc, b);
+  Value bmemV = ToValuesOp::create(rewriter, loc, b);
   Value rowA = genAllocCopy(rewriter, loc, amemR, tokens);
   Value colA = genAllocCopy(rewriter, loc, amemC, tokens);
   Value valA = genAllocCopy(rewriter, loc, amemV, tokens);
@@ -818,7 +818,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   // Sparse matrix C materializes (also assumes beta == 0).
   Value zero = constantIndex(rewriter, loc, 0);
   Value one = constantIndex(rewriter, loc, 1);
-  Value mplus1 = rewriter.create<arith::AddIOp>(loc, szm, one);
+  Value mplus1 = arith::AddIOp::create(rewriter, loc, szm, one);
   auto e1 = genAllocBuffer(rewriter, loc, cTp.getPosType(), mplus1, token);
   Value rowC = e1.getResult(0);
   token = e1.getAsyncToken();
@@ -836,10 +836,10 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
 
   // Precompute buffersizes for SpGEMM.
   Operation *descOp =
-      rewriter.create<gpu::SpGEMMCreateDescrOp>(loc, descTp, tokenTp, token);
+      gpu::SpGEMMCreateDescrOp::create(rewriter, loc, descTp, tokenTp, token);
   Value desc = descOp->getResult(0);
   token = descOp->getResult(1);
-  Operation *work1 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
       valC, gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
@@ -848,7 +848,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   auto buf1 = genAllocBuffer(rewriter, loc, bufferSz1, token);
   Value buffer1 = buf1.getResult(0);
   token = buf1.getAsyncToken();
-  Operation *work2 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
       bufferSz1, buffer1,
@@ -856,7 +856,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = work2->getResult(1);
 
   // Compute step.
-  Operation *compute1 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
       valC, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
@@ -865,14 +865,14 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   auto buf2 = genAllocBuffer(rewriter, loc, bufferSz2, token);
   Value buffer2 = buf2.getResult(0);
   token = buf2.getAsyncToken();
-  Operation *compute2 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
       bufferSz2, buffer2, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
   token = compute2->getResult(1);
 
   // Get sizes.
-  Operation *sizes = rewriter.create<gpu::SpMatGetSizeOp>(
+  Operation *sizes = gpu::SpMatGetSizeOp::create(rewriter,
       loc, indexTp, indexTp, indexTp, tokenTp, token, spMatC);
   Value nnz = sizes->getResult(2);
   token = sizes->getResult(3);
@@ -884,10 +884,10 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = a3.getAsyncToken();
 
   // Update C with new pointers and copy final product back into C.
-  Operation *update = rewriter.create<gpu::SetCsrPointersOp>(
+  Operation *update = gpu::SetCsrPointersOp::create(rewriter,
       loc, tokenTp, token, spMatC, rowC, colC, valC);
   token = update->getResult(0);
-  Operation *copy = rewriter.create<gpu::SpGEMMCopyOp>(
+  Operation *copy = gpu::SpGEMMCopyOp::create(rewriter,
       loc, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType);
   token = copy->getResult(0);
@@ -898,13 +898,13 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   Value valH = genHostBuffer(rewriter, loc, dnCType, nnz);
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::SpGEMMDestroyDescrOp>(loc, tokenTp, token, desc)
+  token = gpu::SpGEMMDestroyDescrOp::create(rewriter, loc, tokenTp, token, desc)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatB)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatC)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatC)
               .getAsyncToken();
   token = genCopyMemRef(rewriter, loc, rowH, rowC, token);
   token = genCopyMemRef(rewriter, loc, colH, colC, token);
@@ -925,11 +925,11 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  Value vt = rewriter.create<bufferization::ToTensorOp>(
+  Value vt = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
-  Value rt = rewriter.create<bufferization::ToTensorOp>(
+  Value rt = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
-  Value ct = rewriter.create<bufferization::ToTensorOp>(
+  Value ct = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
   rewriter.replaceOpWithNewOp<AssembleOp>(op, c.getType(), ValueRange{rt, ct},
                                           vt);
@@ -980,17 +980,17 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  Operation *spGenA = rewriter.create<gpu::Create2To4SpMatOp>(
+  Operation *spGenA = gpu::Create2To4SpMatOp::create(rewriter,
       loc, spMatHandleTp, tokenTp, token, szm, szk,
       gpu::Prune2To4SpMatFlag::PRUNE_AND_CHECK, matA);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matB,
       SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matC,
       SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
@@ -1000,7 +1000,7 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   // Precompute buffersize for SpMM.
   SmallVector<Type> bufferTypes_{indexTp, indexTp, indexTp};
   TypeRange bufferTypes(bufferTypes_);
-  auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
       loc, bufferTypes, tokenTp, token, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
@@ -1022,17 +1022,17 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
 
   // Perform the SpMM.
   auto dnCType = llvm::cast<ShapedType>(matC.getType()).getElementType();
-  auto spmmComp = rewriter.create<gpu::SpMMOp>(
+  auto spmmComp = gpu::SpMMOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType,
       SmallVector<Value>{buffer1, buffer2, buffer3});
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resources.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnC)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, buffer1, token);
   token = genDeallocMemRef(rewriter, loc, buffer2, token);
@@ -1073,7 +1073,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   //   a : bufA           -> matA
   //   b : bufB           -> matB
   //   c : memR/memC/memV -> rowC,colC,valC
-  Value nseC = rewriter.create<NumberOfEntriesOp>(loc, c);
+  Value nseC = NumberOfEntriesOp::create(rewriter, loc, c);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
@@ -1083,7 +1083,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Value matB = genAllocCopy(rewriter, loc, bufB, tokens);
   Value memR = genFirstPosOrCrds(rewriter, loc, c, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, c, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, c);
+  Value memV = ToValuesOp::create(rewriter, loc, c);
   Value rowC = genAllocCopy(rewriter, loc, memR, tokens);
   Value colC = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valC = genAllocCopy(rewriter, loc, memV, tokens);
@@ -1096,11 +1096,11 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  auto dmatA = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatA = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnMatHandleTp, tokenTp, token, matA, SmallVector<Value>{szm, szk});
   Value dnA = dmatA.getResult(0);
   token = dmatA.getAsyncToken();
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnMatHandleTp, tokenTp, token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
@@ -1112,7 +1112,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SDDMM.
-  auto bufferComp = rewriter.create<gpu::SDDMMBufferSizeOp>(
+  auto bufferComp = gpu::SDDMMBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, dnA, dnB, spMatC, dnCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
@@ -1121,16 +1121,16 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SDDMM.
-  auto sddmmComp = rewriter.create<gpu::SDDMMOp>(loc, tokenTp, token, dnA, dnB,
+  auto sddmmComp = gpu::SDDMMOp::create(rewriter, loc, tokenTp, token, dnA, dnB,
                                                  spMatC, dnCType, buffer);
   token = sddmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnA)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatC)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, buffer, token);
   token = genDeallocMemRef(rewriter, loc, matA, token);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
index 2f68008e68b5f..0986a0f2ea0f0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
@@ -67,11 +67,11 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
       op.getRegionDefinedSpace(newBlock->getParent()->getRegionNumber());
   for (unsigned i : caseBits.bits()) {
     SparseIterator *it = iters[i].get();
-    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                 it->getCrd(), loopCrd);
-    casePred = rewriter.create<arith::AndIOp>(loc, casePred, pred);
+    casePred = arith::AndIOp::create(rewriter, loc, casePred, pred);
   }
-  scf::IfOp ifOp = rewriter.create<scf::IfOp>(
+  scf::IfOp ifOp = scf::IfOp::create(rewriter,
       loc, ValueRange(userReduc).getTypes(), casePred, /*else=*/true);
   rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
@@ -103,7 +103,7 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
   ValueRange yields = spY.getResults();
   rewriter.eraseOp(spY);
   rewriter.setInsertionPointToEnd(&ifOp.getThenRegion().front());
-  rewriter.create<scf::YieldOp>(loc, yields);
+  scf::YieldOp::create(rewriter, loc, yields);
 
   // Generates remaining case recursively.
   rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -111,7 +111,7 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
                                           newBlocks.drop_front(),
                                           oldBlocks.drop_front(), userReduc);
   if (!res.empty())
-    rewriter.create<scf::YieldOp>(loc, res);
+    scf::YieldOp::create(rewriter, loc, res);
 
   rewriter.setInsertionPointAfter(ifOp);
   return ifOp.getResults();
@@ -127,7 +127,7 @@ static ValueRange genLoopWithIterator(
   if (it->iteratableByFor()) {
     auto [lo, hi] = it->genForCond(rewriter, loc);
     Value step = constantIndex(rewriter, loc, 1);
-    scf::ForOp forOp = rewriter.create<scf::ForOp>(
+    scf::ForOp forOp = scf::ForOp::create(rewriter,
         loc, lo, hi, step, reduc,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
           // Empty builder function to ensure that no terminator is created.
@@ -140,7 +140,7 @@ static ValueRange genLoopWithIterator(
                                            it, forOp.getRegionIterArgs());
 
       rewriter.setInsertionPointToEnd(forOp.getBody());
-      rewriter.create<scf::YieldOp>(loc, ret);
+      scf::YieldOp::create(rewriter, loc, ret);
     }
     return forOp.getResults();
   }
@@ -149,7 +149,7 @@ static ValueRange genLoopWithIterator(
   llvm::append_range(ivs, it->getCursor());
 
   TypeRange types = ValueRange(ivs).getTypes();
-  auto whileOp = rewriter.create<scf::WhileOp>(loc, types, ivs);
+  auto whileOp = scf::WhileOp::create(rewriter, loc, types, ivs);
   {
     OpBuilder::InsertionGuard guard(rewriter);
     // Generates loop conditions.
@@ -158,7 +158,7 @@ static ValueRange genLoopWithIterator(
     rewriter.setInsertionPointToStart(before);
     ValueRange bArgs = before->getArguments();
     auto [whileCond, remArgs] = it->genWhileCond(rewriter, loc, bArgs);
-    rewriter.create<scf::ConditionOp>(loc, whileCond, before->getArguments());
+    scf::ConditionOp::create(rewriter, loc, whileCond, before->getArguments());
 
     // Delegates loop body generation.
     Region &dstRegion = whileOp.getAfter();
@@ -175,7 +175,7 @@ static ValueRange genLoopWithIterator(
     SmallVector<Value> yields;
     llvm::append_range(yields, ret);
     llvm::append_range(yields, it->forward(rewriter, loc));
-    rewriter.create<scf::YieldOp>(loc, yields);
+    scf::YieldOp::create(rewriter, loc, yields);
   }
   return whileOp.getResults().drop_front(it->getCursor().size());
 }
@@ -212,7 +212,7 @@ class ExtractValOpConverter : public OpConversionPattern<ExtractValOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value pos = adaptor.getIterator().back();
-    Value valBuf = rewriter.create<ToValuesOp>(
+    Value valBuf = ToValuesOp::create(rewriter,
         loc, llvm::getSingleElement(adaptor.getTensor()));
     rewriter.replaceOpWithNewOp<memref::LoadOp>(op, valBuf, pos);
     return success();
@@ -385,12 +385,12 @@ class SparseCoIterateOpConverter : public OpConversionPattern<CoIterateOp> {
         SmallVector<Value> nextIterYields(res);
         // 2nd. foward the loop.
         for (SparseIterator *it : validIters) {
-          Value cmp = rewriter.create<arith::CmpIOp>(
+          Value cmp = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::eq, it->getCrd(), loopCrd);
           it->forwardIf(rewriter, loc, cmp);
           llvm::append_range(nextIterYields, it->getCursor());
         }
-        rewriter.create<scf::YieldOp>(loc, nextIterYields);
+        scf::YieldOp::create(rewriter, loc, nextIterYields);
 
         // Exit the loop, relink the iterator SSA value.
         rewriter.setInsertionPointAfter(loop);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
index 4f554756b3dd2..dc6bef5387b70 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
@@ -43,7 +43,7 @@ struct DemapInsRewriter : public OpRewritePattern<SourceOp> {
     SmallVector<Value> deMappedIns(op->getOperands());
     for (Value &in : deMappedIns) {
       if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) {
-        in = rewriter.create<ReinterpretMapOp>(loc, stt->getDemappedType(), in);
+        in = ReinterpretMapOp::create(rewriter, loc, stt->getDemappedType(), in);
         changed = true;
       }
     }
@@ -337,14 +337,14 @@ translateMap(linalg::GenericOp op, PatternRewriter &rewriter) {
 // Generates a "de"mapping reinterpretation of the map.
 static Value genDemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
-  return builder.create<ReinterpretMapOp>(val.getLoc(), enc.withoutDimToLvl(),
+  return ReinterpretMapOp::create(builder, val.getLoc(), enc.withoutDimToLvl(),
                                           val);
 }
 
 // Generates a "re"mapping reinterpretation of the map.
 static Value genRemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
-  return builder.create<ReinterpretMapOp>(val.getLoc(), enc, val);
+  return ReinterpretMapOp::create(builder, val.getLoc(), enc, val);
 }
 
 static SmallVector<Value> remapValueRange(OpBuilder &rewriter, TypeRange types,
@@ -353,7 +353,7 @@ static SmallVector<Value> remapValueRange(OpBuilder &rewriter, TypeRange types,
   assert(outs.size() == types.size());
   for (auto [r, t] : llvm::zip(ret, types))
     if (r.getType() != t)
-      r = rewriter.create<ReinterpretMapOp>(r.getLoc(), t, r);
+      r = ReinterpretMapOp::create(rewriter, r.getLoc(), t, r);
   return ret;
 }
 
@@ -566,7 +566,7 @@ struct GenericOpScheduler : public OpRewritePattern<linalg::GenericOp> {
       // Inserting the transpose
       rewriter.setInsertionPoint(linalgOp);
       RankedTensorType dstTp = stt.withDimToLvl(dimToLvl).getRankedTensorType();
-      Value dst = rewriter.create<ConvertOp>(tval.getLoc(), dstTp, tval);
+      Value dst = ConvertOp::create(rewriter, tval.getLoc(), dstTp, tval);
       rewriter.modifyOpInPlace(linalgOp, [&]() {
         linalgOp->setOperand(t->getOperandNumber(), dst);
       });
@@ -574,7 +574,7 @@ struct GenericOpScheduler : public OpRewritePattern<linalg::GenericOp> {
       // Release the transposed form afterwards.
       // TODO: CSE when used in more than one following op?
       rewriter.setInsertionPointAfter(linalgOp);
-      rewriter.create<bufferization::DeallocTensorOp>(dst.getLoc(), dst);
+      bufferization::DeallocTensorOp::create(rewriter, dst.getLoc(), dst);
 
       return success();
     }
@@ -604,7 +604,7 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     ValueRange dynSz = op.getDynamicSizes();
     for (int64_t dimSz : stt.getDimShape()) {
       if (ShapedType::isDynamic(dimSz)) {
-        Value maxCrd = rewriter.create<arith::SubIOp>(
+        Value maxCrd = arith::SubIOp::create(rewriter,
             loc, dynSz.front(), constantIndex(rewriter, loc, 1));
         maxDimCrds.push_back(maxCrd);
         dynSz = dynSz.drop_front();
@@ -619,7 +619,7 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     SmallVector<Value> dynLvlSzs;
     for (unsigned i = 0, e = lvlShape.size(); i < e; i++) {
       if (ShapedType::isDynamic(lvlShape[i])) {
-        Value sz = rewriter.create<arith::AddIOp>(
+        Value sz = arith::AddIOp::create(rewriter,
             loc, maxLvlCrds[i], constantIndex(rewriter, loc, 1));
         dynLvlSzs.push_back(sz);
       }
@@ -650,7 +650,7 @@ struct TensorInsertDemapper
     auto stt = getSparseTensorType(op.getResult());
     ValueRange lvlCrd = stt.translateCrds(rewriter, loc, op.getIndices(),
                                           CrdTransDirectionKind::dim2lvl);
-    auto insertOp = rewriter.create<tensor::InsertOp>(
+    auto insertOp = tensor::InsertOp::create(rewriter,
         loc, op.getScalar(), adaptor.getDest(), lvlCrd);
 
     Value out = genRemap(rewriter, stt.getEncoding(), insertOp.getResult());
@@ -765,7 +765,7 @@ struct ForeachOpDemapper
           stt && !stt->isIdentity()) {
         Value y =
             genDemap(rewriter, stt->getEncoding(), yield.getSingleResult());
-        rewriter.create<YieldOp>(loc, y);
+        YieldOp::create(rewriter, loc, y);
         rewriter.eraseOp(yield);
       }
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
index f85c4761a8d52..7b662c2fb0db0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
@@ -126,7 +126,7 @@ void collapseSparseSpace(MutableArrayRef<CollapseSpaceInfo> toCollapse) {
   OpBuilder builder(root);
 
   // Construct the collapsed iteration space.
-  auto collapsedSpace = builder.create<ExtractIterSpaceOp>(
+  auto collapsedSpace = ExtractIterSpaceOp::create(builder,
       loc, root.getTensor(), root.getParentIter(), root.getLoLvl(),
       leaf.getHiLvl());
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
index 01028f71c20bb..a1b9035f3b6e6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
@@ -69,13 +69,13 @@ class SpecifierStructBuilder : public StructBuilder {
   Value extractField(OpBuilder &builder, Location loc,
                      ArrayRef<int64_t> indices) const {
     return genCast(builder, loc,
-                   builder.create<LLVM::ExtractValueOp>(loc, value, indices),
+                   LLVM::ExtractValueOp::create(builder, loc, value, indices),
                    builder.getIndexType());
   }
 
   void insertField(OpBuilder &builder, Location loc, ArrayRef<int64_t> indices,
                    Value v) {
-    value = builder.create<LLVM::InsertValueOp>(
+    value = LLVM::InsertValueOp::create(builder,
         loc, value, genCast(builder, loc, v, builder.getIntegerType(64)),
         indices);
   }
@@ -110,7 +110,7 @@ class SpecifierStructBuilder : public StructBuilder {
 
 Value SpecifierStructBuilder::getInitValue(OpBuilder &builder, Location loc,
                                            Type structType, Value source) {
-  Value metaData = builder.create<LLVM::PoisonOp>(loc, structType);
+  Value metaData = LLVM::PoisonOp::create(builder, loc, structType);
   SpecifierStructBuilder md(metaData);
   if (!source) {
     auto memSizeArrayType =
@@ -204,14 +204,14 @@ void SpecifierStructBuilder::setMemSize(OpBuilder &builder, Location loc,
 /// Builds IR extracting the memory size array from the descriptor.
 Value SpecifierStructBuilder::memSizeArray(OpBuilder &builder,
                                            Location loc) const {
-  return builder.create<LLVM::ExtractValueOp>(loc, value,
+  return LLVM::ExtractValueOp::create(builder, loc, value,
                                               kMemSizePosInSpecifier);
 }
 
 /// Builds IR inserting the memory size array into the descriptor.
 void SpecifierStructBuilder::setMemSizeArray(OpBuilder &builder, Location loc,
                                              Value array) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, array,
+  value = LLVM::InsertValueOp::create(builder, loc, value, array,
                                               kMemSizePosInSpecifier);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 001ea62b07360..0450e1d6e8775 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -50,7 +50,7 @@ static SmallVector<Value> flattenValues(ArrayRef<ValueRange> values) {
 /// Generates a load with proper `index` typing.
 static Value genLoad(OpBuilder &builder, Location loc, Value mem, Value idx) {
   idx = genCast(builder, loc, idx, builder.getIndexType());
-  return builder.create<memref::LoadOp>(loc, mem, idx);
+  return memref::LoadOp::create(builder, loc, mem, idx);
 }
 
 /// Generates a store with proper `index` typing and proper value.
@@ -59,7 +59,7 @@ static void genStore(OpBuilder &builder, Location loc, Value val, Value mem,
   idx = genCast(builder, loc, idx, builder.getIndexType());
   val = genCast(builder, loc, val,
                 cast<ShapedType>(mem.getType()).getElementType());
-  builder.create<memref::StoreOp>(loc, val, mem, idx);
+  memref::StoreOp::create(builder, loc, val, mem, idx);
 }
 
 /// Creates a straightforward counting for-loop.
@@ -70,7 +70,7 @@ static scf::ForOp createFor(OpBuilder &builder, Location loc, Value upper,
   if (!lower)
     lower = constantZero(builder, loc, indexType);
   Value one = constantOne(builder, loc, indexType);
-  scf::ForOp forOp = builder.create<scf::ForOp>(loc, lower, upper, one, fields);
+  scf::ForOp forOp = scf::ForOp::create(builder, loc, lower, upper, one, fields);
   for (unsigned i = 0, e = fields.size(); i < e; i++)
     fields[i] = forOp.getRegionIterArg(i);
   builder.setInsertionPointToStart(forOp.getBody());
@@ -86,7 +86,7 @@ static void createPushback(OpBuilder &builder, Location loc,
   Value field = desc.getMemRefField(kind, lvl);
   StorageSpecifierKind specFieldKind = toSpecifierKind(kind);
 
-  auto pushBackOp = builder.create<PushBackOp>(
+  auto pushBackOp = PushBackOp::create(builder,
       loc, desc.getSpecifierField(builder, loc, specFieldKind, lvl), field,
       genCast(builder, loc, value, etp), repeat);
 
@@ -112,7 +112,7 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
       Value posZero = constantZero(builder, loc, stt.getPosType());
       if (isLooseCompressedLT(lt)) {
         Value two = constantIndex(builder, loc, 2);
-        linear = builder.create<arith::MulIOp>(loc, linear, two);
+        linear = arith::MulIOp::create(builder, loc, linear, two);
       }
       createPushback(builder, loc, desc, SparseTensorFieldKind::PosMemRef, lvl,
                      /*value=*/posZero, /*repeat=*/linear);
@@ -125,7 +125,7 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
     // otherwise the values array for the from-here "all-dense" case.
     assert(isDenseLT(lt));
     Value size = desc.getLvlSize(builder, loc, lvl);
-    linear = builder.create<arith::MulIOp>(loc, linear, size);
+    linear = arith::MulIOp::create(builder, loc, linear, size);
   }
   // Reached values array so prepare for an insertion.
   Value valZero = constantZero(builder, loc, stt.getElementType());
@@ -137,11 +137,11 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
 static Value createAllocation(OpBuilder &builder, Location loc,
                               MemRefType memRefType, Value sz,
                               bool enableInit) {
-  Value buffer = builder.create<memref::AllocOp>(loc, memRefType, sz);
+  Value buffer = memref::AllocOp::create(builder, loc, memRefType, sz);
   Type elemType = memRefType.getElementType();
   if (enableInit) {
     Value fillValue = constantZero(builder, loc, elemType);
-    builder.create<linalg::FillOp>(loc, fillValue, buffer);
+    linalg::FillOp::create(builder, loc, fillValue, buffer);
   }
   return buffer;
 }
@@ -179,14 +179,14 @@ static void createAllocFields(OpBuilder &builder, Location loc,
     valHeuristic = lvlSizesValues[0];
     for (Level lvl = 1; lvl < lvlRank; lvl++)
       valHeuristic =
-          builder.create<arith::MulIOp>(loc, valHeuristic, lvlSizesValues[lvl]);
+          arith::MulIOp::create(builder, loc, valHeuristic, lvlSizesValues[lvl]);
   } else if (sizeHint) {
     if (stt.getAoSCOOStart() == 0) {
       posHeuristic = constantIndex(builder, loc, 2);
-      crdHeuristic = builder.create<arith::MulIOp>(
+      crdHeuristic = arith::MulIOp::create(builder,
           loc, constantIndex(builder, loc, lvlRank), sizeHint); // AOS
     } else if (lvlRank == 2 && stt.isDenseLvl(0) && stt.isCompressedLvl(1)) {
-      posHeuristic = builder.create<arith::AddIOp>(
+      posHeuristic = arith::AddIOp::create(builder,
           loc, sizeHint, constantIndex(builder, loc, 1));
       crdHeuristic = sizeHint;
     } else {
@@ -280,7 +280,7 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   unsigned crdStride;
   std::tie(crdFidx, crdStride) = desc.getCrdMemRefIndexAndStride(lvl);
   const Value one = constantIndex(builder, loc, 1);
-  const Value pp1 = builder.create<arith::AddIOp>(loc, parentPos, one);
+  const Value pp1 = arith::AddIOp::create(builder, loc, parentPos, one);
   const Value positionsAtLvl = desc.getPosMemRef(lvl);
   const Value pstart = genLoad(builder, loc, positionsAtLvl, parentPos);
   const Value pstop = genLoad(builder, loc, positionsAtLvl, pp1);
@@ -288,29 +288,29 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   const Value crdStrideC =
       crdStride > 1 ? constantIndex(builder, loc, crdStride) : Value();
   const Value msz =
-      crdStrideC ? builder.create<arith::DivUIOp>(loc, crdMsz, crdStrideC)
+      crdStrideC ? arith::DivUIOp::create(builder, loc, crdMsz, crdStrideC)
                  : crdMsz;
-  const Value plast = builder.create<arith::SubIOp>(
+  const Value plast = arith::SubIOp::create(builder,
       loc, genCast(builder, loc, pstop, indexType), one);
   // Conditional expression.
-  Value lt = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value lt = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                            pstart, pstop);
   types.push_back(boolType);
-  scf::IfOp ifOp1 = builder.create<scf::IfOp>(loc, types, lt, /*else*/ true);
+  scf::IfOp ifOp1 = scf::IfOp::create(builder, loc, types, lt, /*else*/ true);
   types.pop_back();
   builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
   Value crd =
       genLoad(builder, loc, desc.getMemRefField(crdFidx),
-              crdStrideC ? builder.create<arith::MulIOp>(loc, plast, crdStrideC)
+              crdStrideC ? arith::MulIOp::create(builder, loc, plast, crdStrideC)
                          : plast);
-  Value eq = builder.create<arith::CmpIOp>(
+  Value eq = arith::CmpIOp::create(builder,
       loc, arith::CmpIPredicate::eq, genCast(builder, loc, crd, indexType),
       lvlCoords[lvl]);
-  builder.create<scf::YieldOp>(loc, eq);
+  scf::YieldOp::create(builder, loc, eq);
   builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
   if (lvl > 0)
     genStore(builder, loc, msz, positionsAtLvl, parentPos);
-  builder.create<scf::YieldOp>(loc, constantI1(builder, loc, false));
+  scf::YieldOp::create(builder, loc, constantI1(builder, loc, false));
   builder.setInsertionPointAfter(ifOp1);
   // If present construct. Note that for a non-unique dimension level, we
   // simply set the condition to false and rely on CSE/DCE to clean up the IR.
@@ -322,19 +322,19 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   types.push_back(indexType);
   const Value p = stt.isUniqueLvl(lvl) ? ifOp1.getResult(0)
                                        : constantI1(builder, loc, false);
-  scf::IfOp ifOp2 = builder.create<scf::IfOp>(loc, types, p, /*else*/ true);
+  scf::IfOp ifOp2 = scf::IfOp::create(builder, loc, types, p, /*else*/ true);
   // If present (fields unaffected, update pnext to plast).
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
 
   // FIXME: This does not looks like a clean way, but probably the most
   // efficient way.
   desc.getFields().push_back(plast);
-  builder.create<scf::YieldOp>(loc, desc.getFields());
+  scf::YieldOp::create(builder, loc, desc.getFields());
   desc.getFields().pop_back();
 
   // If !present (changes fields, update pnext).
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
-  Value mszp1 = builder.create<arith::AddIOp>(loc, msz, one);
+  Value mszp1 = arith::AddIOp::create(builder, loc, msz, one);
   genStore(builder, loc, mszp1, positionsAtLvl, pp1);
   createPushback(builder, loc, desc, SparseTensorFieldKind::CrdMemRef, lvl,
                  /*value=*/lvlCoords[lvl]);
@@ -343,7 +343,7 @@ static Value genCompressed(OpBuilder &builder, Location loc,
     allocSchemeForRank(builder, loc, desc, lvl + 1);
 
   desc.getFields().push_back(msz);
-  builder.create<scf::YieldOp>(loc, desc.getFields());
+  scf::YieldOp::create(builder, loc, desc.getFields());
   desc.getFields().pop_back();
 
   // Update fields and return next pos.
@@ -381,17 +381,17 @@ static void genEndInsert(OpBuilder &builder, Location loc,
         Value oldv = loop.getRegionIterArg(0);
         Value newv = genLoad(builder, loc, posMemRef, i);
         Value posZero = constantZero(builder, loc, posType);
-        Value cond = builder.create<arith::CmpIOp>(
+        Value cond = arith::CmpIOp::create(builder,
             loc, arith::CmpIPredicate::eq, newv, posZero);
-        scf::IfOp ifOp = builder.create<scf::IfOp>(loc, TypeRange(posType),
+        scf::IfOp ifOp = scf::IfOp::create(builder, loc, TypeRange(posType),
                                                    cond, /*else*/ true);
         builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         genStore(builder, loc, oldv, posMemRef, i);
-        builder.create<scf::YieldOp>(loc, oldv);
+        scf::YieldOp::create(builder, loc, oldv);
         builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-        builder.create<scf::YieldOp>(loc, newv);
+        scf::YieldOp::create(builder, loc, newv);
         builder.setInsertionPointAfter(ifOp);
-        builder.create<scf::YieldOp>(loc, ifOp.getResult(0));
+        scf::YieldOp::create(builder, loc, ifOp.getResult(0));
         builder.setInsertionPointAfter(loop);
       }
     } else {
@@ -484,7 +484,7 @@ class SparseInsertGenerator
         //   <insert @ positions[lvl] at next level lvl + 1>
         if (isLooseCompressedLT(lt)) {
           Value two = constantIndex(builder, loc, 2);
-          parentPos = builder.create<arith::MulIOp>(loc, parentPos, two);
+          parentPos = arith::MulIOp::create(builder, loc, parentPos, two);
         }
         parentPos =
             genCompressed(builder, loc, desc, coords, value, parentPos, lvl);
@@ -501,8 +501,8 @@ class SparseInsertGenerator
         //   positions[lvl] = size * positions[lvl-1] + coords[lvl]
         //   <insert @ positions[lvl] at next level lvl + 1>
         Value size = desc.getLvlSize(builder, loc, lvl);
-        Value mult = builder.create<arith::MulIOp>(loc, size, parentPos);
-        parentPos = builder.create<arith::AddIOp>(loc, mult, coords[lvl]);
+        Value mult = arith::MulIOp::create(builder, loc, size, parentPos);
+        parentPos = arith::AddIOp::create(builder, loc, mult, coords[lvl]);
       }
     }
     // Reached the actual value append/insert.
@@ -582,7 +582,7 @@ class SparseCallConverter : public OpConversionPattern<func::CallOp> {
       return failure();
 
     // (1) Generates new call with flattened return value.
-    auto newCall = rewriter.create<func::CallOp>(
+    auto newCall = func::CallOp::create(rewriter,
         loc, op.getCallee(), finalRetTy, flattenValues(adaptor.getOperands()));
     // (2) Gather sparse tensor returns.
     SmallVector<SmallVector<Value>> packedResultVals;
@@ -671,7 +671,7 @@ struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
 
     auto id = AffineMap::getMultiDimIdentityMap(srcStt.getLvlRank(), ctx);
 
-    rewriter.create<SortOp>(loc, nnz, crd, ValueRange{val}, id,
+    SortOp::create(rewriter, loc, nnz, crd, ValueRange{val}, id,
                             rewriter.getIndexAttr(0), op.getAlgorithm());
 
     // Since we do in-place sorting, the destinate tensor will have the same set
@@ -757,10 +757,10 @@ class SparseTensorAllocConverter
       // Memcpy on memref fields.
       for (auto field : desc.getMemRefFields()) {
         auto memrefTp = cast<MemRefType>(field.getType());
-        auto size = rewriter.create<memref::DimOp>(loc, field, 0);
+        auto size = memref::DimOp::create(rewriter, loc, field, 0);
         auto copied =
-            rewriter.create<memref::AllocOp>(loc, memrefTp, ValueRange{size});
-        rewriter.create<memref::CopyOp>(loc, field, copied);
+            memref::AllocOp::create(rewriter, loc, memrefTp, ValueRange{size});
+        memref::CopyOp::create(rewriter, loc, field, copied);
         fields.push_back(copied);
       }
       // Reuses specifier.
@@ -863,7 +863,7 @@ class SparseTensorDeallocConverter
           cast<RankedTensorType>(op.getTensor().getType()));
       for (auto input : desc.getMemRefFields())
         // Deallocate every buffer used to store the sparse tensor handler.
-        rewriter.create<memref::DeallocOp>(loc, input);
+        memref::DeallocOp::create(rewriter, loc, input);
     }
     rewriter.eraseOp(op);
     return success();
@@ -917,7 +917,7 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // Generate a memref for `sz` elements of type `t`.
     const auto genAlloc = [&](Type t) {
       const auto memTp = MemRefType::get({ShapedType::kDynamic}, t);
-      return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
+      return memref::AllocOp::create(rewriter, loc, memTp, ValueRange{sz});
     };
     // Allocate temporary buffers for values/filled-switch and added.
     // We do not use stack buffers for this, since the expanded size may
@@ -931,10 +931,10 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, eltType)},
         ValueRange{values});
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, boolType)},
         ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
@@ -965,7 +965,7 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     // If the innermost level is ordered, we need to sort the coordinates
     // in the "added" array prior to applying the compression.
     if (dstType.isOrderedLvl(dstType.getLvlRank() - 1))
-      rewriter.create<SortOp>(
+      SortOp::create(rewriter,
           loc, count, added, ValueRange{}, rewriter.getMultiDimIdentityMap(1),
           rewriter.getIndexAttr(0), SparseTensorSortKind::HybridQuickSort);
     // While performing the insertions, we also need to reset the elements
@@ -1000,15 +1000,15 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     SmallVector<Value> insertRet = insertGen.genCallOrInline(rewriter, loc);
     genStore(rewriter, loc, constantZero(rewriter, loc, eltType), values, crd);
     genStore(rewriter, loc, constantI1(rewriter, loc, false), filled, crd);
-    rewriter.create<scf::YieldOp>(loc, insertRet);
+    scf::YieldOp::create(rewriter, loc, insertRet);
 
     rewriter.setInsertionPointAfter(loop);
     // Deallocate the buffers on exit of the full loop nest.
     Operation *parent = getTop(op);
     rewriter.setInsertionPointAfter(parent);
-    rewriter.create<memref::DeallocOp>(loc, values);
-    rewriter.create<memref::DeallocOp>(loc, filled);
-    rewriter.create<memref::DeallocOp>(loc, added);
+    memref::DeallocOp::create(rewriter, loc, values);
+    memref::DeallocOp::create(rewriter, loc, filled);
+    memref::DeallocOp::create(rewriter, loc, added);
     // Replace operation with resulting memrefs.
     rewriter.replaceOpWithMultiple(op, {loop->getResults()});
     return success();
@@ -1192,7 +1192,7 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
             // would require a subViewOp to avoid overflow when copying
             // values.
             Value sz = linalg::createOrFoldDimOp(rewriter, loc, srcMem, 0);
-            auto dstMem = rewriter.create<memref::AllocOp>(
+            auto dstMem = memref::AllocOp::create(rewriter,
                 loc, cast<MemRefType>(fTp), sz);
             if (fTp != srcMem.getType()) {
               // Converts elements type.
@@ -1201,16 +1201,16 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
                   constantIndex(rewriter, loc, 1),
                   [srcMem, &dstMem](OpBuilder &builder, Location loc,
                                     ValueRange ivs) {
-                    Value v = builder.create<memref::LoadOp>(loc, srcMem, ivs);
+                    Value v = memref::LoadOp::create(builder, loc, srcMem, ivs);
                     Value casted = genCast(builder, loc, v,
                                            dstMem.getType().getElementType());
-                    builder.create<memref::StoreOp>(loc, casted, dstMem, ivs);
+                    memref::StoreOp::create(builder, loc, casted, dstMem, ivs);
                   });
             } else {
               // TODO: We can even reuse the same memref for the new tensor,
               // but that requires a `ref-counting` based memory management
               // for shared memrefs between multiple sparse tensors.
-              rewriter.create<memref::CopyOp>(loc, srcMem, dstMem);
+              memref::CopyOp::create(rewriter, loc, srcMem, dstMem);
             }
             fields.push_back(dstMem);
           }
@@ -1242,7 +1242,7 @@ class SparseExtractSliceConverter
     auto desc = getMutDescriptorFromTensorTuple(adaptor.getSource(), fields,
                                                 op.getSource().getType());
 
-    auto newSpec = rewriter.create<StorageSpecifierInitOp>(
+    auto newSpec = StorageSpecifierInitOp::create(rewriter,
         loc, StorageSpecifierType::get(ctx, dstEnc), desc.getSpecifier());
     desc.setSpecifier(newSpec);
 
@@ -1326,11 +1326,11 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
               // Flattens the buffer to batchLvlRank.
               auto reassoc = getReassociationForFlattening(
                   mem.getType(), stt.getBatchLvlRank());
-              mem = rewriter.create<memref::CastOp>(
+              mem = memref::CastOp::create(rewriter,
                   loc, fType,
-                  rewriter.create<memref::CollapseShapeOp>(loc, mem, reassoc));
+                  memref::CollapseShapeOp::create(rewriter, loc, mem, reassoc));
             } else {
-              mem = rewriter.create<memref::CastOp>(loc, fType, mem);
+              mem = memref::CastOp::create(rewriter, loc, fType, mem);
             }
             fields.push_back(mem);
           }
@@ -1362,8 +1362,8 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       LevelType lt = stt.getLvlType(lvl);
       // Simply forwards the position index when this is a dense level.
       if (lt.isa<LevelFormat::Dense>()) {
-        memSize = rewriter.create<arith::MulIOp>(loc, lvlSize, memSize);
-        posBack = rewriter.create<arith::SubIOp>(loc, memSize, c1);
+        memSize = arith::MulIOp::create(rewriter, loc, lvlSize, memSize);
+        posBack = arith::SubIOp::create(rewriter, loc, memSize, c1);
         continue;
       }
       if (lt.isa<LevelFormat::Batch>()) {
@@ -1376,12 +1376,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       if (isWithPosLT(lt)) {
         assert(isCompressedLT(lt) || isLooseCompressedLT(lt));
         if (isLooseCompressedLT(lt)) {
-          memSize = rewriter.create<arith::MulIOp>(loc, memSize, c2);
-          posBack = rewriter.create<arith::SubIOp>(loc, memSize, c1);
+          memSize = arith::MulIOp::create(rewriter, loc, memSize, c2);
+          posBack = arith::SubIOp::create(rewriter, loc, memSize, c1);
         } else {
           assert(isCompressedLT(lt));
           posBack = memSize;
-          memSize = rewriter.create<arith::AddIOp>(loc, memSize, c1);
+          memSize = arith::AddIOp::create(rewriter, loc, memSize, c1);
         }
         desc.setPosMemSize(rewriter, loc, lvl, memSize);
         // The last value in position array is the memory size for next level.
@@ -1391,12 +1391,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
                                    constantIndex(rewriter, loc, 0));
         batched.push_back(posBack);
         memSize = genIndexLoad(rewriter, loc, desc.getPosMemRef(lvl), batched);
-        posBack = rewriter.create<arith::SubIOp>(loc, posBack, c1);
+        posBack = arith::SubIOp::create(rewriter, loc, posBack, c1);
       }
       assert(isWithCrdLT(lt) && lvl <= trailCOOStart);
       // FIXME: This seems to be unnecessarily complex, can we simplify it?
       if (lvl == trailCOOStart) {
-        Value cooSz = rewriter.create<arith::MulIOp>(
+        Value cooSz = arith::MulIOp::create(rewriter,
             loc, memSize, constantIndex(rewriter, loc, trailCOORank));
         desc.setCrdMemSize(rewriter, loc, lvl, cooSz);
       } else {
@@ -1460,18 +1460,18 @@ struct SparseDisassembleOpConverter
       if (dst.getType().getRank() > stt.getBatchLvlRank() + 1) {
         auto reassoc =
             getReassociationForFlattening(dst.getType(), stt.getBatchLvlRank());
-        flatOut = rewriter.create<memref::CollapseShapeOp>(loc, dst, reassoc);
+        flatOut = memref::CollapseShapeOp::create(rewriter, loc, dst, reassoc);
       }
       Value dstMem = genSliceToSize(rewriter, loc, flatOut, sz);
       Value srcMem = genSliceToSize(rewriter, loc, src, sz);
-      rewriter.create<memref::CopyOp>(loc, srcMem, dstMem);
+      memref::CopyOp::create(rewriter, loc, srcMem, dstMem);
       return true;
     });
 
     // Converts MemRefs back to Tensors.
     SmallVector<Value> retValues = llvm::to_vector(
         llvm::map_range(retMem, [&rewriter, loc](Value v) -> Value {
-          return rewriter.create<bufferization::ToTensorOp>(
+          return bufferization::ToTensorOp::create(rewriter,
               loc, memref::getTensorTypeFromMemRefType(v.getType()), v);
         }));
     // Appends the actual memory length used in each buffer returned.
@@ -1549,13 +1549,13 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Level lvlRank = dstTp.getLvlRank();
     if (dstTp.isOrderedLvl(lvlRank - 1)) {
       Value kFalse = constantI1(rewriter, loc, false);
-      Value notSorted = rewriter.create<arith::CmpIOp>(
+      Value notSorted = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, isSorted, kFalse);
       scf::IfOp ifOp =
-          rewriter.create<scf::IfOp>(loc, notSorted, /*else*/ false);
+          scf::IfOp::create(rewriter, loc, notSorted, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       auto xPerm = rewriter.getMultiDimIdentityMap(lvlRank);
-      rewriter.create<SortOp>(loc, nse, xs, ValueRange{ys}, xPerm,
+      SortOp::create(rewriter, loc, nse, xs, ValueRange{ys}, xPerm,
                               rewriter.getIndexAttr(0),
                               SparseTensorSortKind::HybridQuickSort);
       rewriter.setInsertionPointAfter(ifOp);
@@ -1566,10 +1566,10 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Value posMemref0 = desc.getPosMemRef(0);
     const Type posTp = dstTp.getPosType();
     const Value posNse = genCast(rewriter, loc, nse, posTp);
-    rewriter.create<memref::StoreOp>(loc, posNse, posMemref0, c1);
+    memref::StoreOp::create(rewriter, loc, posNse, posMemref0, c1);
 
     // Update storage specifier.
-    Value coordinatesSize = rewriter.create<arith::MulIOp>(
+    Value coordinatesSize = arith::MulIOp::create(rewriter,
         loc, nse, constantIndex(rewriter, loc, lvlRank));
     desc.setSpecifierField(rewriter, loc, StorageSpecifierKind::CrdMemSize, 0,
                            coordinatesSize);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 50ccb43d432b6..54ea1843b457f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -137,7 +137,7 @@ static SmallVector<Value> getDimSizes(OpBuilder &builder, Location loc,
 /// this buffer must be explicitly deallocated by client.
 static Value genAlloc(RewriterBase &rewriter, Location loc, Value sz, Type tp) {
   auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
-  return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
+  return memref::AllocOp::create(rewriter, loc, memTp, ValueRange{sz});
 }
 
 /// Generates a temporary buffer for the level-types of the given encoding.
@@ -154,7 +154,7 @@ static Value genLvlTypesBuffer(OpBuilder &builder, Location loc,
 static Value extractBarePtrFromTensor(OpBuilder &builder, Location loc,
                                       Value tensor) {
   auto buf = genToMemref(builder, loc, tensor);
-  return builder.create<memref::ExtractAlignedPointerAsIndexOp>(loc, buf);
+  return memref::ExtractAlignedPointerAsIndexOp::create(builder, loc, buf);
 }
 
 /// Generates a temporary buffer for the level-types of the given encoding.
@@ -168,11 +168,11 @@ static Value genLvlPtrsBuffers(OpBuilder &builder, Location loc,
 
   // Passing in value buffer pointers.
   lvlBarePtrs.push_back(extractBarePtrFromTensor(builder, loc, valTensor));
-  Value idxPtr = builder.create<memref::ExtractAlignedPointerAsIndexOp>(
+  Value idxPtr = memref::ExtractAlignedPointerAsIndexOp::create(builder,
       loc, allocaBuffer(builder, loc, lvlBarePtrs));
   Value idxCast =
-      builder.create<arith::IndexCastOp>(loc, builder.getI64Type(), idxPtr);
-  return builder.create<LLVM::IntToPtrOp>(loc, getOpaquePointerType(builder),
+      arith::IndexCastOp::create(builder, loc, builder.getI64Type(), idxPtr);
+  return LLVM::IntToPtrOp::create(builder, loc, getOpaquePointerType(builder),
                                           idxCast);
 }
 
@@ -227,7 +227,7 @@ class NewCallParams final {
     assert(isInitialized() && "Must initialize before genNewCall");
     StringRef name = "newSparseTensor";
     params[kParamAction] = constantAction(builder, loc, action);
-    params[kParamPtr] = ptr ? ptr : builder.create<LLVM::ZeroOp>(loc, pTp);
+    params[kParamPtr] = ptr ? ptr : LLVM::ZeroOp::create(builder, loc, pTp);
     return createFuncCall(builder, loc, name, pTp, params, EmitCInterface::On)
         .getResult(0);
   }
@@ -539,7 +539,7 @@ class SparseTensorToCoordinatesConverter
     // Cast the MemRef type to the type expected by the users, though these
     // two types should be compatible at runtime.
     if (op.getType() != crds.getType())
-      crds = rewriter.create<memref::CastOp>(loc, op.getType(), crds);
+      crds = memref::CastOp::create(rewriter, loc, op.getType(), crds);
     rewriter.replaceOp(op, crds);
     return success();
   }
@@ -560,7 +560,7 @@ class SparseToCoordinatesBufferConverter
     // Cast the MemRef type to the type expected by the users, though these
     // two types should be compatible at runtime.
     if (op.getType() != crds.getType())
-      crds = rewriter.create<memref::CastOp>(loc, op.getType(), crds);
+      crds = memref::CastOp::create(rewriter, loc, op.getType(), crds);
     rewriter.replaceOp(op, crds);
     return success();
   }
@@ -652,7 +652,7 @@ class SparseTensorInsertConverter
       vref = genAllocaScalar(rewriter, loc, elemTp);
     }
     storeAll(rewriter, loc, lvlCoords, adaptor.getIndices());
-    rewriter.create<memref::StoreOp>(loc, adaptor.getScalar(), vref);
+    memref::StoreOp::create(rewriter, loc, adaptor.getScalar(), vref);
     SmallString<12> name{"lexInsert", primaryTypeFunctionSuffix(elemTp)};
     createFuncCall(rewriter, loc, name, {},
                    {adaptor.getDest(), lvlCoords, vref}, EmitCInterface::On);
@@ -690,10 +690,10 @@ class SparseTensorExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, eltType)},
         ValueRange{values});
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, boolType)},
         ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
@@ -733,9 +733,9 @@ class SparseTensorCompressConverter : public OpConversionPattern<CompressOp> {
     rewriter.replaceOp(op, adaptor.getTensor());
     // Deallocate the buffers on exit of the loop nest.
     rewriter.setInsertionPointAfter(parent);
-    rewriter.create<memref::DeallocOp>(loc, values);
-    rewriter.create<memref::DeallocOp>(loc, filled);
-    rewriter.create<memref::DeallocOp>(loc, added);
+    memref::DeallocOp::create(rewriter, loc, values);
+    memref::DeallocOp::create(rewriter, loc, filled);
+    memref::DeallocOp::create(rewriter, loc, added);
     return success();
   }
 };
@@ -837,21 +837,21 @@ class SparseTensorDisassembleConverter
                                       cooStartLvl + 1);
       auto crdLen = linalg::createOrFoldDimOp(rewriter, loc, crds0, 0);
       auto two = constantIndex(rewriter, loc, 2);
-      auto bufLen = rewriter.create<arith::MulIOp>(loc, crdLen, two);
+      auto bufLen = arith::MulIOp::create(rewriter, loc, crdLen, two);
       Type indexType = rewriter.getIndexType();
       auto zero = constantZero(rewriter, loc, indexType);
       auto one = constantOne(rewriter, loc, indexType);
-      scf::ForOp forOp = rewriter.create<scf::ForOp>(loc, zero, crdLen, one);
+      scf::ForOp forOp = scf::ForOp::create(rewriter, loc, zero, crdLen, one);
       auto idx = forOp.getInductionVar();
       rewriter.setInsertionPointToStart(forOp.getBody());
-      auto c0 = rewriter.create<memref::LoadOp>(loc, crds0, idx);
-      auto c1 = rewriter.create<memref::LoadOp>(loc, crds1, idx);
+      auto c0 = memref::LoadOp::create(rewriter, loc, crds0, idx);
+      auto c1 = memref::LoadOp::create(rewriter, loc, crds1, idx);
       SmallVector<Value> args;
       args.push_back(idx);
       args.push_back(zero);
-      rewriter.create<memref::StoreOp>(loc, c0, buf, args);
+      memref::StoreOp::create(rewriter, loc, c0, buf, args);
       args[1] = one;
-      rewriter.create<memref::StoreOp>(loc, c1, buf, args);
+      memref::StoreOp::create(rewriter, loc, c1, buf, args);
       rewriter.setInsertionPointAfter(forOp);
       auto bufLenTp = op.getLvlLens().getTypes()[retLen.size()];
       retVal.push_back(buf);
@@ -867,11 +867,11 @@ class SparseTensorDisassembleConverter
     // Converts MemRefs back to Tensors.
     assert(retVal.size() + retLen.size() == op.getNumResults());
     for (unsigned i = 0, sz = retVal.size(); i < sz; i++) {
-      auto tensor = rewriter.create<bufferization::ToTensorOp>(
+      auto tensor = bufferization::ToTensorOp::create(rewriter,
           loc, memref::getTensorTypeFromMemRefType(retVal[i].getType()),
           retVal[i]);
       retVal[i] =
-          rewriter.create<tensor::CastOp>(loc, op.getResultTypes()[i], tensor);
+          tensor::CastOp::create(rewriter, loc, op.getResultTypes()[i], tensor);
     }
 
     // Appends the actual memory length used in each buffer returned.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index d4a02bf7a70b6..ee963ab5d1ffb 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -127,7 +127,7 @@ static void sizesForTensor(OpBuilder &builder, SmallVectorImpl<Value> &sizes,
   for (const auto &d : enumerate(stp.getShape())) {
     Value dim;
     if (d.value() == ShapedType::kDynamic)
-      dim = builder.create<tensor::DimOp>(loc, tensor, d.index());
+      dim = tensor::DimOp::create(builder, loc, tensor, d.index());
     else
       dim = constantIndex(builder, loc, d.value());
     sizes.push_back(dim);
@@ -198,7 +198,7 @@ static void concatSizesFromInputs(OpBuilder &builder,
     for (const auto &src : srcs.drop_front()) {
       Value srcSz = linalg::createOrFoldDimOp(builder, loc, src, dim);
       // Sum up all the sizes.
-      sizes[dim] = builder.create<arith::AddIOp>(loc, sizes[dim], srcSz);
+      sizes[dim] = arith::AddIOp::create(builder, loc, sizes[dim], srcSz);
     }
   }
 }
@@ -405,7 +405,7 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     inputOps.push_back(op.getDpsInputOperand(1 - other)->get());
     fusedIndexMaps.push_back(fusedIndexMaps.back()); // mimic other
     // Fuse producer and consumer into a new generic op.
-    auto fusedOp = rewriter.create<GenericOp>(
+    auto fusedOp = GenericOp::create(rewriter,
         loc, op.getResult(0).getType(), inputOps, outputOps,
         rewriter.getAffineMapArrayAttr(fusedIndexMaps), prod.getIteratorTypes(),
         /*doc=*/nullptr, /*library_call=*/nullptr);
@@ -430,7 +430,7 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     mapper.map(consBlock.getArgument(other), fusedBlock->back().getResult(0));
     mapper.map(last, rewriter.clone(*sampler, mapper)->getResult(0));
     last = rewriter.clone(*acc, mapper)->getResult(0);
-    rewriter.create<linalg::YieldOp>(loc, last);
+    linalg::YieldOp::create(rewriter, loc, last);
     // Force initial value on merged allocation for dense outputs.
     // TODO: deal with non alloc tensor here one day
     if (!getSparseTensorEncoding(op.getResult(0).getType())) {
@@ -534,7 +534,7 @@ struct GenSemiRingSelect : public OpRewritePattern<GenericOp> {
       assert(t.getType() == f.getType());
       auto selTp = t.getType();
       auto c0 = constantZero(rewriter, loc, selTp);
-      auto binOp = rewriter.create<sparse_tensor::BinaryOp>(loc, selTp, t, f);
+      auto binOp = sparse_tensor::BinaryOp::create(rewriter, loc, selTp, t, f);
       // Initializes all the blocks.
       rewriter.createBlock(&binOp.getOverlapRegion(), {}, {selTp, selTp},
                            {t.getLoc(), f.getLoc()});
@@ -564,7 +564,7 @@ struct GenSemiRingSelect : public OpRewritePattern<GenericOp> {
           irMap.map(f, b->getArgument(1));
         }
         auto y = rewriter.clone(inst, irMap)->getResult(0);
-        rewriter.create<sparse_tensor::YieldOp>(loc, y);
+        sparse_tensor::YieldOp::create(rewriter, loc, y);
       }
 
       // We successfully rewrited a operation. We can not do replacement here
@@ -674,28 +674,28 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     // Identity.
     Location loc = op.getLoc();
     Value identity =
-        rewriter.create<tensor::ExtractOp>(loc, init->get(), ValueRange());
+        tensor::ExtractOp::create(rewriter, loc, init->get(), ValueRange());
     // Unary {
     //    present -> value
     //    absent  -> zero.
     // }
     Type rtp = s0.getType();
     rewriter.setInsertionPointToStart(&op.getRegion().front());
-    auto semiring = rewriter.create<sparse_tensor::UnaryOp>(loc, rtp, s0);
+    auto semiring = sparse_tensor::UnaryOp::create(rewriter, loc, rtp, s0);
     Block *present =
         rewriter.createBlock(&semiring.getPresentRegion(), {}, rtp, loc);
     rewriter.setInsertionPointToStart(&semiring.getPresentRegion().front());
-    rewriter.create<sparse_tensor::YieldOp>(loc, present->getArgument(0));
+    sparse_tensor::YieldOp::create(rewriter, loc, present->getArgument(0));
     rewriter.createBlock(&semiring.getAbsentRegion(), {}, {}, {});
     rewriter.setInsertionPointToStart(&semiring.getAbsentRegion().front());
     auto zero =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(rtp));
-    rewriter.create<sparse_tensor::YieldOp>(loc, zero);
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(rtp));
+    sparse_tensor::YieldOp::create(rewriter, loc, zero);
     rewriter.setInsertionPointAfter(semiring);
     // CustomReduce {
     //    x = x REDUC y, identity
     // }
-    auto custom = rewriter.create<sparse_tensor::ReduceOp>(
+    auto custom = sparse_tensor::ReduceOp::create(rewriter,
         loc, rtp, semiring.getResult(), s1, identity);
     Block *region =
         rewriter.createBlock(&custom.getRegion(), {}, {rtp, rtp}, {loc, loc});
@@ -704,7 +704,7 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     irMap.map(red->getOperand(0), region->getArgument(0));
     irMap.map(red->getOperand(1), region->getArgument(1));
     auto *cloned = rewriter.clone(*red, irMap);
-    rewriter.create<sparse_tensor::YieldOp>(loc, cloned->getResult(0));
+    sparse_tensor::YieldOp::create(rewriter, loc, cloned->getResult(0));
     rewriter.setInsertionPointAfter(custom);
     rewriter.replaceOp(red, custom.getResult());
     return success();
@@ -723,14 +723,14 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto tensor = op.getTensor();
     auto stt = getSparseTensorType(tensor);
     // Header with NSE.
-    auto nse = rewriter.create<NumberOfEntriesOp>(loc, tensor);
-    rewriter.create<vector::PrintOp>(
+    auto nse = NumberOfEntriesOp::create(rewriter, loc, tensor);
+    vector::PrintOp::create(rewriter,
         loc, rewriter.getStringAttr("---- Sparse Tensor ----\nnse = "));
-    rewriter.create<vector::PrintOp>(loc, nse);
+    vector::PrintOp::create(rewriter, loc, nse);
     // Print run-time contents for dim/lvl sizes.
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("dim = "));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("dim = "));
     printSizes(rewriter, loc, tensor, stt.getDimRank(), /*isDim=*/true);
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("lvl = "));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("lvl = "));
     printSizes(rewriter, loc, tensor, stt.getLvlRank(), /*isDim=*/false);
     // Use the "codegen" foreach loop construct to iterate over
     // all typical sparse tensor components for printing.
@@ -744,42 +744,42 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       }
       case SparseTensorFieldKind::PosMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("pos["));
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("pos["));
+        vector::PrintOp::create(rewriter,
             loc, lvl, vector::PrintPunctuation::NoPunctuation);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("] : "));
-        auto pos = rewriter.create<ToPositionsOp>(loc, tensor, l);
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
+        auto pos = ToPositionsOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, pos);
         break;
       }
       case SparseTensorFieldKind::CrdMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("crd["));
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("crd["));
+        vector::PrintOp::create(rewriter,
             loc, lvl, vector::PrintPunctuation::NoPunctuation);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("] : "));
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
         Value crd = nullptr;
         // For COO AoS storage, we want to print a single, linear view of
         // the full coordinate storage at this level. For any other storage,
         // we show the coordinate storage for every indivual level.
         if (stt.getAoSCOOStart() == l)
-          crd = rewriter.create<ToCoordinatesBufferOp>(loc, tensor);
+          crd = ToCoordinatesBufferOp::create(rewriter, loc, tensor);
         else
-          crd = rewriter.create<ToCoordinatesOp>(loc, tensor, l);
+          crd = ToCoordinatesOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, crd);
         break;
       }
       case SparseTensorFieldKind::ValMemRef: {
-        rewriter.create<vector::PrintOp>(loc,
+        vector::PrintOp::create(rewriter, loc,
                                          rewriter.getStringAttr("values : "));
-        auto val = rewriter.create<ToValuesOp>(loc, tensor);
+        auto val = ToValuesOp::create(rewriter, loc, tensor);
         printContents(rewriter, loc, val);
         break;
       }
       }
       return true;
     });
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("----\n"));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("----\n"));
     rewriter.eraseOp(op);
     return success();
   }
@@ -797,7 +797,7 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto shape = cast<ShapedType>(vec.getType()).getShape();
     SmallVector<Value> idxs;
     printContentsLevel(rewriter, loc, vec, 0, shape, idxs);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::NewLine);
   }
 
   // Helper to the helper.
@@ -805,13 +805,13 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
                                  Value vec, unsigned i, ArrayRef<int64_t> shape,
                                  SmallVectorImpl<Value> &idxs) {
     // Open bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
     // Generate for loop.
     auto zero = constantIndex(rewriter, loc, 0);
     auto index = constantIndex(rewriter, loc, i);
-    auto size = rewriter.create<memref::DimOp>(loc, vec, index);
+    auto size = memref::DimOp::create(rewriter, loc, vec, index);
     auto step = constantIndex(rewriter, loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, zero, size, step);
+    auto forOp = scf::ForOp::create(rewriter, loc, zero, size, step);
     idxs.push_back(forOp.getInductionVar());
     rewriter.setInsertionPointToStart(forOp.getBody());
     if (i < shape.size() - 1) {
@@ -819,56 +819,56 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       printContentsLevel(rewriter, loc, vec, i + 1, shape, idxs);
     } else {
       // Actual contents printing.
-      auto val = rewriter.create<memref::LoadOp>(loc, vec, idxs);
+      auto val = memref::LoadOp::create(rewriter, loc, vec, idxs);
       if (llvm::isa<ComplexType>(val.getType())) {
         // Since the vector dialect does not support complex types in any op,
         // we split those into (real, imag) pairs here.
-        Value real = rewriter.create<complex::ReOp>(loc, val);
-        Value imag = rewriter.create<complex::ImOp>(loc, val);
-        rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
-        rewriter.create<vector::PrintOp>(loc, real,
+        Value real = complex::ReOp::create(rewriter, loc, val);
+        Value imag = complex::ImOp::create(rewriter, loc, val);
+        vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
+        vector::PrintOp::create(rewriter, loc, real,
                                          vector::PrintPunctuation::Comma);
-        rewriter.create<vector::PrintOp>(loc, imag,
+        vector::PrintOp::create(rewriter, loc, imag,
                                          vector::PrintPunctuation::Close);
       } else {
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter,
             loc, val, vector::PrintPunctuation::NoPunctuation);
       }
       // Terminating comma (except at end).
-      auto bound = rewriter.create<arith::AddIOp>(loc, idxs.back(), step);
-      Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+      auto bound = arith::AddIOp::create(rewriter, loc, idxs.back(), step);
+      Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
                                                   bound, size);
-      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, cond, /*else*/ false);
+      scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, cond, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Comma);
+      vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Comma);
     }
     idxs.pop_back();
     rewriter.setInsertionPointAfter(forOp);
     // Close bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
   }
 
   // Helper method to print run-time lvl/dim sizes.
   static void printSizes(PatternRewriter &rewriter, Location loc, Value tensor,
                          unsigned size, bool isDim) {
     // Open bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
     // Print unrolled contents (dimop requires constant value).
     for (unsigned i = 0; i < size; i++) {
       auto idx = constantIndex(rewriter, loc, i);
       Value val;
       if (isDim)
-        val = rewriter.create<tensor::DimOp>(loc, tensor, idx);
+        val = tensor::DimOp::create(rewriter, loc, tensor, idx);
       else
-        val = rewriter.create<LvlOp>(loc, tensor, idx);
-      rewriter.create<vector::PrintOp>(
+        val = LvlOp::create(rewriter, loc, tensor, idx);
+      vector::PrintOp::create(rewriter,
           loc, val,
           i != size - 1 ? vector::PrintPunctuation::Comma
                         : vector::PrintPunctuation::NoPunctuation);
     }
     // Close bracket and end of line.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::NewLine);
   }
 };
 
@@ -896,7 +896,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     for (Dimension d : dstTp->getDimShape())
       dstSizes.push_back(constantIndex(rewriter, loc, d));
 
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, srcTensor);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, srcTensor);
     // Only need an unordered COO buffer if input and output are not sorted
     // in the same way.
     Type bufferTp = getBufferType(
@@ -920,7 +920,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp->getEncoding();
-    ForeachOp foreachOp = rewriter.create<ForeachOp>(
+    ForeachOp foreachOp = ForeachOp::create(rewriter,
         loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
@@ -935,7 +935,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
           Value collapseSize = constantIndex(builder, loc, 1);
           for (Dimension d = 0; d < srcRank; d++)
             collapseSize =
-                builder.create<arith::MulIOp>(loc, collapseSize, srcSizes[d]);
+                arith::MulIOp::create(builder, loc, collapseSize, srcSizes[d]);
           SmallVector<Value, 1> collapsedSizes = {collapseSize};
 
           ReassociationIndices collapseIdx;
@@ -955,15 +955,15 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
                      dstSizes, dstDcvs);
 
           auto t =
-              builder.create<tensor::InsertOp>(loc, v, reduc.front(), dstDcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+              tensor::InsertOp::create(builder, loc, v, reduc.front(), dstDcvs);
+          sparse_tensor::YieldOp::create(builder, loc, t);
         });
 
-    Value t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
+    Value t = LoadOp::create(rewriter, loc, foreachOp.getResult(0), true);
     if (bufferTp != *dstTp) {
       auto dstRTT = dstTp->getRankedTensorType();
-      Value converted = rewriter.create<ConvertOp>(loc, dstRTT, t).getResult();
-      rewriter.create<DeallocTensorOp>(loc, t);
+      Value converted = ConvertOp::create(rewriter, loc, dstRTT, t).getResult();
+      DeallocTensorOp::create(rewriter, loc, t);
       t = converted;
     }
     rewriter.replaceOp(op, t);
@@ -1004,7 +1004,7 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           dstDynSizes.push_back(dstSizes[idx]);
       }
     }
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, srcTensor);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, srcTensor);
     // Only need a unordered COO buffer if input and output are not sorted
     // in the same way.
     Type bufferTp = getBufferType(
@@ -1025,7 +1025,7 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp.getEncoding();
-    ForeachOp foreachOp = rewriter.create<ForeachOp>(
+    ForeachOp foreachOp = ForeachOp::create(rewriter,
         loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
@@ -1040,15 +1040,15 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           reshapeCvs(builder, loc, op.getReassociationIndices(), srcSizes,
                      srcDcvs, dstSizes, dstDcvs);
           auto t =
-              builder.create<tensor::InsertOp>(loc, v, reduc.front(), dstDcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+              tensor::InsertOp::create(builder, loc, v, reduc.front(), dstDcvs);
+          sparse_tensor::YieldOp::create(builder, loc, t);
         });
 
-    Value t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
+    Value t = LoadOp::create(rewriter, loc, foreachOp.getResult(0), true);
     if (bufferTp != dstTp) {
       auto dstRTT = dstTp.getRankedTensorType();
-      Value converted = rewriter.create<ConvertOp>(loc, dstRTT, t).getResult();
-      rewriter.create<DeallocTensorOp>(loc, t);
+      Value converted = ConvertOp::create(rewriter, loc, dstRTT, t).getResult();
+      DeallocTensorOp::create(rewriter, loc, t);
       t = converted;
     }
     rewriter.replaceOp(op, t);
@@ -1079,7 +1079,7 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
       auto rtp = getRankedTensorType(op.getSrc());
       auto denseTp =
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
-      auto convert = rewriter.create<ConvertOp>(loc, denseTp, op.getSrc());
+      auto convert = ConvertOp::create(rewriter, loc, denseTp, op.getSrc());
       rewriter.modifyOpInPlace(op, [&]() { op->setOperand(0, convert); });
       return success();
     }
@@ -1089,14 +1089,14 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
       ReshapeOp reshape;
       if constexpr (std::is_same<ReshapeOp, tensor::ExpandShapeOp>::value) {
-        reshape = rewriter.create<ReshapeOp>(
+        reshape = ReshapeOp::create(rewriter,
             loc, denseTp, op.getSrc(), op.getReassociation(),
             op.getOutputShape(), op.getStaticOutputShape());
       } else {
-        reshape = rewriter.create<ReshapeOp>(loc, denseTp, op.getSrc(),
+        reshape = ReshapeOp::create(rewriter, loc, denseTp, op.getSrc(),
                                              op.getReassociation());
       }
-      Value convert = rewriter.create<ConvertOp>(loc, rtp, reshape);
+      Value convert = ConvertOp::create(rewriter, loc, rtp, reshape);
       rewriter.replaceOp(op, convert);
       return success();
     }
@@ -1112,20 +1112,20 @@ struct TensorLike {
     SmallVector<Value> dynSzs;
     getDynamicSizes(rtt, sizes, dynSzs);
 
-    val = builder.create<AllocTensorOp>(loc, rtt, dynSzs);
+    val = AllocTensorOp::create(builder, loc, rtt, dynSzs);
     if (!isSparse()) {
       Value c0 = constantZero(builder, loc, rtt.getElementType());
-      val = builder.create<linalg::FillOp>(loc, c0, val).getResult(0);
+      val = linalg::FillOp::create(builder, loc, c0, val).getResult(0);
     }
   }
 
   void insert(OpBuilder &builder, Location loc, Value v, ValueRange crds) {
-    val = builder.create<tensor::InsertOp>(loc, v, val, crds);
+    val = tensor::InsertOp::create(builder, loc, v, val, crds);
   }
 
   Value finalize(OpBuilder &builder, Location loc, RankedTensorType rtp) const {
     if (isSparse())
-      return builder.create<LoadOp>(loc, val, true);
+      return LoadOp::create(builder, loc, val, true);
     return val;
   }
 
@@ -1160,18 +1160,18 @@ struct SparseTensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
     Location loc = op.getLoc();
     SmallVector<Value> maxLvlCrds;
     for (Level l = 0; l < stt->getLvlRank(); l++) {
-      Value lvlSz = rewriter.create<LvlOp>(loc, op.getSource(), l);
-      Value maxLvlCrd = rewriter.create<arith::SubIOp>(
+      Value lvlSz = LvlOp::create(rewriter, loc, op.getSource(), l);
+      Value maxLvlCrd = arith::SubIOp::create(rewriter,
           loc, lvlSz, constantOne(rewriter, loc, rewriter.getIndexType()));
       maxLvlCrds.push_back(maxLvlCrd);
     }
 
     AffineExpr lvl2DimExp = stt->getLvlToDim().getResult(*dim);
-    Value maxDimCrd = rewriter.create<affine::AffineApplyOp>(
+    Value maxDimCrd = affine::AffineApplyOp::create(rewriter,
         op.getLoc(), AffineMap::get(stt->getLvlRank(), 0, lvl2DimExp),
         maxLvlCrds);
 
-    Value dimSz = rewriter.create<arith::AddIOp>(
+    Value dimSz = arith::AddIOp::create(rewriter,
         loc, maxDimCrd, constantOne(rewriter, loc, rewriter.getIndexType()));
     rewriter.replaceOp(op, dimSz);
     return success();
@@ -1212,26 +1212,26 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
     for (Value input : op.getInputs()) {
       // Builds a for op for each input tensor to append new values into the
       // output tensor.
-      foreachOp = rewriter.create<ForeachOp>(
+      foreachOp = ForeachOp::create(rewriter,
           loc, input, iterArg,
           [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
               ValueRange reduc) {
             SmallVector<Value> offDimCrd(dcvs);
             offDimCrd[conDim] =
-                builder.create<arith::AddIOp>(loc, offDimCrd[conDim], offset);
+                arith::AddIOp::create(builder, loc, offDimCrd[conDim], offset);
 
             // Enters foreach, updates the SSA chain.
             dstBuf.val = reduc.front();
             if (!dstTp.isAllDense()) {
               Value cond = genIsNonzero(builder, loc, v);
-              auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+              auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
                                                     /*else*/ true);
               builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-              builder.create<scf::YieldOp>(loc, dstBuf.val);
+              scf::YieldOp::create(builder, loc, dstBuf.val);
 
               builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
               dstBuf.insert(builder, loc, v, offDimCrd);
-              builder.create<scf::YieldOp>(loc, dstBuf.val);
+              scf::YieldOp::create(builder, loc, dstBuf.val);
 
               // Exits the ifOp, update the sparse tensor SSA value.
               builder.setInsertionPointAfter(ifOp);
@@ -1239,14 +1239,14 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
             } else {
               dstBuf.insert(builder, loc, v, offDimCrd);
             }
-            builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
+            sparse_tensor::YieldOp::create(builder, loc, dstBuf.val);
           });
       // Accumulates the offset. Note that only static-shaped inputs are allowed
       // by concatenate op verifier, which saves us from computing the offset
       // dynamically.
       const Size sz = getSparseTensorType(input).getDynamicDimSize(conDim);
       assert(ShapedType::isStatic(sz));
-      offset = rewriter.create<arith::AddIOp>(loc, offset,
+      offset = arith::AddIOp::create(rewriter, loc, offset,
                                               constantIndex(rewriter, loc, sz));
       iterArg = foreachOp.getResult(0);
       dstBuf.val = iterArg;
@@ -1299,7 +1299,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
     ValueRange vs;
     TensorLike dstBuf(rewriter, loc, dstStt.getRankedTensorType(), sizes);
 
-    auto foreachOp = rewriter.create<ForeachOp>(
+    auto foreachOp = ForeachOp::create(rewriter,
         loc, src, dstBuf.val, foreachOrder,
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
@@ -1307,14 +1307,14 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           dstBuf.val = reduc.front();
           if (!skipZeroCheck) {
             Value cond = genIsNonzero(builder, loc, v);
-            auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+            auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
                                                   /*else*/ true);
             builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-            builder.create<scf::YieldOp>(loc, dstBuf.val);
+            scf::YieldOp::create(builder, loc, dstBuf.val);
 
             builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
             dstBuf.insert(builder, loc, v, dcvs);
-            builder.create<scf::YieldOp>(loc, dstBuf.val);
+            scf::YieldOp::create(builder, loc, dstBuf.val);
 
             // Exits the ifOp, update the sparse tensor SSA value.
             builder.setInsertionPointAfter(ifOp);
@@ -1322,7 +1322,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           } else {
             dstBuf.insert(builder, loc, v, dcvs);
           }
-          builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
+          sparse_tensor::YieldOp::create(builder, loc, dstBuf.val);
         });
 
     rewriter.setInsertionPointAfter(foreachOp);
@@ -1349,7 +1349,7 @@ struct CrdTranslateRewriter : public OpRewritePattern<CrdTranslateOp> {
       // TODO: we should probably expand the affine map to IR using our own
       // rules, since affine.apply assume signed value, while the cooridinates
       // we provided must always be signless.
-      Value trans = rewriter.create<affine::AffineApplyOp>(
+      Value trans = affine::AffineApplyOp::create(rewriter,
           op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
           op.getInCrds());
       outCrds.push_back(trans);
@@ -1412,8 +1412,8 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
     SmallVector<Value> pos = loopEmitter.getValPosits(0);
     // Loads the value from sparse tensor using position-index;
     // loads the value from dense tensor using coords.
-    Value val = enc ? rewriter.create<memref::LoadOp>(loc, vals, pos)
-                    : rewriter.create<memref::LoadOp>(loc, vals, lcvs);
+    Value val = enc ? memref::LoadOp::create(rewriter, loc, vals, pos)
+                    : memref::LoadOp::create(rewriter, loc, vals, lcvs);
 
     // 2. Inline the block in the foreach operator.
     Block *srcBlock = op.getBody();
@@ -1472,22 +1472,22 @@ struct NewRewriter : public OpRewritePattern<NewOp> {
     // with enveloping reinterpreted_map ops for non-permutations.
     RankedTensorType dstTp = stt.getRankedTensorType();
     RankedTensorType cooTp = stt.getCOOType(/*ordered=*/true);
-    Value cooTensor = rewriter.create<NewOp>(loc, cooTp, op.getSource());
+    Value cooTensor = NewOp::create(rewriter, loc, cooTp, op.getSource());
     Value convert = cooTensor;
     auto enc = stt.getEncoding();
     if (!stt.isPermutation()) { // demap coo, demap dstTp
       auto coo = getSparseTensorType(cooTensor).getEncoding().withoutDimToLvl();
-      convert = rewriter.create<ReinterpretMapOp>(loc, coo, convert);
+      convert = ReinterpretMapOp::create(rewriter, loc, coo, convert);
       dstTp = getSparseTensorType(convert).withEncoding(enc.withoutDimToLvl());
     }
-    convert = rewriter.create<ConvertOp>(loc, dstTp, convert);
+    convert = ConvertOp::create(rewriter, loc, dstTp, convert);
     if (!stt.isPermutation()) // remap to original enc
-      convert = rewriter.create<ReinterpretMapOp>(loc, enc, convert);
+      convert = ReinterpretMapOp::create(rewriter, loc, enc, convert);
     rewriter.replaceOp(op, convert);
 
     // Release the temporary ordered COO tensor.
     rewriter.setInsertionPointAfterValue(convert);
-    rewriter.create<DeallocTensorOp>(loc, cooTensor);
+    DeallocTensorOp::create(rewriter, loc, cooTensor);
 
     return success();
   }
@@ -1501,7 +1501,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     Location loc = op.getLoc();
     // Calculate NNZ.
     Value src = op.getTensor();
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, src);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, src);
 
     // Allocate a temporary buffer for storing dimension-sizes/coordinates.
     const auto srcTp = getSparseTensorType(src);
@@ -1514,7 +1514,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     SmallVector<Value> dims;
     sizesForTensor(rewriter, dims, loc, srcTp, src);
     for (Dimension d = 0; d < dimRank; d++) {
-      rewriter.create<memref::StoreOp>(loc, dims[d], dimSizes,
+      memref::StoreOp::create(rewriter, loc, dims[d], dimSizes,
                                        constantIndex(rewriter, loc, d));
     }
 
@@ -1536,20 +1536,20 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     ModuleOp module = op->getParentOfType<ModuleOp>();
 
     // For each element in the source tensor, output the element.
-    rewriter.create<ForeachOp>(
+    ForeachOp::create(rewriter,
         loc, src, ValueRange(),
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
           for (Dimension d = 0; d < dimRank; d++) {
-            rewriter.create<memref::StoreOp>(loc, dcvs[d], dimCoords,
+            memref::StoreOp::create(rewriter, loc, dcvs[d], dimCoords,
                                              constantIndex(builder, loc, d));
           }
-          rewriter.create<memref::StoreOp>(loc, v, value);
+          memref::StoreOp::create(rewriter, loc, v, value);
           SmallVector<Value> operands{writer, rankValue, dimCoords, value};
           FlatSymbolRefAttr fn = getFunc(module, outNextFuncName, {}, operands,
                                          EmitCInterface::On);
-          builder.create<func::CallOp>(loc, TypeRange(), fn, operands);
-          builder.create<sparse_tensor::YieldOp>(loc);
+          func::CallOp::create(builder, loc, TypeRange(), fn, operands);
+          sparse_tensor::YieldOp::create(builder, loc);
         });
 
     // Release the writer.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 52b66badef44b..7bccd9a64c9e4 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -78,7 +78,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
       matchPattern(step, m_Constant(&stepInt))) {
     if (((hiInt.getInt() - loInt.getInt()) % stepInt.getInt()) == 0) {
       Value trueVal = constantI1(rewriter, loc, true);
-      return rewriter.create<vector::BroadcastOp>(loc, mtp, trueVal);
+      return vector::BroadcastOp::create(rewriter, loc, mtp, trueVal);
     }
   }
   // Otherwise, generate a vector mask that avoids overrunning the upperbound
@@ -92,7 +92,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
       rewriter.getContext());
   Value end = rewriter.createOrFold<affine::AffineMinOp>(
       loc, min, ValueRange{hi, iv, step});
-  return rewriter.create<vector::CreateMaskOp>(loc, mtp, end);
+  return vector::CreateMaskOp::create(rewriter, loc, mtp, end);
 }
 
 /// Generates a vectorized invariant. Here we rely on subsequent loop
@@ -100,7 +100,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
 static Value genVectorInvariantValue(PatternRewriter &rewriter, VL vl,
                                      Value val) {
   VectorType vtp = vectorType(vl, val.getType());
-  return rewriter.create<vector::BroadcastOp>(val.getLoc(), vtp, val);
+  return vector::BroadcastOp::create(rewriter, val.getLoc(), vtp, val);
 }
 
 /// Generates a vectorized load lhs = a[ind[lo:hi]] or lhs = a[lo:hi],
@@ -115,10 +115,10 @@ static Value genVectorLoad(PatternRewriter &rewriter, Location loc, VL vl,
     SmallVector<Value> scalarArgs(idxs);
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
-    return rewriter.create<vector::GatherOp>(loc, vtp, mem, scalarArgs,
+    return vector::GatherOp::create(rewriter, loc, vtp, mem, scalarArgs,
                                              indexVec, vmask, pass);
   }
-  return rewriter.create<vector::MaskedLoadOp>(loc, vtp, mem, idxs, vmask,
+  return vector::MaskedLoadOp::create(rewriter, loc, vtp, mem, idxs, vmask,
                                                pass);
 }
 
@@ -132,11 +132,11 @@ static void genVectorStore(PatternRewriter &rewriter, Location loc, Value mem,
     SmallVector<Value> scalarArgs(idxs);
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
-    rewriter.create<vector::ScatterOp>(loc, mem, scalarArgs, indexVec, vmask,
+    vector::ScatterOp::create(rewriter, loc, mem, scalarArgs, indexVec, vmask,
                                        rhs);
     return;
   }
-  rewriter.create<vector::MaskedStoreOp>(loc, mem, idxs, vmask, rhs);
+  vector::MaskedStoreOp::create(rewriter, loc, mem, idxs, vmask, rhs);
 }
 
 /// Detects a vectorizable reduction operations and returns the
@@ -197,18 +197,18 @@ static Value genVectorReducInit(PatternRewriter &rewriter, Location loc,
   case vector::CombiningKind::ADD:
   case vector::CombiningKind::XOR:
     // Initialize reduction vector to: | 0 | .. | 0 | r |
-    return rewriter.create<vector::InsertOp>(loc, r,
+    return vector::InsertOp::create(rewriter, loc, r,
                                              constantZero(rewriter, loc, vtp),
                                              constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::MUL:
     // Initialize reduction vector to: | 1 | .. | 1 | r |
-    return rewriter.create<vector::InsertOp>(loc, r,
+    return vector::InsertOp::create(rewriter, loc, r,
                                              constantOne(rewriter, loc, vtp),
                                              constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::AND:
   case vector::CombiningKind::OR:
     // Initialize reduction vector to: | r | .. | r | r |
-    return rewriter.create<vector::BroadcastOp>(loc, vtp, r);
+    return vector::BroadcastOp::create(rewriter, loc, vtp, r);
   default:
     break;
   }
@@ -300,10 +300,10 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
         Type etp = llvm::cast<VectorType>(vload.getType()).getElementType();
         if (!llvm::isa<IndexType>(etp)) {
           if (etp.getIntOrFloatBitWidth() < 32)
-            vload = rewriter.create<arith::ExtUIOp>(
+            vload = arith::ExtUIOp::create(rewriter,
                 loc, vectorType(vl, rewriter.getI32Type()), vload);
           else if (etp.getIntOrFloatBitWidth() < 64 && !vl.enableSIMDIndex32)
-            vload = rewriter.create<arith::ExtUIOp>(
+            vload = arith::ExtUIOp::create(rewriter,
                 loc, vectorType(vl, rewriter.getI64Type()), vload);
         }
         idxs.push_back(vload);
@@ -328,7 +328,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
             return false;
           if (codegen)
             idxs.push_back(
-                rewriter.create<arith::AddIOp>(forOp.getLoc(), inv, idx));
+                arith::AddIOp::create(rewriter, forOp.getLoc(), inv, idx));
           continue; // success so far
         }
       }
@@ -341,7 +341,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define UNAOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = rewriter.create<xxx>(loc, vx);                                    \
+      vexp = xxx::create(rewriter, loc, vx);                                    \
     return true;                                                               \
   }
 
@@ -349,7 +349,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
   if (auto x = dyn_cast<xxx>(def)) {                                           \
     if (codegen) {                                                             \
       VectorType vtp = vectorType(vl, x.getType());                            \
-      vexp = rewriter.create<xxx>(loc, vtp, vx);                               \
+      vexp = xxx::create(rewriter, loc, vtp, vx);                               \
     }                                                                          \
     return true;                                                               \
   }
@@ -357,7 +357,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define BINOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = rewriter.create<xxx>(loc, vx, vy);                                \
+      vexp = xxx::create(rewriter, loc, vx, vy);                                \
     return true;                                                               \
   }
 
@@ -380,9 +380,9 @@ static bool vectorizeExpr(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       // such as a[i] = i, which must convert to [i, i+1, ...].
       if (codegen) {
         VectorType vtp = vectorType(vl, arg.getType());
-        Value veci = rewriter.create<vector::BroadcastOp>(loc, vtp, arg);
-        Value incr = rewriter.create<vector::StepOp>(loc, vtp);
-        vexp = rewriter.create<arith::AddIOp>(loc, veci, incr);
+        Value veci = vector::BroadcastOp::create(rewriter, loc, vtp, arg);
+        Value incr = vector::StepOp::create(rewriter, loc, vtp);
+        vexp = arith::AddIOp::create(rewriter, loc, veci, incr);
       }
       return true;
     }
@@ -525,15 +525,15 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
     Value step = constantIndex(rewriter, loc, vl.vectorLength);
     if (vl.enableVLAVectorization) {
       Value vscale =
-          rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-      step = rewriter.create<arith::MulIOp>(loc, vscale, step);
+          vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+      step = arith::MulIOp::create(rewriter, loc, vscale, step);
     }
     if (!yield.getResults().empty()) {
       Value init = forOp.getInitArgs()[0];
       VectorType vtp = vectorType(vl, init.getType());
       Value vinit = genVectorReducInit(rewriter, loc, yield->getOperand(0),
                                        forOp.getRegionIterArg(0), init, vtp);
-      forOpNew = rewriter.create<scf::ForOp>(
+      forOpNew = scf::ForOp::create(rewriter,
           loc, forOp.getLowerBound(), forOp.getUpperBound(), step, vinit);
       forOpNew->setAttr(
           LoopEmitter::getLoopEmitterLoopAttrName(),
@@ -562,10 +562,10 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       if (codegen) {
         Value partial = forOpNew.getResult(0);
         Value vpass = genVectorInvariantValue(rewriter, vl, iter);
-        Value vred = rewriter.create<arith::SelectOp>(loc, vmask, vrhs, vpass);
-        rewriter.create<scf::YieldOp>(loc, vred);
+        Value vred = arith::SelectOp::create(rewriter, loc, vmask, vrhs, vpass);
+        scf::YieldOp::create(rewriter, loc, vred);
         rewriter.setInsertionPointAfter(forOpNew);
-        Value vres = rewriter.create<vector::ReductionOp>(loc, kind, partial);
+        Value vres = vector::ReductionOp::create(rewriter, loc, kind, partial);
         // Now do some relinking (last one is not completely type safe
         // but all bad ones are removed right away). This also folds away
         // nop broadcast operations.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
index d0e3e88f131d3..29a3e92fcf7fb 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
@@ -316,7 +316,7 @@ static void genBuffers(CodegenEnv &env, OpBuilder &builder) {
         if (!isInit) {
           Value zero = constantZero(builder, loc,
                                     getElementTypeOrSelf(tensor.getType()));
-          builder.create<linalg::FillOp>(loc, ValueRange{zero},
+          linalg::FillOp::create(builder, loc, ValueRange{zero},
                                          ValueRange{init});
         }
         return init;
@@ -379,7 +379,7 @@ static Value genInsertionLoad(CodegenEnv &env, OpBuilder &builder,
   }
   // Load from expanded access pattern.
   Value index = genIndex(env, t);
-  return builder.create<memref::LoadOp>(loc, env.getExpandValues(), index);
+  return memref::LoadOp::create(builder, loc, env.getExpandValues(), index);
 }
 
 /// Generates insertion code to implement dynamic tensor load for reduction.
@@ -395,22 +395,22 @@ static Value genInsertionLoadReduce(CodegenEnv &env, OpBuilder &builder,
   Value values = env.getExpandValues();
   Value filled = env.getExpandFilled();
   Value index = genIndex(env, t);
-  Value isFilled = builder.create<memref::LoadOp>(loc, filled, index);
-  Value valAtIndex = builder.create<memref::LoadOp>(loc, values, index);
-  return builder.create<arith::SelectOp>(loc, isFilled, valAtIndex, identity);
+  Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
+  Value valAtIndex = memref::LoadOp::create(builder, loc, values, index);
+  return arith::SelectOp::create(builder, loc, isFilled, valAtIndex, identity);
 }
 
 static Value genConditionalInsert(Location loc, OpBuilder &builder, Value cond,
                                   Value sparseOut, ValueRange ivs, Value v) {
   scf::IfOp condInsert =
-      builder.create<scf::IfOp>(loc, sparseOut.getType(), cond, true);
+      scf::IfOp::create(builder, loc, sparseOut.getType(), cond, true);
   // True branch.
   builder.setInsertionPointToStart(condInsert.thenBlock());
-  Value res = builder.create<tensor::InsertOp>(loc, v, sparseOut, ivs);
-  builder.create<scf::YieldOp>(loc, res);
+  Value res = tensor::InsertOp::create(builder, loc, v, sparseOut, ivs);
+  scf::YieldOp::create(builder, loc, res);
   // False branch.
   builder.setInsertionPointToStart(condInsert.elseBlock());
-  builder.create<scf::YieldOp>(loc, sparseOut);
+  scf::YieldOp::create(builder, loc, sparseOut);
   // Value assignment.
   builder.setInsertionPointAfter(condInsert);
   return condInsert.getResult(0);
@@ -447,7 +447,7 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
         Value nz = genIsNonzero(builder, loc, rhs);
         sparseOut = genConditionalInsert(loc, builder, nz, chain, ivs, rhs);
       } else {
-        sparseOut = builder.create<tensor::InsertOp>(loc, rhs, chain, ivs);
+        sparseOut = tensor::InsertOp::create(builder, loc, rhs, chain, ivs);
       }
       // Generates regular insertion chain.
       env.updateInsertionChain(sparseOut);
@@ -468,25 +468,25 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
   Value fval = constantI1(builder, loc, false);
   Value tval = constantI1(builder, loc, true);
   // If statement.
-  Value isFilled = builder.create<memref::LoadOp>(loc, filled, index);
-  Value cond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+  Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
+  Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
                                              isFilled, fval);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIndexType(), cond,
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIndexType(), cond,
                                              /*else=*/true);
   // True branch.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  builder.create<memref::StoreOp>(loc, tval, filled, index);
-  builder.create<memref::StoreOp>(loc, index, added, count);
+  memref::StoreOp::create(builder, loc, tval, filled, index);
+  memref::StoreOp::create(builder, loc, index, added, count);
   Value one = constantIndex(builder, loc, 1);
-  Value add = builder.create<arith::AddIOp>(loc, count, one);
-  builder.create<scf::YieldOp>(loc, add);
+  Value add = arith::AddIOp::create(builder, loc, count, one);
+  scf::YieldOp::create(builder, loc, add);
   // False branch.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, count);
+  scf::YieldOp::create(builder, loc, count);
   builder.setInsertionPointAfter(ifOp);
   // Value assignment.
   env.updateExpandCount(ifOp.getResult(0));
-  builder.create<memref::StoreOp>(loc, rhs, values, index);
+  memref::StoreOp::create(builder, loc, rhs, values, index);
 }
 
 /// Generates a load on a dense or sparse tensor.
@@ -516,9 +516,9 @@ static Value genTensorLoad(CodegenEnv &env, OpBuilder &builder, ExprId exp) {
   if (llvm::isa<TensorType>(ptr.getType())) {
     assert(env.options().sparseEmitStrategy ==
            SparseEmitStrategy::kSparseIterator);
-    return builder.create<ExtractValOp>(loc, ptr, llvm::getSingleElement(args));
+    return ExtractValOp::create(builder, loc, ptr, llvm::getSingleElement(args));
   }
-  return builder.create<memref::LoadOp>(loc, ptr, args);
+  return memref::LoadOp::create(builder, loc, ptr, args);
 }
 
 /// Generates a store on a dense or sparse tensor.
@@ -545,7 +545,7 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   if (!env.isSparseOutput(t)) {
     SmallVector<Value> args;
     Value ptr = genSubscript(env, builder, t, args);
-    builder.create<memref::StoreOp>(loc, rhs, ptr, args);
+    memref::StoreOp::create(builder, loc, rhs, ptr, args);
     return;
   }
   // Store during sparse insertion.
@@ -556,7 +556,7 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   // Select operation insertion.
   Value chain = env.getInsertionChain();
   scf::IfOp ifOp =
-      builder.create<scf::IfOp>(loc, chain.getType(), rhs, /*else=*/true);
+      scf::IfOp::create(builder, loc, chain.getType(), rhs, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   // Existing value was preserved to be used here.
   assert(env.exp(exp).val);
@@ -565,10 +565,10 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   env.merger().clearExprValue(exp);
   // Yield modified insertion chain along true branch.
   Value mchain = env.getInsertionChain();
-  builder.create<scf::YieldOp>(op.getLoc(), mchain);
+  scf::YieldOp::create(builder, op.getLoc(), mchain);
   // Yield original insertion chain along false branch.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, chain);
+  scf::YieldOp::create(builder, loc, chain);
   // Done with if statement.
   env.updateInsertionChain(ifOp->getResult(0));
   builder.setInsertionPointAfter(ifOp);
@@ -597,7 +597,7 @@ static Value relinkBranch(CodegenEnv &env, RewriterBase &rewriter, Block *block,
       assert(!getSparseTensorType(t->get()).hasEncoding()); // dense!
       SmallVector<Value> args;
       Value ptr = genSubscript(env, rewriter, t, args);
-      return rewriter.create<memref::LoadOp>(op.getLoc(), ptr, args);
+      return memref::LoadOp::create(rewriter, op.getLoc(), ptr, args);
     }
   } else if (Operation *def = e.getDefiningOp()) {
     // Handle index computation.
@@ -768,7 +768,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Type t2 = MemRefType::get(dynShape, builder.getI1Type());
     Type t3 = MemRefType::get(dynShape, builder.getIndexType());
     Type t4 = builder.getIndexType();
-    auto r = builder.create<ExpandOp>(loc, TypeRange({t1, t2, t3, t4}), tensor);
+    auto r = ExpandOp::create(builder, loc, TypeRange({t1, t2, t3, t4}), tensor);
     assert(r.getNumResults() == 4);
     env.startExpand(r.getResult(0), r.getResult(1), r.getResult(2),
                     r.getResult(3));
@@ -781,7 +781,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Value added = env.getExpandAdded();
     Value count = env.getExpandCount();
     Value chain = env.getInsertionChain();
-    Value compress = builder.create<CompressOp>(loc, values, filled, added,
+    Value compress = CompressOp::create(builder, loc, values, filled, added,
                                                 count, chain, indices);
     env.updateInsertionChain(compress);
     env.endExpand();
@@ -889,7 +889,7 @@ static void finalizeWhileOp(CodegenEnv &env, OpBuilder &builder,
         env.updateInsertionChain(ifOp->getResult(y++));
       }
       assert(y == yields.size());
-      builder.create<scf::YieldOp>(loc, yields);
+      scf::YieldOp::create(builder, loc, yields);
       builder.setInsertionPointAfter(ifOp);
     }
   }
@@ -942,13 +942,13 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
           assert(lvl.has_value());
           const Value crd = env.emitter().getCoord(tid, *lvl);
           const Value lvar = env.getLoopVar(curr);
-          clause = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+          clause = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
                                                  crd, lvar);
         } else {
           assert(lt.hasDenseSemantic() || isUndefLT(lt));
           clause = constantI1(builder, loc, true);
         }
-        cond = cond ? builder.create<arith::AndIOp>(loc, cond, clause) : clause;
+        cond = cond ? arith::AndIOp::create(builder, loc, cond, clause) : clause;
       });
   if (env.isReduc()) {
     types.push_back(env.getReduc().getType());
@@ -959,7 +959,7 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     types.push_back(builder.getIndexType());
   if (env.getInsertionChain())
     types.push_back(env.getInsertionChain().getType());
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   return ifOp;
 }
@@ -987,7 +987,7 @@ static void endIf(CodegenEnv &env, OpBuilder &builder, scf::IfOp ifOp,
     env.updateInsertionChain(insInput);
   }
   if (!operands.empty())
-    builder.create<scf::YieldOp>(env.op().getLoc(), operands);
+    scf::YieldOp::create(builder, env.op().getLoc(), operands);
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
 }
 
@@ -1301,7 +1301,7 @@ static void genStmt(CodegenEnv &env, RewriterBase &rewriter, ExprId exp,
           genStmt(env, rewriter, ej, curr + 1);
           // TODO: handle yield values.
           assert(reduc.empty() && "Not Implemented");
-          rewriter.create<sparse_tensor::YieldOp>(env.op().getLoc());
+          sparse_tensor::YieldOp::create(rewriter, env.op().getLoc());
           return std::nullopt;
         });
         // endIf(env, rewriter, ifOp, redInput, cntInput, insInput, validIns);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
index 7835c6c3b7797..684a2d418f66c 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
@@ -41,7 +41,7 @@ struct GuardSparseAlloc
     // operation that leaves the underlying storage in a proper state
     // before the tensor escapes across the method boundary.
     rewriter.setInsertionPointAfter(op);
-    auto load = rewriter.create<LoadOp>(op.getLoc(), op.getResult(), true);
+    auto load = LoadOp::create(rewriter, op.getLoc(), op.getResult(), true);
     rewriter.replaceAllUsesExcept(op, load, load);
     return success();
   }
@@ -60,7 +60,7 @@ struct StageUnorderedSparseOps : public OpRewritePattern<StageWithSortOp> {
     // Deallocate tmpBuf.
     // TODO: Delegate to buffer deallocation pass in the future.
     if (succeeded(stageResult) && tmpBuf)
-      rewriter.create<bufferization::DeallocTensorOp>(loc, tmpBuf);
+      bufferization::DeallocTensorOp::create(rewriter, loc, tmpBuf);
 
     return stageResult;
   }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
index 33be62d1d5e7e..27c1b106d71ba 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
@@ -153,7 +153,7 @@ Value sparse_tensor::genCast(OpBuilder &builder, Location loc, Value value,
 
   // int <=> index
   if (isa<IndexType>(srcTp) || isa<IndexType>(dstTp))
-    return builder.create<arith::IndexCastOp>(loc, dstTp, value);
+    return arith::IndexCastOp::create(builder, loc, dstTp, value);
 
   const auto srcIntTp = dyn_cast_or_null<IntegerType>(srcTp);
   const bool isUnsignedCast = srcIntTp ? srcIntTp.isUnsigned() : false;
@@ -166,19 +166,19 @@ Value sparse_tensor::genScalarToTensor(OpBuilder &builder, Location loc,
     // Scalars can only be converted to 0-ranked tensors.
     assert(rtp.getRank() == 0);
     elem = sparse_tensor::genCast(builder, loc, elem, rtp.getElementType());
-    return builder.create<tensor::FromElementsOp>(loc, rtp, elem);
+    return tensor::FromElementsOp::create(builder, loc, rtp, elem);
   }
   return sparse_tensor::genCast(builder, loc, elem, dstTp);
 }
 
 Value sparse_tensor::genIndexLoad(OpBuilder &builder, Location loc, Value mem,
                                   ValueRange s) {
-  Value load = builder.create<memref::LoadOp>(loc, mem, s);
+  Value load = memref::LoadOp::create(builder, loc, mem, s);
   if (!isa<IndexType>(load.getType())) {
     if (load.getType().getIntOrFloatBitWidth() < 64)
-      load = builder.create<arith::ExtUIOp>(loc, builder.getI64Type(), load);
+      load = arith::ExtUIOp::create(builder, loc, builder.getI64Type(), load);
     load =
-        builder.create<arith::IndexCastOp>(loc, builder.getIndexType(), load);
+        arith::IndexCastOp::create(builder, loc, builder.getIndexType(), load);
   }
   return load;
 }
@@ -203,13 +203,13 @@ Value mlir::sparse_tensor::genIsNonzero(OpBuilder &builder, mlir::Location loc,
   Type tp = v.getType();
   Value zero = constantZero(builder, loc, tp);
   if (isa<FloatType>(tp))
-    return builder.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNE, v,
+    return arith::CmpFOp::create(builder, loc, arith::CmpFPredicate::UNE, v,
                                          zero);
   if (tp.isIntOrIndex())
-    return builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, v,
+    return arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, v,
                                          zero);
   if (isa<ComplexType>(tp))
-    return builder.create<complex::NotEqualOp>(loc, v, zero);
+    return complex::NotEqualOp::create(builder, loc, v, zero);
   llvm_unreachable("Non-numeric type");
 }
 
@@ -223,7 +223,7 @@ void mlir::sparse_tensor::genReshapeDstShape(
     for (const auto &map : llvm::enumerate(reassociation)) {
       auto dstDim = constantIndex(builder, loc, 1);
       for (unsigned i = start; i < start + map.value().size(); i++) {
-        dstDim = builder.create<arith::MulIOp>(loc, dstDim, srcShape[i]);
+        dstDim = arith::MulIOp::create(builder, loc, dstDim, srcShape[i]);
       }
       dstShape.push_back(dstDim);
       start = start + map.value().size();
@@ -257,7 +257,7 @@ void mlir::sparse_tensor::genReshapeDstShape(
         // Compute the dynamic dimension size.
         Value productVal = constantIndex(builder, loc, product);
         Value dynamicSize =
-            builder.create<arith::DivUIOp>(loc, srcDim, productVal);
+            arith::DivUIOp::create(builder, loc, srcDim, productVal);
         dstShape.push_back(dynamicSize);
       } else {
         // The expanded dimension is statically known.
@@ -286,7 +286,7 @@ void mlir::sparse_tensor::reshapeCvs(
     // Prepare strides information in dimension slice.
     Value linear = constantIndex(builder, loc, 1);
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
-      linear = builder.create<arith::MulIOp>(loc, linear, sizes[j]);
+      linear = arith::MulIOp::create(builder, loc, linear, sizes[j]);
     }
     // Start expansion.
     Value val;
@@ -294,16 +294,16 @@ void mlir::sparse_tensor::reshapeCvs(
       val = srcCvs[i];
     // Iterate over dimension slice.
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
-      linear = builder.create<arith::DivUIOp>(loc, linear, sizes[j]);
+      linear = arith::DivUIOp::create(builder, loc, linear, sizes[j]);
       if (isCollapse) {
-        const Value mul = builder.create<arith::MulIOp>(loc, srcCvs[j], linear);
-        val = val ? builder.create<arith::AddIOp>(loc, val, mul) : mul;
+        const Value mul = arith::MulIOp::create(builder, loc, srcCvs[j], linear);
+        val = val ? arith::AddIOp::create(builder, loc, val, mul) : mul;
       } else {
         const Value old = val;
-        val = builder.create<arith::DivUIOp>(loc, val, linear);
+        val = arith::DivUIOp::create(builder, loc, val, linear);
         assert(dstCvs.size() == j);
         dstCvs.push_back(val);
-        val = builder.create<arith::RemUIOp>(loc, old, linear);
+        val = arith::RemUIOp::create(builder, loc, old, linear);
       }
     }
     // Finalize collapse.
@@ -326,7 +326,7 @@ FlatSymbolRefAttr mlir::sparse_tensor::getFunc(ModuleOp module, StringRef name,
   auto func = module.lookupSymbol<func::FuncOp>(result.getAttr());
   if (!func) {
     OpBuilder moduleBuilder(module.getBodyRegion());
-    func = moduleBuilder.create<func::FuncOp>(
+    func = func::FuncOp::create(moduleBuilder,
         module.getLoc(), name,
         FunctionType::get(context, operands.getTypes(), resultType));
     func.setPrivate();
@@ -343,7 +343,7 @@ func::CallOp mlir::sparse_tensor::createFuncCall(
   auto module = builder.getBlock()->getParentOp()->getParentOfType<ModuleOp>();
   FlatSymbolRefAttr fn =
       getFunc(module, name, resultType, operands, emitCInterface);
-  return builder.create<func::CallOp>(loc, resultType, fn, operands);
+  return func::CallOp::create(builder, loc, resultType, fn, operands);
 }
 
 Type mlir::sparse_tensor::getOpaquePointerType(MLIRContext *ctx) {
@@ -358,7 +358,7 @@ Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
                                      unsigned sz, Type tp, bool staticShape) {
   if (staticShape) {
     auto memTp = MemRefType::get({sz}, tp);
-    return builder.create<memref::AllocaOp>(loc, memTp);
+    return memref::AllocaOp::create(builder, loc, memTp);
   }
   return genAlloca(builder, loc, constantIndex(builder, loc, sz), tp);
 }
@@ -366,12 +366,12 @@ Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
 Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc, Value sz,
                                      Type tp) {
   auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
-  return builder.create<memref::AllocaOp>(loc, memTp, ValueRange{sz});
+  return memref::AllocaOp::create(builder, loc, memTp, ValueRange{sz});
 }
 
 Value mlir::sparse_tensor::genAllocaScalar(OpBuilder &builder, Location loc,
                                            Type tp) {
-  return builder.create<memref::AllocaOp>(loc, MemRefType::get({}, tp));
+  return memref::AllocaOp::create(builder, loc, MemRefType::get({}, tp));
 }
 
 Value mlir::sparse_tensor::allocaBuffer(OpBuilder &builder, Location loc,
@@ -381,7 +381,7 @@ Value mlir::sparse_tensor::allocaBuffer(OpBuilder &builder, Location loc,
   Value buffer = genAlloca(builder, loc, sz, values[0].getType());
   for (unsigned i = 0; i < sz; i++) {
     Value idx = constantIndex(builder, loc, i);
-    builder.create<memref::StoreOp>(loc, values[i], buffer, idx);
+    memref::StoreOp::create(builder, loc, values[i], buffer, idx);
   }
   return buffer;
 }
@@ -397,15 +397,15 @@ Value mlir::sparse_tensor::allocDenseTensor(OpBuilder &builder, Location loc,
     if (shape[i] == ShapedType::kDynamic)
       dynamicSizes.push_back(sizes[i]);
   }
-  Value mem = builder.create<memref::AllocOp>(loc, memTp, dynamicSizes);
+  Value mem = memref::AllocOp::create(builder, loc, memTp, dynamicSizes);
   Value zero = constantZero(builder, loc, elemTp);
-  builder.create<linalg::FillOp>(loc, ValueRange{zero}, ValueRange{mem});
+  linalg::FillOp::create(builder, loc, ValueRange{zero}, ValueRange{mem});
   return mem;
 }
 
 void mlir::sparse_tensor::deallocDenseTensor(OpBuilder &builder, Location loc,
                                              Value buffer) {
-  builder.create<memref::DeallocOp>(loc, buffer);
+  memref::DeallocOp::create(builder, loc, buffer);
 }
 
 void mlir::sparse_tensor::sizesFromSrc(OpBuilder &builder,
@@ -483,17 +483,17 @@ void sparse_tensor::foreachInSparseConstant(
     cvs.clear();
     for (Dimension d = 0; d < dimRank; d++) {
       auto crd = elems[i].first[d].getInt();
-      cvs.push_back(builder.create<arith::ConstantIndexOp>(loc, crd));
+      cvs.push_back(arith::ConstantIndexOp::create(builder, loc, crd));
     }
     // Remap value.
     Value val;
     if (isa<ComplexType>(attr.getElementType())) {
       auto valAttr = cast<ArrayAttr>(elems[i].second);
-      val = builder.create<complex::ConstantOp>(loc, attr.getElementType(),
+      val = complex::ConstantOp::create(builder, loc, attr.getElementType(),
                                                 valAttr);
     } else {
       auto valAttr = cast<TypedAttr>(elems[i].second);
-      val = builder.create<arith::ConstantOp>(loc, valAttr);
+      val = arith::ConstantOp::create(builder, loc, valAttr);
     }
     assert(val);
     callback(cvs, val);
@@ -513,10 +513,10 @@ SmallVector<Value> sparse_tensor::loadAll(OpBuilder &builder, Location loc,
   SmallVector<Value> vs;
   vs.reserve(size);
   for (unsigned i = 0; i < size; i++) {
-    Value v = builder.create<memref::LoadOp>(loc, mem,
+    Value v = memref::LoadOp::create(builder, loc, mem,
                                              constantIndex(builder, loc, i));
     if (i == offsetIdx && offsetVal)
-      v = builder.create<arith::AddIOp>(loc, v, offsetVal);
+      v = arith::AddIOp::create(builder, loc, v, offsetVal);
     vs.push_back(v);
   }
   return vs;
@@ -535,9 +535,9 @@ void sparse_tensor::storeAll(OpBuilder &builder, Location loc, Value mem,
   for (const auto &v : llvm::enumerate(vs)) {
     const Value w =
         (offsetIdx == v.index() && offsetVal)
-            ? builder.create<arith::AddIOp>(loc, v.value(), offsetVal)
+            ? arith::AddIOp::create(builder, loc, v.value(), offsetVal)
             : v.value();
-    builder.create<memref::StoreOp>(loc, w, mem,
+    memref::StoreOp::create(builder, loc, w, mem,
                                     constantIndex(builder, loc, v.index()));
   }
 }
@@ -547,7 +547,7 @@ sparse_tensor::genToMemref(OpBuilder &builder, Location loc, Value tensor) {
   auto tTp = llvm::cast<TensorType>(tensor.getType());
   auto mTp = MemRefType::get(tTp.getShape(), tTp.getElementType());
   return cast<TypedValue<BaseMemRefType>>(
-      builder.create<bufferization::ToBufferOp>(loc, mTp, tensor).getResult());
+      bufferization::ToBufferOp::create(builder, loc, mTp, tensor).getResult());
 }
 
 Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
@@ -557,7 +557,7 @@ Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
   std::optional<unsigned> offset = enc.getStaticDimSliceOffset(dim);
   if (offset.has_value())
     return constantIndex(builder, loc, *offset);
-  return builder.create<ToSliceOffsetOp>(loc, tensor, APInt(64, dim));
+  return ToSliceOffsetOp::create(builder, loc, tensor, APInt(64, dim));
 }
 
 Value sparse_tensor::createOrFoldSliceStrideOp(OpBuilder &builder, Location loc,
@@ -567,7 +567,7 @@ Value sparse_tensor::createOrFoldSliceStrideOp(OpBuilder &builder, Location loc,
   std::optional<unsigned> stride = enc.getStaticDimSliceStride(dim);
   if (stride.has_value())
     return constantIndex(builder, loc, *stride);
-  return builder.create<ToSliceStrideOp>(loc, tensor, APInt(64, dim));
+  return ToSliceStrideOp::create(builder, loc, tensor, APInt(64, dim));
 }
 
 Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
@@ -609,7 +609,7 @@ Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
     // subsequent clients need the values (DCE will remove unused).
     for (Dimension d = 0; d < dimRank; d++) {
       if (stt.isDynamicDim(d))
-        dimSizesValues[d] = builder.create<memref::LoadOp>(
+        dimSizesValues[d] = memref::LoadOp::create(builder,
             loc, dimSizesBuffer, constantIndex(builder, loc, d));
     }
   }
@@ -686,7 +686,7 @@ Value sparse_tensor::genMapBuffers(
     if (cm == 0) {
       lvlSz = dimSizesValues[d];
       if (cf != 0)
-        lvlSz = builder.create<arith::DivUIOp>(loc, lvlSz,
+        lvlSz = arith::DivUIOp::create(builder, loc, lvlSz,
                                                constantIndex(builder, loc, cf));
     } else {
       lvlSz = constantIndex(builder, loc, cm);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
index dc017e6baa6dc..400be54aeb86f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
@@ -97,7 +97,7 @@ class FuncCallOrInlineGenerator {
       // Create the function if not already exist.
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(getParentOpOf<func::FuncOp>(builder));
-      func = builder.create<func::FuncOp>(
+      func = func::FuncOp::create(builder,
           loc, funcName,
           FunctionType::get(context, params.getTypes(), retTypes));
       func.setPrivate();
@@ -108,10 +108,10 @@ class FuncCallOrInlineGenerator {
       // Delegates to user to generate the actually implementation.
       SmallVector<Value> result =
           genImplementation(retTypes, args, builder, loc);
-      builder.create<func::ReturnOp>(loc, result);
+      func::ReturnOp::create(builder, loc, result);
     }
     // Returns the CallOp result.
-    func::CallOp call = builder.create<func::CallOp>(loc, func, params);
+    func::CallOp call = func::CallOp::create(builder, loc, func, params);
     return call.getResults();
   }
 
@@ -310,9 +310,9 @@ inline Value constantZero(OpBuilder &builder, Location loc, Type tp) {
   if (auto ctp = dyn_cast<ComplexType>(tp)) {
     auto zeroe = builder.getZeroAttr(ctp.getElementType());
     auto zeroa = builder.getArrayAttr({zeroe, zeroe});
-    return builder.create<complex::ConstantOp>(loc, tp, zeroa);
+    return complex::ConstantOp::create(builder, loc, tp, zeroa);
   }
-  return builder.create<arith::ConstantOp>(loc, tp, builder.getZeroAttr(tp));
+  return arith::ConstantOp::create(builder, loc, tp, builder.getZeroAttr(tp));
 }
 
 /// Generates a 1-valued constant of the given type.  This supports all
@@ -322,39 +322,39 @@ inline Value constantOne(OpBuilder &builder, Location loc, Type tp) {
     auto zeroe = builder.getZeroAttr(ctp.getElementType());
     auto onee = getOneAttr(builder, ctp.getElementType());
     auto zeroa = builder.getArrayAttr({onee, zeroe});
-    return builder.create<complex::ConstantOp>(loc, tp, zeroa);
+    return complex::ConstantOp::create(builder, loc, tp, zeroa);
   }
-  return builder.create<arith::ConstantOp>(loc, tp, getOneAttr(builder, tp));
+  return arith::ConstantOp::create(builder, loc, tp, getOneAttr(builder, tp));
 }
 
 /// Generates a constant of `index` type.
 inline Value constantIndex(OpBuilder &builder, Location loc, int64_t i) {
-  return builder.create<arith::ConstantIndexOp>(loc, i);
+  return arith::ConstantIndexOp::create(builder, loc, i);
 }
 
 /// Generates a constant of `i64` type.
 inline Value constantI64(OpBuilder &builder, Location loc, int64_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 64);
+  return arith::ConstantIntOp::create(builder, loc, i, 64);
 }
 
 /// Generates a constant of `i32` type.
 inline Value constantI32(OpBuilder &builder, Location loc, int32_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 32);
+  return arith::ConstantIntOp::create(builder, loc, i, 32);
 }
 
 /// Generates a constant of `i16` type.
 inline Value constantI16(OpBuilder &builder, Location loc, int16_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 16);
+  return arith::ConstantIntOp::create(builder, loc, i, 16);
 }
 
 /// Generates a constant of `i8` type.
 inline Value constantI8(OpBuilder &builder, Location loc, int8_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 8);
+  return arith::ConstantIntOp::create(builder, loc, i, 8);
 }
 
 /// Generates a constant of `i1` type.
 inline Value constantI1(OpBuilder &builder, Location loc, bool b) {
-  return builder.create<arith::ConstantIntOp>(loc, b, 1);
+  return arith::ConstantIntOp::create(builder, loc, b, 1);
 }
 
 /// Generates a constant of the given `Action`.
@@ -400,12 +400,12 @@ inline Value constantLevelTypeEncoding(OpBuilder &builder, Location loc,
 inline Value genValFromAttr(OpBuilder &builder, Location loc, Attribute attr) {
   if (auto complexAttr = dyn_cast<complex::NumberAttr>(attr)) {
     Type tp = cast<ComplexType>(complexAttr.getType()).getElementType();
-    return builder.create<complex::ConstantOp>(
+    return complex::ConstantOp::create(builder,
         loc, complexAttr.getType(),
         builder.getArrayAttr({FloatAttr::get(tp, complexAttr.getReal()),
                               FloatAttr::get(tp, complexAttr.getImag())}));
   }
-  return builder.create<arith::ConstantOp>(loc, cast<TypedAttr>(attr));
+  return arith::ConstantOp::create(builder, loc, cast<TypedAttr>(attr));
 }
 
 // TODO: is this at the right place?
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
index a77e3036ac519..6eaeff91dd5c6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
@@ -25,18 +25,18 @@ using namespace mlir::sparse_tensor;
 //===----------------------------------------------------------------------===//
 
 #define CMPI(p, l, r)                                                          \
-  (builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::p, (l), (r))       \
+  (arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::p, (l), (r))       \
        .getResult())
 
 #define C_IDX(v) (constantIndex(builder, loc, (v)))
-#define YIELD(vs) (builder.create<scf::YieldOp>(loc, (vs)))
-#define ADDI(lhs, rhs) (builder.create<arith::AddIOp>(loc, (lhs), (rhs)))
-#define ANDI(lhs, rhs) (builder.create<arith::AndIOp>(loc, (lhs), (rhs)))
-#define SUBI(lhs, rhs) (builder.create<arith::SubIOp>(loc, (lhs), (rhs)))
-#define MULI(lhs, rhs) (builder.create<arith::MulIOp>(loc, (lhs), (rhs)))
-#define REMUI(lhs, rhs) (builder.create<arith::RemUIOp>(loc, (lhs), (rhs)))
-#define DIVUI(lhs, rhs) (builder.create<arith::DivUIOp>(loc, (lhs), (rhs)))
-#define SELECT(c, l, r) (builder.create<arith::SelectOp>(loc, (c), (l), (r)))
+#define YIELD(vs) (scf::YieldOp::create(builder, loc, (vs)))
+#define ADDI(lhs, rhs) (arith::AddIOp::create(builder, loc, (lhs), (rhs)))
+#define ANDI(lhs, rhs) (arith::AndIOp::create(builder, loc, (lhs), (rhs)))
+#define SUBI(lhs, rhs) (arith::SubIOp::create(builder, loc, (lhs), (rhs)))
+#define MULI(lhs, rhs) (arith::MulIOp::create(builder, loc, (lhs), (rhs)))
+#define REMUI(lhs, rhs) (arith::RemUIOp::create(builder, loc, (lhs), (rhs)))
+#define DIVUI(lhs, rhs) (arith::DivUIOp::create(builder, loc, (lhs), (rhs)))
+#define SELECT(c, l, r) (arith::SelectOp::create(builder, loc, (c), (l), (r)))
 
 //===----------------------------------------------------------------------===//
 // Debugging utils
@@ -45,7 +45,7 @@ using namespace mlir::sparse_tensor;
 #ifndef NDEBUG
 LLVM_ATTRIBUTE_UNUSED static void dumpIndexMemRef(OpBuilder &builder,
                                                   Location loc, Value memref) {
-  memref = builder.create<memref::CastOp>(
+  memref = memref::CastOp::create(builder,
       loc, UnrankedMemRefType::get(builder.getIndexType(), 0), memref);
   createFuncCall(builder, loc, "printMemrefInd", TypeRange{},
                  ValueRange{memref}, EmitCInterface::On);
@@ -261,7 +261,7 @@ void LoopEmitter::initializeLoopEmit(
         denseTp = bufferization::getMemRefTypeWithFullyDynamicLayout(rtp);
 
       Value denseVal =
-          builder.create<bufferization::ToBufferOp>(loc, denseTp, tensor);
+          bufferization::ToBufferOp::create(builder, loc, denseTp, tensor);
       // Dense outputs need special handling.
       if (isOutput && updater)
         denseVal = updater(builder, loc, denseVal, tensor);
@@ -271,7 +271,7 @@ void LoopEmitter::initializeLoopEmit(
       // Annotated sparse tensors.
       // We also need the value buffer for all-dense annotated "sparse"
       // tensors.
-      valBuffer[t] = builder.create<ToValuesOp>(loc, tensor);
+      valBuffer[t] = ToValuesOp::create(builder, loc, tensor);
     }
   }
 
@@ -479,7 +479,7 @@ std::pair<Operation *, Value> LoopEmitter::emitForLoopOverTensorAtLvl(
   Value iv;
   if (isParallel) {
     scf::ParallelOp parOp =
-        builder.create<scf::ParallelOp>(loc, lo, hi, step, reduc);
+        scf::ParallelOp::create(builder, loc, lo, hi, step, reduc);
     builder.setInsertionPointToStart(parOp.getBody());
     assert(parOp.getNumReductions() == reduc.size());
     iv = parOp.getInductionVars()[0];
@@ -495,7 +495,7 @@ std::pair<Operation *, Value> LoopEmitter::emitForLoopOverTensorAtLvl(
       reduc[i] = parOp.getInitVals()[i];
     loop = parOp;
   } else {
-    scf::ForOp forOp = builder.create<scf::ForOp>(loc, lo, hi, step, reduc);
+    scf::ForOp forOp = scf::ForOp::create(builder, loc, lo, hi, step, reduc);
     builder.setInsertionPointToStart(forOp.getBody());
     iv = forOp.getInductionVar();
 
@@ -603,11 +603,11 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
 
       // Extract and iterate over the iteration space.
       ExtractIterSpaceOp extractSpaceOp =
-          lvl == 0 ? builder.create<ExtractIterSpaceOp>(loc, t)
-                   : builder.create<ExtractIterSpaceOp>(
+          lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
+                   : ExtractIterSpaceOp::create(builder,
                          loc, t, spIterVals[tid][lvl - 1], lvl);
 
-      IterateOp iterOp = builder.create<IterateOp>(
+      IterateOp iterOp = IterateOp::create(builder,
           loc, extractSpaceOp.getExtractedSpace(), reduc);
       spIterVals[tid][lvl] = iterOp.getIterator();
 
@@ -625,12 +625,12 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
     for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
       Value t = tensors[tid];
       ExtractIterSpaceOp extractSpaceOp =
-          lvl == 0 ? builder.create<ExtractIterSpaceOp>(loc, t)
-                   : builder.create<ExtractIterSpaceOp>(
+          lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
+                   : ExtractIterSpaceOp::create(builder,
                          loc, t, spIterVals[tid][lvl - 1], lvl);
       spaces.push_back(extractSpaceOp.getExtractedSpace());
     }
-    auto coIterOp = builder.create<CoIterateOp>(loc, spaces, reduc, numCases);
+    auto coIterOp = CoIterateOp::create(builder, loc, spaces, reduc, numCases);
     // The CoIterationOp does not have insertion block nor induction variable.
     // TODO: the `struct LoopInfo` should be simplied after full migration.
     loopStack.emplace_back(tidLvls, coIterOp, /*insertion block*/ nullptr,
@@ -728,7 +728,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   if (emitStrategy == SparseEmitStrategy::kSparseIterator) {
     auto iterateOp = llvm::cast<IterateOp>(loopInfo.loop);
     assert(reduc.size() == iterateOp.getNumResults());
-    rewriter.create<sparse_tensor::YieldOp>(loc, reduc);
+    sparse_tensor::YieldOp::create(rewriter, loc, reduc);
     // Exit the loop.
     rewriter.setInsertionPointAfter(iterateOp);
     // In-place update reduction variables.
@@ -738,7 +738,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   if (auto forOp = llvm::dyn_cast<scf::ForOp>(loopInfo.loop)) {
     if (!reduc.empty()) {
       assert(reduc.size() == forOp.getNumResults());
-      rewriter.create<scf::YieldOp>(loc, reduc);
+      scf::YieldOp::create(rewriter, loc, reduc);
     }
     // Exit the loop.
     rewriter.setInsertionPointAfter(forOp);
@@ -777,7 +777,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
 #endif // NDEBUG
 
       rewriter.setInsertionPointAfter(redExp);
-      auto redOp = rewriter.create<scf::ReduceOp>(loc, curVal);
+      auto redOp = scf::ReduceOp::create(rewriter, loc, curVal);
       // Attach to the reduction op.
       Block *redBlock = &redOp.getReductions().front().front();
       rewriter.setInsertionPointToEnd(redBlock);
@@ -789,7 +789,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
       // Erases the out-dated reduction expression.
       rewriter.eraseOp(redExp);
       rewriter.setInsertionPointToEnd(redBlock);
-      rewriter.create<scf::ReduceReturnOp>(loc, newRed->getResult(0));
+      scf::ReduceReturnOp::create(rewriter, loc, newRed->getResult(0));
     }
     rewriter.setInsertionPointAfter(parOp);
     // In-place update reduction variables.
@@ -863,7 +863,7 @@ void LoopEmitter::exitCurrentLoop(RewriterBase &rewriter, Location loc,
   if (emitStrategy == SparseEmitStrategy::kSparseIterator) {
     Operation *p = loopInfo.loop;
     if (isa<IterateOp>(p))
-      rewriter.create<sparse_tensor::YieldOp>(loc, reduc);
+      sparse_tensor::YieldOp::create(rewriter, loc, reduc);
 
     // Exit the loop.
     rewriter.setInsertionPointAfter(p);
@@ -929,7 +929,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
   // Ensures all operands are valid.
   assert(!llvm::is_contained(ivs, nullptr));
   TypeRange types = ValueRange(ivs).getTypes();
-  auto whileOp = builder.create<scf::WhileOp>(loc, types, ivs);
+  auto whileOp = scf::WhileOp::create(builder, loc, types, ivs);
 
   SmallVector<Location> locs(types.size(), loc);
   Block *before = builder.createBlock(&whileOp.getBefore(), {}, types, locs);
@@ -948,7 +948,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
   // The remaining block arguments are user-provided reduction values and an
   // optional universal index. Make sure their sizes match.
   assert(bArgs.size() == reduc.size() + (uniIdx ? 1 : 0));
-  builder.create<scf::ConditionOp>(loc, whileCond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, whileCond, before->getArguments());
 
   // Generates loop body.
   builder.setInsertionPointToStart(after);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
index 1c8a4789e2065..47fa3f950afd3 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
@@ -78,14 +78,14 @@ SparseTensorTypeToBufferConverter::SparseTensorTypeToBufferConverter() {
 
 Value SparseTensorSpecifier::getInitValue(OpBuilder &builder, Location loc,
                                           SparseTensorType stt) {
-  return builder.create<StorageSpecifierInitOp>(
+  return StorageSpecifierInitOp::create(builder,
       loc, StorageSpecifierType::get(stt.getEncoding()));
 }
 
 Value SparseTensorSpecifier::getSpecifierField(OpBuilder &builder, Location loc,
                                                StorageSpecifierKind kind,
                                                std::optional<Level> lvl) {
-  return builder.create<GetStorageSpecifierOp>(
+  return GetStorageSpecifierOp::create(builder,
       loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl));
 }
 
@@ -95,7 +95,7 @@ void SparseTensorSpecifier::setSpecifierField(OpBuilder &builder, Location loc,
                                               std::optional<Level> lvl) {
   // TODO: make `v` have type `TypedValue<IndexType>` instead.
   assert(v.getType().isIndex());
-  specifier = builder.create<SetStorageSpecifierOp>(
+  specifier = SetStorageSpecifierOp::create(builder,
       loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl), v);
 }
 
@@ -111,8 +111,8 @@ Value sparse_tensor::SparseTensorDescriptor::getCrdMemRefOrView(
 
   Value stride = constantIndex(builder, loc, rType.getLvlRank() - cooStart);
   Value size = getCrdMemSize(builder, loc, cooStart);
-  size = builder.create<arith::DivUIOp>(loc, size, stride);
-  return builder.create<memref::SubViewOp>(
+  size = arith::DivUIOp::create(builder, loc, size, stride);
+  return memref::SubViewOp::create(builder,
       loc, getMemRefField(SparseTensorFieldKind::CrdMemRef, cooStart),
       /*offset=*/ValueRange{constantIndex(builder, loc, lvl - cooStart)},
       /*size=*/ValueRange{size},
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
index 869c7864d7535..45d142a807c36 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
@@ -231,7 +231,7 @@ class MutSparseTensorDescriptor
 /// Packs the given values as a "tuple" value.
 inline Value genTuple(OpBuilder &builder, Location loc, Type tp,
                       ValueRange values) {
-  return builder.create<UnrealizedConversionCastOp>(loc, TypeRange(tp), values)
+  return UnrealizedConversionCastOp::create(builder, loc, TypeRange(tp), values)
       .getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
index aad5e97ed14ab..212ee99647167 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
@@ -22,23 +22,23 @@ using ValueTuple = std::tuple<Value, Value, Value>;
 // File local helper functions/macros.
 //===----------------------------------------------------------------------===//
 #define CMPI(p, lhs, rhs)                                                      \
-  (b.create<arith::CmpIOp>(l, arith::CmpIPredicate::p, (lhs), (rhs))           \
+  (arith::CmpIOp::create(b, l, arith::CmpIPredicate::p, (lhs), (rhs))           \
        .getResult())
 
 #define C_FALSE (constantI1(b, l, false))
 #define C_TRUE (constantI1(b, l, true))
 #define C_IDX(v) (constantIndex(b, l, (v)))
-#define YIELD(vs) (b.create<scf::YieldOp>(l, (vs)))
-#define ADDI(lhs, rhs) (b.create<arith::AddIOp>(l, (lhs), (rhs)).getResult())
-#define ORI(lhs, rhs) (b.create<arith::OrIOp>(l, (lhs), (rhs)).getResult())
-#define ANDI(lhs, rhs) (b.create<arith::AndIOp>(l, (lhs), (rhs)).getResult())
-#define SUBI(lhs, rhs) (b.create<arith::SubIOp>(l, (lhs), (rhs)).getResult())
-#define MULI(lhs, rhs) (b.create<arith::MulIOp>(l, (lhs), (rhs)).getResult())
-#define MINUI(lhs, rhs) (b.create<arith::MinUIOp>(l, (lhs), (rhs)).getResult())
-#define REMUI(lhs, rhs) (b.create<arith::RemUIOp>(l, (lhs), (rhs)).getResult())
-#define DIVUI(lhs, rhs) (b.create<arith::DivUIOp>(l, (lhs), (rhs)).getResult())
+#define YIELD(vs) (scf::YieldOp::create(b, l, (vs)))
+#define ADDI(lhs, rhs) (arith::AddIOp::create(b, l, (lhs), (rhs)).getResult())
+#define ORI(lhs, rhs) (arith::OrIOp::create(b, l, (lhs), (rhs)).getResult())
+#define ANDI(lhs, rhs) (arith::AndIOp::create(b, l, (lhs), (rhs)).getResult())
+#define SUBI(lhs, rhs) (arith::SubIOp::create(b, l, (lhs), (rhs)).getResult())
+#define MULI(lhs, rhs) (arith::MulIOp::create(b, l, (lhs), (rhs)).getResult())
+#define MINUI(lhs, rhs) (arith::MinUIOp::create(b, l, (lhs), (rhs)).getResult())
+#define REMUI(lhs, rhs) (arith::RemUIOp::create(b, l, (lhs), (rhs)).getResult())
+#define DIVUI(lhs, rhs) (arith::DivUIOp::create(b, l, (lhs), (rhs)).getResult())
 #define SELECT(c, lhs, rhs)                                                    \
-  (b.create<arith::SelectOp>(l, (c), (lhs), (rhs)).getResult())
+  (arith::SelectOp::create(b, l, (c), (lhs), (rhs)).getResult())
 
 //===----------------------------------------------------------------------===//
 // SparseTensorLevel derived classes.
@@ -150,19 +150,19 @@ class CompressedLevel : public SparseLevel</*hasPosBuf=*/true> {
       return loadRange();
 
     SmallVector<Type, 2> types{b.getIndexType(), b.getIndexType()};
-    scf::IfOp posRangeIf = b.create<scf::IfOp>(l, types, inPadZone, true);
+    scf::IfOp posRangeIf = scf::IfOp::create(b, l, types, inPadZone, true);
     // True branch, returns a "fake" empty range [0, 0) if parent
     // iterator is in pad zone.
     b.setInsertionPointToStart(posRangeIf.thenBlock());
 
     SmallVector<Value, 2> emptyRange{C_IDX(0), C_IDX(0)};
-    b.create<scf::YieldOp>(l, emptyRange);
+    scf::YieldOp::create(b, l, emptyRange);
 
     // False branch, returns the actual range.
     b.setInsertionPointToStart(posRangeIf.elseBlock());
     auto [pLo, pHi] = loadRange();
     SmallVector<Value, 2> loadedRange{pLo, pHi};
-    b.create<scf::YieldOp>(l, loadedRange);
+    scf::YieldOp::create(b, l, loadedRange);
 
     b.setInsertionPointAfter(posRangeIf);
     ValueRange posRange = posRangeIf.getResults();
@@ -248,7 +248,7 @@ static scf::ValueVector genWhenInBound(
     llvm::function_ref<scf::ValueVector(OpBuilder &, Location, Value)>
         builder) {
   TypeRange ifRetTypes = elseRet.getTypes();
-  auto ifOp = b.create<scf::IfOp>(l, ifRetTypes, it.genNotEnd(b, l), true);
+  auto ifOp = scf::IfOp::create(b, l, ifRetTypes, it.genNotEnd(b, l), true);
 
   b.setInsertionPointToStart(ifOp.thenBlock());
   Value crd = it.deref(b, l);
@@ -732,7 +732,7 @@ class NonEmptySubSectIterator : public SparseIterator {
   //      [itVal0, itVal1, ..., pNx0],
   //      ...]
   Value allocSubSectPosBuf(OpBuilder &b, Location l) {
-    return b.create<memref::AllocaOp>(
+    return memref::AllocaOp::create(b,
         l,
         MemRefType::get({ShapedType::kDynamic, tupleSz + 1}, b.getIndexType()),
         maxTupleCnt);
@@ -740,12 +740,12 @@ class NonEmptySubSectIterator : public SparseIterator {
 
   void storeNxLvlStart(OpBuilder &b, Location l, Value tupleId,
                        Value start) const {
-    b.create<memref::StoreOp>(l, start, subSectPosBuf,
+    memref::StoreOp::create(b, l, start, subSectPosBuf,
                               ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
   Value loadNxLvlStart(OpBuilder &b, Location l, Value tupleId) const {
-    return b.create<memref::LoadOp>(l, subSectPosBuf,
+    return memref::LoadOp::create(b, l, subSectPosBuf,
                                     ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
@@ -753,7 +753,7 @@ class NonEmptySubSectIterator : public SparseIterator {
                        ValueRange itVals) const {
     assert(itVals.size() == tupleSz);
     for (unsigned i = 0; i < tupleSz; i++) {
-      b.create<memref::StoreOp>(l, itVals[i], subSectPosBuf,
+      memref::StoreOp::create(b, l, itVals[i], subSectPosBuf,
                                 ValueRange{tupleId, C_IDX(i)});
     }
   }
@@ -762,7 +762,7 @@ class NonEmptySubSectIterator : public SparseIterator {
                                     Value tupleId) const {
     SmallVector<Value> ret;
     for (unsigned i = 0; i < tupleSz; i++) {
-      Value v = b.create<memref::LoadOp>(l, subSectPosBuf,
+      Value v = memref::LoadOp::create(b, l, subSectPosBuf,
                                          ValueRange{tupleId, C_IDX(i)});
       ret.push_back(v);
     }
@@ -1043,7 +1043,7 @@ ValueRange SparseIterator::forward(OpBuilder &b, Location l) {
 }
 
 ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
-  auto ifOp = b.create<scf::IfOp>(l, getCursor().getTypes(), cond, true);
+  auto ifOp = scf::IfOp::create(b, l, getCursor().getTypes(), cond, true);
   // Generate else branch first, otherwise iterator values will be updated by
   // `forward()`.
   b.setInsertionPointToStart(ifOp.elseBlock());
@@ -1058,12 +1058,12 @@ ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
 }
 
 Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
-  auto whileOp = b.create<scf::WhileOp>(
+  auto whileOp = scf::WhileOp::create(b,
       l, pos.getType(), pos,
       /*beforeBuilder=*/
       [this, pos](OpBuilder &b, Location l, ValueRange ivs) {
         Value inBound = CMPI(ult, ivs.front(), posHi);
-        auto ifInBound = b.create<scf::IfOp>(l, b.getI1Type(), inBound, true);
+        auto ifInBound = scf::IfOp::create(b, l, b.getI1Type(), inBound, true);
         {
           OpBuilder::InsertionGuard guard(b);
           // If in bound, load the next coordinates and check duplication.
@@ -1076,7 +1076,7 @@ Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
           b.setInsertionPointToStart(ifInBound.elseBlock());
           YIELD(constantI1(b, l, false));
         }
-        b.create<scf::ConditionOp>(l, ifInBound.getResults()[0], ivs);
+        scf::ConditionOp::create(b, l, ifInBound.getResults()[0], ivs);
       },
       /*afterBuilder=*/
       [](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1137,7 +1137,7 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
 
   SmallVector<Value> whileArgs(getCursor().begin(), getCursor().end());
   whileArgs.push_back(isFirst);
-  auto whileOp = b.create<scf::WhileOp>(
+  auto whileOp = scf::WhileOp::create(b,
       l, ValueRange(whileArgs).getTypes(), whileArgs,
       /*beforeBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1154,7 +1154,7 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
                              ret = ORI(ret, llvm::getSingleElement(isFirst));
                              return {ret};
                            });
-        b.create<scf::ConditionOp>(l, cont.front(), ivs);
+        scf::ConditionOp::create(b, l, cont.front(), ivs);
       },
       /*afterBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1219,7 +1219,7 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
     SmallVector<Value> iterArgs;
     iterArgs.push_back(C_IDX(0));
     iterArgs.append(reduc.begin(), reduc.end());
-    auto forEachLeaf = b.create<scf::ForOp>(
+    auto forEachLeaf = scf::ForOp::create(b,
         l, /*lb=*/C_IDX(0), /*ub=*/tupleCnt, /*step=*/C_IDX(1), iterArgs,
         [&helper, &builder](OpBuilder &b, Location l, Value tupleId,
                             ValueRange iterArgs) {
@@ -1235,12 +1235,12 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
           SmallVector<Value> whileArgs(helper.wrap.getCursor());
           whileArgs.append(iterArgs.begin(), iterArgs.end());
 
-          auto whileOp = b.create<scf::WhileOp>(
+          auto whileOp = scf::WhileOp::create(b,
               l, ValueRange(whileArgs).getTypes(), whileArgs,
               /*beforeBuilder=*/
               [&helper](OpBuilder &b, Location l, ValueRange ivs) {
                 helper.wrap.linkNewScope(ivs);
-                b.create<scf::ConditionOp>(l, helper.genNotEnd(b, l), ivs);
+                scf::ConditionOp::create(b, l, helper.genNotEnd(b, l), ivs);
               },
               /*afterBuilder=*/
               [&helper, &builder](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1267,7 +1267,7 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
                                      ValueRange reduc) {
     assert(!parent || parent->lvl + 1 == lvl);
     delegate->genInit(b, l, parent);
-    auto forOp = b.create<scf::ForOp>(
+    auto forOp = scf::ForOp::create(b,
         l, /*lb=*/C_IDX(0), /*ub=*/subSectSz, /*step=*/C_IDX(1), reduc,
         [&](OpBuilder &b, Location l, Value crd, ValueRange iterArgs) {
           helper.locate(b, l, crd);
@@ -1411,7 +1411,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
   // if (offset + size > parents.size)
   //   isNonEmpty = false;
   Value fastPathP = CMPI(ugt, getMinCrd(), getAbsOff());
-  auto ifOp = b.create<scf::IfOp>(l, getCursor().getTypes(), fastPathP, true);
+  auto ifOp = scf::IfOp::create(b, l, getCursor().getTypes(), fastPathP, true);
   {
     OpBuilder::InsertionGuard guard(b);
     // Take the fast path
@@ -1448,7 +1448,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 Value isMin = CMPI(eq, crd, getMinCrd());
                 delegate->forwardIf(b, l, isMin);
                 // Update the forwarded iterator values if needed.
-                auto ifIsMin = b.create<scf::IfOp>(l, isMin, false);
+                auto ifIsMin = scf::IfOp::create(b, l, isMin, false);
                 b.setInsertionPointToStart(&ifIsMin.getThenRegion().front());
                 storeCursorVals(b, l, tupleId, delegate->serialize());
                 b.setInsertionPointAfter(ifIsMin);
@@ -1458,7 +1458,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 return genWhenInBound(b, l, *delegate, /*elseRet=*/iterArgs,
                                       [nxMin](OpBuilder &b, Location l,
                                               Value crd) -> scf::ValueVector {
-                                        Value nx = b.create<arith::MinUIOp>(
+                                        Value nx = arith::MinUIOp::create(b,
                                             l, crd, nxMin);
                                         return {nx, C_TRUE};
                                       });
@@ -1480,7 +1480,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
 
   // We should at least forward the offset by one.
   Value minAbsOff = ADDI(getAbsOff(), c1);
-  nxAbsOff = b.create<arith::MaxUIOp>(l, minAbsOff, nxAbsOff);
+  nxAbsOff = arith::MaxUIOp::create(b, l, minAbsOff, nxAbsOff);
 
   seek(ValueRange{nxMinCrd, nxAbsOff, nxNotEnd});
   // The coordinate should not exceeds the space upper bound.
@@ -1581,16 +1581,16 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &b, Location l, Value t,
   auto stt = getSparseTensorType(t);
 
   LevelType lt = stt.getLvlType(lvl);
-  Value sz = stt.hasEncoding() ? b.create<LvlOp>(l, t, lvl).getResult()
-                               : b.create<tensor::DimOp>(l, t, lvl).getResult();
+  Value sz = stt.hasEncoding() ? LvlOp::create(b, l, t, lvl).getResult()
+                               : tensor::DimOp::create(b, l, t, lvl).getResult();
 
   SmallVector<Value, 2> buffers;
   if (lt.isWithPosLT()) {
-    Value pos = b.create<ToPositionsOp>(l, t, lvl);
+    Value pos = ToPositionsOp::create(b, l, t, lvl);
     buffers.push_back(pos);
   }
   if (lt.isWithCrdLT()) {
-    Value pos = b.create<ToCoordinatesOp>(l, t, lvl);
+    Value pos = ToCoordinatesOp::create(b, l, t, lvl);
     buffers.push_back(pos);
   }
   return makeSparseTensorLevel(lt, sz, buffers, tid, lvl);
diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
index 0258f797143cb..fb68620c378e5 100644
--- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
@@ -1563,7 +1563,7 @@ static Value insertYieldOp(RewriterBase &rewriter, Location loc, Region &region,
   Block &clonedBlock = tmpRegion.front();
   YieldOp clonedYield = cast<YieldOp>(clonedBlock.getTerminator());
   // Merge cloned block and return yield value.
-  Operation *placeholder = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Operation *placeholder = arith::ConstantIndexOp::create(rewriter, loc, 0);
   rewriter.inlineBlockBefore(&tmpRegion.front(), placeholder, vals);
   Value val = clonedYield.getSingleResult();
   rewriter.eraseOp(clonedYield);
@@ -1603,16 +1603,16 @@ static Value buildRelu(RewriterBase &rewriter, Location loc, Value v0,
                        Attribute attr) {
   Type tp = v0.getType();
   auto zero =
-      rewriter.create<arith::ConstantOp>(loc, tp, rewriter.getZeroAttr(tp));
+      arith::ConstantOp::create(rewriter, loc, tp, rewriter.getZeroAttr(tp));
   Value cmp;
   if (isa<FloatType>(tp)) {
     auto pred = llvm::cast<arith::CmpFPredicateAttr>(attr);
-    cmp = rewriter.create<arith::CmpFOp>(loc, pred, v0, zero);
+    cmp = arith::CmpFOp::create(rewriter, loc, pred, v0, zero);
   } else {
     auto pred = llvm::cast<arith::CmpIPredicateAttr>(attr);
-    cmp = rewriter.create<arith::CmpIOp>(loc, pred, v0, zero);
+    cmp = arith::CmpIOp::create(rewriter, loc, pred, v0, zero);
   }
-  return rewriter.create<arith::SelectOp>(loc, cmp, v0, zero);
+  return arith::SelectOp::create(rewriter, loc, cmp, v0, zero);
 }
 
 Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
@@ -1627,128 +1627,128 @@ Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
     llvm_unreachable("unexpected non-op");
   // Unary operations.
   case TensorExp::Kind::kAbsF:
-    return rewriter.create<math::AbsFOp>(loc, v0);
+    return math::AbsFOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kAbsC: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::AbsOp>(loc, eltType, v0);
+    return complex::AbsOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kAbsI:
-    return rewriter.create<math::AbsIOp>(loc, v0);
+    return math::AbsIOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kCeilF:
-    return rewriter.create<math::CeilOp>(loc, v0);
+    return math::CeilOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kFloorF:
-    return rewriter.create<math::FloorOp>(loc, v0);
+    return math::FloorOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSqrtF:
-    return rewriter.create<math::SqrtOp>(loc, v0);
+    return math::SqrtOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSqrtC:
-    return rewriter.create<complex::SqrtOp>(loc, v0);
+    return complex::SqrtOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kExpm1F:
-    return rewriter.create<math::ExpM1Op>(loc, v0);
+    return math::ExpM1Op::create(rewriter, loc, v0);
   case TensorExp::Kind::kExpm1C:
-    return rewriter.create<complex::Expm1Op>(loc, v0);
+    return complex::Expm1Op::create(rewriter, loc, v0);
   case TensorExp::Kind::kLog1pF:
-    return rewriter.create<math::Log1pOp>(loc, v0);
+    return math::Log1pOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kLog1pC:
-    return rewriter.create<complex::Log1pOp>(loc, v0);
+    return complex::Log1pOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kRelu:
     return buildRelu(rewriter, loc, v0, expr.attr);
   case TensorExp::Kind::kSinF:
-    return rewriter.create<math::SinOp>(loc, v0);
+    return math::SinOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSinC:
-    return rewriter.create<complex::SinOp>(loc, v0);
+    return complex::SinOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kTanhF:
-    return rewriter.create<math::TanhOp>(loc, v0);
+    return math::TanhOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kTanhC:
-    return rewriter.create<complex::TanhOp>(loc, v0);
+    return complex::TanhOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegF:
-    return rewriter.create<arith::NegFOp>(loc, v0);
+    return arith::NegFOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegC:
-    return rewriter.create<complex::NegOp>(loc, v0);
+    return complex::NegOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegI: // no negi in std
-    return rewriter.create<arith::SubIOp>(
+    return arith::SubIOp::create(rewriter,
         loc,
-        rewriter.create<arith::ConstantOp>(loc, v0.getType(),
+        arith::ConstantOp::create(rewriter, loc, v0.getType(),
                                            rewriter.getZeroAttr(v0.getType())),
         v0);
   case TensorExp::Kind::kTruncF:
-    return rewriter.create<arith::TruncFOp>(loc, inferType(e, v0), v0);
+    return arith::TruncFOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kExtF:
-    return rewriter.create<arith::ExtFOp>(loc, inferType(e, v0), v0);
+    return arith::ExtFOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastFS:
-    return rewriter.create<arith::FPToSIOp>(loc, inferType(e, v0), v0);
+    return arith::FPToSIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastFU:
-    return rewriter.create<arith::FPToUIOp>(loc, inferType(e, v0), v0);
+    return arith::FPToUIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastSF:
-    return rewriter.create<arith::SIToFPOp>(loc, inferType(e, v0), v0);
+    return arith::SIToFPOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastUF:
-    return rewriter.create<arith::UIToFPOp>(loc, inferType(e, v0), v0);
+    return arith::UIToFPOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastS:
-    return rewriter.create<arith::ExtSIOp>(loc, inferType(e, v0), v0);
+    return arith::ExtSIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastU:
-    return rewriter.create<arith::ExtUIOp>(loc, inferType(e, v0), v0);
+    return arith::ExtUIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastIdx:
-    return rewriter.create<arith::IndexCastOp>(loc, inferType(e, v0), v0);
+    return arith::IndexCastOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kTruncI:
-    return rewriter.create<arith::TruncIOp>(loc, inferType(e, v0), v0);
+    return arith::TruncIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCIm: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::ImOp>(loc, eltType, v0);
+    return complex::ImOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kCRe: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::ReOp>(loc, eltType, v0);
+    return complex::ReOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kBitCast:
-    return rewriter.create<arith::BitcastOp>(loc, inferType(e, v0), v0);
+    return arith::BitcastOp::create(rewriter, loc, inferType(e, v0), v0);
   // Binary operations.
   case TensorExp::Kind::kMulF:
-    return rewriter.create<arith::MulFOp>(loc, v0, v1);
+    return arith::MulFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kMulC:
-    return rewriter.create<complex::MulOp>(loc, v0, v1);
+    return complex::MulOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kMulI:
-    return rewriter.create<arith::MulIOp>(loc, v0, v1);
+    return arith::MulIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivF:
-    return rewriter.create<arith::DivFOp>(loc, v0, v1);
+    return arith::DivFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivC:
-    return rewriter.create<complex::DivOp>(loc, v0, v1);
+    return complex::DivOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivS:
-    return rewriter.create<arith::DivSIOp>(loc, v0, v1);
+    return arith::DivSIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivU:
-    return rewriter.create<arith::DivUIOp>(loc, v0, v1);
+    return arith::DivUIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddF:
-    return rewriter.create<arith::AddFOp>(loc, v0, v1);
+    return arith::AddFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddC:
-    return rewriter.create<complex::AddOp>(loc, v0, v1);
+    return complex::AddOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddI:
-    return rewriter.create<arith::AddIOp>(loc, v0, v1);
+    return arith::AddIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubF:
-    return rewriter.create<arith::SubFOp>(loc, v0, v1);
+    return arith::SubFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubC:
-    return rewriter.create<complex::SubOp>(loc, v0, v1);
+    return complex::SubOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubI:
-    return rewriter.create<arith::SubIOp>(loc, v0, v1);
+    return arith::SubIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAndI:
-    return rewriter.create<arith::AndIOp>(loc, v0, v1);
+    return arith::AndIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kOrI:
-    return rewriter.create<arith::OrIOp>(loc, v0, v1);
+    return arith::OrIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kXorI:
-    return rewriter.create<arith::XOrIOp>(loc, v0, v1);
+    return arith::XOrIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShrS:
-    return rewriter.create<arith::ShRSIOp>(loc, v0, v1);
+    return arith::ShRSIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShrU:
-    return rewriter.create<arith::ShRUIOp>(loc, v0, v1);
+    return arith::ShRUIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShlI:
-    return rewriter.create<arith::ShLIOp>(loc, v0, v1);
+    return arith::ShLIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kCmpI: {
     auto predicate = llvm::cast<arith::CmpIPredicateAttr>(expr.attr);
-    return rewriter.create<arith::CmpIOp>(loc, predicate, v0, v1);
+    return arith::CmpIOp::create(rewriter, loc, predicate, v0, v1);
   }
   case TensorExp::Kind::kCmpF: {
     auto predicate = llvm::cast<arith::CmpFPredicateAttr>(expr.attr);
-    return rewriter.create<arith::CmpFOp>(loc, predicate, v0, v1);
+    return arith::CmpFOp::create(rewriter, loc, predicate, v0, v1);
   }
   case TensorExp::Kind::kBinaryBranch: // semi-ring ops with custom logic.
     return insertYieldOp(rewriter, loc, *expr.op->getBlock()->getParent(),
diff --git a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
index 26406ceef082c..78eb50cbdf540 100644
--- a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
+++ b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
@@ -74,11 +74,11 @@ struct CreatorOpShardingInterface
         if (!oldType.isDynamicDim(i) && shardType.isDynamicDim(i)) {
           if (!newSharding) {
             newSharding =
-                builder.create<ShardingOp>(op->getLoc(), resultShardings[0]);
+                ShardingOp::create(builder, op->getLoc(), resultShardings[0]);
             device =
-                builder.create<mesh::ProcessMultiIndexOp>(op->getLoc(), mesh)
+                mesh::ProcessMultiIndexOp::create(builder, op->getLoc(), mesh)
                     .getResults();
-            shapeForDevice = builder.create<mesh::ShardShapeOp>(
+            shapeForDevice = mesh::ShardShapeOp::create(builder,
                 op->getLoc(), oldType.getShape(), spmdizedOperands,
                 newSharding->getResult(0), device);
           }
@@ -88,7 +88,7 @@ struct CreatorOpShardingInterface
           newOperands.emplace_back(spmdizedOperands[++currOldOprndNum]);
         }
       }
-      newOp = builder.create<OpTy>(op->getLoc(), shardType, newOperands);
+      newOp = OpTy::create(builder, op->getLoc(), shardType, newOperands);
       spmdizationMap.map(op->getResult(0), newOp->getResult(0));
     } else {
       // `clone` will populate the mapping of old to new results.
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index b035a53692dcf..df34d9e37bd3f 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -53,7 +53,7 @@ Operation *TensorDialect::materializeConstant(OpBuilder &builder,
   if (auto op = arith::ConstantOp::materialize(builder, value, type, loc))
     return op;
   if (complex::ConstantOp::isBuildableWith(value, type))
-    return builder.create<complex::ConstantOp>(loc, type,
+    return complex::ConstantOp::create(builder, loc, type,
                                                llvm::cast<ArrayAttr>(value));
   return nullptr;
 }
@@ -107,7 +107,7 @@ FailureOr<Value> tensor::getOrCreateDestination(OpBuilder &b, Location loc,
 
   // Create empty tensor.
   Value emptyTensor =
-      b.create<tensor::EmptyOp>(loc, mixedSizes, tensorType.getElementType());
+      tensor::EmptyOp::create(b, loc, mixedSizes, tensorType.getElementType());
   return emptyTensor;
 }
 
@@ -678,7 +678,7 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
     inputShapes.emplace_back(std::move(inputShape));
   }
 
-  Value replacement = builder.create<tensor::EmptyOp>(
+  Value replacement = tensor::EmptyOp::create(builder,
       loc, outputShape, getType().getElementType());
 
   int64_t rank = getType().getRank();
@@ -687,12 +687,12 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
   SmallVector<OpFoldResult> offsets(rank, zero);
   for (auto [index, input] : llvm::enumerate(getInputs())) {
     offsets[concatDim] = concatOffsets[index];
-    auto insertSlice = builder.create<tensor::InsertSliceOp>(
+    auto insertSlice = tensor::InsertSliceOp::create(builder,
         loc, input, replacement, offsets, inputShapes[index], strides);
     replacement = insertSlice.getResult();
   }
   if (replacement.getType() != getType()) {
-    replacement = builder.create<tensor::CastOp>(loc, getType(), replacement);
+    replacement = tensor::CastOp::create(builder, loc, getType(), replacement);
   }
   return SmallVector<Value>{replacement};
 }
@@ -723,7 +723,7 @@ ConcatOp::reifyResultShapes(OpBuilder &builder,
           builder.getIndexAttr(inferredResultType.getDimSize(i)));
     } else {
       reifiedReturnShapes[0][i] =
-          builder.create<tensor::DimOp>(init.getLoc(), init, i).getResult();
+          tensor::DimOp::create(builder, init.getLoc(), init, i).getResult();
     }
   }
 
@@ -823,7 +823,7 @@ struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
 
         // Use refined operand type and create cast from original operand.
         auto castOp =
-            rewriter.create<CastOp>(concatOp->getLoc(), inferredOperandType,
+            CastOp::create(rewriter, concatOp->getLoc(), inferredOperandType,
                                     concatOp.getOperand(operandIdx));
         rewriter.modifyOpInPlace(concatOp, [=, operandIdx = operandIdx] {
           concatOp->setOperand(operandIdx, castOp->getResult(0));
@@ -864,7 +864,7 @@ struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
       return failure();
     }
 
-    auto newConcatOp = rewriter.create<ConcatOp>(
+    auto newConcatOp = ConcatOp::create(rewriter,
         concatOp->getLoc(), inferredResultType, dim, concatOp->getOperands());
     rewriter.replaceOpWithNewOp<CastOp>(concatOp, concatOp.getResultType(),
                                         newConcatOp);
@@ -892,7 +892,7 @@ void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
 void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
                   int64_t index) {
   auto loc = result.location;
-  Value indexValue = builder.create<arith::ConstantIndexOp>(loc, index);
+  Value indexValue = arith::ConstantIndexOp::create(builder, loc, index);
   build(builder, result, source, indexValue);
 }
 
@@ -1036,10 +1036,10 @@ struct DimOfReshapeOp : public OpRewritePattern<DimOp> {
     rewriter.setInsertionPointAfter(dim);
     Location loc = dim.getLoc();
     Value extract =
-        rewriter.create<ExtractOp>(loc, reshape.getShape(), dim.getIndex());
+        ExtractOp::create(rewriter, loc, reshape.getShape(), dim.getIndex());
     if (extract.getType() != dim.getType())
       extract =
-          rewriter.create<arith::IndexCastOp>(loc, dim.getType(), extract);
+          arith::IndexCastOp::create(rewriter, loc, dim.getType(), extract);
     rewriter.replaceOp(dim, extract);
     return success();
   }
@@ -1150,7 +1150,7 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
     if (foldedTensorType == op.getType())
       return failure();
 
-    auto newOp = rewriter.create<EmptyOp>(op.getLoc(), foldedTensorType,
+    auto newOp = EmptyOp::create(rewriter, op.getLoc(), foldedTensorType,
                                           foldedDynamicSizes);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
@@ -1326,7 +1326,7 @@ struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
 
       SmallVector<int64_t> basis =
           llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-      auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+      auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
           extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
       llvm::append_range(sourceIndices, delinearize.getResults());
     }
@@ -1498,7 +1498,7 @@ struct ExtractElementFromIndexCast
 
     Type elementTy = getElementTypeOrSelf(indexCast.getIn());
 
-    auto newExtract = rewriter.create<tensor::ExtractOp>(
+    auto newExtract = tensor::ExtractOp::create(rewriter,
         loc, elementTy, indexCast.getIn(), extract.getIndices());
 
     rewriter.replaceOpWithNewOp<arith::IndexCastOp>(extract, extract.getType(),
@@ -1736,7 +1736,7 @@ struct StaticTensorGenerate : public OpRewritePattern<GenerateOp> {
 
     auto loc = generateOp.getLoc();
     auto newOp =
-        rewriter.create<GenerateOp>(loc, foldedTensorType, foldedDynamicSizes);
+        GenerateOp::create(rewriter, loc, foldedTensorType, foldedDynamicSizes);
     rewriter.inlineRegionBefore(generateOp.getBody(), newOp.getBody(),
                                 newOp.getBody().begin());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(generateOp,
@@ -2161,7 +2161,7 @@ struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
         collapseShapeOp.getSrcMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = rewriter.create<CollapseShapeOp>(
+      auto newOp = CollapseShapeOp::create(rewriter,
           collapseShapeOp.getLoc(), newResultType, castOp.getSource(),
           collapseShapeOp.getReassociation());
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
@@ -2240,9 +2240,9 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
         newInputShape, expandOp.getSrcType().getElementType());
     auto outputType = RankedTensorType::get(
         newOutputShape, expandOp.getSrcType().getElementType());
-    auto inputCast = rewriter.create<CastOp>(expandOp.getLoc(), inputType,
+    auto inputCast = CastOp::create(rewriter, expandOp.getLoc(), inputType,
                                              expandOp.getSrc());
-    auto newExpand = rewriter.create<ExpandShapeOp>(
+    auto newExpand = ExpandShapeOp::create(rewriter,
         expandOp.getLoc(), outputType, inputCast.getResult(),
         expandOp.getReassociationIndices(), outputOfr);
     rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
@@ -2555,7 +2555,7 @@ class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
 
     // Create folded extract.
     Location loc = sliceOp.getLoc();
-    Value newResult = rewriter.create<ExtractSliceOp>(
+    Value newResult = ExtractSliceOp::create(rewriter,
         loc, sliceOp.getType(), castOp.getSource(), sliceOp.getOffsets(),
         sliceOp.getSizes(), sliceOp.getStrides(), sliceOp.getStaticOffsets(),
         sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
@@ -2709,7 +2709,7 @@ struct SliceCanonicalizer {
                   ExtractSliceOp newOp) {
     Value replacement = newOp.getResult();
     if (replacement.getType() != op.getType())
-      replacement = rewriter.create<tensor::CastOp>(op.getLoc(), op.getType(),
+      replacement = tensor::CastOp::create(rewriter, op.getLoc(), op.getType(),
                                                     replacement);
     rewriter.replaceOp(op, replacement);
   }
@@ -2978,7 +2978,7 @@ class InsertSliceOpConstantArgumentFolder final
       // the parallel case.
       if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      toInsert = rewriter.create<tensor::CastOp>(insertSliceOp.getLoc(),
+      toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
                                                  sourceType, toInsert);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
@@ -3075,7 +3075,7 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     if (!sliceResult.isValid)
       return failure();
 
-    Operation *replacement = rewriter.create<InsertOpTy>(
+    Operation *replacement = InsertOpTy::create(rewriter,
         insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
         mixedSizes, insertSliceOp.getMixedStrides());
 
@@ -3083,7 +3083,7 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     bool isParallelInsert =
         std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
     if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
-      replacement = rewriter.create<tensor::CastOp>(insertSliceOp.getLoc(),
+      replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
                                                     insertSliceOp.getDestType(),
                                                     replacement->getResult(0));
     }
@@ -3154,7 +3154,7 @@ struct InsertSliceOpSourceCastInserter final
     // parallel case.
     if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
       rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-    Value cast = rewriter.create<tensor::CastOp>(
+    Value cast = tensor::CastOp::create(rewriter,
         insertSliceOp.getLoc(), newSrcType, insertSliceOp.getSource());
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, cast, insertSliceOp.getDest(),
@@ -3353,7 +3353,7 @@ void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
   // a guard to reset the insertion point of the builder after it is destroyed.
   OpBuilder::InsertionGuard guard(b);
   b.createBlock(region, region->end(), blockArgTypes, blockArgLocs);
-  b.create<tensor::YieldOp>(result.location, constantPadValue);
+  tensor::YieldOp::create(b, result.location, constantPadValue);
 }
 
 llvm::SmallBitVector PadOp::getPaddedDims() {
@@ -3407,7 +3407,7 @@ struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
         padTensorOp.getSourceMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = rewriter.create<PadOp>(
+      auto newOp = PadOp::create(rewriter,
           padTensorOp->getLoc(), newResultType, padTensorOp.getSource(),
           padTensorOp.getStaticLow(), padTensorOp.getStaticHigh(),
           padTensorOp.getLow(), padTensorOp.getHigh(), padTensorOp.getNofold(),
@@ -3439,7 +3439,7 @@ struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
                                             tensorCastOp.getDest().getType()))
       return failure();
 
-    auto replacementOp = rewriter.create<PadOp>(
+    auto replacementOp = PadOp::create(rewriter,
         padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
         padTensorOp.getSource(), padTensorOp.getStaticLow(),
         padTensorOp.getStaticHigh(), padTensorOp.getLow(),
@@ -3597,10 +3597,10 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
 
     // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
     // the two paddings in one step.
-    auto newSliceOp = rewriter.create<ExtractSliceOp>(
+    auto newSliceOp = ExtractSliceOp::create(rewriter,
         padOp.getLoc(), outerSliceOp.getSource(), newOffsets, newSizes,
         innerSliceOp.getMixedStrides());
-    auto newPadOp = rewriter.create<PadOp>(
+    auto newPadOp = PadOp::create(rewriter,
         padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
         padOp.getMixedLowPad(), newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
@@ -3697,7 +3697,7 @@ struct FoldStaticPadding : public OpRewritePattern<PadOp> {
     // Rewrite the op using the new static type.
     auto newResultType = RankedTensorType::get(
         newOutDims, padTensorOp.getType().getElementType());
-    auto newOp = rewriter.create<PadOp>(
+    auto newOp = PadOp::create(rewriter,
         padTensorOp->getLoc(), newResultType, input, staticLow, staticHigh,
         newLows, newHighs, padTensorOp.getNofold(),
         getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
@@ -3777,7 +3777,7 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
     SmallVector<OpFoldResult> newLowPad =
         addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         padOp.getLoc(), padOp.getResultType(), producerPad.getSource(),
         newLowPad, newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, tensor::PadOp::getAttributeNames()));
@@ -3803,7 +3803,7 @@ PadOp::reifyResultShapes(OpBuilder &b,
     }
     Location loc = getLoc();
     Value dim = b.createOrFold<tensor::DimOp>(
-        loc, getSource(), b.create<arith::ConstantIndexOp>(loc, i));
+        loc, getSource(), arith::ConstantIndexOp::create(b, loc, i));
 
     AffineExpr d0, d1, d2;
     bindDims(b.getContext(), d0, d1, d2);
@@ -4108,7 +4108,7 @@ struct FoldTensorCastProducerOp
     for (auto [oldResult, newResult] :
          llvm::zip(op->getResults(), newOp->getResults())) {
       if (newResult.getType() != oldResult.getType()) {
-        replacements.push_back(rewriter.create<tensor::CastOp>(
+        replacements.push_back(tensor::CastOp::create(rewriter,
             op->getLoc(), oldResult.getType(), newResult));
       } else {
         replacements.push_back(newResult);
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 437bc5d00faa8..0a4fa7bfa007f 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -207,13 +207,13 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     if (isZeroInteger(newLength)) {
       hasZeroLen = true;
     } else if (!hasZeroLen) {
-      Value check = b.create<arith::CmpIOp>(
+      Value check = arith::CmpIOp::create(b,
           loc, arith::CmpIPredicate::eq,
           getValueOrCreateConstantIndexOp(b, loc, newLength),
           getValueOrCreateConstantIndexOp(b, loc, zero));
       dynHasZeroLenCond =
           dynHasZeroLenCond
-              ? b.create<arith::OrIOp>(loc, check, dynHasZeroLenCond)
+              ? arith::OrIOp::create(b, loc, check, dynHasZeroLenCond)
               : check;
     }
 
@@ -237,7 +237,7 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   auto castResult = [&](Value val) -> Value {
     if (resultType == val.getType())
       return val;
-    return b.create<tensor::CastOp>(loc, resultType, val);
+    return tensor::CastOp::create(b, loc, resultType, val);
   };
 
   // In cases where the original data source is unused: Emit a GenerateOp and
@@ -245,10 +245,10 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // have a dimension of size 0, the semantics of which is unclear.)
   auto createGenerateOp = [&]() {
     // Create GenerateOp.
-    auto generateOp = b.create<tensor::GenerateOp>(
+    auto generateOp = tensor::GenerateOp::create(b,
         loc, resultType, dynDims,
         [&](OpBuilder &builder, Location gLoc, ValueRange indices) {
-          builder.create<tensor::YieldOp>(gLoc, padValue);
+          tensor::YieldOp::create(builder, gLoc, padValue);
         });
     return generateOp;
   };
@@ -257,9 +257,9 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // the result shape of the new SliceOp has a zero dimension.
   auto createPadOfExtractSlice = [&]() {
     // Create pad(extract_slice(x)).
-    auto newSliceOp = b.create<tensor::ExtractSliceOp>(
+    auto newSliceOp = tensor::ExtractSliceOp::create(b,
         loc, padOp.getSource(), newOffsets, newLengths, newStrides);
-    auto newPadOp = b.create<PadOp>(
+    auto newPadOp = PadOp::create(b,
         loc, Type(), newSliceOp, newLows, newHighs,
         /*nofold=*/padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
@@ -287,17 +287,17 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     Operation *thenOp;
     Operation *elseOp;
     Operation *sliceOp;
-    auto result = b.create<scf::IfOp>(
+    auto result = scf::IfOp::create(b,
         loc, dynHasZeroLenCond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
           thenOp = createGenerateOp();
-          b.create<scf::YieldOp>(loc, castResult(thenOp->getResult(0)));
+          scf::YieldOp::create(b, loc, castResult(thenOp->getResult(0)));
         },
         /*elseBuilder=*/
         [&](OpBuilder &b, Location loc) {
           std::tie(elseOp, sliceOp) = createPadOfExtractSlice();
-          b.create<scf::YieldOp>(loc, castResult(elseOp->getResult(0)));
+          scf::YieldOp::create(b, loc, castResult(elseOp->getResult(0)));
         });
     return TilingResult{
         {elseOp}, SmallVector<Value>(result->getResults()), {sliceOp}};
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index 723731b8bed61..ba258402b61b3 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -165,7 +165,7 @@ void transform::TypeConversionCastShapeDynamicDimsOp::
     if (!tensor::CastOp::areCastCompatible(input.getType(), resultType)) {
       return Value();
     }
-    return builder.create<tensor::CastOp>(loc, resultType, input).getResult();
+    return tensor::CastOp::create(builder, loc, resultType, input).getResult();
   });
   converter.addTargetMaterialization([](OpBuilder &builder, Type resultType,
                                         ValueRange inputs,
@@ -177,7 +177,7 @@ void transform::TypeConversionCastShapeDynamicDimsOp::
     if (!tensor::CastOp::areCastCompatible(input.getType(), resultType)) {
       return Value();
     }
-    return builder.create<tensor::CastOp>(loc, resultType, input).getResult();
+    return tensor::CastOp::create(builder, loc, resultType, input).getResult();
   });
 }
 
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index 47b41efbed83b..e5b4b7a865d87 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -222,7 +222,7 @@ struct CollapseShapeOpInterface
           MemRefType::get(collapseShapeOp.getSrcType().getShape(),
                           collapseShapeOp.getSrcType().getElementType(),
                           AffineMap(), bufferType.getMemorySpace());
-      buffer = rewriter.create<bufferization::ToBufferOp>(
+      buffer = bufferization::ToBufferOp::create(rewriter,
           op->getLoc(), memrefType, *tensorAlloc);
     }
 
@@ -349,7 +349,7 @@ struct ExpandShapeOpInterface
     if (failed(buffer))
       return failure();
 
-    auto memrefExpandShape = rewriter.create<memref::ExpandShapeOp>(
+    auto memrefExpandShape = memref::ExpandShapeOp::create(rewriter,
         op->getLoc(), tensorResultType.getShape(), *buffer,
         expandShapeOp.getReassociationIndices(),
         expandShapeOp.getMixedOutputShape());
@@ -398,7 +398,7 @@ struct ExtractSliceOpInterface
         extractSliceOp.getResult(), options, state);
     if (failed(resultMemrefType))
       return failure();
-    Value subView = rewriter.create<memref::SubViewOp>(
+    Value subView = memref::SubViewOp::create(rewriter,
         loc, llvm::cast<MemRefType>(*resultMemrefType), *srcMemref,
         mixedOffsets, mixedSizes, mixedStrides);
 
@@ -469,7 +469,7 @@ static void createStores(RewriterBase &rewriter, Location loc, int dim,
   if (dim == static_cast<int>(shape.size()) - 1) {
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
-      rewriter.create<memref::StoreOp>(loc, *elementIt, buffer, indices);
+      memref::StoreOp::create(rewriter, loc, *elementIt, buffer, indices);
       ++elementIt;
     }
     return;
@@ -507,7 +507,7 @@ struct FromElementsOpInterface
         bufferization::getBufferType(*tensorAlloc, options, state);
     if (failed(memrefType))
       return failure();
-    Value buffer = rewriter.create<bufferization::ToBufferOp>(
+    Value buffer = bufferization::ToBufferOp::create(rewriter,
         op->getLoc(), *memrefType, *tensorAlloc);
 
     // Case: tensor<0xelem_type>.
@@ -518,7 +518,7 @@ struct FromElementsOpInterface
 
     // Case: tensor<elem_type>.
     if (shape.empty()) {
-      rewriter.create<memref::StoreOp>(
+      memref::StoreOp::create(rewriter,
           loc, fromElementsOp.getElements().front(), buffer);
       replaceOpWithBufferizedValues(rewriter, op, buffer);
       return success();
@@ -529,7 +529,7 @@ struct FromElementsOpInterface
     SmallVector<Value, 2> constants;
     constants.reserve(maxDim);
     for (int i = 0; i < maxDim; ++i)
-      constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
+      constants.push_back(arith::ConstantIndexOp::create(rewriter, loc, i));
 
     // Traverse all `elements` and create `memref.store` ops.
     auto elementIt = fromElementsOp.getElements().begin();
@@ -576,7 +576,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   // Create linalg::MapOp.
   OpBuilder::InsertionGuard g(rewriter);
   auto linalgOp =
-      rewriter.create<linalg::MapOp>(loc, tensorType, /*inputs=*/ValueRange(),
+      linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
                                      /*init=*/tensorDestination);
   Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
@@ -584,7 +584,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   rewriter.setInsertionPointToStart(&linalgBody);
   SmallVector<Value> indices;
   for (int64_t dim = 0; dim < tensorType.getRank(); ++dim)
-    indices.push_back(rewriter.create<linalg::IndexOp>(loc, dim));
+    indices.push_back(linalg::IndexOp::create(rewriter, loc, dim));
 
   // Move over body.
   rewriter.mergeBlocks(&generateBody.front(), &linalgBody, indices);
@@ -644,7 +644,7 @@ struct InsertOpInterface
         getBuffer(rewriter, insertOp.getDest(), options, state);
     if (failed(destMemref))
       return failure();
-    rewriter.create<memref::StoreOp>(insertOp.getLoc(), insertOp.getScalar(),
+    memref::StoreOp::create(rewriter, insertOp.getLoc(), insertOp.getScalar(),
                                      *destMemref, insertOp.getIndices());
     replaceOpWithBufferizedValues(rewriter, op, *destMemref);
     return success();
@@ -713,7 +713,7 @@ struct InsertSliceOpInterface
         memref::SubViewOp::inferRankReducedResultType(
             insertSliceOp.getSourceType().getShape(), dstMemrefType,
             mixedOffsets, mixedSizes, mixedStrides);
-    Value subView = rewriter.create<memref::SubViewOp>(
+    Value subView = memref::SubViewOp::create(rewriter,
         loc, subviewMemRefType, *dstMemref, mixedOffsets, mixedSizes,
         mixedStrides);
 
@@ -796,13 +796,13 @@ struct PadOpInterface
     for (int64_t i = 0; i < resultType.getRank(); ++i) {
       if (!resultType.isDynamicDim(i))
         continue;
-      Value srcDim = rewriter.create<tensor::DimOp>(loc, padOp.getSource(), i);
+      Value srcDim = tensor::DimOp::create(rewriter, loc, padOp.getSource(), i);
       Value lowPad = toValue(mixedLowPad[i]);
       Value highPad = toValue(mixedHighPad[i]);
       AffineExpr s0, s1, s2;
       bindSymbols(op->getContext(), s0, s1, s2);
       AffineExpr sumExpr = s0 + s1 + s2;
-      Value sum = rewriter.create<affine::AffineApplyOp>(
+      Value sum = affine::AffineApplyOp::create(rewriter,
           loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
       dynamicSizes.push_back(sum);
     }
@@ -995,7 +995,7 @@ struct ParallelInsertSliceOpInterface
             parallelInsertSliceOp.getMixedOffsets(),
             parallelInsertSliceOp.getMixedSizes(),
             parallelInsertSliceOp.getMixedStrides());
-    Value subview = rewriter.create<memref::SubViewOp>(
+    Value subview = memref::SubViewOp::create(rewriter,
         parallelInsertSliceOp.getLoc(), subviewMemRefType, *destBuffer,
         parallelInsertSliceOp.getMixedOffsets(),
         parallelInsertSliceOp.getMixedSizes(),
@@ -1066,13 +1066,13 @@ struct SplatOpInterface
       return op->emitError("memory space not implemented yet");
 
     auto linalgOp =
-        rewriter.create<linalg::MapOp>(loc, tensorType, /*inputs=*/ValueRange(),
+        linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
                                        /*init=*/*tensorAlloc);
     Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
     // Create linalg::IndexOps.
     rewriter.setInsertionPointToStart(&linalgBody);
-    rewriter.create<linalg::YieldOp>(loc, splatOp.getInput());
+    linalg::YieldOp::create(rewriter, loc, splatOp.getInput());
     rewriter.replaceOp(splatOp, linalgOp.getResult()[0]);
 
     return success();
@@ -1126,7 +1126,7 @@ struct ConcatOpInterface
     MemRefType memrefType =
         MemRefType::get(concatOp.getResultType().getShape(),
                         concatOp.getResultType().getElementType(), layout);
-    Value dstBuffer = rewriter.create<bufferization::ToBufferOp>(
+    Value dstBuffer = bufferization::ToBufferOp::create(rewriter,
         op->getLoc(), memrefType, *tensorAlloc);
 
     // Extract the dimension for the concat op
@@ -1142,7 +1142,7 @@ struct ConcatOpInterface
     for (const auto &[dimIdx, dimSize] :
          llvm::enumerate(tensorType.getShape())) {
       if (dimSize == ShapedType::kDynamic) {
-        auto dimOp = rewriter.create<memref::DimOp>(loc, dstBuffer, dimIdx);
+        auto dimOp = memref::DimOp::create(rewriter, loc, dstBuffer, dimIdx);
         sizes.push_back(dimOp.getResult());
         if (dimIdx == concatDim)
           dynamicConcatDim = true;
@@ -1157,7 +1157,7 @@ struct ConcatOpInterface
     if (dynamicConcatDim) {
       // One or more operands have dynamic size, so we must accumulate the
       // offset with arith ops.
-      dynamicOffset = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+      dynamicOffset = arith::ConstantIndexOp::create(rewriter, loc, 0);
     }
 
     for (auto operand : concatOp.getInputs()) {
@@ -1174,7 +1174,7 @@ struct ConcatOpInterface
 
       if (dynamicConcatDim) {
         offsets[concatDim] = dynamicOffset.value();
-        dynamicSize = rewriter.create<memref::DimOp>(loc, *srcBuffer, concatDim)
+        dynamicSize = memref::DimOp::create(rewriter, loc, *srcBuffer, concatDim)
                           .getResult();
         sizes[concatDim] = dynamicSize.value();
       } else {
@@ -1188,7 +1188,7 @@ struct ConcatOpInterface
           memref::SubViewOp::inferRankReducedResultType(
               operandTensorType.getShape(), dstMemrefType, offsets, sizes,
               strides);
-      Value subview = rewriter.create<memref::SubViewOp>(
+      Value subview = memref::SubViewOp::create(rewriter,
           loc, subviewMemRefType, dstBuffer, offsets, sizes, strides);
 
       // Copy the source buffer into the destination subview.
@@ -1196,7 +1196,7 @@ struct ConcatOpInterface
         return failure();
 
       if (dynamicConcatDim) {
-        dynamicOffset = rewriter.create<arith::AddIOp>(
+        dynamicOffset = arith::AddIOp::create(rewriter,
             loc, dynamicOffset.value(), dynamicSize.value());
       } else {
         concatDimOffset += operandConcatDimSize;
diff --git a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
index 3c2b0ab42f7a6..24bcb8192f672 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
@@ -42,7 +42,7 @@ struct FoldEmptyTensorWithReshapeOp : public OpRewritePattern<ReshapeOp> {
 
     // Create new tensor.empty op.
     // TODO: Do not drop tensor type encoding.
-    Value emptyTensor = rewriter.create<EmptyOp>(
+    Value emptyTensor = EmptyOp::create(rewriter,
         loc, resultShapes[0], reshapeOp.getResultType().getElementType());
     if (emptyTensor.getType() != reshapeOp.getResultType()) {
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
index dd50ae54d17cc..c3801e0f05c88 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
@@ -53,7 +53,7 @@ static ValueRange invertCollapseShapeIndexing(
   for (int64_t i : reassociation[dim])
     basis.push_back(reshapeSourceShape[i]);
   auto delinearized =
-      b.create<AffineDelinearizeIndexOp>(loc, indexValue, basis);
+      AffineDelinearizeIndexOp::create(b, loc, indexValue, basis);
   return delinearized->getResults();
 }
 
@@ -142,14 +142,14 @@ tensor::ExtractSliceFromCollapseHelper::emitLoopNestBody(
   SmallVector<Range> extractParams =
       helper.getExtractSliceParams(builder.getContext(), multiIndices);
 
-  Value subTileResult = builder.create<tensor::ExtractSliceOp>(
+  Value subTileResult = tensor::ExtractSliceOp::create(builder,
       loc, collapseShapeOp.getSrc(), extractParams);
 
   SmallVector<Range> insertParams =
       helper.getInsertSliceParams(builder.getContext(), tileInductionVars);
 
   // Collapse the dimensions of the source slice back down.
-  Value collapsedResult = builder.create<tensor::CollapseShapeOp>(
+  Value collapsedResult = tensor::CollapseShapeOp::create(builder,
       loc, subTileResult, reassociationIndices);
   return std::make_pair(collapsedResult, insertParams);
 }
@@ -173,7 +173,7 @@ tensor::simplifyCollapseShapeWithRankReducingExtractSlice(
   SmallVector<OpFoldResult> sizes =
       tensor::getMixedSizes(rewriter, op.getLoc(), op.getSrc());
   SmallVector<OpFoldResult> strides(sourceType.getRank(), one);
-  auto sliceOp = rewriter.create<tensor::ExtractSliceOp>(
+  auto sliceOp = tensor::ExtractSliceOp::create(rewriter,
       op.getLoc(), info->sliceResultType, op.getSrc(), offsets, sizes, strides);
 
   if (!info->newReassociationIndices.has_value()) {
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
index 13de55b0672a5..cc32f1193a55b 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
@@ -112,7 +112,7 @@ TransferReadOfExtractSliceOpFolder::matchAndRewriteMaskableOp(
       extractSliceOp.getMixedStrides(), extractSliceOp.getDroppedDims(),
       indices, sourceIndices);
 
-  Operation *newOp = rewriter.create<vector::TransferReadOp>(
+  Operation *newOp = vector::TransferReadOp::create(rewriter,
       readOp.getLoc(), readOp.getVectorType(), extractSliceOp.getSource(),
       sourceIndices,
       AffineMapAttr::get(expandDimsToRank(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
index bad56d4111dca..a6db540ffd8a0 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
@@ -64,7 +64,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     return padOp.getResult();
 
   // Create a new tensor::PadOp.
-  auto newPadOp = b.create<PadOp>(
+  auto newPadOp = PadOp::create(b,
       loc, padOp.getResultType(), padOp.getSource(), newMixedLow, newMixedHigh,
       constantPadding, padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
 
@@ -83,7 +83,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
       offsets.push_back(b.getIndexAttr(0));
     } else {
       offsets.push_back(
-          b.create<affine::AffineApplyOp>(
+          affine::AffineApplyOp::create(b,
                loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
                std::initializer_list<Value>{cast<Value>(newMixedLow[i]),
                                             cast<Value>(prevLow)})
@@ -99,7 +99,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     strides.push_back(b.getIndexAttr(1));
   }
 
-  return b.create<ExtractSliceOp>(loc, newPadOp, offsets, sizes, strides)
+  return ExtractSliceOp::create(b, loc, newPadOp, offsets, sizes, strides)
       .getResult();
 }
 
@@ -124,7 +124,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b,
 
   // Create a new tensor::EmptyOp.
   Value newEmptyOp =
-      b.create<EmptyOp>(loc, newSizes, emptyOp.getType().getElementType());
+      EmptyOp::create(b, loc, newSizes, emptyOp.getType().getElementType());
 
   // Create a tensor::ExtractSliceOp.
   SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index 20bb4d1caf019..3b63c87f2c05a 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -76,7 +76,7 @@ struct FoldUnPaddingCollapseIntoExtract
       return rewriter.notifyMatchFailure(collapseShapeOp,
                                          "expected unpadding collapse");
 
-    Value unPaddedExtractSlice = rewriter.create<tensor::ExtractSliceOp>(
+    Value unPaddedExtractSlice = tensor::ExtractSliceOp::create(rewriter,
         extractSliceOp.getLoc(), collapseShapeOp.getResultType(),
         extractSliceOp.getSource(), extractSliceOp.getMixedOffsets(),
         extractSliceOp.getMixedSizes(), extractSliceOp.getMixedStrides());
@@ -270,7 +270,7 @@ struct BubbleUpExpandThroughParallelCollapse
     // matches the number of dimensions of the result, then the expand_shape
     // is a no-op.
     if (newExpandReInds.size() != newExpandSizes.size()) {
-      newCollapseSrc = rewriter.create<tensor::ExpandShapeOp>(
+      newCollapseSrc = tensor::ExpandShapeOp::create(rewriter,
           loc, expandResultType, newCollapseSrc, newExpandReInds,
           newExpandSizes);
     }
@@ -280,7 +280,7 @@ struct BubbleUpExpandThroughParallelCollapse
     // is a no-op.
     Value replacement = newCollapseSrc;
     if (newCollapseReInds.size() != newExpandSizes.size()) {
-      replacement = rewriter.create<tensor::CollapseShapeOp>(
+      replacement = tensor::CollapseShapeOp::create(rewriter,
           loc, newCollapseSrc, newCollapseReInds);
     }
     rewriter.replaceOp(expandOp, replacement);
@@ -405,7 +405,7 @@ struct BubbleUpExpandShapeThroughExtractSlice
         shape, expandShapeOp.getResultType().getElementType());
 
     // Create a new ExtractSliceOp and ExpandShapeOp.
-    Value newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
         collapsedStrides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -735,7 +735,7 @@ struct BubbleUpCollapseShapeThroughExtractSlice
                              groupExpandedOffsets.rend());
     }
 
-    Value newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         collapseShapeOp->getLoc(), collapseShapeOp.getSrc(), expandedOffsets,
         expandedSizes, expandedStrides);
     rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
index 7c9fced540adb..69e649d2eebe8 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
@@ -196,7 +196,7 @@ struct PadOpToConstant final : public OpRewritePattern<PadOp> {
                                          "tensor type not supported");
 
     if (newOp.getType() != resultType)
-      newOp = rewriter.create<tensor::CastOp>(loc, resultType, newOp);
+      newOp = tensor::CastOp::create(rewriter, loc, resultType, newOp);
 
     rewriter.replaceOp(padTensorOp, newOp);
     return success();
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
index 6e3285abffbfc..7c1dea45843d5 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
@@ -47,12 +47,12 @@ struct CastOpInterface
 
     if (isa<UnrankedTensorType>(srcType)) {
       // Check rank.
-      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
-          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
-      Value isSameRank = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
+      Value isSameRank = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameRank,
           RuntimeVerifiableOpInterface::generateErrorMessage(op,
                                                              "rank mismatch"));
@@ -70,12 +70,12 @@ struct CastOpInterface
         continue;
 
       Value srcDimSz =
-          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+          DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameSz = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameSz = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
@@ -89,9 +89,9 @@ struct DimOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto dimOp = cast<DimOp>(op);
-    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    builder.create<cf::AssertOp>(
+    Value rank = RankOp::create(builder, loc, dimOp.getSource());
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    cf::AssertOp::create(builder,
         loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
@@ -124,7 +124,7 @@ struct ExtractInsertOpInterface
     }
 
     auto indices = extractInsertOp.getIndices();
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     Value assertCond;
     for (auto i : llvm::seq<int64_t>(0, rank)) {
       Value dimOp = builder.createOrFold<tensor::DimOp>(loc, tensor, i);
@@ -134,7 +134,7 @@ struct ExtractInsertOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    builder.create<cf::AssertOp>(
+    cf::AssertOp::create(builder,
         loc, assertCond,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "out-of-bounds access"));
@@ -152,8 +152,8 @@ struct ExtractSliceOpInterface
     // For each dimension, assert that:
     // 0 <= offset < dim_size
     // 0 <= offset + (size - 1) * stride < dim_size
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value one = arith::ConstantIndexOp::create(builder, loc, 1);
     for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
       Value offset = getValueOrCreateConstantIndexOp(
           builder, loc, extractSliceOp.getMixedOffsets()[i]);
@@ -167,20 +167,20 @@ struct ExtractSliceOpInterface
           loc, extractSliceOp.getSource(), i);
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
       // Verify that slice does not run out-of-bounds.
-      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOne = arith::SubIOp::create(builder, loc, size, one);
       Value sizeMinusOneTimesStride =
-          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+          arith::MulIOp::create(builder, loc, sizeMinusOne, stride);
       Value lastPos =
-          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+          arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "extract_slice runs out-of-bounds along dimension " +
diff --git a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
index d50d7c62b789c..21f70e52b8706 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
@@ -59,7 +59,7 @@ struct InsertSliceLikeOpSubsetInsertionOpInterface
   Value buildSubsetExtraction(Operation *op, OpBuilder &builder,
                               Location loc) const {
     auto insertSliceOp = cast<OpTy>(op);
-    auto extractOp = builder.create<tensor::ExtractSliceOp>(
+    auto extractOp = tensor::ExtractSliceOp::create(builder,
         loc, insertSliceOp.getSourceType(), insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
         insertSliceOp.getMixedStrides());
diff --git a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
index 3f6258b5e4d43..fd75e1090174b 100644
--- a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
@@ -56,7 +56,7 @@ PadOp mlir::tensor::createPadHighOp(RankedTensorType resType, Value source,
     high[idx] = affine::makeComposedFoldedAffineApply(b, loc, d0 - d1,
                                                       {outDim, sourceDim});
   }
-  return b.create<PadOp>(loc, resType, source, low, high, pad, nofold);
+  return PadOp::create(b, loc, resType, source, low, high, pad, nofold);
 }
 
 SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
@@ -67,7 +67,7 @@ SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
   for (const auto &en : llvm::enumerate(tensorTy.getShape())) {
     if (en.value() == ShapedType::kDynamic)
       dynamicDims.push_back(
-          b.create<tensor::DimOp>(loc, rankedTensor, en.index()));
+          tensor::DimOp::create(b, loc, rankedTensor, en.index()));
   }
   return dynamicDims;
 }
@@ -119,7 +119,7 @@ mlir::tensor::dropGivenUnitDims(OpBuilder &b, Location loc, Value src,
     reassocMaps.emplace_back(llvm::make_range(seq.begin(), seq.end()));
     nextDimToGroup = setBit + 1;
   }
-  return b.create<tensor::CollapseShapeOp>(loc, src, reassocMaps);
+  return tensor::CollapseShapeOp::create(b, loc, src, reassocMaps);
 }
 
 bool mlir::tensor::isCastLikeInsertSliceOp(InsertSliceOp op) {
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 2dd45d27157cb..7ca4f6496ccdc 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -845,7 +845,7 @@ struct PadSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
         getTosaConstShape(rewriter, sliceOp.getLoc(), newPadPaddings);
     auto newPadTy =
         RankedTensorType::get(newPadShape, inputTy.getElementType());
-    auto newPadOp = rewriter.create<tosa::PadOp>(
+    auto newPadOp = tosa::PadOp::create(rewriter,
         padOp.getLoc(), newPadTy, padOp.getInput1(), newPaddingsOp,
         padOp.getPadConst());
 
@@ -897,7 +897,7 @@ struct SliceDynamicSizeCanonicalization
     }
 
     auto size_op = getTosaConstShape(rewriter, sliceOp.getLoc(), sliceSizes);
-    auto newSliceOp = rewriter.create<tosa::SliceOp>(
+    auto newSliceOp = tosa::SliceOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getType(), sliceOp.getInput1(),
         sliceOp.getStart(), size_op);
 
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 4a952ac062cad..d110e1aeee8e4 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -180,11 +180,11 @@ Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // Tosa dialect constants only support ElementsAttr unlike standard dialect
   // constant which supports all attributes.
   if (llvm::isa<shapeType>(type) && llvm::isa<DenseIntElementsAttr>(value)) {
-    return builder.create<tosa::ConstShapeOp>(
+    return tosa::ConstShapeOp::create(builder,
         loc, type, llvm::cast<DenseIntElementsAttr>(value));
   }
   if (llvm::isa<ElementsAttr>(value))
-    return builder.create<tosa::ConstOp>(loc, type,
+    return tosa::ConstOp::create(builder, loc, type,
                                          llvm::cast<ElementsAttr>(value));
   return nullptr;
 }
@@ -323,7 +323,7 @@ Value mlir::tosa::createPadConstTensor(OpBuilder &builder, Location loc,
                                    builder.getFloatAttr(srcElemType, val))
           : DenseElementsAttr::get(padConstEType,
                                    builder.getIntegerAttr(srcElemType, val))};
-  return builder.create<tosa::ConstOp>(loc, padConstType, padConstAttr);
+  return tosa::ConstOp::create(builder, loc, padConstType, padConstAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2415,7 +2415,7 @@ LogicalResult TransposeOp::reifyResultShapes(
     int32_t dimInInput = transposePerms[dim];
     if (inputType.isDynamicDim(dimInInput))
       returnedDims[dim] =
-          builder.create<tensor::DimOp>(getLoc(), input, dimInInput)
+          tensor::DimOp::create(builder, getLoc(), input, dimInInput)
               .getResult();
     else
       returnedDims[dim] =
@@ -3946,12 +3946,12 @@ std::optional<Value> mlir::tosa::createZeroPointTensor(OpBuilder &builder,
   if (llvm::isa<FloatType>(srcElemType)) {
     auto zpAttr = DenseElementsAttr::get(
         zpType, builder.getFloatAttr(srcElemType, static_cast<double>(zp)));
-    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+    return tosa::ConstOp::create(builder, loc, zpType, zpAttr);
   }
   if (llvm::isa<IntegerType>(srcElemType)) {
     auto zpAttr =
         DenseElementsAttr::get(zpType, builder.getIntegerAttr(srcElemType, zp));
-    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+    return tosa::ConstOp::create(builder, loc, zpType, zpAttr);
   }
   llvm::errs() << "zero point is not allowed for unsupported data types\n";
   return std::nullopt;
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
index f6caa2a985a4d..9b77986257814 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
@@ -90,12 +90,12 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
 
     if (inputETy != resultETy) {
       inputType = inputType.clone(resultETy);
-      input = rewriter.create<tosa::CastOp>(op.getLoc(), inputType, input);
+      input = tosa::CastOp::create(rewriter, op.getLoc(), inputType, input);
     }
 
     if (weightETy != resultETy) {
       weightType = weightType.clone(resultETy);
-      weight = rewriter.create<tosa::CastOp>(op.getLoc(), weightType, weight);
+      weight = tosa::CastOp::create(rewriter, op.getLoc(), weightType, weight);
     }
 
     if (iZp != 0 || wZp != 0) {
@@ -109,8 +109,8 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         auto zpTy = RankedTensorType::get(shape, ety);
         auto zpAttr =
             DenseElementsAttr::get(zpTy, rewriter.getIntegerAttr(ety, zp));
-        auto zpVal = rewriter.create<tosa::ConstOp>(op.getLoc(), zpTy, zpAttr);
-        return rewriter.create<tosa::SubOp>(op.getLoc(), val.getType(), val,
+        auto zpVal = tosa::ConstOp::create(rewriter, op.getLoc(), zpTy, zpAttr);
+        return tosa::SubOp::create(rewriter, op.getLoc(), val.getType(), val,
                                             zpVal);
       };
 
@@ -138,9 +138,9 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
       auto padTy = RankedTensorType::get({1}, inputETy);
       auto padAttr = DenseElementsAttr::get(padTy, zeroAttr);
       Value padVal =
-          rewriter.create<tosa::ConstOp>(op->getLoc(), padTy, padAttr);
+          tosa::ConstOp::create(rewriter, op->getLoc(), padTy, padAttr);
       inputType = RankedTensorType::get(newShape, inputETy);
-      input = rewriter.create<tosa::PadOp>(op->getLoc(), inputType, input,
+      input = tosa::PadOp::create(rewriter, op->getLoc(), inputType, input,
                                            padSizeVal, padVal);
     }
 
@@ -161,7 +161,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
     auto shiftZeroAttr = DenseElementsAttr::get(
         shiftType, rewriter.getIntegerAttr(shiftElementType, 0));
     Value constZero =
-        rewriter.create<tosa::ConstOp>(op.getLoc(), shiftType, shiftZeroAttr);
+        tosa::ConstOp::create(rewriter, op.getLoc(), shiftType, shiftZeroAttr);
     Value mulValue = rewriter
                          .create<tosa::MulOp>(op.getLoc(), mulShapeType, input,
                                               weight, constZero)
@@ -174,7 +174,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         dyn_cast<RankedTensorType>(input.getType()).getElementType());
     auto outputShapeValue =
         getTosaConstShape(rewriter, op->getLoc(), outputShape);
-    Value outputValue = rewriter.create<tosa::ReshapeOp>(
+    Value outputValue = tosa::ReshapeOp::create(rewriter,
         op.getLoc(), outputShapeType, mulValue, outputShapeValue);
 
     Value bias = op.getBias();
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
index df6d52615478e..4a07220b4e181 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
@@ -62,12 +62,12 @@ class TransposeConvNonStridedConverter
     convPad[2] = kernelWidth - 1 + pad[2];
     convPad[3] = kernelWidth - 1 + pad[3];
 
-    auto reverse1 = rewriter.create<tosa::ReverseOp>(
+    auto reverse1 = tosa::ReverseOp::create(rewriter,
         loc, weightTy, weight, /* axis = */ rewriter.getI32IntegerAttr(1));
-    auto reverse2 = rewriter.create<tosa::ReverseOp>(
+    auto reverse2 = tosa::ReverseOp::create(rewriter,
         loc, weightTy, reverse1, /* axis = */ rewriter.getI32IntegerAttr(2));
 
-    Value conv2d = rewriter.create<tosa::Conv2DOp>(
+    Value conv2d = tosa::Conv2DOp::create(rewriter,
         loc, resultTy, input, reverse2, bias, op.getInputZp(), op.getWeightZp(),
         rewriter.getDenseI64ArrayAttr(convPad),
         rewriter.getDenseI64ArrayAttr(stride),
@@ -216,7 +216,7 @@ class TransposeConvStridedConverter
         inputPaddingVal, inputPadConst);
 
     // We use a zero bias as we need to broadcast the bias.
-    auto zeroBias = rewriter.create<tosa::ConstOp>(
+    auto zeroBias = tosa::ConstOp::create(rewriter,
         loc,
         RankedTensorType::get({outputChannels * stride[0] * stride[1]},
                               biasETy),
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index a9e98c8908e15..4d347c02ee16d 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -112,7 +112,7 @@ class TypeModificationState {
           OpBuilder builder{value.getContext()};
           builder.setInsertionPointAfter(value.getDefiningOp());
           castValue =
-              builder.create<tensor::CastOp>(value.getLoc(), oldType, value);
+              tensor::CastOp::create(builder, value.getLoc(), oldType, value);
         }
 
         use->set(castValue);
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index 8ebbbc94eb6a2..a281c6600fdf6 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -403,7 +403,7 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
     return std::nullopt;
   }
   ImplicitLocOpBuilder builder(reshapeOp.getLoc(), rewriter);
-  auto foldedReshape = rewriter.create<ReshapeOp>(
+  auto foldedReshape = ReshapeOp::create(rewriter,
       reshapeOp.getLoc(),
       RankedTensorType::get(applyTOSAPermutation(shape, hoistedPerms),
                             reshapeOutputType.getElementType()),
@@ -423,7 +423,7 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
   if (!maybeNewDenseAttr.has_value())
     return std::nullopt;
   auto newDenseAttr = maybeNewDenseAttr.value();
-  auto newConstOp = rewriter.create<ConstOp>(
+  auto newConstOp = ConstOp::create(rewriter,
       constOp.getLoc(), newDenseAttr.getType(), newDenseAttr);
   return newConstOp->getResult(0);
 }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
index 3b697a2ee3e47..677d8e9904a67 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
@@ -37,7 +37,7 @@ void mlir::tosa::populateTosaTypeConversion(TypeConverter &converter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
   converter.addTargetMaterialization([&](OpBuilder &builder, Type resultType,
@@ -46,7 +46,7 @@ void mlir::tosa::populateTosaTypeConversion(TypeConverter &converter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
 }
diff --git a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
index 9844abcc34cb1..69ef0ba40d72e 100644
--- a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
@@ -33,18 +33,18 @@ mlir::tosa::condenseValues(const SmallVector<Value> &values) {
 
 Value mlir::tosa::clampFloatHelper(Location loc, Value arg, Value min,
                                    Value max, OpBuilder &rewriter) {
-  Value minValue = rewriter.create<arith::MinimumFOp>(loc, arg, max);
-  return rewriter.create<arith::MaximumFOp>(loc, minValue, min);
+  Value minValue = arith::MinimumFOp::create(rewriter, loc, arg, max);
+  return arith::MaximumFOp::create(rewriter, loc, minValue, min);
 }
 
 Value mlir::tosa::clampIntHelper(Location loc, Value arg, Value min, Value max,
                                  OpBuilder &rewriter, bool isUnsigned) {
   if (isUnsigned) {
-    auto minOrArg = rewriter.create<arith::MaxUIOp>(loc, min, arg);
-    return rewriter.create<arith::MinUIOp>(loc, max, minOrArg);
+    auto minOrArg = arith::MaxUIOp::create(rewriter, loc, min, arg);
+    return arith::MinUIOp::create(rewriter, loc, max, minOrArg);
   }
-  auto minOrArg = rewriter.create<arith::MaxSIOp>(loc, min, arg);
-  return rewriter.create<arith::MinSIOp>(loc, max, minOrArg);
+  auto minOrArg = arith::MaxSIOp::create(rewriter, loc, min, arg);
+  return arith::MinSIOp::create(rewriter, loc, max, minOrArg);
 }
 
 bool mlir::tosa::validIntegerRange(IntegerType ty, int64_t value) {
@@ -144,7 +144,7 @@ LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder,
       ArrayRef<int64_t>(reshapeOutputShape), reshapeInputType.getElementType());
   auto reshapeOutputShapeValue = getTosaConstShape(builder, reshapeOutputShape);
 
-  auto reshapeLower = builder.create<tosa::ReshapeOp>(
+  auto reshapeLower = tosa::ReshapeOp::create(builder,
       reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
 
   if (input1Rank > input2Rank) {
@@ -162,7 +162,7 @@ Value mlir::tosa::getTosaConstShape(ImplicitLocOpBuilder &builder,
                                     llvm::ArrayRef<int64_t> shape) {
   auto attr = builder.getIndexTensorAttr(convertFromMlirShape(shape));
   auto type = mlir::tosa::shapeType::get(builder.getContext(), shape.size());
-  mlir::Operation *mlir_op = builder.create<tosa::ConstShapeOp>(type, attr);
+  mlir::Operation *mlir_op = tosa::ConstShapeOp::create(builder, type, attr);
   return mlir_op->getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/UB/IR/UBOps.cpp b/mlir/lib/Dialect/UB/IR/UBOps.cpp
index 5b2cfe7bf4264..ee523f9522953 100644
--- a/mlir/lib/Dialect/UB/IR/UBOps.cpp
+++ b/mlir/lib/Dialect/UB/IR/UBOps.cpp
@@ -52,7 +52,7 @@ void UBDialect::initialize() {
 Operation *UBDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                           Type type, Location loc) {
   if (auto attr = dyn_cast<PoisonAttr>(value))
-    return builder.create<PoisonOp>(loc, type, attr);
+    return PoisonOp::create(builder, loc, type, attr);
 
   return nullptr;
 }
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index b58ed6b5d1144..3497b0d7a88a0 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -125,7 +125,7 @@ static MaskFormat getMaskFormat(Value mask) {
 /// Default callback to build a region with a 'vector.yield' terminator with no
 /// arguments.
 void mlir::vector::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<vector::YieldOp>(loc);
+  vector::YieldOp::create(builder, loc);
 }
 
 // Helper for verifying combining kinds in contractions and reductions.
@@ -597,16 +597,16 @@ struct ElideUnitDimsInMultiDimReduction
         VectorType newMaskType =
             VectorType::get(dstVecType.getShape(), rewriter.getI1Type(),
                             dstVecType.getScalableDims());
-        mask = rewriter.create<vector::ShapeCastOp>(loc, newMaskType, mask);
+        mask = vector::ShapeCastOp::create(rewriter, loc, newMaskType, mask);
       }
-      cast = rewriter.create<vector::ShapeCastOp>(
+      cast = vector::ShapeCastOp::create(rewriter,
           loc, reductionOp.getDestType(), reductionOp.getSource());
     } else {
       // This means we are reducing all the dimensions, and all reduction
       // dimensions are of size 1. So a simple extraction would do.
       if (mask)
-        mask = rewriter.create<vector::ExtractOp>(loc, mask);
-      cast = rewriter.create<vector::ExtractOp>(loc, reductionOp.getSource());
+        mask = vector::ExtractOp::create(rewriter, loc, mask);
+      cast = vector::ExtractOp::create(rewriter, loc, reductionOp.getSource());
     }
 
     Value result =
@@ -673,35 +673,35 @@ Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
   switch (op) {
   case arith::AtomicRMWKind::addf:
   case arith::AtomicRMWKind::addi:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::ADD, vector);
   case arith::AtomicRMWKind::mulf:
   case arith::AtomicRMWKind::muli:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MUL, vector);
   case arith::AtomicRMWKind::minimumf:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINIMUMF, vector);
   case arith::AtomicRMWKind::mins:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINSI, vector);
   case arith::AtomicRMWKind::minu:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINUI, vector);
   case arith::AtomicRMWKind::maximumf:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXIMUMF, vector);
   case arith::AtomicRMWKind::maxs:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXSI, vector);
   case arith::AtomicRMWKind::maxu:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXUI, vector);
   case arith::AtomicRMWKind::andi:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::AND, vector);
   case arith::AtomicRMWKind::ori:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::OR, vector);
   // TODO: Add remaining reduction operations.
   default:
@@ -741,8 +741,8 @@ struct ElideSingleElementReduction : public OpRewritePattern<ReductionOp> {
 
     Location loc = reductionOp.getLoc();
     if (mask)
-      mask = rewriter.create<ExtractOp>(loc, mask);
-    Value result = rewriter.create<ExtractOp>(loc, reductionOp.getVector());
+      mask = ExtractOp::create(rewriter, loc, mask);
+    Value result = ExtractOp::create(rewriter, loc, reductionOp.getVector());
 
     if (Value acc = reductionOp.getAcc())
       result = vector::makeArithReduction(rewriter, loc, reductionOp.getKind(),
@@ -4255,7 +4255,7 @@ class StridedSliceBroadcast final
     // just a single scalar.
     bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
     if (!lowerDimMatch && !isScalarSrc) {
-      source = rewriter.create<ExtractStridedSliceOp>(
+      source = ExtractStridedSliceOp::create(rewriter,
           op->getLoc(), source,
           getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
@@ -4350,7 +4350,7 @@ class ContiguousExtractStridedSliceToExtract final
 
     SmallVector<int64_t> offsets = getI64SubArray(op.getOffsets());
     auto extractOffsets = ArrayRef(offsets).take_front(numOffsets);
-    Value extract = rewriter.create<vector::ExtractOp>(op->getLoc(), source,
+    Value extract = vector::ExtractOp::create(rewriter, op->getLoc(), source,
                                                        extractOffsets);
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(), extract);
     return success();
@@ -4381,7 +4381,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
 
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
         *padding, /*mask=*/Value(), inBoundsAttr);
 }
@@ -4399,7 +4399,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                                 SmallVector<bool>(vectorType.getRank(), false));
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, *padding,
         permutationMapAttr, inBoundsAttr);
 }
@@ -4418,7 +4418,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                                 SmallVector<bool>(vectorType.getRank(), false));
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
         *padding,
         /*mask=*/Value(), inBoundsAttr);
@@ -4943,7 +4943,7 @@ struct TransferReadAfterWriteToBroadcast
     VectorType broadcastedType = VectorType::get(
         broadcastShape, defWrite.getVectorType().getElementType(),
         broadcastScalableFlags);
-    vec = rewriter.create<vector::BroadcastOp>(loc, broadcastedType, vec);
+    vec = vector::BroadcastOp::create(rewriter, loc, broadcastedType, vec);
     SmallVector<int64_t> transposePerm(permutation.begin(), permutation.end());
     rewriter.replaceOpWithNewOp<vector::TransposeOp>(readOp, vec,
                                                      transposePerm);
@@ -5421,11 +5421,11 @@ struct SwapExtractSliceOfTransferWrite
     // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
     // Set all in_bounds to false and let the folder infer them.
     SmallVector<bool> newInBounds(vectorShape.size(), false);
-    auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
         extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
         insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
         insertOp.getMixedStrides());
-    auto newTransferWriteOp = rewriter.create<TransferWriteOp>(
+    auto newTransferWriteOp = TransferWriteOp::create(rewriter,
         transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
         transferOp.getIndices(), transferOp.getPermutationMapAttr(),
         rewriter.getBoolArrayAttr(newInBounds));
@@ -6951,7 +6951,7 @@ void MaskOp::ensureTerminator(Region &region, Builder &builder, Location loc) {
   OpBuilder opBuilder(builder.getContext());
   Operation *maskedOp = &block.front();
   opBuilder.setInsertionPointToEnd(&block);
-  opBuilder.create<vector::YieldOp>(loc, maskedOp->getResults());
+  vector::YieldOp::create(opBuilder, loc, maskedOp->getResults());
 }
 
 LogicalResult MaskOp::verify() {
@@ -7286,7 +7286,7 @@ void mlir::vector::createMaskOpRegion(OpBuilder &builder,
   // Create a block and move the op to that block.
   insBlock->getOperations().splice(
       insBlock->begin(), maskableOp->getBlock()->getOperations(), maskableOp);
-  builder.create<YieldOp>(maskableOp->getLoc(), maskableOp->getResults());
+  YieldOp::create(builder, maskableOp->getLoc(), maskableOp->getResults());
 }
 
 /// Creates a vector.mask operation around a maskable operation. Returns the
@@ -7298,10 +7298,10 @@ Operation *mlir::vector::maskOperation(OpBuilder &builder,
   if (!mask)
     return maskableOp;
   if (passthru)
-    return builder.create<MaskOp>(maskableOp->getLoc(),
+    return MaskOp::create(builder, maskableOp->getLoc(),
                                   maskableOp->getResultTypes(), mask, passthru,
                                   maskableOp, createMaskOpRegion);
-  return builder.create<MaskOp>(maskableOp->getLoc(),
+  return MaskOp::create(builder, maskableOp->getLoc(),
                                 maskableOp->getResultTypes(), mask, maskableOp,
                                 createMaskOpRegion);
 }
@@ -7318,7 +7318,7 @@ Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
   if (!mask)
     return newValue;
 
-  return builder.create<arith::SelectOp>(newValue.getLoc(), newValue.getType(),
+  return arith::SelectOp::create(builder, newValue.getLoc(), newValue.getType(),
                                          mask, newValue, passthru);
 }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
index 9da051150e409..c750017a6b504 100644
--- a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -116,7 +116,7 @@ struct TransferWriteOpInterface
         getBuffer(rewriter, writeOp.getBase(), options, state);
     if (failed(resultBuffer))
       return failure();
-    rewriter.create<vector::TransferWriteOp>(
+    vector::TransferWriteOp::create(rewriter,
         writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
         writeOp.getIndices(), writeOp.getPermutationMapAttr(),
         writeOp.getMask(), writeOp.getInBoundsAttr());
@@ -241,7 +241,7 @@ struct MaskOpInterface
     // Create a new vector.mask op.
     ValueRange newYieldedValuesRange(newYieldedValues);
     TypeRange newResultTypes(newYieldedValuesRange);
-    auto newOp = rewriter.create<vector::MaskOp>(
+    auto newOp = vector::MaskOp::create(rewriter,
         op->getLoc(), newResultTypes, maskOp.getMask(), maskOp.getPassthru(),
         /*maskableOp=*/nullptr,
         /*maskRegionBuilder=*/[](OpBuilder &b, Operation *) {});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
index 89930a6bd35fa..4c3a04cfb5bfa 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
@@ -64,14 +64,14 @@ class UnrollBitCastOp final : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(shape, resultType.getElementType(), scalableDims);
 
     Location loc = op.getLoc();
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
     for (auto position : *unrollIterator) {
       Value extract =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+          vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       Value bitcast =
-          rewriter.create<vector::BitCastOp>(loc, bitcastResType, extract);
+          vector::BitCastOp::create(rewriter, loc, bitcastResType, extract);
       result =
-          rewriter.create<vector::InsertOp>(loc, bitcast, result, position);
+          vector::InsertOp::create(rewriter, loc, bitcast, result, position);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
index 11dcfe421e0c4..cb8e566869cfd 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
@@ -52,7 +52,7 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
 
     // Stretching scalar inside vector (e.g. vector<1xf32>) can use splat.
     if (srcRank <= 1 && dstRank == 1) {
-      Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource());
+      Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource());
       rewriter.replaceOpWithNewOp<vector::SplatOp>(op, dstType, ext);
       return success();
     }
@@ -70,10 +70,10 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
       // Duplication.
       VectorType resType = VectorType::Builder(dstType).dropDim(0);
       Value bcst =
-          rewriter.create<vector::BroadcastOp>(loc, resType, op.getSource());
-      Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
+          vector::BroadcastOp::create(rewriter, loc, resType, op.getSource());
+      Value result = ub::PoisonOp::create(rewriter, loc, dstType);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
       rewriter.replaceOp(op, result);
       return success();
     }
@@ -111,13 +111,13 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType =
         VectorType::get(dstType.getShape().drop_front(), eltType,
                         dstType.getScalableDims().drop_front());
-    Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
+    Value result = ub::PoisonOp::create(rewriter, loc, dstType);
     if (m == 0) {
       // Stetch at start.
-      Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), 0);
-      Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
+      Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource(), 0);
+      Value bcst = vector::BroadcastOp::create(rewriter, loc, resType, ext);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
     } else {
       // Stetch not at start.
       if (dstType.getScalableDims()[0]) {
@@ -125,9 +125,9 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
         return failure();
       }
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d) {
-        Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), d);
-        Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource(), d);
+        Value bcst = vector::BroadcastOp::create(rewriter, loc, resType, ext);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
       }
     }
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
index c6627b5ec0d77..b2486c5cbbcf8 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
@@ -90,17 +90,17 @@ static Value reshapeLoad(Location loc, Value val, VectorType type,
 
   // At extraction dimension?
   if (index == 0)
-    return rewriter.create<vector::ExtractOp>(loc, val, pos);
+    return vector::ExtractOp::create(rewriter, loc, val, pos);
 
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   VectorType resType = VectorType::Builder(type).dropDim(index);
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, resType, rewriter.getZeroAttr(resType));
   for (int64_t d = 0, e = resType.getDimSize(0); d < e; d++) {
-    Value ext = rewriter.create<vector::ExtractOp>(loc, val, d);
+    Value ext = vector::ExtractOp::create(rewriter, loc, val, d);
     Value load = reshapeLoad(loc, ext, vType, index - 1, pos, rewriter);
-    result = rewriter.create<vector::InsertOp>(loc, load, result, d);
+    result = vector::InsertOp::create(rewriter, loc, load, result, d);
   }
   return result;
 }
@@ -115,15 +115,15 @@ static Value reshapeStore(Location loc, Value val, Value result,
     return val;
   // At insertion dimension?
   if (index == 0)
-    return rewriter.create<vector::InsertOp>(loc, val, result, pos);
+    return vector::InsertOp::create(rewriter, loc, val, result, pos);
 
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   for (int64_t d = 0, e = type.getDimSize(0); d < e; d++) {
-    Value ext = rewriter.create<vector::ExtractOp>(loc, result, d);
-    Value ins = rewriter.create<vector::ExtractOp>(loc, val, d);
+    Value ext = vector::ExtractOp::create(rewriter, loc, result, d);
+    Value ins = vector::ExtractOp::create(rewriter, loc, val, d);
     Value sto = reshapeStore(loc, ins, ext, vType, index - 1, pos, rewriter);
-    result = rewriter.create<vector::InsertOp>(loc, sto, result, d);
+    result = vector::InsertOp::create(rewriter, loc, sto, result, d);
   }
   return result;
 }
@@ -141,7 +141,7 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
         kind == CombiningKind::MINIMUMF || kind == CombiningKind::MAXIMUMF)
       // Only valid for floating point types.
       return std::nullopt;
-    mul = rewriter.create<arith::MulIOp>(loc, x, y);
+    mul = arith::MulIOp::create(rewriter, loc, x, y);
   } else {
     // Float case.
     if (kind == CombiningKind::AND || kind == CombiningKind::MINUI ||
@@ -152,14 +152,14 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
       return std::nullopt;
     // Special case for fused multiply-add.
     if (acc && isa<VectorType>(acc.getType()) && kind == CombiningKind::ADD) {
-      Value fma = rewriter.create<vector::FMAOp>(loc, x, y, acc);
+      Value fma = vector::FMAOp::create(rewriter, loc, x, y, acc);
       if (mask)
         // The fma op doesn't need explicit masking. However, fma ops used in
         // reductions must preserve previous 'acc' values for masked-out lanes.
         fma = selectPassthru(rewriter, mask, fma, acc);
       return fma;
     }
-    mul = rewriter.create<arith::MulFOp>(loc, x, y);
+    mul = arith::MulFOp::create(rewriter, loc, x, y);
   }
 
   if (!acc)
@@ -195,8 +195,8 @@ static std::optional<unsigned> getDimPosition(AffineMap map, unsigned dim) {
 static Value createAdd(Location loc, Value x, Value y, bool isInt,
                        PatternRewriter &rewriter) {
   if (isInt)
-    return rewriter.create<arith::AddIOp>(loc, x, y);
-  return rewriter.create<arith::AddFOp>(loc, x, y);
+    return arith::AddIOp::create(rewriter, loc, x, y);
+  return arith::AddFOp::create(rewriter, loc, x, y);
 }
 
 /// Creates a MulIOp if `isInt` is true otherwise create an MulFOp using
@@ -204,8 +204,8 @@ static Value createAdd(Location loc, Value x, Value y, bool isInt,
 static Value createMul(Location loc, Value x, Value y, bool isInt,
                        PatternRewriter &rewriter) {
   if (isInt)
-    return rewriter.create<arith::MulIOp>(loc, x, y);
-  return rewriter.create<arith::MulFOp>(loc, x, y);
+    return arith::MulIOp::create(rewriter, loc, x, y);
+  return arith::MulFOp::create(rewriter, loc, x, y);
 }
 
 namespace {
@@ -411,7 +411,7 @@ struct UnrolledOuterProductGenerator
   Value t(Value v, ArrayRef<int64_t> perm = {1, 0}) {
     if (!v)
       return v;
-    return rewriter.create<vector::TransposeOp>(loc, v, perm);
+    return vector::TransposeOp::create(rewriter, loc, v, perm);
   }
 
   Value promote(Value v, Type dstElementType) {
@@ -425,8 +425,8 @@ struct UnrolledOuterProductGenerator
     if (vecType)
       promotedType = vecType.clone(promotedType);
     if (isa<FloatType>(dstElementType))
-      return rewriter.create<arith::ExtFOp>(loc, promotedType, v);
-    return rewriter.create<arith::ExtSIOp>(loc, promotedType, v);
+      return arith::ExtFOp::create(rewriter, loc, promotedType, v);
+    return arith::ExtSIOp::create(rewriter, loc, promotedType, v);
   }
 
   FailureOr<Value> outerProd(Value lhs, Value rhs, Value res,
@@ -438,16 +438,16 @@ struct UnrolledOuterProductGenerator
 
     Type resElementType = cast<VectorType>(res.getType()).getElementType();
     for (int64_t k = 0; k < reductionSize; ++k) {
-      Value extractA = rewriter.create<vector::ExtractOp>(loc, lhs, k);
-      Value extractB = rewriter.create<vector::ExtractOp>(loc, rhs, k);
+      Value extractA = vector::ExtractOp::create(rewriter, loc, lhs, k);
+      Value extractB = vector::ExtractOp::create(rewriter, loc, rhs, k);
       extractA = promote(extractA, resElementType);
       extractB = promote(extractB, resElementType);
       Value extractMask;
       if (maybeMask.has_value() && maybeMask.value())
         extractMask =
-            rewriter.create<vector::ExtractOp>(loc, maybeMask.value(), k);
+            vector::ExtractOp::create(rewriter, loc, maybeMask.value(), k);
 
-      Operation *outerProdOp = rewriter.create<vector::OuterProductOp>(
+      Operation *outerProdOp = vector::OuterProductOp::create(rewriter,
           loc, res.getType(), extractA, extractB, res, kind);
       res = maskOperation(rewriter, outerProdOp, extractMask)->getResult(0);
     }
@@ -698,28 +698,28 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
     // Two outer parallel, one inner reduction (matmat flavor).
     //
     if (maps == infer({{m, k}, {k, n}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{m, k}, {n, k}, {m, n}})) {
       // No need to permute anything.
     } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
       // This is the classical row-major matmul. Just permute the lhs.
       Value tmp = lhs;
-      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
       rhs = tmp;
     } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
       std::swap(lhs, rhs);
     } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
       Value tmp = lhs;
-      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      rhs = rewriter.create<vector::TransposeOp>(loc, tmp, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, tmp, perm);
     } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
       Value tmp = rhs;
-      rhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
       lhs = tmp;
     } else {
       return failure();
@@ -732,12 +732,12 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
     if (maps == infer({{m, n}, {n}, {m}})) {
       // No need to permute anything.
     } else if (maps == infer({{n, m}, {n}, {m}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{n}, {m, n}, {m}})) {
       std::swap(lhs, rhs);
     } else if (maps == infer({{n}, {n, m}, {m}})) {
       std::swap(lhs, rhs);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else {
       return failure();
     }
@@ -754,31 +754,31 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
   unsigned dstColumns = rank == 1 ? 1 : dstType.getShape()[1];
 
   // ExtractOp does not allow dynamic indexing, we must unroll explicitly.
-  Value res = rewriter.create<arith::ConstantOp>(loc, dstType,
+  Value res = arith::ConstantOp::create(rewriter, loc, dstType,
                                                  rewriter.getZeroAttr(dstType));
   bool isInt = isa<IntegerType>(dstType.getElementType());
   llvm::SmallVector<Value> extractedCols;
   extractedCols.reserve(dstColumns);
   for (unsigned r = 0; r < dstRows; ++r) {
-    Value rowLhs = rewriter.create<vector::ExtractOp>(op.getLoc(), lhs, r);
+    Value rowLhs = vector::ExtractOp::create(rewriter, op.getLoc(), lhs, r);
     for (unsigned c = 0; c < dstColumns; ++c) {
       // Extract each respective row and column of the LHS and RHS once to
       // avoid having duplicate SSA values pointing to the same rows/columns.
       if (r == 0) {
         Value colRhs =
             rank == 1 ? rhs
-                      : rewriter.create<vector::ExtractOp>(op.getLoc(), rhs, c);
+                      : vector::ExtractOp::create(rewriter, op.getLoc(), rhs, c);
         extractedCols.push_back(colRhs);
       }
       Value extractedColRhs = extractedCols[c];
       Value product =
           createMul(op.getLoc(), rowLhs, extractedColRhs, isInt, rewriter);
-      Value sum = rewriter.create<vector::ReductionOp>(
+      Value sum = vector::ReductionOp::create(rewriter,
           op.getLoc(), vector::CombiningKind::ADD, product);
 
       SmallVector<int64_t, 2> pos = rank == 1 ? SmallVector<int64_t, 2>{r}
                                               : SmallVector<int64_t, 2>{r, c};
-      res = rewriter.create<vector::InsertOp>(op.getLoc(), sum, res, pos);
+      res = vector::InsertOp::create(rewriter, op.getLoc(), sum, res, pos);
     }
   }
   if (auto acc = op.getAcc())
@@ -879,21 +879,21 @@ struct ContractOpToElementwise
       lhsDims.append(lhsShape.begin(), lhsShape.end());
       auto expandedType =
           VectorType::get(lhsDims, contractOp.getLhsType().getElementType());
-      newLhs = rewriter.create<vector::BroadcastOp>(loc, expandedType, newLhs);
+      newLhs = vector::BroadcastOp::create(rewriter, loc, expandedType, newLhs);
     }
     if (!rhsDims.empty()) {
       rhsDims.append(rhsShape.begin(), rhsShape.end());
       auto expandedType =
           VectorType::get(rhsDims, contractOp.getRhsType().getElementType());
-      newRhs = rewriter.create<vector::BroadcastOp>(loc, expandedType, newRhs);
+      newRhs = vector::BroadcastOp::create(rewriter, loc, expandedType, newRhs);
     }
     bool isInt = contractOp.getLhsType().getElementType().isIntOrIndex();
-    newLhs = rewriter.create<vector::TransposeOp>(loc, newLhs, lhsTranspose);
-    newRhs = rewriter.create<vector::TransposeOp>(loc, newRhs, rhsTranspose);
+    newLhs = vector::TransposeOp::create(rewriter, loc, newLhs, lhsTranspose);
+    newRhs = vector::TransposeOp::create(rewriter, loc, newRhs, rhsTranspose);
     SmallVector<int64_t> lhsOffsets(lhsReductionDims.size(), 0);
     SmallVector<int64_t> rhsOffsets(rhsReductionDims.size(), 0);
-    newLhs = rewriter.create<vector::ExtractOp>(loc, newLhs, lhsOffsets);
-    newRhs = rewriter.create<vector::ExtractOp>(loc, newRhs, rhsOffsets);
+    newLhs = vector::ExtractOp::create(rewriter, loc, newLhs, lhsOffsets);
+    newRhs = vector::ExtractOp::create(rewriter, loc, newRhs, rhsOffsets);
     std::optional<Value> result =
         createContractArithOp(loc, newLhs, newRhs, contractOp.getAcc(),
                               contractOp.getKind(), rewriter, isInt);
@@ -1097,7 +1097,7 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       rewriter.getArrayAttr(adjustIter(op.getIteratorTypes(), iterIndex));
   // Unroll into a series of lower dimensional vector.contract ops.
   Location loc = op.getLoc();
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, resType, rewriter.getZeroAttr(resType));
 
   for (int64_t d = 0; d < dimSize; ++d) {
@@ -1110,7 +1110,7 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       lowMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *lowContract = rewriter.create<vector::ContractionOp>(
+    Operation *lowContract = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, acc, lowAffine, lowIter);
     lowContract = maskOperation(rewriter, lowContract, lowMask);
     result = reshapeStore(loc, lowContract->getResult(0), result, resType,
@@ -1161,8 +1161,8 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
 
     Value acc = op.getAcc();
     Operation *reductionOp =
-        acc ? rewriter.create<vector::ReductionOp>(loc, kind, m, acc)
-            : rewriter.create<vector::ReductionOp>(loc, kind, m);
+        acc ? vector::ReductionOp::create(rewriter, loc, kind, m, acc)
+            : vector::ReductionOp::create(rewriter, loc, kind, m);
     return maskOperation(rewriter, reductionOp, mask)->getResult(0);
   }
   // Construct new iterator types and affine map array attribute.
@@ -1186,7 +1186,7 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
       newMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *newContract = rewriter.create<vector::ContractionOp>(
+    Operation *newContract = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, result, lowAffine, lowIter);
     result = maskOperation(rewriter, newContract, newMask)->getResult(0);
   }
@@ -1240,7 +1240,7 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
 
     if (!rhsType) {
       // Special case: AXPY operation.
-      Value b = rewriter.create<vector::BroadcastOp>(loc, lhsType, op.getRhs());
+      Value b = vector::BroadcastOp::create(rewriter, loc, lhsType, op.getRhs());
       std::optional<Value> mult = createContractArithOp(
           loc, op.getLhs(), b, acc, kind, rewriter, isInt, mask);
       if (!mult.has_value())
@@ -1249,23 +1249,23 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
       return success();
     }
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
     for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
-      Value x = rewriter.create<vector::ExtractOp>(loc, op.getLhs(), d);
-      Value a = rewriter.create<vector::BroadcastOp>(loc, rhsType, x);
+      Value x = vector::ExtractOp::create(rewriter, loc, op.getLhs(), d);
+      Value a = vector::BroadcastOp::create(rewriter, loc, rhsType, x);
       Value r = nullptr;
       if (acc)
-        r = rewriter.create<vector::ExtractOp>(loc, acc, d);
+        r = vector::ExtractOp::create(rewriter, loc, acc, d);
       Value extrMask;
       if (mask)
-        extrMask = rewriter.create<vector::ExtractOp>(loc, mask, d);
+        extrMask = vector::ExtractOp::create(rewriter, loc, mask, d);
 
       std::optional<Value> m = createContractArithOp(
           loc, a, op.getRhs(), r, kind, rewriter, isInt, extrMask);
       if (!m.has_value())
         return failure();
-      result = rewriter.create<vector::InsertOp>(loc, *m, result, d);
+      result = vector::InsertOp::create(rewriter, loc, *m, result, d);
     }
 
     rewriter.replaceOp(rootOp, result);
@@ -1335,7 +1335,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   Value lhs = op.getLhs();
   auto lhsMap = op.getIndexingMapsArray()[0];
   if (lhsMap == AffineMap::get(3, 0, {k, m}, ctx))
-    lhs = rew.create<vector::TransposeOp>(loc, lhs, ArrayRef<int64_t>{1, 0});
+    lhs = vector::TransposeOp::create(rew, loc, lhs, ArrayRef<int64_t>{1, 0});
   else if (lhsMap != AffineMap::get(3, 0, {m, k}, ctx))
     return failure();
 
@@ -1343,7 +1343,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   Value rhs = op.getRhs();
   auto rhsMap = op.getIndexingMapsArray()[1];
   if (rhsMap == AffineMap::get(3, 0, {n, k}, ctx))
-    rhs = rew.create<vector::TransposeOp>(loc, rhs, ArrayRef<int64_t>{1, 0});
+    rhs = vector::TransposeOp::create(rew, loc, rhs, ArrayRef<int64_t>{1, 0});
   else if (rhsMap != AffineMap::get(3, 0, {k, n}, ctx))
     return failure();
 
@@ -1356,15 +1356,15 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
 
   Type flattenedLHSType =
       VectorType::get(lhsType.getNumElements(), lhsType.getElementType());
-  lhs = rew.create<vector::ShapeCastOp>(loc, flattenedLHSType, lhs);
+  lhs = vector::ShapeCastOp::create(rew, loc, flattenedLHSType, lhs);
 
   Type flattenedRHSType =
       VectorType::get(rhsType.getNumElements(), rhsType.getElementType());
-  rhs = rew.create<vector::ShapeCastOp>(loc, flattenedRHSType, rhs);
+  rhs = vector::ShapeCastOp::create(rew, loc, flattenedRHSType, rhs);
 
-  Value mul = rew.create<vector::MatmulOp>(loc, lhs, rhs, lhsRows, lhsColumns,
+  Value mul = vector::MatmulOp::create(rew, loc, lhs, rhs, lhsRows, lhsColumns,
                                            rhsColumns);
-  mul = rew.create<vector::ShapeCastOp>(
+  mul = vector::ShapeCastOp::create(rew,
       loc,
       VectorType::get({lhsRows, rhsColumns},
                       getElementTypeOrSelf(op.getAcc().getType())),
@@ -1373,15 +1373,15 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   // ACC must be C(m, n) or C(n, m).
   auto accMap = op.getIndexingMapsArray()[2];
   if (accMap == AffineMap::get(3, 0, {n, m}, ctx))
-    mul = rew.create<vector::TransposeOp>(loc, mul, ArrayRef<int64_t>{1, 0});
+    mul = vector::TransposeOp::create(rew, loc, mul, ArrayRef<int64_t>{1, 0});
   else if (accMap != AffineMap::get(3, 0, {m, n}, ctx))
     llvm_unreachable("invalid contraction semantics");
 
   Value res =
       isa<IntegerType>(elementType)
-          ? static_cast<Value>(rew.create<arith::AddIOp>(loc, op.getAcc(), mul))
+          ? static_cast<Value>(arith::AddIOp::create(rew, loc, op.getAcc(), mul))
           : static_cast<Value>(
-                rew.create<arith::AddFOp>(loc, op.getAcc(), mul));
+                arith::AddFOp::create(rew, loc, op.getAcc(), mul));
 
   return res;
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index 3000204c8ce17..c586059ca55bc 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -75,7 +75,7 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
     Value maskVec = op.getMask();
     Value passThruVec = op.getPassThru();
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultTy, rewriter.getZeroAttr(resultTy));
 
     VectorType subTy = VectorType::Builder(resultTy).dropDim(0);
@@ -84,16 +84,16 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
       int64_t thisIdx[1] = {i};
 
       Value indexSubVec =
-          rewriter.create<vector::ExtractOp>(loc, indexVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
       Value maskSubVec =
-          rewriter.create<vector::ExtractOp>(loc, maskVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, maskVec, thisIdx);
       Value passThruSubVec =
-          rewriter.create<vector::ExtractOp>(loc, passThruVec, thisIdx);
-      Value subGather = rewriter.create<vector::GatherOp>(
+          vector::ExtractOp::create(rewriter, loc, passThruVec, thisIdx);
+      Value subGather = vector::GatherOp::create(rewriter,
           loc, subTy, op.getBase(), op.getIndices(), indexSubVec, maskSubVec,
           passThruSubVec);
       result =
-          rewriter.create<vector::InsertOp>(loc, subGather, result, thisIdx);
+          vector::InsertOp::create(rewriter, loc, subGather, result, thisIdx);
     }
 
     rewriter.replaceOp(op, result);
@@ -159,22 +159,22 @@ struct RemoveStrideFromGatherSource : OpRewritePattern<vector::GatherOp> {
 
     // 1. Collapse the input memref so that it's "flat".
     SmallVector<ReassociationIndices> reassoc = {{0, 1}};
-    Value collapsed = rewriter.create<memref::CollapseShapeOp>(
+    Value collapsed = memref::CollapseShapeOp::create(rewriter,
         op.getLoc(), subview.getSource(), reassoc);
 
     // 2. Generate new gather indices that will model the
     // strided access.
     IntegerAttr stride = rewriter.getIndexAttr(srcTrailingDim);
     VectorType vType = op.getIndexVec().getType();
-    Value mulCst = rewriter.create<arith::ConstantOp>(
+    Value mulCst = arith::ConstantOp::create(rewriter,
         op.getLoc(), vType, DenseElementsAttr::get(vType, stride));
 
     Value newIdxs =
-        rewriter.create<arith::MulIOp>(op.getLoc(), op.getIndexVec(), mulCst);
+        arith::MulIOp::create(rewriter, op.getLoc(), op.getIndexVec(), mulCst);
 
     // 3. Create an updated gather op with the collapsed input memref and the
     // updated indices.
-    Value newGather = rewriter.create<vector::GatherOp>(
+    Value newGather = vector::GatherOp::create(rewriter,
         op.getLoc(), op.getResult().getType(), collapsed, op.getIndices(),
         newIdxs, op.getMask(), op.getPassThru());
     rewriter.replaceOp(op, newGather);
@@ -229,8 +229,8 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
     for (int64_t i = 0, e = resultTy.getNumElements(); i < e; ++i) {
       int64_t thisIdx[1] = {i};
       Value condition =
-          rewriter.create<vector::ExtractOp>(loc, condMask, thisIdx);
-      Value index = rewriter.create<vector::ExtractOp>(loc, indexVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, condMask, thisIdx);
+      Value index = vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
       baseOffsets.back() =
           rewriter.createOrFold<arith::AddIOp>(loc, lastBaseOffset, index);
 
@@ -240,19 +240,19 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
           // `vector.load` does not support scalar result; emit a vector load
           // and extract the single result instead.
           Value load =
-              b.create<vector::LoadOp>(loc, elemVecTy, base, baseOffsets);
+              vector::LoadOp::create(b, loc, elemVecTy, base, baseOffsets);
           int64_t zeroIdx[1] = {0};
-          extracted = b.create<vector::ExtractOp>(loc, load, zeroIdx);
+          extracted = vector::ExtractOp::create(b, loc, load, zeroIdx);
         } else {
-          extracted = b.create<tensor::ExtractOp>(loc, base, baseOffsets);
+          extracted = tensor::ExtractOp::create(b, loc, base, baseOffsets);
         }
 
         Value newResult =
-            b.create<vector::InsertOp>(loc, extracted, result, thisIdx);
-        b.create<scf::YieldOp>(loc, newResult);
+            vector::InsertOp::create(b, loc, extracted, result, thisIdx);
+        scf::YieldOp::create(b, loc, newResult);
       };
       auto passThruBuilder = [result](OpBuilder &b, Location loc) {
-        b.create<scf::YieldOp>(loc, result);
+        scf::YieldOp::create(b, loc, result);
       };
 
       result =
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index cab0f213b14a9..b0affbb699d9e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -60,14 +60,14 @@ class UnrollInterleaveOp final : public OpRewritePattern<vector::InterleaveOp> {
       return failure();
 
     auto loc = op.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
     for (auto position : *unrollIterator) {
-      Value extractLhs = rewriter.create<ExtractOp>(loc, op.getLhs(), position);
-      Value extractRhs = rewriter.create<ExtractOp>(loc, op.getRhs(), position);
+      Value extractLhs = ExtractOp::create(rewriter, loc, op.getLhs(), position);
+      Value extractRhs = ExtractOp::create(rewriter, loc, op.getRhs(), position);
       Value interleave =
-          rewriter.create<InterleaveOp>(loc, extractLhs, extractRhs);
-      result = rewriter.create<InsertOp>(loc, interleave, result, position);
+          InterleaveOp::create(rewriter, loc, extractLhs, extractRhs);
+      result = InsertOp::create(rewriter, loc, interleave, result, position);
     }
 
     rewriter.replaceOp(op, result);
@@ -123,19 +123,19 @@ class UnrollDeinterleaveOp final
       return failure();
 
     auto loc = op.getLoc();
-    Value emptyResult = rewriter.create<arith::ConstantOp>(
+    Value emptyResult = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
     Value evenResult = emptyResult;
     Value oddResult = emptyResult;
 
     for (auto position : *unrollIterator) {
       auto extractSrc =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+          vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       auto deinterleave =
-          rewriter.create<vector::DeinterleaveOp>(loc, extractSrc);
-      evenResult = rewriter.create<vector::InsertOp>(
+          vector::DeinterleaveOp::create(rewriter, loc, extractSrc);
+      evenResult = vector::InsertOp::create(rewriter,
           loc, deinterleave.getRes1(), evenResult, position);
-      oddResult = rewriter.create<vector::InsertOp>(loc, deinterleave.getRes2(),
+      oddResult = vector::InsertOp::create(rewriter, loc, deinterleave.getRes2(),
                                                     oddResult, position);
     }
     rewriter.replaceOp(op, ValueRange{evenResult, oddResult});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
index ba21092d2af3c..edd840d8fbc7b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
@@ -67,19 +67,19 @@ class CreateMaskOpLowering : public OpRewritePattern<vector::CreateMaskOp> {
     Value idx = op.getOperand(0);
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = rewriter.create<vector::CreateMaskOp>(
+    Value trueVal = vector::CreateMaskOp::create(rewriter,
         loc, lowType, op.getOperands().drop_front());
-    Value falseVal = rewriter.create<arith::ConstantOp>(
+    Value falseVal = arith::ConstantOp::create(rewriter,
         loc, lowType, rewriter.getZeroAttr(lowType));
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstType, rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < dim; d++) {
       Value bnd =
-          rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(d));
-      Value val = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+          arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(d));
+      Value val = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                  bnd, idx);
-      Value sel = rewriter.create<arith::SelectOp>(loc, val, trueVal, falseVal);
-      result = rewriter.create<vector::InsertOp>(loc, sel, result, d);
+      Value sel = arith::SelectOp::create(rewriter, loc, val, trueVal, falseVal);
+      result = vector::InsertOp::create(rewriter, loc, sel, result, d);
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -146,12 +146,12 @@ class ConstantMaskOpLowering : public OpRewritePattern<vector::ConstantMaskOp> {
           op, "Cannot unroll leading scalable dim in dstType");
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = rewriter.create<vector::ConstantMaskOp>(
+    Value trueVal = vector::ConstantMaskOp::create(rewriter,
         loc, lowType, dimSizes.drop_front());
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstType, rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < trueDimSize; d++)
-      result = rewriter.create<vector::InsertOp>(loc, trueVal, result, d);
+      result = vector::InsertOp::create(rewriter, loc, trueVal, result, d);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -261,7 +261,7 @@ struct MaskedGatherOpPattern : public MaskOpRewritePattern<GatherOp> {
                             PatternRewriter &rewriter) const override {
     Value passthru = maskingOp.hasPassthru()
                          ? maskingOp.getPassthru()
-                         : rewriter.create<arith::ConstantOp>(
+                         : arith::ConstantOp::create(rewriter,
                                gatherOp.getLoc(),
                                rewriter.getZeroAttr(gatherOp.getVectorType()));
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
index ce524b259d8d4..c5631ba00bbc6 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
@@ -103,12 +103,12 @@ class InnerOuterDimReductionConversion
     // If masked, transpose the original mask.
     Value transposedMask;
     if (maskableOp.isMasked()) {
-      transposedMask = rewriter.create<vector::TransposeOp>(
+      transposedMask = vector::TransposeOp::create(rewriter,
           loc, maskableOp.getMaskingOp().getMask(), indices);
     }
 
     // Transpose reduction source.
-    auto transposeOp = rewriter.create<vector::TransposeOp>(loc, src, indices);
+    auto transposeOp = vector::TransposeOp::create(rewriter, loc, src, indices);
     SmallVector<bool> reductionMask(srcRank, false);
     for (int i = 0; i < reductionSize; ++i) {
       if (useInnerDimsForReduction)
@@ -117,7 +117,7 @@ class InnerOuterDimReductionConversion
         reductionMask[i] = true;
     }
 
-    Operation *newMultiRedOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newMultiRedOp = vector::MultiDimReductionOp::create(rewriter,
         multiReductionOp.getLoc(), transposeOp.getResult(),
         multiReductionOp.getAcc(), reductionMask, multiReductionOp.getKind());
     newMultiRedOp =
@@ -256,13 +256,13 @@ class ReduceMultiDimReductionRank
           vectorShape,
           llvm::cast<VectorType>(vectorMask.getType()).getElementType());
       newVectorMask =
-          rewriter.create<vector::ShapeCastOp>(loc, maskCastedType, vectorMask);
+          vector::ShapeCastOp::create(rewriter, loc, maskCastedType, vectorMask);
     }
 
     auto castedType = VectorType::get(
         vectorShape, multiReductionOp.getSourceVectorType().getElementType(),
         scalableDims);
-    Value cast = rewriter.create<vector::ShapeCastOp>(
+    Value cast = vector::ShapeCastOp::create(rewriter,
         loc, castedType, multiReductionOp.getSource());
 
     Value acc = multiReductionOp.getAcc();
@@ -271,11 +271,11 @@ class ReduceMultiDimReductionRank
           {flattenedParallelDim},
           multiReductionOp.getSourceVectorType().getElementType(),
           /*scalableDims=*/{isParallelDimScalable});
-      acc = rewriter.create<vector::ShapeCastOp>(loc, accType, acc);
+      acc = vector::ShapeCastOp::create(rewriter, loc, accType, acc);
     }
     // 6. Creates the flattened form of vector.multi_reduction with inner/outer
     // most dim as reduction.
-    Operation *newMultiDimRedOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(rewriter,
         loc, cast, acc, mask, multiReductionOp.getKind());
     newMultiDimRedOp =
         mlir::vector::maskOperation(rewriter, newMultiDimRedOp, newVectorMask);
@@ -339,11 +339,11 @@ struct TwoDimMultiReductionToElementWise
 
     Value result = multiReductionOp.getAcc();
     for (int64_t i = 0; i < srcShape[0]; i++) {
-      auto operand = rewriter.create<vector::ExtractOp>(
+      auto operand = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getSource(), i);
       Value extractMask = nullptr;
       if (mask) {
-        extractMask = rewriter.create<vector::ExtractOp>(loc, mask, i);
+        extractMask = vector::ExtractOp::create(rewriter, loc, mask, i);
       }
       result =
           makeArithReduction(rewriter, loc, multiReductionOp.getKind(), operand,
@@ -383,27 +383,27 @@ struct TwoDimMultiReductionToReduction
     }
 
     auto loc = multiReductionOp.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, multiReductionOp.getDestType(),
         rewriter.getZeroAttr(multiReductionOp.getDestType()));
     int outerDim = multiReductionOp.getSourceVectorType().getShape()[0];
 
     for (int i = 0; i < outerDim; ++i) {
-      auto v = rewriter.create<vector::ExtractOp>(
+      auto v = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
-      auto acc = rewriter.create<vector::ExtractOp>(
+      auto acc = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getAcc(), ArrayRef<int64_t>{i});
-      Operation *reductionOp = rewriter.create<vector::ReductionOp>(
+      Operation *reductionOp = vector::ReductionOp::create(rewriter,
           loc, multiReductionOp.getKind(), v, acc);
 
       // If masked, slice the mask and mask the new reduction operation.
       if (maskableOp.isMasked()) {
-        Value mask = rewriter.create<vector::ExtractOp>(
+        Value mask = vector::ExtractOp::create(rewriter,
             loc, maskableOp.getMaskingOp().getMask(), ArrayRef<int64_t>{i});
         reductionOp = mlir::vector::maskOperation(rewriter, reductionOp, mask);
       }
 
-      result = rewriter.create<vector::InsertOp>(loc, reductionOp->getResult(0),
+      result = vector::InsertOp::create(rewriter, loc, reductionOp->getResult(0),
                                                  result, i);
     }
 
@@ -459,9 +459,9 @@ struct OneDimMultiReductionToTwoDim
     SmallVector<bool, 2> reductionMask{false, true};
 
     /// vector.extract(vector.multi_reduce(vector.shape_cast(v, 1xk)), 0)
-    Value cast = rewriter.create<vector::ShapeCastOp>(
+    Value cast = vector::ShapeCastOp::create(rewriter,
         loc, castedType, multiReductionOp.getSource());
-    Value castAcc = rewriter.create<vector::BroadcastOp>(
+    Value castAcc = vector::BroadcastOp::create(rewriter,
         loc, accType, multiReductionOp.getAcc());
     Value castMask;
     if (maskableOp.isMasked()) {
@@ -470,10 +470,10 @@ struct OneDimMultiReductionToTwoDim
           ArrayRef<int64_t>{1, maskType.getShape().back()},
           maskType.getElementType(),
           ArrayRef<bool>{false, maskType.getScalableDims().back()});
-      castMask = rewriter.create<vector::BroadcastOp>(loc, castMaskType, mask);
+      castMask = vector::BroadcastOp::create(rewriter, loc, castMaskType, mask);
     }
 
-    Operation *newOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newOp = vector::MultiDimReductionOp::create(rewriter,
         loc, cast, castAcc, reductionMask, multiReductionOp.getKind());
     newOp = vector::maskOperation(rewriter, newOp, castMask);
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
index a1f67bd0e9ed3..a6985b96b59df 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
@@ -122,7 +122,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
       return failure();
 
     VectorType resType = VectorType::get(destShape, elType);
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
     int64_t reductionDim = scanOp.getReductionDim();
     bool inclusive = scanOp.getInclusive();
@@ -144,7 +144,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
     for (int i = 0; i < destShape[reductionDim]; i++) {
       offsets[reductionDim] = i;
       ArrayAttr scanOffsets = rewriter.getI64ArrayAttr(offsets);
-      Value input = rewriter.create<vector::ExtractStridedSliceOp>(
+      Value input = vector::ExtractStridedSliceOp::create(rewriter,
           loc, reductionType, scanOp.getSource(), scanOffsets, scanSizes,
           scanStrides);
       Value output;
@@ -154,10 +154,10 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         } else {
           if (initialValueRank == 0) {
             // ShapeCastOp cannot handle 0-D vectors
-            output = rewriter.create<vector::BroadcastOp>(
+            output = vector::BroadcastOp::create(rewriter,
                 loc, input.getType(), scanOp.getInitialValue());
           } else {
-            output = rewriter.create<vector::ShapeCastOp>(
+            output = vector::ShapeCastOp::create(rewriter,
                 loc, input.getType(), scanOp.getInitialValue());
           }
         }
@@ -166,7 +166,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         output = vector::makeArithReduction(rewriter, loc, scanOp.getKind(),
                                             lastOutput, y);
       }
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = vector::InsertStridedSliceOp::create(rewriter,
           loc, output, result, offsets, strides);
       lastOutput = output;
       lastInput = input;
@@ -174,11 +174,11 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
 
     Value reduction;
     if (initialValueRank == 0) {
-      Value v = rewriter.create<vector::ExtractOp>(loc, lastOutput, 0);
+      Value v = vector::ExtractOp::create(rewriter, loc, lastOutput, 0);
       reduction =
-          rewriter.create<vector::BroadcastOp>(loc, initialValueType, v);
+          vector::BroadcastOp::create(rewriter, loc, initialValueType, v);
     } else {
-      reduction = rewriter.create<vector::ShapeCastOp>(loc, initialValueType,
+      reduction = vector::ShapeCastOp::create(rewriter, loc, initialValueType,
                                                        lastOutput);
     }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
index 39c16fab21c4e..69d3666a13879 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
@@ -137,10 +137,10 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     const int64_t resultLeading = delta > 0 ? 0 : -delta;
 
     const Value source = shapeCast.getSource();
-    const Value poison = rewriter.create<ub::PoisonOp>(loc, resultType);
-    const Value extracted = rewriter.create<vector::ExtractOp>(
+    const Value poison = ub::PoisonOp::create(rewriter, loc, resultType);
+    const Value extracted = vector::ExtractOp::create(rewriter,
         loc, source, SmallVector<int64_t>(sourceLeading, 0));
-    const Value result = rewriter.create<vector::InsertOp>(
+    const Value result = vector::InsertOp::create(rewriter,
         loc, extracted, poison, SmallVector<int64_t>(resultLeading, 0));
 
     rewriter.replaceOp(shapeCast, result);
@@ -171,13 +171,13 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 
     SmallVector<int64_t> extractIndex(sourceDim, 0);
     SmallVector<int64_t> insertIndex(resultDim, 0);
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
 
     for (int i = 0; i < nSlices; ++i) {
       Value extracted =
-          rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+          vector::ExtractOp::create(rewriter, loc, source, extractIndex);
 
-      result = rewriter.create<vector::InsertOp>(loc, extracted, result,
+      result = vector::InsertOp::create(rewriter, loc, extracted, result,
                                                  insertIndex);
 
       inplaceAdd(1, sourceShape.take_front(sourceDim), extractIndex);
@@ -276,9 +276,9 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     Value extracted = {};
     Value extractedStrided = {};
     Value insertedSlice = {};
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
     const Value partResult =
-        rewriter.create<ub::PoisonOp>(loc, insertStridedType);
+        ub::PoisonOp::create(rewriter, loc, insertStridedType);
 
     for (size_t i = 0; i < nAtomicSlices; ++i) {
 
@@ -288,14 +288,14 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
       // vector.extract
       if (extractStridedPhase == 0) {
         extracted =
-            rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+            vector::ExtractOp::create(rewriter, loc, source, extractIndex);
         inplaceAdd(1, sourceShape.take_front(sourceSuffixStartDim),
                    extractIndex);
       }
 
       // vector.extract_strided_slice
       extractOffsets[0] = extractStridedPhase * greatestCommonDivisor;
-      extractedStrided = rewriter.create<vector::ExtractStridedSliceOp>(
+      extractedStrided = vector::ExtractStridedSliceOp::create(rewriter,
           loc, extracted, extractOffsets, atomicShape, sizes);
 
       // vector.insert_strided_slice
@@ -303,12 +303,12 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
         insertedSlice = partResult;
       }
       insertOffsets[0] = insertStridedPhase * greatestCommonDivisor;
-      insertedSlice = rewriter.create<vector::InsertStridedSliceOp>(
+      insertedSlice = vector::InsertStridedSliceOp::create(rewriter,
           loc, extractedStrided, insertedSlice, insertOffsets, sizes);
 
       // vector.insert
       if (insertStridedPhase + 1 == insertPeriod) {
-        result = rewriter.create<vector::InsertOp>(loc, insertedSlice, result,
+        result = vector::InsertOp::create(rewriter, loc, insertedSlice, result,
                                                    insertIndex);
         inplaceAdd(1, resultType.getShape().take_front(resultSuffixStartDim),
                    insertIndex);
@@ -394,7 +394,7 @@ class ScalableShapeCastOpRewritePattern
     auto extractionVectorType = VectorType::get(
         {minExtractionSize}, sourceVectorType.getElementType(), {true});
 
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultVectorType);
     SmallVector<int64_t> srcIdx(srcRank, 0);
     SmallVector<int64_t> resIdx(resRank, 0);
 
@@ -406,7 +406,7 @@ class ScalableShapeCastOpRewritePattern
       // 1. Extract a scalable subvector from the source vector.
       if (!currentSourceScalableVector) {
         if (srcRank != 1) {
-          currentSourceScalableVector = rewriter.create<vector::ExtractOp>(
+          currentSourceScalableVector = vector::ExtractOp::create(rewriter,
               loc, op.getSource(), llvm::ArrayRef(srcIdx).drop_back());
         } else {
           currentSourceScalableVector = op.getSource();
@@ -414,7 +414,7 @@ class ScalableShapeCastOpRewritePattern
       }
       Value sourceSubVector = currentSourceScalableVector;
       if (minExtractionSize < minSourceTrailingSize) {
-        sourceSubVector = rewriter.create<vector::ScalableExtractOp>(
+        sourceSubVector = vector::ScalableExtractOp::create(rewriter,
             loc, extractionVectorType, sourceSubVector, srcIdx.back());
       }
 
@@ -423,14 +423,14 @@ class ScalableShapeCastOpRewritePattern
         if (minExtractionSize == minResultTrailingSize) {
           currentResultScalableVector = sourceSubVector;
         } else if (resRank != 1) {
-          currentResultScalableVector = rewriter.create<vector::ExtractOp>(
+          currentResultScalableVector = vector::ExtractOp::create(rewriter,
               loc, result, llvm::ArrayRef(resIdx).drop_back());
         } else {
           currentResultScalableVector = result;
         }
       }
       if (minExtractionSize < minResultTrailingSize) {
-        currentResultScalableVector = rewriter.create<vector::ScalableInsertOp>(
+        currentResultScalableVector = vector::ScalableInsertOp::create(rewriter,
             loc, sourceSubVector, currentResultScalableVector, resIdx.back());
       }
 
@@ -439,7 +439,7 @@ class ScalableShapeCastOpRewritePattern
           currentResultScalableVector != result) {
         // Finished row of result. Insert complete scalable vector into result
         // (n-D) vector.
-        result = rewriter.create<vector::InsertOp>(
+        result = vector::InsertOp::create(rewriter,
             loc, currentResultScalableVector, result,
             llvm::ArrayRef(resIdx).drop_back());
         currentResultScalableVector = {};
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
index 5b81d0d33d484..fe02cd5656571 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
@@ -47,7 +47,7 @@ static Value extendVectorRank(OpBuilder &builder, Location loc, Value vec,
                          originalVecType.getScalableDims().end());
   VectorType newVecType = VectorType::get(
       newShape, originalVecType.getElementType(), newScalableDims);
-  return builder.create<vector::BroadcastOp>(loc, newVecType, vec);
+  return vector::BroadcastOp::create(builder, loc, newVecType, vec);
 }
 
 /// Extend the rank of a vector Value by `addedRanks` by adding inner unit
@@ -62,7 +62,7 @@ static Value extendMaskRank(OpBuilder &builder, Location loc, Value vec,
     permutation.push_back(i);
   for (int64_t i = 0; i < addedRank; ++i)
     permutation.push_back(i);
-  return builder.create<vector::TransposeOp>(loc, broadcasted, permutation);
+  return vector::TransposeOp::create(builder, loc, broadcasted, permutation);
 }
 
 //===----------------------------------------------------------------------===//
@@ -138,7 +138,7 @@ struct TransferReadPermutationLowering
     // Generate new transfer_read operation.
     VectorType newReadType = VectorType::get(
         newVectorShape, op.getVectorType().getElementType(), newScalableDims);
-    Value newRead = rewriter.create<vector::TransferReadOp>(
+    Value newRead = vector::TransferReadOp::create(rewriter,
         op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
@@ -209,11 +209,11 @@ struct TransferWritePermutationLowering
         inverseTransposeInBoundsAttr(rewriter, op.getInBounds(), permutation);
 
     // Generate new transfer_write operation.
-    Value newVec = rewriter.create<vector::TransposeOp>(
+    Value newVec = vector::TransposeOp::create(rewriter,
         op.getLoc(), op.getVector(), indices);
     auto newMap = AffineMap::getMinorIdentityMap(
         map.getNumDims(), map.getNumResults(), rewriter.getContext());
-    auto newWrite = rewriter.create<vector::TransferWriteOp>(
+    auto newWrite = vector::TransferWriteOp::create(rewriter,
         op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getMask(), newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
@@ -299,7 +299,7 @@ struct TransferWriteNonPermutationLowering
       newInBoundsValues.push_back(op.isDimInBounds(i));
     }
     ArrayAttr newInBoundsAttr = rewriter.getBoolArrayAttr(newInBoundsValues);
-    auto newWrite = rewriter.create<vector::TransferWriteOp>(
+    auto newWrite = vector::TransferWriteOp::create(rewriter,
         op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), newMask, newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
@@ -370,7 +370,7 @@ struct TransferOpReduceRank
             ? rewriter.getArrayAttr(
                   op.getInBoundsAttr().getValue().take_back(reducedShapeRank))
             : ArrayAttr();
-    Value newRead = rewriter.create<vector::TransferReadOp>(
+    Value newRead = vector::TransferReadOp::create(rewriter,
         op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
@@ -471,20 +471,20 @@ struct TransferReadToVectorLoadLowering
             read, "vector type is not rank 1, can't create masked load, needs "
                   "VectorToSCF");
 
-      Value fill = rewriter.create<vector::SplatOp>(
+      Value fill = vector::SplatOp::create(rewriter,
           read.getLoc(), unbroadcastedVectorType, read.getPadding());
-      res = rewriter.create<vector::MaskedLoadOp>(
+      res = vector::MaskedLoadOp::create(rewriter,
           read.getLoc(), unbroadcastedVectorType, read.getBase(),
           read.getIndices(), read.getMask(), fill);
     } else {
-      res = rewriter.create<vector::LoadOp>(read.getLoc(),
+      res = vector::LoadOp::create(rewriter, read.getLoc(),
                                             unbroadcastedVectorType,
                                             read.getBase(), read.getIndices());
     }
 
     // Insert a broadcasting op if required.
     if (!broadcastedDims.empty())
-      res = rewriter.create<vector::BroadcastOp>(
+      res = vector::BroadcastOp::create(rewriter,
           read.getLoc(), read.getVectorType(), res->getResult(0));
     return res->getResult(0);
   }
@@ -569,11 +569,11 @@ struct TransferWriteToVectorStoreLowering
                    << write;
             });
 
-      rewriter.create<vector::MaskedStoreOp>(
+      vector::MaskedStoreOp::create(rewriter,
           write.getLoc(), write.getBase(), write.getIndices(), write.getMask(),
           write.getVector());
     } else {
-      rewriter.create<vector::StoreOp>(write.getLoc(), write.getVector(),
+      vector::StoreOp::create(rewriter, write.getLoc(), write.getVector(),
                                        write.getBase(), write.getIndices());
     }
     // There's no return value for StoreOps. Use Value() to signal success to
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index 732e316c93381..e9591bfcf6feb 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -80,7 +80,7 @@ getUnpackShufflePermFor128Lane(ArrayRef<int64_t> vals, int numBits) {
 static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({0, 1, numElem, numElem + 1}, numBits));
 }
@@ -94,7 +94,7 @@ static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({2, 3, numElem + 2, numElem + 3},
                                      numBits));
@@ -109,7 +109,7 @@ static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  auto shuffle = b.create<vector::ShuffleOp>(
+  auto shuffle = vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({0, numElem, 1, numElem + 1}, numBits));
   return shuffle;
@@ -124,7 +124,7 @@ static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({2, numElem + 2, 3, numElem + 3},
                                      numBits));
@@ -181,7 +181,7 @@ static Value create4x128BitSuffle(ImplicitLocOpBuilder &b, Value v1, Value v2,
   appendToMask(0, b23);
   appendToMask(16, b45);
   appendToMask(16, b67);
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// Lowers the value to a vector.shuffle op. The `source` is expected to be a
@@ -192,7 +192,7 @@ static Value transposeToShuffle1D(OpBuilder &b, Value source, int m, int n) {
   for (int64_t j = 0; j < n; ++j)
     for (int64_t i = 0; i < m; ++i)
       mask.push_back(i * n + j);
-  return b.create<vector::ShuffleOp>(source.getLoc(), source, source, mask);
+  return vector::ShuffleOp::create(b, source.getLoc(), source, source, mask);
 }
 
 /// Lowers the value to a sequence of vector.shuffle ops. The `source` is
@@ -284,9 +284,9 @@ static Value transposeToShuffle16x16(OpBuilder &builder, Value source, int m,
 
   auto reshInputType = VectorType::get(
       {m, n}, cast<VectorType>(source.getType()).getElementType());
-  Value res = b.create<ub::PoisonOp>(reshInputType);
+  Value res = ub::PoisonOp::create(b, reshInputType);
   for (int64_t i = 0; i < m; ++i)
-    res = b.create<vector::InsertOp>(vs[i], res, i);
+    res = vector::InsertOp::create(b, vs[i], res, i);
   return res;
 }
 
@@ -335,10 +335,10 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
       Type flattenedType =
           VectorType::get(resType.getNumElements(), resType.getElementType());
       auto matrix =
-          rewriter.create<vector::ShapeCastOp>(loc, flattenedType, input);
+          vector::ShapeCastOp::create(rewriter, loc, flattenedType, input);
       auto rows = rewriter.getI32IntegerAttr(resType.getShape()[0]);
       auto columns = rewriter.getI32IntegerAttr(resType.getShape()[1]);
-      Value trans = rewriter.create<vector::FlatTransposeOp>(
+      Value trans = vector::FlatTransposeOp::create(rewriter,
           loc, flattenedType, matrix, rows, columns);
       rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, trans);
       return success();
@@ -359,7 +359,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
     // of the leftmost transposed dimensions. We traverse every transpose
     // element using a linearized index that we delinearize to generate the
     // appropriate indices for the extract/insert operations.
-    Value result = rewriter.create<ub::PoisonOp>(loc, resType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resType);
     int64_t numTransposedElements = ShapedType::getNumElements(prunedInShape);
 
     for (int64_t linearIdx = 0; linearIdx < numTransposedElements;
@@ -482,14 +482,14 @@ class TransposeOp2DToShuffleLowering
     Location loc = op.getLoc();
     auto flattenedType = VectorType::get({n * m}, srcType.getElementType());
     auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
-    auto reshInput = rewriter.create<vector::ShapeCastOp>(loc, flattenedType,
+    auto reshInput = vector::ShapeCastOp::create(rewriter, loc, flattenedType,
                                                           op.getVector());
 
     Value res;
     if (vectorTransposeLowering == VectorTransposeLowering::Shuffle16x16 &&
         m == 16 && n == 16) {
       reshInput =
-          rewriter.create<vector::ShapeCastOp>(loc, reshInputType, reshInput);
+          vector::ShapeCastOp::create(rewriter, loc, reshInputType, reshInput);
       res = transposeToShuffle16x16(rewriter, reshInput, m, n);
     } else {
       // Fallback to shuffle on 1D approach.
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index c8566b1ff83ef..2683a85ecdaeb 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -114,7 +114,7 @@ struct DistributedLoadStoreHelper {
            "preregistered sequential value.");
     // Scalar case can directly use memref.store.
     if (!isa<VectorType>(val.getType()))
-      return b.create<memref::StoreOp>(loc, val, buffer, zero);
+      return memref::StoreOp::create(b, loc, val, buffer, zero);
 
     // Vector case must use vector::TransferWriteOp which will later lower to
     //   vector.store of memref.store depending on further lowerings.
@@ -127,7 +127,7 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return b.create<vector::TransferWriteOp>(
+    return vector::TransferWriteOp::create(b,
         loc, val, buffer, indices,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
@@ -156,7 +156,7 @@ struct DistributedLoadStoreHelper {
 
     // Scalar case can directly use memref.store.
     if (!isa<VectorType>(type))
-      return b.create<memref::LoadOp>(loc, buffer, zero);
+      return memref::LoadOp::create(b, loc, buffer, zero);
 
     // Other cases must be vector atm.
     // Vector case must use vector::TransferReadOp which will later lower to
@@ -172,7 +172,7 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return b.create<vector::TransferReadOp>(
+    return vector::TransferReadOp::create(b,
         loc, cast<VectorType>(type), buffer, indices, /*padding=*/std::nullopt,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
@@ -243,10 +243,10 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
     rewriter.setInsertionPoint(warpOp);
 
     // Step 1: Create scf.if op.
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    Value isLane0 = rewriter.create<arith::CmpIOp>(
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    Value isLane0 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, warpOp.getLaneid(), c0);
-    auto ifOp = rewriter.create<scf::IfOp>(loc, isLane0,
+    auto ifOp = scf::IfOp::create(rewriter, loc, isLane0,
                                            /*withElseRegion=*/false);
     rewriter.eraseOp(ifOp.thenBlock()->getTerminator());
 
@@ -325,7 +325,7 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
     // Step 7. Delete terminator and add empty scf.yield.
     rewriter.eraseOp(yieldOp);
     rewriter.setInsertionPointToEnd(ifOp.thenBlock());
-    rewriter.create<scf::YieldOp>(yieldLoc);
+    scf::YieldOp::create(rewriter, yieldLoc);
 
     // Compute replacements for WarpOp results.
     rewriter.replaceOp(warpOp, replacements);
@@ -512,7 +512,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
 
     // Create a second warp op that contains only writeOp.
-    auto secondWarpOp = rewriter.create<WarpExecuteOnLane0Op>(
+    auto secondWarpOp = WarpExecuteOnLane0Op::create(rewriter,
         loc, TypeRange(), newWarpOp.getLaneid(), newWarpOp.getWarpSize());
     Block &body = secondWarpOp.getBodyRegion().front();
     rewriter.setInsertionPointToStart(&body);
@@ -521,7 +521,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     newWriteOp.getValueToStoreMutable().assign(
         newWarpOp.getResult(newRetIndices[0]));
     rewriter.eraseOp(writeOp);
-    rewriter.create<gpu::YieldOp>(newWarpOp.getLoc());
+    gpu::YieldOp::create(rewriter, newWarpOp.getLoc());
     return success();
   }
 
@@ -698,7 +698,7 @@ struct WarpOpConstant : public WarpDistributionPattern {
         cast<ShapedType>(warpOp.getResult(operandIndex).getType()), scalarAttr);
     Location loc = warpOp.getLoc();
     rewriter.setInsertionPointAfter(warpOp);
-    Value distConstant = rewriter.create<arith::ConstantOp>(loc, newAttr);
+    Value distConstant = arith::ConstantOp::create(rewriter, loc, newAttr);
     rewriter.replaceAllUsesWith(warpOp.getResult(operandIndex), distConstant);
     rewriter.finalizeOpModification(warpOp);
     return success();
@@ -823,7 +823,7 @@ struct WarpOpTransferRead : public WarpDistributionPattern {
     Value newMask =
         hasMask ? newWarpOp.getResult(newRetIndices[newRetIndices.size() - 1])
                 : Value();
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         read.getLoc(), distributedVal.getType(), read.getBase(), newIndices,
         read.getPermutationMapAttr(), newPadding, newMask,
         read.getInBoundsAttr());
@@ -965,7 +965,7 @@ struct WarpOpBroadcast : public WarpDistributionPattern {
     WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
         rewriter, warpOp, {broadcastSrc}, {broadcastSrcType}, newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value broadcasted = rewriter.create<vector::BroadcastOp>(
+    Value broadcasted = vector::BroadcastOp::create(rewriter,
         loc, destVecType, newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 broadcasted);
@@ -1008,7 +1008,7 @@ struct WarpOpShapeCast : public WarpDistributionPattern {
         rewriter, warpOp, {oldCastOp.getSource()}, {castDistributedType},
         newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value newCast = rewriter.create<vector::ShapeCastOp>(
+    Value newCast = vector::ShapeCastOp::create(rewriter,
         oldCastOp.getLoc(), castResultType,
         newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newCast);
@@ -1091,7 +1091,7 @@ struct WarpOpCreateMask : public WarpDistributionPattern {
     }
 
     auto newMask =
-        rewriter.create<vector::CreateMaskOp>(loc, distType, newOperands);
+        vector::CreateMaskOp::create(rewriter, loc, distType, newOperands);
     rewriter.replaceAllUsesWith(warpOp.getResult(operandIndex), newMask);
     rewriter.finalizeOpModification(warpOp);
     return success();
@@ -1182,7 +1182,7 @@ struct WarpOpInsertStridedSlice : public WarpDistributionPattern {
     Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
     // Create a new insert strided slice op that inserts distributed source into
     // distributed dest.
-    Value newInsert = rewriter.create<vector::InsertStridedSliceOp>(
+    Value newInsert = vector::InsertStridedSliceOp::create(rewriter,
         insertOp.getLoc(), distributedDest.getType(), distributedSource,
         distributedDest, insertOp.getOffsets(), insertOp.getStrides());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newInsert);
@@ -1277,7 +1277,7 @@ struct WarpOpExtractStridedSlice : public WarpDistributionPattern {
     // Create a new extract strided slice op that extracts from the
     // distributed vector.
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
-    Value newExtract = rewriter.create<vector::ExtractStridedSliceOp>(
+    Value newExtract = vector::ExtractStridedSliceOp::create(rewriter,
         extractOp.getLoc(), distributedType, distributedVec,
         extractOp.getOffsets(),
         ArrayAttr::get(rewriter.getContext(), distributedSizes),
@@ -1323,7 +1323,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
       // Extract from distributed vector.
-      Value newExtract = rewriter.create<vector::ExtractOp>(
+      Value newExtract = vector::ExtractOp::create(rewriter,
           loc, distributedVec, extractOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
@@ -1352,7 +1352,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
     // Extract from distributed vector.
-    Value newExtract = rewriter.create<vector::ExtractOp>(
+    Value newExtract = vector::ExtractOp::create(rewriter,
         loc, distributedVec, extractOp.getMixedPosition());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 newExtract);
@@ -1422,7 +1422,7 @@ struct WarpOpExtractScalar : public WarpDistributionPattern {
       Value newExtract;
       SmallVector<int64_t> indices(extractSrcType.getRank(), 0);
       newExtract =
-          rewriter.create<vector::ExtractOp>(loc, distributedVec, indices);
+          vector::ExtractOp::create(rewriter, loc, distributedVec, indices);
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
       return success();
@@ -1442,11 +1442,11 @@ struct WarpOpExtractScalar : public WarpDistributionPattern {
     // Extract at position: pos % elementsPerLane
     Value newPos =
         elementsPerLane == 1
-            ? rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult()
+            ? arith::ConstantIndexOp::create(rewriter, loc, 0).getResult()
             : affine::makeComposedAffineApply(rewriter, loc,
                                               sym0 % elementsPerLane, pos);
     Value extracted =
-        rewriter.create<vector::ExtractOp>(loc, distributedVec, newPos);
+        vector::ExtractOp::create(rewriter, loc, distributedVec, newPos);
 
     // Shuffle the extracted value to all lanes.
     Value shuffled = warpShuffleFromIdxFn(
@@ -1514,7 +1514,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
       if (pos) {
         indices.push_back(pos);
       }
-      newInsert = rewriter.create<vector::InsertOp>(loc, newSource,
+      newInsert = vector::InsertOp::create(rewriter, loc, newSource,
                                                     distributedVec, indices);
       // Broadcast: Simply move the vector.insert op out.
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
@@ -1531,7 +1531,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
     // Insert position: pos % elementsPerLane
     OpFoldResult newPos = affine::makeComposedFoldedAffineApply(
         rewriter, loc, sym0 % elementsPerLane, pos);
-    Value isInsertingLane = rewriter.create<arith::CmpIOp>(
+    Value isInsertingLane = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
     Value newResult =
         rewriter
@@ -1539,13 +1539,13 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
                 loc, isInsertingLane,
                 /*thenBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  Value newInsert = builder.create<vector::InsertOp>(
+                  Value newInsert = vector::InsertOp::create(builder,
                       loc, newSource, distributedVec, newPos);
-                  builder.create<scf::YieldOp>(loc, newInsert);
+                  scf::YieldOp::create(builder, loc, newInsert);
                 },
                 /*elseBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  builder.create<scf::YieldOp>(loc, distributedVec);
+                  scf::YieldOp::create(builder, loc, distributedVec);
                 })
             .getResult(0);
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newResult);
@@ -1582,7 +1582,7 @@ struct WarpOpInsert : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedSrc = newWarpOp->getResult(newRetIndices[0]);
       Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
-      Value newResult = rewriter.create<vector::InsertOp>(
+      Value newResult = vector::InsertOp::create(rewriter,
           loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newResult);
@@ -1632,7 +1632,7 @@ struct WarpOpInsert : public WarpDistributionPattern {
     Value newResult;
     if (distrSrcDim >= 0) {
       // Every lane inserts a small piece.
-      newResult = rewriter.create<vector::InsertOp>(
+      newResult = vector::InsertOp::create(rewriter,
           loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
     } else {
       // One lane inserts the entire source vector.
@@ -1640,19 +1640,19 @@ struct WarpOpInsert : public WarpDistributionPattern {
       SmallVector<OpFoldResult> pos = insertOp.getMixedPosition();
       SmallVector<int64_t> newPos = getAsIntegers(pos);
       // tid of inserting lane: pos / elementsPerLane
-      Value insertingLane = rewriter.create<arith::ConstantIndexOp>(
+      Value insertingLane = arith::ConstantIndexOp::create(rewriter,
           loc, newPos[distrDestDim] / elementsPerLane);
-      Value isInsertingLane = rewriter.create<arith::CmpIOp>(
+      Value isInsertingLane = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
       // Insert position: pos % elementsPerLane
       newPos[distrDestDim] %= elementsPerLane;
       auto insertingBuilder = [&](OpBuilder &builder, Location loc) {
-        Value newInsert = builder.create<vector::InsertOp>(
+        Value newInsert = vector::InsertOp::create(builder,
             loc, distributedSrc, distributedDest, newPos);
-        builder.create<scf::YieldOp>(loc, newInsert);
+        scf::YieldOp::create(builder, loc, newInsert);
       };
       auto nonInsertingBuilder = [&](OpBuilder &builder, Location loc) {
-        builder.create<scf::YieldOp>(loc, distributedDest);
+        scf::YieldOp::create(builder, loc, distributedDest);
       };
       newResult = rewriter
                       .create<scf::IfOp>(loc, isInsertingLane,
@@ -1761,7 +1761,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
 
     // Create a new for op outside the region with a WarpExecuteOnLane0Op
     // region inside.
-    auto newForOp = rewriter.create<scf::ForOp>(
+    auto newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), newOperands);
     rewriter.setInsertionPointToStart(newForOp.getBody());
@@ -1776,7 +1776,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
       argIndexMapping[escapingValues[i]] = warpInputType.size();
       warpInputType.push_back(inputTypes[i]);
     }
-    auto innerWarp = rewriter.create<WarpExecuteOnLane0Op>(
+    auto innerWarp = WarpExecuteOnLane0Op::create(rewriter,
         newWarpOp.getLoc(), newForOp.getResultTypes(), newWarpOp.getLaneid(),
         newWarpOp.getWarpSize(), warpInput, warpInputType);
 
@@ -1792,10 +1792,10 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
     rewriter.eraseOp(forOp.getBody()->getTerminator());
     rewriter.mergeBlocks(forOp.getBody(), innerWarp.getBody(), argMapping);
     rewriter.setInsertionPointToEnd(innerWarp.getBody());
-    rewriter.create<gpu::YieldOp>(innerWarp.getLoc(), yieldOperands);
+    gpu::YieldOp::create(rewriter, innerWarp.getLoc(), yieldOperands);
     rewriter.setInsertionPointAfter(innerWarp);
     if (!innerWarp.getResults().empty())
-      rewriter.create<scf::YieldOp>(forOp.getLoc(), innerWarp.getResults());
+      scf::YieldOp::create(rewriter, forOp.getLoc(), innerWarp.getResults());
     rewriter.eraseOp(forOp);
     // Replace the warpOp result coming from the original ForOp.
     for (const auto &res : llvm::enumerate(resultIdx)) {
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
index 067d4e3491391..e9718a7795fc5 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
@@ -77,7 +77,7 @@ struct CastAwayExtractStridedSliceLeadingOneDim
 
     Location loc = extractOp.getLoc();
 
-    Value newSrcVector = rewriter.create<vector::ExtractOp>(
+    Value newSrcVector = vector::ExtractOp::create(rewriter,
         loc, extractOp.getVector(), splatZero(dropCount));
 
     // The offsets/sizes/strides attribute can have a less number of elements
@@ -89,7 +89,7 @@ struct CastAwayExtractStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         extractOp.getStrides().getValue().drop_front(dropCount));
 
-    auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
         loc, newDstType, newSrcVector, newOffsets, newSizes, newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(extractOp, oldDstType,
@@ -120,9 +120,9 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     // Trim leading one dimensions from both operands.
     Location loc = insertOp.getLoc();
 
-    Value newSrcVector = rewriter.create<vector::ExtractOp>(
+    Value newSrcVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getValueToStore(), splatZero(srcDropCount));
-    Value newDstVector = rewriter.create<vector::ExtractOp>(
+    Value newDstVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getDest(), splatZero(dstDropCount));
 
     auto newOffsets = rewriter.getArrayAttr(
@@ -130,7 +130,7 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         insertOp.getStrides().getValue().take_back(newSrcType.getRank()));
 
-    auto newInsertOp = rewriter.create<vector::InsertStridedSliceOp>(
+    auto newInsertOp = vector::InsertStridedSliceOp::create(rewriter,
         loc, newDstType, newSrcVector, newDstVector, newOffsets, newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
@@ -169,10 +169,10 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
 
     Value newSrcVector = insertOp.getValueToStore();
     if (oldSrcRank != 0) {
-      newSrcVector = rewriter.create<vector::ExtractOp>(
+      newSrcVector = vector::ExtractOp::create(rewriter,
           loc, insertOp.getValueToStore(), splatZero(srcDropCount));
     }
-    Value newDstVector = rewriter.create<vector::ExtractOp>(
+    Value newDstVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getDest(), splatZero(dstDropCount));
 
     // New position rank needs to be computed in two steps: (1) if destination
@@ -187,7 +187,7 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
     newPosition.resize(newDstType.getRank() - newSrcRank,
                        rewriter.getI64IntegerAttr(0));
 
-    auto newInsertOp = rewriter.create<vector::InsertOp>(
+    auto newInsertOp = vector::InsertOp::create(rewriter,
         loc, newSrcVector, newDstVector, newPosition);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
@@ -209,9 +209,9 @@ static Value dropUnitDimsFromMask(OpBuilder &b, Location loc, Value mask,
   if (vector::isBroadcastableTo(newMaskType, oldMaskType) ==
       BroadcastableToResult::Success) {
     int64_t dropDim = oldMaskType.getRank() - newMaskType.getRank();
-    return b.create<vector::ExtractOp>(loc, mask, splatZero(dropDim));
+    return vector::ExtractOp::create(b, loc, mask, splatZero(dropDim));
   }
-  return b.create<vector::ShapeCastOp>(loc, newMaskType, mask);
+  return vector::ShapeCastOp::create(b, loc, newMaskType, mask);
 }
 
 // Turns vector.transfer_read on vector with leading 1 dimensions into
@@ -259,7 +259,7 @@ struct CastAwayTransferReadLeadingOneDim
                                   newType, newMap, maskType);
     }
 
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         read.getLoc(), newType, read.getBase(), read.getIndices(),
         AffineMapAttr::get(newMap), read.getPadding(), mask, inBoundsAttr);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(read, oldType, newRead);
@@ -306,7 +306,7 @@ struct CastAwayTransferWriteLeadingOneDim
       inBoundsAttr = rewriter.getArrayAttr(
           write.getInBoundsAttr().getValue().take_back(newType.getRank()));
 
-    auto newVector = rewriter.create<vector::ExtractOp>(
+    auto newVector = vector::ExtractOp::create(rewriter,
         write.getLoc(), write.getVector(), splatZero(dropDim));
 
     if (write.getMask()) {
@@ -444,20 +444,20 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
     // Extract if its a valid extraction, otherwise use the operand
     // without extraction.
     newOperands.push_back(
-        validExtract ? rewriter.create<vector::ExtractOp>(
+        validExtract ? vector::ExtractOp::create(rewriter,
                            loc, operands[it.index()], splatZero(dropDim))
                      : operands[it.index()]);
   }
 
   // Depending on whether this vector.contract is masked, the replacing Op
   // should either be a new vector.contract Op or vector.mask Op.
-  Operation *newOp = rewriter.create<vector::ContractionOp>(
+  Operation *newOp = vector::ContractionOp::create(rewriter,
       loc, newOperands[0], newOperands[1], newOperands[2],
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind());
 
   if (maskingOp) {
-    auto newMask = rewriter.create<vector::ExtractOp>(loc, maskingOp.getMask(),
+    auto newMask = vector::ExtractOp::create(rewriter, loc, maskingOp.getMask(),
                                                       splatZero(dropDim));
 
     newOp = mlir::vector::maskOperation(rewriter, newOp, newMask);
@@ -519,7 +519,7 @@ class CastAwayElementwiseLeadingOneDim : public RewritePattern {
     SmallVector<Value, 4> newOperands;
     for (Value operand : op->getOperands()) {
       if (auto opVecType = dyn_cast<VectorType>(operand.getType())) {
-        newOperands.push_back(rewriter.create<vector::ExtractOp>(
+        newOperands.push_back(vector::ExtractOp::create(rewriter,
             op->getLoc(), operand, splatZero(dropDim)));
       } else {
         newOperands.push_back(operand);
@@ -559,7 +559,7 @@ struct CastAwayConstantMaskLeadingOneDim
     SmallVector<int64_t> newDimSizes = {flatLeadingSize};
     newDimSizes.append(dimSizes.begin() + dropDim + 1, dimSizes.end());
 
-    auto newMask = rewriter.create<vector::ConstantMaskOp>(
+    auto newMask = vector::ConstantMaskOp::create(rewriter,
         mask.getLoc(), newType, newDimSizes);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(mask, oldType, newMask);
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
index 8cc7008d80b3e..3411e010f3499 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
@@ -65,26 +65,26 @@ struct VectorMaskedLoadOpConverter final
     Value base = maskedLoadOp.getBase();
     Value iValue = maskedLoadOp.getPassThru();
     auto indices = llvm::to_vector_of<Value>(maskedLoadOp.getIndices());
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, indexType, IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
-      auto maskBit = rewriter.create<vector::ExtractOp>(loc, mask, i);
+      auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = rewriter.create<scf::IfOp>(
+      auto ifOp = scf::IfOp::create(rewriter,
           loc, maskBit,
           [&](OpBuilder &builder, Location loc) {
             auto loadedValue =
-                builder.create<memref::LoadOp>(loc, base, indices);
+                memref::LoadOp::create(builder, loc, base, indices);
             auto combinedValue =
-                builder.create<vector::InsertOp>(loc, loadedValue, iValue, i);
-            builder.create<scf::YieldOp>(loc, combinedValue.getResult());
+                vector::InsertOp::create(builder, loc, loadedValue, iValue, i);
+            scf::YieldOp::create(builder, loc, combinedValue.getResult());
           },
           [&](OpBuilder &builder, Location loc) {
-            builder.create<scf::YieldOp>(loc, iValue);
+            scf::YieldOp::create(builder, loc, iValue);
           });
       iValue = ifOp.getResult(0);
 
-      indices.back() = rewriter.create<arith::AddIOp>(loc, indices.back(), one);
+      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.replaceOp(maskedLoadOp, iValue);
@@ -132,18 +132,18 @@ struct VectorMaskedStoreOpConverter final
     Value base = maskedStoreOp.getBase();
     Value value = maskedStoreOp.getValueToStore();
     auto indices = llvm::to_vector_of<Value>(maskedStoreOp.getIndices());
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, indexType, IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
-      auto maskBit = rewriter.create<vector::ExtractOp>(loc, mask, i);
+      auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = rewriter.create<scf::IfOp>(loc, maskBit, /*else=*/false);
+      auto ifOp = scf::IfOp::create(rewriter, loc, maskBit, /*else=*/false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      auto extractedValue = rewriter.create<vector::ExtractOp>(loc, value, i);
-      rewriter.create<memref::StoreOp>(loc, extractedValue, base, indices);
+      auto extractedValue = vector::ExtractOp::create(rewriter, loc, value, i);
+      memref::StoreOp::create(rewriter, loc, extractedValue, base, indices);
 
       rewriter.setInsertionPointAfter(ifOp);
-      indices.back() = rewriter.create<arith::AddIOp>(loc, indices.back(), one);
+      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.eraseOp(maskedStoreOp);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 0fe08417f818f..e915e85f16910 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -132,7 +132,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 SmallVector<Value> newMaskOperands(maskOperands.drop_back());
                 newMaskOperands.push_back(
                     getValueOrCreateConstantIndexOp(rewriter, loc, maskIndex));
-                return rewriter.create<vector::CreateMaskOp>(loc, newMaskType,
+                return vector::CreateMaskOp::create(rewriter, loc, newMaskType,
                                                              newMaskOperands);
               })
           .Case<vector::ConstantMaskOp>(
@@ -143,7 +143,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 int64_t &maskIndex = maskDimSizes.back();
                 maskIndex = llvm::divideCeil(numFrontPadElems + maskIndex,
                                              numSrcElemsPerDest);
-                return rewriter.create<vector::ConstantMaskOp>(loc, newMaskType,
+                return vector::ConstantMaskOp::create(rewriter, loc, newMaskType,
                                                                maskDimSizes);
               })
           .Case<arith::ConstantOp>([&](auto constantOp)
@@ -182,7 +182,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
               }
               compressedMaskValues.push_back(combinedValue);
             }
-            return rewriter.create<arith::ConstantOp>(
+            return arith::ConstantOp::create(rewriter,
                 loc, DenseElementsAttr::get(newMaskType, compressedMaskValues));
           });
 
@@ -190,7 +190,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
     return failure();
 
   while (!extractOps.empty()) {
-    newMask = rewriter.create<vector::ExtractOp>(
+    newMask = vector::ExtractOp::create(rewriter,
         loc, (*newMask)->getResults()[0], extractOps.back().getMixedPosition());
     extractOps.pop_back();
   }
@@ -258,7 +258,7 @@ static Value staticallyInsertSubvector(OpBuilder &rewriter, Location loc,
 
   auto offsets = rewriter.getI64ArrayAttr({offset});
   auto strides = rewriter.getI64ArrayAttr({1});
-  return rewriter.create<vector::InsertStridedSliceOp>(loc, destVecTy, src,
+  return vector::InsertStridedSliceOp::create(rewriter, loc, destVecTy, src,
                                                        dest, offsets, strides);
 }
 
@@ -301,11 +301,11 @@ static Value dynamicallyExtractSubVector(OpBuilder &rewriter, Location loc,
   for (int i = 0; i < numElemsToExtract; ++i) {
     Value extractLoc =
         (i == 0) ? dyn_cast<Value>(offset)
-                 : rewriter.create<arith::AddIOp>(
+                 : arith::AddIOp::create(rewriter,
                        loc, rewriter.getIndexType(), dyn_cast<Value>(offset),
-                       rewriter.create<arith::ConstantIndexOp>(loc, i));
-    auto extractOp = rewriter.create<vector::ExtractOp>(loc, src, extractLoc);
-    dest = rewriter.create<vector::InsertOp>(loc, extractOp, dest, i);
+                       arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto extractOp = vector::ExtractOp::create(rewriter, loc, src, extractLoc);
+    dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, i);
   }
   return dest;
 }
@@ -346,11 +346,11 @@ static Value dynamicallyInsertSubVector(RewriterBase &rewriter, Location loc,
   for (int64_t i = 0; i < numElemsToInsert; ++i) {
     auto insertLoc = i == 0
                          ? destOffsetVal
-                         : rewriter.create<arith::AddIOp>(
+                         : arith::AddIOp::create(rewriter,
                                loc, rewriter.getIndexType(), destOffsetVal,
-                               rewriter.create<arith::ConstantIndexOp>(loc, i));
-    auto extractOp = rewriter.create<vector::ExtractOp>(loc, src, i);
-    dest = rewriter.create<vector::InsertOp>(loc, extractOp, dest, insertLoc);
+                               arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto extractOp = vector::ExtractOp::create(rewriter, loc, src, i);
+    dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, insertLoc);
   }
   return dest;
 }
@@ -369,10 +369,10 @@ static VectorValue emulatedVectorLoad(OpBuilder &rewriter, Location loc,
                                       Type containerElemTy) {
   auto emulatedPerContainerElem = containerElemTy.getIntOrFloatBitWidth() /
                                   emulatedElemTy.getIntOrFloatBitWidth();
-  auto newLoad = rewriter.create<vector::LoadOp>(
+  auto newLoad = vector::LoadOp::create(rewriter,
       loc, VectorType::get(numContainerElemsToLoad, containerElemTy), base,
       getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
-  return rewriter.create<vector::BitCastOp>(
+  return vector::BitCastOp::create(rewriter,
       loc,
       VectorType::get(numContainerElemsToLoad * emulatedPerContainerElem,
                       emulatedElemTy),
@@ -390,16 +390,16 @@ static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
           upcastType.getNumElements() * upcastType.getElementTypeBitWidth() &&
       "expected input and output number of bits to match");
   if (trueValue.getType() != downcastType) {
-    trueValue = builder.create<vector::BitCastOp>(loc, downcastType, trueValue);
+    trueValue = vector::BitCastOp::create(builder, loc, downcastType, trueValue);
   }
   if (falseValue.getType() != downcastType) {
     falseValue =
-        builder.create<vector::BitCastOp>(loc, downcastType, falseValue);
+        vector::BitCastOp::create(builder, loc, downcastType, falseValue);
   }
   Value selectedType =
-      builder.create<arith::SelectOp>(loc, mask, trueValue, falseValue);
+      arith::SelectOp::create(builder, loc, mask, trueValue, falseValue);
   // Upcast the selected value to the new type.
-  return builder.create<vector::BitCastOp>(loc, upcastType, selectedType);
+  return vector::BitCastOp::create(builder, loc, upcastType, selectedType);
 }
 
 /// Emits `memref.generic_atomic_rmw` op to store a subbyte-sized value to a
@@ -422,7 +422,7 @@ static void atomicRMW(OpBuilder &builder, Location loc,
 
   // Create an atomic load-modify-write region using
   // `memref.generic_atomic_rmw`.
-  auto atomicOp = builder.create<memref::GenericAtomicRMWOp>(
+  auto atomicOp = memref::GenericAtomicRMWOp::create(builder,
       loc, linearizedMemref, ValueRange{storeIdx});
   Value origValue = atomicOp.getCurrentValue();
 
@@ -432,7 +432,7 @@ static void atomicRMW(OpBuilder &builder, Location loc,
   // Load the original value from memory, and cast it to the original element
   // type.
   auto oneElemVecType = VectorType::get({1}, origValue.getType());
-  Value origVecValue = builder.create<vector::FromElementsOp>(
+  Value origVecValue = vector::FromElementsOp::create(builder,
       loc, oneElemVecType, ValueRange{origValue});
 
   // Construct the final masked value and yield it.
@@ -440,8 +440,8 @@ static void atomicRMW(OpBuilder &builder, Location loc,
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
   auto scalarMaskedValue =
-      builder.create<vector::ExtractOp>(loc, maskedValue, 0);
-  builder.create<memref::AtomicYieldOp>(loc, scalarMaskedValue);
+      vector::ExtractOp::create(builder, loc, maskedValue, 0);
+  memref::AtomicYieldOp::create(builder, loc, scalarMaskedValue);
 }
 
 /// Generate a non-atomic read-modify-write sequence for storing to the emulated
@@ -453,15 +453,15 @@ static void nonAtomicRMW(OpBuilder &builder, Location loc,
 
   auto oneElemVecType =
       VectorType::get({1}, linearizedMemref.getType().getElementType());
-  Value origVecValue = builder.create<vector::LoadOp>(
+  Value origVecValue = vector::LoadOp::create(builder,
       loc, oneElemVecType, linearizedMemref, ValueRange{linearizedIndex});
-  origVecValue = builder.create<vector::BitCastOp>(loc, valueToStore.getType(),
+  origVecValue = vector::BitCastOp::create(builder, loc, valueToStore.getType(),
                                                    origVecValue);
 
   Value maskedValue =
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
-  builder.create<vector::StoreOp>(loc, maskedValue, linearizedMemref,
+  vector::StoreOp::create(builder, loc, maskedValue, linearizedMemref,
                                   linearizedIndex);
 }
 
@@ -489,7 +489,7 @@ static Value extractSliceIntoByte(ConversionPatternRewriter &rewriter,
   assert(8 % vectorElementType.getIntOrFloatBitWidth() == 0 &&
          "vector element must be a valid sub-byte type");
   auto emulatedPerContainerElem = 8 / vectorElementType.getIntOrFloatBitWidth();
-  auto emptyByteVector = rewriter.create<arith::ConstantOp>(
+  auto emptyByteVector = arith::ConstantOp::create(rewriter,
       loc, VectorType::get({emulatedPerContainerElem}, vectorElementType),
       rewriter.getZeroAttr(
           VectorType::get({emulatedPerContainerElem}, vectorElementType)));
@@ -602,7 +602,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         ShapedType::isDynamic(trailingDim) || trailingDim == origElements;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     // FIXME: ATM, we do not test cases where offsets, sizes, or strides are
     // non-zero. As such, this is not needed.
@@ -664,7 +664,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     if (!emulationRequiresPartialStores) {
       // Basic case: storing full bytes.
       auto numElements = origElements / emulatedPerContainerElem;
-      auto bitCast = rewriter.create<vector::BitCastOp>(
+      auto bitCast = vector::BitCastOp::create(rewriter,
           loc, VectorType::get(numElements, containerElemTy),
           op.getValueToStore());
       rewriter.replaceOpWithNewOp<vector::StoreOp>(
@@ -732,7 +732,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         std::fill_n(frontMaskValues.end() - frontSubWidthStoreElem,
                     *foldedNumFrontPadElems, true);
       }
-      auto frontMask = rewriter.create<arith::ConstantOp>(
+      auto frontMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
 
       currentSourceIndex = emulatedPerContainerElem - (*foldedNumFrontPadElems);
@@ -751,8 +751,8 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
 
     // Increment the destination index by 1 to align to the emulated width
     // boundary.
-    auto constantOne = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    currentDestIndex = rewriter.create<arith::AddIOp>(
+    auto constantOne = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    currentDestIndex = arith::AddIOp::create(rewriter,
         loc, rewriter.getIndexType(), currentDestIndex, constantOne);
 
     // 2. Full width store for the inner output bytes.
@@ -772,15 +772,15 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       auto storeType = VectorType::get(
           {originType.getNumElements() / emulatedPerContainerElem},
           memrefElemType);
-      auto bitCast = rewriter.create<vector::BitCastOp>(loc, storeType,
+      auto bitCast = vector::BitCastOp::create(rewriter, loc, storeType,
                                                         fullWidthStorePart);
-      rewriter.create<vector::StoreOp>(loc, bitCast.getResult(), memrefBase,
+      vector::StoreOp::create(rewriter, loc, bitCast.getResult(), memrefBase,
                                        currentDestIndex);
 
       currentSourceIndex += numNonFullWidthElements;
-      currentDestIndex = rewriter.create<arith::AddIOp>(
+      currentDestIndex = arith::AddIOp::create(rewriter,
           loc, rewriter.getIndexType(), currentDestIndex,
-          rewriter.create<arith::ConstantIndexOp>(loc, fullWidthStoreSize));
+          arith::ConstantIndexOp::create(rewriter, loc, fullWidthStoreSize));
     }
 
     // 3. Partial width store for the trailing output byte.
@@ -795,7 +795,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       // Generate back mask.
       auto maskValues = SmallVector<bool>(emulatedPerContainerElem, 0);
       std::fill_n(maskValues.begin(), remainingElements, 1);
-      auto backMask = rewriter.create<arith::ConstantOp>(
+      auto backMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
 
       storeFunc(rewriter, loc, memrefBase, currentDestIndex,
@@ -848,7 +848,7 @@ struct ConvertVectorMaskedStore final
       return failure();
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
     OpFoldResult linearizedIndicesOfr;
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndicesOfr) =
@@ -901,21 +901,21 @@ struct ConvertVectorMaskedStore final
     auto numElements = (origElements + emulatedPerContainerElem - 1) /
                        emulatedPerContainerElem;
     auto newType = VectorType::get(numElements, containerElemTy);
-    auto passThru = rewriter.create<arith::ConstantOp>(
+    auto passThru = arith::ConstantOp::create(rewriter,
         loc, newType, rewriter.getZeroAttr(newType));
 
-    auto newLoad = rewriter.create<vector::MaskedLoadOp>(
+    auto newLoad = vector::MaskedLoadOp::create(rewriter,
         loc, newType, adaptor.getBase(), linearizedIndices,
         newMask.value()->getResult(0), passThru);
 
     auto newBitCastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
     Value valueToStore =
-        rewriter.create<vector::BitCastOp>(loc, newBitCastType, newLoad);
-    valueToStore = rewriter.create<arith::SelectOp>(
+        vector::BitCastOp::create(rewriter, loc, newBitCastType, newLoad);
+    valueToStore = arith::SelectOp::create(rewriter,
         loc, op.getMask(), op.getValueToStore(), valueToStore);
     valueToStore =
-        rewriter.create<vector::BitCastOp>(loc, newType, valueToStore);
+        vector::BitCastOp::create(rewriter, loc, newType, valueToStore);
 
     rewriter.replaceOpWithNewOp<vector::MaskedStoreOp>(
         op, adaptor.getBase(), linearizedIndices, newMask.value()->getResult(0),
@@ -990,7 +990,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
     bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
@@ -1016,7 +1016,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
                            numElements, emulatedElemTy, containerElemTy);
 
     if (!foldedIntraVectorOffset) {
-      auto resultVector = rewriter.create<arith::ConstantOp>(
+      auto resultVector = arith::ConstantOp::create(rewriter,
           loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(
           rewriter, loc, dyn_cast<TypedValue<VectorType>>(result), resultVector,
@@ -1111,7 +1111,7 @@ struct ConvertVectorMaskedLoad final
     bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndices) =
@@ -1142,7 +1142,7 @@ struct ConvertVectorMaskedLoad final
     auto newBitcastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
 
-    auto emptyVector = rewriter.create<arith::ConstantOp>(
+    auto emptyVector = arith::ConstantOp::create(rewriter,
         loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
     if (!foldedIntraVectorOffset) {
       passthru = dynamicallyInsertSubVector(
@@ -1153,10 +1153,10 @@ struct ConvertVectorMaskedLoad final
                                            *foldedIntraVectorOffset);
     }
     auto newPassThru =
-        rewriter.create<vector::BitCastOp>(loc, loadType, passthru);
+        vector::BitCastOp::create(rewriter, loc, loadType, passthru);
 
     // Generating the new masked load.
-    auto newLoad = rewriter.create<vector::MaskedLoadOp>(
+    auto newLoad = vector::MaskedLoadOp::create(rewriter,
         loc, loadType, adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newMask.value()->getResult(0), newPassThru);
@@ -1164,13 +1164,13 @@ struct ConvertVectorMaskedLoad final
     // Setting the part that originally was not effectively loaded from memory
     // to pass through.
     auto bitCast =
-        rewriter.create<vector::BitCastOp>(loc, newBitcastType, newLoad);
+        vector::BitCastOp::create(rewriter, loc, newBitcastType, newLoad);
 
     Value mask = op.getMask();
     auto newSelectMaskType = VectorType::get(
         numElements * emulatedPerContainerElem, rewriter.getI1Type());
     // TODO: try to fold if op's mask is constant
-    auto emptyMask = rewriter.create<arith::ConstantOp>(
+    auto emptyMask = arith::ConstantOp::create(rewriter,
         loc, newSelectMaskType, rewriter.getZeroAttr(newSelectMaskType));
     if (!foldedIntraVectorOffset) {
       mask = dynamicallyInsertSubVector(rewriter, loc, mask, emptyMask,
@@ -1182,7 +1182,7 @@ struct ConvertVectorMaskedLoad final
     }
 
     Value result =
-        rewriter.create<arith::SelectOp>(loc, mask, bitCast, passthru);
+        arith::SelectOp::create(rewriter, loc, mask, bitCast, passthru);
     if (!foldedIntraVectorOffset) {
       result = dynamicallyExtractSubVector(
           rewriter, loc, result, op.getPassThru(),
@@ -1279,10 +1279,11 @@ struct ConvertVectorTransferRead final
           padding);
     }
     auto newPadding =
-        rewriter.create<arith::ExtUIOp>(loc, containerElemTy, padding);
+        arith::ExtUIOp::create(rewriter, loc, containerElemTy,
+                                                      padding);
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
@@ -1303,19 +1304,19 @@ struct ConvertVectorTransferRead final
     auto numElements = llvm::divideCeil(maxIntraDataOffset + origElements,
                                         emulatedPerContainerElem);
 
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         loc, VectorType::get(numElements, containerElemTy), adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newPadding);
 
-    auto bitCast = rewriter.create<vector::BitCastOp>(
+    auto bitCast = vector::BitCastOp::create(rewriter,
         loc,
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy),
         newRead);
 
     Value result = bitCast->getResult(0);
     if (!foldedIntraVectorOffset) {
-      auto zeros = rewriter.create<arith::ConstantOp>(
+      auto zeros = arith::ConstantOp::create(rewriter,
           loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(rewriter, loc, bitCast, zeros,
                                            linearizedInfo.intraDataOffset,
@@ -1689,32 +1690,32 @@ Value BitCastRewriter::genericRewriteStep(
     PatternRewriter &rewriter, Location loc, Value initialValue,
     Value runningResult, const BitCastRewriter::Metadata &metadata) {
   // Create vector.shuffle from the metadata.
-  auto shuffleOp = rewriter.create<vector::ShuffleOp>(
+  auto shuffleOp = vector::ShuffleOp::create(rewriter,
       loc, initialValue, initialValue, metadata.shuffles);
 
   // Intersect with the mask.
   VectorType shuffledVectorType = shuffleOp.getResultVectorType();
-  auto constOp = rewriter.create<arith::ConstantOp>(
+  auto constOp = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(shuffledVectorType, metadata.masks));
-  Value andValue = rewriter.create<arith::AndIOp>(loc, shuffleOp, constOp);
+  Value andValue = arith::AndIOp::create(rewriter, loc, shuffleOp, constOp);
 
   // Align right on 0.
-  auto shiftRightConstantOp = rewriter.create<arith::ConstantOp>(
+  auto shiftRightConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftRightAmounts));
   Value shiftedRight =
-      rewriter.create<arith::ShRUIOp>(loc, andValue, shiftRightConstantOp);
+      arith::ShRUIOp::create(rewriter, loc, andValue, shiftRightConstantOp);
 
   // Shift bits left into their final position.
-  auto shiftLeftConstantOp = rewriter.create<arith::ConstantOp>(
+  auto shiftLeftConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftLeftAmounts));
   Value shiftedLeft =
-      rewriter.create<arith::ShLIOp>(loc, shiftedRight, shiftLeftConstantOp);
+      arith::ShLIOp::create(rewriter, loc, shiftedRight, shiftLeftConstantOp);
 
   runningResult =
       runningResult
-          ? rewriter.create<arith::OrIOp>(loc, runningResult, shiftedLeft)
+          ? arith::OrIOp::create(rewriter, loc, runningResult, shiftedLeft)
           : shiftedLeft;
 
   return runningResult;
@@ -1737,7 +1738,7 @@ static Value bitcastSubByteVectorToI8(PatternRewriter &rewriter, Location loc,
   // Adjust last dimension of the vector, so the total size remains the same.
   vecShape.back() = vecShape.back() / numSrcElemsPerByte;
   auto i8VecType = VectorType::get(vecShape, rewriter.getI8Type());
-  return rewriter.create<vector::BitCastOp>(loc, i8VecType, subByteVec);
+  return vector::BitCastOp::create(rewriter, loc, i8VecType, subByteVec);
 }
 
 /// Extracts a signed N-bit sequence from each element of a vector of bytes,
@@ -1765,15 +1766,15 @@ static Value extractNBitsPerByteAndSignExtendToI8(PatternRewriter &rewriter,
   assert(bitIdx >= 0 && bitsToShiftLeft >= 0 && numBits > 0 && numBits <= 8 &&
          "Invalid bitIdx range");
   if (bitsToShiftLeft != 0) {
-    Value shiftLeftValues = rewriter.create<arith::ConstantOp>(
+    Value shiftLeftValues = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(srcType, bitsToShiftLeft));
-    shl = rewriter.create<arith::ShLIOp>(loc, src, shiftLeftValues);
+    shl = arith::ShLIOp::create(rewriter, loc, src, shiftLeftValues);
   }
 
   int8_t bitsToShiftRight = 8 - numBits;
-  Value shiftRightValues = rewriter.create<arith::ConstantOp>(
+  Value shiftRightValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
-  Value shr = rewriter.create<arith::ShRSIOp>(loc, shl, shiftRightValues);
+  Value shr = arith::ShRSIOp::create(rewriter, loc, shl, shiftRightValues);
   return shr;
 }
 
@@ -1807,17 +1808,17 @@ static Value extractNBitsPerByteAndExtendToI8(PatternRewriter &rewriter,
   int8_t bitsToShiftRight = bitIdx;
   Value shr = src;
   if (bitsToShiftRight != 0) {
-    Value shiftRightValues = rewriter.create<arith::ConstantOp>(
+    Value shiftRightValues = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
-    shr = rewriter.create<arith::ShRUIOp>(loc, src, shiftRightValues);
+    shr = arith::ShRUIOp::create(rewriter, loc, src, shiftRightValues);
   }
   if (bitIdx + numBits == 8) {
     return shr;
   }
   uint8_t lowBitsMask = (1 << numBits) - 1;
-  Value lowBitsMaskValues = rewriter.create<arith::ConstantOp>(
+  Value lowBitsMaskValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(srcType, lowBitsMask));
-  return rewriter.create<arith::AndIOp>(loc, shr, lowBitsMaskValues);
+  return arith::AndIOp::create(rewriter, loc, shr, lowBitsMaskValues);
 }
 
 using ExtractNBitsFn =
@@ -1840,7 +1841,7 @@ static Value rewriteI4ToI8Ext(PatternRewriter &rewriter, Location loc,
   Value high = extFn(rewriter, loc, i8Vector, 4, 4);
 
   // 3. Interleave low and high i8 elements.
-  return rewriter.create<vector::InterleaveOp>(loc, low, high);
+  return vector::InterleaveOp::create(rewriter, loc, low, high);
 }
 
 /// Rewrite the i2 -> i8  extension into a sequence of shuffles and
@@ -1873,9 +1874,9 @@ static Value rewriteI2ToI8Ext(PatternRewriter &rewriter, Location loc,
   // 02    = [0,2,0,2,0,2,0,2],...
   // 13    = [1,3,1,3,1,3,1,3],...
   // 0213  = [0,1,2,3,...],...
-  Value interleave02 = rewriter.create<vector::InterleaveOp>(loc, vec0, vec2);
-  Value interleave13 = rewriter.create<vector::InterleaveOp>(loc, vec1, vec3);
-  return rewriter.create<vector::InterleaveOp>(loc, interleave02, interleave13);
+  Value interleave02 = vector::InterleaveOp::create(rewriter, loc, vec0, vec2);
+  Value interleave13 = vector::InterleaveOp::create(rewriter, loc, vec1, vec3);
+  return vector::InterleaveOp::create(rewriter, loc, interleave02, interleave13);
 }
 
 /// Rewrite the i8 -> i4 truncation into a deinterleave and series of bitwise
@@ -1887,29 +1888,29 @@ static Value rewriteI8ToI4Trunc(PatternRewriter &rewriter, Location loc,
          "Expected i8 type");
 
   // 1. De-interleave low and high i8 elements.
-  auto deinterleaveOp = rewriter.create<vector::DeinterleaveOp>(loc, srcValue);
+  auto deinterleaveOp = vector::DeinterleaveOp::create(rewriter, loc, srcValue);
 
   // 2. Zero out the upper side of each low i8 element.
   constexpr int8_t i8LowBitMask = 0x0F;
   VectorType deinterI8VecType = deinterleaveOp.getResultVectorType();
-  Value zeroOutMask = rewriter.create<arith::ConstantOp>(
+  Value zeroOutMask = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(deinterI8VecType, i8LowBitMask));
-  Value zeroOutLow = rewriter.create<arith::AndIOp>(
+  Value zeroOutLow = arith::AndIOp::create(rewriter,
       loc, deinterleaveOp.getRes1(), zeroOutMask);
 
   // 3. Move high i4 values to upper side of the byte.
   constexpr int8_t bitsToShift = 4;
-  auto shiftValues = rewriter.create<arith::ConstantOp>(
+  auto shiftValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(deinterI8VecType, bitsToShift));
-  Value shlHigh = rewriter.create<arith::ShLIOp>(loc, deinterleaveOp.getRes2(),
+  Value shlHigh = arith::ShLIOp::create(rewriter, loc, deinterleaveOp.getRes2(),
                                                  shiftValues);
 
   // 4. Merge high and low i4 values.
-  auto mergedHiLowOp = rewriter.create<arith::OrIOp>(loc, zeroOutLow, shlHigh);
+  auto mergedHiLowOp = arith::OrIOp::create(rewriter, loc, zeroOutLow, shlHigh);
 
   // 5. Generate a bitcast vector<Xxi8> -> vector<2Xxi4>.
   auto i4VecType = srcVecType.cloneWith(std::nullopt, rewriter.getI4Type());
-  return rewriter.create<vector::BitCastOp>(loc, i4VecType, mergedHiLowOp);
+  return vector::BitCastOp::create(rewriter, loc, i4VecType, mergedHiLowOp);
 }
 
 namespace {
@@ -2151,7 +2152,7 @@ struct RewriteAlignedSubByteIntTrunc : OpRewritePattern<arith::TruncIOp> {
     Location loc = truncOp.getLoc();
     auto i8VecType = srcVecType.cloneWith(std::nullopt, rewriter.getI8Type());
     Value i8TruncVal =
-        rewriter.create<arith::TruncIOp>(loc, i8VecType, srcValue);
+        arith::TruncIOp::create(rewriter, loc, i8VecType, srcValue);
 
     // Rewrite the i8 -> i4 truncation part.
     Value subByteTrunc = rewriteI8ToI4Trunc(rewriter, loc, i8TruncVal);
@@ -2199,9 +2200,9 @@ struct RewriteVectorTranspose : OpRewritePattern<vector::TransposeOp> {
     // support is available.
     auto srcNativeVecType = srcSubByteVecType.cloneWith(
         std::nullopt, rewriter.getIntegerType(minNativeBitwidth));
-    Value extOp = rewriter.create<arith::ExtSIOp>(loc, srcNativeVecType,
+    Value extOp = arith::ExtSIOp::create(rewriter, loc, srcNativeVecType,
                                                   transposeOp.getVector());
-    Value newTranspose = rewriter.create<vector::TransposeOp>(
+    Value newTranspose = vector::TransposeOp::create(rewriter,
         loc, extOp, transposeOp.getPermutation());
     VectorType dstSubByteVecType = transposeOp.getResultVectorType();
     rewriter.replaceOpWithNewOp<arith::TruncIOp>(transposeOp, dstSubByteVecType,
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
index d834a99076834..baeca29298334 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
@@ -53,14 +53,14 @@ class DecomposeDifferentRankInsertStridedSlice
     int64_t rankRest = dstType.getRank() - rankDiff;
     // Extract / insert the subvector of matching rank and InsertStridedSlice
     // on it.
-    Value extracted = rewriter.create<ExtractOp>(
+    Value extracted = ExtractOp::create(rewriter,
         loc, op.getDest(),
         getI64SubArray(op.getOffsets(), /*dropFront=*/0,
                        /*dropBack=*/rankRest));
 
     // A different pattern will kick in for InsertStridedSlice with matching
     // ranks.
-    auto stridedSliceInnerOp = rewriter.create<InsertStridedSliceOp>(
+    auto stridedSliceInnerOp = InsertStridedSliceOp::create(rewriter,
         loc, op.getValueToStore(), extracted,
         getI64SubArray(op.getOffsets(), /*dropFront=*/rankDiff),
         getI64SubArray(op.getStrides(), /*dropFront=*/0));
@@ -131,7 +131,7 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
       SmallVector<int64_t> offsets(nDest, 0);
       for (int64_t i = 0; i < nSrc; ++i)
         offsets[i] = i;
-      Value scaledSource = rewriter.create<ShuffleOp>(
+      Value scaledSource = ShuffleOp::create(rewriter,
           loc, op.getValueToStore(), op.getValueToStore(), offsets);
 
       // 2. Create a mask where we take the value from scaledSource of dest
@@ -156,21 +156,21 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
          off += stride, ++idx) {
       // 1. extract the proper subvector (or element) from source
       Value extractedSource =
-          rewriter.create<ExtractOp>(loc, op.getValueToStore(), idx);
+          ExtractOp::create(rewriter, loc, op.getValueToStore(), idx);
       if (isa<VectorType>(extractedSource.getType())) {
         // 2. If we have a vector, extract the proper subvector from destination
         // Otherwise we are at the element level and no need to recurse.
         Value extractedDest =
-            rewriter.create<ExtractOp>(loc, op.getDest(), off);
+            ExtractOp::create(rewriter, loc, op.getDest(), off);
         // 3. Reduce the problem to lowering a new InsertStridedSlice op with
         // smaller rank.
-        extractedSource = rewriter.create<InsertStridedSliceOp>(
+        extractedSource = InsertStridedSliceOp::create(rewriter,
             loc, extractedSource, extractedDest,
             getI64SubArray(op.getOffsets(), /* dropFront=*/1),
             getI64SubArray(op.getStrides(), /* dropFront=*/1));
       }
       // 4. Insert the extractedSource into the res vector.
-      res = rewriter.create<InsertOp>(loc, extractedSource, res, off);
+      res = InsertOp::create(rewriter, loc, extractedSource, res, off);
     }
 
     rewriter.replaceOp(op, res);
@@ -250,12 +250,12 @@ class Convert1DExtractStridedSliceIntoExtractInsertChain final
     SmallVector<Value> elements;
     elements.reserve(size);
     for (int64_t i = offset, e = offset + size * stride; i < e; i += stride)
-      elements.push_back(rewriter.create<ExtractOp>(loc, op.getVector(), i));
+      elements.push_back(ExtractOp::create(rewriter, loc, op.getVector(), i));
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(op.getType()));
     for (int64_t i = 0; i < size; ++i)
-      result = rewriter.create<InsertOp>(loc, elements[i], result, i);
+      result = InsertOp::create(rewriter, loc, elements[i], result, i);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -301,17 +301,17 @@ class DecomposeNDExtractStridedSlice
       return failure();
 
     // Extract/insert on a lower ranked extract strided slice op.
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value res = rewriter.create<SplatOp>(loc, dstType, zero);
+    Value res = SplatOp::create(rewriter, loc, dstType, zero);
     for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
          off += stride, ++idx) {
-      Value one = rewriter.create<ExtractOp>(loc, op.getVector(), off);
-      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+      Value one = ExtractOp::create(rewriter, loc, op.getVector(), off);
+      Value extracted = ExtractStridedSliceOp::create(rewriter,
           loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
           getI64SubArray(op.getSizes(), /* dropFront=*/1),
           getI64SubArray(op.getStrides(), /* dropFront=*/1));
-      res = rewriter.create<InsertOp>(loc, extracted, res, idx);
+      res = InsertOp::create(rewriter, loc, extracted, res, idx);
     }
     rewriter.replaceOp(op, res);
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
index 6d3f5697dfaa0..0de9848c52461 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
@@ -658,7 +658,7 @@ struct LinearizeVectorCreateMask final
     // The result of the comparison is then multiplied with
     // the second operand of create_mask to get the 1D mask.
     auto firstOperand = adaptor.getOperands().front();
-    auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto isNonZero = rewriter.createOrFold<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::sgt, firstOperand, zero);
     auto isNonZeroIndex = rewriter.createOrFold<mlir::arith::IndexCastOp>(
@@ -668,7 +668,7 @@ struct LinearizeVectorCreateMask final
         loc, rewriter.getIndexType(), isNonZeroIndex, secondOperand);
 
     auto newMask =
-        rewriter.create<mlir::vector::CreateMaskOp>(loc, dstTy, maskSize);
+        mlir::vector::CreateMaskOp::create(rewriter, loc, dstTy, maskSize);
     rewriter.replaceOp(createMaskOp, newMask);
     return success();
   }
@@ -745,7 +745,7 @@ void mlir::vector::populateForVectorLinearize(TypeConverter &typeConverter,
     if (!isa<VectorType>(type) || !isa<VectorType>(value.getType()))
       return nullptr;
 
-    return builder.create<vector::ShapeCastOp>(loc, type, value);
+    return vector::ShapeCastOp::create(builder, loc, type, value);
   };
   typeConverter.addSourceMaterialization(materializeCast);
   typeConverter.addTargetMaterialization(materializeCast);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
index 363108238e596..70d786a516859 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
@@ -84,7 +84,7 @@ LogicalResult resolveAllTrueCreateMaskOp(IRRewriter &rewriter,
   // Replace createMaskOp with an all-true constant. This should result in the
   // mask being removed in most cases (as xfer ops + vector.mask have folds to
   // remove all-true masks).
-  auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+  auto allTrue = vector::ConstantMaskOp::create(rewriter,
       createMaskOp.getLoc(), maskType, ConstantMaskKind::AllTrue);
   rewriter.replaceAllUsesWith(createMaskOp, allTrue);
   return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index c20a1b355996c..9bbedc396664b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -286,7 +286,7 @@ static Value rankReducingSubviewDroppingUnitDims(PatternRewriter &rewriter,
   if (resultType.canonicalizeStridedLayout() ==
       inputType.canonicalizeStridedLayout())
     return input;
-  return rewriter.create<memref::SubViewOp>(loc, resultType, input, offsets,
+  return memref::SubViewOp::create(rewriter, loc, resultType, input, offsets,
                                             sizes, strides);
 }
 
@@ -395,11 +395,11 @@ class TransferReadDropUnitDimsPattern
 
     Value reducedShapeSource =
         rankReducingSubviewDroppingUnitDims(rewriter, loc, source);
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
-    Operation *newTransferReadOp = rewriter.create<vector::TransferReadOp>(
+    Operation *newTransferReadOp = vector::TransferReadOp::create(rewriter,
         loc, reducedVectorType, reducedShapeSource, zeros, identityMap,
         transferReadOp.getPadding(), maskOp,
         rewriter.getBoolArrayAttr(inBounds));
@@ -477,13 +477,13 @@ class TransferWriteDropUnitDimsPattern
     }
     Value reducedShapeSource =
         rankReducingSubviewDroppingUnitDims(rewriter, loc, source);
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
     auto shapeCastSrc = rewriter.createOrFold<vector::ShapeCastOp>(
         loc, reducedVectorType, vector);
-    Operation *newXferWrite = rewriter.create<vector::TransferWriteOp>(
+    Operation *newXferWrite = vector::TransferWriteOp::create(rewriter,
         loc, Type(), shapeCastSrc, reducedShapeSource, zeros, identityMap,
         maskOp, rewriter.getBoolArrayAttr(inBounds));
 
@@ -520,7 +520,7 @@ static Value collapseInnerDims(PatternRewriter &rewriter, mlir::Location loc,
   for (int64_t i = firstDimToCollapse; i < inputType.getRank(); ++i)
     collapsedIndices.push_back(i);
   reassociation.push_back(collapsedIndices);
-  return rewriter.create<memref::CollapseShapeOp>(loc, input, reassociation);
+  return memref::CollapseShapeOp::create(rewriter, loc, input, reassociation);
 }
 
 /// Returns the new indices that collapses the inner dimensions starting from
@@ -559,7 +559,7 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   // one would get the following offset:
   //    %offset = %arg0 * 43
   OpFoldResult collapsedOffset =
-      rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult();
+      arith::ConstantIndexOp::create(rewriter, loc, 0).getResult();
 
   auto collapsedStrides = computeSuffixProduct(
       ArrayRef<int64_t>(shape.begin() + firstDimToCollapse, shape.end()));
@@ -573,7 +573,7 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   if (auto value = dyn_cast<Value>(collapsedOffset)) {
     indicesAfterCollapsing.push_back(value);
   } else {
-    indicesAfterCollapsing.push_back(rewriter.create<arith::ConstantIndexOp>(
+    indicesAfterCollapsing.push_back(arith::ConstantIndexOp::create(rewriter,
         loc, *getConstantIntValue(collapsedOffset)));
   }
 
@@ -659,7 +659,7 @@ class FlattenContiguousRowMajorTransferReadPattern
     // 3. Create new vector.transfer_read that reads from the collapsed memref
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
-    vector::TransferReadOp flatRead = rewriter.create<vector::TransferReadOp>(
+    vector::TransferReadOp flatRead = vector::TransferReadOp::create(rewriter,
         loc, flatVectorType, collapsedSource, collapsedIndices,
         transferReadOp.getPadding(), collapsedMap);
     flatRead.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
@@ -757,9 +757,9 @@ class FlattenContiguousRowMajorTransferWritePattern
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
     Value flatVector =
-        rewriter.create<vector::ShapeCastOp>(loc, flatVectorType, vector);
+        vector::ShapeCastOp::create(rewriter, loc, flatVectorType, vector);
     vector::TransferWriteOp flatWrite =
-        rewriter.create<vector::TransferWriteOp>(
+        vector::TransferWriteOp::create(rewriter,
             loc, flatVector, collapsedSource, collapsedIndices, collapsedMap);
     flatWrite.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
@@ -846,7 +846,7 @@ class RewriteScalarExtractOfTransferRead
       if (auto value = dyn_cast<Value>(composedIdx)) {
         newIndices[idx] = value;
       } else {
-        newIndices[idx] = rewriter.create<arith::ConstantIndexOp>(
+        newIndices[idx] = arith::ConstantIndexOp::create(rewriter,
             extractOp.getLoc(), *getConstantIntValue(composedIdx));
       }
     }
@@ -884,7 +884,7 @@ class RewriteScalarWrite : public OpRewritePattern<vector::TransferWriteOp> {
       return failure();
     // Only float and integer element types are supported.
     Value scalar =
-        rewriter.create<vector::ExtractOp>(xferOp.getLoc(), xferOp.getVector());
+        vector::ExtractOp::create(rewriter, xferOp.getLoc(), xferOp.getVector());
     // Construct a scalar store.
     if (isa<MemRefType>(xferOp.getBase().getType())) {
       rewriter.replaceOpWithNewOp<memref::StoreOp>(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
index 256c8cb69b1ba..a1f36a9744932 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
@@ -64,12 +64,12 @@ static Value createInBoundsCond(RewriterBase &b,
     if (maybeCstSum && maybeCstDimSz && *maybeCstSum <= *maybeCstDimSz)
       return;
     Value cond =
-        b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sle,
+        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sle,
                                 getValueOrCreateConstantIndexOp(b, loc, sum),
                                 getValueOrCreateConstantIndexOp(b, loc, dimSz));
     // Conjunction over all dims for which we are in-bounds.
     if (inBoundsCond)
-      inBoundsCond = b.create<arith::AndIOp>(loc, inBoundsCond, cond);
+      inBoundsCond = arith::AndIOp::create(b, loc, inBoundsCond, cond);
     else
       inBoundsCond = cond;
   });
@@ -177,11 +177,11 @@ static Value castToCompatibleMemRefType(OpBuilder &b, Value memref,
     sourceType = MemRefType::get(
         sourceType.getShape(), sourceType.getElementType(),
         sourceType.getLayout(), compatibleMemRefType.getMemorySpace());
-    res = b.create<memref::MemorySpaceCastOp>(memref.getLoc(), sourceType, res);
+    res = memref::MemorySpaceCastOp::create(b, memref.getLoc(), sourceType, res);
   }
   if (sourceType == compatibleMemRefType)
     return res;
-  return b.create<memref::CastOp>(memref.getLoc(), compatibleMemRefType, res);
+  return memref::CastOp::create(b, memref.getLoc(), compatibleMemRefType, res);
 }
 
 /// Operates under a scoped context to build the intersection between the
@@ -203,15 +203,15 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
   xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
     using MapList = ArrayRef<ArrayRef<AffineExpr>>;
     Value dimMemRef =
-        b.create<memref::DimOp>(xferOp.getLoc(), xferOp.getBase(), indicesIdx);
-    Value dimAlloc = b.create<memref::DimOp>(loc, alloc, resultIdx);
+        memref::DimOp::create(b, xferOp.getLoc(), xferOp.getBase(), indicesIdx);
+    Value dimAlloc = memref::DimOp::create(b, loc, alloc, resultIdx);
     Value index = xferOp.getIndices()[indicesIdx];
     AffineExpr i, j, k;
     bindDims(xferOp.getContext(), i, j, k);
     SmallVector<AffineMap, 4> maps =
         AffineMap::inferFromExprList(MapList{{i - j, k}}, b.getContext());
     // affine_min(%dimMemRef - %index, %dimAlloc)
-    Value affineMin = b.create<affine::AffineMinOp>(
+    Value affineMin = affine::AffineMinOp::create(b,
         loc, index.getType(), maps[0], ValueRange{dimMemRef, index, dimAlloc});
     sizes.push_back(affineMin);
   });
@@ -220,9 +220,9 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
       xferOp.getIndices(), [](Value idx) -> OpFoldResult { return idx; }));
   SmallVector<OpFoldResult> destIndices(memrefRank, b.getIndexAttr(0));
   SmallVector<OpFoldResult> strides(memrefRank, b.getIndexAttr(1));
-  auto copySrc = b.create<memref::SubViewOp>(
+  auto copySrc = memref::SubViewOp::create(b,
       loc, isaWrite ? alloc : xferOp.getBase(), srcIndices, sizes, strides);
-  auto copyDest = b.create<memref::SubViewOp>(
+  auto copyDest = memref::SubViewOp::create(b,
       loc, isaWrite ? xferOp.getBase() : alloc, destIndices, sizes, strides);
   return std::make_pair(copySrc, copyDest);
 }
@@ -251,18 +251,18 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
                             TypeRange returnTypes, Value inBoundsCond,
                             MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return b.create<scf::IfOp>(
+  return scf::IfOp::create(b,
       loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
         llvm::append_range(viewAndIndices, xferOp.getIndices());
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       },
       [&](OpBuilder &b, Location loc) {
-        b.create<linalg::FillOp>(loc, ValueRange{xferOp.getPadding()},
+        linalg::FillOp::create(b, loc, ValueRange{xferOp.getPadding()},
                                  ValueRange{alloc});
         // Take partial subview of memref which guarantees no dimension
         // overflows.
@@ -270,13 +270,13 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
         std::pair<Value, Value> copyArgs = createSubViewIntersection(
             rewriter, cast<VectorTransferOpInterface>(xferOp.getOperation()),
             alloc);
-        b.create<memref::CopyOp>(loc, copyArgs.first, copyArgs.second);
+        memref::CopyOp::create(b, loc, copyArgs.first, copyArgs.second);
         Value casted =
             castToCompatibleMemRefType(b, alloc, compatibleMemRefType);
         scf::ValueVector viewAndIndices{casted};
         viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
                               zero);
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       });
 }
 
@@ -304,22 +304,22 @@ static scf::IfOp createFullPartialVectorTransferRead(
     Value inBoundsCond, MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
   scf::IfOp fullPartialIfOp;
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return b.create<scf::IfOp>(
+  return scf::IfOp::create(b,
       loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
         llvm::append_range(viewAndIndices, xferOp.getIndices());
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       },
       [&](OpBuilder &b, Location loc) {
         Operation *newXfer = b.clone(*xferOp.getOperation());
         Value vector = cast<VectorTransferOpInterface>(newXfer).getVector();
-        b.create<memref::StoreOp>(
+        memref::StoreOp::create(b,
             loc, vector,
-            b.create<vector::TypeCastOp>(
+            vector::TypeCastOp::create(b,
                 loc, MemRefType::get({}, vector.getType()), alloc));
 
         Value casted =
@@ -327,7 +327,7 @@ static scf::IfOp createFullPartialVectorTransferRead(
         scf::ValueVector viewAndIndices{casted};
         viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
                               zero);
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       });
 }
 
@@ -351,7 +351,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
                           TypeRange returnTypes, Value inBoundsCond,
                           MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
   return b
       .create<scf::IfOp>(
@@ -361,7 +361,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
                 castToCompatibleMemRefType(b, memref, compatibleMemRefType);
             scf::ValueVector viewAndIndices{res};
             llvm::append_range(viewAndIndices, xferOp.getIndices());
-            b.create<scf::YieldOp>(loc, viewAndIndices);
+            scf::YieldOp::create(b, loc, viewAndIndices);
           },
           [&](OpBuilder &b, Location loc) {
             Value casted =
@@ -369,7 +369,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
             scf::ValueVector viewAndIndices{casted};
             viewAndIndices.insert(viewAndIndices.end(),
                                   xferOp.getTransferRank(), zero);
-            b.create<scf::YieldOp>(loc, viewAndIndices);
+            scf::YieldOp::create(b, loc, viewAndIndices);
           })
       ->getResults();
 }
@@ -391,15 +391,15 @@ static void createFullPartialLinalgCopy(RewriterBase &b,
                                         vector::TransferWriteOp xferOp,
                                         Value inBoundsCond, Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = b.create<arith::XOrIOp>(
-      loc, inBoundsCond, b.create<arith::ConstantIntOp>(loc, true, 1));
-  b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
+  auto notInBounds = arith::XOrIOp::create(b,
+      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRRewriter rewriter(b);
     std::pair<Value, Value> copyArgs = createSubViewIntersection(
         rewriter, cast<VectorTransferOpInterface>(xferOp.getOperation()),
         alloc);
-    b.create<memref::CopyOp>(loc, copyArgs.first, copyArgs.second);
-    b.create<scf::YieldOp>(loc, ValueRange{});
+    memref::CopyOp::create(b, loc, copyArgs.first, copyArgs.second);
+    scf::YieldOp::create(b, loc, ValueRange{});
   });
 }
 
@@ -420,18 +420,18 @@ static void createFullPartialVectorTransferWrite(RewriterBase &b,
                                                  Value inBoundsCond,
                                                  Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = b.create<arith::XOrIOp>(
-      loc, inBoundsCond, b.create<arith::ConstantIntOp>(loc, true, 1));
-  b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
+  auto notInBounds = arith::XOrIOp::create(b,
+      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRMapping mapping;
-    Value load = b.create<memref::LoadOp>(
+    Value load = memref::LoadOp::create(b,
         loc,
-        b.create<vector::TypeCastOp>(
+        vector::TypeCastOp::create(b,
             loc, MemRefType::get({}, xferOp.getVector().getType()), alloc),
         ValueRange());
     mapping.map(xferOp.getVector(), load);
     b.clone(*xferOp.getOperation(), mapping);
-    b.create<scf::YieldOp>(loc, ValueRange{});
+    scf::YieldOp::create(b, loc, ValueRange{});
   });
 }
 
@@ -561,7 +561,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
     b.setInsertionPointToStart(&scope->getRegion(0).front());
     auto shape = xferOp.getVectorType().getShape();
     Type elementType = xferOp.getVectorType().getElementType();
-    alloc = b.create<memref::AllocaOp>(scope->getLoc(),
+    alloc = memref::AllocaOp::create(b, scope->getLoc(),
                                        MemRefType::get(shape, elementType),
                                        ValueRange{}, b.getI64IntegerAttr(32));
   }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index fe2707629d82e..d318b4b16b5aa 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -381,7 +381,7 @@ FailureOr<Value> combineContractAndBroadcast(vector::ContractionOp contractOp,
   if (getUnusedDimsBitVector({maps[0], maps[1]}).any())
     return failure();
 
-  Operation *newOp = rewriter.create<vector::ContractionOp>(
+  Operation *newOp = vector::ContractionOp::create(rewriter,
       contractOp.getLoc(), lhs, rhs, contractOp.getAcc(),
       rewriter.getAffineMapArrayAttr(maps), rewriter.getArrayAttr(iterators));
 
@@ -534,7 +534,7 @@ struct ReorderElementwiseOpsOnTranspose final
         // This is a constant. Create a reverse transpose op for it.
         auto vectorType =
             srcType.clone(cast<VectorType>(operand.getType()).getElementType());
-        srcValues.push_back(rewriter.create<vector::TransposeOp>(
+        srcValues.push_back(vector::TransposeOp::create(rewriter,
             operand.getLoc(), vectorType, operand, invOrder));
       }
     }
@@ -608,12 +608,12 @@ struct BubbleDownVectorBitCastForExtract
     // Get the single scalar (as a vector) in the source value that packs the
     // desired scalar. E.g. extract vector<1xf32> from vector<4xf32>
     Location loc = extractOp.getLoc();
-    Value packedValue = rewriter.create<vector::ExtractOp>(
+    Value packedValue = vector::ExtractOp::create(rewriter,
         loc, castOp.getSource(), index / expandRatio);
     Type packedVecType = VectorType::get(/*shape=*/{1}, packedValue.getType());
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, packedVecType, rewriter.getZeroAttr(packedVecType));
-    packedValue = rewriter.create<vector::InsertOp>(loc, packedValue, zero,
+    packedValue = vector::InsertOp::create(rewriter, loc, packedValue, zero,
                                                     /*position=*/0);
 
     // Cast it to a vector with the desired scalar's type.
@@ -621,7 +621,7 @@ struct BubbleDownVectorBitCastForExtract
     VectorType packedType =
         VectorType::get({expandRatio}, castDstType.getElementType());
     Value castedValue =
-        rewriter.create<vector::BitCastOp>(loc, packedType, packedValue);
+        vector::BitCastOp::create(rewriter, loc, packedType, packedValue);
 
     // Finally extract the desired scalar.
     rewriter.replaceOpWithNewOp<vector::ExtractOp>(extractOp, castedValue,
@@ -700,7 +700,7 @@ struct BubbleDownBitCastForStridedSliceExtract
     VectorType newExtractType =
         VectorType::get(dims, castSrcType.getElementType());
 
-    auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
         extractOp.getLoc(), newExtractType, castOp.getSource(), newOffsets,
         newSizes, extractOp.getStrides());
 
@@ -761,7 +761,7 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         isNumElemsShrink ? srcDims.back() / ratio : srcDims.back() * ratio;
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
-    auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
+    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims(insertOp.getDestVectorType().getShape());
@@ -771,7 +771,7 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(dstDims, castDstType.getElementType());
 
     // Bitcast the destination.
-    auto newCastDstOp = rewriter.create<vector::BitCastOp>(
+    auto newCastDstOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     // Generate new insert.
@@ -852,7 +852,7 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
 
-    auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
+    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims =
@@ -861,7 +861,7 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastDstType =
         VectorType::get(dstDims, castDstType.getElementType());
 
-    auto newCastDstOp = rewriter.create<vector::BitCastOp>(
+    auto newCastDstOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     rewriter.replaceOpWithNewOp<vector::InsertStridedSliceOp>(
@@ -936,9 +936,9 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
     Type elemType = castDstType.getElementType();
     assert(elemType.isSignlessIntOrIndexOrFloat());
 
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value res = rewriter.create<SplatOp>(loc, castDstType, zero);
+    Value res = SplatOp::create(rewriter, loc, castDstType, zero);
 
     SmallVector<int64_t> sliceShape = {castDstLastDim};
     SmallVector<int64_t> strides = {1};
@@ -947,12 +947,12 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
                         castDstType.getElementType());
 
     for (int i = 0, e = shrinkRatio; i < e; ++i) {
-      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+      Value extracted = ExtractStridedSliceOp::create(rewriter,
           loc, bitcastOp.getSource(), ArrayRef<int64_t>{i * castDstLastDim},
           sliceShape, strides);
       Value bitcast =
-          rewriter.create<BitCastOp>(loc, newCastDstType, extracted);
-      res = rewriter.create<InsertStridedSliceOp>(
+          BitCastOp::create(rewriter, loc, newCastDstType, extracted);
+      res = InsertStridedSliceOp::create(rewriter,
           loc, bitcast, res,
           ArrayRef<int64_t>{i * castDstLastDim / shrinkRatio}, strides);
     }
@@ -1103,7 +1103,7 @@ class ExtractOpFromElementwise final
     Location loc = eltwise->getLoc();
     SmallVector<OpFoldResult> pos = op.getMixedPosition();
     for (Value arg : eltwise->getOperands()) {
-      Value newArg = rewriter.create<vector::ExtractOp>(loc, arg, pos);
+      Value newArg = vector::ExtractOp::create(rewriter, loc, arg, pos);
       mapping.map(arg, newArg);
     }
 
@@ -1292,18 +1292,18 @@ static Value buildVectorComparison(PatternRewriter &rewriter, Operation *op,
     indicesAttr = rewriter.getI64VectorAttr(
         llvm::to_vector<4>(llvm::seq<int64_t>(0, dim)));
   }
-  Value indices = rewriter.create<arith::ConstantOp>(loc, indicesAttr);
+  Value indices = arith::ConstantOp::create(rewriter, loc, indicesAttr);
   // Add in an offset if requested.
   if (off) {
     Value o = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, *off);
-    Value ov = rewriter.create<vector::SplatOp>(loc, indices.getType(), o);
-    indices = rewriter.create<arith::AddIOp>(loc, ov, indices);
+    Value ov = vector::SplatOp::create(rewriter, loc, indices.getType(), o);
+    indices = arith::AddIOp::create(rewriter, loc, ov, indices);
   }
   // Construct the vector comparison.
   Value bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, b);
   Value bounds =
-      rewriter.create<vector::SplatOp>(loc, indices.getType(), bound);
-  return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, indices,
+      vector::SplatOp::create(rewriter, loc, indices.getType(), bound);
+  return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, indices,
                                         bounds);
 }
 
@@ -1335,15 +1335,15 @@ struct MaterializeTransferMask : public OpRewritePattern<ConcreteOp> {
     Value off = xferOp.getIndices()[lastIndex];
     Value dim =
         vector::createOrFoldDimOp(rewriter, loc, xferOp.getBase(), lastIndex);
-    Value b = rewriter.create<arith::SubIOp>(loc, dim.getType(), dim, off);
-    Value mask = rewriter.create<vector::CreateMaskOp>(
+    Value b = arith::SubIOp::create(rewriter, loc, dim.getType(), dim, off);
+    Value mask = vector::CreateMaskOp::create(rewriter,
         loc,
         VectorType::get(vtp.getShape(), rewriter.getI1Type(),
                         vtp.getScalableDims()),
         b);
     if (xferOp.getMask()) {
       // Intersect the in-bounds with the mask specified as an op parameter.
-      mask = rewriter.create<arith::AndIOp>(loc, mask, xferOp.getMask());
+      mask = arith::AndIOp::create(rewriter, loc, mask, xferOp.getMask());
     }
 
     rewriter.modifyOpInPlace(xferOp, [&]() {
@@ -1548,11 +1548,11 @@ class DropInnerMostUnitDimsTransferRead
         strides);
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         readOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
-    Value rankedReducedView = rewriter.create<memref::SubViewOp>(
+    Value rankedReducedView = memref::SubViewOp::create(rewriter,
         loc, resultMemrefType, readOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
-    Value result = rewriter.create<vector::TransferReadOp>(
+    Value result = vector::TransferReadOp::create(rewriter,
         loc, resultTargetVecType, rankedReducedView,
         readOp.getIndices().drop_back(dimsToDrop), AffineMapAttr::get(permMap),
         readOp.getPadding(),
@@ -1639,7 +1639,7 @@ class DropInnerMostUnitDimsTransferWrite
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         writeOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
 
-    Value rankedReducedView = rewriter.create<memref::SubViewOp>(
+    Value rankedReducedView = memref::SubViewOp::create(rewriter,
         loc, resultMemrefType, writeOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
@@ -1708,21 +1708,21 @@ struct CanonicalizeContractMatmulToMMT final
     auto createTranspose = [&rewriter, loc](Value mat) -> Value {
       if (auto sext = mat.getDefiningOp<arith::ExtSIOp>()) {
         Value trans =
-            rewriter.create<vector::TransposeOp>(loc, sext.getIn(), perm);
+            vector::TransposeOp::create(rewriter, loc, sext.getIn(), perm);
         VectorType newType =
             cast<VectorType>(trans.getType())
                 .clone(cast<VectorType>(mat.getType()).getElementType());
-        return rewriter.create<arith::ExtSIOp>(loc, newType, trans);
+        return arith::ExtSIOp::create(rewriter, loc, newType, trans);
       }
       if (auto zext = mat.getDefiningOp<arith::ExtUIOp>()) {
         Value trans =
-            rewriter.create<vector::TransposeOp>(loc, zext.getIn(), perm);
+            vector::TransposeOp::create(rewriter, loc, zext.getIn(), perm);
         VectorType newType =
             VectorType::get(cast<VectorType>(trans.getType()).getShape(),
                             cast<VectorType>(mat.getType()).getElementType());
-        return rewriter.create<arith::ExtUIOp>(loc, newType, trans);
+        return arith::ExtUIOp::create(rewriter, loc, newType, trans);
       }
-      return rewriter.create<vector::TransposeOp>(loc, mat, perm);
+      return vector::TransposeOp::create(rewriter, loc, mat, perm);
     };
 
     if (maps == infer({{m, k}, {k, n}, {m, n}})) {
@@ -1836,7 +1836,7 @@ struct ChainedReduction final : OpRewritePattern<vector::ReductionOp> {
       vAdd = rewriter.createOrFold<arith::AddIOp>(
           loc, parentReduction.getVector(), op.getVector());
     } else {
-      vAdd = rewriter.create<arith::AddFOp>(loc, parentReduction.getVector(),
+      vAdd = arith::AddFOp::create(rewriter, loc, parentReduction.getVector(),
                                             op.getVector());
     }
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), vAdd,
@@ -1925,7 +1925,7 @@ struct DropUnitDimFromElementwiseOps final
       if (newVType == opVectorType)
         return rewriter.notifyMatchFailure(op, "No unit dimension to remove.");
 
-      auto opSC = rewriter.create<vector::ShapeCastOp>(loc, newVType, operand);
+      auto opSC = vector::ShapeCastOp::create(rewriter, loc, newVType, operand);
       newOperands.push_back(opSC);
     }
 
@@ -2004,11 +2004,11 @@ struct DropUnitDimsFromTransposeOp final
 
     Location loc = op.getLoc();
     // Drop the unit dims via shape_cast.
-    auto dropDimsShapeCast = rewriter.create<vector::ShapeCastOp>(
+    auto dropDimsShapeCast = vector::ShapeCastOp::create(rewriter,
         loc, sourceTypeWithoutUnitDims, op.getVector());
     // Create the new transpose.
     auto transposeWithoutUnitDims =
-        rewriter.create<vector::TransposeOp>(loc, dropDimsShapeCast, newPerm);
+        vector::TransposeOp::create(rewriter, loc, dropDimsShapeCast, newPerm);
     // Restore the unit dims via shape cast.
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
         op, op.getResultVectorType(), transposeWithoutUnitDims);
@@ -2059,7 +2059,7 @@ struct DropUnitDimsFromScfForOp final : OpRewritePattern<scf::ForOp> {
 
       // Create a new ForOp with that iter operand replaced.
       auto castFn = [](OpBuilder &b, Location loc, Type type, Value source) {
-        return b.create<vector::ShapeCastOp>(loc, type, source);
+        return vector::ShapeCastOp::create(b, loc, type, source);
       };
 
       Value replacement =
@@ -2111,7 +2111,7 @@ struct ReduceRedundantZero final : OpRewritePattern<vector::ReductionOp> {
     if (!matchPattern(addLhs.getRhs(), m_AnyZeroFloat()))
       return failure();
 
-    auto newAdd = rewriter.create<arith::AddFOp>(vAdd.getLoc(), addLhs.getLhs(),
+    auto newAdd = arith::AddFOp::create(rewriter, vAdd.getLoc(), addLhs.getLhs(),
                                                  vAdd.getRhs());
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), newAdd,
                                                      op.getAcc());
@@ -2154,7 +2154,7 @@ struct BreakDownVectorReduction final : OpRewritePattern<vector::ReductionOp> {
     Location loc = op.getLoc();
     SmallVector<Value> extracted(numElems, nullptr);
     for (auto [idx, extractedElem] : llvm::enumerate(extracted))
-      extractedElem = rewriter.create<vector::ExtractOp>(
+      extractedElem = vector::ExtractOp::create(rewriter,
           loc, op.getVector(), static_cast<int64_t>(idx));
 
     Value res = extracted.front();
@@ -2234,7 +2234,7 @@ struct FoldArithToVectorOuterProduct : public OpRewritePattern<MulOpType> {
       if (!broadcastedRhs || !isValidBroadcastSource(broadcastedRhs))
         return failure();
 
-      return rewriter.create<vector::OuterProductOp>(
+      return vector::OuterProductOp::create(rewriter,
           mulOp->getLoc(), resType, broadcastedLhs.getSource(),
           broadcastedRhs.getSource(), Value(), vector::CombiningKind::ADD);
     };
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 693f4f955994d..9fa01061c1bed 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -49,7 +49,7 @@ static SmallVector<Value> sliceTransferIndices(ArrayRef<int64_t> elementOffsets,
                 getAffineConstantExpr(elementOffsets[dim.index()], ctx);
     auto map = AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, expr);
     slicedIndices[pos] =
-        builder.create<affine::AffineApplyOp>(loc, map, indices[pos]);
+        affine::AffineApplyOp::create(builder, loc, map, indices[pos]);
   }
   return slicedIndices;
 }
@@ -68,9 +68,9 @@ static SmallVector<Value> sliceLoadStoreIndices(PatternRewriter &rewriter,
   auto start = indices.size() - offsets.size();
   for (auto [i, offset] : llvm::enumerate(offsets)) {
     if (offset != 0) {
-      indices[start + i] = rewriter.create<arith::AddIOp>(
+      indices[start + i] = arith::AddIOp::create(rewriter,
           loc, originalIndices[start + i],
-          rewriter.create<arith::ConstantIndexOp>(loc, offset));
+          arith::ConstantIndexOp::create(rewriter, loc, offset));
     }
   }
   return indices;
@@ -172,7 +172,7 @@ struct UnrollTransferReadPattern
     ArrayRef<int64_t> originalSize = readOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
@@ -185,7 +185,7 @@ struct UnrollTransferReadPattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                readOp.getPermutationMap(), loc, rewriter);
-      auto slicedRead = rewriter.create<vector::TransferReadOp>(
+      auto slicedRead = vector::TransferReadOp::create(rewriter,
           loc, targetType, readOp.getBase(), indices,
           readOp.getPermutationMapAttr(), readOp.getPadding(), readOp.getMask(),
           readOp.getInBoundsAttr());
@@ -236,7 +236,7 @@ struct UnrollTransferWritePattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                writeOp.getPermutationMap(), loc, rewriter);
-      Operation *slicedWrite = rewriter.create<vector::TransferWriteOp>(
+      Operation *slicedWrite = vector::TransferWriteOp::create(rewriter,
           loc, slicedVector, resultTensor ? resultTensor : writeOp.getBase(),
           indices, writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
       // For the tensor case update the destination for the next transfer write.
@@ -348,7 +348,7 @@ struct UnrollContractionPattern
       accCache[dstOffets] = newOp->getResult(0);
     }
     // Assemble back the accumulator into a single vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
@@ -427,7 +427,7 @@ struct UnrollMultiReductionPattern
       accCache[destOffset] = result;
     }
     // Assemble back the accumulator into a single vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, reductionOp.getDestType(),
         rewriter.getZeroAttr(reductionOp.getDestType()));
     for (const auto &it : accCache) {
@@ -468,7 +468,7 @@ struct UnrollElementwisePattern : public RewritePattern {
           op, "expected input vector rank to match target shape rank");
     Location loc = op->getLoc();
     // Prepare the result vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     SmallVector<int64_t> strides(targetShape->size(), 1);
     VectorType newVecType =
@@ -567,7 +567,7 @@ struct UnrollTransposePattern : public OpRewritePattern<vector::TransposeOp> {
     ArrayRef<int64_t> originalSize = originalVectorType.getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, originalVectorType, rewriter.getZeroAttr(originalVectorType));
     ArrayRef<int64_t> permutation = transposeOp.getPermutation();
 
@@ -618,7 +618,7 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
     ArrayRef<int64_t> originalSize = gatherOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
@@ -638,7 +638,7 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
           rewriter.createOrFold<vector::ExtractStridedSliceOp>(
               loc, gatherOp.getPassThru(), elementOffsets, *targetShape,
               strides);
-      auto slicedGather = rewriter.create<vector::GatherOp>(
+      auto slicedGather = vector::GatherOp::create(rewriter,
           loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
           indexSubVec, maskSubVec, passThruSubVec);
 
@@ -671,7 +671,7 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
     ArrayRef<int64_t> originalShape = vecType.getShape();
     SmallVector<int64_t> strides(targetShape->size(), 1);
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, vecType, rewriter.getZeroAttr(vecType));
 
     SmallVector<int64_t> loopOrder =
@@ -684,7 +684,7 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
          StaticTileOffsetRange(originalShape, *targetShape, loopOrder)) {
       SmallVector<Value> indices =
           sliceLoadStoreIndices(rewriter, loc, loadOp.getIndices(), offsets);
-      Value slicedLoad = rewriter.create<vector::LoadOp>(
+      Value slicedLoad = vector::LoadOp::create(rewriter,
           loc, targetVecType, loadOp.getBase(), indices);
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, slicedLoad, result, offsets, strides);
@@ -727,7 +727,7 @@ struct UnrollStorePattern : public OpRewritePattern<vector::StoreOp> {
           sliceLoadStoreIndices(rewriter, loc, storeOp.getIndices(), offsets);
       Value slice = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
           loc, vector, offsets, *targetShape, strides);
-      rewriter.create<vector::StoreOp>(loc, slice, base, indices);
+      vector::StoreOp::create(rewriter, loc, slice, base, indices);
     }
     rewriter.eraseOp(storeOp);
     return success();
@@ -755,7 +755,7 @@ struct UnrollBroadcastPattern : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType = broadcastOp.getResultVectorType();
     VectorType targetType =
         resType.cloneWith(*targetShape, resType.getElementType());
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
 
     SmallVector<int64_t> originalShape = *broadcastOp.getShapeForUnroll();
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 7e4984582b373..7c31ada1644c5 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -331,7 +331,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
   assert(padValue.getType() == sourceShapedType.getElementType() &&
          "expected same pad element type to match source element type");
   int64_t readRank = inputVectorSizes.size();
-  auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+  auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
   SmallVector<bool> inBoundsVal(readRank, true);
 
   if (useInBoundsInsteadOfMasking) {
@@ -341,7 +341,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
       inBoundsVal[i] = (sourceShape[i] == inputVectorSizes[i]) &&
                        ShapedType::isStatic(sourceShape[i]);
   }
-  auto transferReadOp = builder.create<vector::TransferReadOp>(
+  auto transferReadOp = vector::TransferReadOp::create(builder,
       loc,
       /*vectorType=*/vectorType,
       /*source=*/source,
@@ -356,7 +356,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
 
   auto maskType = VectorType::get(inputVectorSizes, builder.getI1Type());
   Value mask =
-      builder.create<vector::CreateMaskOp>(loc, maskType, mixedSourceDims);
+      vector::CreateMaskOp::create(builder, loc, maskType, mixedSourceDims);
   return mlir::vector::maskOperation(builder, transferReadOp, mask)
       ->getResult(0);
 }
diff --git a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
index 179e6ee8784e6..01acade66a3c4 100644
--- a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
+++ b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
@@ -63,11 +63,11 @@ SmallVector<Value> x86vector::MaskCompressOp::getIntrinsicOperands(
   if (adaptor.getSrc()) {
     src = adaptor.getSrc();
   } else if (adaptor.getConstantSrc()) {
-    src = rewriter.create<LLVM::ConstantOp>(loc, opType,
+    src = LLVM::ConstantOp::create(rewriter, loc, opType,
                                             adaptor.getConstantSrcAttr());
   } else {
     auto zeroAttr = rewriter.getZeroAttr(opType);
-    src = rewriter.create<LLVM::ConstantOp>(loc, opType, zeroAttr);
+    src = LLVM::ConstantOp::create(rewriter, loc, opType, zeroAttr);
   }
 
   return SmallVector<Value>{adaptor.getA(), src, adaptor.getK()};
@@ -80,7 +80,7 @@ x86vector::DotOp::getIntrinsicOperands(ArrayRef<Value> operands,
   SmallVector<Value> intrinsicOperands(operands);
   // Dot product of all elements, broadcasted to all elements.
   Value scale =
-      rewriter.create<LLVM::ConstantOp>(getLoc(), rewriter.getI8Type(), 0xff);
+      LLVM::ConstantOp::create(rewriter, getLoc(), rewriter.getI8Type(), 0xff);
   intrinsicOperands.push_back(scale);
 
   return intrinsicOperands;
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
index 63f725084cabf..8f87661a68d96 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
+++ b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
@@ -38,7 +38,7 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
       "=x,x,x"; // Careful: constraint parser is very brittle: no ws!
   SmallVector<Value> asmVals{v1, v2};
   auto asmStr = llvm::formatv(asmTp, llvm::format_hex(mask, /*width=*/2)).str();
-  auto asmOp = b.create<LLVM::InlineAsmOp>(
+  auto asmOp = LLVM::InlineAsmOp::create(b,
       v1.getType(), /*operands=*/asmVals, /*asm_string=*/asmStr,
       /*constraints=*/asmCstr, /*has_side_effects=*/false,
       /*is_align_stack=*/false, LLVM::TailCallKind::None,
@@ -49,13 +49,13 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackLoPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2, ArrayRef<int64_t>{0, 8, 1, 9, 4, 12, 5, 13});
 }
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackHiPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2, ArrayRef<int64_t>{2, 10, 3, 11, 6, 14, 7, 15});
 }
 ///                            a  a   b   b  a  a   b   b
@@ -69,7 +69,7 @@ Value mlir::x86vector::avx2::intrin::mm256ShufflePs(ImplicitLocOpBuilder &b,
   MaskHelper::extractShuffle(mask, b01, b23, b45, b67);
   SmallVector<int64_t> shuffleMask = {
       b01, b23, b45 + 8, b67 + 8, b01 + 4, b23 + 4, b45 + 8 + 4, b67 + 8 + 4};
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 // imm[0:1] out of imm[0:3] is:
@@ -97,7 +97,7 @@ Value mlir::x86vector::avx2::intrin::mm256Permute2f128Ps(
   MaskHelper::extractPermute(mask, b03, b47);
   appendToMask(b03);
   appendToMask(b47);
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// If bit i of `mask` is zero, take f32 at i from v1 else take it from v2.
@@ -109,7 +109,7 @@ Value mlir::x86vector::avx2::intrin::mm256BlendPs(ImplicitLocOpBuilder &b,
     bool isSet = mask & (1 << i);
     shuffleMask.push_back(!isSet ? i : i + 8);
   }
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// AVX2 4x8xf32-specific transpose lowering using a "C intrinsics" model.
@@ -245,13 +245,13 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
           VectorType::get({n * m}, op.getSourceVectorType().getElementType());
       auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
       auto reshInput =
-          ib.create<vector::ShapeCastOp>(flattenedType, op.getVector());
-      reshInput = ib.create<vector::ShapeCastOp>(reshInputType, reshInput);
+          vector::ShapeCastOp::create(ib, flattenedType, op.getVector());
+      reshInput = vector::ShapeCastOp::create(ib, reshInputType, reshInput);
 
       // Extract 1-D vectors from the higher-order dimension of the input
       // vector.
       for (int64_t i = 0; i < m; ++i)
-        vs.push_back(ib.create<vector::ExtractOp>(reshInput, i));
+        vs.push_back(vector::ExtractOp::create(ib, reshInput, i));
 
       // Transpose set of 1-D vectors.
       if (m == 4)
@@ -261,16 +261,16 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
 
       // Insert transposed 1-D vectors into the higher-order dimension of the
       // output vector.
-      Value res = ib.create<arith::ConstantOp>(reshInputType,
+      Value res = arith::ConstantOp::create(ib, reshInputType,
                                                ib.getZeroAttr(reshInputType));
       for (int64_t i = 0; i < m; ++i)
-        res = ib.create<vector::InsertOp>(vs[i], res, i);
+        res = vector::InsertOp::create(ib, vs[i], res, i);
 
       // The output vector still has the shape of the input vector (e.g., 4x8).
       // We have to transpose their dimensions and retrieve its original rank
       // (e.g., 1x8x1x4x1).
-      res = ib.create<vector::ShapeCastOp>(flattenedType, res);
-      res = ib.create<vector::ShapeCastOp>(op.getResultVectorType(), res);
+      res = vector::ShapeCastOp::create(ib, flattenedType, res);
+      res = vector::ShapeCastOp::create(ib, op.getResultVectorType(), res);
       rewriter.replaceOp(op, res);
       return success();
     };
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index ef7cd1424e7a4..f155b2a363834 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -414,7 +414,7 @@ void CreateDescOp::build(OpBuilder &builder, OperationState &state,
   int64_t size = static_cast<int64_t>(offsets.size());
   auto type = VectorType::get(size, builder.getIndexType());
   auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
-  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  auto offset = vector::FromElementsOp::create(builder, loc, type, values);
   build(builder, state, TensorDesc, source, offset);
 }
 
@@ -534,7 +534,7 @@ void UpdateOffsetOp::build(OpBuilder &builder, OperationState &state,
   int64_t size = static_cast<int64_t>(offsets.size());
   auto type = VectorType::get({size}, builder.getIndexType());
   auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
-  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  auto offset = vector::FromElementsOp::create(builder, loc, type, values);
   build(builder, state, tdescTy, tensorDesc, offset);
 }
 
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index c072557c2bd22..613a33fe63d87 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -120,13 +120,13 @@ static Value resolveDistributedTy(Value orig, T expected,
   // If orig is a vector type, create a shape cast op to reconcile the types.
   if (isa<VectorType>(orig.getType())) {
     auto castOp =
-        rewriter.create<vector::ShapeCastOp>(orig.getLoc(), expected, orig);
+        vector::ShapeCastOp::create(rewriter, orig.getLoc(), expected, orig);
     return castOp.getResult();
   }
   // If orig is a tensor descriptor type, create an unrealized conversion cast
   // op to reconcile the types.
   if (isa<xegpu::TensorDescType>(orig.getType())) {
-    auto castOp = rewriter.create<UnrealizedConversionCastOp>(orig.getLoc(),
+    auto castOp = UnrealizedConversionCastOp::create(rewriter, orig.getLoc(),
                                                               expected, orig);
     castOp->setAttr(resolveSIMTTypeMismatch, rewriter.getUnitAttr());
     return castOp.getResult(0);
@@ -198,16 +198,16 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         }))
       return failure();
     // Create a new function with the same signature.
-    auto newGpuFunc = rewriter.create<gpu::GPUFuncOp>(
+    auto newGpuFunc = gpu::GPUFuncOp::create(rewriter,
         gpuFuncOp.getLoc(), gpuFuncOp.getName(), gpuFuncOp.getFunctionType());
     // Create a WarpExecuteOnLane0Op with same arguments and results as the
     // original gpuFuncOp.
     rewriter.setInsertionPointToEnd(&newGpuFunc.getFunctionBody().front());
-    auto laneId = rewriter.create<gpu::LaneIdOp>(
+    auto laneId = gpu::LaneIdOp::create(rewriter,
         newGpuFunc.getLoc(), rewriter.getIndexType(),
         /** upperBound = **/ mlir::IntegerAttr());
     ArrayRef<Type> gpuFuncResultType = gpuFuncOp.getFunctionType().getResults();
-    auto warpOp = rewriter.create<gpu::WarpExecuteOnLane0Op>(
+    auto warpOp = gpu::WarpExecuteOnLane0Op::create(rewriter,
         laneId.getLoc(), gpuFuncResultType, laneId,
         xegpu::targetinfo::subgroupSize, newGpuFunc.getArguments(),
         newGpuFunc.getArgumentTypes());
@@ -216,7 +216,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
     auto origRetunOp =
         cast<gpu::ReturnOp>(gpuFuncOp.getBlocks().back().getTerminator());
     rewriter.setInsertionPointAfter(origRetunOp);
-    rewriter.create<gpu::YieldOp>(origRetunOp.getLoc(),
+    gpu::YieldOp::create(rewriter, origRetunOp.getLoc(),
                                   origRetunOp.getOperands());
     rewriter.eraseOp(origRetunOp);
     // Move the original function body to the WarpExecuteOnLane0Op body.
@@ -225,7 +225,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
     rewriter.eraseBlock(&warpBodyBlock);
     // Insert a new ReturnOp after the WarpExecuteOnLane0Op.
     rewriter.setInsertionPointAfter(warpOp);
-    rewriter.create<gpu::ReturnOp>(newGpuFunc.getLoc(), warpOp.getResults());
+    gpu::ReturnOp::create(rewriter, newGpuFunc.getLoc(), warpOp.getResults());
     rewriter.replaceOp(gpuFuncOp, newGpuFunc);
     return success();
   }
@@ -301,7 +301,7 @@ struct CreateNdDescDistribution final : public gpu::WarpDistributionPattern {
     xegpu::TensorDescType distributedTensorDescTy =
         descOp.getType().dropLayouts(); // Distributed tensor descriptor type
                                         // does not contain layout info.
-    Value newDescOp = rewriter.create<xegpu::CreateNdDescOp>(
+    Value newDescOp = xegpu::CreateNdDescOp::create(rewriter,
         newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
         descOp->getAttrs());
 
@@ -403,7 +403,7 @@ struct StoreNdDistribution final : public gpu::WarpDistributionPattern {
         resolveDistributedTy(newWarpOp.getResult(newRetIndices[1]),
                              distributedTensorDescTy, rewriter));
 
-    rewriter.create<xegpu::StoreNdOp>(
+    xegpu::StoreNdOp::create(rewriter,
         newWarpOp.getLoc(), TypeRange{}, newStoreOperands,
         removeTemporaryLayoutAttributes(storeOp->getAttrs()));
     rewriter.eraseOp(storeOp);
@@ -494,7 +494,7 @@ struct LoadNdDistribution final : public gpu::WarpDistributionPattern {
         loadOp.getTensorDescType().dropLayouts(); // Distributed tensor
                                                   // descriptor type does not
                                                   // contain layout info.
-    auto newLoadOp = rewriter.create<xegpu::LoadNdOp>(
+    auto newLoadOp = xegpu::LoadNdOp::create(rewriter,
         newWarpOp.getLoc(), loadNdDistValueTyOrFailure.value(),
         resolveDistributedTy(newWarpOp->getResult(newRetIndices[0]),
                              distributedTensorDescTy, rewriter),
@@ -630,7 +630,7 @@ struct DpasDistribution final : public gpu::WarpDistributionPattern {
           resolveDistributedTy(newWarpOp.getResult(newRetIndices[i]),
                                newDpasOperandExpectedTypes[i], rewriter));
     }
-    Value newDpasOp = rewriter.create<xegpu::DpasOp>(
+    Value newDpasOp = xegpu::DpasOp::create(rewriter,
         newWarpOp->getLoc(), distributedResultTy, newDpasOperands,
         removeTemporaryLayoutAttributes(dpasOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -717,7 +717,7 @@ struct UpdateNdOffsetDistribution final : public gpu::WarpDistributionPattern {
       }
     }
     // Create a new update op outside the warp op.
-    Value newUpdateOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+    Value newUpdateOp = xegpu::UpdateNdOffsetOp::create(rewriter,
         newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
         removeTemporaryLayoutAttributes(updateOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -783,7 +783,7 @@ struct PrefetchNdDistribution final : public gpu::WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     SmallVector<Value> newPrefetchOperands = {resolveDistributedTy(
         newWarpOp.getResult(newRetIndices[0]), newTensorDescTy, rewriter)};
-    rewriter.create<xegpu::PrefetchNdOp>(
+    xegpu::PrefetchNdOp::create(rewriter,
         newWarpOp.getLoc(), TypeRange{}, newPrefetchOperands,
         removeTemporaryLayoutAttributes(prefetchOp->getAttrs()));
     rewriter.eraseOp(prefetchOp);
@@ -806,7 +806,7 @@ struct GpuBarrierDistribution final : public gpu::WarpDistributionPattern {
       return failure();
     // Move the barrier op outside of the warp op.
     rewriter.setInsertionPointAfter(subgroupOp);
-    rewriter.create<gpu::BarrierOp>(
+    gpu::BarrierOp::create(rewriter,
         barrierOp.getLoc(), barrierOp->getResultTypes(),
         barrierOp->getOperands(), barrierOp->getAttrs());
     rewriter.eraseOp(barrierOp);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
index 13d49cb0e9d82..6266cca918d4f 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
@@ -83,7 +83,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = rewriter.create<UnrealizedConversionCastOp>(
+      auto castOp = UnrealizedConversionCastOp::create(rewriter,
           loc, destTy, srcs, ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResult(0);
     }
@@ -109,7 +109,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = rewriter.create<UnrealizedConversionCastOp>(
+      auto castOp = UnrealizedConversionCastOp::create(rewriter,
           loc, destTypes, src, ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResults();
     }
@@ -144,10 +144,10 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
     auto addi = [&](OpFoldResult a, int64_t b) -> Value {
       std::optional<int64_t> maybeInt = getConstantIntValue(a);
       if (maybeInt) {
-        return rewriter.create<arith::ConstantIndexOp>(loc, *maybeInt + b);
+        return arith::ConstantIndexOp::create(rewriter, loc, *maybeInt + b);
       } else {
         auto aV = llvm::cast<Value>(a);
-        auto bV = rewriter.create<arith::ConstantIndexOp>(loc, b);
+        auto bV = arith::ConstantIndexOp::create(rewriter, loc, b);
         return rewriter.createOrFold<arith::AddIOp>(loc, aV, bV);
       }
     };
@@ -169,7 +169,7 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
            llvm::zip(validIdxes, oldOffsets, offsets))
         mixedOffsets[idx] = addi(oldOff, offset);
 
-      auto newOp = rewriter.create<xegpu::CreateNdDescOp>(
+      auto newOp = xegpu::CreateNdDescOp::create(rewriter,
           loc, newTdescTy, op.getSource(), mixedOffsets, op.getMixedSizes(),
           op.getMixedStrides());
       newOps.push_back(newOp);
@@ -199,7 +199,7 @@ struct UnrollUpdateNdOffsetOp : public UnrollPattern<xegpu::UpdateNdOffsetOp> {
 
     SmallVector<Value> newOps;
     for (auto t : convertedTdesc) {
-      auto newOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+      auto newOp = xegpu::UpdateNdOffsetOp::create(rewriter,
           loc, t.getType(), t, op.getOffsets(), op.getConstOffsets());
       newOps.push_back(newOp);
     }
@@ -226,7 +226,7 @@ struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      rewriter.create<xegpu::PrefetchNdOp>(loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchNdOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -257,7 +257,7 @@ struct UnrollLoadNdOp : public UnrollPattern<xegpu::LoadNdOp> {
     SmallVector<Value> newOps;
     for (auto t : convertedTdescs) {
       auto newOp =
-          rewriter.create<xegpu::LoadNdOp>(loc, newValueTy, t, op->getAttrs());
+          xegpu::LoadNdOp::create(rewriter, loc, newValueTy, t, op->getAttrs());
       newOps.push_back(newOp);
     }
 
@@ -291,7 +291,7 @@ struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto [v, t] : llvm::zip(convertedValues, convertedTdescs))
-      rewriter.create<xegpu::StoreNdOp>(loc, v, t, op.getL1HintAttr(),
+      xegpu::StoreNdOp::create(rewriter, loc, v, t, op.getL1HintAttr(),
                                         op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
@@ -384,7 +384,7 @@ struct UnrollDpasOp : public UnrollPattern<xegpu::DpasOp> {
           if (tmpC)
             operands.push_back(tmpC);
 
-          tmpC = rewriter.create<xegpu::DpasOp>(loc, vecTy, operands,
+          tmpC = xegpu::DpasOp::create(rewriter, loc, vecTy, operands,
                                                 op->getAttrs());
         }
         newOps.push_back(tmpC);
@@ -436,13 +436,13 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
            llvm::zip(convertedIndiceVec, convertedIndiceTypes)) {
         for (int64_t i = 0; i < numNewChunks; ++i) {
           // Compute the offset
-          Value inc = rewriter.create<arith::ConstantIndexOp>(
+          Value inc = arith::ConstantIndexOp::create(rewriter,
               loc, i * blockedChunkSize);
-          Value incVec = rewriter.create<vector::SplatOp>(loc, indiceType, inc);
+          Value incVec = vector::SplatOp::create(rewriter, loc, indiceType, inc);
           Value offsetIndice =
-              rewriter.create<arith::AddIOp>(loc, indice, incVec);
+              arith::AddIOp::create(rewriter, loc, indice, incVec);
 
-          auto newOp = rewriter.create<xegpu::CreateDescOp>(
+          auto newOp = xegpu::CreateDescOp::create(rewriter,
               loc, newTdescTy, op.getSource(), offsetIndice);
 
           newOps.push_back(newOp);
@@ -450,7 +450,7 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
       }
     } else {
       for (auto indice : convertedIndiceVec) {
-        auto newOp = rewriter.create<xegpu::CreateDescOp>(
+        auto newOp = xegpu::CreateDescOp::create(rewriter,
             loc, newTdescTy, op.getSource(), indice);
         newOps.push_back(newOp);
       }
@@ -515,7 +515,7 @@ struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
 
     SmallVector<Value> newOps;
     for (auto [t, m] : llvm::zip(convertedTdescs, convertedMasks)) {
-      auto newOp = rewriter.create<xegpu::LoadGatherOp>(
+      auto newOp = xegpu::LoadGatherOp::create(rewriter,
           loc, newValueTy, t, m, op.getL1HintAttr(), op.getL2HintAttr(),
           op.getL3HintAttr());
       newOps.push_back(newOp);
@@ -547,7 +547,7 @@ struct UnrollPrefetchOp : public UnrollPattern<xegpu::PrefetchOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      rewriter.create<xegpu::PrefetchOp>(loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -608,7 +608,7 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
       Value v = convertedValues[i];
       Value t = convertedTdescs[i];
       Value m = op.getMask() ? convertedMasks[i] : nullptr;
-      rewriter.create<xegpu::StoreScatterOp>(loc, v, t, m, op.getL1HintAttr(),
+      xegpu::StoreScatterOp::create(rewriter, loc, v, t, m, op.getL1HintAttr(),
                                              op.getL2HintAttr(),
                                              op.getL3HintAttr());
     }
@@ -662,7 +662,7 @@ struct UnrollUpdateOffsetOp : public UnrollPattern<xegpu::UpdateOffsetOp> {
 
     for (auto [t, o] : llvm::zip(convertedTdesc, convertedOffsetVec)) {
       auto newOp =
-          rewriter.create<xegpu::UpdateOffsetOp>(loc, t.getType(), t, o);
+          xegpu::UpdateOffsetOp::create(rewriter, loc, t.getType(), t, o);
       newOps.push_back(newOp);
     }
     Value castOp = unpack(newOps, op.getType(), *targetShape, loc, rewriter);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index be7b860dd1729..aa2fc78c77f39 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -121,11 +121,11 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
     for (size_t i = 0; i < rank; ++i) {
       size_t dimIdx = originalOffsets.size() - rank + i;
       Value constOffset =
-          rewriter.create<arith::ConstantIndexOp>(loc, distUnitBaseAddr[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, distUnitBaseAddr[i]);
       Value offset =
           rewriter.createOrFold<index::AddOp>(loc, localOffset[i], constOffset);
       Value modValue =
-          rewriter.create<arith::ConstantIndexOp>(loc, distUnitShape[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, distUnitShape[i]);
       Value offsetMod =
           rewriter.createOrFold<index::RemUOp>(loc, offset, modValue);
       Value origOffset = getValueOrCreateConstantIndexOp(
@@ -162,7 +162,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
     // TODO : Handle order attribute
     // Get the subgroup ID
     auto linearSgId =
-        rewriter.create<gpu::SubgroupIdOp>(loc, /*upper_bound=*/nullptr);
+        gpu::SubgroupIdOp::create(rewriter, loc, /*upper_bound=*/nullptr);
 
     // Create constants for layout dimensions
     SmallVector<Value> sgLayoutDim(sgLayout.size());
@@ -170,8 +170,8 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
 
     for (size_t i = 0; i < sgLayout.size(); i++) {
       sgLayoutDim[i] =
-          rewriter.create<arith::ConstantIndexOp>(loc, sgLayout[i]);
-      sgDataDim[i] = rewriter.create<arith::ConstantIndexOp>(loc, sgShape[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, sgLayout[i]);
+      sgDataDim[i] = arith::ConstantIndexOp::create(rewriter, loc, sgShape[i]);
     }
 
     auto deLinearizeSgId =
@@ -201,7 +201,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
           calculateGlobalOffsets(rewriter, loc, originalOffsets, localOffset,
                                  distUnitBaseAddr, distUnitShape);
 
-      auto newCreateNdOp = rewriter.create<xegpu::CreateNdDescOp>(
+      auto newCreateNdOp = xegpu::CreateNdDescOp::create(rewriter,
           loc, newTdescTy, op.getSource(), globalOffsets, op.getMixedSizes(),
           op.getMixedStrides());
       newCreateNdOps.push_back(newCreateNdOp);
@@ -224,7 +224,7 @@ struct WgToSgLoadNdOp : public OpConversionPattern<xegpu::LoadNdOp> {
           dyn_cast<xegpu::TensorDescType>(src.getType());
       ArrayRef<int64_t> srcShape = tdescTy.getShape();
       VectorType newResTy = VectorType::get(srcShape, tdescTy.getElementType());
-      auto newLoadOp = rewriter.create<xegpu::LoadNdOp>(op.getLoc(), newResTy,
+      auto newLoadOp = xegpu::LoadNdOp::create(rewriter, op.getLoc(), newResTy,
                                                         src, op->getAttrs());
       newLoadOps.push_back(newLoadOp);
     }
@@ -242,7 +242,7 @@ struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
   matchAndRewrite(xegpu::StoreNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     for (auto [v, t] : llvm::zip(adaptor.getValue(), adaptor.getTensorDesc()))
-      rewriter.create<xegpu::StoreNdOp>(op.getLoc(), v, t, op.getL1HintAttr(),
+      xegpu::StoreNdOp::create(rewriter, op.getLoc(), v, t, op.getL1HintAttr(),
                                         op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
@@ -261,7 +261,7 @@ struct WgToSgUpdateNdOffsetOp
                   ConversionPatternRewriter &rewriter) const override {
     llvm::SmallVector<Value> newUpdateTileOffsetOps;
     for (auto tDesc : adaptor.getTensorDesc()) {
-      auto newUpdateTileOffsetOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+      auto newUpdateTileOffsetOp = xegpu::UpdateNdOffsetOp::create(rewriter,
           op.getLoc(), tDesc.getType(), tDesc, op.getOffsets(),
           op.getConstOffsets());
       newUpdateTileOffsetOps.push_back(newUpdateTileOffsetOp);
@@ -305,7 +305,7 @@ struct WgToSgDpasOp : public OpConversionPattern<xegpu::DpasOp> {
             llvm::cast<VectorType>(bVec.getType()).getShape();
         VectorType resTy = VectorType::get({aVecShape[0], bVecShape[1]},
                                            resultTy.getElementType());
-        tmpC = rewriter.create<xegpu::DpasOp>(loc, resTy, operands);
+        tmpC = xegpu::DpasOp::create(rewriter, loc, resTy, operands);
         xegpu::setLayoutAttr(cast<OpResult>(tmpC),
                              originalLayout.dropSgLayoutAndData());
 
@@ -324,7 +324,7 @@ struct WgToSgPrefetchNdOp : public OpConversionPattern<xegpu::PrefetchNdOp> {
   matchAndRewrite(xegpu::PrefetchNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     for (auto src : adaptor.getTensorDesc())
-      rewriter.create<xegpu::PrefetchNdOp>(op.getLoc(), TypeRange(), src,
+      xegpu::PrefetchNdOp::create(rewriter, op.getLoc(), TypeRange(), src,
                                            op->getAttrs());
     rewriter.eraseOp(op);
     return success();
diff --git a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index 6b85a66a8bd36..9acb209d031bd 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -198,7 +198,7 @@ xegpu::extractVectorsWithShapeFromValue(OpBuilder &builder, Location loc,
   SmallVector<Value> result;
   for (SmallVector<int64_t> offsets : StaticTileOffsetRange(srcShape, shape)) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result.push_back(builder.create<vector::ExtractStridedSliceOp>(
+    result.push_back(vector::ExtractStridedSliceOp::create(builder,
         loc, value, offsets, shape, staticStrides));
   }
 
@@ -218,13 +218,13 @@ Value xegpu::createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
 
   VectorType resultTy = VectorType::get(shape, elemTy);
   auto zeroAttr = builder.getZeroAttr(elemTy);
-  Value result = builder.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(builder,
       loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
 
   for (auto [src, offsets] :
        llvm::zip_equal(values, StaticTileOffsetRange(shape, tileShape))) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result = builder.create<vector::InsertStridedSliceOp>(
+    result = vector::InsertStridedSliceOp::create(builder,
         loc, src, result, offsets, staticStrides);
   }
   return result;
@@ -236,7 +236,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
   auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
                             Location loc) -> Value {
-    return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
         .getResult(0);
   };
 
@@ -343,7 +343,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
         if (isa<RankedTensorType>(inputTy) && isa<VectorType>(outputTy)) {
           SmallVector<Value> values = xegpu::flattenValues(adaptor.getInputs());
-          auto newOp = rewriter.create<UnrealizedConversionCastOp>(
+          auto newOp = UnrealizedConversionCastOp::create(rewriter,
               op.getLoc(), outputTy, values);
           rewriter.replaceOp(op, newOp);
           return success();
@@ -355,7 +355,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
     converter.addSourceMaterialization(materializeCast);
     converter.addTargetMaterialization([&](OpBuilder &builder, TypeRange type,
                                            ValueRange inputs, Location loc) {
-      return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+      return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
           .getResults();
     });
 
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 6d7e2aa0ece7d..c88b328282275 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -132,7 +132,7 @@ void ModuleOp::build(OpBuilder &builder, OperationState &state,
 /// Construct a module from the given context.
 ModuleOp ModuleOp::create(Location loc, std::optional<StringRef> name) {
   OpBuilder builder(loc->getContext());
-  return builder.create<ModuleOp>(loc, name);
+  return ModuleOp::create(builder, loc, name);
 }
 
 DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {
diff --git a/mlir/lib/Query/Query.cpp b/mlir/lib/Query/Query.cpp
index 637e1f3cdef87..8ddb35d75734b 100644
--- a/mlir/lib/Query/Query.cpp
+++ b/mlir/lib/Query/Query.cpp
@@ -78,7 +78,7 @@ static Operation *extractFunction(std::vector<Operation *> &ops,
                       clonedOp->result_end());
   }
   // Add return operation
-  builder.create<func::ReturnOp>(loc, clonedVals);
+  func::ReturnOp::create(builder, loc, clonedVals);
 
   // Remove unused function arguments
   size_t currentIndex = 0;
diff --git a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
index 4e7f1d3185129..c0f19f6dce600 100644
--- a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
+++ b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
@@ -37,7 +37,7 @@ LogicalResult mlir::LLVMImportInterface::convertUnregisteredIntrinsic(
     return failure();
 
   Type resultType = moduleImport.convertType(inst->getType());
-  auto op = builder.create<::mlir::LLVM::CallIntrinsicOp>(
+  auto op = ::mlir::LLVM::CallIntrinsicOp::create(builder,
       moduleImport.translateLoc(inst->getDebugLoc()),
       isa<LLVMVoidType>(resultType) ? TypeRange{} : TypeRange{resultType},
       StringAttr::get(builder.getContext(), intrinName),
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index bfda223fe0f5f..6e47db6d1dc6a 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -185,7 +185,7 @@ ComdatOp ModuleImport::getGlobalComdatOp() {
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(mlirModule.getBody());
   globalComdatOp =
-      builder.create<ComdatOp>(mlirModule.getLoc(), getGlobalComdatOpName());
+      ComdatOp::create(builder, mlirModule.getLoc(), getGlobalComdatOpName());
   globalInsertionOp = globalComdatOp;
   return globalComdatOp;
 }
@@ -864,7 +864,7 @@ LogicalResult ModuleImport::convertModuleFlagsMetadata() {
   }
 
   if (!moduleFlags.empty())
-    builder.create<LLVM::ModuleFlagsOp>(mlirModule.getLoc(),
+    LLVM::ModuleFlagsOp::create(builder, mlirModule.getLoc(),
                                         builder.getArrayAttr(moduleFlags));
 
   return success();
@@ -880,7 +880,7 @@ LogicalResult ModuleImport::convertLinkerOptionsMetadata() {
       options.reserve(node->getNumOperands());
       for (const llvm::MDOperand &option : node->operands())
         options.push_back(cast<llvm::MDString>(option)->getString());
-      builder.create<LLVM::LinkerOptionsOp>(mlirModule.getLoc(),
+      LLVM::LinkerOptionsOp::create(builder, mlirModule.getLoc(),
                                             builder.getStrArrayAttr(options));
     }
   }
@@ -984,7 +984,7 @@ void ModuleImport::processComdat(const llvm::Comdat *comdat) {
   ComdatOp comdatOp = getGlobalComdatOp();
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(&comdatOp.getBody().back());
-  auto selectorOp = builder.create<ComdatSelectorOp>(
+  auto selectorOp = ComdatSelectorOp::create(builder,
       mlirModule.getLoc(), comdat->getName(),
       convertComdatFromLLVM(comdat->getSelectionKind()));
   auto symbolRef =
@@ -1346,7 +1346,7 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   Type type = convertType(alias->getValueType());
-  AliasOp aliasOp = builder.create<AliasOp>(
+  AliasOp aliasOp = AliasOp::create(builder,
       mlirModule.getLoc(), type, convertLinkageFromLLVM(alias->getLinkage()),
       alias->getName(),
       /*dso_local=*/alias->isDSOLocal(),
@@ -1360,7 +1360,7 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   FailureOr<Value> initializer = convertConstantExpr(alias->getAliasee());
   if (failed(initializer))
     return failure();
-  builder.create<ReturnOp>(aliasOp.getLoc(), *initializer);
+  ReturnOp::create(builder, aliasOp.getLoc(), *initializer);
 
   if (alias->hasAtLeastLocalUnnamedAddr())
     aliasOp.setUnnamedAddr(convertUnnamedAddrFromLLVM(alias->getUnnamedAddr()));
@@ -1403,7 +1403,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
   if (globalName.empty())
     globalName = getOrCreateNamelessSymbolName(globalVar).getValue();
 
-  GlobalOp globalOp = builder.create<GlobalOp>(
+  GlobalOp globalOp = GlobalOp::create(builder,
       mlirModule.getLoc(), type, globalVar->isConstant(),
       convertLinkageFromLLVM(globalVar->getLinkage()), StringRef(globalName),
       valueAttr, alignment, /*addr_space=*/globalVar->getAddressSpace(),
@@ -1420,7 +1420,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
         convertConstantExpr(globalVar->getInitializer());
     if (failed(initializer))
       return failure();
-    builder.create<ReturnOp>(globalOp.getLoc(), *initializer);
+    ReturnOp::create(builder, globalOp.getLoc(), *initializer);
   }
   if (globalVar->hasAtLeastLocalUnnamedAddr()) {
     globalOp.setUnnamedAddr(
@@ -1488,12 +1488,12 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   if (globalVar->getName() == getGlobalCtorsVarName()) {
-    globalInsertionOp = builder.create<LLVM::GlobalCtorsOp>(
+    globalInsertionOp = LLVM::GlobalCtorsOp::create(builder,
         mlirModule.getLoc(), builder.getArrayAttr(funcs),
         builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
     return success();
   }
-  globalInsertionOp = builder.create<LLVM::GlobalDtorsOp>(
+  globalInsertionOp = LLVM::GlobalDtorsOp::create(builder,
       mlirModule.getLoc(), builder.getArrayAttr(funcs),
       builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
   return success();
@@ -1569,33 +1569,33 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
   if (Attribute attr = getConstantAsAttr(constant)) {
     Type type = convertType(constant->getType());
     if (auto symbolRef = dyn_cast<FlatSymbolRefAttr>(attr)) {
-      return builder.create<AddressOfOp>(loc, type, symbolRef.getValue())
+      return AddressOfOp::create(builder, loc, type, symbolRef.getValue())
           .getResult();
     }
-    return builder.create<ConstantOp>(loc, type, attr).getResult();
+    return ConstantOp::create(builder, loc, type, attr).getResult();
   }
 
   // Convert null pointer constants.
   if (auto *nullPtr = dyn_cast<llvm::ConstantPointerNull>(constant)) {
     Type type = convertType(nullPtr->getType());
-    return builder.create<ZeroOp>(loc, type).getResult();
+    return ZeroOp::create(builder, loc, type).getResult();
   }
 
   // Convert none token constants.
   if (isa<llvm::ConstantTokenNone>(constant)) {
-    return builder.create<NoneTokenOp>(loc).getResult();
+    return NoneTokenOp::create(builder, loc).getResult();
   }
 
   // Convert poison.
   if (auto *poisonVal = dyn_cast<llvm::PoisonValue>(constant)) {
     Type type = convertType(poisonVal->getType());
-    return builder.create<PoisonOp>(loc, type).getResult();
+    return PoisonOp::create(builder, loc, type).getResult();
   }
 
   // Convert undef.
   if (auto *undefVal = dyn_cast<llvm::UndefValue>(constant)) {
     Type type = convertType(undefVal->getType());
-    return builder.create<UndefOp>(loc, type).getResult();
+    return UndefOp::create(builder, loc, type).getResult();
   }
 
   // Convert dso_local_equivalent.
@@ -1621,7 +1621,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
           getOrCreateNamelessSymbolName(cast<llvm::GlobalVariable>(globalObj));
     else
       symbolRef = FlatSymbolRefAttr::get(context, globalName);
-    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+    return AddressOfOp::create(builder, loc, type, symbolRef).getResult();
   }
 
   // Convert global alias accesses.
@@ -1629,7 +1629,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     Type type = convertType(globalAliasObj->getType());
     StringRef aliaseeName = globalAliasObj->getName();
     FlatSymbolRefAttr symbolRef = FlatSymbolRefAttr::get(context, aliaseeName);
-    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+    return AddressOfOp::create(builder, loc, type, symbolRef).getResult();
   }
 
   // Convert constant expressions.
@@ -1680,15 +1680,15 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     bool isArrayOrStruct = isa<LLVMArrayType, LLVMStructType>(rootType);
     assert((isArrayOrStruct || LLVM::isCompatibleVectorType(rootType)) &&
            "unrecognized aggregate type");
-    Value root = builder.create<UndefOp>(loc, rootType);
+    Value root = UndefOp::create(builder, loc, rootType);
     for (const auto &it : llvm::enumerate(elementValues)) {
       if (isArrayOrStruct) {
-        root = builder.create<InsertValueOp>(loc, root, it.value(), it.index());
+        root = InsertValueOp::create(builder, loc, root, it.value(), it.index());
       } else {
         Attribute indexAttr = builder.getI32IntegerAttr(it.index());
         Value indexValue =
-            builder.create<ConstantOp>(loc, builder.getI32Type(), indexAttr);
-        root = builder.create<InsertElementOp>(loc, rootType, root, it.value(),
+            ConstantOp::create(builder, loc, builder.getI32Type(), indexAttr);
+        root = InsertElementOp::create(builder, loc, rootType, root, it.value(),
                                                indexValue);
       }
     }
@@ -1702,7 +1702,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
            "target extension type does not support zero-initialization");
     // Create llvm.mlir.zero operation to represent zero-initialization of
     // target extension type.
-    return builder.create<LLVM::ZeroOp>(loc, targetExtType).getRes();
+    return LLVM::ZeroOp::create(builder, loc, targetExtType).getRes();
   }
 
   if (auto *blockAddr = dyn_cast<llvm::BlockAddress>(constant)) {
@@ -2124,7 +2124,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     if (!brInst->isConditional()) {
-      auto brOp = builder.create<LLVM::BrOp>(loc, succBlockArgs.front(),
+      auto brOp = LLVM::BrOp::create(builder, loc, succBlockArgs.front(),
                                              succBlocks.front());
       mapNoResultOp(inst, brOp);
       return success();
@@ -2132,7 +2132,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     FailureOr<Value> condition = convertValue(brInst->getCondition());
     if (failed(condition))
       return failure();
-    auto condBrOp = builder.create<LLVM::CondBrOp>(
+    auto condBrOp = LLVM::CondBrOp::create(builder,
         loc, *condition, succBlocks.front(), succBlockArgs.front(),
         succBlocks.back(), succBlockArgs.back());
     mapNoResultOp(inst, condBrOp);
@@ -2166,7 +2166,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       caseBlocks[it.index()] = lookupBlock(succBB);
     }
 
-    auto switchOp = builder.create<SwitchOp>(
+    auto switchOp = SwitchOp::create(builder,
         loc, *condition, lookupBlock(defaultBB), defaultBlockArgs, caseValues,
         caseBlocks, caseOperandRefs);
     mapNoResultOp(inst, switchOp);
@@ -2218,14 +2218,14 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
         // IR). Build the indirect call by passing an empty `callee` operand and
         // insert into `operands` to include the indirect call target.
         FlatSymbolRefAttr calleeSym = convertCalleeName(callInst);
-        Value indirectCallVal = builder.create<LLVM::AddressOfOp>(
+        Value indirectCallVal = LLVM::AddressOfOp::create(builder,
             loc, LLVM::LLVMPointerType::get(context), calleeSym);
         operands->insert(operands->begin(), indirectCallVal);
       } else {
         // Regular direct call using callee name.
         callee = convertCalleeName(callInst);
       }
-      CallOp callOp = builder.create<CallOp>(loc, *funcTy, callee, *operands);
+      CallOp callOp = CallOp::create(builder, loc, *funcTy, callee, *operands);
 
       if (failed(convertCallAttributes(callInst, callOp)))
         return failure();
@@ -2260,7 +2260,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
 
     Type type = convertType(lpInst->getType());
     auto lpOp =
-        builder.create<LandingpadOp>(loc, type, lpInst->isCleanup(), operands);
+        LandingpadOp::create(builder, loc, type, lpInst->isCleanup(), operands);
     mapValue(inst, lpOp);
     return success();
   }
@@ -2310,7 +2310,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // IR). Build the indirect invoke by passing an empty `callee` operand and
       // insert into `operands` to include the indirect invoke target.
       FlatSymbolRefAttr calleeSym = convertCalleeName(invokeInst);
-      Value indirectInvokeVal = builder.create<LLVM::AddressOfOp>(
+      Value indirectInvokeVal = LLVM::AddressOfOp::create(builder,
           loc, LLVM::LLVMPointerType::get(context), calleeSym);
       operands->insert(operands->begin(), indirectInvokeVal);
     } else {
@@ -2320,7 +2320,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     // Create the invoke operation. Normal destination block arguments will be
     // added later on to handle the case in which the operation result is
     // included in this list.
-    auto invokeOp = builder.create<InvokeOp>(
+    auto invokeOp = InvokeOp::create(builder,
         loc, *funcTy, calleeName, *operands, directNormalDest, ValueRange(),
         lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
 
@@ -2348,7 +2348,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // arguments (including the invoke operation's result).
       OpBuilder::InsertionGuard g(builder);
       builder.setInsertionPointToStart(directNormalDest);
-      builder.create<LLVM::BrOp>(loc, normalArgs, normalDest);
+      LLVM::BrOp::create(builder, loc, normalArgs, normalDest);
     } else {
       // If the invoke operation's result is not a block argument to the normal
       // destination block, just add the block arguments as usual.
@@ -2382,7 +2382,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     Type type = convertType(inst->getType());
-    auto gepOp = builder.create<GEPOp>(
+    auto gepOp = GEPOp::create(builder,
         loc, type, sourceElementType, *basePtr, indices,
         static_cast<GEPNoWrapFlags>(gepInst->getNoWrapFlags().getRaw()));
     mapValue(inst, gepOp);
@@ -2409,7 +2409,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     SmallVector<ValueRange> succBlockArgsRange =
         llvm::to_vector_of<ValueRange>(succBlockArgs);
     Location loc = translateLoc(inst->getDebugLoc());
-    auto indBrOp = builder.create<LLVM::IndirectBrOp>(
+    auto indBrOp = LLVM::IndirectBrOp::create(builder,
         loc, *basePtr, succBlockArgsRange, succBlocks);
 
     mapNoResultOp(inst, indBrOp);
@@ -2854,7 +2854,7 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) {
   builder.setInsertionPointToEnd(mlirModule.getBody());
 
   Location loc = debugImporter->translateFuncLocation(func);
-  LLVMFuncOp funcOp = builder.create<LLVMFuncOp>(
+  LLVMFuncOp funcOp = LLVMFuncOp::create(builder,
       loc, func->getName(), functionType,
       convertLinkageFromLLVM(func->getLinkage()), dsoLocal, cconv);
 
@@ -3032,11 +3032,11 @@ ModuleImport::processDebugIntrinsic(llvm::DbgVariableIntrinsic *dbgIntr,
   Operation *op =
       llvm::TypeSwitch<llvm::DbgVariableIntrinsic *, Operation *>(dbgIntr)
           .Case([&](llvm::DbgDeclareInst *) {
-            return builder.create<LLVM::DbgDeclareOp>(
+            return LLVM::DbgDeclareOp::create(builder,
                 loc, *argOperand, localVariableAttr, locationExprAttr);
           })
           .Case([&](llvm::DbgValueInst *) {
-            return builder.create<LLVM::DbgValueOp>(
+            return LLVM::DbgValueOp::create(builder,
                 loc, *argOperand, localVariableAttr, locationExprAttr);
           });
   mapNoResultOp(dbgIntr, op);
@@ -3082,7 +3082,7 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb,
   if (bb->hasAddressTaken()) {
     OpBuilder::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(block);
-    builder.create<BlockTagOp>(block->getParentOp()->getLoc(),
+    BlockTagOp::create(builder, block->getParentOp()->getLoc(),
                                BlockTagAttr::get(context, bb->getNumber()));
   }
   return success();
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index 55d6a380d0bff..6e14b94e87407 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -42,22 +42,22 @@ static inline spirv::Opcode extractOpcode(uint32_t word) {
 Value spirv::Deserializer::getValue(uint32_t id) {
   if (auto constInfo = getConstant(id)) {
     // Materialize a `spirv.Constant` op at every use site.
-    return opBuilder.create<spirv::ConstantOp>(unknownLoc, constInfo->second,
+    return spirv::ConstantOp::create(opBuilder, unknownLoc, constInfo->second,
                                                constInfo->first);
   }
   if (auto varOp = getGlobalVariable(id)) {
-    auto addressOfOp = opBuilder.create<spirv::AddressOfOp>(
+    auto addressOfOp = spirv::AddressOfOp::create(opBuilder,
         unknownLoc, varOp.getType(), SymbolRefAttr::get(varOp.getOperation()));
     return addressOfOp.getPointer();
   }
   if (auto constOp = getSpecConstant(id)) {
-    auto referenceOfOp = opBuilder.create<spirv::ReferenceOfOp>(
+    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
         unknownLoc, constOp.getDefaultValue().getType(),
         SymbolRefAttr::get(constOp.getOperation()));
     return referenceOfOp.getReference();
   }
   if (auto constCompositeOp = getSpecConstantComposite(id)) {
-    auto referenceOfOp = opBuilder.create<spirv::ReferenceOfOp>(
+    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
         unknownLoc, constCompositeOp.getType(),
         SymbolRefAttr::get(constCompositeOp.getOperation()));
     return referenceOfOp.getReference();
@@ -69,7 +69,7 @@ Value spirv::Deserializer::getValue(uint32_t id) {
         specConstOperationInfo->enclosedOpOperands);
   }
   if (auto undef = getUndefType(id)) {
-    return opBuilder.create<spirv::UndefOp>(unknownLoc, undef);
+    return spirv::UndefOp::create(opBuilder, unknownLoc, undef);
   }
   return valueMap.lookup(id);
 }
@@ -369,7 +369,7 @@ Deserializer::processOp<spirv::EntryPointOp>(ArrayRef<uint32_t> words) {
     interface.push_back(SymbolRefAttr::get(arg.getOperation()));
     wordIndex++;
   }
-  opBuilder.create<spirv::EntryPointOp>(
+  spirv::EntryPointOp::create(opBuilder,
       unknownLoc, execModel, SymbolRefAttr::get(opBuilder.getContext(), fnName),
       opBuilder.getArrayAttr(interface));
   return success();
@@ -402,7 +402,7 @@ Deserializer::processOp<spirv::ExecutionModeOp>(ArrayRef<uint32_t> words) {
     attrListElems.push_back(opBuilder.getI32IntegerAttr(words[wordIndex++]));
   }
   auto values = opBuilder.getArrayAttr(attrListElems);
-  opBuilder.create<spirv::ExecutionModeOp>(
+  spirv::ExecutionModeOp::create(opBuilder,
       unknownLoc, SymbolRefAttr::get(opBuilder.getContext(), fn.getName()),
       execMode, values);
   return success();
@@ -441,7 +441,7 @@ Deserializer::processOp<spirv::FunctionCallOp>(ArrayRef<uint32_t> operands) {
     arguments.push_back(value);
   }
 
-  auto opFunctionCall = opBuilder.create<spirv::FunctionCallOp>(
+  auto opFunctionCall = spirv::FunctionCallOp::create(opBuilder,
       unknownLoc, resultType,
       SymbolRefAttr::get(opBuilder.getContext(), functionName), arguments);
 
@@ -518,7 +518,7 @@ Deserializer::processOp<spirv::CopyMemoryOp>(ArrayRef<uint32_t> words) {
   }
 
   Location loc = createFileLineColLoc(opBuilder);
-  opBuilder.create<spirv::CopyMemoryOp>(loc, resultTypes, operands, attributes);
+  spirv::CopyMemoryOp::create(opBuilder, loc, resultTypes, operands, attributes);
 
   return success();
 }
@@ -549,7 +549,7 @@ LogicalResult Deserializer::processOp<spirv::GenericCastToPtrExplicitOp>(
   operands.push_back(arg);
 
   Location loc = createFileLineColLoc(opBuilder);
-  Operation *op = opBuilder.create<spirv::GenericCastToPtrExplicitOp>(
+  Operation *op = spirv::GenericCastToPtrExplicitOp::create(opBuilder,
       loc, resultTypes, operands);
   valueMap[valueID] = op->getResult(0);
   return success();
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index b1abd8b3dffe9..7b1f8055fadce 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -518,7 +518,7 @@ spirv::Deserializer::processFunction(ArrayRef<uint32_t> operands) {
   }
 
   std::string fnName = getFunctionSymbol(fnID);
-  auto funcOp = opBuilder.create<spirv::FuncOp>(
+  auto funcOp = spirv::FuncOp::create(opBuilder,
       unknownLoc, fnName, functionType, fnControl.value());
   // Processing other function attributes.
   if (decorations.count(fnID)) {
@@ -706,7 +706,7 @@ spirv::SpecConstantOp
 spirv::Deserializer::createSpecConstant(Location loc, uint32_t resultID,
                                         TypedAttr defaultValue) {
   auto symName = opBuilder.getStringAttr(getSpecConstantSymbol(resultID));
-  auto op = opBuilder.create<spirv::SpecConstantOp>(unknownLoc, symName,
+  auto op = spirv::SpecConstantOp::create(opBuilder, unknownLoc, symName,
                                                     defaultValue);
   if (decorations.count(resultID)) {
     for (auto attr : decorations[resultID].getAttrs())
@@ -782,7 +782,7 @@ spirv::Deserializer::processGlobalVariable(ArrayRef<uint32_t> operands) {
            << wordIndex << " of " << operands.size() << " processed";
   }
   auto loc = createFileLineColLoc(opBuilder);
-  auto varOp = opBuilder.create<spirv::GlobalVariableOp>(
+  auto varOp = spirv::GlobalVariableOp::create(opBuilder,
       loc, TypeAttr::get(type), opBuilder.getStringAttr(variableName),
       initializer);
 
@@ -1581,7 +1581,7 @@ spirv::Deserializer::processSpecConstantComposite(ArrayRef<uint32_t> operands) {
     elements.push_back(SymbolRefAttr::get(elementInfo));
   }
 
-  auto op = opBuilder.create<spirv::SpecConstantCompositeOp>(
+  auto op = spirv::SpecConstantCompositeOp::create(opBuilder,
       unknownLoc, TypeAttr::get(resultType), symName,
       opBuilder.getArrayAttr(elements));
   specConstCompositeMap[resultID] = op;
@@ -1656,7 +1656,7 @@ Value spirv::Deserializer::materializeSpecConstantOperation(
 
   auto loc = createFileLineColLoc(opBuilder);
   auto specConstOperationOp =
-      opBuilder.create<spirv::SpecConstantOperationOp>(loc, resultType);
+      spirv::SpecConstantOperationOp::create(opBuilder, loc, resultType);
 
   Region &body = specConstOperationOp.getBody();
   // Move the new block into SpecConstantOperation's body.
@@ -1669,7 +1669,7 @@ Value spirv::Deserializer::materializeSpecConstantOperation(
   OpBuilder::InsertionGuard moduleInsertionGuard(opBuilder);
   opBuilder.setInsertionPointToEnd(&block);
 
-  opBuilder.create<spirv::YieldOp>(loc, block.front().getResult(0));
+  spirv::YieldOp::create(opBuilder, loc, block.front().getResult(0));
   return specConstOperationOp.getResult();
 }
 
@@ -1733,7 +1733,7 @@ LogicalResult spirv::Deserializer::processBranch(ArrayRef<uint32_t> operands) {
   // The preceding instruction for the OpBranch instruction could be an
   // OpLoopMerge or an OpSelectionMerge instruction, in this case they will have
   // the same OpLine information.
-  opBuilder.create<spirv::BranchOp>(loc, target);
+  spirv::BranchOp::create(opBuilder, loc, target);
 
   clearDebugLine();
   return success();
@@ -1764,7 +1764,7 @@ spirv::Deserializer::processBranchConditional(ArrayRef<uint32_t> operands) {
   // an OpSelectionMerge instruction, in this case they will have the same
   // OpLine information.
   auto loc = createFileLineColLoc(opBuilder);
-  opBuilder.create<spirv::BranchConditionalOp>(
+  spirv::BranchConditionalOp::create(opBuilder,
       loc, condition, trueBlock,
       /*trueArguments=*/ArrayRef<Value>(), falseBlock,
       /*falseArguments=*/ArrayRef<Value>(), weights);
@@ -1947,7 +1947,7 @@ ControlFlowStructurizer::createSelectionOp(uint32_t selectionControl) {
   OpBuilder builder(&mergeBlock->front());
 
   auto control = static_cast<spirv::SelectionControl>(selectionControl);
-  auto selectionOp = builder.create<spirv::SelectionOp>(location, control);
+  auto selectionOp = spirv::SelectionOp::create(builder, location, control);
   selectionOp.addMergeBlock(builder);
 
   return selectionOp;
@@ -1959,7 +1959,7 @@ spirv::LoopOp ControlFlowStructurizer::createLoopOp(uint32_t loopControl) {
   OpBuilder builder(&mergeBlock->front());
 
   auto control = static_cast<spirv::LoopControl>(loopControl);
-  auto loopOp = builder.create<spirv::LoopOp>(location, control);
+  auto loopOp = spirv::LoopOp::create(builder, location, control);
   loopOp.addEntryAndMergeBlock(builder);
 
   return loopOp;
@@ -2092,7 +2092,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
     // The loop entry block should have a unconditional branch jumping to the
     // loop header block.
     builder.setInsertionPointToEnd(&body.front());
-    builder.create<spirv::BranchOp>(location, mapper.lookupOrNull(headerBlock),
+    spirv::BranchOp::create(builder, location, mapper.lookupOrNull(headerBlock),
                                     ArrayRef<Value>(blockArgs));
   }
 
@@ -2177,11 +2177,11 @@ LogicalResult ControlFlowStructurizer::structurize() {
     Operation *newOp = nullptr;
 
     if (isLoop)
-      newOp = builder.create<spirv::LoopOp>(
+      newOp = spirv::LoopOp::create(builder,
           location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::LoopControl>(control));
     else
-      newOp = builder.create<spirv::SelectionOp>(
+      newOp = spirv::SelectionOp::create(builder,
           location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::SelectionControl>(control));
 
@@ -2308,7 +2308,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
       // but replace all ops inside with a branch to the merge block.
       block->clear();
       builder.setInsertionPointToEnd(block);
-      builder.create<spirv::BranchOp>(location, mergeBlock);
+      spirv::BranchOp::create(builder, location, mergeBlock);
     } else {
       LLVM_DEBUG(logger.startLine() << "[cf] erasing block " << block << "\n");
       block->erase();
@@ -2362,7 +2362,7 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
 
     if (auto branchOp = dyn_cast<spirv::BranchOp>(op)) {
       // Replace the previous branch op with a new one with block arguments.
-      opBuilder.create<spirv::BranchOp>(branchOp.getLoc(), branchOp.getTarget(),
+      spirv::BranchOp::create(opBuilder, branchOp.getLoc(), branchOp.getTarget(),
                                         blockArgs);
       branchOp.erase();
     } else if (auto branchCondOp = dyn_cast<spirv::BranchConditionalOp>(op)) {
@@ -2370,13 +2370,13 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
               branchCondOp.getFalseBlock() == target) &&
              "expected target to be either the true or false target");
       if (target == branchCondOp.getTrueTarget())
-        opBuilder.create<spirv::BranchConditionalOp>(
+        spirv::BranchConditionalOp::create(opBuilder,
             branchCondOp.getLoc(), branchCondOp.getCondition(), blockArgs,
             branchCondOp.getFalseBlockArguments(),
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueTarget(),
             branchCondOp.getFalseTarget());
       else
-        opBuilder.create<spirv::BranchConditionalOp>(
+        spirv::BranchConditionalOp::create(opBuilder,
             branchCondOp.getLoc(), branchCondOp.getCondition(),
             branchCondOp.getTrueBlockArguments(), blockArgs,
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueBlock(),
@@ -2437,7 +2437,7 @@ LogicalResult spirv::Deserializer::splitConditionalBlocks() {
     if (!llvm::hasSingleElement(*block) || splitHeaderMergeBlock) {
       Block *newBlock = block->splitBlock(terminator);
       OpBuilder builder(block, block->end());
-      builder.create<spirv::BranchOp>(block->getParent()->getLoc(), newBlock);
+      spirv::BranchOp::create(builder, block->getParent()->getLoc(), newBlock);
 
       // After splitting we need to update the map to use the new block as a
       // header.
diff --git a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
index 824201d17b5ab..3f4e7762fed0e 100644
--- a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
+++ b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
@@ -134,7 +134,7 @@ class CodeGen {
 
 OwningOpRef<ModuleOp> CodeGen::generate(const ast::Module &module) {
   OwningOpRef<ModuleOp> mlirModule =
-      builder.create<ModuleOp>(genLoc(module.getLoc()));
+      ModuleOp::create(builder, genLoc(module.getLoc()));
   builder.setInsertionPointToStart(mlirModule->getBody());
 
   // Generate code for each of the decls within the module.
@@ -205,7 +205,7 @@ static void checkAndNestUnderRewriteOp(OpBuilder &builder, Value rootExpr,
                                        Location loc) {
   if (isa<pdl::PatternOp>(builder.getInsertionBlock()->getParentOp())) {
     pdl::RewriteOp rewrite =
-        builder.create<pdl::RewriteOp>(loc, rootExpr, /*name=*/StringAttr(),
+        pdl::RewriteOp::create(builder, loc, rootExpr, /*name=*/StringAttr(),
                                        /*externalArgs=*/ValueRange());
     builder.createBlock(&rewrite.getBodyRegion());
   }
@@ -219,7 +219,7 @@ void CodeGen::genImpl(const ast::EraseStmt *stmt) {
   // Make sure we are nested in a RewriteOp.
   OpBuilder::InsertionGuard guard(builder);
   checkAndNestUnderRewriteOp(builder, rootExpr, loc);
-  builder.create<pdl::EraseOp>(loc, rootExpr);
+  pdl::EraseOp::create(builder, loc, rootExpr);
 }
 
 void CodeGen::genImpl(const ast::LetStmt *stmt) { genVar(stmt->getVarDecl()); }
@@ -242,7 +242,7 @@ void CodeGen::genImpl(const ast::ReplaceStmt *stmt) {
   bool usesReplOperation =
       replValues.size() == 1 &&
       isa<pdl::OperationType>(replValues.front().getType());
-  builder.create<pdl::ReplaceOp>(
+  pdl::ReplaceOp::create(builder,
       loc, rootExpr, usesReplOperation ? replValues[0] : Value(),
       usesReplOperation ? ValueRange() : ValueRange(replValues));
 }
@@ -283,7 +283,7 @@ void CodeGen::genImpl(const ast::PatternDecl *decl) {
 
   // FIXME: Properly model HasBoundedRecursion in PDL so that we don't drop it
   // here.
-  pdl::PatternOp pattern = builder.create<pdl::PatternOp>(
+  pdl::PatternOp pattern = pdl::PatternOp::create(builder,
       genLoc(decl->getLoc()), decl->getBenefit(),
       name ? std::optional<StringRef>(name->getName())
            : std::optional<StringRef>());
@@ -338,19 +338,19 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   ast::Type type = varDecl->getType();
   Type mlirType = genType(type);
   if (isa<ast::ValueType>(type))
-    return builder.create<pdl::OperandOp>(loc, mlirType, getTypeConstraint());
+    return pdl::OperandOp::create(builder, loc, mlirType, getTypeConstraint());
   if (isa<ast::TypeType>(type))
-    return builder.create<pdl::TypeOp>(loc, mlirType, /*type=*/TypeAttr());
+    return pdl::TypeOp::create(builder, loc, mlirType, /*type=*/TypeAttr());
   if (isa<ast::AttributeType>(type))
-    return builder.create<pdl::AttributeOp>(loc, getTypeConstraint());
+    return pdl::AttributeOp::create(builder, loc, getTypeConstraint());
   if (ast::OperationType opType = dyn_cast<ast::OperationType>(type)) {
-    Value operands = builder.create<pdl::OperandsOp>(
+    Value operands = pdl::OperandsOp::create(builder,
         loc, pdl::RangeType::get(builder.getType<pdl::ValueType>()),
         /*type=*/Value());
-    Value results = builder.create<pdl::TypesOp>(
+    Value results = pdl::TypesOp::create(builder,
         loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
         /*types=*/ArrayAttr());
-    return builder.create<pdl::OperationOp>(loc, opType.getName(), operands,
+    return pdl::OperationOp::create(builder, loc, opType.getName(), operands,
                                             ArrayRef<StringRef>(), ValueRange(),
                                             results);
   }
@@ -358,10 +358,10 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   if (ast::RangeType rangeTy = dyn_cast<ast::RangeType>(type)) {
     ast::Type eleTy = rangeTy.getElementType();
     if (isa<ast::ValueType>(eleTy))
-      return builder.create<pdl::OperandsOp>(loc, mlirType,
+      return pdl::OperandsOp::create(builder, loc, mlirType,
                                              getTypeConstraint());
     if (isa<ast::TypeType>(eleTy))
-      return builder.create<pdl::TypesOp>(loc, mlirType, /*types=*/ArrayAttr());
+      return pdl::TypesOp::create(builder, loc, mlirType, /*types=*/ArrayAttr());
   }
 
   llvm_unreachable("invalid non-initialized variable type");
@@ -404,7 +404,7 @@ SmallVector<Value> CodeGen::genExpr(const ast::Expr *expr) {
 Value CodeGen::genExprImpl(const ast::AttributeExpr *expr) {
   Attribute attr = parseAttribute(expr->getValue(), builder.getContext());
   assert(attr && "invalid MLIR attribute data");
-  return builder.create<pdl::AttributeOp>(genLoc(expr->getLoc()), attr);
+  return pdl::AttributeOp::create(builder, genLoc(expr->getLoc()), attr);
 }
 
 SmallVector<Value> CodeGen::genExprImpl(const ast::CallExpr *expr) {
@@ -443,9 +443,9 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
     if (isa<ast::AllResultsMemberAccessExpr>(expr)) {
       Type mlirType = genType(expr->getType());
       if (isa<pdl::ValueType>(mlirType))
-        return builder.create<pdl::ResultOp>(loc, mlirType, parentExprs[0],
+        return pdl::ResultOp::create(builder, loc, mlirType, parentExprs[0],
                                              builder.getI32IntegerAttr(0));
-      return builder.create<pdl::ResultsOp>(loc, mlirType, parentExprs[0]);
+      return pdl::ResultsOp::create(builder, loc, mlirType, parentExprs[0]);
     }
 
     const ods::Operation *odsOp = opType.getODSOperation();
@@ -455,7 +455,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
       unsigned resultIndex;
       name.getAsInteger(/*Radix=*/10, resultIndex);
       IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
-      return builder.create<pdl::ResultOp>(loc, genType(expr->getType()),
+      return pdl::ResultOp::create(builder, loc, genType(expr->getType()),
                                            parentExprs[0], index);
     }
 
@@ -474,7 +474,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
 
     // Generate the result access.
     IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
-    return builder.create<pdl::ResultsOp>(loc, genType(expr->getType()),
+    return pdl::ResultsOp::create(builder, loc, genType(expr->getType()),
                                           parentExprs[0], index);
   }
 
@@ -518,7 +518,7 @@ Value CodeGen::genExprImpl(const ast::OperationExpr *expr) {
   for (const ast::Expr *result : expr->getResultTypes())
     results.push_back(genSingleExpr(result));
 
-  return builder.create<pdl::OperationOp>(loc, opName, operands, attrNames,
+  return pdl::OperationOp::create(builder, loc, opName, operands, attrNames,
                                           attrValues, results);
 }
 
@@ -527,7 +527,7 @@ Value CodeGen::genExprImpl(const ast::RangeExpr *expr) {
   for (const ast::Expr *element : expr->getElements())
     llvm::append_range(elements, genExpr(element));
 
-  return builder.create<pdl::RangeOp>(genLoc(expr->getLoc()),
+  return pdl::RangeOp::create(builder, genLoc(expr->getLoc()),
                                       genType(expr->getType()), elements);
 }
 
@@ -541,7 +541,7 @@ SmallVector<Value> CodeGen::genExprImpl(const ast::TupleExpr *expr) {
 Value CodeGen::genExprImpl(const ast::TypeExpr *expr) {
   Type type = parseType(expr->getValue(), builder.getContext());
   assert(type && "invalid MLIR type data");
-  return builder.create<pdl::TypeOp>(genLoc(expr->getLoc()),
+  return pdl::TypeOp::create(builder, genLoc(expr->getLoc()),
                                      builder.getType<pdl::TypeType>(),
                                      TypeAttr::get(type));
 }
@@ -586,7 +586,7 @@ CodeGen::genConstraintOrRewriteCall(const T *decl, Location loc,
     } else {
       resultTypes.push_back(genType(declResultType));
     }
-    PDLOpT pdlOp = builder.create<PDLOpT>(loc, resultTypes,
+    PDLOpT pdlOp = PDLOpT::create(builder, loc, resultTypes,
                                           decl->getName().getName(), inputs);
     if (isNegated && std::is_same_v<PDLOpT, pdl::ApplyNativeConstraintOp>)
       cast<pdl::ApplyNativeConstraintOp>(pdlOp).setIsNegated(true);
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 4d01a83d37165..f83184abe5308 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -1505,7 +1505,7 @@ ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
   OpBuilder builder(outputTypes.front().getContext());
   builder.setInsertionPoint(ip.getBlock(), ip.getPoint());
   auto convertOp =
-      builder.create<UnrealizedConversionCastOp>(loc, outputTypes, inputs);
+      UnrealizedConversionCastOp::create(builder, loc, outputTypes, inputs);
   if (!valuesToMap.empty())
     mapping.map(valuesToMap, convertOp.getResults());
   if (castOp)

>From 46e79c40e547e8a56aa0b9b4e46b3f39741eab64 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 12:56:56 -0400
Subject: [PATCH 2/5] fix cpp custom builders

---
 mlir/include/mlir/Dialect/Arith/IR/Arith.h | 10 +++
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp     | 79 ++++++++++++++++++----
 mlir/lib/IR/BuiltinDialect.cpp             |  2 +-
 3 files changed, 76 insertions(+), 15 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 7c50c2036ffdc..79b09b5fcaccf 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -59,15 +59,21 @@ class ConstantIntOp : public arith::ConstantOp {
   /// Build a constant int op that produces an integer of the specified width.
   static void build(OpBuilder &builder, OperationState &result, int64_t value,
                     unsigned width);
+  static ConstantIntOp create(OpBuilder &builder, Location location,
+                              int64_t value, unsigned width);
 
   /// Build a constant int op that produces an integer of the specified type,
   /// which must be an integer type.
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     int64_t value);
+  static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
+                              int64_t value);
 
   /// Build a constant int op that produces an integer from an APInt
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     const APInt &value);
+  static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
+                              const APInt &value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
@@ -85,6 +91,8 @@ class ConstantFloatOp : public arith::ConstantOp {
   /// Build a constant float op that produces a float of the specified type.
   static void build(OpBuilder &builder, OperationState &result, FloatType type,
                     const APFloat &value);
+  static ConstantFloatOp create(OpBuilder &builder, Location location,
+                                FloatType type, const APFloat &value);
 
   inline APFloat value() {
     return cast<FloatAttr>(arith::ConstantOp::getValue()).getValue();
@@ -100,6 +108,8 @@ class ConstantIndexOp : public arith::ConstantOp {
   static ::mlir::TypeID resolveTypeID() { return TypeID::get<ConstantOp>(); }
   /// Build a constant int op that produces an index.
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
+  static ConstantIndexOp create(OpBuilder &builder, Location location,
+                                int64_t value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 60c1fc6adacff..fdc2d3ac737e7 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -255,18 +255,49 @@ void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location,
+                                                  int64_t value,
+                                                  unsigned width) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, value, width);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, int64_t value) {
   arith::ConstantOp::build(builder, result, type,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location, Type type,
+                                                  int64_t value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, const APInt &value) {
   arith::ConstantOp::build(builder, result, type,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location, Type type,
+                                                  const APInt &value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -279,6 +310,17 @@ void arith::ConstantFloatOp::build(OpBuilder &builder, OperationState &result,
                            builder.getFloatAttr(type, value));
 }
 
+arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
+                                                      Location location,
+                                                      FloatType type,
+                                                      const APFloat &value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantFloatOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantFloatOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return llvm::isa<FloatType>(constOp.getType());
@@ -291,6 +333,16 @@ void arith::ConstantIndexOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIndexAttr(value));
 }
 
+arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
+                                                      Location location,
+                                                      int64_t value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, value);
+  auto result = llvm::dyn_cast<ConstantIndexOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantIndexOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isIndex();
@@ -2334,9 +2386,8 @@ class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
     // comparison.
     rewriter.replaceOpWithNewOp<CmpIOp>(
         op, pred, intVal,
-        ConstantOp::create(rewriter,
-            op.getLoc(), intVal.getType(),
-            rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
+        ConstantOp::create(rewriter, op.getLoc(), intVal.getType(),
+                           rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
     return success();
   }
 };
@@ -2373,10 +2424,10 @@ struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
         matchPattern(op.getFalseValue(), m_One())) {
       rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
           op, op.getType(),
-          arith::XOrIOp::create(rewriter,
-              op.getLoc(), op.getCondition(),
-              arith::ConstantIntOp::create(rewriter,
-                  op.getLoc(), op.getCondition().getType(), 1)));
+          arith::XOrIOp::create(
+              rewriter, op.getLoc(), op.getCondition(),
+              arith::ConstantIntOp::create(rewriter, op.getLoc(),
+                                           op.getCondition().getType(), 1)));
       return success();
     }
 
@@ -2439,12 +2490,12 @@ OpFoldResult arith::SelectOp::fold(FoldAdaptor adaptor) {
 
   // Constant-fold constant operands over non-splat constant condition.
   // select %cst_vec, %cst0, %cst1 => %cst2
-  if (auto cond =
-          llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getCondition())) {
-    if (auto lhs =
-            llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getTrueValue())) {
-      if (auto rhs =
-              llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getFalseValue())) {
+  if (auto cond = llvm::dyn_cast_if_present<DenseElementsAttr>(
+          adaptor.getCondition())) {
+    if (auto lhs = llvm::dyn_cast_if_present<DenseElementsAttr>(
+            adaptor.getTrueValue())) {
+      if (auto rhs = llvm::dyn_cast_if_present<DenseElementsAttr>(
+              adaptor.getFalseValue())) {
         SmallVector<Attribute> results;
         results.reserve(static_cast<size_t>(cond.getNumElements()));
         auto condVals = llvm::make_range(cond.value_begin<BoolAttr>(),
@@ -2712,7 +2763,7 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
     return arith::MaximumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minimumf:
     return arith::MinimumFOp::create(builder, loc, lhs, rhs);
-   case AtomicRMWKind::maxnumf:
+  case AtomicRMWKind::maxnumf:
     return arith::MaxNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minnumf:
     return arith::MinNumFOp::create(builder, loc, lhs, rhs);
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index c88b328282275..6d7e2aa0ece7d 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -132,7 +132,7 @@ void ModuleOp::build(OpBuilder &builder, OperationState &state,
 /// Construct a module from the given context.
 ModuleOp ModuleOp::create(Location loc, std::optional<StringRef> name) {
   OpBuilder builder(loc->getContext());
-  return ModuleOp::create(builder, loc, name);
+  return builder.create<ModuleOp>(loc, name);
 }
 
 DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {

>From 08544213cbd4968a54c8720a4e868ab7a4b81485 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 17:06:46 -0400
Subject: [PATCH 3/5] fix arith+smt+create impmlicit

---
 .../mlir/Dialect/Affine/IR/AffineOps.h        |  21 ++++
 mlir/include/mlir/Dialect/Arith/IR/Arith.h    |   9 ++
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    | 116 +++++++++---------
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    |   2 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      |   2 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  98 ++++++++++-----
 .../Affine/Transforms/SuperVectorize.cpp      |  49 ++++----
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |  28 +++++
 .../Async/Transforms/AsyncParallelFor.cpp     | 105 ++++++++--------
 .../GPU/Transforms/MemoryPromotion.cpp        |   6 +-
 10 files changed, 269 insertions(+), 167 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
index 2091faa6b0b02..333de6bbd8a05 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
@@ -114,6 +114,21 @@ class AffineDmaStartOp
                     AffineMap tagMap, ValueRange tagIndices, Value numElements,
                     Value stride = nullptr, Value elementsPerStride = nullptr);
 
+  static AffineDmaStartOp
+  create(OpBuilder &builder, Location location, Value srcMemRef,
+         AffineMap srcMap, ValueRange srcIndices, Value destMemRef,
+         AffineMap dstMap, ValueRange destIndices, Value tagMemRef,
+         AffineMap tagMap, ValueRange tagIndices, Value numElements,
+         Value stride = nullptr, Value elementsPerStride = nullptr);
+
+  static AffineDmaStartOp create(ImplicitLocOpBuilder &builder, Value srcMemRef,
+                                 AffineMap srcMap, ValueRange srcIndices,
+                                 Value destMemRef, AffineMap dstMap,
+                                 ValueRange destIndices, Value tagMemRef,
+                                 AffineMap tagMap, ValueRange tagIndices,
+                                 Value numElements, Value stride = nullptr,
+                                 Value elementsPerStride = nullptr);
+
   /// Returns the operand index of the source memref.
   unsigned getSrcMemRefOperandIndex() { return 0; }
 
@@ -319,6 +334,12 @@ class AffineDmaWaitOp
 
   static void build(OpBuilder &builder, OperationState &result, Value tagMemRef,
                     AffineMap tagMap, ValueRange tagIndices, Value numElements);
+  static AffineDmaWaitOp create(OpBuilder &builder, Location location,
+                                Value tagMemRef, AffineMap tagMap,
+                                ValueRange tagIndices, Value numElements);
+  static AffineDmaWaitOp create(ImplicitLocOpBuilder &builder, Value tagMemRef,
+                                AffineMap tagMap, ValueRange tagIndices,
+                                Value numElements);
 
   static StringRef getOperationName() { return "affine.dma_wait"; }
 
diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 79b09b5fcaccf..0fc3db8e993d8 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -61,6 +61,8 @@ class ConstantIntOp : public arith::ConstantOp {
                     unsigned width);
   static ConstantIntOp create(OpBuilder &builder, Location location,
                               int64_t value, unsigned width);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, int64_t value,
+                              unsigned width);
 
   /// Build a constant int op that produces an integer of the specified type,
   /// which must be an integer type.
@@ -68,12 +70,16 @@ class ConstantIntOp : public arith::ConstantOp {
                     int64_t value);
   static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
                               int64_t value);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, Type type,
+                              int64_t value);
 
   /// Build a constant int op that produces an integer from an APInt
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     const APInt &value);
   static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
                               const APInt &value);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, Type type,
+                              const APInt &value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
@@ -93,6 +99,8 @@ class ConstantFloatOp : public arith::ConstantOp {
                     const APFloat &value);
   static ConstantFloatOp create(OpBuilder &builder, Location location,
                                 FloatType type, const APFloat &value);
+  static ConstantFloatOp create(ImplicitLocOpBuilder &builder, FloatType type,
+                                const APFloat &value);
 
   inline APFloat value() {
     return cast<FloatAttr>(arith::ConstantOp::getValue()).getValue();
@@ -110,6 +118,7 @@ class ConstantIndexOp : public arith::ConstantOp {
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
   static ConstantIndexOp create(OpBuilder &builder, Location location,
                                 int64_t value);
+  static ConstantIndexOp create(ImplicitLocOpBuilder &builder, int64_t value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 3f82342001ea5..69eec9d9bbf3e 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -20,8 +20,8 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
+#include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
@@ -114,7 +114,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
   auto makeConst = [&](int32_t index) -> Value {
     return LLVM::ConstantOp::create(rewriter, loc, IntegerType::get(ctx, 32),
-                                             rewriter.getI32IntegerAttr(index));
+                                    rewriter.getI32IntegerAttr(index));
   };
 
   if (arrayType) {
@@ -147,11 +147,11 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
         Value x1 =
             LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i * 2);
         Value x2 = LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult,
-                                                         i * 2 + 1);
+                                                i * 2 + 1);
         vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
-                                                     x1, makeConst(0));
+                                            x1, makeConst(0));
         vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
-                                                     x2, makeConst(1));
+                                            x2, makeConst(1));
         elements.push_back(vec);
       }
     }
@@ -160,7 +160,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
     Value result = LLVM::PoisonOp::create(rewriter, loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
       result = LLVM::InsertValueOp::create(rewriter, loc, result, el.value(),
-                                                    el.index());
+                                           el.index());
     }
     return result;
   }
@@ -208,8 +208,8 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
                          innerArrayTy.getElementType() == f32Ty)) {
       for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
            idx < innerSize; idx++) {
-        result.push_back(LLVM::ExtractElementOp::create(b,
-            toUse,
+        result.push_back(LLVM::ExtractElementOp::create(
+            b, toUse,
             LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(idx))));
       }
       continue;
@@ -285,8 +285,8 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
-    Value ldMatrixResult = NVVM::LdMatrixOp::create(b,
-        ldMatrixResultType, srcPtr,
+    Value ldMatrixResult = NVVM::LdMatrixOp::create(
+        b, ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
                                      : NVVM::MMALayout::row);
@@ -375,16 +375,16 @@ struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
     Type desiredRetTy = typeConverter->convertType(op->getResultTypes()[0]);
     Type intrinsicResTy = inferIntrinsicResultType(
         typeConverter->convertType(op->getResultTypes()[0]));
-    Value intrinsicResult = NVVM::MmaOp::create(b,
-        intrinsicResTy, matA, matB, matC,
-        /*shape=*/gemmShape,
-        /*b1Op=*/std::nullopt,
-        /*intOverflow=*/overflow,
-        /*multiplicandPtxTypes=*/
-        std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
-        /*multiplicandLayouts=*/
-        std::array<NVVM::MMALayout, 2>{NVVM::MMALayout::row,
-                                       NVVM::MMALayout::col});
+    Value intrinsicResult =
+        NVVM::MmaOp::create(b, intrinsicResTy, matA, matB, matC,
+                            /*shape=*/gemmShape,
+                            /*b1Op=*/std::nullopt,
+                            /*intOverflow=*/overflow,
+                            /*multiplicandPtxTypes=*/
+                            std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
+                            /*multiplicandLayouts=*/
+                            std::array<NVVM::MMALayout, 2>{
+                                NVVM::MMALayout::row, NVVM::MMALayout::col});
     rewriter.replaceOp(op, convertIntrinsicResult(op.getLoc(), intrinsicResTy,
                                                   desiredRetTy, intrinsicResult,
                                                   rewriter));
@@ -566,14 +566,15 @@ static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
   asmVals.push_back(indexData);
 
   return LLVM::InlineAsmOp::create(b,
-      /*resultTypes=*/intrinsicResultType,
-      /*operands=*/asmVals,
-      /*asm_string=*/asmStr,
-      /*constraints=*/constraintStr,
-      /*has_side_effects=*/true,
-      /*is_align_stack=*/false, LLVM::TailCallKind::None,
-      /*asm_dialect=*/asmDialectAttr,
-      /*operand_attrs=*/ArrayAttr());
+                                   /*resultTypes=*/intrinsicResultType,
+                                   /*operands=*/asmVals,
+                                   /*asm_string=*/asmStr,
+                                   /*constraints=*/constraintStr,
+                                   /*has_side_effects=*/true,
+                                   /*is_align_stack=*/false,
+                                   LLVM::TailCallKind::None,
+                                   /*asm_dialect=*/asmDialectAttr,
+                                   /*operand_attrs=*/ArrayAttr());
 }
 
 /// Lowers `nvgpu.mma.sp.sync` to inline assembly.
@@ -698,12 +699,12 @@ struct NVGPUAsyncCopyLowering
       // filled with zeros.
       Value c3I32 =
           LLVM::ConstantOp::create(b, b.getI32Type(), b.getI32IntegerAttr(3));
-      Value bitwidth = LLVM::ConstantOp::create(b,
-          b.getI32Type(),
+      Value bitwidth = LLVM::ConstantOp::create(
+          b, b.getI32Type(),
           b.getI32IntegerAttr(srcMemrefType.getElementTypeBitWidth()));
       Value srcElementsI32 = LLVM::TruncOp::create(b, b.getI32Type(), srcBytes);
-      srcBytes = LLVM::LShrOp::create(b,
-          LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
+      srcBytes = LLVM::LShrOp::create(
+          b, LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
     }
     // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
     // 16 dst bytes.
@@ -712,14 +713,15 @@ struct NVGPUAsyncCopyLowering
             ? NVVM::LoadCacheModifierKind::CG
             : NVVM::LoadCacheModifierKind::CA;
 
-    NVVM::CpAsyncOp::create(b,
-        dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
+    NVVM::CpAsyncOp::create(
+        b, dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
         NVVM::LoadCacheModifierKindAttr::get(op->getContext(), cacheModifier),
         srcBytes);
 
     // Drop the result token.
-    Value zero = LLVM::ConstantOp::create(b,
-        IntegerType::get(op.getContext(), 32), rewriter.getI32IntegerAttr(0));
+    Value zero =
+        LLVM::ConstantOp::create(b, IntegerType::get(op.getContext(), 32),
+                                 rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
   }
@@ -735,9 +737,9 @@ struct NVGPUAsyncCreateGroupLowering
                   ConversionPatternRewriter &rewriter) const override {
     NVVM::CpAsyncCommitGroupOp::create(rewriter, op.getLoc());
     // Drop the result token.
-    Value zero = LLVM::ConstantOp::create(rewriter,
-        op->getLoc(), IntegerType::get(op.getContext(), 32),
-        rewriter.getI32IntegerAttr(0));
+    Value zero = LLVM::ConstantOp::create(rewriter, op->getLoc(),
+                                          IntegerType::get(op.getContext(), 32),
+                                          rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
   }
@@ -771,8 +773,8 @@ struct NVGPUMBarrierCreateLowering
     SymbolTable symbolTable(moduleOp);
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(&moduleOp.front());
-    auto global = memref::GlobalOp::create(rewriter,
-        funcOp->getLoc(), "__mbarrier",
+    auto global = memref::GlobalOp::create(
+        rewriter, funcOp->getLoc(), "__mbarrier",
         /*sym_visibility=*/rewriter.getStringAttr("private"),
         /*type=*/barrierType,
         /*initial_value=*/ElementsAttr(),
@@ -1119,7 +1121,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 
 static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
   return LLVM::ConstantOp::create(b, b.getIntegerType(64),
-                                    b.getI32IntegerAttr(index));
+                                  b.getI32IntegerAttr(index));
 }
 
 /// Returns a Value that holds data type enum that is expected by CUDA driver.
@@ -1182,11 +1184,11 @@ struct NVGPUTmaCreateDescriptorOpLowering
     auto promotedOperands = getTypeConverter()->promoteOperands(
         b.getLoc(), op->getOperands(), adaptor.getOperands(), b);
 
-    Value boxArrayPtr = LLVM::AllocaOp::create(b, llvmPointerType, llvmInt64Type,
-                                                 makeI64Const(b, 5));
+    Value boxArrayPtr = LLVM::AllocaOp::create(
+        b, llvmPointerType, llvmInt64Type, makeI64Const(b, 5));
     for (auto [index, value] : llvm::enumerate(adaptor.getBoxDimensions())) {
       Value gep = LLVM::GEPOp::create(b, llvmPointerType, llvmPointerType,
-                                        boxArrayPtr, makeI64Const(b, index));
+                                      boxArrayPtr, makeI64Const(b, index));
       LLVM::StoreOp::create(b, value, gep);
     }
 
@@ -1430,9 +1432,9 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      return NVVM::WgmmaMmaAsyncOp::create(b,
-          matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
-          itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
+      return NVVM::WgmmaMmaAsyncOp::create(
+          b, matrixC.getType(), matrixC, descriptorA, descriptorB, shape,
+          itypeA, itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
           overflow);
     }
 
@@ -1444,7 +1446,8 @@ struct NVGPUWarpgroupMmaOpLowering
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
+        Value matrixC =
+            LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
             matrixC = generateWgmma(i, j, k, matrixC);
@@ -1452,7 +1455,7 @@ struct NVGPUWarpgroupMmaOpLowering
       }
       for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
         wgmmaResult = LLVM::InsertValueOp::create(b, wgmmaResult.getType(),
-                                                    wgmmaResult, matrix, idx);
+                                                  wgmmaResult, matrix, idx);
       }
       return wgmmaResult;
     }
@@ -1486,9 +1489,9 @@ struct NVGPUWarpgroupMmaOpLowering
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
     Value generateWarpgroupMma() {
-      NVVM::WgmmaFenceAlignedOp::create(b, );
+      NVVM::WgmmaFenceAlignedOp::create(b);
       Value wgmmaResult = generateWgmmaGroup();
-      NVVM::WgmmaGroupSyncAlignedOp::create(b, );
+      NVVM::WgmmaGroupSyncAlignedOp::create(b);
       NVVM::WgmmaWaitGroupSyncOp::create(b, op.getWaitGroup());
       return wgmmaResult;
     }
@@ -1626,7 +1629,8 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     auto stype = cast<LLVM::LLVMStructType>(matriDValue.getType());
     for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(matrixD);
-      Value innerStructValue = LLVM::ExtractValueOp::create(b, matriDValue, idx);
+      Value innerStructValue =
+          LLVM::ExtractValueOp::create(b, matriDValue, idx);
       storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
@@ -1656,15 +1660,15 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
       auto structType = cast<LLVM::LLVMStructType>(s);
       Value structValue = LLVM::ExtractValueOp::create(b, packStruct, idx);
       for (unsigned i = 0; i < structType.getBody().size(); ++i) {
-        structValue = LLVM::InsertValueOp::create(b,
-            structType, structValue, zero, ArrayRef<int64_t>({i}));
+        structValue = LLVM::InsertValueOp::create(b, structType, structValue,
+                                                  zero, ArrayRef<int64_t>({i}));
       }
       innerStructs.push_back(structValue);
     }
     // Pack the inner structs into a single struct
     for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
       packStruct = LLVM::InsertValueOp::create(b, packStruct.getType(),
-                                                 packStruct, matrix, idx);
+                                               packStruct, matrix, idx);
     }
     rewriter.replaceOp(op, packStruct);
     return success();
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 3eee444725f74..f490e065143e7 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -709,7 +709,7 @@ class ExecutionModePattern
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
-    structValue = LLVM::InsertValueOp::create(rewriter, loc, structValue,
+    structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
                                                        executionMode, 0);
 
     // Insert extra operands if they exist into execution mode info struct.
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 1b31d391cb0bc..d21833736a718 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -670,7 +670,7 @@ static Value createFPReductionComparisonOpLowering(
 
   if (accumulator) {
     result =
-        typename VectorToScalarMapper<LLVMRedIntrinOp>::Type::create(rewriter,
+        rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
             loc, result, accumulator);
   }
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 3113d5896741b..bd1be7c087d05 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -1747,6 +1747,32 @@ void AffineDmaStartOp::build(OpBuilder &builder, OperationState &result,
   }
 }
 
+AffineDmaStartOp AffineDmaStartOp::create(
+    OpBuilder &builder, Location location, Value srcMemRef, AffineMap srcMap,
+    ValueRange srcIndices, Value destMemRef, AffineMap dstMap,
+    ValueRange destIndices, Value tagMemRef, AffineMap tagMap,
+    ValueRange tagIndices, Value numElements, Value stride,
+    Value elementsPerStride) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, srcMemRef, srcMap, srcIndices, destMemRef, dstMap,
+        destIndices, tagMemRef, tagMap, tagIndices, numElements, stride,
+        elementsPerStride);
+  auto result = llvm::dyn_cast<AffineDmaStartOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
+AffineDmaStartOp AffineDmaStartOp::create(
+    ImplicitLocOpBuilder &builder, Value srcMemRef, AffineMap srcMap,
+    ValueRange srcIndices, Value destMemRef, AffineMap dstMap,
+    ValueRange destIndices, Value tagMemRef, AffineMap tagMap,
+    ValueRange tagIndices, Value numElements, Value stride,
+    Value elementsPerStride) {
+  return create(builder, builder.getLoc(), srcMemRef, srcMap, srcIndices,
+                destMemRef, dstMap, destIndices, tagMemRef, tagMap, tagIndices,
+                numElements, stride, elementsPerStride);
+}
+
 void AffineDmaStartOp::print(OpAsmPrinter &p) {
   p << " " << getSrcMemRef() << '[';
   p.printAffineMapOfSSAIds(getSrcMapAttr(), getSrcIndices());
@@ -1917,6 +1943,25 @@ void AffineDmaWaitOp::build(OpBuilder &builder, OperationState &result,
   result.addOperands(numElements);
 }
 
+AffineDmaWaitOp AffineDmaWaitOp::create(OpBuilder &builder, Location location,
+                                        Value tagMemRef, AffineMap tagMap,
+                                        ValueRange tagIndices,
+                                        Value numElements) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, tagMemRef, tagMap, tagIndices, numElements);
+  auto result = llvm::dyn_cast<AffineDmaWaitOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
+AffineDmaWaitOp AffineDmaWaitOp::create(ImplicitLocOpBuilder &builder,
+                                        Value tagMemRef, AffineMap tagMap,
+                                        ValueRange tagIndices,
+                                        Value numElements) {
+  return create(builder, builder.getLoc(), tagMemRef, tagMap, tagIndices,
+                numElements);
+}
+
 void AffineDmaWaitOp::print(OpAsmPrinter &p) {
   p << " " << getTagMemRef() << '[';
   SmallVector<Value, 2> operands(getTagIndices());
@@ -2688,8 +2733,8 @@ FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInits());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  AffineForOp newLoop = AffineForOp::create(rewriter,
-      getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
+  AffineForOp newLoop = AffineForOp::create(
+      rewriter, getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
       getUpperBoundOperands(), getUpperBoundMap(), getStepAsInt(), inits);
 
   // Generate the new yield values and append them to the scf.yield operation.
@@ -2847,7 +2892,7 @@ buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb,
                              int64_t ub, int64_t step,
                              AffineForOp::BodyBuilderFn bodyBuilderFn) {
   return AffineForOp::create(builder, loc, lb, ub, step,
-                                     /*iterArgs=*/ValueRange(), bodyBuilderFn);
+                             /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
 /// Creates an affine loop from the bounds that may or may not be constants.
@@ -2861,8 +2906,8 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
     return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
                                         ubConst.value(), step, bodyBuilderFn);
   return AffineForOp::create(builder, loc, lb, builder.getDimIdentityMap(), ub,
-                                     builder.getDimIdentityMap(), step,
-                                     /*iterArgs=*/ValueRange(), bodyBuilderFn);
+                             builder.getDimIdentityMap(), step,
+                             /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
 void mlir::affine::buildAffineLoopNest(
@@ -4906,8 +4951,8 @@ struct DropUnitExtentBasis
 
     if (!newBasis.empty()) {
       // Will drop the leading nullptr from `basis` if there was no outer bound.
-      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(rewriter,
-          loc, delinearizeOp.getLinearIndex(), newBasis);
+      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(
+          rewriter, loc, delinearizeOp.getLinearIndex(), newBasis);
       int newIndex = 0;
       // Map back the new delinearized indices to the values they replace.
       for (auto &replacement : replacements) {
@@ -4971,12 +5016,12 @@ struct CancelDelinearizeOfLinearizeDisjointExactTail
       return success();
     }
 
-    Value newLinearize = affine::AffineLinearizeIndexOp::create(rewriter,
-        linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
+    Value newLinearize = affine::AffineLinearizeIndexOp::create(
+        rewriter, linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
         ArrayRef<OpFoldResult>{linearizeBasis}.drop_back(numMatches),
         linearizeOp.getDisjoint());
-    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-        delinearizeOp.getLoc(), newLinearize,
+    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), newLinearize,
         ArrayRef<OpFoldResult>{delinearizeBasis}.drop_back(numMatches),
         delinearizeOp.hasOuterBound());
     SmallVector<Value> mergedResults(newDelinearize.getResults());
@@ -5048,19 +5093,16 @@ struct SplitDelinearizeSpanningLastLinearizeArg final
           delinearizeOp,
           "need at least two elements to form the basis product");
 
-    Value linearizeWithoutBack =
-        affine::AffineLinearizeIndexOp::create(rewriter,
-            linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
-            linearizeOp.getDynamicBasis(),
-            linearizeOp.getStaticBasis().drop_back(),
-            linearizeOp.getDisjoint());
-    auto delinearizeWithoutSplitPart =
-        affine::AffineDelinearizeIndexOp::create(rewriter,
-            delinearizeOp.getLoc(), linearizeWithoutBack,
-            delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
-            delinearizeOp.hasOuterBound());
-    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(rewriter,
-        delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
+    Value linearizeWithoutBack = affine::AffineLinearizeIndexOp::create(
+        rewriter, linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
+        linearizeOp.getDynamicBasis(), linearizeOp.getStaticBasis().drop_back(),
+        linearizeOp.getDisjoint());
+    auto delinearizeWithoutSplitPart = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), linearizeWithoutBack,
+        delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
+        delinearizeOp.hasOuterBound());
+    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
         basis.take_back(elemsToSplit), /*hasOuterBound=*/true);
     SmallVector<Value> results = llvm::to_vector(
         llvm::concat<Value>(delinearizeWithoutSplitPart.getResults(),
@@ -5437,16 +5479,16 @@ struct CancelLinearizeOfDelinearizePortion final
       newDelinBasis.erase(newDelinBasis.begin() + m.delinStart,
                           newDelinBasis.begin() + m.delinStart + m.length);
       newDelinBasis.insert(newDelinBasis.begin() + m.delinStart, newSize);
-      auto newDelinearize = AffineDelinearizeIndexOp::create(rewriter,
-          m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
+      auto newDelinearize = AffineDelinearizeIndexOp::create(
+          rewriter, m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
           newDelinBasis);
 
       // Since there may be other uses of the indices we just merged together,
       // create a residual affine.delinearize_index that delinearizes the
       // merged output into its component parts.
       Value combinedElem = newDelinearize.getResult(m.delinStart);
-      auto residualDelinearize = AffineDelinearizeIndexOp::create(rewriter,
-          m.delinearize.getLoc(), combinedElem, basisToMerge);
+      auto residualDelinearize = AffineDelinearizeIndexOp::create(
+          rewriter, m.delinearize.getLoc(), combinedElem, basisToMerge);
 
       // Swap all the uses of the unaffected delinearize outputs to the new
       // delinearization so that the old code can be removed if this
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index b078514cbf978..827e1dff9f319 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -905,8 +905,8 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   for (auto resultExpr : map.getResults()) {
     auto singleResMap =
         AffineMap::get(map.getNumDims(), map.getNumSymbols(), resultExpr);
-    auto afOp = state.AffineApplyOp::create(builder, op->getLoc(), singleResMap,
-                                                    mapOperands);
+    auto afOp = AffineApplyOp::create(state.builder, op->getLoc(), singleResMap,
+                                      mapOperands);
     results.push_back(afOp);
   }
 }
@@ -961,7 +961,7 @@ static arith::ConstantOp vectorizeConstant(arith::ConstantOp constOp,
   auto vecForOp = cast<AffineForOp>(parentOp);
   state.builder.setInsertionPointToStart(vecForOp.getBody());
   auto newConstOp =
-      state.arith::ConstantOp::create(builder, constOp.getLoc(), vecAttr);
+      arith::ConstantOp::create(state.builder, constOp.getLoc(), vecAttr);
 
   // Register vector replacement for future uses in the scope.
   state.registerOpVectorReplacement(constOp, newConstOp);
@@ -986,8 +986,8 @@ static Operation *vectorizeAffineApplyOp(AffineApplyOp applyOp,
     }
   }
 
-  auto newApplyOp = state.AffineApplyOp::create(builder,
-      applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
+  auto newApplyOp = AffineApplyOp::create(
+      state.builder, applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
 
   // Register the new affine.apply result.
   state.registerValueScalarReplacement(applyOp.getResult(),
@@ -1010,7 +1010,7 @@ static arith::ConstantOp createInitialVector(arith::AtomicRMWKind reductionKind,
   auto vecTy = getVectorType(scalarTy, state.strategy);
   auto vecAttr = DenseElementsAttr::get(vecTy, valueAttr);
   auto newConstOp =
-      state.arith::ConstantOp::create(builder, oldOperand.getLoc(), vecAttr);
+      arith::ConstantOp::create(state.builder, oldOperand.getLoc(), vecAttr);
 
   return newConstOp;
 }
@@ -1062,11 +1062,11 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   AffineMap ubMap = vecForOp.getUpperBoundMap();
   Value ub;
   if (ubMap.getNumResults() == 1)
-    ub = state.AffineApplyOp::create(builder, loc, vecForOp.getUpperBoundMap(),
-                                             vecForOp.getUpperBoundOperands());
+    ub = AffineApplyOp::create(state.builder, loc, vecForOp.getUpperBoundMap(),
+                               vecForOp.getUpperBoundOperands());
   else
-    ub = state.AffineMinOp::create(builder, loc, vecForOp.getUpperBoundMap(),
-                                           vecForOp.getUpperBoundOperands());
+    ub = AffineMinOp::create(state.builder, loc, vecForOp.getUpperBoundMap(),
+                             vecForOp.getUpperBoundOperands());
   // Then we compute the number of (original) iterations left in the loop.
   AffineExpr subExpr =
       state.builder.getAffineDimExpr(0) - state.builder.getAffineDimExpr(1);
@@ -1080,7 +1080,7 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   Type maskTy = VectorType::get(state.strategy->vectorSizes,
                                 state.builder.getIntegerType(1));
   Value mask =
-      state.vector::CreateMaskOp::create(builder, loc, maskTy, itersLeft);
+      vector::CreateMaskOp::create(state.builder, loc, maskTy, itersLeft);
 
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ creating a mask:\n"
                     << itersLeft << "\n"
@@ -1123,8 +1123,8 @@ static Operation *vectorizeUniform(Value uniformVal,
   state.builder.setInsertionPointAfterValue(uniformScalarRepl);
 
   auto vectorTy = getVectorType(uniformVal.getType(), state.strategy);
-  auto bcastOp = state.BroadcastOp::create(builder, uniformVal.getLoc(),
-                                                   vectorTy, uniformScalarRepl);
+  auto bcastOp = BroadcastOp::create(state.builder, uniformVal.getLoc(),
+                                     vectorTy, uniformScalarRepl);
   state.registerValueVectorReplacement(uniformVal, bcastOp);
   return bcastOp;
 }
@@ -1256,8 +1256,8 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.vector::TransferReadOp::create(builder,
-      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
+  auto transfer = vector::TransferReadOp::create(
+      state.builder, loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
       /*padding=*/std::nullopt, permutationMap);
 
   // Register replacement for future uses in the scope.
@@ -1303,9 +1303,9 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.vector::TransferWriteOp::create(builder,
-      storeOp.getLoc(), vectorValue, storeOp.getMemRef(), indices,
-      permutationMap);
+  auto transfer = vector::TransferWriteOp::create(
+      state.builder, storeOp.getLoc(), vectorValue, storeOp.getMemRef(),
+      indices, permutationMap);
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
 
   // Register replacement for future uses in the scope.
@@ -1387,10 +1387,10 @@ static Operation *vectorizeAffineForOp(AffineForOp forOp,
     }
   }
 
-  auto vecForOp = state.AffineForOp::create(builder,
-      forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(),
-      forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep,
-      vecIterOperands,
+  auto vecForOp = AffineForOp::create(
+      state.builder, forOp.getLoc(), forOp.getLowerBoundOperands(),
+      forOp.getLowerBoundMap(), forOp.getUpperBoundOperands(),
+      forOp.getUpperBoundMap(), newStep, vecIterOperands,
       /*bodyBuilder=*/[](OpBuilder &, Location, Value, ValueRange) {
         // Make sure we don't create a default terminator in the loop body as
         // the proper terminator will be added during vectorization.
@@ -1512,8 +1512,8 @@ static Operation *vectorizeAffineYieldOp(AffineYieldOp yieldOp,
       // IterOperands are neutral element vectors.
       Value neutralVal = cast<AffineForOp>(newParentOp).getInits()[i];
       state.builder.setInsertionPoint(combinerOps.back());
-      Value maskedReducedVal = state.arith::SelectOp::create(builder,
-          reducedVal.getLoc(), mask, reducedVal, neutralVal);
+      Value maskedReducedVal = arith::SelectOp::create(
+          state.builder, reducedVal.getLoc(), mask, reducedVal, neutralVal);
       LLVM_DEBUG(
           dbgs() << "\n[early-vect]+++++ masking an input to a binary op that"
                     "produces value for a yield Op: "
@@ -1865,7 +1865,6 @@ verifyLoopNesting(const std::vector<SmallVector<AffineForOp, 2>> &loops) {
   return success();
 }
 
-
 /// External utility to vectorize affine loops in 'loops' using the n-D
 /// vectorization factors in 'vectorSizes'. By default, each vectorization
 /// factor is applied inner-to-outer to the loops of each loop nest.
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index fdc2d3ac737e7..ac00917d33a85 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -266,6 +266,12 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  int64_t value,
+                                                  unsigned width) {
+  return create(builder, builder.getLoc(), value, width);
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, int64_t value) {
   arith::ConstantOp::build(builder, result, type,
@@ -282,6 +288,11 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  Type type, int64_t value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, const APInt &value) {
   arith::ConstantOp::build(builder, result, type,
@@ -298,6 +309,12 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  Type type,
+                                                  const APInt &value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -321,6 +338,12 @@ arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantFloatOp
+arith::ConstantFloatOp::create(ImplicitLocOpBuilder &builder, FloatType type,
+                               const APFloat &value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 bool arith::ConstantFloatOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return llvm::isa<FloatType>(constOp.getType());
@@ -343,6 +366,11 @@ arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIndexOp
+arith::ConstantIndexOp::create(ImplicitLocOpBuilder &builder, int64_t value) {
+  return create(builder, builder.getLoc(), value);
+}
+
 bool arith::ConstantIndexOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isIndex();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index 447411ad989ba..09367ab544ff8 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -369,21 +369,22 @@ static ParallelComputeFunction createParallelComputeFunction(
       ImplicitLocOpBuilder b(loc, nestedBuilder);
 
       // Compute induction variable for `loopIdx`.
-      computeBlockInductionVars[loopIdx] = arith::AddIOp::create(b,
-          lowerBounds[loopIdx], arith::MulIOp::create(b, iv, steps[loopIdx]));
+      computeBlockInductionVars[loopIdx] =
+          arith::AddIOp::create(b, lowerBounds[loopIdx],
+                                arith::MulIOp::create(b, iv, steps[loopIdx]));
 
       // Check if we are inside first or last iteration of the loop.
-      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
-      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
+      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
+      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
 
       // Check if the previous loop is in its first or last iteration.
       if (loopIdx > 0) {
-        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(b,
-            isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
-        isBlockLastCoord[loopIdx] = arith::AndIOp::create(b,
-            isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
+        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(
+            b, isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
+        isBlockLastCoord[loopIdx] = arith::AndIOp::create(
+            b, isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
       }
 
       // Keep building loop nest.
@@ -392,20 +393,20 @@ static ParallelComputeFunction createParallelComputeFunction(
           // For block aligned loops we always iterate starting from 0 up to
           // the loop trip counts.
           scf::ForOp::create(b, c0, tripCounts[loopIdx + 1], c1, ValueRange(),
-                               workLoopBuilder(loopIdx + 1));
+                             workLoopBuilder(loopIdx + 1));
 
         } else {
           // Select nested loop lower/upper bounds depending on our position in
           // the multi-dimensional iteration space.
           auto lb = arith::SelectOp::create(b, isBlockFirstCoord[loopIdx],
-                                              blockFirstCoord[loopIdx + 1], c0);
+                                            blockFirstCoord[loopIdx + 1], c0);
 
           auto ub = arith::SelectOp::create(b, isBlockLastCoord[loopIdx],
-                                              blockEndCoord[loopIdx + 1],
-                                              tripCounts[loopIdx + 1]);
+                                            blockEndCoord[loopIdx + 1],
+                                            tripCounts[loopIdx + 1]);
 
           scf::ForOp::create(b, lb, ub, c1, ValueRange(),
-                               workLoopBuilder(loopIdx + 1));
+                             workLoopBuilder(loopIdx + 1));
         }
 
         scf::YieldOp::create(b, loc);
@@ -424,7 +425,7 @@ static ParallelComputeFunction createParallelComputeFunction(
   };
 
   scf::ForOp::create(b, blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
-                       workLoopBuilder(0));
+                     workLoopBuilder(0));
   func::ReturnOp::create(b, ValueRange());
 
   return {op.getNumLoops(), func, std::move(computeFuncType.captures)};
@@ -537,13 +538,13 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
       operands[2] = end;
 
       func::CallOp::create(executeBuilder, executeLoc, func.getSymName(),
-                                          func.getResultTypes(), operands);
+                           func.getResultTypes(), operands);
       async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to dispatch half of the block range.
     auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
-                                       executeBodyBuilder);
+                                     executeBodyBuilder);
     AddToGroupOp::create(b, indexTy, execute.getToken(), group);
     scf::YieldOp::create(b, ValueRange({start, midIndex}));
   }
@@ -558,8 +559,7 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   computeFuncOperands.append(forwardedInputs.begin(), forwardedInputs.end());
 
   func::CallOp::create(b, computeFunc.func.getSymName(),
-                         computeFunc.func.getResultTypes(),
-                         computeFuncOperands);
+                       computeFunc.func.getResultTypes(), computeFuncOperands);
   func::ReturnOp::create(b, ValueRange());
 
   return func;
@@ -605,9 +605,9 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     appendBlockComputeOperands(operands);
 
     func::CallOp::create(b, parallelComputeFunction.func.getSymName(),
-                           parallelComputeFunction.func.getResultTypes(),
-                           operands);
-    scf::YieldOp::create(b, );
+                         parallelComputeFunction.func.getResultTypes(),
+                         operands);
+    scf::YieldOp::create(b);
   };
 
   auto asyncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -624,12 +624,12 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     appendBlockComputeOperands(operands);
 
     func::CallOp::create(b, asyncDispatchFunction.getSymName(),
-                           asyncDispatchFunction.getResultTypes(), operands);
+                         asyncDispatchFunction.getResultTypes(), operands);
 
     // Wait for the completion of all parallel compute operations.
     AwaitAllOp::create(b, group);
 
-    scf::YieldOp::create(b, );
+    scf::YieldOp::create(b);
   };
 
   // Dispatch either single block compute function, or launch async dispatch.
@@ -682,16 +682,15 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
                                   Location executeLoc, ValueRange executeArgs) {
       func::CallOp::create(executeBuilder, executeLoc, compute.getSymName(),
-                                          compute.getResultTypes(),
-                                          computeFuncOperands(iv));
+                           compute.getResultTypes(), computeFuncOperands(iv));
       async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to launch parallel computate function.
     auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
-                                       executeBodyBuilder);
+                                     executeBodyBuilder);
     AddToGroupOp::create(b, rewriter.getIndexType(), execute.getToken(), group);
-    scf::YieldOp::create(b, );
+    scf::YieldOp::create(b);
   };
 
   // Iterate over all compute blocks and launch parallel compute operations.
@@ -699,7 +698,7 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
 
   // Call parallel compute function for the first block in the caller thread.
   func::CallOp::create(b, compute.getSymName(), compute.getResultTypes(),
-                         computeFuncOperands(c0));
+                       computeFuncOperands(c0));
 
   // Wait for the completion of all async compute operations.
   AwaitAllOp::create(b, group);
@@ -800,7 +799,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     if (numWorkerThreads >= 0)
       numWorkerThreadsVal = arith::ConstantIndexOp::create(b, numWorkerThreads);
     else
-      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b, );
+      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b);
 
     // With large number of threads the value of creating many compute blocks
     // is reduced because the problem typically becomes memory bound. For this
@@ -819,16 +818,16 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
         {4, 4.0f}, {8, 2.0f}, {16, 1.0f}, {32, 0.8f}, {64, 0.6f}};
     const float initialOvershardingFactor = 8.0f;
 
-    Value scalingFactor = arith::ConstantFloatOp::create(b,
-        b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
+    Value scalingFactor = arith::ConstantFloatOp::create(
+        b, b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
     for (const std::pair<int, float> &p : overshardingBrackets) {
       Value bracketBegin = arith::ConstantIndexOp::create(b, p.first);
-      Value inBracket = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
-      Value bracketScalingFactor = arith::ConstantFloatOp::create(b,
-          b.getF32Type(), llvm::APFloat(p.second));
-      scalingFactor = arith::SelectOp::create(b, inBracket, bracketScalingFactor,
-                                                scalingFactor);
+      Value inBracket = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
+      Value bracketScalingFactor = arith::ConstantFloatOp::create(
+          b, b.getF32Type(), llvm::APFloat(p.second));
+      scalingFactor = arith::SelectOp::create(
+          b, inBracket, bracketScalingFactor, scalingFactor);
     }
     Value numWorkersIndex =
         arith::IndexCastOp::create(b, b.getI32Type(), numWorkerThreadsVal);
@@ -841,8 +840,8 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     Value scaledWorkers =
         arith::IndexCastOp::create(b, b.getIndexType(), scaledNumInt);
 
-    Value maxComputeBlocks = arith::MaxSIOp::create(b,
-        arith::ConstantIndexOp::create(b, 1), scaledWorkers);
+    Value maxComputeBlocks = arith::MaxSIOp::create(
+        b, arith::ConstantIndexOp::create(b, 1), scaledWorkers);
 
     // Compute parallel block size from the parallel problem size:
     //   blockSize = min(tripCount,
@@ -869,7 +868,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       doDispatch(b, rewriter, compute, op, blockSize, blockCount, tripCounts);
-      scf::YieldOp::create(b, );
+      scf::YieldOp::create(b);
     };
 
     // Dispatch parallel compute function with hints for unrolling inner loops.
@@ -880,27 +879,27 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       // Align the block size to be a multiple of the statically known
       // number of iterations in the inner loops.
-      Value numIters = arith::ConstantIndexOp::create(b,
-          numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value alignedBlockSize = arith::MulIOp::create(b,
-          arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
+      Value numIters = arith::ConstantIndexOp::create(
+          b, numIterations[op.getNumLoops() - numUnrollableLoops]);
+      Value alignedBlockSize = arith::MulIOp::create(
+          b, arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
       doDispatch(b, rewriter, compute, op, alignedBlockSize, blockCount,
                  tripCounts);
-      scf::YieldOp::create(b, );
+      scf::YieldOp::create(b);
     };
 
     // Dispatch to block aligned compute function only if the computed block
     // size is larger than the number of iterations in the unrollable inner
     // loops, because otherwise it can reduce the available parallelism.
     if (numUnrollableLoops > 0) {
-      Value numIters = arith::ConstantIndexOp::create(b,
-          numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value useBlockAlignedComputeFn = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::sge, blockSize, numIters);
+      Value numIters = arith::ConstantIndexOp::create(
+          b, numIterations[op.getNumLoops() - numUnrollableLoops]);
+      Value useBlockAlignedComputeFn = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::sge, blockSize, numIters);
 
       scf::IfOp::create(b, useBlockAlignedComputeFn, dispatchBlockAligned,
-                          dispatchDefault);
-      scf::YieldOp::create(b, );
+                        dispatchDefault);
+      scf::YieldOp::create(b);
     } else {
       dispatchDefault(b, loc);
     }
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 2bee1a1619665..4af08dad08462 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -18,7 +18,7 @@
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
+#include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
@@ -131,10 +131,10 @@ static void insertCopies(Region &region, Location loc, Value from, Value to) {
 
   auto b = ImplicitLocOpBuilder::atBlockBegin(loc, &region.front());
   insertCopyLoops(b, from, to);
-  gpu::BarrierOp::create(b, );
+  gpu::BarrierOp::create(b);
 
   b.setInsertionPoint(&region.front().back());
-  gpu::BarrierOp::create(b, );
+  gpu::BarrierOp::create(b);
   insertCopyLoops(b, to, from);
 }
 

>From 5b7efd80ce8d610d735421d82e5a6befe89532f8 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 18:26:17 -0400
Subject: [PATCH 4/5] implicitbuilder

---
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    |   1 +
 flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp   |   1 +
 flang/lib/Optimizer/Dialect/FIROps.cpp        |   1 +
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp     |   1 +
 .../lib/Standalone/StandaloneDialect.cpp      |   1 +
 .../lib/Standalone/StandaloneOps.cpp          |   1 +
 mlir/examples/toy/Ch2/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch3/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch4/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch5/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch6/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch7/mlir/Dialect.cpp        |   4 +-
 .../transform/Ch2/lib/MyExtension.cpp         |   1 +
 .../transform/Ch3/lib/MyExtension.cpp         |   1 +
 .../transform/Ch4/lib/MyExtension.cpp         |   1 +
 mlir/lib/CAPI/Dialect/PDL.cpp                 |   1 +
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    |   1 +
 mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp  |   1 +
 mlir/lib/Dialect/AMX/IR/AMXDialect.cpp        |   1 +
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |   1 +
 .../TransformOps/AffineTransformOps.cpp       |   1 +
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |   1 +
 .../lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp |   1 +
 .../ArmNeonVectorTransformOps.cpp             |   1 +
 mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp         |   1 +
 .../TransformOps/ArmSVEVectorTransformOps.cpp |   1 +
 mlir/lib/Dialect/Async/IR/Async.cpp           |   1 +
 .../Bufferization/IR/BufferizationOps.cpp     |  20 ++--
 mlir/lib/Dialect/Complex/IR/ComplexOps.cpp    |   5 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.cpp |  16 +--
 .../DLTI/TransformOps/DLTITransformOps.cpp    |   1 +
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |   7 +-
 .../Func/TransformOps/FuncTransformOps.cpp    |   3 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |   9 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |   5 +-
 .../GPU/Transforms/MemoryPromotion.cpp        |   1 +
 mlir/lib/Dialect/IRDL/IR/IRDL.cpp             |   1 +
 mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp          |   1 +
 mlir/lib/Dialect/Index/IR/IndexOps.cpp        |  16 ++-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |   7 +-
 mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp    |   1 +
 mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp   |   1 +
 mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp    |   1 +
 mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp    |   1 +
 .../lib/Dialect/MLProgram/IR/MLProgramOps.cpp |   1 +
 mlir/lib/Dialect/MPI/IR/MPIOps.cpp            |   5 +-
 mlir/lib/Dialect/Math/IR/MathOps.cpp          |   1 +
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |  27 +++--
 .../TransformOps/MemRefTransformOps.cpp       |   8 +-
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |  15 +--
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       |   1 +
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |   1 +
 mlir/lib/Dialect/PDL/IR/PDL.cpp               |   1 +
 mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp   |   1 +
 mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp        |   1 +
 mlir/lib/Dialect/Quant/IR/QuantOps.cpp        |   1 +
 mlir/lib/Dialect/SCF/IR/SCF.cpp               | 106 ++++++++---------
 .../SCF/TransformOps/SCFTransformOps.cpp      |   1 +
 mlir/lib/Dialect/SMT/IR/SMTOps.cpp            |   1 +
 .../Dialect/SPIRV/IR/SPIRVOpDefinition.cpp    |   1 +
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  64 ++++++-----
 .../SparseTensor/IR/SparseTensorDialect.cpp   |   3 +-
 .../TransformOps/SparseTensorTransformOps.cpp |   1 +
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      | 107 +++++++++---------
 .../TransformOps/TensorTransformOps.cpp       |   1 +
 .../DebugExtension/DebugExtension.cpp         |   1 +
 .../DebugExtension/DebugExtensionOps.cpp      |   1 +
 .../lib/Dialect/Transform/IR/TransformOps.cpp |   1 +
 .../IRDLExtension/IRDLExtensionOps.cpp        |   1 +
 .../LoopExtension/LoopExtensionOps.cpp        |   1 +
 mlir/lib/Dialect/UB/IR/UBOps.cpp              |   1 +
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  56 ++++-----
 .../TransformOps/VectorTransformOps.cpp       |   1 +
 .../Dialect/X86Vector/IR/X86VectorDialect.cpp |   3 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   1 +
 mlir/lib/IR/BuiltinDialect.cpp                |   1 +
 .../TestTransformDialectExtension.cpp         |   1 +
 .../TestTilingInterfaceTransformOps.cpp       |   1 +
 .../test/lib/Transforms/TestTransformsOps.cpp |   1 +
 mlir/test/python/lib/PythonTestDialect.cpp    |   1 +
 80 files changed, 342 insertions(+), 234 deletions(-)

diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index 687007d957225..cef95a13c5a50 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
index 19ad6bed512c7..d7d294ab621d8 100644
--- a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Optimizer/Dialect/FIRCG/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index b6bf2753b80ce..db1981b40a6ec 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -26,6 +26,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index df6ce12215d26..7f2ec418af40a 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/APInt.h"
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
index 1ea69f9059321..3da966ef9cc85 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
@@ -8,6 +8,7 @@
 
 #include "Standalone/StandaloneDialect.h"
 #include "Standalone/StandaloneOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "Standalone/StandaloneTypes.h"
 
 using namespace mlir;
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
index 55b66b51232f2..ef20049f1b76b 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
@@ -8,6 +8,7 @@
 
 #include "Standalone/StandaloneOps.h"
 #include "Standalone/StandaloneDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 #define GET_OP_CLASSES
 #include "Standalone/StandaloneOps.cpp.inc"
diff --git a/mlir/examples/toy/Ch2/mlir/Dialect.cpp b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
index 489f348c8be52..fc4c3c75dcfda 100644
--- a/mlir/examples/toy/Ch2/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -144,7 +145,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -280,7 +282,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch3/mlir/Dialect.cpp b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
index 708855f18cf45..af3cbb385d332 100644
--- a/mlir/examples/toy/Ch3/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -144,7 +145,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -280,7 +282,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch4/mlir/Dialect.cpp b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
index 076a75a26619b..e3ab656c193a3 100644
--- a/mlir/examples/toy/Ch4/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch5/mlir/Dialect.cpp b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
index fb7c742a01802..81c9e8f9a143f 100644
--- a/mlir/examples/toy/Ch5/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch6/mlir/Dialect.cpp b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
index fb7c742a01802..81c9e8f9a143f 100644
--- a/mlir/examples/toy/Ch6/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch7/mlir/Dialect.cpp b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
index 52881db87d86b..64aa4d8e8995d 100644
--- a/mlir/examples/toy/Ch7/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpImplementation.h"
@@ -429,7 +430,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/transform/Ch2/lib/MyExtension.cpp b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
index b4b27e97d266e..e24714b408e29 100644
--- a/mlir/examples/transform/Ch2/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/examples/transform/Ch3/lib/MyExtension.cpp b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
index 4b2123fa71d31..5af5cc8ac5052 100644
--- a/mlir/examples/transform/Ch3/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
@@ -17,6 +17,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/examples/transform/Ch4/lib/MyExtension.cpp b/mlir/examples/transform/Ch4/lib/MyExtension.cpp
index fa0ffc9dc2e8a..86884d3bee7a0 100644
--- a/mlir/examples/transform/Ch4/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch4/lib/MyExtension.cpp
@@ -13,6 +13,7 @@
 
 #include "MyExtension.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE_MATCHER "transform-matcher"
diff --git a/mlir/lib/CAPI/Dialect/PDL.cpp b/mlir/lib/CAPI/Dialect/PDL.cpp
index bd8b13c6516e2..d2a83da643a89 100644
--- a/mlir/lib/CAPI/Dialect/PDL.cpp
+++ b/mlir/lib/CAPI/Dialect/PDL.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 69eec9d9bbf3e..cd92e1dbba090 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -22,6 +22,7 @@
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index acaf6a2f8792a..ece11fe1a8fb2 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 748ff1edbfeb2..111d93e39c15e 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index bd1be7c087d05..cf21156085772 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
diff --git a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
index b1e40d9b289ec..267179f26c52e 100644
--- a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
+++ b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index ac00917d33a85..3eb1580e2a1d2 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
index 4bcad5f4ece25..022e8bb45107b 100644
--- a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
+++ b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/Builders.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
index d07e6a52d8b5f..f0b7a8a4b0954 100644
--- a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/ArmNeon/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
index bc505f936d6c5..12018912f6f66 100644
--- a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/Builders.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp b/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
index b2ca4fc1eaa8c..b50679239a840 100644
--- a/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index dc7b07d911c17..7dd1b856fa2b0 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -10,6 +10,7 @@
 
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
+#include "mlir/IR/Builders.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index e64fee235e50b..58bae9fcdc4d3 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include <optional>
 
@@ -321,8 +322,8 @@ struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
         newShape, op.getType().getElementType(), op.getType().getEncoding());
     if (newType == op.getType())
       return failure();
-    auto newOp = AllocTensorOp::create(rewriter,
-        op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
+    auto newOp = AllocTensorOp::create(rewriter, op.getLoc(), newType,
+                                       newDynamicSizes, /*copy=*/Value());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -514,7 +515,7 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
 
     if (source.getType() != cloneOp.getType())
       source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
-                                               cloneOp.getType(), source);
+                                      cloneOp.getType(), source);
     rewriter.replaceOp(cloneOp, source);
     rewriter.eraseOp(redundantDealloc);
     return success();
@@ -643,8 +644,9 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return ToTensorOp::create(builder,
-      loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
+  return ToTensorOp::create(
+      builder, loc, memref::getTensorTypeFromMemRefType(getDest().getType()),
+      getDest(),
       /*restrict=*/true, getWritable());
 }
 
@@ -807,7 +809,7 @@ struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
     auto memrefType = MemRefType::get(srcTensorType.getShape(),
                                       srcTensorType.getElementType());
     Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
-                                               tensorCastOperand.getOperand());
+                                      tensorCastOperand.getOperand());
     rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                 memref);
     return success();
@@ -1015,7 +1017,7 @@ struct DeallocRemoveDuplicateRetainedMemrefs
     // same as the number of condition operands.
     auto newDeallocOp =
         DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
-                                   deallocOp.getConditions(), newRetained);
+                          deallocOp.getConditions(), newRetained);
     SmallVector<Value> replacements(
         llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
           return newDeallocOp.getUpdatedConditions()[idx];
@@ -1036,8 +1038,8 @@ struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     if (deallocOp.getMemrefs().empty()) {
-      Value constFalse = arith::ConstantOp::create(rewriter,
-          deallocOp.getLoc(), rewriter.getBoolAttr(false));
+      Value constFalse = arith::ConstantOp::create(rewriter, deallocOp.getLoc(),
+                                                   rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
                                         constFalse));
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
index fb97045687d65..34008aa5c4d6a 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 
@@ -20,9 +21,7 @@ using namespace mlir::complex;
 // ConstantOp
 //===----------------------------------------------------------------------===//
 
-OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) {
-  return getValue();
-}
+OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValue(); }
 
 void ConstantOp::getAsmResultNames(
     function_ref<void(Value, StringRef)> setNameFn) {
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 38e47cc5e3714..66593783a14f1 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -89,7 +90,7 @@ LogicalResult AssertOp::canonicalize(AssertOp op, PatternRewriter &rewriter) {
   return failure();
 }
 
-// This side effect models "program termination". 
+// This side effect models "program termination".
 void AssertOp::getEffects(
     SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
         &effects) {
@@ -312,8 +313,9 @@ struct SimplifyCondBranchIdenticalSuccessors
       if (std::get<0>(it) == std::get<1>(it))
         mergedOperands.push_back(std::get<0>(it));
       else
-        mergedOperands.push_back(arith::SelectOp::create(rewriter,
-            condbr.getLoc(), condition, std::get<0>(it), std::get<1>(it)));
+        mergedOperands.push_back(
+            arith::SelectOp::create(rewriter, condbr.getLoc(), condition,
+                                    std::get<0>(it), std::get<1>(it)));
     }
 
     rewriter.replaceOpWithNewOp<BranchOp>(condbr, trueDest, mergedOperands);
@@ -412,8 +414,8 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantTrue)
-            constantTrue = arith::ConstantOp::create(rewriter,
-                condbr.getLoc(), ty, rewriter.getBoolAttr(true));
+            constantTrue = arith::ConstantOp::create(
+                rewriter, condbr.getLoc(), ty, rewriter.getBoolAttr(true));
 
           rewriter.modifyOpInPlace(use.getOwner(),
                                    [&] { use.set(constantTrue); });
@@ -427,8 +429,8 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantFalse)
-            constantFalse = arith::ConstantOp::create(rewriter,
-                condbr.getLoc(), ty, rewriter.getBoolAttr(false));
+            constantFalse = arith::ConstantOp::create(
+                rewriter, condbr.getLoc(), ty, rewriter.getBoolAttr(false));
 
           rewriter.modifyOpInPlace(use.getOwner(),
                                    [&] { use.set(constantFalse); });
diff --git a/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp b/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
index 02c41b4fe8113..4dbd1659e24dc 100644
--- a/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
+++ b/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/Utils/Utils.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index 9a33fc8e07dde..f40e1b625694a 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -51,7 +52,7 @@ Operation *FuncDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                             Type type, Location loc) {
   if (ConstantOp::isBuildableWith(value, type))
     return ConstantOp::create(builder, loc, type,
-                                      llvm::cast<FlatSymbolRefAttr>(value));
+                              llvm::cast<FlatSymbolRefAttr>(value));
   return nullptr;
 }
 
@@ -139,9 +140,7 @@ LogicalResult ConstantOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   return success();
 }
 
-OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) {
-  return getValueAttr();
-}
+OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValueAttr(); }
 
 void ConstantOp::getAsmResultNames(
     function_ref<void(Value, StringRef)> setNameFn) {
diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
index 10515251fef7d..d869309cdd63b 100644
--- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
+++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
@@ -171,7 +172,7 @@ transform::CastAndCallOp::apply(transform::TransformRewriter &rewriter,
   }
 
   auto callOp = func::CallOp::create(rewriter, insertionPoint->getLoc(),
-                                              targetFunction, inputs);
+                                     targetFunction, inputs);
 
   // Cast the call results back to the expected types. If any conversions fail
   // this is a definite failure as the call has been constructed at this point.
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index f7eb41be287e4..8d6962d2d2be8 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -23,6 +23,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -1382,9 +1383,9 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
                       int32_t offset, int32_t width, ShuffleMode mode) {
   build(builder, result, value,
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(offset)),
+                                  builder.getI32IntegerAttr(offset)),
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(width)),
+                                  builder.getI32IntegerAttr(width)),
         mode);
 }
 
@@ -1396,9 +1397,9 @@ void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
                      int32_t offset, int32_t width) {
   build(builder, result, value,
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(offset)),
+                                  builder.getI32IntegerAttr(offset)),
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(width)));
+                                  builder.getI32IntegerAttr(width)));
 }
 
 LogicalResult RotateOp::verify() {
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index 2035f371694a7..26adfade879d8 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -31,6 +31,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/Visitors.h"
@@ -561,7 +562,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   if (originalBasisWasProvided) {
     for (Value tmpPredicate : builderResult.predicateOps) {
       predicate = predicate ? arith::AndIOp::create(rewriter, loc, predicate,
-                                                             tmpPredicate)
+                                                    tmpPredicate)
                             : tmpPredicate;
     }
   }
@@ -574,7 +575,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   if (predicate) {
     // Step 6.a. If predicated, move at the beginning.
     auto ifOp = scf::IfOp::create(rewriter, loc, predicate,
-                                           /*withElseRegion=*/false);
+                                  /*withElseRegion=*/false);
     targetBlock = ifOp.thenBlock();
     insertionPoint = ifOp.thenBlock()->begin();
   } else {
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 4af08dad08462..4a670373272a7 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
index 197a5907fc4cb..25df6f876e101 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
index 7ec3aa2741023..c62e5c9c50183 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/IRDL/IR/IRDL.h"
 #include "mlir/Dialect/IRDL/IRDLSymbols.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/ValueRange.h"
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Index/IR/IndexOps.cpp b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
index 7cab93709b9c6..b7d00c553d70a 100644
--- a/mlir/lib/Dialect/Index/IR/IndexOps.cpp
+++ b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Index/IR/IndexAttrs.h"
 #include "mlir/Dialect/Index/IR/IndexDialect.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/Utils/InferIntRangeCommon.h"
@@ -129,15 +130,18 @@ canonicalizeAssociativeCommutativeBinaryOp(BinaryOp op,
 
   auto lhsOp = op.getLhs().template getDefiningOp<BinaryOp>();
   if (!lhsOp)
-    return rewriter.notifyMatchFailure(op.getLoc(), "LHS is not the same BinaryOp");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "LHS is not the same BinaryOp");
 
   if (!mlir::matchPattern(lhsOp.getRhs(), mlir::m_Constant()))
-    return rewriter.notifyMatchFailure(op.getLoc(), "RHS of LHS op is not a constant");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "RHS of LHS op is not a constant");
 
   Value c = rewriter.createOrFold<BinaryOp>(op->getLoc(), op.getRhs(),
-                                           lhsOp.getRhs());
+                                            lhsOp.getRhs());
   if (c.getDefiningOp<BinaryOp>())
-    return rewriter.notifyMatchFailure(op.getLoc(), "new BinaryOp was not folded");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "new BinaryOp was not folded");
 
   rewriter.replaceOpWithNewOp<BinaryOp>(op, lhsOp.getLhs(), c);
   return success();
@@ -716,10 +720,10 @@ LogicalResult CmpOp::canonicalize(CmpOp op, PatternRewriter &rewriter) {
   index::CmpOp newCmp;
   if (rhsIsZero)
     newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
-                                           subOp.getLhs(), subOp.getRhs());
+                                  subOp.getLhs(), subOp.getRhs());
   else
     newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
-                                           subOp.getRhs(), subOp.getLhs());
+                                  subOp.getRhs(), subOp.getLhs());
   rewriter.replaceOp(op, newCmp);
   return success();
 }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 884ae4a2eac64..fbe8468548d61 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
@@ -4304,8 +4305,8 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
   MLIRContext *ctx = builder.getContext();
   auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
-  auto global = LLVM::GlobalOp::create(moduleBuilder,
-      loc, type, /*isConstant=*/true, linkage, name,
+  auto global = LLVM::GlobalOp::create(
+      moduleBuilder, loc, type, /*isConstant=*/true, linkage, name,
       builder.getStringAttr(value), /*alignment=*/0);
 
   LLVMPointerType ptrType = LLVMPointerType::get(ctx);
@@ -4313,7 +4314,7 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   Value globalPtr =
       LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
   return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
-                                     ArrayRef<GEPArg>{0, 0});
+                             ArrayRef<GEPArg>{0, 0});
 }
 
 bool mlir::LLVM::satisfiesLLVMModule(Operation *op) {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 6e29b129e8835..d5713e045fa71 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -24,6 +24,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
index 1a9ccf56008b8..a39457068f319 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
index bd9d3528ceb74..b1676b4e58c39 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
index 9e497829ba723..fbedc714c200b 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp b/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
index 37ddca101f64b..877f8e72caa31 100644
--- a/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
+++ b/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/MLProgram/IR/MLProgram.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index 749cca0575581..da2317cd4eb97 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 
 using namespace mlir;
@@ -60,8 +61,8 @@ struct FoldRank final : public mlir::OpRewritePattern<mlir::mpi::CommRankOp> {
     if (!isa<IntegerAttr>(dltiAttr.value()))
       return op->emitError()
              << "Expected an integer attribute for MPI:comm_world_rank";
-    Value res = arith::ConstantIndexOp::create(b,
-        op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
+    Value res = arith::ConstantIndexOp::create(
+        b, op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
     if (Value retVal = op.getRetval())
       b.replaceOp(op, {retVal, res});
     else
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index a21631cbf8510..b09a67818fff6 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 706ac4b4867f4..cb5fe132494b3 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -213,9 +214,9 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
 
     // Create and insert the alloc op for the new memref.
-    auto newAlloc = AllocLikeOp::create(rewriter,
-        alloc.getLoc(), newMemRefType, dynamicSizes, alloc.getSymbolOperands(),
-        alloc.getAlignmentAttr());
+    auto newAlloc = AllocLikeOp::create(rewriter, alloc.getLoc(), newMemRefType,
+                                        dynamicSizes, alloc.getSymbolOperands(),
+                                        alloc.getAlignmentAttr());
     // Insert a cast so we have the same type as the old alloc.
     rewriter.replaceOpWithNewOp<CastOp>(alloc, alloc.getType(), newAlloc);
     return success();
@@ -1358,8 +1359,9 @@ static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
     assert(isa<Attribute>(maybeConstant) &&
            "The constified value should be either unchanged (i.e., == result) "
            "or a constant");
-    Value constantVal = arith::ConstantIndexOp::create(rewriter,
-        loc, llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
+    Value constantVal = arith::ConstantIndexOp::create(
+        rewriter, loc,
+        llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
     for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
       // modifyOpInPlace: lambda cannot capture structured bindings in C++17
       // yet.
@@ -2587,8 +2589,9 @@ struct CollapseShapeOpMemRefCastFolder
       rewriter.modifyOpInPlace(
           op, [&]() { op.getSrcMutable().assign(cast.getSource()); });
     } else {
-      Value newOp = CollapseShapeOp::create(rewriter,
-          op->getLoc(), cast.getSource(), op.getReassociationIndices());
+      Value newOp =
+          CollapseShapeOp::create(rewriter, op->getLoc(), cast.getSource(),
+                                  op.getReassociationIndices());
       rewriter.replaceOpWithNewOp<CastOp>(op, op.getType(), newOp);
     }
     return success();
@@ -3212,8 +3215,8 @@ class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
     if (!resultType)
       return failure();
 
-    Value newSubView = SubViewOp::create(rewriter,
-        subViewOp.getLoc(), resultType, castOp.getSource(),
+    Value newSubView = SubViewOp::create(
+        rewriter, subViewOp.getLoc(), resultType, castOp.getSource(),
         subViewOp.getOffsets(), subViewOp.getSizes(), subViewOp.getStrides(),
         subViewOp.getStaticOffsets(), subViewOp.getStaticSizes(),
         subViewOp.getStaticStrides());
@@ -3534,9 +3537,9 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
       return failure();
 
     // Create new ViewOp.
-    auto newViewOp = ViewOp::create(rewriter,
-        viewOp.getLoc(), newMemRefType, viewOp.getOperand(0),
-        viewOp.getByteShift(), newOperands);
+    auto newViewOp = ViewOp::create(rewriter, viewOp.getLoc(), newMemRefType,
+                                    viewOp.getOperand(0), viewOp.getByteShift(),
+                                    newOperands);
     // Insert a cast so we have the same type as the old memref type.
     rewriter.replaceOpWithNewOp<CastOp>(viewOp, viewOp.getType(), newViewOp);
     return success();
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 5c556308b1e9f..c583c64f23bc0 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -23,6 +23,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "llvm/Support/Debug.h"
 
@@ -155,9 +156,10 @@ transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
       Type resultType = alloca.getResult().getType();
       OpBuilder builder(rewriter.getContext());
       // TODO: Add a better builder for this.
-      globalOp = memref::GlobalOp::create(builder,
-          loc, StringAttr::get(ctx, "alloca"), StringAttr::get(ctx, "private"),
-          TypeAttr::get(resultType), Attribute{}, UnitAttr{}, IntegerAttr{});
+      globalOp = memref::GlobalOp::create(
+          builder, loc, StringAttr::get(ctx, "alloca"),
+          StringAttr::get(ctx, "private"), TypeAttr::get(resultType),
+          Attribute{}, UnitAttr{}, IntegerAttr{});
       symbolTable.insert(globalOp);
     }
 
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index a440749dbd703..f468e9f6f4b4d 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -317,9 +318,9 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
   if (!newShardOp) {
     auto shardingOp =
         ShardingOp::create(builder, operandValue.getLoc(), sharding);
-    newShardOp =
-        ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                                /*annotate_for_users*/ false);
+    newShardOp = ShardOp::create(builder, operandValue.getLoc(), operandValue,
+                                 shardingOp,
+                                 /*annotate_for_users*/ false);
   }
   operandValue.replaceUsesWithIf(
       newShardOp, [operandOp, operandValue](OpOperand &use) {
@@ -331,8 +332,8 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
   }
 
   auto newShardOp2 = ShardOp::create(builder, operandValue.getLoc(), newShardOp,
-                                             newShardOp.getSharding(),
-                                             /*annotate_for_users*/ true);
+                                     newShardOp.getSharding(),
+                                     /*annotate_for_users*/ true);
   newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
 }
 
@@ -381,7 +382,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
       ShardingOp::create(builder, operand.get().getLoc(), sharding);
   auto newShardOp =
       ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                              /*annotate_for_users*/ true);
+                      /*annotate_for_users*/ true);
   IRRewriter rewriter(builder);
   rewriter.replaceUsesWithIf(
       operandValue, newShardOp, [operandOp, operandValue](OpOperand &use) {
@@ -396,7 +397,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
   builder.setInsertionPoint(newShardOp);
   auto newPreceedingShardOp =
       ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                              /*annotate_for_users*/ false);
+                      /*annotate_for_users*/ false);
   rewriter.replaceUsesWithIf(
       newShardOp.getSrc(), newPreceedingShardOp, [&newShardOp](OpOperand &use) {
         return use.getOwner() == newShardOp.getOperation();
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..69a318f4af79b 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 9bea8b7a732a8..056b73f52cde8 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/SymbolTable.h"
diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp
index d3f7c9798b9b8..ffe53bbe80607 100644
--- a/mlir/lib/Dialect/PDL/IR/PDL.cpp
+++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
index 9b1f11d835282..5cc84ab29e246 100644
--- a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
+++ b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
index c488144508128..3695262155387 100644
--- a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
+++ b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/Dialect/Ptr/IR/PtrOps.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
diff --git a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
index e23a0d6aba825..cd4198b431669 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Quant/IR/Quant.h"
 #include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 82fa50a6ac6e0..8ead566703ab4 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
@@ -247,7 +248,7 @@ struct MultiBlockExecuteInliner : public OpRewritePattern<ExecuteRegionOp> {
       if (YieldOp yieldOp = dyn_cast<YieldOp>(blk.getTerminator())) {
         rewriter.setInsertionPoint(yieldOp);
         cf::BranchOp::create(rewriter, yieldOp.getLoc(), postBlock,
-                                      yieldOp.getResults());
+                             yieldOp.getResults());
         rewriter.eraseOp(yieldOp);
       }
     }
@@ -557,8 +558,8 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  scf::ForOp newLoop = scf::ForOp::create(rewriter,
-      getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
+  scf::ForOp newLoop = scf::ForOp::create(
+      rewriter, getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
       [](OpBuilder &, Location, Value, ValueRange) {});
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
 
@@ -673,8 +674,8 @@ void mlir::scf::promote(RewriterBase &rewriter, scf::ForallOp forallOp) {
     Value dst = parallelInsertSliceOp.getDest();
     Value src = parallelInsertSliceOp.getSource();
     if (llvm::isa<TensorType>(src.getType())) {
-      results.push_back(tensor::InsertSliceOp::create(rewriter,
-          forallOp.getLoc(), dst.getType(), src, dst,
+      results.push_back(tensor::InsertSliceOp::create(
+          rewriter, forallOp.getLoc(), dst.getType(), src, dst,
           parallelInsertSliceOp.getOffsets(), parallelInsertSliceOp.getSizes(),
           parallelInsertSliceOp.getStrides(),
           parallelInsertSliceOp.getStaticOffsets(),
@@ -722,8 +723,8 @@ LoopNest mlir::scf::buildLoopNest(
   ValueRange currentIterArgs = iterArgs;
   Location currentLoc = loc;
   for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
-    auto loop = scf::ForOp::create(builder,
-        currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
+    auto loop = scf::ForOp::create(
+        builder, currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
             ValueRange args) {
           ivs.push_back(iv);
@@ -801,8 +802,8 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   }
 
   // 2. Create the new forOp shell.
-  scf::ForOp newForOp = scf::ForOp::create(rewriter,
-      forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
+  scf::ForOp newForOp = scf::ForOp::create(
+      rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
       forOp.getStep(), newIterOperands);
   newForOp->setAttrs(forOp->getAttrs());
   Block &newBlock = newForOp.getRegion().front();
@@ -926,9 +927,9 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     if (!canonicalize)
       return failure();
 
-    scf::ForOp newForOp = scf::ForOp::create(rewriter,
-        forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-        forOp.getStep(), newIterArgs);
+    scf::ForOp newForOp =
+        scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(),
+                           forOp.getUpperBound(), forOp.getStep(), newIterArgs);
     newForOp->setAttrs(forOp->getAttrs());
     Block &newBlock = newForOp.getRegion().front();
 
@@ -971,7 +972,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
         if (keepMask[idx])
           filteredOperands.push_back(mergedTerminator.getOperand(idx));
       scf::YieldOp::create(rewriter, mergedTerminator.getLoc(),
-                                    filteredOperands);
+                           filteredOperands);
     };
 
     rewriter.mergeBlocks(&oldBlock, &newBlock, newBlockTransferArgs);
@@ -1685,8 +1686,8 @@ struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
 
     // Step 3. Create a new scf.forall op with the new shared_outs' operands
     //         fetched earlier
-    auto newForallOp = scf::ForallOp::create(rewriter,
-        forallOp.getLoc(), forallOp.getMixedLowerBound(),
+    auto newForallOp = scf::ForallOp::create(
+        rewriter, forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
         forallOp.getMapping(),
         /*bodyBuilderFn =*/[](OpBuilder &, Location, ValueRange) {});
@@ -1783,8 +1784,8 @@ struct ForallOpSingleOrZeroIterationDimsFolder
     // Replace the loop by a lower-dimensional loop.
     ForallOp newOp;
     newOp = ForallOp::create(rewriter, loc, newMixedLowerBounds,
-                                      newMixedUpperBounds, newMixedSteps,
-                                      op.getOutputs(), std::nullopt, nullptr);
+                             newMixedUpperBounds, newMixedSteps,
+                             op.getOutputs(), std::nullopt, nullptr);
     newOp.getBodyRegion().getBlocks().clear();
     // The new loop needs to keep all attributes from the old one, except for
     // "operandSegmentSizes" and static loop bound attributes which capture
@@ -1867,16 +1868,17 @@ struct FoldTensorCastOfOutputIntoForallOp
 
     // Create new loop.
     Location loc = forallOp.getLoc();
-    auto newForallOp = ForallOp::create(rewriter,
-        loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
-        forallOp.getMixedStep(), newOutputTensors, forallOp.getMapping(),
+    auto newForallOp = ForallOp::create(
+        rewriter, loc, forallOp.getMixedLowerBound(),
+        forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
+        newOutputTensors, forallOp.getMapping(),
         [&](OpBuilder nestedBuilder, Location nestedLoc, ValueRange bbArgs) {
           auto castBlockArgs =
               llvm::to_vector(bbArgs.take_back(forallOp->getNumResults()));
           for (auto [index, cast] : tensorCastProducers) {
             Value &oldTypeBBArg = castBlockArgs[index];
-            oldTypeBBArg = tensor::CastOp::create(nestedBuilder,
-                nestedLoc, cast.dstType, oldTypeBBArg);
+            oldTypeBBArg = tensor::CastOp::create(nestedBuilder, nestedLoc,
+                                                  cast.dstType, oldTypeBBArg);
           }
 
           // Move old body into new parallel loop.
@@ -1903,7 +1905,7 @@ struct FoldTensorCastOfOutputIntoForallOp
     for (auto &item : tensorCastProducers) {
       Value &oldTypeResult = castResults[item.first];
       oldTypeResult = tensor::CastOp::create(rewriter, loc, item.second.dstType,
-                                                      oldTypeResult);
+                                             oldTypeResult);
     }
     rewriter.replaceOp(forallOp, castResults);
     return success();
@@ -2373,7 +2375,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       return failure();
 
     IfOp replacement = IfOp::create(rewriter, op.getLoc(), nonHoistable, cond,
-                                             /*withElseRegion=*/false);
+                                    /*withElseRegion=*/false);
     if (replacement.thenBlock())
       rewriter.eraseBlock(replacement.thenBlock());
     replacement.getThenRegion().takeBody(op.getThenRegion());
@@ -2398,8 +2400,8 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       } else if (trueVal == falseVal)
         results[it.index()] = trueVal;
       else
-        results[it.index()] = arith::SelectOp::create(rewriter,
-            op.getLoc(), cond, trueVal, falseVal);
+        results[it.index()] = arith::SelectOp::create(rewriter, op.getLoc(),
+                                                      cond, trueVal, falseVal);
     }
 
     rewriter.setInsertionPointToEnd(replacement.thenBlock());
@@ -2450,8 +2452,8 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantTrue)
-          constantTrue = arith::ConstantOp::create(rewriter,
-              op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
+          constantTrue = arith::ConstantOp::create(
+              rewriter, op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
 
         rewriter.modifyOpInPlace(use.getOwner(),
                                  [&]() { use.set(constantTrue); });
@@ -2460,8 +2462,8 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantFalse)
-          constantFalse = arith::ConstantOp::create(rewriter,
-              op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
+          constantFalse = arith::ConstantOp::create(
+              rewriter, op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
 
         rewriter.modifyOpInPlace(use.getOwner(),
                                  [&]() { use.set(constantFalse); });
@@ -2547,8 +2549,8 @@ struct ReplaceIfYieldWithConditionOrValue : public OpRewritePattern<IfOp> {
       if (!trueVal && falseVal) {
         if (!opResult.use_empty()) {
           Dialect *constDialect = trueResult.getDefiningOp()->getDialect();
-          Value notCond = arith::XOrIOp::create(rewriter,
-              op.getLoc(), op.getCondition(),
+          Value notCond = arith::XOrIOp::create(
+              rewriter, op.getLoc(), op.getCondition(),
               constDialect
                   ->materializeConstant(rewriter,
                                         rewriter.getIntegerAttr(i1Ty, 1), i1Ty,
@@ -2661,8 +2663,8 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
     SmallVector<Type> mergedTypes(prevIf.getResultTypes());
     llvm::append_range(mergedTypes, nextIf.getResultTypes());
 
-    IfOp combinedIf = IfOp::create(rewriter,
-        nextIf.getLoc(), mergedTypes, prevIf.getCondition(), /*hasElse=*/false);
+    IfOp combinedIf = IfOp::create(rewriter, nextIf.getLoc(), mergedTypes,
+                                   prevIf.getCondition(), /*hasElse=*/false);
     rewriter.eraseBlock(&combinedIf.getThenRegion().back());
 
     rewriter.inlineRegionBefore(prevIf.getThenRegion(),
@@ -2823,8 +2825,8 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     }
 
     Location loc = op.getLoc();
-    Value newCondition = arith::AndIOp::create(rewriter,
-        loc, op.getCondition(), nestedIf.getCondition());
+    Value newCondition = arith::AndIOp::create(rewriter, loc, op.getCondition(),
+                                               nestedIf.getCondition());
     auto newIf = IfOp::create(rewriter, loc, op.getResultTypes(), newCondition);
     Block *newIfBlock = rewriter.createBlock(&newIf.getThenRegion());
 
@@ -2833,8 +2835,9 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     rewriter.setInsertionPoint(newIf);
 
     for (auto idx : elseYieldsToUpgradeToSelect)
-      results[idx] = arith::SelectOp::create(rewriter,
-          op.getLoc(), op.getCondition(), thenYield[idx], elseYield[idx]);
+      results[idx] =
+          arith::SelectOp::create(rewriter, op.getLoc(), op.getCondition(),
+                                  thenYield[idx], elseYield[idx]);
 
     rewriter.mergeBlocks(nestedIf.thenBlock(), newIfBlock);
     rewriter.setInsertionPointToEnd(newIf.thenBlock());
@@ -3160,8 +3163,8 @@ struct ParallelOpSingleOrZeroIterationDimsFolder
     }
     // Replace the parallel loop by lower-dimensional parallel loop.
     auto newOp =
-        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds, newUpperBounds,
-                                    newSteps, op.getInitVals(), nullptr);
+        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds,
+                           newUpperBounds, newSteps, op.getInitVals(), nullptr);
     // Erase the empty block that was inserted by the builder.
     rewriter.eraseBlock(newOp.getBody());
     // Clone the loop body and remap the block arguments of the collapsed loops
@@ -3541,8 +3544,8 @@ struct WhileConditionTruth : public OpRewritePattern<WhileOp> {
       if (std::get<0>(yieldedAndBlockArgs) == term.getCondition()) {
         if (!std::get<1>(yieldedAndBlockArgs).use_empty()) {
           if (!constantTrue)
-            constantTrue = arith::ConstantOp::create(rewriter,
-                op.getLoc(), term.getCondition().getType(),
+            constantTrue = arith::ConstantOp::create(
+                rewriter, op.getLoc(), term.getCondition().getType(),
                 rewriter.getBoolAttr(true));
 
           rewriter.replaceAllUsesWith(std::get<1>(yieldedAndBlockArgs),
@@ -3684,8 +3687,8 @@ struct RemoveLoopInvariantArgsFromBeforeBlock
       rewriter.replaceOpWithNewOp<YieldOp>(yieldOp, newYieldOpArgs);
     }
 
-    auto newWhile =
-        WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(), newInitArgs);
+    auto newWhile = WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(),
+                                    newInitArgs);
 
     Block &newBeforeBlock = *rewriter.createBlock(
         &newWhile.getBefore(), /*insertPt*/ {},
@@ -3808,7 +3811,7 @@ struct RemoveLoopInvariantValueYielded : public OpRewritePattern<WhileOp> {
     }
 
     auto newWhile = WhileOp::create(rewriter, op.getLoc(), newAfterBlockType,
-                                             op.getOperands());
+                                    op.getOperands());
 
     Block &newAfterBlock =
         *rewriter.createBlock(&newWhile.getAfter(), /*insertPt*/ {},
@@ -4044,7 +4047,7 @@ struct WhileRemoveUnusedArgs : public OpRewritePattern<WhileOp> {
     Location loc = op.getLoc();
     auto newWhileOp =
         WhileOp::create(rewriter, loc, op.getResultTypes(), newInits,
-                                 /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
+                        /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
 
@@ -4091,9 +4094,10 @@ struct WhileRemoveDuplicatedResults : public OpRewritePattern<WhileOp> {
     ValueRange argsRange(newArgs);
 
     Location loc = op.getLoc();
-    auto newWhileOp = scf::WhileOp::create(rewriter,
-        loc, argsRange.getTypes(), op.getInits(), /*beforeBody*/ nullptr,
-        /*afterBody*/ nullptr);
+    auto newWhileOp =
+        scf::WhileOp::create(rewriter, loc, argsRange.getTypes(), op.getInits(),
+                             /*beforeBody*/ nullptr,
+                             /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
 
@@ -4187,8 +4191,8 @@ struct WhileOpAlignBeforeArgs : public OpRewritePattern<WhileOp> {
     for (auto &&[i, j] : llvm::enumerate(*mapping))
       newResultTypes[j] = loop.getResult(i).getType();
 
-    auto newLoop = WhileOp::create(rewriter,
-        loop.getLoc(), newResultTypes, loop.getInits(),
+    auto newLoop = WhileOp::create(
+        rewriter, loop.getLoc(), newResultTypes, loop.getInits(),
         /*beforeBuilder=*/nullptr, /*afterBuilder=*/nullptr);
     auto newBefore = newLoop.getBeforeBody();
     auto newAfter = newLoop.getAfterBody();
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index e9943a3489d46..145afac6df15e 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -25,6 +25,7 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
index c517ef2b18f6f..6af87afd8b83e 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/SMT/IR/SMTOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/APSInt.h"
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
index d8dfe164458e2..8ba2af581e00d 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 #include "SPIRVParsingUtils.h"
 
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 7339ba5c2379c..19dba20f43fa9 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -154,14 +155,14 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
     return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (llvm::isa<ShapeType>(type) || isExtentTensorType(type))
-    return ConstShapeOp::create(builder,
-        loc, type, llvm::cast<DenseIntElementsAttr>(value));
+    return ConstShapeOp::create(builder, loc, type,
+                                llvm::cast<DenseIntElementsAttr>(value));
   if (llvm::isa<SizeType>(type))
     return ConstSizeOp::create(builder, loc, type,
-                                       llvm::cast<IntegerAttr>(value));
+                               llvm::cast<IntegerAttr>(value));
   if (llvm::isa<WitnessType>(type))
     return ConstWitnessOp::create(builder, loc, type,
-                                          llvm::cast<BoolAttr>(value));
+                                  llvm::cast<BoolAttr>(value));
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
@@ -316,8 +317,8 @@ struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
     auto newYieldOp =
         rewriter.replaceOpWithNewOp<AssumingYieldOp>(yieldOp, newYieldOperands);
     rewriter.setInsertionPoint(op);
-    auto newOp = AssumingOp::create(rewriter,
-        op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
+    auto newOp = AssumingOp::create(
+        rewriter, op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
     newOp.getDoRegion().takeBody(op.getDoRegion());
 
     // Use the new results to replace the previously used ones.
@@ -780,9 +781,9 @@ struct BroadcastFoldConstantOperandsPattern
     auto foldedConstantOperandsTy = RankedTensorType::get(
         {static_cast<int64_t>(foldedConstantShape.size())},
         rewriter.getIndexType());
-    newShapeOperands.push_back(ConstShapeOp::create(rewriter,
-        op.getLoc(), foldedConstantOperandsTy,
-        rewriter.getIndexTensorAttr(foldedConstantShape)));
+    newShapeOperands.push_back(
+        ConstShapeOp::create(rewriter, op.getLoc(), foldedConstantOperandsTy,
+                             rewriter.getIndexTensorAttr(foldedConstantShape)));
     rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(),
                                              newShapeOperands);
     return success();
@@ -845,9 +846,9 @@ struct BroadcastConcretizeResultTypePattern
       }
     }
 
-    auto newOp = BroadcastOp::create(rewriter,
-        op.getLoc(), getExtentTensorType(getContext(), maxRank),
-        op.getShapes());
+    auto newOp = BroadcastOp::create(rewriter, op.getLoc(),
+                                     getExtentTensorType(getContext(), maxRank),
+                                     op.getShapes());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -1338,7 +1339,8 @@ std::optional<int64_t> GetExtentOp::getConstantDim() {
 }
 
 OpFoldResult GetExtentOp::fold(FoldAdaptor adaptor) {
-  auto elements = llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
+  auto elements =
+      llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
   if (!elements)
     return nullptr;
   std::optional<int64_t> dim = getConstantDim();
@@ -1357,8 +1359,8 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
     Value dim = ConstSizeOp::create(builder, loc, dimAttr);
     build(builder, result, builder.getType<SizeType>(), shape, dim);
   } else {
-    Value dim =
-        arith::ConstantOp::create(builder, loc, builder.getIndexType(), dimAttr);
+    Value dim = arith::ConstantOp::create(builder, loc, builder.getIndexType(),
+                                          dimAttr);
     build(builder, result, builder.getIndexType(), shape, dim);
   }
 }
@@ -1480,7 +1482,8 @@ bool mlir::shape::MeetOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
 //===----------------------------------------------------------------------===//
 
 OpFoldResult shape::RankOp::fold(FoldAdaptor adaptor) {
-  auto shape = llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
+  auto shape =
+      llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
   if (!shape)
     return {};
   int64_t rank = shape.getNumElements();
@@ -1708,8 +1711,8 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
                                   rewriter.getIndexTensorAttr(type.getShape()))
             .getResult();
     if (constShape.getType() != op.getResult().getType())
-      constShape = tensor::CastOp::create(rewriter,
-          loc, op.getResult().getType(), constShape);
+      constShape = tensor::CastOp::create(rewriter, loc,
+                                          op.getResult().getType(), constShape);
     rewriter.replaceOp(op, constShape);
     return success();
   }
@@ -1717,13 +1720,13 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
 
 // Canonicalize
 //
-// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
-// %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
+// tensor<*xf32> %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
 //
 // to
 //
-// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
-// %1 = %shape
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
+// tensor<*xf32> %1 = %shape
 //
 struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
   using OpRewritePattern<shape::ShapeOfOp>::OpRewritePattern;
@@ -1751,10 +1754,11 @@ struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
 
     if (opTensorTy != shapeTensorTy) {
       if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
-        shape = tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
-      else if (!isExtentTensorType(shapeTensorTy))
         shape =
-            arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
+            tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
+      else if (!isExtentTensorType(shapeTensorTy))
+        shape = arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy,
+                                           shape);
     }
 
     rewriter.replaceOp(op, shape);
@@ -1895,8 +1899,9 @@ LogicalResult SplitAtOp::fold(FoldAdaptor adaptor,
                               SmallVectorImpl<OpFoldResult> &results) {
   if (!adaptor.getOperand() || !adaptor.getIndex())
     return failure();
-  auto shapeVec = llvm::to_vector<6>(
-      llvm::cast<DenseIntElementsAttr>(adaptor.getOperand()).getValues<int64_t>());
+  auto shapeVec =
+      llvm::to_vector<6>(llvm::cast<DenseIntElementsAttr>(adaptor.getOperand())
+                             .getValues<int64_t>());
   auto shape = llvm::ArrayRef(shapeVec);
   auto splitPoint = llvm::cast<IntegerAttr>(adaptor.getIndex()).getInt();
   // Verify that the split point is in the correct range.
@@ -1920,8 +1925,9 @@ OpFoldResult ToExtentTensorOp::fold(FoldAdaptor adaptor) {
   if (!adaptor.getInput())
     return OpFoldResult();
   Builder builder(getContext());
-  auto shape = llvm::to_vector<6>(
-      llvm::cast<DenseIntElementsAttr>(adaptor.getInput()).getValues<int64_t>());
+  auto shape =
+      llvm::to_vector<6>(llvm::cast<DenseIntElementsAttr>(adaptor.getInput())
+                             .getValues<int64_t>());
   auto type = RankedTensorType::get({static_cast<int64_t>(shape.size())},
                                     builder.getIndexType());
   return DenseIntElementsAttr::get(type, shape);
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index e2a6a59636a84..1a9d9e158ee75 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -559,7 +559,8 @@ SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
   SmallVector<Type> retType(
       dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
       builder.getIndexType());
-  auto transOp = CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
+  auto transOp =
+      CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
   return transOp.getOutCrds();
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp b/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
index bdec43825ddc2..33249368f7ed3 100644
--- a/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
+++ b/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h"
 #include "mlir/Dialect/Linalg/TransformOps/Syntax.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 using namespace mlir::sparse_tensor;
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index df34d9e37bd3f..1884d51de5411 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinTypeInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -54,7 +55,7 @@ Operation *TensorDialect::materializeConstant(OpBuilder &builder,
     return op;
   if (complex::ConstantOp::isBuildableWith(value, type))
     return complex::ConstantOp::create(builder, loc, type,
-                                               llvm::cast<ArrayAttr>(value));
+                                       llvm::cast<ArrayAttr>(value));
   return nullptr;
 }
 
@@ -678,8 +679,8 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
     inputShapes.emplace_back(std::move(inputShape));
   }
 
-  Value replacement = tensor::EmptyOp::create(builder,
-      loc, outputShape, getType().getElementType());
+  Value replacement = tensor::EmptyOp::create(builder, loc, outputShape,
+                                              getType().getElementType());
 
   int64_t rank = getType().getRank();
   OpFoldResult one = builder.getIndexAttr(1);
@@ -687,8 +688,8 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
   SmallVector<OpFoldResult> offsets(rank, zero);
   for (auto [index, input] : llvm::enumerate(getInputs())) {
     offsets[concatDim] = concatOffsets[index];
-    auto insertSlice = tensor::InsertSliceOp::create(builder,
-        loc, input, replacement, offsets, inputShapes[index], strides);
+    auto insertSlice = tensor::InsertSliceOp::create(
+        builder, loc, input, replacement, offsets, inputShapes[index], strides);
     replacement = insertSlice.getResult();
   }
   if (replacement.getType() != getType()) {
@@ -824,7 +825,7 @@ struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
         // Use refined operand type and create cast from original operand.
         auto castOp =
             CastOp::create(rewriter, concatOp->getLoc(), inferredOperandType,
-                                    concatOp.getOperand(operandIdx));
+                           concatOp.getOperand(operandIdx));
         rewriter.modifyOpInPlace(concatOp, [=, operandIdx = operandIdx] {
           concatOp->setOperand(operandIdx, castOp->getResult(0));
         });
@@ -864,8 +865,9 @@ struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
       return failure();
     }
 
-    auto newConcatOp = ConcatOp::create(rewriter,
-        concatOp->getLoc(), inferredResultType, dim, concatOp->getOperands());
+    auto newConcatOp =
+        ConcatOp::create(rewriter, concatOp->getLoc(), inferredResultType, dim,
+                         concatOp->getOperands());
     rewriter.replaceOpWithNewOp<CastOp>(concatOp, concatOp.getResultType(),
                                         newConcatOp);
 
@@ -1151,7 +1153,7 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
       return failure();
 
     auto newOp = EmptyOp::create(rewriter, op.getLoc(), foldedTensorType,
-                                          foldedDynamicSizes);
+                                 foldedDynamicSizes);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -1326,8 +1328,8 @@ struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
 
       SmallVector<int64_t> basis =
           llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-      auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-          extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
+      auto delinearize = affine::AffineDelinearizeIndexOp::create(
+          rewriter, extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
       llvm::append_range(sourceIndices, delinearize.getResults());
     }
     if (collapseOp.getReassociationIndices().empty()) {
@@ -1498,8 +1500,8 @@ struct ExtractElementFromIndexCast
 
     Type elementTy = getElementTypeOrSelf(indexCast.getIn());
 
-    auto newExtract = tensor::ExtractOp::create(rewriter,
-        loc, elementTy, indexCast.getIn(), extract.getIndices());
+    auto newExtract = tensor::ExtractOp::create(
+        rewriter, loc, elementTy, indexCast.getIn(), extract.getIndices());
 
     rewriter.replaceOpWithNewOp<arith::IndexCastOp>(extract, extract.getType(),
                                                     newExtract);
@@ -2161,9 +2163,9 @@ struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
         collapseShapeOp.getSrcMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = CollapseShapeOp::create(rewriter,
-          collapseShapeOp.getLoc(), newResultType, castOp.getSource(),
-          collapseShapeOp.getReassociation());
+      auto newOp = CollapseShapeOp::create(rewriter, collapseShapeOp.getLoc(),
+                                           newResultType, castOp.getSource(),
+                                           collapseShapeOp.getReassociation());
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
           collapseShapeOp, collapseShapeOp.getResultType(), newOp);
     }
@@ -2241,9 +2243,9 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
     auto outputType = RankedTensorType::get(
         newOutputShape, expandOp.getSrcType().getElementType());
     auto inputCast = CastOp::create(rewriter, expandOp.getLoc(), inputType,
-                                             expandOp.getSrc());
-    auto newExpand = ExpandShapeOp::create(rewriter,
-        expandOp.getLoc(), outputType, inputCast.getResult(),
+                                    expandOp.getSrc());
+    auto newExpand = ExpandShapeOp::create(
+        rewriter, expandOp.getLoc(), outputType, inputCast.getResult(),
         expandOp.getReassociationIndices(), outputOfr);
     rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
                                         newExpand.getResult());
@@ -2555,10 +2557,11 @@ class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
 
     // Create folded extract.
     Location loc = sliceOp.getLoc();
-    Value newResult = ExtractSliceOp::create(rewriter,
-        loc, sliceOp.getType(), castOp.getSource(), sliceOp.getOffsets(),
-        sliceOp.getSizes(), sliceOp.getStrides(), sliceOp.getStaticOffsets(),
-        sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
+    Value newResult = ExtractSliceOp::create(
+        rewriter, loc, sliceOp.getType(), castOp.getSource(),
+        sliceOp.getOffsets(), sliceOp.getSizes(), sliceOp.getStrides(),
+        sliceOp.getStaticOffsets(), sliceOp.getStaticSizes(),
+        sliceOp.getStaticStrides());
     rewriter.replaceOp(sliceOp, newResult);
     return success();
   }
@@ -2710,7 +2713,7 @@ struct SliceCanonicalizer {
     Value replacement = newOp.getResult();
     if (replacement.getType() != op.getType())
       replacement = tensor::CastOp::create(rewriter, op.getLoc(), op.getType(),
-                                                    replacement);
+                                           replacement);
     rewriter.replaceOp(op, replacement);
   }
 };
@@ -2979,7 +2982,7 @@ class InsertSliceOpConstantArgumentFolder final
       if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
       toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
-                                                 sourceType, toInsert);
+                                        sourceType, toInsert);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, toInsert, insertSliceOp.getDest(), mixedOffsets,
@@ -3075,17 +3078,18 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     if (!sliceResult.isValid)
       return failure();
 
-    Operation *replacement = InsertOpTy::create(rewriter,
-        insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
-        mixedSizes, insertSliceOp.getMixedStrides());
+    Operation *replacement =
+        InsertOpTy::create(rewriter, insertSliceOp.getLoc(), src, dst,
+                           insertSliceOp.getMixedOffsets(), mixedSizes,
+                           insertSliceOp.getMixedStrides());
 
     // In the parallel case there is no result and so nothing to cast.
     bool isParallelInsert =
         std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
     if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
       replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
-                                                    insertSliceOp.getDestType(),
-                                                    replacement->getResult(0));
+                                           insertSliceOp.getDestType(),
+                                           replacement->getResult(0));
     }
     rewriter.replaceOp(insertSliceOp, replacement->getResults());
     return success();
@@ -3154,8 +3158,8 @@ struct InsertSliceOpSourceCastInserter final
     // parallel case.
     if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
       rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-    Value cast = tensor::CastOp::create(rewriter,
-        insertSliceOp.getLoc(), newSrcType, insertSliceOp.getSource());
+    Value cast = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
+                                        newSrcType, insertSliceOp.getSource());
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, cast, insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
@@ -3407,10 +3411,11 @@ struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
         padTensorOp.getSourceMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = PadOp::create(rewriter,
-          padTensorOp->getLoc(), newResultType, padTensorOp.getSource(),
-          padTensorOp.getStaticLow(), padTensorOp.getStaticHigh(),
-          padTensorOp.getLow(), padTensorOp.getHigh(), padTensorOp.getNofold(),
+      auto newOp = PadOp::create(
+          rewriter, padTensorOp->getLoc(), newResultType,
+          padTensorOp.getSource(), padTensorOp.getStaticLow(),
+          padTensorOp.getStaticHigh(), padTensorOp.getLow(),
+          padTensorOp.getHigh(), padTensorOp.getNofold(),
           getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
       IRMapping mapper;
       padTensorOp.getRegion().cloneInto(&newOp.getRegion(), mapper);
@@ -3439,8 +3444,8 @@ struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
                                             tensorCastOp.getDest().getType()))
       return failure();
 
-    auto replacementOp = PadOp::create(rewriter,
-        padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
+    auto replacementOp = PadOp::create(
+        rewriter, padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
         padTensorOp.getSource(), padTensorOp.getStaticLow(),
         padTensorOp.getStaticHigh(), padTensorOp.getLow(),
         padTensorOp.getHigh(), padTensorOp.getNofold(),
@@ -3597,11 +3602,11 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
 
     // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
     // the two paddings in one step.
-    auto newSliceOp = ExtractSliceOp::create(rewriter,
-        padOp.getLoc(), outerSliceOp.getSource(), newOffsets, newSizes,
-        innerSliceOp.getMixedStrides());
-    auto newPadOp = PadOp::create(rewriter,
-        padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
+    auto newSliceOp = ExtractSliceOp::create(
+        rewriter, padOp.getLoc(), outerSliceOp.getSource(), newOffsets,
+        newSizes, innerSliceOp.getMixedStrides());
+    auto newPadOp = PadOp::create(
+        rewriter, padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
         padOp.getMixedLowPad(), newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
     rewriter.inlineRegionBefore(padOp.getRegion(), newPadOp.getRegion(),
@@ -3697,9 +3702,9 @@ struct FoldStaticPadding : public OpRewritePattern<PadOp> {
     // Rewrite the op using the new static type.
     auto newResultType = RankedTensorType::get(
         newOutDims, padTensorOp.getType().getElementType());
-    auto newOp = PadOp::create(rewriter,
-        padTensorOp->getLoc(), newResultType, input, staticLow, staticHigh,
-        newLows, newHighs, padTensorOp.getNofold(),
+    auto newOp = PadOp::create(
+        rewriter, padTensorOp->getLoc(), newResultType, input, staticLow,
+        staticHigh, newLows, newHighs, padTensorOp.getNofold(),
         getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
 
     IRMapping mapper;
@@ -3777,9 +3782,9 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
     SmallVector<OpFoldResult> newLowPad =
         addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        padOp.getLoc(), padOp.getResultType(), producerPad.getSource(),
-        newLowPad, newHighPad, padOp.getNofold(),
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, padOp.getLoc(), padOp.getResultType(),
+        producerPad.getSource(), newLowPad, newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, tensor::PadOp::getAttributeNames()));
     rewriter.inlineRegionBefore(padOp.getRegion(), newPadOp.getRegion(),
                                 newPadOp.getRegion().begin());
@@ -4108,8 +4113,8 @@ struct FoldTensorCastProducerOp
     for (auto [oldResult, newResult] :
          llvm::zip(op->getResults(), newOp->getResults())) {
       if (newResult.getType() != oldResult.getType()) {
-        replacements.push_back(tensor::CastOp::create(rewriter,
-            op->getLoc(), oldResult.getType(), newResult));
+        replacements.push_back(tensor::CastOp::create(
+            rewriter, op->getLoc(), oldResult.getType(), newResult));
       } else {
         replacements.push_back(newResult);
       }
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index ba258402b61b3..b8b1a592fdca2 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
index d69535169f956..003d81fbc7771 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
index a963b3f063a8a..fa64e0fe016a2 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h"
 
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
+#include "mlir/IR/Builders.h"
 #include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 9266a63a0038a..adf1e4c46ea8f 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp b/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
index 93b0bc591ca02..15c2ae22f56ea 100644
--- a/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/ADT/STLExtras.h"
 
diff --git a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
index 95870e8ef87be..696dbe3d1892a 100644
--- a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h"
 
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
+#include "mlir/IR/Builders.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/UB/IR/UBOps.cpp b/mlir/lib/Dialect/UB/IR/UBOps.cpp
index ee523f9522953..442a53ca8162a 100644
--- a/mlir/lib/Dialect/UB/IR/UBOps.cpp
+++ b/mlir/lib/Dialect/UB/IR/UBOps.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "mlir/Dialect/UB/IR/UBOpsDialect.cpp.inc"
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 3497b0d7a88a0..4dc3bebef8cfb 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -30,6 +30,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -599,8 +600,8 @@ struct ElideUnitDimsInMultiDimReduction
                             dstVecType.getScalableDims());
         mask = vector::ShapeCastOp::create(rewriter, loc, newMaskType, mask);
       }
-      cast = vector::ShapeCastOp::create(rewriter,
-          loc, reductionOp.getDestType(), reductionOp.getSource());
+      cast = vector::ShapeCastOp::create(
+          rewriter, loc, reductionOp.getDestType(), reductionOp.getSource());
     } else {
       // This means we are reducing all the dimensions, and all reduction
       // dimensions are of size 1. So a simple extraction would do.
@@ -674,35 +675,35 @@ Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
   case arith::AtomicRMWKind::addf:
   case arith::AtomicRMWKind::addi:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::ADD, vector);
+                                       CombiningKind::ADD, vector);
   case arith::AtomicRMWKind::mulf:
   case arith::AtomicRMWKind::muli:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MUL, vector);
+                                       CombiningKind::MUL, vector);
   case arith::AtomicRMWKind::minimumf:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINIMUMF, vector);
+                                       CombiningKind::MINIMUMF, vector);
   case arith::AtomicRMWKind::mins:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINSI, vector);
+                                       CombiningKind::MINSI, vector);
   case arith::AtomicRMWKind::minu:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINUI, vector);
+                                       CombiningKind::MINUI, vector);
   case arith::AtomicRMWKind::maximumf:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXIMUMF, vector);
+                                       CombiningKind::MAXIMUMF, vector);
   case arith::AtomicRMWKind::maxs:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXSI, vector);
+                                       CombiningKind::MAXSI, vector);
   case arith::AtomicRMWKind::maxu:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXUI, vector);
+                                       CombiningKind::MAXUI, vector);
   case arith::AtomicRMWKind::andi:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::AND, vector);
+                                       CombiningKind::AND, vector);
   case arith::AtomicRMWKind::ori:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::OR, vector);
+                                       CombiningKind::OR, vector);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
@@ -4255,8 +4256,8 @@ class StridedSliceBroadcast final
     // just a single scalar.
     bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
     if (!lowerDimMatch && !isScalarSrc) {
-      source = ExtractStridedSliceOp::create(rewriter,
-          op->getLoc(), source,
+      source = ExtractStridedSliceOp::create(
+          rewriter, op->getLoc(), source,
           getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getStrides(), /* dropFront=*/rankDiff));
@@ -4351,7 +4352,7 @@ class ContiguousExtractStridedSliceToExtract final
     SmallVector<int64_t> offsets = getI64SubArray(op.getOffsets());
     auto extractOffsets = ArrayRef(offsets).take_front(numOffsets);
     Value extract = vector::ExtractOp::create(rewriter, op->getLoc(), source,
-                                                       extractOffsets);
+                                              extractOffsets);
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(), extract);
     return success();
   }
@@ -5421,13 +5422,14 @@ struct SwapExtractSliceOfTransferWrite
     // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
     // Set all in_bounds to false and let the folder infer them.
     SmallVector<bool> newInBounds(vectorShape.size(), false);
-    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
-        extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
-        insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
-        insertOp.getMixedStrides());
-    auto newTransferWriteOp = TransferWriteOp::create(rewriter,
-        transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
-        transferOp.getIndices(), transferOp.getPermutationMapAttr(),
+    auto newExtractOp = tensor::ExtractSliceOp::create(
+        rewriter, extractOp.getLoc(), insertOp.getSourceType(),
+        insertOp.getDest(), insertOp.getMixedOffsets(),
+        insertOp.getMixedSizes(), insertOp.getMixedStrides());
+    auto newTransferWriteOp = TransferWriteOp::create(
+        rewriter, transferOp.getLoc(), transferOp.getVector(),
+        newExtractOp.getResult(), transferOp.getIndices(),
+        transferOp.getPermutationMapAttr(),
         rewriter.getBoolArrayAttr(newInBounds));
     rewriter.modifyOpInPlace(insertOp, [&]() {
       insertOp.getSourceMutable().assign(newTransferWriteOp.getResult());
@@ -7299,11 +7301,11 @@ Operation *mlir::vector::maskOperation(OpBuilder &builder,
     return maskableOp;
   if (passthru)
     return MaskOp::create(builder, maskableOp->getLoc(),
-                                  maskableOp->getResultTypes(), mask, passthru,
-                                  maskableOp, createMaskOpRegion);
+                          maskableOp->getResultTypes(), mask, passthru,
+                          maskableOp, createMaskOpRegion);
   return MaskOp::create(builder, maskableOp->getLoc(),
-                                maskableOp->getResultTypes(), mask, maskableOp,
-                                createMaskOpRegion);
+                        maskableOp->getResultTypes(), mask, maskableOp,
+                        createMaskOpRegion);
 }
 
 /// Creates a vector select operation that picks values from `newValue` or
@@ -7319,7 +7321,7 @@ Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
     return newValue;
 
   return arith::SelectOp::create(builder, newValue.getLoc(), newValue.getType(),
-                                         mask, newValue, passthru);
+                                 mask, newValue, passthru);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 125c3d918284c..bb051650897b4 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
 #include "mlir/Dialect/X86Vector/Transforms.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
index 01acade66a3c4..99654f79ead24 100644
--- a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
+++ b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
@@ -64,7 +65,7 @@ SmallVector<Value> x86vector::MaskCompressOp::getIntrinsicOperands(
     src = adaptor.getSrc();
   } else if (adaptor.getConstantSrc()) {
     src = LLVM::ConstantOp::create(rewriter, loc, opType,
-                                            adaptor.getConstantSrcAttr());
+                                   adaptor.getConstantSrcAttr());
   } else {
     auto zeroAttr = rewriter.getZeroAttr(opType);
     src = LLVM::ConstantOp::create(rewriter, loc, opType, zeroAttr);
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index f155b2a363834..4622f8fc67e94 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
 
 #include "llvm/Support/Debug.h"
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 6d7e2aa0ece7d..23988a00c7a2c 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectResourceBlobManager.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeRange.h"
 
diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
index cdf44c2959d50..9753ac00af88a 100644
--- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
+++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
index 3d24d4ecc4d0d..477219b79d937 100644
--- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
+++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "llvm/Support/Debug.h"
diff --git a/mlir/test/lib/Transforms/TestTransformsOps.cpp b/mlir/test/lib/Transforms/TestTransformsOps.cpp
index 9a5632bb99c06..3d7b6be8a78e9 100644
--- a/mlir/test/lib/Transforms/TestTransformsOps.cpp
+++ b/mlir/test/lib/Transforms/TestTransformsOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Transforms/RegionUtils.h"
 
diff --git a/mlir/test/python/lib/PythonTestDialect.cpp b/mlir/test/python/lib/PythonTestDialect.cpp
index a0ff31504c691..e0a76f1b75d38 100644
--- a/mlir/test/python/lib/PythonTestDialect.cpp
+++ b/mlir/test/python/lib/PythonTestDialect.cpp
@@ -9,6 +9,7 @@
 #include "PythonTestDialect.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "PythonTestDialect.cpp.inc"

>From 5881ed81658ba2959c928a2804388413f0dbca3c Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Fri, 11 Jul 2025 19:41:00 -0400
Subject: [PATCH 5/5] format

---
 flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp   |   2 +-
 .../lib/Standalone/StandaloneDialect.cpp      |   2 +-
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           | 172 +++---
 .../AffineToStandard/AffineToStandard.cpp     |  22 +-
 .../ArithToAMDGPU/ArithToAMDGPU.cpp           |  62 +-
 .../ArithToArmSME/ArithToArmSME.cpp           |   4 +-
 .../Conversion/ArithToEmitC/ArithToEmitC.cpp  |  78 +--
 .../Conversion/ArithToLLVM/ArithToLLVM.cpp    |  16 +-
 .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp  |  45 +-
 .../ArmNeon2dToIntr/ArmNeon2dToIntr.cpp       |   4 +-
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  | 175 +++---
 .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp    |  76 +--
 .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp    |  66 +--
 .../BufferizationToMemRef.cpp                 |   3 +-
 .../ComplexCommon/DivisionConverter.cpp       | 442 +++++++-------
 .../ComplexToLLVM/ComplexToLLVM.cpp           |  28 +-
 .../ComplexToLibm/ComplexToLibm.cpp           |   2 +-
 .../ComplexToStandard/ComplexToStandard.cpp   | 234 ++++----
 .../ControlFlowToLLVM/ControlFlowToLLVM.cpp   |   2 +-
 .../ControlFlowToSCF/ControlFlowToSCF.cpp     |  26 +-
 .../Conversion/FuncToEmitC/FuncToEmitC.cpp    |   4 +-
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp |  45 +-
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |  79 +--
 .../GPUCommon/GPUToLLVMConversion.cpp         |  93 +--
 .../GPUCommon/IndexIntrinsicsOpLowering.h     |  10 +-
 .../GPUCommon/OpToFuncCallLowering.h          |  20 +-
 .../Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp  |  10 +-
 .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp        |  37 +-
 .../Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp    |  32 +-
 .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp      |  38 +-
 mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp |  68 +--
 .../Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp  |   5 +-
 .../Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp  |   8 +-
 .../Conversion/IndexToLLVM/IndexToLLVM.cpp    |   8 +-
 .../Conversion/IndexToSPIRV/IndexToSPIRV.cpp  |  33 +-
 .../Conversion/LLVMCommon/MemRefBuilder.cpp   |  54 +-
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    |  42 +-
 .../Conversion/LLVMCommon/PrintCallHelper.cpp |   6 +-
 .../Conversion/LLVMCommon/TypeConverter.cpp   |   2 +-
 .../Conversion/LLVMCommon/VectorPattern.cpp   |   4 +-
 .../LinalgToStandard/LinalgToStandard.cpp     |   8 +-
 mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp   |  70 +--
 .../Conversion/MathToFuncs/MathToFuncs.cpp    | 143 ++---
 mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp |  70 +--
 mlir/lib/Conversion/MathToLibm/MathToLibm.cpp |   9 +-
 .../Conversion/MathToSPIRV/MathToSPIRV.cpp    |  50 +-
 .../MemRefToEmitC/MemRefToEmitC.cpp           |   8 +-
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  | 128 ++--
 .../MemRefToSPIRV/MemRefToSPIRV.cpp           |  29 +-
 mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp   | 175 +++---
 .../Conversion/OpenACCToSCF/OpenACCToSCF.cpp  |   2 +-
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |   2 +-
 .../PDLToPDLInterp/PDLToPDLInterp.cpp         | 181 +++---
 .../SCFToControlFlow/SCFToControlFlow.cpp     |  24 +-
 mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp |  10 +-
 mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp     |  48 +-
 .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp    |  35 +-
 mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp |  41 +-
 .../ConvertLaunchFuncToLLVMCalls.cpp          |  12 +-
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    | 111 ++--
 .../ShapeToStandard/ShapeToStandard.cpp       | 157 ++---
 .../TensorToSPIRV/TensorToSPIRV.cpp           |   4 +-
 .../Conversion/TosaToArith/TosaToArith.cpp    |  39 +-
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  | 545 +++++++++---------
 .../TosaToLinalg/TosaToLinalgNamed.cpp        | 126 ++--
 .../TosaToMLProgram/TosaToMLProgram.cpp       |  12 +-
 mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp   |  24 +-
 .../Conversion/TosaToTensor/TosaToTensor.cpp  |  27 +-
 .../VectorToArmSME/VectorToArmSME.cpp         |  67 +--
 .../Conversion/VectorToGPU/VectorToGPU.cpp    |  74 +--
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      | 191 +++---
 .../Conversion/VectorToSCF/VectorToSCF.cpp    | 124 ++--
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  79 +--
 .../VectorToXeGPU/VectorToXeGPU.cpp           |  31 +-
 .../AMDGPU/Transforms/EmulateAtomics.cpp      |  15 +-
 .../AMDGPU/Transforms/MaskedloadToLoad.cpp    |  20 +-
 .../Transforms/ResolveStridedMetadata.cpp     |   8 +-
 .../Transforms/AffineExpandIndexOps.cpp       |  21 +-
 .../Affine/Transforms/DecomposeAffineOps.cpp  |   4 +-
 .../Transforms/PipelineDataTransfer.cpp       |   6 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |  76 +--
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |  80 +--
 .../BufferDeallocationOpInterfaceImpl.cpp     |   4 +-
 .../Arith/Transforms/EmulateWideInt.cpp       |  73 +--
 .../Dialect/Arith/Transforms/ExpandOps.cpp    |  69 +--
 .../Arith/Transforms/ReifyValueBounds.cpp     |  12 +-
 mlir/lib/Dialect/Arith/Utils/Utils.cpp        |  14 +-
 .../LowerContractionToNeonI8MMPattern.cpp     |  18 +-
 mlir/lib/Dialect/ArmSME/IR/Utils.cpp          |   8 +-
 .../ArmSME/Transforms/OuterProductFusion.cpp  |   9 +-
 .../ArmSME/Transforms/VectorLegalization.cpp  | 125 ++--
 .../Transforms/LegalizeForLLVMExport.cpp      |  18 +-
 .../Transforms/LegalizeVectorStorage.cpp      |  22 +-
 .../LowerContractionToSVEI8MMPattern.cpp      |  42 +-
 .../Transforms/AsyncRuntimeRefCounting.cpp    |   2 +-
 .../Async/Transforms/AsyncToAsyncRuntime.cpp  |  36 +-
 .../IR/BufferDeallocationOpInterface.cpp      |   8 +-
 .../IR/BufferizableOpInterface.cpp            |  10 +-
 .../BufferDeallocationSimplification.cpp      |  25 +-
 .../Transforms/BufferResultsToOutParams.cpp   |   4 +-
 .../Bufferization/Transforms/BufferUtils.cpp  |   5 +-
 .../Bufferization/Transforms/Bufferize.cpp    |  12 +-
 .../DropEquivalentBufferResults.cpp           |   6 +-
 .../Transforms/EmptyTensorElimination.cpp     |   2 +-
 .../FuncBufferizableOpInterfaceImpl.cpp       |   9 +-
 .../Transforms/LowerDeallocations.cpp         | 202 +++----
 .../OwnershipBasedBufferDeallocation.cpp      |  38 +-
 .../lib/Dialect/Complex/IR/ComplexDialect.cpp |   2 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |  23 +-
 .../EmitC/Transforms/WrapFuncInClass.cpp      |   2 +-
 .../Func/Extensions/InlinerExtension.cpp      |   3 +-
 .../Func/Transforms/FuncConversions.cpp       |   6 +-
 mlir/lib/Dialect/Func/Utils/Utils.cpp         |   9 +-
 mlir/lib/Dialect/GPU/TransformOps/Utils.cpp   |   9 +-
 .../GPU/Transforms/AsyncRegionRewriter.cpp    |   5 +-
 .../GPU/Transforms/DecomposeMemRefs.cpp       |   4 +-
 .../GPU/Transforms/KernelOutlining.cpp        |  12 +-
 .../Dialect/GPU/Transforms/ModuleToBinary.cpp |   2 +-
 .../GPU/Transforms/PromoteShuffleToAMDGPU.cpp |   4 +-
 .../GPU/Transforms/ShuffleRewriter.cpp        |   4 +-
 .../GPU/Transforms/SubgroupIdRewriter.cpp     |  16 +-
 .../GPU/Transforms/SubgroupReduceLowering.cpp | 103 ++--
 .../Dialect/GPU/Utils/DistributionUtils.cpp   |   7 +-
 .../LLVMIR/IR/BasicPtxBuilderInterface.cpp    |   8 +-
 .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp   |   4 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp |  63 +-
 .../Dialect/LLVMIR/Transforms/AddComdats.cpp  |   5 +-
 .../Transforms/InlinerInterfaceImpl.cpp       |  35 +-
 .../LLVMIR/Transforms/LegalizeForExport.cpp   |   2 +-
 .../LLVMIR/Transforms/OptimizeForNVVM.cpp     |  16 +-
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 152 ++---
 .../TransformOps/LinalgTransformOps.cpp       |  24 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/ConvertConv2DToImg2Col.cpp     | 123 ++--
 .../Transforms/ConvertToDestinationStyle.cpp  |  51 +-
 .../Transforms/DataLayoutPropagation.cpp      |  79 +--
 ...DecomposeGenericByUnfoldingPermutation.cpp |  13 +-
 .../Linalg/Transforms/DecomposeLinalgOps.cpp  |  20 +-
 .../Dialect/Linalg/Transforms/Detensorize.cpp |   8 +-
 .../Linalg/Transforms/DropUnitDims.cpp        |  45 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  74 +--
 .../Linalg/Transforms/ElementwiseToLinalg.cpp |   4 +-
 .../EraseUnusedOperandsAndResults.cpp         |   4 +-
 .../FusePadOpWithLinalgProducer.cpp           |  12 +-
 .../Linalg/Transforms/Generalization.cpp      |   5 +-
 .../Linalg/Transforms/HoistPadding.cpp        |  40 +-
 .../Dialect/Linalg/Transforms/Hoisting.cpp    |   6 +-
 .../Transforms/InlineScalarOperands.cpp       |  10 +-
 .../Dialect/Linalg/Transforms/Interchange.cpp |   3 +-
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |   5 +-
 .../Transforms/MeshShardingInterfaceImpl.cpp  |  14 +-
 .../Linalg/Transforms/NamedOpConversions.cpp  |  17 +-
 .../Transforms/PackAndUnpackPatterns.cpp      |  10 +-
 .../Linalg/Transforms/PadTilingInterface.cpp  |  10 +-
 .../lib/Dialect/Linalg/Transforms/Padding.cpp |  10 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |   6 +-
 mlir/lib/Dialect/Linalg/Transforms/Split.cpp  |   4 +-
 .../Linalg/Transforms/SplitReduction.cpp      |  32 +-
 .../SwapExtractSliceWithFillPatterns.cpp      |   8 +-
 mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp |  27 +-
 .../Linalg/Transforms/TilingInterfaceImpl.cpp |  74 +--
 .../Dialect/Linalg/Transforms/Transforms.cpp  | 115 ++--
 .../Linalg/Transforms/TransposeConv2D.cpp     |   2 +-
 .../Linalg/Transforms/TransposeMatmul.cpp     |  34 +-
 .../Linalg/Transforms/Vectorization.cpp       | 255 ++++----
 .../Linalg/Transforms/WinogradConv2D.cpp      | 155 ++---
 mlir/lib/Dialect/Linalg/Utils/Utils.cpp       |  24 +-
 .../BufferizableOpInterfaceImpl.cpp           |   5 +-
 .../Transforms/AlgebraicSimplification.cpp    |  13 +-
 .../Math/Transforms/ExpandPatterns.cpp        |  63 +-
 .../Transforms/PolynomialApproximation.cpp    | 143 ++---
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    |   2 +-
 .../Transforms/AllocationOpInterfaceImpl.cpp  |   5 +-
 .../MemRef/Transforms/ComposeSubView.cpp      |   4 +-
 .../MemRef/Transforms/EmulateNarrowType.cpp   |  34 +-
 .../Dialect/MemRef/Transforms/ExpandOps.cpp   |   9 +-
 .../MemRef/Transforms/ExpandRealloc.cpp       |  22 +-
 .../Transforms/ExpandStridedMetadata.cpp      |  27 +-
 .../Transforms/ExtractAddressComputations.cpp |  26 +-
 .../MemRef/Transforms/FlattenMemRefs.cpp      |  55 +-
 .../MemRef/Transforms/FoldMemRefAliasOps.cpp  |  12 +-
 .../Transforms/IndependenceTransforms.cpp     |  12 +-
 .../Dialect/MemRef/Transforms/MultiBuffer.cpp |  13 +-
 .../MemRef/Transforms/NormalizeMemRefs.cpp    |   2 +-
 .../MemRef/Transforms/ReifyResultShapes.cpp   |   6 +-
 .../Transforms/RuntimeOpVerification.cpp      | 110 ++--
 .../Mesh/Transforms/Simplifications.cpp       |   4 +-
 .../Dialect/Mesh/Transforms/Spmdization.cpp   |  24 +-
 .../Dialect/Mesh/Transforms/Transforms.cpp    |  25 +-
 .../NVGPU/TransformOps/NVGPUTransformOps.cpp  |  51 +-
 .../NVGPU/Transforms/CreateAsyncGroups.cpp    |  19 +-
 .../NVGPU/Transforms/OptimizeSharedMemory.cpp |   3 +-
 .../Quant/Transforms/LowerQuantOps.cpp        |  30 +-
 .../Quant/Transforms/StripFuncQuantTypes.cpp  |   2 +-
 .../BufferizableOpInterfaceImpl.cpp           |  32 +-
 .../lib/Dialect/SCF/Transforms/ForToWhile.cpp |  13 +-
 .../Dialect/SCF/Transforms/LoopPipelining.cpp | 101 ++--
 .../SCF/Transforms/LoopSpecialization.cpp     |   4 +-
 .../SCF/Transforms/ParallelLoopFusion.cpp     |   4 +-
 .../SCF/Transforms/ParallelLoopTiling.cpp     |  30 +-
 .../Transforms/StructuralTypeConversions.cpp  |  18 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  61 +-
 .../SCF/Transforms/WrapInZeroTripCheck.cpp    |   8 +-
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          |  97 ++--
 mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp  |   8 +-
 .../SPIRV/IR/SPIRVCanonicalization.cpp        |   6 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp        |  36 +-
 .../Linking/ModuleCombiner/ModuleCombiner.cpp |   5 +-
 .../Transforms/LowerABIAttributesPass.cpp     |  24 +-
 .../SPIRV/Transforms/RewriteInsertsPass.cpp   |   4 +-
 .../SPIRV/Transforms/SPIRVConversion.cpp      |  59 +-
 .../Transforms/SPIRVWebGPUTransforms.cpp      |  26 +-
 .../Transforms/UnifyAliasedResourcePass.cpp   |  26 +-
 .../BufferizableOpInterfaceImpl.cpp           |   9 +-
 .../Shape/Transforms/ShapeToShapeLowering.cpp |   2 +-
 .../IR/SparseTensorInterfaces.cpp             |   4 +-
 .../Transforms/SparseAssembler.cpp            |  15 +-
 .../Transforms/SparseBufferRewriting.cpp      | 118 ++--
 .../Transforms/SparseGPUCodegen.cpp           | 161 +++---
 .../Transforms/SparseIterationToScf.cpp       |  18 +-
 .../Transforms/SparseReinterpretMap.cpp       |  17 +-
 .../Transforms/SparseSpaceCollapse.cpp        |   4 +-
 .../SparseStorageSpecifierToLLVM.cpp          |  10 +-
 .../Transforms/SparseTensorCodegen.cpp        | 107 ++--
 .../Transforms/SparseTensorConversion.cpp     |  24 +-
 .../Transforms/SparseTensorRewriting.cpp      | 102 ++--
 .../Transforms/SparseVectorization.cpp        |  32 +-
 .../Transforms/Sparsification.cpp             |  19 +-
 .../Transforms/Utils/CodegenUtils.cpp         |  23 +-
 .../Transforms/Utils/CodegenUtils.h           |   8 +-
 .../Transforms/Utils/LoopEmitter.cpp          |  18 +-
 .../Utils/SparseTensorDescriptor.cpp          |  18 +-
 .../Transforms/Utils/SparseTensorIterator.cpp |  43 +-
 .../lib/Dialect/SparseTensor/Utils/Merger.cpp |   6 +-
 .../Extensions/MeshShardingExtensions.cpp     |   4 +-
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp   |  20 +-
 .../BufferizableOpInterfaceImpl.cpp           |  63 +-
 .../Tensor/Transforms/EmptyOpPatterns.cpp     |   5 +-
 .../ExtractSliceFromReshapeUtils.cpp          |  13 +-
 .../Tensor/Transforms/FoldTensorSubsetOps.cpp |   6 +-
 .../Transforms/IndependenceTransforms.cpp     |  15 +-
 .../Tensor/Transforms/ReshapePatterns.cpp     |  22 +-
 .../Transforms/RuntimeOpVerification.cpp      |  39 +-
 .../SubsetInsertionOpInterfaceImpl.cpp        |   4 +-
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |  12 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |   6 +-
 .../Transforms/TosaDecomposeDepthwise.cpp     |   8 +-
 .../Transforms/TosaDecomposeTransposeConv.cpp |  22 +-
 .../Tosa/Transforms/TosaReduceTransposes.cpp  |   8 +-
 .../Dialect/Tosa/Utils/ConversionUtils.cpp    |   4 +-
 .../LoopExtension/LoopExtensionOps.cpp        |   2 +-
 .../BufferizableOpInterfaceImpl.cpp           |   9 +-
 .../Vector/Transforms/LowerVectorContract.cpp |  54 +-
 .../Vector/Transforms/LowerVectorGather.cpp   |  24 +-
 .../Transforms/LowerVectorInterleave.cpp      |  22 +-
 .../Vector/Transforms/LowerVectorMask.cpp     |  31 +-
 .../Transforms/LowerVectorMultiReduction.cpp  |  62 +-
 .../Vector/Transforms/LowerVectorScan.cpp     |  24 +-
 .../Transforms/LowerVectorShapeCast.cpp       |  46 +-
 .../Vector/Transforms/LowerVectorTransfer.cpp |  44 +-
 .../Transforms/LowerVectorTranspose.cpp       |  22 +-
 .../Vector/Transforms/VectorDistribute.cpp    | 100 ++--
 .../Transforms/VectorDropLeadUnitDim.cpp      |  65 ++-
 .../VectorEmulateMaskedLoadStore.cpp          |  18 +-
 .../Transforms/VectorEmulateNarrowType.cpp    | 198 ++++---
 ...sertExtractStridedSliceRewritePatterns.cpp |  32 +-
 .../Transforms/VectorMaskElimination.cpp      |   4 +-
 .../Transforms/VectorTransferOpTransforms.cpp |  36 +-
 .../VectorTransferSplitRewritePatterns.cpp    |  58 +-
 .../Vector/Transforms/VectorTransforms.cpp    |  96 +--
 .../Vector/Transforms/VectorUnroll.cpp        |  58 +-
 mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp |   4 +-
 .../X86Vector/Transforms/AVXTranspose.cpp     |  14 +-
 .../Transforms/XeGPUSubgroupDistribute.cpp    |  47 +-
 .../Dialect/XeGPU/Transforms/XeGPUUnroll.cpp  |  51 +-
 .../Transforms/XeGPUWgToSgDistribute.cpp      |  16 +-
 mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp   |  16 +-
 .../lib/Target/LLVMIR/LLVMImportInterface.cpp |   4 +-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |  85 +--
 .../SPIRV/Deserialization/DeserializeOps.cpp  |  38 +-
 .../SPIRV/Deserialization/Deserializer.cpp    |  46 +-
 mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp       |  43 +-
 mlir/test/python/lib/PythonTestDialect.cpp    |   2 +-
 283 files changed, 5801 insertions(+), 5403 deletions(-)

diff --git a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
index d7d294ab621d8..4cd71f6e1f3c7 100644
--- a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
@@ -12,9 +12,9 @@
 
 #include "flang/Optimizer/Dialect/FIRCG/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 /// FIR codegen dialect constructor.
 fir::FIRCodeGenDialect::FIRCodeGenDialect(mlir::MLIRContext *ctx)
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
index 3da966ef9cc85..4861300c8b546 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
@@ -8,8 +8,8 @@
 
 #include "Standalone/StandaloneDialect.h"
 #include "Standalone/StandaloneOps.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "Standalone/StandaloneTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 using namespace mlir::standalone;
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index e7e0a4ebd2092..fe3dc91328879 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -81,8 +81,8 @@ static Value getLinearIndexI32(ConversionPatternRewriter &rewriter,
               : LLVM::ConstantOp::create(rewriter, loc, i32, stride);
       increment = LLVM::MulOp::create(rewriter, loc, increment, strideValue);
     }
-    index =
-        index ? LLVM::AddOp::create(rewriter, loc, index, increment) : increment;
+    index = index ? LLVM::AddOp::create(rewriter, loc, index, increment)
+                  : increment;
   }
   return index ? index : createI32Constant(rewriter, loc, 0);
 }
@@ -133,13 +133,13 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
   if (chipset.majorVersion == 9 && chipset >= kGfx942 && cacheSwizzleStride) {
     Value cacheStrideZext =
         LLVM::ZExtOp::create(rewriter, loc, i16, cacheSwizzleStride);
-    Value swizzleBit = LLVM::ConstantOp::create(rewriter,
-        loc, i16, rewriter.getI16IntegerAttr(1 << 14));
+    Value swizzleBit = LLVM::ConstantOp::create(
+        rewriter, loc, i16, rewriter.getI16IntegerAttr(1 << 14));
     stride = LLVM::OrOp::create(rewriter, loc, cacheStrideZext, swizzleBit,
-                                         /*isDisjoint=*/true);
+                                /*isDisjoint=*/true);
   } else {
     stride = LLVM::ConstantOp::create(rewriter, loc, i16,
-                                               rewriter.getI16IntegerAttr(0));
+                                      rewriter.getI16IntegerAttr(0));
   }
   // Get the number of elements.
   // Flag word:
@@ -209,20 +209,21 @@ struct FatRawBufferCastLowering
             : descriptor.alignedPtr(rewriter, loc);
 
     Value offset = adaptor.getResetOffset()
-                       ? LLVM::ConstantOp::create(rewriter,
-                             loc, getIndexType(), rewriter.getIndexAttr(0))
+                       ? LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                                  rewriter.getIndexAttr(0))
                        : descriptor.offset(rewriter, loc);
 
     bool hasSizes = memrefType.getRank() > 0;
     // No need to unpack() and pack() all the individual sizes and strides,
     // so we'll just extract the arrays.
-    Value sizes = hasSizes ? LLVM::ExtractValueOp::create(rewriter,
-                                 loc, descriptor, kSizePosInMemRefDescriptor)
-                           : Value{};
-    Value strides = hasSizes
-                        ? LLVM::ExtractValueOp::create(rewriter,
-                              loc, descriptor, kStridePosInMemRefDescriptor)
-                        : Value{};
+    Value sizes = hasSizes
+                      ? LLVM::ExtractValueOp::create(rewriter, loc, descriptor,
+                                                     kSizePosInMemRefDescriptor)
+                      : Value{};
+    Value strides =
+        hasSizes ? LLVM::ExtractValueOp::create(rewriter, loc, descriptor,
+                                                kStridePosInMemRefDescriptor)
+                 : Value{};
 
     Value fatPtr = makeBufferRsrc(
         rewriter, loc, basePointer, numRecords, adaptor.getBoundsCheck(),
@@ -231,17 +232,17 @@ struct FatRawBufferCastLowering
     Value result = MemRefDescriptor::poison(
         rewriter, loc,
         getTypeConverter()->convertType(op.getResult().getType()));
-    result = LLVM::InsertValueOp::create(rewriter,
-        loc, result, fatPtr, kAllocatedPtrPosInMemRefDescriptor);
-    result = LLVM::InsertValueOp::create(rewriter,
-        loc, result, fatPtr, kAlignedPtrPosInMemRefDescriptor);
+    result = LLVM::InsertValueOp::create(rewriter, loc, result, fatPtr,
+                                         kAllocatedPtrPosInMemRefDescriptor);
+    result = LLVM::InsertValueOp::create(rewriter, loc, result, fatPtr,
+                                         kAlignedPtrPosInMemRefDescriptor);
     result = LLVM::InsertValueOp::create(rewriter, loc, result, offset,
-                                                  kOffsetPosInMemRefDescriptor);
+                                         kOffsetPosInMemRefDescriptor);
     if (hasSizes) {
       result = LLVM::InsertValueOp::create(rewriter, loc, result, sizes,
-                                                    kSizePosInMemRefDescriptor);
-      result = LLVM::InsertValueOp::create(rewriter,
-          loc, result, strides, kStridePosInMemRefDescriptor);
+                                           kSizePosInMemRefDescriptor);
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, strides,
+                                           kStridePosInMemRefDescriptor);
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -342,8 +343,8 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     SmallVector<Value, 6> args;
     if (storeData) {
       if (llvmBufferValType != llvmWantedDataType) {
-        Value castForStore =
-            LLVM::BitcastOp::create(rewriter, loc, llvmBufferValType, storeData);
+        Value castForStore = LLVM::BitcastOp::create(
+            rewriter, loc, llvmBufferValType, storeData);
         args.push_back(castForStore);
       } else {
         args.push_back(storeData);
@@ -352,8 +353,8 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     if (atomicCmpData) {
       if (llvmBufferValType != llvmWantedDataType) {
-        Value castForCmp = LLVM::BitcastOp::create(rewriter,
-            loc, llvmBufferValType, atomicCmpData);
+        Value castForCmp = LLVM::BitcastOp::create(
+            rewriter, loc, llvmBufferValType, atomicCmpData);
         args.push_back(castForCmp);
       } else {
         args.push_back(atomicCmpData);
@@ -382,9 +383,9 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     if (std::optional<int32_t> indexOffset = adaptor.getIndexOffset();
         indexOffset && *indexOffset > 0) {
       Value extraOffsetConst = createI32Constant(rewriter, loc, *indexOffset);
-      voffset =
-          voffset ? LLVM::AddOp::create(rewriter, loc, voffset, extraOffsetConst)
-                  : extraOffsetConst;
+      voffset = voffset ? LLVM::AddOp::create(rewriter, loc, voffset,
+                                              extraOffsetConst)
+                        : extraOffsetConst;
     }
     voffset = LLVM::MulOp::create(rewriter, loc, voffset, byteWidthConst);
     args.push_back(voffset);
@@ -404,12 +405,12 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
                                            llvmBufferValType);
     Operation *lowered = Intrinsic::create(rewriter, loc, resultTypes, args,
-                                                    ArrayRef<NamedAttribute>());
+                                           ArrayRef<NamedAttribute>());
     if (lowered->getNumResults() == 1) {
       Value replacement = lowered->getResult(0);
       if (llvmBufferValType != llvmWantedDataType) {
         replacement = LLVM::BitcastOp::create(rewriter, loc, llvmWantedDataType,
-                                                       replacement);
+                                              replacement);
       }
       rewriter.replaceOp(gpuOp, replacement);
     } else {
@@ -516,19 +517,21 @@ static Value convertMFMAVectorOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   if (auto vectorType = dyn_cast<VectorType>(inputType)) {
     if (vectorType.getElementType().isBF16() && !allowBf16)
-      return LLVM::BitcastOp::create(rewriter,
-          loc, vectorType.clone(rewriter.getI16Type()), input);
+      return LLVM::BitcastOp::create(
+          rewriter, loc, vectorType.clone(rewriter.getI16Type()), input);
     if (vectorType.getElementType().isInteger(8) &&
         vectorType.getNumElements() <= 8)
-      return LLVM::BitcastOp::create(rewriter,
-          loc, rewriter.getIntegerType(vectorType.getNumElements() * 8), input);
+      return LLVM::BitcastOp::create(
+          rewriter, loc,
+          rewriter.getIntegerType(vectorType.getNumElements() * 8), input);
     if (isa<IntegerType>(vectorType.getElementType()) &&
         vectorType.getElementTypeBitWidth() <= 8) {
       int64_t numWords = llvm::divideCeil(
           vectorType.getNumElements() * vectorType.getElementTypeBitWidth(),
           32);
-      return LLVM::BitcastOp::create(rewriter,
-          loc, VectorType::get(numWords, rewriter.getI32Type()), input);
+      return LLVM::BitcastOp::create(
+          rewriter, loc, VectorType::get(numWords, rewriter.getI32Type()),
+          input);
     }
   }
   return input;
@@ -576,8 +579,8 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   Type elemType = vectorType.getElementType();
 
   if (elemType.isBF16())
-    llvmInput = LLVM::BitcastOp::create(rewriter,
-        loc, vectorType.clone(rewriter.getI16Type()), llvmInput);
+    llvmInput = LLVM::BitcastOp::create(
+        rewriter, loc, vectorType.clone(rewriter.getI16Type()), llvmInput);
   if (elemType.getIntOrFloatBitWidth() > 8) {
     operands.push_back(llvmInput);
     return;
@@ -633,8 +636,8 @@ static void wmmaPushOutputOperand(ConversionPatternRewriter &rewriter,
   auto vectorType = dyn_cast<VectorType>(inputType);
   Type elemType = vectorType.getElementType();
   if (elemType.isBF16())
-    output = LLVM::BitcastOp::create(rewriter,
-        loc, vectorType.clone(rewriter.getI16Type()), output);
+    output = LLVM::BitcastOp::create(
+        rewriter, loc, vectorType.clone(rewriter.getI16Type()), output);
   operands.push_back(output);
   if (elemType.isF16() || elemType.isBF16() || elemType.isInteger(16)) {
     operands.push_back(createI1Constant(rewriter, loc, subwordOffset));
@@ -1092,8 +1095,8 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
 
     Operation *maybeCastBack = lowered;
     if (rawOutType != outType)
-      maybeCastBack =
-          LLVM::BitcastOp::create(rewriter, loc, outType, lowered->getResult(0));
+      maybeCastBack = LLVM::BitcastOp::create(rewriter, loc, outType,
+                                              lowered->getResult(0));
     rewriter.replaceOp(op, maybeCastBack->getResults());
 
     return success();
@@ -1143,22 +1146,22 @@ struct TransposeLoadOpLowering
     switch (elementTypeSize) {
     case 4: {
       assert(numElements == 16);
-      auto rocdlOp =
-          ROCDL::ds_read_tr4_b64::create(rewriter, loc, rocdlResultType, srcPtr);
+      auto rocdlOp = ROCDL::ds_read_tr4_b64::create(rewriter, loc,
+                                                    rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 6: {
       assert(numElements == 16);
-      auto rocdlOp =
-          ROCDL::ds_read_tr6_b96::create(rewriter, loc, rocdlResultType, srcPtr);
+      auto rocdlOp = ROCDL::ds_read_tr6_b96::create(rewriter, loc,
+                                                    rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 8: {
       assert(numElements == 8);
-      auto rocdlOp =
-          ROCDL::ds_read_tr8_b64::create(rewriter, loc, rocdlResultType, srcPtr);
+      auto rocdlOp = ROCDL::ds_read_tr8_b64::create(rewriter, loc,
+                                                    rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
@@ -1318,8 +1321,8 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
   if (!sourceVecType || sourceVecType.getNumElements() < 4) {
     Value longVec = LLVM::UndefOp::create(rewriter, loc, v4i8);
     if (!sourceVecType) {
-      longVec = LLVM::InsertElementOp::create(rewriter,
-          loc, longVec, source, createI32Constant(rewriter, loc, 0));
+      longVec = LLVM::InsertElementOp::create(
+          rewriter, loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
@@ -1384,8 +1387,8 @@ LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
   if (sourceVecType.getNumElements() < packedVecType.getNumElements()) {
     Value longVec = LLVM::ZeroOp::create(rewriter, loc, packedVecType);
     if (!sourceVecType) {
-      longVec = LLVM::InsertElementOp::create(rewriter,
-          loc, longVec, source, createI32Constant(rewriter, loc, 0));
+      longVec = LLVM::InsertElementOp::create(
+          rewriter, loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
@@ -1476,32 +1479,35 @@ LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
 
   Value result;
   if (sourceElemType.isF32() && isa<Float8E5M2Type>(resultElemType))
-    result = ROCDL::CvtScaleF32PkBf8F32Op::create(rewriter,
-        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkBf8F32Op::create(rewriter, loc, intResultType,
+                                                  existing, sourceA, sourceB,
+                                                  scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E5M2Type>(resultElemType))
-    result = ROCDL::CvtScaleF32PkBf8F16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkBf8F16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E5M2Type>(resultElemType))
-    result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float8E4M3FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp8F32Op::create(rewriter,
-        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp8F32Op::create(rewriter, loc, intResultType,
+                                                  existing, sourceA, sourceB,
+                                                  scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp8F16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp8F16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float4E2M1FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp4F32Op::create(rewriter,
-        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp4F32Op::create(rewriter, loc, intResultType,
+                                                  existing, sourceA, sourceB,
+                                                  scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp4F16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp4F16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else
     return failure();
 
@@ -1536,10 +1542,10 @@ LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
     result = ROCDL::CvtPkBf8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
-                                                   existing, op.getWordIndex());
+                                          existing, op.getWordIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
     result = ROCDL::CvtPkFp8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
-                                                   existing, op.getWordIndex());
+                                          existing, op.getWordIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
       op, getTypeConverter()->convertType(resultType), result);
@@ -1569,11 +1575,11 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = ROCDL::CvtSrBf8F32Op::create(rewriter,
-        loc, i32, source, stoch, existing, op.getStoreIndex());
+    result = ROCDL::CvtSrBf8F32Op::create(rewriter, loc, i32, source, stoch,
+                                          existing, op.getStoreIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = ROCDL::CvtSrFp8F32Op::create(rewriter,
-        loc, i32, source, stoch, existing, op.getStoreIndex());
+    result = ROCDL::CvtSrFp8F32Op::create(rewriter, loc, i32, source, stoch,
+                                          existing, op.getStoreIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
       op, getTypeConverter()->convertType(resultType), result);
@@ -1622,8 +1628,9 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
         auto llvmVecType = typeConverter->convertType(mlir::VectorType::get(
             32 / operandType.getIntOrFloatBitWidth(), llvmSrcIntType));
         Value undefVec = LLVM::UndefOp::create(rewriter, loc, llvmVecType);
-        operand = LLVM::InsertElementOp::create(rewriter,
-            loc, undefVec, operand, createI32Constant(rewriter, loc, 0));
+        operand =
+            LLVM::InsertElementOp::create(rewriter, loc, undefVec, operand,
+                                          createI32Constant(rewriter, loc, 0));
         operand = LLVM::BitcastOp::create(rewriter, loc, llvmType, operand);
       }
       return operand;
@@ -1711,8 +1718,9 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
     bool boundCtrl = DppOp->getAttrOfType<BoolAttr>("bound_ctrl").getValue();
 
     // create a ROCDL_DPPMovOp instruction with the appropriate attributes
-    auto dppMovOp = ROCDL::DPPUpdateOp::create(rewriter,
-        loc, llvmType, old, src, DppCtrl, rowMask, bankMask, boundCtrl);
+    auto dppMovOp =
+        ROCDL::DPPUpdateOp::create(rewriter, loc, llvmType, old, src, DppCtrl,
+                                   rowMask, bankMask, boundCtrl);
 
     Value result = dppMovOp.getRes();
     if (srcType.getIntOrFloatBitWidth() < 32) {
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 5a9d946c2fe4e..3b148f9021666 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -156,7 +156,7 @@ class AffineForLowering : public OpRewritePattern<AffineForOp> {
     Value step =
         arith::ConstantIndexOp::create(rewriter, loc, op.getStepAsInt());
     auto scfForOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound,
-                                                step, op.getInits());
+                                       step, op.getInits());
     rewriter.eraseBlock(scfForOp.getBody());
     rewriter.inlineRegionBefore(op.getRegion(), scfForOp.getRegion(),
                                 scfForOp.getRegion().end());
@@ -206,8 +206,8 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     if (op.getResults().empty()) {
       // Case with no reduction operations/return values.
       parOp = scf::ParallelOp::create(rewriter, loc, lowerBoundTuple,
-                                               upperBoundTuple, steps,
-                                               /*bodyBuilderFn=*/nullptr);
+                                      upperBoundTuple, steps,
+                                      /*bodyBuilderFn=*/nullptr);
       rewriter.eraseBlock(parOp.getBody());
       rewriter.inlineRegionBefore(op.getRegion(), parOp.getRegion(),
                                   parOp.getRegion().end());
@@ -233,9 +233,9 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       identityVals.push_back(
           arith::getIdentityValue(reductionOpValue, resultType, rewriter, loc));
     }
-    parOp = scf::ParallelOp::create(rewriter,
-        loc, lowerBoundTuple, upperBoundTuple, steps, identityVals,
-        /*bodyBuilderFn=*/nullptr);
+    parOp = scf::ParallelOp::create(rewriter, loc, lowerBoundTuple,
+                                    upperBoundTuple, steps, identityVals,
+                                    /*bodyBuilderFn=*/nullptr);
 
     //  Copy the body of the affine.parallel op.
     rewriter.eraseBlock(parOp.getBody());
@@ -299,17 +299,17 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
           isEquality ? arith::CmpIPredicate::eq : arith::CmpIPredicate::sge;
       Value cmpVal =
           arith::CmpIOp::create(rewriter, loc, pred, affResult, zeroConstant);
-      cond = cond
-                 ? arith::AndIOp::create(rewriter, loc, cond, cmpVal).getResult()
-                 : cmpVal;
+      cond =
+          cond ? arith::AndIOp::create(rewriter, loc, cond, cmpVal).getResult()
+               : cmpVal;
     }
     cond = cond ? cond
                 : arith::ConstantIntOp::create(rewriter, loc, /*value=*/1,
-                                                        /*width=*/1);
+                                               /*width=*/1);
 
     bool hasElseRegion = !op.getElseRegion().empty();
     auto ifOp = scf::IfOp::create(rewriter, loc, op.getResultTypes(), cond,
-                                           hasElseRegion);
+                                  hasElseRegion);
     rewriter.inlineRegionBefore(op.getThenRegion(),
                                 &ifOp.getThenRegion().back());
     rewriter.eraseBlock(&ifOp.getThenRegion().back());
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index df3f2c4926614..898a8e558dd3c 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -139,16 +139,16 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
   VectorType extResType = VectorType::get(2, rewriter.getF32Type());
   if (!inVecType) {
-    Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
-        loc, rewriter.getF32Type(), in, 0);
+    Value asFloat = amdgpu::ExtPackedFp8Op::create(
+        rewriter, loc, rewriter.getF32Type(), in, 0);
     Value result = castF32To(outElemType, asFloat, loc, rewriter);
     rewriter.replaceOp(op, result);
     return success();
   }
   int64_t numElements = inVecType.getNumElements();
 
-  Value zero = arith::ConstantOp::create(rewriter,
-      loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
+  Value zero = arith::ConstantOp::create(
+      rewriter, loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   VectorType outType = cast<VectorType>(op.getOut().getType());
 
   if (inVecType.getShape().empty()) {
@@ -158,8 +158,8 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
         vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     Value scalarExt =
         arith::ExtFOp::create(rewriter, loc, outElemType, scalarIn);
-    Value result = vector::InsertOp::create(rewriter, loc, scalarExt, zerodSplat,
-                                                     ArrayRef<int64_t>{});
+    Value result = vector::InsertOp::create(rewriter, loc, scalarExt,
+                                            zerodSplat, ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -176,19 +176,19 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
-    Value inSlice = vector::ExtractStridedSliceOp::create(rewriter,
-        loc, in, i, elemsThisOp, 1);
+    Value inSlice = vector::ExtractStridedSliceOp::create(rewriter, loc, in, i,
+                                                          elemsThisOp, 1);
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
       if (i + j + 1 < numElements) { // Convert two 8-bit elements
-        Value asFloats = amdgpu::ExtPackedFp8Op::create(rewriter,
-            loc, extResType, inSlice, j / 2);
+        Value asFloats = amdgpu::ExtPackedFp8Op::create(
+            rewriter, loc, extResType, inSlice, j / 2);
         Type desType = VectorType::get(2, outElemType);
         Value asType = castF32To(desType, asFloats, loc, rewriter);
-        result = vector::InsertStridedSliceOp::create(rewriter,
-            loc, asType, result, i + j, 1);
+        result = vector::InsertStridedSliceOp::create(rewriter, loc, asType,
+                                                      result, i + j, 1);
       } else { // Convert a 8-bit element
-        Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
-            loc, rewriter.getF32Type(), inSlice, j / 2 * 2);
+        Value asFloat = amdgpu::ExtPackedFp8Op::create(
+            rewriter, loc, rewriter.getF32Type(), inSlice, j / 2 * 2);
         Value asType = castF32To(outElemType, asFloat, loc, rewriter);
         result = vector::InsertOp::create(rewriter, loc, asType, result, i + j);
       }
@@ -250,11 +250,13 @@ static Value clampInput(PatternRewriter &rewriter, Location loc,
       loc, arith::CmpFPredicate::OEQ, source, negInf);
   Value isNan = rewriter.createOrFold<arith::CmpFOp>(
       loc, arith::CmpFPredicate::UNO, source, source);
-  Value isNonFinite = arith::OrIOp::create(rewriter,
-      loc, arith::OrIOp::create(rewriter, loc, isInf, isNegInf), isNan);
+  Value isNonFinite = arith::OrIOp::create(
+      rewriter, loc, arith::OrIOp::create(rewriter, loc, isInf, isNegInf),
+      isNan);
 
   Value clampedBelow = arith::MaximumFOp::create(rewriter, loc, source, minCst);
-  Value clamped = arith::MinimumFOp::create(rewriter, loc, clampedBelow, maxCst);
+  Value clamped =
+      arith::MinimumFOp::create(rewriter, loc, clampedBelow, maxCst);
   Value res =
       arith::SelectOp::create(rewriter, loc, isNonFinite, source, clamped);
   return res;
@@ -290,8 +292,8 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   VectorType truncResType = VectorType::get(4, outElemType);
   if (!inVectorTy) {
     Value asFloat = castToF32(in, loc, rewriter);
-    Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
-        loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
+    Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(
+        rewriter, loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
         /*existing=*/nullptr);
     Value result = vector::ExtractOp::create(rewriter, loc, asF8s, 0);
     rewriter.replaceOp(op, result);
@@ -299,8 +301,8 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   }
 
   int64_t numElements = outVecType.getNumElements();
-  Value zero = arith::ConstantOp::create(rewriter,
-      loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
+  Value zero = arith::ConstantOp::create(
+      rewriter, loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   if (outVecType.getShape().empty()) {
     Value scalarIn =
         vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
@@ -308,7 +310,7 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
     Value scalarTrunc =
         arith::TruncFOp::create(rewriter, loc, outElemType, scalarIn);
     Value result = vector::InsertOp::create(rewriter, loc, scalarTrunc, zero,
-                                                     ArrayRef<int64_t>{});
+                                            ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -334,14 +336,14 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
         Value elemB = vector::ExtractOp::create(rewriter, loc, in, i + j + 1);
         asFloatB = castToF32(elemB, loc, rewriter);
       }
-      thisResult = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
-          loc, truncResType, asFloatA, asFloatB, j / 2, thisResult);
+      thisResult = amdgpu::PackedTrunc2xFp8Op::create(
+          rewriter, loc, truncResType, asFloatA, asFloatB, j / 2, thisResult);
     }
     if (elemsThisOp < 4)
-      thisResult = vector::ExtractStridedSliceOp::create(rewriter,
-          loc, thisResult, 0, elemsThisOp, 1);
+      thisResult = vector::ExtractStridedSliceOp::create(
+          rewriter, loc, thisResult, 0, elemsThisOp, 1);
     result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
-                                                           result, i, 1);
+                                                  result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
@@ -407,10 +409,10 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
         ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, elemA, elemB);
     // Place back the truncated result into the possibly larger vector. If we
     // are operating on a size 2 vector, these operations should be folded away
-    thisResult = vector::ExtractStridedSliceOp::create(rewriter,
-        loc, thisResult, 0, elemsThisOp, 1);
+    thisResult = vector::ExtractStridedSliceOp::create(
+        rewriter, loc, thisResult, 0, elemsThisOp, 1);
     result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
-                                                           result, i, 1);
+                                                  result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
diff --git a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
index 40bf8f1cffb04..ba489436a1a4d 100644
--- a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
+++ b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
@@ -81,8 +81,8 @@ struct ConstantOpToArmSMELowering : public OpRewritePattern<arith::ConstantOp> {
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to write vector to tile
       // slice.
-      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
-          loc, tileType, constantOp1D, currentTile, tileSliceIndex);
+      auto nextTile = arm_sme::InsertTileSliceOp::create(
+          b, loc, tileType, constantOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
     auto forOp = mlir::arm_sme::createLoopOverTileSlices(
diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
index c5b3fdc536f72..59b3fe2e4eaed 100644
--- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
+++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
@@ -110,9 +110,9 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     emitc::CmpPredicate predicate;
     switch (op.getPredicate()) {
     case arith::CmpFPredicate::AlwaysFalse: {
-      auto constant = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), rewriter.getI1Type(),
-          rewriter.getBoolAttr(/*value=*/false));
+      auto constant =
+          emitc::ConstantOp::create(rewriter, op.getLoc(), rewriter.getI1Type(),
+                                    rewriter.getBoolAttr(/*value=*/false));
       rewriter.replaceOp(op, constant);
       return success();
     }
@@ -179,9 +179,9 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
       return success();
     }
     case arith::CmpFPredicate::AlwaysTrue: {
-      auto constant = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), rewriter.getI1Type(),
-          rewriter.getBoolAttr(/*value=*/true));
+      auto constant =
+          emitc::ConstantOp::create(rewriter, op.getLoc(), rewriter.getI1Type(),
+                                    rewriter.getBoolAttr(/*value=*/true));
       rewriter.replaceOp(op, constant);
       return success();
     }
@@ -190,7 +190,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     // Compare the values naively
     auto cmpResult =
         emitc::CmpOp::create(rewriter, op.getLoc(), op.getType(), predicate,
-                                      adaptor.getLhs(), adaptor.getRhs());
+                             adaptor.getLhs(), adaptor.getRhs());
 
     // Adjust the results for unordered/ordered semantics
     if (unordered) {
@@ -213,16 +213,16 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNaN(ConversionPatternRewriter &rewriter, Location loc,
               Value operand) const {
     // A value is NaN exactly when it compares unequal to itself.
-    return emitc::CmpOp::create(rewriter,
-        loc, rewriter.getI1Type(), emitc::CmpPredicate::ne, operand, operand);
+    return emitc::CmpOp::create(rewriter, loc, rewriter.getI1Type(),
+                                emitc::CmpPredicate::ne, operand, operand);
   }
 
   /// Return a value that is true if \p operand is not NaN.
   Value isNotNaN(ConversionPatternRewriter &rewriter, Location loc,
                  Value operand) const {
     // A value is not NaN exactly when it compares equal to itself.
-    return emitc::CmpOp::create(rewriter,
-        loc, rewriter.getI1Type(), emitc::CmpPredicate::eq, operand, operand);
+    return emitc::CmpOp::create(rewriter, loc, rewriter.getI1Type(),
+                                emitc::CmpPredicate::eq, operand, operand);
   }
 
   /// Return a value that is true if the operands \p first and \p second are
@@ -232,7 +232,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     auto firstIsNaN = isNaN(rewriter, loc, first);
     auto secondIsNaN = isNaN(rewriter, loc, second);
     return emitc::LogicalOrOp::create(rewriter, loc, rewriter.getI1Type(),
-                                               firstIsNaN, secondIsNaN);
+                                      firstIsNaN, secondIsNaN);
   }
 
   /// Return a value that is true if the operands \p first and \p second are
@@ -242,7 +242,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     auto firstIsNotNaN = isNotNaN(rewriter, loc, first);
     auto secondIsNotNaN = isNotNaN(rewriter, loc, second);
     return emitc::LogicalAndOp::create(rewriter, loc, rewriter.getI1Type(),
-                                                firstIsNotNaN, secondIsNotNaN);
+                                       firstIsNotNaN, secondIsNotNaN);
   }
 };
 
@@ -378,10 +378,10 @@ class CastConversion : public OpConversionPattern<ArithOp> {
       Type attrType = (emitc::isPointerWideType(operandType))
                           ? rewriter.getIndexType()
                           : operandType;
-      auto constOne = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), operandType, rewriter.getOneAttr(attrType));
-      auto oneAndOperand = emitc::BitwiseAndOp::create(rewriter,
-          op.getLoc(), operandType, adaptor.getIn(), constOne);
+      auto constOne = emitc::ConstantOp::create(
+          rewriter, op.getLoc(), operandType, rewriter.getOneAttr(attrType));
+      auto oneAndOperand = emitc::BitwiseAndOp::create(
+          rewriter, op.getLoc(), operandType, adaptor.getIn(), constOne);
       rewriter.replaceOpWithNewOp<emitc::CastOp>(op, opReturnType,
                                                  oneAndOperand);
       return success();
@@ -466,9 +466,8 @@ class BinaryUIOpConversion final : public OpConversionPattern<ArithOp> {
     Value lhsAdapted = adaptValueType(uiBinOp.getLhs(), rewriter, unsignedType);
     Value rhsAdapted = adaptValueType(uiBinOp.getRhs(), rewriter, unsignedType);
 
-    auto newDivOp =
-        EmitCOp::create(rewriter, uiBinOp.getLoc(), unsignedType,
-                                 ArrayRef<Value>{lhsAdapted, rhsAdapted});
+    auto newDivOp = EmitCOp::create(rewriter, uiBinOp.getLoc(), unsignedType,
+                                    ArrayRef<Value>{lhsAdapted, rhsAdapted});
     Value resultAdapted = adaptValueType(newDivOp, rewriter, newRetTy);
     rewriter.replaceOp(uiBinOp, resultAdapted);
     return success();
@@ -588,37 +587,39 @@ class ShiftOpConversion : public OpConversionPattern<ArithOp> {
     // Add a runtime check for overflow
     Value width;
     if (emitc::isPointerWideType(type)) {
-      Value eight = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), rhsType, rewriter.getIndexAttr(8));
-      emitc::CallOpaqueOp sizeOfCall = emitc::CallOpaqueOp::create(rewriter,
-          op.getLoc(), rhsType, "sizeof", ArrayRef<Value>{eight});
+      Value eight = emitc::ConstantOp::create(rewriter, op.getLoc(), rhsType,
+                                              rewriter.getIndexAttr(8));
+      emitc::CallOpaqueOp sizeOfCall = emitc::CallOpaqueOp::create(
+          rewriter, op.getLoc(), rhsType, "sizeof", ArrayRef<Value>{eight});
       width = emitc::MulOp::create(rewriter, op.getLoc(), rhsType, eight,
-                                            sizeOfCall.getResult(0));
+                                   sizeOfCall.getResult(0));
     } else {
-      width = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), rhsType,
+      width = emitc::ConstantOp::create(
+          rewriter, op.getLoc(), rhsType,
           rewriter.getIntegerAttr(rhsType, type.getIntOrFloatBitWidth()));
     }
 
-    Value excessCheck = emitc::CmpOp::create(rewriter,
-        op.getLoc(), rewriter.getI1Type(), emitc::CmpPredicate::lt, rhs, width);
+    Value excessCheck =
+        emitc::CmpOp::create(rewriter, op.getLoc(), rewriter.getI1Type(),
+                             emitc::CmpPredicate::lt, rhs, width);
 
     // Any concrete value is a valid refinement of poison.
-    Value poison = emitc::ConstantOp::create(rewriter,
-        op.getLoc(), arithmeticType,
+    Value poison = emitc::ConstantOp::create(
+        rewriter, op.getLoc(), arithmeticType,
         (isa<IntegerType>(arithmeticType)
              ? rewriter.getIntegerAttr(arithmeticType, 0)
              : rewriter.getIndexAttr(0)));
 
-    emitc::ExpressionOp ternary = emitc::ExpressionOp::create(rewriter,
-        op.getLoc(), arithmeticType, /*do_not_inline=*/false);
+    emitc::ExpressionOp ternary = emitc::ExpressionOp::create(
+        rewriter, op.getLoc(), arithmeticType, /*do_not_inline=*/false);
     Block &bodyBlock = ternary.getBodyRegion().emplaceBlock();
     auto currentPoint = rewriter.getInsertionPoint();
     rewriter.setInsertionPointToStart(&bodyBlock);
     Value arithmeticResult =
         EmitCOp::create(rewriter, op.getLoc(), arithmeticType, lhs, rhs);
-    Value resultOrPoison = emitc::ConditionalOp::create(rewriter,
-        op.getLoc(), arithmeticType, excessCheck, arithmeticResult, poison);
+    Value resultOrPoison =
+        emitc::ConditionalOp::create(rewriter, op.getLoc(), arithmeticType,
+                                     excessCheck, arithmeticResult, poison);
     emitc::YieldOp::create(rewriter, op.getLoc(), resultOrPoison);
     rewriter.setInsertionPoint(op->getBlock(), currentPoint);
 
@@ -700,11 +701,12 @@ class FtoICastOpConversion : public OpConversionPattern<CastOp> {
                                   /*isSigned=*/false);
     }
 
-    Value result = emitc::CastOp::create(rewriter,
-        castOp.getLoc(), actualResultType, adaptor.getOperands());
+    Value result = emitc::CastOp::create(
+        rewriter, castOp.getLoc(), actualResultType, adaptor.getOperands());
 
     if (isa<arith::FPToUIOp>(castOp)) {
-      result = emitc::CastOp::create(rewriter, castOp.getLoc(), dstType, result);
+      result =
+          emitc::CastOp::create(rewriter, castOp.getLoc(), dstType, result);
     }
     rewriter.replaceOp(castOp, result);
 
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index f7d4963b5e293..18e857c81af8d 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -294,10 +294,10 @@ LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
         typename OpTy::Adaptor adaptor(operands);
         if (targetBits < sourceBits) {
           return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
-                                                adaptor.getIn());
+                                       adaptor.getIn());
         }
         return ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
-                                          adaptor.getIn());
+                                 adaptor.getIn());
       },
       rewriter);
 }
@@ -324,8 +324,8 @@ LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
     Type newOverflowType = typeConverter->convertType(overflowResultType);
     Type structType =
         LLVM::LLVMStructType::getLiteral(ctx, {sumResultType, newOverflowType});
-    Value addOverflow = LLVM::UAddWithOverflowOp::create(rewriter,
-        loc, structType, adaptor.getLhs(), adaptor.getRhs());
+    Value addOverflow = LLVM::UAddWithOverflowOp::create(
+        rewriter, loc, structType, adaptor.getLhs(), adaptor.getRhs());
     Value sumExtracted =
         LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 0);
     Value overflowExtracted =
@@ -435,8 +435,8 @@ CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return LLVM::ICmpOp::create(rewriter,
-            op.getLoc(), llvm1DVectorTy,
+        return LLVM::ICmpOp::create(
+            rewriter, op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs());
       },
@@ -471,8 +471,8 @@ CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return LLVM::FCmpOp::create(rewriter,
-            op.getLoc(), llvm1DVectorTy,
+        return LLVM::FCmpOp::create(
+            rewriter, op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs(), fmf);
       },
diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index c93cf13f831ef..d43e6816641cb 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -121,8 +121,8 @@ static Value getScalarOrVectorConstInt(Type type, uint64_t value,
   }
 
   if (auto intType = dyn_cast<IntegerType>(type))
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getIntegerAttr(type, value));
+    return spirv::ConstantOp::create(builder, loc, type,
+                                     builder.getIntegerAttr(type, value));
 
   return nullptr;
 }
@@ -429,7 +429,8 @@ static Value emulateSignedRemainder(Location loc, Value lhs, Value rhs,
   else
     isPositive = spirv::IEqualOp::create(builder, loc, rhs, rhsAbs);
   Value absNegate = spirv::SNegateOp::create(builder, loc, type, abs);
-  return spirv::SelectOp::create(builder, loc, type, isPositive, abs, absNegate);
+  return spirv::SelectOp::create(builder, loc, type, isPositive, abs,
+                                 absNegate);
 }
 
 /// Converts arith.remsi to GLSL SPIR-V ops.
@@ -601,13 +602,13 @@ struct ExtSII1Pattern final : public OpConversionPattern<arith::ExtSIOp> {
     Value allOnes;
     if (auto intTy = dyn_cast<IntegerType>(dstType)) {
       unsigned componentBitwidth = intTy.getWidth();
-      allOnes = spirv::ConstantOp::create(rewriter,
-          loc, intTy,
+      allOnes = spirv::ConstantOp::create(
+          rewriter, loc, intTy,
           rewriter.getIntegerAttr(intTy, APInt::getAllOnes(componentBitwidth)));
     } else if (auto vectorTy = dyn_cast<VectorType>(dstType)) {
       unsigned componentBitwidth = vectorTy.getElementTypeBitWidth();
-      allOnes = spirv::ConstantOp::create(rewriter,
-          loc, vectorTy,
+      allOnes = spirv::ConstantOp::create(
+          rewriter, loc, vectorTy,
           SplatElementsAttr::get(vectorTy,
                                  APInt::getAllOnes(componentBitwidth)));
     } else {
@@ -653,8 +654,8 @@ struct ExtSIPattern final : public OpConversionPattern<arith::ExtSIOp> {
       // First shift left to sequeeze out all leading bits beyond the original
       // bitwidth. Here we need to use the original source and result type's
       // bitwidth.
-      auto shiftLOp = spirv::ShiftLeftLogicalOp::create(rewriter,
-          op.getLoc(), dstType, adaptor.getIn(), shiftSize);
+      auto shiftLOp = spirv::ShiftLeftLogicalOp::create(
+          rewriter, op.getLoc(), dstType, adaptor.getIn(), shiftSize);
 
       // Then we perform arithmetic right shift to make sure we have the right
       // sign bits for negative values.
@@ -757,8 +758,8 @@ struct TruncII1Pattern final : public OpConversionPattern<arith::TruncIOp> {
     auto srcType = adaptor.getOperands().front().getType();
     // Check if (x & 1) == 1.
     Value mask = spirv::ConstantOp::getOne(srcType, loc, rewriter);
-    Value maskedSrc = spirv::BitwiseAndOp::create(rewriter,
-        loc, srcType, adaptor.getOperands()[0], mask);
+    Value maskedSrc = spirv::BitwiseAndOp::create(
+        rewriter, loc, srcType, adaptor.getOperands()[0], mask);
     Value isOne = spirv::IEqualOp::create(rewriter, loc, maskedSrc, mask);
 
     Value zero = spirv::ConstantOp::getZero(dstType, loc, rewriter);
@@ -1095,12 +1096,12 @@ class AddUIExtendedOpPattern final
     Type dstElemTy = adaptor.getLhs().getType();
     Location loc = op->getLoc();
     Value result = spirv::IAddCarryOp::create(rewriter, loc, adaptor.getLhs(),
-                                                       adaptor.getRhs());
+                                              adaptor.getRhs());
 
-    Value sumResult = spirv::CompositeExtractOp::create(rewriter,
-        loc, result, llvm::ArrayRef(0));
-    Value carryValue = spirv::CompositeExtractOp::create(rewriter,
-        loc, result, llvm::ArrayRef(1));
+    Value sumResult = spirv::CompositeExtractOp::create(rewriter, loc, result,
+                                                        llvm::ArrayRef(0));
+    Value carryValue = spirv::CompositeExtractOp::create(rewriter, loc, result,
+                                                         llvm::ArrayRef(1));
 
     // Convert the carry value to boolean.
     Value one = spirv::ConstantOp::getOne(dstElemTy, loc, rewriter);
@@ -1128,9 +1129,9 @@ class MulIExtendedOpPattern final : public OpConversionPattern<ArithMulOp> {
         SPIRVMulOp::create(rewriter, loc, adaptor.getLhs(), adaptor.getRhs());
 
     Value low = spirv::CompositeExtractOp::create(rewriter, loc, result,
-                                                           llvm::ArrayRef(0));
+                                                  llvm::ArrayRef(0));
     Value high = spirv::CompositeExtractOp::create(rewriter, loc, result,
-                                                            llvm::ArrayRef(1));
+                                                   llvm::ArrayRef(1));
 
     rewriter.replaceOp(op, {low, high});
     return success();
@@ -1194,9 +1195,9 @@ class MinimumMaximumFOpPattern final : public OpConversionPattern<Op> {
     Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
     Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
-                                                     adaptor.getLhs(), spirvOp);
+                                            adaptor.getLhs(), spirvOp);
     Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
-                                                     adaptor.getRhs(), select1);
+                                            adaptor.getRhs(), select1);
 
     rewriter.replaceOp(op, select2);
     return success();
@@ -1249,9 +1250,9 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
     Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
     Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
-                                                     adaptor.getRhs(), spirvOp);
+                                            adaptor.getRhs(), spirvOp);
     Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
-                                                     adaptor.getLhs(), select1);
+                                            adaptor.getLhs(), select1);
 
     rewriter.replaceOp(op, select2);
     return success();
diff --git a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
index 2295d661d44a5..1510b0b16b07d 100644
--- a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
+++ b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
@@ -44,8 +44,8 @@ class Sdot2dLoweringPattern : public OpRewritePattern<Sdot2dOp> {
         vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, b2d);
     Value c1d =
         vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, c2d);
-    Value newOp = SdotOp::create(rewriter, loc, op.getRes().getType(), op.getA(),
-                                          b1d, c1d);
+    Value newOp = SdotOp::create(rewriter, loc, op.getRes().getType(),
+                                 op.getA(), b1d, c1d);
     rewriter.replaceOp(op, {newOp});
     return success();
   }
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index cf46e0c10df09..9bc3fa3473398 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -45,38 +45,38 @@ static Operation *createLoadTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return arm_sme::aarch64_sme_ld1b_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1b_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return arm_sme::aarch64_sme_ld1h_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1h_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return arm_sme::aarch64_sme_ld1w_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1w_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return arm_sme::aarch64_sme_ld1d_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1d_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return arm_sme::aarch64_sme_ld1q_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1q_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return arm_sme::aarch64_sme_ld1b_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1b_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return arm_sme::aarch64_sme_ld1h_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1h_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return arm_sme::aarch64_sme_ld1w_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1w_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return arm_sme::aarch64_sme_ld1d_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1d_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return arm_sme::aarch64_sme_ld1q_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1q_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
       break;
     }
   }
@@ -91,38 +91,38 @@ static Operation *createStoreTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return arm_sme::aarch64_sme_st1b_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1b_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return arm_sme::aarch64_sme_st1h_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1h_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return arm_sme::aarch64_sme_st1w_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1w_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return arm_sme::aarch64_sme_st1d_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1d_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return arm_sme::aarch64_sme_st1q_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1q_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return arm_sme::aarch64_sme_st1b_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1b_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return arm_sme::aarch64_sme_st1h_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1h_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return arm_sme::aarch64_sme_st1w_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1w_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return arm_sme::aarch64_sme_st1d_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1d_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return arm_sme::aarch64_sme_st1q_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1q_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     }
   }
   llvm_unreachable("unknown type in createStoreTileSliceIntrinsic");
@@ -154,8 +154,8 @@ createAllocaForTile(RewriterBase &rewriter, Location loc,
   auto minElementsOp =
       arith::ConstantIndexOp::create(rewriter, loc, minElements);
   auto vectorLen = arith::MulIOp::create(rewriter, loc, vscale, minElementsOp);
-  auto alloca = memref::AllocaOp::create(rewriter,
-      loc, memrefType, ValueRange{vectorLen, vectorLen});
+  auto alloca = memref::AllocaOp::create(rewriter, loc, memrefType,
+                                         ValueRange{vectorLen, vectorLen});
   return alloca;
 }
 
@@ -295,8 +295,8 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     auto descriptor =
         UnrealizedConversionCastOp::create(rewriter, loc, llvmType, tileMemory);
     auto zero = arith::ConstantIntOp::create(rewriter, loc, 0, /*width=*/64);
-    auto sliceIndexI64 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI64Type(), sliceIndex);
+    auto sliceIndexI64 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI64Type(), sliceIndex);
     return getStridedElementPtr(
         static_cast<ConversionPatternRewriter &>(rewriter), loc,
         llvm::cast<MemRefType>(tileMemory.getType()), descriptor.getResult(0),
@@ -309,20 +309,21 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                      arm_sme::ArmSMETileType tileType, VectorType sliceType,
                      IntegerAttr tileId, Value sliceIndex) const {
     // Cast the slice index to an i32.
-    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI32Type(), sliceIndex);
+    auto sliceIndexI32 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI32Type(), sliceIndex);
     // Create an all-true predicate for the slice.
     auto predicateType = sliceType.clone(rewriter.getI1Type());
-    auto allTruePredicate = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(predicateType, true));
+    auto allTruePredicate = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(predicateType, true));
     // Create padding vector (never used due to all-true predicate).
     auto padVector = LLVM::PoisonOp::create(rewriter, loc, sliceType);
     // Get a pointer to the current slice.
     auto slicePtr =
         getInMemoryTileSlicePtr(rewriter, loc, tileAlloca, sliceIndex);
     // Read the value of the current slice from ZA.
-    auto currentTileSlice = arm_sme::aarch64_sme_read_horiz::create(rewriter,
-        loc, sliceType, padVector, allTruePredicate, tileId, sliceIndexI32);
+    auto currentTileSlice = arm_sme::aarch64_sme_read_horiz::create(
+        rewriter, loc, sliceType, padVector, allTruePredicate, tileId,
+        sliceIndexI32);
     // Load the new tile slice back from memory into ZA.
     createLoadTileSliceIntrinsic(
         rewriter, loc, tileType, arm_sme::TileSliceLayout::Horizontal,
@@ -330,7 +331,7 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     // Store the current tile slice to memory.
     auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
     vector::StoreOp::create(rewriter, loc, currentTileSlice, tileAlloca,
-                                     ValueRange{sliceIndex, zero});
+                            ValueRange{sliceIndex, zero});
   }
 
   /// Emits a full in-place swap of the contents of a tile in ZA and a
@@ -343,10 +344,12 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     auto minNumElts =
         arith::ConstantIndexOp::create(rewriter, loc, sliceType.getDimSize(0));
     auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
-    auto upperBound = arith::MulIOp::create(rewriter,
-        loc, minNumElts, vector::VectorScaleOp::create(rewriter, loc));
+    auto upperBound =
+        arith::MulIOp::create(rewriter, loc, minNumElts,
+                              vector::VectorScaleOp::create(rewriter, loc));
     auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
+    auto forOp =
+        scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     // Emit a swap for each tile slice.
     rewriter.setInsertionPointToStart(forOp.getBody());
     auto sliceIndex = forOp.getInductionVar();
@@ -479,8 +482,8 @@ struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
     //
     // This holds for all tile sizes.
     int32_t zeroMask = baseMaskForSize << int32_t(tileId.getInt());
-    arm_sme::aarch64_sme_zero::create(rewriter,
-        loc, rewriter.getI32IntegerAttr(zeroMask));
+    arm_sme::aarch64_sme_zero::create(rewriter, loc,
+                                      rewriter.getI32IntegerAttr(zeroMask));
 
     // Create a placeholder op to preserve dataflow.
     // Note: Place the `get_tile` op at the start of the block. This ensures
@@ -513,8 +516,8 @@ struct LoadTileSliceConversion
     auto tileSlice = loadTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
-        loc, rewriter.getI32Type(), tileSlice);
+    auto tileSliceI32 = arith::IndexCastUIOp::create(
+        rewriter, loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
     auto maskOp = loadTileSliceOp.getMask();
@@ -559,8 +562,8 @@ struct StoreTileSliceConversion
     auto tileSlice = storeTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
-        loc, rewriter.getI32Type(), tileSlice);
+    auto tileSliceI32 = arith::IndexCastUIOp::create(
+        rewriter, loc, rewriter.getI32Type(), tileSlice);
 
     auto maskOp = storeTileSliceOp.getMask();
 
@@ -595,12 +598,12 @@ struct InsertTileSliceConversion
     auto tileSlice = insertTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
-        loc, rewriter.getI32Type(), tileSlice);
+    auto tileSliceI32 = arith::IndexCastUIOp::create(
+        rewriter, loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
-    auto one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getI1Type(),
+    auto one = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getI1Type(),
         rewriter.getIntegerAttr(rewriter.getI1Type(), 1));
     auto predTy = VectorType::get(tileType.getShape()[0], rewriter.getI1Type(),
                                   /*scalableDims=*/{true});
@@ -609,14 +612,14 @@ struct InsertTileSliceConversion
     // Create 'arm_sme.intr.write.(horiz|vert)' to write vector to tile slice.
     switch (insertTileSliceOp.getLayout()) {
     case arm_sme::TileSliceLayout::Horizontal:
-      arm_sme::aarch64_sme_write_horiz::create(rewriter,
-          loc, tileId, tileSliceI32, allActiveMask,
-          insertTileSliceOp.getVector());
+      arm_sme::aarch64_sme_write_horiz::create(rewriter, loc, tileId,
+                                               tileSliceI32, allActiveMask,
+                                               insertTileSliceOp.getVector());
       break;
     case arm_sme::TileSliceLayout::Vertical:
-      arm_sme::aarch64_sme_write_vert::create(rewriter,
-          loc, tileId, tileSliceI32, allActiveMask,
-          insertTileSliceOp.getVector());
+      arm_sme::aarch64_sme_write_vert::create(rewriter, loc, tileId,
+                                              tileSliceI32, allActiveMask,
+                                              insertTileSliceOp.getVector());
       break;
     }
 
@@ -646,16 +649,16 @@ struct ExtractTileSliceConversion
 
     // Create an 'all true' predicate for the tile slice.
     auto predicateType = sliceType.cloneWith({}, rewriter.getI1Type());
-    auto allTruePredicate = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(predicateType, true));
+    auto allTruePredicate = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(predicateType, true));
 
     // Zero destination/fallback for tile slice extraction.
-    auto zeroVector = arith::ConstantOp::create(rewriter,
-        loc, sliceType, rewriter.getZeroAttr(sliceType));
+    auto zeroVector = arith::ConstantOp::create(
+        rewriter, loc, sliceType, rewriter.getZeroAttr(sliceType));
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI32Type(), sliceIndex);
+    auto sliceIndexI32 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI32Type(), sliceIndex);
 
     // Create 'arm_sme.intr.read.(horiz|vert)' to extract the tile slice.
     switch (extractTileSlice.getLayout()) {
@@ -754,16 +757,16 @@ struct OuterProductOpConversion
     if (!lhsMask || !rhsMask) {
       auto predTy =
           outerProductOp.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = arith::ConstantOp::create(rewriter,
-          loc, DenseElementsAttr::get(predTy, true));
+      Value allActiveMask = arith::ConstantOp::create(
+          rewriter, loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
     // Create 'arm_sme.intr.mopa' outer product intrinsic.
     arm_sme::aarch64_sme_mopa::create(rewriter, loc, tileId, lhsMask, rhsMask,
-                                               outerProductOp.getLhs(),
-                                               outerProductOp.getRhs());
+                                      outerProductOp.getLhs(),
+                                      outerProductOp.getRhs());
 
     // The outerproduct intrinsics have no result, replace
     // 'arm_sme.outerproduct' with the input tile to preserve dataflow.
@@ -801,14 +804,14 @@ struct OuterProductWideningOpConversion
     Value rhsMask = op.getRhsMask();
     if (!lhsMask || !rhsMask) {
       auto predTy = op.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = arith::ConstantOp::create(rewriter,
-          loc, DenseElementsAttr::get(predTy, true));
+      Value allActiveMask = arith::ConstantOp::create(
+          rewriter, loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
-    OuterProductWideningIntrOp::create(rewriter,
-        loc, tileId, lhsMask, rhsMask, adaptor.getLhs(), adaptor.getRhs());
+    OuterProductWideningIntrOp::create(rewriter, loc, tileId, lhsMask, rhsMask,
+                                       adaptor.getLhs(), adaptor.getRhs());
 
     // The outerproduct intrinsics have no result, replace
     // 'arm_sme.outerproduct' with the input tile to preserve dataflow.
@@ -872,8 +875,8 @@ static void mergeConsecutiveTileZerosInBlock(Block *block) {
     if (zeroOpsToMerge.size() <= 1)
       return;
     IRRewriter rewriter(zeroOpsToMerge.front());
-    arm_sme::aarch64_sme_zero::create(rewriter,
-        zeroOpsToMerge.front().getLoc(),
+    arm_sme::aarch64_sme_zero::create(
+        rewriter, zeroOpsToMerge.front().getLoc(),
         rewriter.getI32IntegerAttr(mergedZeroMask));
     for (auto zeroOp : zeroOpsToMerge)
       rewriter.eraseOp(zeroOp);
diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 3591ed65f8d84..9a37b30c14813 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -59,8 +59,9 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   if (memrefIndices.size() != 2)
     return rewriter.notifyMatchFailure(loc, "invalid number of indices");
 
-  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
-      loc, arm_sme::getSMETileSliceMinNumElts(tileType.getElementType()));
+  auto minTileSlices = arith::ConstantIndexOp::create(
+      rewriter, loc,
+      arm_sme::getSMETileSliceMinNumElts(tileType.getElementType()));
   auto vscale =
       vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
   auto predicateType =
@@ -82,30 +83,30 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
     // The upper bound of the loop must be clamped at `numTileSlices` as
     // `vector.create_mask` allows operands to be greater than the size of a
     // dimension.
-    auto numRowI64 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI64Type(), maskDim0);
-    auto numTileSlicesI64 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI64Type(), numTileSlices);
+    auto numRowI64 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI64Type(), maskDim0);
+    auto numTileSlicesI64 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI64Type(), numTileSlices);
     auto upperBoundI64 =
         arith::MinSIOp::create(rewriter, loc, numRowI64, numTileSlicesI64);
-    upperBound = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getIndexType(), upperBoundI64);
+    upperBound = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getIndexType(), upperBoundI64);
 
     predicate =
         vector::CreateMaskOp::create(rewriter, loc, predicateType, maskDim1);
   } else {
     upperBound = numTileSlices;
     // No mask. Create an 'all true' predicate for the tile slice.
-    predicate = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(predicateType, true));
+    predicate = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(predicateType, true));
   }
 
   bool hasCarriedArgs = bool(initTile);
   auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
-                                           hasCarriedArgs ? ValueRange{initTile}
-                                                          : ValueRange{});
+  auto forOp =
+      scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
+                         hasCarriedArgs ? ValueRange{initTile} : ValueRange{});
 
   rewriter.setInsertionPointToStart(forOp.getBody());
   Value tileSliceIndex = forOp.getInductionVar();
@@ -207,9 +208,10 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
             Value currentTile) -> Value {
           // Create 'arm_sme.load_tile_slice' to load tile slice from memory
           // into tile.
-          return arm_sme::LoadTileSliceOp::create(rewriter,
-              loc, tileType, tileLoadOp.getBase(), predicate, currentTile,
-              memrefIndices, tileSliceIndex, tileLoadOp.getLayout());
+          return arm_sme::LoadTileSliceOp::create(
+              rewriter, loc, tileType, tileLoadOp.getBase(), predicate,
+              currentTile, memrefIndices, tileSliceIndex,
+              tileLoadOp.getLayout());
         });
 
     if (failed(forOp))
@@ -283,22 +285,22 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto numRows = createMaskOp.getOperands()[0];
     auto numCols = createMaskOp.getOperands()[1];
 
-    auto numColsI32 = arith::IndexCastUIOp::create(rewriter,
-        loc, rewriter.getI32Type(), numCols);
+    auto numColsI32 = arith::IndexCastUIOp::create(
+        rewriter, loc, rewriter.getI32Type(), numCols);
 
     auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     // Create a loop that loads each ZA tile slice from memory.
     auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
-        loc, arm_sme::getSMETileSliceMinNumElts(tileElementType));
+    auto minTileSlices = arith::ConstantIndexOp::create(
+        rewriter, loc, arm_sme::getSMETileSliceMinNumElts(tileElementType));
     auto vscale =
         vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
     auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto numTileSlices =
         arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
     auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices,
-                                             step, ValueRange{initTile});
+                                    step, ValueRange{initTile});
 
     rewriter.setInsertionPointToStart(forOp.getBody());
 
@@ -306,17 +308,18 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto currentTile = forOp.getRegionIterArg(0);
 
     // Combine masks.
-    auto rowIsActive = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ult, tileSliceIndex, numRows);
-    auto rowIsActiveI32 = arith::ExtSIOp::create(rewriter,
-        loc, rewriter.getI32Type(), rowIsActive);
-    auto mask = arith::AndIOp::create(rewriter, loc, rowIsActiveI32, numColsI32);
-    auto maskIndex =
-        arith::IndexCastOp::create(rewriter, loc, rewriter.getIndexType(), mask);
+    auto rowIsActive = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ult, tileSliceIndex, numRows);
+    auto rowIsActiveI32 = arith::ExtSIOp::create(
+        rewriter, loc, rewriter.getI32Type(), rowIsActive);
+    auto mask =
+        arith::AndIOp::create(rewriter, loc, rowIsActiveI32, numColsI32);
+    auto maskIndex = arith::IndexCastOp::create(rewriter, loc,
+                                                rewriter.getIndexType(), mask);
     auto predicateType =
         VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
-    auto maskOp1D = vector::CreateMaskOp::create(rewriter,
-        loc, predicateType, maskIndex.getResult());
+    auto maskOp1D = vector::CreateMaskOp::create(rewriter, loc, predicateType,
+                                                 maskIndex.getResult());
 
     auto memrefIndices = getMemrefIndices(
         tileLoadOp.getIndices(), tileLoadOp.getMemRefType().getRank(),
@@ -326,14 +329,15 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
     auto pad1DOp = vector::SplatOp::create(rewriter, loc, tileSliceType, padOp);
 
-    auto loadSlice = vector::MaskedLoadOp::create(rewriter,
-        loc, tileSliceType, tileLoadOp.getBase(), memrefIndices, maskOp1D,
-        /*passthru=*/pad1DOp);
+    auto loadSlice = vector::MaskedLoadOp::create(rewriter, loc, tileSliceType,
+                                                  tileLoadOp.getBase(),
+                                                  memrefIndices, maskOp1D,
+                                                  /*passthru=*/pad1DOp);
 
     // Create 'arm_sme.insert_tile_slice' to insert slice into tile.
-    auto insertSlice = arm_sme::InsertTileSliceOp::create(rewriter,
-        loc, tileType, loadSlice->getResult(0), currentTile, tileSliceIndex,
-        tileLoadOp.getLayout());
+    auto insertSlice = arm_sme::InsertTileSliceOp::create(
+        rewriter, loc, tileType, loadSlice->getResult(0), currentTile,
+        tileSliceIndex, tileLoadOp.getLayout());
     scf::YieldOp::create(rewriter, loc, insertSlice.getResult());
 
     rewriter.setInsertionPointAfter(forOp);
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index e1d28f19ee236..79e1683b4e2cf 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -254,8 +254,8 @@ static void addResumeFunction(ModuleOp module) {
   auto voidTy = LLVM::LLVMVoidType::get(ctx);
   Type ptrType = AsyncAPI::opaquePointerType(ctx);
 
-  auto resumeOp = LLVM::LLVMFuncOp::create(moduleBuilder,
-      kResume, LLVM::LLVMFunctionType::get(voidTy, {ptrType}));
+  auto resumeOp = LLVM::LLVMFuncOp::create(
+      moduleBuilder, kResume, LLVM::LLVMFunctionType::get(voidTy, {ptrType}));
   resumeOp.setPrivate();
 
   auto *block = resumeOp.addEntryBlock(moduleBuilder);
@@ -282,7 +282,8 @@ class AsyncRuntimeTypeConverter : public TypeConverter {
     // in patterns for other dialects.
     auto addUnrealizedCast = [](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+      auto cast =
+          UnrealizedConversionCastOp::create(builder, loc, type, inputs);
       return cast.getResult(0);
     };
 
@@ -382,7 +383,7 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
     // parameter.
     auto makeConstant = [&](uint64_t c) {
       return LLVM::ConstantOp::create(rewriter, op->getLoc(),
-                                               rewriter.getI64Type(), c);
+                                      rewriter.getI64Type(), c);
     };
     coroSize = LLVM::AddOp::create(rewriter, op->getLoc(), coroSize, coroAlign);
     coroSize =
@@ -397,8 +398,8 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
         rewriter, op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
     if (failed(allocFuncOp))
       return failure();
-    auto coroAlloc = LLVM::CallOp::create(rewriter,
-        loc, allocFuncOp.value(), ValueRange{coroAlign, coroSize});
+    auto coroAlloc = LLVM::CallOp::create(rewriter, loc, allocFuncOp.value(),
+                                          ValueRange{coroAlign, coroSize});
 
     // Begin a coroutine: @llvm.coro.begin.
     auto coroId = CoroBeginOpAdaptor(adaptor.getOperands()).getId();
@@ -455,15 +456,15 @@ class CoroEndOpConversion : public OpConversionPattern<CoroEndOp> {
   matchAndRewrite(CoroEndOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // We are not in the block that is part of the unwind sequence.
-    auto constFalse = LLVM::ConstantOp::create(rewriter,
-        op->getLoc(), rewriter.getI1Type(), rewriter.getBoolAttr(false));
+    auto constFalse =
+        LLVM::ConstantOp::create(rewriter, op->getLoc(), rewriter.getI1Type(),
+                                 rewriter.getBoolAttr(false));
     auto noneToken = LLVM::NoneTokenOp::create(rewriter, op->getLoc());
 
     // Mark the end of a coroutine: @llvm.coro.end.
     auto coroHdl = adaptor.getHandle();
-    LLVM::CoroEndOp::create(rewriter,
-        op->getLoc(), rewriter.getI1Type(),
-        ValueRange({coroHdl, constFalse, noneToken}));
+    LLVM::CoroEndOp::create(rewriter, op->getLoc(), rewriter.getI1Type(),
+                            ValueRange({coroHdl, constFalse, noneToken}));
     rewriter.eraseOp(op);
 
     return success();
@@ -534,13 +535,13 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     auto loc = op->getLoc();
 
     // This is not a final suspension point.
-    auto constFalse = LLVM::ConstantOp::create(rewriter,
-        loc, rewriter.getI1Type(), rewriter.getBoolAttr(false));
+    auto constFalse = LLVM::ConstantOp::create(
+        rewriter, loc, rewriter.getI1Type(), rewriter.getBoolAttr(false));
 
     // Suspend a coroutine: @llvm.coro.suspend
     auto coroState = adaptor.getState();
-    auto coroSuspend = LLVM::CoroSuspendOp::create(rewriter,
-        loc, i8, ValueRange({coroState, constFalse}));
+    auto coroSuspend = LLVM::CoroSuspendOp::create(
+        rewriter, loc, i8, ValueRange({coroState, constFalse}));
 
     // Cast return code to i32.
 
@@ -605,7 +606,7 @@ class RuntimeCreateOpLowering : public ConvertOpToLLVMPattern<RuntimeCreateOp> {
         auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, storagePtrType);
         auto gep =
             LLVM::GEPOp::create(rewriter, loc, storagePtrType, storedType,
-                                         nullPtr, ArrayRef<LLVM::GEPArg>{1});
+                                nullPtr, ArrayRef<LLVM::GEPArg>{1});
         return LLVM::PtrToIntOp::create(rewriter, loc, i64, gep);
       };
 
@@ -740,7 +741,7 @@ class RuntimeAwaitOpLowering : public OpConversionPattern<RuntimeAwaitOp> {
             .Case<GroupType>([](Type) { return kAwaitGroup; });
 
     func::CallOp::create(rewriter, op->getLoc(), apiFuncName, TypeRange(),
-                                  adaptor.getOperands());
+                         adaptor.getOperands());
     rewriter.eraseOp(op);
 
     return success();
@@ -772,13 +773,12 @@ class RuntimeAwaitAndResumeOpLowering
 
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
-        op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
-        kResume);
+    auto resumePtr = LLVM::AddressOfOp::create(
+        rewriter, op->getLoc(),
+        AsyncAPI::opaquePointerType(rewriter.getContext()), kResume);
 
-    func::CallOp::create(rewriter,
-        op->getLoc(), apiFuncName, TypeRange(),
-        ValueRange({operand, handle, resumePtr.getRes()}));
+    func::CallOp::create(rewriter, op->getLoc(), apiFuncName, TypeRange(),
+                         ValueRange({operand, handle, resumePtr.getRes()}));
     rewriter.eraseOp(op);
 
     return success();
@@ -801,9 +801,9 @@ class RuntimeResumeOpLowering
                   ConversionPatternRewriter &rewriter) const override {
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
-        op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
-        kResume);
+    auto resumePtr = LLVM::AddressOfOp::create(
+        rewriter, op->getLoc(),
+        AsyncAPI::opaquePointerType(rewriter.getContext()), kResume);
 
     // Call async runtime API to execute a coroutine in the managed thread.
     auto coroHdl = adaptor.getHandle();
@@ -832,8 +832,8 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = func::CallOp::create(rewriter,
-        loc, kGetValueStorage, TypeRange(ptrType), storage);
+    auto storagePtr = func::CallOp::create(rewriter, loc, kGetValueStorage,
+                                           TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
     auto valueType = op.getValue().getType();
@@ -872,8 +872,8 @@ class RuntimeLoadOpLowering : public ConvertOpToLLVMPattern<RuntimeLoadOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = func::CallOp::create(rewriter,
-        loc, kGetValueStorage, TypeRange(ptrType), storage);
+    auto storagePtr = func::CallOp::create(rewriter, loc, kGetValueStorage,
+                                           TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
     auto valueType = op.getResult().getType();
@@ -960,9 +960,9 @@ class RefCountingOpLowering : public OpConversionPattern<RefCountingOp> {
   LogicalResult
   matchAndRewrite(RefCountingOp op, typename RefCountingOp::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto count = arith::ConstantOp::create(rewriter,
-        op->getLoc(), rewriter.getI64Type(),
-        rewriter.getI64IntegerAttr(op.getCount()));
+    auto count =
+        arith::ConstantOp::create(rewriter, op->getLoc(), rewriter.getI64Type(),
+                                  rewriter.getI64IntegerAttr(op.getCount()));
 
     auto operand = adaptor.getOperand();
     rewriter.replaceOpWithNewOp<func::CallOp>(op, TypeRange(), apiFunctionName,
diff --git a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
index a63a1ceb00ebb..30a7170cf5c6a 100644
--- a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
+++ b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
@@ -103,7 +103,8 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
       }
 
       // Allocate a memref with identity layout.
-      alloc = memref::AllocOp::create(rewriter, loc, allocType, dynamicOperands);
+      alloc =
+          memref::AllocOp::create(rewriter, loc, allocType, dynamicOperands);
       // Cast the allocation to the specified type if needed.
       if (memrefType != allocType)
         alloc =
diff --git a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
index 1961e1004ba8e..14fbb9bf09545 100644
--- a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
+++ b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
@@ -23,35 +23,37 @@ void mlir::complex::convertDivToLLVMUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, LLVM::FastmathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = LLVM::FAddOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+  Value rhsSqNorm = LLVM::FAddOp::create(
+      rewriter, loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
       LLVM::FMulOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = LLVM::FAddOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+  Value realNumerator = LLVM::FAddOp::create(
+      rewriter, loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
       LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
 
-  Value imagNumerator = LLVM::FSubOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+  Value imagNumerator = LLVM::FSubOp::create(
+      rewriter, loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
       LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
-  *resultRe = LLVM::FDivOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
-  *resultIm = LLVM::FDivOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
+  *resultRe =
+      LLVM::FDivOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
+  *resultIm =
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 void mlir::complex::convertDivToStandardUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, arith::FastMathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = arith::AddFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+  Value rhsSqNorm = arith::AddFOp::create(
+      rewriter, loc, arith::MulFOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
       arith::MulFOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = arith::AddFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+  Value realNumerator = arith::AddFOp::create(
+      rewriter, loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
       arith::MulFOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
-  Value imagNumerator = arith::SubFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+  Value imagNumerator = arith::SubFOp::create(
+      rewriter, loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
       arith::MulFOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
   *resultRe =
@@ -95,57 +97,59 @@ void mlir::complex::convertDivToLLVMUsingRangeReduction(
 
   Value rhsRealImagRatio =
       LLVM::FDivOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = LLVM::FAddOp::create(rewriter,
-      loc, rhsIm,
+  Value rhsRealImagDenom = LLVM::FAddOp::create(
+      rewriter, loc, rhsIm,
       LLVM::FMulOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = LLVM::FAddOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
-      lhsIm, fmf);
-  Value resultReal1 =
-      LLVM::FDivOp::create(rewriter, loc, realNumerator1, rhsRealImagDenom, fmf);
-  Value imagNumerator1 = LLVM::FSubOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
-      lhsRe, fmf);
-  Value resultImag1 =
-      LLVM::FDivOp::create(rewriter, loc, imagNumerator1, rhsRealImagDenom, fmf);
+  Value realNumerator1 = LLVM::FAddOp::create(
+      rewriter, loc,
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf), lhsIm,
+      fmf);
+  Value resultReal1 = LLVM::FDivOp::create(rewriter, loc, realNumerator1,
+                                           rhsRealImagDenom, fmf);
+  Value imagNumerator1 = LLVM::FSubOp::create(
+      rewriter, loc,
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf), lhsRe,
+      fmf);
+  Value resultImag1 = LLVM::FDivOp::create(rewriter, loc, imagNumerator1,
+                                           rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
       LLVM::FDivOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = LLVM::FAddOp::create(rewriter,
-      loc, rhsRe,
+  Value rhsImagRealDenom = LLVM::FAddOp::create(
+      rewriter, loc, rhsRe,
       LLVM::FMulOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = LLVM::FAddOp::create(rewriter,
-      loc, lhsRe,
+  Value realNumerator2 = LLVM::FAddOp::create(
+      rewriter, loc, lhsRe,
       LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
-  Value resultReal2 =
-      LLVM::FDivOp::create(rewriter, loc, realNumerator2, rhsImagRealDenom, fmf);
-  Value imagNumerator2 = LLVM::FSubOp::create(rewriter,
-      loc, lhsIm,
+  Value resultReal2 = LLVM::FDivOp::create(rewriter, loc, realNumerator2,
+                                           rhsImagRealDenom, fmf);
+  Value imagNumerator2 = LLVM::FSubOp::create(
+      rewriter, loc, lhsIm,
       LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
-  Value resultImag2 =
-      LLVM::FDivOp::create(rewriter, loc, imagNumerator2, rhsImagRealDenom, fmf);
+  Value resultImag2 = LLVM::FDivOp::create(rewriter, loc, imagNumerator2,
+                                           rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = LLVM::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getZeroAttr(elementType));
+  Value zero = LLVM::ConstantOp::create(rewriter, loc, elementType,
+                                        rewriter.getZeroAttr(elementType));
   Value rhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, rhsRe, fmf);
-  Value rhsRealIsZero = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, zero);
+  Value rhsRealIsZero = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, zero);
   Value rhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, rhsIm, fmf);
-  Value rhsImagIsZero = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, zero);
-  Value lhsRealIsNotNaN =
-      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
-  Value lhsImagIsNotNaN =
-      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
+  Value rhsImagIsZero = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, zero);
+  Value lhsRealIsNotNaN = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
+  Value lhsImagIsNotNaN = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
   Value lhsContainsNotNaNValue =
       LLVM::OrOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = LLVM::AndOp::create(rewriter,
-      loc, lhsContainsNotNaNValue,
+  Value resultIsInfinity = LLVM::AndOp::create(
+      rewriter, loc, lhsContainsNotNaNValue,
       LLVM::AndOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = LLVM::ConstantOp::create(rewriter,
-      loc, elementType,
+  Value inf = LLVM::ConstantOp::create(
+      rewriter, loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfrhsReal =
@@ -156,119 +160,121 @@ void mlir::complex::convertDivToLLVMUsingRangeReduction(
       LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::one, rhsRealAbs, inf);
-  Value rhsImagFinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::one, rhsImagAbs, inf);
+  Value rhsRealFinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::one, rhsRealAbs, inf);
+  Value rhsImagFinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::one, rhsImagAbs, inf);
   Value rhsFinite =
       LLVM::AndOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
   Value lhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, lhsRe, fmf);
-  Value lhsRealInfinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, lhsRealAbs, inf);
+  Value lhsRealInfinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, lhsRealAbs, inf);
   Value lhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, lhsIm, fmf);
-  Value lhsImagInfinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, lhsImagAbs, inf);
+  Value lhsImagInfinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, lhsImagAbs, inf);
   Value lhsInfinite =
       LLVM::OrOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
       LLVM::AndOp::create(rewriter, loc, lhsInfinite, rhsFinite);
-  Value one = LLVM::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
-      loc, LLVM::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
-      lhsRe);
-  Value lhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
-      loc, LLVM::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
-      lhsIm);
+  Value one = LLVM::ConstantOp::create(rewriter, loc, elementType,
+                                       rewriter.getFloatAttr(elementType, 1));
+  Value lhsRealIsInfWithSign = LLVM::CopySignOp::create(
+      rewriter, loc,
+      LLVM::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero), lhsRe);
+  Value lhsImagIsInfWithSign = LLVM::CopySignOp::create(
+      rewriter, loc,
+      LLVM::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero), lhsIm);
   Value lhsRealIsInfWithSignTimesrhsReal =
       LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesrhsImag =
       LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = LLVM::FMulOp::create(rewriter,
-      loc, inf,
+  Value resultReal3 = LLVM::FMulOp::create(
+      rewriter, loc, inf,
       LLVM::FAddOp::create(rewriter, loc, lhsRealIsInfWithSignTimesrhsReal,
-                                    lhsImagIsInfWithSignTimesrhsImag, fmf),
+                           lhsImagIsInfWithSignTimesrhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesrhsImag =
       LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesrhsReal =
       LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = LLVM::FMulOp::create(rewriter,
-      loc, inf,
+  Value resultImag3 = LLVM::FMulOp::create(
+      rewriter, loc, inf,
       LLVM::FSubOp::create(rewriter, loc, lhsImagIsInfWithSignTimesrhsReal,
-                                    lhsRealIsInfWithSignTimesrhsImag, fmf),
+                           lhsRealIsInfWithSignTimesrhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::one, lhsRealAbs, inf);
-  Value lhsImagFinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::one, lhsImagAbs, inf);
+  Value lhsRealFinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::one, lhsRealAbs, inf);
+  Value lhsImagFinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::one, lhsImagAbs, inf);
   Value lhsFinite =
       LLVM::AndOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, inf);
-  Value rhsImagInfinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, inf);
+  Value rhsRealInfinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, inf);
+  Value rhsImagInfinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, inf);
   Value rhsInfinite =
       LLVM::OrOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
       LLVM::AndOp::create(rewriter, loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
-      loc, LLVM::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
-      rhsRe);
-  Value rhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
-      loc, LLVM::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
-      rhsIm);
+  Value rhsRealIsInfWithSign = LLVM::CopySignOp::create(
+      rewriter, loc,
+      LLVM::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero), rhsRe);
+  Value rhsImagIsInfWithSign = LLVM::CopySignOp::create(
+      rewriter, loc,
+      LLVM::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero), rhsIm);
   Value rhsRealIsInfWithSignTimeslhsReal =
       LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsImag =
       LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = LLVM::FMulOp::create(rewriter,
-      loc, zero,
+  Value resultReal4 = LLVM::FMulOp::create(
+      rewriter, loc, zero,
       LLVM::FAddOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsReal,
-                                    rhsImagIsInfWithSignTimeslhsImag, fmf),
+                           rhsImagIsInfWithSignTimeslhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimeslhsImag =
       LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsReal =
       LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = LLVM::FMulOp::create(rewriter,
-      loc, zero,
+  Value resultImag4 = LLVM::FMulOp::create(
+      rewriter, loc, zero,
       LLVM::FSubOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsImag,
-                                    rhsImagIsInfWithSignTimeslhsReal, fmf),
+                           rhsImagIsInfWithSignTimeslhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::olt, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = LLVM::SelectOp::create(rewriter,
-      loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = LLVM::SelectOp::create(rewriter,
-      loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = LLVM::SelectOp::create(rewriter,
-      loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = LLVM::SelectOp::create(rewriter,
-      loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = LLVM::SelectOp::create(rewriter,
-      loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = LLVM::SelectOp::create(rewriter,
-      loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = LLVM::SelectOp::create(rewriter,
-      loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = LLVM::SelectOp::create(rewriter,
-      loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
+  Value realAbsSmallerThanImagAbs = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::olt, rhsRealAbs, rhsImagAbs);
+  Value resultReal5 = LLVM::SelectOp::create(
+      rewriter, loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
+  Value resultImag5 = LLVM::SelectOp::create(
+      rewriter, loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
+  Value resultRealSpecialCase3 = LLVM::SelectOp::create(
+      rewriter, loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
+  Value resultImagSpecialCase3 = LLVM::SelectOp::create(
+      rewriter, loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
+  Value resultRealSpecialCase2 = LLVM::SelectOp::create(
+      rewriter, loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
+  Value resultImagSpecialCase2 = LLVM::SelectOp::create(
+      rewriter, loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
+  Value resultRealSpecialCase1 =
+      LLVM::SelectOp::create(rewriter, loc, resultIsInfinity,
+                             infinityResultReal, resultRealSpecialCase2);
+  Value resultImagSpecialCase1 =
+      LLVM::SelectOp::create(rewriter, loc, resultIsInfinity,
+                             infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::uno, resultReal5, zero);
-  Value resultImagIsNaN = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::uno, resultImag5, zero);
+  Value resultRealIsNaN = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::uno, resultReal5, zero);
+  Value resultImagIsNaN = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::uno, resultImag5, zero);
   Value resultIsNaN =
       LLVM::AndOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = LLVM::SelectOp::create(rewriter,
-      loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = LLVM::SelectOp::create(rewriter,
-      loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
+  *resultRe = LLVM::SelectOp::create(rewriter, loc, resultIsNaN,
+                                     resultRealSpecialCase1, resultReal5);
+  *resultIm = LLVM::SelectOp::create(rewriter, loc, resultIsNaN,
+                                     resultImagSpecialCase1, resultImag5);
 }
 
 void mlir::complex::convertDivToStandardUsingRangeReduction(
@@ -279,57 +285,59 @@ void mlir::complex::convertDivToStandardUsingRangeReduction(
 
   Value rhsRealImagRatio =
       arith::DivFOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = arith::AddFOp::create(rewriter,
-      loc, rhsIm,
+  Value rhsRealImagDenom = arith::AddFOp::create(
+      rewriter, loc, rhsIm,
       arith::MulFOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = arith::AddFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
-      lhsIm, fmf);
+  Value realNumerator1 = arith::AddFOp::create(
+      rewriter, loc,
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf), lhsIm,
+      fmf);
   Value resultReal1 = arith::DivFOp::create(rewriter, loc, realNumerator1,
-                                                     rhsRealImagDenom, fmf);
-  Value imagNumerator1 = arith::SubFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
-      lhsRe, fmf);
+                                            rhsRealImagDenom, fmf);
+  Value imagNumerator1 = arith::SubFOp::create(
+      rewriter, loc,
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf), lhsRe,
+      fmf);
   Value resultImag1 = arith::DivFOp::create(rewriter, loc, imagNumerator1,
-                                                     rhsRealImagDenom, fmf);
+                                            rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
       arith::DivFOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = arith::AddFOp::create(rewriter,
-      loc, rhsRe,
+  Value rhsImagRealDenom = arith::AddFOp::create(
+      rewriter, loc, rhsRe,
       arith::MulFOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = arith::AddFOp::create(rewriter,
-      loc, lhsRe,
+  Value realNumerator2 = arith::AddFOp::create(
+      rewriter, loc, lhsRe,
       arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
   Value resultReal2 = arith::DivFOp::create(rewriter, loc, realNumerator2,
-                                                     rhsImagRealDenom, fmf);
-  Value imagNumerator2 = arith::SubFOp::create(rewriter,
-      loc, lhsIm,
+                                            rhsImagRealDenom, fmf);
+  Value imagNumerator2 = arith::SubFOp::create(
+      rewriter, loc, lhsIm,
       arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
   Value resultImag2 = arith::DivFOp::create(rewriter, loc, imagNumerator2,
-                                                     rhsImagRealDenom, fmf);
+                                            rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = arith::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getZeroAttr(elementType));
+  Value zero = arith::ConstantOp::create(rewriter, loc, elementType,
+                                         rewriter.getZeroAttr(elementType));
   Value rhsRealAbs = math::AbsFOp::create(rewriter, loc, rhsRe, fmf);
-  Value rhsRealIsZero = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, rhsRealAbs, zero);
+  Value rhsRealIsZero = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, rhsRealAbs, zero);
   Value rhsImagAbs = math::AbsFOp::create(rewriter, loc, rhsIm, fmf);
-  Value rhsImagIsZero = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, rhsImagAbs, zero);
-  Value lhsRealIsNotNaN = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ORD, lhsRe, zero);
-  Value lhsImagIsNotNaN = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ORD, lhsIm, zero);
+  Value rhsImagIsZero = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, rhsImagAbs, zero);
+  Value lhsRealIsNotNaN = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ORD, lhsRe, zero);
+  Value lhsImagIsNotNaN = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ORD, lhsIm, zero);
   Value lhsContainsNotNaNValue =
       arith::OrIOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = arith::AndIOp::create(rewriter,
-      loc, lhsContainsNotNaNValue,
+  Value resultIsInfinity = arith::AndIOp::create(
+      rewriter, loc, lhsContainsNotNaNValue,
       arith::AndIOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = arith::ConstantOp::create(rewriter,
-      loc, elementType,
+  Value inf = arith::ConstantOp::create(
+      rewriter, loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfRhsReal =
@@ -340,117 +348,123 @@ void mlir::complex::convertDivToStandardUsingRangeReduction(
       arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ONE, rhsRealAbs, inf);
-  Value rhsImagFinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ONE, rhsImagAbs, inf);
+  Value rhsRealFinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ONE, rhsRealAbs, inf);
+  Value rhsImagFinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ONE, rhsImagAbs, inf);
   Value rhsFinite =
       arith::AndIOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
   Value lhsRealAbs = math::AbsFOp::create(rewriter, loc, lhsRe, fmf);
-  Value lhsRealInfinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, lhsRealAbs, inf);
+  Value lhsRealInfinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, lhsRealAbs, inf);
   Value lhsImagAbs = math::AbsFOp::create(rewriter, loc, lhsIm, fmf);
-  Value lhsImagInfinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, lhsImagAbs, inf);
+  Value lhsImagInfinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, lhsImagAbs, inf);
   Value lhsInfinite =
       arith::OrIOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
       arith::AndIOp::create(rewriter, loc, lhsInfinite, rhsFinite);
-  Value one = arith::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
-      loc, arith::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
+  Value one = arith::ConstantOp::create(rewriter, loc, elementType,
+                                        rewriter.getFloatAttr(elementType, 1));
+  Value lhsRealIsInfWithSign = math::CopySignOp::create(
+      rewriter, loc,
+      arith::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
-      loc, arith::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = math::CopySignOp::create(
+      rewriter, loc,
+      arith::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesRhsReal =
       arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesRhsImag =
       arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = arith::MulFOp::create(rewriter,
-      loc, inf,
+  Value resultReal3 = arith::MulFOp::create(
+      rewriter, loc, inf,
       arith::AddFOp::create(rewriter, loc, lhsRealIsInfWithSignTimesRhsReal,
-                                     lhsImagIsInfWithSignTimesRhsImag, fmf),
+                            lhsImagIsInfWithSignTimesRhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesRhsImag =
       arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesRhsReal =
       arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = arith::MulFOp::create(rewriter,
-      loc, inf,
+  Value resultImag3 = arith::MulFOp::create(
+      rewriter, loc, inf,
       arith::SubFOp::create(rewriter, loc, lhsImagIsInfWithSignTimesRhsReal,
-                                     lhsRealIsInfWithSignTimesRhsImag, fmf),
+                            lhsRealIsInfWithSignTimesRhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ONE, lhsRealAbs, inf);
-  Value lhsImagFinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ONE, lhsImagAbs, inf);
+  Value lhsRealFinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ONE, lhsRealAbs, inf);
+  Value lhsImagFinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ONE, lhsImagAbs, inf);
   Value lhsFinite =
       arith::AndIOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, rhsRealAbs, inf);
-  Value rhsImagInfinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, rhsImagAbs, inf);
+  Value rhsRealInfinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, rhsRealAbs, inf);
+  Value rhsImagInfinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, rhsImagAbs, inf);
   Value rhsInfinite =
       arith::OrIOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
       arith::AndIOp::create(rewriter, loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
-      loc, arith::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
+  Value rhsRealIsInfWithSign = math::CopySignOp::create(
+      rewriter, loc,
+      arith::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
-      loc, arith::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = math::CopySignOp::create(
+      rewriter, loc,
+      arith::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimesLhsReal =
       arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsImag =
       arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = arith::MulFOp::create(rewriter,
-      loc, zero,
+  Value resultReal4 = arith::MulFOp::create(
+      rewriter, loc, zero,
       arith::AddFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsReal,
-                                     rhsImagIsInfWithSignTimesLhsImag, fmf),
+                            rhsImagIsInfWithSignTimesLhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimesLhsImag =
       arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsReal =
       arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = arith::MulFOp::create(rewriter,
-      loc, zero,
+  Value resultImag4 = arith::MulFOp::create(
+      rewriter, loc, zero,
       arith::SubFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsImag,
-                                     rhsImagIsInfWithSignTimesLhsReal, fmf),
+                            rhsImagIsInfWithSignTimesLhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OLT, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = arith::SelectOp::create(rewriter,
-      loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = arith::SelectOp::create(rewriter,
-      loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = arith::SelectOp::create(rewriter,
-      loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = arith::SelectOp::create(rewriter,
-      loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = arith::SelectOp::create(rewriter,
-      loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = arith::SelectOp::create(rewriter,
-      loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = arith::SelectOp::create(rewriter,
-      loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = arith::SelectOp::create(rewriter,
-      loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
+  Value realAbsSmallerThanImagAbs = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OLT, rhsRealAbs, rhsImagAbs);
+  Value resultReal5 = arith::SelectOp::create(
+      rewriter, loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
+  Value resultImag5 = arith::SelectOp::create(
+      rewriter, loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
+  Value resultRealSpecialCase3 = arith::SelectOp::create(
+      rewriter, loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
+  Value resultImagSpecialCase3 = arith::SelectOp::create(
+      rewriter, loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
+  Value resultRealSpecialCase2 = arith::SelectOp::create(
+      rewriter, loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
+  Value resultImagSpecialCase2 = arith::SelectOp::create(
+      rewriter, loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
+  Value resultRealSpecialCase1 =
+      arith::SelectOp::create(rewriter, loc, resultIsInfinity,
+                              infinityResultReal, resultRealSpecialCase2);
+  Value resultImagSpecialCase1 =
+      arith::SelectOp::create(rewriter, loc, resultIsInfinity,
+                              infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::UNO, resultReal5, zero);
-  Value resultImagIsNaN = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::UNO, resultImag5, zero);
+  Value resultRealIsNaN = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::UNO, resultReal5, zero);
+  Value resultImagIsNaN = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::UNO, resultImag5, zero);
   Value resultIsNaN =
       arith::AndIOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = arith::SelectOp::create(rewriter,
-      loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = arith::SelectOp::create(rewriter,
-      loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
+  *resultRe = arith::SelectOp::create(rewriter, loc, resultIsNaN,
+                                      resultRealSpecialCase1, resultReal5);
+  *resultIm = arith::SelectOp::create(rewriter, loc, resultIsNaN,
+                                      resultImagSpecialCase1, resultImag5);
 }
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index a06935c4387c6..86d02e6c6209f 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -79,8 +79,8 @@ struct AbsOpConversion : public ConvertOpToLLVMPattern<complex::AbsOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value sqNorm = LLVM::FAddOp::create(rewriter,
-        loc, LLVM::FMulOp::create(rewriter, loc, real, real, fmf),
+    Value sqNorm = LLVM::FAddOp::create(
+        rewriter, loc, LLVM::FMulOp::create(rewriter, loc, real, real, fmf),
         LLVM::FMulOp::create(rewriter, loc, imag, imag, fmf), fmf);
 
     rewriter.replaceOpWithNewOp<LLVM::SqrtOp>(op, sqNorm);
@@ -191,10 +191,10 @@ struct AddOpConversion : public ConvertOpToLLVMPattern<complex::AddOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value real =
-        LLVM::FAddOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
-    Value imag =
-        LLVM::FAddOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+    Value real = LLVM::FAddOp::create(rewriter, loc, arg.lhs.real(),
+                                      arg.rhs.real(), fmf);
+    Value imag = LLVM::FAddOp::create(rewriter, loc, arg.lhs.imag(),
+                                      arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
@@ -278,12 +278,12 @@ struct MulOpConversion : public ConvertOpToLLVMPattern<complex::MulOp> {
     Value lhsRe = arg.lhs.real();
     Value lhsIm = arg.lhs.imag();
 
-    Value real = LLVM::FSubOp::create(rewriter,
-        loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, lhsRe, fmf),
+    Value real = LLVM::FSubOp::create(
+        rewriter, loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, lhsRe, fmf),
         LLVM::FMulOp::create(rewriter, loc, rhsIm, lhsIm, fmf), fmf);
 
-    Value imag = LLVM::FAddOp::create(rewriter,
-        loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+    Value imag = LLVM::FAddOp::create(
+        rewriter, loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
         LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
     result.setReal(rewriter, loc, real);
@@ -313,10 +313,10 @@ struct SubOpConversion : public ConvertOpToLLVMPattern<complex::SubOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value real =
-        LLVM::FSubOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
-    Value imag =
-        LLVM::FSubOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+    Value real = LLVM::FSubOp::create(rewriter, loc, arg.lhs.real(),
+                                      arg.rhs.real(), fmf);
+    Value imag = LLVM::FSubOp::create(rewriter, loc, arg.lhs.imag(),
+                                      arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
diff --git a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
index 4800f65c675e6..f83cac751ff05 100644
--- a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
+++ b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
@@ -85,7 +85,7 @@ LogicalResult ScalarOpToLibmCall<Op, TypeResolver>::matchAndRewrite(
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
     opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
-                                           opFunctionTy);
+                                  opFunctionTy);
     opFunc.setPrivate();
   }
   assert(isa<FunctionOpInterface>(SymbolTable::lookupSymbolIn(module, name)));
diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index d14e3888b118c..eeff8a93e7a72 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -32,7 +32,7 @@ enum class AbsFn { abs, sqrt, rsqrt };
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
                  ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
   Value one = arith::ConstantOp::create(b, real.getType(),
-                                          b.getFloatAttr(real.getType(), 1.0));
+                                        b.getFloatAttr(real.getType(), 1.0));
 
   Value absReal = math::AbsFOp::create(b, real, fmf);
   Value absImag = math::AbsFOp::create(b, imag, fmf);
@@ -55,11 +55,12 @@ Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
   }
 
   if (fn == AbsFn::sqrt) {
-    Value quarter = arith::ConstantOp::create(b,
-        real.getType(), b.getFloatAttr(real.getType(), 0.25));
+    Value quarter = arith::ConstantOp::create(
+        b, real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
     Value sqrt = math::SqrtOp::create(b, max, fmfWithNaNInf);
-    Value p025 = math::PowFOp::create(b, ratioSqPlusOne, quarter, fmfWithNaNInf);
+    Value p025 =
+        math::PowFOp::create(b, ratioSqPlusOne, quarter, fmfWithNaNInf);
     result = arith::MulFOp::create(b, sqrt, p025, fmfWithNaNInf);
   } else {
     Value sqrt = math::SqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
@@ -67,7 +68,7 @@ Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
   }
 
   Value isNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, result,
-                                        result, fmfWithNaNInf);
+                                      result, fmfWithNaNInf);
   return arith::SelectOp::create(b, isNaN, min, result);
 }
 
@@ -115,17 +116,17 @@ struct Atan2OpConversion : public OpConversionPattern<complex::Atan2Op> {
     Value zero =
         arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
     Value one = arith::ConstantOp::create(b, elementType,
-                                            b.getFloatAttr(elementType, 1));
+                                          b.getFloatAttr(elementType, 1));
     Value i = complex::CreateOp::create(b, type, zero, one);
     Value iTimesLhs = complex::MulOp::create(b, i, lhs, fmf);
     Value rhsPlusILhs = complex::AddOp::create(b, rhs, iTimesLhs, fmf);
 
-    Value divResult = complex::DivOp::create(b,
-        rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf);
+    Value divResult = complex::DivOp::create(
+        b, rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf);
     Value logResult = complex::LogOp::create(b, divResult, fmf);
 
-    Value negativeOne = arith::ConstantOp::create(b,
-        elementType, b.getFloatAttr(elementType, -1));
+    Value negativeOne = arith::ConstantOp::create(
+        b, elementType, b.getFloatAttr(elementType, -1));
     Value negativeI = complex::CreateOp::create(b, type, zero, negativeOne);
 
     rewriter.replaceOpWithNewOp<complex::MulOp>(op, negativeI, logResult, fmf);
@@ -146,10 +147,14 @@ struct ComparisonOpConversion : public OpConversionPattern<ComparisonOp> {
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getLhs().getType()).getElementType();
 
-    Value realLhs = complex::ReOp::create(rewriter, loc, type, adaptor.getLhs());
-    Value imagLhs = complex::ImOp::create(rewriter, loc, type, adaptor.getLhs());
-    Value realRhs = complex::ReOp::create(rewriter, loc, type, adaptor.getRhs());
-    Value imagRhs = complex::ImOp::create(rewriter, loc, type, adaptor.getRhs());
+    Value realLhs =
+        complex::ReOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value imagLhs =
+        complex::ImOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value realRhs =
+        complex::ReOp::create(rewriter, loc, type, adaptor.getRhs());
+    Value imagRhs =
+        complex::ImOp::create(rewriter, loc, type, adaptor.getRhs());
     Value realComparison =
         arith::CmpFOp::create(rewriter, loc, p, realLhs, realRhs);
     Value imagComparison =
@@ -178,12 +183,12 @@ struct BinaryComplexOpConversion : public OpConversionPattern<BinaryComplexOp> {
 
     Value realLhs = complex::ReOp::create(b, elementType, adaptor.getLhs());
     Value realRhs = complex::ReOp::create(b, elementType, adaptor.getRhs());
-    Value resultReal = BinaryStandardOp::create(b, elementType, realLhs, realRhs,
-                                                  fmf.getValue());
+    Value resultReal = BinaryStandardOp::create(b, elementType, realLhs,
+                                                realRhs, fmf.getValue());
     Value imagLhs = complex::ImOp::create(b, elementType, adaptor.getLhs());
     Value imagRhs = complex::ImOp::create(b, elementType, adaptor.getRhs());
-    Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs, imagRhs,
-                                                  fmf.getValue());
+    Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs,
+                                                imagRhs, fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -212,8 +217,8 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
     // Trigonometric ops use a set of common building blocks to convert to real
     // ops. Here we create these building blocks and call into an op-specific
     // implementation in the subclass to combine them.
-    Value half = arith::ConstantOp::create(rewriter,
-        loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
+    Value half = arith::ConstantOp::create(
+        rewriter, loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
     Value exp = math::ExpOp::create(rewriter, loc, imag, fmf);
     Value scaledExp = arith::MulFOp::create(rewriter, loc, half, exp, fmf);
     Value reciprocalExp = arith::DivFOp::create(rewriter, loc, half, exp, fmf);
@@ -342,8 +347,8 @@ Value evaluatePolynomial(ImplicitLocOpBuilder &b, Value arg,
   Value poly =
       arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[0]));
   for (unsigned i = 1; i < coefficients.size(); ++i) {
-    poly = math::FmaOp::create(b,
-        poly, arg,
+    poly = math::FmaOp::create(
+        b, poly, arg,
         arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[i])),
         fmf);
   }
@@ -378,13 +383,13 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
     Value cosm1Imag = emitCosm1(imag, fmf, b);
     Value cosImag = arith::AddFOp::create(b, cosm1Imag, one, fmf);
 
-    Value realResult = arith::AddFOp::create(b,
-        arith::MulFOp::create(b, expm1Real, cosImag, fmf), cosm1Imag, fmf);
+    Value realResult = arith::AddFOp::create(
+        b, arith::MulFOp::create(b, expm1Real, cosImag, fmf), cosm1Imag, fmf);
 
     Value imagIsZero = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag,
-                                               zero, fmf.getValue());
-    Value imagResult = arith::SelectOp::create(b,
-        imagIsZero, zero, arith::MulFOp::create(b, expReal, sinImag, fmf));
+                                             zero, fmf.getValue());
+    Value imagResult = arith::SelectOp::create(
+        b, imagIsZero, zero, arith::MulFOp::create(b, expReal, sinImag, fmf));
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, realResult,
                                                    imagResult);
@@ -414,13 +419,13 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
 
     auto forSmallArg =
         arith::AddFOp::create(b, arith::MulFOp::create(b, argPow4, poly, fmf),
-                                arith::MulFOp::create(b, negHalf, argPow2, fmf));
+                              arith::MulFOp::create(b, negHalf, argPow2, fmf));
 
     // (pi/4)^2 is approximately 0.61685
     Value piOver4Pow2 =
         arith::ConstantOp::create(b, b.getFloatAttr(argType, 0.61685));
     Value cond = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, argPow2,
-                                         piOver4Pow2, fmf.getValue());
+                                       piOver4Pow2, fmf.getValue());
     return arith::SelectOp::create(b, cond, forLargeArg, forSmallArg);
   }
 };
@@ -437,7 +442,7 @@ struct LogOpConversion : public OpConversionPattern<complex::LogOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
     Value abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(),
-                                         fmf.getValue());
+                                       fmf.getValue());
     Value resultReal = math::LogOp::create(b, elementType, abs, fmf.getValue());
     Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
     Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
@@ -464,9 +469,9 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
     Value half = arith::ConstantOp::create(b, elementType,
-                                             b.getFloatAttr(elementType, 0.5));
+                                           b.getFloatAttr(elementType, 0.5));
     Value one = arith::ConstantOp::create(b, elementType,
-                                            b.getFloatAttr(elementType, 1));
+                                          b.getFloatAttr(elementType, 1));
     Value realPlusOne = arith::AddFOp::create(b, real, one, fmf);
     Value absRealPlusOne = math::AbsFOp::create(b, realPlusOne, fmf);
     Value absImag = math::AbsFOp::create(b, imag, fmf);
@@ -475,7 +480,7 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     Value minAbs = arith::MinimumFOp::create(b, absRealPlusOne, absImag, fmf);
 
     Value useReal = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT,
-                                            realPlusOne, absImag, fmf);
+                                          realPlusOne, absImag, fmf);
     Value maxMinusOne = arith::SubFOp::create(b, maxAbs, one, fmf);
     Value maxAbsOfRealPlusOneAndImagMinusOne =
         arith::SelectOp::create(b, useReal, real, maxMinusOne);
@@ -484,14 +489,16 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     Value minMaxRatio = arith::DivFOp::create(b, minAbs, maxAbs, fmfWithNaNInf);
     Value logOfMaxAbsOfRealPlusOneAndImag =
         math::Log1pOp::create(b, maxAbsOfRealPlusOneAndImagMinusOne, fmf);
-    Value logOfSqrtPart = math::Log1pOp::create(b,
-        arith::MulFOp::create(b, minMaxRatio, minMaxRatio, fmfWithNaNInf),
+    Value logOfSqrtPart = math::Log1pOp::create(
+        b, arith::MulFOp::create(b, minMaxRatio, minMaxRatio, fmfWithNaNInf),
         fmfWithNaNInf);
-    Value r = arith::AddFOp::create(b,
-        arith::MulFOp::create(b, half, logOfSqrtPart, fmfWithNaNInf),
+    Value r = arith::AddFOp::create(
+        b, arith::MulFOp::create(b, half, logOfSqrtPart, fmfWithNaNInf),
         logOfMaxAbsOfRealPlusOneAndImag, fmfWithNaNInf);
-    Value resultReal = arith::SelectOp::create(b,
-        arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
+    Value resultReal = arith::SelectOp::create(
+        b,
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, r, r,
+                              fmfWithNaNInf),
         minAbs, r);
     Value resultImag = math::Atan2Op::create(b, imag, realPlusOne, fmf);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
@@ -520,13 +527,13 @@ struct MulOpConversion : public OpConversionPattern<complex::MulOp> {
     Value lhsImagTimesRhsImag =
         arith::MulFOp::create(b, lhsImag, rhsImag, fmfValue);
     Value real = arith::SubFOp::create(b, lhsRealTimesRhsReal,
-                                         lhsImagTimesRhsImag, fmfValue);
+                                       lhsImagTimesRhsImag, fmfValue);
     Value lhsImagTimesRhsReal =
         arith::MulFOp::create(b, lhsImag, rhsReal, fmfValue);
     Value lhsRealTimesRhsImag =
         arith::MulFOp::create(b, lhsReal, rhsImag, fmfValue);
     Value imag = arith::AddFOp::create(b, lhsImagTimesRhsReal,
-                                         lhsRealTimesRhsImag, fmfValue);
+                                       lhsRealTimesRhsImag, fmfValue);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, imag);
     return success();
   }
@@ -594,12 +601,12 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
 
     auto cst = [&](APFloat v) {
       return arith::ConstantOp::create(b, elementType,
-                                         b.getFloatAttr(elementType, v));
+                                       b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
     Value half = arith::ConstantOp::create(b, elementType,
-                                             b.getFloatAttr(elementType, 0.5));
+                                           b.getFloatAttr(elementType, 0.5));
 
     Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
     Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
@@ -614,8 +621,8 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
         arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, sin, zero, fmf);
 
     Value resultReal = arith::MulFOp::create(b, absSqrt, cos, fmf);
-    Value resultImag = arith::SelectOp::create(b,
-        sinIsZero, zero, arith::MulFOp::create(b, absSqrt, sin, fmf));
+    Value resultImag = arith::SelectOp::create(
+        b, sinIsZero, zero, arith::MulFOp::create(b, absSqrt, sin, fmf));
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
       Value inf = cst(APFloat::getInf(floatSemantics));
@@ -623,27 +630,28 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
       Value nan = cst(APFloat::getNaN(floatSemantics));
       Value absImag = math::AbsFOp::create(b, elementType, imag, fmf);
 
-      Value absImagIsInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
-      Value absImagIsNotInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::ONE, absImag, inf, fmf);
+      Value absImagIsInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                                 absImag, inf, fmf);
+      Value absImagIsNotInf = arith::CmpFOp::create(
+          b, arith::CmpFPredicate::ONE, absImag, inf, fmf);
       Value realIsInf =
           arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, inf, fmf);
-      Value realIsNegInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, negInf, fmf);
+      Value realIsNegInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                                 real, negInf, fmf);
 
-      resultReal = arith::SelectOp::create(b,
-          arith::AndIOp::create(b, realIsNegInf, absImagIsNotInf), zero,
+      resultReal = arith::SelectOp::create(
+          b, arith::AndIOp::create(b, realIsNegInf, absImagIsNotInf), zero,
           resultReal);
-      resultReal = arith::SelectOp::create(b,
-          arith::OrIOp::create(b, absImagIsInf, realIsInf), inf, resultReal);
+      resultReal = arith::SelectOp::create(
+          b, arith::OrIOp::create(b, absImagIsInf, realIsInf), inf, resultReal);
 
       Value imagSignInf = math::CopySignOp::create(b, inf, imag, fmf);
-      resultImag = arith::SelectOp::create(b,
+      resultImag = arith::SelectOp::create(
+          b,
           arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, absSqrt, absSqrt),
           nan, resultImag);
-      resultImag = arith::SelectOp::create(b,
-          arith::OrIOp::create(b, absImagIsInf, realIsNegInf), imagSignInf,
+      resultImag = arith::SelectOp::create(
+          b, arith::OrIOp::create(b, absImagIsInf, realIsNegInf), imagSignInf,
           resultImag);
     }
 
@@ -678,7 +686,8 @@ struct SignOpConversion : public OpConversionPattern<complex::SignOp> {
     Value imagIsZero =
         arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero);
     Value isZero = arith::AndIOp::create(b, realIsZero, imagIsZero);
-    auto abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(), fmf);
+    auto abs =
+        complex::AbsOp::create(b, elementType, adaptor.getComplex(), fmf);
     Value realSign = arith::DivFOp::create(b, real, abs, fmf);
     Value imagSign = arith::DivFOp::create(b, imag, abs, fmf);
     Value sign = complex::CreateOp::create(b, type, realSign, imagSign);
@@ -706,8 +715,8 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
         complex::ReOp::create(b, loc, elementType, adaptor.getComplex());
     Value imag =
         complex::ImOp::create(b, loc, elementType, adaptor.getComplex());
-    Value negOne = arith::ConstantOp::create(b,
-        elementType, b.getFloatAttr(elementType, -1.0));
+    Value negOne = arith::ConstantOp::create(b, elementType,
+                                             b.getFloatAttr(elementType, -1.0));
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
@@ -717,60 +726,60 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
 
     auto cst = [&](APFloat v) {
       return arith::ConstantOp::create(b, elementType,
-                                         b.getFloatAttr(elementType, v));
+                                       b.getFloatAttr(elementType, v));
     };
     Value inf = cst(APFloat::getInf(floatSemantics));
     Value four = arith::ConstantOp::create(b, elementType,
-                                             b.getFloatAttr(elementType, 4.0));
+                                           b.getFloatAttr(elementType, 4.0));
     Value twoReal = arith::AddFOp::create(b, real, real, fmf);
     Value negTwoReal = arith::MulFOp::create(b, negOne, twoReal, fmf);
 
     Value expTwoRealMinusOne = math::ExpM1Op::create(b, twoReal, fmf);
     Value expNegTwoRealMinusOne = math::ExpM1Op::create(b, negTwoReal, fmf);
-    Value realNum =
-        arith::SubFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+    Value realNum = arith::SubFOp::create(b, expTwoRealMinusOne,
+                                          expNegTwoRealMinusOne, fmf);
 
     Value cosImag = math::CosOp::create(b, imag, fmf);
     Value cosImagSq = arith::MulFOp::create(b, cosImag, cosImag, fmf);
     Value twoCosTwoImagPlusOne = arith::MulFOp::create(b, cosImagSq, four, fmf);
     Value sinImag = math::SinOp::create(b, imag, fmf);
 
-    Value imagNum = arith::MulFOp::create(b,
-        four, arith::MulFOp::create(b, cosImag, sinImag, fmf), fmf);
+    Value imagNum = arith::MulFOp::create(
+        b, four, arith::MulFOp::create(b, cosImag, sinImag, fmf), fmf);
 
-    Value expSumMinusTwo =
-        arith::AddFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+    Value expSumMinusTwo = arith::AddFOp::create(b, expTwoRealMinusOne,
+                                                 expNegTwoRealMinusOne, fmf);
     Value denom =
         arith::AddFOp::create(b, expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
 
     Value isInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
-                                          expSumMinusTwo, inf, fmf);
+                                        expSumMinusTwo, inf, fmf);
     Value realLimit = math::CopySignOp::create(b, negOne, real, fmf);
 
-    Value resultReal = arith::SelectOp::create(b,
-        isInf, realLimit, arith::DivFOp::create(b, realNum, denom, fmf));
+    Value resultReal = arith::SelectOp::create(
+        b, isInf, realLimit, arith::DivFOp::create(b, realNum, denom, fmf));
     Value resultImag = arith::DivFOp::create(b, imagNum, denom, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
       Value absReal = math::AbsFOp::create(b, real, fmf);
-      Value zero = arith::ConstantOp::create(b,
-          elementType, b.getFloatAttr(elementType, 0.0));
+      Value zero = arith::ConstantOp::create(b, elementType,
+                                             b.getFloatAttr(elementType, 0.0));
       Value nan = cst(APFloat::getNaN(floatSemantics));
 
-      Value absRealIsInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value absRealIsInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                                 absReal, inf, fmf);
       Value imagIsZero =
           arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
-      Value absRealIsNotInf = arith::XOrIOp::create(b,
-          absRealIsInf, arith::ConstantIntOp::create(b, true, /*width=*/1));
+      Value absRealIsNotInf = arith::XOrIOp::create(
+          b, absRealIsInf, arith::ConstantIntOp::create(b, true, /*width=*/1));
 
       Value imagNumIsNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO,
-                                                   imagNum, imagNum, fmf);
+                                                 imagNum, imagNum, fmf);
       Value resultRealIsNaN =
           arith::AndIOp::create(b, imagNumIsNaN, absRealIsNotInf);
-      Value resultImagIsZero = arith::OrIOp::create(b,
-          imagIsZero, arith::AndIOp::create(b, absRealIsInf, imagNumIsNaN));
+      Value resultImagIsZero = arith::OrIOp::create(
+          b, imagIsZero, arith::AndIOp::create(b, absRealIsInf, imagNumIsNaN));
 
       resultReal = arith::SelectOp::create(b, resultRealIsNaN, nan, resultReal);
       resultImag =
@@ -837,14 +846,14 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
   Value cosQ = math::CosOp::create(builder, q, fmf);
   Value sinQ = math::SinOp::create(builder, q, fmf);
 
-  Value inf = arith::ConstantOp::create(builder,
-      elementType,
+  Value inf = arith::ConstantOp::create(
+      builder, elementType,
       builder.getFloatAttr(elementType,
                            APFloat::getInf(elementType.getFloatSemantics())));
-  Value zero = arith::ConstantOp::create(builder,
-      elementType, builder.getFloatAttr(elementType, 0.0));
-  Value one = arith::ConstantOp::create(builder,
-      elementType, builder.getFloatAttr(elementType, 1.0));
+  Value zero = arith::ConstantOp::create(
+      builder, elementType, builder.getFloatAttr(elementType, 0.0));
+  Value one = arith::ConstantOp::create(builder, elementType,
+                                        builder.getFloatAttr(elementType, 1.0));
   Value complexOne = complex::CreateOp::create(builder, type, one, zero);
   Value complexZero = complex::CreateOp::create(builder, type, zero, zero);
   Value complexInf = complex::CreateOp::create(builder, type, inf, zero);
@@ -870,9 +879,10 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
       arith::SelectOp::create(builder, cEqZero, complexOne, complexZero);
   Value coeffCosSin =
       complex::CreateOp::create(builder, type, coeffCosQ, coeffSinQ);
-  Value cutoff0 = arith::SelectOp::create(builder,
-      arith::AndIOp::create(builder,
-          arith::AndIOp::create(builder, absEqZero, dEqZero), zeroLeC),
+  Value cutoff0 = arith::SelectOp::create(
+      builder,
+      arith::AndIOp::create(
+          builder, arith::AndIOp::create(builder, absEqZero, dEqZero), zeroLeC),
       complexOneOrZero, coeffCosSin);
 
   // Case 1:
@@ -885,7 +895,8 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
 
   // Case 2:
   // 1^(c + d*i) = 1 + 0*i
-  Value lhsEqOne = arith::AndIOp::create(builder,
+  Value lhsEqOne = arith::AndIOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, one, fmf),
       bEqZero);
   Value cutoff2 =
@@ -893,22 +904,25 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
 
   // Case 3:
   // inf^(c + 0*i) = inf + 0*i, c > 0
-  Value lhsEqInf = arith::AndIOp::create(builder,
+  Value lhsEqInf = arith::AndIOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, inf, fmf),
       bEqZero);
-  Value rhsGt0 = arith::AndIOp::create(builder,
-      dEqZero,
+  Value rhsGt0 = arith::AndIOp::create(
+      builder, dEqZero,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, c, zero, fmf));
-  Value cutoff3 = arith::SelectOp::create(builder,
-      arith::AndIOp::create(builder, lhsEqInf, rhsGt0), complexInf, cutoff2);
+  Value cutoff3 = arith::SelectOp::create(
+      builder, arith::AndIOp::create(builder, lhsEqInf, rhsGt0), complexInf,
+      cutoff2);
 
   // Case 4:
   // inf^(c + 0*i) = 0 + 0*i, c < 0
-  Value rhsLt0 = arith::AndIOp::create(builder,
-      dEqZero,
+  Value rhsLt0 = arith::AndIOp::create(
+      builder, dEqZero,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, c, zero, fmf));
-  Value cutoff4 = arith::SelectOp::create(builder,
-      arith::AndIOp::create(builder, lhsEqInf, rhsLt0), complexZero, cutoff3);
+  Value cutoff4 = arith::SelectOp::create(
+      builder, arith::AndIOp::create(builder, lhsEqInf, rhsLt0), complexZero,
+      cutoff3);
 
   return cutoff4;
 }
@@ -946,13 +960,13 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
 
     auto cst = [&](APFloat v) {
       return arith::ConstantOp::create(b, elementType,
-                                         b.getFloatAttr(elementType, v));
+                                       b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value negHalf = arith::ConstantOp::create(b,
-        elementType, b.getFloatAttr(elementType, -0.5));
+    Value negHalf = arith::ConstantOp::create(
+        b, elementType, b.getFloatAttr(elementType, -0.5));
     Value nan = cst(APFloat::getNaN(floatSemantics));
 
     Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
@@ -968,8 +982,8 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value negOne = arith::ConstantOp::create(b,
-          elementType, b.getFloatAttr(elementType, -1));
+      Value negOne = arith::ConstantOp::create(b, elementType,
+                                               b.getFloatAttr(elementType, -1));
 
       Value realSignedZero = math::CopySignOp::create(b, zero, real, fmf);
       Value imagSignedZero = math::CopySignOp::create(b, zero, imag, fmf);
@@ -979,12 +993,12 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
       Value absReal = math::AbsFOp::create(b, real, fmf);
       Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-      Value absImagIsInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+      Value absImagIsInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                                 absImag, inf, fmf);
       Value realIsNan =
           arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, real, real, fmf);
-      Value realIsInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value realIsInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                              absReal, inf, fmf);
       Value inIsNanInf = arith::AndIOp::create(b, absImagIsInf, realIsNan);
 
       Value resultIsZero = arith::OrIOp::create(b, inIsNanInf, realIsInf);
@@ -992,7 +1006,7 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
       resultReal =
           arith::SelectOp::create(b, resultIsZero, realSignedZero, resultReal);
       resultImag = arith::SelectOp::create(b, resultIsZero, negImagSignedZero,
-                                             resultImag);
+                                           resultImag);
     }
 
     Value isRealZero =
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index 5c3d26874faed..ff6d369176393 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -74,7 +74,7 @@ struct AssertOpLowering : public ConvertOpToLLVMPattern<cf::AssertOp> {
         rewriter.setInsertionPointToStart(module.getBody());
         auto abortFuncTy = LLVM::LLVMFunctionType::get(getVoidType(), {});
         abortFunc = LLVM::LLVMFuncOp::create(rewriter, rewriter.getUnknownLoc(),
-                                                      "abort", abortFuncTy);
+                                             "abort", abortFuncTy);
       }
       LLVM::CallOp::create(rewriter, loc, abortFunc, ValueRange());
       LLVM::UnreachableOp::create(rewriter, loc);
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index aa468c29536c7..c8311eb5a6433 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -34,7 +34,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
   if (auto condBrOp = dyn_cast<cf::CondBranchOp>(controlFlowCondOp)) {
     assert(regions.size() == 2);
     auto ifOp = scf::IfOp::create(builder, controlFlowCondOp->getLoc(),
-                                          resultTypes, condBrOp.getCondition());
+                                  resultTypes, condBrOp.getCondition());
     ifOp.getThenRegion().takeBody(regions[0]);
     ifOp.getElseRegion().takeBody(regions[1]);
     return ifOp.getOperation();
@@ -43,8 +43,8 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
   if (auto switchOp = dyn_cast<cf::SwitchOp>(controlFlowCondOp)) {
     // `getCFGSwitchValue` returns an i32 that we need to convert to index
     // fist.
-    auto cast = arith::IndexCastUIOp::create(builder,
-        controlFlowCondOp->getLoc(), builder.getIndexType(),
+    auto cast = arith::IndexCastUIOp::create(
+        builder, controlFlowCondOp->getLoc(), builder.getIndexType(),
         switchOp.getFlag());
     SmallVector<int64_t> cases;
     if (auto caseValues = switchOp.getCaseValues())
@@ -55,8 +55,9 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
 
     assert(regions.size() == cases.size() + 1);
 
-    auto indexSwitchOp = scf::IndexSwitchOp::create(builder,
-        controlFlowCondOp->getLoc(), resultTypes, cast, cases, cases.size());
+    auto indexSwitchOp =
+        scf::IndexSwitchOp::create(builder, controlFlowCondOp->getLoc(),
+                                   resultTypes, cast, cases, cases.size());
 
     indexSwitchOp.getDefaultRegion().takeBody(regions[0]);
     for (auto &&[targetRegion, sourceRegion] :
@@ -84,16 +85,17 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
     OpBuilder &builder, Operation *replacedOp, ValueRange loopVariablesInit,
     Value condition, ValueRange loopVariablesNextIter, Region &&loopBody) {
   Location loc = replacedOp->getLoc();
-  auto whileOp = scf::WhileOp::create(builder, loc, loopVariablesInit.getTypes(),
-                                              loopVariablesInit);
+  auto whileOp = scf::WhileOp::create(
+      builder, loc, loopVariablesInit.getTypes(), loopVariablesInit);
 
   whileOp.getBefore().takeBody(loopBody);
 
   builder.setInsertionPointToEnd(&whileOp.getBefore().back());
   // `getCFGSwitchValue` returns a i32. We therefore need to truncate the
   // condition to i1 first. It is guaranteed to be either 0 or 1 already.
-  scf::ConditionOp::create(builder,
-      loc, arith::TruncIOp::create(builder, loc, builder.getI1Type(), condition),
+  scf::ConditionOp::create(
+      builder, loc,
+      arith::TruncIOp::create(builder, loc, builder.getI1Type(), condition),
       loopVariablesNextIter);
 
   Block *afterBlock = builder.createBlock(&whileOp.getAfter());
@@ -109,7 +111,7 @@ Value ControlFlowToSCFTransformation::getCFGSwitchValue(Location loc,
                                                         OpBuilder &builder,
                                                         unsigned int value) {
   return arith::ConstantOp::create(builder, loc,
-                                           builder.getI32IntegerAttr(value));
+                                   builder.getI32IntegerAttr(value));
 }
 
 void ControlFlowToSCFTransformation::createCFGSwitchOp(
@@ -118,8 +120,8 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
     ArrayRef<ValueRange> caseArguments, Block *defaultDest,
     ValueRange defaultArgs) {
   cf::SwitchOp::create(builder, loc, flag, defaultDest, defaultArgs,
-                               llvm::to_vector_of<int32_t>(caseValues),
-                               caseDestinations, caseArguments);
+                       llvm::to_vector_of<int32_t>(caseValues),
+                       caseDestinations, caseArguments);
 }
 
 Value ControlFlowToSCFTransformation::getUndefValue(Location loc,
diff --git a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
index 20167f168185a..197caeb4ffbfa 100644
--- a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
+++ b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
@@ -99,8 +99,8 @@ class FuncOpConversion final : public OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted `emitc.func` op.
-    emitc::FuncOp newFuncOp = emitc::FuncOp::create(rewriter,
-        funcOp.getLoc(), funcOp.getName(),
+    emitc::FuncOp newFuncOp = emitc::FuncOp::create(
+        rewriter, funcOp.getLoc(), funcOp.getName(),
         FunctionType::get(rewriter.getContext(),
                           signatureConverter.getConvertedTypes(),
                           resultType ? TypeRange(resultType) : TypeRange()));
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 51dbe6cbb7de3..67bb1c14c99a2 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -115,8 +115,8 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   SmallVector<NamedAttribute> attributes;
   filterFuncAttributes(funcOp, attributes);
 
-  auto wrapperFuncOp = LLVM::LLVMFuncOp::create(rewriter,
-      loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
+  auto wrapperFuncOp = LLVM::LLVMFuncOp::create(
+      rewriter, loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperFuncType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
   propagateArgResAttrs(rewriter, !!resultStructType, funcOp, wrapperFuncOp);
@@ -129,14 +129,14 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   for (auto [index, argType] : llvm::enumerate(type.getInputs())) {
     Value arg = wrapperFuncOp.getArgument(index + argOffset);
     if (auto memrefType = dyn_cast<MemRefType>(argType)) {
-      Value loaded = LLVM::LoadOp::create(rewriter,
-          loc, typeConverter.convertType(memrefType), arg);
+      Value loaded = LLVM::LoadOp::create(
+          rewriter, loc, typeConverter.convertType(memrefType), arg);
       MemRefDescriptor::unpack(rewriter, loc, loaded, memrefType, args);
       continue;
     }
     if (isa<UnrankedMemRefType>(argType)) {
-      Value loaded = LLVM::LoadOp::create(rewriter,
-          loc, typeConverter.convertType(argType), arg);
+      Value loaded = LLVM::LoadOp::create(
+          rewriter, loc, typeConverter.convertType(argType), arg);
       UnrankedMemRefDescriptor::unpack(rewriter, loc, loaded, args);
       continue;
     }
@@ -148,7 +148,7 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
 
   if (resultStructType) {
     LLVM::StoreOp::create(rewriter, loc, call.getResult(),
-                                   wrapperFuncOp.getArgument(0));
+                          wrapperFuncOp.getArgument(0));
     LLVM::ReturnOp::create(rewriter, loc, ValueRange{});
   } else {
     LLVM::ReturnOp::create(rewriter, loc, call.getResults());
@@ -182,8 +182,8 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   filterFuncAttributes(funcOp, attributes);
 
   // Create the auxiliary function.
-  auto wrapperFunc = LLVM::LLVMFuncOp::create(builder,
-      loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
+  auto wrapperFunc = LLVM::LLVMFuncOp::create(
+      builder, loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
   propagateArgResAttrs(builder, !!resultStructType, funcOp, wrapperFunc);
@@ -201,8 +201,8 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   if (resultStructType) {
     // Allocate the struct on the stack and pass the pointer.
     Type resultType = cast<LLVM::LLVMFunctionType>(wrapperType).getParamType(0);
-    Value one = LLVM::ConstantOp::create(builder,
-        loc, typeConverter.convertType(builder.getIndexType()),
+    Value one = LLVM::ConstantOp::create(
+        builder, loc, typeConverter.convertType(builder.getIndexType()),
         builder.getIntegerAttr(builder.getIndexType(), 1));
     Value result =
         LLVM::AllocaOp::create(builder, loc, resultType, resultStructType, one);
@@ -229,11 +229,11 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
                     wrapperArgsRange.take_front(numToDrop));
 
       auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext());
-      Value one = LLVM::ConstantOp::create(builder,
-          loc, typeConverter.convertType(builder.getIndexType()),
+      Value one = LLVM::ConstantOp::create(
+          builder, loc, typeConverter.convertType(builder.getIndexType()),
           builder.getIntegerAttr(builder.getIndexType(), 1));
-      Value allocated = LLVM::AllocaOp::create(builder,
-          loc, ptrTy, packed.getType(), one, /*alignment=*/0);
+      Value allocated = LLVM::AllocaOp::create(
+          builder, loc, ptrTy, packed.getType(), one, /*alignment=*/0);
       LLVM::StoreOp::create(builder, loc, packed, allocated);
       arg = allocated;
     } else {
@@ -357,8 +357,8 @@ FailureOr<LLVM::LLVMFuncOp> mlir::convertFuncOpToLLVMFuncOp(
     symbolTable.remove(funcOp);
   }
 
-  auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter,
-      funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
+  auto newFuncOp = LLVM::LLVMFuncOp::create(
+      rewriter, funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
       /*dsoLocal=*/false, /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr,
       attributes);
 
@@ -556,9 +556,10 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
     auto promoted = this->getTypeConverter()->promoteOperands(
         callOp.getLoc(), /*opOperands=*/callOp->getOperands(),
         adaptor.getOperands(), rewriter, useBarePtrCallConv);
-    auto newOp = LLVM::CallOp::create(rewriter,
-        callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
-        promoted, callOp->getAttrs());
+    auto newOp = LLVM::CallOp::create(rewriter, callOp.getLoc(),
+                                      packedResult ? TypeRange(packedResult)
+                                                   : TypeRange(),
+                                      promoted, callOp->getAttrs());
 
     newOp.getProperties().operandSegmentSizes = {
         static_cast<int32_t>(promoted.size()), 0};
@@ -573,8 +574,8 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
       // Extract individual results from the structure and return them as list.
       results.reserve(numResults);
       for (unsigned i = 0; i < numResults; ++i) {
-        results.push_back(LLVM::ExtractValueOp::create(rewriter,
-            callOp.getLoc(), newOp->getResult(0), i));
+        results.push_back(LLVM::ExtractValueOp::create(
+            rewriter, callOp.getLoc(), newOp->getResult(0), i));
       }
     }
 
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index f5d5460412da2..1037e296c8128 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -69,8 +69,8 @@ mlir::getOrCreateStringConstant(OpBuilder &b, Location loc,
   b.setInsertionPointToStart(moduleOp.getBody());
   SmallString<16> name = getUniqueSymbolName(moduleOp, namePrefix);
   return LLVM::GlobalOp::create(b, loc, globalType,
-                                  /*isConstant=*/true, LLVM::Linkage::Internal,
-                                  name, attr, alignment, addrSpace);
+                                /*isConstant=*/true, LLVM::Linkage::Internal,
+                                name, attr, alignment, addrSpace);
 }
 
 LogicalResult
@@ -151,8 +151,8 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
               gpuFuncOp.getWorkgroupAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      auto globalOp = LLVM::GlobalOp::create(rewriter,
-          gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
+      auto globalOp = LLVM::GlobalOp::create(
+          rewriter, gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
           LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment,
           workgroupAddrSpace);
       workgroupBuffers.push_back(globalOp);
@@ -220,8 +220,8 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
   LLVM::CConv callingConvention = gpuFuncOp.isKernel()
                                       ? kernelCallingConvention
                                       : nonKernelCallingConvention;
-  auto llvmFuncOp = LLVM::LLVMFuncOp::create(rewriter,
-      gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
+  auto llvmFuncOp = LLVM::LLVMFuncOp::create(
+      rewriter, gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
       LLVM::Linkage::External, /*dsoLocal=*/false, callingConvention,
       /*comdat=*/nullptr, attributes);
 
@@ -266,11 +266,11 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       for (const auto [idx, global] : llvm::enumerate(workgroupBuffers)) {
         auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext(),
                                                   global.getAddrSpace());
-        Value address = LLVM::AddressOfOp::create(rewriter,
-            loc, ptrType, global.getSymNameAttr());
+        Value address = LLVM::AddressOfOp::create(rewriter, loc, ptrType,
+                                                  global.getSymNameAttr());
         Value memory =
             LLVM::GEPOp::create(rewriter, loc, ptrType, global.getType(),
-                                         address, ArrayRef<LLVM::GEPArg>{0, 0});
+                                address, ArrayRef<LLVM::GEPArg>{0, 0});
 
         // Build a memref descriptor pointing to the buffer to plug with the
         // existing memref infrastructure. This may use more registers than
@@ -298,15 +298,16 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       Type elementType = typeConverter->convertType(type.getElementType());
       auto ptrType =
           LLVM::LLVMPointerType::get(rewriter.getContext(), allocaAddrSpace);
-      Value numElements = LLVM::ConstantOp::create(rewriter,
-          gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
+      Value numElements = LLVM::ConstantOp::create(
+          rewriter, gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
       uint64_t alignment = 0;
       if (auto alignAttr =
               dyn_cast_or_null<IntegerAttr>(gpuFuncOp.getPrivateAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      Value allocated = LLVM::AllocaOp::create(rewriter,
-          gpuFuncOp.getLoc(), ptrType, elementType, numElements, alignment);
+      Value allocated =
+          LLVM::AllocaOp::create(rewriter, gpuFuncOp.getLoc(), ptrType,
+                                 elementType, numElements, alignment);
       Value descr = MemRefDescriptor::fromStaticShape(
           rewriter, loc, *getTypeConverter(), type, allocated);
       signatureConversion.remapInput(
@@ -419,7 +420,8 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
 
   /// Start the printf hostcall
   Value zeroI64 = LLVM::ConstantOp::create(rewriter, loc, llvmI64, 0);
-  auto printfBeginCall = LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
+  auto printfBeginCall =
+      LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
   Value printfDesc = printfBeginCall.getResult();
 
   // Create the global op or find an existing one.
@@ -427,21 +429,21 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
       rewriter, loc, moduleOp, llvmI8, "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element and pass it to printf()
-  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
-      loc,
+  Value globalPtr = LLVM::AddressOfOp::create(
+      rewriter, loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
       LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
-                                   globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
-  Value stringLen = LLVM::ConstantOp::create(rewriter,
-      loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
+                          globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+  Value stringLen = LLVM::ConstantOp::create(
+      rewriter, loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
 
   Value oneI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 1);
   Value zeroI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 0);
 
-  auto appendFormatCall = LLVM::CallOp::create(rewriter,
-      loc, ocklAppendStringN,
+  auto appendFormatCall = LLVM::CallOp::create(
+      rewriter, loc, ocklAppendStringN,
       ValueRange{printfDesc, stringStart, stringLen,
                  adaptor.getArgs().empty() ? oneI32 : zeroI32});
   printfDesc = appendFormatCall.getResult();
@@ -461,8 +463,9 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
       Value arg = adaptor.getArgs()[i];
       if (auto floatType = dyn_cast<FloatType>(arg.getType())) {
         if (!floatType.isF64())
-          arg = LLVM::FPExtOp::create(rewriter,
-              loc, typeConverter->convertType(rewriter.getF64Type()), arg);
+          arg = LLVM::FPExtOp::create(
+              rewriter, loc, typeConverter->convertType(rewriter.getF64Type()),
+              arg);
         arg = LLVM::BitcastOp::create(rewriter, loc, llvmI64, arg);
       }
       if (arg.getType().getIntOrFloatBitWidth() != 64)
@@ -510,13 +513,13 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
       /*alignment=*/0, addressSpace);
 
   // Get a pointer to the format string's first element
-  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
-      loc,
+  Value globalPtr = LLVM::AddressOfOp::create(
+      rewriter, loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
       LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
-                                   globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+                          globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
 
   // Construct arguments and function call
   auto argsRange = adaptor.getArgs();
@@ -562,7 +565,7 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   Value globalPtr = LLVM::AddressOfOp::create(rewriter, loc, global);
   Value stringStart =
       LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
-                                   globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+                          globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
   SmallVector<Type> types;
   SmallVector<Value> args;
   // Promote and pack the arguments into a stack allocation.
@@ -580,13 +583,13 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   Type structType =
       LLVM::LLVMStructType::getLiteral(gpuPrintfOp.getContext(), types);
   Value one = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
-                                                rewriter.getIndexAttr(1));
+                                       rewriter.getIndexAttr(1));
   Value tempAlloc =
       LLVM::AllocaOp::create(rewriter, loc, ptrType, structType, one,
-                                      /*alignment=*/0);
+                             /*alignment=*/0);
   for (auto [index, arg] : llvm::enumerate(args)) {
-    Value ptr = LLVM::GEPOp::create(rewriter,
-        loc, ptrType, structType, tempAlloc,
+    Value ptr = LLVM::GEPOp::create(
+        rewriter, loc, ptrType, structType, tempAlloc,
         ArrayRef<LLVM::GEPArg>{0, static_cast<int32_t>(index)});
     LLVM::StoreOp::create(rewriter, loc, arg, ptr);
   }
@@ -622,8 +625,8 @@ static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands,
     auto scalarOperands = llvm::map_to_vector(operands, extractElement);
     Operation *scalarOp =
         rewriter.create(loc, name, scalarOperands, elementType, op->getAttrs());
-    result = LLVM::InsertElementOp::create(rewriter,
-        loc, result, scalarOp->getResult(0), index);
+    result = LLVM::InsertElementOp::create(rewriter, loc, result,
+                                           scalarOp->getResult(0), index);
   }
   return result;
 }
@@ -705,10 +708,10 @@ LLVM::GlobalOp getDynamicSharedMemorySymbol(
   auto zeroSizedArrayType = LLVM::LLVMArrayType::get(
       typeConverter->convertType(memrefType.getElementType()), 0);
 
-  return LLVM::GlobalOp::create(rewriter,
-      op->getLoc(), zeroSizedArrayType, /*isConstant=*/false,
-      LLVM::Linkage::Internal, symName, /*value=*/Attribute(), alignmentByte,
-      addressSpace.value());
+  return LLVM::GlobalOp::create(rewriter, op->getLoc(), zeroSizedArrayType,
+                                /*isConstant=*/false, LLVM::Linkage::Internal,
+                                symName, /*value=*/Attribute(), alignmentByte,
+                                addressSpace.value());
 }
 
 LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
@@ -738,7 +741,7 @@ LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
   // Step 4. Generate GEP using offsets
   SmallVector<LLVM::GEPArg> gepArgs = {0};
   Value shmemPtr = LLVM::GEPOp::create(rewriter, loc, baseType, elementType,
-                                                basePtr, gepArgs);
+                                       basePtr, gepArgs);
   // Step 5. Create a memref descriptor
   SmallVector<Value> shape, strides;
   Value sizeBytes;
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 5e8254a0b6035..63eb6c58e87a7 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -79,8 +79,8 @@ class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern<OpTy> {
     uint64_t rank = type.getRank();
     Value numElements = desc.size(rewriter, loc, /*pos=*/0);
     for (unsigned i = 1; i < rank; i++)
-      numElements = LLVM::MulOp::create(rewriter,
-          loc, numElements, desc.size(rewriter, loc, /*pos=*/i));
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements,
+                                        desc.size(rewriter, loc, /*pos=*/i));
     return numElements;
   }
 
@@ -785,8 +785,8 @@ LogicalResult ConvertAllocOpToGpuRuntimeCallPattern::matchAndRewrite(
                      ? nullPtr
                      : adaptor.getAsyncDependencies().front();
 
-  auto isHostShared = mlir::LLVM::ConstantOp::create(rewriter,
-      loc, llvmInt8Type, rewriter.getI8IntegerAttr(isShared));
+  auto isHostShared = mlir::LLVM::ConstantOp::create(
+      rewriter, loc, llvmInt8Type, rewriter.getI8IntegerAttr(isShared));
 
   Value allocatedPtr =
       allocCallBuilder.create(loc, rewriter, {sizeBytes, stream, isHostShared})
@@ -1012,8 +1012,8 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
       uint64_t staticSize = static_cast<uint64_t>(bitwidth / 8) *
                             static_cast<uint64_t>(memrefTy.getNumElements());
 
-      Value sizeArg = LLVM::ConstantOp::create(rewriter,
-          loc, getIndexType(), rewriter.getIndexAttr(staticSize));
+      Value sizeArg = LLVM::ConstantOp::create(
+          rewriter, loc, getIndexType(), rewriter.getIndexAttr(staticSize));
       llvmArgumentsWithSizes.push_back(llvmArg); // Presumably a bare pointer.
       llvmArgumentsWithSizes.push_back(sizeArg);
     }
@@ -1025,8 +1025,8 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
         gpu::KernelDim3{adaptor.getClusterSizeX(), adaptor.getClusterSizeY(),
                         adaptor.getClusterSizeZ()};
   }
-  gpu::LaunchFuncOp::create(rewriter,
-      launchOp.getLoc(), launchOp.getKernelAttr(),
+  gpu::LaunchFuncOp::create(
+      rewriter, launchOp.getLoc(), launchOp.getKernelAttr(),
       gpu::KernelDim3{adaptor.getGridSizeX(), adaptor.getGridSizeY(),
                       adaptor.getGridSizeZ()},
       gpu::KernelDim3{adaptor.getBlockSizeX(), adaptor.getBlockSizeY(),
@@ -1048,8 +1048,8 @@ static Value bitAndAddrspaceCast(Location loc,
                                  const LLVMTypeConverter &typeConverter) {
   auto sourceTy = cast<LLVM::LLVMPointerType>(sourcePtr.getType());
   if (destinationType.getAddressSpace() != sourceTy.getAddressSpace())
-    sourcePtr = LLVM::AddrSpaceCastOp::create(rewriter,
-        loc,
+    sourcePtr = LLVM::AddrSpaceCastOp::create(
+        rewriter, loc,
         LLVM::LLVMPointerType::get(rewriter.getContext(),
                                    destinationType.getAddressSpace()),
         sourcePtr);
@@ -1073,8 +1073,8 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite(
 
   Type elementPtrType = getElementPtrType(memRefType);
   Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
-  Value gepPtr = LLVM::GEPOp::create(rewriter,
-      loc, elementPtrType,
+  Value gepPtr = LLVM::GEPOp::create(
+      rewriter, loc, elementPtrType,
       typeConverter->convertType(memRefType.getElementType()), nullPtr,
       numElements);
   auto sizeBytes =
@@ -1151,14 +1151,14 @@ template <typename T>
 static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmInt32Type = builder.getIntegerType(32);
   return LLVM::ConstantOp::create(builder, loc, llvmInt32Type,
-                                          static_cast<int32_t>(tValue));
+                                  static_cast<int32_t>(tValue));
 }
 
 template <typename T>
 static Value genConstFloat32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmFloat32Type = builder.getF32Type();
-  return LLVM::ConstantOp::create(builder,
-      loc, llvmFloat32Type,
+  return LLVM::ConstantOp::create(
+      builder, loc, llvmFloat32Type,
       builder.getF32FloatAttr(static_cast<float>(tValue)));
 }
 
@@ -1189,10 +1189,10 @@ LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite(
   // the dnmat is used with spmat with 2:4 sparsity
   if (dims.size() == 2) {
     if (isSpMMCusparseLtOp(op.getDnTensor())) {
-      auto handleSz = LLVM::ConstantOp::create(rewriter,
-          loc, getIndexType(), rewriter.getIndexAttr(11032));
-      handle = LLVM::AllocaOp::create(rewriter,
-          loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
+      auto handleSz = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                               rewriter.getIndexAttr(11032));
+      handle = LLVM::AllocaOp::create(rewriter, loc, llvmPointerType,
+                                      llvmInt8Type, handleSz, /*alignment=*/16);
       handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
       createLtDnMatCallBuilder
@@ -1351,10 +1351,10 @@ LogicalResult ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto dtp = genConstInt32From(rewriter, loc, getCuSparseDataTypeFrom(dType));
 
   // CUDA runner asserts the size is 44104 bytes.
-  auto handleSz = LLVM::ConstantOp::create(rewriter,
-      loc, getIndexType(), rewriter.getIndexAttr(44104));
-  Value handle = LLVM::AllocaOp::create(rewriter,
-      loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
+  auto handleSz = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                           rewriter.getIndexAttr(44104));
+  Value handle = LLVM::AllocaOp::create(
+      rewriter, loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
   handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
   create2To4SpMatCallBuilder
@@ -1442,9 +1442,10 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
     auto computeType = genConstInt32From(
         rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType()));
     auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                   rewriter.getIndexAttr(3));
-    auto bufferSize = LLVM::AllocaOp::create(rewriter,
-        loc, llvmPointerType, llvmPointerType, three, /*alignment=*/16);
+                                          rewriter.getIndexAttr(3));
+    auto bufferSize =
+        LLVM::AllocaOp::create(rewriter, loc, llvmPointerType, llvmPointerType,
+                               three, /*alignment=*/16);
     createCuSparseLtSpMMBufferSizeBuilder
         .create(loc, rewriter,
                 {bufferSize, modeA, modeB, adaptor.getSpmatA(),
@@ -1452,14 +1453,14 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
                  pruneFlag, stream})
         .getResult();
 
-    auto bufferSizePtr1 = LLVM::GEPOp::create(rewriter,
-        loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{LLVM::ConstantOp::create(rewriter,
-            loc, getIndexType(), rewriter.getIndexAttr(1))});
-    auto bufferSizePtr2 = LLVM::GEPOp::create(rewriter,
-        loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{LLVM::ConstantOp::create(rewriter,
-            loc, getIndexType(), rewriter.getIndexAttr(2))});
+    auto bufferSizePtr1 = LLVM::GEPOp::create(
+        rewriter, loc, llvmPointerType, llvmPointerType, bufferSize,
+        ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                            rewriter.getIndexAttr(1))});
+    auto bufferSizePtr2 = LLVM::GEPOp::create(
+        rewriter, loc, llvmPointerType, llvmPointerType, bufferSize,
+        ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                            rewriter.getIndexAttr(2))});
     auto bufferSize0 =
         LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSize);
     auto bufferSize1 =
@@ -1670,22 +1671,22 @@ LogicalResult ConvertSpMatGetSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto stream = adaptor.getAsyncDependencies().front();
 
   auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                 rewriter.getIndexAttr(3));
-  auto buffer = LLVM::AllocaOp::create(rewriter,
-      loc, llvmPointerType, llvmInt64Type, three, /*alignment=*/16);
+                                        rewriter.getIndexAttr(3));
+  auto buffer = LLVM::AllocaOp::create(rewriter, loc, llvmPointerType,
+                                       llvmInt64Type, three, /*alignment=*/16);
 
-  auto rowsPtr = LLVM::GEPOp::create(rewriter,
-      loc, llvmPointerType, llvmPointerType, buffer,
+  auto rowsPtr = LLVM::GEPOp::create(
+      rewriter, loc, llvmPointerType, llvmPointerType, buffer,
       ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                   rewriter.getIndexAttr(0))});
-  auto colsPtr = LLVM::GEPOp::create(rewriter,
-      loc, llvmPointerType, llvmPointerType, buffer,
+                                          rewriter.getIndexAttr(0))});
+  auto colsPtr = LLVM::GEPOp::create(
+      rewriter, loc, llvmPointerType, llvmPointerType, buffer,
       ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                   rewriter.getIndexAttr(1))});
-  auto nnzsPtr = LLVM::GEPOp::create(rewriter,
-      loc, llvmPointerType, llvmPointerType, buffer,
+                                          rewriter.getIndexAttr(1))});
+  auto nnzsPtr = LLVM::GEPOp::create(
+      rewriter, loc, llvmPointerType, llvmPointerType, buffer,
       ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                   rewriter.getIndexAttr(2))});
+                                          rewriter.getIndexAttr(2))});
   createSpMatGetSizeBuilder.create(
       loc, rewriter, {adaptor.getSpmat(), rowsPtr, colsPtr, nnzsPtr, stream});
   auto rows = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, rowsPtr);
diff --git a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
index 70964727351a9..91c43e8bd1117 100644
--- a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
@@ -124,11 +124,13 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
                                   rewriter.getContext(), 32, min, max));
     }
     if (indexBitwidth > 32) {
-      newOp = LLVM::SExtOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
+      newOp = LLVM::SExtOp::create(rewriter, loc,
+                                   IntegerType::get(context, indexBitwidth),
+                                   newOp->getResult(0));
     } else if (indexBitwidth < 32) {
-      newOp = LLVM::TruncOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
+      newOp = LLVM::TruncOp::create(rewriter, loc,
+                                    IntegerType::get(context, indexBitwidth),
+                                    newOp->getResult(0));
     }
 
     rewriter.replaceOp(op, newOp->getResults());
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index 33b09b06649b9..9f36e5c369d06 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -115,19 +115,20 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // there is no guarantee of a specific value being used to indicate true,
     // compare for inequality with zero (rather than truncate or shift).
     if (isResultBool) {
-      Value zero = LLVM::ConstantOp::create(rewriter,
-          op->getLoc(), rewriter.getIntegerType(32),
-          rewriter.getI32IntegerAttr(0));
-      Value truncated = LLVM::ICmpOp::create(rewriter,
-          op->getLoc(), LLVM::ICmpPredicate::ne, callOp.getResult(), zero);
+      Value zero = LLVM::ConstantOp::create(rewriter, op->getLoc(),
+                                            rewriter.getIntegerType(32),
+                                            rewriter.getI32IntegerAttr(0));
+      Value truncated =
+          LLVM::ICmpOp::create(rewriter, op->getLoc(), LLVM::ICmpPredicate::ne,
+                               callOp.getResult(), zero);
       rewriter.replaceOp(op, {truncated});
       return success();
     }
 
     assert(callOp.getResult().getType().isF32() &&
            "only f32 types are supposed to be truncated back");
-    Value truncated = LLVM::FPTruncOp::create(rewriter,
-        op->getLoc(), adaptor.getOperands().front().getType(),
+    Value truncated = LLVM::FPTruncOp::create(
+        rewriter, op->getLoc(), adaptor.getOperands().front().getType(),
         callOp.getResult());
     rewriter.replaceOp(op, {truncated});
     return success();
@@ -142,8 +143,9 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     if (!f16Func.empty() && isa<Float16Type>(type))
       return operand;
 
-    return LLVM::FPExtOp::create(rewriter,
-        operand.getLoc(), Float32Type::get(rewriter.getContext()), operand);
+    return LLVM::FPExtOp::create(rewriter, operand.getLoc(),
+                                 Float32Type::get(rewriter.getContext()),
+                                 operand);
   }
 
   Type getFunctionType(Type resultType, ValueRange operands) const {
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index fc469102a54d2..c2363a1a40294 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -54,8 +54,8 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
       SymbolTable::lookupSymbolIn(symbolTable, name));
   if (!func) {
     OpBuilder b(symbolTable->getRegion(0));
-    func = LLVM::LLVMFuncOp::create(b,
-        symbolTable->getLoc(), name,
+    func = LLVM::LLVMFuncOp::create(
+        b, symbolTable->getLoc(), name,
         LLVM::LLVMFunctionType::get(resultType, paramTypes));
     func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
     func.setNoUnwind(true);
@@ -163,7 +163,7 @@ struct LaunchConfigConversion : ConvertToLLVMPattern {
     Location loc = op->getLoc();
     gpu::Dimension dim = getDimension(op);
     Value dimVal = LLVM::ConstantOp::create(rewriter, loc, dimTy,
-                                                     static_cast<int64_t>(dim));
+                                            static_cast<int64_t>(dim));
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, dimVal));
     return success();
   }
@@ -292,12 +292,12 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     return TypeSwitch<Type, Value>(oldVal.getType())
         .Case([&](BFloat16Type) {
           return LLVM::BitcastOp::create(rewriter, loc, rewriter.getI16Type(),
-                                                  oldVal);
+                                         oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
             return LLVM::ZExtOp::create(rewriter, loc, rewriter.getI8Type(),
-                                                 oldVal);
+                                        oldVal);
           return oldVal;
         })
         .Default(oldVal);
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 7a6601ea754e9..317bfc2970cf5 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -120,8 +120,8 @@ struct GPUSubgroupReduceOpLowering
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     Value offset = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
 
-    auto reduxOp = NVVM::ReduxOp::create(rewriter, loc, int32Type, op.getValue(),
-                                                  mode.value(), offset);
+    auto reduxOp = NVVM::ReduxOp::create(rewriter, loc, int32Type,
+                                         op.getValue(), mode.value(), offset);
 
     rewriter.replaceOp(op, reduxOp->getResult(0));
     return success();
@@ -161,19 +161,19 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     Value one = LLVM::ConstantOp::create(rewriter, loc, int32Type, 1);
     Value minusOne = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
     Value thirtyTwo = LLVM::ConstantOp::create(rewriter, loc, int32Type, 32);
-    Value numLeadInactiveLane = LLVM::SubOp::create(rewriter,
-        loc, int32Type, thirtyTwo, adaptor.getWidth());
+    Value numLeadInactiveLane = LLVM::SubOp::create(
+        rewriter, loc, int32Type, thirtyTwo, adaptor.getWidth());
     // Bit mask of active lanes: `(-1) >> (32 - activeWidth)`.
     Value activeMask = LLVM::LShrOp::create(rewriter, loc, int32Type, minusOne,
-                                                     numLeadInactiveLane);
+                                            numLeadInactiveLane);
     Value maskAndClamp;
     if (op.getMode() == gpu::ShuffleMode::UP) {
       // Clamp lane: `32 - activeWidth`
       maskAndClamp = numLeadInactiveLane;
     } else {
       // Clamp lane: `activeWidth - 1`
-      maskAndClamp =
-          LLVM::SubOp::create(rewriter, loc, int32Type, adaptor.getWidth(), one);
+      maskAndClamp = LLVM::SubOp::create(rewriter, loc, int32Type,
+                                         adaptor.getWidth(), one);
     }
 
     bool predIsUsed = !op->getResult(1).use_empty();
@@ -184,9 +184,10 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
       resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(),
                                                   {valueTy, predTy});
     }
-    Value shfl = NVVM::ShflOp::create(rewriter,
-        loc, resultTy, activeMask, adaptor.getValue(), adaptor.getOffset(),
-        maskAndClamp, convertShflKind(op.getMode()), returnValueAndIsValidAttr);
+    Value shfl = NVVM::ShflOp::create(
+        rewriter, loc, resultTy, activeMask, adaptor.getValue(),
+        adaptor.getOffset(), maskAndClamp, convertShflKind(op.getMode()),
+        returnValueAndIsValidAttr);
     if (predIsUsed) {
       Value shflValue = LLVM::ExtractValueOp::create(rewriter, loc, shfl, 0);
       Value isActiveSrcLane =
@@ -220,11 +221,11 @@ struct GPULaneIdOpToNVVM : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      newOp = LLVM::SExtOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), newOp);
+      newOp = LLVM::SExtOp::create(
+          rewriter, loc, IntegerType::get(context, indexBitwidth), newOp);
     } else if (indexBitwidth < 32) {
-      newOp = LLVM::TruncOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), newOp);
+      newOp = LLVM::TruncOp::create(
+          rewriter, loc, IntegerType::get(context, indexBitwidth), newOp);
     }
     rewriter.replaceOp(op, {newOp});
     return success();
@@ -272,7 +273,7 @@ struct AssertOpToAssertfailLowering
         rewriter.splitBlock(assertBlock, ++assertOp->getIterator());
     rewriter.setInsertionPointToEnd(beforeBlock);
     cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(), afterBlock,
-                                      assertBlock);
+                             assertBlock);
     rewriter.setInsertionPointToEnd(assertBlock);
     cf::BranchOp::create(rewriter, loc, afterBlock);
 
@@ -301,12 +302,12 @@ struct AssertOpToAssertfailLowering
     // Create constants.
     auto getGlobal = [&](LLVM::GlobalOp global) {
       // Get a pointer to the format string's first element.
-      Value globalPtr = LLVM::AddressOfOp::create(rewriter,
-          loc, LLVM::LLVMPointerType::get(ctx, global.getAddrSpace()),
+      Value globalPtr = LLVM::AddressOfOp::create(
+          rewriter, loc, LLVM::LLVMPointerType::get(ctx, global.getAddrSpace()),
           global.getSymNameAttr());
       Value start =
           LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
-                                       globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+                              globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
       return start;
     };
     Value assertMessage = getGlobal(getOrCreateStringConstant(
diff --git a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
index 7d5f65c470386..99c059cb03299 100644
--- a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
@@ -126,8 +126,8 @@ struct WmmaLoadOpToNVVMLowering
         cast<MemRefType>(subgroupMmaLoadMatrixOp.getSrcMemref().getType()),
         adaptor.getSrcMemref(), adaptor.getIndices());
 
-    Value leadingDim = LLVM::ConstantOp::create(rewriter,
-        loc, rewriter.getI32Type(),
+    Value leadingDim = LLVM::ConstantOp::create(
+        rewriter, loc, rewriter.getI32Type(),
         subgroupMmaLoadMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMALoadOp>(
         op, resType, dataPtr, leadingDim, m, n, k, layout, eltype, frag);
@@ -181,8 +181,8 @@ struct WmmaStoreOpToNVVMLowering
         rewriter, loc,
         cast<MemRefType>(subgroupMmaStoreMatrixOp.getDstMemref().getType()),
         adaptor.getDstMemref(), adaptor.getIndices());
-    Value leadingDim = LLVM::ConstantOp::create(rewriter,
-        loc, rewriter.getI32Type(),
+    Value leadingDim = LLVM::ConstantOp::create(
+        rewriter, loc, rewriter.getI32Type(),
         subgroupMmaStoreMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMAStoreOp>(
         op, dataPtr, m, n, k, layout, eltype, storeOpOperands, leadingDim);
@@ -282,10 +282,10 @@ struct WmmaConstantOpToNVVMLowering
     if (auto vecType = dyn_cast<VectorType>(type.getBody()[0])) {
       Value vecCst = LLVM::PoisonOp::create(rewriter, loc, vecType);
       for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) {
-        Value idx = LLVM::ConstantOp::create(rewriter,
-            loc, rewriter.getI32Type(), vecEl);
+        Value idx = LLVM::ConstantOp::create(rewriter, loc,
+                                             rewriter.getI32Type(), vecEl);
         vecCst = LLVM::InsertElementOp::create(rewriter, loc, vecType, vecCst,
-                                                        cst, idx);
+                                               cst, idx);
       }
       cst = vecCst;
     }
@@ -305,14 +305,14 @@ static Value createMinMaxF(OpBuilder &builder, Location loc, Value lhs,
   Type i1Type = builder.getI1Type();
   if (auto vecType = dyn_cast<VectorType>(lhs.getType()))
     i1Type = VectorType::get(vecType.getShape(), i1Type);
-  Value cmp = LLVM::FCmpOp::create(builder,
-      loc, i1Type, isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt,
-      lhs, rhs);
+  Value cmp = LLVM::FCmpOp::create(
+      builder, loc, i1Type,
+      isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt, lhs, rhs);
   Value sel = LLVM::SelectOp::create(builder, loc, cmp, lhs, rhs);
-  Value isNan = LLVM::FCmpOp::create(builder,
-      loc, i1Type, LLVM::FCmpPredicate::uno, lhs, rhs);
-  Value nan = LLVM::ConstantOp::create(builder,
-      loc, lhs.getType(),
+  Value isNan = LLVM::FCmpOp::create(builder, loc, i1Type,
+                                     LLVM::FCmpPredicate::uno, lhs, rhs);
+  Value nan = LLVM::ConstantOp::create(
+      builder, loc, lhs.getType(),
       builder.getFloatAttr(floatType,
                            APFloat::getQNaN(floatType.getFloatSemantics())));
   return LLVM::SelectOp::create(builder, loc, isNan, nan, sel);
@@ -360,8 +360,8 @@ struct WmmaElementwiseOpToNVVMLowering
     for (size_t i = 0, e = destType.getBody().size(); i < e; ++i) {
       SmallVector<Value> extractedOperands;
       for (size_t opIdx = 0; opIdx < numOperands; opIdx++) {
-        extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
-            loc, adaptor.getOperands()[opIdx], i));
+        extractedOperands.push_back(LLVM::ExtractValueOp::create(
+            rewriter, loc, adaptor.getOperands()[opIdx], i));
       }
       Value element =
           createScalarOp(rewriter, loc, subgroupMmaElementwiseOp.getOpType(),
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 9b43d3ef602d1..d22364e1ef441 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -85,9 +85,9 @@ static Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
   Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
   Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
   Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, int32Type,
-                                                    ValueRange{minus1, zero});
+                                           ValueRange{minus1, zero});
   Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, int32Type,
-                                                   ValueRange{minus1, mbcntLo});
+                                          ValueRange{minus1, mbcntLo});
   return laneId;
 }
 static constexpr StringLiteral amdgcnDataLayout =
@@ -112,19 +112,19 @@ struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
     Type intTy = IntegerType::get(context, 32);
     Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
     Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
-    Value mbcntLo =
-        ROCDL::MbcntLoOp::create(rewriter, loc, intTy, ValueRange{minus1, zero});
-    Value laneId = ROCDL::MbcntHiOp::create(rewriter,
-        loc, intTy, ValueRange{minus1, mbcntLo});
+    Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, intTy,
+                                             ValueRange{minus1, zero});
+    Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, intTy,
+                                            ValueRange{minus1, mbcntLo});
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      laneId = LLVM::SExtOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), laneId);
+      laneId = LLVM::SExtOp::create(
+          rewriter, loc, IntegerType::get(context, indexBitwidth), laneId);
     } else if (indexBitwidth < 32) {
-      laneId = LLVM::TruncOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), laneId);
+      laneId = LLVM::TruncOp::create(
+          rewriter, loc, IntegerType::get(context, indexBitwidth), laneId);
     }
     rewriter.replaceOp(op, {laneId});
     return success();
@@ -149,8 +149,8 @@ struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
           /*bitWidth=*/32, /*lower=*/isBeforeGfx10 ? 64 : 32,
           /*upper=*/op.getUpperBoundAttr().getInt() + 1);
     }
-    Value wavefrontOp = ROCDL::WavefrontSizeOp::create(rewriter,
-        op.getLoc(), rewriter.getI32Type(), bounds);
+    Value wavefrontOp = ROCDL::WavefrontSizeOp::create(
+        rewriter, op.getLoc(), rewriter.getI32Type(), bounds);
     wavefrontOp = truncOrExtToLLVMType(rewriter, op.getLoc(), wavefrontOp,
                                        *getTypeConverter());
     rewriter.replaceOp(op, {wavefrontOp});
@@ -200,24 +200,24 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     switch (op.getMode()) {
     case gpu::ShuffleMode::UP:
       dstLane = LLVM::SubOp::create(rewriter, loc, int32Type, srcLaneId,
-                                             adaptor.getOffset());
+                                    adaptor.getOffset());
       break;
     case gpu::ShuffleMode::DOWN:
       dstLane = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId,
-                                             adaptor.getOffset());
+                                    adaptor.getOffset());
       break;
     case gpu::ShuffleMode::XOR:
       dstLane = LLVM::XOrOp::create(rewriter, loc, int32Type, srcLaneId,
-                                             adaptor.getOffset());
+                                    adaptor.getOffset());
       break;
     case gpu::ShuffleMode::IDX:
       dstLane = adaptor.getOffset();
       break;
     }
-    Value isActiveSrcLane = LLVM::ICmpOp::create(rewriter,
-        loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
+    Value isActiveSrcLane = LLVM::ICmpOp::create(
+        rewriter, loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
     Value selectDstLane = LLVM::SelectOp::create(rewriter, loc, isActiveSrcLane,
-                                                          dstLane, srcLaneId);
+                                                 dstLane, srcLaneId);
     Value two = LLVM::ConstantOp::create(rewriter, loc, int32Type, 2);
     Value dwordAlignedDstLane =
         LLVM::ShlOp::create(rewriter, loc, int32Type, selectDstLane, two);
@@ -227,7 +227,7 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
       Value res = ROCDL::DsBpermuteOp::create(rewriter, loc, int32Type,
-                                                       dwordAlignedDstLane, v);
+                                              dwordAlignedDstLane, v);
       swizzled.emplace_back(res);
     }
     Value shflValue =
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index 71618cbf8a2f2..a19194eb181fb 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -169,8 +169,8 @@ LogicalResult LaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
 
   Value vector =
       spirv::getBuiltinVariableValue(op, builtin, builtinType, rewriter);
-  Value dim = spirv::CompositeExtractOp::create(rewriter,
-      op.getLoc(), builtinType, vector,
+  Value dim = spirv::CompositeExtractOp::create(
+      rewriter, op.getLoc(), builtinType, vector,
       rewriter.getI32ArrayAttr({static_cast<int32_t>(op.getDimension())}));
   if (forShader && builtinType != indexType)
     dim = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType, dim);
@@ -199,7 +199,7 @@ SingleDimLaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
       spirv::getBuiltinVariableValue(op, builtin, i32Type, rewriter);
   if (i32Type != indexType)
     builtinValue = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType,
-                                                      builtinValue);
+                                             builtinValue);
   rewriter.replaceOp(op, builtinValue);
   return success();
 }
@@ -257,8 +257,8 @@ lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter,
       signatureConverter.addInputs(argType.index(), convertedType);
     }
   }
-  auto newFuncOp = spirv::FuncOp::create(rewriter,
-      funcOp.getLoc(), funcOp.getName(),
+  auto newFuncOp = spirv::FuncOp::create(
+      rewriter, funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(signatureConverter.getConvertedTypes(), {}));
   for (const auto &namedAttr : funcOp->getAttrs()) {
     if (namedAttr.getName() == funcOp.getFunctionTypeAttrName() ||
@@ -367,8 +367,8 @@ LogicalResult GPUModuleConversion::matchAndRewrite(
 
   // Add a keyword to the module name to avoid symbolic conflict.
   std::string spvModuleName = (kSPIRVModule + moduleOp.getName()).str();
-  auto spvModule = spirv::ModuleOp::create(rewriter,
-      moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
+  auto spvModule = spirv::ModuleOp::create(
+      rewriter, moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
       StringRef(spvModuleName));
 
   // Move the region from the module op into the SPIR-V module.
@@ -452,42 +452,42 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
 
   switch (shuffleOp.getMode()) {
   case gpu::ShuffleMode::XOR: {
-    result = spirv::GroupNonUniformShuffleXorOp::create(rewriter,
-        loc, scope, adaptor.getValue(), adaptor.getOffset());
+    result = spirv::GroupNonUniformShuffleXorOp::create(
+        rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::IDX: {
-    result = spirv::GroupNonUniformShuffleOp::create(rewriter,
-        loc, scope, adaptor.getValue(), adaptor.getOffset());
+    result = spirv::GroupNonUniformShuffleOp::create(
+        rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::DOWN: {
-    result = spirv::GroupNonUniformShuffleDownOp::create(rewriter,
-        loc, scope, adaptor.getValue(), adaptor.getOffset());
+    result = spirv::GroupNonUniformShuffleDownOp::create(
+        rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset());
 
     Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
         arith::AddIOp::create(rewriter, loc, laneId, adaptor.getOffset());
     validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
-                                              resultLaneId, adaptor.getWidth());
+                                     resultLaneId, adaptor.getWidth());
     break;
   }
   case gpu::ShuffleMode::UP: {
-    result = spirv::GroupNonUniformShuffleUpOp::create(rewriter,
-        loc, scope, adaptor.getValue(), adaptor.getOffset());
+    result = spirv::GroupNonUniformShuffleUpOp::create(
+        rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset());
 
     Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
         arith::SubIOp::create(rewriter, loc, laneId, adaptor.getOffset());
     auto i32Type = rewriter.getIntegerType(32);
-    validVal = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sge, resultLaneId,
-        arith::ConstantOp::create(rewriter,
-            loc, i32Type, rewriter.getIntegerAttr(i32Type, 0)));
+    validVal = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sge, resultLaneId,
+        arith::ConstantOp::create(rewriter, loc, i32Type,
+                                  rewriter.getIntegerAttr(i32Type, 0)));
     break;
   }
   }
@@ -516,15 +516,16 @@ LogicalResult GPURotateConversion::matchAndRewrite(
 
   Location loc = rotateOp.getLoc();
   auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
-  Value rotateResult = spirv::GroupNonUniformRotateKHROp::create(rewriter,
-      loc, scope, adaptor.getValue(), adaptor.getOffset(), adaptor.getWidth());
+  Value rotateResult = spirv::GroupNonUniformRotateKHROp::create(
+      rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset(),
+      adaptor.getWidth());
   Value validVal;
   if (widthAttr.getValue().getZExtValue() == subgroupSize) {
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(), loc, rewriter);
   } else {
     Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
-                                              laneId, adaptor.getWidth());
+                                     laneId, adaptor.getWidth());
   }
 
   rewriter.replaceOp(rotateOp, {rotateResult, validVal});
@@ -554,8 +555,8 @@ static Value createGroupReduceOpImpl(OpBuilder &builder, Location loc,
 
   Value clusterSizeValue;
   if (clusterSize.has_value())
-    clusterSizeValue = spirv::ConstantOp::create(builder,
-        loc, builder.getI32Type(),
+    clusterSizeValue = spirv::ConstantOp::create(
+        builder, loc, builder.getI32Type(),
         builder.getIntegerAttr(builder.getI32Type(), *clusterSize));
 
   return builder
@@ -740,8 +741,8 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstName =
         makeVarName(moduleOp, llvm::Twine(globalVarName) + "_sc");
 
-    return spirv::SpecConstantOp::create(rewriter,
-        loc, rewriter.getStringAttr(specCstName), attr);
+    return spirv::SpecConstantOp::create(
+        rewriter, loc, rewriter.getStringAttr(specCstName), attr);
   };
   {
     Operation *parent =
@@ -774,8 +775,8 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstCompositeName =
         (llvm::Twine(globalVarName) + "_scc").str();
 
-    specCstComposite = spirv::SpecConstantCompositeOp::create(rewriter,
-        loc, TypeAttr::get(globalType),
+    specCstComposite = spirv::SpecConstantCompositeOp::create(
+        rewriter, loc, TypeAttr::get(globalType),
         rewriter.getStringAttr(specCstCompositeName),
         rewriter.getArrayAttr(constituents));
 
@@ -785,16 +786,17 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     // Define a GlobalVarOp initialized using specialized constants
     // that is used to specify the printf format string
     // to be passed to the SPIRV CLPrintfOp.
-    globalVar = spirv::GlobalVariableOp::create(rewriter,
-        loc, ptrType, globalVarName, FlatSymbolRefAttr::get(specCstComposite));
+    globalVar = spirv::GlobalVariableOp::create(
+        rewriter, loc, ptrType, globalVarName,
+        FlatSymbolRefAttr::get(specCstComposite));
 
     globalVar->setAttr("Constant", rewriter.getUnitAttr());
   }
   // Get SSA value of Global variable and create pointer to i8 to point to
   // the format string.
   Value globalPtr = spirv::AddressOfOp::create(rewriter, loc, globalVar);
-  Value fmtStr = spirv::BitcastOp::create(rewriter,
-      loc,
+  Value fmtStr = spirv::BitcastOp::create(
+      rewriter, loc,
       spirv::PointerType::get(i8Type, spirv::StorageClass::UniformConstant),
       globalPtr);
 
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index da8ffe47a7d33..a344f88326089 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -144,8 +144,9 @@ void GPUToSPIRVPass::runOnOperation() {
     if (targetEnvSupportsKernelCapability(moduleOp)) {
       moduleOp.walk([&](gpu::GPUFuncOp funcOp) {
         builder.setInsertionPoint(funcOp);
-        auto newFuncOp = func::FuncOp::create(builder,
-            funcOp.getLoc(), funcOp.getName(), funcOp.getFunctionType());
+        auto newFuncOp =
+            func::FuncOp::create(builder, funcOp.getLoc(), funcOp.getName(),
+                                 funcOp.getFunctionType());
         auto entryBlock = newFuncOp.addEntryBlock();
         builder.setInsertionPointToEnd(entryBlock);
         func::ReturnOp::create(builder, funcOp.getLoc());
diff --git a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
index 70646aa5f2e74..51dc50048024f 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
@@ -283,8 +283,8 @@ struct WmmaLoadOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = spirv::ConstantOp::create(rewriter,
-        loc, i32Type, IntegerAttr::get(i32Type, stride));
+    auto strideValue = spirv::ConstantOp::create(
+        rewriter, loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
     auto layout = isColMajor ? spirv::CooperativeMatrixLayoutKHR::ColumnMajor
@@ -315,8 +315,8 @@ struct WmmaStoreOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = spirv::ConstantOp::create(rewriter,
-        loc, i32Type, IntegerAttr::get(i32Type, stride));
+    auto strideValue = spirv::ConstantOp::create(
+        rewriter, loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
     auto layout = isColMajor ? spirv::CooperativeMatrixLayoutKHR::ColumnMajor
diff --git a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
index 0d5b318f739cc..99d2f6ca78c38 100644
--- a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
+++ b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
@@ -59,8 +59,8 @@ struct ConvertIndexCeilDivS : mlir::ConvertOpToLLVMPattern<CeilDivSOp> {
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
     Value nPos =
         LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, n, zero);
-    Value sameSign =
-        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, nPos, mPos);
+    Value sameSign = LLVM::ICmpOp::create(rewriter, loc,
+                                          LLVM::ICmpPredicate::eq, nPos, mPos);
     Value nNonZero =
         LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
     Value cmp = LLVM::AndOp::create(rewriter, loc, sameSign, nNonZero);
@@ -135,8 +135,8 @@ struct ConvertIndexFloorDivS : mlir::ConvertOpToLLVMPattern<FloorDivSOp> {
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
     Value nNeg =
         LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, n, zero);
-    Value diffSign =
-        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
+    Value diffSign = LLVM::ICmpOp::create(rewriter, loc,
+                                          LLVM::ICmpPredicate::ne, nNeg, mNeg);
     Value nNonZero =
         LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
     Value cmp = LLVM::AndOp::create(rewriter, loc, diffSign, nNonZero);
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index 6fc097059c5e5..36cfe9dd6e2db 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -111,12 +111,12 @@ struct ConvertIndexCeilDivSPattern final : OpConversionPattern<CeilDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, -1));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                           IntegerAttr::get(n_type, 0));
+    Value posOne = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                             IntegerAttr::get(n_type, 1));
+    Value negOne = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                             IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
     Value mPos = spirv::SGreaterThanOp::create(rewriter, loc, m, zero);
@@ -161,10 +161,10 @@ struct ConvertIndexCeilDivUPattern final : OpConversionPattern<CeilDivUOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 0));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                           IntegerAttr::get(n_type, 0));
     Value one = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                                   IntegerAttr::get(n_type, 1));
+                                          IntegerAttr::get(n_type, 1));
 
     // Compute the non-zero result.
     Value minusOne = spirv::ISubOp::create(rewriter, loc, n, one);
@@ -197,12 +197,12 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, -1));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                           IntegerAttr::get(n_type, 0));
+    Value posOne = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                             IntegerAttr::get(n_type, 1));
+    Value negOne = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                             IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
     Value mNeg = spirv::SLessThanOp::create(rewriter, loc, m, zero);
@@ -219,7 +219,8 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
     Value nNeg = spirv::SLessThanOp::create(rewriter, loc, n, zero);
-    Value diffSign = spirv::LogicalNotEqualOp::create(rewriter, loc, nNeg, mNeg);
+    Value diffSign =
+        spirv::LogicalNotEqualOp::create(rewriter, loc, nNeg, mNeg);
     Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
 
     Value cmp = spirv::LogicalAndOp::create(rewriter, loc, diffSign, nNonZero);
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index e8ad0d4644389..fce7a3f324b86 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -100,20 +100,20 @@ void MemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc,
 static Value createIndexAttrConstant(OpBuilder &builder, Location loc,
                                      Type resultType, int64_t value) {
   return LLVM::ConstantOp::create(builder, loc, resultType,
-                                          builder.getIndexAttr(value));
+                                  builder.getIndexAttr(value));
 }
 
 /// Builds IR extracting the offset from the descriptor.
 Value MemRefDescriptor::offset(OpBuilder &builder, Location loc) {
   return LLVM::ExtractValueOp::create(builder, loc, value,
-                                              kOffsetPosInMemRefDescriptor);
+                                      kOffsetPosInMemRefDescriptor);
 }
 
 /// Builds IR inserting the offset into the descriptor.
 void MemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                  Value offset) {
   value = LLVM::InsertValueOp::create(builder, loc, value, offset,
-                                              kOffsetPosInMemRefDescriptor);
+                                      kOffsetPosInMemRefDescriptor);
 }
 
 /// Builds IR inserting the offset into the descriptor.
@@ -125,8 +125,9 @@ void MemRefDescriptor::setConstantOffset(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th size from the descriptor.
 Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) {
-  return LLVM::ExtractValueOp::create(builder,
-      loc, value, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
+  return LLVM::ExtractValueOp::create(
+      builder, loc, value,
+      ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
 Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
@@ -137,23 +138,25 @@ Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
 
   // Copy size values to stack-allocated memory.
   auto one = createIndexAttrConstant(builder, loc, indexType, 1);
-  auto sizes = LLVM::ExtractValueOp::create(builder,
-      loc, value, llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
+  auto sizes = LLVM::ExtractValueOp::create(
+      builder, loc, value,
+      llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
   auto sizesPtr = LLVM::AllocaOp::create(builder, loc, ptrTy, arrayTy, one,
-                                                 /*alignment=*/0);
+                                         /*alignment=*/0);
   LLVM::StoreOp::create(builder, loc, sizes, sizesPtr);
 
   // Load an return size value of interest.
   auto resultPtr = LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, sizesPtr,
-                                               ArrayRef<LLVM::GEPArg>{0, pos});
+                                       ArrayRef<LLVM::GEPArg>{0, pos});
   return LLVM::LoadOp::create(builder, loc, indexType, resultPtr);
 }
 
 /// Builds IR inserting the pos-th size into the descriptor
 void MemRefDescriptor::setSize(OpBuilder &builder, Location loc, unsigned pos,
                                Value size) {
-  value = LLVM::InsertValueOp::create(builder,
-      loc, value, size, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
+  value = LLVM::InsertValueOp::create(
+      builder, loc, value, size,
+      ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
 void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
@@ -164,15 +167,16 @@ void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th stride from the descriptor.
 Value MemRefDescriptor::stride(OpBuilder &builder, Location loc, unsigned pos) {
-  return LLVM::ExtractValueOp::create(builder,
-      loc, value, ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
+  return LLVM::ExtractValueOp::create(
+      builder, loc, value,
+      ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
 
 /// Builds IR inserting the pos-th stride into the descriptor
 void MemRefDescriptor::setStride(OpBuilder &builder, Location loc, unsigned pos,
                                  Value stride) {
-  value = LLVM::InsertValueOp::create(builder,
-      loc, value, stride,
+  value = LLVM::InsertValueOp::create(
+      builder, loc, value, stride,
       ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
 
@@ -208,7 +212,7 @@ Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
           : createIndexAttrConstant(builder, loc, indexType, offsetCst);
   Type elementType = converter.convertType(type.getElementType());
   ptr = LLVM::GEPOp::create(builder, loc, ptr.getType(), elementType, ptr,
-                                    offsetVal);
+                            offsetVal);
   return ptr;
 }
 
@@ -387,12 +391,12 @@ void UnrankedMemRefDescriptor::computeSizes(
     Value doubleRank = LLVM::MulOp::create(builder, loc, indexType, two, rank);
     Value doubleRankIncremented =
         LLVM::AddOp::create(builder, loc, indexType, doubleRank, one);
-    Value rankIndexSize = LLVM::MulOp::create(builder,
-        loc, indexType, doubleRankIncremented, indexSize);
+    Value rankIndexSize = LLVM::MulOp::create(builder, loc, indexType,
+                                              doubleRankIncremented, indexSize);
 
     // Total allocation size.
-    Value allocationSize = LLVM::AddOp::create(builder,
-        loc, indexType, doublePointerSize, rankIndexSize);
+    Value allocationSize = LLVM::AddOp::create(
+        builder, loc, indexType, doublePointerSize, rankIndexSize);
     sizes.push_back(allocationSize);
   }
 }
@@ -424,7 +428,7 @@ Value UnrankedMemRefDescriptor::alignedPtr(
 
   Value alignedGep =
       LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
-                                  elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
+                          elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
   return LLVM::LoadOp::create(builder, loc, elemPtrType, alignedGep);
 }
 
@@ -436,7 +440,7 @@ void UnrankedMemRefDescriptor::setAlignedPtr(
 
   Value alignedGep =
       LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
-                                  elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
+                          elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
   LLVM::StoreOp::create(builder, loc, alignedPtr, alignedGep);
 }
 
@@ -447,7 +451,7 @@ Value UnrankedMemRefDescriptor::offsetBasePtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   return LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
-                                     elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
+                             elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
 }
 
 Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
@@ -457,7 +461,7 @@ Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
   return LLVM::LoadOp::create(builder, loc, typeConverter.getIndexType(),
-                                      offsetPtr);
+                              offsetPtr);
 }
 
 void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
@@ -478,7 +482,7 @@ Value UnrankedMemRefDescriptor::sizeBasePtr(
       indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy});
   auto resultType = LLVM::LLVMPointerType::get(builder.getContext());
   return LLVM::GEPOp::create(builder, loc, resultType, structTy, memRefDescPtr,
-                                     ArrayRef<LLVM::GEPArg>{0, 3});
+                             ArrayRef<LLVM::GEPArg>{0, 3});
 }
 
 Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc,
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index 57aed79874ad0..ecd5b6367fba4 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -58,7 +58,7 @@ Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder,
                                                     Type resultType,
                                                     int64_t value) {
   return LLVM::ConstantOp::create(builder, loc, resultType,
-                                          builder.getIndexAttr(value));
+                                  builder.getIndexAttr(value));
 }
 
 Value ConvertToLLVMPattern::getStridedElementPtr(
@@ -132,8 +132,8 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
     Type elementType = typeConverter->convertType(memRefType.getElementType());
     auto elementPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
     Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
-    Value gepPtr = LLVM::GEPOp::create(rewriter,
-        loc, elementPtrType, elementType, nullPtr, runningStride);
+    Value gepPtr = LLVM::GEPOp::create(rewriter, loc, elementPtrType,
+                                       elementType, nullPtr, runningStride);
     size = LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
   } else {
     size = runningStride;
@@ -151,7 +151,7 @@ Value ConvertToLLVMPattern::getSizeInBytes(
   auto convertedPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
   auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, convertedPtrType);
   auto gep = LLVM::GEPOp::create(rewriter, loc, convertedPtrType, llvmType,
-                                          nullPtr, ArrayRef<LLVM::GEPArg>{1});
+                                 nullPtr, ArrayRef<LLVM::GEPArg>{1});
   return LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gep);
 }
 
@@ -277,9 +277,9 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
                   .create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
                   .getResult()
             : LLVM::AllocaOp::create(builder, loc, getPtrType(),
-                                             IntegerType::get(getContext(), 8),
-                                             allocationSize,
-                                             /*alignment=*/0);
+                                     IntegerType::get(getContext(), 8),
+                                     allocationSize,
+                                     /*alignment=*/0);
     Value source = desc.memRefDescPtr(builder, loc);
     LLVM::MemcpyOp::create(builder, loc, memory, source, allocationSize, false);
     if (!toDynamic)
@@ -349,8 +349,8 @@ LogicalResult LLVM::detail::oneToOneRewrite(
   SmallVector<Value, 4> results;
   results.reserve(numResults);
   for (unsigned i = 0; i < numResults; ++i) {
-    results.push_back(LLVM::ExtractValueOp::create(rewriter,
-        op->getLoc(), newOp->getResult(0), i));
+    results.push_back(LLVM::ExtractValueOp::create(rewriter, op->getLoc(),
+                                                   newOp->getResult(0), i));
   }
   rewriter.replaceOp(op, results);
   return success();
@@ -371,8 +371,8 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   if (numResults != 0)
     resType = typeConverter.packOperationResults(op->getResultTypes());
 
-  auto callIntrOp = LLVM::CallIntrinsicOp::create(rewriter,
-      loc, resType, rewriter.getStringAttr(intrinsic), operands);
+  auto callIntrOp = LLVM::CallIntrinsicOp::create(
+      rewriter, loc, resType, rewriter.getStringAttr(intrinsic), operands);
   // Propagate attributes.
   callIntrOp->setAttrs(op->getAttrDictionary());
 
@@ -518,20 +518,20 @@ Value mlir::LLVM::getStridedElementPtr(OpBuilder &builder, Location loc,
       Value stride =
           ShapedType::isDynamic(strides[i])
               ? memRefDescriptor.stride(builder, loc, i)
-              : LLVM::ConstantOp::create(builder,
-                    loc, indexType, builder.getIndexAttr(strides[i]));
-      increment =
-          LLVM::MulOp::create(builder, loc, increment, stride, intOverflowFlags);
+              : LLVM::ConstantOp::create(builder, loc, indexType,
+                                         builder.getIndexAttr(strides[i]));
+      increment = LLVM::MulOp::create(builder, loc, increment, stride,
+                                      intOverflowFlags);
     }
     index = index ? LLVM::AddOp::create(builder, loc, index, increment,
-                                                intOverflowFlags)
+                                        intOverflowFlags)
                   : increment;
   }
 
   Type elementPtrType = memRefDescriptor.getElementPtrType();
-  return index ? LLVM::GEPOp::create(builder,
-                     loc, elementPtrType,
-                     converter.convertType(type.getElementType()), base, index,
-                     noWrapFlags)
-               : base;
+  return index
+             ? LLVM::GEPOp::create(builder, loc, elementPtrType,
+                                   converter.convertType(type.getElementType()),
+                                   base, index, noWrapFlags)
+             : base;
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
index 091fb7e4c2afb..d95aeba8a4488 100644
--- a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
@@ -66,8 +66,8 @@ LogicalResult mlir::LLVM::createPrintStrCall(
       DenseElementsAttr::get(dataAttrType, llvm::ArrayRef(elementVals));
   auto arrayTy =
       LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), elementVals.size());
-  auto globalOp = LLVM::GlobalOp::create(builder,
-      loc, arrayTy, /*constant=*/true, LLVM::Linkage::Private,
+  auto globalOp = LLVM::GlobalOp::create(
+      builder, loc, arrayTy, /*constant=*/true, LLVM::Linkage::Private,
       ensureSymbolNameIsUnique(moduleOp, symbolName, symbolTables), dataAttr);
 
   auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext());
@@ -83,6 +83,6 @@ LogicalResult mlir::LLVM::createPrintStrCall(
   if (failed(printer))
     return failure();
   LLVM::CallOp::create(builder, loc, TypeRange(),
-                               SymbolRefAttr::get(printer.value()), gep);
+                       SymbolRefAttr::get(printer.value()), gep);
   return success();
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 77135188dabcf..1a9bf569086da 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -732,7 +732,7 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand,
   // alloca op and so we omit allocating at the entry block.
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
   Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
-                                               builder.getIndexAttr(1));
+                                       builder.getIndexAttr(1));
   Value allocated =
       LLVM::AllocaOp::create(builder, loc, ptrType, operand.getType(), one);
   // Store into the alloca'ed descriptor.
diff --git a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
index 4fc208e05eb9d..e7dd0b506e12d 100644
--- a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
@@ -93,8 +93,8 @@ LogicalResult LLVM::detail::handleMultidimensionalVectors(
     // element, extract operand vectors
     SmallVector<Value, 4> extractedOperands;
     for (const auto &operand : llvm::enumerate(operands)) {
-      extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
-          loc, operand.value(), position));
+      extractedOperands.push_back(LLVM::ExtractValueOp::create(
+          rewriter, loc, operand.value(), position));
     }
     Value newVal = createOperand(result1DVectorTy, extractedOperands);
     desc = LLVM::InsertValueOp::create(rewriter, loc, desc, newVal, position);
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 498a5cfc40204..3f4b4d6cbc8ab 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -78,8 +78,8 @@ getLibraryCallSymbolRef(Operation *op, PatternRewriter &rewriter) {
   // Insert before module terminator.
   rewriter.setInsertionPoint(module.getBody(),
                              std::prev(module.getBody()->end()));
-  func::FuncOp funcOp = func::FuncOp::create(rewriter,
-      op->getLoc(), fnNameAttr.getValue(), libFnType);
+  func::FuncOp funcOp = func::FuncOp::create(rewriter, op->getLoc(),
+                                             fnNameAttr.getValue(), libFnType);
   // Insert a function attribute that will trigger the emission of the
   // corresponding `_mlir_ciface_xxx` interface so that external libraries see
   // a normalized ABI. This interface is added during std to llvm conversion.
@@ -100,8 +100,8 @@ createTypeCanonicalizedMemRefOperands(OpBuilder &b, Location loc,
       res.push_back(op);
       continue;
     }
-    Value cast =
-        memref::CastOp::create(b, loc, makeStridedLayoutDynamic(memrefType), op);
+    Value cast = memref::CastOp::create(
+        b, loc, makeStridedLayoutDynamic(memrefType), op);
     res.push_back(cast);
   }
   return res;
diff --git a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
index 30521d66a2e4c..5b68eb8188996 100644
--- a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
+++ b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
@@ -55,14 +55,14 @@ std::pair<Value, Value> getRawPtrAndSize(const Location loc,
   Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
   Value dataPtr =
       LLVM::ExtractValueOp::create(rewriter, loc, ptrType, memRef, 1);
-  Value offset = LLVM::ExtractValueOp::create(rewriter,
-      loc, rewriter.getI64Type(), memRef, 2);
+  Value offset = LLVM::ExtractValueOp::create(rewriter, loc,
+                                              rewriter.getI64Type(), memRef, 2);
   Value resPtr =
       LLVM::GEPOp::create(rewriter, loc, ptrType, elType, dataPtr, offset);
   Value size;
   if (cast<LLVM::LLVMStructType>(memRef.getType()).getBody().size() > 3) {
     size = LLVM::ExtractValueOp::create(rewriter, loc, memRef,
-                                                 ArrayRef<int64_t>{3, 0});
+                                        ArrayRef<int64_t>{3, 0});
     size = LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), size);
   } else {
     size = arith::ConstantIntOp::create(rewriter, loc, 1, 32);
@@ -158,7 +158,7 @@ class MPICHImplTraits : public MPIImplTraits {
                      ConversionPatternRewriter &rewriter) override {
     static constexpr int MPI_COMM_WORLD = 0x44000000;
     return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
-                                             MPI_COMM_WORLD);
+                                    MPI_COMM_WORLD);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
@@ -195,7 +195,8 @@ class MPICHImplTraits : public MPIImplTraits {
       mtype = MPI_UINT8_T;
     else
       assert(false && "unsupported type");
-    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), mtype);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
+                                    mtype);
   }
 
   Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter,
@@ -281,16 +282,16 @@ class OMPIImplTraits : public MPIImplTraits {
     getOrDefineExternalStruct(loc, rewriter, name, commStructT);
 
     // get address of symbol
-    auto comm = LLVM::AddressOfOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(context),
-        SymbolRefAttr::get(context, name));
+    auto comm = LLVM::AddressOfOp::create(rewriter, loc,
+                                          LLVM::LLVMPointerType::get(context),
+                                          SymbolRefAttr::get(context, name));
     return LLVM::PtrToIntOp::create(rewriter, loc, rewriter.getI64Type(), comm);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return LLVM::IntToPtrOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(rewriter.getContext()), comm);
+    return LLVM::IntToPtrOp::create(
+        rewriter, loc, LLVM::LLVMPointerType::get(rewriter.getContext()), comm);
   }
 
   intptr_t getStatusIgnore() override { return 0; }
@@ -330,9 +331,9 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, mtype, typeStructT);
     // get address of symbol
-    return LLVM::AddressOfOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(context),
-        SymbolRefAttr::get(context, mtype));
+    return LLVM::AddressOfOp::create(rewriter, loc,
+                                     LLVM::LLVMPointerType::get(context),
+                                     SymbolRefAttr::get(context, mtype));
   }
 
   Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter,
@@ -389,9 +390,9 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, op, opStructT);
     // get address of symbol
-    return LLVM::AddressOfOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(context),
-        SymbolRefAttr::get(context, op));
+    return LLVM::AddressOfOp::create(rewriter, loc,
+                                     LLVM::LLVMPointerType::get(context),
+                                     SymbolRefAttr::get(context, op));
   }
 };
 
@@ -524,10 +525,10 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
     LLVM::LLVMFuncOp funcDecl = getOrDefineFunction(moduleOp, loc, rewriter,
                                                     "MPI_Comm_split", funcType);
 
-    auto callOp = LLVM::CallOp::create(rewriter,
-        loc, funcDecl,
-        ValueRange{comm, adaptor.getColor(), adaptor.getKey(),
-                   outPtr.getRes()});
+    auto callOp =
+        LLVM::CallOp::create(rewriter, loc, funcDecl,
+                             ValueRange{comm, adaptor.getColor(),
+                                        adaptor.getKey(), outPtr.getRes()});
 
     // load the communicator into a register
     Value res = LLVM::LoadOp::create(rewriter, loc, i32, outPtr.getResult());
@@ -582,8 +583,8 @@ struct CommRankOpLowering : public ConvertOpToLLVMPattern<mpi::CommRankOp> {
     // replace with function call
     auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
     auto rankptr = LLVM::AllocaOp::create(rewriter, loc, ptrType, i32, one);
-    auto callOp = LLVM::CallOp::create(rewriter,
-        loc, initDecl, ValueRange{comm, rankptr.getRes()});
+    auto callOp = LLVM::CallOp::create(rewriter, loc, initDecl,
+                                       ValueRange{comm, rankptr.getRes()});
 
     // load the rank into a register
     auto loadedRank =
@@ -641,10 +642,10 @@ struct SendOpLowering : public ConvertOpToLLVMPattern<mpi::SendOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Send", funcType);
 
     // replace op with function call
-    auto funcCall = LLVM::CallOp::create(rewriter,
-        loc, funcDecl,
-        ValueRange{dataPtr, size, dataType, adaptor.getDest(), adaptor.getTag(),
-                   comm});
+    auto funcCall = LLVM::CallOp::create(rewriter, loc, funcDecl,
+                                         ValueRange{dataPtr, size, dataType,
+                                                    adaptor.getDest(),
+                                                    adaptor.getTag(), comm});
     if (op.getRetval())
       rewriter.replaceOp(op, funcCall.getResult());
     else
@@ -683,8 +684,8 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
     auto mpiTraits = MPIImplTraits::get(moduleOp);
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    Value statusIgnore = LLVM::ConstantOp::create(rewriter,
-        loc, i64, mpiTraits->getStatusIgnore());
+    Value statusIgnore = LLVM::ConstantOp::create(rewriter, loc, i64,
+                                                  mpiTraits->getStatusIgnore());
     statusIgnore =
         LLVM::IntToPtrOp::create(rewriter, loc, ptrType, statusIgnore);
 
@@ -698,8 +699,8 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Recv", funcType);
 
     // replace op with function call
-    auto funcCall = LLVM::CallOp::create(rewriter,
-        loc, funcDecl,
+    auto funcCall = LLVM::CallOp::create(
+        rewriter, loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getSource(),
                    adaptor.getTag(), comm, statusIgnore});
     if (op.getRetval())
@@ -738,8 +739,9 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
 
     // If input and output are the same, request in-place operation.
     if (adaptor.getSendbuf() == adaptor.getRecvbuf()) {
-      sendPtr = LLVM::ConstantOp::create(rewriter,
-          loc, i64, reinterpret_cast<int64_t>(mpiTraits->getInPlace()));
+      sendPtr = LLVM::ConstantOp::create(
+          rewriter, loc, i64,
+          reinterpret_cast<int64_t>(mpiTraits->getInPlace()));
       sendPtr = LLVM::IntToPtrOp::create(rewriter, loc, ptrType, sendPtr);
     }
 
@@ -757,8 +759,8 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Allreduce", funcType);
 
     // replace op with function call
-    auto funcCall = LLVM::CallOp::create(rewriter,
-        loc, funcDecl,
+    auto funcCall = LLVM::CallOp::create(
+        rewriter, loc, funcDecl,
         ValueRange{sendPtr, recvPtr, sendSize, dataType, mpiOp, commWorld});
 
     if (op.getRetval())
diff --git a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
index bb3511440e041..08a456691880c 100644
--- a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
+++ b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
@@ -121,8 +121,8 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     initValueAttr = FloatAttr::get(resultElementType, 0.0);
   else
     initValueAttr = IntegerAttr::get(resultElementType, 0);
-  Value result = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(vecType, initValueAttr));
+  Value result = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(vecType, initValueAttr));
   SmallVector<int64_t> strides = computeStrides(shape);
   for (int64_t linearIndex = 0; linearIndex < numElements; ++linearIndex) {
     SmallVector<int64_t> positions = delinearize(linearIndex, strides);
@@ -208,12 +208,12 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value zeroValue = arith::ConstantOp::create(builder,
-      elementType, builder.getIntegerAttr(elementType, 0));
-  Value oneValue = arith::ConstantOp::create(builder,
-      elementType, builder.getIntegerAttr(elementType, 1));
-  Value minusOneValue = arith::ConstantOp::create(builder,
-      elementType,
+  Value zeroValue = arith::ConstantOp::create(
+      builder, elementType, builder.getIntegerAttr(elementType, 0));
+  Value oneValue = arith::ConstantOp::create(
+      builder, elementType, builder.getIntegerAttr(elementType, 1));
+  Value minusOneValue = arith::ConstantOp::create(
+      builder, elementType,
       builder.getIntegerAttr(elementType,
                              APInt(elementType.getIntOrFloatBitWidth(), -1ULL,
                                    /*isSigned=*/true)));
@@ -231,15 +231,16 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
 
   // if (p < T(0)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
-  auto pIsNeg =
-      arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg, zeroValue);
+  auto pIsNeg = arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg,
+                                      zeroValue);
   //   if (b == T(0))
   builder.createBlock(funcBody);
   auto bIsZero =
       arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, zeroValue);
   //     return T(1) / T(0);
   thenBlock = builder.createBlock(funcBody);
-  func::ReturnOp::create(builder,
+  func::ReturnOp::create(
+      builder,
       arith::DivSIOp::create(builder, oneValue, zeroValue).getResult());
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(0)).
@@ -261,12 +262,12 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   //   if (b == T(-1)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto bIsMinusOne = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
-                                                   bArg, minusOneValue);
+                                           bArg, minusOneValue);
   //     if (p & T(1))
   builder.createBlock(funcBody);
-  auto pIsOdd = arith::CmpIOp::create(builder,
-      arith::CmpIPredicate::ne, arith::AndIOp::create(builder, pArg, oneValue),
-      zeroValue);
+  auto pIsOdd = arith::CmpIOp::create(
+      builder, arith::CmpIPredicate::ne,
+      arith::AndIOp::create(builder, pArg, oneValue), zeroValue);
   //       return T(-1);
   thenBlock = builder.createBlock(funcBody);
   func::ReturnOp::create(builder, minusOneValue);
@@ -283,7 +284,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   // Set up conditional branch for (b == T(-1)).
   builder.setInsertionPointToEnd(bIsMinusOne->getBlock());
   cf::CondBranchOp::create(builder, bIsMinusOne, pIsOdd->getBlock(),
-                                   fallthroughBlock);
+                           fallthroughBlock);
 
   //   return T(0);
   // } // (p < T(0))
@@ -296,7 +297,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   builder.setInsertionPointToEnd(pIsNeg->getBlock());
   // Set initial values of 'result', 'b' and 'p' for the loop.
   cf::CondBranchOp::create(builder, pIsNeg, bIsZero->getBlock(), loopHeader,
-                                   ValueRange{oneValue, bArg, pArg});
+                           ValueRange{oneValue, bArg, pArg});
 
   // T result = T(1);
   // while (true) {
@@ -313,8 +314,8 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & T(1))
-  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
-      arith::CmpIPredicate::ne,
+  auto powerTmpIsOdd = arith::CmpIOp::create(
+      builder, arith::CmpIPredicate::ne,
       arith::AndIOp::create(builder, powerTmp, oneValue), zeroValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
@@ -326,7 +327,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
   cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
-                                   resultTmp);
+                           resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
@@ -336,22 +337,23 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
 
   //   if (p == T(0))
   auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
-                                                      newPowerTmp, zeroValue);
+                                              newPowerTmp, zeroValue);
   //     return result;
   thenBlock = builder.createBlock(funcBody);
   func::ReturnOp::create(builder, newResultTmp);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  cf::CondBranchOp::create(builder, newPowerIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, newPowerIsZero, thenBlock,
+                           fallthroughBlock);
 
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
   Value newBaseTmp = arith::MulIOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  cf::BranchOp::create(builder,
-      ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
+  cf::BranchOp::create(
+      builder, ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
   return funcOp;
 }
 
@@ -433,23 +435,25 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value oneBValue = arith::ConstantOp::create(builder,
-      baseType, builder.getFloatAttr(baseType, 1.0));
-  Value zeroPValue = arith::ConstantOp::create(builder,
-      powType, builder.getIntegerAttr(powType, 0));
-  Value onePValue = arith::ConstantOp::create(builder,
-      powType, builder.getIntegerAttr(powType, 1));
-  Value minPValue = arith::ConstantOp::create(builder,
-      powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMinValue(
-                                                   powType.getWidth())));
-  Value maxPValue = arith::ConstantOp::create(builder,
-      powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMaxValue(
-                                                   powType.getWidth())));
+  Value oneBValue = arith::ConstantOp::create(
+      builder, baseType, builder.getFloatAttr(baseType, 1.0));
+  Value zeroPValue = arith::ConstantOp::create(
+      builder, powType, builder.getIntegerAttr(powType, 0));
+  Value onePValue = arith::ConstantOp::create(
+      builder, powType, builder.getIntegerAttr(powType, 1));
+  Value minPValue = arith::ConstantOp::create(
+      builder, powType,
+      builder.getIntegerAttr(
+          powType, llvm::APInt::getSignedMinValue(powType.getWidth())));
+  Value maxPValue = arith::ConstantOp::create(
+      builder, powType,
+      builder.getIntegerAttr(
+          powType, llvm::APInt::getSignedMaxValue(powType.getWidth())));
 
   // if (p == Tp{0})
   //   return Tb{1};
-  auto pIsZero =
-      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroPValue);
+  auto pIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg,
+                                       zeroPValue);
   Block *thenBlock = builder.createBlock(funcBody);
   func::ReturnOp::create(builder, oneBValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
@@ -460,7 +464,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(fallthroughBlock);
   // bool isNegativePower{p < Tp{0}}
   auto pIsNeg = arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg,
-                                              zeroPValue);
+                                      zeroPValue);
   // bool isMin{p == std::numeric_limits<Tp>::min()};
   auto pIsMin =
       arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, minPValue);
@@ -498,8 +502,8 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & Tp{1})
-  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
-      arith::CmpIPredicate::ne,
+  auto powerTmpIsOdd = arith::CmpIOp::create(
+      builder, arith::CmpIPredicate::ne,
       arith::AndIOp::create(builder, powerTmp, onePValue), zeroPValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
@@ -511,7 +515,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   // Set up conditional branch for (p & Tp{1}).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
   cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
-                                   resultTmp);
+                           resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
@@ -521,7 +525,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
 
   //   if (p == Tp{0})
   auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
-                                                      newPowerTmp, zeroPValue);
+                                              newPowerTmp, zeroPValue);
   //     break;
   //
   // The conditional branch is finalized below with a jump to
@@ -533,8 +537,8 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(fallthroughBlock);
   Value newBaseTmp = arith::MulFOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  cf::BranchOp::create(builder,
-      ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
+  cf::BranchOp::create(
+      builder, ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
 
   // Set up conditional branch for early loop exit:
   //   if (p == Tp{0})
@@ -543,7 +547,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
                                         builder.getLoc());
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
   cf::CondBranchOp::create(builder, newPowerIsZero, loopExit, newResultTmp,
-                                   fallthroughBlock, ValueRange{});
+                           fallthroughBlock, ValueRange{});
 
   // if (isMin) {
   //   result *= origBase;
@@ -554,7 +558,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(loopExit);
   cf::CondBranchOp::create(builder, pIsMin, thenBlock, fallthroughBlock,
-                                   newResultTmp);
+                           newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
   newResultTmp = arith::MulFOp::create(builder, newResultTmp, bArg);
   cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
@@ -568,7 +572,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
                                            builder.getLoc());
   builder.setInsertionPointToEnd(fallthroughBlock);
   cf::CondBranchOp::create(builder, pIsNeg, thenBlock, returnBlock,
-                                   newResultTmp);
+                           newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
   newResultTmp = arith::DivFOp::create(builder, oneBValue, newResultTmp);
   cf::BranchOp::create(builder, newResultTmp, returnBlock);
@@ -683,33 +687,34 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
 
   Value arg = funcOp.getArgument(0);
   Type indexType = builder.getIndexType();
-  Value bitWidthValue = arith::ConstantOp::create(builder,
-      elementType, builder.getIntegerAttr(elementType, bitWidth));
-  Value zeroValue = arith::ConstantOp::create(builder,
-      elementType, builder.getIntegerAttr(elementType, 0));
+  Value bitWidthValue = arith::ConstantOp::create(
+      builder, elementType, builder.getIntegerAttr(elementType, bitWidth));
+  Value zeroValue = arith::ConstantOp::create(
+      builder, elementType, builder.getIntegerAttr(elementType, 0));
 
   Value inputEqZero =
       arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, arg, zeroValue);
 
   // if input == 0, return bit width, else enter loop.
-  scf::IfOp ifOp = scf::IfOp::create(builder,
-      elementType, inputEqZero, /*addThenBlock=*/true, /*addElseBlock=*/true);
+  scf::IfOp ifOp =
+      scf::IfOp::create(builder, elementType, inputEqZero,
+                        /*addThenBlock=*/true, /*addElseBlock=*/true);
   ifOp.getThenBodyBuilder().create<scf::YieldOp>(loc, bitWidthValue);
 
   auto elseBuilder =
       ImplicitLocOpBuilder::atBlockEnd(loc, &ifOp.getElseRegion().front());
 
-  Value oneIndex = arith::ConstantOp::create(elseBuilder,
-      indexType, elseBuilder.getIndexAttr(1));
-  Value oneValue = arith::ConstantOp::create(elseBuilder,
-      elementType, elseBuilder.getIntegerAttr(elementType, 1));
-  Value bitWidthIndex = arith::ConstantOp::create(elseBuilder,
-      indexType, elseBuilder.getIndexAttr(bitWidth));
-  Value nValue = arith::ConstantOp::create(elseBuilder,
-      elementType, elseBuilder.getIntegerAttr(elementType, 0));
-
-  auto loop = scf::ForOp::create(elseBuilder,
-      oneIndex, bitWidthIndex, oneIndex,
+  Value oneIndex = arith::ConstantOp::create(elseBuilder, indexType,
+                                             elseBuilder.getIndexAttr(1));
+  Value oneValue = arith::ConstantOp::create(
+      elseBuilder, elementType, elseBuilder.getIntegerAttr(elementType, 1));
+  Value bitWidthIndex = arith::ConstantOp::create(
+      elseBuilder, indexType, elseBuilder.getIndexAttr(bitWidth));
+  Value nValue = arith::ConstantOp::create(
+      elseBuilder, elementType, elseBuilder.getIntegerAttr(elementType, 0));
+
+  auto loop = scf::ForOp::create(
+      elseBuilder, oneIndex, bitWidthIndex, oneIndex,
       // Initial values for two loop induction variables, the arg which is being
       // shifted left in each iteration, and the n value which tracks the count
       // of leading zeros.
@@ -725,10 +730,10 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
         Value argIter = args[0];
         Value nIter = args[1];
 
-        Value argIsNonNegative = arith::CmpIOp::create(b,
-            loc, arith::CmpIPredicate::slt, argIter, zeroValue);
-        scf::IfOp ifOp = scf::IfOp::create(b,
-            loc, argIsNonNegative,
+        Value argIsNonNegative = arith::CmpIOp::create(
+            b, loc, arith::CmpIPredicate::slt, argIter, zeroValue);
+        scf::IfOp ifOp = scf::IfOp::create(
+            b, loc, argIsNonNegative,
             [&](OpBuilder &b, Location loc) {
               // If arg is negative, continue (effectively, break)
               scf::YieldOp::create(b, loc, ValueRange{argIter, nIter});
diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
index 26559f394c221..853f45498ac52 100644
--- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
+++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
@@ -108,7 +108,7 @@ struct IntOpWithFlagLowering : public ConvertOpToLLVMPattern<MathOp> {
         op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
           return LLVMOp::create(rewriter, loc, llvm1DVectorTy, operands[0],
-                                         false);
+                                false);
         },
         rewriter);
   }
@@ -145,15 +145,16 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (LLVM::isCompatibleVectorType(llvmOperandType)) {
-        one = LLVM::ConstantOp::create(rewriter,
-            loc, llvmOperandType,
+        one = LLVM::ConstantOp::create(
+            rewriter, loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
+        one =
+            LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
       auto exp = LLVM::ExpOp::create(rewriter, loc, adaptor.getOperand(),
-                                              expAttrs.getAttrs());
+                                     expAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FSubOp>(
           op, llvmOperandType, ValueRange{exp, one}, subAttrs.getAttrs());
       return success();
@@ -170,12 +171,13 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
               mlir::VectorType::get({numElements.getKnownMinValue()}, floatType,
                                     {numElements.isScalable()}),
               floatOne);
-          auto one =
-              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
-          auto exp = LLVM::ExpOp::create(rewriter,
-              loc, llvm1DVectorTy, operands[0], expAttrs.getAttrs());
-          return LLVM::FSubOp::create(rewriter,
-              loc, llvm1DVectorTy, ValueRange{exp, one}, subAttrs.getAttrs());
+          auto one = LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy,
+                                              splatAttr);
+          auto exp = LLVM::ExpOp::create(rewriter, loc, llvm1DVectorTy,
+                                         operands[0], expAttrs.getAttrs());
+          return LLVM::FSubOp::create(rewriter, loc, llvm1DVectorTy,
+                                      ValueRange{exp, one},
+                                      subAttrs.getAttrs());
         },
         rewriter);
   }
@@ -205,16 +207,16 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one =
           isa<VectorType>(llvmOperandType)
-              ? LLVM::ConstantOp::create(rewriter,
-                    loc, llvmOperandType,
+              ? LLVM::ConstantOp::create(
+                    rewriter, loc, llvmOperandType,
                     SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                            floatOne))
               : LLVM::ConstantOp::create(rewriter, loc, llvmOperandType,
-                                                  floatOne);
+                                         floatOne);
 
-      auto add = LLVM::FAddOp::create(rewriter,
-          loc, llvmOperandType, ValueRange{one, adaptor.getOperand()},
-          addAttrs.getAttrs());
+      auto add = LLVM::FAddOp::create(rewriter, loc, llvmOperandType,
+                                      ValueRange{one, adaptor.getOperand()},
+                                      addAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::LogOp>(
           op, llvmOperandType, ValueRange{add}, logAttrs.getAttrs());
       return success();
@@ -231,13 +233,13 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
               mlir::VectorType::get({numElements.getKnownMinValue()}, floatType,
                                     {numElements.isScalable()}),
               floatOne);
-          auto one =
-              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto one = LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy,
+                                              splatAttr);
           auto add = LLVM::FAddOp::create(rewriter, loc, llvm1DVectorTy,
-                                                   ValueRange{one, operands[0]},
-                                                   addAttrs.getAttrs());
-          return LLVM::LogOp::create(rewriter,
-              loc, llvm1DVectorTy, ValueRange{add}, logAttrs.getAttrs());
+                                          ValueRange{one, operands[0]},
+                                          addAttrs.getAttrs());
+          return LLVM::LogOp::create(rewriter, loc, llvm1DVectorTy,
+                                     ValueRange{add}, logAttrs.getAttrs());
         },
         rewriter);
   }
@@ -267,15 +269,16 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (isa<VectorType>(llvmOperandType)) {
-        one = LLVM::ConstantOp::create(rewriter,
-            loc, llvmOperandType,
+        one = LLVM::ConstantOp::create(
+            rewriter, loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
+        one =
+            LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
       auto sqrt = LLVM::SqrtOp::create(rewriter, loc, adaptor.getOperand(),
-                                                sqrtAttrs.getAttrs());
+                                       sqrtAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FDivOp>(
           op, llvmOperandType, ValueRange{one, sqrt}, divAttrs.getAttrs());
       return success();
@@ -292,12 +295,13 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
               mlir::VectorType::get({numElements.getKnownMinValue()}, floatType,
                                     {numElements.isScalable()}),
               floatOne);
-          auto one =
-              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
-          auto sqrt = LLVM::SqrtOp::create(rewriter,
-              loc, llvm1DVectorTy, operands[0], sqrtAttrs.getAttrs());
-          return LLVM::FDivOp::create(rewriter,
-              loc, llvm1DVectorTy, ValueRange{one, sqrt}, divAttrs.getAttrs());
+          auto one = LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy,
+                                              splatAttr);
+          auto sqrt = LLVM::SqrtOp::create(rewriter, loc, llvm1DVectorTy,
+                                           operands[0], sqrtAttrs.getAttrs());
+          return LLVM::FDivOp::create(rewriter, loc, llvm1DVectorTy,
+                                      ValueRange{one, sqrt},
+                                      divAttrs.getAttrs());
         },
         rewriter);
   }
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index 19303dc3e8b55..f7c0d4fe3a799 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -84,9 +84,10 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto shape = vecType.getShape();
   int64_t numElements = vecType.getNumElements();
 
-  Value result = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(
-               vecType, FloatAttr::get(vecType.getElementType(), 0.0)));
+  Value result = arith::ConstantOp::create(
+      rewriter, loc,
+      DenseElementsAttr::get(vecType,
+                             FloatAttr::get(vecType.getElementType(), 0.0)));
   SmallVector<int64_t> strides = computeStrides(shape);
   for (auto linearIndex = 0; linearIndex < numElements; ++linearIndex) {
     SmallVector<int64_t> positions = delinearize(linearIndex, strides);
@@ -140,7 +141,7 @@ ScalarOpToLibmCall<Op>::matchAndRewrite(Op op,
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
     opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
-                                           opFunctionTy);
+                                  opFunctionTy);
     opFunc.setPrivate();
 
     // By definition Math dialect operations imply LLVM's "readnone"
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 26891aa7c2025..a877ad21734a2 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -37,11 +37,11 @@ static Value getScalarOrVectorI32Constant(Type type, int value,
       return nullptr;
     SmallVector<int> values(vectorType.getNumElements(), value);
     return spirv::ConstantOp::create(builder, loc, type,
-                                             builder.getI32VectorAttr(values));
+                                     builder.getI32VectorAttr(values));
   }
   if (type.isInteger(32))
     return spirv::ConstantOp::create(builder, loc, type,
-                                             builder.getI32IntegerAttr(value));
+                                     builder.getI32IntegerAttr(value));
 
   return nullptr;
 }
@@ -144,10 +144,11 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
     Type intType = rewriter.getIntegerType(bitwidth);
     uint64_t intValue = uint64_t(1) << (bitwidth - 1);
 
-    Value signMask = spirv::ConstantOp::create(rewriter,
-        loc, intType, rewriter.getIntegerAttr(intType, intValue));
-    Value valueMask = spirv::ConstantOp::create(rewriter,
-        loc, intType, rewriter.getIntegerAttr(intType, intValue - 1u));
+    Value signMask = spirv::ConstantOp::create(
+        rewriter, loc, intType, rewriter.getIntegerAttr(intType, intValue));
+    Value valueMask = spirv::ConstantOp::create(
+        rewriter, loc, intType,
+        rewriter.getIntegerAttr(intType, intValue - 1u));
 
     if (auto vectorType = dyn_cast<VectorType>(type)) {
       assert(vectorType.getRank() == 1);
@@ -155,12 +156,12 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
       intType = VectorType::get(count, intType);
 
       SmallVector<Value> signSplat(count, signMask);
-      signMask =
-          spirv::CompositeConstructOp::create(rewriter, loc, intType, signSplat);
+      signMask = spirv::CompositeConstructOp::create(rewriter, loc, intType,
+                                                     signSplat);
 
       SmallVector<Value> valueSplat(count, valueMask);
       valueMask = spirv::CompositeConstructOp::create(rewriter, loc, intType,
-                                                               valueSplat);
+                                                      valueSplat);
     }
 
     Value lhsCast =
@@ -168,13 +169,13 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
     Value rhsCast =
         spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getRhs());
 
-    Value value = spirv::BitwiseAndOp::create(rewriter,
-        loc, intType, ValueRange{lhsCast, valueMask});
-    Value sign = spirv::BitwiseAndOp::create(rewriter,
-        loc, intType, ValueRange{rhsCast, signMask});
+    Value value = spirv::BitwiseAndOp::create(rewriter, loc, intType,
+                                              ValueRange{lhsCast, valueMask});
+    Value sign = spirv::BitwiseAndOp::create(rewriter, loc, intType,
+                                             ValueRange{rhsCast, signMask});
 
     Value result = spirv::BitwiseOrOp::create(rewriter, loc, intType,
-                                                       ValueRange{value, sign});
+                                              ValueRange{value, sign});
     rewriter.replaceOpWithNewOp<spirv::BitcastOp>(copySignOp, type, result);
     return success();
   }
@@ -321,15 +322,15 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
 
     auto getConstantValue = [&](double value) {
       if (auto floatType = dyn_cast<FloatType>(type)) {
-        return spirv::ConstantOp::create(rewriter,
-            loc, type, rewriter.getFloatAttr(floatType, value));
+        return spirv::ConstantOp::create(
+            rewriter, loc, type, rewriter.getFloatAttr(floatType, value));
       }
       if (auto vectorType = dyn_cast<VectorType>(type)) {
         Type elemType = vectorType.getElementType();
 
         if (isa<FloatType>(elemType)) {
-          return spirv::ConstantOp::create(rewriter,
-              loc, type,
+          return spirv::ConstantOp::create(
+              rewriter, loc, type,
               DenseFPElementsAttr::get(
                   vectorType, FloatAttr::get(elemType, value).getValue()));
         }
@@ -408,8 +409,9 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
 
     Value NanValue =
         spirv::ConstantOp::create(rewriter, loc, operandType, nanAttr);
-    Value lhs = spirv::SelectOp::create(rewriter,
-        loc, cmpNegativeWithFractionalExp, NanValue, adaptor.getLhs());
+    Value lhs =
+        spirv::SelectOp::create(rewriter, loc, cmpNegativeWithFractionalExp,
+                                NanValue, adaptor.getLhs());
     Value abs = spirv::GLFAbsOp::create(rewriter, loc, lhs);
 
     // TODO: The following just forcefully casts y into an integer value in
@@ -455,13 +457,13 @@ struct RoundOpPattern final : public OpConversionPattern<math::RoundOp> {
     auto one = spirv::ConstantOp::getOne(ty, loc, rewriter);
     Value half;
     if (VectorType vty = dyn_cast<VectorType>(ty)) {
-      half = spirv::ConstantOp::create(rewriter,
-          loc, vty,
+      half = spirv::ConstantOp::create(
+          rewriter, loc, vty,
           DenseElementsAttr::get(vty,
                                  rewriter.getFloatAttr(ety, 0.5).getValue()));
     } else {
-      half = spirv::ConstantOp::create(rewriter,
-          loc, ty, rewriter.getFloatAttr(ety, 0.5));
+      half = spirv::ConstantOp::create(rewriter, loc, ty,
+                                       rewriter.getFloatAttr(ety, 0.5));
     }
 
     auto abs = spirv::GLFAbsOp::create(rewriter, loc, operand);
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
index db6ce600af76a..bd6136db4376a 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
@@ -161,8 +161,8 @@ struct ConvertLoad final : public OpConversionPattern<memref::LoadOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = emitc::SubscriptOp::create(rewriter,
-        op.getLoc(), arrayValue, operands.getIndices());
+    auto subscript = emitc::SubscriptOp::create(
+        rewriter, op.getLoc(), arrayValue, operands.getIndices());
 
     rewriter.replaceOpWithNewOp<emitc::LoadOp>(op, resultTy, subscript);
     return success();
@@ -181,8 +181,8 @@ struct ConvertStore final : public OpConversionPattern<memref::StoreOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = emitc::SubscriptOp::create(rewriter,
-        op.getLoc(), arrayValue, operands.getIndices());
+    auto subscript = emitc::SubscriptOp::create(
+        rewriter, op.getLoc(), arrayValue, operands.getIndices());
     rewriter.replaceOpWithNewOp<emitc::AssignOp>(op, subscript,
                                                  operands.getValue());
     return success();
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 4ef2168380620..53a19129103a3 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -88,7 +88,7 @@ getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
 static Value createAligned(ConversionPatternRewriter &rewriter, Location loc,
                            Value input, Value alignment) {
   Value one = LLVM::ConstantOp::create(rewriter, loc, alignment.getType(),
-                                                rewriter.getIndexAttr(1));
+                                       rewriter.getIndexAttr(1));
   Value bump = LLVM::SubOp::create(rewriter, loc, alignment, one);
   Value bumped = LLVM::AddOp::create(rewriter, loc, input, bump);
   Value mod = LLVM::URemOp::create(rewriter, loc, bumped, alignment);
@@ -123,8 +123,9 @@ static Value castAllocFuncResult(ConversionPatternRewriter &rewriter,
   assert(succeeded(maybeMemrefAddrSpace) && "unsupported address space");
   unsigned memrefAddrSpace = *maybeMemrefAddrSpace;
   if (allocatedPtrTy.getAddressSpace() != memrefAddrSpace)
-    allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
+    allocatedPtr = LLVM::AddrSpaceCastOp::create(
+        rewriter, loc,
+        LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
         allocatedPtr);
   return allocatedPtr;
 }
@@ -268,8 +269,9 @@ class AlignedAllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
       sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
 
     Type elementPtrType = this->getElementPtrType(memRefType);
-    auto results = LLVM::CallOp::create(rewriter,
-        loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
+    auto results =
+        LLVM::CallOp::create(rewriter, loc, allocFuncOp.value(),
+                             ValueRange({allocAlignment, sizeBytes}));
 
     Value ptr =
         castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
@@ -360,8 +362,9 @@ struct AllocaOpLowering : public ConvertOpToLLVMPattern<memref::AllocaOp> {
     auto elementPtrType =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addrSpace);
 
-    auto allocatedElementPtr = LLVM::AllocaOp::create(rewriter,
-        loc, elementPtrType, elementType, size, op.getAlignment().value_or(0));
+    auto allocatedElementPtr =
+        LLVM::AllocaOp::create(rewriter, loc, elementPtrType, elementType, size,
+                               op.getAlignment().value_or(0));
 
     // Create the MemRef descriptor.
     auto memRefDescriptor = this->createMemRefDescriptor(
@@ -455,7 +458,7 @@ struct AssumeAlignmentOpLowering
     Value alignmentConst =
         createIndexAttrConstant(rewriter, loc, getIndexType(), alignment);
     LLVM::AssumeOp::create(rewriter, loc, trueCond, LLVM::AssumeAlignTag(), ptr,
-                                    alignmentConst);
+                           alignmentConst);
     rewriter.replaceOp(op, memref);
     return success();
   }
@@ -559,18 +562,19 @@ struct DimOpLowering : public ConvertOpToLLVMPattern<memref::DimOp> {
     // Get pointer to offset field of memref<element_type> descriptor.
     auto indexPtrTy =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addressSpace);
-    Value offsetPtr = LLVM::GEPOp::create(rewriter,
-        loc, indexPtrTy, elementType, underlyingRankedDesc,
-        ArrayRef<LLVM::GEPArg>{0, 2});
+    Value offsetPtr =
+        LLVM::GEPOp::create(rewriter, loc, indexPtrTy, elementType,
+                            underlyingRankedDesc, ArrayRef<LLVM::GEPArg>{0, 2});
 
     // The size value that we have to extract can be obtained using GEPop with
     // `dimOp.index() + 1` index argument.
-    Value idxPlusOne = LLVM::AddOp::create(rewriter,
-        loc, createIndexAttrConstant(rewriter, loc, getIndexType(), 1),
+    Value idxPlusOne = LLVM::AddOp::create(
+        rewriter, loc,
+        createIndexAttrConstant(rewriter, loc, getIndexType(), 1),
         adaptor.getIndex());
-    Value sizePtr = LLVM::GEPOp::create(rewriter,
-        loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr,
-        idxPlusOne);
+    Value sizePtr = LLVM::GEPOp::create(rewriter, loc, indexPtrTy,
+                                        getTypeConverter()->getIndexType(),
+                                        offsetPtr, idxPlusOne);
     return rewriter
         .create<LLVM::LoadOp>(loc, getTypeConverter()->getIndexType(), sizePtr)
         .getResult();
@@ -674,8 +678,9 @@ struct GenericAtomicRMWOpLowering
     auto memRefType = cast<MemRefType>(atomicOp.getMemref().getType());
     auto dataPtr = getStridedElementPtr(
         rewriter, loc, memRefType, adaptor.getMemref(), adaptor.getIndices());
-    Value init = LLVM::LoadOp::create(rewriter,
-        loc, typeConverter->convertType(memRefType.getElementType()), dataPtr);
+    Value init = LLVM::LoadOp::create(
+        rewriter, loc, typeConverter->convertType(memRefType.getElementType()),
+        dataPtr);
     LLVM::BrOp::create(rewriter, loc, init, loopBlock);
 
     // Prepare the body of the loop block.
@@ -696,15 +701,16 @@ struct GenericAtomicRMWOpLowering
     // Append the cmpxchg op to the end of the loop block.
     auto successOrdering = LLVM::AtomicOrdering::acq_rel;
     auto failureOrdering = LLVM::AtomicOrdering::monotonic;
-    auto cmpxchg = LLVM::AtomicCmpXchgOp::create(rewriter,
-        loc, dataPtr, loopArgument, result, successOrdering, failureOrdering);
+    auto cmpxchg =
+        LLVM::AtomicCmpXchgOp::create(rewriter, loc, dataPtr, loopArgument,
+                                      result, successOrdering, failureOrdering);
     // Extract the %new_loaded and %ok values from the pair.
     Value newLoaded = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 0);
     Value ok = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 1);
 
     // Conditionally branch to the end or back to the loop depending on %ok.
     LLVM::CondBrOp::create(rewriter, loc, ok, endBlock, ArrayRef<Value>(),
-                                    loopBlock, newLoaded);
+                           loopBlock, newLoaded);
 
     rewriter.setInsertionPointToEnd(endBlock);
 
@@ -846,9 +852,9 @@ struct GetGlobalMemrefOpLowering
 
     // Get the address of the first element in the array by creating a GEP with
     // the address of the GV as the base, and (rank + 1) number of 0 indices.
-    auto gep = LLVM::GEPOp::create(rewriter,
-        loc, ptrTy, arrayTy, addressOf,
-        SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
+    auto gep =
+        LLVM::GEPOp::create(rewriter, loc, ptrTy, arrayTy, addressOf,
+                            SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
 
     // We do not expect the memref obtained using `memref.get_global` to be
     // ever deallocated. Set the allocated pointer to be known bad value to
@@ -1009,8 +1015,8 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
           loc, adaptor.getSource(), rewriter);
 
       // rank = ConstantOp srcRank
-      auto rankVal = LLVM::ConstantOp::create(rewriter,
-          loc, getIndexType(), rewriter.getIndexAttr(rank));
+      auto rankVal = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                              rewriter.getIndexAttr(rank));
       // poison = PoisonOp
       UnrankedMemRefDescriptor memRefDesc =
           UnrankedMemRefDescriptor::poison(rewriter, loc, targetStructType);
@@ -1063,8 +1069,8 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     MemRefDescriptor srcDesc(adaptor.getSource());
 
     // Compute number of elements.
-    Value numElements = LLVM::ConstantOp::create(rewriter,
-        loc, getIndexType(), rewriter.getIndexAttr(1));
+    Value numElements = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                                 rewriter.getIndexAttr(1));
     for (int pos = 0; pos < srcType.getRank(); ++pos) {
       auto size = srcDesc.size(rewriter, loc, pos);
       numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
@@ -1080,15 +1086,16 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
 
     Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc);
     Value srcOffset = srcDesc.offset(rewriter, loc);
-    Value srcPtr = LLVM::GEPOp::create(rewriter,
-        loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset);
+    Value srcPtr = LLVM::GEPOp::create(rewriter, loc, srcBasePtr.getType(),
+                                       elementType, srcBasePtr, srcOffset);
     MemRefDescriptor targetDesc(adaptor.getTarget());
     Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc);
     Value targetOffset = targetDesc.offset(rewriter, loc);
-    Value targetPtr = LLVM::GEPOp::create(rewriter,
-        loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset);
+    Value targetPtr =
+        LLVM::GEPOp::create(rewriter, loc, targetBasePtr.getType(), elementType,
+                            targetBasePtr, targetOffset);
     LLVM::MemcpyOp::create(rewriter, loc, targetPtr, srcPtr, totalSize,
-                                    /*isVolatile=*/false);
+                           /*isVolatile=*/false);
     rewriter.eraseOp(op);
 
     return success();
@@ -1104,7 +1111,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     // First make sure we have an unranked memref descriptor representation.
     auto makeUnranked = [&, this](Value ranked, MemRefType type) {
       auto rank = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                    type.getRank());
+                                           type.getRank());
       auto *typeConverter = getTypeConverter();
       auto ptr =
           typeConverter->promoteOneMemRefDescriptor(loc, ranked, rewriter);
@@ -1129,7 +1136,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
 
     // Now promote the unranked descriptors to the stack.
     auto one = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                 rewriter.getIndexAttr(1));
+                                        rewriter.getIndexAttr(1));
     auto promote = [&](Value desc) {
       auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
       auto allocated =
@@ -1150,7 +1157,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     if (failed(copyFn))
       return failure();
     LLVM::CallOp::create(rewriter, loc, copyFn.value(),
-                                  ValueRange{elemSize, sourcePtr, targetPtr});
+                         ValueRange{elemSize, sourcePtr, targetPtr});
 
     // Restore stack used for descriptors
     LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
@@ -1241,8 +1248,9 @@ struct MemorySpaceCastOpLowering
       UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                              result, resultAddrSpace, sizes);
       Value resultUnderlyingSize = sizes.front();
-      Value resultUnderlyingDesc = LLVM::AllocaOp::create(rewriter,
-          loc, getPtrType(), rewriter.getI8Type(), resultUnderlyingSize);
+      Value resultUnderlyingDesc =
+          LLVM::AllocaOp::create(rewriter, loc, getPtrType(),
+                                 rewriter.getI8Type(), resultUnderlyingSize);
       result.setMemRefDescPtr(rewriter, loc, resultUnderlyingDesc);
 
       // Copy pointers, performing address space casts.
@@ -1256,10 +1264,10 @@ struct MemorySpaceCastOpLowering
       Value alignedPtr =
           sourceDesc.alignedPtr(rewriter, loc, *getTypeConverter(),
                                 sourceUnderlyingDesc, sourceElemPtrType);
-      allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
-          loc, resultElemPtrType, allocatedPtr);
-      alignedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
-          loc, resultElemPtrType, alignedPtr);
+      allocatedPtr = LLVM::AddrSpaceCastOp::create(
+          rewriter, loc, resultElemPtrType, allocatedPtr);
+      alignedPtr = LLVM::AddrSpaceCastOp::create(rewriter, loc,
+                                                 resultElemPtrType, alignedPtr);
 
       result.setAllocatedPtr(rewriter, loc, resultUnderlyingDesc,
                              resultElemPtrType, allocatedPtr);
@@ -1277,12 +1285,13 @@ struct MemorySpaceCastOpLowering
       int64_t bytesToSkip =
           2 * llvm::divideCeil(
                   getTypeConverter()->getPointerBitwidth(resultAddrSpace), 8);
-      Value bytesToSkipConst = LLVM::ConstantOp::create(rewriter,
-          loc, getIndexType(), rewriter.getIndexAttr(bytesToSkip));
-      Value copySize = LLVM::SubOp::create(rewriter,
-          loc, getIndexType(), resultUnderlyingSize, bytesToSkipConst);
+      Value bytesToSkipConst = LLVM::ConstantOp::create(
+          rewriter, loc, getIndexType(), rewriter.getIndexAttr(bytesToSkip));
+      Value copySize =
+          LLVM::SubOp::create(rewriter, loc, getIndexType(),
+                              resultUnderlyingSize, bytesToSkipConst);
       LLVM::MemcpyOp::create(rewriter, loc, resultIndexVals, sourceIndexVals,
-                                      copySize, /*isVolatile=*/false);
+                             copySize, /*isVolatile=*/false);
 
       rewriter.replaceOp(op, ValueRange{result});
       return success();
@@ -1522,8 +1531,9 @@ struct MemRefReshapeOpLowering
     SmallVector<Value, 4> sizes;
     UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                            targetDesc, addressSpace, sizes);
-    Value underlyingDescPtr = LLVM::AllocaOp::create(rewriter,
-        loc, getPtrType(), IntegerType::get(getContext(), 8), sizes.front());
+    Value underlyingDescPtr = LLVM::AllocaOp::create(
+        rewriter, loc, getPtrType(), IntegerType::get(getContext(), 8),
+        sizes.front());
     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
 
     // Extract pointers and offset from the source memref.
@@ -1568,15 +1578,15 @@ struct MemRefReshapeOpLowering
     rewriter.mergeBlocks(remainingBlock, condBlock, ValueRange());
 
     rewriter.setInsertionPointToEnd(initBlock);
-    LLVM::BrOp::create(rewriter, loc, ValueRange({resultRankMinusOne, oneIndex}),
-                                condBlock);
+    LLVM::BrOp::create(rewriter, loc,
+                       ValueRange({resultRankMinusOne, oneIndex}), condBlock);
     rewriter.setInsertionPointToStart(condBlock);
     Value indexArg = condBlock->getArgument(0);
     Value strideArg = condBlock->getArgument(1);
 
     Value zeroIndex = createIndexAttrConstant(rewriter, loc, indexType, 0);
-    Value pred = LLVM::ICmpOp::create(rewriter,
-        loc, IntegerType::get(rewriter.getContext(), 1),
+    Value pred = LLVM::ICmpOp::create(
+        rewriter, loc, IntegerType::get(rewriter.getContext(), 1),
         LLVM::ICmpPredicate::sge, indexArg, zeroIndex);
 
     Block *bodyBlock =
@@ -1585,8 +1595,8 @@ struct MemRefReshapeOpLowering
 
     // Copy size from shape to descriptor.
     auto llvmIndexPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value sizeLoadGep = LLVM::GEPOp::create(rewriter,
-        loc, llvmIndexPtrType,
+    Value sizeLoadGep = LLVM::GEPOp::create(
+        rewriter, loc, llvmIndexPtrType,
         typeConverter->convertType(shapeMemRefType.getElementType()),
         shapeOperandPtr, indexArg);
     Value size = LLVM::LoadOp::create(rewriter, loc, indexType, sizeLoadGep);
@@ -1601,7 +1611,7 @@ struct MemRefReshapeOpLowering
     // Decrement loop counter and branch back.
     Value decrement = LLVM::SubOp::create(rewriter, loc, indexArg, oneIndex);
     LLVM::BrOp::create(rewriter, loc, ValueRange({decrement, nextStride}),
-                                condBlock);
+                       condBlock);
 
     Block *remainder =
         rewriter.splitBlock(bodyBlock, rewriter.getInsertionPoint());
@@ -1609,7 +1619,7 @@ struct MemRefReshapeOpLowering
     // Hook up the cond exit to the remainder.
     rewriter.setInsertionPointToEnd(condBlock);
     LLVM::CondBrOp::create(rewriter, loc, pred, bodyBlock, ValueRange(),
-                                    remainder, ValueRange());
+                           remainder, ValueRange());
 
     // Reset position to beginning of new remainder block.
     rewriter.setInsertionPointToStart(remainder);
@@ -1783,8 +1793,8 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
 
     // Field 2: Copy the actual aligned pointer to payload.
     Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc);
-    alignedPtr = LLVM::GEPOp::create(rewriter,
-        loc, alignedPtr.getType(),
+    alignedPtr = LLVM::GEPOp::create(
+        rewriter, loc, alignedPtr.getType(),
         typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr,
         adaptor.getByteShift());
 
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index 7b7ad9b95b0ec..7a705336bf11c 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -79,7 +79,8 @@ adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter,
   assert(indices.size() == 2);
   indices.back() = builder.createOrFold<spirv::SDivOp>(loc, lastDim, idx);
   Type t = typeConverter.convertType(op.getComponentPtr().getType());
-  return spirv::AccessChainOp::create(builder, loc, t, op.getBasePtr(), indices);
+  return spirv::AccessChainOp::create(builder, loc, t, op.getBasePtr(),
+                                      indices);
 }
 
 /// Casts the given `srcBool` into an integer of `dstType`.
@@ -107,8 +108,8 @@ static Value shiftValue(Location loc, Value value, Value offset, Value mask,
     value = castBoolToIntN(loc, value, dstType, builder);
   } else {
     if (valueBits < targetBits) {
-      value = spirv::UConvertOp::create(builder,
-          loc, builder.getIntegerType(targetBits), value);
+      value = spirv::UConvertOp::create(
+          builder, loc, builder.getIntegerType(targetBits), value);
     }
 
     value = builder.createOrFold<spirv::BitwiseAndOp>(loc, value, mask);
@@ -373,7 +374,7 @@ AllocOpPattern::matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor,
         std::string("__workgroup_mem__") +
         std::to_string(std::distance(varOps.begin(), varOps.end()));
     varOp = spirv::GlobalVariableOp::create(rewriter, loc, spirvType, varName,
-                                                     /*initializer=*/nullptr);
+                                            /*initializer=*/nullptr);
   }
 
   // Get pointer to global variable at the current scope.
@@ -573,7 +574,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
 
     auto [memoryAccess, alignment] = *memoryRequirements;
     Value loadVal = spirv::LoadOp::create(rewriter, loc, accessChain,
-                                                   memoryAccess, alignment);
+                                          memoryAccess, alignment);
     if (isBool)
       loadVal = castIntNToBool(loc, loadVal, rewriter);
     rewriter.replaceOp(loadOp, loadVal);
@@ -602,7 +603,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
 
   auto [memoryAccess, alignment] = *memoryRequirements;
   Value spvLoadOp = spirv::LoadOp::create(rewriter, loc, dstType, adjustedPtr,
-                                                   memoryAccess, alignment);
+                                          memoryAccess, alignment);
 
   // Shift the bits to the rightmost.
   // ____XXXX________ -> ____________XXXX
@@ -770,12 +771,12 @@ IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor,
   if (!scope)
     return rewriter.notifyMatchFailure(storeOp, "atomic scope not available");
 
-  Value result = spirv::AtomicAndOp::create(rewriter,
-      loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
-      clearBitsMask);
-  result = spirv::AtomicOrOp::create(rewriter,
-      loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
-      storeVal);
+  Value result = spirv::AtomicAndOp::create(
+      rewriter, loc, dstType, adjustedPtr, *scope,
+      spirv::MemorySemantics::AcquireRelease, clearBitsMask);
+  result = spirv::AtomicOrOp::create(
+      rewriter, loc, dstType, adjustedPtr, *scope,
+      spirv::MemorySemantics::AcquireRelease, storeVal);
 
   // The AtomicOrOp has no side effect. Since it is already inserted, we can
   // just remove the original StoreOp. Note that rewriter.replaceOp()
@@ -850,8 +851,8 @@ LogicalResult MemorySpaceCastOpPattern::matchAndRewrite(
     genericPtrType = typeConverter.convertType(intermediateType);
   }
   if (sourceSc != spirv::StorageClass::Generic) {
-    result =
-        spirv::PtrCastToGenericOp::create(rewriter, loc, genericPtrType, result);
+    result = spirv::PtrCastToGenericOp::create(rewriter, loc, genericPtrType,
+                                               result);
   }
   if (resultSc != spirv::StorageClass::Generic) {
     result =
diff --git a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
index f16b92a287d0a..63b1fdabaf407 100644
--- a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
+++ b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
@@ -147,8 +147,9 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     Value resSplitAxes = tensor::EmptyOp::create(rewriter, loc, shape, i16);
     auto attr = IntegerAttr::get(i16, -1);
     Value fillValue = arith::ConstantOp::create(rewriter, loc, i16, attr);
-    resSplitAxes = linalg::FillOp::create(rewriter, loc, fillValue, resSplitAxes)
-                       .getResult(0);
+    resSplitAxes =
+        linalg::FillOp::create(rewriter, loc, fillValue, resSplitAxes)
+            .getResult(0);
 
     // explicitly write values into tensor row by row
     std::array<int64_t, 2> strides = {1, 1};
@@ -163,8 +164,9 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       auto tensorType = RankedTensorType::get({size}, i16);
       auto attrs = DenseIntElementsAttr::get(tensorType, axes.asArrayRef());
       auto vals = arith::ConstantOp::create(rewriter, loc, tensorType, attrs);
-      resSplitAxes = tensor::InsertSliceOp::create(rewriter,
-          loc, vals, resSplitAxes, empty, empty, empty, offs, sizes, strides);
+      resSplitAxes = tensor::InsertSliceOp::create(rewriter, loc, vals,
+                                                   resSplitAxes, empty, empty,
+                                                   empty, offs, sizes, strides);
     }
 
     // To hold halos sizes, create 2d Tensor with shape {nSplits, 2}.
@@ -189,8 +191,8 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     // MeshOp)
     Value resOffsets;
     if (adaptor.getStaticShardedDimsOffsets().empty()) {
-      resOffsets = tensor::EmptyOp::create(rewriter,
-          loc, std::array<int64_t, 2>{0, 0}, i64);
+      resOffsets = tensor::EmptyOp::create(rewriter, loc,
+                                           std::array<int64_t, 2>{0, 0}, i64);
     } else {
       SymbolTableCollection symbolTableCollection;
       auto meshOp = getMesh(op, symbolTableCollection);
@@ -204,10 +206,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       assert(maxSplitSize);
       ++maxSplitSize; // add one for the total size
 
-      resOffsets = tensor::EmptyOp::create(rewriter,
-          loc, std::array<int64_t, 2>{nSplits, maxSplitSize}, i64);
-      Value zero = arith::ConstantOp::create(rewriter,
-          loc, i64, rewriter.getI64IntegerAttr(ShapedType::kDynamic));
+      resOffsets = tensor::EmptyOp::create(
+          rewriter, loc, std::array<int64_t, 2>{nSplits, maxSplitSize}, i64);
+      Value zero = arith::ConstantOp::create(
+          rewriter, loc, i64, rewriter.getI64IntegerAttr(ShapedType::kDynamic));
       resOffsets =
           linalg::FillOp::create(rewriter, loc, zero, resOffsets).getResult(0);
       SmallVector<Value> offsets =
@@ -223,8 +225,9 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
         Value vals = tensor::FromElementsOp::create(rewriter, loc, values);
         std::array<int64_t, 2> offs = {static_cast<int64_t>(i), 0};
         std::array<int64_t, 2> sizes = {1, splitSize};
-        resOffsets = tensor::InsertSliceOp::create(rewriter,
-            loc, vals, resOffsets, empty, empty, empty, offs, sizes, strides);
+        resOffsets = tensor::InsertSliceOp::create(rewriter, loc, vals,
+                                                   resOffsets, empty, empty,
+                                                   empty, offs, sizes, strides);
         curr += splitSize;
       }
     }
@@ -346,11 +349,11 @@ struct ConvertNeighborsLinearIndicesOp
     Value dimSz = dims[axes[0]];
     Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
     Value minus1 = arith::ConstantIndexOp::create(rewriter, loc, -1);
-    Value atBorder = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sle, orgIdx,
-        arith::ConstantIndexOp::create(rewriter, loc, 0));
-    auto down = scf::IfOp::create(rewriter,
-        loc, atBorder,
+    Value atBorder =
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sle, orgIdx,
+                              arith::ConstantIndexOp::create(rewriter, loc, 0));
+    auto down = scf::IfOp::create(
+        rewriter, loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
           scf::YieldOp::create(builder, loc, minus1);
         },
@@ -359,14 +362,14 @@ struct ConvertNeighborsLinearIndicesOp
           tmp[axes[0]] =
               arith::SubIOp::create(rewriter, op.getLoc(), orgIdx, one)
                   .getResult();
-          scf::YieldOp::create(builder,
-              loc, multiToLinearIndex(loc, rewriter, tmp, dims));
+          scf::YieldOp::create(builder, loc,
+                               multiToLinearIndex(loc, rewriter, tmp, dims));
         });
-    atBorder = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sge, orgIdx,
+    atBorder = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sge, orgIdx,
         arith::SubIOp::create(rewriter, loc, dimSz, one).getResult());
-    auto up = scf::IfOp::create(rewriter,
-        loc, atBorder,
+    auto up = scf::IfOp::create(
+        rewriter, loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
           scf::YieldOp::create(builder, loc, minus1);
         },
@@ -374,8 +377,8 @@ struct ConvertNeighborsLinearIndicesOp
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
               arith::AddIOp::create(rewriter, op.getLoc(), orgIdx, one);
-          scf::YieldOp::create(builder,
-              loc, multiToLinearIndex(loc, rewriter, tmp, dims));
+          scf::YieldOp::create(builder, loc,
+                               multiToLinearIndex(loc, rewriter, tmp, dims));
         });
     rewriter.replaceOp(op, ValueRange{down.getResult(0), up.getResult(0)});
     return success();
@@ -447,8 +450,9 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           rewriter, loc, sharding.getStaticShardedDimsOffsets(),
           sharding.getDynamicShardedDimsOffsets(), index);
       if (!tmp.empty())
-        shardedDimsOffs = tensor::FromElementsOp::create(rewriter,
-            loc, RankedTensorType::get({(int64_t)tmp.size()}, index), tmp);
+        shardedDimsOffs = tensor::FromElementsOp::create(
+            rewriter, loc, RankedTensorType::get({(int64_t)tmp.size()}, index),
+            tmp);
     }
 
     // With static mesh shape the sizes of the split axes are known.
@@ -469,8 +473,8 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
         auto axes = splitAxes[i];
         // The current dimension might not be sharded.
         // Create a value from the static position in shardDimsOffsets.
-        Value posVal =
-            arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(pos));
+        Value posVal = arith::ConstantOp::create(rewriter, loc,
+                                                 rewriter.getIndexAttr(pos));
         // Get the index of the local shard in the mesh axis.
         Value idx = multiIdx[axes[0]];
         auto numShards =
@@ -492,8 +496,8 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           Value sz = arith::SubIOp::create(rewriter, loc, nextOff, off);
           shardShape.emplace_back(sz);
         } else {
-          Value numShardsVal = arith::ConstantOp::create(rewriter,
-              loc, rewriter.getIndexAttr(numShards));
+          Value numShardsVal = arith::ConstantOp::create(
+              rewriter, loc, rewriter.getIndexAttr(numShards));
           // Compute shard dim size by distributing odd elements to trailing
           // shards:
           // sz = dim / numShards
@@ -501,8 +505,8 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           Value sz = arith::DivSIOp::create(rewriter, loc, dim, numShardsVal);
           Value sz1 = arith::RemSIOp::create(rewriter, loc, dim, numShardsVal);
           sz1 = arith::SubIOp::create(rewriter, loc, numShardsVal, sz1);
-          auto cond = arith::CmpIOp::create(rewriter,
-              loc, arith::CmpIPredicate::sge, idx, sz1);
+          auto cond = arith::CmpIOp::create(
+              rewriter, loc, arith::CmpIPredicate::sge, idx, sz1);
           Value odd = arith::SelectOp::create(rewriter, loc, cond, one, zero);
           sz = arith::AddIOp::create(rewriter, loc, sz, odd);
           shardShape.emplace_back(sz);
@@ -583,8 +587,8 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     }
 
     // Allocate buffer and copy input to buffer.
-    Value buffer = memref::AllocOp::create(iBuilder,
-        shape, cast<ShapedType>(op.getType()).getElementType());
+    Value buffer = memref::AllocOp::create(
+        iBuilder, shape, cast<ShapedType>(op.getType()).getElementType());
     linalg::CopyOp::create(iBuilder, input, buffer);
 
     // Get an MPI_Comm_split for the AllReduce operation.
@@ -623,19 +627,19 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (inType.getRank() > 1) {
       ReassociationIndices reassociation(inType.getRank());
       std::iota(reassociation.begin(), reassociation.end(), 0);
-      buffer1d = memref::CollapseShapeOp::create(iBuilder,
-          buffer, ArrayRef<ReassociationIndices>(reassociation));
+      buffer1d = memref::CollapseShapeOp::create(
+          iBuilder, buffer, ArrayRef<ReassociationIndices>(reassociation));
     }
 
     // Create the MPI AllReduce operation.
-    mpi::AllReduceOp::create(iBuilder,
-        TypeRange(), buffer1d, buffer1d,
-        getMPIReductionOp(adaptor.getReductionAttr()), comm);
+    mpi::AllReduceOp::create(iBuilder, TypeRange(), buffer1d, buffer1d,
+                             getMPIReductionOp(adaptor.getReductionAttr()),
+                             comm);
 
     // If the destination is a memref, cast it to a tensor
     if (isa<RankedTensorType>(op.getType()))
       buffer = bufferization::ToTensorOp::create(iBuilder, op.getType(), buffer,
-                                                          true);
+                                                 true);
 
     rewriter.replaceOp(op, buffer);
     return success();
@@ -676,9 +680,10 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     auto toValue = [&rewriter, &loc](OpFoldResult &v) -> Value {
       if (auto value = dyn_cast<Value>(v))
         return value;
-      return arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexAttr(
-                   cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
+      return arith::ConstantOp::create(
+          rewriter, loc,
+          rewriter.getIndexAttr(
+              cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
     };
 
     auto dest = adaptor.getDestination();
@@ -723,8 +728,8 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         offsets[i] = haloSizes[currHaloDim * 2];
 
         // prepare shape and offsets of highest dim's halo exchange
-        Value _haloSz = arith::AddIOp::create(rewriter,
-            loc, toValue(haloSizes[currHaloDim * 2]),
+        Value _haloSz = arith::AddIOp::create(
+            rewriter, loc, toValue(haloSizes[currHaloDim * 2]),
             toValue(haloSizes[currHaloDim * 2 + 1]));
         // the halo shape of lower dims exlude the halos
         dimSizes[i] =
@@ -758,20 +763,22 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
                                                        splitAxes)
                      .getResults();
       // MPI operates on i32...
-      Value neighbourIDs[2] = {arith::IndexCastOp::create(rewriter,
-                                   loc, rewriter.getI32Type(), tmp[0]),
-                               arith::IndexCastOp::create(rewriter,
-                                   loc, rewriter.getI32Type(), tmp[1])};
+      Value neighbourIDs[2] = {
+          arith::IndexCastOp::create(rewriter, loc, rewriter.getI32Type(),
+                                     tmp[0]),
+          arith::IndexCastOp::create(rewriter, loc, rewriter.getI32Type(),
+                                     tmp[1])};
 
       auto lowerRecvOffset = rewriter.getIndexAttr(0);
       auto lowerSendOffset = toValue(haloSizes[currHaloDim * 2]);
-      auto upperRecvOffset = arith::SubIOp::create(rewriter,
-          loc, toValue(shape[dim]), toValue(haloSizes[currHaloDim * 2 + 1]));
-      auto upperSendOffset = arith::SubIOp::create(rewriter,
-          loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
+      auto upperRecvOffset =
+          arith::SubIOp::create(rewriter, loc, toValue(shape[dim]),
+                                toValue(haloSizes[currHaloDim * 2 + 1]));
+      auto upperSendOffset = arith::SubIOp::create(
+          rewriter, loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
 
-      Value commWorld = mpi::CommWorldOp::create(rewriter,
-          loc, mpi::CommType::get(op->getContext()));
+      Value commWorld = mpi::CommWorldOp::create(
+          rewriter, loc, mpi::CommType::get(op->getContext()));
 
       // Make sure we send/recv in a way that does not lead to a dead-lock.
       // The current approach is by far not optimal, this should be at least
@@ -787,33 +794,34 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         // Processes on the mesh borders have only one neighbor
         auto to = upperHalo ? neighbourIDs[0] : neighbourIDs[1];
         auto from = upperHalo ? neighbourIDs[1] : neighbourIDs[0];
-        auto hasFrom = arith::CmpIOp::create(rewriter,
-            loc, arith::CmpIPredicate::sge, from, zero);
-        auto hasTo = arith::CmpIOp::create(rewriter,
-            loc, arith::CmpIPredicate::sge, to, zero);
-        auto buffer = memref::AllocOp::create(rewriter,
-            loc, dimSizes, cast<ShapedType>(array.getType()).getElementType());
+        auto hasFrom = arith::CmpIOp::create(
+            rewriter, loc, arith::CmpIPredicate::sge, from, zero);
+        auto hasTo = arith::CmpIOp::create(rewriter, loc,
+                                           arith::CmpIPredicate::sge, to, zero);
+        auto buffer = memref::AllocOp::create(
+            rewriter, loc, dimSizes,
+            cast<ShapedType>(array.getType()).getElementType());
         // if has neighbor: copy halo data from array to buffer and send
-        scf::IfOp::create(rewriter,
-            loc, hasTo, [&](OpBuilder &builder, Location loc) {
+        scf::IfOp::create(
+            rewriter, loc, hasTo, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(lowerSendOffset)
                                        : OpFoldResult(upperSendOffset);
-              auto subview = memref::SubViewOp::create(builder,
-                  loc, array, offsets, dimSizes, strides);
+              auto subview = memref::SubViewOp::create(
+                  builder, loc, array, offsets, dimSizes, strides);
               memref::CopyOp::create(builder, loc, subview, buffer);
               mpi::SendOp::create(builder, loc, TypeRange{}, buffer, tag, to,
-                                          commWorld);
+                                  commWorld);
               scf::YieldOp::create(builder, loc);
             });
         // if has neighbor: receive halo data into buffer and copy to array
-        scf::IfOp::create(rewriter,
-            loc, hasFrom, [&](OpBuilder &builder, Location loc) {
+        scf::IfOp::create(
+            rewriter, loc, hasFrom, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(upperRecvOffset)
                                        : OpFoldResult(lowerRecvOffset);
               mpi::RecvOp::create(builder, loc, TypeRange{}, buffer, tag, from,
-                                          commWorld);
-              auto subview = memref::SubViewOp::create(builder,
-                  loc, array, offsets, dimSizes, strides);
+                                  commWorld);
+              auto subview = memref::SubViewOp::create(
+                  builder, loc, array, offsets, dimSizes, strides);
               memref::CopyOp::create(builder, loc, buffer, subview);
               scf::YieldOp::create(builder, loc);
             });
@@ -825,16 +833,17 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         OpFoldResult &v = haloSizes[currHaloDim * 2 + upOrDown];
         Value haloSz = dyn_cast<Value>(v);
         if (!haloSz)
-          haloSz = arith::ConstantOp::create(rewriter,
-              loc, rewriter.getI32IntegerAttr(
-                       cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
-        auto hasSize = arith::CmpIOp::create(rewriter,
-            loc, arith::CmpIPredicate::sgt, haloSz, zero);
+          haloSz = arith::ConstantOp::create(
+              rewriter, loc,
+              rewriter.getI32IntegerAttr(
+                  cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
+        auto hasSize = arith::CmpIOp::create(
+            rewriter, loc, arith::CmpIPredicate::sgt, haloSz, zero);
         scf::IfOp::create(rewriter, loc, hasSize,
-                                   [&](OpBuilder &builder, Location loc) {
-                                     genSendRecv(upOrDown > 0);
-                                     scf::YieldOp::create(builder, loc);
-                                   });
+                          [&](OpBuilder &builder, Location loc) {
+                            genSendRecv(upOrDown > 0);
+                            scf::YieldOp::create(builder, loc);
+                          });
       };
 
       doSendRecv(0);
@@ -852,8 +861,8 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       rewriter.replaceOp(op, array);
     } else {
       assert(isa<RankedTensorType>(op.getResult().getType()));
-      rewriter.replaceOp(op, bufferization::ToTensorOp::create(rewriter,
-                                 loc, op.getResult().getType(), array,
+      rewriter.replaceOp(op, bufferization::ToTensorOp::create(
+                                 rewriter, loc, op.getResult().getType(), array,
                                  /*restrict=*/true, /*writable=*/true));
     }
     return success();
diff --git a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
index a6125fc07dcd5..f5b3689c88d26 100644
--- a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
+++ b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
@@ -40,7 +40,7 @@ class ExpandIfCondition : public OpRewritePattern<OpTy> {
     IntegerAttr constAttr;
     if (!matchPattern(op.getIfCond(), m_Constant(&constAttr))) {
       auto ifOp = scf::IfOp::create(rewriter, op.getLoc(), TypeRange(),
-                                             op.getIfCond(), false);
+                                    op.getIfCond(), false);
       rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });
       auto thenBodyBuilder = ifOp.getThenBodyBuilder(rewriter.getListener());
       thenBodyBuilder.clone(*op.getOperation());
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 4bf59124a9bdb..021e31a8ecd97 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -96,7 +96,7 @@ struct OpenMPOpConversion : public ConvertOpToLLVMPattern<T> {
 
     // Create new operation.
     auto newOp = T::create(rewriter, op.getLoc(), resTypes, convertedOperands,
-                                    convertedAttrs);
+                           convertedAttrs);
 
     // Translate regions.
     for (auto [originalRegion, convertedRegion] :
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 75cd35c0d7876..b711e33cfc0d6 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -272,8 +272,8 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     auto *operationPos = cast<OperationPosition>(pos);
     if (operationPos->isOperandDefiningOp())
       // Standard (downward) traversal which directly follows the defining op.
-      value = pdl_interp::GetDefiningOpOp::create(builder,
-          loc, builder.getType<pdl::OperationType>(), parentVal);
+      value = pdl_interp::GetDefiningOpOp::create(
+          builder, loc, builder.getType<pdl::OperationType>(), parentVal);
     else
       // A passthrough operation position.
       value = parentVal;
@@ -297,8 +297,8 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::ForEachPos: {
     assert(!failureBlockStack.empty() && "expected valid failure block");
-    auto foreach = pdl_interp::ForEachOp::create(builder,
-        loc, parentVal, failureBlockStack.back(), /*initLoop=*/true);
+    auto foreach = pdl_interp::ForEachOp::create(
+        builder, loc, parentVal, failureBlockStack.back(), /*initLoop=*/true);
     value = foreach.getLoopVariable();
 
     // Create the continuation block.
@@ -311,23 +311,24 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::OperandPos: {
     auto *operandPos = cast<OperandPosition>(pos);
-    value = pdl_interp::GetOperandOp::create(builder,
-        loc, builder.getType<pdl::ValueType>(), parentVal,
+    value = pdl_interp::GetOperandOp::create(
+        builder, loc, builder.getType<pdl::ValueType>(), parentVal,
         operandPos->getOperandNumber());
     break;
   }
   case Predicates::OperandGroupPos: {
     auto *operandPos = cast<OperandGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = pdl_interp::GetOperandsOp::create(builder,
-        loc, operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
+    value = pdl_interp::GetOperandsOp::create(
+        builder, loc,
+        operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, operandPos->getOperandGroupNumber());
     break;
   }
   case Predicates::AttributePos: {
     auto *attrPos = cast<AttributePosition>(pos);
-    value = pdl_interp::GetAttributeOp::create(builder,
-        loc, builder.getType<pdl::AttributeType>(), parentVal,
+    value = pdl_interp::GetAttributeOp::create(
+        builder, loc, builder.getType<pdl::AttributeType>(), parentVal,
         attrPos->getName().strref());
     break;
   }
@@ -340,23 +341,24 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::ResultPos: {
     auto *resPos = cast<ResultPosition>(pos);
-    value = pdl_interp::GetResultOp::create(builder,
-        loc, builder.getType<pdl::ValueType>(), parentVal,
+    value = pdl_interp::GetResultOp::create(
+        builder, loc, builder.getType<pdl::ValueType>(), parentVal,
         resPos->getResultNumber());
     break;
   }
   case Predicates::ResultGroupPos: {
     auto *resPos = cast<ResultGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = pdl_interp::GetResultsOp::create(builder,
-        loc, resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
+    value = pdl_interp::GetResultsOp::create(
+        builder, loc,
+        resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, resPos->getResultGroupNumber());
     break;
   }
   case Predicates::AttributeLiteralPos: {
     auto *attrPos = cast<AttributeLiteralPosition>(pos);
-    value =
-        pdl_interp::CreateAttributeOp::create(builder, loc, attrPos->getValue());
+    value = pdl_interp::CreateAttributeOp::create(builder, loc,
+                                                  attrPos->getValue());
     break;
   }
   case Predicates::TypeLiteralPos: {
@@ -365,8 +367,8 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     if (TypeAttr typeAttr = dyn_cast<TypeAttr>(rawTypeAttr))
       value = pdl_interp::CreateTypeOp::create(builder, loc, typeAttr);
     else
-      value = pdl_interp::CreateTypesOp::create(builder,
-          loc, cast<ArrayAttr>(rawTypeAttr));
+      value = pdl_interp::CreateTypesOp::create(builder, loc,
+                                                cast<ArrayAttr>(rawTypeAttr));
     break;
   }
   case Predicates::ConstraintResultPos: {
@@ -417,52 +419,55 @@ void PatternLowering::generate(BoolNode *boolNode, Block *&currentBlock,
     break;
   case Predicates::OperationNameQuestion: {
     auto *opNameAnswer = cast<OperationNameAnswer>(answer);
-    pdl_interp::CheckOperationNameOp::create(builder,
-        loc, val, opNameAnswer->getValue().getStringRef(), success, failure);
+    pdl_interp::CheckOperationNameOp::create(
+        builder, loc, val, opNameAnswer->getValue().getStringRef(), success,
+        failure);
     break;
   }
   case Predicates::TypeQuestion: {
     auto *ans = cast<TypeAnswer>(answer);
     if (isa<pdl::RangeType>(val.getType()))
-      pdl_interp::CheckTypesOp::create(builder,
-          loc, val, llvm::cast<ArrayAttr>(ans->getValue()), success, failure);
+      pdl_interp::CheckTypesOp::create(builder, loc, val,
+                                       llvm::cast<ArrayAttr>(ans->getValue()),
+                                       success, failure);
     else
-      pdl_interp::CheckTypeOp::create(builder,
-          loc, val, llvm::cast<TypeAttr>(ans->getValue()), success, failure);
+      pdl_interp::CheckTypeOp::create(builder, loc, val,
+                                      llvm::cast<TypeAttr>(ans->getValue()),
+                                      success, failure);
     break;
   }
   case Predicates::AttributeQuestion: {
     auto *ans = cast<AttributeAnswer>(answer);
     pdl_interp::CheckAttributeOp::create(builder, loc, val, ans->getValue(),
-                                                 success, failure);
+                                         success, failure);
     break;
   }
   case Predicates::OperandCountAtLeastQuestion:
   case Predicates::OperandCountQuestion:
-    pdl_interp::CheckOperandCountOp::create(builder,
-        loc, val, cast<UnsignedAnswer>(answer)->getValue(),
+    pdl_interp::CheckOperandCountOp::create(
+        builder, loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::OperandCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::ResultCountAtLeastQuestion:
   case Predicates::ResultCountQuestion:
-    pdl_interp::CheckResultCountOp::create(builder,
-        loc, val, cast<UnsignedAnswer>(answer)->getValue(),
+    pdl_interp::CheckResultCountOp::create(
+        builder, loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::ResultCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::EqualToQuestion: {
     bool trueAnswer = isa<TrueAnswer>(answer);
     pdl_interp::AreEqualOp::create(builder, loc, val, args.front(),
-                                           trueAnswer ? success : failure,
-                                           trueAnswer ? failure : success);
+                                   trueAnswer ? success : failure,
+                                   trueAnswer ? failure : success);
     break;
   }
   case Predicates::ConstraintQuestion: {
     auto *cstQuestion = cast<ConstraintQuestion>(question);
-    auto applyConstraintOp = pdl_interp::ApplyConstraintOp::create(builder,
-        loc, cstQuestion->getResultTypes(), cstQuestion->getName(), args,
-        cstQuestion->getIsNegated(), success, failure);
+    auto applyConstraintOp = pdl_interp::ApplyConstraintOp::create(
+        builder, loc, cstQuestion->getResultTypes(), cstQuestion->getName(),
+        args, cstQuestion->getIsNegated(), success, failure);
 
     constraintOpMap.insert({cstQuestion, applyConstraintOp});
     break;
@@ -536,12 +541,14 @@ void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
       unsigned ans = cast<UnsignedAnswer>(child.first)->getValue();
       switch (kind) {
       case Predicates::OperandCountAtLeastQuestion:
-        pdl_interp::CheckOperandCountOp::create(builder,
-            loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
+        pdl_interp::CheckOperandCountOp::create(builder, loc, val, ans,
+                                                /*compareAtLeast=*/true,
+                                                childBlock, defaultDest);
         break;
       case Predicates::ResultCountAtLeastQuestion:
-        pdl_interp::CheckResultCountOp::create(builder,
-            loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
+        pdl_interp::CheckResultCountOp::create(builder, loc, val, ans,
+                                               /*compareAtLeast=*/true,
+                                               childBlock, defaultDest);
         break;
       default:
         llvm_unreachable("Generating invalid AtLeast operation");
@@ -619,8 +626,8 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
       rootKindAttr = builder.getStringAttr(*rootKind);
 
   builder.setInsertionPointToEnd(currentBlock);
-  auto matchOp = pdl_interp::RecordMatchOp::create(builder,
-      pattern.getLoc(), mappedMatchValues, locOps.getArrayRef(),
+  auto matchOp = pdl_interp::RecordMatchOp::create(
+      builder, pattern.getLoc(), mappedMatchValues, locOps.getArrayRef(),
       rewriterFuncRef, rootKindAttr, generatedOpsAttr, pattern.getBenefitAttr(),
       failureBlockStack.back());
 
@@ -632,8 +639,8 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
 SymbolRefAttr PatternLowering::generateRewriter(
     pdl::PatternOp pattern, SmallVectorImpl<Position *> &usedMatchValues) {
   builder.setInsertionPointToEnd(rewriterModule.getBody());
-  auto rewriterFunc = pdl_interp::FuncOp::create(builder,
-      pattern.getLoc(), "pdl_generated_rewriter",
+  auto rewriterFunc = pdl_interp::FuncOp::create(
+      builder, pattern.getLoc(), "pdl_generated_rewriter",
       builder.getFunctionType({}, {}));
   rewriterSymbolTable.insert(rewriterFunc);
 
@@ -651,18 +658,18 @@ SymbolRefAttr PatternLowering::generateRewriter(
     Operation *oldOp = oldValue.getDefiningOp();
     if (pdl::AttributeOp attrOp = dyn_cast<pdl::AttributeOp>(oldOp)) {
       if (Attribute value = attrOp.getValueAttr()) {
-        return newValue = pdl_interp::CreateAttributeOp::create(builder,
-                   attrOp.getLoc(), value);
+        return newValue = pdl_interp::CreateAttributeOp::create(
+                   builder, attrOp.getLoc(), value);
       }
     } else if (pdl::TypeOp typeOp = dyn_cast<pdl::TypeOp>(oldOp)) {
       if (TypeAttr type = typeOp.getConstantTypeAttr()) {
-        return newValue = pdl_interp::CreateTypeOp::create(builder,
-                   typeOp.getLoc(), type);
+        return newValue = pdl_interp::CreateTypeOp::create(
+                   builder, typeOp.getLoc(), type);
       }
     } else if (pdl::TypesOp typeOp = dyn_cast<pdl::TypesOp>(oldOp)) {
       if (ArrayAttr type = typeOp.getConstantTypesAttr()) {
-        return newValue = pdl_interp::CreateTypesOp::create(builder,
-                   typeOp.getLoc(), typeOp.getType(), type);
+        return newValue = pdl_interp::CreateTypesOp::create(
+                   builder, typeOp.getLoc(), typeOp.getType(), type);
       }
     }
 
@@ -684,8 +691,9 @@ SymbolRefAttr PatternLowering::generateRewriter(
     auto mappedArgs =
         llvm::map_range(rewriter.getExternalArgs(), mapRewriteValue);
     args.append(mappedArgs.begin(), mappedArgs.end());
-    pdl_interp::ApplyRewriteOp::create(builder,
-        rewriter.getLoc(), /*resultTypes=*/TypeRange(), rewriteName, args);
+    pdl_interp::ApplyRewriteOp::create(builder, rewriter.getLoc(),
+                                       /*resultTypes=*/TypeRange(), rewriteName,
+                                       args);
   } else {
     // Otherwise this is a dag rewriter defined using PDL operations.
     for (Operation &rewriteOp : *rewriter.getBody()) {
@@ -716,9 +724,9 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 2> arguments;
   for (Value argument : rewriteOp.getArgs())
     arguments.push_back(mapRewriteValue(argument));
-  auto interpOp = pdl_interp::ApplyRewriteOp::create(builder,
-      rewriteOp.getLoc(), rewriteOp.getResultTypes(), rewriteOp.getNameAttr(),
-      arguments);
+  auto interpOp = pdl_interp::ApplyRewriteOp::create(
+      builder, rewriteOp.getLoc(), rewriteOp.getResultTypes(),
+      rewriteOp.getNameAttr(), arguments);
   for (auto it : llvm::zip(rewriteOp.getResults(), interpOp.getResults()))
     rewriteValues[std::get<0>(it)] = std::get<1>(it);
 }
@@ -726,8 +734,8 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::AttributeOp attrOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  Value newAttr = pdl_interp::CreateAttributeOp::create(builder,
-      attrOp.getLoc(), attrOp.getValueAttr());
+  Value newAttr = pdl_interp::CreateAttributeOp::create(
+      builder, attrOp.getLoc(), attrOp.getValueAttr());
   rewriteValues[attrOp] = newAttr;
 }
 
@@ -735,7 +743,7 @@ void PatternLowering::generateRewriter(
     pdl::EraseOp eraseOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
   pdl_interp::EraseOp::create(builder, eraseOp.getLoc(),
-                                      mapRewriteValue(eraseOp.getOpValue()));
+                              mapRewriteValue(eraseOp.getOpValue()));
 }
 
 void PatternLowering::generateRewriter(
@@ -756,9 +764,9 @@ void PatternLowering::generateRewriter(
 
   // Create the new operation.
   Location loc = operationOp.getLoc();
-  Value createdOp = pdl_interp::CreateOperationOp::create(builder,
-      loc, *operationOp.getOpName(), types, hasInferredResultTypes, operands,
-      attributes, operationOp.getAttributeValueNames());
+  Value createdOp = pdl_interp::CreateOperationOp::create(
+      builder, loc, *operationOp.getOpName(), types, hasInferredResultTypes,
+      operands, attributes, operationOp.getAttributeValueNames());
   rewriteValues[operationOp.getOp()] = createdOp;
 
   // Generate accesses for any results that have their types constrained.
@@ -789,11 +797,12 @@ void PatternLowering::generateRewriter(
     // groups because the exact index of the result is not statically known.
     Value resultVal;
     if (seenVariableLength)
-      resultVal = pdl_interp::GetResultsOp::create(builder,
-          loc, isVariadic ? valueRangeTy : valueTy, createdOp, it.index());
+      resultVal = pdl_interp::GetResultsOp::create(
+          builder, loc, isVariadic ? valueRangeTy : valueTy, createdOp,
+          it.index());
     else
-      resultVal = pdl_interp::GetResultOp::create(builder,
-          loc, valueTy, createdOp, it.index());
+      resultVal = pdl_interp::GetResultOp::create(builder, loc, valueTy,
+                                                  createdOp, it.index());
     type = pdl_interp::GetValueTypeOp::create(builder, loc, resultVal);
   }
 }
@@ -804,8 +813,8 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 4> replOperands;
   for (Value operand : rangeOp.getArguments())
     replOperands.push_back(mapRewriteValue(operand));
-  rewriteValues[rangeOp] = pdl_interp::CreateRangeOp::create(builder,
-      rangeOp.getLoc(), rangeOp.getType(), replOperands);
+  rewriteValues[rangeOp] = pdl_interp::CreateRangeOp::create(
+      builder, rangeOp.getLoc(), rangeOp.getType(), replOperands);
 }
 
 void PatternLowering::generateRewriter(
@@ -820,8 +829,8 @@ void PatternLowering::generateRewriter(
     // Don't use replace if we know the replaced operation has no results.
     auto opOp = replaceOp.getOpValue().getDefiningOp<pdl::OperationOp>();
     if (!opOp || !opOp.getTypeValues().empty()) {
-      replOperands.push_back(pdl_interp::GetResultsOp::create(builder,
-          replOp.getLoc(), mapRewriteValue(replOp)));
+      replOperands.push_back(pdl_interp::GetResultsOp::create(
+          builder, replOp.getLoc(), mapRewriteValue(replOp)));
     }
   } else {
     for (Value operand : replaceOp.getReplValues())
@@ -830,29 +839,29 @@ void PatternLowering::generateRewriter(
 
   // If there are no replacement values, just create an erase instead.
   if (replOperands.empty()) {
-    pdl_interp::EraseOp::create(builder,
-        replaceOp.getLoc(), mapRewriteValue(replaceOp.getOpValue()));
+    pdl_interp::EraseOp::create(builder, replaceOp.getLoc(),
+                                mapRewriteValue(replaceOp.getOpValue()));
     return;
   }
 
   pdl_interp::ReplaceOp::create(builder, replaceOp.getLoc(),
-                                        mapRewriteValue(replaceOp.getOpValue()),
-                                        replOperands);
+                                mapRewriteValue(replaceOp.getOpValue()),
+                                replOperands);
 }
 
 void PatternLowering::generateRewriter(
     pdl::ResultOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = pdl_interp::GetResultOp::create(builder,
-      resultOp.getLoc(), builder.getType<pdl::ValueType>(),
+  rewriteValues[resultOp] = pdl_interp::GetResultOp::create(
+      builder, resultOp.getLoc(), builder.getType<pdl::ValueType>(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
 
 void PatternLowering::generateRewriter(
     pdl::ResultsOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = pdl_interp::GetResultsOp::create(builder,
-      resultOp.getLoc(), resultOp.getType(),
+  rewriteValues[resultOp] = pdl_interp::GetResultsOp::create(
+      builder, resultOp.getLoc(), resultOp.getType(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
 
@@ -873,8 +882,8 @@ void PatternLowering::generateRewriter(
   // If the type isn't constant, the users (e.g. OperationOp) will resolve this
   // type.
   if (ArrayAttr typeAttr = typeOp.getConstantTypesAttr()) {
-    rewriteValues[typeOp] = pdl_interp::CreateTypesOp::create(builder,
-        typeOp.getLoc(), typeOp.getType(), typeAttr);
+    rewriteValues[typeOp] = pdl_interp::CreateTypesOp::create(
+        builder, typeOp.getLoc(), typeOp.getType(), typeAttr);
   }
 }
 
@@ -939,10 +948,10 @@ void PatternLowering::generateOperationResultTypeRewriter(
         !replacedOp->isBeforeInBlock(op))
       continue;
 
-    Value replacedOpResults = pdl_interp::GetResultsOp::create(builder,
-        replacedOp->getLoc(), mapRewriteValue(replOpVal));
-    types.push_back(pdl_interp::GetValueTypeOp::create(builder,
-        replacedOp->getLoc(), replacedOpResults));
+    Value replacedOpResults = pdl_interp::GetResultsOp::create(
+        builder, replacedOp->getLoc(), mapRewriteValue(replOpVal));
+    types.push_back(pdl_interp::GetValueTypeOp::create(
+        builder, replacedOp->getLoc(), replacedOpResults));
     return;
   }
 
@@ -985,16 +994,18 @@ void PDLToPDLInterpPass::runOnOperation() {
   // Create the main matcher function This function contains all of the match
   // related functionality from patterns in the module.
   OpBuilder builder = OpBuilder::atBlockBegin(module.getBody());
-  auto matcherFunc = pdl_interp::FuncOp::create(builder,
-      module.getLoc(), pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
+  auto matcherFunc = pdl_interp::FuncOp::create(
+      builder, module.getLoc(),
+      pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
       builder.getFunctionType(builder.getType<pdl::OperationType>(),
                               /*results=*/{}),
       /*attrs=*/ArrayRef<NamedAttribute>());
 
   // Create a nested module to hold the functions invoked for rewriting the IR
   // after a successful match.
-  ModuleOp rewriterModule = ModuleOp::create(builder,
-      module.getLoc(), pdl_interp::PDLInterpDialect::getRewriterModuleName());
+  ModuleOp rewriterModule =
+      ModuleOp::create(builder, module.getLoc(),
+                       pdl_interp::PDLInterpDialect::getRewriterModuleName());
 
   // Generate the code for the patterns within the module.
   PatternLowering generator(matcherFunc, rewriterModule, configMap);
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index 68527da73e1c7..afc25211ad193 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -379,12 +379,11 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
 
   // With the body block done, we can fill in the condition block.
   rewriter.setInsertionPointToEnd(conditionBlock);
-  auto comparison = arith::CmpIOp::create(rewriter,
-      loc, arith::CmpIPredicate::slt, iv, upperBound);
+  auto comparison = arith::CmpIOp::create(
+      rewriter, loc, arith::CmpIPredicate::slt, iv, upperBound);
 
   cf::CondBranchOp::create(rewriter, loc, comparison, firstBodyBlock,
-                                    ArrayRef<Value>(), endBlock,
-                                    ArrayRef<Value>());
+                           ArrayRef<Value>(), endBlock, ArrayRef<Value>());
 
   // The result of the loop operation is the values of the condition block
   // arguments except the induction variable on the last iteration.
@@ -440,8 +439,8 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
 
   rewriter.setInsertionPointToEnd(condBlock);
   cf::CondBranchOp::create(rewriter, loc, ifOp.getCondition(), thenBlock,
-                                    /*trueArgs=*/ArrayRef<Value>(), elseBlock,
-                                    /*falseArgs=*/ArrayRef<Value>());
+                           /*trueArgs=*/ArrayRef<Value>(), elseBlock,
+                           /*falseArgs=*/ArrayRef<Value>());
 
   // Ok, we're done!
   rewriter.replaceOp(ifOp, continueBlock->getArguments());
@@ -465,7 +464,8 @@ ExecuteRegionLowering::matchAndRewrite(ExecuteRegionOp op,
     if (auto terminator = dyn_cast<scf::YieldOp>(block.getTerminator())) {
       ValueRange terminatorOperands = terminator->getOperands();
       rewriter.setInsertionPointToEnd(&block);
-      cf::BranchOp::create(rewriter, loc, remainingOpsBlock, terminatorOperands);
+      cf::BranchOp::create(rewriter, loc, remainingOpsBlock,
+                           terminatorOperands);
       rewriter.eraseOp(terminator);
     }
   }
@@ -695,12 +695,12 @@ IndexSwitchLowering::matchAndRewrite(IndexSwitchOp op,
   SmallVector<ValueRange> caseOperands(caseSuccessors.size(), {});
 
   // Cast switch index to integer case value.
-  Value caseValue = arith::IndexCastOp::create(rewriter,
-      op.getLoc(), rewriter.getI32Type(), op.getArg());
+  Value caseValue = arith::IndexCastOp::create(
+      rewriter, op.getLoc(), rewriter.getI32Type(), op.getArg());
 
-  cf::SwitchOp::create(rewriter,
-      op.getLoc(), caseValue, *defaultBlock, ValueRange(),
-      rewriter.getDenseI32ArrayAttr(caseValues), caseSuccessors, caseOperands);
+  cf::SwitchOp::create(rewriter, op.getLoc(), caseValue, *defaultBlock,
+                       ValueRange(), rewriter.getDenseI32ArrayAttr(caseValues),
+                       caseSuccessors, caseOperands);
   rewriter.replaceOp(op, continueBlock->getArguments());
   return success();
 }
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index 37134fe9d992a..84cbd869c78ef 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -164,8 +164,9 @@ ForLowering::matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
 
   assignValues(adaptor.getInitArgs(), resultVariables, rewriter, loc);
 
-  emitc::ForOp loweredFor = emitc::ForOp::create(rewriter,
-      loc, adaptor.getLowerBound(), adaptor.getUpperBound(), adaptor.getStep());
+  emitc::ForOp loweredFor =
+      emitc::ForOp::create(rewriter, loc, adaptor.getLowerBound(),
+                           adaptor.getUpperBound(), adaptor.getStep());
 
   Block *loweredBody = loweredFor.getBody();
 
@@ -304,8 +305,9 @@ LogicalResult IndexSwitchOpLowering::matchAndRewrite(
                                        "create variables for results failed");
   }
 
-  auto loweredSwitch = emitc::SwitchOp::create(rewriter,
-      loc, adaptor.getArg(), adaptor.getCases(), indexSwitchOp.getNumCases());
+  auto loweredSwitch =
+      emitc::SwitchOp::create(rewriter, loc, adaptor.getArg(),
+                              adaptor.getCases(), indexSwitchOp.getNumCases());
 
   // Lowering all case regions.
   for (auto pair :
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index bfdb5a688b286..f191f3502cf5a 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -85,7 +85,7 @@ static Operation::operand_range getUpperBoundOperands(AffineForOp forOp) {
 // materialize a corresponding constant using builder.
 static Value getOrCreateStep(AffineForOp forOp, OpBuilder &builder) {
   return arith::ConstantIndexOp::create(builder, forOp.getLoc(),
-                                                forOp.getStepAsInt());
+                                        forOp.getStepAsInt());
 }
 
 // Get a Value for the loop lower bound.  If the value requires computation,
@@ -191,11 +191,11 @@ AffineLoopToGpuConverter::collectBounds(AffineForOp forOp, unsigned numLoops) {
     }
 
     Value range = arith::SubIOp::create(builder, currentLoop.getLoc(),
-                                                upperBound, lowerBound);
+                                        upperBound, lowerBound);
     Value step = getOrCreateStep(currentLoop, builder);
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
-      range =
-          arith::CeilDivSIOp::create(builder, currentLoop.getLoc(), range, step);
+      range = arith::CeilDivSIOp::create(builder, currentLoop.getLoc(), range,
+                                         step);
     dims.push_back(range);
 
     lbs.push_back(lowerBound);
@@ -232,9 +232,9 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
 
   // Create a launch op and move the body region of the innermost loop to the
   // launch op.
-  auto launchOp = gpu::LaunchOp::create(builder,
-      rootForOp.getLoc(), gridSizeX, gridSizeY, gridSizeZ, blockSizeX,
-      blockSizeY, blockSizeZ);
+  auto launchOp =
+      gpu::LaunchOp::create(builder, rootForOp.getLoc(), gridSizeX, gridSizeY,
+                            gridSizeZ, blockSizeX, blockSizeY, blockSizeZ);
 
   // Replace the loop terminator (loops contain only a single block) with the
   // gpu terminator and move the operations from the loop body block to the gpu
@@ -320,7 +320,7 @@ static Value deriveStaticUpperBound(Value upperBound,
     for (const AffineExpr &result : minOp.getMap().getResults()) {
       if (auto constExpr = dyn_cast<AffineConstantExpr>(result)) {
         return arith::ConstantIndexOp::create(rewriter, minOp.getLoc(),
-                                                       constExpr.getValue());
+                                              constExpr.getValue());
       }
     }
   }
@@ -344,8 +344,8 @@ static Value deriveStaticUpperBound(Value upperBound,
         if ((lhs.value() < 0) != (rhs.value() < 0))
           return {};
 
-        return arith::ConstantIndexOp::create(rewriter,
-            multiplyOp.getLoc(), lhs.value() * rhs.value());
+        return arith::ConstantIndexOp::create(rewriter, multiplyOp.getLoc(),
+                                              lhs.value() * rhs.value());
       }
   }
 
@@ -423,7 +423,7 @@ static LogicalResult processParallelLoop(
       return val;
     if (auto constOp = val.getDefiningOp<arith::ConstantOp>())
       return arith::ConstantOp::create(rewriter, constOp.getLoc(),
-                                                constOp.getValue());
+                                       constOp.getValue());
     return {};
   };
 
@@ -453,8 +453,8 @@ static LogicalResult processParallelLoop(
           1, 2,
           rewriter.getAffineDimExpr(0) * rewriter.getAffineSymbolExpr(0) +
               rewriter.getAffineSymbolExpr(1));
-      newIndex = AffineApplyOp::create(rewriter,
-          loc, annotation.getMap().compose(lowerAndStep),
+      newIndex = AffineApplyOp::create(
+          rewriter, loc, annotation.getMap().compose(lowerAndStep),
           ValueRange{operand, ensureLaunchIndependent(step),
                      ensureLaunchIndependent(lowerBound)});
       // If there was also a bound, insert that, too.
@@ -498,8 +498,8 @@ static LogicalResult processParallelLoop(
               1, 2,
               ((rewriter.getAffineDimExpr(0) - rewriter.getAffineSymbolExpr(0))
                    .ceilDiv(rewriter.getAffineSymbolExpr(1))));
-          Value launchBound = AffineApplyOp::create(rewriter,
-              loc, annotation.getBound().compose(stepMap),
+          Value launchBound = AffineApplyOp::create(
+              rewriter, loc, annotation.getBound().compose(stepMap),
               ValueRange{
                   ensureLaunchIndependent(
                       cloningMap.lookupOrDefault(upperBound)),
@@ -517,8 +517,8 @@ static LogicalResult processParallelLoop(
         if (!boundIsPrecise) {
           // We are using an approximation, create a surrounding conditional.
           Value originalBound = std::get<3>(config);
-          arith::CmpIOp pred = arith::CmpIOp::create(rewriter,
-              loc, arith::CmpIPredicate::slt, newIndex,
+          arith::CmpIOp pred = arith::CmpIOp::create(
+              rewriter, loc, arith::CmpIPredicate::slt, newIndex,
               cloningMap.lookupOrDefault(originalBound));
           scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, pred, false);
           rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
@@ -530,10 +530,10 @@ static LogicalResult processParallelLoop(
       }
     } else {
       // Create a sequential for loop.
-      auto loopOp = scf::ForOp::create(rewriter,
-          loc, cloningMap.lookupOrDefault(lowerBound),
-          cloningMap.lookupOrDefault(upperBound),
-          cloningMap.lookupOrDefault(step));
+      auto loopOp = scf::ForOp::create(rewriter, loc,
+                                       cloningMap.lookupOrDefault(lowerBound),
+                                       cloningMap.lookupOrDefault(upperBound),
+                                       cloningMap.lookupOrDefault(step));
       newIndex = loopOp.getInductionVar();
       rewriter.setInsertionPointToStart(loopOp.getBody());
       // Put a sentinel into the worklist so we know when to pop out of the loop
@@ -609,9 +609,9 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
   Location loc = parallelOp.getLoc();
   Value constantOne =
       arith::ConstantIndexOp::create(rewriter, parallelOp.getLoc(), 1);
-  gpu::LaunchOp launchOp = gpu::LaunchOp::create(rewriter,
-      parallelOp.getLoc(), constantOne, constantOne, constantOne, constantOne,
-      constantOne, constantOne);
+  gpu::LaunchOp launchOp = gpu::LaunchOp::create(
+      rewriter, parallelOp.getLoc(), constantOne, constantOne, constantOne,
+      constantOne, constantOne, constantOne);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
   gpu::TerminatorOp::create(rewriter, loc);
   rewriter.setInsertionPointToStart(&launchOp.getBody().front());
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 7b4c37d44824d..34f372af1e4b5 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -188,7 +188,7 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
   OpBuilder::InsertionGuard guard(builder);
   Type type = reduce.getOperands()[reductionIndex].getType();
   auto decl = omp::DeclareReductionOp::create(builder, reduce.getLoc(),
-                                                      "__scf_reduction", type);
+                                              "__scf_reduction", type);
   symbolTable.insert(decl);
 
   builder.createBlock(&decl.getInitializerRegion(),
@@ -228,10 +228,10 @@ static omp::DeclareReductionOp addAtomicRMW(OpBuilder &builder,
   Block *atomicBlock = &decl.getAtomicReductionRegion().back();
   builder.setInsertionPointToEnd(atomicBlock);
   Value loaded = LLVM::LoadOp::create(builder, reduce.getLoc(), decl.getType(),
-                                              atomicBlock->getArgument(1));
+                                      atomicBlock->getArgument(1));
   LLVM::AtomicRMWOp::create(builder, reduce.getLoc(), atomicKind,
-                                    atomicBlock->getArgument(0), loaded,
-                                    LLVM::AtomicOrdering::monotonic);
+                            atomicBlock->getArgument(0), loaded,
+                            LLVM::AtomicOrdering::monotonic);
   omp::YieldOp::create(builder, reduce.getLoc(), ArrayRef<Value>());
   return decl;
 }
@@ -380,8 +380,9 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     // Allocate reduction variables. Make sure the we don't overflow the stack
     // with local `alloca`s by saving and restoring the stack pointer.
     Location loc = parallelOp.getLoc();
-    Value one = LLVM::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerType(64), rewriter.getI64IntegerAttr(1));
+    Value one =
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getIntegerType(64),
+                                 rewriter.getI64IntegerAttr(1));
     SmallVector<Value> reductionVariables;
     reductionVariables.reserve(parallelOp.getNumReductions());
     auto ptrType = LLVM::LLVMPointerType::get(parallelOp.getContext());
@@ -390,8 +391,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
               isa<LLVM::PointerElementTypeInterface>(init.getType())) &&
              "cannot create a reduction variable if the type is not an LLVM "
              "pointer element");
-      Value storage =
-          LLVM::AllocaOp::create(rewriter, loc, ptrType, init.getType(), one, 0);
+      Value storage = LLVM::AllocaOp::create(rewriter, loc, ptrType,
+                                             init.getType(), one, 0);
       LLVM::StoreOp::create(rewriter, loc, init, storage);
       reductionVariables.push_back(storage);
     }
@@ -412,7 +413,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
              "expect reduction region to have one block");
       Value pvtRedVar = parallelOp.getRegion().addArgument(x.getType(), loc);
       Value pvtRedVal = LLVM::LoadOp::create(rewriter, reduce.getLoc(),
-                                                      rD.getType(), pvtRedVar);
+                                             rD.getType(), pvtRedVar);
       // Make a copy of the reduction combiner region in the body
       mlir::OpBuilder builder(rewriter.getContext());
       builder.setInsertionPoint(reduce);
@@ -437,12 +438,12 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
 
     Value numThreadsVar;
     if (numThreads > 0) {
-      numThreadsVar = LLVM::ConstantOp::create(rewriter,
-          loc, rewriter.getI32IntegerAttr(numThreads));
+      numThreadsVar = LLVM::ConstantOp::create(
+          rewriter, loc, rewriter.getI32IntegerAttr(numThreads));
     }
     // Create the parallel wrapper.
-    auto ompParallel = omp::ParallelOp::create(rewriter,
-        loc,
+    auto ompParallel = omp::ParallelOp::create(
+        rewriter, loc,
         /* allocate_vars = */ llvm::SmallVector<Value>{},
         /* allocator_vars = */ llvm::SmallVector<Value>{},
         /* if_expr = */ Value{},
@@ -490,8 +491,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
                                               parallelOp.getLoc()));
 
         // Create loop nest and populate region with contents of scf.parallel.
-        auto loopOp = omp::LoopNestOp::create(rewriter,
-            parallelOp.getLoc(), parallelOp.getLowerBound(),
+        auto loopOp = omp::LoopNestOp::create(
+            rewriter, parallelOp.getLoc(), parallelOp.getLowerBound(),
             parallelOp.getUpperBound(), parallelOp.getStep());
 
         rewriter.inlineRegionBefore(parallelOp.getRegion(), loopOp.getRegion(),
@@ -511,8 +512,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
             rewriter.splitBlock(&loopOpEntryBlock, loopOpEntryBlock.begin());
         rewriter.setInsertionPointToStart(&loopOpEntryBlock);
 
-        auto scope = memref::AllocaScopeOp::create(rewriter, parallelOp.getLoc(),
-                                                            TypeRange());
+        auto scope = memref::AllocaScopeOp::create(
+            rewriter, parallelOp.getLoc(), TypeRange());
         omp::YieldOp::create(rewriter, loc, ValueRange());
         Block *scopeBlock = rewriter.createBlock(&scope.getBodyRegion());
         rewriter.mergeBlocks(ops, scopeBlock);
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index b494fa98f1362..dc92367fc58cd 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -71,9 +71,9 @@ void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp,
     auto pointerType =
         spirv::PointerType::get(convertedType, spirv::StorageClass::Function);
     rewriter.setInsertionPoint(newOp);
-    auto alloc = spirv::VariableOp::create(rewriter,
-        loc, pointerType, spirv::StorageClass::Function,
-        /*initializer=*/nullptr);
+    auto alloc = spirv::VariableOp::create(rewriter, loc, pointerType,
+                                           spirv::StorageClass::Function,
+                                           /*initializer=*/nullptr);
     allocas.push_back(alloc);
     rewriter.setInsertionPointAfter(newOp);
     Value loadResult = spirv::LoadOp::create(rewriter, loc, alloc);
@@ -135,7 +135,8 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // a single back edge from the continue to header block, and a single exit
     // from header to merge.
     auto loc = forOp.getLoc();
-    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
+    auto loopOp =
+        spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     OpBuilder::InsertionGuard guard(rewriter);
@@ -177,11 +178,12 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // Generate the rest of the loop header.
     rewriter.setInsertionPointToEnd(header);
     auto *mergeBlock = loopOp.getMergeBlock();
-    auto cmpOp = spirv::SLessThanOp::create(rewriter,
-        loc, rewriter.getI1Type(), newIndVar, adaptor.getUpperBound());
+    auto cmpOp = spirv::SLessThanOp::create(rewriter, loc, rewriter.getI1Type(),
+                                            newIndVar, adaptor.getUpperBound());
 
-    spirv::BranchConditionalOp::create(rewriter,
-        loc, cmpOp, body, ArrayRef<Value>(), mergeBlock, ArrayRef<Value>());
+    spirv::BranchConditionalOp::create(rewriter, loc, cmpOp, body,
+                                       ArrayRef<Value>(), mergeBlock,
+                                       ArrayRef<Value>());
 
     // Generate instructions to increment the step of the induction variable and
     // branch to the header.
@@ -189,8 +191,8 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     rewriter.setInsertionPointToEnd(continueBlock);
 
     // Add the step to the induction variable and branch to the header.
-    Value updatedIndVar = spirv::IAddOp::create(rewriter,
-        loc, newIndVar.getType(), newIndVar, adaptor.getStep());
+    Value updatedIndVar = spirv::IAddOp::create(
+        rewriter, loc, newIndVar.getType(), newIndVar, adaptor.getStep());
     spirv::BranchOp::create(rewriter, loc, header, updatedIndVar);
 
     // Infer the return types from the init operands. Vector type may get
@@ -237,8 +239,8 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
 
     // Create `spirv.selection` operation, selection header block and merge
     // block.
-    auto selectionOp =
-        spirv::SelectionOp::create(rewriter, loc, spirv::SelectionControl::None);
+    auto selectionOp = spirv::SelectionOp::create(
+        rewriter, loc, spirv::SelectionControl::None);
     auto *mergeBlock = rewriter.createBlock(&selectionOp.getBody(),
                                             selectionOp.getBody().end());
     spirv::MergeOp::create(rewriter, loc);
@@ -268,8 +270,8 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     // Create a `spirv.BranchConditional` operation for selection header block.
     rewriter.setInsertionPointToEnd(selectionHeaderBlock);
     spirv::BranchConditionalOp::create(rewriter, loc, adaptor.getCondition(),
-                                                thenBlock, ArrayRef<Value>(),
-                                                elseBlock, ArrayRef<Value>());
+                                       thenBlock, ArrayRef<Value>(), elseBlock,
+                                       ArrayRef<Value>());
 
     replaceSCFOutputValue(ifOp, selectionOp, rewriter, scfToSPIRVContext,
                           returnTypes);
@@ -320,7 +322,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
         args.append(operands.begin(), operands.end());
         rewriter.setInsertionPoint(br);
         spirv::BranchOp::create(rewriter, terminatorOp.getLoc(), br.getTarget(),
-                                         args);
+                                args);
         rewriter.eraseOp(br);
       }
     }
@@ -340,7 +342,8 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
   matchAndRewrite(scf::WhileOp whileOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto loc = whileOp.getLoc();
-    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
+    auto loopOp =
+        spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     Region &beforeRegion = whileOp.getBefore();
@@ -403,9 +406,9 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
       // Create local variables before the scf.while op.
       rewriter.setInsertionPoint(loopOp);
-      auto alloc = spirv::VariableOp::create(rewriter,
-          condLoc, pointerType, spirv::StorageClass::Function,
-          /*initializer=*/nullptr);
+      auto alloc = spirv::VariableOp::create(rewriter, condLoc, pointerType,
+                                             spirv::StorageClass::Function,
+                                             /*initializer=*/nullptr);
 
       // Load the final result values after the scf.while op.
       rewriter.setInsertionPointAfter(loopOp);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index 934bb0d1c37fd..035f197b1eac2 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -194,8 +194,8 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
     if (!kernelFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(module.getBody());
-      kernelFunc = LLVM::LLVMFuncOp::create(rewriter,
-          rewriter.getUnknownLoc(), newKernelFuncName,
+      kernelFunc = LLVM::LLVMFuncOp::create(
+          rewriter, rewriter.getUnknownLoc(), newKernelFuncName,
           LLVM::LLVMFunctionType::get(LLVM::LLVMVoidType::get(context),
                                       ArrayRef<Type>()));
       rewriter.setInsertionPoint(launchOp);
@@ -245,8 +245,8 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       if (!dstGlobal) {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
-        dstGlobal = LLVM::GlobalOp::create(rewriter,
-            loc, dstGlobalType,
+        dstGlobal = LLVM::GlobalOp::create(
+            rewriter, loc, dstGlobalType,
             /*isConstant=*/false, LLVM::Linkage::Linkonce, name, Attribute(),
             /*alignment=*/0);
         rewriter.setInsertionPoint(launchOp);
@@ -255,8 +255,8 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       // Copy the data from src operand pointer to dst global variable. Save
       // src, dst and size so that we can copy data back after emulating the
       // kernel call.
-      Value dst = LLVM::AddressOfOp::create(rewriter,
-          loc, typeConverter->convertType(spirvGlobal.getType()),
+      Value dst = LLVM::AddressOfOp::create(
+          rewriter, loc, typeConverter->convertType(spirvGlobal.getType()),
           dstGlobal.getSymName());
       copy(loc, dst, src, sizeBytes, rewriter);
 
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index f490e065143e7..788d9f95ec331 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -94,13 +94,13 @@ static IntegerAttr minusOneIntegerAttribute(Type type, Builder builder) {
 static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
                                       PatternRewriter &rewriter) {
   if (isa<VectorType>(srcType)) {
-    return LLVM::ConstantOp::create(rewriter,
-        loc, dstType,
+    return LLVM::ConstantOp::create(
+        rewriter, loc, dstType,
         SplatElementsAttr::get(cast<ShapedType>(srcType),
                                minusOneIntegerAttribute(srcType, rewriter)));
   }
-  return LLVM::ConstantOp::create(rewriter,
-      loc, dstType, minusOneIntegerAttribute(srcType, rewriter));
+  return LLVM::ConstantOp::create(rewriter, loc, dstType,
+                                  minusOneIntegerAttribute(srcType, rewriter));
 }
 
 /// Creates `llvm.mlir.constant` with a floating-point scalar or vector value.
@@ -108,14 +108,14 @@ static Value createFPConstant(Location loc, Type srcType, Type dstType,
                               PatternRewriter &rewriter, double value) {
   if (auto vecType = dyn_cast<VectorType>(srcType)) {
     auto floatType = cast<FloatType>(vecType.getElementType());
-    return LLVM::ConstantOp::create(rewriter,
-        loc, dstType,
+    return LLVM::ConstantOp::create(
+        rewriter, loc, dstType,
         SplatElementsAttr::get(vecType,
                                rewriter.getFloatAttr(floatType, value)));
   }
   auto floatType = cast<FloatType>(srcType);
-  return LLVM::ConstantOp::create(rewriter,
-      loc, dstType, rewriter.getFloatAttr(floatType, value));
+  return LLVM::ConstantOp::create(rewriter, loc, dstType,
+                                  rewriter.getFloatAttr(floatType, value));
 }
 
 /// Utility function for bitfield ops:
@@ -153,10 +153,10 @@ static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
   auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
   Value broadcasted = LLVM::PoisonOp::create(rewriter, loc, llvmVectorType);
   for (unsigned i = 0; i < numElements; ++i) {
-    auto index = LLVM::ConstantOp::create(rewriter,
-        loc, llvmI32Type, rewriter.getI32IntegerAttr(i));
-    broadcasted = LLVM::InsertElementOp::create(rewriter,
-        loc, llvmVectorType, broadcasted, toBroadcast, index);
+    auto index = LLVM::ConstantOp::create(rewriter, loc, llvmI32Type,
+                                          rewriter.getI32IntegerAttr(i));
+    broadcasted = LLVM::InsertElementOp::create(
+        rewriter, loc, llvmVectorType, broadcasted, toBroadcast, index);
   }
   return broadcasted;
 }
@@ -217,8 +217,8 @@ static Type convertStructTypePacked(spirv::StructType type,
 /// Creates LLVM dialect constant with the given value.
 static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
                                  unsigned value) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, IntegerType::get(rewriter.getContext(), 32),
+  return LLVM::ConstantOp::create(
+      rewriter, loc, IntegerType::get(rewriter.getContext(), 32),
       rewriter.getIntegerAttr(rewriter.getI32Type(), value));
 }
 
@@ -322,8 +322,9 @@ class AccessChainPattern : public SPIRVToLLVMConversion<spirv::AccessChainOp> {
     auto llvmIndexType = getTypeConverter()->convertType(indexType);
     if (!llvmIndexType)
       return rewriter.notifyMatchFailure(op, "type conversion failed");
-    Value zero = LLVM::ConstantOp::create(rewriter,
-        op.getLoc(), llvmIndexType, rewriter.getIntegerAttr(indexType, 0));
+    Value zero =
+        LLVM::ConstantOp::create(rewriter, op.getLoc(), llvmIndexType,
+                                 rewriter.getIntegerAttr(indexType, 0));
     indices.insert(indices.begin(), zero);
 
     auto elementType = getTypeConverter()->convertType(
@@ -377,11 +378,11 @@ class BitFieldInsertPattern
     Value maskShiftedByCount =
         LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
     Value negated = LLVM::XOrOp::create(rewriter, loc, dstType,
-                                                 maskShiftedByCount, minusOne);
+                                        maskShiftedByCount, minusOne);
     Value maskShiftedByCountAndOffset =
         LLVM::ShlOp::create(rewriter, loc, dstType, negated, offset);
-    Value mask = LLVM::XOrOp::create(rewriter,
-        loc, dstType, maskShiftedByCountAndOffset, minusOne);
+    Value mask = LLVM::XOrOp::create(rewriter, loc, dstType,
+                                     maskShiftedByCountAndOffset, minusOne);
 
     // Extract unchanged bits from the `Base`  that are outside of
     // [Offset, Offset + Count - 1]. Then `or` with shifted `Insert`.
@@ -470,8 +471,8 @@ class BitFieldSExtractPattern
     auto baseSize = rewriter.getIntegerAttr(integerType, getBitWidth(srcType));
     Value size =
         isa<VectorType>(srcType)
-            ? LLVM::ConstantOp::create(rewriter,
-                  loc, dstType,
+            ? LLVM::ConstantOp::create(
+                  rewriter, loc, dstType,
                   SplatElementsAttr::get(cast<ShapedType>(srcType), baseSize))
             : LLVM::ConstantOp::create(rewriter, loc, dstType, baseSize);
 
@@ -481,8 +482,8 @@ class BitFieldSExtractPattern
         LLVM::AddOp::create(rewriter, loc, dstType, count, offset);
     Value amountToShiftLeft =
         LLVM::SubOp::create(rewriter, loc, dstType, size, countPlusOffset);
-    Value baseShiftedLeft = LLVM::ShlOp::create(rewriter,
-        loc, dstType, op.getBase(), amountToShiftLeft);
+    Value baseShiftedLeft = LLVM::ShlOp::create(
+        rewriter, loc, dstType, op.getBase(), amountToShiftLeft);
 
     // Shift the result right, filling the bits with the sign bit.
     Value amountToShiftRight =
@@ -518,7 +519,7 @@ class BitFieldUExtractPattern
     Value maskShiftedByCount =
         LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
     Value mask = LLVM::XOrOp::create(rewriter, loc, dstType, maskShiftedByCount,
-                                              minusOne);
+                                     minusOne);
 
     // Shift `Base` by `Offset` and apply the mask on it.
     Value shiftedBase =
@@ -694,8 +695,8 @@ class ExecutionModePattern
     auto structType = LLVM::LLVMStructType::getLiteral(context, fields);
 
     // Create `llvm.mlir.global` with initializer region containing one block.
-    auto global = LLVM::GlobalOp::create(rewriter,
-        UnknownLoc::get(context), structType, /*isConstant=*/true,
+    auto global = LLVM::GlobalOp::create(
+        rewriter, UnknownLoc::get(context), structType, /*isConstant=*/true,
         LLVM::Linkage::External, executionModeInfoName, Attribute(),
         /*alignment=*/0);
     Location loc = global.getLoc();
@@ -705,8 +706,8 @@ class ExecutionModePattern
     // Initialize the struct and set the execution mode value.
     rewriter.setInsertionPointToStart(block);
     Value structValue = LLVM::PoisonOp::create(rewriter, loc, structType);
-    Value executionMode = LLVM::ConstantOp::create(rewriter,
-        loc, llvmI32Type,
+    Value executionMode = LLVM::ConstantOp::create(
+        rewriter, loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
     structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
@@ -716,8 +717,8 @@ class ExecutionModePattern
     for (unsigned i = 0, e = values.size(); i < e; ++i) {
       auto attr = values.getValue()[i];
       Value entry = LLVM::ConstantOp::create(rewriter, loc, llvmI32Type, attr);
-      structValue = LLVM::InsertValueOp::create(rewriter,
-          loc, structValue, entry, ArrayRef<int64_t>({1, i}));
+      structValue = LLVM::InsertValueOp::create(
+          rewriter, loc, structValue, entry, ArrayRef<int64_t>({1, i}));
     }
     LLVM::ReturnOp::create(rewriter, loc, ArrayRef<Value>({structValue}));
     rewriter.eraseOp(op);
@@ -973,8 +974,8 @@ class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
     IntegerAttr minusOne = minusOneIntegerAttribute(srcType, rewriter);
     auto mask =
         isa<VectorType>(srcType)
-            ? LLVM::ConstantOp::create(rewriter,
-                  loc, dstType,
+            ? LLVM::ConstantOp::create(
+                  rewriter, loc, dstType,
                   SplatElementsAttr::get(cast<VectorType>(srcType), minusOne))
             : LLVM::ConstantOp::create(rewriter, loc, dstType, minusOne);
     rewriter.template replaceOpWithNewOp<LLVM::XOrOp>(notOp, dstType,
@@ -1034,8 +1035,8 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
     return func;
 
   OpBuilder b(symbolTable->getRegion(0));
-  func = LLVM::LLVMFuncOp::create(b,
-      symbolTable->getLoc(), name,
+  func = LLVM::LLVMFuncOp::create(
+      b, symbolTable->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes));
   func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
   func.setConvergent(convergent);
@@ -1078,12 +1079,12 @@ class ControlBarrierPattern : public SPIRVToLLVMConversion<BarrierOpTy> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, {i32, i32, i32}, voidTy);
 
     Location loc = controlBarrierOp->getLoc();
-    Value execution = LLVM::ConstantOp::create(rewriter,
-        loc, i32, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value memory = LLVM::ConstantOp::create(rewriter,
-        loc, i32, static_cast<int32_t>(adaptor.getMemoryScope()));
-    Value semantics = LLVM::ConstantOp::create(rewriter,
-        loc, i32, static_cast<int32_t>(adaptor.getMemorySemantics()));
+    Value execution = LLVM::ConstantOp::create(
+        rewriter, loc, i32, static_cast<int32_t>(adaptor.getExecutionScope()));
+    Value memory = LLVM::ConstantOp::create(
+        rewriter, loc, i32, static_cast<int32_t>(adaptor.getMemoryScope()));
+    Value semantics = LLVM::ConstantOp::create(
+        rewriter, loc, i32, static_cast<int32_t>(adaptor.getMemorySemantics()));
 
     auto call = createSPIRVBuiltinCall(loc, rewriter, func,
                                        {execution, memory, semantics});
@@ -1255,10 +1256,12 @@ class GroupReducePattern : public SPIRVToLLVMConversion<ReduceOp> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, paramTypes, retTy);
 
     Location loc = op.getLoc();
-    Value scope = LLVM::ConstantOp::create(rewriter,
-        loc, i32Ty, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value groupOp = LLVM::ConstantOp::create(rewriter,
-        loc, i32Ty, static_cast<int32_t>(adaptor.getGroupOperation()));
+    Value scope = LLVM::ConstantOp::create(
+        rewriter, loc, i32Ty,
+        static_cast<int32_t>(adaptor.getExecutionScope()));
+    Value groupOp = LLVM::ConstantOp::create(
+        rewriter, loc, i32Ty,
+        static_cast<int32_t>(adaptor.getGroupOperation()));
     SmallVector<Value> operands{scope, groupOp};
     operands.append(adaptor.getOperands().begin(), adaptor.getOperands().end());
 
@@ -1441,9 +1444,8 @@ class SelectionPattern : public SPIRVToLLVMConversion<spirv::SelectionOp> {
     Block *falseBlock = condBrOp.getFalseBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
     LLVM::CondBrOp::create(rewriter, loc, condBrOp.getCondition(), trueBlock,
-                                    condBrOp.getTrueTargetOperands(),
-                                    falseBlock,
-                                    condBrOp.getFalseTargetOperands());
+                           condBrOp.getTrueTargetOperands(), falseBlock,
+                           condBrOp.getFalseTargetOperands());
 
     rewriter.eraseBlock(headerBlock);
     rewriter.inlineRegionBefore(op.getBody(), continueBlock);
@@ -1767,16 +1769,17 @@ class VectorShufflePattern
         baseVector = vector2;
       }
 
-      Value dstIndex = LLVM::ConstantOp::create(rewriter,
-          loc, llvmI32Type, rewriter.getIntegerAttr(rewriter.getI32Type(), i));
-      Value index = LLVM::ConstantOp::create(rewriter,
-          loc, llvmI32Type,
+      Value dstIndex = LLVM::ConstantOp::create(
+          rewriter, loc, llvmI32Type,
+          rewriter.getIntegerAttr(rewriter.getI32Type(), i));
+      Value index = LLVM::ConstantOp::create(
+          rewriter, loc, llvmI32Type,
           rewriter.getIntegerAttr(rewriter.getI32Type(), indexVal - offsetVal));
 
-      auto extractOp = LLVM::ExtractElementOp::create(rewriter,
-          loc, scalarType, baseVector, index);
+      auto extractOp = LLVM::ExtractElementOp::create(rewriter, loc, scalarType,
+                                                      baseVector, index);
       targetOp = LLVM::InsertElementOp::create(rewriter, loc, dstType, targetOp,
-                                                        extractOp, dstIndex);
+                                               extractOp, dstIndex);
     }
     rewriter.replaceOp(op, targetOp);
     return success();
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index 58cffa6119d93..7025c5a7daf93 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -88,34 +88,34 @@ Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
     Value shape = std::get<0>(tup);
     Value rankDiff = std::get<1>(tup);
     Value outOfBounds = arith::CmpIOp::create(lb, arith::CmpIPredicate::ult,
-                                                 outputDimension, rankDiff);
+                                              outputDimension, rankDiff);
     Type indexTy = lb.getIndexType();
     broadcastedDim =
-        IfOp::create(lb,
-              outOfBounds,
-              [&](OpBuilder &b, Location loc) {
-                scf::YieldOp::create(b, loc, broadcastedDim);
-              },
-              [&](OpBuilder &b, Location loc) {
-                // The broadcasting logic is:
-                // - if one extent (here we arbitrarily choose the
-                // extent from the greater-rank operand) is equal to 1,
-                // then take the extent from the other operand
-                // - otherwise, take the extent as-is.
-                // Note that this logic remains correct in the presence
-                // of dimensions of zero extent.
-                Value lesserRankOperandDimension = arith::SubIOp::create(b,
-                    loc, indexTy, outputDimension, rankDiff);
-                Value lesserRankOperandExtent = tensor::ExtractOp::create(b,
-                    loc, shape, ValueRange{lesserRankOperandDimension});
-
-                Value dimIsOne =
-                    arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
-                                            lesserRankOperandExtent, one);
-                Value dim = arith::SelectOp::create(b,
-                    loc, dimIsOne, broadcastedDim, lesserRankOperandExtent);
-                scf::YieldOp::create(b, loc, dim);
-              })
+        IfOp::create(
+            lb, outOfBounds,
+            [&](OpBuilder &b, Location loc) {
+              scf::YieldOp::create(b, loc, broadcastedDim);
+            },
+            [&](OpBuilder &b, Location loc) {
+              // The broadcasting logic is:
+              // - if one extent (here we arbitrarily choose the
+              // extent from the greater-rank operand) is equal to 1,
+              // then take the extent from the other operand
+              // - otherwise, take the extent as-is.
+              // Note that this logic remains correct in the presence
+              // of dimensions of zero extent.
+              Value lesserRankOperandDimension = arith::SubIOp::create(
+                  b, loc, indexTy, outputDimension, rankDiff);
+              Value lesserRankOperandExtent = tensor::ExtractOp::create(
+                  b, loc, shape, ValueRange{lesserRankOperandDimension});
+
+              Value dimIsOne =
+                  arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
+                                        lesserRankOperandExtent, one);
+              Value dim = arith::SelectOp::create(
+                  b, loc, dimIsOne, broadcastedDim, lesserRankOperandExtent);
+              scf::YieldOp::create(b, loc, dim);
+            })
             .getResult(0);
   }
   return broadcastedDim;
@@ -155,8 +155,8 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
                        return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
-  Value replacement = tensor::GenerateOp::create(lb,
-      getExtentTensorType(lb.getContext()), ValueRange{maxRank},
+  Value replacement = tensor::GenerateOp::create(
+      lb, getExtentTensorType(lb.getContext()), ValueRange{maxRank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value broadcastedDim =
             getBroadcastedDim(ImplicitLocOpBuilder(loc, b), adaptor.getShapes(),
@@ -193,8 +193,8 @@ LogicalResult ConstShapeOpConverter::matchAndRewrite(
   auto loc = op.getLoc();
   SmallVector<Value, 4> extentOperands;
   for (auto extent : op.getShape()) {
-    extentOperands.push_back(
-        arith::ConstantIndexOp::create(rewriter, loc, extent.getLimitedValue()));
+    extentOperands.push_back(arith::ConstantIndexOp::create(
+        rewriter, loc, extent.getLimitedValue()));
   }
   Type resultTy =
       RankedTensorType::get({op.getShape().size()}, rewriter.getIndexType());
@@ -269,11 +269,11 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
                      }));
 
   Type i1Ty = rewriter.getI1Type();
-  Value trueVal =
-      arith::ConstantOp::create(rewriter, loc, i1Ty, rewriter.getBoolAttr(true));
+  Value trueVal = arith::ConstantOp::create(rewriter, loc, i1Ty,
+                                            rewriter.getBoolAttr(true));
 
-  auto reduceResult = ForOp::create(lb,
-      loc, zero, maxRank, one, ValueRange{trueVal},
+  auto reduceResult = ForOp::create(
+      lb, loc, zero, maxRank, one, ValueRange{trueVal},
       [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
         // Find a non-1 dim, if it exists. Note that the first part of this
         // could reuse the Broadcast lowering entirely, but we redo the work
@@ -285,34 +285,34 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
         for (auto tup : llvm::zip(adaptor.getShapes(), rankDiffs)) {
           Value shape, rankDiff;
           std::tie(shape, rankDiff) = tup;
-          Value outOfBounds = arith::CmpIOp::create(b,
-              loc, arith::CmpIPredicate::ult, iv, rankDiff);
+          Value outOfBounds = arith::CmpIOp::create(
+              b, loc, arith::CmpIPredicate::ult, iv, rankDiff);
           broadcastable =
-              IfOp::create(b,
-                   loc, outOfBounds,
-                   [&](OpBuilder &b, Location loc) {
-                     // Non existent dimensions are always broadcastable
-                     scf::YieldOp::create(b, loc, broadcastable);
-                   },
-                   [&](OpBuilder &b, Location loc) {
-                     // Every value needs to be either 1, or the same non-1
-                     // value to be broadcastable in this dim.
-                     Value operandDimension =
-                         arith::SubIOp::create(b, loc, indexTy, iv, rankDiff);
-                     Value dimensionExtent = tensor::ExtractOp::create(b,
-                         loc, shape, ValueRange{operandDimension});
-
-                     Value equalOne = arith::CmpIOp::create(b,
-                         loc, arith::CmpIPredicate::eq, dimensionExtent, one);
-                     Value equalBroadcasted = arith::CmpIOp::create(b,
-                         loc, arith::CmpIPredicate::eq, dimensionExtent,
-                         broadcastedDim);
-                     Value result = arith::AndIOp::create(b,
-                         loc, broadcastable,
-                         arith::OrIOp::create(b, loc, equalOne,
-                                                equalBroadcasted));
-                     scf::YieldOp::create(b, loc, result);
-                   })
+              IfOp::create(
+                  b, loc, outOfBounds,
+                  [&](OpBuilder &b, Location loc) {
+                    // Non existent dimensions are always broadcastable
+                    scf::YieldOp::create(b, loc, broadcastable);
+                  },
+                  [&](OpBuilder &b, Location loc) {
+                    // Every value needs to be either 1, or the same non-1
+                    // value to be broadcastable in this dim.
+                    Value operandDimension =
+                        arith::SubIOp::create(b, loc, indexTy, iv, rankDiff);
+                    Value dimensionExtent = tensor::ExtractOp::create(
+                        b, loc, shape, ValueRange{operandDimension});
+
+                    Value equalOne = arith::CmpIOp::create(
+                        b, loc, arith::CmpIPredicate::eq, dimensionExtent, one);
+                    Value equalBroadcasted =
+                        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
+                                              dimensionExtent, broadcastedDim);
+                    Value result = arith::AndIOp::create(
+                        b, loc, broadcastable,
+                        arith::OrIOp::create(b, loc, equalOne,
+                                             equalBroadcasted));
+                    scf::YieldOp::create(b, loc, result);
+                  })
                   .getResult(0);
         }
 
@@ -427,10 +427,11 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
   Value rank =
       tensor::DimOp::create(rewriter, loc, indexTy, adaptor.getShape(), zero);
 
-  auto loop = scf::ForOp::create(rewriter,
-      loc, zero, rank, one, op.getInitVals(),
+  auto loop = scf::ForOp::create(
+      rewriter, loc, zero, rank, one, op.getInitVals(),
       [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
-        Value extent = tensor::ExtractOp::create(b, loc, adaptor.getShape(), iv);
+        Value extent =
+            tensor::ExtractOp::create(b, loc, adaptor.getShape(), iv);
 
         SmallVector<Value, 2> mappedValues{iv, extent};
         mappedValues.append(args.begin(), args.end());
@@ -515,23 +516,23 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, OpAdaptor adaptor,
   // Generate a linear sequence of compares, all with firstShape as lhs.
   for (Value shape : adaptor.getShapes().drop_front(1)) {
     Value rank = tensor::DimOp::create(rewriter, loc, indexTy, shape, zero);
-    Value eqRank = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                                  firstRank, rank);
-    auto same = IfOp::create(rewriter,
-        loc, eqRank,
+    Value eqRank = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, firstRank, rank);
+    auto same = IfOp::create(
+        rewriter, loc, eqRank,
         [&](OpBuilder &b, Location loc) {
           Value one = arith::ConstantIndexOp::create(b, loc, 1);
           Value init =
               arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(true));
-          auto loop = scf::ForOp::create(b,
-              loc, zero, firstRank, one, ValueRange{init},
+          auto loop = scf::ForOp::create(
+              b, loc, zero, firstRank, one, ValueRange{init},
               [&](OpBuilder &b, Location nestedLoc, Value iv, ValueRange args) {
                 Value conj = args[0];
                 Value lhsExtent =
                     tensor::ExtractOp::create(b, loc, firstShape, iv);
                 Value rhsExtent = tensor::ExtractOp::create(b, loc, shape, iv);
-                Value eqExtent = arith::CmpIOp::create(b,
-                    loc, arith::CmpIPredicate::eq, lhsExtent, rhsExtent);
+                Value eqExtent = arith::CmpIOp::create(
+                    b, loc, arith::CmpIPredicate::eq, lhsExtent, rhsExtent);
                 Value conjNext = arith::AndIOp::create(b, loc, conj, eqExtent);
                 scf::YieldOp::create(b, loc, ValueRange({conjNext}));
               });
@@ -544,7 +545,7 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, OpAdaptor adaptor,
         });
     result = !result ? same.getResult(0)
                      : arith::AndIOp::create(rewriter, loc, result,
-                                                      same.getResult(0));
+                                             same.getResult(0));
   }
   rewriter.replaceOp(op, result);
   return success();
@@ -584,15 +585,15 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
         Value extent = tensor::DimOp::create(rewriter, loc, tensor, i);
         extentValues.push_back(extent);
       } else {
-        Value extent = arith::ConstantIndexOp::create(rewriter,
-            loc, rankedTensorTy.getDimSize(i));
+        Value extent = arith::ConstantIndexOp::create(
+            rewriter, loc, rankedTensorTy.getDimSize(i));
         extentValues.push_back(extent);
       }
     }
 
     // Materialize extent tensor.
-    Value staticExtentTensor = tensor::FromElementsOp::create(rewriter,
-        loc, RankedTensorType::get({rank}, rewriter.getIndexType()),
+    Value staticExtentTensor = tensor::FromElementsOp::create(
+        rewriter, loc, RankedTensorType::get({rank}, rewriter.getIndexType()),
         extentValues);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(),
                                                 staticExtentTensor);
@@ -649,7 +650,7 @@ LogicalResult SplitAtOpConversion::matchAndRewrite(
       tensor::ExtractSliceOp::create(b, adaptor.getOperand(), zero, index, one);
   Value tailSize = arith::SubIOp::create(b, rank, index);
   Value tail = tensor::ExtractSliceOp::create(b, adaptor.getOperand(), index,
-                                                tailSize, one);
+                                              tailSize, one);
   rewriter.replaceOp(op, {head, tail});
   return success();
 }
diff --git a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
index f06c3e1f7cb18..f24972f6b6ee1 100644
--- a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
+++ b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
@@ -69,8 +69,8 @@ class TensorExtractPattern final
       // have bugs dealing with that. So for now, use spirv.Store for
       // initialization.
       varOp = spirv::VariableOp::create(rewriter, loc, varType,
-                                                 spirv::StorageClass::Function,
-                                                 /*initializer=*/nullptr);
+                                        spirv::StorageClass::Function,
+                                        /*initializer=*/nullptr);
       spirv::StoreOp::create(rewriter, loc, varOp, adaptor.getTensor());
     } else {
       // Need to store the value to the local variable. It's questionable
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 44872c8d7540d..044b725c7d805 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -51,8 +51,8 @@ TypedAttr getConstantAttr(Type type, int64_t value, PatternRewriter &rewriter) {
 
 Value getConstantValue(Location loc, Type type, int64_t value,
                        PatternRewriter &rewriter) {
-  return arith::ConstantOp::create(rewriter,
-      loc, getConstantAttr(type, value, rewriter));
+  return arith::ConstantOp::create(rewriter, loc,
+                                   getConstantAttr(type, value, rewriter));
 }
 
 // This converts the TOSA ApplyScale operator to a set of arithmetic ops,
@@ -104,13 +104,13 @@ class ApplyScaleGenericOpConverter
       int64_t roundInt = 1 << 30;
       Value roundUp = getConstantValue(loc, i64Ty, roundInt, rewriter);
       Value roundDown = getConstantValue(loc, i64Ty, -roundInt, rewriter);
-      Value positive = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::sge, value, zero);
+      Value positive = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::sge, value, zero);
       Value dir =
           arith::SelectOp::create(rewriter, loc, positive, roundUp, roundDown);
       Value val = arith::AddIOp::create(rewriter, loc, dir, multiply64);
-      Value valid = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::sgt, shift32, thirtyOne32);
+      Value valid = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::sgt, shift32, thirtyOne32);
       multiply64 =
           arith::SelectOp::create(rewriter, loc, valid, val, multiply64);
     }
@@ -163,10 +163,10 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
     Value high32 = value64.getHigh();
 
     // Determine the direction and amount to shift the high bits.
-    Value shiftOver32 = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sge, shift32, thirtyTwo32);
-    Value roundHighBits = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sgt, shift32, thirtyTwo32);
+    Value shiftOver32 = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sge, shift32, thirtyTwo32);
+    Value roundHighBits = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sgt, shift32, thirtyTwo32);
 
     Value shiftHighL =
         arith::SubIOp::create(rewriter, loc, thirtyTwo32, shift32);
@@ -181,11 +181,11 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
     // Conditionally perform our double round.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       Value negOne32 = getConstantValue(loc, i32Ty, -1, rewriter);
-      Value valuePositive = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::sge, value32, zero32);
+      Value valuePositive = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::sge, value32, zero32);
 
-      Value roundDir =
-          arith::SelectOp::create(rewriter, loc, valuePositive, one32, negOne32);
+      Value roundDir = arith::SelectOp::create(rewriter, loc, valuePositive,
+                                               one32, negOne32);
       roundDir =
           arith::SelectOp::create(rewriter, loc, shiftOver32, roundDir, zero32);
 
@@ -205,14 +205,15 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
       Value shiftSubOne = arith::SubIOp::create(rewriter, loc, shift32, one32);
       Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
       roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, zero32,
-                                                  roundBit);
+                                         roundBit);
 
       Value newLow32 = arith::AddIOp::create(rewriter, loc, low32, roundBit);
-      Value wasRounded = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::ugt, low32, newLow32);
+      Value wasRounded = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::ugt, low32, newLow32);
       low32 = newLow32;
 
-      Value rounded32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, wasRounded);
+      Value rounded32 =
+          arith::ExtUIOp::create(rewriter, loc, i32Ty, wasRounded);
       high32 = arith::AddIOp::create(rewriter, loc, high32, rounded32);
     }
 
@@ -222,7 +223,7 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
           arith::SubIOp::create(rewriter, loc, shiftHighR, one32);
       Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
       roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, roundBit,
-                                                  zero32);
+                                         zero32);
       high32 = arith::AddIOp::create(rewriter, loc, high32, roundBit);
     }
 
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 2aec35ed9d8ca..ec55091cd7eb8 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -70,14 +70,14 @@ materializeBinaryNanCheckIfRequired(OpTy op, PatternRewriter &rewriter,
     return result;
 
   // Unordered comparison of NaN against itself will always return true.
-  Value lhsIsNaN = arith::CmpFOp::create(rewriter,
-      op.getLoc(), arith::CmpFPredicate::UNO, lhs, lhs);
-  Value rhsIsNaN = arith::CmpFOp::create(rewriter,
-      op.getLoc(), arith::CmpFPredicate::UNO, rhs, rhs);
+  Value lhsIsNaN = arith::CmpFOp::create(rewriter, op.getLoc(),
+                                         arith::CmpFPredicate::UNO, lhs, lhs);
+  Value rhsIsNaN = arith::CmpFOp::create(rewriter, op.getLoc(),
+                                         arith::CmpFPredicate::UNO, rhs, rhs);
   Value rhsOrResult =
       arith::SelectOp::create(rewriter, op.getLoc(), lhsIsNaN, rhs, result);
   return arith::SelectOp::create(rewriter, op.getLoc(), rhsIsNaN, lhs,
-                                          rhsOrResult);
+                                 rhsOrResult);
 }
 
 static Value createLinalgBodyCalculationForElementwiseOp(
@@ -92,8 +92,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     return math::AbsFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AbsOp>(op) && isa<IntegerType>(elementTy)) {
-    auto zero = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(elementTy));
+    auto zero = arith::ConstantOp::create(rewriter, loc,
+                                          rewriter.getZeroAttr(elementTy));
     auto neg = arith::SubIOp::create(rewriter, loc, zero, args[0]);
     return arith::MaxSIOp::create(rewriter, loc, args[0], neg);
   }
@@ -140,7 +140,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                                           "Cannot have shift value for float");
         return nullptr;
       }
-      return arith::MulFOp::create(rewriter, loc, resultTypes, args[0], args[1]);
+      return arith::MulFOp::create(rewriter, loc, resultTypes, args[0],
+                                   args[1]);
     }
 
     if (isa<IntegerType>(elementTy)) {
@@ -156,8 +157,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         if (!b.getType().isInteger(32))
           b = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), b);
 
-        auto result = tosa::ApplyScaleOp::create(rewriter,
-            loc, rewriter.getI32Type(), a, b, shiftConst,
+        auto result = tosa::ApplyScaleOp::create(
+            rewriter, loc, rewriter.getI32Type(), a, b, shiftConst,
             rewriter.getStringAttr("SINGLE_ROUND"));
 
         if (elementTy.isInteger(32))
@@ -205,10 +206,10 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     if (isa<IntegerType>(elementTy)) {
       if (!inZp && !outZp) {
-        auto constant = arith::ConstantOp::create(rewriter,
-            loc, IntegerAttr::get(elementTy, 0));
+        auto constant = arith::ConstantOp::create(
+            rewriter, loc, IntegerAttr::get(elementTy, 0));
         return arith::SubIOp::create(rewriter, loc, resultTypes, constant,
-                                              args[0]);
+                                     args[0]);
       }
 
       // Compute the maximum value that can occur in the intermediate buffer.
@@ -231,8 +232,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       }
 
       Type intermediateType = rewriter.getIntegerType(intermediateBitWidth);
-      Value zpAddValue = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
+      Value zpAddValue = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
 
       // The negation can be applied by doing:
       //  outputValue = inZp + outZp - inputValue
@@ -241,11 +242,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       auto sub = arith::SubIOp::create(rewriter, loc, zpAddValue, ext);
 
       // Clamp to the negation range.
-      Value min = arith::ConstantIntOp::create(rewriter,
-          loc, intermediateType,
+      Value min = arith::ConstantIntOp::create(
+          rewriter, loc, intermediateType,
           APInt::getSignedMinValue(inputBitWidth).getSExtValue());
-      Value max = arith::ConstantIntOp::create(rewriter,
-          loc, intermediateType,
+      Value max = arith::ConstantIntOp::create(
+          rewriter, loc, intermediateType,
           APInt::getSignedMaxValue(inputBitWidth).getSExtValue());
       auto clamp = clampIntHelper(loc, sub, min, max, rewriter, false);
 
@@ -291,16 +292,16 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     }
 
     Type i1Ty = IntegerType::get(rewriter.getContext(), /*width=*/1);
-    auto one =
-        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 1));
-    auto zero =
-        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 0));
+    auto one = arith::ConstantOp::create(rewriter, loc,
+                                         IntegerAttr::get(elementTy, 1));
+    auto zero = arith::ConstantOp::create(rewriter, loc,
+                                          IntegerAttr::get(elementTy, 0));
     auto i1one =
         arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(i1Ty, 1));
 
     // Checking that input2 != 0
-    auto shiftValueGreaterThanZero = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sgt, args[1], zero);
+    auto shiftValueGreaterThanZero = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sgt, args[1], zero);
 
     // Checking for the last bit of input1 to be 1
     auto subtract =
@@ -308,13 +309,13 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     auto shifted =
         arith::ShRSIOp::create(rewriter, loc, resultTypes, args[0], subtract)
             ->getResults();
-    auto truncated = arith::TruncIOp::create(rewriter,
-        loc, i1Ty, shifted, ArrayRef<NamedAttribute>());
+    auto truncated = arith::TruncIOp::create(rewriter, loc, i1Ty, shifted,
+                                             ArrayRef<NamedAttribute>());
     auto isInputOdd =
         arith::AndIOp::create(rewriter, loc, i1Ty, truncated, i1one);
 
-    auto shouldRound = arith::AndIOp::create(rewriter,
-        loc, i1Ty, shiftValueGreaterThanZero, isInputOdd);
+    auto shouldRound = arith::AndIOp::create(
+        rewriter, loc, i1Ty, shiftValueGreaterThanZero, isInputOdd);
     auto extended =
         arith::ExtUIOp::create(rewriter, loc, resultTypes, shouldRound);
     return arith::AddIOp::create(rewriter, loc, resultTypes, result, extended);
@@ -331,8 +332,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
   // tosa::LogicalNot
   if (isa<tosa::LogicalNotOp>(op) && elementTy.isInteger(1)) {
-    auto one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerAttr(elementTy, 1));
+    auto one = arith::ConstantOp::create(rewriter, loc,
+                                         rewriter.getIntegerAttr(elementTy, 1));
     return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], one);
   }
 
@@ -379,29 +380,29 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   // tosa::GreaterOp
   if (isa<tosa::GreaterOp>(op) && isa<FloatType>(elementTy))
     return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGT,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   if (isa<tosa::GreaterOp>(op) && elementTy.isSignlessInteger())
     return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sgt,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   // tosa::GreaterEqualOp
   if (isa<tosa::GreaterEqualOp>(op) && isa<FloatType>(elementTy))
     return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGE,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   if (isa<tosa::GreaterEqualOp>(op) && elementTy.isSignlessInteger())
     return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sge,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   // tosa::EqualOp
   if (isa<tosa::EqualOp>(op) && isa<FloatType>(elementTy))
     return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OEQ,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   if (isa<tosa::EqualOp>(op) && elementTy.isSignlessInteger())
     return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   // tosa::SelectOp
   if (isa<tosa::SelectOp>(op)) {
@@ -449,10 +450,10 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                    APFloat::rmNearestTiesToEven, &losesInfo);
     maxApf.convert(cast<FloatType>(elementTy).getFloatSemantics(),
                    APFloat::rmNearestTiesToEven, &losesInfo);
-    auto min = arith::ConstantOp::create(rewriter,
-        loc, elementTy, rewriter.getFloatAttr(elementTy, minApf));
-    auto max = arith::ConstantOp::create(rewriter,
-        loc, elementTy, rewriter.getFloatAttr(elementTy, maxApf));
+    auto min = arith::ConstantOp::create(
+        rewriter, loc, elementTy, rewriter.getFloatAttr(elementTy, minApf));
+    auto max = arith::ConstantOp::create(
+        rewriter, loc, elementTy, rewriter.getFloatAttr(elementTy, maxApf));
     auto result = clampFloatHelper(loc, args[0], min, max, rewriter);
 
     auto clampOp = llvm::cast<tosa::ClampOp>(op);
@@ -478,8 +479,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     //   return init if x == NaN else result
 
     // Unordered comparison of NaN against itself will always return true.
-    Value isNaN = arith::CmpFOp::create(rewriter,
-        op->getLoc(), arith::CmpFPredicate::UNO, args[0], args[0]);
+    Value isNaN = arith::CmpFOp::create(
+        rewriter, op->getLoc(), arith::CmpFPredicate::UNO, args[0], args[0]);
     // TOSA specifies that in "ignore" NaN mode the result is "min" if the input
     // is NaN.
     return arith::SelectOp::create(rewriter, op->getLoc(), isNaN, min, result);
@@ -515,10 +516,10 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     min = std::min(min, maxRepresentable);
     max = std::min(max, maxRepresentable);
 
-    auto minVal = arith::ConstantIntOp::create(rewriter,
-        loc, min, intTy.getIntOrFloatBitWidth());
-    auto maxVal = arith::ConstantIntOp::create(rewriter,
-        loc, max, intTy.getIntOrFloatBitWidth());
+    auto minVal = arith::ConstantIntOp::create(rewriter, loc, min,
+                                               intTy.getIntOrFloatBitWidth());
+    auto maxVal = arith::ConstantIntOp::create(rewriter, loc, max,
+                                               intTy.getIntOrFloatBitWidth());
     return clampIntHelper(loc, args[0], minVal, maxVal, rewriter,
                           intTy.isUnsignedInteger());
   }
@@ -550,20 +551,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && bitExtend)
       return arith::ExtFOp::create(rewriter, loc, resultTypes, args,
-                                            ArrayRef<NamedAttribute>());
+                                   ArrayRef<NamedAttribute>());
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && !bitExtend)
       return arith::TruncFOp::create(rewriter, loc, resultTypes, args,
-                                              ArrayRef<NamedAttribute>());
+                                     ArrayRef<NamedAttribute>());
 
     // 1-bit integers need to be treated as signless.
     if (srcTy.isInteger(1) && arith::UIToFPOp::areCastCompatible(srcTy, dstTy))
       return arith::UIToFPOp::create(rewriter, loc, resultTypes, args,
-                                              ArrayRef<NamedAttribute>());
+                                     ArrayRef<NamedAttribute>());
 
     if (srcTy.isInteger(1) && isa<IntegerType>(dstTy) && bitExtend)
       return arith::ExtUIOp::create(rewriter, loc, resultTypes, args,
-                                             ArrayRef<NamedAttribute>());
+                                    ArrayRef<NamedAttribute>());
 
     // Unsigned integers need an unrealized cast so that they can be passed
     // to UIToFP.
@@ -575,20 +576,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                   args[0])
               .getResult(0);
       return arith::UIToFPOp::create(rewriter, loc, resultTypes[0],
-                                              unrealizedCast);
+                                     unrealizedCast);
     }
 
     // All other si-to-fp conversions should be handled by SIToFP.
     if (arith::SIToFPOp::areCastCompatible(srcTy, dstTy))
       return arith::SIToFPOp::create(rewriter, loc, resultTypes, args,
-                                              ArrayRef<NamedAttribute>());
+                                     ArrayRef<NamedAttribute>());
 
     // Casting to boolean, floats need to only be checked as not-equal to zero.
     if (isa<FloatType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getFloatAttr(srcTy, 0.0));
+      Value zero = arith::ConstantOp::create(rewriter, loc,
+                                             rewriter.getFloatAttr(srcTy, 0.0));
       return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNE,
-                                            args.front(), zero);
+                                   args.front(), zero);
     }
 
     if (arith::FPToSIOp::areCastCompatible(srcTy, dstTy)) {
@@ -602,36 +603,41 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         // Use cmp + select to replace infinites by int min / int max. Other
         // integral values can be represented in the integer space.
         auto conv = arith::FPToSIOp::create(rewriter, loc, dstTy, rounded);
-        auto posInf = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getFloatAttr(getElementTypeOrSelf(srcTy),
-                                       APFloat::getInf(fltSemantics)));
-        auto negInf = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getFloatAttr(
-                     getElementTypeOrSelf(srcTy),
-                     APFloat::getInf(fltSemantics, /*Negative=*/true)));
-        auto overflow = arith::CmpFOp::create(rewriter,
-            loc, arith::CmpFPredicate::UEQ, rounded, posInf);
-        auto underflow = arith::CmpFOp::create(rewriter,
-            loc, arith::CmpFPredicate::UEQ, rounded, negInf);
-        auto intMin = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getIntegerAttr(
-                     getElementTypeOrSelf(dstTy),
-                     APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())));
-        auto intMax = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getIntegerAttr(
-                     getElementTypeOrSelf(dstTy),
-                     APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
+        auto posInf = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getFloatAttr(getElementTypeOrSelf(srcTy),
+                                  APFloat::getInf(fltSemantics)));
+        auto negInf = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getFloatAttr(
+                getElementTypeOrSelf(srcTy),
+                APFloat::getInf(fltSemantics, /*Negative=*/true)));
+        auto overflow = arith::CmpFOp::create(
+            rewriter, loc, arith::CmpFPredicate::UEQ, rounded, posInf);
+        auto underflow = arith::CmpFOp::create(
+            rewriter, loc, arith::CmpFPredicate::UEQ, rounded, negInf);
+        auto intMin = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getIntegerAttr(
+                getElementTypeOrSelf(dstTy),
+                APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())));
+        auto intMax = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getIntegerAttr(
+                getElementTypeOrSelf(dstTy),
+                APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
         auto maxClamped =
             arith::SelectOp::create(rewriter, loc, overflow, intMax, conv);
         return arith::SelectOp::create(rewriter, loc, underflow, intMin,
-                                                maxClamped);
+                                       maxClamped);
       }
 
-      auto intMinFP = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getFloatAttr(
-                   getElementTypeOrSelf(srcTy),
-                   APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())
-                       .getSExtValue()));
+      auto intMinFP = arith::ConstantOp::create(
+          rewriter, loc,
+          rewriter.getFloatAttr(
+              getElementTypeOrSelf(srcTy),
+              APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())
+                  .getSExtValue()));
 
       // Check whether the mantissa has enough bits to represent int max.
       if (cast<FloatType>(srcTy).getFPMantissaWidth() >=
@@ -640,11 +646,12 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         // consists of a single leading bit. Therefore we can clamp the input
         // in the floating-point domain.
 
-        auto intMaxFP = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getFloatAttr(
-                     getElementTypeOrSelf(srcTy),
-                     APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
-                         .getSExtValue()));
+        auto intMaxFP = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getFloatAttr(
+                getElementTypeOrSelf(srcTy),
+                APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
+                    .getSExtValue()));
 
         Value clamped =
             clampFloatHelper(loc, rounded, intMinFP, intMaxFP, rewriter);
@@ -655,40 +662,42 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       // int min. We can therefore rely on int max + 1 being representable as
       // well because it's just int min with a positive sign. So clamp the min
       // value and compare against that to select the max int value if needed.
-      auto intMaxPlusOneFP = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getFloatAttr(
-                   getElementTypeOrSelf(srcTy),
-                   static_cast<double>(
-                       APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
-                           .getSExtValue()) +
-                       1.0f));
-
-      auto intMax = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIntegerAttr(
-                   getElementTypeOrSelf(dstTy),
-                   APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
+      auto intMaxPlusOneFP = arith::ConstantOp::create(
+          rewriter, loc,
+          rewriter.getFloatAttr(
+              getElementTypeOrSelf(srcTy),
+              static_cast<double>(
+                  APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
+                      .getSExtValue()) +
+                  1.0f));
+
+      auto intMax = arith::ConstantOp::create(
+          rewriter, loc,
+          rewriter.getIntegerAttr(
+              getElementTypeOrSelf(dstTy),
+              APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
       auto minClampedFP =
           arith::MaximumFOp::create(rewriter, loc, rounded, intMinFP);
       auto minClamped =
           arith::FPToSIOp::create(rewriter, loc, dstTy, minClampedFP);
-      auto overflow = arith::CmpFOp::create(rewriter,
-          loc, arith::CmpFPredicate::UGE, rounded, intMaxPlusOneFP);
+      auto overflow = arith::CmpFOp::create(
+          rewriter, loc, arith::CmpFPredicate::UGE, rounded, intMaxPlusOneFP);
       return arith::SelectOp::create(rewriter, loc, overflow, intMax,
-                                              minClamped);
+                                     minClamped);
     }
 
     // Casting to boolean, integers need to only be checked as not-equal to
     // zero.
     if (isa<IntegerType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = arith::ConstantIntOp::create(rewriter,
-          loc, 0, srcTy.getIntOrFloatBitWidth());
+      Value zero = arith::ConstantIntOp::create(rewriter, loc, 0,
+                                                srcTy.getIntOrFloatBitWidth());
       return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
-                                            args.front(), zero);
+                                   args.front(), zero);
     }
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && bitExtend)
       return arith::ExtSIOp::create(rewriter, loc, resultTypes, args,
-                                             ArrayRef<NamedAttribute>());
+                                    ArrayRef<NamedAttribute>());
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && !bitExtend) {
       return arith::TruncIOp::create(rewriter, loc, dstTy, args[0]);
@@ -838,8 +847,8 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
   // Check if broadcast is necessary
   auto one = createIndex(rewriter, loc, indexPool, 1);
   auto runtimeSize = getTensorDim(rewriter, loc, indexPool, operand, dim);
-  auto broadcastNecessary = arith::CmpIOp::create(rewriter,
-      loc, arith::CmpIPredicate::eq, runtimeSize, one);
+  auto broadcastNecessary = arith::CmpIOp::create(
+      rewriter, loc, arith::CmpIPredicate::eq, runtimeSize, one);
 
   // Emit 'then' region of 'scf.if'
   auto emitThenRegion = [&](OpBuilder &opBuilder, Location loc) {
@@ -855,8 +864,8 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                                                     operand, index);
       outputTensorShape.push_back(size);
     }
-    Value outputTensor = tensor::EmptyOp::create(opBuilder,
-        loc, outputTensorShape, rankedTensorType.getElementType());
+    Value outputTensor = tensor::EmptyOp::create(
+        opBuilder, loc, outputTensorShape, rankedTensorType.getElementType());
 
     // Emit 'linalg.generic' op
     auto resultTensor =
@@ -885,7 +894,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
 
   // Emit 'scf.if' op
   auto ifOp = scf::IfOp::create(rewriter, loc, broadcastNecessary,
-                                         emitThenRegion, emitElseRegion);
+                                emitThenRegion, emitElseRegion);
   return ifOp.getResult(0);
 }
 
@@ -930,8 +939,8 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
   if (!resultType) {
     return rewriter.notifyMatchFailure(operation, "failed to convert type");
   }
-  Value outputTensor = tensor::EmptyOp::create(rewriter,
-      loc, targetShape, resultType.getElementType());
+  Value outputTensor = tensor::EmptyOp::create(rewriter, loc, targetShape,
+                                               resultType.getElementType());
 
   // Create affine maps. Input affine maps broadcast static dimensions of size
   // 1. The output affine map is an identity map.
@@ -957,8 +966,8 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
 
   // Emit 'linalg.generic' op
   bool encounteredError = false;
-  auto linalgOp = linalg::GenericOp::create(rewriter,
-      loc, outputTensor.getType(), operands, outputTensor, affineMaps,
+  auto linalgOp = linalg::GenericOp::create(
+      rewriter, loc, outputTensor.getType(), operands, outputTensor, affineMaps,
       getNParallelLoopsAttrs(rank),
       [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
         Value opResult = createLinalgBodyCalculationForElementwiseOp(
@@ -1202,8 +1211,8 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
   }
 
   bool didEncounterError = false;
-  linalg::LinalgOp linalgOp = linalg::ReduceOp::create(rewriter,
-      loc, inputs, outputs, axis,
+  linalg::LinalgOp linalgOp = linalg::ReduceOp::create(
+      rewriter, loc, inputs, outputs, axis,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange blockArgs) {
         std::array<Value, 2> binaryArgs{
             blockArgs[0], isNanIgnoreMode ? blockArgs[2] : blockArgs[1]};
@@ -1219,15 +1228,16 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
           auto oldAllResultsNanFlagValue = blockArgs[3];
 
           // Unordered comparison of NaN against itself will always return true.
-          Value isNaN = arith::CmpFOp::create(nestedBuilder,
-              op->getLoc(), arith::CmpFPredicate::UNO, inputValue, inputValue);
+          Value isNaN = arith::CmpFOp::create(nestedBuilder, op->getLoc(),
+                                              arith::CmpFPredicate::UNO,
+                                              inputValue, inputValue);
           // If we've encountered a NaN, take the non-NaN value.
-          auto selectOp = arith::SelectOp::create(nestedBuilder,
-              op->getLoc(), isNaN, initialValue, result);
+          auto selectOp = arith::SelectOp::create(nestedBuilder, op->getLoc(),
+                                                  isNaN, initialValue, result);
           // Update the flag which keeps track of whether we have seen a non-NaN
           // value.
-          auto newAllResultsNanFlagValue = arith::AndIOp::create(nestedBuilder,
-              op->getLoc(), oldAllResultsNanFlagValue, isNaN);
+          auto newAllResultsNanFlagValue = arith::AndIOp::create(
+              nestedBuilder, op->getLoc(), oldAllResultsNanFlagValue, isNaN);
           resultsToYield.push_back(selectOp);
           resultsToYield.push_back(newAllResultsNanFlagValue);
         } else {
@@ -1401,16 +1411,17 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value multiplierConstant;
     int64_t multiplierArg = 0;
     if (multiplierValues.size() == 1) {
-      multiplierConstant = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32IntegerAttr(multiplierValues.front()));
+      multiplierConstant = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32IntegerAttr(multiplierValues.front()));
     } else {
       SmallVector<AffineExpr, 2> multiplierExprs{
           rewriter.getAffineDimExpr(rank - 1)};
       auto multiplierType =
           RankedTensorType::get({static_cast<int64_t>(multiplierValues.size())},
                                 rewriter.getI32Type());
-      genericInputs.push_back(arith::ConstantOp::create(rewriter,
-          loc, DenseIntElementsAttr::get(multiplierType, multiplierValues)));
+      genericInputs.push_back(arith::ConstantOp::create(
+          rewriter, loc,
+          DenseIntElementsAttr::get(multiplierType, multiplierValues)));
 
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
                                             /*symbolCount=*/0, multiplierExprs,
@@ -1424,16 +1435,16 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value shiftConstant;
     int64_t shiftArg = 0;
     if (shiftValues.size() == 1) {
-      shiftConstant = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI8IntegerAttr(shiftValues.front()));
+      shiftConstant = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI8IntegerAttr(shiftValues.front()));
     } else {
       SmallVector<AffineExpr, 2> shiftExprs = {
           rewriter.getAffineDimExpr(rank - 1)};
       auto shiftType =
           RankedTensorType::get({static_cast<int64_t>(shiftValues.size())},
                                 rewriter.getIntegerType(8));
-      genericInputs.push_back(arith::ConstantOp::create(rewriter,
-          loc, DenseIntElementsAttr::get(shiftType, shiftValues)));
+      genericInputs.push_back(arith::ConstantOp::create(
+          rewriter, loc, DenseIntElementsAttr::get(shiftType, shiftValues)));
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
                                             /*symbolCount=*/0, shiftExprs,
                                             rewriter.getContext()));
@@ -1444,13 +1455,13 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(rank));
 
     // Construct the indexing maps needed for linalg.generic ops.
-    Value emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, outputTy.getShape(), outputTy.getElementType(),
+    Value emptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, outputTy.getShape(), outputTy.getElementType(),
         ArrayRef<Value>({dynDims}));
 
-    auto linalgOp = linalg::GenericOp::create(rewriter,
-        loc, outputTy, genericInputs, ValueRange{emptyTensor}, indexingMaps,
-        getNParallelLoopsAttrs(rank),
+    auto linalgOp = linalg::GenericOp::create(
+        rewriter, loc, outputTy, genericInputs, ValueRange{emptyTensor},
+        indexingMaps, getNParallelLoopsAttrs(rank),
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
             ValueRange blockArgs) {
           Value value = blockArgs[0];
@@ -1466,9 +1477,10 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           const int32_t inBitwidth = valueTy.getIntOrFloatBitWidth();
           // Extend zeropoint for sub-32bits widths.
           const int32_t inAttrBitwidth = inBitwidth > 32 ? inBitwidth : 32;
-          auto inputZp = arith::ConstantOp::create(nestedBuilder,
-              loc, IntegerAttr::get(rewriter.getIntegerType(inAttrBitwidth),
-                                    *maybeIZp));
+          auto inputZp = arith::ConstantOp::create(
+              nestedBuilder, loc,
+              IntegerAttr::get(rewriter.getIntegerType(inAttrBitwidth),
+                               *maybeIZp));
 
           FailureOr<int64_t> maybeOZp = op.getOutputZeroPoint();
           if (failed(maybeOZp)) {
@@ -1482,9 +1494,10 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           unsigned outBitWidth = outIntType.getWidth();
           const int32_t outAttrBitwidth = 32;
           assert(outBitWidth <= 32 && "Unexpected output zeropoint bitwidth");
-          auto outputZp = arith::ConstantOp::create(nestedBuilder,
-              loc, IntegerAttr::get(rewriter.getIntegerType(outAttrBitwidth),
-                                    *maybeOZp));
+          auto outputZp = arith::ConstantOp::create(
+              nestedBuilder, loc,
+              IntegerAttr::get(rewriter.getIntegerType(outAttrBitwidth),
+                               *maybeOZp));
 
           Value multiplier = multiplierConstant ? multiplierConstant
                                                 : blockArgs[multiplierArg];
@@ -1501,20 +1514,20 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           }
           if (valueTy.getIntOrFloatBitWidth() < 32) {
             if (op.getInputUnsigned()) {
-              value = arith::ExtUIOp::create(nestedBuilder,
-                  nestedLoc, nestedBuilder.getI32Type(), value);
+              value = arith::ExtUIOp::create(nestedBuilder, nestedLoc,
+                                             nestedBuilder.getI32Type(), value);
             } else {
-              value = arith::ExtSIOp::create(nestedBuilder,
-                  nestedLoc, nestedBuilder.getI32Type(), value);
+              value = arith::ExtSIOp::create(nestedBuilder, nestedLoc,
+                                             nestedBuilder.getI32Type(), value);
             }
           }
 
           value =
               arith::SubIOp::create(nestedBuilder, nestedLoc, value, inputZp);
 
-          value = tosa::ApplyScaleOp::create(nestedBuilder,
-              loc, nestedBuilder.getI32Type(), value, multiplier, shift,
-              roundingMode);
+          value = tosa::ApplyScaleOp::create(nestedBuilder, loc,
+                                             nestedBuilder.getI32Type(), value,
+                                             multiplier, shift, roundingMode);
 
           // Move to the new zero-point.
           value =
@@ -1530,18 +1543,18 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
             intMax = APInt::getMaxValue(outBitWidth).getZExtValue();
           }
 
-          auto intMinVal = arith::ConstantOp::create(nestedBuilder,
-              loc, nestedBuilder.getI32IntegerAttr(intMin));
-          auto intMaxVal = arith::ConstantOp::create(nestedBuilder,
-              loc, nestedBuilder.getI32IntegerAttr(intMax));
+          auto intMinVal = arith::ConstantOp::create(
+              nestedBuilder, loc, nestedBuilder.getI32IntegerAttr(intMin));
+          auto intMaxVal = arith::ConstantOp::create(
+              nestedBuilder, loc, nestedBuilder.getI32IntegerAttr(intMax));
 
           value = clampIntHelper(nestedLoc, value, intMinVal, intMaxVal,
                                  nestedBuilder, /*isUnsigned=*/false);
 
           if (outIntType.getWidth() < 32) {
-            value = arith::TruncIOp::create(nestedBuilder,
-                nestedLoc, rewriter.getIntegerType(outIntType.getWidth()),
-                value);
+            value = arith::TruncIOp::create(
+                nestedBuilder, nestedLoc,
+                rewriter.getIntegerType(outIntType.getWidth()), value);
           }
 
           if (outIntType.isUnsignedInteger()) {
@@ -1609,7 +1622,7 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
         RankedTensorType::get({inputTy.getDimSize(0), inputTy.getDimSize(3)},
                               inputTy.getElementType());
     Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, input,
-                                                             reassociationMap);
+                                                     reassociationMap);
 
     // Get any dynamic shapes that appear in the input format.
     llvm::SmallVector<Value> outputDynSize;
@@ -1620,31 +1633,32 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
 
     // Generate the elementwise operation for casting scaling the input value.
     auto genericTy = collapseTy.clone(resultTy.getElementType());
-    Value empty = tensor::EmptyOp::create(builder,
-        genericTy.getShape(), resultTy.getElementType(), outputDynSize);
+    Value empty =
+        tensor::EmptyOp::create(builder, genericTy.getShape(),
+                                resultTy.getElementType(), outputDynSize);
     auto genericMap = rewriter.getMultiDimIdentityMap(genericTy.getRank());
     SmallVector<utils::IteratorType> iterators(genericTy.getRank(),
                                                utils::IteratorType::parallel);
 
-    auto generic = linalg::GenericOp::create(builder,
-        genericTy, ValueRange{collapse}, ValueRange{empty},
+    auto generic = linalg::GenericOp::create(
+        builder, genericTy, ValueRange{collapse}, ValueRange{empty},
         ArrayRef<AffineMap>{genericMap, genericMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
           Value value = args[0];
           // This is the quantized case.
           if (inputTy.getElementType() != resultTy.getElementType()) {
-            value =
-                arith::ExtSIOp::create(b, loc, resultTy.getElementType(), value);
+            value = arith::ExtSIOp::create(b, loc, resultTy.getElementType(),
+                                           value);
 
             if (isBilinear && scale[0] != 0) {
-              Value scaleY = arith::ConstantOp::create(b,
-                  loc, b.getI32IntegerAttr(scale[0]));
+              Value scaleY = arith::ConstantOp::create(
+                  b, loc, b.getI32IntegerAttr(scale[0]));
               value = arith::MulIOp::create(b, loc, value, scaleY);
             }
 
             if (isBilinear && scale[2] != 0) {
-              Value scaleX = arith::ConstantOp::create(b,
-                  loc, b.getI32IntegerAttr(scale[2]));
+              Value scaleX = arith::ConstantOp::create(
+                  b, loc, b.getI32IntegerAttr(scale[2]));
               value = arith::MulIOp::create(b, loc, value, scaleX);
             }
           }
@@ -1697,9 +1711,9 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     resizeShape.push_back(channels);
 
     auto resizeTy = resultTy.clone(resizeShape);
-    auto resize = tosa::ResizeOp::create(builder, resizeTy, input, op.getScale(),
-                                                 op.getOffset(), op.getBorder(),
-                                                 op.getMode());
+    auto resize =
+        tosa::ResizeOp::create(builder, resizeTy, input, op.getScale(),
+                               op.getOffset(), op.getBorder(), op.getMode());
 
     // Collapse an unit result dims.
     SmallVector<ReassociationExprs, 4> reassociationMap(2);
@@ -1720,8 +1734,8 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     collapseShape.push_back(channels);
 
     auto collapseTy = resultTy.clone(collapseShape);
-    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, resize,
-                                                             reassociationMap);
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy,
+                                                     resize, reassociationMap);
 
     // Broadcast the collapsed shape to the output result.
     llvm::SmallVector<Value> outputDynSize;
@@ -1732,8 +1746,8 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
 
     SmallVector<utils::IteratorType> iterators(resultTy.getRank(),
                                                utils::IteratorType::parallel);
-    Value empty = tensor::EmptyOp::create(builder,
-        resultTy.getShape(), resultTy.getElementType(), outputDynSize);
+    Value empty = tensor::EmptyOp::create(
+        builder, resultTy.getShape(), resultTy.getElementType(), outputDynSize);
 
     SmallVector<AffineExpr, 4> inputExprs{rewriter.getAffineDimExpr(0)};
     if (inputH != 1)
@@ -1789,10 +1803,10 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
 
     SmallVector<AffineMap, 2> affineMaps = {
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
-    auto emptyTensor = tensor::EmptyOp::create(b, resultTy.getShape(), resultETy,
-                                                 *dynamicDimsOr);
-    auto genericOp = linalg::GenericOp::create(b,
-        resultTy, ValueRange({}), ValueRange{emptyTensor}, affineMaps,
+    auto emptyTensor = tensor::EmptyOp::create(b, resultTy.getShape(),
+                                               resultETy, *dynamicDimsOr);
+    auto genericOp = linalg::GenericOp::create(
+        b, resultTy, ValueRange({}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     Value resize = genericOp.getResult(0);
 
@@ -1808,8 +1822,10 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       Value zeroI32 =
           arith::ConstantOp::create(b, b.getZeroAttr(b.getI32Type()));
       Value zeroFp = arith::ConstantOp::create(b, b.getZeroAttr(floatTy));
-      Value hMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageH - 1));
-      Value wMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageW - 1));
+      Value hMax =
+          arith::ConstantOp::create(b, b.getI32IntegerAttr(imageH - 1));
+      Value wMax =
+          arith::ConstantOp::create(b, b.getI32IntegerAttr(imageW - 1));
 
       Value inY = arith::IndexCastOp::create(b, b.getI32Type(), y);
       Value inX = arith::IndexCastOp::create(b, b.getI32Type(), x);
@@ -1898,12 +1914,13 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
 
           Value pred;
           if (floatingPointMode) {
-            auto h = arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 0.5f));
+            auto h =
+                arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 0.5f));
             pred = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, dval, h);
           } else {
             Value dvalDouble = arith::ShLIOp::create(b, dval, one);
             pred = arith::CmpIOp::create(b, arith::CmpIPredicate::sge,
-                                           dvalDouble, scale);
+                                         dvalDouble, scale);
           }
 
           auto offset = arith::SelectOp::create(b, pred, one, zeroI32);
@@ -1915,8 +1932,8 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         iy = getNearestIndexAndClamp(iy, dy, yScaleN, hMax, imageH, b);
         ix = getNearestIndexAndClamp(ix, dx, xScaleN, wMax, imageW, b);
 
-        Value result = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, iy, ix, channel});
+        Value result = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, iy, ix, channel});
 
         linalg::YieldOp::create(b, result);
       } else {
@@ -1946,14 +1963,14 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         getClampedIdxs(y0, y1, imageH, iy, hMax, b);
         getClampedIdxs(x0, x1, imageW, ix, wMax, b);
 
-        Value y0x0 = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, y0, x0, channel});
-        Value y0x1 = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, y0, x1, channel});
-        Value y1x0 = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, y1, x0, channel});
-        Value y1x1 = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, y1, x1, channel});
+        Value y0x0 = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, y0, x0, channel});
+        Value y0x1 = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, y0, x1, channel});
+        Value y1x0 = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, y1, x0, channel});
+        Value y1x1 = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, y1, x1, channel});
 
         if (floatingPointMode) {
           auto oneVal =
@@ -2100,16 +2117,16 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
               auto sizeMinusOne =
                   arith::SubIOp::create(rewriter, nestedLoc, axisDimSize, one);
               index = arith::SubIOp::create(rewriter, nestedLoc, sizeMinusOne,
-                                                     index);
+                                            index);
             }
 
             indices.push_back(index);
           }
 
-          auto extract = tensor::ExtractOp::create(nestedBuilder,
-              nestedLoc, input, indices);
+          auto extract = tensor::ExtractOp::create(nestedBuilder, nestedLoc,
+                                                   input, indices);
           linalg::YieldOp::create(nestedBuilder, op.getLoc(),
-                                                extract.getResult());
+                                  extract.getResult());
         });
     return success();
   }
@@ -2152,8 +2169,8 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
       }
     }
 
-    auto emptyTensor = tensor::EmptyOp::create(rewriter,
-        op.getLoc(), genericShape, elementTy, dynDims);
+    auto emptyTensor = tensor::EmptyOp::create(
+        rewriter, op.getLoc(), genericShape, elementTy, dynDims);
 
     // We needs to map the input shape to the non-broadcasted dimensions.
     SmallVector<AffineExpr, 4> dimExprs;
@@ -2168,8 +2185,8 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<AffineMap, 2> affineMaps = {
         readAffineMap, rewriter.getMultiDimIdentityMap(genericShape.size())};
 
-    auto genericOp = linalg::GenericOp::create(rewriter,
-        loc, RankedTensorType::get(genericShape, elementTy), input,
+    auto genericOp = linalg::GenericOp::create(
+        rewriter, loc, RankedTensorType::get(genericShape, elementTy), input,
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(genericShape.size()),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -2229,8 +2246,8 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
                               .create<tensor::EmptyOp>(loc, resultTy.getShape(),
                                                        outElementTy, dynDims)
                               .getResult();
-    auto fillValueIdx = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerAttr(outElementTy, 0));
+    auto fillValueIdx = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getIntegerAttr(outElementTy, 0));
     auto filledTensorIdx =
         rewriter
             .create<linalg::FillOp>(loc, ValueRange{fillValueIdx},
@@ -2274,8 +2291,8 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     bool didEncounterError = false;
     auto maps = AffineMap::inferFromExprList({srcExprs, dstExprs, dstExprs},
                                              rewriter.getContext());
-    auto linalgOp = linalg::GenericOp::create(rewriter,
-        loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
+    auto linalgOp = linalg::GenericOp::create(
+        rewriter, loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
         ValueRange({filledTensorIdx, filledTensorMax}), maps, iteratorTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
             ValueRange blockArgs) {
@@ -2283,8 +2300,8 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           auto oldIndex = blockArgs[1];
           auto oldValue = blockArgs[2];
 
-          Value newIndex = arith::IndexCastOp::create(rewriter,
-              nestedLoc, oldIndex.getType(),
+          Value newIndex = arith::IndexCastOp::create(
+              rewriter, nestedLoc, oldIndex.getType(),
               linalg::IndexOp::create(rewriter, loc, axis));
 
           Value predicate;
@@ -2292,33 +2309,37 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
             if (argmaxOp.getNanMode() == "IGNORE") {
               // Only update index & max value for non NaN values. If all
               // values are NaNs, the initial index will be return which is 0.
-              predicate = arith::CmpFOp::create(rewriter,
-                  nestedLoc, arith::CmpFPredicate::OGT, newValue, oldValue);
+              predicate = arith::CmpFOp::create(rewriter, nestedLoc,
+                                                arith::CmpFPredicate::OGT,
+                                                newValue, oldValue);
             } else {
               // Update max value if either of the following is true:
               // - new value is bigger
               // - cur max is not NaN and new value is NaN
-              Value gt = arith::CmpFOp::create(rewriter,
-                  nestedLoc, arith::CmpFPredicate::UGT, newValue, oldValue);
-              Value oldNonNaN = arith::CmpFOp::create(rewriter,
-                  nestedLoc, arith::CmpFPredicate::ORD, oldValue, oldValue);
-              predicate = arith::AndIOp::create(rewriter,
-                  nestedLoc, rewriter.getI1Type(), gt, oldNonNaN);
+              Value gt = arith::CmpFOp::create(rewriter, nestedLoc,
+                                               arith::CmpFPredicate::UGT,
+                                               newValue, oldValue);
+              Value oldNonNaN = arith::CmpFOp::create(rewriter, nestedLoc,
+                                                      arith::CmpFPredicate::ORD,
+                                                      oldValue, oldValue);
+              predicate = arith::AndIOp::create(
+                  rewriter, nestedLoc, rewriter.getI1Type(), gt, oldNonNaN);
             }
           } else if (isa<IntegerType>(inElementTy)) {
-            predicate = arith::CmpIOp::create(rewriter,
-                nestedLoc, arith::CmpIPredicate::sgt, newValue, oldValue);
+            predicate = arith::CmpIOp::create(rewriter, nestedLoc,
+                                              arith::CmpIPredicate::sgt,
+                                              newValue, oldValue);
           } else {
             didEncounterError = true;
             return;
           }
 
-          auto resultMax = arith::SelectOp::create(rewriter,
-              nestedLoc, predicate, newValue, oldValue);
-          auto resultIndex = arith::SelectOp::create(rewriter,
-              nestedLoc, predicate, newIndex, oldIndex);
-          linalg::YieldOp::create(nestedBuilder,
-              nestedLoc, ValueRange({resultIndex, resultMax}));
+          auto resultMax = arith::SelectOp::create(
+              rewriter, nestedLoc, predicate, newValue, oldValue);
+          auto resultIndex = arith::SelectOp::create(
+              rewriter, nestedLoc, predicate, newIndex, oldIndex);
+          linalg::YieldOp::create(nestedBuilder, nestedLoc,
+                                  ValueRange({resultIndex, resultMax}));
         });
 
     if (didEncounterError)
@@ -2363,18 +2384,18 @@ class GatherConverter : public OpConversionPattern<tosa::GatherOp> {
             rewriter.getContext()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = linalg::GenericOp::create(rewriter,
-        loc, ArrayRef<Type>({resultTy}), ValueRange{indices},
+    auto genericOp = linalg::GenericOp::create(
+        rewriter, loc, ArrayRef<Type>({resultTy}), ValueRange{indices},
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
           auto indexValue = args[0];
           auto index0 = linalg::IndexOp::create(rewriter, loc, 0);
-          Value index1 = arith::IndexCastOp::create(rewriter,
-              loc, rewriter.getIndexType(), indexValue);
+          Value index1 = arith::IndexCastOp::create(
+              rewriter, loc, rewriter.getIndexType(), indexValue);
           auto index2 = linalg::IndexOp::create(rewriter, loc, 2);
-          Value extract = tensor::ExtractOp::create(rewriter,
-              loc, input, ValueRange{index0, index1, index2});
+          Value extract = tensor::ExtractOp::create(
+              rewriter, loc, input, ValueRange{index0, index1, index2});
           linalg::YieldOp::create(rewriter, loc, extract);
         });
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -2437,9 +2458,9 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
         rewriter.getMultiDimIdentityMap(resultTy.getRank()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = linalg::GenericOp::create(rewriter,
-        loc, resultTy, ValueRange({input}), ValueRange{emptyTensor}, affineMaps,
-        getNParallelLoopsAttrs(resultTy.getRank()));
+    auto genericOp = linalg::GenericOp::create(
+        rewriter, loc, resultTy, ValueRange({input}), ValueRange{emptyTensor},
+        affineMaps, getNParallelLoopsAttrs(resultTy.getRank()));
     rewriter.replaceOp(op, genericOp.getResult(0));
 
     {
@@ -2452,11 +2473,11 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
       rewriter.setInsertionPointToStart(block);
       if (inputElementTy.isInteger(8) && tableElementTy.isInteger(8) &&
           resultElementTy.isInteger(8)) {
-        Value index = arith::IndexCastOp::create(rewriter,
-            loc, rewriter.getIndexType(), inputValue);
+        Value index = arith::IndexCastOp::create(
+            rewriter, loc, rewriter.getIndexType(), inputValue);
         Value offset = arith::ConstantIndexOp::create(rewriter, loc, 128);
         index = arith::AddIOp::create(rewriter, loc, rewriter.getIndexType(),
-                                               index, offset);
+                                      index, offset);
         Value extract =
             tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
         linalg::YieldOp::create(rewriter, loc, extract);
@@ -2465,17 +2486,17 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
 
       if (inputElementTy.isInteger(16) && tableElementTy.isInteger(16) &&
           resultElementTy.isInteger(32)) {
-        Value extend = arith::ExtSIOp::create(rewriter,
-            loc, rewriter.getI32Type(), inputValue);
-
-        auto offset = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getI32IntegerAttr(32768));
-        auto seven = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getI32IntegerAttr(7));
-        auto one = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getI32IntegerAttr(1));
-        auto b1111111 = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getI32IntegerAttr(127));
+        Value extend = arith::ExtSIOp::create(
+            rewriter, loc, rewriter.getI32Type(), inputValue);
+
+        auto offset = arith::ConstantOp::create(
+            rewriter, loc, rewriter.getI32IntegerAttr(32768));
+        auto seven = arith::ConstantOp::create(rewriter, loc,
+                                               rewriter.getI32IntegerAttr(7));
+        auto one = arith::ConstantOp::create(rewriter, loc,
+                                             rewriter.getI32IntegerAttr(1));
+        auto b1111111 = arith::ConstantOp::create(
+            rewriter, loc, rewriter.getI32IntegerAttr(127));
 
         // Compute the index and fractional part from the input value:
         // value = value + 32768
@@ -2491,15 +2512,15 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
         // next = (int32_t) table[index + 1];
         Value indexPlusOne = arith::AddIOp::create(rewriter, loc, index, one);
 
-        index = arith::IndexCastOp::create(rewriter,
-            loc, rewriter.getIndexType(), index);
-        indexPlusOne = arith::IndexCastOp::create(rewriter,
-            loc, rewriter.getIndexType(), indexPlusOne);
+        index = arith::IndexCastOp::create(rewriter, loc,
+                                           rewriter.getIndexType(), index);
+        indexPlusOne = arith::IndexCastOp::create(
+            rewriter, loc, rewriter.getIndexType(), indexPlusOne);
 
         Value base =
             tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
-        Value next = tensor::ExtractOp::create(rewriter,
-            loc, table, ValueRange{indexPlusOne});
+        Value next = tensor::ExtractOp::create(rewriter, loc, table,
+                                               ValueRange{indexPlusOne});
 
         base =
             arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), base);
@@ -2574,8 +2595,8 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static Value castIndexToFloat(OpBuilder &builder, Location loc,
                                 FloatType type, Value value) {
-    auto integerVal = arith::IndexCastUIOp::create(builder,
-        loc,
+    auto integerVal = arith::IndexCastUIOp::create(
+        builder, loc,
         type.getIntOrFloatBitWidth() > 32 ? builder.getI64Type()
                                           : builder.getI32Type(),
         value);
@@ -2683,8 +2704,10 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
-      auto outImag = arith::SubFOp::create(builder, loc, sumImag, imagComponent);
+      auto outReal =
+          arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag =
+          arith::SubFOp::create(builder, loc, sumImag, imagComponent);
 
       linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
@@ -2798,10 +2821,10 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
       auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       if (inverse.getValue()) {
-        angle = arith::MulFOp::create(builder,
-            loc, angle,
-            arith::ConstantOp::create(rewriter,
-                loc, rewriter.getFloatAttr(real_el_ty, -1.0)));
+        angle = arith::MulFOp::create(
+            builder, loc, angle,
+            arith::ConstantOp::create(rewriter, loc,
+                                      rewriter.getFloatAttr(real_el_ty, -1.0)));
       }
 
       // realComponent = val_real * cos(a) + val_imag * sin(a);
@@ -2820,8 +2843,10 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
-      auto outImag = arith::AddFOp::create(builder, loc, sumImag, imagComponent);
+      auto outReal =
+          arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag =
+          arith::AddFOp::create(builder, loc, sumImag, imagComponent);
 
       linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index bfe43e8dcc274..5d66c3900030f 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -54,9 +54,9 @@ static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
 
   Value padValue = arith::ConstantOp::create(rewriter, loc, padAttr);
 
-  return tensor::PadOp::create(rewriter,
-      loc, RankedTensorType::get(paddedShape, inputETy), input, lowIndices,
-      highIndices, padValue);
+  return tensor::PadOp::create(rewriter, loc,
+                               RankedTensorType::get(paddedShape, inputETy),
+                               input, lowIndices, highIndices, padValue);
 }
 
 static mlir::Value
@@ -160,8 +160,8 @@ static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim,
                                           int64_t dilationAttr,
                                           OpBuilder &rewriter) {
   ImplicitLocOpBuilder builder(loc, rewriter);
-  auto one = arith::ConstantOp::create(rewriter,
-      loc, IntegerAttr::get(inputDim.getType(), 1));
+  auto one = arith::ConstantOp::create(rewriter, loc,
+                                       IntegerAttr::get(inputDim.getType(), 1));
   Value padBefore = reifyConstantDim(padBeforeAttr, builder);
   Value paddedBefore = arith::AddIOp::create(builder, inputDim, padBefore);
   Value padAfter = reifyConstantDim(padAfterAttr, builder);
@@ -351,7 +351,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
         Type newWeightTy =
             RankedTensorType::get(newWeightShape, weightTy.getElementType());
         weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
-                                                    weightPermAttr);
+                                           weightPermAttr);
       }
     }
 
@@ -373,7 +373,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       Type newWeightTy =
           RankedTensorType::get(newWeightShape, weightTy.getElementType());
       weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
-                                                  weightPermAttr);
+                                         weightPermAttr);
     }
 
     // Extract the attributes for convolution.
@@ -384,8 +384,8 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     auto strideAttr = rewriter.getI64TensorAttr(stride);
     auto dilationAttr = rewriter.getI64TensorAttr(dilation);
 
-    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultTy.getShape(), accETy, filteredDims);
+    Value biasEmptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, resultTy.getShape(), accETy, filteredDims);
 
     Value broadcastBias =
         linalgBroadcastAndMaybeExt(rewriter, loc, bias, biasEmptyTensor);
@@ -526,16 +526,16 @@ class DepthwiseConvConverter
                               accETy);
 
     auto resultZeroAttr = rewriter.getZeroAttr(accETy);
-    Value emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, linalgConvTy.getShape(), accETy, filteredDims);
+    Value emptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, linalgConvTy.getShape(), accETy, filteredDims);
     Value zero = arith::ConstantOp::create(rewriter, loc, resultZeroAttr);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
                            .result();
 
-    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultTy.getShape(), resultETy, filteredDims);
+    Value biasEmptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, resultTy.getShape(), resultETy, filteredDims);
 
     // Broadcast the initial value to the output tensor before convolving.
     SmallVector<AffineMap, 4> indexingMaps;
@@ -553,16 +553,16 @@ class DepthwiseConvConverter
       // We may need to truncate back to the result type if the accumulator was
       // wider than the result.
       if (accETy != resultETy)
-        conv = tosa::CastOp::create(rewriter,
-            loc,
+        conv = tosa::CastOp::create(
+            rewriter, loc,
             RankedTensorType::get(cast<ShapedType>(conv.getType()).getShape(),
                                   resultETy),
             conv);
 
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
-          loc, resultTy, conv, reassociationMap);
+      Value convReshape = tensor::CollapseShapeOp::create(
+          rewriter, loc, resultTy, conv, reassociationMap);
 
       Value result =
           rewriter
@@ -575,10 +575,10 @@ class DepthwiseConvConverter
                     Value added;
                     if (llvm::isa<FloatType>(inputETy))
                       added = arith::AddFOp::create(nestedBuilder, loc, args[0],
-                                                                  args[1]);
+                                                    args[1]);
                     else
                       added = arith::AddIOp::create(nestedBuilder, loc, args[0],
-                                                                  args[1]);
+                                                    args[1]);
                     linalg::YieldOp::create(nestedBuilder, nestedLoc, added);
                   })
               .getResult(0);
@@ -596,8 +596,8 @@ class DepthwiseConvConverter
               .getResult(0);
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
-          loc, resultTy, conv, reassociationMap);
+      Value convReshape = tensor::CollapseShapeOp::create(
+          rewriter, loc, resultTy, conv, reassociationMap);
       Value result = linalgIntBroadcastExtSIAdd(
           rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
       rewriter.replaceOp(op, result);
@@ -636,8 +636,9 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
 
     auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
     Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
-    auto emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
+    auto emptyTensor =
+        tensor::EmptyOp::create(rewriter, loc, outputTy.getShape(),
+                                outputTy.getElementType(), filteredDims);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
@@ -670,10 +671,10 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
       return success();
     }
 
-    auto aZp = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getI32IntegerAttr(aZpVal));
-    auto bZp = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getI32IntegerAttr(bZpVal));
+    auto aZp = arith::ConstantOp::create(rewriter, loc,
+                                         rewriter.getI32IntegerAttr(aZpVal));
+    auto bZp = arith::ConstantOp::create(rewriter, loc,
+                                         rewriter.getI32IntegerAttr(bZpVal));
     rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
         op, TypeRange{op.getType()},
         ValueRange{adaptor.getA(), adaptor.getB(), aZp, bZp}, zeroTensor);
@@ -785,8 +786,9 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
+    Value emptyTensor =
+        tensor::EmptyOp::create(rewriter, loc, resultTy.getShape(),
+                                resultTy.getElementType(), dynamicDims);
 
     Value filledEmptyTensor =
         linalg::FillOp::create(rewriter, loc, initialValue, emptyTensor)
@@ -802,8 +804,8 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       return llvm::success();
     }
 
-    auto resultOp = linalg::PoolingNhwcMaxOp::create(rewriter,
-        op->getLoc(), ArrayRef<Type>{resultTy},
+    auto resultOp = linalg::PoolingNhwcMaxOp::create(
+        rewriter, op->getLoc(), ArrayRef<Type>{resultTy},
         ValueRange{paddedInput, fakeWindowDims}, filledEmptyTensor, strideAttr,
         dilationAttr);
 
@@ -903,8 +905,8 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value poolEmptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, accTy.getShape(), accETy, dynamicDims);
+    Value poolEmptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, accTy.getShape(), accETy, dynamicDims);
 
     Value filledEmptyTensor =
         rewriter
@@ -932,12 +934,12 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     iH = arith::SubIOp::create(rewriter, loc, iH, one);
     iW = arith::SubIOp::create(rewriter, loc, iW, one);
 
-    Value genericEmptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultTy.getShape(), resultETy, dynamicDims);
+    Value genericEmptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, resultTy.getShape(), resultETy, dynamicDims);
 
     auto affineMap = rewriter.getMultiDimIdentityMap(resultTy.getRank());
-    auto genericOp = linalg::GenericOp::create(rewriter,
-        loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
+    auto genericOp = linalg::GenericOp::create(
+        rewriter, loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
         ValueRange{genericEmptyTensor},
         ArrayRef<AffineMap>({affineMap, affineMap}),
         getNParallelLoopsAttrs(resultTy.getRank()),
@@ -981,8 +983,8 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
           Value kW3 = coverageFn(2, iW);
 
           // Compute the total number of elements and normalize.
-          auto count = arith::IndexCastOp::create(rewriter,
-              loc, rewriter.getI32Type(),
+          auto count = arith::IndexCastOp::create(
+              rewriter, loc, rewriter.getI32Type(),
               arith::MulIOp::create(rewriter, loc, kH3, kW3));
 
           // Divide by the number of summed values. For floats this is just
@@ -1002,8 +1004,8 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // If we have quantization information we need to apply an offset
             // for the input zp value.
             if (inputZpVal != 0) {
-              auto inputZp = arith::ConstantOp::create(rewriter,
-                  loc, b.getIntegerAttr(accETy, inputZpVal));
+              auto inputZp = arith::ConstantOp::create(
+                  rewriter, loc, b.getIntegerAttr(accETy, inputZpVal));
               Value offset =
                   arith::MulIOp::create(rewriter, loc, accETy, count, inputZp);
               poolVal =
@@ -1011,10 +1013,10 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             }
 
             // Compute: k = 32 - count_leading_zeros(value - 1)
-            Value one32 = arith::ConstantOp::create(rewriter,
-                loc, rewriter.getI32IntegerAttr(1));
-            Value thirtyTwo32 = arith::ConstantOp::create(rewriter,
-                loc, rewriter.getI32IntegerAttr(32));
+            Value one32 = arith::ConstantOp::create(
+                rewriter, loc, rewriter.getI32IntegerAttr(1));
+            Value thirtyTwo32 = arith::ConstantOp::create(
+                rewriter, loc, rewriter.getI32IntegerAttr(32));
 
             Value countSubOne =
                 arith::SubIOp::create(rewriter, loc, count, one32);
@@ -1026,24 +1028,24 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // Compute: numerator = ((1 << 30) + 1) << k
             Value k64 =
                 arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), k);
-            Value thirtyShiftPlusOne = arith::ConstantOp::create(rewriter,
-                loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
+            Value thirtyShiftPlusOne = arith::ConstantOp::create(
+                rewriter, loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
             Value numerator =
                 arith::ShLIOp::create(rewriter, loc, thirtyShiftPlusOne, k64);
 
             // Compute: scale.multiplier = numerator / value;
-            Value count64 = arith::ExtUIOp::create(rewriter,
-                loc, rewriter.getI64Type(), count);
+            Value count64 = arith::ExtUIOp::create(
+                rewriter, loc, rewriter.getI64Type(), count);
             Value multiplier =
                 arith::DivUIOp::create(rewriter, loc, numerator, count64);
-            multiplier = arith::TruncIOp::create(rewriter,
-                loc, rewriter.getI32Type(), multiplier);
+            multiplier = arith::TruncIOp::create(
+                rewriter, loc, rewriter.getI32Type(), multiplier);
 
             // Compute: scale.shift = 30 + k
             Value k8 =
                 arith::TruncIOp::create(rewriter, loc, rewriter.getI8Type(), k);
-            Value thirty8 = arith::ConstantOp::create(rewriter,
-                loc, rewriter.getI8IntegerAttr(30));
+            Value thirty8 = arith::ConstantOp::create(
+                rewriter, loc, rewriter.getI8IntegerAttr(30));
             Value shift = arith::AddIOp::create(rewriter, loc, k8, thirty8);
 
             auto scaled =
@@ -1056,8 +1058,9 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // If we have quantization information we need to apply output
             // zeropoint.
             if (outputZpVal != 0) {
-              auto outputZp = arith::ConstantOp::create(rewriter,
-                  loc, b.getIntegerAttr(scaled.getType(), outputZpVal));
+              auto outputZp = arith::ConstantOp::create(
+                  rewriter, loc,
+                  b.getIntegerAttr(scaled.getType(), outputZpVal));
               scaled = arith::AddIOp::create(rewriter, loc, scaled, outputZp)
                            .getResult();
             }
@@ -1065,11 +1068,11 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // Apply Clip.
             int64_t outBitwidth = resultETy.getIntOrFloatBitWidth();
 
-            auto min = arith::ConstantIntOp::create(rewriter,
-                loc, accETy,
+            auto min = arith::ConstantIntOp::create(
+                rewriter, loc, accETy,
                 APInt::getSignedMinValue(outBitwidth).getSExtValue());
-            auto max = arith::ConstantIntOp::create(rewriter,
-                loc, accETy,
+            auto max = arith::ConstantIntOp::create(
+                rewriter, loc, accETy,
                 APInt::getSignedMaxValue(outBitwidth).getSExtValue());
             auto clamp = clampIntHelper(loc, scaled, min, max, rewriter,
                                         /*isUnsigned=*/false);
@@ -1107,8 +1110,9 @@ class TransposeConverter : public OpRewritePattern<tosa::TransposeOp> {
     auto permutedSizes =
         applyTOSAPermutation<OpFoldResult>(inputSizes, constantPerms);
 
-    auto permutedInit = tensor::EmptyOp::create(rewriter,
-        loc, permutedSizes, op.getInput1().getType().getElementType());
+    auto permutedInit =
+        tensor::EmptyOp::create(rewriter, loc, permutedSizes,
+                                op.getInput1().getType().getElementType());
     rewriter.replaceOpWithNewOp<linalg::TransposeOp>(
         op, op.getInput1(), permutedInit,
         llvm::to_vector(llvm::map_range(
diff --git a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
index cd36aadce410a..b83f5ec9b0283 100644
--- a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
+++ b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
@@ -27,8 +27,8 @@ class VariableOpConverter : public OpRewritePattern<tosa::VariableOp> {
   LogicalResult matchAndRewrite(tosa::VariableOp op,
                                 PatternRewriter &rewriter) const final {
     auto variableType = tosa::getVariableType(op);
-    auto newVariable = mlir::ml_program::GlobalOp::create(rewriter,
-        op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
+    auto newVariable = mlir::ml_program::GlobalOp::create(
+        rewriter, op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
         op.getInitialValueAttr(), /*sym_visibility=*/nullptr);
     newVariable.setPrivate();
     rewriter.replaceOp(op, newVariable);
@@ -45,8 +45,8 @@ class VariableWriteOpConverter
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableWrite = ml_program::GlobalStoreOp::create(rewriter,
-        op.getLoc(), globalSymbolRef, op.getInput1());
+    auto newVariableWrite = ml_program::GlobalStoreOp::create(
+        rewriter, op.getLoc(), globalSymbolRef, op.getInput1());
     rewriter.replaceOp(op, newVariableWrite);
     return success();
   }
@@ -60,8 +60,8 @@ class VariableReadOpConverter : public OpRewritePattern<tosa::VariableReadOp> {
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableRead = ml_program::GlobalLoadOp::create(rewriter,
-        op.getLoc(), op.getType(), globalSymbolRef);
+    auto newVariableRead = ml_program::GlobalLoadOp::create(
+        rewriter, op.getLoc(), op.getType(), globalSymbolRef);
     rewriter.replaceOp(op, newVariableRead);
 
     return success();
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index 9188b8d1368da..aa6b4164e9876 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -46,10 +46,10 @@ static void inlineWhileCase(Region &srcRegion, Region &dstRegion,
   auto yield = cast<YieldOp>(headBlock->getTerminator());
   rewriter.setInsertionPoint(yield);
   if (isCond) {
-    auto condition =
-        tensor::ExtractOp::create(rewriter, yield.getLoc(), yield.getOperand(0));
+    auto condition = tensor::ExtractOp::create(rewriter, yield.getLoc(),
+                                               yield.getOperand(0));
     scf::ConditionOp::create(rewriter, yield.getLoc(), condition,
-                                      headBlock->getArguments());
+                             headBlock->getArguments());
   } else {
     rewriter.setInsertionPoint(yield);
     scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
@@ -68,7 +68,7 @@ class IfOpConverter : public OpRewritePattern<tosa::IfOp> {
     auto condition =
         tensor::ExtractOp::create(rewriter, op.getLoc(), op.getCondition());
     auto newIf = scf::IfOp::create(rewriter, op.getLoc(), op.getResultTypes(),
-                                            condition, true);
+                                   condition, true);
 
     inlineIfCase(op.getThenGraph(), newIf.getThenRegion(), op.getInputList(),
                  rewriter);
@@ -120,8 +120,8 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
 
       // Read the index and cast it to index type
       auto index = tensor::ExtractOp::create(builder, loc, indices, ivs);
-      auto castIndex = arith::IndexCastOp::create(builder,
-          loc, builder.getIndexType(), index);
+      auto castIndex = arith::IndexCastOp::create(
+          builder, loc, builder.getIndexType(), index);
 
       // Offset, sizes, and strides for the input tensor
       auto inputOffset = llvm::to_vector(ivs);
@@ -130,13 +130,13 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       llvm::SmallVector<Value> sizes = {one, one, dimC};
       llvm::SmallVector<Value> strides = {one, one, one};
 
-      auto slice = tensor::ExtractSliceOp::create(builder,
-          loc, input, inputOffset, sizes, strides);
+      auto slice = tensor::ExtractSliceOp::create(builder, loc, input,
+                                                  inputOffset, sizes, strides);
 
       // Insert the slice into the output accumulator tensor.
       llvm::SmallVector<Value> outputOffset = {n, castIndex, zero};
-      auto updated = tensor::InsertSliceOp::create(builder,
-          loc, slice, args[0], outputOffset, sizes, strides);
+      auto updated = tensor::InsertSliceOp::create(
+          builder, loc, slice, args[0], outputOffset, sizes, strides);
 
       return {updated};
     };
@@ -155,8 +155,8 @@ class WhileOpConverter : public OpRewritePattern<tosa::WhileOp> {
 
   LogicalResult matchAndRewrite(tosa::WhileOp op,
                                 PatternRewriter &rewriter) const final {
-    auto newWhile = scf::WhileOp::create(rewriter,
-        op.getLoc(), op.getResultTypes(), op.getInputList());
+    auto newWhile = scf::WhileOp::create(
+        rewriter, op.getLoc(), op.getResultTypes(), op.getInputList());
     rewriter.createBlock(&newWhile.getBefore());
     rewriter.createBlock(&newWhile.getAfter());
 
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index a7219d423a77a..2945ae3b49f1f 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -309,14 +309,14 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
         continue;
 
       auto dim = tensor::DimOp::create(rewriter, loc, input, index);
-      auto offset = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexAttr(sliceStarts[index]));
+      auto offset = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIndexAttr(sliceStarts[index]));
       dynSizes.push_back(arith::SubIOp::create(rewriter, loc, dim, offset));
     }
 
-    auto newSliceOp = tensor::ExtractSliceOp::create(rewriter,
-        sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}), dynSizes,
-        ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
+    auto newSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}),
+        dynSizes, ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
         rewriter.getDenseI64ArrayAttr(sizes),
         rewriter.getDenseI64ArrayAttr(strides));
 
@@ -375,16 +375,16 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
     highValues.reserve(rank);
 
     for (int i = 0; i < rank; i++) {
-      Value lowVal = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexAttr(paddingVals[2 * i]));
-      Value highVal = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexAttr(paddingVals[2 * i + 1]));
+      Value lowVal = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIndexAttr(paddingVals[2 * i]));
+      Value highVal = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIndexAttr(paddingVals[2 * i + 1]));
       lowValues.push_back(lowVal);
       highValues.push_back(highVal);
     }
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, padOp.getType(), input, lowValues, highValues, padConstant);
+    auto newPadOp = tensor::PadOp::create(rewriter, loc, padOp.getType(), input,
+                                          lowValues, highValues, padConstant);
 
     rewriter.replaceOp(padOp, newPadOp.getResult());
     return success();
@@ -439,8 +439,9 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
       }
     }
 
-    Value result = tensor::EmptyOp::create(rewriter,
-        loc, resultType.getShape(), resultType.getElementType(), dynDims);
+    Value result =
+        tensor::EmptyOp::create(rewriter, loc, resultType.getShape(),
+                                resultType.getElementType(), dynDims);
 
     for (auto [arg, offset] : llvm::zip(adaptor.getOperands(), axisOffsets)) {
       auto sizes = tensor::getMixedSizes(rewriter, op.getLoc(), arg);
diff --git a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
index 9e4eb347b3689..125ea1eb60ed6 100644
--- a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
+++ b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
@@ -226,8 +226,8 @@ struct BroadcastOpToArmSMELowering
         (srcVectorType && (srcVectorType.getRank() == 0))) {
       // Broadcast scalar or 0-d vector to 1-d vector.
       VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-      broadcastOp1D = vector::BroadcastOp::create(rewriter,
-          loc, tileSliceType, broadcastOp.getSource());
+      broadcastOp1D = vector::BroadcastOp::create(rewriter, loc, tileSliceType,
+                                                  broadcastOp.getSource());
     } else if (srcVectorType && (srcVectorType.getRank() == 1))
       // Value to broadcast is already a 1-d vector, nothing to do.
       broadcastOp1D = broadcastOp.getSource();
@@ -240,8 +240,8 @@ struct BroadcastOpToArmSMELowering
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to broadcast the value
       // to each tile slice.
-      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
-          loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
+      auto nextTile = arm_sme::InsertTileSliceOp::create(
+          b, loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
 
@@ -292,15 +292,15 @@ struct SplatOpToArmSMELowering : public OpRewritePattern<vector::SplatOp> {
 
     // First, broadcast the scalar to a 1-d vector.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    Value broadcastOp1D = vector::BroadcastOp::create(rewriter,
-        loc, tileSliceType, splatOp.getInput());
+    Value broadcastOp1D = vector::BroadcastOp::create(
+        rewriter, loc, tileSliceType, splatOp.getInput());
 
     auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
-      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
-          loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
+      auto nextTile = arm_sme::InsertTileSliceOp::create(
+          b, loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
 
@@ -371,8 +371,8 @@ struct TransposeOpToArmSMELowering
     // Allocate buffer to store input tile to.
     Value vscale =
         vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
-    Value minTileSlices = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIndexAttr(tileType.getDimSize(0)));
+    Value minTileSlices = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getIndexAttr(tileType.getDimSize(0)));
     Value c0 =
         arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(0));
     Value numTileSlices =
@@ -380,12 +380,12 @@ struct TransposeOpToArmSMELowering
     auto bufferType =
         MemRefType::get({ShapedType::kDynamic, ShapedType::kDynamic},
                         tileType.getElementType());
-    auto buffer = memref::AllocaOp::create(rewriter,
-        loc, bufferType, ValueRange{numTileSlices, numTileSlices});
+    auto buffer = memref::AllocaOp::create(
+        rewriter, loc, bufferType, ValueRange{numTileSlices, numTileSlices});
 
     // Store input tile.
-    auto tileStoreOp = arm_sme::TileStoreOp::create(rewriter,
-        loc, input, buffer, ValueRange{c0, c0});
+    auto tileStoreOp = arm_sme::TileStoreOp::create(rewriter, loc, input,
+                                                    buffer, ValueRange{c0, c0});
 
     // Reload input tile vertically.
     rewriter.replaceOpWithNewOp<arm_sme::TileLoadOp>(
@@ -488,10 +488,10 @@ struct VectorOuterProductToArmSMELowering
     Value rhsMaskDim = createMaskOp.getOperand(1);
 
     VectorType operandMaskType = VectorType::Builder(maskType).dropDim(0);
-    Value lhsMask =
-        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, lhsMaskDim);
-    Value rhsMask =
-        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, rhsMaskDim);
+    Value lhsMask = vector::CreateMaskOp::create(rewriter, loc, operandMaskType,
+                                                 lhsMaskDim);
+    Value rhsMask = vector::CreateMaskOp::create(rewriter, loc, operandMaskType,
+                                                 rhsMaskDim);
 
     return std::make_pair(lhsMask, rhsMask);
   }
@@ -531,8 +531,8 @@ struct VectorExtractToArmSMELowering
     }
 
     Value sliceIndex = vector::getAsValues(rewriter, loc, position[0]).front();
-    auto extractTileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
-        loc, sourceVector, sliceIndex);
+    auto extractTileSlice = arm_sme::ExtractTileSliceOp::create(
+        rewriter, loc, sourceVector, sliceIndex);
 
     if (position.size() == 1) {
       // Single index case: Extracts a 1D slice.
@@ -593,10 +593,10 @@ struct VectorInsertToArmSMELowering
     if (position.size() == 2) {
       // Two indices case: Insert single element into tile.
       // We need to first extract the existing slice and update the element.
-      tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
-          loc, insertOp.getDest(), sliceIndex);
+      tileSlice = arm_sme::ExtractTileSliceOp::create(
+          rewriter, loc, insertOp.getDest(), sliceIndex);
       tileSlice = vector::InsertOp::create(rewriter, loc, source, tileSlice,
-                                                    position[1]);
+                                           position[1]);
     }
 
     // Insert the slice into the destination tile.
@@ -648,17 +648,18 @@ struct VectorPrintToArmSMELowering : public OpRewritePattern<vector::PrintOp> {
     auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto upperBound = arith::MulIOp::create(rewriter, loc, minTileRows, vscale);
     auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
+    auto forOp =
+        scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     {
       // Loop body.
       rewriter.setInsertionPointToStart(forOp.getBody());
       // Extract the current row from the tile.
       Value rowIndex = forOp.getInductionVar();
-      auto tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
-          loc, printOp.getSource(), rowIndex);
+      auto tileSlice = arm_sme::ExtractTileSliceOp::create(
+          rewriter, loc, printOp.getSource(), rowIndex);
       // Print the row with a 1D vector.print.
       vector::PrintOp::create(rewriter, loc, tileSlice,
-                                       printOp.getPunctuation());
+                              printOp.getPunctuation());
     }
 
     rewriter.eraseOp(printOp);
@@ -707,8 +708,8 @@ struct FoldTransferWriteOfExtractTileSlice
     Value mask = writeOp.getMask();
     if (!mask) {
       auto maskType = writeOp.getVectorType().clone(rewriter.getI1Type());
-      mask = arith::ConstantOp::create(rewriter,
-          writeOp.getLoc(), maskType, DenseElementsAttr::get(maskType, true));
+      mask = arith::ConstantOp::create(rewriter, writeOp.getLoc(), maskType,
+                                       DenseElementsAttr::get(maskType, true));
     }
 
     rewriter.replaceOpWithNewOp<arm_sme::StoreTileSliceOp>(
@@ -776,10 +777,10 @@ struct ExtractFromCreateMaskToPselLowering
     // Create the two 1-D masks at the location of the 2-D create_mask (which is
     // usually outside a loop). This prevents the need for later hoisting.
     rewriter.setInsertionPoint(createMaskOp);
-    auto rowMask = vector::CreateMaskOp::create(rewriter,
-        loc, rowMaskType, createMaskOp.getOperand(0));
-    auto colMask = vector::CreateMaskOp::create(rewriter,
-        loc, colMaskType, createMaskOp.getOperand(1));
+    auto rowMask = vector::CreateMaskOp::create(rewriter, loc, rowMaskType,
+                                                createMaskOp.getOperand(0));
+    auto colMask = vector::CreateMaskOp::create(rewriter, loc, colMaskType,
+                                                createMaskOp.getOperand(1));
 
     rewriter.setInsertionPoint(extractOp);
     auto position =
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 5b3e67509672a..d09d76a63b5ec 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -579,8 +579,8 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op,
   }
   gpu::MMAMatrixType type =
       gpu::MMAMatrixType::get(op.getVectorType().getShape(), elType, fragType);
-  Value load = gpu::SubgroupMmaLoadMatrixOp::create(rewriter,
-      op.getLoc(), type, op.getBase(), op.getIndices(),
+  Value load = gpu::SubgroupMmaLoadMatrixOp::create(
+      rewriter, op.getLoc(), type, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride),
       isTranspose ? rewriter.getUnitAttr() : UnitAttr());
   valueMapping[mappingResult] = load;
@@ -610,8 +610,8 @@ convertTransferWriteOp(RewriterBase &rewriter, vector::TransferWriteOp op,
   }
 
   Value matrix = it->second;
-  auto store = gpu::SubgroupMmaStoreMatrixOp::create(rewriter,
-      op.getLoc(), matrix, op.getBase(), op.getIndices(),
+  auto store = gpu::SubgroupMmaStoreMatrixOp::create(
+      rewriter, op.getLoc(), matrix, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride), /*transpose=*/UnitAttr());
   (void)store;
 
@@ -661,8 +661,8 @@ convertConstantOpMmaSync(RewriterBase &rewriter, arith::ConstantOp op,
     return rewriter.notifyMatchFailure(op, "not a splat");
   }
 
-  Value result = arith::ConstantOp::create(rewriter,
-      op.getLoc(), vectorType,
+  Value result = arith::ConstantOp::create(
+      rewriter, op.getLoc(), vectorType,
       DenseElementsAttr::get(vectorType, dense.getSplatValue<Attribute>()));
   valueMapping[op.getResult()] = result;
   return success();
@@ -757,8 +757,9 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   getXferIndices<vector::TransferReadOp>(rewriter, op, *offsets, {laneId},
                                          indices);
 
-  nvgpu::LdMatrixOp newOp = nvgpu::LdMatrixOp::create(rewriter,
-      loc, vectorType, op.getBase(), indices, *transpose, params->numTiles);
+  nvgpu::LdMatrixOp newOp =
+      nvgpu::LdMatrixOp::create(rewriter, loc, vectorType, op.getBase(),
+                                indices, *transpose, params->numTiles);
   valueMapping[op] = newOp->getResult(0);
   return success();
 }
@@ -788,8 +789,8 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   Type loadedElType = regInfo->registerLLVMType;
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
 
-  Value fill = arith::ConstantOp::create(rewriter,
-      op.getLoc(), vectorType.getElementType(),
+  Value fill = arith::ConstantOp::create(
+      rewriter, op.getLoc(), vectorType.getElementType(),
       rewriter.getZeroAttr(vectorType.getElementType()));
   Value result =
       vector::SplatOp::create(rewriter, op.getLoc(), fill, vectorType);
@@ -809,15 +810,15 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       if (failed(coords))
         return rewriter.notifyMatchFailure(op, "no coords");
 
-      Value logicalValueId = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexType(),
+      Value logicalValueId = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIndexType(),
           rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
       SmallVector<Value, 4> newIndices;
       getXferIndices<vector::TransferReadOp>(
           rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
 
       Value el = vector::LoadOp::create(rewriter, loc, loadedElType,
-                                                 op.getBase(), newIndices);
+                                        op.getBase(), newIndices);
       result = vector::InsertOp::create(rewriter, loc, el, result, i);
     }
   } else {
@@ -828,8 +829,8 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       for (unsigned innerIdx = 0; innerIdx < vectorType.getShape()[1];
            innerIdx++) {
 
-        Value logicalValueId = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getIndexType(),
+        Value logicalValueId = arith::ConstantOp::create(
+            rewriter, loc, rewriter.getIndexType(),
             rewriter.getIndexAttr(i * regInfo->elementsPerRegister + innerIdx));
         FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
             rewriter, op.getLoc(), *warpMatrixInfo);
@@ -840,9 +841,9 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
         getXferIndices<vector::TransferReadOp>(
             rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
         Value el = memref::LoadOp::create(rewriter, op.getLoc(), loadedElType,
-                                                   op.getBase(), newIndices);
-        result = vector::InsertOp::create(rewriter,
-            op.getLoc(), el, result, ArrayRef<int64_t>{i, innerIdx});
+                                          op.getBase(), newIndices);
+        result = vector::InsertOp::create(rewriter, op.getLoc(), el, result,
+                                          ArrayRef<int64_t>{i, innerIdx});
       }
     }
   }
@@ -919,8 +920,8 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
   Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   for (unsigned i = 0; i < vectorType.getShape()[0]; i++) {
-    Value logicalValueId = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIndexType(),
+    Value logicalValueId = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getIndexType(),
         rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
     FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
         rewriter, op.getLoc(), *warpMatrixInfo);
@@ -1015,8 +1016,8 @@ convertExtractStridedSlice(RewriterBase &rewriter,
   else if (offsets[1])
     sliceOffset[0] = (warpVectorShape[1] / offsets[1]);
 
-  Value newOp = vector::ExtractStridedSliceOp::create(rewriter,
-      loc, sourceVector, sliceOffset, sliceShape, strides);
+  Value newOp = vector::ExtractStridedSliceOp::create(
+      rewriter, loc, sourceVector, sliceOffset, sliceShape, strides);
 
   valueMapping[op] = newOp;
   return success();
@@ -1035,9 +1036,10 @@ convertContractOp(RewriterBase &rewriter, vector::ContractionOp op,
       itC == valueMapping.end())
     return rewriter.notifyMatchFailure(op, "no mapping");
   Value opA = itA->second, opB = itB->second, opC = itC->second;
-  Value matmul = gpu::SubgroupMmaComputeOp::create(rewriter,
-      op.getLoc(), opC.getType(), opA, opB, opC, /*a_transpose=*/UnitAttr(),
-      /*b_transpose=*/UnitAttr());
+  Value matmul = gpu::SubgroupMmaComputeOp::create(rewriter, op.getLoc(),
+                                                   opC.getType(), opA, opB, opC,
+                                                   /*a_transpose=*/UnitAttr(),
+                                                   /*b_transpose=*/UnitAttr());
   valueMapping[op.getResult()] = matmul;
   return success();
 }
@@ -1058,8 +1060,8 @@ convertContractOpToMmaSync(RewriterBase &rewriter, vector::ContractionOp op,
   int64_t m = cast<VectorType>(op.getLhs().getType()).getShape()[0];
   int64_t n = cast<VectorType>(op.getRhs().getType()).getShape()[0];
   int64_t k = cast<VectorType>(op.getLhs().getType()).getShape()[1];
-  Value matmul = nvgpu::MmaSyncOp::create(rewriter,
-      op.getLoc(), opA, opB, opC, rewriter.getI64ArrayAttr({m, n, k}));
+  Value matmul = nvgpu::MmaSyncOp::create(rewriter, op.getLoc(), opA, opB, opC,
+                                          rewriter.getI64ArrayAttr({m, n, k}));
   valueMapping[op.getResult()] = matmul;
   return success();
 }
@@ -1081,8 +1083,8 @@ convertConstantOp(RewriterBase &rewriter, arith::ConstantOp op,
   auto vecType = cast<VectorType>(op.getType());
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
-      op.getLoc(), type, scalarConstant);
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter, op.getLoc(),
+                                                         type, scalarConstant);
   valueMapping[op.getResult()] = matrix;
   return success();
 }
@@ -1100,8 +1102,8 @@ convertBroadcastOp(RewriterBase &rewriter, vector::BroadcastOp op,
   auto vecType = op.getResultVectorType();
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
-      op.getLoc(), type, op.getSource());
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter, op.getLoc(),
+                                                         type, op.getSource());
   valueMapping[op.getResult()] = matrix;
   return success();
 }
@@ -1118,9 +1120,9 @@ static scf::ForOp replaceForOpWithNewSignature(RewriterBase &rewriter,
   rewriter.setInsertionPoint(loop);
   auto operands = llvm::to_vector<4>(loop.getInitArgs());
   llvm::append_range(operands, newInitArgs);
-  scf::ForOp newLoop = scf::ForOp::create(rewriter,
-      loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
-      operands);
+  scf::ForOp newLoop =
+      scf::ForOp::create(rewriter, loop.getLoc(), loop.getLowerBound(),
+                         loop.getUpperBound(), loop.getStep(), operands);
   rewriter.eraseBlock(newLoop.getBody());
 
   newLoop.getRegion().getBlocks().splice(
@@ -1220,8 +1222,8 @@ convertElementwiseOp(RewriterBase &rewriter, Operation *op,
                                          resultType.getOperand());
   }
 
-  Value newOp = gpu::SubgroupMmaElementwiseOp::create(rewriter,
-      op->getLoc(), resultType, matrixOperands, opType);
+  Value newOp = gpu::SubgroupMmaElementwiseOp::create(
+      rewriter, op->getLoc(), resultType, matrixOperands, opType);
   valueMapping[op->getResult(0)] = newOp;
   return success();
 }
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index d21833736a718..c43466b25af6e 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -43,11 +43,11 @@ static Value insertOne(ConversionPatternRewriter &rewriter,
   assert(rank > 0 && "0-D vector corner case should have been handled already");
   if (rank == 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = LLVM::ConstantOp::create(rewriter,
-        loc, typeConverter.convertType(idxType),
+    auto constant = LLVM::ConstantOp::create(
+        rewriter, loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
     return LLVM::InsertElementOp::create(rewriter, loc, llvmType, val1, val2,
-                                                  constant);
+                                         constant);
   }
   return LLVM::InsertValueOp::create(rewriter, loc, val1, val2, pos);
 }
@@ -58,11 +58,11 @@ static Value extractOne(ConversionPatternRewriter &rewriter,
                         Value val, Type llvmType, int64_t rank, int64_t pos) {
   if (rank <= 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = LLVM::ConstantOp::create(rewriter,
-        loc, typeConverter.convertType(idxType),
+    auto constant = LLVM::ConstantOp::create(
+        rewriter, loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
     return LLVM::ExtractElementOp::create(rewriter, loc, llvmType, val,
-                                                   constant);
+                                          constant);
   }
   return LLVM::ExtractValueOp::create(rewriter, loc, val, pos);
 }
@@ -141,9 +141,9 @@ static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc,
   auto ptrsType =
       LLVM::getVectorType(pType, vectorType.getDimSize(0),
                           /*isScalable=*/vectorType.getScalableDims()[0]);
-  return LLVM::GEPOp::create(rewriter,
-      loc, ptrsType, typeConverter.convertType(memRefType.getElementType()),
-      base, index);
+  return LLVM::GEPOp::create(
+      rewriter, loc, ptrsType,
+      typeConverter.convertType(memRefType.getElementType()), base, index);
 }
 
 /// Convert `foldResult` into a Value. Integer attribute is converted to
@@ -476,31 +476,31 @@ static Value createReductionNeutralValue(ReductionNeutralZero neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   return LLVM::ConstantOp::create(rewriter, loc, llvmType,
-                                           rewriter.getZeroAttr(llvmType));
+                                  rewriter.getZeroAttr(llvmType));
 }
 
 /// Create the reduction neutral integer one value.
 static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType, rewriter.getIntegerAttr(llvmType, 1));
+  return LLVM::ConstantOp::create(rewriter, loc, llvmType,
+                                  rewriter.getIntegerAttr(llvmType, 1));
 }
 
 /// Create the reduction neutral fp one value.
 static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
+  return LLVM::ConstantOp::create(rewriter, loc, llvmType,
+                                  rewriter.getFloatAttr(llvmType, 1.0));
 }
 
 /// Create the reduction neutral all-ones value.
 static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(
           llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth())));
 }
@@ -509,8 +509,8 @@ static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
 }
@@ -519,8 +519,8 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
 }
@@ -529,8 +529,8 @@ static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
 }
@@ -539,8 +539,8 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
 }
@@ -550,8 +550,8 @@ static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
                                            /*Negative=*/false)));
@@ -562,8 +562,8 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
                                            /*Negative=*/true)));
@@ -591,8 +591,8 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
   auto vShape = vType.getShape();
   assert(vShape.size() == 1 && "Unexpected multi-dim vector type");
 
-  Value baseVecLength = LLVM::ConstantOp::create(rewriter,
-      loc, rewriter.getI32Type(),
+  Value baseVecLength = LLVM::ConstantOp::create(
+      rewriter, loc, rewriter.getI32Type(),
       rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0]));
 
   if (!vType.getScalableDims()[0])
@@ -616,7 +616,8 @@ static Value createIntegerReductionArithmeticOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator) {
 
-  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
+  Value result =
+      LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
 
   if (accumulator)
     result = ScalarOp::create(rewriter, loc, accumulator, result);
@@ -631,7 +632,8 @@ template <class LLVMRedIntrinOp>
 static Value createIntegerReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) {
-  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
+  Value result =
+      LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
   if (accumulator) {
     Value cmp =
         LLVM::ICmpOp::create(rewriter, loc, predicate, accumulator, result);
@@ -717,8 +719,8 @@ lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
                                 Value mask, LLVM::FastmathFlagsAttr fmf) {
   const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
       rewriter, loc, llvmType, vectorOperand.getType());
-  const Value selectedVectorByMask = LLVM::SelectOp::create(rewriter,
-      loc, mask, vectorOperand, vectorMaskNeutral);
+  const Value selectedVectorByMask = LLVM::SelectOp::create(
+      rewriter, loc, mask, vectorOperand, vectorMaskNeutral);
   return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
       rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
 }
@@ -731,8 +733,8 @@ lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
   return LLVMRedIntrinOp::create(rewriter, loc, llvmType,
-                                          /*startValue=*/accumulator,
-                                          vectorOperand, fmf);
+                                 /*startValue=*/accumulator, vectorOperand,
+                                 fmf);
 }
 
 /// Overloaded methods to lower a *predicated* reduction to an llvm intrinsic
@@ -746,8 +748,7 @@ lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter,
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
   return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
-                                            /*startValue=*/accumulator,
-                                            vectorOperand);
+                                   /*startValue=*/accumulator, vectorOperand);
 }
 
 template <class LLVMVPRedIntrinOp, class ReductionNeutral>
@@ -759,8 +760,8 @@ static Value lowerPredicatedReductionWithStartValue(
   Value vectorLength =
       createVectorLengthValue(rewriter, loc, vectorOperand.getType());
   return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
-                                            /*startValue=*/accumulator,
-                                            vectorOperand, mask, vectorLength);
+                                   /*startValue=*/accumulator, vectorOperand,
+                                   mask, vectorLength);
 }
 
 template <class LLVMIntVPRedIntrinOp, class IntReductionNeutral,
@@ -1071,8 +1072,8 @@ class VectorShuffleOpConversion
     // For rank 0 and 1, where both operands have *exactly* the same vector
     // type, there is direct shuffle support in LLVM. Use it!
     if (rank <= 1 && v1Type == v2Type) {
-      Value llvmShuffleOp = LLVM::ShuffleVectorOp::create(rewriter,
-          loc, adaptor.getV1(), adaptor.getV2(),
+      Value llvmShuffleOp = LLVM::ShuffleVectorOp::create(
+          rewriter, loc, adaptor.getV1(), adaptor.getV2(),
           llvm::to_vector_of<int32_t>(mask));
       rewriter.replaceOp(shuffleOp, llvmShuffleOp);
       return success();
@@ -1122,9 +1123,9 @@ class VectorExtractElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = extractEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = LLVM::ConstantOp::create(rewriter,
-          loc, typeConverter->convertType(idxType),
-          rewriter.getIntegerAttr(idxType, 0));
+      auto zero = LLVM::ConstantOp::create(rewriter, loc,
+                                           typeConverter->convertType(idxType),
+                                           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
           extractEltOp, llvmType, adaptor.getVector(), zero);
       return success();
@@ -1193,13 +1194,14 @@ class VectorExtractOpConversion
       if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) {
         return failure();
       }
-      extracted = LLVM::ExtractValueOp::create(rewriter,
-          loc, extracted, getAsIntegers(position));
+      extracted = LLVM::ExtractValueOp::create(rewriter, loc, extracted,
+                                               getAsIntegers(position));
     }
 
     if (extractsScalar) {
-      extracted = LLVM::ExtractElementOp::create(rewriter,
-          loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back()));
+      extracted = LLVM::ExtractElementOp::create(
+          rewriter, loc, extracted,
+          getAsLLVMValue(rewriter, loc, positionVec.back()));
     }
 
     rewriter.replaceOp(extractOp, extracted);
@@ -1256,9 +1258,9 @@ class VectorInsertElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = insertEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = LLVM::ConstantOp::create(rewriter,
-          loc, typeConverter->convertType(idxType),
-          rewriter.getIntegerAttr(idxType, 0));
+      auto zero = LLVM::ConstantOp::create(rewriter, loc,
+                                           typeConverter->convertType(idxType),
+                                           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
           insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(), zero);
       return success();
@@ -1342,8 +1344,8 @@ class VectorInsertOpConversion
           // llvm.extractvalue does not support dynamic dimensions.
           return failure();
         }
-        sourceAggregate = LLVM::ExtractValueOp::create(rewriter,
-            loc, adaptor.getDest(),
+        sourceAggregate = LLVM::ExtractValueOp::create(
+            rewriter, loc, adaptor.getDest(),
             getAsIntegers(positionOf1DVectorWithinAggregate));
       } else {
         // No-aggregate case. The destination for the InsertElementOp is just
@@ -1351,16 +1353,16 @@ class VectorInsertOpConversion
         sourceAggregate = adaptor.getDest();
       }
       // Insert the scalar into the 1D vector.
-      sourceAggregate = LLVM::InsertElementOp::create(rewriter,
-          loc, sourceAggregate.getType(), sourceAggregate,
+      sourceAggregate = LLVM::InsertElementOp::create(
+          rewriter, loc, sourceAggregate.getType(), sourceAggregate,
           adaptor.getValueToStore(),
           getAsLLVMValue(rewriter, loc, positionOfScalarWithin1DVector));
     }
 
     Value result = sourceAggregate;
     if (isNestedAggregate) {
-      result = LLVM::InsertValueOp::create(rewriter,
-          loc, adaptor.getDest(), sourceAggregate,
+      result = LLVM::InsertValueOp::create(
+          rewriter, loc, adaptor.getDest(), sourceAggregate,
           getAsIntegers(positionOf1DVectorWithinAggregate));
     }
 
@@ -1439,8 +1441,8 @@ class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
 
     auto loc = op.getLoc();
     auto elemType = vType.getElementType();
-    Value zero = arith::ConstantOp::create(rewriter,
-        loc, elemType, rewriter.getZeroAttr(elemType));
+    Value zero = arith::ConstantOp::create(rewriter, loc, elemType,
+                                           rewriter.getZeroAttr(elemType));
     Value desc = vector::SplatOp::create(rewriter, loc, vType, zero);
     for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
       Value extrLHS = ExtractOp::create(rewriter, loc, op.getLhs(), i);
@@ -1550,7 +1552,8 @@ class VectorTypeCastOpConversion
       desc.setSize(rewriter, loc, index, size);
       auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(),
                                                 (*targetStrides)[index]);
-      auto stride = LLVM::ConstantOp::create(rewriter, loc, int64Ty, strideAttr);
+      auto stride =
+          LLVM::ConstantOp::create(rewriter, loc, int64Ty, strideAttr);
       desc.setStride(rewriter, loc, index, stride);
     }
 
@@ -1578,14 +1581,15 @@ class VectorCreateMaskOpConversion
     IntegerType idxType =
         force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
     auto loc = op->getLoc();
-    Value indices = LLVM::StepVectorOp::create(rewriter,
-        loc, LLVM::getVectorType(idxType, dstType.getShape()[0],
-                                 /*isScalable=*/true));
+    Value indices = LLVM::StepVectorOp::create(
+        rewriter, loc,
+        LLVM::getVectorType(idxType, dstType.getShape()[0],
+                            /*isScalable=*/true));
     auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType,
                                                  adaptor.getOperands()[0]);
     Value bounds = SplatOp::create(rewriter, loc, indices.getType(), bound);
     Value comp = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
-                                                indices, bounds);
+                                       indices, bounds);
     rewriter.replaceOp(op, comp);
     return success();
   }
@@ -1741,16 +1745,16 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 
     switch (conversion) {
     case PrintConversion::ZeroExt64:
-      value = arith::ExtUIOp::create(rewriter,
-          loc, IntegerType::get(rewriter.getContext(), 64), value);
+      value = arith::ExtUIOp::create(
+          rewriter, loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::SignExt64:
-      value = arith::ExtSIOp::create(rewriter,
-          loc, IntegerType::get(rewriter.getContext(), 64), value);
+      value = arith::ExtSIOp::create(
+          rewriter, loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::Bitcast16:
-      value = LLVM::BitcastOp::create(rewriter,
-          loc, IntegerType::get(rewriter.getContext(), 16), value);
+      value = LLVM::BitcastOp::create(
+          rewriter, loc, IntegerType::get(rewriter.getContext(), 16), value);
       break;
     case PrintConversion::None:
       break;
@@ -1763,7 +1767,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
   static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
                        Operation *ref, ValueRange params = ValueRange()) {
     LLVM::CallOp::create(rewriter, loc, TypeRange(), SymbolRefAttr::get(ref),
-                                  params);
+                         params);
   }
 };
 
@@ -1783,8 +1787,8 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     auto vectorType = typeConverter->convertType(splatOp.getType());
     Value poison =
         LLVM::PoisonOp::create(rewriter, splatOp.getLoc(), vectorType);
-    auto zero = LLVM::ConstantOp::create(rewriter,
-        splatOp.getLoc(),
+    auto zero = LLVM::ConstantOp::create(
+        rewriter, splatOp.getLoc(),
         typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
 
@@ -1796,8 +1800,9 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     }
 
     // For 1-d vector, we additionally do a `vectorshuffle`.
-    auto v = LLVM::InsertElementOp::create(rewriter,
-        splatOp.getLoc(), vectorType, poison, adaptor.getInput(), zero);
+    auto v =
+        LLVM::InsertElementOp::create(rewriter, splatOp.getLoc(), vectorType,
+                                      poison, adaptor.getInput(), zero);
 
     int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
     SmallVector<int32_t> zeroValues(width, 0);
@@ -1837,11 +1842,11 @@ struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
     // Construct a 1-D vector with the splatted value that we insert in all the
     // places within the returned descriptor.
     Value vdesc = LLVM::PoisonOp::create(rewriter, loc, llvm1DVectorTy);
-    auto zero = LLVM::ConstantOp::create(rewriter,
-        loc, typeConverter->convertType(rewriter.getIntegerType(32)),
+    auto zero = LLVM::ConstantOp::create(
+        rewriter, loc, typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
-    Value v = LLVM::InsertElementOp::create(rewriter, loc, llvm1DVectorTy, vdesc,
-                                                     adaptor.getInput(), zero);
+    Value v = LLVM::InsertElementOp::create(rewriter, loc, llvm1DVectorTy,
+                                            vdesc, adaptor.getInput(), zero);
 
     // Shuffle the value across the desired number of elements.
     int64_t width = resultType.getDimSize(resultType.getRank() - 1);
@@ -1921,13 +1926,13 @@ struct VectorDeinterleaveOpLowering
       auto deinterleaveResults = deinterleaveOp.getResultTypes();
       auto packedOpResults =
           llvmTypeConverter->packOperationResults(deinterleaveResults);
-      auto intrinsic = LLVM::vector_deinterleave2::create(rewriter,
-          loc, packedOpResults, adaptor.getSource());
+      auto intrinsic = LLVM::vector_deinterleave2::create(
+          rewriter, loc, packedOpResults, adaptor.getSource());
 
-      auto evenResult = LLVM::ExtractValueOp::create(rewriter,
-          loc, intrinsic->getResult(0), 0);
-      auto oddResult = LLVM::ExtractValueOp::create(rewriter,
-          loc, intrinsic->getResult(0), 1);
+      auto evenResult = LLVM::ExtractValueOp::create(
+          rewriter, loc, intrinsic->getResult(0), 0);
+      auto oddResult = LLVM::ExtractValueOp::create(rewriter, loc,
+                                                    intrinsic->getResult(0), 1);
 
       rewriter.replaceOp(deinterleaveOp, ValueRange{evenResult, oddResult});
       return success();
@@ -1951,10 +1956,10 @@ struct VectorDeinterleaveOpLowering
     }
 
     auto poison = LLVM::PoisonOp::create(rewriter, loc, sourceType);
-    auto evenShuffle = LLVM::ShuffleVectorOp::create(rewriter,
-        loc, adaptor.getSource(), poison, evenShuffleMask);
-    auto oddShuffle = LLVM::ShuffleVectorOp::create(rewriter,
-        loc, adaptor.getSource(), poison, oddShuffleMask);
+    auto evenShuffle = LLVM::ShuffleVectorOp::create(
+        rewriter, loc, adaptor.getSource(), poison, evenShuffleMask);
+    auto oddShuffle = LLVM::ShuffleVectorOp::create(
+        rewriter, loc, adaptor.getSource(), poison, oddShuffleMask);
 
     rewriter.replaceOp(deinterleaveOp, ValueRange{evenShuffle, oddShuffle});
     return success();
@@ -2003,12 +2008,12 @@ struct VectorToElementsLowering
       if (element.use_empty())
         continue;
 
-      auto constIdx = LLVM::ConstantOp::create(rewriter,
-          loc, idxType, rewriter.getIntegerAttr(idxType, idx));
+      auto constIdx = LLVM::ConstantOp::create(
+          rewriter, loc, idxType, rewriter.getIntegerAttr(idxType, idx));
       auto llvmType = typeConverter->convertType(element.getType());
 
       Value result = LLVM::ExtractElementOp::create(rewriter, loc, llvmType,
-                                                             source, constIdx);
+                                                    source, constIdx);
       results[idx] = result;
     }
 
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index 7385e28aeb04a..f11fca77bdbe4 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -202,7 +202,7 @@ static Value generateInBoundsCheck(
     Value memrefIdx =
         affine::makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
     cond = arith::CmpIOp::create(lb, arith::CmpIPredicate::sgt, memrefDim,
-                                    memrefIdx);
+                                 memrefIdx);
   }
 
   // Condition check 2: Masked in?
@@ -215,8 +215,8 @@ static Value generateInBoundsCheck(
 
   // If the condition is non-empty, generate an SCF::IfOp.
   if (cond) {
-    auto check = scf::IfOp::create(lb,
-        cond,
+    auto check = scf::IfOp::create(
+        lb, cond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
           maybeYieldValue(b, loc, hasRetVal, inBoundsCase(b, loc));
@@ -310,7 +310,8 @@ static BufferAllocs allocBuffers(OpBuilder &b, OpTy xferOp) {
     auto maskBuffer = memref::AllocaOp::create(b, loc, maskType);
     b.setInsertionPoint(xferOp);
     memref::StoreOp::create(b, loc, xferOp.getMask(), maskBuffer);
-    result.maskBuffer = memref::LoadOp::create(b, loc, maskBuffer, ValueRange());
+    result.maskBuffer =
+        memref::LoadOp::create(b, loc, maskBuffer, ValueRange());
   }
 
   return result;
@@ -421,14 +422,15 @@ struct Strategy<TransferReadOp> {
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
-    auto newXferOp = vector::TransferReadOp::create(b,
-        loc, vecType, xferOp.getBase(), xferIndices,
+    auto newXferOp = vector::TransferReadOp::create(
+        b, loc, vecType, xferOp.getBase(), xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
         xferOp.getPadding(), Value(), inBoundsAttr);
 
     maybeApplyPassLabel(b, newXferOp, options.targetRank);
 
-    memref::StoreOp::create(b, loc, newXferOp.getVector(), buffer, storeIndices);
+    memref::StoreOp::create(b, loc, newXferOp.getVector(), buffer,
+                            storeIndices);
     return newXferOp;
   }
 
@@ -510,8 +512,8 @@ struct Strategy<TransferWriteOp> {
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
     auto source = loopState.empty() ? xferOp.getBase() : loopState[0];
     Type type = isTensorOp(xferOp) ? xferOp.getShapedType() : Type();
-    auto newXferOp = vector::TransferWriteOp::create(b,
-        loc, type, vec, source, xferIndices,
+    auto newXferOp = vector::TransferWriteOp::create(
+        b, loc, type, vec, source, xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
         inBoundsAttr);
 
@@ -611,7 +613,7 @@ struct PrepareTransferReadConversion
 
     Location loc = xferOp.getLoc();
     memref::StoreOp::create(rewriter, loc, newXfer->getResult(0),
-                                     buffers.dataBuffer);
+                            buffers.dataBuffer);
     rewriter.replaceOpWithNewOp<memref::LoadOp>(xferOp, buffers.dataBuffer);
 
     return success();
@@ -654,7 +656,7 @@ struct PrepareTransferWriteConversion
     Location loc = xferOp.getLoc();
     auto buffers = allocBuffers(rewriter, xferOp);
     memref::StoreOp::create(rewriter, loc, xferOp.getVector(),
-                                     buffers.dataBuffer);
+                            buffers.dataBuffer);
     auto loadedVec = memref::LoadOp::create(rewriter, loc, buffers.dataBuffer);
     rewriter.modifyOpInPlace(xferOp, [&]() {
       xferOp.getValueToStoreMutable().assign(loadedVec);
@@ -736,14 +738,14 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
           vectorType.cloneWith({}, getIntTypeWithSignlessSemantics(legalIntTy));
       auto targetVectorType = vectorType.cloneWith({}, legalIntTy);
       value = vector::BitCastOp::create(rewriter, loc, signlessSourceVectorType,
-                                                 value);
+                                        value);
       if (value.getType() != signlessTargetVectorType) {
         if (width == 1 || intTy.isUnsigned())
-          value = arith::ExtUIOp::create(rewriter, loc, signlessTargetVectorType,
-                                                  value);
+          value = arith::ExtUIOp::create(rewriter, loc,
+                                         signlessTargetVectorType, value);
         else
-          value = arith::ExtSIOp::create(rewriter, loc, signlessTargetVectorType,
-                                                  value);
+          value = arith::ExtSIOp::create(rewriter, loc,
+                                         signlessTargetVectorType, value);
       }
       value = vector::BitCastOp::create(rewriter, loc, targetVectorType, value);
       vectorType = targetVectorType;
@@ -770,11 +772,12 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     for (unsigned d = 0; d < shape.size(); d++) {
       // Setup loop bounds and step.
       Value lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
-      Value upperBound = arith::ConstantIndexOp::create(rewriter, loc, shape[d]);
+      Value upperBound =
+          arith::ConstantIndexOp::create(rewriter, loc, shape[d]);
       Value step = arith::ConstantIndexOp::create(rewriter, loc, 1);
       if (!scalableDimensions.empty() && scalableDimensions[d]) {
-        auto vscale = vector::VectorScaleOp::create(rewriter,
-            loc, rewriter.getIndexType());
+        auto vscale = vector::VectorScaleOp::create(rewriter, loc,
+                                                    rewriter.getIndexType());
         upperBound = arith::MulIOp::create(rewriter, loc, upperBound, vscale);
       }
       auto lastIndex = arith::SubIOp::create(rewriter, loc, upperBound, step);
@@ -783,8 +786,8 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
       vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
       auto loop =
           scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
-      auto printClose = vector::PrintOp::create(rewriter,
-          loc, vector::PrintPunctuation::Close);
+      auto printClose = vector::PrintOp::create(
+          rewriter, loc, vector::PrintPunctuation::Close);
       if (!firstClose)
         firstClose = printClose;
 
@@ -793,14 +796,14 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
 
       // Print a comma after all but the last element.
       rewriter.setInsertionPointToStart(loop.getBody());
-      auto notLastIndex = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::ult, loopIdx, lastIndex);
+      auto notLastIndex = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::ult, loopIdx, lastIndex);
       scf::IfOp::create(rewriter, loc, notLastIndex,
-                                 [&](OpBuilder &builder, Location loc) {
-                                   vector::PrintOp::create(builder,
-                                       loc, vector::PrintPunctuation::Comma);
-                                   scf::YieldOp::create(builder, loc);
-                                 });
+                        [&](OpBuilder &builder, Location loc) {
+                          vector::PrintOp::create(
+                              builder, loc, vector::PrintPunctuation::Comma);
+                          scf::YieldOp::create(builder, loc);
+                        });
 
       rewriter.setInsertionPointToStart(loop.getBody());
     }
@@ -810,7 +813,8 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     Value flatIndex;
     auto currentStride = 1;
     for (int d = shape.size() - 1; d >= 0; d--) {
-      auto stride = arith::ConstantIndexOp::create(rewriter, loc, currentStride);
+      auto stride =
+          arith::ConstantIndexOp::create(rewriter, loc, currentStride);
       auto index = arith::MulIOp::create(rewriter, loc, stride, loopIndices[d]);
       if (flatIndex)
         flatIndex = arith::AddIOp::create(rewriter, loc, flatIndex, index);
@@ -820,10 +824,9 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     }
 
     // Print the scalar elements in the inner most loop.
-    auto element =
-        vector::ExtractOp::create(rewriter, loc, value, flatIndex);
+    auto element = vector::ExtractOp::create(rewriter, loc, value, flatIndex);
     vector::PrintOp::create(rewriter, loc, element,
-                                     vector::PrintPunctuation::NoPunctuation);
+                            vector::PrintPunctuation::NoPunctuation);
 
     rewriter.setInsertionPointAfter(firstClose);
     vector::PrintOp::create(rewriter, loc, printOp.getPunctuation());
@@ -942,16 +945,16 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
 
     // Loop bounds and step.
     auto lb = arith::ConstantIndexOp::create(locB, 0);
-    auto ub = arith::ConstantIndexOp::create(locB,
-        castedDataType->getDimSize(castedDataType->getRank() - 1));
+    auto ub = arith::ConstantIndexOp::create(
+        locB, castedDataType->getDimSize(castedDataType->getRank() - 1));
     auto step = arith::ConstantIndexOp::create(locB, 1);
     // TransferWriteOps that operate on tensors return the modified tensor and
     // require a loop state.
     auto loopState = Strategy<OpTy>::initialLoopState(xferOp);
 
     // Generate for loop.
-    auto result = scf::ForOp::create(locB,
-        lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(),
+    auto result = scf::ForOp::create(
+        locB, lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(),
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopState) {
           Type stateType = loopState.empty() ? Type() : loopState[0].getType();
 
@@ -977,7 +980,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                   getMaskBufferLoadIndices(xferOp, castedMaskBuffer,
                                            loadIndices, iv);
                   auto mask = memref::LoadOp::create(b, loc, castedMaskBuffer,
-                                                       loadIndices);
+                                                     loadIndices);
                   rewriter.modifyOpInPlace(newXfer, [&]() {
                     newXfer.getMaskMutable().assign(mask);
                   });
@@ -1135,15 +1138,15 @@ struct ScalableTransposeTransferWriteConversion
     VectorType sliceType = VectorType::Builder(vectorType).dropDim(0);
     Value sliceMask = nullptr;
     if (!maskDims->empty()) {
-      sliceMask = vector::CreateMaskOp::create(rewriter,
-          loc, sliceType.clone(rewriter.getI1Type()),
+      sliceMask = vector::CreateMaskOp::create(
+          rewriter, loc, sliceType.clone(rewriter.getI1Type()),
           ArrayRef<OpFoldResult>(*maskDims).drop_front());
     }
 
     Value initDest = isTensorOp(writeOp) ? writeOp.getBase() : Value{};
     ValueRange initLoopArgs = initDest ? initDest : ValueRange{};
-    auto result = scf::ForOp::create(rewriter,
-        loc, lb, ub, step, initLoopArgs,
+    auto result = scf::ForOp::create(
+        rewriter, loc, lb, ub, step, initLoopArgs,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopIterArgs) {
           // Indices for the new transfer op.
           SmallVector<Value, 8> xferIndices;
@@ -1152,25 +1155,25 @@ struct ScalableTransposeTransferWriteConversion
           // Extract a transposed slice from the source vector.
           SmallVector<Value> transposeElements =
               llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-                return vector::ExtractOp::create(b,
-                    loc, transposeSourceSlices[idx], iv);
+                return vector::ExtractOp::create(
+                    b, loc, transposeSourceSlices[idx], iv);
               });
           auto sliceVec = vector::FromElementsOp::create(b, loc, sliceType,
-                                                           transposeElements);
+                                                         transposeElements);
 
           // Create the transfer_write for the slice.
           Value dest =
               loopIterArgs.empty() ? writeOp.getBase() : loopIterArgs.front();
-          auto newWriteOp = vector::TransferWriteOp::create(b,
-              loc, sliceVec, dest, xferIndices,
+          auto newWriteOp = vector::TransferWriteOp::create(
+              b, loc, sliceVec, dest, xferIndices,
               ArrayRef<bool>(writeOp.getInBoundsValues()).drop_front());
           if (sliceMask)
             newWriteOp.getMaskMutable().assign(sliceMask);
 
           // Yield from the loop.
-          scf::YieldOp::create(b, loc, loopIterArgs.empty()
-                                          ? ValueRange{}
-                                          : newWriteOp.getResult());
+          scf::YieldOp::create(b, loc,
+                               loopIterArgs.empty() ? ValueRange{}
+                                                    : newWriteOp.getResult());
         });
 
     if (isTensorOp(writeOp))
@@ -1263,7 +1266,7 @@ struct UnrollTransferReadConversion
       return insertOp.getDest();
     Location loc = xferOp.getLoc();
     return vector::SplatOp::create(rewriter, loc, xferOp.getVectorType(),
-                                            xferOp.getPadding());
+                                   xferOp.getPadding());
   }
 
   /// If the result of the TransferReadOp has exactly one user, which is a
@@ -1337,8 +1340,8 @@ struct UnrollTransferReadConversion
 
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
 
-            auto newXferOp = vector::TransferReadOp::create(b,
-                loc, newXferVecType, xferOp.getBase(), xferIndices,
+            auto newXferOp = vector::TransferReadOp::create(
+                b, loc, newXferVecType, xferOp.getBase(), xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
                 xferOp.getPadding(), Value(), inBoundsAttr);
             maybeAssignMask(b, xferOp, newXferOp, i);
@@ -1348,10 +1351,10 @@ struct UnrollTransferReadConversion
               // vector.insert does not accept rank-0 as the non-indexed
               // argument. Extract the scalar before inserting.
               valToInser = vector::ExtractOp::create(b, loc, valToInser,
-                                                       SmallVector<int64_t>());
+                                                     SmallVector<int64_t>());
             }
             return vector::InsertOp::create(b, loc, valToInser, vec,
-                                              insertionIndices);
+                                            insertionIndices);
           },
           /*outOfBoundsCase=*/
           [&](OpBuilder &b, Location loc) {
@@ -1485,13 +1488,13 @@ struct UnrollTransferWriteConversion
               // When target-rank=0, unrolling would causes the vector input
               // argument into `transfer_write` to become a scalar. We solve
               // this by broadcasting the scalar to a 0D vector.
-              xferVec = vector::BroadcastOp::create(b,
-                  loc, VectorType::get({}, extracted.getType()), extracted);
+              xferVec = vector::BroadcastOp::create(
+                  b, loc, VectorType::get({}, extracted.getType()), extracted);
             } else {
               xferVec = extracted;
             }
-            auto newXferOp = vector::TransferWriteOp::create(b,
-                loc, sourceType, xferVec, source, xferIndices,
+            auto newXferOp = vector::TransferWriteOp::create(
+                b, loc, sourceType, xferVec, source, xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
                 inBoundsAttr);
 
@@ -1585,7 +1588,7 @@ struct Strategy1d<TransferReadOp> {
     // Inititalize vector with padding value.
     Location loc = xferOp.getLoc();
     return vector::SplatOp::create(b, loc, xferOp.getVectorType(),
-                                     xferOp.getPadding());
+                                   xferOp.getPadding());
   }
 };
 
@@ -1602,8 +1605,7 @@ struct Strategy1d<TransferWriteOp> {
     generateInBoundsCheck(
         b, xferOp, iv, dim,
         /*inBoundsCase=*/[&](OpBuilder &b, Location loc) {
-          auto val =
-              vector::ExtractOp::create(b, loc, xferOp.getVector(), iv);
+          auto val = vector::ExtractOp::create(b, loc, xferOp.getVector(), iv);
           memref::StoreOp::create(b, loc, val, xferOp.getBase(), indices);
         });
     scf::YieldOp::create(b, loc);
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 2466e72277441..ea1b9c1c4203e 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -147,19 +147,19 @@ static Value sanitizeDynamicIndex(ConversionPatternRewriter &rewriter,
                                   Location loc, Value dynamicIndex,
                                   int64_t kPoisonIndex, unsigned vectorSize) {
   if (llvm::isPowerOf2_32(vectorSize)) {
-    Value inBoundsMask = spirv::ConstantOp::create(rewriter,
-        loc, dynamicIndex.getType(),
+    Value inBoundsMask = spirv::ConstantOp::create(
+        rewriter, loc, dynamicIndex.getType(),
         rewriter.getIntegerAttr(dynamicIndex.getType(), vectorSize - 1));
     return spirv::BitwiseAndOp::create(rewriter, loc, dynamicIndex,
-                                                inBoundsMask);
+                                       inBoundsMask);
   }
-  Value poisonIndex = spirv::ConstantOp::create(rewriter,
-      loc, dynamicIndex.getType(),
+  Value poisonIndex = spirv::ConstantOp::create(
+      rewriter, loc, dynamicIndex.getType(),
       rewriter.getIntegerAttr(dynamicIndex.getType(), kPoisonIndex));
   Value cmpResult =
       spirv::IEqualOp::create(rewriter, loc, dynamicIndex, poisonIndex);
-  return spirv::SelectOp::create(rewriter,
-      loc, cmpResult,
+  return spirv::SelectOp::create(
+      rewriter, loc, cmpResult,
       spirv::ConstantOp::getZero(dynamicIndex.getType(), loc, rewriter),
       dynamicIndex);
 }
@@ -427,8 +427,8 @@ static SmallVector<Value> extractAllElements(
   Location loc = reduceOp.getLoc();
 
   for (int i = 0; i < numElements; ++i) {
-    values.push_back(spirv::CompositeExtractOp::create(rewriter,
-        loc, srcVectorType.getElementType(), adaptor.getVector(),
+    values.push_back(spirv::CompositeExtractOp::create(
+        rewriter, loc, srcVectorType.getElementType(), adaptor.getVector(),
         rewriter.getI32ArrayAttr({i})));
   }
   if (Value acc = adaptor.getAcc())
@@ -481,16 +481,16 @@ struct VectorReductionPattern final : OpConversionPattern<vector::ReductionOp> {
 #define INT_AND_FLOAT_CASE(kind, iop, fop)                                     \
   case vector::CombiningKind::kind:                                            \
     if (llvm::isa<IntegerType>(resultType)) {                                  \
-      result = spirv::iop::create(rewriter, loc, resultType, result, next);     \
+      result = spirv::iop::create(rewriter, loc, resultType, result, next);    \
     } else {                                                                   \
       assert(llvm::isa<FloatType>(resultType));                                \
-      result = spirv::fop::create(rewriter, loc, resultType, result, next);     \
+      result = spirv::fop::create(rewriter, loc, resultType, result, next);    \
     }                                                                          \
     break
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = fop::create(rewriter, loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);             \
     break
 
         INT_AND_FLOAT_CASE(ADD, IAddOp, FAddOp);
@@ -537,7 +537,7 @@ struct VectorReductionFloatMinMax final
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = fop::create(rewriter, loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);             \
     break
 
         INT_OR_FLOAT_CASE(MAXIMUMF, SPIRVFMaxOp);
@@ -614,7 +614,7 @@ struct VectorShuffleOpConvert final
                                Value scalarOrVec, int32_t idx) -> Value {
       if (auto vecTy = dyn_cast<VectorType>(scalarOrVec.getType()))
         return spirv::CompositeExtractOp::create(rewriter, loc, scalarOrVec,
-                                                          idx);
+                                                 idx);
 
       assert(idx == 0 && "Invalid scalar element index");
       return scalarOrVec;
@@ -712,11 +712,13 @@ struct VectorDeinterleaveOpConvert final
     // We cannot use `spirv::VectorShuffleOp` directly in this case, and need to
     // use `spirv::CompositeExtractOp`.
     if (n == 2) {
-      auto elem0 = spirv::CompositeExtractOp::create(rewriter,
-          loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({0}));
+      auto elem0 = spirv::CompositeExtractOp::create(
+          rewriter, loc, newResultType, sourceVector,
+          rewriter.getI32ArrayAttr({0}));
 
-      auto elem1 = spirv::CompositeExtractOp::create(rewriter,
-          loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({1}));
+      auto elem1 = spirv::CompositeExtractOp::create(
+          rewriter, loc, newResultType, sourceVector,
+          rewriter.getI32ArrayAttr({1}));
 
       rewriter.replaceOp(deinterleaveOp, {elem0, elem1});
       return success();
@@ -733,12 +735,12 @@ struct VectorDeinterleaveOpConvert final
         llvm::map_to_vector(seqOdd, [](int i) { return i * 2 + 1; });
 
     // Create two SPIR-V shuffles.
-    auto shuffleEven = spirv::VectorShuffleOp::create(rewriter,
-        loc, newResultType, sourceVector, sourceVector,
+    auto shuffleEven = spirv::VectorShuffleOp::create(
+        rewriter, loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesEven));
 
-    auto shuffleOdd = spirv::VectorShuffleOp::create(rewriter,
-        loc, newResultType, sourceVector, sourceVector,
+    auto shuffleOdd = spirv::VectorShuffleOp::create(
+        rewriter, loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesOdd));
 
     rewriter.replaceOp(deinterleaveOp, {shuffleEven, shuffleOdd});
@@ -779,10 +781,11 @@ struct VectorLoadOpConverter final
     // For single element vectors, we don't need to bitcast the access chain to
     // the original vector type. Both is going to be the same, a pointer
     // to a scalar.
-    Value castedAccessChain = (vectorType.getNumElements() == 1)
-                                  ? accessChain
-                                  : spirv::BitcastOp::create(rewriter,
-                                        loc, vectorPtrType, accessChain);
+    Value castedAccessChain =
+        (vectorType.getNumElements() == 1)
+            ? accessChain
+            : spirv::BitcastOp::create(rewriter, loc, vectorPtrType,
+                                       accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, spirvVectorType,
                                                castedAccessChain);
@@ -821,10 +824,11 @@ struct VectorStoreOpConverter final
     // For single element vectors, we don't need to bitcast the access chain to
     // the original vector type. Both is going to be the same, a pointer
     // to a scalar.
-    Value castedAccessChain = (vectorType.getNumElements() == 1)
-                                  ? accessChain
-                                  : spirv::BitcastOp::create(rewriter,
-                                        loc, vectorPtrType, accessChain);
+    Value castedAccessChain =
+        (vectorType.getNumElements() == 1)
+            ? accessChain
+            : spirv::BitcastOp::create(rewriter, loc, vectorPtrType,
+                                       accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, castedAccessChain,
                                                 adaptor.getValueToStore());
@@ -905,10 +909,10 @@ struct VectorReductionToIntDotProd final
       auto v4i8Type = VectorType::get({4}, i8Type);
       Location loc = op.getLoc();
       Value zero = spirv::ConstantOp::getZero(i8Type, loc, rewriter);
-      lhsIn = spirv::CompositeConstructOp::create(rewriter,
-          loc, v4i8Type, ValueRange{lhsIn, zero});
-      rhsIn = spirv::CompositeConstructOp::create(rewriter,
-          loc, v4i8Type, ValueRange{rhsIn, zero});
+      lhsIn = spirv::CompositeConstructOp::create(rewriter, loc, v4i8Type,
+                                                  ValueRange{lhsIn, zero});
+      rhsIn = spirv::CompositeConstructOp::create(rewriter, loc, v4i8Type,
+                                                  ValueRange{rhsIn, zero});
     }
 
     // There's no variant of dot prod ops for unsigned LHS and signed RHS, so
@@ -1013,7 +1017,8 @@ struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
     source.reserve(numElements);
     for (int64_t i = 0; i < numElements; ++i) {
       Attribute intAttr = rewriter.getIntegerAttr(intType, i);
-      Value constOp = spirv::ConstantOp::create(rewriter, loc, intType, intAttr);
+      Value constOp =
+          spirv::ConstantOp::create(rewriter, loc, intType, intAttr);
       source.push_back(constOp);
     }
     rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(stepOp, dstType,
@@ -1056,8 +1061,8 @@ struct VectorToElementOpConvert final
       if (element.use_empty())
         continue;
 
-      Value result = spirv::CompositeExtractOp::create(rewriter,
-          loc, elementType, adaptor.getSource(),
+      Value result = spirv::CompositeExtractOp::create(
+          rewriter, loc, elementType, adaptor.getSource(),
           rewriter.getI32ArrayAttr({static_cast<int32_t>(idx)}));
       results[idx] = result;
     }
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index 8e074f91744e8..80107554144cf 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -109,7 +109,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
   xegpu::CreateNdDescOp ndDesc;
   if (srcTy.hasStaticShape()) {
     ndDesc = xegpu::CreateNdDescOp::create(rewriter, loc, descType, src,
-                                                    getAsOpFoldResult(offsets));
+                                           getAsOpFoldResult(offsets));
   } else {
     // In case of any dynamic shapes, source's shape and strides have to be
     // explicitly provided.
@@ -145,8 +145,8 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
     }
     std::reverse(dynStrides.begin(), dynStrides.end());
 
-    ndDesc = xegpu::CreateNdDescOp::create(rewriter,
-        loc, descType, src, dynOffsets, dynShapes, dynStrides,
+    ndDesc = xegpu::CreateNdDescOp::create(
+        rewriter, loc, descType, src, dynOffsets, dynShapes, dynStrides,
         DenseI64ArrayAttr::get(rewriter.getContext(), constOffsets),
         DenseI64ArrayAttr::get(rewriter.getContext(), srcTy.getShape()),
         DenseI64ArrayAttr::get(rewriter.getContext(), strides));
@@ -200,10 +200,10 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
                                                   ArrayRef<int64_t>{1, 0});
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadOp = xegpu::LoadNdOp::create(rewriter,
-        loc, vecTy, ndDesc, /*packed=*/nullptr, transposeAttr,
-        /*l1_hint=*/hint,
-        /*l2_hint=*/hint, /*l3_hint=*/hint);
+    auto loadOp = xegpu::LoadNdOp::create(rewriter, loc, vecTy, ndDesc,
+                                          /*packed=*/nullptr, transposeAttr,
+                                          /*l1_hint=*/hint,
+                                          /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(readOp, loadOp);
 
     return success();
@@ -239,8 +239,8 @@ struct TransferWriteLowering
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeOp =
         xegpu::StoreNdOp::create(rewriter, loc, writeOp.getVector(), ndDesc,
-                                          /*l1_hint=*/hint,
-                                          /*l2_hint=*/hint, /*l3_hint=*/hint);
+                                 /*l1_hint=*/hint,
+                                 /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(writeOp, storeOp);
 
     return success();
@@ -269,8 +269,8 @@ struct LoadLowering : public OpRewritePattern<vector::LoadOp> {
 
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadNdOp = xegpu::LoadNdOp::create(rewriter,
-        loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
+    auto loadNdOp = xegpu::LoadNdOp::create(
+        rewriter, loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(loadOp, loadNdOp);
@@ -304,8 +304,8 @@ struct StoreLowering : public OpRewritePattern<vector::StoreOp> {
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeNdOp =
         xegpu::StoreNdOp::create(rewriter, loc, vector, ndDesc,
-                                          /*l1_hint=*/hint,
-                                          /*l2_hint=*/hint, /*l3_hint=*/hint);
+                                 /*l1_hint=*/hint,
+                                 /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(storeOp, storeNdOp);
 
     return success();
@@ -339,8 +339,9 @@ struct ContractionLowering : public OpRewritePattern<vector::ContractionOp> {
     if (!isRowMajorMatmul(contractOp.getIndexingMapsAttr()))
       return rewriter.notifyMatchFailure(contractOp, "Invalid indexing maps");
 
-    auto dpasOp = xegpu::DpasOp::create(rewriter,
-        loc, TypeRange{contractOp.getResultType()}, ValueRange{lhs, rhs, acc});
+    auto dpasOp = xegpu::DpasOp::create(rewriter, loc,
+                                        TypeRange{contractOp.getResultType()},
+                                        ValueRange{lhs, rhs, acc});
     rewriter.replaceOp(contractOp, dpasOp);
 
     return success();
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
index 0230810f260ca..6d1f64e94df15 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -118,8 +118,8 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
 
   SmallVector<NamedAttribute> loadAttrs;
   patchOperandSegmentSizes(origAttrs, loadAttrs, DataArgAction::Drop);
-  Value initialLoad =
-      RawBufferLoadOp::create(rewriter, loc, dataType, invariantArgs, loadAttrs);
+  Value initialLoad = RawBufferLoadOp::create(rewriter, loc, dataType,
+                                              invariantArgs, loadAttrs);
   Block *currentBlock = rewriter.getInsertionBlock();
   Block *afterAtomic =
       rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
@@ -137,8 +137,8 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   patchOperandSegmentSizes(origAttrs, cmpswapAttrs, DataArgAction::Duplicate);
   SmallVector<Value> cmpswapArgs = {operated, prevLoad};
   cmpswapArgs.append(invariantArgs.begin(), invariantArgs.end());
-  Value atomicRes = RawBufferAtomicCmpswapOp::create(rewriter,
-      loc, dataType, cmpswapArgs, cmpswapAttrs);
+  Value atomicRes = RawBufferAtomicCmpswapOp::create(rewriter, loc, dataType,
+                                                     cmpswapArgs, cmpswapAttrs);
 
   // We care about exact bitwise equality here, so do some bitcasts.
   // These will fold away during lowering to the ROCDL dialect, where
@@ -154,10 +154,11 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
     atomicResForCompare =
         arith::BitcastOp::create(rewriter, loc, equivInt, atomicRes);
   }
-  Value canLeave = arith::CmpIOp::create(rewriter,
-      loc, arith::CmpIPredicate::eq, atomicResForCompare, prevLoadForCompare);
+  Value canLeave =
+      arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
+                            atomicResForCompare, prevLoadForCompare);
   cf::CondBranchOp::create(rewriter, loc, canLeave, afterAtomic, ValueRange{},
-                                    loopBlock, atomicRes);
+                           loopBlock, atomicRes);
   rewriter.eraseOp(atomicOp);
   return success();
 }
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
index a11804c1d7741..86b2025e2471b 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -56,11 +56,11 @@ static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
                                            vector::MaskedLoadOp maskedOp,
                                            bool passthru) {
   VectorType vectorType = maskedOp.getVectorType();
-  Value load = vector::LoadOp::create(builder,
-      loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
+  Value load = vector::LoadOp::create(
+      builder, loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
   if (passthru)
-    load = arith::SelectOp::create(builder,loc, vectorType, maskedOp.getMask(),
-                                           load, maskedOp.getPassThru());
+    load = arith::SelectOp::create(builder, loc, vectorType, maskedOp.getMask(),
+                                   load, maskedOp.getPassThru());
   return load;
 }
 
@@ -132,14 +132,14 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
     Value delta = arith::SubIOp::create(rewriter, loc, totalSize, linearIndex);
 
     // 1) check if delta < vectorSize
-    Value isOutofBounds = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
+    Value isOutofBounds = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
 
     // 2) check if (detla % elements_per_word != 0)
-    Value elementsPerWord = arith::ConstantIndexOp::create(rewriter,
-        loc, llvm::divideCeil(32, elementBitWidth));
-    Value isNotWordAligned = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ne,
+    Value elementsPerWord = arith::ConstantIndexOp::create(
+        rewriter, loc, llvm::divideCeil(32, elementBitWidth));
+    Value isNotWordAligned = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ne,
         arith::RemUIOp::create(rewriter, loc, delta, elementsPerWord),
         arith::ConstantIndexOp::create(rewriter, loc, 0));
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
index 4771d655e36c2..f8bab8289cbc6 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
@@ -37,8 +37,8 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       return rewriter.notifyMatchFailure(metadataOp,
                                          "not a fat raw buffer cast");
     Location loc = castOp.getLoc();
-    auto sourceMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
-        loc, castOp.getSource());
+    auto sourceMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, castOp.getSource());
     SmallVector<Value> results;
     if (metadataOp.getBaseBuffer().use_empty()) {
       results.push_back(nullptr);
@@ -48,8 +48,8 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       if (baseBufferType == castOp.getResult().getType()) {
         results.push_back(castOp.getResult());
       } else {
-        results.push_back(memref::ReinterpretCastOp::create(rewriter,
-            loc, baseBufferType, castOp.getResult(), /*offset=*/0,
+        results.push_back(memref::ReinterpretCastOp::create(
+            rewriter, loc, baseBufferType, castOp.getResult(), /*offset=*/0,
             /*sizes=*/ArrayRef<int64_t>{}, /*strides=*/ArrayRef<int64_t>{}));
       }
     }
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
index fb0d0d5027085..c0d174a04abf9 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
@@ -58,8 +58,9 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
       // Note: basis elements and their products are, definitionally,
       // non-negative, so `nuw` is justified.
       if (dynamicPart)
-        dynamicPart = arith::MulIOp::create(rewriter,
-            loc, dynamicPart, dynamicBasis[dynamicIndex - 1], ovflags);
+        dynamicPart =
+            arith::MulIOp::create(rewriter, loc, dynamicPart,
+                                  dynamicBasis[dynamicIndex - 1], ovflags);
       else
         dynamicPart = dynamicBasis[dynamicIndex - 1];
       --dynamicIndex;
@@ -111,15 +112,15 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
 
   auto emitModTerm = [&](Value stride) -> Value {
     Value remainder = arith::RemSIOp::create(rewriter, loc, linearIdx, stride);
-    Value remainderNegative = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::slt, remainder, zero);
+    Value remainderNegative = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::slt, remainder, zero);
     // If the correction is relevant, this term is <= stride, which is known
     // to be positive in `index`. Otherwise, while 2 * stride might overflow,
     // this branch won't be taken, so the risk of `poison` is fine.
-    Value corrected = arith::AddIOp::create(rewriter,
-        loc, remainder, stride, arith::IntegerOverflowFlags::nsw);
+    Value corrected = arith::AddIOp::create(rewriter, loc, remainder, stride,
+                                            arith::IntegerOverflowFlags::nsw);
     Value mod = arith::SelectOp::create(rewriter, loc, remainderNegative,
-                                                 corrected, remainder);
+                                        corrected, remainder);
     return mod;
   };
 
@@ -167,8 +168,8 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   // our hands on an `OpOperand&` for the loop invariant counting function.
   for (auto [stride, idxOp] :
        llvm::zip_equal(strides, llvm::drop_end(op.getMultiIndexMutable()))) {
-    Value scaledIdx = arith::MulIOp::create(rewriter,
-        loc, idxOp.get(), stride, arith::IntegerOverflowFlags::nsw);
+    Value scaledIdx = arith::MulIOp::create(rewriter, loc, idxOp.get(), stride,
+                                            arith::IntegerOverflowFlags::nsw);
     int64_t numHoistableLoops = numEnclosingInvariantLoops(idxOp);
     scaledValues.emplace_back(scaledIdx, numHoistableLoops);
   }
@@ -185,7 +186,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   for (auto [scaledValue, numHoistableLoops] : llvm::drop_begin(scaledValues)) {
     std::ignore = numHoistableLoops;
     result = arith::AddIOp::create(rewriter, loc, result, scaledValue,
-                                            arith::IntegerOverflowFlags::nsw);
+                                   arith::IntegerOverflowFlags::nsw);
   }
   rewriter.replaceOp(op, result);
   return success();
diff --git a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
index a78cea5bff6dd..3c00b323473d2 100644
--- a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
@@ -89,7 +89,7 @@ static AffineApplyOp createSubApply(RewriterBase &rewriter,
   SmallVector<Value> rhsOperands = originalOp->getOperands();
   canonicalizeMapAndOperands(&rhsMap, &rhsOperands);
   return AffineApplyOp::create(rewriter, originalOp.getLoc(), rhsMap,
-                                        rhsOperands);
+                               rhsOperands);
 }
 
 FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
@@ -161,7 +161,7 @@ FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
   for (int64_t i = 1, e = subExpressions.size(); i < e; ++i) {
     Value tmp = createSubApply(rewriter, op, subExpressions[i]);
     current = AffineApplyOp::create(rewriter, op.getLoc(), binMap,
-                                             ValueRange{current, tmp});
+                                    ValueRange{current, tmp});
     LLVM_DEBUG(DBGS() << "--reassociate into: " << current << "\n");
   }
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
index 17384654586c4..c942c0248fefd 100644
--- a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
@@ -100,8 +100,8 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
 
   // Create and place the alloc right before the 'affine.for' operation.
-  Value newMemRef = memref::AllocOp::create(bOuter,
-      forOp.getLoc(), newMemRefType, allocOperands);
+  Value newMemRef = memref::AllocOp::create(bOuter, forOp.getLoc(),
+                                            newMemRefType, allocOperands);
 
   // Create 'iv mod 2' value to index the leading dimension.
   auto d0 = bInner.getAffineDimExpr(0);
@@ -109,7 +109,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   auto modTwoMap =
       AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, d0.floorDiv(step) % 2);
   auto ivModTwoOp = AffineApplyOp::create(bInner, forOp.getLoc(), modTwoMap,
-                                                 forOp.getInductionVar());
+                                          forOp.getInductionVar());
 
   // replaceAllMemRefUsesWith will succeed unless the forOp body has
   // non-dereferencing uses of the memref (dealloc's are fine though).
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 0f16e84263ec1..f90aa3a68a1ea 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -56,7 +56,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
   OpBuilder b(forOp);
   auto lbMap = forOp.getLowerBoundMap();
   auto lb = AffineApplyOp::create(b, forOp.getLoc(), lbMap,
-                                    forOp.getLowerBoundOperands());
+                                  forOp.getLowerBoundOperands());
 
   // For each upper bound expr, get the range.
   // Eg: affine.for %i = lb to min (ub1, ub2),
@@ -135,8 +135,8 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         builder.setInsertionPointToStart(&func.getFunctionBody().front());
       else
         builder.setInsertionPoint(forOp);
-      auto constOp = arith::ConstantIndexOp::create(builder,
-          forOp.getLoc(), forOp.getConstantLowerBound());
+      auto constOp = arith::ConstantIndexOp::create(
+          builder, forOp.getLoc(), forOp.getConstantLowerBound());
       iv.replaceAllUsesWith(constOp);
     } else {
       auto lbOperands = forOp.getLowerBoundOperands();
@@ -183,7 +183,7 @@ static AffineForOp generateShiftedLoop(
 
   auto loopChunk =
       AffineForOp::create(b, srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
-                            ubMap, srcForOp.getStepAsInt());
+                          ubMap, srcForOp.getStepAsInt());
   auto loopChunkIV = loopChunk.getInductionVar();
   auto srcIV = srcForOp.getInductionVar();
 
@@ -198,8 +198,8 @@ static AffineForOp generateShiftedLoop(
     // Generate the remapping if the shift is not zero: remappedIV = newIV -
     // shift.
     if (!srcIV.use_empty() && shift != 0) {
-      auto ivRemap = AffineApplyOp::create(bodyBuilder,
-          srcForOp.getLoc(),
+      auto ivRemap = AffineApplyOp::create(
+          bodyBuilder, srcForOp.getLoc(),
           bodyBuilder.getSingleDimShiftAffineMap(
               -static_cast<int64_t>(srcForOp.getStepAsInt() * shift)),
           loopChunkIV);
@@ -1557,7 +1557,7 @@ stripmineSink(AffineForOp forOp, uint64_t factor,
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator(t.getBody());
     auto newForOp = AffineForOp::create(b, t.getLoc(), lbOperands, lbMap,
-                                          ubOperands, ubMap, originalStep);
+                                        ubOperands, ubMap, originalStep);
     auto begin = t.getBody()->begin();
     // Skip terminator and `newForOp` which is just before the terminator.
     auto nOps = t.getBody()->getOperations().size() - 2;
@@ -1653,8 +1653,8 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
     operands.push_back(prev);
     operands.push_back(upperBound);
     // Maintain running product of loop upper bounds.
-    prev = AffineApplyOp::create(builder,
-        loc,
+    prev = AffineApplyOp::create(
+        builder, loc,
         AffineMap::get(/*dimCount=*/1,
                        /*symbolCount=*/1,
                        builder.getAffineDimExpr(0) *
@@ -1683,13 +1683,12 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> operands;
       operands.push_back(previous);
       operands.push_back(upperBoundSymbols[idx]);
-      previous = AffineApplyOp::create(builder,
-          loc,
-          AffineMap::get(
-              /*dimCount=*/1, /*symbolCount=*/1,
-              builder.getAffineDimExpr(0).floorDiv(
-                  builder.getAffineSymbolExpr(0))),
-          operands);
+      previous = AffineApplyOp::create(builder, loc,
+                                       AffineMap::get(
+                                           /*dimCount=*/1, /*symbolCount=*/1,
+                                           builder.getAffineDimExpr(0).floorDiv(
+                                               builder.getAffineSymbolExpr(0))),
+                                       operands);
     }
     // Modified value of the induction variables of the nested loops after
     // coalescing.
@@ -1700,8 +1699,8 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> applyOperands;
       applyOperands.push_back(previous);
       applyOperands.push_back(upperBoundSymbols[idx - 1]);
-      inductionVariable = AffineApplyOp::create(builder,
-          loc,
+      inductionVariable = AffineApplyOp::create(
+          builder, loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
               builder.getAffineDimExpr(0) % builder.getAffineSymbolExpr(0)),
@@ -1738,16 +1737,16 @@ void mlir::affine::mapLoopToProcessorIds(scf::ForOp forOp,
 
   Value linearIndex = processorId.front();
   for (unsigned i = 1, e = processorId.size(); i < e; ++i) {
-    auto mulApplyOp = AffineApplyOp::create(b,
-        loc, mulMap, ValueRange{linearIndex, numProcessors[i]});
-    linearIndex = AffineApplyOp::create(b,
-        loc, addMap, ValueRange{mulApplyOp, processorId[i]});
+    auto mulApplyOp = AffineApplyOp::create(
+        b, loc, mulMap, ValueRange{linearIndex, numProcessors[i]});
+    linearIndex = AffineApplyOp::create(b, loc, addMap,
+                                        ValueRange{mulApplyOp, processorId[i]});
   }
 
-  auto mulApplyOp = AffineApplyOp::create(b,
-      loc, mulMap, ValueRange{linearIndex, forOp.getStep()});
-  Value lb = AffineApplyOp::create(b,
-      loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()});
+  auto mulApplyOp = AffineApplyOp::create(
+      b, loc, mulMap, ValueRange{linearIndex, forOp.getStep()});
+  Value lb = AffineApplyOp::create(
+      b, loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()});
   forOp.setLowerBound(lb);
 
   Value step = forOp.getStep();
@@ -1918,7 +1917,7 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
     // Copy in.
     auto load = AffineLoadOp::create(b, loc, memref, memIndices);
     AffineStoreOp::create(b, loc, load, fastMemRef, fastBufMap,
-                            fastBufMapOperands);
+                          fastBufMapOperands);
     return copyNestRoot;
   }
 
@@ -2085,7 +2084,8 @@ static LogicalResult generateCopy(
       lbs[d] = lbs[d].replaceDimsAndSymbols(
           /*dimReplacements=*/{}, symReplacements, lbs[d].getNumSymbols(),
           /*numResultSyms=*/0);
-      memIndices.push_back(AffineApplyOp::create(b, loc, lbs[d], regionSymbols));
+      memIndices.push_back(
+          AffineApplyOp::create(b, loc, lbs[d], regionSymbols));
     }
     // The fast buffer is copied into at location zero; addressing is relative.
     bufIndices.push_back(zeroIndex);
@@ -2144,8 +2144,8 @@ static LogicalResult generateCopy(
     if (!dmaStrideInfos.empty()) {
       dmaStride =
           arith::ConstantIndexOp::create(top, loc, dmaStrideInfos[0].stride);
-      numEltPerDmaStride = arith::ConstantIndexOp::create(top,
-          loc, dmaStrideInfos[0].numEltPerStride);
+      numEltPerDmaStride = arith::ConstantIndexOp::create(
+          top, loc, dmaStrideInfos[0].numEltPerStride);
     }
   }
 
@@ -2183,13 +2183,13 @@ static LogicalResult generateCopy(
     if (!region.isWrite()) {
       // DMA non-blocking read from original buffer to fast buffer.
       AffineDmaStartOp::create(b, loc, memref, memAffineMap, memIndices,
-                                 fastMemRef, bufAffineMap, bufIndices,
-                                 tagMemRef, tagAffineMap, tagIndices,
-                                 numElementsSSA, dmaStride, numEltPerDmaStride);
+                               fastMemRef, bufAffineMap, bufIndices, tagMemRef,
+                               tagAffineMap, tagIndices, numElementsSSA,
+                               dmaStride, numEltPerDmaStride);
     } else {
       // DMA non-blocking write from fast buffer to the original memref.
-      auto op = AffineDmaStartOp::create(b,
-          loc, fastMemRef, bufAffineMap, bufIndices, memref, memAffineMap,
+      auto op = AffineDmaStartOp::create(
+          b, loc, fastMemRef, bufAffineMap, bufIndices, memref, memAffineMap,
           memIndices, tagMemRef, tagAffineMap, tagIndices, numElementsSSA,
           dmaStride, numEltPerDmaStride);
       // Since new ops may be appended at 'end' (for outgoing DMAs), adjust the
@@ -2200,7 +2200,7 @@ static LogicalResult generateCopy(
 
     // Matching DMA wait to block on completion; tag always has a 0 index.
     AffineDmaWaitOp::create(b, loc, tagMemRef, tagAffineMap, zeroIndex,
-                              numElementsSSA);
+                            numElementsSSA);
 
     // Generate dealloc for the tag.
     auto tagDeallocOp = memref::DeallocOp::create(epilogue, loc, tagMemRef);
@@ -2583,7 +2583,7 @@ AffineForOp mlir::affine::createCanonicalizedAffineForOp(
   ubMap = removeDuplicateExprs(ubMap);
 
   return AffineForOp::create(b, loc, lowerOperands, lbMap, upperOperands, ubMap,
-                               step);
+                             step);
 }
 
 /// Creates an AffineIfOp that encodes the conditional to choose between
@@ -2667,7 +2667,7 @@ static AffineIfOp createSeparationCondition(MutableArrayRef<AffineForOp> loops,
   cst.getValues(0, cst.getNumDimAndSymbolVars(), &setOperands);
   canonicalizeSetAndOperands(&ifCondSet, &setOperands);
   return AffineIfOp::create(b, loops[0].getLoc(), ifCondSet, setOperands,
-                              /*withElseRegion=*/true);
+                            /*withElseRegion=*/true);
 }
 
 /// Create the full tile loop nest (along with its body).
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 0b0950968ecba..048d197722d33 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -93,12 +93,12 @@ class AffineApplyExpander
 
     Value remainder = arith::RemSIOp::create(builder, loc, lhs, rhs);
     Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
-    Value isRemainderNegative = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::slt, remainder, zeroCst);
+    Value isRemainderNegative = arith::CmpIOp::create(
+        builder, loc, arith::CmpIPredicate::slt, remainder, zeroCst);
     Value correctedRemainder =
         arith::AddIOp::create(builder, loc, remainder, rhs);
-    Value result = arith::SelectOp::create(builder,
-        loc, isRemainderNegative, correctedRemainder, remainder);
+    Value result = arith::SelectOp::create(builder, loc, isRemainderNegative,
+                                           correctedRemainder, remainder);
     return result;
   }
 
@@ -132,16 +132,17 @@ class AffineApplyExpander
 
     Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
     Value noneCst = arith::ConstantIndexOp::create(builder, loc, -1);
-    Value negative = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::slt, lhs, zeroCst);
-    Value negatedDecremented = arith::SubIOp::create(builder, loc, noneCst, lhs);
-    Value dividend =
-        arith::SelectOp::create(builder, loc, negative, negatedDecremented, lhs);
+    Value negative = arith::CmpIOp::create(
+        builder, loc, arith::CmpIPredicate::slt, lhs, zeroCst);
+    Value negatedDecremented =
+        arith::SubIOp::create(builder, loc, noneCst, lhs);
+    Value dividend = arith::SelectOp::create(builder, loc, negative,
+                                             negatedDecremented, lhs);
     Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value correctedQuotient =
         arith::SubIOp::create(builder, loc, noneCst, quotient);
     Value result = arith::SelectOp::create(builder, loc, negative,
-                                                   correctedQuotient, quotient);
+                                           correctedQuotient, quotient);
     return result;
   }
 
@@ -171,19 +172,19 @@ class AffineApplyExpander
 
     Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
     Value oneCst = arith::ConstantIndexOp::create(builder, loc, 1);
-    Value nonPositive = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::sle, lhs, zeroCst);
+    Value nonPositive = arith::CmpIOp::create(
+        builder, loc, arith::CmpIPredicate::sle, lhs, zeroCst);
     Value negated = arith::SubIOp::create(builder, loc, zeroCst, lhs);
     Value decremented = arith::SubIOp::create(builder, loc, lhs, oneCst);
-    Value dividend =
-        arith::SelectOp::create(builder, loc, nonPositive, negated, decremented);
+    Value dividend = arith::SelectOp::create(builder, loc, nonPositive, negated,
+                                             decremented);
     Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value negatedQuotient =
         arith::SubIOp::create(builder, loc, zeroCst, quotient);
     Value incrementedQuotient =
         arith::AddIOp::create(builder, loc, quotient, oneCst);
-    Value result = arith::SelectOp::create(builder,
-        loc, nonPositive, negatedQuotient, incrementedQuotient);
+    Value result = arith::SelectOp::create(
+        builder, loc, nonPositive, negatedQuotient, incrementedQuotient);
     return result;
   }
 
@@ -299,8 +300,8 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   IRMapping operandMap;
   OpBuilder b(hoistOverOp);
   auto hoistedIfOp = AffineIfOp::create(b, ifOp.getLoc(), ifOp.getIntegerSet(),
-                                          ifOp.getOperands(),
-                                          /*elseBlock=*/true);
+                                        ifOp.getOperands(),
+                                        /*elseBlock=*/true);
 
   // Create a clone of hoistOverOp to use for the else branch of the hoisted
   // conditional. The else block may get optimized away if empty.
@@ -369,8 +370,8 @@ mlir::affine::affineParallelize(AffineForOp forOp,
       parallelReductions, [](const LoopReduction &red) { return red.value; }));
   auto reductionKinds = llvm::to_vector<4>(llvm::map_range(
       parallelReductions, [](const LoopReduction &red) { return red.kind; }));
-  AffineParallelOp newPloop = AffineParallelOp::create(outsideBuilder,
-      loc, ValueRange(reducedValues).getTypes(), reductionKinds,
+  AffineParallelOp newPloop = AffineParallelOp::create(
+      outsideBuilder, loc, ValueRange(reducedValues).getTypes(), reductionKinds,
       llvm::ArrayRef(lowerBoundMap), lowerBoundOperands,
       llvm::ArrayRef(upperBoundMap), upperBoundOperands,
       llvm::ArrayRef(forOp.getStepAsInt()));
@@ -541,7 +542,8 @@ void mlir::affine::normalizeAffineParallel(AffineParallelOp op) {
     SmallVector<Value, 8> applyOperands{dimOperands};
     applyOperands.push_back(iv);
     applyOperands.append(symbolOperands.begin(), symbolOperands.end());
-    auto apply = AffineApplyOp::create(builder, op.getLoc(), map, applyOperands);
+    auto apply =
+        AffineApplyOp::create(builder, op.getLoc(), map, applyOperands);
     iv.replaceAllUsesExcept(apply, apply);
   }
 
@@ -622,8 +624,9 @@ LogicalResult mlir::affine::normalizeAffineFor(AffineForOp op,
   AffineValueMap newIvToOldIvMap;
   AffineValueMap::difference(lbMap, scaleIvValueMap, &newIvToOldIvMap);
   (void)newIvToOldIvMap.canonicalize();
-  auto newIV = AffineApplyOp::create(opBuilder,
-      loc, newIvToOldIvMap.getAffineMap(), newIvToOldIvMap.getOperands());
+  auto newIV =
+      AffineApplyOp::create(opBuilder, loc, newIvToOldIvMap.getAffineMap(),
+                            newIvToOldIvMap.getOperands());
   op.getInductionVar().replaceAllUsesExcept(newIV->getResult(0), newIV);
   return success();
 }
@@ -1188,7 +1191,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
       auto singleResMap = AffineMap::get(oldMap.getNumDims(),
                                          oldMap.getNumSymbols(), resultExpr);
       auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
-                                                oldMapOperands);
+                                        oldMapOperands);
       oldMemRefOperands.push_back(afOp);
       affineApplyOps.push_back(afOp);
     }
@@ -1215,7 +1218,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
       auto singleResMap = AffineMap::get(
           indexRemap.getNumDims(), indexRemap.getNumSymbols(), resultExpr);
       auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
-                                                remapOperands);
+                                        remapOperands);
       remapOutputs.push_back(afOp);
       affineApplyOps.push_back(afOp);
     }
@@ -1264,8 +1267,8 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     // AffineMapAccessInterface, we need to apply the values of `newMapOperands`
     // to the `newMap` to get the correct indices.
     for (unsigned i = 0; i < newMemRefRank; i++) {
-      state.operands.push_back(AffineApplyOp::create(builder,
-          op->getLoc(),
+      state.operands.push_back(AffineApplyOp::create(
+          builder, op->getLoc(),
           AffineMap::get(newMap.getNumDims(), newMap.getNumSymbols(),
                          newMap.getResult(i)),
           newMapOperands));
@@ -1450,8 +1453,8 @@ void mlir::affine::createAffineComputationSlice(
   for (auto resultExpr : composedMap.getResults()) {
     auto singleResMap = AffineMap::get(composedMap.getNumDims(),
                                        composedMap.getNumSymbols(), resultExpr);
-    sliceOps->push_back(AffineApplyOp::create(builder,
-        opInst->getLoc(), singleResMap, composedOpOperands));
+    sliceOps->push_back(AffineApplyOp::create(
+        builder, opInst->getLoc(), singleResMap, composedOpOperands));
   }
 
   // Construct the new operands that include the results from the composed
@@ -1740,12 +1743,11 @@ LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp allocOp) {
     createNewDynamicSizes(oldMemRefType, newMemRefType, layoutMap, allocOp, b,
                           newDynamicSizes);
     // Add the new dynamic sizes in new AllocOp.
-    newAlloc =
-        AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType, newDynamicSizes,
-                              allocOp.getAlignmentAttr());
+    newAlloc = AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType,
+                                   newDynamicSizes, allocOp.getAlignmentAttr());
   } else {
     newAlloc = AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType,
-                                     allocOp.getAlignmentAttr());
+                                   allocOp.getAlignmentAttr());
   }
   // Replace all uses of the old memref.
   if (failed(replaceAllMemRefUsesWith(oldMemRef, /*newMemRef=*/newAlloc,
@@ -1804,7 +1806,7 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
     if (memrefType.isDynamicDim(i))
       mapOperands[i] =
           arith::SubIOp::create(b, loc, oldSizes[0].getType(), oldSizes[idx++],
-                                  arith::ConstantIndexOp::create(b, loc, 1));
+                                arith::ConstantIndexOp::create(b, loc, 1));
     else
       mapOperands[i] = arith::ConstantIndexOp::create(b, loc, oldShape[i] - 1);
   }
@@ -1816,8 +1818,8 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0; i < newRank; i++) {
     if (!newMemRefType.isDynamicDim(i))
       continue;
-    newSizes.push_back(AffineApplyOp::create(b,
-        loc,
+    newSizes.push_back(AffineApplyOp::create(
+        b, loc,
         AffineMap::get(oldLayoutMap.getNumDims(), oldLayoutMap.getNumSymbols(),
                        oldLayoutMap.getResult(i)),
         mapOperands));
@@ -1825,11 +1827,11 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0, e = newSizes.size(); i < e; i++) {
     newSizes[i] =
         arith::AddIOp::create(b, loc, newSizes[i].getType(), newSizes[i],
-                                arith::ConstantIndexOp::create(b, loc, 1));
+                              arith::ConstantIndexOp::create(b, loc, 1));
   }
   // Create the new reinterpret_cast op.
-  auto newReinterpretCast = memref::ReinterpretCastOp::create(b,
-      loc, newMemRefType, reinterpretCastOp.getSource(),
+  auto newReinterpretCast = memref::ReinterpretCastOp::create(
+      b, loc, newMemRefType, reinterpretCastOp.getSource(),
       /*offsets=*/ValueRange(), newSizes,
       /*strides=*/ValueRange(),
       /*static_offsets=*/newStaticOffsets,
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index d21da23903544..9199dccdcaff3 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -67,8 +67,8 @@ struct SelectOpInterface
       return state.getMemrefWithUniqueOwnership(builder, value,
                                                 value.getParentBlock());
 
-    Value ownership = arith::SelectOp::create(builder,
-        op->getLoc(), selectOp.getCondition(),
+    Value ownership = arith::SelectOp::create(
+        builder, op->getLoc(), selectOp.getCondition(),
         state.getOwnership(selectOp.getTrueValue(), block).getIndicator(),
         state.getOwnership(selectOp.getFalseValue(), block).getIndicator());
     return {selectOp.getResult(), ownership};
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
index 44e896a97cb4b..efe6ad2579055 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
@@ -81,7 +81,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
   SmallVector<int64_t> strides(shape.size(), 1);
 
   return vector::ExtractStridedSliceOp::create(rewriter, loc, input, offsets,
-                                                        sizes, strides);
+                                               sizes, strides);
 }
 
 /// Extracts two vector slices from the `input` whose type is `vector<...x2T>`,
@@ -142,7 +142,7 @@ static Value insertLastDimSlice(ConversionPatternRewriter &rewriter,
   offsets.back() = lastOffset;
   SmallVector<int64_t> strides(shape.size(), 1);
   return vector::InsertStridedSliceOp::create(rewriter, loc, source, dest,
-                                                       offsets, strides);
+                                              offsets, strides);
 }
 
 /// Constructs a new vector of type `resultType` by creating a series of
@@ -362,8 +362,8 @@ struct ConvertCmpI final : OpConversionPattern<arith::CmpIOp> {
     }
     default: {
       // Handle inequality checks.
-      Value highEq = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::eq, lhsElem1, rhsElem1);
+      Value highEq = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::eq, lhsElem1, rhsElem1);
       cmpResult =
           arith::SelectOp::create(rewriter, loc, highEq, lowCmp, highCmp);
       break;
@@ -443,8 +443,8 @@ struct ConvertExtSI final : OpConversionPattern<arith::ExtSIOp> {
         loc, newResultComponentTy, newOperand);
     Value operandZeroCst =
         createScalarOrSplatConstant(rewriter, loc, newResultComponentTy, 0);
-    Value signBit = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::slt, extended, operandZeroCst);
+    Value signBit = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::slt, extended, operandZeroCst);
     Value signValue =
         arith::ExtSIOp::create(rewriter, loc, newResultComponentTy, signBit);
 
@@ -680,16 +680,16 @@ struct ConvertShLI final : OpConversionPattern<arith::ShLIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
+    Value illegalElemShift = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
         arith::ShLIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     Value resElem0 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
-                                                      zeroCst, shiftedElem0);
+                                             zeroCst, shiftedElem0);
 
-    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, elemBitWidth, rhsElem0);
+    Value cappedShiftAmount = arith::SelectOp::create(
+        rewriter, loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value rightShiftAmount =
         arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedRight =
@@ -701,10 +701,10 @@ struct ConvertShLI final : OpConversionPattern<arith::ShLIOp> {
 
     Value shiftedElem1 =
         arith::ShLIOp::create(rewriter, loc, lhsElem1, rhsElem0);
-    Value resElem1High = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, zeroCst, shiftedElem1);
-    Value resElem1Low = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, shiftedLeft, shiftedRight);
+    Value resElem1High = arith::SelectOp::create(
+        rewriter, loc, illegalElemShift, zeroCst, shiftedElem1);
+    Value resElem1Low = arith::SelectOp::create(rewriter, loc, illegalElemShift,
+                                                shiftedLeft, shiftedRight);
     Value resElem1 =
         arith::OrIOp::create(rewriter, loc, resElem1Low, resElem1High);
 
@@ -769,20 +769,20 @@ struct ConvertShRUI final : OpConversionPattern<arith::ShRUIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
+    Value illegalElemShift = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
         arith::ShRUIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     Value resElem0Low = arith::SelectOp::create(rewriter, loc, illegalElemShift,
-                                                         zeroCst, shiftedElem0);
+                                                zeroCst, shiftedElem0);
     Value shiftedElem1 =
         arith::ShRUIOp::create(rewriter, loc, lhsElem1, rhsElem0);
     Value resElem1 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
-                                                      zeroCst, shiftedElem1);
+                                             zeroCst, shiftedElem1);
 
-    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, elemBitWidth, rhsElem0);
+    Value cappedShiftAmount = arith::SelectOp::create(
+        rewriter, loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value leftShiftAmount =
         arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedLeft =
@@ -792,8 +792,8 @@ struct ConvertShRUI final : OpConversionPattern<arith::ShRUIOp> {
     Value shiftedRight =
         arith::ShRUIOp::create(rewriter, loc, lhsElem1, overshotShiftAmount);
 
-    Value resElem0High = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, shiftedRight, shiftedLeft);
+    Value resElem0High = arith::SelectOp::create(
+        rewriter, loc, illegalElemShift, shiftedRight, shiftedLeft);
     Value resElem0 =
         arith::OrIOp::create(rewriter, loc, resElem0Low, resElem0High);
 
@@ -832,8 +832,8 @@ struct ConvertShRSI final : OpConversionPattern<arith::ShRSIOp> {
     // Perform as many ops over the narrow integer type as possible and let the
     // other emulation patterns convert the rest.
     Value elemZero = createScalarOrSplatConstant(rewriter, loc, narrowTy, 0);
-    Value signBit = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::slt, lhsElem1, elemZero);
+    Value signBit = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::slt, lhsElem1, elemZero);
     signBit = dropTrailingX1Dim(rewriter, loc, signBit);
 
     // Create a bit pattern of either all ones or all zeros. Then shift it left
@@ -857,8 +857,8 @@ struct ConvertShRSI final : OpConversionPattern<arith::ShRSIOp> {
 
     // Handle shifting by zero. This is necessary when the `signBits` shift is
     // invalid.
-    Value isNoop = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                                  rhsElem0, elemZero);
+    Value isNoop = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, rhsElem0, elemZero);
     isNoop = dropTrailingX1Dim(rewriter, loc, isNoop);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNoop, op.getLhs(),
                                                  shrsi);
@@ -894,8 +894,8 @@ struct ConvertSubI final : OpConversionPattern<arith::SubIOp> {
     // CARRY is 1 or 0.
     Value low = arith::SubIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     // We have a carry if lhsElem0 < rhsElem0.
-    Value carry0 = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ult, lhsElem0, rhsElem0);
+    Value carry0 = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ult, lhsElem0, rhsElem0);
     Value carryVal = arith::ExtUIOp::create(rewriter, loc, newElemTy, carry0);
 
     Value high0 = arith::SubIOp::create(rewriter, loc, lhsElem1, carryVal);
@@ -933,8 +933,8 @@ struct ConvertSIToFP final : OpConversionPattern<arith::SIToFPOp> {
     // result or not based on that sign bit. We implement negation by
     // subtracting from zero. Note that this relies on the the other conversion
     // patterns to legalize created ops and narrow the bit widths.
-    Value isNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
-                                                 in, zeroCst);
+    Value isNeg = arith::CmpIOp::create(rewriter, loc,
+                                        arith::CmpIPredicate::slt, in, zeroCst);
     Value neg = arith::SubIOp::create(rewriter, loc, zeroCst, in);
     Value abs = arith::SelectOp::create(rewriter, loc, isNeg, neg, in);
 
@@ -985,8 +985,8 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     //
     // Note 2: We do not strictly need the `hi == 0`, case, but it makes
     // constant folding easier.
-    Value hiEqZero = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::eq, hiInt, zeroCst);
+    Value hiEqZero = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, hiInt, zeroCst);
 
     Type resultTy = op.getType();
     Type resultElemTy = getElementTypeOrSelf(resultTy);
@@ -999,7 +999,8 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     if (auto vecTy = dyn_cast<VectorType>(resultTy))
       pow2Attr = SplatElementsAttr::get(vecTy, pow2Attr);
 
-    Value pow2Val = arith::ConstantOp::create(rewriter, loc, resultTy, pow2Attr);
+    Value pow2Val =
+        arith::ConstantOp::create(rewriter, loc, resultTy, pow2Attr);
 
     Value hiVal = arith::MulFOp::create(rewriter, loc, hiFp, pow2Val);
     Value result = arith::AddFOp::create(rewriter, loc, lowFp, hiVal);
@@ -1042,8 +1043,8 @@ struct ConvertFPToSI final : OpConversionPattern<arith::FPToSIOp> {
 
     // Get the absolute value. One could have used math.absf here, but that
     // introduces an extra dependency.
-    Value isNeg = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OLT,
-                                                 inFp, zeroCst);
+    Value isNeg = arith::CmpFOp::create(
+        rewriter, loc, arith::CmpFPredicate::OLT, inFp, zeroCst);
     Value negInFp = arith::NegFOp::create(rewriter, loc, inFp);
 
     Value absVal = arith::SelectOp::create(rewriter, loc, isNeg, negInFp, inFp);
diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
index dcdb0d1687504..f304354e6f45d 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
@@ -29,8 +29,8 @@ static Value createConst(Location loc, Type type, int value,
                          PatternRewriter &rewriter) {
   auto attr = rewriter.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(shapedTy, attr));
+    return arith::ConstantOp::create(rewriter, loc,
+                                     DenseElementsAttr::get(shapedTy, attr));
   }
   return arith::ConstantOp::create(rewriter, loc, attr);
 }
@@ -40,8 +40,8 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                               PatternRewriter &rewriter) {
   auto attr = rewriter.getFloatAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(shapedTy, attr));
+    return arith::ConstantOp::create(rewriter, loc,
+                                     DenseElementsAttr::get(shapedTy, attr));
   }
 
   return arith::ConstantOp::create(rewriter, loc, attr);
@@ -99,16 +99,16 @@ struct CeilDivSIOpConverter : public OpRewritePattern<arith::CeilDivSIOp> {
 
     Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
     Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
-    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ne, a, product);
+    Value notEqualDivisor = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ne, a, product);
 
-    Value aNeg =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
-    Value bNeg =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
+    Value aNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                                       a, zero);
+    Value bNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                                       b, zero);
 
-    Value signEqual = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::eq, aNeg, bNeg);
+    Value signEqual = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, aNeg, bNeg);
     Value cond =
         arith::AndIOp::create(rewriter, loc, notEqualDivisor, signEqual);
 
@@ -138,17 +138,17 @@ struct FloorDivSIOpConverter : public OpRewritePattern<arith::FloorDivSIOp> {
 
     Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
     Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
-    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ne, a, product);
+    Value notEqualDivisor = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ne, a, product);
     Value zero = createConst(loc, type, 0, rewriter);
 
-    Value aNeg =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
-    Value bNeg =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
+    Value aNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                                       a, zero);
+    Value bNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                                       b, zero);
 
-    Value signOpposite = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ne, aNeg, bNeg);
+    Value signOpposite = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ne, aNeg, bNeg);
     Value cond =
         arith::AndIOp::create(rewriter, loc, notEqualDivisor, signOpposite);
 
@@ -197,8 +197,8 @@ struct MaximumMinimumFOpConverter : public OpRewritePattern<OpTy> {
     Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'.
-    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
-                                                 rhs, rhs);
+    Value isNaN = arith::CmpFOp::create(rewriter, loc,
+                                        arith::CmpFPredicate::UNO, rhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
   }
@@ -223,8 +223,8 @@ struct MaxNumMinNumFOpConverter : public OpRewritePattern<OpTy> {
     Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where lhs is NaN: 'isNaN(lhs) ? rhs : select'.
-    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
-                                                 lhs, lhs);
+    Value isNaN = arith::CmpFOp::create(rewriter, loc,
+                                        arith::CmpFPredicate::UNO, lhs, lhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
   }
@@ -415,7 +415,8 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Value c0x8 = createConst(loc, i4Ty, 0x8, rewriter);
     Value negative =
         arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x8);
-    Value bit32 = arith::SelectOp::create(b, negative, c0x80000000, zeroExpBits);
+    Value bit32 =
+        arith::SelectOp::create(b, negative, c0x80000000, zeroExpBits);
 
     // Add segments together.
     Value bits1To31 = arith::AddIOp::create(b, bits1To24, bits25To31);
@@ -464,7 +465,7 @@ struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Value result = arith::BitcastOp::create(b, f32Ty, f32Bits);
     if (resultETy.getIntOrFloatBitWidth() < 32) {
       result = arith::TruncFOp::create(b, resultTy, result, nullptr,
-                                         op.getFastmathAttr());
+                                       op.getFastmathAttr());
     } else if (resultETy.getIntOrFloatBitWidth() > 32) {
       result = arith::ExtFOp::create(b, resultTy, result, op.getFastmathAttr());
     }
@@ -570,7 +571,7 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
         arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0xff);
     Value man23Bits = arith::AndIOp::create(b, f32Bits, cF32MantissaMask);
     Value isNonZeroMan = arith::CmpIOp::create(b, arith::CmpIPredicate::ugt,
-                                                 man23Bits, zeroExpBits);
+                                               man23Bits, zeroExpBits);
     Value roundToHalf = arith::AndIOp::create(b, isNegOneExp, isNonZeroMan);
     Value isZeroExp =
         arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0x00);
@@ -628,8 +629,8 @@ struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     if (operandETy.getIntOrFloatBitWidth() < 32) {
       operand = arith::ExtFOp::create(b, f32Ty, operand, op.getFastmathAttr());
     } else if (operandETy.getIntOrFloatBitWidth() > 32) {
-      operand = arith::TruncFOp::create(b,
-          f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
+      operand = arith::TruncFOp::create(
+          b, f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
     }
     Value f32Bits = arith::BitcastOp::create(b, i32Ty, operand);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
@@ -655,7 +656,7 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
       scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
-                                               op.getFastmathAttr());
+                                             op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
       return rewriter.notifyMatchFailure(
@@ -696,7 +697,7 @@ struct ScalingTruncFOpConverter
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
       scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
-                                               op.getFastmathAttr());
+                                             op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
       return rewriter.notifyMatchFailure(
@@ -710,9 +711,9 @@ struct ScalingTruncFOpConverter
     scaleOperand =
         arith::ExtFOp::create(b, inputTy, scaleOperand, op.getFastmathAttr());
     Value result = arith::DivFOp::create(b, inputOperand, scaleOperand,
-                                           op.getFastmathAttr());
-    Value resultCast = arith::TruncFOp::create(b,
-        resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
+                                         op.getFastmathAttr());
+    Value resultCast = arith::TruncFOp::create(
+        b, resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
     rewriter.replaceOp(op, resultCast);
     return success();
   }
diff --git a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
index 904aab33345f0..4bdd1e6a54d69 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
@@ -24,7 +24,7 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
     switch (e.getKind()) {
     case AffineExprKind::Constant:
       return ConstantIndexOp::create(b, loc,
-                                       cast<AffineConstantExpr>(e).getValue());
+                                     cast<AffineConstantExpr>(e).getValue());
     case AffineExprKind::DimId:
       return operands[cast<AffineDimExpr>(e).getPosition()];
     case AffineExprKind::SymbolId:
@@ -33,27 +33,27 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
     case AffineExprKind::Add: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return AddIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                              buildExpr(binaryExpr.getRHS()));
+                            buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mul: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return MulIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                              buildExpr(binaryExpr.getRHS()));
+                            buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::FloorDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return DivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                               buildExpr(binaryExpr.getRHS()));
+                             buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::CeilDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return CeilDivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                                   buildExpr(binaryExpr.getRHS()));
+                                 buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mod: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return RemSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                               buildExpr(binaryExpr.getRHS()));
+                             buildExpr(binaryExpr.getRHS()));
     }
     }
     llvm_unreachable("unsupported AffineExpr kind");
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index 6e9cc14c6ada5..208e8f0d3bca3 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -105,8 +105,8 @@ Value mlir::getValueOrCreateConstantIntOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return arith::ConstantOp::create(b,
-      loc, b.getIntegerAttr(attr.getType(), attr.getValue().getSExtValue()));
+  return arith::ConstantOp::create(
+      b, loc, b.getIntegerAttr(attr.getType(), attr.getValue().getSExtValue()));
 }
 
 Value mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc,
@@ -215,8 +215,8 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
     if (from.getType().getIntOrFloatBitWidth() > toBitwidth) {
       from = arith::TruncFOp::create(b, toFpTy, from);
     }
-    Value zero = mlir::arith::ConstantFloatOp::create(b,
-        toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
+    Value zero = mlir::arith::ConstantFloatOp::create(
+        b, toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
     return complex::CreateOp::create(b, targetType, from, zero);
   }
 
@@ -228,8 +228,8 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
     } else {
       from = arith::SIToFPOp::create(b, toFpTy, from);
     }
-    Value zero = mlir::arith::ConstantFloatOp::create(b,
-        toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
+    Value zero = mlir::arith::ConstantFloatOp::create(
+        b, toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
     return complex::CreateOp::create(b, targetType, from, zero);
   }
 
@@ -350,7 +350,7 @@ Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values) {
 Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
                     Type resultType) {
   Value one = ConstantOp::create(builder, loc, resultType,
-                                         builder.getOneAttr(resultType));
+                                 builder.getOneAttr(resultType));
   ArithBuilder arithBuilder(builder, loc);
   return std::accumulate(
       values.begin(), values.end(), one,
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
index b9a81b71fa301..2b80441b33309 100644
--- a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
@@ -229,8 +229,9 @@ class LowerContractionToNeonI8MMPattern
 
     // Initial accumulator for the final result. This is the un-tiled result if
     // tiling is done.
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, op.getResultType(), rewriter.getZeroAttr(op.getResultType()));
+    Value result =
+        arith::ConstantOp::create(rewriter, loc, op.getResultType(),
+                                  rewriter.getZeroAttr(op.getResultType()));
 
     SmallVector<int64_t> unrolledSize = *op.getShapeForUnroll();
     SmallVector<int64_t> smmlaShape = {2, 8};
@@ -281,8 +282,9 @@ class LowerContractionToNeonI8MMPattern
       if (isVecmat) {
         auto expandForSMMLA = [&](Value tiledOperand,
                                   VectorType expandedTypeType) {
-          auto emptyOperand = arith::ConstantOp::create(rewriter,
-              loc, expandedTypeType, rewriter.getZeroAttr(expandedTypeType));
+          auto emptyOperand =
+              arith::ConstantOp::create(rewriter, loc, expandedTypeType,
+                                        rewriter.getZeroAttr(expandedTypeType));
           SmallVector<int64_t> offsets(
               cast<ShapedType>(emptyOperand.getType()).getRank(), 0);
           SmallVector<int64_t> strides(
@@ -298,8 +300,8 @@ class LowerContractionToNeonI8MMPattern
       // using the instruction for unsigned by signed multiplication with
       // reversed operands.
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledAcc = vector::TransposeOp::create(rewriter,
-            loc, tiledAcc, ArrayRef<int64_t>({1, 0}));
+        tiledAcc = vector::TransposeOp::create(rewriter, loc, tiledAcc,
+                                               ArrayRef<int64_t>({1, 0}));
 
       // Collapse tiled operands to 1D vectors required by smmla intrinsic
       auto collapsedInputType =
@@ -331,8 +333,8 @@ class LowerContractionToNeonI8MMPattern
       // Because of the reversed operands the result is obtained transposed.
       // Transpose it back,
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledRes = vector::TransposeOp::create(rewriter,
-            loc, tiledRes, ArrayRef<int64_t>({1, 0}));
+        tiledRes = vector::TransposeOp::create(rewriter, loc, tiledRes,
+                                               ArrayRef<int64_t>({1, 0}));
 
       // With vecmat, only one row of tiled ACC can be inserted into the final
       // result
diff --git a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
index 23ec00196f714..e5e1312f0eb04 100644
--- a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
@@ -76,15 +76,15 @@ scf::ForOp createLoopOverTileSlices(
     std::function<Value(OpBuilder &, Location, Value, Value)> makeLoopBody) {
   OpBuilder::InsertionGuard g(rewriter);
   auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
-      loc, llvm::cast<VectorType>(initTile.getType()).getDimSize(0));
+  auto minTileSlices = arith::ConstantIndexOp::create(
+      rewriter, loc, llvm::cast<VectorType>(initTile.getType()).getDimSize(0));
   auto vscale =
       vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
   auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto numTileSlices =
       arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
-  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices, step,
-                                           ValueRange{initTile});
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices,
+                                  step, ValueRange{initTile});
   rewriter.setInsertionPointToStart(forOp.getBody());
   Value nextTile =
       makeLoopBody(rewriter, loc, /*tileSliceIndex=*/forOp.getInductionVar(),
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
index 4d03bbdabdf09..9bf026563c255 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
@@ -456,8 +456,8 @@ struct SwapVectorExtractOfArithExtend
     Value extendSource = extendOp->getOperand(0);
 
     // Create new extract from source of extend.
-    Value newExtract = vector::ExtractOp::create(rewriter,
-        loc, extendSource, extractOp.getMixedPosition());
+    Value newExtract = vector::ExtractOp::create(rewriter, loc, extendSource,
+                                                 extractOp.getMixedPosition());
 
     // Extend new extract to original result type.
     Operation *newExtend =
@@ -503,8 +503,9 @@ struct SwapVectorScalableExtractOfArithExtend
     // Create new extract from source of extend.
     VectorType extractResultVectorType =
         resultType.clone(extendSourceVectorType.getElementType());
-    Value newExtract = vector::ScalableExtractOp::create(rewriter,
-        loc, extractResultVectorType, extendSource, extractOp.getPos());
+    Value newExtract = vector::ScalableExtractOp::create(
+        rewriter, loc, extractResultVectorType, extendSource,
+        extractOp.getPos());
 
     // Extend new extract to original result type.
     Operation *newExtend =
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 3362d0ce13533..1c0eced43dc00 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -86,8 +86,9 @@ SmallVector<Value, 2> addConstantScalableOffset(OpBuilder &builder,
   return llvm::map_to_vector(
       llvm::zip_equal(indices, scalableOffsets), [&](auto pair) -> Value {
         auto [index, base] = pair;
-        auto offset = arith::MulIOp::create(builder,
-            loc, arith::ConstantIndexOp::create(builder, loc, base), vscale);
+        auto offset = arith::MulIOp::create(
+            builder, loc, arith::ConstantIndexOp::create(builder, loc, base),
+            vscale);
         return arith::AddIOp::create(builder, loc, index, offset);
       });
 }
@@ -132,8 +133,8 @@ Value extractSMEMask(OpBuilder &builder, Location loc, Value mask,
   // from the mask operands to get the parameters for this sub-tile.
   auto smeTileMaskDims = addConstantScalableOffset(
       builder, loc, createMask.getOperands(), {-smeTile.row, -smeTile.col});
-  auto smeTileCreateMask = vector::CreateMaskOp::create(builder,
-      loc, smeTile.type.clone(builder.getI1Type()), smeTileMaskDims);
+  auto smeTileCreateMask = vector::CreateMaskOp::create(
+      builder, loc, smeTile.type.clone(builder.getI1Type()), smeTileMaskDims);
   return smeTileCreateMask.getResult();
 }
 
@@ -190,8 +191,8 @@ struct LegalizeArithConstantOpsByDecomposition
 
     auto smeTileType = getSMETileTypeForElement(vectorType.getElementType());
     auto tileCount = getNumberOfSMETilesForVectorType(vectorType);
-    auto tileSplat = arith::ConstantOp::create(rewriter,
-        constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
+    auto tileSplat = arith::ConstantOp::create(
+        rewriter, constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
     SmallVector<Value> repl(tileCount, tileSplat);
     rewriter.replaceOpWithMultiple(constantOp, {repl});
 
@@ -237,12 +238,12 @@ struct LegalizeVectorOuterProductOpsByDecomposition
              decomposeToSMETiles(rewriter, vectorType, smeTileType))) {
 
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto lhs = vector::ScalableExtractOp::create(rewriter,
-          loc, sliceType, outerProductOp.getLhs(), smeTile.row);
-      auto rhs = vector::ScalableExtractOp::create(rewriter,
-          loc, sliceType, outerProductOp.getRhs(), smeTile.col);
-      auto smeOuterProduct = vector::OuterProductOp::create(rewriter,
-          loc, smeTileType, lhs, rhs,
+      auto lhs = vector::ScalableExtractOp::create(
+          rewriter, loc, sliceType, outerProductOp.getLhs(), smeTile.row);
+      auto rhs = vector::ScalableExtractOp::create(
+          rewriter, loc, sliceType, outerProductOp.getRhs(), smeTile.col);
+      auto smeOuterProduct = vector::OuterProductOp::create(
+          rewriter, loc, smeTileType, lhs, rhs,
           !accSMETiles.empty() ? accSMETiles[index] : Value{},
           outerProductOp.getKind());
 
@@ -314,8 +315,8 @@ struct LegalizeTransferReadOpsByDecomposition
     for (SMESubTile smeTile :
          decomposeToSMETiles(rewriter, vectorType, smeTileType, transposed)) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeRead = vector::TransferReadOp::create(rewriter,
-          loc, smeTileType, readOp.getBase(),
+      auto smeRead = vector::TransferReadOp::create(
+          rewriter, loc, smeTileType, readOp.getBase(),
           getSMESubTileIndices(rewriter, loc, readOp.getIndices(), smeTile),
           readOp.getPermutationMapAttr(), readOp.getPadding(), smeMask,
           readOp.getInBoundsAttr());
@@ -363,8 +364,8 @@ struct LegalizeTransferWriteOpsByDecomposition
     for (auto [index, smeTile] : llvm::enumerate(decomposeToSMETiles(
              rewriter, vectorType, smeTileType, transposed))) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeWrite = vector::TransferWriteOp::create(rewriter,
-          loc, inputSMETiles[index], destTensorOrMemref,
+      auto smeWrite = vector::TransferWriteOp::create(
+          rewriter, loc, inputSMETiles[index], destTensorOrMemref,
           getSMESubTileIndices(rewriter, loc, writeOp.getIndices(), smeTile),
           writeOp.getPermutationMapAttr(), smeMask, writeOp.getInBoundsAttr());
       if (writeOp.hasPureTensorSemantics())
@@ -478,26 +479,27 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
 
       // Where in the destination memref the current slice will be stored.
       auto storeRow = arith::AddIOp::create(rewriter, loc, sliceIndex,
-                                                     writeOp.getIndices()[0]);
-      auto storeCol =
-          arith::AddIOp::create(rewriter, loc, tileCol, writeOp.getIndices()[1]);
+                                            writeOp.getIndices()[0]);
+      auto storeCol = arith::AddIOp::create(rewriter, loc, tileCol,
+                                            writeOp.getIndices()[1]);
 
       // Extract the mask for the current slice.
       Value sliceMask = nullptr;
       if (mask) {
-        sliceMask = vector::ExtractOp::create(rewriter,
-            loc, mask, OpFoldResult(sliceIndex));
+        sliceMask = vector::ExtractOp::create(rewriter, loc, mask,
+                                              OpFoldResult(sliceIndex));
         if (sliceMaskType != sliceMask.getType())
-          sliceMask = vector::ScalableExtractOp::create(rewriter,
-              loc, sliceMaskType, sliceMask, smeTile.col);
+          sliceMask = vector::ScalableExtractOp::create(
+              rewriter, loc, sliceMaskType, sliceMask, smeTile.col);
       }
 
       // Extract and store the current slice.
       Value tile = inputSMETiles[index];
       auto slice =
           vector::ExtractOp::create(rewriter, loc, tile, tileSliceIndex);
-      vector::TransferWriteOp::create(rewriter,
-          loc, slice, writeOp.getBase(), ValueRange{storeRow, storeCol},
+      vector::TransferWriteOp::create(
+          rewriter, loc, slice, writeOp.getBase(),
+          ValueRange{storeRow, storeCol},
           AffineMapAttr::get(writeOp.getPermutationMap().dropResult(0)),
           sliceMask,
           rewriter.getBoolArrayAttr(
@@ -570,11 +572,12 @@ struct FoldExtractFromVectorOfSMELikeCreateMasks
     auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto extractionIndex = getValueOrCreateConstantIndexOp(
         rewriter, loc, extractOp.getMixedPosition()[0]);
-    auto extractionInTrueRegion = arith::CmpIOp::create(rewriter,
-        loc, rewriter.getI1Type(), arith::CmpIPredicate::slt, extractionIndex,
-        frontMaskDim);
-    auto newMaskFrontDim = arith::SelectOp::create(rewriter,
-        loc, extractionInTrueRegion, createMaskOp.getOperand(1), zero);
+    auto extractionInTrueRegion = arith::CmpIOp::create(
+        rewriter, loc, rewriter.getI1Type(), arith::CmpIPredicate::slt,
+        extractionIndex, frontMaskDim);
+    auto newMaskFrontDim =
+        arith::SelectOp::create(rewriter, loc, extractionInTrueRegion,
+                                createMaskOp.getOperand(1), zero);
 
     rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(
         extractOp, extractedMaskType,
@@ -676,9 +679,9 @@ struct LiftIllegalVectorTransposeToMemory
           return arith::MulIOp::create(rewriter, loc, vscale, dimSize);
         });
     SmallVector<Value> strides(readType.getRank(), Value(one));
-    auto readSubview = memref::SubViewOp::create(rewriter,
-        loc, illegalRead.getBase(), illegalRead.getIndices(), readSizes,
-        strides);
+    auto readSubview =
+        memref::SubViewOp::create(rewriter, loc, illegalRead.getBase(),
+                                  illegalRead.getIndices(), readSizes, strides);
 
     // Apply the transpose to all values/attributes of the transfer_read:
     // - The mask
@@ -687,13 +690,13 @@ struct LiftIllegalVectorTransposeToMemory
       // Note: The transpose for the mask should fold into the
       // vector.create_mask/constant_mask op, which will then become legal.
       mask = vector::TransposeOp::create(rewriter, loc, mask,
-                                                  transposeOp.getPermutation());
+                                         transposeOp.getPermutation());
     }
     // - The source memref
     mlir::AffineMap transposeMap = AffineMap::getPermutationMap(
         transposeOp.getPermutation(), getContext());
-    auto transposedSubview = memref::TransposeOp::create(rewriter,
-        loc, readSubview, AffineMapAttr::get(transposeMap));
+    auto transposedSubview = memref::TransposeOp::create(
+        rewriter, loc, readSubview, AffineMapAttr::get(transposeMap));
     ArrayAttr inBoundsAttr = illegalRead.getInBoundsAttr();
     // - The `in_bounds` attribute
     if (inBoundsAttr) {
@@ -706,8 +709,8 @@ struct LiftIllegalVectorTransposeToMemory
     VectorType legalReadType = resultType.clone(readType.getElementType());
     // Note: The indices are all zero as the subview is already offset.
     SmallVector<Value> readIndices(illegalRead.getIndices().size(), zero);
-    auto legalRead = vector::TransferReadOp::create(rewriter,
-        loc, legalReadType, transposedSubview, readIndices,
+    auto legalRead = vector::TransferReadOp::create(
+        rewriter, loc, legalReadType, transposedSubview, readIndices,
         illegalRead.getPermutationMapAttr(), illegalRead.getPadding(), mask,
         inBoundsAttr);
 
@@ -811,12 +814,12 @@ struct LowerIllegalTransposeStoreViaZA
       // rows of the tile after 1*vscale rows.
       Value tile = undefTile;
       for (int d = 0; d < numSlicesPerTile; ++d) {
-        Value vector = vector::ExtractOp::create(rewriter,
-            loc, transposeOp.getVector(),
-            rewriter.getIndexAttr(d + smeTile.row));
+        Value vector =
+            vector::ExtractOp::create(rewriter, loc, transposeOp.getVector(),
+                                      rewriter.getIndexAttr(d + smeTile.row));
         if (vector.getType() != smeSliceType) {
-          vector = vector::ScalableExtractOp::create(rewriter,
-              loc, smeSliceType, vector, smeTile.col);
+          vector = vector::ScalableExtractOp::create(
+              rewriter, loc, smeSliceType, vector, smeTile.col);
         }
         tile = vector::InsertOp::create(rewriter, loc, vector, tile, d);
       }
@@ -831,17 +834,17 @@ struct LowerIllegalTransposeStoreViaZA
       Value maskCols;
       if (auto mask = writeOp.getMask()) {
         auto createMask = mask.getDefiningOp<vector::CreateMaskOp>();
-        maskRows = arith::SubIOp::create(rewriter, loc, createMask.getOperand(0),
-                                                  transposedRow);
-        maskCols = arith::SubIOp::create(rewriter, loc, createMask.getOperand(1),
-                                                  transposedCol);
+        maskRows = arith::SubIOp::create(
+            rewriter, loc, createMask.getOperand(0), transposedRow);
+        maskCols = arith::SubIOp::create(
+            rewriter, loc, createMask.getOperand(1), transposedCol);
         maskCols = index::MinSOp::create(rewriter, loc, maskCols, numSlices);
       } else {
         maskRows = createVscaleMultiple(smeTileType.getDimSize(0));
         maskCols = numSlices;
       }
-      auto subMask = vector::CreateMaskOp::create(rewriter,
-          loc, smeTileType.clone(rewriter.getI1Type()),
+      auto subMask = vector::CreateMaskOp::create(
+          rewriter, loc, smeTileType.clone(rewriter.getI1Type()),
           ValueRange{maskRows, maskCols});
 
       // 4. Emit a transposed tile write.
@@ -850,8 +853,8 @@ struct LowerIllegalTransposeStoreViaZA
           arith::AddIOp::create(rewriter, loc, transposedRow, writeIndices[0]);
       Value destCol =
           arith::AddIOp::create(rewriter, loc, transposedCol, writeIndices[1]);
-      auto smeWrite = vector::TransferWriteOp::create(rewriter,
-          loc, tile, destTensorOrMemref, ValueRange{destRow, destCol},
+      auto smeWrite = vector::TransferWriteOp::create(
+          rewriter, loc, tile, destTensorOrMemref, ValueRange{destRow, destCol},
           transposeMap, subMask, writeOp.getInBounds());
 
       if (writeOp.hasPureTensorSemantics())
@@ -939,27 +942,27 @@ struct LowerColumnTransferReadToLoops
         vector::makeVscaleConstantBuilder(rewriter, loc);
     auto upperBound = createVscaleMultiple(numRows);
     auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    Value init = arith::ConstantOp::create(rewriter,
-        loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
+    Value init = arith::ConstantOp::create(
+        rewriter, loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
 
     scf::ForOp loadLoop;
     {
       OpBuilder::InsertionGuard g(rewriter);
       loadLoop = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
-                                             ValueRange{init});
+                                    ValueRange{init});
       rewriter.setInsertionPointToStart(loadLoop.getBody());
 
       auto tileSliceIndex = loadLoop.getInductionVar();
 
       auto idx0 = arith::AddIOp::create(rewriter, loc, tileSliceIndex,
-                                                 readOp.getIndices()[0]);
+                                        readOp.getIndices()[0]);
       auto idx1 = readOp.getIndices()[1];
 
-      Value scalar = memref::LoadOp::create(rewriter,
-          loc, readOp.getBase(), SmallVector<Value>({idx0, idx1}));
+      Value scalar = memref::LoadOp::create(rewriter, loc, readOp.getBase(),
+                                            SmallVector<Value>({idx0, idx1}));
 
-      Operation *updateInit = vector::InsertOp::create(rewriter,
-          loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
+      Operation *updateInit = vector::InsertOp::create(
+          rewriter, loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
 
       scf::YieldOp::create(rewriter, loc, updateInit->getResult(0));
     }
@@ -968,8 +971,8 @@ struct LowerColumnTransferReadToLoops
     // type was a 2D vector, we need to cast before returning the result. This
     // ShapeCast should cancel-out with some other ShapeCast (i.e. it's a
     // no-op).
-    auto sc = vector::ShapeCastOp::create(rewriter,
-        loc, readOp.getResult().getType(), loadLoop.getResult(0));
+    auto sc = vector::ShapeCastOp::create(
+        rewriter, loc, readOp.getResult().getType(), loadLoop.getResult(0));
 
     rewriter.replaceOp(readOp, sc);
 
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index 3ee211fc09bee..a7c69811fb7bd 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -87,8 +87,8 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     VectorType sourceType = source.getType();
     VectorType resultType = convertOp.getResult().getType();
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resultType, rewriter.getZeroAttr(resultType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resultType,
+                                             rewriter.getZeroAttr(resultType));
 
     // We want to iterate over the input vector in steps of the trailing
     // dimension. So this creates tile shape where all leading dimensions are 1,
@@ -100,15 +100,15 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     for (SmallVector<int64_t> index :
          StaticTileOffsetRange(sourceType.getShape(), tileShape)) {
       auto extractOrInsertPosition = ArrayRef(index).drop_back();
-      auto sourceVector = vector::ExtractOp::create(rewriter,
-          loc, source, extractOrInsertPosition);
+      auto sourceVector = vector::ExtractOp::create(rewriter, loc, source,
+                                                    extractOrInsertPosition);
       VectorType convertedType =
           VectorType::Builder(llvm::cast<VectorType>(sourceVector.getType()))
               .setDim(0, resultType.getShape().back());
       auto convertedVector =
           IntrOp::create(rewriter, loc, TypeRange{convertedType}, sourceVector);
       result = vector::InsertOp::create(rewriter, loc, convertedVector, result,
-                                                 extractOrInsertPosition);
+                                        extractOrInsertPosition);
     }
 
     rewriter.replaceOp(convertOp, result);
@@ -136,11 +136,11 @@ struct PselOpLowering : public ConvertOpToLLVMPattern<PselOp> {
     auto svboolType = VectorType::get(16, rewriter.getI1Type(), true);
     auto loc = pselOp.getLoc();
     auto svboolP1 = ConvertToSvboolIntrOp::create(rewriter, loc, svboolType,
-                                                           adaptor.getP1());
-    auto indexI32 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI32Type(), pselOp.getIndex());
+                                                  adaptor.getP1());
+    auto indexI32 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI32Type(), pselOp.getIndex());
     auto pselIntr = PselIntrOp::create(rewriter, loc, svboolType, svboolP1,
-                                                pselOp.getP2(), indexI32);
+                                       pselOp.getP2(), indexI32);
     rewriter.replaceOpWithNewOp<ConvertFromSvboolIntrOp>(
         pselOp, adaptor.getP1().getType(), pselIntr);
     return success();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
index 33de12ca17cfc..3a409ad9ed9d6 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
@@ -71,8 +71,8 @@ void replaceOpWithUnrealizedConversion(PatternRewriter &rewriter, TOp op,
                                        TLegalizerCallback callback) {
   replaceOpWithLegalizedOp(rewriter, op, [&](TOp newOp) {
     // Mark our `unrealized_conversion_casts` with a pass label.
-    return UnrealizedConversionCastOp::create(rewriter,
-        op.getLoc(), TypeRange{op.getResult().getType()},
+    return UnrealizedConversionCastOp::create(
+        rewriter, op.getLoc(), TypeRange{op.getResult().getType()},
         ValueRange{callback(newOp)},
         NamedAttribute(rewriter.getStringAttr(kSVELegalizerTag),
                        rewriter.getUnitAttr()));
@@ -239,8 +239,8 @@ struct LegalizeSVEMaskStoreConversion
 
     auto legalMaskType = widenScalableMaskTypeToSvbool(
         llvm::cast<VectorType>(valueToStore.getType()));
-    auto convertToSvbool = arm_sve::ConvertToSvboolOp::create(rewriter,
-        loc, legalMaskType, valueToStore);
+    auto convertToSvbool = arm_sve::ConvertToSvboolOp::create(
+        rewriter, loc, legalMaskType, valueToStore);
     // Replace this store with a conversion to a storable svbool mask [1],
     // followed by a wider store.
     replaceOpWithLegalizedOp(rewriter, storeOp,
@@ -290,8 +290,8 @@ struct LegalizeSVEMaskLoadConversion : public OpRewritePattern<memref::LoadOp> {
     replaceOpWithLegalizedOp(rewriter, loadOp, [&](memref::LoadOp newLoadOp) {
       newLoadOp.setMemRef(*legalMemref);
       newLoadOp.getResult().setType(legalMaskType);
-      return arm_sve::ConvertFromSvboolOp::create(rewriter,
-          loc, loadedMask.getType(), newLoadOp);
+      return arm_sve::ConvertFromSvboolOp::create(
+          rewriter, loc, loadedMask.getType(), newLoadOp);
     });
 
     return success();
@@ -408,8 +408,8 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
       reassoc.back().push_back(i);
     if (!memref::CollapseShapeOp::isGuaranteedCollapsible(memTy, reassoc))
       return failure();
-    Value collapsedMem = memref::CollapseShapeOp::create(rewriter,
-        readOp.getLoc(), readOp.getBase(), reassoc);
+    Value collapsedMem = memref::CollapseShapeOp::create(
+        rewriter, readOp.getLoc(), readOp.getBase(), reassoc);
 
     // Get a vector type with collapsed trailing dimensions.
     SmallVector<int64_t> shape(origVT.getShape());
@@ -424,14 +424,14 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
     auto indices = readOp.getIndices().drop_back(numCollapseDims - 1);
 
     // Create the new `transfer_read`.
-    auto newReadOp = vector::TransferReadOp::create(rewriter,
-        readOp.getLoc(), collapsedVT, collapsedMem, indices,
+    auto newReadOp = vector::TransferReadOp::create(
+        rewriter, readOp.getLoc(), collapsedVT, collapsedMem, indices,
         readOp.getPadding(),
         ArrayRef<bool>(origInBounds).drop_back(numCollapseDims - 1));
 
     // Cast back to the original vector type.
     auto toOrigShape = vector::ShapeCastOp::create(rewriter, readOp.getLoc(),
-                                                            origVT, newReadOp);
+                                                   origVT, newReadOp);
 
     rewriter.replaceOp(readOp, toOrigShape);
     return success();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
index 0dfd58c582dfd..26113693536df 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
@@ -234,25 +234,25 @@ class LowerContractionToSVEI8MMPattern
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the LHS tile.
       auto r0 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
-                                                   ArrayRef<int64_t>{i});
+                                          ArrayRef<int64_t>{i});
       auto r1 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
-                                                   ArrayRef<int64_t>{i + 1});
+                                          ArrayRef<int64_t>{i + 1});
       // Concatenate to obtain a 16 x i8 flattened sub-tile.
-      auto t = vector::ShuffleOp::create(rewriter,
-          loc, r0, r1,
+      auto t = vector::ShuffleOp::create(
+          rewriter, loc, r0, r1,
           llvm::ArrayRef<int64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
                                   14, 15});
       // Turn it into a scalable vector.
-      auto s = vector::ScalableInsertOp::create(rewriter,
-          loc, t, ub::PoisonOp::create(rewriter, loc, nxv16i8), 0);
+      auto s = vector::ScalableInsertOp::create(
+          rewriter, loc, t, ub::PoisonOp::create(rewriter, loc, nxv16i8), 0);
       // Replicate the sub-tile VSCALE times to fill the entire vector.
       auto r = arm_sve::DupQLaneOp::create(rewriter, loc, s, 0);
       lhsTile.push_back(r);
     }
 
     // "Flatten" the RHS tile from <[N]x8> to <[8*N]>.
-    auto rhs = vector::ShapeCastOp::create(rewriter,
-        maybeRhs->getLoc(),
+    auto rhs = vector::ShapeCastOp::create(
+        rewriter, maybeRhs->getLoc(),
         VectorType::get(/*shape=*/8 * N, rewriter.getI8Type(),
                         /*scalableDims=*/{true}),
         *maybeRhs);
@@ -260,8 +260,8 @@ class LowerContractionToSVEI8MMPattern
     // Extract the RHS sub-tiles with logical shape <8x[2]>.
     SmallVector<Value> rhsTile;
     for (int64_t j = 0; j < N; j += 2)
-      rhsTile.push_back(
-          vector::ScalableExtractOp::create(rewriter, loc, nxv16i8, rhs, j * 8));
+      rhsTile.push_back(vector::ScalableExtractOp::create(rewriter, loc,
+                                                          nxv16i8, rhs, j * 8));
 
     // Handy types for packing/unpacking of the accumulator tile.
     auto accRowTy = VectorType::get(/*shape=*/N, rewriter.getI32Type(),
@@ -278,9 +278,9 @@ class LowerContractionToSVEI8MMPattern
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the accumulator tile.
       auto r0 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
-                                                   ArrayRef<int64_t>{i});
+                                          ArrayRef<int64_t>{i});
       auto r1 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
-                                                   ArrayRef<int64_t>{i + 1});
+                                          ArrayRef<int64_t>{i + 1});
       Value accTileVec;
       if (mmlaOp == MMLA::MixedSwapped) {
         // We need to swap the positions of the LHS and RHS (since we don't have
@@ -295,7 +295,8 @@ class LowerContractionToSVEI8MMPattern
 
         // Interleave the rows, effectively flattening each 2x2 tile into 4
         // consecutive elements.
-        auto intrI64 = vector::InterleaveOp::create(rewriter, loc, r0I64, r1I64);
+        auto intrI64 =
+            vector::InterleaveOp::create(rewriter, loc, r0I64, r1I64);
 
         // Bitcast back to 32-bit elements.
         accTileVec =
@@ -303,8 +304,8 @@ class LowerContractionToSVEI8MMPattern
       }
       // Extract ACC sub-tiles.
       for (int64_t j = 0; j < N; j += 2)
-        accTile.push_back(vector::ScalableExtractOp::create(rewriter,
-            loc, nxv4i32, accTileVec, j * 2));
+        accTile.push_back(vector::ScalableExtractOp::create(
+            rewriter, loc, nxv4i32, accTileVec, j * 2));
     }
 
     // Emit sub-tile matrix multiplications.
@@ -322,8 +323,8 @@ class LowerContractionToSVEI8MMPattern
       // Collect a number of sub-tiles in a row.
       Value row = ub::PoisonOp::create(rewriter, loc, accRowX2Ty);
       for (int64_t j = 0; j < N / 2; ++j)
-        row = vector::ScalableInsertOp::create(rewriter,
-            loc, outTile[i * N / 2 + j], row, j * 4);
+        row = vector::ScalableInsertOp::create(
+            rewriter, loc, outTile[i * N / 2 + j], row, j * 4);
 
       // Unpack the row to obtain two rows of the output. If we have the out
       // sub-tiles transposed we obtain two consecutive output rows by
@@ -336,14 +337,15 @@ class LowerContractionToSVEI8MMPattern
         out1 = tmp.getRes2();
       } else {
         // Deinterleave by pairs.
-        auto row64 = vector::BitCastOp::create(rewriter, loc, accRowX264Ty, row);
+        auto row64 =
+            vector::BitCastOp::create(rewriter, loc, accRowX264Ty, row);
         auto deintr64 = vector::DeinterleaveOp::create(rewriter, loc, row64);
 
         // Bitcast back into 32-bit elements and insert into the result.
         out0 = vector::BitCastOp::create(rewriter, loc, accRowTy,
-                                                  deintr64.getRes1());
+                                         deintr64.getRes1());
         out1 = vector::BitCastOp::create(rewriter, loc, accRowTy,
-                                                  deintr64.getRes2());
+                                         deintr64.getRes2());
       }
       result = vector::InsertOp::create(rewriter, loc, out0, result, i * 2);
       result = vector::InsertOp::create(rewriter, loc, out1, result, i * 2 + 1);
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
index 25ce0fea50581..ddc64ea7ebf9a 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
@@ -416,7 +416,7 @@ AsyncRuntimeRefCountingPass::addDropRefInDivergentLivenessSuccessor(
 
       OpBuilder builder = OpBuilder::atBlockBegin(refCountingBlock);
       RuntimeDropRefOp::create(builder, value.getLoc(), value,
-                                       builder.getI64IntegerAttr(1));
+                               builder.getI64IntegerAttr(1));
 
       // No need to update the terminator operation.
       if (successor == refCountingBlock)
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index 0485ad99cf7d2..a01303be3e0d8 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -375,7 +375,7 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Add async.coro.suspend as a suspended block terminator.
     CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend,
-                                  branch.getDest(), coro.cleanupForDestroy);
+                          branch.getDest(), coro.cleanupForDestroy);
 
     branch.erase();
   }
@@ -383,8 +383,9 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
   // Replace the original `async.execute` with a call to outlined function.
   {
     ImplicitLocOpBuilder callBuilder(loc, execute);
-    auto callOutlinedFunc = func::CallOp::create(callBuilder,
-        func.getName(), execute.getResultTypes(), functionInputs.getArrayRef());
+    auto callOutlinedFunc = func::CallOp::create(callBuilder, func.getName(),
+                                                 execute.getResultTypes(),
+                                                 functionInputs.getArrayRef());
     execute.replaceAllUsesWith(callOutlinedFunc.getResults());
     execute.erase();
   }
@@ -586,12 +587,13 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
 
       // Assert that the awaited operands is not in the error state.
       Value isError = RuntimeIsErrorOp::create(builder, i1, operand);
-      Value notError = arith::XOrIOp::create(builder,
-          isError, arith::ConstantOp::create(builder,
-                       loc, i1, builder.getIntegerAttr(i1, 1)));
+      Value notError = arith::XOrIOp::create(
+          builder, isError,
+          arith::ConstantOp::create(builder, loc, i1,
+                                    builder.getIntegerAttr(i1, 1)));
 
       cf::AssertOp::create(builder, notError,
-                                   "Awaited async operand is in error state");
+                           "Awaited async operand is in error state");
     }
 
     // Inside the coroutine we convert await operation into coroutine suspension
@@ -614,8 +616,8 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
 
       // Add async.coro.suspend as a suspended block terminator.
       builder.setInsertionPointToEnd(suspended);
-      CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend, resume,
-                                    coro.cleanupForDestroy);
+      CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend,
+                            resume, coro.cleanupForDestroy);
 
       // Split the resume block into error checking and continuation.
       Block *continuation = rewriter.splitBlock(resume, Block::iterator(op));
@@ -624,10 +626,10 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
       builder.setInsertionPointToStart(resume);
       auto isError = RuntimeIsErrorOp::create(builder, loc, i1, operand);
       cf::CondBranchOp::create(builder, isError,
-                                       /*trueDest=*/setupSetErrorBlock(coro),
-                                       /*trueArgs=*/ArrayRef<Value>(),
-                                       /*falseDest=*/continuation,
-                                       /*falseArgs=*/ArrayRef<Value>());
+                               /*trueDest=*/setupSetErrorBlock(coro),
+                               /*trueArgs=*/ArrayRef<Value>(),
+                               /*falseDest=*/continuation,
+                               /*falseArgs=*/ArrayRef<Value>());
 
       // Make sure that replacement value will be constructed in the
       // continuation block.
@@ -757,10 +759,10 @@ class AssertOpLowering : public OpConversionPattern<cf::AssertOp> {
     Block *cont = rewriter.splitBlock(op->getBlock(), Block::iterator(op));
     rewriter.setInsertionPointToEnd(cont->getPrevNode());
     cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(),
-                                      /*trueDest=*/cont,
-                                      /*trueArgs=*/ArrayRef<Value>(),
-                                      /*falseDest=*/setupSetErrorBlock(coro),
-                                      /*falseArgs=*/ArrayRef<Value>());
+                             /*trueDest=*/cont,
+                             /*trueArgs=*/ArrayRef<Value>(),
+                             /*falseDest=*/setupSetErrorBlock(coro),
+                             /*falseArgs=*/ArrayRef<Value>());
     rewriter.eraseOp(op);
 
     return success();
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index a1deab67f6bfa..4dfba74a6dd3c 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -196,8 +196,8 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // Simply cast unranked MemRefs to ranked memrefs with 0 dimensions such
     // that we can call extract_strided_metadata on it.
     if (auto unrankedMemRefTy = dyn_cast<UnrankedMemRefType>(memref.getType()))
-      memref = memref::ReinterpretCastOp::create(builder,
-          loc, memref,
+      memref = memref::ReinterpretCastOp::create(
+          builder, loc, memref,
           /*offset=*/builder.getIndexAttr(0),
           /*sizes=*/ArrayRef<OpFoldResult>{},
           /*strides=*/ArrayRef<OpFoldResult>{});
@@ -296,8 +296,8 @@ FailureOr<Operation *> deallocation_impl::insertDeallocOpForReturnLike(
   if (memrefs.empty() && toRetain.empty())
     return op;
 
-  auto deallocOp = bufferization::DeallocOp::create(builder,
-      op->getLoc(), memrefs, conditions, toRetain);
+  auto deallocOp = bufferization::DeallocOp::create(
+      builder, op->getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
   // result values of the dealloc operation as they are always unique.
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 05a1367a6b126..825f63e7147c8 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -170,8 +170,8 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   if (llvm::isa<RankedTensorType>(shapedValue.getType())) {
     tensor = shapedValue;
   } else if (llvm::isa<MemRefType>(shapedValue.getType())) {
-    tensor = ToTensorOp::create(b,
-        loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
+    tensor = ToTensorOp::create(
+        b, loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
         shapedValue);
   } else if (llvm::isa<UnrankedTensorType>(shapedValue.getType()) ||
              llvm::isa<UnrankedMemRefType>(shapedValue.getType())) {
@@ -210,7 +210,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
 
   // Create AllocTensorOp.
   auto allocTensorOp = AllocTensorOp::create(b, loc, tensorType, dynamicSizes,
-                                               copy ? tensor : Value());
+                                             copy ? tensor : Value());
 
   // Add 'memory_space' attribute. Not needed if 'copy' operand is specified.
   if (copy)
@@ -753,8 +753,8 @@ void bufferization::replaceOpWithBufferizedValues(RewriterBase &rewriter,
       // ToTensorOp. Throughout bufferization, this ToTensorOp will gradually
       // loose all of its users and eventually DCE away.
       rewriter.setInsertionPointAfter(op);
-      replacement = bufferization::ToTensorOp::create(rewriter,
-          replacement.getLoc(), opResult.getType(), replacement);
+      replacement = bufferization::ToTensorOp::create(
+          rewriter, replacement.getLoc(), opResult.getType(), replacement);
     }
     replacements.push_back(replacement);
   }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index 835ad61f54da2..891652670dd5b 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -167,8 +167,8 @@ struct RemoveDeallocMemrefsContainedInRetained
       std::optional<bool> analysisResult =
           analysis.isSameAllocation(retained, memref);
       if (analysisResult == true) {
-        auto disjunction = arith::OrIOp::create(rewriter,
-            deallocOp.getLoc(), updatedCondition, cond);
+        auto disjunction = arith::OrIOp::create(rewriter, deallocOp.getLoc(),
+                                                updatedCondition, cond);
         rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(),
                                       disjunction);
       }
@@ -247,16 +247,16 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias
         continue;
       }
 
-      replacements.push_back(arith::ConstantOp::create(rewriter,
-          deallocOp.getLoc(), rewriter.getBoolAttr(false)));
+      replacements.push_back(arith::ConstantOp::create(
+          rewriter, deallocOp.getLoc(), rewriter.getBoolAttr(false)));
     }
 
     if (newRetainedMemrefs.size() == deallocOp.getRetained().size())
       return failure();
 
-    auto newDeallocOp = DeallocOp::create(rewriter,
-        deallocOp.getLoc(), deallocOp.getMemrefs(), deallocOp.getConditions(),
-        newRetainedMemrefs);
+    auto newDeallocOp =
+        DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
+                          deallocOp.getConditions(), newRetainedMemrefs);
     int i = 0;
     for (auto &repl : replacements) {
       if (!repl)
@@ -327,7 +327,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
 
       // Create new bufferization.dealloc op for `memref`.
       auto newDeallocOp = DeallocOp::create(rewriter, loc, memref, cond,
-                                                     deallocOp.getRetained());
+                                            deallocOp.getRetained());
       updatedConditions.push_back(
           llvm::to_vector(ValueRange(newDeallocOp.getUpdatedConditions())));
     }
@@ -337,8 +337,9 @@ struct SplitDeallocWhenNotAliasingAnyOther
       return failure();
 
     // Create bufferization.dealloc op for all remaining memrefs.
-    auto newDeallocOp = DeallocOp::create(rewriter,
-        loc, remainingMemrefs, remainingConditions, deallocOp.getRetained());
+    auto newDeallocOp =
+        DeallocOp::create(rewriter, loc, remainingMemrefs, remainingConditions,
+                          deallocOp.getRetained());
 
     // Bit-or all conditions.
     SmallVector<Value> replacements =
@@ -347,8 +348,8 @@ struct SplitDeallocWhenNotAliasingAnyOther
       assert(replacements.size() == additionalConditions.size() &&
              "expected same number of updated conditions");
       for (int64_t i = 0, e = replacements.size(); i < e; ++i) {
-        replacements[i] = arith::OrIOp::create(rewriter,
-            loc, replacements[i], additionalConditions[i]);
+        replacements[i] = arith::OrIOp::create(rewriter, loc, replacements[i],
+                                               additionalConditions[i]);
       }
     }
     rewriter.replaceOp(deallocOp, replacements);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index 52490a0b2293b..e30e094c28467 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -200,8 +200,8 @@ updateCalls(ModuleOp module,
     newOperands.append(outParams.begin(), outParams.end());
     auto newResultTypes = llvm::to_vector<6>(llvm::map_range(
         replaceWithNewCallResults, [](Value v) { return v.getType(); }));
-    auto newCall = func::CallOp::create(builder, op.getLoc(), op.getCalleeAttr(),
-                                                newResultTypes, newOperands);
+    auto newCall = func::CallOp::create(
+        builder, op.getLoc(), op.getCalleeAttr(), newResultTypes, newOperands);
     for (auto t : llvm::zip(replaceWithNewCallResults, newCall.getResults()))
       std::get<0>(t).replaceAllUsesWith(std::get<1>(t));
     op.erase();
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
index 80984f21df551..c0e0809fa4111 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
@@ -141,8 +141,9 @@ bufferization::getGlobalFor(arith::ConstantOp constantOp,
       cast<MemRefType>(getMemRefTypeWithStaticIdentityLayout(type));
   if (memorySpace)
     memrefType = MemRefType::Builder(memrefType).setMemorySpace(memorySpace);
-  auto global = memref::GlobalOp::create(globalBuilder,
-      constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
+  auto global = memref::GlobalOp::create(
+      globalBuilder, constantOp.getLoc(),
+      (Twine("__constant_") + os.str()).str(),
       /*sym_visibility=*/globalBuilder.getStringAttr("private"),
       /*type=*/memrefType,
       /*initial_value=*/cast<ElementsAttr>(constantOp.getValue()),
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 4a68ecba79fb7..91f6f25e3d096 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -434,8 +434,8 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
     // Replace all uses of the original tensor bbArg.
     rewriter.setInsertionPointToStart(block);
     if (!bbArgUses.empty()) {
-      Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
-          bbArg.getLoc(), tensorType, bbArg);
+      Value toTensorOp = bufferization::ToTensorOp::create(
+          rewriter, bbArg.getLoc(), tensorType, bbArg);
       for (OpOperand *use : bbArgUses)
         use->set(toTensorOp);
     }
@@ -466,13 +466,13 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
       if (failed(operandBufferType))
         return failure();
       rewriter.setInsertionPointAfterValue(operand);
-      Value bufferizedOperand = bufferization::ToBufferOp::create(rewriter,
-          operand.getLoc(), *operandBufferType, operand);
+      Value bufferizedOperand = bufferization::ToBufferOp::create(
+          rewriter, operand.getLoc(), *operandBufferType, operand);
       // A cast is needed if the operand and the block argument have different
       // bufferized types.
       if (type != *operandBufferType)
-        bufferizedOperand = memref::CastOp::create(rewriter,
-            operand.getLoc(), type, bufferizedOperand);
+        bufferizedOperand = memref::CastOp::create(rewriter, operand.getLoc(),
+                                                   type, bufferizedOperand);
       newOperands.push_back(bufferizedOperand);
     }
     operands.getMutableForwardedOperands().assign(newOperands);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index 42d63b66485d5..a50ddbe3a8ee8 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -119,7 +119,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
     for (func::CallOp callOp : callerMap[funcOp]) {
       rewriter.setInsertionPoint(callOp);
       auto newCallOp = func::CallOp::create(rewriter, callOp.getLoc(), funcOp,
-                                                     callOp.getOperands());
+                                            callOp.getOperands());
       SmallVector<Value> newResults;
       int64_t nextResult = 0;
       for (int64_t i = 0; i < callOp.getNumResults(); ++i) {
@@ -134,8 +134,8 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
         Type expectedType = callOp.getResult(i).getType();
         if (replacement.getType() != expectedType) {
           // A cast must be inserted at the call site.
-          replacement = memref::CastOp::create(rewriter,
-              callOp.getLoc(), expectedType, replacement);
+          replacement = memref::CastOp::create(rewriter, callOp.getLoc(),
+                                               expectedType, replacement);
         }
         newResults.push_back(replacement);
       }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
index 923cbc0e2259d..1784964cf9b95 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
@@ -169,7 +169,7 @@ LogicalResult mlir::bufferization::eliminateEmptyTensors(
           continue;
         rewriter.setInsertionPointAfterValue(replacement);
         replacement = tensor::CastOp::create(rewriter, v.getLoc(), v.getType(),
-                                                      replacement);
+                                             replacement);
       }
       // Replace the specific use of the tensor::EmptyOp.
       rewriter.modifyOpInPlace(user, [&]() {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index 025d5764c6d9e..f69efd1b3fa8c 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -319,8 +319,9 @@ struct CallOpInterface
     }
 
     // 3. Create the new CallOp.
-    Operation *newCallOp = func::CallOp::create(rewriter,
-        callOp.getLoc(), funcOp.getSymName(), resultTypes, newOperands);
+    Operation *newCallOp =
+        func::CallOp::create(rewriter, callOp.getLoc(), funcOp.getSymName(),
+                             resultTypes, newOperands);
     newCallOp->setAttrs(callOp->getAttrs());
 
     // 4. Replace the old op with the new op.
@@ -483,8 +484,8 @@ struct FuncOpInterface
 
         // Note: If `inferFunctionResultLayout = true`, casts are later folded
         // away.
-        Value toBufferOp = bufferization::ToBufferOp::create(rewriter,
-            returnOp.getLoc(), bufferizedType, returnVal);
+        Value toBufferOp = bufferization::ToBufferOp::create(
+            rewriter, returnOp.getLoc(), bufferizedType, returnVal);
         returnValues.push_back(toBufferOp);
       }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
index 83ccd573a2734..f0d65b04ee447 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
@@ -108,14 +108,14 @@ class DeallocOpConversion
     // Compute the base pointer indices, compare all retained indices to the
     // memref index to check if they alias.
     SmallVector<Value> doesNotAliasList;
-    Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(rewriter,
-        op->getLoc(), adaptor.getMemrefs()[0]);
+    Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
+        rewriter, op->getLoc(), adaptor.getMemrefs()[0]);
     for (Value retained : adaptor.getRetained()) {
-      Value retainedAsIdx =
-          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op->getLoc(),
-                                                                  retained);
-      Value doesNotAlias = arith::CmpIOp::create(rewriter,
-          op->getLoc(), arith::CmpIPredicate::ne, memrefAsIdx, retainedAsIdx);
+      Value retainedAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
+          rewriter, op->getLoc(), retained);
+      Value doesNotAlias = arith::CmpIOp::create(rewriter, op->getLoc(),
+                                                 arith::CmpIPredicate::ne,
+                                                 memrefAsIdx, retainedAsIdx);
       doesNotAliasList.push_back(doesNotAlias);
     }
 
@@ -126,27 +126,28 @@ class DeallocOpConversion
 
     // Also consider the condition given by the dealloc operation and perform a
     // conditional deallocation guarded by that value.
-    Value shouldDealloc = arith::AndIOp::create(rewriter,
-        op->getLoc(), prev, adaptor.getConditions()[0]);
+    Value shouldDealloc = arith::AndIOp::create(rewriter, op->getLoc(), prev,
+                                                adaptor.getConditions()[0]);
 
-    scf::IfOp::create(rewriter,
-        op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
-          scf::YieldOp::create(builder, loc);
-        });
+    scf::IfOp::create(rewriter, op.getLoc(), shouldDealloc,
+                      [&](OpBuilder &builder, Location loc) {
+                        memref::DeallocOp::create(builder, loc,
+                                                  adaptor.getMemrefs()[0]);
+                        scf::YieldOp::create(builder, loc);
+                      });
 
     // Compute the replacement values for the dealloc operation results. This
     // inserts an already canonicalized form of
     // `select(does_alias_with_memref(r), memref_cond, false)` for each retained
     // value r.
     SmallVector<Value> replacements;
-    Value trueVal = arith::ConstantOp::create(rewriter,
-        op->getLoc(), rewriter.getBoolAttr(true));
+    Value trueVal = arith::ConstantOp::create(rewriter, op->getLoc(),
+                                              rewriter.getBoolAttr(true));
     for (Value doesNotAlias : doesNotAliasList) {
       Value aliases =
           arith::XOrIOp::create(rewriter, op->getLoc(), doesNotAlias, trueVal);
       Value result = arith::AndIOp::create(rewriter, op->getLoc(), aliases,
-                                                    adaptor.getConditions()[0]);
+                                           adaptor.getConditions()[0]);
       replacements.push_back(result);
     }
 
@@ -230,98 +231,102 @@ class DeallocOpConversion
     // Without storing them to memrefs, we could not use for-loops but only a
     // completely unrolled version of it, potentially leading to code-size
     // blow-up.
-    Value toDeallocMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
-                                     rewriter.getIndexType()));
-    Value conditionMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getConditions().size()},
-                                     rewriter.getI1Type()));
-    Value toRetainMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
-                                     rewriter.getIndexType()));
+    Value toDeallocMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
+                        rewriter.getIndexType()));
+    Value conditionMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getConditions().size()},
+                        rewriter.getI1Type()));
+    Value toRetainMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getRetained().size()},
+                        rewriter.getIndexType()));
 
     auto getConstValue = [&](uint64_t value) -> Value {
       return arith::ConstantOp::create(rewriter, op.getLoc(),
-                                                rewriter.getIndexAttr(value));
+                                       rewriter.getIndexAttr(value));
     };
 
     // Extract the base pointers of the memrefs as indices to check for aliasing
     // at runtime.
     for (auto [i, toDealloc] : llvm::enumerate(adaptor.getMemrefs())) {
-      Value memrefAsIdx =
-          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
-                                                                  toDealloc);
+      Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
+          rewriter, op.getLoc(), toDealloc);
       memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx,
-                                       toDeallocMemref, getConstValue(i));
+                              toDeallocMemref, getConstValue(i));
     }
 
     for (auto [i, cond] : llvm::enumerate(adaptor.getConditions()))
       memref::StoreOp::create(rewriter, op.getLoc(), cond, conditionMemref,
-                                       getConstValue(i));
+                              getConstValue(i));
 
     for (auto [i, toRetain] : llvm::enumerate(adaptor.getRetained())) {
-      Value memrefAsIdx =
-          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
-                                                                  toRetain);
-      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx, toRetainMemref,
-                                       getConstValue(i));
+      Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
+          rewriter, op.getLoc(), toRetain);
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx,
+                              toRetainMemref, getConstValue(i));
     }
 
     // Cast the allocated memrefs to dynamic shape because we want only one
     // helper function no matter how many operands the bufferization.dealloc
     // has.
-    Value castedDeallocMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value castedDeallocMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toDeallocMemref);
-    Value castedCondsMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value castedCondsMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         conditionMemref);
-    Value castedRetainMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value castedRetainMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toRetainMemref);
 
-    Value deallocCondsMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
-                                     rewriter.getI1Type()));
-    Value retainCondsMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
-                                     rewriter.getI1Type()));
-
-    Value castedDeallocCondsMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value deallocCondsMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
+                        rewriter.getI1Type()));
+    Value retainCondsMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getRetained().size()},
+                        rewriter.getI1Type()));
+
+    Value castedDeallocCondsMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         deallocCondsMemref);
-    Value castedRetainCondsMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value castedRetainCondsMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         retainCondsMemref);
 
     Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>();
-    func::CallOp::create(rewriter,
-        op.getLoc(), deallocHelperFuncMap.lookup(symtableOp),
+    func::CallOp::create(
+        rewriter, op.getLoc(), deallocHelperFuncMap.lookup(symtableOp),
         SmallVector<Value>{castedDeallocMemref, castedRetainMemref,
                            castedCondsMemref, castedDeallocCondsMemref,
                            castedRetainCondsMemref});
 
     for (unsigned i = 0, e = adaptor.getMemrefs().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value shouldDealloc = memref::LoadOp::create(rewriter,
-          op.getLoc(), deallocCondsMemref, idxValue);
-      scf::IfOp::create(rewriter,
-          op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-            memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[i]);
-            scf::YieldOp::create(builder, loc);
-          });
+      Value shouldDealloc = memref::LoadOp::create(
+          rewriter, op.getLoc(), deallocCondsMemref, idxValue);
+      scf::IfOp::create(rewriter, op.getLoc(), shouldDealloc,
+                        [&](OpBuilder &builder, Location loc) {
+                          memref::DeallocOp::create(builder, loc,
+                                                    adaptor.getMemrefs()[i]);
+                          scf::YieldOp::create(builder, loc);
+                        });
     }
 
     SmallVector<Value> replacements;
     for (unsigned i = 0, e = adaptor.getRetained().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value ownership = memref::LoadOp::create(rewriter,
-          op.getLoc(), retainCondsMemref, idxValue);
+      Value ownership = memref::LoadOp::create(rewriter, op.getLoc(),
+                                               retainCondsMemref, idxValue);
       replacements.push_back(ownership);
     }
 
@@ -349,8 +354,8 @@ class DeallocOpConversion
                   ConversionPatternRewriter &rewriter) const override {
     // Lower the trivial case.
     if (adaptor.getMemrefs().empty()) {
-      Value falseVal = arith::ConstantOp::create(rewriter,
-          op.getLoc(), rewriter.getBoolAttr(false));
+      Value falseVal = arith::ConstantOp::create(rewriter, op.getLoc(),
+                                                 rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           op, SmallVector<Value>(adaptor.getRetained().size(), falseVal));
       return success();
@@ -455,18 +460,19 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
       arith::ConstantOp::create(builder, loc, builder.getBoolAttr(true));
   Value falseValue =
       arith::ConstantOp::create(builder, loc, builder.getBoolAttr(false));
-  Value toDeallocSize = memref::DimOp::create(builder, loc, toDeallocMemref, c0);
+  Value toDeallocSize =
+      memref::DimOp::create(builder, loc, toDeallocMemref, c0);
   Value toRetainSize = memref::DimOp::create(builder, loc, toRetainMemref, c0);
 
-  scf::ForOp::create(builder,
-      loc, c0, toRetainSize, c1, ValueRange(),
+  scf::ForOp::create(
+      builder, loc, c0, toRetainSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value i, ValueRange iterArgs) {
         memref::StoreOp::create(builder, loc, falseValue, retainCondsMemref, i);
         scf::YieldOp::create(builder, loc);
       });
 
-  scf::ForOp::create(builder,
-      loc, c0, toDeallocSize, c1, ValueRange(),
+  scf::ForOp::create(
+      builder, loc, c0, toDeallocSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value outerIter,
           ValueRange iterArgs) {
         Value toDealloc =
@@ -482,30 +488,28 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, toRetainSize, c1, trueValue,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value retainValue = memref::LoadOp::create(builder,
-                          loc, toRetainMemref, i);
-                      Value doesAlias = arith::CmpIOp::create(builder,
-                          loc, arith::CmpIPredicate::eq, retainValue,
+                      Value retainValue = memref::LoadOp::create(
+                          builder, loc, toRetainMemref, i);
+                      Value doesAlias = arith::CmpIOp::create(
+                          builder, loc, arith::CmpIPredicate::eq, retainValue,
                           toDealloc);
-                      scf::IfOp::create(builder,
-                          loc, doesAlias,
+                      scf::IfOp::create(
+                          builder, loc, doesAlias,
                           [&](OpBuilder &builder, Location loc) {
-                            Value retainCondValue =
-                                memref::LoadOp::create(builder,
-                                    loc, retainCondsMemref, i);
-                            Value aggregatedRetainCond =
-                                arith::OrIOp::create(builder,
-                                    loc, retainCondValue, cond);
-                            memref::StoreOp::create(builder,
-                                loc, aggregatedRetainCond, retainCondsMemref,
-                                i);
+                            Value retainCondValue = memref::LoadOp::create(
+                                builder, loc, retainCondsMemref, i);
+                            Value aggregatedRetainCond = arith::OrIOp::create(
+                                builder, loc, retainCondValue, cond);
+                            memref::StoreOp::create(builder, loc,
+                                                    aggregatedRetainCond,
+                                                    retainCondsMemref, i);
                             scf::YieldOp::create(builder, loc);
                           });
-                      Value doesntAlias = arith::CmpIOp::create(builder,
-                          loc, arith::CmpIPredicate::ne, retainValue,
+                      Value doesntAlias = arith::CmpIOp::create(
+                          builder, loc, arith::CmpIPredicate::ne, retainValue,
                           toDealloc);
-                      Value yieldValue = arith::AndIOp::create(builder,
-                          loc, iterArgs[0], doesntAlias);
+                      Value yieldValue = arith::AndIOp::create(
+                          builder, loc, iterArgs[0], doesntAlias);
                       scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
@@ -518,20 +522,20 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, outerIter, c1, noRetainAlias,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value prevDeallocValue = memref::LoadOp::create(builder,
-                          loc, toDeallocMemref, i);
-                      Value doesntAlias = arith::CmpIOp::create(builder,
-                          loc, arith::CmpIPredicate::ne, prevDeallocValue,
-                          toDealloc);
-                      Value yieldValue = arith::AndIOp::create(builder,
-                          loc, iterArgs[0], doesntAlias);
+                      Value prevDeallocValue = memref::LoadOp::create(
+                          builder, loc, toDeallocMemref, i);
+                      Value doesntAlias = arith::CmpIOp::create(
+                          builder, loc, arith::CmpIPredicate::ne,
+                          prevDeallocValue, toDealloc);
+                      Value yieldValue = arith::AndIOp::create(
+                          builder, loc, iterArgs[0], doesntAlias);
                       scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
         Value shouldDealoc = arith::AndIOp::create(builder, loc, noAlias, cond);
         memref::StoreOp::create(builder, loc, shouldDealoc, deallocCondsMemref,
-                                        outerIter);
+                                outerIter);
         scf::YieldOp::create(builder, loc);
       });
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index 71890bfea4e5d..64c178dfe76d8 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -750,19 +750,18 @@ Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
 
   // Insert a runtime check and only clone if we still don't have ownership at
   // runtime.
-  Value maybeClone =
-      builder
-          .create<scf::IfOp>(
-              memref.getLoc(), condition,
-              [&](OpBuilder &builder, Location loc) {
-                scf::YieldOp::create(builder, loc, newMemref);
-              },
-              [&](OpBuilder &builder, Location loc) {
-                Value clone =
-                    bufferization::CloneOp::create(builder, loc, newMemref);
-                scf::YieldOp::create(builder, loc, clone);
-              })
-          .getResult(0);
+  Value maybeClone = builder
+                         .create<scf::IfOp>(
+                             memref.getLoc(), condition,
+                             [&](OpBuilder &builder, Location loc) {
+                               scf::YieldOp::create(builder, loc, newMemref);
+                             },
+                             [&](OpBuilder &builder, Location loc) {
+                               Value clone = bufferization::CloneOp::create(
+                                   builder, loc, newMemref);
+                               scf::YieldOp::create(builder, loc, clone);
+                             })
+                         .getResult(0);
   Value trueVal = buildBoolValue(builder, memref.getLoc(), true);
   state.updateOwnership(maybeClone, trueVal);
   state.addMemrefToDeallocate(maybeClone, maybeClone.getParentBlock());
@@ -797,8 +796,8 @@ BufferDeallocation::handleInterface(BranchOpInterface op) {
   state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
                            toRetain);
 
-  auto deallocOp = bufferization::DeallocOp::create(builder,
-      op.getLoc(), memrefs, conditions, toRetain);
+  auto deallocOp = bufferization::DeallocOp::create(
+      builder, op.getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
   // result values of the dealloc operation as they are always unique.
@@ -885,12 +884,11 @@ BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
       builder.setInsertionPoint(op);
       Ownership ownership = state.getOwnership(operand, block);
       if (ownership.isUnique()) {
-        Value ownershipInverted = arith::XOrIOp::create(builder,
-            op.getLoc(), ownership.getIndicator(),
+        Value ownershipInverted = arith::XOrIOp::create(
+            builder, op.getLoc(), ownership.getIndicator(),
             buildBoolValue(builder, op.getLoc(), true));
-        cf::AssertOp::create(builder,
-            op.getLoc(), ownershipInverted,
-            "expected that the block does not have ownership");
+        cf::AssertOp::create(builder, op.getLoc(), ownershipInverted,
+                             "expected that the block does not have ownership");
       }
     }
   }
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
index 925c72959bd3a..0adfb51a228bb 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
@@ -49,7 +49,7 @@ Operation *complex::ComplexDialect::materializeConstant(OpBuilder &builder,
                                                         Location loc) {
   if (complex::ConstantOp::isBuildableWith(value, type)) {
     return complex::ConstantOp::create(builder, loc, type,
-                                               llvm::cast<ArrayAttr>(value));
+                                       llvm::cast<ArrayAttr>(value));
   }
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index 23f50930cb3b3..80dc0c597562d 100644
--- a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -87,8 +87,8 @@ struct CondBranchOpInterface
                                destOperands.getAsOperandRange(), toRetain);
       SmallVector<Value> adaptedConditions(
           llvm::map_range(conditions, conditionModifier));
-      auto deallocOp = bufferization::DeallocOp::create(builder,
-          condBr.getLoc(), memrefs, adaptedConditions, toRetain);
+      auto deallocOp = bufferization::DeallocOp::create(
+          builder, condBr.getLoc(), memrefs, adaptedConditions, toRetain);
       state.resetOwnerships(deallocOp.getRetained(), condBr->getBlock());
       for (auto [retained, ownership] : llvm::zip(
                deallocOp.getRetained(), deallocOp.getUpdatedConditions())) {
@@ -116,17 +116,18 @@ struct CondBranchOpInterface
         condBr.getTrueDest(), condBr.getTrueDestOperandsMutable(),
         [&](Value cond) {
           return arith::AndIOp::create(builder, condBr.getLoc(), cond,
-                                               condBr.getCondition());
+                                       condBr.getCondition());
         },
         thenMapping);
     DeallocOp elseTakenDeallocOp = insertDeallocForBranch(
         condBr.getFalseDest(), condBr.getFalseDestOperandsMutable(),
         [&](Value cond) {
-          Value trueVal = arith::ConstantOp::create(builder,
-              condBr.getLoc(), builder.getBoolAttr(true));
-          Value negation = arith::XOrIOp::create(builder,
-              condBr.getLoc(), trueVal, condBr.getCondition());
-          return arith::AndIOp::create(builder, condBr.getLoc(), cond, negation);
+          Value trueVal = arith::ConstantOp::create(builder, condBr.getLoc(),
+                                                    builder.getBoolAttr(true));
+          Value negation = arith::XOrIOp::create(
+              builder, condBr.getLoc(), trueVal, condBr.getCondition());
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond,
+                                       negation);
         },
         elseMapping);
 
@@ -143,9 +144,9 @@ struct CondBranchOpInterface
 
     for (Value retained : commonValues) {
       state.resetOwnerships(retained, condBr->getBlock());
-      Value combinedOwnership = arith::SelectOp::create(builder,
-          condBr.getLoc(), condBr.getCondition(), thenMapping[retained],
-          elseMapping[retained]);
+      Value combinedOwnership = arith::SelectOp::create(
+          builder, condBr.getLoc(), condBr.getCondition(),
+          thenMapping[retained], elseMapping[retained]);
       state.updateOwnership(retained, combinedOwnership, condBr->getBlock());
     }
 
diff --git a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
index 72c331ad96607..612e8099eaf35 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
@@ -68,7 +68,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
       TypeAttr typeAttr = TypeAttr::get(val.getType());
       fields.push_back({fieldName, typeAttr});
       emitc::FieldOp::create(rewriter, funcOp.getLoc(), fieldName, typeAttr,
-                                      argAttr);
+                             argAttr);
     }
 
     rewriter.setInsertionPointToEnd(&newClassOp.getBody().front());
diff --git a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
index 7485ba40b4344..c39e77d823b78 100644
--- a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
+++ b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
@@ -61,7 +61,8 @@ struct FuncInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    cf::BranchOp::create(builder, op->getLoc(), newDest, returnOp.getOperands());
+    cf::BranchOp::create(builder, op->getLoc(), newDest,
+                         returnOp.getOperands());
     op->erase();
   }
 
diff --git a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
index 4b70cf172b407..b6c8cdf2f495a 100644
--- a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
@@ -46,9 +46,9 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 
     // Substitute with the new result types from the corresponding FuncType
     // conversion.
-    auto newCallOp = CallOp::create(rewriter,
-        callOp.getLoc(), callOp.getCallee(), convertedResults,
-        flattenValues(adaptor.getOperands()));
+    auto newCallOp =
+        CallOp::create(rewriter, callOp.getLoc(), callOp.getCallee(),
+                       convertedResults, flattenValues(adaptor.getOperands()));
     SmallVector<ValueRange> replacements;
     size_t offset = 0;
     for (int i = 0, e = callOp->getNumResults(); i < e; ++i) {
diff --git a/mlir/lib/Dialect/Func/Utils/Utils.cpp b/mlir/lib/Dialect/Func/Utils/Utils.cpp
index 1eba777921828..f781ed2d591b4 100644
--- a/mlir/lib/Dialect/Func/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Func/Utils/Utils.cpp
@@ -44,8 +44,8 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
   for (unsigned int idx : newResultsOrder)
     newOutputTypes.push_back(origOutputTypes[idx]);
   rewriter.setInsertionPoint(funcOp);
-  auto newFuncOp = func::FuncOp::create(rewriter,
-      funcOp.getLoc(), funcOp.getName(),
+  auto newFuncOp = func::FuncOp::create(
+      rewriter, funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(newInputTypes, newOutputTypes));
 
   Region &newRegion = newFuncOp.getBody();
@@ -109,8 +109,9 @@ func::replaceCallOpWithNewOrder(RewriterBase &rewriter, func::CallOp callOp,
   // Replace the kernel call operation with a new one that has the
   // reordered arguments.
   rewriter.setInsertionPoint(callOp);
-  auto newCallOp = func::CallOp::create(rewriter,
-      callOp.getLoc(), callOp.getCallee(), newResultTypes, newArgsOrderValues);
+  auto newCallOp =
+      func::CallOp::create(rewriter, callOp.getLoc(), callOp.getCallee(),
+                           newResultTypes, newArgsOrderValues);
   newCallOp.setNoInlineAttr(callOp.getNoInlineAttr());
   for (auto &&[newIndex, origIndex] : llvm::enumerate(newResultsOrder))
     rewriter.replaceAllUsesWith(callOp.getResult(origIndex),
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index 6f0510138da20..224130736c945 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -76,9 +76,10 @@ buildPredicates(RewriterBase &rewriter, Location loc, ArrayRef<Value> activeIds,
     }
     if (activeMappingSize == availableMappingSize)
       continue;
-    Value idx = arith::ConstantIndexOp::create(rewriter, loc, activeMappingSize);
+    Value idx =
+        arith::ConstantIndexOp::create(rewriter, loc, activeMappingSize);
     Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
-                                                activeId, idx);
+                                       activeId, idx);
     predicateOps.push_back(pred);
   }
   return predicateOps;
@@ -422,7 +423,7 @@ DiagnosedSilenceableFailure createGpuLaunch(
   Value blkSizeY = blockDimY.has_value() ? createConst(blockDimY.value()) : one;
   Value blkSizeZ = blockDimZ.has_value() ? createConst(blockDimZ.value()) : one;
   launchOp = LaunchOp::create(rewriter, loc, gridSizeX, gridSizeY, gridSizeZ,
-                                       blkSizeX, blkSizeY, blkSizeZ);
+                              blkSizeX, blkSizeY, blkSizeZ);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
   TerminatorOp::create(rewriter, loc);
   return DiagnosedSilenceableFailure::success();
@@ -446,7 +447,7 @@ DiagnosedSilenceableFailure alterGpuLaunch(
   rewriter.setInsertionPointAfterValue(currentBlockdim.x);
   auto createConstValue = [&](int dim) {
     return arith::ConstantIndexOp::create(rewriter, currentBlockdim.x.getLoc(),
-                                                   dim);
+                                          dim);
   };
 
   if (gridDimX.has_value())
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index ccb5245442206..cd138401e3177 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -165,8 +165,9 @@ async::ExecuteOp addExecuteResults(async::ExecuteOp executeOp,
 
   // Clone executeOp with the extra results.
   OpBuilder builder(executeOp);
-  auto newOp = async::ExecuteOp::create(builder,
-      executeOp.getLoc(), TypeRange{resultTypes}.drop_front() /*drop token*/,
+  auto newOp = async::ExecuteOp::create(
+      builder, executeOp.getLoc(),
+      TypeRange{resultTypes}.drop_front() /*drop token*/,
       executeOp.getDependencies(), executeOp.getBodyOperands());
   IRMapping mapper;
   newOp.getRegion().getBlocks().clear();
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index 5c84a340f0e08..7b30906abc2fd 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -109,8 +109,8 @@ static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
       getFlatOffsetAndStrides(rewriter, loc, source, offsetsTemp);
   MemRefType retType = inferCastResultType(base, offset);
   return memref::ReinterpretCastOp::create(rewriter, loc, retType, base, offset,
-                                                    ArrayRef<OpFoldResult>(),
-                                                    ArrayRef<OpFoldResult>());
+                                           ArrayRef<OpFoldResult>(),
+                                           ArrayRef<OpFoldResult>());
 }
 
 static bool needFlatten(Value val) {
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index b98c060ac1da5..99f5c5b0cf139 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -195,8 +195,8 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   }
   FunctionType type =
       FunctionType::get(launchOp.getContext(), kernelOperandTypes, {});
-  auto outlinedFunc = gpu::GPUFuncOp::create(builder,
-      loc, kernelFnName, type,
+  auto outlinedFunc = gpu::GPUFuncOp::create(
+      builder, loc, kernelFnName, type,
       TypeRange(ValueRange(launchOp.getWorkgroupAttributions())),
       TypeRange(ValueRange(launchOp.getPrivateAttributions())));
   outlinedFunc->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
@@ -287,9 +287,9 @@ static void convertToLaunchFuncOp(gpu::LaunchOp launchOp,
   Value asyncToken = launchOp.getAsyncToken();
   std::optional<gpu::KernelDim3> clusterSize =
       launchOp.getClusterSizeOperandValues();
-  auto launchFunc = gpu::LaunchFuncOp::create(builder,
-      launchOp.getLoc(), kernelFunc, launchOp.getGridSizeOperandValues(),
-      launchOp.getBlockSizeOperandValues(),
+  auto launchFunc = gpu::LaunchFuncOp::create(
+      builder, launchOp.getLoc(), kernelFunc,
+      launchOp.getGridSizeOperandValues(), launchOp.getBlockSizeOperandValues(),
       launchOp.getDynamicSharedMemorySize(), operands,
       asyncToken ? asyncToken.getType() : nullptr,
       launchOp.getAsyncDependencies(), clusterSize);
@@ -416,7 +416,7 @@ class GpuKernelOutliningPass
     if (!kernelModule) {
       // If not found, create a new GPU module
       kernelModule = gpu::GPUModuleOp::create(builder, kernelFunc.getLoc(),
-                                                      kernelModuleName);
+                                              kernelModuleName);
     }
 
     // If a valid data layout spec was provided, attach it to the kernel module.
diff --git a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
index dd164e5aaa7c4..3c447337d821f 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
@@ -109,7 +109,7 @@ LogicalResult moduleSerializer(GPUModuleOp op,
     handler = moduleHandler;
   builder.setInsertionPointAfter(op);
   gpu::BinaryOp::create(builder, op.getLoc(), op.getName(), handler,
-                                builder.getArrayAttr(objects));
+                        builder.getArrayAttr(objects));
   op->erase();
   return success();
 }
diff --git a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
index 5a55a39a7748e..18c69f5f30e5d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
@@ -48,8 +48,8 @@ struct PromoteShuffleToSwizzlePattern
                                          "offset must be in the range [0, 31]");
 
     Location loc = op.getLoc();
-    Value res = amdgpu::SwizzleBitModeOp::create(rewriter,
-        loc, op.getResult(0).getType(), op.getValue(), /*andMask=*/31,
+    Value res = amdgpu::SwizzleBitModeOp::create(
+        rewriter, loc, op.getResult(0).getType(), op.getValue(), /*andMask=*/31,
         /*orMask=*/0, /*xorMask=*/offsetValue);
     Value valid = arith::ConstantIntOp::create(rewriter, loc, 1, /*width*/ 1);
     rewriter.replaceOp(op, {res, valid});
diff --git a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
index 7aa5288d8260d..d88f4d56d9009 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
@@ -53,8 +53,8 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
     lo = arith::TruncIOp::create(rewriter, valueLoc, i32, value);
 
     // Get the high bits by trunc(value >> 32).
-    auto c32 = arith::ConstantOp::create(rewriter,
-        valueLoc, rewriter.getIntegerAttr(i64, 32));
+    auto c32 = arith::ConstantOp::create(rewriter, valueLoc,
+                                         rewriter.getIntegerAttr(i64, 32));
     hi = arith::ShRUIOp::create(rewriter, valueLoc, value, c32);
     hi = arith::TruncIOp::create(rewriter, valueLoc, i32, hi);
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
index f401e6a267eef..79be247c2a6b5 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
@@ -60,17 +60,19 @@ struct GpuSubgroupIdRewriter final : OpRewritePattern<gpu::SubgroupIdOp> {
     Value tidY = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::y);
     Value tidZ = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::z);
 
-    Value dimYxIdZ = arith::MulIOp::create(rewriter, loc, indexType, dimY, tidZ);
+    Value dimYxIdZ =
+        arith::MulIOp::create(rewriter, loc, indexType, dimY, tidZ);
     Value dimYxIdZPlusIdY =
         arith::AddIOp::create(rewriter, loc, indexType, dimYxIdZ, tidY);
     Value dimYxIdZPlusIdYTimesDimX =
         arith::MulIOp::create(rewriter, loc, indexType, dimX, dimYxIdZPlusIdY);
-    Value IdXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(rewriter,
-        loc, indexType, tidX, dimYxIdZPlusIdYTimesDimX);
-    Value subgroupSize = gpu::SubgroupSizeOp::create(rewriter,
-        loc, rewriter.getIndexType(), /*upper_bound = */ nullptr);
-    Value subgroupIdOp = arith::DivUIOp::create(rewriter,
-        loc, indexType, IdXPlusDimYxIdZPlusIdYTimesDimX, subgroupSize);
+    Value IdXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(
+        rewriter, loc, indexType, tidX, dimYxIdZPlusIdYTimesDimX);
+    Value subgroupSize = gpu::SubgroupSizeOp::create(
+        rewriter, loc, rewriter.getIndexType(), /*upper_bound = */ nullptr);
+    Value subgroupIdOp =
+        arith::DivUIOp::create(rewriter, loc, indexType,
+                               IdXPlusDimYxIdZPlusIdYTimesDimX, subgroupSize);
     rewriter.replaceOp(op, {subgroupIdOp});
     return success();
   }
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index 7290f92f6fe08..b9e2dd5b19a6f 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -92,21 +92,22 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
         extracted =
             vector::ExtractOp::create(rewriter, loc, op.getValue(), startIdx);
       } else {
-        extracted = vector::ExtractStridedSliceOp::create(rewriter,
-            loc, op.getValue(), /*offsets=*/startIdx, /*sizes=*/numElems,
+        extracted = vector::ExtractStridedSliceOp::create(
+            rewriter, loc, op.getValue(), /*offsets=*/startIdx,
+            /*sizes=*/numElems,
             /*strides=*/1);
       }
 
-      Value reduce = gpu::SubgroupReduceOp::create(rewriter,
-          loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
-          op.getClusterStride());
+      Value reduce = gpu::SubgroupReduceOp::create(
+          rewriter, loc, extracted, op.getOp(), op.getUniform(),
+          op.getClusterSize(), op.getClusterStride());
       if (numElems == 1) {
         res = vector::InsertOp::create(rewriter, loc, reduce, res, startIdx);
         continue;
       }
 
-      res = vector::InsertStridedSliceOp::create(rewriter,
-          loc, reduce, res, /*offsets=*/startIdx, /*strides=*/1);
+      res = vector::InsertStridedSliceOp::create(
+          rewriter, loc, reduce, res, /*offsets=*/startIdx, /*strides=*/1);
     }
 
     rewriter.replaceOp(op, res);
@@ -138,10 +139,11 @@ struct ScalarizeSingleElementReduce final
     assert(vecTy.getRank() == 1 && "Unexpected vector type");
     assert(!vecTy.isScalable() && "Unexpected vector type");
     Location loc = op.getLoc();
-    Value extracted = vector::ExtractOp::create(rewriter, loc, op.getValue(), 0);
-    Value reduce = gpu::SubgroupReduceOp::create(rewriter,
-        loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
-        op.getClusterStride());
+    Value extracted =
+        vector::ExtractOp::create(rewriter, loc, op.getValue(), 0);
+    Value reduce = gpu::SubgroupReduceOp::create(
+        rewriter, loc, extracted, op.getOp(), op.getUniform(),
+        op.getClusterSize(), op.getClusterStride());
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, vecTy, reduce);
     return success();
   }
@@ -326,10 +328,10 @@ struct VectorSubgroupReduceToShuffles final
         static_cast<int64_t>(elementsPerShuffle), vecTy.getElementType());
     Value extendedInput = op.getValue();
     if (vecBitwidth < shuffleBitwidth) {
-      auto zero = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getZeroAttr(extendedVecTy));
-      extendedInput = vector::InsertStridedSliceOp::create(rewriter,
-          loc, extendedInput, zero, /*offsets=*/0, /*strides=*/1);
+      auto zero = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getZeroAttr(extendedVecTy));
+      extendedInput = vector::InsertStridedSliceOp::create(
+          rewriter, loc, extendedInput, zero, /*offsets=*/0, /*strides=*/1);
     }
 
     auto shuffleIntType = rewriter.getIntegerType(shuffleBitwidth);
@@ -351,8 +353,8 @@ struct VectorSubgroupReduceToShuffles final
         rewriter, loc, extendedInput, op.getOp(), *ci, packFn, unpackFn);
 
     if (vecBitwidth < shuffleBitwidth) {
-      res = vector::ExtractStridedSliceOp::create(rewriter,
-          loc, res, /*offsets=*/0, /*sizes=*/vecTy.getNumElements(),
+      res = vector::ExtractStridedSliceOp::create(
+          rewriter, loc, res, /*offsets=*/0, /*sizes=*/vecTy.getNumElements(),
           /*strides=*/1);
     }
 
@@ -378,8 +380,8 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   const bool boundCtrl = true;
   if (ci.clusterSize >= 2) {
     // Perform reduction between all lanes N <-> N+1.
-    dpp = amdgpu::DPPOp::create(rewriter,
-        loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
+    dpp = amdgpu::DPPOp::create(
+        rewriter, loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({1, 0, 3, 2}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
                                      gpu::convertReductionKind(mode), res, dpp);
@@ -387,8 +389,8 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
 
   if (ci.clusterSize >= 4) {
     // Perform reduction between all lanes N <-> N+2.
-    dpp = amdgpu::DPPOp::create(rewriter,
-        loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
+    dpp = amdgpu::DPPOp::create(
+        rewriter, loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({2, 3, 0, 1}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
                                      gpu::convertReductionKind(mode), res, dpp);
@@ -396,17 +398,18 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 8) {
     // Perform reduction between all lanes N <-> 7-N,
     // e.g lane[0] <-> lane[7], lane[1] <-> lane[6]..., lane[3] <-> lane[4].
-    dpp = amdgpu::DPPOp::create(rewriter,
-        loc, res.getType(), res, res, amdgpu::DPPPerm::row_half_mirror,
-        rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
+    dpp = amdgpu::DPPOp::create(rewriter, loc, res.getType(), res, res,
+                                amdgpu::DPPPerm::row_half_mirror,
+                                rewriter.getUnitAttr(), allRows, allBanks,
+                                boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
                                      gpu::convertReductionKind(mode), res, dpp);
   }
   if (ci.clusterSize >= 16) {
     // Perform reduction between all lanes N <-> 15-N,
     // e.g lane[0] <-> lane[15], lane[1] <-> lane[14]..., lane[7] <-> lane[8].
-    dpp = amdgpu::DPPOp::create(rewriter,
-        loc, res.getType(), res, res, amdgpu::DPPPerm::row_mirror,
+    dpp = amdgpu::DPPOp::create(
+        rewriter, loc, res.getType(), res, res, amdgpu::DPPPerm::row_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
                                      gpu::convertReductionKind(mode), res, dpp);
@@ -415,20 +418,20 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
     if (chipset.majorVersion <= 9) {
       // Broadcast last value from each row to next row.
       // Use row mask to avoid polluting rows 1 and 3.
-      dpp = amdgpu::DPPOp::create(rewriter,
-          loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_15,
-          rewriter.getUnitAttr(), 0xa, allBanks,
-          /*bound_ctrl*/ false);
+      dpp = amdgpu::DPPOp::create(rewriter, loc, res.getType(), res, res,
+                                  amdgpu::DPPPerm::row_bcast_15,
+                                  rewriter.getUnitAttr(), 0xa, allBanks,
+                                  /*bound_ctrl*/ false);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), res, dpp);
     } else if (chipset.majorVersion <= 12) {
       // Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2).
-      Value uint32Max = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
+      Value uint32Max = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
       dpp = ROCDL::PermlaneX16Op::create(rewriter, loc, res.getType(), res, res,
-                                                  uint32Max, uint32Max,
-                                                  /*fi=*/true,
-                                                  /*bound_ctrl=*/false);
+                                         uint32Max, uint32Max,
+                                         /*fi=*/true,
+                                         /*bound_ctrl=*/false);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), res, dpp);
     } else {
@@ -437,33 +440,35 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
               "this device.");
     }
     if (ci.subgroupSize == 32) {
-      Value lane31 = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
+      Value lane31 = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
+      res =
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
     }
   }
   if (ci.clusterSize >= 64) {
     if (chipset.majorVersion <= 9) {
       // Broadcast 31st lane value to rows 2 and 3.
-      dpp = amdgpu::DPPOp::create(rewriter,
-          loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_31,
-          rewriter.getUnitAttr(), 0xf, allBanks,
-          /*bound_ctrl*/ true);
+      dpp = amdgpu::DPPOp::create(rewriter, loc, res.getType(), res, res,
+                                  amdgpu::DPPPerm::row_bcast_31,
+                                  rewriter.getUnitAttr(), 0xf, allBanks,
+                                  /*bound_ctrl*/ true);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), dpp, res);
       // Obtain reduction from last rows, the previous rows are polluted.
-      Value lane63 = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
-      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
+      Value lane63 = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
+      res =
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
 
     } else if (chipset.majorVersion <= 12) {
       // Assume reduction across 32 lanes has been done.
       // Perform final reduction manually by summing values in lane 0 and
       // lane 32.
-      Value lane31 = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      Value lane63 = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
+      Value lane31 = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
+      Value lane63 = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
       lane31 =
           ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
       lane63 =
diff --git a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
index 473a2e515c12b..384d1a0ddccd2 100644
--- a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
+++ b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
@@ -27,9 +27,10 @@ WarpDistributionPattern::moveRegionToNewWarpOpAndReplaceReturns(
   // Create a new op before the existing one, with the extra operands.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(warpOp);
-  auto newWarpOp = WarpExecuteOnLane0Op::create(rewriter,
-      warpOp.getLoc(), newReturnTypes, warpOp.getLaneid(), warpOp.getWarpSize(),
-      warpOp.getArgs(), warpOp.getBody()->getArgumentTypes());
+  auto newWarpOp = WarpExecuteOnLane0Op::create(
+      rewriter, warpOp.getLoc(), newReturnTypes, warpOp.getLaneid(),
+      warpOp.getWarpSize(), warpOp.getArgs(),
+      warpOp.getBody()->getArgumentTypes());
 
   Region &opBody = warpOp.getBodyRegion();
   Region &newOpBody = newWarpOp.getBodyRegion();
diff --git a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
index b1d376c7c03db..894de4408c375 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
@@ -90,8 +90,8 @@ void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
     }
     for (auto [idx, t] : llvm::enumerate(stype.getBody())) {
       if (itype != PTXRegisterMod::Write) {
-        Value extractValue = LLVM::ExtractValueOp::create(rewriter,
-            interfaceOp->getLoc(), v, idx);
+        Value extractValue = LLVM::ExtractValueOp::create(
+            rewriter, interfaceOp->getLoc(), v, idx);
         addValue(extractValue);
       }
       if (itype == PTXRegisterMod::ReadWrite) {
@@ -132,8 +132,8 @@ LLVM::InlineAsmOp PtxBuilder::build() {
   // Replace all % with $
   llvm::replace(ptxInstruction, '%', '$');
 
-  return LLVM::InlineAsmOp::create(rewriter,
-      interfaceOp->getLoc(),
+  return LLVM::InlineAsmOp::create(
+      rewriter, interfaceOp->getLoc(),
       /*result types=*/resultTypes,
       /*operands=*/ptxOperands,
       /*asm_string=*/llvm::StringRef(ptxInstruction),
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 7d3f59d128367..feaffa34897b6 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -89,8 +89,8 @@ mlir::LLVM::lookupOrCreateFn(OpBuilder &b, Operation *moduleOp, StringRef name,
   OpBuilder::InsertionGuard g(b);
   assert(!moduleOp->getRegion(0).empty() && "expected non-empty region");
   b.setInsertionPointToStart(&moduleOp->getRegion(0).front());
-  auto funcOp = LLVM::LLVMFuncOp::create(b,
-      moduleOp->getLoc(), name,
+  auto funcOp = LLVM::LLVMFuncOp::create(
+      b, moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 
   if (symbolTables) {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index 98fb9b09f0cd5..d09c59ca3f24d 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -46,8 +46,8 @@ void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
   for (Operation *user : getOperation()->getUsers())
     if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
       LLVM::DbgValueOp::create(builder, declareOp.getLoc(), argument,
-                                       declareOp.getVarInfo(),
-                                       declareOp.getLocationExpr());
+                               declareOp.getVarInfo(),
+                               declareOp.getLocationExpr());
 }
 
 std::optional<PromotableAllocationOpInterface>
@@ -89,8 +89,8 @@ DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
   for (Attribute index : usedIndices) {
     Type elemType = destructurableType.getTypeAtIndex(index);
     assert(elemType && "used index must exist");
-    auto subAlloca = LLVM::AllocaOp::create(builder,
-        getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
+    auto subAlloca = LLVM::AllocaOp::create(
+        builder, getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
         getArraySize());
     newAllocators.push_back(subAlloca);
     slotMap.try_emplace<MemorySlot>(index, {subAlloca.getResult(), elemType});
@@ -260,14 +260,14 @@ static Value createExtractAndCast(OpBuilder &builder, Location loc,
   // Truncate the integer if the size of the target is less than the value.
   if (isBigEndian(dataLayout)) {
     uint64_t shiftAmount = srcTypeSize - targetTypeSize;
-    auto shiftConstant = LLVM::ConstantOp::create(builder,
-        loc, builder.getIntegerAttr(srcType, shiftAmount));
+    auto shiftConstant = LLVM::ConstantOp::create(
+        builder, loc, builder.getIntegerAttr(srcType, shiftAmount));
     replacement =
         builder.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
   }
 
-  replacement = LLVM::TruncOp::create(builder,
-      loc, builder.getIntegerType(targetTypeSize), replacement);
+  replacement = LLVM::TruncOp::create(
+      builder, loc, builder.getIntegerType(targetTypeSize), replacement);
 
   // Now cast the integer to the actual target type if required.
   return castIntValueToSameSizedType(builder, loc, replacement, targetType);
@@ -304,8 +304,9 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
     // On big endian systems, a store to the base pointer overwrites the most
     // significant bits. To accomodate for this, the stored value needs to be
     // shifted into the according position.
-    Value bigEndianShift = LLVM::ConstantOp::create(builder,
-        loc, builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
+    Value bigEndianShift = LLVM::ConstantOp::create(
+        builder, loc,
+        builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
     valueAsInt =
         builder.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
   }
@@ -325,8 +326,8 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
   }
 
   // Mask out the affected bits ...
-  Value mask = LLVM::ConstantOp::create(builder,
-      loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
+  Value mask = LLVM::ConstantOp::create(
+      builder, loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
   Value masked = builder.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
 
   // ... and combine the result with the new value.
@@ -656,7 +657,7 @@ void LLVM::DbgDeclareOp::visitReplacedValues(
   for (auto [op, value] : definitions) {
     builder.setInsertionPointAfter(op);
     LLVM::DbgValueOp::create(builder, getLoc(), value, getVarInfo(),
-                                     getLocationExpr());
+                             getLocationExpr());
   }
 }
 
@@ -979,8 +980,8 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetOp toReplace,
           .getResult();
 
   LLVM::MemsetOp::create(builder, toReplace.getLoc(), subslots.at(index).ptr,
-                                 toReplace.getVal(), newMemsetSizeValue,
-                                 toReplace.getIsVolatile());
+                         toReplace.getVal(), newMemsetSizeValue,
+                         toReplace.getIsVolatile());
 }
 
 template <>
@@ -991,9 +992,9 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
   auto newMemsetSizeValue =
       IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize);
 
-  LLVM::MemsetInlineOp::create(builder,
-      toReplace.getLoc(), subslots.at(index).ptr, toReplace.getVal(),
-      newMemsetSizeValue, toReplace.getIsVolatile());
+  LLVM::MemsetInlineOp::create(builder, toReplace.getLoc(),
+                               subslots.at(index).ptr, toReplace.getVal(),
+                               newMemsetSizeValue, toReplace.getIsVolatile());
 }
 
 } // namespace
@@ -1063,8 +1064,8 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       APInt memsetVal(/*numBits=*/width, /*val=*/0);
       for (unsigned loBit = 0; loBit < width; loBit += 8)
         memsetVal.insertBits(constantPattern.getValue(), loBit);
-      return LLVM::ConstantOp::create(builder,
-          op.getLoc(), IntegerAttr::get(intType, memsetVal));
+      return LLVM::ConstantOp::create(builder, op.getLoc(),
+                                      IntegerAttr::get(intType, memsetVal));
     }
 
     // If the output is a single byte, we can return the pattern directly.
@@ -1309,7 +1310,8 @@ memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot,
                          const SmallPtrSetImpl<OpOperand *> &blockingUses,
                          OpBuilder &builder, Value reachingDefinition) {
   if (op.loadsFrom(slot))
-    LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition, op.getDst());
+    LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition,
+                          op.getDst());
   return DeletionKind::Delete;
 }
 
@@ -1354,11 +1356,12 @@ template <class MemcpyLike>
 void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                MemcpyLike toReplace, Value dst, Value src,
                                Type toCpy, bool isVolatile) {
-  Value memcpySize = LLVM::ConstantOp::create(builder,
-      toReplace.getLoc(), IntegerAttr::get(toReplace.getLen().getType(),
-                                           layout.getTypeSize(toCpy)));
+  Value memcpySize =
+      LLVM::ConstantOp::create(builder, toReplace.getLoc(),
+                               IntegerAttr::get(toReplace.getLen().getType(),
+                                                layout.getTypeSize(toCpy)));
   MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize,
-                             isVolatile);
+                     isVolatile);
 }
 
 template <>
@@ -1367,8 +1370,8 @@ void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                Value src, Type toCpy, bool isVolatile) {
   Type lenType = IntegerType::get(toReplace->getContext(),
                                   toReplace.getLen().getBitWidth());
-  LLVM::MemcpyInlineOp::create(builder,
-      toReplace.getLoc(), dst, src,
+  LLVM::MemcpyInlineOp::create(
+      builder, toReplace.getLoc(), dst, src,
       IntegerAttr::get(lenType, layout.getTypeSize(toCpy)), isVolatile);
 }
 
@@ -1409,9 +1412,9 @@ memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
     SmallVector<LLVM::GEPArg> gepIndices{
         0, static_cast<int32_t>(
                cast<IntegerAttr>(index).getValue().getZExtValue())};
-    Value subslotPtrInOther = LLVM::GEPOp::create(builder,
-        op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()), slot.elemType,
-        isDst ? op.getSrc() : op.getDst(), gepIndices);
+    Value subslotPtrInOther = LLVM::GEPOp::create(
+        builder, op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()),
+        slot.elemType, isDst ? op.getSrc() : op.getDst(), gepIndices);
 
     // Then create a new memcpy out of this source pointer.
     createMemcpyLikeToReplace(builder, dataLayout, op,
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
index 1b1268da15646..1fb482b63670f 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
@@ -34,8 +34,9 @@ static void addComdat(LLVM::LLVMFuncOp &op, OpBuilder &builder,
 
   PatternRewriter::InsertionGuard guard(builder);
   builder.setInsertionPointToStart(&comdatOp.getBody().back());
-  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(builder,
-      comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
+  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(
+      builder, comdatOp.getLoc(), op.getSymName(),
+      mlir::LLVM::comdat::Comdat::Any);
   op.setComdatAttr(mlir::SymbolRefAttr::get(
       builder.getContext(), comdatName,
       mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr())));
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index 2ecef4b4c8a51..47c2441100cf9 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -113,20 +113,22 @@ handleInlinedAllocas(Operation *call,
     // scope if some are already present in the body of the caller. This is not
     // invalid IR, but LLVM cleans these up in InstCombineCalls.cpp, along with
     // other cases where the stacksave/stackrestore is redundant.
-    stackPtr = LLVM::StackSaveOp::create(builder,
-        call->getLoc(), LLVM::LLVMPointerType::get(call->getContext()));
+    stackPtr = LLVM::StackSaveOp::create(
+        builder, call->getLoc(),
+        LLVM::LLVMPointerType::get(call->getContext()));
   }
   builder.setInsertionPointToStart(callerEntryBlock);
   for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
-    auto newConstant = LLVM::ConstantOp::create(builder,
-        allocaOp->getLoc(), allocaOp.getArraySize().getType(), arraySize);
+    auto newConstant =
+        LLVM::ConstantOp::create(builder, allocaOp->getLoc(),
+                                 allocaOp.getArraySize().getType(), arraySize);
     // Insert a lifetime start intrinsic where the alloca was before moving it.
     if (shouldInsertLifetime) {
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(allocaOp);
-      LLVM::LifetimeStartOp::create(builder,
-          allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
-          allocaOp.getResult());
+      LLVM::LifetimeStartOp::create(builder, allocaOp.getLoc(),
+                                    arraySize.getValue().getLimitedValue(),
+                                    allocaOp.getResult());
     }
     allocaOp->moveAfter(newConstant);
     allocaOp.getArraySizeMutable().assign(newConstant.getResult());
@@ -142,9 +144,9 @@ handleInlinedAllocas(Operation *call,
       LLVM::StackRestoreOp::create(builder, call->getLoc(), stackPtr);
     for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
       if (shouldInsertLifetime)
-        LLVM::LifetimeEndOp::create(builder,
-            allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
-            allocaOp.getResult());
+        LLVM::LifetimeEndOp::create(builder, allocaOp.getLoc(),
+                                    arraySize.getValue().getLimitedValue(),
+                                    allocaOp.getResult());
     }
   }
 }
@@ -604,15 +606,16 @@ static Value handleByValArgumentInit(OpBuilder &builder, Location loc,
     Block *entryBlock = &(*argument.getParentRegion()->begin());
     builder.setInsertionPointToStart(entryBlock);
     Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
-                                                 builder.getI64IntegerAttr(1));
-    allocaOp = LLVM::AllocaOp::create(builder,
-        loc, argument.getType(), elementType, one, targetAlignment);
+                                         builder.getI64IntegerAttr(1));
+    allocaOp = LLVM::AllocaOp::create(builder, loc, argument.getType(),
+                                      elementType, one, targetAlignment);
   }
   // Copy the pointee to the newly allocated value.
-  Value copySize = LLVM::ConstantOp::create(builder,
-      loc, builder.getI64Type(), builder.getI64IntegerAttr(elementTypeSize));
+  Value copySize =
+      LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
+                               builder.getI64IntegerAttr(elementTypeSize));
   LLVM::MemcpyOp::create(builder, loc, allocaOp, argument, copySize,
-                                 /*isVolatile=*/false);
+                         /*isVolatile=*/false);
   return allocaOp;
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
index 6fa4f6935f11c..38a4bc844ea89 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
@@ -59,7 +59,7 @@ static void ensureDistinctSuccessors(Block &bb) {
       for (BlockArgument arg : successor.first->getArguments())
         dummyBlock->addArgument(arg.getType(), arg.getLoc());
       LLVM::BrOp::create(builder, terminator->getLoc(),
-                                 dummyBlock->getArguments(), successor.first);
+                         dummyBlock->getArguments(), successor.first);
     }
   }
 }
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
index de6c3043bbcdd..7f34f7d142d73 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
@@ -68,19 +68,19 @@ LogicalResult ExpandDivF16::matchAndRewrite(LLVM::FDivOp op,
 
   // Refine the approximation with one Newton iteration:
   // float refined = approx + (lhs - approx * rhs) * rcp;
-  Value err = LLVM::FMAOp::create(rewriter,
-      loc, approx, LLVM::FNegOp::create(rewriter, loc, rhs), lhs);
+  Value err = LLVM::FMAOp::create(
+      rewriter, loc, approx, LLVM::FNegOp::create(rewriter, loc, rhs), lhs);
   Value refined = LLVM::FMAOp::create(rewriter, loc, err, rcp, approx);
 
   // Use refined value if approx is normal (exponent neither all 0 or all 1).
-  Value mask = LLVM::ConstantOp::create(rewriter,
-      loc, i32Type, rewriter.getUI32IntegerAttr(0x7f800000));
+  Value mask = LLVM::ConstantOp::create(
+      rewriter, loc, i32Type, rewriter.getUI32IntegerAttr(0x7f800000));
   Value cast = LLVM::BitcastOp::create(rewriter, loc, i32Type, approx);
   Value exp = LLVM::AndOp::create(rewriter, loc, i32Type, cast, mask);
-  Value zero = LLVM::ConstantOp::create(rewriter,
-      loc, i32Type, rewriter.getUI32IntegerAttr(0));
-  Value pred = LLVM::OrOp::create(rewriter,
-      loc,
+  Value zero = LLVM::ConstantOp::create(rewriter, loc, i32Type,
+                                        rewriter.getUI32IntegerAttr(0));
+  Value pred = LLVM::OrOp::create(
+      rewriter, loc,
       LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, zero),
       LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, mask));
   Value result =
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 51cc1232df891..d5e2ed6bad7b1 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -78,11 +78,11 @@ static Operation *getSlice(OpBuilder &b, Location loc, Value source,
   return TypeSwitch<Type, Operation *>(source.getType())
       .Case<RankedTensorType>([&](RankedTensorType t) -> Operation * {
         return tensor::ExtractSliceOp::create(b, loc, source, offsets, sizes,
-                                                strides);
+                                              strides);
       })
       .Case<MemRefType>([&](MemRefType type) -> Operation * {
         return memref::SubViewOp::create(b, loc, source, offsets, sizes,
-                                           strides);
+                                         strides);
       })
       .Default([&](Type t) -> Operation * { return nullptr; });
 }
@@ -467,7 +467,7 @@ class RegionBuilderHelper {
     case UnaryFn::reciprocal: {
       Attribute oneAttr = builder.getOneAttr(arg.getType());
       auto one = arith::ConstantOp::create(builder, arg.getLoc(),
-                                                   ::cast<TypedAttr>(oneAttr));
+                                           ::cast<TypedAttr>(oneAttr));
       return arith::DivFOp::create(builder, arg.getLoc(), one, arg);
     }
     case UnaryFn::round:
@@ -647,7 +647,8 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
     Attribute valueAttr = parseAttribute(value, builder.getContext());
-    return arith::ConstantOp::create(builder, loc, ::cast<TypedAttr>(valueAttr));
+    return arith::ConstantOp::create(builder, loc,
+                                     ::cast<TypedAttr>(valueAttr));
   }
 
   Value index(int64_t dim) {
@@ -749,14 +750,14 @@ struct FoldFillWithTensorReshape : OpRewritePattern<TensorReshapeOp> {
     TensorReshapeOp newInit;
     if constexpr (std::is_same<TensorReshapeOp, tensor::ExpandShapeOp>::value) {
 
-      newInit = TensorReshapeOp::create(rewriter,
-          loc, reshapeOp.getResultType(), oldFill.output(),
+      newInit = TensorReshapeOp::create(
+          rewriter, loc, reshapeOp.getResultType(), oldFill.output(),
           reshapeOp.getReassociation(), reshapeOp.getOutputShape(),
           reshapeOp.getStaticOutputShape());
     } else {
-      newInit = TensorReshapeOp::create(rewriter, loc, reshapeOp.getResultType(),
-                                                 oldFill.output(),
-                                                 reshapeOp.getReassociation());
+      newInit = TensorReshapeOp::create(
+          rewriter, loc, reshapeOp.getResultType(), oldFill.output(),
+          reshapeOp.getReassociation());
     }
     rewriter.replaceOpWithNewOp<FillOp>(reshapeOp, ValueRange{oldFill.value()},
                                         ValueRange{newInit});
@@ -786,17 +787,17 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
       return rewriter.notifyMatchFailure(
           padOp, "failed to reify tensor.pad op result shape");
 
-    auto emptyTensor = tensor::EmptyOp::create(rewriter,
-        padOp.getLoc(), reifiedShape.front(),
-        padOp.getResultType().getElementType());
+    auto emptyTensor =
+        tensor::EmptyOp::create(rewriter, padOp.getLoc(), reifiedShape.front(),
+                                padOp.getResultType().getElementType());
     Value replacement =
         rewriter
             .create<FillOp>(fillOp.getLoc(), ValueRange{padValue},
                             ValueRange{emptyTensor})
             .getResult(0);
     if (replacement.getType() != padOp.getResultType()) {
-      replacement = tensor::CastOp::create(rewriter,
-          fillOp.getLoc(), padOp.getResultType(), replacement);
+      replacement = tensor::CastOp::create(rewriter, fillOp.getLoc(),
+                                           padOp.getResultType(), replacement);
     }
     rewriter.replaceOp(padOp, replacement);
     return success();
@@ -943,7 +944,7 @@ static FailureOr<FillOp> foldFillPackIntoFillOp(RewriterBase &rewriter,
     return failure();
 
   return linalg::FillOp::create(rewriter, packOp.getLoc(), fillOp.getInputs(),
-                                         packOp.getDest());
+                                packOp.getDest());
 }
 
 /// Wrapper pattern that applies foldFillPackIntoFillOp method.
@@ -1042,8 +1043,8 @@ struct FoldConcatsOfFill : public OpRewritePattern<tensor::ConcatOp> {
           concatOp, "not all operands are defined by a compatible fill op");
     }
 
-    Value outsConcat = tensor::ConcatOp::create(rewriter,
-        concatOp.getLoc(), concatOp.getDim(), allOuts);
+    Value outsConcat = tensor::ConcatOp::create(rewriter, concatOp.getLoc(),
+                                                concatOp.getDim(), allOuts);
     rewriter.replaceOpWithNewOp<linalg::FillOp>(
         concatOp, firstFillOp.getDpsInputOperand(0)->get(), outsConcat);
     return success();
@@ -1407,14 +1408,14 @@ struct EraseIdentityLinalgOp : public OpRewritePattern<OpTy> {
         // TODO: unify the two ops?
         if (sparse_tensor::getSparseTensorEncoding(returnType) ||
             sparse_tensor::getSparseTensorEncoding(resultType))
-          returnedArg = sparse_tensor::ConvertOp::create(rewriter,
-              linalgOp.getLoc(), resultType, returnedArg);
+          returnedArg = sparse_tensor::ConvertOp::create(
+              rewriter, linalgOp.getLoc(), resultType, returnedArg);
         else {
           if (!tensor::CastOp::areCastCompatible(returnedArg.getType(),
                                                  resultType))
             return failure();
-          returnedArg = tensor::CastOp::create(rewriter,
-              linalgOp.getLoc(), resultType, returnedArg);
+          returnedArg = tensor::CastOp::create(rewriter, linalgOp.getLoc(),
+                                               resultType, returnedArg);
         }
       }
       returnedArgs.push_back(returnedArg);
@@ -2147,8 +2148,8 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     }
     SmallVector<OpFoldResult> transposeResultShapes =
         applyPermutation(dims, resultPerms);
-    Value transposeInit = tensor::EmptyOp::create(rewriter,
-        transposeOp.getLoc(), transposeResultShapes,
+    Value transposeInit = tensor::EmptyOp::create(
+        rewriter, transposeOp.getLoc(), transposeResultShapes,
         broadcastInputTy.getElementType());
 
     // Create broadcast(transpose(input)).
@@ -2560,8 +2561,8 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     Operation *newOp = clone(rewriter, linalgOp, resultTypes, newOperands);
 
     // Create a tensor.cast operation back to the original type.
-    Value castBack = tensor::CastOp::create(rewriter,
-        loc, resultValue.getType(), newOp->getResult(resultNumber));
+    Value castBack = tensor::CastOp::create(
+        rewriter, loc, resultValue.getType(), newOp->getResult(resultNumber));
 
     SmallVector<Value> results(newOp->result_begin(), newOp->result_end());
     results[resultNumber] = castBack;
@@ -2924,9 +2925,9 @@ static Value reduce(OpBuilder &builder, Location loc, Value input, Value output,
          "We should have two maps: 1 for the input, 1 for the output");
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
 
-  auto genericOp = linalg::GenericOp::create(builder,
-      loc, output.getType(), input, output, indexingMaps, iteratorTypes,
-      [&](OpBuilder &b, Location loc, ValueRange args) {
+  auto genericOp = linalg::GenericOp::create(
+      builder, loc, output.getType(), input, output, indexingMaps,
+      iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
         Value result = T::create(b, loc, args[0], args[1]);
         linalg::YieldOp::create(b, loc, result);
       });
@@ -2947,9 +2948,10 @@ static Value buildSubAndExpOp(OpBuilder &builder, Location loc, Value input,
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
   // Add the affine map for the output argument.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = linalg::GenericOp::create(builder,
-      loc, input.getType(), ValueRange{input, max}, output, indexingMaps,
-      iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
+  auto genericOp = linalg::GenericOp::create(
+      builder, loc, input.getType(), ValueRange{input, max}, output,
+      indexingMaps, iteratorTypes,
+      [&](OpBuilder &b, Location loc, ValueRange args) {
         Value diff = arith::SubFOp::create(b, loc, args[0], args[1]);
         Value result = math::ExpOp::create(b, loc, diff);
         linalg::YieldOp::create(b, loc, result);
@@ -2974,9 +2976,9 @@ static Value buildDivOp(OpBuilder &builder, Location loc, Value numerator,
   assert(indexingMaps[0].isIdentity() && "Numerator map should be identity");
   // Add the affine map for the output tensor.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = linalg::GenericOp::create(builder,
-      loc, numerator.getType(), ValueRange{numerator, denominator}, output,
-      indexingMaps, iteratorTypes,
+  auto genericOp = linalg::GenericOp::create(
+      builder, loc, numerator.getType(), ValueRange{numerator, denominator},
+      output, indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value result = arith::DivFOp::create(b, loc, args[0], args[1]);
         linalg::YieldOp::create(b, loc, result);
@@ -3153,8 +3155,8 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
   int64_t filterRank = getFilterOperandRank();
   SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
   Location loc = getLoc();
-  auto filterSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getFilter(), sliceOffsets, sliceSizes, filterStrides);
+  auto filterSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getFilter(), sliceOffsets, sliceSizes, filterStrides);
   tiledOperands.emplace_back(filterSlice);
 
   SmallVector<OpFoldResult> resultOffsets, resultSizes;
@@ -3164,8 +3166,8 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getOutput(), resultOffsets, resultSizes, outputStrides);
+  auto outputSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
   SmallVector<Type> resultTypes;
@@ -3333,8 +3335,8 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
       {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
   int64_t inputRank = getInputOperandRank();
   SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
-  auto inputSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getInput(), sliceOffsets, sliceSizes, inputStrides);
+  auto inputSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getInput(), sliceOffsets, sliceSizes, inputStrides);
   tiledOperands.emplace_back(inputSlice);
 
   SmallVector<OpFoldResult> resultOffsets, resultSizes;
@@ -3344,8 +3346,8 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getOutput(), resultOffsets, resultSizes, outputStrides);
+  auto outputSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
   SmallVector<Type> resultTypes;
@@ -3504,8 +3506,8 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
                      sizes[getValueFDim()]});
   int64_t valueRank = getValueOperandRank();
   SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
-  auto valueSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getValue(), sliceOffsets, sliceSizes, sliceStrides);
+  auto valueSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getValue(), sliceOffsets, sliceSizes, sliceStrides);
   tiledOperands.emplace_back(valueSlice);
 
   SmallVector<OpFoldResult> resultOffsets, resultSizes;
@@ -3515,8 +3517,8 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> strides(outputRank, oneAttr);
-  auto outputSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getOutput(), resultOffsets, resultSizes, strides);
+  auto outputSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getOutput(), resultOffsets, resultSizes, strides);
   tiledOperands.emplace_back(outputSlice);
 
   SmallVector<Type> resultTypes;
@@ -4997,8 +4999,8 @@ PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
       createDestinationTensor(b, loc, getSource(), metadata.innerTiles,
                               metadata.innerDimsPos, metadata.outerDimsPerm);
   return PackOp::create(b, loc, getSource(), transposedDest,
-                          metadata.innerDimsPos, metadata.innerTiles,
-                          getPaddingValue(), metadata.outerDimsPerm);
+                        metadata.innerDimsPos, metadata.innerTiles,
+                        getPaddingValue(), metadata.outerDimsPerm);
 }
 
 /// Returns true if the tiles and the tiled dims are constant.
@@ -5250,18 +5252,20 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    PackOp newOp = PackOp::create(rewriter,
-        op.getLoc(), newOperands[0], newOperands[1], op.getInnerDimsPos(),
-        newMixedTileSizes, op.getPaddingValue(), op.getOuterDimsPerm());
+    PackOp newOp =
+        PackOp::create(rewriter, op.getLoc(), newOperands[0], newOperands[1],
+                       op.getInnerDimsPos(), newMixedTileSizes,
+                       op.getPaddingValue(), op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
 
     // Replace op.
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
-    Value replacement = (newResult.getType() != oldResult.getType())
-                            ? tensor::CastOp::create(rewriter,
-                                  op->getLoc(), oldResult.getType(), newResult)
-                            : newResult;
+    Value replacement =
+        (newResult.getType() != oldResult.getType())
+            ? tensor::CastOp::create(rewriter, op->getLoc(),
+                                     oldResult.getType(), newResult)
+            : newResult;
 
     rewriter.replaceOp(op, {replacement});
 
@@ -5358,7 +5362,8 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
   for (auto i :
        llvm::seq<unsigned>(0, srcType.getRank() - innerTileSizes.size())) {
     if (srcType.isDynamicDim(i))
-      mixedSizes.push_back(tensor::DimOp::create(b, loc, source, i).getResult());
+      mixedSizes.push_back(
+          tensor::DimOp::create(b, loc, source, i).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(srcType.getDimSize(i)));
   }
@@ -5381,8 +5386,8 @@ UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
   PackOrUnPackTransposeResult metadata = commonPermutationOfPackAndUnPackOp(
       *this, innerPermutation, outerPermutation);
   return UnPackOp::create(b, loc, transposedSource, getDest(),
-                            metadata.innerDimsPos, metadata.innerTiles,
-                            metadata.outerDimsPerm);
+                          metadata.innerDimsPos, metadata.innerTiles,
+                          metadata.outerDimsPerm);
 }
 
 /// Returns true if the `srcShape` or `destShape` is different from the one in
@@ -5454,8 +5459,8 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
             extractSliceUser.getResultType().getRank()) {
       OpBuilder::InsertionGuard g(rewriter);
       rewriter.setInsertionPoint(unPackOp);
-      auto newDest = tensor::ExtractSliceOp::create(rewriter,
-          unPackOp->getLoc(), unPackOp.getDest(),
+      auto newDest = tensor::ExtractSliceOp::create(
+          rewriter, unPackOp->getLoc(), unPackOp.getDest(),
           extractSliceUser.getMixedOffsets(), extractSliceUser.getMixedSizes(),
           extractSliceUser.getMixedStrides());
       rewriter.modifyOpInPlace(unPackOp, [&]() {
@@ -5475,17 +5480,17 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
     if (srcShape != unPackOp.getSourceType().getShape()) {
       auto newSrcType = unPackOp.getSourceType().clone(srcShape);
       source = tensor::CastOp::create(rewriter, loc, newSrcType,
-                                               unPackOp.getSource());
+                                      unPackOp.getSource());
     }
     Value dest = unPackOp.getDest();
     if (destShape != unPackOp.getDestType().getShape()) {
       auto newDestType = unPackOp.getDestType().clone(destShape);
-      dest =
-          tensor::CastOp::create(rewriter, loc, newDestType, unPackOp.getDest());
+      dest = tensor::CastOp::create(rewriter, loc, newDestType,
+                                    unPackOp.getDest());
     }
-    Value newOp = UnPackOp::create(rewriter,
-        loc, source, dest, unPackOp.getInnerDimsPos(), unPackOp.getMixedTiles(),
-        unPackOp.getOuterDimsPerm());
+    Value newOp = UnPackOp::create(
+        rewriter, loc, source, dest, unPackOp.getInnerDimsPos(),
+        unPackOp.getMixedTiles(), unPackOp.getOuterDimsPerm());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(
         unPackOp, unPackOp.getResult().getType(), newOp);
     return success();
@@ -5542,18 +5547,19 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    UnPackOp newOp = UnPackOp::create(rewriter,
-        op.getLoc(), sourceTensor, newOperands[1], op.getInnerDimsPos(),
-        newMixedTileSizes, op.getOuterDimsPerm());
+    UnPackOp newOp = UnPackOp::create(rewriter, op.getLoc(), sourceTensor,
+                                      newOperands[1], op.getInnerDimsPos(),
+                                      newMixedTileSizes, op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
 
     // Replace op.
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
-    Value replacement = (newResult.getType() != oldResult.getType())
-                            ? tensor::CastOp::create(rewriter,
-                                  op->getLoc(), oldResult.getType(), newResult)
-                            : newResult;
+    Value replacement =
+        (newResult.getType() != oldResult.getType())
+            ? tensor::CastOp::create(rewriter, op->getLoc(),
+                                     oldResult.getType(), newResult)
+            : newResult;
 
     rewriter.replaceOp(op, {replacement});
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index ba03d09e5072e..9c7099404deef 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -672,9 +672,10 @@ static Operation *replaceForAllWithNewSignature(
   newOuts.push_back(outputs[resultNumber]);
 
   // Create new scf.forall op
-  auto newforallOp = scf::ForallOp::create(rewriter,
-      loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
-      forallOp.getMixedStep(), newOuts, forallOp.getMapping());
+  auto newforallOp = scf::ForallOp::create(
+      rewriter, loc, forallOp.getMixedLowerBound(),
+      forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
+      forallOp.getMapping());
   rewriter.eraseBlock(newforallOp.getBody());
   newforallOp.getRegion().takeBody(forallOp.getRegion());
 
@@ -700,7 +701,7 @@ static Operation *replaceForAllWithNewSignature(
   Value dst = newforallOp.getRegionIterArgs().back();
   SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
   tensor::ParallelInsertSliceOp::create(rewriter, firstYieldOp->getLoc(), src,
-                                                 dst, offsets, sizes, strides);
+                                        dst, offsets, sizes, strides);
 
   for (auto result : llvm::enumerate(forallOp.getResults())) {
     rewriter.replaceAllUsesWith(result.value(),
@@ -3410,8 +3411,8 @@ transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
         for (auto [ofrIdx, ofr] : llvm::enumerate(getMixedSizes())) {
           if (auto attr = llvm::dyn_cast_if_present<Attribute>(ofr)) {
             if (scalableSizes[ofrIdx]) {
-              auto val = arith::ConstantIndexOp::create(b,
-                  getLoc(), cast<IntegerAttr>(attr).getInt());
+              auto val = arith::ConstantIndexOp::create(
+                  b, getLoc(), cast<IntegerAttr>(attr).getInt());
               Value vscale =
                   vector::VectorScaleOp::create(b, getLoc(), b.getIndexType());
               sizes.push_back(
@@ -3626,9 +3627,10 @@ static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter,
   SmallVector<OpFoldResult> normalizedSteps(normalizedUbs.size(),
                                             rewriter.getIndexAttr(1));
 
-  auto normalizedForallOp = scf::ForallOp::create(rewriter,
-      loc, normalizedLbs, normalizedUbs, normalizedSteps, loop.getOutputs(),
-      loop.getMapping(), [](OpBuilder &, Location, ValueRange) {});
+  auto normalizedForallOp = scf::ForallOp::create(
+      rewriter, loc, normalizedLbs, normalizedUbs, normalizedSteps,
+      loop.getOutputs(), loop.getMapping(),
+      [](OpBuilder &, Location, ValueRange) {});
 
   auto normalizedLoopIvs = normalizedForallOp.getInductionVars();
   OpBuilder::InsertionGuard g(rewriter);
@@ -4128,8 +4130,8 @@ DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target,
         target->template getParentOfType<scf::InParallelOp>());
   }
 
-  Value extracted = tensor::ExtractSliceOp::create(rewriter,
-      target.getLoc(), target.getDest(), target.getMixedOffsets(),
+  Value extracted = tensor::ExtractSliceOp::create(
+      rewriter, target.getLoc(), target.getDest(), target.getMixedOffsets(),
       target.getMixedSizes(), target.getMixedStrides());
   Value copied = rewriter
                      .create<linalg::CopyOp>(target.getLoc(),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
index 2cb8ad367f93c..3512ecd9d2eb2 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -185,8 +185,8 @@ struct SoftmaxOpInterface
     if (failed(outputBuffer))
       return failure();
     linalg::SoftmaxOp::create(rewriter, softmaxOp.getLoc(),
-                                       /*result=*/TypeRange(), *inputBuffer,
-                                       *outputBuffer, softmaxOp.getDimension());
+                              /*result=*/TypeRange(), *inputBuffer,
+                              *outputBuffer, softmaxOp.getDimension());
     replaceOpWithBufferizedValues(rewriter, op, *outputBuffer);
     return success();
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index 61c779850366f..d1eb27099db61 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -115,18 +115,18 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0, 1, 2}, {3}};
   auto reshapedFilterType =
       RankedTensorType::get({fh * fw * ic, oc}, filterType.getElementType());
-  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedFilterType, filter, filterReassocIndices);
+  Value reshapedFilter = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedOutputType, output, outputReassocIndices);
+  Value reshapedOutput = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = tensor::EmptyOp::create(rewriter,
-      loc, colTensorShape, inputType.getElementType());
+  Value colTensor = tensor::EmptyOp::create(rewriter, loc, colTensorShape,
+                                            inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
   auto nloops = colTensorShape.size();
@@ -138,8 +138,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = linalg::GenericOp::create(rewriter,
-      loc, colTensor.getType(),
+  auto img2ColTensor = linalg::GenericOp::create(
+      rewriter, loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -170,8 +170,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
-            loc, input, extractionIndices);
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder, loc, input,
+                                                   extractionIndices);
         linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
@@ -187,8 +187,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, reshapedOutputType,
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
       ArrayRef<AffineMap>{lhsMap, rhsMap, resultMap}, genericIterators,
@@ -200,8 +200,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
-      loc, outputType, result, outputReassocIndices);
+  auto reshapedResult = tensor::ExpandShapeOp::create(
+      rewriter, loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
 
@@ -244,8 +244,8 @@ rewriteInIm2Col(RewriterBase &rewriter,
     SmallVector<int64_t> targetShape = llvm::to_vector<4>(llvm::map_range(
         indices, [&](int64_t index) -> int64_t { return inputShape[index]; }));
 
-    Value outputTensor = tensor::EmptyOp::create(rewriter,
-        loc, targetShape, operandTensorType.getElementType());
+    Value outputTensor = tensor::EmptyOp::create(
+        rewriter, loc, targetShape, operandTensorType.getElementType());
 
     SmallVector<utils::IteratorType> loopAttributeTypes(
         nloops, utils::IteratorType::parallel);
@@ -255,8 +255,8 @@ rewriteInIm2Col(RewriterBase &rewriter,
             AffineMap::get(nloops, 0, exprs, rewriter.getContext())),
         AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-    auto transposedOp = linalg::GenericOp::create(rewriter,
-        loc, outputTensor.getType(),
+    auto transposedOp = linalg::GenericOp::create(
+        rewriter, loc, outputTensor.getType(),
         /*inputs=*/operand, /*outputs=*/outputTensor, indexingMaps,
         loopAttributeTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -307,11 +307,11 @@ rewriteInIm2Col(RewriterBase &rewriter,
       AffineMap::get(nloops, 0, inputExprs, rewriter.getContext()),
       AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-  Value colTensor = tensor::EmptyOp::create(rewriter,
-      loc, colTensorShape, inputType.getElementType());
+  Value colTensor = tensor::EmptyOp::create(rewriter, loc, colTensorShape,
+                                            inputType.getElementType());
 
-  auto img2ColTensor = linalg::GenericOp::create(rewriter,
-      loc, colTensor.getType(),
+  auto img2ColTensor = linalg::GenericOp::create(
+      rewriter, loc, colTensor.getType(),
       /*inputs=*/inputT, /*outputs=*/colTensor, indexingMaps,
       loopAttributeTypes,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -331,26 +331,27 @@ rewriteInIm2Col(RewriterBase &rewriter,
   auto reshapedOutputTensorType =
       RankedTensorType::get({n * c, oh * ow}, outputType.getElementType());
 
-  Value reshapedImg2ColTensor = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedImg2ColTensorType, img2ColTensor.getResult(0),
+  Value reshapedImg2ColTensor = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedImg2ColTensorType, img2ColTensor.getResult(0),
       img2ColTensorReassocIndices);
-  Value reshapedFilterTensor = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedFilterTensorType, filterT, filterReassociationIndice);
-  Value reshapedoutputTensor = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedOutputTensorType, transposedOutputTensor,
+  Value reshapedFilterTensor =
+      tensor::CollapseShapeOp::create(rewriter, loc, reshapedFilterTensorType,
+                                      filterT, filterReassociationIndice);
+  Value reshapedoutputTensor = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedOutputTensorType, transposedOutputTensor,
       outputReassociationIndice);
 
-  auto batchMatVecResult = linalg::BatchMatvecOp::create(rewriter,
-      loc, TypeRange{reshapedoutputTensor.getType()},
+  auto batchMatVecResult = linalg::BatchMatvecOp::create(
+      rewriter, loc, TypeRange{reshapedoutputTensor.getType()},
       ValueRange{reshapedImg2ColTensor, reshapedFilterTensor},
       ValueRange{reshapedoutputTensor});
 
   SmallVector<ReassociationIndices> batchMatVecReassociationIndice = {{0, 1},
                                                                       {2, 3}};
 
-  auto batchMatVecResultReshaped = tensor::ExpandShapeOp::create(rewriter,
-      loc, transposedOutputTensor.getType(), batchMatVecResult.getResult(0),
-      batchMatVecReassociationIndice);
+  auto batchMatVecResultReshaped = tensor::ExpandShapeOp::create(
+      rewriter, loc, transposedOutputTensor.getType(),
+      batchMatVecResult.getResult(0), batchMatVecReassociationIndice);
 
   Value transposedResult =
       transposeOperand(batchMatVecResultReshaped, {0, 2, 3, 1});
@@ -400,19 +401,19 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, ic * fh * fw}, inputType.getElementType());
-  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedFilterType, filter, filterReassocIndices);
+  Value reshapedFilter = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1}, {2, 3}};
   auto reshapedOutputType =
       RankedTensorType::get({n, oc, oh * ow}, outputType.getElementType());
-  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedOutputType, output, outputReassocIndices);
+  Value reshapedOutput = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedOutputType, output, outputReassocIndices);
 
   // Convert the input to a (BKN) tensor.
   SmallVector<int64_t, 4> colTensorShape = {n, ic * fh * fw, oh * ow};
-  Value colTensor = tensor::EmptyOp::create(rewriter,
-      loc, colTensorShape, inputType.getElementType());
+  Value colTensor = tensor::EmptyOp::create(rewriter, loc, colTensorShape,
+                                            inputType.getElementType());
 
   auto nloops = colTensorShape.size();
 
@@ -423,8 +424,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<AffineMap, 4> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = linalg::GenericOp::create(rewriter,
-      loc, colTensor.getType(),
+  auto img2ColTensor = linalg::GenericOp::create(
+      rewriter, loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -455,8 +456,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
 
         // im2col[n, ic*fh*fw, oh*ow] = input[n, ic, sh*oh + fh, sw*ow + fw]
         SmallVector<Value> extractionIndices{bIndex, icIndex, hIndex, wIndex};
-        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
-            loc, input, extractionIndices);
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder, loc, input,
+                                                   extractionIndices);
         linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
@@ -471,8 +472,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   auto resultMap = AffineMap::get(4, 0, {bDim, mDim, nDim}, context);
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, reshapedOutputType,
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, reshapedOutputType,
       /*inputs=*/ValueRange{reshapedFilter, img2ColTensor.getResult(0)},
       /*outputs=*/ValueRange{reshapedOutput},
       ArrayRef<AffineMap>{lhsMap, rhsMap, resultMap}, genericIterators,
@@ -484,8 +485,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
-      loc, outputType, result, outputReassocIndices);
+  auto reshapedResult = tensor::ExpandShapeOp::create(
+      rewriter, loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
 
@@ -535,18 +536,18 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, fh * fw * ic}, filterType.getElementType());
-  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedFilterType, filter, filterReassocIndices);
+  Value reshapedFilter = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedOutputType, output, outputReassocIndices);
+  Value reshapedOutput = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = tensor::EmptyOp::create(rewriter,
-      loc, colTensorShape, inputType.getElementType());
+  Value colTensor = tensor::EmptyOp::create(rewriter, loc, colTensorShape,
+                                            inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
   auto nloops = colTensorShape.size();
@@ -558,8 +559,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = linalg::GenericOp::create(rewriter,
-      loc, colTensor.getType(),
+  auto img2ColTensor = linalg::GenericOp::create(
+      rewriter, loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -590,8 +591,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
-            loc, input, extractionIndices);
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder, loc, input,
+                                                   extractionIndices);
         linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
@@ -606,8 +607,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, reshapedOutputType,
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
       ArrayRef<AffineMap>{lhsMap, rhsMap, resultMap}, genericIterators,
@@ -619,8 +620,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
-      loc, outputType, result, outputReassocIndices);
+  auto reshapedResult = tensor::ExpandShapeOp::create(
+      rewriter, loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index f21563c323c1b..76ddee4f0e9cf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -38,7 +38,7 @@ static Value createInserts(RewriterBase &rewriter, Location loc, int dim,
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
       destination = tensor::InsertOp::create(rewriter, loc, *elementIt,
-                                                      destination, indices);
+                                             destination, indices);
       ++elementIt;
     }
     return destination;
@@ -65,16 +65,16 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
       MaterializeInDestination: {
     // Note: This is the preferred way of memcpy'ing because no layout map
     // and/or memory space must be specified for the source.
-    auto materializeOp = bufferization::MaterializeInDestinationOp::create(b,
-        loc, tensorSource, memrefDest);
+    auto materializeOp = bufferization::MaterializeInDestinationOp::create(
+        b, loc, tensorSource, memrefDest);
     materializeOp.setWritable(true);
   } break;
   case linalg::BufferizeToAllocationOptions::MemcpyOp::MemrefCopy: {
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = bufferization::ToBufferOp::create(b,
-        loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
+    Value toBuffer = bufferization::ToBufferOp::create(
+        b, loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
     memref::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
@@ -82,8 +82,8 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = bufferization::ToBufferOp::create(b,
-        loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
+    Value toBuffer = bufferization::ToBufferOp::create(
+        b, loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
     linalg::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
@@ -121,14 +121,14 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
                                   yieldedValue.getType(), yieldedValue.getLoc())
             ->getResult(0);
     auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(fillValue),
-                                                  ValueRange(dest));
+                                         ValueRange(dest));
     return fillOp;
   }
 
   if (invariantYieldedValue) {
     // Padding with an invariant value.
-    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(yieldedValue),
-                                                  ValueRange(dest));
+    auto fillOp = linalg::FillOp::create(
+        rewriter, loc, ValueRange(yieldedValue), ValueRange(dest));
     return fillOp;
   }
 
@@ -137,8 +137,8 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(resultType.getRank()));
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, resultType, /*inputs=*/ValueRange(),
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, resultType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{dest}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
   Block *body = rewriter.createBlock(&genericOp->getRegion(0), {},
@@ -180,7 +180,7 @@ static SmallVector<Value> reifyOrComputeDynamicSizes(OpBuilder &b,
     if (tensorType.isDynamicDim(i))
       dynSizes.push_back(
           DimOp::create(b, value.getLoc(), value,
-                          arith::ConstantIndexOp::create(b, value.getLoc(), i)));
+                        arith::ConstantIndexOp::create(b, value.getLoc(), i)));
   }
   return dynSizes;
 }
@@ -243,14 +243,14 @@ Value linalg::bufferizeToAllocation(
       getMixedSizes(rewriter, loc, padOp.getSource());
   SmallVector<OpFoldResult> strides(padOp.getResultType().getRank(),
                                     rewriter.getIndexAttr(1));
-  Value subview = memref::SubViewOp::create(rewriter,
-      loc, alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
+  Value subview = memref::SubViewOp::create(
+      rewriter, loc, alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
   createMemcpy(rewriter, loc, padOp.getSource(), subview, options);
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
-      loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
+  Value toTensorOp = bufferization::ToTensorOp::create(
+      rewriter, loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(padOp, toTensorOp);
   return alloc;
@@ -338,8 +338,9 @@ Value linalg::bufferizeToAllocation(
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
-      loc, allocTensorOp.getResult().getType(), alloc, /*restrict=*/true,
+  Value toTensorOp = bufferization::ToTensorOp::create(
+      rewriter, loc, allocTensorOp.getResult().getType(), alloc,
+      /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(allocTensorOp, toTensorOp);
   return alloc;
@@ -394,16 +395,16 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
   RankedTensorType tensorType = cast<RankedTensorType>(generateOp.getType());
 
   // Create tensor.empty.
-  auto emptyOp =
-      EmptyOp::create(rewriter, loc, tensorType, generateOp.getDynamicExtents());
+  auto emptyOp = EmptyOp::create(rewriter, loc, tensorType,
+                                 generateOp.getDynamicExtents());
 
   // Create linalg.generic.
   SmallVector<utils::IteratorType> iteratorTypes(tensorType.getRank(),
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(tensorType.getRank()));
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, tensorType, /*inputs=*/ValueRange(),
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, tensorType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{emptyOp.getResult()}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
   Block *body = rewriter.createBlock(&genericOp->getRegion(0), {},
@@ -567,8 +568,8 @@ Value linalg::bufferizeToAllocation(
       createMemcpy(rewriter, op->getLoc(), operand->get(), alloc, options);
     }
     rewriter.modifyOpInPlace(op, [&]() {
-      auto toTensorOp = ToTensorOp::create(rewriter,
-          op->getLoc(), operand->get().getType(), alloc);
+      auto toTensorOp = ToTensorOp::create(rewriter, op->getLoc(),
+                                           operand->get().getType(), alloc);
       operand->set(toTensorOp);
       if (options.bufferizeDestinationOnly) {
         rewriter.modifyOpInPlace(toTensorOp, [&]() {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index eb2c7756dc6ae..91a297f7b9db7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -287,8 +287,8 @@ getOrCreatePackedViewOfOperand(OpBuilder &b, Location loc, PackInfo packInfo,
 
   auto empty = linalg::PackOp::createDestinationTensor(
       b, loc, opOperand->get(), innerTileSizes, innerDimsPos, outerDimsPerm);
-  auto packedOperand = linalg::PackOp::create(b,
-      loc, opOperand->get(), empty, innerDimsPos, innerTileSizes,
+  auto packedOperand = linalg::PackOp::create(
+      b, loc, opOperand->get(), empty, innerDimsPos, innerTileSizes,
       /*padding=*/std::nullopt, outerDimsPerm);
   return std::make_tuple(packedOperand, indexingMap);
 }
@@ -345,8 +345,9 @@ static GenericOp packGenericOp(RewriterBase &rewriter, GenericOp genericOp,
 
   indexingMaps.push_back(packedOutIndexingMap);
 
-  auto newGenericOp = linalg::GenericOp::create(rewriter,
-      loc, dest.getType(), inputOperands, dest, indexingMaps, iterTypes,
+  auto newGenericOp = linalg::GenericOp::create(
+      rewriter, loc, dest.getType(), inputOperands, dest, indexingMaps,
+      iterTypes,
       /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
   rewriter.cloneRegionBefore(genericOp.getRegion(), newGenericOp.getRegion(),
                              newGenericOp.getRegion().begin());
@@ -457,9 +458,9 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, linalg::PackOp packOp,
   if (!packOpDest.hasOneUse())
     return failure();
   if (auto emptyOp = packOpDest.getDefiningOp<tensor::EmptyOp>()) {
-    packOpDest = tensor::EmptyOp::create(rewriter,
-        genericOp->getLoc(), emptyOp.getMixedSizes(),
-        emptyOp.getType().getElementType());
+    packOpDest = tensor::EmptyOp::create(rewriter, genericOp->getLoc(),
+                                         emptyOp.getMixedSizes(),
+                                         emptyOp.getType().getElementType());
   } else {
     DominanceInfo dom(genericOp);
     if (!dom.properlyDominates(packOpDest, genericOp))
@@ -562,8 +563,8 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     auto empty = linalg::PackOp::createDestinationTensor(
         rewriter, loc, padOp.getSource(), mixedTiles, innerDimsPos,
         outerDimsPerm);
-    auto sourcePack = linalg::PackOp::create(rewriter,
-        loc, padOp.getSource(), empty, innerDimsPos, mixedTiles,
+    auto sourcePack = linalg::PackOp::create(
+        rewriter, loc, padOp.getSource(), empty, innerDimsPos, mixedTiles,
         /*padding=*/std::nullopt, outerDimsPerm);
 
     // If we have `outer_dims_perms` we need to adjust the padded dimensions.
@@ -579,17 +580,18 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, /*result=*/Type(), sourcePack, lowPad, highPad, paddingVal,
-        padOp.getNofold());
+    auto newPadOp =
+        tensor::PadOp::create(rewriter, loc, /*result=*/Type(), sourcePack,
+                              lowPad, highPad, paddingVal, padOp.getNofold());
 
     // If the pad has more than one user, create an unpack on the new pad to
     // replace the other uses.
     if (!padOp->hasOneUse()) {
       auto unpackEmpty = linalg::UnPackOp::createDestinationTensor(
           rewriter, loc, newPadOp, mixedTiles, innerDimsPos, outerDimsPerm);
-      Value unpackedPad = linalg::UnPackOp::create(rewriter,
-          loc, newPadOp, unpackEmpty, innerDimsPos, mixedTiles, outerDimsPerm);
+      Value unpackedPad =
+          linalg::UnPackOp::create(rewriter, loc, newPadOp, unpackEmpty,
+                                   innerDimsPos, mixedTiles, outerDimsPerm);
       rewriter.replaceAllUsesExcept(padOp, unpackedPad, sourcePack);
     }
 
@@ -719,9 +721,10 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
   auto emptyOp = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), collapseOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newPackOp = linalg::PackOp::create(rewriter,
-      packOp.getLoc(), collapseOp.getSrc(), emptyOp, projectedInnerDimsPos,
-      packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPerm);
+  auto newPackOp = linalg::PackOp::create(
+      rewriter, packOp.getLoc(), collapseOp.getSrc(), emptyOp,
+      projectedInnerDimsPos, packOp.getMixedTiles(), packOp.getPaddingValue(),
+      newOuterDimsPerm);
 
   SmallVector<ReassociationIndices> newReassocIndices = reassocIndices;
   // First apply the permutation on the reassociations of the outer dims.
@@ -735,8 +738,9 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
     nextPos += 1;
   }
 
-  auto newCollapseOp = tensor::CollapseShapeOp::create(rewriter,
-      collapseOp.getLoc(), packOp.getType(), newPackOp, newReassocIndices);
+  auto newCollapseOp = tensor::CollapseShapeOp::create(
+      rewriter, collapseOp.getLoc(), packOp.getType(), newPackOp,
+      newReassocIndices);
   rewriter.replaceOp(packOp, newCollapseOp);
 
   return success();
@@ -853,13 +857,14 @@ bubbleUpPackOpThroughExpandShape(tensor::ExpandShapeOp expandOp,
   Value destTensor = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), expandOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, /*outerDimsPerm=*/SmallVector<int64_t>{});
-  Value packedVal = linalg::PackOp::create(rewriter,
-      packOp.getLoc(), expandOp.getSrc(), destTensor, projectedInnerDimsPos,
-      packOp.getMixedTiles(), packOp.getPaddingValue(),
+  Value packedVal = linalg::PackOp::create(
+      rewriter, packOp.getLoc(), expandOp.getSrc(), destTensor,
+      projectedInnerDimsPos, packOp.getMixedTiles(), packOp.getPaddingValue(),
       /*outerDimsPerm=*/SmallVector<int64_t>{});
 
-  Value newExpandOp = tensor::ExpandShapeOp::create(rewriter,
-      packOp.getLoc(), packOp.getDestType(), packedVal, *reassocExpand);
+  Value newExpandOp = tensor::ExpandShapeOp::create(rewriter, packOp.getLoc(),
+                                                    packOp.getDestType(),
+                                                    packedVal, *reassocExpand);
   rewriter.replaceOp(packOp, newExpandOp);
 
   return success();
@@ -972,15 +977,15 @@ static LogicalResult pushDownUnPackOpThroughExpandShape(
 
   RankedTensorType newExpandType = linalg::PackOp::inferPackedType(
       expandTy, innerTileSizes, projectedInnerDimsPos, newOuterDimsPerm);
-  auto newExpandOp = tensor::ExpandShapeOp::create(rewriter,
-      expandOp.getLoc(), newExpandType, unPackOp.getSource(),
-      newReassocIndices);
+  auto newExpandOp =
+      tensor::ExpandShapeOp::create(rewriter, expandOp.getLoc(), newExpandType,
+                                    unPackOp.getSource(), newReassocIndices);
 
   auto emptyOp = linalg::UnPackOp::createDestinationTensor(
       rewriter, unPackOp.getLoc(), newExpandOp, unPackOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newUnPackOp = linalg::UnPackOp::create(rewriter,
-      unPackOp.getLoc(), newExpandOp.getResult(), emptyOp,
+  auto newUnPackOp = linalg::UnPackOp::create(
+      rewriter, unPackOp.getLoc(), newExpandOp.getResult(), emptyOp,
       projectedInnerDimsPos, unPackOp.getMixedTiles(), newOuterDimsPerm);
   rewriter.replaceOp(expandOp, newUnPackOp);
 
@@ -1212,17 +1217,17 @@ struct PushDownUnPackThroughPadOp : public OpRewritePattern<tensor::PadOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, /*result=*/Type(), unpackOp.getSource(), lowPad, highPad,
-        paddingVal, padOp.getNofold());
+    auto newPadOp = tensor::PadOp::create(rewriter, loc, /*result=*/Type(),
+                                          unpackOp.getSource(), lowPad, highPad,
+                                          paddingVal, padOp.getNofold());
 
     // Inject the linalg.unpack right after the packed padOp.
-    Value outputUnPack = tensor::EmptyOp::create(rewriter,
-        loc, padOp.getResultType().getShape(),
-        padOp.getResultType().getElementType());
+    Value outputUnPack =
+        tensor::EmptyOp::create(rewriter, loc, padOp.getResultType().getShape(),
+                                padOp.getResultType().getElementType());
 
-    Value replacement = linalg::UnPackOp::create(rewriter,
-        loc, newPadOp.getResult(), outputUnPack, innerDimsPos,
+    Value replacement = linalg::UnPackOp::create(
+        rewriter, loc, newPadOp.getResult(), outputUnPack, innerDimsPos,
         unpackOp.getMixedTiles(), outerDimsPerm);
     rewriter.replaceOp(padOp, replacement);
     return success();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
index 82d386b3b9099..b7da20c108f3e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
@@ -201,7 +201,7 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
           tensor::EmptyOp::create(rewriter, loc, transposedShape, elType);
 
       auto transposeOp = TransposeOp::create(rewriter, loc, newInitValues[i],
-                                                      emptyTensor, permutation);
+                                             emptyTensor, permutation);
       newInitValues[i] = transposeOp->getResult(0);
       isChanged = true;
     }
@@ -209,11 +209,11 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
     // Does it require broadcast?
     if (!broadcastedDims.empty()) {
       assert(broadcastedDims.size() && "should have non size broadcast");
-      Value emptyTensor = tensor::EmptyOp::create(rewriter,
-          loc, outputShape, inputRTType.getElementType());
+      Value emptyTensor = tensor::EmptyOp::create(rewriter, loc, outputShape,
+                                                  inputRTType.getElementType());
 
-      auto broadcastOp = linalg::BroadcastOp::create(rewriter,
-          loc, newInitValues[i], emptyTensor, broadcastedDims);
+      auto broadcastOp = linalg::BroadcastOp::create(
+          rewriter, loc, newInitValues[i], emptyTensor, broadcastedDims);
 
       newInitValues[i] = broadcastOp->getResult(0);
       isChanged = true;
@@ -227,7 +227,8 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
   SmallVector<Value> operands = op->getOperands();
   ValueRange operandsRef(operands);
 
-  auto newOp = linalg::GenericOp::create(rewriter,
+  auto newOp = linalg::GenericOp::create(
+      rewriter,
       /*location=*/op.getLoc(),
       /*resultTensorTypes=*/op->getResultTypes(),
       /*inputs=*/newInitValues,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
index bd3fad01aa48b..c92a27f0af858 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
@@ -138,8 +138,8 @@ static Value getZero(OpBuilder &b, Location loc, Type elementType) {
     return arith::ConstantIndexOp::create(b, loc, 0);
   // Assume float.
   auto floatType = cast<FloatType>(elementType);
-  return arith::ConstantFloatOp::create(b,
-      loc, floatType, APFloat::getZero(floatType.getFloatSemantics()));
+  return arith::ConstantFloatOp::create(
+      b, loc, floatType, APFloat::getZero(floatType.getFloatSemantics()));
 }
 
 GenericOp
@@ -188,8 +188,8 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
 
     // Fall back path, use an `init_tensor` and identity indexing map.
     AffineMap indexingMap = rewriter.getMultiDimIdentityMap(domain.size());
-    Value emptyTensor =
-        tensor::EmptyOp::create(rewriter, loc, domain, scalarOpResult.getType());
+    Value emptyTensor = tensor::EmptyOp::create(rewriter, loc, domain,
+                                                scalarOpResult.getType());
     newInitValues.push_back(emptyTensor);
     newResultTypes.push_back(emptyTensor.getType());
     peeledGenericOpIndexingMaps.push_back(indexingMap);
@@ -202,10 +202,10 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
   resultTypes.append(newResultTypes.begin(), newResultTypes.end());
   auto indexingMapAttr =
       rewriter.getAffineMapArrayAttr(peeledGenericOpIndexingMaps);
-  return GenericOp::create(rewriter,
-      loc, resultTypes, genericOp.getInputs(), outsOperands, indexingMapAttr,
-      genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
-      [](OpBuilder, Location, ValueRange) {});
+  return GenericOp::create(
+      rewriter, loc, resultTypes, genericOp.getInputs(), outsOperands,
+      indexingMapAttr, genericOp.getIteratorTypes(), /*doc=*/nullptr,
+      /*libraryCall=*/nullptr, [](OpBuilder, Location, ValueRange) {});
 }
 
 GenericOp
@@ -239,8 +239,8 @@ DecomposeLinalgOp::createResidualGenericOp(GenericOp genericOp,
     indexingMaps.push_back(genericOp.getMatchingIndexingMap(&outOperand));
 
   auto indexingMapAttr = rewriter.getAffineMapArrayAttr(indexingMaps);
-  return GenericOp::create(rewriter,
-      genericOp->getLoc(), genericOp->getResultTypes(),
+  return GenericOp::create(
+      rewriter, genericOp->getLoc(), genericOp->getResultTypes(),
       residualGenericOpOperands, genericOp.getOutputs(), indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
       [](OpBuilder, Location, ValueRange) {});
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index b8e73bd5929dd..830905495e759 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -34,8 +34,8 @@ static Value sourceMaterializationCallback(OpBuilder &builder, Type type,
 
   // A detensored value is converted back by creating a new tensor from its
   // element(s).
-  return tensor::FromElementsOp::create(builder,
-      loc, RankedTensorType::get({}, inputType), inputs[0]);
+  return tensor::FromElementsOp::create(
+      builder, loc, RankedTensorType::get({}, inputType), inputs[0]);
 }
 
 namespace {
@@ -480,8 +480,8 @@ struct LinalgDetensorize
     Block *postEntryBlock =
         rewriter.splitBlock(entryBlock, entryBlock->begin());
     rewriter.setInsertionPointToStart(entryBlock);
-    auto branch =
-        cf::BranchOp::create(rewriter, rewriter.getUnknownLoc(), postEntryBlock);
+    auto branch = cf::BranchOp::create(rewriter, rewriter.getUnknownLoc(),
+                                       postEntryBlock);
 
     if (aggressiveMode.getValue()) {
       AggressiveDetensoringModel costModel;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index fd8c177cc7247..7dbae83e993e4 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -118,16 +118,17 @@ struct MoveInitOperandsToInput : public OpRewritePattern<GenericOp> {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointAfterValue(op->get());
       auto elemType = cast<ShapedType>(op->get().getType()).getElementType();
-      auto empty = tensor::EmptyOp::create(rewriter,
-          loc, tensor::getMixedSizes(rewriter, loc, op->get()), elemType);
+      auto empty = tensor::EmptyOp::create(
+          rewriter, loc, tensor::getMixedSizes(rewriter, loc, op->get()),
+          elemType);
 
       unsigned start = genericOp.getDpsInits().getBeginOperandIndex();
       newOutputOperands[op->getOperandNumber() - start] = empty.getResult();
     }
 
-    auto newOp = GenericOp::create(rewriter,
-        loc, genericOp.getResultTypes(), newInputOperands, newOutputOperands,
-        newIndexingMaps, genericOp.getIteratorTypesArray(),
+    auto newOp = GenericOp::create(
+        rewriter, loc, genericOp.getResultTypes(), newInputOperands,
+        newOutputOperands, newIndexingMaps, genericOp.getIteratorTypesArray(),
         /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
 
     OpBuilder::InsertionGuard guard(rewriter);
@@ -296,7 +297,7 @@ static Value collapseValue(
     auto targetType = MemRefType::get(targetShape, memrefType.getElementType(),
                                       layout, memrefType.getMemorySpace());
     return memref::CollapseShapeOp::create(rewriter, loc, targetType, operand,
-                                                    reassociation);
+                                           reassociation);
   }
   if (auto tensorType = dyn_cast<RankedTensorType>(operand.getType())) {
     if (rankReductionStrategy ==
@@ -315,7 +316,7 @@ static Value collapseValue(
     auto targetType =
         RankedTensorType::get(targetShape, tensorType.getElementType());
     return tensor::CollapseShapeOp::create(rewriter, loc, targetType, operand,
-                                                    reassociation);
+                                           reassociation);
   }
   llvm_unreachable("unsupported operand type");
 }
@@ -520,7 +521,7 @@ linalg::dropUnitDims(RewriterBase &rewriter, GenericOp genericOp,
     resultTypes.push_back(newOutputs[i].getType());
   GenericOp replacementOp =
       GenericOp::create(rewriter, loc, resultTypes, newInputs, newOutputs,
-                                 newIndexingMaps, newIteratorTypes);
+                        newIndexingMaps, newIteratorTypes);
   rewriter.inlineRegionBefore(genericOp.getRegion(), replacementOp.getRegion(),
                               replacementOp.getRegion().begin());
   // 5a. Replace `linalg.index` operations that refer to the dropped unit
@@ -652,8 +653,8 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         collapseValue(rewriter, padOp.getLoc(), padOp.getSource(), newShape,
                       reassociationMap, options.rankReductionStrategy);
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        padOp.getLoc(), /*result=*/Type(), collapsedSource, newLowPad,
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, padOp.getLoc(), /*result=*/Type(), collapsedSource, newLowPad,
         newHighPad, paddingVal, padOp.getNofold());
 
     Value dest = padOp.getResult();
@@ -670,9 +671,8 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         expandedSizes.push_back(tensor::getMixedSize(
             rewriter, padOp.getLoc(), newPadOp, dim - numUnitDims));
       }
-      dest = tensor::EmptyOp::create(rewriter,
-          padOp.getLoc(), expandedSizes,
-          padOp.getResultType().getElementType());
+      dest = tensor::EmptyOp::create(rewriter, padOp.getLoc(), expandedSizes,
+                                     padOp.getResultType().getElementType());
     }
 
     Value expandedValue =
@@ -713,8 +713,9 @@ struct RankReducedExtractSliceOp
             strides));
 
     Location loc = sliceOp.getLoc();
-    Value newSlice = tensor::ExtractSliceOp::create(rewriter,
-        loc, rankReducedType, sliceOp.getSource(), offsets, sizes, strides);
+    Value newSlice = tensor::ExtractSliceOp::create(
+        rewriter, loc, rankReducedType, sliceOp.getSource(), offsets, sizes,
+        strides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSlice, *reassociation);
     return success();
@@ -747,8 +748,8 @@ struct RankReducedInsertSliceOp : public OpRewritePattern<InsertOpTy> {
       // parallel case.
       if (std::is_same<InsertOpTy, tensor::ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      reshapedSource = tensor::CollapseShapeOp::create(rewriter,
-          loc, insertSliceOp.getSource(), *reassociation);
+      reshapedSource = tensor::CollapseShapeOp::create(
+          rewriter, loc, insertSliceOp.getSource(), *reassociation);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, reshapedSource, insertSliceOp.getDest(),
@@ -898,8 +899,8 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
   /// Expand result tensor.
   Value expandResult(PatternRewriter &rewriter, Value result,
                      RankedTensorType expandedType, int64_t dim) const {
-    return tensor::ExpandShapeOp::create(rewriter,
-        result.getLoc(), expandedType, result,
+    return tensor::ExpandShapeOp::create(
+        rewriter, result.getLoc(), expandedType, result,
         getReassociationForReshapeAtDim(expandedType.getRank(), dim));
   }
 
@@ -934,9 +935,9 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
     SmallVector<Type, 1> collapsedResultTy;
     if (isa<RankedTensorType>(collapsedInit.getType()))
       collapsedResultTy.push_back(collapsedInit.getType());
-    auto collapsedOp = ToOpTy::create(rewriter,
-        loc, collapsedResultTy, ValueRange{collapsedLhs, collapsedRhs},
-        ValueRange{collapsedInit});
+    auto collapsedOp = ToOpTy::create(rewriter, loc, collapsedResultTy,
+                                      ValueRange{collapsedLhs, collapsedRhs},
+                                      ValueRange{collapsedInit});
     for (auto attr : contractionOp->getAttrs()) {
       if (attr.getName() == LinalgDialect::kMemoizedIndexingMapsAttrName ||
           attr.getName() == "indexing_maps")
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index 754f7652a776b..4ec4deeeabe66 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -240,8 +240,8 @@ static void generateFusedElementwiseOpRegion(
                     });
     for (IndexOp indexOp :
          llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
-      Value newIndex = affine::AffineApplyOp::create(rewriter,
-          producer.getLoc(),
+      Value newIndex = affine::AffineApplyOp::create(
+          rewriter, producer.getLoc(),
           consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
       mapper.map(indexOp.getResult(), newIndex);
     }
@@ -416,8 +416,8 @@ mlir::linalg::fuseElementwiseOps(RewriterBase &rewriter,
   }
 
   // Generate the fused op.
-  auto fusedOp = GenericOp::create(rewriter,
-      consumer.getLoc(), fusedResultTypes, fusedInputOperands,
+  auto fusedOp = GenericOp::create(
+      rewriter, consumer.getLoc(), fusedResultTypes, fusedInputOperands,
       fusedOutputOperands, rewriter.getAffineMapArrayAttr(fusedIndexMaps),
       consumer.getIteratorTypes(),
       /*doc=*/nullptr,
@@ -797,7 +797,7 @@ static Operation *createExpandedTransposeOp(PatternRewriter &rewriter,
     }
   }
   return TransposeOp::create(rewriter, transposeOp.getLoc(), expandedInput,
-                                      output, invertPermutationVector(newPerm));
+                             output, invertPermutationVector(newPerm));
 }
 
 // Create an expanded generic op.
@@ -813,9 +813,9 @@ static Operation *createExpandedGenericOp(
     for (auto j : expansionInfo.getExpandedDims(i))
       iteratorTypes[j] = type;
 
-  Operation *fused = GenericOp::create(rewriter,
-      linalgOp.getLoc(), resultTypes, expandedOpOperands, outputs,
-      expandedOpIndexingMaps, iteratorTypes);
+  Operation *fused = GenericOp::create(rewriter, linalgOp.getLoc(), resultTypes,
+                                       expandedOpOperands, outputs,
+                                       expandedOpIndexingMaps, iteratorTypes);
 
   Region &fusedRegion = fused->getRegion(0);
   Region &originalRegion = linalgOp->getRegion(0);
@@ -933,8 +933,8 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
                 reassociation,
                 /*isExpandingReshape=*/true)))
           return std::nullopt;
-        expandedOpOperands.push_back(tensor::ExpandShapeOp::create(rewriter,
-            loc, expandedOperandType, opOperand->get(), reassociation,
+        expandedOpOperands.push_back(tensor::ExpandShapeOp::create(
+            rewriter, loc, expandedOperandType, opOperand->get(), reassociation,
             expandedOperandShape));
         continue;
       }
@@ -961,8 +961,8 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               reassociation,
               /*isExpandingReshape=*/true)))
         return std::nullopt;
-      outputs.push_back(tensor::ExpandShapeOp::create(rewriter,
-          loc, expandedOutputType, opOperand.get(), reassociation,
+      outputs.push_back(tensor::ExpandShapeOp::create(
+          rewriter, loc, expandedOutputType, opOperand.get(), reassociation,
           expandedOutputShape));
     } else {
       outputs.push_back(opOperand.get());
@@ -984,8 +984,8 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               linalgOp.getMatchingIndexingMap(
                   linalgOp.getDpsInitOperand(resultNumber)),
               expansionInfo);
-      resultVals.push_back(tensor::CollapseShapeOp::create(rewriter,
-          linalgOp.getLoc(), opResult.getType(),
+      resultVals.push_back(tensor::CollapseShapeOp::create(
+          rewriter, linalgOp.getLoc(), opResult.getType(),
           fusedOp->getResult(resultNumber), reassociation));
     } else {
       resultVals.push_back(fusedOp->getResult(resultNumber));
@@ -1086,8 +1086,8 @@ class FoldPadWithProducerReshapeOpByExpansion
 
     Location loc = padOp->getLoc();
     RankedTensorType expandedPaddedType = paddedType.clone(expandedPaddedShape);
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
     rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
@@ -1687,9 +1687,10 @@ GenericOp cloneToCollapsedOp<GenericOp>(RewriterBase &rewriter,
   SmallVector<utils::IteratorType> iteratorTypes(getCollapsedOpIteratorTypes(
       origOp.getIteratorTypesArray(), collapsingInfo));
 
-  GenericOp collapsedOp = linalg::GenericOp::create(rewriter,
-      origOp.getLoc(), resultTypes, inputOperands, outputOperands, indexingMaps,
-      iteratorTypes, [](OpBuilder &builder, Location loc, ValueRange args) {});
+  GenericOp collapsedOp = linalg::GenericOp::create(
+      rewriter, origOp.getLoc(), resultTypes, inputOperands, outputOperands,
+      indexingMaps, iteratorTypes,
+      [](OpBuilder &builder, Location loc, ValueRange args) {});
   Block *origOpBlock = &origOp->getRegion(0).front();
   Block *collapsedOpBlock = &collapsedOp->getRegion(0).front();
   rewriter.mergeBlocks(origOpBlock, collapsedOpBlock,
@@ -1794,12 +1795,12 @@ FailureOr<CollapseResult> mlir::linalg::collapseOpIterationDims(
       if (isa<MemRefType>(collapsedOpResult.getType())) {
         MemRefType expandShapeResultType = MemRefType::get(
             originalResultType.getShape(), originalResultType.getElementType());
-        result = memref::ExpandShapeOp::create(rewriter,
-            loc, expandShapeResultType, collapsedOpResult, reassociation,
-            resultShape);
+        result = memref::ExpandShapeOp::create(
+            rewriter, loc, expandShapeResultType, collapsedOpResult,
+            reassociation, resultShape);
       } else {
-        result = tensor::ExpandShapeOp::create(rewriter,
-            loc, originalResultType, collapsedOpResult, reassociation,
+        result = tensor::ExpandShapeOp::create(
+            rewriter, loc, originalResultType, collapsedOpResult, reassociation,
             resultShape);
       }
       results.push_back(result);
@@ -1982,8 +1983,8 @@ class FoldPadWithProducerReshapeOpByCollapsing
 
     RankedTensorType collapsedPaddedType =
         paddedType.clone(collapsedPaddedShape);
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -2120,14 +2121,15 @@ class FoldScalarOrSplatConstant : public OpRewritePattern<GenericOp> {
           arith::ConstantOp::create(rewriter, def->getLoc(), constantAttr);
 
       SmallVector<Value> outputOperands = genericOp.getOutputs();
-      auto fusedOp = GenericOp::create(rewriter,
-          rewriter.getFusedLoc(fusedLocs), genericOp->getResultTypes(),
-          /*inputs=*/fusedOperands,
-          /*outputs=*/outputOperands,
-          rewriter.getAffineMapArrayAttr(fusedIndexMaps),
-          genericOp.getIteratorTypes(),
-          /*doc=*/nullptr,
-          /*library_call=*/nullptr);
+      auto fusedOp =
+          GenericOp::create(rewriter, rewriter.getFusedLoc(fusedLocs),
+                            genericOp->getResultTypes(),
+                            /*inputs=*/fusedOperands,
+                            /*outputs=*/outputOperands,
+                            rewriter.getAffineMapArrayAttr(fusedIndexMaps),
+                            genericOp.getIteratorTypes(),
+                            /*doc=*/nullptr,
+                            /*library_call=*/nullptr);
 
       // Map the block argument corresponding to the replaced argument with the
       // scalar constant.
@@ -2183,8 +2185,8 @@ struct RemoveOutsDependency : public OpRewritePattern<GenericOp> {
         modifiedOutput = true;
         SmallVector<OpFoldResult> mixedSizes =
             tensor::getMixedSizes(rewriter, loc, operandVal);
-        Value emptyTensor = tensor::EmptyOp::create(rewriter,
-            loc, mixedSizes, operandType.getElementType());
+        Value emptyTensor = tensor::EmptyOp::create(
+            rewriter, loc, mixedSizes, operandType.getElementType());
         op->setOperand(opOperand.getOperandNumber(), emptyTensor);
       }
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index d93f8c26a9f0a..c52315333c5b3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -64,8 +64,8 @@ getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) {
       continue;
 
     // Extract static / dynamic shape mix from the first operand.
-    res.push_back(tensor::EmptyOp::create(b,
-        loc, tensor::getMixedSizes(b, loc, operands.front()),
+    res.push_back(tensor::EmptyOp::create(
+        b, loc, tensor::getMixedSizes(b, loc, operands.front()),
         cast<RankedTensorType>(t).getElementType()));
   }
   return res;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
index 4e95aae86e670..9974ccd8ae79e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
@@ -259,8 +259,8 @@ mlir::linalg::deduplicateOperandsAndRemoveDeadResults(
   for (Value v : newOutputOperands)
     if (isa<TensorType>(v.getType()))
       newResultTypes.push_back(v.getType());
-  auto newOp = GenericOp::create(rewriter,
-      loc, newResultTypes, newInputOperands, newOutputOperands,
+  auto newOp = GenericOp::create(
+      rewriter, loc, newResultTypes, newInputOperands, newOutputOperands,
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       genericOp.getIteratorTypes(), genericOp.getDocAttr(),
       genericOp.getLibraryCallAttr(),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
index e19e224c25028..0ca89040f3453 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
@@ -72,14 +72,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
     // Create the tensor of same size as output of the pad op.
     RankedTensorType padResultType = padOp.getResultType();
     auto resultSizes = resultShape[0];
-    auto emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultSizes, padResultType.getElementType());
+    auto emptyTensor = tensor::EmptyOp::create(rewriter, loc, resultSizes,
+                                               padResultType.getElementType());
 
     // Fill the tensor with the pad value.
     // TODO: There is an option to fill only the boundaries. For now just
     // filling the whole tensor.
-    auto fillTensor =
-        linalg::FillOp::create(rewriter, loc, padValue, emptyTensor.getResult());
+    auto fillTensor = linalg::FillOp::create(rewriter, loc, padValue,
+                                             emptyTensor.getResult());
 
     // Construct a slice of the fill result that is to be replaced with the
     // result of the generic op. The low pad values are the offsets, the size of
@@ -100,8 +100,8 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
       }
     }
     SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-    auto slice = tensor::ExtractSliceOp::create(rewriter,
-        loc, fillTensor.getResult(0), offsets, sizes, strides);
+    auto slice = tensor::ExtractSliceOp::create(
+        rewriter, loc, fillTensor.getResult(0), offsets, sizes, strides);
 
     // Clone the generic op.
     auto clonedOp =
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 7994af8834990..3e31393fd51ed 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -61,8 +61,9 @@ FailureOr<GenericOp> mlir::linalg::generalizeNamedOp(RewriterBase &rewriter,
   // All named ops have a region attached that can be inlined.
   assert(linalgOp->getNumRegions() == 1 &&
          "expect named op to have one region attached");
-  GenericOp genericOp = GenericOp::create(rewriter,
-      linalgOp.getLoc(), resultTypes, inputs, outputs, indexingMaps, iterators);
+  GenericOp genericOp =
+      GenericOp::create(rewriter, linalgOp.getLoc(), resultTypes, inputs,
+                        outputs, indexingMaps, iterators);
   rewriter.inlineRegionBefore(linalgOp->getRegion(0), genericOp.getRegion(),
                               genericOp.getRegion().begin());
   rewriter.replaceOp(linalgOp, genericOp->getResults());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index 86f24578376c1..19729af1bc9fa 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -591,8 +591,8 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
 
     // Create a packing loop that takes `hoistedPackedTensor` as iteration
     // argument.
-    auto clonedForOp = scf::ForOp::create(rewriter,
-        loc, bvm.lookupOrDefault(forOp.getLowerBound()),
+    auto clonedForOp = scf::ForOp::create(
+        rewriter, loc, bvm.lookupOrDefault(forOp.getLowerBound()),
         bvm.lookupOrDefault(forOp.getUpperBound()),
         bvm.lookupOrDefault(forOp.getStep()), hoistedPackedTensor);
 
@@ -640,11 +640,11 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   TransposeOp maybeTransposeOp;
   Value paddedTensor = bvm.lookup(opToHoist.getResult());
   if (!transposeVector.empty()) {
-    Value outputTensor = tensor::ExtractSliceOp::create(rewriter,
-        loc, transposedTensorType, hoistedPackedTensor, offsets, sizes,
-        strides);
-    maybeTransposeOp = linalg::TransposeOp::create(rewriter,
-        loc, paddedTensor, outputTensor, transposeVector);
+    Value outputTensor = tensor::ExtractSliceOp::create(
+        rewriter, loc, transposedTensorType, hoistedPackedTensor, offsets,
+        sizes, strides);
+    maybeTransposeOp = linalg::TransposeOp::create(
+        rewriter, loc, paddedTensor, outputTensor, transposeVector);
     paddedTensor = maybeTransposeOp.getResult()[0];
   }
 
@@ -652,8 +652,9 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   if (nPackedLoops > 0) {
     // Step 4. Create InsertSliceOp at the innermost loop level, inserting an
     // optionally transposed padded slice into the packed tensor.
-    Value inserted = tensor::InsertSliceOp::create(rewriter,
-        loc, paddedTensor, hoistedPackedTensor, offsets, sizes, strides);
+    Value inserted = tensor::InsertSliceOp::create(rewriter, loc, paddedTensor,
+                                                   hoistedPackedTensor, offsets,
+                                                   sizes, strides);
 
     // Step 5. Iteratively pop the stack and propagate the yield.
     Value valueToYield = inserted;
@@ -712,8 +713,8 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   rewriter.setInsertionPoint(outerLoop);
   SmallVector<Value> dynamicTensorSizes =
       analysis.getHoistedPackedTensorSizes(rewriter, loc);
-  auto emptyOp = tensor::EmptyOp::create(rewriter,
-      loc, hoistedPackedTensorType.getShape(),
+  auto emptyOp = tensor::EmptyOp::create(
+      rewriter, loc, hoistedPackedTensorType.getShape(),
       hoistedPackedTensorType.getElementType(), dynamicTensorSizes);
 
   return buildPackingLoopNestImpl(rewriter, bvm, opToHoist, transposeVector,
@@ -840,8 +841,8 @@ padThroughLoopIterArg(RewriterBase &rewriter, Value paddedValueBeforeHoisting,
   {
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(forOp);
-    extracted = tensor::ExtractSliceOp::create(rewriter,
-        hoistedPackedTensor.getLoc(), hoistedPackedTensor,
+    extracted = tensor::ExtractSliceOp::create(
+        rewriter, hoistedPackedTensor.getLoc(), hoistedPackedTensor,
         outerSliceOp.getMixedOffsets(), outerSliceOp.getMixedSizes(),
         outerSliceOp.getMixedStrides());
     rewriter.replaceAllUsesWith(forOp.getResult(iterArgNumber), extracted);
@@ -934,8 +935,8 @@ static Value replaceByPackingResult(RewriterBase &rewriter,
   // offsets = [maybe_leading_ivs, 0 .. 0].
   // sizes = [1 .. 1, transposedShape] (defined above).
   // strides = [1 .. 1] (defined above)
-  return tensor::ExtractSliceOp::create(rewriter,
-      loc, transposedTensorType, hoistedPackedTensor, offsets,
+  return tensor::ExtractSliceOp::create(
+      rewriter, loc, transposedTensorType, hoistedPackedTensor, offsets,
       packingResult.sizes, packingResult.strides);
 }
 
@@ -982,10 +983,11 @@ FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(newResult.getDefiningOp());
     // Transpose the packed tensor back to the original storage order.
-    Value emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, paddedTensorType.getShape(), paddedTensorType.getElementType());
-    TransposeOp unTransposeOp = linalg::TransposeOp::create(rewriter,
-        loc, newResult, emptyTensor, transposeVector);
+    Value emptyTensor =
+        tensor::EmptyOp::create(rewriter, loc, paddedTensorType.getShape(),
+                                paddedTensorType.getElementType());
+    TransposeOp unTransposeOp = linalg::TransposeOp::create(
+        rewriter, loc, newResult, emptyTensor, transposeVector);
     newResult = unTransposeOp.getResult()[0];
     transposeOps.push_back(unTransposeOp);
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 5397fc572e558..70bc7b6360ceb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -53,9 +53,9 @@ static scf::ForOp replaceWithDifferentYield(RewriterBase &rewriter,
   assert(index < inits.size());
   inits[index] = newInitOperand;
 
-  scf::ForOp newLoop = scf::ForOp::create(rewriter,
-      loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
-      inits, [](OpBuilder &, Location, Value, ValueRange) {});
+  scf::ForOp newLoop = scf::ForOp::create(
+      rewriter, loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(),
+      loop.getStep(), inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
   // Generate the new yield with the replaced operand.
   auto yieldOp = cast<scf::YieldOp>(loop.getBody()->getTerminator());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
index a900215ccf014..39cc21df40583 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
@@ -60,9 +60,9 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
 
     Location loc = genericOp->getLoc();
     SmallVector<Value> outputOperands = genericOp.getOutputs();
-    auto newOp = GenericOp::create(rewriter,
-        loc, genericOp->getResultTypes(), newOperands, outputOperands,
-        newIndexingMaps, genericOp.getIteratorTypesArray());
+    auto newOp = GenericOp::create(rewriter, loc, genericOp->getResultTypes(),
+                                   newOperands, outputOperands, newIndexingMaps,
+                                   genericOp.getIteratorTypesArray());
     rewriter.cloneRegionBefore(genericOp.getRegion(), newOp.getRegion(),
                                newOp.getRegion().begin());
 
@@ -80,8 +80,8 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
             arith::ConstantIndexOp::create(rewriter, loc, idx));
       Value scalarValue = opOperand->get();
       if (isa<RankedTensorType>(scalarValue.getType())) {
-        scalarValue =
-            tensor::ExtractOp::create(rewriter, loc, scalarValue, indicesValues);
+        scalarValue = tensor::ExtractOp::create(rewriter, loc, scalarValue,
+                                                indicesValues);
       }
       body->getArgument(idx).replaceAllUsesWith(scalarValue);
       body->eraseArgument(idx);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index 2372313e29c58..96e6eee6d7a0d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -88,7 +88,8 @@ mlir::linalg::interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp,
       allIndices.reserve(genericOp.getNumLoops());
       llvm::transform(llvm::seq<uint64_t>(0, genericOp.getNumLoops()),
                       std::back_inserter(allIndices), [&](uint64_t dim) {
-                        return IndexOp::create(rewriter, indexOp->getLoc(), dim);
+                        return IndexOp::create(rewriter, indexOp->getLoc(),
+                                               dim);
                       });
       rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
           indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 0c08c1d8b997d..38f1a8b7247eb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -70,8 +70,9 @@ static void inlineRegionAndEmitStore(OpBuilder &b, Location loc, OpType op,
   Operation *terminator = block.getTerminator();
   for (OpOperand &operand : terminator->getOpOperands()) {
     Value toStore = map.lookupOrDefault(operand.get());
-    StoreOpTy::create(b, loc, toStore, outputBuffers[operand.getOperandNumber()],
-                        indexing[operand.getOperandNumber()]);
+    StoreOpTy::create(b, loc, toStore,
+                      outputBuffers[operand.getOperandNumber()],
+                      indexing[operand.getOperandNumber()]);
   }
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
index 28b246714df53..f1c9d79a80804 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
@@ -132,11 +132,11 @@ static Value createDestinationPassingStyleInitOperand(
   Value processLinearIndexInReductionGroup = mesh::createProcessLinearIndex(
       meshOp.getSymName(), reductionMeshAxes, builder);
   Value zero = arith::ConstantIndexOp::create(builder, 0);
-  Value isLeadProcess = arith::CmpIOp::create(builder,
-      builder.getI1Type(), arith::CmpIPredicate::eq,
+  Value isLeadProcess = arith::CmpIOp::create(
+      builder, builder.getI1Type(), arith::CmpIPredicate::eq,
       processLinearIndexInReductionGroup, zero);
   scf::IfOp ifOp = scf::IfOp::create(builder, spmdizedOperand.getType(),
-                                             isLeadProcess, true, true);
+                                     isLeadProcess, true, true);
   // Then block.
   {
     OpBuilder::InsertionGuard insertionGuard(builder);
@@ -158,7 +158,7 @@ static Value createDestinationPassingStyleInitOperand(
         arith::getNeutralElement(combinerOps[0]);
 
     Value init = tensor::EmptyOp::create(builder, op.getLoc(), shape,
-                                                 neutralEl.value().getType());
+                                         neutralEl.value().getType());
     Value constant =
         arith::ConstantOp::create(builder, op.getLoc(), neutralEl.value());
     Value fill = linalg::FillOp::create(builder, op.getLoc(), constant, init)
@@ -202,9 +202,9 @@ static void createAllReduceForResultWithoutPartialSharding(
   }
 
   Value spmdizedLinalgOpResult = spmdizationMap.lookup(unshardedLinalgOpResult);
-  Value reducedValue = mesh::AllReduceOp::create(builder,
-      spmdizedLinalgOpResult, resultSharding.getMesh(), allReduceMeshAxes,
-      reductionKind);
+  Value reducedValue = mesh::AllReduceOp::create(
+      builder, spmdizedLinalgOpResult, resultSharding.getMesh(),
+      allReduceMeshAxes, reductionKind);
   spmdizationMap.map(unshardedLinalgOpResult, reducedValue);
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
index 019c845b4e0e7..a2bd9d92815a0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
@@ -59,8 +59,8 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
   auto newKernelTy = RankedTensorType::get(
       {kernelTy.getDimSize(0), kernelTy.getDimSize(1), kernelTy.getDimSize(2)},
       kernelTy.getElementType());
-  auto collapsedKernel = tensor::CollapseShapeOp::create(rewriter,
-      loc, newKernelTy, kernel, collapsedKernelDims);
+  auto collapsedKernel = tensor::CollapseShapeOp::create(
+      rewriter, loc, newKernelTy, kernel, collapsedKernelDims);
 
   // Collapse init dims.
   SmallVector<ReassociationIndices, 4> collapsedInitDims = {
@@ -70,22 +70,23 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       RankedTensorType::get({initTy.getDimSize(0), initTy.getDimSize(1),
                              initTy.getDimSize(2), initTy.getDimSize(3)},
                             initTy.getElementType());
-  auto collapsedInit = tensor::CollapseShapeOp::create(rewriter,
-      loc, newInitTy, init, collapsedInitDims);
+  auto collapsedInit = tensor::CollapseShapeOp::create(rewriter, loc, newInitTy,
+                                                       init, collapsedInitDims);
 
   SmallVector<NamedAttribute> preservedAttrs;
   Operation *newConv =
       TypeSwitch<Operation *, Operation *>(operation)
           .Case<DepthwiseConv2DNhwcHwcmOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return DepthwiseConv2DNhwcHwcOp::create(rewriter,
-                loc, newInitTy, ValueRange{input, collapsedKernel},
+            return DepthwiseConv2DNhwcHwcOp::create(
+                rewriter, loc, newInitTy, ValueRange{input, collapsedKernel},
                 ValueRange{collapsedInit}, stride, dilation);
           })
           .Case<DepthwiseConv2DNhwcHwcmQOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return DepthwiseConv2DNhwcHwcQOp::create(rewriter,
-                loc, newInitTy, ValueRange{input, collapsedKernel, iZp, kZp},
+            return DepthwiseConv2DNhwcHwcQOp::create(
+                rewriter, loc, newInitTy,
+                ValueRange{input, collapsedKernel, iZp, kZp},
                 ValueRange{collapsedInit}, stride, dilation);
           })
           .Default([](Operation *op) { return nullptr; });
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
index 1368bb7e77db0..0415057eda86b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
@@ -144,7 +144,7 @@ struct SimplifyUnPackToCollapseShape : public OpRewritePattern<UnPackOp> {
     if (operand.getType() == newOperandType)
       return operand;
     return tensor::CollapseShapeOp::create(rewriter, loc, newOperandType,
-                                                    operand, reassociation);
+                                           operand, reassociation);
   }
 
   /// Returns success() if it is unpacking on the innermost dimension.
@@ -265,8 +265,8 @@ struct FoldUnpackWithExtractSliceOp
 
     // Create a new empty output tensor.
     Type elementType = unpackOp.getDestType().getElementType();
-    Value output = tensor::EmptyOp::create(rewriter,
-        sliceOp.getLoc(), sliceOp.getMixedSizes(), elementType);
+    Value output = tensor::EmptyOp::create(
+        rewriter, sliceOp.getLoc(), sliceOp.getMixedSizes(), elementType);
     rewriter.replaceOpWithNewOp<UnPackOp>(
         sliceOp, unpackOp.getSource(), output, unpackOp.getInnerDimsPos(),
         unpackOp.getMixedTiles(), unpackOp.getOuterDimsPerm());
@@ -529,8 +529,8 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
     auto elemType =
         cast<ShapedType>(unPackOp->getResultTypes()[0]).getElementType();
-    Value output = tensor::EmptyOp::create(rewriter,
-        unPackOp->getLoc(), unpackOpResultDims[0], elemType);
+    Value output = tensor::EmptyOp::create(rewriter, unPackOp->getLoc(),
+                                           unpackOpResultDims[0], elemType);
 
     rewriter.replaceOpWithNewOp<UnPackOp>(
         unPackOp, linalgOp->getOperand(0), output, newInnerDimsPosVec,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index fb43c1f92e5b3..2c62cb628a7dd 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -193,10 +193,10 @@ static Value padOperand(RewriterBase &rewriter, TilingInterface opToPad,
           dyn_cast<ComplexType>(getElementTypeOrSelf(v.getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingValueAttr);
     paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
-                                                        complexTy, complexAttr);
+                                               complexTy, complexAttr);
   } else {
-    paddingValue = arith::ConstantOp::create(rewriter,
-        opToPad.getLoc(), cast<TypedAttr>(paddingValueAttr));
+    paddingValue = arith::ConstantOp::create(rewriter, opToPad.getLoc(),
+                                             cast<TypedAttr>(paddingValueAttr));
   }
 
   // Pad the operand to the bounding box defined by `paddedShape`.
@@ -323,8 +323,8 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
-        loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(
+        rewriter, loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index af614e88fb3d5..b5c5aea56a998 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -220,10 +220,10 @@ static FailureOr<Value> padOperandToSmallestStaticBoundingBox(
           getElementTypeOrSelf(opOperand->get().getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingAttr);
     paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
-                                                        complexTy, complexAttr);
+                                               complexTy, complexAttr);
   } else {
-    paddingValue = arith::ConstantOp::create(rewriter,
-        opToPad.getLoc(), cast<TypedAttr>(paddingAttr));
+    paddingValue = arith::ConstantOp::create(rewriter, opToPad.getLoc(),
+                                             cast<TypedAttr>(paddingAttr));
   }
 
   // Computes the padded shape.
@@ -313,8 +313,8 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
-        loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(
+        rewriter, loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index 4042a43b04091..7014d02a70377 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -64,10 +64,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
         MemRefType::Builder(staticBufferType).setMemorySpace(memorySpaceAttr);
     if (options.useAlloca) {
       return memref::AllocaOp::create(b, staticBufferType, ValueRange{},
-                                        alignmentAttr);
+                                      alignmentAttr);
     }
     return memref::AllocOp::create(b, staticBufferType, ValueRange{},
-                                     alignmentAttr);
+                                   alignmentAttr);
   }
 
   // Fallback dynamic buffer.
@@ -124,7 +124,7 @@ defaultDeallocBufferCallBack(const LinalgPromotionOptions &options,
   if (!options.useAlloca) {
     auto viewOp = cast<memref::ViewOp>(fullLocalView.getDefiningOp());
     memref::DeallocOp::create(b, viewOp.getSource().getLoc(),
-                                viewOp.getSource());
+                              viewOp.getSource());
   }
   return success();
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
index 1fac80142dddf..76d0ba97be4db 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
@@ -52,8 +52,8 @@ createSplitPart(RewriterBase &b, Location loc, TilingInterface op,
       return nullptr;
     SmallVector<OpFoldResult> resultStrides(resultOffsets.size(),
                                             b.getIndexAttr(1));
-    Value inserted = tensor::InsertSliceOp::create(b,
-        loc, result, resultOperands[index], resultOffsets, resultSizes,
+    Value inserted = tensor::InsertSliceOp::create(
+        b, loc, result, resultOperands[index], resultOffsets, resultSizes,
         resultStrides);
     results.push_back(inserted);
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
index 1b0bfbbec8624..b8f862001bccf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
@@ -115,8 +115,8 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
         newShape,
         cast<RankedTensorType>(operand->get().getType()).getElementType());
 
-    Value newInput = tensor::ExpandShapeOp::create(b,
-        loc, newType, operand->get(), reassociation);
+    Value newInput = tensor::ExpandShapeOp::create(
+        b, loc, newType, operand->get(), reassociation);
     newInputs.push_back(newInput);
   }
 
@@ -140,14 +140,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   Value emptyOrAllocTensor;
   if (useAlloc) {
-    emptyOrAllocTensor = bufferization::AllocTensorOp::create(b,
-        loc,
+    emptyOrAllocTensor = bufferization::AllocTensorOp::create(
+        b, loc,
         RankedTensorType::get(newOutputShape,
                               op.getRegionOutputArgs()[0].getType()),
         ValueRange{});
   } else {
-    emptyOrAllocTensor = tensor::EmptyOp::create(b,
-        loc, newOutputShape, op.getRegionOutputArgs()[0].getType());
+    emptyOrAllocTensor = tensor::EmptyOp::create(
+        b, loc, newOutputShape, op.getRegionOutputArgs()[0].getType());
   }
   Value constantOp = arith::ConstantOp::create(b, loc, *identity);
   Value identityTensor =
@@ -168,8 +168,8 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   // Create the new op matching the original op with an extra parallel
   // dimension.
-  GenericOp genericOp = GenericOp::create(b,
-      loc, TypeRange({emptyOrAllocTensor.getType()}), newInputs,
+  GenericOp genericOp = GenericOp::create(
+      b, loc, TypeRange({emptyOrAllocTensor.getType()}), newInputs,
       ValueRange({identityTensor}), newMaps, newIteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
                        genericOp.getRegion().begin());
@@ -191,8 +191,8 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   AffineMap outputMap = AffineMap::get(intermRank, 0, exprs, op.getContext());
   SmallVector<AffineMap> reductionMaps = {inputMap, outputMap};
 
-  auto reduction = GenericOp::create(b,
-      loc, op->getResultTypes(), ValueRange({genericOp.getResult(0)}),
+  auto reduction = GenericOp::create(
+      b, loc, op->getResultTypes(), ValueRange({genericOp.getResult(0)}),
       op.getDpsInits(), reductionMaps, reductionIteratorTypes,
       [reductionOp](OpBuilder &b, Location loc, ValueRange inputs) {
         Operation *clonedReductionOp = b.clone(*reductionOp);
@@ -321,11 +321,11 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
           bufferization::AllocTensorOp::create(b, loc, newT, dims);
     } else {
       emptyOrAllocTensor = tensor::EmptyOp::create(b, loc, newT.getShape(),
-                                                     t.getElementType(), dims);
+                                                   t.getElementType(), dims);
     }
     Value constantOp = arith::ConstantOp::create(b, loc, std::get<1>(it));
-    fillOps.push_back(
-        linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor));
+    fillOps.push_back(linalg::FillOp::create(b, op->getLoc(), constantOp,
+                                             emptyOrAllocTensor));
     newOutputs.push_back(fillOps.back().getResult(0));
     emptyOrAllocTensorOps.push_back(emptyOrAllocTensor.getDefiningOp());
   }
@@ -354,8 +354,8 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   SmallVector<Value> newInputs = op.getDpsInputs();
   // Add a single shape-only tensor to carry the dimensions without resorting to
   // more complex inversions.
-  newInputs.push_back(tensor::EmptyOp::create(b,
-      loc, ArrayRef<int64_t>{reductionDimSize / splitFactor, splitFactor},
+  newInputs.push_back(tensor::EmptyOp::create(
+      b, loc, ArrayRef<int64_t>{reductionDimSize / splitFactor, splitFactor},
       b.getIntegerType(1)));
   // Output tensors are already good to go.
 
@@ -366,7 +366,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
                        utils::IteratorType::parallel);
   GenericOp genericOp =
       GenericOp::create(b, loc, ValueRange(newOutputs).getTypes(), newInputs,
-                          newOutputs, newMaps, iteratorTypes);
+                        newOutputs, newMaps, iteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
                        genericOp.getRegion().begin());
   genericOp.getRegion().front().insertArgument(reductionDimPos,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
index 854d3855d654d..792ca3e5123ef 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
@@ -29,10 +29,10 @@ struct SwapExtractSliceOfFill final
     if (!fillOp || !fillOp->hasOneUse())
       return failure();
 
-    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
-        extractOp.getLoc(), extractOp.getType(), fillOp.getOutputs()[0],
-        extractOp.getMixedOffsets(), extractOp.getMixedSizes(),
-        extractOp.getMixedStrides());
+    auto newExtractOp = tensor::ExtractSliceOp::create(
+        rewriter, extractOp.getLoc(), extractOp.getType(),
+        fillOp.getOutputs()[0], extractOp.getMixedOffsets(),
+        extractOp.getMixedSizes(), extractOp.getMixedStrides());
     rewriter.replaceOpWithNewOp<FillOp>(extractOp, fillOp.getInputs(),
                                         ValueRange{newExtractOp.getResult()});
     return success();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index c2803423fafd7..705d6f26efd29 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -96,9 +96,9 @@ static void emitIsPositiveIndexAssertion(ImplicitLocOpBuilder &b,
 
   Value zero = arith::ConstantIndexOp::create(b, 0);
   Value condition = arith::CmpIOp::create(b, arith::CmpIPredicate::sgt,
-                                            cast<Value>(value), zero);
-  cf::AssertOp::create(b,
-      condition,
+                                          cast<Value>(value), zero);
+  cf::AssertOp::create(
+      b, condition,
       b.getStringAttr("expected strictly positive tile size and divisor"));
 }
 
@@ -318,10 +318,11 @@ mlir::linalg::computeMultiTileSizes(OpBuilder &builder, LinalgOp op,
         apply(s0 * s1 + s2 * s3, {spec.lowTileSize, spec.lowTripCount,
                                   spec.highTileSize, spec.highTripCount});
     Value equals = arith::CmpIOp::create(b, arith::CmpIPredicate::eq,
-                                           coveredSize, tripCount);
-    cf::AssertOp::create(b,
-        equals, builder.getStringAttr(
-                    "could not compute dynamic multi-size tile shapes"));
+                                         coveredSize, tripCount);
+    cf::AssertOp::create(
+        b, equals,
+        builder.getStringAttr(
+            "could not compute dynamic multi-size tile shapes"));
   }
 
   return spec;
@@ -656,8 +657,8 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       getValueOrCreateConstantIndexOp(b, loc, nonZeroNumThreads);
 
   // 2. Create the ForallOp with an empty region.
-  scf::ForallOp forallOp = scf::ForallOp::create(b,
-      loc, getAsOpFoldResult(materializedNonZeroNumThreads), initTensors,
+  scf::ForallOp forallOp = scf::ForallOp::create(
+      b, loc, getAsOpFoldResult(materializedNonZeroNumThreads), initTensors,
       mapping);
 
   // 3. Calculate the tile offsets and sizes for the subsequent loop that will
@@ -689,8 +690,8 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       sizes[reductionDim] = b.getIndexAttr(1);
       outOffsets[reductionDim] = forallOp.getInductionVars()[0];
       // TODO: use SubsetExtractOpInterface once it is available.
-      tiledDpsInitOperands.push_back(tensor::ExtractSliceOp::create(b,
-          loc, cast<RankedTensorType>(initOperand.getType()),
+      tiledDpsInitOperands.push_back(tensor::ExtractSliceOp::create(
+          b, loc, cast<RankedTensorType>(initOperand.getType()),
           destBbArgs[destNum], outOffsets, sizes, strides));
     }
 
@@ -768,8 +769,8 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
     // 6.b. Parallel insertions are inserted at the end of the combining
     // terminator.
     b.setInsertionPointToEnd(forallOp.getTerminator().getBody());
-    tensor::ParallelInsertSliceOp::create(b,
-        loc, result, bbArg, resultOffsetsRank, resultSizesRank, strides);
+    tensor::ParallelInsertSliceOp::create(
+        b, loc, result, bbArg, resultOffsetsRank, resultSizesRank, strides);
   }
 
   // 7. Merge the partial reductions.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
index 94449fae260ee..b65f9d9fb9f7f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
@@ -72,9 +72,9 @@ static LogicalResult inlinePayload(OpBuilder &b, LinalgOp linalgOp,
     OpOperand *storeInto = linalgOp.getDpsInitOperand(operand.index());
     auto indices = getIndicesForAccess(
         b, loc, linalgOp.getMatchingIndexingMap(storeInto), ivs);
-    memref::StoreOp::create(b,
-        loc, toStore, linalgOp.getDpsInitOperand(operand.index())->get(),
-        indices);
+    memref::StoreOp::create(b, loc, toStore,
+                            linalgOp.getDpsInitOperand(operand.index())->get(),
+                            indices);
   }
   return success();
 }
@@ -574,9 +574,9 @@ struct LinalgOpPartialReductionInterface
       RankedTensorType sliceResultType = RankedTensorType::get(
           sliceInfo.resultShape, valueToTileType.getElementType(),
           valueToTileType.getEncoding());
-      auto sliceOp = tensor::ExtractSliceOp::create(b,
-          loc, sliceResultType, valueToTile, sliceInfo.offsets, sliceInfo.sizes,
-          sliceInfo.strides);
+      auto sliceOp = tensor::ExtractSliceOp::create(
+          b, loc, sliceResultType, valueToTile, sliceInfo.offsets,
+          sliceInfo.sizes, sliceInfo.strides);
       tiledInits.push_back(sliceOp.getResult());
       generatedSlices.push_back(sliceOp);
     }
@@ -603,8 +603,8 @@ struct LinalgOpPartialReductionInterface
     auto resultTypes = ValueRange(tiledInits).getTypes();
     if (tilingStrategy ==
         ReductionTilingStrategy::PartialReductionOuterReduction) {
-      auto genericOp = GenericOp::create(b,
-          loc, resultTypes, tiledInputs, tiledInits, newMaps, newIteratorTypes);
+      auto genericOp = GenericOp::create(b, loc, resultTypes, tiledInputs,
+                                         tiledInits, newMaps, newIteratorTypes);
       IRMapping mapping;
       op->getRegion(0).cloneInto(&genericOp.getRegion(),
                                  genericOp.getRegion().begin(), mapping);
@@ -648,8 +648,8 @@ struct LinalgOpPartialReductionInterface
         }
       }
 
-      auto reduction = linalg::ReduceOp::create(b,
-          loc, partialResult, init, partialReductionDims,
+      auto reduction = linalg::ReduceOp::create(
+          b, loc, partialResult, init, partialReductionDims,
           [&linalgOp, &initIdx](OpBuilder &b, Location loc, ValueRange inputs) {
             // Get the combiner op.
             SmallVector<Operation *, 4> combinerOps;
@@ -790,8 +790,8 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = tensor::ExtractSliceOp::create(b,
-        loc, packOp.getSource(), inputIndices, inputSizes, strides);
+    auto sourceSlice = tensor::ExtractSliceOp::create(
+        b, loc, packOp.getSource(), inputIndices, inputSizes, strides);
     tiledOperands.push_back(sourceSlice);
 
     SmallVector<OpFoldResult> outputOffsets, outputSizes;
@@ -800,8 +800,8 @@ struct PackOpTiling
       return {};
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = tensor::ExtractSliceOp::create(b,
-        loc, packOp.getDest(), outputOffsets, outputSizes, strides);
+    auto outSlice = tensor::ExtractSliceOp::create(
+        b, loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
     if (auto val = packOp.getPaddingValue())
@@ -809,8 +809,8 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = PackOp::create(b,
-        loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
+    Operation *tiledPackOp = PackOp::create(
+        b, loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
         {tiledPackOp},
@@ -968,8 +968,8 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = tensor::ExtractSliceOp::create(b,
-        loc, packOp.getSource(), offsets, sizes, strides);
+    auto sourceSlice = tensor::ExtractSliceOp::create(
+        b, loc, packOp.getSource(), offsets, sizes, strides);
     tiledOperands.push_back(sourceSlice);
 
     SmallVector<OpFoldResult> outerDimOffsets, outerDimSizes;
@@ -984,16 +984,16 @@ struct PackOpTiling
       return failure();
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = tensor::ExtractSliceOp::create(b,
-        loc, packOp.getDest(), outputOffsets, outputSizes, strides);
+    auto outSlice = tensor::ExtractSliceOp::create(
+        b, loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
     assert(!packOp.getPaddingValue() && "Expect no padding semantic");
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = PackOp::create(b,
-        loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
+    Operation *tiledPackOp = PackOp::create(
+        b, loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
         {tiledPackOp},
@@ -1172,37 +1172,37 @@ struct UnPackOpTiling
     sliceSrcSizes.append(unpackOp.getMixedTiles());
     sliceSrcStrides.append(numInnerTiles, oneAttr);
     SmallVector<Operation *> generatedSlices;
-    tensor::ExtractSliceOp sliceSource = tensor::ExtractSliceOp::create(b,
-        loc, unpackOp.getSource(), sliceSrcIndices, sliceSrcSizes,
+    tensor::ExtractSliceOp sliceSource = tensor::ExtractSliceOp::create(
+        b, loc, unpackOp.getSource(), sliceSrcIndices, sliceSrcSizes,
         sliceSrcStrides);
     generatedSlices.push_back(sliceSource);
 
     SmallVector<OpFoldResult> destStrides(destRank, oneAttr);
     Value sliceDest;
     if (isPerfectTilingCase) {
-      auto destSliceOp = tensor::ExtractSliceOp::create(b,
-          loc, unpackOp.getDest(), offsets, sizes, destStrides);
+      auto destSliceOp = tensor::ExtractSliceOp::create(
+          b, loc, unpackOp.getDest(), offsets, sizes, destStrides);
       sliceDest = destSliceOp;
       generatedSlices.push_back(destSliceOp);
     } else {
-      sliceDest = tensor::EmptyOp::create(b,
-          loc, destExpandedSizes, unpackOp.getDestType().getElementType());
+      sliceDest = tensor::EmptyOp::create(
+          b, loc, destExpandedSizes, unpackOp.getDestType().getElementType());
     }
 
     SmallVector<Value> tiledOperands = {sliceSource.getResult(), sliceDest};
     for (auto tile : unpackOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledUnpackOp = UnPackOp::create(b,
-        loc, TypeRange{sliceDest.getType()}, tiledOperands, op->getAttrs());
+    Operation *tiledUnpackOp = UnPackOp::create(
+        b, loc, TypeRange{sliceDest.getType()}, tiledOperands, op->getAttrs());
 
     if (isPerfectTilingCase)
       return TilingResult{{tiledUnpackOp},
                           SmallVector<Value>(tiledUnpackOp->getResults()),
                           generatedSlices};
 
-    auto extractSlice = tensor::ExtractSliceOp::create(b,
-        loc, tiledUnpackOp->getResult(0), resultOffsetsFromDest, sizes,
+    auto extractSlice = tensor::ExtractSliceOp::create(
+        b, loc, tiledUnpackOp->getResult(0), resultOffsetsFromDest, sizes,
         destStrides);
     return TilingResult{
         {tiledUnpackOp}, {extractSlice.getResult()}, generatedSlices};
@@ -1337,14 +1337,14 @@ struct UnPackOpTiling
 
     SmallVector<Value> tiledOperands;
     // Create slice of the dest operand.
-    auto extractDestSlice = tensor::ExtractSliceOp::create(b,
-        loc, unPackOp.getDest(), outputOffsets, outputSizes, strides);
+    auto extractDestSlice = tensor::ExtractSliceOp::create(
+        b, loc, unPackOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(extractDestSlice);
 
     strides.append(unPackOp.getSourceRank() - outputRank, oneAttr);
     // Create slice of the source operand.
-    auto extractSourceSlice = tensor::ExtractSliceOp::create(b,
-        loc, unPackOp.getSource(), offsets, sizes, strides);
+    auto extractSourceSlice = tensor::ExtractSliceOp::create(
+        b, loc, unPackOp.getSource(), offsets, sizes, strides);
     tiledOperands.insert(tiledOperands.begin(), extractSourceSlice);
     for (auto tile : unPackOp.getInnerTiles())
       tiledOperands.push_back(tile);
@@ -1352,7 +1352,7 @@ struct UnPackOpTiling
     // Create tiled unpack op.
     Operation *tiledUnPackOp =
         UnPackOp::create(b, loc, TypeRange{extractDestSlice.getType()},
-                           tiledOperands, op->getAttrs());
+                         tiledOperands, op->getAttrs());
 
     return TilingResult{{tiledUnPackOp},
                         SmallVector<Value>(tiledUnPackOp->getResults()),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 3526f62f770dc..1f1e617738981 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -269,12 +269,12 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       packingMetadata.reassociations);
   Value paddingValue = packOp.getPaddingValue();
   if (!paddingValue) {
-    paddingValue = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(getElementTypeOrSelf(collapsed)));
+    paddingValue = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getZeroAttr(getElementTypeOrSelf(collapsed)));
   }
   auto padOp =
       tensor::PadOp::create(rewriter, loc, collapsed, packOp.getSource(), lows,
-                                     highs, paddingValue, /*nofold=*/false);
+                            highs, paddingValue, /*nofold=*/false);
 
   LLVM_DEBUG(
       DBGSNL(); DBGSNL();
@@ -313,8 +313,8 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       SmallVector<OpFoldResult> sizes =
           tensor::getMixedSizes(rewriter, loc, packOp.getDest());
 
-      auto insertSliceOp = tensor::InsertSliceOp::create(rewriter,
-          loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
+      auto insertSliceOp = tensor::InsertSliceOp::create(
+          rewriter, loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
           /*offsets=*/zeros, sizes, /*strides=*/ones);
 
       LLVM_DEBUG(DBGS() << "insert_slice op: " << insertSliceOp; DBGSNL(););
@@ -329,15 +329,15 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   // 5. Expand from the padded result to the stripMinedShape.
   auto expandShapeResultType =
       RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape);
-  auto reshapeOp = tensor::ExpandShapeOp::create(rewriter,
-      loc, expandShapeResultType, padOp.getResult(),
+  auto reshapeOp = tensor::ExpandShapeOp::create(
+      rewriter, loc, expandShapeResultType, padOp.getResult(),
       packingMetadata.reassociations);
 
   // 6. Transpose stripMinedShape to packedShape.
   SmallVector<int64_t> transpPerm =
       invertPermutationVector(packedToStripMinedShapePerm);
-  auto transposeOp = linalg::TransposeOp::create(rewriter,
-      loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
+  auto transposeOp = linalg::TransposeOp::create(
+      rewriter, loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
 
   LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
              DBGS() << "reshape op: " << reshapeOp; DBGSNL();
@@ -371,8 +371,8 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
     SmallVector<OpFoldResult> sizes(packedRank - destShape.size(), one);
     sizes.append(tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()));
 
-    auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
-        loc, destTensorType, unPackOp.getSource(),
+    auto extractSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, loc, destTensorType, unPackOp.getSource(),
         SmallVector<OpFoldResult>(packedRank, zero), sizes,
         SmallVector<OpFoldResult>(packedRank, one));
 
@@ -404,10 +404,11 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
   SmallVector<OpFoldResult, 4> dims =
       tensor::getMixedSizes(rewriter, loc, unPackOp.getSource());
   applyPermutationToVector(dims, packedToStripMinedShapePerm);
-  auto emptyOp = tensor::EmptyOp::create(rewriter,
-      loc, dims, stripMinedTensorType.getElementType());
-  auto transposeOp = linalg::TransposeOp::create(rewriter,
-      loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm);
+  auto emptyOp = tensor::EmptyOp::create(rewriter, loc, dims,
+                                         stripMinedTensorType.getElementType());
+  auto transposeOp =
+      linalg::TransposeOp::create(rewriter, loc, unPackOp.getSource(), emptyOp,
+                                  packedToStripMinedShapePerm);
 
   LLVM_DEBUG(
       DBGSNL(); DBGSNL();
@@ -426,21 +427,21 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
       DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
 
   // 4. Collapse from the stripMinedShape to the padded result.
-  auto reshapeOp = tensor::CollapseShapeOp::create(rewriter,
-      loc, collapsedType, transposeOp->getResult(0),
+  auto reshapeOp = tensor::CollapseShapeOp::create(
+      rewriter, loc, collapsedType, transposeOp->getResult(0),
       packingMetadata.reassociations);
 
   // 5. ExtractSlice.
   int64_t destRank = destTensorType.getRank();
-  auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
-      loc, destTensorType, reshapeOp->getResult(0),
+  auto extractSliceOp = tensor::ExtractSliceOp::create(
+      rewriter, loc, destTensorType, reshapeOp->getResult(0),
       SmallVector<OpFoldResult>(destRank, zero),
       tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()),
       SmallVector<OpFoldResult>(destRank, one));
 
   // 6. Inject a copy to preserve DPS.
-  auto copyOp = linalg::CopyOp::create(rewriter,
-      loc, extractSliceOp->getResult(0), unPackOp.getDest());
+  auto copyOp = linalg::CopyOp::create(
+      rewriter, loc, extractSliceOp->getResult(0), unPackOp.getDest());
 
   // 7. Replace unPackOp by copyOp.
   rewriter.replaceOp(unPackOp, copyOp->getResults());
@@ -554,16 +555,16 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
               operandType.getShape(), innerPos,
               cast<ShapedType>(dest.getType()).getShape(), {},
               innerPackSizes)) {
-        packOps.push_back(linalg::PackOp::create(rewriter,
-            loc, operand, dest, innerPos, innerPackSizes));
+        packOps.push_back(linalg::PackOp::create(rewriter, loc, operand, dest,
+                                                 innerPos, innerPackSizes));
       } else {
         // TODO: value of the padding attribute should be determined by
         // consumers.
         auto zeroAttr =
             rewriter.getZeroAttr(getElementTypeOrSelf(dest.getType()));
         Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
-        packOps.push_back(linalg::PackOp::create(rewriter,
-            loc, operand, dest, innerPos, innerPackSizes, zero));
+        packOps.push_back(linalg::PackOp::create(
+            rewriter, loc, operand, dest, innerPos, innerPackSizes, zero));
       }
       inputsAndInits.push_back(packOps.back());
     }
@@ -574,9 +575,9 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       ValueRange{inputsAndInits}.take_front(linalgOp.getNumDpsInputs());
   ValueRange inits =
       ValueRange{inputsAndInits}.take_back(linalgOp.getNumDpsInits());
-  auto packedLinalgOp = linalg::GenericOp::create(rewriter,
-      linalgOp.getLoc(), inits.getTypes(), inputs, inits, indexingMaps,
-      iteratorTypes);
+  auto packedLinalgOp =
+      linalg::GenericOp::create(rewriter, linalgOp.getLoc(), inits.getTypes(),
+                                inputs, inits, indexingMaps, iteratorTypes);
   packedLinalgOp.getRegion().takeBody(linalgOp->getRegion(0));
 
   // Step 4. Propagate packing to all the op results.
@@ -589,8 +590,8 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       continue;
     }
     // Build the symmetrical UnPackOp to the existing PackOp.
-    unPackOps.push_back(linalg::UnPackOp::create(rewriter,
-        packedLinalgOp->getLoc(), result, maybePackedInit.getSource(),
+    unPackOps.push_back(linalg::UnPackOp::create(
+        rewriter, packedLinalgOp->getLoc(), result, maybePackedInit.getSource(),
         maybePackedInit.getInnerDimsPos(), maybePackedInit.getMixedTiles()));
     results.push_back(unPackOps.back());
   }
@@ -655,7 +656,8 @@ static LinalgOp transposeOneLinalgOperandAndReplace(
   operands[opOperand.getOperandNumber()] = transposedValue;
 
   ValueRange operandsRef(operands);
-  auto transposedGenericOp = linalg::GenericOp::create(rewriter,
+  auto transposedGenericOp = linalg::GenericOp::create(
+      rewriter,
       /*location=*/linalgOp->getLoc(),
       /*resultTensorTypes=*/
       operandsRef.drop_front(linalgOp.getNumDpsInputs()).getTypes(),
@@ -930,8 +932,8 @@ Value DecomposePadOpPattern::createFillOrGenerateOp(
   }
 
   // Fill could not be optimized: Lower to tensor::GenerateOp with region.
-  auto generateOp = tensor::GenerateOp::create(rewriter,
-      padOp.getLoc(), padOp.getResultType(), dynSizes);
+  auto generateOp = tensor::GenerateOp::create(rewriter, padOp.getLoc(),
+                                               padOp.getResultType(), dynSizes);
   // Copy region to new op.
   IRMapping bvm;
   padOp.getRegion().cloneInto(&generateOp.getRegion(), bvm);
@@ -970,8 +972,9 @@ DecomposePadOpPattern::matchAndRewrite(tensor::PadOp padOp,
   }
 
   // Init tensor and fill it with padding.
-  Value emptyTensor = tensor::EmptyOp::create(rewriter,
-      padOp.getLoc(), staticSizes, resultType.getElementType(), dynSizes);
+  Value emptyTensor =
+      tensor::EmptyOp::create(rewriter, padOp.getLoc(), staticSizes,
+                              resultType.getElementType(), dynSizes);
   Value fill = createFillOrGenerateOp(rewriter, padOp, emptyTensor, dynSizes);
 
   // Generate a InsertSliceOp for copying the PadOp source.
@@ -1222,12 +1225,13 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
 
   applyPermutationToVector<OpFoldResult>(transShapeForEmptyOp,
                                          srcPermForTranspose);
-  Value empty = tensor::EmptyOp::create(rewriter,
-      loc, transShapeForEmptyOp, packOp.getSourceType().getElementType());
+  Value empty =
+      tensor::EmptyOp::create(rewriter, loc, transShapeForEmptyOp,
+                              packOp.getSourceType().getElementType());
 
   // 2.2 Create linalg.transpose
   auto transposedOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
-                                                           srcPermForTranspose);
+                                                  srcPermForTranspose);
 
   // 3. Insert the inner tile to the destination:
   //  %inserted_tile = tensor.insert_slice(%transposed_tile)
@@ -1246,9 +1250,9 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   }
 
   // 4. Replace tensor.packOp with tensor.insert_slice created above
-  auto insert = tensor::InsertSliceOp::create(rewriter,
-      loc, transposedOp.getResult()[0], packOp.getDest(), writeOffsets,
-      writeSizes, writeStrides);
+  auto insert = tensor::InsertSliceOp::create(
+      rewriter, loc, transposedOp.getResult()[0], packOp.getDest(),
+      writeOffsets, writeSizes, writeStrides);
   rewriter.replaceOp(packOp, insert.getResult());
 
   return success();
@@ -1340,8 +1344,8 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   readShapeForExtractSlice.append(tileShape.begin(), tileShape.end());
   Type elemType = unpackOp.getSourceType().getElementType();
   auto readType = RankedTensorType::get(readShapeForExtractSlice, elemType);
-  Value innerTile = tensor::ExtractSliceOp::create(rewriter,
-      loc, readType, unpackOp.getSource(), extractSliceOffsets,
+  Value innerTile = tensor::ExtractSliceOp::create(
+      rewriter, loc, readType, unpackOp.getSource(), extractSliceOffsets,
       extractSliceSizes, extractSliceStrides);
 
   // 2. Transpose the tile to match the outer corresponding tile order.
@@ -1369,8 +1373,9 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
           tensor::getMixedSize(rewriter, loc, unpackOp.getDest(), i));
   }
 
-  auto partialTile = tensor::ExtractSliceOp::create(rewriter,
-      loc, transposedOp.getResult()[0], tileOffsets, tileSizes, tileStrides);
+  auto partialTile =
+      tensor::ExtractSliceOp::create(rewriter, loc, transposedOp.getResult()[0],
+                                     tileOffsets, tileSizes, tileStrides);
 
   // 4. Insert the result to the destination tensor.
   SmallVector<OpFoldResult> writeSizes;
@@ -1382,9 +1387,9 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     else
       writeSizes.push_back(oneIdxAttr);
   }
-  auto insert = tensor::InsertSliceOp::create(rewriter,
-      loc, partialTile, unpackOp.getDest(), writeOffsets, writeSizes,
-      writeStrides);
+  auto insert = tensor::InsertSliceOp::create(rewriter, loc, partialTile,
+                                              unpackOp.getDest(), writeOffsets,
+                                              writeSizes, writeStrides);
   rewriter.replaceOp(unpackOp, insert.getResult());
 
   return success();
@@ -1491,8 +1496,8 @@ FailureOr<Conv1DOp> DownscaleSizeOneWindowed2DConvolution<Conv2DOp, Conv1DOp>::
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = Conv1DOp::create(rewriter,
-      loc, newOutputType, ValueRange{newInput, newKernel},
+  auto conv1DOp = Conv1DOp::create(
+      rewriter, loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
   // Insert back.
@@ -1578,8 +1583,8 @@ DownscaleDepthwiseConv2DNhwcHwcOp::returningMatchAndRewrite(
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = DepthwiseConv1DNwcWcOp::create(rewriter,
-      loc, newOutputType, ValueRange{newInput, newKernel},
+  auto conv1DOp = DepthwiseConv1DNwcWcOp::create(
+      rewriter, loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
   // Insert back.
@@ -1635,9 +1640,9 @@ DownscaleConv2DOp::returningMatchAndRewrite(Conv2DOp convOp,
   Value newOutput = tensor::createCanonicalRankReducingExtractSliceOp(
       rewriter, loc, output, newOutputType);
 
-  auto conv1DOp = Conv1DOp::create(rewriter, loc, newOutputType,
-                                            ValueRange{newInput, newKernel},
-                                            ValueRange{newOutput});
+  auto conv1DOp =
+      Conv1DOp::create(rewriter, loc, newOutputType,
+                       ValueRange{newInput, newKernel}, ValueRange{newOutput});
 
   // Insert back.
   Value inserted = tensor::createCanonicalRankReducingInsertSliceOp(
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index b933a2d143925..99fb8c796cf06 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -99,7 +99,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
   }
   auto newConv =
       HWCFConvOp::create(rewriter, loc, resultTy, newInputs, op.getOutputs(),
-                                  op.getStrides(), op.getDilations());
+                         op.getStrides(), op.getDilations());
   rewriter.replaceOp(op, newConv);
   return newConv.getOperation();
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index 6b1e89c1594d1..a2a4335ef8e22 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -52,20 +52,20 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
     dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = tensor::EmptyOp::create(rewriter,
-      loc, ArrayRef<int64_t>{shape[1], shape[0]}, type.getElementType(),
-      dynamicDims);
-  auto transposeOp = linalg::TransposeOp::create(rewriter,
-      loc, input, empty, ArrayRef<int64_t>{1, 0});
+  Value empty = tensor::EmptyOp::create(rewriter, loc,
+                                        ArrayRef<int64_t>{shape[1], shape[0]},
+                                        type.getElementType(), dynamicDims);
+  auto transposeOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
+                                                 ArrayRef<int64_t>{1, 0});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = linalg::MatmulTransposeAOp::create(rewriter,
-        loc, matmulOp.getResultTypes(),
+    newMatmulOp = linalg::MatmulTransposeAOp::create(
+        rewriter, loc, matmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), matmulOp.getInputs()[1]},
         matmulOp.getOutputs());
   } else {
-    newMatmulOp = linalg::MatmulTransposeBOp::create(rewriter,
-        loc, matmulOp.getResultTypes(),
+    newMatmulOp = linalg::MatmulTransposeBOp::create(
+        rewriter, loc, matmulOp.getResultTypes(),
         ValueRange{matmulOp.getInputs()[0], transposeOp->getResult(0)},
         matmulOp.getOutputs());
   }
@@ -109,20 +109,20 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
     dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = tensor::EmptyOp::create(rewriter,
-      loc, ArrayRef<int64_t>{shape[0], shape[2], shape[1]},
+  Value empty = tensor::EmptyOp::create(
+      rewriter, loc, ArrayRef<int64_t>{shape[0], shape[2], shape[1]},
       type.getElementType(), dynamicDims);
-  auto transposeOp = linalg::TransposeOp::create(rewriter,
-      loc, input, empty, ArrayRef<int64_t>{0, 2, 1});
+  auto transposeOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
+                                                 ArrayRef<int64_t>{0, 2, 1});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(rewriter,
-        loc, batchMatmulOp.getResultTypes(),
+    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(
+        rewriter, loc, batchMatmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), batchMatmulOp.getInputs()[1]},
         batchMatmulOp.getOutputs());
   } else {
-    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(rewriter,
-        loc, batchMatmulOp.getResultTypes(),
+    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(
+        rewriter, loc, batchMatmulOp.getResultTypes(),
         ValueRange{batchMatmulOp.getInputs()[0], transposeOp->getResult(0)},
         batchMatmulOp.getOutputs());
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 2fff28ffc7474..272537f22deb3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -119,8 +119,9 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-            loc, input, /*offsets=*/ArrayRef<int64_t>{w + kw}, sizes, strides));
+        result.push_back(vector::ExtractStridedSliceOp::create(
+            rewriter, loc, input, /*offsets=*/ArrayRef<int64_t>{w + kw}, sizes,
+            strides));
       }
     }
   } else {
@@ -130,8 +131,8 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-            loc, input,
+        result.push_back(vector::ExtractStridedSliceOp::create(
+            rewriter, loc, input,
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             sizes, strides));
       }
@@ -149,8 +150,8 @@ static SmallVector<Value> extractConvFilterSlices(RewriterBase &rewriter,
   // Extract rhs slice of size [{c, f} for channeled convolutions and {1} for
   // non-chanelled convolution] @ [kw].
   for (int64_t kw = 0; kw < kwSize; ++kw) {
-    result.push_back(vector::ExtractOp::create(rewriter,
-        loc, filter, /*offsets=*/ArrayRef<int64_t>{kw}));
+    result.push_back(vector::ExtractOp::create(
+        rewriter, loc, filter, /*offsets=*/ArrayRef<int64_t>{kw}));
   }
   return result;
 }
@@ -167,8 +168,9 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {wSizeStep};
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-          loc, res, /*offsets=*/ArrayRef<int64_t>{w}, sizes, strides));
+      result.push_back(vector::ExtractStridedSliceOp::create(
+          rewriter, loc, res, /*offsets=*/ArrayRef<int64_t>{w}, sizes,
+          strides));
     }
   } else {
     // Extract res slice: {n, wSizeStep, f} @ [0, w, 0] for channeled
@@ -176,8 +178,9 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {nSize, wSizeStep, fSize};
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-          loc, res, /*offsets=*/ArrayRef<int64_t>{0, w, 0}, sizes, strides));
+      result.push_back(vector::ExtractStridedSliceOp::create(
+          rewriter, loc, res, /*offsets=*/ArrayRef<int64_t>{0, w, 0}, sizes,
+          strides));
     }
   }
   return result;
@@ -194,17 +197,18 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // This does not depend on kw.
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = vector::InsertStridedSliceOp::create(rewriter,
-          loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{w}, strides);
+      res = vector::InsertStridedSliceOp::create(
+          rewriter, loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{w},
+          strides);
     }
   } else {
     // Write back res slice: {n, wSizeStep, f} @ [0, w, 0] for channeled
     // convolution. This does not depend on kw.
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = vector::InsertStridedSliceOp::create(rewriter,
-          loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{0, w, 0},
-          strides);
+      res = vector::InsertStridedSliceOp::create(
+          rewriter, loc, resVals[w], res,
+          /*offsets=*/ArrayRef<int64_t>{0, w, 0}, strides);
     }
   }
   return res;
@@ -337,8 +341,8 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
   for (int vecDim = 0, end = canonicalVecShape.size(); vecDim < end; ++vecDim) {
     if (ShapedType::isStatic(iterSpaceStaticSizes[vecDim])) {
       // Create constant index op for static dimensions.
-      iterSpaceValueSizes.push_back(arith::ConstantIndexOp::create(rewriter,
-          linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
+      iterSpaceValueSizes.push_back(arith::ConstantIndexOp::create(
+          rewriter, linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
       continue;
     }
 
@@ -350,11 +354,12 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
                                                          operandDimPos)))
       return failure();
 
-    Value dynamicDim = linalgOp.hasPureTensorSemantics()
-                           ? (Value)tensor::DimOp::create(rewriter,
-                                 linalgOp.getLoc(), operand, operandDimPos)
-                           : (Value)memref::DimOp::create(rewriter,
-                                 linalgOp.getLoc(), operand, operandDimPos);
+    Value dynamicDim =
+        linalgOp.hasPureTensorSemantics()
+            ? (Value)tensor::DimOp::create(rewriter, linalgOp.getLoc(), operand,
+                                           operandDimPos)
+            : (Value)memref::DimOp::create(rewriter, linalgOp.getLoc(), operand,
+                                           operandDimPos);
     iterSpaceValueSizes.push_back(dynamicDim);
   }
 
@@ -475,7 +480,7 @@ Value VectorizationState::getOrCreateMaskFor(
 
   // Create the mask based on the dimension values.
   Value mask = vector::CreateMaskOp::create(rewriter, linalgOp.getLoc(),
-                                                     maskType, upperBounds);
+                                            maskType, upperBounds);
   LDBG("Creating new mask: " << mask << "\n");
   activeMaskCache[maskingMap] = mask;
   return mask;
@@ -643,8 +648,8 @@ static Operation *buildMultiDimReduce(OpBuilder &b, Operation *reduceOp,
                                       ArrayRef<bool> dimsToMask) {
   auto maybeKind = getCombinerOpKind(reduceOp);
   assert(maybeKind && "Failed precondition: could not get reduction kind");
-  return vector::MultiDimReductionOp::create(b,
-      reduceOp->getLoc(), valueToReduce, acc, dimsToMask, *maybeKind);
+  return vector::MultiDimReductionOp::create(
+      b, reduceOp->getLoc(), valueToReduce, acc, dimsToMask, *maybeKind);
 }
 
 static SmallVector<bool> getDimsToReduce(LinalgOp linalgOp) {
@@ -688,19 +693,20 @@ static Value buildVectorWrite(RewriterBase &rewriter, Value value,
   Operation *write;
   if (vectorType.getRank() > 0) {
     AffineMap writeMap = inversePermutation(reindexIndexingMap(opOperandMap));
-    SmallVector<Value> indices(linalgOp.getRank(outputOperand),
-                               arith::ConstantIndexOp::create(rewriter, loc, 0));
+    SmallVector<Value> indices(
+        linalgOp.getRank(outputOperand),
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
     value = broadcastIfNeeded(rewriter, value, vectorType);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = vector::TransferWriteOp::create(rewriter,
-        loc, value, outputOperand->get(), indices, writeMap);
+    write = vector::TransferWriteOp::create(
+        rewriter, loc, value, outputOperand->get(), indices, writeMap);
   } else {
     // 0-d case is still special: do not invert the reindexing writeMap.
     if (!isa<VectorType>(value.getType()))
       value = vector::BroadcastOp::create(rewriter, loc, vectorType, value);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = vector::TransferWriteOp::create(rewriter,
-        loc, value, outputOperand->get(), ValueRange{});
+    write = vector::TransferWriteOp::create(rewriter, loc, value,
+                                            outputOperand->get(), ValueRange{});
   }
 
   write = state.maskOperation(rewriter, write, linalgOp, opOperandMap);
@@ -793,9 +799,9 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto permMap =
       AffineMap::getPermutationMap(permPattern, linalgOp.getContext());
 
-  auto broadCastOp = vector::BroadcastOp::create(rewriter,
-      loc, state.getCanonicalVecType(rewriter.getIndexType(), permMap),
-      indexSteps);
+  auto broadCastOp = vector::BroadcastOp::create(
+      rewriter, loc,
+      state.getCanonicalVecType(rewriter.getIndexType(), permMap), indexSteps);
   SmallVector<int64_t> transposition =
       llvm::to_vector<16>(llvm::seq<int64_t>(0, linalgOp.getNumLoops()));
   std::swap(transposition.back(), transposition[dim]);
@@ -1110,12 +1116,12 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
 
   // Compute the static loop sizes of the extract op.
   auto resultType = state.getCanonicalVecType(extractOp.getResult().getType());
-  auto maskConstantOp = arith::ConstantOp::create(rewriter,
-      loc,
+  auto maskConstantOp = arith::ConstantOp::create(
+      rewriter, loc,
       DenseIntElementsAttr::get(state.getCanonicalVecType(rewriter.getI1Type()),
                                 /*value=*/true));
-  auto passThruConstantOp =
-      arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(resultType));
+  auto passThruConstantOp = arith::ConstantOp::create(
+      rewriter, loc, rewriter.getZeroAttr(resultType));
 
   // Base indices are currently set to 0. We will need to re-visit if more
   // generic scenarios are to be supported.
@@ -1131,8 +1137,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     Value offset = calculateGatherOffset(rewriter, state, extractOp, bvm);
 
     // Generate the gather load
-    Operation *gatherOp = vector::GatherOp::create(rewriter,
-        loc, resultType, extractOp.getTensor(), baseIndices, offset,
+    Operation *gatherOp = vector::GatherOp::create(
+        rewriter, loc, resultType, extractOp.getTensor(), baseIndices, offset,
         maskConstantOp, passThruConstantOp);
     gatherOp = state.maskOperation(rewriter, gatherOp, linalgOp);
 
@@ -1166,8 +1172,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
       continue;
     }
 
-    auto indexAs1dVector = vector::ShapeCastOp::create(rewriter,
-        loc,
+    auto indexAs1dVector = vector::ShapeCastOp::create(
+        rewriter, loc,
         VectorType::get(resultType.getShape().back(), rewriter.getIndexType(),
                         resultType.getScalableDims().back()),
         idx);
@@ -1186,8 +1192,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     SmallVector<AffineExpr> exprs(dstRank, getAffineConstantExpr(0, ctx));
     auto permutationMap = AffineMap::get(srcRank, 0, exprs, ctx);
 
-    auto transferReadOp = vector::TransferReadOp::create(rewriter,
-        loc, resultType, extractOp.getTensor(), transferReadIdxs,
+    auto transferReadOp = vector::TransferReadOp::create(
+        rewriter, loc, resultType, extractOp.getTensor(), transferReadIdxs,
         /*padding=*/std::nullopt, permutationMap, inBounds);
 
     // Mask this broadcasting xfer_read here rather than relying on the generic
@@ -1195,8 +1201,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     // valid here).
     SmallVector<int64_t> readMaskShape = {1};
     auto readMaskType = VectorType::get(readMaskShape, rewriter.getI1Type());
-    auto allTrue = vector::ConstantMaskOp::create(rewriter,
-        loc, readMaskType, vector::ConstantMaskKind::AllTrue);
+    auto allTrue = vector::ConstantMaskOp::create(
+        rewriter, loc, readMaskType, vector::ConstantMaskKind::AllTrue);
     auto *maskedReadOp =
         mlir::vector::maskOperation(rewriter, transferReadOp, allTrue);
 
@@ -1223,8 +1229,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     rankDiff--;
   }
 
-  auto transferReadOp = vector::TransferReadOp::create(rewriter,
-      loc, resultType, extractOp.getTensor(), transferReadIdxs,
+  auto transferReadOp = vector::TransferReadOp::create(
+      rewriter, loc, resultType, extractOp.getTensor(), transferReadIdxs,
       /*padding=*/std::nullopt, permutationMap, inBounds);
 
   LDBG("Vectorised as contiguous load: " << extractOp);
@@ -1435,8 +1441,8 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
-    Operation *read = vector::TransferReadOp::create(rewriter,
-        loc, readType, opOperand->get(), indices,
+    Operation *read = vector::TransferReadOp::create(
+        rewriter, loc, readType, opOperand->get(), indices,
         /*padding=*/std::nullopt, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
     Value readValue = read->getResult(0);
@@ -1453,7 +1459,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     // TODO: remove this.
     if (readType.getRank() == 0)
       readValue = vector::ExtractOp::create(rewriter, loc, readValue,
-                                                     ArrayRef<int64_t>());
+                                            ArrayRef<int64_t>());
 
     LDBG("New vectorized bbarg(" << bbarg.getArgNumber() << "): " << readValue
                                  << "\n");
@@ -1665,12 +1671,11 @@ createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
   }
 
   // Generate the xfer_write Op
-  Operation *write =
-      vector::TransferWriteOp::create(builder, loc,
-                                              /*vector=*/vecToStore,
-                                              /*source=*/dest,
-                                              /*indices=*/writeIndices,
-                                              /*inBounds=*/inBoundsVal);
+  Operation *write = vector::TransferWriteOp::create(builder, loc,
+                                                     /*vector=*/vecToStore,
+                                                     /*source=*/dest,
+                                                     /*indices=*/writeIndices,
+                                                     /*inBounds=*/inBoundsVal);
 
   // If masking is disabled, exit.
   if (useInBoundsInsteadOfMasking)
@@ -1742,8 +1747,9 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   Location loc = packOp.getLoc();
   auto padValue = packOp.getPaddingValue();
   if (!padValue) {
-    padValue = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(packOp.getSourceType().getElementType()));
+    padValue = arith::ConstantOp::create(
+        rewriter, loc,
+        rewriter.getZeroAttr(packOp.getSourceType().getElementType()));
   }
   ReifiedRankedShapedTypeDims reifiedReturnShapes;
   LogicalResult status =
@@ -1787,12 +1793,12 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   // Create TransposeOp.
   auto destPermutation =
       invertPermutationVector(getPackInverseDestPerm(packOp));
-  auto transposeOp = vector::TransposeOp::create(rewriter,
-      loc, shapeCastOp.getResult(), destPermutation);
+  auto transposeOp = vector::TransposeOp::create(
+      rewriter, loc, shapeCastOp.getResult(), destPermutation);
 
   // Create TransferWriteOp.
-  Value dest = tensor::EmptyOp::create(rewriter,
-      loc, reifiedReturnShapes[0],
+  Value dest = tensor::EmptyOp::create(
+      rewriter, loc, reifiedReturnShapes[0],
       transposeOp.getResult().getType().getElementType());
   Operation *write =
       createWriteOrMaskedWrite(rewriter, loc, transposeOp.getResult(), dest);
@@ -1892,8 +1898,9 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   }
   Location loc = unpackOp->getLoc();
 
-  auto padValue = arith::ConstantOp::create(rewriter,
-      loc, rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
+  auto padValue = arith::ConstantOp::create(
+      rewriter, loc,
+      rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
 
   // Read result, mask if necessary. If transferReadOp shape is not equal
   // to shape of source, then a mask is necessary.
@@ -1911,16 +1918,16 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   RankedTensorType stripMineTensorType =
       RankedTensorType::get(stripMineShape, stripMineElemType);
   // Transpose the appropriate rows to match output.
-  vector::TransposeOp transposeOp = vector::TransposeOp::create(rewriter,
-      loc, readResult, lastDimToInsertPosPerm);
+  vector::TransposeOp transposeOp = vector::TransposeOp::create(
+      rewriter, loc, readResult, lastDimToInsertPosPerm);
 
   // Collapse the vector to the size required by result.
   RankedTensorType collapsedType = tensor::CollapseShapeOp::inferCollapsedType(
       stripMineTensorType, packMetadata.reassociations);
   mlir::VectorType vecCollapsedType =
       VectorType::get(collapsedType.getShape(), collapsedType.getElementType());
-  vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(rewriter,
-      loc, vecCollapsedType, transposeOp->getResult(0));
+  vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(
+      rewriter, loc, vecCollapsedType, transposeOp->getResult(0));
 
   // writeVectorSizes had to match the shapecast shape for dynamic sizes,
   // otherwise the validator complains that the mask size is invalid.
@@ -1928,8 +1935,8 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       unpackOp.getDestType().hasStaticShape()
           ? vectorSizes
           : shapeCastOp.getResultVectorType().getShape());
-  Value dest = tensor::EmptyOp::create(rewriter,
-      loc, reifiedRetShapes[0],
+  Value dest = tensor::EmptyOp::create(
+      rewriter, loc, reifiedRetShapes[0],
       shapeCastOp.getResult().getType().getElementType());
   Operation *write = createWriteOrMaskedWrite(
       rewriter, loc, shapeCastOp.getResult(), dest,
@@ -1963,8 +1970,8 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
       /*useInBoundsInsteadOfMasking=*/false);
 
   // Create Xfer write Op
-  Value dest = tensor::EmptyOp::create(rewriter,
-      loc, reifiedReturnShapes[0], padOp.getResultType().getElementType());
+  Value dest = tensor::EmptyOp::create(rewriter, loc, reifiedReturnShapes[0],
+                                       padOp.getResultType().getElementType());
   Operation *write = createWriteOrMaskedWrite(rewriter, loc, maskedRead, dest);
   newResults.push_back(write->getResult(0));
   return success();
@@ -2637,17 +2644,18 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   SmallVector<Value> indices(srcType.getRank(), zero);
 
-  Value readValue = vector::TransferReadOp::create(rewriter,
-      loc, readType, copyOp.getSource(), indices,
+  Value readValue = vector::TransferReadOp::create(
+      rewriter, loc, readType, copyOp.getSource(), indices,
       /*padding=*/std::nullopt,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   if (cast<VectorType>(readValue.getType()).getRank() == 0) {
+    readValue = vector::ExtractOp::create(rewriter, loc, readValue,
+                                          ArrayRef<int64_t>());
     readValue =
-        vector::ExtractOp::create(rewriter, loc, readValue, ArrayRef<int64_t>());
-    readValue = vector::BroadcastOp::create(rewriter, loc, writeType, readValue);
+        vector::BroadcastOp::create(rewriter, loc, writeType, readValue);
   }
-  Operation *writeValue = vector::TransferWriteOp::create(rewriter,
-      loc, readValue, copyOp.getTarget(), indices,
+  Operation *writeValue = vector::TransferWriteOp::create(
+      rewriter, loc, readValue, copyOp.getTarget(), indices,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   rewriter.replaceOp(copyOp, writeValue->getResults());
   return success();
@@ -2957,8 +2965,8 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   if (!padValue) {
     auto elemType = sourceType.getElementType();
-    padValue = arith::ConstantOp::create(rewriter,
-        sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
+    padValue = arith::ConstantOp::create(rewriter, sliceOp.getLoc(), elemType,
+                                         rewriter.getZeroAttr(elemType));
   }
 
   // 2. Get the vector shape
@@ -3077,8 +3085,9 @@ struct PadOpVectorizationWithInsertSlicePattern
     // padding.
     SmallVector<Value> readIndices(
         vecRank, arith::ConstantIndexOp::create(rewriter, padOp.getLoc(), 0));
-    auto read = vector::TransferReadOp::create(rewriter,
-        padOp.getLoc(), vecType, padOp.getSource(), readIndices, padValue);
+    auto read = vector::TransferReadOp::create(rewriter, padOp.getLoc(),
+                                               vecType, padOp.getSource(),
+                                               readIndices, padValue);
 
     // Generate TransferWriteOp: Write to InsertSliceOp's dest tensor at
     // specified offsets. Write is fully in-bounds because a InsertSliceOp's
@@ -3212,8 +3221,8 @@ LogicalResult LinalgCopyVTRForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_read, the attribute must be reset
   // conservatively.
   auto vectorType = xferOp.getVectorType();
-  Value res = vector::TransferReadOp::create(rewriter,
-      xferOp.getLoc(), vectorType, in, xferOp.getIndices(),
+  Value res = vector::TransferReadOp::create(
+      rewriter, xferOp.getLoc(), vectorType, in, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getPadding(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(
           SmallVector<bool>(vectorType.getRank(), false)));
@@ -3271,8 +3280,8 @@ LogicalResult LinalgCopyVTWForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_write, the attribute must be reset
   // conservatively.
   auto vector = xferOp.getVector();
-  vector::TransferWriteOp::create(rewriter,
-      xferOp.getLoc(), vector, out, xferOp.getIndices(),
+  vector::TransferWriteOp::create(
+      rewriter, xferOp.getLoc(), vector, out, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(SmallVector<bool>(
           dyn_cast<VectorType>(vector.getType()).getRank(), false)));
@@ -3486,17 +3495,17 @@ struct Conv1DGenerator
     SmallVector<Value> resPadding(resShape.size(), zero);
 
     // Read the whole lhs, rhs and res in one shot (with zero padding).
-    Value lhs = vector::TransferReadOp::create(rewriter,
-        loc, lhsType, lhsShaped, lhsPadding,
+    Value lhs = vector::TransferReadOp::create(
+        rewriter, loc, lhsType, lhsShaped, lhsPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     // This is needed only for Conv.
     Value rhs = nullptr;
     if (oper == ConvOperationKind::Conv)
-      rhs = vector::TransferReadOp::create(rewriter,
-          loc, rhsType, rhsShaped, rhsPadding,
+      rhs = vector::TransferReadOp::create(
+          rewriter, loc, rhsType, rhsShaped, rhsPadding,
           /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
-    Value res = vector::TransferReadOp::create(rewriter,
-        loc, resType, resShaped, resPadding,
+    Value res = vector::TransferReadOp::create(
+        rewriter, loc, resType, resShaped, resPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
 
     // The base vectorization case for channeled convolution is input:
@@ -3633,8 +3642,8 @@ struct Conv1DGenerator
     bindDims(ctx, n, w, f, c);
     lhs = promote(rewriter, loc, lhs, res.getType());
     rhs = promote(rewriter, loc, rhs, res.getType());
-    auto contrationOp = vector::ContractionOp::create(rewriter,
-        loc, lhs, rhs, res,
+    auto contrationOp = vector::ContractionOp::create(
+        rewriter, loc, lhs, rhs, res,
         /*indexingMaps=*/MapList{{n, w, c}, {c, f}, {n, w, f}},
         /*iteratorTypes=*/ArrayRef<vector::IteratorType>{par, par, par, red});
     contrationOp.setKind(reductionKind);
@@ -3645,8 +3654,8 @@ struct Conv1DGenerator
   // convolution.
   Value conv1dSliceAsOuterProduct(RewriterBase &rewriter, Location loc,
                                   Value lhs, Value rhs, Value res) {
-    return vector::OuterProductOp::create(rewriter,
-        loc, res.getType(), lhs, rhs, res, vector::CombiningKind::ADD);
+    return vector::OuterProductOp::create(rewriter, loc, res.getType(), lhs,
+                                          rhs, res, vector::CombiningKind::ADD);
   }
 
   // Create a reduction: lhs{n, w, c} -> res{n, w, c}
@@ -3743,22 +3752,22 @@ struct Conv1DGenerator
 
     // Read lhs slice of size {n, w * strideW + kw * dilationW, c} @ [0, 0,
     // 0].
-    Value lhs = vector::TransferReadOp::create(rewriter,
-        loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
+    Value lhs = vector::TransferReadOp::create(
+        rewriter, loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     auto maybeMaskedLhs = maybeMaskXferOp(
         lhsType.getShape(), lhsType.getScalableDims(), lhs.getDefiningOp());
 
     // Read rhs slice of size {kw, c} @ [0, 0].
-    Value rhs = vector::TransferReadOp::create(rewriter,
-        loc, rhsType, rhsShaped, ValueRange{zero, zero},
+    Value rhs = vector::TransferReadOp::create(
+        rewriter, loc, rhsType, rhsShaped, ValueRange{zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
     auto maybeMaskedRhs = maybeMaskXferOp(
         rhsType.getShape(), rhsType.getScalableDims(), rhs.getDefiningOp());
 
     // Read res slice of size {n, w, c} @ [0, 0, 0].
-    Value res = vector::TransferReadOp::create(rewriter,
-        loc, resType, resShaped, ValueRange{zero, zero, zero},
+    Value res = vector::TransferReadOp::create(
+        rewriter, loc, resType, resShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
     auto maybeMaskedRes = maybeMaskXferOp(
         resType.getShape(), resType.getScalableDims(), res.getDefiningOp());
@@ -3775,22 +3784,22 @@ struct Conv1DGenerator
     //   @ [0, sw * w + dw * kw, 0].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        lhsVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-            loc, maybeMaskedLhs->getResult(0),
+        lhsVals.push_back(vector::ExtractStridedSliceOp::create(
+            rewriter, loc, maybeMaskedLhs->getResult(0),
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             inOutSliceSizes, inOutStrides));
       }
     }
     // Extract rhs slice of size {c} @ [kw].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
-      rhsVals.push_back(vector::ExtractOp::create(rewriter,
-          loc, maybeMaskedRhs->getResult(0),
-          /*offsets=*/ArrayRef<int64_t>{kw}));
+      rhsVals.push_back(
+          vector::ExtractOp::create(rewriter, loc, maybeMaskedRhs->getResult(0),
+                                    /*offsets=*/ArrayRef<int64_t>{kw}));
     }
     // Extract res slice: {n, wSizeStep, c} @ [0, w, 0].
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      resVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-          loc, maybeMaskedRes->getResult(0),
+      resVals.push_back(vector::ExtractStridedSliceOp::create(
+          rewriter, loc, maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0}, inOutSliceSizes,
           inOutStrides));
     }
@@ -3815,17 +3824,19 @@ struct Conv1DGenerator
         if (flatten) {
           // Flatten the input and output vectors (collapse the channel
           // dimension)
-          lhsVal = vector::ShapeCastOp::create(rewriter,
-              loc, lhsTypeAfterFlattening, lhsVals[linearIndex(kw, w)]);
-          resVal = vector::ShapeCastOp::create(rewriter,
-              loc, resTypeAfterFlattening, resVals[w]);
+          lhsVal =
+              vector::ShapeCastOp::create(rewriter, loc, lhsTypeAfterFlattening,
+                                          lhsVals[linearIndex(kw, w)]);
+          resVal = vector::ShapeCastOp::create(
+              rewriter, loc, resTypeAfterFlattening, resVals[w]);
         }
         resVals[w] = depthwiseConv1dSliceAsMulAcc(rewriter, loc, lhsVal,
                                                   rhsVals[kw], resVal, flatten);
         if (flatten) {
           // Un-flatten the output vector (restore the channel dimension)
-          resVals[w] = vector::ShapeCastOp::create(rewriter,
-              loc, VectorType::get(inOutSliceSizes, resEltType), resVals[w]);
+          resVals[w] = vector::ShapeCastOp::create(
+              rewriter, loc, VectorType::get(inOutSliceSizes, resEltType),
+              resVals[w]);
         }
       }
     }
@@ -3843,8 +3854,8 @@ struct Conv1DGenerator
     // Write back res slice: {n, wSizeStep, c} @ [0, w, 0].
     // This does not depend on kw.
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      maybeMaskedRes = vector::InsertStridedSliceOp::create(rewriter,
-          loc, resVals[w], maybeMaskedRes->getResult(0),
+      maybeMaskedRes = vector::InsertStridedSliceOp::create(
+          rewriter, loc, resVals[w], maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           /*strides=*/ArrayRef<int64_t>{1, 1, 1});
     }
@@ -3853,8 +3864,8 @@ struct Conv1DGenerator
     //===------------------------------------------------------------------===//
 
     // Write back res slice of size {n, w, c} @ [0, 0, 0].
-    Operation *resOut = vector::TransferWriteOp::create(rewriter,
-        loc, maybeMaskedRes->getResult(0), resShaped,
+    Operation *resOut = vector::TransferWriteOp::create(
+        rewriter, loc, maybeMaskedRes->getResult(0), resShaped,
         ValueRange{zero, zero, zero});
     return maybeMaskXferOp(resType.getShape(), resType.getScalableDims(),
                            resOut);
@@ -3894,8 +3905,8 @@ struct Conv1DGenerator
       rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, indices);
     }
     // Broadcast the filter to match the output vector
-    rhs = vector::BroadcastOp::create(rewriter,
-        loc, resTy.clone(rhsTy.getElementType()), rhs);
+    rhs = vector::BroadcastOp::create(rewriter, loc,
+                                      resTy.clone(rhsTy.getElementType()), rhs);
 
     rhs = promote(rewriter, loc, rhs, resTy);
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index f9c6aae9374a8..669fefcd86de1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -201,11 +201,12 @@ Value create2DTransformMatrix(OpBuilder &builder, Location loc,
                               TransformMatrix transform, Type type) {
   ArrayRef<float> constVec(transform.table, transform.rows * transform.cols);
 
-  return arith::ConstantOp::create(builder,
-      loc, DenseFPElementsAttr::get(
-               RankedTensorType::get(
-                   SmallVector<int64_t>{transform.rows, transform.cols}, type),
-               constVec));
+  return arith::ConstantOp::create(
+      builder, loc,
+      DenseFPElementsAttr::get(
+          RankedTensorType::get(
+              SmallVector<int64_t>{transform.rows, transform.cols}, type),
+          constVec));
 }
 
 /// Extract height x width data from 4D tensors.
@@ -233,8 +234,8 @@ Value extract2DDataFrom4D(OpBuilder &builder, Location loc, Value source,
 
   auto extractFilterType =
       RankedTensorType::get({extractHeight, extractWidth}, elementType);
-  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
-      loc, extractFilterType, source, offsets, sizes, strides);
+  auto extractFilterOp = tensor::ExtractSliceOp::create(
+      builder, loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
 }
@@ -267,8 +268,8 @@ Value extract2DDataFrom6D(OpBuilder &builder, Location loc, Value source,
   SmallVector<OpFoldResult> strides(srcSize, oneIndex);
 
   auto extractFilterType = RankedTensorType::get({height, width}, elementType);
-  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
-      loc, extractFilterType, source, offsets, sizes, strides);
+  auto extractFilterOp = tensor::ExtractSliceOp::create(
+      builder, loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
 }
@@ -293,8 +294,8 @@ Value insert2DDataTo4D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
-      loc, source, dest, retOffsets, retSizes, strides);
+  auto insertSliceOp = tensor::InsertSliceOp::create(
+      builder, loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
 }
@@ -321,8 +322,8 @@ Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
-      loc, source, dest, retOffsets, retSizes, strides);
+  auto insertSliceOp = tensor::InsertSliceOp::create(
+      builder, loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
 }
@@ -386,8 +387,8 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
 
     int64_t retRows = 1;
     Value matmulRetValue = extractFilter;
-    Value zero = arith::ConstantOp::create(builder,
-        loc, rewriter.getZeroAttr(elementType));
+    Value zero = arith::ConstantOp::create(builder, loc,
+                                           rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix G.
       auto it = GMatrices.find(fmr);
@@ -401,12 +402,14 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
+      auto init =
+          linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value G = create2DTransformMatrix(builder, loc, GMatrix, elementType);
       // Multiply G x g.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{G, extractFilter}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{G, extractFilter},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -423,12 +426,14 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
+      auto init =
+          linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value GT = create2DTransformMatrix(builder, loc, GTMatrix, elementType);
       // Multiply u = (G x g) x GT.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{matmulRetValue, GT}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{matmulRetValue, GT},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -516,10 +521,11 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = affine::AffineApplyOp::create(builder,
-        loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = affine::AffineApplyOp::create(builder,
-        loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
+    Value heightOffset = affine::AffineApplyOp::create(
+        builder, loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
+    Value widthOffset = affine::AffineApplyOp::create(
+        builder, loc, rightTransform ? affineMap : identityAffineMap,
+        tileWIter);
 
     // Extract (H, W) from (N, H, W, C).
     auto extractInput =
@@ -530,8 +536,8 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     int64_t retRows = 1;
     int64_t retCols = 1;
     Value matmulRetValue = extractInput;
-    Value zero = arith::ConstantOp::create(builder,
-        loc, rewriter.getZeroAttr(elementType));
+    Value zero = arith::ConstantOp::create(builder, loc,
+                                           rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix BT.
       auto it = BTMatrices.find(fmr);
@@ -545,13 +551,15 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
+      auto init =
+          linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value BT =
           create2DTransformMatrix(builder, loc, BTMatrix, builder.getF32Type());
       // Multiply BT x d.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{BT, matmulRetValue}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{BT, matmulRetValue},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -568,12 +576,14 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
+      auto init =
+          linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       Value B =
           create2DTransformMatrix(builder, loc, BMatrix, builder.getF32Type());
       // Multiply v = (BT x d) x B.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{matmulRetValue, B}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{matmulRetValue, B},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -629,8 +639,8 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       {filterShape[0] * filterShape[1], filterShape[2], filterShape[3]},
       filterElementType);
   SmallVector<ReassociationIndices> filterReassoc = {{0, 1}, {2}, {3}};
-  Value collapseFilter = tensor::CollapseShapeOp::create(rewriter,
-      loc, filterReassocType, transformedFilter, filterReassoc);
+  Value collapseFilter = tensor::CollapseShapeOp::create(
+      rewriter, loc, filterReassocType, transformedFilter, filterReassoc);
 
   // Convert (alphaH, alphaW, tileH, tileW, N, C) to
   // (alphaH x alphaW, tileH x tileW x N, C) for input.
@@ -643,8 +653,8 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
        inputShape[2] * inputShape[3] * inputShape[4], inputShape[5]},
       inputElementType);
   SmallVector<ReassociationIndices> inputReassoc = {{0, 1}, {2, 3, 4}, {5}};
-  Value collapseInput = tensor::CollapseShapeOp::create(rewriter,
-      loc, inputReassocType, transformedInput, inputReassoc);
+  Value collapseInput = tensor::CollapseShapeOp::create(
+      rewriter, loc, inputReassocType, transformedInput, inputReassoc);
 
   // Batched matrix multiply.
   auto matmulType = RankedTensorType::get(
@@ -655,12 +665,12 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
                     .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                              outputElementType)
                     .getResult();
-  Value zero = arith::ConstantOp::create(rewriter,
-      loc, rewriter.getZeroAttr(outputElementType));
+  Value zero = arith::ConstantOp::create(
+      rewriter, loc, rewriter.getZeroAttr(outputElementType));
   Value init = linalg::FillOp::create(rewriter, loc, zero, empty).getResult(0);
 
-  auto matmulOp = linalg::BatchMatmulOp::create(rewriter,
-      loc, matmulType, ValueRange({collapseInput, collapseFilter}),
+  auto matmulOp = linalg::BatchMatmulOp::create(
+      rewriter, loc, matmulType, ValueRange({collapseInput, collapseFilter}),
       ValueRange{init});
 
   // The result shape of batch matmul is (alphaH x alphaW, tileH x tileW x N, F)
@@ -670,8 +680,8 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       RankedTensorType::get({inputShape[0], inputShape[1], inputShape[2],
                              inputShape[3], inputShape[4], filterShape[3]},
                             outputElementType);
-  auto expandOutput = tensor::ExpandShapeOp::create(rewriter,
-      loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
+  auto expandOutput = tensor::ExpandShapeOp::create(
+      rewriter, loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
   return expandOutput;
 }
 
@@ -750,16 +760,17 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
     int64_t retRows = leftTransform ? ATMatrix.rows : 1;
 
     Value matmulRetValue = extractValue;
-    Value zero = arith::ConstantOp::create(builder,
-        loc, rewriter.getZeroAttr(elementType));
+    Value zero = arith::ConstantOp::create(builder, loc,
+                                           rewriter.getZeroAttr(elementType));
 
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = affine::AffineApplyOp::create(builder,
-        loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = affine::AffineApplyOp::create(builder,
-        loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
+    Value heightOffset = affine::AffineApplyOp::create(
+        builder, loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
+    Value widthOffset = affine::AffineApplyOp::create(
+        builder, loc, rightTransform ? affineMap : identityAffineMap,
+        tileWIter);
 
     Value outInitVal =
         extract2DDataFrom4D(builder, loc, args[0], NIter, FIter, heightOffset,
@@ -780,8 +791,9 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
       Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
       // Multiply AT x m.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{AT, matmulRetValue}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{AT, matmulRetValue},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -799,15 +811,16 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
       Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
       // Multiply y = (AT x m) x A.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{matmulRetValue, A}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{matmulRetValue, A},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
     if (scalarFactor != 1) {
       // Multiply by scalar factor and add outInitVal.
-      Value scalarFactorValue = arith::ConstantOp::create(builder,
-          loc, FloatAttr::get(elementType, scalarFactor));
+      Value scalarFactorValue = arith::ConstantOp::create(
+          builder, loc, FloatAttr::get(elementType, scalarFactor));
       auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
       auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
       SmallVector<AffineMap> affineMaps = {
@@ -824,12 +837,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                       utils::IteratorType::parallel},
                   [&](OpBuilder &nestedBuilder, Location nestedLoc,
                       ValueRange args) {
-                    auto mulf = arith::MulFOp::create(nestedBuilder,
-                        nestedLoc, args[0], args[1]);
-                    auto addf = arith::AddFOp::create(nestedBuilder,
-                        nestedLoc, mulf.getResult(), args[2]);
+                    auto mulf = arith::MulFOp::create(nestedBuilder, nestedLoc,
+                                                      args[0], args[1]);
+                    auto addf = arith::AddFOp::create(
+                        nestedBuilder, nestedLoc, mulf.getResult(), args[2]);
                     linalg::YieldOp::create(nestedBuilder, nestedLoc,
-                                                          addf.getResult());
+                                            addf.getResult());
                   })
               .getResult(0);
     }
@@ -867,8 +880,8 @@ static Value padToAlignedTensor(RewriterBase &rewriter, Location loc,
   auto valueType = cast<ShapedType>(value.getType());
   Type elementType = valueType.getElementType();
   auto alignedType = RankedTensorType::get(alignedShape, elementType);
-  Value padValue = arith::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getZeroAttr(elementType));
+  Value padValue = arith::ConstantOp::create(rewriter, loc, elementType,
+                                             rewriter.getZeroAttr(elementType));
 
   return linalg::makeComposedPadHighOp(rewriter, loc, alignedType, value,
                                        padValue, false);
@@ -888,7 +901,7 @@ static Value extractFromAlignedTensor(RewriterBase &rewriter, Location loc,
       getAsOpFoldResult(rewriter.getI64ArrayAttr(extractedShape));
 
   return tensor::ExtractSliceOp::create(rewriter, loc, extractedType, value,
-                                                 offsets, sizes, strides);
+                                        offsets, sizes, strides);
 }
 
 /// Utility function to check all values in the attribute are 1.
@@ -980,9 +993,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   auto retType = RankedTensorType::get({alphaH, alphaW, filterC, filterF},
                                        filterElementType);
   Value retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
-                                                    filterElementType);
-  auto transformedFilter = linalg::WinogradFilterTransformOp::create(rewriter,
-      loc, retType, filter, retValue, fmr);
+                                           filterElementType);
+  auto transformedFilter = linalg::WinogradFilterTransformOp::create(
+      rewriter, loc, retType, filter, retValue, fmr);
 
   // --- Create operation for input transform ---
 
@@ -999,9 +1012,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   retType = RankedTensorType::get(
       {alphaH, alphaW, tileH, tileW, inputN, inputC}, inputElementType);
   retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
-                                              inputElementType);
-  auto transformedInput = linalg::WinogradInputTransformOp::create(rewriter,
-      loc, retType, input, retValue, fmr);
+                                     inputElementType);
+  auto transformedInput = linalg::WinogradInputTransformOp::create(
+      rewriter, loc, retType, input, retValue, fmr);
 
   Type outputElementType = outputType.getElementType();
   Value matmulRet = matrixMultiply(rewriter, loc, transformedFilter,
@@ -1023,8 +1036,8 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
     outputType = alignedOutputType;
   }
 
-  Value transformedOutput = linalg::WinogradOutputTransformOp::create(rewriter,
-      loc, outputType, matmulRet, output, fmr);
+  Value transformedOutput = linalg::WinogradOutputTransformOp::create(
+      rewriter, loc, outputType, matmulRet, output, fmr);
 
   // When output size is not aligned with output tile size, extract the
   // value from the padded buffer.
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index d964aa5cb4e81..999a6381cd2e0 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -320,8 +320,8 @@ GenericOp makeMemRefCopyOp(OpBuilder &b, Location loc, Value from, Value to) {
       AffineMap::getMultiDimIdentityMap(memrefTypeTo.getRank(), b.getContext());
   SmallVector<utils::IteratorType> iteratorTypes(memrefTypeTo.getRank(),
                                                  utils::IteratorType::parallel);
-  return linalg::GenericOp::create(b,
-      loc,
+  return linalg::GenericOp::create(
+      b, loc,
       /*inputs=*/from,
       /*outputs=*/to,
       /*indexingMaps=*/llvm::ArrayRef({id, id}),
@@ -483,8 +483,8 @@ static void generateParallelLoopNest(
   case DistributionMethod::None: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    scf::ParallelOp::create(b,
-        loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
+    scf::ParallelOp::create(
+        b, loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
           ivStorage.append(localIvs.begin(), localIvs.end());
@@ -499,8 +499,8 @@ static void generateParallelLoopNest(
   case DistributionMethod::Cyclic: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    scf::ParallelOp::create(b,
-        loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
+    scf::ParallelOp::create(
+        b, loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
           ivStorage.append(localIvs.begin(), localIvs.end());
@@ -595,13 +595,13 @@ static Operation *materializeTiledShape(OpBuilder &builder, Location loc,
   auto shapedType = dyn_cast<ShapedType>(valueToTile.getType());
   auto *sliceOp = TypeSwitch<ShapedType, Operation *>(shapedType)
                       .Case([&](MemRefType) {
-                        return memref::SubViewOp::create(builder,
-                            loc, valueToTile, sliceParams.offsets,
+                        return memref::SubViewOp::create(
+                            builder, loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
                       .Case([&](RankedTensorType) {
-                        return tensor::ExtractSliceOp::create(builder,
-                            loc, valueToTile, sliceParams.offsets,
+                        return tensor::ExtractSliceOp::create(
+                            builder, loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
                       .Default([](ShapedType) -> Operation * {
@@ -793,8 +793,8 @@ SmallVector<Value> insertSlicesBack(OpBuilder &builder, Location loc,
     // `tiledOperands`.
     Value outputTensor = operands[opOperand.getOperandNumber()];
     if (auto sliceOp = outputTensor.getDefiningOp<tensor::ExtractSliceOp>()) {
-      Value inserted = tensor::InsertSliceOp::create(builder,
-          loc, sliceOp.getSource().getType(), results[resultIdx],
+      Value inserted = tensor::InsertSliceOp::create(
+          builder, loc, sliceOp.getSource().getType(), results[resultIdx],
           sliceOp.getSource(), sliceOp.getOffsets(), sliceOp.getSizes(),
           sliceOp.getStrides(), sliceOp.getStaticOffsets(),
           sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
diff --git a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
index 9dadd9bea2f67..364e4d385fd62 100644
--- a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -135,8 +135,9 @@ struct GlobalStoreOpInterface
     auto memrefType = getMemRefTypeWithStaticIdentityLayout(tensorType);
 
     auto loc = globalStoreOp.getLoc();
-    auto targetMemref = memref::GetGlobalOp::create(rewriter,
-        loc, memrefType, globalStoreOp.getGlobalAttr().getLeafReference());
+    auto targetMemref = memref::GetGlobalOp::create(
+        rewriter, loc, memrefType,
+        globalStoreOp.getGlobalAttr().getLeafReference());
 
     auto sourceMemref =
         getBuffer(rewriter, globalStoreOp.getValue(), options, state);
diff --git a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
index 770f32841c0f6..31785eb20a642 100644
--- a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
@@ -91,8 +91,9 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
 
   // Replace `pow(x, -1.0)` with `1.0 / x`.
   if (isExponentValue(-1.0)) {
-    Value one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getFloatAttr(getElementTypeOrSelf(op.getType()), 1.0));
+    Value one = arith::ConstantOp::create(
+        rewriter, loc,
+        rewriter.getFloatAttr(getElementTypeOrSelf(op.getType()), 1.0));
     rewriter.replaceOpWithNewOp<arith::DivFOp>(op, ValueRange({bcast(one), x}));
     return success();
   }
@@ -175,11 +176,11 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   Value one;
   Type opType = getElementTypeOrSelf(op.getType());
   if constexpr (std::is_same_v<PowIOpTy, math::FPowIOp>)
-    one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getFloatAttr(opType, 1.0));
+    one = arith::ConstantOp::create(rewriter, loc,
+                                    rewriter.getFloatAttr(opType, 1.0));
   else
-    one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerAttr(opType, 1));
+    one = arith::ConstantOp::create(rewriter, loc,
+                                    rewriter.getIntegerAttr(opType, 1));
 
   // Replace `[fi]powi(x, 0)` with `1`.
   if (exponentValue == 0) {
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index b5c0ddd617805..5edb6e28fb018 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -33,7 +33,7 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
   auto attr = b.getFloatAttr(eltType, value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
     return arith::ConstantOp::create(b, loc,
-                                       DenseElementsAttr::get(shapedTy, attr));
+                                     DenseElementsAttr::get(shapedTy, attr));
   }
 
   return arith::ConstantOp::create(b, loc, attr);
@@ -50,7 +50,7 @@ static Value createIntConst(Location loc, Type type, int64_t value,
   auto attr = b.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
     return arith::ConstantOp::create(b, loc,
-                                       DenseElementsAttr::get(shapedTy, attr));
+                                     DenseElementsAttr::get(shapedTy, attr));
   }
 
   return arith::ConstantOp::create(b, loc, attr);
@@ -116,8 +116,8 @@ static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) {
   Value negTwo = createFloatConst(loc, floatType, -2.0, rewriter);
 
   // Compute sign(x) = cast<float_type>(x < 0) * (-2) + 1
-  Value isNegative = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OLT, op.getOperand(), zero);
+  Value isNegative = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OLT, op.getOperand(), zero);
   Value isNegativeFloat =
       arith::UIToFPOp::create(rewriter, loc, floatType, isNegative);
   Value isNegativeTimesNegTwo =
@@ -236,8 +236,9 @@ static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) {
   Value one = createFloatConst(op->getLoc(), opType, 1.00, rewriter);
 
   Value gtCheck = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT, operand,
-                                          fpFixedConvert);
-  Value incrValue = arith::SelectOp::create(b, op->getLoc(), gtCheck, one, zero);
+                                        fpFixedConvert);
+  Value incrValue =
+      arith::SelectOp::create(b, op->getLoc(), gtCheck, one, zero);
 
   Value ret = arith::AddFOp::create(b, opType, fpFixedConvert, incrValue);
   rewriter.replaceOp(op, ret);
@@ -259,7 +260,7 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
     Value castPowerToFp =
         arith::SIToFPOp::create(rewriter, op.getLoc(), baseType, power);
     Value res = math::PowFOp::create(rewriter, op.getLoc(), baseType, base,
-                                              castPowerToFp);
+                                     castPowerToFp);
     rewriter.replaceOp(op, res);
     return success();
   };
@@ -309,7 +310,7 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
     res =
         arith::SelectOp::create(b, op->getLoc(), zeroEqCheck, posInfinity, res);
     res = arith::SelectOp::create(b, op->getLoc(), negZeroEqCheck, negInfinity,
-                                    res);
+                                  res);
   }
 
   rewriter.replaceOp(op, res);
@@ -451,14 +452,14 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   //
   // All three cases satisfy the property `biasedExp >= 23`.
   Value operandBitcast = arith::BitcastOp::create(b, i32Ty, operand);
-  Value operandExp = arith::AndIOp::create(b,
-      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandExp = arith::AndIOp::create(
+      b, arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
   Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
-  Value isSpecialValOrLargeVal =
-      arith::CmpIOp::create(b, arith::CmpIPredicate::sge, operandBiasedExp, c23);
+  Value isSpecialValOrLargeVal = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::sge, operandBiasedExp, c23);
 
   Value result = arith::SelectOp::create(b, isSpecialValOrLargeVal, operand,
-                                           fpFixedConvert);
+                                         fpFixedConvert);
   rewriter.replaceOp(op, result);
   return success();
 }
@@ -488,8 +489,8 @@ static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
     auto bits = createIntConst(loc, operandTy, half, rewriter);
     auto mask = createIntConst(loc, operandTy, allbits >> half, rewriter);
 
-    Value pred =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ule, x, mask);
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ule,
+                                       x, mask);
     Value add = arith::AddIOp::create(rewriter, loc, count, bits);
     Value shift = arith::ShLIOp::create(rewriter, loc, x, bits);
 
@@ -499,7 +500,7 @@ static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
 
   Value zero = createIntConst(loc, operandTy, 0, rewriter);
   Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                              operand, zero);
+                                     operand, zero);
 
   Value bwval = createIntConst(loc, operandTy, bitwidth, rewriter);
   Value sel = arith::SelectOp::create(rewriter, loc, pred, bwval, count);
@@ -554,11 +555,11 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   Value roundBitcast = arith::BitcastOp::create(b, iTy, round);
 
   // Get biased exponents for operand and round(operand)
-  Value operandExp = arith::AndIOp::create(b,
-      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandExp = arith::AndIOp::create(
+      b, arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
   Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
-  Value roundExp = arith::AndIOp::create(b,
-      arith::ShRUIOp::create(b, roundBitcast, c23), expMask);
+  Value roundExp = arith::AndIOp::create(
+      b, arith::ShRUIOp::create(b, roundBitcast, c23), expMask);
   Value roundBiasedExp = arith::SubIOp::create(b, roundExp, c127);
 
   auto safeShiftRight = [&](Value x, Value shift) -> Value {
@@ -592,8 +593,8 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
       arith::CmpIOp::create(b, arith::CmpIPredicate::eq, roundBiasedExp, c0);
   Value roundBiasedExpMinus1 = arith::SubIOp::create(b, roundBiasedExp, c1);
   Value roundMaskedMantissa = maskMantissa(roundBitcast, roundBiasedExpMinus1);
-  Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(b,
-      arith::CmpIPredicate::ne, roundMaskedMantissa, c0);
+  Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::ne, roundMaskedMantissa, c0);
   roundIsNotEvenOrSpecialVal =
       arith::OrIOp::create(b, roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
 
@@ -604,19 +605,19 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // values +-0.5 are the only halfway values that have `biasedExp == -1 < 0`,
   // so these are handled separately. In particular, if `biasedExp == -1`, the
   // value is halfway if the entire mantissa is zero.
-  Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(b,
-      arith::CmpIPredicate::eq, operandBiasedExp, cNeg1);
-  Value expectedOperandMaskedMantissa = arith::SelectOp::create(b,
-      operandBiasedExpEqNeg1, c0, safeShiftRight(c2To22, operandBiasedExp));
+  Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::eq, operandBiasedExp, cNeg1);
+  Value expectedOperandMaskedMantissa = arith::SelectOp::create(
+      b, operandBiasedExpEqNeg1, c0, safeShiftRight(c2To22, operandBiasedExp));
   Value operandMaskedMantissa = maskMantissa(operandBitcast, operandBiasedExp);
   Value operandIsHalfway =
       arith::CmpIOp::create(b, arith::CmpIPredicate::eq, operandMaskedMantissa,
-                              expectedOperandMaskedMantissa);
+                            expectedOperandMaskedMantissa);
   // Ensure `biasedExp` is in the valid range for half values.
-  Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(b,
-      arith::CmpIPredicate::sge, operandBiasedExp, cNeg1);
-  Value operandBiasedExpLt23 =
-      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, operandBiasedExp, c23);
+  Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::sge, operandBiasedExp, cNeg1);
+  Value operandBiasedExpLt23 = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::slt, operandBiasedExp, c23);
   operandIsHalfway =
       arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpLt23);
   operandIsHalfway =
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index baa35fb5a2c5d..76720cfd4a98c 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -156,16 +156,16 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
   // Stitch results together into one large vector.
   Type resultEltType = cast<VectorType>(results[0].getType()).getElementType();
   Type resultExpandedType = VectorType::get(expandedShape, resultEltType);
-  Value result = arith::ConstantOp::create(builder,
-      resultExpandedType, builder.getZeroAttr(resultExpandedType));
+  Value result = arith::ConstantOp::create(
+      builder, resultExpandedType, builder.getZeroAttr(resultExpandedType));
 
   for (int64_t i = 0; i < maxIndex; ++i)
     result = vector::InsertOp::create(builder, results[i], result,
-                                              delinearize(i, strides));
+                                      delinearize(i, strides));
 
   // Reshape back to the original vector shape.
-  return vector::ShapeCastOp::create(builder,
-      VectorType::get(inputShape, resultEltType), result);
+  return vector::ShapeCastOp::create(
+      builder, VectorType::get(inputShape, resultEltType), result);
 }
 
 //----------------------------------------------------------------------------//
@@ -181,7 +181,7 @@ static Value floatCst(ImplicitLocOpBuilder &builder, float value,
   assert((elementType.isF16() || elementType.isF32()) &&
          "x must be f16 or f32 type.");
   return arith::ConstantOp::create(builder,
-      builder.getFloatAttr(elementType, value));
+                                   builder.getFloatAttr(elementType, value));
 }
 
 static Value f32Cst(ImplicitLocOpBuilder &builder, double value) {
@@ -203,14 +203,16 @@ static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
 
 // Return the minimum of the two values or NaN if value is NaN
 static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return arith::SelectOp::create(builder,
+  return arith::SelectOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT, value, bound),
       value, bound);
 }
 
 // Return the maximum of the two values or NaN if value is NaN
 static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return arith::SelectOp::create(builder,
+  return arith::SelectOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::UGT, value, bound),
       value, bound);
 }
@@ -252,8 +254,8 @@ static std::pair<Value, Value> frexp(ImplicitLocOpBuilder &builder, Value arg,
 
   // Compute exponent.
   Value arg0 = isPositive ? arg : math::AbsFOp::create(builder, arg);
-  Value biasedExponentBits = arith::ShRUIOp::create(builder,
-      arith::BitcastOp::create(builder, i32Vec, arg0),
+  Value biasedExponentBits = arith::ShRUIOp::create(
+      builder, arith::BitcastOp::create(builder, i32Vec, arg0),
       bcast(i32Cst(builder, 23)));
   Value biasedExponent =
       arith::SIToFPOp::create(builder, f32Vec, biasedExponentBits);
@@ -301,7 +303,7 @@ Value makePolynomialCalculation(ImplicitLocOpBuilder &builder,
     return coeffs[0];
 
   Value res = math::FmaOp::create(builder, x, coeffs[coeffs.size() - 1],
-                                          coeffs[coeffs.size() - 2]);
+                                  coeffs[coeffs.size() - 2]);
   for (auto i = ptrdiff_t(coeffs.size()) - 3; i >= 0; --i) {
     res = math::FmaOp::create(builder, x, res, coeffs[i]);
   }
@@ -511,18 +513,20 @@ Atan2Approximation::matchAndRewrite(math::Atan2Op op,
   // Handle x = 0, y > 0
   Value xZero =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, x, zero);
-  Value yGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, y, zero);
+  Value yGt =
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, y, zero);
   Value isHalfPi = arith::AndIOp::create(builder, xZero, yGt);
   auto halfPi = broadcast(builder, f32Cst(builder, 1.57079632679f), shape);
   result = arith::SelectOp::create(builder, isHalfPi, halfPi, result);
 
   // Handle x = 0, y < 0
-  Value yLt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, y, zero);
+  Value yLt =
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, y, zero);
   Value isNegativeHalfPiPi = arith::AndIOp::create(builder, xZero, yLt);
   auto negativeHalfPiPi =
       broadcast(builder, f32Cst(builder, -1.57079632679f), shape);
-  result = arith::SelectOp::create(builder, isNegativeHalfPiPi, negativeHalfPiPi,
-                                           result);
+  result = arith::SelectOp::create(builder, isNegativeHalfPiPi,
+                                   negativeHalfPiPi, result);
 
   // Handle x = 0, y = 0;
   Value yZero =
@@ -569,9 +573,9 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
 
   // Mask for tiny values that are approximated with `operand`.
   Value tiny = bcast(f32Cst(builder, 0.0004f));
-  Value tinyMask = arith::CmpFOp::create(builder,
-      arith::CmpFPredicate::OLT, math::AbsFOp::create(builder, op.getOperand()),
-      tiny);
+  Value tinyMask = arith::CmpFOp::create(
+      builder, arith::CmpFPredicate::OLT,
+      math::AbsFOp::create(builder, op.getOperand()), tiny);
 
   // The monomial coefficients of the numerator polynomial (odd).
   Value alpha1 = bcast(f32Cst(builder, 4.89352455891786e-03f));
@@ -606,8 +610,8 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
   q = math::FmaOp::create(builder, x2, q, beta0);
 
   // Divide the numerator by the denominator.
-  Value res = arith::SelectOp::create(builder,
-      tinyMask, x, arith::DivFOp::create(builder, p, q));
+  Value res = arith::SelectOp::create(builder, tinyMask, x,
+                                      arith::DivFOp::create(builder, p, q));
 
   rewriter.replaceOp(op, res);
 
@@ -691,12 +695,12 @@ LogApproximationBase<Op>::logMatchAndRewrite(Op op, PatternRewriter &rewriter,
   //     x = x + x - 1.0;
   //   } else { x = x - 1.0; }
   Value mask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x,
-                                             cstCephesSQRTHF);
+                                     cstCephesSQRTHF);
   Value tmp = arith::SelectOp::create(builder, mask, x, cstZero);
 
   x = arith::SubFOp::create(builder, x, cstOne);
-  e = arith::SubFOp::create(builder,
-      e, arith::SelectOp::create(builder, mask, cstOne, cstZero));
+  e = arith::SubFOp::create(
+      builder, e, arith::SelectOp::create(builder, mask, cstOne, cstZero));
   x = arith::AddFOp::create(builder, x, tmp);
 
   Value x2 = arith::MulFOp::create(builder, x, x);
@@ -726,20 +730,20 @@ LogApproximationBase<Op>::logMatchAndRewrite(Op op, PatternRewriter &rewriter,
   }
 
   Value invalidMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT,
-                                                    op.getOperand(), cstZero);
+                                            op.getOperand(), cstZero);
   Value zeroMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
-                                                 op.getOperand(), cstZero);
+                                         op.getOperand(), cstZero);
   Value posInfMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
-                                                   op.getOperand(), cstPosInf);
+                                           op.getOperand(), cstPosInf);
 
   // Filter out invalid values:
   //  • x == 0     -> -INF
   //  • x < 0      ->  NAN
   //  • x == +INF  -> +INF
-  Value aproximation = arith::SelectOp::create(builder,
-      zeroMask, cstMinusInf,
-      arith::SelectOp::create(builder,
-          invalidMask, cstNan,
+  Value aproximation = arith::SelectOp::create(
+      builder, zeroMask, cstMinusInf,
+      arith::SelectOp::create(
+          builder, invalidMask, cstNan,
           arith::SelectOp::create(builder, posInfMask, cstPosInf, x)));
 
   rewriter.replaceOp(op, aproximation);
@@ -811,11 +815,12 @@ Log1pApproximation::matchAndRewrite(math::Log1pOp op,
   Value logU = math::LogOp::create(builder, u);
   Value uInf =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, logU);
-  Value logLarge = arith::MulFOp::create(builder,
-      x, arith::DivFOp::create(builder,
-             logU, arith::SubFOp::create(builder, u, cstOne)));
-  Value approximation = arith::SelectOp::create(builder,
-      arith::OrIOp::create(builder, uSmall, uInf), x, logLarge);
+  Value logLarge = arith::MulFOp::create(
+      builder, x,
+      arith::DivFOp::create(builder, logU,
+                            arith::SubFOp::create(builder, u, cstOne)));
+  Value approximation = arith::SelectOp::create(
+      builder, arith::OrIOp::create(builder, uSmall, uInf), x, logLarge);
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -866,7 +871,9 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
 
   auto abs = [&](Value a) -> Value { return math::AbsFOp::create(builder, a); };
 
-  auto sqrt = [&](Value a) -> Value { return math::SqrtOp::create(builder, a); };
+  auto sqrt = [&](Value a) -> Value {
+    return math::SqrtOp::create(builder, a);
+  };
 
   auto scopy = [&](Value a, Value b) -> Value {
     return math::CopySignOp::create(builder, a, b);
@@ -880,9 +887,8 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   Value aa = mul(operand, operand);
   Value opp = sqrt(sub(bcast(floatCst(builder, 1.0, elementType)), aa));
 
-  Value gt =
-      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, aa,
-                                    bcast(floatCst(builder, 0.5, elementType)));
+  Value gt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, aa,
+                                   bcast(floatCst(builder, 0.5, elementType)));
 
   Value x = sel(gt, opp, abso);
 
@@ -978,8 +984,8 @@ AcosPolynomialApproximation::matchAndRewrite(math::AcosOp op,
   r = arith::SelectOp::create(builder, firstPred, trueVal, falseVal);
 
   // Check whether the operand lies in between [-1.0, 0.0).
-  Value greaterThanNegOne =
-      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGE, operand, negOne);
+  Value greaterThanNegOne = arith::CmpFOp::create(
+      builder, arith::CmpFPredicate::OGE, operand, negOne);
 
   Value lessThanZero =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
@@ -1094,22 +1100,22 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
         arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, bounds[j]);
     for (int i = 0; i <= polyDegree; ++i) {
       p[i] = arith::SelectOp::create(builder, isLessThanBound[j], p[i],
-                                             pp[j + 1][i]);
+                                     pp[j + 1][i]);
       q[i] = arith::SelectOp::create(builder, isLessThanBound[j], q[i],
-                                             qq[j + 1][i]);
+                                     qq[j + 1][i]);
     }
     offset = arith::SelectOp::create(builder, isLessThanBound[j], offset,
-                                             offsets[j + 1]);
+                                     offsets[j + 1]);
   }
-  isLessThanBound[intervalsCount - 1] = arith::CmpFOp::create(builder,
-      arith::CmpFPredicate::ULT, x, bounds[intervalsCount - 1]);
+  isLessThanBound[intervalsCount - 1] = arith::CmpFOp::create(
+      builder, arith::CmpFPredicate::ULT, x, bounds[intervalsCount - 1]);
 
   Value pPoly = makePolynomialCalculation(builder, p, x);
   Value qPoly = makePolynomialCalculation(builder, q, x);
   Value rationalPoly = arith::DivFOp::create(builder, pPoly, qPoly);
   Value formula = arith::AddFOp::create(builder, offset, rationalPoly);
-  formula = arith::SelectOp::create(builder, isLessThanBound[intervalsCount - 1],
-                                            formula, one);
+  formula = arith::SelectOp::create(
+      builder, isLessThanBound[intervalsCount - 1], formula, one);
 
   // erf is odd function: erf(x) = -erf(-x).
   Value negFormula = arith::NegFOp::create(builder, formula);
@@ -1160,8 +1166,9 @@ ErfcPolynomialApproximation::matchAndRewrite(math::ErfcOp op,
   Value r = arith::DivFOp::create(builder, one, p);
   Value q = math::FmaOp::create(builder, neg4, r, one);
   Value t = math::FmaOp::create(builder, arith::AddFOp::create(builder, q, one),
-                                        neg2, a);
-  Value e = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), q, t);
+                                neg2, a);
+  Value e =
+      math::FmaOp::create(builder, arith::NegFOp::create(builder, a), q, t);
   q = math::FmaOp::create(builder, r, e, q);
 
   p = bcast(floatCst(builder, -0x1.a4a000p-12f, et));        // -4.01139259e-4
@@ -1197,23 +1204,24 @@ ErfcPolynomialApproximation::matchAndRewrite(math::ErfcOp op,
   e = math::ExpOp::create(builder, arith::NegFOp::create(builder, s));
 
   t = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), a, s);
-  r = math::FmaOp::create(builder,
-      r, e,
+  r = math::FmaOp::create(
+      builder, r, e,
       arith::MulFOp::create(builder, arith::MulFOp::create(builder, r, e), t));
 
-  Value isNotLessThanInf = arith::XOrIOp::create(builder,
+  Value isNotLessThanInf = arith::XOrIOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, a, posInf),
       trueValue);
   r = arith::SelectOp::create(builder, isNotLessThanInf,
-                                      arith::AddFOp::create(builder, x, x), r);
+                              arith::AddFOp::create(builder, x, x), r);
   Value isGreaterThanClamp =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, a, clampVal);
   r = arith::SelectOp::create(builder, isGreaterThanClamp, zero, r);
 
   Value isNegative =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, zero);
-  r = arith::SelectOp::create(builder,
-      isNegative, arith::SubFOp::create(builder, pos2, r), r);
+  r = arith::SelectOp::create(builder, isNegative,
+                              arith::SubFOp::create(builder, pos2, r), r);
 
   rewriter.replaceOp(op, r);
   return success();
@@ -1235,8 +1243,9 @@ Value clampWithNormals(ImplicitLocOpBuilder &builder,
   };
 
   auto selectCmp = [&builder](auto pred, Value value, Value bound) {
-    return arith::SelectOp::create(builder,
-        arith::CmpFOp::create(builder, pred, value, bound), value, bound);
+    return arith::SelectOp::create(
+        builder, arith::CmpFOp::create(builder, pred, value, bound), value,
+        bound);
   };
 
   // Note: prefer UGE/ULE vs. UGT/ULT, since they generate vmaxps/vminps vs.
@@ -1434,8 +1443,8 @@ ExpM1Approximation::matchAndRewrite(math::ExpM1Op op,
   Value uEqOneOrNaN =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::UEQ, u, cstOne);
   Value uMinusOne = arith::SubFOp::create(builder, u, cstOne);
-  Value uMinusOneEqNegOne = arith::CmpFOp::create(builder,
-      arith::CmpFPredicate::OEQ, uMinusOne, cstNegOne);
+  Value uMinusOneEqNegOne = arith::CmpFOp::create(
+      builder, arith::CmpFPredicate::OEQ, uMinusOne, cstNegOne);
   // logU = log(u) ~= x
   Value logU = math::LogOp::create(builder, u);
 
@@ -1444,11 +1453,11 @@ ExpM1Approximation::matchAndRewrite(math::ExpM1Op op,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, logU, u);
 
   // (u - 1) * (x / ~x)
-  Value expm1 = arith::MulFOp::create(builder,
-      uMinusOne, arith::DivFOp::create(builder, x, logU));
+  Value expm1 = arith::MulFOp::create(builder, uMinusOne,
+                                      arith::DivFOp::create(builder, x, logU));
   expm1 = arith::SelectOp::create(builder, isInf, u, expm1);
-  Value approximation = arith::SelectOp::create(builder,
-      uEqOneOrNaN, x,
+  Value approximation = arith::SelectOp::create(
+      builder, uEqOneOrNaN, x,
       arith::SelectOp::create(builder, uMinusOneEqNegOne, cstNegOne, expm1));
   rewriter.replaceOp(op, approximation);
   return success();
@@ -1723,10 +1732,10 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
 
   // Select only the inverse sqrt of positive normals (denormals are
   // flushed to zero).
-  Value ltMinMask = arith::CmpFOp::create(builder,
-      arith::CmpFPredicate::OLT, op.getOperand(), cstMinNormPos);
+  Value ltMinMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT,
+                                          op.getOperand(), cstMinNormPos);
   Value infMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
-                                                op.getOperand(), cstPosInf);
+                                        op.getOperand(), cstPosInf);
   Value notNormalFiniteMask = arith::OrIOp::create(builder, ltMinMask, infMask);
 
   // Compute an approximate result.
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index 46298fb189e41..dfa2e4e0376ed 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -89,7 +89,7 @@ Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
       })
       .Default([&](Type t) {
         return arith::ConstantOp::create(builder, getLoc(), t,
-                                                 builder.getZeroAttr(t));
+                                         builder.getZeroAttr(t));
       });
 }
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
index a2b689ed13cbf..75cc39e61656a 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
@@ -35,8 +35,9 @@ struct DefaultAllocationInterface
   static ::std::optional<::mlir::Operation *>
   buildPromotedAlloc(OpBuilder &builder, Value alloc) {
     Operation *definingOp = alloc.getDefiningOp();
-    return memref::AllocaOp::create(builder,
-        definingOp->getLoc(), cast<MemRefType>(definingOp->getResultTypes()[0]),
+    return memref::AllocaOp::create(
+        builder, definingOp->getLoc(),
+        cast<MemRefType>(definingOp->getResultTypes()[0]),
         definingOp->getOperands(), definingOp->getAttrs());
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
index 9d9923bfd0541..106c3b458dbac 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
@@ -124,8 +124,8 @@ struct ComposeSubViewOpPattern : public OpRewritePattern<memref::SubViewOp> {
         }
 
         AffineMap map = AffineMap::get(0, affineApplyOperands.size(), expr);
-        Value result = affine::AffineApplyOp::create(rewriter,
-            op.getLoc(), map, affineApplyOperands);
+        Value result = affine::AffineApplyOp::create(rewriter, op.getLoc(), map,
+                                                     affineApplyOperands);
         offsets.push_back(result);
       }
     }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
index c35036d7631c8..b8923bb50583e 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
@@ -112,8 +112,8 @@ static Value getSubByteWriteMask(Location loc, OpFoldResult linearizedIndices,
   auto dstIntegerType = builder.getIntegerType(dstBits);
   auto maskRightAlignedAttr =
       builder.getIntegerAttr(dstIntegerType, (1 << srcBits) - 1);
-  Value maskRightAligned = arith::ConstantOp::create(builder,
-      loc, dstIntegerType, maskRightAlignedAttr);
+  Value maskRightAligned = arith::ConstantOp::create(
+      builder, loc, dstIntegerType, maskRightAlignedAttr);
   Value writeMaskInverse =
       arith::ShLIOp::create(builder, loc, maskRightAligned, bitwidthOffset);
   auto flipValAttr = builder.getIntegerAttr(dstIntegerType, -1);
@@ -299,15 +299,15 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
     Value bitsLoad;
     if (convertedType.getRank() == 0) {
       bitsLoad = memref::LoadOp::create(rewriter, loc, adaptor.getMemref(),
-                                                 ValueRange{});
+                                        ValueRange{});
     } else {
       // Linearize the indices of the original load instruction. Do not account
       // for the scaling yet. This will be accounted for later.
       OpFoldResult linearizedIndices = getLinearizedSrcIndices(
           rewriter, loc, srcBits, adaptor.getIndices(), op.getMemRef());
 
-      Value newLoad = memref::LoadOp::create(rewriter,
-          loc, adaptor.getMemref(),
+      Value newLoad = memref::LoadOp::create(
+          rewriter, loc, adaptor.getMemref(),
           getIndicesForLoadOrStore(rewriter, loc, linearizedIndices, srcBits,
                                    dstBits));
 
@@ -331,8 +331,8 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
             : IntegerType::get(rewriter.getContext(),
                                resultTy.getIntOrFloatBitWidth());
     if (conversionTy == convertedElementType) {
-      auto mask = arith::ConstantOp::create(rewriter,
-          loc, convertedElementType,
+      auto mask = arith::ConstantOp::create(
+          rewriter, loc, convertedElementType,
           rewriter.getIntegerAttr(convertedElementType, (1 << srcBits) - 1));
 
       result = arith::AndIOp::create(rewriter, loc, bitsLoad, mask);
@@ -433,16 +433,14 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
           IntegerType::get(rewriter.getContext(),
                            input.getType().getIntOrFloatBitWidth()),
           input);
-    }
-    Value extendedInput =
-        arith::ExtUIOp::create(rewriter, loc, dstIntegerType,
-                                                          input);
+    }    Value extendedInput = arith::ExtUIOp::create(rewriter, loc, dstIntegerType,
+                                                 input);
 
     // Special case 0-rank memref stores. No need for masking.
     if (convertedType.getRank() == 0) {
       memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::assign,
-                                           extendedInput, adaptor.getMemref(),
-                                           ValueRange{});
+                                  extendedInput, adaptor.getMemref(),
+                                  ValueRange{});
       rewriter.eraseOp(op);
       return success();
     }
@@ -461,12 +459,10 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
 
     // Clear destination bits
     memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::andi,
-                                         writeMask, adaptor.getMemref(),
-                                         storeIndices);
+                                writeMask, adaptor.getMemref(), storeIndices);
     // Write srcs bits to destination
     memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::ori,
-                                         alignedVal, adaptor.getMemref(),
-                                         storeIndices);
+                                alignedVal, adaptor.getMemref(), storeIndices);
     rewriter.eraseOp(op);
     return success();
   }
@@ -526,8 +522,8 @@ struct ConvertMemRefSubview final : OpConversionPattern<memref::SubViewOp> {
     }
 
     // Transform the offsets, sizes and strides according to the emulation.
-    auto stridedMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
-        loc, subViewOp.getViewSource());
+    auto stridedMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, subViewOp.getViewSource());
 
     OpFoldResult linearizedIndices;
     auto strides = stridedMetadata.getConstifiedMixedStrides();
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
index 34f9afae9d2ad..17a148cc31dc0 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
@@ -51,8 +51,8 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
         Value index = arith::ConstantIndexOp::create(rewriter, loc, i);
         size = memref::LoadOp::create(rewriter, loc, op.getShape(), index);
         if (!isa<IndexType>(size.getType()))
-          size = arith::IndexCastOp::create(rewriter,
-              loc, rewriter.getIndexType(), size);
+          size = arith::IndexCastOp::create(rewriter, loc,
+                                            rewriter.getIndexType(), size);
         sizes[i] = size;
       } else {
         auto sizeAttr = rewriter.getIndexAttr(op.getType().getDimSize(i));
@@ -68,8 +68,9 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
         if (stride) {
           stride = arith::MulIOp::create(rewriter, loc, stride, size);
         } else if (op.getType().isDynamicDim(i)) {
-          stride = arith::MulIOp::create(rewriter,
-              loc, arith::ConstantIndexOp::create(rewriter, loc, staticStride),
+          stride = arith::MulIOp::create(
+              rewriter, loc,
+              arith::ConstantIndexOp::create(rewriter, loc, staticStride),
               size);
         } else {
           staticStride *= op.getType().getDimSize(i);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
index b54fa61714323..01d32621b2055 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
@@ -89,9 +89,9 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     Value lhs = getValueOrCreateConstantIndexOp(rewriter, loc, currSize);
     Value rhs = getValueOrCreateConstantIndexOp(rewriter, loc, targetSize);
     Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
-                                                lhs, rhs);
-    auto ifOp = scf::IfOp::create(rewriter,
-        loc, cond,
+                                       lhs, rhs);
+    auto ifOp = scf::IfOp::create(
+        rewriter, loc, cond,
         [&](OpBuilder &builder, Location loc) {
           // Allocate the new buffer. If it is a dynamic memref we need to pass
           // an additional operand for the size at runtime, otherwise the static
@@ -100,15 +100,16 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           if (op.getDynamicResultSize())
             dynamicSizeOperands.push_back(op.getDynamicResultSize());
 
-          Value newAlloc = memref::AllocOp::create(builder,
-              loc, op.getResult().getType(), dynamicSizeOperands,
+          Value newAlloc = memref::AllocOp::create(
+              builder, loc, op.getResult().getType(), dynamicSizeOperands,
               op.getAlignmentAttr());
 
           // Take a subview of the new (bigger) buffer such that we can copy the
           // old values over (the copy operation requires both operands to have
           // the same shape).
-          Value subview = memref::SubViewOp::create(builder,
-              loc, newAlloc, ArrayRef<OpFoldResult>{rewriter.getIndexAttr(0)},
+          Value subview = memref::SubViewOp::create(
+              builder, loc, newAlloc,
+              ArrayRef<OpFoldResult>{rewriter.getIndexAttr(0)},
               ArrayRef<OpFoldResult>{currSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
           memref::CopyOp::create(builder, loc, op.getSource(), subview);
@@ -126,9 +127,10 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           // dynamic or vice-versa. If both are static and the original buffer
           // is already bigger than the requested size, the cast represents a
           // subview operation.
-          Value casted = memref::ReinterpretCastOp::create(builder,
-              loc, cast<MemRefType>(op.getResult().getType()), op.getSource(),
-              rewriter.getIndexAttr(0), ArrayRef<OpFoldResult>{targetSize},
+          Value casted = memref::ReinterpretCastOp::create(
+              builder, loc, cast<MemRefType>(op.getResult().getType()),
+              op.getSource(), rewriter.getIndexAttr(0),
+              ArrayRef<OpFoldResult>{targetSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
           scf::YieldOp::create(builder, loc, casted);
         });
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index f581b6490fadb..9771bd2aaa143 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -828,8 +828,8 @@ struct ExtractStridedMetadataOpAllocFolder
       if (allocLikeOp.getType() == baseBufferType)
         results.push_back(allocLikeOp);
       else
-        results.push_back(memref::ReinterpretCastOp::create(rewriter,
-            loc, baseBufferType, allocLikeOp, offset,
+        results.push_back(memref::ReinterpretCastOp::create(
+            rewriter, loc, baseBufferType, allocLikeOp, offset,
             /*sizes=*/ArrayRef<int64_t>(),
             /*strides=*/ArrayRef<int64_t>()));
     }
@@ -900,8 +900,8 @@ struct ExtractStridedMetadataOpGetGlobalFolder
     if (getGlobalOp.getType() == baseBufferType)
       results.push_back(getGlobalOp);
     else
-      results.push_back(memref::ReinterpretCastOp::create(rewriter,
-          loc, baseBufferType, getGlobalOp, offset,
+      results.push_back(memref::ReinterpretCastOp::create(
+          rewriter, loc, baseBufferType, getGlobalOp, offset,
           /*sizes=*/ArrayRef<int64_t>(),
           /*strides=*/ArrayRef<int64_t>()));
 
@@ -1008,9 +1008,8 @@ class ExtractStridedMetadataOpReinterpretCastFolder
     SmallVector<OpFoldResult> results;
     results.resize_for_overwrite(rank * 2 + 2);
 
-    auto newExtractStridedMetadata =
-        memref::ExtractStridedMetadataOp::create(rewriter,
-            loc, reinterpretCastOp.getSource());
+    auto newExtractStridedMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, reinterpretCastOp.getSource());
 
     // Register the base_buffer.
     results[0] = newExtractStridedMetadata.getBaseBuffer();
@@ -1082,9 +1081,8 @@ class ExtractStridedMetadataOpCastFolder
     SmallVector<OpFoldResult> results;
     results.resize_for_overwrite(rank * 2 + 2);
 
-    auto newExtractStridedMetadata =
-        memref::ExtractStridedMetadataOp::create(rewriter, loc,
-                                                          castOp.getSource());
+    auto newExtractStridedMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, castOp.getSource());
 
     // Register the base_buffer.
     results[0] = newExtractStridedMetadata.getBaseBuffer();
@@ -1142,9 +1140,8 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
     auto memSpaceCastOp = source.getDefiningOp<memref::MemorySpaceCastOp>();
     if (!memSpaceCastOp)
       return failure();
-    auto newExtractStridedMetadata =
-        memref::ExtractStridedMetadataOp::create(rewriter,
-            loc, memSpaceCastOp.getSource());
+    auto newExtractStridedMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, memSpaceCastOp.getSource());
     SmallVector<Value> results(newExtractStridedMetadata.getResults());
     // As with most other strided metadata rewrite patterns, don't introduce
     // a use of the base pointer where non existed. This needs to happen here,
@@ -1158,8 +1155,8 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
       MemRefType::Builder newTypeBuilder(baseBufferType);
       newTypeBuilder.setMemorySpace(
           memSpaceCastOp.getResult().getType().getMemorySpace());
-      results[0] = memref::MemorySpaceCastOp::create(rewriter,
-          loc, Type{newTypeBuilder}, baseBuffer);
+      results[0] = memref::MemorySpaceCastOp::create(
+          rewriter, loc, Type{newTypeBuilder}, baseBuffer);
     } else {
       results[0] = nullptr;
     }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
index 65f398f6dc5a8..0946da8e4e919 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
@@ -43,7 +43,7 @@ static memref::LoadOp rebuildLoadOp(RewriterBase &rewriter,
                                     ArrayRef<Value> indices) {
   Location loc = loadOp.getLoc();
   return memref::LoadOp::create(rewriter, loc, srcMemRef, indices,
-                                         loadOp.getNontemporal());
+                                loadOp.getNontemporal());
 }
 
 // Matches getViewSizeForEachDim specs for LoadOp.
@@ -73,8 +73,7 @@ static memref::StoreOp rebuildStoreOp(RewriterBase &rewriter,
                                       ArrayRef<Value> indices) {
   Location loc = storeOp.getLoc();
   return memref::StoreOp::create(rewriter, loc, storeOp.getValueToStore(),
-                                          srcMemRef, indices,
-                                          storeOp.getNontemporal());
+                                 srcMemRef, indices, storeOp.getNontemporal());
 }
 
 // Matches getViewSizeForEachDim specs for StoreOp.
@@ -104,8 +103,8 @@ static nvgpu::LdMatrixOp rebuildLdMatrixOp(RewriterBase &rewriter,
                                            Value srcMemRef,
                                            ArrayRef<Value> indices) {
   Location loc = ldMatrixOp.getLoc();
-  return nvgpu::LdMatrixOp::create(rewriter,
-      loc, ldMatrixOp.getResult().getType(), srcMemRef, indices,
+  return nvgpu::LdMatrixOp::create(
+      rewriter, loc, ldMatrixOp.getResult().getType(), srcMemRef, indices,
       ldMatrixOp.getTranspose(), ldMatrixOp.getNumTiles());
 }
 
@@ -132,8 +131,8 @@ rebuildTransferReadOp(RewriterBase &rewriter,
                       vector::TransferReadOp transferReadOp, Value srcMemRef,
                       ArrayRef<Value> indices) {
   Location loc = transferReadOp.getLoc();
-  return vector::TransferReadOp::create(rewriter,
-      loc, transferReadOp.getResult().getType(), srcMemRef, indices,
+  return vector::TransferReadOp::create(
+      rewriter, loc, transferReadOp.getResult().getType(), srcMemRef, indices,
       transferReadOp.getPermutationMap(), transferReadOp.getPadding(),
       transferReadOp.getMask(), transferReadOp.getInBoundsAttr());
 }
@@ -150,8 +149,8 @@ rebuildTransferWriteOp(RewriterBase &rewriter,
                        vector::TransferWriteOp transferWriteOp, Value srcMemRef,
                        ArrayRef<Value> indices) {
   Location loc = transferWriteOp.getLoc();
-  return vector::TransferWriteOp::create(rewriter,
-      loc, transferWriteOp.getValue(), srcMemRef, indices,
+  return vector::TransferWriteOp::create(
+      rewriter, loc, transferWriteOp.getValue(), srcMemRef, indices,
       transferWriteOp.getPermutationMapAttr(), transferWriteOp.getMask(),
       transferWriteOp.getInBoundsAttr());
 }
@@ -182,9 +181,8 @@ static SmallVector<OpFoldResult>
 getGenericOpViewSizeForEachDim(RewriterBase &rewriter,
                                LoadStoreLikeOp loadStoreLikeOp) {
   Location loc = loadStoreLikeOp.getLoc();
-  auto extractStridedMetadataOp =
-      memref::ExtractStridedMetadataOp::create(rewriter,
-          loc, getSrcMemRef(loadStoreLikeOp));
+  auto extractStridedMetadataOp = memref::ExtractStridedMetadataOp::create(
+      rewriter, loc, getSrcMemRef(loadStoreLikeOp));
   SmallVector<OpFoldResult> srcSizes =
       extractStridedMetadataOp.getConstifiedMixedSizes();
   SmallVector<OpFoldResult> indices =
@@ -268,8 +266,8 @@ struct LoadStoreLikeOpRewriter : public OpRewritePattern<LoadStoreLikeOp> {
     // Therefore the strides multipliers are simply ones.
     auto subview =
         memref::SubViewOp::create(rewriter, loc, /*source=*/srcMemRef,
-                                           /*offsets=*/indices,
-                                           /*sizes=*/sizes, /*strides=*/ones);
+                                  /*offsets=*/indices,
+                                  /*sizes=*/sizes, /*strides=*/ones);
     // Rewrite the load/store with the subview as the base pointer.
     SmallVector<Value> zeros(loadStoreRank,
                              arith::ConstantIndexOp::create(rewriter, loc, 0));
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
index 48dbabfdf890a..42be847811d52 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
@@ -40,8 +40,8 @@ using namespace mlir;
 static Value getValueFromOpFoldResult(OpBuilder &rewriter, Location loc,
                                       OpFoldResult in) {
   if (Attribute offsetAttr = dyn_cast<Attribute>(in)) {
-    return arith::ConstantIndexOp::create(rewriter,
-        loc, cast<IntegerAttr>(offsetAttr).getInt());
+    return arith::ConstantIndexOp::create(
+        rewriter, loc, cast<IntegerAttr>(offsetAttr).getInt());
   }
   return cast<Value>(in);
 }
@@ -74,8 +74,8 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
           getAsOpFoldResult(indices));
 
   return std::make_pair(
-      memref::ReinterpretCastOp::create(rewriter,
-          loc, source,
+      memref::ReinterpretCastOp::create(
+          rewriter, loc, source,
           /* offset = */ linearizedInfo.linearizedOffset,
           /* shapes = */
           ArrayRef<OpFoldResult>{linearizedInfo.linearizedSize},
@@ -125,63 +125,68 @@ static void replaceOp(T op, PatternRewriter &rewriter, Value flatMemref,
   Location loc = op->getLoc();
   llvm::TypeSwitch<Operation *>(op.getOperation())
       .template Case<memref::AllocOp>([&](auto oper) {
-        auto newAlloc = memref::AllocOp::create(rewriter,
-            loc, cast<MemRefType>(flatMemref.getType()),
+        auto newAlloc = memref::AllocOp::create(
+            rewriter, loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloc, loc, rewriter);
       })
       .template Case<memref::AllocaOp>([&](auto oper) {
-        auto newAlloca = memref::AllocaOp::create(rewriter,
-            loc, cast<MemRefType>(flatMemref.getType()),
+        auto newAlloca = memref::AllocaOp::create(
+            rewriter, loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloca, loc, rewriter);
       })
       .template Case<memref::LoadOp>([&](auto op) {
-        auto newLoad = memref::LoadOp::create(rewriter,
-            loc, op->getResultTypes(), flatMemref, ValueRange{offset});
+        auto newLoad =
+            memref::LoadOp::create(rewriter, loc, op->getResultTypes(),
+                                   flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<memref::StoreOp>([&](auto op) {
-        auto newStore = memref::StoreOp::create(rewriter,
-            loc, op->getOperands().front(), flatMemref, ValueRange{offset});
+        auto newStore =
+            memref::StoreOp::create(rewriter, loc, op->getOperands().front(),
+                                    flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::LoadOp>([&](auto op) {
-        auto newLoad = vector::LoadOp::create(rewriter,
-            loc, op->getResultTypes(), flatMemref, ValueRange{offset});
+        auto newLoad =
+            vector::LoadOp::create(rewriter, loc, op->getResultTypes(),
+                                   flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<vector::StoreOp>([&](auto op) {
-        auto newStore = vector::StoreOp::create(rewriter,
-            loc, op->getOperands().front(), flatMemref, ValueRange{offset});
+        auto newStore =
+            vector::StoreOp::create(rewriter, loc, op->getOperands().front(),
+                                    flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::MaskedLoadOp>([&](auto op) {
-        auto newMaskedLoad = vector::MaskedLoadOp::create(rewriter,
-            loc, op.getType(), flatMemref, ValueRange{offset}, op.getMask(),
-            op.getPassThru());
+        auto newMaskedLoad = vector::MaskedLoadOp::create(
+            rewriter, loc, op.getType(), flatMemref, ValueRange{offset},
+            op.getMask(), op.getPassThru());
         newMaskedLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedLoad.getResult());
       })
       .template Case<vector::MaskedStoreOp>([&](auto op) {
-        auto newMaskedStore = vector::MaskedStoreOp::create(rewriter,
-            loc, flatMemref, ValueRange{offset}, op.getMask(),
+        auto newMaskedStore = vector::MaskedStoreOp::create(
+            rewriter, loc, flatMemref, ValueRange{offset}, op.getMask(),
             op.getValueToStore());
         newMaskedStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedStore);
       })
       .template Case<vector::TransferReadOp>([&](auto op) {
-        auto newTransferRead = vector::TransferReadOp::create(rewriter,
-            loc, op.getType(), flatMemref, ValueRange{offset}, op.getPadding());
+        auto newTransferRead = vector::TransferReadOp::create(
+            rewriter, loc, op.getType(), flatMemref, ValueRange{offset},
+            op.getPadding());
         rewriter.replaceOp(op, newTransferRead.getResult());
       })
       .template Case<vector::TransferWriteOp>([&](auto op) {
-        auto newTransferWrite = vector::TransferWriteOp::create(rewriter,
-            loc, op.getVector(), flatMemref, ValueRange{offset});
+        auto newTransferWrite = vector::TransferWriteOp::create(
+            rewriter, loc, op.getVector(), flatMemref, ValueRange{offset});
         rewriter.replaceOp(op, newTransferWrite);
       })
       .Default([&](auto op) {
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index 8e99a101f28a6..991065c9506a1 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -76,8 +76,8 @@ static LogicalResult resolveSourceIndicesExpandShape(
         llvm::map_to_vector(group, [&](int64_t d) { return destShape[d]; });
     SmallVector<Value> groupIndices =
         llvm::map_to_vector(group, [&](int64_t d) { return indices[d]; });
-    Value collapsedIndex = affine::AffineLinearizeIndexOp::create(rewriter,
-        loc, groupIndices, groupBasis, /*disjoint=*/startsInbounds);
+    Value collapsedIndex = affine::AffineLinearizeIndexOp::create(
+        rewriter, loc, groupIndices, groupBasis, /*disjoint=*/startsInbounds);
     sourceIndices.push_back(collapsedIndex);
   }
   return success();
@@ -102,8 +102,8 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
                                   ValueRange indices,
                                   SmallVectorImpl<Value> &sourceIndices) {
   // Note: collapse_shape requires a strided memref, we can do this.
-  auto metadata = memref::ExtractStridedMetadataOp::create(rewriter,
-      loc, collapseShapeOp.getSrc());
+  auto metadata = memref::ExtractStridedMetadataOp::create(
+      rewriter, loc, collapseShapeOp.getSrc());
   SmallVector<OpFoldResult> sourceSizes = metadata.getConstifiedMixedSizes();
   for (auto [index, group] :
        llvm::zip(indices, collapseShapeOp.getReassociationIndices())) {
@@ -117,8 +117,8 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
 
     SmallVector<OpFoldResult> basis =
         llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-    auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-        loc, index, basis, /*hasOuterBound=*/true);
+    auto delinearize = affine::AffineDelinearizeIndexOp::create(
+        rewriter, loc, index, basis, /*hasOuterBound=*/true);
     llvm::append_range(sourceIndices, delinearize.getResults());
   }
   if (collapseShapeOp.getReassociationIndices().empty()) {
diff --git a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
index 953024d70d0aa..66c1aa6bf3fe1 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
@@ -71,11 +71,11 @@ propagateSubViewOp(RewriterBase &rewriter,
   MemRefType newResultType = SubViewOp::inferRankReducedResultType(
       op.getType().getShape(), op.getSourceType(), op.getMixedOffsets(),
       op.getMixedSizes(), op.getMixedStrides());
-  Value newSubview = SubViewOp::create(rewriter,
-      op.getLoc(), newResultType, conversionOp.getOperand(0),
+  Value newSubview = SubViewOp::create(
+      rewriter, op.getLoc(), newResultType, conversionOp.getOperand(0),
       op.getMixedOffsets(), op.getMixedSizes(), op.getMixedStrides());
-  auto newConversionOp = UnrealizedConversionCastOp::create(rewriter,
-      op.getLoc(), op.getType(), newSubview);
+  auto newConversionOp = UnrealizedConversionCastOp::create(
+      rewriter, op.getLoc(), op.getType(), newSubview);
   rewriter.replaceAllUsesWith(op.getResult(), newConversionOp->getResult(0));
   return newConversionOp;
 }
@@ -106,8 +106,8 @@ static void replaceAndPropagateMemRefType(RewriterBase &rewriter,
   SmallVector<UnrealizedConversionCastOp> unrealizedConversions;
   for (const auto &it :
        llvm::enumerate(llvm::zip(from->getResults(), to->getResults()))) {
-    unrealizedConversions.push_back(UnrealizedConversionCastOp::create(rewriter,
-        to->getLoc(), std::get<0>(it.value()).getType(),
+    unrealizedConversions.push_back(UnrealizedConversionCastOp::create(
+        rewriter, to->getLoc(), std::get<0>(it.value()).getType(),
         std::get<1>(it.value())));
     rewriter.replaceAllUsesWith(from->getResult(it.index()),
                                 unrealizedConversions.back()->getResult(0));
diff --git a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
index b7e4388193a06..5d3cec402cab1 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
@@ -63,9 +63,10 @@ static void replaceUsesAndPropagateType(RewriterBase &rewriter,
         subviewUse.getType().getShape(), cast<MemRefType>(val.getType()),
         subviewUse.getStaticOffsets(), subviewUse.getStaticSizes(),
         subviewUse.getStaticStrides());
-    Value newSubview = memref::SubViewOp::create(rewriter,
-        subviewUse->getLoc(), newType, val, subviewUse.getMixedOffsets(),
-        subviewUse.getMixedSizes(), subviewUse.getMixedStrides());
+    Value newSubview = memref::SubViewOp::create(
+        rewriter, subviewUse->getLoc(), newType, val,
+        subviewUse.getMixedOffsets(), subviewUse.getMixedSizes(),
+        subviewUse.getMixedStrides());
 
     // Ouch recursion ... is this really necessary?
     replaceUsesAndPropagateType(rewriter, subviewUse, newSubview);
@@ -177,8 +178,8 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   Location loc = allocOp->getLoc();
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(allocOp);
-  auto mbAlloc = memref::AllocOp::create(rewriter,
-      loc, mbMemRefType, ValueRange{}, allocOp->getAttrs());
+  auto mbAlloc = memref::AllocOp::create(rewriter, loc, mbMemRefType,
+                                         ValueRange{}, allocOp->getAttrs());
   LLVM_DEBUG(DBGS() << "--multi-buffered alloc: " << mbAlloc << "\n");
 
   // 3. Within the loop, build the modular leading index (i.e. each loop
@@ -212,7 +213,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   MemRefType dstMemref = memref::SubViewOp::inferRankReducedResultType(
       originalShape, mbMemRefType, offsets, sizes, strides);
   Value subview = memref::SubViewOp::create(rewriter, loc, dstMemref, mbAlloc,
-                                                     offsets, sizes, strides);
+                                            offsets, sizes, strides);
   LLVM_DEBUG(DBGS() << "--multi-buffered slice: " << subview << "\n");
 
   // 5. Due to the recursive nature of replaceUsesAndPropagateType , we need to
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index c522b2bd5ddec..fa7991e6c6a80 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -277,7 +277,7 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
       continue;
     Operation *newCallOp =
         func::CallOp::create(builder, userOp->getLoc(), callOp.getCalleeAttr(),
-                                     resultTypes, userOp->getOperands());
+                             resultTypes, userOp->getOperands());
     bool replacingMemRefUsesFailed = false;
     bool returnTypeChanged = false;
     for (unsigned resIndex : llvm::seq<unsigned>(0, userOp->getNumResults())) {
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
index ea7f394f48ba4..40d45b57257c6 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
@@ -116,10 +116,12 @@ static LogicalResult reifyOpResultShapes(RewriterBase &rewriter,
     // Update the type.
     newRes.setType(reifiedTy);
     if (isa<RankedTensorType>(reifiedTy)) {
-      newResults.push_back(tensor::CastOp::create(rewriter, loc, oldTy, newRes));
+      newResults.push_back(
+          tensor::CastOp::create(rewriter, loc, oldTy, newRes));
     } else {
       assert(isa<MemRefType>(reifiedTy) && "expected a memref type");
-      newResults.push_back(memref::CastOp::create(rewriter, loc, oldTy, newRes));
+      newResults.push_back(
+          memref::CastOp::create(rewriter, loc, oldTy, newRes));
     }
   }
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
index fe9555a7a16fd..1f03e9ae8d6a1 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
@@ -40,19 +40,18 @@ struct AssumeAlignmentOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto assumeOp = cast<AssumeAlignmentOp>(op);
-    Value ptr = ExtractAlignedPointerAsIndexOp::create(builder,
-        loc, assumeOp.getMemref());
-    Value rest = arith::RemUIOp::create(builder,
-        loc, ptr,
+    Value ptr = ExtractAlignedPointerAsIndexOp::create(builder, loc,
+                                                       assumeOp.getMemref());
+    Value rest = arith::RemUIOp::create(
+        builder, loc, ptr,
         arith::ConstantIndexOp::create(builder, loc, assumeOp.getAlignment()));
-    Value isAligned = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::eq, rest,
-        arith::ConstantIndexOp::create(builder, loc, 0));
-    cf::AssertOp::create(builder,
-        loc, isAligned,
-        RuntimeVerifiableOpInterface::generateErrorMessage(
-            op, "memref is not aligned to " +
-                    std::to_string(assumeOp.getAlignment())));
+    Value isAligned =
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, rest,
+                              arith::ConstantIndexOp::create(builder, loc, 0));
+    cf::AssertOp::create(builder, loc, isAligned,
+                         RuntimeVerifiableOpInterface::generateErrorMessage(
+                             op, "memref is not aligned to " +
+                                     std::to_string(assumeOp.getAlignment())));
   }
 };
 
@@ -74,12 +73,11 @@ struct CastOpInterface
       Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
           arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
-      Value isSameRank = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      cf::AssertOp::create(builder,
-          loc, isSameRank,
-          RuntimeVerifiableOpInterface::generateErrorMessage(op,
-                                                             "rank mismatch"));
+      Value isSameRank = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcRank, resultRank);
+      cf::AssertOp::create(builder, loc, isSameRank,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "rank mismatch"));
     }
 
     // Get source offset and strides. We do not have an op to get offsets and
@@ -96,7 +94,8 @@ struct CastOpInterface
                         stridedLayout, resultType.getMemorySpace());
     Value helperCast =
         CastOp::create(builder, loc, dynStridesType, castOp.getSource());
-    auto metadataOp = ExtractStridedMetadataOp::create(builder, loc, helperCast);
+    auto metadataOp =
+        ExtractStridedMetadataOp::create(builder, loc, helperCast);
 
     // Check dimension sizes.
     for (const auto &it : llvm::enumerate(resultType.getShape())) {
@@ -113,10 +112,10 @@ struct CastOpInterface
           DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
           arith::ConstantIndexOp::create(builder, loc, it.value());
-      Value isSameSz = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      cf::AssertOp::create(builder,
-          loc, isSameSz,
+      Value isSameSz = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
+      cf::AssertOp::create(
+          builder, loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
     }
@@ -133,12 +132,11 @@ struct CastOpInterface
       Value srcOffset = metadataOp.getResult(1);
       Value resultOffsetVal =
           arith::ConstantIndexOp::create(builder, loc, resultOffset);
-      Value isSameOffset = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcOffset, resultOffsetVal);
-      cf::AssertOp::create(builder,
-          loc, isSameOffset,
-          RuntimeVerifiableOpInterface::generateErrorMessage(
-              op, "offset mismatch"));
+      Value isSameOffset = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcOffset, resultOffsetVal);
+      cf::AssertOp::create(builder, loc, isSameOffset,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "offset mismatch"));
     }
 
     // Check strides.
@@ -151,10 +149,10 @@ struct CastOpInterface
           metadataOp.getResult(2 + resultType.getRank() + it.index());
       Value resultStrideVal =
           arith::ConstantIndexOp::create(builder, loc, it.value());
-      Value isSameStride = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcStride, resultStrideVal);
-      cf::AssertOp::create(builder,
-          loc, isSameStride,
+      Value isSameStride = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcStride, resultStrideVal);
+      cf::AssertOp::create(
+          builder, loc, isSameStride,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "stride mismatch of dim " + std::to_string(it.index())));
     }
@@ -194,13 +192,12 @@ struct CopyOpInterface
       };
       Value sourceDim = getDimSize(copyOp.getSource(), rankedSourceType, i);
       Value targetDim = getDimSize(copyOp.getTarget(), rankedTargetType, i);
-      Value sameDimSize = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
-      cf::AssertOp::create(builder,
-          loc, sameDimSize,
-          RuntimeVerifiableOpInterface::generateErrorMessage(
-              op, "size of " + std::to_string(i) +
-                      "-th source/target dim does not match"));
+      Value sameDimSize = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
+      cf::AssertOp::create(builder, loc, sameDimSize,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "size of " + std::to_string(i) +
+                                       "-th source/target dim does not match"));
     }
   }
 };
@@ -213,8 +210,9 @@ struct DimOpInterface
     auto dimOp = cast<DimOp>(op);
     Value rank = RankOp::create(builder, loc, dimOp.getSource());
     Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
-    cf::AssertOp::create(builder,
-        loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
+    cf::AssertOp::create(
+        builder, loc,
+        generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
   }
@@ -247,10 +245,9 @@ struct LoadStoreOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    cf::AssertOp::create(builder,
-        loc, assertCond,
-        RuntimeVerifiableOpInterface::generateErrorMessage(
-            op, "out-of-bounds access"));
+    cf::AssertOp::create(builder, loc, assertCond,
+                         RuntimeVerifiableOpInterface::generateErrorMessage(
+                             op, "out-of-bounds access"));
   }
 };
 
@@ -281,8 +278,8 @@ struct SubViewOpInterface
       Value dimSize = metadataOp.getSizes()[i];
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      cf::AssertOp::create(builder,
-          loc, offsetInBounds,
+      cf::AssertOp::create(
+          builder, loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
@@ -294,8 +291,8 @@ struct SubViewOpInterface
           arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      cf::AssertOp::create(builder,
-          loc, lastPosInBounds,
+      cf::AssertOp::create(
+          builder, loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "subview runs out-of-bounds along dimension " +
                       std::to_string(i)));
@@ -334,14 +331,13 @@ struct ExpandShapeOpInterface
       // staticResultDimSz must divide srcDimSz evenly.
       Value mod =
           arith::RemSIOp::create(builder, loc, srcDimSz, staticResultDimSz);
-      Value isModZero = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, mod,
+      Value isModZero = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, mod,
           arith::ConstantIndexOp::create(builder, loc, 0));
-      cf::AssertOp::create(builder,
-          loc, isModZero,
-          RuntimeVerifiableOpInterface::generateErrorMessage(
-              op, "static result dims in reassoc group do not "
-                  "divide src dim evenly"));
+      cf::AssertOp::create(builder, loc, isModZero,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "static result dims in reassoc group do not "
+                                   "divide src dim evenly"));
     }
   }
 };
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
index 65446142aabcc..db5fd6e494da1 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
@@ -91,8 +91,8 @@ struct MeshShapeFolder
         newShapeOpMeshAxes.push_back(opMeshAxes[i]);
       } else {
         // Fold static mesh axes.
-        newResults[i] = arith::ConstantOp::create(builder,
-            builder.getIndexAttr(meshAxisSize));
+        newResults[i] = arith::ConstantOp::create(
+            builder, builder.getIndexAttr(meshAxisSize));
       }
     }
 
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index 9229bfaa1dffb..1e54affa8198f 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -265,7 +265,8 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
       targetShardingInUnsplitLastAxis(ctx, sourceSharding, splitTensorAxis);
   ShapedType allGatherResultShape = allGatherResultShapeInUnsplitLastAxis(
       sourceShard.getType(), mesh.getShape()[splitMeshAxis], splitTensorAxis);
-  Value allGatherResult = AllGatherOp::create(builder,
+  Value allGatherResult = AllGatherOp::create(
+      builder,
       RankedTensorType::get(allGatherResultShape.getShape(),
                             allGatherResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({splitMeshAxis}), sourceShard,
@@ -273,7 +274,8 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      tensor::CastOp::create(builder, targetShape, allGatherResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allGatherResult)
+          .getResult());
   return {targetShard, targetSharding};
 }
 
@@ -398,7 +400,8 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType allToAllResultShape = allToAllResultShapeInMoveLastAxis(
       sourceShard.getType(), mesh.getShape()[meshAxis], sourceTensorAxis,
       targetTensorAxis);
-  Value allToAllResult = AllToAllOp::create(builder,
+  Value allToAllResult = AllToAllOp::create(
+      builder,
       RankedTensorType::get(allToAllResultShape.getShape(),
                             allToAllResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({meshAxis}), sourceShard,
@@ -477,15 +480,16 @@ tryUpdateHaloInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
 
   // Extract core from source and copy into destination core.
   auto noVals = ValueRange{};
-  auto initVal = tensor::EmptyOp::create(builder,
-      sourceShard.getLoc(), outShape, sourceShard.getType().getElementType());
-  auto core = tensor::ExtractSliceOp::create(builder,
-      sourceShard.getLoc(),
+  auto initVal =
+      tensor::EmptyOp::create(builder, sourceShard.getLoc(), outShape,
+                              sourceShard.getType().getElementType());
+  auto core = tensor::ExtractSliceOp::create(
+      builder, sourceShard.getLoc(),
       RankedTensorType::get(coreShape, sourceShard.getType().getElementType()),
       sourceShard, noVals, noVals, noVals, srcCoreOffs, coreShape, strides);
-  auto initOprnd = tensor::InsertSliceOp::create(builder,
-      sourceShard.getLoc(), core, initVal, noVals, noVals, noVals, tgtCoreOffs,
-      coreShape, strides);
+  auto initOprnd = tensor::InsertSliceOp::create(
+      builder, sourceShard.getLoc(), core, initVal, noVals, noVals, noVals,
+      tgtCoreOffs, coreShape, strides);
 
   // Finally update the halo.
   auto updateHaloResult =
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
index c85394924608d..6ae95ae1f8a49 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
@@ -52,7 +52,8 @@ struct ProcessMultiIndexOpLowering
     Value linearIndex = ProcessLinearIndexOp::create(builder, mesh);
     ValueRange meshShape = MeshShapeOp::create(builder, mesh).getResults();
     SmallVector<Value> completeMultiIndex =
-        affine::AffineDelinearizeIndexOp::create(builder, linearIndex, meshShape)
+        affine::AffineDelinearizeIndexOp::create(builder, linearIndex,
+                                                 meshShape)
             .getMultiIndex();
     SmallVector<Value> multiIndex;
     ArrayRef<MeshAxis> opMeshAxes = op.getAxes();
@@ -104,7 +105,8 @@ struct AllSliceOpLowering
     Value zero = arith::ConstantOp::create(builder, builder.getIndexAttr(0));
 
     Operation::result_range processInGroupMultiIndex =
-        ProcessMultiIndexOp::create(builder, mesh.getSymName(), op.getMeshAxes())
+        ProcessMultiIndexOp::create(builder, mesh.getSymName(),
+                                    op.getMeshAxes())
             .getResults();
 
     Operation::result_range processGroupShape =
@@ -118,13 +120,13 @@ struct AllSliceOpLowering
         tensor::DimOp::create(builder, op.getOperand(), sliceAxis);
     Value operandSliceAxisSizeModProcessGroupSize =
         arith::RemUIOp::create(builder, operandSliceAxisSize, processGroupSize);
-    Value isTargetShapeExactlyDivisible = arith::CmpIOp::create(builder,
-        arith::CmpIPredicate::eq, operandSliceAxisSizeModProcessGroupSize,
-        zero);
+    Value isTargetShapeExactlyDivisible =
+        arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
+                              operandSliceAxisSizeModProcessGroupSize, zero);
     cf::AssertOp::create(builder, isTargetShapeExactlyDivisible,
-                                 "Slicing a tensor with axis size that is "
-                                 "not exactly divisible by the "
-                                 "mesh process group size is not supported.");
+                         "Slicing a tensor with axis size that is "
+                         "not exactly divisible by the "
+                         "mesh process group size is not supported.");
     Value resultSliceAxisSize =
         arith::DivUIOp::create(builder, operandSliceAxisSize, processGroupSize);
     OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
@@ -152,8 +154,8 @@ struct AllSliceOpLowering
                  resultSliceAxisSize);
     SmallVector<OpFoldResult> strides(
         operandType.getRank(), getAsIndexOpFoldResult(builder.getContext(), 1));
-    Value slice = tensor::ExtractSliceOp::create(builder,
-        op.getOperand(), offsets, sizes, strides);
+    Value slice = tensor::ExtractSliceOp::create(builder, op.getOperand(),
+                                                 offsets, sizes, strides);
     Value newResult =
         tensor::CastOp::create(builder, op.getResult().getType(), slice);
     rewriter.replaceAllUsesWith(op.getResult(), newResult);
@@ -218,7 +220,8 @@ createProcessLinearIndex(StringRef mesh, ValueRange processInGroupMultiIndex,
       llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
   auto res = dyn_cast<Value>(processInGroupLinearIndex);
   if (!res)
-    res = arith::ConstantIndexOp::create(builder,
+    res = arith::ConstantIndexOp::create(
+        builder,
         cast<IntegerAttr>(cast<Attribute>(processInGroupLinearIndex)).getInt());
   return cast<TypedValue<IndexType>>(res);
 }
diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
index 85317a68c9f00..5d253c1199dc0 100644
--- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
+++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
@@ -333,15 +333,15 @@ static Operation *replaceOpWithPredicatedOp(RewriterBase &rewriter,
   //   srcElement = (pred) ?  prevSrcElements : 0;
   //
   Location loc = asyncCopyOp->getLoc();
-  Value dstElements =
-      arith::ConstantOp::create(rewriter, loc, asyncCopyOp.getDstElementsAttr());
+  Value dstElements = arith::ConstantOp::create(
+      rewriter, loc, asyncCopyOp.getDstElementsAttr());
   Value originalSrcElement =
       asyncCopyOp.getSrcElements() ? asyncCopyOp.getSrcElements() : dstElements;
   Value c0Index = arith::ConstantIndexOp::create(rewriter, loc, 0);
-  auto srcElements = arith::SelectOp::create(rewriter,
-      loc, predicate, originalSrcElement, c0Index);
-  auto asyncCopyZeroFillOp = nvgpu::DeviceAsyncCopyOp::create(rewriter,
-      loc, nvgpu::DeviceAsyncTokenType::get(asyncCopyOp.getContext()),
+  auto srcElements = arith::SelectOp::create(rewriter, loc, predicate,
+                                             originalSrcElement, c0Index);
+  auto asyncCopyZeroFillOp = nvgpu::DeviceAsyncCopyOp::create(
+      rewriter, loc, nvgpu::DeviceAsyncTokenType::get(asyncCopyOp.getContext()),
       asyncCopyOp.getDst(), asyncCopyOp.getDstIndices(), asyncCopyOp.getSrc(),
       asyncCopyOp.getSrcIndices(), asyncCopyOp.getDstElements(), srcElements,
       UnitAttr());
@@ -811,7 +811,7 @@ FailureOr<Operation *> MmaSyncBuilder::buildMmaSync(LinalgOp linalgOp) {
   Value res = buildMmaSyncMemRefLoadOperand(b, loc, laneId, resMemRef,
                                             resIndexFn, resShape);
   res = nvgpu::MmaSyncOp::create(b, loc, lhs, rhs, res, info.mmaShape,
-                                   info.tf32Enabled);
+                                 info.tf32Enabled);
   buildMmaSyncMemRefStoreOperand(b, loc, res, laneId, resMemRef, resIndexFn,
                                  resShape);
   return res.getDefiningOp();
@@ -832,8 +832,8 @@ DiagnosedSilenceableFailure transform::RewriteMatmulAsMmaSyncOp::applyToOne(
     }
     Location loc = linalgOp.getLoc();
     // TODO: more robust computation of laneId, for now assume a single warp.
-    Value laneId = gpu::ThreadIdOp::create(rewriter,
-        loc, rewriter.getIndexType(), gpu::Dimension::x);
+    Value laneId = gpu::ThreadIdOp::create(
+        rewriter, loc, rewriter.getIndexType(), gpu::Dimension::x);
     if (succeeded(MmaSyncBuilder(rewriter, loc, laneId).buildMmaSync(linalgOp)))
       fail = false;
   }
@@ -899,8 +899,8 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
   SmallVector<Operation *> loadOps;
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   Value tidx = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
-  Value cond =
-      arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, tidx, zero);
+  Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
+                                     tidx, zero);
   // clang-format off
   scf::IfOp::create(rewriter,
     /*location=*/loc,
@@ -939,13 +939,14 @@ static Attribute getSharedAddressSpaceAttribute(OpBuilder &b) {
 TypedValue<nvgpu::MBarrierGroupType>
 HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) {
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value barrier = nvgpu::MBarrierCreateOp::create(rewriter,
-      loc,
+  Value barrier = nvgpu::MBarrierCreateOp::create(
+      rewriter, loc,
       nvgpu::MBarrierGroupType::get(rewriter.getContext(), sharedMemorySpace));
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
-  nvgpu::MBarrierInitOp::create(rewriter,
-      loc, barrier, getValueOrCreateConstantIndexOp(rewriter, loc, numThreads),
-      zero, Value());
+  nvgpu::MBarrierInitOp::create(
+      rewriter, loc, barrier,
+      getValueOrCreateConstantIndexOp(rewriter, loc, numThreads), zero,
+      Value());
   gpu::BarrierOp::create(rewriter, loc);
   return cast<TypedValue<nvgpu::MBarrierGroupType>>(barrier);
 }
@@ -955,8 +956,8 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
                                            gpu::LaunchOp launchOp) {
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(launchOp);
-  Value unrankedMemRef = memref::CastOp::create(rewriter,
-      loc,
+  Value unrankedMemRef = memref::CastOp::create(
+      rewriter, loc,
       UnrankedMemRefType::get(memref.getType().getElementType(),
                               memref.getType().getMemorySpace()),
       memref);
@@ -966,8 +967,8 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
       getValueOrCreateConstantIndexOp(rewriter, loc, mixedSizes);
 
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value desc = nvgpu::TmaCreateDescriptorOp::create(rewriter,
-      loc,
+  Value desc = nvgpu::TmaCreateDescriptorOp::create(
+      rewriter, loc,
       nvgpu::TensorMapDescriptorType::get(
           rewriter.getContext(),
           MemRefType::Builder(memref.getType())
@@ -986,9 +987,9 @@ OpFoldResult HopperBuilder::buildTmaAsyncLoad(
     SmallVectorImpl<Operation *> &loadOps) {
   MLIRContext *ctx = rewriter.getContext();
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
-  Operation *loadOp = nvgpu::TmaAsyncLoadOp::create(rewriter,
-      loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero,
-      Value(), Value());
+  Operation *loadOp = nvgpu::TmaAsyncLoadOp::create(
+      rewriter, loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero},
+      zero, Value(), Value());
   loadOps.push_back(loadOp);
   auto mixedSizes = memref::getMixedSizes(rewriter, loc, sharedMemref);
   SmallVector<AffineExpr> symbols(mixedSizes.size());
@@ -1014,7 +1015,7 @@ void HopperBuilder::buildBarrierArriveTx(
   Value sizeVal = getValueOrCreateConstantIndexOp(rewriter, loc, size);
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   nvgpu::MBarrierArriveExpectTxOp::create(rewriter, loc, barrier, sizeVal, zero,
-                                                   Value());
+                                          Value());
 }
 
 void HopperBuilder::buildTryWaitParity(
@@ -1028,7 +1029,7 @@ void HopperBuilder::buildTryWaitParity(
       arith::ConstantIndexOp::create(rewriter, loc, 10000000);
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   nvgpu::MBarrierTryWaitParityOp::create(rewriter, loc, barrier, parity,
-                                                  ticksBeforeRetry, zero);
+                                         ticksBeforeRetry, zero);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
index b8baadb4ef241..b392ffeb13de6 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
@@ -110,15 +110,15 @@ static Value buildNumReadElements(OpBuilder &b, Location loc,
                                   transferMask->createMaskOp->getOperands())) {
     Value cmp =
         arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt,
-                                arith::ConstantIndexOp::create(b, loc, pos), sz);
+                              arith::ConstantIndexOp::create(b, loc, pos), sz);
     if (!cond) {
       cond = cmp;
       continue;
     }
     cond = arith::AndIOp::create(b, loc, cmp, cond);
   }
-  return arith::SelectOp::create(b,
-      loc, cond, transferMask->createMaskOp->getOperands().back(),
+  return arith::SelectOp::create(
+      b, loc, cond, transferMask->createMaskOp->getOperands().back(),
       arith::ConstantIndexOp::create(b, loc, 0));
 }
 
@@ -251,8 +251,9 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
       int64_t sizeInBytes =
           (dstMemref.getElementTypeBitWidth() * numElements) / 8;
       // bypass_l1 only possible with 16 byte transfer.
-      Value token = nvgpu::DeviceAsyncCopyOp::create(rewriter,
-          writeOp->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
+      Value token = nvgpu::DeviceAsyncCopyOp::create(
+          rewriter, writeOp->getLoc(),
+          nvgpu::DeviceAsyncTokenType::get(op->getContext()),
           /*dst=*/storeBase, /*dstIndices=*/nvgpu::getIndices(writeOp),
           /*src=*/loadBase,
           /*srcIndices=*/nvgpu::getIndices(readOp),
@@ -264,11 +265,11 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
     }
 
     // Create the group and wait for it right after.
-    Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(rewriter,
-        op->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
-        tokens);
+    Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(
+        rewriter, op->getLoc(),
+        nvgpu::DeviceAsyncTokenType::get(op->getContext()), tokens);
     nvgpu::DeviceAsyncWaitOp::create(rewriter, op->getLoc(), groupToken,
-                                              nullptr);
+                                     nullptr);
     // Clean up old stores.
     for (Operation *writeOp : group)
       rewriter.eraseOp(writeOp);
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
index 2d2ff94a25f92..957b9632422a6 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
@@ -85,7 +85,8 @@ static Value permuteVectorOffset(OpBuilder &b, Location loc,
       Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, shlBits);
       srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
     } else if (shlBits < 0) {
-      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, -1 * shlBits);
+      Value finalShiftVal =
+          arith::ConstantIndexOp::create(b, loc, -1 * shlBits);
       srcBits = b.createOrFold<arith::ShRUIOp>(loc, srcBits, finalShiftVal);
     }
   } else {
diff --git a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
index a378b09c27c41..58cd160948f7f 100644
--- a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
@@ -95,8 +95,8 @@ std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
   auto *context = builder.getContext();
   auto shapeType = shape::getExtentTensorType(context);
   auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
-  Value inputSize = shape::NumElementsOp::create(builder,
-      loc, builder.getIndexType(), inputShape);
+  Value inputSize = shape::NumElementsOp::create(
+      builder, loc, builder.getIndexType(), inputShape);
 
   // Turn input size into 1D tensor
   auto flatShapeType = shape::getExtentTensorType(context, 1);
@@ -109,7 +109,7 @@ std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
   auto flatInputType =
       RankedTensorType::get({ShapedType::kDynamic}, elementType);
   auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
-                                                     flatInputShape);
+                                             flatInputShape);
   return std::make_pair(flatInput, inputShape);
 }
 
@@ -165,8 +165,9 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   // Compute flat input shape as a 3-element 1D tensor
   auto axisSizeValue = arith::ConstantIndexOp::create(builder, loc, axisSize);
   auto flatShapeType = shape::getExtentTensorType(context, 3);
-  auto flatInputShape = tensor::FromElementsOp::create(builder,
-      loc, flatShapeType, ValueRange{sizeLeft, axisSizeValue, sizeRight});
+  auto flatInputShape = tensor::FromElementsOp::create(
+      builder, loc, flatShapeType,
+      ValueRange{sizeLeft, axisSizeValue, sizeRight});
 
   // Reshape input to 3D tensor
   auto inputType = cast<UnrankedTensorType>(input.getType());
@@ -174,7 +175,7 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto flatInputType = RankedTensorType::get(
       {ShapedType::kDynamic, axisSize, ShapedType::kDynamic}, elementType);
   auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
-                                                     flatInputShape);
+                                             flatInputShape);
 
   return std::make_pair(flatInput, inputShape);
 }
@@ -193,7 +194,7 @@ Value restoreUnrankedTensorShape(OpBuilder &builder, Location loc, Value input,
   auto elementType = inputType.getElementType();
   auto unrankedType = UnrankedTensorType::get(elementType);
   return tensor::ReshapeOp::create(builder, loc, unrankedType, input,
-                                           inputShape);
+                                   inputShape);
 }
 
 // Create a tensor constant containing all scales in a per-channel quantized
@@ -314,10 +315,10 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
   // Materialize bounds
   auto inputType = input.getType();
   auto storageType = quantizedType.getStorageType();
-  auto storageMinScalar = arith::ConstantIntOp::create(builder,
-      loc, storageType, quantizedType.getStorageTypeMin());
-  auto storageMaxScalar = arith::ConstantIntOp::create(builder,
-      loc, storageType, quantizedType.getStorageTypeMax());
+  auto storageMinScalar = arith::ConstantIntOp::create(
+      builder, loc, storageType, quantizedType.getStorageTypeMin());
+  auto storageMaxScalar = arith::ConstantIntOp::create(
+      builder, loc, storageType, quantizedType.getStorageTypeMax());
   auto storageMin = getScalarOrTensorConstant(builder, loc, storageMinScalar,
                                               inputType, inputShape);
   auto storageMax = getScalarOrTensorConstant(builder, loc, storageMaxScalar,
@@ -477,7 +478,8 @@ Value convertPerLayerRanked(OpBuilder &builder, Location loc, Operation *op,
   auto storageType = quantizedType.getStorageType();
   auto scaleAttr =
       builder.getFloatAttr(expressedType, quantizedType.getScale());
-  auto scale = arith::ConstantOp::create(builder, loc, expressedType, scaleAttr);
+  auto scale =
+      arith::ConstantOp::create(builder, loc, expressedType, scaleAttr);
   auto zeroPointAttr =
       builder.getIntegerAttr(storageType, quantizedType.getZeroPoint());
   auto zeroPoint =
@@ -729,8 +731,8 @@ struct DequantizeCastOpConversion
     // Convert quantized input to storage type
     auto storageScalarOrTensorType =
         getScalarOrTensorType(quantizedType.getStorageType(), input.getType());
-    input = quant::StorageCastOp::create(rewriter,
-        loc, storageScalarOrTensorType, input);
+    input = quant::StorageCastOp::create(rewriter, loc,
+                                         storageScalarOrTensorType, input);
 
     auto result = convertQuantized(rewriter, loc, op, input, quantizedType);
 
diff --git a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
index f5e0aef365ff7..e84cca5b0c1f6 100644
--- a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
@@ -47,7 +47,7 @@ class QuantizedTypeConverter : public TypeConverter {
   static Value materializeConversion(OpBuilder &builder, Type type,
                                      ValueRange inputs, Location loc) {
     return quant::StorageCastOp::create(builder, loc, type,
-                                                llvm::getSingleElement(inputs));
+                                        llvm::getSingleElement(inputs));
   }
 
 public:
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index 0214ce3090656..0325597c39a02 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -205,8 +205,8 @@ struct ExecuteRegionOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(executeRegionOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
-            executeRegionOp.getLoc(), it.value(),
+        newResults.push_back(bufferization::ToTensorOp::create(
+            rewriter, executeRegionOp.getLoc(), it.value(),
             newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
@@ -260,9 +260,9 @@ struct IfOpInterface
 
     // Create new op.
     rewriter.setInsertionPoint(ifOp);
-    auto newIfOp =
-        scf::IfOp::create(rewriter, ifOp.getLoc(), newTypes, ifOp.getCondition(),
-                                   /*withElseRegion=*/true);
+    auto newIfOp = scf::IfOp::create(rewriter, ifOp.getLoc(), newTypes,
+                                     ifOp.getCondition(),
+                                     /*withElseRegion=*/true);
 
     // Move over then/else blocks.
     rewriter.mergeBlocks(ifOp.thenBlock(), newIfOp.thenBlock());
@@ -374,9 +374,9 @@ struct IndexSwitchOpInterface
 
     // Create new op.
     rewriter.setInsertionPoint(switchOp);
-    auto newSwitchOp = scf::IndexSwitchOp::create(rewriter,
-        switchOp.getLoc(), newTypes, switchOp.getArg(), switchOp.getCases(),
-        switchOp.getCases().size());
+    auto newSwitchOp = scf::IndexSwitchOp::create(
+        rewriter, switchOp.getLoc(), newTypes, switchOp.getArg(),
+        switchOp.getCases(), switchOp.getCases().size());
 
     // Move over blocks.
     for (auto [src, dest] :
@@ -769,8 +769,8 @@ struct ForOpInterface
     }
 
     // Construct a new scf.for op with memref instead of tensor values.
-    auto newForOp = scf::ForOp::create(rewriter,
-        forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
+    auto newForOp = scf::ForOp::create(
+        rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), castedInitArgs);
     newForOp->setAttrs(forOp->getAttrs());
     Block *loopBody = newForOp.getBody();
@@ -1005,8 +1005,8 @@ struct WhileOpInterface
     // Construct a new scf.while op with memref instead of tensor values.
     ValueRange argsRangeBefore(castedInitArgs);
     TypeRange argsTypesBefore(argsRangeBefore);
-    auto newWhileOp = scf::WhileOp::create(rewriter,
-        whileOp.getLoc(), argsTypesAfter, castedInitArgs);
+    auto newWhileOp = scf::WhileOp::create(rewriter, whileOp.getLoc(),
+                                           argsTypesAfter, castedInitArgs);
 
     // Add before/after regions to the new op.
     SmallVector<Location> bbArgLocsBefore(castedInitArgs.size(),
@@ -1265,8 +1265,8 @@ struct ForallOpInterface
              forallOp.getBody()->getArguments().drop_front(rank), buffers)) {
       BlockArgument bbArg = std::get<0>(it);
       Value buffer = std::get<1>(it);
-      Value bufferAsTensor = ToTensorOp::create(rewriter,
-          forallOp.getLoc(), bbArg.getType(), buffer);
+      Value bufferAsTensor = ToTensorOp::create(rewriter, forallOp.getLoc(),
+                                                bbArg.getType(), buffer);
       bbArg.replaceAllUsesWith(bufferAsTensor);
     }
 
@@ -1274,8 +1274,8 @@ struct ForallOpInterface
     // introduced terminator.
     rewriter.setInsertionPoint(forallOp);
     ForallOp newForallOp;
-    newForallOp = ForallOp::create(rewriter,
-        forallOp.getLoc(), forallOp.getMixedLowerBound(),
+    newForallOp = ForallOp::create(
+        rewriter, forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
         /*outputs=*/ValueRange(), forallOp.getMapping());
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
index bf4eba3319a4d..bee77800e9d44 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
@@ -51,18 +51,18 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     initArgs.push_back(forOp.getLowerBound());
     llvm::append_range(initArgs, forOp.getInitArgs());
     auto whileOp = WhileOp::create(rewriter, forOp.getLoc(), lcvTypes, initArgs,
-                                            forOp->getAttrs());
+                                   forOp->getAttrs());
 
     // 'before' region contains the loop condition and forwarding of iteration
     // arguments to the 'after' region.
     auto *beforeBlock = rewriter.createBlock(
         &whileOp.getBefore(), whileOp.getBefore().begin(), lcvTypes, lcvLocs);
     rewriter.setInsertionPointToStart(whileOp.getBeforeBody());
-    auto cmpOp = arith::CmpIOp::create(rewriter,
-        whileOp.getLoc(), arith::CmpIPredicate::slt,
+    auto cmpOp = arith::CmpIOp::create(
+        rewriter, whileOp.getLoc(), arith::CmpIPredicate::slt,
         beforeBlock->getArgument(0), forOp.getUpperBound());
     scf::ConditionOp::create(rewriter, whileOp.getLoc(), cmpOp.getResult(),
-                                      beforeBlock->getArguments());
+                             beforeBlock->getArguments());
 
     // Inline for-loop body into an executeRegion operation in the "after"
     // region. The return type of the execRegionOp does not contain the
@@ -72,8 +72,9 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
 
     // Add induction variable incrementation
     rewriter.setInsertionPointToEnd(afterBlock);
-    auto ivIncOp = arith::AddIOp::create(rewriter,
-        whileOp.getLoc(), afterBlock->getArgument(0), forOp.getStep());
+    auto ivIncOp =
+        arith::AddIOp::create(rewriter, whileOp.getLoc(),
+                              afterBlock->getArgument(0), forOp.getStep());
 
     // Rewrite uses of the for-loop block arguments to the new while-loop
     // "after" arguments
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index f1ede64d01e0f..59828563a8196 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -279,25 +279,25 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
     if (dynamicLoop) {
       Type t = ub.getType();
       // pred = ub > lb + (i * step)
-      Value iv = arith::AddIOp::create(rewriter,
-          loc, lb,
-          arith::MulIOp::create(rewriter,
-              loc, step,
-              arith::ConstantOp::create(rewriter,
-                  loc, rewriter.getIntegerAttr(t, i))));
-      predicates[i] = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::slt, iv, ub);
+      Value iv = arith::AddIOp::create(
+          rewriter, loc, lb,
+          arith::MulIOp::create(
+              rewriter, loc, step,
+              arith::ConstantOp::create(rewriter, loc,
+                                        rewriter.getIntegerAttr(t, i))));
+      predicates[i] = arith::CmpIOp::create(rewriter, loc,
+                                            arith::CmpIPredicate::slt, iv, ub);
     }
 
     // special handling for induction variable as the increment is implicit.
     // iv = lb + i * step
     Type t = lb.getType();
-    Value iv = arith::AddIOp::create(rewriter,
-        loc, lb,
-        arith::MulIOp::create(rewriter,
-            loc, step,
+    Value iv = arith::AddIOp::create(
+        rewriter, loc, lb,
+        arith::MulIOp::create(
+            rewriter, loc, step,
             arith::ConstantOp::create(rewriter, loc,
-                                               rewriter.getIntegerAttr(t, i))));
+                                      rewriter.getIntegerAttr(t, i))));
     setValueMapping(forOp.getInductionVar(), iv, i);
     for (Operation *op : opOrder) {
       if (stages[op] > i)
@@ -332,8 +332,8 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
             Value prevValue = valueMapping
                 [forOp.getRegionIterArgs()[operand.getOperandNumber()]]
                 [i - stages[op]];
-            source = arith::SelectOp::create(rewriter,
-                loc, predicates[predicateIdx], source, prevValue);
+            source = arith::SelectOp::create(
+                rewriter, loc, predicates[predicateIdx], source, prevValue);
           }
           setValueMapping(forOp.getRegionIterArgs()[operand.getOperandNumber()],
                           source, i - stages[op] + 1);
@@ -444,15 +444,15 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop(
     Type t = ub.getType();
     Location loc = forOp.getLoc();
     // newUb = ub - maxStage * step
-    Value maxStageValue = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerAttr(t, maxStage));
+    Value maxStageValue = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getIntegerAttr(t, maxStage));
     Value maxStageByStep =
         arith::MulIOp::create(rewriter, loc, step, maxStageValue);
     newUb = arith::SubIOp::create(rewriter, loc, ub, maxStageByStep);
   }
   auto newForOp =
       scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(), newUb,
-                                  forOp.getStep(), newLoopArg);
+                         forOp.getStep(), newLoopArg);
   // When there are no iter args, the loop body terminator will be created.
   // Since we always create it below, remove the terminator if it was created.
   if (!newForOp.getBody()->empty())
@@ -483,16 +483,17 @@ LogicalResult LoopPipelinerInternal::createKernel(
     Type t = ub.getType();
     for (unsigned i = 0; i < maxStage; i++) {
       // c = ub - (maxStage - i) * step
-      Value c = arith::SubIOp::create(rewriter,
-          loc, ub,
-          arith::MulIOp::create(rewriter,
-              loc, step,
-              arith::ConstantOp::create(rewriter,
-                  loc, rewriter.getIntegerAttr(t, int64_t(maxStage - i)))));
-
-      Value pred = arith::CmpIOp::create(rewriter,
-          newForOp.getLoc(), arith::CmpIPredicate::slt,
-          newForOp.getInductionVar(), c);
+      Value c = arith::SubIOp::create(
+          rewriter, loc, ub,
+          arith::MulIOp::create(
+              rewriter, loc, step,
+              arith::ConstantOp::create(
+                  rewriter, loc,
+                  rewriter.getIntegerAttr(t, int64_t(maxStage - i)))));
+
+      Value pred = arith::CmpIOp::create(rewriter, newForOp.getLoc(),
+                                         arith::CmpIPredicate::slt,
+                                         newForOp.getInductionVar(), c);
       predicates[i] = pred;
     }
   }
@@ -515,13 +516,13 @@ LogicalResult LoopPipelinerInternal::createKernel(
 
         // offset = (maxStage - stages[op]) * step
         Type t = step.getType();
-        Value offset = arith::MulIOp::create(rewriter,
-            forOp.getLoc(), step,
-            arith::ConstantOp::create(rewriter,
-                forOp.getLoc(),
+        Value offset = arith::MulIOp::create(
+            rewriter, forOp.getLoc(), step,
+            arith::ConstantOp::create(
+                rewriter, forOp.getLoc(),
                 rewriter.getIntegerAttr(t, maxStage - stages[op])));
-        Value iv = arith::AddIOp::create(rewriter,
-            forOp.getLoc(), newForOp.getInductionVar(), offset);
+        Value iv = arith::AddIOp::create(rewriter, forOp.getLoc(),
+                                         newForOp.getInductionVar(), offset);
         nestedNewOp->setOperand(operand->getOperandNumber(), iv);
         rewriter.setInsertionPointAfter(newOp);
         continue;
@@ -594,8 +595,8 @@ LogicalResult LoopPipelinerInternal::createKernel(
         auto defStage = stages.find(def);
         if (defStage != stages.end() && defStage->second < maxStage) {
           Value pred = predicates[defStage->second];
-          source = arith::SelectOp::create(rewriter,
-              pred.getLoc(), pred, source,
+          source = arith::SelectOp::create(
+              rewriter, pred.getLoc(), pred, source,
               newForOp.getBody()
                   ->getArguments()[yieldOperand.getOperandNumber() + 1]);
         }
@@ -653,7 +654,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
 
   auto createConst = [&](int v) {
     return arith::ConstantOp::create(rewriter, loc,
-                                              rewriter.getIntegerAttr(t, v));
+                                     rewriter.getIntegerAttr(t, v));
   };
 
   // total_iterations = cdiv(range_diff, step);
@@ -661,22 +662,23 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // - total_iterations = (range_diff + step + (step < 0 ? 1 : -1)) / step
   Value zero = createConst(0);
   Value one = createConst(1);
-  Value stepLessZero = arith::CmpIOp::create(rewriter,
-      loc, arith::CmpIPredicate::slt, step, zero);
-  Value stepDecr =
-      arith::SelectOp::create(rewriter, loc, stepLessZero, one, createConst(-1));
+  Value stepLessZero = arith::CmpIOp::create(
+      rewriter, loc, arith::CmpIPredicate::slt, step, zero);
+  Value stepDecr = arith::SelectOp::create(rewriter, loc, stepLessZero, one,
+                                           createConst(-1));
 
   Value rangeDiff = arith::SubIOp::create(rewriter, loc, ub, lb);
   Value rangeIncrStep = arith::AddIOp::create(rewriter, loc, rangeDiff, step);
   Value rangeDecr =
       arith::AddIOp::create(rewriter, loc, rangeIncrStep, stepDecr);
-  Value totalIterations = arith::DivSIOp::create(rewriter, loc, rangeDecr, step);
+  Value totalIterations =
+      arith::DivSIOp::create(rewriter, loc, rangeDecr, step);
 
   // If total_iters < max_stage, start the epilogue at zero to match the
   // ramp-up in the prologue.
   // start_iter = max(0, total_iters - max_stage)
   Value iterI = arith::SubIOp::create(rewriter, loc, totalIterations,
-                                               createConst(maxStage));
+                                      createConst(maxStage));
   iterI = arith::MaxSIOp::create(rewriter, loc, zero, iterI);
 
   // Capture predicates for dynamic loops.
@@ -684,8 +686,8 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
 
   for (int64_t i = 1; i <= maxStage; i++) {
     // newLastIter = lb + step * iterI
-    Value newlastIter = arith::AddIOp::create(rewriter,
-        loc, lb, arith::MulIOp::create(rewriter, loc, step, iterI));
+    Value newlastIter = arith::AddIOp::create(
+        rewriter, loc, lb, arith::MulIOp::create(rewriter, loc, step, iterI));
 
     setValueMapping(forOp.getInductionVar(), newlastIter, i);
 
@@ -695,8 +697,9 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
     if (dynamicLoop) {
       // Disable stages when `i` is greater than total_iters.
       // pred = total_iters >= i
-      predicates[i] = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::sge, totalIterations, createConst(i));
+      predicates[i] =
+          arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sge,
+                                totalIterations, createConst(i));
     }
   }
 
@@ -758,8 +761,8 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
           unsigned nextVersion = currentVersion + 1;
           Value pred = predicates[currentVersion];
           Value prevValue = valueMapping[mapVal][currentVersion];
-          auto selOp = arith::SelectOp::create(rewriter, loc, pred, pair.value(),
-                                                        prevValue);
+          auto selOp = arith::SelectOp::create(rewriter, loc, pred,
+                                               pair.value(), prevValue);
           returnValues[ri] = selOp;
           if (nextVersion <= maxStage)
             setValueMapping(mapVal, selOp, nextVersion);
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index b24e276272c2c..15476d5e2bfc2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -66,7 +66,7 @@ static void specializeParallelLoopForUnrolling(ParallelOp op) {
     Value constant =
         arith::ConstantIndexOp::create(b, op.getLoc(), std::get<1>(bound));
     Value cmp = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
-                                        std::get<0>(bound), constant);
+                                      std::get<0>(bound), constant);
     cond = cond ? arith::AndIOp::create(b, op.getLoc(), cond, cmp) : cmp;
     map.map(std::get<0>(bound), constant);
   }
@@ -97,7 +97,7 @@ static void specializeForLoopForUnrolling(ForOp op) {
   IRMapping map;
   Value constant = arith::ConstantIndexOp::create(b, op.getLoc(), minConstant);
   Value cond = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
-                                       bound, constant);
+                                     bound, constant);
   map.map(bound, constant);
   auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index 013c3b69620e2..694cd85435f63 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -190,8 +190,8 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
 
   IRRewriter b(builder);
   b.setInsertionPoint(secondPloop);
-  auto newSecondPloop = ParallelOp::create(b,
-      secondPloop.getLoc(), secondPloop.getLowerBound(),
+  auto newSecondPloop = ParallelOp::create(
+      b, secondPloop.getLoc(), secondPloop.getLowerBound(),
       secondPloop.getUpperBound(), secondPloop.getStep(), newInitVars);
 
   Block *newBlock = newSecondPloop.getBody();
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
index 2ca3b1e366bdf..84a779b90f6c2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
@@ -76,10 +76,10 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
   newSteps.reserve(op.getStep().size());
   for (auto step : llvm::zip(op.getStep(), tileSizeConstants)) {
     newSteps.push_back(arith::MulIOp::create(b, op.getLoc(), std::get<0>(step),
-                                               std::get<1>(step)));
+                                             std::get<1>(step)));
   }
   auto outerLoop = ParallelOp::create(b, op.getLoc(), op.getLowerBound(),
-                                        op.getUpperBound(), newSteps);
+                                      op.getUpperBound(), newSteps);
   b.setInsertionPointToStart(outerLoop.getBody());
 
   // Compute min(size, dim - offset) to avoid out-of-bounds accesses.
@@ -131,10 +131,10 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // each iteration of the outer loop.
     newBounds.push_back(
         affine::AffineMinOp::create(b, op.getLoc(), b.getIndexType(), minMap,
-                                      ValueRange{newStep, upperBound, iv}));
+                                    ValueRange{newStep, upperBound, iv}));
   }
-  auto innerLoop = ParallelOp::create(b,
-      op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
+  auto innerLoop = ParallelOp::create(
+      b, op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
       op.getStep());
 
   if (noMinMaxBounds && needInboundCheck) {
@@ -147,16 +147,16 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
                    innerLoop.getInductionVars(), innerLoop.getStep())) {
       // %in_bound = %in_bound &&
       //             (%inner_iv * %inner_step + %outer_iv < %outer_upper_bound)
-      Value index = arith::AddIOp::create(b,
-          op.getLoc(), arith::MulIOp::create(b, op.getLoc(), innerIV, innerStep),
-          outerIV);
-      Value dimInbound = arith::CmpIOp::create(b,
-          op.getLoc(), arith::CmpIPredicate::ult, index, outerUpperBound);
+      Value index = arith::AddIOp::create(
+          b, op.getLoc(),
+          arith::MulIOp::create(b, op.getLoc(), innerIV, innerStep), outerIV);
+      Value dimInbound = arith::CmpIOp::create(
+          b, op.getLoc(), arith::CmpIPredicate::ult, index, outerUpperBound);
       inbound = arith::AndIOp::create(b, op.getLoc(), inbound, dimInbound);
     }
     auto ifInbound = IfOp::create(b, op.getLoc(),
-                                    /*resultTypes*/ ArrayRef<Type>{}, inbound,
-                                    /*hasElseRegion*/ false);
+                                  /*resultTypes*/ ArrayRef<Type>{}, inbound,
+                                  /*hasElseRegion*/ false);
     ifInbound.getThenRegion().takeBody(op.getRegion());
     Block &thenBlock = ifInbound.getThenRegion().front();
     // Replace the scf.reduce terminator with an scf.yield terminator.
@@ -167,8 +167,8 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     b.setInsertionPointToStart(innerLoop.getBody());
     for (const auto &ivs : llvm::enumerate(llvm::zip(
              innerLoop.getInductionVars(), outerLoop.getInductionVars()))) {
-      auto newIndex = arith::AddIOp::create(b,
-          op.getLoc(), std::get<0>(ivs.value()), std::get<1>(ivs.value()));
+      auto newIndex = arith::AddIOp::create(
+          b, op.getLoc(), std::get<0>(ivs.value()), std::get<1>(ivs.value()));
       thenBlock.getArgument(ivs.index())
           .replaceAllUsesExcept(newIndex, newIndex);
     }
@@ -180,7 +180,7 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
                               outerLoop.getInductionVars())) {
       Value innerIndex = std::get<0>(ivs);
       auto newIndex = arith::AddIOp::create(b, op.getLoc(), std::get<0>(ivs),
-                                              std::get<1>(ivs));
+                                            std::get<1>(ivs));
       innerIndex.replaceAllUsesExcept(newIndex, newIndex);
     }
   }
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 41895e46d953d..1b07b77546030 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -112,11 +112,11 @@ class ConvertForOpTypes
 
     // We can not do clone as the number of result types after conversion
     // might be different.
-    ForOp newOp = ForOp::create(rewriter,
-        op.getLoc(), llvm::getSingleElement(adaptor.getLowerBound()),
-        llvm::getSingleElement(adaptor.getUpperBound()),
-        llvm::getSingleElement(adaptor.getStep()),
-        flattenValues(adaptor.getInitArgs()));
+    ForOp newOp = ForOp::create(rewriter, op.getLoc(),
+                                llvm::getSingleElement(adaptor.getLowerBound()),
+                                llvm::getSingleElement(adaptor.getUpperBound()),
+                                llvm::getSingleElement(adaptor.getStep()),
+                                flattenValues(adaptor.getInitArgs()));
 
     // Reserve whatever attributes in the original op.
     newOp->setAttrs(op->getAttrs());
@@ -142,9 +142,9 @@ class ConvertIfOpTypes
                                       ConversionPatternRewriter &rewriter,
                                       TypeRange dstTypes) const {
 
-    IfOp newOp = IfOp::create(rewriter,
-        op.getLoc(), dstTypes, llvm::getSingleElement(adaptor.getCondition()),
-        true);
+    IfOp newOp =
+        IfOp::create(rewriter, op.getLoc(), dstTypes,
+                     llvm::getSingleElement(adaptor.getCondition()), true);
     newOp->setAttrs(op->getAttrs());
 
     // We do not need the empty blocks created by rewriter.
@@ -172,7 +172,7 @@ class ConvertWhileOpTypes
                                          ConversionPatternRewriter &rewriter,
                                          TypeRange dstTypes) const {
     auto newOp = WhileOp::create(rewriter, op.getLoc(), dstTypes,
-                                          flattenValues(adaptor.getOperands()));
+                                 flattenValues(adaptor.getOperands()));
 
     for (auto i : {0u, 1u}) {
       if (failed(rewriter.convertRegionTypes(&op.getRegion(i), *typeConverter)))
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index 3171b96d46b8a..6a9e02d844e94 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -451,8 +451,8 @@ static LogicalResult generateLoopNestUsingForOp(
   for (auto [lb, ub, step] : llvm::zip_equal(lbVals, ubVals, stepVals)) {
     auto loop =
         scf::ForOp::create(rewriter, loc, lb, ub, step, destinationTensors,
-                                    [](OpBuilder &bodyBuilder, Location bodyLoc,
-                                       Value iv, ValueRange /*iterArgs*/) {});
+                           [](OpBuilder &bodyBuilder, Location bodyLoc,
+                              Value iv, ValueRange /*iterArgs*/) {});
     loops.push_back(loop);
     ivs.push_back(loop.getInductionVar());
     rewriter.setInsertionPointToEnd(loop.getBody());
@@ -479,8 +479,8 @@ static LogicalResult generateLoopNestUsingForOp(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    auto insertSlice = tensor::InsertSliceOp::create(rewriter,
-        loc, tiledValue, destinationTensor, resultOffset, resultSize,
+    auto insertSlice = tensor::InsertSliceOp::create(
+        rewriter, loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
     yieldedValues.push_back(insertSlice);
   }
@@ -534,13 +534,13 @@ static LogicalResult generateLoopNestUsingForallOp(
       nonZeroNumThreads.push_back(nt);
     }
     forallOp = scf::ForallOp::create(rewriter, loc, nonZeroNumThreads,
-                                              destinationTensors, mappingAttr);
+                                     destinationTensors, mappingAttr);
   } else {
     SmallVector<OpFoldResult> lbs, ubs, steps;
     std::tie(lbs, ubs, steps) =
         getLoopBounds(rewriter, loc, loopRanges, tileSizes);
     forallOp = scf::ForallOp::create(rewriter, loc, lbs, ubs, steps,
-                                              destinationTensors, mappingAttr);
+                                     destinationTensors, mappingAttr);
   }
   loops.push_back(forallOp);
 
@@ -561,9 +561,9 @@ static LogicalResult generateLoopNestUsingForallOp(
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
 
-    tensor::ParallelInsertSliceOp::create(rewriter,
-        loc, tiledValue, destinationTensor, resultOffset, resultSize,
-        resultStride);
+    tensor::ParallelInsertSliceOp::create(rewriter, loc, tiledValue,
+                                          destinationTensor, resultOffset,
+                                          resultSize, resultStride);
   }
   return success();
 }
@@ -798,9 +798,9 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
 
   auto inits = llvm::to_vector(loopOp.getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = scf::ForOp::create(rewriter,
-      loc, loopOp.getLowerBound(), loopOp.getUpperBound(), loopOp.getStep(),
-      inits, [](OpBuilder &, Location, Value, ValueRange) {});
+  auto newLoop = scf::ForOp::create(
+      rewriter, loc, loopOp.getLowerBound(), loopOp.getUpperBound(),
+      loopOp.getStep(), inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
   // Move the loop body to the new op.
   Block *loopBody = loopOp.getBody();
@@ -829,9 +829,9 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    Value insert = tensor::InsertSliceOp::create(rewriter,
-        yieldOp->getLoc(), tiledValue, regionIterArg, resultOffset, resultSize,
-        resultStride);
+    Value insert = tensor::InsertSliceOp::create(
+        rewriter, yieldOp->getLoc(), tiledValue, regionIterArg, resultOffset,
+        resultSize, resultStride);
     newYieldValues.push_back(insert);
   }
 
@@ -851,8 +851,8 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
   rewriter.setInsertionPoint(loopOp);
   auto inits = llvm::to_vector(loopOp.getOutputs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = scf::ForallOp::create(rewriter,
-      loc, loopOp.getMixedLowerBound(), loopOp.getMixedUpperBound(),
+  auto newLoop = scf::ForallOp::create(
+      rewriter, loc, loopOp.getMixedLowerBound(), loopOp.getMixedUpperBound(),
       loopOp.getMixedStep(), inits, loopOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
 
@@ -884,9 +884,9 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
            tiledValues, regionIterArgs, resultOffsets, resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    tensor::ParallelInsertSliceOp::create(rewriter,
-        terminator.getLoc(), tiledValue, iterArg, resultOffset, resultSize,
-        resultStride);
+    tensor::ParallelInsertSliceOp::create(rewriter, terminator.getLoc(),
+                                          tiledValue, iterArg, resultOffset,
+                                          resultSize, resultStride);
   }
 
   rewriter.replaceOp(loopOp,
@@ -935,9 +935,9 @@ static LogicalResult addInitOperandsToLoopNest(
     // Create a new loop with the new init values for this loop.
     SmallVector<Value> newInits = llvm::to_vector(forLoop.getInitArgs());
     newInits.append(newInitValues.begin(), newInitValues.end());
-    auto newLoop = scf::ForOp::create(rewriter,
-        forLoop.getLoc(), forLoop.getLowerBound(), forLoop.getUpperBound(),
-        forLoop.getStep(), newInits,
+    auto newLoop = scf::ForOp::create(
+        rewriter, forLoop.getLoc(), forLoop.getLowerBound(),
+        forLoop.getUpperBound(), forLoop.getStep(), newInits,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {});
 
     // Merge the body of the new loop with the body of the old loops.
@@ -1419,8 +1419,8 @@ FailureOr<SmallVector<Operation *>> mlir::scf::yieldReplacementForFusedProducer(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
-            loc, newRegionArg, offsetList[index], sizesList[index],
+        auto destSlice = tensor::ExtractSliceOp::create(
+            rewriter, loc, newRegionArg, offsetList[index], sizesList[index],
             SmallVector<OpFoldResult>(offsetList[index].size(),
                                       rewriter.getIndexAttr(1)));
         generatedSlices.push_back(destSlice);
@@ -2092,8 +2092,8 @@ cloneAsInsertSlice<tensor::InsertSliceOp>(RewriterBase &rewriter,
 template <>
 tensor::InsertSliceOp cloneAsInsertSlice<tensor::ParallelInsertSliceOp>(
     RewriterBase &rewriter, tensor::ParallelInsertSliceOp insertSliceOp) {
-  return tensor::InsertSliceOp::create(rewriter,
-      insertSliceOp->getLoc(), insertSliceOp.getSource(),
+  return tensor::InsertSliceOp::create(
+      rewriter, insertSliceOp->getLoc(), insertSliceOp.getSource(),
       insertSliceOp.getDest(), insertSliceOp.getMixedOffsets(),
       insertSliceOp.getMixedSizes(), insertSliceOp.getMixedStrides());
 }
@@ -2314,8 +2314,9 @@ mlir::scf::tileAndFuseConsumerOfSlices(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
-            loc, newRegionArg, resultOffsets[index], resultSizes[index],
+        auto destSlice = tensor::ExtractSliceOp::create(
+            rewriter, loc, newRegionArg, resultOffsets[index],
+            resultSizes[index],
             SmallVector<OpFoldResult>(resultOffsets[index].size(),
                                       rewriter.getIndexAttr(1)));
         // Make a copy of index to avoid a capturing structured binding, which
@@ -2392,7 +2393,7 @@ mlir::scf::lowerToLoopsUsingSCFForOp(RewriterBase &rewriter,
     Value strideVal =
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.stride);
     auto loop = scf::ForOp::create(rewriter, op.getLoc(), offsetVal, sizeVal,
-                                            strideVal, ValueRange{});
+                                   strideVal, ValueRange{});
     loops.push_back(loop);
     ivs.push_back(loop.getInductionVar());
     rewriter.setInsertionPoint(loop.getBody()->getTerminator());
diff --git a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
index 707ed836cf63e..db504fe157dc9 100644
--- a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
@@ -96,8 +96,8 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       condOp.getArgs(), [&](Value arg) { return mapper.lookupOrDefault(arg); });
 
   // Create rotated while loop.
-  auto newLoopOp = scf::WhileOp::create(rewriter,
-      whileOp.getLoc(), whileOp.getResultTypes(), clonedCondArgs,
+  auto newLoopOp = scf::WhileOp::create(
+      rewriter, whileOp.getLoc(), whileOp.getResultTypes(), clonedCondArgs,
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Rotate and move the loop body into before block.
         auto newBlock = builder.getBlock();
@@ -113,8 +113,8 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       });
 
   // Create zero-trip-check and move the while loop in.
-  auto ifOp = scf::IfOp::create(rewriter,
-      whileOp.getLoc(), clonedCondition,
+  auto ifOp = scf::IfOp::create(
+      rewriter, whileOp.getLoc(), clonedCondition,
       [&](OpBuilder &builder, Location loc) {
         // Then runs the while loop.
         rewriter.moveOpBefore(newLoopOp, builder.getInsertionBlock(),
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index 6a387ddf8908a..2b7459b7addec 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -169,7 +169,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
     // Explicitly set up a new ReturnOp terminator.
     rewriter.setInsertionPointToEnd(outlinedFuncBody);
     func::ReturnOp::create(rewriter, loc, originalTerminator->getResultTypes(),
-                                    originalTerminator->getOperands());
+                           originalTerminator->getOperands());
   }
 
   // Reconstruct the block that was deleted and add a
@@ -274,10 +274,10 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
 
-  Value divisorMinusOneCst = arith::ConstantOp::create(builder,
-      loc, builder.getIntegerAttr(dividend.getType(), divisor - 1));
-  Value divisorCst = arith::ConstantOp::create(builder,
-      loc, builder.getIntegerAttr(dividend.getType(), divisor));
+  Value divisorMinusOneCst = arith::ConstantOp::create(
+      builder, loc, builder.getIntegerAttr(dividend.getType(), divisor - 1));
+  Value divisorCst = arith::ConstantOp::create(
+      builder, loc, builder.getIntegerAttr(dividend.getType(), divisor));
   Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOneCst);
   return arith::DivUIOp::create(builder, loc, sum, divisorCst);
 }
@@ -290,8 +290,8 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
                              Value divisor) {
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
-  Value cstOne = arith::ConstantOp::create(builder,
-      loc, builder.getOneAttr(dividend.getType()));
+  Value cstOne = arith::ConstantOp::create(
+      builder, loc, builder.getOneAttr(dividend.getType()));
   Value divisorMinusOne = arith::SubIOp::create(builder, loc, divisor, cstOne);
   Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOne);
   return arith::DivUIOp::create(builder, loc, sum, divisor);
@@ -404,18 +404,20 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'upperBoundUnrolled' and set epilogue loop flag.
     generateEpilogueLoop = upperBoundUnrolledCst < ubCst;
     if (generateEpilogueLoop)
-      upperBoundUnrolled = arith::ConstantOp::create(boundsBuilder,
-          loc, boundsBuilder.getIntegerAttr(forOp.getUpperBound().getType(),
-                                            upperBoundUnrolledCst));
+      upperBoundUnrolled = arith::ConstantOp::create(
+          boundsBuilder, loc,
+          boundsBuilder.getIntegerAttr(forOp.getUpperBound().getType(),
+                                       upperBoundUnrolledCst));
     else
       upperBoundUnrolled = forOp.getUpperBound();
 
     // Create constant for 'stepUnrolled'.
-    stepUnrolled = stepCst == stepUnrolledCst
-                       ? step
-                       : arith::ConstantOp::create(boundsBuilder,
-                             loc, boundsBuilder.getIntegerAttr(
-                                      step.getType(), stepUnrolledCst));
+    stepUnrolled =
+        stepCst == stepUnrolledCst
+            ? step
+            : arith::ConstantOp::create(boundsBuilder, loc,
+                                        boundsBuilder.getIntegerAttr(
+                                            step.getType(), stepUnrolledCst));
   } else {
     // Dynamic loop bounds computation.
     // TODO: Add dynamic asserts for negative lb/ub/step, or
@@ -425,16 +427,17 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     Value diff =
         arith::SubIOp::create(boundsBuilder, loc, upperBound, lowerBound);
     Value tripCount = ceilDivPositive(boundsBuilder, loc, diff, step);
-    Value unrollFactorCst = arith::ConstantOp::create(boundsBuilder,
-        loc, boundsBuilder.getIntegerAttr(tripCount.getType(), unrollFactor));
+    Value unrollFactorCst = arith::ConstantOp::create(
+        boundsBuilder, loc,
+        boundsBuilder.getIntegerAttr(tripCount.getType(), unrollFactor));
     Value tripCountRem =
         arith::RemSIOp::create(boundsBuilder, loc, tripCount, unrollFactorCst);
     // Compute tripCountEvenMultiple = tripCount - (tripCount % unrollFactor)
     Value tripCountEvenMultiple =
         arith::SubIOp::create(boundsBuilder, loc, tripCount, tripCountRem);
     // Compute upperBoundUnrolled = lowerBound + tripCountEvenMultiple * step
-    upperBoundUnrolled = arith::AddIOp::create(boundsBuilder,
-        loc, lowerBound,
+    upperBoundUnrolled = arith::AddIOp::create(
+        boundsBuilder, loc, lowerBound,
         arith::MulIOp::create(boundsBuilder, loc, tripCountEvenMultiple, step));
     // Scale 'step' by 'unrollFactor'.
     stepUnrolled =
@@ -474,10 +477,10 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
       forOp.getBody(), forOp.getInductionVar(), unrollFactor,
       [&](unsigned i, Value iv, OpBuilder b) {
         // iv' = iv + step * i;
-        auto stride = arith::MulIOp::create(b,
-            loc, step,
+        auto stride = arith::MulIOp::create(
+            b, loc, step,
             arith::ConstantOp::create(b, loc,
-                                        b.getIntegerAttr(iv.getType(), i)));
+                                      b.getIntegerAttr(iv.getType(), i)));
         return arith::AddIOp::create(b, loc, iv, stride);
       },
       annotateFn, iterArgs, yieldedValues);
@@ -823,8 +826,8 @@ static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc,
     if (vOne && vOne.value() == 1)
       continue;
     if (productOf)
-      productOf =
-          arith::MulIOp::create(rewriter, loc, productOf.value(), v).getResult();
+      productOf = arith::MulIOp::create(rewriter, loc, productOf.value(), v)
+                      .getResult();
     else
       productOf = v;
   }
@@ -850,9 +853,8 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
                              Value linearizedIv, ArrayRef<Value> ubs) {
 
   if (linearizedIv.getType().isIndex()) {
-    Operation *delinearizedOp =
-        affine::AffineDelinearizeIndexOp::create(rewriter, loc, linearizedIv,
-                                                          ubs);
+    Operation *delinearizedOp = affine::AffineDelinearizeIndexOp::create(
+        rewriter, loc, linearizedIv, ubs);
     auto resultVals = llvm::map_to_vector(
         delinearizedOp->getResults(), [](OpResult r) -> Value { return r; });
     return {resultVals, SmallPtrSet<Operation *, 2>{delinearizedOp}};
@@ -874,8 +876,8 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
     if (!isUbOne.test(index)) {
       break;
     }
-    delinearizedIvs[index] = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(ub.getType()));
+    delinearizedIvs[index] = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getZeroAttr(ub.getType()));
     numLeadingOneUbs++;
   }
 
@@ -892,8 +894,8 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
         iv = arith::RemSIOp::create(rewriter, loc, previous, ubs[idx]);
         preservedUsers.insert(iv.getDefiningOp());
       } else {
-        iv = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getZeroAttr(ubs[idx].getType()));
+        iv = arith::ConstantOp::create(
+            rewriter, loc, rewriter.getZeroAttr(ubs[idx].getType()));
       }
     }
     delinearizedIvs[idx] = iv;
@@ -1098,8 +1100,8 @@ void mlir::collapseParallelLoops(
   for (auto &sortedDimension : sortedDimensions) {
     Value newUpperBound = arith::ConstantIndexOp::create(rewriter, loc, 1);
     for (auto idx : sortedDimension) {
-      newUpperBound = arith::MulIOp::create(rewriter,
-          loc, newUpperBound, normalizedUpperBounds[idx]);
+      newUpperBound = arith::MulIOp::create(rewriter, loc, newUpperBound,
+                                            normalizedUpperBounds[idx]);
     }
     lowerBounds.push_back(cst0);
     steps.push_back(cst1);
@@ -1112,8 +1114,8 @@ void mlir::collapseParallelLoops(
   // value. The remainders then determine based on that range, which iteration
   // of the original induction value this represents. This is a normalized value
   // that is un-normalized already by the previous logic.
-  auto newPloop = scf::ParallelOp::create(rewriter,
-      loc, lowerBounds, upperBounds, steps,
+  auto newPloop = scf::ParallelOp::create(
+      rewriter, loc, lowerBounds, upperBounds, steps,
       [&](OpBuilder &insideBuilder, Location, ValueRange ploopIVs) {
         for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) {
           Value previous = ploopIVs[i];
@@ -1123,15 +1125,15 @@ void mlir::collapseParallelLoops(
             unsigned idx = combinedDimensions[i][j];
 
             // Determine the current induction value's current loop iteration
-            Value iv = arith::RemSIOp::create(insideBuilder,
-                loc, previous, normalizedUpperBounds[idx]);
+            Value iv = arith::RemSIOp::create(insideBuilder, loc, previous,
+                                              normalizedUpperBounds[idx]);
             replaceAllUsesInRegionWith(loops.getBody()->getArgument(idx), iv,
                                        loops.getRegion());
 
             // Remove the effect of the current induction value to prepare for
             // the next value.
-            previous = arith::DivSIOp::create(insideBuilder,
-                loc, previous, normalizedUpperBounds[idx]);
+            previous = arith::DivSIOp::create(insideBuilder, loc, previous,
+                                              normalizedUpperBounds[idx]);
           }
 
           // The final induction value is just the remaining value.
@@ -1344,7 +1346,7 @@ TileLoops mlir::extractFixedOuterLoops(scf::ForOp rootForOp,
     OpBuilder builder(forOp);
     auto loc = forOp.getLoc();
     Value diff = arith::SubIOp::create(builder, loc, forOp.getUpperBound(),
-                                               forOp.getLowerBound());
+                                       forOp.getLowerBound());
     Value numIterations = ceilDivPositive(builder, loc, diff, forOp.getStep());
     Value iterationsPerBlock =
         ceilDivPositive(builder, loc, numIterations, sizes[i]);
@@ -1376,9 +1378,10 @@ scf::ForallOp mlir::fuseIndependentSiblingForallLoops(scf::ForallOp target,
 
   // Create a new scf.forall op after the source loop.
   rewriter.setInsertionPointAfter(source);
-  scf::ForallOp fusedLoop = scf::ForallOp::create(rewriter,
-      source.getLoc(), source.getMixedLowerBound(), source.getMixedUpperBound(),
-      source.getMixedStep(), fusedOuts, source.getMapping());
+  scf::ForallOp fusedLoop = scf::ForallOp::create(
+      rewriter, source.getLoc(), source.getMixedLowerBound(),
+      source.getMixedUpperBound(), source.getMixedStep(), fusedOuts,
+      source.getMapping());
 
   // Map control operands.
   IRMapping mapping;
@@ -1429,8 +1432,8 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   // Create a new scf.for op after the source loop (with scf.yield terminator
   // (without arguments) only in case its init_args is empty).
   rewriter.setInsertionPointAfter(source);
-  scf::ForOp fusedLoop = scf::ForOp::create(rewriter,
-      source.getLoc(), source.getLowerBound(), source.getUpperBound(),
+  scf::ForOp fusedLoop = scf::ForOp::create(
+      rewriter, source.getLoc(), source.getLowerBound(), source.getUpperBound(),
       source.getStep(), fusedInitArgs);
 
   // Map original induction variables and operands to those of the fused loop.
@@ -1487,8 +1490,8 @@ FailureOr<scf::ForallOp> mlir::normalizeForallOp(RewriterBase &rewriter,
 
   // Use the normalized builder since the lower bounds are always 0 and the
   // steps are always 1.
-  auto normalizedForallOp = scf::ForallOp::create(rewriter,
-      loc, newUbs, forallOp.getOutputs(), forallOp.getMapping(),
+  auto normalizedForallOp = scf::ForallOp::create(
+      rewriter, loc, newUbs, forallOp.getOutputs(), forallOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
 
   rewriter.inlineRegionBefore(forallOp.getBodyRegion(),
diff --git a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
index cacc52d0feaf0..890406df74e72 100644
--- a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
@@ -569,10 +569,10 @@ SelectionOp::createIfThen(Location loc, Value condition,
   {
     OpBuilder::InsertionGuard guard(builder);
     builder.createBlock(thenBlock);
-    spirv::BranchConditionalOp::create(builder,
-        loc, condition, thenBlock,
-        /*trueArguments=*/ArrayRef<Value>(), mergeBlock,
-        /*falseArguments=*/ArrayRef<Value>());
+    spirv::BranchConditionalOp::create(builder, loc, condition, thenBlock,
+                                       /*trueArguments=*/ArrayRef<Value>(),
+                                       mergeBlock,
+                                       /*falseArguments=*/ArrayRef<Value>());
   }
 
   return selectionOp;
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index 53c07fc88e2ab..2bde44baf961e 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -1309,11 +1309,11 @@ struct ConvertSelectionOpToSelect final : OpRewritePattern<spirv::SelectionOp> {
     auto storeOpAttributes =
         cast<spirv::StoreOp>(trueBlock->front())->getAttrs();
 
-    auto selectOp = spirv::SelectOp::create(rewriter,
-        selectionOp.getLoc(), trueValue.getType(),
+    auto selectOp = spirv::SelectOp::create(
+        rewriter, selectionOp.getLoc(), trueValue.getType(),
         brConditionalOp.getCondition(), trueValue, falseValue);
     spirv::StoreOp::create(rewriter, selectOp.getLoc(), ptrValue,
-                                    selectOp.getResult(), storeOpAttributes);
+                           selectOp.getResult(), storeOpAttributes);
 
     // `spirv.mlir.selection` is not needed anymore.
     rewriter.eraseOp(op);
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 945f95b0f1dc6..5d011b237c0ab 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -652,25 +652,25 @@ spirv::ConstantOp spirv::ConstantOp::getZero(Type type, Location loc,
     unsigned width = intType.getWidth();
     if (width == 1)
       return spirv::ConstantOp::create(builder, loc, type,
-                                               builder.getBoolAttr(false));
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
+                                       builder.getBoolAttr(false));
+    return spirv::ConstantOp::create(
+        builder, loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getFloatAttr(floatType, 0.0));
+    return spirv::ConstantOp::create(builder, loc, type,
+                                     builder.getFloatAttr(floatType, 0.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return spirv::ConstantOp::create(builder,
-          loc, type,
+      return spirv::ConstantOp::create(
+          builder, loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 0).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return spirv::ConstantOp::create(builder,
-          loc, type,
+      return spirv::ConstantOp::create(
+          builder, loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 0.0).getValue()));
     }
@@ -685,25 +685,25 @@ spirv::ConstantOp spirv::ConstantOp::getOne(Type type, Location loc,
     unsigned width = intType.getWidth();
     if (width == 1)
       return spirv::ConstantOp::create(builder, loc, type,
-                                               builder.getBoolAttr(true));
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
+                                       builder.getBoolAttr(true));
+    return spirv::ConstantOp::create(
+        builder, loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getFloatAttr(floatType, 1.0));
+    return spirv::ConstantOp::create(builder, loc, type,
+                                     builder.getFloatAttr(floatType, 1.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return spirv::ConstantOp::create(builder,
-          loc, type,
+      return spirv::ConstantOp::create(
+          builder, loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 1).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return spirv::ConstantOp::create(builder,
-          loc, type,
+      return spirv::ConstantOp::create(
+          builder, loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 1.0).getValue()));
     }
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 0795678a9c66b..2b9c7296830dc 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -105,8 +105,9 @@ OwningOpRef<spirv::ModuleOp> combine(ArrayRef<spirv::ModuleOp> inputModules,
     }
   }
 
-  auto combinedModule = spirv::ModuleOp::create(combinedModuleBuilder,
-      firstModule.getLoc(), addressingModel, memoryModel, vceTriple);
+  auto combinedModule =
+      spirv::ModuleOp::create(combinedModuleBuilder, firstModule.getLoc(),
+                              addressingModel, memoryModel, vceTriple);
   combinedModuleBuilder.setInsertionPointToStart(combinedModule.getBody());
 
   // In some cases, a symbol in the (current state of the) combined module is
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 574f9a7c24b12..81365b44a3aad 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -70,9 +70,9 @@ createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
   varType =
       spirv::PointerType::get(varPointeeType, varPtrType.getStorageClass());
 
-  return spirv::GlobalVariableOp::create(builder,
-      funcOp.getLoc(), varType, varName, abiInfo.getDescriptorSet(),
-      abiInfo.getBinding());
+  return spirv::GlobalVariableOp::create(builder, funcOp.getLoc(), varType,
+                                         varName, abiInfo.getDescriptorSet(),
+                                         abiInfo.getBinding());
 }
 
 /// Gets the global variables that need to be specified as interface variable
@@ -147,7 +147,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
                              "execution model based on 'spirv.target_env'");
 
   spirv::EntryPointOp::create(builder, funcOp.getLoc(), *executionModel, funcOp,
-                                      interfaceVars);
+                              interfaceVars);
 
   // Specifies the spirv.ExecutionModeOp.
   if (DenseI32ArrayAttr workgroupSizeAttr = entryPointAttr.getWorkgroupSize()) {
@@ -155,8 +155,8 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
         spirv::getCapabilities(spirv::ExecutionMode::LocalSize);
     if (!caps || targetEnv.allows(*caps)) {
       spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
-                                             spirv::ExecutionMode::LocalSize,
-                                             workgroupSizeAttr.asArrayRef());
+                                     spirv::ExecutionMode::LocalSize,
+                                     workgroupSizeAttr.asArrayRef());
       // Erase workgroup size.
       entryPointAttr = spirv::EntryPointABIAttr::get(
           entryPointAttr.getContext(), DenseI32ArrayAttr(),
@@ -168,8 +168,8 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
         spirv::getCapabilities(spirv::ExecutionMode::SubgroupSize);
     if (!caps || targetEnv.allows(*caps)) {
       spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
-                                             spirv::ExecutionMode::SubgroupSize,
-                                             *subgroupSize);
+                                     spirv::ExecutionMode::SubgroupSize,
+                                     *subgroupSize);
       // Erase subgroup size.
       entryPointAttr = spirv::EntryPointABIAttr::get(
           entryPointAttr.getContext(), entryPointAttr.getWorkgroupSize(),
@@ -180,8 +180,8 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SignedZeroInfNanPreserve);
     if (!caps || targetEnv.allows(*caps)) {
-      spirv::ExecutionModeOp::create(builder,
-          funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(
+          builder, funcOp.getLoc(), funcOp,
           spirv::ExecutionMode::SignedZeroInfNanPreserve, *targetWidth);
       // Erase target width.
       entryPointAttr = spirv::EntryPointABIAttr::get(
@@ -269,8 +269,8 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     if (cast<spirv::SPIRVType>(argType.value()).isScalarOrVector()) {
       auto zero =
           spirv::ConstantOp::getZero(indexType, funcOp.getLoc(), rewriter);
-      auto loadPtr = spirv::AccessChainOp::create(rewriter,
-          funcOp.getLoc(), replacement, zero.getConstant());
+      auto loadPtr = spirv::AccessChainOp::create(
+          rewriter, funcOp.getLoc(), replacement, zero.getConstant());
       replacement = spirv::LoadOp::create(rewriter, funcOp.getLoc(), loadPtr);
     }
     signatureConverter.remapInput(argType.index(), replacement);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
index 8bb34de8988b8..38ef547f0769f 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
@@ -65,8 +65,8 @@ void RewriteInsertsPass::runOnOperation() {
       operands.push_back(insertionOp.getObject());
 
     OpBuilder builder(lastCompositeInsertOp);
-    auto compositeConstructOp = spirv::CompositeConstructOp::create(builder,
-        location, compositeType, operands);
+    auto compositeConstructOp = spirv::CompositeConstructOp::create(
+        builder, location, compositeType, operands);
 
     lastCompositeInsertOp.replaceAllUsesWith(
         compositeConstructOp->getResult(0));
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index d1fa4bbc208d7..35ec0190b5a61 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -669,21 +669,24 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
                               Location loc) {
   // We can only cast one value in SPIR-V.
   if (inputs.size() != 1) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   Value input = inputs.front();
 
   // Only support integer types for now. Floating point types to be implemented.
   if (!isa<IntegerType>(type)) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   auto inputType = cast<IntegerType>(input.getType());
 
   auto scalarType = dyn_cast<spirv::ScalarType>(type);
   if (!scalarType) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -691,7 +694,8 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // truncating to go back so we don't need to worry about the signedness.
   // For extension, we cannot have enough signal here to decide which op to use.
   if (inputType.getIntOrFloatBitWidth() < scalarType.getIntOrFloatBitWidth()) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -708,7 +712,8 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   scalarType.getCapabilities(caps);
   if (failed(checkCapabilityRequirements(type, targetEnv, caps)) ||
       failed(checkExtensionRequirements(type, targetEnv, exts))) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -843,7 +848,7 @@ getOrInsertPushConstantVariable(Location loc, Block &block,
   auto type = getPushConstantStorageType(elementCount, builder, indexType);
   const char *name = "__push_constant_var__";
   return spirv::GlobalVariableOp::create(builder, loc, type, name,
-                                                 /*initializer=*/nullptr);
+                                         /*initializer=*/nullptr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -879,8 +884,8 @@ struct FuncOpConversion final : OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted spirv.func op.
-    auto newFuncOp = spirv::FuncOp::create(rewriter,
-        funcOp.getLoc(), funcOp.getName(),
+    auto newFuncOp = spirv::FuncOp::create(
+        rewriter, funcOp.getLoc(), funcOp.getName(),
         rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
                                  resultType ? TypeRange(resultType)
                                             : TypeRange()));
@@ -920,7 +925,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
 
     // Create a new func op with the original type and copy the function body.
     auto newFuncOp = func::FuncOp::create(rewriter, funcOp.getLoc(),
-                                                   funcOp.getName(), fnType);
+                                          funcOp.getName(), fnType);
     rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
                                 newFuncOp.end());
 
@@ -954,8 +959,8 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto origVecType = dyn_cast<VectorType>(origType);
       if (!origVecType) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = arith::ConstantOp::create(rewriter,
-            loc, origType, rewriter.getZeroAttr(origType));
+        Value result = arith::ConstantOp::create(
+            rewriter, loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
         oneToNTypeMapping.addInputs(origInputNo, origType);
@@ -967,8 +972,8 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto targetShape = getTargetShape(origVecType);
       if (!targetShape) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = arith::ConstantOp::create(rewriter,
-            loc, origType, rewriter.getZeroAttr(origType));
+        Value result = arith::ConstantOp::create(
+            rewriter, loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
         oneToNTypeMapping.addInputs(origInputNo, origType);
@@ -982,12 +987,12 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
           llvm::to_vector_of<int64_t, 4>(origVecType.getShape());
 
       // Prepare the result vector.
-      Value result = arith::ConstantOp::create(rewriter,
-          loc, origVecType, rewriter.getZeroAttr(origVecType));
+      Value result = arith::ConstantOp::create(
+          rewriter, loc, origVecType, rewriter.getZeroAttr(origVecType));
       ++newOpCount;
       // Prepare the placeholder for the new arguments that will be added later.
-      Value dummy = arith::ConstantOp::create(rewriter,
-          loc, unrolledType, rewriter.getZeroAttr(unrolledType));
+      Value dummy = arith::ConstantOp::create(
+          rewriter, loc, unrolledType, rewriter.getZeroAttr(unrolledType));
       ++newOpCount;
 
       // Create the `vector.insert_strided_slice` ops.
@@ -995,8 +1000,8 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       SmallVector<Type> newTypes;
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        result = vector::InsertStridedSliceOp::create(rewriter,
-            loc, dummy, result, offsets, strides);
+        result = vector::InsertStridedSliceOp::create(rewriter, loc, dummy,
+                                                      result, offsets, strides);
         newTypes.push_back(unrolledType);
         unrolledInputNums.push_back(newInputNo);
         ++newInputNo;
@@ -1109,8 +1114,8 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
       Value returnValue = returnOp.getOperand(origResultNo);
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        Value result = vector::ExtractStridedSliceOp::create(rewriter,
-            loc, returnValue, offsets, extractShape, strides);
+        Value result = vector::ExtractStridedSliceOp::create(
+            rewriter, loc, returnValue, offsets, extractShape, strides);
         if (originalShape.size() > 1) {
           SmallVector<int64_t> extractIndices(originalShape.size() - 1, 0);
           result =
@@ -1179,11 +1184,11 @@ Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
       loc, parent->getRegion(0).front(), elementCount, builder, integerType);
 
   Value zeroOp = spirv::ConstantOp::getZero(integerType, loc, builder);
-  Value offsetOp = spirv::ConstantOp::create(builder,
-      loc, integerType, builder.getI32IntegerAttr(offset));
+  Value offsetOp = spirv::ConstantOp::create(builder, loc, integerType,
+                                             builder.getI32IntegerAttr(offset));
   auto addrOp = spirv::AddressOfOp::create(builder, loc, varOp);
-  auto acOp = spirv::AccessChainOp::create(builder,
-      loc, addrOp, llvm::ArrayRef({zeroOp, offsetOp}));
+  auto acOp = spirv::AccessChainOp::create(builder, loc, addrOp,
+                                           llvm::ArrayRef({zeroOp, offsetOp}));
   return spirv::LoadOp::create(builder, loc, acOp);
 }
 
@@ -1276,10 +1281,10 @@ Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
   if (isa<spirv::ArrayType>(pointeeType)) {
     linearizedIndices.push_back(linearIndex);
     return spirv::AccessChainOp::create(builder, loc, basePtr,
-                                                linearizedIndices);
+                                        linearizedIndices);
   }
   return spirv::PtrAccessChainOp::create(builder, loc, basePtr, linearIndex,
-                                                 linearizedIndices);
+                                         linearizedIndices);
 }
 
 Value mlir::spirv::getElementPtr(const SPIRVTypeConverter &typeConverter,
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
index 17f79780afa20..e0900005ea1bb 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
@@ -64,14 +64,14 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   //     and 4 additions after constant folding.
   //   - With sign-extended arguments, we end up emitting 8 multiplications and
   //     and 12 additions after CSE.
-  Value cstLowMask = ConstantOp::create(rewriter,
-      loc, lhs.getType(), getScalarOrSplatAttr(argTy, (1 << 16) - 1));
+  Value cstLowMask = ConstantOp::create(
+      rewriter, loc, lhs.getType(), getScalarOrSplatAttr(argTy, (1 << 16) - 1));
   auto getLowDigit = [&rewriter, loc, cstLowMask](Value val) {
     return BitwiseAndOp::create(rewriter, loc, val, cstLowMask);
   };
 
   Value cst16 = ConstantOp::create(rewriter, loc, lhs.getType(),
-                                            getScalarOrSplatAttr(argTy, 16));
+                                   getScalarOrSplatAttr(argTy, 16));
   auto getHighDigit = [&rewriter, loc, cst16](Value val) {
     return ShiftRightLogicalOp::create(rewriter, loc, val, cst16);
   };
@@ -86,7 +86,7 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   };
 
   Value cst0 = ConstantOp::create(rewriter, loc, lhs.getType(),
-                                           getScalarOrSplatAttr(argTy, 0));
+                                  getScalarOrSplatAttr(argTy, 0));
 
   Value lhsLow = getLowDigit(lhs);
   Value lhsHigh = getHighDigit(lhs);
@@ -128,8 +128,9 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   Value low = combineDigits(resultDigits[0], resultDigits[1]);
   Value high = combineDigits(resultDigits[2], resultDigits[3]);
 
-  return CompositeConstructOp::create(rewriter,
-      loc, mulOp->getResultTypes().front(), llvm::ArrayRef({low, high}));
+  return CompositeConstructOp::create(rewriter, loc,
+                                      mulOp->getResultTypes().front(),
+                                      llvm::ArrayRef({low, high}));
 }
 
 //===----------------------------------------------------------------------===//
@@ -184,18 +185,19 @@ struct ExpandAddCarryPattern final : OpRewritePattern<IAddCarryOp> {
           loc,
           llvm::formatv("Unexpected integer type for WebGPU: '{0}'", elemTy));
 
-    Value one =
-        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 1));
-    Value zero =
-        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 0));
+    Value one = ConstantOp::create(rewriter, loc, argTy,
+                                   getScalarOrSplatAttr(argTy, 1));
+    Value zero = ConstantOp::create(rewriter, loc, argTy,
+                                    getScalarOrSplatAttr(argTy, 0));
 
     // Calculate the carry by checking if the addition resulted in an overflow.
     Value out = IAddOp::create(rewriter, loc, lhs, rhs);
     Value cmp = ULessThanOp::create(rewriter, loc, out, lhs);
     Value carry = SelectOp::create(rewriter, loc, cmp, one, zero);
 
-    Value add = CompositeConstructOp::create(rewriter,
-        loc, op->getResultTypes().front(), llvm::ArrayRef({out, carry}));
+    Value add = CompositeConstructOp::create(rewriter, loc,
+                                             op->getResultTypes().front(),
+                                             llvm::ArrayRef({out, carry}));
 
     rewriter.replaceOp(op, add);
     return success();
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
index e7bb2d76666fb..692f2e7616e5a 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
@@ -380,13 +380,13 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = dstNumBytes / srcNumBytes;
-      auto ratioValue = spirv::ConstantOp::create(rewriter,
-          loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
+      auto ratioValue = spirv::ConstantOp::create(
+          rewriter, loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
           spirv::SDivOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
-      indices.push_back(
-          spirv::SModOp::create(rewriter, loc, indexType, oldIndex, ratioValue));
+      indices.push_back(spirv::SModOp::create(rewriter, loc, indexType,
+                                              oldIndex, ratioValue));
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -407,8 +407,8 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = srcNumBytes / dstNumBytes;
-      auto ratioValue = spirv::ConstantOp::create(rewriter,
-          loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
+      auto ratioValue = spirv::ConstantOp::create(
+          rewriter, loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
           spirv::IMulOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
@@ -443,7 +443,7 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
 
     if (areSameBitwidthScalarType(srcElemType, dstElemType)) {
       auto castOp = spirv::BitcastOp::create(rewriter, loc, srcElemType,
-                                                      newLoadOp.getValue());
+                                             newLoadOp.getValue());
       rewriter.replaceOp(loadOp, castOp->getResults());
 
       return success();
@@ -475,10 +475,10 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
       auto indices = llvm::to_vector<4>(acOp.getIndices());
       for (int i = 1; i < ratio; ++i) {
         // Load all subsequent components belonging to this element.
-        indices.back() = spirv::IAddOp::create(rewriter,
-            loc, i32Type, indices.back(), oneValue);
-        auto componentAcOp = spirv::AccessChainOp::create(rewriter,
-            loc, acOp.getBasePtr(), indices);
+        indices.back() = spirv::IAddOp::create(rewriter, loc, i32Type,
+                                               indices.back(), oneValue);
+        auto componentAcOp = spirv::AccessChainOp::create(
+            rewriter, loc, acOp.getBasePtr(), indices);
         // Assuming little endian, this reads lower-ordered bits of the number
         // to lower-numbered components of the vector.
         components.push_back(
@@ -513,8 +513,8 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
               c = spirv::BitcastOp::create(rewriter, loc, castType, c);
           }
         }
-      Value vectorValue = spirv::CompositeConstructOp::create(rewriter,
-          loc, vectorType, components);
+      Value vectorValue = spirv::CompositeConstructOp::create(
+          rewriter, loc, vectorType, components);
 
       if (!isa<VectorType>(srcElemType))
         vectorValue =
diff --git a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
index 5533a7e914bf0..301f943559409 100644
--- a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -57,8 +57,8 @@ struct AssumingOpInterface
 
     // Create new op and move over region.
     TypeRange newResultTypes(yieldOp.getOperands());
-    auto newOp = shape::AssumingOp::create(rewriter,
-        op->getLoc(), newResultTypes, assumingOp.getWitness());
+    auto newOp = shape::AssumingOp::create(
+        rewriter, op->getLoc(), newResultTypes, assumingOp.getWitness());
     newOp.getDoRegion().takeBody(assumingOp.getRegion());
 
     // Update all uses of the old op.
@@ -66,8 +66,9 @@ struct AssumingOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(assumingOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
-            assumingOp.getLoc(), it.value(), newOp->getResult(it.index())));
+        newResults.push_back(bufferization::ToTensorOp::create(
+            rewriter, assumingOp.getLoc(), it.value(),
+            newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
       }
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index b5f6230bc2362..9e9cf62c57fa9 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -50,7 +50,7 @@ NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
   Block *body = reduce.getBody();
   OpBuilder b = OpBuilder::atBlockEnd(body);
   Value product = MulOp::create(b, loc, valueType, body->getArgument(1),
-                                  body->getArgument(2));
+                                body->getArgument(2));
   shape::YieldOp::create(b, loc, product);
 
   rewriter.replaceOp(op, reduce.getResult());
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
index 74842cc1f310b..abb37a5e10b9a 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
@@ -41,8 +41,8 @@ LogicalResult sparse_tensor::detail::stageWithSortImpl(
 
   // -> sort
   Type dstCOOTp = dstStt.getCOOType(/*ordered=*/true);
-  Value dstCOO = ReorderCOOOp::create(rewriter,
-      loc, dstCOOTp, srcCOO, SparseTensorSortKind::HybridQuickSort);
+  Value dstCOO = ReorderCOOOp::create(rewriter, loc, dstCOOTp, srcCOO,
+                                      SparseTensorSortKind::HybridQuickSort);
 
   // -> dest.
   if (dstCOO.getType() == finalTp) {
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
index 6545bfbcac5c7..40c182f9dbb37 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
@@ -89,10 +89,10 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
           Value mem;
           if (kind == SparseTensorFieldKind::PosMemRef)
             mem = sparse_tensor::ToPositionsOp::create(builder, loc, inputs[0],
-                                                               lv);
+                                                       lv);
           else if (kind == SparseTensorFieldKind::CrdMemRef)
-            mem = sparse_tensor::ToCoordinatesOp::create(builder, loc, inputs[0],
-                                                                 lv);
+            mem = sparse_tensor::ToCoordinatesOp::create(builder, loc,
+                                                         inputs[0], lv);
           else
             mem = sparse_tensor::ToValuesOp::create(builder, loc, inputs[0]);
           toVals.push_back(mem);
@@ -199,8 +199,9 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     OpBuilder moduleBuilder(modOp.getBodyRegion());
     unsigned extra = inputTypes.size();
     inputTypes.append(extraTypes);
-    auto func = func::FuncOp::create(moduleBuilder,
-        loc, orgName, FunctionType::get(context, inputTypes, outputTypes));
+    auto func = func::FuncOp::create(
+        moduleBuilder, loc, orgName,
+        FunctionType::get(context, inputTypes, outputTypes));
     func.setPublic();
 
     // Construct new wrapper method body.
@@ -216,8 +217,8 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     // Call the original, now private method. A subsequent inlining pass can
     // determine whether cloning the method body in place is worthwhile.
     auto org = SymbolRefAttr::get(context, wrapper);
-    auto call = func::CallOp::create(rewriter, loc, funcOp.getResultTypes(), org,
-                                              inputs);
+    auto call = func::CallOp::create(rewriter, loc, funcOp.getResultTypes(),
+                                     org, inputs);
 
     // Convert outputs and return.
     SmallVector<Value> outputs;
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
index 12507ff1c5251..02623198c25b5 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
@@ -94,8 +94,8 @@ static FlatSymbolRefAttr getMangledSortHelperFunc(
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPoint(insertPoint);
     Location loc = insertPoint.getLoc();
-    func = func::FuncOp::create(builder,
-        loc, nameOstream.str(),
+    func = func::FuncOp::create(
+        builder, loc, nameOstream.str(),
         FunctionType::get(context, operands.getTypes(), resultTypes));
     func.setPrivate();
     createFunc(builder, module, func, xPerm, ny, nTrailingP);
@@ -221,7 +221,7 @@ static Value createEqCompare(OpBuilder &builder, Location loc, Value i, Value j,
     Value ne =
         arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
     scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
-                                               ne, /*else=*/true);
+                                       ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value f = constantI1(builder, loc, false);
@@ -266,7 +266,8 @@ static Value createLessThanCompare(OpBuilder &builder, Location loc, Value i,
 
   Value res;
   if (isLastDim) {
-    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
+    res =
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
@@ -275,7 +276,7 @@ static Value createLessThanCompare(OpBuilder &builder, Location loc, Value i,
     Value ne =
         arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
     scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
-                                               ne, /*else=*/true);
+                                       ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value lt =
@@ -337,16 +338,16 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value p = args[hiIdx];
   SmallVector<Type, 2> types(2, p.getType()); // Only two types.
-  scf::WhileOp whileOp = scf::WhileOp::create(builder,
-      loc, types, SmallVector<Value, 2>{args[loIdx], args[hiIdx]});
+  scf::WhileOp whileOp = scf::WhileOp::create(
+      builder, loc, types, SmallVector<Value, 2>{args[loIdx], args[hiIdx]});
 
   // The before-region of the WhileOp.
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, types, {loc, loc});
   builder.setInsertionPointToEnd(before);
-  Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
-                                              before->getArgument(0),
-                                              before->getArgument(1));
+  Value cond1 =
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
+                            before->getArgument(0), before->getArgument(1));
   scf::ConditionOp::create(builder, loc, cond1, before->getArguments());
 
   // The after-region of the WhileOp.
@@ -357,8 +358,8 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   Value hi = after->getArgument(1);
   // Compute mid = (lo + hi) >> 1.
   Value c1 = constantIndex(builder, loc, 1);
-  Value mid = arith::ShRUIOp::create(builder,
-      loc, arith::AddIOp::create(builder, loc, lo, hi), c1);
+  Value mid = arith::ShRUIOp::create(
+      builder, loc, arith::AddIOp::create(builder, loc, lo, hi), c1);
   Value midp1 = arith::AddIOp::create(builder, loc, mid, c1);
 
   // Compare xs[p] < xs[mid].
@@ -521,7 +522,7 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
   Value len = arith::SubIOp::create(builder, loc, hiP1, lo);
   Value lenThreshold = constantIndex(builder, loc, 1000);
   Value lenCond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
-                                                len, lenThreshold);
+                                        len, lenThreshold);
   scf::IfOp lenIf = scf::IfOp::create(builder, loc, lenCond, /*else=*/true);
 
   // When len < 1000, choose pivot from median of 3 values.
@@ -607,7 +608,7 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
                                       {loc, loc, loc, loc});
   builder.setInsertionPointToEnd(before);
   scf::ConditionOp::create(builder, loc, before->getArgument(3),
-                                   before->getArguments());
+                           before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -639,14 +640,14 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   Value icond =
       arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, i, p);
   scf::IfOp ifOpI = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
-                                              icond, /*else=*/true);
+                                      icond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpI.getThenRegion().front());
   scf::YieldOp::create(builder, loc, ValueRange{j});
   builder.setInsertionPointToStart(&ifOpI.getElseRegion().front());
   Value jcond =
       arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, j, p);
   scf::IfOp ifOpJ = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
-                                              jcond, /*else=*/true);
+                                      jcond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpJ.getThenRegion().front());
   scf::YieldOp::create(builder, loc, ValueRange{i});
   builder.setInsertionPointToStart(&ifOpJ.getElseRegion().front());
@@ -656,8 +657,9 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   builder.setInsertionPointAfter(ifOpI);
   Value compareEqIJ =
       arith::AndIOp::create(builder, loc, iCompareEq, jCompareEq);
-  scf::IfOp ifOp2 = scf::IfOp::create(builder,
-      loc, TypeRange{i.getType(), j.getType()}, compareEqIJ, /*else=*/true);
+  scf::IfOp ifOp2 =
+      scf::IfOp::create(builder, loc, TypeRange{i.getType(), j.getType()},
+                        compareEqIJ, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
   Value i2 = arith::AddIOp::create(builder, loc, i, c1);
   Value j2 = arith::SubIOp::create(builder, loc, j, c1);
@@ -665,17 +667,18 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
   scf::YieldOp::create(builder, loc, ValueRange{i, j});
   builder.setInsertionPointAfter(ifOp2);
-  scf::YieldOp::create(builder,
-      loc,
-      ValueRange{ifOp2.getResult(0), ifOp2.getResult(1), ifOpI.getResult(0),
-                 /*cont=*/constantI1(builder, loc, true)});
+  scf::YieldOp::create(builder, loc,
+                       ValueRange{ifOp2.getResult(0), ifOp2.getResult(1),
+                                  ifOpI.getResult(0),
+                                  /*cont=*/constantI1(builder, loc, true)});
 
   // False branch for if i < j (i.e., i >= j):
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
   p = arith::AddIOp::create(builder, loc, j,
-                                    constantOne(builder, loc, j.getType()));
-  scf::YieldOp::create(builder,
-      loc, ValueRange{i, j, p, /*cont=*/constantI1(builder, loc, false)});
+                            constantOne(builder, loc, j.getType()));
+  scf::YieldOp::create(
+      builder, loc,
+      ValueRange{i, j, p, /*cont=*/constantI1(builder, loc, false)});
 
   // Return for the whileOp.
   builder.setInsertionPointAfter(ifOp);
@@ -773,7 +776,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
     Value lChildIdx = arith::AddIOp::create(builder, loc, lChild, first);
     Value rChild = arith::AddIOp::create(builder, loc, lChild, c1);
     Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
-                                                rChild, n);
+                                        rChild, n);
     SmallVector<Type, 2> ifTypes(2, r.getType());
     scf::IfOp if1 =
         scf::IfOp::create(builder, loc, ifTypes, cond1, /*else=*/true);
@@ -803,8 +806,8 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
 
   // While (data[start] < data[childIndex]).
   SmallVector<Type, 3> types(3, child.getType());
-  scf::WhileOp whileOp = scf::WhileOp::create(builder,
-      loc, types, SmallVector<Value, 2>{start, child, childIdx});
+  scf::WhileOp whileOp = scf::WhileOp::create(
+      builder, loc, types, SmallVector<Value, 2>{start, child, childIdx});
 
   // The before-region of the WhileOp.
   SmallVector<Location, 3> locs(3, loc);
@@ -828,16 +831,17 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   start = childIdx;
   Value cond2 =
       arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp if2 = scf::IfOp::create(builder,
-      loc, TypeRange{child.getType(), child.getType()}, cond2, /*else=*/true);
+  scf::IfOp if2 = scf::IfOp::create(builder, loc,
+                                    TypeRange{child.getType(), child.getType()},
+                                    cond2, /*else=*/true);
   builder.setInsertionPointToStart(&if2.getThenRegion().front());
   auto [newChild, newChildIdx] = getLargerChild(child);
   scf::YieldOp::create(builder, loc, ValueRange{newChild, newChildIdx});
   builder.setInsertionPointToStart(&if2.getElseRegion().front());
   scf::YieldOp::create(builder, loc, ValueRange{child, childIdx});
   builder.setInsertionPointAfter(if2);
-  scf::YieldOp::create(builder,
-      loc, ValueRange{start, if2.getResult(0), if2.getResult(1)});
+  scf::YieldOp::create(builder, loc,
+                       ValueRange{start, if2.getResult(0), if2.getResult(1)});
 
   builder.setInsertionPointAfter(ifN);
   func::ReturnOp::create(builder, loc);
@@ -888,7 +892,7 @@ static void createHeapSortFunc(OpBuilder &builder, ModuleOp module,
       builder, func, TypeRange(), kShiftDownFuncNamePrefix, xPerm, ny,
       shiftDownOperands, createShiftDownFunc, /*nTrailingP=*/1);
   func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
-                               shiftDownOperands);
+                       shiftDownOperands);
 
   builder.setInsertionPointAfter(forI);
   // For l = n downto 2.
@@ -905,7 +909,7 @@ static void createHeapSortFunc(OpBuilder &builder, ModuleOp module,
   shiftDownOperands[shiftDownOperands.size() - 1] =
       arith::SubIOp::create(builder, loc, l, c1);
   func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
-                               shiftDownOperands);
+                       shiftDownOperands);
 
   builder.setInsertionPointAfter(forL);
   func::ReturnOp::create(builder, loc);
@@ -949,7 +953,7 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
   // Else len > 2, need recursion.
   builder.setInsertionPointToStart(&ifLenGtTwo.getThenRegion().front());
   Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
-                                             lenLow, lenHigh);
+                                     lenLow, lenHigh);
 
   Value c0 = constantIndex(builder, loc, 0);
   scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
@@ -1153,8 +1157,8 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   if (isHybrid) {
     Value len = arith::SubIOp::create(builder, loc, hi, lo);
     Value lenLimit = constantIndex(builder, loc, 30);
-    Value lenCond = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::ule, len, lenLimit);
+    Value lenCond = arith::CmpIOp::create(
+        builder, loc, arith::CmpIPredicate::ule, len, lenLimit);
     scf::IfOp lenIf =
         scf::IfOp::create(builder, loc, types, lenCond, /*else=*/true);
 
@@ -1164,17 +1168,17 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
         builder, func, TypeRange(), kSortStableFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createSortStableFunc);
     func::CallOp::create(builder, loc, insertionSortFunc, TypeRange(),
-                                 ValueRange(args).drop_back(nTrailingP));
+                         ValueRange(args).drop_back(nTrailingP));
     scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When len > limit.
     builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
     Value depthLimit = args.back();
     depthLimit = arith::SubIOp::create(builder, loc, depthLimit,
-                                               constantI64(builder, loc, 1));
+                                       constantI64(builder, loc, 1));
     Value depthCond =
         arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
-                                      depthLimit, constantI64(builder, loc, 0));
+                              depthLimit, constantI64(builder, loc, 0));
     scf::IfOp depthIf =
         scf::IfOp::create(builder, loc, types, depthCond, /*else=*/true);
 
@@ -1184,7 +1188,7 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
         builder, func, TypeRange(), kHeapSortFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createHeapSortFunc);
     func::CallOp::create(builder, loc, heapSortFunc, TypeRange(),
-                                 ValueRange(args).drop_back(nTrailingP));
+                         ValueRange(args).drop_back(nTrailingP));
     scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When depth doesn't exceed limit.
@@ -1248,11 +1252,11 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     // As a heuristics, set depthLimit = 2 * log2(n).
     Value lo = operands[loIdx];
     Value hi = operands[hiIdx];
-    Value len = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI64Type(),
+    Value len = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI64Type(),
         arith::SubIOp::create(rewriter, loc, hi, lo));
-    Value depthLimit = arith::SubIOp::create(rewriter,
-        loc, constantI64(rewriter, loc, 64),
+    Value depthLimit = arith::SubIOp::create(
+        rewriter, loc, constantI64(rewriter, loc, 64),
         math::CountLeadingZerosOp::create(rewriter, loc, len));
     operands.push_back(depthLimit);
     break;
@@ -1317,14 +1321,14 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
     bool nIsOne = (nValue && nValue.value() == 1);
 
     if (!op.getInbounds()) {
-      Value cond = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::ugt, newSize, capacity);
+      Value cond = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::ugt, newSize, capacity);
 
       Value c2 = constantIndex(rewriter, loc, 2);
       auto bufferType =
           MemRefType::get({ShapedType::kDynamic}, value.getType());
       scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, bufferType, cond,
-                                                  /*else=*/true);
+                                         /*else=*/true);
       // True branch.
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       if (nIsOne) {
@@ -1343,7 +1347,7 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
         capacity =
             arith::MulIOp::create(rewriter, loc, before->getArgument(0), c2);
         cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ugt,
-                                              newSize, capacity);
+                                     newSize, capacity);
         scf::ConditionOp::create(rewriter, loc, cond, ValueRange{capacity});
         // The after-region of the WhileOp.
         Block *after = rewriter.createBlock(&whileOp.getAfter(), {},
@@ -1355,13 +1359,14 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
         capacity = whileOp.getResult(0);
       }
 
-      Value newBuffer =
-          memref::ReallocOp::create(rewriter, loc, bufferType, buffer, capacity);
+      Value newBuffer = memref::ReallocOp::create(rewriter, loc, bufferType,
+                                                  buffer, capacity);
       if (enableBufferInitialization) {
-        Value fillSize = arith::SubIOp::create(rewriter, loc, capacity, newSize);
+        Value fillSize =
+            arith::SubIOp::create(rewriter, loc, capacity, newSize);
         Value fillValue = constantZero(rewriter, loc, value.getType());
-        Value subBuffer = memref::SubViewOp::create(rewriter,
-            loc, newBuffer, /*offset=*/ValueRange{newSize},
+        Value subBuffer = memref::SubViewOp::create(
+            rewriter, loc, newBuffer, /*offset=*/ValueRange{newSize},
             /*size=*/ValueRange{fillSize},
             /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
         linalg::FillOp::create(rewriter, loc, fillValue, subBuffer);
@@ -1381,8 +1386,9 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
     if (nIsOne) {
       memref::StoreOp::create(rewriter, loc, value, buffer, size);
     } else {
-      Value subBuffer = memref::SubViewOp::create(rewriter,
-          loc, buffer, /*offset=*/ValueRange{size}, /*size=*/ValueRange{n},
+      Value subBuffer = memref::SubViewOp::create(
+          rewriter, loc, buffer, /*offset=*/ValueRange{size},
+          /*size=*/ValueRange{n},
           /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
       linalg::FillOp::create(rewriter, loc, value, subBuffer);
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index 001754b5531be..a317abd6c560b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -60,7 +60,7 @@ static gpu::GPUModuleOp genGPUModule(OpBuilder &builder, ModuleOp topModule) {
   markAsGPUContainer(topModule);
   builder.setInsertionPointToStart(topModule.getBody());
   return gpu::GPUModuleOp::create(builder, topModule->getLoc(),
-                                          "sparse_kernels");
+                                  "sparse_kernels");
 }
 
 /// Constructs a new GPU kernel in the given GPU module.
@@ -157,7 +157,7 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
     }
   }
   return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
-                                      token, dynamicSizes, ValueRange());
+                              token, dynamicSizes, ValueRange());
 }
 
 // Allocates a typed buffer on the host with given size.
@@ -172,7 +172,7 @@ static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Type type,
                                    Value size, Value token) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
   return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
-                                      token, size, ValueRange());
+                              token, size, ValueRange());
 }
 
 // Allocates a void buffer on the device with given size.
@@ -524,12 +524,12 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
     if (enableRT) {
       assert(colA);
       return gpu::CreateCooOp::create(builder, loc, handleTp, tokenTp, token,
-                                              sz1, sz2, nseA, rowA, colA, valA);
+                                      sz1, sz2, nseA, rowA, colA, valA);
     }
 #ifdef CUSPARSE_COO_AOS
     assert(!colA);
     return gpu::CreateCooAoSOp::create(builder, loc, handleTp, tokenTp, token,
-                                               sz1, sz2, nseA, rowA, valA);
+                                       sz1, sz2, nseA, rowA, valA);
 #else
     llvm_unreachable("gpu::CreateCooAoSOp is deprecated");
 #endif
@@ -537,10 +537,10 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   assert(colA);
   if (format == CuSparseFormat::kCSR)
     return gpu::CreateCsrOp::create(builder, loc, handleTp, tokenTp, token, sz1,
-                                            sz2, nseA, rowA, colA, valA);
+                                    sz2, nseA, rowA, colA, valA);
   if (format == CuSparseFormat::kCSC)
     return gpu::CreateCscOp::create(builder, loc, handleTp, tokenTp, token, sz1,
-                                            sz2, nseA, rowA, colA, valA);
+                                    sz2, nseA, rowA, colA, valA);
   // BSR requires a bit more work since we need to pass in the block size
   // and all others sizes in terms of blocks (#block-rows, #block-cols,
   // #nonzero-blocks).
@@ -551,11 +551,10 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   Value bSz = constantIndex(builder, loc, b);
   Value bRows = arith::DivUIOp::create(builder, loc, sz1, bSz);
   Value bCols = arith::DivUIOp::create(builder, loc, sz2, bSz);
-  Value bNum = arith::DivUIOp::create(builder,
-      loc, nseA, constantIndex(builder, loc, b * b));
+  Value bNum = arith::DivUIOp::create(builder, loc, nseA,
+                                      constantIndex(builder, loc, b * b));
   return gpu::CreateBsrOp::create(builder, loc, handleTp, tokenTp, token, bRows,
-                                          bCols, bNum, bSz, bSz, rowA, colA,
-                                          valA);
+                                  bCols, bNum, bSz, bSz, rowA, colA, valA);
 }
 
 /// Match and rewrite SpMV kernel.
@@ -606,19 +605,19 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dvecX = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, vecX, szX);
+  auto dvecX = gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp,
+                                             tokenTp, token, vecX, szX);
   Value dnX = dvecX.getResult(0);
   token = dvecX.getAsyncToken();
-  auto dvecY = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, vecY, szY);
+  auto dvecY = gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp,
+                                             tokenTp, token, vecY, szY);
   Value dnY = dvecY.getResult(0);
   token = dvecY.getAsyncToken();
   auto dnYType = llvm::cast<ShapedType>(y.getType()).getElementType();
 
   // Precompute buffersize for SpMV.
-  auto bufferComp = gpu::SpMVBufferSizeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, spMatA, dnX, dnY,
+  auto bufferComp = gpu::SpMVBufferSizeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, spMatA, dnX, dnY,
       /*computeType=*/dnYType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
@@ -627,8 +626,9 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SpMV.
-  auto spmvComp = gpu::SpMVOp::create(rewriter,
-      loc, tokenTp, token, spMatA, dnX, dnY, /*computeType=*/dnYType, buffer);
+  auto spmvComp =
+      gpu::SpMVOp::create(rewriter, loc, tokenTp, token, spMatA, dnX, dnY,
+                          /*computeType=*/dnYType, buffer);
   token = spmvComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
@@ -704,21 +704,21 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, matB,
-      SmallVector<Value>{szk, szn});
+  auto dmatB =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp, tokenTp,
+                                    token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, matC,
-      SmallVector<Value>{szm, szn});
+  auto dmatC =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp, tokenTp,
+                                    token, matC, SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
   token = dmatC.getAsyncToken();
   auto dmatCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SpMM.
-  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, spMatA, dnB, dnC,
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
@@ -728,8 +728,9 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Perform the SpMM.
-  auto spmmComp = gpu::SpMMOp::create(rewriter,
-      loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType, buffer);
+  auto spmmComp =
+      gpu::SpMMOp::create(rewriter, loc, tokenTp, token, spMatA, dnB, dnC,
+                          /*computeType=*/dnCType, buffer);
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
@@ -839,41 +840,44 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
       gpu::SpGEMMCreateDescrOp::create(rewriter, loc, descTp, tokenTp, token);
   Value desc = descOp->getResult(0);
   token = descOp->getResult(1);
-  Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
-      valC, gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
+  Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, desc,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, spMatB, spMatC, dnCType, zero, valC,
+      gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
   Value bufferSz1 = work1->getResult(0);
   token = work1->getResult(1);
   auto buf1 = genAllocBuffer(rewriter, loc, bufferSz1, token);
   Value buffer1 = buf1.getResult(0);
   token = buf1.getAsyncToken();
-  Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
-      bufferSz1, buffer1,
+  Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, desc,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, spMatB, spMatC, dnCType, bufferSz1, buffer1,
       gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
   token = work2->getResult(1);
 
   // Compute step.
-  Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
-      valC, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
+  Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, desc,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, spMatB, spMatC, dnCType, zero, valC,
+      gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
   Value bufferSz2 = compute1->getResult(0);
   token = compute1->getResult(1);
   auto buf2 = genAllocBuffer(rewriter, loc, bufferSz2, token);
   Value buffer2 = buf2.getResult(0);
   token = buf2.getAsyncToken();
-  Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
-      bufferSz2, buffer2, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
+  Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, desc,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, spMatB, spMatC, dnCType, bufferSz2, buffer2,
+      gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
   token = compute2->getResult(1);
 
   // Get sizes.
-  Operation *sizes = gpu::SpMatGetSizeOp::create(rewriter,
-      loc, indexTp, indexTp, indexTp, tokenTp, token, spMatC);
+  Operation *sizes = gpu::SpMatGetSizeOp::create(
+      rewriter, loc, indexTp, indexTp, indexTp, tokenTp, token, spMatC);
   Value nnz = sizes->getResult(2);
   token = sizes->getResult(3);
   auto a2 = genAllocBuffer(rewriter, loc, cTp.getCrdType(), nnz, token);
@@ -884,11 +888,11 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = a3.getAsyncToken();
 
   // Update C with new pointers and copy final product back into C.
-  Operation *update = gpu::SetCsrPointersOp::create(rewriter,
-      loc, tokenTp, token, spMatC, rowC, colC, valC);
+  Operation *update = gpu::SetCsrPointersOp::create(
+      rewriter, loc, tokenTp, token, spMatC, rowC, colC, valC);
   token = update->getResult(0);
-  Operation *copy = gpu::SpGEMMCopyOp::create(rewriter,
-      loc, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
+  Operation *copy = gpu::SpGEMMCopyOp::create(
+      rewriter, loc, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType);
   token = copy->getResult(0);
 
@@ -925,12 +929,12 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  Value vt = bufferization::ToTensorOp::create(rewriter,
-      loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
-  Value rt = bufferization::ToTensorOp::create(rewriter,
-      loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
-  Value ct = bufferization::ToTensorOp::create(rewriter,
-      loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
+  Value vt = bufferization::ToTensorOp::create(
+      rewriter, loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
+  Value rt = bufferization::ToTensorOp::create(
+      rewriter, loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
+  Value ct = bufferization::ToTensorOp::create(
+      rewriter, loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
   rewriter.replaceOpWithNewOp<AssembleOp>(op, c.getType(), ValueRange{rt, ct},
                                           vt);
   return success();
@@ -980,19 +984,19 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  Operation *spGenA = gpu::Create2To4SpMatOp::create(rewriter,
-      loc, spMatHandleTp, tokenTp, token, szm, szk,
+  Operation *spGenA = gpu::Create2To4SpMatOp::create(
+      rewriter, loc, spMatHandleTp, tokenTp, token, szm, szk,
       gpu::Prune2To4SpMatFlag::PRUNE_AND_CHECK, matA);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, matB,
-      SmallVector<Value>{szk, szn});
+  auto dmatB =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp, tokenTp,
+                                    token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, matC,
-      SmallVector<Value>{szm, szn});
+  auto dmatC =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp, tokenTp,
+                                    token, matC, SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
   token = dmatC.getAsyncToken();
   auto dmatCType = llvm::cast<ShapedType>(matC.getType()).getElementType();
@@ -1000,9 +1004,10 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   // Precompute buffersize for SpMM.
   SmallVector<Type> bufferTypes_{indexTp, indexTp, indexTp};
   TypeRange bufferTypes(bufferTypes_);
-  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
-      loc, bufferTypes, tokenTp, token, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, dnB, dnC,
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(
+      rewriter, loc, bufferTypes, tokenTp, token,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
   token = bufferComp.getAsyncToken();
 
@@ -1022,8 +1027,8 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
 
   // Perform the SpMM.
   auto dnCType = llvm::cast<ShapedType>(matC.getType()).getElementType();
-  auto spmmComp = gpu::SpMMOp::create(rewriter,
-      loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType,
+  auto spmmComp = gpu::SpMMOp::create(
+      rewriter, loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType,
       SmallVector<Value>{buffer1, buffer2, buffer3});
   token = spmmComp.getAsyncToken();
 
@@ -1096,12 +1101,14 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  auto dmatA = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnMatHandleTp, tokenTp, token, matA, SmallVector<Value>{szm, szk});
+  auto dmatA =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnMatHandleTp, tokenTp,
+                                    token, matA, SmallVector<Value>{szm, szk});
   Value dnA = dmatA.getResult(0);
   token = dmatA.getAsyncToken();
-  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnMatHandleTp, tokenTp, token, matB, SmallVector<Value>{szk, szn});
+  auto dmatB =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnMatHandleTp, tokenTp,
+                                    token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
   Operation *spGenC =
@@ -1112,8 +1119,8 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SDDMM.
-  auto bufferComp = gpu::SDDMMBufferSizeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, dnA, dnB, spMatC, dnCType);
+  auto bufferComp = gpu::SDDMMBufferSizeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, dnA, dnB, spMatC, dnCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
   auto buf = genAllocBuffer(rewriter, loc, bufferSz, token);
@@ -1122,7 +1129,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
 
   // Perform the SDDMM.
   auto sddmmComp = gpu::SDDMMOp::create(rewriter, loc, tokenTp, token, dnA, dnB,
-                                                 spMatC, dnCType, buffer);
+                                        spMatC, dnCType, buffer);
   token = sddmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
index 0986a0f2ea0f0..dfb127444e281 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
@@ -68,11 +68,11 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
   for (unsigned i : caseBits.bits()) {
     SparseIterator *it = iters[i].get();
     Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                                it->getCrd(), loopCrd);
+                                       it->getCrd(), loopCrd);
     casePred = arith::AndIOp::create(rewriter, loc, casePred, pred);
   }
-  scf::IfOp ifOp = scf::IfOp::create(rewriter,
-      loc, ValueRange(userReduc).getTypes(), casePred, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(
+      rewriter, loc, ValueRange(userReduc).getTypes(), casePred, /*else=*/true);
   rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
   // Erase the empty block.
@@ -127,8 +127,8 @@ static ValueRange genLoopWithIterator(
   if (it->iteratableByFor()) {
     auto [lo, hi] = it->genForCond(rewriter, loc);
     Value step = constantIndex(rewriter, loc, 1);
-    scf::ForOp forOp = scf::ForOp::create(rewriter,
-        loc, lo, hi, step, reduc,
+    scf::ForOp forOp = scf::ForOp::create(
+        rewriter, loc, lo, hi, step, reduc,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
           // Empty builder function to ensure that no terminator is created.
         });
@@ -212,8 +212,8 @@ class ExtractValOpConverter : public OpConversionPattern<ExtractValOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value pos = adaptor.getIterator().back();
-    Value valBuf = ToValuesOp::create(rewriter,
-        loc, llvm::getSingleElement(adaptor.getTensor()));
+    Value valBuf = ToValuesOp::create(
+        rewriter, loc, llvm::getSingleElement(adaptor.getTensor()));
     rewriter.replaceOpWithNewOp<memref::LoadOp>(op, valBuf, pos);
     return success();
   }
@@ -385,8 +385,8 @@ class SparseCoIterateOpConverter : public OpConversionPattern<CoIterateOp> {
         SmallVector<Value> nextIterYields(res);
         // 2nd. foward the loop.
         for (SparseIterator *it : validIters) {
-          Value cmp = arith::CmpIOp::create(rewriter,
-              loc, arith::CmpIPredicate::eq, it->getCrd(), loopCrd);
+          Value cmp = arith::CmpIOp::create(
+              rewriter, loc, arith::CmpIPredicate::eq, it->getCrd(), loopCrd);
           it->forwardIf(rewriter, loc, cmp);
           llvm::append_range(nextIterYields, it->getCursor());
         }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
index dc6bef5387b70..df9b6cf040efa 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
@@ -43,7 +43,8 @@ struct DemapInsRewriter : public OpRewritePattern<SourceOp> {
     SmallVector<Value> deMappedIns(op->getOperands());
     for (Value &in : deMappedIns) {
       if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) {
-        in = ReinterpretMapOp::create(rewriter, loc, stt->getDemappedType(), in);
+        in =
+            ReinterpretMapOp::create(rewriter, loc, stt->getDemappedType(), in);
         changed = true;
       }
     }
@@ -338,7 +339,7 @@ translateMap(linalg::GenericOp op, PatternRewriter &rewriter) {
 static Value genDemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
   return ReinterpretMapOp::create(builder, val.getLoc(), enc.withoutDimToLvl(),
-                                          val);
+                                  val);
 }
 
 // Generates a "re"mapping reinterpretation of the map.
@@ -604,8 +605,8 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     ValueRange dynSz = op.getDynamicSizes();
     for (int64_t dimSz : stt.getDimShape()) {
       if (ShapedType::isDynamic(dimSz)) {
-        Value maxCrd = arith::SubIOp::create(rewriter,
-            loc, dynSz.front(), constantIndex(rewriter, loc, 1));
+        Value maxCrd = arith::SubIOp::create(rewriter, loc, dynSz.front(),
+                                             constantIndex(rewriter, loc, 1));
         maxDimCrds.push_back(maxCrd);
         dynSz = dynSz.drop_front();
       } else {
@@ -619,8 +620,8 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     SmallVector<Value> dynLvlSzs;
     for (unsigned i = 0, e = lvlShape.size(); i < e; i++) {
       if (ShapedType::isDynamic(lvlShape[i])) {
-        Value sz = arith::AddIOp::create(rewriter,
-            loc, maxLvlCrds[i], constantIndex(rewriter, loc, 1));
+        Value sz = arith::AddIOp::create(rewriter, loc, maxLvlCrds[i],
+                                         constantIndex(rewriter, loc, 1));
         dynLvlSzs.push_back(sz);
       }
     }
@@ -650,8 +651,8 @@ struct TensorInsertDemapper
     auto stt = getSparseTensorType(op.getResult());
     ValueRange lvlCrd = stt.translateCrds(rewriter, loc, op.getIndices(),
                                           CrdTransDirectionKind::dim2lvl);
-    auto insertOp = tensor::InsertOp::create(rewriter,
-        loc, op.getScalar(), adaptor.getDest(), lvlCrd);
+    auto insertOp = tensor::InsertOp::create(rewriter, loc, op.getScalar(),
+                                             adaptor.getDest(), lvlCrd);
 
     Value out = genRemap(rewriter, stt.getEncoding(), insertOp.getResult());
     rewriter.replaceOp(op, out);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
index 7b662c2fb0db0..81cd3296de294 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
@@ -126,8 +126,8 @@ void collapseSparseSpace(MutableArrayRef<CollapseSpaceInfo> toCollapse) {
   OpBuilder builder(root);
 
   // Construct the collapsed iteration space.
-  auto collapsedSpace = ExtractIterSpaceOp::create(builder,
-      loc, root.getTensor(), root.getParentIter(), root.getLoLvl(),
+  auto collapsedSpace = ExtractIterSpaceOp::create(
+      builder, loc, root.getTensor(), root.getParentIter(), root.getLoLvl(),
       leaf.getHiLvl());
 
   auto rItOp = llvm::cast<IterateOp>(*root->getUsers().begin());
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
index a1b9035f3b6e6..6dfffbb6e7442 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
@@ -75,9 +75,9 @@ class SpecifierStructBuilder : public StructBuilder {
 
   void insertField(OpBuilder &builder, Location loc, ArrayRef<int64_t> indices,
                    Value v) {
-    value = LLVM::InsertValueOp::create(builder,
-        loc, value, genCast(builder, loc, v, builder.getIntegerType(64)),
-        indices);
+    value = LLVM::InsertValueOp::create(
+        builder, loc, value,
+        genCast(builder, loc, v, builder.getIntegerType(64)), indices);
   }
 
 public:
@@ -205,14 +205,14 @@ void SpecifierStructBuilder::setMemSize(OpBuilder &builder, Location loc,
 Value SpecifierStructBuilder::memSizeArray(OpBuilder &builder,
                                            Location loc) const {
   return LLVM::ExtractValueOp::create(builder, loc, value,
-                                              kMemSizePosInSpecifier);
+                                      kMemSizePosInSpecifier);
 }
 
 /// Builds IR inserting the memory size array into the descriptor.
 void SpecifierStructBuilder::setMemSizeArray(OpBuilder &builder, Location loc,
                                              Value array) {
   value = LLVM::InsertValueOp::create(builder, loc, value, array,
-                                              kMemSizePosInSpecifier);
+                                      kMemSizePosInSpecifier);
 }
 
 } // namespace
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 0450e1d6e8775..70795e2eb211b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -70,7 +70,8 @@ static scf::ForOp createFor(OpBuilder &builder, Location loc, Value upper,
   if (!lower)
     lower = constantZero(builder, loc, indexType);
   Value one = constantOne(builder, loc, indexType);
-  scf::ForOp forOp = scf::ForOp::create(builder, loc, lower, upper, one, fields);
+  scf::ForOp forOp =
+      scf::ForOp::create(builder, loc, lower, upper, one, fields);
   for (unsigned i = 0, e = fields.size(); i < e; i++)
     fields[i] = forOp.getRegionIterArg(i);
   builder.setInsertionPointToStart(forOp.getBody());
@@ -86,9 +87,9 @@ static void createPushback(OpBuilder &builder, Location loc,
   Value field = desc.getMemRefField(kind, lvl);
   StorageSpecifierKind specFieldKind = toSpecifierKind(kind);
 
-  auto pushBackOp = PushBackOp::create(builder,
-      loc, desc.getSpecifierField(builder, loc, specFieldKind, lvl), field,
-      genCast(builder, loc, value, etp), repeat);
+  auto pushBackOp = PushBackOp::create(
+      builder, loc, desc.getSpecifierField(builder, loc, specFieldKind, lvl),
+      field, genCast(builder, loc, value, etp), repeat);
 
   desc.setMemRefField(kind, lvl, pushBackOp.getOutBuffer());
   desc.setSpecifierField(builder, loc, specFieldKind, lvl,
@@ -178,16 +179,16 @@ static void createAllocFields(OpBuilder &builder, Location loc,
   if (stt.isAllDense()) {
     valHeuristic = lvlSizesValues[0];
     for (Level lvl = 1; lvl < lvlRank; lvl++)
-      valHeuristic =
-          arith::MulIOp::create(builder, loc, valHeuristic, lvlSizesValues[lvl]);
+      valHeuristic = arith::MulIOp::create(builder, loc, valHeuristic,
+                                           lvlSizesValues[lvl]);
   } else if (sizeHint) {
     if (stt.getAoSCOOStart() == 0) {
       posHeuristic = constantIndex(builder, loc, 2);
-      crdHeuristic = arith::MulIOp::create(builder,
-          loc, constantIndex(builder, loc, lvlRank), sizeHint); // AOS
+      crdHeuristic = arith::MulIOp::create(
+          builder, loc, constantIndex(builder, loc, lvlRank), sizeHint); // AOS
     } else if (lvlRank == 2 && stt.isDenseLvl(0) && stt.isCompressedLvl(1)) {
-      posHeuristic = arith::AddIOp::create(builder,
-          loc, sizeHint, constantIndex(builder, loc, 1));
+      posHeuristic = arith::AddIOp::create(builder, loc, sizeHint,
+                                           constantIndex(builder, loc, 1));
       crdHeuristic = sizeHint;
     } else {
       posHeuristic = crdHeuristic = constantIndex(builder, loc, 16);
@@ -290,22 +291,22 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   const Value msz =
       crdStrideC ? arith::DivUIOp::create(builder, loc, crdMsz, crdStrideC)
                  : crdMsz;
-  const Value plast = arith::SubIOp::create(builder,
-      loc, genCast(builder, loc, pstop, indexType), one);
+  const Value plast = arith::SubIOp::create(
+      builder, loc, genCast(builder, loc, pstop, indexType), one);
   // Conditional expression.
   Value lt = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
-                                           pstart, pstop);
+                                   pstart, pstop);
   types.push_back(boolType);
   scf::IfOp ifOp1 = scf::IfOp::create(builder, loc, types, lt, /*else*/ true);
   types.pop_back();
   builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
-  Value crd =
-      genLoad(builder, loc, desc.getMemRefField(crdFidx),
-              crdStrideC ? arith::MulIOp::create(builder, loc, plast, crdStrideC)
-                         : plast);
-  Value eq = arith::CmpIOp::create(builder,
-      loc, arith::CmpIPredicate::eq, genCast(builder, loc, crd, indexType),
-      lvlCoords[lvl]);
+  Value crd = genLoad(
+      builder, loc, desc.getMemRefField(crdFidx),
+      crdStrideC ? arith::MulIOp::create(builder, loc, plast, crdStrideC)
+                 : plast);
+  Value eq = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
+                                   genCast(builder, loc, crd, indexType),
+                                   lvlCoords[lvl]);
   scf::YieldOp::create(builder, loc, eq);
   builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
   if (lvl > 0)
@@ -381,10 +382,10 @@ static void genEndInsert(OpBuilder &builder, Location loc,
         Value oldv = loop.getRegionIterArg(0);
         Value newv = genLoad(builder, loc, posMemRef, i);
         Value posZero = constantZero(builder, loc, posType);
-        Value cond = arith::CmpIOp::create(builder,
-            loc, arith::CmpIPredicate::eq, newv, posZero);
+        Value cond = arith::CmpIOp::create(
+            builder, loc, arith::CmpIPredicate::eq, newv, posZero);
         scf::IfOp ifOp = scf::IfOp::create(builder, loc, TypeRange(posType),
-                                                   cond, /*else*/ true);
+                                           cond, /*else*/ true);
         builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         genStore(builder, loc, oldv, posMemRef, i);
         scf::YieldOp::create(builder, loc, oldv);
@@ -582,8 +583,9 @@ class SparseCallConverter : public OpConversionPattern<func::CallOp> {
       return failure();
 
     // (1) Generates new call with flattened return value.
-    auto newCall = func::CallOp::create(rewriter,
-        loc, op.getCallee(), finalRetTy, flattenValues(adaptor.getOperands()));
+    auto newCall =
+        func::CallOp::create(rewriter, loc, op.getCallee(), finalRetTy,
+                             flattenValues(adaptor.getOperands()));
     // (2) Gather sparse tensor returns.
     SmallVector<SmallVector<Value>> packedResultVals;
     // Tracks the offset of current return value (of the original call)
@@ -672,7 +674,7 @@ struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
     auto id = AffineMap::getMultiDimIdentityMap(srcStt.getLvlRank(), ctx);
 
     SortOp::create(rewriter, loc, nnz, crd, ValueRange{val}, id,
-                            rewriter.getIndexAttr(0), op.getAlgorithm());
+                   rewriter.getIndexAttr(0), op.getAlgorithm());
 
     // Since we do in-place sorting, the destinate tensor will have the same set
     // of memrefs as the source tensor.
@@ -931,12 +933,12 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    linalg::FillOp::create(rewriter,
-        loc, ValueRange{constantZero(rewriter, loc, eltType)},
-        ValueRange{values});
-    linalg::FillOp::create(rewriter,
-        loc, ValueRange{constantZero(rewriter, loc, boolType)},
-        ValueRange{filled});
+    linalg::FillOp::create(rewriter, loc,
+                           ValueRange{constantZero(rewriter, loc, eltType)},
+                           ValueRange{values});
+    linalg::FillOp::create(rewriter, loc,
+                           ValueRange{constantZero(rewriter, loc, boolType)},
+                           ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
     assert(op.getNumResults() == 4);
     rewriter.replaceOp(op, {values, filled, added, zero});
@@ -965,9 +967,10 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     // If the innermost level is ordered, we need to sort the coordinates
     // in the "added" array prior to applying the compression.
     if (dstType.isOrderedLvl(dstType.getLvlRank() - 1))
-      SortOp::create(rewriter,
-          loc, count, added, ValueRange{}, rewriter.getMultiDimIdentityMap(1),
-          rewriter.getIndexAttr(0), SparseTensorSortKind::HybridQuickSort);
+      SortOp::create(rewriter, loc, count, added, ValueRange{},
+                     rewriter.getMultiDimIdentityMap(1),
+                     rewriter.getIndexAttr(0),
+                     SparseTensorSortKind::HybridQuickSort);
     // While performing the insertions, we also need to reset the elements
     // of the values/filled-switch by only iterating over the set elements,
     // to ensure that the runtime complexity remains proportional to the
@@ -1192,8 +1195,8 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
             // would require a subViewOp to avoid overflow when copying
             // values.
             Value sz = linalg::createOrFoldDimOp(rewriter, loc, srcMem, 0);
-            auto dstMem = memref::AllocOp::create(rewriter,
-                loc, cast<MemRefType>(fTp), sz);
+            auto dstMem = memref::AllocOp::create(rewriter, loc,
+                                                  cast<MemRefType>(fTp), sz);
             if (fTp != srcMem.getType()) {
               // Converts elements type.
               scf::buildLoopNest(
@@ -1242,8 +1245,9 @@ class SparseExtractSliceConverter
     auto desc = getMutDescriptorFromTensorTuple(adaptor.getSource(), fields,
                                                 op.getSource().getType());
 
-    auto newSpec = StorageSpecifierInitOp::create(rewriter,
-        loc, StorageSpecifierType::get(ctx, dstEnc), desc.getSpecifier());
+    auto newSpec = StorageSpecifierInitOp::create(
+        rewriter, loc, StorageSpecifierType::get(ctx, dstEnc),
+        desc.getSpecifier());
     desc.setSpecifier(newSpec);
 
     // Fills in slice information.
@@ -1326,8 +1330,8 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
               // Flattens the buffer to batchLvlRank.
               auto reassoc = getReassociationForFlattening(
                   mem.getType(), stt.getBatchLvlRank());
-              mem = memref::CastOp::create(rewriter,
-                  loc, fType,
+              mem = memref::CastOp::create(
+                  rewriter, loc, fType,
                   memref::CollapseShapeOp::create(rewriter, loc, mem, reassoc));
             } else {
               mem = memref::CastOp::create(rewriter, loc, fType, mem);
@@ -1396,8 +1400,8 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       assert(isWithCrdLT(lt) && lvl <= trailCOOStart);
       // FIXME: This seems to be unnecessarily complex, can we simplify it?
       if (lvl == trailCOOStart) {
-        Value cooSz = arith::MulIOp::create(rewriter,
-            loc, memSize, constantIndex(rewriter, loc, trailCOORank));
+        Value cooSz = arith::MulIOp::create(
+            rewriter, loc, memSize, constantIndex(rewriter, loc, trailCOORank));
         desc.setCrdMemSize(rewriter, loc, lvl, cooSz);
       } else {
         desc.setCrdMemSize(rewriter, loc, lvl, memSize);
@@ -1471,8 +1475,9 @@ struct SparseDisassembleOpConverter
     // Converts MemRefs back to Tensors.
     SmallVector<Value> retValues = llvm::to_vector(
         llvm::map_range(retMem, [&rewriter, loc](Value v) -> Value {
-          return bufferization::ToTensorOp::create(rewriter,
-              loc, memref::getTensorTypeFromMemRefType(v.getType()), v);
+          return bufferization::ToTensorOp::create(
+              rewriter, loc, memref::getTensorTypeFromMemRefType(v.getType()),
+              v);
         }));
     // Appends the actual memory length used in each buffer returned.
     retValues.append(retLen.begin(), retLen.end());
@@ -1549,15 +1554,15 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Level lvlRank = dstTp.getLvlRank();
     if (dstTp.isOrderedLvl(lvlRank - 1)) {
       Value kFalse = constantI1(rewriter, loc, false);
-      Value notSorted = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::eq, isSorted, kFalse);
+      Value notSorted = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::eq, isSorted, kFalse);
       scf::IfOp ifOp =
           scf::IfOp::create(rewriter, loc, notSorted, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       auto xPerm = rewriter.getMultiDimIdentityMap(lvlRank);
       SortOp::create(rewriter, loc, nse, xs, ValueRange{ys}, xPerm,
-                              rewriter.getIndexAttr(0),
-                              SparseTensorSortKind::HybridQuickSort);
+                     rewriter.getIndexAttr(0),
+                     SparseTensorSortKind::HybridQuickSort);
       rewriter.setInsertionPointAfter(ifOp);
     }
 
@@ -1569,8 +1574,8 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     memref::StoreOp::create(rewriter, loc, posNse, posMemref0, c1);
 
     // Update storage specifier.
-    Value coordinatesSize = arith::MulIOp::create(rewriter,
-        loc, nse, constantIndex(rewriter, loc, lvlRank));
+    Value coordinatesSize = arith::MulIOp::create(
+        rewriter, loc, nse, constantIndex(rewriter, loc, lvlRank));
     desc.setSpecifierField(rewriter, loc, StorageSpecifierKind::CrdMemSize, 0,
                            coordinatesSize);
     desc.setSpecifierField(rewriter, loc, StorageSpecifierKind::ValMemSize,
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 54ea1843b457f..134aef3a6c719 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -168,12 +168,12 @@ static Value genLvlPtrsBuffers(OpBuilder &builder, Location loc,
 
   // Passing in value buffer pointers.
   lvlBarePtrs.push_back(extractBarePtrFromTensor(builder, loc, valTensor));
-  Value idxPtr = memref::ExtractAlignedPointerAsIndexOp::create(builder,
-      loc, allocaBuffer(builder, loc, lvlBarePtrs));
+  Value idxPtr = memref::ExtractAlignedPointerAsIndexOp::create(
+      builder, loc, allocaBuffer(builder, loc, lvlBarePtrs));
   Value idxCast =
       arith::IndexCastOp::create(builder, loc, builder.getI64Type(), idxPtr);
   return LLVM::IntToPtrOp::create(builder, loc, getOpaquePointerType(builder),
-                                          idxCast);
+                                  idxCast);
 }
 
 /// This class abstracts over the API of `_mlir_ciface_newSparseTensor`:
@@ -690,12 +690,12 @@ class SparseTensorExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    linalg::FillOp::create(rewriter,
-        loc, ValueRange{constantZero(rewriter, loc, eltType)},
-        ValueRange{values});
-    linalg::FillOp::create(rewriter,
-        loc, ValueRange{constantZero(rewriter, loc, boolType)},
-        ValueRange{filled});
+    linalg::FillOp::create(rewriter, loc,
+                           ValueRange{constantZero(rewriter, loc, eltType)},
+                           ValueRange{values});
+    linalg::FillOp::create(rewriter, loc,
+                           ValueRange{constantZero(rewriter, loc, boolType)},
+                           ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
     assert(op.getNumResults() == 4);
     rewriter.replaceOp(op, {values, filled, lastLvlCoordinates, zero});
@@ -867,9 +867,9 @@ class SparseTensorDisassembleConverter
     // Converts MemRefs back to Tensors.
     assert(retVal.size() + retLen.size() == op.getNumResults());
     for (unsigned i = 0, sz = retVal.size(); i < sz; i++) {
-      auto tensor = bufferization::ToTensorOp::create(rewriter,
-          loc, memref::getTensorTypeFromMemRefType(retVal[i].getType()),
-          retVal[i]);
+      auto tensor = bufferization::ToTensorOp::create(
+          rewriter, loc,
+          memref::getTensorTypeFromMemRefType(retVal[i].getType()), retVal[i]);
       retVal[i] =
           tensor::CastOp::create(rewriter, loc, op.getResultTypes()[i], tensor);
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index ee963ab5d1ffb..b444ac5ba1285 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -405,8 +405,8 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     inputOps.push_back(op.getDpsInputOperand(1 - other)->get());
     fusedIndexMaps.push_back(fusedIndexMaps.back()); // mimic other
     // Fuse producer and consumer into a new generic op.
-    auto fusedOp = GenericOp::create(rewriter,
-        loc, op.getResult(0).getType(), inputOps, outputOps,
+    auto fusedOp = GenericOp::create(
+        rewriter, loc, op.getResult(0).getType(), inputOps, outputOps,
         rewriter.getAffineMapArrayAttr(fusedIndexMaps), prod.getIteratorTypes(),
         /*doc=*/nullptr, /*library_call=*/nullptr);
     Block &prodBlock = prod.getRegion().front();
@@ -695,8 +695,8 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     // CustomReduce {
     //    x = x REDUC y, identity
     // }
-    auto custom = sparse_tensor::ReduceOp::create(rewriter,
-        loc, rtp, semiring.getResult(), s1, identity);
+    auto custom = sparse_tensor::ReduceOp::create(
+        rewriter, loc, rtp, semiring.getResult(), s1, identity);
     Block *region =
         rewriter.createBlock(&custom.getRegion(), {}, {rtp, rtp}, {loc, loc});
     rewriter.setInsertionPointToStart(&custom.getRegion().front());
@@ -724,8 +724,9 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto stt = getSparseTensorType(tensor);
     // Header with NSE.
     auto nse = NumberOfEntriesOp::create(rewriter, loc, tensor);
-    vector::PrintOp::create(rewriter,
-        loc, rewriter.getStringAttr("---- Sparse Tensor ----\nnse = "));
+    vector::PrintOp::create(
+        rewriter, loc,
+        rewriter.getStringAttr("---- Sparse Tensor ----\nnse = "));
     vector::PrintOp::create(rewriter, loc, nse);
     // Print run-time contents for dim/lvl sizes.
     vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("dim = "));
@@ -745,8 +746,8 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       case SparseTensorFieldKind::PosMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
         vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("pos["));
-        vector::PrintOp::create(rewriter,
-            loc, lvl, vector::PrintPunctuation::NoPunctuation);
+        vector::PrintOp::create(rewriter, loc, lvl,
+                                vector::PrintPunctuation::NoPunctuation);
         vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
         auto pos = ToPositionsOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, pos);
@@ -755,8 +756,8 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       case SparseTensorFieldKind::CrdMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
         vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("crd["));
-        vector::PrintOp::create(rewriter,
-            loc, lvl, vector::PrintPunctuation::NoPunctuation);
+        vector::PrintOp::create(rewriter, loc, lvl,
+                                vector::PrintPunctuation::NoPunctuation);
         vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
         Value crd = nullptr;
         // For COO AoS storage, we want to print a single, linear view of
@@ -771,7 +772,7 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       }
       case SparseTensorFieldKind::ValMemRef: {
         vector::PrintOp::create(rewriter, loc,
-                                         rewriter.getStringAttr("values : "));
+                                rewriter.getStringAttr("values : "));
         auto val = ToValuesOp::create(rewriter, loc, tensor);
         printContents(rewriter, loc, val);
         break;
@@ -827,17 +828,17 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
         Value imag = complex::ImOp::create(rewriter, loc, val);
         vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
         vector::PrintOp::create(rewriter, loc, real,
-                                         vector::PrintPunctuation::Comma);
+                                vector::PrintPunctuation::Comma);
         vector::PrintOp::create(rewriter, loc, imag,
-                                         vector::PrintPunctuation::Close);
+                                vector::PrintPunctuation::Close);
       } else {
-        vector::PrintOp::create(rewriter,
-            loc, val, vector::PrintPunctuation::NoPunctuation);
+        vector::PrintOp::create(rewriter, loc, val,
+                                vector::PrintPunctuation::NoPunctuation);
       }
       // Terminating comma (except at end).
       auto bound = arith::AddIOp::create(rewriter, loc, idxs.back(), step);
-      Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
-                                                  bound, size);
+      Value cond = arith::CmpIOp::create(rewriter, loc,
+                                         arith::CmpIPredicate::ne, bound, size);
       scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, cond, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Comma);
@@ -861,10 +862,10 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
         val = tensor::DimOp::create(rewriter, loc, tensor, idx);
       else
         val = LvlOp::create(rewriter, loc, tensor, idx);
-      vector::PrintOp::create(rewriter,
-          loc, val,
-          i != size - 1 ? vector::PrintPunctuation::Comma
-                        : vector::PrintPunctuation::NoPunctuation);
+      vector::PrintOp::create(rewriter, loc, val,
+                              i != size - 1
+                                  ? vector::PrintPunctuation::Comma
+                                  : vector::PrintPunctuation::NoPunctuation);
     }
     // Close bracket and end of line.
     vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
@@ -920,8 +921,8 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp->getEncoding();
-    ForeachOp foreachOp = ForeachOp::create(rewriter,
-        loc, srcTensor, buffer,
+    ForeachOp foreachOp = ForeachOp::create(
+        rewriter, loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
           const Dimension srcRank = srcTp->getDimRank();
@@ -1025,8 +1026,8 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp.getEncoding();
-    ForeachOp foreachOp = ForeachOp::create(rewriter,
-        loc, srcTensor, buffer,
+    ForeachOp foreachOp = ForeachOp::create(
+        rewriter, loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
           const Dimension dimRank = srcTp.getDimRank();
@@ -1089,12 +1090,12 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
       ReshapeOp reshape;
       if constexpr (std::is_same<ReshapeOp, tensor::ExpandShapeOp>::value) {
-        reshape = ReshapeOp::create(rewriter,
-            loc, denseTp, op.getSrc(), op.getReassociation(),
-            op.getOutputShape(), op.getStaticOutputShape());
+        reshape = ReshapeOp::create(rewriter, loc, denseTp, op.getSrc(),
+                                    op.getReassociation(), op.getOutputShape(),
+                                    op.getStaticOutputShape());
       } else {
         reshape = ReshapeOp::create(rewriter, loc, denseTp, op.getSrc(),
-                                             op.getReassociation());
+                                    op.getReassociation());
       }
       Value convert = ConvertOp::create(rewriter, loc, rtp, reshape);
       rewriter.replaceOp(op, convert);
@@ -1161,18 +1162,20 @@ struct SparseTensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
     SmallVector<Value> maxLvlCrds;
     for (Level l = 0; l < stt->getLvlRank(); l++) {
       Value lvlSz = LvlOp::create(rewriter, loc, op.getSource(), l);
-      Value maxLvlCrd = arith::SubIOp::create(rewriter,
-          loc, lvlSz, constantOne(rewriter, loc, rewriter.getIndexType()));
+      Value maxLvlCrd = arith::SubIOp::create(
+          rewriter, loc, lvlSz,
+          constantOne(rewriter, loc, rewriter.getIndexType()));
       maxLvlCrds.push_back(maxLvlCrd);
     }
 
     AffineExpr lvl2DimExp = stt->getLvlToDim().getResult(*dim);
-    Value maxDimCrd = affine::AffineApplyOp::create(rewriter,
-        op.getLoc(), AffineMap::get(stt->getLvlRank(), 0, lvl2DimExp),
+    Value maxDimCrd = affine::AffineApplyOp::create(
+        rewriter, op.getLoc(), AffineMap::get(stt->getLvlRank(), 0, lvl2DimExp),
         maxLvlCrds);
 
-    Value dimSz = arith::AddIOp::create(rewriter,
-        loc, maxDimCrd, constantOne(rewriter, loc, rewriter.getIndexType()));
+    Value dimSz = arith::AddIOp::create(
+        rewriter, loc, maxDimCrd,
+        constantOne(rewriter, loc, rewriter.getIndexType()));
     rewriter.replaceOp(op, dimSz);
     return success();
   }
@@ -1212,8 +1215,8 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
     for (Value input : op.getInputs()) {
       // Builds a for op for each input tensor to append new values into the
       // output tensor.
-      foreachOp = ForeachOp::create(rewriter,
-          loc, input, iterArg,
+      foreachOp = ForeachOp::create(
+          rewriter, loc, input, iterArg,
           [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
               ValueRange reduc) {
             SmallVector<Value> offDimCrd(dcvs);
@@ -1224,8 +1227,9 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
             dstBuf.val = reduc.front();
             if (!dstTp.isAllDense()) {
               Value cond = genIsNonzero(builder, loc, v);
-              auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
-                                                    /*else*/ true);
+              auto ifOp =
+                  scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
+                                    /*else*/ true);
               builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
               scf::YieldOp::create(builder, loc, dstBuf.val);
 
@@ -1247,7 +1251,7 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
       const Size sz = getSparseTensorType(input).getDynamicDimSize(conDim);
       assert(ShapedType::isStatic(sz));
       offset = arith::AddIOp::create(rewriter, loc, offset,
-                                              constantIndex(rewriter, loc, sz));
+                                     constantIndex(rewriter, loc, sz));
       iterArg = foreachOp.getResult(0);
       dstBuf.val = iterArg;
     }
@@ -1299,8 +1303,8 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
     ValueRange vs;
     TensorLike dstBuf(rewriter, loc, dstStt.getRankedTensorType(), sizes);
 
-    auto foreachOp = ForeachOp::create(rewriter,
-        loc, src, dstBuf.val, foreachOrder,
+    auto foreachOp = ForeachOp::create(
+        rewriter, loc, src, dstBuf.val, foreachOrder,
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
           // Enters the loop, update the SSA value for insertion chain.
@@ -1308,7 +1312,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           if (!skipZeroCheck) {
             Value cond = genIsNonzero(builder, loc, v);
             auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
-                                                  /*else*/ true);
+                                          /*else*/ true);
             builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
             scf::YieldOp::create(builder, loc, dstBuf.val);
 
@@ -1349,8 +1353,8 @@ struct CrdTranslateRewriter : public OpRewritePattern<CrdTranslateOp> {
       // TODO: we should probably expand the affine map to IR using our own
       // rules, since affine.apply assume signed value, while the cooridinates
       // we provided must always be signless.
-      Value trans = affine::AffineApplyOp::create(rewriter,
-          op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
+      Value trans = affine::AffineApplyOp::create(
+          rewriter, op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
           op.getInCrds());
       outCrds.push_back(trans);
     }
@@ -1515,7 +1519,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     sizesForTensor(rewriter, dims, loc, srcTp, src);
     for (Dimension d = 0; d < dimRank; d++) {
       memref::StoreOp::create(rewriter, loc, dims[d], dimSizes,
-                                       constantIndex(rewriter, loc, d));
+                              constantIndex(rewriter, loc, d));
     }
 
     // Create a sparse tensor writer and output meta data.
@@ -1536,13 +1540,13 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     ModuleOp module = op->getParentOfType<ModuleOp>();
 
     // For each element in the source tensor, output the element.
-    ForeachOp::create(rewriter,
-        loc, src, ValueRange(),
+    ForeachOp::create(
+        rewriter, loc, src, ValueRange(),
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
           for (Dimension d = 0; d < dimRank; d++) {
             memref::StoreOp::create(rewriter, loc, dcvs[d], dimCoords,
-                                             constantIndex(builder, loc, d));
+                                    constantIndex(builder, loc, d));
           }
           memref::StoreOp::create(rewriter, loc, v, value);
           SmallVector<Value> operands{writer, rankValue, dimCoords, value};
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 7bccd9a64c9e4..4464450fd328f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -116,10 +116,10 @@ static Value genVectorLoad(PatternRewriter &rewriter, Location loc, VL vl,
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
     return vector::GatherOp::create(rewriter, loc, vtp, mem, scalarArgs,
-                                             indexVec, vmask, pass);
+                                    indexVec, vmask, pass);
   }
   return vector::MaskedLoadOp::create(rewriter, loc, vtp, mem, idxs, vmask,
-                                               pass);
+                                      pass);
 }
 
 /// Generates a vectorized store a[ind[lo:hi]] = rhs or a[lo:hi] = rhs
@@ -133,7 +133,7 @@ static void genVectorStore(PatternRewriter &rewriter, Location loc, Value mem,
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
     vector::ScatterOp::create(rewriter, loc, mem, scalarArgs, indexVec, vmask,
-                                       rhs);
+                              rhs);
     return;
   }
   vector::MaskedStoreOp::create(rewriter, loc, mem, idxs, vmask, rhs);
@@ -198,13 +198,13 @@ static Value genVectorReducInit(PatternRewriter &rewriter, Location loc,
   case vector::CombiningKind::XOR:
     // Initialize reduction vector to: | 0 | .. | 0 | r |
     return vector::InsertOp::create(rewriter, loc, r,
-                                             constantZero(rewriter, loc, vtp),
-                                             constantIndex(rewriter, loc, 0));
+                                    constantZero(rewriter, loc, vtp),
+                                    constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::MUL:
     // Initialize reduction vector to: | 1 | .. | 1 | r |
     return vector::InsertOp::create(rewriter, loc, r,
-                                             constantOne(rewriter, loc, vtp),
-                                             constantIndex(rewriter, loc, 0));
+                                    constantOne(rewriter, loc, vtp),
+                                    constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::AND:
   case vector::CombiningKind::OR:
     // Initialize reduction vector to: | r | .. | r | r |
@@ -300,11 +300,11 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
         Type etp = llvm::cast<VectorType>(vload.getType()).getElementType();
         if (!llvm::isa<IndexType>(etp)) {
           if (etp.getIntOrFloatBitWidth() < 32)
-            vload = arith::ExtUIOp::create(rewriter,
-                loc, vectorType(vl, rewriter.getI32Type()), vload);
+            vload = arith::ExtUIOp::create(
+                rewriter, loc, vectorType(vl, rewriter.getI32Type()), vload);
           else if (etp.getIntOrFloatBitWidth() < 64 && !vl.enableSIMDIndex32)
-            vload = arith::ExtUIOp::create(rewriter,
-                loc, vectorType(vl, rewriter.getI64Type()), vload);
+            vload = arith::ExtUIOp::create(
+                rewriter, loc, vectorType(vl, rewriter.getI64Type()), vload);
         }
         idxs.push_back(vload);
       }
@@ -341,7 +341,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define UNAOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = xxx::create(rewriter, loc, vx);                                    \
+      vexp = xxx::create(rewriter, loc, vx);                                   \
     return true;                                                               \
   }
 
@@ -349,7 +349,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
   if (auto x = dyn_cast<xxx>(def)) {                                           \
     if (codegen) {                                                             \
       VectorType vtp = vectorType(vl, x.getType());                            \
-      vexp = xxx::create(rewriter, loc, vtp, vx);                               \
+      vexp = xxx::create(rewriter, loc, vtp, vx);                              \
     }                                                                          \
     return true;                                                               \
   }
@@ -357,7 +357,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define BINOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = xxx::create(rewriter, loc, vx, vy);                                \
+      vexp = xxx::create(rewriter, loc, vx, vy);                               \
     return true;                                                               \
   }
 
@@ -533,8 +533,8 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       VectorType vtp = vectorType(vl, init.getType());
       Value vinit = genVectorReducInit(rewriter, loc, yield->getOperand(0),
                                        forOp.getRegionIterArg(0), init, vtp);
-      forOpNew = scf::ForOp::create(rewriter,
-          loc, forOp.getLowerBound(), forOp.getUpperBound(), step, vinit);
+      forOpNew = scf::ForOp::create(rewriter, loc, forOp.getLowerBound(),
+                                    forOp.getUpperBound(), step, vinit);
       forOpNew->setAttr(
           LoopEmitter::getLoopEmitterLoopAttrName(),
           forOp->getAttr(LoopEmitter::getLoopEmitterLoopAttrName()));
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
index 29a3e92fcf7fb..0a5f5595bba56 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
@@ -317,7 +317,7 @@ static void genBuffers(CodegenEnv &env, OpBuilder &builder) {
           Value zero = constantZero(builder, loc,
                                     getElementTypeOrSelf(tensor.getType()));
           linalg::FillOp::create(builder, loc, ValueRange{zero},
-                                         ValueRange{init});
+                                 ValueRange{init});
         }
         return init;
       },
@@ -470,9 +470,9 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
   // If statement.
   Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
   Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
-                                             isFilled, fval);
+                                     isFilled, fval);
   scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIndexType(), cond,
-                                             /*else=*/true);
+                                     /*else=*/true);
   // True branch.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   memref::StoreOp::create(builder, loc, tval, filled, index);
@@ -516,7 +516,8 @@ static Value genTensorLoad(CodegenEnv &env, OpBuilder &builder, ExprId exp) {
   if (llvm::isa<TensorType>(ptr.getType())) {
     assert(env.options().sparseEmitStrategy ==
            SparseEmitStrategy::kSparseIterator);
-    return ExtractValOp::create(builder, loc, ptr, llvm::getSingleElement(args));
+    return ExtractValOp::create(builder, loc, ptr,
+                                llvm::getSingleElement(args));
   }
   return memref::LoadOp::create(builder, loc, ptr, args);
 }
@@ -768,7 +769,8 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Type t2 = MemRefType::get(dynShape, builder.getI1Type());
     Type t3 = MemRefType::get(dynShape, builder.getIndexType());
     Type t4 = builder.getIndexType();
-    auto r = ExpandOp::create(builder, loc, TypeRange({t1, t2, t3, t4}), tensor);
+    auto r =
+        ExpandOp::create(builder, loc, TypeRange({t1, t2, t3, t4}), tensor);
     assert(r.getNumResults() == 4);
     env.startExpand(r.getResult(0), r.getResult(1), r.getResult(2),
                     r.getResult(3));
@@ -782,7 +784,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Value count = env.getExpandCount();
     Value chain = env.getInsertionChain();
     Value compress = CompressOp::create(builder, loc, values, filled, added,
-                                                count, chain, indices);
+                                        count, chain, indices);
     env.updateInsertionChain(compress);
     env.endExpand();
   }
@@ -943,12 +945,13 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
           const Value crd = env.emitter().getCoord(tid, *lvl);
           const Value lvar = env.getLoopVar(curr);
           clause = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
-                                                 crd, lvar);
+                                         crd, lvar);
         } else {
           assert(lt.hasDenseSemantic() || isUndefLT(lt));
           clause = constantI1(builder, loc, true);
         }
-        cond = cond ? arith::AndIOp::create(builder, loc, cond, clause) : clause;
+        cond =
+            cond ? arith::AndIOp::create(builder, loc, cond, clause) : clause;
       });
   if (env.isReduc()) {
     types.push_back(env.getReduc().getType());
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
index 27c1b106d71ba..f57f7f7fc0946 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
@@ -204,10 +204,10 @@ Value mlir::sparse_tensor::genIsNonzero(OpBuilder &builder, mlir::Location loc,
   Value zero = constantZero(builder, loc, tp);
   if (isa<FloatType>(tp))
     return arith::CmpFOp::create(builder, loc, arith::CmpFPredicate::UNE, v,
-                                         zero);
+                                 zero);
   if (tp.isIntOrIndex())
     return arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, v,
-                                         zero);
+                                 zero);
   if (isa<ComplexType>(tp))
     return complex::NotEqualOp::create(builder, loc, v, zero);
   llvm_unreachable("Non-numeric type");
@@ -296,7 +296,8 @@ void mlir::sparse_tensor::reshapeCvs(
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
       linear = arith::DivUIOp::create(builder, loc, linear, sizes[j]);
       if (isCollapse) {
-        const Value mul = arith::MulIOp::create(builder, loc, srcCvs[j], linear);
+        const Value mul =
+            arith::MulIOp::create(builder, loc, srcCvs[j], linear);
         val = val ? arith::AddIOp::create(builder, loc, val, mul) : mul;
       } else {
         const Value old = val;
@@ -326,8 +327,8 @@ FlatSymbolRefAttr mlir::sparse_tensor::getFunc(ModuleOp module, StringRef name,
   auto func = module.lookupSymbol<func::FuncOp>(result.getAttr());
   if (!func) {
     OpBuilder moduleBuilder(module.getBodyRegion());
-    func = func::FuncOp::create(moduleBuilder,
-        module.getLoc(), name,
+    func = func::FuncOp::create(
+        moduleBuilder, module.getLoc(), name,
         FunctionType::get(context, operands.getTypes(), resultType));
     func.setPrivate();
     if (static_cast<bool>(emitCInterface))
@@ -490,7 +491,7 @@ void sparse_tensor::foreachInSparseConstant(
     if (isa<ComplexType>(attr.getElementType())) {
       auto valAttr = cast<ArrayAttr>(elems[i].second);
       val = complex::ConstantOp::create(builder, loc, attr.getElementType(),
-                                                valAttr);
+                                        valAttr);
     } else {
       auto valAttr = cast<TypedAttr>(elems[i].second);
       val = arith::ConstantOp::create(builder, loc, valAttr);
@@ -514,7 +515,7 @@ SmallVector<Value> sparse_tensor::loadAll(OpBuilder &builder, Location loc,
   vs.reserve(size);
   for (unsigned i = 0; i < size; i++) {
     Value v = memref::LoadOp::create(builder, loc, mem,
-                                             constantIndex(builder, loc, i));
+                                     constantIndex(builder, loc, i));
     if (i == offsetIdx && offsetVal)
       v = arith::AddIOp::create(builder, loc, v, offsetVal);
     vs.push_back(v);
@@ -538,7 +539,7 @@ void sparse_tensor::storeAll(OpBuilder &builder, Location loc, Value mem,
             ? arith::AddIOp::create(builder, loc, v.value(), offsetVal)
             : v.value();
     memref::StoreOp::create(builder, loc, w, mem,
-                                    constantIndex(builder, loc, v.index()));
+                            constantIndex(builder, loc, v.index()));
   }
 }
 
@@ -609,8 +610,8 @@ Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
     // subsequent clients need the values (DCE will remove unused).
     for (Dimension d = 0; d < dimRank; d++) {
       if (stt.isDynamicDim(d))
-        dimSizesValues[d] = memref::LoadOp::create(builder,
-            loc, dimSizesBuffer, constantIndex(builder, loc, d));
+        dimSizesValues[d] = memref::LoadOp::create(
+            builder, loc, dimSizesBuffer, constantIndex(builder, loc, d));
     }
   }
   return reader;
@@ -687,7 +688,7 @@ Value sparse_tensor::genMapBuffers(
       lvlSz = dimSizesValues[d];
       if (cf != 0)
         lvlSz = arith::DivUIOp::create(builder, loc, lvlSz,
-                                               constantIndex(builder, loc, cf));
+                                       constantIndex(builder, loc, cf));
     } else {
       lvlSz = constantIndex(builder, loc, cm);
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
index 400be54aeb86f..1c10dd5566184 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
@@ -97,8 +97,8 @@ class FuncCallOrInlineGenerator {
       // Create the function if not already exist.
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(getParentOpOf<func::FuncOp>(builder));
-      func = func::FuncOp::create(builder,
-          loc, funcName,
+      func = func::FuncOp::create(
+          builder, loc, funcName,
           FunctionType::get(context, params.getTypes(), retTypes));
       func.setPrivate();
       // Set the insertion point to the body of the function.
@@ -400,8 +400,8 @@ inline Value constantLevelTypeEncoding(OpBuilder &builder, Location loc,
 inline Value genValFromAttr(OpBuilder &builder, Location loc, Attribute attr) {
   if (auto complexAttr = dyn_cast<complex::NumberAttr>(attr)) {
     Type tp = cast<ComplexType>(complexAttr.getType()).getElementType();
-    return complex::ConstantOp::create(builder,
-        loc, complexAttr.getType(),
+    return complex::ConstantOp::create(
+        builder, loc, complexAttr.getType(),
         builder.getArrayAttr({FloatAttr::get(tp, complexAttr.getReal()),
                               FloatAttr::get(tp, complexAttr.getImag())}));
   }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
index 6eaeff91dd5c6..659282a995123 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
@@ -25,7 +25,7 @@ using namespace mlir::sparse_tensor;
 //===----------------------------------------------------------------------===//
 
 #define CMPI(p, l, r)                                                          \
-  (arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::p, (l), (r))       \
+  (arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::p, (l), (r))      \
        .getResult())
 
 #define C_IDX(v) (constantIndex(builder, loc, (v)))
@@ -45,8 +45,8 @@ using namespace mlir::sparse_tensor;
 #ifndef NDEBUG
 LLVM_ATTRIBUTE_UNUSED static void dumpIndexMemRef(OpBuilder &builder,
                                                   Location loc, Value memref) {
-  memref = memref::CastOp::create(builder,
-      loc, UnrankedMemRefType::get(builder.getIndexType(), 0), memref);
+  memref = memref::CastOp::create(
+      builder, loc, UnrankedMemRefType::get(builder.getIndexType(), 0), memref);
   createFuncCall(builder, loc, "printMemrefInd", TypeRange{},
                  ValueRange{memref}, EmitCInterface::On);
 }
@@ -604,11 +604,11 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
       // Extract and iterate over the iteration space.
       ExtractIterSpaceOp extractSpaceOp =
           lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
-                   : ExtractIterSpaceOp::create(builder,
-                         loc, t, spIterVals[tid][lvl - 1], lvl);
+                   : ExtractIterSpaceOp::create(builder, loc, t,
+                                                spIterVals[tid][lvl - 1], lvl);
 
-      IterateOp iterOp = IterateOp::create(builder,
-          loc, extractSpaceOp.getExtractedSpace(), reduc);
+      IterateOp iterOp = IterateOp::create(
+          builder, loc, extractSpaceOp.getExtractedSpace(), reduc);
       spIterVals[tid][lvl] = iterOp.getIterator();
 
       // Update the reduction varaibles.
@@ -626,8 +626,8 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
       Value t = tensors[tid];
       ExtractIterSpaceOp extractSpaceOp =
           lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
-                   : ExtractIterSpaceOp::create(builder,
-                         loc, t, spIterVals[tid][lvl - 1], lvl);
+                   : ExtractIterSpaceOp::create(builder, loc, t,
+                                                spIterVals[tid][lvl - 1], lvl);
       spaces.push_back(extractSpaceOp.getExtractedSpace());
     }
     auto coIterOp = CoIterateOp::create(builder, loc, spaces, reduc, numCases);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
index 47fa3f950afd3..3b3b0aadf638c 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
@@ -78,15 +78,16 @@ SparseTensorTypeToBufferConverter::SparseTensorTypeToBufferConverter() {
 
 Value SparseTensorSpecifier::getInitValue(OpBuilder &builder, Location loc,
                                           SparseTensorType stt) {
-  return StorageSpecifierInitOp::create(builder,
-      loc, StorageSpecifierType::get(stt.getEncoding()));
+  return StorageSpecifierInitOp::create(
+      builder, loc, StorageSpecifierType::get(stt.getEncoding()));
 }
 
 Value SparseTensorSpecifier::getSpecifierField(OpBuilder &builder, Location loc,
                                                StorageSpecifierKind kind,
                                                std::optional<Level> lvl) {
-  return GetStorageSpecifierOp::create(builder,
-      loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl));
+  return GetStorageSpecifierOp::create(
+      builder, loc, specifier, kind,
+      optionalLevelAttr(specifier.getContext(), lvl));
 }
 
 void SparseTensorSpecifier::setSpecifierField(OpBuilder &builder, Location loc,
@@ -95,8 +96,9 @@ void SparseTensorSpecifier::setSpecifierField(OpBuilder &builder, Location loc,
                                               std::optional<Level> lvl) {
   // TODO: make `v` have type `TypedValue<IndexType>` instead.
   assert(v.getType().isIndex());
-  specifier = SetStorageSpecifierOp::create(builder,
-      loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl), v);
+  specifier = SetStorageSpecifierOp::create(
+      builder, loc, specifier, kind,
+      optionalLevelAttr(specifier.getContext(), lvl), v);
 }
 
 //===----------------------------------------------------------------------===//
@@ -112,8 +114,8 @@ Value sparse_tensor::SparseTensorDescriptor::getCrdMemRefOrView(
   Value stride = constantIndex(builder, loc, rType.getLvlRank() - cooStart);
   Value size = getCrdMemSize(builder, loc, cooStart);
   size = arith::DivUIOp::create(builder, loc, size, stride);
-  return memref::SubViewOp::create(builder,
-      loc, getMemRefField(SparseTensorFieldKind::CrdMemRef, cooStart),
+  return memref::SubViewOp::create(
+      builder, loc, getMemRefField(SparseTensorFieldKind::CrdMemRef, cooStart),
       /*offset=*/ValueRange{constantIndex(builder, loc, lvl - cooStart)},
       /*size=*/ValueRange{size},
       /*step=*/ValueRange{stride});
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
index 212ee99647167..46d0baac58f06 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
@@ -22,7 +22,7 @@ using ValueTuple = std::tuple<Value, Value, Value>;
 // File local helper functions/macros.
 //===----------------------------------------------------------------------===//
 #define CMPI(p, lhs, rhs)                                                      \
-  (arith::CmpIOp::create(b, l, arith::CmpIPredicate::p, (lhs), (rhs))           \
+  (arith::CmpIOp::create(b, l, arith::CmpIPredicate::p, (lhs), (rhs))          \
        .getResult())
 
 #define C_FALSE (constantI1(b, l, false))
@@ -732,8 +732,8 @@ class NonEmptySubSectIterator : public SparseIterator {
   //      [itVal0, itVal1, ..., pNx0],
   //      ...]
   Value allocSubSectPosBuf(OpBuilder &b, Location l) {
-    return memref::AllocaOp::create(b,
-        l,
+    return memref::AllocaOp::create(
+        b, l,
         MemRefType::get({ShapedType::kDynamic, tupleSz + 1}, b.getIndexType()),
         maxTupleCnt);
   }
@@ -741,12 +741,12 @@ class NonEmptySubSectIterator : public SparseIterator {
   void storeNxLvlStart(OpBuilder &b, Location l, Value tupleId,
                        Value start) const {
     memref::StoreOp::create(b, l, start, subSectPosBuf,
-                              ValueRange{tupleId, C_IDX(tupleSz)});
+                            ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
   Value loadNxLvlStart(OpBuilder &b, Location l, Value tupleId) const {
     return memref::LoadOp::create(b, l, subSectPosBuf,
-                                    ValueRange{tupleId, C_IDX(tupleSz)});
+                                  ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
   void storeCursorVals(OpBuilder &b, Location l, Value tupleId,
@@ -754,7 +754,7 @@ class NonEmptySubSectIterator : public SparseIterator {
     assert(itVals.size() == tupleSz);
     for (unsigned i = 0; i < tupleSz; i++) {
       memref::StoreOp::create(b, l, itVals[i], subSectPosBuf,
-                                ValueRange{tupleId, C_IDX(i)});
+                              ValueRange{tupleId, C_IDX(i)});
     }
   }
 
@@ -763,7 +763,7 @@ class NonEmptySubSectIterator : public SparseIterator {
     SmallVector<Value> ret;
     for (unsigned i = 0; i < tupleSz; i++) {
       Value v = memref::LoadOp::create(b, l, subSectPosBuf,
-                                         ValueRange{tupleId, C_IDX(i)});
+                                       ValueRange{tupleId, C_IDX(i)});
       ret.push_back(v);
     }
     return ret;
@@ -1058,8 +1058,8 @@ ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
 }
 
 Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
-  auto whileOp = scf::WhileOp::create(b,
-      l, pos.getType(), pos,
+  auto whileOp = scf::WhileOp::create(
+      b, l, pos.getType(), pos,
       /*beforeBuilder=*/
       [this, pos](OpBuilder &b, Location l, ValueRange ivs) {
         Value inBound = CMPI(ult, ivs.front(), posHi);
@@ -1137,8 +1137,8 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
 
   SmallVector<Value> whileArgs(getCursor().begin(), getCursor().end());
   whileArgs.push_back(isFirst);
-  auto whileOp = scf::WhileOp::create(b,
-      l, ValueRange(whileArgs).getTypes(), whileArgs,
+  auto whileOp = scf::WhileOp::create(
+      b, l, ValueRange(whileArgs).getTypes(), whileArgs,
       /*beforeBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
         ValueRange isFirst = linkNewScope(ivs);
@@ -1219,8 +1219,8 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
     SmallVector<Value> iterArgs;
     iterArgs.push_back(C_IDX(0));
     iterArgs.append(reduc.begin(), reduc.end());
-    auto forEachLeaf = scf::ForOp::create(b,
-        l, /*lb=*/C_IDX(0), /*ub=*/tupleCnt, /*step=*/C_IDX(1), iterArgs,
+    auto forEachLeaf = scf::ForOp::create(
+        b, l, /*lb=*/C_IDX(0), /*ub=*/tupleCnt, /*step=*/C_IDX(1), iterArgs,
         [&helper, &builder](OpBuilder &b, Location l, Value tupleId,
                             ValueRange iterArgs) {
           // Deserialize the iterator at the cached position (tupleId).
@@ -1235,8 +1235,8 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
           SmallVector<Value> whileArgs(helper.wrap.getCursor());
           whileArgs.append(iterArgs.begin(), iterArgs.end());
 
-          auto whileOp = scf::WhileOp::create(b,
-              l, ValueRange(whileArgs).getTypes(), whileArgs,
+          auto whileOp = scf::WhileOp::create(
+              b, l, ValueRange(whileArgs).getTypes(), whileArgs,
               /*beforeBuilder=*/
               [&helper](OpBuilder &b, Location l, ValueRange ivs) {
                 helper.wrap.linkNewScope(ivs);
@@ -1267,8 +1267,8 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
                                      ValueRange reduc) {
     assert(!parent || parent->lvl + 1 == lvl);
     delegate->genInit(b, l, parent);
-    auto forOp = scf::ForOp::create(b,
-        l, /*lb=*/C_IDX(0), /*ub=*/subSectSz, /*step=*/C_IDX(1), reduc,
+    auto forOp = scf::ForOp::create(
+        b, l, /*lb=*/C_IDX(0), /*ub=*/subSectSz, /*step=*/C_IDX(1), reduc,
         [&](OpBuilder &b, Location l, Value crd, ValueRange iterArgs) {
           helper.locate(b, l, crd);
           scf::ValueVector nx = builder(b, l, &helper.wrap, iterArgs);
@@ -1458,8 +1458,8 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 return genWhenInBound(b, l, *delegate, /*elseRet=*/iterArgs,
                                       [nxMin](OpBuilder &b, Location l,
                                               Value crd) -> scf::ValueVector {
-                                        Value nx = arith::MinUIOp::create(b,
-                                            l, crd, nxMin);
+                                        Value nx = arith::MinUIOp::create(
+                                            b, l, crd, nxMin);
                                         return {nx, C_TRUE};
                                       });
               });
@@ -1581,8 +1581,9 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &b, Location l, Value t,
   auto stt = getSparseTensorType(t);
 
   LevelType lt = stt.getLvlType(lvl);
-  Value sz = stt.hasEncoding() ? LvlOp::create(b, l, t, lvl).getResult()
-                               : tensor::DimOp::create(b, l, t, lvl).getResult();
+  Value sz = stt.hasEncoding()
+                 ? LvlOp::create(b, l, t, lvl).getResult()
+                 : tensor::DimOp::create(b, l, t, lvl).getResult();
 
   SmallVector<Value, 2> buffers;
   if (lt.isWithPosLT()) {
diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
index fb68620c378e5..5847fecc45404 100644
--- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
@@ -1666,10 +1666,10 @@ Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
   case TensorExp::Kind::kNegC:
     return complex::NegOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegI: // no negi in std
-    return arith::SubIOp::create(rewriter,
-        loc,
+    return arith::SubIOp::create(
+        rewriter, loc,
         arith::ConstantOp::create(rewriter, loc, v0.getType(),
-                                           rewriter.getZeroAttr(v0.getType())),
+                                  rewriter.getZeroAttr(v0.getType())),
         v0);
   case TensorExp::Kind::kTruncF:
     return arith::TruncFOp::create(rewriter, loc, inferType(e, v0), v0);
diff --git a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
index 78eb50cbdf540..7e4a5acb9867d 100644
--- a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
+++ b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
@@ -78,8 +78,8 @@ struct CreatorOpShardingInterface
             device =
                 mesh::ProcessMultiIndexOp::create(builder, op->getLoc(), mesh)
                     .getResults();
-            shapeForDevice = mesh::ShardShapeOp::create(builder,
-                op->getLoc(), oldType.getShape(), spmdizedOperands,
+            shapeForDevice = mesh::ShardShapeOp::create(
+                builder, op->getLoc(), oldType.getShape(), spmdizedOperands,
                 newSharding->getResult(0), device);
           }
           newOperands.emplace_back(shapeForDevice.getResult()[i]);
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 0a4fa7bfa007f..124a63281a37c 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -207,8 +207,8 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     if (isZeroInteger(newLength)) {
       hasZeroLen = true;
     } else if (!hasZeroLen) {
-      Value check = arith::CmpIOp::create(b,
-          loc, arith::CmpIPredicate::eq,
+      Value check = arith::CmpIOp::create(
+          b, loc, arith::CmpIPredicate::eq,
           getValueOrCreateConstantIndexOp(b, loc, newLength),
           getValueOrCreateConstantIndexOp(b, loc, zero));
       dynHasZeroLenCond =
@@ -245,8 +245,8 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // have a dimension of size 0, the semantics of which is unclear.)
   auto createGenerateOp = [&]() {
     // Create GenerateOp.
-    auto generateOp = tensor::GenerateOp::create(b,
-        loc, resultType, dynDims,
+    auto generateOp = tensor::GenerateOp::create(
+        b, loc, resultType, dynDims,
         [&](OpBuilder &builder, Location gLoc, ValueRange indices) {
           tensor::YieldOp::create(builder, gLoc, padValue);
         });
@@ -257,10 +257,10 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // the result shape of the new SliceOp has a zero dimension.
   auto createPadOfExtractSlice = [&]() {
     // Create pad(extract_slice(x)).
-    auto newSliceOp = tensor::ExtractSliceOp::create(b,
-        loc, padOp.getSource(), newOffsets, newLengths, newStrides);
-    auto newPadOp = PadOp::create(b,
-        loc, Type(), newSliceOp, newLows, newHighs,
+    auto newSliceOp = tensor::ExtractSliceOp::create(
+        b, loc, padOp.getSource(), newOffsets, newLengths, newStrides);
+    auto newPadOp = PadOp::create(
+        b, loc, Type(), newSliceOp, newLows, newHighs,
         /*nofold=*/padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
 
@@ -287,8 +287,8 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     Operation *thenOp;
     Operation *elseOp;
     Operation *sliceOp;
-    auto result = scf::IfOp::create(b,
-        loc, dynHasZeroLenCond,
+    auto result = scf::IfOp::create(
+        b, loc, dynHasZeroLenCond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
           thenOp = createGenerateOp();
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index e5b4b7a865d87..bc11e567fa2d8 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -222,8 +222,8 @@ struct CollapseShapeOpInterface
           MemRefType::get(collapseShapeOp.getSrcType().getShape(),
                           collapseShapeOp.getSrcType().getElementType(),
                           AffineMap(), bufferType.getMemorySpace());
-      buffer = bufferization::ToBufferOp::create(rewriter,
-          op->getLoc(), memrefType, *tensorAlloc);
+      buffer = bufferization::ToBufferOp::create(rewriter, op->getLoc(),
+                                                 memrefType, *tensorAlloc);
     }
 
     // Result type is inferred by the builder.
@@ -349,8 +349,8 @@ struct ExpandShapeOpInterface
     if (failed(buffer))
       return failure();
 
-    auto memrefExpandShape = memref::ExpandShapeOp::create(rewriter,
-        op->getLoc(), tensorResultType.getShape(), *buffer,
+    auto memrefExpandShape = memref::ExpandShapeOp::create(
+        rewriter, op->getLoc(), tensorResultType.getShape(), *buffer,
         expandShapeOp.getReassociationIndices(),
         expandShapeOp.getMixedOutputShape());
     replaceOpWithBufferizedValues(rewriter, op,
@@ -398,8 +398,8 @@ struct ExtractSliceOpInterface
         extractSliceOp.getResult(), options, state);
     if (failed(resultMemrefType))
       return failure();
-    Value subView = memref::SubViewOp::create(rewriter,
-        loc, llvm::cast<MemRefType>(*resultMemrefType), *srcMemref,
+    Value subView = memref::SubViewOp::create(
+        rewriter, loc, llvm::cast<MemRefType>(*resultMemrefType), *srcMemref,
         mixedOffsets, mixedSizes, mixedStrides);
 
     replaceOpWithBufferizedValues(rewriter, op, subView);
@@ -507,8 +507,8 @@ struct FromElementsOpInterface
         bufferization::getBufferType(*tensorAlloc, options, state);
     if (failed(memrefType))
       return failure();
-    Value buffer = bufferization::ToBufferOp::create(rewriter,
-        op->getLoc(), *memrefType, *tensorAlloc);
+    Value buffer = bufferization::ToBufferOp::create(rewriter, op->getLoc(),
+                                                     *memrefType, *tensorAlloc);
 
     // Case: tensor<0xelem_type>.
     if (fromElementsOp.getElements().empty()) {
@@ -518,8 +518,8 @@ struct FromElementsOpInterface
 
     // Case: tensor<elem_type>.
     if (shape.empty()) {
-      memref::StoreOp::create(rewriter,
-          loc, fromElementsOp.getElements().front(), buffer);
+      memref::StoreOp::create(rewriter, loc,
+                              fromElementsOp.getElements().front(), buffer);
       replaceOpWithBufferizedValues(rewriter, op, buffer);
       return success();
     }
@@ -577,7 +577,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   OpBuilder::InsertionGuard g(rewriter);
   auto linalgOp =
       linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
-                                     /*init=*/tensorDestination);
+                            /*init=*/tensorDestination);
   Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
   // Create linalg::IndexOps.
@@ -645,7 +645,7 @@ struct InsertOpInterface
     if (failed(destMemref))
       return failure();
     memref::StoreOp::create(rewriter, insertOp.getLoc(), insertOp.getScalar(),
-                                     *destMemref, insertOp.getIndices());
+                            *destMemref, insertOp.getIndices());
     replaceOpWithBufferizedValues(rewriter, op, *destMemref);
     return success();
   }
@@ -713,9 +713,9 @@ struct InsertSliceOpInterface
         memref::SubViewOp::inferRankReducedResultType(
             insertSliceOp.getSourceType().getShape(), dstMemrefType,
             mixedOffsets, mixedSizes, mixedStrides);
-    Value subView = memref::SubViewOp::create(rewriter,
-        loc, subviewMemRefType, *dstMemref, mixedOffsets, mixedSizes,
-        mixedStrides);
+    Value subView =
+        memref::SubViewOp::create(rewriter, loc, subviewMemRefType, *dstMemref,
+                                  mixedOffsets, mixedSizes, mixedStrides);
 
     // Copy tensor. If this tensor.insert_slice has a matching
     // tensor.extract_slice, the copy operation will eventually fold away.
@@ -802,8 +802,8 @@ struct PadOpInterface
       AffineExpr s0, s1, s2;
       bindSymbols(op->getContext(), s0, s1, s2);
       AffineExpr sumExpr = s0 + s1 + s2;
-      Value sum = affine::AffineApplyOp::create(rewriter,
-          loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
+      Value sum = affine::AffineApplyOp::create(
+          rewriter, loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
       dynamicSizes.push_back(sum);
     }
 
@@ -995,9 +995,9 @@ struct ParallelInsertSliceOpInterface
             parallelInsertSliceOp.getMixedOffsets(),
             parallelInsertSliceOp.getMixedSizes(),
             parallelInsertSliceOp.getMixedStrides());
-    Value subview = memref::SubViewOp::create(rewriter,
-        parallelInsertSliceOp.getLoc(), subviewMemRefType, *destBuffer,
-        parallelInsertSliceOp.getMixedOffsets(),
+    Value subview = memref::SubViewOp::create(
+        rewriter, parallelInsertSliceOp.getLoc(), subviewMemRefType,
+        *destBuffer, parallelInsertSliceOp.getMixedOffsets(),
         parallelInsertSliceOp.getMixedSizes(),
         parallelInsertSliceOp.getMixedStrides());
 
@@ -1065,9 +1065,9 @@ struct SplatOpInterface
     if (options.defaultMemorySpaceFn(tensorType) != Attribute())
       return op->emitError("memory space not implemented yet");
 
-    auto linalgOp =
-        linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
-                                       /*init=*/*tensorAlloc);
+    auto linalgOp = linalg::MapOp::create(rewriter, loc, tensorType,
+                                          /*inputs=*/ValueRange(),
+                                          /*init=*/*tensorAlloc);
     Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
     // Create linalg::IndexOps.
@@ -1126,8 +1126,8 @@ struct ConcatOpInterface
     MemRefType memrefType =
         MemRefType::get(concatOp.getResultType().getShape(),
                         concatOp.getResultType().getElementType(), layout);
-    Value dstBuffer = bufferization::ToBufferOp::create(rewriter,
-        op->getLoc(), memrefType, *tensorAlloc);
+    Value dstBuffer = bufferization::ToBufferOp::create(
+        rewriter, op->getLoc(), memrefType, *tensorAlloc);
 
     // Extract the dimension for the concat op
     uint64_t concatDim = concatOp.getDim();
@@ -1174,8 +1174,9 @@ struct ConcatOpInterface
 
       if (dynamicConcatDim) {
         offsets[concatDim] = dynamicOffset.value();
-        dynamicSize = memref::DimOp::create(rewriter, loc, *srcBuffer, concatDim)
-                          .getResult();
+        dynamicSize =
+            memref::DimOp::create(rewriter, loc, *srcBuffer, concatDim)
+                .getResult();
         sizes[concatDim] = dynamicSize.value();
       } else {
         sizes[concatDim] = rewriter.getIndexAttr(operandConcatDimSize);
@@ -1188,16 +1189,16 @@ struct ConcatOpInterface
           memref::SubViewOp::inferRankReducedResultType(
               operandTensorType.getShape(), dstMemrefType, offsets, sizes,
               strides);
-      Value subview = memref::SubViewOp::create(rewriter,
-          loc, subviewMemRefType, dstBuffer, offsets, sizes, strides);
+      Value subview = memref::SubViewOp::create(
+          rewriter, loc, subviewMemRefType, dstBuffer, offsets, sizes, strides);
 
       // Copy the source buffer into the destination subview.
       if (failed(options.createMemCpy(rewriter, loc, *srcBuffer, subview)))
         return failure();
 
       if (dynamicConcatDim) {
-        dynamicOffset = arith::AddIOp::create(rewriter,
-            loc, dynamicOffset.value(), dynamicSize.value());
+        dynamicOffset = arith::AddIOp::create(
+            rewriter, loc, dynamicOffset.value(), dynamicSize.value());
       } else {
         concatDimOffset += operandConcatDimSize;
       }
diff --git a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
index 24bcb8192f672..670865de6031f 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
@@ -42,8 +42,9 @@ struct FoldEmptyTensorWithReshapeOp : public OpRewritePattern<ReshapeOp> {
 
     // Create new tensor.empty op.
     // TODO: Do not drop tensor type encoding.
-    Value emptyTensor = EmptyOp::create(rewriter,
-        loc, resultShapes[0], reshapeOp.getResultType().getElementType());
+    Value emptyTensor =
+        EmptyOp::create(rewriter, loc, resultShapes[0],
+                        reshapeOp.getResultType().getElementType());
     if (emptyTensor.getType() != reshapeOp.getResultType()) {
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
           reshapeOp, reshapeOp.getResultType(), emptyTensor);
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
index c3801e0f05c88..840b4f3121428 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
@@ -142,15 +142,15 @@ tensor::ExtractSliceFromCollapseHelper::emitLoopNestBody(
   SmallVector<Range> extractParams =
       helper.getExtractSliceParams(builder.getContext(), multiIndices);
 
-  Value subTileResult = tensor::ExtractSliceOp::create(builder,
-      loc, collapseShapeOp.getSrc(), extractParams);
+  Value subTileResult = tensor::ExtractSliceOp::create(
+      builder, loc, collapseShapeOp.getSrc(), extractParams);
 
   SmallVector<Range> insertParams =
       helper.getInsertSliceParams(builder.getContext(), tileInductionVars);
 
   // Collapse the dimensions of the source slice back down.
-  Value collapsedResult = tensor::CollapseShapeOp::create(builder,
-      loc, subTileResult, reassociationIndices);
+  Value collapsedResult = tensor::CollapseShapeOp::create(
+      builder, loc, subTileResult, reassociationIndices);
   return std::make_pair(collapsedResult, insertParams);
 }
 
@@ -173,8 +173,9 @@ tensor::simplifyCollapseShapeWithRankReducingExtractSlice(
   SmallVector<OpFoldResult> sizes =
       tensor::getMixedSizes(rewriter, op.getLoc(), op.getSrc());
   SmallVector<OpFoldResult> strides(sourceType.getRank(), one);
-  auto sliceOp = tensor::ExtractSliceOp::create(rewriter,
-      op.getLoc(), info->sliceResultType, op.getSrc(), offsets, sizes, strides);
+  auto sliceOp = tensor::ExtractSliceOp::create(
+      rewriter, op.getLoc(), info->sliceResultType, op.getSrc(), offsets, sizes,
+      strides);
 
   if (!info->newReassociationIndices.has_value()) {
     rewriter.replaceOp(op, sliceOp.getResult());
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
index cc32f1193a55b..d76c02af7ab16 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
@@ -112,9 +112,9 @@ TransferReadOfExtractSliceOpFolder::matchAndRewriteMaskableOp(
       extractSliceOp.getMixedStrides(), extractSliceOp.getDroppedDims(),
       indices, sourceIndices);
 
-  Operation *newOp = vector::TransferReadOp::create(rewriter,
-      readOp.getLoc(), readOp.getVectorType(), extractSliceOp.getSource(),
-      sourceIndices,
+  Operation *newOp = vector::TransferReadOp::create(
+      rewriter, readOp.getLoc(), readOp.getVectorType(),
+      extractSliceOp.getSource(), sourceIndices,
       AffineMapAttr::get(expandDimsToRank(
           readOp.getPermutationMap(), extractSliceOp.getSourceType().getRank(),
           extractSliceOp.getDroppedDims())),
diff --git a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
index a6db540ffd8a0..43d9d704a29ff 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
@@ -64,9 +64,10 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     return padOp.getResult();
 
   // Create a new tensor::PadOp.
-  auto newPadOp = PadOp::create(b,
-      loc, padOp.getResultType(), padOp.getSource(), newMixedLow, newMixedHigh,
-      constantPadding, padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
+  auto newPadOp =
+      PadOp::create(b, loc, padOp.getResultType(), padOp.getSource(),
+                    newMixedLow, newMixedHigh, constantPadding,
+                    padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
 
   // Create a tensor::ExtractSliceOp.
   // Reify the result sizes of the old tensor::PadOp.
@@ -83,10 +84,10 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
       offsets.push_back(b.getIndexAttr(0));
     } else {
       offsets.push_back(
-          affine::AffineApplyOp::create(b,
-               loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
-               std::initializer_list<Value>{cast<Value>(newMixedLow[i]),
-                                            cast<Value>(prevLow)})
+          affine::AffineApplyOp::create(
+              b, loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
+              std::initializer_list<Value>{cast<Value>(newMixedLow[i]),
+                                           cast<Value>(prevLow)})
               .getResult());
     }
     // size = reified result size
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index 3b63c87f2c05a..e0af2f77d44b8 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -76,8 +76,8 @@ struct FoldUnPaddingCollapseIntoExtract
       return rewriter.notifyMatchFailure(collapseShapeOp,
                                          "expected unpadding collapse");
 
-    Value unPaddedExtractSlice = tensor::ExtractSliceOp::create(rewriter,
-        extractSliceOp.getLoc(), collapseShapeOp.getResultType(),
+    Value unPaddedExtractSlice = tensor::ExtractSliceOp::create(
+        rewriter, extractSliceOp.getLoc(), collapseShapeOp.getResultType(),
         extractSliceOp.getSource(), extractSliceOp.getMixedOffsets(),
         extractSliceOp.getMixedSizes(), extractSliceOp.getMixedStrides());
     rewriter.replaceOp(collapseShapeOp, unPaddedExtractSlice);
@@ -270,8 +270,8 @@ struct BubbleUpExpandThroughParallelCollapse
     // matches the number of dimensions of the result, then the expand_shape
     // is a no-op.
     if (newExpandReInds.size() != newExpandSizes.size()) {
-      newCollapseSrc = tensor::ExpandShapeOp::create(rewriter,
-          loc, expandResultType, newCollapseSrc, newExpandReInds,
+      newCollapseSrc = tensor::ExpandShapeOp::create(
+          rewriter, loc, expandResultType, newCollapseSrc, newExpandReInds,
           newExpandSizes);
     }
 
@@ -280,8 +280,8 @@ struct BubbleUpExpandThroughParallelCollapse
     // is a no-op.
     Value replacement = newCollapseSrc;
     if (newCollapseReInds.size() != newExpandSizes.size()) {
-      replacement = tensor::CollapseShapeOp::create(rewriter,
-          loc, newCollapseSrc, newCollapseReInds);
+      replacement = tensor::CollapseShapeOp::create(
+          rewriter, loc, newCollapseSrc, newCollapseReInds);
     }
     rewriter.replaceOp(expandOp, replacement);
     return success();
@@ -405,8 +405,8 @@ struct BubbleUpExpandShapeThroughExtractSlice
         shape, expandShapeOp.getResultType().getElementType());
 
     // Create a new ExtractSliceOp and ExpandShapeOp.
-    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
-        loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
+    Value newSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
         collapsedStrides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSliceOp,
@@ -735,9 +735,9 @@ struct BubbleUpCollapseShapeThroughExtractSlice
                              groupExpandedOffsets.rend());
     }
 
-    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
-        collapseShapeOp->getLoc(), collapseShapeOp.getSrc(), expandedOffsets,
-        expandedSizes, expandedStrides);
+    Value newSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, collapseShapeOp->getLoc(), collapseShapeOp.getSrc(),
+        expandedOffsets, expandedSizes, expandedStrides);
     rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
         sliceOp, sliceOp.getResultType(), newSliceOp,
         collapseShapeOp.getReassociationIndices());
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
index 7c1dea45843d5..838ff1f987c63 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
@@ -50,12 +50,11 @@ struct CastOpInterface
       Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
           arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
-      Value isSameRank = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      cf::AssertOp::create(builder,
-          loc, isSameRank,
-          RuntimeVerifiableOpInterface::generateErrorMessage(op,
-                                                             "rank mismatch"));
+      Value isSameRank = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcRank, resultRank);
+      cf::AssertOp::create(builder, loc, isSameRank,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "rank mismatch"));
     }
 
     // Check dimension sizes.
@@ -73,10 +72,10 @@ struct CastOpInterface
           DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
           arith::ConstantIndexOp::create(builder, loc, it.value());
-      Value isSameSz = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      cf::AssertOp::create(builder,
-          loc, isSameSz,
+      Value isSameSz = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
+      cf::AssertOp::create(
+          builder, loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
     }
@@ -91,8 +90,9 @@ struct DimOpInterface
     auto dimOp = cast<DimOp>(op);
     Value rank = RankOp::create(builder, loc, dimOp.getSource());
     Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
-    cf::AssertOp::create(builder,
-        loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
+    cf::AssertOp::create(
+        builder, loc,
+        generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
   }
@@ -134,10 +134,9 @@ struct ExtractInsertOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    cf::AssertOp::create(builder,
-        loc, assertCond,
-        RuntimeVerifiableOpInterface::generateErrorMessage(
-            op, "out-of-bounds access"));
+    cf::AssertOp::create(builder, loc, assertCond,
+                         RuntimeVerifiableOpInterface::generateErrorMessage(
+                             op, "out-of-bounds access"));
   }
 };
 
@@ -167,8 +166,8 @@ struct ExtractSliceOpInterface
           loc, extractSliceOp.getSource(), i);
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      cf::AssertOp::create(builder,
-          loc, offsetInBounds,
+      cf::AssertOp::create(
+          builder, loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
@@ -180,8 +179,8 @@ struct ExtractSliceOpInterface
           arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      cf::AssertOp::create(builder,
-          loc, lastPosInBounds,
+      cf::AssertOp::create(
+          builder, loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "extract_slice runs out-of-bounds along dimension " +
                       std::to_string(i)));
diff --git a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
index 21f70e52b8706..b6fdba360deea 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
@@ -59,8 +59,8 @@ struct InsertSliceLikeOpSubsetInsertionOpInterface
   Value buildSubsetExtraction(Operation *op, OpBuilder &builder,
                               Location loc) const {
     auto insertSliceOp = cast<OpTy>(op);
-    auto extractOp = tensor::ExtractSliceOp::create(builder,
-        loc, insertSliceOp.getSourceType(), insertSliceOp.getDest(),
+    auto extractOp = tensor::ExtractSliceOp::create(
+        builder, loc, insertSliceOp.getSourceType(), insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
         insertSliceOp.getMixedStrides());
     return extractOp.getResult();
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 7ca4f6496ccdc..967f027915e42 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -845,9 +845,9 @@ struct PadSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
         getTosaConstShape(rewriter, sliceOp.getLoc(), newPadPaddings);
     auto newPadTy =
         RankedTensorType::get(newPadShape, inputTy.getElementType());
-    auto newPadOp = tosa::PadOp::create(rewriter,
-        padOp.getLoc(), newPadTy, padOp.getInput1(), newPaddingsOp,
-        padOp.getPadConst());
+    auto newPadOp = tosa::PadOp::create(rewriter, padOp.getLoc(), newPadTy,
+                                        padOp.getInput1(), newPaddingsOp,
+                                        padOp.getPadConst());
 
     // Update SliceOp and point to new PadOp
     auto newStartOp =
@@ -897,9 +897,9 @@ struct SliceDynamicSizeCanonicalization
     }
 
     auto size_op = getTosaConstShape(rewriter, sliceOp.getLoc(), sliceSizes);
-    auto newSliceOp = tosa::SliceOp::create(rewriter,
-        sliceOp.getLoc(), sliceOp.getType(), sliceOp.getInput1(),
-        sliceOp.getStart(), size_op);
+    auto newSliceOp =
+        tosa::SliceOp::create(rewriter, sliceOp.getLoc(), sliceOp.getType(),
+                              sliceOp.getInput1(), sliceOp.getStart(), size_op);
 
     rewriter.replaceOp(sliceOp, newSliceOp.getResult());
     return success();
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index d110e1aeee8e4..2ff1e6b6d1666 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -180,12 +180,12 @@ Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // Tosa dialect constants only support ElementsAttr unlike standard dialect
   // constant which supports all attributes.
   if (llvm::isa<shapeType>(type) && llvm::isa<DenseIntElementsAttr>(value)) {
-    return tosa::ConstShapeOp::create(builder,
-        loc, type, llvm::cast<DenseIntElementsAttr>(value));
+    return tosa::ConstShapeOp::create(builder, loc, type,
+                                      llvm::cast<DenseIntElementsAttr>(value));
   }
   if (llvm::isa<ElementsAttr>(value))
     return tosa::ConstOp::create(builder, loc, type,
-                                         llvm::cast<ElementsAttr>(value));
+                                 llvm::cast<ElementsAttr>(value));
   return nullptr;
 }
 
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
index 9b77986257814..9474299a39582 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
@@ -111,7 +111,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
             DenseElementsAttr::get(zpTy, rewriter.getIntegerAttr(ety, zp));
         auto zpVal = tosa::ConstOp::create(rewriter, op.getLoc(), zpTy, zpAttr);
         return tosa::SubOp::create(rewriter, op.getLoc(), val.getType(), val,
-                                            zpVal);
+                                   zpVal);
       };
 
       input = applyZp(input, iZp);
@@ -141,7 +141,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
           tosa::ConstOp::create(rewriter, op->getLoc(), padTy, padAttr);
       inputType = RankedTensorType::get(newShape, inputETy);
       input = tosa::PadOp::create(rewriter, op->getLoc(), inputType, input,
-                                           padSizeVal, padVal);
+                                  padSizeVal, padVal);
     }
 
     // Perform an elementwise mul over the reshaped input and weight.
@@ -174,8 +174,8 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         dyn_cast<RankedTensorType>(input.getType()).getElementType());
     auto outputShapeValue =
         getTosaConstShape(rewriter, op->getLoc(), outputShape);
-    Value outputValue = tosa::ReshapeOp::create(rewriter,
-        op.getLoc(), outputShapeType, mulValue, outputShapeValue);
+    Value outputValue = tosa::ReshapeOp::create(
+        rewriter, op.getLoc(), outputShapeType, mulValue, outputShapeValue);
 
     Value bias = op.getBias();
     if (EqualizeRanks(rewriter, op.getLoc(), outputValue, bias).failed()) {
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
index 4a07220b4e181..dc5c51b0abad5 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
@@ -62,14 +62,16 @@ class TransposeConvNonStridedConverter
     convPad[2] = kernelWidth - 1 + pad[2];
     convPad[3] = kernelWidth - 1 + pad[3];
 
-    auto reverse1 = tosa::ReverseOp::create(rewriter,
-        loc, weightTy, weight, /* axis = */ rewriter.getI32IntegerAttr(1));
-    auto reverse2 = tosa::ReverseOp::create(rewriter,
-        loc, weightTy, reverse1, /* axis = */ rewriter.getI32IntegerAttr(2));
-
-    Value conv2d = tosa::Conv2DOp::create(rewriter,
-        loc, resultTy, input, reverse2, bias, op.getInputZp(), op.getWeightZp(),
-        rewriter.getDenseI64ArrayAttr(convPad),
+    auto reverse1 =
+        tosa::ReverseOp::create(rewriter, loc, weightTy, weight,
+                                /* axis = */ rewriter.getI32IntegerAttr(1));
+    auto reverse2 =
+        tosa::ReverseOp::create(rewriter, loc, weightTy, reverse1,
+                                /* axis = */ rewriter.getI32IntegerAttr(2));
+
+    Value conv2d = tosa::Conv2DOp::create(
+        rewriter, loc, resultTy, input, reverse2, bias, op.getInputZp(),
+        op.getWeightZp(), rewriter.getDenseI64ArrayAttr(convPad),
         rewriter.getDenseI64ArrayAttr(stride),
         rewriter.getDenseI64ArrayAttr({1, 1}),
         /* acc_type = */ op.getAccType());
@@ -216,8 +218,8 @@ class TransposeConvStridedConverter
         inputPaddingVal, inputPadConst);
 
     // We use a zero bias as we need to broadcast the bias.
-    auto zeroBias = tosa::ConstOp::create(rewriter,
-        loc,
+    auto zeroBias = tosa::ConstOp::create(
+        rewriter, loc,
         RankedTensorType::get({outputChannels * stride[0] * stride[1]},
                               biasETy),
         DenseElementsAttr::get(
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index a281c6600fdf6..581ac0466aa7a 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -403,8 +403,8 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
     return std::nullopt;
   }
   ImplicitLocOpBuilder builder(reshapeOp.getLoc(), rewriter);
-  auto foldedReshape = ReshapeOp::create(rewriter,
-      reshapeOp.getLoc(),
+  auto foldedReshape = ReshapeOp::create(
+      rewriter, reshapeOp.getLoc(),
       RankedTensorType::get(applyTOSAPermutation(shape, hoistedPerms),
                             reshapeOutputType.getElementType()),
       reshapeOp.getInput1(),
@@ -423,8 +423,8 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
   if (!maybeNewDenseAttr.has_value())
     return std::nullopt;
   auto newDenseAttr = maybeNewDenseAttr.value();
-  auto newConstOp = ConstOp::create(rewriter,
-      constOp.getLoc(), newDenseAttr.getType(), newDenseAttr);
+  auto newConstOp = ConstOp::create(rewriter, constOp.getLoc(),
+                                    newDenseAttr.getType(), newDenseAttr);
   return newConstOp->getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
index 69ef0ba40d72e..69eda03e03ab3 100644
--- a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
@@ -144,8 +144,8 @@ LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder,
       ArrayRef<int64_t>(reshapeOutputShape), reshapeInputType.getElementType());
   auto reshapeOutputShapeValue = getTosaConstShape(builder, reshapeOutputShape);
 
-  auto reshapeLower = tosa::ReshapeOp::create(builder,
-      reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
+  auto reshapeLower = tosa::ReshapeOp::create(
+      builder, reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
 
   if (input1Rank > input2Rank) {
     input1 = higherTensorValue;
diff --git a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
index 696dbe3d1892a..2bb2ea046fc1f 100644
--- a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h"
 
-#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
index c750017a6b504..66196194b0585 100644
--- a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -116,8 +116,8 @@ struct TransferWriteOpInterface
         getBuffer(rewriter, writeOp.getBase(), options, state);
     if (failed(resultBuffer))
       return failure();
-    vector::TransferWriteOp::create(rewriter,
-        writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
+    vector::TransferWriteOp::create(
+        rewriter, writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
         writeOp.getIndices(), writeOp.getPermutationMapAttr(),
         writeOp.getMask(), writeOp.getInBoundsAttr());
     replaceOpWithBufferizedValues(rewriter, op, *resultBuffer);
@@ -241,8 +241,9 @@ struct MaskOpInterface
     // Create a new vector.mask op.
     ValueRange newYieldedValuesRange(newYieldedValues);
     TypeRange newResultTypes(newYieldedValuesRange);
-    auto newOp = vector::MaskOp::create(rewriter,
-        op->getLoc(), newResultTypes, maskOp.getMask(), maskOp.getPassthru(),
+    auto newOp = vector::MaskOp::create(
+        rewriter, op->getLoc(), newResultTypes, maskOp.getMask(),
+        maskOp.getPassthru(),
         /*maskableOp=*/nullptr,
         /*maskRegionBuilder=*/[](OpBuilder &b, Operation *) {});
     newOp.getRegion().takeBody(maskOp.getMaskRegion());
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
index b2486c5cbbcf8..ad4d12eed0d3e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
@@ -95,8 +95,8 @@ static Value reshapeLoad(Location loc, Value val, VectorType type,
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   VectorType resType = VectorType::Builder(type).dropDim(index);
-  Value result = arith::ConstantOp::create(rewriter,
-      loc, resType, rewriter.getZeroAttr(resType));
+  Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                           rewriter.getZeroAttr(resType));
   for (int64_t d = 0, e = resType.getDimSize(0); d < e; d++) {
     Value ext = vector::ExtractOp::create(rewriter, loc, val, d);
     Value load = reshapeLoad(loc, ext, vType, index - 1, pos, rewriter);
@@ -447,8 +447,8 @@ struct UnrolledOuterProductGenerator
         extractMask =
             vector::ExtractOp::create(rewriter, loc, maybeMask.value(), k);
 
-      Operation *outerProdOp = vector::OuterProductOp::create(rewriter,
-          loc, res.getType(), extractA, extractB, res, kind);
+      Operation *outerProdOp = vector::OuterProductOp::create(
+          rewriter, loc, res.getType(), extractA, extractB, res, kind);
       res = maskOperation(rewriter, outerProdOp, extractMask)->getResult(0);
     }
     return res;
@@ -755,7 +755,7 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
 
   // ExtractOp does not allow dynamic indexing, we must unroll explicitly.
   Value res = arith::ConstantOp::create(rewriter, loc, dstType,
-                                                 rewriter.getZeroAttr(dstType));
+                                        rewriter.getZeroAttr(dstType));
   bool isInt = isa<IntegerType>(dstType.getElementType());
   llvm::SmallVector<Value> extractedCols;
   extractedCols.reserve(dstColumns);
@@ -766,15 +766,16 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
       // avoid having duplicate SSA values pointing to the same rows/columns.
       if (r == 0) {
         Value colRhs =
-            rank == 1 ? rhs
-                      : vector::ExtractOp::create(rewriter, op.getLoc(), rhs, c);
+            rank == 1
+                ? rhs
+                : vector::ExtractOp::create(rewriter, op.getLoc(), rhs, c);
         extractedCols.push_back(colRhs);
       }
       Value extractedColRhs = extractedCols[c];
       Value product =
           createMul(op.getLoc(), rowLhs, extractedColRhs, isInt, rewriter);
-      Value sum = vector::ReductionOp::create(rewriter,
-          op.getLoc(), vector::CombiningKind::ADD, product);
+      Value sum = vector::ReductionOp::create(
+          rewriter, op.getLoc(), vector::CombiningKind::ADD, product);
 
       SmallVector<int64_t, 2> pos = rank == 1 ? SmallVector<int64_t, 2>{r}
                                               : SmallVector<int64_t, 2>{r, c};
@@ -1097,8 +1098,8 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       rewriter.getArrayAttr(adjustIter(op.getIteratorTypes(), iterIndex));
   // Unroll into a series of lower dimensional vector.contract ops.
   Location loc = op.getLoc();
-  Value result = arith::ConstantOp::create(rewriter,
-      loc, resType, rewriter.getZeroAttr(resType));
+  Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                           rewriter.getZeroAttr(resType));
 
   for (int64_t d = 0; d < dimSize; ++d) {
     auto lhs = reshapeLoad(loc, op.getLhs(), lhsType, lhsIndex, d, rewriter);
@@ -1110,8 +1111,8 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       lowMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *lowContract = vector::ContractionOp::create(rewriter,
-        loc, lhs, rhs, acc, lowAffine, lowIter);
+    Operation *lowContract = vector::ContractionOp::create(
+        rewriter, loc, lhs, rhs, acc, lowAffine, lowIter);
     lowContract = maskOperation(rewriter, lowContract, lowMask);
     result = reshapeStore(loc, lowContract->getResult(0), result, resType,
                           resIndex, d, rewriter);
@@ -1186,8 +1187,8 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
       newMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *newContract = vector::ContractionOp::create(rewriter,
-        loc, lhs, rhs, result, lowAffine, lowIter);
+    Operation *newContract = vector::ContractionOp::create(
+        rewriter, loc, lhs, rhs, result, lowAffine, lowIter);
     result = maskOperation(rewriter, newContract, newMask)->getResult(0);
   }
   return result;
@@ -1240,7 +1241,8 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
 
     if (!rhsType) {
       // Special case: AXPY operation.
-      Value b = vector::BroadcastOp::create(rewriter, loc, lhsType, op.getRhs());
+      Value b =
+          vector::BroadcastOp::create(rewriter, loc, lhsType, op.getRhs());
       std::optional<Value> mult = createContractArithOp(
           loc, op.getLhs(), b, acc, kind, rewriter, isInt, mask);
       if (!mult.has_value())
@@ -1249,8 +1251,8 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
       return success();
     }
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resType, rewriter.getZeroAttr(resType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                             rewriter.getZeroAttr(resType));
     for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
       Value x = vector::ExtractOp::create(rewriter, loc, op.getLhs(), d);
       Value a = vector::BroadcastOp::create(rewriter, loc, rhsType, x);
@@ -1363,9 +1365,9 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   rhs = vector::ShapeCastOp::create(rew, loc, flattenedRHSType, rhs);
 
   Value mul = vector::MatmulOp::create(rew, loc, lhs, rhs, lhsRows, lhsColumns,
-                                           rhsColumns);
-  mul = vector::ShapeCastOp::create(rew,
-      loc,
+                                       rhsColumns);
+  mul = vector::ShapeCastOp::create(
+      rew, loc,
       VectorType::get({lhsRows, rhsColumns},
                       getElementTypeOrSelf(op.getAcc().getType())),
       mul);
@@ -1377,11 +1379,11 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   else if (accMap != AffineMap::get(3, 0, {m, n}, ctx))
     llvm_unreachable("invalid contraction semantics");
 
-  Value res =
-      isa<IntegerType>(elementType)
-          ? static_cast<Value>(arith::AddIOp::create(rew, loc, op.getAcc(), mul))
-          : static_cast<Value>(
-                arith::AddFOp::create(rew, loc, op.getAcc(), mul));
+  Value res = isa<IntegerType>(elementType)
+                  ? static_cast<Value>(
+                        arith::AddIOp::create(rew, loc, op.getAcc(), mul))
+                  : static_cast<Value>(
+                        arith::AddFOp::create(rew, loc, op.getAcc(), mul));
 
   return res;
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index c586059ca55bc..b028940de1741 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -75,8 +75,8 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
     Value maskVec = op.getMask();
     Value passThruVec = op.getPassThru();
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resultTy, rewriter.getZeroAttr(resultTy));
+    Value result = arith::ConstantOp::create(rewriter, loc, resultTy,
+                                             rewriter.getZeroAttr(resultTy));
 
     VectorType subTy = VectorType::Builder(resultTy).dropDim(0);
 
@@ -89,9 +89,9 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
           vector::ExtractOp::create(rewriter, loc, maskVec, thisIdx);
       Value passThruSubVec =
           vector::ExtractOp::create(rewriter, loc, passThruVec, thisIdx);
-      Value subGather = vector::GatherOp::create(rewriter,
-          loc, subTy, op.getBase(), op.getIndices(), indexSubVec, maskSubVec,
-          passThruSubVec);
+      Value subGather = vector::GatherOp::create(
+          rewriter, loc, subTy, op.getBase(), op.getIndices(), indexSubVec,
+          maskSubVec, passThruSubVec);
       result =
           vector::InsertOp::create(rewriter, loc, subGather, result, thisIdx);
     }
@@ -159,24 +159,24 @@ struct RemoveStrideFromGatherSource : OpRewritePattern<vector::GatherOp> {
 
     // 1. Collapse the input memref so that it's "flat".
     SmallVector<ReassociationIndices> reassoc = {{0, 1}};
-    Value collapsed = memref::CollapseShapeOp::create(rewriter,
-        op.getLoc(), subview.getSource(), reassoc);
+    Value collapsed = memref::CollapseShapeOp::create(
+        rewriter, op.getLoc(), subview.getSource(), reassoc);
 
     // 2. Generate new gather indices that will model the
     // strided access.
     IntegerAttr stride = rewriter.getIndexAttr(srcTrailingDim);
     VectorType vType = op.getIndexVec().getType();
-    Value mulCst = arith::ConstantOp::create(rewriter,
-        op.getLoc(), vType, DenseElementsAttr::get(vType, stride));
+    Value mulCst = arith::ConstantOp::create(
+        rewriter, op.getLoc(), vType, DenseElementsAttr::get(vType, stride));
 
     Value newIdxs =
         arith::MulIOp::create(rewriter, op.getLoc(), op.getIndexVec(), mulCst);
 
     // 3. Create an updated gather op with the collapsed input memref and the
     // updated indices.
-    Value newGather = vector::GatherOp::create(rewriter,
-        op.getLoc(), op.getResult().getType(), collapsed, op.getIndices(),
-        newIdxs, op.getMask(), op.getPassThru());
+    Value newGather = vector::GatherOp::create(
+        rewriter, op.getLoc(), op.getResult().getType(), collapsed,
+        op.getIndices(), newIdxs, op.getMask(), op.getPassThru());
     rewriter.replaceOp(op, newGather);
 
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index b0affbb699d9e..9d6a865a9301f 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -60,11 +60,13 @@ class UnrollInterleaveOp final : public OpRewritePattern<vector::InterleaveOp> {
       return failure();
 
     auto loc = op.getLoc();
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resultType, rewriter.getZeroAttr(resultType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resultType,
+                                             rewriter.getZeroAttr(resultType));
     for (auto position : *unrollIterator) {
-      Value extractLhs = ExtractOp::create(rewriter, loc, op.getLhs(), position);
-      Value extractRhs = ExtractOp::create(rewriter, loc, op.getRhs(), position);
+      Value extractLhs =
+          ExtractOp::create(rewriter, loc, op.getLhs(), position);
+      Value extractRhs =
+          ExtractOp::create(rewriter, loc, op.getRhs(), position);
       Value interleave =
           InterleaveOp::create(rewriter, loc, extractLhs, extractRhs);
       result = InsertOp::create(rewriter, loc, interleave, result, position);
@@ -123,8 +125,8 @@ class UnrollDeinterleaveOp final
       return failure();
 
     auto loc = op.getLoc();
-    Value emptyResult = arith::ConstantOp::create(rewriter,
-        loc, resultType, rewriter.getZeroAttr(resultType));
+    Value emptyResult = arith::ConstantOp::create(
+        rewriter, loc, resultType, rewriter.getZeroAttr(resultType));
     Value evenResult = emptyResult;
     Value oddResult = emptyResult;
 
@@ -133,10 +135,10 @@ class UnrollDeinterleaveOp final
           vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       auto deinterleave =
           vector::DeinterleaveOp::create(rewriter, loc, extractSrc);
-      evenResult = vector::InsertOp::create(rewriter,
-          loc, deinterleave.getRes1(), evenResult, position);
-      oddResult = vector::InsertOp::create(rewriter, loc, deinterleave.getRes2(),
-                                                    oddResult, position);
+      evenResult = vector::InsertOp::create(
+          rewriter, loc, deinterleave.getRes1(), evenResult, position);
+      oddResult = vector::InsertOp::create(
+          rewriter, loc, deinterleave.getRes2(), oddResult, position);
     }
     rewriter.replaceOp(op, ValueRange{evenResult, oddResult});
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
index edd840d8fbc7b..45ef7f01a85f1 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
@@ -67,18 +67,19 @@ class CreateMaskOpLowering : public OpRewritePattern<vector::CreateMaskOp> {
     Value idx = op.getOperand(0);
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = vector::CreateMaskOp::create(rewriter,
-        loc, lowType, op.getOperands().drop_front());
-    Value falseVal = arith::ConstantOp::create(rewriter,
-        loc, lowType, rewriter.getZeroAttr(lowType));
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, dstType, rewriter.getZeroAttr(dstType));
+    Value trueVal = vector::CreateMaskOp::create(rewriter, loc, lowType,
+                                                 op.getOperands().drop_front());
+    Value falseVal = arith::ConstantOp::create(rewriter, loc, lowType,
+                                               rewriter.getZeroAttr(lowType));
+    Value result = arith::ConstantOp::create(rewriter, loc, dstType,
+                                             rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < dim; d++) {
       Value bnd =
           arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(d));
-      Value val = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
-                                                 bnd, idx);
-      Value sel = arith::SelectOp::create(rewriter, loc, val, trueVal, falseVal);
+      Value val = arith::CmpIOp::create(rewriter, loc,
+                                        arith::CmpIPredicate::slt, bnd, idx);
+      Value sel =
+          arith::SelectOp::create(rewriter, loc, val, trueVal, falseVal);
       result = vector::InsertOp::create(rewriter, loc, sel, result, d);
     }
     rewriter.replaceOp(op, result);
@@ -146,10 +147,10 @@ class ConstantMaskOpLowering : public OpRewritePattern<vector::ConstantMaskOp> {
           op, "Cannot unroll leading scalable dim in dstType");
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = vector::ConstantMaskOp::create(rewriter,
-        loc, lowType, dimSizes.drop_front());
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, dstType, rewriter.getZeroAttr(dstType));
+    Value trueVal = vector::ConstantMaskOp::create(rewriter, loc, lowType,
+                                                   dimSizes.drop_front());
+    Value result = arith::ConstantOp::create(rewriter, loc, dstType,
+                                             rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < trueDimSize; d++)
       result = vector::InsertOp::create(rewriter, loc, trueVal, result, d);
 
@@ -261,8 +262,8 @@ struct MaskedGatherOpPattern : public MaskOpRewritePattern<GatherOp> {
                             PatternRewriter &rewriter) const override {
     Value passthru = maskingOp.hasPassthru()
                          ? maskingOp.getPassthru()
-                         : arith::ConstantOp::create(rewriter,
-                               gatherOp.getLoc(),
+                         : arith::ConstantOp::create(
+                               rewriter, gatherOp.getLoc(),
                                rewriter.getZeroAttr(gatherOp.getVectorType()));
 
     // Replace the `vector.mask` operation.
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
index c5631ba00bbc6..4773732d8d9a6 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
@@ -103,8 +103,8 @@ class InnerOuterDimReductionConversion
     // If masked, transpose the original mask.
     Value transposedMask;
     if (maskableOp.isMasked()) {
-      transposedMask = vector::TransposeOp::create(rewriter,
-          loc, maskableOp.getMaskingOp().getMask(), indices);
+      transposedMask = vector::TransposeOp::create(
+          rewriter, loc, maskableOp.getMaskingOp().getMask(), indices);
     }
 
     // Transpose reduction source.
@@ -117,8 +117,8 @@ class InnerOuterDimReductionConversion
         reductionMask[i] = true;
     }
 
-    Operation *newMultiRedOp = vector::MultiDimReductionOp::create(rewriter,
-        multiReductionOp.getLoc(), transposeOp.getResult(),
+    Operation *newMultiRedOp = vector::MultiDimReductionOp::create(
+        rewriter, multiReductionOp.getLoc(), transposeOp.getResult(),
         multiReductionOp.getAcc(), reductionMask, multiReductionOp.getKind());
     newMultiRedOp =
         mlir::vector::maskOperation(rewriter, newMultiRedOp, transposedMask);
@@ -255,15 +255,15 @@ class ReduceMultiDimReductionRank
       auto maskCastedType = VectorType::get(
           vectorShape,
           llvm::cast<VectorType>(vectorMask.getType()).getElementType());
-      newVectorMask =
-          vector::ShapeCastOp::create(rewriter, loc, maskCastedType, vectorMask);
+      newVectorMask = vector::ShapeCastOp::create(rewriter, loc, maskCastedType,
+                                                  vectorMask);
     }
 
     auto castedType = VectorType::get(
         vectorShape, multiReductionOp.getSourceVectorType().getElementType(),
         scalableDims);
-    Value cast = vector::ShapeCastOp::create(rewriter,
-        loc, castedType, multiReductionOp.getSource());
+    Value cast = vector::ShapeCastOp::create(rewriter, loc, castedType,
+                                             multiReductionOp.getSource());
 
     Value acc = multiReductionOp.getAcc();
     if (flattenedParallelDim) {
@@ -275,8 +275,8 @@ class ReduceMultiDimReductionRank
     }
     // 6. Creates the flattened form of vector.multi_reduction with inner/outer
     // most dim as reduction.
-    Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(rewriter,
-        loc, cast, acc, mask, multiReductionOp.getKind());
+    Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(
+        rewriter, loc, cast, acc, mask, multiReductionOp.getKind());
     newMultiDimRedOp =
         mlir::vector::maskOperation(rewriter, newMultiDimRedOp, newVectorMask);
 
@@ -339,8 +339,8 @@ struct TwoDimMultiReductionToElementWise
 
     Value result = multiReductionOp.getAcc();
     for (int64_t i = 0; i < srcShape[0]; i++) {
-      auto operand = vector::ExtractOp::create(rewriter,
-          loc, multiReductionOp.getSource(), i);
+      auto operand = vector::ExtractOp::create(rewriter, loc,
+                                               multiReductionOp.getSource(), i);
       Value extractMask = nullptr;
       if (mask) {
         extractMask = vector::ExtractOp::create(rewriter, loc, mask, i);
@@ -383,28 +383,29 @@ struct TwoDimMultiReductionToReduction
     }
 
     auto loc = multiReductionOp.getLoc();
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, multiReductionOp.getDestType(),
+    Value result = arith::ConstantOp::create(
+        rewriter, loc, multiReductionOp.getDestType(),
         rewriter.getZeroAttr(multiReductionOp.getDestType()));
     int outerDim = multiReductionOp.getSourceVectorType().getShape()[0];
 
     for (int i = 0; i < outerDim; ++i) {
-      auto v = vector::ExtractOp::create(rewriter,
-          loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
-      auto acc = vector::ExtractOp::create(rewriter,
-          loc, multiReductionOp.getAcc(), ArrayRef<int64_t>{i});
-      Operation *reductionOp = vector::ReductionOp::create(rewriter,
-          loc, multiReductionOp.getKind(), v, acc);
+      auto v = vector::ExtractOp::create(
+          rewriter, loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
+      auto acc = vector::ExtractOp::create(
+          rewriter, loc, multiReductionOp.getAcc(), ArrayRef<int64_t>{i});
+      Operation *reductionOp = vector::ReductionOp::create(
+          rewriter, loc, multiReductionOp.getKind(), v, acc);
 
       // If masked, slice the mask and mask the new reduction operation.
       if (maskableOp.isMasked()) {
-        Value mask = vector::ExtractOp::create(rewriter,
-            loc, maskableOp.getMaskingOp().getMask(), ArrayRef<int64_t>{i});
+        Value mask = vector::ExtractOp::create(
+            rewriter, loc, maskableOp.getMaskingOp().getMask(),
+            ArrayRef<int64_t>{i});
         reductionOp = mlir::vector::maskOperation(rewriter, reductionOp, mask);
       }
 
-      result = vector::InsertOp::create(rewriter, loc, reductionOp->getResult(0),
-                                                 result, i);
+      result = vector::InsertOp::create(rewriter, loc,
+                                        reductionOp->getResult(0), result, i);
     }
 
     rewriter.replaceOp(rootOp, result);
@@ -459,10 +460,10 @@ struct OneDimMultiReductionToTwoDim
     SmallVector<bool, 2> reductionMask{false, true};
 
     /// vector.extract(vector.multi_reduce(vector.shape_cast(v, 1xk)), 0)
-    Value cast = vector::ShapeCastOp::create(rewriter,
-        loc, castedType, multiReductionOp.getSource());
-    Value castAcc = vector::BroadcastOp::create(rewriter,
-        loc, accType, multiReductionOp.getAcc());
+    Value cast = vector::ShapeCastOp::create(rewriter, loc, castedType,
+                                             multiReductionOp.getSource());
+    Value castAcc = vector::BroadcastOp::create(rewriter, loc, accType,
+                                                multiReductionOp.getAcc());
     Value castMask;
     if (maskableOp.isMasked()) {
       auto maskType = llvm::cast<VectorType>(mask.getType());
@@ -473,8 +474,9 @@ struct OneDimMultiReductionToTwoDim
       castMask = vector::BroadcastOp::create(rewriter, loc, castMaskType, mask);
     }
 
-    Operation *newOp = vector::MultiDimReductionOp::create(rewriter,
-        loc, cast, castAcc, reductionMask, multiReductionOp.getKind());
+    Operation *newOp = vector::MultiDimReductionOp::create(
+        rewriter, loc, cast, castAcc, reductionMask,
+        multiReductionOp.getKind());
     newOp = vector::maskOperation(rewriter, newOp, castMask);
 
     rewriter.replaceOpWithNewOp<vector::ExtractOp>(rootOp, newOp->getResult(0),
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
index a6985b96b59df..4f6391b78b947 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
@@ -122,8 +122,8 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
       return failure();
 
     VectorType resType = VectorType::get(destShape, elType);
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resType, rewriter.getZeroAttr(resType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                             rewriter.getZeroAttr(resType));
     int64_t reductionDim = scanOp.getReductionDim();
     bool inclusive = scanOp.getInclusive();
     int64_t destRank = destType.getRank();
@@ -144,9 +144,9 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
     for (int i = 0; i < destShape[reductionDim]; i++) {
       offsets[reductionDim] = i;
       ArrayAttr scanOffsets = rewriter.getI64ArrayAttr(offsets);
-      Value input = vector::ExtractStridedSliceOp::create(rewriter,
-          loc, reductionType, scanOp.getSource(), scanOffsets, scanSizes,
-          scanStrides);
+      Value input = vector::ExtractStridedSliceOp::create(
+          rewriter, loc, reductionType, scanOp.getSource(), scanOffsets,
+          scanSizes, scanStrides);
       Value output;
       if (i == 0) {
         if (inclusive) {
@@ -154,11 +154,11 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         } else {
           if (initialValueRank == 0) {
             // ShapeCastOp cannot handle 0-D vectors
-            output = vector::BroadcastOp::create(rewriter,
-                loc, input.getType(), scanOp.getInitialValue());
+            output = vector::BroadcastOp::create(rewriter, loc, input.getType(),
+                                                 scanOp.getInitialValue());
           } else {
-            output = vector::ShapeCastOp::create(rewriter,
-                loc, input.getType(), scanOp.getInitialValue());
+            output = vector::ShapeCastOp::create(rewriter, loc, input.getType(),
+                                                 scanOp.getInitialValue());
           }
         }
       } else {
@@ -166,8 +166,8 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         output = vector::makeArithReduction(rewriter, loc, scanOp.getKind(),
                                             lastOutput, y);
       }
-      result = vector::InsertStridedSliceOp::create(rewriter,
-          loc, output, result, offsets, strides);
+      result = vector::InsertStridedSliceOp::create(rewriter, loc, output,
+                                                    result, offsets, strides);
       lastOutput = output;
       lastInput = input;
     }
@@ -179,7 +179,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
           vector::BroadcastOp::create(rewriter, loc, initialValueType, v);
     } else {
       reduction = vector::ShapeCastOp::create(rewriter, loc, initialValueType,
-                                                       lastOutput);
+                                              lastOutput);
     }
 
     rewriter.replaceOp(scanOp, {result, reduction});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
index 69d3666a13879..603ea41d43360 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
@@ -138,10 +138,11 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 
     const Value source = shapeCast.getSource();
     const Value poison = ub::PoisonOp::create(rewriter, loc, resultType);
-    const Value extracted = vector::ExtractOp::create(rewriter,
-        loc, source, SmallVector<int64_t>(sourceLeading, 0));
-    const Value result = vector::InsertOp::create(rewriter,
-        loc, extracted, poison, SmallVector<int64_t>(resultLeading, 0));
+    const Value extracted = vector::ExtractOp::create(
+        rewriter, loc, source, SmallVector<int64_t>(sourceLeading, 0));
+    const Value result =
+        vector::InsertOp::create(rewriter, loc, extracted, poison,
+                                 SmallVector<int64_t>(resultLeading, 0));
 
     rewriter.replaceOp(shapeCast, result);
     return success();
@@ -178,7 +179,7 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
           vector::ExtractOp::create(rewriter, loc, source, extractIndex);
 
       result = vector::InsertOp::create(rewriter, loc, extracted, result,
-                                                 insertIndex);
+                                        insertIndex);
 
       inplaceAdd(1, sourceShape.take_front(sourceDim), extractIndex);
       inplaceAdd(1, resultShape.take_front(resultDim), insertIndex);
@@ -295,21 +296,21 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 
       // vector.extract_strided_slice
       extractOffsets[0] = extractStridedPhase * greatestCommonDivisor;
-      extractedStrided = vector::ExtractStridedSliceOp::create(rewriter,
-          loc, extracted, extractOffsets, atomicShape, sizes);
+      extractedStrided = vector::ExtractStridedSliceOp::create(
+          rewriter, loc, extracted, extractOffsets, atomicShape, sizes);
 
       // vector.insert_strided_slice
       if (insertStridedPhase == 0) {
         insertedSlice = partResult;
       }
       insertOffsets[0] = insertStridedPhase * greatestCommonDivisor;
-      insertedSlice = vector::InsertStridedSliceOp::create(rewriter,
-          loc, extractedStrided, insertedSlice, insertOffsets, sizes);
+      insertedSlice = vector::InsertStridedSliceOp::create(
+          rewriter, loc, extractedStrided, insertedSlice, insertOffsets, sizes);
 
       // vector.insert
       if (insertStridedPhase + 1 == insertPeriod) {
         result = vector::InsertOp::create(rewriter, loc, insertedSlice, result,
-                                                   insertIndex);
+                                          insertIndex);
         inplaceAdd(1, resultType.getShape().take_front(resultSuffixStartDim),
                    insertIndex);
       }
@@ -406,16 +407,18 @@ class ScalableShapeCastOpRewritePattern
       // 1. Extract a scalable subvector from the source vector.
       if (!currentSourceScalableVector) {
         if (srcRank != 1) {
-          currentSourceScalableVector = vector::ExtractOp::create(rewriter,
-              loc, op.getSource(), llvm::ArrayRef(srcIdx).drop_back());
+          currentSourceScalableVector =
+              vector::ExtractOp::create(rewriter, loc, op.getSource(),
+                                        llvm::ArrayRef(srcIdx).drop_back());
         } else {
           currentSourceScalableVector = op.getSource();
         }
       }
       Value sourceSubVector = currentSourceScalableVector;
       if (minExtractionSize < minSourceTrailingSize) {
-        sourceSubVector = vector::ScalableExtractOp::create(rewriter,
-            loc, extractionVectorType, sourceSubVector, srcIdx.back());
+        sourceSubVector = vector::ScalableExtractOp::create(
+            rewriter, loc, extractionVectorType, sourceSubVector,
+            srcIdx.back());
       }
 
       // 2. Insert the scalable subvector into the result vector.
@@ -423,15 +426,16 @@ class ScalableShapeCastOpRewritePattern
         if (minExtractionSize == minResultTrailingSize) {
           currentResultScalableVector = sourceSubVector;
         } else if (resRank != 1) {
-          currentResultScalableVector = vector::ExtractOp::create(rewriter,
-              loc, result, llvm::ArrayRef(resIdx).drop_back());
+          currentResultScalableVector = vector::ExtractOp::create(
+              rewriter, loc, result, llvm::ArrayRef(resIdx).drop_back());
         } else {
           currentResultScalableVector = result;
         }
       }
       if (minExtractionSize < minResultTrailingSize) {
-        currentResultScalableVector = vector::ScalableInsertOp::create(rewriter,
-            loc, sourceSubVector, currentResultScalableVector, resIdx.back());
+        currentResultScalableVector = vector::ScalableInsertOp::create(
+            rewriter, loc, sourceSubVector, currentResultScalableVector,
+            resIdx.back());
       }
 
       // 3. Update the source and result scalable vectors if needed.
@@ -439,9 +443,9 @@ class ScalableShapeCastOpRewritePattern
           currentResultScalableVector != result) {
         // Finished row of result. Insert complete scalable vector into result
         // (n-D) vector.
-        result = vector::InsertOp::create(rewriter,
-            loc, currentResultScalableVector, result,
-            llvm::ArrayRef(resIdx).drop_back());
+        result = vector::InsertOp::create(rewriter, loc,
+                                          currentResultScalableVector, result,
+                                          llvm::ArrayRef(resIdx).drop_back());
         currentResultScalableVector = {};
       }
       if (srcIdx.back() + minExtractionSize >= minSourceTrailingSize) {
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
index fe02cd5656571..290a131bc496d 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
@@ -138,8 +138,8 @@ struct TransferReadPermutationLowering
     // Generate new transfer_read operation.
     VectorType newReadType = VectorType::get(
         newVectorShape, op.getVectorType().getElementType(), newScalableDims);
-    Value newRead = vector::TransferReadOp::create(rewriter,
-        op.getLoc(), newReadType, op.getBase(), op.getIndices(),
+    Value newRead = vector::TransferReadOp::create(
+        rewriter, op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
 
@@ -209,12 +209,12 @@ struct TransferWritePermutationLowering
         inverseTransposeInBoundsAttr(rewriter, op.getInBounds(), permutation);
 
     // Generate new transfer_write operation.
-    Value newVec = vector::TransposeOp::create(rewriter,
-        op.getLoc(), op.getVector(), indices);
+    Value newVec = vector::TransposeOp::create(rewriter, op.getLoc(),
+                                               op.getVector(), indices);
     auto newMap = AffineMap::getMinorIdentityMap(
         map.getNumDims(), map.getNumResults(), rewriter.getContext());
-    auto newWrite = vector::TransferWriteOp::create(rewriter,
-        op.getLoc(), newVec, op.getBase(), op.getIndices(),
+    auto newWrite = vector::TransferWriteOp::create(
+        rewriter, op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getMask(), newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
       return newWrite.getResult();
@@ -299,8 +299,8 @@ struct TransferWriteNonPermutationLowering
       newInBoundsValues.push_back(op.isDimInBounds(i));
     }
     ArrayAttr newInBoundsAttr = rewriter.getBoolArrayAttr(newInBoundsValues);
-    auto newWrite = vector::TransferWriteOp::create(rewriter,
-        op.getLoc(), newVec, op.getBase(), op.getIndices(),
+    auto newWrite = vector::TransferWriteOp::create(
+        rewriter, op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), newMask, newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
       return newWrite.getResult();
@@ -370,8 +370,8 @@ struct TransferOpReduceRank
             ? rewriter.getArrayAttr(
                   op.getInBoundsAttr().getValue().take_back(reducedShapeRank))
             : ArrayAttr();
-    Value newRead = vector::TransferReadOp::create(rewriter,
-        op.getLoc(), newReadType, op.getBase(), op.getIndices(),
+    Value newRead = vector::TransferReadOp::create(
+        rewriter, op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
     return rewriter
@@ -471,21 +471,21 @@ struct TransferReadToVectorLoadLowering
             read, "vector type is not rank 1, can't create masked load, needs "
                   "VectorToSCF");
 
-      Value fill = vector::SplatOp::create(rewriter,
-          read.getLoc(), unbroadcastedVectorType, read.getPadding());
-      res = vector::MaskedLoadOp::create(rewriter,
-          read.getLoc(), unbroadcastedVectorType, read.getBase(),
+      Value fill = vector::SplatOp::create(
+          rewriter, read.getLoc(), unbroadcastedVectorType, read.getPadding());
+      res = vector::MaskedLoadOp::create(
+          rewriter, read.getLoc(), unbroadcastedVectorType, read.getBase(),
           read.getIndices(), read.getMask(), fill);
     } else {
       res = vector::LoadOp::create(rewriter, read.getLoc(),
-                                            unbroadcastedVectorType,
-                                            read.getBase(), read.getIndices());
+                                   unbroadcastedVectorType, read.getBase(),
+                                   read.getIndices());
     }
 
     // Insert a broadcasting op if required.
     if (!broadcastedDims.empty())
-      res = vector::BroadcastOp::create(rewriter,
-          read.getLoc(), read.getVectorType(), res->getResult(0));
+      res = vector::BroadcastOp::create(
+          rewriter, read.getLoc(), read.getVectorType(), res->getResult(0));
     return res->getResult(0);
   }
 
@@ -569,12 +569,12 @@ struct TransferWriteToVectorStoreLowering
                    << write;
             });
 
-      vector::MaskedStoreOp::create(rewriter,
-          write.getLoc(), write.getBase(), write.getIndices(), write.getMask(),
-          write.getVector());
+      vector::MaskedStoreOp::create(rewriter, write.getLoc(), write.getBase(),
+                                    write.getIndices(), write.getMask(),
+                                    write.getVector());
     } else {
       vector::StoreOp::create(rewriter, write.getLoc(), write.getVector(),
-                                       write.getBase(), write.getIndices());
+                              write.getBase(), write.getIndices());
     }
     // There's no return value for StoreOps. Use Value() to signal success to
     // matchAndRewrite.
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index e9591bfcf6feb..da2aed653a080 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -80,8 +80,8 @@ getUnpackShufflePermFor128Lane(ArrayRef<int64_t> vals, int numBits) {
 static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return vector::ShuffleOp::create(b,
-      v1, v2,
+  return vector::ShuffleOp::create(
+      b, v1, v2,
       getUnpackShufflePermFor128Lane({0, 1, numElem, numElem + 1}, numBits));
 }
 
@@ -94,8 +94,8 @@ static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return vector::ShuffleOp::create(b,
-      v1, v2,
+  return vector::ShuffleOp::create(
+      b, v1, v2,
       getUnpackShufflePermFor128Lane({2, 3, numElem + 2, numElem + 3},
                                      numBits));
 }
@@ -109,8 +109,8 @@ static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  auto shuffle = vector::ShuffleOp::create(b,
-      v1, v2,
+  auto shuffle = vector::ShuffleOp::create(
+      b, v1, v2,
       getUnpackShufflePermFor128Lane({0, numElem, 1, numElem + 1}, numBits));
   return shuffle;
 }
@@ -124,8 +124,8 @@ static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return vector::ShuffleOp::create(b,
-      v1, v2,
+  return vector::ShuffleOp::create(
+      b, v1, v2,
       getUnpackShufflePermFor128Lane({2, numElem + 2, 3, numElem + 3},
                                      numBits));
 }
@@ -338,8 +338,8 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
           vector::ShapeCastOp::create(rewriter, loc, flattenedType, input);
       auto rows = rewriter.getI32IntegerAttr(resType.getShape()[0]);
       auto columns = rewriter.getI32IntegerAttr(resType.getShape()[1]);
-      Value trans = vector::FlatTransposeOp::create(rewriter,
-          loc, flattenedType, matrix, rows, columns);
+      Value trans = vector::FlatTransposeOp::create(
+          rewriter, loc, flattenedType, matrix, rows, columns);
       rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, trans);
       return success();
     }
@@ -483,7 +483,7 @@ class TransposeOp2DToShuffleLowering
     auto flattenedType = VectorType::get({n * m}, srcType.getElementType());
     auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
     auto reshInput = vector::ShapeCastOp::create(rewriter, loc, flattenedType,
-                                                          op.getVector());
+                                                 op.getVector());
 
     Value res;
     if (vectorTransposeLowering == VectorTransposeLowering::Shuffle16x16 &&
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index 2683a85ecdaeb..e5d53cd87506a 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -127,8 +127,8 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return vector::TransferWriteOp::create(b,
-        loc, val, buffer, indices,
+    return vector::TransferWriteOp::create(
+        b, loc, val, buffer, indices,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
 
@@ -172,8 +172,9 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return vector::TransferReadOp::create(b,
-        loc, cast<VectorType>(type), buffer, indices, /*padding=*/std::nullopt,
+    return vector::TransferReadOp::create(
+        b, loc, cast<VectorType>(type), buffer, indices,
+        /*padding=*/std::nullopt,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
 
@@ -244,10 +245,10 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
 
     // Step 1: Create scf.if op.
     Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
-    Value isLane0 = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::eq, warpOp.getLaneid(), c0);
+    Value isLane0 = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, warpOp.getLaneid(), c0);
     auto ifOp = scf::IfOp::create(rewriter, loc, isLane0,
-                                           /*withElseRegion=*/false);
+                                  /*withElseRegion=*/false);
     rewriter.eraseOp(ifOp.thenBlock()->getTerminator());
 
     // Step 2: insert appropriate (alloc, write)-pairs before the scf.if and
@@ -512,8 +513,9 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
 
     // Create a second warp op that contains only writeOp.
-    auto secondWarpOp = WarpExecuteOnLane0Op::create(rewriter,
-        loc, TypeRange(), newWarpOp.getLaneid(), newWarpOp.getWarpSize());
+    auto secondWarpOp = WarpExecuteOnLane0Op::create(rewriter, loc, TypeRange(),
+                                                     newWarpOp.getLaneid(),
+                                                     newWarpOp.getWarpSize());
     Block &body = secondWarpOp.getBodyRegion().front();
     rewriter.setInsertionPointToStart(&body);
     auto newWriteOp =
@@ -823,9 +825,9 @@ struct WarpOpTransferRead : public WarpDistributionPattern {
     Value newMask =
         hasMask ? newWarpOp.getResult(newRetIndices[newRetIndices.size() - 1])
                 : Value();
-    auto newRead = vector::TransferReadOp::create(rewriter,
-        read.getLoc(), distributedVal.getType(), read.getBase(), newIndices,
-        read.getPermutationMapAttr(), newPadding, newMask,
+    auto newRead = vector::TransferReadOp::create(
+        rewriter, read.getLoc(), distributedVal.getType(), read.getBase(),
+        newIndices, read.getPermutationMapAttr(), newPadding, newMask,
         read.getInBoundsAttr());
 
     rewriter.replaceAllUsesWith(distributedVal, newRead);
@@ -965,8 +967,8 @@ struct WarpOpBroadcast : public WarpDistributionPattern {
     WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
         rewriter, warpOp, {broadcastSrc}, {broadcastSrcType}, newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value broadcasted = vector::BroadcastOp::create(rewriter,
-        loc, destVecType, newWarpOp->getResult(newRetIndices[0]));
+    Value broadcasted = vector::BroadcastOp::create(
+        rewriter, loc, destVecType, newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 broadcasted);
     return success();
@@ -1008,8 +1010,8 @@ struct WarpOpShapeCast : public WarpDistributionPattern {
         rewriter, warpOp, {oldCastOp.getSource()}, {castDistributedType},
         newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value newCast = vector::ShapeCastOp::create(rewriter,
-        oldCastOp.getLoc(), castResultType,
+    Value newCast = vector::ShapeCastOp::create(
+        rewriter, oldCastOp.getLoc(), castResultType,
         newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newCast);
     return success();
@@ -1182,9 +1184,10 @@ struct WarpOpInsertStridedSlice : public WarpDistributionPattern {
     Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
     // Create a new insert strided slice op that inserts distributed source into
     // distributed dest.
-    Value newInsert = vector::InsertStridedSliceOp::create(rewriter,
-        insertOp.getLoc(), distributedDest.getType(), distributedSource,
-        distributedDest, insertOp.getOffsets(), insertOp.getStrides());
+    Value newInsert = vector::InsertStridedSliceOp::create(
+        rewriter, insertOp.getLoc(), distributedDest.getType(),
+        distributedSource, distributedDest, insertOp.getOffsets(),
+        insertOp.getStrides());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newInsert);
     return success();
   }
@@ -1277,8 +1280,8 @@ struct WarpOpExtractStridedSlice : public WarpDistributionPattern {
     // Create a new extract strided slice op that extracts from the
     // distributed vector.
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
-    Value newExtract = vector::ExtractStridedSliceOp::create(rewriter,
-        extractOp.getLoc(), distributedType, distributedVec,
+    Value newExtract = vector::ExtractStridedSliceOp::create(
+        rewriter, extractOp.getLoc(), distributedType, distributedVec,
         extractOp.getOffsets(),
         ArrayAttr::get(rewriter.getContext(), distributedSizes),
         extractOp.getStrides());
@@ -1323,8 +1326,8 @@ struct WarpOpExtract : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
       // Extract from distributed vector.
-      Value newExtract = vector::ExtractOp::create(rewriter,
-          loc, distributedVec, extractOp.getMixedPosition());
+      Value newExtract = vector::ExtractOp::create(
+          rewriter, loc, distributedVec, extractOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
       return success();
@@ -1352,8 +1355,8 @@ struct WarpOpExtract : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
     // Extract from distributed vector.
-    Value newExtract = vector::ExtractOp::create(rewriter,
-        loc, distributedVec, extractOp.getMixedPosition());
+    Value newExtract = vector::ExtractOp::create(rewriter, loc, distributedVec,
+                                                 extractOp.getMixedPosition());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 newExtract);
     return success();
@@ -1515,7 +1518,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
         indices.push_back(pos);
       }
       newInsert = vector::InsertOp::create(rewriter, loc, newSource,
-                                                    distributedVec, indices);
+                                           distributedVec, indices);
       // Broadcast: Simply move the vector.insert op out.
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newInsert);
@@ -1531,16 +1534,17 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
     // Insert position: pos % elementsPerLane
     OpFoldResult newPos = affine::makeComposedFoldedAffineApply(
         rewriter, loc, sym0 % elementsPerLane, pos);
-    Value isInsertingLane = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
+    Value isInsertingLane =
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
+                              newWarpOp.getLaneid(), insertingLane);
     Value newResult =
         rewriter
             .create<scf::IfOp>(
                 loc, isInsertingLane,
                 /*thenBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  Value newInsert = vector::InsertOp::create(builder,
-                      loc, newSource, distributedVec, newPos);
+                  Value newInsert = vector::InsertOp::create(
+                      builder, loc, newSource, distributedVec, newPos);
                   scf::YieldOp::create(builder, loc, newInsert);
                 },
                 /*elseBuilder=*/
@@ -1582,8 +1586,9 @@ struct WarpOpInsert : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedSrc = newWarpOp->getResult(newRetIndices[0]);
       Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
-      Value newResult = vector::InsertOp::create(rewriter,
-          loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
+      Value newResult = vector::InsertOp::create(rewriter, loc, distributedSrc,
+                                                 distributedDest,
+                                                 insertOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newResult);
       return success();
@@ -1632,23 +1637,25 @@ struct WarpOpInsert : public WarpDistributionPattern {
     Value newResult;
     if (distrSrcDim >= 0) {
       // Every lane inserts a small piece.
-      newResult = vector::InsertOp::create(rewriter,
-          loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
+      newResult = vector::InsertOp::create(rewriter, loc, distributedSrc,
+                                           distributedDest,
+                                           insertOp.getMixedPosition());
     } else {
       // One lane inserts the entire source vector.
       int64_t elementsPerLane = distrDestType.getDimSize(distrDestDim);
       SmallVector<OpFoldResult> pos = insertOp.getMixedPosition();
       SmallVector<int64_t> newPos = getAsIntegers(pos);
       // tid of inserting lane: pos / elementsPerLane
-      Value insertingLane = arith::ConstantIndexOp::create(rewriter,
-          loc, newPos[distrDestDim] / elementsPerLane);
-      Value isInsertingLane = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
+      Value insertingLane = arith::ConstantIndexOp::create(
+          rewriter, loc, newPos[distrDestDim] / elementsPerLane);
+      Value isInsertingLane =
+          arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
+                                newWarpOp.getLaneid(), insertingLane);
       // Insert position: pos % elementsPerLane
       newPos[distrDestDim] %= elementsPerLane;
       auto insertingBuilder = [&](OpBuilder &builder, Location loc) {
-        Value newInsert = vector::InsertOp::create(builder,
-            loc, distributedSrc, distributedDest, newPos);
+        Value newInsert = vector::InsertOp::create(builder, loc, distributedSrc,
+                                                   distributedDest, newPos);
         scf::YieldOp::create(builder, loc, newInsert);
       };
       auto nonInsertingBuilder = [&](OpBuilder &builder, Location loc) {
@@ -1761,9 +1768,9 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
 
     // Create a new for op outside the region with a WarpExecuteOnLane0Op
     // region inside.
-    auto newForOp = scf::ForOp::create(rewriter,
-        forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-        forOp.getStep(), newOperands);
+    auto newForOp =
+        scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(),
+                           forOp.getUpperBound(), forOp.getStep(), newOperands);
     rewriter.setInsertionPointToStart(newForOp.getBody());
 
     SmallVector<Value> warpInput(newForOp.getRegionIterArgs().begin(),
@@ -1776,9 +1783,10 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
       argIndexMapping[escapingValues[i]] = warpInputType.size();
       warpInputType.push_back(inputTypes[i]);
     }
-    auto innerWarp = WarpExecuteOnLane0Op::create(rewriter,
-        newWarpOp.getLoc(), newForOp.getResultTypes(), newWarpOp.getLaneid(),
-        newWarpOp.getWarpSize(), warpInput, warpInputType);
+    auto innerWarp = WarpExecuteOnLane0Op::create(
+        rewriter, newWarpOp.getLoc(), newForOp.getResultTypes(),
+        newWarpOp.getLaneid(), newWarpOp.getWarpSize(), warpInput,
+        warpInputType);
 
     SmallVector<Value> argMapping;
     argMapping.push_back(newForOp.getInductionVar());
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
index e9718a7795fc5..73388a5da3e4f 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
@@ -77,8 +77,8 @@ struct CastAwayExtractStridedSliceLeadingOneDim
 
     Location loc = extractOp.getLoc();
 
-    Value newSrcVector = vector::ExtractOp::create(rewriter,
-        loc, extractOp.getVector(), splatZero(dropCount));
+    Value newSrcVector = vector::ExtractOp::create(
+        rewriter, loc, extractOp.getVector(), splatZero(dropCount));
 
     // The offsets/sizes/strides attribute can have a less number of elements
     // than the input vector's rank: it is meant for the leading dimensions.
@@ -89,8 +89,9 @@ struct CastAwayExtractStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         extractOp.getStrides().getValue().drop_front(dropCount));
 
-    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
-        loc, newDstType, newSrcVector, newOffsets, newSizes, newStrides);
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(
+        rewriter, loc, newDstType, newSrcVector, newOffsets, newSizes,
+        newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(extractOp, oldDstType,
                                                      newExtractOp);
@@ -120,18 +121,19 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     // Trim leading one dimensions from both operands.
     Location loc = insertOp.getLoc();
 
-    Value newSrcVector = vector::ExtractOp::create(rewriter,
-        loc, insertOp.getValueToStore(), splatZero(srcDropCount));
-    Value newDstVector = vector::ExtractOp::create(rewriter,
-        loc, insertOp.getDest(), splatZero(dstDropCount));
+    Value newSrcVector = vector::ExtractOp::create(
+        rewriter, loc, insertOp.getValueToStore(), splatZero(srcDropCount));
+    Value newDstVector = vector::ExtractOp::create(
+        rewriter, loc, insertOp.getDest(), splatZero(dstDropCount));
 
     auto newOffsets = rewriter.getArrayAttr(
         insertOp.getOffsets().getValue().take_back(newDstType.getRank()));
     auto newStrides = rewriter.getArrayAttr(
         insertOp.getStrides().getValue().take_back(newSrcType.getRank()));
 
-    auto newInsertOp = vector::InsertStridedSliceOp::create(rewriter,
-        loc, newDstType, newSrcVector, newDstVector, newOffsets, newStrides);
+    auto newInsertOp = vector::InsertStridedSliceOp::create(
+        rewriter, loc, newDstType, newSrcVector, newDstVector, newOffsets,
+        newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
                                                      newInsertOp);
@@ -169,11 +171,11 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
 
     Value newSrcVector = insertOp.getValueToStore();
     if (oldSrcRank != 0) {
-      newSrcVector = vector::ExtractOp::create(rewriter,
-          loc, insertOp.getValueToStore(), splatZero(srcDropCount));
+      newSrcVector = vector::ExtractOp::create(
+          rewriter, loc, insertOp.getValueToStore(), splatZero(srcDropCount));
     }
-    Value newDstVector = vector::ExtractOp::create(rewriter,
-        loc, insertOp.getDest(), splatZero(dstDropCount));
+    Value newDstVector = vector::ExtractOp::create(
+        rewriter, loc, insertOp.getDest(), splatZero(dstDropCount));
 
     // New position rank needs to be computed in two steps: (1) if destination
     // type has leading unit dims, we also trim the position array accordingly,
@@ -187,8 +189,8 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
     newPosition.resize(newDstType.getRank() - newSrcRank,
                        rewriter.getI64IntegerAttr(0));
 
-    auto newInsertOp = vector::InsertOp::create(rewriter,
-        loc, newSrcVector, newDstVector, newPosition);
+    auto newInsertOp = vector::InsertOp::create(rewriter, loc, newSrcVector,
+                                                newDstVector, newPosition);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
                                                      newInsertOp);
@@ -259,8 +261,8 @@ struct CastAwayTransferReadLeadingOneDim
                                   newType, newMap, maskType);
     }
 
-    auto newRead = vector::TransferReadOp::create(rewriter,
-        read.getLoc(), newType, read.getBase(), read.getIndices(),
+    auto newRead = vector::TransferReadOp::create(
+        rewriter, read.getLoc(), newType, read.getBase(), read.getIndices(),
         AffineMapAttr::get(newMap), read.getPadding(), mask, inBoundsAttr);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(read, oldType, newRead);
 
@@ -306,8 +308,8 @@ struct CastAwayTransferWriteLeadingOneDim
       inBoundsAttr = rewriter.getArrayAttr(
           write.getInBoundsAttr().getValue().take_back(newType.getRank()));
 
-    auto newVector = vector::ExtractOp::create(rewriter,
-        write.getLoc(), write.getVector(), splatZero(dropDim));
+    auto newVector = vector::ExtractOp::create(
+        rewriter, write.getLoc(), write.getVector(), splatZero(dropDim));
 
     if (write.getMask()) {
       VectorType maskType = write.getMaskType();
@@ -443,22 +445,23 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
                                              contractOp.getContext()));
     // Extract if its a valid extraction, otherwise use the operand
     // without extraction.
-    newOperands.push_back(
-        validExtract ? vector::ExtractOp::create(rewriter,
-                           loc, operands[it.index()], splatZero(dropDim))
-                     : operands[it.index()]);
+    newOperands.push_back(validExtract
+                              ? vector::ExtractOp::create(rewriter, loc,
+                                                          operands[it.index()],
+                                                          splatZero(dropDim))
+                              : operands[it.index()]);
   }
 
   // Depending on whether this vector.contract is masked, the replacing Op
   // should either be a new vector.contract Op or vector.mask Op.
-  Operation *newOp = vector::ContractionOp::create(rewriter,
-      loc, newOperands[0], newOperands[1], newOperands[2],
+  Operation *newOp = vector::ContractionOp::create(
+      rewriter, loc, newOperands[0], newOperands[1], newOperands[2],
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind());
 
   if (maskingOp) {
     auto newMask = vector::ExtractOp::create(rewriter, loc, maskingOp.getMask(),
-                                                      splatZero(dropDim));
+                                             splatZero(dropDim));
 
     newOp = mlir::vector::maskOperation(rewriter, newOp, newMask);
   }
@@ -519,8 +522,8 @@ class CastAwayElementwiseLeadingOneDim : public RewritePattern {
     SmallVector<Value, 4> newOperands;
     for (Value operand : op->getOperands()) {
       if (auto opVecType = dyn_cast<VectorType>(operand.getType())) {
-        newOperands.push_back(vector::ExtractOp::create(rewriter,
-            op->getLoc(), operand, splatZero(dropDim)));
+        newOperands.push_back(vector::ExtractOp::create(
+            rewriter, op->getLoc(), operand, splatZero(dropDim)));
       } else {
         newOperands.push_back(operand);
       }
@@ -559,8 +562,8 @@ struct CastAwayConstantMaskLeadingOneDim
     SmallVector<int64_t> newDimSizes = {flatLeadingSize};
     newDimSizes.append(dimSizes.begin() + dropDim + 1, dimSizes.end());
 
-    auto newMask = vector::ConstantMaskOp::create(rewriter,
-        mask.getLoc(), newType, newDimSizes);
+    auto newMask = vector::ConstantMaskOp::create(rewriter, mask.getLoc(),
+                                                  newType, newDimSizes);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(mask, oldType, newMask);
     return success();
   }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
index 3411e010f3499..cb3e8dc67a1ae 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
@@ -65,13 +65,13 @@ struct VectorMaskedLoadOpConverter final
     Value base = maskedLoadOp.getBase();
     Value iValue = maskedLoadOp.getPassThru();
     auto indices = llvm::to_vector_of<Value>(maskedLoadOp.getIndices());
-    Value one = arith::ConstantOp::create(rewriter,
-        loc, indexType, IntegerAttr::get(indexType, 1));
+    Value one = arith::ConstantOp::create(rewriter, loc, indexType,
+                                          IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
       auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = scf::IfOp::create(rewriter,
-          loc, maskBit,
+      auto ifOp = scf::IfOp::create(
+          rewriter, loc, maskBit,
           [&](OpBuilder &builder, Location loc) {
             auto loadedValue =
                 memref::LoadOp::create(builder, loc, base, indices);
@@ -84,7 +84,8 @@ struct VectorMaskedLoadOpConverter final
           });
       iValue = ifOp.getResult(0);
 
-      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
+      indices.back() =
+          arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.replaceOp(maskedLoadOp, iValue);
@@ -132,8 +133,8 @@ struct VectorMaskedStoreOpConverter final
     Value base = maskedStoreOp.getBase();
     Value value = maskedStoreOp.getValueToStore();
     auto indices = llvm::to_vector_of<Value>(maskedStoreOp.getIndices());
-    Value one = arith::ConstantOp::create(rewriter,
-        loc, indexType, IntegerAttr::get(indexType, 1));
+    Value one = arith::ConstantOp::create(rewriter, loc, indexType,
+                                          IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
       auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
@@ -143,7 +144,8 @@ struct VectorMaskedStoreOpConverter final
       memref::StoreOp::create(rewriter, loc, extractedValue, base, indices);
 
       rewriter.setInsertionPointAfter(ifOp);
-      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
+      indices.back() =
+          arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.eraseOp(maskedStoreOp);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index e915e85f16910..0a8178fe1cabf 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -133,7 +133,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 newMaskOperands.push_back(
                     getValueOrCreateConstantIndexOp(rewriter, loc, maskIndex));
                 return vector::CreateMaskOp::create(rewriter, loc, newMaskType,
-                                                             newMaskOperands);
+                                                    newMaskOperands);
               })
           .Case<vector::ConstantMaskOp>(
               [&](auto constantMaskOp) -> std::optional<Operation *> {
@@ -143,8 +143,8 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 int64_t &maskIndex = maskDimSizes.back();
                 maskIndex = llvm::divideCeil(numFrontPadElems + maskIndex,
                                              numSrcElemsPerDest);
-                return vector::ConstantMaskOp::create(rewriter, loc, newMaskType,
-                                                               maskDimSizes);
+                return vector::ConstantMaskOp::create(
+                    rewriter, loc, newMaskType, maskDimSizes);
               })
           .Case<arith::ConstantOp>([&](auto constantOp)
                                        -> std::optional<Operation *> {
@@ -182,16 +182,18 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
               }
               compressedMaskValues.push_back(combinedValue);
             }
-            return arith::ConstantOp::create(rewriter,
-                loc, DenseElementsAttr::get(newMaskType, compressedMaskValues));
+            return arith::ConstantOp::create(
+                rewriter, loc,
+                DenseElementsAttr::get(newMaskType, compressedMaskValues));
           });
 
   if (!newMask)
     return failure();
 
   while (!extractOps.empty()) {
-    newMask = vector::ExtractOp::create(rewriter,
-        loc, (*newMask)->getResults()[0], extractOps.back().getMixedPosition());
+    newMask =
+        vector::ExtractOp::create(rewriter, loc, (*newMask)->getResults()[0],
+                                  extractOps.back().getMixedPosition());
     extractOps.pop_back();
   }
 
@@ -259,7 +261,7 @@ static Value staticallyInsertSubvector(OpBuilder &rewriter, Location loc,
   auto offsets = rewriter.getI64ArrayAttr({offset});
   auto strides = rewriter.getI64ArrayAttr({1});
   return vector::InsertStridedSliceOp::create(rewriter, loc, destVecTy, src,
-                                                       dest, offsets, strides);
+                                              dest, offsets, strides);
 }
 
 /// Extracts 1-D subvector from a 1-D vector.
@@ -301,8 +303,9 @@ static Value dynamicallyExtractSubVector(OpBuilder &rewriter, Location loc,
   for (int i = 0; i < numElemsToExtract; ++i) {
     Value extractLoc =
         (i == 0) ? dyn_cast<Value>(offset)
-                 : arith::AddIOp::create(rewriter,
-                       loc, rewriter.getIndexType(), dyn_cast<Value>(offset),
+                 : arith::AddIOp::create(
+                       rewriter, loc, rewriter.getIndexType(),
+                       dyn_cast<Value>(offset),
                        arith::ConstantIndexOp::create(rewriter, loc, i));
     auto extractOp = vector::ExtractOp::create(rewriter, loc, src, extractLoc);
     dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, i);
@@ -344,11 +347,11 @@ static Value dynamicallyInsertSubVector(RewriterBase &rewriter, Location loc,
 
   Value destOffsetVal = getValueOrCreateConstantIndexOp(rewriter, loc, offset);
   for (int64_t i = 0; i < numElemsToInsert; ++i) {
-    auto insertLoc = i == 0
-                         ? destOffsetVal
-                         : arith::AddIOp::create(rewriter,
-                               loc, rewriter.getIndexType(), destOffsetVal,
-                               arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto insertLoc =
+        i == 0 ? destOffsetVal
+               : arith::AddIOp::create(
+                     rewriter, loc, rewriter.getIndexType(), destOffsetVal,
+                     arith::ConstantIndexOp::create(rewriter, loc, i));
     auto extractOp = vector::ExtractOp::create(rewriter, loc, src, i);
     dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, insertLoc);
   }
@@ -369,11 +372,11 @@ static VectorValue emulatedVectorLoad(OpBuilder &rewriter, Location loc,
                                       Type containerElemTy) {
   auto emulatedPerContainerElem = containerElemTy.getIntOrFloatBitWidth() /
                                   emulatedElemTy.getIntOrFloatBitWidth();
-  auto newLoad = vector::LoadOp::create(rewriter,
-      loc, VectorType::get(numContainerElemsToLoad, containerElemTy), base,
-      getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
-  return vector::BitCastOp::create(rewriter,
-      loc,
+  auto newLoad = vector::LoadOp::create(
+      rewriter, loc, VectorType::get(numContainerElemsToLoad, containerElemTy),
+      base, getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
+  return vector::BitCastOp::create(
+      rewriter, loc,
       VectorType::get(numContainerElemsToLoad * emulatedPerContainerElem,
                       emulatedElemTy),
       newLoad);
@@ -390,7 +393,8 @@ static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
           upcastType.getNumElements() * upcastType.getElementTypeBitWidth() &&
       "expected input and output number of bits to match");
   if (trueValue.getType() != downcastType) {
-    trueValue = vector::BitCastOp::create(builder, loc, downcastType, trueValue);
+    trueValue =
+        vector::BitCastOp::create(builder, loc, downcastType, trueValue);
   }
   if (falseValue.getType() != downcastType) {
     falseValue =
@@ -422,8 +426,8 @@ static void atomicRMW(OpBuilder &builder, Location loc,
 
   // Create an atomic load-modify-write region using
   // `memref.generic_atomic_rmw`.
-  auto atomicOp = memref::GenericAtomicRMWOp::create(builder,
-      loc, linearizedMemref, ValueRange{storeIdx});
+  auto atomicOp = memref::GenericAtomicRMWOp::create(
+      builder, loc, linearizedMemref, ValueRange{storeIdx});
   Value origValue = atomicOp.getCurrentValue();
 
   OpBuilder::InsertionGuard guard(builder);
@@ -432,8 +436,8 @@ static void atomicRMW(OpBuilder &builder, Location loc,
   // Load the original value from memory, and cast it to the original element
   // type.
   auto oneElemVecType = VectorType::get({1}, origValue.getType());
-  Value origVecValue = vector::FromElementsOp::create(builder,
-      loc, oneElemVecType, ValueRange{origValue});
+  Value origVecValue = vector::FromElementsOp::create(
+      builder, loc, oneElemVecType, ValueRange{origValue});
 
   // Construct the final masked value and yield it.
   Value maskedValue =
@@ -453,16 +457,17 @@ static void nonAtomicRMW(OpBuilder &builder, Location loc,
 
   auto oneElemVecType =
       VectorType::get({1}, linearizedMemref.getType().getElementType());
-  Value origVecValue = vector::LoadOp::create(builder,
-      loc, oneElemVecType, linearizedMemref, ValueRange{linearizedIndex});
+  Value origVecValue =
+      vector::LoadOp::create(builder, loc, oneElemVecType, linearizedMemref,
+                             ValueRange{linearizedIndex});
   origVecValue = vector::BitCastOp::create(builder, loc, valueToStore.getType(),
-                                                   origVecValue);
+                                           origVecValue);
 
   Value maskedValue =
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
   vector::StoreOp::create(builder, loc, maskedValue, linearizedMemref,
-                                  linearizedIndex);
+                          linearizedIndex);
 }
 
 /// Extract `sliceNumElements` from source `vector` at `extractOffset`,
@@ -489,8 +494,9 @@ static Value extractSliceIntoByte(ConversionPatternRewriter &rewriter,
   assert(8 % vectorElementType.getIntOrFloatBitWidth() == 0 &&
          "vector element must be a valid sub-byte type");
   auto emulatedPerContainerElem = 8 / vectorElementType.getIntOrFloatBitWidth();
-  auto emptyByteVector = arith::ConstantOp::create(rewriter,
-      loc, VectorType::get({emulatedPerContainerElem}, vectorElementType),
+  auto emptyByteVector = arith::ConstantOp::create(
+      rewriter, loc,
+      VectorType::get({emulatedPerContainerElem}, vectorElementType),
       rewriter.getZeroAttr(
           VectorType::get({emulatedPerContainerElem}, vectorElementType)));
   auto extracted = staticallyExtractSubvector(rewriter, loc, vector,
@@ -664,8 +670,8 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     if (!emulationRequiresPartialStores) {
       // Basic case: storing full bytes.
       auto numElements = origElements / emulatedPerContainerElem;
-      auto bitCast = vector::BitCastOp::create(rewriter,
-          loc, VectorType::get(numElements, containerElemTy),
+      auto bitCast = vector::BitCastOp::create(
+          rewriter, loc, VectorType::get(numElements, containerElemTy),
           op.getValueToStore());
       rewriter.replaceOpWithNewOp<vector::StoreOp>(
           op, bitCast.getResult(), memrefBase,
@@ -732,8 +738,9 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         std::fill_n(frontMaskValues.end() - frontSubWidthStoreElem,
                     *foldedNumFrontPadElems, true);
       }
-      auto frontMask = arith::ConstantOp::create(rewriter,
-          loc, DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
+      auto frontMask = arith::ConstantOp::create(
+          rewriter, loc,
+          DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
 
       currentSourceIndex = emulatedPerContainerElem - (*foldedNumFrontPadElems);
       auto value =
@@ -752,8 +759,8 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     // Increment the destination index by 1 to align to the emulated width
     // boundary.
     auto constantOne = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    currentDestIndex = arith::AddIOp::create(rewriter,
-        loc, rewriter.getIndexType(), currentDestIndex, constantOne);
+    currentDestIndex = arith::AddIOp::create(
+        rewriter, loc, rewriter.getIndexType(), currentDestIndex, constantOne);
 
     // 2. Full width store for the inner output bytes.
     // After the previous step, the store address is aligned to the emulated
@@ -773,13 +780,13 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
           {originType.getNumElements() / emulatedPerContainerElem},
           memrefElemType);
       auto bitCast = vector::BitCastOp::create(rewriter, loc, storeType,
-                                                        fullWidthStorePart);
+                                               fullWidthStorePart);
       vector::StoreOp::create(rewriter, loc, bitCast.getResult(), memrefBase,
-                                       currentDestIndex);
+                              currentDestIndex);
 
       currentSourceIndex += numNonFullWidthElements;
-      currentDestIndex = arith::AddIOp::create(rewriter,
-          loc, rewriter.getIndexType(), currentDestIndex,
+      currentDestIndex = arith::AddIOp::create(
+          rewriter, loc, rewriter.getIndexType(), currentDestIndex,
           arith::ConstantIndexOp::create(rewriter, loc, fullWidthStoreSize));
     }
 
@@ -795,8 +802,9 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       // Generate back mask.
       auto maskValues = SmallVector<bool>(emulatedPerContainerElem, 0);
       std::fill_n(maskValues.begin(), remainingElements, 1);
-      auto backMask = arith::ConstantOp::create(rewriter,
-          loc, DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
+      auto backMask = arith::ConstantOp::create(
+          rewriter, loc,
+          DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
 
       storeFunc(rewriter, loc, memrefBase, currentDestIndex,
                 cast<VectorValue>(subWidthStorePart), backMask.getResult());
@@ -901,19 +909,19 @@ struct ConvertVectorMaskedStore final
     auto numElements = (origElements + emulatedPerContainerElem - 1) /
                        emulatedPerContainerElem;
     auto newType = VectorType::get(numElements, containerElemTy);
-    auto passThru = arith::ConstantOp::create(rewriter,
-        loc, newType, rewriter.getZeroAttr(newType));
+    auto passThru = arith::ConstantOp::create(rewriter, loc, newType,
+                                              rewriter.getZeroAttr(newType));
 
-    auto newLoad = vector::MaskedLoadOp::create(rewriter,
-        loc, newType, adaptor.getBase(), linearizedIndices,
+    auto newLoad = vector::MaskedLoadOp::create(
+        rewriter, loc, newType, adaptor.getBase(), linearizedIndices,
         newMask.value()->getResult(0), passThru);
 
     auto newBitCastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
     Value valueToStore =
         vector::BitCastOp::create(rewriter, loc, newBitCastType, newLoad);
-    valueToStore = arith::SelectOp::create(rewriter,
-        loc, op.getMask(), op.getValueToStore(), valueToStore);
+    valueToStore = arith::SelectOp::create(rewriter, loc, op.getMask(),
+                                           op.getValueToStore(), valueToStore);
     valueToStore =
         vector::BitCastOp::create(rewriter, loc, newType, valueToStore);
 
@@ -1016,8 +1024,8 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
                            numElements, emulatedElemTy, containerElemTy);
 
     if (!foldedIntraVectorOffset) {
-      auto resultVector = arith::ConstantOp::create(rewriter,
-          loc, op.getType(), rewriter.getZeroAttr(op.getType()));
+      auto resultVector = arith::ConstantOp::create(
+          rewriter, loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(
           rewriter, loc, dyn_cast<TypedValue<VectorType>>(result), resultVector,
           linearizedInfo.intraDataOffset, origElements);
@@ -1142,8 +1150,8 @@ struct ConvertVectorMaskedLoad final
     auto newBitcastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
 
-    auto emptyVector = arith::ConstantOp::create(rewriter,
-        loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
+    auto emptyVector = arith::ConstantOp::create(
+        rewriter, loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
     if (!foldedIntraVectorOffset) {
       passthru = dynamicallyInsertSubVector(
           rewriter, loc, passthru, emptyVector, linearizedInfo.intraDataOffset,
@@ -1156,8 +1164,8 @@ struct ConvertVectorMaskedLoad final
         vector::BitCastOp::create(rewriter, loc, loadType, passthru);
 
     // Generating the new masked load.
-    auto newLoad = vector::MaskedLoadOp::create(rewriter,
-        loc, loadType, adaptor.getBase(),
+    auto newLoad = vector::MaskedLoadOp::create(
+        rewriter, loc, loadType, adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newMask.value()->getResult(0), newPassThru);
 
@@ -1170,8 +1178,9 @@ struct ConvertVectorMaskedLoad final
     auto newSelectMaskType = VectorType::get(
         numElements * emulatedPerContainerElem, rewriter.getI1Type());
     // TODO: try to fold if op's mask is constant
-    auto emptyMask = arith::ConstantOp::create(rewriter,
-        loc, newSelectMaskType, rewriter.getZeroAttr(newSelectMaskType));
+    auto emptyMask =
+        arith::ConstantOp::create(rewriter, loc, newSelectMaskType,
+                                  rewriter.getZeroAttr(newSelectMaskType));
     if (!foldedIntraVectorOffset) {
       mask = dynamicallyInsertSubVector(rewriter, loc, mask, emptyMask,
                                         linearizedInfo.intraDataOffset,
@@ -1277,10 +1286,8 @@ struct ConvertVectorTransferRead final
           IntegerType::get(rewriter.getContext(),
                            padding.getType().getIntOrFloatBitWidth()),
           padding);
-    }
-    auto newPadding =
-        arith::ExtUIOp::create(rewriter, loc, containerElemTy,
-                                                      padding);
+    }auto newPadding = arith::ExtUIOp::create(rewriter, loc, containerElemTy,
+                                             padding);
 
     auto stridedMetadata =
         memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
@@ -1304,20 +1311,21 @@ struct ConvertVectorTransferRead final
     auto numElements = llvm::divideCeil(maxIntraDataOffset + origElements,
                                         emulatedPerContainerElem);
 
-    auto newRead = vector::TransferReadOp::create(rewriter,
-        loc, VectorType::get(numElements, containerElemTy), adaptor.getBase(),
+    auto newRead = vector::TransferReadOp::create(
+        rewriter, loc, VectorType::get(numElements, containerElemTy),
+        adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newPadding);
 
-    auto bitCast = vector::BitCastOp::create(rewriter,
-        loc,
+    auto bitCast = vector::BitCastOp::create(
+        rewriter, loc,
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy),
         newRead);
 
     Value result = bitCast->getResult(0);
     if (!foldedIntraVectorOffset) {
-      auto zeros = arith::ConstantOp::create(rewriter,
-          loc, op.getType(), rewriter.getZeroAttr(op.getType()));
+      auto zeros = arith::ConstantOp::create(
+          rewriter, loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(rewriter, loc, bitCast, zeros,
                                            linearizedInfo.intraDataOffset,
                                            origElements);
@@ -1690,25 +1698,26 @@ Value BitCastRewriter::genericRewriteStep(
     PatternRewriter &rewriter, Location loc, Value initialValue,
     Value runningResult, const BitCastRewriter::Metadata &metadata) {
   // Create vector.shuffle from the metadata.
-  auto shuffleOp = vector::ShuffleOp::create(rewriter,
-      loc, initialValue, initialValue, metadata.shuffles);
+  auto shuffleOp = vector::ShuffleOp::create(rewriter, loc, initialValue,
+                                             initialValue, metadata.shuffles);
 
   // Intersect with the mask.
   VectorType shuffledVectorType = shuffleOp.getResultVectorType();
-  auto constOp = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(shuffledVectorType, metadata.masks));
+  auto constOp = arith::ConstantOp::create(
+      rewriter, loc,
+      DenseElementsAttr::get(shuffledVectorType, metadata.masks));
   Value andValue = arith::AndIOp::create(rewriter, loc, shuffleOp, constOp);
 
   // Align right on 0.
-  auto shiftRightConstantOp = arith::ConstantOp::create(rewriter,
-      loc,
+  auto shiftRightConstantOp = arith::ConstantOp::create(
+      rewriter, loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftRightAmounts));
   Value shiftedRight =
       arith::ShRUIOp::create(rewriter, loc, andValue, shiftRightConstantOp);
 
   // Shift bits left into their final position.
-  auto shiftLeftConstantOp = arith::ConstantOp::create(rewriter,
-      loc,
+  auto shiftLeftConstantOp = arith::ConstantOp::create(
+      rewriter, loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftLeftAmounts));
   Value shiftedLeft =
       arith::ShLIOp::create(rewriter, loc, shiftedRight, shiftLeftConstantOp);
@@ -1766,14 +1775,14 @@ static Value extractNBitsPerByteAndSignExtendToI8(PatternRewriter &rewriter,
   assert(bitIdx >= 0 && bitsToShiftLeft >= 0 && numBits > 0 && numBits <= 8 &&
          "Invalid bitIdx range");
   if (bitsToShiftLeft != 0) {
-    Value shiftLeftValues = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(srcType, bitsToShiftLeft));
+    Value shiftLeftValues = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(srcType, bitsToShiftLeft));
     shl = arith::ShLIOp::create(rewriter, loc, src, shiftLeftValues);
   }
 
   int8_t bitsToShiftRight = 8 - numBits;
-  Value shiftRightValues = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
+  Value shiftRightValues = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
   Value shr = arith::ShRSIOp::create(rewriter, loc, shl, shiftRightValues);
   return shr;
 }
@@ -1808,16 +1817,16 @@ static Value extractNBitsPerByteAndExtendToI8(PatternRewriter &rewriter,
   int8_t bitsToShiftRight = bitIdx;
   Value shr = src;
   if (bitsToShiftRight != 0) {
-    Value shiftRightValues = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
+    Value shiftRightValues = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
     shr = arith::ShRUIOp::create(rewriter, loc, src, shiftRightValues);
   }
   if (bitIdx + numBits == 8) {
     return shr;
   }
   uint8_t lowBitsMask = (1 << numBits) - 1;
-  Value lowBitsMaskValues = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(srcType, lowBitsMask));
+  Value lowBitsMaskValues = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(srcType, lowBitsMask));
   return arith::AndIOp::create(rewriter, loc, shr, lowBitsMaskValues);
 }
 
@@ -1876,7 +1885,8 @@ static Value rewriteI2ToI8Ext(PatternRewriter &rewriter, Location loc,
   // 0213  = [0,1,2,3,...],...
   Value interleave02 = vector::InterleaveOp::create(rewriter, loc, vec0, vec2);
   Value interleave13 = vector::InterleaveOp::create(rewriter, loc, vec1, vec3);
-  return vector::InterleaveOp::create(rewriter, loc, interleave02, interleave13);
+  return vector::InterleaveOp::create(rewriter, loc, interleave02,
+                                      interleave13);
 }
 
 /// Rewrite the i8 -> i4 truncation into a deinterleave and series of bitwise
@@ -1893,17 +1903,17 @@ static Value rewriteI8ToI4Trunc(PatternRewriter &rewriter, Location loc,
   // 2. Zero out the upper side of each low i8 element.
   constexpr int8_t i8LowBitMask = 0x0F;
   VectorType deinterI8VecType = deinterleaveOp.getResultVectorType();
-  Value zeroOutMask = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(deinterI8VecType, i8LowBitMask));
-  Value zeroOutLow = arith::AndIOp::create(rewriter,
-      loc, deinterleaveOp.getRes1(), zeroOutMask);
+  Value zeroOutMask = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(deinterI8VecType, i8LowBitMask));
+  Value zeroOutLow = arith::AndIOp::create(
+      rewriter, loc, deinterleaveOp.getRes1(), zeroOutMask);
 
   // 3. Move high i4 values to upper side of the byte.
   constexpr int8_t bitsToShift = 4;
-  auto shiftValues = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(deinterI8VecType, bitsToShift));
+  auto shiftValues = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(deinterI8VecType, bitsToShift));
   Value shlHigh = arith::ShLIOp::create(rewriter, loc, deinterleaveOp.getRes2(),
-                                                 shiftValues);
+                                        shiftValues);
 
   // 4. Merge high and low i4 values.
   auto mergedHiLowOp = arith::OrIOp::create(rewriter, loc, zeroOutLow, shlHigh);
@@ -2201,9 +2211,9 @@ struct RewriteVectorTranspose : OpRewritePattern<vector::TransposeOp> {
     auto srcNativeVecType = srcSubByteVecType.cloneWith(
         std::nullopt, rewriter.getIntegerType(minNativeBitwidth));
     Value extOp = arith::ExtSIOp::create(rewriter, loc, srcNativeVecType,
-                                                  transposeOp.getVector());
-    Value newTranspose = vector::TransposeOp::create(rewriter,
-        loc, extOp, transposeOp.getPermutation());
+                                         transposeOp.getVector());
+    Value newTranspose = vector::TransposeOp::create(
+        rewriter, loc, extOp, transposeOp.getPermutation());
     VectorType dstSubByteVecType = transposeOp.getResultVectorType();
     rewriter.replaceOpWithNewOp<arith::TruncIOp>(transposeOp, dstSubByteVecType,
                                                  newTranspose);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
index baeca29298334..72352d72bfe77 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
@@ -53,15 +53,15 @@ class DecomposeDifferentRankInsertStridedSlice
     int64_t rankRest = dstType.getRank() - rankDiff;
     // Extract / insert the subvector of matching rank and InsertStridedSlice
     // on it.
-    Value extracted = ExtractOp::create(rewriter,
-        loc, op.getDest(),
-        getI64SubArray(op.getOffsets(), /*dropFront=*/0,
-                       /*dropBack=*/rankRest));
+    Value extracted =
+        ExtractOp::create(rewriter, loc, op.getDest(),
+                          getI64SubArray(op.getOffsets(), /*dropFront=*/0,
+                                         /*dropBack=*/rankRest));
 
     // A different pattern will kick in for InsertStridedSlice with matching
     // ranks.
-    auto stridedSliceInnerOp = InsertStridedSliceOp::create(rewriter,
-        loc, op.getValueToStore(), extracted,
+    auto stridedSliceInnerOp = InsertStridedSliceOp::create(
+        rewriter, loc, op.getValueToStore(), extracted,
         getI64SubArray(op.getOffsets(), /*dropFront=*/rankDiff),
         getI64SubArray(op.getStrides(), /*dropFront=*/0));
 
@@ -131,8 +131,8 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
       SmallVector<int64_t> offsets(nDest, 0);
       for (int64_t i = 0; i < nSrc; ++i)
         offsets[i] = i;
-      Value scaledSource = ShuffleOp::create(rewriter,
-          loc, op.getValueToStore(), op.getValueToStore(), offsets);
+      Value scaledSource = ShuffleOp::create(
+          rewriter, loc, op.getValueToStore(), op.getValueToStore(), offsets);
 
       // 2. Create a mask where we take the value from scaledSource of dest
       // depending on the offset.
@@ -164,8 +164,8 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
             ExtractOp::create(rewriter, loc, op.getDest(), off);
         // 3. Reduce the problem to lowering a new InsertStridedSlice op with
         // smaller rank.
-        extractedSource = InsertStridedSliceOp::create(rewriter,
-            loc, extractedSource, extractedDest,
+        extractedSource = InsertStridedSliceOp::create(
+            rewriter, loc, extractedSource, extractedDest,
             getI64SubArray(op.getOffsets(), /* dropFront=*/1),
             getI64SubArray(op.getStrides(), /* dropFront=*/1));
       }
@@ -252,8 +252,8 @@ class Convert1DExtractStridedSliceIntoExtractInsertChain final
     for (int64_t i = offset, e = offset + size * stride; i < e; i += stride)
       elements.push_back(ExtractOp::create(rewriter, loc, op.getVector(), i));
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(op.getType()));
+    Value result = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getZeroAttr(op.getType()));
     for (int64_t i = 0; i < size; ++i)
       result = InsertOp::create(rewriter, loc, elements[i], result, i);
 
@@ -301,14 +301,14 @@ class DecomposeNDExtractStridedSlice
       return failure();
 
     // Extract/insert on a lower ranked extract strided slice op.
-    Value zero = arith::ConstantOp::create(rewriter,
-        loc, elemType, rewriter.getZeroAttr(elemType));
+    Value zero = arith::ConstantOp::create(rewriter, loc, elemType,
+                                           rewriter.getZeroAttr(elemType));
     Value res = SplatOp::create(rewriter, loc, dstType, zero);
     for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
          off += stride, ++idx) {
       Value one = ExtractOp::create(rewriter, loc, op.getVector(), off);
-      Value extracted = ExtractStridedSliceOp::create(rewriter,
-          loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
+      Value extracted = ExtractStridedSliceOp::create(
+          rewriter, loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
           getI64SubArray(op.getSizes(), /* dropFront=*/1),
           getI64SubArray(op.getStrides(), /* dropFront=*/1));
       res = InsertOp::create(rewriter, loc, extracted, res, idx);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
index 70d786a516859..4233212b5cbbe 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
@@ -84,8 +84,8 @@ LogicalResult resolveAllTrueCreateMaskOp(IRRewriter &rewriter,
   // Replace createMaskOp with an all-true constant. This should result in the
   // mask being removed in most cases (as xfer ops + vector.mask have folds to
   // remove all-true masks).
-  auto allTrue = vector::ConstantMaskOp::create(rewriter,
-      createMaskOp.getLoc(), maskType, ConstantMaskKind::AllTrue);
+  auto allTrue = vector::ConstantMaskOp::create(
+      rewriter, createMaskOp.getLoc(), maskType, ConstantMaskKind::AllTrue);
   rewriter.replaceAllUsesWith(createMaskOp, allTrue);
   return success();
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 9bbedc396664b..2676d254c9b64 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -287,7 +287,7 @@ static Value rankReducingSubviewDroppingUnitDims(PatternRewriter &rewriter,
       inputType.canonicalizeStridedLayout())
     return input;
   return memref::SubViewOp::create(rewriter, loc, resultType, input, offsets,
-                                            sizes, strides);
+                                   sizes, strides);
 }
 
 /// Returns the number of dims that aren't unit dims.
@@ -399,9 +399,9 @@ class TransferReadDropUnitDimsPattern
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
-    Operation *newTransferReadOp = vector::TransferReadOp::create(rewriter,
-        loc, reducedVectorType, reducedShapeSource, zeros, identityMap,
-        transferReadOp.getPadding(), maskOp,
+    Operation *newTransferReadOp = vector::TransferReadOp::create(
+        rewriter, loc, reducedVectorType, reducedShapeSource, zeros,
+        identityMap, transferReadOp.getPadding(), maskOp,
         rewriter.getBoolArrayAttr(inBounds));
 
     if (maskingOp) {
@@ -483,9 +483,9 @@ class TransferWriteDropUnitDimsPattern
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
     auto shapeCastSrc = rewriter.createOrFold<vector::ShapeCastOp>(
         loc, reducedVectorType, vector);
-    Operation *newXferWrite = vector::TransferWriteOp::create(rewriter,
-        loc, Type(), shapeCastSrc, reducedShapeSource, zeros, identityMap,
-        maskOp, rewriter.getBoolArrayAttr(inBounds));
+    Operation *newXferWrite = vector::TransferWriteOp::create(
+        rewriter, loc, Type(), shapeCastSrc, reducedShapeSource, zeros,
+        identityMap, maskOp, rewriter.getBoolArrayAttr(inBounds));
 
     if (maskingOp) {
       auto shapeCastMask = rewriter.createOrFold<vector::ShapeCastOp>(
@@ -573,8 +573,8 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   if (auto value = dyn_cast<Value>(collapsedOffset)) {
     indicesAfterCollapsing.push_back(value);
   } else {
-    indicesAfterCollapsing.push_back(arith::ConstantIndexOp::create(rewriter,
-        loc, *getConstantIntValue(collapsedOffset)));
+    indicesAfterCollapsing.push_back(arith::ConstantIndexOp::create(
+        rewriter, loc, *getConstantIntValue(collapsedOffset)));
   }
 
   return indicesAfterCollapsing;
@@ -659,8 +659,8 @@ class FlattenContiguousRowMajorTransferReadPattern
     // 3. Create new vector.transfer_read that reads from the collapsed memref
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
-    vector::TransferReadOp flatRead = vector::TransferReadOp::create(rewriter,
-        loc, flatVectorType, collapsedSource, collapsedIndices,
+    vector::TransferReadOp flatRead = vector::TransferReadOp::create(
+        rewriter, loc, flatVectorType, collapsedSource, collapsedIndices,
         transferReadOp.getPadding(), collapsedMap);
     flatRead.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
@@ -758,9 +758,9 @@ class FlattenContiguousRowMajorTransferWritePattern
                                                 vectorType.getElementType());
     Value flatVector =
         vector::ShapeCastOp::create(rewriter, loc, flatVectorType, vector);
-    vector::TransferWriteOp flatWrite =
-        vector::TransferWriteOp::create(rewriter,
-            loc, flatVector, collapsedSource, collapsedIndices, collapsedMap);
+    vector::TransferWriteOp flatWrite = vector::TransferWriteOp::create(
+        rewriter, loc, flatVector, collapsedSource, collapsedIndices,
+        collapsedMap);
     flatWrite.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
     // 4. Replace the old transfer_write with the new one writing the
@@ -846,8 +846,8 @@ class RewriteScalarExtractOfTransferRead
       if (auto value = dyn_cast<Value>(composedIdx)) {
         newIndices[idx] = value;
       } else {
-        newIndices[idx] = arith::ConstantIndexOp::create(rewriter,
-            extractOp.getLoc(), *getConstantIntValue(composedIdx));
+        newIndices[idx] = arith::ConstantIndexOp::create(
+            rewriter, extractOp.getLoc(), *getConstantIntValue(composedIdx));
       }
     }
     if (isa<MemRefType>(xferOp.getBase().getType())) {
@@ -883,8 +883,8 @@ class RewriteScalarWrite : public OpRewritePattern<vector::TransferWriteOp> {
     if (!xferOp.getPermutationMap().isMinorIdentity())
       return failure();
     // Only float and integer element types are supported.
-    Value scalar =
-        vector::ExtractOp::create(rewriter, xferOp.getLoc(), xferOp.getVector());
+    Value scalar = vector::ExtractOp::create(rewriter, xferOp.getLoc(),
+                                             xferOp.getVector());
     // Construct a scalar store.
     if (isa<MemRefType>(xferOp.getBase().getType())) {
       rewriter.replaceOpWithNewOp<memref::StoreOp>(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
index a1f36a9744932..71de9839f0f5c 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
@@ -65,8 +65,8 @@ static Value createInBoundsCond(RewriterBase &b,
       return;
     Value cond =
         arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sle,
-                                getValueOrCreateConstantIndexOp(b, loc, sum),
-                                getValueOrCreateConstantIndexOp(b, loc, dimSz));
+                              getValueOrCreateConstantIndexOp(b, loc, sum),
+                              getValueOrCreateConstantIndexOp(b, loc, dimSz));
     // Conjunction over all dims for which we are in-bounds.
     if (inBoundsCond)
       inBoundsCond = arith::AndIOp::create(b, loc, inBoundsCond, cond);
@@ -177,7 +177,8 @@ static Value castToCompatibleMemRefType(OpBuilder &b, Value memref,
     sourceType = MemRefType::get(
         sourceType.getShape(), sourceType.getElementType(),
         sourceType.getLayout(), compatibleMemRefType.getMemorySpace());
-    res = memref::MemorySpaceCastOp::create(b, memref.getLoc(), sourceType, res);
+    res =
+        memref::MemorySpaceCastOp::create(b, memref.getLoc(), sourceType, res);
   }
   if (sourceType == compatibleMemRefType)
     return res;
@@ -211,8 +212,9 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
     SmallVector<AffineMap, 4> maps =
         AffineMap::inferFromExprList(MapList{{i - j, k}}, b.getContext());
     // affine_min(%dimMemRef - %index, %dimAlloc)
-    Value affineMin = affine::AffineMinOp::create(b,
-        loc, index.getType(), maps[0], ValueRange{dimMemRef, index, dimAlloc});
+    Value affineMin =
+        affine::AffineMinOp::create(b, loc, index.getType(), maps[0],
+                                    ValueRange{dimMemRef, index, dimAlloc});
     sizes.push_back(affineMin);
   });
 
@@ -220,10 +222,10 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
       xferOp.getIndices(), [](Value idx) -> OpFoldResult { return idx; }));
   SmallVector<OpFoldResult> destIndices(memrefRank, b.getIndexAttr(0));
   SmallVector<OpFoldResult> strides(memrefRank, b.getIndexAttr(1));
-  auto copySrc = memref::SubViewOp::create(b,
-      loc, isaWrite ? alloc : xferOp.getBase(), srcIndices, sizes, strides);
-  auto copyDest = memref::SubViewOp::create(b,
-      loc, isaWrite ? xferOp.getBase() : alloc, destIndices, sizes, strides);
+  auto copySrc = memref::SubViewOp::create(
+      b, loc, isaWrite ? alloc : xferOp.getBase(), srcIndices, sizes, strides);
+  auto copyDest = memref::SubViewOp::create(
+      b, loc, isaWrite ? xferOp.getBase() : alloc, destIndices, sizes, strides);
   return std::make_pair(copySrc, copyDest);
 }
 
@@ -253,8 +255,8 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
   Location loc = xferOp.getLoc();
   Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return scf::IfOp::create(b,
-      loc, inBoundsCond,
+  return scf::IfOp::create(
+      b, loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
@@ -263,7 +265,7 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
       },
       [&](OpBuilder &b, Location loc) {
         linalg::FillOp::create(b, loc, ValueRange{xferOp.getPadding()},
-                                 ValueRange{alloc});
+                               ValueRange{alloc});
         // Take partial subview of memref which guarantees no dimension
         // overflows.
         IRRewriter rewriter(b);
@@ -306,8 +308,8 @@ static scf::IfOp createFullPartialVectorTransferRead(
   scf::IfOp fullPartialIfOp;
   Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return scf::IfOp::create(b,
-      loc, inBoundsCond,
+  return scf::IfOp::create(
+      b, loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
@@ -317,10 +319,10 @@ static scf::IfOp createFullPartialVectorTransferRead(
       [&](OpBuilder &b, Location loc) {
         Operation *newXfer = b.clone(*xferOp.getOperation());
         Value vector = cast<VectorTransferOpInterface>(newXfer).getVector();
-        memref::StoreOp::create(b,
-            loc, vector,
-            vector::TypeCastOp::create(b,
-                loc, MemRefType::get({}, vector.getType()), alloc));
+        memref::StoreOp::create(
+            b, loc, vector,
+            vector::TypeCastOp::create(
+                b, loc, MemRefType::get({}, vector.getType()), alloc));
 
         Value casted =
             castToCompatibleMemRefType(b, alloc, compatibleMemRefType);
@@ -391,8 +393,8 @@ static void createFullPartialLinalgCopy(RewriterBase &b,
                                         vector::TransferWriteOp xferOp,
                                         Value inBoundsCond, Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = arith::XOrIOp::create(b,
-      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  auto notInBounds = arith::XOrIOp::create(
+      b, loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
   scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRRewriter rewriter(b);
     std::pair<Value, Value> copyArgs = createSubViewIntersection(
@@ -420,14 +422,14 @@ static void createFullPartialVectorTransferWrite(RewriterBase &b,
                                                  Value inBoundsCond,
                                                  Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = arith::XOrIOp::create(b,
-      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  auto notInBounds = arith::XOrIOp::create(
+      b, loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
   scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRMapping mapping;
-    Value load = memref::LoadOp::create(b,
-        loc,
-        vector::TypeCastOp::create(b,
-            loc, MemRefType::get({}, xferOp.getVector().getType()), alloc),
+    Value load = memref::LoadOp::create(
+        b, loc,
+        vector::TypeCastOp::create(
+            b, loc, MemRefType::get({}, xferOp.getVector().getType()), alloc),
         ValueRange());
     mapping.map(xferOp.getVector(), load);
     b.clone(*xferOp.getOperation(), mapping);
@@ -562,8 +564,8 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
     auto shape = xferOp.getVectorType().getShape();
     Type elementType = xferOp.getVectorType().getElementType();
     alloc = memref::AllocaOp::create(b, scope->getLoc(),
-                                       MemRefType::get(shape, elementType),
-                                       ValueRange{}, b.getI64IntegerAttr(32));
+                                     MemRefType::get(shape, elementType),
+                                     ValueRange{}, b.getI64IntegerAttr(32));
   }
 
   MemRefType compatibleMemRefType =
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index d318b4b16b5aa..73ca327bb49c5 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -381,8 +381,8 @@ FailureOr<Value> combineContractAndBroadcast(vector::ContractionOp contractOp,
   if (getUnusedDimsBitVector({maps[0], maps[1]}).any())
     return failure();
 
-  Operation *newOp = vector::ContractionOp::create(rewriter,
-      contractOp.getLoc(), lhs, rhs, contractOp.getAcc(),
+  Operation *newOp = vector::ContractionOp::create(
+      rewriter, contractOp.getLoc(), lhs, rhs, contractOp.getAcc(),
       rewriter.getAffineMapArrayAttr(maps), rewriter.getArrayAttr(iterators));
 
   // Handle the mask.
@@ -534,8 +534,8 @@ struct ReorderElementwiseOpsOnTranspose final
         // This is a constant. Create a reverse transpose op for it.
         auto vectorType =
             srcType.clone(cast<VectorType>(operand.getType()).getElementType());
-        srcValues.push_back(vector::TransposeOp::create(rewriter,
-            operand.getLoc(), vectorType, operand, invOrder));
+        srcValues.push_back(vector::TransposeOp::create(
+            rewriter, operand.getLoc(), vectorType, operand, invOrder));
       }
     }
 
@@ -608,13 +608,13 @@ struct BubbleDownVectorBitCastForExtract
     // Get the single scalar (as a vector) in the source value that packs the
     // desired scalar. E.g. extract vector<1xf32> from vector<4xf32>
     Location loc = extractOp.getLoc();
-    Value packedValue = vector::ExtractOp::create(rewriter,
-        loc, castOp.getSource(), index / expandRatio);
+    Value packedValue = vector::ExtractOp::create(
+        rewriter, loc, castOp.getSource(), index / expandRatio);
     Type packedVecType = VectorType::get(/*shape=*/{1}, packedValue.getType());
-    Value zero = arith::ConstantOp::create(rewriter,
-        loc, packedVecType, rewriter.getZeroAttr(packedVecType));
+    Value zero = arith::ConstantOp::create(rewriter, loc, packedVecType,
+                                           rewriter.getZeroAttr(packedVecType));
     packedValue = vector::InsertOp::create(rewriter, loc, packedValue, zero,
-                                                    /*position=*/0);
+                                           /*position=*/0);
 
     // Cast it to a vector with the desired scalar's type.
     // E.g. f32 -> vector<2xf16>
@@ -700,9 +700,9 @@ struct BubbleDownBitCastForStridedSliceExtract
     VectorType newExtractType =
         VectorType::get(dims, castSrcType.getElementType());
 
-    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
-        extractOp.getLoc(), newExtractType, castOp.getSource(), newOffsets,
-        newSizes, extractOp.getStrides());
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(
+        rewriter, extractOp.getLoc(), newExtractType, castOp.getSource(),
+        newOffsets, newSizes, extractOp.getStrides());
 
     rewriter.replaceOpWithNewOp<vector::BitCastOp>(
         extractOp, extractOp.getType(), newExtractOp);
@@ -761,8 +761,9 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         isNumElemsShrink ? srcDims.back() / ratio : srcDims.back() * ratio;
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
-    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
-        bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
+    auto newCastSrcOp =
+        vector::BitCastOp::create(rewriter, bitcastOp.getLoc(), newCastSrcType,
+                                  insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims(insertOp.getDestVectorType().getShape());
     dstDims.back() =
@@ -771,8 +772,8 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(dstDims, castDstType.getElementType());
 
     // Bitcast the destination.
-    auto newCastDstOp = vector::BitCastOp::create(rewriter,
-        bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
+    auto newCastDstOp = vector::BitCastOp::create(
+        rewriter, bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     // Generate new insert.
     rewriter.replaceOpWithNewOp<vector::InsertOp>(
@@ -852,8 +853,9 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
 
-    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
-        bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
+    auto newCastSrcOp =
+        vector::BitCastOp::create(rewriter, bitcastOp.getLoc(), newCastSrcType,
+                                  insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims =
         llvm::to_vector<4>(insertOp.getDestVectorType().getShape());
@@ -861,8 +863,8 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastDstType =
         VectorType::get(dstDims, castDstType.getElementType());
 
-    auto newCastDstOp = vector::BitCastOp::create(rewriter,
-        bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
+    auto newCastDstOp = vector::BitCastOp::create(
+        rewriter, bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     rewriter.replaceOpWithNewOp<vector::InsertStridedSliceOp>(
         bitcastOp, bitcastOp.getType(), newCastSrcOp, newCastDstOp, newOffsets,
@@ -936,8 +938,8 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
     Type elemType = castDstType.getElementType();
     assert(elemType.isSignlessIntOrIndexOrFloat());
 
-    Value zero = arith::ConstantOp::create(rewriter,
-        loc, elemType, rewriter.getZeroAttr(elemType));
+    Value zero = arith::ConstantOp::create(rewriter, loc, elemType,
+                                           rewriter.getZeroAttr(elemType));
     Value res = SplatOp::create(rewriter, loc, castDstType, zero);
 
     SmallVector<int64_t> sliceShape = {castDstLastDim};
@@ -947,13 +949,13 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
                         castDstType.getElementType());
 
     for (int i = 0, e = shrinkRatio; i < e; ++i) {
-      Value extracted = ExtractStridedSliceOp::create(rewriter,
-          loc, bitcastOp.getSource(), ArrayRef<int64_t>{i * castDstLastDim},
-          sliceShape, strides);
+      Value extracted = ExtractStridedSliceOp::create(
+          rewriter, loc, bitcastOp.getSource(),
+          ArrayRef<int64_t>{i * castDstLastDim}, sliceShape, strides);
       Value bitcast =
           BitCastOp::create(rewriter, loc, newCastDstType, extracted);
-      res = InsertStridedSliceOp::create(rewriter,
-          loc, bitcast, res,
+      res = InsertStridedSliceOp::create(
+          rewriter, loc, bitcast, res,
           ArrayRef<int64_t>{i * castDstLastDim / shrinkRatio}, strides);
     }
     rewriter.replaceOp(bitcastOp, res);
@@ -1303,8 +1305,8 @@ static Value buildVectorComparison(PatternRewriter &rewriter, Operation *op,
   Value bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, b);
   Value bounds =
       vector::SplatOp::create(rewriter, loc, indices.getType(), bound);
-  return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, indices,
-                                        bounds);
+  return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                               indices, bounds);
 }
 
 template <typename ConcreteOp>
@@ -1336,8 +1338,8 @@ struct MaterializeTransferMask : public OpRewritePattern<ConcreteOp> {
     Value dim =
         vector::createOrFoldDimOp(rewriter, loc, xferOp.getBase(), lastIndex);
     Value b = arith::SubIOp::create(rewriter, loc, dim.getType(), dim, off);
-    Value mask = vector::CreateMaskOp::create(rewriter,
-        loc,
+    Value mask = vector::CreateMaskOp::create(
+        rewriter, loc,
         VectorType::get(vtp.getShape(), rewriter.getI1Type(),
                         vtp.getScalableDims()),
         b);
@@ -1548,12 +1550,13 @@ class DropInnerMostUnitDimsTransferRead
         strides);
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         readOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
-    Value rankedReducedView = memref::SubViewOp::create(rewriter,
-        loc, resultMemrefType, readOp.getBase(), offsets, sizes, strides);
+    Value rankedReducedView =
+        memref::SubViewOp::create(rewriter, loc, resultMemrefType,
+                                  readOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
-    Value result = vector::TransferReadOp::create(rewriter,
-        loc, resultTargetVecType, rankedReducedView,
+    Value result = vector::TransferReadOp::create(
+        rewriter, loc, resultTargetVecType, rankedReducedView,
         readOp.getIndices().drop_back(dimsToDrop), AffineMapAttr::get(permMap),
         readOp.getPadding(),
         // TODO: support mask.
@@ -1639,8 +1642,9 @@ class DropInnerMostUnitDimsTransferWrite
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         writeOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
 
-    Value rankedReducedView = memref::SubViewOp::create(rewriter,
-        loc, resultMemrefType, writeOp.getBase(), offsets, sizes, strides);
+    Value rankedReducedView =
+        memref::SubViewOp::create(rewriter, loc, resultMemrefType,
+                                  writeOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
 
@@ -1837,7 +1841,7 @@ struct ChainedReduction final : OpRewritePattern<vector::ReductionOp> {
           loc, parentReduction.getVector(), op.getVector());
     } else {
       vAdd = arith::AddFOp::create(rewriter, loc, parentReduction.getVector(),
-                                            op.getVector());
+                                   op.getVector());
     }
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), vAdd,
                                                      parentReduction.getAcc());
@@ -2004,8 +2008,8 @@ struct DropUnitDimsFromTransposeOp final
 
     Location loc = op.getLoc();
     // Drop the unit dims via shape_cast.
-    auto dropDimsShapeCast = vector::ShapeCastOp::create(rewriter,
-        loc, sourceTypeWithoutUnitDims, op.getVector());
+    auto dropDimsShapeCast = vector::ShapeCastOp::create(
+        rewriter, loc, sourceTypeWithoutUnitDims, op.getVector());
     // Create the new transpose.
     auto transposeWithoutUnitDims =
         vector::TransposeOp::create(rewriter, loc, dropDimsShapeCast, newPerm);
@@ -2111,8 +2115,8 @@ struct ReduceRedundantZero final : OpRewritePattern<vector::ReductionOp> {
     if (!matchPattern(addLhs.getRhs(), m_AnyZeroFloat()))
       return failure();
 
-    auto newAdd = arith::AddFOp::create(rewriter, vAdd.getLoc(), addLhs.getLhs(),
-                                                 vAdd.getRhs());
+    auto newAdd = arith::AddFOp::create(rewriter, vAdd.getLoc(),
+                                        addLhs.getLhs(), vAdd.getRhs());
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), newAdd,
                                                      op.getAcc());
     return success();
@@ -2154,8 +2158,8 @@ struct BreakDownVectorReduction final : OpRewritePattern<vector::ReductionOp> {
     Location loc = op.getLoc();
     SmallVector<Value> extracted(numElems, nullptr);
     for (auto [idx, extractedElem] : llvm::enumerate(extracted))
-      extractedElem = vector::ExtractOp::create(rewriter,
-          loc, op.getVector(), static_cast<int64_t>(idx));
+      extractedElem = vector::ExtractOp::create(rewriter, loc, op.getVector(),
+                                                static_cast<int64_t>(idx));
 
     Value res = extracted.front();
     for (auto extractedElem : llvm::drop_begin(extracted))
@@ -2234,8 +2238,8 @@ struct FoldArithToVectorOuterProduct : public OpRewritePattern<MulOpType> {
       if (!broadcastedRhs || !isValidBroadcastSource(broadcastedRhs))
         return failure();
 
-      return vector::OuterProductOp::create(rewriter,
-          mulOp->getLoc(), resType, broadcastedLhs.getSource(),
+      return vector::OuterProductOp::create(
+          rewriter, mulOp->getLoc(), resType, broadcastedLhs.getSource(),
           broadcastedRhs.getSource(), Value(), vector::CombiningKind::ADD);
     };
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 9fa01061c1bed..fceba65fa3e3a 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -68,8 +68,8 @@ static SmallVector<Value> sliceLoadStoreIndices(PatternRewriter &rewriter,
   auto start = indices.size() - offsets.size();
   for (auto [i, offset] : llvm::enumerate(offsets)) {
     if (offset != 0) {
-      indices[start + i] = arith::AddIOp::create(rewriter,
-          loc, originalIndices[start + i],
+      indices[start + i] = arith::AddIOp::create(
+          rewriter, loc, originalIndices[start + i],
           arith::ConstantIndexOp::create(rewriter, loc, offset));
     }
   }
@@ -172,8 +172,9 @@ struct UnrollTransferReadPattern
     ArrayRef<int64_t> originalSize = readOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
+    Value result =
+        arith::ConstantOp::create(rewriter, loc, sourceVectorType,
+                                  rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
     SmallVector<Value> originalIndices(readOp.getIndices().begin(),
@@ -185,8 +186,8 @@ struct UnrollTransferReadPattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                readOp.getPermutationMap(), loc, rewriter);
-      auto slicedRead = vector::TransferReadOp::create(rewriter,
-          loc, targetType, readOp.getBase(), indices,
+      auto slicedRead = vector::TransferReadOp::create(
+          rewriter, loc, targetType, readOp.getBase(), indices,
           readOp.getPermutationMapAttr(), readOp.getPadding(), readOp.getMask(),
           readOp.getInBoundsAttr());
 
@@ -236,9 +237,10 @@ struct UnrollTransferWritePattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                writeOp.getPermutationMap(), loc, rewriter);
-      Operation *slicedWrite = vector::TransferWriteOp::create(rewriter,
-          loc, slicedVector, resultTensor ? resultTensor : writeOp.getBase(),
-          indices, writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
+      Operation *slicedWrite = vector::TransferWriteOp::create(
+          rewriter, loc, slicedVector,
+          resultTensor ? resultTensor : writeOp.getBase(), indices,
+          writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
       // For the tensor case update the destination for the next transfer write.
       if (!slicedWrite->getResults().empty())
         resultTensor = slicedWrite->getResult(0);
@@ -348,8 +350,8 @@ struct UnrollContractionPattern
       accCache[dstOffets] = newOp->getResult(0);
     }
     // Assemble back the accumulator into a single vector.
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, dstVecType, rewriter.getZeroAttr(dstVecType));
+    Value result = arith::ConstantOp::create(rewriter, loc, dstVecType,
+                                             rewriter.getZeroAttr(dstVecType));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
@@ -427,8 +429,8 @@ struct UnrollMultiReductionPattern
       accCache[destOffset] = result;
     }
     // Assemble back the accumulator into a single vector.
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, reductionOp.getDestType(),
+    Value result = arith::ConstantOp::create(
+        rewriter, loc, reductionOp.getDestType(),
         rewriter.getZeroAttr(reductionOp.getDestType()));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
@@ -468,8 +470,8 @@ struct UnrollElementwisePattern : public RewritePattern {
           op, "expected input vector rank to match target shape rank");
     Location loc = op->getLoc();
     // Prepare the result vector.
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, dstVecType, rewriter.getZeroAttr(dstVecType));
+    Value result = arith::ConstantOp::create(rewriter, loc, dstVecType,
+                                             rewriter.getZeroAttr(dstVecType));
     SmallVector<int64_t> strides(targetShape->size(), 1);
     VectorType newVecType =
         VectorType::get(*targetShape, dstVecType.getElementType());
@@ -567,8 +569,9 @@ struct UnrollTransposePattern : public OpRewritePattern<vector::TransposeOp> {
     ArrayRef<int64_t> originalSize = originalVectorType.getShape();
 
     // Prepare the result vector;
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, originalVectorType, rewriter.getZeroAttr(originalVectorType));
+    Value result =
+        arith::ConstantOp::create(rewriter, loc, originalVectorType,
+                                  rewriter.getZeroAttr(originalVectorType));
     ArrayRef<int64_t> permutation = transposeOp.getPermutation();
 
     // Unroll the computation.
@@ -618,8 +621,9 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
     ArrayRef<int64_t> originalSize = gatherOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
+    Value result =
+        arith::ConstantOp::create(rewriter, loc, sourceVectorType,
+                                  rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
 
@@ -638,8 +642,8 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
           rewriter.createOrFold<vector::ExtractStridedSliceOp>(
               loc, gatherOp.getPassThru(), elementOffsets, *targetShape,
               strides);
-      auto slicedGather = vector::GatherOp::create(rewriter,
-          loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
+      auto slicedGather = vector::GatherOp::create(
+          rewriter, loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
           indexSubVec, maskSubVec, passThruSubVec);
 
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
@@ -671,8 +675,8 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
     ArrayRef<int64_t> originalShape = vecType.getShape();
     SmallVector<int64_t> strides(targetShape->size(), 1);
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, vecType, rewriter.getZeroAttr(vecType));
+    Value result = arith::ConstantOp::create(rewriter, loc, vecType,
+                                             rewriter.getZeroAttr(vecType));
 
     SmallVector<int64_t> loopOrder =
         getUnrollOrder(originalShape.size(), loadOp, options);
@@ -684,8 +688,8 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
          StaticTileOffsetRange(originalShape, *targetShape, loopOrder)) {
       SmallVector<Value> indices =
           sliceLoadStoreIndices(rewriter, loc, loadOp.getIndices(), offsets);
-      Value slicedLoad = vector::LoadOp::create(rewriter,
-          loc, targetVecType, loadOp.getBase(), indices);
+      Value slicedLoad = vector::LoadOp::create(rewriter, loc, targetVecType,
+                                                loadOp.getBase(), indices);
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, slicedLoad, result, offsets, strides);
     }
@@ -755,8 +759,8 @@ struct UnrollBroadcastPattern : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType = broadcastOp.getResultVectorType();
     VectorType targetType =
         resType.cloneWith(*targetShape, resType.getElementType());
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resType, rewriter.getZeroAttr(resType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                             rewriter.getZeroAttr(resType));
 
     SmallVector<int64_t> originalShape = *broadcastOp.getShapeForUnroll();
     SmallVector<int64_t> strides(originalShape.size(), 1);
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 7c31ada1644c5..ac542ddd7d1c4 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -341,8 +341,8 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
       inBoundsVal[i] = (sourceShape[i] == inputVectorSizes[i]) &&
                        ShapedType::isStatic(sourceShape[i]);
   }
-  auto transferReadOp = vector::TransferReadOp::create(builder,
-      loc,
+  auto transferReadOp = vector::TransferReadOp::create(
+      builder, loc,
       /*vectorType=*/vectorType,
       /*source=*/source,
       /*indices=*/SmallVector<Value>(readRank, zero),
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
index 8f87661a68d96..e11595a09cc45 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
+++ b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
@@ -38,8 +38,8 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
       "=x,x,x"; // Careful: constraint parser is very brittle: no ws!
   SmallVector<Value> asmVals{v1, v2};
   auto asmStr = llvm::formatv(asmTp, llvm::format_hex(mask, /*width=*/2)).str();
-  auto asmOp = LLVM::InlineAsmOp::create(b,
-      v1.getType(), /*operands=*/asmVals, /*asm_string=*/asmStr,
+  auto asmOp = LLVM::InlineAsmOp::create(
+      b, v1.getType(), /*operands=*/asmVals, /*asm_string=*/asmStr,
       /*constraints=*/asmCstr, /*has_side_effects=*/false,
       /*is_align_stack=*/false, LLVM::TailCallKind::None,
       /*asm_dialect=*/asmDialectAttr,
@@ -49,14 +49,14 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackLoPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return vector::ShuffleOp::create(b,
-      v1, v2, ArrayRef<int64_t>{0, 8, 1, 9, 4, 12, 5, 13});
+  return vector::ShuffleOp::create(b, v1, v2,
+                                   ArrayRef<int64_t>{0, 8, 1, 9, 4, 12, 5, 13});
 }
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackHiPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return vector::ShuffleOp::create(b,
-      v1, v2, ArrayRef<int64_t>{2, 10, 3, 11, 6, 14, 7, 15});
+  return vector::ShuffleOp::create(
+      b, v1, v2, ArrayRef<int64_t>{2, 10, 3, 11, 6, 14, 7, 15});
 }
 ///                            a  a   b   b  a  a   b   b
 /// Takes an 8 bit mask, 2 bit for each position of a[0, 3)  **and** b[0, 4):
@@ -262,7 +262,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
       // Insert transposed 1-D vectors into the higher-order dimension of the
       // output vector.
       Value res = arith::ConstantOp::create(ib, reshInputType,
-                                               ib.getZeroAttr(reshInputType));
+                                            ib.getZeroAttr(reshInputType));
       for (int64_t i = 0; i < m; ++i)
         res = vector::InsertOp::create(ib, vs[i], res, i);
 
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index 613a33fe63d87..efcafa7d33d2b 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -127,7 +127,7 @@ static Value resolveDistributedTy(Value orig, T expected,
   // op to reconcile the types.
   if (isa<xegpu::TensorDescType>(orig.getType())) {
     auto castOp = UnrealizedConversionCastOp::create(rewriter, orig.getLoc(),
-                                                              expected, orig);
+                                                     expected, orig);
     castOp->setAttr(resolveSIMTTypeMismatch, rewriter.getUnitAttr());
     return castOp.getResult(0);
   }
@@ -198,17 +198,18 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         }))
       return failure();
     // Create a new function with the same signature.
-    auto newGpuFunc = gpu::GPUFuncOp::create(rewriter,
-        gpuFuncOp.getLoc(), gpuFuncOp.getName(), gpuFuncOp.getFunctionType());
+    auto newGpuFunc = gpu::GPUFuncOp::create(rewriter, gpuFuncOp.getLoc(),
+                                             gpuFuncOp.getName(),
+                                             gpuFuncOp.getFunctionType());
     // Create a WarpExecuteOnLane0Op with same arguments and results as the
     // original gpuFuncOp.
     rewriter.setInsertionPointToEnd(&newGpuFunc.getFunctionBody().front());
-    auto laneId = gpu::LaneIdOp::create(rewriter,
-        newGpuFunc.getLoc(), rewriter.getIndexType(),
+    auto laneId = gpu::LaneIdOp::create(
+        rewriter, newGpuFunc.getLoc(), rewriter.getIndexType(),
         /** upperBound = **/ mlir::IntegerAttr());
     ArrayRef<Type> gpuFuncResultType = gpuFuncOp.getFunctionType().getResults();
-    auto warpOp = gpu::WarpExecuteOnLane0Op::create(rewriter,
-        laneId.getLoc(), gpuFuncResultType, laneId,
+    auto warpOp = gpu::WarpExecuteOnLane0Op::create(
+        rewriter, laneId.getLoc(), gpuFuncResultType, laneId,
         xegpu::targetinfo::subgroupSize, newGpuFunc.getArguments(),
         newGpuFunc.getArgumentTypes());
     Block &warpBodyBlock = warpOp.getBodyRegion().front();
@@ -217,7 +218,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         cast<gpu::ReturnOp>(gpuFuncOp.getBlocks().back().getTerminator());
     rewriter.setInsertionPointAfter(origRetunOp);
     gpu::YieldOp::create(rewriter, origRetunOp.getLoc(),
-                                  origRetunOp.getOperands());
+                         origRetunOp.getOperands());
     rewriter.eraseOp(origRetunOp);
     // Move the original function body to the WarpExecuteOnLane0Op body.
     rewriter.inlineRegionBefore(gpuFuncOp.getBody(), warpOp.getBodyRegion(),
@@ -301,8 +302,8 @@ struct CreateNdDescDistribution final : public gpu::WarpDistributionPattern {
     xegpu::TensorDescType distributedTensorDescTy =
         descOp.getType().dropLayouts(); // Distributed tensor descriptor type
                                         // does not contain layout info.
-    Value newDescOp = xegpu::CreateNdDescOp::create(rewriter,
-        newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
+    Value newDescOp = xegpu::CreateNdDescOp::create(
+        rewriter, newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
         descOp->getAttrs());
 
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -403,8 +404,8 @@ struct StoreNdDistribution final : public gpu::WarpDistributionPattern {
         resolveDistributedTy(newWarpOp.getResult(newRetIndices[1]),
                              distributedTensorDescTy, rewriter));
 
-    xegpu::StoreNdOp::create(rewriter,
-        newWarpOp.getLoc(), TypeRange{}, newStoreOperands,
+    xegpu::StoreNdOp::create(
+        rewriter, newWarpOp.getLoc(), TypeRange{}, newStoreOperands,
         removeTemporaryLayoutAttributes(storeOp->getAttrs()));
     rewriter.eraseOp(storeOp);
     return success();
@@ -494,8 +495,8 @@ struct LoadNdDistribution final : public gpu::WarpDistributionPattern {
         loadOp.getTensorDescType().dropLayouts(); // Distributed tensor
                                                   // descriptor type does not
                                                   // contain layout info.
-    auto newLoadOp = xegpu::LoadNdOp::create(rewriter,
-        newWarpOp.getLoc(), loadNdDistValueTyOrFailure.value(),
+    auto newLoadOp = xegpu::LoadNdOp::create(
+        rewriter, newWarpOp.getLoc(), loadNdDistValueTyOrFailure.value(),
         resolveDistributedTy(newWarpOp->getResult(newRetIndices[0]),
                              distributedTensorDescTy, rewriter),
         removeTemporaryLayoutAttributes(loadOp->getAttrs()));
@@ -630,8 +631,8 @@ struct DpasDistribution final : public gpu::WarpDistributionPattern {
           resolveDistributedTy(newWarpOp.getResult(newRetIndices[i]),
                                newDpasOperandExpectedTypes[i], rewriter));
     }
-    Value newDpasOp = xegpu::DpasOp::create(rewriter,
-        newWarpOp->getLoc(), distributedResultTy, newDpasOperands,
+    Value newDpasOp = xegpu::DpasOp::create(
+        rewriter, newWarpOp->getLoc(), distributedResultTy, newDpasOperands,
         removeTemporaryLayoutAttributes(dpasOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
     // Resolve the output type.
@@ -717,8 +718,8 @@ struct UpdateNdOffsetDistribution final : public gpu::WarpDistributionPattern {
       }
     }
     // Create a new update op outside the warp op.
-    Value newUpdateOp = xegpu::UpdateNdOffsetOp::create(rewriter,
-        newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
+    Value newUpdateOp = xegpu::UpdateNdOffsetOp::create(
+        rewriter, newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
         removeTemporaryLayoutAttributes(updateOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
     // Resolve the distributed type with the original type.
@@ -783,8 +784,8 @@ struct PrefetchNdDistribution final : public gpu::WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     SmallVector<Value> newPrefetchOperands = {resolveDistributedTy(
         newWarpOp.getResult(newRetIndices[0]), newTensorDescTy, rewriter)};
-    xegpu::PrefetchNdOp::create(rewriter,
-        newWarpOp.getLoc(), TypeRange{}, newPrefetchOperands,
+    xegpu::PrefetchNdOp::create(
+        rewriter, newWarpOp.getLoc(), TypeRange{}, newPrefetchOperands,
         removeTemporaryLayoutAttributes(prefetchOp->getAttrs()));
     rewriter.eraseOp(prefetchOp);
     return success();
@@ -806,9 +807,9 @@ struct GpuBarrierDistribution final : public gpu::WarpDistributionPattern {
       return failure();
     // Move the barrier op outside of the warp op.
     rewriter.setInsertionPointAfter(subgroupOp);
-    gpu::BarrierOp::create(rewriter,
-        barrierOp.getLoc(), barrierOp->getResultTypes(),
-        barrierOp->getOperands(), barrierOp->getAttrs());
+    gpu::BarrierOp::create(rewriter, barrierOp.getLoc(),
+                           barrierOp->getResultTypes(),
+                           barrierOp->getOperands(), barrierOp->getAttrs());
     rewriter.eraseOp(barrierOp);
     return success();
   }
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
index 6266cca918d4f..83aeb9dc189ef 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
@@ -83,8 +83,9 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = UnrealizedConversionCastOp::create(rewriter,
-          loc, destTy, srcs, ArrayRef<NamedAttribute>({attr, blkAttr}));
+      auto castOp = UnrealizedConversionCastOp::create(
+          rewriter, loc, destTy, srcs,
+          ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResult(0);
     }
 
@@ -109,8 +110,9 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = UnrealizedConversionCastOp::create(rewriter,
-          loc, destTypes, src, ArrayRef<NamedAttribute>({attr, blkAttr}));
+      auto castOp = UnrealizedConversionCastOp::create(
+          rewriter, loc, destTypes, src,
+          ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResults();
     }
 
@@ -169,9 +171,9 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
            llvm::zip(validIdxes, oldOffsets, offsets))
         mixedOffsets[idx] = addi(oldOff, offset);
 
-      auto newOp = xegpu::CreateNdDescOp::create(rewriter,
-          loc, newTdescTy, op.getSource(), mixedOffsets, op.getMixedSizes(),
-          op.getMixedStrides());
+      auto newOp = xegpu::CreateNdDescOp::create(
+          rewriter, loc, newTdescTy, op.getSource(), mixedOffsets,
+          op.getMixedSizes(), op.getMixedStrides());
       newOps.push_back(newOp);
     }
     Value castOp = unpack(newOps, tdescTy, *targetShape, loc, rewriter);
@@ -199,8 +201,8 @@ struct UnrollUpdateNdOffsetOp : public UnrollPattern<xegpu::UpdateNdOffsetOp> {
 
     SmallVector<Value> newOps;
     for (auto t : convertedTdesc) {
-      auto newOp = xegpu::UpdateNdOffsetOp::create(rewriter,
-          loc, t.getType(), t, op.getOffsets(), op.getConstOffsets());
+      auto newOp = xegpu::UpdateNdOffsetOp::create(
+          rewriter, loc, t.getType(), t, op.getOffsets(), op.getConstOffsets());
       newOps.push_back(newOp);
     }
     Value castOp = unpack(newOps, op.getType(), *targetShape, loc, rewriter);
@@ -226,7 +228,8 @@ struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      xegpu::PrefetchNdOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchNdOp::create(rewriter, loc, TypeRange(), t,
+                                  op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -292,7 +295,7 @@ struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> {
 
     for (auto [v, t] : llvm::zip(convertedValues, convertedTdescs))
       xegpu::StoreNdOp::create(rewriter, loc, v, t, op.getL1HintAttr(),
-                                        op.getL2HintAttr(), op.getL3HintAttr());
+                               op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
     return success();
@@ -385,7 +388,7 @@ struct UnrollDpasOp : public UnrollPattern<xegpu::DpasOp> {
             operands.push_back(tmpC);
 
           tmpC = xegpu::DpasOp::create(rewriter, loc, vecTy, operands,
-                                                op->getAttrs());
+                                       op->getAttrs());
         }
         newOps.push_back(tmpC);
       }
@@ -436,22 +439,23 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
            llvm::zip(convertedIndiceVec, convertedIndiceTypes)) {
         for (int64_t i = 0; i < numNewChunks; ++i) {
           // Compute the offset
-          Value inc = arith::ConstantIndexOp::create(rewriter,
-              loc, i * blockedChunkSize);
-          Value incVec = vector::SplatOp::create(rewriter, loc, indiceType, inc);
+          Value inc = arith::ConstantIndexOp::create(rewriter, loc,
+                                                     i * blockedChunkSize);
+          Value incVec =
+              vector::SplatOp::create(rewriter, loc, indiceType, inc);
           Value offsetIndice =
               arith::AddIOp::create(rewriter, loc, indice, incVec);
 
-          auto newOp = xegpu::CreateDescOp::create(rewriter,
-              loc, newTdescTy, op.getSource(), offsetIndice);
+          auto newOp = xegpu::CreateDescOp::create(
+              rewriter, loc, newTdescTy, op.getSource(), offsetIndice);
 
           newOps.push_back(newOp);
         }
       }
     } else {
       for (auto indice : convertedIndiceVec) {
-        auto newOp = xegpu::CreateDescOp::create(rewriter,
-            loc, newTdescTy, op.getSource(), indice);
+        auto newOp = xegpu::CreateDescOp::create(rewriter, loc, newTdescTy,
+                                                 op.getSource(), indice);
         newOps.push_back(newOp);
       }
     }
@@ -515,9 +519,9 @@ struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
 
     SmallVector<Value> newOps;
     for (auto [t, m] : llvm::zip(convertedTdescs, convertedMasks)) {
-      auto newOp = xegpu::LoadGatherOp::create(rewriter,
-          loc, newValueTy, t, m, op.getL1HintAttr(), op.getL2HintAttr(),
-          op.getL3HintAttr());
+      auto newOp = xegpu::LoadGatherOp::create(
+          rewriter, loc, newValueTy, t, m, op.getL1HintAttr(),
+          op.getL2HintAttr(), op.getL3HintAttr());
       newOps.push_back(newOp);
     }
 
@@ -609,8 +613,7 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
       Value t = convertedTdescs[i];
       Value m = op.getMask() ? convertedMasks[i] : nullptr;
       xegpu::StoreScatterOp::create(rewriter, loc, v, t, m, op.getL1HintAttr(),
-                                             op.getL2HintAttr(),
-                                             op.getL3HintAttr());
+                                    op.getL2HintAttr(), op.getL3HintAttr());
     }
 
     rewriter.eraseOp(op);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index aa2fc78c77f39..80bb5e888bdc7 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -201,9 +201,9 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
           calculateGlobalOffsets(rewriter, loc, originalOffsets, localOffset,
                                  distUnitBaseAddr, distUnitShape);
 
-      auto newCreateNdOp = xegpu::CreateNdDescOp::create(rewriter,
-          loc, newTdescTy, op.getSource(), globalOffsets, op.getMixedSizes(),
-          op.getMixedStrides());
+      auto newCreateNdOp = xegpu::CreateNdDescOp::create(
+          rewriter, loc, newTdescTy, op.getSource(), globalOffsets,
+          op.getMixedSizes(), op.getMixedStrides());
       newCreateNdOps.push_back(newCreateNdOp);
     }
 
@@ -225,7 +225,7 @@ struct WgToSgLoadNdOp : public OpConversionPattern<xegpu::LoadNdOp> {
       ArrayRef<int64_t> srcShape = tdescTy.getShape();
       VectorType newResTy = VectorType::get(srcShape, tdescTy.getElementType());
       auto newLoadOp = xegpu::LoadNdOp::create(rewriter, op.getLoc(), newResTy,
-                                                        src, op->getAttrs());
+                                               src, op->getAttrs());
       newLoadOps.push_back(newLoadOp);
     }
     rewriter.replaceOpWithMultiple(op, {newLoadOps});
@@ -243,7 +243,7 @@ struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
                   ConversionPatternRewriter &rewriter) const override {
     for (auto [v, t] : llvm::zip(adaptor.getValue(), adaptor.getTensorDesc()))
       xegpu::StoreNdOp::create(rewriter, op.getLoc(), v, t, op.getL1HintAttr(),
-                                        op.getL2HintAttr(), op.getL3HintAttr());
+                               op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
     return success();
@@ -261,8 +261,8 @@ struct WgToSgUpdateNdOffsetOp
                   ConversionPatternRewriter &rewriter) const override {
     llvm::SmallVector<Value> newUpdateTileOffsetOps;
     for (auto tDesc : adaptor.getTensorDesc()) {
-      auto newUpdateTileOffsetOp = xegpu::UpdateNdOffsetOp::create(rewriter,
-          op.getLoc(), tDesc.getType(), tDesc, op.getOffsets(),
+      auto newUpdateTileOffsetOp = xegpu::UpdateNdOffsetOp::create(
+          rewriter, op.getLoc(), tDesc.getType(), tDesc, op.getOffsets(),
           op.getConstOffsets());
       newUpdateTileOffsetOps.push_back(newUpdateTileOffsetOp);
     }
@@ -325,7 +325,7 @@ struct WgToSgPrefetchNdOp : public OpConversionPattern<xegpu::PrefetchNdOp> {
                   ConversionPatternRewriter &rewriter) const override {
     for (auto src : adaptor.getTensorDesc())
       xegpu::PrefetchNdOp::create(rewriter, op.getLoc(), TypeRange(), src,
-                                           op->getAttrs());
+                                  op->getAttrs());
     rewriter.eraseOp(op);
     return success();
   }
diff --git a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index 9acb209d031bd..ca7c2d63b8726 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -198,8 +198,8 @@ xegpu::extractVectorsWithShapeFromValue(OpBuilder &builder, Location loc,
   SmallVector<Value> result;
   for (SmallVector<int64_t> offsets : StaticTileOffsetRange(srcShape, shape)) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result.push_back(vector::ExtractStridedSliceOp::create(builder,
-        loc, value, offsets, shape, staticStrides));
+    result.push_back(vector::ExtractStridedSliceOp::create(
+        builder, loc, value, offsets, shape, staticStrides));
   }
 
   return result;
@@ -218,14 +218,14 @@ Value xegpu::createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
 
   VectorType resultTy = VectorType::get(shape, elemTy);
   auto zeroAttr = builder.getZeroAttr(elemTy);
-  Value result = arith::ConstantOp::create(builder,
-      loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
+  Value result = arith::ConstantOp::create(
+      builder, loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
 
   for (auto [src, offsets] :
        llvm::zip_equal(values, StaticTileOffsetRange(shape, tileShape))) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result = vector::InsertStridedSliceOp::create(builder,
-        loc, src, result, offsets, staticStrides);
+    result = vector::InsertStridedSliceOp::create(builder, loc, src, result,
+                                                  offsets, staticStrides);
   }
   return result;
 }
@@ -343,8 +343,8 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
         if (isa<RankedTensorType>(inputTy) && isa<VectorType>(outputTy)) {
           SmallVector<Value> values = xegpu::flattenValues(adaptor.getInputs());
-          auto newOp = UnrealizedConversionCastOp::create(rewriter,
-              op.getLoc(), outputTy, values);
+          auto newOp = UnrealizedConversionCastOp::create(rewriter, op.getLoc(),
+                                                          outputTy, values);
           rewriter.replaceOp(op, newOp);
           return success();
         }
diff --git a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
index c0f19f6dce600..0fb1adbbf75aa 100644
--- a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
+++ b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
@@ -37,8 +37,8 @@ LogicalResult mlir::LLVMImportInterface::convertUnregisteredIntrinsic(
     return failure();
 
   Type resultType = moduleImport.convertType(inst->getType());
-  auto op = ::mlir::LLVM::CallIntrinsicOp::create(builder,
-      moduleImport.translateLoc(inst->getDebugLoc()),
+  auto op = ::mlir::LLVM::CallIntrinsicOp::create(
+      builder, moduleImport.translateLoc(inst->getDebugLoc()),
       isa<LLVMVoidType>(resultType) ? TypeRange{} : TypeRange{resultType},
       StringAttr::get(builder.getContext(), intrinName),
       ValueRange{mlirOperands}, FastmathFlagsAttr{});
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 6e47db6d1dc6a..51901191fe863 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -865,7 +865,7 @@ LogicalResult ModuleImport::convertModuleFlagsMetadata() {
 
   if (!moduleFlags.empty())
     LLVM::ModuleFlagsOp::create(builder, mlirModule.getLoc(),
-                                        builder.getArrayAttr(moduleFlags));
+                                builder.getArrayAttr(moduleFlags));
 
   return success();
 }
@@ -881,7 +881,7 @@ LogicalResult ModuleImport::convertLinkerOptionsMetadata() {
       for (const llvm::MDOperand &option : node->operands())
         options.push_back(cast<llvm::MDString>(option)->getString());
       LLVM::LinkerOptionsOp::create(builder, mlirModule.getLoc(),
-                                            builder.getStrArrayAttr(options));
+                                    builder.getStrArrayAttr(options));
     }
   }
   return success();
@@ -984,8 +984,8 @@ void ModuleImport::processComdat(const llvm::Comdat *comdat) {
   ComdatOp comdatOp = getGlobalComdatOp();
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(&comdatOp.getBody().back());
-  auto selectorOp = ComdatSelectorOp::create(builder,
-      mlirModule.getLoc(), comdat->getName(),
+  auto selectorOp = ComdatSelectorOp::create(
+      builder, mlirModule.getLoc(), comdat->getName(),
       convertComdatFromLLVM(comdat->getSelectionKind()));
   auto symbolRef =
       SymbolRefAttr::get(builder.getContext(), getGlobalComdatOpName(),
@@ -1346,12 +1346,12 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   Type type = convertType(alias->getValueType());
-  AliasOp aliasOp = AliasOp::create(builder,
-      mlirModule.getLoc(), type, convertLinkageFromLLVM(alias->getLinkage()),
-      alias->getName(),
-      /*dso_local=*/alias->isDSOLocal(),
-      /*thread_local=*/alias->isThreadLocal(),
-      /*attrs=*/ArrayRef<NamedAttribute>());
+  AliasOp aliasOp = AliasOp::create(builder, mlirModule.getLoc(), type,
+                                    convertLinkageFromLLVM(alias->getLinkage()),
+                                    alias->getName(),
+                                    /*dso_local=*/alias->isDSOLocal(),
+                                    /*thread_local=*/alias->isThreadLocal(),
+                                    /*attrs=*/ArrayRef<NamedAttribute>());
   globalInsertionOp = aliasOp;
 
   clearRegionState();
@@ -1403,8 +1403,8 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
   if (globalName.empty())
     globalName = getOrCreateNamelessSymbolName(globalVar).getValue();
 
-  GlobalOp globalOp = GlobalOp::create(builder,
-      mlirModule.getLoc(), type, globalVar->isConstant(),
+  GlobalOp globalOp = GlobalOp::create(
+      builder, mlirModule.getLoc(), type, globalVar->isConstant(),
       convertLinkageFromLLVM(globalVar->getLinkage()), StringRef(globalName),
       valueAttr, alignment, /*addr_space=*/globalVar->getAddressSpace(),
       /*dso_local=*/globalVar->isDSOLocal(),
@@ -1488,13 +1488,13 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   if (globalVar->getName() == getGlobalCtorsVarName()) {
-    globalInsertionOp = LLVM::GlobalCtorsOp::create(builder,
-        mlirModule.getLoc(), builder.getArrayAttr(funcs),
+    globalInsertionOp = LLVM::GlobalCtorsOp::create(
+        builder, mlirModule.getLoc(), builder.getArrayAttr(funcs),
         builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
     return success();
   }
-  globalInsertionOp = LLVM::GlobalDtorsOp::create(builder,
-      mlirModule.getLoc(), builder.getArrayAttr(funcs),
+  globalInsertionOp = LLVM::GlobalDtorsOp::create(
+      builder, mlirModule.getLoc(), builder.getArrayAttr(funcs),
       builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
   return success();
 }
@@ -1683,13 +1683,14 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     Value root = UndefOp::create(builder, loc, rootType);
     for (const auto &it : llvm::enumerate(elementValues)) {
       if (isArrayOrStruct) {
-        root = InsertValueOp::create(builder, loc, root, it.value(), it.index());
+        root =
+            InsertValueOp::create(builder, loc, root, it.value(), it.index());
       } else {
         Attribute indexAttr = builder.getI32IntegerAttr(it.index());
         Value indexValue =
             ConstantOp::create(builder, loc, builder.getI32Type(), indexAttr);
         root = InsertElementOp::create(builder, loc, rootType, root, it.value(),
-                                               indexValue);
+                                       indexValue);
       }
     }
     return root;
@@ -2125,15 +2126,15 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
 
     if (!brInst->isConditional()) {
       auto brOp = LLVM::BrOp::create(builder, loc, succBlockArgs.front(),
-                                             succBlocks.front());
+                                     succBlocks.front());
       mapNoResultOp(inst, brOp);
       return success();
     }
     FailureOr<Value> condition = convertValue(brInst->getCondition());
     if (failed(condition))
       return failure();
-    auto condBrOp = LLVM::CondBrOp::create(builder,
-        loc, *condition, succBlocks.front(), succBlockArgs.front(),
+    auto condBrOp = LLVM::CondBrOp::create(
+        builder, loc, *condition, succBlocks.front(), succBlockArgs.front(),
         succBlocks.back(), succBlockArgs.back());
     mapNoResultOp(inst, condBrOp);
     return success();
@@ -2166,9 +2167,9 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       caseBlocks[it.index()] = lookupBlock(succBB);
     }
 
-    auto switchOp = SwitchOp::create(builder,
-        loc, *condition, lookupBlock(defaultBB), defaultBlockArgs, caseValues,
-        caseBlocks, caseOperandRefs);
+    auto switchOp = SwitchOp::create(builder, loc, *condition,
+                                     lookupBlock(defaultBB), defaultBlockArgs,
+                                     caseValues, caseBlocks, caseOperandRefs);
     mapNoResultOp(inst, switchOp);
     return success();
   }
@@ -2218,8 +2219,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
         // IR). Build the indirect call by passing an empty `callee` operand and
         // insert into `operands` to include the indirect call target.
         FlatSymbolRefAttr calleeSym = convertCalleeName(callInst);
-        Value indirectCallVal = LLVM::AddressOfOp::create(builder,
-            loc, LLVM::LLVMPointerType::get(context), calleeSym);
+        Value indirectCallVal = LLVM::AddressOfOp::create(
+            builder, loc, LLVM::LLVMPointerType::get(context), calleeSym);
         operands->insert(operands->begin(), indirectCallVal);
       } else {
         // Regular direct call using callee name.
@@ -2310,8 +2311,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // IR). Build the indirect invoke by passing an empty `callee` operand and
       // insert into `operands` to include the indirect invoke target.
       FlatSymbolRefAttr calleeSym = convertCalleeName(invokeInst);
-      Value indirectInvokeVal = LLVM::AddressOfOp::create(builder,
-          loc, LLVM::LLVMPointerType::get(context), calleeSym);
+      Value indirectInvokeVal = LLVM::AddressOfOp::create(
+          builder, loc, LLVM::LLVMPointerType::get(context), calleeSym);
       operands->insert(operands->begin(), indirectInvokeVal);
     } else {
       // Regular direct invoke using callee name.
@@ -2320,9 +2321,9 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     // Create the invoke operation. Normal destination block arguments will be
     // added later on to handle the case in which the operation result is
     // included in this list.
-    auto invokeOp = InvokeOp::create(builder,
-        loc, *funcTy, calleeName, *operands, directNormalDest, ValueRange(),
-        lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
+    auto invokeOp = InvokeOp::create(
+        builder, loc, *funcTy, calleeName, *operands, directNormalDest,
+        ValueRange(), lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
 
     if (failed(convertInvokeAttributes(invokeInst, invokeOp)))
       return failure();
@@ -2382,8 +2383,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     Type type = convertType(inst->getType());
-    auto gepOp = GEPOp::create(builder,
-        loc, type, sourceElementType, *basePtr, indices,
+    auto gepOp = GEPOp::create(
+        builder, loc, type, sourceElementType, *basePtr, indices,
         static_cast<GEPNoWrapFlags>(gepInst->getNoWrapFlags().getRaw()));
     mapValue(inst, gepOp);
     return success();
@@ -2409,8 +2410,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     SmallVector<ValueRange> succBlockArgsRange =
         llvm::to_vector_of<ValueRange>(succBlockArgs);
     Location loc = translateLoc(inst->getDebugLoc());
-    auto indBrOp = LLVM::IndirectBrOp::create(builder,
-        loc, *basePtr, succBlockArgsRange, succBlocks);
+    auto indBrOp = LLVM::IndirectBrOp::create(builder, loc, *basePtr,
+                                              succBlockArgsRange, succBlocks);
 
     mapNoResultOp(inst, indBrOp);
     return success();
@@ -2854,8 +2855,8 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) {
   builder.setInsertionPointToEnd(mlirModule.getBody());
 
   Location loc = debugImporter->translateFuncLocation(func);
-  LLVMFuncOp funcOp = LLVMFuncOp::create(builder,
-      loc, func->getName(), functionType,
+  LLVMFuncOp funcOp = LLVMFuncOp::create(
+      builder, loc, func->getName(), functionType,
       convertLinkageFromLLVM(func->getLinkage()), dsoLocal, cconv);
 
   convertParameterAttributes(func, funcOp, builder);
@@ -3032,12 +3033,12 @@ ModuleImport::processDebugIntrinsic(llvm::DbgVariableIntrinsic *dbgIntr,
   Operation *op =
       llvm::TypeSwitch<llvm::DbgVariableIntrinsic *, Operation *>(dbgIntr)
           .Case([&](llvm::DbgDeclareInst *) {
-            return LLVM::DbgDeclareOp::create(builder,
-                loc, *argOperand, localVariableAttr, locationExprAttr);
+            return LLVM::DbgDeclareOp::create(
+                builder, loc, *argOperand, localVariableAttr, locationExprAttr);
           })
           .Case([&](llvm::DbgValueInst *) {
-            return LLVM::DbgValueOp::create(builder,
-                loc, *argOperand, localVariableAttr, locationExprAttr);
+            return LLVM::DbgValueOp::create(
+                builder, loc, *argOperand, localVariableAttr, locationExprAttr);
           });
   mapNoResultOp(dbgIntr, op);
   setNonDebugMetadataAttrs(dbgIntr, op);
@@ -3083,7 +3084,7 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb,
     OpBuilder::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(block);
     BlockTagOp::create(builder, block->getParentOp()->getLoc(),
-                               BlockTagAttr::get(context, bb->getNumber()));
+                       BlockTagAttr::get(context, bb->getNumber()));
   }
   return success();
 }
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index 6e14b94e87407..85b0e04754acf 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -43,22 +43,23 @@ Value spirv::Deserializer::getValue(uint32_t id) {
   if (auto constInfo = getConstant(id)) {
     // Materialize a `spirv.Constant` op at every use site.
     return spirv::ConstantOp::create(opBuilder, unknownLoc, constInfo->second,
-                                               constInfo->first);
+                                     constInfo->first);
   }
   if (auto varOp = getGlobalVariable(id)) {
-    auto addressOfOp = spirv::AddressOfOp::create(opBuilder,
-        unknownLoc, varOp.getType(), SymbolRefAttr::get(varOp.getOperation()));
+    auto addressOfOp =
+        spirv::AddressOfOp::create(opBuilder, unknownLoc, varOp.getType(),
+                                   SymbolRefAttr::get(varOp.getOperation()));
     return addressOfOp.getPointer();
   }
   if (auto constOp = getSpecConstant(id)) {
-    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
-        unknownLoc, constOp.getDefaultValue().getType(),
+    auto referenceOfOp = spirv::ReferenceOfOp::create(
+        opBuilder, unknownLoc, constOp.getDefaultValue().getType(),
         SymbolRefAttr::get(constOp.getOperation()));
     return referenceOfOp.getReference();
   }
   if (auto constCompositeOp = getSpecConstantComposite(id)) {
-    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
-        unknownLoc, constCompositeOp.getType(),
+    auto referenceOfOp = spirv::ReferenceOfOp::create(
+        opBuilder, unknownLoc, constCompositeOp.getType(),
         SymbolRefAttr::get(constCompositeOp.getOperation()));
     return referenceOfOp.getReference();
   }
@@ -369,8 +370,9 @@ Deserializer::processOp<spirv::EntryPointOp>(ArrayRef<uint32_t> words) {
     interface.push_back(SymbolRefAttr::get(arg.getOperation()));
     wordIndex++;
   }
-  spirv::EntryPointOp::create(opBuilder,
-      unknownLoc, execModel, SymbolRefAttr::get(opBuilder.getContext(), fnName),
+  spirv::EntryPointOp::create(
+      opBuilder, unknownLoc, execModel,
+      SymbolRefAttr::get(opBuilder.getContext(), fnName),
       opBuilder.getArrayAttr(interface));
   return success();
 }
@@ -402,9 +404,10 @@ Deserializer::processOp<spirv::ExecutionModeOp>(ArrayRef<uint32_t> words) {
     attrListElems.push_back(opBuilder.getI32IntegerAttr(words[wordIndex++]));
   }
   auto values = opBuilder.getArrayAttr(attrListElems);
-  spirv::ExecutionModeOp::create(opBuilder,
-      unknownLoc, SymbolRefAttr::get(opBuilder.getContext(), fn.getName()),
-      execMode, values);
+  spirv::ExecutionModeOp::create(
+      opBuilder, unknownLoc,
+      SymbolRefAttr::get(opBuilder.getContext(), fn.getName()), execMode,
+      values);
   return success();
 }
 
@@ -441,8 +444,8 @@ Deserializer::processOp<spirv::FunctionCallOp>(ArrayRef<uint32_t> operands) {
     arguments.push_back(value);
   }
 
-  auto opFunctionCall = spirv::FunctionCallOp::create(opBuilder,
-      unknownLoc, resultType,
+  auto opFunctionCall = spirv::FunctionCallOp::create(
+      opBuilder, unknownLoc, resultType,
       SymbolRefAttr::get(opBuilder.getContext(), functionName), arguments);
 
   if (resultType)
@@ -518,7 +521,8 @@ Deserializer::processOp<spirv::CopyMemoryOp>(ArrayRef<uint32_t> words) {
   }
 
   Location loc = createFileLineColLoc(opBuilder);
-  spirv::CopyMemoryOp::create(opBuilder, loc, resultTypes, operands, attributes);
+  spirv::CopyMemoryOp::create(opBuilder, loc, resultTypes, operands,
+                              attributes);
 
   return success();
 }
@@ -549,8 +553,8 @@ LogicalResult Deserializer::processOp<spirv::GenericCastToPtrExplicitOp>(
   operands.push_back(arg);
 
   Location loc = createFileLineColLoc(opBuilder);
-  Operation *op = spirv::GenericCastToPtrExplicitOp::create(opBuilder,
-      loc, resultTypes, operands);
+  Operation *op = spirv::GenericCastToPtrExplicitOp::create(
+      opBuilder, loc, resultTypes, operands);
   valueMap[valueID] = op->getResult(0);
   return success();
 }
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index 7b1f8055fadce..2eaed616773db 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -518,8 +518,8 @@ spirv::Deserializer::processFunction(ArrayRef<uint32_t> operands) {
   }
 
   std::string fnName = getFunctionSymbol(fnID);
-  auto funcOp = spirv::FuncOp::create(opBuilder,
-      unknownLoc, fnName, functionType, fnControl.value());
+  auto funcOp = spirv::FuncOp::create(opBuilder, unknownLoc, fnName,
+                                      functionType, fnControl.value());
   // Processing other function attributes.
   if (decorations.count(fnID)) {
     for (auto attr : decorations[fnID].getAttrs()) {
@@ -707,7 +707,7 @@ spirv::Deserializer::createSpecConstant(Location loc, uint32_t resultID,
                                         TypedAttr defaultValue) {
   auto symName = opBuilder.getStringAttr(getSpecConstantSymbol(resultID));
   auto op = spirv::SpecConstantOp::create(opBuilder, unknownLoc, symName,
-                                                    defaultValue);
+                                          defaultValue);
   if (decorations.count(resultID)) {
     for (auto attr : decorations[resultID].getAttrs())
       op->setAttr(attr.getName(), attr.getValue());
@@ -782,9 +782,9 @@ spirv::Deserializer::processGlobalVariable(ArrayRef<uint32_t> operands) {
            << wordIndex << " of " << operands.size() << " processed";
   }
   auto loc = createFileLineColLoc(opBuilder);
-  auto varOp = spirv::GlobalVariableOp::create(opBuilder,
-      loc, TypeAttr::get(type), opBuilder.getStringAttr(variableName),
-      initializer);
+  auto varOp = spirv::GlobalVariableOp::create(
+      opBuilder, loc, TypeAttr::get(type),
+      opBuilder.getStringAttr(variableName), initializer);
 
   // Decorations.
   if (decorations.count(variableID)) {
@@ -1581,8 +1581,8 @@ spirv::Deserializer::processSpecConstantComposite(ArrayRef<uint32_t> operands) {
     elements.push_back(SymbolRefAttr::get(elementInfo));
   }
 
-  auto op = spirv::SpecConstantCompositeOp::create(opBuilder,
-      unknownLoc, TypeAttr::get(resultType), symName,
+  auto op = spirv::SpecConstantCompositeOp::create(
+      opBuilder, unknownLoc, TypeAttr::get(resultType), symName,
       opBuilder.getArrayAttr(elements));
   specConstCompositeMap[resultID] = op;
 
@@ -1764,8 +1764,8 @@ spirv::Deserializer::processBranchConditional(ArrayRef<uint32_t> operands) {
   // an OpSelectionMerge instruction, in this case they will have the same
   // OpLine information.
   auto loc = createFileLineColLoc(opBuilder);
-  spirv::BranchConditionalOp::create(opBuilder,
-      loc, condition, trueBlock,
+  spirv::BranchConditionalOp::create(
+      opBuilder, loc, condition, trueBlock,
       /*trueArguments=*/ArrayRef<Value>(), falseBlock,
       /*falseArguments=*/ArrayRef<Value>(), weights);
 
@@ -2093,7 +2093,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
     // loop header block.
     builder.setInsertionPointToEnd(&body.front());
     spirv::BranchOp::create(builder, location, mapper.lookupOrNull(headerBlock),
-                                    ArrayRef<Value>(blockArgs));
+                            ArrayRef<Value>(blockArgs));
   }
 
   // Values defined inside the selection region that need to be yielded outside
@@ -2177,12 +2177,12 @@ LogicalResult ControlFlowStructurizer::structurize() {
     Operation *newOp = nullptr;
 
     if (isLoop)
-      newOp = spirv::LoopOp::create(builder,
-          location, TypeRange(ValueRange(outsideUses)),
-          static_cast<spirv::LoopControl>(control));
+      newOp = spirv::LoopOp::create(builder, location,
+                                    TypeRange(ValueRange(outsideUses)),
+                                    static_cast<spirv::LoopControl>(control));
     else
-      newOp = spirv::SelectionOp::create(builder,
-          location, TypeRange(ValueRange(outsideUses)),
+      newOp = spirv::SelectionOp::create(
+          builder, location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::SelectionControl>(control));
 
     newOp->getRegion(0).takeBody(body);
@@ -2362,22 +2362,22 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
 
     if (auto branchOp = dyn_cast<spirv::BranchOp>(op)) {
       // Replace the previous branch op with a new one with block arguments.
-      spirv::BranchOp::create(opBuilder, branchOp.getLoc(), branchOp.getTarget(),
-                                        blockArgs);
+      spirv::BranchOp::create(opBuilder, branchOp.getLoc(),
+                              branchOp.getTarget(), blockArgs);
       branchOp.erase();
     } else if (auto branchCondOp = dyn_cast<spirv::BranchConditionalOp>(op)) {
       assert((branchCondOp.getTrueBlock() == target ||
               branchCondOp.getFalseBlock() == target) &&
              "expected target to be either the true or false target");
       if (target == branchCondOp.getTrueTarget())
-        spirv::BranchConditionalOp::create(opBuilder,
-            branchCondOp.getLoc(), branchCondOp.getCondition(), blockArgs,
-            branchCondOp.getFalseBlockArguments(),
+        spirv::BranchConditionalOp::create(
+            opBuilder, branchCondOp.getLoc(), branchCondOp.getCondition(),
+            blockArgs, branchCondOp.getFalseBlockArguments(),
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueTarget(),
             branchCondOp.getFalseTarget());
       else
-        spirv::BranchConditionalOp::create(opBuilder,
-            branchCondOp.getLoc(), branchCondOp.getCondition(),
+        spirv::BranchConditionalOp::create(
+            opBuilder, branchCondOp.getLoc(), branchCondOp.getCondition(),
             branchCondOp.getTrueBlockArguments(), blockArgs,
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueBlock(),
             branchCondOp.getFalseBlock());
diff --git a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
index 3f4e7762fed0e..0677828b635d4 100644
--- a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
+++ b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
@@ -206,7 +206,7 @@ static void checkAndNestUnderRewriteOp(OpBuilder &builder, Value rootExpr,
   if (isa<pdl::PatternOp>(builder.getInsertionBlock()->getParentOp())) {
     pdl::RewriteOp rewrite =
         pdl::RewriteOp::create(builder, loc, rootExpr, /*name=*/StringAttr(),
-                                       /*externalArgs=*/ValueRange());
+                               /*externalArgs=*/ValueRange());
     builder.createBlock(&rewrite.getBodyRegion());
   }
 }
@@ -242,8 +242,8 @@ void CodeGen::genImpl(const ast::ReplaceStmt *stmt) {
   bool usesReplOperation =
       replValues.size() == 1 &&
       isa<pdl::OperationType>(replValues.front().getType());
-  pdl::ReplaceOp::create(builder,
-      loc, rootExpr, usesReplOperation ? replValues[0] : Value(),
+  pdl::ReplaceOp::create(
+      builder, loc, rootExpr, usesReplOperation ? replValues[0] : Value(),
       usesReplOperation ? ValueRange() : ValueRange(replValues));
 }
 
@@ -283,8 +283,8 @@ void CodeGen::genImpl(const ast::PatternDecl *decl) {
 
   // FIXME: Properly model HasBoundedRecursion in PDL so that we don't drop it
   // here.
-  pdl::PatternOp pattern = pdl::PatternOp::create(builder,
-      genLoc(decl->getLoc()), decl->getBenefit(),
+  pdl::PatternOp pattern = pdl::PatternOp::create(
+      builder, genLoc(decl->getLoc()), decl->getBenefit(),
       name ? std::optional<StringRef>(name->getName())
            : std::optional<StringRef>());
 
@@ -344,24 +344,25 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   if (isa<ast::AttributeType>(type))
     return pdl::AttributeOp::create(builder, loc, getTypeConstraint());
   if (ast::OperationType opType = dyn_cast<ast::OperationType>(type)) {
-    Value operands = pdl::OperandsOp::create(builder,
-        loc, pdl::RangeType::get(builder.getType<pdl::ValueType>()),
+    Value operands = pdl::OperandsOp::create(
+        builder, loc, pdl::RangeType::get(builder.getType<pdl::ValueType>()),
         /*type=*/Value());
-    Value results = pdl::TypesOp::create(builder,
-        loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
+    Value results = pdl::TypesOp::create(
+        builder, loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
         /*types=*/ArrayAttr());
     return pdl::OperationOp::create(builder, loc, opType.getName(), operands,
-                                            ArrayRef<StringRef>(), ValueRange(),
-                                            results);
+                                    ArrayRef<StringRef>(), ValueRange(),
+                                    results);
   }
 
   if (ast::RangeType rangeTy = dyn_cast<ast::RangeType>(type)) {
     ast::Type eleTy = rangeTy.getElementType();
     if (isa<ast::ValueType>(eleTy))
       return pdl::OperandsOp::create(builder, loc, mlirType,
-                                             getTypeConstraint());
+                                     getTypeConstraint());
     if (isa<ast::TypeType>(eleTy))
-      return pdl::TypesOp::create(builder, loc, mlirType, /*types=*/ArrayAttr());
+      return pdl::TypesOp::create(builder, loc, mlirType,
+                                  /*types=*/ArrayAttr());
   }
 
   llvm_unreachable("invalid non-initialized variable type");
@@ -444,7 +445,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
       Type mlirType = genType(expr->getType());
       if (isa<pdl::ValueType>(mlirType))
         return pdl::ResultOp::create(builder, loc, mlirType, parentExprs[0],
-                                             builder.getI32IntegerAttr(0));
+                                     builder.getI32IntegerAttr(0));
       return pdl::ResultsOp::create(builder, loc, mlirType, parentExprs[0]);
     }
 
@@ -456,7 +457,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
       name.getAsInteger(/*Radix=*/10, resultIndex);
       IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
       return pdl::ResultOp::create(builder, loc, genType(expr->getType()),
-                                           parentExprs[0], index);
+                                   parentExprs[0], index);
     }
 
     // Find the result with the member name or by index.
@@ -475,7 +476,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
     // Generate the result access.
     IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
     return pdl::ResultsOp::create(builder, loc, genType(expr->getType()),
-                                          parentExprs[0], index);
+                                  parentExprs[0], index);
   }
 
   // Handle tuple based member access.
@@ -519,7 +520,7 @@ Value CodeGen::genExprImpl(const ast::OperationExpr *expr) {
     results.push_back(genSingleExpr(result));
 
   return pdl::OperationOp::create(builder, loc, opName, operands, attrNames,
-                                          attrValues, results);
+                                  attrValues, results);
 }
 
 Value CodeGen::genExprImpl(const ast::RangeExpr *expr) {
@@ -528,7 +529,7 @@ Value CodeGen::genExprImpl(const ast::RangeExpr *expr) {
     llvm::append_range(elements, genExpr(element));
 
   return pdl::RangeOp::create(builder, genLoc(expr->getLoc()),
-                                      genType(expr->getType()), elements);
+                              genType(expr->getType()), elements);
 }
 
 SmallVector<Value> CodeGen::genExprImpl(const ast::TupleExpr *expr) {
@@ -542,8 +543,8 @@ Value CodeGen::genExprImpl(const ast::TypeExpr *expr) {
   Type type = parseType(expr->getValue(), builder.getContext());
   assert(type && "invalid MLIR type data");
   return pdl::TypeOp::create(builder, genLoc(expr->getLoc()),
-                                     builder.getType<pdl::TypeType>(),
-                                     TypeAttr::get(type));
+                             builder.getType<pdl::TypeType>(),
+                             TypeAttr::get(type));
 }
 
 SmallVector<Value>
@@ -587,7 +588,7 @@ CodeGen::genConstraintOrRewriteCall(const T *decl, Location loc,
       resultTypes.push_back(genType(declResultType));
     }
     PDLOpT pdlOp = PDLOpT::create(builder, loc, resultTypes,
-                                          decl->getName().getName(), inputs);
+                                  decl->getName().getName(), inputs);
     if (isNegated && std::is_same_v<PDLOpT, pdl::ApplyNativeConstraintOp>)
       cast<pdl::ApplyNativeConstraintOp>(pdlOp).setIsNegated(true);
     return pdlOp->getResults();
diff --git a/mlir/test/python/lib/PythonTestDialect.cpp b/mlir/test/python/lib/PythonTestDialect.cpp
index e0a76f1b75d38..0c29902024ca6 100644
--- a/mlir/test/python/lib/PythonTestDialect.cpp
+++ b/mlir/test/python/lib/PythonTestDialect.cpp
@@ -8,8 +8,8 @@
 
 #include "PythonTestDialect.h"
 #include "mlir/IR/DialectImplementation.h"
-#include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
+#include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "PythonTestDialect.cpp.inc"


From flang-commits at lists.llvm.org  Mon Jul 14 09:56:16 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 09:56:16 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Lower] fix warning (PR #148667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148667@github.com>
Message-ID: <687536b0.630a0220.29aaa.9fc4@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Tom Eccles (tblah)

<details>
<summary>Changes</summary>

GCC 9.3.0
```
.../flang/lib/Lower/Support/ReductionProcessor.cpp:137:1: error: control reaches end of non-void function [-Werror=return-type]
```

---
Full diff: https://github.com/llvm/llvm-project/pull/148667.diff


1 Files Affected:

- (modified) flang/lib/Lower/Support/ReductionProcessor.cpp (+1) 


``````````diff
diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 77cb15a2a77ca..c0be1e229f825 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -134,6 +134,7 @@ ReductionProcessor::getReductionType(const fir::ReduceOperationEnum &redOp) {
   case fir::ReduceOperationEnum::MIN:
     return ReductionIdentifier::MIN;
   }
+  llvm_unreachable("Unhandled ReductionIdentifier case");
 }
 
 bool ReductionProcessor::supportedIntrinsicProcReduction(

``````````

</details>


https://github.com/llvm/llvm-project/pull/148667

From flang-commits at lists.llvm.org  Mon Jul 14 09:59:56 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 14 Jul 2025 09:59:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Lower] fix warning (PR #148667)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148667@github.com>
Message-ID: <6875378c.050a0220.106949.c4b2@mx.google.com>


https://github.com/tblah closed https://github.com/llvm/llvm-project/pull/148667

From flang-commits at lists.llvm.org  Mon Jul 14 10:01:58 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 14 Jul 2025 10:01:58 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [mlir][core] NFC update builder
	create API usage (PR #147311)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147311@github.com>
Message-ID: <68753806.a70a0220.2dbd94.a4ec@mx.google.com>


https://github.com/makslevental updated https://github.com/llvm/llvm-project/pull/147311

>From 2f00b126ca57868f4fe83bf5694d586f891aa891 Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 10:35:48 -0400
Subject: [PATCH 1/5] [mlir][core] update builder create API

---
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           | 176 ++---
 .../AffineToStandard/AffineToStandard.cpp     |  26 +-
 .../ArithToAMDGPU/ArithToAMDGPU.cpp           |  90 +--
 .../ArithToArmSME/ArithToArmSME.cpp           |   6 +-
 .../Conversion/ArithToEmitC/ArithToEmitC.cpp  |  44 +-
 .../Conversion/ArithToLLVM/ArithToLLVM.cpp    |  28 +-
 .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp  |  74 +-
 .../ArmNeon2dToIntr/ArmNeon2dToIntr.cpp       |   6 +-
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  | 122 +--
 .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp    |  68 +-
 .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp    |  66 +-
 .../BufferizationToMemRef.cpp                 |  26 +-
 .../ComplexCommon/DivisionConverter.cpp       | 392 ++++-----
 .../ComplexToLLVM/ComplexToLLVM.cpp           |  28 +-
 .../ComplexToLibm/ComplexToLibm.cpp           |   2 +-
 .../ComplexToStandard/ComplexToStandard.cpp   | 598 +++++++-------
 .../ControlFlowToLLVM/ControlFlowToLLVM.cpp   |   8 +-
 .../ControlFlowToSCF/ControlFlowToSCF.cpp     |  22 +-
 .../Conversion/FuncToEmitC/FuncToEmitC.cpp    |   2 +-
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp |  48 +-
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |  82 +-
 .../GPUCommon/GPUToLLVMConversion.cpp         |  78 +-
 .../GPUCommon/IndexIntrinsicsOpLowering.h     |  10 +-
 .../GPUCommon/OpToFuncCallLowering.h          |  12 +-
 .../Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp  |  20 +-
 .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp        |  40 +-
 .../Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp    |  40 +-
 .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp      |  50 +-
 mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp |  54 +-
 .../Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp  |   4 +-
 .../Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp  |   4 +-
 .../Conversion/IndexToLLVM/IndexToLLVM.cpp    |  68 +-
 .../Conversion/IndexToSPIRV/IndexToSPIRV.cpp  |  68 +-
 .../Conversion/LLVMCommon/MemRefBuilder.cpp   |  78 +-
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    |  62 +-
 .../Conversion/LLVMCommon/PrintCallHelper.cpp |   8 +-
 .../Conversion/LLVMCommon/StructBuilder.cpp   |   4 +-
 .../Conversion/LLVMCommon/TypeConverter.cpp   |  14 +-
 .../Conversion/LLVMCommon/VectorPattern.cpp   |   6 +-
 .../LinalgToStandard/LinalgToStandard.cpp     |   4 +-
 mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp   |  64 +-
 .../Conversion/MathToFuncs/MathToFuncs.cpp    | 180 ++---
 mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp |  38 +-
 mlir/lib/Conversion/MathToLibm/MathToLibm.cpp |  14 +-
 .../Conversion/MathToSPIRV/MathToSPIRV.cpp    |  82 +-
 .../MemRefToEmitC/MemRefToEmitC.cpp           |   6 +-
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  | 138 ++--
 .../MemRefToSPIRV/MemRefToSPIRV.cpp           |  18 +-
 mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp   | 204 ++---
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    | 158 ++--
 .../Conversion/OpenACCToSCF/OpenACCToSCF.cpp  |   2 +-
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |   2 +-
 .../PDLToPDLInterp/PDLToPDLInterp.cpp         | 112 +--
 .../SCFToControlFlow/SCFToControlFlow.cpp     |  36 +-
 mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp |  14 +-
 mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp     |  40 +-
 .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp    |  40 +-
 mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp |  40 +-
 .../ConvertLaunchFuncToLLVMCalls.cpp          |   8 +-
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    | 124 +--
 .../ConvertShapeConstraints.cpp               |   2 +-
 .../ShapeToStandard/ShapeToStandard.cpp       | 150 ++--
 .../TensorToSPIRV/TensorToSPIRV.cpp           |   6 +-
 .../Conversion/TosaToArith/TosaToArith.cpp    | 100 +--
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  | 742 +++++++++---------
 .../TosaToLinalg/TosaToLinalgNamed.cpp        | 218 ++---
 .../TosaToMLProgram/TosaToMLProgram.cpp       |   6 +-
 mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp   |  24 +-
 .../Conversion/TosaToTensor/TosaToTensor.cpp  |  20 +-
 .../VectorToArmSME/VectorToArmSME.cpp         |  56 +-
 .../Conversion/VectorToGPU/VectorToGPU.cpp    |  76 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      | 158 ++--
 .../Conversion/VectorToSCF/VectorToSCF.cpp    | 166 ++--
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  48 +-
 .../VectorToXeGPU/VectorToXeGPU.cpp           |  20 +-
 .../AMDGPU/Transforms/EmulateAtomics.cpp      |  20 +-
 .../AMDGPU/Transforms/MaskedloadToLoad.cpp    |  28 +-
 .../Transforms/ResolveStridedMetadata.cpp     |   6 +-
 mlir/lib/Dialect/AMX/IR/AMXDialect.cpp        |   8 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  34 +-
 .../Transforms/AffineDataCopyGeneration.cpp   |   2 +-
 .../Transforms/AffineExpandIndexOps.cpp       |  20 +-
 .../Affine/Transforms/DecomposeAffineOps.cpp  |   4 +-
 .../Dialect/Affine/Transforms/LoopFusion.cpp  |   2 +-
 .../Transforms/PipelineDataTransfer.cpp       |   6 +-
 .../Affine/Transforms/ReifyValueBounds.cpp    |   6 +-
 .../Affine/Transforms/SuperVectorize.cpp      |  24 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |  86 +-
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |  88 +--
 mlir/lib/Dialect/Arith/IR/ArithDialect.cpp    |   2 +-
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |  40 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |   2 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/EmulateUnsupportedFloats.cpp   |   4 +-
 .../Arith/Transforms/EmulateWideInt.cpp       | 176 ++---
 .../Dialect/Arith/Transforms/ExpandOps.cpp    | 228 +++---
 .../Transforms/IntRangeOptimizations.cpp      |  10 +-
 .../Arith/Transforms/ReifyValueBounds.cpp     |  16 +-
 .../Transforms/ShardingInterfaceImpl.cpp      |   2 +-
 mlir/lib/Dialect/Arith/Utils/Utils.cpp        |  90 +--
 .../LowerContractionToNeonI8MMPattern.cpp     |   8 +-
 mlir/lib/Dialect/ArmSME/IR/Utils.cpp          |  14 +-
 .../ArmSME/Transforms/OuterProductFusion.cpp  |   8 +-
 .../ArmSME/Transforms/TileAllocation.cpp      |   4 +-
 .../ArmSME/Transforms/VectorLegalization.cpp  | 110 +--
 .../Transforms/LegalizeForLLVMExport.cpp      |  16 +-
 .../Transforms/LegalizeVectorStorage.cpp      |  12 +-
 .../LowerContractionToSVEI8MMPattern.cpp      |  60 +-
 mlir/lib/Dialect/Async/IR/Async.cpp           |   2 +-
 .../Async/Transforms/AsyncParallelFor.cpp     | 194 ++---
 .../Transforms/AsyncRuntimeRefCounting.cpp    |  14 +-
 .../Async/Transforms/AsyncToAsyncRuntime.cpp  |  78 +-
 .../IR/BufferDeallocationOpInterface.cpp      |  10 +-
 .../IR/BufferizableOpInterface.cpp            |  10 +-
 .../Bufferization/IR/BufferizationOps.cpp     |  30 +-
 .../BufferizationTransformOps.cpp             |   4 +-
 .../BufferDeallocationSimplification.cpp      |  12 +-
 .../Transforms/BufferResultsToOutParams.cpp   |   6 +-
 .../Bufferization/Transforms/BufferUtils.cpp  |   2 +-
 .../Bufferization/Transforms/Bufferize.cpp    |   6 +-
 .../DropEquivalentBufferResults.cpp           |   4 +-
 .../Transforms/EmptyTensorElimination.cpp     |   2 +-
 .../FuncBufferizableOpInterfaceImpl.cpp       |   4 +-
 .../Transforms/LowerDeallocations.cpp         | 142 ++--
 .../OwnershipBasedBufferDeallocation.cpp      |  14 +-
 .../lib/Dialect/Complex/IR/ComplexDialect.cpp |   2 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.cpp |   6 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |  12 +-
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp           |   4 +-
 .../Dialect/EmitC/Transforms/Transforms.cpp   |   4 +-
 .../EmitC/Transforms/TypeConversions.cpp      |   2 +-
 .../EmitC/Transforms/WrapFuncInClass.cpp      |   8 +-
 .../Func/Extensions/InlinerExtension.cpp      |   2 +-
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |   2 +-
 .../Func/TransformOps/FuncTransformOps.cpp    |   2 +-
 .../Func/Transforms/FuncConversions.cpp       |   2 +-
 mlir/lib/Dialect/Func/Utils/Utils.cpp         |   6 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |  10 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |  10 +-
 mlir/lib/Dialect/GPU/TransformOps/Utils.cpp   |  24 +-
 .../GPU/Transforms/AllReduceLowering.cpp      |   2 +-
 .../GPU/Transforms/AsyncRegionRewriter.cpp    |   8 +-
 .../GPU/Transforms/DecomposeMemRefs.cpp       |   4 +-
 .../GPU/Transforms/GlobalIdRewriter.cpp       |   8 +-
 .../GPU/Transforms/KernelOutlining.cpp        |  10 +-
 .../GPU/Transforms/MemoryPromotion.cpp        |  16 +-
 .../Dialect/GPU/Transforms/ModuleToBinary.cpp |   2 +-
 .../GPU/Transforms/PromoteShuffleToAMDGPU.cpp |   4 +-
 .../GPU/Transforms/ShuffleRewriter.cpp        |  22 +-
 .../GPU/Transforms/SubgroupIdRewriter.cpp     |  22 +-
 .../GPU/Transforms/SubgroupReduceLowering.cpp |  70 +-
 .../Dialect/GPU/Utils/DistributionUtils.cpp   |   4 +-
 mlir/lib/Dialect/Index/IR/IndexOps.cpp        |   8 +-
 .../LLVMIR/IR/BasicPtxBuilderInterface.cpp    |   4 +-
 .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp   |   2 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |  16 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp |  46 +-
 .../Dialect/LLVMIR/Transforms/AddComdats.cpp  |   4 +-
 .../Transforms/InlinerInterfaceImpl.cpp       |  22 +-
 .../LLVMIR/Transforms/LegalizeForExport.cpp   |   2 +-
 .../LLVMIR/Transforms/OptimizeForNVVM.cpp     |  30 +-
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 202 ++---
 .../TransformOps/LinalgTransformOps.cpp       |  14 +-
 .../BufferizableOpInterfaceImpl.cpp           |   2 +-
 .../Transforms/ConvertConv2DToImg2Col.cpp     | 108 +--
 .../Transforms/ConvertToDestinationStyle.cpp  |  52 +-
 .../Transforms/DataLayoutPropagation.cpp      |  30 +-
 ...DecomposeGenericByUnfoldingPermutation.cpp |  10 +-
 .../Linalg/Transforms/DecomposeLinalgOps.cpp  |  14 +-
 .../Dialect/Linalg/Transforms/Detensorize.cpp |   6 +-
 .../Linalg/Transforms/DropUnitDims.cpp        |  22 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  40 +-
 .../Linalg/Transforms/ElementwiseToLinalg.cpp |   4 +-
 .../EraseUnusedOperandsAndResults.cpp         |   2 +-
 .../FusePadOpWithLinalgProducer.cpp           |   8 +-
 mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp |   2 +-
 .../Linalg/Transforms/Generalization.cpp      |   2 +-
 .../Linalg/Transforms/HoistPadding.cpp        |  20 +-
 .../Dialect/Linalg/Transforms/Hoisting.cpp    |   2 +-
 .../Transforms/InlineScalarOperands.cpp       |   6 +-
 .../Dialect/Linalg/Transforms/Interchange.cpp |   2 +-
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |   8 +-
 .../Transforms/MeshShardingInterfaceImpl.cpp  |  18 +-
 .../Linalg/Transforms/NamedOpConversions.cpp  |   8 +-
 .../Transforms/PackAndUnpackPatterns.cpp      |   6 +-
 .../Linalg/Transforms/PadTilingInterface.cpp  |   6 +-
 .../lib/Dialect/Linalg/Transforms/Padding.cpp |   6 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |  44 +-
 .../Transforms/RuntimeOpVerification.cpp      |   4 +-
 mlir/lib/Dialect/Linalg/Transforms/Split.cpp  |   2 +-
 .../Linalg/Transforms/SplitReduction.cpp      |  32 +-
 .../SwapExtractSliceWithFillPatterns.cpp      |   2 +-
 mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp |  16 +-
 .../Linalg/Transforms/TilingInterfaceImpl.cpp |  48 +-
 .../Dialect/Linalg/Transforms/Transforms.cpp  |  66 +-
 .../Linalg/Transforms/TransposeConv2D.cpp     |   6 +-
 .../Linalg/Transforms/TransposeMatmul.cpp     |  26 +-
 .../Linalg/Transforms/Vectorization.cpp       | 176 ++---
 .../Linalg/Transforms/WinogradConv2D.cpp      | 114 +--
 mlir/lib/Dialect/Linalg/Utils/Utils.cpp       |  18 +-
 .../BufferizableOpInterfaceImpl.cpp           |   2 +-
 mlir/lib/Dialect/MPI/IR/MPIOps.cpp            |   2 +-
 mlir/lib/Dialect/Math/IR/MathOps.cpp          |   2 +-
 .../Transforms/AlgebraicSimplification.cpp    |  20 +-
 .../Math/Transforms/ExpandPatterns.cpp        | 226 +++---
 .../Transforms/ExtendToSupportedTypes.cpp     |   4 +-
 .../Transforms/PolynomialApproximation.cpp    | 536 ++++++-------
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    |   6 +-
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |  22 +-
 .../TransformOps/MemRefTransformOps.cpp       |   2 +-
 .../Transforms/AllocationOpInterfaceImpl.cpp  |   8 +-
 .../MemRef/Transforms/ComposeSubView.cpp      |   2 +-
 .../MemRef/Transforms/EmulateNarrowType.cpp   |  37 +-
 .../Dialect/MemRef/Transforms/ExpandOps.cpp   |  14 +-
 .../MemRef/Transforms/ExpandRealloc.cpp       |  20 +-
 .../Transforms/ExpandStridedMetadata.cpp      |  24 +-
 .../Transforms/ExtractAddressComputations.cpp |  16 +-
 .../MemRef/Transforms/FlattenMemRefs.cpp      |  28 +-
 .../MemRef/Transforms/FoldMemRefAliasOps.cpp  |   6 +-
 .../Transforms/IndependenceTransforms.cpp     |   8 +-
 .../Dialect/MemRef/Transforms/MultiBuffer.cpp |   8 +-
 .../MemRef/Transforms/NormalizeMemRefs.cpp    |   2 +-
 .../MemRef/Transforms/ReifyResultShapes.cpp   |   4 +-
 .../ResolveShapedTypeResultDims.cpp           |   2 +-
 .../Transforms/RuntimeOpVerification.cpp      |  88 +--
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |  14 +-
 .../Mesh/Transforms/Simplifications.cpp       |   4 +-
 .../Dialect/Mesh/Transforms/Spmdization.cpp   |  14 +-
 .../Dialect/Mesh/Transforms/Transforms.cpp    |  36 +-
 .../NVGPU/TransformOps/NVGPUTransformOps.cpp  |  62 +-
 .../NVGPU/Transforms/CreateAsyncGroups.cpp    |  16 +-
 .../NVGPU/Transforms/OptimizeSharedMemory.cpp |  12 +-
 .../Quant/Transforms/LowerQuantOps.cpp        |  78 +-
 .../Quant/Transforms/StripFuncQuantTypes.cpp  |   2 +-
 mlir/lib/Dialect/SCF/IR/SCF.cpp               |  78 +-
 .../SCF/TransformOps/SCFTransformOps.cpp      |   4 +-
 .../BufferizableOpInterfaceImpl.cpp           |  18 +-
 .../lib/Dialect/SCF/Transforms/ForToWhile.cpp |   8 +-
 .../SCF/Transforms/ForallToParallel.cpp       |   2 +-
 .../Dialect/SCF/Transforms/LoopPipelining.cpp |  70 +-
 .../SCF/Transforms/LoopSpecialization.cpp     |  14 +-
 .../SCF/Transforms/ParallelLoopFusion.cpp     |   4 +-
 .../SCF/Transforms/ParallelLoopTiling.cpp     |  32 +-
 .../Transforms/StructuralTypeConversions.cpp  |   6 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  32 +-
 .../SCF/Transforms/UpliftWhileToFor.cpp       |  20 +-
 .../SCF/Transforms/WrapInZeroTripCheck.cpp    |  10 +-
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          |  94 +--
 mlir/lib/Dialect/SMT/IR/SMTDialect.cpp        |   4 +-
 mlir/lib/Dialect/SMT/IR/SMTOps.cpp            |   4 +-
 mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp  |  10 +-
 .../SPIRV/IR/SPIRVCanonicalization.cpp        |  20 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp    |   4 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp        |  22 +-
 .../Linking/ModuleCombiner/ModuleCombiner.cpp |   2 +-
 .../Transforms/LowerABIAttributesPass.cpp     |  18 +-
 .../SPIRV/Transforms/RewriteInsertsPass.cpp   |   2 +-
 .../SPIRV/Transforms/SPIRVConversion.cpp      |  62 +-
 .../Transforms/SPIRVWebGPUTransforms.cpp      |  32 +-
 .../Transforms/UnifyAliasedResourcePass.cpp   |  28 +-
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  30 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/OutlineShapeComputation.cpp    |   6 +-
 .../Shape/Transforms/ShapeToShapeLowering.cpp |   6 +-
 .../SparseTensor/IR/SparseTensorDialect.cpp   |   4 +-
 .../IR/SparseTensorInterfaces.cpp             |   2 +-
 .../Transforms/SparseAssembler.cpp            |  16 +-
 .../Transforms/SparseBufferRewriting.cpp      | 366 ++++-----
 .../Transforms/SparseGPUCodegen.cpp           | 178 ++---
 .../Transforms/SparseIterationToScf.cpp       |  26 +-
 .../Transforms/SparseReinterpretMap.cpp       |  20 +-
 .../Transforms/SparseSpaceCollapse.cpp        |   2 +-
 .../SparseStorageSpecifierToLLVM.cpp          |  10 +-
 .../Transforms/SparseTensorCodegen.cpp        | 138 ++--
 .../Transforms/SparseTensorConversion.cpp     |  44 +-
 .../Transforms/SparseTensorRewriting.cpp      | 212 ++---
 .../Transforms/SparseVectorization.cpp        |  50 +-
 .../Transforms/Sparsification.cpp             |  68 +-
 .../Transforms/StageSparseOperations.cpp      |   4 +-
 .../Transforms/Utils/CodegenUtils.cpp         |  74 +-
 .../Transforms/Utils/CodegenUtils.h           |  30 +-
 .../Transforms/Utils/LoopEmitter.cpp          |  54 +-
 .../Utils/SparseTensorDescriptor.cpp          |  10 +-
 .../Transforms/Utils/SparseTensorDescriptor.h |   2 +-
 .../Transforms/Utils/SparseTensorIterator.cpp |  76 +-
 .../lib/Dialect/SparseTensor/Utils/Merger.cpp | 116 +--
 .../Extensions/MeshShardingExtensions.cpp     |   8 +-
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      |  66 +-
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp   |  20 +-
 .../TransformOps/TensorTransformOps.cpp       |   4 +-
 .../BufferizableOpInterfaceImpl.cpp           |  44 +-
 .../Tensor/Transforms/EmptyOpPatterns.cpp     |   2 +-
 .../ExtractSliceFromReshapeUtils.cpp          |   8 +-
 .../Tensor/Transforms/FoldTensorSubsetOps.cpp |   2 +-
 .../Transforms/IndependenceTransforms.cpp     |   8 +-
 .../Tensor/Transforms/ReshapePatterns.cpp     |  10 +-
 .../Tensor/Transforms/RewriteAsConstant.cpp   |   2 +-
 .../Transforms/RuntimeOpVerification.cpp      |  40 +-
 .../SubsetInsertionOpInterfaceImpl.cpp        |   2 +-
 mlir/lib/Dialect/Tensor/Utils/Utils.cpp       |   6 +-
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |   4 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |  12 +-
 .../Transforms/TosaDecomposeDepthwise.cpp     |  16 +-
 .../Transforms/TosaDecomposeTransposeConv.cpp |   8 +-
 .../Tosa/Transforms/TosaInferShapes.cpp       |   2 +-
 .../Tosa/Transforms/TosaReduceTransposes.cpp  |   4 +-
 .../Tosa/Transforms/TosaTypeConverters.cpp    |   4 +-
 .../Dialect/Tosa/Utils/ConversionUtils.cpp    |  16 +-
 mlir/lib/Dialect/UB/IR/UBOps.cpp              |   2 +-
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  60 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Vector/Transforms/LowerVectorBitCast.cpp  |   8 +-
 .../Transforms/LowerVectorBroadcast.cpp       |  22 +-
 .../Vector/Transforms/LowerVectorContract.cpp | 128 +--
 .../Vector/Transforms/LowerVectorGather.cpp   |  36 +-
 .../Transforms/LowerVectorInterleave.cpp      |  20 +-
 .../Vector/Transforms/LowerVectorMask.cpp     |  22 +-
 .../Transforms/LowerVectorMultiReduction.cpp  |  38 +-
 .../Vector/Transforms/LowerVectorScan.cpp     |  16 +-
 .../Transforms/LowerVectorShapeCast.cpp       |  36 +-
 .../Vector/Transforms/LowerVectorTransfer.cpp |  26 +-
 .../Transforms/LowerVectorTranspose.cpp       |  26 +-
 .../Vector/Transforms/VectorDistribute.cpp    |  76 +-
 .../Transforms/VectorDropLeadUnitDim.cpp      |  34 +-
 .../VectorEmulateMaskedLoadStore.cpp          |  28 +-
 .../Transforms/VectorEmulateNarrowType.cpp    | 177 ++---
 ...sertExtractStridedSliceRewritePatterns.cpp |  30 +-
 .../Vector/Transforms/VectorLinearize.cpp     |   6 +-
 .../Transforms/VectorMaskElimination.cpp      |   2 +-
 .../Transforms/VectorTransferOpTransforms.cpp |  26 +-
 .../VectorTransferSplitRewritePatterns.cpp    |  72 +-
 .../Vector/Transforms/VectorTransforms.cpp    |  82 +-
 .../Vector/Transforms/VectorUnroll.cpp        |  32 +-
 mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp |   6 +-
 .../Dialect/X86Vector/IR/X86VectorDialect.cpp |   6 +-
 .../X86Vector/Transforms/AVXTranspose.cpp     |  26 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   4 +-
 .../Transforms/XeGPUSubgroupDistribute.cpp    |  28 +-
 .../Dialect/XeGPU/Transforms/XeGPUUnroll.cpp  |  38 +-
 .../Transforms/XeGPUWgToSgDistribute.cpp      |  22 +-
 mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp   |  12 +-
 mlir/lib/IR/BuiltinDialect.cpp                |   2 +-
 mlir/lib/Query/Query.cpp                      |   2 +-
 .../lib/Target/LLVMIR/LLVMImportInterface.cpp |   2 +-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |  76 +-
 .../SPIRV/Deserialization/DeserializeOps.cpp  |  20 +-
 .../SPIRV/Deserialization/Deserializer.cpp    |  36 +-
 mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp       |  44 +-
 .../Transforms/Utils/DialectConversion.cpp    |   2 +-
 349 files changed, 6991 insertions(+), 6987 deletions(-)

diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index ef35ee208f002..e7e0a4ebd2092 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -50,20 +50,20 @@ static Value convertUnsignedToI32(ConversionPatternRewriter &rewriter,
   if (i32 == valTy)
     return val;
   return valTy.getWidth() > 32
-             ? Value(rewriter.create<LLVM::TruncOp>(loc, i32, val))
-             : Value(rewriter.create<LLVM::ZExtOp>(loc, i32, val));
+             ? Value(LLVM::TruncOp::create(rewriter, loc, i32, val))
+             : Value(LLVM::ZExtOp::create(rewriter, loc, i32, val));
 }
 
 static Value createI32Constant(ConversionPatternRewriter &rewriter,
                                Location loc, int32_t value) {
   Type i32 = rewriter.getI32Type();
-  return rewriter.create<LLVM::ConstantOp>(loc, i32, value);
+  return LLVM::ConstantOp::create(rewriter, loc, i32, value);
 }
 
 static Value createI1Constant(ConversionPatternRewriter &rewriter, Location loc,
                               bool value) {
   Type llvmI1 = rewriter.getI1Type();
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmI1, value);
+  return LLVM::ConstantOp::create(rewriter, loc, llvmI1, value);
 }
 
 /// Returns the linear index used to access an element in the memref.
@@ -78,11 +78,11 @@ static Value getLinearIndexI32(ConversionPatternRewriter &rewriter,
           ShapedType::isDynamic(stride)
               ? convertUnsignedToI32(rewriter, loc,
                                      memRefDescriptor.stride(rewriter, loc, i))
-              : rewriter.create<LLVM::ConstantOp>(loc, i32, stride);
-      increment = rewriter.create<LLVM::MulOp>(loc, increment, strideValue);
+              : LLVM::ConstantOp::create(rewriter, loc, i32, stride);
+      increment = LLVM::MulOp::create(rewriter, loc, increment, strideValue);
     }
     index =
-        index ? rewriter.create<LLVM::AddOp>(loc, index, increment) : increment;
+        index ? LLVM::AddOp::create(rewriter, loc, index, increment) : increment;
   }
   return index ? index : createI32Constant(rewriter, loc, 0);
 }
@@ -110,14 +110,14 @@ static Value getNumRecords(ConversionPatternRewriter &rewriter, Location loc,
   for (uint32_t i = 0, e = memrefType.getRank(); i < e; ++i) {
     Value size = memrefDescriptor.size(rewriter, loc, i);
     Value stride = memrefDescriptor.stride(rewriter, loc, i);
-    Value maxThisDim = rewriter.create<LLVM::MulOp>(loc, size, stride);
+    Value maxThisDim = LLVM::MulOp::create(rewriter, loc, size, stride);
     maxIndex = maxIndex
-                   ? rewriter.create<LLVM::UMaxOp>(loc, maxIndex, maxThisDim)
+                   ? LLVM::UMaxOp::create(rewriter, loc, maxIndex, maxThisDim)
                    : maxThisDim;
   }
   Value maxIndexI32 = convertUnsignedToI32(rewriter, loc, maxIndex);
   Value byteWidthConst = createI32Constant(rewriter, loc, elementByteWidth);
-  return rewriter.create<LLVM::MulOp>(loc, maxIndexI32, byteWidthConst);
+  return LLVM::MulOp::create(rewriter, loc, maxIndexI32, byteWidthConst);
 }
 
 static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
@@ -132,13 +132,13 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
   Value stride;
   if (chipset.majorVersion == 9 && chipset >= kGfx942 && cacheSwizzleStride) {
     Value cacheStrideZext =
-        rewriter.create<LLVM::ZExtOp>(loc, i16, cacheSwizzleStride);
-    Value swizzleBit = rewriter.create<LLVM::ConstantOp>(
+        LLVM::ZExtOp::create(rewriter, loc, i16, cacheSwizzleStride);
+    Value swizzleBit = LLVM::ConstantOp::create(rewriter,
         loc, i16, rewriter.getI16IntegerAttr(1 << 14));
-    stride = rewriter.create<LLVM::OrOp>(loc, cacheStrideZext, swizzleBit,
+    stride = LLVM::OrOp::create(rewriter, loc, cacheStrideZext, swizzleBit,
                                          /*isDisjoint=*/true);
   } else {
-    stride = rewriter.create<LLVM::ConstantOp>(loc, i16,
+    stride = LLVM::ConstantOp::create(rewriter, loc, i16,
                                                rewriter.getI16IntegerAttr(0));
   }
   // Get the number of elements.
@@ -209,18 +209,18 @@ struct FatRawBufferCastLowering
             : descriptor.alignedPtr(rewriter, loc);
 
     Value offset = adaptor.getResetOffset()
-                       ? rewriter.create<LLVM::ConstantOp>(
+                       ? LLVM::ConstantOp::create(rewriter,
                              loc, getIndexType(), rewriter.getIndexAttr(0))
                        : descriptor.offset(rewriter, loc);
 
     bool hasSizes = memrefType.getRank() > 0;
     // No need to unpack() and pack() all the individual sizes and strides,
     // so we'll just extract the arrays.
-    Value sizes = hasSizes ? rewriter.create<LLVM::ExtractValueOp>(
+    Value sizes = hasSizes ? LLVM::ExtractValueOp::create(rewriter,
                                  loc, descriptor, kSizePosInMemRefDescriptor)
                            : Value{};
     Value strides = hasSizes
-                        ? rewriter.create<LLVM::ExtractValueOp>(
+                        ? LLVM::ExtractValueOp::create(rewriter,
                               loc, descriptor, kStridePosInMemRefDescriptor)
                         : Value{};
 
@@ -231,16 +231,16 @@ struct FatRawBufferCastLowering
     Value result = MemRefDescriptor::poison(
         rewriter, loc,
         getTypeConverter()->convertType(op.getResult().getType()));
-    result = rewriter.create<LLVM::InsertValueOp>(
+    result = LLVM::InsertValueOp::create(rewriter,
         loc, result, fatPtr, kAllocatedPtrPosInMemRefDescriptor);
-    result = rewriter.create<LLVM::InsertValueOp>(
+    result = LLVM::InsertValueOp::create(rewriter,
         loc, result, fatPtr, kAlignedPtrPosInMemRefDescriptor);
-    result = rewriter.create<LLVM::InsertValueOp>(loc, result, offset,
+    result = LLVM::InsertValueOp::create(rewriter, loc, result, offset,
                                                   kOffsetPosInMemRefDescriptor);
     if (hasSizes) {
-      result = rewriter.create<LLVM::InsertValueOp>(loc, result, sizes,
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, sizes,
                                                     kSizePosInMemRefDescriptor);
-      result = rewriter.create<LLVM::InsertValueOp>(
+      result = LLVM::InsertValueOp::create(rewriter,
           loc, result, strides, kStridePosInMemRefDescriptor);
     }
     rewriter.replaceOp(op, result);
@@ -343,7 +343,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     if (storeData) {
       if (llvmBufferValType != llvmWantedDataType) {
         Value castForStore =
-            rewriter.create<LLVM::BitcastOp>(loc, llvmBufferValType, storeData);
+            LLVM::BitcastOp::create(rewriter, loc, llvmBufferValType, storeData);
         args.push_back(castForStore);
       } else {
         args.push_back(storeData);
@@ -352,7 +352,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     if (atomicCmpData) {
       if (llvmBufferValType != llvmWantedDataType) {
-        Value castForCmp = rewriter.create<LLVM::BitcastOp>(
+        Value castForCmp = LLVM::BitcastOp::create(rewriter,
             loc, llvmBufferValType, atomicCmpData);
         args.push_back(castForCmp);
       } else {
@@ -383,17 +383,17 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
         indexOffset && *indexOffset > 0) {
       Value extraOffsetConst = createI32Constant(rewriter, loc, *indexOffset);
       voffset =
-          voffset ? rewriter.create<LLVM::AddOp>(loc, voffset, extraOffsetConst)
+          voffset ? LLVM::AddOp::create(rewriter, loc, voffset, extraOffsetConst)
                   : extraOffsetConst;
     }
-    voffset = rewriter.create<LLVM::MulOp>(loc, voffset, byteWidthConst);
+    voffset = LLVM::MulOp::create(rewriter, loc, voffset, byteWidthConst);
     args.push_back(voffset);
 
     // SGPR offset.
     Value sgprOffset = adaptor.getSgprOffset();
     if (!sgprOffset)
       sgprOffset = createI32Constant(rewriter, loc, 0);
-    sgprOffset = rewriter.create<LLVM::MulOp>(loc, sgprOffset, byteWidthConst);
+    sgprOffset = LLVM::MulOp::create(rewriter, loc, sgprOffset, byteWidthConst);
     args.push_back(sgprOffset);
 
     // bit 0: GLC = 0 (atomics drop value, less coherency)
@@ -403,12 +403,12 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
                                            llvmBufferValType);
-    Operation *lowered = rewriter.create<Intrinsic>(loc, resultTypes, args,
+    Operation *lowered = Intrinsic::create(rewriter, loc, resultTypes, args,
                                                     ArrayRef<NamedAttribute>());
     if (lowered->getNumResults() == 1) {
       Value replacement = lowered->getResult(0);
       if (llvmBufferValType != llvmWantedDataType) {
-        replacement = rewriter.create<LLVM::BitcastOp>(loc, llvmWantedDataType,
+        replacement = LLVM::BitcastOp::create(rewriter, loc, llvmWantedDataType,
                                                        replacement);
       }
       rewriter.replaceOp(gpuOp, replacement);
@@ -465,12 +465,12 @@ struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
                << chipset.majorVersion;
 
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::SWaitcntOp>(loc, ldsOnlyBits);
+      ROCDL::SWaitcntOp::create(rewriter, loc, ldsOnlyBits);
       rewriter.replaceOpWithNewOp<ROCDL::SBarrierOp>(op);
     } else {
       Location loc = op->getLoc();
-      rewriter.create<ROCDL::WaitDscntOp>(loc, 0);
-      rewriter.create<ROCDL::BarrierSignalOp>(loc, -1);
+      ROCDL::WaitDscntOp::create(rewriter, loc, 0);
+      ROCDL::BarrierSignalOp::create(rewriter, loc, -1);
       rewriter.replaceOpWithNewOp<ROCDL::BarrierWaitOp>(op, -1);
     }
 
@@ -516,18 +516,18 @@ static Value convertMFMAVectorOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   if (auto vectorType = dyn_cast<VectorType>(inputType)) {
     if (vectorType.getElementType().isBF16() && !allowBf16)
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, vectorType.clone(rewriter.getI16Type()), input);
     if (vectorType.getElementType().isInteger(8) &&
         vectorType.getNumElements() <= 8)
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, rewriter.getIntegerType(vectorType.getNumElements() * 8), input);
     if (isa<IntegerType>(vectorType.getElementType()) &&
         vectorType.getElementTypeBitWidth() <= 8) {
       int64_t numWords = llvm::divideCeil(
           vectorType.getNumElements() * vectorType.getElementTypeBitWidth(),
           32);
-      return rewriter.create<LLVM::BitcastOp>(
+      return LLVM::BitcastOp::create(rewriter,
           loc, VectorType::get(numWords, rewriter.getI32Type()), input);
     }
   }
@@ -549,8 +549,8 @@ static Value castMFMAScaleOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   Type outputType = rewriter.getI32Type();
   if (auto intType = dyn_cast<IntegerType>(inputType))
-    return rewriter.create<LLVM::ZExtOp>(loc, outputType, input);
-  return rewriter.create<LLVM::BitcastOp>(loc, outputType, input);
+    return LLVM::ZExtOp::create(rewriter, loc, outputType, input);
+  return LLVM::BitcastOp::create(rewriter, loc, outputType, input);
 }
 
 /// Push an input operand. If it is a float type, nothing to do. If it is
@@ -576,7 +576,7 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   Type elemType = vectorType.getElementType();
 
   if (elemType.isBF16())
-    llvmInput = rewriter.create<LLVM::BitcastOp>(
+    llvmInput = LLVM::BitcastOp::create(rewriter,
         loc, vectorType.clone(rewriter.getI16Type()), llvmInput);
   if (elemType.getIntOrFloatBitWidth() > 8) {
     operands.push_back(llvmInput);
@@ -613,7 +613,7 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   // (256 / 64) * 4 = 16 bits of input (on gfx12+) but take i32 arguments.
   // Add in the zeros here.
   if (numBits < 32)
-    castInput = rewriter.create<LLVM::ZExtOp>(loc, i32, castInput);
+    castInput = LLVM::ZExtOp::create(rewriter, loc, i32, castInput);
   operands.push_back(castInput);
 }
 
@@ -633,7 +633,7 @@ static void wmmaPushOutputOperand(ConversionPatternRewriter &rewriter,
   auto vectorType = dyn_cast<VectorType>(inputType);
   Type elemType = vectorType.getElementType();
   if (elemType.isBF16())
-    output = rewriter.create<LLVM::BitcastOp>(
+    output = LLVM::BitcastOp::create(rewriter,
         loc, vectorType.clone(rewriter.getI16Type()), output);
   operands.push_back(output);
   if (elemType.isF16() || elemType.isBF16() || elemType.isInteger(16)) {
@@ -992,7 +992,7 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
     };
     Value lowered = rewriter.create(loweredOp)->getResult(0);
     if (outType != intrinsicOutType)
-      lowered = rewriter.create<LLVM::BitcastOp>(loc, outType, lowered);
+      lowered = LLVM::BitcastOp::create(rewriter, loc, outType, lowered);
     rewriter.replaceOp(op, lowered);
     return success();
   }
@@ -1093,7 +1093,7 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
     Operation *maybeCastBack = lowered;
     if (rawOutType != outType)
       maybeCastBack =
-          rewriter.create<LLVM::BitcastOp>(loc, outType, lowered->getResult(0));
+          LLVM::BitcastOp::create(rewriter, loc, outType, lowered->getResult(0));
     rewriter.replaceOp(op, maybeCastBack->getResults());
 
     return success();
@@ -1144,21 +1144,21 @@ struct TransposeLoadOpLowering
     case 4: {
       assert(numElements == 16);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr4_b64>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr4_b64::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 6: {
       assert(numElements == 16);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr6_b96>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr6_b96::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 8: {
       assert(numElements == 8);
       auto rocdlOp =
-          rewriter.create<ROCDL::ds_read_tr8_b64>(loc, rocdlResultType, srcPtr);
+          ROCDL::ds_read_tr8_b64::create(rewriter, loc, rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
@@ -1316,21 +1316,21 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
   Type sourceElemType = getElementTypeOrSelf(op.getSource());
   // Extend to a v4i8
   if (!sourceVecType || sourceVecType.getNumElements() < 4) {
-    Value longVec = rewriter.create<LLVM::UndefOp>(loc, v4i8);
+    Value longVec = LLVM::UndefOp::create(rewriter, loc, v4i8);
     if (!sourceVecType) {
-      longVec = rewriter.create<LLVM::InsertElementOp>(
+      longVec = LLVM::InsertElementOp::create(rewriter,
           loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
-        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        Value elem = LLVM::ExtractElementOp::create(rewriter, loc, source, idx);
         longVec =
-            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+            LLVM::InsertElementOp::create(rewriter, loc, longVec, elem, idx);
       }
     }
     source = longVec;
   }
-  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+  Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
   if (resultVecType) {
     if (typeIsExpectedBf8ForChipset(chipset, sourceElemType)) {
       rewriter.replaceOpWithNewOp<ROCDL::CvtPkF32Bf8Op>(op, f32, i32Source,
@@ -1382,21 +1382,21 @@ LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
 
   // Extend to a packedVectorType
   if (sourceVecType.getNumElements() < packedVecType.getNumElements()) {
-    Value longVec = rewriter.create<LLVM::ZeroOp>(loc, packedVecType);
+    Value longVec = LLVM::ZeroOp::create(rewriter, loc, packedVecType);
     if (!sourceVecType) {
-      longVec = rewriter.create<LLVM::InsertElementOp>(
+      longVec = LLVM::InsertElementOp::create(rewriter,
           loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
-        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        Value elem = LLVM::ExtractElementOp::create(rewriter, loc, source, idx);
         longVec =
-            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+            LLVM::InsertElementOp::create(rewriter, loc, longVec, elem, idx);
       }
     }
     source = longVec;
   }
-  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+  Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
 
   if (isa<Float8E5M2Type>(sourceElemType) && destElemType.isF32())
     rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Bf8Op>(
@@ -1454,53 +1454,53 @@ LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
   Value scale = adaptor.getScale();
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, intResultType, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, intResultType, existing);
   else
-    existing = rewriter.create<LLVM::ZeroOp>(loc, intResultType);
+    existing = LLVM::ZeroOp::create(rewriter, loc, intResultType);
 
   if (sourceVecType.getNumElements() < 2) {
     Value c0 = createI32Constant(rewriter, loc, 0);
-    Value elem0 = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
+    Value elem0 = LLVM::ExtractElementOp::create(rewriter, loc, source, c0);
     VectorType v2 = VectorType::get(2, sourceElemType);
-    source = rewriter.create<LLVM::ZeroOp>(loc, v2);
-    source = rewriter.create<LLVM::InsertElementOp>(loc, source, elem0, c0);
+    source = LLVM::ZeroOp::create(rewriter, loc, v2);
+    source = LLVM::InsertElementOp::create(rewriter, loc, source, elem0, c0);
   }
 
   Value sourceA, sourceB;
   if (sourceElemType.isF32()) {
     Value c0 = createI32Constant(rewriter, loc, 0);
     Value c1 = createI32Constant(rewriter, loc, 1);
-    sourceA = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
-    sourceB = rewriter.create<LLVM::ExtractElementOp>(loc, source, c1);
+    sourceA = LLVM::ExtractElementOp::create(rewriter, loc, source, c0);
+    sourceB = LLVM::ExtractElementOp::create(rewriter, loc, source, c1);
   }
 
   Value result;
   if (sourceElemType.isF32() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F32Op>(
+    result = ROCDL::CvtScaleF32PkBf8F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F16Op>(
+    result = ROCDL::CvtScaleF32PkBf8F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E5M2Type>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkBf8Bf16Op>(
+    result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F32Op>(
+    result = ROCDL::CvtScaleF32PkFp8F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F16Op>(
+    result = ROCDL::CvtScaleF32PkFp8F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp8Bf16Op>(
+    result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F32Op>(
+    result = ROCDL::CvtScaleF32PkFp4F32Op::create(rewriter,
         loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F16Op>(
+    result = ROCDL::CvtScaleF32PkFp4F16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = rewriter.create<ROCDL::CvtScaleF32PkFp4Bf16Op>(
+    result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(rewriter,
         loc, intResultType, existing, source, scale, op.getIndex());
   else
     return failure();
@@ -1526,19 +1526,19 @@ LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
   Value sourceA = adaptor.getSourceA();
   Value sourceB = adaptor.getSourceB();
   if (!sourceB)
-    sourceB = rewriter.create<LLVM::UndefOp>(loc, sourceA.getType());
+    sourceB = LLVM::UndefOp::create(rewriter, loc, sourceA.getType());
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, i32, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, i32, existing);
   else
-    existing = rewriter.create<LLVM::UndefOp>(loc, i32);
+    existing = LLVM::UndefOp::create(rewriter, loc, i32);
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtPkBf8F32Op>(loc, i32, sourceA, sourceB,
+    result = ROCDL::CvtPkBf8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
                                                    existing, op.getWordIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtPkFp8F32Op>(loc, i32, sourceA, sourceB,
+    result = ROCDL::CvtPkFp8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
                                                    existing, op.getWordIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
@@ -1563,16 +1563,16 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
   Value stoch = adaptor.getStochiasticParam();
   Value existing = adaptor.getExisting();
   if (existing)
-    existing = rewriter.create<LLVM::BitcastOp>(loc, i32, existing);
+    existing = LLVM::BitcastOp::create(rewriter, loc, i32, existing);
   else
-    existing = rewriter.create<LLVM::UndefOp>(loc, i32);
+    existing = LLVM::UndefOp::create(rewriter, loc, i32);
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtSrBf8F32Op>(
+    result = ROCDL::CvtSrBf8F32Op::create(rewriter,
         loc, i32, source, stoch, existing, op.getStoreIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = rewriter.create<ROCDL::CvtSrFp8F32Op>(
+    result = ROCDL::CvtSrFp8F32Op::create(rewriter,
         loc, i32, source, stoch, existing, op.getStoreIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
@@ -1617,14 +1617,14 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
       if (operandType.getIntOrFloatBitWidth() <= 16) {
         if (llvm::isa<FloatType>(operandType)) {
           operand =
-              rewriter.create<LLVM::BitcastOp>(loc, llvmSrcIntType, operand);
+              LLVM::BitcastOp::create(rewriter, loc, llvmSrcIntType, operand);
         }
         auto llvmVecType = typeConverter->convertType(mlir::VectorType::get(
             32 / operandType.getIntOrFloatBitWidth(), llvmSrcIntType));
-        Value undefVec = rewriter.create<LLVM::UndefOp>(loc, llvmVecType);
-        operand = rewriter.create<LLVM::InsertElementOp>(
+        Value undefVec = LLVM::UndefOp::create(rewriter, loc, llvmVecType);
+        operand = LLVM::InsertElementOp::create(rewriter,
             loc, undefVec, operand, createI32Constant(rewriter, loc, 0));
-        operand = rewriter.create<LLVM::BitcastOp>(loc, llvmType, operand);
+        operand = LLVM::BitcastOp::create(rewriter, loc, llvmType, operand);
       }
       return operand;
     };
@@ -1711,14 +1711,14 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
     bool boundCtrl = DppOp->getAttrOfType<BoolAttr>("bound_ctrl").getValue();
 
     // create a ROCDL_DPPMovOp instruction with the appropriate attributes
-    auto dppMovOp = rewriter.create<ROCDL::DPPUpdateOp>(
+    auto dppMovOp = ROCDL::DPPUpdateOp::create(rewriter,
         loc, llvmType, old, src, DppCtrl, rowMask, bankMask, boundCtrl);
 
     Value result = dppMovOp.getRes();
     if (srcType.getIntOrFloatBitWidth() < 32) {
-      result = rewriter.create<LLVM::TruncOp>(loc, llvmSrcIntType, result);
+      result = LLVM::TruncOp::create(rewriter, loc, llvmSrcIntType, result);
       if (!llvm::isa<IntegerType>(srcType)) {
-        result = rewriter.create<LLVM::BitcastOp>(loc, srcType, result);
+        result = LLVM::BitcastOp::create(rewriter, loc, srcType, result);
       }
     }
 
@@ -1752,7 +1752,7 @@ struct AMDGPUSwizzleBitModeLowering
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
       Value res =
-          rewriter.create<ROCDL::DsSwizzleOp>(loc, v.getType(), v, maskValue);
+          ROCDL::DsSwizzleOp::create(rewriter, loc, v.getType(), v, maskValue);
       swizzled.emplace_back(res);
     }
 
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 3b143ca1ef9ce..5a9d946c2fe4e 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -50,9 +50,9 @@ static Value buildMinMaxReductionSeq(Location loc,
   Value value = *valueIt++;
   for (; valueIt != values.end(); ++valueIt) {
     if (predicate == arith::CmpIPredicate::sgt)
-      value = builder.create<arith::MaxSIOp>(loc, value, *valueIt);
+      value = arith::MaxSIOp::create(builder, loc, value, *valueIt);
     else
-      value = builder.create<arith::MinSIOp>(loc, value, *valueIt);
+      value = arith::MinSIOp::create(builder, loc, value, *valueIt);
   }
 
   return value;
@@ -154,8 +154,8 @@ class AffineForLowering : public OpRewritePattern<AffineForOp> {
     Value lowerBound = lowerAffineLowerBound(op, rewriter);
     Value upperBound = lowerAffineUpperBound(op, rewriter);
     Value step =
-        rewriter.create<arith::ConstantIndexOp>(loc, op.getStepAsInt());
-    auto scfForOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound,
+        arith::ConstantIndexOp::create(rewriter, loc, op.getStepAsInt());
+    auto scfForOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound,
                                                 step, op.getInits());
     rewriter.eraseBlock(scfForOp.getBody());
     rewriter.inlineRegionBefore(op.getRegion(), scfForOp.getRegion(),
@@ -197,7 +197,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     }
     steps.reserve(op.getSteps().size());
     for (int64_t step : op.getSteps())
-      steps.push_back(rewriter.create<arith::ConstantIndexOp>(loc, step));
+      steps.push_back(arith::ConstantIndexOp::create(rewriter, loc, step));
 
     // Get the terminator op.
     auto affineParOpTerminator =
@@ -205,7 +205,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     scf::ParallelOp parOp;
     if (op.getResults().empty()) {
       // Case with no reduction operations/return values.
-      parOp = rewriter.create<scf::ParallelOp>(loc, lowerBoundTuple,
+      parOp = scf::ParallelOp::create(rewriter, loc, lowerBoundTuple,
                                                upperBoundTuple, steps,
                                                /*bodyBuilderFn=*/nullptr);
       rewriter.eraseBlock(parOp.getBody());
@@ -233,7 +233,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       identityVals.push_back(
           arith::getIdentityValue(reductionOpValue, resultType, rewriter, loc));
     }
-    parOp = rewriter.create<scf::ParallelOp>(
+    parOp = scf::ParallelOp::create(rewriter,
         loc, lowerBoundTuple, upperBoundTuple, steps, identityVals,
         /*bodyBuilderFn=*/nullptr);
 
@@ -261,7 +261,7 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       Value reductionResult = arith::getReductionOp(
           reductionOpValue, rewriter, loc, reductionBody.getArgument(0),
           reductionBody.getArgument(1));
-      rewriter.create<scf::ReduceReturnOp>(loc, reductionResult);
+      scf::ReduceReturnOp::create(rewriter, loc, reductionResult);
     }
     rewriter.replaceOp(op, parOp.getResults());
     return success();
@@ -278,7 +278,7 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
 
     // Now we just have to handle the condition logic.
     auto integerSet = op.getIntegerSet();
-    Value zeroConstant = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zeroConstant = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value, 8> operands(op.getOperands());
     auto operandsRef = llvm::ArrayRef(operands);
 
@@ -298,17 +298,17 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
       auto pred =
           isEquality ? arith::CmpIPredicate::eq : arith::CmpIPredicate::sge;
       Value cmpVal =
-          rewriter.create<arith::CmpIOp>(loc, pred, affResult, zeroConstant);
+          arith::CmpIOp::create(rewriter, loc, pred, affResult, zeroConstant);
       cond = cond
-                 ? rewriter.create<arith::AndIOp>(loc, cond, cmpVal).getResult()
+                 ? arith::AndIOp::create(rewriter, loc, cond, cmpVal).getResult()
                  : cmpVal;
     }
     cond = cond ? cond
-                : rewriter.create<arith::ConstantIntOp>(loc, /*value=*/1,
+                : arith::ConstantIntOp::create(rewriter, loc, /*value=*/1,
                                                         /*width=*/1);
 
     bool hasElseRegion = !op.getElseRegion().empty();
-    auto ifOp = rewriter.create<scf::IfOp>(loc, op.getResultTypes(), cond,
+    auto ifOp = scf::IfOp::create(rewriter, loc, op.getResultTypes(), cond,
                                            hasElseRegion);
     rewriter.inlineRegionBefore(op.getThenRegion(),
                                 &ifOp.getThenRegion().back());
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index cf9bb3a000050..df3f2c4926614 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -115,9 +115,9 @@ static Value castF32To(Type desType, Value f32, Location loc,
   if (elementType.isF32())
     return f32;
   if (elementType.getIntOrFloatBitWidth() < 32)
-    return rewriter.create<arith::TruncFOp>(loc, desType, f32);
+    return arith::TruncFOp::create(rewriter, loc, desType, f32);
   if (elementType.getIntOrFloatBitWidth() > 32)
-    return rewriter.create<arith::ExtFOp>(loc, desType, f32);
+    return arith::ExtFOp::create(rewriter, loc, desType, f32);
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
@@ -139,7 +139,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
   VectorType extResType = VectorType::get(2, rewriter.getF32Type());
   if (!inVecType) {
-    Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
+    Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
         loc, rewriter.getF32Type(), in, 0);
     Value result = castF32To(outElemType, asFloat, loc, rewriter);
     rewriter.replaceOp(op, result);
@@ -147,7 +147,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   }
   int64_t numElements = inVecType.getNumElements();
 
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   VectorType outType = cast<VectorType>(op.getOut().getType());
 
@@ -155,10 +155,10 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
     Value zerodSplat =
         rewriter.createOrFold<vector::SplatOp>(loc, outType, zero);
     Value scalarIn =
-        rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
+        vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     Value scalarExt =
-        rewriter.create<arith::ExtFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarExt, zerodSplat,
+        arith::ExtFOp::create(rewriter, loc, outElemType, scalarIn);
+    Value result = vector::InsertOp::create(rewriter, loc, scalarExt, zerodSplat,
                                                      ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
@@ -171,32 +171,32 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   if (inVecType.getRank() > 1) {
     inVecType = VectorType::get(SmallVector<int64_t>{numElements},
                                 inVecType.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVecType, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVecType, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
-    Value inSlice = rewriter.create<vector::ExtractStridedSliceOp>(
+    Value inSlice = vector::ExtractStridedSliceOp::create(rewriter,
         loc, in, i, elemsThisOp, 1);
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
       if (i + j + 1 < numElements) { // Convert two 8-bit elements
-        Value asFloats = rewriter.create<amdgpu::ExtPackedFp8Op>(
+        Value asFloats = amdgpu::ExtPackedFp8Op::create(rewriter,
             loc, extResType, inSlice, j / 2);
         Type desType = VectorType::get(2, outElemType);
         Value asType = castF32To(desType, asFloats, loc, rewriter);
-        result = rewriter.create<vector::InsertStridedSliceOp>(
+        result = vector::InsertStridedSliceOp::create(rewriter,
             loc, asType, result, i + j, 1);
       } else { // Convert a 8-bit element
-        Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
+        Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
             loc, rewriter.getF32Type(), inSlice, j / 2 * 2);
         Value asType = castF32To(outElemType, asFloat, loc, rewriter);
-        result = rewriter.create<vector::InsertOp>(loc, asType, result, i + j);
+        result = vector::InsertOp::create(rewriter, loc, asType, result, i + j);
       }
     }
   }
 
   if (inVecType.getRank() != outType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outType, result);
   }
 
   rewriter.replaceOp(op, result);
@@ -208,9 +208,9 @@ static Value castToF32(Value value, Location loc, PatternRewriter &rewriter) {
   if (type.isF32())
     return value;
   if (type.getIntOrFloatBitWidth() < 32)
-    return rewriter.create<arith::ExtFOp>(loc, rewriter.getF32Type(), value);
+    return arith::ExtFOp::create(rewriter, loc, rewriter.getF32Type(), value);
   if (type.getIntOrFloatBitWidth() > 32)
-    return rewriter.create<arith::TruncFOp>(loc, rewriter.getF32Type(), value);
+    return arith::TruncFOp::create(rewriter, loc, rewriter.getF32Type(), value);
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
@@ -250,13 +250,13 @@ static Value clampInput(PatternRewriter &rewriter, Location loc,
       loc, arith::CmpFPredicate::OEQ, source, negInf);
   Value isNan = rewriter.createOrFold<arith::CmpFOp>(
       loc, arith::CmpFPredicate::UNO, source, source);
-  Value isNonFinite = rewriter.create<arith::OrIOp>(
-      loc, rewriter.create<arith::OrIOp>(loc, isInf, isNegInf), isNan);
+  Value isNonFinite = arith::OrIOp::create(rewriter,
+      loc, arith::OrIOp::create(rewriter, loc, isInf, isNegInf), isNan);
 
-  Value clampedBelow = rewriter.create<arith::MaximumFOp>(loc, source, minCst);
-  Value clamped = rewriter.create<arith::MinimumFOp>(loc, clampedBelow, maxCst);
+  Value clampedBelow = arith::MaximumFOp::create(rewriter, loc, source, minCst);
+  Value clamped = arith::MinimumFOp::create(rewriter, loc, clampedBelow, maxCst);
   Value res =
-      rewriter.create<arith::SelectOp>(loc, isNonFinite, source, clamped);
+      arith::SelectOp::create(rewriter, loc, isNonFinite, source, clamped);
   return res;
 }
 
@@ -290,24 +290,24 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   VectorType truncResType = VectorType::get(4, outElemType);
   if (!inVectorTy) {
     Value asFloat = castToF32(in, loc, rewriter);
-    Value asF8s = rewriter.create<amdgpu::PackedTrunc2xFp8Op>(
+    Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
         loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
         /*existing=*/nullptr);
-    Value result = rewriter.create<vector::ExtractOp>(loc, asF8s, 0);
+    Value result = vector::ExtractOp::create(rewriter, loc, asF8s, 0);
     rewriter.replaceOp(op, result);
     return success();
   }
 
   int64_t numElements = outVecType.getNumElements();
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   if (outVecType.getShape().empty()) {
     Value scalarIn =
-        rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
+        vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     // Recurse to send the 0-D vector case to the 1-D vector case
     Value scalarTrunc =
-        rewriter.create<arith::TruncFOp>(loc, outElemType, scalarIn);
-    Value result = rewriter.create<vector::InsertOp>(loc, scalarTrunc, zero,
+        arith::TruncFOp::create(rewriter, loc, outElemType, scalarIn);
+    Value result = vector::InsertOp::create(rewriter, loc, scalarTrunc, zero,
                                                      ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
@@ -320,32 +320,32 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
                                  inVectorTy.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVectorTy, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVectorTy, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
     Value thisResult = nullptr;
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
-      Value elemA = rewriter.create<vector::ExtractOp>(loc, in, i + j);
+      Value elemA = vector::ExtractOp::create(rewriter, loc, in, i + j);
       Value asFloatA = castToF32(elemA, loc, rewriter);
       Value asFloatB = nullptr;
       if (j + 1 < elemsThisOp) {
-        Value elemB = rewriter.create<vector::ExtractOp>(loc, in, i + j + 1);
+        Value elemB = vector::ExtractOp::create(rewriter, loc, in, i + j + 1);
         asFloatB = castToF32(elemB, loc, rewriter);
       }
-      thisResult = rewriter.create<amdgpu::PackedTrunc2xFp8Op>(
+      thisResult = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
           loc, truncResType, asFloatA, asFloatB, j / 2, thisResult);
     }
     if (elemsThisOp < 4)
-      thisResult = rewriter.create<vector::ExtractStridedSliceOp>(
+      thisResult = vector::ExtractStridedSliceOp::create(rewriter,
           loc, thisResult, 0, elemsThisOp, 1);
-    result = rewriter.create<vector::InsertStridedSliceOp>(loc, thisResult,
+    result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
                                                            result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
@@ -373,10 +373,10 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
 
   // Handle the case where input type is not a vector type
   if (!inVectorTy) {
-    auto sourceB = rewriter.create<LLVM::PoisonOp>(loc, rewriter.getF32Type());
+    auto sourceB = LLVM::PoisonOp::create(rewriter, loc, rewriter.getF32Type());
     Value asF16s =
-        rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, in, sourceB);
-    Value result = rewriter.create<vector::ExtractOp>(loc, asF16s, 0);
+        ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, in, sourceB);
+    Value result = vector::ExtractOp::create(rewriter, loc, asF16s, 0);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -388,7 +388,7 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
                                  inVectorTy.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inVectorTy, in);
+    in = vector::ShapeCastOp::create(rewriter, loc, inVectorTy, in);
   }
 
   // Handle the vector case. We also handle the (uncommon) case where the vector
@@ -396,25 +396,25 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   for (int64_t i = 0; i < numElements; i += 2) {
     int64_t elemsThisOp = std::min(numElements, i + 2) - i;
     Value thisResult = nullptr;
-    Value elemA = rewriter.create<vector::ExtractOp>(loc, in, i);
-    Value elemB = rewriter.create<LLVM::PoisonOp>(loc, rewriter.getF32Type());
+    Value elemA = vector::ExtractOp::create(rewriter, loc, in, i);
+    Value elemB = LLVM::PoisonOp::create(rewriter, loc, rewriter.getF32Type());
 
     if (elemsThisOp == 2) {
-      elemB = rewriter.create<vector::ExtractOp>(loc, in, i + 1);
+      elemB = vector::ExtractOp::create(rewriter, loc, in, i + 1);
     }
 
     thisResult =
-        rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, elemA, elemB);
+        ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, elemA, elemB);
     // Place back the truncated result into the possibly larger vector. If we
     // are operating on a size 2 vector, these operations should be folded away
-    thisResult = rewriter.create<vector::ExtractStridedSliceOp>(
+    thisResult = vector::ExtractStridedSliceOp::create(rewriter,
         loc, thisResult, 0, elemsThisOp, 1);
-    result = rewriter.create<vector::InsertStridedSliceOp>(loc, thisResult,
+    result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
                                                            result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
+    result = vector::ShapeCastOp::create(rewriter, loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
index cbe0b3fda3410..40bf8f1cffb04 100644
--- a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
+++ b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
@@ -74,14 +74,14 @@ struct ConstantOpToArmSMELowering : public OpRewritePattern<arith::ConstantOp> {
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
     auto denseAttr1D = DenseElementsAttr::get(
         tileSliceType, denseAttr.getSplatValue<Attribute>());
-    auto constantOp1D = rewriter.create<arith::ConstantOp>(loc, denseAttr1D);
+    auto constantOp1D = arith::ConstantOp::create(rewriter, loc, denseAttr1D);
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to write vector to tile
       // slice.
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, constantOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
index a5c08a6378021..c5b3fdc536f72 100644
--- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
+++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
@@ -110,7 +110,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     emitc::CmpPredicate predicate;
     switch (op.getPredicate()) {
     case arith::CmpFPredicate::AlwaysFalse: {
-      auto constant = rewriter.create<emitc::ConstantOp>(
+      auto constant = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getI1Type(),
           rewriter.getBoolAttr(/*value=*/false));
       rewriter.replaceOp(op, constant);
@@ -179,7 +179,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
       return success();
     }
     case arith::CmpFPredicate::AlwaysTrue: {
-      auto constant = rewriter.create<emitc::ConstantOp>(
+      auto constant = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getI1Type(),
           rewriter.getBoolAttr(/*value=*/true));
       rewriter.replaceOp(op, constant);
@@ -189,7 +189,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
 
     // Compare the values naively
     auto cmpResult =
-        rewriter.create<emitc::CmpOp>(op.getLoc(), op.getType(), predicate,
+        emitc::CmpOp::create(rewriter, op.getLoc(), op.getType(), predicate,
                                       adaptor.getLhs(), adaptor.getRhs());
 
     // Adjust the results for unordered/ordered semantics
@@ -213,7 +213,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNaN(ConversionPatternRewriter &rewriter, Location loc,
               Value operand) const {
     // A value is NaN exactly when it compares unequal to itself.
-    return rewriter.create<emitc::CmpOp>(
+    return emitc::CmpOp::create(rewriter,
         loc, rewriter.getI1Type(), emitc::CmpPredicate::ne, operand, operand);
   }
 
@@ -221,7 +221,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNotNaN(ConversionPatternRewriter &rewriter, Location loc,
                  Value operand) const {
     // A value is not NaN exactly when it compares equal to itself.
-    return rewriter.create<emitc::CmpOp>(
+    return emitc::CmpOp::create(rewriter,
         loc, rewriter.getI1Type(), emitc::CmpPredicate::eq, operand, operand);
   }
 
@@ -231,7 +231,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
                                Location loc, Value first, Value second) const {
     auto firstIsNaN = isNaN(rewriter, loc, first);
     auto secondIsNaN = isNaN(rewriter, loc, second);
-    return rewriter.create<emitc::LogicalOrOp>(loc, rewriter.getI1Type(),
+    return emitc::LogicalOrOp::create(rewriter, loc, rewriter.getI1Type(),
                                                firstIsNaN, secondIsNaN);
   }
 
@@ -241,7 +241,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
                              Value first, Value second) const {
     auto firstIsNotNaN = isNotNaN(rewriter, loc, first);
     auto secondIsNotNaN = isNotNaN(rewriter, loc, second);
-    return rewriter.create<emitc::LogicalAndOp>(loc, rewriter.getI1Type(),
+    return emitc::LogicalAndOp::create(rewriter, loc, rewriter.getI1Type(),
                                                 firstIsNotNaN, secondIsNotNaN);
   }
 };
@@ -378,9 +378,9 @@ class CastConversion : public OpConversionPattern<ArithOp> {
       Type attrType = (emitc::isPointerWideType(operandType))
                           ? rewriter.getIndexType()
                           : operandType;
-      auto constOne = rewriter.create<emitc::ConstantOp>(
+      auto constOne = emitc::ConstantOp::create(rewriter,
           op.getLoc(), operandType, rewriter.getOneAttr(attrType));
-      auto oneAndOperand = rewriter.create<emitc::BitwiseAndOp>(
+      auto oneAndOperand = emitc::BitwiseAndOp::create(rewriter,
           op.getLoc(), operandType, adaptor.getIn(), constOne);
       rewriter.replaceOpWithNewOp<emitc::CastOp>(op, opReturnType,
                                                  oneAndOperand);
@@ -467,7 +467,7 @@ class BinaryUIOpConversion final : public OpConversionPattern<ArithOp> {
     Value rhsAdapted = adaptValueType(uiBinOp.getRhs(), rewriter, unsignedType);
 
     auto newDivOp =
-        rewriter.create<EmitCOp>(uiBinOp.getLoc(), unsignedType,
+        EmitCOp::create(rewriter, uiBinOp.getLoc(), unsignedType,
                                  ArrayRef<Value>{lhsAdapted, rhsAdapted});
     Value resultAdapted = adaptValueType(newDivOp, rewriter, newRetTy);
     rewriter.replaceOp(uiBinOp, resultAdapted);
@@ -588,38 +588,38 @@ class ShiftOpConversion : public OpConversionPattern<ArithOp> {
     // Add a runtime check for overflow
     Value width;
     if (emitc::isPointerWideType(type)) {
-      Value eight = rewriter.create<emitc::ConstantOp>(
+      Value eight = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rhsType, rewriter.getIndexAttr(8));
-      emitc::CallOpaqueOp sizeOfCall = rewriter.create<emitc::CallOpaqueOp>(
+      emitc::CallOpaqueOp sizeOfCall = emitc::CallOpaqueOp::create(rewriter,
           op.getLoc(), rhsType, "sizeof", ArrayRef<Value>{eight});
-      width = rewriter.create<emitc::MulOp>(op.getLoc(), rhsType, eight,
+      width = emitc::MulOp::create(rewriter, op.getLoc(), rhsType, eight,
                                             sizeOfCall.getResult(0));
     } else {
-      width = rewriter.create<emitc::ConstantOp>(
+      width = emitc::ConstantOp::create(rewriter,
           op.getLoc(), rhsType,
           rewriter.getIntegerAttr(rhsType, type.getIntOrFloatBitWidth()));
     }
 
-    Value excessCheck = rewriter.create<emitc::CmpOp>(
+    Value excessCheck = emitc::CmpOp::create(rewriter,
         op.getLoc(), rewriter.getI1Type(), emitc::CmpPredicate::lt, rhs, width);
 
     // Any concrete value is a valid refinement of poison.
-    Value poison = rewriter.create<emitc::ConstantOp>(
+    Value poison = emitc::ConstantOp::create(rewriter,
         op.getLoc(), arithmeticType,
         (isa<IntegerType>(arithmeticType)
              ? rewriter.getIntegerAttr(arithmeticType, 0)
              : rewriter.getIndexAttr(0)));
 
-    emitc::ExpressionOp ternary = rewriter.create<emitc::ExpressionOp>(
+    emitc::ExpressionOp ternary = emitc::ExpressionOp::create(rewriter,
         op.getLoc(), arithmeticType, /*do_not_inline=*/false);
     Block &bodyBlock = ternary.getBodyRegion().emplaceBlock();
     auto currentPoint = rewriter.getInsertionPoint();
     rewriter.setInsertionPointToStart(&bodyBlock);
     Value arithmeticResult =
-        rewriter.create<EmitCOp>(op.getLoc(), arithmeticType, lhs, rhs);
-    Value resultOrPoison = rewriter.create<emitc::ConditionalOp>(
+        EmitCOp::create(rewriter, op.getLoc(), arithmeticType, lhs, rhs);
+    Value resultOrPoison = emitc::ConditionalOp::create(rewriter,
         op.getLoc(), arithmeticType, excessCheck, arithmeticResult, poison);
-    rewriter.create<emitc::YieldOp>(op.getLoc(), resultOrPoison);
+    emitc::YieldOp::create(rewriter, op.getLoc(), resultOrPoison);
     rewriter.setInsertionPoint(op->getBlock(), currentPoint);
 
     Value result = adaptValueType(ternary, rewriter, type);
@@ -700,11 +700,11 @@ class FtoICastOpConversion : public OpConversionPattern<CastOp> {
                                   /*isSigned=*/false);
     }
 
-    Value result = rewriter.create<emitc::CastOp>(
+    Value result = emitc::CastOp::create(rewriter,
         castOp.getLoc(), actualResultType, adaptor.getOperands());
 
     if (isa<arith::FPToUIOp>(castOp)) {
-      result = rewriter.create<emitc::CastOp>(castOp.getLoc(), dstType, result);
+      result = emitc::CastOp::create(rewriter, castOp.getLoc(), dstType, result);
     }
     rewriter.replaceOp(castOp, result);
 
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index f7bf581adc9e3..f7d4963b5e293 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -293,10 +293,10 @@ LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
       [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
         typename OpTy::Adaptor adaptor(operands);
         if (targetBits < sourceBits) {
-          return rewriter.create<LLVM::TruncOp>(op.getLoc(), llvm1DVectorTy,
+          return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
                                                 adaptor.getIn());
         }
-        return rewriter.create<ExtCastTy>(op.getLoc(), llvm1DVectorTy,
+        return ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
                                           adaptor.getIn());
       },
       rewriter);
@@ -324,12 +324,12 @@ LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
     Type newOverflowType = typeConverter->convertType(overflowResultType);
     Type structType =
         LLVM::LLVMStructType::getLiteral(ctx, {sumResultType, newOverflowType});
-    Value addOverflow = rewriter.create<LLVM::UAddWithOverflowOp>(
+    Value addOverflow = LLVM::UAddWithOverflowOp::create(rewriter,
         loc, structType, adaptor.getLhs(), adaptor.getRhs());
     Value sumExtracted =
-        rewriter.create<LLVM::ExtractValueOp>(loc, addOverflow, 0);
+        LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 0);
     Value overflowExtracted =
-        rewriter.create<LLVM::ExtractValueOp>(loc, addOverflow, 1);
+        LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 1);
     rewriter.replaceOp(op, {sumExtracted, overflowExtracted});
     return success();
   }
@@ -381,15 +381,15 @@ LogicalResult MulIExtendedOpLowering<ArithMulOp, IsSigned>::matchAndRewrite(
            "LLVM dialect should support all signless integer types");
 
     using LLVMExtOp = std::conditional_t<IsSigned, LLVM::SExtOp, LLVM::ZExtOp>;
-    Value lhsExt = rewriter.create<LLVMExtOp>(loc, wideType, adaptor.getLhs());
-    Value rhsExt = rewriter.create<LLVMExtOp>(loc, wideType, adaptor.getRhs());
-    Value mulExt = rewriter.create<LLVM::MulOp>(loc, wideType, lhsExt, rhsExt);
+    Value lhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getLhs());
+    Value rhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getRhs());
+    Value mulExt = LLVM::MulOp::create(rewriter, loc, wideType, lhsExt, rhsExt);
 
     // Split the 2*N-bit wide result into two N-bit values.
-    Value low = rewriter.create<LLVM::TruncOp>(loc, resultType, mulExt);
-    Value shiftVal = rewriter.create<LLVM::ConstantOp>(loc, shiftValAttr);
-    Value highExt = rewriter.create<LLVM::LShrOp>(loc, mulExt, shiftVal);
-    Value high = rewriter.create<LLVM::TruncOp>(loc, resultType, highExt);
+    Value low = LLVM::TruncOp::create(rewriter, loc, resultType, mulExt);
+    Value shiftVal = LLVM::ConstantOp::create(rewriter, loc, shiftValAttr);
+    Value highExt = LLVM::LShrOp::create(rewriter, loc, mulExt, shiftVal);
+    Value high = LLVM::TruncOp::create(rewriter, loc, resultType, highExt);
 
     rewriter.replaceOp(op, {low, high});
     return success();
@@ -435,7 +435,7 @@ CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return rewriter.create<LLVM::ICmpOp>(
+        return LLVM::ICmpOp::create(rewriter,
             op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs());
@@ -471,7 +471,7 @@ CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return rewriter.create<LLVM::FCmpOp>(
+        return LLVM::FCmpOp::create(rewriter,
             op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs(), fmf);
diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index 434d7df853a5e..c93cf13f831ef 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -117,11 +117,11 @@ static Value getScalarOrVectorConstInt(Type type, uint64_t value,
   if (auto vectorType = dyn_cast<VectorType>(type)) {
     Attribute element = IntegerAttr::get(vectorType.getElementType(), value);
     auto attr = SplatElementsAttr::get(vectorType, element);
-    return builder.create<spirv::ConstantOp>(loc, vectorType, attr);
+    return spirv::ConstantOp::create(builder, loc, vectorType, attr);
   }
 
   if (auto intType = dyn_cast<IntegerType>(type))
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, value));
 
   return nullptr;
@@ -418,18 +418,18 @@ static Value emulateSignedRemainder(Location loc, Value lhs, Value rhs,
   Type type = lhs.getType();
 
   // Calculate the remainder with spirv.UMod.
-  Value lhsAbs = builder.create<SignedAbsOp>(loc, type, lhs);
-  Value rhsAbs = builder.create<SignedAbsOp>(loc, type, rhs);
-  Value abs = builder.create<spirv::UModOp>(loc, lhsAbs, rhsAbs);
+  Value lhsAbs = SignedAbsOp::create(builder, loc, type, lhs);
+  Value rhsAbs = SignedAbsOp::create(builder, loc, type, rhs);
+  Value abs = spirv::UModOp::create(builder, loc, lhsAbs, rhsAbs);
 
   // Fix the sign.
   Value isPositive;
   if (lhs == signOperand)
-    isPositive = builder.create<spirv::IEqualOp>(loc, lhs, lhsAbs);
+    isPositive = spirv::IEqualOp::create(builder, loc, lhs, lhsAbs);
   else
-    isPositive = builder.create<spirv::IEqualOp>(loc, rhs, rhsAbs);
-  Value absNegate = builder.create<spirv::SNegateOp>(loc, type, abs);
-  return builder.create<spirv::SelectOp>(loc, type, isPositive, abs, absNegate);
+    isPositive = spirv::IEqualOp::create(builder, loc, rhs, rhsAbs);
+  Value absNegate = spirv::SNegateOp::create(builder, loc, type, abs);
+  return spirv::SelectOp::create(builder, loc, type, isPositive, abs, absNegate);
 }
 
 /// Converts arith.remsi to GLSL SPIR-V ops.
@@ -601,12 +601,12 @@ struct ExtSII1Pattern final : public OpConversionPattern<arith::ExtSIOp> {
     Value allOnes;
     if (auto intTy = dyn_cast<IntegerType>(dstType)) {
       unsigned componentBitwidth = intTy.getWidth();
-      allOnes = rewriter.create<spirv::ConstantOp>(
+      allOnes = spirv::ConstantOp::create(rewriter,
           loc, intTy,
           rewriter.getIntegerAttr(intTy, APInt::getAllOnes(componentBitwidth)));
     } else if (auto vectorTy = dyn_cast<VectorType>(dstType)) {
       unsigned componentBitwidth = vectorTy.getElementTypeBitWidth();
-      allOnes = rewriter.create<spirv::ConstantOp>(
+      allOnes = spirv::ConstantOp::create(rewriter,
           loc, vectorTy,
           SplatElementsAttr::get(vectorTy,
                                  APInt::getAllOnes(componentBitwidth)));
@@ -653,7 +653,7 @@ struct ExtSIPattern final : public OpConversionPattern<arith::ExtSIOp> {
       // First shift left to sequeeze out all leading bits beyond the original
       // bitwidth. Here we need to use the original source and result type's
       // bitwidth.
-      auto shiftLOp = rewriter.create<spirv::ShiftLeftLogicalOp>(
+      auto shiftLOp = spirv::ShiftLeftLogicalOp::create(rewriter,
           op.getLoc(), dstType, adaptor.getIn(), shiftSize);
 
       // Then we perform arithmetic right shift to make sure we have the right
@@ -757,9 +757,9 @@ struct TruncII1Pattern final : public OpConversionPattern<arith::TruncIOp> {
     auto srcType = adaptor.getOperands().front().getType();
     // Check if (x & 1) == 1.
     Value mask = spirv::ConstantOp::getOne(srcType, loc, rewriter);
-    Value maskedSrc = rewriter.create<spirv::BitwiseAndOp>(
+    Value maskedSrc = spirv::BitwiseAndOp::create(rewriter,
         loc, srcType, adaptor.getOperands()[0], mask);
-    Value isOne = rewriter.create<spirv::IEqualOp>(loc, maskedSrc, mask);
+    Value isOne = spirv::IEqualOp::create(rewriter, loc, maskedSrc, mask);
 
     Value zero = spirv::ConstantOp::getZero(dstType, loc, rewriter);
     Value one = spirv::ConstantOp::getOne(dstType, loc, rewriter);
@@ -914,9 +914,9 @@ class CmpIOpBooleanPattern final : public OpConversionPattern<arith::CmpIOp> {
       if (auto vectorType = dyn_cast<VectorType>(dstType))
         type = VectorType::get(vectorType.getShape(), type);
       Value extLhs =
-          rewriter.create<arith::ExtUIOp>(op.getLoc(), type, adaptor.getLhs());
+          arith::ExtUIOp::create(rewriter, op.getLoc(), type, adaptor.getLhs());
       Value extRhs =
-          rewriter.create<arith::ExtUIOp>(op.getLoc(), type, adaptor.getRhs());
+          arith::ExtUIOp::create(rewriter, op.getLoc(), type, adaptor.getRhs());
 
       rewriter.replaceOpWithNewOp<arith::CmpIOp>(op, op.getPredicate(), extLhs,
                                                  extRhs);
@@ -1067,12 +1067,12 @@ class CmpFOpNanNonePattern final : public OpConversionPattern<arith::CmpFOp> {
         replace = spirv::ConstantOp::getZero(op.getType(), loc, rewriter);
       }
     } else {
-      Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-      Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+      Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+      Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-      replace = rewriter.create<spirv::LogicalOrOp>(loc, lhsIsNan, rhsIsNan);
+      replace = spirv::LogicalOrOp::create(rewriter, loc, lhsIsNan, rhsIsNan);
       if (op.getPredicate() == arith::CmpFPredicate::ORD)
-        replace = rewriter.create<spirv::LogicalNotOp>(loc, replace);
+        replace = spirv::LogicalNotOp::create(rewriter, loc, replace);
     }
 
     rewriter.replaceOp(op, replace);
@@ -1094,17 +1094,17 @@ class AddUIExtendedOpPattern final
                   ConversionPatternRewriter &rewriter) const override {
     Type dstElemTy = adaptor.getLhs().getType();
     Location loc = op->getLoc();
-    Value result = rewriter.create<spirv::IAddCarryOp>(loc, adaptor.getLhs(),
+    Value result = spirv::IAddCarryOp::create(rewriter, loc, adaptor.getLhs(),
                                                        adaptor.getRhs());
 
-    Value sumResult = rewriter.create<spirv::CompositeExtractOp>(
+    Value sumResult = spirv::CompositeExtractOp::create(rewriter,
         loc, result, llvm::ArrayRef(0));
-    Value carryValue = rewriter.create<spirv::CompositeExtractOp>(
+    Value carryValue = spirv::CompositeExtractOp::create(rewriter,
         loc, result, llvm::ArrayRef(1));
 
     // Convert the carry value to boolean.
     Value one = spirv::ConstantOp::getOne(dstElemTy, loc, rewriter);
-    Value carryResult = rewriter.create<spirv::IEqualOp>(loc, carryValue, one);
+    Value carryResult = spirv::IEqualOp::create(rewriter, loc, carryValue, one);
 
     rewriter.replaceOp(op, {sumResult, carryResult});
     return success();
@@ -1125,11 +1125,11 @@ class MulIExtendedOpPattern final : public OpConversionPattern<ArithMulOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op->getLoc();
     Value result =
-        rewriter.create<SPIRVMulOp>(loc, adaptor.getLhs(), adaptor.getRhs());
+        SPIRVMulOp::create(rewriter, loc, adaptor.getLhs(), adaptor.getRhs());
 
-    Value low = rewriter.create<spirv::CompositeExtractOp>(loc, result,
+    Value low = spirv::CompositeExtractOp::create(rewriter, loc, result,
                                                            llvm::ArrayRef(0));
-    Value high = rewriter.create<spirv::CompositeExtractOp>(loc, result,
+    Value high = spirv::CompositeExtractOp::create(rewriter, loc, result,
                                                             llvm::ArrayRef(1));
 
     rewriter.replaceOp(op, {low, high});
@@ -1183,19 +1183,19 @@ class MinimumMaximumFOpPattern final : public OpConversionPattern<Op> {
 
     Location loc = op.getLoc();
     Value spirvOp =
-        rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
+        SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
 
     if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
       rewriter.replaceOp(op, spirvOp);
       return success();
     }
 
-    Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-    Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+    Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+    Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-    Value select1 = rewriter.create<spirv::SelectOp>(loc, dstType, lhsIsNan,
+    Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
                                                      adaptor.getLhs(), spirvOp);
-    Value select2 = rewriter.create<spirv::SelectOp>(loc, dstType, rhsIsNan,
+    Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
                                                      adaptor.getRhs(), select1);
 
     rewriter.replaceOp(op, select2);
@@ -1237,7 +1237,7 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
 
     Location loc = op.getLoc();
     Value spirvOp =
-        rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
+        SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
 
     if (!shouldInsertNanGuards<SPIRVOp>() ||
         bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
@@ -1245,12 +1245,12 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
       return success();
     }
 
-    Value lhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getLhs());
-    Value rhsIsNan = rewriter.create<spirv::IsNanOp>(loc, adaptor.getRhs());
+    Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
+    Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
-    Value select1 = rewriter.create<spirv::SelectOp>(loc, dstType, lhsIsNan,
+    Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
                                                      adaptor.getRhs(), spirvOp);
-    Value select2 = rewriter.create<spirv::SelectOp>(loc, dstType, rhsIsNan,
+    Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
                                                      adaptor.getLhs(), select1);
 
     rewriter.replaceOp(op, select2);
diff --git a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
index 9c6de938a7108..2295d661d44a5 100644
--- a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
+++ b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
@@ -41,10 +41,10 @@ class Sdot2dLoweringPattern : public OpRewritePattern<Sdot2dOp> {
     Value c2d = op.getC();
     Location loc = op.getLoc();
     Value b1d =
-        rewriter.create<vector::ShapeCastOp>(loc, flattenedVectorType, b2d);
+        vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, b2d);
     Value c1d =
-        rewriter.create<vector::ShapeCastOp>(loc, flattenedVectorType, c2d);
-    Value newOp = rewriter.create<SdotOp>(loc, op.getRes().getType(), op.getA(),
+        vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, c2d);
+    Value newOp = SdotOp::create(rewriter, loc, op.getRes().getType(), op.getA(),
                                           b1d, c1d);
     rewriter.replaceOp(op, {newOp});
     return success();
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 21ea444e31821..cf46e0c10df09 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -45,37 +45,37 @@ static Operation *createLoadTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_ld1b_horiz>(
+      return arm_sme::aarch64_sme_ld1b_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_ld1h_horiz>(
+      return arm_sme::aarch64_sme_ld1h_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_ld1w_horiz>(
+      return arm_sme::aarch64_sme_ld1w_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_ld1d_horiz>(
+      return arm_sme::aarch64_sme_ld1d_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_ld1q_horiz>(
+      return arm_sme::aarch64_sme_ld1q_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_ld1b_vert>(
+      return arm_sme::aarch64_sme_ld1b_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_ld1h_vert>(
+      return arm_sme::aarch64_sme_ld1h_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_ld1w_vert>(
+      return arm_sme::aarch64_sme_ld1w_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_ld1d_vert>(
+      return arm_sme::aarch64_sme_ld1d_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_ld1q_vert>(
+      return arm_sme::aarch64_sme_ld1q_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
       break;
     }
@@ -91,37 +91,37 @@ static Operation *createStoreTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_st1b_horiz>(
+      return arm_sme::aarch64_sme_st1b_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_st1h_horiz>(
+      return arm_sme::aarch64_sme_st1h_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_st1w_horiz>(
+      return arm_sme::aarch64_sme_st1w_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_st1d_horiz>(
+      return arm_sme::aarch64_sme_st1d_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_st1q_horiz>(
+      return arm_sme::aarch64_sme_st1q_horiz::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return rewriter.create<arm_sme::aarch64_sme_st1b_vert>(
+      return arm_sme::aarch64_sme_st1b_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return rewriter.create<arm_sme::aarch64_sme_st1h_vert>(
+      return arm_sme::aarch64_sme_st1h_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return rewriter.create<arm_sme::aarch64_sme_st1w_vert>(
+      return arm_sme::aarch64_sme_st1w_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return rewriter.create<arm_sme::aarch64_sme_st1d_vert>(
+      return arm_sme::aarch64_sme_st1d_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return rewriter.create<arm_sme::aarch64_sme_st1q_vert>(
+      return arm_sme::aarch64_sme_st1q_vert::create(rewriter,
           loc, maskOp, ptr, tileId, tileSliceI32);
     }
   }
@@ -146,15 +146,15 @@ createAllocaForTile(RewriterBase &rewriter, Location loc,
   // Move to the first operation in the function.
   rewriter.setInsertionPointToStart(&func.getBlocks().front());
   // Create an alloca matching the tile size of the `tileOp`.
-  auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
+  auto vscale = vector::VectorScaleOp::create(rewriter, loc);
   auto tileElementType = tileOp.getTileType().getElementType();
   auto memrefType = MemRefType::get(
       {ShapedType::kDynamic, ShapedType::kDynamic}, tileElementType);
   unsigned minElements = arm_sme::getSMETileSliceMinNumElts(tileElementType);
   auto minElementsOp =
-      rewriter.create<arith::ConstantIndexOp>(loc, minElements);
-  auto vectorLen = rewriter.create<arith::MulIOp>(loc, vscale, minElementsOp);
-  auto alloca = rewriter.create<memref::AllocaOp>(
+      arith::ConstantIndexOp::create(rewriter, loc, minElements);
+  auto vectorLen = arith::MulIOp::create(rewriter, loc, vscale, minElementsOp);
+  auto alloca = memref::AllocaOp::create(rewriter,
       loc, memrefType, ValueRange{vectorLen, vectorLen});
   return alloca;
 }
@@ -293,9 +293,9 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                                 Value tileMemory, Value sliceIndex) const {
     auto llvmType = getTypeConverter()->convertType(tileMemory.getType());
     auto descriptor =
-        rewriter.create<UnrealizedConversionCastOp>(loc, llvmType, tileMemory);
-    auto zero = rewriter.create<arith::ConstantIntOp>(loc, 0, /*width=*/64);
-    auto sliceIndexI64 = rewriter.create<arith::IndexCastOp>(
+        UnrealizedConversionCastOp::create(rewriter, loc, llvmType, tileMemory);
+    auto zero = arith::ConstantIntOp::create(rewriter, loc, 0, /*width=*/64);
+    auto sliceIndexI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), sliceIndex);
     return getStridedElementPtr(
         static_cast<ConversionPatternRewriter &>(rewriter), loc,
@@ -309,27 +309,27 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                      arm_sme::ArmSMETileType tileType, VectorType sliceType,
                      IntegerAttr tileId, Value sliceIndex) const {
     // Cast the slice index to an i32.
-    auto sliceIndexI32 = rewriter.create<arith::IndexCastOp>(
+    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), sliceIndex);
     // Create an all-true predicate for the slice.
     auto predicateType = sliceType.clone(rewriter.getI1Type());
-    auto allTruePredicate = rewriter.create<arith::ConstantOp>(
+    auto allTruePredicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
     // Create padding vector (never used due to all-true predicate).
-    auto padVector = rewriter.create<LLVM::PoisonOp>(loc, sliceType);
+    auto padVector = LLVM::PoisonOp::create(rewriter, loc, sliceType);
     // Get a pointer to the current slice.
     auto slicePtr =
         getInMemoryTileSlicePtr(rewriter, loc, tileAlloca, sliceIndex);
     // Read the value of the current slice from ZA.
-    auto currentTileSlice = rewriter.create<arm_sme::aarch64_sme_read_horiz>(
+    auto currentTileSlice = arm_sme::aarch64_sme_read_horiz::create(rewriter,
         loc, sliceType, padVector, allTruePredicate, tileId, sliceIndexI32);
     // Load the new tile slice back from memory into ZA.
     createLoadTileSliceIntrinsic(
         rewriter, loc, tileType, arm_sme::TileSliceLayout::Horizontal,
         allTruePredicate, slicePtr, tileId, sliceIndexI32);
     // Store the current tile slice to memory.
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    rewriter.create<vector::StoreOp>(loc, currentTileSlice, tileAlloca,
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    vector::StoreOp::create(rewriter, loc, currentTileSlice, tileAlloca,
                                      ValueRange{sliceIndex, zero});
   }
 
@@ -341,12 +341,12 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     RewriterBase::InsertionGuard guard(rewriter);
     // Create an scf.for over all tile slices.
     auto minNumElts =
-        rewriter.create<arith::ConstantIndexOp>(loc, sliceType.getDimSize(0));
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto upperBound = rewriter.create<arith::MulIOp>(
-        loc, minNumElts, rewriter.create<vector::VectorScaleOp>(loc));
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        arith::ConstantIndexOp::create(rewriter, loc, sliceType.getDimSize(0));
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto upperBound = arith::MulIOp::create(rewriter,
+        loc, minNumElts, vector::VectorScaleOp::create(rewriter, loc));
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     // Emit a swap for each tile slice.
     rewriter.setInsertionPointToStart(forOp.getBody());
     auto sliceIndex = forOp.getInductionVar();
@@ -479,7 +479,7 @@ struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
     //
     // This holds for all tile sizes.
     int32_t zeroMask = baseMaskForSize << int32_t(tileId.getInt());
-    rewriter.create<arm_sme::aarch64_sme_zero>(
+    arm_sme::aarch64_sme_zero::create(rewriter,
         loc, rewriter.getI32IntegerAttr(zeroMask));
 
     // Create a placeholder op to preserve dataflow.
@@ -513,7 +513,7 @@ struct LoadTileSliceConversion
     auto tileSlice = loadTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
@@ -559,7 +559,7 @@ struct StoreTileSliceConversion
     auto tileSlice = storeTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     auto maskOp = storeTileSliceOp.getMask();
@@ -595,26 +595,26 @@ struct InsertTileSliceConversion
     auto tileSlice = insertTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto tileSliceI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
-    auto one = rewriter.create<arith::ConstantOp>(
+    auto one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI1Type(),
         rewriter.getIntegerAttr(rewriter.getI1Type(), 1));
     auto predTy = VectorType::get(tileType.getShape()[0], rewriter.getI1Type(),
                                   /*scalableDims=*/{true});
-    auto allActiveMask = rewriter.create<vector::SplatOp>(loc, predTy, one);
+    auto allActiveMask = vector::SplatOp::create(rewriter, loc, predTy, one);
 
     // Create 'arm_sme.intr.write.(horiz|vert)' to write vector to tile slice.
     switch (insertTileSliceOp.getLayout()) {
     case arm_sme::TileSliceLayout::Horizontal:
-      rewriter.create<arm_sme::aarch64_sme_write_horiz>(
+      arm_sme::aarch64_sme_write_horiz::create(rewriter,
           loc, tileId, tileSliceI32, allActiveMask,
           insertTileSliceOp.getVector());
       break;
     case arm_sme::TileSliceLayout::Vertical:
-      rewriter.create<arm_sme::aarch64_sme_write_vert>(
+      arm_sme::aarch64_sme_write_vert::create(rewriter,
           loc, tileId, tileSliceI32, allActiveMask,
           insertTileSliceOp.getVector());
       break;
@@ -646,15 +646,15 @@ struct ExtractTileSliceConversion
 
     // Create an 'all true' predicate for the tile slice.
     auto predicateType = sliceType.cloneWith({}, rewriter.getI1Type());
-    auto allTruePredicate = rewriter.create<arith::ConstantOp>(
+    auto allTruePredicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
 
     // Zero destination/fallback for tile slice extraction.
-    auto zeroVector = rewriter.create<arith::ConstantOp>(
+    auto zeroVector = arith::ConstantOp::create(rewriter,
         loc, sliceType, rewriter.getZeroAttr(sliceType));
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto sliceIndexI32 = rewriter.create<arith::IndexCastOp>(
+    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), sliceIndex);
 
     // Create 'arm_sme.intr.read.(horiz|vert)' to extract the tile slice.
@@ -743,7 +743,7 @@ struct OuterProductOpConversion
     Value acc = outerProductOp.getAcc();
     if (!acc) {
       // Initalize accumulator with zero.
-      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, resultVectorType);
+      auto zero = arm_sme::ZeroOp::create(rewriter, loc, resultVectorType);
       zero.setTileId(tileId);
       acc = zero;
     }
@@ -754,14 +754,14 @@ struct OuterProductOpConversion
     if (!lhsMask || !rhsMask) {
       auto predTy =
           outerProductOp.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = rewriter.create<arith::ConstantOp>(
+      Value allActiveMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
     // Create 'arm_sme.intr.mopa' outer product intrinsic.
-    rewriter.create<arm_sme::aarch64_sme_mopa>(loc, tileId, lhsMask, rhsMask,
+    arm_sme::aarch64_sme_mopa::create(rewriter, loc, tileId, lhsMask, rhsMask,
                                                outerProductOp.getLhs(),
                                                outerProductOp.getRhs());
 
@@ -792,7 +792,7 @@ struct OuterProductWideningOpConversion
     Value acc = op.getAcc();
     if (!acc) {
       // Initalize accumulator with zero.
-      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, op.getResultType());
+      auto zero = arm_sme::ZeroOp::create(rewriter, loc, op.getResultType());
       zero.setTileId(tileId);
       acc = zero;
     }
@@ -801,13 +801,13 @@ struct OuterProductWideningOpConversion
     Value rhsMask = op.getRhsMask();
     if (!lhsMask || !rhsMask) {
       auto predTy = op.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = rewriter.create<arith::ConstantOp>(
+      Value allActiveMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
-    rewriter.create<OuterProductWideningIntrOp>(
+    OuterProductWideningIntrOp::create(rewriter,
         loc, tileId, lhsMask, rhsMask, adaptor.getLhs(), adaptor.getRhs());
 
     // The outerproduct intrinsics have no result, replace
@@ -843,13 +843,13 @@ struct StreamingVLOpConversion
     auto *intrOp = [&]() -> Operation * {
       switch (streamingVlOp.getTypeSize()) {
       case arm_sme::TypeSize::Byte:
-        return rewriter.create<arm_sme::aarch64_sme_cntsb>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsb::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Half:
-        return rewriter.create<arm_sme::aarch64_sme_cntsh>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsh::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Word:
-        return rewriter.create<arm_sme::aarch64_sme_cntsw>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsw::create(rewriter, loc, i64Type);
       case arm_sme::TypeSize::Double:
-        return rewriter.create<arm_sme::aarch64_sme_cntsd>(loc, i64Type);
+        return arm_sme::aarch64_sme_cntsd::create(rewriter, loc, i64Type);
       }
       llvm_unreachable("unknown type size in StreamingVLOpConversion");
     }();
@@ -872,7 +872,7 @@ static void mergeConsecutiveTileZerosInBlock(Block *block) {
     if (zeroOpsToMerge.size() <= 1)
       return;
     IRRewriter rewriter(zeroOpsToMerge.front());
-    rewriter.create<arm_sme::aarch64_sme_zero>(
+    arm_sme::aarch64_sme_zero::create(rewriter,
         zeroOpsToMerge.front().getLoc(),
         rewriter.getI32IntegerAttr(mergedZeroMask));
     for (auto zeroOp : zeroOpsToMerge)
diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 458628c29c6ac..3591ed65f8d84 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -39,7 +39,7 @@ SmallVector<Value, 2> getMemrefIndices(ValueRange indices, unsigned rank,
   auto tileSliceOffset = tileSliceIndex;
 
   auto baseIndexPlusTileSliceOffset =
-      rewriter.create<arith::AddIOp>(loc, indices[0], tileSliceOffset);
+      arith::AddIOp::create(rewriter, loc, indices[0], tileSliceOffset);
   outIndices.push_back(baseIndexPlusTileSliceOffset);
   outIndices.push_back(indices[1]);
 
@@ -59,10 +59,10 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   if (memrefIndices.size() != 2)
     return rewriter.notifyMatchFailure(loc, "invalid number of indices");
 
-  auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
       loc, arm_sme::getSMETileSliceMinNumElts(tileType.getElementType()));
   auto vscale =
-      rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
+      vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
   auto predicateType =
       VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
 
@@ -70,7 +70,7 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   // elements in a vector of SVL bits for a given element type (SVL_B,
   // SVL_H, ..., SVL_Q).
   auto numTileSlices =
-      rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
+      arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
 
   Value predicate;
   Value upperBound;
@@ -82,28 +82,28 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
     // The upper bound of the loop must be clamped at `numTileSlices` as
     // `vector.create_mask` allows operands to be greater than the size of a
     // dimension.
-    auto numRowI64 = rewriter.create<arith::IndexCastOp>(
+    auto numRowI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), maskDim0);
-    auto numTileSlicesI64 = rewriter.create<arith::IndexCastOp>(
+    auto numTileSlicesI64 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(), numTileSlices);
     auto upperBoundI64 =
-        rewriter.create<arith::MinSIOp>(loc, numRowI64, numTileSlicesI64);
-    upperBound = rewriter.create<arith::IndexCastOp>(
+        arith::MinSIOp::create(rewriter, loc, numRowI64, numTileSlicesI64);
+    upperBound = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getIndexType(), upperBoundI64);
 
     predicate =
-        rewriter.create<vector::CreateMaskOp>(loc, predicateType, maskDim1);
+        vector::CreateMaskOp::create(rewriter, loc, predicateType, maskDim1);
   } else {
     upperBound = numTileSlices;
     // No mask. Create an 'all true' predicate for the tile slice.
-    predicate = rewriter.create<arith::ConstantOp>(
+    predicate = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(predicateType, true));
   }
 
   bool hasCarriedArgs = bool(initTile);
-  auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-  auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+  auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
                                            hasCarriedArgs ? ValueRange{initTile}
                                                           : ValueRange{});
 
@@ -118,7 +118,7 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
 
   assert(bool(nextTile) == hasCarriedArgs);
   if (nextTile)
-    rewriter.create<scf::YieldOp>(loc, nextTile);
+    scf::YieldOp::create(rewriter, loc, nextTile);
 
   return forOp;
 }
@@ -194,9 +194,9 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
       // Initialize tile with zero to satisfy padding. Inactive cols will be
       // zeroed anyway since the loads use zeroing predication. For inactive
       // rows however, no load will occur so these need to be zeroed.
-      initTile = rewriter.create<arm_sme::ZeroOp>(loc, tileType);
+      initTile = arm_sme::ZeroOp::create(rewriter, loc, tileType);
     } else {
-      initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+      initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
     }
 
     // Create a loop to load the active tile slices from memory.
@@ -207,7 +207,7 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
             Value currentTile) -> Value {
           // Create 'arm_sme.load_tile_slice' to load tile slice from memory
           // into tile.
-          return rewriter.create<arm_sme::LoadTileSliceOp>(
+          return arm_sme::LoadTileSliceOp::create(rewriter,
               loc, tileType, tileLoadOp.getBase(), predicate, currentTile,
               memrefIndices, tileSliceIndex, tileLoadOp.getLayout());
         });
@@ -283,21 +283,21 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto numRows = createMaskOp.getOperands()[0];
     auto numCols = createMaskOp.getOperands()[1];
 
-    auto numColsI32 = rewriter.create<arith::IndexCastUIOp>(
+    auto numColsI32 = arith::IndexCastUIOp::create(rewriter,
         loc, rewriter.getI32Type(), numCols);
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     // Create a loop that loads each ZA tile slice from memory.
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
         loc, arm_sme::getSMETileSliceMinNumElts(tileElementType));
     auto vscale =
-        rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+        vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto numTileSlices =
-        rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, numTileSlices,
+        arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices,
                                              step, ValueRange{initTile});
 
     rewriter.setInsertionPointToStart(forOp.getBody());
@@ -306,16 +306,16 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto currentTile = forOp.getRegionIterArg(0);
 
     // Combine masks.
-    auto rowIsActive = rewriter.create<arith::CmpIOp>(
+    auto rowIsActive = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, tileSliceIndex, numRows);
-    auto rowIsActiveI32 = rewriter.create<arith::ExtSIOp>(
+    auto rowIsActiveI32 = arith::ExtSIOp::create(rewriter,
         loc, rewriter.getI32Type(), rowIsActive);
-    auto mask = rewriter.create<arith::AndIOp>(loc, rowIsActiveI32, numColsI32);
+    auto mask = arith::AndIOp::create(rewriter, loc, rowIsActiveI32, numColsI32);
     auto maskIndex =
-        rewriter.create<arith::IndexCastOp>(loc, rewriter.getIndexType(), mask);
+        arith::IndexCastOp::create(rewriter, loc, rewriter.getIndexType(), mask);
     auto predicateType =
         VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
-    auto maskOp1D = rewriter.create<vector::CreateMaskOp>(
+    auto maskOp1D = vector::CreateMaskOp::create(rewriter,
         loc, predicateType, maskIndex.getResult());
 
     auto memrefIndices = getMemrefIndices(
@@ -324,17 +324,17 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
 
     // Splat pad into 1-D vector matching type of tile slice.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    auto pad1DOp = rewriter.create<vector::SplatOp>(loc, tileSliceType, padOp);
+    auto pad1DOp = vector::SplatOp::create(rewriter, loc, tileSliceType, padOp);
 
-    auto loadSlice = rewriter.create<vector::MaskedLoadOp>(
+    auto loadSlice = vector::MaskedLoadOp::create(rewriter,
         loc, tileSliceType, tileLoadOp.getBase(), memrefIndices, maskOp1D,
         /*passthru=*/pad1DOp);
 
     // Create 'arm_sme.insert_tile_slice' to insert slice into tile.
-    auto insertSlice = rewriter.create<arm_sme::InsertTileSliceOp>(
+    auto insertSlice = arm_sme::InsertTileSliceOp::create(rewriter,
         loc, tileType, loadSlice->getResult(0), currentTile, tileSliceIndex,
         tileLoadOp.getLayout());
-    rewriter.create<scf::YieldOp>(loc, insertSlice.getResult());
+    scf::YieldOp::create(rewriter, loc, insertSlice.getResult());
 
     rewriter.setInsertionPointAfter(forOp);
 
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index 94f7caa315cf7..e1d28f19ee236 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -203,7 +203,7 @@ static void addAsyncRuntimeApiDeclarations(ModuleOp module) {
   auto addFuncDecl = [&](StringRef name, FunctionType type) {
     if (module.lookupSymbol(name))
       return;
-    builder.create<func::FuncOp>(name, type).setPrivate();
+    func::FuncOp::create(builder, name, type).setPrivate();
   };
 
   MLIRContext *ctx = module.getContext();
@@ -254,15 +254,15 @@ static void addResumeFunction(ModuleOp module) {
   auto voidTy = LLVM::LLVMVoidType::get(ctx);
   Type ptrType = AsyncAPI::opaquePointerType(ctx);
 
-  auto resumeOp = moduleBuilder.create<LLVM::LLVMFuncOp>(
+  auto resumeOp = LLVM::LLVMFuncOp::create(moduleBuilder,
       kResume, LLVM::LLVMFunctionType::get(voidTy, {ptrType}));
   resumeOp.setPrivate();
 
   auto *block = resumeOp.addEntryBlock(moduleBuilder);
   auto blockBuilder = ImplicitLocOpBuilder::atBlockEnd(loc, block);
 
-  blockBuilder.create<LLVM::CoroResumeOp>(resumeOp.getArgument(0));
-  blockBuilder.create<LLVM::ReturnOp>(ValueRange());
+  LLVM::CoroResumeOp::create(blockBuilder, resumeOp.getArgument(0));
+  LLVM::ReturnOp::create(blockBuilder, ValueRange());
 }
 
 //===----------------------------------------------------------------------===//
@@ -282,7 +282,7 @@ class AsyncRuntimeTypeConverter : public TypeConverter {
     // in patterns for other dialects.
     auto addUnrealizedCast = [](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+      auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
       return cast.getResult(0);
     };
 
@@ -343,8 +343,8 @@ class CoroIdOpConversion : public AsyncOpConversionPattern<CoroIdOp> {
 
     // Constants for initializing coroutine frame.
     auto constZero =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), 0);
-    auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, ptrType);
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), 0);
+    auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, ptrType);
 
     // Get coroutine id: @llvm.coro.id.
     rewriter.replaceOpWithNewOp<LLVM::CoroIdOp>(
@@ -372,32 +372,32 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
 
     // Get coroutine frame size: @llvm.coro.size.i64.
     Value coroSize =
-        rewriter.create<LLVM::CoroSizeOp>(loc, rewriter.getI64Type());
+        LLVM::CoroSizeOp::create(rewriter, loc, rewriter.getI64Type());
     // Get coroutine frame alignment: @llvm.coro.align.i64.
     Value coroAlign =
-        rewriter.create<LLVM::CoroAlignOp>(loc, rewriter.getI64Type());
+        LLVM::CoroAlignOp::create(rewriter, loc, rewriter.getI64Type());
 
     // Round up the size to be multiple of the alignment. Since aligned_alloc
     // requires the size parameter be an integral multiple of the alignment
     // parameter.
     auto makeConstant = [&](uint64_t c) {
-      return rewriter.create<LLVM::ConstantOp>(op->getLoc(),
+      return LLVM::ConstantOp::create(rewriter, op->getLoc(),
                                                rewriter.getI64Type(), c);
     };
-    coroSize = rewriter.create<LLVM::AddOp>(op->getLoc(), coroSize, coroAlign);
+    coroSize = LLVM::AddOp::create(rewriter, op->getLoc(), coroSize, coroAlign);
     coroSize =
-        rewriter.create<LLVM::SubOp>(op->getLoc(), coroSize, makeConstant(1));
+        LLVM::SubOp::create(rewriter, op->getLoc(), coroSize, makeConstant(1));
     Value negCoroAlign =
-        rewriter.create<LLVM::SubOp>(op->getLoc(), makeConstant(0), coroAlign);
+        LLVM::SubOp::create(rewriter, op->getLoc(), makeConstant(0), coroAlign);
     coroSize =
-        rewriter.create<LLVM::AndOp>(op->getLoc(), coroSize, negCoroAlign);
+        LLVM::AndOp::create(rewriter, op->getLoc(), coroSize, negCoroAlign);
 
     // Allocate memory for the coroutine frame.
     auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn(
         rewriter, op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
     if (failed(allocFuncOp))
       return failure();
-    auto coroAlloc = rewriter.create<LLVM::CallOp>(
+    auto coroAlloc = LLVM::CallOp::create(rewriter,
         loc, allocFuncOp.value(), ValueRange{coroAlign, coroSize});
 
     // Begin a coroutine: @llvm.coro.begin.
@@ -427,7 +427,7 @@ class CoroFreeOpConversion : public AsyncOpConversionPattern<CoroFreeOp> {
 
     // Get a pointer to the coroutine frame memory: @llvm.coro.free.
     auto coroMem =
-        rewriter.create<LLVM::CoroFreeOp>(loc, ptrType, adaptor.getOperands());
+        LLVM::CoroFreeOp::create(rewriter, loc, ptrType, adaptor.getOperands());
 
     // Free the memory.
     auto freeFuncOp =
@@ -455,13 +455,13 @@ class CoroEndOpConversion : public OpConversionPattern<CoroEndOp> {
   matchAndRewrite(CoroEndOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // We are not in the block that is part of the unwind sequence.
-    auto constFalse = rewriter.create<LLVM::ConstantOp>(
+    auto constFalse = LLVM::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getI1Type(), rewriter.getBoolAttr(false));
-    auto noneToken = rewriter.create<LLVM::NoneTokenOp>(op->getLoc());
+    auto noneToken = LLVM::NoneTokenOp::create(rewriter, op->getLoc());
 
     // Mark the end of a coroutine: @llvm.coro.end.
     auto coroHdl = adaptor.getHandle();
-    rewriter.create<LLVM::CoroEndOp>(
+    LLVM::CoroEndOp::create(rewriter,
         op->getLoc(), rewriter.getI1Type(),
         ValueRange({coroHdl, constFalse, noneToken}));
     rewriter.eraseOp(op);
@@ -534,12 +534,12 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     auto loc = op->getLoc();
 
     // This is not a final suspension point.
-    auto constFalse = rewriter.create<LLVM::ConstantOp>(
+    auto constFalse = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI1Type(), rewriter.getBoolAttr(false));
 
     // Suspend a coroutine: @llvm.coro.suspend
     auto coroState = adaptor.getState();
-    auto coroSuspend = rewriter.create<LLVM::CoroSuspendOp>(
+    auto coroSuspend = LLVM::CoroSuspendOp::create(rewriter,
         loc, i8, ValueRange({coroState, constFalse}));
 
     // Cast return code to i32.
@@ -551,7 +551,7 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     llvm::SmallVector<Block *, 2> caseDest = {op.getResumeDest(),
                                               op.getCleanupDest()};
     rewriter.replaceOpWithNewOp<LLVM::SwitchOp>(
-        op, rewriter.create<LLVM::SExtOp>(loc, i32, coroSuspend.getResult()),
+        op, LLVM::SExtOp::create(rewriter, loc, i32, coroSuspend.getResult()),
         /*defaultDestination=*/op.getSuspendDest(),
         /*defaultOperands=*/ValueRange(),
         /*caseValues=*/caseValues,
@@ -602,11 +602,11 @@ class RuntimeCreateOpLowering : public ConvertOpToLLVMPattern<RuntimeCreateOp> {
 
         // %Size = getelementptr %T* null, int 1
         // %SizeI = ptrtoint %T* %Size to i64
-        auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, storagePtrType);
+        auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, storagePtrType);
         auto gep =
-            rewriter.create<LLVM::GEPOp>(loc, storagePtrType, storedType,
+            LLVM::GEPOp::create(rewriter, loc, storagePtrType, storedType,
                                          nullPtr, ArrayRef<LLVM::GEPArg>{1});
-        return rewriter.create<LLVM::PtrToIntOp>(loc, i64, gep);
+        return LLVM::PtrToIntOp::create(rewriter, loc, i64, gep);
       };
 
       rewriter.replaceOpWithNewOp<func::CallOp>(op, kCreateValue, resultType,
@@ -739,7 +739,7 @@ class RuntimeAwaitOpLowering : public OpConversionPattern<RuntimeAwaitOp> {
             .Case<ValueType>([](Type) { return kAwaitValue; })
             .Case<GroupType>([](Type) { return kAwaitGroup; });
 
-    rewriter.create<func::CallOp>(op->getLoc(), apiFuncName, TypeRange(),
+    func::CallOp::create(rewriter, op->getLoc(), apiFuncName, TypeRange(),
                                   adaptor.getOperands());
     rewriter.eraseOp(op);
 
@@ -772,11 +772,11 @@ class RuntimeAwaitAndResumeOpLowering
 
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = rewriter.create<LLVM::AddressOfOp>(
+    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
         op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
         kResume);
 
-    rewriter.create<func::CallOp>(
+    func::CallOp::create(rewriter,
         op->getLoc(), apiFuncName, TypeRange(),
         ValueRange({operand, handle, resumePtr.getRes()}));
     rewriter.eraseOp(op);
@@ -801,7 +801,7 @@ class RuntimeResumeOpLowering
                   ConversionPatternRewriter &rewriter) const override {
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = rewriter.create<LLVM::AddressOfOp>(
+    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
         op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
         kResume);
 
@@ -832,7 +832,7 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = rewriter.create<func::CallOp>(
+    auto storagePtr = func::CallOp::create(rewriter,
         loc, kGetValueStorage, TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
@@ -845,7 +845,7 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     Value castedStoragePtr = storagePtr.getResult(0);
     // Store the yielded value into the async value storage.
     auto value = adaptor.getValue();
-    rewriter.create<LLVM::StoreOp>(loc, value, castedStoragePtr);
+    LLVM::StoreOp::create(rewriter, loc, value, castedStoragePtr);
 
     // Erase the original runtime store operation.
     rewriter.eraseOp(op);
@@ -872,7 +872,7 @@ class RuntimeLoadOpLowering : public ConvertOpToLLVMPattern<RuntimeLoadOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = rewriter.create<func::CallOp>(
+    auto storagePtr = func::CallOp::create(rewriter,
         loc, kGetValueStorage, TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
@@ -960,7 +960,7 @@ class RefCountingOpLowering : public OpConversionPattern<RefCountingOp> {
   LogicalResult
   matchAndRewrite(RefCountingOp op, typename RefCountingOp::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto count = rewriter.create<arith::ConstantOp>(
+    auto count = arith::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getI64Type(),
         rewriter.getI64IntegerAttr(op.getCount()));
 
diff --git a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
index b9991f36cdaaf..a63a1ceb00ebb 100644
--- a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
+++ b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
@@ -47,26 +47,26 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
 
     if (auto unrankedType = dyn_cast<UnrankedMemRefType>(type)) {
       // Constants
-      Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-      Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+      Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+      Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
       // Dynamically evaluate the size and shape of the unranked memref
-      Value rank = rewriter.create<memref::RankOp>(loc, op.getInput());
+      Value rank = memref::RankOp::create(rewriter, loc, op.getInput());
       MemRefType allocType =
           MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType());
-      Value shape = rewriter.create<memref::AllocaOp>(loc, allocType, rank);
+      Value shape = memref::AllocaOp::create(rewriter, loc, allocType, rank);
 
       // Create a loop to query dimension sizes, store them as a shape, and
       // compute the total size of the memref
       auto loopBody = [&](OpBuilder &builder, Location loc, Value i,
                           ValueRange args) {
         auto acc = args.front();
-        auto dim = rewriter.create<memref::DimOp>(loc, op.getInput(), i);
+        auto dim = memref::DimOp::create(rewriter, loc, op.getInput(), i);
 
-        rewriter.create<memref::StoreOp>(loc, dim, shape, i);
-        acc = rewriter.create<arith::MulIOp>(loc, acc, dim);
+        memref::StoreOp::create(rewriter, loc, dim, shape, i);
+        acc = arith::MulIOp::create(rewriter, loc, acc, dim);
 
-        rewriter.create<scf::YieldOp>(loc, acc);
+        scf::YieldOp::create(rewriter, loc, acc);
       };
       auto size = rewriter
                       .create<scf::ForOp>(loc, zero, rank, one, ValueRange(one),
@@ -78,9 +78,9 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
 
       // Allocate new memref with 1D dynamic shape, then reshape into the
       // shape of the original unranked memref
-      alloc = rewriter.create<memref::AllocOp>(loc, memrefType, size);
+      alloc = memref::AllocOp::create(rewriter, loc, memrefType, size);
       alloc =
-          rewriter.create<memref::ReshapeOp>(loc, unrankedType, alloc, shape);
+          memref::ReshapeOp::create(rewriter, loc, unrankedType, alloc, shape);
     } else {
       MemRefType memrefType = cast<MemRefType>(type);
       MemRefLayoutAttrInterface layout;
@@ -103,14 +103,14 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
       }
 
       // Allocate a memref with identity layout.
-      alloc = rewriter.create<memref::AllocOp>(loc, allocType, dynamicOperands);
+      alloc = memref::AllocOp::create(rewriter, loc, allocType, dynamicOperands);
       // Cast the allocation to the specified type if needed.
       if (memrefType != allocType)
         alloc =
-            rewriter.create<memref::CastOp>(op->getLoc(), memrefType, alloc);
+            memref::CastOp::create(rewriter, op->getLoc(), memrefType, alloc);
     }
 
-    rewriter.create<memref::CopyOp>(loc, op.getInput(), alloc);
+    memref::CopyOp::create(rewriter, loc, op.getInput(), alloc);
     rewriter.replaceOp(op, alloc);
     return success();
   }
diff --git a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
index 70b22386f1eea..1961e1004ba8e 100644
--- a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
+++ b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
@@ -23,41 +23,41 @@ void mlir::complex::convertDivToLLVMUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, LLVM::FastmathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, rhsRe, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, rhsIm, rhsIm, fmf), fmf);
+  Value rhsSqNorm = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsIm, fmf), fmf);
+  Value realNumerator = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
 
-  Value imagNumerator = rewriter.create<LLVM::FSubOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRe, fmf),
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsIm, fmf), fmf);
+  Value imagNumerator = LLVM::FSubOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
-  *resultRe = rewriter.create<LLVM::FDivOp>(loc, realNumerator, rhsSqNorm, fmf);
-  *resultIm = rewriter.create<LLVM::FDivOp>(loc, imagNumerator, rhsSqNorm, fmf);
+  *resultRe = LLVM::FDivOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
+  *resultIm = LLVM::FDivOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 void mlir::complex::convertDivToStandardUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, arith::FastMathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, rhsRe, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, rhsIm, rhsIm, fmf), fmf);
+  Value rhsSqNorm = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsIm, fmf), fmf);
-  Value imagNumerator = rewriter.create<arith::SubFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRe, fmf),
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsIm, fmf), fmf);
+  Value realNumerator = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
+  Value imagNumerator = arith::SubFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
   *resultRe =
-      rewriter.create<arith::DivFOp>(loc, realNumerator, rhsSqNorm, fmf);
+      arith::DivFOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
   *resultIm =
-      rewriter.create<arith::DivFOp>(loc, imagNumerator, rhsSqNorm, fmf);
+      arith::DivFOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 // Smith's algorithm to divide complex numbers. It is just a bit smarter
@@ -94,180 +94,180 @@ void mlir::complex::convertDivToLLVMUsingRangeReduction(
   auto elementType = cast<FloatType>(rhsRe.getType());
 
   Value rhsRealImagRatio =
-      rewriter.create<LLVM::FDivOp>(loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = rewriter.create<LLVM::FAddOp>(
+      LLVM::FDivOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
+  Value rhsRealImagDenom = LLVM::FAddOp::create(rewriter,
       loc, rhsIm,
-      rewriter.create<LLVM::FMulOp>(loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = rewriter.create<LLVM::FAddOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRealImagRatio, fmf),
+      LLVM::FMulOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
+  Value realNumerator1 = LLVM::FAddOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
       lhsIm, fmf);
   Value resultReal1 =
-      rewriter.create<LLVM::FDivOp>(loc, realNumerator1, rhsRealImagDenom, fmf);
-  Value imagNumerator1 = rewriter.create<LLVM::FSubOp>(
-      loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRealImagRatio, fmf),
+      LLVM::FDivOp::create(rewriter, loc, realNumerator1, rhsRealImagDenom, fmf);
+  Value imagNumerator1 = LLVM::FSubOp::create(rewriter,
+      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
       lhsRe, fmf);
   Value resultImag1 =
-      rewriter.create<LLVM::FDivOp>(loc, imagNumerator1, rhsRealImagDenom, fmf);
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator1, rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
-      rewriter.create<LLVM::FDivOp>(loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = rewriter.create<LLVM::FAddOp>(
+      LLVM::FDivOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
+  Value rhsImagRealDenom = LLVM::FAddOp::create(rewriter,
       loc, rhsRe,
-      rewriter.create<LLVM::FMulOp>(loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = rewriter.create<LLVM::FAddOp>(
+      LLVM::FMulOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
+  Value realNumerator2 = LLVM::FAddOp::create(rewriter,
       loc, lhsRe,
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsImagRealRatio, fmf), fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
   Value resultReal2 =
-      rewriter.create<LLVM::FDivOp>(loc, realNumerator2, rhsImagRealDenom, fmf);
-  Value imagNumerator2 = rewriter.create<LLVM::FSubOp>(
+      LLVM::FDivOp::create(rewriter, loc, realNumerator2, rhsImagRealDenom, fmf);
+  Value imagNumerator2 = LLVM::FSubOp::create(rewriter,
       loc, lhsIm,
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsImagRealRatio, fmf), fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
   Value resultImag2 =
-      rewriter.create<LLVM::FDivOp>(loc, imagNumerator2, rhsImagRealDenom, fmf);
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator2, rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = rewriter.create<LLVM::ConstantOp>(
+  Value zero = LLVM::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
-  Value rhsRealAbs = rewriter.create<LLVM::FAbsOp>(loc, rhsRe, fmf);
-  Value rhsRealIsZero = rewriter.create<LLVM::FCmpOp>(
+  Value rhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, rhsRe, fmf);
+  Value rhsRealIsZero = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, zero);
-  Value rhsImagAbs = rewriter.create<LLVM::FAbsOp>(loc, rhsIm, fmf);
-  Value rhsImagIsZero = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, rhsIm, fmf);
+  Value rhsImagIsZero = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, zero);
   Value lhsRealIsNotNaN =
-      rewriter.create<LLVM::FCmpOp>(loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
+      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
   Value lhsImagIsNotNaN =
-      rewriter.create<LLVM::FCmpOp>(loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
+      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
   Value lhsContainsNotNaNValue =
-      rewriter.create<LLVM::OrOp>(loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = rewriter.create<LLVM::AndOp>(
+      LLVM::OrOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
+  Value resultIsInfinity = LLVM::AndOp::create(rewriter,
       loc, lhsContainsNotNaNValue,
-      rewriter.create<LLVM::AndOp>(loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = rewriter.create<LLVM::ConstantOp>(
+      LLVM::AndOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
+  Value inf = LLVM::ConstantOp::create(rewriter,
       loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfrhsReal =
-      rewriter.create<LLVM::CopySignOp>(loc, inf, rhsRe);
+      LLVM::CopySignOp::create(rewriter, loc, inf, rhsRe);
   Value infinityResultReal =
-      rewriter.create<LLVM::FMulOp>(loc, infWithSignOfrhsReal, lhsRe, fmf);
+      LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsRe, fmf);
   Value infinityResultImag =
-      rewriter.create<LLVM::FMulOp>(loc, infWithSignOfrhsReal, lhsIm, fmf);
+      LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsRealFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, rhsRealAbs, inf);
-  Value rhsImagFinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, rhsImagAbs, inf);
   Value rhsFinite =
-      rewriter.create<LLVM::AndOp>(loc, rhsRealFinite, rhsImagFinite);
-  Value lhsRealAbs = rewriter.create<LLVM::FAbsOp>(loc, lhsRe, fmf);
-  Value lhsRealInfinite = rewriter.create<LLVM::FCmpOp>(
+      LLVM::AndOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
+  Value lhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, lhsRe, fmf);
+  Value lhsRealInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, lhsRealAbs, inf);
-  Value lhsImagAbs = rewriter.create<LLVM::FAbsOp>(loc, lhsIm, fmf);
-  Value lhsImagInfinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, lhsIm, fmf);
+  Value lhsImagInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, lhsImagAbs, inf);
   Value lhsInfinite =
-      rewriter.create<LLVM::OrOp>(loc, lhsRealInfinite, lhsImagInfinite);
+      LLVM::OrOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
-      rewriter.create<LLVM::AndOp>(loc, lhsInfinite, rhsFinite);
-  Value one = rewriter.create<LLVM::ConstantOp>(
+      LLVM::AndOp::create(rewriter, loc, lhsInfinite, rhsFinite);
+  Value one = LLVM::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, lhsRealInfinite, one, zero),
+  Value lhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesrhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRealIsInfWithSign, rhsRe, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesrhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
+  Value resultReal3 = LLVM::FMulOp::create(rewriter,
       loc, inf,
-      rewriter.create<LLVM::FAddOp>(loc, lhsRealIsInfWithSignTimesrhsReal,
+      LLVM::FAddOp::create(rewriter, loc, lhsRealIsInfWithSignTimesrhsReal,
                                     lhsImagIsInfWithSignTimesrhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesrhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRealIsInfWithSign, rhsIm, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesrhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
+  Value resultImag3 = LLVM::FMulOp::create(rewriter,
       loc, inf,
-      rewriter.create<LLVM::FSubOp>(loc, lhsImagIsInfWithSignTimesrhsReal,
+      LLVM::FSubOp::create(rewriter, loc, lhsImagIsInfWithSignTimesrhsReal,
                                     lhsRealIsInfWithSignTimesrhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsRealFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, lhsRealAbs, inf);
-  Value lhsImagFinite = rewriter.create<LLVM::FCmpOp>(
+  Value lhsImagFinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::one, lhsImagAbs, inf);
   Value lhsFinite =
-      rewriter.create<LLVM::AndOp>(loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = rewriter.create<LLVM::FCmpOp>(
+      LLVM::AndOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
+  Value rhsRealInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, inf);
-  Value rhsImagInfinite = rewriter.create<LLVM::FCmpOp>(
+  Value rhsImagInfinite = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, inf);
   Value rhsInfinite =
-      rewriter.create<LLVM::OrOp>(loc, rhsRealInfinite, rhsImagInfinite);
+      LLVM::OrOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
-      rewriter.create<LLVM::AndOp>(loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, rhsRealInfinite, one, zero),
+      LLVM::AndOp::create(rewriter, loc, lhsFinite, rhsInfinite);
+  Value rhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = rewriter.create<LLVM::CopySignOp>(
-      loc, rewriter.create<LLVM::SelectOp>(loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
+      loc, LLVM::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimeslhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsRealIsInfWithSign, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
+  Value resultReal4 = LLVM::FMulOp::create(rewriter,
       loc, zero,
-      rewriter.create<LLVM::FAddOp>(loc, rhsRealIsInfWithSignTimeslhsReal,
+      LLVM::FAddOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsReal,
                                     rhsImagIsInfWithSignTimeslhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimeslhsImag =
-      rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRealIsInfWithSign, fmf);
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsReal =
-      rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = rewriter.create<LLVM::FMulOp>(
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
+  Value resultImag4 = LLVM::FMulOp::create(rewriter,
       loc, zero,
-      rewriter.create<LLVM::FSubOp>(loc, rhsRealIsInfWithSignTimeslhsImag,
+      LLVM::FSubOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsImag,
                                     rhsImagIsInfWithSignTimeslhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = rewriter.create<LLVM::FCmpOp>(
+  Value realAbsSmallerThanImagAbs = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::olt, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = rewriter.create<LLVM::SelectOp>(
+  Value resultReal5 = LLVM::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = rewriter.create<LLVM::SelectOp>(
+  Value resultImag5 = LLVM::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase3 = LLVM::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase3 = LLVM::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase2 = LLVM::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase2 = LLVM::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = rewriter.create<LLVM::SelectOp>(
+  Value resultRealSpecialCase1 = LLVM::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = rewriter.create<LLVM::SelectOp>(
+  Value resultImagSpecialCase1 = LLVM::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = rewriter.create<LLVM::FCmpOp>(
+  Value resultRealIsNaN = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::uno, resultReal5, zero);
-  Value resultImagIsNaN = rewriter.create<LLVM::FCmpOp>(
+  Value resultImagIsNaN = LLVM::FCmpOp::create(rewriter,
       loc, LLVM::FCmpPredicate::uno, resultImag5, zero);
   Value resultIsNaN =
-      rewriter.create<LLVM::AndOp>(loc, resultRealIsNaN, resultImagIsNaN);
+      LLVM::AndOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = rewriter.create<LLVM::SelectOp>(
+  *resultRe = LLVM::SelectOp::create(rewriter,
       loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = rewriter.create<LLVM::SelectOp>(
+  *resultIm = LLVM::SelectOp::create(rewriter,
       loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
 }
 
@@ -278,179 +278,179 @@ void mlir::complex::convertDivToStandardUsingRangeReduction(
   auto elementType = cast<FloatType>(rhsRe.getType());
 
   Value rhsRealImagRatio =
-      rewriter.create<arith::DivFOp>(loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = rewriter.create<arith::AddFOp>(
+      arith::DivFOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
+  Value rhsRealImagDenom = arith::AddFOp::create(rewriter,
       loc, rhsIm,
-      rewriter.create<arith::MulFOp>(loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = rewriter.create<arith::AddFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRealImagRatio, fmf),
+      arith::MulFOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
+  Value realNumerator1 = arith::AddFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
       lhsIm, fmf);
-  Value resultReal1 = rewriter.create<arith::DivFOp>(loc, realNumerator1,
+  Value resultReal1 = arith::DivFOp::create(rewriter, loc, realNumerator1,
                                                      rhsRealImagDenom, fmf);
-  Value imagNumerator1 = rewriter.create<arith::SubFOp>(
-      loc, rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRealImagRatio, fmf),
+  Value imagNumerator1 = arith::SubFOp::create(rewriter,
+      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
       lhsRe, fmf);
-  Value resultImag1 = rewriter.create<arith::DivFOp>(loc, imagNumerator1,
+  Value resultImag1 = arith::DivFOp::create(rewriter, loc, imagNumerator1,
                                                      rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
-      rewriter.create<arith::DivFOp>(loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = rewriter.create<arith::AddFOp>(
+      arith::DivFOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
+  Value rhsImagRealDenom = arith::AddFOp::create(rewriter,
       loc, rhsRe,
-      rewriter.create<arith::MulFOp>(loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = rewriter.create<arith::AddFOp>(
+      arith::MulFOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
+  Value realNumerator2 = arith::AddFOp::create(rewriter,
       loc, lhsRe,
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsImagRealRatio, fmf), fmf);
-  Value resultReal2 = rewriter.create<arith::DivFOp>(loc, realNumerator2,
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
+  Value resultReal2 = arith::DivFOp::create(rewriter, loc, realNumerator2,
                                                      rhsImagRealDenom, fmf);
-  Value imagNumerator2 = rewriter.create<arith::SubFOp>(
+  Value imagNumerator2 = arith::SubFOp::create(rewriter,
       loc, lhsIm,
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsImagRealRatio, fmf), fmf);
-  Value resultImag2 = rewriter.create<arith::DivFOp>(loc, imagNumerator2,
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
+  Value resultImag2 = arith::DivFOp::create(rewriter, loc, imagNumerator2,
                                                      rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
-  Value rhsRealAbs = rewriter.create<math::AbsFOp>(loc, rhsRe, fmf);
-  Value rhsRealIsZero = rewriter.create<arith::CmpFOp>(
+  Value rhsRealAbs = math::AbsFOp::create(rewriter, loc, rhsRe, fmf);
+  Value rhsRealIsZero = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsRealAbs, zero);
-  Value rhsImagAbs = rewriter.create<math::AbsFOp>(loc, rhsIm, fmf);
-  Value rhsImagIsZero = rewriter.create<arith::CmpFOp>(
+  Value rhsImagAbs = math::AbsFOp::create(rewriter, loc, rhsIm, fmf);
+  Value rhsImagIsZero = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsImagAbs, zero);
-  Value lhsRealIsNotNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsRealIsNotNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ORD, lhsRe, zero);
-  Value lhsImagIsNotNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsImagIsNotNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ORD, lhsIm, zero);
   Value lhsContainsNotNaNValue =
-      rewriter.create<arith::OrIOp>(loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = rewriter.create<arith::AndIOp>(
+      arith::OrIOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
+  Value resultIsInfinity = arith::AndIOp::create(rewriter,
       loc, lhsContainsNotNaNValue,
-      rewriter.create<arith::AndIOp>(loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = rewriter.create<arith::ConstantOp>(
+      arith::AndIOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
+  Value inf = arith::ConstantOp::create(rewriter,
       loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfRhsReal =
-      rewriter.create<math::CopySignOp>(loc, inf, rhsRe);
+      math::CopySignOp::create(rewriter, loc, inf, rhsRe);
   Value infinityResultReal =
-      rewriter.create<arith::MulFOp>(loc, infWithSignOfRhsReal, lhsRe, fmf);
+      arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsRe, fmf);
   Value infinityResultImag =
-      rewriter.create<arith::MulFOp>(loc, infWithSignOfRhsReal, lhsIm, fmf);
+      arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = rewriter.create<arith::CmpFOp>(
+  Value rhsRealFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, rhsRealAbs, inf);
-  Value rhsImagFinite = rewriter.create<arith::CmpFOp>(
+  Value rhsImagFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, rhsImagAbs, inf);
   Value rhsFinite =
-      rewriter.create<arith::AndIOp>(loc, rhsRealFinite, rhsImagFinite);
-  Value lhsRealAbs = rewriter.create<math::AbsFOp>(loc, lhsRe, fmf);
-  Value lhsRealInfinite = rewriter.create<arith::CmpFOp>(
+      arith::AndIOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
+  Value lhsRealAbs = math::AbsFOp::create(rewriter, loc, lhsRe, fmf);
+  Value lhsRealInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, lhsRealAbs, inf);
-  Value lhsImagAbs = rewriter.create<math::AbsFOp>(loc, lhsIm, fmf);
-  Value lhsImagInfinite = rewriter.create<arith::CmpFOp>(
+  Value lhsImagAbs = math::AbsFOp::create(rewriter, loc, lhsIm, fmf);
+  Value lhsImagInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, lhsImagAbs, inf);
   Value lhsInfinite =
-      rewriter.create<arith::OrIOp>(loc, lhsRealInfinite, lhsImagInfinite);
+      arith::OrIOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
-      rewriter.create<arith::AndIOp>(loc, lhsInfinite, rhsFinite);
-  Value one = rewriter.create<arith::ConstantOp>(
+      arith::AndIOp::create(rewriter, loc, lhsInfinite, rhsFinite);
+  Value one = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, lhsRealInfinite, one, zero),
+  Value lhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesRhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRealIsInfWithSign, rhsRe, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesRhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
+  Value resultReal3 = arith::MulFOp::create(rewriter,
       loc, inf,
-      rewriter.create<arith::AddFOp>(loc, lhsRealIsInfWithSignTimesRhsReal,
+      arith::AddFOp::create(rewriter, loc, lhsRealIsInfWithSignTimesRhsReal,
                                      lhsImagIsInfWithSignTimesRhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesRhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsRealIsInfWithSign, rhsIm, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesRhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
+  Value resultImag3 = arith::MulFOp::create(rewriter,
       loc, inf,
-      rewriter.create<arith::SubFOp>(loc, lhsImagIsInfWithSignTimesRhsReal,
+      arith::SubFOp::create(rewriter, loc, lhsImagIsInfWithSignTimesRhsReal,
                                      lhsRealIsInfWithSignTimesRhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = rewriter.create<arith::CmpFOp>(
+  Value lhsRealFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, lhsRealAbs, inf);
-  Value lhsImagFinite = rewriter.create<arith::CmpFOp>(
+  Value lhsImagFinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::ONE, lhsImagAbs, inf);
   Value lhsFinite =
-      rewriter.create<arith::AndIOp>(loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = rewriter.create<arith::CmpFOp>(
+      arith::AndIOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
+  Value rhsRealInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsRealAbs, inf);
-  Value rhsImagInfinite = rewriter.create<arith::CmpFOp>(
+  Value rhsImagInfinite = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OEQ, rhsImagAbs, inf);
   Value rhsInfinite =
-      rewriter.create<arith::OrIOp>(loc, rhsRealInfinite, rhsImagInfinite);
+      arith::OrIOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
-      rewriter.create<arith::AndIOp>(loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, rhsRealInfinite, one, zero),
+      arith::AndIOp::create(rewriter, loc, lhsFinite, rhsInfinite);
+  Value rhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = rewriter.create<math::CopySignOp>(
-      loc, rewriter.create<arith::SelectOp>(loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
+      loc, arith::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimesLhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsRealIsInfWithSign, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
+  Value resultReal4 = arith::MulFOp::create(rewriter,
       loc, zero,
-      rewriter.create<arith::AddFOp>(loc, rhsRealIsInfWithSignTimesLhsReal,
+      arith::AddFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsReal,
                                      rhsImagIsInfWithSignTimesLhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimesLhsImag =
-      rewriter.create<arith::MulFOp>(loc, lhsIm, rhsRealIsInfWithSign, fmf);
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsReal =
-      rewriter.create<arith::MulFOp>(loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = rewriter.create<arith::MulFOp>(
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
+  Value resultImag4 = arith::MulFOp::create(rewriter,
       loc, zero,
-      rewriter.create<arith::SubFOp>(loc, rhsRealIsInfWithSignTimesLhsImag,
+      arith::SubFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsImag,
                                      rhsImagIsInfWithSignTimesLhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = rewriter.create<arith::CmpFOp>(
+  Value realAbsSmallerThanImagAbs = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OLT, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = rewriter.create<arith::SelectOp>(
+  Value resultReal5 = arith::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = rewriter.create<arith::SelectOp>(
+  Value resultImag5 = arith::SelectOp::create(rewriter,
       loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase3 = arith::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase3 = arith::SelectOp::create(rewriter,
       loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase2 = arith::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase2 = arith::SelectOp::create(rewriter,
       loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = rewriter.create<arith::SelectOp>(
+  Value resultRealSpecialCase1 = arith::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = rewriter.create<arith::SelectOp>(
+  Value resultImagSpecialCase1 = arith::SelectOp::create(rewriter,
       loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = rewriter.create<arith::CmpFOp>(
+  Value resultRealIsNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::UNO, resultReal5, zero);
-  Value resultImagIsNaN = rewriter.create<arith::CmpFOp>(
+  Value resultImagIsNaN = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::UNO, resultImag5, zero);
   Value resultIsNaN =
-      rewriter.create<arith::AndIOp>(loc, resultRealIsNaN, resultImagIsNaN);
+      arith::AndIOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = rewriter.create<arith::SelectOp>(
+  *resultRe = arith::SelectOp::create(rewriter,
       loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = rewriter.create<arith::SelectOp>(
+  *resultIm = arith::SelectOp::create(rewriter,
       loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
 }
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index e5e862315941d..a06935c4387c6 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -35,7 +35,7 @@ static constexpr unsigned kImaginaryPosInComplexNumberStruct = 1;
 
 ComplexStructBuilder ComplexStructBuilder::poison(OpBuilder &builder,
                                                   Location loc, Type type) {
-  Value val = builder.create<LLVM::PoisonOp>(loc, type);
+  Value val = LLVM::PoisonOp::create(builder, loc, type);
   return ComplexStructBuilder(val);
 }
 
@@ -79,9 +79,9 @@ struct AbsOpConversion : public ConvertOpToLLVMPattern<complex::AbsOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value sqNorm = rewriter.create<LLVM::FAddOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, real, real, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, imag, imag, fmf), fmf);
+    Value sqNorm = LLVM::FAddOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, real, real, fmf),
+        LLVM::FMulOp::create(rewriter, loc, imag, imag, fmf), fmf);
 
     rewriter.replaceOpWithNewOp<LLVM::SqrtOp>(op, sqNorm);
     return success();
@@ -192,9 +192,9 @@ struct AddOpConversion : public ConvertOpToLLVMPattern<complex::AddOp> {
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
     Value real =
-        rewriter.create<LLVM::FAddOp>(loc, arg.lhs.real(), arg.rhs.real(), fmf);
+        LLVM::FAddOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
     Value imag =
-        rewriter.create<LLVM::FAddOp>(loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+        LLVM::FAddOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
@@ -278,13 +278,13 @@ struct MulOpConversion : public ConvertOpToLLVMPattern<complex::MulOp> {
     Value lhsRe = arg.lhs.real();
     Value lhsIm = arg.lhs.imag();
 
-    Value real = rewriter.create<LLVM::FSubOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, rhsRe, lhsRe, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, rhsIm, lhsIm, fmf), fmf);
+    Value real = LLVM::FSubOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, lhsRe, fmf),
+        LLVM::FMulOp::create(rewriter, loc, rhsIm, lhsIm, fmf), fmf);
 
-    Value imag = rewriter.create<LLVM::FAddOp>(
-        loc, rewriter.create<LLVM::FMulOp>(loc, lhsIm, rhsRe, fmf),
-        rewriter.create<LLVM::FMulOp>(loc, lhsRe, rhsIm, fmf), fmf);
+    Value imag = LLVM::FAddOp::create(rewriter,
+        loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+        LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
@@ -314,9 +314,9 @@ struct SubOpConversion : public ConvertOpToLLVMPattern<complex::SubOp> {
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
     Value real =
-        rewriter.create<LLVM::FSubOp>(loc, arg.lhs.real(), arg.rhs.real(), fmf);
+        LLVM::FSubOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
     Value imag =
-        rewriter.create<LLVM::FSubOp>(loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+        LLVM::FSubOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
diff --git a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
index 56269d189873a..4800f65c675e6 100644
--- a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
+++ b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
@@ -84,7 +84,7 @@ LogicalResult ScalarOpToLibmCall<Op, TypeResolver>::matchAndRewrite(
     rewriter.setInsertionPointToStart(&module->getRegion(0).front());
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-    opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name,
+    opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
                                            opFunctionTy);
     opFunc.setPrivate();
   }
diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 0c832c452718b..d14e3888b118c 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -31,44 +31,44 @@ enum class AbsFn { abs, sqrt, rsqrt };
 // Returns the absolute value, its square root or its reciprocal square root.
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
                  ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
-  Value one = b.create<arith::ConstantOp>(real.getType(),
+  Value one = arith::ConstantOp::create(b, real.getType(),
                                           b.getFloatAttr(real.getType(), 1.0));
 
-  Value absReal = b.create<math::AbsFOp>(real, fmf);
-  Value absImag = b.create<math::AbsFOp>(imag, fmf);
+  Value absReal = math::AbsFOp::create(b, real, fmf);
+  Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-  Value max = b.create<arith::MaximumFOp>(absReal, absImag, fmf);
-  Value min = b.create<arith::MinimumFOp>(absReal, absImag, fmf);
+  Value max = arith::MaximumFOp::create(b, absReal, absImag, fmf);
+  Value min = arith::MinimumFOp::create(b, absReal, absImag, fmf);
 
   // The lowering below requires NaNs and infinities to work correctly.
   arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear(
       fmf, arith::FastMathFlags::nnan | arith::FastMathFlags::ninf);
-  Value ratio = b.create<arith::DivFOp>(min, max, fmfWithNaNInf);
-  Value ratioSq = b.create<arith::MulFOp>(ratio, ratio, fmfWithNaNInf);
-  Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmfWithNaNInf);
+  Value ratio = arith::DivFOp::create(b, min, max, fmfWithNaNInf);
+  Value ratioSq = arith::MulFOp::create(b, ratio, ratio, fmfWithNaNInf);
+  Value ratioSqPlusOne = arith::AddFOp::create(b, ratioSq, one, fmfWithNaNInf);
   Value result;
 
   if (fn == AbsFn::rsqrt) {
-    ratioSqPlusOne = b.create<math::RsqrtOp>(ratioSqPlusOne, fmfWithNaNInf);
-    min = b.create<math::RsqrtOp>(min, fmfWithNaNInf);
-    max = b.create<math::RsqrtOp>(max, fmfWithNaNInf);
+    ratioSqPlusOne = math::RsqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
+    min = math::RsqrtOp::create(b, min, fmfWithNaNInf);
+    max = math::RsqrtOp::create(b, max, fmfWithNaNInf);
   }
 
   if (fn == AbsFn::sqrt) {
-    Value quarter = b.create<arith::ConstantOp>(
+    Value quarter = arith::ConstantOp::create(b,
         real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
-    Value sqrt = b.create<math::SqrtOp>(max, fmfWithNaNInf);
-    Value p025 = b.create<math::PowFOp>(ratioSqPlusOne, quarter, fmfWithNaNInf);
-    result = b.create<arith::MulFOp>(sqrt, p025, fmfWithNaNInf);
+    Value sqrt = math::SqrtOp::create(b, max, fmfWithNaNInf);
+    Value p025 = math::PowFOp::create(b, ratioSqPlusOne, quarter, fmfWithNaNInf);
+    result = arith::MulFOp::create(b, sqrt, p025, fmfWithNaNInf);
   } else {
-    Value sqrt = b.create<math::SqrtOp>(ratioSqPlusOne, fmfWithNaNInf);
-    result = b.create<arith::MulFOp>(max, sqrt, fmfWithNaNInf);
+    Value sqrt = math::SqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
+    result = arith::MulFOp::create(b, max, sqrt, fmfWithNaNInf);
   }
 
-  Value isNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, result,
+  Value isNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, result,
                                         result, fmfWithNaNInf);
-  return b.create<arith::SelectOp>(isNaN, min, result);
+  return arith::SelectOp::create(b, isNaN, min, result);
 }
 
 struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
@@ -81,8 +81,8 @@ struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
 
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
     rewriter.replaceOp(op, computeAbs(real, imag, fmf, b));
 
     return success();
@@ -105,28 +105,28 @@ struct Atan2OpConversion : public OpConversionPattern<complex::Atan2Op> {
     Value lhs = adaptor.getLhs();
     Value rhs = adaptor.getRhs();
 
-    Value rhsSquared = b.create<complex::MulOp>(type, rhs, rhs, fmf);
-    Value lhsSquared = b.create<complex::MulOp>(type, lhs, lhs, fmf);
+    Value rhsSquared = complex::MulOp::create(b, type, rhs, rhs, fmf);
+    Value lhsSquared = complex::MulOp::create(b, type, lhs, lhs, fmf);
     Value rhsSquaredPlusLhsSquared =
-        b.create<complex::AddOp>(type, rhsSquared, lhsSquared, fmf);
+        complex::AddOp::create(b, type, rhsSquared, lhsSquared, fmf);
     Value sqrtOfRhsSquaredPlusLhsSquared =
-        b.create<complex::SqrtOp>(type, rhsSquaredPlusLhsSquared, fmf);
+        complex::SqrtOp::create(b, type, rhsSquaredPlusLhsSquared, fmf);
 
     Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
-    Value one = b.create<arith::ConstantOp>(elementType,
+        arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
+    Value one = arith::ConstantOp::create(b, elementType,
                                             b.getFloatAttr(elementType, 1));
-    Value i = b.create<complex::CreateOp>(type, zero, one);
-    Value iTimesLhs = b.create<complex::MulOp>(i, lhs, fmf);
-    Value rhsPlusILhs = b.create<complex::AddOp>(rhs, iTimesLhs, fmf);
+    Value i = complex::CreateOp::create(b, type, zero, one);
+    Value iTimesLhs = complex::MulOp::create(b, i, lhs, fmf);
+    Value rhsPlusILhs = complex::AddOp::create(b, rhs, iTimesLhs, fmf);
 
-    Value divResult = b.create<complex::DivOp>(
+    Value divResult = complex::DivOp::create(b,
         rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf);
-    Value logResult = b.create<complex::LogOp>(divResult, fmf);
+    Value logResult = complex::LogOp::create(b, divResult, fmf);
 
-    Value negativeOne = b.create<arith::ConstantOp>(
+    Value negativeOne = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -1));
-    Value negativeI = b.create<complex::CreateOp>(type, zero, negativeOne);
+    Value negativeI = complex::CreateOp::create(b, type, zero, negativeOne);
 
     rewriter.replaceOpWithNewOp<complex::MulOp>(op, negativeI, logResult, fmf);
     return success();
@@ -146,14 +146,14 @@ struct ComparisonOpConversion : public OpConversionPattern<ComparisonOp> {
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getLhs().getType()).getElementType();
 
-    Value realLhs = rewriter.create<complex::ReOp>(loc, type, adaptor.getLhs());
-    Value imagLhs = rewriter.create<complex::ImOp>(loc, type, adaptor.getLhs());
-    Value realRhs = rewriter.create<complex::ReOp>(loc, type, adaptor.getRhs());
-    Value imagRhs = rewriter.create<complex::ImOp>(loc, type, adaptor.getRhs());
+    Value realLhs = complex::ReOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value imagLhs = complex::ImOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value realRhs = complex::ReOp::create(rewriter, loc, type, adaptor.getRhs());
+    Value imagRhs = complex::ImOp::create(rewriter, loc, type, adaptor.getRhs());
     Value realComparison =
-        rewriter.create<arith::CmpFOp>(loc, p, realLhs, realRhs);
+        arith::CmpFOp::create(rewriter, loc, p, realLhs, realRhs);
     Value imagComparison =
-        rewriter.create<arith::CmpFOp>(loc, p, imagLhs, imagRhs);
+        arith::CmpFOp::create(rewriter, loc, p, imagLhs, imagRhs);
 
     rewriter.replaceOpWithNewOp<ResultCombiner>(op, realComparison,
                                                 imagComparison);
@@ -176,13 +176,13 @@ struct BinaryComplexOpConversion : public OpConversionPattern<BinaryComplexOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
-    Value realLhs = b.create<complex::ReOp>(elementType, adaptor.getLhs());
-    Value realRhs = b.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value resultReal = b.create<BinaryStandardOp>(elementType, realLhs, realRhs,
+    Value realLhs = complex::ReOp::create(b, elementType, adaptor.getLhs());
+    Value realRhs = complex::ReOp::create(b, elementType, adaptor.getRhs());
+    Value resultReal = BinaryStandardOp::create(b, elementType, realLhs, realRhs,
                                                   fmf.getValue());
-    Value imagLhs = b.create<complex::ImOp>(elementType, adaptor.getLhs());
-    Value imagRhs = b.create<complex::ImOp>(elementType, adaptor.getRhs());
-    Value resultImag = b.create<BinaryStandardOp>(elementType, imagLhs, imagRhs,
+    Value imagLhs = complex::ImOp::create(b, elementType, adaptor.getLhs());
+    Value imagRhs = complex::ImOp::create(b, elementType, adaptor.getRhs());
+    Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs, imagRhs,
                                                   fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -205,20 +205,20 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
 
     // Trigonometric ops use a set of common building blocks to convert to real
     // ops. Here we create these building blocks and call into an op-specific
     // implementation in the subclass to combine them.
-    Value half = rewriter.create<arith::ConstantOp>(
+    Value half = arith::ConstantOp::create(rewriter,
         loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
-    Value exp = rewriter.create<math::ExpOp>(loc, imag, fmf);
-    Value scaledExp = rewriter.create<arith::MulFOp>(loc, half, exp, fmf);
-    Value reciprocalExp = rewriter.create<arith::DivFOp>(loc, half, exp, fmf);
-    Value sin = rewriter.create<math::SinOp>(loc, real, fmf);
-    Value cos = rewriter.create<math::CosOp>(loc, real, fmf);
+    Value exp = math::ExpOp::create(rewriter, loc, imag, fmf);
+    Value scaledExp = arith::MulFOp::create(rewriter, loc, half, exp, fmf);
+    Value reciprocalExp = arith::DivFOp::create(rewriter, loc, half, exp, fmf);
+    Value sin = math::SinOp::create(rewriter, loc, real, fmf);
+    Value cos = math::CosOp::create(rewriter, loc, real, fmf);
 
     auto resultPair =
         combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf);
@@ -251,11 +251,11 @@ struct CosOpConversion : public TrigonometricOpConversion<complex::CosOp> {
     //   Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x
     //   Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x
     Value sum =
-        rewriter.create<arith::AddFOp>(loc, reciprocalExp, scaledExp, fmf);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, cos, fmf);
+        arith::AddFOp::create(rewriter, loc, reciprocalExp, scaledExp, fmf);
+    Value resultReal = arith::MulFOp::create(rewriter, loc, sum, cos, fmf);
     Value diff =
-        rewriter.create<arith::SubFOp>(loc, reciprocalExp, scaledExp, fmf);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, sin, fmf);
+        arith::SubFOp::create(rewriter, loc, reciprocalExp, scaledExp, fmf);
+    Value resultImag = arith::MulFOp::create(rewriter, loc, diff, sin, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -275,13 +275,13 @@ struct DivOpConversion : public OpConversionPattern<complex::DivOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value lhsReal =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getLhs());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getLhs());
     Value lhsImag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getLhs());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getLhs());
     Value rhsReal =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getRhs());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getRhs());
     Value rhsImag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getRhs());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getRhs());
 
     Value resultReal, resultImag;
 
@@ -318,16 +318,16 @@ struct ExpOpConversion : public OpConversionPattern<complex::ExpOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value expReal = rewriter.create<math::ExpOp>(loc, real, fmf.getValue());
-    Value cosImag = rewriter.create<math::CosOp>(loc, imag, fmf.getValue());
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value expReal = math::ExpOp::create(rewriter, loc, real, fmf.getValue());
+    Value cosImag = math::CosOp::create(rewriter, loc, imag, fmf.getValue());
     Value resultReal =
-        rewriter.create<arith::MulFOp>(loc, expReal, cosImag, fmf.getValue());
-    Value sinImag = rewriter.create<math::SinOp>(loc, imag, fmf.getValue());
+        arith::MulFOp::create(rewriter, loc, expReal, cosImag, fmf.getValue());
+    Value sinImag = math::SinOp::create(rewriter, loc, imag, fmf.getValue());
     Value resultImag =
-        rewriter.create<arith::MulFOp>(loc, expReal, sinImag, fmf.getValue());
+        arith::MulFOp::create(rewriter, loc, expReal, sinImag, fmf.getValue());
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -340,11 +340,11 @@ Value evaluatePolynomial(ImplicitLocOpBuilder &b, Value arg,
                          arith::FastMathFlagsAttr fmf) {
   auto argType = mlir::cast<FloatType>(arg.getType());
   Value poly =
-      b.create<arith::ConstantOp>(b.getFloatAttr(argType, coefficients[0]));
+      arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[0]));
   for (unsigned i = 1; i < coefficients.size(); ++i) {
-    poly = b.create<math::FmaOp>(
+    poly = math::FmaOp::create(b,
         poly, arg,
-        b.create<arith::ConstantOp>(b.getFloatAttr(argType, coefficients[i])),
+        arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[i])),
         fmf);
   }
   return poly;
@@ -365,26 +365,26 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
 
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     ImplicitLocOpBuilder b(op.getLoc(), rewriter);
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
-    Value zero = b.create<arith::ConstantOp>(b.getFloatAttr(elemType, 0.0));
-    Value one = b.create<arith::ConstantOp>(b.getFloatAttr(elemType, 1.0));
+    Value zero = arith::ConstantOp::create(b, b.getFloatAttr(elemType, 0.0));
+    Value one = arith::ConstantOp::create(b, b.getFloatAttr(elemType, 1.0));
 
-    Value expm1Real = b.create<math::ExpM1Op>(real, fmf);
-    Value expReal = b.create<arith::AddFOp>(expm1Real, one, fmf);
+    Value expm1Real = math::ExpM1Op::create(b, real, fmf);
+    Value expReal = arith::AddFOp::create(b, expm1Real, one, fmf);
 
-    Value sinImag = b.create<math::SinOp>(imag, fmf);
+    Value sinImag = math::SinOp::create(b, imag, fmf);
     Value cosm1Imag = emitCosm1(imag, fmf, b);
-    Value cosImag = b.create<arith::AddFOp>(cosm1Imag, one, fmf);
+    Value cosImag = arith::AddFOp::create(b, cosm1Imag, one, fmf);
 
-    Value realResult = b.create<arith::AddFOp>(
-        b.create<arith::MulFOp>(expm1Real, cosImag, fmf), cosm1Imag, fmf);
+    Value realResult = arith::AddFOp::create(b,
+        arith::MulFOp::create(b, expm1Real, cosImag, fmf), cosm1Imag, fmf);
 
-    Value imagIsZero = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag,
+    Value imagIsZero = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag,
                                                zero, fmf.getValue());
-    Value imagResult = b.create<arith::SelectOp>(
-        imagIsZero, zero, b.create<arith::MulFOp>(expReal, sinImag, fmf));
+    Value imagResult = arith::SelectOp::create(b,
+        imagIsZero, zero, arith::MulFOp::create(b, expReal, sinImag, fmf));
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, realResult,
                                                    imagResult);
@@ -395,8 +395,8 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
   Value emitCosm1(Value arg, arith::FastMathFlagsAttr fmf,
                   ImplicitLocOpBuilder &b) const {
     auto argType = mlir::cast<FloatType>(arg.getType());
-    auto negHalf = b.create<arith::ConstantOp>(b.getFloatAttr(argType, -0.5));
-    auto negOne = b.create<arith::ConstantOp>(b.getFloatAttr(argType, -1.0));
+    auto negHalf = arith::ConstantOp::create(b, b.getFloatAttr(argType, -0.5));
+    auto negOne = arith::ConstantOp::create(b, b.getFloatAttr(argType, -1.0));
 
     // Algorithm copied from cephes cosm1.
     SmallVector<double, 7> kCoeffs{
@@ -405,23 +405,23 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
         2.4801587301570552304991E-5,  -1.3888888888888872993737E-3,
         4.1666666666666666609054E-2,
     };
-    Value cos = b.create<math::CosOp>(arg, fmf);
-    Value forLargeArg = b.create<arith::AddFOp>(cos, negOne, fmf);
+    Value cos = math::CosOp::create(b, arg, fmf);
+    Value forLargeArg = arith::AddFOp::create(b, cos, negOne, fmf);
 
-    Value argPow2 = b.create<arith::MulFOp>(arg, arg, fmf);
-    Value argPow4 = b.create<arith::MulFOp>(argPow2, argPow2, fmf);
+    Value argPow2 = arith::MulFOp::create(b, arg, arg, fmf);
+    Value argPow4 = arith::MulFOp::create(b, argPow2, argPow2, fmf);
     Value poly = evaluatePolynomial(b, argPow2, kCoeffs, fmf);
 
     auto forSmallArg =
-        b.create<arith::AddFOp>(b.create<arith::MulFOp>(argPow4, poly, fmf),
-                                b.create<arith::MulFOp>(negHalf, argPow2, fmf));
+        arith::AddFOp::create(b, arith::MulFOp::create(b, argPow4, poly, fmf),
+                                arith::MulFOp::create(b, negHalf, argPow2, fmf));
 
     // (pi/4)^2 is approximately 0.61685
     Value piOver4Pow2 =
-        b.create<arith::ConstantOp>(b.getFloatAttr(argType, 0.61685));
-    Value cond = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, argPow2,
+        arith::ConstantOp::create(b, b.getFloatAttr(argType, 0.61685));
+    Value cond = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, argPow2,
                                          piOver4Pow2, fmf.getValue());
-    return b.create<arith::SelectOp>(cond, forLargeArg, forSmallArg);
+    return arith::SelectOp::create(b, cond, forLargeArg, forSmallArg);
   }
 };
 
@@ -436,13 +436,13 @@ struct LogOpConversion : public OpConversionPattern<complex::LogOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
-    Value abs = b.create<complex::AbsOp>(elementType, adaptor.getComplex(),
+    Value abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(),
                                          fmf.getValue());
-    Value resultReal = b.create<math::LogOp>(elementType, abs, fmf.getValue());
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value resultReal = math::LogOp::create(b, elementType, abs, fmf.getValue());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value resultImag =
-        b.create<math::Atan2Op>(elementType, imag, real, fmf.getValue());
+        math::Atan2Op::create(b, elementType, imag, real, fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -460,40 +460,40 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
-    Value real = b.create<complex::ReOp>(adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(adaptor.getComplex());
+    Value real = complex::ReOp::create(b, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
-    Value half = b.create<arith::ConstantOp>(elementType,
+    Value half = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 0.5));
-    Value one = b.create<arith::ConstantOp>(elementType,
+    Value one = arith::ConstantOp::create(b, elementType,
                                             b.getFloatAttr(elementType, 1));
-    Value realPlusOne = b.create<arith::AddFOp>(real, one, fmf);
-    Value absRealPlusOne = b.create<math::AbsFOp>(realPlusOne, fmf);
-    Value absImag = b.create<math::AbsFOp>(imag, fmf);
+    Value realPlusOne = arith::AddFOp::create(b, real, one, fmf);
+    Value absRealPlusOne = math::AbsFOp::create(b, realPlusOne, fmf);
+    Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-    Value maxAbs = b.create<arith::MaximumFOp>(absRealPlusOne, absImag, fmf);
-    Value minAbs = b.create<arith::MinimumFOp>(absRealPlusOne, absImag, fmf);
+    Value maxAbs = arith::MaximumFOp::create(b, absRealPlusOne, absImag, fmf);
+    Value minAbs = arith::MinimumFOp::create(b, absRealPlusOne, absImag, fmf);
 
-    Value useReal = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGT,
+    Value useReal = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT,
                                             realPlusOne, absImag, fmf);
-    Value maxMinusOne = b.create<arith::SubFOp>(maxAbs, one, fmf);
+    Value maxMinusOne = arith::SubFOp::create(b, maxAbs, one, fmf);
     Value maxAbsOfRealPlusOneAndImagMinusOne =
-        b.create<arith::SelectOp>(useReal, real, maxMinusOne);
+        arith::SelectOp::create(b, useReal, real, maxMinusOne);
     arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear(
         fmf, arith::FastMathFlags::nnan | arith::FastMathFlags::ninf);
-    Value minMaxRatio = b.create<arith::DivFOp>(minAbs, maxAbs, fmfWithNaNInf);
+    Value minMaxRatio = arith::DivFOp::create(b, minAbs, maxAbs, fmfWithNaNInf);
     Value logOfMaxAbsOfRealPlusOneAndImag =
-        b.create<math::Log1pOp>(maxAbsOfRealPlusOneAndImagMinusOne, fmf);
-    Value logOfSqrtPart = b.create<math::Log1pOp>(
-        b.create<arith::MulFOp>(minMaxRatio, minMaxRatio, fmfWithNaNInf),
+        math::Log1pOp::create(b, maxAbsOfRealPlusOneAndImagMinusOne, fmf);
+    Value logOfSqrtPart = math::Log1pOp::create(b,
+        arith::MulFOp::create(b, minMaxRatio, minMaxRatio, fmfWithNaNInf),
         fmfWithNaNInf);
-    Value r = b.create<arith::AddFOp>(
-        b.create<arith::MulFOp>(half, logOfSqrtPart, fmfWithNaNInf),
+    Value r = arith::AddFOp::create(b,
+        arith::MulFOp::create(b, half, logOfSqrtPart, fmfWithNaNInf),
         logOfMaxAbsOfRealPlusOneAndImag, fmfWithNaNInf);
-    Value resultReal = b.create<arith::SelectOp>(
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
+    Value resultReal = arith::SelectOp::create(b,
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
         minAbs, r);
-    Value resultImag = b.create<math::Atan2Op>(imag, realPlusOne, fmf);
+    Value resultImag = math::Atan2Op::create(b, imag, realPlusOne, fmf);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -511,21 +511,21 @@ struct MulOpConversion : public OpConversionPattern<complex::MulOp> {
     auto elementType = cast<FloatType>(type.getElementType());
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
     auto fmfValue = fmf.getValue();
-    Value lhsReal = b.create<complex::ReOp>(elementType, adaptor.getLhs());
-    Value lhsImag = b.create<complex::ImOp>(elementType, adaptor.getLhs());
-    Value rhsReal = b.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value rhsImag = b.create<complex::ImOp>(elementType, adaptor.getRhs());
+    Value lhsReal = complex::ReOp::create(b, elementType, adaptor.getLhs());
+    Value lhsImag = complex::ImOp::create(b, elementType, adaptor.getLhs());
+    Value rhsReal = complex::ReOp::create(b, elementType, adaptor.getRhs());
+    Value rhsImag = complex::ImOp::create(b, elementType, adaptor.getRhs());
     Value lhsRealTimesRhsReal =
-        b.create<arith::MulFOp>(lhsReal, rhsReal, fmfValue);
+        arith::MulFOp::create(b, lhsReal, rhsReal, fmfValue);
     Value lhsImagTimesRhsImag =
-        b.create<arith::MulFOp>(lhsImag, rhsImag, fmfValue);
-    Value real = b.create<arith::SubFOp>(lhsRealTimesRhsReal,
+        arith::MulFOp::create(b, lhsImag, rhsImag, fmfValue);
+    Value real = arith::SubFOp::create(b, lhsRealTimesRhsReal,
                                          lhsImagTimesRhsImag, fmfValue);
     Value lhsImagTimesRhsReal =
-        b.create<arith::MulFOp>(lhsImag, rhsReal, fmfValue);
+        arith::MulFOp::create(b, lhsImag, rhsReal, fmfValue);
     Value lhsRealTimesRhsImag =
-        b.create<arith::MulFOp>(lhsReal, rhsImag, fmfValue);
-    Value imag = b.create<arith::AddFOp>(lhsImagTimesRhsReal,
+        arith::MulFOp::create(b, lhsReal, rhsImag, fmfValue);
+    Value imag = arith::AddFOp::create(b, lhsImagTimesRhsReal,
                                          lhsRealTimesRhsImag, fmfValue);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, imag);
     return success();
@@ -543,11 +543,11 @@ struct NegOpConversion : public OpConversionPattern<complex::NegOp> {
     auto elementType = cast<FloatType>(type.getElementType());
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negReal = rewriter.create<arith::NegFOp>(loc, real);
-    Value negImag = rewriter.create<arith::NegFOp>(loc, imag);
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value negReal = arith::NegFOp::create(rewriter, loc, real);
+    Value negImag = arith::NegFOp::create(rewriter, loc, imag);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, negReal, negImag);
     return success();
   }
@@ -570,11 +570,11 @@ struct SinOpConversion : public TrigonometricOpConversion<complex::SinOp> {
     //   Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x
     //   Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x
     Value sum =
-        rewriter.create<arith::AddFOp>(loc, scaledExp, reciprocalExp, fmf);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, sin, fmf);
+        arith::AddFOp::create(rewriter, loc, scaledExp, reciprocalExp, fmf);
+    Value resultReal = arith::MulFOp::create(rewriter, loc, sum, sin, fmf);
     Value diff =
-        rewriter.create<arith::SubFOp>(loc, scaledExp, reciprocalExp, fmf);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, cos, fmf);
+        arith::SubFOp::create(rewriter, loc, scaledExp, reciprocalExp, fmf);
+    Value resultImag = arith::MulFOp::create(rewriter, loc, diff, cos, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -593,64 +593,64 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
-    Value half = b.create<arith::ConstantOp>(elementType,
+    Value half = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 0.5));
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value absSqrt = computeAbs(real, imag, fmf, b, AbsFn::sqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value sqrtArg = b.create<arith::MulFOp>(argArg, half, fmf);
-    Value cos = b.create<math::CosOp>(sqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(sqrtArg, fmf);
+    Value argArg = math::Atan2Op::create(b, imag, real, fmf);
+    Value sqrtArg = arith::MulFOp::create(b, argArg, half, fmf);
+    Value cos = math::CosOp::create(b, sqrtArg, fmf);
+    Value sin = math::SinOp::create(b, sqrtArg, fmf);
     // sin(atan2(0, inf)) = 0, sqrt(abs(inf)) = inf, but we can't multiply
     // 0 * inf.
     Value sinIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, sin, zero, fmf);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, sin, zero, fmf);
 
-    Value resultReal = b.create<arith::MulFOp>(absSqrt, cos, fmf);
-    Value resultImag = b.create<arith::SelectOp>(
-        sinIsZero, zero, b.create<arith::MulFOp>(absSqrt, sin, fmf));
+    Value resultReal = arith::MulFOp::create(b, absSqrt, cos, fmf);
+    Value resultImag = arith::SelectOp::create(b,
+        sinIsZero, zero, arith::MulFOp::create(b, absSqrt, sin, fmf));
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
       Value inf = cst(APFloat::getInf(floatSemantics));
       Value negInf = cst(APFloat::getInf(floatSemantics, true));
       Value nan = cst(APFloat::getNaN(floatSemantics));
-      Value absImag = b.create<math::AbsFOp>(elementType, imag, fmf);
+      Value absImag = math::AbsFOp::create(b, elementType, imag, fmf);
 
       Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
       Value absImagIsNotInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::ONE, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::ONE, absImag, inf, fmf);
       Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, inf, fmf);
       Value realIsNegInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, negInf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, negInf, fmf);
 
-      resultReal = b.create<arith::SelectOp>(
-          b.create<arith::AndIOp>(realIsNegInf, absImagIsNotInf), zero,
+      resultReal = arith::SelectOp::create(b,
+          arith::AndIOp::create(b, realIsNegInf, absImagIsNotInf), zero,
           resultReal);
-      resultReal = b.create<arith::SelectOp>(
-          b.create<arith::OrIOp>(absImagIsInf, realIsInf), inf, resultReal);
+      resultReal = arith::SelectOp::create(b,
+          arith::OrIOp::create(b, absImagIsInf, realIsInf), inf, resultReal);
 
-      Value imagSignInf = b.create<math::CopySignOp>(inf, imag, fmf);
-      resultImag = b.create<arith::SelectOp>(
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, absSqrt, absSqrt),
+      Value imagSignInf = math::CopySignOp::create(b, inf, imag, fmf);
+      resultImag = arith::SelectOp::create(b,
+          arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, absSqrt, absSqrt),
           nan, resultImag);
-      resultImag = b.create<arith::SelectOp>(
-          b.create<arith::OrIOp>(absImagIsInf, realIsNegInf), imagSignInf,
+      resultImag = arith::SelectOp::create(b,
+          arith::OrIOp::create(b, absImagIsInf, realIsNegInf), imagSignInf,
           resultImag);
     }
 
     Value resultIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
-    resultReal = b.create<arith::SelectOp>(resultIsZero, zero, resultReal);
-    resultImag = b.create<arith::SelectOp>(resultIsZero, zero, resultImag);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
+    resultReal = arith::SelectOp::create(b, resultIsZero, zero, resultReal);
+    resultImag = arith::SelectOp::create(b, resultIsZero, zero, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -669,19 +669,19 @@ struct SignOpConversion : public OpConversionPattern<complex::SignOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
+        arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
     Value realIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, zero);
     Value imagIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero);
-    Value isZero = b.create<arith::AndIOp>(realIsZero, imagIsZero);
-    auto abs = b.create<complex::AbsOp>(elementType, adaptor.getComplex(), fmf);
-    Value realSign = b.create<arith::DivFOp>(real, abs, fmf);
-    Value imagSign = b.create<arith::DivFOp>(imag, abs, fmf);
-    Value sign = b.create<complex::CreateOp>(type, realSign, imagSign);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero);
+    Value isZero = arith::AndIOp::create(b, realIsZero, imagIsZero);
+    auto abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(), fmf);
+    Value realSign = arith::DivFOp::create(b, real, abs, fmf);
+    Value imagSign = arith::DivFOp::create(b, imag, abs, fmf);
+    Value sign = complex::CreateOp::create(b, type, realSign, imagSign);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isZero,
                                                  adaptor.getComplex(), sign);
     return success();
@@ -703,84 +703,84 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
     const auto &floatSemantics = elementType.getFloatSemantics();
 
     Value real =
-        b.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(b, loc, elementType, adaptor.getComplex());
     Value imag =
-        b.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negOne = b.create<arith::ConstantOp>(
+        complex::ImOp::create(b, loc, elementType, adaptor.getComplex());
+    Value negOne = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -1.0));
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
       std::swap(real, imag);
-      real = b.create<arith::MulFOp>(real, negOne, fmf);
+      real = arith::MulFOp::create(b, real, negOne, fmf);
     }
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value four = b.create<arith::ConstantOp>(elementType,
+    Value four = arith::ConstantOp::create(b, elementType,
                                              b.getFloatAttr(elementType, 4.0));
-    Value twoReal = b.create<arith::AddFOp>(real, real, fmf);
-    Value negTwoReal = b.create<arith::MulFOp>(negOne, twoReal, fmf);
+    Value twoReal = arith::AddFOp::create(b, real, real, fmf);
+    Value negTwoReal = arith::MulFOp::create(b, negOne, twoReal, fmf);
 
-    Value expTwoRealMinusOne = b.create<math::ExpM1Op>(twoReal, fmf);
-    Value expNegTwoRealMinusOne = b.create<math::ExpM1Op>(negTwoReal, fmf);
+    Value expTwoRealMinusOne = math::ExpM1Op::create(b, twoReal, fmf);
+    Value expNegTwoRealMinusOne = math::ExpM1Op::create(b, negTwoReal, fmf);
     Value realNum =
-        b.create<arith::SubFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+        arith::SubFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
 
-    Value cosImag = b.create<math::CosOp>(imag, fmf);
-    Value cosImagSq = b.create<arith::MulFOp>(cosImag, cosImag, fmf);
-    Value twoCosTwoImagPlusOne = b.create<arith::MulFOp>(cosImagSq, four, fmf);
-    Value sinImag = b.create<math::SinOp>(imag, fmf);
+    Value cosImag = math::CosOp::create(b, imag, fmf);
+    Value cosImagSq = arith::MulFOp::create(b, cosImag, cosImag, fmf);
+    Value twoCosTwoImagPlusOne = arith::MulFOp::create(b, cosImagSq, four, fmf);
+    Value sinImag = math::SinOp::create(b, imag, fmf);
 
-    Value imagNum = b.create<arith::MulFOp>(
-        four, b.create<arith::MulFOp>(cosImag, sinImag, fmf), fmf);
+    Value imagNum = arith::MulFOp::create(b,
+        four, arith::MulFOp::create(b, cosImag, sinImag, fmf), fmf);
 
     Value expSumMinusTwo =
-        b.create<arith::AddFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+        arith::AddFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
     Value denom =
-        b.create<arith::AddFOp>(expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
+        arith::AddFOp::create(b, expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
 
-    Value isInf = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+    Value isInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
                                           expSumMinusTwo, inf, fmf);
-    Value realLimit = b.create<math::CopySignOp>(negOne, real, fmf);
+    Value realLimit = math::CopySignOp::create(b, negOne, real, fmf);
 
-    Value resultReal = b.create<arith::SelectOp>(
-        isInf, realLimit, b.create<arith::DivFOp>(realNum, denom, fmf));
-    Value resultImag = b.create<arith::DivFOp>(imagNum, denom, fmf);
+    Value resultReal = arith::SelectOp::create(b,
+        isInf, realLimit, arith::DivFOp::create(b, realNum, denom, fmf));
+    Value resultImag = arith::DivFOp::create(b, imagNum, denom, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value zero = b.create<arith::ConstantOp>(
+      Value absReal = math::AbsFOp::create(b, real, fmf);
+      Value zero = arith::ConstantOp::create(b,
           elementType, b.getFloatAttr(elementType, 0.0));
       Value nan = cst(APFloat::getNaN(floatSemantics));
 
       Value absRealIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
       Value imagIsZero =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-      Value absRealIsNotInf = b.create<arith::XOrIOp>(
-          absRealIsInf, b.create<arith::ConstantIntOp>(true, /*width=*/1));
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
+      Value absRealIsNotInf = arith::XOrIOp::create(b,
+          absRealIsInf, arith::ConstantIntOp::create(b, true, /*width=*/1));
 
-      Value imagNumIsNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO,
+      Value imagNumIsNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO,
                                                    imagNum, imagNum, fmf);
       Value resultRealIsNaN =
-          b.create<arith::AndIOp>(imagNumIsNaN, absRealIsNotInf);
-      Value resultImagIsZero = b.create<arith::OrIOp>(
-          imagIsZero, b.create<arith::AndIOp>(absRealIsInf, imagNumIsNaN));
+          arith::AndIOp::create(b, imagNumIsNaN, absRealIsNotInf);
+      Value resultImagIsZero = arith::OrIOp::create(b,
+          imagIsZero, arith::AndIOp::create(b, absRealIsInf, imagNumIsNaN));
 
-      resultReal = b.create<arith::SelectOp>(resultRealIsNaN, nan, resultReal);
+      resultReal = arith::SelectOp::create(b, resultRealIsNaN, nan, resultReal);
       resultImag =
-          b.create<arith::SelectOp>(resultImagIsZero, zero, resultImag);
+          arith::SelectOp::create(b, resultImagIsZero, zero, resultImag);
     }
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
       std::swap(resultReal, resultImag);
-      resultImag = b.create<arith::MulFOp>(resultImag, negOne, fmf);
+      resultImag = arith::MulFOp::create(b, resultImag, negOne, fmf);
     }
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
@@ -799,10 +799,10 @@ struct ConjOpConversion : public OpConversionPattern<complex::ConjOp> {
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value negImag = rewriter.create<arith::NegFOp>(loc, elementType, imag);
+        complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex());
+    Value negImag = arith::NegFOp::create(rewriter, loc, elementType, imag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, negImag);
 
@@ -818,97 +818,97 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
                                  arith::FastMathFlags fmf) {
   auto elementType = cast<FloatType>(type.getElementType());
 
-  Value a = builder.create<complex::ReOp>(lhs);
-  Value b = builder.create<complex::ImOp>(lhs);
+  Value a = complex::ReOp::create(builder, lhs);
+  Value b = complex::ImOp::create(builder, lhs);
 
-  Value abs = builder.create<complex::AbsOp>(lhs, fmf);
-  Value absToC = builder.create<math::PowFOp>(abs, c, fmf);
+  Value abs = complex::AbsOp::create(builder, lhs, fmf);
+  Value absToC = math::PowFOp::create(builder, abs, c, fmf);
 
-  Value negD = builder.create<arith::NegFOp>(d, fmf);
-  Value argLhs = builder.create<math::Atan2Op>(b, a, fmf);
-  Value negDArgLhs = builder.create<arith::MulFOp>(negD, argLhs, fmf);
-  Value expNegDArgLhs = builder.create<math::ExpOp>(negDArgLhs, fmf);
+  Value negD = arith::NegFOp::create(builder, d, fmf);
+  Value argLhs = math::Atan2Op::create(builder, b, a, fmf);
+  Value negDArgLhs = arith::MulFOp::create(builder, negD, argLhs, fmf);
+  Value expNegDArgLhs = math::ExpOp::create(builder, negDArgLhs, fmf);
 
-  Value coeff = builder.create<arith::MulFOp>(absToC, expNegDArgLhs, fmf);
-  Value lnAbs = builder.create<math::LogOp>(abs, fmf);
-  Value cArgLhs = builder.create<arith::MulFOp>(c, argLhs, fmf);
-  Value dLnAbs = builder.create<arith::MulFOp>(d, lnAbs, fmf);
-  Value q = builder.create<arith::AddFOp>(cArgLhs, dLnAbs, fmf);
-  Value cosQ = builder.create<math::CosOp>(q, fmf);
-  Value sinQ = builder.create<math::SinOp>(q, fmf);
+  Value coeff = arith::MulFOp::create(builder, absToC, expNegDArgLhs, fmf);
+  Value lnAbs = math::LogOp::create(builder, abs, fmf);
+  Value cArgLhs = arith::MulFOp::create(builder, c, argLhs, fmf);
+  Value dLnAbs = arith::MulFOp::create(builder, d, lnAbs, fmf);
+  Value q = arith::AddFOp::create(builder, cArgLhs, dLnAbs, fmf);
+  Value cosQ = math::CosOp::create(builder, q, fmf);
+  Value sinQ = math::SinOp::create(builder, q, fmf);
 
-  Value inf = builder.create<arith::ConstantOp>(
+  Value inf = arith::ConstantOp::create(builder,
       elementType,
       builder.getFloatAttr(elementType,
                            APFloat::getInf(elementType.getFloatSemantics())));
-  Value zero = builder.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(builder,
       elementType, builder.getFloatAttr(elementType, 0.0));
-  Value one = builder.create<arith::ConstantOp>(
+  Value one = arith::ConstantOp::create(builder,
       elementType, builder.getFloatAttr(elementType, 1.0));
-  Value complexOne = builder.create<complex::CreateOp>(type, one, zero);
-  Value complexZero = builder.create<complex::CreateOp>(type, zero, zero);
-  Value complexInf = builder.create<complex::CreateOp>(type, inf, zero);
+  Value complexOne = complex::CreateOp::create(builder, type, one, zero);
+  Value complexZero = complex::CreateOp::create(builder, type, zero, zero);
+  Value complexInf = complex::CreateOp::create(builder, type, inf, zero);
 
   // Case 0:
   // d^c is 0 if d is 0 and c > 0. 0^0 is defined to be 1.0, see
   // Branch Cuts for Complex Elementary Functions or Much Ado About
   // Nothing's Sign Bit, W. Kahan, Section 10.
   Value absEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, abs, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, abs, zero, fmf);
   Value dEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, d, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, d, zero, fmf);
   Value cEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, c, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, c, zero, fmf);
   Value bEqZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, b, zero, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, b, zero, fmf);
 
   Value zeroLeC =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLE, zero, c, fmf);
-  Value coeffCosQ = builder.create<arith::MulFOp>(coeff, cosQ, fmf);
-  Value coeffSinQ = builder.create<arith::MulFOp>(coeff, sinQ, fmf);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLE, zero, c, fmf);
+  Value coeffCosQ = arith::MulFOp::create(builder, coeff, cosQ, fmf);
+  Value coeffSinQ = arith::MulFOp::create(builder, coeff, sinQ, fmf);
   Value complexOneOrZero =
-      builder.create<arith::SelectOp>(cEqZero, complexOne, complexZero);
+      arith::SelectOp::create(builder, cEqZero, complexOne, complexZero);
   Value coeffCosSin =
-      builder.create<complex::CreateOp>(type, coeffCosQ, coeffSinQ);
-  Value cutoff0 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(
-          builder.create<arith::AndIOp>(absEqZero, dEqZero), zeroLeC),
+      complex::CreateOp::create(builder, type, coeffCosQ, coeffSinQ);
+  Value cutoff0 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder,
+          arith::AndIOp::create(builder, absEqZero, dEqZero), zeroLeC),
       complexOneOrZero, coeffCosSin);
 
   // Case 1:
   // x^0 is defined to be 1 for any x, see
   // Branch Cuts for Complex Elementary Functions or Much Ado About
   // Nothing's Sign Bit, W. Kahan, Section 10.
-  Value rhsEqZero = builder.create<arith::AndIOp>(cEqZero, dEqZero);
+  Value rhsEqZero = arith::AndIOp::create(builder, cEqZero, dEqZero);
   Value cutoff1 =
-      builder.create<arith::SelectOp>(rhsEqZero, complexOne, cutoff0);
+      arith::SelectOp::create(builder, rhsEqZero, complexOne, cutoff0);
 
   // Case 2:
   // 1^(c + d*i) = 1 + 0*i
-  Value lhsEqOne = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, one, fmf),
+  Value lhsEqOne = arith::AndIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, one, fmf),
       bEqZero);
   Value cutoff2 =
-      builder.create<arith::SelectOp>(lhsEqOne, complexOne, cutoff1);
+      arith::SelectOp::create(builder, lhsEqOne, complexOne, cutoff1);
 
   // Case 3:
   // inf^(c + 0*i) = inf + 0*i, c > 0
-  Value lhsEqInf = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, inf, fmf),
+  Value lhsEqInf = arith::AndIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, inf, fmf),
       bEqZero);
-  Value rhsGt0 = builder.create<arith::AndIOp>(
+  Value rhsGt0 = arith::AndIOp::create(builder,
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, c, zero, fmf));
-  Value cutoff3 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(lhsEqInf, rhsGt0), complexInf, cutoff2);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, c, zero, fmf));
+  Value cutoff3 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder, lhsEqInf, rhsGt0), complexInf, cutoff2);
 
   // Case 4:
   // inf^(c + 0*i) = 0 + 0*i, c < 0
-  Value rhsLt0 = builder.create<arith::AndIOp>(
+  Value rhsLt0 = arith::AndIOp::create(builder,
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, c, zero, fmf));
-  Value cutoff4 = builder.create<arith::SelectOp>(
-      builder.create<arith::AndIOp>(lhsEqInf, rhsLt0), complexZero, cutoff3);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, c, zero, fmf));
+  Value cutoff4 = arith::SelectOp::create(builder,
+      arith::AndIOp::create(builder, lhsEqInf, rhsLt0), complexZero, cutoff3);
 
   return cutoff4;
 }
@@ -923,8 +923,8 @@ struct PowOpConversion : public OpConversionPattern<complex::PowOp> {
     auto type = cast<ComplexType>(adaptor.getLhs().getType());
     auto elementType = cast<FloatType>(type.getElementType());
 
-    Value c = builder.create<complex::ReOp>(elementType, adaptor.getRhs());
-    Value d = builder.create<complex::ImOp>(elementType, adaptor.getRhs());
+    Value c = complex::ReOp::create(builder, elementType, adaptor.getRhs());
+    Value d = complex::ImOp::create(builder, elementType, adaptor.getRhs());
 
     rewriter.replaceOp(op, {powOpConversionImpl(builder, type, adaptor.getLhs(),
                                                 c, d, op.getFastmath())});
@@ -945,64 +945,64 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
     auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
+      return arith::ConstantOp::create(b, elementType,
                                          b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value negHalf = b.create<arith::ConstantOp>(
+    Value negHalf = arith::ConstantOp::create(b,
         elementType, b.getFloatAttr(elementType, -0.5));
     Value nan = cst(APFloat::getNaN(floatSemantics));
 
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
+    Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
     Value absRsqrt = computeAbs(real, imag, fmf, b, AbsFn::rsqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value rsqrtArg = b.create<arith::MulFOp>(argArg, negHalf, fmf);
-    Value cos = b.create<math::CosOp>(rsqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(rsqrtArg, fmf);
+    Value argArg = math::Atan2Op::create(b, imag, real, fmf);
+    Value rsqrtArg = arith::MulFOp::create(b, argArg, negHalf, fmf);
+    Value cos = math::CosOp::create(b, rsqrtArg, fmf);
+    Value sin = math::SinOp::create(b, rsqrtArg, fmf);
 
-    Value resultReal = b.create<arith::MulFOp>(absRsqrt, cos, fmf);
-    Value resultImag = b.create<arith::MulFOp>(absRsqrt, sin, fmf);
+    Value resultReal = arith::MulFOp::create(b, absRsqrt, cos, fmf);
+    Value resultImag = arith::MulFOp::create(b, absRsqrt, sin, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value negOne = b.create<arith::ConstantOp>(
+      Value negOne = arith::ConstantOp::create(b,
           elementType, b.getFloatAttr(elementType, -1));
 
-      Value realSignedZero = b.create<math::CopySignOp>(zero, real, fmf);
-      Value imagSignedZero = b.create<math::CopySignOp>(zero, imag, fmf);
+      Value realSignedZero = math::CopySignOp::create(b, zero, real, fmf);
+      Value imagSignedZero = math::CopySignOp::create(b, zero, imag, fmf);
       Value negImagSignedZero =
-          b.create<arith::MulFOp>(negOne, imagSignedZero, fmf);
+          arith::MulFOp::create(b, negOne, imagSignedZero, fmf);
 
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value absImag = b.create<math::AbsFOp>(imag, fmf);
+      Value absReal = math::AbsFOp::create(b, real, fmf);
+      Value absImag = math::AbsFOp::create(b, imag, fmf);
 
       Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
       Value realIsNan =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, real, real, fmf);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, real, real, fmf);
       Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
-      Value inIsNanInf = b.create<arith::AndIOp>(absImagIsInf, realIsNan);
+          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value inIsNanInf = arith::AndIOp::create(b, absImagIsInf, realIsNan);
 
-      Value resultIsZero = b.create<arith::OrIOp>(inIsNanInf, realIsInf);
+      Value resultIsZero = arith::OrIOp::create(b, inIsNanInf, realIsInf);
 
       resultReal =
-          b.create<arith::SelectOp>(resultIsZero, realSignedZero, resultReal);
-      resultImag = b.create<arith::SelectOp>(resultIsZero, negImagSignedZero,
+          arith::SelectOp::create(b, resultIsZero, realSignedZero, resultReal);
+      resultImag = arith::SelectOp::create(b, resultIsZero, negImagSignedZero,
                                              resultImag);
     }
 
     Value isRealZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero, fmf);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, zero, fmf);
     Value isImagZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-    Value isZero = b.create<arith::AndIOp>(isRealZero, isImagZero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
+    Value isZero = arith::AndIOp::create(b, isRealZero, isImagZero);
 
-    resultReal = b.create<arith::SelectOp>(isZero, inf, resultReal);
-    resultImag = b.create<arith::SelectOp>(isZero, nan, resultImag);
+    resultReal = arith::SelectOp::create(b, isZero, inf, resultReal);
+    resultImag = arith::SelectOp::create(b, isZero, nan, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -1021,9 +1021,9 @@ struct AngleOpConversion : public OpConversionPattern<complex::AngleOp> {
     arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
-        rewriter.create<complex::ReOp>(loc, type, adaptor.getComplex());
+        complex::ReOp::create(rewriter, loc, type, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, type, adaptor.getComplex());
+        complex::ImOp::create(rewriter, loc, type, adaptor.getComplex());
 
     rewriter.replaceOpWithNewOp<math::Atan2Op>(op, imag, real, fmf);
 
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index 13a084407e53f..5c3d26874faed 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -73,13 +73,13 @@ struct AssertOpLowering : public ConvertOpToLLVMPattern<cf::AssertOp> {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
         auto abortFuncTy = LLVM::LLVMFunctionType::get(getVoidType(), {});
-        abortFunc = rewriter.create<LLVM::LLVMFuncOp>(rewriter.getUnknownLoc(),
+        abortFunc = LLVM::LLVMFuncOp::create(rewriter, rewriter.getUnknownLoc(),
                                                       "abort", abortFuncTy);
       }
-      rewriter.create<LLVM::CallOp>(loc, abortFunc, ValueRange());
-      rewriter.create<LLVM::UnreachableOp>(loc);
+      LLVM::CallOp::create(rewriter, loc, abortFunc, ValueRange());
+      LLVM::UnreachableOp::create(rewriter, loc);
     } else {
-      rewriter.create<LLVM::BrOp>(loc, ValueRange(), continuationBlock);
+      LLVM::BrOp::create(rewriter, loc, ValueRange(), continuationBlock);
     }
 
     // Generate assertion test.
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index 9831dcaaaccc8..aa468c29536c7 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -33,7 +33,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
     MutableArrayRef<Region> regions) {
   if (auto condBrOp = dyn_cast<cf::CondBranchOp>(controlFlowCondOp)) {
     assert(regions.size() == 2);
-    auto ifOp = builder.create<scf::IfOp>(controlFlowCondOp->getLoc(),
+    auto ifOp = scf::IfOp::create(builder, controlFlowCondOp->getLoc(),
                                           resultTypes, condBrOp.getCondition());
     ifOp.getThenRegion().takeBody(regions[0]);
     ifOp.getElseRegion().takeBody(regions[1]);
@@ -43,7 +43,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
   if (auto switchOp = dyn_cast<cf::SwitchOp>(controlFlowCondOp)) {
     // `getCFGSwitchValue` returns an i32 that we need to convert to index
     // fist.
-    auto cast = builder.create<arith::IndexCastUIOp>(
+    auto cast = arith::IndexCastUIOp::create(builder,
         controlFlowCondOp->getLoc(), builder.getIndexType(),
         switchOp.getFlag());
     SmallVector<int64_t> cases;
@@ -55,7 +55,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
 
     assert(regions.size() == cases.size() + 1);
 
-    auto indexSwitchOp = builder.create<scf::IndexSwitchOp>(
+    auto indexSwitchOp = scf::IndexSwitchOp::create(builder,
         controlFlowCondOp->getLoc(), resultTypes, cast, cases, cases.size());
 
     indexSwitchOp.getDefaultRegion().takeBody(regions[0]);
@@ -75,7 +75,7 @@ LogicalResult
 ControlFlowToSCFTransformation::createStructuredBranchRegionTerminatorOp(
     Location loc, OpBuilder &builder, Operation *branchRegionOp,
     Operation *replacedControlFlowOp, ValueRange results) {
-  builder.create<scf::YieldOp>(loc, results);
+  scf::YieldOp::create(builder, loc, results);
   return success();
 }
 
@@ -84,7 +84,7 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
     OpBuilder &builder, Operation *replacedOp, ValueRange loopVariablesInit,
     Value condition, ValueRange loopVariablesNextIter, Region &&loopBody) {
   Location loc = replacedOp->getLoc();
-  auto whileOp = builder.create<scf::WhileOp>(loc, loopVariablesInit.getTypes(),
+  auto whileOp = scf::WhileOp::create(builder, loc, loopVariablesInit.getTypes(),
                                               loopVariablesInit);
 
   whileOp.getBefore().takeBody(loopBody);
@@ -92,15 +92,15 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
   builder.setInsertionPointToEnd(&whileOp.getBefore().back());
   // `getCFGSwitchValue` returns a i32. We therefore need to truncate the
   // condition to i1 first. It is guaranteed to be either 0 or 1 already.
-  builder.create<scf::ConditionOp>(
-      loc, builder.create<arith::TruncIOp>(loc, builder.getI1Type(), condition),
+  scf::ConditionOp::create(builder,
+      loc, arith::TruncIOp::create(builder, loc, builder.getI1Type(), condition),
       loopVariablesNextIter);
 
   Block *afterBlock = builder.createBlock(&whileOp.getAfter());
   afterBlock->addArguments(
       loopVariablesInit.getTypes(),
       SmallVector<Location>(loopVariablesInit.size(), loc));
-  builder.create<scf::YieldOp>(loc, afterBlock->getArguments());
+  scf::YieldOp::create(builder, loc, afterBlock->getArguments());
 
   return whileOp.getOperation();
 }
@@ -108,7 +108,7 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
 Value ControlFlowToSCFTransformation::getCFGSwitchValue(Location loc,
                                                         OpBuilder &builder,
                                                         unsigned int value) {
-  return builder.create<arith::ConstantOp>(loc,
+  return arith::ConstantOp::create(builder, loc,
                                            builder.getI32IntegerAttr(value));
 }
 
@@ -117,7 +117,7 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
     ArrayRef<unsigned int> caseValues, BlockRange caseDestinations,
     ArrayRef<ValueRange> caseArguments, Block *defaultDest,
     ValueRange defaultArgs) {
-  builder.create<cf::SwitchOp>(loc, flag, defaultDest, defaultArgs,
+  cf::SwitchOp::create(builder, loc, flag, defaultDest, defaultArgs,
                                llvm::to_vector_of<int32_t>(caseValues),
                                caseDestinations, caseArguments);
 }
@@ -125,7 +125,7 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
 Value ControlFlowToSCFTransformation::getUndefValue(Location loc,
                                                     OpBuilder &builder,
                                                     Type type) {
-  return builder.create<ub::PoisonOp>(loc, type, nullptr);
+  return ub::PoisonOp::create(builder, loc, type, nullptr);
 }
 
 FailureOr<Operation *>
diff --git a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
index f8dc06f41ab87..20167f168185a 100644
--- a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
+++ b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
@@ -99,7 +99,7 @@ class FuncOpConversion final : public OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted `emitc.func` op.
-    emitc::FuncOp newFuncOp = rewriter.create<emitc::FuncOp>(
+    emitc::FuncOp newFuncOp = emitc::FuncOp::create(rewriter,
         funcOp.getLoc(), funcOp.getName(),
         FunctionType::get(rewriter.getContext(),
                           signatureConverter.getConvertedTypes(),
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 36235636d6ba2..51dbe6cbb7de3 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -115,7 +115,7 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   SmallVector<NamedAttribute> attributes;
   filterFuncAttributes(funcOp, attributes);
 
-  auto wrapperFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto wrapperFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperFuncType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
@@ -129,13 +129,13 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   for (auto [index, argType] : llvm::enumerate(type.getInputs())) {
     Value arg = wrapperFuncOp.getArgument(index + argOffset);
     if (auto memrefType = dyn_cast<MemRefType>(argType)) {
-      Value loaded = rewriter.create<LLVM::LoadOp>(
+      Value loaded = LLVM::LoadOp::create(rewriter,
           loc, typeConverter.convertType(memrefType), arg);
       MemRefDescriptor::unpack(rewriter, loc, loaded, memrefType, args);
       continue;
     }
     if (isa<UnrankedMemRefType>(argType)) {
-      Value loaded = rewriter.create<LLVM::LoadOp>(
+      Value loaded = LLVM::LoadOp::create(rewriter,
           loc, typeConverter.convertType(argType), arg);
       UnrankedMemRefDescriptor::unpack(rewriter, loc, loaded, args);
       continue;
@@ -144,14 +144,14 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
     args.push_back(arg);
   }
 
-  auto call = rewriter.create<LLVM::CallOp>(loc, newFuncOp, args);
+  auto call = LLVM::CallOp::create(rewriter, loc, newFuncOp, args);
 
   if (resultStructType) {
-    rewriter.create<LLVM::StoreOp>(loc, call.getResult(),
+    LLVM::StoreOp::create(rewriter, loc, call.getResult(),
                                    wrapperFuncOp.getArgument(0));
-    rewriter.create<LLVM::ReturnOp>(loc, ValueRange{});
+    LLVM::ReturnOp::create(rewriter, loc, ValueRange{});
   } else {
-    rewriter.create<LLVM::ReturnOp>(loc, call.getResults());
+    LLVM::ReturnOp::create(rewriter, loc, call.getResults());
   }
 }
 
@@ -182,7 +182,7 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   filterFuncAttributes(funcOp, attributes);
 
   // Create the auxiliary function.
-  auto wrapperFunc = builder.create<LLVM::LLVMFuncOp>(
+  auto wrapperFunc = LLVM::LLVMFuncOp::create(builder,
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
@@ -201,11 +201,11 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   if (resultStructType) {
     // Allocate the struct on the stack and pass the pointer.
     Type resultType = cast<LLVM::LLVMFunctionType>(wrapperType).getParamType(0);
-    Value one = builder.create<LLVM::ConstantOp>(
+    Value one = LLVM::ConstantOp::create(builder,
         loc, typeConverter.convertType(builder.getIndexType()),
         builder.getIntegerAttr(builder.getIndexType(), 1));
     Value result =
-        builder.create<LLVM::AllocaOp>(loc, resultType, resultStructType, one);
+        LLVM::AllocaOp::create(builder, loc, resultType, resultStructType, one);
     args.push_back(result);
   }
 
@@ -229,12 +229,12 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
                     wrapperArgsRange.take_front(numToDrop));
 
       auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext());
-      Value one = builder.create<LLVM::ConstantOp>(
+      Value one = LLVM::ConstantOp::create(builder,
           loc, typeConverter.convertType(builder.getIndexType()),
           builder.getIntegerAttr(builder.getIndexType(), 1));
-      Value allocated = builder.create<LLVM::AllocaOp>(
+      Value allocated = LLVM::AllocaOp::create(builder,
           loc, ptrTy, packed.getType(), one, /*alignment=*/0);
-      builder.create<LLVM::StoreOp>(loc, packed, allocated);
+      LLVM::StoreOp::create(builder, loc, packed, allocated);
       arg = allocated;
     } else {
       arg = wrapperArgsRange[0];
@@ -245,14 +245,14 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   }
   assert(wrapperArgsRange.empty() && "did not map some of the arguments");
 
-  auto call = builder.create<LLVM::CallOp>(loc, wrapperFunc, args);
+  auto call = LLVM::CallOp::create(builder, loc, wrapperFunc, args);
 
   if (resultStructType) {
     Value result =
-        builder.create<LLVM::LoadOp>(loc, resultStructType, args.front());
-    builder.create<LLVM::ReturnOp>(loc, result);
+        LLVM::LoadOp::create(builder, loc, resultStructType, args.front());
+    LLVM::ReturnOp::create(builder, loc, result);
   } else {
-    builder.create<LLVM::ReturnOp>(loc, call.getResults());
+    LLVM::ReturnOp::create(builder, loc, call.getResults());
   }
 }
 
@@ -283,7 +283,7 @@ static void restoreByValRefArgumentType(
     Type resTy = typeConverter.convertType(
         cast<TypeAttr>(byValRefAttr->getValue()).getValue());
 
-    Value valueArg = rewriter.create<LLVM::LoadOp>(arg.getLoc(), resTy, arg);
+    Value valueArg = LLVM::LoadOp::create(rewriter, arg.getLoc(), resTy, arg);
     rewriter.replaceUsesOfBlockArgument(arg, valueArg);
   }
 }
@@ -357,7 +357,7 @@ FailureOr<LLVM::LLVMFuncOp> mlir::convertFuncOpToLLVMFuncOp(
     symbolTable.remove(funcOp);
   }
 
-  auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
       /*dsoLocal=*/false, /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr,
       attributes);
@@ -509,7 +509,7 @@ struct ConstantOpLowering : public ConvertOpToLLVMPattern<func::ConstantOp> {
       return rewriter.notifyMatchFailure(op, "failed to convert result type");
 
     auto newOp =
-        rewriter.create<LLVM::AddressOfOp>(op.getLoc(), type, op.getValue());
+        LLVM::AddressOfOp::create(rewriter, op.getLoc(), type, op.getValue());
     for (const NamedAttribute &attr : op->getAttrs()) {
       if (attr.getName().strref() == "value")
         continue;
@@ -556,7 +556,7 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
     auto promoted = this->getTypeConverter()->promoteOperands(
         callOp.getLoc(), /*opOperands=*/callOp->getOperands(),
         adaptor.getOperands(), rewriter, useBarePtrCallConv);
-    auto newOp = rewriter.create<LLVM::CallOp>(
+    auto newOp = LLVM::CallOp::create(rewriter,
         callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
         promoted, callOp->getAttrs());
 
@@ -573,7 +573,7 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
       // Extract individual results from the structure and return them as list.
       results.reserve(numResults);
       for (unsigned i = 0; i < numResults; ++i) {
-        results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        results.push_back(LLVM::ExtractValueOp::create(rewriter,
             callOp.getLoc(), newOp->getResult(0), i));
       }
     }
@@ -726,9 +726,9 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<func::ReturnOp> {
       return rewriter.notifyMatchFailure(op, "could not convert result types");
     }
 
-    Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
+    Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
     for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
-      packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
+      packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
     }
     rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, TypeRange(), packed,
                                                 op->getAttrs());
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index 01ca5e99a9aff..f5d5460412da2 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -28,7 +28,7 @@ LLVM::LLVMFuncOp mlir::getOrDefineFunction(gpu::GPUModuleOp moduleOp,
   if (!(ret = moduleOp.template lookupSymbol<LLVM::LLVMFuncOp>(name))) {
     OpBuilder::InsertionGuard guard(b);
     b.setInsertionPointToStart(moduleOp.getBody());
-    ret = b.create<LLVM::LLVMFuncOp>(loc, name, type, LLVM::Linkage::External);
+    ret = LLVM::LLVMFuncOp::create(b, loc, name, type, LLVM::Linkage::External);
   }
   return ret;
 }
@@ -68,7 +68,7 @@ mlir::getOrCreateStringConstant(OpBuilder &b, Location loc,
   OpBuilder::InsertionGuard guard(b);
   b.setInsertionPointToStart(moduleOp.getBody());
   SmallString<16> name = getUniqueSymbolName(moduleOp, namePrefix);
-  return b.create<LLVM::GlobalOp>(loc, globalType,
+  return LLVM::GlobalOp::create(b, loc, globalType,
                                   /*isConstant=*/true, LLVM::Linkage::Internal,
                                   name, attr, alignment, addrSpace);
 }
@@ -151,7 +151,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
               gpuFuncOp.getWorkgroupAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      auto globalOp = rewriter.create<LLVM::GlobalOp>(
+      auto globalOp = LLVM::GlobalOp::create(rewriter,
           gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
           LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment,
           workgroupAddrSpace);
@@ -220,7 +220,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
   LLVM::CConv callingConvention = gpuFuncOp.isKernel()
                                       ? kernelCallingConvention
                                       : nonKernelCallingConvention;
-  auto llvmFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+  auto llvmFuncOp = LLVM::LLVMFuncOp::create(rewriter,
       gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
       LLVM::Linkage::External, /*dsoLocal=*/false, callingConvention,
       /*comdat=*/nullptr, attributes);
@@ -266,10 +266,10 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       for (const auto [idx, global] : llvm::enumerate(workgroupBuffers)) {
         auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext(),
                                                   global.getAddrSpace());
-        Value address = rewriter.create<LLVM::AddressOfOp>(
+        Value address = LLVM::AddressOfOp::create(rewriter,
             loc, ptrType, global.getSymNameAttr());
         Value memory =
-            rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getType(),
+            LLVM::GEPOp::create(rewriter, loc, ptrType, global.getType(),
                                          address, ArrayRef<LLVM::GEPArg>{0, 0});
 
         // Build a memref descriptor pointing to the buffer to plug with the
@@ -298,14 +298,14 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       Type elementType = typeConverter->convertType(type.getElementType());
       auto ptrType =
           LLVM::LLVMPointerType::get(rewriter.getContext(), allocaAddrSpace);
-      Value numElements = rewriter.create<LLVM::ConstantOp>(
+      Value numElements = LLVM::ConstantOp::create(rewriter,
           gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
       uint64_t alignment = 0;
       if (auto alignAttr =
               dyn_cast_or_null<IntegerAttr>(gpuFuncOp.getPrivateAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      Value allocated = rewriter.create<LLVM::AllocaOp>(
+      Value allocated = LLVM::AllocaOp::create(rewriter,
           gpuFuncOp.getLoc(), ptrType, elementType, numElements, alignment);
       Value descr = MemRefDescriptor::fromStaticShape(
           rewriter, loc, *getTypeConverter(), type, allocated);
@@ -418,8 +418,8 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
           {llvmI64, ptrType, /*length (bytes)*/ llvmI64, /*isLast*/ llvmI32}));
 
   /// Start the printf hostcall
-  Value zeroI64 = rewriter.create<LLVM::ConstantOp>(loc, llvmI64, 0);
-  auto printfBeginCall = rewriter.create<LLVM::CallOp>(loc, ocklBegin, zeroI64);
+  Value zeroI64 = LLVM::ConstantOp::create(rewriter, loc, llvmI64, 0);
+  auto printfBeginCall = LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
   Value printfDesc = printfBeginCall.getResult();
 
   // Create the global op or find an existing one.
@@ -427,20 +427,20 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
       rewriter, loc, moduleOp, llvmI8, "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element and pass it to printf()
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
       loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
-  Value stringLen = rewriter.create<LLVM::ConstantOp>(
+  Value stringLen = LLVM::ConstantOp::create(rewriter,
       loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
 
-  Value oneI32 = rewriter.create<LLVM::ConstantOp>(loc, llvmI32, 1);
-  Value zeroI32 = rewriter.create<LLVM::ConstantOp>(loc, llvmI32, 0);
+  Value oneI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 1);
+  Value zeroI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 0);
 
-  auto appendFormatCall = rewriter.create<LLVM::CallOp>(
+  auto appendFormatCall = LLVM::CallOp::create(rewriter,
       loc, ocklAppendStringN,
       ValueRange{printfDesc, stringStart, stringLen,
                  adaptor.getArgs().empty() ? oneI32 : zeroI32});
@@ -456,17 +456,17 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
     SmallVector<mlir::Value, 2 + argsPerAppend + 1> arguments;
     arguments.push_back(printfDesc);
     arguments.push_back(
-        rewriter.create<LLVM::ConstantOp>(loc, llvmI32, numArgsThisCall));
+        LLVM::ConstantOp::create(rewriter, loc, llvmI32, numArgsThisCall));
     for (size_t i = group; i < bound; ++i) {
       Value arg = adaptor.getArgs()[i];
       if (auto floatType = dyn_cast<FloatType>(arg.getType())) {
         if (!floatType.isF64())
-          arg = rewriter.create<LLVM::FPExtOp>(
+          arg = LLVM::FPExtOp::create(rewriter,
               loc, typeConverter->convertType(rewriter.getF64Type()), arg);
-        arg = rewriter.create<LLVM::BitcastOp>(loc, llvmI64, arg);
+        arg = LLVM::BitcastOp::create(rewriter, loc, llvmI64, arg);
       }
       if (arg.getType().getIntOrFloatBitWidth() != 64)
-        arg = rewriter.create<LLVM::ZExtOp>(loc, llvmI64, arg);
+        arg = LLVM::ZExtOp::create(rewriter, loc, llvmI64, arg);
 
       arguments.push_back(arg);
     }
@@ -477,7 +477,7 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
 
     auto isLast = (bound == nArgs) ? oneI32 : zeroI32;
     arguments.push_back(isLast);
-    auto call = rewriter.create<LLVM::CallOp>(loc, ocklAppendArgs, arguments);
+    auto call = LLVM::CallOp::create(rewriter, loc, ocklAppendArgs, arguments);
     printfDesc = call.getResult();
   }
   rewriter.eraseOp(gpuPrintfOp);
@@ -510,12 +510,12 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
       /*alignment=*/0, addressSpace);
 
   // Get a pointer to the format string's first element
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
       loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
 
   // Construct arguments and function call
@@ -525,7 +525,7 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
   printfArgs.push_back(stringStart);
   printfArgs.append(argsRange.begin(), argsRange.end());
 
-  rewriter.create<LLVM::CallOp>(loc, printfDecl, printfArgs);
+  LLVM::CallOp::create(rewriter, loc, printfDecl, printfArgs);
   rewriter.eraseOp(gpuPrintfOp);
   return success();
 }
@@ -559,9 +559,9 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
                                 "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element
-  Value globalPtr = rewriter.create<LLVM::AddressOfOp>(loc, global);
+  Value globalPtr = LLVM::AddressOfOp::create(rewriter, loc, global);
   Value stringStart =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+      LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                    globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
   SmallVector<Type> types;
   SmallVector<Value> args;
@@ -572,27 +572,27 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
     assert(type.isIntOrFloat());
     if (isa<FloatType>(type)) {
       type = rewriter.getF64Type();
-      promotedArg = rewriter.create<LLVM::FPExtOp>(loc, type, arg);
+      promotedArg = LLVM::FPExtOp::create(rewriter, loc, type, arg);
     }
     types.push_back(type);
     args.push_back(promotedArg);
   }
   Type structType =
       LLVM::LLVMStructType::getLiteral(gpuPrintfOp.getContext(), types);
-  Value one = rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
+  Value one = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
                                                 rewriter.getIndexAttr(1));
   Value tempAlloc =
-      rewriter.create<LLVM::AllocaOp>(loc, ptrType, structType, one,
+      LLVM::AllocaOp::create(rewriter, loc, ptrType, structType, one,
                                       /*alignment=*/0);
   for (auto [index, arg] : llvm::enumerate(args)) {
-    Value ptr = rewriter.create<LLVM::GEPOp>(
+    Value ptr = LLVM::GEPOp::create(rewriter,
         loc, ptrType, structType, tempAlloc,
         ArrayRef<LLVM::GEPArg>{0, static_cast<int32_t>(index)});
-    rewriter.create<LLVM::StoreOp>(loc, arg, ptr);
+    LLVM::StoreOp::create(rewriter, loc, arg, ptr);
   }
   std::array<Value, 2> printfArgs = {stringStart, tempAlloc};
 
-  rewriter.create<LLVM::CallOp>(loc, vprintfDecl, printfArgs);
+  LLVM::CallOp::create(rewriter, loc, vprintfDecl, printfArgs);
   rewriter.eraseOp(gpuPrintfOp);
   return success();
 }
@@ -607,22 +607,22 @@ static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands,
   TypeRange operandTypes(operands);
   VectorType vectorType = cast<VectorType>(llvm1DVectorTy);
   Location loc = op->getLoc();
-  Value result = rewriter.create<LLVM::PoisonOp>(loc, vectorType);
+  Value result = LLVM::PoisonOp::create(rewriter, loc, vectorType);
   Type indexType = converter.convertType(rewriter.getIndexType());
   StringAttr name = op->getName().getIdentifier();
   Type elementType = vectorType.getElementType();
 
   for (int64_t i = 0; i < vectorType.getNumElements(); ++i) {
-    Value index = rewriter.create<LLVM::ConstantOp>(loc, indexType, i);
+    Value index = LLVM::ConstantOp::create(rewriter, loc, indexType, i);
     auto extractElement = [&](Value operand) -> Value {
       if (!isa<VectorType>(operand.getType()))
         return operand;
-      return rewriter.create<LLVM::ExtractElementOp>(loc, operand, index);
+      return LLVM::ExtractElementOp::create(rewriter, loc, operand, index);
     };
     auto scalarOperands = llvm::map_to_vector(operands, extractElement);
     Operation *scalarOp =
         rewriter.create(loc, name, scalarOperands, elementType, op->getAttrs());
-    result = rewriter.create<LLVM::InsertElementOp>(
+    result = LLVM::InsertElementOp::create(rewriter,
         loc, result, scalarOp->getResult(0), index);
   }
   return result;
@@ -705,7 +705,7 @@ LLVM::GlobalOp getDynamicSharedMemorySymbol(
   auto zeroSizedArrayType = LLVM::LLVMArrayType::get(
       typeConverter->convertType(memrefType.getElementType()), 0);
 
-  return rewriter.create<LLVM::GlobalOp>(
+  return LLVM::GlobalOp::create(rewriter,
       op->getLoc(), zeroSizedArrayType, /*isConstant=*/false,
       LLVM::Linkage::Internal, symName, /*value=*/Attribute(), alignmentByte,
       addressSpace.value());
@@ -732,12 +732,12 @@ LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
   // Step 3. Get address of the global symbol
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(op);
-  auto basePtr = rewriter.create<LLVM::AddressOfOp>(loc, shmemOp);
+  auto basePtr = LLVM::AddressOfOp::create(rewriter, loc, shmemOp);
   Type baseType = basePtr->getResultTypes().front();
 
   // Step 4. Generate GEP using offsets
   SmallVector<LLVM::GEPArg> gepArgs = {0};
-  Value shmemPtr = rewriter.create<LLVM::GEPOp>(loc, baseType, elementType,
+  Value shmemPtr = LLVM::GEPOp::create(rewriter, loc, baseType, elementType,
                                                 basePtr, gepArgs);
   // Step 5. Create a memref descriptor
   SmallVector<Value> shape, strides;
@@ -799,9 +799,9 @@ LogicalResult GPUReturnOpLowering::matchAndRewrite(
     return rewriter.notifyMatchFailure(op, "could not convert result types");
   }
 
-  Value packed = rewriter.create<LLVM::PoisonOp>(loc, packedType);
+  Value packed = LLVM::PoisonOp::create(rewriter, loc, packedType);
   for (auto [idx, operand] : llvm::enumerate(updatedOperands)) {
-    packed = rewriter.create<LLVM::InsertValueOp>(loc, packed, operand, idx);
+    packed = LLVM::InsertValueOp::create(rewriter, loc, packed, operand, idx);
   }
   rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, TypeRange(), packed,
                                               op->getAttrs());
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 167cabbc57db9..5e8254a0b6035 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -79,7 +79,7 @@ class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern<OpTy> {
     uint64_t rank = type.getRank();
     Value numElements = desc.size(rewriter, loc, /*pos=*/0);
     for (unsigned i = 1; i < rank; i++)
-      numElements = rewriter.create<LLVM::MulOp>(
+      numElements = LLVM::MulOp::create(rewriter,
           loc, numElements, desc.size(rewriter, loc, /*pos=*/i));
     return numElements;
   }
@@ -582,7 +582,7 @@ LLVM::CallOp FunctionCallBuilder::create(Location loc, OpBuilder &builder,
     return OpBuilder::atBlockEnd(module.getBody())
         .create<LLVM::LLVMFuncOp>(loc, functionName, functionType);
   }();
-  return builder.create<LLVM::CallOp>(loc, function, arguments);
+  return LLVM::CallOp::create(builder, loc, function, arguments);
 }
 
 // Corresponding to cusparseIndexType_t defined in cusparse.h.
@@ -780,12 +780,12 @@ LogicalResult ConvertAllocOpToGpuRuntimeCallPattern::matchAndRewrite(
 
   // Allocate the underlying buffer and store a pointer to it in the MemRef
   // descriptor.
-  auto nullPtr = rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPointerType);
+  auto nullPtr = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPointerType);
   Value stream = adaptor.getAsyncDependencies().empty()
                      ? nullPtr
                      : adaptor.getAsyncDependencies().front();
 
-  auto isHostShared = rewriter.create<mlir::LLVM::ConstantOp>(
+  auto isHostShared = mlir::LLVM::ConstantOp::create(rewriter,
       loc, llvmInt8Type, rewriter.getI8IntegerAttr(isShared));
 
   Value allocatedPtr =
@@ -1012,7 +1012,7 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
       uint64_t staticSize = static_cast<uint64_t>(bitwidth / 8) *
                             static_cast<uint64_t>(memrefTy.getNumElements());
 
-      Value sizeArg = rewriter.create<LLVM::ConstantOp>(
+      Value sizeArg = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(staticSize));
       llvmArgumentsWithSizes.push_back(llvmArg); // Presumably a bare pointer.
       llvmArgumentsWithSizes.push_back(sizeArg);
@@ -1025,7 +1025,7 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
         gpu::KernelDim3{adaptor.getClusterSizeX(), adaptor.getClusterSizeY(),
                         adaptor.getClusterSizeZ()};
   }
-  rewriter.create<gpu::LaunchFuncOp>(
+  gpu::LaunchFuncOp::create(rewriter,
       launchOp.getLoc(), launchOp.getKernelAttr(),
       gpu::KernelDim3{adaptor.getGridSizeX(), adaptor.getGridSizeY(),
                       adaptor.getGridSizeZ()},
@@ -1048,7 +1048,7 @@ static Value bitAndAddrspaceCast(Location loc,
                                  const LLVMTypeConverter &typeConverter) {
   auto sourceTy = cast<LLVM::LLVMPointerType>(sourcePtr.getType());
   if (destinationType.getAddressSpace() != sourceTy.getAddressSpace())
-    sourcePtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+    sourcePtr = LLVM::AddrSpaceCastOp::create(rewriter,
         loc,
         LLVM::LLVMPointerType::get(rewriter.getContext(),
                                    destinationType.getAddressSpace()),
@@ -1072,13 +1072,13 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite(
   Value numElements = getNumElements(rewriter, loc, memRefType, srcDesc);
 
   Type elementPtrType = getElementPtrType(memRefType);
-  Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, elementPtrType);
-  Value gepPtr = rewriter.create<LLVM::GEPOp>(
+  Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
+  Value gepPtr = LLVM::GEPOp::create(rewriter,
       loc, elementPtrType,
       typeConverter->convertType(memRefType.getElementType()), nullPtr,
       numElements);
   auto sizeBytes =
-      rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);
+      LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
 
   auto src = bitAndAddrspaceCast(loc, rewriter, llvmPointerType,
                                  srcDesc.alignedPtr(rewriter, loc),
@@ -1123,7 +1123,7 @@ LogicalResult ConvertMemsetOpToGpuRuntimeCallPattern::matchAndRewrite(
   Value numElements = getNumElements(rewriter, loc, memRefType, dstDesc);
 
   auto value =
-      rewriter.create<LLVM::BitcastOp>(loc, bitCastType, adaptor.getValue());
+      LLVM::BitcastOp::create(rewriter, loc, bitCastType, adaptor.getValue());
   auto dst = bitAndAddrspaceCast(loc, rewriter, llvmPointerType,
                                  dstDesc.alignedPtr(rewriter, loc),
                                  *getTypeConverter());
@@ -1150,14 +1150,14 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite(
 template <typename T>
 static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmInt32Type = builder.getIntegerType(32);
-  return builder.create<LLVM::ConstantOp>(loc, llvmInt32Type,
+  return LLVM::ConstantOp::create(builder, loc, llvmInt32Type,
                                           static_cast<int32_t>(tValue));
 }
 
 template <typename T>
 static Value genConstFloat32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmFloat32Type = builder.getF32Type();
-  return builder.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(builder,
       loc, llvmFloat32Type,
       builder.getF32FloatAttr(static_cast<float>(tValue)));
 }
@@ -1189,11 +1189,11 @@ LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite(
   // the dnmat is used with spmat with 2:4 sparsity
   if (dims.size() == 2) {
     if (isSpMMCusparseLtOp(op.getDnTensor())) {
-      auto handleSz = rewriter.create<LLVM::ConstantOp>(
+      auto handleSz = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(11032));
-      handle = rewriter.create<LLVM::AllocaOp>(
+      handle = LLVM::AllocaOp::create(rewriter,
           loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
-      handle = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, handle);
+      handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
       createLtDnMatCallBuilder
           .create(loc, rewriter,
@@ -1351,11 +1351,11 @@ LogicalResult ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto dtp = genConstInt32From(rewriter, loc, getCuSparseDataTypeFrom(dType));
 
   // CUDA runner asserts the size is 44104 bytes.
-  auto handleSz = rewriter.create<LLVM::ConstantOp>(
+  auto handleSz = LLVM::ConstantOp::create(rewriter,
       loc, getIndexType(), rewriter.getIndexAttr(44104));
-  Value handle = rewriter.create<LLVM::AllocaOp>(
+  Value handle = LLVM::AllocaOp::create(rewriter,
       loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
-  handle = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, handle);
+  handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
   create2To4SpMatCallBuilder
       .create(loc, rewriter,
@@ -1441,9 +1441,9 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
         genConstInt32From(rewriter, loc, get2To4PruneFlag(op.getSpmatA()));
     auto computeType = genConstInt32From(
         rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType()));
-    auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+    auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(3));
-    auto bufferSize = rewriter.create<LLVM::AllocaOp>(
+    auto bufferSize = LLVM::AllocaOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, three, /*alignment=*/16);
     createCuSparseLtSpMMBufferSizeBuilder
         .create(loc, rewriter,
@@ -1452,20 +1452,20 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
                  pruneFlag, stream})
         .getResult();
 
-    auto bufferSizePtr1 = rewriter.create<LLVM::GEPOp>(
+    auto bufferSizePtr1 = LLVM::GEPOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{rewriter.create<LLVM::ConstantOp>(
+        ValueRange{LLVM::ConstantOp::create(rewriter,
             loc, getIndexType(), rewriter.getIndexAttr(1))});
-    auto bufferSizePtr2 = rewriter.create<LLVM::GEPOp>(
+    auto bufferSizePtr2 = LLVM::GEPOp::create(rewriter,
         loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{rewriter.create<LLVM::ConstantOp>(
+        ValueRange{LLVM::ConstantOp::create(rewriter,
             loc, getIndexType(), rewriter.getIndexAttr(2))});
     auto bufferSize0 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSize);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSize);
     auto bufferSize1 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr1);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSizePtr1);
     auto bufferSize2 =
-        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr2);
+        LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSizePtr2);
 
     rewriter.replaceOp(op, {bufferSize0, bufferSize1, bufferSize2, stream});
   } else {
@@ -1669,28 +1669,28 @@ LogicalResult ConvertSpMatGetSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
   Location loc = op.getLoc();
   auto stream = adaptor.getAsyncDependencies().front();
 
-  auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+  auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                  rewriter.getIndexAttr(3));
-  auto buffer = rewriter.create<LLVM::AllocaOp>(
+  auto buffer = LLVM::AllocaOp::create(rewriter,
       loc, llvmPointerType, llvmInt64Type, three, /*alignment=*/16);
 
-  auto rowsPtr = rewriter.create<LLVM::GEPOp>(
+  auto rowsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(0))});
-  auto colsPtr = rewriter.create<LLVM::GEPOp>(
+  auto colsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(1))});
-  auto nnzsPtr = rewriter.create<LLVM::GEPOp>(
+  auto nnzsPtr = LLVM::GEPOp::create(rewriter,
       loc, llvmPointerType, llvmPointerType, buffer,
-      ValueRange{rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                    rewriter.getIndexAttr(2))});
   createSpMatGetSizeBuilder.create(
       loc, rewriter, {adaptor.getSpmat(), rowsPtr, colsPtr, nnzsPtr, stream});
-  auto rows = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, rowsPtr);
-  auto cols = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, colsPtr);
-  auto nnzs = rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, nnzsPtr);
+  auto rows = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, rowsPtr);
+  auto cols = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, colsPtr);
+  auto nnzs = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, nnzsPtr);
 
   rewriter.replaceOp(op, {rows, cols, nnzs, stream});
   return success();
diff --git a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
index aab2409ed6328..70964727351a9 100644
--- a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
@@ -59,13 +59,13 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
     Operation *newOp;
     switch (op.getDimension()) {
     case gpu::Dimension::x:
-      newOp = rewriter.create<XOp>(loc, IntegerType::get(context, 32));
+      newOp = XOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     case gpu::Dimension::y:
-      newOp = rewriter.create<YOp>(loc, IntegerType::get(context, 32));
+      newOp = YOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     case gpu::Dimension::z:
-      newOp = rewriter.create<ZOp>(loc, IntegerType::get(context, 32));
+      newOp = ZOp::create(rewriter, loc, IntegerType::get(context, 32));
       break;
     }
 
@@ -124,10 +124,10 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
                                   rewriter.getContext(), 32, min, max));
     }
     if (indexBitwidth > 32) {
-      newOp = rewriter.create<LLVM::SExtOp>(
+      newOp = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
     } else if (indexBitwidth < 32) {
-      newOp = rewriter.create<LLVM::TruncOp>(
+      newOp = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
     }
 
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index 64cf09e600b88..33b09b06649b9 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -103,7 +103,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
     LLVMFuncOp funcOp = appendOrGetFuncOp(funcName, funcType, op);
     auto callOp =
-        rewriter.create<LLVM::CallOp>(op->getLoc(), funcOp, castedOperands);
+        LLVM::CallOp::create(rewriter, op->getLoc(), funcOp, castedOperands);
 
     if (resultType == adaptor.getOperands().front().getType()) {
       rewriter.replaceOp(op, {callOp.getResult()});
@@ -115,10 +115,10 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // there is no guarantee of a specific value being used to indicate true,
     // compare for inequality with zero (rather than truncate or shift).
     if (isResultBool) {
-      Value zero = rewriter.create<LLVM::ConstantOp>(
+      Value zero = LLVM::ConstantOp::create(rewriter,
           op->getLoc(), rewriter.getIntegerType(32),
           rewriter.getI32IntegerAttr(0));
-      Value truncated = rewriter.create<LLVM::ICmpOp>(
+      Value truncated = LLVM::ICmpOp::create(rewriter,
           op->getLoc(), LLVM::ICmpPredicate::ne, callOp.getResult(), zero);
       rewriter.replaceOp(op, {truncated});
       return success();
@@ -126,7 +126,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
     assert(callOp.getResult().getType().isF32() &&
            "only f32 types are supposed to be truncated back");
-    Value truncated = rewriter.create<LLVM::FPTruncOp>(
+    Value truncated = LLVM::FPTruncOp::create(rewriter,
         op->getLoc(), adaptor.getOperands().front().getType(),
         callOp.getResult());
     rewriter.replaceOp(op, {truncated});
@@ -142,7 +142,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     if (!f16Func.empty() && isa<Float16Type>(type))
       return operand;
 
-    return rewriter.create<LLVM::FPExtOp>(
+    return LLVM::FPExtOp::create(rewriter,
         operand.getLoc(), Float32Type::get(rewriter.getContext()), operand);
   }
 
@@ -169,7 +169,7 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // location as debug info metadata inside of a function cannot be used
     // outside of that function.
     auto globalloc = op->getLoc()->findInstanceOfOrUnknown<FileLineColLoc>();
-    return b.create<LLVMFuncOp>(globalloc, funcName, funcType);
+    return LLVMFuncOp::create(b, globalloc, funcName, funcType);
   }
 
   StringRef getFunctionName(Type type, SourceOp op) const {
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index 8b6b553f6eed0..fc469102a54d2 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -54,7 +54,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
       SymbolTable::lookupSymbolIn(symbolTable, name));
   if (!func) {
     OpBuilder b(symbolTable->getRegion(0));
-    func = b.create<LLVM::LLVMFuncOp>(
+    func = LLVM::LLVMFuncOp::create(b,
         symbolTable->getLoc(), name,
         LLVM::LLVMFunctionType::get(resultType, paramTypes));
     func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
@@ -79,7 +79,7 @@ static LLVM::CallOp createSPIRVBuiltinCall(Location loc,
                                            ConversionPatternRewriter &rewriter,
                                            LLVM::LLVMFuncOp func,
                                            ValueRange args) {
-  auto call = rewriter.create<LLVM::CallOp>(loc, func, args);
+  auto call = LLVM::CallOp::create(rewriter, loc, func, args);
   call.setCConv(func.getCConv());
   call.setConvergentAttr(func.getConvergentAttr());
   call.setNoUnwindAttr(func.getNoUnwindAttr());
@@ -121,7 +121,7 @@ struct GPUBarrierConversion final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
     constexpr int64_t localMemFenceFlag = 1;
     Location loc = op->getLoc();
     Value flag =
-        rewriter.create<LLVM::ConstantOp>(loc, flagTy, localMemFenceFlag);
+        LLVM::ConstantOp::create(rewriter, loc, flagTy, localMemFenceFlag);
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, flag));
     return success();
   }
@@ -162,7 +162,7 @@ struct LaunchConfigConversion : ConvertToLLVMPattern {
 
     Location loc = op->getLoc();
     gpu::Dimension dim = getDimension(op);
-    Value dimVal = rewriter.create<LLVM::ConstantOp>(loc, dimTy,
+    Value dimVal = LLVM::ConstantOp::create(rewriter, loc, dimTy,
                                                      static_cast<int64_t>(dim));
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, dimVal));
     return success();
@@ -291,12 +291,12 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                                          ConversionPatternRewriter &rewriter) {
     return TypeSwitch<Type, Value>(oldVal.getType())
         .Case([&](BFloat16Type) {
-          return rewriter.create<LLVM::BitcastOp>(loc, rewriter.getI16Type(),
+          return LLVM::BitcastOp::create(rewriter, loc, rewriter.getI16Type(),
                                                   oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
-            return rewriter.create<LLVM::ZExtOp>(loc, rewriter.getI8Type(),
+            return LLVM::ZExtOp::create(rewriter, loc, rewriter.getI8Type(),
                                                  oldVal);
           return oldVal;
         })
@@ -308,11 +308,11 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                                           ConversionPatternRewriter &rewriter) {
     return TypeSwitch<Type, Value>(newTy)
         .Case([&](BFloat16Type) {
-          return rewriter.create<LLVM::BitcastOp>(loc, newTy, oldVal);
+          return LLVM::BitcastOp::create(rewriter, loc, newTy, oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
-            return rewriter.create<LLVM::TruncOp>(loc, newTy, oldVal);
+            return LLVM::TruncOp::create(rewriter, loc, newTy, oldVal);
           return oldVal;
         })
         .Default(oldVal);
@@ -349,7 +349,7 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
         bitcastOrTruncAfterShuffle(result, op.getType(0), loc, rewriter);
 
     Value trueVal =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI1Type(), true);
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getI1Type(), true);
     rewriter.replaceOp(op, {resultOrConversion, trueVal});
     return success();
   }
@@ -426,7 +426,7 @@ struct GPUSubgroupOpConversion final : ConvertOpToLLVMPattern<SubgroupOp> {
       if (indexTy.getIntOrFloatBitWidth() < resultTy.getIntOrFloatBitWidth()) {
         return failure();
       }
-      result = rewriter.create<LLVM::ZExtOp>(loc, indexTy, result);
+      result = LLVM::ZExtOp::create(rewriter, loc, indexTy, result);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 1ef6edea93c58..7a6601ea754e9 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -118,9 +118,9 @@ struct GPUSubgroupReduceOpLowering
 
     Location loc = op->getLoc();
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
-    Value offset = rewriter.create<LLVM::ConstantOp>(loc, int32Type, -1);
+    Value offset = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
 
-    auto reduxOp = rewriter.create<NVVM::ReduxOp>(loc, int32Type, op.getValue(),
+    auto reduxOp = NVVM::ReduxOp::create(rewriter, loc, int32Type, op.getValue(),
                                                   mode.value(), offset);
 
     rewriter.replaceOp(op, reduxOp->getResult(0));
@@ -158,13 +158,13 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     auto predTy = IntegerType::get(rewriter.getContext(), 1);
 
-    Value one = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 1);
-    Value minusOne = rewriter.create<LLVM::ConstantOp>(loc, int32Type, -1);
-    Value thirtyTwo = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 32);
-    Value numLeadInactiveLane = rewriter.create<LLVM::SubOp>(
+    Value one = LLVM::ConstantOp::create(rewriter, loc, int32Type, 1);
+    Value minusOne = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
+    Value thirtyTwo = LLVM::ConstantOp::create(rewriter, loc, int32Type, 32);
+    Value numLeadInactiveLane = LLVM::SubOp::create(rewriter,
         loc, int32Type, thirtyTwo, adaptor.getWidth());
     // Bit mask of active lanes: `(-1) >> (32 - activeWidth)`.
-    Value activeMask = rewriter.create<LLVM::LShrOp>(loc, int32Type, minusOne,
+    Value activeMask = LLVM::LShrOp::create(rewriter, loc, int32Type, minusOne,
                                                      numLeadInactiveLane);
     Value maskAndClamp;
     if (op.getMode() == gpu::ShuffleMode::UP) {
@@ -173,7 +173,7 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     } else {
       // Clamp lane: `activeWidth - 1`
       maskAndClamp =
-          rewriter.create<LLVM::SubOp>(loc, int32Type, adaptor.getWidth(), one);
+          LLVM::SubOp::create(rewriter, loc, int32Type, adaptor.getWidth(), one);
     }
 
     bool predIsUsed = !op->getResult(1).use_empty();
@@ -184,13 +184,13 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
       resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(),
                                                   {valueTy, predTy});
     }
-    Value shfl = rewriter.create<NVVM::ShflOp>(
+    Value shfl = NVVM::ShflOp::create(rewriter,
         loc, resultTy, activeMask, adaptor.getValue(), adaptor.getOffset(),
         maskAndClamp, convertShflKind(op.getMode()), returnValueAndIsValidAttr);
     if (predIsUsed) {
-      Value shflValue = rewriter.create<LLVM::ExtractValueOp>(loc, shfl, 0);
+      Value shflValue = LLVM::ExtractValueOp::create(rewriter, loc, shfl, 0);
       Value isActiveSrcLane =
-          rewriter.create<LLVM::ExtractValueOp>(loc, shfl, 1);
+          LLVM::ExtractValueOp::create(rewriter, loc, shfl, 1);
       rewriter.replaceOp(op, {shflValue, isActiveSrcLane});
     } else {
       rewriter.replaceOp(op, {shfl, nullptr});
@@ -215,15 +215,15 @@ struct GPULaneIdOpToNVVM : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
       bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
           /*bitWidth=*/32, /*lower=*/0, /*upper=*/kWarpSize);
     Value newOp =
-        rewriter.create<NVVM::LaneIdOp>(loc, rewriter.getI32Type(), bounds);
+        NVVM::LaneIdOp::create(rewriter, loc, rewriter.getI32Type(), bounds);
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      newOp = rewriter.create<LLVM::SExtOp>(
+      newOp = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp);
     } else if (indexBitwidth < 32) {
-      newOp = rewriter.create<LLVM::TruncOp>(
+      newOp = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), newOp);
     }
     rewriter.replaceOp(op, {newOp});
@@ -271,10 +271,10 @@ struct AssertOpToAssertfailLowering
     Block *afterBlock =
         rewriter.splitBlock(assertBlock, ++assertOp->getIterator());
     rewriter.setInsertionPointToEnd(beforeBlock);
-    rewriter.create<cf::CondBranchOp>(loc, adaptor.getArg(), afterBlock,
+    cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(), afterBlock,
                                       assertBlock);
     rewriter.setInsertionPointToEnd(assertBlock);
-    rewriter.create<cf::BranchOp>(loc, afterBlock);
+    cf::BranchOp::create(rewriter, loc, afterBlock);
 
     // Continue cf.assert lowering.
     rewriter.setInsertionPoint(assertOp);
@@ -301,11 +301,11 @@ struct AssertOpToAssertfailLowering
     // Create constants.
     auto getGlobal = [&](LLVM::GlobalOp global) {
       // Get a pointer to the format string's first element.
-      Value globalPtr = rewriter.create<LLVM::AddressOfOp>(
+      Value globalPtr = LLVM::AddressOfOp::create(rewriter,
           loc, LLVM::LLVMPointerType::get(ctx, global.getAddrSpace()),
           global.getSymNameAttr());
       Value start =
-          rewriter.create<LLVM::GEPOp>(loc, ptrType, global.getGlobalType(),
+          LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
                                        globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
       return start;
     };
@@ -316,8 +316,8 @@ struct AssertOpToAssertfailLowering
     Value assertFunc = getGlobal(getOrCreateStringConstant(
         rewriter, loc, moduleOp, i8Type, "assert_func_", funcName));
     Value assertLine =
-        rewriter.create<LLVM::ConstantOp>(loc, i32Type, fileLine);
-    Value c1 = rewriter.create<LLVM::ConstantOp>(loc, i64Type, 1);
+        LLVM::ConstantOp::create(rewriter, loc, i32Type, fileLine);
+    Value c1 = LLVM::ConstantOp::create(rewriter, loc, i64Type, 1);
 
     // Insert function call to __assertfail.
     SmallVector<Value> arguments{assertMessage, assertFile, assertLine,
diff --git a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
index 45fd933d58857..7d5f65c470386 100644
--- a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
@@ -126,7 +126,7 @@ struct WmmaLoadOpToNVVMLowering
         cast<MemRefType>(subgroupMmaLoadMatrixOp.getSrcMemref().getType()),
         adaptor.getSrcMemref(), adaptor.getIndices());
 
-    Value leadingDim = rewriter.create<LLVM::ConstantOp>(
+    Value leadingDim = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI32Type(),
         subgroupMmaLoadMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMALoadOp>(
@@ -173,7 +173,7 @@ struct WmmaStoreOpToNVVMLowering
     auto matrixType = cast<LLVM::LLVMStructType>(adaptor.getSrc().getType());
     for (unsigned i = 0, e = matrixType.getBody().size(); i < e; ++i) {
       Value toUse =
-          rewriter.create<LLVM::ExtractValueOp>(loc, adaptor.getSrc(), i);
+          LLVM::ExtractValueOp::create(rewriter, loc, adaptor.getSrc(), i);
       storeOpOperands.push_back(toUse);
     }
 
@@ -181,7 +181,7 @@ struct WmmaStoreOpToNVVMLowering
         rewriter, loc,
         cast<MemRefType>(subgroupMmaStoreMatrixOp.getDstMemref().getType()),
         adaptor.getDstMemref(), adaptor.getIndices());
-    Value leadingDim = rewriter.create<LLVM::ConstantOp>(
+    Value leadingDim = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getI32Type(),
         subgroupMmaStoreMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMAStoreOp>(
@@ -216,7 +216,7 @@ struct WmmaMmaOpToNVVMLowering
     auto unpackOp = [&](Value operand) {
       auto structType = cast<LLVM::LLVMStructType>(operand.getType());
       for (size_t i = 0, e = structType.getBody().size(); i < e; ++i) {
-        Value toUse = rewriter.create<LLVM::ExtractValueOp>(loc, operand, i);
+        Value toUse = LLVM::ExtractValueOp::create(rewriter, loc, operand, i);
         unpackedOps.push_back(toUse);
       }
     };
@@ -280,19 +280,19 @@ struct WmmaConstantOpToNVVMLowering
         cast<gpu::MMAMatrixType>(subgroupMmaConstantOp.getType()));
     // If the element type is a vector create a vector from the operand.
     if (auto vecType = dyn_cast<VectorType>(type.getBody()[0])) {
-      Value vecCst = rewriter.create<LLVM::PoisonOp>(loc, vecType);
+      Value vecCst = LLVM::PoisonOp::create(rewriter, loc, vecType);
       for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) {
-        Value idx = rewriter.create<LLVM::ConstantOp>(
+        Value idx = LLVM::ConstantOp::create(rewriter,
             loc, rewriter.getI32Type(), vecEl);
-        vecCst = rewriter.create<LLVM::InsertElementOp>(loc, vecType, vecCst,
+        vecCst = LLVM::InsertElementOp::create(rewriter, loc, vecType, vecCst,
                                                         cst, idx);
       }
       cst = vecCst;
     }
-    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, type);
+    Value matrixStruct = LLVM::PoisonOp::create(rewriter, loc, type);
     for (size_t i : llvm::seq(size_t(0), type.getBody().size())) {
       matrixStruct =
-          rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, cst, i);
+          LLVM::InsertValueOp::create(rewriter, loc, matrixStruct, cst, i);
     }
     rewriter.replaceOp(subgroupMmaConstantOp, matrixStruct);
     return success();
@@ -305,17 +305,17 @@ static Value createMinMaxF(OpBuilder &builder, Location loc, Value lhs,
   Type i1Type = builder.getI1Type();
   if (auto vecType = dyn_cast<VectorType>(lhs.getType()))
     i1Type = VectorType::get(vecType.getShape(), i1Type);
-  Value cmp = builder.create<LLVM::FCmpOp>(
+  Value cmp = LLVM::FCmpOp::create(builder,
       loc, i1Type, isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt,
       lhs, rhs);
-  Value sel = builder.create<LLVM::SelectOp>(loc, cmp, lhs, rhs);
-  Value isNan = builder.create<LLVM::FCmpOp>(
+  Value sel = LLVM::SelectOp::create(builder, loc, cmp, lhs, rhs);
+  Value isNan = LLVM::FCmpOp::create(builder,
       loc, i1Type, LLVM::FCmpPredicate::uno, lhs, rhs);
-  Value nan = builder.create<LLVM::ConstantOp>(
+  Value nan = LLVM::ConstantOp::create(builder,
       loc, lhs.getType(),
       builder.getFloatAttr(floatType,
                            APFloat::getQNaN(floatType.getFloatSemantics())));
-  return builder.create<LLVM::SelectOp>(loc, isNan, nan, sel);
+  return LLVM::SelectOp::create(builder, loc, isNan, nan, sel);
 }
 
 static Value createScalarOp(OpBuilder &builder, Location loc,
@@ -323,11 +323,11 @@ static Value createScalarOp(OpBuilder &builder, Location loc,
                             ArrayRef<Value> operands) {
   switch (op) {
   case gpu::MMAElementwiseOp::ADDF:
-    return builder.create<LLVM::FAddOp>(loc, operands[0].getType(), operands);
+    return LLVM::FAddOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::MULF:
-    return builder.create<LLVM::FMulOp>(loc, operands[0].getType(), operands);
+    return LLVM::FMulOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::DIVF:
-    return builder.create<LLVM::FDivOp>(loc, operands[0].getType(), operands);
+    return LLVM::FDivOp::create(builder, loc, operands[0].getType(), operands);
   case gpu::MMAElementwiseOp::MAXF:
     return createMinMaxF(builder, loc, operands[0], operands[1],
                          /*isMin=*/false);
@@ -356,18 +356,18 @@ struct WmmaElementwiseOpToNVVMLowering
     size_t numOperands = adaptor.getOperands().size();
     LLVM::LLVMStructType destType = convertMMAToLLVMType(
         cast<gpu::MMAMatrixType>(subgroupMmaElementwiseOp.getType()));
-    Value matrixStruct = rewriter.create<LLVM::PoisonOp>(loc, destType);
+    Value matrixStruct = LLVM::PoisonOp::create(rewriter, loc, destType);
     for (size_t i = 0, e = destType.getBody().size(); i < e; ++i) {
       SmallVector<Value> extractedOperands;
       for (size_t opIdx = 0; opIdx < numOperands; opIdx++) {
-        extractedOperands.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
             loc, adaptor.getOperands()[opIdx], i));
       }
       Value element =
           createScalarOp(rewriter, loc, subgroupMmaElementwiseOp.getOpType(),
                          extractedOperands);
       matrixStruct =
-          rewriter.create<LLVM::InsertValueOp>(loc, matrixStruct, element, i);
+          LLVM::InsertValueOp::create(rewriter, loc, matrixStruct, element, i);
     }
     rewriter.replaceOp(subgroupMmaElementwiseOp, matrixStruct);
     return success();
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 456bfaba980ca..9b43d3ef602d1 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -61,10 +61,10 @@ static Value truncOrExtToLLVMType(ConversionPatternRewriter &rewriter,
       IntegerType::get(rewriter.getContext(), converter.getIndexTypeBitwidth());
   // TODO: use <=> in C++20.
   if (indexBitwidth > intWidth) {
-    return rewriter.create<LLVM::SExtOp>(loc, indexBitwidthType, value);
+    return LLVM::SExtOp::create(rewriter, loc, indexBitwidthType, value);
   }
   if (indexBitwidth < intWidth) {
-    return rewriter.create<LLVM::TruncOp>(loc, indexBitwidthType, value);
+    return LLVM::TruncOp::create(rewriter, loc, indexBitwidthType, value);
   }
   return value;
 }
@@ -82,11 +82,11 @@ static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func) {
 static Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
                        const unsigned indexBitwidth) {
   auto int32Type = IntegerType::get(rewriter.getContext(), 32);
-  Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
-  Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
-  Value mbcntLo = rewriter.create<ROCDL::MbcntLoOp>(loc, int32Type,
+  Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
+  Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
+  Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, int32Type,
                                                     ValueRange{minus1, zero});
-  Value laneId = rewriter.create<ROCDL::MbcntHiOp>(loc, int32Type,
+  Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, int32Type,
                                                    ValueRange{minus1, mbcntLo});
   return laneId;
 }
@@ -110,20 +110,20 @@ struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
     // followed by: %lid = call @llvm.amdgcn.mbcnt.hi(-1, %mlo)
 
     Type intTy = IntegerType::get(context, 32);
-    Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
-    Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
+    Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
+    Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
     Value mbcntLo =
-        rewriter.create<ROCDL::MbcntLoOp>(loc, intTy, ValueRange{minus1, zero});
-    Value laneId = rewriter.create<ROCDL::MbcntHiOp>(
+        ROCDL::MbcntLoOp::create(rewriter, loc, intTy, ValueRange{minus1, zero});
+    Value laneId = ROCDL::MbcntHiOp::create(rewriter,
         loc, intTy, ValueRange{minus1, mbcntLo});
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      laneId = rewriter.create<LLVM::SExtOp>(
+      laneId = LLVM::SExtOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), laneId);
     } else if (indexBitwidth < 32) {
-      laneId = rewriter.create<LLVM::TruncOp>(
+      laneId = LLVM::TruncOp::create(rewriter,
           loc, IntegerType::get(context, indexBitwidth), laneId);
     }
     rewriter.replaceOp(op, {laneId});
@@ -149,7 +149,7 @@ struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
           /*bitWidth=*/32, /*lower=*/isBeforeGfx10 ? 64 : 32,
           /*upper=*/op.getUpperBoundAttr().getInt() + 1);
     }
-    Value wavefrontOp = rewriter.create<ROCDL::WavefrontSizeOp>(
+    Value wavefrontOp = ROCDL::WavefrontSizeOp::create(rewriter,
         op.getLoc(), rewriter.getI32Type(), bounds);
     wavefrontOp = truncOrExtToLLVMType(rewriter, op.getLoc(), wavefrontOp,
                                        *getTypeConverter());
@@ -190,43 +190,43 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
 
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     Value width = adaptor.getWidth();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 0);
-    Value negwidth = rewriter.create<LLVM::SubOp>(loc, int32Type, zero, width);
-    Value add = rewriter.create<LLVM::AddOp>(loc, int32Type, srcLaneId, width);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, int32Type, 0);
+    Value negwidth = LLVM::SubOp::create(rewriter, loc, int32Type, zero, width);
+    Value add = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId, width);
     Value widthOrZeroIfOutside =
-        rewriter.create<LLVM::AndOp>(loc, int32Type, add, negwidth);
+        LLVM::AndOp::create(rewriter, loc, int32Type, add, negwidth);
     Value dstLane;
 
     switch (op.getMode()) {
     case gpu::ShuffleMode::UP:
-      dstLane = rewriter.create<LLVM::SubOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::SubOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::DOWN:
-      dstLane = rewriter.create<LLVM::AddOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::XOR:
-      dstLane = rewriter.create<LLVM::XOrOp>(loc, int32Type, srcLaneId,
+      dstLane = LLVM::XOrOp::create(rewriter, loc, int32Type, srcLaneId,
                                              adaptor.getOffset());
       break;
     case gpu::ShuffleMode::IDX:
       dstLane = adaptor.getOffset();
       break;
     }
-    Value isActiveSrcLane = rewriter.create<LLVM::ICmpOp>(
+    Value isActiveSrcLane = LLVM::ICmpOp::create(rewriter,
         loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
-    Value selectDstLane = rewriter.create<LLVM::SelectOp>(loc, isActiveSrcLane,
+    Value selectDstLane = LLVM::SelectOp::create(rewriter, loc, isActiveSrcLane,
                                                           dstLane, srcLaneId);
-    Value two = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 2);
+    Value two = LLVM::ConstantOp::create(rewriter, loc, int32Type, 2);
     Value dwordAlignedDstLane =
-        rewriter.create<LLVM::ShlOp>(loc, int32Type, selectDstLane, two);
+        LLVM::ShlOp::create(rewriter, loc, int32Type, selectDstLane, two);
 
     SmallVector<Value> decomposed =
         LLVM::decomposeValue(rewriter, loc, initShflValue, int32Type);
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
-      Value res = rewriter.create<ROCDL::DsBpermuteOp>(loc, int32Type,
+      Value res = ROCDL::DsBpermuteOp::create(rewriter, loc, int32Type,
                                                        dwordAlignedDstLane, v);
       swizzled.emplace_back(res);
     }
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index b99ed261ecfa3..71618cbf8a2f2 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -169,11 +169,11 @@ LogicalResult LaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
 
   Value vector =
       spirv::getBuiltinVariableValue(op, builtin, builtinType, rewriter);
-  Value dim = rewriter.create<spirv::CompositeExtractOp>(
+  Value dim = spirv::CompositeExtractOp::create(rewriter,
       op.getLoc(), builtinType, vector,
       rewriter.getI32ArrayAttr({static_cast<int32_t>(op.getDimension())}));
   if (forShader && builtinType != indexType)
-    dim = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType, dim);
+    dim = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType, dim);
   rewriter.replaceOp(op, dim);
   return success();
 }
@@ -198,7 +198,7 @@ SingleDimLaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
   Value builtinValue =
       spirv::getBuiltinVariableValue(op, builtin, i32Type, rewriter);
   if (i32Type != indexType)
-    builtinValue = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType,
+    builtinValue = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType,
                                                       builtinValue);
   rewriter.replaceOp(op, builtinValue);
   return success();
@@ -257,7 +257,7 @@ lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter,
       signatureConverter.addInputs(argType.index(), convertedType);
     }
   }
-  auto newFuncOp = rewriter.create<spirv::FuncOp>(
+  auto newFuncOp = spirv::FuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(signatureConverter.getConvertedTypes(), {}));
   for (const auto &namedAttr : funcOp->getAttrs()) {
@@ -367,7 +367,7 @@ LogicalResult GPUModuleConversion::matchAndRewrite(
 
   // Add a keyword to the module name to avoid symbolic conflict.
   std::string spvModuleName = (kSPIRVModule + moduleOp.getName()).str();
-  auto spvModule = rewriter.create<spirv::ModuleOp>(
+  auto spvModule = spirv::ModuleOp::create(rewriter,
       moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
       StringRef(spvModuleName));
 
@@ -452,41 +452,41 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
 
   switch (shuffleOp.getMode()) {
   case gpu::ShuffleMode::XOR: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleXorOp>(
+    result = spirv::GroupNonUniformShuffleXorOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::IDX: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleOp>(
+    result = spirv::GroupNonUniformShuffleOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::DOWN: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleDownOp>(
+    result = spirv::GroupNonUniformShuffleDownOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
 
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
-        rewriter.create<arith::AddIOp>(loc, laneId, adaptor.getOffset());
-    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+        arith::AddIOp::create(rewriter, loc, laneId, adaptor.getOffset());
+    validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                               resultLaneId, adaptor.getWidth());
     break;
   }
   case gpu::ShuffleMode::UP: {
-    result = rewriter.create<spirv::GroupNonUniformShuffleUpOp>(
+    result = spirv::GroupNonUniformShuffleUpOp::create(rewriter,
         loc, scope, adaptor.getValue(), adaptor.getOffset());
 
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
-        rewriter.create<arith::SubIOp>(loc, laneId, adaptor.getOffset());
+        arith::SubIOp::create(rewriter, loc, laneId, adaptor.getOffset());
     auto i32Type = rewriter.getIntegerType(32);
-    validVal = rewriter.create<arith::CmpIOp>(
+    validVal = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, resultLaneId,
-        rewriter.create<arith::ConstantOp>(
+        arith::ConstantOp::create(rewriter,
             loc, i32Type, rewriter.getIntegerAttr(i32Type, 0)));
     break;
   }
@@ -516,14 +516,14 @@ LogicalResult GPURotateConversion::matchAndRewrite(
 
   Location loc = rotateOp.getLoc();
   auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
-  Value rotateResult = rewriter.create<spirv::GroupNonUniformRotateKHROp>(
+  Value rotateResult = spirv::GroupNonUniformRotateKHROp::create(rewriter,
       loc, scope, adaptor.getValue(), adaptor.getOffset(), adaptor.getWidth());
   Value validVal;
   if (widthAttr.getValue().getZExtValue() == subgroupSize) {
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(), loc, rewriter);
   } else {
-    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
-    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
+    validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                               laneId, adaptor.getWidth());
   }
 
@@ -548,13 +548,13 @@ static Value createGroupReduceOpImpl(OpBuilder &builder, Location loc,
                                 ? spirv::GroupOperation::ClusteredReduce
                                 : spirv::GroupOperation::Reduce);
   if (isUniform) {
-    return builder.create<UniformOp>(loc, type, scope, groupOp, arg)
+    return UniformOp::create(builder, loc, type, scope, groupOp, arg)
         .getResult();
   }
 
   Value clusterSizeValue;
   if (clusterSize.has_value())
-    clusterSizeValue = builder.create<spirv::ConstantOp>(
+    clusterSizeValue = spirv::ConstantOp::create(builder,
         loc, builder.getI32Type(),
         builder.getIntegerAttr(builder.getI32Type(), *clusterSize));
 
@@ -740,7 +740,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstName =
         makeVarName(moduleOp, llvm::Twine(globalVarName) + "_sc");
 
-    return rewriter.create<spirv::SpecConstantOp>(
+    return spirv::SpecConstantOp::create(rewriter,
         loc, rewriter.getStringAttr(specCstName), attr);
   };
   {
@@ -774,7 +774,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstCompositeName =
         (llvm::Twine(globalVarName) + "_scc").str();
 
-    specCstComposite = rewriter.create<spirv::SpecConstantCompositeOp>(
+    specCstComposite = spirv::SpecConstantCompositeOp::create(rewriter,
         loc, TypeAttr::get(globalType),
         rewriter.getStringAttr(specCstCompositeName),
         rewriter.getArrayAttr(constituents));
@@ -785,15 +785,15 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     // Define a GlobalVarOp initialized using specialized constants
     // that is used to specify the printf format string
     // to be passed to the SPIRV CLPrintfOp.
-    globalVar = rewriter.create<spirv::GlobalVariableOp>(
+    globalVar = spirv::GlobalVariableOp::create(rewriter,
         loc, ptrType, globalVarName, FlatSymbolRefAttr::get(specCstComposite));
 
     globalVar->setAttr("Constant", rewriter.getUnitAttr());
   }
   // Get SSA value of Global variable and create pointer to i8 to point to
   // the format string.
-  Value globalPtr = rewriter.create<spirv::AddressOfOp>(loc, globalVar);
-  Value fmtStr = rewriter.create<spirv::BitcastOp>(
+  Value globalPtr = spirv::AddressOfOp::create(rewriter, loc, globalVar);
+  Value fmtStr = spirv::BitcastOp::create(rewriter,
       loc,
       spirv::PointerType::get(i8Type, spirv::StorageClass::UniformConstant),
       globalPtr);
@@ -801,7 +801,7 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
   // Get printf arguments.
   auto printfArgs = llvm::to_vector_of<Value, 4>(adaptor.getArgs());
 
-  rewriter.create<spirv::CLPrintfOp>(loc, i32Type, fmtStr, printfArgs);
+  spirv::CLPrintfOp::create(rewriter, loc, i32Type, fmtStr, printfArgs);
 
   // Need to erase the gpu.printf op as gpu.printf does not use result vs
   // spirv::CLPrintfOp has i32 resultType so cannot replace with new SPIR-V
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index 0b2c06a08db2d..da8ffe47a7d33 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -144,11 +144,11 @@ void GPUToSPIRVPass::runOnOperation() {
     if (targetEnvSupportsKernelCapability(moduleOp)) {
       moduleOp.walk([&](gpu::GPUFuncOp funcOp) {
         builder.setInsertionPoint(funcOp);
-        auto newFuncOp = builder.create<func::FuncOp>(
+        auto newFuncOp = func::FuncOp::create(builder,
             funcOp.getLoc(), funcOp.getName(), funcOp.getFunctionType());
         auto entryBlock = newFuncOp.addEntryBlock();
         builder.setInsertionPointToEnd(entryBlock);
-        builder.create<func::ReturnOp>(funcOp.getLoc());
+        func::ReturnOp::create(builder, funcOp.getLoc());
         newFuncOp->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                            builder.getUnitAttr());
         funcOp.erase();
diff --git a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
index 7bb86b5ce1ddd..70646aa5f2e74 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
@@ -283,7 +283,7 @@ struct WmmaLoadOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = rewriter.create<spirv::ConstantOp>(
+    auto strideValue = spirv::ConstantOp::create(rewriter,
         loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
@@ -315,7 +315,7 @@ struct WmmaStoreOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = rewriter.create<spirv::ConstantOp>(
+    auto strideValue = spirv::ConstantOp::create(rewriter,
         loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
diff --git a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
index 0473bb59fa6aa..0d5b318f739cc 100644
--- a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
+++ b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
@@ -36,34 +36,34 @@ struct ConvertIndexCeilDivS : mlir::ConvertOpToLLVMPattern<CeilDivSOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value posOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
-    Value negOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), -1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value posOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
+    Value negOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), -1);
 
     // Compute `x`.
     Value mPos =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::sgt, m, zero);
-    Value x = rewriter.create<LLVM::SelectOp>(loc, mPos, negOne, posOne);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, m, zero);
+    Value x = LLVM::SelectOp::create(rewriter, loc, mPos, negOne, posOne);
 
     // Compute the positive result.
-    Value nPlusX = rewriter.create<LLVM::AddOp>(loc, n, x);
-    Value nPlusXDivM = rewriter.create<LLVM::SDivOp>(loc, nPlusX, m);
-    Value posRes = rewriter.create<LLVM::AddOp>(loc, nPlusXDivM, posOne);
+    Value nPlusX = LLVM::AddOp::create(rewriter, loc, n, x);
+    Value nPlusXDivM = LLVM::SDivOp::create(rewriter, loc, nPlusX, m);
+    Value posRes = LLVM::AddOp::create(rewriter, loc, nPlusXDivM, posOne);
 
     // Compute the negative result.
-    Value negN = rewriter.create<LLVM::SubOp>(loc, zero, n);
-    Value negNDivM = rewriter.create<LLVM::SDivOp>(loc, negN, m);
-    Value negRes = rewriter.create<LLVM::SubOp>(loc, zero, negNDivM);
+    Value negN = LLVM::SubOp::create(rewriter, loc, zero, n);
+    Value negNDivM = LLVM::SDivOp::create(rewriter, loc, negN, m);
+    Value negRes = LLVM::SubOp::create(rewriter, loc, zero, negNDivM);
 
     // Pick the positive result if `n` and `m` have the same sign and `n` is
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
     Value nPos =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::sgt, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, n, zero);
     Value sameSign =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, nPos, mPos);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, nPos, mPos);
     Value nNonZero =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, n, zero);
-    Value cmp = rewriter.create<LLVM::AndOp>(loc, sameSign, nNonZero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
+    Value cmp = LLVM::AndOp::create(rewriter, loc, sameSign, nNonZero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
@@ -83,17 +83,17 @@ struct ConvertIndexCeilDivU : mlir::ConvertOpToLLVMPattern<CeilDivUOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value one = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value one = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
 
     // Compute the non-zero result.
-    Value minusOne = rewriter.create<LLVM::SubOp>(loc, n, one);
-    Value quotient = rewriter.create<LLVM::UDivOp>(loc, minusOne, m);
-    Value plusOne = rewriter.create<LLVM::AddOp>(loc, quotient, one);
+    Value minusOne = LLVM::SubOp::create(rewriter, loc, n, one);
+    Value quotient = LLVM::UDivOp::create(rewriter, loc, minusOne, m);
+    Value plusOne = LLVM::AddOp::create(rewriter, loc, quotient, one);
 
     // Pick the result.
     Value cmp =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, n, zero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, zero, plusOne);
     return success();
   }
@@ -114,32 +114,32 @@ struct ConvertIndexFloorDivS : mlir::ConvertOpToLLVMPattern<FloorDivSOp> {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
     Value m = adaptor.getRhs();
-    Value zero = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 0);
-    Value posOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), 1);
-    Value negOne = rewriter.create<LLVM::ConstantOp>(loc, n.getType(), -1);
+    Value zero = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 0);
+    Value posOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), 1);
+    Value negOne = LLVM::ConstantOp::create(rewriter, loc, n.getType(), -1);
 
     // Compute `x`.
     Value mNeg =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::slt, m, zero);
-    Value x = rewriter.create<LLVM::SelectOp>(loc, mNeg, posOne, negOne);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, m, zero);
+    Value x = LLVM::SelectOp::create(rewriter, loc, mNeg, posOne, negOne);
 
     // Compute the negative result.
-    Value xMinusN = rewriter.create<LLVM::SubOp>(loc, x, n);
-    Value xMinusNDivM = rewriter.create<LLVM::SDivOp>(loc, xMinusN, m);
-    Value negRes = rewriter.create<LLVM::SubOp>(loc, negOne, xMinusNDivM);
+    Value xMinusN = LLVM::SubOp::create(rewriter, loc, x, n);
+    Value xMinusNDivM = LLVM::SDivOp::create(rewriter, loc, xMinusN, m);
+    Value negRes = LLVM::SubOp::create(rewriter, loc, negOne, xMinusNDivM);
 
     // Compute the positive result.
-    Value posRes = rewriter.create<LLVM::SDivOp>(loc, n, m);
+    Value posRes = LLVM::SDivOp::create(rewriter, loc, n, m);
 
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
     Value nNeg =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::slt, n, zero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, n, zero);
     Value diffSign =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
     Value nNonZero =
-        rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::ne, n, zero);
-    Value cmp = rewriter.create<LLVM::AndOp>(loc, diffSign, nNonZero);
+        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
+    Value cmp = LLVM::AndOp::create(rewriter, loc, diffSign, nNonZero);
     rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, cmp, negRes, posRes);
     return success();
   }
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index 4821962f989e6..6fc097059c5e5 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -111,33 +111,33 @@ struct ConvertIndexCeilDivSPattern final : OpConversionPattern<CeilDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = rewriter.create<spirv::ConstantOp>(
+    Value posOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = rewriter.create<spirv::ConstantOp>(
+    Value negOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
-    Value mPos = rewriter.create<spirv::SGreaterThanOp>(loc, m, zero);
-    Value x = rewriter.create<spirv::SelectOp>(loc, mPos, negOne, posOne);
+    Value mPos = spirv::SGreaterThanOp::create(rewriter, loc, m, zero);
+    Value x = spirv::SelectOp::create(rewriter, loc, mPos, negOne, posOne);
 
     // Compute the positive result.
-    Value nPlusX = rewriter.create<spirv::IAddOp>(loc, n, x);
-    Value nPlusXDivM = rewriter.create<spirv::SDivOp>(loc, nPlusX, m);
-    Value posRes = rewriter.create<spirv::IAddOp>(loc, nPlusXDivM, posOne);
+    Value nPlusX = spirv::IAddOp::create(rewriter, loc, n, x);
+    Value nPlusXDivM = spirv::SDivOp::create(rewriter, loc, nPlusX, m);
+    Value posRes = spirv::IAddOp::create(rewriter, loc, nPlusXDivM, posOne);
 
     // Compute the negative result.
-    Value negN = rewriter.create<spirv::ISubOp>(loc, zero, n);
-    Value negNDivM = rewriter.create<spirv::SDivOp>(loc, negN, m);
-    Value negRes = rewriter.create<spirv::ISubOp>(loc, zero, negNDivM);
+    Value negN = spirv::ISubOp::create(rewriter, loc, zero, n);
+    Value negNDivM = spirv::SDivOp::create(rewriter, loc, negN, m);
+    Value negRes = spirv::ISubOp::create(rewriter, loc, zero, negNDivM);
 
     // Pick the positive result if `n` and `m` have the same sign and `n` is
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
-    Value nPos = rewriter.create<spirv::SGreaterThanOp>(loc, n, zero);
-    Value sameSign = rewriter.create<spirv::LogicalEqualOp>(loc, nPos, mPos);
-    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
-    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, sameSign, nNonZero);
+    Value nPos = spirv::SGreaterThanOp::create(rewriter, loc, n, zero);
+    Value sameSign = spirv::LogicalEqualOp::create(rewriter, loc, nPos, mPos);
+    Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
+    Value cmp = spirv::LogicalAndOp::create(rewriter, loc, sameSign, nNonZero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
@@ -161,18 +161,18 @@ struct ConvertIndexCeilDivUPattern final : OpConversionPattern<CeilDivUOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value one = rewriter.create<spirv::ConstantOp>(loc, n_type,
+    Value one = spirv::ConstantOp::create(rewriter, loc, n_type,
                                                    IntegerAttr::get(n_type, 1));
 
     // Compute the non-zero result.
-    Value minusOne = rewriter.create<spirv::ISubOp>(loc, n, one);
-    Value quotient = rewriter.create<spirv::UDivOp>(loc, minusOne, m);
-    Value plusOne = rewriter.create<spirv::IAddOp>(loc, quotient, one);
+    Value minusOne = spirv::ISubOp::create(rewriter, loc, n, one);
+    Value quotient = spirv::UDivOp::create(rewriter, loc, minusOne, m);
+    Value plusOne = spirv::IAddOp::create(rewriter, loc, quotient, one);
 
     // Pick the result
-    Value cmp = rewriter.create<spirv::IEqualOp>(loc, n, zero);
+    Value cmp = spirv::IEqualOp::create(rewriter, loc, n, zero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, zero, plusOne);
     return success();
   }
@@ -197,32 +197,32 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = rewriter.create<spirv::ConstantOp>(
+    Value zero = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = rewriter.create<spirv::ConstantOp>(
+    Value posOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = rewriter.create<spirv::ConstantOp>(
+    Value negOne = spirv::ConstantOp::create(rewriter,
         loc, n_type, IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
-    Value mNeg = rewriter.create<spirv::SLessThanOp>(loc, m, zero);
-    Value x = rewriter.create<spirv::SelectOp>(loc, mNeg, posOne, negOne);
+    Value mNeg = spirv::SLessThanOp::create(rewriter, loc, m, zero);
+    Value x = spirv::SelectOp::create(rewriter, loc, mNeg, posOne, negOne);
 
     // Compute the negative result
-    Value xMinusN = rewriter.create<spirv::ISubOp>(loc, x, n);
-    Value xMinusNDivM = rewriter.create<spirv::SDivOp>(loc, xMinusN, m);
-    Value negRes = rewriter.create<spirv::ISubOp>(loc, negOne, xMinusNDivM);
+    Value xMinusN = spirv::ISubOp::create(rewriter, loc, x, n);
+    Value xMinusNDivM = spirv::SDivOp::create(rewriter, loc, xMinusN, m);
+    Value negRes = spirv::ISubOp::create(rewriter, loc, negOne, xMinusNDivM);
 
     // Compute the positive result.
-    Value posRes = rewriter.create<spirv::SDivOp>(loc, n, m);
+    Value posRes = spirv::SDivOp::create(rewriter, loc, n, m);
 
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
-    Value nNeg = rewriter.create<spirv::SLessThanOp>(loc, n, zero);
-    Value diffSign = rewriter.create<spirv::LogicalNotEqualOp>(loc, nNeg, mNeg);
-    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
+    Value nNeg = spirv::SLessThanOp::create(rewriter, loc, n, zero);
+    Value diffSign = spirv::LogicalNotEqualOp::create(rewriter, loc, nNeg, mNeg);
+    Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
 
-    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, diffSign, nNonZero);
+    Value cmp = spirv::LogicalAndOp::create(rewriter, loc, diffSign, nNonZero);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
     return success();
   }
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index e34d5f74d232f..e8ad0d4644389 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -32,7 +32,7 @@ MemRefDescriptor::MemRefDescriptor(Value descriptor)
 MemRefDescriptor MemRefDescriptor::poison(OpBuilder &builder, Location loc,
                                           Type descriptorType) {
 
-  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
+  Value descriptor = LLVM::PoisonOp::create(builder, loc, descriptorType);
   return MemRefDescriptor(descriptor);
 }
 
@@ -99,20 +99,20 @@ void MemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc,
 // integer attribute.
 static Value createIndexAttrConstant(OpBuilder &builder, Location loc,
                                      Type resultType, int64_t value) {
-  return builder.create<LLVM::ConstantOp>(loc, resultType,
+  return LLVM::ConstantOp::create(builder, loc, resultType,
                                           builder.getIndexAttr(value));
 }
 
 /// Builds IR extracting the offset from the descriptor.
 Value MemRefDescriptor::offset(OpBuilder &builder, Location loc) {
-  return builder.create<LLVM::ExtractValueOp>(loc, value,
+  return LLVM::ExtractValueOp::create(builder, loc, value,
                                               kOffsetPosInMemRefDescriptor);
 }
 
 /// Builds IR inserting the offset into the descriptor.
 void MemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                  Value offset) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, offset,
+  value = LLVM::InsertValueOp::create(builder, loc, value, offset,
                                               kOffsetPosInMemRefDescriptor);
 }
 
@@ -125,7 +125,7 @@ void MemRefDescriptor::setConstantOffset(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th size from the descriptor.
 Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) {
-  return builder.create<LLVM::ExtractValueOp>(
+  return LLVM::ExtractValueOp::create(builder,
       loc, value, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
@@ -137,22 +137,22 @@ Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
 
   // Copy size values to stack-allocated memory.
   auto one = createIndexAttrConstant(builder, loc, indexType, 1);
-  auto sizes = builder.create<LLVM::ExtractValueOp>(
+  auto sizes = LLVM::ExtractValueOp::create(builder,
       loc, value, llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
-  auto sizesPtr = builder.create<LLVM::AllocaOp>(loc, ptrTy, arrayTy, one,
+  auto sizesPtr = LLVM::AllocaOp::create(builder, loc, ptrTy, arrayTy, one,
                                                  /*alignment=*/0);
-  builder.create<LLVM::StoreOp>(loc, sizes, sizesPtr);
+  LLVM::StoreOp::create(builder, loc, sizes, sizesPtr);
 
   // Load an return size value of interest.
-  auto resultPtr = builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, sizesPtr,
+  auto resultPtr = LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, sizesPtr,
                                                ArrayRef<LLVM::GEPArg>{0, pos});
-  return builder.create<LLVM::LoadOp>(loc, indexType, resultPtr);
+  return LLVM::LoadOp::create(builder, loc, indexType, resultPtr);
 }
 
 /// Builds IR inserting the pos-th size into the descriptor
 void MemRefDescriptor::setSize(OpBuilder &builder, Location loc, unsigned pos,
                                Value size) {
-  value = builder.create<LLVM::InsertValueOp>(
+  value = LLVM::InsertValueOp::create(builder,
       loc, value, size, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
@@ -164,14 +164,14 @@ void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th stride from the descriptor.
 Value MemRefDescriptor::stride(OpBuilder &builder, Location loc, unsigned pos) {
-  return builder.create<LLVM::ExtractValueOp>(
+  return LLVM::ExtractValueOp::create(builder,
       loc, value, ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
 
 /// Builds IR inserting the pos-th stride into the descriptor
 void MemRefDescriptor::setStride(OpBuilder &builder, Location loc, unsigned pos,
                                  Value stride) {
-  value = builder.create<LLVM::InsertValueOp>(
+  value = LLVM::InsertValueOp::create(builder,
       loc, value, stride,
       ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
@@ -207,7 +207,7 @@ Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
           ? offset(builder, loc)
           : createIndexAttrConstant(builder, loc, indexType, offsetCst);
   Type elementType = converter.convertType(type.getElementType());
-  ptr = builder.create<LLVM::GEPOp>(loc, ptr.getType(), elementType, ptr,
+  ptr = LLVM::GEPOp::create(builder, loc, ptr.getType(), elementType, ptr,
                                     offsetVal);
   return ptr;
 }
@@ -303,7 +303,7 @@ UnrankedMemRefDescriptor::UnrankedMemRefDescriptor(Value descriptor)
 UnrankedMemRefDescriptor UnrankedMemRefDescriptor::poison(OpBuilder &builder,
                                                           Location loc,
                                                           Type descriptorType) {
-  Value descriptor = builder.create<LLVM::PoisonOp>(loc, descriptorType);
+  Value descriptor = LLVM::PoisonOp::create(builder, loc, descriptorType);
   return UnrankedMemRefDescriptor(descriptor);
 }
 Value UnrankedMemRefDescriptor::rank(OpBuilder &builder, Location loc) const {
@@ -380,18 +380,18 @@ void UnrankedMemRefDescriptor::computeSizes(
         builder, loc, indexType,
         llvm::divideCeil(typeConverter.getPointerBitwidth(addressSpace), 8));
     Value doublePointerSize =
-        builder.create<LLVM::MulOp>(loc, indexType, two, pointerSize);
+        LLVM::MulOp::create(builder, loc, indexType, two, pointerSize);
 
     // (1 + 2 * rank) * sizeof(index)
     Value rank = desc.rank(builder, loc);
-    Value doubleRank = builder.create<LLVM::MulOp>(loc, indexType, two, rank);
+    Value doubleRank = LLVM::MulOp::create(builder, loc, indexType, two, rank);
     Value doubleRankIncremented =
-        builder.create<LLVM::AddOp>(loc, indexType, doubleRank, one);
-    Value rankIndexSize = builder.create<LLVM::MulOp>(
+        LLVM::AddOp::create(builder, loc, indexType, doubleRank, one);
+    Value rankIndexSize = LLVM::MulOp::create(builder,
         loc, indexType, doubleRankIncremented, indexSize);
 
     // Total allocation size.
-    Value allocationSize = builder.create<LLVM::AddOp>(
+    Value allocationSize = LLVM::AddOp::create(builder,
         loc, indexType, doublePointerSize, rankIndexSize);
     sizes.push_back(allocationSize);
   }
@@ -400,13 +400,13 @@ void UnrankedMemRefDescriptor::computeSizes(
 Value UnrankedMemRefDescriptor::allocatedPtr(
     OpBuilder &builder, Location loc, Value memRefDescPtr,
     LLVM::LLVMPointerType elemPtrType) {
-  return builder.create<LLVM::LoadOp>(loc, elemPtrType, memRefDescPtr);
+  return LLVM::LoadOp::create(builder, loc, elemPtrType, memRefDescPtr);
 }
 
 void UnrankedMemRefDescriptor::setAllocatedPtr(
     OpBuilder &builder, Location loc, Value memRefDescPtr,
     LLVM::LLVMPointerType elemPtrType, Value allocatedPtr) {
-  builder.create<LLVM::StoreOp>(loc, allocatedPtr, memRefDescPtr);
+  LLVM::StoreOp::create(builder, loc, allocatedPtr, memRefDescPtr);
 }
 
 static std::pair<Value, Type>
@@ -423,9 +423,9 @@ Value UnrankedMemRefDescriptor::alignedPtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   Value alignedGep =
-      builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+      LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                   elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
-  return builder.create<LLVM::LoadOp>(loc, elemPtrType, alignedGep);
+  return LLVM::LoadOp::create(builder, loc, elemPtrType, alignedGep);
 }
 
 void UnrankedMemRefDescriptor::setAlignedPtr(
@@ -435,9 +435,9 @@ void UnrankedMemRefDescriptor::setAlignedPtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   Value alignedGep =
-      builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+      LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                   elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
-  builder.create<LLVM::StoreOp>(loc, alignedPtr, alignedGep);
+  LLVM::StoreOp::create(builder, loc, alignedPtr, alignedGep);
 }
 
 Value UnrankedMemRefDescriptor::offsetBasePtr(
@@ -446,7 +446,7 @@ Value UnrankedMemRefDescriptor::offsetBasePtr(
   auto [elementPtrPtr, elemPtrPtrType] =
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
-  return builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
+  return LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
                                      elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
 }
 
@@ -456,7 +456,7 @@ Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
                                        LLVM::LLVMPointerType elemPtrType) {
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
-  return builder.create<LLVM::LoadOp>(loc, typeConverter.getIndexType(),
+  return LLVM::LoadOp::create(builder, loc, typeConverter.getIndexType(),
                                       offsetPtr);
 }
 
@@ -467,7 +467,7 @@ void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                          Value offset) {
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
-  builder.create<LLVM::StoreOp>(loc, offset, offsetPtr);
+  LLVM::StoreOp::create(builder, loc, offset, offsetPtr);
 }
 
 Value UnrankedMemRefDescriptor::sizeBasePtr(
@@ -477,7 +477,7 @@ Value UnrankedMemRefDescriptor::sizeBasePtr(
   Type structTy = LLVM::LLVMStructType::getLiteral(
       indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy});
   auto resultType = LLVM::LLVMPointerType::get(builder.getContext());
-  return builder.create<LLVM::GEPOp>(loc, resultType, structTy, memRefDescPtr,
+  return LLVM::GEPOp::create(builder, loc, resultType, structTy, memRefDescPtr,
                                      ArrayRef<LLVM::GEPArg>{0, 3});
 }
 
@@ -489,8 +489,8 @@ Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value sizeStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, index);
-  return builder.create<LLVM::LoadOp>(loc, indexTy, sizeStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, index);
+  return LLVM::LoadOp::create(builder, loc, indexTy, sizeStoreGep);
 }
 
 void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
@@ -501,8 +501,8 @@ void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value sizeStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, index);
-  builder.create<LLVM::StoreOp>(loc, size, sizeStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, index);
+  LLVM::StoreOp::create(builder, loc, size, sizeStoreGep);
 }
 
 Value UnrankedMemRefDescriptor::strideBasePtr(
@@ -511,7 +511,7 @@ Value UnrankedMemRefDescriptor::strideBasePtr(
   Type indexTy = typeConverter.getIndexType();
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
-  return builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, sizeBasePtr, rank);
+  return LLVM::GEPOp::create(builder, loc, ptrType, indexTy, sizeBasePtr, rank);
 }
 
 Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
@@ -522,8 +522,8 @@ Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value strideStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, strideBasePtr, index);
-  return builder.create<LLVM::LoadOp>(loc, indexTy, strideStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, strideBasePtr, index);
+  return LLVM::LoadOp::create(builder, loc, indexTy, strideStoreGep);
 }
 
 void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
@@ -534,6 +534,6 @@ void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
 
   Value strideStoreGep =
-      builder.create<LLVM::GEPOp>(loc, ptrType, indexTy, strideBasePtr, index);
-  builder.create<LLVM::StoreOp>(loc, stride, strideStoreGep);
+      LLVM::GEPOp::create(builder, loc, ptrType, indexTy, strideBasePtr, index);
+  LLVM::StoreOp::create(builder, loc, stride, strideStoreGep);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index c5f72f7e10b8c..57aed79874ad0 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -57,7 +57,7 @@ Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder,
                                                     Location loc,
                                                     Type resultType,
                                                     int64_t value) {
-  return builder.create<LLVM::ConstantOp>(loc, resultType,
+  return LLVM::ConstantOp::create(builder, loc, resultType,
                                           builder.getIndexAttr(value));
 }
 
@@ -123,7 +123,7 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
       runningStride = sizes[i];
     else if (stride == ShapedType::kDynamic)
       runningStride =
-          rewriter.create<LLVM::MulOp>(loc, runningStride, sizes[i]);
+          LLVM::MulOp::create(rewriter, loc, runningStride, sizes[i]);
     else
       runningStride = createIndexAttrConstant(rewriter, loc, indexType, stride);
   }
@@ -131,10 +131,10 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
     // Buffer size in bytes.
     Type elementType = typeConverter->convertType(memRefType.getElementType());
     auto elementPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, elementPtrType);
-    Value gepPtr = rewriter.create<LLVM::GEPOp>(
+    Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
+    Value gepPtr = LLVM::GEPOp::create(rewriter,
         loc, elementPtrType, elementType, nullPtr, runningStride);
-    size = rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);
+    size = LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
   } else {
     size = runningStride;
   }
@@ -149,10 +149,10 @@ Value ConvertToLLVMPattern::getSizeInBytes(
   // which is a common pattern of getting the size of a type in bytes.
   Type llvmType = typeConverter->convertType(type);
   auto convertedPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-  auto nullPtr = rewriter.create<LLVM::ZeroOp>(loc, convertedPtrType);
-  auto gep = rewriter.create<LLVM::GEPOp>(loc, convertedPtrType, llvmType,
+  auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, convertedPtrType);
+  auto gep = LLVM::GEPOp::create(rewriter, loc, convertedPtrType, llvmType,
                                           nullPtr, ArrayRef<LLVM::GEPArg>{1});
-  return rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gep);
+  return LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gep);
 }
 
 Value ConvertToLLVMPattern::getNumElements(
@@ -175,7 +175,7 @@ Value ConvertToLLVMPattern::getNumElements(
           staticSize == ShapedType::kDynamic
               ? dynamicSizes[dynamicIndex++]
               : createIndexAttrConstant(rewriter, loc, indexType, staticSize);
-      numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
     } else {
       numElements =
           staticSize == ShapedType::kDynamic
@@ -276,14 +276,14 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
             ? builder
                   .create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
                   .getResult()
-            : builder.create<LLVM::AllocaOp>(loc, getPtrType(),
+            : LLVM::AllocaOp::create(builder, loc, getPtrType(),
                                              IntegerType::get(getContext(), 8),
                                              allocationSize,
                                              /*alignment=*/0);
     Value source = desc.memRefDescPtr(builder, loc);
-    builder.create<LLVM::MemcpyOp>(loc, memory, source, allocationSize, false);
+    LLVM::MemcpyOp::create(builder, loc, memory, source, allocationSize, false);
     if (!toDynamic)
-      builder.create<LLVM::CallOp>(loc, freeFunc.value(), source);
+      LLVM::CallOp::create(builder, loc, freeFunc.value(), source);
 
     // Create a new descriptor. The same descriptor can be returned multiple
     // times, attempting to modify its pointer can lead to memory leaks
@@ -349,7 +349,7 @@ LogicalResult LLVM::detail::oneToOneRewrite(
   SmallVector<Value, 4> results;
   results.reserve(numResults);
   for (unsigned i = 0; i < numResults; ++i) {
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+    results.push_back(LLVM::ExtractValueOp::create(rewriter,
         op->getLoc(), newOp->getResult(0), i));
   }
   rewriter.replaceOp(op, results);
@@ -371,7 +371,7 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   if (numResults != 0)
     resType = typeConverter.packOperationResults(op->getResultTypes());
 
-  auto callIntrOp = rewriter.create<LLVM::CallIntrinsicOp>(
+  auto callIntrOp = LLVM::CallIntrinsicOp::create(rewriter,
       loc, resType, rewriter.getStringAttr(intrinsic), operands);
   // Propagate attributes.
   callIntrOp->setAttrs(op->getAttrDictionary());
@@ -388,7 +388,7 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   results.reserve(numResults);
   Value intrRes = callIntrOp.getResults();
   for (unsigned i = 0; i < numResults; ++i)
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(loc, intrRes, i));
+    results.push_back(LLVM::ExtractValueOp::create(rewriter, loc, intrRes, i));
   rewriter.replaceOp(op, results);
 
   return success();
@@ -406,7 +406,7 @@ static unsigned getBitWidth(Type type) {
 static Value createI32Constant(OpBuilder &builder, Location loc,
                                int32_t value) {
   Type i32 = builder.getI32Type();
-  return builder.create<LLVM::ConstantOp>(loc, i32, value);
+  return LLVM::ConstantOp::create(builder, loc, i32, value);
 }
 
 SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
@@ -418,17 +418,17 @@ SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
   unsigned srcBitWidth = getBitWidth(srcType);
   unsigned dstBitWidth = getBitWidth(dstType);
   if (srcBitWidth == dstBitWidth) {
-    Value cast = builder.create<LLVM::BitcastOp>(loc, dstType, src);
+    Value cast = LLVM::BitcastOp::create(builder, loc, dstType, src);
     return {cast};
   }
 
   if (dstBitWidth > srcBitWidth) {
     auto smallerInt = builder.getIntegerType(srcBitWidth);
     if (srcType != smallerInt)
-      src = builder.create<LLVM::BitcastOp>(loc, smallerInt, src);
+      src = LLVM::BitcastOp::create(builder, loc, smallerInt, src);
 
     auto largerInt = builder.getIntegerType(dstBitWidth);
-    Value res = builder.create<LLVM::ZExtOp>(loc, largerInt, src);
+    Value res = LLVM::ZExtOp::create(builder, loc, largerInt, src);
     return {res};
   }
   assert(srcBitWidth % dstBitWidth == 0 &&
@@ -436,12 +436,12 @@ SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
   int64_t numElements = srcBitWidth / dstBitWidth;
   auto vecType = VectorType::get(numElements, dstType);
 
-  src = builder.create<LLVM::BitcastOp>(loc, vecType, src);
+  src = LLVM::BitcastOp::create(builder, loc, vecType, src);
 
   SmallVector<Value> res;
   for (auto i : llvm::seq(numElements)) {
     Value idx = createI32Constant(builder, loc, i);
-    Value elem = builder.create<LLVM::ExtractElementOp>(loc, src, idx);
+    Value elem = LLVM::ExtractElementOp::create(builder, loc, src, idx);
     res.emplace_back(elem);
   }
 
@@ -461,28 +461,28 @@ Value mlir::LLVM::composeValue(OpBuilder &builder, Location loc, ValueRange src,
     if (dstBitWidth < srcBitWidth) {
       auto largerInt = builder.getIntegerType(srcBitWidth);
       if (res.getType() != largerInt)
-        res = builder.create<LLVM::BitcastOp>(loc, largerInt, res);
+        res = LLVM::BitcastOp::create(builder, loc, largerInt, res);
 
       auto smallerInt = builder.getIntegerType(dstBitWidth);
-      res = builder.create<LLVM::TruncOp>(loc, smallerInt, res);
+      res = LLVM::TruncOp::create(builder, loc, smallerInt, res);
     }
 
     if (res.getType() != dstType)
-      res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+      res = LLVM::BitcastOp::create(builder, loc, dstType, res);
 
     return res;
   }
 
   int64_t numElements = src.size();
   auto srcType = VectorType::get(numElements, src.front().getType());
-  Value res = builder.create<LLVM::PoisonOp>(loc, srcType);
+  Value res = LLVM::PoisonOp::create(builder, loc, srcType);
   for (auto &&[i, elem] : llvm::enumerate(src)) {
     Value idx = createI32Constant(builder, loc, i);
-    res = builder.create<LLVM::InsertElementOp>(loc, srcType, res, elem, idx);
+    res = LLVM::InsertElementOp::create(builder, loc, srcType, res, elem, idx);
   }
 
   if (res.getType() != dstType)
-    res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+    res = LLVM::BitcastOp::create(builder, loc, dstType, res);
 
   return res;
 }
@@ -518,18 +518,18 @@ Value mlir::LLVM::getStridedElementPtr(OpBuilder &builder, Location loc,
       Value stride =
           ShapedType::isDynamic(strides[i])
               ? memRefDescriptor.stride(builder, loc, i)
-              : builder.create<LLVM::ConstantOp>(
+              : LLVM::ConstantOp::create(builder,
                     loc, indexType, builder.getIndexAttr(strides[i]));
       increment =
-          builder.create<LLVM::MulOp>(loc, increment, stride, intOverflowFlags);
+          LLVM::MulOp::create(builder, loc, increment, stride, intOverflowFlags);
     }
-    index = index ? builder.create<LLVM::AddOp>(loc, index, increment,
+    index = index ? LLVM::AddOp::create(builder, loc, index, increment,
                                                 intOverflowFlags)
                   : increment;
   }
 
   Type elementPtrType = memRefDescriptor.getElementPtrType();
-  return index ? builder.create<LLVM::GEPOp>(
+  return index ? LLVM::GEPOp::create(builder,
                      loc, elementPtrType,
                      converter.convertType(type.getElementType()), base, index,
                      noWrapFlags)
diff --git a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
index 49c73fbc9dd79..091fb7e4c2afb 100644
--- a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
@@ -66,7 +66,7 @@ LogicalResult mlir::LLVM::createPrintStrCall(
       DenseElementsAttr::get(dataAttrType, llvm::ArrayRef(elementVals));
   auto arrayTy =
       LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), elementVals.size());
-  auto globalOp = builder.create<LLVM::GlobalOp>(
+  auto globalOp = LLVM::GlobalOp::create(builder,
       loc, arrayTy, /*constant=*/true, LLVM::Linkage::Private,
       ensureSymbolNameIsUnique(moduleOp, symbolName, symbolTables), dataAttr);
 
@@ -74,15 +74,15 @@ LogicalResult mlir::LLVM::createPrintStrCall(
   // Emit call to `printStr` in runtime library.
   builder.restoreInsertionPoint(ip);
   auto msgAddr =
-      builder.create<LLVM::AddressOfOp>(loc, ptrTy, globalOp.getName());
+      LLVM::AddressOfOp::create(builder, loc, ptrTy, globalOp.getName());
   SmallVector<LLVM::GEPArg> indices(1, 0);
   Value gep =
-      builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, msgAddr, indices);
+      LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, msgAddr, indices);
   FailureOr<LLVM::LLVMFuncOp> printer =
       LLVM::lookupOrCreatePrintStringFn(builder, moduleOp, runtimeFunctionName);
   if (failed(printer))
     return failure();
-  builder.create<LLVM::CallOp>(loc, TypeRange(),
+  LLVM::CallOp::create(builder, loc, TypeRange(),
                                SymbolRefAttr::get(printer.value()), gep);
   return success();
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
index 1cd0bd85f9894..13ed4628c3c9e 100644
--- a/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/StructBuilder.cpp
@@ -24,10 +24,10 @@ StructBuilder::StructBuilder(Value v) : value(v), structType(v.getType()) {
 
 Value StructBuilder::extractPtr(OpBuilder &builder, Location loc,
                                 unsigned pos) const {
-  return builder.create<LLVM::ExtractValueOp>(loc, value, pos);
+  return LLVM::ExtractValueOp::create(builder, loc, value, pos);
 }
 
 void StructBuilder::setPtr(OpBuilder &builder, Location loc, unsigned pos,
                            Value ptr) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, ptr, pos);
+  value = LLVM::InsertValueOp::create(builder, loc, value, ptr, pos);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 7312594c761f7..77135188dabcf 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -91,7 +91,7 @@ static Value unrankedMemRefMaterialization(OpBuilder &builder,
       packUnrankedMemRefDesc(builder, resultType, inputs, loc, converter);
   if (!packed)
     return Value();
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, packed)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, packed)
       .getResult(0);
 }
 
@@ -107,7 +107,7 @@ static Value rankedMemRefMaterialization(OpBuilder &builder,
       packRankedMemRefDesc(builder, resultType, inputs, loc, converter);
   if (!packed)
     return Value();
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, packed)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, packed)
       .getResult(0);
 }
 
@@ -224,12 +224,12 @@ LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
   // non-LLVM types persist after an LLVM conversion.
   addSourceMaterialization([&](OpBuilder &builder, Type resultType,
                                ValueRange inputs, Location loc) {
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
   addTargetMaterialization([&](OpBuilder &builder, Type resultType,
                                ValueRange inputs, Location loc) {
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
 
@@ -731,12 +731,12 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand,
   // Alloca with proper alignment. We do not expect optimizations of this
   // alloca op and so we omit allocating at the entry block.
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
-  Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
+  Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
                                                builder.getIndexAttr(1));
   Value allocated =
-      builder.create<LLVM::AllocaOp>(loc, ptrType, operand.getType(), one);
+      LLVM::AllocaOp::create(builder, loc, ptrType, operand.getType(), one);
   // Store into the alloca'ed descriptor.
-  builder.create<LLVM::StoreOp>(loc, operand, allocated);
+  LLVM::StoreOp::create(builder, loc, operand, allocated);
   return allocated;
 }
 
diff --git a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
index bf3f31729c3da..4fc208e05eb9d 100644
--- a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
@@ -87,17 +87,17 @@ LogicalResult LLVM::detail::handleMultidimensionalVectors(
   auto result1DVectorTy = resultTypeInfo.llvm1DVectorTy;
   auto resultNDVectoryTy = resultTypeInfo.llvmNDVectorTy;
   auto loc = op->getLoc();
-  Value desc = rewriter.create<LLVM::PoisonOp>(loc, resultNDVectoryTy);
+  Value desc = LLVM::PoisonOp::create(rewriter, loc, resultNDVectoryTy);
   nDVectorIterate(resultTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
     // For this unrolled `position` corresponding to the `linearIndex`^th
     // element, extract operand vectors
     SmallVector<Value, 4> extractedOperands;
     for (const auto &operand : llvm::enumerate(operands)) {
-      extractedOperands.push_back(rewriter.create<LLVM::ExtractValueOp>(
+      extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
           loc, operand.value(), position));
     }
     Value newVal = createOperand(result1DVectorTy, extractedOperands);
-    desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, newVal, position);
+    desc = LLVM::InsertValueOp::create(rewriter, loc, desc, newVal, position);
   });
   rewriter.replaceOp(op, desc);
   return success();
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index c3f213147b7a7..498a5cfc40204 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -78,7 +78,7 @@ getLibraryCallSymbolRef(Operation *op, PatternRewriter &rewriter) {
   // Insert before module terminator.
   rewriter.setInsertionPoint(module.getBody(),
                              std::prev(module.getBody()->end()));
-  func::FuncOp funcOp = rewriter.create<func::FuncOp>(
+  func::FuncOp funcOp = func::FuncOp::create(rewriter,
       op->getLoc(), fnNameAttr.getValue(), libFnType);
   // Insert a function attribute that will trigger the emission of the
   // corresponding `_mlir_ciface_xxx` interface so that external libraries see
@@ -101,7 +101,7 @@ createTypeCanonicalizedMemRefOperands(OpBuilder &b, Location loc,
       continue;
     }
     Value cast =
-        b.create<memref::CastOp>(loc, makeStridedLayoutDynamic(memrefType), op);
+        memref::CastOp::create(b, loc, makeStridedLayoutDynamic(memrefType), op);
     res.push_back(cast);
   }
   return res;
diff --git a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
index d4deff5b88070..30521d66a2e4c 100644
--- a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
+++ b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
@@ -54,18 +54,18 @@ std::pair<Value, Value> getRawPtrAndSize(const Location loc,
                                          Value memRef, Type elType) {
   Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
   Value dataPtr =
-      rewriter.create<LLVM::ExtractValueOp>(loc, ptrType, memRef, 1);
-  Value offset = rewriter.create<LLVM::ExtractValueOp>(
+      LLVM::ExtractValueOp::create(rewriter, loc, ptrType, memRef, 1);
+  Value offset = LLVM::ExtractValueOp::create(rewriter,
       loc, rewriter.getI64Type(), memRef, 2);
   Value resPtr =
-      rewriter.create<LLVM::GEPOp>(loc, ptrType, elType, dataPtr, offset);
+      LLVM::GEPOp::create(rewriter, loc, ptrType, elType, dataPtr, offset);
   Value size;
   if (cast<LLVM::LLVMStructType>(memRef.getType()).getBody().size() > 3) {
-    size = rewriter.create<LLVM::ExtractValueOp>(loc, memRef,
+    size = LLVM::ExtractValueOp::create(rewriter, loc, memRef,
                                                  ArrayRef<int64_t>{3, 0});
-    size = rewriter.create<LLVM::TruncOp>(loc, rewriter.getI32Type(), size);
+    size = LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), size);
   } else {
-    size = rewriter.create<arith::ConstantIntOp>(loc, 1, 32);
+    size = arith::ConstantIntOp::create(rewriter, loc, 1, 32);
   }
   return {resPtr, size};
 }
@@ -157,13 +157,13 @@ class MPICHImplTraits : public MPIImplTraits {
   Value getCommWorld(const Location loc,
                      ConversionPatternRewriter &rewriter) override {
     static constexpr int MPI_COMM_WORLD = 0x44000000;
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
                                              MPI_COMM_WORLD);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return rewriter.create<LLVM::TruncOp>(loc, rewriter.getI32Type(), comm);
+    return LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), comm);
   }
 
   intptr_t getStatusIgnore() override { return 1; }
@@ -195,7 +195,7 @@ class MPICHImplTraits : public MPIImplTraits {
       mtype = MPI_UINT8_T;
     else
       assert(false && "unsupported type");
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), mtype);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), mtype);
   }
 
   Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter,
@@ -245,7 +245,7 @@ class MPICHImplTraits : public MPIImplTraits {
       op = MPI_REPLACE;
       break;
     }
-    return rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI32Type(), op);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), op);
   }
 };
 
@@ -281,15 +281,15 @@ class OMPIImplTraits : public MPIImplTraits {
     getOrDefineExternalStruct(loc, rewriter, name, commStructT);
 
     // get address of symbol
-    auto comm = rewriter.create<LLVM::AddressOfOp>(
+    auto comm = LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, name));
-    return rewriter.create<LLVM::PtrToIntOp>(loc, rewriter.getI64Type(), comm);
+    return LLVM::PtrToIntOp::create(rewriter, loc, rewriter.getI64Type(), comm);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return rewriter.create<LLVM::IntToPtrOp>(
+    return LLVM::IntToPtrOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(rewriter.getContext()), comm);
   }
 
@@ -330,7 +330,7 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, mtype, typeStructT);
     // get address of symbol
-    return rewriter.create<LLVM::AddressOfOp>(
+    return LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, mtype));
   }
@@ -389,7 +389,7 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, op, opStructT);
     // get address of symbol
-    return rewriter.create<LLVM::AddressOfOp>(
+    return LLVM::AddressOfOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(context),
         SymbolRefAttr::get(context, op));
   }
@@ -424,7 +424,7 @@ struct InitOpLowering : public ConvertOpToLLVMPattern<mpi::InitOp> {
     Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
 
     // instantiate nullptr `%nullptr = llvm.mlir.zero : !llvm.ptr`
-    auto nullPtrOp = rewriter.create<LLVM::ZeroOp>(loc, ptrType);
+    auto nullPtrOp = LLVM::ZeroOp::create(rewriter, loc, ptrType);
     Value llvmnull = nullPtrOp.getRes();
 
     // grab a reference to the global module op:
@@ -513,9 +513,9 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
 
     // get communicator
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, i32, 1);
+    auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
     auto outPtr =
-        rewriter.create<LLVM::AllocaOp>(loc, ptrType, comm.getType(), one);
+        LLVM::AllocaOp::create(rewriter, loc, ptrType, comm.getType(), one);
 
     // int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm * newcomm)
     auto funcType =
@@ -524,14 +524,14 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
     LLVM::LLVMFuncOp funcDecl = getOrDefineFunction(moduleOp, loc, rewriter,
                                                     "MPI_Comm_split", funcType);
 
-    auto callOp = rewriter.create<LLVM::CallOp>(
+    auto callOp = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{comm, adaptor.getColor(), adaptor.getKey(),
                    outPtr.getRes()});
 
     // load the communicator into a register
-    Value res = rewriter.create<LLVM::LoadOp>(loc, i32, outPtr.getResult());
-    res = rewriter.create<LLVM::SExtOp>(loc, rewriter.getI64Type(), res);
+    Value res = LLVM::LoadOp::create(rewriter, loc, i32, outPtr.getResult());
+    res = LLVM::SExtOp::create(rewriter, loc, rewriter.getI64Type(), res);
 
     // if retval is checked, replace uses of retval with the results from the
     // call op
@@ -580,14 +580,14 @@ struct CommRankOpLowering : public ConvertOpToLLVMPattern<mpi::CommRankOp> {
         moduleOp, loc, rewriter, "MPI_Comm_rank", rankFuncType);
 
     // replace with function call
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, i32, 1);
-    auto rankptr = rewriter.create<LLVM::AllocaOp>(loc, ptrType, i32, one);
-    auto callOp = rewriter.create<LLVM::CallOp>(
+    auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
+    auto rankptr = LLVM::AllocaOp::create(rewriter, loc, ptrType, i32, one);
+    auto callOp = LLVM::CallOp::create(rewriter,
         loc, initDecl, ValueRange{comm, rankptr.getRes()});
 
     // load the rank into a register
     auto loadedRank =
-        rewriter.create<LLVM::LoadOp>(loc, i32, rankptr.getResult());
+        LLVM::LoadOp::create(rewriter, loc, i32, rankptr.getResult());
 
     // if retval is checked, replace uses of retval with the results from the
     // call op
@@ -641,7 +641,7 @@ struct SendOpLowering : public ConvertOpToLLVMPattern<mpi::SendOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Send", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getDest(), adaptor.getTag(),
                    comm});
@@ -683,10 +683,10 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
     auto mpiTraits = MPIImplTraits::get(moduleOp);
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    Value statusIgnore = rewriter.create<LLVM::ConstantOp>(
+    Value statusIgnore = LLVM::ConstantOp::create(rewriter,
         loc, i64, mpiTraits->getStatusIgnore());
     statusIgnore =
-        rewriter.create<LLVM::IntToPtrOp>(loc, ptrType, statusIgnore);
+        LLVM::IntToPtrOp::create(rewriter, loc, ptrType, statusIgnore);
 
     // LLVM Function type representing `i32 MPI_Recv(data, count, datatype, dst,
     // tag, comm)`
@@ -698,7 +698,7 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Recv", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getSource(),
                    adaptor.getTag(), comm, statusIgnore});
@@ -738,9 +738,9 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
 
     // If input and output are the same, request in-place operation.
     if (adaptor.getSendbuf() == adaptor.getRecvbuf()) {
-      sendPtr = rewriter.create<LLVM::ConstantOp>(
+      sendPtr = LLVM::ConstantOp::create(rewriter,
           loc, i64, reinterpret_cast<int64_t>(mpiTraits->getInPlace()));
-      sendPtr = rewriter.create<LLVM::IntToPtrOp>(loc, ptrType, sendPtr);
+      sendPtr = LLVM::IntToPtrOp::create(rewriter, loc, ptrType, sendPtr);
     }
 
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
@@ -757,7 +757,7 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Allreduce", funcType);
 
     // replace op with function call
-    auto funcCall = rewriter.create<LLVM::CallOp>(
+    auto funcCall = LLVM::CallOp::create(rewriter,
         loc, funcDecl,
         ValueRange{sendPtr, recvPtr, sendSize, dataType, mpiOp, commWorld});
 
diff --git a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
index 7f4655e53609e..bb3511440e041 100644
--- a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
+++ b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
@@ -121,7 +121,7 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     initValueAttr = FloatAttr::get(resultElementType, 0.0);
   else
     initValueAttr = IntegerAttr::get(resultElementType, 0);
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(vecType, initValueAttr));
   SmallVector<int64_t> strides = computeStrides(shape);
   for (int64_t linearIndex = 0; linearIndex < numElements; ++linearIndex) {
@@ -129,11 +129,11 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     SmallVector<Value> operands;
     for (Value input : op->getOperands())
       operands.push_back(
-          rewriter.create<vector::ExtractOp>(loc, input, positions));
+          vector::ExtractOp::create(rewriter, loc, input, positions));
     Value scalarOp =
-        rewriter.create<Op>(loc, vecType.getElementType(), operands);
+        Op::create(rewriter, loc, vecType.getElementType(), operands);
     result =
-        rewriter.create<vector::InsertOp>(loc, scalarOp, result, positions);
+        vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
   }
   rewriter.replaceOp(op, result);
   return success();
@@ -195,7 +195,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
 
   FunctionType funcType = FunctionType::get(
       builder.getContext(), {elementType, elementType}, elementType);
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
   LLVM::linkage::Linkage inlineLinkage = LLVM::linkage::Linkage::LinkonceODR;
   Attribute linkage =
       LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
@@ -208,11 +208,11 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value zeroValue = builder.create<arith::ConstantOp>(
+  Value zeroValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 0));
-  Value oneValue = builder.create<arith::ConstantOp>(
+  Value oneValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 1));
-  Value minusOneValue = builder.create<arith::ConstantOp>(
+  Value minusOneValue = arith::ConstantOp::create(builder,
       elementType,
       builder.getIntegerAttr(elementType,
                              APInt(elementType.getIntOrFloatBitWidth(), -1ULL,
@@ -221,81 +221,81 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   // if (p == T(0))
   //   return T(1);
   auto pIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroValue);
   Block *thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(pIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(pIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsZero, thenBlock, fallthroughBlock);
 
   // if (p < T(0)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto pIsNeg =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, pArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg, zeroValue);
   //   if (b == T(0))
   builder.createBlock(funcBody);
   auto bIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bArg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, zeroValue);
   //     return T(1) / T(0);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(
-      builder.create<arith::DivSIOp>(oneValue, zeroValue).getResult());
+  func::ReturnOp::create(builder,
+      arith::DivSIOp::create(builder, oneValue, zeroValue).getResult());
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(0)).
   builder.setInsertionPointToEnd(bIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(bIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, bIsZero, thenBlock, fallthroughBlock);
 
   //   if (b == T(1))
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto bIsOne =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bArg, oneValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, oneValue);
   //    return T(1);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(1)).
   builder.setInsertionPointToEnd(bIsOne->getBlock());
-  builder.create<cf::CondBranchOp>(bIsOne, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, bIsOne, thenBlock, fallthroughBlock);
 
   //   if (b == T(-1)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
-  auto bIsMinusOne = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto bIsMinusOne = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                    bArg, minusOneValue);
   //     if (p & T(1))
   builder.createBlock(funcBody);
-  auto pIsOdd = builder.create<arith::CmpIOp>(
-      arith::CmpIPredicate::ne, builder.create<arith::AndIOp>(pArg, oneValue),
+  auto pIsOdd = arith::CmpIOp::create(builder,
+      arith::CmpIPredicate::ne, arith::AndIOp::create(builder, pArg, oneValue),
       zeroValue);
   //       return T(-1);
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(minusOneValue);
+  func::ReturnOp::create(builder, minusOneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(pIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(pIsOdd, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsOdd, thenBlock, fallthroughBlock);
 
   //     return T(1);
   //   } // b == T(-1)
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<func::ReturnOp>(oneValue);
+  func::ReturnOp::create(builder, oneValue);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(-1)).
   builder.setInsertionPointToEnd(bIsMinusOne->getBlock());
-  builder.create<cf::CondBranchOp>(bIsMinusOne, pIsOdd->getBlock(),
+  cf::CondBranchOp::create(builder, bIsMinusOne, pIsOdd->getBlock(),
                                    fallthroughBlock);
 
   //   return T(0);
   // } // (p < T(0))
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<func::ReturnOp>(zeroValue);
+  func::ReturnOp::create(builder, zeroValue);
   Block *loopHeader = builder.createBlock(
       funcBody, funcBody->end(), {elementType, elementType, elementType},
       {builder.getLoc(), builder.getLoc(), builder.getLoc()});
   // Set up conditional branch for (p < T(0)).
   builder.setInsertionPointToEnd(pIsNeg->getBlock());
   // Set initial values of 'result', 'b' and 'p' for the loop.
-  builder.create<cf::CondBranchOp>(pIsNeg, bIsZero->getBlock(), loopHeader,
+  cf::CondBranchOp::create(builder, pIsNeg, bIsZero->getBlock(), loopHeader,
                                    ValueRange{oneValue, bArg, pArg});
 
   // T result = T(1);
@@ -313,44 +313,44 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & T(1))
-  auto powerTmpIsOdd = builder.create<arith::CmpIOp>(
+  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
       arith::CmpIPredicate::ne,
-      builder.create<arith::AndIOp>(powerTmp, oneValue), zeroValue);
+      arith::AndIOp::create(builder, powerTmp, oneValue), zeroValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
-  Value newResultTmp = builder.create<arith::MulIOp>(resultTmp, baseTmp);
+  Value newResultTmp = arith::MulIOp::create(builder, resultTmp, baseTmp);
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), elementType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(thenBlock);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(powerTmpIsOdd, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
                                    resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
   //   p >>= T(1);
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newPowerTmp = builder.create<arith::ShRUIOp>(powerTmp, oneValue);
+  Value newPowerTmp = arith::ShRUIOp::create(builder, powerTmp, oneValue);
 
   //   if (p == T(0))
-  auto newPowerIsZero = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                       newPowerTmp, zeroValue);
   //     return result;
   thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(newResultTmp);
+  func::ReturnOp::create(builder, newResultTmp);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(newPowerIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, newPowerIsZero, thenBlock, fallthroughBlock);
 
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newBaseTmp = builder.create<arith::MulIOp>(baseTmp, baseTmp);
+  Value newBaseTmp = arith::MulIOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  builder.create<cf::BranchOp>(
+  cf::BranchOp::create(builder,
       ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
   return funcOp;
 }
@@ -420,7 +420,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   llvm::raw_string_ostream nameOS(funcName);
   nameOS << '_' << baseType;
   nameOS << '_' << powType;
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
   LLVM::linkage::Linkage inlineLinkage = LLVM::linkage::Linkage::LinkonceODR;
   Attribute linkage =
       LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
@@ -433,46 +433,46 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value oneBValue = builder.create<arith::ConstantOp>(
+  Value oneBValue = arith::ConstantOp::create(builder,
       baseType, builder.getFloatAttr(baseType, 1.0));
-  Value zeroPValue = builder.create<arith::ConstantOp>(
+  Value zeroPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, 0));
-  Value onePValue = builder.create<arith::ConstantOp>(
+  Value onePValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, 1));
-  Value minPValue = builder.create<arith::ConstantOp>(
+  Value minPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMinValue(
                                                    powType.getWidth())));
-  Value maxPValue = builder.create<arith::ConstantOp>(
+  Value maxPValue = arith::ConstantOp::create(builder,
       powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMaxValue(
                                                    powType.getWidth())));
 
   // if (p == Tp{0})
   //   return Tb{1};
   auto pIsZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, zeroPValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroPValue);
   Block *thenBlock = builder.createBlock(funcBody);
-  builder.create<func::ReturnOp>(oneBValue);
+  func::ReturnOp::create(builder, oneBValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == Tp{0}).
   builder.setInsertionPointToEnd(pIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(pIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, pIsZero, thenBlock, fallthroughBlock);
 
   builder.setInsertionPointToEnd(fallthroughBlock);
   // bool isNegativePower{p < Tp{0}}
-  auto pIsNeg = builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, pArg,
+  auto pIsNeg = arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg,
                                               zeroPValue);
   // bool isMin{p == std::numeric_limits<Tp>::min()};
   auto pIsMin =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, pArg, minPValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, minPValue);
 
   // if (isMin) {
   //   p = std::numeric_limits<Tp>::max();
   // } else if (isNegativePower) {
   //   p = -p;
   // }
-  Value negP = builder.create<arith::SubIOp>(zeroPValue, pArg);
-  auto pInit = builder.create<arith::SelectOp>(pIsNeg, negP, pArg);
-  pInit = builder.create<arith::SelectOp>(pIsMin, maxPValue, pInit);
+  Value negP = arith::SubIOp::create(builder, zeroPValue, pArg);
+  auto pInit = arith::SelectOp::create(builder, pIsNeg, negP, pArg);
+  pInit = arith::SelectOp::create(builder, pIsMin, maxPValue, pInit);
 
   // Tb result = Tb{1};
   // Tb origBase = Tb{b};
@@ -489,7 +489,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
       {builder.getLoc(), builder.getLoc(), builder.getLoc()});
   // Set initial values of 'result', 'b' and 'p' for the loop.
   builder.setInsertionPointToEnd(pInit->getBlock());
-  builder.create<cf::BranchOp>(loopHeader, ValueRange{oneBValue, bArg, pInit});
+  cf::BranchOp::create(builder, loopHeader, ValueRange{oneBValue, bArg, pInit});
 
   // Create loop body.
   Value resultTmp = loopHeader->getArgument(0);
@@ -498,29 +498,29 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & Tp{1})
-  auto powerTmpIsOdd = builder.create<arith::CmpIOp>(
+  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
       arith::CmpIPredicate::ne,
-      builder.create<arith::AndIOp>(powerTmp, onePValue), zeroPValue);
+      arith::AndIOp::create(builder, powerTmp, onePValue), zeroPValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
-  Value newResultTmp = builder.create<arith::MulFOp>(resultTmp, baseTmp);
+  Value newResultTmp = arith::MulFOp::create(builder, resultTmp, baseTmp);
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(thenBlock);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
   // Set up conditional branch for (p & Tp{1}).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
-  builder.create<cf::CondBranchOp>(powerTmpIsOdd, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
                                    resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
   //   p >>= Tp{1};
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newPowerTmp = builder.create<arith::ShRUIOp>(powerTmp, onePValue);
+  Value newPowerTmp = arith::ShRUIOp::create(builder, powerTmp, onePValue);
 
   //   if (p == Tp{0})
-  auto newPowerIsZero = builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+  auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
                                                       newPowerTmp, zeroPValue);
   //     break;
   //
@@ -531,9 +531,9 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
-  Value newBaseTmp = builder.create<arith::MulFOp>(baseTmp, baseTmp);
+  Value newBaseTmp = arith::MulFOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  builder.create<cf::BranchOp>(
+  cf::BranchOp::create(builder,
       ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
 
   // Set up conditional branch for early loop exit:
@@ -542,7 +542,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Block *loopExit = builder.createBlock(funcBody, funcBody->end(), baseType,
                                         builder.getLoc());
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  builder.create<cf::CondBranchOp>(newPowerIsZero, loopExit, newResultTmp,
+  cf::CondBranchOp::create(builder, newPowerIsZero, loopExit, newResultTmp,
                                    fallthroughBlock, ValueRange{});
 
   // if (isMin) {
@@ -553,11 +553,11 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   fallthroughBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(loopExit);
-  builder.create<cf::CondBranchOp>(pIsMin, thenBlock, fallthroughBlock,
+  cf::CondBranchOp::create(builder, pIsMin, thenBlock, fallthroughBlock,
                                    newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
-  newResultTmp = builder.create<arith::MulFOp>(newResultTmp, bArg);
-  builder.create<cf::BranchOp>(newResultTmp, fallthroughBlock);
+  newResultTmp = arith::MulFOp::create(builder, newResultTmp, bArg);
+  cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
 
   /// if (isNegativePower) {
   ///   result = Tb{1} / result;
@@ -567,15 +567,15 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Block *returnBlock = builder.createBlock(funcBody, funcBody->end(), baseType,
                                            builder.getLoc());
   builder.setInsertionPointToEnd(fallthroughBlock);
-  builder.create<cf::CondBranchOp>(pIsNeg, thenBlock, returnBlock,
+  cf::CondBranchOp::create(builder, pIsNeg, thenBlock, returnBlock,
                                    newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
-  newResultTmp = builder.create<arith::DivFOp>(oneBValue, newResultTmp);
-  builder.create<cf::BranchOp>(newResultTmp, returnBlock);
+  newResultTmp = arith::DivFOp::create(builder, oneBValue, newResultTmp);
+  cf::BranchOp::create(builder, newResultTmp, returnBlock);
 
   // return result;
   builder.setInsertionPointToEnd(returnBlock);
-  builder.create<func::ReturnOp>(returnBlock->getArgument(0));
+  func::ReturnOp::create(builder, returnBlock->getArgument(0));
 
   return funcOp;
 }
@@ -667,7 +667,7 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
   nameOS << '_' << elementType;
   FunctionType funcType =
       FunctionType::get(builder.getContext(), {elementType}, elementType);
-  auto funcOp = builder.create<func::FuncOp>(funcName, funcType);
+  auto funcOp = func::FuncOp::create(builder, funcName, funcType);
 
   // LinkonceODR ensures that there is only one implementation of this function
   // across all math.ctlz functions that are lowered in this way.
@@ -683,32 +683,32 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
 
   Value arg = funcOp.getArgument(0);
   Type indexType = builder.getIndexType();
-  Value bitWidthValue = builder.create<arith::ConstantOp>(
+  Value bitWidthValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, bitWidth));
-  Value zeroValue = builder.create<arith::ConstantOp>(
+  Value zeroValue = arith::ConstantOp::create(builder,
       elementType, builder.getIntegerAttr(elementType, 0));
 
   Value inputEqZero =
-      builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, arg, zeroValue);
+      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, arg, zeroValue);
 
   // if input == 0, return bit width, else enter loop.
-  scf::IfOp ifOp = builder.create<scf::IfOp>(
+  scf::IfOp ifOp = scf::IfOp::create(builder,
       elementType, inputEqZero, /*addThenBlock=*/true, /*addElseBlock=*/true);
   ifOp.getThenBodyBuilder().create<scf::YieldOp>(loc, bitWidthValue);
 
   auto elseBuilder =
       ImplicitLocOpBuilder::atBlockEnd(loc, &ifOp.getElseRegion().front());
 
-  Value oneIndex = elseBuilder.create<arith::ConstantOp>(
+  Value oneIndex = arith::ConstantOp::create(elseBuilder,
       indexType, elseBuilder.getIndexAttr(1));
-  Value oneValue = elseBuilder.create<arith::ConstantOp>(
+  Value oneValue = arith::ConstantOp::create(elseBuilder,
       elementType, elseBuilder.getIntegerAttr(elementType, 1));
-  Value bitWidthIndex = elseBuilder.create<arith::ConstantOp>(
+  Value bitWidthIndex = arith::ConstantOp::create(elseBuilder,
       indexType, elseBuilder.getIndexAttr(bitWidth));
-  Value nValue = elseBuilder.create<arith::ConstantOp>(
+  Value nValue = arith::ConstantOp::create(elseBuilder,
       elementType, elseBuilder.getIntegerAttr(elementType, 0));
 
-  auto loop = elseBuilder.create<scf::ForOp>(
+  auto loop = scf::ForOp::create(elseBuilder,
       oneIndex, bitWidthIndex, oneIndex,
       // Initial values for two loop induction variables, the arg which is being
       // shifted left in each iteration, and the n value which tracks the count
@@ -725,25 +725,25 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
         Value argIter = args[0];
         Value nIter = args[1];
 
-        Value argIsNonNegative = b.create<arith::CmpIOp>(
+        Value argIsNonNegative = arith::CmpIOp::create(b,
             loc, arith::CmpIPredicate::slt, argIter, zeroValue);
-        scf::IfOp ifOp = b.create<scf::IfOp>(
+        scf::IfOp ifOp = scf::IfOp::create(b,
             loc, argIsNonNegative,
             [&](OpBuilder &b, Location loc) {
               // If arg is negative, continue (effectively, break)
-              b.create<scf::YieldOp>(loc, ValueRange{argIter, nIter});
+              scf::YieldOp::create(b, loc, ValueRange{argIter, nIter});
             },
             [&](OpBuilder &b, Location loc) {
               // Otherwise, increment n and shift arg left.
-              Value nNext = b.create<arith::AddIOp>(loc, nIter, oneValue);
-              Value argNext = b.create<arith::ShLIOp>(loc, argIter, oneValue);
-              b.create<scf::YieldOp>(loc, ValueRange{argNext, nNext});
+              Value nNext = arith::AddIOp::create(b, loc, nIter, oneValue);
+              Value argNext = arith::ShLIOp::create(b, loc, argIter, oneValue);
+              scf::YieldOp::create(b, loc, ValueRange{argNext, nNext});
             });
-        b.create<scf::YieldOp>(loc, ifOp.getResults());
+        scf::YieldOp::create(b, loc, ifOp.getResults());
       });
-  elseBuilder.create<scf::YieldOp>(loop.getResult(1));
+  scf::YieldOp::create(elseBuilder, loop.getResult(1));
 
-  builder.create<func::ReturnOp>(ifOp.getResult(0));
+  func::ReturnOp::create(builder, ifOp.getResult(0));
   return funcOp;
 }
 
diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
index f4d69ce8235bb..26559f394c221 100644
--- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
+++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
@@ -107,7 +107,7 @@ struct IntOpWithFlagLowering : public ConvertOpToLLVMPattern<MathOp> {
     return LLVM::detail::handleMultidimensionalVectors(
         op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
-          return rewriter.create<LLVMOp>(loc, llvm1DVectorTy, operands[0],
+          return LLVMOp::create(rewriter, loc, llvm1DVectorTy, operands[0],
                                          false);
         },
         rewriter);
@@ -145,14 +145,14 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (LLVM::isCompatibleVectorType(llvmOperandType)) {
-        one = rewriter.create<LLVM::ConstantOp>(
+        one = LLVM::ConstantOp::create(rewriter,
             loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
+        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
-      auto exp = rewriter.create<LLVM::ExpOp>(loc, adaptor.getOperand(),
+      auto exp = LLVM::ExpOp::create(rewriter, loc, adaptor.getOperand(),
                                               expAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FSubOp>(
           op, llvmOperandType, ValueRange{exp, one}, subAttrs.getAttrs());
@@ -171,10 +171,10 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto exp = rewriter.create<LLVM::ExpOp>(
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto exp = LLVM::ExpOp::create(rewriter,
               loc, llvm1DVectorTy, operands[0], expAttrs.getAttrs());
-          return rewriter.create<LLVM::FSubOp>(
+          return LLVM::FSubOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{exp, one}, subAttrs.getAttrs());
         },
         rewriter);
@@ -205,14 +205,14 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one =
           isa<VectorType>(llvmOperandType)
-              ? rewriter.create<LLVM::ConstantOp>(
+              ? LLVM::ConstantOp::create(rewriter,
                     loc, llvmOperandType,
                     SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                            floatOne))
-              : rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType,
+              : LLVM::ConstantOp::create(rewriter, loc, llvmOperandType,
                                                   floatOne);
 
-      auto add = rewriter.create<LLVM::FAddOp>(
+      auto add = LLVM::FAddOp::create(rewriter,
           loc, llvmOperandType, ValueRange{one, adaptor.getOperand()},
           addAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::LogOp>(
@@ -232,11 +232,11 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto add = rewriter.create<LLVM::FAddOp>(loc, llvm1DVectorTy,
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto add = LLVM::FAddOp::create(rewriter, loc, llvm1DVectorTy,
                                                    ValueRange{one, operands[0]},
                                                    addAttrs.getAttrs());
-          return rewriter.create<LLVM::LogOp>(
+          return LLVM::LogOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{add}, logAttrs.getAttrs());
         },
         rewriter);
@@ -267,14 +267,14 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (isa<VectorType>(llvmOperandType)) {
-        one = rewriter.create<LLVM::ConstantOp>(
+        one = LLVM::ConstantOp::create(rewriter,
             loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
+        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
-      auto sqrt = rewriter.create<LLVM::SqrtOp>(loc, adaptor.getOperand(),
+      auto sqrt = LLVM::SqrtOp::create(rewriter, loc, adaptor.getOperand(),
                                                 sqrtAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FDivOp>(
           op, llvmOperandType, ValueRange{one, sqrt}, divAttrs.getAttrs());
@@ -293,10 +293,10 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
                                     {numElements.isScalable()}),
               floatOne);
           auto one =
-              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
-          auto sqrt = rewriter.create<LLVM::SqrtOp>(
+              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto sqrt = LLVM::SqrtOp::create(rewriter,
               loc, llvm1DVectorTy, operands[0], sqrtAttrs.getAttrs());
-          return rewriter.create<LLVM::FDivOp>(
+          return LLVM::FDivOp::create(rewriter,
               loc, llvm1DVectorTy, ValueRange{one, sqrt}, divAttrs.getAttrs());
         },
         rewriter);
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index a0ce7d3b75fc2..19303dc3e8b55 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -84,7 +84,7 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto shape = vecType.getShape();
   int64_t numElements = vecType.getNumElements();
 
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(
                vecType, FloatAttr::get(vecType.getElementType(), 0.0)));
   SmallVector<int64_t> strides = computeStrides(shape);
@@ -93,11 +93,11 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     SmallVector<Value> operands;
     for (auto input : op->getOperands())
       operands.push_back(
-          rewriter.create<vector::ExtractOp>(loc, input, positions));
+          vector::ExtractOp::create(rewriter, loc, input, positions));
     Value scalarOp =
-        rewriter.create<Op>(loc, vecType.getElementType(), operands);
+        Op::create(rewriter, loc, vecType.getElementType(), operands);
     result =
-        rewriter.create<vector::InsertOp>(loc, scalarOp, result, positions);
+        vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
   }
   rewriter.replaceOp(op, {result});
   return success();
@@ -114,9 +114,9 @@ PromoteOpToF32<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto f32 = rewriter.getF32Type();
   auto extendedOperands = llvm::to_vector(
       llvm::map_range(op->getOperands(), [&](Value operand) -> Value {
-        return rewriter.create<arith::ExtFOp>(loc, f32, operand);
+        return arith::ExtFOp::create(rewriter, loc, f32, operand);
       }));
-  auto newOp = rewriter.create<Op>(loc, f32, extendedOperands);
+  auto newOp = Op::create(rewriter, loc, f32, extendedOperands);
   rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, opType, newOp);
   return success();
 }
@@ -139,7 +139,7 @@ ScalarOpToLibmCall<Op>::matchAndRewrite(Op op,
     rewriter.setInsertionPointToStart(&module->getRegion(0).front());
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
-    opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), name,
+    opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
                                            opFunctionTy);
     opFunc.setPrivate();
 
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 59db14ed816be..26891aa7c2025 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -36,11 +36,11 @@ static Value getScalarOrVectorI32Constant(Type type, int value,
     if (!vectorType.getElementType().isInteger(32))
       return nullptr;
     SmallVector<int> values(vectorType.getNumElements(), value);
-    return builder.create<spirv::ConstantOp>(loc, type,
+    return spirv::ConstantOp::create(builder, loc, type,
                                              builder.getI32VectorAttr(values));
   }
   if (type.isInteger(32))
-    return builder.create<spirv::ConstantOp>(loc, type,
+    return spirv::ConstantOp::create(builder, loc, type,
                                              builder.getI32IntegerAttr(value));
 
   return nullptr;
@@ -144,9 +144,9 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
     Type intType = rewriter.getIntegerType(bitwidth);
     uint64_t intValue = uint64_t(1) << (bitwidth - 1);
 
-    Value signMask = rewriter.create<spirv::ConstantOp>(
+    Value signMask = spirv::ConstantOp::create(rewriter,
         loc, intType, rewriter.getIntegerAttr(intType, intValue));
-    Value valueMask = rewriter.create<spirv::ConstantOp>(
+    Value valueMask = spirv::ConstantOp::create(rewriter,
         loc, intType, rewriter.getIntegerAttr(intType, intValue - 1u));
 
     if (auto vectorType = dyn_cast<VectorType>(type)) {
@@ -156,24 +156,24 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
 
       SmallVector<Value> signSplat(count, signMask);
       signMask =
-          rewriter.create<spirv::CompositeConstructOp>(loc, intType, signSplat);
+          spirv::CompositeConstructOp::create(rewriter, loc, intType, signSplat);
 
       SmallVector<Value> valueSplat(count, valueMask);
-      valueMask = rewriter.create<spirv::CompositeConstructOp>(loc, intType,
+      valueMask = spirv::CompositeConstructOp::create(rewriter, loc, intType,
                                                                valueSplat);
     }
 
     Value lhsCast =
-        rewriter.create<spirv::BitcastOp>(loc, intType, adaptor.getLhs());
+        spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getLhs());
     Value rhsCast =
-        rewriter.create<spirv::BitcastOp>(loc, intType, adaptor.getRhs());
+        spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getRhs());
 
-    Value value = rewriter.create<spirv::BitwiseAndOp>(
+    Value value = spirv::BitwiseAndOp::create(rewriter,
         loc, intType, ValueRange{lhsCast, valueMask});
-    Value sign = rewriter.create<spirv::BitwiseAndOp>(
+    Value sign = spirv::BitwiseAndOp::create(rewriter,
         loc, intType, ValueRange{rhsCast, signMask});
 
-    Value result = rewriter.create<spirv::BitwiseOrOp>(loc, intType,
+    Value result = spirv::BitwiseOrOp::create(rewriter, loc, intType,
                                                        ValueRange{value, sign});
     rewriter.replaceOpWithNewOp<spirv::BitcastOp>(copySignOp, type, result);
     return success();
@@ -214,18 +214,18 @@ struct CountLeadingZerosPattern final
     Value val31 = getScalarOrVectorI32Constant(type, 31, rewriter, loc);
     Value val32 = getScalarOrVectorI32Constant(type, 32, rewriter, loc);
 
-    Value msb = rewriter.create<spirv::GLFindUMsbOp>(loc, input);
+    Value msb = spirv::GLFindUMsbOp::create(rewriter, loc, input);
     // We need to subtract from 31 given that the index returned by GLSL
     // FindUMsb is counted from the least significant bit. Theoretically this
     // also gives the correct result even if the integer has all zero bits, in
     // which case GL FindUMsb would return -1.
-    Value subMsb = rewriter.create<spirv::ISubOp>(loc, val31, msb);
+    Value subMsb = spirv::ISubOp::create(rewriter, loc, val31, msb);
     // However, certain Vulkan implementations have driver bugs for the corner
     // case where the input is zero. And.. it can be smart to optimize a select
     // only involving the corner case. So separately compute the result when the
     // input is either zero or one.
-    Value subInput = rewriter.create<spirv::ISubOp>(loc, val32, input);
-    Value cmp = rewriter.create<spirv::ULessThanEqualOp>(loc, input, val1);
+    Value subInput = spirv::ISubOp::create(rewriter, loc, val32, input);
+    Value cmp = spirv::ULessThanEqualOp::create(rewriter, loc, input, val1);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(countOp, cmp, subInput,
                                                  subMsb);
     return success();
@@ -253,7 +253,7 @@ struct ExpM1OpPattern final : public OpConversionPattern<math::ExpM1Op> {
     if (!type)
       return failure();
 
-    Value exp = rewriter.create<ExpOp>(loc, type, adaptor.getOperand());
+    Value exp = ExpOp::create(rewriter, loc, type, adaptor.getOperand());
     auto one = spirv::ConstantOp::getOne(type, loc, rewriter);
     rewriter.replaceOpWithNewOp<spirv::FSubOp>(operation, exp, one);
     return success();
@@ -283,7 +283,7 @@ struct Log1pOpPattern final : public OpConversionPattern<math::Log1pOp> {
 
     auto one = spirv::ConstantOp::getOne(type, operation.getLoc(), rewriter);
     Value onePlus =
-        rewriter.create<spirv::FAddOp>(loc, one, adaptor.getOperand());
+        spirv::FAddOp::create(rewriter, loc, one, adaptor.getOperand());
     rewriter.replaceOpWithNewOp<LogOp>(operation, type, onePlus);
     return success();
   }
@@ -321,14 +321,14 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
 
     auto getConstantValue = [&](double value) {
       if (auto floatType = dyn_cast<FloatType>(type)) {
-        return rewriter.create<spirv::ConstantOp>(
+        return spirv::ConstantOp::create(rewriter,
             loc, type, rewriter.getFloatAttr(floatType, value));
       }
       if (auto vectorType = dyn_cast<VectorType>(type)) {
         Type elemType = vectorType.getElementType();
 
         if (isa<FloatType>(elemType)) {
-          return rewriter.create<spirv::ConstantOp>(
+          return spirv::ConstantOp::create(rewriter,
               loc, type,
               DenseFPElementsAttr::get(
                   vectorType, FloatAttr::get(elemType, value).getValue()));
@@ -341,7 +341,7 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
     Value constantValue = getConstantValue(
         std::is_same<MathLogOp, math::Log2Op>() ? log2Reciprocal
                                                 : log10Reciprocal);
-    Value log = rewriter.create<SpirvLogOp>(loc, adaptor.getOperand());
+    Value log = SpirvLogOp::create(rewriter, loc, adaptor.getOperand());
     rewriter.replaceOpWithNewOp<spirv::FMulOp>(operation, type, log,
                                                constantValue);
     return success();
@@ -386,7 +386,7 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
     Location loc = powfOp.getLoc();
     Value zero = spirv::ConstantOp::getZero(operandType, loc, rewriter);
     Value lessThan =
-        rewriter.create<spirv::FOrdLessThanOp>(loc, adaptor.getLhs(), zero);
+        spirv::FOrdLessThanOp::create(rewriter, loc, adaptor.getLhs(), zero);
 
     // Per C/C++ spec:
     // > pow(base, exponent) returns NaN (and raises FE_INVALID) if base is
@@ -394,11 +394,11 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
     // Calculate the reminder from the exponent and check whether it is zero.
     Value floatOne = spirv::ConstantOp::getOne(operandType, loc, rewriter);
     Value expRem =
-        rewriter.create<spirv::FRemOp>(loc, adaptor.getRhs(), floatOne);
+        spirv::FRemOp::create(rewriter, loc, adaptor.getRhs(), floatOne);
     Value expRemNonZero =
-        rewriter.create<spirv::FOrdNotEqualOp>(loc, expRem, zero);
+        spirv::FOrdNotEqualOp::create(rewriter, loc, expRem, zero);
     Value cmpNegativeWithFractionalExp =
-        rewriter.create<spirv::LogicalAndOp>(loc, expRemNonZero, lessThan);
+        spirv::LogicalAndOp::create(rewriter, loc, expRemNonZero, lessThan);
     // Create NaN result and replace base value if conditions are met.
     const auto &floatSemantics = scalarFloatType.getFloatSemantics();
     const auto nan = APFloat::getNaN(floatSemantics);
@@ -407,10 +407,10 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
       nanAttr = DenseElementsAttr::get(vectorType, nan);
 
     Value NanValue =
-        rewriter.create<spirv::ConstantOp>(loc, operandType, nanAttr);
-    Value lhs = rewriter.create<spirv::SelectOp>(
+        spirv::ConstantOp::create(rewriter, loc, operandType, nanAttr);
+    Value lhs = spirv::SelectOp::create(rewriter,
         loc, cmpNegativeWithFractionalExp, NanValue, adaptor.getLhs());
-    Value abs = rewriter.create<spirv::GLFAbsOp>(loc, lhs);
+    Value abs = spirv::GLFAbsOp::create(rewriter, loc, lhs);
 
     // TODO: The following just forcefully casts y into an integer value in
     // order to properly propagate the sign, assuming integer y cases. It
@@ -418,18 +418,18 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
 
     // Cast exponent to integer and calculate exponent % 2 != 0.
     Value intRhs =
-        rewriter.create<spirv::ConvertFToSOp>(loc, intType, adaptor.getRhs());
+        spirv::ConvertFToSOp::create(rewriter, loc, intType, adaptor.getRhs());
     Value intOne = spirv::ConstantOp::getOne(intType, loc, rewriter);
     Value bitwiseAndOne =
-        rewriter.create<spirv::BitwiseAndOp>(loc, intRhs, intOne);
-    Value isOdd = rewriter.create<spirv::IEqualOp>(loc, bitwiseAndOne, intOne);
+        spirv::BitwiseAndOp::create(rewriter, loc, intRhs, intOne);
+    Value isOdd = spirv::IEqualOp::create(rewriter, loc, bitwiseAndOne, intOne);
 
     // calculate pow based on abs(lhs)^rhs.
-    Value pow = rewriter.create<spirv::GLPowOp>(loc, abs, adaptor.getRhs());
-    Value negate = rewriter.create<spirv::FNegateOp>(loc, pow);
+    Value pow = spirv::GLPowOp::create(rewriter, loc, abs, adaptor.getRhs());
+    Value negate = spirv::FNegateOp::create(rewriter, loc, pow);
     // if the exponent is odd and lhs < 0, negate the result.
     Value shouldNegate =
-        rewriter.create<spirv::LogicalAndOp>(loc, lessThan, isOdd);
+        spirv::LogicalAndOp::create(rewriter, loc, lessThan, isOdd);
     rewriter.replaceOpWithNewOp<spirv::SelectOp>(powfOp, shouldNegate, negate,
                                                  pow);
     return success();
@@ -455,22 +455,22 @@ struct RoundOpPattern final : public OpConversionPattern<math::RoundOp> {
     auto one = spirv::ConstantOp::getOne(ty, loc, rewriter);
     Value half;
     if (VectorType vty = dyn_cast<VectorType>(ty)) {
-      half = rewriter.create<spirv::ConstantOp>(
+      half = spirv::ConstantOp::create(rewriter,
           loc, vty,
           DenseElementsAttr::get(vty,
                                  rewriter.getFloatAttr(ety, 0.5).getValue()));
     } else {
-      half = rewriter.create<spirv::ConstantOp>(
+      half = spirv::ConstantOp::create(rewriter,
           loc, ty, rewriter.getFloatAttr(ety, 0.5));
     }
 
-    auto abs = rewriter.create<spirv::GLFAbsOp>(loc, operand);
-    auto floor = rewriter.create<spirv::GLFloorOp>(loc, abs);
-    auto sub = rewriter.create<spirv::FSubOp>(loc, abs, floor);
+    auto abs = spirv::GLFAbsOp::create(rewriter, loc, operand);
+    auto floor = spirv::GLFloorOp::create(rewriter, loc, abs);
+    auto sub = spirv::FSubOp::create(rewriter, loc, abs, floor);
     auto greater =
-        rewriter.create<spirv::FOrdGreaterThanEqualOp>(loc, sub, half);
-    auto select = rewriter.create<spirv::SelectOp>(loc, greater, one, zero);
-    auto add = rewriter.create<spirv::FAddOp>(loc, floor, select);
+        spirv::FOrdGreaterThanEqualOp::create(rewriter, loc, sub, half);
+    auto select = spirv::SelectOp::create(rewriter, loc, greater, one, zero);
+    auto add = spirv::FAddOp::create(rewriter, loc, floor, select);
     rewriter.replaceOpWithNewOp<math::CopySignOp>(roundOp, add, operand);
     return success();
   }
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
index db244d1d1cac8..db6ce600af76a 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
@@ -161,7 +161,7 @@ struct ConvertLoad final : public OpConversionPattern<memref::LoadOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = rewriter.create<emitc::SubscriptOp>(
+    auto subscript = emitc::SubscriptOp::create(rewriter,
         op.getLoc(), arrayValue, operands.getIndices());
 
     rewriter.replaceOpWithNewOp<emitc::LoadOp>(op, resultTy, subscript);
@@ -181,7 +181,7 @@ struct ConvertStore final : public OpConversionPattern<memref::StoreOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = rewriter.create<emitc::SubscriptOp>(
+    auto subscript = emitc::SubscriptOp::create(rewriter,
         op.getLoc(), arrayValue, operands.getIndices());
     rewriter.replaceOpWithNewOp<emitc::AssignOp>(op, subscript,
                                                  operands.getValue());
@@ -212,7 +212,7 @@ void mlir::populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   };
 
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 83681b2d5fd87..4ef2168380620 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -87,12 +87,12 @@ getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
 ///   aligned = bumped - bumped % alignment
 static Value createAligned(ConversionPatternRewriter &rewriter, Location loc,
                            Value input, Value alignment) {
-  Value one = rewriter.create<LLVM::ConstantOp>(loc, alignment.getType(),
+  Value one = LLVM::ConstantOp::create(rewriter, loc, alignment.getType(),
                                                 rewriter.getIndexAttr(1));
-  Value bump = rewriter.create<LLVM::SubOp>(loc, alignment, one);
-  Value bumped = rewriter.create<LLVM::AddOp>(loc, input, bump);
-  Value mod = rewriter.create<LLVM::URemOp>(loc, bumped, alignment);
-  return rewriter.create<LLVM::SubOp>(loc, bumped, mod);
+  Value bump = LLVM::SubOp::create(rewriter, loc, alignment, one);
+  Value bumped = LLVM::AddOp::create(rewriter, loc, input, bump);
+  Value mod = LLVM::URemOp::create(rewriter, loc, bumped, alignment);
+  return LLVM::SubOp::create(rewriter, loc, bumped, mod);
 }
 
 /// Computes the byte size for the MemRef element type.
@@ -123,7 +123,7 @@ static Value castAllocFuncResult(ConversionPatternRewriter &rewriter,
   assert(succeeded(maybeMemrefAddrSpace) && "unsupported address space");
   unsigned memrefAddrSpace = *maybeMemrefAddrSpace;
   if (allocatedPtrTy.getAddressSpace() != memrefAddrSpace)
-    allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+    allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
         loc, LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
         allocatedPtr);
   return allocatedPtr;
@@ -168,14 +168,14 @@ class AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
     Value alignment = getAlignment(rewriter, loc, op);
     if (alignment) {
       // Adjust the allocation size to consider alignment.
-      sizeBytes = rewriter.create<LLVM::AddOp>(loc, sizeBytes, alignment);
+      sizeBytes = LLVM::AddOp::create(rewriter, loc, sizeBytes, alignment);
     }
 
     // Allocate the underlying buffer.
     Type elementPtrType = this->getElementPtrType(memRefType);
     assert(elementPtrType && "could not compute element ptr type");
     auto results =
-        rewriter.create<LLVM::CallOp>(loc, allocFuncOp.value(), sizeBytes);
+        LLVM::CallOp::create(rewriter, loc, allocFuncOp.value(), sizeBytes);
 
     Value allocatedPtr =
         castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
@@ -184,11 +184,11 @@ class AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
     if (alignment) {
       // Compute the aligned pointer.
       Value allocatedInt =
-          rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), allocatedPtr);
+          LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), allocatedPtr);
       Value alignmentInt =
           createAligned(rewriter, loc, allocatedInt, alignment);
       alignedPtr =
-          rewriter.create<LLVM::IntToPtrOp>(loc, elementPtrType, alignmentInt);
+          LLVM::IntToPtrOp::create(rewriter, loc, elementPtrType, alignmentInt);
     }
 
     // Create the MemRef descriptor.
@@ -268,7 +268,7 @@ class AlignedAllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
       sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
 
     Type elementPtrType = this->getElementPtrType(memRefType);
-    auto results = rewriter.create<LLVM::CallOp>(
+    auto results = LLVM::CallOp::create(rewriter,
         loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
 
     Value ptr =
@@ -360,7 +360,7 @@ struct AllocaOpLowering : public ConvertOpToLLVMPattern<memref::AllocaOp> {
     auto elementPtrType =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addrSpace);
 
-    auto allocatedElementPtr = rewriter.create<LLVM::AllocaOp>(
+    auto allocatedElementPtr = LLVM::AllocaOp::create(rewriter,
         loc, elementPtrType, elementType, size, op.getAlignment().value_or(0));
 
     // Create the MemRef descriptor.
@@ -397,7 +397,7 @@ struct AllocaScopeOpLowering
           remainingOpsBlock, allocaScopeOp.getResultTypes(),
           SmallVector<Location>(allocaScopeOp->getNumResults(),
                                 allocaScopeOp.getLoc()));
-      rewriter.create<LLVM::BrOp>(loc, ValueRange(), remainingOpsBlock);
+      LLVM::BrOp::create(rewriter, loc, ValueRange(), remainingOpsBlock);
     }
 
     // Inline body region.
@@ -407,8 +407,8 @@ struct AllocaScopeOpLowering
 
     // Save stack and then branch into the body of the region.
     rewriter.setInsertionPointToEnd(currentBlock);
-    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
-    rewriter.create<LLVM::BrOp>(loc, ValueRange(), beforeBody);
+    auto stackSaveOp = LLVM::StackSaveOp::create(rewriter, loc, getPtrType());
+    LLVM::BrOp::create(rewriter, loc, ValueRange(), beforeBody);
 
     // Replace the alloca_scope return with a branch that jumps out of the body.
     // Stack restore before leaving the body region.
@@ -420,7 +420,7 @@ struct AllocaScopeOpLowering
 
     // Insert stack restore before jumping out the body of the region.
     rewriter.setInsertionPoint(branchOp);
-    rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);
+    LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
 
     // Replace the op with values return from the body region.
     rewriter.replaceOp(allocaScopeOp, continueBlock->getArguments());
@@ -451,10 +451,10 @@ struct AssumeAlignmentOpLowering
     // This is more direct than ptrtoint-based checks, is explicitly supported,
     // and works with non-integral address spaces.
     Value trueCond =
-        rewriter.create<LLVM::ConstantOp>(loc, rewriter.getBoolAttr(true));
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getBoolAttr(true));
     Value alignmentConst =
         createIndexAttrConstant(rewriter, loc, getIndexType(), alignment);
-    rewriter.create<LLVM::AssumeOp>(loc, trueCond, LLVM::AssumeAlignTag(), ptr,
+    LLVM::AssumeOp::create(rewriter, loc, trueCond, LLVM::AssumeAlignTag(), ptr,
                                     alignmentConst);
     rewriter.replaceOp(op, memref);
     return success();
@@ -559,16 +559,16 @@ struct DimOpLowering : public ConvertOpToLLVMPattern<memref::DimOp> {
     // Get pointer to offset field of memref<element_type> descriptor.
     auto indexPtrTy =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addressSpace);
-    Value offsetPtr = rewriter.create<LLVM::GEPOp>(
+    Value offsetPtr = LLVM::GEPOp::create(rewriter,
         loc, indexPtrTy, elementType, underlyingRankedDesc,
         ArrayRef<LLVM::GEPArg>{0, 2});
 
     // The size value that we have to extract can be obtained using GEPop with
     // `dimOp.index() + 1` index argument.
-    Value idxPlusOne = rewriter.create<LLVM::AddOp>(
+    Value idxPlusOne = LLVM::AddOp::create(rewriter,
         loc, createIndexAttrConstant(rewriter, loc, getIndexType(), 1),
         adaptor.getIndex());
-    Value sizePtr = rewriter.create<LLVM::GEPOp>(
+    Value sizePtr = LLVM::GEPOp::create(rewriter,
         loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr,
         idxPlusOne);
     return rewriter
@@ -674,9 +674,9 @@ struct GenericAtomicRMWOpLowering
     auto memRefType = cast<MemRefType>(atomicOp.getMemref().getType());
     auto dataPtr = getStridedElementPtr(
         rewriter, loc, memRefType, adaptor.getMemref(), adaptor.getIndices());
-    Value init = rewriter.create<LLVM::LoadOp>(
+    Value init = LLVM::LoadOp::create(rewriter,
         loc, typeConverter->convertType(memRefType.getElementType()), dataPtr);
-    rewriter.create<LLVM::BrOp>(loc, init, loopBlock);
+    LLVM::BrOp::create(rewriter, loc, init, loopBlock);
 
     // Prepare the body of the loop block.
     rewriter.setInsertionPointToStart(loopBlock);
@@ -696,14 +696,14 @@ struct GenericAtomicRMWOpLowering
     // Append the cmpxchg op to the end of the loop block.
     auto successOrdering = LLVM::AtomicOrdering::acq_rel;
     auto failureOrdering = LLVM::AtomicOrdering::monotonic;
-    auto cmpxchg = rewriter.create<LLVM::AtomicCmpXchgOp>(
+    auto cmpxchg = LLVM::AtomicCmpXchgOp::create(rewriter,
         loc, dataPtr, loopArgument, result, successOrdering, failureOrdering);
     // Extract the %new_loaded and %ok values from the pair.
-    Value newLoaded = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 0);
-    Value ok = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 1);
+    Value newLoaded = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 0);
+    Value ok = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 1);
 
     // Conditionally branch to the end or back to the loop depending on %ok.
-    rewriter.create<LLVM::CondBrOp>(loc, ok, endBlock, ArrayRef<Value>(),
+    LLVM::CondBrOp::create(rewriter, loc, ok, endBlock, ArrayRef<Value>(),
                                     loopBlock, newLoaded);
 
     rewriter.setInsertionPointToEnd(endBlock);
@@ -796,8 +796,8 @@ class GlobalMemrefOpLowering : public ConvertOpToLLVMPattern<memref::GlobalOp> {
     if (!isExternal && isUninitialized) {
       rewriter.createBlock(&newGlobal.getInitializerRegion());
       Value undef[] = {
-          rewriter.create<LLVM::UndefOp>(newGlobal.getLoc(), arrayTy)};
-      rewriter.create<LLVM::ReturnOp>(newGlobal.getLoc(), undef);
+          LLVM::UndefOp::create(rewriter, newGlobal.getLoc(), arrayTy)};
+      LLVM::ReturnOp::create(rewriter, newGlobal.getLoc(), undef);
     }
     return success();
   }
@@ -842,11 +842,11 @@ struct GetGlobalMemrefOpLowering
     Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter());
     auto ptrTy = LLVM::LLVMPointerType::get(rewriter.getContext(), memSpace);
     auto addressOf =
-        rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, op.getName());
+        LLVM::AddressOfOp::create(rewriter, loc, ptrTy, op.getName());
 
     // Get the address of the first element in the array by creating a GEP with
     // the address of the GV as the base, and (rank + 1) number of 0 indices.
-    auto gep = rewriter.create<LLVM::GEPOp>(
+    auto gep = LLVM::GEPOp::create(rewriter,
         loc, ptrTy, arrayTy, addressOf,
         SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
 
@@ -857,7 +857,7 @@ struct GetGlobalMemrefOpLowering
     Value deadBeefConst =
         createIndexAttrConstant(rewriter, op->getLoc(), intPtrType, 0xdeadbeef);
     auto deadBeefPtr =
-        rewriter.create<LLVM::IntToPtrOp>(loc, ptrTy, deadBeefConst);
+        LLVM::IntToPtrOp::create(rewriter, loc, ptrTy, deadBeefConst);
 
     // Both allocated and aligned pointers are same. We could potentially stash
     // a nullptr for the allocated pointer since we do not expect any dealloc.
@@ -1009,7 +1009,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
           loc, adaptor.getSource(), rewriter);
 
       // rank = ConstantOp srcRank
-      auto rankVal = rewriter.create<LLVM::ConstantOp>(
+      auto rankVal = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(rank));
       // poison = PoisonOp
       UnrankedMemRefDescriptor memRefDesc =
@@ -1029,7 +1029,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
       auto ptr = memRefDesc.memRefDescPtr(rewriter, loc);
 
       // struct = LoadOp ptr
-      auto loadOp = rewriter.create<LLVM::LoadOp>(loc, targetStructType, ptr);
+      auto loadOp = LLVM::LoadOp::create(rewriter, loc, targetStructType, ptr);
       rewriter.replaceOp(memRefCastOp, loadOp.getResult());
     } else {
       llvm_unreachable("Unsupported unranked memref to unranked memref cast");
@@ -1063,31 +1063,31 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     MemRefDescriptor srcDesc(adaptor.getSource());
 
     // Compute number of elements.
-    Value numElements = rewriter.create<LLVM::ConstantOp>(
+    Value numElements = LLVM::ConstantOp::create(rewriter,
         loc, getIndexType(), rewriter.getIndexAttr(1));
     for (int pos = 0; pos < srcType.getRank(); ++pos) {
       auto size = srcDesc.size(rewriter, loc, pos);
-      numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
     }
 
     // Get element size.
     auto sizeInBytes = getSizeInBytes(loc, srcType.getElementType(), rewriter);
     // Compute total.
     Value totalSize =
-        rewriter.create<LLVM::MulOp>(loc, numElements, sizeInBytes);
+        LLVM::MulOp::create(rewriter, loc, numElements, sizeInBytes);
 
     Type elementType = typeConverter->convertType(srcType.getElementType());
 
     Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc);
     Value srcOffset = srcDesc.offset(rewriter, loc);
-    Value srcPtr = rewriter.create<LLVM::GEPOp>(
+    Value srcPtr = LLVM::GEPOp::create(rewriter,
         loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset);
     MemRefDescriptor targetDesc(adaptor.getTarget());
     Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc);
     Value targetOffset = targetDesc.offset(rewriter, loc);
-    Value targetPtr = rewriter.create<LLVM::GEPOp>(
+    Value targetPtr = LLVM::GEPOp::create(rewriter,
         loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset);
-    rewriter.create<LLVM::MemcpyOp>(loc, targetPtr, srcPtr, totalSize,
+    LLVM::MemcpyOp::create(rewriter, loc, targetPtr, srcPtr, totalSize,
                                     /*isVolatile=*/false);
     rewriter.eraseOp(op);
 
@@ -1103,7 +1103,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
 
     // First make sure we have an unranked memref descriptor representation.
     auto makeUnranked = [&, this](Value ranked, MemRefType type) {
-      auto rank = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+      auto rank = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                     type.getRank());
       auto *typeConverter = getTypeConverter();
       auto ptr =
@@ -1116,7 +1116,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     };
 
     // Save stack position before promoting descriptors
-    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
+    auto stackSaveOp = LLVM::StackSaveOp::create(rewriter, loc, getPtrType());
 
     auto srcMemRefType = dyn_cast<MemRefType>(srcType);
     Value unrankedSource =
@@ -1128,13 +1128,13 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
                          : adaptor.getTarget();
 
     // Now promote the unranked descriptors to the stack.
-    auto one = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
+    auto one = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
                                                  rewriter.getIndexAttr(1));
     auto promote = [&](Value desc) {
       auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
       auto allocated =
-          rewriter.create<LLVM::AllocaOp>(loc, ptrType, desc.getType(), one);
-      rewriter.create<LLVM::StoreOp>(loc, desc, allocated);
+          LLVM::AllocaOp::create(rewriter, loc, ptrType, desc.getType(), one);
+      LLVM::StoreOp::create(rewriter, loc, desc, allocated);
       return allocated;
     };
 
@@ -1149,11 +1149,11 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
         sourcePtr.getType(), symbolTables);
     if (failed(copyFn))
       return failure();
-    rewriter.create<LLVM::CallOp>(loc, copyFn.value(),
+    LLVM::CallOp::create(rewriter, loc, copyFn.value(),
                                   ValueRange{elemSize, sourcePtr, targetPtr});
 
     // Restore stack used for descriptors
-    rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);
+    LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
 
     rewriter.eraseOp(op);
 
@@ -1204,9 +1204,9 @@ struct MemorySpaceCastOpLowering
       MemRefDescriptor::unpack(rewriter, loc, adaptor.getSource(), resultTypeR,
                                descVals);
       descVals[0] =
-          rewriter.create<LLVM::AddrSpaceCastOp>(loc, newPtrType, descVals[0]);
+          LLVM::AddrSpaceCastOp::create(rewriter, loc, newPtrType, descVals[0]);
       descVals[1] =
-          rewriter.create<LLVM::AddrSpaceCastOp>(loc, newPtrType, descVals[1]);
+          LLVM::AddrSpaceCastOp::create(rewriter, loc, newPtrType, descVals[1]);
       Value result = MemRefDescriptor::pack(rewriter, loc, *getTypeConverter(),
                                             resultTypeR, descVals);
       rewriter.replaceOp(op, result);
@@ -1241,7 +1241,7 @@ struct MemorySpaceCastOpLowering
       UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                              result, resultAddrSpace, sizes);
       Value resultUnderlyingSize = sizes.front();
-      Value resultUnderlyingDesc = rewriter.create<LLVM::AllocaOp>(
+      Value resultUnderlyingDesc = LLVM::AllocaOp::create(rewriter,
           loc, getPtrType(), rewriter.getI8Type(), resultUnderlyingSize);
       result.setMemRefDescPtr(rewriter, loc, resultUnderlyingDesc);
 
@@ -1256,9 +1256,9 @@ struct MemorySpaceCastOpLowering
       Value alignedPtr =
           sourceDesc.alignedPtr(rewriter, loc, *getTypeConverter(),
                                 sourceUnderlyingDesc, sourceElemPtrType);
-      allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+      allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
           loc, resultElemPtrType, allocatedPtr);
-      alignedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+      alignedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
           loc, resultElemPtrType, alignedPtr);
 
       result.setAllocatedPtr(rewriter, loc, resultUnderlyingDesc,
@@ -1277,11 +1277,11 @@ struct MemorySpaceCastOpLowering
       int64_t bytesToSkip =
           2 * llvm::divideCeil(
                   getTypeConverter()->getPointerBitwidth(resultAddrSpace), 8);
-      Value bytesToSkipConst = rewriter.create<LLVM::ConstantOp>(
+      Value bytesToSkipConst = LLVM::ConstantOp::create(rewriter,
           loc, getIndexType(), rewriter.getIndexAttr(bytesToSkip));
-      Value copySize = rewriter.create<LLVM::SubOp>(
+      Value copySize = LLVM::SubOp::create(rewriter,
           loc, getIndexType(), resultUnderlyingSize, bytesToSkipConst);
-      rewriter.create<LLVM::MemcpyOp>(loc, resultIndexVals, sourceIndexVals,
+      LLVM::MemcpyOp::create(rewriter, loc, resultIndexVals, sourceIndexVals,
                                       copySize, /*isVolatile=*/false);
 
       rewriter.replaceOp(op, ValueRange{result});
@@ -1485,7 +1485,7 @@ struct MemRefReshapeOpLowering
         } else {
           Value shapeOp = reshapeOp.getShape();
           Value index = createIndexAttrConstant(rewriter, loc, indexType, i);
-          dimSize = rewriter.create<memref::LoadOp>(loc, shapeOp, index);
+          dimSize = memref::LoadOp::create(rewriter, loc, shapeOp, index);
           Type indexType = getIndexType();
           if (dimSize.getType() != indexType)
             dimSize = typeConverter->materializeTargetConversion(
@@ -1497,7 +1497,7 @@ struct MemRefReshapeOpLowering
         desc.setStride(rewriter, loc, i, stride);
 
         // Prepare the stride value for the next dimension.
-        stride = rewriter.create<LLVM::MulOp>(loc, stride, dimSize);
+        stride = LLVM::MulOp::create(rewriter, loc, stride, dimSize);
       }
 
       *descriptor = desc;
@@ -1522,7 +1522,7 @@ struct MemRefReshapeOpLowering
     SmallVector<Value, 4> sizes;
     UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                            targetDesc, addressSpace, sizes);
-    Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>(
+    Value underlyingDescPtr = LLVM::AllocaOp::create(rewriter,
         loc, getPtrType(), IntegerType::get(getContext(), 8), sizes.front());
     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
 
@@ -1554,7 +1554,7 @@ struct MemRefReshapeOpLowering
     Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc);
     Value oneIndex = createIndexAttrConstant(rewriter, loc, getIndexType(), 1);
     Value resultRankMinusOne =
-        rewriter.create<LLVM::SubOp>(loc, resultRank, oneIndex);
+        LLVM::SubOp::create(rewriter, loc, resultRank, oneIndex);
 
     Block *initBlock = rewriter.getInsertionBlock();
     Type indexType = getTypeConverter()->getIndexType();
@@ -1568,14 +1568,14 @@ struct MemRefReshapeOpLowering
     rewriter.mergeBlocks(remainingBlock, condBlock, ValueRange());
 
     rewriter.setInsertionPointToEnd(initBlock);
-    rewriter.create<LLVM::BrOp>(loc, ValueRange({resultRankMinusOne, oneIndex}),
+    LLVM::BrOp::create(rewriter, loc, ValueRange({resultRankMinusOne, oneIndex}),
                                 condBlock);
     rewriter.setInsertionPointToStart(condBlock);
     Value indexArg = condBlock->getArgument(0);
     Value strideArg = condBlock->getArgument(1);
 
     Value zeroIndex = createIndexAttrConstant(rewriter, loc, indexType, 0);
-    Value pred = rewriter.create<LLVM::ICmpOp>(
+    Value pred = LLVM::ICmpOp::create(rewriter,
         loc, IntegerType::get(rewriter.getContext(), 1),
         LLVM::ICmpPredicate::sge, indexArg, zeroIndex);
 
@@ -1585,22 +1585,22 @@ struct MemRefReshapeOpLowering
 
     // Copy size from shape to descriptor.
     auto llvmIndexPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value sizeLoadGep = rewriter.create<LLVM::GEPOp>(
+    Value sizeLoadGep = LLVM::GEPOp::create(rewriter,
         loc, llvmIndexPtrType,
         typeConverter->convertType(shapeMemRefType.getElementType()),
         shapeOperandPtr, indexArg);
-    Value size = rewriter.create<LLVM::LoadOp>(loc, indexType, sizeLoadGep);
+    Value size = LLVM::LoadOp::create(rewriter, loc, indexType, sizeLoadGep);
     UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(),
                                       targetSizesBase, indexArg, size);
 
     // Write stride value and compute next one.
     UnrankedMemRefDescriptor::setStride(rewriter, loc, *getTypeConverter(),
                                         targetStridesBase, indexArg, strideArg);
-    Value nextStride = rewriter.create<LLVM::MulOp>(loc, strideArg, size);
+    Value nextStride = LLVM::MulOp::create(rewriter, loc, strideArg, size);
 
     // Decrement loop counter and branch back.
-    Value decrement = rewriter.create<LLVM::SubOp>(loc, indexArg, oneIndex);
-    rewriter.create<LLVM::BrOp>(loc, ValueRange({decrement, nextStride}),
+    Value decrement = LLVM::SubOp::create(rewriter, loc, indexArg, oneIndex);
+    LLVM::BrOp::create(rewriter, loc, ValueRange({decrement, nextStride}),
                                 condBlock);
 
     Block *remainder =
@@ -1608,7 +1608,7 @@ struct MemRefReshapeOpLowering
 
     // Hook up the cond exit to the remainder.
     rewriter.setInsertionPointToEnd(condBlock);
-    rewriter.create<LLVM::CondBrOp>(loc, pred, bodyBlock, ValueRange(),
+    LLVM::CondBrOp::create(rewriter, loc, pred, bodyBlock, ValueRange(),
                                     remainder, ValueRange());
 
     // Reset position to beginning of new remainder block.
@@ -1738,7 +1738,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
       return createIndexAttrConstant(rewriter, loc, indexType, strides[idx]);
     if (nextSize)
       return runningStride
-                 ? rewriter.create<LLVM::MulOp>(loc, runningStride, nextSize)
+                 ? LLVM::MulOp::create(rewriter, loc, runningStride, nextSize)
                  : nextSize;
     assert(!runningStride);
     return createIndexAttrConstant(rewriter, loc, indexType, 1);
@@ -1783,7 +1783,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
 
     // Field 2: Copy the actual aligned pointer to payload.
     Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc);
-    alignedPtr = rewriter.create<LLVM::GEPOp>(
+    alignedPtr = LLVM::GEPOp::create(rewriter,
         loc, alignedPtr.getType(),
         typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr,
         adaptor.getByteShift());
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index b866afbce98b0..7b7ad9b95b0ec 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -79,7 +79,7 @@ adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter,
   assert(indices.size() == 2);
   indices.back() = builder.createOrFold<spirv::SDivOp>(loc, lastDim, idx);
   Type t = typeConverter.convertType(op.getComponentPtr().getType());
-  return builder.create<spirv::AccessChainOp>(loc, t, op.getBasePtr(), indices);
+  return spirv::AccessChainOp::create(builder, loc, t, op.getBasePtr(), indices);
 }
 
 /// Casts the given `srcBool` into an integer of `dstType`.
@@ -107,7 +107,7 @@ static Value shiftValue(Location loc, Value value, Value offset, Value mask,
     value = castBoolToIntN(loc, value, dstType, builder);
   } else {
     if (valueBits < targetBits) {
-      value = builder.create<spirv::UConvertOp>(
+      value = spirv::UConvertOp::create(builder,
           loc, builder.getIntegerType(targetBits), value);
     }
 
@@ -372,7 +372,7 @@ AllocOpPattern::matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor,
     std::string varName =
         std::string("__workgroup_mem__") +
         std::to_string(std::distance(varOps.begin(), varOps.end()));
-    varOp = rewriter.create<spirv::GlobalVariableOp>(loc, spirvType, varName,
+    varOp = spirv::GlobalVariableOp::create(rewriter, loc, spirvType, varName,
                                                      /*initializer=*/nullptr);
   }
 
@@ -572,7 +572,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
           loadOp, "failed to determine memory requirements");
 
     auto [memoryAccess, alignment] = *memoryRequirements;
-    Value loadVal = rewriter.create<spirv::LoadOp>(loc, accessChain,
+    Value loadVal = spirv::LoadOp::create(rewriter, loc, accessChain,
                                                    memoryAccess, alignment);
     if (isBool)
       loadVal = castIntNToBool(loc, loadVal, rewriter);
@@ -601,7 +601,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
         loadOp, "failed to determine memory requirements");
 
   auto [memoryAccess, alignment] = *memoryRequirements;
-  Value spvLoadOp = rewriter.create<spirv::LoadOp>(loc, dstType, adjustedPtr,
+  Value spvLoadOp = spirv::LoadOp::create(rewriter, loc, dstType, adjustedPtr,
                                                    memoryAccess, alignment);
 
   // Shift the bits to the rightmost.
@@ -770,10 +770,10 @@ IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor,
   if (!scope)
     return rewriter.notifyMatchFailure(storeOp, "atomic scope not available");
 
-  Value result = rewriter.create<spirv::AtomicAndOp>(
+  Value result = spirv::AtomicAndOp::create(rewriter,
       loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
       clearBitsMask);
-  result = rewriter.create<spirv::AtomicOrOp>(
+  result = spirv::AtomicOrOp::create(rewriter,
       loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
       storeVal);
 
@@ -851,11 +851,11 @@ LogicalResult MemorySpaceCastOpPattern::matchAndRewrite(
   }
   if (sourceSc != spirv::StorageClass::Generic) {
     result =
-        rewriter.create<spirv::PtrCastToGenericOp>(loc, genericPtrType, result);
+        spirv::PtrCastToGenericOp::create(rewriter, loc, genericPtrType, result);
   }
   if (resultSc != spirv::StorageClass::Generic) {
     result =
-        rewriter.create<spirv::GenericCastToPtrOp>(loc, resultPtrType, result);
+        spirv::GenericCastToPtrOp::create(rewriter, loc, resultPtrType, result);
   }
   rewriter.replaceOp(addrCastOp, result);
   return success();
diff --git a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
index b93128441f2b5..f16b92a287d0a 100644
--- a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
+++ b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
@@ -65,7 +65,7 @@ static SmallVector<Value> getMixedAsValues(OpBuilder b, const Location &loc,
       values.emplace_back(*(dyn++));
     } else {
       TypedAttr val = type == i64 ? b.getI64IntegerAttr(s) : b.getIndexAttr(s);
-      values.emplace_back(b.create<arith::ConstantOp>(loc, type, val));
+      values.emplace_back(arith::ConstantOp::create(b, loc, type, val));
     }
   }
   return values;
@@ -79,9 +79,9 @@ static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
   SmallVector<Value> multiIndex(n);
 
   for (int i = n - 1; i >= 0; --i) {
-    multiIndex[i] = b.create<arith::RemSIOp>(loc, linearIndex, dimensions[i]);
+    multiIndex[i] = arith::RemSIOp::create(b, loc, linearIndex, dimensions[i]);
     if (i > 0)
-      linearIndex = b.create<arith::DivSIOp>(loc, linearIndex, dimensions[i]);
+      linearIndex = arith::DivSIOp::create(b, loc, linearIndex, dimensions[i]);
   }
 
   return multiIndex;
@@ -91,13 +91,13 @@ static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
 Value multiToLinearIndex(Location loc, OpBuilder b, ValueRange multiIndex,
                          ValueRange dimensions) {
 
-  Value linearIndex = b.create<arith::ConstantIndexOp>(loc, 0);
-  Value stride = b.create<arith::ConstantIndexOp>(loc, 1);
+  Value linearIndex = arith::ConstantIndexOp::create(b, loc, 0);
+  Value stride = arith::ConstantIndexOp::create(b, loc, 1);
 
   for (int i = multiIndex.size() - 1; i >= 0; --i) {
-    Value off = b.create<arith::MulIOp>(loc, multiIndex[i], stride);
-    linearIndex = b.create<arith::AddIOp>(loc, linearIndex, off);
-    stride = b.create<arith::MulIOp>(loc, stride, dimensions[i]);
+    Value off = arith::MulIOp::create(b, loc, multiIndex[i], stride);
+    linearIndex = arith::AddIOp::create(b, loc, linearIndex, off);
+    stride = arith::MulIOp::create(b, loc, stride, dimensions[i]);
   }
 
   return linearIndex;
@@ -144,10 +144,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     auto i64 = rewriter.getI64Type();
     std::array<int64_t, 2> shape = {static_cast<int64_t>(splitAxes.size()),
                                     maxNAxes};
-    Value resSplitAxes = rewriter.create<tensor::EmptyOp>(loc, shape, i16);
+    Value resSplitAxes = tensor::EmptyOp::create(rewriter, loc, shape, i16);
     auto attr = IntegerAttr::get(i16, -1);
-    Value fillValue = rewriter.create<arith::ConstantOp>(loc, i16, attr);
-    resSplitAxes = rewriter.create<linalg::FillOp>(loc, fillValue, resSplitAxes)
+    Value fillValue = arith::ConstantOp::create(rewriter, loc, i16, attr);
+    resSplitAxes = linalg::FillOp::create(rewriter, loc, fillValue, resSplitAxes)
                        .getResult(0);
 
     // explicitly write values into tensor row by row
@@ -162,8 +162,8 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       std::array<int64_t, 2> sizes = {1, size};
       auto tensorType = RankedTensorType::get({size}, i16);
       auto attrs = DenseIntElementsAttr::get(tensorType, axes.asArrayRef());
-      auto vals = rewriter.create<arith::ConstantOp>(loc, tensorType, attrs);
-      resSplitAxes = rewriter.create<tensor::InsertSliceOp>(
+      auto vals = arith::ConstantOp::create(rewriter, loc, tensorType, attrs);
+      resSplitAxes = tensor::InsertSliceOp::create(rewriter,
           loc, vals, resSplitAxes, empty, empty, empty, offs, sizes, strides);
     }
 
@@ -179,7 +179,7 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
                   .create<tensor::EmptyOp>(loc, std::array<int64_t, 2>{0, 0},
                                            i64)
                   .getResult()
-            : rewriter.create<tensor::FromElementsOp>(loc, type, haloSizes)
+            : tensor::FromElementsOp::create(rewriter, loc, type, haloSizes)
                   .getResult();
 
     // To hold sharded dims offsets, create Tensor with shape {nSplits,
@@ -189,7 +189,7 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     // MeshOp)
     Value resOffsets;
     if (adaptor.getStaticShardedDimsOffsets().empty()) {
-      resOffsets = rewriter.create<tensor::EmptyOp>(
+      resOffsets = tensor::EmptyOp::create(rewriter,
           loc, std::array<int64_t, 2>{0, 0}, i64);
     } else {
       SymbolTableCollection symbolTableCollection;
@@ -204,12 +204,12 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       assert(maxSplitSize);
       ++maxSplitSize; // add one for the total size
 
-      resOffsets = rewriter.create<tensor::EmptyOp>(
+      resOffsets = tensor::EmptyOp::create(rewriter,
           loc, std::array<int64_t, 2>{nSplits, maxSplitSize}, i64);
-      Value zero = rewriter.create<arith::ConstantOp>(
+      Value zero = arith::ConstantOp::create(rewriter,
           loc, i64, rewriter.getI64IntegerAttr(ShapedType::kDynamic));
       resOffsets =
-          rewriter.create<linalg::FillOp>(loc, zero, resOffsets).getResult(0);
+          linalg::FillOp::create(rewriter, loc, zero, resOffsets).getResult(0);
       SmallVector<Value> offsets =
           getMixedAsValues(rewriter, loc, adaptor.getStaticShardedDimsOffsets(),
                            adaptor.getDynamicShardedDimsOffsets());
@@ -220,10 +220,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
         assert(splitSize != ShapedType::kDynamic && splitSize < maxSplitSize);
         ++splitSize; // add one for the total size
         ArrayRef<Value> values(&offsets[curr], splitSize);
-        Value vals = rewriter.create<tensor::FromElementsOp>(loc, values);
+        Value vals = tensor::FromElementsOp::create(rewriter, loc, values);
         std::array<int64_t, 2> offs = {static_cast<int64_t>(i), 0};
         std::array<int64_t, 2> sizes = {1, splitSize};
-        resOffsets = rewriter.create<tensor::InsertSliceOp>(
+        resOffsets = tensor::InsertSliceOp::create(rewriter,
             loc, vals, resOffsets, empty, empty, empty, offs, sizes, strides);
         curr += splitSize;
       }
@@ -236,10 +236,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       return failure();
 
     resSplitAxes =
-        rewriter.create<tensor::CastOp>(loc, resTypes[0], resSplitAxes);
+        tensor::CastOp::create(rewriter, loc, resTypes[0], resSplitAxes);
     resHaloSizes =
-        rewriter.create<tensor::CastOp>(loc, resTypes[1], resHaloSizes);
-    resOffsets = rewriter.create<tensor::CastOp>(loc, resTypes[2], resOffsets);
+        tensor::CastOp::create(rewriter, loc, resTypes[1], resHaloSizes);
+    resOffsets = tensor::CastOp::create(rewriter, loc, resTypes[2], resOffsets);
 
     rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
         op, TupleType::get(op.getContext(), resTypes),
@@ -269,9 +269,9 @@ struct ConvertProcessMultiIndexOp
     SmallVector<Value> dims;
     llvm::transform(
         meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
-          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+          return arith::ConstantIndexOp::create(rewriter, loc, i).getResult();
         });
-    Value rank = rewriter.create<ProcessLinearIndexOp>(op.getLoc(), meshOp);
+    Value rank = ProcessLinearIndexOp::create(rewriter, op.getLoc(), meshOp);
     auto mIdx = linearToMultiIndex(loc, rewriter, rank, dims);
 
     // optionally extract subset of mesh axes
@@ -302,7 +302,7 @@ class ConvertProcessLinearIndexOp
     Location loc = op.getLoc();
     auto ctx = op.getContext();
     Value commWorld =
-        rewriter.create<mpi::CommWorldOp>(loc, mpi::CommType::get(ctx));
+        mpi::CommWorldOp::create(rewriter, loc, mpi::CommType::get(ctx));
     auto rank =
         rewriter
             .create<mpi::CommRankOp>(
@@ -341,40 +341,40 @@ struct ConvertNeighborsLinearIndicesOp
     SmallVector<Value> dims;
     llvm::transform(
         meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
-          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+          return arith::ConstantIndexOp::create(rewriter, loc, i).getResult();
         });
     Value dimSz = dims[axes[0]];
-    Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    Value minus1 = rewriter.create<arith::ConstantIndexOp>(loc, -1);
-    Value atBorder = rewriter.create<arith::CmpIOp>(
+    Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    Value minus1 = arith::ConstantIndexOp::create(rewriter, loc, -1);
+    Value atBorder = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sle, orgIdx,
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
-    auto down = rewriter.create<scf::IfOp>(
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
+    auto down = scf::IfOp::create(rewriter,
         loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
-          builder.create<scf::YieldOp>(loc, minus1);
+          scf::YieldOp::create(builder, loc, minus1);
         },
         [&](OpBuilder &builder, Location loc) {
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
-              rewriter.create<arith::SubIOp>(op.getLoc(), orgIdx, one)
+              arith::SubIOp::create(rewriter, op.getLoc(), orgIdx, one)
                   .getResult();
-          builder.create<scf::YieldOp>(
+          scf::YieldOp::create(builder,
               loc, multiToLinearIndex(loc, rewriter, tmp, dims));
         });
-    atBorder = rewriter.create<arith::CmpIOp>(
+    atBorder = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, orgIdx,
-        rewriter.create<arith::SubIOp>(loc, dimSz, one).getResult());
-    auto up = rewriter.create<scf::IfOp>(
+        arith::SubIOp::create(rewriter, loc, dimSz, one).getResult());
+    auto up = scf::IfOp::create(rewriter,
         loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
-          builder.create<scf::YieldOp>(loc, minus1);
+          scf::YieldOp::create(builder, loc, minus1);
         },
         [&](OpBuilder &builder, Location loc) {
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
-              rewriter.create<arith::AddIOp>(op.getLoc(), orgIdx, one);
-          builder.create<scf::YieldOp>(
+              arith::AddIOp::create(rewriter, op.getLoc(), orgIdx, one);
+          scf::YieldOp::create(builder,
               loc, multiToLinearIndex(loc, rewriter, tmp, dims));
         });
     rewriter.replaceOp(op, ValueRange{down.getResult(0), up.getResult(0)});
@@ -447,7 +447,7 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           rewriter, loc, sharding.getStaticShardedDimsOffsets(),
           sharding.getDynamicShardedDimsOffsets(), index);
       if (!tmp.empty())
-        shardedDimsOffs = rewriter.create<tensor::FromElementsOp>(
+        shardedDimsOffs = tensor::FromElementsOp::create(rewriter,
             loc, RankedTensorType::get({(int64_t)tmp.size()}, index), tmp);
     }
 
@@ -457,9 +457,9 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
     int64_t pos = 0;
     SmallVector<Value> shardShape;
     Value zero =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(index));
     Value one =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getOneAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getOneAttr(index));
 
     // Iterate over the dimensions of the tensor shape, get their split Axes,
     // and compute the sharded shape.
@@ -470,7 +470,7 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
         // The current dimension might not be sharded.
         // Create a value from the static position in shardDimsOffsets.
         Value posVal =
-            rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(pos));
+            arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(pos));
         // Get the index of the local shard in the mesh axis.
         Value idx = multiIdx[axes[0]];
         auto numShards =
@@ -482,29 +482,29 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
             return op->emitError() << "Only single axis sharding is "
                                    << "supported for each dimension.";
           }
-          idx = rewriter.create<arith::AddIOp>(loc, posVal, idx);
+          idx = arith::AddIOp::create(rewriter, loc, posVal, idx);
           // Compute size = shardedDimsOffs[idx+1] - shardedDimsOffs[idx].
           Value off =
-              rewriter.create<tensor::ExtractOp>(loc, shardedDimsOffs, idx);
-          idx = rewriter.create<arith::AddIOp>(loc, idx, one);
+              tensor::ExtractOp::create(rewriter, loc, shardedDimsOffs, idx);
+          idx = arith::AddIOp::create(rewriter, loc, idx, one);
           Value nextOff =
-              rewriter.create<tensor::ExtractOp>(loc, shardedDimsOffs, idx);
-          Value sz = rewriter.create<arith::SubIOp>(loc, nextOff, off);
+              tensor::ExtractOp::create(rewriter, loc, shardedDimsOffs, idx);
+          Value sz = arith::SubIOp::create(rewriter, loc, nextOff, off);
           shardShape.emplace_back(sz);
         } else {
-          Value numShardsVal = rewriter.create<arith::ConstantOp>(
+          Value numShardsVal = arith::ConstantOp::create(rewriter,
               loc, rewriter.getIndexAttr(numShards));
           // Compute shard dim size by distributing odd elements to trailing
           // shards:
           // sz = dim / numShards
           //      + (idx >= (numShards - (dim % numShards)) ? 1 : 0)
-          Value sz = rewriter.create<arith::DivSIOp>(loc, dim, numShardsVal);
-          Value sz1 = rewriter.create<arith::RemSIOp>(loc, dim, numShardsVal);
-          sz1 = rewriter.create<arith::SubIOp>(loc, numShardsVal, sz1);
-          auto cond = rewriter.create<arith::CmpIOp>(
+          Value sz = arith::DivSIOp::create(rewriter, loc, dim, numShardsVal);
+          Value sz1 = arith::RemSIOp::create(rewriter, loc, dim, numShardsVal);
+          sz1 = arith::SubIOp::create(rewriter, loc, numShardsVal, sz1);
+          auto cond = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::sge, idx, sz1);
-          Value odd = rewriter.create<arith::SelectOp>(loc, cond, one, zero);
-          sz = rewriter.create<arith::AddIOp>(loc, sz, odd);
+          Value odd = arith::SelectOp::create(rewriter, loc, cond, one, zero);
+          sz = arith::AddIOp::create(rewriter, loc, sz, odd);
           shardShape.emplace_back(sz);
         }
         pos += numShards + 1; // add one for the total size.
@@ -568,7 +568,7 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (isa<RankedTensorType>(input.getType())) {
       auto memrefType = MemRefType::get(
           inputShape, cast<ShapedType>(input.getType()).getElementType());
-      input = iBuilder.create<bufferization::ToBufferOp>(memrefType, input);
+      input = bufferization::ToBufferOp::create(iBuilder, memrefType, input);
     }
     MemRefType inType = cast<MemRefType>(input.getType());
 
@@ -577,15 +577,15 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     for (auto i = 0; i < inType.getRank(); ++i) {
       auto s = inputShape[i];
       if (ShapedType::isDynamic(s))
-        shape[i] = iBuilder.create<memref::DimOp>(input, s).getResult();
+        shape[i] = memref::DimOp::create(iBuilder, input, s).getResult();
       else
         shape[i] = iBuilder.getIndexAttr(s);
     }
 
     // Allocate buffer and copy input to buffer.
-    Value buffer = iBuilder.create<memref::AllocOp>(
+    Value buffer = memref::AllocOp::create(iBuilder,
         shape, cast<ShapedType>(op.getType()).getElementType());
-    iBuilder.create<linalg::CopyOp>(input, buffer);
+    linalg::CopyOp::create(iBuilder, input, buffer);
 
     // Get an MPI_Comm_split for the AllReduce operation.
     // The color is the linear index of the process in the mesh along the
@@ -594,9 +594,9 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
                                        iBuilder.getIndexType());
     SmallVector<Value> myMultiIndex =
-        iBuilder.create<ProcessMultiIndexOp>(indexResultTypes, mesh)
+        ProcessMultiIndexOp::create(iBuilder, indexResultTypes, mesh)
             .getResult();
-    Value zero = iBuilder.create<arith::ConstantIndexOp>(0);
+    Value zero = arith::ConstantIndexOp::create(iBuilder, 0);
     SmallVector<Value> multiKey(myMultiIndex.size(), zero);
 
     auto redAxes = adaptor.getMeshAxes();
@@ -607,15 +607,15 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
 
     Value color =
         createProcessLinearIndex(mesh, myMultiIndex, redAxes, iBuilder);
-    color = iBuilder.create<arith::IndexCastOp>(iBuilder.getI32Type(), color);
+    color = arith::IndexCastOp::create(iBuilder, iBuilder.getI32Type(), color);
     Value key = createProcessLinearIndex(mesh, multiKey, redAxes, iBuilder);
-    key = iBuilder.create<arith::IndexCastOp>(iBuilder.getI32Type(), key);
+    key = arith::IndexCastOp::create(iBuilder, iBuilder.getI32Type(), key);
 
     // Finally split the communicator
     auto commType = mpi::CommType::get(op->getContext());
-    Value commWorld = iBuilder.create<mpi::CommWorldOp>(commType);
+    Value commWorld = mpi::CommWorldOp::create(iBuilder, commType);
     auto comm =
-        iBuilder.create<mpi::CommSplitOp>(commType, commWorld, color, key)
+        mpi::CommSplitOp::create(iBuilder, commType, commWorld, color, key)
             .getNewcomm();
 
     Value buffer1d = buffer;
@@ -623,18 +623,18 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (inType.getRank() > 1) {
       ReassociationIndices reassociation(inType.getRank());
       std::iota(reassociation.begin(), reassociation.end(), 0);
-      buffer1d = iBuilder.create<memref::CollapseShapeOp>(
+      buffer1d = memref::CollapseShapeOp::create(iBuilder,
           buffer, ArrayRef<ReassociationIndices>(reassociation));
     }
 
     // Create the MPI AllReduce operation.
-    iBuilder.create<mpi::AllReduceOp>(
+    mpi::AllReduceOp::create(iBuilder,
         TypeRange(), buffer1d, buffer1d,
         getMPIReductionOp(adaptor.getReductionAttr()), comm);
 
     // If the destination is a memref, cast it to a tensor
     if (isa<RankedTensorType>(op.getType()))
-      buffer = iBuilder.create<bufferization::ToTensorOp>(op.getType(), buffer,
+      buffer = bufferization::ToTensorOp::create(iBuilder, op.getType(), buffer,
                                                           true);
 
     rewriter.replaceOp(op, buffer);
@@ -676,7 +676,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     auto toValue = [&rewriter, &loc](OpFoldResult &v) -> Value {
       if (auto value = dyn_cast<Value>(v))
         return value;
-      return rewriter.create<arith::ConstantOp>(
+      return arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(
                    cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
     };
@@ -689,7 +689,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       auto mmemrefType = MemRefType::get(
           dstShape, cast<ShapedType>(array.getType()).getElementType());
       array =
-          rewriter.create<bufferization::ToBufferOp>(loc, mmemrefType, array);
+          bufferization::ToBufferOp::create(rewriter, loc, mmemrefType, array);
     }
     auto rank = cast<ShapedType>(array.getType()).getRank();
     auto opSplitAxes = adaptor.getSplitAxes().getAxes();
@@ -713,7 +713,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     for (auto i = 0; i < rank; ++i) {
       auto s = dstShape[i];
       if (ShapedType::isDynamic(s))
-        shape[i] = rewriter.create<memref::DimOp>(loc, array, s).getResult();
+        shape[i] = memref::DimOp::create(rewriter, loc, array, s).getResult();
       else
         shape[i] = rewriter.getIndexAttr(s);
 
@@ -723,12 +723,12 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         offsets[i] = haloSizes[currHaloDim * 2];
 
         // prepare shape and offsets of highest dim's halo exchange
-        Value _haloSz = rewriter.create<arith::AddIOp>(
+        Value _haloSz = arith::AddIOp::create(rewriter,
             loc, toValue(haloSizes[currHaloDim * 2]),
             toValue(haloSizes[currHaloDim * 2 + 1]));
         // the halo shape of lower dims exlude the halos
         dimSizes[i] =
-            rewriter.create<arith::SubIOp>(loc, toValue(shape[i]), _haloSz)
+            arith::SubIOp::create(rewriter, loc, toValue(shape[i]), _haloSz)
                 .getResult();
       } else {
         dimSizes[i] = shape[i];
@@ -736,14 +736,14 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     }
 
     auto tagAttr = rewriter.getI32IntegerAttr(91); // we just pick something
-    auto tag = rewriter.create<arith::ConstantOp>(loc, tagAttr);
+    auto tag = arith::ConstantOp::create(rewriter, loc, tagAttr);
     auto zeroAttr = rewriter.getI32IntegerAttr(0); // for detecting v<0
-    auto zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
+    auto zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
 
     SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
                                        rewriter.getIndexType());
     auto myMultiIndex =
-        rewriter.create<ProcessMultiIndexOp>(loc, indexResultTypes, mesh)
+        ProcessMultiIndexOp::create(rewriter, loc, indexResultTypes, mesh)
             .getResult();
     // traverse all split axes from high to low dim
     for (ssize_t dim = opSplitAxes.size() - 1; dim >= 0; --dim) {
@@ -758,19 +758,19 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
                                                        splitAxes)
                      .getResults();
       // MPI operates on i32...
-      Value neighbourIDs[2] = {rewriter.create<arith::IndexCastOp>(
+      Value neighbourIDs[2] = {arith::IndexCastOp::create(rewriter,
                                    loc, rewriter.getI32Type(), tmp[0]),
-                               rewriter.create<arith::IndexCastOp>(
+                               arith::IndexCastOp::create(rewriter,
                                    loc, rewriter.getI32Type(), tmp[1])};
 
       auto lowerRecvOffset = rewriter.getIndexAttr(0);
       auto lowerSendOffset = toValue(haloSizes[currHaloDim * 2]);
-      auto upperRecvOffset = rewriter.create<arith::SubIOp>(
+      auto upperRecvOffset = arith::SubIOp::create(rewriter,
           loc, toValue(shape[dim]), toValue(haloSizes[currHaloDim * 2 + 1]));
-      auto upperSendOffset = rewriter.create<arith::SubIOp>(
+      auto upperSendOffset = arith::SubIOp::create(rewriter,
           loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
 
-      Value commWorld = rewriter.create<mpi::CommWorldOp>(
+      Value commWorld = mpi::CommWorldOp::create(rewriter,
           loc, mpi::CommType::get(op->getContext()));
 
       // Make sure we send/recv in a way that does not lead to a dead-lock.
@@ -787,37 +787,37 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         // Processes on the mesh borders have only one neighbor
         auto to = upperHalo ? neighbourIDs[0] : neighbourIDs[1];
         auto from = upperHalo ? neighbourIDs[1] : neighbourIDs[0];
-        auto hasFrom = rewriter.create<arith::CmpIOp>(
+        auto hasFrom = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sge, from, zero);
-        auto hasTo = rewriter.create<arith::CmpIOp>(
+        auto hasTo = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sge, to, zero);
-        auto buffer = rewriter.create<memref::AllocOp>(
+        auto buffer = memref::AllocOp::create(rewriter,
             loc, dimSizes, cast<ShapedType>(array.getType()).getElementType());
         // if has neighbor: copy halo data from array to buffer and send
-        rewriter.create<scf::IfOp>(
+        scf::IfOp::create(rewriter,
             loc, hasTo, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(lowerSendOffset)
                                        : OpFoldResult(upperSendOffset);
-              auto subview = builder.create<memref::SubViewOp>(
+              auto subview = memref::SubViewOp::create(builder,
                   loc, array, offsets, dimSizes, strides);
-              builder.create<memref::CopyOp>(loc, subview, buffer);
-              builder.create<mpi::SendOp>(loc, TypeRange{}, buffer, tag, to,
+              memref::CopyOp::create(builder, loc, subview, buffer);
+              mpi::SendOp::create(builder, loc, TypeRange{}, buffer, tag, to,
                                           commWorld);
-              builder.create<scf::YieldOp>(loc);
+              scf::YieldOp::create(builder, loc);
             });
         // if has neighbor: receive halo data into buffer and copy to array
-        rewriter.create<scf::IfOp>(
+        scf::IfOp::create(rewriter,
             loc, hasFrom, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(upperRecvOffset)
                                        : OpFoldResult(lowerRecvOffset);
-              builder.create<mpi::RecvOp>(loc, TypeRange{}, buffer, tag, from,
+              mpi::RecvOp::create(builder, loc, TypeRange{}, buffer, tag, from,
                                           commWorld);
-              auto subview = builder.create<memref::SubViewOp>(
+              auto subview = memref::SubViewOp::create(builder,
                   loc, array, offsets, dimSizes, strides);
-              builder.create<memref::CopyOp>(loc, buffer, subview);
-              builder.create<scf::YieldOp>(loc);
+              memref::CopyOp::create(builder, loc, buffer, subview);
+              scf::YieldOp::create(builder, loc);
             });
-        rewriter.create<memref::DeallocOp>(loc, buffer);
+        memref::DeallocOp::create(rewriter, loc, buffer);
         offsets[dim] = orgOffset;
       };
 
@@ -825,15 +825,15 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         OpFoldResult &v = haloSizes[currHaloDim * 2 + upOrDown];
         Value haloSz = dyn_cast<Value>(v);
         if (!haloSz)
-          haloSz = rewriter.create<arith::ConstantOp>(
+          haloSz = arith::ConstantOp::create(rewriter,
               loc, rewriter.getI32IntegerAttr(
                        cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
-        auto hasSize = rewriter.create<arith::CmpIOp>(
+        auto hasSize = arith::CmpIOp::create(rewriter,
             loc, arith::CmpIPredicate::sgt, haloSz, zero);
-        rewriter.create<scf::IfOp>(loc, hasSize,
+        scf::IfOp::create(rewriter, loc, hasSize,
                                    [&](OpBuilder &builder, Location loc) {
                                      genSendRecv(upOrDown > 0);
-                                     builder.create<scf::YieldOp>(loc);
+                                     scf::YieldOp::create(builder, loc);
                                    });
       };
 
@@ -852,7 +852,7 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       rewriter.replaceOp(op, array);
     } else {
       assert(isa<RankedTensorType>(op.getResult().getType()));
-      rewriter.replaceOp(op, rewriter.create<bufferization::ToTensorOp>(
+      rewriter.replaceOp(op, bufferization::ToTensorOp::create(rewriter,
                                  loc, op.getResult().getType(), array,
                                  /*restrict=*/true, /*writable=*/true));
     }
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 80b3d85488495..3f82342001ea5 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -53,7 +53,7 @@ static Value truncToI32(ImplicitLocOpBuilder &b, Value value) {
   assert(llvm::isa<IntegerType>(type) && "expected an integer Value");
   if (type.getIntOrFloatBitWidth() <= 32)
     return value;
-  return b.create<LLVM::TruncOp>(b.getI32Type(), value);
+  return LLVM::TruncOp::create(b, b.getI32Type(), value);
 }
 
 /// Returns the type for the intrinsic given the vectorResultType of the
@@ -113,7 +113,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
   Type f32x1Ty = VectorType::get(1, f32Ty);
 
   auto makeConst = [&](int32_t index) -> Value {
-    return rewriter.create<LLVM::ConstantOp>(loc, IntegerType::get(ctx, 32),
+    return LLVM::ConstantOp::create(rewriter, loc, IntegerType::get(ctx, 32),
                                              rewriter.getI32IntegerAttr(index));
   };
 
@@ -126,7 +126,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
         arrayType.getElementType() == f32x1Ty) {
       for (unsigned i = 0; i < structType.getBody().size(); i++) {
         Value el =
-            rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i);
+            LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i);
         el = rewriter.createOrFold<LLVM::BitcastOp>(
             loc, arrayType.getElementType(), el);
         elements.push_back(el);
@@ -143,23 +143,23 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
       for (unsigned i = 0, e = structType.getBody().size() / 2; i < e; i++) {
         Value vec =
-            rewriter.create<LLVM::PoisonOp>(loc, arrayType.getElementType());
+            LLVM::PoisonOp::create(rewriter, loc, arrayType.getElementType());
         Value x1 =
-            rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult, i * 2);
-        Value x2 = rewriter.create<LLVM::ExtractValueOp>(loc, intrinsicResult,
+            LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i * 2);
+        Value x2 = LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult,
                                                          i * 2 + 1);
-        vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec,
+        vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
                                                      x1, makeConst(0));
-        vec = rewriter.create<LLVM::InsertElementOp>(loc, vec.getType(), vec,
+        vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
                                                      x2, makeConst(1));
         elements.push_back(vec);
       }
     }
 
     // Create the final vectorized result.
-    Value result = rewriter.create<LLVM::PoisonOp>(loc, arrayType);
+    Value result = LLVM::PoisonOp::create(rewriter, loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
-      result = rewriter.create<LLVM::InsertValueOp>(loc, result, el.value(),
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, el.value(),
                                                     el.index());
     }
     return result;
@@ -187,7 +187,7 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
   auto arrayTy = cast<LLVM::LLVMArrayType>(operand.getType());
 
   for (unsigned i = 0, e = arrayTy.getNumElements(); i < e; ++i) {
-    Value toUse = b.create<LLVM::ExtractValueOp>(operand, i);
+    Value toUse = LLVM::ExtractValueOp::create(b, operand, i);
 
     // For 4xi8 vectors, the intrinsic expects these to be provided as i32
     // scalar types.
@@ -195,7 +195,7 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
         arrayTy.getElementType() == i4x8Ty ||
         (arrayTy.getElementType() == f32x1Ty &&
          operandPtxType == NVVM::MMATypes::tf32)) {
-      result.push_back(b.create<LLVM::BitcastOp>(i32Ty, toUse));
+      result.push_back(LLVM::BitcastOp::create(b, i32Ty, toUse));
       continue;
     }
 
@@ -208,9 +208,9 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
                          innerArrayTy.getElementType() == f32Ty)) {
       for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
            idx < innerSize; idx++) {
-        result.push_back(b.create<LLVM::ExtractElementOp>(
+        result.push_back(LLVM::ExtractElementOp::create(b,
             toUse,
-            b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(idx))));
+            LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(idx))));
       }
       continue;
     }
@@ -285,7 +285,7 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
-    Value ldMatrixResult = b.create<NVVM::LdMatrixOp>(
+    Value ldMatrixResult = NVVM::LdMatrixOp::create(b,
         ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
@@ -296,13 +296,13 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     // actual vector type (still of width 32b) and repack them into a result
     // struct.
     Type finalResultType = typeConverter->convertType(vectorResultType);
-    Value result = b.create<LLVM::PoisonOp>(finalResultType);
+    Value result = LLVM::PoisonOp::create(b, finalResultType);
     for (int64_t i = 0, e = vectorResultType.getDimSize(0); i < e; i++) {
       Value i32Register =
-          num32BitRegs > 1 ? b.create<LLVM::ExtractValueOp>(ldMatrixResult, i)
+          num32BitRegs > 1 ? LLVM::ExtractValueOp::create(b, ldMatrixResult, i)
                            : ldMatrixResult;
-      Value casted = b.create<LLVM::BitcastOp>(innerVectorType, i32Register);
-      result = b.create<LLVM::InsertValueOp>(result, casted, i);
+      Value casted = LLVM::BitcastOp::create(b, innerVectorType, i32Register);
+      result = LLVM::InsertValueOp::create(b, result, casted, i);
     }
 
     rewriter.replaceOp(op, result);
@@ -375,7 +375,7 @@ struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
     Type desiredRetTy = typeConverter->convertType(op->getResultTypes()[0]);
     Type intrinsicResTy = inferIntrinsicResultType(
         typeConverter->convertType(op->getResultTypes()[0]));
-    Value intrinsicResult = b.create<NVVM::MmaOp>(
+    Value intrinsicResult = NVVM::MmaOp::create(b,
         intrinsicResTy, matA, matB, matC,
         /*shape=*/gemmShape,
         /*b1Op=*/std::nullopt,
@@ -565,7 +565,7 @@ static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
     llvm::append_range(asmVals, args);
   asmVals.push_back(indexData);
 
-  return b.create<LLVM::InlineAsmOp>(
+  return LLVM::InlineAsmOp::create(b,
       /*resultTypes=*/intrinsicResultType,
       /*operands=*/asmVals,
       /*asm_string=*/asmStr,
@@ -631,7 +631,7 @@ struct NVGPUMmaSparseSyncLowering
       return op->emitOpError() << "Expected metadata type to be LLVM "
                                   "VectorType of 2 i16 elements";
     sparseMetadata =
-        b.create<LLVM::BitcastOp>(rewriter.getI32Type(), sparseMetadata);
+        LLVM::BitcastOp::create(b, rewriter.getI32Type(), sparseMetadata);
 
     FailureOr<LLVM::InlineAsmOp> intrinsicResult = emitMmaSparseSyncOpAsm(
         b, *ptxTypeA, *ptxTypeB, *ptxTypeC, *ptxTypeC, overflow, matA, matB,
@@ -682,7 +682,7 @@ struct NVGPUAsyncCopyLowering
     // Intrinsics takes a global pointer so we need an address space cast.
     auto srcPointerGlobalType = LLVM::LLVMPointerType::get(
         op->getContext(), NVVM::NVVMMemorySpace::kGlobalMemorySpace);
-    scrPtr = b.create<LLVM::AddrSpaceCastOp>(srcPointerGlobalType, scrPtr);
+    scrPtr = LLVM::AddrSpaceCastOp::create(b, srcPointerGlobalType, scrPtr);
     int64_t dstElements = adaptor.getDstElements().getZExtValue();
     int64_t sizeInBytes =
         (dstMemrefType.getElementTypeBitWidth() * dstElements) / 8;
@@ -697,13 +697,13 @@ struct NVGPUAsyncCopyLowering
       // The rest of the DstElements in the destination (shared memory) are
       // filled with zeros.
       Value c3I32 =
-          b.create<LLVM::ConstantOp>(b.getI32Type(), b.getI32IntegerAttr(3));
-      Value bitwidth = b.create<LLVM::ConstantOp>(
+          LLVM::ConstantOp::create(b, b.getI32Type(), b.getI32IntegerAttr(3));
+      Value bitwidth = LLVM::ConstantOp::create(b,
           b.getI32Type(),
           b.getI32IntegerAttr(srcMemrefType.getElementTypeBitWidth()));
-      Value srcElementsI32 = b.create<LLVM::TruncOp>(b.getI32Type(), srcBytes);
-      srcBytes = b.create<LLVM::LShrOp>(
-          b.create<LLVM::MulOp>(bitwidth, srcElementsI32), c3I32);
+      Value srcElementsI32 = LLVM::TruncOp::create(b, b.getI32Type(), srcBytes);
+      srcBytes = LLVM::LShrOp::create(b,
+          LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
     }
     // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
     // 16 dst bytes.
@@ -712,13 +712,13 @@ struct NVGPUAsyncCopyLowering
             ? NVVM::LoadCacheModifierKind::CG
             : NVVM::LoadCacheModifierKind::CA;
 
-    b.create<NVVM::CpAsyncOp>(
+    NVVM::CpAsyncOp::create(b,
         dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
         NVVM::LoadCacheModifierKindAttr::get(op->getContext(), cacheModifier),
         srcBytes);
 
     // Drop the result token.
-    Value zero = b.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(b,
         IntegerType::get(op.getContext(), 32), rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
@@ -733,9 +733,9 @@ struct NVGPUAsyncCreateGroupLowering
   LogicalResult
   matchAndRewrite(nvgpu::DeviceAsyncCreateGroupOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    rewriter.create<NVVM::CpAsyncCommitGroupOp>(op.getLoc());
+    NVVM::CpAsyncCommitGroupOp::create(rewriter, op.getLoc());
     // Drop the result token.
-    Value zero = rewriter.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(rewriter,
         op->getLoc(), IntegerType::get(op.getContext(), 32),
         rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
@@ -753,7 +753,7 @@ struct NVGPUAsyncWaitLowering
                   ConversionPatternRewriter &rewriter) const override {
     // If numGroup is not present pick 0 as a conservative correct value.
     int32_t numGroups = adaptor.getNumGroups().value_or(0);
-    rewriter.create<NVVM::CpAsyncWaitGroupOp>(op.getLoc(), numGroups);
+    NVVM::CpAsyncWaitGroupOp::create(rewriter, op.getLoc(), numGroups);
     rewriter.eraseOp(op);
     return success();
   }
@@ -771,7 +771,7 @@ struct NVGPUMBarrierCreateLowering
     SymbolTable symbolTable(moduleOp);
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(&moduleOp.front());
-    auto global = rewriter.create<memref::GlobalOp>(
+    auto global = memref::GlobalOp::create(rewriter,
         funcOp->getLoc(), "__mbarrier",
         /*sym_visibility=*/rewriter.getStringAttr("private"),
         /*type=*/barrierType,
@@ -974,7 +974,7 @@ struct NVGPUMBarrierTryWaitParityLowering
                        adaptor.getMbarId(), rewriter);
     Value ticks = truncToI32(b, adaptor.getTicks());
     Value phase =
-        b.create<LLVM::ZExtOp>(b.getI32Type(), adaptor.getPhaseParity());
+        LLVM::ZExtOp::create(b, b.getI32Type(), adaptor.getPhaseParity());
 
     if (isMbarrierShared(op.getBarriers().getType())) {
       rewriter.replaceOpWithNewOp<NVVM::MBarrierTryWaitParitySharedOp>(
@@ -1063,16 +1063,16 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 
     auto ti64 = b.getIntegerType(64);
     auto makeConst = [&](uint64_t index) -> Value {
-      return b.create<LLVM::ConstantOp>(ti64, b.getI64IntegerAttr(index));
+      return LLVM::ConstantOp::create(b, ti64, b.getI64IntegerAttr(index));
     };
     auto shiftLeft = [&](Value value, unsigned shift) -> Value {
-      return b.create<LLVM::ShlOp>(ti64, value, makeConst(shift));
+      return LLVM::ShlOp::create(b, ti64, value, makeConst(shift));
     };
     auto shiftRight = [&](Value value, unsigned shift) -> Value {
-      return b.create<LLVM::LShrOp>(ti64, value, makeConst(shift));
+      return LLVM::LShrOp::create(b, ti64, value, makeConst(shift));
     };
     auto insertBit = [&](Value desc, Value val, int startBit) {
-      return b.create<LLVM::OrOp>(ti64, desc, shiftLeft(val, startBit));
+      return LLVM::OrOp::create(b, ti64, desc, shiftLeft(val, startBit));
     };
 
     int64_t sizeN = op.getTensorMap().getType().getTensor().getDimSize(0);
@@ -1086,7 +1086,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
     Value baseAddr = getStridedElementPtr(
         rewriter, op->getLoc(), cast<MemRefType>(op.getTensor().getType()),
         adaptor.getTensor(), {});
-    Value basePtr = b.create<LLVM::PtrToIntOp>(ti64, baseAddr);
+    Value basePtr = LLVM::PtrToIntOp::create(b, ti64, baseAddr);
     // Just use 14 bits for base address
     Value basePtr14bit = shiftRight(shiftLeft(basePtr, 46), 50);
 
@@ -1118,7 +1118,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 };
 
 static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
-  return b.create<LLVM::ConstantOp>(b.getIntegerType(64),
+  return LLVM::ConstantOp::create(b, b.getIntegerType(64),
                                     b.getI32IntegerAttr(index));
 }
 
@@ -1182,12 +1182,12 @@ struct NVGPUTmaCreateDescriptorOpLowering
     auto promotedOperands = getTypeConverter()->promoteOperands(
         b.getLoc(), op->getOperands(), adaptor.getOperands(), b);
 
-    Value boxArrayPtr = b.create<LLVM::AllocaOp>(llvmPointerType, llvmInt64Type,
+    Value boxArrayPtr = LLVM::AllocaOp::create(b, llvmPointerType, llvmInt64Type,
                                                  makeI64Const(b, 5));
     for (auto [index, value] : llvm::enumerate(adaptor.getBoxDimensions())) {
-      Value gep = b.create<LLVM::GEPOp>(llvmPointerType, llvmPointerType,
+      Value gep = LLVM::GEPOp::create(b, llvmPointerType, llvmPointerType,
                                         boxArrayPtr, makeI64Const(b, index));
-      b.create<LLVM::StoreOp>(value, gep);
+      LLVM::StoreOp::create(b, value, gep);
     }
 
     nvgpu::TensorMapDescriptorType desc = op.getTensorMap().getType();
@@ -1337,7 +1337,7 @@ struct NVGPUWarpgroupMmaOpLowering
 
     /// Basic function to generate Add
     Value makeAdd(Value lhs, Value rhs) {
-      return b.create<LLVM::AddOp>(lhs.getType(), lhs, rhs);
+      return LLVM::AddOp::create(b, lhs.getType(), lhs, rhs);
     };
 
     /// Moves the descriptor pointer of matrix-A for the next wgmma instruction.
@@ -1430,7 +1430,7 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      return b.create<NVVM::WgmmaMmaAsyncOp>(
+      return NVVM::WgmmaMmaAsyncOp::create(b,
           matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
           itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
           overflow);
@@ -1439,19 +1439,19 @@ struct NVGPUWarpgroupMmaOpLowering
     /// Generates multiple wgmma instructions to complete the given GEMM shape
     Value generateWgmmaGroup() {
       Value wgmmaResult =
-          b.create<LLVM::PoisonOp>(adaptor.getMatrixC().getType());
+          LLVM::PoisonOp::create(b, adaptor.getMatrixC().getType());
 
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = b.create<LLVM::ExtractValueOp>(adaptor.getMatrixC(), i);
+        Value matrixC = LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
             matrixC = generateWgmma(i, j, k, matrixC);
         wgmmaResults.push_back(matrixC);
       }
       for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
-        wgmmaResult = b.create<LLVM::InsertValueOp>(wgmmaResult.getType(),
+        wgmmaResult = LLVM::InsertValueOp::create(b, wgmmaResult.getType(),
                                                     wgmmaResult, matrix, idx);
       }
       return wgmmaResult;
@@ -1486,10 +1486,10 @@ struct NVGPUWarpgroupMmaOpLowering
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
     Value generateWarpgroupMma() {
-      b.create<NVVM::WgmmaFenceAlignedOp>();
+      NVVM::WgmmaFenceAlignedOp::create(b, );
       Value wgmmaResult = generateWgmmaGroup();
-      b.create<NVVM::WgmmaGroupSyncAlignedOp>();
-      b.create<NVVM::WgmmaWaitGroupSyncOp>(op.getWaitGroup());
+      NVVM::WgmmaGroupSyncAlignedOp::create(b, );
+      NVVM::WgmmaWaitGroupSyncOp::create(b, op.getWaitGroup());
       return wgmmaResult;
     }
   };
@@ -1557,7 +1557,7 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     Type i32 = b.getI32Type();
 
     auto makeConst = [&](int32_t index) -> Value {
-      return b.create<LLVM::ConstantOp>(i32, b.getI32IntegerAttr(index));
+      return LLVM::ConstantOp::create(b, i32, b.getI32IntegerAttr(index));
     };
     Value c1 = makeConst(1);
     Value c2 = makeConst(2);
@@ -1567,29 +1567,29 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     Value warpSize = makeConst(kWarpSize);
 
     auto makeMul = [&](Value lhs, Value rhs) -> Value {
-      return b.create<LLVM::MulOp>(lhs.getType(), lhs, rhs);
+      return LLVM::MulOp::create(b, lhs.getType(), lhs, rhs);
     };
     auto makeAdd = [&](Value lhs, Value rhs) -> Value {
-      return b.create<LLVM::AddOp>(lhs.getType(), lhs, rhs);
+      return LLVM::AddOp::create(b, lhs.getType(), lhs, rhs);
     };
 
     auto makeExtractAndStore = [&](int i, Value wgmmaResult, Value x, Value y,
                                    TypedValue<::mlir::MemRefType> memref) {
       Type it = b.getIndexType();
-      Value idx = b.create<arith::IndexCastOp>(it, x);
-      Value idy0 = b.create<arith::IndexCastOp>(it, y);
-      Value idy1 = b.create<arith::IndexCastOp>(it, makeAdd(y, c1));
-      Value d0 = b.create<LLVM::ExtractValueOp>(wgmmaResult, i);
-      Value d1 = b.create<LLVM::ExtractValueOp>(wgmmaResult, i + 1);
-      b.create<memref::StoreOp>(d0, memref, ValueRange{idx, idy0});
-      b.create<memref::StoreOp>(d1, memref, ValueRange{idx, idy1});
+      Value idx = arith::IndexCastOp::create(b, it, x);
+      Value idy0 = arith::IndexCastOp::create(b, it, y);
+      Value idy1 = arith::IndexCastOp::create(b, it, makeAdd(y, c1));
+      Value d0 = LLVM::ExtractValueOp::create(b, wgmmaResult, i);
+      Value d1 = LLVM::ExtractValueOp::create(b, wgmmaResult, i + 1);
+      memref::StoreOp::create(b, d0, memref, ValueRange{idx, idy0});
+      memref::StoreOp::create(b, d1, memref, ValueRange{idx, idy1});
     };
 
-    Value tidx = b.create<NVVM::ThreadIdXOp>(i32);
-    Value laneId = b.create<LLVM::URemOp>(i32, tidx, warpSize);
-    Value warpId = b.create<LLVM::UDivOp>(i32, tidx, warpSize);
-    Value lane4Id = b.create<LLVM::UDivOp>(i32, laneId, c4);
-    Value lane4modId = b.create<LLVM::URemOp>(i32, laneId, c4);
+    Value tidx = NVVM::ThreadIdXOp::create(b, i32);
+    Value laneId = LLVM::URemOp::create(b, i32, tidx, warpSize);
+    Value warpId = LLVM::UDivOp::create(b, i32, tidx, warpSize);
+    Value lane4Id = LLVM::UDivOp::create(b, i32, laneId, c4);
+    Value lane4modId = LLVM::URemOp::create(b, i32, laneId, c4);
 
     Value tj = makeMul(lane4modId, c2);
     Value ti = makeAdd(lane4Id, makeMul(warpId, c16));
@@ -1626,7 +1626,7 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     auto stype = cast<LLVM::LLVMStructType>(matriDValue.getType());
     for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(matrixD);
-      Value innerStructValue = b.create<LLVM::ExtractValueOp>(matriDValue, idx);
+      Value innerStructValue = LLVM::ExtractValueOp::create(b, matriDValue, idx);
       storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
@@ -1648,22 +1648,22 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
     Type elemType = cast<LLVM::LLVMStructType>(packStructType.getBody().front())
                         .getBody()
                         .front();
-    Value zero = b.create<LLVM::ConstantOp>(elemType, b.getZeroAttr(elemType));
-    Value packStruct = b.create<LLVM::PoisonOp>(packStructType);
+    Value zero = LLVM::ConstantOp::create(b, elemType, b.getZeroAttr(elemType));
+    Value packStruct = LLVM::PoisonOp::create(b, packStructType);
     SmallVector<Value> innerStructs;
     // Unpack the structs and set all values to zero
     for (auto [idx, s] : llvm::enumerate(packStructType.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(s);
-      Value structValue = b.create<LLVM::ExtractValueOp>(packStruct, idx);
+      Value structValue = LLVM::ExtractValueOp::create(b, packStruct, idx);
       for (unsigned i = 0; i < structType.getBody().size(); ++i) {
-        structValue = b.create<LLVM::InsertValueOp>(
+        structValue = LLVM::InsertValueOp::create(b,
             structType, structValue, zero, ArrayRef<int64_t>({i}));
       }
       innerStructs.push_back(structValue);
     }
     // Pack the inner structs into a single struct
     for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
-      packStruct = b.create<LLVM::InsertValueOp>(packStruct.getType(),
+      packStruct = LLVM::InsertValueOp::create(b, packStruct.getType(),
                                                  packStruct, matrix, idx);
     }
     rewriter.replaceOp(op, packStruct);
@@ -1681,7 +1681,7 @@ struct NVGPUTmaFenceOpLowering
     ImplicitLocOpBuilder b(op->getLoc(), rewriter);
     auto i32Ty = b.getI32Type();
     Value tensormapSize =
-        b.create<LLVM::ConstantOp>(i32Ty, rewriter.getI32IntegerAttr(128));
+        LLVM::ConstantOp::create(b, i32Ty, rewriter.getI32IntegerAttr(128));
 
     auto memscope =
         NVVM::MemScopeKindAttr::get(ctx, ::mlir::NVVM::MemScopeKind::SYS);
@@ -1716,13 +1716,13 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
     VectorType inTy = op.getIn().getType();
     // apply rcp.approx.ftz.f on each element in vector.
     auto convert1DVec = [&](Type llvm1DVectorTy, Value inVec) {
-      Value ret1DVec = b.create<LLVM::PoisonOp>(llvm1DVectorTy);
+      Value ret1DVec = LLVM::PoisonOp::create(b, llvm1DVectorTy);
       int numElems = llvm::cast<VectorType>(llvm1DVectorTy).getNumElements();
       for (int i = 0; i < numElems; i++) {
-        Value idx = b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(i));
-        Value elem = b.create<LLVM::ExtractElementOp>(inVec, idx);
-        Value dst = b.create<NVVM::RcpApproxFtzF32Op>(f32Ty, elem);
-        ret1DVec = b.create<LLVM::InsertElementOp>(ret1DVec, dst, idx);
+        Value idx = LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(i));
+        Value elem = LLVM::ExtractElementOp::create(b, inVec, idx);
+        Value dst = NVVM::RcpApproxFtzF32Op::create(b, f32Ty, elem);
+        ret1DVec = LLVM::InsertElementOp::create(b, ret1DVec, dst, idx);
       }
       return ret1DVec;
     };
diff --git a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
index 479725aae8afd..a6125fc07dcd5 100644
--- a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
+++ b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
@@ -39,7 +39,7 @@ class ExpandIfCondition : public OpRewritePattern<OpTy> {
 
     IntegerAttr constAttr;
     if (!matchPattern(op.getIfCond(), m_Constant(&constAttr))) {
-      auto ifOp = rewriter.create<scf::IfOp>(op.getLoc(), TypeRange(),
+      auto ifOp = scf::IfOp::create(rewriter, op.getLoc(), TypeRange(),
                                              op.getIfCond(), false);
       rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });
       auto thenBodyBuilder = ifOp.getThenBodyBuilder(rewriter.getListener());
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 7ac9687c4eeda..4bf59124a9bdb 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -95,7 +95,7 @@ struct OpenMPOpConversion : public ConvertOpToLLVMPattern<T> {
     }
 
     // Create new operation.
-    auto newOp = rewriter.create<T>(op.getLoc(), resTypes, convertedOperands,
+    auto newOp = T::create(rewriter, op.getLoc(), resTypes, convertedOperands,
                                     convertedAttrs);
 
     // Translate regions.
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 7d20109b3db59..75cd35c0d7876 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -196,7 +196,7 @@ Block *PatternLowering::generateMatcher(MatcherNode &node, Region &region,
   // finalize.
   if (isa<ExitNode>(node)) {
     builder.setInsertionPointToEnd(block);
-    builder.create<pdl_interp::FinalizeOp>(matcherFunc.getLoc());
+    pdl_interp::FinalizeOp::create(builder, matcherFunc.getLoc());
     return block;
   }
 
@@ -272,7 +272,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     auto *operationPos = cast<OperationPosition>(pos);
     if (operationPos->isOperandDefiningOp())
       // Standard (downward) traversal which directly follows the defining op.
-      value = builder.create<pdl_interp::GetDefiningOpOp>(
+      value = pdl_interp::GetDefiningOpOp::create(builder,
           loc, builder.getType<pdl::OperationType>(), parentVal);
     else
       // A passthrough operation position.
@@ -287,23 +287,23 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     // requested to use a representative value (e.g., upward traversal).
     if (isa<pdl::RangeType>(parentVal.getType()) &&
         usersPos->useRepresentative())
-      value = builder.create<pdl_interp::ExtractOp>(loc, parentVal, 0);
+      value = pdl_interp::ExtractOp::create(builder, loc, parentVal, 0);
     else
       value = parentVal;
 
     // The second operation retrieves the users.
-    value = builder.create<pdl_interp::GetUsersOp>(loc, value);
+    value = pdl_interp::GetUsersOp::create(builder, loc, value);
     break;
   }
   case Predicates::ForEachPos: {
     assert(!failureBlockStack.empty() && "expected valid failure block");
-    auto foreach = builder.create<pdl_interp::ForEachOp>(
+    auto foreach = pdl_interp::ForEachOp::create(builder,
         loc, parentVal, failureBlockStack.back(), /*initLoop=*/true);
     value = foreach.getLoopVariable();
 
     // Create the continuation block.
     Block *continueBlock = builder.createBlock(&foreach.getRegion());
-    builder.create<pdl_interp::ContinueOp>(loc);
+    pdl_interp::ContinueOp::create(builder, loc);
     failureBlockStack.push_back(continueBlock);
 
     currentBlock = &foreach.getRegion().front();
@@ -311,7 +311,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::OperandPos: {
     auto *operandPos = cast<OperandPosition>(pos);
-    value = builder.create<pdl_interp::GetOperandOp>(
+    value = pdl_interp::GetOperandOp::create(builder,
         loc, builder.getType<pdl::ValueType>(), parentVal,
         operandPos->getOperandNumber());
     break;
@@ -319,28 +319,28 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::OperandGroupPos: {
     auto *operandPos = cast<OperandGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = builder.create<pdl_interp::GetOperandsOp>(
+    value = pdl_interp::GetOperandsOp::create(builder,
         loc, operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, operandPos->getOperandGroupNumber());
     break;
   }
   case Predicates::AttributePos: {
     auto *attrPos = cast<AttributePosition>(pos);
-    value = builder.create<pdl_interp::GetAttributeOp>(
+    value = pdl_interp::GetAttributeOp::create(builder,
         loc, builder.getType<pdl::AttributeType>(), parentVal,
         attrPos->getName().strref());
     break;
   }
   case Predicates::TypePos: {
     if (isa<pdl::AttributeType>(parentVal.getType()))
-      value = builder.create<pdl_interp::GetAttributeTypeOp>(loc, parentVal);
+      value = pdl_interp::GetAttributeTypeOp::create(builder, loc, parentVal);
     else
-      value = builder.create<pdl_interp::GetValueTypeOp>(loc, parentVal);
+      value = pdl_interp::GetValueTypeOp::create(builder, loc, parentVal);
     break;
   }
   case Predicates::ResultPos: {
     auto *resPos = cast<ResultPosition>(pos);
-    value = builder.create<pdl_interp::GetResultOp>(
+    value = pdl_interp::GetResultOp::create(builder,
         loc, builder.getType<pdl::ValueType>(), parentVal,
         resPos->getResultNumber());
     break;
@@ -348,7 +348,7 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::ResultGroupPos: {
     auto *resPos = cast<ResultGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = builder.create<pdl_interp::GetResultsOp>(
+    value = pdl_interp::GetResultsOp::create(builder,
         loc, resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, resPos->getResultGroupNumber());
     break;
@@ -356,16 +356,16 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   case Predicates::AttributeLiteralPos: {
     auto *attrPos = cast<AttributeLiteralPosition>(pos);
     value =
-        builder.create<pdl_interp::CreateAttributeOp>(loc, attrPos->getValue());
+        pdl_interp::CreateAttributeOp::create(builder, loc, attrPos->getValue());
     break;
   }
   case Predicates::TypeLiteralPos: {
     auto *typePos = cast<TypeLiteralPosition>(pos);
     Attribute rawTypeAttr = typePos->getValue();
     if (TypeAttr typeAttr = dyn_cast<TypeAttr>(rawTypeAttr))
-      value = builder.create<pdl_interp::CreateTypeOp>(loc, typeAttr);
+      value = pdl_interp::CreateTypeOp::create(builder, loc, typeAttr);
     else
-      value = builder.create<pdl_interp::CreateTypesOp>(
+      value = pdl_interp::CreateTypesOp::create(builder,
           loc, cast<ArrayAttr>(rawTypeAttr));
     break;
   }
@@ -413,54 +413,54 @@ void PatternLowering::generate(BoolNode *boolNode, Block *&currentBlock,
   Predicates::Kind kind = question->getKind();
   switch (kind) {
   case Predicates::IsNotNullQuestion:
-    builder.create<pdl_interp::IsNotNullOp>(loc, val, success, failure);
+    pdl_interp::IsNotNullOp::create(builder, loc, val, success, failure);
     break;
   case Predicates::OperationNameQuestion: {
     auto *opNameAnswer = cast<OperationNameAnswer>(answer);
-    builder.create<pdl_interp::CheckOperationNameOp>(
+    pdl_interp::CheckOperationNameOp::create(builder,
         loc, val, opNameAnswer->getValue().getStringRef(), success, failure);
     break;
   }
   case Predicates::TypeQuestion: {
     auto *ans = cast<TypeAnswer>(answer);
     if (isa<pdl::RangeType>(val.getType()))
-      builder.create<pdl_interp::CheckTypesOp>(
+      pdl_interp::CheckTypesOp::create(builder,
           loc, val, llvm::cast<ArrayAttr>(ans->getValue()), success, failure);
     else
-      builder.create<pdl_interp::CheckTypeOp>(
+      pdl_interp::CheckTypeOp::create(builder,
           loc, val, llvm::cast<TypeAttr>(ans->getValue()), success, failure);
     break;
   }
   case Predicates::AttributeQuestion: {
     auto *ans = cast<AttributeAnswer>(answer);
-    builder.create<pdl_interp::CheckAttributeOp>(loc, val, ans->getValue(),
+    pdl_interp::CheckAttributeOp::create(builder, loc, val, ans->getValue(),
                                                  success, failure);
     break;
   }
   case Predicates::OperandCountAtLeastQuestion:
   case Predicates::OperandCountQuestion:
-    builder.create<pdl_interp::CheckOperandCountOp>(
+    pdl_interp::CheckOperandCountOp::create(builder,
         loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::OperandCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::ResultCountAtLeastQuestion:
   case Predicates::ResultCountQuestion:
-    builder.create<pdl_interp::CheckResultCountOp>(
+    pdl_interp::CheckResultCountOp::create(builder,
         loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::ResultCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::EqualToQuestion: {
     bool trueAnswer = isa<TrueAnswer>(answer);
-    builder.create<pdl_interp::AreEqualOp>(loc, val, args.front(),
+    pdl_interp::AreEqualOp::create(builder, loc, val, args.front(),
                                            trueAnswer ? success : failure,
                                            trueAnswer ? failure : success);
     break;
   }
   case Predicates::ConstraintQuestion: {
     auto *cstQuestion = cast<ConstraintQuestion>(question);
-    auto applyConstraintOp = builder.create<pdl_interp::ApplyConstraintOp>(
+    auto applyConstraintOp = pdl_interp::ApplyConstraintOp::create(builder,
         loc, cstQuestion->getResultTypes(), cstQuestion->getName(), args,
         cstQuestion->getIsNegated(), success, failure);
 
@@ -487,7 +487,7 @@ static void createSwitchOp(Value val, Block *defaultDest, OpBuilder &builder,
     blocks.push_back(it.second);
     values.push_back(cast<PredT>(it.first)->getValue());
   }
-  builder.create<OpT>(val.getLoc(), val, values, defaultDest, blocks);
+  OpT::create(builder, val.getLoc(), val, values, defaultDest, blocks);
 }
 
 void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
@@ -536,11 +536,11 @@ void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
       unsigned ans = cast<UnsignedAnswer>(child.first)->getValue();
       switch (kind) {
       case Predicates::OperandCountAtLeastQuestion:
-        builder.create<pdl_interp::CheckOperandCountOp>(
+        pdl_interp::CheckOperandCountOp::create(builder,
             loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
         break;
       case Predicates::ResultCountAtLeastQuestion:
-        builder.create<pdl_interp::CheckResultCountOp>(
+        pdl_interp::CheckResultCountOp::create(builder,
             loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
         break;
       default:
@@ -619,7 +619,7 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
       rootKindAttr = builder.getStringAttr(*rootKind);
 
   builder.setInsertionPointToEnd(currentBlock);
-  auto matchOp = builder.create<pdl_interp::RecordMatchOp>(
+  auto matchOp = pdl_interp::RecordMatchOp::create(builder,
       pattern.getLoc(), mappedMatchValues, locOps.getArrayRef(),
       rewriterFuncRef, rootKindAttr, generatedOpsAttr, pattern.getBenefitAttr(),
       failureBlockStack.back());
@@ -632,7 +632,7 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
 SymbolRefAttr PatternLowering::generateRewriter(
     pdl::PatternOp pattern, SmallVectorImpl<Position *> &usedMatchValues) {
   builder.setInsertionPointToEnd(rewriterModule.getBody());
-  auto rewriterFunc = builder.create<pdl_interp::FuncOp>(
+  auto rewriterFunc = pdl_interp::FuncOp::create(builder,
       pattern.getLoc(), "pdl_generated_rewriter",
       builder.getFunctionType({}, {}));
   rewriterSymbolTable.insert(rewriterFunc);
@@ -651,17 +651,17 @@ SymbolRefAttr PatternLowering::generateRewriter(
     Operation *oldOp = oldValue.getDefiningOp();
     if (pdl::AttributeOp attrOp = dyn_cast<pdl::AttributeOp>(oldOp)) {
       if (Attribute value = attrOp.getValueAttr()) {
-        return newValue = builder.create<pdl_interp::CreateAttributeOp>(
+        return newValue = pdl_interp::CreateAttributeOp::create(builder,
                    attrOp.getLoc(), value);
       }
     } else if (pdl::TypeOp typeOp = dyn_cast<pdl::TypeOp>(oldOp)) {
       if (TypeAttr type = typeOp.getConstantTypeAttr()) {
-        return newValue = builder.create<pdl_interp::CreateTypeOp>(
+        return newValue = pdl_interp::CreateTypeOp::create(builder,
                    typeOp.getLoc(), type);
       }
     } else if (pdl::TypesOp typeOp = dyn_cast<pdl::TypesOp>(oldOp)) {
       if (ArrayAttr type = typeOp.getConstantTypesAttr()) {
-        return newValue = builder.create<pdl_interp::CreateTypesOp>(
+        return newValue = pdl_interp::CreateTypesOp::create(builder,
                    typeOp.getLoc(), typeOp.getType(), type);
       }
     }
@@ -684,7 +684,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
     auto mappedArgs =
         llvm::map_range(rewriter.getExternalArgs(), mapRewriteValue);
     args.append(mappedArgs.begin(), mappedArgs.end());
-    builder.create<pdl_interp::ApplyRewriteOp>(
+    pdl_interp::ApplyRewriteOp::create(builder,
         rewriter.getLoc(), /*resultTypes=*/TypeRange(), rewriteName, args);
   } else {
     // Otherwise this is a dag rewriter defined using PDL operations.
@@ -703,7 +703,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
       llvm::to_vector<8>(rewriterFunc.front().getArgumentTypes()),
       /*results=*/{}));
 
-  builder.create<pdl_interp::FinalizeOp>(rewriter.getLoc());
+  pdl_interp::FinalizeOp::create(builder, rewriter.getLoc());
   return SymbolRefAttr::get(
       builder.getContext(),
       pdl_interp::PDLInterpDialect::getRewriterModuleName(),
@@ -716,7 +716,7 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 2> arguments;
   for (Value argument : rewriteOp.getArgs())
     arguments.push_back(mapRewriteValue(argument));
-  auto interpOp = builder.create<pdl_interp::ApplyRewriteOp>(
+  auto interpOp = pdl_interp::ApplyRewriteOp::create(builder,
       rewriteOp.getLoc(), rewriteOp.getResultTypes(), rewriteOp.getNameAttr(),
       arguments);
   for (auto it : llvm::zip(rewriteOp.getResults(), interpOp.getResults()))
@@ -726,7 +726,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::AttributeOp attrOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  Value newAttr = builder.create<pdl_interp::CreateAttributeOp>(
+  Value newAttr = pdl_interp::CreateAttributeOp::create(builder,
       attrOp.getLoc(), attrOp.getValueAttr());
   rewriteValues[attrOp] = newAttr;
 }
@@ -734,7 +734,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::EraseOp eraseOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  builder.create<pdl_interp::EraseOp>(eraseOp.getLoc(),
+  pdl_interp::EraseOp::create(builder, eraseOp.getLoc(),
                                       mapRewriteValue(eraseOp.getOpValue()));
 }
 
@@ -756,7 +756,7 @@ void PatternLowering::generateRewriter(
 
   // Create the new operation.
   Location loc = operationOp.getLoc();
-  Value createdOp = builder.create<pdl_interp::CreateOperationOp>(
+  Value createdOp = pdl_interp::CreateOperationOp::create(builder,
       loc, *operationOp.getOpName(), types, hasInferredResultTypes, operands,
       attributes, operationOp.getAttributeValueNames());
   rewriteValues[operationOp.getOp()] = createdOp;
@@ -768,8 +768,8 @@ void PatternLowering::generateRewriter(
   if (resultTys.size() == 1 && isa<pdl::RangeType>(resultTys[0].getType())) {
     Value &type = rewriteValues[resultTys[0]];
     if (!type) {
-      auto results = builder.create<pdl_interp::GetResultsOp>(loc, createdOp);
-      type = builder.create<pdl_interp::GetValueTypeOp>(loc, results);
+      auto results = pdl_interp::GetResultsOp::create(builder, loc, createdOp);
+      type = pdl_interp::GetValueTypeOp::create(builder, loc, results);
     }
     return;
   }
@@ -789,12 +789,12 @@ void PatternLowering::generateRewriter(
     // groups because the exact index of the result is not statically known.
     Value resultVal;
     if (seenVariableLength)
-      resultVal = builder.create<pdl_interp::GetResultsOp>(
+      resultVal = pdl_interp::GetResultsOp::create(builder,
           loc, isVariadic ? valueRangeTy : valueTy, createdOp, it.index());
     else
-      resultVal = builder.create<pdl_interp::GetResultOp>(
+      resultVal = pdl_interp::GetResultOp::create(builder,
           loc, valueTy, createdOp, it.index());
-    type = builder.create<pdl_interp::GetValueTypeOp>(loc, resultVal);
+    type = pdl_interp::GetValueTypeOp::create(builder, loc, resultVal);
   }
 }
 
@@ -804,7 +804,7 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 4> replOperands;
   for (Value operand : rangeOp.getArguments())
     replOperands.push_back(mapRewriteValue(operand));
-  rewriteValues[rangeOp] = builder.create<pdl_interp::CreateRangeOp>(
+  rewriteValues[rangeOp] = pdl_interp::CreateRangeOp::create(builder,
       rangeOp.getLoc(), rangeOp.getType(), replOperands);
 }
 
@@ -820,7 +820,7 @@ void PatternLowering::generateRewriter(
     // Don't use replace if we know the replaced operation has no results.
     auto opOp = replaceOp.getOpValue().getDefiningOp<pdl::OperationOp>();
     if (!opOp || !opOp.getTypeValues().empty()) {
-      replOperands.push_back(builder.create<pdl_interp::GetResultsOp>(
+      replOperands.push_back(pdl_interp::GetResultsOp::create(builder,
           replOp.getLoc(), mapRewriteValue(replOp)));
     }
   } else {
@@ -830,12 +830,12 @@ void PatternLowering::generateRewriter(
 
   // If there are no replacement values, just create an erase instead.
   if (replOperands.empty()) {
-    builder.create<pdl_interp::EraseOp>(
+    pdl_interp::EraseOp::create(builder,
         replaceOp.getLoc(), mapRewriteValue(replaceOp.getOpValue()));
     return;
   }
 
-  builder.create<pdl_interp::ReplaceOp>(replaceOp.getLoc(),
+  pdl_interp::ReplaceOp::create(builder, replaceOp.getLoc(),
                                         mapRewriteValue(replaceOp.getOpValue()),
                                         replOperands);
 }
@@ -843,7 +843,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::ResultOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultOp>(
+  rewriteValues[resultOp] = pdl_interp::GetResultOp::create(builder,
       resultOp.getLoc(), builder.getType<pdl::ValueType>(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
@@ -851,7 +851,7 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::ResultsOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultsOp>(
+  rewriteValues[resultOp] = pdl_interp::GetResultsOp::create(builder,
       resultOp.getLoc(), resultOp.getType(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
@@ -863,7 +863,7 @@ void PatternLowering::generateRewriter(
   // type.
   if (TypeAttr typeAttr = typeOp.getConstantTypeAttr()) {
     rewriteValues[typeOp] =
-        builder.create<pdl_interp::CreateTypeOp>(typeOp.getLoc(), typeAttr);
+        pdl_interp::CreateTypeOp::create(builder, typeOp.getLoc(), typeAttr);
   }
 }
 
@@ -873,7 +873,7 @@ void PatternLowering::generateRewriter(
   // If the type isn't constant, the users (e.g. OperationOp) will resolve this
   // type.
   if (ArrayAttr typeAttr = typeOp.getConstantTypesAttr()) {
-    rewriteValues[typeOp] = builder.create<pdl_interp::CreateTypesOp>(
+    rewriteValues[typeOp] = pdl_interp::CreateTypesOp::create(builder,
         typeOp.getLoc(), typeOp.getType(), typeAttr);
   }
 }
@@ -939,9 +939,9 @@ void PatternLowering::generateOperationResultTypeRewriter(
         !replacedOp->isBeforeInBlock(op))
       continue;
 
-    Value replacedOpResults = builder.create<pdl_interp::GetResultsOp>(
+    Value replacedOpResults = pdl_interp::GetResultsOp::create(builder,
         replacedOp->getLoc(), mapRewriteValue(replOpVal));
-    types.push_back(builder.create<pdl_interp::GetValueTypeOp>(
+    types.push_back(pdl_interp::GetValueTypeOp::create(builder,
         replacedOp->getLoc(), replacedOpResults));
     return;
   }
@@ -985,7 +985,7 @@ void PDLToPDLInterpPass::runOnOperation() {
   // Create the main matcher function This function contains all of the match
   // related functionality from patterns in the module.
   OpBuilder builder = OpBuilder::atBlockBegin(module.getBody());
-  auto matcherFunc = builder.create<pdl_interp::FuncOp>(
+  auto matcherFunc = pdl_interp::FuncOp::create(builder,
       module.getLoc(), pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
       builder.getFunctionType(builder.getType<pdl::OperationType>(),
                               /*results=*/{}),
@@ -993,7 +993,7 @@ void PDLToPDLInterpPass::runOnOperation() {
 
   // Create a nested module to hold the functions invoked for rewriting the IR
   // after a successful match.
-  ModuleOp rewriterModule = builder.create<ModuleOp>(
+  ModuleOp rewriterModule = ModuleOp::create(builder,
       module.getLoc(), pdl_interp::PDLInterpDialect::getRewriterModuleName());
 
   // Generate the code for the patterns within the module.
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index 0df91a243d07a..68527da73e1c7 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -340,7 +340,7 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   Operation *terminator = lastBodyBlock->getTerminator();
   rewriter.setInsertionPointToEnd(lastBodyBlock);
   auto step = forOp.getStep();
-  auto stepped = rewriter.create<arith::AddIOp>(loc, iv, step).getResult();
+  auto stepped = arith::AddIOp::create(rewriter, loc, iv, step).getResult();
   if (!stepped)
     return failure();
 
@@ -348,7 +348,7 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   loopCarried.push_back(stepped);
   loopCarried.append(terminator->operand_begin(), terminator->operand_end());
   auto branchOp =
-      rewriter.create<cf::BranchOp>(loc, conditionBlock, loopCarried);
+      cf::BranchOp::create(rewriter, loc, conditionBlock, loopCarried);
 
   // Let the CondBranchOp carry the LLVM attributes from the ForOp, such as the
   // llvm.loop_annotation attribute.
@@ -375,14 +375,14 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   SmallVector<Value, 8> destOperands;
   destOperands.push_back(lowerBound);
   llvm::append_range(destOperands, forOp.getInitArgs());
-  rewriter.create<cf::BranchOp>(loc, conditionBlock, destOperands);
+  cf::BranchOp::create(rewriter, loc, conditionBlock, destOperands);
 
   // With the body block done, we can fill in the condition block.
   rewriter.setInsertionPointToEnd(conditionBlock);
-  auto comparison = rewriter.create<arith::CmpIOp>(
+  auto comparison = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::slt, iv, upperBound);
 
-  rewriter.create<cf::CondBranchOp>(loc, comparison, firstBodyBlock,
+  cf::CondBranchOp::create(rewriter, loc, comparison, firstBodyBlock,
                                     ArrayRef<Value>(), endBlock,
                                     ArrayRef<Value>());
 
@@ -409,7 +409,7 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
     continueBlock =
         rewriter.createBlock(remainingOpsBlock, ifOp.getResultTypes(),
                              SmallVector<Location>(ifOp.getNumResults(), loc));
-    rewriter.create<cf::BranchOp>(loc, remainingOpsBlock);
+    cf::BranchOp::create(rewriter, loc, remainingOpsBlock);
   }
 
   // Move blocks from the "then" region to the region containing 'scf.if',
@@ -419,7 +419,7 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
   Operation *thenTerminator = thenRegion.back().getTerminator();
   ValueRange thenTerminatorOperands = thenTerminator->getOperands();
   rewriter.setInsertionPointToEnd(&thenRegion.back());
-  rewriter.create<cf::BranchOp>(loc, continueBlock, thenTerminatorOperands);
+  cf::BranchOp::create(rewriter, loc, continueBlock, thenTerminatorOperands);
   rewriter.eraseOp(thenTerminator);
   rewriter.inlineRegionBefore(thenRegion, continueBlock);
 
@@ -433,13 +433,13 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
     Operation *elseTerminator = elseRegion.back().getTerminator();
     ValueRange elseTerminatorOperands = elseTerminator->getOperands();
     rewriter.setInsertionPointToEnd(&elseRegion.back());
-    rewriter.create<cf::BranchOp>(loc, continueBlock, elseTerminatorOperands);
+    cf::BranchOp::create(rewriter, loc, continueBlock, elseTerminatorOperands);
     rewriter.eraseOp(elseTerminator);
     rewriter.inlineRegionBefore(elseRegion, continueBlock);
   }
 
   rewriter.setInsertionPointToEnd(condBlock);
-  rewriter.create<cf::CondBranchOp>(loc, ifOp.getCondition(), thenBlock,
+  cf::CondBranchOp::create(rewriter, loc, ifOp.getCondition(), thenBlock,
                                     /*trueArgs=*/ArrayRef<Value>(), elseBlock,
                                     /*falseArgs=*/ArrayRef<Value>());
 
@@ -459,13 +459,13 @@ ExecuteRegionLowering::matchAndRewrite(ExecuteRegionOp op,
 
   auto &region = op.getRegion();
   rewriter.setInsertionPointToEnd(condBlock);
-  rewriter.create<cf::BranchOp>(loc, &region.front());
+  cf::BranchOp::create(rewriter, loc, &region.front());
 
   for (Block &block : region) {
     if (auto terminator = dyn_cast<scf::YieldOp>(block.getTerminator())) {
       ValueRange terminatorOperands = terminator->getOperands();
       rewriter.setInsertionPointToEnd(&block);
-      rewriter.create<cf::BranchOp>(loc, remainingOpsBlock, terminatorOperands);
+      cf::BranchOp::create(rewriter, loc, remainingOpsBlock, terminatorOperands);
       rewriter.eraseOp(terminator);
     }
   }
@@ -503,7 +503,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
   for (auto [iv, lower, upper, step] :
        llvm::zip(parallelOp.getInductionVars(), parallelOp.getLowerBound(),
                  parallelOp.getUpperBound(), parallelOp.getStep())) {
-    ForOp forOp = rewriter.create<ForOp>(loc, lower, upper, step, iterArgs);
+    ForOp forOp = ForOp::create(rewriter, loc, lower, upper, step, iterArgs);
     ivs.push_back(forOp.getInductionVar());
     auto iterRange = forOp.getRegionIterArgs();
     iterArgs.assign(iterRange.begin(), iterRange.end());
@@ -517,7 +517,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
       // A loop is constructed with an empty "yield" terminator if there are
       // no results.
       rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
-      rewriter.create<scf::YieldOp>(loc, forOp.getResults());
+      scf::YieldOp::create(rewriter, loc, forOp.getResults());
     }
 
     rewriter.setInsertionPointToStart(forOp.getBody());
@@ -549,7 +549,7 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
   // has been already created in loop construction).
   if (!yieldOperands.empty()) {
     rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
-    rewriter.create<scf::YieldOp>(loc, yieldOperands);
+    scf::YieldOp::create(rewriter, loc, yieldOperands);
   }
 
   rewriter.replaceOp(parallelOp, loopResults);
@@ -575,7 +575,7 @@ LogicalResult WhileLowering::matchAndRewrite(WhileOp whileOp,
 
   // Branch to the "before" region.
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(loc, before, whileOp.getInits());
+  cf::BranchOp::create(rewriter, loc, before, whileOp.getInits());
 
   // Replace terminators with branches. Assuming bodies are SESE, which holds
   // given only the patterns from this file, we only need to look at the last
@@ -625,7 +625,7 @@ DoWhileLowering::matchAndRewrite(WhileOp whileOp,
 
   // Branch to the "before" region.
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(whileOp.getLoc(), before, whileOp.getInits());
+  cf::BranchOp::create(rewriter, whileOp.getLoc(), before, whileOp.getInits());
 
   // Loop around the "before" region based on condition.
   rewriter.setInsertionPointToEnd(before);
@@ -695,10 +695,10 @@ IndexSwitchLowering::matchAndRewrite(IndexSwitchOp op,
   SmallVector<ValueRange> caseOperands(caseSuccessors.size(), {});
 
   // Cast switch index to integer case value.
-  Value caseValue = rewriter.create<arith::IndexCastOp>(
+  Value caseValue = arith::IndexCastOp::create(rewriter,
       op.getLoc(), rewriter.getI32Type(), op.getArg());
 
-  rewriter.create<cf::SwitchOp>(
+  cf::SwitchOp::create(rewriter,
       op.getLoc(), caseValue, *defaultBlock, ValueRange(),
       rewriter.getDenseI32ArrayAttr(caseValues), caseSuccessors, caseOperands);
   rewriter.replaceOp(op, continueBlock->getArguments());
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index dcb48529a74e6..37134fe9d992a 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -91,7 +91,7 @@ createVariablesForResults(T op, const TypeConverter *typeConverter,
     Type varType = emitc::LValueType::get(resultType);
     emitc::OpaqueAttr noInit = emitc::OpaqueAttr::get(context, "");
     emitc::VariableOp var =
-        rewriter.create<emitc::VariableOp>(loc, varType, noInit);
+        emitc::VariableOp::create(rewriter, loc, varType, noInit);
     resultVariables.push_back(var);
   }
 
@@ -103,14 +103,14 @@ createVariablesForResults(T op, const TypeConverter *typeConverter,
 static void assignValues(ValueRange values, ValueRange variables,
                          ConversionPatternRewriter &rewriter, Location loc) {
   for (auto [value, var] : llvm::zip(values, variables))
-    rewriter.create<emitc::AssignOp>(loc, var, value);
+    emitc::AssignOp::create(rewriter, loc, var, value);
 }
 
 SmallVector<Value> loadValues(const SmallVector<Value> &variables,
                               PatternRewriter &rewriter, Location loc) {
   return llvm::map_to_vector<>(variables, [&](Value var) {
     Type type = cast<emitc::LValueType>(var.getType()).getValueType();
-    return rewriter.create<emitc::LoadOp>(loc, type, var).getResult();
+    return emitc::LoadOp::create(rewriter, loc, type, var).getResult();
   });
 }
 
@@ -129,7 +129,7 @@ static LogicalResult lowerYield(Operation *op, ValueRange resultVariables,
 
   assignValues(yieldOperands, resultVariables, rewriter, loc);
 
-  rewriter.create<emitc::YieldOp>(loc);
+  emitc::YieldOp::create(rewriter, loc);
   rewriter.eraseOp(yield);
 
   return success();
@@ -164,7 +164,7 @@ ForLowering::matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
 
   assignValues(adaptor.getInitArgs(), resultVariables, rewriter, loc);
 
-  emitc::ForOp loweredFor = rewriter.create<emitc::ForOp>(
+  emitc::ForOp loweredFor = emitc::ForOp::create(rewriter,
       loc, adaptor.getLowerBound(), adaptor.getUpperBound(), adaptor.getStep());
 
   Block *loweredBody = loweredFor.getBody();
@@ -257,7 +257,7 @@ IfLowering::matchAndRewrite(IfOp ifOp, OpAdaptor adaptor,
   bool hasElseBlock = !elseRegion.empty();
 
   auto loweredIf =
-      rewriter.create<emitc::IfOp>(loc, adaptor.getCondition(), false, false);
+      emitc::IfOp::create(rewriter, loc, adaptor.getCondition(), false, false);
 
   Region &loweredThenRegion = loweredIf.getThenRegion();
   auto result = lowerRegion(thenRegion, loweredThenRegion);
@@ -304,7 +304,7 @@ LogicalResult IndexSwitchOpLowering::matchAndRewrite(
                                        "create variables for results failed");
   }
 
-  auto loweredSwitch = rewriter.create<emitc::SwitchOp>(
+  auto loweredSwitch = emitc::SwitchOp::create(rewriter,
       loc, adaptor.getArg(), adaptor.getCases(), indexSwitchOp.getNumCases());
 
   // Lowering all case regions.
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index 844e66e927c4d..bfdb5a688b286 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -84,7 +84,7 @@ static Operation::operand_range getUpperBoundOperands(AffineForOp forOp) {
 // Get a Value that corresponds to the loop step.  If the step is an attribute,
 // materialize a corresponding constant using builder.
 static Value getOrCreateStep(AffineForOp forOp, OpBuilder &builder) {
-  return builder.create<arith::ConstantIndexOp>(forOp.getLoc(),
+  return arith::ConstantIndexOp::create(builder, forOp.getLoc(),
                                                 forOp.getStepAsInt());
 }
 
@@ -190,12 +190,12 @@ AffineLoopToGpuConverter::collectBounds(AffineForOp forOp, unsigned numLoops) {
       return std::nullopt;
     }
 
-    Value range = builder.create<arith::SubIOp>(currentLoop.getLoc(),
+    Value range = arith::SubIOp::create(builder, currentLoop.getLoc(),
                                                 upperBound, lowerBound);
     Value step = getOrCreateStep(currentLoop, builder);
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
       range =
-          builder.create<arith::CeilDivSIOp>(currentLoop.getLoc(), range, step);
+          arith::CeilDivSIOp::create(builder, currentLoop.getLoc(), range, step);
     dims.push_back(range);
 
     lbs.push_back(lowerBound);
@@ -221,7 +221,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
   // no loop mapped to a specific dimension, use constant "1" as its size.
   Value constOne =
       (numBlockDims < 3 || numThreadDims < 3)
-          ? builder.create<arith::ConstantIndexOp>(rootForOp.getLoc(), 1)
+          ? arith::ConstantIndexOp::create(builder, rootForOp.getLoc(), 1)
           : nullptr;
   Value gridSizeX = numBlockDims > 0 ? dims[0] : constOne;
   Value gridSizeY = numBlockDims > 1 ? dims[1] : constOne;
@@ -232,7 +232,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
 
   // Create a launch op and move the body region of the innermost loop to the
   // launch op.
-  auto launchOp = builder.create<gpu::LaunchOp>(
+  auto launchOp = gpu::LaunchOp::create(builder,
       rootForOp.getLoc(), gridSizeX, gridSizeY, gridSizeZ, blockSizeX,
       blockSizeY, blockSizeZ);
 
@@ -244,7 +244,7 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
   Location terminatorLoc = terminator.getLoc();
   terminator.erase();
   builder.setInsertionPointToEnd(innermostForOp.getBody());
-  builder.create<gpu::TerminatorOp>(terminatorLoc, TypeRange());
+  gpu::TerminatorOp::create(builder, terminatorLoc, TypeRange());
   launchOp.getBody().front().getOperations().splice(
       launchOp.getBody().front().begin(),
       innermostForOp.getBody()->getOperations());
@@ -263,10 +263,10 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
             : getDim3Value(launchOp.getThreadIds(), en.index() - numBlockDims);
     Value step = steps[en.index()];
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
-      id = builder.create<arith::MulIOp>(rootForOp.getLoc(), step, id);
+      id = arith::MulIOp::create(builder, rootForOp.getLoc(), step, id);
 
     Value ivReplacement =
-        builder.create<arith::AddIOp>(rootForOp.getLoc(), *lbArgumentIt, id);
+        arith::AddIOp::create(builder, rootForOp.getLoc(), *lbArgumentIt, id);
     en.value().replaceAllUsesWith(ivReplacement);
     std::advance(lbArgumentIt, 1);
     std::advance(stepArgumentIt, 1);
@@ -319,7 +319,7 @@ static Value deriveStaticUpperBound(Value upperBound,
   if (auto minOp = upperBound.getDefiningOp<AffineMinOp>()) {
     for (const AffineExpr &result : minOp.getMap().getResults()) {
       if (auto constExpr = dyn_cast<AffineConstantExpr>(result)) {
-        return rewriter.create<arith::ConstantIndexOp>(minOp.getLoc(),
+        return arith::ConstantIndexOp::create(rewriter, minOp.getLoc(),
                                                        constExpr.getValue());
       }
     }
@@ -344,7 +344,7 @@ static Value deriveStaticUpperBound(Value upperBound,
         if ((lhs.value() < 0) != (rhs.value() < 0))
           return {};
 
-        return rewriter.create<arith::ConstantIndexOp>(
+        return arith::ConstantIndexOp::create(rewriter,
             multiplyOp.getLoc(), lhs.value() * rhs.value());
       }
   }
@@ -422,7 +422,7 @@ static LogicalResult processParallelLoop(
     if (launchIndependent(val))
       return val;
     if (auto constOp = val.getDefiningOp<arith::ConstantOp>())
-      return rewriter.create<arith::ConstantOp>(constOp.getLoc(),
+      return arith::ConstantOp::create(rewriter, constOp.getLoc(),
                                                 constOp.getValue());
     return {};
   };
@@ -453,7 +453,7 @@ static LogicalResult processParallelLoop(
           1, 2,
           rewriter.getAffineDimExpr(0) * rewriter.getAffineSymbolExpr(0) +
               rewriter.getAffineSymbolExpr(1));
-      newIndex = rewriter.create<AffineApplyOp>(
+      newIndex = AffineApplyOp::create(rewriter,
           loc, annotation.getMap().compose(lowerAndStep),
           ValueRange{operand, ensureLaunchIndependent(step),
                      ensureLaunchIndependent(lowerBound)});
@@ -498,7 +498,7 @@ static LogicalResult processParallelLoop(
               1, 2,
               ((rewriter.getAffineDimExpr(0) - rewriter.getAffineSymbolExpr(0))
                    .ceilDiv(rewriter.getAffineSymbolExpr(1))));
-          Value launchBound = rewriter.create<AffineApplyOp>(
+          Value launchBound = AffineApplyOp::create(rewriter,
               loc, annotation.getBound().compose(stepMap),
               ValueRange{
                   ensureLaunchIndependent(
@@ -517,10 +517,10 @@ static LogicalResult processParallelLoop(
         if (!boundIsPrecise) {
           // We are using an approximation, create a surrounding conditional.
           Value originalBound = std::get<3>(config);
-          arith::CmpIOp pred = rewriter.create<arith::CmpIOp>(
+          arith::CmpIOp pred = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::slt, newIndex,
               cloningMap.lookupOrDefault(originalBound));
-          scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, pred, false);
+          scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, pred, false);
           rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
           // Put a sentinel into the worklist so we know when to pop out of the
           // if body again. We use the launchOp here, as that cannot be part of
@@ -530,7 +530,7 @@ static LogicalResult processParallelLoop(
       }
     } else {
       // Create a sequential for loop.
-      auto loopOp = rewriter.create<scf::ForOp>(
+      auto loopOp = scf::ForOp::create(rewriter,
           loc, cloningMap.lookupOrDefault(lowerBound),
           cloningMap.lookupOrDefault(upperBound),
           cloningMap.lookupOrDefault(step));
@@ -608,12 +608,12 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
   // sizes. Those will be refined later as we discover them from mappings.
   Location loc = parallelOp.getLoc();
   Value constantOne =
-      rewriter.create<arith::ConstantIndexOp>(parallelOp.getLoc(), 1);
-  gpu::LaunchOp launchOp = rewriter.create<gpu::LaunchOp>(
+      arith::ConstantIndexOp::create(rewriter, parallelOp.getLoc(), 1);
+  gpu::LaunchOp launchOp = gpu::LaunchOp::create(rewriter,
       parallelOp.getLoc(), constantOne, constantOne, constantOne, constantOne,
       constantOne, constantOne);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
-  rewriter.create<gpu::TerminatorOp>(loc);
+  gpu::TerminatorOp::create(rewriter, loc);
   rewriter.setInsertionPointToStart(&launchOp.getBody().front());
 
   IRMapping cloningMap;
@@ -667,7 +667,7 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
       if (externalValues.size())
         return failure();
       // Replace by gpu.all_reduce.
-      auto gpuRedOp = rewriter.create<gpu::AllReduceOp>(loc, newValue);
+      auto gpuRedOp = gpu::AllReduceOp::create(rewriter, loc, newValue);
       cloningMap.map(parentLoop->getResult(0), gpuRedOp.getResult());
       // Copy region.
       rewriter.inlineRegionBefore(reduceOp.getRegion(0), gpuRedOp.getRegion(),
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 584ac2f11b670..7b4c37d44824d 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -187,7 +187,7 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
            scf::ReduceOp reduce, int64_t reductionIndex, Attribute initValue) {
   OpBuilder::InsertionGuard guard(builder);
   Type type = reduce.getOperands()[reductionIndex].getType();
-  auto decl = builder.create<omp::DeclareReductionOp>(reduce.getLoc(),
+  auto decl = omp::DeclareReductionOp::create(builder, reduce.getLoc(),
                                                       "__scf_reduction", type);
   symbolTable.insert(decl);
 
@@ -196,8 +196,8 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
                       {reduce.getOperands()[reductionIndex].getLoc()});
   builder.setInsertionPointToEnd(&decl.getInitializerRegion().back());
   Value init =
-      builder.create<LLVM::ConstantOp>(reduce.getLoc(), type, initValue);
-  builder.create<omp::YieldOp>(reduce.getLoc(), init);
+      LLVM::ConstantOp::create(builder, reduce.getLoc(), type, initValue);
+  omp::YieldOp::create(builder, reduce.getLoc(), init);
 
   Operation *terminator =
       &reduce.getReductions()[reductionIndex].front().back();
@@ -227,12 +227,12 @@ static omp::DeclareReductionOp addAtomicRMW(OpBuilder &builder,
                       {reduceOperandLoc, reduceOperandLoc});
   Block *atomicBlock = &decl.getAtomicReductionRegion().back();
   builder.setInsertionPointToEnd(atomicBlock);
-  Value loaded = builder.create<LLVM::LoadOp>(reduce.getLoc(), decl.getType(),
+  Value loaded = LLVM::LoadOp::create(builder, reduce.getLoc(), decl.getType(),
                                               atomicBlock->getArgument(1));
-  builder.create<LLVM::AtomicRMWOp>(reduce.getLoc(), atomicKind,
+  LLVM::AtomicRMWOp::create(builder, reduce.getLoc(), atomicKind,
                                     atomicBlock->getArgument(0), loaded,
                                     LLVM::AtomicOrdering::monotonic);
-  builder.create<omp::YieldOp>(reduce.getLoc(), ArrayRef<Value>());
+  omp::YieldOp::create(builder, reduce.getLoc(), ArrayRef<Value>());
   return decl;
 }
 
@@ -380,7 +380,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     // Allocate reduction variables. Make sure the we don't overflow the stack
     // with local `alloca`s by saving and restoring the stack pointer.
     Location loc = parallelOp.getLoc();
-    Value one = rewriter.create<LLVM::ConstantOp>(
+    Value one = LLVM::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerType(64), rewriter.getI64IntegerAttr(1));
     SmallVector<Value> reductionVariables;
     reductionVariables.reserve(parallelOp.getNumReductions());
@@ -391,8 +391,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
              "cannot create a reduction variable if the type is not an LLVM "
              "pointer element");
       Value storage =
-          rewriter.create<LLVM::AllocaOp>(loc, ptrType, init.getType(), one, 0);
-      rewriter.create<LLVM::StoreOp>(loc, init, storage);
+          LLVM::AllocaOp::create(rewriter, loc, ptrType, init.getType(), one, 0);
+      LLVM::StoreOp::create(rewriter, loc, init, storage);
       reductionVariables.push_back(storage);
     }
 
@@ -411,7 +411,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
       assert(redRegion.hasOneBlock() &&
              "expect reduction region to have one block");
       Value pvtRedVar = parallelOp.getRegion().addArgument(x.getType(), loc);
-      Value pvtRedVal = rewriter.create<LLVM::LoadOp>(reduce.getLoc(),
+      Value pvtRedVal = LLVM::LoadOp::create(rewriter, reduce.getLoc(),
                                                       rD.getType(), pvtRedVar);
       // Make a copy of the reduction combiner region in the body
       mlir::OpBuilder builder(rewriter.getContext());
@@ -427,7 +427,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           assert(yieldOp && yieldOp.getResults().size() == 1 &&
                  "expect YieldOp in reduction region to return one result");
           Value redVal = yieldOp.getResults()[0];
-          rewriter.create<LLVM::StoreOp>(loc, redVal, pvtRedVar);
+          LLVM::StoreOp::create(rewriter, loc, redVal, pvtRedVar);
           rewriter.eraseOp(yieldOp);
           break;
         }
@@ -437,11 +437,11 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
 
     Value numThreadsVar;
     if (numThreads > 0) {
-      numThreadsVar = rewriter.create<LLVM::ConstantOp>(
+      numThreadsVar = LLVM::ConstantOp::create(rewriter,
           loc, rewriter.getI32IntegerAttr(numThreads));
     }
     // Create the parallel wrapper.
-    auto ompParallel = rewriter.create<omp::ParallelOp>(
+    auto ompParallel = omp::ParallelOp::create(rewriter,
         loc,
         /* allocate_vars = */ llvm::SmallVector<Value>{},
         /* allocator_vars = */ llvm::SmallVector<Value>{},
@@ -464,7 +464,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
       {
         OpBuilder::InsertionGuard allocaGuard(rewriter);
         // Create worksharing loop wrapper.
-        auto wsloopOp = rewriter.create<omp::WsloopOp>(parallelOp.getLoc());
+        auto wsloopOp = omp::WsloopOp::create(rewriter, parallelOp.getLoc());
         if (!reductionVariables.empty()) {
           wsloopOp.setReductionSymsAttr(
               ArrayAttr::get(rewriter.getContext(), reductionSyms));
@@ -476,7 +476,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           wsloopOp.setReductionByref(
               DenseBoolArrayAttr::get(rewriter.getContext(), reductionByRef));
         }
-        rewriter.create<omp::TerminatorOp>(loc); // omp.parallel terminator.
+        omp::TerminatorOp::create(rewriter, loc); // omp.parallel terminator.
 
         // The wrapper's entry block arguments will define the reduction
         // variables.
@@ -490,7 +490,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
                                               parallelOp.getLoc()));
 
         // Create loop nest and populate region with contents of scf.parallel.
-        auto loopOp = rewriter.create<omp::LoopNestOp>(
+        auto loopOp = omp::LoopNestOp::create(rewriter,
             parallelOp.getLoc(), parallelOp.getLowerBound(),
             parallelOp.getUpperBound(), parallelOp.getStep());
 
@@ -511,13 +511,13 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
             rewriter.splitBlock(&loopOpEntryBlock, loopOpEntryBlock.begin());
         rewriter.setInsertionPointToStart(&loopOpEntryBlock);
 
-        auto scope = rewriter.create<memref::AllocaScopeOp>(parallelOp.getLoc(),
+        auto scope = memref::AllocaScopeOp::create(rewriter, parallelOp.getLoc(),
                                                             TypeRange());
-        rewriter.create<omp::YieldOp>(loc, ValueRange());
+        omp::YieldOp::create(rewriter, loc, ValueRange());
         Block *scopeBlock = rewriter.createBlock(&scope.getBodyRegion());
         rewriter.mergeBlocks(ops, scopeBlock);
         rewriter.setInsertionPointToEnd(&*scope.getBodyRegion().begin());
-        rewriter.create<memref::AllocaScopeReturnOp>(loc, ValueRange());
+        memref::AllocaScopeReturnOp::create(rewriter, loc, ValueRange());
       }
     }
 
@@ -526,7 +526,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     results.reserve(reductionVariables.size());
     for (auto [variable, type] :
          llvm::zip(reductionVariables, parallelOp.getResultTypes())) {
-      Value res = rewriter.create<LLVM::LoadOp>(loc, type, variable);
+      Value res = LLVM::LoadOp::create(rewriter, loc, type, variable);
       results.push_back(res);
     }
     rewriter.replaceOp(parallelOp, results);
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index 78d13278fef53..b494fa98f1362 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -71,12 +71,12 @@ void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp,
     auto pointerType =
         spirv::PointerType::get(convertedType, spirv::StorageClass::Function);
     rewriter.setInsertionPoint(newOp);
-    auto alloc = rewriter.create<spirv::VariableOp>(
+    auto alloc = spirv::VariableOp::create(rewriter,
         loc, pointerType, spirv::StorageClass::Function,
         /*initializer=*/nullptr);
     allocas.push_back(alloc);
     rewriter.setInsertionPointAfter(newOp);
-    Value loadResult = rewriter.create<spirv::LoadOp>(loc, alloc);
+    Value loadResult = spirv::LoadOp::create(rewriter, loc, alloc);
     resultValue.push_back(loadResult);
   }
   rewriter.replaceOp(scfOp, resultValue);
@@ -135,7 +135,7 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // a single back edge from the continue to header block, and a single exit
     // from header to merge.
     auto loc = forOp.getLoc();
-    auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
+    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     OpBuilder::InsertionGuard guard(rewriter);
@@ -172,15 +172,15 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     args.append(adaptor.getInitArgs().begin(), adaptor.getInitArgs().end());
     // Branch into it from the entry.
     rewriter.setInsertionPointToEnd(&(loopOp.getBody().front()));
-    rewriter.create<spirv::BranchOp>(loc, header, args);
+    spirv::BranchOp::create(rewriter, loc, header, args);
 
     // Generate the rest of the loop header.
     rewriter.setInsertionPointToEnd(header);
     auto *mergeBlock = loopOp.getMergeBlock();
-    auto cmpOp = rewriter.create<spirv::SLessThanOp>(
+    auto cmpOp = spirv::SLessThanOp::create(rewriter,
         loc, rewriter.getI1Type(), newIndVar, adaptor.getUpperBound());
 
-    rewriter.create<spirv::BranchConditionalOp>(
+    spirv::BranchConditionalOp::create(rewriter,
         loc, cmpOp, body, ArrayRef<Value>(), mergeBlock, ArrayRef<Value>());
 
     // Generate instructions to increment the step of the induction variable and
@@ -189,9 +189,9 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     rewriter.setInsertionPointToEnd(continueBlock);
 
     // Add the step to the induction variable and branch to the header.
-    Value updatedIndVar = rewriter.create<spirv::IAddOp>(
+    Value updatedIndVar = spirv::IAddOp::create(rewriter,
         loc, newIndVar.getType(), newIndVar, adaptor.getStep());
-    rewriter.create<spirv::BranchOp>(loc, header, updatedIndVar);
+    spirv::BranchOp::create(rewriter, loc, header, updatedIndVar);
 
     // Infer the return types from the init operands. Vector type may get
     // converted to CooperativeMatrix or to Vector type, to avoid having complex
@@ -238,10 +238,10 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     // Create `spirv.selection` operation, selection header block and merge
     // block.
     auto selectionOp =
-        rewriter.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
+        spirv::SelectionOp::create(rewriter, loc, spirv::SelectionControl::None);
     auto *mergeBlock = rewriter.createBlock(&selectionOp.getBody(),
                                             selectionOp.getBody().end());
-    rewriter.create<spirv::MergeOp>(loc);
+    spirv::MergeOp::create(rewriter, loc);
 
     OpBuilder::InsertionGuard guard(rewriter);
     auto *selectionHeaderBlock =
@@ -251,7 +251,7 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     auto &thenRegion = ifOp.getThenRegion();
     auto *thenBlock = &thenRegion.front();
     rewriter.setInsertionPointToEnd(&thenRegion.back());
-    rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+    spirv::BranchOp::create(rewriter, loc, mergeBlock);
     rewriter.inlineRegionBefore(thenRegion, mergeBlock);
 
     auto *elseBlock = mergeBlock;
@@ -261,13 +261,13 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
       auto &elseRegion = ifOp.getElseRegion();
       elseBlock = &elseRegion.front();
       rewriter.setInsertionPointToEnd(&elseRegion.back());
-      rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+      spirv::BranchOp::create(rewriter, loc, mergeBlock);
       rewriter.inlineRegionBefore(elseRegion, mergeBlock);
     }
 
     // Create a `spirv.BranchConditional` operation for selection header block.
     rewriter.setInsertionPointToEnd(selectionHeaderBlock);
-    rewriter.create<spirv::BranchConditionalOp>(loc, adaptor.getCondition(),
+    spirv::BranchConditionalOp::create(rewriter, loc, adaptor.getCondition(),
                                                 thenBlock, ArrayRef<Value>(),
                                                 elseBlock, ArrayRef<Value>());
 
@@ -310,7 +310,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
 
       auto loc = terminatorOp.getLoc();
       for (unsigned i = 0, e = operands.size(); i < e; i++)
-        rewriter.create<spirv::StoreOp>(loc, allocas[i], operands[i]);
+        spirv::StoreOp::create(rewriter, loc, allocas[i], operands[i]);
       if (isa<spirv::LoopOp>(parent)) {
         // For loops we also need to update the branch jumping back to the
         // header.
@@ -319,7 +319,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
         SmallVector<Value, 8> args(br.getBlockArguments());
         args.append(operands.begin(), operands.end());
         rewriter.setInsertionPoint(br);
-        rewriter.create<spirv::BranchOp>(terminatorOp.getLoc(), br.getTarget(),
+        spirv::BranchOp::create(rewriter, terminatorOp.getLoc(), br.getTarget(),
                                          args);
         rewriter.eraseOp(br);
       }
@@ -340,7 +340,7 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
   matchAndRewrite(scf::WhileOp whileOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto loc = whileOp.getLoc();
-    auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
+    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     Region &beforeRegion = whileOp.getBefore();
@@ -382,7 +382,7 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
     // Jump from the loop entry block to the loop header block.
     rewriter.setInsertionPointToEnd(&entryBlock);
-    rewriter.create<spirv::BranchOp>(loc, &beforeBlock, adaptor.getInits());
+    spirv::BranchOp::create(rewriter, loc, &beforeBlock, adaptor.getInits());
 
     auto condLoc = cond.getLoc();
 
@@ -403,18 +403,18 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
       // Create local variables before the scf.while op.
       rewriter.setInsertionPoint(loopOp);
-      auto alloc = rewriter.create<spirv::VariableOp>(
+      auto alloc = spirv::VariableOp::create(rewriter,
           condLoc, pointerType, spirv::StorageClass::Function,
           /*initializer=*/nullptr);
 
       // Load the final result values after the scf.while op.
       rewriter.setInsertionPointAfter(loopOp);
-      auto loadResult = rewriter.create<spirv::LoadOp>(condLoc, alloc);
+      auto loadResult = spirv::LoadOp::create(rewriter, condLoc, alloc);
       resultValues[i] = loadResult;
 
       // Store the current iteration's result value.
       rewriter.setInsertionPointToEnd(&beforeBlock);
-      rewriter.create<spirv::StoreOp>(condLoc, alloc, res);
+      spirv::StoreOp::create(rewriter, condLoc, alloc, res);
     }
 
     rewriter.setInsertionPointToEnd(&beforeBlock);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index d7ae9f0e94fe8..934bb0d1c37fd 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -68,7 +68,7 @@ static unsigned calculateGlobalIndex(spirv::GlobalVariableOp op) {
 /// Copies the given number of bytes from src to dst pointers.
 static void copy(Location loc, Value dst, Value src, Value size,
                  OpBuilder &builder) {
-  builder.create<LLVM::MemcpyOp>(loc, dst, src, size, /*isVolatile=*/false);
+  LLVM::MemcpyOp::create(builder, loc, dst, src, size, /*isVolatile=*/false);
 }
 
 /// Encodes the binding and descriptor set numbers into a new symbolic name.
@@ -194,7 +194,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
     if (!kernelFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(module.getBody());
-      kernelFunc = rewriter.create<LLVM::LLVMFuncOp>(
+      kernelFunc = LLVM::LLVMFuncOp::create(rewriter,
           rewriter.getUnknownLoc(), newKernelFuncName,
           LLVM::LLVMFunctionType::get(LLVM::LLVMVoidType::get(context),
                                       ArrayRef<Type>()));
@@ -245,7 +245,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       if (!dstGlobal) {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
-        dstGlobal = rewriter.create<LLVM::GlobalOp>(
+        dstGlobal = LLVM::GlobalOp::create(rewriter,
             loc, dstGlobalType,
             /*isConstant=*/false, LLVM::Linkage::Linkonce, name, Attribute(),
             /*alignment=*/0);
@@ -255,7 +255,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       // Copy the data from src operand pointer to dst global variable. Save
       // src, dst and size so that we can copy data back after emulating the
       // kernel call.
-      Value dst = rewriter.create<LLVM::AddressOfOp>(
+      Value dst = LLVM::AddressOfOp::create(rewriter,
           loc, typeConverter->convertType(spirvGlobal.getType()),
           dstGlobal.getSymName());
       copy(loc, dst, src, sizeBytes, rewriter);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 1d92b5d5562b5..3eee444725f74 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -94,12 +94,12 @@ static IntegerAttr minusOneIntegerAttribute(Type type, Builder builder) {
 static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
                                       PatternRewriter &rewriter) {
   if (isa<VectorType>(srcType)) {
-    return rewriter.create<LLVM::ConstantOp>(
+    return LLVM::ConstantOp::create(rewriter,
         loc, dstType,
         SplatElementsAttr::get(cast<ShapedType>(srcType),
                                minusOneIntegerAttribute(srcType, rewriter)));
   }
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, dstType, minusOneIntegerAttribute(srcType, rewriter));
 }
 
@@ -108,13 +108,13 @@ static Value createFPConstant(Location loc, Type srcType, Type dstType,
                               PatternRewriter &rewriter, double value) {
   if (auto vecType = dyn_cast<VectorType>(srcType)) {
     auto floatType = cast<FloatType>(vecType.getElementType());
-    return rewriter.create<LLVM::ConstantOp>(
+    return LLVM::ConstantOp::create(rewriter,
         loc, dstType,
         SplatElementsAttr::get(vecType,
                                rewriter.getFloatAttr(floatType, value)));
   }
   auto floatType = cast<FloatType>(srcType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, dstType, rewriter.getFloatAttr(floatType, value));
 }
 
@@ -134,13 +134,13 @@ static Value optionallyTruncateOrExtend(Location loc, Value value,
                                : getBitWidth(srcType);
 
   if (valueBitWidth < targetBitWidth)
-    return rewriter.create<LLVM::ZExtOp>(loc, llvmType, value);
+    return LLVM::ZExtOp::create(rewriter, loc, llvmType, value);
   // If the bit widths of `Count` and `Offset` are greater than the bit width
   // of the target type, they are truncated. Truncation is safe since `Count`
   // and `Offset` must be no more than 64 for op behaviour to be defined. Hence,
   // both values can be expressed in 8 bits.
   if (valueBitWidth > targetBitWidth)
-    return rewriter.create<LLVM::TruncOp>(loc, llvmType, value);
+    return LLVM::TruncOp::create(rewriter, loc, llvmType, value);
   return value;
 }
 
@@ -151,11 +151,11 @@ static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
   auto vectorType = VectorType::get(numElements, toBroadcast.getType());
   auto llvmVectorType = typeConverter.convertType(vectorType);
   auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
-  Value broadcasted = rewriter.create<LLVM::PoisonOp>(loc, llvmVectorType);
+  Value broadcasted = LLVM::PoisonOp::create(rewriter, loc, llvmVectorType);
   for (unsigned i = 0; i < numElements; ++i) {
-    auto index = rewriter.create<LLVM::ConstantOp>(
+    auto index = LLVM::ConstantOp::create(rewriter,
         loc, llvmI32Type, rewriter.getI32IntegerAttr(i));
-    broadcasted = rewriter.create<LLVM::InsertElementOp>(
+    broadcasted = LLVM::InsertElementOp::create(rewriter,
         loc, llvmVectorType, broadcasted, toBroadcast, index);
   }
   return broadcasted;
@@ -217,7 +217,7 @@ static Type convertStructTypePacked(spirv::StructType type,
 /// Creates LLVM dialect constant with the given value.
 static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
                                  unsigned value) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, IntegerType::get(rewriter.getContext(), 32),
       rewriter.getIntegerAttr(rewriter.getI32Type(), value));
 }
@@ -322,7 +322,7 @@ class AccessChainPattern : public SPIRVToLLVMConversion<spirv::AccessChainOp> {
     auto llvmIndexType = getTypeConverter()->convertType(indexType);
     if (!llvmIndexType)
       return rewriter.notifyMatchFailure(op, "type conversion failed");
-    Value zero = rewriter.create<LLVM::ConstantOp>(
+    Value zero = LLVM::ConstantOp::create(rewriter,
         op.getLoc(), llvmIndexType, rewriter.getIntegerAttr(indexType, 0));
     indices.insert(indices.begin(), zero);
 
@@ -375,20 +375,20 @@ class BitFieldInsertPattern
     // Create a mask with bits set outside [Offset, Offset + Count - 1].
     Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
     Value maskShiftedByCount =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
-    Value negated = rewriter.create<LLVM::XOrOp>(loc, dstType,
+        LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
+    Value negated = LLVM::XOrOp::create(rewriter, loc, dstType,
                                                  maskShiftedByCount, minusOne);
     Value maskShiftedByCountAndOffset =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, negated, offset);
-    Value mask = rewriter.create<LLVM::XOrOp>(
+        LLVM::ShlOp::create(rewriter, loc, dstType, negated, offset);
+    Value mask = LLVM::XOrOp::create(rewriter,
         loc, dstType, maskShiftedByCountAndOffset, minusOne);
 
     // Extract unchanged bits from the `Base`  that are outside of
     // [Offset, Offset + Count - 1]. Then `or` with shifted `Insert`.
     Value baseAndMask =
-        rewriter.create<LLVM::AndOp>(loc, dstType, op.getBase(), mask);
+        LLVM::AndOp::create(rewriter, loc, dstType, op.getBase(), mask);
     Value insertShiftedByOffset =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, op.getInsert(), offset);
+        LLVM::ShlOp::create(rewriter, loc, dstType, op.getInsert(), offset);
     rewriter.replaceOpWithNewOp<LLVM::OrOp>(op, dstType, baseAndMask,
                                             insertShiftedByOffset);
     return success();
@@ -470,23 +470,23 @@ class BitFieldSExtractPattern
     auto baseSize = rewriter.getIntegerAttr(integerType, getBitWidth(srcType));
     Value size =
         isa<VectorType>(srcType)
-            ? rewriter.create<LLVM::ConstantOp>(
+            ? LLVM::ConstantOp::create(rewriter,
                   loc, dstType,
                   SplatElementsAttr::get(cast<ShapedType>(srcType), baseSize))
-            : rewriter.create<LLVM::ConstantOp>(loc, dstType, baseSize);
+            : LLVM::ConstantOp::create(rewriter, loc, dstType, baseSize);
 
     // Shift `Base` left by [sizeof(Base) - (Count + Offset)], so that the bit
     // at Offset + Count - 1 is the most significant bit now.
     Value countPlusOffset =
-        rewriter.create<LLVM::AddOp>(loc, dstType, count, offset);
+        LLVM::AddOp::create(rewriter, loc, dstType, count, offset);
     Value amountToShiftLeft =
-        rewriter.create<LLVM::SubOp>(loc, dstType, size, countPlusOffset);
-    Value baseShiftedLeft = rewriter.create<LLVM::ShlOp>(
+        LLVM::SubOp::create(rewriter, loc, dstType, size, countPlusOffset);
+    Value baseShiftedLeft = LLVM::ShlOp::create(rewriter,
         loc, dstType, op.getBase(), amountToShiftLeft);
 
     // Shift the result right, filling the bits with the sign bit.
     Value amountToShiftRight =
-        rewriter.create<LLVM::AddOp>(loc, dstType, offset, amountToShiftLeft);
+        LLVM::AddOp::create(rewriter, loc, dstType, offset, amountToShiftLeft);
     rewriter.replaceOpWithNewOp<LLVM::AShrOp>(op, dstType, baseShiftedLeft,
                                               amountToShiftRight);
     return success();
@@ -516,13 +516,13 @@ class BitFieldUExtractPattern
     // Create a mask with bits set at [0, Count - 1].
     Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
     Value maskShiftedByCount =
-        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
-    Value mask = rewriter.create<LLVM::XOrOp>(loc, dstType, maskShiftedByCount,
+        LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
+    Value mask = LLVM::XOrOp::create(rewriter, loc, dstType, maskShiftedByCount,
                                               minusOne);
 
     // Shift `Base` by `Offset` and apply the mask on it.
     Value shiftedBase =
-        rewriter.create<LLVM::LShrOp>(loc, dstType, op.getBase(), offset);
+        LLVM::LShrOp::create(rewriter, loc, dstType, op.getBase(), offset);
     rewriter.replaceOpWithNewOp<LLVM::AndOp>(op, dstType, shiftedBase, mask);
     return success();
   }
@@ -694,7 +694,7 @@ class ExecutionModePattern
     auto structType = LLVM::LLVMStructType::getLiteral(context, fields);
 
     // Create `llvm.mlir.global` with initializer region containing one block.
-    auto global = rewriter.create<LLVM::GlobalOp>(
+    auto global = LLVM::GlobalOp::create(rewriter,
         UnknownLoc::get(context), structType, /*isConstant=*/true,
         LLVM::Linkage::External, executionModeInfoName, Attribute(),
         /*alignment=*/0);
@@ -704,22 +704,22 @@ class ExecutionModePattern
 
     // Initialize the struct and set the execution mode value.
     rewriter.setInsertionPointToStart(block);
-    Value structValue = rewriter.create<LLVM::PoisonOp>(loc, structType);
-    Value executionMode = rewriter.create<LLVM::ConstantOp>(
+    Value structValue = LLVM::PoisonOp::create(rewriter, loc, structType);
+    Value executionMode = LLVM::ConstantOp::create(rewriter,
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
-    structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
+    structValue = LLVM::InsertValueOp::create(rewriter, loc, structValue,
                                                        executionMode, 0);
 
     // Insert extra operands if they exist into execution mode info struct.
     for (unsigned i = 0, e = values.size(); i < e; ++i) {
       auto attr = values.getValue()[i];
-      Value entry = rewriter.create<LLVM::ConstantOp>(loc, llvmI32Type, attr);
-      structValue = rewriter.create<LLVM::InsertValueOp>(
+      Value entry = LLVM::ConstantOp::create(rewriter, loc, llvmI32Type, attr);
+      structValue = LLVM::InsertValueOp::create(rewriter,
           loc, structValue, entry, ArrayRef<int64_t>({1, i}));
     }
-    rewriter.create<LLVM::ReturnOp>(loc, ArrayRef<Value>({structValue}));
+    LLVM::ReturnOp::create(rewriter, loc, ArrayRef<Value>({structValue}));
     rewriter.eraseOp(op);
     return success();
   }
@@ -913,7 +913,7 @@ class InverseSqrtPattern
 
     Location loc = op.getLoc();
     Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
-    Value sqrt = rewriter.create<LLVM::SqrtOp>(loc, dstType, op.getOperand());
+    Value sqrt = LLVM::SqrtOp::create(rewriter, loc, dstType, op.getOperand());
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(op, dstType, one, sqrt);
     return success();
   }
@@ -973,10 +973,10 @@ class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
     IntegerAttr minusOne = minusOneIntegerAttribute(srcType, rewriter);
     auto mask =
         isa<VectorType>(srcType)
-            ? rewriter.create<LLVM::ConstantOp>(
+            ? LLVM::ConstantOp::create(rewriter,
                   loc, dstType,
                   SplatElementsAttr::get(cast<VectorType>(srcType), minusOne))
-            : rewriter.create<LLVM::ConstantOp>(loc, dstType, minusOne);
+            : LLVM::ConstantOp::create(rewriter, loc, dstType, minusOne);
     rewriter.template replaceOpWithNewOp<LLVM::XOrOp>(notOp, dstType,
                                                       notOp.getOperand(), mask);
     return success();
@@ -1034,7 +1034,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
     return func;
 
   OpBuilder b(symbolTable->getRegion(0));
-  func = b.create<LLVM::LLVMFuncOp>(
+  func = LLVM::LLVMFuncOp::create(b,
       symbolTable->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes));
   func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
@@ -1047,7 +1047,7 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
 static LLVM::CallOp createSPIRVBuiltinCall(Location loc, OpBuilder &builder,
                                            LLVM::LLVMFuncOp func,
                                            ValueRange args) {
-  auto call = builder.create<LLVM::CallOp>(loc, func, args);
+  auto call = LLVM::CallOp::create(builder, loc, func, args);
   call.setCConv(func.getCConv());
   call.setConvergentAttr(func.getConvergentAttr());
   call.setNoUnwindAttr(func.getNoUnwindAttr());
@@ -1078,11 +1078,11 @@ class ControlBarrierPattern : public SPIRVToLLVMConversion<BarrierOpTy> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, {i32, i32, i32}, voidTy);
 
     Location loc = controlBarrierOp->getLoc();
-    Value execution = rewriter.create<LLVM::ConstantOp>(
+    Value execution = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value memory = rewriter.create<LLVM::ConstantOp>(
+    Value memory = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getMemoryScope()));
-    Value semantics = rewriter.create<LLVM::ConstantOp>(
+    Value semantics = LLVM::ConstantOp::create(rewriter,
         loc, i32, static_cast<int32_t>(adaptor.getMemorySemantics()));
 
     auto call = createSPIRVBuiltinCall(loc, rewriter, func,
@@ -1255,9 +1255,9 @@ class GroupReducePattern : public SPIRVToLLVMConversion<ReduceOp> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, paramTypes, retTy);
 
     Location loc = op.getLoc();
-    Value scope = rewriter.create<LLVM::ConstantOp>(
+    Value scope = LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value groupOp = rewriter.create<LLVM::ConstantOp>(
+    Value groupOp = LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, static_cast<int32_t>(adaptor.getGroupOperation()));
     SmallVector<Value> operands{scope, groupOp};
     operands.append(adaptor.getOperands().begin(), adaptor.getOperands().end());
@@ -1368,7 +1368,7 @@ class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
       return failure();
     Block *headerBlock = loopOp.getHeaderBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
-    rewriter.create<LLVM::BrOp>(loc, brOp.getBlockArguments(), headerBlock);
+    LLVM::BrOp::create(rewriter, loc, brOp.getBlockArguments(), headerBlock);
     rewriter.eraseBlock(entryBlock);
 
     // Branch from merge block to end block.
@@ -1376,7 +1376,7 @@ class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
     Operation *terminator = mergeBlock->getTerminator();
     ValueRange terminatorOperands = terminator->getOperands();
     rewriter.setInsertionPointToEnd(mergeBlock);
-    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, endBlock);
+    LLVM::BrOp::create(rewriter, loc, terminatorOperands, endBlock);
 
     rewriter.inlineRegionBefore(loopOp.getBody(), endBlock);
     rewriter.replaceOp(loopOp, endBlock->getArguments());
@@ -1434,13 +1434,13 @@ class SelectionPattern : public SPIRVToLLVMConversion<spirv::SelectionOp> {
     Operation *terminator = mergeBlock->getTerminator();
     ValueRange terminatorOperands = terminator->getOperands();
     rewriter.setInsertionPointToEnd(mergeBlock);
-    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, continueBlock);
+    LLVM::BrOp::create(rewriter, loc, terminatorOperands, continueBlock);
 
     // Link current block to `true` and `false` blocks within the selection.
     Block *trueBlock = condBrOp.getTrueBlock();
     Block *falseBlock = condBrOp.getFalseBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
-    rewriter.create<LLVM::CondBrOp>(loc, condBrOp.getCondition(), trueBlock,
+    LLVM::CondBrOp::create(rewriter, loc, condBrOp.getCondition(), trueBlock,
                                     condBrOp.getTrueTargetOperands(),
                                     falseBlock,
                                     condBrOp.getFalseTargetOperands());
@@ -1521,8 +1521,8 @@ class TanPattern : public SPIRVToLLVMConversion<spirv::GLTanOp> {
       return rewriter.notifyMatchFailure(tanOp, "type conversion failed");
 
     Location loc = tanOp.getLoc();
-    Value sin = rewriter.create<LLVM::SinOp>(loc, dstType, tanOp.getOperand());
-    Value cos = rewriter.create<LLVM::CosOp>(loc, dstType, tanOp.getOperand());
+    Value sin = LLVM::SinOp::create(rewriter, loc, dstType, tanOp.getOperand());
+    Value cos = LLVM::CosOp::create(rewriter, loc, dstType, tanOp.getOperand());
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanOp, dstType, sin, cos);
     return success();
   }
@@ -1549,13 +1549,13 @@ class TanhPattern : public SPIRVToLLVMConversion<spirv::GLTanhOp> {
     Location loc = tanhOp.getLoc();
     Value two = createFPConstant(loc, srcType, dstType, rewriter, 2.0);
     Value multiplied =
-        rewriter.create<LLVM::FMulOp>(loc, dstType, two, tanhOp.getOperand());
-    Value exponential = rewriter.create<LLVM::ExpOp>(loc, dstType, multiplied);
+        LLVM::FMulOp::create(rewriter, loc, dstType, two, tanhOp.getOperand());
+    Value exponential = LLVM::ExpOp::create(rewriter, loc, dstType, multiplied);
     Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
     Value numerator =
-        rewriter.create<LLVM::FSubOp>(loc, dstType, exponential, one);
+        LLVM::FSubOp::create(rewriter, loc, dstType, exponential, one);
     Value denominator =
-        rewriter.create<LLVM::FAddOp>(loc, dstType, exponential, one);
+        LLVM::FAddOp::create(rewriter, loc, dstType, exponential, one);
     rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanhOp, dstType, numerator,
                                               denominator);
     return success();
@@ -1594,8 +1594,8 @@ class VariablePattern : public SPIRVToLLVMConversion<spirv::VariableOp> {
     if (!elementType)
       return rewriter.notifyMatchFailure(varOp, "type conversion failed");
     Value allocated =
-        rewriter.create<LLVM::AllocaOp>(loc, dstType, elementType, size);
-    rewriter.create<LLVM::StoreOp>(loc, adaptor.getInitializer(), allocated);
+        LLVM::AllocaOp::create(rewriter, loc, dstType, elementType, size);
+    LLVM::StoreOp::create(rewriter, loc, adaptor.getInitializer(), allocated);
     rewriter.replaceOp(varOp, allocated);
     return success();
   }
@@ -1656,7 +1656,7 @@ class FuncConversionPattern : public SPIRVToLLVMConversion<spirv::FuncOp> {
     // Create a new `LLVMFuncOp`
     Location loc = funcOp.getLoc();
     StringRef name = funcOp.getName();
-    auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(loc, name, llvmType);
+    auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter, loc, name, llvmType);
 
     // Convert SPIR-V Function Control to equivalent LLVM function attribute
     MLIRContext *context = funcOp.getContext();
@@ -1710,7 +1710,7 @@ class ModuleConversionPattern : public SPIRVToLLVMConversion<spirv::ModuleOp> {
                   ConversionPatternRewriter &rewriter) const override {
 
     auto newModuleOp =
-        rewriter.create<ModuleOp>(spvModuleOp.getLoc(), spvModuleOp.getName());
+        ModuleOp::create(rewriter, spvModuleOp.getLoc(), spvModuleOp.getName());
     rewriter.inlineRegionBefore(spvModuleOp.getRegion(), newModuleOp.getBody());
 
     // Remove the terminator block that was automatically added by builder
@@ -1751,7 +1751,7 @@ class VectorShufflePattern
     auto componentsArray = components.getValue();
     auto *context = rewriter.getContext();
     auto llvmI32Type = IntegerType::get(context, 32);
-    Value targetOp = rewriter.create<LLVM::PoisonOp>(loc, dstType);
+    Value targetOp = LLVM::PoisonOp::create(rewriter, loc, dstType);
     for (unsigned i = 0; i < componentsArray.size(); i++) {
       if (!isa<IntegerAttr>(componentsArray[i]))
         return op.emitError("unable to support non-constant component");
@@ -1767,15 +1767,15 @@ class VectorShufflePattern
         baseVector = vector2;
       }
 
-      Value dstIndex = rewriter.create<LLVM::ConstantOp>(
+      Value dstIndex = LLVM::ConstantOp::create(rewriter,
           loc, llvmI32Type, rewriter.getIntegerAttr(rewriter.getI32Type(), i));
-      Value index = rewriter.create<LLVM::ConstantOp>(
+      Value index = LLVM::ConstantOp::create(rewriter,
           loc, llvmI32Type,
           rewriter.getIntegerAttr(rewriter.getI32Type(), indexVal - offsetVal));
 
-      auto extractOp = rewriter.create<LLVM::ExtractElementOp>(
+      auto extractOp = LLVM::ExtractElementOp::create(rewriter,
           loc, scalarType, baseVector, index);
-      targetOp = rewriter.create<LLVM::InsertElementOp>(loc, dstType, targetOp,
+      targetOp = LLVM::InsertElementOp::create(rewriter, loc, dstType, targetOp,
                                                         extractOp, dstIndex);
     }
     rewriter.replaceOp(op, targetOp);
diff --git a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
index da9ad3dd67328..245e60b04ec31 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
@@ -32,7 +32,7 @@ class ConvertCstrRequireOp : public OpRewritePattern<shape::CstrRequireOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(shape::CstrRequireOp op,
                                 PatternRewriter &rewriter) const override {
-    rewriter.create<cf::AssertOp>(op.getLoc(), op.getPred(), op.getMsgAttr());
+    cf::AssertOp::create(rewriter, op.getLoc(), op.getPred(), op.getMsgAttr());
     rewriter.replaceOpWithNewOp<shape::ConstWitnessOp>(op, true);
     return success();
   }
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index bbe1490137bf8..58cffa6119d93 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -82,19 +82,19 @@ struct BroadcastOpConverter : public OpConversionPattern<BroadcastOp> {
 // number of extent tensors and shifted offsets into them.
 Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
                         ValueRange rankDiffs, Value outputDimension) {
-  Value one = lb.create<arith::ConstantIndexOp>(1);
+  Value one = arith::ConstantIndexOp::create(lb, 1);
   Value broadcastedDim = one;
   for (auto tup : llvm::zip(extentTensors, rankDiffs)) {
     Value shape = std::get<0>(tup);
     Value rankDiff = std::get<1>(tup);
-    Value outOfBounds = lb.create<arith::CmpIOp>(arith::CmpIPredicate::ult,
+    Value outOfBounds = arith::CmpIOp::create(lb, arith::CmpIPredicate::ult,
                                                  outputDimension, rankDiff);
     Type indexTy = lb.getIndexType();
     broadcastedDim =
-        lb.create<IfOp>(
+        IfOp::create(lb,
               outOfBounds,
               [&](OpBuilder &b, Location loc) {
-                b.create<scf::YieldOp>(loc, broadcastedDim);
+                scf::YieldOp::create(b, loc, broadcastedDim);
               },
               [&](OpBuilder &b, Location loc) {
                 // The broadcasting logic is:
@@ -104,17 +104,17 @@ Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
                 // - otherwise, take the extent as-is.
                 // Note that this logic remains correct in the presence
                 // of dimensions of zero extent.
-                Value lesserRankOperandDimension = b.create<arith::SubIOp>(
+                Value lesserRankOperandDimension = arith::SubIOp::create(b,
                     loc, indexTy, outputDimension, rankDiff);
-                Value lesserRankOperandExtent = b.create<tensor::ExtractOp>(
+                Value lesserRankOperandExtent = tensor::ExtractOp::create(b,
                     loc, shape, ValueRange{lesserRankOperandDimension});
 
                 Value dimIsOne =
-                    b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+                    arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
                                             lesserRankOperandExtent, one);
-                Value dim = b.create<arith::SelectOp>(
+                Value dim = arith::SelectOp::create(b,
                     loc, dimIsOne, broadcastedDim, lesserRankOperandExtent);
-                b.create<scf::YieldOp>(loc, dim);
+                scf::YieldOp::create(b, loc, dim);
               })
             .getResult(0);
   }
@@ -133,7 +133,7 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
   auto loc = op.getLoc();
   ImplicitLocOpBuilder lb(loc, rewriter);
 
-  Value zero = lb.create<arith::ConstantIndexOp>(0);
+  Value zero = arith::ConstantIndexOp::create(lb, 0);
   Type indexTy = lb.getIndexType();
 
   // Save all the ranks for bounds checking. Because this is a tensor
@@ -141,31 +141,31 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
   // dimension in the tensor.
   SmallVector<Value> ranks, rankDiffs;
   llvm::append_range(ranks, llvm::map_range(adaptor.getShapes(), [&](Value v) {
-                       return lb.create<tensor::DimOp>(v, zero);
+                       return tensor::DimOp::create(lb, v, zero);
                      }));
 
   // Find the maximum rank
   Value maxRank = ranks.front();
   for (Value v : llvm::drop_begin(ranks, 1)) {
-    maxRank = lb.create<arith::MaxUIOp>(v, maxRank);
+    maxRank = arith::MaxUIOp::create(lb, v, maxRank);
   }
 
   // Calculate the difference of ranks and the maximum rank for later offsets.
   llvm::append_range(rankDiffs, llvm::map_range(ranks, [&](Value v) {
-                       return lb.create<arith::SubIOp>(indexTy, maxRank, v);
+                       return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
-  Value replacement = lb.create<tensor::GenerateOp>(
+  Value replacement = tensor::GenerateOp::create(lb,
       getExtentTensorType(lb.getContext()), ValueRange{maxRank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value broadcastedDim =
             getBroadcastedDim(ImplicitLocOpBuilder(loc, b), adaptor.getShapes(),
                               rankDiffs, args[0]);
 
-        b.create<tensor::YieldOp>(loc, broadcastedDim);
+        tensor::YieldOp::create(b, loc, broadcastedDim);
       });
   if (replacement.getType() != op.getType())
-    replacement = lb.create<tensor::CastOp>(op.getType(), replacement);
+    replacement = tensor::CastOp::create(lb, op.getType(), replacement);
   rewriter.replaceOp(op, replacement);
   return success();
 }
@@ -194,12 +194,12 @@ LogicalResult ConstShapeOpConverter::matchAndRewrite(
   SmallVector<Value, 4> extentOperands;
   for (auto extent : op.getShape()) {
     extentOperands.push_back(
-        rewriter.create<arith::ConstantIndexOp>(loc, extent.getLimitedValue()));
+        arith::ConstantIndexOp::create(rewriter, loc, extent.getLimitedValue()));
   }
   Type resultTy =
       RankedTensorType::get({op.getShape().size()}, rewriter.getIndexType());
   Value tensor =
-      rewriter.create<tensor::FromElementsOp>(loc, resultTy, extentOperands);
+      tensor::FromElementsOp::create(rewriter, loc, resultTy, extentOperands);
   rewriter.replaceOpWithNewOp<tensor::CastOp>(op, resultTy, tensor);
   return success();
 }
@@ -245,8 +245,8 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
 
   auto loc = op.getLoc();
   ImplicitLocOpBuilder lb(loc, rewriter);
-  Value zero = lb.create<arith::ConstantIndexOp>(0);
-  Value one = lb.create<arith::ConstantIndexOp>(1);
+  Value zero = arith::ConstantIndexOp::create(lb, 0);
+  Value one = arith::ConstantIndexOp::create(lb, 1);
   Type indexTy = lb.getIndexType();
 
   // Save all the ranks for bounds checking. Because this is a tensor
@@ -254,25 +254,25 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
   // dimension in the tensor.
   SmallVector<Value> ranks, rankDiffs;
   llvm::append_range(ranks, llvm::map_range(adaptor.getShapes(), [&](Value v) {
-                       return lb.create<tensor::DimOp>(v, zero);
+                       return tensor::DimOp::create(lb, v, zero);
                      }));
 
   // Find the maximum rank
   Value maxRank = ranks.front();
   for (Value v : llvm::drop_begin(ranks, 1)) {
-    maxRank = lb.create<arith::MaxUIOp>(v, maxRank);
+    maxRank = arith::MaxUIOp::create(lb, v, maxRank);
   }
 
   // Calculate the difference of ranks and the maximum rank for later offsets.
   llvm::append_range(rankDiffs, llvm::map_range(ranks, [&](Value v) {
-                       return lb.create<arith::SubIOp>(indexTy, maxRank, v);
+                       return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
   Type i1Ty = rewriter.getI1Type();
   Value trueVal =
-      rewriter.create<arith::ConstantOp>(loc, i1Ty, rewriter.getBoolAttr(true));
+      arith::ConstantOp::create(rewriter, loc, i1Ty, rewriter.getBoolAttr(true));
 
-  auto reduceResult = lb.create<ForOp>(
+  auto reduceResult = ForOp::create(lb,
       loc, zero, maxRank, one, ValueRange{trueVal},
       [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
         // Find a non-1 dim, if it exists. Note that the first part of this
@@ -285,38 +285,38 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
         for (auto tup : llvm::zip(adaptor.getShapes(), rankDiffs)) {
           Value shape, rankDiff;
           std::tie(shape, rankDiff) = tup;
-          Value outOfBounds = b.create<arith::CmpIOp>(
+          Value outOfBounds = arith::CmpIOp::create(b,
               loc, arith::CmpIPredicate::ult, iv, rankDiff);
           broadcastable =
-              b.create<IfOp>(
+              IfOp::create(b,
                    loc, outOfBounds,
                    [&](OpBuilder &b, Location loc) {
                      // Non existent dimensions are always broadcastable
-                     b.create<scf::YieldOp>(loc, broadcastable);
+                     scf::YieldOp::create(b, loc, broadcastable);
                    },
                    [&](OpBuilder &b, Location loc) {
                      // Every value needs to be either 1, or the same non-1
                      // value to be broadcastable in this dim.
                      Value operandDimension =
-                         b.create<arith::SubIOp>(loc, indexTy, iv, rankDiff);
-                     Value dimensionExtent = b.create<tensor::ExtractOp>(
+                         arith::SubIOp::create(b, loc, indexTy, iv, rankDiff);
+                     Value dimensionExtent = tensor::ExtractOp::create(b,
                          loc, shape, ValueRange{operandDimension});
 
-                     Value equalOne = b.create<arith::CmpIOp>(
+                     Value equalOne = arith::CmpIOp::create(b,
                          loc, arith::CmpIPredicate::eq, dimensionExtent, one);
-                     Value equalBroadcasted = b.create<arith::CmpIOp>(
+                     Value equalBroadcasted = arith::CmpIOp::create(b,
                          loc, arith::CmpIPredicate::eq, dimensionExtent,
                          broadcastedDim);
-                     Value result = b.create<arith::AndIOp>(
+                     Value result = arith::AndIOp::create(b,
                          loc, broadcastable,
-                         b.create<arith::OrIOp>(loc, equalOne,
+                         arith::OrIOp::create(b, loc, equalOne,
                                                 equalBroadcasted));
-                     b.create<scf::YieldOp>(loc, result);
+                     scf::YieldOp::create(b, loc, result);
                    })
                   .getResult(0);
         }
 
-        b.create<scf::YieldOp>(loc, broadcastable);
+        scf::YieldOp::create(b, loc, broadcastable);
       });
 
   rewriter.replaceOp(op, reduceResult.getResults().front());
@@ -339,7 +339,7 @@ DimOpConverter::matchAndRewrite(DimOp op, OpAdaptor adaptor,
   // Lower to dim(X, i) to get_extent(shape_of(X), i) and rely on further
   // lowerings. This can be further optimized if needed to avoid intermediate
   // steps.
-  auto shapeOf = rewriter.create<shape::ShapeOfOp>(op.getLoc(), op.getValue());
+  auto shapeOf = shape::ShapeOfOp::create(rewriter, op.getLoc(), op.getValue());
   rewriter.replaceOpWithNewOp<shape::GetExtentOp>(op, op.getType(), shapeOf,
                                                   op.getIndex());
   return success();
@@ -421,16 +421,16 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
 
   auto loc = op.getLoc();
 
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Value one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
   Type indexTy = rewriter.getIndexType();
   Value rank =
-      rewriter.create<tensor::DimOp>(loc, indexTy, adaptor.getShape(), zero);
+      tensor::DimOp::create(rewriter, loc, indexTy, adaptor.getShape(), zero);
 
-  auto loop = rewriter.create<scf::ForOp>(
+  auto loop = scf::ForOp::create(rewriter,
       loc, zero, rank, one, op.getInitVals(),
       [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
-        Value extent = b.create<tensor::ExtractOp>(loc, adaptor.getShape(), iv);
+        Value extent = tensor::ExtractOp::create(b, loc, adaptor.getShape(), iv);
 
         SmallVector<Value, 2> mappedValues{iv, extent};
         mappedValues.append(args.begin(), args.end());
@@ -444,7 +444,7 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
         SmallVector<Value, 2> mappedResults;
         for (auto result : reduceBody->getTerminator()->getOperands())
           mappedResults.push_back(mapping.lookup(result));
-        b.create<scf::YieldOp>(loc, mappedResults);
+        scf::YieldOp::create(b, loc, mappedResults);
       });
 
   rewriter.replaceOp(op, loop.getResults());
@@ -507,43 +507,43 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, OpAdaptor adaptor,
 
   auto loc = op.getLoc();
   Type indexTy = rewriter.getIndexType();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   Value firstShape = adaptor.getShapes().front();
   Value firstRank =
-      rewriter.create<tensor::DimOp>(loc, indexTy, firstShape, zero);
+      tensor::DimOp::create(rewriter, loc, indexTy, firstShape, zero);
   Value result = nullptr;
   // Generate a linear sequence of compares, all with firstShape as lhs.
   for (Value shape : adaptor.getShapes().drop_front(1)) {
-    Value rank = rewriter.create<tensor::DimOp>(loc, indexTy, shape, zero);
-    Value eqRank = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value rank = tensor::DimOp::create(rewriter, loc, indexTy, shape, zero);
+    Value eqRank = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                   firstRank, rank);
-    auto same = rewriter.create<IfOp>(
+    auto same = IfOp::create(rewriter,
         loc, eqRank,
         [&](OpBuilder &b, Location loc) {
-          Value one = b.create<arith::ConstantIndexOp>(loc, 1);
+          Value one = arith::ConstantIndexOp::create(b, loc, 1);
           Value init =
-              b.create<arith::ConstantOp>(loc, i1Ty, b.getBoolAttr(true));
-          auto loop = b.create<scf::ForOp>(
+              arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(true));
+          auto loop = scf::ForOp::create(b,
               loc, zero, firstRank, one, ValueRange{init},
               [&](OpBuilder &b, Location nestedLoc, Value iv, ValueRange args) {
                 Value conj = args[0];
                 Value lhsExtent =
-                    b.create<tensor::ExtractOp>(loc, firstShape, iv);
-                Value rhsExtent = b.create<tensor::ExtractOp>(loc, shape, iv);
-                Value eqExtent = b.create<arith::CmpIOp>(
+                    tensor::ExtractOp::create(b, loc, firstShape, iv);
+                Value rhsExtent = tensor::ExtractOp::create(b, loc, shape, iv);
+                Value eqExtent = arith::CmpIOp::create(b,
                     loc, arith::CmpIPredicate::eq, lhsExtent, rhsExtent);
-                Value conjNext = b.create<arith::AndIOp>(loc, conj, eqExtent);
-                b.create<scf::YieldOp>(loc, ValueRange({conjNext}));
+                Value conjNext = arith::AndIOp::create(b, loc, conj, eqExtent);
+                scf::YieldOp::create(b, loc, ValueRange({conjNext}));
               });
-          b.create<scf::YieldOp>(loc, loop.getResults());
+          scf::YieldOp::create(b, loc, loop.getResults());
         },
         [&](OpBuilder &b, Location loc) {
           Value result =
-              b.create<arith::ConstantOp>(loc, i1Ty, b.getBoolAttr(false));
-          b.create<scf::YieldOp>(loc, result);
+              arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(false));
+          scf::YieldOp::create(b, loc, result);
         });
     result = !result ? same.getResult(0)
-                     : rewriter.create<arith::AndIOp>(loc, result,
+                     : arith::AndIOp::create(rewriter, loc, result,
                                                       same.getResult(0));
   }
   rewriter.replaceOp(op, result);
@@ -581,17 +581,17 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
     int64_t rank = rankedTensorTy.getRank();
     for (int64_t i = 0; i < rank; i++) {
       if (rankedTensorTy.isDynamicDim(i)) {
-        Value extent = rewriter.create<tensor::DimOp>(loc, tensor, i);
+        Value extent = tensor::DimOp::create(rewriter, loc, tensor, i);
         extentValues.push_back(extent);
       } else {
-        Value extent = rewriter.create<arith::ConstantIndexOp>(
+        Value extent = arith::ConstantIndexOp::create(rewriter,
             loc, rankedTensorTy.getDimSize(i));
         extentValues.push_back(extent);
       }
     }
 
     // Materialize extent tensor.
-    Value staticExtentTensor = rewriter.create<tensor::FromElementsOp>(
+    Value staticExtentTensor = tensor::FromElementsOp::create(rewriter,
         loc, RankedTensorType::get({rank}, rewriter.getIndexType()),
         extentValues);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(),
@@ -601,13 +601,13 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
 
   // Lower to `tensor.generate` otherwise.
   auto *ctx = rewriter.getContext();
-  Value rank = rewriter.create<tensor::RankOp>(loc, tensor);
+  Value rank = tensor::RankOp::create(rewriter, loc, tensor);
   rewriter.replaceOpWithNewOp<tensor::GenerateOp>(
       op, getExtentTensorType(ctx), ValueRange{rank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value dim = args.front();
-        Value extent = b.create<tensor::DimOp>(loc, tensor, dim);
-        b.create<tensor::YieldOp>(loc, extent);
+        Value extent = tensor::DimOp::create(b, loc, tensor, dim);
+        tensor::YieldOp::create(b, loc, extent);
       });
 
   return success();
@@ -634,21 +634,21 @@ LogicalResult SplitAtOpConversion::matchAndRewrite(
     return failure();
 
   ImplicitLocOpBuilder b(op.getLoc(), rewriter);
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value rank = b.create<tensor::DimOp>(adaptor.getOperand(), zero);
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value rank = tensor::DimOp::create(b, adaptor.getOperand(), zero);
 
   // index < 0 ? index + rank : index
   Value originalIndex = adaptor.getIndex();
-  Value add = b.create<arith::AddIOp>(originalIndex, rank);
+  Value add = arith::AddIOp::create(b, originalIndex, rank);
   Value indexIsNegative =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::slt, originalIndex, zero);
-  Value index = b.create<arith::SelectOp>(indexIsNegative, add, originalIndex);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, originalIndex, zero);
+  Value index = arith::SelectOp::create(b, indexIsNegative, add, originalIndex);
 
-  Value one = b.create<arith::ConstantIndexOp>(1);
+  Value one = arith::ConstantIndexOp::create(b, 1);
   Value head =
-      b.create<tensor::ExtractSliceOp>(adaptor.getOperand(), zero, index, one);
-  Value tailSize = b.create<arith::SubIOp>(rank, index);
-  Value tail = b.create<tensor::ExtractSliceOp>(adaptor.getOperand(), index,
+      tensor::ExtractSliceOp::create(b, adaptor.getOperand(), zero, index, one);
+  Value tailSize = arith::SubIOp::create(b, rank, index);
+  Value tail = tensor::ExtractSliceOp::create(b, adaptor.getOperand(), index,
                                                 tailSize, one);
   rewriter.replaceOp(op, {head, tail});
   return success();
diff --git a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
index 2c4d27502a521..f06c3e1f7cb18 100644
--- a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
+++ b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
@@ -68,10 +68,10 @@ class TensorExtractPattern final
       // We could use the initializer directly; but certain driver compilers
       // have bugs dealing with that. So for now, use spirv.Store for
       // initialization.
-      varOp = rewriter.create<spirv::VariableOp>(loc, varType,
+      varOp = spirv::VariableOp::create(rewriter, loc, varType,
                                                  spirv::StorageClass::Function,
                                                  /*initializer=*/nullptr);
-      rewriter.create<spirv::StoreOp>(loc, varOp, adaptor.getTensor());
+      spirv::StoreOp::create(rewriter, loc, varOp, adaptor.getTensor());
     } else {
       // Need to store the value to the local variable. It's questionable
       // whether we want to support such case though.
@@ -83,7 +83,7 @@ class TensorExtractPattern final
 
     Value index = spirv::linearizeIndex(adaptor.getIndices(), strides,
                                         /*offset=*/0, indexType, loc, rewriter);
-    auto acOp = rewriter.create<spirv::AccessChainOp>(loc, varOp, index);
+    auto acOp = spirv::AccessChainOp::create(rewriter, loc, varOp, index);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(extractOp, acOp);
 
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 40ad63610e23f..44872c8d7540d 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -51,7 +51,7 @@ TypedAttr getConstantAttr(Type type, int64_t value, PatternRewriter &rewriter) {
 
 Value getConstantValue(Location loc, Type type, int64_t value,
                        PatternRewriter &rewriter) {
-  return rewriter.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(rewriter,
       loc, getConstantAttr(type, value, rewriter));
 }
 
@@ -82,41 +82,41 @@ class ApplyScaleGenericOpConverter
     Value one64 = getConstantValue(loc, i64Ty, 1, rewriter);
     Value thirtyOne32 = getConstantValue(loc, i32Ty, 31, rewriter);
 
-    Value shift32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, op.getShift());
+    Value shift32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, op.getShift());
 
     // Compute the multiplication in 64-bits then select the high / low parts.
     Value value64 = value;
     if (getElementTypeOrSelf(valueTy) != rewriter.getI64Type())
-      value64 = rewriter.create<arith::ExtSIOp>(loc, i64Ty, value);
+      value64 = arith::ExtSIOp::create(rewriter, loc, i64Ty, value);
     Value multiplier64 =
-        rewriter.create<arith::ExtSIOp>(loc, i64Ty, multiplier32);
+        arith::ExtSIOp::create(rewriter, loc, i64Ty, multiplier32);
     Value multiply64 =
-        rewriter.create<arith::MulIOp>(loc, value64, multiplier64);
+        arith::MulIOp::create(rewriter, loc, value64, multiplier64);
 
     // Apply normal rounding.
-    Value shift64 = rewriter.create<arith::ExtUIOp>(loc, i64Ty, shift32);
-    Value round = rewriter.create<arith::ShLIOp>(loc, one64, shift64);
-    round = rewriter.create<arith::ShRUIOp>(loc, round, one64);
-    multiply64 = rewriter.create<arith::AddIOp>(loc, multiply64, round);
+    Value shift64 = arith::ExtUIOp::create(rewriter, loc, i64Ty, shift32);
+    Value round = arith::ShLIOp::create(rewriter, loc, one64, shift64);
+    round = arith::ShRUIOp::create(rewriter, loc, round, one64);
+    multiply64 = arith::AddIOp::create(rewriter, loc, multiply64, round);
 
     // Apply double rounding if necessary.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       int64_t roundInt = 1 << 30;
       Value roundUp = getConstantValue(loc, i64Ty, roundInt, rewriter);
       Value roundDown = getConstantValue(loc, i64Ty, -roundInt, rewriter);
-      Value positive = rewriter.create<arith::CmpIOp>(
+      Value positive = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, value, zero);
       Value dir =
-          rewriter.create<arith::SelectOp>(loc, positive, roundUp, roundDown);
-      Value val = rewriter.create<arith::AddIOp>(loc, dir, multiply64);
-      Value valid = rewriter.create<arith::CmpIOp>(
+          arith::SelectOp::create(rewriter, loc, positive, roundUp, roundDown);
+      Value val = arith::AddIOp::create(rewriter, loc, dir, multiply64);
+      Value valid = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sgt, shift32, thirtyOne32);
       multiply64 =
-          rewriter.create<arith::SelectOp>(loc, valid, val, multiply64);
+          arith::SelectOp::create(rewriter, loc, valid, val, multiply64);
     }
 
-    Value result64 = rewriter.create<arith::ShRSIOp>(loc, multiply64, shift64);
-    Value result32 = rewriter.create<arith::TruncIOp>(loc, i32Ty, result64);
+    Value result64 = arith::ShRSIOp::create(rewriter, loc, multiply64, shift64);
+    Value result32 = arith::TruncIOp::create(rewriter, loc, i32Ty, result64);
 
     rewriter.replaceOp(op, result32);
     return success();
@@ -146,7 +146,7 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
 
     Value value32 = op.getValue();
     Value multiplier32 = op.getMultiplier();
-    Value shift32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, op.getShift());
+    Value shift32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, op.getShift());
 
     // Constants used during the scaling operation.
     Value zero32 = getConstantValue(loc, i32Ty, 0, rewriter);
@@ -158,86 +158,86 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
     // Compute the multiplication in 64-bits then select the high / low parts.
     // Grab out the high/low of the computation
     auto value64 =
-        rewriter.create<arith::MulSIExtendedOp>(loc, value32, multiplier32);
+        arith::MulSIExtendedOp::create(rewriter, loc, value32, multiplier32);
     Value low32 = value64.getLow();
     Value high32 = value64.getHigh();
 
     // Determine the direction and amount to shift the high bits.
-    Value shiftOver32 = rewriter.create<arith::CmpIOp>(
+    Value shiftOver32 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sge, shift32, thirtyTwo32);
-    Value roundHighBits = rewriter.create<arith::CmpIOp>(
+    Value roundHighBits = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, shift32, thirtyTwo32);
 
     Value shiftHighL =
-        rewriter.create<arith::SubIOp>(loc, thirtyTwo32, shift32);
+        arith::SubIOp::create(rewriter, loc, thirtyTwo32, shift32);
     Value shiftHighR =
-        rewriter.create<arith::SubIOp>(loc, shift32, thirtyTwo32);
+        arith::SubIOp::create(rewriter, loc, shift32, thirtyTwo32);
 
     shiftHighL =
-        rewriter.create<arith::SelectOp>(loc, shiftOver32, zero32, shiftHighL);
+        arith::SelectOp::create(rewriter, loc, shiftOver32, zero32, shiftHighL);
     shiftHighR =
-        rewriter.create<arith::SelectOp>(loc, shiftOver32, shiftHighR, zero32);
+        arith::SelectOp::create(rewriter, loc, shiftOver32, shiftHighR, zero32);
 
     // Conditionally perform our double round.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       Value negOne32 = getConstantValue(loc, i32Ty, -1, rewriter);
-      Value valuePositive = rewriter.create<arith::CmpIOp>(
+      Value valuePositive = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, value32, zero32);
 
       Value roundDir =
-          rewriter.create<arith::SelectOp>(loc, valuePositive, one32, negOne32);
+          arith::SelectOp::create(rewriter, loc, valuePositive, one32, negOne32);
       roundDir =
-          rewriter.create<arith::SelectOp>(loc, shiftOver32, roundDir, zero32);
+          arith::SelectOp::create(rewriter, loc, shiftOver32, roundDir, zero32);
 
-      Value shiftLow = rewriter.create<arith::ShRUIOp>(loc, low32, thirty32);
-      Value rounded = rewriter.create<arith::AddIOp>(loc, shiftLow, roundDir);
-      Value carry = rewriter.create<arith::ShRSIOp>(loc, rounded, two32);
+      Value shiftLow = arith::ShRUIOp::create(rewriter, loc, low32, thirty32);
+      Value rounded = arith::AddIOp::create(rewriter, loc, shiftLow, roundDir);
+      Value carry = arith::ShRSIOp::create(rewriter, loc, rounded, two32);
 
       Value shiftRound =
-          rewriter.create<arith::ShLIOp>(loc, roundDir, thirty32);
+          arith::ShLIOp::create(rewriter, loc, roundDir, thirty32);
 
-      low32 = rewriter.create<arith::AddIOp>(loc, low32, shiftRound);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, carry);
+      low32 = arith::AddIOp::create(rewriter, loc, low32, shiftRound);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, carry);
     }
 
     // Conditionally apply rounding in the low bits.
     {
-      Value shiftSubOne = rewriter.create<arith::SubIOp>(loc, shift32, one32);
-      Value roundBit = rewriter.create<arith::ShLIOp>(loc, one32, shiftSubOne);
-      roundBit = rewriter.create<arith::SelectOp>(loc, roundHighBits, zero32,
+      Value shiftSubOne = arith::SubIOp::create(rewriter, loc, shift32, one32);
+      Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
+      roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, zero32,
                                                   roundBit);
 
-      Value newLow32 = rewriter.create<arith::AddIOp>(loc, low32, roundBit);
-      Value wasRounded = rewriter.create<arith::CmpIOp>(
+      Value newLow32 = arith::AddIOp::create(rewriter, loc, low32, roundBit);
+      Value wasRounded = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ugt, low32, newLow32);
       low32 = newLow32;
 
-      Value rounded32 = rewriter.create<arith::ExtUIOp>(loc, i32Ty, wasRounded);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, rounded32);
+      Value rounded32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, wasRounded);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, rounded32);
     }
 
     // Conditionally apply rounding in the high bits.
     {
       Value shiftSubOne =
-          rewriter.create<arith::SubIOp>(loc, shiftHighR, one32);
-      Value roundBit = rewriter.create<arith::ShLIOp>(loc, one32, shiftSubOne);
-      roundBit = rewriter.create<arith::SelectOp>(loc, roundHighBits, roundBit,
+          arith::SubIOp::create(rewriter, loc, shiftHighR, one32);
+      Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
+      roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, roundBit,
                                                   zero32);
-      high32 = rewriter.create<arith::AddIOp>(loc, high32, roundBit);
+      high32 = arith::AddIOp::create(rewriter, loc, high32, roundBit);
     }
 
     // Combine the correct high/low bits into the final rescale result.
-    high32 = rewriter.create<arith::ShLIOp>(loc, high32, shiftHighL);
-    high32 = rewriter.create<arith::ShRSIOp>(loc, high32, shiftHighR);
-    low32 = rewriter.create<arith::ShRUIOp>(loc, low32, shift32);
-    low32 = rewriter.create<arith::SelectOp>(loc, shiftOver32, zero32, low32);
+    high32 = arith::ShLIOp::create(rewriter, loc, high32, shiftHighL);
+    high32 = arith::ShRSIOp::create(rewriter, loc, high32, shiftHighR);
+    low32 = arith::ShRUIOp::create(rewriter, loc, low32, shift32);
+    low32 = arith::SelectOp::create(rewriter, loc, shiftOver32, zero32, low32);
 
     // Apply the rounding behavior and shift to the final alignment.
-    Value result = rewriter.create<arith::AddIOp>(loc, low32, high32);
+    Value result = arith::AddIOp::create(rewriter, loc, low32, high32);
 
     // Truncate if necessary.
     if (!getElementTypeOrSelf(resultTy).isInteger(32)) {
-      result = rewriter.create<arith::TruncIOp>(loc, resultTy, result);
+      result = arith::TruncIOp::create(rewriter, loc, resultTy, result);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 2f608bbd637b4..2aec35ed9d8ca 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -70,13 +70,13 @@ materializeBinaryNanCheckIfRequired(OpTy op, PatternRewriter &rewriter,
     return result;
 
   // Unordered comparison of NaN against itself will always return true.
-  Value lhsIsNaN = rewriter.create<arith::CmpFOp>(
+  Value lhsIsNaN = arith::CmpFOp::create(rewriter,
       op.getLoc(), arith::CmpFPredicate::UNO, lhs, lhs);
-  Value rhsIsNaN = rewriter.create<arith::CmpFOp>(
+  Value rhsIsNaN = arith::CmpFOp::create(rewriter,
       op.getLoc(), arith::CmpFPredicate::UNO, rhs, rhs);
   Value rhsOrResult =
-      rewriter.create<arith::SelectOp>(op.getLoc(), lhsIsNaN, rhs, result);
-  return rewriter.create<arith::SelectOp>(op.getLoc(), rhsIsNaN, lhs,
+      arith::SelectOp::create(rewriter, op.getLoc(), lhsIsNaN, rhs, result);
+  return arith::SelectOp::create(rewriter, op.getLoc(), rhsIsNaN, lhs,
                                           rhsOrResult);
 }
 
@@ -89,38 +89,38 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
   // tosa::AbsOp
   if (isa<tosa::AbsOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::AbsFOp>(loc, resultTypes, args);
+    return math::AbsFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AbsOp>(op) && isa<IntegerType>(elementTy)) {
-    auto zero = rewriter.create<arith::ConstantOp>(
+    auto zero = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(elementTy));
-    auto neg = rewriter.create<arith::SubIOp>(loc, zero, args[0]);
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], neg);
+    auto neg = arith::SubIOp::create(rewriter, loc, zero, args[0]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], neg);
   }
 
   // tosa::AddOp
   if (isa<tosa::AddOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::AddFOp>(loc, resultTypes, args);
+    return arith::AddFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AddOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::AddIOp>(loc, resultTypes, args);
+    return arith::AddIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::SubOp
   if (isa<tosa::SubOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::SubFOp>(loc, resultTypes, args);
+    return arith::SubFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::SubOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::SubIOp>(loc, resultTypes, args);
+    return arith::SubIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::IntDivOp
   if (isa<tosa::IntDivOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::DivSIOp>(loc, resultTypes, args);
+    return arith::DivSIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ReciprocalOp
   if (isa<tosa::ReciprocalOp>(op) && isa<FloatType>(elementTy)) {
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, FloatAttr::get(elementTy, 1));
-    return rewriter.create<arith::DivFOp>(loc, resultTypes, one, args[0]);
+        arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
+    return arith::DivFOp::create(rewriter, loc, resultTypes, one, args[0]);
   }
 
   // tosa::MulOp
@@ -140,7 +140,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                                           "Cannot have shift value for float");
         return nullptr;
       }
-      return rewriter.create<arith::MulFOp>(loc, resultTypes, args[0], args[1]);
+      return arith::MulFOp::create(rewriter, loc, resultTypes, args[0], args[1]);
     }
 
     if (isa<IntegerType>(elementTy)) {
@@ -149,21 +149,21 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
       if (shift > 0) {
         auto shiftConst =
-            rewriter.create<arith::ConstantIntOp>(loc, shift, /*bitwidth=*/8);
+            arith::ConstantIntOp::create(rewriter, loc, shift, /*bitwidth=*/8);
         if (!a.getType().isInteger(32))
-          a = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), a);
+          a = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), a);
 
         if (!b.getType().isInteger(32))
-          b = rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), b);
+          b = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), b);
 
-        auto result = rewriter.create<tosa::ApplyScaleOp>(
+        auto result = tosa::ApplyScaleOp::create(rewriter,
             loc, rewriter.getI32Type(), a, b, shiftConst,
             rewriter.getStringAttr("SINGLE_ROUND"));
 
         if (elementTy.isInteger(32))
           return result;
 
-        return rewriter.create<arith::TruncIOp>(loc, elementTy, result);
+        return arith::TruncIOp::create(rewriter, loc, elementTy, result);
       }
 
       int aWidth = a.getType().getIntOrFloatBitWidth();
@@ -171,11 +171,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       int cWidth = resultTypes[0].getIntOrFloatBitWidth();
 
       if (aWidth < cWidth)
-        a = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], a);
+        a = arith::ExtSIOp::create(rewriter, loc, resultTypes[0], a);
       if (bWidth < cWidth)
-        b = rewriter.create<arith::ExtSIOp>(loc, resultTypes[0], b);
+        b = arith::ExtSIOp::create(rewriter, loc, resultTypes[0], b);
 
-      return rewriter.create<arith::MulIOp>(loc, resultTypes, a, b);
+      return arith::MulIOp::create(rewriter, loc, resultTypes, a, b);
     }
   }
 
@@ -201,13 +201,13 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     int64_t outZp = *maybeOutZp;
 
     if (isa<FloatType>(elementTy))
-      return rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
+      return arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
 
     if (isa<IntegerType>(elementTy)) {
       if (!inZp && !outZp) {
-        auto constant = rewriter.create<arith::ConstantOp>(
+        auto constant = arith::ConstantOp::create(rewriter,
             loc, IntegerAttr::get(elementTy, 0));
-        return rewriter.create<arith::SubIOp>(loc, resultTypes, constant,
+        return arith::SubIOp::create(rewriter, loc, resultTypes, constant,
                                               args[0]);
       }
 
@@ -231,60 +231,60 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       }
 
       Type intermediateType = rewriter.getIntegerType(intermediateBitWidth);
-      Value zpAddValue = rewriter.create<arith::ConstantOp>(
+      Value zpAddValue = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
 
       // The negation can be applied by doing:
       //  outputValue = inZp + outZp - inputValue
       auto ext =
-          rewriter.create<arith::ExtSIOp>(loc, intermediateType, args[0]);
-      auto sub = rewriter.create<arith::SubIOp>(loc, zpAddValue, ext);
+          arith::ExtSIOp::create(rewriter, loc, intermediateType, args[0]);
+      auto sub = arith::SubIOp::create(rewriter, loc, zpAddValue, ext);
 
       // Clamp to the negation range.
-      Value min = rewriter.create<arith::ConstantIntOp>(
+      Value min = arith::ConstantIntOp::create(rewriter,
           loc, intermediateType,
           APInt::getSignedMinValue(inputBitWidth).getSExtValue());
-      Value max = rewriter.create<arith::ConstantIntOp>(
+      Value max = arith::ConstantIntOp::create(rewriter,
           loc, intermediateType,
           APInt::getSignedMaxValue(inputBitWidth).getSExtValue());
       auto clamp = clampIntHelper(loc, sub, min, max, rewriter, false);
 
       // Truncate to the final value.
-      return rewriter.create<arith::TruncIOp>(loc, elementTy, clamp);
+      return arith::TruncIOp::create(rewriter, loc, elementTy, clamp);
     }
   }
 
   // tosa::BitwiseAndOp
   if (isa<tosa::BitwiseAndOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::AndIOp>(loc, resultTypes, args);
+    return arith::AndIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::BitwiseOrOp
   if (isa<tosa::BitwiseOrOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::OrIOp>(loc, resultTypes, args);
+    return arith::OrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::BitwiseNotOp
   if (isa<tosa::BitwiseNotOp>(op) && isa<IntegerType>(elementTy)) {
     auto allOnesAttr = rewriter.getIntegerAttr(
         elementTy, APInt::getAllOnes(elementTy.getIntOrFloatBitWidth()));
-    auto allOnes = rewriter.create<arith::ConstantOp>(loc, allOnesAttr);
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args[0], allOnes);
+    auto allOnes = arith::ConstantOp::create(rewriter, loc, allOnesAttr);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], allOnes);
   }
 
   // tosa::BitwiseXOrOp
   if (isa<tosa::BitwiseXorOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalLeftShiftOp
   if (isa<tosa::LogicalLeftShiftOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::ShLIOp>(loc, resultTypes, args);
+    return arith::ShLIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalRightShiftOp
   if (isa<tosa::LogicalRightShiftOp>(op) && isa<IntegerType>(elementTy))
-    return rewriter.create<arith::ShRUIOp>(loc, resultTypes, args);
+    return arith::ShRUIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ArithmeticRightShiftOp
   if (isa<tosa::ArithmeticRightShiftOp>(op) && isa<IntegerType>(elementTy)) {
-    auto result = rewriter.create<arith::ShRSIOp>(loc, resultTypes, args);
+    auto result = arith::ShRSIOp::create(rewriter, loc, resultTypes, args);
     auto round = cast<BoolAttr>(op->getAttr("round")).getValue();
     if (!round) {
       return result;
@@ -292,153 +292,153 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     Type i1Ty = IntegerType::get(rewriter.getContext(), /*width=*/1);
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(elementTy, 1));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 1));
     auto zero =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(elementTy, 0));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 0));
     auto i1one =
-        rewriter.create<arith::ConstantOp>(loc, IntegerAttr::get(i1Ty, 1));
+        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(i1Ty, 1));
 
     // Checking that input2 != 0
-    auto shiftValueGreaterThanZero = rewriter.create<arith::CmpIOp>(
+    auto shiftValueGreaterThanZero = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, args[1], zero);
 
     // Checking for the last bit of input1 to be 1
     auto subtract =
-        rewriter.create<arith::SubIOp>(loc, resultTypes, args[1], one);
+        arith::SubIOp::create(rewriter, loc, resultTypes, args[1], one);
     auto shifted =
-        rewriter.create<arith::ShRSIOp>(loc, resultTypes, args[0], subtract)
+        arith::ShRSIOp::create(rewriter, loc, resultTypes, args[0], subtract)
             ->getResults();
-    auto truncated = rewriter.create<arith::TruncIOp>(
+    auto truncated = arith::TruncIOp::create(rewriter,
         loc, i1Ty, shifted, ArrayRef<NamedAttribute>());
     auto isInputOdd =
-        rewriter.create<arith::AndIOp>(loc, i1Ty, truncated, i1one);
+        arith::AndIOp::create(rewriter, loc, i1Ty, truncated, i1one);
 
-    auto shouldRound = rewriter.create<arith::AndIOp>(
+    auto shouldRound = arith::AndIOp::create(rewriter,
         loc, i1Ty, shiftValueGreaterThanZero, isInputOdd);
     auto extended =
-        rewriter.create<arith::ExtUIOp>(loc, resultTypes, shouldRound);
-    return rewriter.create<arith::AddIOp>(loc, resultTypes, result, extended);
+        arith::ExtUIOp::create(rewriter, loc, resultTypes, shouldRound);
+    return arith::AddIOp::create(rewriter, loc, resultTypes, result, extended);
   }
 
   // tosa::ClzOp
   if (isa<tosa::ClzOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<math::CountLeadingZerosOp>(loc, elementTy, args[0]);
+    return math::CountLeadingZerosOp::create(rewriter, loc, elementTy, args[0]);
   }
 
   // tosa::LogicalAnd
   if (isa<tosa::LogicalAndOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::AndIOp>(loc, resultTypes, args);
+    return arith::AndIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalNot
   if (isa<tosa::LogicalNotOp>(op) && elementTy.isInteger(1)) {
-    auto one = rewriter.create<arith::ConstantOp>(
+    auto one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(elementTy, 1));
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args[0], one);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], one);
   }
 
   // tosa::LogicalOr
   if (isa<tosa::LogicalOrOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::OrIOp>(loc, resultTypes, args);
+    return arith::OrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogicalXor
   if (isa<tosa::LogicalXorOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::XOrIOp>(loc, resultTypes, args);
+    return arith::XOrIOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::PowOp
   if (isa<tosa::PowOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::PowFOp>(loc, resultTypes, args);
+    return mlir::math::PowFOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::RsqrtOp
   if (isa<tosa::RsqrtOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::RsqrtOp>(loc, resultTypes, args);
+    return mlir::math::RsqrtOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::LogOp
   if (isa<tosa::LogOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::LogOp>(loc, resultTypes, args);
+    return mlir::math::LogOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ExpOp
   if (isa<tosa::ExpOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::ExpOp>(loc, resultTypes, args);
+    return mlir::math::ExpOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::SinOp
   if (isa<tosa::SinOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::SinOp>(loc, resultTypes, args);
+    return mlir::math::SinOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::CosOp
   if (isa<tosa::CosOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::CosOp>(loc, resultTypes, args);
+    return mlir::math::CosOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::TanhOp
   if (isa<tosa::TanhOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::TanhOp>(loc, resultTypes, args);
+    return mlir::math::TanhOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ErfOp
   if (isa<tosa::ErfOp>(op) && llvm::isa<FloatType>(elementTy))
-    return rewriter.create<mlir::math::ErfOp>(loc, resultTypes, args);
+    return mlir::math::ErfOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::GreaterOp
   if (isa<tosa::GreaterOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGT,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGT,
                                           args[0], args[1]);
 
   if (isa<tosa::GreaterOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sgt,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sgt,
                                           args[0], args[1]);
 
   // tosa::GreaterEqualOp
   if (isa<tosa::GreaterEqualOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGE,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGE,
                                           args[0], args[1]);
 
   if (isa<tosa::GreaterEqualOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sge,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sge,
                                           args[0], args[1]);
 
   // tosa::EqualOp
   if (isa<tosa::EqualOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OEQ,
+    return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OEQ,
                                           args[0], args[1]);
 
   if (isa<tosa::EqualOp>(op) && elementTy.isSignlessInteger())
-    return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                           args[0], args[1]);
 
   // tosa::SelectOp
   if (isa<tosa::SelectOp>(op)) {
     elementTy = cast<ShapedType>(op->getOperand(1).getType()).getElementType();
     if (isa<FloatType>(elementTy) || isa<IntegerType>(elementTy))
-      return rewriter.create<arith::SelectOp>(loc, args[0], args[1], args[2]);
+      return arith::SelectOp::create(rewriter, loc, args[0], args[1], args[2]);
   }
 
   // tosa::MaximumOp
   if (isa<tosa::MaximumOp>(op) && isa<FloatType>(elementTy)) {
-    auto max = rewriter.create<arith::MaximumFOp>(loc, args[0], args[1]);
+    auto max = arith::MaximumFOp::create(rewriter, loc, args[0], args[1]);
     return materializeBinaryNanCheckIfRequired(llvm::cast<tosa::MaximumOp>(op),
                                                rewriter, args[0], args[1], max);
   }
 
   if (isa<tosa::MaximumOp>(op) && elementTy.isSignlessInteger()) {
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], args[1]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   // tosa::MinimumOp
   if (isa<tosa::MinimumOp>(op) && isa<FloatType>(elementTy)) {
-    auto min = rewriter.create<arith::MinimumFOp>(loc, args[0], args[1]);
+    auto min = arith::MinimumFOp::create(rewriter, loc, args[0], args[1]);
     return materializeBinaryNanCheckIfRequired(llvm::cast<tosa::MinimumOp>(op),
                                                rewriter, args[0], args[1], min);
   }
 
   if (isa<tosa::MinimumOp>(op) && elementTy.isSignlessInteger()) {
-    return rewriter.create<arith::MinSIOp>(loc, args[0], args[1]);
+    return arith::MinSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   // tosa::CeilOp
   if (isa<tosa::CeilOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::CeilOp>(loc, resultTypes, args);
+    return math::CeilOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::FloorOp
   if (isa<tosa::FloorOp>(op) && isa<FloatType>(elementTy))
-    return rewriter.create<math::FloorOp>(loc, resultTypes, args);
+    return math::FloorOp::create(rewriter, loc, resultTypes, args);
 
   // tosa::ClampOp
   if (isa<tosa::ClampOp>(op) && isa<FloatType>(elementTy)) {
@@ -449,9 +449,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                    APFloat::rmNearestTiesToEven, &losesInfo);
     maxApf.convert(cast<FloatType>(elementTy).getFloatSemantics(),
                    APFloat::rmNearestTiesToEven, &losesInfo);
-    auto min = rewriter.create<arith::ConstantOp>(
+    auto min = arith::ConstantOp::create(rewriter,
         loc, elementTy, rewriter.getFloatAttr(elementTy, minApf));
-    auto max = rewriter.create<arith::ConstantOp>(
+    auto max = arith::ConstantOp::create(rewriter,
         loc, elementTy, rewriter.getFloatAttr(elementTy, maxApf));
     auto result = clampFloatHelper(loc, args[0], min, max, rewriter);
 
@@ -478,11 +478,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     //   return init if x == NaN else result
 
     // Unordered comparison of NaN against itself will always return true.
-    Value isNaN = rewriter.create<arith::CmpFOp>(
+    Value isNaN = arith::CmpFOp::create(rewriter,
         op->getLoc(), arith::CmpFPredicate::UNO, args[0], args[0]);
     // TOSA specifies that in "ignore" NaN mode the result is "min" if the input
     // is NaN.
-    return rewriter.create<arith::SelectOp>(op->getLoc(), isNaN, min, result);
+    return arith::SelectOp::create(rewriter, op->getLoc(), isNaN, min, result);
   }
 
   if (isa<tosa::ClampOp>(op) && isa<IntegerType>(elementTy)) {
@@ -515,9 +515,9 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     min = std::min(min, maxRepresentable);
     max = std::min(max, maxRepresentable);
 
-    auto minVal = rewriter.create<arith::ConstantIntOp>(
+    auto minVal = arith::ConstantIntOp::create(rewriter,
         loc, min, intTy.getIntOrFloatBitWidth());
-    auto maxVal = rewriter.create<arith::ConstantIntOp>(
+    auto maxVal = arith::ConstantIntOp::create(rewriter,
         loc, max, intTy.getIntOrFloatBitWidth());
     return clampIntHelper(loc, args[0], minVal, maxVal, rewriter,
                           intTy.isUnsignedInteger());
@@ -526,11 +526,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   // tosa::SigmoidOp
   if (isa<tosa::SigmoidOp>(op) && isa<FloatType>(elementTy)) {
     auto one =
-        rewriter.create<arith::ConstantOp>(loc, FloatAttr::get(elementTy, 1));
-    auto negate = rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
-    auto exp = rewriter.create<mlir::math::ExpOp>(loc, resultTypes, negate);
-    auto added = rewriter.create<arith::AddFOp>(loc, resultTypes, exp, one);
-    return rewriter.create<arith::DivFOp>(loc, resultTypes, one, added);
+        arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
+    auto negate = arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
+    auto exp = mlir::math::ExpOp::create(rewriter, loc, resultTypes, negate);
+    auto added = arith::AddFOp::create(rewriter, loc, resultTypes, exp, one);
+    return arith::DivFOp::create(rewriter, loc, resultTypes, one, added);
   }
 
   // tosa::CastOp
@@ -549,20 +549,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       return args.front();
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtFOp>(loc, resultTypes, args,
+      return arith::ExtFOp::create(rewriter, loc, resultTypes, args,
                                             ArrayRef<NamedAttribute>());
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && !bitExtend)
-      return rewriter.create<arith::TruncFOp>(loc, resultTypes, args,
+      return arith::TruncFOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     // 1-bit integers need to be treated as signless.
     if (srcTy.isInteger(1) && arith::UIToFPOp::areCastCompatible(srcTy, dstTy))
-      return rewriter.create<arith::UIToFPOp>(loc, resultTypes, args,
+      return arith::UIToFPOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     if (srcTy.isInteger(1) && isa<IntegerType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtUIOp>(loc, resultTypes, args,
+      return arith::ExtUIOp::create(rewriter, loc, resultTypes, args,
                                              ArrayRef<NamedAttribute>());
 
     // Unsigned integers need an unrealized cast so that they can be passed
@@ -574,25 +574,25 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                   loc, rewriter.getIntegerType(srcTy.getIntOrFloatBitWidth()),
                   args[0])
               .getResult(0);
-      return rewriter.create<arith::UIToFPOp>(loc, resultTypes[0],
+      return arith::UIToFPOp::create(rewriter, loc, resultTypes[0],
                                               unrealizedCast);
     }
 
     // All other si-to-fp conversions should be handled by SIToFP.
     if (arith::SIToFPOp::areCastCompatible(srcTy, dstTy))
-      return rewriter.create<arith::SIToFPOp>(loc, resultTypes, args,
+      return arith::SIToFPOp::create(rewriter, loc, resultTypes, args,
                                               ArrayRef<NamedAttribute>());
 
     // Casting to boolean, floats need to only be checked as not-equal to zero.
     if (isa<FloatType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = rewriter.create<arith::ConstantOp>(
+      Value zero = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(srcTy, 0.0));
-      return rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNE,
+      return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNE,
                                             args.front(), zero);
     }
 
     if (arith::FPToSIOp::areCastCompatible(srcTy, dstTy)) {
-      auto rounded = rewriter.create<math::RoundEvenOp>(loc, args[0]);
+      auto rounded = math::RoundEvenOp::create(rewriter, loc, args[0]);
 
       const auto &fltSemantics = cast<FloatType>(srcTy).getFloatSemantics();
       // Check whether neither int min nor int max can be represented in the
@@ -601,33 +601,33 @@ static Value createLinalgBodyCalculationForElementwiseOp(
           APFloat::semanticsMaxExponent(fltSemantics)) {
         // Use cmp + select to replace infinites by int min / int max. Other
         // integral values can be represented in the integer space.
-        auto conv = rewriter.create<arith::FPToSIOp>(loc, dstTy, rounded);
-        auto posInf = rewriter.create<arith::ConstantOp>(
+        auto conv = arith::FPToSIOp::create(rewriter, loc, dstTy, rounded);
+        auto posInf = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(getElementTypeOrSelf(srcTy),
                                        APFloat::getInf(fltSemantics)));
-        auto negInf = rewriter.create<arith::ConstantOp>(
+        auto negInf = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(
                      getElementTypeOrSelf(srcTy),
                      APFloat::getInf(fltSemantics, /*Negative=*/true)));
-        auto overflow = rewriter.create<arith::CmpFOp>(
+        auto overflow = arith::CmpFOp::create(rewriter,
             loc, arith::CmpFPredicate::UEQ, rounded, posInf);
-        auto underflow = rewriter.create<arith::CmpFOp>(
+        auto underflow = arith::CmpFOp::create(rewriter,
             loc, arith::CmpFPredicate::UEQ, rounded, negInf);
-        auto intMin = rewriter.create<arith::ConstantOp>(
+        auto intMin = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIntegerAttr(
                      getElementTypeOrSelf(dstTy),
                      APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())));
-        auto intMax = rewriter.create<arith::ConstantOp>(
+        auto intMax = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIntegerAttr(
                      getElementTypeOrSelf(dstTy),
                      APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
         auto maxClamped =
-            rewriter.create<arith::SelectOp>(loc, overflow, intMax, conv);
-        return rewriter.create<arith::SelectOp>(loc, underflow, intMin,
+            arith::SelectOp::create(rewriter, loc, overflow, intMax, conv);
+        return arith::SelectOp::create(rewriter, loc, underflow, intMin,
                                                 maxClamped);
       }
 
-      auto intMinFP = rewriter.create<arith::ConstantOp>(
+      auto intMinFP = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
                    APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())
@@ -640,7 +640,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         // consists of a single leading bit. Therefore we can clamp the input
         // in the floating-point domain.
 
-        auto intMaxFP = rewriter.create<arith::ConstantOp>(
+        auto intMaxFP = arith::ConstantOp::create(rewriter,
             loc, rewriter.getFloatAttr(
                      getElementTypeOrSelf(srcTy),
                      APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
@@ -648,14 +648,14 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
         Value clamped =
             clampFloatHelper(loc, rounded, intMinFP, intMaxFP, rewriter);
-        return rewriter.create<arith::FPToSIOp>(loc, dstTy, clamped);
+        return arith::FPToSIOp::create(rewriter, loc, dstTy, clamped);
       }
 
       // Due to earlier check we know exponant range is big enough to represent
       // int min. We can therefore rely on int max + 1 being representable as
       // well because it's just int min with a positive sign. So clamp the min
       // value and compare against that to select the max int value if needed.
-      auto intMaxPlusOneFP = rewriter.create<arith::ConstantOp>(
+      auto intMaxPlusOneFP = arith::ConstantOp::create(rewriter,
           loc, rewriter.getFloatAttr(
                    getElementTypeOrSelf(srcTy),
                    static_cast<double>(
@@ -663,35 +663,35 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                            .getSExtValue()) +
                        1.0f));
 
-      auto intMax = rewriter.create<arith::ConstantOp>(
+      auto intMax = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIntegerAttr(
                    getElementTypeOrSelf(dstTy),
                    APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
       auto minClampedFP =
-          rewriter.create<arith::MaximumFOp>(loc, rounded, intMinFP);
+          arith::MaximumFOp::create(rewriter, loc, rounded, intMinFP);
       auto minClamped =
-          rewriter.create<arith::FPToSIOp>(loc, dstTy, minClampedFP);
-      auto overflow = rewriter.create<arith::CmpFOp>(
+          arith::FPToSIOp::create(rewriter, loc, dstTy, minClampedFP);
+      auto overflow = arith::CmpFOp::create(rewriter,
           loc, arith::CmpFPredicate::UGE, rounded, intMaxPlusOneFP);
-      return rewriter.create<arith::SelectOp>(loc, overflow, intMax,
+      return arith::SelectOp::create(rewriter, loc, overflow, intMax,
                                               minClamped);
     }
 
     // Casting to boolean, integers need to only be checked as not-equal to
     // zero.
     if (isa<IntegerType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = rewriter.create<arith::ConstantIntOp>(
+      Value zero = arith::ConstantIntOp::create(rewriter,
           loc, 0, srcTy.getIntOrFloatBitWidth());
-      return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+      return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
                                             args.front(), zero);
     }
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && bitExtend)
-      return rewriter.create<arith::ExtSIOp>(loc, resultTypes, args,
+      return arith::ExtSIOp::create(rewriter, loc, resultTypes, args,
                                              ArrayRef<NamedAttribute>());
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && !bitExtend) {
-      return rewriter.create<arith::TruncIOp>(loc, dstTy, args[0]);
+      return arith::TruncIOp::create(rewriter, loc, dstTy, args[0]);
     }
   }
 
@@ -710,14 +710,14 @@ static Value createIndex(PatternRewriter &rewriter, Location loc,
   auto [it, inserted] = indexPool.try_emplace(index);
   if (inserted)
     it->second =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(index));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(index));
   return it->second;
 }
 
 static Value getTensorDim(PatternRewriter &rewriter, Location loc,
                           IndexPool &indexPool, Value tensor, int64_t index) {
   auto indexValue = createIndex(rewriter, loc, indexPool, index);
-  return rewriter.create<tensor::DimOp>(loc, tensor, indexValue).getResult();
+  return tensor::DimOp::create(rewriter, loc, tensor, indexValue).getResult();
 }
 
 static OpFoldResult getOrFoldTensorDim(PatternRewriter &rewriter, Location loc,
@@ -783,7 +783,7 @@ computeTargetSize(PatternRewriter &rewriter, Location loc, IndexPool &indexPool,
   for (size_t i = 1; i < operandsWithDynamicDim.size(); i++) {
     auto nextSize =
         getTensorDim(rewriter, loc, indexPool, operandsWithDynamicDim[i], dim);
-    targetSize = rewriter.create<arith::MaxUIOp>(loc, targetSize, nextSize);
+    targetSize = arith::MaxUIOp::create(rewriter, loc, targetSize, nextSize);
   }
   return {targetSize, nullptr};
 }
@@ -838,7 +838,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
   // Check if broadcast is necessary
   auto one = createIndex(rewriter, loc, indexPool, 1);
   auto runtimeSize = getTensorDim(rewriter, loc, indexPool, operand, dim);
-  auto broadcastNecessary = rewriter.create<arith::CmpIOp>(
+  auto broadcastNecessary = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::eq, runtimeSize, one);
 
   // Emit 'then' region of 'scf.if'
@@ -855,7 +855,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                                                     operand, index);
       outputTensorShape.push_back(size);
     }
-    Value outputTensor = opBuilder.create<tensor::EmptyOp>(
+    Value outputTensor = tensor::EmptyOp::create(opBuilder,
         loc, outputTensorShape, rankedTensorType.getElementType());
 
     // Emit 'linalg.generic' op
@@ -866,7 +866,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                 getNParallelLoopsAttrs(rank),
                 [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
                   // Emit 'linalg.yield' op
-                  opBuilder.create<linalg::YieldOp>(loc, blockArgs.front());
+                  linalg::YieldOp::create(opBuilder, loc, blockArgs.front());
                 })
             .getResult(0);
 
@@ -875,16 +875,16 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
         loc, operand.getType(), resultTensor);
 
     // Emit 'scf.yield' op
-    opBuilder.create<scf::YieldOp>(loc, castResultTensor);
+    scf::YieldOp::create(opBuilder, loc, castResultTensor);
   };
 
   // Emit 'else' region of 'scf.if'
   auto emitElseRegion = [&](OpBuilder &opBuilder, Location loc) {
-    opBuilder.create<scf::YieldOp>(loc, operand);
+    scf::YieldOp::create(opBuilder, loc, operand);
   };
 
   // Emit 'scf.if' op
-  auto ifOp = rewriter.create<scf::IfOp>(loc, broadcastNecessary,
+  auto ifOp = scf::IfOp::create(rewriter, loc, broadcastNecessary,
                                          emitThenRegion, emitElseRegion);
   return ifOp.getResult(0);
 }
@@ -930,7 +930,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
   if (!resultType) {
     return rewriter.notifyMatchFailure(operation, "failed to convert type");
   }
-  Value outputTensor = rewriter.create<tensor::EmptyOp>(
+  Value outputTensor = tensor::EmptyOp::create(rewriter,
       loc, targetShape, resultType.getElementType());
 
   // Create affine maps. Input affine maps broadcast static dimensions of size
@@ -957,7 +957,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
 
   // Emit 'linalg.generic' op
   bool encounteredError = false;
-  auto linalgOp = rewriter.create<linalg::GenericOp>(
+  auto linalgOp = linalg::GenericOp::create(rewriter,
       loc, outputTensor.getType(), operands, outputTensor, affineMaps,
       getNParallelLoopsAttrs(rank),
       [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
@@ -968,7 +968,7 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
           encounteredError = true;
           return;
         }
-        opBuilder.create<linalg::YieldOp>(loc, opResult);
+        linalg::YieldOp::create(opBuilder, loc, opResult);
       });
   if (encounteredError)
     return rewriter.notifyMatchFailure(
@@ -1078,42 +1078,42 @@ static Value createLinalgBodyCalculationForReduceOp(Operation *op,
                                                     PatternRewriter &rewriter) {
   Location loc = op->getLoc();
   if (isa<tosa::ReduceSumOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::AddFOp>(loc, args);
+    return arith::AddFOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceSumOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::AddIOp>(loc, args);
+    return arith::AddIOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceProductOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MulFOp>(loc, args);
+    return arith::MulFOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceProductOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MulIOp>(loc, args);
+    return arith::MulIOp::create(rewriter, loc, args);
   }
 
   if (isa<tosa::ReduceMinOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MinimumFOp>(loc, args[0], args[1]);
+    return arith::MinimumFOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMinOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MinSIOp>(loc, args[0], args[1]);
+    return arith::MinSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMaxOp>(op) && isa<FloatType>(elementTy)) {
-    return rewriter.create<arith::MaximumFOp>(loc, args[0], args[1]);
+    return arith::MaximumFOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceMaxOp>(op) && isa<IntegerType>(elementTy)) {
-    return rewriter.create<arith::MaxSIOp>(loc, args[0], args[1]);
+    return arith::MaxSIOp::create(rewriter, loc, args[0], args[1]);
   }
 
   if (isa<tosa::ReduceAllOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::AndIOp>(loc, args);
+    return arith::AndIOp::create(rewriter, loc, args);
 
   if (isa<tosa::ReduceAnyOp>(op) && elementTy.isInteger(1))
-    return rewriter.create<arith::OrIOp>(loc, args);
+    return arith::OrIOp::create(rewriter, loc, args);
 
   return {};
 }
@@ -1139,7 +1139,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     if (axis != i) {
       reduceShape.push_back(inputTy.getDimSize(i));
       if (inputTy.isDynamicDim(i))
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
     }
   }
 
@@ -1158,7 +1158,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     return rewriter.notifyMatchFailure(
         op, "No initial value found for reduction operation");
 
-  auto fillValue = rewriter.create<arith::ConstantOp>(loc, fillValueAttr);
+  auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
   auto filledTensor = rewriter
                           .create<linalg::FillOp>(loc, ValueRange{fillValue},
                                                   ValueRange{emptyTensor})
@@ -1176,7 +1176,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
       // Additionally we have to keep track of whether we've seen any non-NaN
       // values and then do a final select based on this predicate.
       auto trueAttr = rewriter.getBoolAttr(true);
-      auto trueValue = rewriter.create<arith::ConstantOp>(loc, trueAttr);
+      auto trueValue = arith::ConstantOp::create(rewriter, loc, trueAttr);
       auto emptyBoolTensor =
           rewriter
               .create<tensor::EmptyOp>(loc, reduceShape, trueValue.getType(),
@@ -1202,7 +1202,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
   }
 
   bool didEncounterError = false;
-  linalg::LinalgOp linalgOp = rewriter.create<linalg::ReduceOp>(
+  linalg::LinalgOp linalgOp = linalg::ReduceOp::create(rewriter,
       loc, inputs, outputs, axis,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange blockArgs) {
         std::array<Value, 2> binaryArgs{
@@ -1219,21 +1219,21 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
           auto oldAllResultsNanFlagValue = blockArgs[3];
 
           // Unordered comparison of NaN against itself will always return true.
-          Value isNaN = nestedBuilder.create<arith::CmpFOp>(
+          Value isNaN = arith::CmpFOp::create(nestedBuilder,
               op->getLoc(), arith::CmpFPredicate::UNO, inputValue, inputValue);
           // If we've encountered a NaN, take the non-NaN value.
-          auto selectOp = nestedBuilder.create<arith::SelectOp>(
+          auto selectOp = arith::SelectOp::create(nestedBuilder,
               op->getLoc(), isNaN, initialValue, result);
           // Update the flag which keeps track of whether we have seen a non-NaN
           // value.
-          auto newAllResultsNanFlagValue = nestedBuilder.create<arith::AndIOp>(
+          auto newAllResultsNanFlagValue = arith::AndIOp::create(nestedBuilder,
               op->getLoc(), oldAllResultsNanFlagValue, isNaN);
           resultsToYield.push_back(selectOp);
           resultsToYield.push_back(newAllResultsNanFlagValue);
         } else {
           resultsToYield.push_back(result);
         }
-        nestedBuilder.create<linalg::YieldOp>(loc, resultsToYield);
+        linalg::YieldOp::create(nestedBuilder, loc, resultsToYield);
       });
 
   if (!didEncounterError)
@@ -1250,7 +1250,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     auto nanValueAttr = rewriter.getFloatAttr(
         elementTy,
         APFloat::getNaN(cast<FloatType>(elementTy).getFloatSemantics(), false));
-    auto nanValue = rewriter.create<arith::ConstantOp>(loc, nanValueAttr);
+    auto nanValue = arith::ConstantOp::create(rewriter, loc, nanValueAttr);
     auto emptyNanTensor =
         rewriter
             .create<tensor::EmptyOp>(loc, reduceShape,
@@ -1278,7 +1278,7 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
     ins.push_back(linalgOp->getResult(0));
     outs.push_back(finalEmptyTensor);
     auto linalgSelect =
-        rewriter.create<linalg::SelectOp>(op->getLoc(), ins, outs);
+        linalg::SelectOp::create(rewriter, op->getLoc(), ins, outs);
     linalgOp = linalgSelect;
   }
 
@@ -1350,7 +1350,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < outputTy.getRank(); i++) {
       if (outputTy.isDynamicDim(i)) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
@@ -1401,7 +1401,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value multiplierConstant;
     int64_t multiplierArg = 0;
     if (multiplierValues.size() == 1) {
-      multiplierConstant = rewriter.create<arith::ConstantOp>(
+      multiplierConstant = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32IntegerAttr(multiplierValues.front()));
     } else {
       SmallVector<AffineExpr, 2> multiplierExprs{
@@ -1409,7 +1409,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
       auto multiplierType =
           RankedTensorType::get({static_cast<int64_t>(multiplierValues.size())},
                                 rewriter.getI32Type());
-      genericInputs.push_back(rewriter.create<arith::ConstantOp>(
+      genericInputs.push_back(arith::ConstantOp::create(rewriter,
           loc, DenseIntElementsAttr::get(multiplierType, multiplierValues)));
 
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
@@ -1424,7 +1424,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value shiftConstant;
     int64_t shiftArg = 0;
     if (shiftValues.size() == 1) {
-      shiftConstant = rewriter.create<arith::ConstantOp>(
+      shiftConstant = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI8IntegerAttr(shiftValues.front()));
     } else {
       SmallVector<AffineExpr, 2> shiftExprs = {
@@ -1432,7 +1432,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
       auto shiftType =
           RankedTensorType::get({static_cast<int64_t>(shiftValues.size())},
                                 rewriter.getIntegerType(8));
-      genericInputs.push_back(rewriter.create<arith::ConstantOp>(
+      genericInputs.push_back(arith::ConstantOp::create(rewriter,
           loc, DenseIntElementsAttr::get(shiftType, shiftValues)));
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
                                             /*symbolCount=*/0, shiftExprs,
@@ -1444,11 +1444,11 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(rank));
 
     // Construct the indexing maps needed for linalg.generic ops.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, outputTy.getShape(), outputTy.getElementType(),
         ArrayRef<Value>({dynDims}));
 
-    auto linalgOp = rewriter.create<linalg::GenericOp>(
+    auto linalgOp = linalg::GenericOp::create(rewriter,
         loc, outputTy, genericInputs, ValueRange{emptyTensor}, indexingMaps,
         getNParallelLoopsAttrs(rank),
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
@@ -1466,7 +1466,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           const int32_t inBitwidth = valueTy.getIntOrFloatBitWidth();
           // Extend zeropoint for sub-32bits widths.
           const int32_t inAttrBitwidth = inBitwidth > 32 ? inBitwidth : 32;
-          auto inputZp = nestedBuilder.create<arith::ConstantOp>(
+          auto inputZp = arith::ConstantOp::create(nestedBuilder,
               loc, IntegerAttr::get(rewriter.getIntegerType(inAttrBitwidth),
                                     *maybeIZp));
 
@@ -1482,7 +1482,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           unsigned outBitWidth = outIntType.getWidth();
           const int32_t outAttrBitwidth = 32;
           assert(outBitWidth <= 32 && "Unexpected output zeropoint bitwidth");
-          auto outputZp = nestedBuilder.create<arith::ConstantOp>(
+          auto outputZp = arith::ConstantOp::create(nestedBuilder,
               loc, IntegerAttr::get(rewriter.getIntegerType(outAttrBitwidth),
                                     *maybeOZp));
 
@@ -1501,24 +1501,24 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           }
           if (valueTy.getIntOrFloatBitWidth() < 32) {
             if (op.getInputUnsigned()) {
-              value = nestedBuilder.create<arith::ExtUIOp>(
+              value = arith::ExtUIOp::create(nestedBuilder,
                   nestedLoc, nestedBuilder.getI32Type(), value);
             } else {
-              value = nestedBuilder.create<arith::ExtSIOp>(
+              value = arith::ExtSIOp::create(nestedBuilder,
                   nestedLoc, nestedBuilder.getI32Type(), value);
             }
           }
 
           value =
-              nestedBuilder.create<arith::SubIOp>(nestedLoc, value, inputZp);
+              arith::SubIOp::create(nestedBuilder, nestedLoc, value, inputZp);
 
-          value = nestedBuilder.create<tosa::ApplyScaleOp>(
+          value = tosa::ApplyScaleOp::create(nestedBuilder,
               loc, nestedBuilder.getI32Type(), value, multiplier, shift,
               roundingMode);
 
           // Move to the new zero-point.
           value =
-              nestedBuilder.create<arith::AddIOp>(nestedLoc, value, outputZp);
+              arith::AddIOp::create(nestedBuilder, nestedLoc, value, outputZp);
 
           // Saturate to the output size.
           int32_t intMin = APInt::getSignedMinValue(outBitWidth).getSExtValue();
@@ -1530,16 +1530,16 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
             intMax = APInt::getMaxValue(outBitWidth).getZExtValue();
           }
 
-          auto intMinVal = nestedBuilder.create<arith::ConstantOp>(
+          auto intMinVal = arith::ConstantOp::create(nestedBuilder,
               loc, nestedBuilder.getI32IntegerAttr(intMin));
-          auto intMaxVal = nestedBuilder.create<arith::ConstantOp>(
+          auto intMaxVal = arith::ConstantOp::create(nestedBuilder,
               loc, nestedBuilder.getI32IntegerAttr(intMax));
 
           value = clampIntHelper(nestedLoc, value, intMinVal, intMaxVal,
                                  nestedBuilder, /*isUnsigned=*/false);
 
           if (outIntType.getWidth() < 32) {
-            value = nestedBuilder.create<arith::TruncIOp>(
+            value = arith::TruncIOp::create(nestedBuilder,
                 nestedLoc, rewriter.getIntegerType(outIntType.getWidth()),
                 value);
           }
@@ -1550,7 +1550,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
                                                             outIntType, value)
                         .getResult(0);
           }
-          nestedBuilder.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(nestedBuilder, loc, value);
         });
 
     rewriter.replaceOp(op, linalgOp->getResults());
@@ -1608,25 +1608,25 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
     auto collapseTy =
         RankedTensorType::get({inputTy.getDimSize(0), inputTy.getDimSize(3)},
                               inputTy.getElementType());
-    Value collapse = builder.create<tensor::CollapseShapeOp>(collapseTy, input,
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, input,
                                                              reassociationMap);
 
     // Get any dynamic shapes that appear in the input format.
     llvm::SmallVector<Value> outputDynSize;
     if (inputTy.isDynamicDim(0))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 0));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 0));
     if (inputTy.isDynamicDim(3))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 3));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 3));
 
     // Generate the elementwise operation for casting scaling the input value.
     auto genericTy = collapseTy.clone(resultTy.getElementType());
-    Value empty = builder.create<tensor::EmptyOp>(
+    Value empty = tensor::EmptyOp::create(builder,
         genericTy.getShape(), resultTy.getElementType(), outputDynSize);
     auto genericMap = rewriter.getMultiDimIdentityMap(genericTy.getRank());
     SmallVector<utils::IteratorType> iterators(genericTy.getRank(),
                                                utils::IteratorType::parallel);
 
-    auto generic = builder.create<linalg::GenericOp>(
+    auto generic = linalg::GenericOp::create(builder,
         genericTy, ValueRange{collapse}, ValueRange{empty},
         ArrayRef<AffineMap>{genericMap, genericMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
@@ -1634,22 +1634,22 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
           // This is the quantized case.
           if (inputTy.getElementType() != resultTy.getElementType()) {
             value =
-                b.create<arith::ExtSIOp>(loc, resultTy.getElementType(), value);
+                arith::ExtSIOp::create(b, loc, resultTy.getElementType(), value);
 
             if (isBilinear && scale[0] != 0) {
-              Value scaleY = b.create<arith::ConstantOp>(
+              Value scaleY = arith::ConstantOp::create(b,
                   loc, b.getI32IntegerAttr(scale[0]));
-              value = b.create<arith::MulIOp>(loc, value, scaleY);
+              value = arith::MulIOp::create(b, loc, value, scaleY);
             }
 
             if (isBilinear && scale[2] != 0) {
-              Value scaleX = b.create<arith::ConstantOp>(
+              Value scaleX = arith::ConstantOp::create(b,
                   loc, b.getI32IntegerAttr(scale[2]));
-              value = b.create<arith::MulIOp>(loc, value, scaleX);
+              value = arith::MulIOp::create(b, loc, value, scaleX);
             }
           }
 
-          b.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(b, loc, value);
         });
 
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -1697,7 +1697,7 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     resizeShape.push_back(channels);
 
     auto resizeTy = resultTy.clone(resizeShape);
-    auto resize = builder.create<tosa::ResizeOp>(resizeTy, input, op.getScale(),
+    auto resize = tosa::ResizeOp::create(builder, resizeTy, input, op.getScale(),
                                                  op.getOffset(), op.getBorder(),
                                                  op.getMode());
 
@@ -1720,19 +1720,19 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     collapseShape.push_back(channels);
 
     auto collapseTy = resultTy.clone(collapseShape);
-    Value collapse = builder.create<tensor::CollapseShapeOp>(collapseTy, resize,
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, resize,
                                                              reassociationMap);
 
     // Broadcast the collapsed shape to the output result.
     llvm::SmallVector<Value> outputDynSize;
     if (inputTy.isDynamicDim(0))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 0));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 0));
     if (inputTy.isDynamicDim(3))
-      outputDynSize.push_back(builder.create<tensor::DimOp>(input, 3));
+      outputDynSize.push_back(tensor::DimOp::create(builder, input, 3));
 
     SmallVector<utils::IteratorType> iterators(resultTy.getRank(),
                                                utils::IteratorType::parallel);
-    Value empty = builder.create<tensor::EmptyOp>(
+    Value empty = tensor::EmptyOp::create(builder,
         resultTy.getShape(), resultTy.getElementType(), outputDynSize);
 
     SmallVector<AffineExpr, 4> inputExprs{rewriter.getAffineDimExpr(0)};
@@ -1751,7 +1751,7 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
         ArrayRef<AffineMap>{inputMap, outputMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
           Value value = args[0];
-          b.create<linalg::YieldOp>(loc, value);
+          linalg::YieldOp::create(b, loc, value);
         });
 
     return success();
@@ -1789,9 +1789,9 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
 
     SmallVector<AffineMap, 2> affineMaps = {
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
-    auto emptyTensor = b.create<tensor::EmptyOp>(resultTy.getShape(), resultETy,
+    auto emptyTensor = tensor::EmptyOp::create(b, resultTy.getShape(), resultETy,
                                                  *dynamicDimsOr);
-    auto genericOp = b.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(b,
         resultTy, ValueRange({}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     Value resize = genericOp.getResult(0);
@@ -1800,19 +1800,19 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       OpBuilder::InsertionGuard regionGuard(b);
       b.createBlock(&genericOp.getRegion(), genericOp.getRegion().end(),
                     TypeRange({resultETy}), loc);
-      Value batch = b.create<linalg::IndexOp>(0);
-      Value y = b.create<linalg::IndexOp>(1);
-      Value x = b.create<linalg::IndexOp>(2);
-      Value channel = b.create<linalg::IndexOp>(3);
+      Value batch = linalg::IndexOp::create(b, 0);
+      Value y = linalg::IndexOp::create(b, 1);
+      Value x = linalg::IndexOp::create(b, 2);
+      Value channel = linalg::IndexOp::create(b, 3);
 
       Value zeroI32 =
-          b.create<arith::ConstantOp>(b.getZeroAttr(b.getI32Type()));
-      Value zeroFp = b.create<arith::ConstantOp>(b.getZeroAttr(floatTy));
-      Value hMax = b.create<arith::ConstantOp>(b.getI32IntegerAttr(imageH - 1));
-      Value wMax = b.create<arith::ConstantOp>(b.getI32IntegerAttr(imageW - 1));
+          arith::ConstantOp::create(b, b.getZeroAttr(b.getI32Type()));
+      Value zeroFp = arith::ConstantOp::create(b, b.getZeroAttr(floatTy));
+      Value hMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageH - 1));
+      Value wMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageW - 1));
 
-      Value inY = b.create<arith::IndexCastOp>(b.getI32Type(), y);
-      Value inX = b.create<arith::IndexCastOp>(b.getI32Type(), x);
+      Value inY = arith::IndexCastOp::create(b, b.getI32Type(), y);
+      Value inX = arith::IndexCastOp::create(b, b.getI32Type(), x);
 
       SmallVector<int64_t> scale, offset, border;
       if (!tosa::getConstShapeValues(op.getScale().getDefiningOp(), scale) ||
@@ -1824,16 +1824,16 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       }
 
       Value yScaleN, yScaleD, xScaleN, xScaleD;
-      yScaleN = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[0]));
-      yScaleD = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[1]));
-      xScaleN = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[2]));
-      xScaleD = b.create<arith::ConstantOp>(b.getI32IntegerAttr(scale[3]));
+      yScaleN = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[0]));
+      yScaleD = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[1]));
+      xScaleN = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[2]));
+      xScaleD = arith::ConstantOp::create(b, b.getI32IntegerAttr(scale[3]));
 
       Value yOffset, xOffset, yBorder, xBorder;
-      yOffset = b.create<arith::ConstantOp>(b.getI32IntegerAttr(offset[0]));
-      xOffset = b.create<arith::ConstantOp>(b.getI32IntegerAttr(offset[1]));
-      yBorder = b.create<arith::ConstantOp>(b.getI32IntegerAttr(border[0]));
-      xBorder = b.create<arith::ConstantOp>(b.getI32IntegerAttr(border[1]));
+      yOffset = arith::ConstantOp::create(b, b.getI32IntegerAttr(offset[0]));
+      xOffset = arith::ConstantOp::create(b, b.getI32IntegerAttr(offset[1]));
+      yBorder = arith::ConstantOp::create(b, b.getI32IntegerAttr(border[0]));
+      xBorder = arith::ConstantOp::create(b, b.getI32IntegerAttr(border[1]));
 
       // Compute the ix and dx values for both the X and Y dimensions.
       auto getIndexAndDeltaFp = [&](Value &index, Value &delta, Value in,
@@ -1846,16 +1846,16 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         }
         // x = x * scale_d + offset;
         // ix = floor(x / scale_n)
-        Value val = b.create<arith::MulIOp>(in, scaleD);
-        val = b.create<arith::AddIOp>(val, offset);
-        index = b.create<arith::FloorDivSIOp>(val, scaleN);
+        Value val = arith::MulIOp::create(b, in, scaleD);
+        val = arith::AddIOp::create(b, val, offset);
+        index = arith::FloorDivSIOp::create(b, val, scaleN);
 
         // rx = x % scale_n
         // dx = rx / scale_n
-        Value r = b.create<arith::RemSIOp>(val, scaleN);
-        Value rFp = b.create<arith::SIToFPOp>(floatTy, r);
-        Value scaleNfp = b.create<arith::UIToFPOp>(floatTy, scaleN);
-        delta = b.create<arith::DivFOp>(rFp, scaleNfp);
+        Value r = arith::RemSIOp::create(b, val, scaleN);
+        Value rFp = arith::SIToFPOp::create(b, floatTy, r);
+        Value scaleNfp = arith::UIToFPOp::create(b, floatTy, scaleN);
+        delta = arith::DivFOp::create(b, rFp, scaleNfp);
       };
 
       // Compute the ix and dx values for the X and Y dimensions - int case.
@@ -1870,11 +1870,11 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         // x = x * scale_d + offset;
         // ix = floor(x / scale_n)
         //  dx = x - ix * scale_n;
-        Value val = b.create<arith::MulIOp>(in, scaleD);
-        val = b.create<arith::AddIOp>(val, offset);
-        index = b.create<arith::DivSIOp>(val, scaleN);
-        delta = b.create<arith::MulIOp>(index, scaleN);
-        delta = b.create<arith::SubIOp>(val, delta);
+        Value val = arith::MulIOp::create(b, in, scaleD);
+        val = arith::AddIOp::create(b, val, offset);
+        index = arith::DivSIOp::create(b, val, scaleN);
+        delta = arith::MulIOp::create(b, index, scaleN);
+        delta = arith::SubIOp::create(b, val, delta);
       };
 
       Value ix, iy, dx, dy;
@@ -1887,54 +1887,54 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       }
 
       if (op.getMode() == "NEAREST_NEIGHBOR") {
-        auto one = b.create<arith::ConstantOp>(b.getI32IntegerAttr(1));
+        auto one = arith::ConstantOp::create(b, b.getI32IntegerAttr(1));
 
         auto getNearestIndexAndClamp = [&](Value val, Value dval, Value scale,
                                            Value max, int size,
                                            ImplicitLocOpBuilder &b) -> Value {
           if (size == 1) {
-            return b.create<arith::ConstantIndexOp>(0);
+            return arith::ConstantIndexOp::create(b, 0);
           }
 
           Value pred;
           if (floatingPointMode) {
-            auto h = b.create<arith::ConstantOp>(b.getFloatAttr(floatTy, 0.5f));
-            pred = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, dval, h);
+            auto h = arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 0.5f));
+            pred = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, dval, h);
           } else {
-            Value dvalDouble = b.create<arith::ShLIOp>(dval, one);
-            pred = b.create<arith::CmpIOp>(arith::CmpIPredicate::sge,
+            Value dvalDouble = arith::ShLIOp::create(b, dval, one);
+            pred = arith::CmpIOp::create(b, arith::CmpIPredicate::sge,
                                            dvalDouble, scale);
           }
 
-          auto offset = b.create<arith::SelectOp>(pred, one, zeroI32);
-          val = b.create<arith::AddIOp>(val, offset);
+          auto offset = arith::SelectOp::create(b, pred, one, zeroI32);
+          val = arith::AddIOp::create(b, val, offset);
           val = clampIntHelper(loc, val, zeroI32, max, b, /*isUnsigned=*/false);
-          return b.create<arith::IndexCastOp>(b.getIndexType(), val);
+          return arith::IndexCastOp::create(b, b.getIndexType(), val);
         };
 
         iy = getNearestIndexAndClamp(iy, dy, yScaleN, hMax, imageH, b);
         ix = getNearestIndexAndClamp(ix, dx, xScaleN, wMax, imageW, b);
 
-        Value result = b.create<tensor::ExtractOp>(
+        Value result = tensor::ExtractOp::create(b,
             input, ValueRange{batch, iy, ix, channel});
 
-        b.create<linalg::YieldOp>(result);
+        linalg::YieldOp::create(b, result);
       } else {
         // The mode here must be BILINEAR.
         assert(op.getMode() == "BILINEAR");
 
-        auto oneVal = b.create<arith::ConstantOp>(b.getI32IntegerAttr(1));
+        auto oneVal = arith::ConstantOp::create(b, b.getI32IntegerAttr(1));
 
         auto getClampedIdxs = [&](Value &val0, Value &val1, int size, Value in,
                                   Value max, ImplicitLocOpBuilder &b) {
           val0 = in;
-          val1 = b.create<arith::AddIOp>(val0, oneVal);
+          val1 = arith::AddIOp::create(b, val0, oneVal);
           val0 =
               clampIntHelper(loc, val0, zeroI32, max, b, /*isUnsigned=*/false);
           val1 =
               clampIntHelper(loc, val1, zeroI32, max, b, /*isUnsigned=*/false);
-          val0 = b.create<arith::IndexCastOp>(b.getIndexType(), val0);
-          val1 = b.create<arith::IndexCastOp>(b.getIndexType(), val1);
+          val0 = arith::IndexCastOp::create(b, b.getIndexType(), val0);
+          val1 = arith::IndexCastOp::create(b, b.getIndexType(), val1);
         };
 
         // Linalg equivalent to the section below:
@@ -1946,27 +1946,27 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         getClampedIdxs(y0, y1, imageH, iy, hMax, b);
         getClampedIdxs(x0, x1, imageW, ix, wMax, b);
 
-        Value y0x0 = b.create<tensor::ExtractOp>(
+        Value y0x0 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y0, x0, channel});
-        Value y0x1 = b.create<tensor::ExtractOp>(
+        Value y0x1 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y0, x1, channel});
-        Value y1x0 = b.create<tensor::ExtractOp>(
+        Value y1x0 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y1, x0, channel});
-        Value y1x1 = b.create<tensor::ExtractOp>(
+        Value y1x1 = tensor::ExtractOp::create(b,
             input, ValueRange{batch, y1, x1, channel});
 
         if (floatingPointMode) {
           auto oneVal =
-              b.create<arith::ConstantOp>(b.getFloatAttr(floatTy, 1.0f));
+              arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 1.0f));
           auto interpolate = [&](Value val0, Value val1, Value delta,
                                  int inputSize,
                                  ImplicitLocOpBuilder &b) -> Value {
             if (inputSize == 1)
               return val0;
-            Value oneMinusDelta = b.create<arith::SubFOp>(oneVal, delta);
-            Value mul0 = b.create<arith::MulFOp>(val0, oneMinusDelta);
-            Value mul1 = b.create<arith::MulFOp>(val1, delta);
-            return b.create<arith::AddFOp>(mul0, mul1);
+            Value oneMinusDelta = arith::SubFOp::create(b, oneVal, delta);
+            Value mul0 = arith::MulFOp::create(b, val0, oneMinusDelta);
+            Value mul1 = arith::MulFOp::create(b, val1, delta);
+            return arith::AddFOp::create(b, mul0, mul1);
           };
 
           // Linalg equivalent to the section below:
@@ -1982,18 +1982,18 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
           // Linalg equivalent to the section below:
           //   result = topAcc * (unit_y - dy) + bottomAcc * dy
           Value result = interpolate(topAcc, bottomAcc, dy, imageH, b);
-          b.create<linalg::YieldOp>(result);
+          linalg::YieldOp::create(b, result);
         } else {
           // Perform in quantized space.
-          y0x0 = b.create<arith::ExtSIOp>(resultETy, y0x0);
-          y0x1 = b.create<arith::ExtSIOp>(resultETy, y0x1);
-          y1x0 = b.create<arith::ExtSIOp>(resultETy, y1x0);
-          y1x1 = b.create<arith::ExtSIOp>(resultETy, y1x1);
+          y0x0 = arith::ExtSIOp::create(b, resultETy, y0x0);
+          y0x1 = arith::ExtSIOp::create(b, resultETy, y0x1);
+          y1x0 = arith::ExtSIOp::create(b, resultETy, y1x0);
+          y1x1 = arith::ExtSIOp::create(b, resultETy, y1x1);
 
           const int64_t deltaBitwidth = dx.getType().getIntOrFloatBitWidth();
           if (resultETy.getIntOrFloatBitWidth() > deltaBitwidth) {
-            dx = b.create<arith::ExtSIOp>(resultETy, dx);
-            dy = b.create<arith::ExtSIOp>(resultETy, dy);
+            dx = arith::ExtSIOp::create(b, resultETy, dx);
+            dy = arith::ExtSIOp::create(b, resultETy, dy);
           }
 
           Value yScaleNExt = yScaleN;
@@ -2002,26 +2002,26 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
           const int64_t scaleBitwidth =
               xScaleN.getType().getIntOrFloatBitWidth();
           if (resultETy.getIntOrFloatBitWidth() > scaleBitwidth) {
-            yScaleNExt = b.create<arith::ExtSIOp>(resultETy, yScaleN);
-            xScaleNExt = b.create<arith::ExtSIOp>(resultETy, xScaleN);
+            yScaleNExt = arith::ExtSIOp::create(b, resultETy, yScaleN);
+            xScaleNExt = arith::ExtSIOp::create(b, resultETy, xScaleN);
           }
 
           auto interpolate = [](Value val0, Value val1, Value weight1,
                                 Value scale, int inputSize,
                                 ImplicitLocOpBuilder &b) -> Value {
             if (inputSize == 1)
-              return b.create<arith::MulIOp>(val0, scale);
-            Value weight0 = b.create<arith::SubIOp>(scale, weight1);
-            Value mul0 = b.create<arith::MulIOp>(val0, weight0);
-            Value mul1 = b.create<arith::MulIOp>(val1, weight1);
-            return b.create<arith::AddIOp>(mul0, mul1);
+              return arith::MulIOp::create(b, val0, scale);
+            Value weight0 = arith::SubIOp::create(b, scale, weight1);
+            Value mul0 = arith::MulIOp::create(b, val0, weight0);
+            Value mul1 = arith::MulIOp::create(b, val1, weight1);
+            return arith::AddIOp::create(b, mul0, mul1);
           };
 
           Value topAcc = interpolate(y0x0, y0x1, dx, xScaleNExt, imageW, b);
           Value bottomAcc = interpolate(y1x0, y1x1, dx, xScaleNExt, imageW, b);
           Value result =
               interpolate(topAcc, bottomAcc, dy, yScaleNExt, imageH, b);
-          b.create<linalg::YieldOp>(result);
+          linalg::YieldOp::create(b, result);
         }
       }
     }
@@ -2072,11 +2072,11 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i)) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
-    Value axisDimSize = rewriter.create<tensor::DimOp>(loc, input, axis);
+    Value axisDimSize = tensor::DimOp::create(rewriter, loc, input, axis);
 
     // First fill the output buffer with the init value.
     auto emptyTensor = rewriter
@@ -2094,21 +2094,21 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
           llvm::SmallVector<Value> indices;
           for (unsigned int i = 0; i < inputTy.getRank(); i++) {
             Value index =
-                rewriter.create<linalg::IndexOp>(nestedLoc, i).getResult();
+                linalg::IndexOp::create(rewriter, nestedLoc, i).getResult();
             if (i == axis) {
-              auto one = rewriter.create<arith::ConstantIndexOp>(nestedLoc, 1);
+              auto one = arith::ConstantIndexOp::create(rewriter, nestedLoc, 1);
               auto sizeMinusOne =
-                  rewriter.create<arith::SubIOp>(nestedLoc, axisDimSize, one);
-              index = rewriter.create<arith::SubIOp>(nestedLoc, sizeMinusOne,
+                  arith::SubIOp::create(rewriter, nestedLoc, axisDimSize, one);
+              index = arith::SubIOp::create(rewriter, nestedLoc, sizeMinusOne,
                                                      index);
             }
 
             indices.push_back(index);
           }
 
-          auto extract = nestedBuilder.create<tensor::ExtractOp>(
+          auto extract = tensor::ExtractOp::create(nestedBuilder,
               nestedLoc, input, indices);
-          nestedBuilder.create<linalg::YieldOp>(op.getLoc(),
+          linalg::YieldOp::create(nestedBuilder, op.getLoc(),
                                                 extract.getResult());
         });
     return success();
@@ -2148,11 +2148,11 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i) || multiples[i] == -1) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         op.getLoc(), genericShape, elementTy, dynDims);
 
     // We needs to map the input shape to the non-broadcasted dimensions.
@@ -2168,12 +2168,12 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<AffineMap, 2> affineMaps = {
         readAffineMap, rewriter.getMultiDimIdentityMap(genericShape.size())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, RankedTensorType::get(genericShape, elementTy), input,
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(genericShape.size()),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-          nestedBuilder.create<linalg::YieldOp>(op.getLoc(), *args.begin());
+          linalg::YieldOp::create(nestedBuilder, op.getLoc(), *args.begin());
         });
 
     auto shapeValue = getTosaConstShape(
@@ -2220,7 +2220,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     SmallVector<Value> dynDims;
     for (int i = 0; i < inputTy.getRank(); i++) {
       if (inputTy.isDynamicDim(i) && i != axis) {
-        dynDims.push_back(rewriter.create<tensor::DimOp>(loc, input, i));
+        dynDims.push_back(tensor::DimOp::create(rewriter, loc, input, i));
       }
     }
 
@@ -2229,7 +2229,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
                               .create<tensor::EmptyOp>(loc, resultTy.getShape(),
                                                        outElementTy, dynDims)
                               .getResult();
-    auto fillValueIdx = rewriter.create<arith::ConstantOp>(
+    auto fillValueIdx = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(outElementTy, 0));
     auto filledTensorIdx =
         rewriter
@@ -2250,7 +2250,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           argmaxOp, "unsupported tosa.argmax element type");
 
     auto fillValueMax =
-        rewriter.create<arith::ConstantOp>(loc, fillValueMaxAttr);
+        arith::ConstantOp::create(rewriter, loc, fillValueMaxAttr);
     auto filledTensorMax =
         rewriter
             .create<linalg::FillOp>(loc, ValueRange{fillValueMax},
@@ -2274,7 +2274,7 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     bool didEncounterError = false;
     auto maps = AffineMap::inferFromExprList({srcExprs, dstExprs, dstExprs},
                                              rewriter.getContext());
-    auto linalgOp = rewriter.create<linalg::GenericOp>(
+    auto linalgOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
         ValueRange({filledTensorIdx, filledTensorMax}), maps, iteratorTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
@@ -2283,41 +2283,41 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           auto oldIndex = blockArgs[1];
           auto oldValue = blockArgs[2];
 
-          Value newIndex = rewriter.create<arith::IndexCastOp>(
+          Value newIndex = arith::IndexCastOp::create(rewriter,
               nestedLoc, oldIndex.getType(),
-              rewriter.create<linalg::IndexOp>(loc, axis));
+              linalg::IndexOp::create(rewriter, loc, axis));
 
           Value predicate;
           if (isa<FloatType>(inElementTy)) {
             if (argmaxOp.getNanMode() == "IGNORE") {
               // Only update index & max value for non NaN values. If all
               // values are NaNs, the initial index will be return which is 0.
-              predicate = rewriter.create<arith::CmpFOp>(
+              predicate = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::OGT, newValue, oldValue);
             } else {
               // Update max value if either of the following is true:
               // - new value is bigger
               // - cur max is not NaN and new value is NaN
-              Value gt = rewriter.create<arith::CmpFOp>(
+              Value gt = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::UGT, newValue, oldValue);
-              Value oldNonNaN = rewriter.create<arith::CmpFOp>(
+              Value oldNonNaN = arith::CmpFOp::create(rewriter,
                   nestedLoc, arith::CmpFPredicate::ORD, oldValue, oldValue);
-              predicate = rewriter.create<arith::AndIOp>(
+              predicate = arith::AndIOp::create(rewriter,
                   nestedLoc, rewriter.getI1Type(), gt, oldNonNaN);
             }
           } else if (isa<IntegerType>(inElementTy)) {
-            predicate = rewriter.create<arith::CmpIOp>(
+            predicate = arith::CmpIOp::create(rewriter,
                 nestedLoc, arith::CmpIPredicate::sgt, newValue, oldValue);
           } else {
             didEncounterError = true;
             return;
           }
 
-          auto resultMax = rewriter.create<arith::SelectOp>(
+          auto resultMax = arith::SelectOp::create(rewriter,
               nestedLoc, predicate, newValue, oldValue);
-          auto resultIndex = rewriter.create<arith::SelectOp>(
+          auto resultIndex = arith::SelectOp::create(rewriter,
               nestedLoc, predicate, newIndex, oldIndex);
-          nestedBuilder.create<linalg::YieldOp>(
+          linalg::YieldOp::create(nestedBuilder,
               nestedLoc, ValueRange({resultIndex, resultMax}));
         });
 
@@ -2363,19 +2363,19 @@ class GatherConverter : public OpConversionPattern<tosa::GatherOp> {
             rewriter.getContext()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy}), ValueRange{indices},
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
           auto indexValue = args[0];
-          auto index0 = rewriter.create<linalg::IndexOp>(loc, 0);
-          Value index1 = rewriter.create<arith::IndexCastOp>(
+          auto index0 = linalg::IndexOp::create(rewriter, loc, 0);
+          Value index1 = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getIndexType(), indexValue);
-          auto index2 = rewriter.create<linalg::IndexOp>(loc, 2);
-          Value extract = rewriter.create<tensor::ExtractOp>(
+          auto index2 = linalg::IndexOp::create(rewriter, loc, 2);
+          Value extract = tensor::ExtractOp::create(rewriter,
               loc, input, ValueRange{index0, index1, index2});
-          rewriter.create<linalg::YieldOp>(loc, extract);
+          linalg::YieldOp::create(rewriter, loc, extract);
         });
     rewriter.replaceOp(op, genericOp.getResult(0));
     return success();
@@ -2424,7 +2424,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
     for (int i = 0; i < resultTy.getRank(); ++i) {
       if (inputTy.isDynamicDim(i)) {
         dynDims.push_back(
-            rewriter.create<tensor::DimOp>(loc, op.getOperand(0), i));
+            tensor::DimOp::create(rewriter, loc, op.getOperand(0), i));
       }
     }
 
@@ -2437,7 +2437,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
         rewriter.getMultiDimIdentityMap(resultTy.getRank()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, resultTy, ValueRange({input}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -2452,69 +2452,69 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
       rewriter.setInsertionPointToStart(block);
       if (inputElementTy.isInteger(8) && tableElementTy.isInteger(8) &&
           resultElementTy.isInteger(8)) {
-        Value index = rewriter.create<arith::IndexCastOp>(
+        Value index = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), inputValue);
-        Value offset = rewriter.create<arith::ConstantIndexOp>(loc, 128);
-        index = rewriter.create<arith::AddIOp>(loc, rewriter.getIndexType(),
+        Value offset = arith::ConstantIndexOp::create(rewriter, loc, 128);
+        index = arith::AddIOp::create(rewriter, loc, rewriter.getIndexType(),
                                                index, offset);
         Value extract =
-            rewriter.create<tensor::ExtractOp>(loc, table, ValueRange{index});
-        rewriter.create<linalg::YieldOp>(loc, extract);
+            tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
+        linalg::YieldOp::create(rewriter, loc, extract);
         return success();
       }
 
       if (inputElementTy.isInteger(16) && tableElementTy.isInteger(16) &&
           resultElementTy.isInteger(32)) {
-        Value extend = rewriter.create<arith::ExtSIOp>(
+        Value extend = arith::ExtSIOp::create(rewriter,
             loc, rewriter.getI32Type(), inputValue);
 
-        auto offset = rewriter.create<arith::ConstantOp>(
+        auto offset = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(32768));
-        auto seven = rewriter.create<arith::ConstantOp>(
+        auto seven = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(7));
-        auto one = rewriter.create<arith::ConstantOp>(
+        auto one = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(1));
-        auto b1111111 = rewriter.create<arith::ConstantOp>(
+        auto b1111111 = arith::ConstantOp::create(rewriter,
             loc, rewriter.getI32IntegerAttr(127));
 
         // Compute the index and fractional part from the input value:
         // value = value + 32768
         // index = value >> 7;
         // fraction = 0x01111111 & value
-        auto extendAdd = rewriter.create<arith::AddIOp>(loc, extend, offset);
-        Value index = rewriter.create<arith::ShRUIOp>(loc, extendAdd, seven);
+        auto extendAdd = arith::AddIOp::create(rewriter, loc, extend, offset);
+        Value index = arith::ShRUIOp::create(rewriter, loc, extendAdd, seven);
         Value fraction =
-            rewriter.create<arith::AndIOp>(loc, extendAdd, b1111111);
+            arith::AndIOp::create(rewriter, loc, extendAdd, b1111111);
 
         // Extract the base and next values from the table.
         // base = (int32_t) table[index];
         // next = (int32_t) table[index + 1];
-        Value indexPlusOne = rewriter.create<arith::AddIOp>(loc, index, one);
+        Value indexPlusOne = arith::AddIOp::create(rewriter, loc, index, one);
 
-        index = rewriter.create<arith::IndexCastOp>(
+        index = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), index);
-        indexPlusOne = rewriter.create<arith::IndexCastOp>(
+        indexPlusOne = arith::IndexCastOp::create(rewriter,
             loc, rewriter.getIndexType(), indexPlusOne);
 
         Value base =
-            rewriter.create<tensor::ExtractOp>(loc, table, ValueRange{index});
-        Value next = rewriter.create<tensor::ExtractOp>(
+            tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
+        Value next = tensor::ExtractOp::create(rewriter,
             loc, table, ValueRange{indexPlusOne});
 
         base =
-            rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), base);
+            arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), base);
         next =
-            rewriter.create<arith::ExtSIOp>(loc, rewriter.getI32Type(), next);
+            arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), next);
 
         // Use the fractional part to interpolate between the input values:
         // result = (base << 7) + (next - base) * fraction
-        Value baseScaled = rewriter.create<arith::ShLIOp>(loc, base, seven);
-        Value diff = rewriter.create<arith::SubIOp>(loc, next, base);
-        Value diffScaled = rewriter.create<arith::MulIOp>(loc, diff, fraction);
+        Value baseScaled = arith::ShLIOp::create(rewriter, loc, base, seven);
+        Value diff = arith::SubIOp::create(rewriter, loc, next, base);
+        Value diffScaled = arith::MulIOp::create(rewriter, loc, diff, fraction);
         Value result =
-            rewriter.create<arith::AddIOp>(loc, baseScaled, diffScaled);
+            arith::AddIOp::create(rewriter, loc, baseScaled, diffScaled);
 
-        rewriter.create<linalg::YieldOp>(loc, result);
+        linalg::YieldOp::create(rewriter, loc, result);
 
         return success();
       }
@@ -2532,8 +2532,8 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static OpFoldResult halfPlusOne(OpBuilder &builder, Location loc,
                                   OpFoldResult ofr) {
-    auto one = builder.create<arith::ConstantIndexOp>(loc, 1);
-    auto two = builder.create<arith::ConstantIndexOp>(loc, 2);
+    auto one = arith::ConstantIndexOp::create(builder, loc, 1);
+    auto two = arith::ConstantIndexOp::create(builder, loc, 2);
 
     auto value = getValueOrCreateConstantIndexOp(builder, loc, ofr);
     auto divBy2 = builder.createOrFold<arith::DivUIOp>(loc, value, two);
@@ -2562,9 +2562,9 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
                                 RankedTensorType type,
                                 llvm::ArrayRef<Value> dynamicSizes) {
     auto emptyTensor =
-        rewriter.create<tensor::EmptyOp>(loc, type, dynamicSizes);
+        tensor::EmptyOp::create(rewriter, loc, type, dynamicSizes);
     auto fillValueAttr = rewriter.getZeroAttr(type.getElementType());
-    auto fillValue = rewriter.create<arith::ConstantOp>(loc, fillValueAttr);
+    auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
     auto filledTensor = rewriter
                             .create<linalg::FillOp>(loc, ValueRange{fillValue},
                                                     ValueRange{emptyTensor})
@@ -2574,18 +2574,18 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static Value castIndexToFloat(OpBuilder &builder, Location loc,
                                 FloatType type, Value value) {
-    auto integerVal = builder.create<arith::IndexCastUIOp>(
+    auto integerVal = arith::IndexCastUIOp::create(builder,
         loc,
         type.getIntOrFloatBitWidth() > 32 ? builder.getI64Type()
                                           : builder.getI32Type(),
         value);
 
-    return builder.create<arith::UIToFPOp>(loc, type, integerVal);
+    return arith::UIToFPOp::create(builder, loc, type, integerVal);
   }
 
   static Value createLinalgIndex(OpBuilder &builder, Location loc,
                                  FloatType type, int64_t index) {
-    auto indexVal = builder.create<linalg::IndexOp>(loc, index);
+    auto indexVal = linalg::IndexOp::create(builder, loc, index);
     return castIndexToFloat(builder, loc, type, indexVal);
   }
 
@@ -2640,7 +2640,7 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
     // Constants and dimension sizes
     auto twoPiAttr = rewriter.getFloatAttr(elementType, 6.283185307179586);
-    auto twoPi = rewriter.create<arith::ConstantOp>(loc, twoPiAttr);
+    auto twoPi = arith::ConstantOp::create(rewriter, loc, twoPiAttr);
     auto constH = castIndexToFloat(rewriter, loc, elementType, dimH);
     auto constW = castIndexToFloat(rewriter, loc, elementType, dimW);
 
@@ -2650,43 +2650,43 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
       Value sumImag = args[2];
 
       // Indices for angle computation
-      Value oy = builder.create<linalg::IndexOp>(loc, 1);
-      Value ox = builder.create<linalg::IndexOp>(loc, 2);
-      Value iy = builder.create<linalg::IndexOp>(loc, 3);
-      Value ix = builder.create<linalg::IndexOp>(loc, 4);
+      Value oy = linalg::IndexOp::create(builder, loc, 1);
+      Value ox = linalg::IndexOp::create(builder, loc, 2);
+      Value iy = linalg::IndexOp::create(builder, loc, 3);
+      Value ix = linalg::IndexOp::create(builder, loc, 4);
 
       // Calculating angle without integer parts of components as sin/cos are
       // periodic: angle = 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix * ox) % W )
       // / W);
-      auto iyXoy = builder.create<index::MulOp>(loc, iy, oy);
-      auto ixXox = builder.create<index::MulOp>(loc, ix, ox);
+      auto iyXoy = index::MulOp::create(builder, loc, iy, oy);
+      auto ixXox = index::MulOp::create(builder, loc, ix, ox);
 
-      auto iyRem = builder.create<index::RemUOp>(loc, iyXoy, dimH);
-      auto ixRem = builder.create<index::RemUOp>(loc, ixXox, dimW);
+      auto iyRem = index::RemUOp::create(builder, loc, iyXoy, dimH);
+      auto ixRem = index::RemUOp::create(builder, loc, ixXox, dimW);
 
       auto iyRemFloat = castIndexToFloat(builder, loc, elementType, iyRem);
       auto ixRemFloat = castIndexToFloat(builder, loc, elementType, ixRem);
 
-      auto yComponent = builder.create<arith::DivFOp>(loc, iyRemFloat, constH);
-      auto xComponent = builder.create<arith::DivFOp>(loc, ixRemFloat, constW);
-      auto sumXY = builder.create<arith::AddFOp>(loc, yComponent, xComponent);
-      auto angle = builder.create<arith::MulFOp>(loc, twoPi, sumXY);
+      auto yComponent = arith::DivFOp::create(builder, loc, iyRemFloat, constH);
+      auto xComponent = arith::DivFOp::create(builder, loc, ixRemFloat, constW);
+      auto sumXY = arith::AddFOp::create(builder, loc, yComponent, xComponent);
+      auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       // realComponent = valReal * cos(angle)
       // imagComponent = valReal * sin(angle)
-      auto cosAngle = builder.create<math::CosOp>(loc, angle);
-      auto sinAngle = builder.create<math::SinOp>(loc, angle);
+      auto cosAngle = math::CosOp::create(builder, loc, angle);
+      auto sinAngle = math::SinOp::create(builder, loc, angle);
       auto realComponent =
-          builder.create<arith::MulFOp>(loc, valReal, cosAngle);
+          arith::MulFOp::create(builder, loc, valReal, cosAngle);
       auto imagComponent =
-          builder.create<arith::MulFOp>(loc, valReal, sinAngle);
+          arith::MulFOp::create(builder, loc, valReal, sinAngle);
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = builder.create<arith::AddFOp>(loc, sumReal, realComponent);
-      auto outImag = builder.create<arith::SubFOp>(loc, sumImag, imagComponent);
+      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag = arith::SubFOp::create(builder, loc, sumImag, imagComponent);
 
-      builder.create<linalg::YieldOp>(loc, ValueRange{outReal, outImag});
+      linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
 
     rewriter.replaceOpWithNewOp<linalg::GenericOp>(
@@ -2760,7 +2760,7 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
 
     // Constants and dimension sizes
     auto twoPiAttr = rewriter.getFloatAttr(real_el_ty, 6.283185307179586);
-    auto twoPi = rewriter.create<arith::ConstantOp>(loc, twoPiAttr);
+    auto twoPi = arith::ConstantOp::create(rewriter, loc, twoPiAttr);
     Value constH =
         RFFT2dConverter::castIndexToFloat(rewriter, loc, real_el_ty, dimH);
     Value constW =
@@ -2773,57 +2773,57 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
       Value sumImag = args[3];
 
       // Indices for angle computation
-      Value oy = builder.create<linalg::IndexOp>(loc, 1);
-      Value ox = builder.create<linalg::IndexOp>(loc, 2);
-      Value iy = builder.create<linalg::IndexOp>(loc, 3);
-      Value ix = builder.create<linalg::IndexOp>(loc, 4);
+      Value oy = linalg::IndexOp::create(builder, loc, 1);
+      Value ox = linalg::IndexOp::create(builder, loc, 2);
+      Value iy = linalg::IndexOp::create(builder, loc, 3);
+      Value ix = linalg::IndexOp::create(builder, loc, 4);
 
       // float_t angle = sign_val * 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix *
       // ox) % W ) / W);
-      auto iyXoy = builder.create<index::MulOp>(loc, iy, oy);
-      auto ixXox = builder.create<index::MulOp>(loc, ix, ox);
+      auto iyXoy = index::MulOp::create(builder, loc, iy, oy);
+      auto ixXox = index::MulOp::create(builder, loc, ix, ox);
 
-      auto iyRem = builder.create<index::RemUOp>(loc, iyXoy, dimH);
-      auto ixRem = builder.create<index::RemUOp>(loc, ixXox, dimW);
+      auto iyRem = index::RemUOp::create(builder, loc, iyXoy, dimH);
+      auto ixRem = index::RemUOp::create(builder, loc, ixXox, dimW);
 
       auto iyRemFloat =
           RFFT2dConverter::castIndexToFloat(builder, loc, real_el_ty, iyRem);
       auto ixRemFloat =
           RFFT2dConverter::castIndexToFloat(builder, loc, real_el_ty, ixRem);
 
-      auto yComponent = builder.create<arith::DivFOp>(loc, iyRemFloat, constH);
-      auto xComponent = builder.create<arith::DivFOp>(loc, ixRemFloat, constW);
+      auto yComponent = arith::DivFOp::create(builder, loc, iyRemFloat, constH);
+      auto xComponent = arith::DivFOp::create(builder, loc, ixRemFloat, constW);
 
-      auto sumXY = builder.create<arith::AddFOp>(loc, yComponent, xComponent);
-      auto angle = builder.create<arith::MulFOp>(loc, twoPi, sumXY);
+      auto sumXY = arith::AddFOp::create(builder, loc, yComponent, xComponent);
+      auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       if (inverse.getValue()) {
-        angle = builder.create<arith::MulFOp>(
+        angle = arith::MulFOp::create(builder,
             loc, angle,
-            rewriter.create<arith::ConstantOp>(
+            arith::ConstantOp::create(rewriter,
                 loc, rewriter.getFloatAttr(real_el_ty, -1.0)));
       }
 
       // realComponent = val_real * cos(a) + val_imag * sin(a);
       // imagComponent = -val_real * sin(a) + val_imag * cos(a);
-      auto cosAngle = builder.create<math::CosOp>(loc, angle);
-      auto sinAngle = builder.create<math::SinOp>(loc, angle);
+      auto cosAngle = math::CosOp::create(builder, loc, angle);
+      auto sinAngle = math::SinOp::create(builder, loc, angle);
 
-      auto rcos = builder.create<arith::MulFOp>(loc, valReal, cosAngle);
-      auto rsin = builder.create<arith::MulFOp>(loc, valImag, sinAngle);
-      auto realComponent = builder.create<arith::AddFOp>(loc, rcos, rsin);
+      auto rcos = arith::MulFOp::create(builder, loc, valReal, cosAngle);
+      auto rsin = arith::MulFOp::create(builder, loc, valImag, sinAngle);
+      auto realComponent = arith::AddFOp::create(builder, loc, rcos, rsin);
 
-      auto icos = builder.create<arith::MulFOp>(loc, valImag, cosAngle);
-      auto isin = builder.create<arith::MulFOp>(loc, valReal, sinAngle);
+      auto icos = arith::MulFOp::create(builder, loc, valImag, cosAngle);
+      auto isin = arith::MulFOp::create(builder, loc, valReal, sinAngle);
 
-      auto imagComponent = builder.create<arith::SubFOp>(loc, icos, isin);
+      auto imagComponent = arith::SubFOp::create(builder, loc, icos, isin);
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = builder.create<arith::AddFOp>(loc, sumReal, realComponent);
-      auto outImag = builder.create<arith::AddFOp>(loc, sumImag, imagComponent);
+      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag = arith::AddFOp::create(builder, loc, sumImag, imagComponent);
 
-      builder.create<linalg::YieldOp>(loc, ValueRange{outReal, outImag});
+      linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
 
     rewriter.replaceOpWithNewOp<linalg::GenericOp>(
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index 00b9a065dfb3d..bfe43e8dcc274 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -52,9 +52,9 @@ static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
     highIndices.push_back(rewriter.getIndexAttr(highPad));
   }
 
-  Value padValue = rewriter.create<arith::ConstantOp>(loc, padAttr);
+  Value padValue = arith::ConstantOp::create(rewriter, loc, padAttr);
 
-  return rewriter.create<tensor::PadOp>(
+  return tensor::PadOp::create(rewriter,
       loc, RankedTensorType::get(paddedShape, inputETy), input, lowIndices,
       highIndices, padValue);
 }
@@ -72,10 +72,10 @@ linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias,
             Value biasVal = args[0];
             Type resType = args[1].getType();
             if (resType != biasVal.getType()) {
-              biasVal = builder.create<arith::ExtSIOp>(loc, resType, biasVal);
+              biasVal = arith::ExtSIOp::create(builder, loc, resType, biasVal);
             }
-            Value added = builder.create<arith::AddIOp>(loc, biasVal, args[1]);
-            builder.create<linalg::YieldOp>(loc, added);
+            Value added = arith::AddIOp::create(builder, loc, biasVal, args[1]);
+            linalg::YieldOp::create(builder, loc, added);
           })
       .getResult(0);
 }
@@ -134,19 +134,19 @@ static mlir::Value linalgBroadcastAndMaybeExt(PatternRewriter &rewriter,
             if (resType != biasVal.getType()) {
               biasVal =
                   resultTy.getElementType().isFloat()
-                      ? builder.create<arith::ExtFOp>(loc, resType, biasVal)
+                      ? arith::ExtFOp::create(builder, loc, resType, biasVal)
                             .getResult()
-                      : builder.create<arith::ExtSIOp>(loc, resType, biasVal)
+                      : arith::ExtSIOp::create(builder, loc, resType, biasVal)
                             .getResult();
             }
-            builder.create<linalg::YieldOp>(loc, biasVal);
+            linalg::YieldOp::create(builder, loc, biasVal);
           })
       .getResult(0);
 }
 
 static mlir::Value reifyConstantDim(int64_t attr,
                                     ImplicitLocOpBuilder &builder) {
-  return builder.create<arith::ConstantIndexOp>(attr);
+  return arith::ConstantIndexOp::create(builder, attr);
 }
 
 // Calculating the output width/height using the formula:
@@ -160,22 +160,22 @@ static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim,
                                           int64_t dilationAttr,
                                           OpBuilder &rewriter) {
   ImplicitLocOpBuilder builder(loc, rewriter);
-  auto one = rewriter.create<arith::ConstantOp>(
+  auto one = arith::ConstantOp::create(rewriter,
       loc, IntegerAttr::get(inputDim.getType(), 1));
   Value padBefore = reifyConstantDim(padBeforeAttr, builder);
-  Value paddedBefore = builder.create<arith::AddIOp>(inputDim, padBefore);
+  Value paddedBefore = arith::AddIOp::create(builder, inputDim, padBefore);
   Value padAfter = reifyConstantDim(padAfterAttr, builder);
-  Value paddedAfter = builder.create<arith::AddIOp>(paddedBefore, padAfter);
+  Value paddedAfter = arith::AddIOp::create(builder, paddedBefore, padAfter);
 
-  Value subOne = builder.create<arith::SubIOp>(kernelDim, one);
+  Value subOne = arith::SubIOp::create(builder, kernelDim, one);
   Value dilation = reifyConstantDim(dilationAttr, builder);
-  Value dilated = builder.create<arith::MulIOp>(dilation, subOne);
-  Value addOne = builder.create<arith::AddIOp>(dilated, one);
+  Value dilated = arith::MulIOp::create(builder, dilation, subOne);
+  Value addOne = arith::AddIOp::create(builder, dilated, one);
 
-  Value subtract = builder.create<arith::SubIOp>(paddedAfter, addOne);
+  Value subtract = arith::SubIOp::create(builder, paddedAfter, addOne);
   Value stride = reifyConstantDim(strideAttr, builder);
-  Value divide = builder.create<arith::DivUIOp>(subtract, stride);
-  return builder.create<arith::AddIOp>(divide, one);
+  Value divide = arith::DivUIOp::create(builder, subtract, stride);
+  return arith::AddIOp::create(builder, divide, one);
 }
 
 // Creates a vector of the dynamic output dims for Conv2D and Depthwise_Conv2D
@@ -198,9 +198,9 @@ static SmallVector<Value> inferDynamicDimsForConv(
       auto padBottom = padAttr[i * 2 + 1];
       auto stride = strideAttr[i];
       auto dilation = dilationAttr[i];
-      Value initDynDim = rewriter.create<tensor::DimOp>(loc, input, inputDim);
+      Value initDynDim = tensor::DimOp::create(rewriter, loc, input, inputDim);
       Value kernelDynDim =
-          rewriter.create<tensor::DimOp>(loc, weight, kernelDim);
+          tensor::DimOp::create(rewriter, loc, weight, kernelDim);
       // H = F(IH, pad_top, pad_bottom, dilation_y, KH, stride_y)
       dynDims[inputDim] =
           getConvOrPoolOutputDim(loc, initDynDim, padTop, padBottom,
@@ -211,7 +211,7 @@ static SmallVector<Value> inferDynamicDimsForConv(
   // Get the batch/channels dimensions.
   for (int i = 0; i < inputRank; i++) {
     if (resultTy.isDynamicDim(i) && !dynDims[i])
-      dynDims[i] = rewriter.create<tensor::DimOp>(loc, input, i);
+      dynDims[i] = tensor::DimOp::create(rewriter, loc, input, i);
   }
 
   SmallVector<Value> filteredDims = condenseValues(dynDims);
@@ -350,7 +350,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
         auto weightPermAttr = rewriter.getDenseI32ArrayAttr(weightPerm);
         Type newWeightTy =
             RankedTensorType::get(newWeightShape, weightTy.getElementType());
-        weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
+        weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
                                                     weightPermAttr);
       }
     }
@@ -372,7 +372,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       auto weightPermAttr = rewriter.getDenseI32ArrayAttr(weightPerm);
       Type newWeightTy =
           RankedTensorType::get(newWeightShape, weightTy.getElementType());
-      weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
+      weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
                                                   weightPermAttr);
     }
 
@@ -384,7 +384,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     auto strideAttr = rewriter.getI64TensorAttr(stride);
     auto dilationAttr = rewriter.getI64TensorAttr(dilation);
 
-    Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), accETy, filteredDims);
 
     Value broadcastBias =
@@ -394,8 +394,8 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       auto iZp = rewriter.getI32IntegerAttr(inputZpVal);
       auto kZp = rewriter.getI32IntegerAttr(weightZpVal);
 
-      auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
+      auto iZpVal = arith::ConstantOp::create(rewriter, loc, iZp);
+      auto kZpVal = arith::ConstantOp::create(rewriter, loc, kZp);
 
       Value conv =
           rewriter
@@ -417,7 +417,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     // We may need to truncate back to the result type if the accumulator was
     // wider than the result.
     if (resultTy != accTy)
-      conv = rewriter.create<tosa::CastOp>(loc, resultTy, conv);
+      conv = tosa::CastOp::create(rewriter, loc, resultTy, conv);
 
     rewriter.replaceOp(op, conv);
     return success();
@@ -526,15 +526,15 @@ class DepthwiseConvConverter
                               accETy);
 
     auto resultZeroAttr = rewriter.getZeroAttr(accETy);
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, linalgConvTy.getShape(), accETy, filteredDims);
-    Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
+    Value zero = arith::ConstantOp::create(rewriter, loc, resultZeroAttr);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
                            .result();
 
-    Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultETy, filteredDims);
 
     // Broadcast the initial value to the output tensor before convolving.
@@ -553,7 +553,7 @@ class DepthwiseConvConverter
       // We may need to truncate back to the result type if the accumulator was
       // wider than the result.
       if (accETy != resultETy)
-        conv = rewriter.create<tosa::CastOp>(
+        conv = tosa::CastOp::create(rewriter,
             loc,
             RankedTensorType::get(cast<ShapedType>(conv.getType()).getShape(),
                                   resultETy),
@@ -561,7 +561,7 @@ class DepthwiseConvConverter
 
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
+      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
           loc, resultTy, conv, reassociationMap);
 
       Value result =
@@ -574,20 +574,20 @@ class DepthwiseConvConverter
                       ValueRange args) {
                     Value added;
                     if (llvm::isa<FloatType>(inputETy))
-                      added = nestedBuilder.create<arith::AddFOp>(loc, args[0],
+                      added = arith::AddFOp::create(nestedBuilder, loc, args[0],
                                                                   args[1]);
                     else
-                      added = nestedBuilder.create<arith::AddIOp>(loc, args[0],
+                      added = arith::AddIOp::create(nestedBuilder, loc, args[0],
                                                                   args[1]);
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
+                    linalg::YieldOp::create(nestedBuilder, nestedLoc, added);
                   })
               .getResult(0);
       rewriter.replaceOp(op, result);
     } else {
       IntegerAttr iZp = rewriter.getI32IntegerAttr(inputZpVal);
       IntegerAttr wZp = rewriter.getI32IntegerAttr(weightZpVal);
-      auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
-      auto kZpVal = rewriter.create<arith::ConstantOp>(loc, wZp);
+      auto iZpVal = arith::ConstantOp::create(rewriter, loc, iZp);
+      auto kZpVal = arith::ConstantOp::create(rewriter, loc, wZp);
       Value conv =
           rewriter
               .create<linalg::DepthwiseConv2DNhwcHwcmQOp>(
@@ -596,7 +596,7 @@ class DepthwiseConvConverter
               .getResult(0);
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
+      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
           loc, resultTy, conv, reassociationMap);
       Value result = linalgIntBroadcastExtSIAdd(
           rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
@@ -621,22 +621,22 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
     dynDims.resize(cast<ShapedType>(op->getResult(0).getType()).getRank());
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(0)) {
-      dynDims[0] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 0);
+      dynDims[0] = tensor::DimOp::create(rewriter, loc, op->getOperand(0), 0);
     }
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(1)) {
-      dynDims[1] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 1);
+      dynDims[1] = tensor::DimOp::create(rewriter, loc, op->getOperand(0), 1);
     }
 
     if (!outputTy.hasRank() || outputTy.isDynamicDim(2)) {
-      dynDims[2] = rewriter.create<tensor::DimOp>(loc, op->getOperand(1), 2);
+      dynDims[2] = tensor::DimOp::create(rewriter, loc, op->getOperand(1), 2);
     }
 
     SmallVector<Value> filteredDims = condenseValues(dynDims);
 
     auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
-    Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
@@ -670,9 +670,9 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
       return success();
     }
 
-    auto aZp = rewriter.create<arith::ConstantOp>(
+    auto aZp = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI32IntegerAttr(aZpVal));
-    auto bZp = rewriter.create<arith::ConstantOp>(
+    auto bZp = arith::ConstantOp::create(rewriter,
         loc, rewriter.getI32IntegerAttr(bZpVal));
     rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
         op, TypeRange{op.getType()},
@@ -702,7 +702,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     // Batch dimension
     if (resultTy.isDynamicDim(0))
-      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+      dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
     // Height/width dimensions
     for (int64_t dim : {1, 2}) {
@@ -713,10 +713,10 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       int64_t index = dim - 1;
 
       // Input height/width
-      Value ihw = rewriter.create<tensor::DimOp>(loc, input, dim);
+      Value ihw = tensor::DimOp::create(rewriter, loc, input, dim);
 
       // Kernel height/width
-      Value khw = rewriter.create<arith::ConstantIndexOp>(loc, kernel[index]);
+      Value khw = arith::ConstantIndexOp::create(rewriter, loc, kernel[index]);
 
       // Output height/width
       Value ohw = getConvOrPoolOutputDim(loc, ihw, pad[index * 2],
@@ -727,7 +727,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     // Channel dimension
     if (resultTy.isDynamicDim(3))
-      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 3));
+      dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 3));
 
     return dynamicDims;
   }
@@ -776,7 +776,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
 
     Value paddedInput = applyPad(loc, input, pad, initialAttr, rewriter);
 
-    Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
+    Value initialValue = arith::ConstantOp::create(rewriter, loc, initialAttr);
 
     ArrayRef<int64_t> kernel = op.getKernel();
     ArrayRef<int64_t> stride = op.getStride();
@@ -785,15 +785,15 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
 
     Value filledEmptyTensor =
-        rewriter.create<linalg::FillOp>(loc, initialValue, emptyTensor)
+        linalg::FillOp::create(rewriter, loc, initialValue, emptyTensor)
             .result();
 
     Value fakeWindowDims =
-        rewriter.create<tensor::EmptyOp>(loc, kernel, resultETy);
+        tensor::EmptyOp::create(rewriter, loc, kernel, resultETy);
 
     if (isUnsigned) {
       rewriter.replaceOpWithNewOp<linalg::PoolingNhwcMaxUnsignedOp>(
@@ -802,7 +802,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       return llvm::success();
     }
 
-    auto resultOp = rewriter.create<linalg::PoolingNhwcMaxOp>(
+    auto resultOp = linalg::PoolingNhwcMaxOp::create(rewriter,
         op->getLoc(), ArrayRef<Type>{resultTy},
         ValueRange{paddedInput, fakeWindowDims}, filledEmptyTensor, strideAttr,
         dilationAttr);
@@ -823,7 +823,7 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     // it to include the appropriate checks. If the current value is NaN the
     // old value of pool will be taken otherwise we use the result.
     if (nanMode == "IGNORE") {
-      auto genericOp = rewriter.create<linalg::GenericOp>(
+      auto genericOp = linalg::GenericOp::create(rewriter,
           loc, resultOp.getType(0), resultOp.getInputs(), resultOp.getOutputs(),
           resultOp.getIndexingMapsArray(), resultOp.getIteratorTypesArray(),
           [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
@@ -833,12 +833,12 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
             auto &oldMaxOp = *resultOp.getBlock()->begin();
             map.map(oldArgs, blockArgs);
             auto *newOp = opBuilder.clone(oldMaxOp, map);
-            Value isNaN = opBuilder.create<arith::CmpFOp>(
+            Value isNaN = arith::CmpFOp::create(opBuilder,
                 loc, arith::CmpFPredicate::UNO, blockArgs.front(),
                 blockArgs.front());
-            auto selectOp = opBuilder.create<arith::SelectOp>(
+            auto selectOp = arith::SelectOp::create(opBuilder,
                 loc, isNaN, blockArgs.back(), newOp->getResult(0));
-            opBuilder.create<linalg::YieldOp>(loc, selectOp.getResult());
+            linalg::YieldOp::create(opBuilder, loc, selectOp.getResult());
           });
       rewriter.replaceOp(resultOp, genericOp);
     }
@@ -894,7 +894,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Value paddedInput = applyPad(loc, input, pad, padAttr, rewriter);
 
     auto initialAttr = rewriter.getZeroAttr(accETy);
-    Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
+    Value initialValue = arith::ConstantOp::create(rewriter, loc, initialAttr);
 
     ArrayRef<int64_t> kernel = op.getKernel();
     ArrayRef<int64_t> stride = op.getStride();
@@ -903,7 +903,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value poolEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value poolEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, accTy.getShape(), accETy, dynamicDims);
 
     Value filledEmptyTensor =
@@ -913,7 +913,7 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             .result();
 
     Value fakeWindowDims =
-        rewriter.create<tensor::EmptyOp>(loc, kernel, accETy);
+        tensor::EmptyOp::create(rewriter, loc, kernel, accETy);
 
     // Sum across the pooled region.
     Value poolingOp = rewriter
@@ -925,24 +925,24 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
 
     // Normalize the summed value by the number of elements grouped in each
     // pool.
-    Value iH = rewriter.create<tensor::DimOp>(loc, poolingOp, 1);
-    Value iW = rewriter.create<tensor::DimOp>(loc, poolingOp, 2);
+    Value iH = tensor::DimOp::create(rewriter, loc, poolingOp, 1);
+    Value iW = tensor::DimOp::create(rewriter, loc, poolingOp, 2);
 
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    iH = rewriter.create<arith::SubIOp>(loc, iH, one);
-    iW = rewriter.create<arith::SubIOp>(loc, iW, one);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    iH = arith::SubIOp::create(rewriter, loc, iH, one);
+    iW = arith::SubIOp::create(rewriter, loc, iW, one);
 
-    Value genericEmptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value genericEmptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultTy.getShape(), resultETy, dynamicDims);
 
     auto affineMap = rewriter.getMultiDimIdentityMap(resultTy.getRank());
-    auto genericOp = rewriter.create<linalg::GenericOp>(
+    auto genericOp = linalg::GenericOp::create(rewriter,
         loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
         ValueRange{genericEmptyTensor},
         ArrayRef<AffineMap>({affineMap, affineMap}),
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
-          auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+          auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
           // Determines what the portion of valid input is covered by the
           // kernel.
@@ -950,30 +950,30 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             if (pad == 0)
               return valid;
 
-            auto padVal = rewriter.create<arith::ConstantIndexOp>(loc, pad);
-            Value dpos = rewriter.create<arith::SubIOp>(loc, pos, padVal);
+            auto padVal = arith::ConstantIndexOp::create(rewriter, loc, pad);
+            Value dpos = arith::SubIOp::create(rewriter, loc, pos, padVal);
 
-            Value offset = rewriter.create<arith::MinSIOp>(loc, dpos, zero);
-            return rewriter.create<arith::AddIOp>(loc, valid, offset)
+            Value offset = arith::MinSIOp::create(rewriter, loc, dpos, zero);
+            return arith::AddIOp::create(rewriter, loc, valid, offset)
                 ->getResult(0);
           };
 
           auto coverageFn = [&](int64_t i, Value isize) -> Value {
             Value strideVal =
-                rewriter.create<arith::ConstantIndexOp>(loc, stride[i - 1]);
+                arith::ConstantIndexOp::create(rewriter, loc, stride[i - 1]);
             Value val =
-                rewriter.create<arith::ConstantIndexOp>(loc, kernel[i - 1]);
+                arith::ConstantIndexOp::create(rewriter, loc, kernel[i - 1]);
 
             // Find the position relative to the input tensor's ends.
-            Value left = rewriter.create<linalg::IndexOp>(loc, i);
-            Value right = rewriter.create<arith::SubIOp>(loc, isize, left);
-            left = rewriter.create<arith::MulIOp>(loc, left, strideVal);
-            right = rewriter.create<arith::MulIOp>(loc, right, strideVal);
+            Value left = linalg::IndexOp::create(rewriter, loc, i);
+            Value right = arith::SubIOp::create(rewriter, loc, isize, left);
+            left = arith::MulIOp::create(rewriter, loc, left, strideVal);
+            right = arith::MulIOp::create(rewriter, loc, right, strideVal);
 
             // Determine how much padding was included.
             val = padFn(val, left, pad[i * 2]);
             val = padFn(val, right, pad[i * 2 + 1]);
-            return rewriter.create<arith::MaxSIOp>(loc, one, val);
+            return arith::MaxSIOp::create(rewriter, loc, one, val);
           };
 
           // Compute the indices from either end.
@@ -981,70 +981,70 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
           Value kW3 = coverageFn(2, iW);
 
           // Compute the total number of elements and normalize.
-          auto count = rewriter.create<arith::IndexCastOp>(
+          auto count = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getI32Type(),
-              rewriter.create<arith::MulIOp>(loc, kH3, kW3));
+              arith::MulIOp::create(rewriter, loc, kH3, kW3));
 
           // Divide by the number of summed values. For floats this is just
           // a div however for quantized values input normalization had
           // to be applied.
           Value poolVal = args[0];
           if (isa<FloatType>(accETy)) {
-            auto countF = rewriter.create<arith::SIToFPOp>(loc, accETy, count);
-            poolVal = rewriter.create<arith::DivFOp>(loc, poolVal, countF)
+            auto countF = arith::SIToFPOp::create(rewriter, loc, accETy, count);
+            poolVal = arith::DivFOp::create(rewriter, loc, poolVal, countF)
                           ->getResult(0);
             if (accETy.getIntOrFloatBitWidth() >
                 resultETy.getIntOrFloatBitWidth())
               poolVal =
-                  rewriter.create<arith::TruncFOp>(loc, resultETy, poolVal);
+                  arith::TruncFOp::create(rewriter, loc, resultETy, poolVal);
           } else {
 
             // If we have quantization information we need to apply an offset
             // for the input zp value.
             if (inputZpVal != 0) {
-              auto inputZp = rewriter.create<arith::ConstantOp>(
+              auto inputZp = arith::ConstantOp::create(rewriter,
                   loc, b.getIntegerAttr(accETy, inputZpVal));
               Value offset =
-                  rewriter.create<arith::MulIOp>(loc, accETy, count, inputZp);
+                  arith::MulIOp::create(rewriter, loc, accETy, count, inputZp);
               poolVal =
-                  rewriter.create<arith::SubIOp>(loc, accETy, poolVal, offset);
+                  arith::SubIOp::create(rewriter, loc, accETy, poolVal, offset);
             }
 
             // Compute: k = 32 - count_leading_zeros(value - 1)
-            Value one32 = rewriter.create<arith::ConstantOp>(
+            Value one32 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI32IntegerAttr(1));
-            Value thirtyTwo32 = rewriter.create<arith::ConstantOp>(
+            Value thirtyTwo32 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI32IntegerAttr(32));
 
             Value countSubOne =
-                rewriter.create<arith::SubIOp>(loc, count, one32);
+                arith::SubIOp::create(rewriter, loc, count, one32);
             Value leadingZeros =
-                rewriter.create<math::CountLeadingZerosOp>(loc, countSubOne);
+                math::CountLeadingZerosOp::create(rewriter, loc, countSubOne);
             Value k =
-                rewriter.create<arith::SubIOp>(loc, thirtyTwo32, leadingZeros);
+                arith::SubIOp::create(rewriter, loc, thirtyTwo32, leadingZeros);
 
             // Compute: numerator = ((1 << 30) + 1) << k
             Value k64 =
-                rewriter.create<arith::ExtUIOp>(loc, rewriter.getI64Type(), k);
-            Value thirtyShiftPlusOne = rewriter.create<arith::ConstantOp>(
+                arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), k);
+            Value thirtyShiftPlusOne = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
             Value numerator =
-                rewriter.create<arith::ShLIOp>(loc, thirtyShiftPlusOne, k64);
+                arith::ShLIOp::create(rewriter, loc, thirtyShiftPlusOne, k64);
 
             // Compute: scale.multiplier = numerator / value;
-            Value count64 = rewriter.create<arith::ExtUIOp>(
+            Value count64 = arith::ExtUIOp::create(rewriter,
                 loc, rewriter.getI64Type(), count);
             Value multiplier =
-                rewriter.create<arith::DivUIOp>(loc, numerator, count64);
-            multiplier = rewriter.create<arith::TruncIOp>(
+                arith::DivUIOp::create(rewriter, loc, numerator, count64);
+            multiplier = arith::TruncIOp::create(rewriter,
                 loc, rewriter.getI32Type(), multiplier);
 
             // Compute: scale.shift = 30 + k
             Value k8 =
-                rewriter.create<arith::TruncIOp>(loc, rewriter.getI8Type(), k);
-            Value thirty8 = rewriter.create<arith::ConstantOp>(
+                arith::TruncIOp::create(rewriter, loc, rewriter.getI8Type(), k);
+            Value thirty8 = arith::ConstantOp::create(rewriter,
                 loc, rewriter.getI8IntegerAttr(30));
-            Value shift = rewriter.create<arith::AddIOp>(loc, k8, thirty8);
+            Value shift = arith::AddIOp::create(rewriter, loc, k8, thirty8);
 
             auto scaled =
                 rewriter
@@ -1056,19 +1056,19 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // If we have quantization information we need to apply output
             // zeropoint.
             if (outputZpVal != 0) {
-              auto outputZp = rewriter.create<arith::ConstantOp>(
+              auto outputZp = arith::ConstantOp::create(rewriter,
                   loc, b.getIntegerAttr(scaled.getType(), outputZpVal));
-              scaled = rewriter.create<arith::AddIOp>(loc, scaled, outputZp)
+              scaled = arith::AddIOp::create(rewriter, loc, scaled, outputZp)
                            .getResult();
             }
 
             // Apply Clip.
             int64_t outBitwidth = resultETy.getIntOrFloatBitWidth();
 
-            auto min = rewriter.create<arith::ConstantIntOp>(
+            auto min = arith::ConstantIntOp::create(rewriter,
                 loc, accETy,
                 APInt::getSignedMinValue(outBitwidth).getSExtValue());
-            auto max = rewriter.create<arith::ConstantIntOp>(
+            auto max = arith::ConstantIntOp::create(rewriter,
                 loc, accETy,
                 APInt::getSignedMaxValue(outBitwidth).getSExtValue());
             auto clamp = clampIntHelper(loc, scaled, min, max, rewriter,
@@ -1078,11 +1078,11 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // Convert type.
             if (resultETy != clamp.getType()) {
               poolVal =
-                  rewriter.create<arith::TruncIOp>(loc, resultETy, poolVal);
+                  arith::TruncIOp::create(rewriter, loc, resultETy, poolVal);
             }
           }
 
-          rewriter.create<linalg::YieldOp>(loc, poolVal);
+          linalg::YieldOp::create(rewriter, loc, poolVal);
         });
 
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -1107,7 +1107,7 @@ class TransposeConverter : public OpRewritePattern<tosa::TransposeOp> {
     auto permutedSizes =
         applyTOSAPermutation<OpFoldResult>(inputSizes, constantPerms);
 
-    auto permutedInit = rewriter.create<tensor::EmptyOp>(
+    auto permutedInit = tensor::EmptyOp::create(rewriter,
         loc, permutedSizes, op.getInput1().getType().getElementType());
     rewriter.replaceOpWithNewOp<linalg::TransposeOp>(
         op, op.getInput1(), permutedInit,
diff --git a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
index 7dbccd19a0518..cd36aadce410a 100644
--- a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
+++ b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
@@ -27,7 +27,7 @@ class VariableOpConverter : public OpRewritePattern<tosa::VariableOp> {
   LogicalResult matchAndRewrite(tosa::VariableOp op,
                                 PatternRewriter &rewriter) const final {
     auto variableType = tosa::getVariableType(op);
-    auto newVariable = rewriter.create<mlir::ml_program::GlobalOp>(
+    auto newVariable = mlir::ml_program::GlobalOp::create(rewriter,
         op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
         op.getInitialValueAttr(), /*sym_visibility=*/nullptr);
     newVariable.setPrivate();
@@ -45,7 +45,7 @@ class VariableWriteOpConverter
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableWrite = rewriter.create<ml_program::GlobalStoreOp>(
+    auto newVariableWrite = ml_program::GlobalStoreOp::create(rewriter,
         op.getLoc(), globalSymbolRef, op.getInput1());
     rewriter.replaceOp(op, newVariableWrite);
     return success();
@@ -60,7 +60,7 @@ class VariableReadOpConverter : public OpRewritePattern<tosa::VariableReadOp> {
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableRead = rewriter.create<ml_program::GlobalLoadOp>(
+    auto newVariableRead = ml_program::GlobalLoadOp::create(rewriter,
         op.getLoc(), op.getType(), globalSymbolRef);
     rewriter.replaceOp(op, newVariableRead);
 
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index 03f9d20ad69de..9188b8d1368da 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -30,7 +30,7 @@ static void inlineIfCase(Region &srcRegion, Region &dstRegion,
 
   auto yield = cast<YieldOp>(headBlock->getTerminator());
   rewriter.setInsertionPoint(yield);
-  rewriter.create<scf::YieldOp>(yield.getLoc(), yield.getInputs());
+  scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
   rewriter.eraseOp(yield);
 
   headBlock->eraseArguments(0, headBlock->getNumArguments());
@@ -47,12 +47,12 @@ static void inlineWhileCase(Region &srcRegion, Region &dstRegion,
   rewriter.setInsertionPoint(yield);
   if (isCond) {
     auto condition =
-        rewriter.create<tensor::ExtractOp>(yield.getLoc(), yield.getOperand(0));
-    rewriter.create<scf::ConditionOp>(yield.getLoc(), condition,
+        tensor::ExtractOp::create(rewriter, yield.getLoc(), yield.getOperand(0));
+    scf::ConditionOp::create(rewriter, yield.getLoc(), condition,
                                       headBlock->getArguments());
   } else {
     rewriter.setInsertionPoint(yield);
-    rewriter.create<scf::YieldOp>(yield.getLoc(), yield.getInputs());
+    scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
   }
   rewriter.eraseOp(yield);
 }
@@ -66,8 +66,8 @@ class IfOpConverter : public OpRewritePattern<tosa::IfOp> {
   LogicalResult matchAndRewrite(tosa::IfOp op,
                                 PatternRewriter &rewriter) const final {
     auto condition =
-        rewriter.create<tensor::ExtractOp>(op.getLoc(), op.getCondition());
-    auto newIf = rewriter.create<scf::IfOp>(op.getLoc(), op.getResultTypes(),
+        tensor::ExtractOp::create(rewriter, op.getLoc(), op.getCondition());
+    auto newIf = scf::IfOp::create(rewriter, op.getLoc(), op.getResultTypes(),
                                             condition, true);
 
     inlineIfCase(op.getThenGraph(), newIf.getThenRegion(), op.getInputList(),
@@ -88,7 +88,7 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
 
   static Value createIndexConst(OpBuilder &builder, Location loc,
                                 int64_t value) {
-    return builder.create<arith::ConstantIndexOp>(loc, value);
+    return arith::ConstantIndexOp::create(builder, loc, value);
   }
 
 public:
@@ -119,8 +119,8 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       auto n = ivs[0];
 
       // Read the index and cast it to index type
-      auto index = builder.create<tensor::ExtractOp>(loc, indices, ivs);
-      auto castIndex = builder.create<arith::IndexCastOp>(
+      auto index = tensor::ExtractOp::create(builder, loc, indices, ivs);
+      auto castIndex = arith::IndexCastOp::create(builder,
           loc, builder.getIndexType(), index);
 
       // Offset, sizes, and strides for the input tensor
@@ -130,12 +130,12 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       llvm::SmallVector<Value> sizes = {one, one, dimC};
       llvm::SmallVector<Value> strides = {one, one, one};
 
-      auto slice = builder.create<tensor::ExtractSliceOp>(
+      auto slice = tensor::ExtractSliceOp::create(builder,
           loc, input, inputOffset, sizes, strides);
 
       // Insert the slice into the output accumulator tensor.
       llvm::SmallVector<Value> outputOffset = {n, castIndex, zero};
-      auto updated = builder.create<tensor::InsertSliceOp>(
+      auto updated = tensor::InsertSliceOp::create(builder,
           loc, slice, args[0], outputOffset, sizes, strides);
 
       return {updated};
@@ -155,7 +155,7 @@ class WhileOpConverter : public OpRewritePattern<tosa::WhileOp> {
 
   LogicalResult matchAndRewrite(tosa::WhileOp op,
                                 PatternRewriter &rewriter) const final {
-    auto newWhile = rewriter.create<scf::WhileOp>(
+    auto newWhile = scf::WhileOp::create(rewriter,
         op.getLoc(), op.getResultTypes(), op.getInputList());
     rewriter.createBlock(&newWhile.getBefore());
     rewriter.createBlock(&newWhile.getAfter());
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index c6cbcb0f8ab2b..a7219d423a77a 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -308,13 +308,13 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
       if (ShapedType::isStatic(sizes.back()))
         continue;
 
-      auto dim = rewriter.create<tensor::DimOp>(loc, input, index);
-      auto offset = rewriter.create<arith::ConstantOp>(
+      auto dim = tensor::DimOp::create(rewriter, loc, input, index);
+      auto offset = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(sliceStarts[index]));
-      dynSizes.push_back(rewriter.create<arith::SubIOp>(loc, dim, offset));
+      dynSizes.push_back(arith::SubIOp::create(rewriter, loc, dim, offset));
     }
 
-    auto newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}), dynSizes,
         ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
         rewriter.getDenseI64ArrayAttr(sizes),
@@ -361,7 +361,7 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
 
     Value padConstant = rewriter.createOrFold<tensor::ExtractOp>(
         loc, padOp.getPadConst(),
-        ValueRange({rewriter.create<arith::ConstantIndexOp>(loc, 0)}));
+        ValueRange({arith::ConstantIndexOp::create(rewriter, loc, 0)}));
 
     if (!padConstant) {
       return rewriter.notifyMatchFailure(
@@ -375,15 +375,15 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
     highValues.reserve(rank);
 
     for (int i = 0; i < rank; i++) {
-      Value lowVal = rewriter.create<arith::ConstantOp>(
+      Value lowVal = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(paddingVals[2 * i]));
-      Value highVal = rewriter.create<arith::ConstantOp>(
+      Value highVal = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexAttr(paddingVals[2 * i + 1]));
       lowValues.push_back(lowVal);
       highValues.push_back(highVal);
     }
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, padOp.getType(), input, lowValues, highValues, padConstant);
 
     rewriter.replaceOp(padOp, newPadOp.getResult());
@@ -402,7 +402,7 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
     Location loc = op.getLoc();
     int axis = op.getAxis();
     Value axisValue =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(axis));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(axis));
     int64_t rank = resultType.getRank();
 
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
@@ -439,7 +439,7 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
       }
     }
 
-    Value result = rewriter.create<tensor::EmptyOp>(
+    Value result = tensor::EmptyOp::create(rewriter,
         loc, resultType.getShape(), resultType.getElementType(), dynDims);
 
     for (auto [arg, offset] : llvm::zip(adaptor.getOperands(), axisOffsets)) {
diff --git a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
index d6f9495b2567c..9e4eb347b3689 100644
--- a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
+++ b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
@@ -226,7 +226,7 @@ struct BroadcastOpToArmSMELowering
         (srcVectorType && (srcVectorType.getRank() == 0))) {
       // Broadcast scalar or 0-d vector to 1-d vector.
       VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-      broadcastOp1D = rewriter.create<vector::BroadcastOp>(
+      broadcastOp1D = vector::BroadcastOp::create(rewriter,
           loc, tileSliceType, broadcastOp.getSource());
     } else if (srcVectorType && (srcVectorType.getRank() == 1))
       // Value to broadcast is already a 1-d vector, nothing to do.
@@ -234,13 +234,13 @@ struct BroadcastOpToArmSMELowering
     else
       return failure();
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to broadcast the value
       // to each tile slice.
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
@@ -292,14 +292,14 @@ struct SplatOpToArmSMELowering : public OpRewritePattern<vector::SplatOp> {
 
     // First, broadcast the scalar to a 1-d vector.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    Value broadcastOp1D = rewriter.create<vector::BroadcastOp>(
+    Value broadcastOp1D = vector::BroadcastOp::create(rewriter,
         loc, tileSliceType, splatOp.getInput());
 
-    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
+    auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
-      auto nextTile = b.create<arm_sme::InsertTileSliceOp>(
+      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
           loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
@@ -370,21 +370,21 @@ struct TransposeOpToArmSMELowering
 
     // Allocate buffer to store input tile to.
     Value vscale =
-        rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-    Value minTileSlices = rewriter.create<arith::ConstantOp>(
+        vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+    Value minTileSlices = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexAttr(tileType.getDimSize(0)));
     Value c0 =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(0));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(0));
     Value numTileSlices =
-        rewriter.create<arith::MulIOp>(loc, vscale, minTileSlices);
+        arith::MulIOp::create(rewriter, loc, vscale, minTileSlices);
     auto bufferType =
         MemRefType::get({ShapedType::kDynamic, ShapedType::kDynamic},
                         tileType.getElementType());
-    auto buffer = rewriter.create<memref::AllocaOp>(
+    auto buffer = memref::AllocaOp::create(rewriter,
         loc, bufferType, ValueRange{numTileSlices, numTileSlices});
 
     // Store input tile.
-    auto tileStoreOp = rewriter.create<arm_sme::TileStoreOp>(
+    auto tileStoreOp = arm_sme::TileStoreOp::create(rewriter,
         loc, input, buffer, ValueRange{c0, c0});
 
     // Reload input tile vertically.
@@ -489,9 +489,9 @@ struct VectorOuterProductToArmSMELowering
 
     VectorType operandMaskType = VectorType::Builder(maskType).dropDim(0);
     Value lhsMask =
-        rewriter.create<vector::CreateMaskOp>(loc, operandMaskType, lhsMaskDim);
+        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, lhsMaskDim);
     Value rhsMask =
-        rewriter.create<vector::CreateMaskOp>(loc, operandMaskType, rhsMaskDim);
+        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, rhsMaskDim);
 
     return std::make_pair(lhsMask, rhsMask);
   }
@@ -531,7 +531,7 @@ struct VectorExtractToArmSMELowering
     }
 
     Value sliceIndex = vector::getAsValues(rewriter, loc, position[0]).front();
-    auto extractTileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+    auto extractTileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
         loc, sourceVector, sliceIndex);
 
     if (position.size() == 1) {
@@ -593,9 +593,9 @@ struct VectorInsertToArmSMELowering
     if (position.size() == 2) {
       // Two indices case: Insert single element into tile.
       // We need to first extract the existing slice and update the element.
-      tileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+      tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
           loc, insertOp.getDest(), sliceIndex);
-      tileSlice = rewriter.create<vector::InsertOp>(loc, source, tileSlice,
+      tileSlice = vector::InsertOp::create(rewriter, loc, source, tileSlice,
                                                     position[1]);
     }
 
@@ -642,22 +642,22 @@ struct VectorPrintToArmSMELowering : public OpRewritePattern<vector::PrintOp> {
     auto loc = printOp.getLoc();
 
     // Create a loop over the rows of the tile.
-    auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
+    auto vscale = vector::VectorScaleOp::create(rewriter, loc);
     auto minTileRows =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorType.getDimSize(0));
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto upperBound = rewriter.create<arith::MulIOp>(loc, minTileRows, vscale);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        arith::ConstantIndexOp::create(rewriter, loc, vectorType.getDimSize(0));
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto upperBound = arith::MulIOp::create(rewriter, loc, minTileRows, vscale);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     {
       // Loop body.
       rewriter.setInsertionPointToStart(forOp.getBody());
       // Extract the current row from the tile.
       Value rowIndex = forOp.getInductionVar();
-      auto tileSlice = rewriter.create<arm_sme::ExtractTileSliceOp>(
+      auto tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
           loc, printOp.getSource(), rowIndex);
       // Print the row with a 1D vector.print.
-      rewriter.create<vector::PrintOp>(loc, tileSlice,
+      vector::PrintOp::create(rewriter, loc, tileSlice,
                                        printOp.getPunctuation());
     }
 
@@ -707,7 +707,7 @@ struct FoldTransferWriteOfExtractTileSlice
     Value mask = writeOp.getMask();
     if (!mask) {
       auto maskType = writeOp.getVectorType().clone(rewriter.getI1Type());
-      mask = rewriter.create<arith::ConstantOp>(
+      mask = arith::ConstantOp::create(rewriter,
           writeOp.getLoc(), maskType, DenseElementsAttr::get(maskType, true));
     }
 
@@ -776,9 +776,9 @@ struct ExtractFromCreateMaskToPselLowering
     // Create the two 1-D masks at the location of the 2-D create_mask (which is
     // usually outside a loop). This prevents the need for later hoisting.
     rewriter.setInsertionPoint(createMaskOp);
-    auto rowMask = rewriter.create<vector::CreateMaskOp>(
+    auto rowMask = vector::CreateMaskOp::create(rewriter,
         loc, rowMaskType, createMaskOp.getOperand(0));
-    auto colMask = rewriter.create<vector::CreateMaskOp>(
+    auto colMask = vector::CreateMaskOp::create(rewriter,
         loc, colMaskType, createMaskOp.getOperand(1));
 
     rewriter.setInsertionPoint(extractOp);
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 18adaa793787c..5b3e67509672a 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -412,22 +412,22 @@ struct PrepareContractToGPUMMA
     if (maps == infer({{m, k}, {k, n}, {m, n}}))
       return rewriter.notifyMatchFailure(op, "contraction already prepared");
     if (maps == infer({{m, k}, {n, k}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
       std::swap(rhs, lhs);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
       std::swap(rhs, lhs);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
       std::swap(lhs, rhs);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
       std::swap(lhs, rhs);
     } else {
@@ -494,13 +494,13 @@ struct CombineTransferReadOpTranspose final
     // Fuse through the integer extend op.
     if (extOp) {
       if (isa<arith::ExtSIOp>(extOp))
-        result = rewriter.create<arith::ExtSIOp>(loc, op.getType(), result)
+        result = arith::ExtSIOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
       else if (isa<arith::ExtUIOp>(extOp))
-        result = rewriter.create<arith::ExtUIOp>(loc, op.getType(), result)
+        result = arith::ExtUIOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
       else
-        result = rewriter.create<arith::ExtFOp>(loc, op.getType(), result)
+        result = arith::ExtFOp::create(rewriter, loc, op.getType(), result)
                      .getResult();
     }
 
@@ -579,7 +579,7 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op,
   }
   gpu::MMAMatrixType type =
       gpu::MMAMatrixType::get(op.getVectorType().getShape(), elType, fragType);
-  Value load = rewriter.create<gpu::SubgroupMmaLoadMatrixOp>(
+  Value load = gpu::SubgroupMmaLoadMatrixOp::create(rewriter,
       op.getLoc(), type, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride),
       isTranspose ? rewriter.getUnitAttr() : UnitAttr());
@@ -610,7 +610,7 @@ convertTransferWriteOp(RewriterBase &rewriter, vector::TransferWriteOp op,
   }
 
   Value matrix = it->second;
-  auto store = rewriter.create<gpu::SubgroupMmaStoreMatrixOp>(
+  auto store = gpu::SubgroupMmaStoreMatrixOp::create(rewriter,
       op.getLoc(), matrix, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride), /*transpose=*/UnitAttr());
   (void)store;
@@ -661,7 +661,7 @@ convertConstantOpMmaSync(RewriterBase &rewriter, arith::ConstantOp op,
     return rewriter.notifyMatchFailure(op, "not a splat");
   }
 
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       op.getLoc(), vectorType,
       DenseElementsAttr::get(vectorType, dense.getSplatValue<Attribute>()));
   valueMapping[op.getResult()] = result;
@@ -743,7 +743,7 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   }
 
   // Adjust the load offset.
-  auto laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  auto laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
   FailureOr<AffineMap> offsets =
       nvgpu::getLaneIdToLdMatrixMatrixCoord(rewriter, loc, *params);
   if (failed(offsets)) {
@@ -757,7 +757,7 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   getXferIndices<vector::TransferReadOp>(rewriter, op, *offsets, {laneId},
                                          indices);
 
-  nvgpu::LdMatrixOp newOp = rewriter.create<nvgpu::LdMatrixOp>(
+  nvgpu::LdMatrixOp newOp = nvgpu::LdMatrixOp::create(rewriter,
       loc, vectorType, op.getBase(), indices, *transpose, params->numTiles);
   valueMapping[op] = newOp->getResult(0);
   return success();
@@ -782,17 +782,17 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
             "conversion to distributed non-ldmatrix compatible load");
   }
 
-  Value laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   // This is the individual element type.
   Type loadedElType = regInfo->registerLLVMType;
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
 
-  Value fill = rewriter.create<arith::ConstantOp>(
+  Value fill = arith::ConstantOp::create(rewriter,
       op.getLoc(), vectorType.getElementType(),
       rewriter.getZeroAttr(vectorType.getElementType()));
   Value result =
-      rewriter.create<vector::SplatOp>(op.getLoc(), fill, vectorType);
+      vector::SplatOp::create(rewriter, op.getLoc(), fill, vectorType);
 
   bool isTransposeLoad = !op.getPermutationMap().isMinorIdentity();
 
@@ -809,16 +809,16 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       if (failed(coords))
         return rewriter.notifyMatchFailure(op, "no coords");
 
-      Value logicalValueId = rewriter.create<arith::ConstantOp>(
+      Value logicalValueId = arith::ConstantOp::create(rewriter,
           loc, rewriter.getIndexType(),
           rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
       SmallVector<Value, 4> newIndices;
       getXferIndices<vector::TransferReadOp>(
           rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
 
-      Value el = rewriter.create<vector::LoadOp>(loc, loadedElType,
+      Value el = vector::LoadOp::create(rewriter, loc, loadedElType,
                                                  op.getBase(), newIndices);
-      result = rewriter.create<vector::InsertOp>(loc, el, result, i);
+      result = vector::InsertOp::create(rewriter, loc, el, result, i);
     }
   } else {
     if (auto vecType = dyn_cast<VectorType>(loadedElType)) {
@@ -828,7 +828,7 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       for (unsigned innerIdx = 0; innerIdx < vectorType.getShape()[1];
            innerIdx++) {
 
-        Value logicalValueId = rewriter.create<arith::ConstantOp>(
+        Value logicalValueId = arith::ConstantOp::create(rewriter,
             loc, rewriter.getIndexType(),
             rewriter.getIndexAttr(i * regInfo->elementsPerRegister + innerIdx));
         FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
@@ -839,9 +839,9 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
         SmallVector<Value, 4> newIndices;
         getXferIndices<vector::TransferReadOp>(
             rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
-        Value el = rewriter.create<memref::LoadOp>(op.getLoc(), loadedElType,
+        Value el = memref::LoadOp::create(rewriter, op.getLoc(), loadedElType,
                                                    op.getBase(), newIndices);
-        result = rewriter.create<vector::InsertOp>(
+        result = vector::InsertOp::create(rewriter,
             op.getLoc(), el, result, ArrayRef<int64_t>{i, innerIdx});
       }
     }
@@ -916,10 +916,10 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
     return rewriter.notifyMatchFailure(op, "not mma sync reg info");
 
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
-  Value laneId = rewriter.create<gpu::LaneIdOp>(loc, /*upperBound=*/nullptr);
+  Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   for (unsigned i = 0; i < vectorType.getShape()[0]; i++) {
-    Value logicalValueId = rewriter.create<arith::ConstantOp>(
+    Value logicalValueId = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexType(),
         rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
     FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
@@ -928,11 +928,11 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
       return rewriter.notifyMatchFailure(op, "no coords");
 
     Value el =
-        rewriter.create<vector::ExtractOp>(loc, matrix, ArrayRef<int64_t>{i});
+        vector::ExtractOp::create(rewriter, loc, matrix, ArrayRef<int64_t>{i});
     SmallVector<Value, 4> newIndices;
     getXferIndices<vector::TransferWriteOp>(
         rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
-    rewriter.create<vector::StoreOp>(loc, el, op.getBase(), newIndices);
+    vector::StoreOp::create(rewriter, loc, el, op.getBase(), newIndices);
   }
 
   LLVM_DEBUG(DBGS() << "erase: " << op << "\n");
@@ -1015,7 +1015,7 @@ convertExtractStridedSlice(RewriterBase &rewriter,
   else if (offsets[1])
     sliceOffset[0] = (warpVectorShape[1] / offsets[1]);
 
-  Value newOp = rewriter.create<vector::ExtractStridedSliceOp>(
+  Value newOp = vector::ExtractStridedSliceOp::create(rewriter,
       loc, sourceVector, sliceOffset, sliceShape, strides);
 
   valueMapping[op] = newOp;
@@ -1035,7 +1035,7 @@ convertContractOp(RewriterBase &rewriter, vector::ContractionOp op,
       itC == valueMapping.end())
     return rewriter.notifyMatchFailure(op, "no mapping");
   Value opA = itA->second, opB = itB->second, opC = itC->second;
-  Value matmul = rewriter.create<gpu::SubgroupMmaComputeOp>(
+  Value matmul = gpu::SubgroupMmaComputeOp::create(rewriter,
       op.getLoc(), opC.getType(), opA, opB, opC, /*a_transpose=*/UnitAttr(),
       /*b_transpose=*/UnitAttr());
   valueMapping[op.getResult()] = matmul;
@@ -1058,7 +1058,7 @@ convertContractOpToMmaSync(RewriterBase &rewriter, vector::ContractionOp op,
   int64_t m = cast<VectorType>(op.getLhs().getType()).getShape()[0];
   int64_t n = cast<VectorType>(op.getRhs().getType()).getShape()[0];
   int64_t k = cast<VectorType>(op.getLhs().getType()).getShape()[1];
-  Value matmul = rewriter.create<nvgpu::MmaSyncOp>(
+  Value matmul = nvgpu::MmaSyncOp::create(rewriter,
       op.getLoc(), opA, opB, opC, rewriter.getI64ArrayAttr({m, n, k}));
   valueMapping[op.getResult()] = matmul;
   return success();
@@ -1076,12 +1076,12 @@ convertConstantOp(RewriterBase &rewriter, arith::ConstantOp op,
   auto splat =
       cast<SplatElementsAttr>(op.getValue()).getSplatValue<TypedAttr>();
   auto scalarConstant =
-      rewriter.create<arith::ConstantOp>(op.getLoc(), splat.getType(), splat);
+      arith::ConstantOp::create(rewriter, op.getLoc(), splat.getType(), splat);
   const char *fragType = inferFragType(op);
   auto vecType = cast<VectorType>(op.getType());
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = rewriter.create<gpu::SubgroupMmaConstantMatrixOp>(
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
       op.getLoc(), type, scalarConstant);
   valueMapping[op.getResult()] = matrix;
   return success();
@@ -1100,7 +1100,7 @@ convertBroadcastOp(RewriterBase &rewriter, vector::BroadcastOp op,
   auto vecType = op.getResultVectorType();
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = rewriter.create<gpu::SubgroupMmaConstantMatrixOp>(
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
       op.getLoc(), type, op.getSource());
   valueMapping[op.getResult()] = matrix;
   return success();
@@ -1118,7 +1118,7 @@ static scf::ForOp replaceForOpWithNewSignature(RewriterBase &rewriter,
   rewriter.setInsertionPoint(loop);
   auto operands = llvm::to_vector<4>(loop.getInitArgs());
   llvm::append_range(operands, newInitArgs);
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
       operands);
   rewriter.eraseBlock(newLoop.getBody());
@@ -1189,7 +1189,7 @@ convertYieldOp(RewriterBase &rewriter, scf::YieldOp op,
     yieldOperands[operand.index()] = loop.getInitArgs()[operand.index()];
     yieldOperands.push_back(it->second);
   }
-  rewriter.create<scf::YieldOp>(op.getLoc(), yieldOperands);
+  scf::YieldOp::create(rewriter, op.getLoc(), yieldOperands);
 
   LLVM_DEBUG(DBGS() << "erase: " << op << "\n");
   rewriter.eraseOp(op);
@@ -1220,7 +1220,7 @@ convertElementwiseOp(RewriterBase &rewriter, Operation *op,
                                          resultType.getOperand());
   }
 
-  Value newOp = rewriter.create<gpu::SubgroupMmaElementwiseOp>(
+  Value newOp = gpu::SubgroupMmaElementwiseOp::create(rewriter,
       op->getLoc(), resultType, matrixOperands, opType);
   valueMapping[op->getResult(0)] = newOp;
   return success();
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 501d98862672d..1b31d391cb0bc 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -43,13 +43,13 @@ static Value insertOne(ConversionPatternRewriter &rewriter,
   assert(rank > 0 && "0-D vector corner case should have been handled already");
   if (rank == 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = rewriter.create<LLVM::ConstantOp>(
+    auto constant = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
-    return rewriter.create<LLVM::InsertElementOp>(loc, llvmType, val1, val2,
+    return LLVM::InsertElementOp::create(rewriter, loc, llvmType, val1, val2,
                                                   constant);
   }
-  return rewriter.create<LLVM::InsertValueOp>(loc, val1, val2, pos);
+  return LLVM::InsertValueOp::create(rewriter, loc, val1, val2, pos);
 }
 
 // Helper that picks the proper sequence for extracting.
@@ -58,13 +58,13 @@ static Value extractOne(ConversionPatternRewriter &rewriter,
                         Value val, Type llvmType, int64_t rank, int64_t pos) {
   if (rank <= 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = rewriter.create<LLVM::ConstantOp>(
+    auto constant = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
-    return rewriter.create<LLVM::ExtractElementOp>(loc, llvmType, val,
+    return LLVM::ExtractElementOp::create(rewriter, loc, llvmType, val,
                                                    constant);
   }
-  return rewriter.create<LLVM::ExtractValueOp>(loc, val, pos);
+  return LLVM::ExtractValueOp::create(rewriter, loc, val, pos);
 }
 
 // Helper that returns data layout alignment of a vector.
@@ -141,7 +141,7 @@ static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc,
   auto ptrsType =
       LLVM::getVectorType(pType, vectorType.getDimSize(0),
                           /*isScalable=*/vectorType.getScalableDims()[0]);
-  return rewriter.create<LLVM::GEPOp>(
+  return LLVM::GEPOp::create(rewriter,
       loc, ptrsType, typeConverter.convertType(memRefType.getElementType()),
       base, index);
 }
@@ -152,7 +152,7 @@ static Value getAsLLVMValue(OpBuilder &builder, Location loc,
                             OpFoldResult foldResult) {
   if (auto attr = dyn_cast<Attribute>(foldResult)) {
     auto intAttr = cast<IntegerAttr>(attr);
-    return builder.create<LLVM::ConstantOp>(loc, intAttr).getResult();
+    return LLVM::ConstantOp::create(builder, loc, intAttr).getResult();
   }
 
   return cast<Value>(foldResult);
@@ -475,7 +475,7 @@ class ReductionNeutralFPMax {};
 static Value createReductionNeutralValue(ReductionNeutralZero neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmType,
+  return LLVM::ConstantOp::create(rewriter, loc, llvmType,
                                            rewriter.getZeroAttr(llvmType));
 }
 
@@ -483,7 +483,7 @@ static Value createReductionNeutralValue(ReductionNeutralZero neutral,
 static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType, rewriter.getIntegerAttr(llvmType, 1));
 }
 
@@ -491,7 +491,7 @@ static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
 static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
 }
 
@@ -499,7 +499,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
 static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(
           llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth())));
@@ -509,7 +509,7 @@ static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -519,7 +519,7 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -529,7 +529,7 @@ static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -539,7 +539,7 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
@@ -550,7 +550,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
@@ -562,7 +562,7 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return rewriter.create<LLVM::ConstantOp>(
+  return LLVM::ConstantOp::create(rewriter,
       loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
@@ -591,7 +591,7 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
   auto vShape = vType.getShape();
   assert(vShape.size() == 1 && "Unexpected multi-dim vector type");
 
-  Value baseVecLength = rewriter.create<LLVM::ConstantOp>(
+  Value baseVecLength = LLVM::ConstantOp::create(rewriter,
       loc, rewriter.getI32Type(),
       rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0]));
 
@@ -599,11 +599,11 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
     return baseVecLength;
 
   // For a scalable vector type, create and return `vScale * baseVecLength`.
-  Value vScale = rewriter.create<vector::VectorScaleOp>(loc);
+  Value vScale = vector::VectorScaleOp::create(rewriter, loc);
   vScale =
-      rewriter.create<arith::IndexCastOp>(loc, rewriter.getI32Type(), vScale);
+      arith::IndexCastOp::create(rewriter, loc, rewriter.getI32Type(), vScale);
   Value scalableVecLength =
-      rewriter.create<arith::MulIOp>(loc, baseVecLength, vScale);
+      arith::MulIOp::create(rewriter, loc, baseVecLength, vScale);
   return scalableVecLength;
 }
 
@@ -616,10 +616,10 @@ static Value createIntegerReductionArithmeticOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator) {
 
-  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
 
   if (accumulator)
-    result = rewriter.create<ScalarOp>(loc, accumulator, result);
+    result = ScalarOp::create(rewriter, loc, accumulator, result);
   return result;
 }
 
@@ -631,11 +631,11 @@ template <class LLVMRedIntrinOp>
 static Value createIntegerReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) {
-  Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
   if (accumulator) {
     Value cmp =
-        rewriter.create<LLVM::ICmpOp>(loc, predicate, accumulator, result);
-    result = rewriter.create<LLVM::SelectOp>(loc, cmp, accumulator, result);
+        LLVM::ICmpOp::create(rewriter, loc, predicate, accumulator, result);
+    result = LLVM::SelectOp::create(rewriter, loc, cmp, accumulator, result);
   }
   return result;
 }
@@ -666,11 +666,11 @@ static Value createFPReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::FastmathFlagsAttr fmf) {
   Value result =
-      rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand, fmf);
+      LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand, fmf);
 
   if (accumulator) {
     result =
-        rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
+        typename VectorToScalarMapper<LLVMRedIntrinOp>::Type::create(rewriter,
             loc, result, accumulator);
   }
 
@@ -702,7 +702,7 @@ static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
   const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
   auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
   auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
-  return rewriter.create<LLVM::ConstantOp>(loc, vectorType, denseValue);
+  return LLVM::ConstantOp::create(rewriter, loc, vectorType, denseValue);
 }
 
 /// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
@@ -717,7 +717,7 @@ lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
                                 Value mask, LLVM::FastmathFlagsAttr fmf) {
   const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
       rewriter, loc, llvmType, vectorOperand.getType());
-  const Value selectedVectorByMask = rewriter.create<LLVM::SelectOp>(
+  const Value selectedVectorByMask = LLVM::SelectOp::create(rewriter,
       loc, mask, vectorOperand, vectorMaskNeutral);
   return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
       rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
@@ -730,7 +730,7 @@ lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
                              Value accumulator, LLVM::FastmathFlagsAttr fmf) {
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
-  return rewriter.create<LLVMRedIntrinOp>(loc, llvmType,
+  return LLVMRedIntrinOp::create(rewriter, loc, llvmType,
                                           /*startValue=*/accumulator,
                                           vectorOperand, fmf);
 }
@@ -745,7 +745,7 @@ lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter,
                                        Value vectorOperand, Value accumulator) {
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
-  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
+  return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
                                             /*startValue=*/accumulator,
                                             vectorOperand);
 }
@@ -758,7 +758,7 @@ static Value lowerPredicatedReductionWithStartValue(
                                                          llvmType, accumulator);
   Value vectorLength =
       createVectorLengthValue(rewriter, loc, vectorOperand.getType());
-  return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
+  return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
                                             /*startValue=*/accumulator,
                                             vectorOperand, mask, vectorLength);
 }
@@ -1071,7 +1071,7 @@ class VectorShuffleOpConversion
     // For rank 0 and 1, where both operands have *exactly* the same vector
     // type, there is direct shuffle support in LLVM. Use it!
     if (rank <= 1 && v1Type == v2Type) {
-      Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>(
+      Value llvmShuffleOp = LLVM::ShuffleVectorOp::create(rewriter,
           loc, adaptor.getV1(), adaptor.getV2(),
           llvm::to_vector_of<int32_t>(mask));
       rewriter.replaceOp(shuffleOp, llvmShuffleOp);
@@ -1085,7 +1085,7 @@ class VectorShuffleOpConversion
       eltType = arrayType.getElementType();
     else
       eltType = cast<VectorType>(llvmType).getElementType();
-    Value insert = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
+    Value insert = LLVM::PoisonOp::create(rewriter, loc, llvmType);
     int64_t insPos = 0;
     for (int64_t extPos : mask) {
       Value value = adaptor.getV1();
@@ -1122,7 +1122,7 @@ class VectorExtractElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = extractEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = rewriter.create<LLVM::ConstantOp>(
+      auto zero = LLVM::ConstantOp::create(rewriter,
           loc, typeConverter->convertType(idxType),
           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
@@ -1193,12 +1193,12 @@ class VectorExtractOpConversion
       if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) {
         return failure();
       }
-      extracted = rewriter.create<LLVM::ExtractValueOp>(
+      extracted = LLVM::ExtractValueOp::create(rewriter,
           loc, extracted, getAsIntegers(position));
     }
 
     if (extractsScalar) {
-      extracted = rewriter.create<LLVM::ExtractElementOp>(
+      extracted = LLVM::ExtractElementOp::create(rewriter,
           loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back()));
     }
 
@@ -1256,7 +1256,7 @@ class VectorInsertElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = insertEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = rewriter.create<LLVM::ConstantOp>(
+      auto zero = LLVM::ConstantOp::create(rewriter,
           loc, typeConverter->convertType(idxType),
           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
@@ -1342,7 +1342,7 @@ class VectorInsertOpConversion
           // llvm.extractvalue does not support dynamic dimensions.
           return failure();
         }
-        sourceAggregate = rewriter.create<LLVM::ExtractValueOp>(
+        sourceAggregate = LLVM::ExtractValueOp::create(rewriter,
             loc, adaptor.getDest(),
             getAsIntegers(positionOf1DVectorWithinAggregate));
       } else {
@@ -1351,7 +1351,7 @@ class VectorInsertOpConversion
         sourceAggregate = adaptor.getDest();
       }
       // Insert the scalar into the 1D vector.
-      sourceAggregate = rewriter.create<LLVM::InsertElementOp>(
+      sourceAggregate = LLVM::InsertElementOp::create(rewriter,
           loc, sourceAggregate.getType(), sourceAggregate,
           adaptor.getValueToStore(),
           getAsLLVMValue(rewriter, loc, positionOfScalarWithin1DVector));
@@ -1359,7 +1359,7 @@ class VectorInsertOpConversion
 
     Value result = sourceAggregate;
     if (isNestedAggregate) {
-      result = rewriter.create<LLVM::InsertValueOp>(
+      result = LLVM::InsertValueOp::create(rewriter,
           loc, adaptor.getDest(), sourceAggregate,
           getAsIntegers(positionOf1DVectorWithinAggregate));
     }
@@ -1439,15 +1439,15 @@ class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
 
     auto loc = op.getLoc();
     auto elemType = vType.getElementType();
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero);
+    Value desc = vector::SplatOp::create(rewriter, loc, vType, zero);
     for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
-      Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i);
-      Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i);
-      Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i);
-      Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC);
-      desc = rewriter.create<InsertOp>(loc, fma, desc, i);
+      Value extrLHS = ExtractOp::create(rewriter, loc, op.getLhs(), i);
+      Value extrRHS = ExtractOp::create(rewriter, loc, op.getRhs(), i);
+      Value extrACC = ExtractOp::create(rewriter, loc, op.getAcc(), i);
+      Value fma = FMAOp::create(rewriter, loc, extrLHS, extrRHS, extrACC);
+      desc = InsertOp::create(rewriter, loc, fma, desc, i);
     }
     rewriter.replaceOp(op, desc);
     return success();
@@ -1537,7 +1537,7 @@ class VectorTypeCastOpConversion
     desc.setAlignedPtr(rewriter, loc, ptr);
     // Fill offset 0.
     auto attr = rewriter.getIntegerAttr(rewriter.getIndexType(), 0);
-    auto zero = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, attr);
+    auto zero = LLVM::ConstantOp::create(rewriter, loc, int64Ty, attr);
     desc.setOffset(rewriter, loc, zero);
 
     // Fill size and stride descriptors in memref.
@@ -1546,11 +1546,11 @@ class VectorTypeCastOpConversion
       int64_t index = indexedSize.index();
       auto sizeAttr =
           rewriter.getIntegerAttr(rewriter.getIndexType(), indexedSize.value());
-      auto size = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, sizeAttr);
+      auto size = LLVM::ConstantOp::create(rewriter, loc, int64Ty, sizeAttr);
       desc.setSize(rewriter, loc, index, size);
       auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(),
                                                 (*targetStrides)[index]);
-      auto stride = rewriter.create<LLVM::ConstantOp>(loc, int64Ty, strideAttr);
+      auto stride = LLVM::ConstantOp::create(rewriter, loc, int64Ty, strideAttr);
       desc.setStride(rewriter, loc, index, stride);
     }
 
@@ -1578,13 +1578,13 @@ class VectorCreateMaskOpConversion
     IntegerType idxType =
         force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
     auto loc = op->getLoc();
-    Value indices = rewriter.create<LLVM::StepVectorOp>(
+    Value indices = LLVM::StepVectorOp::create(rewriter,
         loc, LLVM::getVectorType(idxType, dstType.getShape()[0],
                                  /*isScalable=*/true));
     auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType,
                                                  adaptor.getOperands()[0]);
-    Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound);
-    Value comp = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+    Value bounds = SplatOp::create(rewriter, loc, indices.getType(), bound);
+    Value comp = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                 indices, bounds);
     rewriter.replaceOp(op, comp);
     return success();
@@ -1741,15 +1741,15 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 
     switch (conversion) {
     case PrintConversion::ZeroExt64:
-      value = rewriter.create<arith::ExtUIOp>(
+      value = arith::ExtUIOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::SignExt64:
-      value = rewriter.create<arith::ExtSIOp>(
+      value = arith::ExtSIOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::Bitcast16:
-      value = rewriter.create<LLVM::BitcastOp>(
+      value = LLVM::BitcastOp::create(rewriter,
           loc, IntegerType::get(rewriter.getContext(), 16), value);
       break;
     case PrintConversion::None:
@@ -1762,7 +1762,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
   // Helper to emit a call.
   static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
                        Operation *ref, ValueRange params = ValueRange()) {
-    rewriter.create<LLVM::CallOp>(loc, TypeRange(), SymbolRefAttr::get(ref),
+    LLVM::CallOp::create(rewriter, loc, TypeRange(), SymbolRefAttr::get(ref),
                                   params);
   }
 };
@@ -1782,8 +1782,8 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     // First insert it into a poison vector so we can shuffle it.
     auto vectorType = typeConverter->convertType(splatOp.getType());
     Value poison =
-        rewriter.create<LLVM::PoisonOp>(splatOp.getLoc(), vectorType);
-    auto zero = rewriter.create<LLVM::ConstantOp>(
+        LLVM::PoisonOp::create(rewriter, splatOp.getLoc(), vectorType);
+    auto zero = LLVM::ConstantOp::create(rewriter,
         splatOp.getLoc(),
         typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
@@ -1796,7 +1796,7 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     }
 
     // For 1-d vector, we additionally do a `vectorshuffle`.
-    auto v = rewriter.create<LLVM::InsertElementOp>(
+    auto v = LLVM::InsertElementOp::create(rewriter,
         splatOp.getLoc(), vectorType, poison, adaptor.getInput(), zero);
 
     int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
@@ -1832,26 +1832,26 @@ struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
       return failure();
 
     // Construct returned value.
-    Value desc = rewriter.create<LLVM::PoisonOp>(loc, llvmNDVectorTy);
+    Value desc = LLVM::PoisonOp::create(rewriter, loc, llvmNDVectorTy);
 
     // Construct a 1-D vector with the splatted value that we insert in all the
     // places within the returned descriptor.
-    Value vdesc = rewriter.create<LLVM::PoisonOp>(loc, llvm1DVectorTy);
-    auto zero = rewriter.create<LLVM::ConstantOp>(
+    Value vdesc = LLVM::PoisonOp::create(rewriter, loc, llvm1DVectorTy);
+    auto zero = LLVM::ConstantOp::create(rewriter,
         loc, typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
-    Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc,
+    Value v = LLVM::InsertElementOp::create(rewriter, loc, llvm1DVectorTy, vdesc,
                                                      adaptor.getInput(), zero);
 
     // Shuffle the value across the desired number of elements.
     int64_t width = resultType.getDimSize(resultType.getRank() - 1);
     SmallVector<int32_t> zeroValues(width, 0);
-    v = rewriter.create<LLVM::ShuffleVectorOp>(loc, v, v, zeroValues);
+    v = LLVM::ShuffleVectorOp::create(rewriter, loc, v, v, zeroValues);
 
     // Iterate of linear index, convert to coords space and insert splatted 1-D
     // vector in each position.
     nDVectorIterate(vectorTypeInfo, rewriter, [&](ArrayRef<int64_t> position) {
-      desc = rewriter.create<LLVM::InsertValueOp>(loc, desc, v, position);
+      desc = LLVM::InsertValueOp::create(rewriter, loc, desc, v, position);
     });
     rewriter.replaceOp(splatOp, desc);
     return success();
@@ -1921,12 +1921,12 @@ struct VectorDeinterleaveOpLowering
       auto deinterleaveResults = deinterleaveOp.getResultTypes();
       auto packedOpResults =
           llvmTypeConverter->packOperationResults(deinterleaveResults);
-      auto intrinsic = rewriter.create<LLVM::vector_deinterleave2>(
+      auto intrinsic = LLVM::vector_deinterleave2::create(rewriter,
           loc, packedOpResults, adaptor.getSource());
 
-      auto evenResult = rewriter.create<LLVM::ExtractValueOp>(
+      auto evenResult = LLVM::ExtractValueOp::create(rewriter,
           loc, intrinsic->getResult(0), 0);
-      auto oddResult = rewriter.create<LLVM::ExtractValueOp>(
+      auto oddResult = LLVM::ExtractValueOp::create(rewriter,
           loc, intrinsic->getResult(0), 1);
 
       rewriter.replaceOp(deinterleaveOp, ValueRange{evenResult, oddResult});
@@ -1950,10 +1950,10 @@ struct VectorDeinterleaveOpLowering
         oddShuffleMask.push_back(i);
     }
 
-    auto poison = rewriter.create<LLVM::PoisonOp>(loc, sourceType);
-    auto evenShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+    auto poison = LLVM::PoisonOp::create(rewriter, loc, sourceType);
+    auto evenShuffle = LLVM::ShuffleVectorOp::create(rewriter,
         loc, adaptor.getSource(), poison, evenShuffleMask);
-    auto oddShuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+    auto oddShuffle = LLVM::ShuffleVectorOp::create(rewriter,
         loc, adaptor.getSource(), poison, oddShuffleMask);
 
     rewriter.replaceOp(deinterleaveOp, ValueRange{evenShuffle, oddShuffle});
@@ -1977,9 +1977,9 @@ struct VectorFromElementsLowering
       return rewriter.notifyMatchFailure(fromElementsOp,
                                          "rank > 1 vectors are not supported");
     Type llvmType = typeConverter->convertType(vectorType);
-    Value result = rewriter.create<LLVM::PoisonOp>(loc, llvmType);
+    Value result = LLVM::PoisonOp::create(rewriter, loc, llvmType);
     for (auto [idx, val] : llvm::enumerate(adaptor.getElements()))
-      result = rewriter.create<vector::InsertOp>(loc, val, result, idx);
+      result = vector::InsertOp::create(rewriter, loc, val, result, idx);
     rewriter.replaceOp(fromElementsOp, result);
     return success();
   }
@@ -2003,11 +2003,11 @@ struct VectorToElementsLowering
       if (element.use_empty())
         continue;
 
-      auto constIdx = rewriter.create<LLVM::ConstantOp>(
+      auto constIdx = LLVM::ConstantOp::create(rewriter,
           loc, idxType, rewriter.getIntegerAttr(idxType, idx));
       auto llvmType = typeConverter->convertType(element.getType());
 
-      Value result = rewriter.create<LLVM::ExtractElementOp>(loc, llvmType,
+      Value result = LLVM::ExtractElementOp::create(rewriter, loc, llvmType,
                                                              source, constIdx);
       results[idx] = result;
     }
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index ed51b2126dcdd..7385e28aeb04a 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -132,9 +132,9 @@ static void maybeYieldValue(OpBuilder &b, Location loc, bool hasRetVal,
                             Value value) {
   if (hasRetVal) {
     assert(value && "Expected non-empty value");
-    b.create<scf::YieldOp>(loc, value);
+    scf::YieldOp::create(b, loc, value);
   } else {
-    b.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(b, loc);
   }
 }
 
@@ -154,7 +154,7 @@ static Value generateMaskCheck(OpBuilder &b, OpTy xferOp, Value iv) {
     return Value();
 
   Location loc = xferOp.getLoc();
-  return b.create<vector::ExtractOp>(loc, xferOp.getMask(), iv);
+  return vector::ExtractOp::create(b, loc, xferOp.getMask(), iv);
 }
 
 /// Helper function TransferOpConversion and TransferOp1dConversion.
@@ -201,21 +201,21 @@ static Value generateInBoundsCheck(
     Value base = xferOp.getIndices()[*dim];
     Value memrefIdx =
         affine::makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
-    cond = lb.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, memrefDim,
+    cond = arith::CmpIOp::create(lb, arith::CmpIPredicate::sgt, memrefDim,
                                     memrefIdx);
   }
 
   // Condition check 2: Masked in?
   if (auto maskCond = generateMaskCheck(b, xferOp, iv)) {
     if (cond)
-      cond = lb.create<arith::AndIOp>(cond, maskCond);
+      cond = arith::AndIOp::create(lb, cond, maskCond);
     else
       cond = maskCond;
   }
 
   // If the condition is non-empty, generate an SCF::IfOp.
   if (cond) {
-    auto check = lb.create<scf::IfOp>(
+    auto check = scf::IfOp::create(lb,
         cond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
@@ -226,7 +226,7 @@ static Value generateInBoundsCheck(
           if (outOfBoundsCase) {
             maybeYieldValue(b, loc, hasRetVal, outOfBoundsCase(b, loc));
           } else {
-            b.create<scf::YieldOp>(loc);
+            scf::YieldOp::create(b, loc);
           }
         });
 
@@ -303,14 +303,14 @@ static BufferAllocs allocBuffers(OpBuilder &b, OpTy xferOp) {
 
   BufferAllocs result;
   auto bufferType = MemRefType::get({}, xferOp.getVectorType());
-  result.dataBuffer = b.create<memref::AllocaOp>(loc, bufferType);
+  result.dataBuffer = memref::AllocaOp::create(b, loc, bufferType);
 
   if (xferOp.getMask()) {
     auto maskType = MemRefType::get({}, xferOp.getMask().getType());
-    auto maskBuffer = b.create<memref::AllocaOp>(loc, maskType);
+    auto maskBuffer = memref::AllocaOp::create(b, loc, maskType);
     b.setInsertionPoint(xferOp);
-    b.create<memref::StoreOp>(loc, xferOp.getMask(), maskBuffer);
-    result.maskBuffer = b.create<memref::LoadOp>(loc, maskBuffer, ValueRange());
+    memref::StoreOp::create(b, loc, xferOp.getMask(), maskBuffer);
+    result.maskBuffer = memref::LoadOp::create(b, loc, maskBuffer, ValueRange());
   }
 
   return result;
@@ -421,14 +421,14 @@ struct Strategy<TransferReadOp> {
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
-    auto newXferOp = b.create<vector::TransferReadOp>(
+    auto newXferOp = vector::TransferReadOp::create(b,
         loc, vecType, xferOp.getBase(), xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
         xferOp.getPadding(), Value(), inBoundsAttr);
 
     maybeApplyPassLabel(b, newXferOp, options.targetRank);
 
-    b.create<memref::StoreOp>(loc, newXferOp.getVector(), buffer, storeIndices);
+    memref::StoreOp::create(b, loc, newXferOp.getVector(), buffer, storeIndices);
     return newXferOp;
   }
 
@@ -444,8 +444,8 @@ struct Strategy<TransferReadOp> {
     Location loc = xferOp.getLoc();
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
-    auto vec = b.create<vector::SplatOp>(loc, vecType, xferOp.getPadding());
-    b.create<memref::StoreOp>(loc, vec, buffer, storeIndices);
+    auto vec = vector::SplatOp::create(b, loc, vecType, xferOp.getPadding());
+    memref::StoreOp::create(b, loc, vec, buffer, storeIndices);
 
     return Value();
   }
@@ -506,11 +506,11 @@ struct Strategy<TransferWriteOp> {
     getXferIndices(b, xferOp, iv, xferIndices);
 
     Location loc = xferOp.getLoc();
-    auto vec = b.create<memref::LoadOp>(loc, buffer, loadIndices);
+    auto vec = memref::LoadOp::create(b, loc, buffer, loadIndices);
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
     auto source = loopState.empty() ? xferOp.getBase() : loopState[0];
     Type type = isTensorOp(xferOp) ? xferOp.getShapedType() : Type();
-    auto newXferOp = b.create<vector::TransferWriteOp>(
+    auto newXferOp = vector::TransferWriteOp::create(b,
         loc, type, vec, source, xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
         inBoundsAttr);
@@ -610,7 +610,7 @@ struct PrepareTransferReadConversion
     }
 
     Location loc = xferOp.getLoc();
-    rewriter.create<memref::StoreOp>(loc, newXfer->getResult(0),
+    memref::StoreOp::create(rewriter, loc, newXfer->getResult(0),
                                      buffers.dataBuffer);
     rewriter.replaceOpWithNewOp<memref::LoadOp>(xferOp, buffers.dataBuffer);
 
@@ -653,9 +653,9 @@ struct PrepareTransferWriteConversion
 
     Location loc = xferOp.getLoc();
     auto buffers = allocBuffers(rewriter, xferOp);
-    rewriter.create<memref::StoreOp>(loc, xferOp.getVector(),
+    memref::StoreOp::create(rewriter, loc, xferOp.getVector(),
                                      buffers.dataBuffer);
-    auto loadedVec = rewriter.create<memref::LoadOp>(loc, buffers.dataBuffer);
+    auto loadedVec = memref::LoadOp::create(rewriter, loc, buffers.dataBuffer);
     rewriter.modifyOpInPlace(xferOp, [&]() {
       xferOp.getValueToStoreMutable().assign(loadedVec);
       xferOp->setAttr(kPassLabel, rewriter.getUnitAttr());
@@ -735,17 +735,17 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
       auto signlessTargetVectorType =
           vectorType.cloneWith({}, getIntTypeWithSignlessSemantics(legalIntTy));
       auto targetVectorType = vectorType.cloneWith({}, legalIntTy);
-      value = rewriter.create<vector::BitCastOp>(loc, signlessSourceVectorType,
+      value = vector::BitCastOp::create(rewriter, loc, signlessSourceVectorType,
                                                  value);
       if (value.getType() != signlessTargetVectorType) {
         if (width == 1 || intTy.isUnsigned())
-          value = rewriter.create<arith::ExtUIOp>(loc, signlessTargetVectorType,
+          value = arith::ExtUIOp::create(rewriter, loc, signlessTargetVectorType,
                                                   value);
         else
-          value = rewriter.create<arith::ExtSIOp>(loc, signlessTargetVectorType,
+          value = arith::ExtSIOp::create(rewriter, loc, signlessTargetVectorType,
                                                   value);
       }
-      value = rewriter.create<vector::BitCastOp>(loc, targetVectorType, value);
+      value = vector::BitCastOp::create(rewriter, loc, targetVectorType, value);
       vectorType = targetVectorType;
     }
 
@@ -762,28 +762,28 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
                                         std::multiplies<int64_t>());
       auto flatVectorType =
           VectorType::get({flatLength}, vectorType.getElementType());
-      value = rewriter.create<vector::ShapeCastOp>(loc, flatVectorType, value);
+      value = vector::ShapeCastOp::create(rewriter, loc, flatVectorType, value);
     }
 
     vector::PrintOp firstClose;
     SmallVector<Value, 8> loopIndices;
     for (unsigned d = 0; d < shape.size(); d++) {
       // Setup loop bounds and step.
-      Value lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-      Value upperBound = rewriter.create<arith::ConstantIndexOp>(loc, shape[d]);
-      Value step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+      Value lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
+      Value upperBound = arith::ConstantIndexOp::create(rewriter, loc, shape[d]);
+      Value step = arith::ConstantIndexOp::create(rewriter, loc, 1);
       if (!scalableDimensions.empty() && scalableDimensions[d]) {
-        auto vscale = rewriter.create<vector::VectorScaleOp>(
+        auto vscale = vector::VectorScaleOp::create(rewriter,
             loc, rewriter.getIndexType());
-        upperBound = rewriter.create<arith::MulIOp>(loc, upperBound, vscale);
+        upperBound = arith::MulIOp::create(rewriter, loc, upperBound, vscale);
       }
-      auto lastIndex = rewriter.create<arith::SubIOp>(loc, upperBound, step);
+      auto lastIndex = arith::SubIOp::create(rewriter, loc, upperBound, step);
 
       // Create a loop to print the elements surrounded by parentheses.
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+      vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
       auto loop =
-          rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
-      auto printClose = rewriter.create<vector::PrintOp>(
+          scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
+      auto printClose = vector::PrintOp::create(rewriter,
           loc, vector::PrintPunctuation::Close);
       if (!firstClose)
         firstClose = printClose;
@@ -793,13 +793,13 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
 
       // Print a comma after all but the last element.
       rewriter.setInsertionPointToStart(loop.getBody());
-      auto notLastIndex = rewriter.create<arith::CmpIOp>(
+      auto notLastIndex = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ult, loopIdx, lastIndex);
-      rewriter.create<scf::IfOp>(loc, notLastIndex,
+      scf::IfOp::create(rewriter, loc, notLastIndex,
                                  [&](OpBuilder &builder, Location loc) {
-                                   builder.create<vector::PrintOp>(
+                                   vector::PrintOp::create(builder,
                                        loc, vector::PrintPunctuation::Comma);
-                                   builder.create<scf::YieldOp>(loc);
+                                   scf::YieldOp::create(builder, loc);
                                  });
 
       rewriter.setInsertionPointToStart(loop.getBody());
@@ -810,22 +810,23 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     Value flatIndex;
     auto currentStride = 1;
     for (int d = shape.size() - 1; d >= 0; d--) {
-      auto stride = rewriter.create<arith::ConstantIndexOp>(loc, currentStride);
-      auto index = rewriter.create<arith::MulIOp>(loc, stride, loopIndices[d]);
+      auto stride = arith::ConstantIndexOp::create(rewriter, loc, currentStride);
+      auto index = arith::MulIOp::create(rewriter, loc, stride, loopIndices[d]);
       if (flatIndex)
-        flatIndex = rewriter.create<arith::AddIOp>(loc, flatIndex, index);
+        flatIndex = arith::AddIOp::create(rewriter, loc, flatIndex, index);
       else
         flatIndex = index;
       currentStride *= shape[d];
     }
 
     // Print the scalar elements in the inner most loop.
-    auto element = rewriter.create<vector::ExtractOp>(loc, value, flatIndex);
-    rewriter.create<vector::PrintOp>(loc, element,
+    auto element =
+        vector::ExtractOp::create(rewriter, loc, value, flatIndex);
+    vector::PrintOp::create(rewriter, loc, element,
                                      vector::PrintPunctuation::NoPunctuation);
 
     rewriter.setInsertionPointAfter(firstClose);
-    rewriter.create<vector::PrintOp>(loc, printOp.getPunctuation());
+    vector::PrintOp::create(rewriter, loc, printOp.getPunctuation());
     rewriter.eraseOp(printOp);
     return success();
   }
@@ -916,7 +917,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                                          "Failed to unpack one vector dim.");
 
     auto castedDataBuffer =
-        locB.create<vector::TypeCastOp>(*castedDataType, dataBuffer);
+        vector::TypeCastOp::create(locB, *castedDataType, dataBuffer);
 
     // If the xferOp has a mask: Find and cast mask buffer.
     Value castedMaskBuffer;
@@ -935,21 +936,21 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
         auto maskBufferType = cast<MemRefType>(maskBuffer.getType());
         MemRefType castedMaskType = *unpackOneDim(maskBufferType);
         castedMaskBuffer =
-            locB.create<vector::TypeCastOp>(castedMaskType, maskBuffer);
+            vector::TypeCastOp::create(locB, castedMaskType, maskBuffer);
       }
     }
 
     // Loop bounds and step.
-    auto lb = locB.create<arith::ConstantIndexOp>(0);
-    auto ub = locB.create<arith::ConstantIndexOp>(
+    auto lb = arith::ConstantIndexOp::create(locB, 0);
+    auto ub = arith::ConstantIndexOp::create(locB,
         castedDataType->getDimSize(castedDataType->getRank() - 1));
-    auto step = locB.create<arith::ConstantIndexOp>(1);
+    auto step = arith::ConstantIndexOp::create(locB, 1);
     // TransferWriteOps that operate on tensors return the modified tensor and
     // require a loop state.
     auto loopState = Strategy<OpTy>::initialLoopState(xferOp);
 
     // Generate for loop.
-    auto result = locB.create<scf::ForOp>(
+    auto result = scf::ForOp::create(locB,
         lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(),
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopState) {
           Type stateType = loopState.empty() ? Type() : loopState[0].getType();
@@ -975,7 +976,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                   SmallVector<Value, 8> loadIndices;
                   getMaskBufferLoadIndices(xferOp, castedMaskBuffer,
                                            loadIndices, iv);
-                  auto mask = b.create<memref::LoadOp>(loc, castedMaskBuffer,
+                  auto mask = memref::LoadOp::create(b, loc, castedMaskBuffer,
                                                        loadIndices);
                   rewriter.modifyOpInPlace(newXfer, [&]() {
                     newXfer.getMaskMutable().assign(mask);
@@ -1119,29 +1120,29 @@ struct ScalableTransposeTransferWriteConversion
     auto transposeSource = transposeOp.getVector();
     SmallVector<Value> transposeSourceSlices =
         llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-          return rewriter.create<vector::ExtractOp>(loc, transposeSource, idx);
+          return vector::ExtractOp::create(rewriter, loc, transposeSource, idx);
         });
 
     // Loop bounds and step.
-    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lb = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto ub =
         maskDims->empty()
             ? Value(createVscaleMultiple(vectorType.getDimSize(0)))
             : vector::getAsValues(rewriter, loc, maskDims->front()).front();
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
     // Generate a new mask for the slice.
     VectorType sliceType = VectorType::Builder(vectorType).dropDim(0);
     Value sliceMask = nullptr;
     if (!maskDims->empty()) {
-      sliceMask = rewriter.create<vector::CreateMaskOp>(
+      sliceMask = vector::CreateMaskOp::create(rewriter,
           loc, sliceType.clone(rewriter.getI1Type()),
           ArrayRef<OpFoldResult>(*maskDims).drop_front());
     }
 
     Value initDest = isTensorOp(writeOp) ? writeOp.getBase() : Value{};
     ValueRange initLoopArgs = initDest ? initDest : ValueRange{};
-    auto result = rewriter.create<scf::ForOp>(
+    auto result = scf::ForOp::create(rewriter,
         loc, lb, ub, step, initLoopArgs,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopIterArgs) {
           // Indices for the new transfer op.
@@ -1151,23 +1152,23 @@ struct ScalableTransposeTransferWriteConversion
           // Extract a transposed slice from the source vector.
           SmallVector<Value> transposeElements =
               llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-                return b.create<vector::ExtractOp>(
+                return vector::ExtractOp::create(b,
                     loc, transposeSourceSlices[idx], iv);
               });
-          auto sliceVec = b.create<vector::FromElementsOp>(loc, sliceType,
+          auto sliceVec = vector::FromElementsOp::create(b, loc, sliceType,
                                                            transposeElements);
 
           // Create the transfer_write for the slice.
           Value dest =
               loopIterArgs.empty() ? writeOp.getBase() : loopIterArgs.front();
-          auto newWriteOp = b.create<vector::TransferWriteOp>(
+          auto newWriteOp = vector::TransferWriteOp::create(b,
               loc, sliceVec, dest, xferIndices,
               ArrayRef<bool>(writeOp.getInBoundsValues()).drop_front());
           if (sliceMask)
             newWriteOp.getMaskMutable().assign(sliceMask);
 
           // Yield from the loop.
-          b.create<scf::YieldOp>(loc, loopIterArgs.empty()
+          scf::YieldOp::create(b, loc, loopIterArgs.empty()
                                           ? ValueRange{}
                                           : newWriteOp.getResult());
         });
@@ -1207,7 +1208,7 @@ static void maybeAssignMask(OpBuilder &b, OpTy xferOp, OpTy newXferOp,
 
     llvm::SmallVector<int64_t, 1> indices({i});
     Location loc = xferOp.getLoc();
-    auto newMask = b.create<vector::ExtractOp>(loc, xferOp.getMask(), indices);
+    auto newMask = vector::ExtractOp::create(b, loc, xferOp.getMask(), indices);
     newXferOp.getMaskMutable().assign(newMask);
   }
 
@@ -1261,7 +1262,7 @@ struct UnrollTransferReadConversion
     if (auto insertOp = getInsertOp(xferOp))
       return insertOp.getDest();
     Location loc = xferOp.getLoc();
-    return rewriter.create<vector::SplatOp>(loc, xferOp.getVectorType(),
+    return vector::SplatOp::create(rewriter, loc, xferOp.getVectorType(),
                                             xferOp.getPadding());
   }
 
@@ -1317,7 +1318,7 @@ struct UnrollTransferReadConversion
     // Generate fully unrolled loop of transfer ops.
     Location loc = xferOp.getLoc();
     for (int64_t i = 0; i < dimSize; ++i) {
-      Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
+      Value iv = arith::ConstantIndexOp::create(rewriter, loc, i);
 
       // FIXME: Rename this lambda - it does much more than just
       // in-bounds-check generation.
@@ -1336,7 +1337,7 @@ struct UnrollTransferReadConversion
 
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
 
-            auto newXferOp = b.create<vector::TransferReadOp>(
+            auto newXferOp = vector::TransferReadOp::create(b,
                 loc, newXferVecType, xferOp.getBase(), xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
                 xferOp.getPadding(), Value(), inBoundsAttr);
@@ -1346,10 +1347,10 @@ struct UnrollTransferReadConversion
             if (newXferVecType.getRank() == 0) {
               // vector.insert does not accept rank-0 as the non-indexed
               // argument. Extract the scalar before inserting.
-              valToInser = b.create<vector::ExtractOp>(loc, valToInser,
+              valToInser = vector::ExtractOp::create(b, loc, valToInser,
                                                        SmallVector<int64_t>());
             }
-            return b.create<vector::InsertOp>(loc, valToInser, vec,
+            return vector::InsertOp::create(b, loc, valToInser, vec,
                                               insertionIndices);
           },
           /*outOfBoundsCase=*/
@@ -1460,7 +1461,7 @@ struct UnrollTransferWriteConversion
     // Generate fully unrolled loop of transfer ops.
     Location loc = xferOp.getLoc();
     for (int64_t i = 0; i < dimSize; ++i) {
-      Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
+      Value iv = arith::ConstantIndexOp::create(rewriter, loc, i);
 
       auto updatedSource = generateInBoundsCheck(
           rewriter, xferOp, iv, unpackedDim(xferOp),
@@ -1477,19 +1478,19 @@ struct UnrollTransferWriteConversion
             extractionIndices.push_back(b.getI64IntegerAttr(i));
 
             auto extracted =
-                b.create<vector::ExtractOp>(loc, vec, extractionIndices);
+                vector::ExtractOp::create(b, loc, vec, extractionIndices);
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
             Value xferVec;
             if (inputVectorTy.getRank() == 1) {
               // When target-rank=0, unrolling would causes the vector input
               // argument into `transfer_write` to become a scalar. We solve
               // this by broadcasting the scalar to a 0D vector.
-              xferVec = b.create<vector::BroadcastOp>(
+              xferVec = vector::BroadcastOp::create(b,
                   loc, VectorType::get({}, extracted.getType()), extracted);
             } else {
               xferVec = extracted;
             }
-            auto newXferOp = b.create<vector::TransferWriteOp>(
+            auto newXferOp = vector::TransferWriteOp::create(b,
                 loc, sourceType, xferVec, source, xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
                 inBoundsAttr);
@@ -1572,18 +1573,18 @@ struct Strategy1d<TransferReadOp> {
         b, xferOp, iv, dim, TypeRange(xferOp.getVectorType()),
         /*inBoundsCase=*/
         [&](OpBuilder &b, Location loc) {
-          Value val = b.create<memref::LoadOp>(loc, xferOp.getBase(), indices);
-          return b.create<vector::InsertOp>(loc, val, vec, iv);
+          Value val = memref::LoadOp::create(b, loc, xferOp.getBase(), indices);
+          return vector::InsertOp::create(b, loc, val, vec, iv);
         },
         /*outOfBoundsCase=*/
         [&](OpBuilder & /*b*/, Location loc) { return vec; });
-    b.create<scf::YieldOp>(loc, nextVec);
+    scf::YieldOp::create(b, loc, nextVec);
   }
 
   static Value initialLoopState(OpBuilder &b, TransferReadOp xferOp) {
     // Inititalize vector with padding value.
     Location loc = xferOp.getLoc();
-    return b.create<vector::SplatOp>(loc, xferOp.getVectorType(),
+    return vector::SplatOp::create(b, loc, xferOp.getVectorType(),
                                      xferOp.getPadding());
   }
 };
@@ -1601,10 +1602,11 @@ struct Strategy1d<TransferWriteOp> {
     generateInBoundsCheck(
         b, xferOp, iv, dim,
         /*inBoundsCase=*/[&](OpBuilder &b, Location loc) {
-          auto val = b.create<vector::ExtractOp>(loc, xferOp.getVector(), iv);
-          b.create<memref::StoreOp>(loc, val, xferOp.getBase(), indices);
+          auto val =
+              vector::ExtractOp::create(b, loc, xferOp.getVector(), iv);
+          memref::StoreOp::create(b, loc, val, xferOp.getBase(), indices);
         });
-    b.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(b, loc);
   }
 
   static Value initialLoopState(OpBuilder &b, TransferWriteOp xferOp) {
@@ -1665,15 +1667,15 @@ struct TransferOp1dConversion : public VectorToSCFPattern<OpTy> {
     // Loop bounds, step, state...
     Location loc = xferOp.getLoc();
     auto vecType = xferOp.getVectorType();
-    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lb = arith::ConstantIndexOp::create(rewriter, loc, 0);
     Value ub =
-        rewriter.create<arith::ConstantIndexOp>(loc, vecType.getDimSize(0));
+        arith::ConstantIndexOp::create(rewriter, loc, vecType.getDimSize(0));
     if (vecType.isScalable()) {
       Value vscale =
-          rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-      ub = rewriter.create<arith::MulIOp>(loc, ub, vscale);
+          vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+      ub = arith::MulIOp::create(rewriter, loc, ub, vscale);
     }
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto loopState = Strategy1d<OpTy>::initialLoopState(rewriter, xferOp);
 
     // Generate for loop.
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 21d8e1d9f1156..2466e72277441 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -147,18 +147,18 @@ static Value sanitizeDynamicIndex(ConversionPatternRewriter &rewriter,
                                   Location loc, Value dynamicIndex,
                                   int64_t kPoisonIndex, unsigned vectorSize) {
   if (llvm::isPowerOf2_32(vectorSize)) {
-    Value inBoundsMask = rewriter.create<spirv::ConstantOp>(
+    Value inBoundsMask = spirv::ConstantOp::create(rewriter,
         loc, dynamicIndex.getType(),
         rewriter.getIntegerAttr(dynamicIndex.getType(), vectorSize - 1));
-    return rewriter.create<spirv::BitwiseAndOp>(loc, dynamicIndex,
+    return spirv::BitwiseAndOp::create(rewriter, loc, dynamicIndex,
                                                 inBoundsMask);
   }
-  Value poisonIndex = rewriter.create<spirv::ConstantOp>(
+  Value poisonIndex = spirv::ConstantOp::create(rewriter,
       loc, dynamicIndex.getType(),
       rewriter.getIntegerAttr(dynamicIndex.getType(), kPoisonIndex));
   Value cmpResult =
-      rewriter.create<spirv::IEqualOp>(loc, dynamicIndex, poisonIndex);
-  return rewriter.create<spirv::SelectOp>(
+      spirv::IEqualOp::create(rewriter, loc, dynamicIndex, poisonIndex);
+  return spirv::SelectOp::create(rewriter,
       loc, cmpResult,
       spirv::ConstantOp::getZero(dynamicIndex.getType(), loc, rewriter),
       dynamicIndex);
@@ -427,7 +427,7 @@ static SmallVector<Value> extractAllElements(
   Location loc = reduceOp.getLoc();
 
   for (int i = 0; i < numElements; ++i) {
-    values.push_back(rewriter.create<spirv::CompositeExtractOp>(
+    values.push_back(spirv::CompositeExtractOp::create(rewriter,
         loc, srcVectorType.getElementType(), adaptor.getVector(),
         rewriter.getI32ArrayAttr({i})));
   }
@@ -481,16 +481,16 @@ struct VectorReductionPattern final : OpConversionPattern<vector::ReductionOp> {
 #define INT_AND_FLOAT_CASE(kind, iop, fop)                                     \
   case vector::CombiningKind::kind:                                            \
     if (llvm::isa<IntegerType>(resultType)) {                                  \
-      result = rewriter.create<spirv::iop>(loc, resultType, result, next);     \
+      result = spirv::iop::create(rewriter, loc, resultType, result, next);     \
     } else {                                                                   \
       assert(llvm::isa<FloatType>(resultType));                                \
-      result = rewriter.create<spirv::fop>(loc, resultType, result, next);     \
+      result = spirv::fop::create(rewriter, loc, resultType, result, next);     \
     }                                                                          \
     break
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = rewriter.create<fop>(loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);              \
     break
 
         INT_AND_FLOAT_CASE(ADD, IAddOp, FAddOp);
@@ -537,7 +537,7 @@ struct VectorReductionFloatMinMax final
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = rewriter.create<fop>(loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);              \
     break
 
         INT_OR_FLOAT_CASE(MAXIMUMF, SPIRVFMaxOp);
@@ -613,7 +613,7 @@ struct VectorShuffleOpConvert final
     auto getElementAtIdx = [&rewriter, loc = shuffleOp.getLoc()](
                                Value scalarOrVec, int32_t idx) -> Value {
       if (auto vecTy = dyn_cast<VectorType>(scalarOrVec.getType()))
-        return rewriter.create<spirv::CompositeExtractOp>(loc, scalarOrVec,
+        return spirv::CompositeExtractOp::create(rewriter, loc, scalarOrVec,
                                                           idx);
 
       assert(idx == 0 && "Invalid scalar element index");
@@ -712,10 +712,10 @@ struct VectorDeinterleaveOpConvert final
     // We cannot use `spirv::VectorShuffleOp` directly in this case, and need to
     // use `spirv::CompositeExtractOp`.
     if (n == 2) {
-      auto elem0 = rewriter.create<spirv::CompositeExtractOp>(
+      auto elem0 = spirv::CompositeExtractOp::create(rewriter,
           loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({0}));
 
-      auto elem1 = rewriter.create<spirv::CompositeExtractOp>(
+      auto elem1 = spirv::CompositeExtractOp::create(rewriter,
           loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({1}));
 
       rewriter.replaceOp(deinterleaveOp, {elem0, elem1});
@@ -733,11 +733,11 @@ struct VectorDeinterleaveOpConvert final
         llvm::map_to_vector(seqOdd, [](int i) { return i * 2 + 1; });
 
     // Create two SPIR-V shuffles.
-    auto shuffleEven = rewriter.create<spirv::VectorShuffleOp>(
+    auto shuffleEven = spirv::VectorShuffleOp::create(rewriter,
         loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesEven));
 
-    auto shuffleOdd = rewriter.create<spirv::VectorShuffleOp>(
+    auto shuffleOdd = spirv::VectorShuffleOp::create(rewriter,
         loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesOdd));
 
@@ -781,7 +781,7 @@ struct VectorLoadOpConverter final
     // to a scalar.
     Value castedAccessChain = (vectorType.getNumElements() == 1)
                                   ? accessChain
-                                  : rewriter.create<spirv::BitcastOp>(
+                                  : spirv::BitcastOp::create(rewriter,
                                         loc, vectorPtrType, accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, spirvVectorType,
@@ -823,7 +823,7 @@ struct VectorStoreOpConverter final
     // to a scalar.
     Value castedAccessChain = (vectorType.getNumElements() == 1)
                                   ? accessChain
-                                  : rewriter.create<spirv::BitcastOp>(
+                                  : spirv::BitcastOp::create(rewriter,
                                         loc, vectorPtrType, accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, castedAccessChain,
@@ -905,9 +905,9 @@ struct VectorReductionToIntDotProd final
       auto v4i8Type = VectorType::get({4}, i8Type);
       Location loc = op.getLoc();
       Value zero = spirv::ConstantOp::getZero(i8Type, loc, rewriter);
-      lhsIn = rewriter.create<spirv::CompositeConstructOp>(
+      lhsIn = spirv::CompositeConstructOp::create(rewriter,
           loc, v4i8Type, ValueRange{lhsIn, zero});
-      rhsIn = rewriter.create<spirv::CompositeConstructOp>(
+      rhsIn = spirv::CompositeConstructOp::create(rewriter,
           loc, v4i8Type, ValueRange{rhsIn, zero});
     }
 
@@ -971,14 +971,14 @@ struct VectorReductionToFPDotProd final
       Attribute oneAttr =
           rewriter.getFloatAttr(vectorType.getElementType(), 1.0);
       oneAttr = SplatElementsAttr::get(vectorType, oneAttr);
-      rhs = rewriter.create<spirv::ConstantOp>(loc, vectorType, oneAttr);
+      rhs = spirv::ConstantOp::create(rewriter, loc, vectorType, oneAttr);
     }
     assert(lhs);
     assert(rhs);
 
-    Value res = rewriter.create<spirv::DotOp>(loc, resultType, lhs, rhs);
+    Value res = spirv::DotOp::create(rewriter, loc, resultType, lhs, rhs);
     if (acc)
-      res = rewriter.create<spirv::FAddOp>(loc, acc, res);
+      res = spirv::FAddOp::create(rewriter, loc, acc, res);
 
     rewriter.replaceOp(op, res);
     return success();
@@ -1013,7 +1013,7 @@ struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
     source.reserve(numElements);
     for (int64_t i = 0; i < numElements; ++i) {
       Attribute intAttr = rewriter.getIntegerAttr(intType, i);
-      Value constOp = rewriter.create<spirv::ConstantOp>(loc, intType, intAttr);
+      Value constOp = spirv::ConstantOp::create(rewriter, loc, intType, intAttr);
       source.push_back(constOp);
     }
     rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(stepOp, dstType,
@@ -1056,7 +1056,7 @@ struct VectorToElementOpConvert final
       if (element.use_empty())
         continue;
 
-      Value result = rewriter.create<spirv::CompositeExtractOp>(
+      Value result = spirv::CompositeExtractOp::create(rewriter,
           loc, elementType, adaptor.getSource(),
           rewriter.getI32ArrayAttr({static_cast<int32_t>(idx)}));
       results[idx] = result;
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index 2e6a16ddbfdaa..8e074f91744e8 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -108,7 +108,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
 
   xegpu::CreateNdDescOp ndDesc;
   if (srcTy.hasStaticShape()) {
-    ndDesc = rewriter.create<xegpu::CreateNdDescOp>(loc, descType, src,
+    ndDesc = xegpu::CreateNdDescOp::create(rewriter, loc, descType, src,
                                                     getAsOpFoldResult(offsets));
   } else {
     // In case of any dynamic shapes, source's shape and strides have to be
@@ -116,7 +116,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
     SmallVector<Value> sourceDims;
     unsigned srcRank = srcTy.getRank();
     for (unsigned i = 0; i < srcRank; ++i)
-      sourceDims.push_back(rewriter.create<memref::DimOp>(loc, src, i));
+      sourceDims.push_back(memref::DimOp::create(rewriter, loc, src, i));
 
     SmallVector<int64_t> constOffsets;
     SmallVector<Value> dynOffsets;
@@ -135,17 +135,17 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
 
     // Compute strides in reverse order.
     SmallVector<Value> dynStrides;
-    Value accStride = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value accStride = arith::ConstantIndexOp::create(rewriter, loc, 1);
     // Last stride is guaranteed to be static and unit.
     for (int i = static_cast<int>(strides.size()) - 2; i >= 0; --i) {
       accStride =
-          rewriter.create<arith::MulIOp>(loc, accStride, sourceDims[i + 1]);
+          arith::MulIOp::create(rewriter, loc, accStride, sourceDims[i + 1]);
       if (strides[i] == ShapedType::kDynamic)
         dynStrides.push_back(accStride);
     }
     std::reverse(dynStrides.begin(), dynStrides.end());
 
-    ndDesc = rewriter.create<xegpu::CreateNdDescOp>(
+    ndDesc = xegpu::CreateNdDescOp::create(rewriter,
         loc, descType, src, dynOffsets, dynShapes, dynStrides,
         DenseI64ArrayAttr::get(rewriter.getContext(), constOffsets),
         DenseI64ArrayAttr::get(rewriter.getContext(), srcTy.getShape()),
@@ -200,7 +200,7 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
                                                   ArrayRef<int64_t>{1, 0});
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadOp = rewriter.create<xegpu::LoadNdOp>(
+    auto loadOp = xegpu::LoadNdOp::create(rewriter,
         loc, vecTy, ndDesc, /*packed=*/nullptr, transposeAttr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
@@ -238,7 +238,7 @@ struct TransferWriteLowering
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeOp =
-        rewriter.create<xegpu::StoreNdOp>(loc, writeOp.getVector(), ndDesc,
+        xegpu::StoreNdOp::create(rewriter, loc, writeOp.getVector(), ndDesc,
                                           /*l1_hint=*/hint,
                                           /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(writeOp, storeOp);
@@ -269,7 +269,7 @@ struct LoadLowering : public OpRewritePattern<vector::LoadOp> {
 
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadNdOp = rewriter.create<xegpu::LoadNdOp>(
+    auto loadNdOp = xegpu::LoadNdOp::create(rewriter,
         loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
@@ -303,7 +303,7 @@ struct StoreLowering : public OpRewritePattern<vector::StoreOp> {
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeNdOp =
-        rewriter.create<xegpu::StoreNdOp>(loc, vector, ndDesc,
+        xegpu::StoreNdOp::create(rewriter, loc, vector, ndDesc,
                                           /*l1_hint=*/hint,
                                           /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(storeOp, storeNdOp);
@@ -339,7 +339,7 @@ struct ContractionLowering : public OpRewritePattern<vector::ContractionOp> {
     if (!isRowMajorMatmul(contractOp.getIndexingMapsAttr()))
       return rewriter.notifyMatchFailure(contractOp, "Invalid indexing maps");
 
-    auto dpasOp = rewriter.create<xegpu::DpasOp>(
+    auto dpasOp = xegpu::DpasOp::create(rewriter,
         loc, TypeRange{contractOp.getResultType()}, ValueRange{lhs, rhs, acc});
     rewriter.replaceOp(contractOp, dpasOp);
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
index 37e0d2af55fe1..0230810f260ca 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -99,8 +99,8 @@ static Value flattenVecToBits(ConversionPatternRewriter &rewriter, Location loc,
       vectorType.getElementTypeBitWidth() * vectorType.getNumElements();
   Type allBitsType = rewriter.getIntegerType(bitwidth);
   auto allBitsVecType = VectorType::get({1}, allBitsType);
-  Value bitcast = rewriter.create<vector::BitCastOp>(loc, allBitsVecType, val);
-  Value scalar = rewriter.create<vector::ExtractOp>(loc, bitcast, 0);
+  Value bitcast = vector::BitCastOp::create(rewriter, loc, allBitsVecType, val);
+  Value scalar = vector::ExtractOp::create(rewriter, loc, bitcast, 0);
   return scalar;
 }
 
@@ -119,25 +119,25 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   SmallVector<NamedAttribute> loadAttrs;
   patchOperandSegmentSizes(origAttrs, loadAttrs, DataArgAction::Drop);
   Value initialLoad =
-      rewriter.create<RawBufferLoadOp>(loc, dataType, invariantArgs, loadAttrs);
+      RawBufferLoadOp::create(rewriter, loc, dataType, invariantArgs, loadAttrs);
   Block *currentBlock = rewriter.getInsertionBlock();
   Block *afterAtomic =
       rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
   Block *loopBlock = rewriter.createBlock(afterAtomic, {dataType}, {loc});
 
   rewriter.setInsertionPointToEnd(currentBlock);
-  rewriter.create<cf::BranchOp>(loc, loopBlock, initialLoad);
+  cf::BranchOp::create(rewriter, loc, loopBlock, initialLoad);
 
   rewriter.setInsertionPointToEnd(loopBlock);
   Value prevLoad = loopBlock->getArgument(0);
-  Value operated = rewriter.create<ArithOp>(loc, data, prevLoad);
+  Value operated = ArithOp::create(rewriter, loc, data, prevLoad);
   dataType = operated.getType();
 
   SmallVector<NamedAttribute> cmpswapAttrs;
   patchOperandSegmentSizes(origAttrs, cmpswapAttrs, DataArgAction::Duplicate);
   SmallVector<Value> cmpswapArgs = {operated, prevLoad};
   cmpswapArgs.append(invariantArgs.begin(), invariantArgs.end());
-  Value atomicRes = rewriter.create<RawBufferAtomicCmpswapOp>(
+  Value atomicRes = RawBufferAtomicCmpswapOp::create(rewriter,
       loc, dataType, cmpswapArgs, cmpswapAttrs);
 
   // We care about exact bitwise equality here, so do some bitcasts.
@@ -150,13 +150,13 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   if (auto floatDataTy = dyn_cast<FloatType>(dataType)) {
     Type equivInt = rewriter.getIntegerType(floatDataTy.getWidth());
     prevLoadForCompare =
-        rewriter.create<arith::BitcastOp>(loc, equivInt, prevLoad);
+        arith::BitcastOp::create(rewriter, loc, equivInt, prevLoad);
     atomicResForCompare =
-        rewriter.create<arith::BitcastOp>(loc, equivInt, atomicRes);
+        arith::BitcastOp::create(rewriter, loc, equivInt, atomicRes);
   }
-  Value canLeave = rewriter.create<arith::CmpIOp>(
+  Value canLeave = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::eq, atomicResForCompare, prevLoadForCompare);
-  rewriter.create<cf::CondBranchOp>(loc, canLeave, afterAtomic, ValueRange{},
+  cf::CondBranchOp::create(rewriter, loc, canLeave, afterAtomic, ValueRange{},
                                     loopBlock, atomicRes);
   rewriter.eraseOp(atomicOp);
   return success();
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
index 60c8660658a95..a11804c1d7741 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -56,10 +56,10 @@ static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
                                            vector::MaskedLoadOp maskedOp,
                                            bool passthru) {
   VectorType vectorType = maskedOp.getVectorType();
-  Value load = builder.create<vector::LoadOp>(
+  Value load = vector::LoadOp::create(builder,
       loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
   if (passthru)
-    load = builder.create<arith::SelectOp>(loc, vectorType, maskedOp.getMask(),
+    load = arith::SelectOp::create(builder,loc, vectorType, maskedOp.getMask(),
                                            load, maskedOp.getPassThru());
   return load;
 }
@@ -110,7 +110,7 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
     SmallVector<OpFoldResult> indices = maskedOp.getIndices();
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, src);
     SmallVector<OpFoldResult> strides =
         stridedMetadata.getConstifiedMixedStrides();
     SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
@@ -124,47 +124,47 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
 
     // delta = bufferSize - linearizedOffset
     Value vectorSizeOffset =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
+        arith::ConstantIndexOp::create(rewriter, loc, vectorSize);
     Value linearIndex =
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
     Value totalSize = getValueOrCreateConstantIndexOp(
         rewriter, loc, linearizedInfo.linearizedSize);
-    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
+    Value delta = arith::SubIOp::create(rewriter, loc, totalSize, linearIndex);
 
     // 1) check if delta < vectorSize
-    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
+    Value isOutofBounds = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
 
     // 2) check if (detla % elements_per_word != 0)
-    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
+    Value elementsPerWord = arith::ConstantIndexOp::create(rewriter,
         loc, llvm::divideCeil(32, elementBitWidth));
-    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
+    Value isNotWordAligned = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne,
-        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
+        arith::RemUIOp::create(rewriter, loc, delta, elementsPerWord),
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
 
     // We take the fallback of maskedload default lowering only it is both
     // out-of-bounds and not word aligned. The fallback ensures correct results
     // when loading at the boundary of the buffer since buffer load returns
     // inconsistent zeros for the whole word when boundary is crossed.
     Value ifCondition =
-        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
+        arith::AndIOp::create(rewriter, loc, isOutofBounds, isNotWordAligned);
 
     auto thenBuilder = [&](OpBuilder &builder, Location loc) {
       Operation *read = builder.clone(*maskedOp.getOperation());
       read->setAttr(kMaskedloadNeedsMask, builder.getUnitAttr());
       Value readResult = read->getResult(0);
-      builder.create<scf::YieldOp>(loc, readResult);
+      scf::YieldOp::create(builder, loc, readResult);
     };
 
     auto elseBuilder = [&](OpBuilder &builder, Location loc) {
       Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp,
                                                 /*passthru=*/true);
-      rewriter.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(rewriter, loc, res);
     };
 
     auto ifOp =
-        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
+        scf::IfOp::create(rewriter, loc, ifCondition, thenBuilder, elseBuilder);
 
     rewriter.replaceOp(maskedOp, ifOp);
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
index 195f59d625554..4771d655e36c2 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
@@ -37,7 +37,7 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       return rewriter.notifyMatchFailure(metadataOp,
                                          "not a fat raw buffer cast");
     Location loc = castOp.getLoc();
-    auto sourceMetadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+    auto sourceMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
         loc, castOp.getSource());
     SmallVector<Value> results;
     if (metadataOp.getBaseBuffer().use_empty()) {
@@ -48,13 +48,13 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       if (baseBufferType == castOp.getResult().getType()) {
         results.push_back(castOp.getResult());
       } else {
-        results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+        results.push_back(memref::ReinterpretCastOp::create(rewriter,
             loc, baseBufferType, castOp.getResult(), /*offset=*/0,
             /*sizes=*/ArrayRef<int64_t>{}, /*strides=*/ArrayRef<int64_t>{}));
       }
     }
     if (castOp.getResetOffset())
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, 0));
     else
       results.push_back(sourceMetadata.getOffset());
     llvm::append_range(results, sourceMetadata.getSizes());
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 12b375b373fa9..748ff1edbfeb2 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -76,8 +76,8 @@ static SmallVector<Value> getTileSizes(Location loc, amx::TileType tType,
   auto mattr = rewriter.getI16IntegerAttr(tType.getDimSize(0));
   auto nattr = rewriter.getI16IntegerAttr(tType.getDimSize(1) * bytes);
   return SmallVector<Value>{
-      rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, mattr),
-      rewriter.create<LLVM::ConstantOp>(loc, llvmInt16Type, nattr)};
+      LLVM::ConstantOp::create(rewriter, loc, llvmInt16Type, mattr),
+      LLVM::ConstantOp::create(rewriter, loc, llvmInt16Type, nattr)};
 }
 
 /// Maps the 2-dim memref shape to the 64-bit stride. Note that the buffer
@@ -95,7 +95,7 @@ static Value getStride(Location loc, MemRefType mType, Value base,
     // Dynamic stride needs code to compute the stride at runtime.
     MemRefDescriptor memrefDescriptor(base);
     auto attr = rewriter.getI64IntegerAttr(bytes);
-    Value scale = rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr);
+    Value scale = LLVM::ConstantOp::create(rewriter, loc, llvmInt64Type, attr);
     return rewriter
         .create<LLVM::MulOp>(loc, llvmInt64Type, scale,
                              memrefDescriptor.stride(rewriter, loc, preLast))
@@ -103,7 +103,7 @@ static Value getStride(Location loc, MemRefType mType, Value base,
   }
   // Use direct constant for static stride.
   auto attr = rewriter.getI64IntegerAttr(strides[preLast] * bytes);
-  return rewriter.create<LLVM::ConstantOp>(loc, llvmInt64Type, attr)
+  return LLVM::ConstantOp::create(rewriter, loc, llvmInt64Type, attr)
       .getResult();
 }
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index e8e8f624d806e..3113d5896741b 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -240,7 +240,7 @@ Operation *AffineDialect::materializeConstant(OpBuilder &builder,
                                               Attribute value, Type type,
                                               Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
 
@@ -1282,7 +1282,7 @@ mlir::affine::makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map,
   map = foldAttributesIntoMap(b, map, operands, valueOperands);
   composeAffineMapAndOperands(&map, &valueOperands, composeAffineMin);
   assert(map);
-  return b.create<AffineApplyOp>(loc, map, valueOperands);
+  return AffineApplyOp::create(b, loc, map, valueOperands);
 }
 
 AffineApplyOp
@@ -1389,7 +1389,7 @@ static OpTy makeComposedMinMax(OpBuilder &b, Location loc, AffineMap map,
   SmallVector<Value> valueOperands;
   map = foldAttributesIntoMap(b, map, operands, valueOperands);
   composeMultiResultAffineMap(map, valueOperands);
-  return b.create<OpTy>(loc, b.getIndexType(), map, valueOperands);
+  return OpTy::create(b, loc, b.getIndexType(), map, valueOperands);
 }
 
 AffineMinOp
@@ -2688,7 +2688,7 @@ FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInits());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  AffineForOp newLoop = rewriter.create<AffineForOp>(
+  AffineForOp newLoop = AffineForOp::create(rewriter,
       getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
       getUpperBoundOperands(), getUpperBoundMap(), getStepAsInt(), inits);
 
@@ -2831,7 +2831,7 @@ static void buildAffineLoopNestImpl(
         OpBuilder::InsertionGuard nestedGuard(nestedBuilder);
         bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
       }
-      nestedBuilder.create<AffineYieldOp>(nestedLoc);
+      AffineYieldOp::create(nestedBuilder, nestedLoc);
     };
 
     // Delegate actual loop creation to the callback in order to dispatch
@@ -2846,7 +2846,7 @@ static AffineForOp
 buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb,
                              int64_t ub, int64_t step,
                              AffineForOp::BodyBuilderFn bodyBuilderFn) {
-  return builder.create<AffineForOp>(loc, lb, ub, step,
+  return AffineForOp::create(builder, loc, lb, ub, step,
                                      /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
@@ -2860,7 +2860,7 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
   if (lbConst && ubConst)
     return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
                                         ubConst.value(), step, bodyBuilderFn);
-  return builder.create<AffineForOp>(loc, lb, builder.getDimIdentityMap(), ub,
+  return AffineForOp::create(builder, loc, lb, builder.getDimIdentityMap(), ub,
                                      builder.getDimIdentityMap(), step,
                                      /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
@@ -4883,7 +4883,7 @@ struct DropUnitExtentBasis
     Location loc = delinearizeOp->getLoc();
     auto getZero = [&]() -> Value {
       if (!zero)
-        zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+        zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
       return zero.value();
     };
 
@@ -4906,7 +4906,7 @@ struct DropUnitExtentBasis
 
     if (!newBasis.empty()) {
       // Will drop the leading nullptr from `basis` if there was no outer bound.
-      auto newDelinearizeOp = rewriter.create<affine::AffineDelinearizeIndexOp>(
+      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(rewriter,
           loc, delinearizeOp.getLinearIndex(), newBasis);
       int newIndex = 0;
       // Map back the new delinearized indices to the values they replace.
@@ -4971,11 +4971,11 @@ struct CancelDelinearizeOfLinearizeDisjointExactTail
       return success();
     }
 
-    Value newLinearize = rewriter.create<affine::AffineLinearizeIndexOp>(
+    Value newLinearize = affine::AffineLinearizeIndexOp::create(rewriter,
         linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
         ArrayRef<OpFoldResult>{linearizeBasis}.drop_back(numMatches),
         linearizeOp.getDisjoint());
-    auto newDelinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
         delinearizeOp.getLoc(), newLinearize,
         ArrayRef<OpFoldResult>{delinearizeBasis}.drop_back(numMatches),
         delinearizeOp.hasOuterBound());
@@ -5049,17 +5049,17 @@ struct SplitDelinearizeSpanningLastLinearizeArg final
           "need at least two elements to form the basis product");
 
     Value linearizeWithoutBack =
-        rewriter.create<affine::AffineLinearizeIndexOp>(
+        affine::AffineLinearizeIndexOp::create(rewriter,
             linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
             linearizeOp.getDynamicBasis(),
             linearizeOp.getStaticBasis().drop_back(),
             linearizeOp.getDisjoint());
     auto delinearizeWithoutSplitPart =
-        rewriter.create<affine::AffineDelinearizeIndexOp>(
+        affine::AffineDelinearizeIndexOp::create(rewriter,
             delinearizeOp.getLoc(), linearizeWithoutBack,
             delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
             delinearizeOp.hasOuterBound());
-    auto delinearizeBack = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(rewriter,
         delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
         basis.take_back(elemsToSplit), /*hasOuterBound=*/true);
     SmallVector<Value> results = llvm::to_vector(
@@ -5272,7 +5272,7 @@ OpFoldResult computeProduct(Location loc, OpBuilder &builder,
   }
   if (auto constant = dyn_cast<AffineConstantExpr>(result))
     return getAsIndexOpFoldResult(builder.getContext(), constant.getValue());
-  return builder.create<AffineApplyOp>(loc, result, dynamicPart).getResult();
+  return AffineApplyOp::create(builder, loc, result, dynamicPart).getResult();
 }
 
 /// If conseceutive outputs of a delinearize_index are linearized with the same
@@ -5437,7 +5437,7 @@ struct CancelLinearizeOfDelinearizePortion final
       newDelinBasis.erase(newDelinBasis.begin() + m.delinStart,
                           newDelinBasis.begin() + m.delinStart + m.length);
       newDelinBasis.insert(newDelinBasis.begin() + m.delinStart, newSize);
-      auto newDelinearize = rewriter.create<AffineDelinearizeIndexOp>(
+      auto newDelinearize = AffineDelinearizeIndexOp::create(rewriter,
           m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
           newDelinBasis);
 
@@ -5445,7 +5445,7 @@ struct CancelLinearizeOfDelinearizePortion final
       // create a residual affine.delinearize_index that delinearizes the
       // merged output into its component parts.
       Value combinedElem = newDelinearize.getResult(m.delinStart);
-      auto residualDelinearize = rewriter.create<AffineDelinearizeIndexOp>(
+      auto residualDelinearize = AffineDelinearizeIndexOp::create(rewriter,
           m.delinearize.getLoc(), combinedElem, basisToMerge);
 
       // Swap all the uses of the unaffected delinearize outputs to the new
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index f18cec5a14fae..df39544aeaa09 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -202,7 +202,7 @@ void AffineDataCopyGeneration::runOnBlock(Block *block,
 void AffineDataCopyGeneration::runOnOperation() {
   func::FuncOp f = getOperation();
   OpBuilder topBuilder(f.getBody());
-  zeroIndex = topBuilder.create<arith::ConstantIndexOp>(f.getLoc(), 0);
+  zeroIndex = arith::ConstantIndexOp::create(topBuilder, f.getLoc(), 0);
 
   // Nests that are copy-in's or copy-out's; the root AffineForOps of those
   // nests are stored herein.
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
index 5430bdc4ff858..fb0d0d5027085 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
@@ -58,7 +58,7 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
       // Note: basis elements and their products are, definitionally,
       // non-negative, so `nuw` is justified.
       if (dynamicPart)
-        dynamicPart = rewriter.create<arith::MulIOp>(
+        dynamicPart = arith::MulIOp::create(rewriter,
             loc, dynamicPart, dynamicBasis[dynamicIndex - 1], ovflags);
       else
         dynamicPart = dynamicBasis[dynamicIndex - 1];
@@ -74,7 +74,7 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
           rewriter.createOrFold<arith::ConstantIndexOp>(loc, staticPart);
       if (dynamicPart)
         stride =
-            rewriter.create<arith::MulIOp>(loc, dynamicPart, stride, ovflags);
+            arith::MulIOp::create(rewriter, loc, dynamicPart, stride, ovflags);
       result.push_back(stride);
     }
   }
@@ -106,19 +106,19 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
   Value zero = rewriter.createOrFold<arith::ConstantIndexOp>(loc, 0);
 
   Value initialPart =
-      rewriter.create<arith::FloorDivSIOp>(loc, linearIdx, strides.front());
+      arith::FloorDivSIOp::create(rewriter, loc, linearIdx, strides.front());
   results.push_back(initialPart);
 
   auto emitModTerm = [&](Value stride) -> Value {
-    Value remainder = rewriter.create<arith::RemSIOp>(loc, linearIdx, stride);
-    Value remainderNegative = rewriter.create<arith::CmpIOp>(
+    Value remainder = arith::RemSIOp::create(rewriter, loc, linearIdx, stride);
+    Value remainderNegative = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, remainder, zero);
     // If the correction is relevant, this term is <= stride, which is known
     // to be positive in `index`. Otherwise, while 2 * stride might overflow,
     // this branch won't be taken, so the risk of `poison` is fine.
-    Value corrected = rewriter.create<arith::AddIOp>(
+    Value corrected = arith::AddIOp::create(rewriter,
         loc, remainder, stride, arith::IntegerOverflowFlags::nsw);
-    Value mod = rewriter.create<arith::SelectOp>(loc, remainderNegative,
+    Value mod = arith::SelectOp::create(rewriter, loc, remainderNegative,
                                                  corrected, remainder);
     return mod;
   };
@@ -131,7 +131,7 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
     // We know both inputs are positive, so floorDiv == div.
     // This could potentially be a divui, but it's not clear if that would
     // cause issues.
-    Value divided = rewriter.create<arith::DivSIOp>(loc, modulus, nextStride);
+    Value divided = arith::DivSIOp::create(rewriter, loc, modulus, nextStride);
     results.push_back(divided);
   }
 
@@ -167,7 +167,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   // our hands on an `OpOperand&` for the loop invariant counting function.
   for (auto [stride, idxOp] :
        llvm::zip_equal(strides, llvm::drop_end(op.getMultiIndexMutable()))) {
-    Value scaledIdx = rewriter.create<arith::MulIOp>(
+    Value scaledIdx = arith::MulIOp::create(rewriter,
         loc, idxOp.get(), stride, arith::IntegerOverflowFlags::nsw);
     int64_t numHoistableLoops = numEnclosingInvariantLoops(idxOp);
     scaledValues.emplace_back(scaledIdx, numHoistableLoops);
@@ -184,7 +184,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   Value result = scaledValues.front().first;
   for (auto [scaledValue, numHoistableLoops] : llvm::drop_begin(scaledValues)) {
     std::ignore = numHoistableLoops;
-    result = rewriter.create<arith::AddIOp>(loc, result, scaledValue,
+    result = arith::AddIOp::create(rewriter, loc, result, scaledValue,
                                             arith::IntegerOverflowFlags::nsw);
   }
   rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
index 4fd0cf9b3cd25..a78cea5bff6dd 100644
--- a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
@@ -88,7 +88,7 @@ static AffineApplyOp createSubApply(RewriterBase &rewriter,
   auto rhsMap = AffineMap::get(m.getNumDims(), m.getNumSymbols(), expr, ctx);
   SmallVector<Value> rhsOperands = originalOp->getOperands();
   canonicalizeMapAndOperands(&rhsMap, &rhsOperands);
-  return rewriter.create<AffineApplyOp>(originalOp.getLoc(), rhsMap,
+  return AffineApplyOp::create(rewriter, originalOp.getLoc(), rhsMap,
                                         rhsOperands);
 }
 
@@ -160,7 +160,7 @@ FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
   auto current = createSubApply(rewriter, op, subExpressions[0]);
   for (int64_t i = 1, e = subExpressions.size(); i < e; ++i) {
     Value tmp = createSubApply(rewriter, op, subExpressions[i]);
-    current = rewriter.create<AffineApplyOp>(op.getLoc(), binMap,
+    current = AffineApplyOp::create(rewriter, op.getLoc(), binMap,
                                              ValueRange{current, tmp});
     LLVM_DEBUG(DBGS() << "--reassociate into: " << current << "\n");
   }
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 95848d0b67547..b19206c7afb92 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -424,7 +424,7 @@ static Value createPrivateMemRef(AffineForOp forOp,
   // consumer loop nests to reduce their live range. Currently they are added
   // at the beginning of the block, because loop nests can be reordered
   // during the fusion pass.
-  Value newMemRef = top.create<memref::AllocOp>(forOp.getLoc(), newMemRefType);
+  Value newMemRef = memref::AllocOp::create(top, forOp.getLoc(), newMemRefType);
 
   // Build an AffineMap to remap access functions based on lower bound offsets.
   SmallVector<AffineExpr, 4> remapExprs;
diff --git a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
index 05a352f39a93c..17384654586c4 100644
--- a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
@@ -100,7 +100,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
 
   // Create and place the alloc right before the 'affine.for' operation.
-  Value newMemRef = bOuter.create<memref::AllocOp>(
+  Value newMemRef = memref::AllocOp::create(bOuter,
       forOp.getLoc(), newMemRefType, allocOperands);
 
   // Create 'iv mod 2' value to index the leading dimension.
@@ -108,7 +108,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   int64_t step = forOp.getStepAsInt();
   auto modTwoMap =
       AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, d0.floorDiv(step) % 2);
-  auto ivModTwoOp = bInner.create<AffineApplyOp>(forOp.getLoc(), modTwoMap,
+  auto ivModTwoOp = AffineApplyOp::create(bInner, forOp.getLoc(), modTwoMap,
                                                  forOp.getInductionVar());
 
   // replaceAllMemRefUsesWith will succeed unless the forOp body has
@@ -130,7 +130,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
   // Insert the dealloc op right after the for loop.
   bOuter.setInsertionPointAfter(forOp);
-  bOuter.create<memref::DeallocOp>(forOp.getLoc(), newMemRef);
+  memref::DeallocOp::create(bOuter, forOp.getLoc(), newMemRef);
 
   return true;
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
index 1a266b72d1f8d..9537d3e75c26a 100644
--- a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
@@ -51,10 +51,10 @@ OpFoldResult affine::materializeComputedBound(
            "expected dynamic dim");
     if (isa<RankedTensorType>(value.getType())) {
       // A tensor dimension is used: generate a tensor.dim.
-      operands.push_back(b.create<tensor::DimOp>(loc, value, *dim));
+      operands.push_back(tensor::DimOp::create(b, loc, value, *dim));
     } else if (isa<MemRefType>(value.getType())) {
       // A memref dimension is used: generate a memref.dim.
-      operands.push_back(b.create<memref::DimOp>(loc, value, *dim));
+      operands.push_back(memref::DimOp::create(b, loc, value, *dim));
     } else {
       llvm_unreachable("cannot generate DimOp for unsupported shaped type");
     }
@@ -76,7 +76,7 @@ OpFoldResult affine::materializeComputedBound(
         operands[expr.getPosition() + boundMap.getNumDims()]);
   // General case: build affine.apply op.
   return static_cast<OpFoldResult>(
-      b.create<affine::AffineApplyOp>(loc, boundMap, operands).getResult());
+      affine::AffineApplyOp::create(b, loc, boundMap, operands).getResult());
 }
 
 FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound(
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index 7fae260767e0a..b078514cbf978 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -905,7 +905,7 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   for (auto resultExpr : map.getResults()) {
     auto singleResMap =
         AffineMap::get(map.getNumDims(), map.getNumSymbols(), resultExpr);
-    auto afOp = state.builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+    auto afOp = state.AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                     mapOperands);
     results.push_back(afOp);
   }
@@ -961,7 +961,7 @@ static arith::ConstantOp vectorizeConstant(arith::ConstantOp constOp,
   auto vecForOp = cast<AffineForOp>(parentOp);
   state.builder.setInsertionPointToStart(vecForOp.getBody());
   auto newConstOp =
-      state.builder.create<arith::ConstantOp>(constOp.getLoc(), vecAttr);
+      state.arith::ConstantOp::create(builder, constOp.getLoc(), vecAttr);
 
   // Register vector replacement for future uses in the scope.
   state.registerOpVectorReplacement(constOp, newConstOp);
@@ -986,7 +986,7 @@ static Operation *vectorizeAffineApplyOp(AffineApplyOp applyOp,
     }
   }
 
-  auto newApplyOp = state.builder.create<AffineApplyOp>(
+  auto newApplyOp = state.AffineApplyOp::create(builder,
       applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
 
   // Register the new affine.apply result.
@@ -1010,7 +1010,7 @@ static arith::ConstantOp createInitialVector(arith::AtomicRMWKind reductionKind,
   auto vecTy = getVectorType(scalarTy, state.strategy);
   auto vecAttr = DenseElementsAttr::get(vecTy, valueAttr);
   auto newConstOp =
-      state.builder.create<arith::ConstantOp>(oldOperand.getLoc(), vecAttr);
+      state.arith::ConstantOp::create(builder, oldOperand.getLoc(), vecAttr);
 
   return newConstOp;
 }
@@ -1062,10 +1062,10 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   AffineMap ubMap = vecForOp.getUpperBoundMap();
   Value ub;
   if (ubMap.getNumResults() == 1)
-    ub = state.builder.create<AffineApplyOp>(loc, vecForOp.getUpperBoundMap(),
+    ub = state.AffineApplyOp::create(builder, loc, vecForOp.getUpperBoundMap(),
                                              vecForOp.getUpperBoundOperands());
   else
-    ub = state.builder.create<AffineMinOp>(loc, vecForOp.getUpperBoundMap(),
+    ub = state.AffineMinOp::create(builder, loc, vecForOp.getUpperBoundMap(),
                                            vecForOp.getUpperBoundOperands());
   // Then we compute the number of (original) iterations left in the loop.
   AffineExpr subExpr =
@@ -1080,7 +1080,7 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   Type maskTy = VectorType::get(state.strategy->vectorSizes,
                                 state.builder.getIntegerType(1));
   Value mask =
-      state.builder.create<vector::CreateMaskOp>(loc, maskTy, itersLeft);
+      state.vector::CreateMaskOp::create(builder, loc, maskTy, itersLeft);
 
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ creating a mask:\n"
                     << itersLeft << "\n"
@@ -1123,7 +1123,7 @@ static Operation *vectorizeUniform(Value uniformVal,
   state.builder.setInsertionPointAfterValue(uniformScalarRepl);
 
   auto vectorTy = getVectorType(uniformVal.getType(), state.strategy);
-  auto bcastOp = state.builder.create<BroadcastOp>(uniformVal.getLoc(),
+  auto bcastOp = state.BroadcastOp::create(builder, uniformVal.getLoc(),
                                                    vectorTy, uniformScalarRepl);
   state.registerValueVectorReplacement(uniformVal, bcastOp);
   return bcastOp;
@@ -1256,7 +1256,7 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.builder.create<vector::TransferReadOp>(
+  auto transfer = state.vector::TransferReadOp::create(builder,
       loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
       /*padding=*/std::nullopt, permutationMap);
 
@@ -1303,7 +1303,7 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.builder.create<vector::TransferWriteOp>(
+  auto transfer = state.vector::TransferWriteOp::create(builder,
       storeOp.getLoc(), vectorValue, storeOp.getMemRef(), indices,
       permutationMap);
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
@@ -1387,7 +1387,7 @@ static Operation *vectorizeAffineForOp(AffineForOp forOp,
     }
   }
 
-  auto vecForOp = state.builder.create<AffineForOp>(
+  auto vecForOp = state.AffineForOp::create(builder,
       forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(),
       forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep,
       vecIterOperands,
@@ -1512,7 +1512,7 @@ static Operation *vectorizeAffineYieldOp(AffineYieldOp yieldOp,
       // IterOperands are neutral element vectors.
       Value neutralVal = cast<AffineForOp>(newParentOp).getInits()[i];
       state.builder.setInsertionPoint(combinerOps.back());
-      Value maskedReducedVal = state.builder.create<arith::SelectOp>(
+      Value maskedReducedVal = state.arith::SelectOp::create(builder,
           reducedVal.getLoc(), mask, reducedVal, neutralVal);
       LLVM_DEBUG(
           dbgs() << "\n[early-vect]+++++ masking an input to a binary op that"
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 0501616ad912c..0f16e84263ec1 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -55,7 +55,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
 
   OpBuilder b(forOp);
   auto lbMap = forOp.getLowerBoundMap();
-  auto lb = b.create<AffineApplyOp>(forOp.getLoc(), lbMap,
+  auto lb = AffineApplyOp::create(b, forOp.getLoc(), lbMap,
                                     forOp.getLowerBoundOperands());
 
   // For each upper bound expr, get the range.
@@ -72,7 +72,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
     auto bumpMap = AffineMap::get(tripCountMap.getNumDims(),
                                   tripCountMap.getNumSymbols(), bumpExprs[i]);
     bumpValues[i] =
-        b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, tripCountOperands);
+        AffineApplyOp::create(b, forOp.getLoc(), bumpMap, tripCountOperands);
   }
 
   SmallVector<AffineExpr, 4> newUbExprs(tripCountMap.getNumResults());
@@ -135,7 +135,7 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         builder.setInsertionPointToStart(&func.getFunctionBody().front());
       else
         builder.setInsertionPoint(forOp);
-      auto constOp = builder.create<arith::ConstantIndexOp>(
+      auto constOp = arith::ConstantIndexOp::create(builder,
           forOp.getLoc(), forOp.getConstantLowerBound());
       iv.replaceAllUsesWith(constOp);
     } else {
@@ -147,7 +147,7 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         iv.replaceAllUsesWith(lbOperands[0]);
       } else {
         auto affineApplyOp =
-            builder.create<AffineApplyOp>(forOp.getLoc(), lbMap, lbOperands);
+            AffineApplyOp::create(builder, forOp.getLoc(), lbMap, lbOperands);
         iv.replaceAllUsesWith(affineApplyOp);
       }
     }
@@ -182,7 +182,7 @@ static AffineForOp generateShiftedLoop(
   assert(ubMap.getNumInputs() == ubOperands.size());
 
   auto loopChunk =
-      b.create<AffineForOp>(srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
+      AffineForOp::create(b, srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
                             ubMap, srcForOp.getStepAsInt());
   auto loopChunkIV = loopChunk.getInductionVar();
   auto srcIV = srcForOp.getInductionVar();
@@ -198,7 +198,7 @@ static AffineForOp generateShiftedLoop(
     // Generate the remapping if the shift is not zero: remappedIV = newIV -
     // shift.
     if (!srcIV.use_empty() && shift != 0) {
-      auto ivRemap = bodyBuilder.create<AffineApplyOp>(
+      auto ivRemap = AffineApplyOp::create(bodyBuilder,
           srcForOp.getLoc(),
           bodyBuilder.getSingleDimShiftAffineMap(
               -static_cast<int64_t>(srcForOp.getStepAsInt() * shift)),
@@ -434,7 +434,7 @@ static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
   for (unsigned i = 0; i < width; i++) {
     OpBuilder b(topLoop);
     // Loop bounds will be set later.
-    AffineForOp pointLoop = b.create<AffineForOp>(loc, 0, 0);
+    AffineForOp pointLoop = AffineForOp::create(b, loc, 0, 0);
     pointLoop.getBody()->getOperations().splice(
         pointLoop.getBody()->begin(), topLoop->getBlock()->getOperations(),
         topLoop);
@@ -448,7 +448,7 @@ static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
   for (unsigned i = width; i < 2 * width; i++) {
     OpBuilder b(topLoop);
     // Loop bounds will be set later.
-    AffineForOp tileSpaceLoop = b.create<AffineForOp>(loc, 0, 0);
+    AffineForOp tileSpaceLoop = AffineForOp::create(b, loc, 0, 0);
     tileSpaceLoop.getBody()->getOperations().splice(
         tileSpaceLoop.getBody()->begin(), topLoop->getBlock()->getOperations(),
         topLoop);
@@ -1063,7 +1063,7 @@ LogicalResult mlir::affine::loopUnrollByFactor(
         // iv' = iv + i * step
         auto d0 = b.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, d0 + i * step);
-        return b.create<AffineApplyOp>(forOp.getLoc(), bumpMap, iv);
+        return AffineApplyOp::create(b, forOp.getLoc(), bumpMap, iv);
       },
       /*annotateFn=*/annotateFn,
       /*iterArgs=*/iterArgs, /*yieldedValues=*/yieldedValues);
@@ -1227,7 +1227,7 @@ LogicalResult mlir::affine::loopUnrollJamByFactor(AffineForOp forOp,
         auto d0 = builder.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, d0 + i * step);
         auto ivUnroll =
-            builder.create<AffineApplyOp>(forOp.getLoc(), bumpMap, forOpIV);
+            AffineApplyOp::create(builder, forOp.getLoc(), bumpMap, forOpIV);
         operandMaps[i - 1].map(forOpIV, ivUnroll);
       }
       // Clone the sub-block being unroll-jammed.
@@ -1556,7 +1556,7 @@ stripmineSink(AffineForOp forOp, uint64_t factor,
   for (auto t : targets) {
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator(t.getBody());
-    auto newForOp = b.create<AffineForOp>(t.getLoc(), lbOperands, lbMap,
+    auto newForOp = AffineForOp::create(b, t.getLoc(), lbOperands, lbMap,
                                           ubOperands, ubMap, originalStep);
     auto begin = t.getBody()->begin();
     // Skip terminator and `newForOp` which is just before the terminator.
@@ -1631,9 +1631,9 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
   // 1. Store the upper bound of the outermost loop in a variable.
   Value prev;
   if (!llvm::hasSingleElement(origUbMap.getResults()))
-    prev = builder.create<AffineMinOp>(loc, origUbMap, ubOperands);
+    prev = AffineMinOp::create(builder, loc, origUbMap, ubOperands);
   else
-    prev = builder.create<AffineApplyOp>(loc, origUbMap, ubOperands);
+    prev = AffineApplyOp::create(builder, loc, origUbMap, ubOperands);
   upperBoundSymbols.push_back(prev);
 
   // 2. Emit code computing the upper bound of the coalesced loop as product of
@@ -1645,15 +1645,15 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
     Value upperBound;
     // If upper bound map has more than one result, take their minimum.
     if (!llvm::hasSingleElement(origUbMap.getResults()))
-      upperBound = builder.create<AffineMinOp>(loc, origUbMap, ubOperands);
+      upperBound = AffineMinOp::create(builder, loc, origUbMap, ubOperands);
     else
-      upperBound = builder.create<AffineApplyOp>(loc, origUbMap, ubOperands);
+      upperBound = AffineApplyOp::create(builder, loc, origUbMap, ubOperands);
     upperBoundSymbols.push_back(upperBound);
     SmallVector<Value, 4> operands;
     operands.push_back(prev);
     operands.push_back(upperBound);
     // Maintain running product of loop upper bounds.
-    prev = builder.create<AffineApplyOp>(
+    prev = AffineApplyOp::create(builder,
         loc,
         AffineMap::get(/*dimCount=*/1,
                        /*symbolCount=*/1,
@@ -1683,7 +1683,7 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> operands;
       operands.push_back(previous);
       operands.push_back(upperBoundSymbols[idx]);
-      previous = builder.create<AffineApplyOp>(
+      previous = AffineApplyOp::create(builder,
           loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
@@ -1700,7 +1700,7 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> applyOperands;
       applyOperands.push_back(previous);
       applyOperands.push_back(upperBoundSymbols[idx - 1]);
-      inductionVariable = builder.create<AffineApplyOp>(
+      inductionVariable = AffineApplyOp::create(builder,
           loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
@@ -1738,21 +1738,21 @@ void mlir::affine::mapLoopToProcessorIds(scf::ForOp forOp,
 
   Value linearIndex = processorId.front();
   for (unsigned i = 1, e = processorId.size(); i < e; ++i) {
-    auto mulApplyOp = b.create<AffineApplyOp>(
+    auto mulApplyOp = AffineApplyOp::create(b,
         loc, mulMap, ValueRange{linearIndex, numProcessors[i]});
-    linearIndex = b.create<AffineApplyOp>(
+    linearIndex = AffineApplyOp::create(b,
         loc, addMap, ValueRange{mulApplyOp, processorId[i]});
   }
 
-  auto mulApplyOp = b.create<AffineApplyOp>(
+  auto mulApplyOp = AffineApplyOp::create(b,
       loc, mulMap, ValueRange{linearIndex, forOp.getStep()});
-  Value lb = b.create<AffineApplyOp>(
+  Value lb = AffineApplyOp::create(b,
       loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()});
   forOp.setLowerBound(lb);
 
   Value step = forOp.getStep();
   for (auto numProcs : numProcessors)
-    step = b.create<AffineApplyOp>(loc, mulMap, ValueRange{numProcs, step});
+    step = AffineApplyOp::create(b, loc, mulMap, ValueRange{numProcs, step});
   forOp.setStep(step);
 }
 
@@ -1889,7 +1889,7 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
 
     auto fastBufOffsetMap =
         AffineMap::get(lbOperands.size(), 0, fastBufOffsets[d]);
-    auto offset = b.create<AffineApplyOp>(loc, fastBufOffsetMap, lbOperands);
+    auto offset = AffineApplyOp::create(b, loc, fastBufOffsetMap, lbOperands);
 
     // Construct the subscript for the fast memref being copied into/from:
     // x - offset_x.
@@ -1916,16 +1916,16 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
 
   if (!isCopyOut) {
     // Copy in.
-    auto load = b.create<AffineLoadOp>(loc, memref, memIndices);
-    b.create<AffineStoreOp>(loc, load, fastMemRef, fastBufMap,
+    auto load = AffineLoadOp::create(b, loc, memref, memIndices);
+    AffineStoreOp::create(b, loc, load, fastMemRef, fastBufMap,
                             fastBufMapOperands);
     return copyNestRoot;
   }
 
   // Copy out.
   auto load =
-      b.create<AffineLoadOp>(loc, fastMemRef, fastBufMap, fastBufMapOperands);
-  b.create<AffineStoreOp>(loc, load, memref, memIndices);
+      AffineLoadOp::create(b, loc, fastMemRef, fastBufMap, fastBufMapOperands);
+  AffineStoreOp::create(b, loc, load, memref, memIndices);
   return copyNestRoot;
 }
 
@@ -1960,7 +1960,7 @@ static LogicalResult generateCopy(
 
   auto f = begin->getParentOfType<FunctionOpInterface>();
   OpBuilder topBuilder(f.getFunctionBody());
-  Value zeroIndex = topBuilder.create<arith::ConstantIndexOp>(f.getLoc(), 0);
+  Value zeroIndex = arith::ConstantIndexOp::create(topBuilder, f.getLoc(), 0);
 
   *sizeInBytes = 0;
 
@@ -2071,7 +2071,7 @@ static LogicalResult generateCopy(
         memIndices.push_back(zeroIndex);
       } else {
         memIndices.push_back(
-            top.create<arith::ConstantIndexOp>(loc, indexVal).getResult());
+            arith::ConstantIndexOp::create(top, loc, indexVal).getResult());
       }
     } else {
       // The coordinate for the start location is just the lower bound along the
@@ -2085,7 +2085,7 @@ static LogicalResult generateCopy(
       lbs[d] = lbs[d].replaceDimsAndSymbols(
           /*dimReplacements=*/{}, symReplacements, lbs[d].getNumSymbols(),
           /*numResultSyms=*/0);
-      memIndices.push_back(b.create<AffineApplyOp>(loc, lbs[d], regionSymbols));
+      memIndices.push_back(AffineApplyOp::create(b, loc, lbs[d], regionSymbols));
     }
     // The fast buffer is copied into at location zero; addressing is relative.
     bufIndices.push_back(zeroIndex);
@@ -2109,7 +2109,7 @@ static LogicalResult generateCopy(
     // Create the fast memory space buffer just before the 'affine.for'
     // operation.
     fastMemRef =
-        prologue.create<memref::AllocOp>(loc, fastMemRefType).getResult();
+        memref::AllocOp::create(prologue, loc, fastMemRefType).getResult();
     // Record it.
     fastBufferMap[memref] = fastMemRef;
     // fastMemRefType is a constant shaped memref.
@@ -2126,7 +2126,7 @@ static LogicalResult generateCopy(
     fastMemRef = fastBufferMap[memref];
   }
 
-  auto numElementsSSA = top.create<arith::ConstantIndexOp>(loc, *numElements);
+  auto numElementsSSA = arith::ConstantIndexOp::create(top, loc, *numElements);
 
   Value dmaStride;
   Value numEltPerDmaStride;
@@ -2143,8 +2143,8 @@ static LogicalResult generateCopy(
 
     if (!dmaStrideInfos.empty()) {
       dmaStride =
-          top.create<arith::ConstantIndexOp>(loc, dmaStrideInfos[0].stride);
-      numEltPerDmaStride = top.create<arith::ConstantIndexOp>(
+          arith::ConstantIndexOp::create(top, loc, dmaStrideInfos[0].stride);
+      numEltPerDmaStride = arith::ConstantIndexOp::create(top,
           loc, dmaStrideInfos[0].numEltPerStride);
     }
   }
@@ -2175,20 +2175,20 @@ static LogicalResult generateCopy(
     // Create a tag (single element 1-d memref) for the DMA.
     auto tagMemRefType = MemRefType::get({1}, top.getIntegerType(32), {},
                                          copyOptions.tagMemorySpace);
-    auto tagMemRef = prologue.create<memref::AllocOp>(loc, tagMemRefType);
+    auto tagMemRef = memref::AllocOp::create(prologue, loc, tagMemRefType);
 
     SmallVector<Value, 4> tagIndices({zeroIndex});
     auto tagAffineMap = b.getMultiDimIdentityMap(tagIndices.size());
     fullyComposeAffineMapAndOperands(&tagAffineMap, &tagIndices);
     if (!region.isWrite()) {
       // DMA non-blocking read from original buffer to fast buffer.
-      b.create<AffineDmaStartOp>(loc, memref, memAffineMap, memIndices,
+      AffineDmaStartOp::create(b, loc, memref, memAffineMap, memIndices,
                                  fastMemRef, bufAffineMap, bufIndices,
                                  tagMemRef, tagAffineMap, tagIndices,
                                  numElementsSSA, dmaStride, numEltPerDmaStride);
     } else {
       // DMA non-blocking write from fast buffer to the original memref.
-      auto op = b.create<AffineDmaStartOp>(
+      auto op = AffineDmaStartOp::create(b,
           loc, fastMemRef, bufAffineMap, bufIndices, memref, memAffineMap,
           memIndices, tagMemRef, tagAffineMap, tagIndices, numElementsSSA,
           dmaStride, numEltPerDmaStride);
@@ -2199,11 +2199,11 @@ static LogicalResult generateCopy(
     }
 
     // Matching DMA wait to block on completion; tag always has a 0 index.
-    b.create<AffineDmaWaitOp>(loc, tagMemRef, tagAffineMap, zeroIndex,
+    AffineDmaWaitOp::create(b, loc, tagMemRef, tagAffineMap, zeroIndex,
                               numElementsSSA);
 
     // Generate dealloc for the tag.
-    auto tagDeallocOp = epilogue.create<memref::DeallocOp>(loc, tagMemRef);
+    auto tagDeallocOp = memref::DeallocOp::create(epilogue, loc, tagMemRef);
     if (*nEnd == end && isCopyOutAtEndOfBlock)
       // Since new ops are being appended (for outgoing DMAs), adjust the end to
       // mark end of range of the original.
@@ -2212,7 +2212,7 @@ static LogicalResult generateCopy(
 
   // Generate dealloc for the buffer.
   if (!existingBuf) {
-    auto bufDeallocOp = epilogue.create<memref::DeallocOp>(loc, fastMemRef);
+    auto bufDeallocOp = memref::DeallocOp::create(epilogue, loc, fastMemRef);
     // When generating pointwise copies, `nEnd' has to be set to deallocOp on
     // the fast buffer (since it marks the new end insertion point).
     if (!copyOptions.generateDma && *nEnd == end && isCopyOutAtEndOfBlock)
@@ -2582,7 +2582,7 @@ AffineForOp mlir::affine::createCanonicalizedAffineForOp(
   canonicalizeMapAndOperands(&ubMap, &upperOperands);
   ubMap = removeDuplicateExprs(ubMap);
 
-  return b.create<AffineForOp>(loc, lowerOperands, lbMap, upperOperands, ubMap,
+  return AffineForOp::create(b, loc, lowerOperands, lbMap, upperOperands, ubMap,
                                step);
 }
 
@@ -2666,7 +2666,7 @@ static AffineIfOp createSeparationCondition(MutableArrayRef<AffineForOp> loops,
   SmallVector<Value, 4> setOperands;
   cst.getValues(0, cst.getNumDimAndSymbolVars(), &setOperands);
   canonicalizeSetAndOperands(&ifCondSet, &setOperands);
-  return b.create<AffineIfOp>(loops[0].getLoc(), ifCondSet, setOperands,
+  return AffineIfOp::create(b, loops[0].getLoc(), ifCondSet, setOperands,
                               /*withElseRegion=*/true);
 }
 
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 83816a1543708..0b0950968ecba 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -57,7 +57,7 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     if (!lhs || !rhs)
       return nullptr;
-    auto op = builder.create<OpTy>(loc, lhs, rhs, overflowFlags);
+    auto op = OpTy::create(builder, loc, lhs, rhs, overflowFlags);
     return op.getResult();
   }
 
@@ -91,13 +91,13 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value remainder = builder.create<arith::RemSIOp>(loc, lhs, rhs);
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value isRemainderNegative = builder.create<arith::CmpIOp>(
+    Value remainder = arith::RemSIOp::create(builder, loc, lhs, rhs);
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value isRemainderNegative = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::slt, remainder, zeroCst);
     Value correctedRemainder =
-        builder.create<arith::AddIOp>(loc, remainder, rhs);
-    Value result = builder.create<arith::SelectOp>(
+        arith::AddIOp::create(builder, loc, remainder, rhs);
+    Value result = arith::SelectOp::create(builder,
         loc, isRemainderNegative, correctedRemainder, remainder);
     return result;
   }
@@ -130,17 +130,17 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value noneCst = builder.create<arith::ConstantIndexOp>(loc, -1);
-    Value negative = builder.create<arith::CmpIOp>(
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value noneCst = arith::ConstantIndexOp::create(builder, loc, -1);
+    Value negative = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::slt, lhs, zeroCst);
-    Value negatedDecremented = builder.create<arith::SubIOp>(loc, noneCst, lhs);
+    Value negatedDecremented = arith::SubIOp::create(builder, loc, noneCst, lhs);
     Value dividend =
-        builder.create<arith::SelectOp>(loc, negative, negatedDecremented, lhs);
-    Value quotient = builder.create<arith::DivSIOp>(loc, dividend, rhs);
+        arith::SelectOp::create(builder, loc, negative, negatedDecremented, lhs);
+    Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value correctedQuotient =
-        builder.create<arith::SubIOp>(loc, noneCst, quotient);
-    Value result = builder.create<arith::SelectOp>(loc, negative,
+        arith::SubIOp::create(builder, loc, noneCst, quotient);
+    Value result = arith::SelectOp::create(builder, loc, negative,
                                                    correctedQuotient, quotient);
     return result;
   }
@@ -169,26 +169,26 @@ class AffineApplyExpander
     auto rhs = visit(expr.getRHS());
     assert(lhs && rhs && "unexpected affine expr lowering failure");
 
-    Value zeroCst = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value oneCst = builder.create<arith::ConstantIndexOp>(loc, 1);
-    Value nonPositive = builder.create<arith::CmpIOp>(
+    Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value oneCst = arith::ConstantIndexOp::create(builder, loc, 1);
+    Value nonPositive = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::sle, lhs, zeroCst);
-    Value negated = builder.create<arith::SubIOp>(loc, zeroCst, lhs);
-    Value decremented = builder.create<arith::SubIOp>(loc, lhs, oneCst);
+    Value negated = arith::SubIOp::create(builder, loc, zeroCst, lhs);
+    Value decremented = arith::SubIOp::create(builder, loc, lhs, oneCst);
     Value dividend =
-        builder.create<arith::SelectOp>(loc, nonPositive, negated, decremented);
-    Value quotient = builder.create<arith::DivSIOp>(loc, dividend, rhs);
+        arith::SelectOp::create(builder, loc, nonPositive, negated, decremented);
+    Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value negatedQuotient =
-        builder.create<arith::SubIOp>(loc, zeroCst, quotient);
+        arith::SubIOp::create(builder, loc, zeroCst, quotient);
     Value incrementedQuotient =
-        builder.create<arith::AddIOp>(loc, quotient, oneCst);
-    Value result = builder.create<arith::SelectOp>(
+        arith::AddIOp::create(builder, loc, quotient, oneCst);
+    Value result = arith::SelectOp::create(builder,
         loc, nonPositive, negatedQuotient, incrementedQuotient);
     return result;
   }
 
   Value visitConstantExpr(AffineConstantExpr expr) {
-    auto op = builder.create<arith::ConstantIndexOp>(loc, expr.getValue());
+    auto op = arith::ConstantIndexOp::create(builder, loc, expr.getValue());
     return op.getResult();
   }
 
@@ -298,7 +298,7 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   // block.
   IRMapping operandMap;
   OpBuilder b(hoistOverOp);
-  auto hoistedIfOp = b.create<AffineIfOp>(ifOp.getLoc(), ifOp.getIntegerSet(),
+  auto hoistedIfOp = AffineIfOp::create(b, ifOp.getLoc(), ifOp.getIntegerSet(),
                                           ifOp.getOperands(),
                                           /*elseBlock=*/true);
 
@@ -369,7 +369,7 @@ mlir::affine::affineParallelize(AffineForOp forOp,
       parallelReductions, [](const LoopReduction &red) { return red.value; }));
   auto reductionKinds = llvm::to_vector<4>(llvm::map_range(
       parallelReductions, [](const LoopReduction &red) { return red.kind; }));
-  AffineParallelOp newPloop = outsideBuilder.create<AffineParallelOp>(
+  AffineParallelOp newPloop = AffineParallelOp::create(outsideBuilder,
       loc, ValueRange(reducedValues).getTypes(), reductionKinds,
       llvm::ArrayRef(lowerBoundMap), lowerBoundOperands,
       llvm::ArrayRef(upperBoundMap), upperBoundOperands,
@@ -541,7 +541,7 @@ void mlir::affine::normalizeAffineParallel(AffineParallelOp op) {
     SmallVector<Value, 8> applyOperands{dimOperands};
     applyOperands.push_back(iv);
     applyOperands.append(symbolOperands.begin(), symbolOperands.end());
-    auto apply = builder.create<AffineApplyOp>(op.getLoc(), map, applyOperands);
+    auto apply = AffineApplyOp::create(builder, op.getLoc(), map, applyOperands);
     iv.replaceAllUsesExcept(apply, apply);
   }
 
@@ -622,7 +622,7 @@ LogicalResult mlir::affine::normalizeAffineFor(AffineForOp op,
   AffineValueMap newIvToOldIvMap;
   AffineValueMap::difference(lbMap, scaleIvValueMap, &newIvToOldIvMap);
   (void)newIvToOldIvMap.canonicalize();
-  auto newIV = opBuilder.create<AffineApplyOp>(
+  auto newIV = AffineApplyOp::create(opBuilder,
       loc, newIvToOldIvMap.getAffineMap(), newIvToOldIvMap.getOperands());
   op.getInductionVar().replaceAllUsesExcept(newIV->getResult(0), newIV);
   return success();
@@ -1187,7 +1187,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     for (auto resultExpr : oldMap.getResults()) {
       auto singleResMap = AffineMap::get(oldMap.getNumDims(),
                                          oldMap.getNumSymbols(), resultExpr);
-      auto afOp = builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+      auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                 oldMapOperands);
       oldMemRefOperands.push_back(afOp);
       affineApplyOps.push_back(afOp);
@@ -1214,7 +1214,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     for (auto resultExpr : indexRemap.getResults()) {
       auto singleResMap = AffineMap::get(
           indexRemap.getNumDims(), indexRemap.getNumSymbols(), resultExpr);
-      auto afOp = builder.create<AffineApplyOp>(op->getLoc(), singleResMap,
+      auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
                                                 remapOperands);
       remapOutputs.push_back(afOp);
       affineApplyOps.push_back(afOp);
@@ -1264,7 +1264,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     // AffineMapAccessInterface, we need to apply the values of `newMapOperands`
     // to the `newMap` to get the correct indices.
     for (unsigned i = 0; i < newMemRefRank; i++) {
-      state.operands.push_back(builder.create<AffineApplyOp>(
+      state.operands.push_back(AffineApplyOp::create(builder,
           op->getLoc(),
           AffineMap::get(newMap.getNumDims(), newMap.getNumSymbols(),
                          newMap.getResult(i)),
@@ -1450,7 +1450,7 @@ void mlir::affine::createAffineComputationSlice(
   for (auto resultExpr : composedMap.getResults()) {
     auto singleResMap = AffineMap::get(composedMap.getNumDims(),
                                        composedMap.getNumSymbols(), resultExpr);
-    sliceOps->push_back(builder.create<AffineApplyOp>(
+    sliceOps->push_back(AffineApplyOp::create(builder,
         opInst->getLoc(), singleResMap, composedOpOperands));
   }
 
@@ -1681,7 +1681,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
       // Create ConstantOp for static dimension.
       auto constantAttr = b.getIntegerAttr(b.getIndexType(), oldMemRefShape[d]);
       inAffineApply.emplace_back(
-          b.create<arith::ConstantOp>(allocOp.getLoc(), constantAttr));
+          arith::ConstantOp::create(b, allocOp.getLoc(), constantAttr));
     }
   }
 
@@ -1705,7 +1705,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
       AffineMap newMap =
           AffineMap::get(map.getNumInputs(), map.getNumSymbols(), newMapOutput);
       Value affineApp =
-          b.create<AffineApplyOp>(allocOp.getLoc(), newMap, inAffineApply);
+          AffineApplyOp::create(b, allocOp.getLoc(), newMap, inAffineApply);
       newDynamicSizes.emplace_back(affineApp);
     }
     newDimIdx++;
@@ -1741,10 +1741,10 @@ LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp allocOp) {
                           newDynamicSizes);
     // Add the new dynamic sizes in new AllocOp.
     newAlloc =
-        b.create<AllocLikeOp>(allocOp.getLoc(), newMemRefType, newDynamicSizes,
+        AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType, newDynamicSizes,
                               allocOp.getAlignmentAttr());
   } else {
-    newAlloc = b.create<AllocLikeOp>(allocOp.getLoc(), newMemRefType,
+    newAlloc = AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType,
                                      allocOp.getAlignmentAttr());
   }
   // Replace all uses of the old memref.
@@ -1803,10 +1803,10 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0, e = memrefType.getRank(); i < e; i++) {
     if (memrefType.isDynamicDim(i))
       mapOperands[i] =
-          b.create<arith::SubIOp>(loc, oldSizes[0].getType(), oldSizes[idx++],
-                                  b.create<arith::ConstantIndexOp>(loc, 1));
+          arith::SubIOp::create(b, loc, oldSizes[0].getType(), oldSizes[idx++],
+                                  arith::ConstantIndexOp::create(b, loc, 1));
     else
-      mapOperands[i] = b.create<arith::ConstantIndexOp>(loc, oldShape[i] - 1);
+      mapOperands[i] = arith::ConstantIndexOp::create(b, loc, oldShape[i] - 1);
   }
   for (unsigned i = 0, e = oldStrides.size(); i < e; i++)
     mapOperands[memrefType.getRank() + i] = oldStrides[i];
@@ -1816,7 +1816,7 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0; i < newRank; i++) {
     if (!newMemRefType.isDynamicDim(i))
       continue;
-    newSizes.push_back(b.create<AffineApplyOp>(
+    newSizes.push_back(AffineApplyOp::create(b,
         loc,
         AffineMap::get(oldLayoutMap.getNumDims(), oldLayoutMap.getNumSymbols(),
                        oldLayoutMap.getResult(i)),
@@ -1824,11 +1824,11 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   }
   for (unsigned i = 0, e = newSizes.size(); i < e; i++) {
     newSizes[i] =
-        b.create<arith::AddIOp>(loc, newSizes[i].getType(), newSizes[i],
-                                b.create<arith::ConstantIndexOp>(loc, 1));
+        arith::AddIOp::create(b, loc, newSizes[i].getType(), newSizes[i],
+                                arith::ConstantIndexOp::create(b, loc, 1));
   }
   // Create the new reinterpret_cast op.
-  auto newReinterpretCast = b.create<memref::ReinterpretCastOp>(
+  auto newReinterpretCast = memref::ReinterpretCastOp::create(b,
       loc, newMemRefType, reinterpretCastOp.getSource(),
       /*offsets=*/ValueRange(), newSizes,
       /*strides=*/ValueRange(),
diff --git a/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp b/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
index ebcb951cf3518..e7cbee6b06c45 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithDialect.cpp
@@ -64,7 +64,7 @@ Operation *arith::ArithDialect::materializeConstant(OpBuilder &builder,
                                                     Attribute value, Type type,
                                                     Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   return ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 4e40d4ebda004..60c1fc6adacff 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -242,7 +242,7 @@ bool arith::ConstantOp::isBuildableWith(Attribute value, Type type) {
 ConstantOp arith::ConstantOp::materialize(OpBuilder &builder, Attribute value,
                                           Type type, Location loc) {
   if (isBuildableWith(value, type))
-    return builder.create<arith::ConstantOp>(loc, cast<TypedAttr>(value));
+    return arith::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
   return nullptr;
 }
 
@@ -304,7 +304,7 @@ Value mlir::arith::getZeroConstant(OpBuilder &builder, Location loc,
          "type doesn't have a zero representation");
   TypedAttr zeroAttr = builder.getZeroAttr(type);
   assert(zeroAttr && "unsupported type for zero attribute");
-  return builder.create<arith::ConstantOp>(loc, zeroAttr);
+  return arith::ConstantOp::create(builder, loc, zeroAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2334,7 +2334,7 @@ class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
     // comparison.
     rewriter.replaceOpWithNewOp<CmpIOp>(
         op, pred, intVal,
-        rewriter.create<ConstantOp>(
+        ConstantOp::create(rewriter,
             op.getLoc(), intVal.getType(),
             rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
     return success();
@@ -2373,9 +2373,9 @@ struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
         matchPattern(op.getFalseValue(), m_One())) {
       rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
           op, op.getType(),
-          rewriter.create<arith::XOrIOp>(
+          arith::XOrIOp::create(rewriter,
               op.getLoc(), op.getCondition(),
-              rewriter.create<arith::ConstantIntOp>(
+              arith::ConstantIntOp::create(rewriter,
                   op.getLoc(), op.getCondition().getType(), 1)));
       return success();
     }
@@ -2692,7 +2692,7 @@ Value mlir::arith::getIdentityValue(AtomicRMWKind op, Type resultType,
                                     bool useOnlyFiniteValue) {
   auto attr =
       getIdentityValueAttr(op, resultType, builder, loc, useOnlyFiniteValue);
-  return builder.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(builder, loc, attr);
 }
 
 /// Return the value obtained by applying the reduction operation kind
@@ -2701,33 +2701,33 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
                                   Location loc, Value lhs, Value rhs) {
   switch (op) {
   case AtomicRMWKind::addf:
-    return builder.create<arith::AddFOp>(loc, lhs, rhs);
+    return arith::AddFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::addi:
-    return builder.create<arith::AddIOp>(loc, lhs, rhs);
+    return arith::AddIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::mulf:
-    return builder.create<arith::MulFOp>(loc, lhs, rhs);
+    return arith::MulFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::muli:
-    return builder.create<arith::MulIOp>(loc, lhs, rhs);
+    return arith::MulIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maximumf:
-    return builder.create<arith::MaximumFOp>(loc, lhs, rhs);
+    return arith::MaximumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minimumf:
-    return builder.create<arith::MinimumFOp>(loc, lhs, rhs);
+    return arith::MinimumFOp::create(builder, loc, lhs, rhs);
    case AtomicRMWKind::maxnumf:
-    return builder.create<arith::MaxNumFOp>(loc, lhs, rhs);
+    return arith::MaxNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minnumf:
-    return builder.create<arith::MinNumFOp>(loc, lhs, rhs);
+    return arith::MinNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maxs:
-    return builder.create<arith::MaxSIOp>(loc, lhs, rhs);
+    return arith::MaxSIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::mins:
-    return builder.create<arith::MinSIOp>(loc, lhs, rhs);
+    return arith::MinSIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::maxu:
-    return builder.create<arith::MaxUIOp>(loc, lhs, rhs);
+    return arith::MaxUIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minu:
-    return builder.create<arith::MinUIOp>(loc, lhs, rhs);
+    return arith::MinUIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::ori:
-    return builder.create<arith::OrIOp>(loc, lhs, rhs);
+    return arith::OrIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::andi:
-    return builder.create<arith::AndIOp>(loc, lhs, rhs);
+    return arith::AndIOp::create(builder, loc, lhs, rhs);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index f2e7732e8ea4a..d21da23903544 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -67,7 +67,7 @@ struct SelectOpInterface
       return state.getMemrefWithUniqueOwnership(builder, value,
                                                 value.getParentBlock());
 
-    Value ownership = builder.create<arith::SelectOp>(
+    Value ownership = arith::SelectOp::create(builder,
         op->getLoc(), selectOp.getCondition(),
         state.getOwnership(selectOp.getTrueValue(), block).getIndicator(),
         state.getOwnership(selectOp.getFalseValue(), block).getIndicator());
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
index afee162053bea..b073a31850678 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -170,10 +170,10 @@ struct SelectOpInterface
         return failure();
       if (trueBuffer.getType() != *targetType)
         trueBuffer =
-            rewriter.create<memref::CastOp>(loc, *targetType, trueBuffer);
+            memref::CastOp::create(rewriter, loc, *targetType, trueBuffer);
       if (falseBuffer.getType() != *targetType)
         falseBuffer =
-            rewriter.create<memref::CastOp>(loc, *targetType, falseBuffer);
+            memref::CastOp::create(rewriter, loc, *targetType, falseBuffer);
     }
 
     replaceOpWithNewBufferizedOp<arith::SelectOp>(
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
index 62022bfb7df1e..674bd7b8ee201 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
@@ -75,7 +75,7 @@ LogicalResult EmulateFloatPattern::matchAndRewrite(
   for (auto [res, oldType, newType] : llvm::zip_equal(
            MutableArrayRef{newResults}, op->getResultTypes(), resultTypes)) {
     if (oldType != newType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, oldType, res);
+      auto truncFOp = arith::TruncFOp::create(rewriter, loc, oldType, res);
       truncFOp.setFastmath(arith::FastMathFlags::contract);
       res = truncFOp.getResult();
     }
@@ -98,7 +98,7 @@ void mlir::arith::populateEmulateUnsupportedFloatsConversions(
   });
   converter.addTargetMaterialization(
       [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        auto extFOp = arith::ExtFOp::create(b, loc, target, input);
         extFOp.setFastmath(arith::FastMathFlags::contract);
         return extFOp;
       });
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
index d5d1559c658ff..44e896a97cb4b 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
@@ -72,7 +72,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
 
   // Scalarize the result in case of 1D vectors.
   if (shape.size() == 1)
-    return rewriter.create<vector::ExtractOp>(loc, input, lastOffset);
+    return vector::ExtractOp::create(rewriter, loc, input, lastOffset);
 
   SmallVector<int64_t> offsets(shape.size(), 0);
   offsets.back() = lastOffset;
@@ -80,7 +80,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
   sizes.back() = 1;
   SmallVector<int64_t> strides(shape.size(), 1);
 
-  return rewriter.create<vector::ExtractStridedSliceOp>(loc, input, offsets,
+  return vector::ExtractStridedSliceOp::create(rewriter, loc, input, offsets,
                                                         sizes, strides);
 }
 
@@ -107,7 +107,7 @@ static Value dropTrailingX1Dim(ConversionPatternRewriter &rewriter,
   assert(shape.back() == 1 && "Expected the last vector dim to be x1");
 
   auto newVecTy = VectorType::get(shape.drop_back(), vecTy.getElementType());
-  return rewriter.create<vector::ShapeCastOp>(loc, newVecTy, input);
+  return vector::ShapeCastOp::create(rewriter, loc, newVecTy, input);
 }
 
 /// Performs a vector shape cast to append an x1 dimension. If the
@@ -122,7 +122,7 @@ static Value appendX1Dim(ConversionPatternRewriter &rewriter, Location loc,
   auto newShape = llvm::to_vector(vecTy.getShape());
   newShape.push_back(1);
   auto newTy = VectorType::get(newShape, vecTy.getElementType());
-  return rewriter.create<vector::ShapeCastOp>(loc, newTy, input);
+  return vector::ShapeCastOp::create(rewriter, loc, newTy, input);
 }
 
 /// Inserts the `source` vector slice into the `dest` vector at offset
@@ -136,12 +136,12 @@ static Value insertLastDimSlice(ConversionPatternRewriter &rewriter,
 
   // Handle scalar source.
   if (isa<IntegerType>(source.getType()))
-    return rewriter.create<vector::InsertOp>(loc, source, dest, lastOffset);
+    return vector::InsertOp::create(rewriter, loc, source, dest, lastOffset);
 
   SmallVector<int64_t> offsets(shape.size(), 0);
   offsets.back() = lastOffset;
   SmallVector<int64_t> strides(shape.size(), 1);
-  return rewriter.create<vector::InsertStridedSliceOp>(loc, source, dest,
+  return vector::InsertStridedSliceOp::create(rewriter, loc, source, dest,
                                                        offsets, strides);
 }
 
@@ -254,12 +254,12 @@ struct ConvertAddI final : OpConversionPattern<arith::AddIOp> {
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
     auto lowSum =
-        rewriter.create<arith::AddUIExtendedOp>(loc, lhsElem0, rhsElem0);
+        arith::AddUIExtendedOp::create(rewriter, loc, lhsElem0, rhsElem0);
     Value overflowVal =
-        rewriter.create<arith::ExtUIOp>(loc, newElemTy, lowSum.getOverflow());
+        arith::ExtUIOp::create(rewriter, loc, newElemTy, lowSum.getOverflow());
 
-    Value high0 = rewriter.create<arith::AddIOp>(loc, overflowVal, lhsElem1);
-    Value high = rewriter.create<arith::AddIOp>(loc, high0, rhsElem1);
+    Value high0 = arith::AddIOp::create(rewriter, loc, overflowVal, lhsElem1);
+    Value high = arith::AddIOp::create(rewriter, loc, high0, rhsElem1);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {lowSum.getSum(), high});
@@ -293,8 +293,8 @@ struct ConvertBitwiseBinary final : OpConversionPattern<BinaryOp> {
     auto [rhsElem0, rhsElem1] =
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
-    Value resElem0 = rewriter.create<BinaryOp>(loc, lhsElem0, rhsElem0);
-    Value resElem1 = rewriter.create<BinaryOp>(loc, lhsElem1, rhsElem1);
+    Value resElem0 = BinaryOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem1 = BinaryOp::create(rewriter, loc, lhsElem1, rhsElem1);
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
     rewriter.replaceOp(op, resultVec);
@@ -346,26 +346,26 @@ struct ConvertCmpI final : OpConversionPattern<arith::CmpIOp> {
         extractLastDimHalves(rewriter, loc, adaptor.getRhs());
 
     Value lowCmp =
-        rewriter.create<arith::CmpIOp>(loc, lowPred, lhsElem0, rhsElem0);
+        arith::CmpIOp::create(rewriter, loc, lowPred, lhsElem0, rhsElem0);
     Value highCmp =
-        rewriter.create<arith::CmpIOp>(loc, highPred, lhsElem1, rhsElem1);
+        arith::CmpIOp::create(rewriter, loc, highPred, lhsElem1, rhsElem1);
 
     Value cmpResult{};
     switch (highPred) {
     case arith::CmpIPredicate::eq: {
-      cmpResult = rewriter.create<arith::AndIOp>(loc, lowCmp, highCmp);
+      cmpResult = arith::AndIOp::create(rewriter, loc, lowCmp, highCmp);
       break;
     }
     case arith::CmpIPredicate::ne: {
-      cmpResult = rewriter.create<arith::OrIOp>(loc, lowCmp, highCmp);
+      cmpResult = arith::OrIOp::create(rewriter, loc, lowCmp, highCmp);
       break;
     }
     default: {
       // Handle inequality checks.
-      Value highEq = rewriter.create<arith::CmpIOp>(
+      Value highEq = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, lhsElem1, rhsElem1);
       cmpResult =
-          rewriter.create<arith::SelectOp>(loc, highEq, lowCmp, highCmp);
+          arith::SelectOp::create(rewriter, loc, highEq, lowCmp, highCmp);
       break;
     }
     }
@@ -401,14 +401,14 @@ struct ConvertMulI final : OpConversionPattern<arith::MulIOp> {
     // Multiplying two i2N integers produces (at most) an i4N result, but
     // because the calculation of top i2N is not necessary, we omit it.
     auto mulLowLow =
-        rewriter.create<arith::MulUIExtendedOp>(loc, lhsElem0, rhsElem0);
-    Value mulLowHi = rewriter.create<arith::MulIOp>(loc, lhsElem0, rhsElem1);
-    Value mulHiLow = rewriter.create<arith::MulIOp>(loc, lhsElem1, rhsElem0);
+        arith::MulUIExtendedOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value mulLowHi = arith::MulIOp::create(rewriter, loc, lhsElem0, rhsElem1);
+    Value mulHiLow = arith::MulIOp::create(rewriter, loc, lhsElem1, rhsElem0);
 
     Value resLow = mulLowLow.getLow();
     Value resHi =
-        rewriter.create<arith::AddIOp>(loc, mulLowLow.getHigh(), mulLowHi);
-    resHi = rewriter.create<arith::AddIOp>(loc, resHi, mulHiLow);
+        arith::AddIOp::create(rewriter, loc, mulLowLow.getHigh(), mulLowHi);
+    resHi = arith::AddIOp::create(rewriter, loc, resHi, mulHiLow);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resLow, resHi});
@@ -443,10 +443,10 @@ struct ConvertExtSI final : OpConversionPattern<arith::ExtSIOp> {
         loc, newResultComponentTy, newOperand);
     Value operandZeroCst =
         createScalarOrSplatConstant(rewriter, loc, newResultComponentTy, 0);
-    Value signBit = rewriter.create<arith::CmpIOp>(
+    Value signBit = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, extended, operandZeroCst);
     Value signValue =
-        rewriter.create<arith::ExtSIOp>(loc, newResultComponentTy, signBit);
+        arith::ExtSIOp::create(rewriter, loc, newResultComponentTy, signBit);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {extended, signValue});
@@ -508,7 +508,7 @@ struct ConvertMaxMin final : OpConversionPattern<SourceOp> {
     // Rewrite Max*I/Min*I as compare and select over original operands. Let
     // the CmpI and Select emulation patterns handle the final legalization.
     Value cmp =
-        rewriter.create<arith::CmpIOp>(loc, CmpPred, op.getLhs(), op.getRhs());
+        arith::CmpIOp::create(rewriter, loc, CmpPred, op.getLhs(), op.getRhs());
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, op.getLhs(),
                                                  op.getRhs());
     return success();
@@ -587,7 +587,7 @@ struct ConvertIndexCastIndexToInt final : OpConversionPattern<CastOp> {
     // Sign or zero-extend the result. Let the matching conversion pattern
     // legalize the extension op.
     Value underlyingVal =
-        rewriter.create<CastOp>(loc, narrowTy, adaptor.getIn());
+        CastOp::create(rewriter, loc, narrowTy, adaptor.getIn());
     rewriter.replaceOpWithNewOp<ExtensionOp>(op, resultType, underlyingVal);
     return success();
   }
@@ -616,9 +616,9 @@ struct ConvertSelect final : OpConversionPattern<arith::SelectOp> {
     Value cond = appendX1Dim(rewriter, loc, adaptor.getCondition());
 
     Value resElem0 =
-        rewriter.create<arith::SelectOp>(loc, cond, trueElem0, falseElem0);
+        arith::SelectOp::create(rewriter, loc, cond, trueElem0, falseElem0);
     Value resElem1 =
-        rewriter.create<arith::SelectOp>(loc, cond, trueElem1, falseElem1);
+        arith::SelectOp::create(rewriter, loc, cond, trueElem1, falseElem1);
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
     rewriter.replaceOp(op, resultVec);
@@ -680,33 +680,33 @@ struct ConvertShLI final : OpConversionPattern<arith::ShLIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = rewriter.create<arith::CmpIOp>(
+    Value illegalElemShift = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem0, rhsElem0);
-    Value resElem0 = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShLIOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem0 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                       zeroCst, shiftedElem0);
 
-    Value cappedShiftAmount = rewriter.create<arith::SelectOp>(
+    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value rightShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, elemBitWidth, cappedShiftAmount);
+        arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedRight =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem0, rightShiftAmount);
+        arith::ShRUIOp::create(rewriter, loc, lhsElem0, rightShiftAmount);
     Value overshotShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, rhsElem0, elemBitWidth);
+        arith::SubIOp::create(rewriter, loc, rhsElem0, elemBitWidth);
     Value shiftedLeft =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem0, overshotShiftAmount);
+        arith::ShLIOp::create(rewriter, loc, lhsElem0, overshotShiftAmount);
 
     Value shiftedElem1 =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem1, rhsElem0);
-    Value resElem1High = rewriter.create<arith::SelectOp>(
+        arith::ShLIOp::create(rewriter, loc, lhsElem1, rhsElem0);
+    Value resElem1High = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, zeroCst, shiftedElem1);
-    Value resElem1Low = rewriter.create<arith::SelectOp>(
+    Value resElem1Low = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, shiftedLeft, shiftedRight);
     Value resElem1 =
-        rewriter.create<arith::OrIOp>(loc, resElem1Low, resElem1High);
+        arith::OrIOp::create(rewriter, loc, resElem1Low, resElem1High);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
@@ -769,33 +769,33 @@ struct ConvertShRUI final : OpConversionPattern<arith::ShRUIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = rewriter.create<arith::CmpIOp>(
+    Value illegalElemShift = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem0, rhsElem0);
-    Value resElem0Low = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShRUIOp::create(rewriter, loc, lhsElem0, rhsElem0);
+    Value resElem0Low = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                          zeroCst, shiftedElem0);
     Value shiftedElem1 =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem1, rhsElem0);
-    Value resElem1 = rewriter.create<arith::SelectOp>(loc, illegalElemShift,
+        arith::ShRUIOp::create(rewriter, loc, lhsElem1, rhsElem0);
+    Value resElem1 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
                                                       zeroCst, shiftedElem1);
 
-    Value cappedShiftAmount = rewriter.create<arith::SelectOp>(
+    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value leftShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, elemBitWidth, cappedShiftAmount);
+        arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedLeft =
-        rewriter.create<arith::ShLIOp>(loc, lhsElem1, leftShiftAmount);
+        arith::ShLIOp::create(rewriter, loc, lhsElem1, leftShiftAmount);
     Value overshotShiftAmount =
-        rewriter.create<arith::SubIOp>(loc, rhsElem0, elemBitWidth);
+        arith::SubIOp::create(rewriter, loc, rhsElem0, elemBitWidth);
     Value shiftedRight =
-        rewriter.create<arith::ShRUIOp>(loc, lhsElem1, overshotShiftAmount);
+        arith::ShRUIOp::create(rewriter, loc, lhsElem1, overshotShiftAmount);
 
-    Value resElem0High = rewriter.create<arith::SelectOp>(
+    Value resElem0High = arith::SelectOp::create(rewriter,
         loc, illegalElemShift, shiftedRight, shiftedLeft);
     Value resElem0 =
-        rewriter.create<arith::OrIOp>(loc, resElem0Low, resElem0High);
+        arith::OrIOp::create(rewriter, loc, resElem0Low, resElem0High);
 
     Value resultVec =
         constructResultVector(rewriter, loc, newTy, {resElem0, resElem1});
@@ -832,32 +832,32 @@ struct ConvertShRSI final : OpConversionPattern<arith::ShRSIOp> {
     // Perform as many ops over the narrow integer type as possible and let the
     // other emulation patterns convert the rest.
     Value elemZero = createScalarOrSplatConstant(rewriter, loc, narrowTy, 0);
-    Value signBit = rewriter.create<arith::CmpIOp>(
+    Value signBit = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, lhsElem1, elemZero);
     signBit = dropTrailingX1Dim(rewriter, loc, signBit);
 
     // Create a bit pattern of either all ones or all zeros. Then shift it left
     // to calculate the sign extension bits created by shifting the original
     // sign bit right.
-    Value allSign = rewriter.create<arith::ExtSIOp>(loc, oldTy, signBit);
+    Value allSign = arith::ExtSIOp::create(rewriter, loc, oldTy, signBit);
     Value maxShift =
         createScalarOrSplatConstant(rewriter, loc, narrowTy, origBitwidth);
     Value numNonSignExtBits =
-        rewriter.create<arith::SubIOp>(loc, maxShift, rhsElem0);
+        arith::SubIOp::create(rewriter, loc, maxShift, rhsElem0);
     numNonSignExtBits = dropTrailingX1Dim(rewriter, loc, numNonSignExtBits);
     numNonSignExtBits =
-        rewriter.create<arith::ExtUIOp>(loc, oldTy, numNonSignExtBits);
+        arith::ExtUIOp::create(rewriter, loc, oldTy, numNonSignExtBits);
     Value signBits =
-        rewriter.create<arith::ShLIOp>(loc, allSign, numNonSignExtBits);
+        arith::ShLIOp::create(rewriter, loc, allSign, numNonSignExtBits);
 
     // Use original arguments to create the right shift.
     Value shrui =
-        rewriter.create<arith::ShRUIOp>(loc, op.getLhs(), op.getRhs());
-    Value shrsi = rewriter.create<arith::OrIOp>(loc, shrui, signBits);
+        arith::ShRUIOp::create(rewriter, loc, op.getLhs(), op.getRhs());
+    Value shrsi = arith::OrIOp::create(rewriter, loc, shrui, signBits);
 
     // Handle shifting by zero. This is necessary when the `signBits` shift is
     // invalid.
-    Value isNoop = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value isNoop = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                   rhsElem0, elemZero);
     isNoop = dropTrailingX1Dim(rewriter, loc, isNoop);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNoop, op.getLhs(),
@@ -892,14 +892,14 @@ struct ConvertSubI final : OpConversionPattern<arith::SubIOp> {
 
     // Emulates LHS - RHS by [LHS0 - RHS0, LHS1 - RHS1 - CARRY] where
     // CARRY is 1 or 0.
-    Value low = rewriter.create<arith::SubIOp>(loc, lhsElem0, rhsElem0);
+    Value low = arith::SubIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     // We have a carry if lhsElem0 < rhsElem0.
-    Value carry0 = rewriter.create<arith::CmpIOp>(
+    Value carry0 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ult, lhsElem0, rhsElem0);
-    Value carryVal = rewriter.create<arith::ExtUIOp>(loc, newElemTy, carry0);
+    Value carryVal = arith::ExtUIOp::create(rewriter, loc, newElemTy, carry0);
 
-    Value high0 = rewriter.create<arith::SubIOp>(loc, lhsElem1, carryVal);
-    Value high = rewriter.create<arith::SubIOp>(loc, high0, rhsElem1);
+    Value high0 = arith::SubIOp::create(rewriter, loc, lhsElem1, carryVal);
+    Value high = arith::SubIOp::create(rewriter, loc, high0, rhsElem1);
 
     Value resultVec = constructResultVector(rewriter, loc, newTy, {low, high});
     rewriter.replaceOp(op, resultVec);
@@ -933,13 +933,13 @@ struct ConvertSIToFP final : OpConversionPattern<arith::SIToFPOp> {
     // result or not based on that sign bit. We implement negation by
     // subtracting from zero. Note that this relies on the the other conversion
     // patterns to legalize created ops and narrow the bit widths.
-    Value isNeg = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+    Value isNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                  in, zeroCst);
-    Value neg = rewriter.create<arith::SubIOp>(loc, zeroCst, in);
-    Value abs = rewriter.create<arith::SelectOp>(loc, isNeg, neg, in);
+    Value neg = arith::SubIOp::create(rewriter, loc, zeroCst, in);
+    Value abs = arith::SelectOp::create(rewriter, loc, isNeg, neg, in);
 
-    Value absResult = rewriter.create<arith::UIToFPOp>(loc, op.getType(), abs);
-    Value negResult = rewriter.create<arith::NegFOp>(loc, absResult);
+    Value absResult = arith::UIToFPOp::create(rewriter, loc, op.getType(), abs);
+    Value negResult = arith::NegFOp::create(rewriter, loc, absResult);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNeg, negResult,
                                                  absResult);
     return success();
@@ -985,13 +985,13 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     //
     // Note 2: We do not strictly need the `hi == 0`, case, but it makes
     // constant folding easier.
-    Value hiEqZero = rewriter.create<arith::CmpIOp>(
+    Value hiEqZero = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, hiInt, zeroCst);
 
     Type resultTy = op.getType();
     Type resultElemTy = getElementTypeOrSelf(resultTy);
-    Value lowFp = rewriter.create<arith::UIToFPOp>(loc, resultTy, lowInt);
-    Value hiFp = rewriter.create<arith::UIToFPOp>(loc, resultTy, hiInt);
+    Value lowFp = arith::UIToFPOp::create(rewriter, loc, resultTy, lowInt);
+    Value hiFp = arith::UIToFPOp::create(rewriter, loc, resultTy, hiInt);
 
     int64_t pow2Int = int64_t(1) << newBitWidth;
     TypedAttr pow2Attr =
@@ -999,10 +999,10 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     if (auto vecTy = dyn_cast<VectorType>(resultTy))
       pow2Attr = SplatElementsAttr::get(vecTy, pow2Attr);
 
-    Value pow2Val = rewriter.create<arith::ConstantOp>(loc, resultTy, pow2Attr);
+    Value pow2Val = arith::ConstantOp::create(rewriter, loc, resultTy, pow2Attr);
 
-    Value hiVal = rewriter.create<arith::MulFOp>(loc, hiFp, pow2Val);
-    Value result = rewriter.create<arith::AddFOp>(loc, lowFp, hiVal);
+    Value hiVal = arith::MulFOp::create(rewriter, loc, hiFp, pow2Val);
+    Value result = arith::AddFOp::create(rewriter, loc, lowFp, hiVal);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, hiEqZero, lowFp, result);
     return success();
@@ -1037,22 +1037,22 @@ struct ConvertFPToSI final : OpConversionPattern<arith::FPToSIOp> {
     // result is UB.
 
     TypedAttr zeroAttr = rewriter.getZeroAttr(fpTy);
-    Value zeroCst = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
+    Value zeroCst = arith::ConstantOp::create(rewriter, loc, zeroAttr);
     Value zeroCstInt = createScalarOrSplatConstant(rewriter, loc, intTy, 0);
 
     // Get the absolute value. One could have used math.absf here, but that
     // introduces an extra dependency.
-    Value isNeg = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OLT,
+    Value isNeg = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OLT,
                                                  inFp, zeroCst);
-    Value negInFp = rewriter.create<arith::NegFOp>(loc, inFp);
+    Value negInFp = arith::NegFOp::create(rewriter, loc, inFp);
 
-    Value absVal = rewriter.create<arith::SelectOp>(loc, isNeg, negInFp, inFp);
+    Value absVal = arith::SelectOp::create(rewriter, loc, isNeg, negInFp, inFp);
 
     // Defer the absolute value to fptoui.
-    Value res = rewriter.create<arith::FPToUIOp>(loc, intTy, absVal);
+    Value res = arith::FPToUIOp::create(rewriter, loc, intTy, absVal);
 
     // Negate the value if < 0 .
-    Value neg = rewriter.create<arith::SubIOp>(loc, zeroCstInt, res);
+    Value neg = arith::SubIOp::create(rewriter, loc, zeroCstInt, res);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNeg, neg, res);
     return success();
@@ -1109,17 +1109,17 @@ struct ConvertFPToUI final : OpConversionPattern<arith::FPToUIOp> {
     if (auto vecType = dyn_cast<VectorType>(fpTy))
       powBitwidthAttr = SplatElementsAttr::get(vecType, powBitwidthAttr);
     Value powBitwidthFloatCst =
-        rewriter.create<arith::ConstantOp>(loc, powBitwidthAttr);
+        arith::ConstantOp::create(rewriter, loc, powBitwidthAttr);
 
     Value fpDivPowBitwidth =
-        rewriter.create<arith::DivFOp>(loc, inFp, powBitwidthFloatCst);
+        arith::DivFOp::create(rewriter, loc, inFp, powBitwidthFloatCst);
     Value resHigh =
-        rewriter.create<arith::FPToUIOp>(loc, newHalfType, fpDivPowBitwidth);
+        arith::FPToUIOp::create(rewriter, loc, newHalfType, fpDivPowBitwidth);
     // Calculate fp - resHigh * 2^N by getting the remainder of the division
     Value remainder =
-        rewriter.create<arith::RemFOp>(loc, inFp, powBitwidthFloatCst);
+        arith::RemFOp::create(rewriter, loc, inFp, powBitwidthFloatCst);
     Value resLow =
-        rewriter.create<arith::FPToUIOp>(loc, newHalfType, remainder);
+        arith::FPToUIOp::create(rewriter, loc, newHalfType, remainder);
 
     Value high = appendX1Dim(rewriter, loc, resHigh);
     Value low = appendX1Dim(rewriter, loc, resLow);
diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
index f497d2db3bf7c..dcdb0d1687504 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
@@ -29,10 +29,10 @@ static Value createConst(Location loc, Type type, int value,
                          PatternRewriter &rewriter) {
   auto attr = rewriter.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return rewriter.create<arith::ConstantOp>(
+    return arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
-  return rewriter.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(rewriter, loc, attr);
 }
 
 /// Create a float constant.
@@ -40,11 +40,11 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                               PatternRewriter &rewriter) {
   auto attr = rewriter.getFloatAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return rewriter.create<arith::ConstantOp>(
+    return arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return rewriter.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(rewriter, loc, attr);
 }
 
 /// Creates shapedType using shape from cloneFrom and base type from cloneTo
@@ -68,11 +68,11 @@ struct CeilDivUIOpConverter : public OpRewritePattern<arith::CeilDivUIOp> {
     Value b = op.getRhs();
     Value zero = createConst(loc, a.getType(), 0, rewriter);
     Value compare =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, a, zero);
     Value one = createConst(loc, a.getType(), 1, rewriter);
-    Value minusOne = rewriter.create<arith::SubIOp>(loc, a, one);
-    Value quotient = rewriter.create<arith::DivUIOp>(loc, minusOne, b);
-    Value plusOne = rewriter.create<arith::AddIOp>(loc, quotient, one);
+    Value minusOne = arith::SubIOp::create(rewriter, loc, a, one);
+    Value quotient = arith::DivUIOp::create(rewriter, loc, minusOne, b);
+    Value plusOne = arith::AddIOp::create(rewriter, loc, quotient, one);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, compare, zero, plusOne);
     return success();
   }
@@ -97,22 +97,22 @@ struct CeilDivSIOpConverter : public OpRewritePattern<arith::CeilDivSIOp> {
     Value zero = createConst(loc, type, 0, rewriter);
     Value one = createConst(loc, type, 1, rewriter);
 
-    Value quotient = rewriter.create<arith::DivSIOp>(loc, a, b);
-    Value product = rewriter.create<arith::MulIOp>(loc, quotient, b);
-    Value notEqualDivisor = rewriter.create<arith::CmpIOp>(
+    Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
+    Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
+    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, a, product);
 
     Value aNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
     Value bNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, b, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
 
-    Value signEqual = rewriter.create<arith::CmpIOp>(
+    Value signEqual = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, aNeg, bNeg);
     Value cond =
-        rewriter.create<arith::AndIOp>(loc, notEqualDivisor, signEqual);
+        arith::AndIOp::create(rewriter, loc, notEqualDivisor, signEqual);
 
-    Value quotientPlusOne = rewriter.create<arith::AddIOp>(loc, quotient, one);
+    Value quotientPlusOne = arith::AddIOp::create(rewriter, loc, quotient, one);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cond, quotientPlusOne,
                                                  quotient);
@@ -136,25 +136,25 @@ struct FloorDivSIOpConverter : public OpRewritePattern<arith::FloorDivSIOp> {
     Value a = op.getLhs();
     Value b = op.getRhs();
 
-    Value quotient = rewriter.create<arith::DivSIOp>(loc, a, b);
-    Value product = rewriter.create<arith::MulIOp>(loc, quotient, b);
-    Value notEqualDivisor = rewriter.create<arith::CmpIOp>(
+    Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
+    Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
+    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, a, product);
     Value zero = createConst(loc, type, 0, rewriter);
 
     Value aNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, a, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
     Value bNeg =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, b, zero);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
 
-    Value signOpposite = rewriter.create<arith::CmpIOp>(
+    Value signOpposite = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::ne, aNeg, bNeg);
     Value cond =
-        rewriter.create<arith::AndIOp>(loc, notEqualDivisor, signOpposite);
+        arith::AndIOp::create(rewriter, loc, notEqualDivisor, signOpposite);
 
     Value minusOne = createConst(loc, type, -1, rewriter);
     Value quotientMinusOne =
-        rewriter.create<arith::AddIOp>(loc, quotient, minusOne);
+        arith::AddIOp::create(rewriter, loc, quotient, minusOne);
 
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cond, quotientMinusOne,
                                                  quotient);
@@ -172,7 +172,7 @@ struct MaxMinIOpConverter : public OpRewritePattern<OpTy> {
     Value lhs = op.getLhs();
     Value rhs = op.getRhs();
 
-    Value cmp = rewriter.create<arith::CmpIOp>(op.getLoc(), pred, lhs, rhs);
+    Value cmp = arith::CmpIOp::create(rewriter, op.getLoc(), pred, lhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, lhs, rhs);
     return success();
   }
@@ -193,11 +193,11 @@ struct MaximumMinimumFOpConverter : public OpRewritePattern<OpTy> {
     static_assert(pred == arith::CmpFPredicate::UGT ||
                       pred == arith::CmpFPredicate::ULT,
                   "pred must be either UGT or ULT");
-    Value cmp = rewriter.create<arith::CmpFOp>(loc, pred, lhs, rhs);
-    Value select = rewriter.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+    Value cmp = arith::CmpFOp::create(rewriter, loc, pred, lhs, rhs);
+    Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'.
-    Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
+    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
                                                  rhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
@@ -219,11 +219,11 @@ struct MaxNumMinNumFOpConverter : public OpRewritePattern<OpTy> {
     static_assert(pred == arith::CmpFPredicate::UGT ||
                       pred == arith::CmpFPredicate::ULT,
                   "pred must be either UGT or ULT");
-    Value cmp = rewriter.create<arith::CmpFOp>(loc, pred, lhs, rhs);
-    Value select = rewriter.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+    Value cmp = arith::CmpFOp::create(rewriter, loc, pred, lhs, rhs);
+    Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where lhs is NaN: 'isNaN(lhs) ? rhs : select'.
-    Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
+    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
                                                  lhs, lhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
@@ -248,12 +248,12 @@ struct BFloat16ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type i16Ty = cloneToShapedType(operandTy, b.getI16Type());
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
 
-    Value bitcast = b.create<arith::BitcastOp>(i16Ty, operand);
-    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
+    Value bitcast = arith::BitcastOp::create(b, i16Ty, operand);
+    Value exti = arith::ExtUIOp::create(b, i32Ty, bitcast);
 
     Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
-    Value shl = b.create<arith::ShLIOp>(exti, c16);
-    Value result = b.create<arith::BitcastOp>(resultTy, shl);
+    Value shl = arith::ShLIOp::create(b, exti, c16);
+    Value result = arith::BitcastOp::create(b, resultTy, shl);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -297,7 +297,7 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     // exponent bits, that simple truncation is the desired outcome for
     // infinities.
     Value isNan =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNE, operand, operand);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNE, operand, operand);
     // Constant used to make the rounding bias.
     Value c7FFF = createConst(op.getLoc(), i32Ty, 0x7fff, rewriter);
     // Constant used to generate a quiet NaN.
@@ -306,30 +306,30 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
     Value c1 = createConst(op.getLoc(), i32Ty, 1, rewriter);
     // Reinterpret the input f32 value as bits.
-    Value bitcast = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value bitcast = arith::BitcastOp::create(b, i32Ty, operand);
     // Read bit 16 as a value in {0,1}.
     Value bit16 =
-        b.create<arith::AndIOp>(b.create<arith::ShRUIOp>(bitcast, c16), c1);
+        arith::AndIOp::create(b, arith::ShRUIOp::create(b, bitcast, c16), c1);
     // Determine the rounding bias to add as either 0x7fff or 0x8000 depending
     // on bit 16, implementing the tie-breaking "to nearest even".
-    Value roundingBias = b.create<arith::AddIOp>(bit16, c7FFF);
+    Value roundingBias = arith::AddIOp::create(b, bit16, c7FFF);
     // Add the rounding bias. Generally we want this to be added to the
     // mantissa, but nothing prevents this to from carrying into the exponent
     // bits, which would feel like a bug, but this is the magic trick here:
     // when that happens, the mantissa gets reset to zero and the exponent
     // gets incremented by the carry... which is actually exactly what we
     // want.
-    Value biased = b.create<arith::AddIOp>(bitcast, roundingBias);
+    Value biased = arith::AddIOp::create(b, bitcast, roundingBias);
     // Now that the rounding-bias has been added, truncating the low bits
     // yields the correctly rounded result.
-    Value biasedAndShifted = b.create<arith::ShRUIOp>(biased, c16);
+    Value biasedAndShifted = arith::ShRUIOp::create(b, biased, c16);
     Value normalCaseResultI16 =
-        b.create<arith::TruncIOp>(i16Ty, biasedAndShifted);
+        arith::TruncIOp::create(b, i16Ty, biasedAndShifted);
     // Select either the above-computed result, or a quiet NaN constant
     // if the input was NaN.
     Value select =
-        b.create<arith::SelectOp>(isNan, c7FC0I16, normalCaseResultI16);
-    Value result = b.create<arith::BitcastOp>(resultTy, select);
+        arith::SelectOp::create(b, isNan, c7FC0I16, normalCaseResultI16);
+    Value result = arith::BitcastOp::create(b, resultTy, select);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -382,7 +382,7 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
     Type i4Ty = cloneToShapedType(operandTy, b.getI4Type());
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
-    Value i4Bits = b.create<arith::BitcastOp>(i4Ty, operand);
+    Value i4Bits = arith::BitcastOp::create(b, i4Ty, operand);
 
     Value c0x0 = createConst(loc, i4Ty, 0x0, rewriter);
     Value c0x1 = createConst(loc, i4Ty, 0x1, rewriter);
@@ -391,38 +391,38 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
 
     // Set last Exponent bit and Mantissa.
     Value c0x00000014 = createConst(loc, i32Ty, 0x14, rewriter);
-    Value bits1To24 = b.create<arith::ShLIOp>(i4Bits, c0x2);
+    Value bits1To24 = arith::ShLIOp::create(b, i4Bits, c0x2);
     Value isHalf =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x1);
-    bits1To24 = b.create<arith::SelectOp>(isHalf, c0x0, bits1To24);
-    bits1To24 = b.create<arith::ExtUIOp>(i32Ty, bits1To24);
-    bits1To24 = b.create<arith::ShLIOp>(bits1To24, c0x00000014);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, i4Bits, c0x1);
+    bits1To24 = arith::SelectOp::create(b, isHalf, c0x0, bits1To24);
+    bits1To24 = arith::ExtUIOp::create(b, i32Ty, bits1To24);
+    bits1To24 = arith::ShLIOp::create(b, bits1To24, c0x00000014);
 
     // Set first 7 bits of Exponent.
     Value zeroExpBits = createConst(loc, i32Ty, 0x00000000, rewriter);
     Value highExpBits = createConst(loc, i32Ty, 0x40000000, rewriter);
     Value lowExpBits = createConst(loc, i32Ty, 0x3f000000, rewriter);
     Value useLargerExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x4);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x4);
     Value bits25To31 =
-        b.create<arith::SelectOp>(useLargerExp, highExpBits, lowExpBits);
+        arith::SelectOp::create(b, useLargerExp, highExpBits, lowExpBits);
     Value zeroExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x0);
-    bits25To31 = b.create<arith::SelectOp>(zeroExp, zeroExpBits, bits25To31);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, i4Bits, c0x0);
+    bits25To31 = arith::SelectOp::create(b, zeroExp, zeroExpBits, bits25To31);
 
     // Set sign.
     Value c0x80000000 = createConst(loc, i32Ty, 0x80000000, rewriter);
     Value c0x8 = createConst(loc, i4Ty, 0x8, rewriter);
     Value negative =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x8);
-    Value bit32 = b.create<arith::SelectOp>(negative, c0x80000000, zeroExpBits);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x8);
+    Value bit32 = arith::SelectOp::create(b, negative, c0x80000000, zeroExpBits);
 
     // Add segments together.
-    Value bits1To31 = b.create<arith::AddIOp>(bits1To24, bits25To31);
-    Value bits1To32 = b.create<arith::AddIOp>(bits1To31, bit32);
-    Value result = b.create<arith::BitcastOp>(f32Ty, bits1To32);
+    Value bits1To31 = arith::AddIOp::create(b, bits1To24, bits25To31);
+    Value bits1To32 = arith::AddIOp::create(b, bits1To31, bit32);
+    Value result = arith::BitcastOp::create(b, f32Ty, bits1To32);
     if (!isa<Float32Type>(resultETy))
-      result = b.create<arith::TruncFOp>(resultTy, result);
+      result = arith::TruncFOp::create(b, resultTy, result);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -448,25 +448,25 @@ struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
-    Value bitcast = b.create<arith::BitcastOp>(i8Ty, operand);
+    Value bitcast = arith::BitcastOp::create(b, i8Ty, operand);
     // create constants for NaNs
     Value cF8NaN = createConst(op.getLoc(), i8Ty, 0xff, rewriter);
     Value cF32NaN = createConst(op.getLoc(), i32Ty, 0xffffffff, rewriter);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
 
-    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
-    Value f32Bits = b.create<arith::ShLIOp>(exti, cF32MantissaWidth);
+    Value exti = arith::ExtUIOp::create(b, i32Ty, bitcast);
+    Value f32Bits = arith::ShLIOp::create(b, exti, cF32MantissaWidth);
 
     Value isNan =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, bitcast, cF8NaN);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, bitcast, cF8NaN);
     // select for NaNs
-    f32Bits = b.create<arith::SelectOp>(isNan, cF32NaN, f32Bits);
-    Value result = b.create<arith::BitcastOp>(f32Ty, f32Bits);
+    f32Bits = arith::SelectOp::create(b, isNan, cF32NaN, f32Bits);
+    Value result = arith::BitcastOp::create(b, f32Ty, f32Bits);
     if (resultETy.getIntOrFloatBitWidth() < 32) {
-      result = b.create<arith::TruncFOp>(resultTy, result, nullptr,
+      result = arith::TruncFOp::create(b, resultTy, result, nullptr,
                                          op.getFastmathAttr());
     } else if (resultETy.getIntOrFloatBitWidth() > 32) {
-      result = b.create<arith::ExtFOp>(resultTy, result, op.getFastmathAttr());
+      result = arith::ExtFOp::create(b, resultTy, result, op.getFastmathAttr());
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -519,7 +519,7 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (!isa<Float32Type>(operandETy))
-      operand = b.create<arith::ExtFOp>(f32Ty, operand);
+      operand = arith::ExtFOp::create(b, f32Ty, operand);
     if (!isa<Float4E2M1FNType>(resultETy))
       return rewriter.notifyMatchFailure(op, "not a trunc of F4E2M1FN");
 
@@ -533,65 +533,65 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     // Step 0: Clamp to bounds.
     Value cHigherBound = createFloatConst(loc, f32Ty, APFloat(6.0f), rewriter);
     Value cLowerBound = createFloatConst(loc, f32Ty, APFloat(-6.0f), rewriter);
-    Value operandClamped = b.create<arith::MinNumFOp>(cHigherBound, operand);
-    operandClamped = b.create<arith::MaxNumFOp>(cLowerBound, operandClamped);
-    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operandClamped);
+    Value operandClamped = arith::MinNumFOp::create(b, cHigherBound, operand);
+    operandClamped = arith::MaxNumFOp::create(b, cLowerBound, operandClamped);
+    Value f32Bits = arith::BitcastOp::create(b, i32Ty, operandClamped);
 
     // Step 1: Set sign bit.
     Value cF32ExpManWidth = createConst(loc, i32Ty, 31, rewriter); // 23
-    Value f32Sign = b.create<arith::ShRUIOp>(f32Bits, cF32ExpManWidth);
-    Value f4Sign = b.create<arith::TruncIOp>(i4Ty, f32Sign);
-    Value f4Bits = b.create<arith::ShLIOp>(f4Sign, c0x3);
+    Value f32Sign = arith::ShRUIOp::create(b, f32Bits, cF32ExpManWidth);
+    Value f4Sign = arith::TruncIOp::create(b, i4Ty, f32Sign);
+    Value f4Bits = arith::ShLIOp::create(b, f4Sign, c0x3);
 
     // Step 2: Convert exponent by adjusting bias.
     Value biasAdjustment = createConst(loc, i32Ty, 0x7e, rewriter);
     Value cF4MantissaWidth = c0x1;                                   // 1
     Value cF32MantissaWidth = createConst(loc, i32Ty, 23, rewriter); // 23
-    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
+    Value f32SignExp = arith::ShRUIOp::create(b, f32Bits, cF32MantissaWidth);
     Value biasAdjustedSignExp =
-        b.create<arith::SubIOp>(f32SignExp, biasAdjustment);
-    Value f4Exp = b.create<arith::TruncIOp>(i4Ty, biasAdjustedSignExp);
-    f4Exp = b.create<arith::ShLIOp>(f4Exp, cF4MantissaWidth);
-    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Exp);
+        arith::SubIOp::create(b, f32SignExp, biasAdjustment);
+    Value f4Exp = arith::TruncIOp::create(b, i4Ty, biasAdjustedSignExp);
+    f4Exp = arith::ShLIOp::create(b, f4Exp, cF4MantissaWidth);
+    f4Bits = arith::AddIOp::create(b, f4Bits, f4Exp);
 
     // Step 3: Set mantissa to first bit.
     Value cF32FirstBitMask = createConst(loc, i32Ty, 0x400000, rewriter);
-    Value man1Bit = b.create<arith::AndIOp>(f32Bits, cF32FirstBitMask);
-    man1Bit = b.create<arith::ShRUIOp>(man1Bit, c0x00000016);
-    Value f4Man = b.create<arith::TruncIOp>(i4Ty, man1Bit);
-    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Man);
+    Value man1Bit = arith::AndIOp::create(b, f32Bits, cF32FirstBitMask);
+    man1Bit = arith::ShRUIOp::create(b, man1Bit, c0x00000016);
+    Value f4Man = arith::TruncIOp::create(b, i4Ty, man1Bit);
+    f4Bits = arith::AddIOp::create(b, f4Bits, f4Man);
 
     // Step 4: Special consideration for conversion to 0.5.
     Value cF32MantissaMask = createConst(loc, i32Ty, 0x7fffff, rewriter);
-    Value f8Exp = b.create<arith::TruncIOp>(i8Ty, biasAdjustedSignExp);
+    Value f8Exp = arith::TruncIOp::create(b, i8Ty, biasAdjustedSignExp);
     Value isSubnormal =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::sle, f8Exp, c0x00);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::sle, f8Exp, c0x00);
     Value isNegOneExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0xff);
-    Value man23Bits = b.create<arith::AndIOp>(f32Bits, cF32MantissaMask);
-    Value isNonZeroMan = b.create<arith::CmpIOp>(arith::CmpIPredicate::ugt,
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0xff);
+    Value man23Bits = arith::AndIOp::create(b, f32Bits, cF32MantissaMask);
+    Value isNonZeroMan = arith::CmpIOp::create(b, arith::CmpIPredicate::ugt,
                                                  man23Bits, zeroExpBits);
-    Value roundToHalf = b.create<arith::AndIOp>(isNegOneExp, isNonZeroMan);
+    Value roundToHalf = arith::AndIOp::create(b, isNegOneExp, isNonZeroMan);
     Value isZeroExp =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0x00);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0x00);
     Value subnormalF4Bits = createConst(loc, i4Ty, 0xf, rewriter);
     Value halfF4Bits = createConst(loc, i4Ty, 0x0, rewriter);
     Value subResult =
-        b.create<arith::SelectOp>(isSubnormal, subnormalF4Bits, f4Bits);
-    subResult = b.create<arith::SelectOp>(roundToHalf, halfF4Bits, subResult);
-    f4Bits = b.create<arith::SelectOp>(isZeroExp, f4Bits, subResult);
+        arith::SelectOp::create(b, isSubnormal, subnormalF4Bits, f4Bits);
+    subResult = arith::SelectOp::create(b, roundToHalf, halfF4Bits, subResult);
+    f4Bits = arith::SelectOp::create(b, isZeroExp, f4Bits, subResult);
 
     // Step 5: Round up if necessary.
     Value cF32Last22BitMask = createConst(loc, i32Ty, 0x3fffff, rewriter);
     Value cRound = createConst(loc, i32Ty, 0x200000, rewriter); // 010 0000...
-    Value man22Bits = b.create<arith::AndIOp>(f32Bits, cF32Last22BitMask);
+    Value man22Bits = arith::AndIOp::create(b, f32Bits, cF32Last22BitMask);
     Value shouldRound =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, man22Bits, cRound);
-    shouldRound = b.create<arith::OrIOp>(shouldRound, isSubnormal);
-    Value roundedF4Bits = b.create<arith::AddIOp>(f4Bits, c0x1);
-    f4Bits = b.create<arith::SelectOp>(shouldRound, roundedF4Bits, f4Bits);
+        arith::CmpIOp::create(b, arith::CmpIPredicate::uge, man22Bits, cRound);
+    shouldRound = arith::OrIOp::create(b, shouldRound, isSubnormal);
+    Value roundedF4Bits = arith::AddIOp::create(b, f4Bits, c0x1);
+    f4Bits = arith::SelectOp::create(b, shouldRound, roundedF4Bits, f4Bits);
 
-    Value result = b.create<arith::BitcastOp>(resultTy, f4Bits);
+    Value result = arith::BitcastOp::create(b, resultTy, f4Bits);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -626,16 +626,16 @@ struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (operandETy.getIntOrFloatBitWidth() < 32) {
-      operand = b.create<arith::ExtFOp>(f32Ty, operand, op.getFastmathAttr());
+      operand = arith::ExtFOp::create(b, f32Ty, operand, op.getFastmathAttr());
     } else if (operandETy.getIntOrFloatBitWidth() > 32) {
-      operand = b.create<arith::TruncFOp>(
+      operand = arith::TruncFOp::create(b,
           f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
     }
-    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value f32Bits = arith::BitcastOp::create(b, i32Ty, operand);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
-    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
-    Value exp8Bits = b.create<arith::TruncIOp>(i8Ty, f32SignExp);
-    Value result = b.create<arith::BitcastOp>(resultTy, exp8Bits);
+    Value f32SignExp = arith::ShRUIOp::create(b, f32Bits, cF32MantissaWidth);
+    Value exp8Bits = arith::TruncIOp::create(b, i8Ty, f32SignExp);
+    Value result = arith::BitcastOp::create(b, resultTy, exp8Bits);
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -654,7 +654,7 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
     if (scaleETy.getIntOrFloatBitWidth() >= 16) {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
-      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+      scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
                                                op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
@@ -666,11 +666,11 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
     // extf on scale will essentially create floating point number
     // of type resulTy that is 2^scale and will also propagate NaNs
     Value scaleExt =
-        b.create<arith::ExtFOp>(resultTy, scaleOperand, op.getFastmathAttr());
+        arith::ExtFOp::create(b, resultTy, scaleOperand, op.getFastmathAttr());
     Value inputExt =
-        b.create<arith::ExtFOp>(resultTy, inputOperand, op.getFastmathAttr());
+        arith::ExtFOp::create(b, resultTy, inputOperand, op.getFastmathAttr());
     Value result =
-        b.create<arith::MulFOp>(inputExt, scaleExt, op.getFastmathAttr());
+        arith::MulFOp::create(b, inputExt, scaleExt, op.getFastmathAttr());
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -695,7 +695,7 @@ struct ScalingTruncFOpConverter
     if (scaleETy.getIntOrFloatBitWidth() >= 16) {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
-      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+      scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
                                                op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
@@ -708,10 +708,10 @@ struct ScalingTruncFOpConverter
     // this will create a floating point number of type
     // inputTy that is 2^scale and will also propagate NaNs
     scaleOperand =
-        b.create<arith::ExtFOp>(inputTy, scaleOperand, op.getFastmathAttr());
-    Value result = b.create<arith::DivFOp>(inputOperand, scaleOperand,
+        arith::ExtFOp::create(b, inputTy, scaleOperand, op.getFastmathAttr());
+    Value result = arith::DivFOp::create(b, inputOperand, scaleOperand,
                                            op.getFastmathAttr());
-    Value resultCast = b.create<arith::TruncFOp>(
+    Value resultCast = arith::TruncFOp::create(b,
         resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
     rewriter.replaceOp(op, resultCast);
     return success();
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index f2f93883eb2b7..777ff0ecaa314 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -305,18 +305,18 @@ static Value doCast(OpBuilder &builder, Location loc, Value src, Type dstType,
 
   if (isa<IndexType>(srcElemType) || isa<IndexType>(dstElemType)) {
     if (castKind == CastKind::Signed)
-      return builder.create<arith::IndexCastOp>(loc, dstType, src);
-    return builder.create<arith::IndexCastUIOp>(loc, dstType, src);
+      return arith::IndexCastOp::create(builder, loc, dstType, src);
+    return arith::IndexCastUIOp::create(builder, loc, dstType, src);
   }
 
   auto srcInt = cast<IntegerType>(srcElemType);
   auto dstInt = cast<IntegerType>(dstElemType);
   if (dstInt.getWidth() < srcInt.getWidth())
-    return builder.create<arith::TruncIOp>(loc, dstType, src);
+    return arith::TruncIOp::create(builder, loc, dstType, src);
 
   if (castKind == CastKind::Signed)
-    return builder.create<arith::ExtSIOp>(loc, dstType, src);
-  return builder.create<arith::ExtUIOp>(loc, dstType, src);
+    return arith::ExtSIOp::create(builder, loc, dstType, src);
+  return arith::ExtUIOp::create(builder, loc, dstType, src);
 }
 
 struct NarrowElementwise final : OpTraitRewritePattern<OpTrait::Elementwise> {
diff --git a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
index 5fb7953f93700..904aab33345f0 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
@@ -23,7 +23,7 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
   std::function<Value(AffineExpr)> buildExpr = [&](AffineExpr e) -> Value {
     switch (e.getKind()) {
     case AffineExprKind::Constant:
-      return b.create<ConstantIndexOp>(loc,
+      return ConstantIndexOp::create(b, loc,
                                        cast<AffineConstantExpr>(e).getValue());
     case AffineExprKind::DimId:
       return operands[cast<AffineDimExpr>(e).getPosition()];
@@ -32,27 +32,27 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
                       map.getNumDims()];
     case AffineExprKind::Add: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<AddIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return AddIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                               buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mul: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<MulIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return MulIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                               buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::FloorDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<DivSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return DivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::CeilDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<CeilDivSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return CeilDivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                    buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mod: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
-      return b.create<RemSIOp>(loc, buildExpr(binaryExpr.getLHS()),
+      return RemSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
                                buildExpr(binaryExpr.getRHS()));
     }
     }
@@ -89,10 +89,10 @@ FailureOr<OpFoldResult> mlir::arith::reifyValueBound(
            "expected dynamic dim");
     if (isa<RankedTensorType>(value.getType())) {
       // A tensor dimension is used: generate a tensor.dim.
-      operands.push_back(b.create<tensor::DimOp>(loc, value, *dim));
+      operands.push_back(tensor::DimOp::create(b, loc, value, *dim));
     } else if (isa<MemRefType>(value.getType())) {
       // A memref dimension is used: generate a memref.dim.
-      operands.push_back(b.create<memref::DimOp>(loc, value, *dim));
+      operands.push_back(memref::DimOp::create(b, loc, value, *dim));
     } else {
       llvm_unreachable("cannot generate DimOp for unsupported shaped type");
     }
diff --git a/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
index 3478adcb4a128..dd6efe6d6bc31 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ShardingInterfaceImpl.cpp
@@ -83,7 +83,7 @@ struct ConstantShardingInterface
           cOp.getType(), getMesh(op, sharding.getMeshAttr(), symbolTable),
           sharding));
       auto newValue = value.resizeSplat(newType);
-      auto newOp = builder.create<ConstantOp>(op->getLoc(), newType, newValue);
+      auto newOp = ConstantOp::create(builder, op->getLoc(), newType, newValue);
       spmdizationMap.map(op->getResult(0), newOp.getResult());
       spmdizationMap.map(op, newOp.getOperation());
     } else {
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index 3cd8684878a11..6e9cc14c6ada5 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -68,7 +68,7 @@ mlir::inferExpandShapeOutputShape(OpBuilder &b, Location loc,
     // dynamism.
     Value indexGroupSize = cast<Value>(inputShape[inputIndex]);
     Value indexGroupStaticSizesProduct =
-        b.create<arith::ConstantIndexOp>(loc, indexGroupStaticSizesProductInt);
+        arith::ConstantIndexOp::create(b, loc, indexGroupStaticSizesProductInt);
     Value dynamicDimSize = b.createOrFold<arith::DivSIOp>(
         loc, indexGroupSize, indexGroupStaticSizesProduct);
     outputShapeValues.push_back(dynamicDimSize);
@@ -105,7 +105,7 @@ Value mlir::getValueOrCreateConstantIntOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return b.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(b,
       loc, b.getIntegerAttr(attr.getType(), attr.getValue().getSExtValue()));
 }
 
@@ -114,7 +114,7 @@ Value mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return b.create<arith::ConstantIndexOp>(loc, attr.getValue().getSExtValue());
+  return arith::ConstantIndexOp::create(b, loc, attr.getValue().getSExtValue());
 }
 
 Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
@@ -125,7 +125,7 @@ Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
   bool targetIsIndex = targetType.isIndex();
   bool valueIsIndex = value.getType().isIndex();
   if (targetIsIndex ^ valueIsIndex)
-    return b.create<arith::IndexCastOp>(loc, targetType, value);
+    return arith::IndexCastOp::create(b, loc, targetType, value);
 
   auto targetIntegerType = dyn_cast<IntegerType>(targetType);
   auto valueIntegerType = dyn_cast<IntegerType>(value.getType());
@@ -134,8 +134,8 @@ Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
   assert(targetIntegerType.getSignedness() == valueIntegerType.getSignedness());
 
   if (targetIntegerType.getWidth() > valueIntegerType.getWidth())
-    return b.create<arith::ExtSIOp>(loc, targetIntegerType, value);
-  return b.create<arith::TruncIOp>(loc, targetIntegerType, value);
+    return arith::ExtSIOp::create(b, loc, targetIntegerType, value);
+  return arith::TruncIOp::create(b, loc, targetIntegerType, value);
 }
 
 static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
@@ -143,21 +143,21 @@ static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
   // If operand is floating point, cast directly to the int type.
   if (isa<FloatType>(operand.getType())) {
     if (isUnsigned)
-      return b.create<arith::FPToUIOp>(toType, operand);
-    return b.create<arith::FPToSIOp>(toType, operand);
+      return arith::FPToUIOp::create(b, toType, operand);
+    return arith::FPToSIOp::create(b, toType, operand);
   }
   // Cast index operands directly to the int type.
   if (operand.getType().isIndex())
-    return b.create<arith::IndexCastOp>(toType, operand);
+    return arith::IndexCastOp::create(b, toType, operand);
   if (auto fromIntType = dyn_cast<IntegerType>(operand.getType())) {
     // Either extend or truncate.
     if (toType.getWidth() > fromIntType.getWidth()) {
       if (isUnsigned)
-        return b.create<arith::ExtUIOp>(toType, operand);
-      return b.create<arith::ExtSIOp>(toType, operand);
+        return arith::ExtUIOp::create(b, toType, operand);
+      return arith::ExtSIOp::create(b, toType, operand);
     }
     if (toType.getWidth() < fromIntType.getWidth())
-      return b.create<arith::TruncIOp>(toType, operand);
+      return arith::TruncIOp::create(b, toType, operand);
     return operand;
   }
 
@@ -170,14 +170,14 @@ static Value convertScalarToFpDtype(ImplicitLocOpBuilder &b, Value operand,
   // Note that it is unclear how to cast from BF16<->FP16.
   if (isa<IntegerType>(operand.getType())) {
     if (isUnsigned)
-      return b.create<arith::UIToFPOp>(toType, operand);
-    return b.create<arith::SIToFPOp>(toType, operand);
+      return arith::UIToFPOp::create(b, toType, operand);
+    return arith::SIToFPOp::create(b, toType, operand);
   }
   if (auto fromFpTy = dyn_cast<FloatType>(operand.getType())) {
     if (toType.getWidth() > fromFpTy.getWidth())
-      return b.create<arith::ExtFOp>(toType, operand);
+      return arith::ExtFOp::create(b, toType, operand);
     if (toType.getWidth() < fromFpTy.getWidth())
-      return b.create<arith::TruncFOp>(toType, operand);
+      return arith::TruncFOp::create(b, toType, operand);
     return operand;
   }
 
@@ -190,18 +190,18 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
   if (auto fromComplexType = dyn_cast<ComplexType>(operand.getType())) {
     if (isa<FloatType>(targetType.getElementType()) &&
         isa<FloatType>(fromComplexType.getElementType())) {
-      Value real = b.create<complex::ReOp>(operand);
-      Value imag = b.create<complex::ImOp>(operand);
+      Value real = complex::ReOp::create(b, operand);
+      Value imag = complex::ImOp::create(b, operand);
       Type targetETy = targetType.getElementType();
       if (targetType.getElementType().getIntOrFloatBitWidth() <
           fromComplexType.getElementType().getIntOrFloatBitWidth()) {
-        real = b.create<arith::TruncFOp>(targetETy, real);
-        imag = b.create<arith::TruncFOp>(targetETy, imag);
+        real = arith::TruncFOp::create(b, targetETy, real);
+        imag = arith::TruncFOp::create(b, targetETy, imag);
       } else {
-        real = b.create<arith::ExtFOp>(targetETy, real);
-        imag = b.create<arith::ExtFOp>(targetETy, imag);
+        real = arith::ExtFOp::create(b, targetETy, real);
+        imag = arith::ExtFOp::create(b, targetETy, imag);
       }
-      return b.create<complex::CreateOp>(targetType, real, imag);
+      return complex::CreateOp::create(b, targetType, real, imag);
     }
   }
 
@@ -210,27 +210,27 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
     auto toBitwidth = toFpTy.getIntOrFloatBitWidth();
     Value from = operand;
     if (from.getType().getIntOrFloatBitWidth() < toBitwidth) {
-      from = b.create<arith::ExtFOp>(toFpTy, from);
+      from = arith::ExtFOp::create(b, toFpTy, from);
     }
     if (from.getType().getIntOrFloatBitWidth() > toBitwidth) {
-      from = b.create<arith::TruncFOp>(toFpTy, from);
+      from = arith::TruncFOp::create(b, toFpTy, from);
     }
-    Value zero = b.create<mlir::arith::ConstantFloatOp>(
+    Value zero = mlir::arith::ConstantFloatOp::create(b,
         toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
-    return b.create<complex::CreateOp>(targetType, from, zero);
+    return complex::CreateOp::create(b, targetType, from, zero);
   }
 
   if (isa<IntegerType>(operand.getType())) {
     FloatType toFpTy = cast<FloatType>(targetType.getElementType());
     Value from = operand;
     if (isUnsigned) {
-      from = b.create<arith::UIToFPOp>(toFpTy, from);
+      from = arith::UIToFPOp::create(b, toFpTy, from);
     } else {
-      from = b.create<arith::SIToFPOp>(toFpTy, from);
+      from = arith::SIToFPOp::create(b, toFpTy, from);
     }
-    Value zero = b.create<mlir::arith::ConstantFloatOp>(
+    Value zero = mlir::arith::ConstantFloatOp::create(b,
         toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
-    return b.create<complex::CreateOp>(targetType, from, zero);
+    return complex::CreateOp::create(b, targetType, from, zero);
   }
 
   return {};
@@ -278,7 +278,7 @@ Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
     attr = SplatElementsAttr::get(vecTy, value);
   }
 
-  return builder.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(builder, loc, attr);
 }
 
 Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
@@ -310,35 +310,35 @@ Type mlir::getType(OpFoldResult ofr) {
 }
 
 Value ArithBuilder::_and(Value lhs, Value rhs) {
-  return b.create<arith::AndIOp>(loc, lhs, rhs);
+  return arith::AndIOp::create(b, loc, lhs, rhs);
 }
 Value ArithBuilder::add(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::AddFOp>(loc, lhs, rhs);
-  return b.create<arith::AddIOp>(loc, lhs, rhs, ovf);
+    return arith::AddFOp::create(b, loc, lhs, rhs);
+  return arith::AddIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::sub(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::SubFOp>(loc, lhs, rhs);
-  return b.create<arith::SubIOp>(loc, lhs, rhs, ovf);
+    return arith::SubFOp::create(b, loc, lhs, rhs);
+  return arith::SubIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::mul(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::MulFOp>(loc, lhs, rhs);
-  return b.create<arith::MulIOp>(loc, lhs, rhs, ovf);
+    return arith::MulFOp::create(b, loc, lhs, rhs);
+  return arith::MulIOp::create(b, loc, lhs, rhs, ovf);
 }
 Value ArithBuilder::sgt(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OGT, lhs, rhs);
-  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sgt, lhs, rhs);
+    return arith::CmpFOp::create(b, loc, arith::CmpFPredicate::OGT, lhs, rhs);
+  return arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sgt, lhs, rhs);
 }
 Value ArithBuilder::slt(Value lhs, Value rhs) {
   if (isa<FloatType>(lhs.getType()))
-    return b.create<arith::CmpFOp>(loc, arith::CmpFPredicate::OLT, lhs, rhs);
-  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, lhs, rhs);
+    return arith::CmpFOp::create(b, loc, arith::CmpFPredicate::OLT, lhs, rhs);
+  return arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, lhs, rhs);
 }
 Value ArithBuilder::select(Value cmp, Value lhs, Value rhs) {
-  return b.create<arith::SelectOp>(loc, cmp, lhs, rhs);
+  return arith::SelectOp::create(b, loc, cmp, lhs, rhs);
 }
 
 namespace mlir::arith {
@@ -349,7 +349,7 @@ Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values) {
 
 Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
                     Type resultType) {
-  Value one = builder.create<ConstantOp>(loc, resultType,
+  Value one = ConstantOp::create(builder, loc, resultType,
                                          builder.getOneAttr(resultType));
   ArithBuilder arithBuilder(builder, loc);
   return std::accumulate(
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
index a07be7801869f..b9a81b71fa301 100644
--- a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
@@ -229,7 +229,7 @@ class LowerContractionToNeonI8MMPattern
 
     // Initial accumulator for the final result. This is the un-tiled result if
     // tiling is done.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, op.getResultType(), rewriter.getZeroAttr(op.getResultType()));
 
     SmallVector<int64_t> unrolledSize = *op.getShapeForUnroll();
@@ -281,7 +281,7 @@ class LowerContractionToNeonI8MMPattern
       if (isVecmat) {
         auto expandForSMMLA = [&](Value tiledOperand,
                                   VectorType expandedTypeType) {
-          auto emptyOperand = rewriter.create<arith::ConstantOp>(
+          auto emptyOperand = arith::ConstantOp::create(rewriter,
               loc, expandedTypeType, rewriter.getZeroAttr(expandedTypeType));
           SmallVector<int64_t> offsets(
               cast<ShapedType>(emptyOperand.getType()).getRank(), 0);
@@ -298,7 +298,7 @@ class LowerContractionToNeonI8MMPattern
       // using the instruction for unsigned by signed multiplication with
       // reversed operands.
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledAcc = rewriter.create<vector::TransposeOp>(
+        tiledAcc = vector::TransposeOp::create(rewriter,
             loc, tiledAcc, ArrayRef<int64_t>({1, 0}));
 
       // Collapse tiled operands to 1D vectors required by smmla intrinsic
@@ -331,7 +331,7 @@ class LowerContractionToNeonI8MMPattern
       // Because of the reversed operands the result is obtained transposed.
       // Transpose it back,
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledRes = rewriter.create<vector::TransposeOp>(
+        tiledRes = vector::TransposeOp::create(rewriter,
             loc, tiledRes, ArrayRef<int64_t>({1, 0}));
 
       // With vecmat, only one row of tiled ACC can be inserted into the final
diff --git a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
index 5f00cef90159e..23ec00196f714 100644
--- a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
@@ -75,21 +75,21 @@ scf::ForOp createLoopOverTileSlices(
     PatternRewriter &rewriter, Location loc, Value initTile,
     std::function<Value(OpBuilder &, Location, Value, Value)> makeLoopBody) {
   OpBuilder::InsertionGuard g(rewriter);
-  auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-  auto minTileSlices = rewriter.create<arith::ConstantIndexOp>(
+  auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
       loc, llvm::cast<VectorType>(initTile.getType()).getDimSize(0));
   auto vscale =
-      rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-  auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+      vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+  auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto numTileSlices =
-      rewriter.create<arith::MulIOp>(loc, minTileSlices, vscale);
-  auto forOp = rewriter.create<scf::ForOp>(loc, lowerBound, numTileSlices, step,
+      arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices, step,
                                            ValueRange{initTile});
   rewriter.setInsertionPointToStart(forOp.getBody());
   Value nextTile =
       makeLoopBody(rewriter, loc, /*tileSliceIndex=*/forOp.getInductionVar(),
                    /*currentTile=*/forOp.getRegionIterArg(0));
-  rewriter.create<scf::YieldOp>(loc, nextTile);
+  scf::YieldOp::create(rewriter, loc, nextTile);
   return forOp;
 }
 
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
index 23f2c2bf65e47..4d03bbdabdf09 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
@@ -136,7 +136,7 @@ class OuterProductFusion2Way
 
     auto loc = op.getLoc();
     auto packInputs = [&](Value lhs, Value rhs) {
-      return rewriter.create<vector::InterleaveOp>(loc, lhs, rhs);
+      return vector::InterleaveOp::create(rewriter, loc, lhs, rhs);
     };
 
     auto lhs = packInputs(op1.getLhs().getDefiningOp()->getOperand(0),
@@ -284,7 +284,7 @@ class OuterProductFusion4Way
 
     auto loc = op.getLoc();
     auto packInputs = [&](Value lhs, Value rhs) {
-      return rewriter.create<vector::InterleaveOp>(loc, lhs, rhs);
+      return vector::InterleaveOp::create(rewriter, loc, lhs, rhs);
     };
 
     auto lhs0 = packInputs(op1.getLhs().getDefiningOp()->getOperand(0),
@@ -456,7 +456,7 @@ struct SwapVectorExtractOfArithExtend
     Value extendSource = extendOp->getOperand(0);
 
     // Create new extract from source of extend.
-    Value newExtract = rewriter.create<vector::ExtractOp>(
+    Value newExtract = vector::ExtractOp::create(rewriter,
         loc, extendSource, extractOp.getMixedPosition());
 
     // Extend new extract to original result type.
@@ -503,7 +503,7 @@ struct SwapVectorScalableExtractOfArithExtend
     // Create new extract from source of extend.
     VectorType extractResultVectorType =
         resultType.clone(extendSourceVectorType.getElementType());
-    Value newExtract = rewriter.create<vector::ScalableExtractOp>(
+    Value newExtract = vector::ScalableExtractOp::create(rewriter,
         loc, extractResultVectorType, extendSource, extractOp.getPos());
 
     // Extend new extract to original result type.
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index b3c988d455420..d925c19852679 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -210,7 +210,7 @@ void splitCondBranches(IRRewriter &rewriter, FunctionOpInterface function) {
 
   auto insertJump = [&](Location loc, Block *source, Block *dest, auto args) {
     rewriter.setInsertionPointToEnd(source);
-    rewriter.create<cf::BranchOp>(loc, dest, args);
+    cf::BranchOp::create(rewriter, loc, dest, args);
   };
 
   for (auto condBranch : worklist) {
@@ -253,7 +253,7 @@ void insertCopiesAtBranches(IRRewriter &rewriter,
     for (OpOperand &operand : terminator->getOpOperands()) {
       if (isValidSMETileVectorType(operand.get().getType())) {
         auto copy =
-            rewriter.create<CopyTileOp>(terminator->getLoc(), operand.get());
+            CopyTileOp::create(rewriter, terminator->getLoc(), operand.get());
         rewriter.modifyOpInPlace(terminator, [&] { operand.assign(copy); });
       }
     }
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 1e8e1265affa0..3362d0ce13533 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -82,13 +82,13 @@ SmallVector<Value, 2> addConstantScalableOffset(OpBuilder &builder,
                                                 Location loc,
                                                 ValueRange indices,
                                                 ArrayRef<int> scalableOffsets) {
-  auto vscale = builder.create<vector::VectorScaleOp>(loc);
+  auto vscale = vector::VectorScaleOp::create(builder, loc);
   return llvm::map_to_vector(
       llvm::zip_equal(indices, scalableOffsets), [&](auto pair) -> Value {
         auto [index, base] = pair;
-        auto offset = builder.create<arith::MulIOp>(
-            loc, builder.create<arith::ConstantIndexOp>(loc, base), vscale);
-        return builder.create<arith::AddIOp>(loc, index, offset);
+        auto offset = arith::MulIOp::create(builder,
+            loc, arith::ConstantIndexOp::create(builder, loc, base), vscale);
+        return arith::AddIOp::create(builder, loc, index, offset);
       });
 }
 
@@ -132,7 +132,7 @@ Value extractSMEMask(OpBuilder &builder, Location loc, Value mask,
   // from the mask operands to get the parameters for this sub-tile.
   auto smeTileMaskDims = addConstantScalableOffset(
       builder, loc, createMask.getOperands(), {-smeTile.row, -smeTile.col});
-  auto smeTileCreateMask = builder.create<vector::CreateMaskOp>(
+  auto smeTileCreateMask = vector::CreateMaskOp::create(builder,
       loc, smeTile.type.clone(builder.getI1Type()), smeTileMaskDims);
   return smeTileCreateMask.getResult();
 }
@@ -190,7 +190,7 @@ struct LegalizeArithConstantOpsByDecomposition
 
     auto smeTileType = getSMETileTypeForElement(vectorType.getElementType());
     auto tileCount = getNumberOfSMETilesForVectorType(vectorType);
-    auto tileSplat = rewriter.create<arith::ConstantOp>(
+    auto tileSplat = arith::ConstantOp::create(rewriter,
         constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
     SmallVector<Value> repl(tileCount, tileSplat);
     rewriter.replaceOpWithMultiple(constantOp, {repl});
@@ -237,11 +237,11 @@ struct LegalizeVectorOuterProductOpsByDecomposition
              decomposeToSMETiles(rewriter, vectorType, smeTileType))) {
 
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto lhs = rewriter.create<vector::ScalableExtractOp>(
+      auto lhs = vector::ScalableExtractOp::create(rewriter,
           loc, sliceType, outerProductOp.getLhs(), smeTile.row);
-      auto rhs = rewriter.create<vector::ScalableExtractOp>(
+      auto rhs = vector::ScalableExtractOp::create(rewriter,
           loc, sliceType, outerProductOp.getRhs(), smeTile.col);
-      auto smeOuterProduct = rewriter.create<vector::OuterProductOp>(
+      auto smeOuterProduct = vector::OuterProductOp::create(rewriter,
           loc, smeTileType, lhs, rhs,
           !accSMETiles.empty() ? accSMETiles[index] : Value{},
           outerProductOp.getKind());
@@ -314,7 +314,7 @@ struct LegalizeTransferReadOpsByDecomposition
     for (SMESubTile smeTile :
          decomposeToSMETiles(rewriter, vectorType, smeTileType, transposed)) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeRead = rewriter.create<vector::TransferReadOp>(
+      auto smeRead = vector::TransferReadOp::create(rewriter,
           loc, smeTileType, readOp.getBase(),
           getSMESubTileIndices(rewriter, loc, readOp.getIndices(), smeTile),
           readOp.getPermutationMapAttr(), readOp.getPadding(), smeMask,
@@ -363,7 +363,7 @@ struct LegalizeTransferWriteOpsByDecomposition
     for (auto [index, smeTile] : llvm::enumerate(decomposeToSMETiles(
              rewriter, vectorType, smeTileType, transposed))) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeWrite = rewriter.create<vector::TransferWriteOp>(
+      auto smeWrite = vector::TransferWriteOp::create(rewriter,
           loc, inputSMETiles[index], destTensorOrMemref,
           getSMESubTileIndices(rewriter, loc, writeOp.getIndices(), smeTile),
           writeOp.getPermutationMapAttr(), smeMask, writeOp.getInBoundsAttr());
@@ -456,11 +456,11 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
         VectorType::get(minTileSlices, rewriter.getI1Type(), true);
 
     // Create loop over all tile slices.
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto upperBound = createVscaleMultiple(minTileSlices);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto storeLoop =
-        rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
+        scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     rewriter.setInsertionPointToStart(storeLoop.getBody());
 
     // For each sub-tile of the multi-tile `vectorType`.
@@ -474,29 +474,29 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
 
       // The current slice of `vectorType` we are processing.
       auto sliceIndex =
-          rewriter.create<arith::AddIOp>(loc, tileRow, tileSliceIndex);
+          arith::AddIOp::create(rewriter, loc, tileRow, tileSliceIndex);
 
       // Where in the destination memref the current slice will be stored.
-      auto storeRow = rewriter.create<arith::AddIOp>(loc, sliceIndex,
+      auto storeRow = arith::AddIOp::create(rewriter, loc, sliceIndex,
                                                      writeOp.getIndices()[0]);
       auto storeCol =
-          rewriter.create<arith::AddIOp>(loc, tileCol, writeOp.getIndices()[1]);
+          arith::AddIOp::create(rewriter, loc, tileCol, writeOp.getIndices()[1]);
 
       // Extract the mask for the current slice.
       Value sliceMask = nullptr;
       if (mask) {
-        sliceMask = rewriter.create<vector::ExtractOp>(
+        sliceMask = vector::ExtractOp::create(rewriter,
             loc, mask, OpFoldResult(sliceIndex));
         if (sliceMaskType != sliceMask.getType())
-          sliceMask = rewriter.create<vector::ScalableExtractOp>(
+          sliceMask = vector::ScalableExtractOp::create(rewriter,
               loc, sliceMaskType, sliceMask, smeTile.col);
       }
 
       // Extract and store the current slice.
       Value tile = inputSMETiles[index];
       auto slice =
-          rewriter.create<vector::ExtractOp>(loc, tile, tileSliceIndex);
-      rewriter.create<vector::TransferWriteOp>(
+          vector::ExtractOp::create(rewriter, loc, tile, tileSliceIndex);
+      vector::TransferWriteOp::create(rewriter,
           loc, slice, writeOp.getBase(), ValueRange{storeRow, storeCol},
           AffineMapAttr::get(writeOp.getPermutationMap().dropResult(0)),
           sliceMask,
@@ -567,13 +567,13 @@ struct FoldExtractFromVectorOfSMELikeCreateMasks
           extractOp,
           "constant vector.create_masks dims should be folded elsewhere");
 
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto extractionIndex = getValueOrCreateConstantIndexOp(
         rewriter, loc, extractOp.getMixedPosition()[0]);
-    auto extractionInTrueRegion = rewriter.create<arith::CmpIOp>(
+    auto extractionInTrueRegion = arith::CmpIOp::create(rewriter,
         loc, rewriter.getI1Type(), arith::CmpIPredicate::slt, extractionIndex,
         frontMaskDim);
-    auto newMaskFrontDim = rewriter.create<arith::SelectOp>(
+    auto newMaskFrontDim = arith::SelectOp::create(rewriter,
         loc, extractionInTrueRegion, createMaskOp.getOperand(1), zero);
 
     rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(
@@ -660,8 +660,8 @@ struct LiftIllegalVectorTransposeToMemory
           illegalRead, "expected read to have identity permutation map");
 
     auto loc = transposeOp.getLoc();
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
 
     // Create a subview that matches the size of the illegal read vector type.
     auto readType = illegalRead.getVectorType();
@@ -669,14 +669,14 @@ struct LiftIllegalVectorTransposeToMemory
         llvm::zip_equal(readType.getShape(), readType.getScalableDims()),
         [&](auto dim) -> Value {
           auto [size, isScalable] = dim;
-          auto dimSize = rewriter.create<arith::ConstantIndexOp>(loc, size);
+          auto dimSize = arith::ConstantIndexOp::create(rewriter, loc, size);
           if (!isScalable)
             return dimSize;
-          auto vscale = rewriter.create<vector::VectorScaleOp>(loc);
-          return rewriter.create<arith::MulIOp>(loc, vscale, dimSize);
+          auto vscale = vector::VectorScaleOp::create(rewriter, loc);
+          return arith::MulIOp::create(rewriter, loc, vscale, dimSize);
         });
     SmallVector<Value> strides(readType.getRank(), Value(one));
-    auto readSubview = rewriter.create<memref::SubViewOp>(
+    auto readSubview = memref::SubViewOp::create(rewriter,
         loc, illegalRead.getBase(), illegalRead.getIndices(), readSizes,
         strides);
 
@@ -686,13 +686,13 @@ struct LiftIllegalVectorTransposeToMemory
     if (mask) {
       // Note: The transpose for the mask should fold into the
       // vector.create_mask/constant_mask op, which will then become legal.
-      mask = rewriter.create<vector::TransposeOp>(loc, mask,
+      mask = vector::TransposeOp::create(rewriter, loc, mask,
                                                   transposeOp.getPermutation());
     }
     // - The source memref
     mlir::AffineMap transposeMap = AffineMap::getPermutationMap(
         transposeOp.getPermutation(), getContext());
-    auto transposedSubview = rewriter.create<memref::TransposeOp>(
+    auto transposedSubview = memref::TransposeOp::create(rewriter,
         loc, readSubview, AffineMapAttr::get(transposeMap));
     ArrayAttr inBoundsAttr = illegalRead.getInBoundsAttr();
     // - The `in_bounds` attribute
@@ -706,7 +706,7 @@ struct LiftIllegalVectorTransposeToMemory
     VectorType legalReadType = resultType.clone(readType.getElementType());
     // Note: The indices are all zero as the subview is already offset.
     SmallVector<Value> readIndices(illegalRead.getIndices().size(), zero);
-    auto legalRead = rewriter.create<vector::TransferReadOp>(
+    auto legalRead = vector::TransferReadOp::create(rewriter,
         loc, legalReadType, transposedSubview, readIndices,
         illegalRead.getPermutationMapAttr(), illegalRead.getPadding(), mask,
         inBoundsAttr);
@@ -797,12 +797,12 @@ struct LowerIllegalTransposeStoreViaZA
         AffineMap::getPermutationMap(ArrayRef<int64_t>{1, 0}, getContext()));
 
     // Note: We need to use `get_tile` as there's no vector-level `undef`.
-    Value undefTile = rewriter.create<arm_sme::GetTileOp>(loc, smeTileType);
+    Value undefTile = arm_sme::GetTileOp::create(rewriter, loc, smeTileType);
     Value destTensorOrMemref = writeOp.getBase();
     auto numSlicesPerTile =
         std::min(sourceType.getDimSize(0), smeTileType.getDimSize(0));
     auto numSlices =
-        rewriter.create<arith::ConstantIndexOp>(loc, numSlicesPerTile);
+        arith::ConstantIndexOp::create(rewriter, loc, numSlicesPerTile);
     for (auto [index, smeTile] : llvm::enumerate(
              decomposeToSMETiles(rewriter, sourceType, smeTileType))) {
       // 1. _Deliberately_ drop a scalable dimension and insert a fixed number
@@ -811,46 +811,46 @@ struct LowerIllegalTransposeStoreViaZA
       // rows of the tile after 1*vscale rows.
       Value tile = undefTile;
       for (int d = 0; d < numSlicesPerTile; ++d) {
-        Value vector = rewriter.create<vector::ExtractOp>(
+        Value vector = vector::ExtractOp::create(rewriter,
             loc, transposeOp.getVector(),
             rewriter.getIndexAttr(d + smeTile.row));
         if (vector.getType() != smeSliceType) {
-          vector = rewriter.create<vector::ScalableExtractOp>(
+          vector = vector::ScalableExtractOp::create(rewriter,
               loc, smeSliceType, vector, smeTile.col);
         }
-        tile = rewriter.create<vector::InsertOp>(loc, vector, tile, d);
+        tile = vector::InsertOp::create(rewriter, loc, vector, tile, d);
       }
 
       // 2. Transpose the tile position.
       auto transposedRow = createVscaleMultiple(smeTile.col);
       auto transposedCol =
-          rewriter.create<arith::ConstantIndexOp>(loc, smeTile.row);
+          arith::ConstantIndexOp::create(rewriter, loc, smeTile.row);
 
       // 3. Compute mask for tile store.
       Value maskRows;
       Value maskCols;
       if (auto mask = writeOp.getMask()) {
         auto createMask = mask.getDefiningOp<vector::CreateMaskOp>();
-        maskRows = rewriter.create<arith::SubIOp>(loc, createMask.getOperand(0),
+        maskRows = arith::SubIOp::create(rewriter, loc, createMask.getOperand(0),
                                                   transposedRow);
-        maskCols = rewriter.create<arith::SubIOp>(loc, createMask.getOperand(1),
+        maskCols = arith::SubIOp::create(rewriter, loc, createMask.getOperand(1),
                                                   transposedCol);
-        maskCols = rewriter.create<index::MinSOp>(loc, maskCols, numSlices);
+        maskCols = index::MinSOp::create(rewriter, loc, maskCols, numSlices);
       } else {
         maskRows = createVscaleMultiple(smeTileType.getDimSize(0));
         maskCols = numSlices;
       }
-      auto subMask = rewriter.create<vector::CreateMaskOp>(
+      auto subMask = vector::CreateMaskOp::create(rewriter,
           loc, smeTileType.clone(rewriter.getI1Type()),
           ValueRange{maskRows, maskCols});
 
       // 4. Emit a transposed tile write.
       auto writeIndices = writeOp.getIndices();
       Value destRow =
-          rewriter.create<arith::AddIOp>(loc, transposedRow, writeIndices[0]);
+          arith::AddIOp::create(rewriter, loc, transposedRow, writeIndices[0]);
       Value destCol =
-          rewriter.create<arith::AddIOp>(loc, transposedCol, writeIndices[1]);
-      auto smeWrite = rewriter.create<vector::TransferWriteOp>(
+          arith::AddIOp::create(rewriter, loc, transposedCol, writeIndices[1]);
+      auto smeWrite = vector::TransferWriteOp::create(rewriter,
           loc, tile, destTensorOrMemref, ValueRange{destRow, destCol},
           transposeMap, subMask, writeOp.getInBounds());
 
@@ -934,41 +934,41 @@ struct LowerColumnTransferReadToLoops
 
     // Create a loop over all rows and load one element at a time.
     auto loc = readOp.getLoc();
-    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto createVscaleMultiple =
         vector::makeVscaleConstantBuilder(rewriter, loc);
     auto upperBound = createVscaleMultiple(numRows);
-    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    Value init = rewriter.create<arith::ConstantOp>(
+    auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    Value init = arith::ConstantOp::create(rewriter,
         loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
 
     scf::ForOp loadLoop;
     {
       OpBuilder::InsertionGuard g(rewriter);
-      loadLoop = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+      loadLoop = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
                                              ValueRange{init});
       rewriter.setInsertionPointToStart(loadLoop.getBody());
 
       auto tileSliceIndex = loadLoop.getInductionVar();
 
-      auto idx0 = rewriter.create<arith::AddIOp>(loc, tileSliceIndex,
+      auto idx0 = arith::AddIOp::create(rewriter, loc, tileSliceIndex,
                                                  readOp.getIndices()[0]);
       auto idx1 = readOp.getIndices()[1];
 
-      Value scalar = rewriter.create<memref::LoadOp>(
+      Value scalar = memref::LoadOp::create(rewriter,
           loc, readOp.getBase(), SmallVector<Value>({idx0, idx1}));
 
-      Operation *updateInit = rewriter.create<vector::InsertOp>(
+      Operation *updateInit = vector::InsertOp::create(rewriter,
           loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
 
-      rewriter.create<scf::YieldOp>(loc, updateInit->getResult(0));
+      scf::YieldOp::create(rewriter, loc, updateInit->getResult(0));
     }
 
     // The read operation has been "legalized", but since the original result
     // type was a 2D vector, we need to cast before returning the result. This
     // ShapeCast should cancel-out with some other ShapeCast (i.e. it's a
     // no-op).
-    auto sc = rewriter.create<vector::ShapeCastOp>(
+    auto sc = vector::ShapeCastOp::create(rewriter,
         loc, readOp.getResult().getType(), loadLoop.getResult(0));
 
     rewriter.replaceOp(readOp, sc);
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index 7b64e5745fcef..3ee211fc09bee 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -87,7 +87,7 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     VectorType sourceType = source.getType();
     VectorType resultType = convertOp.getResult().getType();
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
 
     // We want to iterate over the input vector in steps of the trailing
@@ -100,14 +100,14 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     for (SmallVector<int64_t> index :
          StaticTileOffsetRange(sourceType.getShape(), tileShape)) {
       auto extractOrInsertPosition = ArrayRef(index).drop_back();
-      auto sourceVector = rewriter.create<vector::ExtractOp>(
+      auto sourceVector = vector::ExtractOp::create(rewriter,
           loc, source, extractOrInsertPosition);
       VectorType convertedType =
           VectorType::Builder(llvm::cast<VectorType>(sourceVector.getType()))
               .setDim(0, resultType.getShape().back());
       auto convertedVector =
-          rewriter.create<IntrOp>(loc, TypeRange{convertedType}, sourceVector);
-      result = rewriter.create<vector::InsertOp>(loc, convertedVector, result,
+          IntrOp::create(rewriter, loc, TypeRange{convertedType}, sourceVector);
+      result = vector::InsertOp::create(rewriter, loc, convertedVector, result,
                                                  extractOrInsertPosition);
     }
 
@@ -135,11 +135,11 @@ struct PselOpLowering : public ConvertOpToLLVMPattern<PselOp> {
                   ConversionPatternRewriter &rewriter) const override {
     auto svboolType = VectorType::get(16, rewriter.getI1Type(), true);
     auto loc = pselOp.getLoc();
-    auto svboolP1 = rewriter.create<ConvertToSvboolIntrOp>(loc, svboolType,
+    auto svboolP1 = ConvertToSvboolIntrOp::create(rewriter, loc, svboolType,
                                                            adaptor.getP1());
-    auto indexI32 = rewriter.create<arith::IndexCastOp>(
+    auto indexI32 = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI32Type(), pselOp.getIndex());
-    auto pselIntr = rewriter.create<PselIntrOp>(loc, svboolType, svboolP1,
+    auto pselIntr = PselIntrOp::create(rewriter, loc, svboolType, svboolP1,
                                                 pselOp.getP2(), indexI32);
     rewriter.replaceOpWithNewOp<ConvertFromSvboolIntrOp>(
         pselOp, adaptor.getP1().getType(), pselIntr);
@@ -174,7 +174,7 @@ struct CreateMaskOpLowering
                                          "not SVE predicate-sized");
 
     auto loc = createMaskOp.getLoc();
-    auto zero = rewriter.create<LLVM::ZeroOp>(loc, rewriter.getI64Type());
+    auto zero = LLVM::ZeroOp::create(rewriter, loc, rewriter.getI64Type());
     rewriter.replaceOpWithNewOp<WhileLTIntrOp>(createMaskOp, maskType, zero,
                                                adaptor.getOperands()[0]);
     return success();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
index 3dbb93b8a0669..33de12ca17cfc 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
@@ -71,7 +71,7 @@ void replaceOpWithUnrealizedConversion(PatternRewriter &rewriter, TOp op,
                                        TLegalizerCallback callback) {
   replaceOpWithLegalizedOp(rewriter, op, [&](TOp newOp) {
     // Mark our `unrealized_conversion_casts` with a pass label.
-    return rewriter.create<UnrealizedConversionCastOp>(
+    return UnrealizedConversionCastOp::create(rewriter,
         op.getLoc(), TypeRange{op.getResult().getType()},
         ValueRange{callback(newOp)},
         NamedAttribute(rewriter.getStringAttr(kSVELegalizerTag),
@@ -239,7 +239,7 @@ struct LegalizeSVEMaskStoreConversion
 
     auto legalMaskType = widenScalableMaskTypeToSvbool(
         llvm::cast<VectorType>(valueToStore.getType()));
-    auto convertToSvbool = rewriter.create<arm_sve::ConvertToSvboolOp>(
+    auto convertToSvbool = arm_sve::ConvertToSvboolOp::create(rewriter,
         loc, legalMaskType, valueToStore);
     // Replace this store with a conversion to a storable svbool mask [1],
     // followed by a wider store.
@@ -290,7 +290,7 @@ struct LegalizeSVEMaskLoadConversion : public OpRewritePattern<memref::LoadOp> {
     replaceOpWithLegalizedOp(rewriter, loadOp, [&](memref::LoadOp newLoadOp) {
       newLoadOp.setMemRef(*legalMemref);
       newLoadOp.getResult().setType(legalMaskType);
-      return rewriter.create<arm_sve::ConvertFromSvboolOp>(
+      return arm_sve::ConvertFromSvboolOp::create(rewriter,
           loc, loadedMask.getType(), newLoadOp);
     });
 
@@ -408,7 +408,7 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
       reassoc.back().push_back(i);
     if (!memref::CollapseShapeOp::isGuaranteedCollapsible(memTy, reassoc))
       return failure();
-    Value collapsedMem = rewriter.create<memref::CollapseShapeOp>(
+    Value collapsedMem = memref::CollapseShapeOp::create(rewriter,
         readOp.getLoc(), readOp.getBase(), reassoc);
 
     // Get a vector type with collapsed trailing dimensions.
@@ -424,13 +424,13 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
     auto indices = readOp.getIndices().drop_back(numCollapseDims - 1);
 
     // Create the new `transfer_read`.
-    auto newReadOp = rewriter.create<vector::TransferReadOp>(
+    auto newReadOp = vector::TransferReadOp::create(rewriter,
         readOp.getLoc(), collapsedVT, collapsedMem, indices,
         readOp.getPadding(),
         ArrayRef<bool>(origInBounds).drop_back(numCollapseDims - 1));
 
     // Cast back to the original vector type.
-    auto toOrigShape = rewriter.create<vector::ShapeCastOp>(readOp.getLoc(),
+    auto toOrigShape = vector::ShapeCastOp::create(rewriter, readOp.getLoc(),
                                                             origVT, newReadOp);
 
     rewriter.replaceOp(readOp, toOrigShape);
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
index bd051b100a91b..0dfd58c582dfd 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
@@ -87,15 +87,15 @@ Value createMMLA(PatternRewriter &rewriter, MMLA op, Location loc,
                  mlir::VectorType accType, Value acc, Value lhs, Value rhs) {
   switch (op) {
   case MMLA::Signed:
-    return rewriter.create<arm_sve::SmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::SmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::Unsigned:
-    return rewriter.create<arm_sve::UmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::UmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::Mixed:
-    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, lhs, rhs);
+    return arm_sve::UsmmlaOp::create(rewriter, loc, accType, acc, lhs, rhs);
   case MMLA::MixedSwapped:
     // The accumulator comes transposed and the result will be transposed
     // later, so all we have to do here is swap the operands.
-    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, rhs, lhs);
+    return arm_sve::UsmmlaOp::create(rewriter, loc, accType, acc, rhs, lhs);
   }
 }
 
@@ -233,25 +233,25 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> lhsTile;
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the LHS tile.
-      auto r0 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+      auto r0 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
                                                    ArrayRef<int64_t>{i});
-      auto r1 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+      auto r1 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
                                                    ArrayRef<int64_t>{i + 1});
       // Concatenate to obtain a 16 x i8 flattened sub-tile.
-      auto t = rewriter.create<vector::ShuffleOp>(
+      auto t = vector::ShuffleOp::create(rewriter,
           loc, r0, r1,
           llvm::ArrayRef<int64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
                                   14, 15});
       // Turn it into a scalable vector.
-      auto s = rewriter.create<vector::ScalableInsertOp>(
-          loc, t, rewriter.create<ub::PoisonOp>(loc, nxv16i8), 0);
+      auto s = vector::ScalableInsertOp::create(rewriter,
+          loc, t, ub::PoisonOp::create(rewriter, loc, nxv16i8), 0);
       // Replicate the sub-tile VSCALE times to fill the entire vector.
-      auto r = rewriter.create<arm_sve::DupQLaneOp>(loc, s, 0);
+      auto r = arm_sve::DupQLaneOp::create(rewriter, loc, s, 0);
       lhsTile.push_back(r);
     }
 
     // "Flatten" the RHS tile from <[N]x8> to <[8*N]>.
-    auto rhs = rewriter.create<vector::ShapeCastOp>(
+    auto rhs = vector::ShapeCastOp::create(rewriter,
         maybeRhs->getLoc(),
         VectorType::get(/*shape=*/8 * N, rewriter.getI8Type(),
                         /*scalableDims=*/{true}),
@@ -261,7 +261,7 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> rhsTile;
     for (int64_t j = 0; j < N; j += 2)
       rhsTile.push_back(
-          rewriter.create<vector::ScalableExtractOp>(loc, nxv16i8, rhs, j * 8));
+          vector::ScalableExtractOp::create(rewriter, loc, nxv16i8, rhs, j * 8));
 
     // Handy types for packing/unpacking of the accumulator tile.
     auto accRowTy = VectorType::get(/*shape=*/N, rewriter.getI32Type(),
@@ -277,33 +277,33 @@ class LowerContractionToSVEI8MMPattern
     SmallVector<Value> accTile;
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the accumulator tile.
-      auto r0 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+      auto r0 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
                                                    ArrayRef<int64_t>{i});
-      auto r1 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+      auto r1 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
                                                    ArrayRef<int64_t>{i + 1});
       Value accTileVec;
       if (mmlaOp == MMLA::MixedSwapped) {
         // We need to swap the positions of the LHS and RHS (since we don't have
         // a signed * unsigned operation), but then each individual 2x2 tile of
         // the acumulator and (later) the result need to be transposed.
-        accTileVec = rewriter.create<vector::InterleaveOp>(loc, r0, r1);
+        accTileVec = vector::InterleaveOp::create(rewriter, loc, r0, r1);
       } else {
         // Bitcast them to 64-bit elements, so subsequent
         // interleave/deinterleave work on pairs of 32-bit numbers.
-        auto r0I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r0);
-        auto r1I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r1);
+        auto r0I64 = vector::BitCastOp::create(rewriter, loc, accRow64Ty, r0);
+        auto r1I64 = vector::BitCastOp::create(rewriter, loc, accRow64Ty, r1);
 
         // Interleave the rows, effectively flattening each 2x2 tile into 4
         // consecutive elements.
-        auto intrI64 = rewriter.create<vector::InterleaveOp>(loc, r0I64, r1I64);
+        auto intrI64 = vector::InterleaveOp::create(rewriter, loc, r0I64, r1I64);
 
         // Bitcast back to 32-bit elements.
         accTileVec =
-            rewriter.create<vector::BitCastOp>(loc, accRowX2Ty, intrI64);
+            vector::BitCastOp::create(rewriter, loc, accRowX2Ty, intrI64);
       }
       // Extract ACC sub-tiles.
       for (int64_t j = 0; j < N; j += 2)
-        accTile.push_back(rewriter.create<vector::ScalableExtractOp>(
+        accTile.push_back(vector::ScalableExtractOp::create(rewriter,
             loc, nxv4i32, accTileVec, j * 2));
     }
 
@@ -317,12 +317,12 @@ class LowerContractionToSVEI8MMPattern
       }
 
     // Unpack the OUT sub-tiles and insert into the result.
-    Value result = rewriter.create<ub::PoisonOp>(loc, op.getResultType());
+    Value result = ub::PoisonOp::create(rewriter, loc, op.getResultType());
     for (int64_t i = 0; i < M / 2; ++i) {
       // Collect a number of sub-tiles in a row.
-      Value row = rewriter.create<ub::PoisonOp>(loc, accRowX2Ty);
+      Value row = ub::PoisonOp::create(rewriter, loc, accRowX2Ty);
       for (int64_t j = 0; j < N / 2; ++j)
-        row = rewriter.create<vector::ScalableInsertOp>(
+        row = vector::ScalableInsertOp::create(rewriter,
             loc, outTile[i * N / 2 + j], row, j * 4);
 
       // Unpack the row to obtain two rows of the output. If we have the out
@@ -331,22 +331,22 @@ class LowerContractionToSVEI8MMPattern
       // Otherwise, the interleave is by pairs.
       Value out0, out1;
       if (mmlaOp == MMLA::MixedSwapped) {
-        auto tmp = rewriter.create<vector::DeinterleaveOp>(loc, row);
+        auto tmp = vector::DeinterleaveOp::create(rewriter, loc, row);
         out0 = tmp.getRes1();
         out1 = tmp.getRes2();
       } else {
         // Deinterleave by pairs.
-        auto row64 = rewriter.create<vector::BitCastOp>(loc, accRowX264Ty, row);
-        auto deintr64 = rewriter.create<vector::DeinterleaveOp>(loc, row64);
+        auto row64 = vector::BitCastOp::create(rewriter, loc, accRowX264Ty, row);
+        auto deintr64 = vector::DeinterleaveOp::create(rewriter, loc, row64);
 
         // Bitcast back into 32-bit elements and insert into the result.
-        out0 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+        out0 = vector::BitCastOp::create(rewriter, loc, accRowTy,
                                                   deintr64.getRes1());
-        out1 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+        out1 = vector::BitCastOp::create(rewriter, loc, accRowTy,
                                                   deintr64.getRes2());
       }
-      result = rewriter.create<vector::InsertOp>(loc, out0, result, i * 2);
-      result = rewriter.create<vector::InsertOp>(loc, out1, result, i * 2 + 1);
+      result = vector::InsertOp::create(rewriter, loc, out0, result, i * 2);
+      result = vector::InsertOp::create(rewriter, loc, out1, result, i * 2 + 1);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index 08a57db6853a0..dc7b07d911c17 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -97,7 +97,7 @@ void ExecuteOp::build(OpBuilder &builder, OperationState &result,
   // expected result is empty. Otherwise, leave this to the caller
   // because we don't know which values to return from the execute op.
   if (resultTypes.empty() && !bodyBuilder) {
-    builder.create<async::YieldOp>(result.location, ValueRange());
+    async::YieldOp::create(builder, result.location, ValueRange());
   } else if (bodyBuilder) {
     bodyBuilder(builder, result.location, bodyBlock->getArguments());
   }
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index 27fa92cee79c2..447411ad989ba 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -191,8 +191,8 @@ static SmallVector<Value> delinearize(ImplicitLocOpBuilder &b, Value index,
   assert(!tripCounts.empty() && "tripCounts must be not empty");
 
   for (ssize_t i = tripCounts.size() - 1; i >= 0; --i) {
-    coords[i] = b.create<arith::RemSIOp>(index, tripCounts[i]);
-    index = b.create<arith::DivSIOp>(index, tripCounts[i]);
+    coords[i] = arith::RemSIOp::create(b, index, tripCounts[i]);
+    index = arith::DivSIOp::create(b, index, tripCounts[i]);
   }
 
   return coords;
@@ -276,15 +276,15 @@ static ParallelComputeFunction createParallelComputeFunction(
   BlockArgument blockSize = args.blockSize();
 
   // Constants used below.
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Materialize known constants as constant operation in the function body.
   auto values = [&](ArrayRef<BlockArgument> args, ArrayRef<IntegerAttr> attrs) {
     return llvm::to_vector(
         llvm::map_range(llvm::zip(args, attrs), [&](auto tuple) -> Value {
           if (IntegerAttr attr = std::get<1>(tuple))
-            return b.create<arith::ConstantOp>(attr);
+            return arith::ConstantOp::create(b, attr);
           return std::get<0>(tuple);
         }));
   };
@@ -303,17 +303,17 @@ static ParallelComputeFunction createParallelComputeFunction(
   // one-dimensional iteration space.
   Value tripCount = tripCounts[0];
   for (unsigned i = 1; i < tripCounts.size(); ++i)
-    tripCount = b.create<arith::MulIOp>(tripCount, tripCounts[i]);
+    tripCount = arith::MulIOp::create(b, tripCount, tripCounts[i]);
 
   // Find one-dimensional iteration bounds: [blockFirstIndex, blockLastIndex]:
   //   blockFirstIndex = blockIndex * blockSize
-  Value blockFirstIndex = b.create<arith::MulIOp>(blockIndex, blockSize);
+  Value blockFirstIndex = arith::MulIOp::create(b, blockIndex, blockSize);
 
   // The last one-dimensional index in the block defined by the `blockIndex`:
   //   blockLastIndex = min(blockFirstIndex + blockSize, tripCount) - 1
-  Value blockEnd0 = b.create<arith::AddIOp>(blockFirstIndex, blockSize);
-  Value blockEnd1 = b.create<arith::MinSIOp>(blockEnd0, tripCount);
-  Value blockLastIndex = b.create<arith::SubIOp>(blockEnd1, c1);
+  Value blockEnd0 = arith::AddIOp::create(b, blockFirstIndex, blockSize);
+  Value blockEnd1 = arith::MinSIOp::create(b, blockEnd0, tripCount);
+  Value blockLastIndex = arith::SubIOp::create(b, blockEnd1, c1);
 
   // Convert one-dimensional indices to multi-dimensional coordinates.
   auto blockFirstCoord = delinearize(b, blockFirstIndex, tripCounts);
@@ -326,7 +326,7 @@ static ParallelComputeFunction createParallelComputeFunction(
   // dimension when inner compute dimension contains multiple blocks.
   SmallVector<Value> blockEndCoord(op.getNumLoops());
   for (size_t i = 0; i < blockLastCoord.size(); ++i)
-    blockEndCoord[i] = b.create<arith::AddIOp>(blockLastCoord[i], c1);
+    blockEndCoord[i] = arith::AddIOp::create(b, blockLastCoord[i], c1);
 
   // Construct a loop nest out of scf.for operations that will iterate over
   // all coordinates in [blockFirstCoord, blockLastCoord] range.
@@ -369,20 +369,20 @@ static ParallelComputeFunction createParallelComputeFunction(
       ImplicitLocOpBuilder b(loc, nestedBuilder);
 
       // Compute induction variable for `loopIdx`.
-      computeBlockInductionVars[loopIdx] = b.create<arith::AddIOp>(
-          lowerBounds[loopIdx], b.create<arith::MulIOp>(iv, steps[loopIdx]));
+      computeBlockInductionVars[loopIdx] = arith::AddIOp::create(b,
+          lowerBounds[loopIdx], arith::MulIOp::create(b, iv, steps[loopIdx]));
 
       // Check if we are inside first or last iteration of the loop.
-      isBlockFirstCoord[loopIdx] = b.create<arith::CmpIOp>(
+      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(b,
           arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
-      isBlockLastCoord[loopIdx] = b.create<arith::CmpIOp>(
+      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(b,
           arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
 
       // Check if the previous loop is in its first or last iteration.
       if (loopIdx > 0) {
-        isBlockFirstCoord[loopIdx] = b.create<arith::AndIOp>(
+        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(b,
             isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
-        isBlockLastCoord[loopIdx] = b.create<arith::AndIOp>(
+        isBlockLastCoord[loopIdx] = arith::AndIOp::create(b,
             isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
       }
 
@@ -391,24 +391,24 @@ static ParallelComputeFunction createParallelComputeFunction(
         if (loopIdx + 1 >= op.getNumLoops() - numBlockAlignedInnerLoops) {
           // For block aligned loops we always iterate starting from 0 up to
           // the loop trip counts.
-          b.create<scf::ForOp>(c0, tripCounts[loopIdx + 1], c1, ValueRange(),
+          scf::ForOp::create(b, c0, tripCounts[loopIdx + 1], c1, ValueRange(),
                                workLoopBuilder(loopIdx + 1));
 
         } else {
           // Select nested loop lower/upper bounds depending on our position in
           // the multi-dimensional iteration space.
-          auto lb = b.create<arith::SelectOp>(isBlockFirstCoord[loopIdx],
+          auto lb = arith::SelectOp::create(b, isBlockFirstCoord[loopIdx],
                                               blockFirstCoord[loopIdx + 1], c0);
 
-          auto ub = b.create<arith::SelectOp>(isBlockLastCoord[loopIdx],
+          auto ub = arith::SelectOp::create(b, isBlockLastCoord[loopIdx],
                                               blockEndCoord[loopIdx + 1],
                                               tripCounts[loopIdx + 1]);
 
-          b.create<scf::ForOp>(lb, ub, c1, ValueRange(),
+          scf::ForOp::create(b, lb, ub, c1, ValueRange(),
                                workLoopBuilder(loopIdx + 1));
         }
 
-        b.create<scf::YieldOp>(loc);
+        scf::YieldOp::create(b, loc);
         return;
       }
 
@@ -419,13 +419,13 @@ static ParallelComputeFunction createParallelComputeFunction(
 
       for (auto &bodyOp : op.getRegion().front().without_terminator())
         b.clone(bodyOp, mapping);
-      b.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(b, loc);
     };
   };
 
-  b.create<scf::ForOp>(blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
+  scf::ForOp::create(b, blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
                        workLoopBuilder(0));
-  b.create<func::ReturnOp>(ValueRange());
+  func::ReturnOp::create(b, ValueRange());
 
   return {op.getNumLoops(), func, std::move(computeFuncType.captures)};
 }
@@ -485,8 +485,8 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   b.setInsertionPointToEnd(block);
 
   Type indexTy = b.getIndexType();
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
-  Value c2 = b.create<arith::ConstantIndexOp>(2);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
+  Value c2 = arith::ConstantIndexOp::create(b, 2);
 
   // Get the async group that will track async dispatch completion.
   Value group = block->getArgument(0);
@@ -501,7 +501,7 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   SmallVector<Location> locations = {loc, loc};
 
   // Create a recursive dispatch loop.
-  scf::WhileOp whileOp = b.create<scf::WhileOp>(types, operands);
+  scf::WhileOp whileOp = scf::WhileOp::create(b, types, operands);
   Block *before = b.createBlock(&whileOp.getBefore(), {}, types, locations);
   Block *after = b.createBlock(&whileOp.getAfter(), {}, types, locations);
 
@@ -511,10 +511,10 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
     b.setInsertionPointToEnd(before);
     Value start = before->getArgument(0);
     Value end = before->getArgument(1);
-    Value distance = b.create<arith::SubIOp>(end, start);
+    Value distance = arith::SubIOp::create(b, end, start);
     Value dispatch =
-        b.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, distance, c1);
-    b.create<scf::ConditionOp>(dispatch, before->getArguments());
+        arith::CmpIOp::create(b, arith::CmpIPredicate::sgt, distance, c1);
+    scf::ConditionOp::create(b, dispatch, before->getArguments());
   }
 
   // Setup the async dispatch loop body: recursively call dispatch function
@@ -523,9 +523,9 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
     b.setInsertionPointToEnd(after);
     Value start = after->getArgument(0);
     Value end = after->getArgument(1);
-    Value distance = b.create<arith::SubIOp>(end, start);
-    Value halfDistance = b.create<arith::DivSIOp>(distance, c2);
-    Value midIndex = b.create<arith::AddIOp>(start, halfDistance);
+    Value distance = arith::SubIOp::create(b, end, start);
+    Value halfDistance = arith::DivSIOp::create(b, distance, c2);
+    Value midIndex = arith::AddIOp::create(b, start, halfDistance);
 
     // Call parallel compute function inside the async.execute region.
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
@@ -536,16 +536,16 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
       operands[1] = midIndex;
       operands[2] = end;
 
-      executeBuilder.create<func::CallOp>(executeLoc, func.getSymName(),
+      func::CallOp::create(executeBuilder, executeLoc, func.getSymName(),
                                           func.getResultTypes(), operands);
-      executeBuilder.create<async::YieldOp>(executeLoc, ValueRange());
+      async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to dispatch half of the block range.
-    auto execute = b.create<ExecuteOp>(TypeRange(), ValueRange(), ValueRange(),
+    auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
                                        executeBodyBuilder);
-    b.create<AddToGroupOp>(indexTy, execute.getToken(), group);
-    b.create<scf::YieldOp>(ValueRange({start, midIndex}));
+    AddToGroupOp::create(b, indexTy, execute.getToken(), group);
+    scf::YieldOp::create(b, ValueRange({start, midIndex}));
   }
 
   // After dispatching async operations to process the tail of the block range
@@ -557,10 +557,10 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   SmallVector<Value> computeFuncOperands = {blockStart};
   computeFuncOperands.append(forwardedInputs.begin(), forwardedInputs.end());
 
-  b.create<func::CallOp>(computeFunc.func.getSymName(),
+  func::CallOp::create(b, computeFunc.func.getSymName(),
                          computeFunc.func.getResultTypes(),
                          computeFuncOperands);
-  b.create<func::ReturnOp>(ValueRange());
+  func::ReturnOp::create(b, ValueRange());
 
   return func;
 }
@@ -578,8 +578,8 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
   func::FuncOp asyncDispatchFunction =
       createAsyncDispatchFunction(parallelComputeFunction, rewriter);
 
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Appends operands shared by async dispatch and parallel compute functions to
   // the given operands vector.
@@ -595,7 +595,7 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
   // completely. If this will be known statically, then canonicalization will
   // erase async group operations.
   Value isSingleBlock =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, blockCount, c1);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, blockCount, c1);
 
   auto syncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
     ImplicitLocOpBuilder b(loc, nestedBuilder);
@@ -604,10 +604,10 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     SmallVector<Value> operands = {c0, blockSize};
     appendBlockComputeOperands(operands);
 
-    b.create<func::CallOp>(parallelComputeFunction.func.getSymName(),
+    func::CallOp::create(b, parallelComputeFunction.func.getSymName(),
                            parallelComputeFunction.func.getResultTypes(),
                            operands);
-    b.create<scf::YieldOp>();
+    scf::YieldOp::create(b, );
   };
 
   auto asyncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -616,24 +616,24 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     // Create an async.group to wait on all async tokens from the concurrent
     // execution of multiple parallel compute function. First block will be
     // executed synchronously in the caller thread.
-    Value groupSize = b.create<arith::SubIOp>(blockCount, c1);
-    Value group = b.create<CreateGroupOp>(GroupType::get(ctx), groupSize);
+    Value groupSize = arith::SubIOp::create(b, blockCount, c1);
+    Value group = CreateGroupOp::create(b, GroupType::get(ctx), groupSize);
 
     // Launch async dispatch function for [0, blockCount) range.
     SmallVector<Value> operands = {group, c0, blockCount, blockSize};
     appendBlockComputeOperands(operands);
 
-    b.create<func::CallOp>(asyncDispatchFunction.getSymName(),
+    func::CallOp::create(b, asyncDispatchFunction.getSymName(),
                            asyncDispatchFunction.getResultTypes(), operands);
 
     // Wait for the completion of all parallel compute operations.
-    b.create<AwaitAllOp>(group);
+    AwaitAllOp::create(b, group);
 
-    b.create<scf::YieldOp>();
+    scf::YieldOp::create(b, );
   };
 
   // Dispatch either single block compute function, or launch async dispatch.
-  b.create<scf::IfOp>(isSingleBlock, syncDispatch, asyncDispatch);
+  scf::IfOp::create(b, isSingleBlock, syncDispatch, asyncDispatch);
 }
 
 // Dispatch parallel compute functions by submitting all async compute tasks
@@ -647,14 +647,14 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
 
   func::FuncOp compute = parallelComputeFunction.func;
 
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
-  Value c1 = b.create<arith::ConstantIndexOp>(1);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
+  Value c1 = arith::ConstantIndexOp::create(b, 1);
 
   // Create an async.group to wait on all async tokens from the concurrent
   // execution of multiple parallel compute function. First block will be
   // executed synchronously in the caller thread.
-  Value groupSize = b.create<arith::SubIOp>(blockCount, c1);
-  Value group = b.create<CreateGroupOp>(GroupType::get(ctx), groupSize);
+  Value groupSize = arith::SubIOp::create(b, blockCount, c1);
+  Value group = CreateGroupOp::create(b, GroupType::get(ctx), groupSize);
 
   // Call parallel compute function for all blocks.
   using LoopBodyBuilder =
@@ -681,28 +681,28 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     // Call parallel compute function inside the async.execute region.
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
                                   Location executeLoc, ValueRange executeArgs) {
-      executeBuilder.create<func::CallOp>(executeLoc, compute.getSymName(),
+      func::CallOp::create(executeBuilder, executeLoc, compute.getSymName(),
                                           compute.getResultTypes(),
                                           computeFuncOperands(iv));
-      executeBuilder.create<async::YieldOp>(executeLoc, ValueRange());
+      async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to launch parallel computate function.
-    auto execute = b.create<ExecuteOp>(TypeRange(), ValueRange(), ValueRange(),
+    auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
                                        executeBodyBuilder);
-    b.create<AddToGroupOp>(rewriter.getIndexType(), execute.getToken(), group);
-    b.create<scf::YieldOp>();
+    AddToGroupOp::create(b, rewriter.getIndexType(), execute.getToken(), group);
+    scf::YieldOp::create(b, );
   };
 
   // Iterate over all compute blocks and launch parallel compute operations.
-  b.create<scf::ForOp>(c1, blockCount, c1, ValueRange(), loopBuilder);
+  scf::ForOp::create(b, c1, blockCount, c1, ValueRange(), loopBuilder);
 
   // Call parallel compute function for the first block in the caller thread.
-  b.create<func::CallOp>(compute.getSymName(), compute.getResultTypes(),
+  func::CallOp::create(b, compute.getSymName(), compute.getResultTypes(),
                          computeFuncOperands(c0));
 
   // Wait for the completion of all async compute operations.
-  b.create<AwaitAllOp>(group);
+  AwaitAllOp::create(b, group);
 }
 
 LogicalResult
@@ -738,17 +738,17 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
   // for the scf.parallel operation.
   Value tripCount = tripCounts[0];
   for (size_t i = 1; i < tripCounts.size(); ++i)
-    tripCount = b.create<arith::MulIOp>(tripCount, tripCounts[i]);
+    tripCount = arith::MulIOp::create(b, tripCount, tripCounts[i]);
 
   // Short circuit no-op parallel loops (zero iterations) that can arise from
   // the memrefs with dynamic dimension(s) equal to zero.
-  Value c0 = b.create<arith::ConstantIndexOp>(0);
+  Value c0 = arith::ConstantIndexOp::create(b, 0);
   Value isZeroIterations =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, tripCount, c0);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, tripCount, c0);
 
   // Do absolutely nothing if the trip count is zero.
   auto noOp = [&](OpBuilder &nestedBuilder, Location loc) {
-    nestedBuilder.create<scf::YieldOp>(loc);
+    scf::YieldOp::create(nestedBuilder, loc);
   };
 
   // Compute the parallel block size and dispatch concurrent tasks computing
@@ -798,9 +798,9 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
     Value numWorkerThreadsVal;
     if (numWorkerThreads >= 0)
-      numWorkerThreadsVal = b.create<arith::ConstantIndexOp>(numWorkerThreads);
+      numWorkerThreadsVal = arith::ConstantIndexOp::create(b, numWorkerThreads);
     else
-      numWorkerThreadsVal = b.create<async::RuntimeNumWorkerThreadsOp>();
+      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b, );
 
     // With large number of threads the value of creating many compute blocks
     // is reduced because the problem typically becomes memory bound. For this
@@ -819,38 +819,38 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
         {4, 4.0f}, {8, 2.0f}, {16, 1.0f}, {32, 0.8f}, {64, 0.6f}};
     const float initialOvershardingFactor = 8.0f;
 
-    Value scalingFactor = b.create<arith::ConstantFloatOp>(
+    Value scalingFactor = arith::ConstantFloatOp::create(b,
         b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
     for (const std::pair<int, float> &p : overshardingBrackets) {
-      Value bracketBegin = b.create<arith::ConstantIndexOp>(p.first);
-      Value inBracket = b.create<arith::CmpIOp>(
+      Value bracketBegin = arith::ConstantIndexOp::create(b, p.first);
+      Value inBracket = arith::CmpIOp::create(b,
           arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
-      Value bracketScalingFactor = b.create<arith::ConstantFloatOp>(
+      Value bracketScalingFactor = arith::ConstantFloatOp::create(b,
           b.getF32Type(), llvm::APFloat(p.second));
-      scalingFactor = b.create<arith::SelectOp>(inBracket, bracketScalingFactor,
+      scalingFactor = arith::SelectOp::create(b, inBracket, bracketScalingFactor,
                                                 scalingFactor);
     }
     Value numWorkersIndex =
-        b.create<arith::IndexCastOp>(b.getI32Type(), numWorkerThreadsVal);
+        arith::IndexCastOp::create(b, b.getI32Type(), numWorkerThreadsVal);
     Value numWorkersFloat =
-        b.create<arith::SIToFPOp>(b.getF32Type(), numWorkersIndex);
+        arith::SIToFPOp::create(b, b.getF32Type(), numWorkersIndex);
     Value scaledNumWorkers =
-        b.create<arith::MulFOp>(scalingFactor, numWorkersFloat);
+        arith::MulFOp::create(b, scalingFactor, numWorkersFloat);
     Value scaledNumInt =
-        b.create<arith::FPToSIOp>(b.getI32Type(), scaledNumWorkers);
+        arith::FPToSIOp::create(b, b.getI32Type(), scaledNumWorkers);
     Value scaledWorkers =
-        b.create<arith::IndexCastOp>(b.getIndexType(), scaledNumInt);
+        arith::IndexCastOp::create(b, b.getIndexType(), scaledNumInt);
 
-    Value maxComputeBlocks = b.create<arith::MaxSIOp>(
-        b.create<arith::ConstantIndexOp>(1), scaledWorkers);
+    Value maxComputeBlocks = arith::MaxSIOp::create(b,
+        arith::ConstantIndexOp::create(b, 1), scaledWorkers);
 
     // Compute parallel block size from the parallel problem size:
     //   blockSize = min(tripCount,
     //                   max(ceil_div(tripCount, maxComputeBlocks),
     //                       minTaskSize))
-    Value bs0 = b.create<arith::CeilDivSIOp>(tripCount, maxComputeBlocks);
-    Value bs1 = b.create<arith::MaxSIOp>(bs0, minTaskSize);
-    Value blockSize = b.create<arith::MinSIOp>(tripCount, bs1);
+    Value bs0 = arith::CeilDivSIOp::create(b, tripCount, maxComputeBlocks);
+    Value bs1 = arith::MaxSIOp::create(b, bs0, minTaskSize);
+    Value blockSize = arith::MinSIOp::create(b, tripCount, bs1);
 
     // Dispatch parallel compute function using async recursive work splitting,
     // or by submitting compute task sequentially from a caller thread.
@@ -860,7 +860,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     // the parallel operation body for a subset of iteration space.
 
     // Compute the number of parallel compute blocks.
-    Value blockCount = b.create<arith::CeilDivSIOp>(tripCount, blockSize);
+    Value blockCount = arith::CeilDivSIOp::create(b, tripCount, blockSize);
 
     // Dispatch parallel compute function without hints to unroll inner loops.
     auto dispatchDefault = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -869,7 +869,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       doDispatch(b, rewriter, compute, op, blockSize, blockCount, tripCounts);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     };
 
     // Dispatch parallel compute function with hints for unrolling inner loops.
@@ -880,34 +880,34 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       // Align the block size to be a multiple of the statically known
       // number of iterations in the inner loops.
-      Value numIters = b.create<arith::ConstantIndexOp>(
+      Value numIters = arith::ConstantIndexOp::create(b,
           numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value alignedBlockSize = b.create<arith::MulIOp>(
-          b.create<arith::CeilDivSIOp>(blockSize, numIters), numIters);
+      Value alignedBlockSize = arith::MulIOp::create(b,
+          arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
       doDispatch(b, rewriter, compute, op, alignedBlockSize, blockCount,
                  tripCounts);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     };
 
     // Dispatch to block aligned compute function only if the computed block
     // size is larger than the number of iterations in the unrollable inner
     // loops, because otherwise it can reduce the available parallelism.
     if (numUnrollableLoops > 0) {
-      Value numIters = b.create<arith::ConstantIndexOp>(
+      Value numIters = arith::ConstantIndexOp::create(b,
           numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value useBlockAlignedComputeFn = b.create<arith::CmpIOp>(
+      Value useBlockAlignedComputeFn = arith::CmpIOp::create(b,
           arith::CmpIPredicate::sge, blockSize, numIters);
 
-      b.create<scf::IfOp>(useBlockAlignedComputeFn, dispatchBlockAligned,
+      scf::IfOp::create(b, useBlockAlignedComputeFn, dispatchBlockAligned,
                           dispatchDefault);
-      b.create<scf::YieldOp>();
+      scf::YieldOp::create(b, );
     } else {
       dispatchDefault(b, loc);
     }
   };
 
   // Replace the `scf.parallel` operation with the parallel compute function.
-  b.create<scf::IfOp>(isZeroIterations, noOp, dispatch);
+  scf::IfOp::create(b, isZeroIterations, noOp, dispatch);
 
   // Parallel operation was replaced with a block iteration loop.
   rewriter.eraseOp(op);
@@ -922,7 +922,7 @@ void AsyncParallelForPass::runOnOperation() {
   populateAsyncParallelForPatterns(
       patterns, asyncDispatch, numWorkerThreads,
       [&](ImplicitLocOpBuilder builder, scf::ParallelOp op) {
-        return builder.create<arith::ConstantIndexOp>(minTaskSize);
+        return arith::ConstantIndexOp::create(builder, minTaskSize);
       });
   if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
     signalPassFailure();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
index 0da9b3a730f85..25ce0fea50581 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
@@ -48,7 +48,7 @@ static LogicalResult dropRefIfNoUses(Value value, unsigned count = 1) {
   else
     b.setInsertionPointToStart(value.getParentBlock());
 
-  b.create<RuntimeDropRefOp>(value.getLoc(), value, b.getI64IntegerAttr(1));
+  RuntimeDropRefOp::create(b, value.getLoc(), value, b.getI64IntegerAttr(1));
   return success();
 }
 
@@ -309,7 +309,7 @@ LogicalResult AsyncRuntimeRefCountingPass::addDropRefAfterLastUse(Value value) {
 
     // Add a drop_ref immediately after the last user.
     builder.setInsertionPointAfter(lastUser);
-    builder.create<RuntimeDropRefOp>(loc, value, builder.getI64IntegerAttr(1));
+    RuntimeDropRefOp::create(builder, loc, value, builder.getI64IntegerAttr(1));
   }
 
   return success();
@@ -327,7 +327,7 @@ AsyncRuntimeRefCountingPass::addAddRefBeforeFunctionCall(Value value) {
     // Add a reference before the function call to pass the value at `+1`
     // reference to the function entry block.
     builder.setInsertionPoint(user);
-    builder.create<RuntimeAddRefOp>(loc, value, builder.getI64IntegerAttr(1));
+    RuntimeAddRefOp::create(builder, loc, value, builder.getI64IntegerAttr(1));
   }
 
   return success();
@@ -411,11 +411,11 @@ AsyncRuntimeRefCountingPass::addDropRefInDivergentLivenessSuccessor(
         refCountingBlock = &successor->getParent()->emplaceBlock();
         refCountingBlock->moveBefore(successor);
         OpBuilder builder = OpBuilder::atBlockEnd(refCountingBlock);
-        builder.create<cf::BranchOp>(value.getLoc(), successor);
+        cf::BranchOp::create(builder, value.getLoc(), successor);
       }
 
       OpBuilder builder = OpBuilder::atBlockBegin(refCountingBlock);
-      builder.create<RuntimeDropRefOp>(value.getLoc(), value,
+      RuntimeDropRefOp::create(builder, value.getLoc(), value,
                                        builder.getI64IntegerAttr(1));
 
       // No need to update the terminator operation.
@@ -507,13 +507,13 @@ AsyncRuntimePolicyBasedRefCountingPass::addRefCounting(Value value) {
       // Create `add_ref` operation before the operand owner.
       if (cnt > 0) {
         b.setInsertionPoint(operand.getOwner());
-        b.create<RuntimeAddRefOp>(loc, value, b.getI64IntegerAttr(cnt));
+        RuntimeAddRefOp::create(b, loc, value, b.getI64IntegerAttr(cnt));
       }
 
       // Create `drop_ref` operation after the operand owner.
       if (cnt < 0) {
         b.setInsertionPointAfter(operand.getOwner());
-        b.create<RuntimeDropRefOp>(loc, value, b.getI64IntegerAttr(-cnt));
+        RuntimeDropRefOp::create(b, loc, value, b.getI64IntegerAttr(-cnt));
       }
     }
   }
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index 832d2ec65cf7e..0485ad99cf7d2 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -187,22 +187,22 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
 
   std::optional<Value> retToken;
   if (isStateful)
-    retToken.emplace(builder.create<RuntimeCreateOp>(TokenType::get(ctx)));
+    retToken.emplace(RuntimeCreateOp::create(builder, TokenType::get(ctx)));
 
   llvm::SmallVector<Value, 4> retValues;
   ArrayRef<Type> resValueTypes =
       isStateful ? func.getResultTypes().drop_front() : func.getResultTypes();
   for (auto resType : resValueTypes)
     retValues.emplace_back(
-        builder.create<RuntimeCreateOp>(resType).getResult());
+        RuntimeCreateOp::create(builder, resType).getResult());
 
   // ------------------------------------------------------------------------ //
   // Initialize coroutine: get coroutine id and coroutine handle.
   // ------------------------------------------------------------------------ //
-  auto coroIdOp = builder.create<CoroIdOp>(CoroIdType::get(ctx));
+  auto coroIdOp = CoroIdOp::create(builder, CoroIdType::get(ctx));
   auto coroHdlOp =
-      builder.create<CoroBeginOp>(CoroHandleType::get(ctx), coroIdOp.getId());
-  builder.create<cf::BranchOp>(originalEntryBlock);
+      CoroBeginOp::create(builder, CoroHandleType::get(ctx), coroIdOp.getId());
+  cf::BranchOp::create(builder, originalEntryBlock);
 
   Block *cleanupBlock = func.addBlock();
   Block *cleanupBlockForDestroy = func.addBlock();
@@ -213,10 +213,10 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   // ------------------------------------------------------------------------ //
   auto buildCleanupBlock = [&](Block *cb) {
     builder.setInsertionPointToStart(cb);
-    builder.create<CoroFreeOp>(coroIdOp.getId(), coroHdlOp.getHandle());
+    CoroFreeOp::create(builder, coroIdOp.getId(), coroHdlOp.getHandle());
 
     // Branch into the suspend block.
-    builder.create<cf::BranchOp>(suspendBlock);
+    cf::BranchOp::create(builder, suspendBlock);
   };
   buildCleanupBlock(cleanupBlock);
   buildCleanupBlock(cleanupBlockForDestroy);
@@ -228,7 +228,7 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   builder.setInsertionPointToStart(suspendBlock);
 
   // Mark the end of a coroutine: async.coro.end
-  builder.create<CoroEndOp>(coroHdlOp.getHandle());
+  CoroEndOp::create(builder, coroHdlOp.getHandle());
 
   // Return created optional `async.token` and `async.values` from the suspend
   // block. This will be the return value of a coroutine ramp function.
@@ -236,7 +236,7 @@ static CoroMachinery setupCoroMachinery(func::FuncOp func) {
   if (retToken)
     ret.push_back(*retToken);
   llvm::append_range(ret, retValues);
-  builder.create<func::ReturnOp>(ret);
+  func::ReturnOp::create(builder, ret);
 
   // `async.await` op lowering will create resume blocks for async
   // continuations, and will conditionally branch to cleanup or suspend blocks.
@@ -273,13 +273,13 @@ static Block *setupSetErrorBlock(CoroMachinery &coro) {
 
   // Coroutine set_error block: set error on token and all returned values.
   if (coro.asyncToken)
-    builder.create<RuntimeSetErrorOp>(*coro.asyncToken);
+    RuntimeSetErrorOp::create(builder, *coro.asyncToken);
 
   for (Value retValue : coro.returnValues)
-    builder.create<RuntimeSetErrorOp>(retValue);
+    RuntimeSetErrorOp::create(builder, retValue);
 
   // Branch into the cleanup block.
-  builder.create<cf::BranchOp>(coro.cleanup);
+  cf::BranchOp::create(builder, coro.cleanup);
 
   return *coro.setError;
 }
@@ -334,13 +334,13 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Await on all dependencies before starting to execute the body region.
     for (size_t i = 0; i < numDependencies; ++i)
-      builder.create<AwaitOp>(func.getArgument(i));
+      AwaitOp::create(builder, func.getArgument(i));
 
     // Await on all async value operands and unwrap the payload.
     SmallVector<Value, 4> unwrappedOperands(numOperands);
     for (size_t i = 0; i < numOperands; ++i) {
       Value operand = func.getArgument(numDependencies + i);
-      unwrappedOperands[i] = builder.create<AwaitOp>(loc, operand).getResult();
+      unwrappedOperands[i] = AwaitOp::create(builder, loc, operand).getResult();
     }
 
     // Map from function inputs defined above the execute op to the function
@@ -367,14 +367,14 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Save the coroutine state: async.coro.save
     auto coroSaveOp =
-        builder.create<CoroSaveOp>(CoroStateType::get(ctx), coro.coroHandle);
+        CoroSaveOp::create(builder, CoroStateType::get(ctx), coro.coroHandle);
 
     // Pass coroutine to the runtime to be resumed on a runtime managed
     // thread.
-    builder.create<RuntimeResumeOp>(coro.coroHandle);
+    RuntimeResumeOp::create(builder, coro.coroHandle);
 
     // Add async.coro.suspend as a suspended block terminator.
-    builder.create<CoroSuspendOp>(coroSaveOp.getState(), coro.suspend,
+    CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend,
                                   branch.getDest(), coro.cleanupForDestroy);
 
     branch.erase();
@@ -383,7 +383,7 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
   // Replace the original `async.execute` with a call to outlined function.
   {
     ImplicitLocOpBuilder callBuilder(loc, execute);
-    auto callOutlinedFunc = callBuilder.create<func::CallOp>(
+    auto callOutlinedFunc = func::CallOp::create(callBuilder,
         func.getName(), execute.getResultTypes(), functionInputs.getArrayRef());
     execute.replaceAllUsesWith(callOutlinedFunc.getResults());
     execute.erase();
@@ -452,7 +452,7 @@ class AsyncFuncOpLowering : public OpConversionPattern<async::FuncOp> {
     Location loc = op->getLoc();
 
     auto newFuncOp =
-        rewriter.create<func::FuncOp>(loc, op.getName(), op.getFunctionType());
+        func::FuncOp::create(rewriter, loc, op.getName(), op.getFunctionType());
 
     SymbolTable::setSymbolVisibility(newFuncOp,
                                      SymbolTable::getSymbolVisibility(op));
@@ -522,16 +522,16 @@ class AsyncReturnOpLowering : public OpConversionPattern<async::ReturnOp> {
     for (auto tuple : llvm::zip(adaptor.getOperands(), coro.returnValues)) {
       Value returnValue = std::get<0>(tuple);
       Value asyncValue = std::get<1>(tuple);
-      rewriter.create<RuntimeStoreOp>(loc, returnValue, asyncValue);
-      rewriter.create<RuntimeSetAvailableOp>(loc, asyncValue);
+      RuntimeStoreOp::create(rewriter, loc, returnValue, asyncValue);
+      RuntimeSetAvailableOp::create(rewriter, loc, asyncValue);
     }
 
     if (coro.asyncToken)
       // Switch the coroutine completion token to available state.
-      rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
+      RuntimeSetAvailableOp::create(rewriter, loc, *coro.asyncToken);
 
     rewriter.eraseOp(op);
-    rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    cf::BranchOp::create(rewriter, loc, coro.cleanup);
     return success();
   }
 
@@ -582,15 +582,15 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
     // the async object (token, value or group) to become available.
     if (!isInCoroutine) {
       ImplicitLocOpBuilder builder(loc, rewriter);
-      builder.create<RuntimeAwaitOp>(loc, operand);
+      RuntimeAwaitOp::create(builder, loc, operand);
 
       // Assert that the awaited operands is not in the error state.
-      Value isError = builder.create<RuntimeIsErrorOp>(i1, operand);
-      Value notError = builder.create<arith::XOrIOp>(
-          isError, builder.create<arith::ConstantOp>(
+      Value isError = RuntimeIsErrorOp::create(builder, i1, operand);
+      Value notError = arith::XOrIOp::create(builder,
+          isError, arith::ConstantOp::create(builder,
                        loc, i1, builder.getIntegerAttr(i1, 1)));
 
-      builder.create<cf::AssertOp>(notError,
+      cf::AssertOp::create(builder, notError,
                                    "Awaited async operand is in error state");
     }
 
@@ -606,15 +606,15 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
       // Save the coroutine state and resume on a runtime managed thread when
       // the operand becomes available.
       auto coroSaveOp =
-          builder.create<CoroSaveOp>(CoroStateType::get(ctx), coro.coroHandle);
-      builder.create<RuntimeAwaitAndResumeOp>(operand, coro.coroHandle);
+          CoroSaveOp::create(builder, CoroStateType::get(ctx), coro.coroHandle);
+      RuntimeAwaitAndResumeOp::create(builder, operand, coro.coroHandle);
 
       // Split the entry block before the await operation.
       Block *resume = rewriter.splitBlock(suspended, Block::iterator(op));
 
       // Add async.coro.suspend as a suspended block terminator.
       builder.setInsertionPointToEnd(suspended);
-      builder.create<CoroSuspendOp>(coroSaveOp.getState(), coro.suspend, resume,
+      CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend, resume,
                                     coro.cleanupForDestroy);
 
       // Split the resume block into error checking and continuation.
@@ -622,8 +622,8 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
 
       // Check if the awaited value is in the error state.
       builder.setInsertionPointToStart(resume);
-      auto isError = builder.create<RuntimeIsErrorOp>(loc, i1, operand);
-      builder.create<cf::CondBranchOp>(isError,
+      auto isError = RuntimeIsErrorOp::create(builder, loc, i1, operand);
+      cf::CondBranchOp::create(builder, isError,
                                        /*trueDest=*/setupSetErrorBlock(coro),
                                        /*trueArgs=*/ArrayRef<Value>(),
                                        /*falseDest=*/continuation,
@@ -673,7 +673,7 @@ class AwaitValueOpLowering : public AwaitOpLoweringBase<AwaitOp, ValueType> {
                       ConversionPatternRewriter &rewriter) const override {
     // Load from the async value storage.
     auto valueType = cast<ValueType>(operand.getType()).getValueType();
-    return rewriter.create<RuntimeLoadOp>(op->getLoc(), valueType, operand);
+    return RuntimeLoadOp::create(rewriter, op->getLoc(), valueType, operand);
   }
 };
 
@@ -714,15 +714,15 @@ class YieldOpLowering : public OpConversionPattern<async::YieldOp> {
     for (auto tuple : llvm::zip(adaptor.getOperands(), coro.returnValues)) {
       Value yieldValue = std::get<0>(tuple);
       Value asyncValue = std::get<1>(tuple);
-      rewriter.create<RuntimeStoreOp>(loc, yieldValue, asyncValue);
-      rewriter.create<RuntimeSetAvailableOp>(loc, asyncValue);
+      RuntimeStoreOp::create(rewriter, loc, yieldValue, asyncValue);
+      RuntimeSetAvailableOp::create(rewriter, loc, asyncValue);
     }
 
     if (coro.asyncToken)
       // Switch the coroutine completion token to available state.
-      rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
+      RuntimeSetAvailableOp::create(rewriter, loc, *coro.asyncToken);
 
-    rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    cf::BranchOp::create(rewriter, loc, coro.cleanup);
     rewriter.eraseOp(op);
 
     return success();
@@ -756,7 +756,7 @@ class AssertOpLowering : public OpConversionPattern<cf::AssertOp> {
 
     Block *cont = rewriter.splitBlock(op->getBlock(), Block::iterator(op));
     rewriter.setInsertionPointToEnd(cont->getPrevNode());
-    rewriter.create<cf::CondBranchOp>(loc, adaptor.getArg(),
+    cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(),
                                       /*trueDest=*/cont,
                                       /*trueArgs=*/ArrayRef<Value>(),
                                       /*falseDest=*/setupSetErrorBlock(coro),
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index 2bf326a07783e..a1deab67f6bfa 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -35,7 +35,7 @@ using namespace bufferization;
 //===----------------------------------------------------------------------===//
 
 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
-  return builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
 }
 
 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
@@ -150,7 +150,7 @@ DeallocationState::getMemrefWithUniqueOwnership(OpBuilder &builder,
   // ownerships more intelligently to not end up with an 'Unknown' ownership in
   // the first place.
   auto cloneOp =
-      builder.create<bufferization::CloneOp>(memref.getLoc(), memref);
+      bufferization::CloneOp::create(builder, memref.getLoc(), memref);
   Value condition = buildBoolValue(builder, memref.getLoc(), true);
   Value newMemref = cloneOp.getResult();
   updateOwnership(newMemref, condition);
@@ -196,7 +196,7 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // Simply cast unranked MemRefs to ranked memrefs with 0 dimensions such
     // that we can call extract_strided_metadata on it.
     if (auto unrankedMemRefTy = dyn_cast<UnrankedMemRefType>(memref.getType()))
-      memref = builder.create<memref::ReinterpretCastOp>(
+      memref = memref::ReinterpretCastOp::create(builder,
           loc, memref,
           /*offset=*/builder.getIndexAttr(0),
           /*sizes=*/ArrayRef<OpFoldResult>{},
@@ -207,7 +207,7 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // alloc operation has to be passed to the dealloc operation. Passing
     // subviews, etc. to a dealloc operation is not allowed.
     memrefs.push_back(
-        builder.create<memref::ExtractStridedMetadataOp>(loc, memref)
+        memref::ExtractStridedMetadataOp::create(builder, loc, memref)
             .getResult(0));
     conditions.push_back(ownership.getIndicator());
   }
@@ -296,7 +296,7 @@ FailureOr<Operation *> deallocation_impl::insertDeallocOpForReturnLike(
   if (memrefs.empty() && toRetain.empty())
     return op;
 
-  auto deallocOp = builder.create<bufferization::DeallocOp>(
+  auto deallocOp = bufferization::DeallocOp::create(builder,
       op->getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 8f17a82fabe03..05a1367a6b126 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -170,7 +170,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   if (llvm::isa<RankedTensorType>(shapedValue.getType())) {
     tensor = shapedValue;
   } else if (llvm::isa<MemRefType>(shapedValue.getType())) {
-    tensor = b.create<ToTensorOp>(
+    tensor = ToTensorOp::create(b,
         loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
         shapedValue);
   } else if (llvm::isa<UnrankedTensorType>(shapedValue.getType()) ||
@@ -209,7 +209,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   }
 
   // Create AllocTensorOp.
-  auto allocTensorOp = b.create<AllocTensorOp>(loc, tensorType, dynamicSizes,
+  auto allocTensorOp = AllocTensorOp::create(b, loc, tensorType, dynamicSizes,
                                                copy ? tensor : Value());
 
   // Add 'memory_space' attribute. Not needed if 'copy' operand is specified.
@@ -753,7 +753,7 @@ void bufferization::replaceOpWithBufferizedValues(RewriterBase &rewriter,
       // ToTensorOp. Throughout bufferization, this ToTensorOp will gradually
       // loose all of its users and eventually DCE away.
       rewriter.setInsertionPointAfter(op);
-      replacement = rewriter.create<bufferization::ToTensorOp>(
+      replacement = bufferization::ToTensorOp::create(rewriter,
           replacement.getLoc(), opResult.getType(), replacement);
     }
     replacements.push_back(replacement);
@@ -779,7 +779,7 @@ FailureOr<Value> BufferizationOptions::createAlloc(OpBuilder &b, Location loc,
         .create<memref::AllocOp>(loc, type, dynShape,
                                  b.getI64IntegerAttr(bufferAlignment))
         .getResult();
-  return b.create<memref::AllocOp>(loc, type, dynShape).getResult();
+  return memref::AllocOp::create(b, loc, type, dynShape).getResult();
 }
 
 /// Create a memory copy between two memref buffers.
@@ -788,7 +788,7 @@ LogicalResult BufferizationOptions::createMemCpy(OpBuilder &b, Location loc,
   if (memCpyFn)
     return (*memCpyFn)(b, loc, from, to);
 
-  b.create<memref::CopyOp>(loc, from, to);
+  memref::CopyOp::create(b, loc, from, to);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 875a06546c9f0..e64fee235e50b 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -58,7 +58,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
   // a fix extra conditions in `isGuaranteedCastCompatible`.
   if (memref::CastOp::areCastCompatible(srcType, destType) &&
       isGuaranteedCastCompatible(srcType, destType)) {
-    Value casted = b.create<memref::CastOp>(value.getLoc(), destType, value);
+    Value casted = memref::CastOp::create(b, value.getLoc(), destType, value);
     return casted;
   }
 
@@ -67,7 +67,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
   for (int i = 0; i < destType.getRank(); ++i) {
     if (destType.getShape()[i] != ShapedType::kDynamic)
       continue;
-    Value size = b.create<memref::DimOp>(loc, value, i);
+    Value size = memref::DimOp::create(b, loc, value, i);
     dynamicOperands.push_back(size);
   }
 
@@ -134,10 +134,10 @@ void mlir::bufferization::populateDynamicDimSizes(
   for (int64_t i = 0; i < shapedType.getRank(); ++i) {
     if (shapedType.isDynamicDim(i)) {
       if (llvm::isa<MemRefType>(shapedType)) {
-        dynamicDims.push_back(b.create<memref::DimOp>(loc, shapedValue, i));
+        dynamicDims.push_back(memref::DimOp::create(b, loc, shapedValue, i));
       } else {
         assert(llvm::isa<RankedTensorType>(shapedType) && "expected tensor");
-        dynamicDims.push_back(b.create<tensor::DimOp>(loc, shapedValue, i));
+        dynamicDims.push_back(tensor::DimOp::create(b, loc, shapedValue, i));
       }
     }
   }
@@ -321,7 +321,7 @@ struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
         newShape, op.getType().getElementType(), op.getType().getEncoding());
     if (newType == op.getType())
       return failure();
-    auto newOp = rewriter.create<AllocTensorOp>(
+    auto newOp = AllocTensorOp::create(rewriter,
         op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
@@ -427,7 +427,7 @@ void AllocTensorOp::print(OpAsmPrinter &p) {
 Value AllocTensorOp::getDynamicSize(OpBuilder &b, unsigned idx) {
   assert(isDynamicDim(idx) && "expected dynamic dim");
   if (getCopy())
-    return b.create<tensor::DimOp>(getLoc(), getCopy(), idx);
+    return tensor::DimOp::create(b, getLoc(), getCopy(), idx);
   return getOperand(getIndexOfDynamicSize(idx));
 }
 
@@ -513,7 +513,7 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
     }
 
     if (source.getType() != cloneOp.getType())
-      source = rewriter.create<memref::CastOp>(cloneOp.getLoc(),
+      source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
                                                cloneOp.getType(), source);
     rewriter.replaceOp(cloneOp, source);
     rewriter.eraseOp(redundantDealloc);
@@ -538,7 +538,7 @@ LogicalResult DeallocTensorOp::bufferize(RewriterBase &rewriter,
   FailureOr<Value> buffer = getBuffer(rewriter, getTensor(), options, state);
   if (failed(buffer))
     return failure();
-  rewriter.create<memref::DeallocOp>(getLoc(), *buffer);
+  memref::DeallocOp::create(rewriter, getLoc(), *buffer);
   rewriter.eraseOp(getOperation());
   return success();
 }
@@ -643,7 +643,7 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return builder.create<ToTensorOp>(
+  return ToTensorOp::create(builder,
       loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
       /*restrict=*/true, getWritable());
 }
@@ -806,7 +806,7 @@ struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
       return failure();
     auto memrefType = MemRefType::get(srcTensorType.getShape(),
                                       srcTensorType.getElementType());
-    Value memref = rewriter.create<ToBufferOp>(toBuffer.getLoc(), memrefType,
+    Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
                                                tensorCastOperand.getOperand());
     rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                 memref);
@@ -880,12 +880,12 @@ LogicalResult ToBufferOp::bufferize(RewriterBase &rewriter,
 
 std::optional<Operation *> CloneOp::buildDealloc(OpBuilder &builder,
                                                  Value alloc) {
-  return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
+  return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
       .getOperation();
 }
 
 std::optional<Value> CloneOp::buildClone(OpBuilder &builder, Value alloc) {
-  return builder.create<CloneOp>(alloc.getLoc(), alloc).getResult();
+  return CloneOp::create(builder, alloc.getLoc(), alloc).getResult();
 }
 
 //===----------------------------------------------------------------------===//
@@ -959,7 +959,7 @@ struct DeallocRemoveDuplicateDeallocMemrefs
         Value &newCond = newConditions[memrefToCondition[memref]];
         if (newCond != cond)
           newCond =
-              rewriter.create<arith::OrIOp>(deallocOp.getLoc(), newCond, cond);
+              arith::OrIOp::create(rewriter, deallocOp.getLoc(), newCond, cond);
       } else {
         memrefToCondition.insert({memref, newConditions.size()});
         newMemrefs.push_back(memref);
@@ -1014,7 +1014,7 @@ struct DeallocRemoveDuplicateRetainedMemrefs
     // We need to create a new op because the number of results is always the
     // same as the number of condition operands.
     auto newDeallocOp =
-        rewriter.create<DeallocOp>(deallocOp.getLoc(), deallocOp.getMemrefs(),
+        DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
                                    deallocOp.getConditions(), newRetained);
     SmallVector<Value> replacements(
         llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
@@ -1036,7 +1036,7 @@ struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     if (deallocOp.getMemrefs().empty()) {
-      Value constFalse = rewriter.create<arith::ConstantOp>(
+      Value constFalse = arith::ConstantOp::create(rewriter,
           deallocOp.getLoc(), rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
diff --git a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
index db1eb20512033..7f495b0ac164c 100644
--- a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
@@ -70,12 +70,12 @@ transform::OneShotBufferizeOp::apply(transform::TransformRewriter &rewriter,
         *getFunctionBoundaryTypeConversion());
   if (getMemcpyOp() == "memref.copy") {
     options.memCpyFn = [](OpBuilder &b, Location loc, Value from, Value to) {
-      b.create<memref::CopyOp>(loc, from, to);
+      memref::CopyOp::create(b, loc, from, to);
       return success();
     };
   } else if (getMemcpyOp() == "linalg.copy") {
     options.memCpyFn = [](OpBuilder &b, Location loc, Value from, Value to) {
-      b.create<linalg::CopyOp>(loc, from, to);
+      linalg::CopyOp::create(b, loc, from, to);
       return success();
     };
   } else {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index c5fab80ecaa08..835ad61f54da2 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -167,7 +167,7 @@ struct RemoveDeallocMemrefsContainedInRetained
       std::optional<bool> analysisResult =
           analysis.isSameAllocation(retained, memref);
       if (analysisResult == true) {
-        auto disjunction = rewriter.create<arith::OrIOp>(
+        auto disjunction = arith::OrIOp::create(rewriter,
             deallocOp.getLoc(), updatedCondition, cond);
         rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(),
                                       disjunction);
@@ -247,14 +247,14 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias
         continue;
       }
 
-      replacements.push_back(rewriter.create<arith::ConstantOp>(
+      replacements.push_back(arith::ConstantOp::create(rewriter,
           deallocOp.getLoc(), rewriter.getBoolAttr(false)));
     }
 
     if (newRetainedMemrefs.size() == deallocOp.getRetained().size())
       return failure();
 
-    auto newDeallocOp = rewriter.create<DeallocOp>(
+    auto newDeallocOp = DeallocOp::create(rewriter,
         deallocOp.getLoc(), deallocOp.getMemrefs(), deallocOp.getConditions(),
         newRetainedMemrefs);
     int i = 0;
@@ -326,7 +326,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       }
 
       // Create new bufferization.dealloc op for `memref`.
-      auto newDeallocOp = rewriter.create<DeallocOp>(loc, memref, cond,
+      auto newDeallocOp = DeallocOp::create(rewriter, loc, memref, cond,
                                                      deallocOp.getRetained());
       updatedConditions.push_back(
           llvm::to_vector(ValueRange(newDeallocOp.getUpdatedConditions())));
@@ -337,7 +337,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       return failure();
 
     // Create bufferization.dealloc op for all remaining memrefs.
-    auto newDeallocOp = rewriter.create<DeallocOp>(
+    auto newDeallocOp = DeallocOp::create(rewriter,
         loc, remainingMemrefs, remainingConditions, deallocOp.getRetained());
 
     // Bit-or all conditions.
@@ -347,7 +347,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       assert(replacements.size() == additionalConditions.size() &&
              "expected same number of updated conditions");
       for (int64_t i = 0, e = replacements.size(); i < e; ++i) {
-        replacements[i] = rewriter.create<arith::OrIOp>(
+        replacements[i] = arith::OrIOp::create(rewriter,
             loc, replacements[i], additionalConditions[i]);
       }
     }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index 6924e88c50e1b..52490a0b2293b 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -132,7 +132,7 @@ updateReturnOps(func::FuncOp func, ArrayRef<BlockArgument> appendedEntryArgs,
           return WalkResult::interrupt();
       }
     }
-    builder.create<func::ReturnOp>(op.getLoc(), keepAsReturnOperands);
+    func::ReturnOp::create(builder, op.getLoc(), keepAsReturnOperands);
     op.erase();
     return WalkResult::advance();
   });
@@ -190,7 +190,7 @@ updateCalls(ModuleOp module,
         assert(hasFullyDynamicLayoutMap(memrefType) &&
                "layout map not supported");
         outParam =
-            builder.create<memref::CastOp>(op.getLoc(), memrefType, outParam);
+            memref::CastOp::create(builder, op.getLoc(), memrefType, outParam);
       }
       memref.replaceAllUsesWith(outParam);
       outParams.push_back(outParam);
@@ -200,7 +200,7 @@ updateCalls(ModuleOp module,
     newOperands.append(outParams.begin(), outParams.end());
     auto newResultTypes = llvm::to_vector<6>(llvm::map_range(
         replaceWithNewCallResults, [](Value v) { return v.getType(); }));
-    auto newCall = builder.create<func::CallOp>(op.getLoc(), op.getCalleeAttr(),
+    auto newCall = func::CallOp::create(builder, op.getLoc(), op.getCalleeAttr(),
                                                 newResultTypes, newOperands);
     for (auto t : llvm::zip(replaceWithNewCallResults, newCall.getResults()))
       std::get<0>(t).replaceAllUsesWith(std::get<1>(t));
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
index a66be7d3b2f52..80984f21df551 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
@@ -141,7 +141,7 @@ bufferization::getGlobalFor(arith::ConstantOp constantOp,
       cast<MemRefType>(getMemRefTypeWithStaticIdentityLayout(type));
   if (memorySpace)
     memrefType = MemRefType::Builder(memrefType).setMemorySpace(memorySpace);
-  auto global = globalBuilder.create<memref::GlobalOp>(
+  auto global = memref::GlobalOp::create(globalBuilder,
       constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
       /*sym_visibility=*/globalBuilder.getStringAttr("private"),
       /*type=*/memrefType,
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 246555dc8c699..4a68ecba79fb7 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -434,7 +434,7 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
     // Replace all uses of the original tensor bbArg.
     rewriter.setInsertionPointToStart(block);
     if (!bbArgUses.empty()) {
-      Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+      Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
           bbArg.getLoc(), tensorType, bbArg);
       for (OpOperand *use : bbArgUses)
         use->set(toTensorOp);
@@ -466,12 +466,12 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
       if (failed(operandBufferType))
         return failure();
       rewriter.setInsertionPointAfterValue(operand);
-      Value bufferizedOperand = rewriter.create<bufferization::ToBufferOp>(
+      Value bufferizedOperand = bufferization::ToBufferOp::create(rewriter,
           operand.getLoc(), *operandBufferType, operand);
       // A cast is needed if the operand and the block argument have different
       // bufferized types.
       if (type != *operandBufferType)
-        bufferizedOperand = rewriter.create<memref::CastOp>(
+        bufferizedOperand = memref::CastOp::create(rewriter,
             operand.getLoc(), type, bufferizedOperand);
       newOperands.push_back(bufferizedOperand);
     }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index c10d290f5d0ed..42d63b66485d5 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -118,7 +118,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
     // Update function calls.
     for (func::CallOp callOp : callerMap[funcOp]) {
       rewriter.setInsertionPoint(callOp);
-      auto newCallOp = rewriter.create<func::CallOp>(callOp.getLoc(), funcOp,
+      auto newCallOp = func::CallOp::create(rewriter, callOp.getLoc(), funcOp,
                                                      callOp.getOperands());
       SmallVector<Value> newResults;
       int64_t nextResult = 0;
@@ -134,7 +134,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
         Type expectedType = callOp.getResult(i).getType();
         if (replacement.getType() != expectedType) {
           // A cast must be inserted at the call site.
-          replacement = rewriter.create<memref::CastOp>(
+          replacement = memref::CastOp::create(rewriter,
               callOp.getLoc(), expectedType, replacement);
         }
         newResults.push_back(replacement);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
index b7db2e847a335..923cbc0e2259d 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
@@ -168,7 +168,7 @@ LogicalResult mlir::bufferization::eliminateEmptyTensors(
             cast<ShapedType>(v.getType()).getElementType())
           continue;
         rewriter.setInsertionPointAfterValue(replacement);
-        replacement = rewriter.create<tensor::CastOp>(v.getLoc(), v.getType(),
+        replacement = tensor::CastOp::create(rewriter, v.getLoc(), v.getType(),
                                                       replacement);
       }
       // Replace the specific use of the tensor::EmptyOp.
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index 2a98203da9d7d..025d5764c6d9e 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -319,7 +319,7 @@ struct CallOpInterface
     }
 
     // 3. Create the new CallOp.
-    Operation *newCallOp = rewriter.create<func::CallOp>(
+    Operation *newCallOp = func::CallOp::create(rewriter,
         callOp.getLoc(), funcOp.getSymName(), resultTypes, newOperands);
     newCallOp->setAttrs(callOp->getAttrs());
 
@@ -483,7 +483,7 @@ struct FuncOpInterface
 
         // Note: If `inferFunctionResultLayout = true`, casts are later folded
         // away.
-        Value toBufferOp = rewriter.create<bufferization::ToBufferOp>(
+        Value toBufferOp = bufferization::ToBufferOp::create(rewriter,
             returnOp.getLoc(), bufferizedType, returnVal);
         returnValues.push_back(toBufferOp);
       }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
index a611126e320ad..83ccd573a2734 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
@@ -64,8 +64,8 @@ class DeallocOpConversion
 
     rewriter.replaceOpWithNewOp<scf::IfOp>(
         op, adaptor.getConditions()[0], [&](OpBuilder &builder, Location loc) {
-          builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[0]);
-          builder.create<scf::YieldOp>(loc);
+          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
+          scf::YieldOp::create(builder, loc);
         });
     return success();
   }
@@ -108,13 +108,13 @@ class DeallocOpConversion
     // Compute the base pointer indices, compare all retained indices to the
     // memref index to check if they alias.
     SmallVector<Value> doesNotAliasList;
-    Value memrefAsIdx = rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(
+    Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(rewriter,
         op->getLoc(), adaptor.getMemrefs()[0]);
     for (Value retained : adaptor.getRetained()) {
       Value retainedAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op->getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op->getLoc(),
                                                                   retained);
-      Value doesNotAlias = rewriter.create<arith::CmpIOp>(
+      Value doesNotAlias = arith::CmpIOp::create(rewriter,
           op->getLoc(), arith::CmpIPredicate::ne, memrefAsIdx, retainedAsIdx);
       doesNotAliasList.push_back(doesNotAlias);
     }
@@ -122,17 +122,17 @@ class DeallocOpConversion
     // AND-reduce the list of booleans from above.
     Value prev = doesNotAliasList.front();
     for (Value doesNotAlias : ArrayRef(doesNotAliasList).drop_front())
-      prev = rewriter.create<arith::AndIOp>(op->getLoc(), prev, doesNotAlias);
+      prev = arith::AndIOp::create(rewriter, op->getLoc(), prev, doesNotAlias);
 
     // Also consider the condition given by the dealloc operation and perform a
     // conditional deallocation guarded by that value.
-    Value shouldDealloc = rewriter.create<arith::AndIOp>(
+    Value shouldDealloc = arith::AndIOp::create(rewriter,
         op->getLoc(), prev, adaptor.getConditions()[0]);
 
-    rewriter.create<scf::IfOp>(
+    scf::IfOp::create(rewriter,
         op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-          builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[0]);
-          builder.create<scf::YieldOp>(loc);
+          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
+          scf::YieldOp::create(builder, loc);
         });
 
     // Compute the replacement values for the dealloc operation results. This
@@ -140,12 +140,12 @@ class DeallocOpConversion
     // `select(does_alias_with_memref(r), memref_cond, false)` for each retained
     // value r.
     SmallVector<Value> replacements;
-    Value trueVal = rewriter.create<arith::ConstantOp>(
+    Value trueVal = arith::ConstantOp::create(rewriter,
         op->getLoc(), rewriter.getBoolAttr(true));
     for (Value doesNotAlias : doesNotAliasList) {
       Value aliases =
-          rewriter.create<arith::XOrIOp>(op->getLoc(), doesNotAlias, trueVal);
-      Value result = rewriter.create<arith::AndIOp>(op->getLoc(), aliases,
+          arith::XOrIOp::create(rewriter, op->getLoc(), doesNotAlias, trueVal);
+      Value result = arith::AndIOp::create(rewriter, op->getLoc(), aliases,
                                                     adaptor.getConditions()[0]);
       replacements.push_back(result);
     }
@@ -230,18 +230,18 @@ class DeallocOpConversion
     // Without storing them to memrefs, we could not use for-loops but only a
     // completely unrolled version of it, potentially leading to code-size
     // blow-up.
-    Value toDeallocMemref = rewriter.create<memref::AllocOp>(
+    Value toDeallocMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
                                      rewriter.getIndexType()));
-    Value conditionMemref = rewriter.create<memref::AllocOp>(
+    Value conditionMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getConditions().size()},
                                      rewriter.getI1Type()));
-    Value toRetainMemref = rewriter.create<memref::AllocOp>(
+    Value toRetainMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
                                      rewriter.getIndexType()));
 
     auto getConstValue = [&](uint64_t value) -> Value {
-      return rewriter.create<arith::ConstantOp>(op.getLoc(),
+      return arith::ConstantOp::create(rewriter, op.getLoc(),
                                                 rewriter.getIndexAttr(value));
     };
 
@@ -249,58 +249,58 @@ class DeallocOpConversion
     // at runtime.
     for (auto [i, toDealloc] : llvm::enumerate(adaptor.getMemrefs())) {
       Value memrefAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op.getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
                                                                   toDealloc);
-      rewriter.create<memref::StoreOp>(op.getLoc(), memrefAsIdx,
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx,
                                        toDeallocMemref, getConstValue(i));
     }
 
     for (auto [i, cond] : llvm::enumerate(adaptor.getConditions()))
-      rewriter.create<memref::StoreOp>(op.getLoc(), cond, conditionMemref,
+      memref::StoreOp::create(rewriter, op.getLoc(), cond, conditionMemref,
                                        getConstValue(i));
 
     for (auto [i, toRetain] : llvm::enumerate(adaptor.getRetained())) {
       Value memrefAsIdx =
-          rewriter.create<memref::ExtractAlignedPointerAsIndexOp>(op.getLoc(),
+          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
                                                                   toRetain);
-      rewriter.create<memref::StoreOp>(op.getLoc(), memrefAsIdx, toRetainMemref,
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx, toRetainMemref,
                                        getConstValue(i));
     }
 
     // Cast the allocated memrefs to dynamic shape because we want only one
     // helper function no matter how many operands the bufferization.dealloc
     // has.
-    Value castedDeallocMemref = rewriter.create<memref::CastOp>(
+    Value castedDeallocMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toDeallocMemref);
-    Value castedCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         conditionMemref);
-    Value castedRetainMemref = rewriter.create<memref::CastOp>(
+    Value castedRetainMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toRetainMemref);
 
-    Value deallocCondsMemref = rewriter.create<memref::AllocOp>(
+    Value deallocCondsMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
                                      rewriter.getI1Type()));
-    Value retainCondsMemref = rewriter.create<memref::AllocOp>(
+    Value retainCondsMemref = memref::AllocOp::create(rewriter,
         op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
                                      rewriter.getI1Type()));
 
-    Value castedDeallocCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedDeallocCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         deallocCondsMemref);
-    Value castedRetainCondsMemref = rewriter.create<memref::CastOp>(
+    Value castedRetainCondsMemref = memref::CastOp::create(rewriter,
         op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         retainCondsMemref);
 
     Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>();
-    rewriter.create<func::CallOp>(
+    func::CallOp::create(rewriter,
         op.getLoc(), deallocHelperFuncMap.lookup(symtableOp),
         SmallVector<Value>{castedDeallocMemref, castedRetainMemref,
                            castedCondsMemref, castedDeallocCondsMemref,
@@ -308,30 +308,30 @@ class DeallocOpConversion
 
     for (unsigned i = 0, e = adaptor.getMemrefs().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value shouldDealloc = rewriter.create<memref::LoadOp>(
+      Value shouldDealloc = memref::LoadOp::create(rewriter,
           op.getLoc(), deallocCondsMemref, idxValue);
-      rewriter.create<scf::IfOp>(
+      scf::IfOp::create(rewriter,
           op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-            builder.create<memref::DeallocOp>(loc, adaptor.getMemrefs()[i]);
-            builder.create<scf::YieldOp>(loc);
+            memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[i]);
+            scf::YieldOp::create(builder, loc);
           });
     }
 
     SmallVector<Value> replacements;
     for (unsigned i = 0, e = adaptor.getRetained().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value ownership = rewriter.create<memref::LoadOp>(
+      Value ownership = memref::LoadOp::create(rewriter,
           op.getLoc(), retainCondsMemref, idxValue);
       replacements.push_back(ownership);
     }
 
     // Deallocate above allocated memrefs again to avoid memory leaks.
     // Deallocation will not be run on code after this stage.
-    rewriter.create<memref::DeallocOp>(op.getLoc(), toDeallocMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), toRetainMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), conditionMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), deallocCondsMemref);
-    rewriter.create<memref::DeallocOp>(op.getLoc(), retainCondsMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), toDeallocMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), toRetainMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), conditionMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), deallocCondsMemref);
+    memref::DeallocOp::create(rewriter, op.getLoc(), retainCondsMemref);
 
     rewriter.replaceOp(op, replacements);
     return success();
@@ -349,7 +349,7 @@ class DeallocOpConversion
                   ConversionPatternRewriter &rewriter) const override {
     // Lower the trivial case.
     if (adaptor.getMemrefs().empty()) {
-      Value falseVal = rewriter.create<arith::ConstantOp>(
+      Value falseVal = arith::ConstantOp::create(rewriter,
           op.getLoc(), rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           op, SmallVector<Value>(adaptor.getRetained().size(), falseVal));
@@ -449,30 +449,30 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
   Value retainCondsMemref = helperFuncOp.getArguments()[4];
 
   // Insert some prerequisites.
-  Value c0 = builder.create<arith::ConstantOp>(loc, builder.getIndexAttr(0));
-  Value c1 = builder.create<arith::ConstantOp>(loc, builder.getIndexAttr(1));
+  Value c0 = arith::ConstantOp::create(builder, loc, builder.getIndexAttr(0));
+  Value c1 = arith::ConstantOp::create(builder, loc, builder.getIndexAttr(1));
   Value trueValue =
-      builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(true));
+      arith::ConstantOp::create(builder, loc, builder.getBoolAttr(true));
   Value falseValue =
-      builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(false));
-  Value toDeallocSize = builder.create<memref::DimOp>(loc, toDeallocMemref, c0);
-  Value toRetainSize = builder.create<memref::DimOp>(loc, toRetainMemref, c0);
+      arith::ConstantOp::create(builder, loc, builder.getBoolAttr(false));
+  Value toDeallocSize = memref::DimOp::create(builder, loc, toDeallocMemref, c0);
+  Value toRetainSize = memref::DimOp::create(builder, loc, toRetainMemref, c0);
 
-  builder.create<scf::ForOp>(
+  scf::ForOp::create(builder,
       loc, c0, toRetainSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value i, ValueRange iterArgs) {
-        builder.create<memref::StoreOp>(loc, falseValue, retainCondsMemref, i);
-        builder.create<scf::YieldOp>(loc);
+        memref::StoreOp::create(builder, loc, falseValue, retainCondsMemref, i);
+        scf::YieldOp::create(builder, loc);
       });
 
-  builder.create<scf::ForOp>(
+  scf::ForOp::create(builder,
       loc, c0, toDeallocSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value outerIter,
           ValueRange iterArgs) {
         Value toDealloc =
-            builder.create<memref::LoadOp>(loc, toDeallocMemref, outerIter);
+            memref::LoadOp::create(builder, loc, toDeallocMemref, outerIter);
         Value cond =
-            builder.create<memref::LoadOp>(loc, conditionMemref, outerIter);
+            memref::LoadOp::create(builder, loc, conditionMemref, outerIter);
 
         // Build the first for loop that computes aliasing with retained
         // memrefs.
@@ -482,31 +482,31 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, toRetainSize, c1, trueValue,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value retainValue = builder.create<memref::LoadOp>(
+                      Value retainValue = memref::LoadOp::create(builder,
                           loc, toRetainMemref, i);
-                      Value doesAlias = builder.create<arith::CmpIOp>(
+                      Value doesAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::eq, retainValue,
                           toDealloc);
-                      builder.create<scf::IfOp>(
+                      scf::IfOp::create(builder,
                           loc, doesAlias,
                           [&](OpBuilder &builder, Location loc) {
                             Value retainCondValue =
-                                builder.create<memref::LoadOp>(
+                                memref::LoadOp::create(builder,
                                     loc, retainCondsMemref, i);
                             Value aggregatedRetainCond =
-                                builder.create<arith::OrIOp>(
+                                arith::OrIOp::create(builder,
                                     loc, retainCondValue, cond);
-                            builder.create<memref::StoreOp>(
+                            memref::StoreOp::create(builder,
                                 loc, aggregatedRetainCond, retainCondsMemref,
                                 i);
-                            builder.create<scf::YieldOp>(loc);
+                            scf::YieldOp::create(builder, loc);
                           });
-                      Value doesntAlias = builder.create<arith::CmpIOp>(
+                      Value doesntAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::ne, retainValue,
                           toDealloc);
-                      Value yieldValue = builder.create<arith::AndIOp>(
+                      Value yieldValue = arith::AndIOp::create(builder,
                           loc, iterArgs[0], doesntAlias);
-                      builder.create<scf::YieldOp>(loc, yieldValue);
+                      scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
@@ -518,24 +518,24 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, outerIter, c1, noRetainAlias,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value prevDeallocValue = builder.create<memref::LoadOp>(
+                      Value prevDeallocValue = memref::LoadOp::create(builder,
                           loc, toDeallocMemref, i);
-                      Value doesntAlias = builder.create<arith::CmpIOp>(
+                      Value doesntAlias = arith::CmpIOp::create(builder,
                           loc, arith::CmpIPredicate::ne, prevDeallocValue,
                           toDealloc);
-                      Value yieldValue = builder.create<arith::AndIOp>(
+                      Value yieldValue = arith::AndIOp::create(builder,
                           loc, iterArgs[0], doesntAlias);
-                      builder.create<scf::YieldOp>(loc, yieldValue);
+                      scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
-        Value shouldDealoc = builder.create<arith::AndIOp>(loc, noAlias, cond);
-        builder.create<memref::StoreOp>(loc, shouldDealoc, deallocCondsMemref,
+        Value shouldDealoc = arith::AndIOp::create(builder, loc, noAlias, cond);
+        memref::StoreOp::create(builder, loc, shouldDealoc, deallocCondsMemref,
                                         outerIter);
-        builder.create<scf::YieldOp>(loc);
+        scf::YieldOp::create(builder, loc);
       });
 
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
   return helperFuncOp;
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index 1eeafc4df8cf1..71890bfea4e5d 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -43,7 +43,7 @@ using namespace mlir::bufferization;
 //===----------------------------------------------------------------------===//
 
 static Value buildBoolValue(OpBuilder &builder, Location loc, bool value) {
-  return builder.create<arith::ConstantOp>(loc, builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, loc, builder.getBoolAttr(value));
 }
 
 static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
@@ -755,12 +755,12 @@ Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
           .create<scf::IfOp>(
               memref.getLoc(), condition,
               [&](OpBuilder &builder, Location loc) {
-                builder.create<scf::YieldOp>(loc, newMemref);
+                scf::YieldOp::create(builder, loc, newMemref);
               },
               [&](OpBuilder &builder, Location loc) {
                 Value clone =
-                    builder.create<bufferization::CloneOp>(loc, newMemref);
-                builder.create<scf::YieldOp>(loc, clone);
+                    bufferization::CloneOp::create(builder, loc, newMemref);
+                scf::YieldOp::create(builder, loc, clone);
               })
           .getResult(0);
   Value trueVal = buildBoolValue(builder, memref.getLoc(), true);
@@ -797,7 +797,7 @@ BufferDeallocation::handleInterface(BranchOpInterface op) {
   state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
                            toRetain);
 
-  auto deallocOp = builder.create<bufferization::DeallocOp>(
+  auto deallocOp = bufferization::DeallocOp::create(builder,
       op.getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
@@ -885,10 +885,10 @@ BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
       builder.setInsertionPoint(op);
       Ownership ownership = state.getOwnership(operand, block);
       if (ownership.isUnique()) {
-        Value ownershipInverted = builder.create<arith::XOrIOp>(
+        Value ownershipInverted = arith::XOrIOp::create(builder,
             op.getLoc(), ownership.getIndicator(),
             buildBoolValue(builder, op.getLoc(), true));
-        builder.create<cf::AssertOp>(
+        cf::AssertOp::create(builder,
             op.getLoc(), ownershipInverted,
             "expected that the block does not have ownership");
       }
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
index f5a42c572ff96..925c72959bd3a 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
@@ -48,7 +48,7 @@ Operation *complex::ComplexDialect::materializeConstant(OpBuilder &builder,
                                                         Type type,
                                                         Location loc) {
   if (complex::ConstantOp::isBuildableWith(value, type)) {
-    return builder.create<complex::ConstantOp>(loc, type,
+    return complex::ConstantOp::create(builder, loc, type,
                                                llvm::cast<ArrayAttr>(value));
   }
   return arith::ConstantOp::materialize(builder, value, type, loc);
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 0c11c76cf1f71..38e47cc5e3714 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -312,7 +312,7 @@ struct SimplifyCondBranchIdenticalSuccessors
       if (std::get<0>(it) == std::get<1>(it))
         mergedOperands.push_back(std::get<0>(it));
       else
-        mergedOperands.push_back(rewriter.create<arith::SelectOp>(
+        mergedOperands.push_back(arith::SelectOp::create(rewriter,
             condbr.getLoc(), condition, std::get<0>(it), std::get<1>(it)));
     }
 
@@ -412,7 +412,7 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantTrue)
-            constantTrue = rewriter.create<arith::ConstantOp>(
+            constantTrue = arith::ConstantOp::create(rewriter,
                 condbr.getLoc(), ty, rewriter.getBoolAttr(true));
 
           rewriter.modifyOpInPlace(use.getOwner(),
@@ -427,7 +427,7 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantFalse)
-            constantFalse = rewriter.create<arith::ConstantOp>(
+            constantFalse = arith::ConstantOp::create(rewriter,
                 condbr.getLoc(), ty, rewriter.getBoolAttr(false));
 
           rewriter.modifyOpInPlace(use.getOwner(),
diff --git a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index a077f56f4f472..23f50930cb3b3 100644
--- a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -87,7 +87,7 @@ struct CondBranchOpInterface
                                destOperands.getAsOperandRange(), toRetain);
       SmallVector<Value> adaptedConditions(
           llvm::map_range(conditions, conditionModifier));
-      auto deallocOp = builder.create<bufferization::DeallocOp>(
+      auto deallocOp = bufferization::DeallocOp::create(builder,
           condBr.getLoc(), memrefs, adaptedConditions, toRetain);
       state.resetOwnerships(deallocOp.getRetained(), condBr->getBlock());
       for (auto [retained, ownership] : llvm::zip(
@@ -115,18 +115,18 @@ struct CondBranchOpInterface
     DeallocOp thenTakenDeallocOp = insertDeallocForBranch(
         condBr.getTrueDest(), condBr.getTrueDestOperandsMutable(),
         [&](Value cond) {
-          return builder.create<arith::AndIOp>(condBr.getLoc(), cond,
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond,
                                                condBr.getCondition());
         },
         thenMapping);
     DeallocOp elseTakenDeallocOp = insertDeallocForBranch(
         condBr.getFalseDest(), condBr.getFalseDestOperandsMutable(),
         [&](Value cond) {
-          Value trueVal = builder.create<arith::ConstantOp>(
+          Value trueVal = arith::ConstantOp::create(builder,
               condBr.getLoc(), builder.getBoolAttr(true));
-          Value negation = builder.create<arith::XOrIOp>(
+          Value negation = arith::XOrIOp::create(builder,
               condBr.getLoc(), trueVal, condBr.getCondition());
-          return builder.create<arith::AndIOp>(condBr.getLoc(), cond, negation);
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond, negation);
         },
         elseMapping);
 
@@ -143,7 +143,7 @@ struct CondBranchOpInterface
 
     for (Value retained : commonValues) {
       state.resetOwnerships(retained, condBr->getBlock());
-      Value combinedOwnership = builder.create<arith::SelectOp>(
+      Value combinedOwnership = arith::SelectOp::create(builder,
           condBr.getLoc(), condBr.getCondition(), thenMapping[retained],
           elseMapping[retained]);
       state.updateOwnership(retained, combinedOwnership, condBr->getBlock());
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index fccbca6ed05dd..568da8905cbc8 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -47,13 +47,13 @@ void EmitCDialect::initialize() {
 Operation *EmitCDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
-  return builder.create<emitc::ConstantOp>(loc, type, value);
+  return emitc::ConstantOp::create(builder, loc, type, value);
 }
 
 /// Default callback for builders of ops carrying a region. Inserts a yield
 /// without arguments.
 void mlir::emitc::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<emitc::YieldOp>(loc);
+  emitc::YieldOp::create(builder, loc);
 }
 
 bool mlir::emitc::isSupportedEmitCType(Type type) {
diff --git a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
index 12218f5072982..d5fe3b4ae1e7f 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
@@ -24,7 +24,7 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
   Location loc = op->getLoc();
 
   builder.setInsertionPointAfter(op);
-  auto expressionOp = builder.create<emitc::ExpressionOp>(loc, resultType);
+  auto expressionOp = emitc::ExpressionOp::create(builder, loc, resultType);
 
   // Replace all op's uses with the new expression's result.
   result.replaceAllUsesWith(expressionOp.getResult());
@@ -33,7 +33,7 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
   Region &region = expressionOp.getRegion();
   Block &block = region.emplaceBlock();
   builder.setInsertionPointToEnd(&block);
-  auto yieldOp = builder.create<emitc::YieldOp>(loc, result);
+  auto yieldOp = emitc::YieldOp::create(builder, loc, result);
 
   // Move op into the new expression.
   op->moveBefore(yieldOp);
diff --git a/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp b/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
index 72c8fd0f32485..ab7be8d6cedd9 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/TypeConversions.cpp
@@ -21,7 +21,7 @@ Value materializeAsUnrealizedCast(OpBuilder &builder, Type resultType,
   if (inputs.size() != 1)
     return Value();
 
-  return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+  return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
       .getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
index 17d436f6df028..72c331ad96607 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
@@ -50,7 +50,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
                                 PatternRewriter &rewriter) const override {
 
     auto className = funcOp.getSymNameAttr().str() + "Class";
-    ClassOp newClassOp = rewriter.create<ClassOp>(funcOp.getLoc(), className);
+    ClassOp newClassOp = ClassOp::create(rewriter, funcOp.getLoc(), className);
 
     SmallVector<std::pair<StringAttr, TypeAttr>> fields;
     rewriter.createBlock(&newClassOp.getBody());
@@ -67,7 +67,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
 
       TypeAttr typeAttr = TypeAttr::get(val.getType());
       fields.push_back({fieldName, typeAttr});
-      rewriter.create<emitc::FieldOp>(funcOp.getLoc(), fieldName, typeAttr,
+      emitc::FieldOp::create(rewriter, funcOp.getLoc(), fieldName, typeAttr,
                                       argAttr);
     }
 
@@ -75,7 +75,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
     FunctionType funcType = funcOp.getFunctionType();
     Location loc = funcOp.getLoc();
     FuncOp newFuncOp =
-        rewriter.create<emitc::FuncOp>(loc, ("execute"), funcType);
+        emitc::FuncOp::create(rewriter, loc, ("execute"), funcType);
 
     rewriter.createBlock(&newFuncOp.getBody());
     newFuncOp.getBody().takeBody(funcOp.getBody());
@@ -85,7 +85,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
     newArguments.reserve(fields.size());
     for (auto &[fieldName, attr] : fields) {
       GetFieldOp arg =
-          rewriter.create<emitc::GetFieldOp>(loc, attr.getValue(), fieldName);
+          emitc::GetFieldOp::create(rewriter, loc, attr.getValue(), fieldName);
       newArguments.push_back(arg);
     }
 
diff --git a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
index 3328d58551bff..7485ba40b4344 100644
--- a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
+++ b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
@@ -61,7 +61,7 @@ struct FuncInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    builder.create<cf::BranchOp>(op->getLoc(), newDest, returnOp.getOperands());
+    cf::BranchOp::create(builder, op->getLoc(), newDest, returnOp.getOperands());
     op->erase();
   }
 
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index d8309d81f4a3f..9a33fc8e07dde 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -50,7 +50,7 @@ void FuncDialect::initialize() {
 Operation *FuncDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                             Type type, Location loc) {
   if (ConstantOp::isBuildableWith(value, type))
-    return builder.create<ConstantOp>(loc, type,
+    return ConstantOp::create(builder, loc, type,
                                       llvm::cast<FlatSymbolRefAttr>(value));
   return nullptr;
 }
diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
index 11fc696a258c0..10515251fef7d 100644
--- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
+++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
@@ -170,7 +170,7 @@ transform::CastAndCallOp::apply(transform::TransformRewriter &rewriter,
     }
   }
 
-  auto callOp = rewriter.create<func::CallOp>(insertionPoint->getLoc(),
+  auto callOp = func::CallOp::create(rewriter, insertionPoint->getLoc(),
                                               targetFunction, inputs);
 
   // Cast the call results back to the expected types. If any conversions fail
diff --git a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
index a3638c8766a5c..4b70cf172b407 100644
--- a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
@@ -46,7 +46,7 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 
     // Substitute with the new result types from the corresponding FuncType
     // conversion.
-    auto newCallOp = rewriter.create<CallOp>(
+    auto newCallOp = CallOp::create(rewriter,
         callOp.getLoc(), callOp.getCallee(), convertedResults,
         flattenValues(adaptor.getOperands()));
     SmallVector<ValueRange> replacements;
diff --git a/mlir/lib/Dialect/Func/Utils/Utils.cpp b/mlir/lib/Dialect/Func/Utils/Utils.cpp
index 0e9662689ef78..1eba777921828 100644
--- a/mlir/lib/Dialect/Func/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Func/Utils/Utils.cpp
@@ -44,7 +44,7 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
   for (unsigned int idx : newResultsOrder)
     newOutputTypes.push_back(origOutputTypes[idx]);
   rewriter.setInsertionPoint(funcOp);
-  auto newFuncOp = rewriter.create<func::FuncOp>(
+  auto newFuncOp = func::FuncOp::create(rewriter,
       funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(newInputTypes, newOutputTypes));
 
@@ -80,7 +80,7 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
     newReturnValues.push_back(returnOp.getOperand(idx));
   rewriter.setInsertionPoint(returnOp);
   auto newReturnOp =
-      rewriter.create<func::ReturnOp>(newFuncOp.getLoc(), newReturnValues);
+      func::ReturnOp::create(rewriter, newFuncOp.getLoc(), newReturnValues);
   newReturnOp->setDiscardableAttrs(returnOp->getDiscardableAttrDictionary());
   rewriter.eraseOp(returnOp);
 
@@ -109,7 +109,7 @@ func::replaceCallOpWithNewOrder(RewriterBase &rewriter, func::CallOp callOp,
   // Replace the kernel call operation with a new one that has the
   // reordered arguments.
   rewriter.setInsertionPoint(callOp);
-  auto newCallOp = rewriter.create<func::CallOp>(
+  auto newCallOp = func::CallOp::create(rewriter,
       callOp.getLoc(), callOp.getCallee(), newResultTypes, newArgsOrderValues);
   newCallOp.setNoInlineAttr(callOp.getNoInlineAttr());
   for (auto &&[newIndex, origIndex] : llvm::enumerate(newResultsOrder))
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 30b5ac9809139..f7eb41be287e4 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -1137,7 +1137,7 @@ struct FoldLaunchArguments : public OpRewritePattern<LaunchOp> {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(&op.getBody().front());
         zero =
-            rewriter.create<arith::ConstantIndexOp>(op.getLoc(), /*value=*/0);
+            arith::ConstantIndexOp::create(rewriter, op.getLoc(), /*value=*/0);
       }
       rewriter.replaceAllUsesWith(id, zero);
       simplified = true;
@@ -1381,9 +1381,9 @@ static void printLaunchFuncOperands(OpAsmPrinter &printer, Operation *,
 void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
                       int32_t offset, int32_t width, ShuffleMode mode) {
   build(builder, result, value,
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(offset)),
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(width)),
         mode);
 }
@@ -1395,9 +1395,9 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
 void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
                      int32_t offset, int32_t width) {
   build(builder, result, value,
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(offset)),
-        builder.create<arith::ConstantOp>(result.location,
+        arith::ConstantOp::create(builder, result.location,
                                           builder.getI32IntegerAttr(width)));
 }
 
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index c9e91535df946..2035f371694a7 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -560,7 +560,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   Value predicate;
   if (originalBasisWasProvided) {
     for (Value tmpPredicate : builderResult.predicateOps) {
-      predicate = predicate ? rewriter.create<arith::AndIOp>(loc, predicate,
+      predicate = predicate ? arith::AndIOp::create(rewriter, loc, predicate,
                                                              tmpPredicate)
                             : tmpPredicate;
     }
@@ -573,7 +573,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   Block::iterator insertionPoint;
   if (predicate) {
     // Step 6.a. If predicated, move at the beginning.
-    auto ifOp = rewriter.create<scf::IfOp>(loc, predicate,
+    auto ifOp = scf::IfOp::create(rewriter, loc, predicate,
                                            /*withElseRegion=*/false);
     targetBlock = ifOp.thenBlock();
     insertionPoint = ifOp.thenBlock()->begin();
@@ -632,7 +632,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapForallToBlocksImpl(
     // the insertion point.
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPointToStart(parentBlock);
-    zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   }
 
   ForallRewriteResult rewriteResult;
@@ -884,7 +884,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapOneForallToThreadsImpl(
     return diag;
   // Add a syncthreads if needed. TODO: warpsync
   if (syncAfterDistribute)
-    rewriter.create<BarrierOp>(loc);
+    BarrierOp::create(rewriter, loc);
 
   return DiagnosedSilenceableFailure::success();
 }
@@ -901,7 +901,7 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapNestedForallToThreadsImpl(
 
   // Create an early zero index value for replacements.
   Location loc = target->getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   DiagnosedSilenceableFailure diag = DiagnosedSilenceableFailure::success();
   WalkResult walkResult = target->walk([&](scf::ForallOp forallOp) {
     diag = mlir::transform::gpu::mapOneForallToThreadsImpl(
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index f6bdbe384c08f..6f0510138da20 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -76,8 +76,8 @@ buildPredicates(RewriterBase &rewriter, Location loc, ArrayRef<Value> activeIds,
     }
     if (activeMappingSize == availableMappingSize)
       continue;
-    Value idx = rewriter.create<arith::ConstantIndexOp>(loc, activeMappingSize);
-    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value idx = arith::ConstantIndexOp::create(rewriter, loc, activeMappingSize);
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                                 activeId, idx);
     predicateOps.push_back(pred);
   }
@@ -98,11 +98,11 @@ static Value buildLinearId(RewriterBase &rewriter, Location loc,
   bindDims(rewriter.getContext(), tx, ty, tz);
   bindSymbols(rewriter.getContext(), bdx, bdy);
   SmallVector<OpFoldResult> vals{
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::x)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::x)
           .getResult(),
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::y)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::y)
           .getResult(),
-      rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::z)
+      ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::z)
           .getResult(),
       originalBasisOfr[0], originalBasisOfr[1]};
   OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
@@ -209,9 +209,9 @@ static GpuIdBuilderFnType common3DIdBuilderFn(int64_t multiplicity = 1) {
                             ArrayRef<int64_t> originalBasis) {
     IndexType indexType = rewriter.getIndexType();
     SmallVector<Value> ids{
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::x),
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::y),
-        rewriter.create<ThreadOrBlockIdOp>(loc, indexType, Dimension::z)};
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::x),
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::y),
+        ThreadOrBlockIdOp::create(rewriter, loc, indexType, Dimension::z)};
     // In the 3-D mapping case, scale the first dimension by the multiplicity.
     SmallVector<Value> scaledIds = ids;
     AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
@@ -411,7 +411,7 @@ DiagnosedSilenceableFailure createGpuLaunch(
     return diag;
 
   auto createConst = [&](int dim) {
-    return rewriter.create<arith::ConstantIndexOp>(loc, dim);
+    return arith::ConstantIndexOp::create(rewriter, loc, dim);
   };
   OpBuilder::InsertionGuard guard(rewriter);
   Value one = createConst(1);
@@ -421,10 +421,10 @@ DiagnosedSilenceableFailure createGpuLaunch(
   Value blkSizeX = blockDimX.has_value() ? createConst(blockDimX.value()) : one;
   Value blkSizeY = blockDimY.has_value() ? createConst(blockDimY.value()) : one;
   Value blkSizeZ = blockDimZ.has_value() ? createConst(blockDimZ.value()) : one;
-  launchOp = rewriter.create<LaunchOp>(loc, gridSizeX, gridSizeY, gridSizeZ,
+  launchOp = LaunchOp::create(rewriter, loc, gridSizeX, gridSizeY, gridSizeZ,
                                        blkSizeX, blkSizeY, blkSizeZ);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
-  rewriter.create<TerminatorOp>(loc);
+  TerminatorOp::create(rewriter, loc);
   return DiagnosedSilenceableFailure::success();
 }
 
@@ -445,7 +445,7 @@ DiagnosedSilenceableFailure alterGpuLaunch(
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPointAfterValue(currentBlockdim.x);
   auto createConstValue = [&](int dim) {
-    return rewriter.create<arith::ConstantIndexOp>(currentBlockdim.x.getLoc(),
+    return arith::ConstantIndexOp::create(rewriter, currentBlockdim.x.getLoc(),
                                                    dim);
   };
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
index 98dc8ad3aa416..8c449144af3a9 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
@@ -145,7 +145,7 @@ struct GpuAllReduceRewriter {
   // Shortcut to create an op from rewriter using loc as the first argument.
   template <typename T, typename... Args>
   T create(Args... args) {
-    return rewriter.create<T>(loc, std::forward<Args>(args)...);
+    return T::create(rewriter, loc, std::forward<Args>(args)...);
   }
 
   // Creates dimension op of type T, with the result casted to int32.
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index c39ba4a41898d..ccb5245442206 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -129,7 +129,7 @@ struct GpuAsyncRegionPass::ThreadTokenCallback {
   }
 
   Value createWaitOp(Location loc, Type resultType, ValueRange operands) {
-    return builder.create<gpu::WaitOp>(loc, resultType, operands)
+    return gpu::WaitOp::create(builder, loc, resultType, operands)
         .getAsyncToken();
   }
 
@@ -165,7 +165,7 @@ async::ExecuteOp addExecuteResults(async::ExecuteOp executeOp,
 
   // Clone executeOp with the extra results.
   OpBuilder builder(executeOp);
-  auto newOp = builder.create<async::ExecuteOp>(
+  auto newOp = async::ExecuteOp::create(builder,
       executeOp.getLoc(), TypeRange{resultTypes}.drop_front() /*drop token*/,
       executeOp.getDependencies(), executeOp.getBodyOperands());
   IRMapping mapper;
@@ -247,7 +247,7 @@ struct GpuAsyncRegionPass::DeferWaitCallback {
           builder.setInsertionPointAfter(op);
           for (auto asyncToken : asyncTokens)
             tokens.push_back(
-                builder.create<async::AwaitOp>(loc, asyncToken).getResult());
+                async::AwaitOp::create(builder, loc, asyncToken).getResult());
           // Set `it` after the inserted async.await ops.
           it = builder.getInsertionPoint();
         })
@@ -279,7 +279,7 @@ struct GpuAsyncRegionPass::DeferWaitCallback {
 
     // Otherwise, insert a gpu.wait before 'it'.
     builder.setInsertionPoint(it->getBlock(), it);
-    auto waitOp = builder.create<gpu::WaitOp>(loc, Type{}, tokens);
+    auto waitOp = gpu::WaitOp::create(builder, loc, Type{}, tokens);
 
     // If the new waitOp is at the end of an async.execute region, add it to the
     // worklist. 'operator()(executeOp)' would do the same, but this is faster.
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index 65b9407a7efba..5c84a340f0e08 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -62,7 +62,7 @@ getFlatOffsetAndStrides(OpBuilder &rewriter, Location loc, Value source,
     OpBuilder::InsertionGuard g(rewriter);
     setInsertionPointToStart(rewriter, source);
     newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, source);
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, source);
   }
 
   auto &&[sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
@@ -108,7 +108,7 @@ static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
   auto &&[base, offset, ignore] =
       getFlatOffsetAndStrides(rewriter, loc, source, offsetsTemp);
   MemRefType retType = inferCastResultType(base, offset);
-  return rewriter.create<memref::ReinterpretCastOp>(loc, retType, base, offset,
+  return memref::ReinterpretCastOp::create(rewriter, loc, retType, base, offset,
                                                     ArrayRef<OpFoldResult>(),
                                                     ArrayRef<OpFoldResult>());
 }
diff --git a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
index 153ceb23a6ecd..6519b65cec465 100644
--- a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
@@ -26,11 +26,11 @@ struct GpuGlobalIdRewriter : public OpRewritePattern<gpu::GlobalIdOp> {
                                 PatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     auto dim = op.getDimension();
-    auto blockId = rewriter.create<gpu::BlockIdOp>(loc, dim);
-    auto blockDim = rewriter.create<gpu::BlockDimOp>(loc, dim);
+    auto blockId = gpu::BlockIdOp::create(rewriter, loc, dim);
+    auto blockDim = gpu::BlockDimOp::create(rewriter, loc, dim);
     // Compute blockId.x * blockDim.x
-    auto tmp = rewriter.create<index::MulOp>(op.getLoc(), blockId, blockDim);
-    auto threadId = rewriter.create<gpu::ThreadIdOp>(loc, dim);
+    auto tmp = index::MulOp::create(rewriter, op.getLoc(), blockId, blockDim);
+    auto threadId = gpu::ThreadIdOp::create(rewriter, loc, dim);
     // Compute threadId.x + blockId.x * blockDim.x
     rewriter.replaceOpWithNewOp<index::AddOp>(op, threadId, tmp);
     return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index 34ea9fcab4188..b98c060ac1da5 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -40,7 +40,7 @@ template <typename OpTy>
 static void createForAllDimensions(OpBuilder &builder, Location loc,
                                    SmallVectorImpl<Value> &values) {
   for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z})
-    values.push_back(builder.create<OpTy>(loc, builder.getIndexType(), dim));
+    values.push_back(OpTy::create(builder, loc, builder.getIndexType(), dim));
 }
 
 /// Adds operations generating block/thread ids and grid/block dimensions at the
@@ -195,7 +195,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   }
   FunctionType type =
       FunctionType::get(launchOp.getContext(), kernelOperandTypes, {});
-  auto outlinedFunc = builder.create<gpu::GPUFuncOp>(
+  auto outlinedFunc = gpu::GPUFuncOp::create(builder,
       loc, kernelFnName, type,
       TypeRange(ValueRange(launchOp.getWorkgroupAttributions())),
       TypeRange(ValueRange(launchOp.getPrivateAttributions())));
@@ -247,7 +247,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
     if (!terminator)
       continue;
     OpBuilder replacer(terminator);
-    replacer.create<gpu::ReturnOp>(terminator->getLoc());
+    gpu::ReturnOp::create(replacer, terminator->getLoc());
     terminator->erase();
   }
 
@@ -287,7 +287,7 @@ static void convertToLaunchFuncOp(gpu::LaunchOp launchOp,
   Value asyncToken = launchOp.getAsyncToken();
   std::optional<gpu::KernelDim3> clusterSize =
       launchOp.getClusterSizeOperandValues();
-  auto launchFunc = builder.create<gpu::LaunchFuncOp>(
+  auto launchFunc = gpu::LaunchFuncOp::create(builder,
       launchOp.getLoc(), kernelFunc, launchOp.getGridSizeOperandValues(),
       launchOp.getBlockSizeOperandValues(),
       launchOp.getDynamicSharedMemorySize(), operands,
@@ -415,7 +415,7 @@ class GpuKernelOutliningPass
     // Check if the module already exists in the symbol table
     if (!kernelModule) {
       // If not found, create a new GPU module
-      kernelModule = builder.create<gpu::GPUModuleOp>(kernelFunc.getLoc(),
+      kernelModule = gpu::GPUModuleOp::create(builder, kernelFunc.getLoc(),
                                                       kernelModuleName);
     }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 14c44f27a6249..2bee1a1619665 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -34,8 +34,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
   auto rank = memRefType.getRank();
 
   SmallVector<Value, 4> lbs, ubs, steps;
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value one = b.create<arith::ConstantIndexOp>(1);
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value one = arith::ConstantIndexOp::create(b, 1);
 
   // Make sure we have enough loops to use all thread dimensions, these trivial
   // loops should be outermost and therefore inserted first.
@@ -59,8 +59,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
   auto indexType = b.getIndexType();
   SmallVector<Value, 3> threadIds, blockDims;
   for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z}) {
-    threadIds.push_back(b.create<gpu::ThreadIdOp>(indexType, dim));
-    blockDims.push_back(b.create<gpu::BlockDimOp>(indexType, dim));
+    threadIds.push_back(gpu::ThreadIdOp::create(b, indexType, dim));
+    blockDims.push_back(gpu::BlockDimOp::create(b, indexType, dim));
   }
 
   // Produce the loop nest with copies.
@@ -70,8 +70,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
       [&](OpBuilder &b, Location loc, ValueRange loopIvs) {
         ivs.assign(loopIvs.begin(), loopIvs.end());
         auto activeIvs = llvm::ArrayRef(ivs).take_back(rank);
-        Value loaded = b.create<memref::LoadOp>(loc, from, activeIvs);
-        b.create<memref::StoreOp>(loc, loaded, to, activeIvs);
+        Value loaded = memref::LoadOp::create(b, loc, from, activeIvs);
+        memref::StoreOp::create(b, loc, loaded, to, activeIvs);
       });
 
   // Map the innermost loops to threads in reverse order.
@@ -131,10 +131,10 @@ static void insertCopies(Region &region, Location loc, Value from, Value to) {
 
   auto b = ImplicitLocOpBuilder::atBlockBegin(loc, &region.front());
   insertCopyLoops(b, from, to);
-  b.create<gpu::BarrierOp>();
+  gpu::BarrierOp::create(b, );
 
   b.setInsertionPoint(&region.front().back());
-  b.create<gpu::BarrierOp>();
+  gpu::BarrierOp::create(b, );
   insertCopyLoops(b, to, from);
 }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
index 9a69e6dde4274..dd164e5aaa7c4 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
@@ -108,7 +108,7 @@ LogicalResult moduleSerializer(GPUModuleOp op,
       !handler && moduleHandler)
     handler = moduleHandler;
   builder.setInsertionPointAfter(op);
-  builder.create<gpu::BinaryOp>(op.getLoc(), op.getName(), handler,
+  gpu::BinaryOp::create(builder, op.getLoc(), op.getName(), handler,
                                 builder.getArrayAttr(objects));
   op->erase();
   return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
index 171e64346f155..5a55a39a7748e 100644
--- a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
@@ -48,10 +48,10 @@ struct PromoteShuffleToSwizzlePattern
                                          "offset must be in the range [0, 31]");
 
     Location loc = op.getLoc();
-    Value res = rewriter.create<amdgpu::SwizzleBitModeOp>(
+    Value res = amdgpu::SwizzleBitModeOp::create(rewriter,
         loc, op.getResult(0).getType(), op.getValue(), /*andMask=*/31,
         /*orMask=*/0, /*xorMask=*/offsetValue);
-    Value valid = rewriter.create<arith::ConstantIntOp>(loc, 1, /*width*/ 1);
+    Value valid = arith::ConstantIntOp::create(rewriter, loc, 1, /*width*/ 1);
     rewriter.replaceOp(op, {res, valid});
     return success();
   }
diff --git a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
index 2d6df0ff6d02d..7aa5288d8260d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
@@ -47,16 +47,16 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
 
     // Float types must be converted to i64 to extract the bits.
     if (isa<FloatType>(valueType))
-      value = rewriter.create<arith::BitcastOp>(valueLoc, i64, value);
+      value = arith::BitcastOp::create(rewriter, valueLoc, i64, value);
 
     // Get the low bits by trunc(value).
-    lo = rewriter.create<arith::TruncIOp>(valueLoc, i32, value);
+    lo = arith::TruncIOp::create(rewriter, valueLoc, i32, value);
 
     // Get the high bits by trunc(value >> 32).
-    auto c32 = rewriter.create<arith::ConstantOp>(
+    auto c32 = arith::ConstantOp::create(rewriter,
         valueLoc, rewriter.getIntegerAttr(i64, 32));
-    hi = rewriter.create<arith::ShRUIOp>(valueLoc, value, c32);
-    hi = rewriter.create<arith::TruncIOp>(valueLoc, i32, hi);
+    hi = arith::ShRUIOp::create(rewriter, valueLoc, value, c32);
+    hi = arith::TruncIOp::create(rewriter, valueLoc, i32, hi);
 
     // Shuffle the values.
     ValueRange loRes =
@@ -71,21 +71,21 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
             .getResults();
 
     // Convert lo back to i64.
-    lo = rewriter.create<arith::ExtUIOp>(valueLoc, i64, loRes[0]);
+    lo = arith::ExtUIOp::create(rewriter, valueLoc, i64, loRes[0]);
 
     // Convert hi back to i64.
-    hi = rewriter.create<arith::ExtUIOp>(valueLoc, i64, hiRes[0]);
-    hi = rewriter.create<arith::ShLIOp>(valueLoc, hi, c32);
+    hi = arith::ExtUIOp::create(rewriter, valueLoc, i64, hiRes[0]);
+    hi = arith::ShLIOp::create(rewriter, valueLoc, hi, c32);
 
     // Obtain the shuffled bits hi | lo.
-    value = rewriter.create<arith::OrIOp>(loc, hi, lo);
+    value = arith::OrIOp::create(rewriter, loc, hi, lo);
 
     // Convert the value back to float.
     if (isa<FloatType>(valueType))
-      value = rewriter.create<arith::BitcastOp>(valueLoc, valueType, value);
+      value = arith::BitcastOp::create(rewriter, valueLoc, valueType, value);
 
     // Obtain the shuffle validity by combining both validities.
-    auto validity = rewriter.create<arith::AndIOp>(loc, loRes[1], hiRes[1]);
+    auto validity = arith::AndIOp::create(rewriter, loc, loRes[1], hiRes[1]);
 
     // Replace the op.
     rewriter.replaceOp(op, {value, validity});
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
index 05631ad87dd71..f401e6a267eef 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
@@ -54,22 +54,22 @@ struct GpuSubgroupIdRewriter final : OpRewritePattern<gpu::SubgroupIdOp> {
     Location loc = op->getLoc();
     Type indexType = rewriter.getIndexType();
 
-    Value dimX = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::x);
-    Value dimY = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::y);
-    Value tidX = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
-    Value tidY = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::y);
-    Value tidZ = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::z);
+    Value dimX = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::x);
+    Value dimY = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::y);
+    Value tidX = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
+    Value tidY = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::y);
+    Value tidZ = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::z);
 
-    Value dimYxIdZ = rewriter.create<arith::MulIOp>(loc, indexType, dimY, tidZ);
+    Value dimYxIdZ = arith::MulIOp::create(rewriter, loc, indexType, dimY, tidZ);
     Value dimYxIdZPlusIdY =
-        rewriter.create<arith::AddIOp>(loc, indexType, dimYxIdZ, tidY);
+        arith::AddIOp::create(rewriter, loc, indexType, dimYxIdZ, tidY);
     Value dimYxIdZPlusIdYTimesDimX =
-        rewriter.create<arith::MulIOp>(loc, indexType, dimX, dimYxIdZPlusIdY);
-    Value IdXPlusDimYxIdZPlusIdYTimesDimX = rewriter.create<arith::AddIOp>(
+        arith::MulIOp::create(rewriter, loc, indexType, dimX, dimYxIdZPlusIdY);
+    Value IdXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(rewriter,
         loc, indexType, tidX, dimYxIdZPlusIdYTimesDimX);
-    Value subgroupSize = rewriter.create<gpu::SubgroupSizeOp>(
+    Value subgroupSize = gpu::SubgroupSizeOp::create(rewriter,
         loc, rewriter.getIndexType(), /*upper_bound = */ nullptr);
-    Value subgroupIdOp = rewriter.create<arith::DivUIOp>(
+    Value subgroupIdOp = arith::DivUIOp::create(rewriter,
         loc, indexType, IdXPlusDimYxIdZPlusIdYTimesDimX, subgroupSize);
     rewriter.replaceOp(op, {subgroupIdOp});
     return success();
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index 1b3d13623c548..7290f92f6fe08 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -79,7 +79,7 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
 
     Location loc = op.getLoc();
     Value res =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(vecTy));
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(vecTy));
 
     for (unsigned i = 0; i != numNewReductions; ++i) {
       int64_t startIdx = i * elementsPerShuffle;
@@ -90,22 +90,22 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
       Value extracted;
       if (numElems == 1) {
         extracted =
-            rewriter.create<vector::ExtractOp>(loc, op.getValue(), startIdx);
+            vector::ExtractOp::create(rewriter, loc, op.getValue(), startIdx);
       } else {
-        extracted = rewriter.create<vector::ExtractStridedSliceOp>(
+        extracted = vector::ExtractStridedSliceOp::create(rewriter,
             loc, op.getValue(), /*offsets=*/startIdx, /*sizes=*/numElems,
             /*strides=*/1);
       }
 
-      Value reduce = rewriter.create<gpu::SubgroupReduceOp>(
+      Value reduce = gpu::SubgroupReduceOp::create(rewriter,
           loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
           op.getClusterStride());
       if (numElems == 1) {
-        res = rewriter.create<vector::InsertOp>(loc, reduce, res, startIdx);
+        res = vector::InsertOp::create(rewriter, loc, reduce, res, startIdx);
         continue;
       }
 
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, reduce, res, /*offsets=*/startIdx, /*strides=*/1);
     }
 
@@ -138,8 +138,8 @@ struct ScalarizeSingleElementReduce final
     assert(vecTy.getRank() == 1 && "Unexpected vector type");
     assert(!vecTy.isScalable() && "Unexpected vector type");
     Location loc = op.getLoc();
-    Value extracted = rewriter.create<vector::ExtractOp>(loc, op.getValue(), 0);
-    Value reduce = rewriter.create<gpu::SubgroupReduceOp>(
+    Value extracted = vector::ExtractOp::create(rewriter, loc, op.getValue(), 0);
+    Value reduce = gpu::SubgroupReduceOp::create(rewriter,
         loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
         op.getClusterStride());
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, vecTy, reduce);
@@ -254,14 +254,14 @@ struct ScalarSubgroupReduceToShuffles final
     auto packFn = [loc, &rewriter, equivIntType,
                    shuffleIntType](Value unpackedVal) -> Value {
       auto asInt =
-          rewriter.create<arith::BitcastOp>(loc, equivIntType, unpackedVal);
-      return rewriter.create<arith::ExtUIOp>(loc, shuffleIntType, asInt);
+          arith::BitcastOp::create(rewriter, loc, equivIntType, unpackedVal);
+      return arith::ExtUIOp::create(rewriter, loc, shuffleIntType, asInt);
     };
     auto unpackFn = [loc, &rewriter, equivIntType,
                      valueTy](Value packedVal) -> Value {
       auto asInt =
-          rewriter.create<arith::TruncIOp>(loc, equivIntType, packedVal);
-      return rewriter.create<arith::BitcastOp>(loc, valueTy, asInt);
+          arith::TruncIOp::create(rewriter, loc, equivIntType, packedVal);
+      return arith::BitcastOp::create(rewriter, loc, valueTy, asInt);
     };
 
     rewriter.replaceOp(
@@ -326,9 +326,9 @@ struct VectorSubgroupReduceToShuffles final
         static_cast<int64_t>(elementsPerShuffle), vecTy.getElementType());
     Value extendedInput = op.getValue();
     if (vecBitwidth < shuffleBitwidth) {
-      auto zero = rewriter.create<arith::ConstantOp>(
+      auto zero = arith::ConstantOp::create(rewriter,
           loc, rewriter.getZeroAttr(extendedVecTy));
-      extendedInput = rewriter.create<vector::InsertStridedSliceOp>(
+      extendedInput = vector::InsertStridedSliceOp::create(rewriter,
           loc, extendedInput, zero, /*offsets=*/0, /*strides=*/1);
     }
 
@@ -337,21 +337,21 @@ struct VectorSubgroupReduceToShuffles final
 
     auto packFn = [loc, &rewriter, shuffleVecType](Value unpackedVal) -> Value {
       auto asIntVec =
-          rewriter.create<vector::BitCastOp>(loc, shuffleVecType, unpackedVal);
-      return rewriter.create<vector::ExtractOp>(loc, asIntVec, 0);
+          vector::BitCastOp::create(rewriter, loc, shuffleVecType, unpackedVal);
+      return vector::ExtractOp::create(rewriter, loc, asIntVec, 0);
     };
     auto unpackFn = [loc, &rewriter, shuffleVecType,
                      extendedVecTy](Value packedVal) -> Value {
       auto asIntVec =
-          rewriter.create<vector::BroadcastOp>(loc, shuffleVecType, packedVal);
-      return rewriter.create<vector::BitCastOp>(loc, extendedVecTy, asIntVec);
+          vector::BroadcastOp::create(rewriter, loc, shuffleVecType, packedVal);
+      return vector::BitCastOp::create(rewriter, loc, extendedVecTy, asIntVec);
     };
 
     Value res = createSubgroupShuffleReduction(
         rewriter, loc, extendedInput, op.getOp(), *ci, packFn, unpackFn);
 
     if (vecBitwidth < shuffleBitwidth) {
-      res = rewriter.create<vector::ExtractStridedSliceOp>(
+      res = vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/0, /*sizes=*/vecTy.getNumElements(),
           /*strides=*/1);
     }
@@ -378,7 +378,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   const bool boundCtrl = true;
   if (ci.clusterSize >= 2) {
     // Perform reduction between all lanes N <-> N+1.
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({1, 0, 3, 2}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -387,7 +387,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
 
   if (ci.clusterSize >= 4) {
     // Perform reduction between all lanes N <-> N+2.
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({2, 3, 0, 1}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -396,7 +396,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 8) {
     // Perform reduction between all lanes N <-> 7-N,
     // e.g lane[0] <-> lane[7], lane[1] <-> lane[6]..., lane[3] <-> lane[4].
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::row_half_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -405,7 +405,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 16) {
     // Perform reduction between all lanes N <-> 15-N,
     // e.g lane[0] <-> lane[15], lane[1] <-> lane[14]..., lane[7] <-> lane[8].
-    dpp = rewriter.create<amdgpu::DPPOp>(
+    dpp = amdgpu::DPPOp::create(rewriter,
         loc, res.getType(), res, res, amdgpu::DPPPerm::row_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
@@ -415,7 +415,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
     if (chipset.majorVersion <= 9) {
       // Broadcast last value from each row to next row.
       // Use row mask to avoid polluting rows 1 and 3.
-      dpp = rewriter.create<amdgpu::DPPOp>(
+      dpp = amdgpu::DPPOp::create(rewriter,
           loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_15,
           rewriter.getUnitAttr(), 0xa, allBanks,
           /*bound_ctrl*/ false);
@@ -423,9 +423,9 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
           rewriter, loc, gpu::convertReductionKind(mode), res, dpp);
     } else if (chipset.majorVersion <= 12) {
       // Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2).
-      Value uint32Max = rewriter.create<arith::ConstantOp>(
+      Value uint32Max = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
-      dpp = rewriter.create<ROCDL::PermlaneX16Op>(loc, res.getType(), res, res,
+      dpp = ROCDL::PermlaneX16Op::create(rewriter, loc, res.getType(), res, res,
                                                   uint32Max, uint32Max,
                                                   /*fi=*/true,
                                                   /*bound_ctrl=*/false);
@@ -437,37 +437,37 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
               "this device.");
     }
     if (ci.subgroupSize == 32) {
-      Value lane31 = rewriter.create<arith::ConstantOp>(
+      Value lane31 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      res = rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane31);
+      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
     }
   }
   if (ci.clusterSize >= 64) {
     if (chipset.majorVersion <= 9) {
       // Broadcast 31st lane value to rows 2 and 3.
-      dpp = rewriter.create<amdgpu::DPPOp>(
+      dpp = amdgpu::DPPOp::create(rewriter,
           loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_31,
           rewriter.getUnitAttr(), 0xf, allBanks,
           /*bound_ctrl*/ true);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), dpp, res);
       // Obtain reduction from last rows, the previous rows are polluted.
-      Value lane63 = rewriter.create<arith::ConstantOp>(
+      Value lane63 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
-      res = rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane63);
+      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
 
     } else if (chipset.majorVersion <= 12) {
       // Assume reduction across 32 lanes has been done.
       // Perform final reduction manually by summing values in lane 0 and
       // lane 32.
-      Value lane31 = rewriter.create<arith::ConstantOp>(
+      Value lane31 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      Value lane63 = rewriter.create<arith::ConstantOp>(
+      Value lane63 = arith::ConstantOp::create(rewriter,
           loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
       lane31 =
-          rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane31);
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
       lane63 =
-          rewriter.create<ROCDL::ReadlaneOp>(loc, res.getType(), res, lane63);
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), lane31, lane63);
     } else {
diff --git a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
index 29f6f32892f72..473a2e515c12b 100644
--- a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
+++ b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
@@ -27,7 +27,7 @@ WarpDistributionPattern::moveRegionToNewWarpOpAndReplaceReturns(
   // Create a new op before the existing one, with the extra operands.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(warpOp);
-  auto newWarpOp = rewriter.create<WarpExecuteOnLane0Op>(
+  auto newWarpOp = WarpExecuteOnLane0Op::create(rewriter,
       warpOp.getLoc(), newReturnTypes, warpOp.getLaneid(), warpOp.getWarpSize(),
       warpOp.getArgs(), warpOp.getBody()->getArgumentTypes());
 
@@ -124,7 +124,7 @@ bool WarpDistributionPattern::delinearizeLaneId(
 
   int64_t usedThreads = 1;
 
-  Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
   delinearizedIds.assign(sizes.size(), zero);
 
   for (int i = sizes.size() - 1; i >= 0; --i) {
diff --git a/mlir/lib/Dialect/Index/IR/IndexOps.cpp b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
index bab9e2852a460..7cab93709b9c6 100644
--- a/mlir/lib/Dialect/Index/IR/IndexOps.cpp
+++ b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
@@ -36,7 +36,7 @@ Operation *IndexDialect::materializeConstant(OpBuilder &b, Attribute value,
   if (auto boolValue = dyn_cast<BoolAttr>(value)) {
     if (!type.isSignlessInteger(1))
       return nullptr;
-    return b.create<BoolConstantOp>(loc, type, boolValue);
+    return BoolConstantOp::create(b, loc, type, boolValue);
   }
 
   // Materialize integer attributes as `index`.
@@ -46,7 +46,7 @@ Operation *IndexDialect::materializeConstant(OpBuilder &b, Attribute value,
       return nullptr;
     assert(indexValue.getValue().getBitWidth() ==
            IndexType::kInternalStorageBitWidth);
-    return b.create<ConstantOp>(loc, indexValue);
+    return ConstantOp::create(b, loc, indexValue);
   }
 
   return nullptr;
@@ -715,10 +715,10 @@ LogicalResult CmpOp::canonicalize(CmpOp op, PatternRewriter &rewriter) {
 
   index::CmpOp newCmp;
   if (rhsIsZero)
-    newCmp = rewriter.create<index::CmpOp>(op.getLoc(), op.getPred(),
+    newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
                                            subOp.getLhs(), subOp.getRhs());
   else
-    newCmp = rewriter.create<index::CmpOp>(op.getLoc(), op.getPred(),
+    newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
                                            subOp.getRhs(), subOp.getLhs());
   rewriter.replaceOp(op, newCmp);
   return success();
diff --git a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
index c17ef1029faf6..b1d376c7c03db 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
@@ -90,7 +90,7 @@ void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
     }
     for (auto [idx, t] : llvm::enumerate(stype.getBody())) {
       if (itype != PTXRegisterMod::Write) {
-        Value extractValue = rewriter.create<LLVM::ExtractValueOp>(
+        Value extractValue = LLVM::ExtractValueOp::create(rewriter,
             interfaceOp->getLoc(), v, idx);
         addValue(extractValue);
       }
@@ -132,7 +132,7 @@ LLVM::InlineAsmOp PtxBuilder::build() {
   // Replace all % with $
   llvm::replace(ptxInstruction, '%', '$');
 
-  return rewriter.create<LLVM::InlineAsmOp>(
+  return LLVM::InlineAsmOp::create(rewriter,
       interfaceOp->getLoc(),
       /*result types=*/resultTypes,
       /*operands=*/ptxOperands,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 89f765dacda35..7d3f59d128367 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -89,7 +89,7 @@ mlir::LLVM::lookupOrCreateFn(OpBuilder &b, Operation *moduleOp, StringRef name,
   OpBuilder::InsertionGuard g(b);
   assert(!moduleOp->getRegion(0).empty() && "expected non-empty region");
   b.setInsertionPointToStart(&moduleOp->getRegion(0).front());
-  auto funcOp = b.create<LLVM::LLVMFuncOp>(
+  auto funcOp = LLVM::LLVMFuncOp::create(b,
       moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 62dce32bc4531..884ae4a2eac64 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3309,7 +3309,7 @@ bool LLVM::ConstantOp::isBuildableWith(Attribute value, Type type) {
 ConstantOp LLVM::ConstantOp::materialize(OpBuilder &builder, Attribute value,
                                          Type type, Location loc) {
   if (isBuildableWith(value, type))
-    return builder.create<LLVM::ConstantOp>(loc, cast<TypedAttr>(value));
+    return LLVM::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
   return nullptr;
 }
 
@@ -4275,13 +4275,13 @@ Operation *LLVMDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // a builtin zero attribute and thus will materialize as a llvm.mlir.constant.
   if (auto symbol = dyn_cast<FlatSymbolRefAttr>(value))
     if (isa<LLVM::LLVMPointerType>(type))
-      return builder.create<LLVM::AddressOfOp>(loc, type, symbol);
+      return LLVM::AddressOfOp::create(builder, loc, type, symbol);
   if (isa<LLVM::UndefAttr>(value))
-    return builder.create<LLVM::UndefOp>(loc, type);
+    return LLVM::UndefOp::create(builder, loc, type);
   if (isa<LLVM::PoisonAttr>(value))
-    return builder.create<LLVM::PoisonOp>(loc, type);
+    return LLVM::PoisonOp::create(builder, loc, type);
   if (isa<LLVM::ZeroAttr>(value))
-    return builder.create<LLVM::ZeroOp>(loc, type);
+    return LLVM::ZeroOp::create(builder, loc, type);
   // Otherwise try materializing it as a regular llvm.mlir.constant op.
   return LLVM::ConstantOp::materialize(builder, value, type, loc);
 }
@@ -4304,15 +4304,15 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
   MLIRContext *ctx = builder.getContext();
   auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
-  auto global = moduleBuilder.create<LLVM::GlobalOp>(
+  auto global = LLVM::GlobalOp::create(moduleBuilder,
       loc, type, /*isConstant=*/true, linkage, name,
       builder.getStringAttr(value), /*alignment=*/0);
 
   LLVMPointerType ptrType = LLVMPointerType::get(ctx);
   // Get the pointer to the first character in the global string.
   Value globalPtr =
-      builder.create<LLVM::AddressOfOp>(loc, ptrType, global.getSymNameAttr());
-  return builder.create<LLVM::GEPOp>(loc, ptrType, type, globalPtr,
+      LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
+  return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
                                      ArrayRef<GEPArg>{0, 0});
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index bc451f8b028fc..98fb9b09f0cd5 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -37,7 +37,7 @@ llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() {
 
 Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot,
                                       OpBuilder &builder) {
-  return builder.create<LLVM::UndefOp>(getLoc(), slot.elemType);
+  return LLVM::UndefOp::create(builder, getLoc(), slot.elemType);
 }
 
 void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
@@ -45,7 +45,7 @@ void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
                                          OpBuilder &builder) {
   for (Operation *user : getOperation()->getUsers())
     if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
-      builder.create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
+      LLVM::DbgValueOp::create(builder, declareOp.getLoc(), argument,
                                        declareOp.getVarInfo(),
                                        declareOp.getLocationExpr());
 }
@@ -89,7 +89,7 @@ DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
   for (Attribute index : usedIndices) {
     Type elemType = destructurableType.getTypeAtIndex(index);
     assert(elemType && "used index must exist");
-    auto subAlloca = builder.create<LLVM::AllocaOp>(
+    auto subAlloca = LLVM::AllocaOp::create(builder,
         getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
         getArraySize());
     newAllocators.push_back(subAlloca);
@@ -260,13 +260,13 @@ static Value createExtractAndCast(OpBuilder &builder, Location loc,
   // Truncate the integer if the size of the target is less than the value.
   if (isBigEndian(dataLayout)) {
     uint64_t shiftAmount = srcTypeSize - targetTypeSize;
-    auto shiftConstant = builder.create<LLVM::ConstantOp>(
+    auto shiftConstant = LLVM::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(srcType, shiftAmount));
     replacement =
         builder.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
   }
 
-  replacement = builder.create<LLVM::TruncOp>(
+  replacement = LLVM::TruncOp::create(builder,
       loc, builder.getIntegerType(targetTypeSize), replacement);
 
   // Now cast the integer to the actual target type if required.
@@ -304,7 +304,7 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
     // On big endian systems, a store to the base pointer overwrites the most
     // significant bits. To accomodate for this, the stored value needs to be
     // shifted into the according position.
-    Value bigEndianShift = builder.create<LLVM::ConstantOp>(
+    Value bigEndianShift = LLVM::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
     valueAsInt =
         builder.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
@@ -325,7 +325,7 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
   }
 
   // Mask out the affected bits ...
-  Value mask = builder.create<LLVM::ConstantOp>(
+  Value mask = LLVM::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
   Value masked = builder.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
 
@@ -644,7 +644,7 @@ DeletionKind LLVM::DbgValueOp::removeBlockingUses(
   // debug local variable info. This allows the debugger to inform the user that
   // the variable has been optimized out.
   auto undef =
-      builder.create<UndefOp>(getValue().getLoc(), getValue().getType());
+      UndefOp::create(builder, getValue().getLoc(), getValue().getType());
   getValueMutable().assign(undef);
   return DeletionKind::Keep;
 }
@@ -655,7 +655,7 @@ void LLVM::DbgDeclareOp::visitReplacedValues(
     ArrayRef<std::pair<Operation *, Value>> definitions, OpBuilder &builder) {
   for (auto [op, value] : definitions) {
     builder.setInsertionPointAfter(op);
-    builder.create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(),
+    LLVM::DbgValueOp::create(builder, getLoc(), value, getVarInfo(),
                                      getLocationExpr());
   }
 }
@@ -978,7 +978,7 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetOp toReplace,
               IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize))
           .getResult();
 
-  builder.create<LLVM::MemsetOp>(toReplace.getLoc(), subslots.at(index).ptr,
+  LLVM::MemsetOp::create(builder, toReplace.getLoc(), subslots.at(index).ptr,
                                  toReplace.getVal(), newMemsetSizeValue,
                                  toReplace.getIsVolatile());
 }
@@ -991,7 +991,7 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
   auto newMemsetSizeValue =
       IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize);
 
-  builder.create<LLVM::MemsetInlineOp>(
+  LLVM::MemsetInlineOp::create(builder,
       toReplace.getLoc(), subslots.at(index).ptr, toReplace.getVal(),
       newMemsetSizeValue, toReplace.getIsVolatile());
 }
@@ -1063,7 +1063,7 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       APInt memsetVal(/*numBits=*/width, /*val=*/0);
       for (unsigned loBit = 0; loBit < width; loBit += 8)
         memsetVal.insertBits(constantPattern.getValue(), loBit);
-      return builder.create<LLVM::ConstantOp>(
+      return LLVM::ConstantOp::create(builder,
           op.getLoc(), IntegerAttr::get(intType, memsetVal));
     }
 
@@ -1075,14 +1075,14 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
     // value and or-ing it with the previous value.
     uint64_t coveredBits = 8;
     Value currentValue =
-        builder.create<LLVM::ZExtOp>(op.getLoc(), intType, op.getVal());
+        LLVM::ZExtOp::create(builder, op.getLoc(), intType, op.getVal());
     while (coveredBits < width) {
       Value shiftBy =
-          builder.create<LLVM::ConstantOp>(op.getLoc(), intType, coveredBits);
+          LLVM::ConstantOp::create(builder, op.getLoc(), intType, coveredBits);
       Value shifted =
-          builder.create<LLVM::ShlOp>(op.getLoc(), currentValue, shiftBy);
+          LLVM::ShlOp::create(builder, op.getLoc(), currentValue, shiftBy);
       currentValue =
-          builder.create<LLVM::OrOp>(op.getLoc(), currentValue, shifted);
+          LLVM::OrOp::create(builder, op.getLoc(), currentValue, shifted);
       coveredBits *= 2;
     }
 
@@ -1094,7 +1094,7 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       })
       .Case([&](FloatType type) -> Value {
         Value intVal = buildMemsetValue(type.getWidth());
-        return builder.create<LLVM::BitcastOp>(op.getLoc(), type, intVal);
+        return LLVM::BitcastOp::create(builder, op.getLoc(), type, intVal);
       })
       .Default([](Type) -> Value {
         llvm_unreachable(
@@ -1282,7 +1282,7 @@ static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot) {
 template <class MemcpyLike>
 static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot,
                              OpBuilder &builder) {
-  return builder.create<LLVM::LoadOp>(op.getLoc(), slot.elemType, op.getSrc());
+  return LLVM::LoadOp::create(builder, op.getLoc(), slot.elemType, op.getSrc());
 }
 
 template <class MemcpyLike>
@@ -1309,7 +1309,7 @@ memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot,
                          const SmallPtrSetImpl<OpOperand *> &blockingUses,
                          OpBuilder &builder, Value reachingDefinition) {
   if (op.loadsFrom(slot))
-    builder.create<LLVM::StoreOp>(op.getLoc(), reachingDefinition, op.getDst());
+    LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition, op.getDst());
   return DeletionKind::Delete;
 }
 
@@ -1354,10 +1354,10 @@ template <class MemcpyLike>
 void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                MemcpyLike toReplace, Value dst, Value src,
                                Type toCpy, bool isVolatile) {
-  Value memcpySize = builder.create<LLVM::ConstantOp>(
+  Value memcpySize = LLVM::ConstantOp::create(builder,
       toReplace.getLoc(), IntegerAttr::get(toReplace.getLen().getType(),
                                            layout.getTypeSize(toCpy)));
-  builder.create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize,
+  MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize,
                              isVolatile);
 }
 
@@ -1367,7 +1367,7 @@ void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                Value src, Type toCpy, bool isVolatile) {
   Type lenType = IntegerType::get(toReplace->getContext(),
                                   toReplace.getLen().getBitWidth());
-  builder.create<LLVM::MemcpyInlineOp>(
+  LLVM::MemcpyInlineOp::create(builder,
       toReplace.getLoc(), dst, src,
       IntegerAttr::get(lenType, layout.getTypeSize(toCpy)), isVolatile);
 }
@@ -1409,7 +1409,7 @@ memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
     SmallVector<LLVM::GEPArg> gepIndices{
         0, static_cast<int32_t>(
                cast<IntegerAttr>(index).getValue().getZExtValue())};
-    Value subslotPtrInOther = builder.create<LLVM::GEPOp>(
+    Value subslotPtrInOther = LLVM::GEPOp::create(builder,
         op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()), slot.elemType,
         isDst ? op.getSrc() : op.getDst(), gepIndices);
 
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
index 6fbb0d24826d0..1b1268da15646 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
@@ -28,13 +28,13 @@ static void addComdat(LLVM::LLVMFuncOp &op, OpBuilder &builder,
     PatternRewriter::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(module.getBody());
     comdatOp =
-        builder.create<mlir::LLVM::ComdatOp>(module.getLoc(), comdatName);
+        mlir::LLVM::ComdatOp::create(builder, module.getLoc(), comdatName);
     symbolTable.insert(comdatOp);
   }
 
   PatternRewriter::InsertionGuard guard(builder);
   builder.setInsertionPointToStart(&comdatOp.getBody().back());
-  auto selectorOp = builder.create<mlir::LLVM::ComdatSelectorOp>(
+  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(builder,
       comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
   op.setComdatAttr(mlir::SymbolRefAttr::get(
       builder.getContext(), comdatName,
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index 7f3afffc9645e..2ecef4b4c8a51 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -113,18 +113,18 @@ handleInlinedAllocas(Operation *call,
     // scope if some are already present in the body of the caller. This is not
     // invalid IR, but LLVM cleans these up in InstCombineCalls.cpp, along with
     // other cases where the stacksave/stackrestore is redundant.
-    stackPtr = builder.create<LLVM::StackSaveOp>(
+    stackPtr = LLVM::StackSaveOp::create(builder,
         call->getLoc(), LLVM::LLVMPointerType::get(call->getContext()));
   }
   builder.setInsertionPointToStart(callerEntryBlock);
   for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
-    auto newConstant = builder.create<LLVM::ConstantOp>(
+    auto newConstant = LLVM::ConstantOp::create(builder,
         allocaOp->getLoc(), allocaOp.getArraySize().getType(), arraySize);
     // Insert a lifetime start intrinsic where the alloca was before moving it.
     if (shouldInsertLifetime) {
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(allocaOp);
-      builder.create<LLVM::LifetimeStartOp>(
+      LLVM::LifetimeStartOp::create(builder,
           allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
           allocaOp.getResult());
     }
@@ -139,10 +139,10 @@ handleInlinedAllocas(Operation *call,
       continue;
     builder.setInsertionPoint(block.getTerminator());
     if (hasDynamicAlloca)
-      builder.create<LLVM::StackRestoreOp>(call->getLoc(), stackPtr);
+      LLVM::StackRestoreOp::create(builder, call->getLoc(), stackPtr);
     for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
       if (shouldInsertLifetime)
-        builder.create<LLVM::LifetimeEndOp>(
+        LLVM::LifetimeEndOp::create(builder,
             allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
             allocaOp.getResult());
     }
@@ -603,15 +603,15 @@ static Value handleByValArgumentInit(OpBuilder &builder, Location loc,
     OpBuilder::InsertionGuard insertionGuard(builder);
     Block *entryBlock = &(*argument.getParentRegion()->begin());
     builder.setInsertionPointToStart(entryBlock);
-    Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
+    Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
                                                  builder.getI64IntegerAttr(1));
-    allocaOp = builder.create<LLVM::AllocaOp>(
+    allocaOp = LLVM::AllocaOp::create(builder,
         loc, argument.getType(), elementType, one, targetAlignment);
   }
   // Copy the pointee to the newly allocated value.
-  Value copySize = builder.create<LLVM::ConstantOp>(
+  Value copySize = LLVM::ConstantOp::create(builder,
       loc, builder.getI64Type(), builder.getI64IntegerAttr(elementTypeSize));
-  builder.create<LLVM::MemcpyOp>(loc, allocaOp, argument, copySize,
+  LLVM::MemcpyOp::create(builder, loc, allocaOp, argument, copySize,
                                  /*isVolatile=*/false);
   return allocaOp;
 }
@@ -747,7 +747,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    builder.create<LLVM::BrOp>(op->getLoc(), returnOp.getOperands(), newDest);
+    LLVM::BrOp::create(builder, op->getLoc(), returnOp.getOperands(), newDest);
     op->erase();
   }
 
@@ -801,7 +801,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
     // and is extremely unlikely to exist in the code prior to inlining, using
     // this to communicate between this method and `processInlinedCallBlocks`.
     // TODO: Fix this by refactoring the inliner interface.
-    auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
+    auto copyOp = LLVM::SSACopyOp::create(builder, call->getLoc(), argument);
     if (argumentAttrs.contains(LLVM::LLVMDialect::getNoAliasAttrName()))
       copyOp->setDiscardableAttr(
           builder.getStringAttr(LLVM::LLVMDialect::getNoAliasAttrName()),
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
index 1a5a6e4ea1a3c..6fa4f6935f11c 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
@@ -58,7 +58,7 @@ static void ensureDistinctSuccessors(Block &bb) {
       terminator->setSuccessor(dummyBlock, position);
       for (BlockArgument arg : successor.first->getArguments())
         dummyBlock->addArgument(arg.getType(), arg.getLoc());
-      builder.create<LLVM::BrOp>(terminator->getLoc(),
+      LLVM::BrOp::create(builder, terminator->getLoc(),
                                  dummyBlock->getArguments(), successor.first);
     }
   }
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
index 8db32ec1526c4..de6c3043bbcdd 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
@@ -59,32 +59,32 @@ LogicalResult ExpandDivF16::matchAndRewrite(LLVM::FDivOp op,
   Type i32Type = rewriter.getI32Type();
 
   // Extend lhs and rhs to fp32.
-  Value lhs = rewriter.create<LLVM::FPExtOp>(loc, f32Type, op.getLhs());
-  Value rhs = rewriter.create<LLVM::FPExtOp>(loc, f32Type, op.getRhs());
+  Value lhs = LLVM::FPExtOp::create(rewriter, loc, f32Type, op.getLhs());
+  Value rhs = LLVM::FPExtOp::create(rewriter, loc, f32Type, op.getRhs());
 
   // float rcp = rcp.approx.ftz.f32(rhs), approx = lhs * rcp.
-  Value rcp = rewriter.create<NVVM::RcpApproxFtzF32Op>(loc, f32Type, rhs);
-  Value approx = rewriter.create<LLVM::FMulOp>(loc, lhs, rcp);
+  Value rcp = NVVM::RcpApproxFtzF32Op::create(rewriter, loc, f32Type, rhs);
+  Value approx = LLVM::FMulOp::create(rewriter, loc, lhs, rcp);
 
   // Refine the approximation with one Newton iteration:
   // float refined = approx + (lhs - approx * rhs) * rcp;
-  Value err = rewriter.create<LLVM::FMAOp>(
-      loc, approx, rewriter.create<LLVM::FNegOp>(loc, rhs), lhs);
-  Value refined = rewriter.create<LLVM::FMAOp>(loc, err, rcp, approx);
+  Value err = LLVM::FMAOp::create(rewriter,
+      loc, approx, LLVM::FNegOp::create(rewriter, loc, rhs), lhs);
+  Value refined = LLVM::FMAOp::create(rewriter, loc, err, rcp, approx);
 
   // Use refined value if approx is normal (exponent neither all 0 or all 1).
-  Value mask = rewriter.create<LLVM::ConstantOp>(
+  Value mask = LLVM::ConstantOp::create(rewriter,
       loc, i32Type, rewriter.getUI32IntegerAttr(0x7f800000));
-  Value cast = rewriter.create<LLVM::BitcastOp>(loc, i32Type, approx);
-  Value exp = rewriter.create<LLVM::AndOp>(loc, i32Type, cast, mask);
-  Value zero = rewriter.create<LLVM::ConstantOp>(
+  Value cast = LLVM::BitcastOp::create(rewriter, loc, i32Type, approx);
+  Value exp = LLVM::AndOp::create(rewriter, loc, i32Type, cast, mask);
+  Value zero = LLVM::ConstantOp::create(rewriter,
       loc, i32Type, rewriter.getUI32IntegerAttr(0));
-  Value pred = rewriter.create<LLVM::OrOp>(
+  Value pred = LLVM::OrOp::create(rewriter,
       loc,
-      rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, exp, zero),
-      rewriter.create<LLVM::ICmpOp>(loc, LLVM::ICmpPredicate::eq, exp, mask));
+      LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, zero),
+      LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, mask));
   Value result =
-      rewriter.create<LLVM::SelectOp>(loc, f32Type, pred, approx, refined);
+      LLVM::SelectOp::create(rewriter, loc, f32Type, pred, approx, refined);
 
   // Replace with trucation back to fp16.
   rewriter.replaceOpWithNewOp<LLVM::FPTruncOp>(op, op.getType(), result);
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 3aa6ac3ea0918..51cc1232df891 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -62,10 +62,10 @@ static OpFoldResult getDimValue(OpBuilder &builder, Location loc, Value v,
   return getAsOpFoldResult(
       TypeSwitch<Type, Value>(v.getType())
           .Case<RankedTensorType>([&](RankedTensorType t) -> Value {
-            return builder.create<tensor::DimOp>(loc, v, dim);
+            return tensor::DimOp::create(builder, loc, v, dim);
           })
           .Case<MemRefType>([&](MemRefType t) -> Value {
-            return builder.create<memref::DimOp>(loc, v, dim);
+            return memref::DimOp::create(builder, loc, v, dim);
           }));
 }
 
@@ -77,11 +77,11 @@ static Operation *getSlice(OpBuilder &b, Location loc, Value source,
                            ArrayRef<OpFoldResult> strides) {
   return TypeSwitch<Type, Operation *>(source.getType())
       .Case<RankedTensorType>([&](RankedTensorType t) -> Operation * {
-        return b.create<tensor::ExtractSliceOp>(loc, source, offsets, sizes,
+        return tensor::ExtractSliceOp::create(b, loc, source, offsets, sizes,
                                                 strides);
       })
       .Case<MemRefType>([&](MemRefType type) -> Operation * {
-        return b.create<memref::SubViewOp>(loc, source, offsets, sizes,
+        return memref::SubViewOp::create(b, loc, source, offsets, sizes,
                                            strides);
       })
       .Default([&](Type t) -> Operation * { return nullptr; });
@@ -453,35 +453,35 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     switch (unaryFn) {
     case UnaryFn::exp:
-      return builder.create<math::ExpOp>(arg.getLoc(), arg);
+      return math::ExpOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::log:
-      return builder.create<math::LogOp>(arg.getLoc(), arg);
+      return math::LogOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::abs:
-      return builder.create<math::AbsFOp>(arg.getLoc(), arg);
+      return math::AbsFOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::ceil:
-      return builder.create<math::CeilOp>(arg.getLoc(), arg);
+      return math::CeilOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::floor:
-      return builder.create<math::FloorOp>(arg.getLoc(), arg);
+      return math::FloorOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::negf:
-      return builder.create<arith::NegFOp>(arg.getLoc(), arg);
+      return arith::NegFOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::reciprocal: {
       Attribute oneAttr = builder.getOneAttr(arg.getType());
-      auto one = builder.create<arith::ConstantOp>(arg.getLoc(),
+      auto one = arith::ConstantOp::create(builder, arg.getLoc(),
                                                    ::cast<TypedAttr>(oneAttr));
-      return builder.create<arith::DivFOp>(arg.getLoc(), one, arg);
+      return arith::DivFOp::create(builder, arg.getLoc(), one, arg);
     }
     case UnaryFn::round:
-      return builder.create<math::RoundOp>(arg.getLoc(), arg);
+      return math::RoundOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::sqrt:
-      return builder.create<math::SqrtOp>(arg.getLoc(), arg);
+      return math::SqrtOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::rsqrt:
-      return builder.create<math::RsqrtOp>(arg.getLoc(), arg);
+      return math::RsqrtOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::square:
-      return builder.create<arith::MulFOp>(arg.getLoc(), arg, arg);
+      return arith::MulFOp::create(builder, arg.getLoc(), arg, arg);
     case UnaryFn::tanh:
-      return builder.create<math::TanhOp>(arg.getLoc(), arg);
+      return math::TanhOp::create(builder, arg.getLoc(), arg);
     case UnaryFn::erf:
-      return builder.create<math::ErfOp>(arg.getLoc(), arg);
+      return math::ErfOp::create(builder, arg.getLoc(), arg);
     }
     if (emitError) {
       emitError() << "unsupported unary function";
@@ -516,17 +516,17 @@ class RegionBuilderHelper {
     switch (binaryFn) {
     case BinaryFn::add:
       if (allComplex)
-        return builder.create<complex::AddOp>(arg0.getLoc(), arg0, arg1);
+        return complex::AddOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::AddFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::AddFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool)
-        return builder.create<arith::OrIOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::AddIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::OrIOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::AddIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::sub:
       if (allComplex)
-        return builder.create<complex::SubOp>(arg0.getLoc(), arg0, arg1);
+        return complex::SubOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::SubFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::SubFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool) {
         if (emitError) {
           emitError() << "unsupported operation: sub with bools";
@@ -534,20 +534,20 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: sub with bools");
       }
-      return builder.create<arith::SubIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::SubIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::mul:
       if (allComplex)
-        return builder.create<complex::MulOp>(arg0.getLoc(), arg0, arg1);
+        return complex::MulOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::MulFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MulFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool)
-        return builder.create<arith::AndIOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MulIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::AndIOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MulIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::div:
       if (allComplex)
-        return builder.create<complex::DivOp>(arg0.getLoc(), arg0, arg1);
+        return complex::DivOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
-        return builder.create<arith::DivFOp>(arg0.getLoc(), arg0, arg1);
+        return arith::DivFOp::create(builder, arg0.getLoc(), arg0, arg1);
       if (allBool) {
         if (emitError) {
           emitError() << "unsupported operation: div with bools";
@@ -555,7 +555,7 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: div with bools");
       }
-      return builder.create<arith::DivSIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::DivSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::div_unsigned:
       if (!allInteger || allBool) {
         if (emitError) {
@@ -564,30 +564,30 @@ class RegionBuilderHelper {
         }
         llvm_unreachable("unsupported operation: unsigned div not on uint");
       }
-      return builder.create<arith::DivUIOp>(arg0.getLoc(), arg0, arg1);
+      return arith::DivUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_signed:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MaximumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MaxSIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MaximumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MaxSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::min_signed:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MinimumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MinSIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MinimumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MinSIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_unsigned:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MaximumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MaxUIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MaximumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MaxUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::min_unsigned:
       assert(!allComplex);
       if (allFloatingPoint)
-        return builder.create<arith::MinimumFOp>(arg0.getLoc(), arg0, arg1);
-      return builder.create<arith::MinUIOp>(arg0.getLoc(), arg0, arg1);
+        return arith::MinimumFOp::create(builder, arg0.getLoc(), arg0, arg1);
+      return arith::MinUIOp::create(builder, arg0.getLoc(), arg0, arg1);
     case BinaryFn::powf:
       assert(allFloatingPoint);
-      return builder.create<math::PowFOp>(arg0.getLoc(), arg0, arg1);
+      return math::PowFOp::create(builder, arg0.getLoc(), arg0, arg1);
     }
     if (emitError) {
       emitError() << "unsupported binary function";
@@ -610,7 +610,7 @@ class RegionBuilderHelper {
     case TernaryFn::select:
       if (!headBool && !(tailFloatingPoint || tailInteger))
         llvm_unreachable("unsupported non numeric type");
-      return builder.create<arith::SelectOp>(arg0.getLoc(), arg0, arg1, arg2);
+      return arith::SelectOp::create(builder, arg0.getLoc(), arg0, arg1, arg2);
     }
     if (emitError) {
       emitError() << "unsupported ternary function";
@@ -639,7 +639,7 @@ class RegionBuilderHelper {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
-    builder.create<YieldOp>(loc, values);
+    YieldOp::create(builder, loc, values);
   }
 
   Value constant(const std::string &value) {
@@ -647,13 +647,13 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
     Attribute valueAttr = parseAttribute(value, builder.getContext());
-    return builder.create<arith::ConstantOp>(loc, ::cast<TypedAttr>(valueAttr));
+    return arith::ConstantOp::create(builder, loc, ::cast<TypedAttr>(valueAttr));
   }
 
   Value index(int64_t dim) {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
-    return builder.create<IndexOp>(builder.getUnknownLoc(), dim);
+    return IndexOp::create(builder, builder.getUnknownLoc(), dim);
   }
 
   Type getIntegerType(unsigned width) {
@@ -749,12 +749,12 @@ struct FoldFillWithTensorReshape : OpRewritePattern<TensorReshapeOp> {
     TensorReshapeOp newInit;
     if constexpr (std::is_same<TensorReshapeOp, tensor::ExpandShapeOp>::value) {
 
-      newInit = rewriter.create<TensorReshapeOp>(
+      newInit = TensorReshapeOp::create(rewriter,
           loc, reshapeOp.getResultType(), oldFill.output(),
           reshapeOp.getReassociation(), reshapeOp.getOutputShape(),
           reshapeOp.getStaticOutputShape());
     } else {
-      newInit = rewriter.create<TensorReshapeOp>(loc, reshapeOp.getResultType(),
+      newInit = TensorReshapeOp::create(rewriter, loc, reshapeOp.getResultType(),
                                                  oldFill.output(),
                                                  reshapeOp.getReassociation());
     }
@@ -786,7 +786,7 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
       return rewriter.notifyMatchFailure(
           padOp, "failed to reify tensor.pad op result shape");
 
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         padOp.getLoc(), reifiedShape.front(),
         padOp.getResultType().getElementType());
     Value replacement =
@@ -795,7 +795,7 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
                             ValueRange{emptyTensor})
             .getResult(0);
     if (replacement.getType() != padOp.getResultType()) {
-      replacement = rewriter.create<tensor::CastOp>(
+      replacement = tensor::CastOp::create(rewriter,
           fillOp.getLoc(), padOp.getResultType(), replacement);
     }
     rewriter.replaceOp(padOp, replacement);
@@ -889,7 +889,7 @@ struct FoldInsertPadIntoFill : public OpRewritePattern<tensor::InsertSliceOp> {
     for (int i = 0, e = srcPadType.getRank(); i < e; ++i) {
       if (srcPadType.isDynamicDim(i)) {
         newSizes.push_back(
-            rewriter.create<tensor::DimOp>(loc, srcPadOp.getSource(), i)
+            tensor::DimOp::create(rewriter, loc, srcPadOp.getSource(), i)
                 .getResult());
       } else {
         newSizes.push_back(rewriter.getIndexAttr(srcPadType.getDimSize(i)));
@@ -942,7 +942,7 @@ static FailureOr<FillOp> foldFillPackIntoFillOp(RewriterBase &rewriter,
   if (!packOpDest.hasOneUse())
     return failure();
 
-  return rewriter.create<linalg::FillOp>(packOp.getLoc(), fillOp.getInputs(),
+  return linalg::FillOp::create(rewriter, packOp.getLoc(), fillOp.getInputs(),
                                          packOp.getDest());
 }
 
@@ -1042,7 +1042,7 @@ struct FoldConcatsOfFill : public OpRewritePattern<tensor::ConcatOp> {
           concatOp, "not all operands are defined by a compatible fill op");
     }
 
-    Value outsConcat = rewriter.create<tensor::ConcatOp>(
+    Value outsConcat = tensor::ConcatOp::create(rewriter,
         concatOp.getLoc(), concatOp.getDim(), allOuts);
     rewriter.replaceOpWithNewOp<linalg::FillOp>(
         concatOp, firstFillOp.getDpsInputOperand(0)->get(), outsConcat);
@@ -1407,13 +1407,13 @@ struct EraseIdentityLinalgOp : public OpRewritePattern<OpTy> {
         // TODO: unify the two ops?
         if (sparse_tensor::getSparseTensorEncoding(returnType) ||
             sparse_tensor::getSparseTensorEncoding(resultType))
-          returnedArg = rewriter.create<sparse_tensor::ConvertOp>(
+          returnedArg = sparse_tensor::ConvertOp::create(rewriter,
               linalgOp.getLoc(), resultType, returnedArg);
         else {
           if (!tensor::CastOp::areCastCompatible(returnedArg.getType(),
                                                  resultType))
             return failure();
-          returnedArg = rewriter.create<tensor::CastOp>(
+          returnedArg = tensor::CastOp::create(rewriter,
               linalgOp.getLoc(), resultType, returnedArg);
         }
       }
@@ -1528,7 +1528,7 @@ static void addBodyWithPayloadOp(OpAsmParser &parser, OperationState &result,
       TypeRange{llvm::cast<ShapedType>(result.operands.back().getType())
                     .getElementType()},
       payloadOpAttrs);
-  b.create<YieldOp>(result.location, payloadOp->getResults());
+  YieldOp::create(b, result.location, payloadOp->getResults());
 }
 
 ParseResult MapOp::parse(OpAsmParser &parser, OperationState &result) {
@@ -1945,7 +1945,7 @@ static void buildIdentityRegion(OpBuilder &builder, Location loc,
   buildGenericRegion(builder, loc, region, inputs, outputs,
                      [](OpBuilder &b, Location loc, ValueRange args) {
                        if (!args.empty())
-                         b.create<linalg::YieldOp>(loc, args[0]);
+                         linalg::YieldOp::create(b, loc, args[0]);
                      });
 }
 
@@ -2138,7 +2138,7 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     unsigned inputRank = broadcastInputTy.getRank();
     for (unsigned i = 0; i < inputRank; ++i) {
       if (broadcastInputTy.isDynamicDim(i)) {
-        dims.push_back(rewriter.create<tensor::DimOp>(loc, broadcastInput, i)
+        dims.push_back(tensor::DimOp::create(rewriter, loc, broadcastInput, i)
                            ->getResult(0));
       } else {
         dims.push_back(IntegerAttr::get(IndexType::get(ctx),
@@ -2147,7 +2147,7 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     }
     SmallVector<OpFoldResult> transposeResultShapes =
         applyPermutation(dims, resultPerms);
-    Value transposeInit = rewriter.create<tensor::EmptyOp>(
+    Value transposeInit = tensor::EmptyOp::create(rewriter,
         transposeOp.getLoc(), transposeResultShapes,
         broadcastInputTy.getElementType());
 
@@ -2547,7 +2547,7 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     // continue to propagate as far up the stack as it can go.
     OpOperand *outOperand = linalgOp.getDpsInitOperand(resultNumber);
     Value newOperand =
-        rewriter.create<tensor::CastOp>(loc, resultType, outOperand->get());
+        tensor::CastOp::create(rewriter, loc, resultType, outOperand->get());
     SmallVector<Value> newOperands = linalgOp.getDpsInputs();
     SmallVector<Value> outputOperands(linalgOp.getDpsInits().begin(),
                                       linalgOp.getDpsInits().end());
@@ -2560,7 +2560,7 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     Operation *newOp = clone(rewriter, linalgOp, resultTypes, newOperands);
 
     // Create a tensor.cast operation back to the original type.
-    Value castBack = rewriter.create<tensor::CastOp>(
+    Value castBack = tensor::CastOp::create(rewriter,
         loc, resultValue.getType(), newOp->getResult(resultNumber));
 
     SmallVector<Value> results(newOp->result_begin(), newOp->result_end());
@@ -2653,7 +2653,7 @@ static void createNewOperandWithStaticSizes(
     changeNeeded = true;
     // Get the new operand value given its size and element type by
     // casting it.
-    Value newOperand = rewriter.create<tensor::CastOp>(loc, resultType, src);
+    Value newOperand = tensor::CastOp::create(rewriter, loc, resultType, src);
     unsigned index = opOperand->getOperandNumber();
     newOperands[index] = newOperand;
   }
@@ -2718,7 +2718,7 @@ struct InferStaticShapeOfOperands : public OpInterfaceRewritePattern<LinalgOp> {
       Type oldType = oldResult.getType();
       replacements.push_back(
           (newType != oldType)
-              ? rewriter.create<tensor::CastOp>(loc, oldType, newResult)
+              ? tensor::CastOp::create(rewriter, loc, oldType, newResult)
               : newResult);
     }
     rewriter.replaceOp(linalgOp, replacements);
@@ -2756,8 +2756,8 @@ SmallVector<Range> SoftmaxOp::getIterationDomain(OpBuilder &builder) {
   int64_t operandRank = getInputOperandRank();
   SmallVector<Range> loopBounds(operandRank);
   Location loc = getLoc();
-  Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-  Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+  Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+  Value one = arith::ConstantIndexOp::create(builder, loc, 1);
   Value source = getInput();
   for (auto dim : llvm::seq<int64_t>(0, operandRank)) {
     loopBounds[dim].offset = zero;
@@ -2924,11 +2924,11 @@ static Value reduce(OpBuilder &builder, Location loc, Value input, Value output,
          "We should have two maps: 1 for the input, 1 for the output");
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
 
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, output.getType(), input, output, indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value result = b.create<T>(loc, args[0], args[1]);
-        b.create<linalg::YieldOp>(loc, result);
+        Value result = T::create(b, loc, args[0], args[1]);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -2947,12 +2947,12 @@ static Value buildSubAndExpOp(OpBuilder &builder, Location loc, Value input,
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
   // Add the affine map for the output argument.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, input.getType(), ValueRange{input, max}, output, indexingMaps,
       iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value diff = b.create<arith::SubFOp>(loc, args[0], args[1]);
-        Value result = b.create<math::ExpOp>(loc, diff);
-        b.create<linalg::YieldOp>(loc, result);
+        Value diff = arith::SubFOp::create(b, loc, args[0], args[1]);
+        Value result = math::ExpOp::create(b, loc, diff);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -2974,12 +2974,12 @@ static Value buildDivOp(OpBuilder &builder, Location loc, Value numerator,
   assert(indexingMaps[0].isIdentity() && "Numerator map should be identity");
   // Add the affine map for the output tensor.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = builder.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(builder,
       loc, numerator.getType(), ValueRange{numerator, denominator}, output,
       indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
-        Value result = b.create<arith::DivFOp>(loc, args[0], args[1]);
-        b.create<linalg::YieldOp>(loc, result);
+        Value result = arith::DivFOp::create(b, loc, args[0], args[1]);
+        linalg::YieldOp::create(b, loc, result);
       });
   return genericOp.getResult(0);
 }
@@ -3015,12 +3015,12 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
   Value output = getOutput();
   dims.erase(dims.begin() + reductionDim);
   // Step 1: Compute max along dim.
-  Value outputReduce = b.create<tensor::EmptyOp>(loc, dims, elementType);
+  Value outputReduce = tensor::EmptyOp::create(b, loc, dims, elementType);
   Value neutralForMaxF = arith::getIdentityValue(arith::AtomicRMWKind::maxnumf,
                                                  elementType, b, loc,
                                                  /*useOnlyFiniteValue=*/true);
   Value neutralForMaxFInit =
-      b.create<linalg::FillOp>(loc, Value{neutralForMaxF}, outputReduce)
+      linalg::FillOp::create(b, loc, Value{neutralForMaxF}, outputReduce)
           .result();
   Value max =
       reduce<arith::MaxNumFOp>(b, loc, input, neutralForMaxFInit, reductionDim);
@@ -3032,7 +3032,7 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
   Value zero = arith::getIdentityValue(arith::AtomicRMWKind::addf, elementType,
                                        b, loc, /*useOnlyFiniteValue=*/true);
   Value zeroInit =
-      b.create<linalg::FillOp>(loc, Value{zero}, outputReduce).result();
+      linalg::FillOp::create(b, loc, Value{zero}, outputReduce).result();
   Value denominator =
       reduce<arith::AddFOp>(b, loc, numerator, zeroInit, reductionDim);
 
@@ -3153,7 +3153,7 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
   int64_t filterRank = getFilterOperandRank();
   SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
   Location loc = getLoc();
-  auto filterSlice = builder.create<tensor::ExtractSliceOp>(
+  auto filterSlice = tensor::ExtractSliceOp::create(builder,
       loc, getFilter(), sliceOffsets, sliceSizes, filterStrides);
   tiledOperands.emplace_back(filterSlice);
 
@@ -3164,7 +3164,7 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -3333,7 +3333,7 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
       {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
   int64_t inputRank = getInputOperandRank();
   SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
-  auto inputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto inputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getInput(), sliceOffsets, sliceSizes, inputStrides);
   tiledOperands.emplace_back(inputSlice);
 
@@ -3344,7 +3344,7 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -3504,7 +3504,7 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
                      sizes[getValueFDim()]});
   int64_t valueRank = getValueOperandRank();
   SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
-  auto valueSlice = builder.create<tensor::ExtractSliceOp>(
+  auto valueSlice = tensor::ExtractSliceOp::create(builder,
       loc, getValue(), sliceOffsets, sliceSizes, sliceStrides);
   tiledOperands.emplace_back(valueSlice);
 
@@ -3515,7 +3515,7 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> strides(outputRank, oneAttr);
-  auto outputSlice = builder.create<tensor::ExtractSliceOp>(
+  auto outputSlice = tensor::ExtractSliceOp::create(builder,
       loc, getOutput(), resultOffsets, resultSizes, strides);
   tiledOperands.emplace_back(outputSlice);
 
@@ -4971,7 +4971,7 @@ Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
            llvm::cast<RankedTensorType>(source.getType()).getShape())) {
     if (ShapedType::isDynamic(value))
       mixedSizes.push_back(
-          b.create<tensor::DimOp>(loc, source, index).getResult());
+          tensor::DimOp::create(b, loc, source, index).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(value));
   }
@@ -4985,7 +4985,7 @@ Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
 
   mixedSizes.append(innerTileSizes.begin(), innerTileSizes.end());
   auto elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
-  return b.create<tensor::EmptyOp>(loc, mixedSizes, elemType);
+  return tensor::EmptyOp::create(b, loc, mixedSizes, elemType);
 }
 
 PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
@@ -4996,7 +4996,7 @@ PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
   Value transposedDest =
       createDestinationTensor(b, loc, getSource(), metadata.innerTiles,
                               metadata.innerDimsPos, metadata.outerDimsPerm);
-  return b.create<PackOp>(loc, getSource(), transposedDest,
+  return PackOp::create(b, loc, getSource(), transposedDest,
                           metadata.innerDimsPos, metadata.innerTiles,
                           getPaddingValue(), metadata.outerDimsPerm);
 }
@@ -5138,7 +5138,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (srcShape != packOp.getSourceType().getShape()) {
       auto newSrcType = packOp.getSourceType().clone(srcShape);
       source =
-          rewriter.create<tensor::CastOp>(loc, newSrcType, packOp.getSource());
+          tensor::CastOp::create(rewriter, loc, newSrcType, packOp.getSource());
     }
     Value dest = packOp.getDest();
     RankedTensorType originalResultType = packOp.getDestType();
@@ -5146,7 +5146,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (needUpdateDestType) {
       auto newDestType = packOp.getDestType().clone(destShape);
       dest =
-          rewriter.create<tensor::CastOp>(loc, newDestType, packOp.getDest());
+          tensor::CastOp::create(rewriter, loc, newDestType, packOp.getDest());
     }
     rewriter.modifyOpInPlace(packOp, [&] {
       packOp.getSourceMutable().assign(source);
@@ -5157,7 +5157,7 @@ LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
     if (needUpdateDestType) {
       rewriter.setInsertionPointAfter(packOp);
       auto castOp =
-          rewriter.create<tensor::CastOp>(loc, originalResultType, packOp);
+          tensor::CastOp::create(rewriter, loc, originalResultType, packOp);
       rewriter.replaceAllUsesExcept(packOp, castOp, castOp);
     }
     return success();
@@ -5250,7 +5250,7 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    PackOp newOp = rewriter.create<PackOp>(
+    PackOp newOp = PackOp::create(rewriter,
         op.getLoc(), newOperands[0], newOperands[1], op.getInnerDimsPos(),
         newMixedTileSizes, op.getPaddingValue(), op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
@@ -5259,7 +5259,7 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
     Value replacement = (newResult.getType() != oldResult.getType())
-                            ? rewriter.create<tensor::CastOp>(
+                            ? tensor::CastOp::create(rewriter,
                                   op->getLoc(), oldResult.getType(), newResult)
                             : newResult;
 
@@ -5358,7 +5358,7 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
   for (auto i :
        llvm::seq<unsigned>(0, srcType.getRank() - innerTileSizes.size())) {
     if (srcType.isDynamicDim(i))
-      mixedSizes.push_back(b.create<tensor::DimOp>(loc, source, i).getResult());
+      mixedSizes.push_back(tensor::DimOp::create(b, loc, source, i).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(srcType.getDimSize(i)));
   }
@@ -5371,7 +5371,7 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
     mixedSizes[dimPos] = dimMul(mixedSizes[dimPos], tileSize);
 
   auto elemType = srcType.getElementType();
-  return b.create<tensor::EmptyOp>(loc, mixedSizes, elemType);
+  return tensor::EmptyOp::create(b, loc, mixedSizes, elemType);
 }
 
 UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
@@ -5380,7 +5380,7 @@ UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
                                          ArrayRef<int64_t> outerPermutation) {
   PackOrUnPackTransposeResult metadata = commonPermutationOfPackAndUnPackOp(
       *this, innerPermutation, outerPermutation);
-  return b.create<UnPackOp>(loc, transposedSource, getDest(),
+  return UnPackOp::create(b, loc, transposedSource, getDest(),
                             metadata.innerDimsPos, metadata.innerTiles,
                             metadata.outerDimsPerm);
 }
@@ -5454,7 +5454,7 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
             extractSliceUser.getResultType().getRank()) {
       OpBuilder::InsertionGuard g(rewriter);
       rewriter.setInsertionPoint(unPackOp);
-      auto newDest = rewriter.create<tensor::ExtractSliceOp>(
+      auto newDest = tensor::ExtractSliceOp::create(rewriter,
           unPackOp->getLoc(), unPackOp.getDest(),
           extractSliceUser.getMixedOffsets(), extractSliceUser.getMixedSizes(),
           extractSliceUser.getMixedStrides());
@@ -5474,16 +5474,16 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
     Value source = unPackOp.getSource();
     if (srcShape != unPackOp.getSourceType().getShape()) {
       auto newSrcType = unPackOp.getSourceType().clone(srcShape);
-      source = rewriter.create<tensor::CastOp>(loc, newSrcType,
+      source = tensor::CastOp::create(rewriter, loc, newSrcType,
                                                unPackOp.getSource());
     }
     Value dest = unPackOp.getDest();
     if (destShape != unPackOp.getDestType().getShape()) {
       auto newDestType = unPackOp.getDestType().clone(destShape);
       dest =
-          rewriter.create<tensor::CastOp>(loc, newDestType, unPackOp.getDest());
+          tensor::CastOp::create(rewriter, loc, newDestType, unPackOp.getDest());
     }
-    Value newOp = rewriter.create<UnPackOp>(
+    Value newOp = UnPackOp::create(rewriter,
         loc, source, dest, unPackOp.getInnerDimsPos(), unPackOp.getMixedTiles(),
         unPackOp.getOuterDimsPerm());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(
@@ -5542,7 +5542,7 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    UnPackOp newOp = rewriter.create<UnPackOp>(
+    UnPackOp newOp = UnPackOp::create(rewriter,
         op.getLoc(), sourceTensor, newOperands[1], op.getInnerDimsPos(),
         newMixedTileSizes, op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
@@ -5551,7 +5551,7 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
     Value replacement = (newResult.getType() != oldResult.getType())
-                            ? rewriter.create<tensor::CastOp>(
+                            ? tensor::CastOp::create(rewriter,
                                   op->getLoc(), oldResult.getType(), newResult)
                             : newResult;
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 5d5f9de465561..ba03d09e5072e 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -672,7 +672,7 @@ static Operation *replaceForAllWithNewSignature(
   newOuts.push_back(outputs[resultNumber]);
 
   // Create new scf.forall op
-  auto newforallOp = rewriter.create<scf::ForallOp>(
+  auto newforallOp = scf::ForallOp::create(rewriter,
       loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
       forallOp.getMixedStep(), newOuts, forallOp.getMapping());
   rewriter.eraseBlock(newforallOp.getBody());
@@ -699,7 +699,7 @@ static Operation *replaceForAllWithNewSignature(
   Value src = tileAndFuseResult.tiledValues[0];
   Value dst = newforallOp.getRegionIterArgs().back();
   SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-  rewriter.create<tensor::ParallelInsertSliceOp>(firstYieldOp->getLoc(), src,
+  tensor::ParallelInsertSliceOp::create(rewriter, firstYieldOp->getLoc(), src,
                                                  dst, offsets, sizes, strides);
 
   for (auto result : llvm::enumerate(forallOp.getResults())) {
@@ -3410,12 +3410,12 @@ transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
         for (auto [ofrIdx, ofr] : llvm::enumerate(getMixedSizes())) {
           if (auto attr = llvm::dyn_cast_if_present<Attribute>(ofr)) {
             if (scalableSizes[ofrIdx]) {
-              auto val = b.create<arith::ConstantIndexOp>(
+              auto val = arith::ConstantIndexOp::create(b,
                   getLoc(), cast<IntegerAttr>(attr).getInt());
               Value vscale =
-                  b.create<vector::VectorScaleOp>(getLoc(), b.getIndexType());
+                  vector::VectorScaleOp::create(b, getLoc(), b.getIndexType());
               sizes.push_back(
-                  b.create<arith::MulIOp>(getLoc(), val, vscale).getResult());
+                  arith::MulIOp::create(b, getLoc(), val, vscale).getResult());
             } else {
               sizes.push_back(attr);
             }
@@ -3626,7 +3626,7 @@ static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter,
   SmallVector<OpFoldResult> normalizedSteps(normalizedUbs.size(),
                                             rewriter.getIndexAttr(1));
 
-  auto normalizedForallOp = rewriter.create<scf::ForallOp>(
+  auto normalizedForallOp = scf::ForallOp::create(rewriter,
       loc, normalizedLbs, normalizedUbs, normalizedSteps, loop.getOutputs(),
       loop.getMapping(), [](OpBuilder &, Location, ValueRange) {});
 
@@ -4128,7 +4128,7 @@ DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target,
         target->template getParentOfType<scf::InParallelOp>());
   }
 
-  Value extracted = rewriter.create<tensor::ExtractSliceOp>(
+  Value extracted = tensor::ExtractSliceOp::create(rewriter,
       target.getLoc(), target.getDest(), target.getMixedOffsets(),
       target.getMixedSizes(), target.getMixedStrides());
   Value copied = rewriter
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
index 1f6d96ca0f81f..2cb8ad367f93c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -184,7 +184,7 @@ struct SoftmaxOpInterface
         getBuffer(rewriter, softmaxOp.getOutput(), options, state);
     if (failed(outputBuffer))
       return failure();
-    rewriter.create<linalg::SoftmaxOp>(softmaxOp.getLoc(),
+    linalg::SoftmaxOp::create(rewriter, softmaxOp.getLoc(),
                                        /*result=*/TypeRange(), *inputBuffer,
                                        *outputBuffer, softmaxOp.getDimension());
     replaceOpWithBufferizedValues(rewriter, op, *outputBuffer);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index a7732b939e70d..61c779850366f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -30,10 +30,10 @@ static bool hasAllOneValues(DenseIntElementsAttr attr) {
 
 static Value createAdd(Location loc, Value x, Value y, OpBuilder &builder) {
   if (isa<IntegerType>(x.getType()))
-    return builder.create<arith::AddIOp>(loc, x, y);
+    return arith::AddIOp::create(builder, loc, x, y);
   if (isa<ComplexType>(x.getType()))
-    return builder.create<complex::AddOp>(loc, x, y);
-  return builder.create<arith::AddFOp>(loc, x, y);
+    return complex::AddOp::create(builder, loc, x, y);
+  return arith::AddFOp::create(builder, loc, x, y);
 }
 
 static Value createMul(Location loc, Value x, Value y, Type accType,
@@ -44,10 +44,10 @@ static Value createMul(Location loc, Value x, Value y, Type accType,
   Value yConvert =
       convertScalarToDtype(builder, loc, y, accType, /*isUnsignedCast=*/false);
   if (isa<ComplexType>(accType))
-    return builder.create<complex::MulOp>(loc, xConvert, yConvert);
+    return complex::MulOp::create(builder, loc, xConvert, yConvert);
   if (isa<IntegerType>(accType))
-    return builder.create<arith::MulIOp>(loc, xConvert, yConvert);
-  return builder.create<arith::MulFOp>(loc, xConvert, yConvert);
+    return arith::MulIOp::create(builder, loc, xConvert, yConvert);
+  return arith::MulFOp::create(builder, loc, xConvert, yConvert);
 }
 
 // Delinearizes the given composite `index` by the basis specified in `factors`.
@@ -56,7 +56,7 @@ static SmallVector<Value> unrollIndex(OpBuilder &b, Location loc, Value index,
   assert(!factors.empty() && "empty factor list");
   SmallVector<Value> basis;
   for (int64_t f : factors)
-    basis.push_back(b.create<arith::ConstantOp>(loc, b.getIndexAttr(f)));
+    basis.push_back(arith::ConstantOp::create(b, loc, b.getIndexAttr(f)));
   FailureOr<SmallVector<Value>> multiIndex =
       affine::delinearizeIndex(b, loc, index, basis);
   assert(!failed(multiIndex) && "Failed to linearize img2col index");
@@ -115,17 +115,17 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0, 1, 2}, {3}};
   auto reshapedFilterType =
       RankedTensorType::get({fh * fw * ic, oc}, filterType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
@@ -138,15 +138,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value mIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> mIndices = unrollIndex(
@@ -170,9 +170,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because the filter does not share the same batch dimension,
@@ -187,7 +187,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -196,11 +196,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
@@ -244,7 +244,7 @@ rewriteInIm2Col(RewriterBase &rewriter,
     SmallVector<int64_t> targetShape = llvm::to_vector<4>(llvm::map_range(
         indices, [&](int64_t index) -> int64_t { return inputShape[index]; }));
 
-    Value outputTensor = rewriter.create<tensor::EmptyOp>(
+    Value outputTensor = tensor::EmptyOp::create(rewriter,
         loc, targetShape, operandTensorType.getElementType());
 
     SmallVector<utils::IteratorType> loopAttributeTypes(
@@ -255,12 +255,12 @@ rewriteInIm2Col(RewriterBase &rewriter,
             AffineMap::get(nloops, 0, exprs, rewriter.getContext())),
         AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-    auto transposedOp = rewriter.create<linalg::GenericOp>(
+    auto transposedOp = linalg::GenericOp::create(rewriter,
         loc, outputTensor.getType(),
         /*inputs=*/operand, /*outputs=*/outputTensor, indexingMaps,
         loopAttributeTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-          nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+          linalg::YieldOp::create(nestedBuilder, nestedLoc, args[0]);
         });
 
     return transposedOp.getResult(0);
@@ -307,15 +307,15 @@ rewriteInIm2Col(RewriterBase &rewriter,
       AffineMap::get(nloops, 0, inputExprs, rewriter.getContext()),
       AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/inputT, /*outputs=*/colTensor, indexingMaps,
       loopAttributeTypes,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, args[0]);
       });
 
   SmallVector<ReassociationIndices> img2ColTensorReassocIndices = {
@@ -331,16 +331,16 @@ rewriteInIm2Col(RewriterBase &rewriter,
   auto reshapedOutputTensorType =
       RankedTensorType::get({n * c, oh * ow}, outputType.getElementType());
 
-  Value reshapedImg2ColTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedImg2ColTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedImg2ColTensorType, img2ColTensor.getResult(0),
       img2ColTensorReassocIndices);
-  Value reshapedFilterTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilterTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterTensorType, filterT, filterReassociationIndice);
-  Value reshapedoutputTensor = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedoutputTensor = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputTensorType, transposedOutputTensor,
       outputReassociationIndice);
 
-  auto batchMatVecResult = rewriter.create<linalg::BatchMatvecOp>(
+  auto batchMatVecResult = linalg::BatchMatvecOp::create(rewriter,
       loc, TypeRange{reshapedoutputTensor.getType()},
       ValueRange{reshapedImg2ColTensor, reshapedFilterTensor},
       ValueRange{reshapedoutputTensor});
@@ -348,7 +348,7 @@ rewriteInIm2Col(RewriterBase &rewriter,
   SmallVector<ReassociationIndices> batchMatVecReassociationIndice = {{0, 1},
                                                                       {2, 3}};
 
-  auto batchMatVecResultReshaped = rewriter.create<tensor::ExpandShapeOp>(
+  auto batchMatVecResultReshaped = tensor::ExpandShapeOp::create(rewriter,
       loc, transposedOutputTensor.getType(), batchMatVecResult.getResult(0),
       batchMatVecReassociationIndice);
 
@@ -400,18 +400,18 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, ic * fh * fw}, inputType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1}, {2, 3}};
   auto reshapedOutputType =
       RankedTensorType::get({n, oc, oh * ow}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   // Convert the input to a (BKN) tensor.
   SmallVector<int64_t, 4> colTensorShape = {n, ic * fh * fw, oh * ow};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   auto nloops = colTensorShape.size();
@@ -423,15 +423,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<AffineMap, 4> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value nIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value nIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> kIndices = unrollIndex(
@@ -455,9 +455,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
 
         // im2col[n, ic*fh*fw, oh*ow] = input[n, ic, sh*oh + fh, sw*ow + fw]
         SmallVector<Value> extractionIndices{bIndex, icIndex, hIndex, wIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because the filter does not share the same batch dimension,
@@ -471,7 +471,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   auto resultMap = AffineMap::get(4, 0, {bDim, mDim, nDim}, context);
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{reshapedFilter, img2ColTensor.getResult(0)},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -480,11 +480,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
@@ -535,17 +535,17 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, fh * fw * ic}, filterType.getElementType());
-  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
       loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = rewriter.create<tensor::EmptyOp>(
+  Value colTensor = tensor::EmptyOp::create(rewriter,
       loc, colTensorShape, inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
@@ -558,15 +558,15 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+  auto img2ColTensor = linalg::GenericOp::create(rewriter,
       loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
         // Get the iterators named based on the matmul (batch, m, k).
-        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
-        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
-        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+        Value bIndex = linalg::IndexOp::create(nestedBuilder, loc, 0);
+        Value mIndex = linalg::IndexOp::create(nestedBuilder, loc, 1);
+        Value kIndex = linalg::IndexOp::create(nestedBuilder, loc, 2);
 
         // Recover the original iteration indices from the problem/input sizes.
         SmallVector<Value> mIndices = unrollIndex(
@@ -590,9 +590,9 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
             loc, input, extractionIndices);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
   // Because we didn't transpose the filters we don't actually have a batched
@@ -606,7 +606,7 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
@@ -615,11 +615,11 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
         Value mul =
             createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
         Value add = createAdd(loc, mul, args[2], nestedBuilder);
-        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+        linalg::YieldOp::create(nestedBuilder, nestedLoc, add);
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
       loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index 39e2aac27e213..f21563c323c1b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -37,7 +37,7 @@ static Value createInserts(RewriterBase &rewriter, Location loc, int dim,
   if (dim == static_cast<int>(shape.size()) - 1) {
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
-      destination = rewriter.create<tensor::InsertOp>(loc, *elementIt,
+      destination = tensor::InsertOp::create(rewriter, loc, *elementIt,
                                                       destination, indices);
       ++elementIt;
     }
@@ -65,7 +65,7 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
       MaterializeInDestination: {
     // Note: This is the preferred way of memcpy'ing because no layout map
     // and/or memory space must be specified for the source.
-    auto materializeOp = b.create<bufferization::MaterializeInDestinationOp>(
+    auto materializeOp = bufferization::MaterializeInDestinationOp::create(b,
         loc, tensorSource, memrefDest);
     materializeOp.setWritable(true);
   } break;
@@ -73,19 +73,19 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = b.create<bufferization::ToBufferOp>(
+    Value toBuffer = bufferization::ToBufferOp::create(b,
         loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
-    b.create<memref::CopyOp>(loc, toBuffer, memrefDest);
+    memref::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
   case linalg::BufferizeToAllocationOptions::MemcpyOp::LinalgCopy: {
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = b.create<bufferization::ToBufferOp>(
+    Value toBuffer = bufferization::ToBufferOp::create(b,
         loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
-    b.create<linalg::CopyOp>(loc, toBuffer, memrefDest);
+    linalg::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
   };
 }
@@ -120,14 +120,14 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
             ->materializeConstant(rewriter, constYieldedValue,
                                   yieldedValue.getType(), yieldedValue.getLoc())
             ->getResult(0);
-    auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(fillValue),
+    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(fillValue),
                                                   ValueRange(dest));
     return fillOp;
   }
 
   if (invariantYieldedValue) {
     // Padding with an invariant value.
-    auto fillOp = rewriter.create<linalg::FillOp>(loc, ValueRange(yieldedValue),
+    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(yieldedValue),
                                                   ValueRange(dest));
     return fillOp;
   }
@@ -137,7 +137,7 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(resultType.getRank()));
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, resultType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{dest}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
@@ -146,7 +146,7 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
   rewriter.setInsertionPointToStart(body);
   SmallVector<Value> bbArgReplacements;
   for (int64_t i = 0; i < resultType.getRank(); ++i)
-    bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
+    bbArgReplacements.push_back(linalg::IndexOp::create(rewriter, loc, i));
   rewriter.mergeBlocks(padOp.getBody(), body, bbArgReplacements);
 
   // Update terminator.
@@ -179,8 +179,8 @@ static SmallVector<Value> reifyOrComputeDynamicSizes(OpBuilder &b,
   for (int64_t i = 0; i < tensorType.getRank(); ++i) {
     if (tensorType.isDynamicDim(i))
       dynSizes.push_back(
-          b.create<DimOp>(value.getLoc(), value,
-                          b.create<arith::ConstantIndexOp>(value.getLoc(), i)));
+          DimOp::create(b, value.getLoc(), value,
+                          arith::ConstantIndexOp::create(b, value.getLoc(), i)));
   }
   return dynSizes;
 }
@@ -201,15 +201,15 @@ createAllocationForTensor(RewriterBase &rewriter, Location loc, Value value,
   Value alloc;
   if (options.allocOp ==
       linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloc) {
-    alloc = rewriter.create<memref::AllocOp>(loc, memrefType, dynamicSizes);
+    alloc = memref::AllocOp::create(rewriter, loc, memrefType, dynamicSizes);
     if (options.emitDealloc) {
       // Place deallocation at the end of the block.
       rewriter.setInsertionPoint(rewriter.getInsertionBlock()->getTerminator());
-      rewriter.create<memref::DeallocOp>(loc, alloc);
+      memref::DeallocOp::create(rewriter, loc, alloc);
     }
   } else if (options.allocOp ==
              linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloca) {
-    alloc = rewriter.create<memref::AllocaOp>(loc, memrefType, dynamicSizes);
+    alloc = memref::AllocaOp::create(rewriter, loc, memrefType, dynamicSizes);
     // No dealloc is needed.
   }
 
@@ -243,13 +243,13 @@ Value linalg::bufferizeToAllocation(
       getMixedSizes(rewriter, loc, padOp.getSource());
   SmallVector<OpFoldResult> strides(padOp.getResultType().getRank(),
                                     rewriter.getIndexAttr(1));
-  Value subview = rewriter.create<memref::SubViewOp>(
+  Value subview = memref::SubViewOp::create(rewriter,
       loc, alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
   createMemcpy(rewriter, loc, padOp.getSource(), subview, options);
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
       loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(padOp, toTensorOp);
@@ -338,7 +338,7 @@ Value linalg::bufferizeToAllocation(
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
+  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
       loc, allocTensorOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(allocTensorOp, toTensorOp);
@@ -354,7 +354,7 @@ FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
   auto shape = tensorType.getShape();
 
   // Create tensor.empty.
-  auto emptyOp = rewriter.create<EmptyOp>(loc, tensorType, ValueRange());
+  auto emptyOp = EmptyOp::create(rewriter, loc, tensorType, ValueRange());
 
   // Case: tensor<elem_type>.
   if (shape.empty()) {
@@ -369,7 +369,7 @@ FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
   SmallVector<Value, 2> constants;
   constants.reserve(maxDim);
   for (int i = 0; i < maxDim; ++i)
-    constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
+    constants.push_back(arith::ConstantIndexOp::create(rewriter, loc, i));
 
   // Traverse all elements and create tensor.insert ops.
   auto elementIt = fromElementsOp.getElements().begin();
@@ -395,14 +395,14 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
 
   // Create tensor.empty.
   auto emptyOp =
-      rewriter.create<EmptyOp>(loc, tensorType, generateOp.getDynamicExtents());
+      EmptyOp::create(rewriter, loc, tensorType, generateOp.getDynamicExtents());
 
   // Create linalg.generic.
   SmallVector<utils::IteratorType> iteratorTypes(tensorType.getRank(),
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(tensorType.getRank()));
-  auto genericOp = rewriter.create<linalg::GenericOp>(
+  auto genericOp = linalg::GenericOp::create(rewriter,
       loc, tensorType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{emptyOp.getResult()}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
@@ -411,7 +411,7 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
   rewriter.setInsertionPointToStart(body);
   SmallVector<Value> bbArgReplacements;
   for (int64_t i = 0; i < tensorType.getRank(); ++i)
-    bbArgReplacements.push_back(rewriter.create<linalg::IndexOp>(loc, i));
+    bbArgReplacements.push_back(linalg::IndexOp::create(rewriter, loc, i));
   rewriter.mergeBlocks(&generateOp.getBody().front(), body, bbArgReplacements);
 
   // Update terminator.
@@ -450,13 +450,13 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
       llvm::all_of(padOp.getMixedHighPad(), isZeroInteger)) {
     using bufferization::AllocTensorOp;
     Value allocated =
-        rewriter.create<AllocTensorOp>(loc, resultType, dynamicSizes);
+        AllocTensorOp::create(rewriter, loc, resultType, dynamicSizes);
     auto copyOp = rewriter.replaceOpWithNewOp<linalg::CopyOp>(
         padOp, padOp.getSource(), allocated);
     return copyOp.getOperation();
   }
 
-  Value empty = rewriter.create<EmptyOp>(loc, resultType, dynamicSizes);
+  Value empty = EmptyOp::create(rewriter, loc, resultType, dynamicSizes);
   // Create linalg.fill or linalg.generic.
   Operation *fillOp = movePaddingToFillOrGenericOp(rewriter, loc, padOp, empty);
   rewriter.setInsertionPointAfter(fillOp);
@@ -567,7 +567,7 @@ Value linalg::bufferizeToAllocation(
       createMemcpy(rewriter, op->getLoc(), operand->get(), alloc, options);
     }
     rewriter.modifyOpInPlace(op, [&]() {
-      auto toTensorOp = rewriter.create<ToTensorOp>(
+      auto toTensorOp = ToTensorOp::create(rewriter,
           op->getLoc(), operand->get().getType(), alloc);
       operand->set(toTensorOp);
       if (options.bufferizeDestinationOnly) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index 70574903f7111..eb2c7756dc6ae 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -287,7 +287,7 @@ getOrCreatePackedViewOfOperand(OpBuilder &b, Location loc, PackInfo packInfo,
 
   auto empty = linalg::PackOp::createDestinationTensor(
       b, loc, opOperand->get(), innerTileSizes, innerDimsPos, outerDimsPerm);
-  auto packedOperand = b.create<linalg::PackOp>(
+  auto packedOperand = linalg::PackOp::create(b,
       loc, opOperand->get(), empty, innerDimsPos, innerTileSizes,
       /*padding=*/std::nullopt, outerDimsPerm);
   return std::make_tuple(packedOperand, indexingMap);
@@ -345,7 +345,7 @@ static GenericOp packGenericOp(RewriterBase &rewriter, GenericOp genericOp,
 
   indexingMaps.push_back(packedOutIndexingMap);
 
-  auto newGenericOp = rewriter.create<linalg::GenericOp>(
+  auto newGenericOp = linalg::GenericOp::create(rewriter,
       loc, dest.getType(), inputOperands, dest, indexingMaps, iterTypes,
       /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
   rewriter.cloneRegionBefore(genericOp.getRegion(), newGenericOp.getRegion(),
@@ -457,7 +457,7 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, linalg::PackOp packOp,
   if (!packOpDest.hasOneUse())
     return failure();
   if (auto emptyOp = packOpDest.getDefiningOp<tensor::EmptyOp>()) {
-    packOpDest = rewriter.create<tensor::EmptyOp>(
+    packOpDest = tensor::EmptyOp::create(rewriter,
         genericOp->getLoc(), emptyOp.getMixedSizes(),
         emptyOp.getType().getElementType());
   } else {
@@ -562,7 +562,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     auto empty = linalg::PackOp::createDestinationTensor(
         rewriter, loc, padOp.getSource(), mixedTiles, innerDimsPos,
         outerDimsPerm);
-    auto sourcePack = rewriter.create<linalg::PackOp>(
+    auto sourcePack = linalg::PackOp::create(rewriter,
         loc, padOp.getSource(), empty, innerDimsPos, mixedTiles,
         /*padding=*/std::nullopt, outerDimsPerm);
 
@@ -579,7 +579,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, /*result=*/Type(), sourcePack, lowPad, highPad, paddingVal,
         padOp.getNofold());
 
@@ -588,7 +588,7 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     if (!padOp->hasOneUse()) {
       auto unpackEmpty = linalg::UnPackOp::createDestinationTensor(
           rewriter, loc, newPadOp, mixedTiles, innerDimsPos, outerDimsPerm);
-      Value unpackedPad = rewriter.create<linalg::UnPackOp>(
+      Value unpackedPad = linalg::UnPackOp::create(rewriter,
           loc, newPadOp, unpackEmpty, innerDimsPos, mixedTiles, outerDimsPerm);
       rewriter.replaceAllUsesExcept(padOp, unpackedPad, sourcePack);
     }
@@ -719,7 +719,7 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
   auto emptyOp = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), collapseOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newPackOp = rewriter.create<linalg::PackOp>(
+  auto newPackOp = linalg::PackOp::create(rewriter,
       packOp.getLoc(), collapseOp.getSrc(), emptyOp, projectedInnerDimsPos,
       packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPerm);
 
@@ -735,7 +735,7 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
     nextPos += 1;
   }
 
-  auto newCollapseOp = rewriter.create<tensor::CollapseShapeOp>(
+  auto newCollapseOp = tensor::CollapseShapeOp::create(rewriter,
       collapseOp.getLoc(), packOp.getType(), newPackOp, newReassocIndices);
   rewriter.replaceOp(packOp, newCollapseOp);
 
@@ -853,12 +853,12 @@ bubbleUpPackOpThroughExpandShape(tensor::ExpandShapeOp expandOp,
   Value destTensor = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), expandOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, /*outerDimsPerm=*/SmallVector<int64_t>{});
-  Value packedVal = rewriter.create<linalg::PackOp>(
+  Value packedVal = linalg::PackOp::create(rewriter,
       packOp.getLoc(), expandOp.getSrc(), destTensor, projectedInnerDimsPos,
       packOp.getMixedTiles(), packOp.getPaddingValue(),
       /*outerDimsPerm=*/SmallVector<int64_t>{});
 
-  Value newExpandOp = rewriter.create<tensor::ExpandShapeOp>(
+  Value newExpandOp = tensor::ExpandShapeOp::create(rewriter,
       packOp.getLoc(), packOp.getDestType(), packedVal, *reassocExpand);
   rewriter.replaceOp(packOp, newExpandOp);
 
@@ -972,14 +972,14 @@ static LogicalResult pushDownUnPackOpThroughExpandShape(
 
   RankedTensorType newExpandType = linalg::PackOp::inferPackedType(
       expandTy, innerTileSizes, projectedInnerDimsPos, newOuterDimsPerm);
-  auto newExpandOp = rewriter.create<tensor::ExpandShapeOp>(
+  auto newExpandOp = tensor::ExpandShapeOp::create(rewriter,
       expandOp.getLoc(), newExpandType, unPackOp.getSource(),
       newReassocIndices);
 
   auto emptyOp = linalg::UnPackOp::createDestinationTensor(
       rewriter, unPackOp.getLoc(), newExpandOp, unPackOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newUnPackOp = rewriter.create<linalg::UnPackOp>(
+  auto newUnPackOp = linalg::UnPackOp::create(rewriter,
       unPackOp.getLoc(), newExpandOp.getResult(), emptyOp,
       projectedInnerDimsPos, unPackOp.getMixedTiles(), newOuterDimsPerm);
   rewriter.replaceOp(expandOp, newUnPackOp);
@@ -1212,16 +1212,16 @@ struct PushDownUnPackThroughPadOp : public OpRewritePattern<tensor::PadOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, /*result=*/Type(), unpackOp.getSource(), lowPad, highPad,
         paddingVal, padOp.getNofold());
 
     // Inject the linalg.unpack right after the packed padOp.
-    Value outputUnPack = rewriter.create<tensor::EmptyOp>(
+    Value outputUnPack = tensor::EmptyOp::create(rewriter,
         loc, padOp.getResultType().getShape(),
         padOp.getResultType().getElementType());
 
-    Value replacement = rewriter.create<linalg::UnPackOp>(
+    Value replacement = linalg::UnPackOp::create(rewriter,
         loc, newPadOp.getResult(), outputUnPack, innerDimsPos,
         unpackOp.getMixedTiles(), outerDimsPerm);
     rewriter.replaceOp(padOp, replacement);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
index 692bf595267d4..82d386b3b9099 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
@@ -198,9 +198,9 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
         transposedShape[i] = inputRTType.getShape()[permutation[i]];
 
       Value emptyTensor =
-          rewriter.create<tensor::EmptyOp>(loc, transposedShape, elType);
+          tensor::EmptyOp::create(rewriter, loc, transposedShape, elType);
 
-      auto transposeOp = rewriter.create<TransposeOp>(loc, newInitValues[i],
+      auto transposeOp = TransposeOp::create(rewriter, loc, newInitValues[i],
                                                       emptyTensor, permutation);
       newInitValues[i] = transposeOp->getResult(0);
       isChanged = true;
@@ -209,10 +209,10 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
     // Does it require broadcast?
     if (!broadcastedDims.empty()) {
       assert(broadcastedDims.size() && "should have non size broadcast");
-      Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+      Value emptyTensor = tensor::EmptyOp::create(rewriter,
           loc, outputShape, inputRTType.getElementType());
 
-      auto broadcastOp = rewriter.create<linalg::BroadcastOp>(
+      auto broadcastOp = linalg::BroadcastOp::create(rewriter,
           loc, newInitValues[i], emptyTensor, broadcastedDims);
 
       newInitValues[i] = broadcastOp->getResult(0);
@@ -227,7 +227,7 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
   SmallVector<Value> operands = op->getOperands();
   ValueRange operandsRef(operands);
 
-  auto newOp = rewriter.create<linalg::GenericOp>(
+  auto newOp = linalg::GenericOp::create(rewriter,
       /*location=*/op.getLoc(),
       /*resultTensorTypes=*/op->getResultTypes(),
       /*inputs=*/newInitValues,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
index 1419175304899..bd3fad01aa48b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
@@ -133,12 +133,12 @@ static Value getZero(OpBuilder &b, Location loc, Type elementType) {
   assert(elementType.isIntOrIndexOrFloat() &&
          "expected scalar type while computing zero value");
   if (isa<IntegerType>(elementType))
-    return b.create<arith::ConstantIntOp>(loc, elementType, 0);
+    return arith::ConstantIntOp::create(b, loc, elementType, 0);
   if (elementType.isIndex())
-    return b.create<arith::ConstantIndexOp>(loc, 0);
+    return arith::ConstantIndexOp::create(b, loc, 0);
   // Assume float.
   auto floatType = cast<FloatType>(elementType);
-  return b.create<arith::ConstantFloatOp>(
+  return arith::ConstantFloatOp::create(b,
       loc, floatType, APFloat::getZero(floatType.getFloatSemantics()));
 }
 
@@ -189,7 +189,7 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
     // Fall back path, use an `init_tensor` and identity indexing map.
     AffineMap indexingMap = rewriter.getMultiDimIdentityMap(domain.size());
     Value emptyTensor =
-        rewriter.create<tensor::EmptyOp>(loc, domain, scalarOpResult.getType());
+        tensor::EmptyOp::create(rewriter, loc, domain, scalarOpResult.getType());
     newInitValues.push_back(emptyTensor);
     newResultTypes.push_back(emptyTensor.getType());
     peeledGenericOpIndexingMaps.push_back(indexingMap);
@@ -202,7 +202,7 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
   resultTypes.append(newResultTypes.begin(), newResultTypes.end());
   auto indexingMapAttr =
       rewriter.getAffineMapArrayAttr(peeledGenericOpIndexingMaps);
-  return rewriter.create<GenericOp>(
+  return GenericOp::create(rewriter,
       loc, resultTypes, genericOp.getInputs(), outsOperands, indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
       [](OpBuilder, Location, ValueRange) {});
@@ -239,7 +239,7 @@ DecomposeLinalgOp::createResidualGenericOp(GenericOp genericOp,
     indexingMaps.push_back(genericOp.getMatchingIndexingMap(&outOperand));
 
   auto indexingMapAttr = rewriter.getAffineMapArrayAttr(indexingMaps);
-  return rewriter.create<GenericOp>(
+  return GenericOp::create(rewriter,
       genericOp->getLoc(), genericOp->getResultTypes(),
       residualGenericOpOperands, genericOp.getOutputs(), indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
@@ -324,7 +324,7 @@ DecomposeLinalgOp::matchAndRewrite(GenericOp genericOp,
     yieldedVals.append(llvm::to_vector(
         llvm::map_range(peeledScalarOperation->getResults(),
                         [](OpResult opr) -> Value { return opr; })));
-    rewriter.create<YieldOp>(genericOp.getLoc(), yieldedVals);
+    YieldOp::create(rewriter, genericOp.getLoc(), yieldedVals);
   }
 
   /// In the split operations, replace block arguments uses that refer to
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index ef24eb881d68b..b8e73bd5929dd 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -34,7 +34,7 @@ static Value sourceMaterializationCallback(OpBuilder &builder, Type type,
 
   // A detensored value is converted back by creating a new tensor from its
   // element(s).
-  return builder.create<tensor::FromElementsOp>(
+  return tensor::FromElementsOp::create(builder,
       loc, RankedTensorType::get({}, inputType), inputs[0]);
 }
 
@@ -147,7 +147,7 @@ class DetensorizeTypeConverter : public TypeConverter {
     // A tensor value is detensoried by extracting its element(s).
     addTargetMaterialization([](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      return builder.create<tensor::ExtractOp>(loc, inputs[0], ValueRange{});
+      return tensor::ExtractOp::create(builder, loc, inputs[0], ValueRange{});
     });
 
     addSourceMaterialization(sourceMaterializationCallback);
@@ -481,7 +481,7 @@ struct LinalgDetensorize
         rewriter.splitBlock(entryBlock, entryBlock->begin());
     rewriter.setInsertionPointToStart(entryBlock);
     auto branch =
-        rewriter.create<cf::BranchOp>(rewriter.getUnknownLoc(), postEntryBlock);
+        cf::BranchOp::create(rewriter, rewriter.getUnknownLoc(), postEntryBlock);
 
     if (aggressiveMode.getValue()) {
       AggressiveDetensoringModel costModel;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index e0062d15e61ca..fd8c177cc7247 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -118,14 +118,14 @@ struct MoveInitOperandsToInput : public OpRewritePattern<GenericOp> {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointAfterValue(op->get());
       auto elemType = cast<ShapedType>(op->get().getType()).getElementType();
-      auto empty = rewriter.create<tensor::EmptyOp>(
+      auto empty = tensor::EmptyOp::create(rewriter,
           loc, tensor::getMixedSizes(rewriter, loc, op->get()), elemType);
 
       unsigned start = genericOp.getDpsInits().getBeginOperandIndex();
       newOutputOperands[op->getOperandNumber() - start] = empty.getResult();
     }
 
-    auto newOp = rewriter.create<GenericOp>(
+    auto newOp = GenericOp::create(rewriter,
         loc, genericOp.getResultTypes(), newInputOperands, newOutputOperands,
         newIndexingMaps, genericOp.getIteratorTypesArray(),
         /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
@@ -295,7 +295,7 @@ static Value collapseValue(
     MemRefLayoutAttrInterface layout;
     auto targetType = MemRefType::get(targetShape, memrefType.getElementType(),
                                       layout, memrefType.getMemorySpace());
-    return rewriter.create<memref::CollapseShapeOp>(loc, targetType, operand,
+    return memref::CollapseShapeOp::create(rewriter, loc, targetType, operand,
                                                     reassociation);
   }
   if (auto tensorType = dyn_cast<RankedTensorType>(operand.getType())) {
@@ -314,7 +314,7 @@ static Value collapseValue(
         "unknown rank reduction strategy");
     auto targetType =
         RankedTensorType::get(targetShape, tensorType.getElementType());
-    return rewriter.create<tensor::CollapseShapeOp>(loc, targetType, operand,
+    return tensor::CollapseShapeOp::create(rewriter, loc, targetType, operand,
                                                     reassociation);
   }
   llvm_unreachable("unsupported operand type");
@@ -519,7 +519,7 @@ linalg::dropUnitDims(RewriterBase &rewriter, GenericOp genericOp,
   for (unsigned i : llvm::seq<unsigned>(0, genericOp.getNumResults()))
     resultTypes.push_back(newOutputs[i].getType());
   GenericOp replacementOp =
-      rewriter.create<GenericOp>(loc, resultTypes, newInputs, newOutputs,
+      GenericOp::create(rewriter, loc, resultTypes, newInputs, newOutputs,
                                  newIndexingMaps, newIteratorTypes);
   rewriter.inlineRegionBefore(genericOp.getRegion(), replacementOp.getRegion(),
                               replacementOp.getRegion().begin());
@@ -652,7 +652,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         collapseValue(rewriter, padOp.getLoc(), padOp.getSource(), newShape,
                       reassociationMap, options.rankReductionStrategy);
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         padOp.getLoc(), /*result=*/Type(), collapsedSource, newLowPad,
         newHighPad, paddingVal, padOp.getNofold());
 
@@ -670,7 +670,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         expandedSizes.push_back(tensor::getMixedSize(
             rewriter, padOp.getLoc(), newPadOp, dim - numUnitDims));
       }
-      dest = rewriter.create<tensor::EmptyOp>(
+      dest = tensor::EmptyOp::create(rewriter,
           padOp.getLoc(), expandedSizes,
           padOp.getResultType().getElementType());
     }
@@ -713,7 +713,7 @@ struct RankReducedExtractSliceOp
             strides));
 
     Location loc = sliceOp.getLoc();
-    Value newSlice = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSlice = tensor::ExtractSliceOp::create(rewriter,
         loc, rankReducedType, sliceOp.getSource(), offsets, sizes, strides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSlice, *reassociation);
@@ -747,7 +747,7 @@ struct RankReducedInsertSliceOp : public OpRewritePattern<InsertOpTy> {
       // parallel case.
       if (std::is_same<InsertOpTy, tensor::ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      reshapedSource = rewriter.create<tensor::CollapseShapeOp>(
+      reshapedSource = tensor::CollapseShapeOp::create(rewriter,
           loc, insertSliceOp.getSource(), *reassociation);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
@@ -898,7 +898,7 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
   /// Expand result tensor.
   Value expandResult(PatternRewriter &rewriter, Value result,
                      RankedTensorType expandedType, int64_t dim) const {
-    return rewriter.create<tensor::ExpandShapeOp>(
+    return tensor::ExpandShapeOp::create(rewriter,
         result.getLoc(), expandedType, result,
         getReassociationForReshapeAtDim(expandedType.getRank(), dim));
   }
@@ -934,7 +934,7 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
     SmallVector<Type, 1> collapsedResultTy;
     if (isa<RankedTensorType>(collapsedInit.getType()))
       collapsedResultTy.push_back(collapsedInit.getType());
-    auto collapsedOp = rewriter.create<ToOpTy>(
+    auto collapsedOp = ToOpTy::create(rewriter,
         loc, collapsedResultTy, ValueRange{collapsedLhs, collapsedRhs},
         ValueRange{collapsedInit});
     for (auto attr : contractionOp->getAttrs()) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index 9c0f6e5d6469e..754f7652a776b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -236,11 +236,11 @@ static void generateFusedElementwiseOpRegion(
     fusedIndices.reserve(numFusedOpLoops);
     llvm::transform(llvm::seq<uint64_t>(0, numFusedOpLoops),
                     std::back_inserter(fusedIndices), [&](uint64_t dim) {
-                      return rewriter.create<IndexOp>(producer.getLoc(), dim);
+                      return IndexOp::create(rewriter, producer.getLoc(), dim);
                     });
     for (IndexOp indexOp :
          llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
-      Value newIndex = rewriter.create<affine::AffineApplyOp>(
+      Value newIndex = affine::AffineApplyOp::create(rewriter,
           producer.getLoc(),
           consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
       mapper.map(indexOp.getResult(), newIndex);
@@ -327,7 +327,7 @@ static void generateFusedElementwiseOpRegion(
   }
   for (auto consumerYieldVal : consumerYieldOp.getOperands())
     fusedYieldValues.push_back(mapper.lookupOrDefault(consumerYieldVal));
-  rewriter.create<YieldOp>(fusedOp.getLoc(), fusedYieldValues);
+  YieldOp::create(rewriter, fusedOp.getLoc(), fusedYieldValues);
 
   // Sanity checks.
   assert(fusedBlock->getNumArguments() == fusedOp.getNumOperands() &&
@@ -416,7 +416,7 @@ mlir::linalg::fuseElementwiseOps(RewriterBase &rewriter,
   }
 
   // Generate the fused op.
-  auto fusedOp = rewriter.create<GenericOp>(
+  auto fusedOp = GenericOp::create(rewriter,
       consumer.getLoc(), fusedResultTypes, fusedInputOperands,
       fusedOutputOperands, rewriter.getAffineMapArrayAttr(fusedIndexMaps),
       consumer.getIteratorTypes(),
@@ -750,9 +750,9 @@ static void updateExpandedGenericOpRegion(PatternRewriter &rewriter,
     expandedIndices.reserve(expandedDims.size() - 1);
     llvm::transform(
         expandedDims.drop_front(), std::back_inserter(expandedIndices),
-        [&](int64_t dim) { return rewriter.create<IndexOp>(loc, dim); });
+        [&](int64_t dim) { return IndexOp::create(rewriter, loc, dim); });
     OpFoldResult newIndex =
-        rewriter.create<IndexOp>(loc, expandedDims.front()).getResult();
+        IndexOp::create(rewriter, loc, expandedDims.front()).getResult();
     for (auto [expandedShape, expandedIndex] :
          llvm::zip(expandedDimsShape, expandedIndices)) {
       AffineExpr idx, acc, shape;
@@ -796,7 +796,7 @@ static Operation *createExpandedTransposeOp(PatternRewriter &rewriter,
       newPerm.push_back(dim);
     }
   }
-  return rewriter.create<TransposeOp>(transposeOp.getLoc(), expandedInput,
+  return TransposeOp::create(rewriter, transposeOp.getLoc(), expandedInput,
                                       output, invertPermutationVector(newPerm));
 }
 
@@ -813,7 +813,7 @@ static Operation *createExpandedGenericOp(
     for (auto j : expansionInfo.getExpandedDims(i))
       iteratorTypes[j] = type;
 
-  Operation *fused = rewriter.create<GenericOp>(
+  Operation *fused = GenericOp::create(rewriter,
       linalgOp.getLoc(), resultTypes, expandedOpOperands, outputs,
       expandedOpIndexingMaps, iteratorTypes);
 
@@ -933,7 +933,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
                 reassociation,
                 /*isExpandingReshape=*/true)))
           return std::nullopt;
-        expandedOpOperands.push_back(rewriter.create<tensor::ExpandShapeOp>(
+        expandedOpOperands.push_back(tensor::ExpandShapeOp::create(rewriter,
             loc, expandedOperandType, opOperand->get(), reassociation,
             expandedOperandShape));
         continue;
@@ -961,7 +961,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               reassociation,
               /*isExpandingReshape=*/true)))
         return std::nullopt;
-      outputs.push_back(rewriter.create<tensor::ExpandShapeOp>(
+      outputs.push_back(tensor::ExpandShapeOp::create(rewriter,
           loc, expandedOutputType, opOperand.get(), reassociation,
           expandedOutputShape));
     } else {
@@ -984,7 +984,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               linalgOp.getMatchingIndexingMap(
                   linalgOp.getDpsInitOperand(resultNumber)),
               expansionInfo);
-      resultVals.push_back(rewriter.create<tensor::CollapseShapeOp>(
+      resultVals.push_back(tensor::CollapseShapeOp::create(rewriter,
           linalgOp.getLoc(), opResult.getType(),
           fusedOp->getResult(resultNumber), reassociation));
     } else {
@@ -1086,7 +1086,7 @@ class FoldPadWithProducerReshapeOpByExpansion
 
     Location loc = padOp->getLoc();
     RankedTensorType expandedPaddedType = paddedType.clone(expandedPaddedShape);
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
@@ -1603,7 +1603,7 @@ static void generateCollapsedIndexingRegion(
        enumerate(collapsingInfo.getCollapsedOpToOrigOpMapping())) {
     ReassociationIndicesRef foldedDimsRef(foldedDims.value());
     Value newIndexVal =
-        rewriter.create<linalg::IndexOp>(loc, foldedDims.index());
+        linalg::IndexOp::create(rewriter, loc, foldedDims.index());
     for (auto dim : llvm::reverse(foldedDimsRef.drop_front())) {
       Value loopDim =
           getValueOrCreateConstantIndexOp(rewriter, loc, loopRange[dim]);
@@ -1687,7 +1687,7 @@ GenericOp cloneToCollapsedOp<GenericOp>(RewriterBase &rewriter,
   SmallVector<utils::IteratorType> iteratorTypes(getCollapsedOpIteratorTypes(
       origOp.getIteratorTypesArray(), collapsingInfo));
 
-  GenericOp collapsedOp = rewriter.create<linalg::GenericOp>(
+  GenericOp collapsedOp = linalg::GenericOp::create(rewriter,
       origOp.getLoc(), resultTypes, inputOperands, outputOperands, indexingMaps,
       iteratorTypes, [](OpBuilder &builder, Location loc, ValueRange args) {});
   Block *origOpBlock = &origOp->getRegion(0).front();
@@ -1794,11 +1794,11 @@ FailureOr<CollapseResult> mlir::linalg::collapseOpIterationDims(
       if (isa<MemRefType>(collapsedOpResult.getType())) {
         MemRefType expandShapeResultType = MemRefType::get(
             originalResultType.getShape(), originalResultType.getElementType());
-        result = rewriter.create<memref::ExpandShapeOp>(
+        result = memref::ExpandShapeOp::create(rewriter,
             loc, expandShapeResultType, collapsedOpResult, reassociation,
             resultShape);
       } else {
-        result = rewriter.create<tensor::ExpandShapeOp>(
+        result = tensor::ExpandShapeOp::create(rewriter,
             loc, originalResultType, collapsedOpResult, reassociation,
             resultShape);
       }
@@ -1982,7 +1982,7 @@ class FoldPadWithProducerReshapeOpByCollapsing
 
     RankedTensorType collapsedPaddedType =
         paddedType.clone(collapsedPaddedShape);
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
@@ -2117,10 +2117,10 @@ class FoldScalarOrSplatConstant : public OpRewritePattern<GenericOp> {
 
       // Create a constant scalar value from the splat constant.
       Value scalarConstant =
-          rewriter.create<arith::ConstantOp>(def->getLoc(), constantAttr);
+          arith::ConstantOp::create(rewriter, def->getLoc(), constantAttr);
 
       SmallVector<Value> outputOperands = genericOp.getOutputs();
-      auto fusedOp = rewriter.create<GenericOp>(
+      auto fusedOp = GenericOp::create(rewriter,
           rewriter.getFusedLoc(fusedLocs), genericOp->getResultTypes(),
           /*inputs=*/fusedOperands,
           /*outputs=*/outputOperands,
@@ -2183,7 +2183,7 @@ struct RemoveOutsDependency : public OpRewritePattern<GenericOp> {
         modifiedOutput = true;
         SmallVector<OpFoldResult> mixedSizes =
             tensor::getMixedSizes(rewriter, loc, operandVal);
-        Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+        Value emptyTensor = tensor::EmptyOp::create(rewriter,
             loc, mixedSizes, operandType.getElementType());
         op->setOperand(opOperand.getOperandNumber(), emptyTensor);
       }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index c4af09ca01421..d93f8c26a9f0a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -64,7 +64,7 @@ getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) {
       continue;
 
     // Extract static / dynamic shape mix from the first operand.
-    res.push_back(b.create<tensor::EmptyOp>(
+    res.push_back(tensor::EmptyOp::create(b,
         loc, tensor::getMixedSizes(b, loc, operands.front()),
         cast<RankedTensorType>(t).getElementType()));
   }
@@ -104,7 +104,7 @@ struct ConvertAnyElementwiseMappableOpOnRankedTensors : public RewritePattern {
               builder.create(loc, op->getName().getIdentifier(),
                              regionArgs.take_front(op->getNumOperands()),
                              resultTypes, op->getAttrs());
-          builder.create<linalg::YieldOp>(loc, scalarOp->getResults());
+          linalg::YieldOp::create(builder, loc, scalarOp->getResults());
         });
     return success();
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
index d375878fb2c91..4e95aae86e670 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
@@ -259,7 +259,7 @@ mlir::linalg::deduplicateOperandsAndRemoveDeadResults(
   for (Value v : newOutputOperands)
     if (isa<TensorType>(v.getType()))
       newResultTypes.push_back(v.getType());
-  auto newOp = rewriter.create<GenericOp>(
+  auto newOp = GenericOp::create(rewriter,
       loc, newResultTypes, newInputOperands, newOutputOperands,
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       genericOp.getIteratorTypes(), genericOp.getDocAttr(),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
index 44469bc404a7c..e19e224c25028 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
@@ -72,14 +72,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
     // Create the tensor of same size as output of the pad op.
     RankedTensorType padResultType = padOp.getResultType();
     auto resultSizes = resultShape[0];
-    auto emptyTensor = rewriter.create<tensor::EmptyOp>(
+    auto emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, resultSizes, padResultType.getElementType());
 
     // Fill the tensor with the pad value.
     // TODO: There is an option to fill only the boundaries. For now just
     // filling the whole tensor.
     auto fillTensor =
-        rewriter.create<linalg::FillOp>(loc, padValue, emptyTensor.getResult());
+        linalg::FillOp::create(rewriter, loc, padValue, emptyTensor.getResult());
 
     // Construct a slice of the fill result that is to be replaced with the
     // result of the generic op. The low pad values are the offsets, the size of
@@ -93,14 +93,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
          llvm::enumerate(cast<RankedTensorType>(source.getType()).getShape())) {
       if (ShapedType::isDynamic(shape.value())) {
         sizes.push_back(
-            rewriter.create<tensor::DimOp>(loc, source, shape.index())
+            tensor::DimOp::create(rewriter, loc, source, shape.index())
                 .getResult());
       } else {
         sizes.push_back(rewriter.getIndexAttr(shape.value()));
       }
     }
     SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-    auto slice = rewriter.create<tensor::ExtractSliceOp>(
+    auto slice = tensor::ExtractSliceOp::create(rewriter,
         loc, fillTensor.getResult(0), offsets, sizes, strides);
 
     // Clone the generic op.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 9bc7be2623849..41252c68ffda9 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -277,7 +277,7 @@ mlir::linalg::fuseProducerOfTensor(OpBuilder &b, OpResult producerOpResult,
   // mismatches. Insert a `tensor.cast` op to propagate the transformation
   // invariant that types are compatible.
   if (consumerType != def.getType())
-    def = b.create<tensor::CastOp>(fusedProducer.getLoc(), consumerType, def);
+    def = tensor::CastOp::create(b, fusedProducer.getLoc(), consumerType, def);
   consumerOpOperand.set(def);
   return FusionInfo{cast<LinalgOp>(producerOpResult.getOwner()), fusedProducer};
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 05f2157b77aeb..7994af8834990 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -61,7 +61,7 @@ FailureOr<GenericOp> mlir::linalg::generalizeNamedOp(RewriterBase &rewriter,
   // All named ops have a region attached that can be inlined.
   assert(linalgOp->getNumRegions() == 1 &&
          "expect named op to have one region attached");
-  GenericOp genericOp = rewriter.create<GenericOp>(
+  GenericOp genericOp = GenericOp::create(rewriter,
       linalgOp.getLoc(), resultTypes, inputs, outputs, indexingMaps, iterators);
   rewriter.inlineRegionBefore(linalgOp->getRegion(0), genericOp.getRegion(),
                               genericOp.getRegion().begin());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index 94ed46442180c..86f24578376c1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -591,7 +591,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
 
     // Create a packing loop that takes `hoistedPackedTensor` as iteration
     // argument.
-    auto clonedForOp = rewriter.create<scf::ForOp>(
+    auto clonedForOp = scf::ForOp::create(rewriter,
         loc, bvm.lookupOrDefault(forOp.getLowerBound()),
         bvm.lookupOrDefault(forOp.getUpperBound()),
         bvm.lookupOrDefault(forOp.getStep()), hoistedPackedTensor);
@@ -640,10 +640,10 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   TransposeOp maybeTransposeOp;
   Value paddedTensor = bvm.lookup(opToHoist.getResult());
   if (!transposeVector.empty()) {
-    Value outputTensor = rewriter.create<tensor::ExtractSliceOp>(
+    Value outputTensor = tensor::ExtractSliceOp::create(rewriter,
         loc, transposedTensorType, hoistedPackedTensor, offsets, sizes,
         strides);
-    maybeTransposeOp = rewriter.create<linalg::TransposeOp>(
+    maybeTransposeOp = linalg::TransposeOp::create(rewriter,
         loc, paddedTensor, outputTensor, transposeVector);
     paddedTensor = maybeTransposeOp.getResult()[0];
   }
@@ -652,7 +652,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   if (nPackedLoops > 0) {
     // Step 4. Create InsertSliceOp at the innermost loop level, inserting an
     // optionally transposed padded slice into the packed tensor.
-    Value inserted = rewriter.create<tensor::InsertSliceOp>(
+    Value inserted = tensor::InsertSliceOp::create(rewriter,
         loc, paddedTensor, hoistedPackedTensor, offsets, sizes, strides);
 
     // Step 5. Iteratively pop the stack and propagate the yield.
@@ -660,7 +660,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
     for (Value iv : llvm::reverse(clonedLoopIvs)) {
       auto forOp = scf::getForInductionVarOwner(iv);
       rewriter.setInsertionPointToEnd(&forOp.getRegion().front());
-      rewriter.create<scf::YieldOp>(loc, valueToYield);
+      scf::YieldOp::create(rewriter, loc, valueToYield);
       valueToYield = forOp.getResult(0);
     }
   }
@@ -712,7 +712,7 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   rewriter.setInsertionPoint(outerLoop);
   SmallVector<Value> dynamicTensorSizes =
       analysis.getHoistedPackedTensorSizes(rewriter, loc);
-  auto emptyOp = rewriter.create<tensor::EmptyOp>(
+  auto emptyOp = tensor::EmptyOp::create(rewriter,
       loc, hoistedPackedTensorType.getShape(),
       hoistedPackedTensorType.getElementType(), dynamicTensorSizes);
 
@@ -840,7 +840,7 @@ padThroughLoopIterArg(RewriterBase &rewriter, Value paddedValueBeforeHoisting,
   {
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(forOp);
-    extracted = rewriter.create<tensor::ExtractSliceOp>(
+    extracted = tensor::ExtractSliceOp::create(rewriter,
         hoistedPackedTensor.getLoc(), hoistedPackedTensor,
         outerSliceOp.getMixedOffsets(), outerSliceOp.getMixedSizes(),
         outerSliceOp.getMixedStrides());
@@ -934,7 +934,7 @@ static Value replaceByPackingResult(RewriterBase &rewriter,
   // offsets = [maybe_leading_ivs, 0 .. 0].
   // sizes = [1 .. 1, transposedShape] (defined above).
   // strides = [1 .. 1] (defined above)
-  return rewriter.create<tensor::ExtractSliceOp>(
+  return tensor::ExtractSliceOp::create(rewriter,
       loc, transposedTensorType, hoistedPackedTensor, offsets,
       packingResult.sizes, packingResult.strides);
 }
@@ -982,9 +982,9 @@ FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(newResult.getDefiningOp());
     // Transpose the packed tensor back to the original storage order.
-    Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+    Value emptyTensor = tensor::EmptyOp::create(rewriter,
         loc, paddedTensorType.getShape(), paddedTensorType.getElementType());
-    TransposeOp unTransposeOp = rewriter.create<linalg::TransposeOp>(
+    TransposeOp unTransposeOp = linalg::TransposeOp::create(rewriter,
         loc, newResult, emptyTensor, transposeVector);
     newResult = unTransposeOp.getResult()[0];
     transposeOps.push_back(unTransposeOp);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index f2e51c29f3241..5397fc572e558 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -53,7 +53,7 @@ static scf::ForOp replaceWithDifferentYield(RewriterBase &rewriter,
   assert(index < inits.size());
   inits[index] = newInitOperand;
 
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
       inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
index 1f3336d2bfbb9..a900215ccf014 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
@@ -60,7 +60,7 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
 
     Location loc = genericOp->getLoc();
     SmallVector<Value> outputOperands = genericOp.getOutputs();
-    auto newOp = rewriter.create<GenericOp>(
+    auto newOp = GenericOp::create(rewriter,
         loc, genericOp->getResultTypes(), newOperands, outputOperands,
         newIndexingMaps, genericOp.getIteratorTypesArray());
     rewriter.cloneRegionBefore(genericOp.getRegion(), newOp.getRegion(),
@@ -77,11 +77,11 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
       SmallVector<Value> indicesValues;
       for (auto idx : indices)
         indicesValues.emplace_back(
-            rewriter.create<arith::ConstantIndexOp>(loc, idx));
+            arith::ConstantIndexOp::create(rewriter, loc, idx));
       Value scalarValue = opOperand->get();
       if (isa<RankedTensorType>(scalarValue.getType())) {
         scalarValue =
-            rewriter.create<tensor::ExtractOp>(loc, scalarValue, indicesValues);
+            tensor::ExtractOp::create(rewriter, loc, scalarValue, indicesValues);
       }
       body->getArgument(idx).replaceAllUsesWith(scalarValue);
       body->eraseArgument(idx);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index a92a0c83e0316..2372313e29c58 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -88,7 +88,7 @@ mlir::linalg::interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp,
       allIndices.reserve(genericOp.getNumLoops());
       llvm::transform(llvm::seq<uint64_t>(0, genericOp.getNumLoops()),
                       std::back_inserter(allIndices), [&](uint64_t dim) {
-                        return rewriter.create<IndexOp>(indexOp->getLoc(), dim);
+                        return IndexOp::create(rewriter, indexOp->getLoc(), dim);
                       });
       rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
           indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 488041a43a2ef..0c08c1d8b997d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -49,7 +49,7 @@ static SmallVector<Value> makeCanonicalAffineApplies(OpBuilder &b, Location loc,
     auto exprMap = AffineMap::get(dims, map.getNumSymbols(), e);
     SmallVector<Value> operands(vals);
     affine::canonicalizeMapAndOperands(&exprMap, &operands);
-    res.push_back(b.create<affine::AffineApplyOp>(loc, exprMap, operands));
+    res.push_back(affine::AffineApplyOp::create(b, loc, exprMap, operands));
   }
   return res;
 }
@@ -70,7 +70,7 @@ static void inlineRegionAndEmitStore(OpBuilder &b, Location loc, OpType op,
   Operation *terminator = block.getTerminator();
   for (OpOperand &operand : terminator->getOpOperands()) {
     Value toStore = map.lookupOrDefault(operand.get());
-    b.create<StoreOpTy>(loc, toStore, outputBuffers[operand.getOperandNumber()],
+    StoreOpTy::create(b, loc, toStore, outputBuffers[operand.getOperandNumber()],
                         indexing[operand.getOperandNumber()]);
   }
 }
@@ -145,7 +145,7 @@ static void emitScalarImplementation(OpBuilder &b, Location loc,
     auto indexing = makeCanonicalAffineApplies(
         b, loc, linalgOp.getMatchingIndexingMap(inputOperand), allIvsPlusDims);
     indexedValues.push_back(
-        b.create<LoadOpTy>(loc, inputOperand->get(), indexing));
+        LoadOpTy::create(b, loc, inputOperand->get(), indexing));
   }
   // 1.b. Emit load from output views.
   for (OpOperand &outputOperand : linalgOp.getDpsInitsMutable()) {
@@ -153,7 +153,7 @@ static void emitScalarImplementation(OpBuilder &b, Location loc,
         b, loc, linalgOp.getMatchingIndexingMap(&outputOperand),
         allIvsPlusDims);
     indexedValues.push_back(
-        b.create<LoadOpTy>(loc, outputOperand.get(), indexing));
+        LoadOpTy::create(b, loc, outputOperand.get(), indexing));
   }
 
   // TODO: When a region inliner exists, use it.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
index ee1957aaa6a53..28b246714df53 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
@@ -131,17 +131,17 @@ static Value createDestinationPassingStyleInitOperand(
     ImplicitLocOpBuilder &builder) {
   Value processLinearIndexInReductionGroup = mesh::createProcessLinearIndex(
       meshOp.getSymName(), reductionMeshAxes, builder);
-  Value zero = builder.create<arith::ConstantIndexOp>(0);
-  Value isLeadProcess = builder.create<arith::CmpIOp>(
+  Value zero = arith::ConstantIndexOp::create(builder, 0);
+  Value isLeadProcess = arith::CmpIOp::create(builder,
       builder.getI1Type(), arith::CmpIPredicate::eq,
       processLinearIndexInReductionGroup, zero);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(spmdizedOperand.getType(),
+  scf::IfOp ifOp = scf::IfOp::create(builder, spmdizedOperand.getType(),
                                              isLeadProcess, true, true);
   // Then block.
   {
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPointToEnd(&ifOp.getThenRegion().front());
-    builder.create<scf::YieldOp>(spmdizedOperand);
+    scf::YieldOp::create(builder, spmdizedOperand);
   }
 
   // Else block.
@@ -157,14 +157,14 @@ static Value createDestinationPassingStyleInitOperand(
     std::optional<TypedAttr> neutralEl =
         arith::getNeutralElement(combinerOps[0]);
 
-    Value init = builder.create<tensor::EmptyOp>(op.getLoc(), shape,
+    Value init = tensor::EmptyOp::create(builder, op.getLoc(), shape,
                                                  neutralEl.value().getType());
     Value constant =
-        builder.create<arith::ConstantOp>(op.getLoc(), neutralEl.value());
-    Value fill = builder.create<linalg::FillOp>(op.getLoc(), constant, init)
+        arith::ConstantOp::create(builder, op.getLoc(), neutralEl.value());
+    Value fill = linalg::FillOp::create(builder, op.getLoc(), constant, init)
                      .getResult(0);
 
-    builder.create<scf::YieldOp>(fill);
+    scf::YieldOp::create(builder, fill);
   }
   return ifOp.getResult(0);
 }
@@ -202,7 +202,7 @@ static void createAllReduceForResultWithoutPartialSharding(
   }
 
   Value spmdizedLinalgOpResult = spmdizationMap.lookup(unshardedLinalgOpResult);
-  Value reducedValue = builder.create<mesh::AllReduceOp>(
+  Value reducedValue = mesh::AllReduceOp::create(builder,
       spmdizedLinalgOpResult, resultSharding.getMesh(), allReduceMeshAxes,
       reductionKind);
   spmdizationMap.map(unshardedLinalgOpResult, reducedValue);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
index bb1e974391878..019c845b4e0e7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
@@ -59,7 +59,7 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
   auto newKernelTy = RankedTensorType::get(
       {kernelTy.getDimSize(0), kernelTy.getDimSize(1), kernelTy.getDimSize(2)},
       kernelTy.getElementType());
-  auto collapsedKernel = rewriter.create<tensor::CollapseShapeOp>(
+  auto collapsedKernel = tensor::CollapseShapeOp::create(rewriter,
       loc, newKernelTy, kernel, collapsedKernelDims);
 
   // Collapse init dims.
@@ -70,7 +70,7 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       RankedTensorType::get({initTy.getDimSize(0), initTy.getDimSize(1),
                              initTy.getDimSize(2), initTy.getDimSize(3)},
                             initTy.getElementType());
-  auto collapsedInit = rewriter.create<tensor::CollapseShapeOp>(
+  auto collapsedInit = tensor::CollapseShapeOp::create(rewriter,
       loc, newInitTy, init, collapsedInitDims);
 
   SmallVector<NamedAttribute> preservedAttrs;
@@ -78,13 +78,13 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       TypeSwitch<Operation *, Operation *>(operation)
           .Case<DepthwiseConv2DNhwcHwcmOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return rewriter.create<DepthwiseConv2DNhwcHwcOp>(
+            return DepthwiseConv2DNhwcHwcOp::create(rewriter,
                 loc, newInitTy, ValueRange{input, collapsedKernel},
                 ValueRange{collapsedInit}, stride, dilation);
           })
           .Case<DepthwiseConv2DNhwcHwcmQOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return rewriter.create<DepthwiseConv2DNhwcHwcQOp>(
+            return DepthwiseConv2DNhwcHwcQOp::create(rewriter,
                 loc, newInitTy, ValueRange{input, collapsedKernel, iZp, kZp},
                 ValueRange{collapsedInit}, stride, dilation);
           })
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
index 2afa2f9b71c2a..1368bb7e77db0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
@@ -143,7 +143,7 @@ struct SimplifyUnPackToCollapseShape : public OpRewritePattern<UnPackOp> {
                        Type newOperandType, ArrayAttr reassociation) const {
     if (operand.getType() == newOperandType)
       return operand;
-    return rewriter.create<tensor::CollapseShapeOp>(loc, newOperandType,
+    return tensor::CollapseShapeOp::create(rewriter, loc, newOperandType,
                                                     operand, reassociation);
   }
 
@@ -265,7 +265,7 @@ struct FoldUnpackWithExtractSliceOp
 
     // Create a new empty output tensor.
     Type elementType = unpackOp.getDestType().getElementType();
-    Value output = rewriter.create<tensor::EmptyOp>(
+    Value output = tensor::EmptyOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getMixedSizes(), elementType);
     rewriter.replaceOpWithNewOp<UnPackOp>(
         sliceOp, unpackOp.getSource(), output, unpackOp.getInnerDimsPos(),
@@ -529,7 +529,7 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
     auto elemType =
         cast<ShapedType>(unPackOp->getResultTypes()[0]).getElementType();
-    Value output = rewriter.create<tensor::EmptyOp>(
+    Value output = tensor::EmptyOp::create(rewriter,
         unPackOp->getLoc(), unpackOpResultDims[0], elemType);
 
     rewriter.replaceOpWithNewOp<UnPackOp>(
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index 5eb3761f7aca1..fb43c1f92e5b3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -192,10 +192,10 @@ static Value padOperand(RewriterBase &rewriter, TilingInterface opToPad,
   if (auto complexTy =
           dyn_cast<ComplexType>(getElementTypeOrSelf(v.getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingValueAttr);
-    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
                                                         complexTy, complexAttr);
   } else {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         opToPad.getLoc(), cast<TypedAttr>(paddingValueAttr));
   }
 
@@ -323,7 +323,7 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
         loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index dc9e11eccac4d..af614e88fb3d5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -219,10 +219,10 @@ static FailureOr<Value> padOperandToSmallestStaticBoundingBox(
   if (auto complexTy = dyn_cast<ComplexType>(
           getElementTypeOrSelf(opOperand->get().getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingAttr);
-    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
                                                         complexTy, complexAttr);
   } else {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         opToPad.getLoc(), cast<TypedAttr>(paddingAttr));
   }
 
@@ -313,7 +313,7 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
         loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index b274502e16903..4042a43b04091 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -63,10 +63,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
     staticBufferType =
         MemRefType::Builder(staticBufferType).setMemorySpace(memorySpaceAttr);
     if (options.useAlloca) {
-      return b.create<memref::AllocaOp>(staticBufferType, ValueRange{},
+      return memref::AllocaOp::create(b, staticBufferType, ValueRange{},
                                         alignmentAttr);
     }
-    return b.create<memref::AllocOp>(staticBufferType, ValueRange{},
+    return memref::AllocOp::create(b, staticBufferType, ValueRange{},
                                      alignmentAttr);
   }
 
@@ -76,10 +76,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
   dynamicBufferType =
       MemRefType::Builder(dynamicBufferType).setMemorySpace(memorySpaceAttr);
   Value mul = b.createOrFold<arith::MulIOp>(
-      b.create<arith::ConstantIndexOp>(width), allocSize);
+      arith::ConstantIndexOp::create(b, width), allocSize);
   if (options.useAlloca)
-    return b.create<memref::AllocaOp>(dynamicBufferType, mul, alignmentAttr);
-  return b.create<memref::AllocOp>(dynamicBufferType, mul, alignmentAttr);
+    return memref::AllocaOp::create(b, dynamicBufferType, mul, alignmentAttr);
+  return memref::AllocOp::create(b, dynamicBufferType, mul, alignmentAttr);
 }
 
 /// Default allocation callback function. This allocates a promoted buffer when
@@ -92,8 +92,8 @@ static std::optional<Value> defaultAllocBufferCallBack(
     std::optional<unsigned> alignment, DataLayout &layout) {
   ShapedType viewType = subView.getType();
   ImplicitLocOpBuilder b(subView.getLoc(), builder);
-  auto zero = b.create<arith::ConstantIndexOp>(0);
-  auto one = b.create<arith::ConstantIndexOp>(1);
+  auto zero = arith::ConstantIndexOp::create(b, 0);
+  auto one = arith::ConstantIndexOp::create(b, 1);
 
   Attribute memorySpaceAttr;
   if (options.memorySpace.has_value())
@@ -123,7 +123,7 @@ defaultDeallocBufferCallBack(const LinalgPromotionOptions &options,
                              OpBuilder &b, Value fullLocalView) {
   if (!options.useAlloca) {
     auto viewOp = cast<memref::ViewOp>(fullLocalView.getDefiningOp());
-    b.create<memref::DeallocOp>(viewOp.getSource().getLoc(),
+    memref::DeallocOp::create(b, viewOp.getSource().getLoc(),
                                 viewOp.getSource());
   }
   return success();
@@ -211,7 +211,7 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
   Location loc = linalgOp.getLoc();
   auto defaultCopyCallBack = [loc](OpBuilder &b, Value src,
                                    Value dst) -> LogicalResult {
-    b.create<linalg::CopyOp>(loc, src, dst);
+    linalg::CopyOp::create(b, loc, src, dst);
     return success();
   };
   copyInFn = (options.copyInFn ? *(options.copyInFn) : defaultCopyCallBack);
@@ -265,7 +265,7 @@ FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
               /*stopCondition=*/nullptr, /*closedUB=*/true);
       size = failed(upperBound)
                  ? getValueOrCreateConstantIndexOp(b, loc, rangeValue.size)
-                 : b.create<arith::ConstantIndexOp>(loc, *upperBound);
+                 : arith::ConstantIndexOp::create(b, loc, *upperBound);
     }
     LLVM_DEBUG(llvm::dbgs() << "Extracted tightest: " << size << "\n");
     fullSizes.push_back(size);
@@ -310,23 +310,23 @@ promoteSubViews(ImplicitLocOpBuilder &b,
     Value fillVal =
         llvm::TypeSwitch<Type, Value>(subviewEltType)
             .Case([&](FloatType t) {
-              return b.create<arith::ConstantOp>(FloatAttr::get(t, 0.0));
+              return arith::ConstantOp::create(b, FloatAttr::get(t, 0.0));
             })
             .Case([&](IntegerType t) {
-              return b.create<arith::ConstantOp>(IntegerAttr::get(t, 0));
+              return arith::ConstantOp::create(b, IntegerAttr::get(t, 0));
             })
             .Case([&](ComplexType t) {
               Value tmp;
               if (auto et = dyn_cast<FloatType>(t.getElementType()))
-                tmp = b.create<arith::ConstantOp>(FloatAttr::get(et, 0.0));
+                tmp = arith::ConstantOp::create(b, FloatAttr::get(et, 0.0));
               else if (auto et = cast<IntegerType>(t.getElementType()))
-                tmp = b.create<arith::ConstantOp>(IntegerAttr::get(et, 0));
-              return b.create<complex::CreateOp>(t, tmp, tmp);
+                tmp = arith::ConstantOp::create(b, IntegerAttr::get(et, 0));
+              return complex::CreateOp::create(b, t, tmp, tmp);
             })
             .Default([](auto) { return Value(); });
     if (!fillVal)
       return failure();
-    b.create<linalg::FillOp>(fillVal, promotionInfo->fullLocalView);
+    linalg::FillOp::create(b, fillVal, promotionInfo->fullLocalView);
   }
 
   // Copy data into the promoted buffers. Use callback if provided.
@@ -459,9 +459,9 @@ static std::optional<Value> allocateSubviewGPUMemoryInAddressSpace(
       gpu::AddressSpaceAttr::get(builder.getContext(), addressSpace));
   Value buffer;
   if (addressSpace == gpu::GPUDialect::getWorkgroupAddressSpace()) {
-    buffer = builder.create<memref::AllocOp>(funcOp.getLoc(), type);
+    buffer = memref::AllocOp::create(builder, funcOp.getLoc(), type);
   } else if (addressSpace == gpu::GPUDialect::getPrivateAddressSpace()) {
-    buffer = builder.create<memref::AllocaOp>(funcOp.getLoc(), type);
+    buffer = memref::AllocaOp::create(builder, funcOp.getLoc(), type);
   } else {
     return std::nullopt;
   }
@@ -487,9 +487,9 @@ LogicalResult mlir::linalg::deallocateWorkgroupMemory(OpBuilder &,
 /// the copy operation to ensure data integrity.
 LogicalResult mlir::linalg::copyToWorkgroupMemory(OpBuilder &b, Value src,
                                                   Value dst) {
-  b.create<gpu::BarrierOp>(src.getLoc());
-  Operation *copyOp = b.create<memref::CopyOp>(src.getLoc(), src, dst);
-  b.create<gpu::BarrierOp>(copyOp->getLoc());
+  gpu::BarrierOp::create(b, src.getLoc());
+  Operation *copyOp = memref::CopyOp::create(b, src.getLoc(), src, dst);
+  gpu::BarrierOp::create(b, copyOp->getLoc());
   return success();
 }
 
@@ -504,7 +504,7 @@ std::optional<Value> mlir::linalg::allocateGPUPrivateMemory(
 /// Normal copy to between src and dst.
 LogicalResult mlir::linalg::copyToGPUPrivateMemory(OpBuilder &b, Value src,
                                                    Value dst) {
-  b.create<memref::CopyOp>(src.getLoc(), src, dst);
+  memref::CopyOp::create(b, src.getLoc(), src, dst);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
index b30182dc84079..eac0e47b18a7d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/RuntimeOpVerification.cpp
@@ -38,8 +38,8 @@ struct StructuredOpInterface
     SmallVector<Range> loopRanges = linalgOp.createLoopRanges(builder, loc);
     auto [starts, ends, _] = getOffsetsSizesAndStrides(loopRanges);
 
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    auto one = arith::ConstantIndexOp::create(builder, loc, 1);
 
     // Subtract one from the loop ends before composing with the indexing map
     transform(ends, ends.begin(), [&](OpFoldResult end) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
index 671dea8bb415f..1fac80142dddf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
@@ -52,7 +52,7 @@ createSplitPart(RewriterBase &b, Location loc, TilingInterface op,
       return nullptr;
     SmallVector<OpFoldResult> resultStrides(resultOffsets.size(),
                                             b.getIndexAttr(1));
-    Value inserted = b.create<tensor::InsertSliceOp>(
+    Value inserted = tensor::InsertSliceOp::create(b,
         loc, result, resultOperands[index], resultOffsets, resultSizes,
         resultStrides);
     results.push_back(inserted);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
index 5bfdbc6d0bb59..1b0bfbbec8624 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
@@ -115,7 +115,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
         newShape,
         cast<RankedTensorType>(operand->get().getType()).getElementType());
 
-    Value newInput = b.create<tensor::ExpandShapeOp>(
+    Value newInput = tensor::ExpandShapeOp::create(b,
         loc, newType, operand->get(), reassociation);
     newInputs.push_back(newInput);
   }
@@ -140,18 +140,18 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   Value emptyOrAllocTensor;
   if (useAlloc) {
-    emptyOrAllocTensor = b.create<bufferization::AllocTensorOp>(
+    emptyOrAllocTensor = bufferization::AllocTensorOp::create(b,
         loc,
         RankedTensorType::get(newOutputShape,
                               op.getRegionOutputArgs()[0].getType()),
         ValueRange{});
   } else {
-    emptyOrAllocTensor = b.create<tensor::EmptyOp>(
+    emptyOrAllocTensor = tensor::EmptyOp::create(b,
         loc, newOutputShape, op.getRegionOutputArgs()[0].getType());
   }
-  Value constantOp = b.create<arith::ConstantOp>(loc, *identity);
+  Value constantOp = arith::ConstantOp::create(b, loc, *identity);
   Value identityTensor =
-      b.create<linalg::FillOp>(op->getLoc(), constantOp, emptyOrAllocTensor)
+      linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor)
           .getResult(0);
 
   newMaps.push_back(AffineMap::get(oldOutputMap.getNumDims() + 1, 0, outputExpr,
@@ -168,7 +168,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   // Create the new op matching the original op with an extra parallel
   // dimension.
-  GenericOp genericOp = b.create<GenericOp>(
+  GenericOp genericOp = GenericOp::create(b,
       loc, TypeRange({emptyOrAllocTensor.getType()}), newInputs,
       ValueRange({identityTensor}), newMaps, newIteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
@@ -191,14 +191,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   AffineMap outputMap = AffineMap::get(intermRank, 0, exprs, op.getContext());
   SmallVector<AffineMap> reductionMaps = {inputMap, outputMap};
 
-  auto reduction = b.create<GenericOp>(
+  auto reduction = GenericOp::create(b,
       loc, op->getResultTypes(), ValueRange({genericOp.getResult(0)}),
       op.getDpsInits(), reductionMaps, reductionIteratorTypes,
       [reductionOp](OpBuilder &b, Location loc, ValueRange inputs) {
         Operation *clonedReductionOp = b.clone(*reductionOp);
         clonedReductionOp->setOperand(0, inputs[0]);
         clonedReductionOp->setOperand(1, inputs[1]);
-        b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+        linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
       });
   b.replaceOp(op, reduction.getResults());
 
@@ -318,14 +318,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
     Value emptyOrAllocTensor;
     if (useAlloc) {
       emptyOrAllocTensor =
-          b.create<bufferization::AllocTensorOp>(loc, newT, dims);
+          bufferization::AllocTensorOp::create(b, loc, newT, dims);
     } else {
-      emptyOrAllocTensor = b.create<tensor::EmptyOp>(loc, newT.getShape(),
+      emptyOrAllocTensor = tensor::EmptyOp::create(b, loc, newT.getShape(),
                                                      t.getElementType(), dims);
     }
-    Value constantOp = b.create<arith::ConstantOp>(loc, std::get<1>(it));
+    Value constantOp = arith::ConstantOp::create(b, loc, std::get<1>(it));
     fillOps.push_back(
-        b.create<linalg::FillOp>(op->getLoc(), constantOp, emptyOrAllocTensor));
+        linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor));
     newOutputs.push_back(fillOps.back().getResult(0));
     emptyOrAllocTensorOps.push_back(emptyOrAllocTensor.getDefiningOp());
   }
@@ -354,7 +354,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   SmallVector<Value> newInputs = op.getDpsInputs();
   // Add a single shape-only tensor to carry the dimensions without resorting to
   // more complex inversions.
-  newInputs.push_back(b.create<tensor::EmptyOp>(
+  newInputs.push_back(tensor::EmptyOp::create(b,
       loc, ArrayRef<int64_t>{reductionDimSize / splitFactor, splitFactor},
       b.getIntegerType(1)));
   // Output tensors are already good to go.
@@ -365,7 +365,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   iteratorTypes.insert(iteratorTypes.begin() + reductionDimPos,
                        utils::IteratorType::parallel);
   GenericOp genericOp =
-      b.create<GenericOp>(loc, ValueRange(newOutputs).getTypes(), newInputs,
+      GenericOp::create(b, loc, ValueRange(newOutputs).getTypes(), newInputs,
                           newOutputs, newMaps, iteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
                        genericOp.getRegion().begin());
@@ -396,7 +396,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
         utils::IteratorType::reduction;
 
     // clang-format off
-    auto reductionOp = b.create<GenericOp>(
+    auto reductionOp = GenericOp::create(b,
         loc,
         originalOutputType,
         reindexedOutput,
@@ -407,7 +407,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
           Operation *clonedReductionOp = b.clone(*combinerOp);
           clonedReductionOp->setOperand(0, bbArgs[0]);
           clonedReductionOp->setOperand(1, bbArgs[1]);
-          b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+          linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
         });
     // clang-format on
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
index d35aad514e884..854d3855d654d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
@@ -29,7 +29,7 @@ struct SwapExtractSliceOfFill final
     if (!fillOp || !fillOp->hasOneUse())
       return failure();
 
-    auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
         extractOp.getLoc(), extractOp.getType(), fillOp.getOutputs()[0],
         extractOp.getMixedOffsets(), extractOp.getMixedSizes(),
         extractOp.getMixedStrides());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 4741afe8a417d..c2803423fafd7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -94,10 +94,10 @@ static void emitIsPositiveIndexAssertion(ImplicitLocOpBuilder &b,
     return;
   }
 
-  Value zero = b.create<arith::ConstantIndexOp>(0);
-  Value condition = b.create<arith::CmpIOp>(arith::CmpIPredicate::sgt,
+  Value zero = arith::ConstantIndexOp::create(b, 0);
+  Value condition = arith::CmpIOp::create(b, arith::CmpIPredicate::sgt,
                                             cast<Value>(value), zero);
-  b.create<cf::AssertOp>(
+  cf::AssertOp::create(b,
       condition,
       b.getStringAttr("expected strictly positive tile size and divisor"));
 }
@@ -317,9 +317,9 @@ mlir::linalg::computeMultiTileSizes(OpBuilder &builder, LinalgOp op,
     Value coveredSize =
         apply(s0 * s1 + s2 * s3, {spec.lowTileSize, spec.lowTripCount,
                                   spec.highTileSize, spec.highTripCount});
-    Value equals = b.create<arith::CmpIOp>(arith::CmpIPredicate::eq,
+    Value equals = arith::CmpIOp::create(b, arith::CmpIPredicate::eq,
                                            coveredSize, tripCount);
-    b.create<cf::AssertOp>(
+    cf::AssertOp::create(b,
         equals, builder.getStringAttr(
                     "could not compute dynamic multi-size tile shapes"));
   }
@@ -656,7 +656,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       getValueOrCreateConstantIndexOp(b, loc, nonZeroNumThreads);
 
   // 2. Create the ForallOp with an empty region.
-  scf::ForallOp forallOp = b.create<scf::ForallOp>(
+  scf::ForallOp forallOp = scf::ForallOp::create(b,
       loc, getAsOpFoldResult(materializedNonZeroNumThreads), initTensors,
       mapping);
 
@@ -689,7 +689,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       sizes[reductionDim] = b.getIndexAttr(1);
       outOffsets[reductionDim] = forallOp.getInductionVars()[0];
       // TODO: use SubsetExtractOpInterface once it is available.
-      tiledDpsInitOperands.push_back(b.create<tensor::ExtractSliceOp>(
+      tiledDpsInitOperands.push_back(tensor::ExtractSliceOp::create(b,
           loc, cast<RankedTensorType>(initOperand.getType()),
           destBbArgs[destNum], outOffsets, sizes, strides));
     }
@@ -768,7 +768,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
     // 6.b. Parallel insertions are inserted at the end of the combining
     // terminator.
     b.setInsertionPointToEnd(forallOp.getTerminator().getBody());
-    b.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(b,
         loc, result, bbArg, resultOffsetsRank, resultSizesRank, strides);
   }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
index 513cecef29b61..94449fae260ee 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
@@ -44,7 +44,7 @@ static SmallVector<Value> getIndicesForAccess(OpBuilder &b, Location loc,
   for (auto result : indexingMap.getResults()) {
     AffineMap m = AffineMap::get(indexingMap.getNumDims(),
                                  indexingMap.getNumSymbols(), result);
-    Value v = b.create<affine::AffineApplyOp>(loc, m, ivs);
+    Value v = affine::AffineApplyOp::create(b, loc, m, ivs);
     indices.push_back(v);
   }
   return indices;
@@ -72,7 +72,7 @@ static LogicalResult inlinePayload(OpBuilder &b, LinalgOp linalgOp,
     OpOperand *storeInto = linalgOp.getDpsInitOperand(operand.index());
     auto indices = getIndicesForAccess(
         b, loc, linalgOp.getMatchingIndexingMap(storeInto), ivs);
-    b.create<memref::StoreOp>(
+    memref::StoreOp::create(b,
         loc, toStore, linalgOp.getDpsInitOperand(operand.index())->get(),
         indices);
   }
@@ -351,7 +351,7 @@ struct LinalgOpTilingInterface
       SmallVector<Value> indices = getIndicesForAccess(
           builder, linalgOpLoc, linalgOp.getMatchingIndexingMap(&operand), ivs);
       Value load =
-          builder.create<memref::LoadOp>(linalgOpLoc, operand.get(), indices);
+          memref::LoadOp::create(builder, linalgOpLoc, operand.get(), indices);
       indexedValues.push_back(load);
     }
 
@@ -519,10 +519,10 @@ struct LinalgOpPartialReductionInterface
 
       Type elType = getElementTypeOrSelf(result.getType());
       Value emptyTensor =
-          b.create<tensor::EmptyOp>(loc, partialResultShape, elType);
-      Value constantOp = b.create<arith::ConstantOp>(loc, *identity);
+          tensor::EmptyOp::create(b, loc, partialResultShape, elType);
+      Value constantOp = arith::ConstantOp::create(b, loc, *identity);
       auto identityTensor =
-          b.create<linalg::FillOp>(loc, constantOp, emptyTensor);
+          linalg::FillOp::create(b, loc, constantOp, emptyTensor);
       inits.push_back(identityTensor.getResult(0));
     }
 
@@ -574,7 +574,7 @@ struct LinalgOpPartialReductionInterface
       RankedTensorType sliceResultType = RankedTensorType::get(
           sliceInfo.resultShape, valueToTileType.getElementType(),
           valueToTileType.getEncoding());
-      auto sliceOp = b.create<tensor::ExtractSliceOp>(
+      auto sliceOp = tensor::ExtractSliceOp::create(b,
           loc, sliceResultType, valueToTile, sliceInfo.offsets, sliceInfo.sizes,
           sliceInfo.strides);
       tiledInits.push_back(sliceOp.getResult());
@@ -603,7 +603,7 @@ struct LinalgOpPartialReductionInterface
     auto resultTypes = ValueRange(tiledInits).getTypes();
     if (tilingStrategy ==
         ReductionTilingStrategy::PartialReductionOuterReduction) {
-      auto genericOp = b.create<GenericOp>(
+      auto genericOp = GenericOp::create(b,
           loc, resultTypes, tiledInputs, tiledInits, newMaps, newIteratorTypes);
       IRMapping mapping;
       op->getRegion(0).cloneInto(&genericOp.getRegion(),
@@ -648,7 +648,7 @@ struct LinalgOpPartialReductionInterface
         }
       }
 
-      auto reduction = b.create<linalg::ReduceOp>(
+      auto reduction = linalg::ReduceOp::create(b,
           loc, partialResult, init, partialReductionDims,
           [&linalgOp, &initIdx](OpBuilder &b, Location loc, ValueRange inputs) {
             // Get the combiner op.
@@ -659,7 +659,7 @@ struct LinalgOpPartialReductionInterface
             // Combine the input at idx and output at numInits + idx.
             clonedReductionOp->setOperand(0, inputs[0]);
             clonedReductionOp->setOperand(1, inputs[1]);
-            b.create<linalg::YieldOp>(loc, clonedReductionOp->getResult(0));
+            linalg::YieldOp::create(b, loc, clonedReductionOp->getResult(0));
           });
 
       mergeOperations.push_back(reduction);
@@ -790,7 +790,7 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto sourceSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getSource(), inputIndices, inputSizes, strides);
     tiledOperands.push_back(sourceSlice);
 
@@ -800,7 +800,7 @@ struct PackOpTiling
       return {};
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = b.create<tensor::ExtractSliceOp>(
+    auto outSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
@@ -809,7 +809,7 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = b.create<PackOp>(
+    Operation *tiledPackOp = PackOp::create(b,
         loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
@@ -968,7 +968,7 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto sourceSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getSource(), offsets, sizes, strides);
     tiledOperands.push_back(sourceSlice);
 
@@ -984,7 +984,7 @@ struct PackOpTiling
       return failure();
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = b.create<tensor::ExtractSliceOp>(
+    auto outSlice = tensor::ExtractSliceOp::create(b,
         loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
@@ -992,7 +992,7 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = b.create<PackOp>(
+    Operation *tiledPackOp = PackOp::create(b,
         loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
@@ -1172,7 +1172,7 @@ struct UnPackOpTiling
     sliceSrcSizes.append(unpackOp.getMixedTiles());
     sliceSrcStrides.append(numInnerTiles, oneAttr);
     SmallVector<Operation *> generatedSlices;
-    tensor::ExtractSliceOp sliceSource = b.create<tensor::ExtractSliceOp>(
+    tensor::ExtractSliceOp sliceSource = tensor::ExtractSliceOp::create(b,
         loc, unpackOp.getSource(), sliceSrcIndices, sliceSrcSizes,
         sliceSrcStrides);
     generatedSlices.push_back(sliceSource);
@@ -1180,12 +1180,12 @@ struct UnPackOpTiling
     SmallVector<OpFoldResult> destStrides(destRank, oneAttr);
     Value sliceDest;
     if (isPerfectTilingCase) {
-      auto destSliceOp = b.create<tensor::ExtractSliceOp>(
+      auto destSliceOp = tensor::ExtractSliceOp::create(b,
           loc, unpackOp.getDest(), offsets, sizes, destStrides);
       sliceDest = destSliceOp;
       generatedSlices.push_back(destSliceOp);
     } else {
-      sliceDest = b.create<tensor::EmptyOp>(
+      sliceDest = tensor::EmptyOp::create(b,
           loc, destExpandedSizes, unpackOp.getDestType().getElementType());
     }
 
@@ -1193,7 +1193,7 @@ struct UnPackOpTiling
     for (auto tile : unpackOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledUnpackOp = b.create<UnPackOp>(
+    Operation *tiledUnpackOp = UnPackOp::create(b,
         loc, TypeRange{sliceDest.getType()}, tiledOperands, op->getAttrs());
 
     if (isPerfectTilingCase)
@@ -1201,7 +1201,7 @@ struct UnPackOpTiling
                           SmallVector<Value>(tiledUnpackOp->getResults()),
                           generatedSlices};
 
-    auto extractSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractSlice = tensor::ExtractSliceOp::create(b,
         loc, tiledUnpackOp->getResult(0), resultOffsetsFromDest, sizes,
         destStrides);
     return TilingResult{
@@ -1337,13 +1337,13 @@ struct UnPackOpTiling
 
     SmallVector<Value> tiledOperands;
     // Create slice of the dest operand.
-    auto extractDestSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractDestSlice = tensor::ExtractSliceOp::create(b,
         loc, unPackOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(extractDestSlice);
 
     strides.append(unPackOp.getSourceRank() - outputRank, oneAttr);
     // Create slice of the source operand.
-    auto extractSourceSlice = b.create<tensor::ExtractSliceOp>(
+    auto extractSourceSlice = tensor::ExtractSliceOp::create(b,
         loc, unPackOp.getSource(), offsets, sizes, strides);
     tiledOperands.insert(tiledOperands.begin(), extractSourceSlice);
     for (auto tile : unPackOp.getInnerTiles())
@@ -1351,7 +1351,7 @@ struct UnPackOpTiling
 
     // Create tiled unpack op.
     Operation *tiledUnPackOp =
-        b.create<UnPackOp>(loc, TypeRange{extractDestSlice.getType()},
+        UnPackOp::create(b, loc, TypeRange{extractDestSlice.getType()},
                            tiledOperands, op->getAttrs());
 
     return TilingResult{{tiledUnPackOp},
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index eab74dab4eb75..3526f62f770dc 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -269,11 +269,11 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       packingMetadata.reassociations);
   Value paddingValue = packOp.getPaddingValue();
   if (!paddingValue) {
-    paddingValue = rewriter.create<arith::ConstantOp>(
+    paddingValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(getElementTypeOrSelf(collapsed)));
   }
   auto padOp =
-      rewriter.create<tensor::PadOp>(loc, collapsed, packOp.getSource(), lows,
+      tensor::PadOp::create(rewriter, loc, collapsed, packOp.getSource(), lows,
                                      highs, paddingValue, /*nofold=*/false);
 
   LLVM_DEBUG(
@@ -313,7 +313,7 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       SmallVector<OpFoldResult> sizes =
           tensor::getMixedSizes(rewriter, loc, packOp.getDest());
 
-      auto insertSliceOp = rewriter.create<tensor::InsertSliceOp>(
+      auto insertSliceOp = tensor::InsertSliceOp::create(rewriter,
           loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
           /*offsets=*/zeros, sizes, /*strides=*/ones);
 
@@ -329,14 +329,14 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   // 5. Expand from the padded result to the stripMinedShape.
   auto expandShapeResultType =
       RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape);
-  auto reshapeOp = rewriter.create<tensor::ExpandShapeOp>(
+  auto reshapeOp = tensor::ExpandShapeOp::create(rewriter,
       loc, expandShapeResultType, padOp.getResult(),
       packingMetadata.reassociations);
 
   // 6. Transpose stripMinedShape to packedShape.
   SmallVector<int64_t> transpPerm =
       invertPermutationVector(packedToStripMinedShapePerm);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
 
   LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
@@ -371,7 +371,7 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
     SmallVector<OpFoldResult> sizes(packedRank - destShape.size(), one);
     sizes.append(tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()));
 
-    auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
         loc, destTensorType, unPackOp.getSource(),
         SmallVector<OpFoldResult>(packedRank, zero), sizes,
         SmallVector<OpFoldResult>(packedRank, one));
@@ -404,9 +404,9 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
   SmallVector<OpFoldResult, 4> dims =
       tensor::getMixedSizes(rewriter, loc, unPackOp.getSource());
   applyPermutationToVector(dims, packedToStripMinedShapePerm);
-  auto emptyOp = rewriter.create<tensor::EmptyOp>(
+  auto emptyOp = tensor::EmptyOp::create(rewriter,
       loc, dims, stripMinedTensorType.getElementType());
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm);
 
   LLVM_DEBUG(
@@ -426,20 +426,20 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
       DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
 
   // 4. Collapse from the stripMinedShape to the padded result.
-  auto reshapeOp = rewriter.create<tensor::CollapseShapeOp>(
+  auto reshapeOp = tensor::CollapseShapeOp::create(rewriter,
       loc, collapsedType, transposeOp->getResult(0),
       packingMetadata.reassociations);
 
   // 5. ExtractSlice.
   int64_t destRank = destTensorType.getRank();
-  auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+  auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
       loc, destTensorType, reshapeOp->getResult(0),
       SmallVector<OpFoldResult>(destRank, zero),
       tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()),
       SmallVector<OpFoldResult>(destRank, one));
 
   // 6. Inject a copy to preserve DPS.
-  auto copyOp = rewriter.create<linalg::CopyOp>(
+  auto copyOp = linalg::CopyOp::create(rewriter,
       loc, extractSliceOp->getResult(0), unPackOp.getDest());
 
   // 7. Replace unPackOp by copyOp.
@@ -554,15 +554,15 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
               operandType.getShape(), innerPos,
               cast<ShapedType>(dest.getType()).getShape(), {},
               innerPackSizes)) {
-        packOps.push_back(rewriter.create<linalg::PackOp>(
+        packOps.push_back(linalg::PackOp::create(rewriter,
             loc, operand, dest, innerPos, innerPackSizes));
       } else {
         // TODO: value of the padding attribute should be determined by
         // consumers.
         auto zeroAttr =
             rewriter.getZeroAttr(getElementTypeOrSelf(dest.getType()));
-        Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
-        packOps.push_back(rewriter.create<linalg::PackOp>(
+        Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
+        packOps.push_back(linalg::PackOp::create(rewriter,
             loc, operand, dest, innerPos, innerPackSizes, zero));
       }
       inputsAndInits.push_back(packOps.back());
@@ -574,7 +574,7 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       ValueRange{inputsAndInits}.take_front(linalgOp.getNumDpsInputs());
   ValueRange inits =
       ValueRange{inputsAndInits}.take_back(linalgOp.getNumDpsInits());
-  auto packedLinalgOp = rewriter.create<linalg::GenericOp>(
+  auto packedLinalgOp = linalg::GenericOp::create(rewriter,
       linalgOp.getLoc(), inits.getTypes(), inputs, inits, indexingMaps,
       iteratorTypes);
   packedLinalgOp.getRegion().takeBody(linalgOp->getRegion(0));
@@ -589,7 +589,7 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       continue;
     }
     // Build the symmetrical UnPackOp to the existing PackOp.
-    unPackOps.push_back(rewriter.create<linalg::UnPackOp>(
+    unPackOps.push_back(linalg::UnPackOp::create(rewriter,
         packedLinalgOp->getLoc(), result, maybePackedInit.getSource(),
         maybePackedInit.getInnerDimsPos(), maybePackedInit.getMixedTiles()));
     results.push_back(unPackOps.back());
@@ -655,7 +655,7 @@ static LinalgOp transposeOneLinalgOperandAndReplace(
   operands[opOperand.getOperandNumber()] = transposedValue;
 
   ValueRange operandsRef(operands);
-  auto transposedGenericOp = rewriter.create<linalg::GenericOp>(
+  auto transposedGenericOp = linalg::GenericOp::create(rewriter,
       /*location=*/linalgOp->getLoc(),
       /*resultTensorTypes=*/
       operandsRef.drop_front(linalgOp.getNumDpsInputs()).getTypes(),
@@ -904,7 +904,7 @@ mlir::linalg::LinalgTilingOptions::setTileSizes(ArrayRef<int64_t> ts) {
     b.setInsertionPointToStart(
         &op->getParentOfType<func::FuncOp>().getBody().front());
     return llvm::to_vector<4>(map_range(tileSizes, [&](int64_t s) {
-      Value v = b.create<arith::ConstantIndexOp>(op->getLoc(), s);
+      Value v = arith::ConstantIndexOp::create(b, op->getLoc(), s);
       return v;
     }));
   };
@@ -926,11 +926,11 @@ Value DecomposePadOpPattern::createFillOrGenerateOp(
     // Move the padding value defined inside the PadOp block to outside.
     if (padValue.getParentBlock() == &padOp.getRegion().front())
       rewriter.moveOpBefore(padValue.getDefiningOp(), padOp);
-    return rewriter.create<FillOp>(padOp.getLoc(), padValue, dest).result();
+    return FillOp::create(rewriter, padOp.getLoc(), padValue, dest).result();
   }
 
   // Fill could not be optimized: Lower to tensor::GenerateOp with region.
-  auto generateOp = rewriter.create<tensor::GenerateOp>(
+  auto generateOp = tensor::GenerateOp::create(rewriter,
       padOp.getLoc(), padOp.getResultType(), dynSizes);
   // Copy region to new op.
   IRMapping bvm;
@@ -970,7 +970,7 @@ DecomposePadOpPattern::matchAndRewrite(tensor::PadOp padOp,
   }
 
   // Init tensor and fill it with padding.
-  Value emptyTensor = rewriter.create<tensor::EmptyOp>(
+  Value emptyTensor = tensor::EmptyOp::create(rewriter,
       padOp.getLoc(), staticSizes, resultType.getElementType(), dynSizes);
   Value fill = createFillOrGenerateOp(rewriter, padOp, emptyTensor, dynSizes);
 
@@ -1222,11 +1222,11 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
 
   applyPermutationToVector<OpFoldResult>(transShapeForEmptyOp,
                                          srcPermForTranspose);
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, transShapeForEmptyOp, packOp.getSourceType().getElementType());
 
   // 2.2 Create linalg.transpose
-  auto transposedOp = rewriter.create<linalg::TransposeOp>(loc, input, empty,
+  auto transposedOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
                                                            srcPermForTranspose);
 
   // 3. Insert the inner tile to the destination:
@@ -1246,7 +1246,7 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   }
 
   // 4. Replace tensor.packOp with tensor.insert_slice created above
-  auto insert = rewriter.create<tensor::InsertSliceOp>(
+  auto insert = tensor::InsertSliceOp::create(rewriter,
       loc, transposedOp.getResult()[0], packOp.getDest(), writeOffsets,
       writeSizes, writeStrides);
   rewriter.replaceOp(packOp, insert.getResult());
@@ -1313,7 +1313,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     // outer-untiled-dims
     if (ShapedType::isDynamic(srcShape[i])) {
       OpFoldResult dynamicDim =
-          rewriter.create<tensor::DimOp>(loc, source, i).getResult();
+          tensor::DimOp::create(rewriter, loc, source, i).getResult();
       extractSliceSizes.push_back(dynamicDim);
       shapeForEmptyOp.push_back(dynamicDim);
     } else {
@@ -1340,7 +1340,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   readShapeForExtractSlice.append(tileShape.begin(), tileShape.end());
   Type elemType = unpackOp.getSourceType().getElementType();
   auto readType = RankedTensorType::get(readShapeForExtractSlice, elemType);
-  Value innerTile = rewriter.create<tensor::ExtractSliceOp>(
+  Value innerTile = tensor::ExtractSliceOp::create(rewriter,
       loc, readType, unpackOp.getSource(), extractSliceOffsets,
       extractSliceSizes, extractSliceStrides);
 
@@ -1352,9 +1352,9 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   applyPermutationToVector<OpFoldResult>(shapeForEmptyOp, perm);
 
   Value empty =
-      rewriter.create<tensor::EmptyOp>(loc, shapeForEmptyOp, elemType);
+      tensor::EmptyOp::create(rewriter, loc, shapeForEmptyOp, elemType);
   auto transposedOp =
-      rewriter.create<linalg::TransposeOp>(loc, innerTile, empty, perm);
+      linalg::TransposeOp::create(rewriter, loc, innerTile, empty, perm);
 
   // 3. Handle in-complete tiles if needed. It truncates trailing data from the
   // transposed tile.
@@ -1369,7 +1369,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
           tensor::getMixedSize(rewriter, loc, unpackOp.getDest(), i));
   }
 
-  auto partialTile = rewriter.create<tensor::ExtractSliceOp>(
+  auto partialTile = tensor::ExtractSliceOp::create(rewriter,
       loc, transposedOp.getResult()[0], tileOffsets, tileSizes, tileStrides);
 
   // 4. Insert the result to the destination tensor.
@@ -1382,7 +1382,7 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     else
       writeSizes.push_back(oneIdxAttr);
   }
-  auto insert = rewriter.create<tensor::InsertSliceOp>(
+  auto insert = tensor::InsertSliceOp::create(rewriter,
       loc, partialTile, unpackOp.getDest(), writeOffsets, writeSizes,
       writeStrides);
   rewriter.replaceOp(unpackOp, insert.getResult());
@@ -1491,7 +1491,7 @@ FailureOr<Conv1DOp> DownscaleSizeOneWindowed2DConvolution<Conv2DOp, Conv1DOp>::
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = rewriter.create<Conv1DOp>(
+  auto conv1DOp = Conv1DOp::create(rewriter,
       loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
@@ -1578,7 +1578,7 @@ DownscaleDepthwiseConv2DNhwcHwcOp::returningMatchAndRewrite(
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = rewriter.create<DepthwiseConv1DNwcWcOp>(
+  auto conv1DOp = DepthwiseConv1DNwcWcOp::create(rewriter,
       loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
@@ -1635,7 +1635,7 @@ DownscaleConv2DOp::returningMatchAndRewrite(Conv2DOp convOp,
   Value newOutput = tensor::createCanonicalRankReducingExtractSliceOp(
       rewriter, loc, output, newOutputType);
 
-  auto conv1DOp = rewriter.create<Conv1DOp>(loc, newOutputType,
+  auto conv1DOp = Conv1DOp::create(rewriter, loc, newOutputType,
                                             ValueRange{newInput, newKernel},
                                             ValueRange{newOutput});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index 092aecceef6b3..b933a2d143925 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -67,7 +67,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
   Value input;
   if (isTensorOp) {
 
-    input = rewriter.create<tensor::EmptyOp>(loc, newFilterShape, elementTy)
+    input = tensor::EmptyOp::create(rewriter, loc, newFilterShape, elementTy)
                 .getResult();
   } else {
     input = rewriter
@@ -78,7 +78,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
 
   // We can then construct the transposition on our filter.
   auto transpose =
-      rewriter.create<linalg::TransposeOp>(loc, filter, input, filterPerm);
+      linalg::TransposeOp::create(rewriter, loc, filter, input, filterPerm);
 
   Value newFilter;
   if (isTensorOp) {
@@ -98,7 +98,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
     resultTy.push_back(op->getResult(0).getType());
   }
   auto newConv =
-      rewriter.create<HWCFConvOp>(loc, resultTy, newInputs, op.getOutputs(),
+      HWCFConvOp::create(rewriter, loc, resultTy, newInputs, op.getOutputs(),
                                   op.getStrides(), op.getDilations());
   rewriter.replaceOp(op, newConv);
   return newConv.getOperation();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index 934781d1cab75..6b1e89c1594d1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -47,24 +47,24 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
 
   SmallVector<Value> dynamicDims;
   if (type.isDynamicDim(1))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 1));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
   if (type.isDynamicDim(0))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, ArrayRef<int64_t>{shape[1], shape[0]}, type.getElementType(),
       dynamicDims);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, input, empty, ArrayRef<int64_t>{1, 0});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = rewriter.create<linalg::MatmulTransposeAOp>(
+    newMatmulOp = linalg::MatmulTransposeAOp::create(rewriter,
         loc, matmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), matmulOp.getInputs()[1]},
         matmulOp.getOutputs());
   } else {
-    newMatmulOp = rewriter.create<linalg::MatmulTransposeBOp>(
+    newMatmulOp = linalg::MatmulTransposeBOp::create(rewriter,
         loc, matmulOp.getResultTypes(),
         ValueRange{matmulOp.getInputs()[0], transposeOp->getResult(0)},
         matmulOp.getOutputs());
@@ -102,26 +102,26 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
 
   SmallVector<Value> dynamicDims;
   if (type.isDynamicDim(0))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
   if (type.isDynamicDim(2))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 2));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 2));
   if (type.isDynamicDim(1))
-    dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 1));
+    dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = rewriter.create<tensor::EmptyOp>(
+  Value empty = tensor::EmptyOp::create(rewriter,
       loc, ArrayRef<int64_t>{shape[0], shape[2], shape[1]},
       type.getElementType(), dynamicDims);
-  auto transposeOp = rewriter.create<linalg::TransposeOp>(
+  auto transposeOp = linalg::TransposeOp::create(rewriter,
       loc, input, empty, ArrayRef<int64_t>{0, 2, 1});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = rewriter.create<linalg::BatchMatmulTransposeAOp>(
+    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(rewriter,
         loc, batchMatmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), batchMatmulOp.getInputs()[1]},
         batchMatmulOp.getOutputs());
   } else {
-    newMatmulOp = rewriter.create<linalg::BatchMatmulTransposeBOp>(
+    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(rewriter,
         loc, batchMatmulOp.getResultTypes(),
         ValueRange{batchMatmulOp.getInputs()[0], transposeOp->getResult(0)},
         batchMatmulOp.getOutputs());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 5a8c5eab3f444..2fff28ffc7474 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -119,7 +119,7 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, input, /*offsets=*/ArrayRef<int64_t>{w + kw}, sizes, strides));
       }
     }
@@ -130,7 +130,7 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, input,
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             sizes, strides));
@@ -149,7 +149,7 @@ static SmallVector<Value> extractConvFilterSlices(RewriterBase &rewriter,
   // Extract rhs slice of size [{c, f} for channeled convolutions and {1} for
   // non-chanelled convolution] @ [kw].
   for (int64_t kw = 0; kw < kwSize; ++kw) {
-    result.push_back(rewriter.create<vector::ExtractOp>(
+    result.push_back(vector::ExtractOp::create(rewriter,
         loc, filter, /*offsets=*/ArrayRef<int64_t>{kw}));
   }
   return result;
@@ -167,7 +167,7 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {wSizeStep};
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/ArrayRef<int64_t>{w}, sizes, strides));
     }
   } else {
@@ -176,7 +176,7 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {nSize, wSizeStep, fSize};
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, res, /*offsets=*/ArrayRef<int64_t>{0, w, 0}, sizes, strides));
     }
   }
@@ -194,7 +194,7 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // This does not depend on kw.
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{w}, strides);
     }
   } else {
@@ -202,7 +202,7 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // convolution. This does not depend on kw.
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = rewriter.create<vector::InsertStridedSliceOp>(
+      res = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           strides);
     }
@@ -337,7 +337,7 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
   for (int vecDim = 0, end = canonicalVecShape.size(); vecDim < end; ++vecDim) {
     if (ShapedType::isStatic(iterSpaceStaticSizes[vecDim])) {
       // Create constant index op for static dimensions.
-      iterSpaceValueSizes.push_back(rewriter.create<arith::ConstantIndexOp>(
+      iterSpaceValueSizes.push_back(arith::ConstantIndexOp::create(rewriter,
           linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
       continue;
     }
@@ -351,9 +351,9 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
       return failure();
 
     Value dynamicDim = linalgOp.hasPureTensorSemantics()
-                           ? (Value)rewriter.create<tensor::DimOp>(
+                           ? (Value)tensor::DimOp::create(rewriter,
                                  linalgOp.getLoc(), operand, operandDimPos)
-                           : (Value)rewriter.create<memref::DimOp>(
+                           : (Value)memref::DimOp::create(rewriter,
                                  linalgOp.getLoc(), operand, operandDimPos);
     iterSpaceValueSizes.push_back(dynamicDim);
   }
@@ -474,7 +474,7 @@ Value VectorizationState::getOrCreateMaskFor(
          "Masked 0-d vectors are not supported yet");
 
   // Create the mask based on the dimension values.
-  Value mask = rewriter.create<vector::CreateMaskOp>(linalgOp.getLoc(),
+  Value mask = vector::CreateMaskOp::create(rewriter, linalgOp.getLoc(),
                                                      maskType, upperBounds);
   LDBG("Creating new mask: " << mask << "\n");
   activeMaskCache[maskingMap] = mask;
@@ -643,7 +643,7 @@ static Operation *buildMultiDimReduce(OpBuilder &b, Operation *reduceOp,
                                       ArrayRef<bool> dimsToMask) {
   auto maybeKind = getCombinerOpKind(reduceOp);
   assert(maybeKind && "Failed precondition: could not get reduction kind");
-  return b.create<vector::MultiDimReductionOp>(
+  return vector::MultiDimReductionOp::create(b,
       reduceOp->getLoc(), valueToReduce, acc, dimsToMask, *maybeKind);
 }
 
@@ -689,17 +689,17 @@ static Value buildVectorWrite(RewriterBase &rewriter, Value value,
   if (vectorType.getRank() > 0) {
     AffineMap writeMap = inversePermutation(reindexIndexingMap(opOperandMap));
     SmallVector<Value> indices(linalgOp.getRank(outputOperand),
-                               rewriter.create<arith::ConstantIndexOp>(loc, 0));
+                               arith::ConstantIndexOp::create(rewriter, loc, 0));
     value = broadcastIfNeeded(rewriter, value, vectorType);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = rewriter.create<vector::TransferWriteOp>(
+    write = vector::TransferWriteOp::create(rewriter,
         loc, value, outputOperand->get(), indices, writeMap);
   } else {
     // 0-d case is still special: do not invert the reindexing writeMap.
     if (!isa<VectorType>(value.getType()))
-      value = rewriter.create<vector::BroadcastOp>(loc, vectorType, value);
+      value = vector::BroadcastOp::create(rewriter, loc, vectorType, value);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = rewriter.create<vector::TransferWriteOp>(
+    write = vector::TransferWriteOp::create(rewriter,
         loc, value, outputOperand->get(), ValueRange{});
   }
 
@@ -778,7 +778,7 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto indexVectorType =
       VectorType::get({targetShape[dim]}, rewriter.getIndexType(),
                       state.getScalableVecDims()[dim]);
-  auto indexSteps = rewriter.create<vector::StepOp>(loc, indexVectorType);
+  auto indexSteps = vector::StepOp::create(rewriter, loc, indexVectorType);
   // Return the one-dimensional index vector if it lives in the trailing
   // dimension of the iteration space since the vectorization algorithm in this
   // case can handle the broadcast.
@@ -793,14 +793,14 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto permMap =
       AffineMap::getPermutationMap(permPattern, linalgOp.getContext());
 
-  auto broadCastOp = rewriter.create<vector::BroadcastOp>(
+  auto broadCastOp = vector::BroadcastOp::create(rewriter,
       loc, state.getCanonicalVecType(rewriter.getIndexType(), permMap),
       indexSteps);
   SmallVector<int64_t> transposition =
       llvm::to_vector<16>(llvm::seq<int64_t>(0, linalgOp.getNumLoops()));
   std::swap(transposition.back(), transposition[dim]);
   auto transposeOp =
-      rewriter.create<vector::TransposeOp>(loc, broadCastOp, transposition);
+      vector::TransposeOp::create(rewriter, loc, broadCastOp, transposition);
   return VectorizationHookResult{VectorizationHookStatus::NewOp, transposeOp};
 }
 
@@ -853,19 +853,19 @@ static Value calculateGatherOffset(RewriterBase &rewriter,
 
   const size_t numIndices = extractOp.getIndices().size();
   for (size_t i = 1; i < numIndices; i++) {
-    Value dimIdx = rewriter.create<arith::ConstantIndexOp>(loc, i);
+    Value dimIdx = arith::ConstantIndexOp::create(rewriter, loc, i);
 
     auto dimSize = broadcastIfNeeded(
         rewriter,
-        rewriter.create<tensor::DimOp>(loc, extractOp.getTensor(), dimIdx),
+        tensor::DimOp::create(rewriter, loc, extractOp.getTensor(), dimIdx),
         indexVecType);
 
-    offset = rewriter.create<arith::MulIOp>(loc, offset, dimSize);
+    offset = arith::MulIOp::create(rewriter, loc, offset, dimSize);
 
     auto extractOpIndex = broadcastIfNeeded(
         rewriter, bvm.lookup(extractOp.getIndices()[i]), indexVecType);
 
-    offset = rewriter.create<arith::AddIOp>(loc, extractOpIndex, offset);
+    offset = arith::AddIOp::create(rewriter, loc, extractOpIndex, offset);
   }
 
   return offset;
@@ -1110,18 +1110,18 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
 
   // Compute the static loop sizes of the extract op.
   auto resultType = state.getCanonicalVecType(extractOp.getResult().getType());
-  auto maskConstantOp = rewriter.create<arith::ConstantOp>(
+  auto maskConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseIntElementsAttr::get(state.getCanonicalVecType(rewriter.getI1Type()),
                                 /*value=*/true));
   auto passThruConstantOp =
-      rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(resultType));
+      arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(resultType));
 
   // Base indices are currently set to 0. We will need to re-visit if more
   // generic scenarios are to be supported.
   SmallVector<Value> baseIndices(
       extractOp.getIndices().size(),
-      rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      arith::ConstantIndexOp::create(rewriter, loc, 0));
 
   VectorMemoryAccessKind memAccessKind =
       getTensorExtractMemoryAccessPattern(extractOp, linalgOp, resultType);
@@ -1131,7 +1131,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     Value offset = calculateGatherOffset(rewriter, state, extractOp, bvm);
 
     // Generate the gather load
-    Operation *gatherOp = rewriter.create<vector::GatherOp>(
+    Operation *gatherOp = vector::GatherOp::create(rewriter,
         loc, resultType, extractOp.getTensor(), baseIndices, offset,
         maskConstantOp, passThruConstantOp);
     gatherOp = state.maskOperation(rewriter, gatherOp, linalgOp);
@@ -1166,13 +1166,13 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
       continue;
     }
 
-    auto indexAs1dVector = rewriter.create<vector::ShapeCastOp>(
+    auto indexAs1dVector = vector::ShapeCastOp::create(rewriter,
         loc,
         VectorType::get(resultType.getShape().back(), rewriter.getIndexType(),
                         resultType.getScalableDims().back()),
         idx);
     transferReadIdxs.push_back(
-        rewriter.create<vector::ExtractOp>(loc, indexAs1dVector, 0));
+        vector::ExtractOp::create(rewriter, loc, indexAs1dVector, 0));
   }
 
   // `tensor.extract_element` is always in-bounds, hence the following holds.
@@ -1186,7 +1186,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     SmallVector<AffineExpr> exprs(dstRank, getAffineConstantExpr(0, ctx));
     auto permutationMap = AffineMap::get(srcRank, 0, exprs, ctx);
 
-    auto transferReadOp = rewriter.create<vector::TransferReadOp>(
+    auto transferReadOp = vector::TransferReadOp::create(rewriter,
         loc, resultType, extractOp.getTensor(), transferReadIdxs,
         /*padding=*/std::nullopt, permutationMap, inBounds);
 
@@ -1195,7 +1195,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     // valid here).
     SmallVector<int64_t> readMaskShape = {1};
     auto readMaskType = VectorType::get(readMaskShape, rewriter.getI1Type());
-    auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+    auto allTrue = vector::ConstantMaskOp::create(rewriter,
         loc, readMaskType, vector::ConstantMaskKind::AllTrue);
     auto *maskedReadOp =
         mlir::vector::maskOperation(rewriter, transferReadOp, allTrue);
@@ -1223,7 +1223,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     rankDiff--;
   }
 
-  auto transferReadOp = rewriter.create<vector::TransferReadOp>(
+  auto transferReadOp = vector::TransferReadOp::create(rewriter,
       loc, resultType, extractOp.getTensor(), transferReadIdxs,
       /*padding=*/std::nullopt, permutationMap, inBounds);
 
@@ -1405,7 +1405,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
   // 3. Turn all BBArgs into vector.transfer_read / load.
   Location loc = linalgOp.getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   for (OpOperand *opOperand : linalgOp.getOpOperandsMatchingBBargs()) {
     BlockArgument bbarg = linalgOp.getMatchingBlockArgument(opOperand);
     if (linalgOp.isScalar(opOperand)) {
@@ -1435,7 +1435,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
-    Operation *read = rewriter.create<vector::TransferReadOp>(
+    Operation *read = vector::TransferReadOp::create(rewriter,
         loc, readType, opOperand->get(), indices,
         /*padding=*/std::nullopt, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
@@ -1452,7 +1452,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     // 3.c. Not all ops support 0-d vectors, extract the scalar for now.
     // TODO: remove this.
     if (readType.getRank() == 0)
-      readValue = rewriter.create<vector::ExtractOp>(loc, readValue,
+      readValue = vector::ExtractOp::create(rewriter, loc, readValue,
                                                      ArrayRef<int64_t>());
 
     LDBG("New vectorized bbarg(" << bbarg.getArgNumber() << "): " << readValue
@@ -1660,13 +1660,13 @@ createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
           writeIndices.size() == static_cast<size_t>(destRank)) &&
          "Invalid number of write indices!");
   if (writeIndices.empty()) {
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     writeIndices.assign(destRank, zero);
   }
 
   // Generate the xfer_write Op
   Operation *write =
-      builder.create<vector::TransferWriteOp>(loc,
+      vector::TransferWriteOp::create(builder, loc,
                                               /*vector=*/vecToStore,
                                               /*source=*/dest,
                                               /*indices=*/writeIndices,
@@ -1742,7 +1742,7 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   Location loc = packOp.getLoc();
   auto padValue = packOp.getPaddingValue();
   if (!padValue) {
-    padValue = rewriter.create<arith::ConstantOp>(
+    padValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(packOp.getSourceType().getElementType()));
   }
   ReifiedRankedShapedTypeDims reifiedReturnShapes;
@@ -1782,16 +1782,16 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   auto tiledPackType = VectorType::get(getTiledPackShape(packOp, destShape),
                                        packOp.getDestType().getElementType());
   auto shapeCastOp =
-      rewriter.create<vector::ShapeCastOp>(loc, tiledPackType, maskedRead);
+      vector::ShapeCastOp::create(rewriter, loc, tiledPackType, maskedRead);
 
   // Create TransposeOp.
   auto destPermutation =
       invertPermutationVector(getPackInverseDestPerm(packOp));
-  auto transposeOp = rewriter.create<vector::TransposeOp>(
+  auto transposeOp = vector::TransposeOp::create(rewriter,
       loc, shapeCastOp.getResult(), destPermutation);
 
   // Create TransferWriteOp.
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedReturnShapes[0],
       transposeOp.getResult().getType().getElementType());
   Operation *write =
@@ -1892,7 +1892,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   }
   Location loc = unpackOp->getLoc();
 
-  auto padValue = rewriter.create<arith::ConstantOp>(
+  auto padValue = arith::ConstantOp::create(rewriter,
       loc, rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
 
   // Read result, mask if necessary. If transferReadOp shape is not equal
@@ -1911,7 +1911,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   RankedTensorType stripMineTensorType =
       RankedTensorType::get(stripMineShape, stripMineElemType);
   // Transpose the appropriate rows to match output.
-  vector::TransposeOp transposeOp = rewriter.create<vector::TransposeOp>(
+  vector::TransposeOp transposeOp = vector::TransposeOp::create(rewriter,
       loc, readResult, lastDimToInsertPosPerm);
 
   // Collapse the vector to the size required by result.
@@ -1919,7 +1919,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       stripMineTensorType, packMetadata.reassociations);
   mlir::VectorType vecCollapsedType =
       VectorType::get(collapsedType.getShape(), collapsedType.getElementType());
-  vector::ShapeCastOp shapeCastOp = rewriter.create<vector::ShapeCastOp>(
+  vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(rewriter,
       loc, vecCollapsedType, transposeOp->getResult(0));
 
   // writeVectorSizes had to match the shapecast shape for dynamic sizes,
@@ -1928,7 +1928,7 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       unpackOp.getDestType().hasStaticShape()
           ? vectorSizes
           : shapeCastOp.getResultVectorType().getShape());
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedRetShapes[0],
       shapeCastOp.getResult().getType().getElementType());
   Operation *write = createWriteOrMaskedWrite(
@@ -1963,7 +1963,7 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
       /*useInBoundsInsteadOfMasking=*/false);
 
   // Create Xfer write Op
-  Value dest = rewriter.create<tensor::EmptyOp>(
+  Value dest = tensor::EmptyOp::create(rewriter,
       loc, reifiedReturnShapes[0], padOp.getResultType().getElementType());
   Operation *write = createWriteOrMaskedWrite(rewriter, loc, maskedRead, dest);
   newResults.push_back(write->getResult(0));
@@ -2634,19 +2634,19 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
   auto writeType = VectorType::get(dstType.getShape(), dstElementType);
 
   Location loc = copyOp->getLoc();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   SmallVector<Value> indices(srcType.getRank(), zero);
 
-  Value readValue = rewriter.create<vector::TransferReadOp>(
+  Value readValue = vector::TransferReadOp::create(rewriter,
       loc, readType, copyOp.getSource(), indices,
       /*padding=*/std::nullopt,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   if (cast<VectorType>(readValue.getType()).getRank() == 0) {
     readValue =
-        rewriter.create<vector::ExtractOp>(loc, readValue, ArrayRef<int64_t>());
-    readValue = rewriter.create<vector::BroadcastOp>(loc, writeType, readValue);
+        vector::ExtractOp::create(rewriter, loc, readValue, ArrayRef<int64_t>());
+    readValue = vector::BroadcastOp::create(rewriter, loc, writeType, readValue);
   }
-  Operation *writeValue = rewriter.create<vector::TransferWriteOp>(
+  Operation *writeValue = vector::TransferWriteOp::create(rewriter,
       loc, readValue, copyOp.getTarget(), indices,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   rewriter.replaceOp(copyOp, writeValue->getResults());
@@ -2957,7 +2957,7 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   if (!padValue) {
     auto elemType = sourceType.getElementType();
-    padValue = rewriter.create<arith::ConstantOp>(
+    padValue = arith::ConstantOp::create(rewriter,
         sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
   }
 
@@ -2989,7 +2989,7 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   // Create read
   SmallVector<Value> readIndices(
-      vecType.getRank(), rewriter.create<arith::ConstantIndexOp>(loc, 0));
+      vecType.getRank(), arith::ConstantIndexOp::create(rewriter, loc, 0));
   Value read = mlir::vector::createReadOrMaskedRead(
       rewriter, loc, source, vecType.getShape(), padValue,
       /*useInBoundsInsteadOfMasking=*/inputVectorSizes.empty());
@@ -3076,8 +3076,8 @@ struct PadOpVectorizationWithInsertSlicePattern
     // Generate TransferReadOp: Read entire source tensor and add high
     // padding.
     SmallVector<Value> readIndices(
-        vecRank, rewriter.create<arith::ConstantIndexOp>(padOp.getLoc(), 0));
-    auto read = rewriter.create<vector::TransferReadOp>(
+        vecRank, arith::ConstantIndexOp::create(rewriter, padOp.getLoc(), 0));
+    auto read = vector::TransferReadOp::create(rewriter,
         padOp.getLoc(), vecType, padOp.getSource(), readIndices, padValue);
 
     // Generate TransferWriteOp: Write to InsertSliceOp's dest tensor at
@@ -3212,7 +3212,7 @@ LogicalResult LinalgCopyVTRForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_read, the attribute must be reset
   // conservatively.
   auto vectorType = xferOp.getVectorType();
-  Value res = rewriter.create<vector::TransferReadOp>(
+  Value res = vector::TransferReadOp::create(rewriter,
       xferOp.getLoc(), vectorType, in, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getPadding(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(
@@ -3271,7 +3271,7 @@ LogicalResult LinalgCopyVTWForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_write, the attribute must be reset
   // conservatively.
   auto vector = xferOp.getVector();
-  rewriter.create<vector::TransferWriteOp>(
+  vector::TransferWriteOp::create(rewriter,
       xferOp.getLoc(), vector, out, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(SmallVector<bool>(
@@ -3467,7 +3467,7 @@ struct Conv1DGenerator
     }
 
     vector::TransferWriteOp write;
-    Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
     // w is unrolled (i.e. wSizeStep == 1) iff strideW > 1.
     // When strideW == 1, we can batch the contiguous loads and avoid
@@ -3486,16 +3486,16 @@ struct Conv1DGenerator
     SmallVector<Value> resPadding(resShape.size(), zero);
 
     // Read the whole lhs, rhs and res in one shot (with zero padding).
-    Value lhs = rewriter.create<vector::TransferReadOp>(
+    Value lhs = vector::TransferReadOp::create(rewriter,
         loc, lhsType, lhsShaped, lhsPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     // This is needed only for Conv.
     Value rhs = nullptr;
     if (oper == ConvOperationKind::Conv)
-      rhs = rewriter.create<vector::TransferReadOp>(
+      rhs = vector::TransferReadOp::create(rewriter,
           loc, rhsType, rhsShaped, rhsPadding,
           /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
-    Value res = rewriter.create<vector::TransferReadOp>(
+    Value res = vector::TransferReadOp::create(rewriter,
         loc, resType, resShaped, resPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
 
@@ -3511,16 +3511,16 @@ struct Conv1DGenerator
       // To match base vectorization case, we pre-transpose current case.
       // ncw -> nwc
       static constexpr std::array<int64_t, 3> permLhs = {0, 2, 1};
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, permLhs);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, permLhs);
       // fcw -> wcf
       static constexpr std::array<int64_t, 3> permRhs = {2, 1, 0};
 
       // This is needed only for Conv.
       if (oper == ConvOperationKind::Conv)
-        rhs = rewriter.create<vector::TransposeOp>(loc, rhs, permRhs);
+        rhs = vector::TransposeOp::create(rewriter, loc, rhs, permRhs);
       // nfw -> nwf
       static constexpr std::array<int64_t, 3> permRes = {0, 2, 1};
-      res = rewriter.create<vector::TransposeOp>(loc, res, permRes);
+      res = vector::TransposeOp::create(rewriter, loc, res, permRes);
       break;
     }
     }
@@ -3585,7 +3585,7 @@ struct Conv1DGenerator
     case Conv1DOpOrder::Ncw: {
       // nwf -> nfw
       static constexpr std::array<int64_t, 3> perm = {0, 2, 1};
-      res = rewriter.create<vector::TransposeOp>(loc, res, perm);
+      res = vector::TransposeOp::create(rewriter, loc, res, perm);
       break;
     }
     }
@@ -3609,16 +3609,16 @@ struct Conv1DGenerator
         cast<ShapedType>(val.getType()).cloneWith(std::nullopt, dstElementType);
 
     if (isa<IntegerType>(srcElementType) && isa<FloatType>(dstElementType)) {
-      return rewriter.create<arith::SIToFPOp>(loc, dstType, val);
+      return arith::SIToFPOp::create(rewriter, loc, dstType, val);
     }
 
     if (isa<FloatType>(srcElementType) && isa<FloatType>(dstElementType) &&
         srcWidth < dstWidth)
-      return rewriter.create<arith::ExtFOp>(loc, dstType, val);
+      return arith::ExtFOp::create(rewriter, loc, dstType, val);
 
     if (isa<IntegerType>(srcElementType) && isa<IntegerType>(dstElementType) &&
         srcWidth < dstWidth)
-      return rewriter.create<arith::ExtSIOp>(loc, dstType, val);
+      return arith::ExtSIOp::create(rewriter, loc, dstType, val);
 
     assert(false && "unhandled promotion case");
     return nullptr;
@@ -3633,7 +3633,7 @@ struct Conv1DGenerator
     bindDims(ctx, n, w, f, c);
     lhs = promote(rewriter, loc, lhs, res.getType());
     rhs = promote(rewriter, loc, rhs, res.getType());
-    auto contrationOp = rewriter.create<vector::ContractionOp>(
+    auto contrationOp = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, res,
         /*indexingMaps=*/MapList{{n, w, c}, {c, f}, {n, w, f}},
         /*iteratorTypes=*/ArrayRef<vector::IteratorType>{par, par, par, red});
@@ -3645,7 +3645,7 @@ struct Conv1DGenerator
   // convolution.
   Value conv1dSliceAsOuterProduct(RewriterBase &rewriter, Location loc,
                                   Value lhs, Value rhs, Value res) {
-    return rewriter.create<vector::OuterProductOp>(
+    return vector::OuterProductOp::create(rewriter,
         loc, res.getType(), lhs, rhs, res, vector::CombiningKind::ADD);
   }
 
@@ -3693,7 +3693,7 @@ struct Conv1DGenerator
     bindShapeDims(resShapedType, nSize, wSize);
 
     vector::TransferWriteOp write;
-    Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
     // w is unrolled (i.e. wSizeStep == 1) iff strideW > 1.
     // When strideW == 1, we can batch the contiguous loads and avoid
@@ -3736,28 +3736,28 @@ struct Conv1DGenerator
           cast<LinalgOp>(op).hasPureTensorSemantics(), opToMask, rewriter);
 
       Value maskOp =
-          rewriter.create<vector::CreateMaskOp>(loc, maskType, mixedDims);
+          vector::CreateMaskOp::create(rewriter, loc, maskType, mixedDims);
 
       return mlir::vector::maskOperation(rewriter, opToMask, maskOp);
     };
 
     // Read lhs slice of size {n, w * strideW + kw * dilationW, c} @ [0, 0,
     // 0].
-    Value lhs = rewriter.create<vector::TransferReadOp>(
+    Value lhs = vector::TransferReadOp::create(rewriter,
         loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     auto maybeMaskedLhs = maybeMaskXferOp(
         lhsType.getShape(), lhsType.getScalableDims(), lhs.getDefiningOp());
 
     // Read rhs slice of size {kw, c} @ [0, 0].
-    Value rhs = rewriter.create<vector::TransferReadOp>(
+    Value rhs = vector::TransferReadOp::create(rewriter,
         loc, rhsType, rhsShaped, ValueRange{zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
     auto maybeMaskedRhs = maybeMaskXferOp(
         rhsType.getShape(), rhsType.getScalableDims(), rhs.getDefiningOp());
 
     // Read res slice of size {n, w, c} @ [0, 0, 0].
-    Value res = rewriter.create<vector::TransferReadOp>(
+    Value res = vector::TransferReadOp::create(rewriter,
         loc, resType, resShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
     auto maybeMaskedRes = maybeMaskXferOp(
@@ -3775,7 +3775,7 @@ struct Conv1DGenerator
     //   @ [0, sw * w + dw * kw, 0].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        lhsVals.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+        lhsVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
             loc, maybeMaskedLhs->getResult(0),
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             inOutSliceSizes, inOutStrides));
@@ -3783,13 +3783,13 @@ struct Conv1DGenerator
     }
     // Extract rhs slice of size {c} @ [kw].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
-      rhsVals.push_back(rewriter.create<vector::ExtractOp>(
+      rhsVals.push_back(vector::ExtractOp::create(rewriter,
           loc, maybeMaskedRhs->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{kw}));
     }
     // Extract res slice: {n, wSizeStep, c} @ [0, w, 0].
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      resVals.push_back(rewriter.create<vector::ExtractStridedSliceOp>(
+      resVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
           loc, maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0}, inOutSliceSizes,
           inOutStrides));
@@ -3815,16 +3815,16 @@ struct Conv1DGenerator
         if (flatten) {
           // Flatten the input and output vectors (collapse the channel
           // dimension)
-          lhsVal = rewriter.create<vector::ShapeCastOp>(
+          lhsVal = vector::ShapeCastOp::create(rewriter,
               loc, lhsTypeAfterFlattening, lhsVals[linearIndex(kw, w)]);
-          resVal = rewriter.create<vector::ShapeCastOp>(
+          resVal = vector::ShapeCastOp::create(rewriter,
               loc, resTypeAfterFlattening, resVals[w]);
         }
         resVals[w] = depthwiseConv1dSliceAsMulAcc(rewriter, loc, lhsVal,
                                                   rhsVals[kw], resVal, flatten);
         if (flatten) {
           // Un-flatten the output vector (restore the channel dimension)
-          resVals[w] = rewriter.create<vector::ShapeCastOp>(
+          resVals[w] = vector::ShapeCastOp::create(rewriter,
               loc, VectorType::get(inOutSliceSizes, resEltType), resVals[w]);
         }
       }
@@ -3843,7 +3843,7 @@ struct Conv1DGenerator
     // Write back res slice: {n, wSizeStep, c} @ [0, w, 0].
     // This does not depend on kw.
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      maybeMaskedRes = rewriter.create<vector::InsertStridedSliceOp>(
+      maybeMaskedRes = vector::InsertStridedSliceOp::create(rewriter,
           loc, resVals[w], maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           /*strides=*/ArrayRef<int64_t>{1, 1, 1});
@@ -3853,7 +3853,7 @@ struct Conv1DGenerator
     //===------------------------------------------------------------------===//
 
     // Write back res slice of size {n, w, c} @ [0, 0, 0].
-    Operation *resOut = rewriter.create<vector::TransferWriteOp>(
+    Operation *resOut = vector::TransferWriteOp::create(rewriter,
         loc, maybeMaskedRes->getResult(0), resShaped,
         ValueRange{zero, zero, zero});
     return maybeMaskXferOp(resType.getShape(), resType.getScalableDims(),
@@ -3891,10 +3891,10 @@ struct Conv1DGenerator
           indices.push_back(j);
       }
 
-      rhs = rewriter.create<vector::ShuffleOp>(loc, rhs, rhs, indices);
+      rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, indices);
     }
     // Broadcast the filter to match the output vector
-    rhs = rewriter.create<vector::BroadcastOp>(
+    rhs = vector::BroadcastOp::create(rewriter,
         loc, resTy.clone(rhsTy.getElementType()), rhs);
 
     rhs = promote(rewriter, loc, rhs, resTy);
@@ -3903,10 +3903,10 @@ struct Conv1DGenerator
       return nullptr;
 
     if (isa<FloatType>(resTy.getElementType()))
-      return rewriter.create<vector::FMAOp>(loc, lhs, rhs, res);
+      return vector::FMAOp::create(rewriter, loc, lhs, rhs, res);
 
-    auto mul = rewriter.create<arith::MulIOp>(loc, lhs, rhs);
-    return rewriter.create<arith::AddIOp>(loc, mul, res);
+    auto mul = arith::MulIOp::create(rewriter, loc, lhs, rhs);
+    return arith::AddIOp::create(rewriter, loc, mul, res);
   }
 
   /// Entry point for non-channeled convolution:
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index 9fd084487e3fd..f9c6aae9374a8 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -201,7 +201,7 @@ Value create2DTransformMatrix(OpBuilder &builder, Location loc,
                               TransformMatrix transform, Type type) {
   ArrayRef<float> constVec(transform.table, transform.rows * transform.cols);
 
-  return builder.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(builder,
       loc, DenseFPElementsAttr::get(
                RankedTensorType::get(
                    SmallVector<int64_t>{transform.rows, transform.cols}, type),
@@ -233,7 +233,7 @@ Value extract2DDataFrom4D(OpBuilder &builder, Location loc, Value source,
 
   auto extractFilterType =
       RankedTensorType::get({extractHeight, extractWidth}, elementType);
-  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
+  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
       loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
@@ -267,7 +267,7 @@ Value extract2DDataFrom6D(OpBuilder &builder, Location loc, Value source,
   SmallVector<OpFoldResult> strides(srcSize, oneIndex);
 
   auto extractFilterType = RankedTensorType::get({height, width}, elementType);
-  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
+  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
       loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
@@ -293,7 +293,7 @@ Value insert2DDataTo4D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
+  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
       loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
@@ -321,7 +321,7 @@ Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
+  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
       loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
@@ -372,7 +372,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
   if (filterW != r && filterW != 1)
     return Value();
 
-  Value zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Value zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
                        ValueRange args) -> scf::ValueVector {
     Value FIter = ivs[0];
@@ -386,7 +386,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
 
     int64_t retRows = 1;
     Value matmulRetValue = extractFilter;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix G.
@@ -401,11 +401,11 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value G = create2DTransformMatrix(builder, loc, GMatrix, elementType);
       // Multiply G x g.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{G, extractFilter}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -423,11 +423,11 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value GT = create2DTransformMatrix(builder, loc, GTMatrix, elementType);
       // Multiply u = (G x g) x GT.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, GT}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -445,9 +445,9 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
     return {insertSliceOp};
   };
 
-  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterF);
-  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterC);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterF);
+  auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterC);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx}, {fUpperBound, cUpperBound},
       {oneStep, oneStep}, {retValue}, buildBody);
@@ -516,9 +516,9 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = builder.create<affine::AffineApplyOp>(
+    Value heightOffset = affine::AffineApplyOp::create(builder,
         loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = builder.create<affine::AffineApplyOp>(
+    Value widthOffset = affine::AffineApplyOp::create(builder,
         loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     // Extract (H, W) from (N, H, W, C).
@@ -530,7 +530,7 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     int64_t retRows = 1;
     int64_t retCols = 1;
     Value matmulRetValue = extractInput;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix BT.
@@ -545,12 +545,12 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value BT =
           create2DTransformMatrix(builder, loc, BTMatrix, builder.getF32Type());
       // Multiply BT x d.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{BT, matmulRetValue}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -568,11 +568,11 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       Value B =
           create2DTransformMatrix(builder, loc, BMatrix, builder.getF32Type());
       // Multiply v = (BT x d) x B.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, B}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -586,12 +586,12 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     return {combinedVal};
   };
 
-  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tileH);
-  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
-  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputN);
-  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputC);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tileH);
+  auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
+  auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputN);
+  auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputC);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
       {tileHBound, tileWBound, nUpperBound, cUpperBound},
@@ -629,7 +629,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       {filterShape[0] * filterShape[1], filterShape[2], filterShape[3]},
       filterElementType);
   SmallVector<ReassociationIndices> filterReassoc = {{0, 1}, {2}, {3}};
-  Value collapseFilter = rewriter.create<tensor::CollapseShapeOp>(
+  Value collapseFilter = tensor::CollapseShapeOp::create(rewriter,
       loc, filterReassocType, transformedFilter, filterReassoc);
 
   // Convert (alphaH, alphaW, tileH, tileW, N, C) to
@@ -643,7 +643,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
        inputShape[2] * inputShape[3] * inputShape[4], inputShape[5]},
       inputElementType);
   SmallVector<ReassociationIndices> inputReassoc = {{0, 1}, {2, 3, 4}, {5}};
-  Value collapseInput = rewriter.create<tensor::CollapseShapeOp>(
+  Value collapseInput = tensor::CollapseShapeOp::create(rewriter,
       loc, inputReassocType, transformedInput, inputReassoc);
 
   // Batched matrix multiply.
@@ -655,11 +655,11 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
                     .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                              outputElementType)
                     .getResult();
-  Value zero = rewriter.create<arith::ConstantOp>(
+  Value zero = arith::ConstantOp::create(rewriter,
       loc, rewriter.getZeroAttr(outputElementType));
-  Value init = rewriter.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+  Value init = linalg::FillOp::create(rewriter, loc, zero, empty).getResult(0);
 
-  auto matmulOp = rewriter.create<linalg::BatchMatmulOp>(
+  auto matmulOp = linalg::BatchMatmulOp::create(rewriter,
       loc, matmulType, ValueRange({collapseInput, collapseFilter}),
       ValueRange{init});
 
@@ -670,7 +670,7 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       RankedTensorType::get({inputShape[0], inputShape[1], inputShape[2],
                              inputShape[3], inputShape[4], filterShape[3]},
                             outputElementType);
-  auto expandOutput = rewriter.create<tensor::ExpandShapeOp>(
+  auto expandOutput = tensor::ExpandShapeOp::create(rewriter,
       loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
   return expandOutput;
 }
@@ -750,15 +750,15 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
     int64_t retRows = leftTransform ? ATMatrix.rows : 1;
 
     Value matmulRetValue = extractValue;
-    Value zero = builder.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(builder,
         loc, rewriter.getZeroAttr(elementType));
 
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = builder.create<affine::AffineApplyOp>(
+    Value heightOffset = affine::AffineApplyOp::create(builder,
         loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = builder.create<affine::AffineApplyOp>(
+    Value widthOffset = affine::AffineApplyOp::create(builder,
         loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
 
     Value outInitVal =
@@ -775,12 +775,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                          .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                                   elementType)
                          .getResult();
-        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+        init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       }
 
       Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
       // Multiply AT x m.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{AT, matmulRetValue}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
@@ -794,19 +794,19 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                          .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                                   elementType)
                          .getResult();
-        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+        init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       }
 
       Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
       // Multiply y = (AT x m) x A.
-      auto matmulOp = builder.create<linalg::MatmulOp>(
+      auto matmulOp = linalg::MatmulOp::create(builder,
           loc, matmulType, ValueRange{matmulRetValue, A}, ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
     if (scalarFactor != 1) {
       // Multiply by scalar factor and add outInitVal.
-      Value scalarFactorValue = builder.create<arith::ConstantOp>(
+      Value scalarFactorValue = arith::ConstantOp::create(builder,
           loc, FloatAttr::get(elementType, scalarFactor));
       auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
       auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
@@ -824,11 +824,11 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                       utils::IteratorType::parallel},
                   [&](OpBuilder &nestedBuilder, Location nestedLoc,
                       ValueRange args) {
-                    auto mulf = nestedBuilder.create<arith::MulFOp>(
+                    auto mulf = arith::MulFOp::create(nestedBuilder,
                         nestedLoc, args[0], args[1]);
-                    auto addf = nestedBuilder.create<arith::AddFOp>(
+                    auto addf = arith::AddFOp::create(nestedBuilder,
                         nestedLoc, mulf.getResult(), args[2]);
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc,
+                    linalg::YieldOp::create(nestedBuilder, nestedLoc,
                                                           addf.getResult());
                   })
               .getResult(0);
@@ -847,12 +847,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
   int64_t tilwH = valueShape[2];
   int64_t tileW = valueShape[3];
-  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tilwH);
-  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
-  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueN);
-  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueF);
-  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tilwH);
+  auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
+  auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueN);
+  auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueF);
+  auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
   scf::LoopNest loops = scf::buildLoopNest(
       rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
       {tileHBound, tileWBound, nUpperBound, fUpperBound},
@@ -867,7 +867,7 @@ static Value padToAlignedTensor(RewriterBase &rewriter, Location loc,
   auto valueType = cast<ShapedType>(value.getType());
   Type elementType = valueType.getElementType();
   auto alignedType = RankedTensorType::get(alignedShape, elementType);
-  Value padValue = rewriter.create<arith::ConstantOp>(
+  Value padValue = arith::ConstantOp::create(rewriter,
       loc, elementType, rewriter.getZeroAttr(elementType));
 
   return linalg::makeComposedPadHighOp(rewriter, loc, alignedType, value,
@@ -887,7 +887,7 @@ static Value extractFromAlignedTensor(RewriterBase &rewriter, Location loc,
   SmallVector<OpFoldResult> sizes =
       getAsOpFoldResult(rewriter.getI64ArrayAttr(extractedShape));
 
-  return rewriter.create<tensor::ExtractSliceOp>(loc, extractedType, value,
+  return tensor::ExtractSliceOp::create(rewriter, loc, extractedType, value,
                                                  offsets, sizes, strides);
 }
 
@@ -979,9 +979,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   int64_t tileW = llvm::divideCeilSigned(outputW, widthM);
   auto retType = RankedTensorType::get({alphaH, alphaW, filterC, filterF},
                                        filterElementType);
-  Value retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
+  Value retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
                                                     filterElementType);
-  auto transformedFilter = rewriter.create<linalg::WinogradFilterTransformOp>(
+  auto transformedFilter = linalg::WinogradFilterTransformOp::create(rewriter,
       loc, retType, filter, retValue, fmr);
 
   // --- Create operation for input transform ---
@@ -998,9 +998,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
 
   retType = RankedTensorType::get(
       {alphaH, alphaW, tileH, tileW, inputN, inputC}, inputElementType);
-  retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
+  retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
                                               inputElementType);
-  auto transformedInput = rewriter.create<linalg::WinogradInputTransformOp>(
+  auto transformedInput = linalg::WinogradInputTransformOp::create(rewriter,
       loc, retType, input, retValue, fmr);
 
   Type outputElementType = outputType.getElementType();
@@ -1023,7 +1023,7 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
     outputType = alignedOutputType;
   }
 
-  Value transformedOutput = rewriter.create<linalg::WinogradOutputTransformOp>(
+  Value transformedOutput = linalg::WinogradOutputTransformOp::create(rewriter,
       loc, outputType, matmulRet, output, fmr);
 
   // When output size is not aligned with output tile size, extract the
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index e89e80b07d20d..d964aa5cb4e81 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -320,14 +320,14 @@ GenericOp makeMemRefCopyOp(OpBuilder &b, Location loc, Value from, Value to) {
       AffineMap::getMultiDimIdentityMap(memrefTypeTo.getRank(), b.getContext());
   SmallVector<utils::IteratorType> iteratorTypes(memrefTypeTo.getRank(),
                                                  utils::IteratorType::parallel);
-  return b.create<linalg::GenericOp>(
+  return linalg::GenericOp::create(b,
       loc,
       /*inputs=*/from,
       /*outputs=*/to,
       /*indexingMaps=*/llvm::ArrayRef({id, id}),
       /*iteratorTypes=*/iteratorTypes,
       [](OpBuilder &b, Location loc, ValueRange args) {
-        b.create<linalg::YieldOp>(loc, args.front());
+        linalg::YieldOp::create(b, loc, args.front());
       });
 }
 
@@ -483,7 +483,7 @@ static void generateParallelLoopNest(
   case DistributionMethod::None: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    b.create<scf::ParallelOp>(
+    scf::ParallelOp::create(b,
         loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
@@ -499,7 +499,7 @@ static void generateParallelLoopNest(
   case DistributionMethod::Cyclic: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    b.create<scf::ParallelOp>(
+    scf::ParallelOp::create(b,
         loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
@@ -519,13 +519,13 @@ static void generateParallelLoopNest(
     for (unsigned i = 1; i < numProcessed; ++i)
       cond = ab._and(cond, ab.slt(lbs[i], ubs[i]));
     ivStorage.append(lbs.begin(), std::next(lbs.begin(), numProcessed));
-    b.create<scf::IfOp>(loc, cond, [&](OpBuilder &b, Location loc) {
+    scf::IfOp::create(b, loc, cond, [&](OpBuilder &b, Location loc) {
       generateParallelLoopNest(b, loc, lbs.drop_front(numProcessed),
                                ubs.drop_front(numProcessed),
                                steps.drop_front(numProcessed),
                                iteratorTypes.drop_front(numProcessed),
                                remainderProcInfo, bodyBuilderFn, ivStorage);
-      b.create<scf::YieldOp>(loc, ValueRange{});
+      scf::YieldOp::create(b, loc, ValueRange{});
     });
     return;
   }
@@ -595,12 +595,12 @@ static Operation *materializeTiledShape(OpBuilder &builder, Location loc,
   auto shapedType = dyn_cast<ShapedType>(valueToTile.getType());
   auto *sliceOp = TypeSwitch<ShapedType, Operation *>(shapedType)
                       .Case([&](MemRefType) {
-                        return builder.create<memref::SubViewOp>(
+                        return memref::SubViewOp::create(builder,
                             loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
                       .Case([&](RankedTensorType) {
-                        return builder.create<tensor::ExtractSliceOp>(
+                        return tensor::ExtractSliceOp::create(builder,
                             loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
@@ -793,7 +793,7 @@ SmallVector<Value> insertSlicesBack(OpBuilder &builder, Location loc,
     // `tiledOperands`.
     Value outputTensor = operands[opOperand.getOperandNumber()];
     if (auto sliceOp = outputTensor.getDefiningOp<tensor::ExtractSliceOp>()) {
-      Value inserted = builder.create<tensor::InsertSliceOp>(
+      Value inserted = tensor::InsertSliceOp::create(builder,
           loc, sliceOp.getSource().getType(), results[resultIdx],
           sliceOp.getSource(), sliceOp.getOffsets(), sliceOp.getSizes(),
           sliceOp.getStrides(), sliceOp.getStaticOffsets(),
diff --git a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
index ff6af63eee531..9dadd9bea2f67 100644
--- a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -135,7 +135,7 @@ struct GlobalStoreOpInterface
     auto memrefType = getMemRefTypeWithStaticIdentityLayout(tensorType);
 
     auto loc = globalStoreOp.getLoc();
-    auto targetMemref = rewriter.create<memref::GetGlobalOp>(
+    auto targetMemref = memref::GetGlobalOp::create(rewriter,
         loc, memrefType, globalStoreOp.getGlobalAttr().getLeafReference());
 
     auto sourceMemref =
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index 7940ff60a48e7..749cca0575581 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -60,7 +60,7 @@ struct FoldRank final : public mlir::OpRewritePattern<mlir::mpi::CommRankOp> {
     if (!isa<IntegerAttr>(dltiAttr.value()))
       return op->emitError()
              << "Expected an integer attribute for MPI:comm_world_rank";
-    Value res = b.create<arith::ConstantIndexOp>(
+    Value res = arith::ConstantIndexOp::create(b,
         op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
     if (Value retVal = op.getRetval())
       b.replaceOp(op, {retVal, res});
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index 26441a9d78658..a21631cbf8510 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -746,7 +746,7 @@ Operation *math::MathDialect::materializeConstant(OpBuilder &builder,
                                                   Attribute value, Type type,
                                                   Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
index 13e2a4b5541b2..770f32841c0f6 100644
--- a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
@@ -65,7 +65,7 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
   // Maybe broadcasts scalar value into vector type compatible with `op`.
   auto bcast = [&](Value value) -> Value {
     if (auto vec = dyn_cast<VectorType>(op.getType()))
-      return rewriter.create<vector::BroadcastOp>(op.getLoc(), vec, value);
+      return vector::BroadcastOp::create(rewriter, op.getLoc(), vec, value);
     return value;
   };
 
@@ -84,14 +84,14 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
   // Replace `pow(x, 3.0)` with `x * x * x`.
   if (isExponentValue(3.0)) {
     Value square =
-        rewriter.create<arith::MulFOp>(op.getLoc(), ValueRange({x, x}));
+        arith::MulFOp::create(rewriter, op.getLoc(), ValueRange({x, x}));
     rewriter.replaceOpWithNewOp<arith::MulFOp>(op, ValueRange({x, square}));
     return success();
   }
 
   // Replace `pow(x, -1.0)` with `1.0 / x`.
   if (isExponentValue(-1.0)) {
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getFloatAttr(getElementTypeOrSelf(op.getType()), 1.0));
     rewriter.replaceOpWithNewOp<arith::DivFOp>(op, ValueRange({bcast(one), x}));
     return success();
@@ -111,8 +111,8 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
 
   // Replace `pow(x, 0.75)` with `sqrt(sqrt(x)) * sqrt(x)`.
   if (isExponentValue(0.75)) {
-    Value powHalf = rewriter.create<math::SqrtOp>(op.getLoc(), x);
-    Value powQuarter = rewriter.create<math::SqrtOp>(op.getLoc(), powHalf);
+    Value powHalf = math::SqrtOp::create(rewriter, op.getLoc(), x);
+    Value powQuarter = math::SqrtOp::create(rewriter, op.getLoc(), powHalf);
     rewriter.replaceOpWithNewOp<arith::MulFOp>(op,
                                                ValueRange{powHalf, powQuarter});
     return success();
@@ -168,17 +168,17 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   // Maybe broadcasts scalar value into vector type compatible with `op`.
   auto bcast = [&loc, &op, &rewriter](Value value) -> Value {
     if (auto vec = dyn_cast<VectorType>(op.getType()))
-      return rewriter.create<vector::BroadcastOp>(loc, vec, value);
+      return vector::BroadcastOp::create(rewriter, loc, vec, value);
     return value;
   };
 
   Value one;
   Type opType = getElementTypeOrSelf(op.getType());
   if constexpr (std::is_same_v<PowIOpTy, math::FPowIOp>)
-    one = rewriter.create<arith::ConstantOp>(
+    one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getFloatAttr(opType, 1.0));
   else
-    one = rewriter.create<arith::ConstantOp>(
+    one = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(opType, 1));
 
   // Replace `[fi]powi(x, 0)` with `1`.
@@ -208,12 +208,12 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   //     with:
   //       (1 / x) * (1 / x) * (1 / x) * ...
   for (unsigned i = 1; i < exponentValue; ++i)
-    result = rewriter.create<MulOpTy>(loc, result, base);
+    result = MulOpTy::create(rewriter, loc, result, base);
 
   // Inverse the base for negative exponent, i.e. for
   // `[fi]powi(x, negative_exponent)` set `x` to `1 / x`.
   if (exponentIsNegative)
-    result = rewriter.create<DivOpTy>(loc, bcast(one), result);
+    result = DivOpTy::create(rewriter, loc, bcast(one), result);
 
   rewriter.replaceOp(op, result);
   return success();
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index bccd486def4bf..b5c0ddd617805 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -32,11 +32,11 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                 APFloat::rmNearestTiesToEven, &losesInfo);
   auto attr = b.getFloatAttr(eltType, value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return b.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(b, loc,
                                        DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return b.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(b, loc, attr);
 }
 
 static Value createFloatConst(Location loc, Type type, double value,
@@ -49,11 +49,11 @@ static Value createIntConst(Location loc, Type type, int64_t value,
                             OpBuilder &b) {
   auto attr = b.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return b.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(b, loc,
                                        DenseElementsAttr::get(shapedTy, attr));
   }
 
-  return b.create<arith::ConstantOp>(loc, attr);
+  return arith::ConstantOp::create(b, loc, attr);
 }
 
 static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
@@ -61,11 +61,11 @@ static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
   Type i64Ty = b.getI64Type();
   if (auto shapedTy = dyn_cast<ShapedType>(opType))
     i64Ty = shapedTy.clone(i64Ty);
-  Value fixedConvert = b.create<arith::FPToSIOp>(i64Ty, operand);
-  Value fpFixedConvert = b.create<arith::SIToFPOp>(opType, fixedConvert);
+  Value fixedConvert = arith::FPToSIOp::create(b, i64Ty, operand);
+  Value fpFixedConvert = arith::SIToFPOp::create(b, opType, fixedConvert);
   // The truncation does not preserve the sign when the truncated
   // value is -0. So here the sign is copied again.
-  return b.create<math::CopySignOp>(fpFixedConvert, operand);
+  return math::CopySignOp::create(b, fpFixedConvert, operand);
 }
 
 // sinhf(float x) -> (exp(x) - exp(-x)) / 2
@@ -74,12 +74,12 @@ static LogicalResult convertSinhOp(math::SinhOp op, PatternRewriter &rewriter) {
   Value operand = op.getOperand();
   Type opType = operand.getType();
 
-  Value exp = b.create<math::ExpOp>(operand);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value nexp = b.create<math::ExpOp>(neg);
-  Value sub = b.create<arith::SubFOp>(exp, nexp);
+  Value exp = math::ExpOp::create(b, operand);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value nexp = math::ExpOp::create(b, neg);
+  Value sub = arith::SubFOp::create(b, exp, nexp);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(sub, half);
+  Value res = arith::MulFOp::create(b, sub, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -90,12 +90,12 @@ static LogicalResult convertCoshOp(math::CoshOp op, PatternRewriter &rewriter) {
   Value operand = op.getOperand();
   Type opType = operand.getType();
 
-  Value exp = b.create<math::ExpOp>(operand);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value nexp = b.create<math::ExpOp>(neg);
-  Value add = b.create<arith::AddFOp>(exp, nexp);
+  Value exp = math::ExpOp::create(b, operand);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value nexp = math::ExpOp::create(b, neg);
+  Value add = arith::AddFOp::create(b, exp, nexp);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(add, half);
+  Value res = arith::MulFOp::create(b, add, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -116,23 +116,23 @@ static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) {
   Value negTwo = createFloatConst(loc, floatType, -2.0, rewriter);
 
   // Compute sign(x) = cast<float_type>(x < 0) * (-2) + 1
-  Value isNegative = rewriter.create<arith::CmpFOp>(
+  Value isNegative = arith::CmpFOp::create(rewriter,
       loc, arith::CmpFPredicate::OLT, op.getOperand(), zero);
   Value isNegativeFloat =
-      rewriter.create<arith::UIToFPOp>(loc, floatType, isNegative);
+      arith::UIToFPOp::create(rewriter, loc, floatType, isNegative);
   Value isNegativeTimesNegTwo =
-      rewriter.create<arith::MulFOp>(loc, isNegativeFloat, negTwo);
-  Value sign = rewriter.create<arith::AddFOp>(loc, isNegativeTimesNegTwo, one);
+      arith::MulFOp::create(rewriter, loc, isNegativeFloat, negTwo);
+  Value sign = arith::AddFOp::create(rewriter, loc, isNegativeTimesNegTwo, one);
 
   // Normalize input to positive value: y = sign(x) * x
-  Value positiveX = rewriter.create<arith::MulFOp>(loc, sign, op.getOperand());
+  Value positiveX = arith::MulFOp::create(rewriter, loc, sign, op.getOperand());
 
   // Decompose on normalized input
-  Value negDoubledX = rewriter.create<arith::MulFOp>(loc, negTwo, positiveX);
-  Value exp2x = rewriter.create<math::ExpOp>(loc, negDoubledX);
-  Value dividend = rewriter.create<arith::SubFOp>(loc, one, exp2x);
-  Value divisor = rewriter.create<arith::AddFOp>(loc, one, exp2x);
-  Value positiveRes = rewriter.create<arith::DivFOp>(loc, dividend, divisor);
+  Value negDoubledX = arith::MulFOp::create(rewriter, loc, negTwo, positiveX);
+  Value exp2x = math::ExpOp::create(rewriter, loc, negDoubledX);
+  Value dividend = arith::SubFOp::create(rewriter, loc, one, exp2x);
+  Value divisor = arith::AddFOp::create(rewriter, loc, one, exp2x);
+  Value positiveRes = arith::DivFOp::create(rewriter, loc, dividend, divisor);
 
   // Multiply result by sign(x) to retain signs from negative inputs
   rewriter.replaceOpWithNewOp<arith::MulFOp>(op, sign, positiveRes);
@@ -145,9 +145,9 @@ static LogicalResult convertTanOp(math::TanOp op, PatternRewriter &rewriter) {
   ImplicitLocOpBuilder b(op->getLoc(), rewriter);
   Value operand = op.getOperand();
   Type type = operand.getType();
-  Value sin = b.create<math::SinOp>(type, operand);
-  Value cos = b.create<math::CosOp>(type, operand);
-  Value div = b.create<arith::DivFOp>(type, sin, cos);
+  Value sin = math::SinOp::create(b, type, operand);
+  Value cos = math::CosOp::create(b, type, operand);
+  Value div = arith::DivFOp::create(b, type, sin, cos);
   rewriter.replaceOp(op, div);
   return success();
 }
@@ -160,10 +160,10 @@ static LogicalResult convertAsinhOp(math::AsinhOp op,
   Type opType = operand.getType();
 
   Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value fma = b.create<math::FmaOp>(operand, operand, one);
-  Value sqrt = b.create<math::SqrtOp>(fma);
-  Value add = b.create<arith::AddFOp>(operand, sqrt);
-  Value res = b.create<math::LogOp>(add);
+  Value fma = math::FmaOp::create(b, operand, operand, one);
+  Value sqrt = math::SqrtOp::create(b, fma);
+  Value add = arith::AddFOp::create(b, operand, sqrt);
+  Value res = math::LogOp::create(b, add);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -176,10 +176,10 @@ static LogicalResult convertAcoshOp(math::AcoshOp op,
   Type opType = operand.getType();
 
   Value negOne = createFloatConst(op->getLoc(), opType, -1.0, rewriter);
-  Value fma = b.create<math::FmaOp>(operand, operand, negOne);
-  Value sqrt = b.create<math::SqrtOp>(fma);
-  Value add = b.create<arith::AddFOp>(operand, sqrt);
-  Value res = b.create<math::LogOp>(add);
+  Value fma = math::FmaOp::create(b, operand, operand, negOne);
+  Value sqrt = math::SqrtOp::create(b, fma);
+  Value add = arith::AddFOp::create(b, operand, sqrt);
+  Value res = math::LogOp::create(b, add);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -192,13 +192,13 @@ static LogicalResult convertAtanhOp(math::AtanhOp op,
   Type opType = operand.getType();
 
   Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value add = b.create<arith::AddFOp>(operand, one);
-  Value neg = b.create<arith::NegFOp>(operand);
-  Value sub = b.create<arith::AddFOp>(neg, one);
-  Value div = b.create<arith::DivFOp>(add, sub);
-  Value log = b.create<math::LogOp>(div);
+  Value add = arith::AddFOp::create(b, operand, one);
+  Value neg = arith::NegFOp::create(b, operand);
+  Value sub = arith::AddFOp::create(b, neg, one);
+  Value div = arith::DivFOp::create(b, add, sub);
+  Value log = math::LogOp::create(b, div);
   Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
-  Value res = b.create<arith::MulFOp>(log, half);
+  Value res = arith::MulFOp::create(b, log, half);
   rewriter.replaceOp(op, res);
   return success();
 }
@@ -209,8 +209,8 @@ static LogicalResult convertFmaFOp(math::FmaOp op, PatternRewriter &rewriter) {
   Value operandB = op.getOperand(1);
   Value operandC = op.getOperand(2);
   Type type = op.getType();
-  Value mult = b.create<arith::MulFOp>(type, operandA, operandB);
-  Value add = b.create<arith::AddFOp>(type, mult, operandC);
+  Value mult = arith::MulFOp::create(b, type, operandA, operandB);
+  Value add = arith::AddFOp::create(b, type, mult, operandC);
   rewriter.replaceOp(op, add);
   return success();
 }
@@ -235,11 +235,11 @@ static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) {
   Value zero = createFloatConst(op->getLoc(), opType, 0.00, rewriter);
   Value one = createFloatConst(op->getLoc(), opType, 1.00, rewriter);
 
-  Value gtCheck = b.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand,
+  Value gtCheck = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT, operand,
                                           fpFixedConvert);
-  Value incrValue = b.create<arith::SelectOp>(op->getLoc(), gtCheck, one, zero);
+  Value incrValue = arith::SelectOp::create(b, op->getLoc(), gtCheck, one, zero);
 
-  Value ret = b.create<arith::AddFOp>(opType, fpFixedConvert, incrValue);
+  Value ret = arith::AddFOp::create(b, opType, fpFixedConvert, incrValue);
   rewriter.replaceOp(op, ret);
   return success();
 }
@@ -257,8 +257,8 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
 
   auto convertFPowItoPowf = [&]() -> LogicalResult {
     Value castPowerToFp =
-        rewriter.create<arith::SIToFPOp>(op.getLoc(), baseType, power);
-    Value res = rewriter.create<math::PowFOp>(op.getLoc(), baseType, base,
+        arith::SIToFPOp::create(rewriter, op.getLoc(), baseType, power);
+    Value res = math::PowFOp::create(rewriter, op.getLoc(), baseType, base,
                                               castPowerToFp);
     rewriter.replaceOp(op, res);
     return success();
@@ -280,9 +280,9 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
 
   while (absPower > 0) {
     if (absPower & 1)
-      res = b.create<arith::MulFOp>(baseType, base, res);
+      res = arith::MulFOp::create(b, baseType, base, res);
     absPower >>= 1;
-    base = b.create<arith::MulFOp>(baseType, base, base);
+    base = arith::MulFOp::create(b, baseType, base, base);
   }
 
   // Make sure not to introduce UB in case of negative power.
@@ -302,13 +302,13 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
         createFloatConst(op->getLoc(), baseType,
                          APFloat::getInf(sem, /*Negative=*/true), rewriter);
     Value zeroEqCheck =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, res, zero);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, res, zero);
     Value negZeroEqCheck =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, res, negZero);
-    res = b.create<arith::DivFOp>(baseType, one, res);
+        arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, res, negZero);
+    res = arith::DivFOp::create(b, baseType, one, res);
     res =
-        b.create<arith::SelectOp>(op->getLoc(), zeroEqCheck, posInfinity, res);
-    res = b.create<arith::SelectOp>(op->getLoc(), negZeroEqCheck, negInfinity,
+        arith::SelectOp::create(b, op->getLoc(), zeroEqCheck, posInfinity, res);
+    res = arith::SelectOp::create(b, op->getLoc(), negZeroEqCheck, negInfinity,
                                     res);
   }
 
@@ -330,7 +330,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       cast<mlir::FloatType>(getElementTypeOrSelf(typeB)).getFloatSemantics();
   APFloat valueB(sem);
   auto mulf = [&](Value x, Value y) -> Value {
-    return b.create<arith::MulFOp>(x, y);
+    return arith::MulFOp::create(b, x, y);
   };
   if (matchPattern(operandB, m_ConstantFloat(&valueB))) {
     if (valueB.isZero()) {
@@ -347,19 +347,19 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
     if (valueB.isExactlyValue(-1.0)) {
       // a^(-1) -> 1 / a
       Value one = createFloatConst(op->getLoc(), typeA, 1.0, rewriter);
-      Value div = b.create<arith::DivFOp>(one, operandA);
+      Value div = arith::DivFOp::create(b, one, operandA);
       rewriter.replaceOp(op, div);
       return success();
     }
     if (valueB.isExactlyValue(0.5)) {
       // a^(1/2) -> sqrt(a)
-      Value sqrt = b.create<math::SqrtOp>(operandA);
+      Value sqrt = math::SqrtOp::create(b, operandA);
       rewriter.replaceOp(op, sqrt);
       return success();
     }
     if (valueB.isExactlyValue(-0.5)) {
       // a^(-1/2) -> 1 / sqrt(a)
-      Value rsqrt = b.create<math::RsqrtOp>(operandA);
+      Value rsqrt = math::RsqrtOp::create(b, operandA);
       rewriter.replaceOp(op, rsqrt);
       return success();
     }
@@ -372,7 +372,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       // a^(-2) -> 1 / (a * a)
       Value one =
           createFloatConst(op->getLoc(), operandA.getType(), 1.0, rewriter);
-      Value div = b.create<arith::DivFOp>(one, mulf(operandA, operandA));
+      Value div = arith::DivFOp::create(b, one, mulf(operandA, operandA));
       rewriter.replaceOp(op, div);
       return success();
     }
@@ -382,9 +382,9 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
     }
   }
 
-  Value logA = b.create<math::LogOp>(operandA);
-  Value mult = b.create<arith::MulFOp>(operandB, logA);
-  Value expResult = b.create<math::ExpOp>(mult);
+  Value logA = math::LogOp::create(b, operandA);
+  Value mult = arith::MulFOp::create(b, operandB, logA);
+  Value expResult = math::ExpOp::create(b, mult);
   rewriter.replaceOp(op, expResult);
   return success();
 }
@@ -399,8 +399,8 @@ static LogicalResult convertExp2fOp(math::Exp2Op op,
   Value operand = op.getOperand();
   Type opType = operand.getType();
   Value ln2 = createFloatConst(op->getLoc(), opType, llvm::numbers::ln2, b);
-  Value mult = b.create<arith::MulFOp>(opType, operand, ln2);
-  Value exp = b.create<math::ExpOp>(op->getLoc(), mult);
+  Value mult = arith::MulFOp::create(b, opType, operand, ln2);
+  Value exp = math::ExpOp::create(b, op->getLoc(), mult);
   rewriter.replaceOp(op, exp);
   return success();
 }
@@ -426,8 +426,8 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   Value c127 = createIntConst(loc, i32Ty, 127, b);
   Value expMask = createIntConst(loc, i32Ty, (1 << 8) - 1, b);
 
-  Value incrValue = b.create<math::CopySignOp>(half, operand);
-  Value add = b.create<arith::AddFOp>(opType, operand, incrValue);
+  Value incrValue = math::CopySignOp::create(b, half, operand);
+  Value add = arith::AddFOp::create(b, opType, operand, incrValue);
   Value fpFixedConvert = createTruncatedFPValue(add, b);
 
   // There are three cases where adding 0.5 to the value and truncating by
@@ -450,14 +450,14 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   //     i64 leading to wrong outputs.
   //
   // All three cases satisfy the property `biasedExp >= 23`.
-  Value operandBitcast = b.create<arith::BitcastOp>(i32Ty, operand);
-  Value operandExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(operandBitcast, c23), expMask);
-  Value operandBiasedExp = b.create<arith::SubIOp>(operandExp, c127);
+  Value operandBitcast = arith::BitcastOp::create(b, i32Ty, operand);
+  Value operandExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
   Value isSpecialValOrLargeVal =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::sge, operandBiasedExp, c23);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::sge, operandBiasedExp, c23);
 
-  Value result = b.create<arith::SelectOp>(isSpecialValOrLargeVal, operand,
+  Value result = arith::SelectOp::create(b, isSpecialValOrLargeVal, operand,
                                            fpFixedConvert);
   rewriter.replaceOp(op, result);
   return success();
@@ -489,20 +489,20 @@ static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
     auto mask = createIntConst(loc, operandTy, allbits >> half, rewriter);
 
     Value pred =
-        rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule, x, mask);
-    Value add = rewriter.create<arith::AddIOp>(loc, count, bits);
-    Value shift = rewriter.create<arith::ShLIOp>(loc, x, bits);
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ule, x, mask);
+    Value add = arith::AddIOp::create(rewriter, loc, count, bits);
+    Value shift = arith::ShLIOp::create(rewriter, loc, x, bits);
 
-    x = rewriter.create<arith::SelectOp>(loc, pred, shift, x);
-    count = rewriter.create<arith::SelectOp>(loc, pred, add, count);
+    x = arith::SelectOp::create(rewriter, loc, pred, shift, x);
+    count = arith::SelectOp::create(rewriter, loc, pred, add, count);
   }
 
   Value zero = createIntConst(loc, operandTy, 0, rewriter);
-  Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+  Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                               operand, zero);
 
   Value bwval = createIntConst(loc, operandTy, bitwidth, rewriter);
-  Value sel = rewriter.create<arith::SelectOp>(loc, pred, bwval, count);
+  Value sel = arith::SelectOp::create(rewriter, loc, pred, bwval, count);
   rewriter.replaceOp(op, sel);
   return success();
 }
@@ -549,29 +549,29 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   Value c23Mask = createIntConst(loc, iTy, (1ull << mantissaWidth) - 1, b);
   Value expMask = createIntConst(loc, iTy, (1ull << exponentWidth) - 1, b);
 
-  Value operandBitcast = b.create<arith::BitcastOp>(iTy, operand);
-  Value round = b.create<math::RoundOp>(operand);
-  Value roundBitcast = b.create<arith::BitcastOp>(iTy, round);
+  Value operandBitcast = arith::BitcastOp::create(b, iTy, operand);
+  Value round = math::RoundOp::create(b, operand);
+  Value roundBitcast = arith::BitcastOp::create(b, iTy, round);
 
   // Get biased exponents for operand and round(operand)
-  Value operandExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(operandBitcast, c23), expMask);
-  Value operandBiasedExp = b.create<arith::SubIOp>(operandExp, c127);
-  Value roundExp = b.create<arith::AndIOp>(
-      b.create<arith::ShRUIOp>(roundBitcast, c23), expMask);
-  Value roundBiasedExp = b.create<arith::SubIOp>(roundExp, c127);
+  Value operandExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
+  Value roundExp = arith::AndIOp::create(b,
+      arith::ShRUIOp::create(b, roundBitcast, c23), expMask);
+  Value roundBiasedExp = arith::SubIOp::create(b, roundExp, c127);
 
   auto safeShiftRight = [&](Value x, Value shift) -> Value {
     // Clamp shift to valid range [0, bitwidth - 1] to avoid undefined behavior
-    Value clampedShift = b.create<arith::MaxSIOp>(shift, c0);
-    clampedShift = b.create<arith::MinSIOp>(clampedShift, c31);
-    return b.create<arith::ShRUIOp>(x, clampedShift);
+    Value clampedShift = arith::MaxSIOp::create(b, shift, c0);
+    clampedShift = arith::MinSIOp::create(b, clampedShift, c31);
+    return arith::ShRUIOp::create(b, x, clampedShift);
   };
 
   auto maskMantissa = [&](Value mantissa,
                           Value mantissaMaskRightShift) -> Value {
     Value shiftedMantissaMask = safeShiftRight(c23Mask, mantissaMaskRightShift);
-    return b.create<arith::AndIOp>(mantissa, shiftedMantissaMask);
+    return arith::AndIOp::create(b, mantissa, shiftedMantissaMask);
   };
 
   // A whole number `x`, such that `|x| != 1`, is even if the mantissa, ignoring
@@ -589,13 +589,13 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // `biasedExp > 23`, so they get treated as large numbers with no room for
   // decimals, which are always even.
   Value roundBiasedExpEq0 =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, roundBiasedExp, c0);
-  Value roundBiasedExpMinus1 = b.create<arith::SubIOp>(roundBiasedExp, c1);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, roundBiasedExp, c0);
+  Value roundBiasedExpMinus1 = arith::SubIOp::create(b, roundBiasedExp, c1);
   Value roundMaskedMantissa = maskMantissa(roundBitcast, roundBiasedExpMinus1);
-  Value roundIsNotEvenOrSpecialVal = b.create<arith::CmpIOp>(
+  Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(b,
       arith::CmpIPredicate::ne, roundMaskedMantissa, c0);
   roundIsNotEvenOrSpecialVal =
-      b.create<arith::OrIOp>(roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
+      arith::OrIOp::create(b, roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
 
   // A value `x` with `0 <= biasedExp < 23`, is halfway between two consecutive
   // integers if the bit at index `biasedExp` starting from the left in the
@@ -604,37 +604,37 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // values +-0.5 are the only halfway values that have `biasedExp == -1 < 0`,
   // so these are handled separately. In particular, if `biasedExp == -1`, the
   // value is halfway if the entire mantissa is zero.
-  Value operandBiasedExpEqNeg1 = b.create<arith::CmpIOp>(
+  Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(b,
       arith::CmpIPredicate::eq, operandBiasedExp, cNeg1);
-  Value expectedOperandMaskedMantissa = b.create<arith::SelectOp>(
+  Value expectedOperandMaskedMantissa = arith::SelectOp::create(b,
       operandBiasedExpEqNeg1, c0, safeShiftRight(c2To22, operandBiasedExp));
   Value operandMaskedMantissa = maskMantissa(operandBitcast, operandBiasedExp);
   Value operandIsHalfway =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, operandMaskedMantissa,
+      arith::CmpIOp::create(b, arith::CmpIPredicate::eq, operandMaskedMantissa,
                               expectedOperandMaskedMantissa);
   // Ensure `biasedExp` is in the valid range for half values.
-  Value operandBiasedExpGeNeg1 = b.create<arith::CmpIOp>(
+  Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(b,
       arith::CmpIPredicate::sge, operandBiasedExp, cNeg1);
   Value operandBiasedExpLt23 =
-      b.create<arith::CmpIOp>(arith::CmpIPredicate::slt, operandBiasedExp, c23);
+      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, operandBiasedExp, c23);
   operandIsHalfway =
-      b.create<arith::AndIOp>(operandIsHalfway, operandBiasedExpLt23);
+      arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpLt23);
   operandIsHalfway =
-      b.create<arith::AndIOp>(operandIsHalfway, operandBiasedExpGeNeg1);
+      arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpGeNeg1);
 
   // Adjust rounded operand with `round(operand) - sign(operand)` to correct the
   // case where `round` rounded in the opposite direction of `roundeven`.
-  Value sign = b.create<math::CopySignOp>(c1Float, operand);
-  Value roundShifted = b.create<arith::SubFOp>(round, sign);
+  Value sign = math::CopySignOp::create(b, c1Float, operand);
+  Value roundShifted = arith::SubFOp::create(b, round, sign);
   // If the rounded value is even or a special value, we default to the behavior
   // of `math.round`.
   Value needsShift =
-      b.create<arith::AndIOp>(roundIsNotEvenOrSpecialVal, operandIsHalfway);
-  Value result = b.create<arith::SelectOp>(needsShift, roundShifted, round);
+      arith::AndIOp::create(b, roundIsNotEvenOrSpecialVal, operandIsHalfway);
+  Value result = arith::SelectOp::create(b, needsShift, roundShifted, round);
   // The `x - sign` adjustment does not preserve the sign when we are adjusting
   // the value -1 to -0. So here the sign is copied again to ensure that -0.5 is
   // rounded to -0.0.
-  result = b.create<math::CopySignOp>(result, operand);
+  result = math::CopySignOp::create(b, result, operand);
   rewriter.replaceOp(op, result);
   return success();
 }
@@ -656,7 +656,7 @@ static LogicalResult convertRsqrtOp(math::RsqrtOp op,
 
   Location loc = op->getLoc();
   auto constOneFloat = createFloatConst(loc, operandTy, 1.0, rewriter);
-  auto sqrtOp = rewriter.create<math::SqrtOp>(loc, operand);
+  auto sqrtOp = math::SqrtOp::create(rewriter, loc, operand);
   rewriter.replaceOpWithNewOp<arith::DivFOp>(op, constOneFloat, sqrtOp);
   return success();
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
index a570ed5118ef0..9d6ad613fc945 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
@@ -73,7 +73,7 @@ void mlir::math::populateExtendToSupportedTypesTypeConverter(
       });
   typeConverter.addTargetMaterialization(
       [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        auto extFOp = arith::ExtFOp::create(b, loc, target, input);
         extFOp.setFastmath(arith::FastMathFlags::contract);
         return extFOp;
       });
@@ -104,7 +104,7 @@ LogicalResult ExtendToSupportedTypesRewritePattern::matchAndRewrite(
   for (auto [result, newType, origType] : llvm::zip_equal(
            results, (*legalized)->getResultTypes(), op->getResultTypes())) {
     if (newType != origType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, origType, result);
+      auto truncFOp = arith::TruncFOp::create(rewriter, loc, origType, result);
       truncFOp.setFastmath(arith::FastMathFlags::contract);
       result = truncFOp.getResult();
     }
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index dd2dfe372b683..baa35fb5a2c5d 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -72,7 +72,7 @@ static Value broadcast(ImplicitLocOpBuilder &builder, Value value,
                        std::optional<VectorShape> shape) {
   assert(!isa<VectorType>(value.getType()) && "must be scalar value");
   auto type = broadcast(value.getType(), shape);
-  return shape ? builder.create<BroadcastOp>(type, value) : value;
+  return shape ? BroadcastOp::create(builder, type, value) : value;
 }
 
 //----------------------------------------------------------------------------//
@@ -130,7 +130,7 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
       auto eltType = cast<VectorType>(operand.getType()).getElementType();
       auto expandedType = VectorType::get(expandedShape, eltType);
       expandedOperands[i] =
-          builder.create<vector::ShapeCastOp>(expandedType, operand);
+          vector::ShapeCastOp::create(builder, expandedType, operand);
     }
   }
 
@@ -148,7 +148,7 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
     SmallVector<Value> extracted(expandedOperands.size());
     for (const auto &tuple : llvm::enumerate(expandedOperands))
       extracted[tuple.index()] =
-          builder.create<vector::ExtractOp>(tuple.value(), offsets);
+          vector::ExtractOp::create(builder, tuple.value(), offsets);
 
     results[i] = compute(extracted);
   }
@@ -156,15 +156,15 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
   // Stitch results together into one large vector.
   Type resultEltType = cast<VectorType>(results[0].getType()).getElementType();
   Type resultExpandedType = VectorType::get(expandedShape, resultEltType);
-  Value result = builder.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(builder,
       resultExpandedType, builder.getZeroAttr(resultExpandedType));
 
   for (int64_t i = 0; i < maxIndex; ++i)
-    result = builder.create<vector::InsertOp>(results[i], result,
+    result = vector::InsertOp::create(builder, results[i], result,
                                               delinearize(i, strides));
 
   // Reshape back to the original vector shape.
-  return builder.create<vector::ShapeCastOp>(
+  return vector::ShapeCastOp::create(builder,
       VectorType::get(inputShape, resultEltType), result);
 }
 
@@ -173,28 +173,28 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
 //----------------------------------------------------------------------------//
 
 static Value boolCst(ImplicitLocOpBuilder &builder, bool value) {
-  return builder.create<arith::ConstantOp>(builder.getBoolAttr(value));
+  return arith::ConstantOp::create(builder, builder.getBoolAttr(value));
 }
 
 static Value floatCst(ImplicitLocOpBuilder &builder, float value,
                       Type elementType) {
   assert((elementType.isF16() || elementType.isF32()) &&
          "x must be f16 or f32 type.");
-  return builder.create<arith::ConstantOp>(
+  return arith::ConstantOp::create(builder,
       builder.getFloatAttr(elementType, value));
 }
 
 static Value f32Cst(ImplicitLocOpBuilder &builder, double value) {
-  return builder.create<arith::ConstantOp>(builder.getF32FloatAttr(value));
+  return arith::ConstantOp::create(builder, builder.getF32FloatAttr(value));
 }
 
 static Value i32Cst(ImplicitLocOpBuilder &builder, int32_t value) {
-  return builder.create<arith::ConstantOp>(builder.getI32IntegerAttr(value));
+  return arith::ConstantOp::create(builder, builder.getI32IntegerAttr(value));
 }
 
 static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
   Value i32Value = i32Cst(builder, static_cast<int32_t>(bits));
-  return builder.create<arith::BitcastOp>(builder.getF32Type(), i32Value);
+  return arith::BitcastOp::create(builder, builder.getF32Type(), i32Value);
 }
 
 //----------------------------------------------------------------------------//
@@ -203,15 +203,15 @@ static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
 
 // Return the minimum of the two values or NaN if value is NaN
 static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return builder.create<arith::SelectOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT, value, bound),
+  return arith::SelectOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT, value, bound),
       value, bound);
 }
 
 // Return the maximum of the two values or NaN if value is NaN
 static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return builder.create<arith::SelectOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::UGT, value, bound),
+  return arith::SelectOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::UGT, value, bound),
       value, bound);
 }
 
@@ -241,24 +241,24 @@ static std::pair<Value, Value> frexp(ImplicitLocOpBuilder &builder, Value arg,
   Value cstInvMantMask = f32FromBits(builder, ~0x7f800000u);
 
   // Bitcast to i32 for bitwise operations.
-  Value i32Half = builder.create<arith::BitcastOp>(i32, cstHalf);
-  Value i32InvMantMask = builder.create<arith::BitcastOp>(i32, cstInvMantMask);
-  Value i32Arg = builder.create<arith::BitcastOp>(i32Vec, arg);
+  Value i32Half = arith::BitcastOp::create(builder, i32, cstHalf);
+  Value i32InvMantMask = arith::BitcastOp::create(builder, i32, cstInvMantMask);
+  Value i32Arg = arith::BitcastOp::create(builder, i32Vec, arg);
 
   // Compute normalized fraction.
-  Value tmp0 = builder.create<arith::AndIOp>(i32Arg, bcast(i32InvMantMask));
-  Value tmp1 = builder.create<arith::OrIOp>(tmp0, bcast(i32Half));
-  Value normalizedFraction = builder.create<arith::BitcastOp>(f32Vec, tmp1);
+  Value tmp0 = arith::AndIOp::create(builder, i32Arg, bcast(i32InvMantMask));
+  Value tmp1 = arith::OrIOp::create(builder, tmp0, bcast(i32Half));
+  Value normalizedFraction = arith::BitcastOp::create(builder, f32Vec, tmp1);
 
   // Compute exponent.
-  Value arg0 = isPositive ? arg : builder.create<math::AbsFOp>(arg);
-  Value biasedExponentBits = builder.create<arith::ShRUIOp>(
-      builder.create<arith::BitcastOp>(i32Vec, arg0),
+  Value arg0 = isPositive ? arg : math::AbsFOp::create(builder, arg);
+  Value biasedExponentBits = arith::ShRUIOp::create(builder,
+      arith::BitcastOp::create(builder, i32Vec, arg0),
       bcast(i32Cst(builder, 23)));
   Value biasedExponent =
-      builder.create<arith::SIToFPOp>(f32Vec, biasedExponentBits);
+      arith::SIToFPOp::create(builder, f32Vec, biasedExponentBits);
   Value exponent =
-      builder.create<arith::SubFOp>(biasedExponent, bcast(cst126f));
+      arith::SubFOp::create(builder, biasedExponent, bcast(cst126f));
 
   return {normalizedFraction, exponent};
 }
@@ -278,10 +278,10 @@ static Value exp2I32(ImplicitLocOpBuilder &builder, Value arg) {
   // Set the exponent bias to zero.
   auto bias = bcast(i32Cst(builder, 127));
 
-  Value biasedArg = builder.create<arith::AddIOp>(arg, bias);
+  Value biasedArg = arith::AddIOp::create(builder, arg, bias);
   Value exp2ValueInt =
-      builder.create<arith::ShLIOp>(biasedArg, exponetBitLocation);
-  Value exp2ValueF32 = builder.create<arith::BitcastOp>(f32Vec, exp2ValueInt);
+      arith::ShLIOp::create(builder, biasedArg, exponetBitLocation);
+  Value exp2ValueF32 = arith::BitcastOp::create(builder, f32Vec, exp2ValueInt);
 
   return exp2ValueF32;
 }
@@ -300,10 +300,10 @@ Value makePolynomialCalculation(ImplicitLocOpBuilder &builder,
   if (coeffs.size() == 1)
     return coeffs[0];
 
-  Value res = builder.create<math::FmaOp>(x, coeffs[coeffs.size() - 1],
+  Value res = math::FmaOp::create(builder, x, coeffs[coeffs.size() - 1],
                                           coeffs[coeffs.size() - 2]);
   for (auto i = ptrdiff_t(coeffs.size()) - 3; i >= 0; --i) {
-    res = builder.create<math::FmaOp>(x, res, coeffs[i]);
+    res = math::FmaOp::create(builder, x, res, coeffs[i]);
   }
   return res;
 }
@@ -343,9 +343,9 @@ LogicalResult insertCasts(Operation *op, PatternRewriter &rewriter) {
   Location loc = op->getLoc();
   SmallVector<Value> operands;
   for (auto operand : op->getOperands())
-    operands.push_back(rewriter.create<arith::ExtFOp>(loc, newType, operand));
+    operands.push_back(arith::ExtFOp::create(rewriter, loc, newType, operand));
   auto result =
-      rewriter.create<T>(loc, TypeRange{newType}, operands, op->getAttrs());
+      T::create(rewriter, loc, TypeRange{newType}, operands, op->getAttrs());
   rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, origType, result);
   return success();
 }
@@ -393,18 +393,18 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
   std::optional<VectorShape> shape = vectorShape(op.getOperand());
 
   ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
-  Value abs = builder.create<math::AbsFOp>(operand);
+  Value abs = math::AbsFOp::create(builder, operand);
 
   auto one = broadcast(builder, f32Cst(builder, 1.0), shape);
 
   // When 0.66 < x <= 2.41 we do (x-1) / (x+1):
   auto twoThirds = broadcast(builder, f32Cst(builder, 0.66), shape);
   Value cmp2 =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, abs, twoThirds);
-  Value addone = builder.create<arith::AddFOp>(abs, one);
-  Value subone = builder.create<arith::SubFOp>(abs, one);
-  Value xnum = builder.create<arith::SelectOp>(cmp2, subone, abs);
-  Value xden = builder.create<arith::SelectOp>(cmp2, addone, one);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, abs, twoThirds);
+  Value addone = arith::AddFOp::create(builder, abs, one);
+  Value subone = arith::SubFOp::create(builder, abs, one);
+  Value xnum = arith::SelectOp::create(builder, cmp2, subone, abs);
+  Value xden = arith::SelectOp::create(builder, cmp2, addone, one);
 
   auto bcast = [&](Value value) -> Value {
     return broadcast(builder, value, shape);
@@ -413,12 +413,12 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
   // Break into the <= 0.66 or > 2.41 we do x or 1/x:
   auto tan3pio8 = bcast(f32Cst(builder, 2.41421356237309504880));
   Value cmp1 =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, abs, tan3pio8);
-  xnum = builder.create<arith::SelectOp>(cmp1, one, xnum);
-  xden = builder.create<arith::SelectOp>(cmp1, abs, xden);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, abs, tan3pio8);
+  xnum = arith::SelectOp::create(builder, cmp1, one, xnum);
+  xden = arith::SelectOp::create(builder, cmp1, abs, xden);
 
-  Value x = builder.create<arith::DivFOp>(xnum, xden);
-  Value xx = builder.create<arith::MulFOp>(x, x);
+  Value x = arith::DivFOp::create(builder, xnum, xden);
+  Value xx = arith::MulFOp::create(builder, x, x);
 
   // Perform the Taylor series approximation for atan over the range
   // [0.0, 0.66].
@@ -435,31 +435,31 @@ AtanApproximation::matchAndRewrite(math::AtanOp op,
 
   // Apply the polynomial approximation for the numerator:
   Value n = p0;
-  n = builder.create<math::FmaOp>(xx, n, p1);
-  n = builder.create<math::FmaOp>(xx, n, p2);
-  n = builder.create<math::FmaOp>(xx, n, p3);
-  n = builder.create<math::FmaOp>(xx, n, p4);
-  n = builder.create<arith::MulFOp>(n, xx);
+  n = math::FmaOp::create(builder, xx, n, p1);
+  n = math::FmaOp::create(builder, xx, n, p2);
+  n = math::FmaOp::create(builder, xx, n, p3);
+  n = math::FmaOp::create(builder, xx, n, p4);
+  n = arith::MulFOp::create(builder, n, xx);
 
   // Apply the polynomial approximation for the denominator:
   Value d = q0;
-  d = builder.create<math::FmaOp>(xx, d, q1);
-  d = builder.create<math::FmaOp>(xx, d, q2);
-  d = builder.create<math::FmaOp>(xx, d, q3);
-  d = builder.create<math::FmaOp>(xx, d, q4);
+  d = math::FmaOp::create(builder, xx, d, q1);
+  d = math::FmaOp::create(builder, xx, d, q2);
+  d = math::FmaOp::create(builder, xx, d, q3);
+  d = math::FmaOp::create(builder, xx, d, q4);
 
   // Compute approximation of theta:
-  Value ans0 = builder.create<arith::DivFOp>(n, d);
-  ans0 = builder.create<math::FmaOp>(ans0, x, x);
+  Value ans0 = arith::DivFOp::create(builder, n, d);
+  ans0 = math::FmaOp::create(builder, ans0, x, x);
 
   // Correct for the input mapping's angles:
   Value mpi4 = bcast(f32Cst(builder, llvm::numbers::pi / 4));
-  Value ans2 = builder.create<arith::AddFOp>(mpi4, ans0);
-  Value ans = builder.create<arith::SelectOp>(cmp2, ans2, ans0);
+  Value ans2 = arith::AddFOp::create(builder, mpi4, ans0);
+  Value ans = arith::SelectOp::create(builder, cmp2, ans2, ans0);
 
   Value mpi2 = bcast(f32Cst(builder, llvm::numbers::pi / 2));
-  Value ans1 = builder.create<arith::SubFOp>(mpi2, ans0);
-  ans = builder.create<arith::SelectOp>(cmp1, ans1, ans);
+  Value ans1 = arith::SubFOp::create(builder, mpi2, ans0);
+  ans = arith::SelectOp::create(builder, cmp1, ans1, ans);
 
   // Correct for signing of the input.
   rewriter.replaceOpWithNewOp<math::CopySignOp>(op, ans, operand);
@@ -492,44 +492,44 @@ Atan2Approximation::matchAndRewrite(math::Atan2Op op,
   std::optional<VectorShape> shape = vectorShape(op.getResult());
 
   // Compute atan in the valid range.
-  auto div = builder.create<arith::DivFOp>(y, x);
-  auto atan = builder.create<math::AtanOp>(div);
+  auto div = arith::DivFOp::create(builder, y, x);
+  auto atan = math::AtanOp::create(builder, div);
 
   // Determine what the atan would be for a 180 degree rotation.
   auto zero = broadcast(builder, f32Cst(builder, 0.0f), shape);
   auto pi = broadcast(builder, f32Cst(builder, 3.14159265359f), shape);
-  auto addPi = builder.create<arith::AddFOp>(atan, pi);
-  auto subPi = builder.create<arith::SubFOp>(atan, pi);
+  auto addPi = arith::AddFOp::create(builder, atan, pi);
+  auto subPi = arith::SubFOp::create(builder, atan, pi);
   auto atanGt =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, atan, zero);
-  auto flippedAtan = builder.create<arith::SelectOp>(atanGt, subPi, addPi);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, atan, zero);
+  auto flippedAtan = arith::SelectOp::create(builder, atanGt, subPi, addPi);
 
   // Determine whether to directly use atan or use the 180 degree flip
-  auto xGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, x, zero);
-  Value result = builder.create<arith::SelectOp>(xGt, atan, flippedAtan);
+  auto xGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, x, zero);
+  Value result = arith::SelectOp::create(builder, xGt, atan, flippedAtan);
 
   // Handle x = 0, y > 0
   Value xZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, x, zero);
-  Value yGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, y, zero);
-  Value isHalfPi = builder.create<arith::AndIOp>(xZero, yGt);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, x, zero);
+  Value yGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, y, zero);
+  Value isHalfPi = arith::AndIOp::create(builder, xZero, yGt);
   auto halfPi = broadcast(builder, f32Cst(builder, 1.57079632679f), shape);
-  result = builder.create<arith::SelectOp>(isHalfPi, halfPi, result);
+  result = arith::SelectOp::create(builder, isHalfPi, halfPi, result);
 
   // Handle x = 0, y < 0
-  Value yLt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, y, zero);
-  Value isNegativeHalfPiPi = builder.create<arith::AndIOp>(xZero, yLt);
+  Value yLt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, y, zero);
+  Value isNegativeHalfPiPi = arith::AndIOp::create(builder, xZero, yLt);
   auto negativeHalfPiPi =
       broadcast(builder, f32Cst(builder, -1.57079632679f), shape);
-  result = builder.create<arith::SelectOp>(isNegativeHalfPiPi, negativeHalfPiPi,
+  result = arith::SelectOp::create(builder, isNegativeHalfPiPi, negativeHalfPiPi,
                                            result);
 
   // Handle x = 0, y = 0;
   Value yZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, y, zero);
-  Value isNan = builder.create<arith::AndIOp>(xZero, yZero);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, y, zero);
+  Value isNan = arith::AndIOp::create(builder, xZero, yZero);
   Value cstNan = broadcast(builder, f32FromBits(builder, 0x7fc00000), shape);
-  result = builder.create<arith::SelectOp>(isNan, cstNan, result);
+  result = arith::SelectOp::create(builder, isNan, cstNan, result);
 
   rewriter.replaceOp(op, result);
   return success();
@@ -569,8 +569,8 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
 
   // Mask for tiny values that are approximated with `operand`.
   Value tiny = bcast(f32Cst(builder, 0.0004f));
-  Value tinyMask = builder.create<arith::CmpFOp>(
-      arith::CmpFPredicate::OLT, builder.create<math::AbsFOp>(op.getOperand()),
+  Value tinyMask = arith::CmpFOp::create(builder,
+      arith::CmpFPredicate::OLT, math::AbsFOp::create(builder, op.getOperand()),
       tiny);
 
   // The monomial coefficients of the numerator polynomial (odd).
@@ -589,25 +589,25 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
   Value beta6 = bcast(f32Cst(builder, 1.19825839466702e-06f));
 
   // Since the polynomials are odd/even, we need x^2.
-  Value x2 = builder.create<arith::MulFOp>(x, x);
+  Value x2 = arith::MulFOp::create(builder, x, x);
 
   // Evaluate the numerator polynomial p.
-  Value p = builder.create<math::FmaOp>(x2, alpha13, alpha11);
-  p = builder.create<math::FmaOp>(x2, p, alpha9);
-  p = builder.create<math::FmaOp>(x2, p, alpha7);
-  p = builder.create<math::FmaOp>(x2, p, alpha5);
-  p = builder.create<math::FmaOp>(x2, p, alpha3);
-  p = builder.create<math::FmaOp>(x2, p, alpha1);
-  p = builder.create<arith::MulFOp>(x, p);
+  Value p = math::FmaOp::create(builder, x2, alpha13, alpha11);
+  p = math::FmaOp::create(builder, x2, p, alpha9);
+  p = math::FmaOp::create(builder, x2, p, alpha7);
+  p = math::FmaOp::create(builder, x2, p, alpha5);
+  p = math::FmaOp::create(builder, x2, p, alpha3);
+  p = math::FmaOp::create(builder, x2, p, alpha1);
+  p = arith::MulFOp::create(builder, x, p);
 
   // Evaluate the denominator polynomial q.
-  Value q = builder.create<math::FmaOp>(x2, beta6, beta4);
-  q = builder.create<math::FmaOp>(x2, q, beta2);
-  q = builder.create<math::FmaOp>(x2, q, beta0);
+  Value q = math::FmaOp::create(builder, x2, beta6, beta4);
+  q = math::FmaOp::create(builder, x2, q, beta2);
+  q = math::FmaOp::create(builder, x2, q, beta0);
 
   // Divide the numerator by the denominator.
-  Value res = builder.create<arith::SelectOp>(
-      tinyMask, x, builder.create<arith::DivFOp>(p, q));
+  Value res = arith::SelectOp::create(builder,
+      tinyMask, x, arith::DivFOp::create(builder, p, q));
 
   rewriter.replaceOp(op, res);
 
@@ -690,57 +690,57 @@ LogApproximationBase<Op>::logMatchAndRewrite(Op op, PatternRewriter &rewriter,
   //     e -= 1;
   //     x = x + x - 1.0;
   //   } else { x = x - 1.0; }
-  Value mask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x,
+  Value mask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x,
                                              cstCephesSQRTHF);
-  Value tmp = builder.create<arith::SelectOp>(mask, x, cstZero);
+  Value tmp = arith::SelectOp::create(builder, mask, x, cstZero);
 
-  x = builder.create<arith::SubFOp>(x, cstOne);
-  e = builder.create<arith::SubFOp>(
-      e, builder.create<arith::SelectOp>(mask, cstOne, cstZero));
-  x = builder.create<arith::AddFOp>(x, tmp);
+  x = arith::SubFOp::create(builder, x, cstOne);
+  e = arith::SubFOp::create(builder,
+      e, arith::SelectOp::create(builder, mask, cstOne, cstZero));
+  x = arith::AddFOp::create(builder, x, tmp);
 
-  Value x2 = builder.create<arith::MulFOp>(x, x);
-  Value x3 = builder.create<arith::MulFOp>(x2, x);
+  Value x2 = arith::MulFOp::create(builder, x, x);
+  Value x3 = arith::MulFOp::create(builder, x2, x);
 
   // Evaluate the polynomial approximant of degree 8 in three parts.
   Value y0, y1, y2;
-  y0 = builder.create<math::FmaOp>(cstCephesLogP0, x, cstCephesLogP1);
-  y1 = builder.create<math::FmaOp>(cstCephesLogP3, x, cstCephesLogP4);
-  y2 = builder.create<math::FmaOp>(cstCephesLogP6, x, cstCephesLogP7);
-  y0 = builder.create<math::FmaOp>(y0, x, cstCephesLogP2);
-  y1 = builder.create<math::FmaOp>(y1, x, cstCephesLogP5);
-  y2 = builder.create<math::FmaOp>(y2, x, cstCephesLogP8);
-  y0 = builder.create<math::FmaOp>(y0, x3, y1);
-  y0 = builder.create<math::FmaOp>(y0, x3, y2);
-  y0 = builder.create<arith::MulFOp>(y0, x3);
-
-  y0 = builder.create<math::FmaOp>(cstNegHalf, x2, y0);
-  x = builder.create<arith::AddFOp>(x, y0);
+  y0 = math::FmaOp::create(builder, cstCephesLogP0, x, cstCephesLogP1);
+  y1 = math::FmaOp::create(builder, cstCephesLogP3, x, cstCephesLogP4);
+  y2 = math::FmaOp::create(builder, cstCephesLogP6, x, cstCephesLogP7);
+  y0 = math::FmaOp::create(builder, y0, x, cstCephesLogP2);
+  y1 = math::FmaOp::create(builder, y1, x, cstCephesLogP5);
+  y2 = math::FmaOp::create(builder, y2, x, cstCephesLogP8);
+  y0 = math::FmaOp::create(builder, y0, x3, y1);
+  y0 = math::FmaOp::create(builder, y0, x3, y2);
+  y0 = arith::MulFOp::create(builder, y0, x3);
+
+  y0 = math::FmaOp::create(builder, cstNegHalf, x2, y0);
+  x = arith::AddFOp::create(builder, x, y0);
 
   if (base2) {
     Value cstLog2e = bcast(f32Cst(builder, static_cast<float>(LOG2E_VALUE)));
-    x = builder.create<math::FmaOp>(x, cstLog2e, e);
+    x = math::FmaOp::create(builder, x, cstLog2e, e);
   } else {
     Value cstLn2 = bcast(f32Cst(builder, static_cast<float>(LN2_VALUE)));
-    x = builder.create<math::FmaOp>(e, cstLn2, x);
+    x = math::FmaOp::create(builder, e, cstLn2, x);
   }
 
-  Value invalidMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT,
+  Value invalidMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT,
                                                     op.getOperand(), cstZero);
-  Value zeroMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value zeroMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                  op.getOperand(), cstZero);
-  Value posInfMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value posInfMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                    op.getOperand(), cstPosInf);
 
   // Filter out invalid values:
   //  • x == 0     -> -INF
   //  • x < 0      ->  NAN
   //  • x == +INF  -> +INF
-  Value aproximation = builder.create<arith::SelectOp>(
+  Value aproximation = arith::SelectOp::create(builder,
       zeroMask, cstMinusInf,
-      builder.create<arith::SelectOp>(
+      arith::SelectOp::create(builder,
           invalidMask, cstNan,
-          builder.create<arith::SelectOp>(posInfMask, cstPosInf, x)));
+          arith::SelectOp::create(builder, posInfMask, cstPosInf, x)));
 
   rewriter.replaceOp(op, aproximation);
 
@@ -805,17 +805,17 @@ Log1pApproximation::matchAndRewrite(math::Log1pOp op,
   //             "logLarge" below.
   Value cstOne = bcast(f32Cst(builder, 1.0f));
   Value x = op.getOperand();
-  Value u = builder.create<arith::AddFOp>(x, cstOne);
+  Value u = arith::AddFOp::create(builder, x, cstOne);
   Value uSmall =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, cstOne);
-  Value logU = builder.create<math::LogOp>(u);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, cstOne);
+  Value logU = math::LogOp::create(builder, u);
   Value uInf =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, logU);
-  Value logLarge = builder.create<arith::MulFOp>(
-      x, builder.create<arith::DivFOp>(
-             logU, builder.create<arith::SubFOp>(u, cstOne)));
-  Value approximation = builder.create<arith::SelectOp>(
-      builder.create<arith::OrIOp>(uSmall, uInf), x, logLarge);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, logU);
+  Value logLarge = arith::MulFOp::create(builder,
+      x, arith::DivFOp::create(builder,
+             logU, arith::SubFOp::create(builder, u, cstOne)));
+  Value approximation = arith::SelectOp::create(builder,
+      arith::OrIOp::create(builder, uSmall, uInf), x, logLarge);
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -853,27 +853,27 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   };
 
   auto fma = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
   auto sub = [&](Value a, Value b) -> Value {
-    return builder.create<arith::SubFOp>(a, b);
+    return arith::SubFOp::create(builder, a, b);
   };
 
-  auto abs = [&](Value a) -> Value { return builder.create<math::AbsFOp>(a); };
+  auto abs = [&](Value a) -> Value { return math::AbsFOp::create(builder, a); };
 
-  auto sqrt = [&](Value a) -> Value { return builder.create<math::SqrtOp>(a); };
+  auto sqrt = [&](Value a) -> Value { return math::SqrtOp::create(builder, a); };
 
   auto scopy = [&](Value a, Value b) -> Value {
-    return builder.create<math::CopySignOp>(a, b);
+    return math::CopySignOp::create(builder, a, b);
   };
 
   auto sel = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<arith::SelectOp>(a, b, c);
+    return arith::SelectOp::create(builder, a, b, c);
   };
 
   Value abso = abs(operand);
@@ -881,7 +881,7 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   Value opp = sqrt(sub(bcast(floatCst(builder, 1.0, elementType)), aa));
 
   Value gt =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, aa,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, aa,
                                     bcast(floatCst(builder, 0.5, elementType)));
 
   Value x = sel(gt, opp, abso);
@@ -948,51 +948,51 @@ AcosPolynomialApproximation::matchAndRewrite(math::AcosOp op,
   };
 
   auto fma = [&](Value a, Value b, Value c) -> Value {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
-  Value negOperand = builder.create<arith::NegFOp>(operand);
+  Value negOperand = arith::NegFOp::create(builder, operand);
   Value zero = bcast(floatCst(builder, 0.0, elementType));
   Value half = bcast(floatCst(builder, 0.5, elementType));
   Value negOne = bcast(floatCst(builder, -1.0, elementType));
   Value selR =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand, zero);
-  Value r = builder.create<arith::SelectOp>(selR, negOperand, operand);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, operand, zero);
+  Value r = arith::SelectOp::create(builder, selR, negOperand, operand);
   Value chkConst = bcast(floatCst(builder, -0.5625, elementType));
   Value firstPred =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, r, chkConst);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, r, chkConst);
 
   Value trueVal =
       fma(bcast(floatCst(builder, 9.3282184640716537e-1, elementType)),
           bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
-          builder.create<math::AsinOp>(r));
+          math::AsinOp::create(builder, r));
 
-  Value falseVal = builder.create<math::SqrtOp>(fma(half, r, half));
-  falseVal = builder.create<math::AsinOp>(falseVal);
+  Value falseVal = math::SqrtOp::create(builder, fma(half, r, half));
+  falseVal = math::AsinOp::create(builder, falseVal);
   falseVal = mul(bcast(floatCst(builder, 2.0, elementType)), falseVal);
 
-  r = builder.create<arith::SelectOp>(firstPred, trueVal, falseVal);
+  r = arith::SelectOp::create(builder, firstPred, trueVal, falseVal);
 
   // Check whether the operand lies in between [-1.0, 0.0).
   Value greaterThanNegOne =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, operand, negOne);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGE, operand, negOne);
 
   Value lessThanZero =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
 
   Value betweenNegOneZero =
-      builder.create<arith::AndIOp>(greaterThanNegOne, lessThanZero);
+      arith::AndIOp::create(builder, greaterThanNegOne, lessThanZero);
 
   trueVal = fma(bcast(floatCst(builder, 1.8656436928143307e+0, elementType)),
                 bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
-                builder.create<arith::NegFOp>(r));
+                arith::NegFOp::create(builder, r));
 
   Value finalVal =
-      builder.create<arith::SelectOp>(betweenNegOneZero, trueVal, r);
+      arith::SelectOp::create(builder, betweenNegOneZero, trueVal, r);
 
   rewriter.replaceOp(op, finalVal);
   return success();
@@ -1075,9 +1075,9 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
   bounds[2] = bcast(floatCst(builder, 3.75f, elementType));
 
   Value isNegativeArg =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
-  Value negArg = builder.create<arith::NegFOp>(operand);
-  Value x = builder.create<arith::SelectOp>(isNegativeArg, negArg, operand);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
+  Value negArg = arith::NegFOp::create(builder, operand);
+  Value x = arith::SelectOp::create(builder, isNegativeArg, negArg, operand);
 
   Value offset = offsets[0];
   Value p[polyDegree + 1];
@@ -1091,30 +1091,30 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
   Value isLessThanBound[intervalsCount];
   for (int j = 0; j < intervalsCount - 1; ++j) {
     isLessThanBound[j] =
-        builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, bounds[j]);
+        arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, bounds[j]);
     for (int i = 0; i <= polyDegree; ++i) {
-      p[i] = builder.create<arith::SelectOp>(isLessThanBound[j], p[i],
+      p[i] = arith::SelectOp::create(builder, isLessThanBound[j], p[i],
                                              pp[j + 1][i]);
-      q[i] = builder.create<arith::SelectOp>(isLessThanBound[j], q[i],
+      q[i] = arith::SelectOp::create(builder, isLessThanBound[j], q[i],
                                              qq[j + 1][i]);
     }
-    offset = builder.create<arith::SelectOp>(isLessThanBound[j], offset,
+    offset = arith::SelectOp::create(builder, isLessThanBound[j], offset,
                                              offsets[j + 1]);
   }
-  isLessThanBound[intervalsCount - 1] = builder.create<arith::CmpFOp>(
+  isLessThanBound[intervalsCount - 1] = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::ULT, x, bounds[intervalsCount - 1]);
 
   Value pPoly = makePolynomialCalculation(builder, p, x);
   Value qPoly = makePolynomialCalculation(builder, q, x);
-  Value rationalPoly = builder.create<arith::DivFOp>(pPoly, qPoly);
-  Value formula = builder.create<arith::AddFOp>(offset, rationalPoly);
-  formula = builder.create<arith::SelectOp>(isLessThanBound[intervalsCount - 1],
+  Value rationalPoly = arith::DivFOp::create(builder, pPoly, qPoly);
+  Value formula = arith::AddFOp::create(builder, offset, rationalPoly);
+  formula = arith::SelectOp::create(builder, isLessThanBound[intervalsCount - 1],
                                             formula, one);
 
   // erf is odd function: erf(x) = -erf(-x).
-  Value negFormula = builder.create<arith::NegFOp>(formula);
+  Value negFormula = arith::NegFOp::create(builder, formula);
   Value res =
-      builder.create<arith::SelectOp>(isNegativeArg, negFormula, formula);
+      arith::SelectOp::create(builder, isNegativeArg, negFormula, formula);
 
   rewriter.replaceOp(op, res);
 
@@ -1155,65 +1155,65 @@ ErfcPolynomialApproximation::matchAndRewrite(math::ErfcOp op,
   Value posInf = bcast(floatCst(builder, INFINITY, et));
   Value clampVal = bcast(floatCst(builder, 10.0546875f, et));
 
-  Value a = builder.create<math::AbsFOp>(x);
-  Value p = builder.create<arith::AddFOp>(a, pos2);
-  Value r = builder.create<arith::DivFOp>(one, p);
-  Value q = builder.create<math::FmaOp>(neg4, r, one);
-  Value t = builder.create<math::FmaOp>(builder.create<arith::AddFOp>(q, one),
+  Value a = math::AbsFOp::create(builder, x);
+  Value p = arith::AddFOp::create(builder, a, pos2);
+  Value r = arith::DivFOp::create(builder, one, p);
+  Value q = math::FmaOp::create(builder, neg4, r, one);
+  Value t = math::FmaOp::create(builder, arith::AddFOp::create(builder, q, one),
                                         neg2, a);
-  Value e = builder.create<math::FmaOp>(builder.create<arith::NegFOp>(a), q, t);
-  q = builder.create<math::FmaOp>(r, e, q);
+  Value e = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), q, t);
+  q = math::FmaOp::create(builder, r, e, q);
 
   p = bcast(floatCst(builder, -0x1.a4a000p-12f, et));        // -4.01139259e-4
   Value c1 = bcast(floatCst(builder, -0x1.42a260p-10f, et)); // -1.23075210e-3
-  p = builder.create<math::FmaOp>(p, q, c1);
+  p = math::FmaOp::create(builder, p, q, c1);
   Value c2 = bcast(floatCst(builder, 0x1.585714p-10f, et)); //  1.31355342e-3
-  p = builder.create<math::FmaOp>(p, q, c2);
+  p = math::FmaOp::create(builder, p, q, c2);
   Value c3 = bcast(floatCst(builder, 0x1.1adcc4p-07f, et)); // 8.63227434e-3
-  p = builder.create<math::FmaOp>(p, q, c3);
+  p = math::FmaOp::create(builder, p, q, c3);
   Value c4 = bcast(floatCst(builder, -0x1.081b82p-07f, et)); // -8.05991981e-3
-  p = builder.create<math::FmaOp>(p, q, c4);
+  p = math::FmaOp::create(builder, p, q, c4);
   Value c5 = bcast(floatCst(builder, -0x1.bc0b6ap-05f, et)); // -5.42046614e-2
-  p = builder.create<math::FmaOp>(p, q, c5);
+  p = math::FmaOp::create(builder, p, q, c5);
   Value c6 = bcast(floatCst(builder, 0x1.4ffc46p-03f, et)); //  1.64055392e-1
-  p = builder.create<math::FmaOp>(p, q, c6);
+  p = math::FmaOp::create(builder, p, q, c6);
   Value c7 = bcast(floatCst(builder, -0x1.540840p-03f, et)); // -1.66031361e-1
-  p = builder.create<math::FmaOp>(p, q, c7);
+  p = math::FmaOp::create(builder, p, q, c7);
   Value c8 = bcast(floatCst(builder, -0x1.7bf616p-04f, et)); // -9.27639827e-2
-  p = builder.create<math::FmaOp>(p, q, c8);
+  p = math::FmaOp::create(builder, p, q, c8);
   Value c9 = bcast(floatCst(builder, 0x1.1ba03ap-02f, et)); // 2.76978403e-1
-  p = builder.create<math::FmaOp>(p, q, c9);
-
-  Value d = builder.create<math::FmaOp>(pos2, a, one);
-  r = builder.create<arith::DivFOp>(one, d);
-  q = builder.create<math::FmaOp>(p, r, r);
-  Value negfa = builder.create<arith::NegFOp>(a);
-  Value fmaqah = builder.create<math::FmaOp>(q, negfa, onehalf);
-  Value psubq = builder.create<arith::SubFOp>(p, q);
-  e = builder.create<math::FmaOp>(fmaqah, pos2, psubq);
-  r = builder.create<math::FmaOp>(e, r, q);
-
-  Value s = builder.create<arith::MulFOp>(a, a);
-  e = builder.create<math::ExpOp>(builder.create<arith::NegFOp>(s));
-
-  t = builder.create<math::FmaOp>(builder.create<arith::NegFOp>(a), a, s);
-  r = builder.create<math::FmaOp>(
+  p = math::FmaOp::create(builder, p, q, c9);
+
+  Value d = math::FmaOp::create(builder, pos2, a, one);
+  r = arith::DivFOp::create(builder, one, d);
+  q = math::FmaOp::create(builder, p, r, r);
+  Value negfa = arith::NegFOp::create(builder, a);
+  Value fmaqah = math::FmaOp::create(builder, q, negfa, onehalf);
+  Value psubq = arith::SubFOp::create(builder, p, q);
+  e = math::FmaOp::create(builder, fmaqah, pos2, psubq);
+  r = math::FmaOp::create(builder, e, r, q);
+
+  Value s = arith::MulFOp::create(builder, a, a);
+  e = math::ExpOp::create(builder, arith::NegFOp::create(builder, s));
+
+  t = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), a, s);
+  r = math::FmaOp::create(builder,
       r, e,
-      builder.create<arith::MulFOp>(builder.create<arith::MulFOp>(r, e), t));
+      arith::MulFOp::create(builder, arith::MulFOp::create(builder, r, e), t));
 
-  Value isNotLessThanInf = builder.create<arith::XOrIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, a, posInf),
+  Value isNotLessThanInf = arith::XOrIOp::create(builder,
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, a, posInf),
       trueValue);
-  r = builder.create<arith::SelectOp>(isNotLessThanInf,
-                                      builder.create<arith::AddFOp>(x, x), r);
+  r = arith::SelectOp::create(builder, isNotLessThanInf,
+                                      arith::AddFOp::create(builder, x, x), r);
   Value isGreaterThanClamp =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, a, clampVal);
-  r = builder.create<arith::SelectOp>(isGreaterThanClamp, zero, r);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, a, clampVal);
+  r = arith::SelectOp::create(builder, isGreaterThanClamp, zero, r);
 
   Value isNegative =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, zero);
-  r = builder.create<arith::SelectOp>(
-      isNegative, builder.create<arith::SubFOp>(pos2, r), r);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, zero);
+  r = arith::SelectOp::create(builder,
+      isNegative, arith::SubFOp::create(builder, pos2, r), r);
 
   rewriter.replaceOp(op, r);
   return success();
@@ -1235,8 +1235,8 @@ Value clampWithNormals(ImplicitLocOpBuilder &builder,
   };
 
   auto selectCmp = [&builder](auto pred, Value value, Value bound) {
-    return builder.create<arith::SelectOp>(
-        builder.create<arith::CmpFOp>(pred, value, bound), value, bound);
+    return arith::SelectOp::create(builder,
+        arith::CmpFOp::create(builder, pred, value, bound), value, bound);
   };
 
   // Note: prefer UGE/ULE vs. UGT/ULT, since they generate vmaxps/vminps vs.
@@ -1268,17 +1268,17 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
   ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
 
   auto add = [&](Value a, Value b) -> Value {
-    return builder.create<arith::AddFOp>(a, b);
+    return arith::AddFOp::create(builder, a, b);
   };
   auto bcast = [&](Value value) -> Value {
     return broadcast(builder, value, shape);
   };
-  auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); };
+  auto floor = [&](Value a) { return math::FloorOp::create(builder, a); };
   auto fmla = [&](Value a, Value b, Value c) {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
 
   // Polynomial approximation from Cephes.
@@ -1382,7 +1382,7 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
 
   // Convert n' to an i32.  This is safe because we clamped it above.
   auto i32Vec = broadcast(builder.getI32Type(), shape);
-  Value nI32 = builder.create<arith::FPToSIOp>(i32Vec, n);
+  Value nI32 = arith::FPToSIOp::create(builder, i32Vec, n);
 
   // Creates the value 2^n' if -126 <= n' <= 127 and 0 if n' = -127.
   Value pow2 = exp2I32(builder, nI32);
@@ -1430,26 +1430,26 @@ ExpM1Approximation::matchAndRewrite(math::ExpM1Op op,
   Value cstOne = bcast(f32Cst(builder, 1.0f));
   Value cstNegOne = bcast(f32Cst(builder, -1.0f));
   Value x = op.getOperand();
-  Value u = builder.create<math::ExpOp>(x);
+  Value u = math::ExpOp::create(builder, x);
   Value uEqOneOrNaN =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::UEQ, u, cstOne);
-  Value uMinusOne = builder.create<arith::SubFOp>(u, cstOne);
-  Value uMinusOneEqNegOne = builder.create<arith::CmpFOp>(
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::UEQ, u, cstOne);
+  Value uMinusOne = arith::SubFOp::create(builder, u, cstOne);
+  Value uMinusOneEqNegOne = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::OEQ, uMinusOne, cstNegOne);
   // logU = log(u) ~= x
-  Value logU = builder.create<math::LogOp>(u);
+  Value logU = math::LogOp::create(builder, u);
 
   // Detect exp(x) = +inf; written this way to avoid having to form +inf.
   Value isInf =
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, logU, u);
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, logU, u);
 
   // (u - 1) * (x / ~x)
-  Value expm1 = builder.create<arith::MulFOp>(
-      uMinusOne, builder.create<arith::DivFOp>(x, logU));
-  expm1 = builder.create<arith::SelectOp>(isInf, u, expm1);
-  Value approximation = builder.create<arith::SelectOp>(
+  Value expm1 = arith::MulFOp::create(builder,
+      uMinusOne, arith::DivFOp::create(builder, x, logU));
+  expm1 = arith::SelectOp::create(builder, isInf, u, expm1);
+  Value approximation = arith::SelectOp::create(builder,
       uEqOneOrNaN, x,
-      builder.create<arith::SelectOp>(uMinusOneEqNegOne, cstNegOne, expm1));
+      arith::SelectOp::create(builder, uMinusOneEqNegOne, cstNegOne, expm1));
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -1494,40 +1494,40 @@ LogicalResult SinAndCosApproximation<isSine, OpTy>::matchAndRewrite(
     return broadcast(builder, value, shape);
   };
   auto mul = [&](Value a, Value b) -> Value {
-    return builder.create<arith::MulFOp>(a, b);
+    return arith::MulFOp::create(builder, a, b);
   };
   auto sub = [&](Value a, Value b) -> Value {
-    return builder.create<arith::SubFOp>(a, b);
+    return arith::SubFOp::create(builder, a, b);
   };
-  auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); };
+  auto floor = [&](Value a) { return math::FloorOp::create(builder, a); };
 
   auto i32Vec = broadcast(builder.getI32Type(), shape);
   auto fPToSingedInteger = [&](Value a) -> Value {
-    return builder.create<arith::FPToSIOp>(i32Vec, a);
+    return arith::FPToSIOp::create(builder, i32Vec, a);
   };
 
   auto modulo4 = [&](Value a) -> Value {
-    return builder.create<arith::AndIOp>(a, bcast(i32Cst(builder, 3)));
+    return arith::AndIOp::create(builder, a, bcast(i32Cst(builder, 3)));
   };
 
   auto isEqualTo = [&](Value a, Value b) -> Value {
-    return builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, a, b);
+    return arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, a, b);
   };
 
   auto isGreaterThan = [&](Value a, Value b) -> Value {
-    return builder.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, a, b);
+    return arith::CmpIOp::create(builder, arith::CmpIPredicate::sgt, a, b);
   };
 
   auto select = [&](Value cond, Value t, Value f) -> Value {
-    return builder.create<arith::SelectOp>(cond, t, f);
+    return arith::SelectOp::create(builder, cond, t, f);
   };
 
   auto fmla = [&](Value a, Value b, Value c) {
-    return builder.create<math::FmaOp>(a, b, c);
+    return math::FmaOp::create(builder, a, b, c);
   };
 
   auto bitwiseOr = [&](Value a, Value b) {
-    return builder.create<arith::OrIOp>(a, b);
+    return arith::OrIOp::create(builder, a, b);
   };
 
   Value twoOverPi = bcast(f32Cst(builder, (float)TWO_OVER_PI));
@@ -1624,7 +1624,7 @@ CbrtApproximation::matchAndRewrite(math::CbrtOp op,
   intTy = broadcast(intTy, shape);
 
   auto bconst = [&](TypedAttr attr) -> Value {
-    Value value = b.create<arith::ConstantOp>(attr);
+    Value value = arith::ConstantOp::create(b, attr);
     return broadcast(b, value, shape);
   };
 
@@ -1641,44 +1641,44 @@ CbrtApproximation::matchAndRewrite(math::CbrtOp op,
   // union {int ix; float x;};
   // x = x0;
   // ix = ix/4 + ix/16;
-  Value absValue = b.create<math::AbsFOp>(operand);
-  Value intValue = b.create<arith::BitcastOp>(intTy, absValue);
-  Value divideBy4 = b.create<arith::ShRSIOp>(intValue, intTwo);
-  Value divideBy16 = b.create<arith::ShRSIOp>(intValue, intFour);
-  intValue = b.create<arith::AddIOp>(divideBy4, divideBy16);
+  Value absValue = math::AbsFOp::create(b, operand);
+  Value intValue = arith::BitcastOp::create(b, intTy, absValue);
+  Value divideBy4 = arith::ShRSIOp::create(b, intValue, intTwo);
+  Value divideBy16 = arith::ShRSIOp::create(b, intValue, intFour);
+  intValue = arith::AddIOp::create(b, divideBy4, divideBy16);
 
   // ix = ix + ix/16;
-  divideBy16 = b.create<arith::ShRSIOp>(intValue, intFour);
-  intValue = b.create<arith::AddIOp>(intValue, divideBy16);
+  divideBy16 = arith::ShRSIOp::create(b, intValue, intFour);
+  intValue = arith::AddIOp::create(b, intValue, divideBy16);
 
   // ix = ix + ix/256;
-  Value divideBy256 = b.create<arith::ShRSIOp>(intValue, intEight);
-  intValue = b.create<arith::AddIOp>(intValue, divideBy256);
+  Value divideBy256 = arith::ShRSIOp::create(b, intValue, intEight);
+  intValue = arith::AddIOp::create(b, intValue, divideBy256);
 
   // ix = 0x2a5137a0 + ix;
-  intValue = b.create<arith::AddIOp>(intValue, intMagic);
+  intValue = arith::AddIOp::create(b, intValue, intMagic);
 
   // Perform one newtons step:
   // x = 0.33333333f*(2.0f*x + x0/(x*x));
-  Value floatValue = b.create<arith::BitcastOp>(floatTy, intValue);
-  Value squared = b.create<arith::MulFOp>(floatValue, floatValue);
-  Value mulTwo = b.create<arith::MulFOp>(floatValue, fpTwo);
-  Value divSquared = b.create<arith::DivFOp>(absValue, squared);
-  floatValue = b.create<arith::AddFOp>(mulTwo, divSquared);
-  floatValue = b.create<arith::MulFOp>(floatValue, fpThird);
+  Value floatValue = arith::BitcastOp::create(b, floatTy, intValue);
+  Value squared = arith::MulFOp::create(b, floatValue, floatValue);
+  Value mulTwo = arith::MulFOp::create(b, floatValue, fpTwo);
+  Value divSquared = arith::DivFOp::create(b, absValue, squared);
+  floatValue = arith::AddFOp::create(b, mulTwo, divSquared);
+  floatValue = arith::MulFOp::create(b, floatValue, fpThird);
 
   // x = 0.33333333f*(2.0f*x + x0/(x*x));
-  squared = b.create<arith::MulFOp>(floatValue, floatValue);
-  mulTwo = b.create<arith::MulFOp>(floatValue, fpTwo);
-  divSquared = b.create<arith::DivFOp>(absValue, squared);
-  floatValue = b.create<arith::AddFOp>(mulTwo, divSquared);
-  floatValue = b.create<arith::MulFOp>(floatValue, fpThird);
+  squared = arith::MulFOp::create(b, floatValue, floatValue);
+  mulTwo = arith::MulFOp::create(b, floatValue, fpTwo);
+  divSquared = arith::DivFOp::create(b, absValue, squared);
+  floatValue = arith::AddFOp::create(b, mulTwo, divSquared);
+  floatValue = arith::MulFOp::create(b, floatValue, fpThird);
 
   // Check for zero and restore sign.
   Value isZero =
-      b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absValue, fpZero);
-  floatValue = b.create<arith::SelectOp>(isZero, fpZero, floatValue);
-  floatValue = b.create<math::CopySignOp>(floatValue, operand);
+      arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absValue, fpZero);
+  floatValue = arith::SelectOp::create(b, isZero, fpZero, floatValue);
+  floatValue = math::CopySignOp::create(b, floatValue, operand);
 
   rewriter.replaceOp(op, floatValue);
   return success();
@@ -1719,29 +1719,29 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
   Value cstNegHalf = bcast(f32Cst(builder, -0.5f));
   Value cstMinNormPos = bcast(f32FromBits(builder, 0x00800000u));
 
-  Value negHalf = builder.create<arith::MulFOp>(op.getOperand(), cstNegHalf);
+  Value negHalf = arith::MulFOp::create(builder, op.getOperand(), cstNegHalf);
 
   // Select only the inverse sqrt of positive normals (denormals are
   // flushed to zero).
-  Value ltMinMask = builder.create<arith::CmpFOp>(
+  Value ltMinMask = arith::CmpFOp::create(builder,
       arith::CmpFPredicate::OLT, op.getOperand(), cstMinNormPos);
-  Value infMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+  Value infMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
                                                 op.getOperand(), cstPosInf);
-  Value notNormalFiniteMask = builder.create<arith::OrIOp>(ltMinMask, infMask);
+  Value notNormalFiniteMask = arith::OrIOp::create(builder, ltMinMask, infMask);
 
   // Compute an approximate result.
   Value yApprox = handleMultidimensionalVectors(
       builder, op->getOperands(), 8, [&builder](ValueRange operands) -> Value {
-        return builder.create<x86vector::RsqrtOp>(operands);
+        return x86vector::RsqrtOp::create(builder, operands);
       });
 
   // Do a single step of Newton-Raphson iteration to improve the approximation.
   // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n).
   // It is essential to evaluate the inner term like this because forming
   // y_n^2 may over- or underflow.
-  Value inner = builder.create<arith::MulFOp>(negHalf, yApprox);
-  Value fma = builder.create<math::FmaOp>(yApprox, inner, cstOnePointFive);
-  Value yNewton = builder.create<arith::MulFOp>(yApprox, fma);
+  Value inner = arith::MulFOp::create(builder, negHalf, yApprox);
+  Value fma = math::FmaOp::create(builder, yApprox, inner, cstOnePointFive);
+  Value yNewton = arith::MulFOp::create(builder, yApprox, fma);
 
   // Select the result of the Newton-Raphson step for positive normal arguments.
   // For other arguments, choose the output of the intrinsic. This will
@@ -1749,7 +1749,7 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
   // x is zero or a positive denormalized float (equivalent to flushing positive
   // denormalized inputs to zero).
   Value res =
-      builder.create<arith::SelectOp>(notNormalFiniteMask, yApprox, yNewton);
+      arith::SelectOp::create(builder, notNormalFiniteMask, yApprox, yNewton);
   rewriter.replaceOp(op, res);
 
   return success();
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index c5643f6e2f830..46298fb189e41 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -85,10 +85,10 @@ Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
   // TODO: support more types.
   return TypeSwitch<Type, Value>(slot.elemType)
       .Case([&](MemRefType t) {
-        return builder.create<memref::AllocaOp>(getLoc(), t);
+        return memref::AllocaOp::create(builder, getLoc(), t);
       })
       .Default([&](Type t) {
-        return builder.create<arith::ConstantOp>(getLoc(), t,
+        return arith::ConstantOp::create(builder, getLoc(), t,
                                                  builder.getZeroAttr(t));
       });
 }
@@ -135,7 +135,7 @@ DenseMap<Attribute, MemorySlot> memref::AllocaOp::destructure(
   for (Attribute usedIndex : usedIndices) {
     Type elemType = memrefType.getTypeAtIndex(usedIndex);
     MemRefType elemPtr = MemRefType::get({}, elemType);
-    auto subAlloca = builder.create<memref::AllocaOp>(getLoc(), elemPtr);
+    auto subAlloca = memref::AllocaOp::create(builder, getLoc(), elemPtr);
     newAllocators.push_back(subAlloca);
     slotMap.try_emplace<MemorySlot>(usedIndex,
                                     {subAlloca.getResult(), elemType});
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index d1a9920aa66c5..706ac4b4867f4 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -213,7 +213,7 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
 
     // Create and insert the alloc op for the new memref.
-    auto newAlloc = rewriter.create<AllocLikeOp>(
+    auto newAlloc = AllocLikeOp::create(rewriter,
         alloc.getLoc(), newMemRefType, dynamicSizes, alloc.getSymbolOperands(),
         alloc.getAlignmentAttr());
     // Insert a cast so we have the same type as the old alloc.
@@ -836,7 +836,7 @@ void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
 void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
                   int64_t index) {
   auto loc = result.location;
-  Value indexValue = builder.create<arith::ConstantIndexOp>(loc, index);
+  Value indexValue = arith::ConstantIndexOp::create(builder, loc, index);
   build(builder, result, source, indexValue);
 }
 
@@ -1083,9 +1083,9 @@ struct DimOfMemRefReshape : public OpRewritePattern<DimOp> {
     rewriter.setInsertionPointAfter(reshape);
     Location loc = dim.getLoc();
     Value load =
-        rewriter.create<LoadOp>(loc, reshape.getShape(), dim.getIndex());
+        LoadOp::create(rewriter, loc, reshape.getShape(), dim.getIndex());
     if (load.getType() != dim.getType())
-      load = rewriter.create<arith::IndexCastOp>(loc, dim.getType(), load);
+      load = arith::IndexCastOp::create(rewriter, loc, dim.getType(), load);
     rewriter.replaceOp(dim, load);
     return success();
   }
@@ -1358,7 +1358,7 @@ static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
     assert(isa<Attribute>(maybeConstant) &&
            "The constified value should be either unchanged (i.e., == result) "
            "or a constant");
-    Value constantVal = rewriter.create<arith::ConstantIndexOp>(
+    Value constantVal = arith::ConstantIndexOp::create(rewriter,
         loc, llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
     for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
       // modifyOpInPlace: lambda cannot capture structured bindings in C++17
@@ -2587,7 +2587,7 @@ struct CollapseShapeOpMemRefCastFolder
       rewriter.modifyOpInPlace(
           op, [&]() { op.getSrcMutable().assign(cast.getSource()); });
     } else {
-      Value newOp = rewriter.create<CollapseShapeOp>(
+      Value newOp = CollapseShapeOp::create(rewriter,
           op->getLoc(), cast.getSource(), op.getReassociationIndices());
       rewriter.replaceOpWithNewOp<CastOp>(op, op.getType(), newOp);
     }
@@ -3045,15 +3045,15 @@ SmallVector<Range, 8> mlir::getOrCreateRanges(OffsetSizeAndStrideOpInterface op,
     Value offset =
         op.isDynamicOffset(idx)
             ? op.getDynamicOffset(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticOffset(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticOffset(idx));
     Value size =
         op.isDynamicSize(idx)
             ? op.getDynamicSize(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticSize(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticSize(idx));
     Value stride =
         op.isDynamicStride(idx)
             ? op.getDynamicStride(idx)
-            : b.create<arith::ConstantIndexOp>(loc, op.getStaticStride(idx));
+            : arith::ConstantIndexOp::create(b, loc, op.getStaticStride(idx));
     res.emplace_back(Range{offset, size, stride});
   }
   return res;
@@ -3212,7 +3212,7 @@ class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
     if (!resultType)
       return failure();
 
-    Value newSubView = rewriter.create<SubViewOp>(
+    Value newSubView = SubViewOp::create(rewriter,
         subViewOp.getLoc(), resultType, castOp.getSource(),
         subViewOp.getOffsets(), subViewOp.getSizes(), subViewOp.getStrides(),
         subViewOp.getStaticOffsets(), subViewOp.getStaticSizes(),
@@ -3534,7 +3534,7 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
       return failure();
 
     // Create new ViewOp.
-    auto newViewOp = rewriter.create<ViewOp>(
+    auto newViewOp = ViewOp::create(rewriter,
         viewOp.getLoc(), newMemRefType, viewOp.getOperand(0),
         viewOp.getByteShift(), newOperands);
     // Insert a cast so we have the same type as the old memref type.
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 0c03670b4535f..5c556308b1e9f 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -155,7 +155,7 @@ transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
       Type resultType = alloca.getResult().getType();
       OpBuilder builder(rewriter.getContext());
       // TODO: Add a better builder for this.
-      globalOp = builder.create<memref::GlobalOp>(
+      globalOp = memref::GlobalOp::create(builder,
           loc, StringAttr::get(ctx, "alloca"), StringAttr::get(ctx, "private"),
           TypeAttr::get(resultType), Attribute{}, UnitAttr{}, IntegerAttr{});
       symbolTable.insert(globalOp);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
index c433415944323..a2b689ed13cbf 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
@@ -22,11 +22,11 @@ struct DefaultAllocationInterface
           DefaultAllocationInterface, memref::AllocOp> {
   static std::optional<Operation *> buildDealloc(OpBuilder &builder,
                                                  Value alloc) {
-    return builder.create<memref::DeallocOp>(alloc.getLoc(), alloc)
+    return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
         .getOperation();
   }
   static std::optional<Value> buildClone(OpBuilder &builder, Value alloc) {
-    return builder.create<bufferization::CloneOp>(alloc.getLoc(), alloc)
+    return bufferization::CloneOp::create(builder, alloc.getLoc(), alloc)
         .getResult();
   }
   static ::mlir::HoistingKind getHoistingKind() {
@@ -35,7 +35,7 @@ struct DefaultAllocationInterface
   static ::std::optional<::mlir::Operation *>
   buildPromotedAlloc(OpBuilder &builder, Value alloc) {
     Operation *definingOp = alloc.getDefiningOp();
-    return builder.create<memref::AllocaOp>(
+    return memref::AllocaOp::create(builder,
         definingOp->getLoc(), cast<MemRefType>(definingOp->getResultTypes()[0]),
         definingOp->getOperands(), definingOp->getAttrs());
   }
@@ -52,7 +52,7 @@ struct DefaultReallocationInterface
           DefaultAllocationInterface, memref::ReallocOp> {
   static std::optional<Operation *> buildDealloc(OpBuilder &builder,
                                                  Value realloc) {
-    return builder.create<memref::DeallocOp>(realloc.getLoc(), realloc)
+    return memref::DeallocOp::create(builder, realloc.getLoc(), realloc)
         .getOperation();
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
index 7c777e807f08c..9d9923bfd0541 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
@@ -124,7 +124,7 @@ struct ComposeSubViewOpPattern : public OpRewritePattern<memref::SubViewOp> {
         }
 
         AffineMap map = AffineMap::get(0, affineApplyOperands.size(), expr);
-        Value result = rewriter.create<affine::AffineApplyOp>(
+        Value result = affine::AffineApplyOp::create(rewriter,
             op.getLoc(), map, affineApplyOperands);
         offsets.push_back(result);
       }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
index ec2bc95291455..c35036d7631c8 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
@@ -99,7 +99,7 @@ static Value getOffsetForBitwidth(Location loc, OpFoldResult srcIdx,
       affine::makeComposedFoldedAffineApply(builder, loc, offsetExpr, {srcIdx});
   Value bitOffset = getValueOrCreateConstantIndexOp(builder, loc, offsetVal);
   IntegerType dstType = builder.getIntegerType(targetBits);
-  return builder.create<arith::IndexCastOp>(loc, dstType, bitOffset);
+  return arith::IndexCastOp::create(builder, loc, dstType, bitOffset);
 }
 
 /// When writing a subbyte size, masked bitwise operations are used to only
@@ -112,14 +112,14 @@ static Value getSubByteWriteMask(Location loc, OpFoldResult linearizedIndices,
   auto dstIntegerType = builder.getIntegerType(dstBits);
   auto maskRightAlignedAttr =
       builder.getIntegerAttr(dstIntegerType, (1 << srcBits) - 1);
-  Value maskRightAligned = builder.create<arith::ConstantOp>(
+  Value maskRightAligned = arith::ConstantOp::create(builder,
       loc, dstIntegerType, maskRightAlignedAttr);
   Value writeMaskInverse =
-      builder.create<arith::ShLIOp>(loc, maskRightAligned, bitwidthOffset);
+      arith::ShLIOp::create(builder, loc, maskRightAligned, bitwidthOffset);
   auto flipValAttr = builder.getIntegerAttr(dstIntegerType, -1);
   Value flipVal =
-      builder.create<arith::ConstantOp>(loc, dstIntegerType, flipValAttr);
-  return builder.create<arith::XOrIOp>(loc, writeMaskInverse, flipVal);
+      arith::ConstantOp::create(builder, loc, dstIntegerType, flipValAttr);
+  return arith::XOrIOp::create(builder, loc, writeMaskInverse, flipVal);
 }
 
 /// Returns the scaled linearized index based on the `srcBits` and `dstBits`
@@ -141,7 +141,7 @@ getLinearizedSrcIndices(OpBuilder &builder, Location loc, int64_t srcBits,
                         const SmallVector<OpFoldResult> &indices,
                         Value memref) {
   auto stridedMetadata =
-      builder.create<memref::ExtractStridedMetadataOp>(loc, memref);
+      memref::ExtractStridedMetadataOp::create(builder, loc, memref);
   OpFoldResult linearizedIndices;
   std::tie(std::ignore, linearizedIndices) =
       memref::getLinearizedMemRefOffsetAndSize(
@@ -298,7 +298,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
     // Special case 0-rank memref loads.
     Value bitsLoad;
     if (convertedType.getRank() == 0) {
-      bitsLoad = rewriter.create<memref::LoadOp>(loc, adaptor.getMemref(),
+      bitsLoad = memref::LoadOp::create(rewriter, loc, adaptor.getMemref(),
                                                  ValueRange{});
     } else {
       // Linearize the indices of the original load instruction. Do not account
@@ -306,7 +306,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
       OpFoldResult linearizedIndices = getLinearizedSrcIndices(
           rewriter, loc, srcBits, adaptor.getIndices(), op.getMemRef());
 
-      Value newLoad = rewriter.create<memref::LoadOp>(
+      Value newLoad = memref::LoadOp::create(rewriter,
           loc, adaptor.getMemref(),
           getIndicesForLoadOrStore(rewriter, loc, linearizedIndices, srcBits,
                                    dstBits));
@@ -315,7 +315,7 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
       // Note, currently only the big-endian is supported.
       Value bitwidthOffset = getOffsetForBitwidth(loc, linearizedIndices,
                                                   srcBits, dstBits, rewriter);
-      bitsLoad = rewriter.create<arith::ShRSIOp>(loc, newLoad, bitwidthOffset);
+      bitsLoad = arith::ShRSIOp::create(rewriter, loc, newLoad, bitwidthOffset);
     }
 
     // Get the corresponding bits. If the arith computation bitwidth equals
@@ -331,13 +331,13 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
             : IntegerType::get(rewriter.getContext(),
                                resultTy.getIntOrFloatBitWidth());
     if (conversionTy == convertedElementType) {
-      auto mask = rewriter.create<arith::ConstantOp>(
+      auto mask = arith::ConstantOp::create(rewriter,
           loc, convertedElementType,
           rewriter.getIntegerAttr(convertedElementType, (1 << srcBits) - 1));
 
-      result = rewriter.create<arith::AndIOp>(loc, bitsLoad, mask);
+      result = arith::AndIOp::create(rewriter, loc, bitsLoad, mask);
     } else {
-      result = rewriter.create<arith::TruncIOp>(loc, conversionTy, bitsLoad);
+      result = arith::TruncIOp::create(rewriter, loc, conversionTy, bitsLoad);
     }
 
     if (conversionTy != resultTy) {
@@ -435,11 +435,12 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
           input);
     }
     Value extendedInput =
-        rewriter.create<arith::ExtUIOp>(loc, dstIntegerType, input);
+        arith::ExtUIOp::create(rewriter, loc, dstIntegerType,
+                                                          input);
 
     // Special case 0-rank memref stores. No need for masking.
     if (convertedType.getRank() == 0) {
-      rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::assign,
+      memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::assign,
                                            extendedInput, adaptor.getMemref(),
                                            ValueRange{});
       rewriter.eraseOp(op);
@@ -456,14 +457,14 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
                                           dstBits, bitwidthOffset, rewriter);
     // Align the value to write with the destination bits
     Value alignedVal =
-        rewriter.create<arith::ShLIOp>(loc, extendedInput, bitwidthOffset);
+        arith::ShLIOp::create(rewriter, loc, extendedInput, bitwidthOffset);
 
     // Clear destination bits
-    rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::andi,
+    memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::andi,
                                          writeMask, adaptor.getMemref(),
                                          storeIndices);
     // Write srcs bits to destination
-    rewriter.create<memref::AtomicRMWOp>(loc, arith::AtomicRMWKind::ori,
+    memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::ori,
                                          alignedVal, adaptor.getMemref(),
                                          storeIndices);
     rewriter.eraseOp(op);
@@ -525,7 +526,7 @@ struct ConvertMemRefSubview final : OpConversionPattern<memref::SubViewOp> {
     }
 
     // Transform the offsets, sizes and strides according to the emulation.
-    auto stridedMetadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+    auto stridedMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
         loc, subViewOp.getViewSource());
 
     OpFoldResult linearizedIndices;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
index e6e4c3b07ecb8..34f9afae9d2ad 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
@@ -48,15 +48,15 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
       Value size;
       // Load dynamic sizes from the shape input, use constants for static dims.
       if (op.getType().isDynamicDim(i)) {
-        Value index = rewriter.create<arith::ConstantIndexOp>(loc, i);
-        size = rewriter.create<memref::LoadOp>(loc, op.getShape(), index);
+        Value index = arith::ConstantIndexOp::create(rewriter, loc, i);
+        size = memref::LoadOp::create(rewriter, loc, op.getShape(), index);
         if (!isa<IndexType>(size.getType()))
-          size = rewriter.create<arith::IndexCastOp>(
+          size = arith::IndexCastOp::create(rewriter,
               loc, rewriter.getIndexType(), size);
         sizes[i] = size;
       } else {
         auto sizeAttr = rewriter.getIndexAttr(op.getType().getDimSize(i));
-        size = rewriter.create<arith::ConstantOp>(loc, sizeAttr);
+        size = arith::ConstantOp::create(rewriter, loc, sizeAttr);
         sizes[i] = sizeAttr;
       }
       if (stride)
@@ -66,10 +66,10 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
 
       if (i > 0) {
         if (stride) {
-          stride = rewriter.create<arith::MulIOp>(loc, stride, size);
+          stride = arith::MulIOp::create(rewriter, loc, stride, size);
         } else if (op.getType().isDynamicDim(i)) {
-          stride = rewriter.create<arith::MulIOp>(
-              loc, rewriter.create<arith::ConstantIndexOp>(loc, staticStride),
+          stride = arith::MulIOp::create(rewriter,
+              loc, arith::ConstantIndexOp::create(rewriter, loc, staticStride),
               size);
         } else {
           staticStride *= op.getType().getDimSize(i);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
index 7475d442b7b9a..b54fa61714323 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
@@ -73,7 +73,7 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     if (ShapedType::isDynamic(inputSize)) {
       Value dimZero = getValueOrCreateConstantIndexOp(rewriter, loc,
                                                       rewriter.getIndexAttr(0));
-      currSize = rewriter.create<memref::DimOp>(loc, op.getSource(), dimZero)
+      currSize = memref::DimOp::create(rewriter, loc, op.getSource(), dimZero)
                      .getResult();
     }
 
@@ -88,9 +88,9 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     // the old buffer is smaller than the requested size.
     Value lhs = getValueOrCreateConstantIndexOp(rewriter, loc, currSize);
     Value rhs = getValueOrCreateConstantIndexOp(rewriter, loc, targetSize);
-    Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
                                                 lhs, rhs);
-    auto ifOp = rewriter.create<scf::IfOp>(
+    auto ifOp = scf::IfOp::create(rewriter,
         loc, cond,
         [&](OpBuilder &builder, Location loc) {
           // Allocate the new buffer. If it is a dynamic memref we need to pass
@@ -100,25 +100,25 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           if (op.getDynamicResultSize())
             dynamicSizeOperands.push_back(op.getDynamicResultSize());
 
-          Value newAlloc = builder.create<memref::AllocOp>(
+          Value newAlloc = memref::AllocOp::create(builder,
               loc, op.getResult().getType(), dynamicSizeOperands,
               op.getAlignmentAttr());
 
           // Take a subview of the new (bigger) buffer such that we can copy the
           // old values over (the copy operation requires both operands to have
           // the same shape).
-          Value subview = builder.create<memref::SubViewOp>(
+          Value subview = memref::SubViewOp::create(builder,
               loc, newAlloc, ArrayRef<OpFoldResult>{rewriter.getIndexAttr(0)},
               ArrayRef<OpFoldResult>{currSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
-          builder.create<memref::CopyOp>(loc, op.getSource(), subview);
+          memref::CopyOp::create(builder, loc, op.getSource(), subview);
 
           // Insert the deallocation of the old buffer only if requested
           // (enabled by default).
           if (emitDeallocs)
-            builder.create<memref::DeallocOp>(loc, op.getSource());
+            memref::DeallocOp::create(builder, loc, op.getSource());
 
-          builder.create<scf::YieldOp>(loc, newAlloc);
+          scf::YieldOp::create(builder, loc, newAlloc);
         },
         [&](OpBuilder &builder, Location loc) {
           // We need to reinterpret-cast here because either the input or output
@@ -126,11 +126,11 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           // dynamic or vice-versa. If both are static and the original buffer
           // is already bigger than the requested size, the cast represents a
           // subview operation.
-          Value casted = builder.create<memref::ReinterpretCastOp>(
+          Value casted = memref::ReinterpretCastOp::create(builder,
               loc, cast<MemRefType>(op.getResult().getType()), op.getSource(),
               rewriter.getIndexAttr(0), ArrayRef<OpFoldResult>{targetSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
-          builder.create<scf::YieldOp>(loc, casted);
+          scf::YieldOp::create(builder, loc, casted);
         });
 
     rewriter.replaceOp(op, ifOp.getResult(0));
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index 2ba798f48ac7c..f581b6490fadb 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -66,7 +66,7 @@ resolveSubviewStridedMetadata(RewriterBase &rewriter,
   unsigned sourceRank = sourceType.getRank();
 
   auto newExtractStridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(origLoc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, origLoc, source);
 
   auto [sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
 #ifndef NDEBUG
@@ -577,7 +577,7 @@ static FailureOr<StridedMetadata> resolveReshapeStridedMetadata(
   unsigned sourceRank = sourceType.getRank();
 
   auto newExtractStridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(origLoc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, origLoc, source);
 
   // Collect statically known information.
   auto [strides, offset] = sourceType.getStridesAndOffset();
@@ -828,14 +828,14 @@ struct ExtractStridedMetadataOpAllocFolder
       if (allocLikeOp.getType() == baseBufferType)
         results.push_back(allocLikeOp);
       else
-        results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+        results.push_back(memref::ReinterpretCastOp::create(rewriter,
             loc, baseBufferType, allocLikeOp, offset,
             /*sizes=*/ArrayRef<int64_t>(),
             /*strides=*/ArrayRef<int64_t>()));
     }
 
     // Offset.
-    results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    results.push_back(arith::ConstantIndexOp::create(rewriter, loc, offset));
 
     for (OpFoldResult size : sizes)
       results.push_back(getValueOrCreateConstantIndexOp(rewriter, loc, size));
@@ -900,19 +900,19 @@ struct ExtractStridedMetadataOpGetGlobalFolder
     if (getGlobalOp.getType() == baseBufferType)
       results.push_back(getGlobalOp);
     else
-      results.push_back(rewriter.create<memref::ReinterpretCastOp>(
+      results.push_back(memref::ReinterpretCastOp::create(rewriter,
           loc, baseBufferType, getGlobalOp, offset,
           /*sizes=*/ArrayRef<int64_t>(),
           /*strides=*/ArrayRef<int64_t>()));
 
     // Offset.
-    results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    results.push_back(arith::ConstantIndexOp::create(rewriter, loc, offset));
 
     for (auto size : sizes)
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, size));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, size));
 
     for (auto stride : strides)
-      results.push_back(rewriter.create<arith::ConstantIndexOp>(loc, stride));
+      results.push_back(arith::ConstantIndexOp::create(rewriter, loc, stride));
 
     rewriter.replaceOp(op, results);
     return success();
@@ -1009,7 +1009,7 @@ class ExtractStridedMetadataOpReinterpretCastFolder
     results.resize_for_overwrite(rank * 2 + 2);
 
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(
+        memref::ExtractStridedMetadataOp::create(rewriter,
             loc, reinterpretCastOp.getSource());
 
     // Register the base_buffer.
@@ -1083,7 +1083,7 @@ class ExtractStridedMetadataOpCastFolder
     results.resize_for_overwrite(rank * 2 + 2);
 
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc,
+        memref::ExtractStridedMetadataOp::create(rewriter, loc,
                                                           castOp.getSource());
 
     // Register the base_buffer.
@@ -1143,7 +1143,7 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
     if (!memSpaceCastOp)
       return failure();
     auto newExtractStridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(
+        memref::ExtractStridedMetadataOp::create(rewriter,
             loc, memSpaceCastOp.getSource());
     SmallVector<Value> results(newExtractStridedMetadata.getResults());
     // As with most other strided metadata rewrite patterns, don't introduce
@@ -1158,7 +1158,7 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
       MemRefType::Builder newTypeBuilder(baseBufferType);
       newTypeBuilder.setMemorySpace(
           memSpaceCastOp.getResult().getType().getMemorySpace());
-      results[0] = rewriter.create<memref::MemorySpaceCastOp>(
+      results[0] = memref::MemorySpaceCastOp::create(rewriter,
           loc, Type{newTypeBuilder}, baseBuffer);
     } else {
       results[0] = nullptr;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
index 2f5c9436fb8c7..65f398f6dc5a8 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
@@ -42,7 +42,7 @@ static memref::LoadOp rebuildLoadOp(RewriterBase &rewriter,
                                     memref::LoadOp loadOp, Value srcMemRef,
                                     ArrayRef<Value> indices) {
   Location loc = loadOp.getLoc();
-  return rewriter.create<memref::LoadOp>(loc, srcMemRef, indices,
+  return memref::LoadOp::create(rewriter, loc, srcMemRef, indices,
                                          loadOp.getNontemporal());
 }
 
@@ -72,7 +72,7 @@ static memref::StoreOp rebuildStoreOp(RewriterBase &rewriter,
                                       memref::StoreOp storeOp, Value srcMemRef,
                                       ArrayRef<Value> indices) {
   Location loc = storeOp.getLoc();
-  return rewriter.create<memref::StoreOp>(loc, storeOp.getValueToStore(),
+  return memref::StoreOp::create(rewriter, loc, storeOp.getValueToStore(),
                                           srcMemRef, indices,
                                           storeOp.getNontemporal());
 }
@@ -104,7 +104,7 @@ static nvgpu::LdMatrixOp rebuildLdMatrixOp(RewriterBase &rewriter,
                                            Value srcMemRef,
                                            ArrayRef<Value> indices) {
   Location loc = ldMatrixOp.getLoc();
-  return rewriter.create<nvgpu::LdMatrixOp>(
+  return nvgpu::LdMatrixOp::create(rewriter,
       loc, ldMatrixOp.getResult().getType(), srcMemRef, indices,
       ldMatrixOp.getTranspose(), ldMatrixOp.getNumTiles());
 }
@@ -132,7 +132,7 @@ rebuildTransferReadOp(RewriterBase &rewriter,
                       vector::TransferReadOp transferReadOp, Value srcMemRef,
                       ArrayRef<Value> indices) {
   Location loc = transferReadOp.getLoc();
-  return rewriter.create<vector::TransferReadOp>(
+  return vector::TransferReadOp::create(rewriter,
       loc, transferReadOp.getResult().getType(), srcMemRef, indices,
       transferReadOp.getPermutationMap(), transferReadOp.getPadding(),
       transferReadOp.getMask(), transferReadOp.getInBoundsAttr());
@@ -150,7 +150,7 @@ rebuildTransferWriteOp(RewriterBase &rewriter,
                        vector::TransferWriteOp transferWriteOp, Value srcMemRef,
                        ArrayRef<Value> indices) {
   Location loc = transferWriteOp.getLoc();
-  return rewriter.create<vector::TransferWriteOp>(
+  return vector::TransferWriteOp::create(rewriter,
       loc, transferWriteOp.getValue(), srcMemRef, indices,
       transferWriteOp.getPermutationMapAttr(), transferWriteOp.getMask(),
       transferWriteOp.getInBoundsAttr());
@@ -183,7 +183,7 @@ getGenericOpViewSizeForEachDim(RewriterBase &rewriter,
                                LoadStoreLikeOp loadStoreLikeOp) {
   Location loc = loadStoreLikeOp.getLoc();
   auto extractStridedMetadataOp =
-      rewriter.create<memref::ExtractStridedMetadataOp>(
+      memref::ExtractStridedMetadataOp::create(rewriter,
           loc, getSrcMemRef(loadStoreLikeOp));
   SmallVector<OpFoldResult> srcSizes =
       extractStridedMetadataOp.getConstifiedMixedSizes();
@@ -267,12 +267,12 @@ struct LoadStoreLikeOpRewriter : public OpRewritePattern<LoadStoreLikeOp> {
     // apply them properly to the input indices.
     // Therefore the strides multipliers are simply ones.
     auto subview =
-        rewriter.create<memref::SubViewOp>(loc, /*source=*/srcMemRef,
+        memref::SubViewOp::create(rewriter, loc, /*source=*/srcMemRef,
                                            /*offsets=*/indices,
                                            /*sizes=*/sizes, /*strides=*/ones);
     // Rewrite the load/store with the subview as the base pointer.
     SmallVector<Value> zeros(loadStoreRank,
-                             rewriter.create<arith::ConstantIndexOp>(loc, 0));
+                             arith::ConstantIndexOp::create(rewriter, loc, 0));
     LoadStoreLikeOp newLoadStore = rebuildOpFromAddressAndIndices(
         rewriter, loadStoreLikeOp, subview.getResult(), zeros);
     rewriter.replaceOp(loadStoreLikeOp, newLoadStore->getResults());
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
index 76f7788c4dcc5..48dbabfdf890a 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
@@ -40,7 +40,7 @@ using namespace mlir;
 static Value getValueFromOpFoldResult(OpBuilder &rewriter, Location loc,
                                       OpFoldResult in) {
   if (Attribute offsetAttr = dyn_cast<Attribute>(in)) {
-    return rewriter.create<arith::ConstantIndexOp>(
+    return arith::ConstantIndexOp::create(rewriter,
         loc, cast<IntegerAttr>(offsetAttr).getInt());
   }
   return cast<Value>(in);
@@ -60,7 +60,7 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
   }
 
   memref::ExtractStridedMetadataOp stridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(loc, source);
+      memref::ExtractStridedMetadataOp::create(rewriter, loc, source);
 
   auto typeBit = sourceType.getElementType().getIntOrFloatBitWidth();
   OpFoldResult linearizedIndices;
@@ -74,7 +74,7 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
           getAsOpFoldResult(indices));
 
   return std::make_pair(
-      rewriter.create<memref::ReinterpretCastOp>(
+      memref::ReinterpretCastOp::create(rewriter,
           loc, source,
           /* offset = */ linearizedInfo.linearizedOffset,
           /* shapes = */
@@ -111,7 +111,7 @@ template <typename T>
 static void castAllocResult(T oper, T newOper, Location loc,
                             PatternRewriter &rewriter) {
   memref::ExtractStridedMetadataOp stridedMetadata =
-      rewriter.create<memref::ExtractStridedMetadataOp>(loc, oper);
+      memref::ExtractStridedMetadataOp::create(rewriter, loc, oper);
   rewriter.replaceOpWithNewOp<memref::ReinterpretCastOp>(
       oper, cast<MemRefType>(oper.getType()), newOper,
       /*offset=*/rewriter.getIndexAttr(0),
@@ -125,62 +125,62 @@ static void replaceOp(T op, PatternRewriter &rewriter, Value flatMemref,
   Location loc = op->getLoc();
   llvm::TypeSwitch<Operation *>(op.getOperation())
       .template Case<memref::AllocOp>([&](auto oper) {
-        auto newAlloc = rewriter.create<memref::AllocOp>(
+        auto newAlloc = memref::AllocOp::create(rewriter,
             loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloc, loc, rewriter);
       })
       .template Case<memref::AllocaOp>([&](auto oper) {
-        auto newAlloca = rewriter.create<memref::AllocaOp>(
+        auto newAlloca = memref::AllocaOp::create(rewriter,
             loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloca, loc, rewriter);
       })
       .template Case<memref::LoadOp>([&](auto op) {
-        auto newLoad = rewriter.create<memref::LoadOp>(
+        auto newLoad = memref::LoadOp::create(rewriter,
             loc, op->getResultTypes(), flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<memref::StoreOp>([&](auto op) {
-        auto newStore = rewriter.create<memref::StoreOp>(
+        auto newStore = memref::StoreOp::create(rewriter,
             loc, op->getOperands().front(), flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::LoadOp>([&](auto op) {
-        auto newLoad = rewriter.create<vector::LoadOp>(
+        auto newLoad = vector::LoadOp::create(rewriter,
             loc, op->getResultTypes(), flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<vector::StoreOp>([&](auto op) {
-        auto newStore = rewriter.create<vector::StoreOp>(
+        auto newStore = vector::StoreOp::create(rewriter,
             loc, op->getOperands().front(), flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::MaskedLoadOp>([&](auto op) {
-        auto newMaskedLoad = rewriter.create<vector::MaskedLoadOp>(
+        auto newMaskedLoad = vector::MaskedLoadOp::create(rewriter,
             loc, op.getType(), flatMemref, ValueRange{offset}, op.getMask(),
             op.getPassThru());
         newMaskedLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedLoad.getResult());
       })
       .template Case<vector::MaskedStoreOp>([&](auto op) {
-        auto newMaskedStore = rewriter.create<vector::MaskedStoreOp>(
+        auto newMaskedStore = vector::MaskedStoreOp::create(rewriter,
             loc, flatMemref, ValueRange{offset}, op.getMask(),
             op.getValueToStore());
         newMaskedStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedStore);
       })
       .template Case<vector::TransferReadOp>([&](auto op) {
-        auto newTransferRead = rewriter.create<vector::TransferReadOp>(
+        auto newTransferRead = vector::TransferReadOp::create(rewriter,
             loc, op.getType(), flatMemref, ValueRange{offset}, op.getPadding());
         rewriter.replaceOp(op, newTransferRead.getResult());
       })
       .template Case<vector::TransferWriteOp>([&](auto op) {
-        auto newTransferWrite = rewriter.create<vector::TransferWriteOp>(
+        auto newTransferWrite = vector::TransferWriteOp::create(rewriter,
             loc, op.getVector(), flatMemref, ValueRange{offset});
         rewriter.replaceOp(op, newTransferWrite);
       })
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index 89be188af9129..8e99a101f28a6 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -76,7 +76,7 @@ static LogicalResult resolveSourceIndicesExpandShape(
         llvm::map_to_vector(group, [&](int64_t d) { return destShape[d]; });
     SmallVector<Value> groupIndices =
         llvm::map_to_vector(group, [&](int64_t d) { return indices[d]; });
-    Value collapsedIndex = rewriter.create<affine::AffineLinearizeIndexOp>(
+    Value collapsedIndex = affine::AffineLinearizeIndexOp::create(rewriter,
         loc, groupIndices, groupBasis, /*disjoint=*/startsInbounds);
     sourceIndices.push_back(collapsedIndex);
   }
@@ -102,7 +102,7 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
                                   ValueRange indices,
                                   SmallVectorImpl<Value> &sourceIndices) {
   // Note: collapse_shape requires a strided memref, we can do this.
-  auto metadata = rewriter.create<memref::ExtractStridedMetadataOp>(
+  auto metadata = memref::ExtractStridedMetadataOp::create(rewriter,
       loc, collapseShapeOp.getSrc());
   SmallVector<OpFoldResult> sourceSizes = metadata.getConstifiedMixedSizes();
   for (auto [index, group] :
@@ -117,7 +117,7 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
 
     SmallVector<OpFoldResult> basis =
         llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-    auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+    auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
         loc, index, basis, /*hasOuterBound=*/true);
     llvm::append_range(sourceIndices, delinearize.getResults());
   }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
index 35c661ecb886d..953024d70d0aa 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
@@ -51,7 +51,7 @@ FailureOr<Value> memref::buildIndependentOp(OpBuilder &b,
 
   // Create a new memref::AllocaOp.
   Value newAllocaOp =
-      b.create<AllocaOp>(loc, newSizes, allocaOp.getType().getElementType());
+      AllocaOp::create(b, loc, newSizes, allocaOp.getType().getElementType());
 
   // Create a memref::SubViewOp.
   SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
@@ -71,10 +71,10 @@ propagateSubViewOp(RewriterBase &rewriter,
   MemRefType newResultType = SubViewOp::inferRankReducedResultType(
       op.getType().getShape(), op.getSourceType(), op.getMixedOffsets(),
       op.getMixedSizes(), op.getMixedStrides());
-  Value newSubview = rewriter.create<SubViewOp>(
+  Value newSubview = SubViewOp::create(rewriter,
       op.getLoc(), newResultType, conversionOp.getOperand(0),
       op.getMixedOffsets(), op.getMixedSizes(), op.getMixedStrides());
-  auto newConversionOp = rewriter.create<UnrealizedConversionCastOp>(
+  auto newConversionOp = UnrealizedConversionCastOp::create(rewriter,
       op.getLoc(), op.getType(), newSubview);
   rewriter.replaceAllUsesWith(op.getResult(), newConversionOp->getResult(0));
   return newConversionOp;
@@ -106,7 +106,7 @@ static void replaceAndPropagateMemRefType(RewriterBase &rewriter,
   SmallVector<UnrealizedConversionCastOp> unrealizedConversions;
   for (const auto &it :
        llvm::enumerate(llvm::zip(from->getResults(), to->getResults()))) {
-    unrealizedConversions.push_back(rewriter.create<UnrealizedConversionCastOp>(
+    unrealizedConversions.push_back(UnrealizedConversionCastOp::create(rewriter,
         to->getLoc(), std::get<0>(it.value()).getType(),
         std::get<1>(it.value())));
     rewriter.replaceAllUsesWith(from->getResult(it.index()),
diff --git a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
index 0a84962150ead..b7e4388193a06 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
@@ -63,7 +63,7 @@ static void replaceUsesAndPropagateType(RewriterBase &rewriter,
         subviewUse.getType().getShape(), cast<MemRefType>(val.getType()),
         subviewUse.getStaticOffsets(), subviewUse.getStaticSizes(),
         subviewUse.getStaticStrides());
-    Value newSubview = rewriter.create<memref::SubViewOp>(
+    Value newSubview = memref::SubViewOp::create(rewriter,
         subviewUse->getLoc(), newType, val, subviewUse.getMixedOffsets(),
         subviewUse.getMixedSizes(), subviewUse.getMixedStrides());
 
@@ -177,7 +177,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   Location loc = allocOp->getLoc();
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(allocOp);
-  auto mbAlloc = rewriter.create<memref::AllocOp>(
+  auto mbAlloc = memref::AllocOp::create(rewriter,
       loc, mbMemRefType, ValueRange{}, allocOp->getAttrs());
   LLVM_DEBUG(DBGS() << "--multi-buffered alloc: " << mbAlloc << "\n");
 
@@ -211,7 +211,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   // Strides is [1, 1 ... 1 ].
   MemRefType dstMemref = memref::SubViewOp::inferRankReducedResultType(
       originalShape, mbMemRefType, offsets, sizes, strides);
-  Value subview = rewriter.create<memref::SubViewOp>(loc, dstMemref, mbAlloc,
+  Value subview = memref::SubViewOp::create(rewriter, loc, dstMemref, mbAlloc,
                                                      offsets, sizes, strides);
   LLVM_DEBUG(DBGS() << "--multi-buffered slice: " << subview << "\n");
 
@@ -224,7 +224,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPoint(deallocOp);
     auto newDeallocOp =
-        rewriter.create<memref::DeallocOp>(deallocOp->getLoc(), mbAlloc);
+        memref::DeallocOp::create(rewriter, deallocOp->getLoc(), mbAlloc);
     (void)newDeallocOp;
     LLVM_DEBUG(DBGS() << "----Created dealloc: " << newDeallocOp << "\n");
     rewriter.eraseOp(deallocOp);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index 4ec04321dd3e2..c522b2bd5ddec 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -276,7 +276,7 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
     if (!callOp)
       continue;
     Operation *newCallOp =
-        builder.create<func::CallOp>(userOp->getLoc(), callOp.getCalleeAttr(),
+        func::CallOp::create(builder, userOp->getLoc(), callOp.getCalleeAttr(),
                                      resultTypes, userOp->getOperands());
     bool replacingMemRefUsesFailed = false;
     bool returnTypeChanged = false;
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
index e6b9e2f7e8213..ea7f394f48ba4 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
@@ -116,10 +116,10 @@ static LogicalResult reifyOpResultShapes(RewriterBase &rewriter,
     // Update the type.
     newRes.setType(reifiedTy);
     if (isa<RankedTensorType>(reifiedTy)) {
-      newResults.push_back(rewriter.create<tensor::CastOp>(loc, oldTy, newRes));
+      newResults.push_back(tensor::CastOp::create(rewriter, loc, oldTy, newRes));
     } else {
       assert(isa<MemRefType>(reifiedTy) && "expected a memref type");
-      newResults.push_back(rewriter.create<memref::CastOp>(loc, oldTy, newRes));
+      newResults.push_back(memref::CastOp::create(rewriter, loc, oldTy, newRes));
     }
   }
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
index 89a3895d06ba5..6a81a15f30e47 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
@@ -69,7 +69,7 @@ struct DimOfShapedTypeOpInterface : public OpRewritePattern<OpTy> {
     Location loc = dimOp->getLoc();
     rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
         dimOp, resultShape,
-        rewriter.create<arith::ConstantIndexOp>(loc, *dimIndex).getResult());
+        arith::ConstantIndexOp::create(rewriter, loc, *dimIndex).getResult());
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
index d231516884c7d..fe9555a7a16fd 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
@@ -40,15 +40,15 @@ struct AssumeAlignmentOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto assumeOp = cast<AssumeAlignmentOp>(op);
-    Value ptr = builder.create<ExtractAlignedPointerAsIndexOp>(
+    Value ptr = ExtractAlignedPointerAsIndexOp::create(builder,
         loc, assumeOp.getMemref());
-    Value rest = builder.create<arith::RemUIOp>(
+    Value rest = arith::RemUIOp::create(builder,
         loc, ptr,
-        builder.create<arith::ConstantIndexOp>(loc, assumeOp.getAlignment()));
-    Value isAligned = builder.create<arith::CmpIOp>(
+        arith::ConstantIndexOp::create(builder, loc, assumeOp.getAlignment()));
+    Value isAligned = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::eq, rest,
-        builder.create<arith::ConstantIndexOp>(loc, 0));
-    builder.create<cf::AssertOp>(
+        arith::ConstantIndexOp::create(builder, loc, 0));
+    cf::AssertOp::create(builder,
         loc, isAligned,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "memref is not aligned to " +
@@ -71,12 +71,12 @@ struct CastOpInterface
 
     if (isa<UnrankedMemRefType>(srcType)) {
       // Check rank.
-      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
-          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
-      Value isSameRank = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
+      Value isSameRank = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameRank,
           RuntimeVerifiableOpInterface::generateErrorMessage(op,
                                                              "rank mismatch"));
@@ -95,8 +95,8 @@ struct CastOpInterface
         MemRefType::get(dynamicShape, resultType.getElementType(),
                         stridedLayout, resultType.getMemorySpace());
     Value helperCast =
-        builder.create<CastOp>(loc, dynStridesType, castOp.getSource());
-    auto metadataOp = builder.create<ExtractStridedMetadataOp>(loc, helperCast);
+        CastOp::create(builder, loc, dynStridesType, castOp.getSource());
+    auto metadataOp = ExtractStridedMetadataOp::create(builder, loc, helperCast);
 
     // Check dimension sizes.
     for (const auto &it : llvm::enumerate(resultType.getShape())) {
@@ -110,12 +110,12 @@ struct CastOpInterface
         continue;
 
       Value srcDimSz =
-          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+          DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameSz = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameSz = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
@@ -132,10 +132,10 @@ struct CastOpInterface
       // Static/dynamic offset -> dynamic offset does not need verification.
       Value srcOffset = metadataOp.getResult(1);
       Value resultOffsetVal =
-          builder.create<arith::ConstantIndexOp>(loc, resultOffset);
-      Value isSameOffset = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultOffset);
+      Value isSameOffset = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcOffset, resultOffsetVal);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameOffset,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset mismatch"));
@@ -150,10 +150,10 @@ struct CastOpInterface
       Value srcStride =
           metadataOp.getResult(2 + resultType.getRank() + it.index());
       Value resultStrideVal =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameStride = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameStride = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcStride, resultStrideVal);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameStride,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "stride mismatch of dim " + std::to_string(it.index())));
@@ -186,7 +186,7 @@ struct CopyOpInterface
       auto getDimSize = [&](Value memRef, MemRefType type,
                             int64_t dim) -> Value {
         return type.isDynamicDim(dim)
-                   ? builder.create<DimOp>(loc, memRef, dim).getResult()
+                   ? DimOp::create(builder, loc, memRef, dim).getResult()
                    : builder
                          .create<arith::ConstantIndexOp>(loc,
                                                          type.getDimSize(dim))
@@ -194,9 +194,9 @@ struct CopyOpInterface
       };
       Value sourceDim = getDimSize(copyOp.getSource(), rankedSourceType, i);
       Value targetDim = getDimSize(copyOp.getTarget(), rankedTargetType, i);
-      Value sameDimSize = builder.create<arith::CmpIOp>(
+      Value sameDimSize = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, sameDimSize,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size of " + std::to_string(i) +
@@ -211,9 +211,9 @@ struct DimOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto dimOp = cast<DimOp>(op);
-    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    builder.create<cf::AssertOp>(
+    Value rank = RankOp::create(builder, loc, dimOp.getSource());
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    cf::AssertOp::create(builder,
         loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
@@ -237,7 +237,7 @@ struct LoadStoreOpInterface
     }
     auto indices = loadStoreOp.getIndices();
 
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     Value assertCond;
     for (auto i : llvm::seq<int64_t>(0, rank)) {
       Value dimOp = builder.createOrFold<memref::DimOp>(loc, memref, i);
@@ -247,7 +247,7 @@ struct LoadStoreOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    builder.create<cf::AssertOp>(
+    cf::AssertOp::create(builder,
         loc, assertCond,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "out-of-bounds access"));
@@ -265,10 +265,10 @@ struct SubViewOpInterface
     // For each dimension, assert that:
     // 0 <= offset < dim_size
     // 0 <= offset + (size - 1) * stride < dim_size
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value one = arith::ConstantIndexOp::create(builder, loc, 1);
     auto metadataOp =
-        builder.create<ExtractStridedMetadataOp>(loc, subView.getSource());
+        ExtractStridedMetadataOp::create(builder, loc, subView.getSource());
     for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
       Value offset = getValueOrCreateConstantIndexOp(
           builder, loc, subView.getMixedOffsets()[i]);
@@ -281,20 +281,20 @@ struct SubViewOpInterface
       Value dimSize = metadataOp.getSizes()[i];
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
       // Verify that slice does not run out-of-bounds.
-      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOne = arith::SubIOp::create(builder, loc, size, one);
       Value sizeMinusOneTimesStride =
-          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+          arith::MulIOp::create(builder, loc, sizeMinusOne, stride);
       Value lastPos =
-          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+          arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "subview runs out-of-bounds along dimension " +
@@ -315,7 +315,7 @@ struct ExpandShapeOpInterface
     for (const auto &it :
          llvm::enumerate(expandShapeOp.getReassociationIndices())) {
       Value srcDimSz =
-          builder.create<DimOp>(loc, expandShapeOp.getSrc(), it.index());
+          DimOp::create(builder, loc, expandShapeOp.getSrc(), it.index());
       int64_t groupSz = 1;
       bool foundDynamicDim = false;
       for (int64_t resultDim : it.value()) {
@@ -330,14 +330,14 @@ struct ExpandShapeOpInterface
         groupSz *= expandShapeOp.getResultType().getDimSize(resultDim);
       }
       Value staticResultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, groupSz);
+          arith::ConstantIndexOp::create(builder, loc, groupSz);
       // staticResultDimSz must divide srcDimSz evenly.
       Value mod =
-          builder.create<arith::RemSIOp>(loc, srcDimSz, staticResultDimSz);
-      Value isModZero = builder.create<arith::CmpIOp>(
+          arith::RemSIOp::create(builder, loc, srcDimSz, staticResultDimSz);
+      Value isModZero = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, mod,
-          builder.create<arith::ConstantIndexOp>(loc, 0));
-      builder.create<cf::AssertOp>(
+          arith::ConstantIndexOp::create(builder, loc, 0));
+      cf::AssertOp::create(builder,
           loc, isModZero,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "static result dims in reassoc group do not "
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index cf506d1e7812b..a440749dbd703 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -91,7 +91,7 @@ SmallVector<Value> mlir::mesh::getMixedAsValues(OpBuilder b,
       values.emplace_back(*(dyn++));
     } else {
       TypedAttr val = type == i64 ? b.getI64IntegerAttr(s) : b.getIndexAttr(s);
-      values.emplace_back(b.create<arith::ConstantOp>(loc, type, val));
+      values.emplace_back(arith::ConstantOp::create(b, loc, type, val));
     }
   }
   return values;
@@ -316,9 +316,9 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
 
   if (!newShardOp) {
     auto shardingOp =
-        builder.create<ShardingOp>(operandValue.getLoc(), sharding);
+        ShardingOp::create(builder, operandValue.getLoc(), sharding);
     newShardOp =
-        builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+        ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                                 /*annotate_for_users*/ false);
   }
   operandValue.replaceUsesWithIf(
@@ -330,7 +330,7 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
     return;
   }
 
-  auto newShardOp2 = builder.create<ShardOp>(operandValue.getLoc(), newShardOp,
+  auto newShardOp2 = ShardOp::create(builder, operandValue.getLoc(), newShardOp,
                                              newShardOp.getSharding(),
                                              /*annotate_for_users*/ true);
   newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
@@ -378,9 +378,9 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
 
   builder.setInsertionPoint(operandOp);
   auto shardingOp =
-      builder.create<ShardingOp>(operand.get().getLoc(), sharding);
+      ShardingOp::create(builder, operand.get().getLoc(), sharding);
   auto newShardOp =
-      builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+      ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                               /*annotate_for_users*/ true);
   IRRewriter rewriter(builder);
   rewriter.replaceUsesWithIf(
@@ -395,7 +395,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
 
   builder.setInsertionPoint(newShardOp);
   auto newPreceedingShardOp =
-      builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
+      ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
                               /*annotate_for_users*/ false);
   rewriter.replaceUsesWithIf(
       newShardOp.getSrc(), newPreceedingShardOp, [&newShardOp](OpOperand &use) {
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
index 9da3c9a3dd160..65446142aabcc 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
@@ -91,7 +91,7 @@ struct MeshShapeFolder
         newShapeOpMeshAxes.push_back(opMeshAxes[i]);
       } else {
         // Fold static mesh axes.
-        newResults[i] = builder.create<arith::ConstantOp>(
+        newResults[i] = arith::ConstantOp::create(builder,
             builder.getIndexAttr(meshAxisSize));
       }
     }
@@ -99,7 +99,7 @@ struct MeshShapeFolder
     // Leave only the dynamic mesh axes to be queried.
     if (!newShapeOpMeshAxes.empty()) {
       MeshShapeOp newShapeOp =
-          builder.create<MeshShapeOp>(mesh.getSymName(), newShapeOpMeshAxes);
+          MeshShapeOp::create(builder, mesh.getSymName(), newShapeOpMeshAxes);
       for (size_t i = 0; i < newShapeOp->getResults().size(); ++i) {
         newResults[newToOldResultsIndexMap[i]] = newShapeOp->getResults()[i];
       }
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index d7b7234f69347..9229bfaa1dffb 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -265,7 +265,7 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
       targetShardingInUnsplitLastAxis(ctx, sourceSharding, splitTensorAxis);
   ShapedType allGatherResultShape = allGatherResultShapeInUnsplitLastAxis(
       sourceShard.getType(), mesh.getShape()[splitMeshAxis], splitTensorAxis);
-  Value allGatherResult = builder.create<AllGatherOp>(
+  Value allGatherResult = AllGatherOp::create(builder,
       RankedTensorType::get(allGatherResultShape.getShape(),
                             allGatherResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({splitMeshAxis}), sourceShard,
@@ -273,7 +273,7 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      builder.create<tensor::CastOp>(targetShape, allGatherResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allGatherResult).getResult());
   return {targetShard, targetSharding};
 }
 
@@ -398,7 +398,7 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType allToAllResultShape = allToAllResultShapeInMoveLastAxis(
       sourceShard.getType(), mesh.getShape()[meshAxis], sourceTensorAxis,
       targetTensorAxis);
-  Value allToAllResult = builder.create<AllToAllOp>(
+  Value allToAllResult = AllToAllOp::create(builder,
       RankedTensorType::get(allToAllResultShape.getShape(),
                             allToAllResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({meshAxis}), sourceShard,
@@ -406,7 +406,7 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      builder.create<tensor::CastOp>(targetShape, allToAllResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allToAllResult).getResult());
   return {targetShard, targetSharding};
 }
 
@@ -477,13 +477,13 @@ tryUpdateHaloInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
 
   // Extract core from source and copy into destination core.
   auto noVals = ValueRange{};
-  auto initVal = builder.create<tensor::EmptyOp>(
+  auto initVal = tensor::EmptyOp::create(builder,
       sourceShard.getLoc(), outShape, sourceShard.getType().getElementType());
-  auto core = builder.create<tensor::ExtractSliceOp>(
+  auto core = tensor::ExtractSliceOp::create(builder,
       sourceShard.getLoc(),
       RankedTensorType::get(coreShape, sourceShard.getType().getElementType()),
       sourceShard, noVals, noVals, noVals, srcCoreOffs, coreShape, strides);
-  auto initOprnd = builder.create<tensor::InsertSliceOp>(
+  auto initOprnd = tensor::InsertSliceOp::create(builder,
       sourceShard.getLoc(), core, initVal, noVals, noVals, noVals, tgtCoreOffs,
       coreShape, strides);
 
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
index f08ef75d8a004..c85394924608d 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
@@ -49,10 +49,10 @@ struct ProcessMultiIndexOpLowering
 
     ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     builder.setInsertionPointAfter(op.getOperation());
-    Value linearIndex = builder.create<ProcessLinearIndexOp>(mesh);
-    ValueRange meshShape = builder.create<MeshShapeOp>(mesh).getResults();
+    Value linearIndex = ProcessLinearIndexOp::create(builder, mesh);
+    ValueRange meshShape = MeshShapeOp::create(builder, mesh).getResults();
     SmallVector<Value> completeMultiIndex =
-        builder.create<affine::AffineDelinearizeIndexOp>(linearIndex, meshShape)
+        affine::AffineDelinearizeIndexOp::create(builder, linearIndex, meshShape)
             .getMultiIndex();
     SmallVector<Value> multiIndex;
     ArrayRef<MeshAxis> opMeshAxes = op.getAxes();
@@ -101,32 +101,32 @@ struct AllSliceOpLowering
     ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
     builder.setInsertionPointAfter(op.getOperation());
 
-    Value zero = builder.create<arith::ConstantOp>(builder.getIndexAttr(0));
+    Value zero = arith::ConstantOp::create(builder, builder.getIndexAttr(0));
 
     Operation::result_range processInGroupMultiIndex =
-        builder.create<ProcessMultiIndexOp>(mesh.getSymName(), op.getMeshAxes())
+        ProcessMultiIndexOp::create(builder, mesh.getSymName(), op.getMeshAxes())
             .getResults();
 
     Operation::result_range processGroupShape =
-        builder.create<MeshShapeOp>(mesh.getSymName(), op.getMeshAxes())
+        MeshShapeOp::create(builder, mesh.getSymName(), op.getMeshAxes())
             .getResult();
     Value processGroupSize =
         createCollectiveProcessGroupSize(mesh, op.getMeshAxes(), builder);
 
     int64_t sliceAxis = op.getSliceAxis().getSExtValue();
     Value operandSliceAxisSize =
-        builder.create<tensor::DimOp>(op.getOperand(), sliceAxis);
+        tensor::DimOp::create(builder, op.getOperand(), sliceAxis);
     Value operandSliceAxisSizeModProcessGroupSize =
-        builder.create<arith::RemUIOp>(operandSliceAxisSize, processGroupSize);
-    Value isTargetShapeExactlyDivisible = builder.create<arith::CmpIOp>(
+        arith::RemUIOp::create(builder, operandSliceAxisSize, processGroupSize);
+    Value isTargetShapeExactlyDivisible = arith::CmpIOp::create(builder,
         arith::CmpIPredicate::eq, operandSliceAxisSizeModProcessGroupSize,
         zero);
-    builder.create<cf::AssertOp>(isTargetShapeExactlyDivisible,
+    cf::AssertOp::create(builder, isTargetShapeExactlyDivisible,
                                  "Slicing a tensor with axis size that is "
                                  "not exactly divisible by the "
                                  "mesh process group size is not supported.");
     Value resultSliceAxisSize =
-        builder.create<arith::DivUIOp>(operandSliceAxisSize, processGroupSize);
+        arith::DivUIOp::create(builder, operandSliceAxisSize, processGroupSize);
     OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
         llvm::to_vector_of<OpFoldResult>(processInGroupMultiIndex),
         llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
@@ -139,7 +139,7 @@ struct AllSliceOpLowering
       if (i == sliceAxis) {
         sizes.emplace_back(resultSliceAxisSize);
       } else {
-        Value dimSize = builder.create<tensor::DimOp>(op.getOperand(), i);
+        Value dimSize = tensor::DimOp::create(builder, op.getOperand(), i);
         sizes.emplace_back(dimSize);
       }
     }
@@ -152,10 +152,10 @@ struct AllSliceOpLowering
                  resultSliceAxisSize);
     SmallVector<OpFoldResult> strides(
         operandType.getRank(), getAsIndexOpFoldResult(builder.getContext(), 1));
-    Value slice = builder.create<tensor::ExtractSliceOp>(
+    Value slice = tensor::ExtractSliceOp::create(builder,
         op.getOperand(), offsets, sizes, strides);
     Value newResult =
-        builder.create<tensor::CastOp>(op.getResult().getType(), slice);
+        tensor::CastOp::create(builder, op.getResult().getType(), slice);
     rewriter.replaceAllUsesWith(op.getResult(), newResult);
 
     return success();
@@ -201,7 +201,7 @@ TypedValue<IndexType>
 createCollectiveProcessGroupSize(MeshOp mesh, ArrayRef<MeshAxis> axes,
                                  ImplicitLocOpBuilder &builder) {
   Operation::result_range meshShape =
-      builder.create<mesh::MeshShapeOp>(mesh, axes).getResults();
+      mesh::MeshShapeOp::create(builder, mesh, axes).getResults();
   return cast<TypedValue<IndexType>>(arith::createProduct(
       builder, builder.getLoc(), llvm::to_vector_of<Value>(meshShape),
       builder.getIndexType()));
@@ -212,13 +212,13 @@ createProcessLinearIndex(StringRef mesh, ValueRange processInGroupMultiIndex,
                          ArrayRef<MeshAxis> meshAxes,
                          ImplicitLocOpBuilder &builder) {
   Operation::result_range processGroupShape =
-      builder.create<MeshShapeOp>(mesh, meshAxes).getResult();
+      MeshShapeOp::create(builder, mesh, meshAxes).getResult();
   OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
       llvm::to_vector_of<OpFoldResult>(processInGroupMultiIndex),
       llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
   auto res = dyn_cast<Value>(processInGroupLinearIndex);
   if (!res)
-    res = builder.create<arith::ConstantIndexOp>(
+    res = arith::ConstantIndexOp::create(builder,
         cast<IntegerAttr>(cast<Attribute>(processInGroupLinearIndex)).getInt());
   return cast<TypedValue<IndexType>>(res);
 }
@@ -227,7 +227,7 @@ TypedValue<IndexType> createProcessLinearIndex(StringRef mesh,
                                                ArrayRef<MeshAxis> meshAxes,
                                                ImplicitLocOpBuilder &builder) {
   return createProcessLinearIndex(
-      mesh, builder.create<ProcessMultiIndexOp>(mesh, meshAxes).getResults(),
+      mesh, ProcessMultiIndexOp::create(builder, mesh, meshAxes).getResults(),
       meshAxes, builder);
 }
 } // namespace mlir::mesh
diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
index d2c94b124cdfb..85317a68c9f00 100644
--- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
+++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
@@ -334,13 +334,13 @@ static Operation *replaceOpWithPredicatedOp(RewriterBase &rewriter,
   //
   Location loc = asyncCopyOp->getLoc();
   Value dstElements =
-      rewriter.create<arith::ConstantOp>(loc, asyncCopyOp.getDstElementsAttr());
+      arith::ConstantOp::create(rewriter, loc, asyncCopyOp.getDstElementsAttr());
   Value originalSrcElement =
       asyncCopyOp.getSrcElements() ? asyncCopyOp.getSrcElements() : dstElements;
-  Value c0Index = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto srcElements = rewriter.create<arith::SelectOp>(
+  Value c0Index = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto srcElements = arith::SelectOp::create(rewriter,
       loc, predicate, originalSrcElement, c0Index);
-  auto asyncCopyZeroFillOp = rewriter.create<nvgpu::DeviceAsyncCopyOp>(
+  auto asyncCopyZeroFillOp = nvgpu::DeviceAsyncCopyOp::create(rewriter,
       loc, nvgpu::DeviceAsyncTokenType::get(asyncCopyOp.getContext()),
       asyncCopyOp.getDst(), asyncCopyOp.getDstIndices(), asyncCopyOp.getSrc(),
       asyncCopyOp.getSrcIndices(), asyncCopyOp.getDstElements(), srcElements,
@@ -675,7 +675,7 @@ MmaSyncBuilder::buildMemRefLoads(OpBuilder &b, Location loc,
   for (auto indexing : indexings) {
     Value row = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.row()));
     Value col = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.col()));
-    auto load = b.create<memref::LoadOp>(loc, memref, ValueRange{row, col});
+    auto load = memref::LoadOp::create(b, loc, memref, ValueRange{row, col});
     res.push_back(load);
   }
   return res;
@@ -688,7 +688,7 @@ Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
 
   Type elementType = getElementTypeOrSelf(memref.getType());
   auto vt = VectorType::get(vectorShape, elementType);
-  Value res = b.create<vector::SplatOp>(loc, vt, loads[0]);
+  Value res = vector::SplatOp::create(b, loc, vt, loads[0]);
   foreachIndividualVectorElement(
       res,
       /*applyFn=*/
@@ -697,7 +697,7 @@ Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
       },
       /*reduceFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
-        res = b.create<vector::InsertOp>(loc, v, res, indices);
+        res = vector::InsertOp::create(b, loc, v, res, indices);
       });
 
   return res;
@@ -715,7 +715,7 @@ SmallVector<Operation *> MmaSyncBuilder::buildMemRefStores(
     Value row = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.row()));
     Value col = getValueOrCreateConstantIndexOp(b, loc, aff(indexing.col()));
     Operation *store =
-        b.create<memref::StoreOp>(loc, val, memref, ValueRange{row, col});
+        memref::StoreOp::create(b, loc, val, memref, ValueRange{row, col});
     res.push_back(store);
   }
   return res;
@@ -730,7 +730,7 @@ SmallVector<Operation *> MmaSyncBuilder::buildMmaSyncMemRefStoreOperand(
       vectorToStore,
       /*applyFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
-        return b.create<vector::ExtractOp>(loc, vectorToStore, indices);
+        return vector::ExtractOp::create(b, loc, vectorToStore, indices);
       },
       /*reduceFn=*/
       [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
@@ -810,7 +810,7 @@ FailureOr<Operation *> MmaSyncBuilder::buildMmaSync(LinalgOp linalgOp) {
                                             rhsIndexFn, rhsShape);
   Value res = buildMmaSyncMemRefLoadOperand(b, loc, laneId, resMemRef,
                                             resIndexFn, resShape);
-  res = b.create<nvgpu::MmaSyncOp>(loc, lhs, rhs, res, info.mmaShape,
+  res = nvgpu::MmaSyncOp::create(b, loc, lhs, rhs, res, info.mmaShape,
                                    info.tf32Enabled);
   buildMmaSyncMemRefStoreOperand(b, loc, res, laneId, resMemRef, resIndexFn,
                                  resShape);
@@ -832,7 +832,7 @@ DiagnosedSilenceableFailure transform::RewriteMatmulAsMmaSyncOp::applyToOne(
     }
     Location loc = linalgOp.getLoc();
     // TODO: more robust computation of laneId, for now assume a single warp.
-    Value laneId = rewriter.create<gpu::ThreadIdOp>(
+    Value laneId = gpu::ThreadIdOp::create(rewriter,
         loc, rewriter.getIndexType(), gpu::Dimension::x);
     if (succeeded(MmaSyncBuilder(rewriter, loc, laneId).buildMmaSync(linalgOp)))
       fail = false;
@@ -897,12 +897,12 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
     ArrayRef<TypedValue<MemRefType>> sharedMemBuffers,
     TypedValue<nvgpu::MBarrierGroupType> barrier) {
   SmallVector<Operation *> loadOps;
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Value tidx = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Value tidx = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
   Value cond =
-      rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, tidx, zero);
+      arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, tidx, zero);
   // clang-format off
-  rewriter.create<scf::IfOp>(
+  scf::IfOp::create(rewriter,
     /*location=*/loc,
     /*conditional=*/cond,
     /*thenBuilder=*/
@@ -917,14 +917,14 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
       // TODO: Note that cutlass predeclares the barrier arrive tx before the tma.async.load.
       // This may or may not have perf implications.
       buildBarrierArriveTx(barrier, sizes);
-      rewriter.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(rewriter, loc);
     },
     /*elseBuilder=*/
     [&](OpBuilder &lb, Location loc) {
       // TODO: is this for no-thread divergence?
       // Should we just yield the size and hoist?
       buildBarrierArriveTx(barrier, getAsIndexOpFoldResult(rewriter.getContext(), 0));
-      rewriter.create<scf::YieldOp>(loc);
+      scf::YieldOp::create(rewriter, loc);
     });
   // clang-format on
   return loadOps;
@@ -939,14 +939,14 @@ static Attribute getSharedAddressSpaceAttribute(OpBuilder &b) {
 TypedValue<nvgpu::MBarrierGroupType>
 HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) {
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value barrier = rewriter.create<nvgpu::MBarrierCreateOp>(
+  Value barrier = nvgpu::MBarrierCreateOp::create(rewriter,
       loc,
       nvgpu::MBarrierGroupType::get(rewriter.getContext(), sharedMemorySpace));
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierInitOp>(
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierInitOp::create(rewriter,
       loc, barrier, getValueOrCreateConstantIndexOp(rewriter, loc, numThreads),
       zero, Value());
-  rewriter.create<gpu::BarrierOp>(loc);
+  gpu::BarrierOp::create(rewriter, loc);
   return cast<TypedValue<nvgpu::MBarrierGroupType>>(barrier);
 }
 
@@ -955,7 +955,7 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
                                            gpu::LaunchOp launchOp) {
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(launchOp);
-  Value unrankedMemRef = rewriter.create<memref::CastOp>(
+  Value unrankedMemRef = memref::CastOp::create(rewriter,
       loc,
       UnrankedMemRefType::get(memref.getType().getElementType(),
                               memref.getType().getMemorySpace()),
@@ -966,7 +966,7 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
       getValueOrCreateConstantIndexOp(rewriter, loc, mixedSizes);
 
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value desc = rewriter.create<nvgpu::TmaCreateDescriptorOp>(
+  Value desc = nvgpu::TmaCreateDescriptorOp::create(rewriter,
       loc,
       nvgpu::TensorMapDescriptorType::get(
           rewriter.getContext(),
@@ -985,8 +985,8 @@ OpFoldResult HopperBuilder::buildTmaAsyncLoad(
     TypedValue<nvgpu::MBarrierGroupType> barrier,
     SmallVectorImpl<Operation *> &loadOps) {
   MLIRContext *ctx = rewriter.getContext();
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  Operation *loadOp = rewriter.create<nvgpu::TmaAsyncLoadOp>(
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  Operation *loadOp = nvgpu::TmaAsyncLoadOp::create(rewriter,
       loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero,
       Value(), Value());
   loadOps.push_back(loadOp);
@@ -1012,22 +1012,22 @@ void HopperBuilder::buildBarrierArriveTx(
   OpFoldResult size =
       affine::makeComposedFoldedAffineApply(rewriter, loc, sumExpr, mixedSizes);
   Value sizeVal = getValueOrCreateConstantIndexOp(rewriter, loc, size);
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierArriveExpectTxOp>(loc, barrier, sizeVal, zero,
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierArriveExpectTxOp::create(rewriter, loc, barrier, sizeVal, zero,
                                                    Value());
 }
 
 void HopperBuilder::buildTryWaitParity(
     TypedValue<nvgpu::MBarrierGroupType> barrier) {
   Type i1 = rewriter.getI1Type();
-  Value parity = rewriter.create<LLVM::ConstantOp>(loc, i1, 0);
+  Value parity = LLVM::ConstantOp::create(rewriter, loc, i1, 0);
   // 10M is an arbitrary, not too small or too big number to specify the number
   // of ticks before retry.
   // TODO: hoist this in a default dialect constant.
   Value ticksBeforeRetry =
-      rewriter.create<arith::ConstantIndexOp>(loc, 10000000);
-  Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierTryWaitParityOp>(loc, barrier, parity,
+      arith::ConstantIndexOp::create(rewriter, loc, 10000000);
+  Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  nvgpu::MBarrierTryWaitParityOp::create(rewriter, loc, barrier, parity,
                                                   ticksBeforeRetry, zero);
 }
 
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
index 47a0c7096de95..b8baadb4ef241 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
@@ -109,17 +109,17 @@ static Value buildNumReadElements(OpBuilder &b, Location loc,
   for (auto [pos, sz] : llvm::zip(transferMask->extractPosition,
                                   transferMask->createMaskOp->getOperands())) {
     Value cmp =
-        b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
-                                b.create<arith::ConstantIndexOp>(loc, pos), sz);
+        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt,
+                                arith::ConstantIndexOp::create(b, loc, pos), sz);
     if (!cond) {
       cond = cmp;
       continue;
     }
-    cond = b.create<arith::AndIOp>(loc, cmp, cond);
+    cond = arith::AndIOp::create(b, loc, cmp, cond);
   }
-  return b.create<arith::SelectOp>(
+  return arith::SelectOp::create(b,
       loc, cond, transferMask->createMaskOp->getOperands().back(),
-      b.create<arith::ConstantIndexOp>(loc, 0));
+      arith::ConstantIndexOp::create(b, loc, 0));
 }
 
 /// Return "true" if the conversion to async copy is supported by "async copy".
@@ -251,7 +251,7 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
       int64_t sizeInBytes =
           (dstMemref.getElementTypeBitWidth() * numElements) / 8;
       // bypass_l1 only possible with 16 byte transfer.
-      Value token = rewriter.create<nvgpu::DeviceAsyncCopyOp>(
+      Value token = nvgpu::DeviceAsyncCopyOp::create(rewriter,
           writeOp->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
           /*dst=*/storeBase, /*dstIndices=*/nvgpu::getIndices(writeOp),
           /*src=*/loadBase,
@@ -264,10 +264,10 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
     }
 
     // Create the group and wait for it right after.
-    Value groupToken = rewriter.create<nvgpu::DeviceAsyncCreateGroupOp>(
+    Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(rewriter,
         op->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
         tokens);
-    rewriter.create<nvgpu::DeviceAsyncWaitOp>(op->getLoc(), groupToken,
+    nvgpu::DeviceAsyncWaitOp::create(rewriter, op->getLoc(), groupToken,
                                               nullptr);
     // Clean up old stores.
     for (Operation *writeOp : group)
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
index 44e7fa961da12..2d2ff94a25f92 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
@@ -74,27 +74,27 @@ static Value permuteVectorOffset(OpBuilder &b, Location loc,
   int64_t mask = (1LL << (m - n)) - 1;
   if (permuteEveryN > 1)
     mask = mask << llvm::Log2_64(permuteEveryN);
-  Value srcBits = b.create<arith::ConstantIndexOp>(loc, mask);
-  srcBits = b.create<arith::AndIOp>(loc, src, srcBits);
+  Value srcBits = arith::ConstantIndexOp::create(b, loc, mask);
+  srcBits = arith::AndIOp::create(b, loc, src, srcBits);
 
   // Use the src bits to permute the target bits b[N:M] containing the
   // vector offset.
   if (permuteEveryN > 1) {
     int64_t shlBits = n - llvm::Log2_64(permuteEveryN);
     if (shlBits > 0) {
-      Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, shlBits);
+      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, shlBits);
       srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
     } else if (shlBits < 0) {
-      Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, -1 * shlBits);
+      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, -1 * shlBits);
       srcBits = b.createOrFold<arith::ShRUIOp>(loc, srcBits, finalShiftVal);
     }
   } else {
-    Value finalShiftVal = b.create<arith::ConstantIndexOp>(loc, n);
+    Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, n);
     srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
   }
 
   Value permutedVectorIdx =
-      b.create<arith::XOrIOp>(loc, indices[tgtDim], srcBits);
+      arith::XOrIOp::create(b, loc, indices[tgtDim], srcBits);
   return permutedVectorIdx;
 }
 
diff --git a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
index 793db73575b4f..a378b09c27c41 100644
--- a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
@@ -72,7 +72,7 @@ Value getScalarOrTensorConstant(OpBuilder &builder, Location loc, Value scalar,
 
   // Create tensor splat
   auto tensorConstant =
-      builder.create<tensor::SplatOp>(loc, scalar, referenceShape);
+      tensor::SplatOp::create(builder, loc, scalar, referenceShape);
   return tensorConstant;
 }
 
@@ -94,21 +94,21 @@ std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
   // Get unranked input shape and total size
   auto *context = builder.getContext();
   auto shapeType = shape::getExtentTensorType(context);
-  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
-  Value inputSize = builder.create<shape::NumElementsOp>(
+  auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
+  Value inputSize = shape::NumElementsOp::create(builder,
       loc, builder.getIndexType(), inputShape);
 
   // Turn input size into 1D tensor
   auto flatShapeType = shape::getExtentTensorType(context, 1);
   auto flatInputShape =
-      builder.create<tensor::FromElementsOp>(loc, flatShapeType, inputSize);
+      tensor::FromElementsOp::create(builder, loc, flatShapeType, inputSize);
 
   // Reshape input tensor into 1D
   auto inputType = cast<UnrankedTensorType>(input.getType());
   auto elementType = inputType.getElementType();
   auto flatInputType =
       RankedTensorType::get({ShapedType::kDynamic}, elementType);
-  auto flatInput = builder.create<tensor::ReshapeOp>(loc, flatInputType, input,
+  auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
                                                      flatInputShape);
   return std::make_pair(flatInput, inputShape);
 }
@@ -142,30 +142,30 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto *context = builder.getContext();
   auto indexType = builder.getIndexType();
   auto shapeType = shape::getExtentTensorType(context);
-  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
+  auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
 
   // Get shape and sizes on left and right of axis
-  auto axisValue = builder.create<arith::ConstantIndexOp>(loc, axis);
-  auto axisNextValue = builder.create<arith::ConstantIndexOp>(loc, axis + 1);
+  auto axisValue = arith::ConstantIndexOp::create(builder, loc, axis);
+  auto axisNextValue = arith::ConstantIndexOp::create(builder, loc, axis + 1);
   auto shapeLeft =
       builder
           .create<shape::SplitAtOp>(loc, TypeRange{shapeType, shapeType},
                                     inputShape, axisValue)
           .getResult(0);
   auto sizeLeft =
-      builder.create<shape::NumElementsOp>(loc, indexType, shapeLeft);
+      shape::NumElementsOp::create(builder, loc, indexType, shapeLeft);
   auto shapeRight =
       builder
           .create<shape::SplitAtOp>(loc, TypeRange{shapeType, shapeType},
                                     inputShape, axisNextValue)
           .getResult(1);
   auto sizeRight =
-      builder.create<shape::NumElementsOp>(loc, indexType, shapeRight);
+      shape::NumElementsOp::create(builder, loc, indexType, shapeRight);
 
   // Compute flat input shape as a 3-element 1D tensor
-  auto axisSizeValue = builder.create<arith::ConstantIndexOp>(loc, axisSize);
+  auto axisSizeValue = arith::ConstantIndexOp::create(builder, loc, axisSize);
   auto flatShapeType = shape::getExtentTensorType(context, 3);
-  auto flatInputShape = builder.create<tensor::FromElementsOp>(
+  auto flatInputShape = tensor::FromElementsOp::create(builder,
       loc, flatShapeType, ValueRange{sizeLeft, axisSizeValue, sizeRight});
 
   // Reshape input to 3D tensor
@@ -173,7 +173,7 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto elementType = inputType.getElementType();
   auto flatInputType = RankedTensorType::get(
       {ShapedType::kDynamic, axisSize, ShapedType::kDynamic}, elementType);
-  auto flatInput = builder.create<tensor::ReshapeOp>(loc, flatInputType, input,
+  auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
                                                      flatInputShape);
 
   return std::make_pair(flatInput, inputShape);
@@ -192,7 +192,7 @@ Value restoreUnrankedTensorShape(OpBuilder &builder, Location loc, Value input,
   auto inputType = cast<RankedTensorType>(input.getType());
   auto elementType = inputType.getElementType();
   auto unrankedType = UnrankedTensorType::get(elementType);
-  return builder.create<tensor::ReshapeOp>(loc, unrankedType, input,
+  return tensor::ReshapeOp::create(builder, loc, unrankedType, input,
                                            inputShape);
 }
 
@@ -215,7 +215,7 @@ Value materializePerChannelScales(OpBuilder &builder, Location loc,
   auto tensorType =
       RankedTensorType::get({(int64_t)scales.size()}, expressedType);
   auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, scalesAttr);
 }
 
 // Create a tensor constant containing all zero points in a per-channel
@@ -239,7 +239,7 @@ Value materializePerChannelZeroPoints(
   auto tensorType =
       RankedTensorType::get({(int64_t)zeroPoints.size()}, storageType);
   auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, zeroPointsAttr);
 }
 
 // Create a tensor constant containing all scales in a sub-channel quantized
@@ -263,7 +263,7 @@ Value materializeSubChannelScales(
   auto tensorType =
       RankedTensorType::get(scales.getType().getShape(), expressedType);
   auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, scalesAttr);
 }
 
 // Create a tensor constant containing all zero points in a sub-channel
@@ -287,7 +287,7 @@ Value materializeSubChannelZeroPoints(
   auto tensorType =
       RankedTensorType::get(zeroPoints.getType().getShape(), storageType);
   auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
-  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+  return arith::ConstantOp::create(builder, loc, tensorType, zeroPointsAttr);
 }
 
 // Clamp the given scalar or tensor input using the storage bounds encoded in
@@ -314,9 +314,9 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
   // Materialize bounds
   auto inputType = input.getType();
   auto storageType = quantizedType.getStorageType();
-  auto storageMinScalar = builder.create<arith::ConstantIntOp>(
+  auto storageMinScalar = arith::ConstantIntOp::create(builder,
       loc, storageType, quantizedType.getStorageTypeMin());
-  auto storageMaxScalar = builder.create<arith::ConstantIntOp>(
+  auto storageMaxScalar = arith::ConstantIntOp::create(builder,
       loc, storageType, quantizedType.getStorageTypeMax());
   auto storageMin = getScalarOrTensorConstant(builder, loc, storageMinScalar,
                                               inputType, inputShape);
@@ -325,11 +325,11 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
 
   // Clamp
   if (quantizedType.isSigned()) {
-    input = builder.create<arith::MaxSIOp>(loc, input, storageMin);
-    input = builder.create<arith::MinSIOp>(loc, input, storageMax);
+    input = arith::MaxSIOp::create(builder, loc, input, storageMin);
+    input = arith::MinSIOp::create(builder, loc, input, storageMax);
   } else {
-    input = builder.create<arith::MaxUIOp>(loc, input, storageMin);
-    input = builder.create<arith::MinUIOp>(loc, input, storageMax);
+    input = arith::MaxUIOp::create(builder, loc, input, storageMin);
+    input = arith::MinUIOp::create(builder, loc, input, storageMax);
   }
   return input;
 }
@@ -338,16 +338,16 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
 Value convertFloatToInteger(OpBuilder &builder, Location loc, Value input,
                             Type resultType, bool isSigned) {
   if (isSigned)
-    return builder.create<arith::FPToSIOp>(loc, resultType, input);
-  return builder.create<arith::FPToUIOp>(loc, resultType, input);
+    return arith::FPToSIOp::create(builder, loc, resultType, input);
+  return arith::FPToUIOp::create(builder, loc, resultType, input);
 }
 
 // Emit op 'arith.sitofp' or 'arith.uitofp'.
 Value convertIntegerToFloat(OpBuilder &builder, Location loc, Value input,
                             Type resultType, bool isSigned) {
   if (isSigned)
-    return builder.create<arith::SIToFPOp>(loc, resultType, input);
-  return builder.create<arith::UIToFPOp>(loc, resultType, input);
+    return arith::SIToFPOp::create(builder, loc, resultType, input);
+  return arith::UIToFPOp::create(builder, loc, resultType, input);
 }
 
 // Quantize a scalar or ranked tensor value. The stored value is clamped using
@@ -362,7 +362,7 @@ Value quantizeValue(OpBuilder &builder, Location loc, Value input,
   scale = getScalarOrTensorConstant(builder, loc, scale, inputType, inputShape);
 
   // Scale input
-  auto scaledValue = builder.create<arith::DivFOp>(loc, input, scale);
+  auto scaledValue = arith::DivFOp::create(builder, loc, input, scale);
 
   // Skip unnecessary computations if no zero point is given
   Value storedValueFloat = scaledValue;
@@ -377,7 +377,7 @@ Value quantizeValue(OpBuilder &builder, Location loc, Value input,
 
     // Add zero point to stored value
     storedValueFloat =
-        builder.create<arith::AddFOp>(loc, scaledValue, zeroPoint);
+        arith::AddFOp::create(builder, loc, scaledValue, zeroPoint);
   }
 
   // Convert stored value to storage type
@@ -418,11 +418,11 @@ Value dequantizeValue(OpBuilder &builder, Location loc, Value input,
                                       quantizedType.isSigned());
 
     // Subtract zero point to stored value
-    result = builder.create<arith::SubFOp>(loc, result, zeroPoint);
+    result = arith::SubFOp::create(builder, loc, result, zeroPoint);
   }
 
   // Multiply by scale
-  result = builder.create<arith::MulFOp>(loc, result, scale);
+  result = arith::MulFOp::create(builder, loc, result, scale);
   return result;
 }
 
@@ -477,11 +477,11 @@ Value convertPerLayerRanked(OpBuilder &builder, Location loc, Operation *op,
   auto storageType = quantizedType.getStorageType();
   auto scaleAttr =
       builder.getFloatAttr(expressedType, quantizedType.getScale());
-  auto scale = builder.create<arith::ConstantOp>(loc, expressedType, scaleAttr);
+  auto scale = arith::ConstantOp::create(builder, loc, expressedType, scaleAttr);
   auto zeroPointAttr =
       builder.getIntegerAttr(storageType, quantizedType.getZeroPoint());
   auto zeroPoint =
-      builder.create<arith::ConstantOp>(loc, storageType, zeroPointAttr);
+      arith::ConstantOp::create(builder, loc, storageType, zeroPointAttr);
 
   auto inputShape = getScalarOrTensorShape(builder, loc, input);
   return convertRanked(builder, loc, op, input, inputShape, scale, zeroPoint,
@@ -546,7 +546,7 @@ Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
                          ? quantizedType.getStorageType()
                          : quantizedType.getExpressedType();
   auto initShape = tensor::getMixedSizes(builder, loc, input);
-  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+  Value init = tensor::EmptyOp::create(builder, loc, initShape, elementType);
 
   SmallVector<utils::IteratorType> iteratorTypes(inputRank,
                                                  utils::IteratorType::parallel);
@@ -572,7 +572,7 @@ Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
                               convertRanked(builder, loc, op, input, {}, scale,
                                             zeroPoint, quantizedType);
 
-                          builder.create<linalg::YieldOp>(loc, result);
+                          linalg::YieldOp::create(builder, loc, result);
                         })
                     .getResult(0);
 
@@ -642,7 +642,7 @@ Value convertSubChannel(OpBuilder &builder, Location loc, Operation *op,
                          ? quantizedType.getStorageType()
                          : quantizedType.getExpressedType();
   auto initShape = tensor::getMixedSizes(builder, loc, input);
-  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+  Value init = tensor::EmptyOp::create(builder, loc, initShape, elementType);
 
   SmallVector<utils::IteratorType> iteratorTypes(inputRank,
                                                  utils::IteratorType::parallel);
@@ -675,7 +675,7 @@ Value convertSubChannel(OpBuilder &builder, Location loc, Operation *op,
                               convertRanked(builder, loc, op, input, {}, scale,
                                             zeroPoint, quantizedType);
 
-                          builder.create<linalg::YieldOp>(loc, result);
+                          linalg::YieldOp::create(builder, loc, result);
                         })
                     .getResult(0);
 
@@ -729,7 +729,7 @@ struct DequantizeCastOpConversion
     // Convert quantized input to storage type
     auto storageScalarOrTensorType =
         getScalarOrTensorType(quantizedType.getStorageType(), input.getType());
-    input = rewriter.create<quant::StorageCastOp>(
+    input = quant::StorageCastOp::create(rewriter,
         loc, storageScalarOrTensorType, input);
 
     auto result = convertQuantized(rewriter, loc, op, input, quantizedType);
diff --git a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
index 4009faa21576d..f5e0aef365ff7 100644
--- a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
@@ -46,7 +46,7 @@ class QuantizedTypeConverter : public TypeConverter {
 
   static Value materializeConversion(OpBuilder &builder, Type type,
                                      ValueRange inputs, Location loc) {
-    return builder.create<quant::StorageCastOp>(loc, type,
+    return quant::StorageCastOp::create(builder, loc, type,
                                                 llvm::getSingleElement(inputs));
   }
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 5a3bd984530db..82fa50a6ac6e0 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -85,7 +85,7 @@ void SCFDialect::initialize() {
 
 /// Default callback for IfOp builders. Inserts a yield without arguments.
 void mlir::scf::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<scf::YieldOp>(loc);
+  scf::YieldOp::create(builder, loc);
 }
 
 /// Verifies that the first block of the given `region` is terminated by a
@@ -241,12 +241,12 @@ struct MultiBlockExecuteInliner : public OpRewritePattern<ExecuteRegionOp> {
     Block *postBlock = rewriter.splitBlock(prevBlock, op->getIterator());
     rewriter.setInsertionPointToEnd(prevBlock);
 
-    rewriter.create<cf::BranchOp>(op.getLoc(), &op.getRegion().front());
+    cf::BranchOp::create(rewriter, op.getLoc(), &op.getRegion().front());
 
     for (Block &blk : op.getRegion()) {
       if (YieldOp yieldOp = dyn_cast<YieldOp>(blk.getTerminator())) {
         rewriter.setInsertionPoint(yieldOp);
-        rewriter.create<cf::BranchOp>(yieldOp.getLoc(), postBlock,
+        cf::BranchOp::create(rewriter, yieldOp.getLoc(), postBlock,
                                       yieldOp.getResults());
         rewriter.eraseOp(yieldOp);
       }
@@ -557,7 +557,7 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  scf::ForOp newLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp newLoop = scf::ForOp::create(rewriter,
       getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
       [](OpBuilder &, Location, Value, ValueRange) {});
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
@@ -673,7 +673,7 @@ void mlir::scf::promote(RewriterBase &rewriter, scf::ForallOp forallOp) {
     Value dst = parallelInsertSliceOp.getDest();
     Value src = parallelInsertSliceOp.getSource();
     if (llvm::isa<TensorType>(src.getType())) {
-      results.push_back(rewriter.create<tensor::InsertSliceOp>(
+      results.push_back(tensor::InsertSliceOp::create(rewriter,
           forallOp.getLoc(), dst.getType(), src, dst,
           parallelInsertSliceOp.getOffsets(), parallelInsertSliceOp.getSizes(),
           parallelInsertSliceOp.getStrides(),
@@ -722,7 +722,7 @@ LoopNest mlir::scf::buildLoopNest(
   ValueRange currentIterArgs = iterArgs;
   Location currentLoc = loc;
   for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
-    auto loop = builder.create<scf::ForOp>(
+    auto loop = scf::ForOp::create(builder,
         currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
             ValueRange args) {
@@ -742,7 +742,7 @@ LoopNest mlir::scf::buildLoopNest(
   // For all loops but the innermost, yield the results of the nested loop.
   for (unsigned i = 0, e = loops.size() - 1; i < e; ++i) {
     builder.setInsertionPointToEnd(loops[i].getBody());
-    builder.create<scf::YieldOp>(loc, loops[i + 1].getResults());
+    scf::YieldOp::create(builder, loc, loops[i + 1].getResults());
   }
 
   // In the body of the innermost loop, call the body building function if any
@@ -756,7 +756,7 @@ LoopNest mlir::scf::buildLoopNest(
          "loop nest body must return as many values as loop has iteration "
          "arguments");
   builder.setInsertionPointToEnd(loops.back().getBody());
-  builder.create<scf::YieldOp>(loc, results);
+  scf::YieldOp::create(builder, loc, results);
 
   // Return the loops.
   ValueVector nestResults;
@@ -801,7 +801,7 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   }
 
   // 2. Create the new forOp shell.
-  scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+  scf::ForOp newForOp = scf::ForOp::create(rewriter,
       forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
       forOp.getStep(), newIterOperands);
   newForOp->setAttrs(forOp->getAttrs());
@@ -831,7 +831,7 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
                          clonedYieldOp.getOperand(yieldIdx));
   SmallVector<Value> newYieldOperands = clonedYieldOp.getOperands();
   newYieldOperands[yieldIdx] = castOut;
-  rewriter.create<scf::YieldOp>(newForOp.getLoc(), newYieldOperands);
+  scf::YieldOp::create(rewriter, newForOp.getLoc(), newYieldOperands);
   rewriter.eraseOp(clonedYieldOp);
 
   // 6. Inject an outgoing cast op after the forOp.
@@ -926,7 +926,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     if (!canonicalize)
       return failure();
 
-    scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+    scf::ForOp newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), newIterArgs);
     newForOp->setAttrs(forOp->getAttrs());
@@ -970,7 +970,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
       for (unsigned idx = 0, e = keepMask.size(); idx < e; ++idx)
         if (keepMask[idx])
           filteredOperands.push_back(mergedTerminator.getOperand(idx));
-      rewriter.create<scf::YieldOp>(mergedTerminator.getLoc(),
+      scf::YieldOp::create(rewriter, mergedTerminator.getLoc(),
                                     filteredOperands);
     };
 
@@ -1111,7 +1111,7 @@ struct ForOpTensorCastFolder : public OpRewritePattern<ForOp> {
           op, replaceAndCastForOpIterArg(
                   rewriter, op, iterOpOperand, incomingCast.getSource(),
                   [](OpBuilder &b, Location loc, Type type, Value source) {
-                    return b.create<tensor::CastOp>(loc, type, source);
+                    return tensor::CastOp::create(b, loc, type, source);
                   }));
       return success();
     }
@@ -1685,7 +1685,7 @@ struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
 
     // Step 3. Create a new scf.forall op with the new shared_outs' operands
     //         fetched earlier
-    auto newForallOp = rewriter.create<scf::ForallOp>(
+    auto newForallOp = scf::ForallOp::create(rewriter,
         forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
         forallOp.getMapping(),
@@ -1782,7 +1782,7 @@ struct ForallOpSingleOrZeroIterationDimsFolder
 
     // Replace the loop by a lower-dimensional loop.
     ForallOp newOp;
-    newOp = rewriter.create<ForallOp>(loc, newMixedLowerBounds,
+    newOp = ForallOp::create(rewriter, loc, newMixedLowerBounds,
                                       newMixedUpperBounds, newMixedSteps,
                                       op.getOutputs(), std::nullopt, nullptr);
     newOp.getBodyRegion().getBlocks().clear();
@@ -1867,7 +1867,7 @@ struct FoldTensorCastOfOutputIntoForallOp
 
     // Create new loop.
     Location loc = forallOp.getLoc();
-    auto newForallOp = rewriter.create<ForallOp>(
+    auto newForallOp = ForallOp::create(rewriter,
         loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
         forallOp.getMixedStep(), newOutputTensors, forallOp.getMapping(),
         [&](OpBuilder nestedBuilder, Location nestedLoc, ValueRange bbArgs) {
@@ -1875,7 +1875,7 @@ struct FoldTensorCastOfOutputIntoForallOp
               llvm::to_vector(bbArgs.take_back(forallOp->getNumResults()));
           for (auto [index, cast] : tensorCastProducers) {
             Value &oldTypeBBArg = castBlockArgs[index];
-            oldTypeBBArg = nestedBuilder.create<tensor::CastOp>(
+            oldTypeBBArg = tensor::CastOp::create(nestedBuilder,
                 nestedLoc, cast.dstType, oldTypeBBArg);
           }
 
@@ -1902,7 +1902,7 @@ struct FoldTensorCastOfOutputIntoForallOp
     SmallVector<Value> castResults = newForallOp.getResults();
     for (auto &item : tensorCastProducers) {
       Value &oldTypeResult = castResults[item.first];
-      oldTypeResult = rewriter.create<tensor::CastOp>(loc, item.second.dstType,
+      oldTypeResult = tensor::CastOp::create(rewriter, loc, item.second.dstType,
                                                       oldTypeResult);
     }
     rewriter.replaceOp(forallOp, castResults);
@@ -2309,7 +2309,7 @@ struct RemoveUnusedResults : public OpRewritePattern<IfOp> {
 
     // Create a replacement operation with empty then and else regions.
     auto newOp =
-        rewriter.create<IfOp>(op.getLoc(), newTypes, op.getCondition());
+        IfOp::create(rewriter, op.getLoc(), newTypes, op.getCondition());
     rewriter.createBlock(&newOp.getThenRegion());
     rewriter.createBlock(&newOp.getElseRegion());
 
@@ -2372,7 +2372,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
     if (nonHoistable.size() == op->getNumResults())
       return failure();
 
-    IfOp replacement = rewriter.create<IfOp>(op.getLoc(), nonHoistable, cond,
+    IfOp replacement = IfOp::create(rewriter, op.getLoc(), nonHoistable, cond,
                                              /*withElseRegion=*/false);
     if (replacement.thenBlock())
       rewriter.eraseBlock(replacement.thenBlock());
@@ -2398,7 +2398,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       } else if (trueVal == falseVal)
         results[it.index()] = trueVal;
       else
-        results[it.index()] = rewriter.create<arith::SelectOp>(
+        results[it.index()] = arith::SelectOp::create(rewriter,
             op.getLoc(), cond, trueVal, falseVal);
     }
 
@@ -2450,7 +2450,7 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantTrue)
-          constantTrue = rewriter.create<arith::ConstantOp>(
+          constantTrue = arith::ConstantOp::create(rewriter,
               op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
 
         rewriter.modifyOpInPlace(use.getOwner(),
@@ -2460,7 +2460,7 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantFalse)
-          constantFalse = rewriter.create<arith::ConstantOp>(
+          constantFalse = arith::ConstantOp::create(rewriter,
               op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
 
         rewriter.modifyOpInPlace(use.getOwner(),
@@ -2547,7 +2547,7 @@ struct ReplaceIfYieldWithConditionOrValue : public OpRewritePattern<IfOp> {
       if (!trueVal && falseVal) {
         if (!opResult.use_empty()) {
           Dialect *constDialect = trueResult.getDefiningOp()->getDialect();
-          Value notCond = rewriter.create<arith::XOrIOp>(
+          Value notCond = arith::XOrIOp::create(rewriter,
               op.getLoc(), op.getCondition(),
               constDialect
                   ->materializeConstant(rewriter,
@@ -2661,7 +2661,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
     SmallVector<Type> mergedTypes(prevIf.getResultTypes());
     llvm::append_range(mergedTypes, nextIf.getResultTypes());
 
-    IfOp combinedIf = rewriter.create<IfOp>(
+    IfOp combinedIf = IfOp::create(rewriter,
         nextIf.getLoc(), mergedTypes, prevIf.getCondition(), /*hasElse=*/false);
     rewriter.eraseBlock(&combinedIf.getThenRegion().back());
 
@@ -2677,7 +2677,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
 
       SmallVector<Value> mergedYields(thenYield.getOperands());
       llvm::append_range(mergedYields, thenYield2.getOperands());
-      rewriter.create<YieldOp>(thenYield2.getLoc(), mergedYields);
+      YieldOp::create(rewriter, thenYield2.getLoc(), mergedYields);
       rewriter.eraseOp(thenYield);
       rewriter.eraseOp(thenYield2);
     }
@@ -2701,7 +2701,7 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
         SmallVector<Value> mergedElseYields(elseYield.getOperands());
         llvm::append_range(mergedElseYields, elseYield2.getOperands());
 
-        rewriter.create<YieldOp>(elseYield2.getLoc(), mergedElseYields);
+        YieldOp::create(rewriter, elseYield2.getLoc(), mergedElseYields);
         rewriter.eraseOp(elseYield);
         rewriter.eraseOp(elseYield2);
       }
@@ -2823,9 +2823,9 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     }
 
     Location loc = op.getLoc();
-    Value newCondition = rewriter.create<arith::AndIOp>(
+    Value newCondition = arith::AndIOp::create(rewriter,
         loc, op.getCondition(), nestedIf.getCondition());
-    auto newIf = rewriter.create<IfOp>(loc, op.getResultTypes(), newCondition);
+    auto newIf = IfOp::create(rewriter, loc, op.getResultTypes(), newCondition);
     Block *newIfBlock = rewriter.createBlock(&newIf.getThenRegion());
 
     SmallVector<Value> results;
@@ -2833,7 +2833,7 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     rewriter.setInsertionPoint(newIf);
 
     for (auto idx : elseYieldsToUpgradeToSelect)
-      results[idx] = rewriter.create<arith::SelectOp>(
+      results[idx] = arith::SelectOp::create(rewriter,
           op.getLoc(), op.getCondition(), thenYield[idx], elseYield[idx]);
 
     rewriter.mergeBlocks(nestedIf.thenBlock(), newIfBlock);
@@ -2842,7 +2842,7 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     if (!elseYield.empty()) {
       rewriter.createBlock(&newIf.getElseRegion());
       rewriter.setInsertionPointToEnd(newIf.elseBlock());
-      rewriter.create<YieldOp>(loc, elseYield);
+      YieldOp::create(rewriter, loc, elseYield);
     }
     rewriter.replaceOp(op, results);
     return success();
@@ -3160,7 +3160,7 @@ struct ParallelOpSingleOrZeroIterationDimsFolder
     }
     // Replace the parallel loop by lower-dimensional parallel loop.
     auto newOp =
-        rewriter.create<ParallelOp>(op.getLoc(), newLowerBounds, newUpperBounds,
+        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds, newUpperBounds,
                                     newSteps, op.getInitVals(), nullptr);
     // Erase the empty block that was inserted by the builder.
     rewriter.eraseBlock(newOp.getBody());
@@ -3541,7 +3541,7 @@ struct WhileConditionTruth : public OpRewritePattern<WhileOp> {
       if (std::get<0>(yieldedAndBlockArgs) == term.getCondition()) {
         if (!std::get<1>(yieldedAndBlockArgs).use_empty()) {
           if (!constantTrue)
-            constantTrue = rewriter.create<arith::ConstantOp>(
+            constantTrue = arith::ConstantOp::create(rewriter,
                 op.getLoc(), term.getCondition().getType(),
                 rewriter.getBoolAttr(true));
 
@@ -3685,7 +3685,7 @@ struct RemoveLoopInvariantArgsFromBeforeBlock
     }
 
     auto newWhile =
-        rewriter.create<WhileOp>(op.getLoc(), op.getResultTypes(), newInitArgs);
+        WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(), newInitArgs);
 
     Block &newBeforeBlock = *rewriter.createBlock(
         &newWhile.getBefore(), /*insertPt*/ {},
@@ -3807,7 +3807,7 @@ struct RemoveLoopInvariantValueYielded : public OpRewritePattern<WhileOp> {
                                                newCondOpArgs);
     }
 
-    auto newWhile = rewriter.create<WhileOp>(op.getLoc(), newAfterBlockType,
+    auto newWhile = WhileOp::create(rewriter, op.getLoc(), newAfterBlockType,
                                              op.getOperands());
 
     Block &newAfterBlock =
@@ -3914,7 +3914,7 @@ struct WhileUnusedResult : public OpRewritePattern<WhileOp> {
     }
 
     auto newWhile =
-        rewriter.create<WhileOp>(op.getLoc(), newResultTypes, op.getInits());
+        WhileOp::create(rewriter, op.getLoc(), newResultTypes, op.getInits());
 
     Block &newAfterBlock = *rewriter.createBlock(
         &newWhile.getAfter(), /*insertPt*/ {}, newResultTypes, newArgLocs);
@@ -4043,7 +4043,7 @@ struct WhileRemoveUnusedArgs : public OpRewritePattern<WhileOp> {
 
     Location loc = op.getLoc();
     auto newWhileOp =
-        rewriter.create<WhileOp>(loc, op.getResultTypes(), newInits,
+        WhileOp::create(rewriter, loc, op.getResultTypes(), newInits,
                                  /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
@@ -4091,7 +4091,7 @@ struct WhileRemoveDuplicatedResults : public OpRewritePattern<WhileOp> {
     ValueRange argsRange(newArgs);
 
     Location loc = op.getLoc();
-    auto newWhileOp = rewriter.create<scf::WhileOp>(
+    auto newWhileOp = scf::WhileOp::create(rewriter,
         loc, argsRange.getTypes(), op.getInits(), /*beforeBody*/ nullptr,
         /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
@@ -4187,7 +4187,7 @@ struct WhileOpAlignBeforeArgs : public OpRewritePattern<WhileOp> {
     for (auto &&[i, j] : llvm::enumerate(*mapping))
       newResultTypes[j] = loop.getResult(i).getType();
 
-    auto newLoop = rewriter.create<WhileOp>(
+    auto newLoop = WhileOp::create(rewriter,
         loop.getLoc(), newResultTypes, loop.getInits(),
         /*beforeBuilder=*/nullptr, /*afterBuilder=*/nullptr);
     auto newBefore = newLoop.getBeforeBody();
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index 57c27231f2144..e9943a3489d46 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -163,7 +163,7 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
   OpBuilder::InsertionGuard g(b);
   b.setInsertionPoint(op);
   scf::ExecuteRegionOp executeRegionOp =
-      b.create<scf::ExecuteRegionOp>(op->getLoc(), op->getResultTypes());
+      scf::ExecuteRegionOp::create(b, op->getLoc(), op->getResultTypes());
   {
     OpBuilder::InsertionGuard g(b);
     b.setInsertionPointToStart(&executeRegionOp.getRegion().emplaceBlock());
@@ -172,7 +172,7 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
     assert(clonedRegion.empty() && "expected empty region");
     b.inlineRegionBefore(op->getRegions().front(), clonedRegion,
                          clonedRegion.end());
-    b.create<scf::YieldOp>(op->getLoc(), clonedOp->getResults());
+    scf::YieldOp::create(b, op->getLoc(), clonedOp->getResults());
   }
   b.replaceOp(op, executeRegionOp.getResults());
   return executeRegionOp;
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index d36d91249ed36..0214ce3090656 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -42,7 +42,7 @@ static Value castBuffer(OpBuilder &b, Value buffer, Type type) {
   // iter_arg's layout map must be changed (see uses of `castBuffer`).
   assert(memref::CastOp::areCastCompatible(buffer.getType(), type) &&
          "scf.while op bufferization: cast incompatible");
-  return b.create<memref::CastOp>(buffer.getLoc(), type, buffer).getResult();
+  return memref::CastOp::create(b, buffer.getLoc(), type, buffer).getResult();
 }
 
 /// Helper function for loop bufferization. Return "true" if the given value
@@ -191,7 +191,7 @@ struct ExecuteRegionOpInterface
 
     // Create new op and move over region.
     auto newOp =
-        rewriter.create<scf::ExecuteRegionOp>(op->getLoc(), newResultTypes);
+        scf::ExecuteRegionOp::create(rewriter, op->getLoc(), newResultTypes);
     newOp.getRegion().takeBody(executeRegionOp.getRegion());
 
     // Bufferize every block.
@@ -205,7 +205,7 @@ struct ExecuteRegionOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(executeRegionOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
+        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
             executeRegionOp.getLoc(), it.value(),
             newOp->getResult(it.index())));
       } else {
@@ -261,7 +261,7 @@ struct IfOpInterface
     // Create new op.
     rewriter.setInsertionPoint(ifOp);
     auto newIfOp =
-        rewriter.create<scf::IfOp>(ifOp.getLoc(), newTypes, ifOp.getCondition(),
+        scf::IfOp::create(rewriter, ifOp.getLoc(), newTypes, ifOp.getCondition(),
                                    /*withElseRegion=*/true);
 
     // Move over then/else blocks.
@@ -374,7 +374,7 @@ struct IndexSwitchOpInterface
 
     // Create new op.
     rewriter.setInsertionPoint(switchOp);
-    auto newSwitchOp = rewriter.create<scf::IndexSwitchOp>(
+    auto newSwitchOp = scf::IndexSwitchOp::create(rewriter,
         switchOp.getLoc(), newTypes, switchOp.getArg(), switchOp.getCases(),
         switchOp.getCases().size());
 
@@ -769,7 +769,7 @@ struct ForOpInterface
     }
 
     // Construct a new scf.for op with memref instead of tensor values.
-    auto newForOp = rewriter.create<scf::ForOp>(
+    auto newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), castedInitArgs);
     newForOp->setAttrs(forOp->getAttrs());
@@ -1005,7 +1005,7 @@ struct WhileOpInterface
     // Construct a new scf.while op with memref instead of tensor values.
     ValueRange argsRangeBefore(castedInitArgs);
     TypeRange argsTypesBefore(argsRangeBefore);
-    auto newWhileOp = rewriter.create<scf::WhileOp>(
+    auto newWhileOp = scf::WhileOp::create(rewriter,
         whileOp.getLoc(), argsTypesAfter, castedInitArgs);
 
     // Add before/after regions to the new op.
@@ -1265,7 +1265,7 @@ struct ForallOpInterface
              forallOp.getBody()->getArguments().drop_front(rank), buffers)) {
       BlockArgument bbArg = std::get<0>(it);
       Value buffer = std::get<1>(it);
-      Value bufferAsTensor = rewriter.create<ToTensorOp>(
+      Value bufferAsTensor = ToTensorOp::create(rewriter,
           forallOp.getLoc(), bbArg.getType(), buffer);
       bbArg.replaceAllUsesWith(bufferAsTensor);
     }
@@ -1274,7 +1274,7 @@ struct ForallOpInterface
     // introduced terminator.
     rewriter.setInsertionPoint(forallOp);
     ForallOp newForallOp;
-    newForallOp = rewriter.create<ForallOp>(
+    newForallOp = ForallOp::create(rewriter,
         forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
         /*outputs=*/ValueRange(), forallOp.getMapping());
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
index 3e93dc80b18ec..bf4eba3319a4d 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
@@ -50,7 +50,7 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     SmallVector<Value> initArgs;
     initArgs.push_back(forOp.getLowerBound());
     llvm::append_range(initArgs, forOp.getInitArgs());
-    auto whileOp = rewriter.create<WhileOp>(forOp.getLoc(), lcvTypes, initArgs,
+    auto whileOp = WhileOp::create(rewriter, forOp.getLoc(), lcvTypes, initArgs,
                                             forOp->getAttrs());
 
     // 'before' region contains the loop condition and forwarding of iteration
@@ -58,10 +58,10 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     auto *beforeBlock = rewriter.createBlock(
         &whileOp.getBefore(), whileOp.getBefore().begin(), lcvTypes, lcvLocs);
     rewriter.setInsertionPointToStart(whileOp.getBeforeBody());
-    auto cmpOp = rewriter.create<arith::CmpIOp>(
+    auto cmpOp = arith::CmpIOp::create(rewriter,
         whileOp.getLoc(), arith::CmpIPredicate::slt,
         beforeBlock->getArgument(0), forOp.getUpperBound());
-    rewriter.create<scf::ConditionOp>(whileOp.getLoc(), cmpOp.getResult(),
+    scf::ConditionOp::create(rewriter, whileOp.getLoc(), cmpOp.getResult(),
                                       beforeBlock->getArguments());
 
     // Inline for-loop body into an executeRegion operation in the "after"
@@ -72,7 +72,7 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
 
     // Add induction variable incrementation
     rewriter.setInsertionPointToEnd(afterBlock);
-    auto ivIncOp = rewriter.create<arith::AddIOp>(
+    auto ivIncOp = arith::AddIOp::create(rewriter,
         whileOp.getLoc(), afterBlock->getArgument(0), forOp.getStep());
 
     // Rewrite uses of the for-loop block arguments to the new while-loop
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
index 44e6840b03a3d..b95604fa44cb9 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
@@ -40,7 +40,7 @@ LogicalResult mlir::scf::forallToParallelLoop(RewriterBase &rewriter,
   SmallVector<Value> steps = forallOp.getStep(rewriter);
 
   // Create empty scf.parallel op.
-  auto parallelOp = rewriter.create<scf::ParallelOp>(loc, lbs, ubs, steps);
+  auto parallelOp = scf::ParallelOp::create(rewriter, loc, lbs, ubs, steps);
   rewriter.eraseBlock(&parallelOp.getRegion().front());
   rewriter.inlineRegionBefore(forallOp.getRegion(), parallelOp.getRegion(),
                               parallelOp.getRegion().begin());
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index bcecef5e6e0a9..f1ede64d01e0f 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -279,24 +279,24 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
     if (dynamicLoop) {
       Type t = ub.getType();
       // pred = ub > lb + (i * step)
-      Value iv = rewriter.create<arith::AddIOp>(
+      Value iv = arith::AddIOp::create(rewriter,
           loc, lb,
-          rewriter.create<arith::MulIOp>(
+          arith::MulIOp::create(rewriter,
               loc, step,
-              rewriter.create<arith::ConstantOp>(
+              arith::ConstantOp::create(rewriter,
                   loc, rewriter.getIntegerAttr(t, i))));
-      predicates[i] = rewriter.create<arith::CmpIOp>(
+      predicates[i] = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::slt, iv, ub);
     }
 
     // special handling for induction variable as the increment is implicit.
     // iv = lb + i * step
     Type t = lb.getType();
-    Value iv = rewriter.create<arith::AddIOp>(
+    Value iv = arith::AddIOp::create(rewriter,
         loc, lb,
-        rewriter.create<arith::MulIOp>(
+        arith::MulIOp::create(rewriter,
             loc, step,
-            rewriter.create<arith::ConstantOp>(loc,
+            arith::ConstantOp::create(rewriter, loc,
                                                rewriter.getIntegerAttr(t, i))));
     setValueMapping(forOp.getInductionVar(), iv, i);
     for (Operation *op : opOrder) {
@@ -332,7 +332,7 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
             Value prevValue = valueMapping
                 [forOp.getRegionIterArgs()[operand.getOperandNumber()]]
                 [i - stages[op]];
-            source = rewriter.create<arith::SelectOp>(
+            source = arith::SelectOp::create(rewriter,
                 loc, predicates[predicateIdx], source, prevValue);
           }
           setValueMapping(forOp.getRegionIterArgs()[operand.getOperandNumber()],
@@ -444,14 +444,14 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop(
     Type t = ub.getType();
     Location loc = forOp.getLoc();
     // newUb = ub - maxStage * step
-    Value maxStageValue = rewriter.create<arith::ConstantOp>(
+    Value maxStageValue = arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerAttr(t, maxStage));
     Value maxStageByStep =
-        rewriter.create<arith::MulIOp>(loc, step, maxStageValue);
-    newUb = rewriter.create<arith::SubIOp>(loc, ub, maxStageByStep);
+        arith::MulIOp::create(rewriter, loc, step, maxStageValue);
+    newUb = arith::SubIOp::create(rewriter, loc, ub, maxStageByStep);
   }
   auto newForOp =
-      rewriter.create<scf::ForOp>(forOp.getLoc(), forOp.getLowerBound(), newUb,
+      scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(), newUb,
                                   forOp.getStep(), newLoopArg);
   // When there are no iter args, the loop body terminator will be created.
   // Since we always create it below, remove the terminator if it was created.
@@ -483,14 +483,14 @@ LogicalResult LoopPipelinerInternal::createKernel(
     Type t = ub.getType();
     for (unsigned i = 0; i < maxStage; i++) {
       // c = ub - (maxStage - i) * step
-      Value c = rewriter.create<arith::SubIOp>(
+      Value c = arith::SubIOp::create(rewriter,
           loc, ub,
-          rewriter.create<arith::MulIOp>(
+          arith::MulIOp::create(rewriter,
               loc, step,
-              rewriter.create<arith::ConstantOp>(
+              arith::ConstantOp::create(rewriter,
                   loc, rewriter.getIntegerAttr(t, int64_t(maxStage - i)))));
 
-      Value pred = rewriter.create<arith::CmpIOp>(
+      Value pred = arith::CmpIOp::create(rewriter,
           newForOp.getLoc(), arith::CmpIPredicate::slt,
           newForOp.getInductionVar(), c);
       predicates[i] = pred;
@@ -515,12 +515,12 @@ LogicalResult LoopPipelinerInternal::createKernel(
 
         // offset = (maxStage - stages[op]) * step
         Type t = step.getType();
-        Value offset = rewriter.create<arith::MulIOp>(
+        Value offset = arith::MulIOp::create(rewriter,
             forOp.getLoc(), step,
-            rewriter.create<arith::ConstantOp>(
+            arith::ConstantOp::create(rewriter,
                 forOp.getLoc(),
                 rewriter.getIntegerAttr(t, maxStage - stages[op])));
-        Value iv = rewriter.create<arith::AddIOp>(
+        Value iv = arith::AddIOp::create(rewriter,
             forOp.getLoc(), newForOp.getInductionVar(), offset);
         nestedNewOp->setOperand(operand->getOperandNumber(), iv);
         rewriter.setInsertionPointAfter(newOp);
@@ -594,7 +594,7 @@ LogicalResult LoopPipelinerInternal::createKernel(
         auto defStage = stages.find(def);
         if (defStage != stages.end() && defStage->second < maxStage) {
           Value pred = predicates[defStage->second];
-          source = rewriter.create<arith::SelectOp>(
+          source = arith::SelectOp::create(rewriter,
               pred.getLoc(), pred, source,
               newForOp.getBody()
                   ->getArguments()[yieldOperand.getOperandNumber() + 1]);
@@ -638,7 +638,7 @@ LogicalResult LoopPipelinerInternal::createKernel(
                       maxStage - defStage->second + 1);
     }
   }
-  rewriter.create<scf::YieldOp>(forOp.getLoc(), yieldOperands);
+  scf::YieldOp::create(rewriter, forOp.getLoc(), yieldOperands);
   return success();
 }
 
@@ -652,7 +652,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // removed by dead code if not used.
 
   auto createConst = [&](int v) {
-    return rewriter.create<arith::ConstantOp>(loc,
+    return arith::ConstantOp::create(rewriter, loc,
                                               rewriter.getIntegerAttr(t, v));
   };
 
@@ -661,41 +661,41 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // - total_iterations = (range_diff + step + (step < 0 ? 1 : -1)) / step
   Value zero = createConst(0);
   Value one = createConst(1);
-  Value stepLessZero = rewriter.create<arith::CmpIOp>(
+  Value stepLessZero = arith::CmpIOp::create(rewriter,
       loc, arith::CmpIPredicate::slt, step, zero);
   Value stepDecr =
-      rewriter.create<arith::SelectOp>(loc, stepLessZero, one, createConst(-1));
+      arith::SelectOp::create(rewriter, loc, stepLessZero, one, createConst(-1));
 
-  Value rangeDiff = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  Value rangeIncrStep = rewriter.create<arith::AddIOp>(loc, rangeDiff, step);
+  Value rangeDiff = arith::SubIOp::create(rewriter, loc, ub, lb);
+  Value rangeIncrStep = arith::AddIOp::create(rewriter, loc, rangeDiff, step);
   Value rangeDecr =
-      rewriter.create<arith::AddIOp>(loc, rangeIncrStep, stepDecr);
-  Value totalIterations = rewriter.create<arith::DivSIOp>(loc, rangeDecr, step);
+      arith::AddIOp::create(rewriter, loc, rangeIncrStep, stepDecr);
+  Value totalIterations = arith::DivSIOp::create(rewriter, loc, rangeDecr, step);
 
   // If total_iters < max_stage, start the epilogue at zero to match the
   // ramp-up in the prologue.
   // start_iter = max(0, total_iters - max_stage)
-  Value iterI = rewriter.create<arith::SubIOp>(loc, totalIterations,
+  Value iterI = arith::SubIOp::create(rewriter, loc, totalIterations,
                                                createConst(maxStage));
-  iterI = rewriter.create<arith::MaxSIOp>(loc, zero, iterI);
+  iterI = arith::MaxSIOp::create(rewriter, loc, zero, iterI);
 
   // Capture predicates for dynamic loops.
   SmallVector<Value> predicates(maxStage + 1);
 
   for (int64_t i = 1; i <= maxStage; i++) {
     // newLastIter = lb + step * iterI
-    Value newlastIter = rewriter.create<arith::AddIOp>(
-        loc, lb, rewriter.create<arith::MulIOp>(loc, step, iterI));
+    Value newlastIter = arith::AddIOp::create(rewriter,
+        loc, lb, arith::MulIOp::create(rewriter, loc, step, iterI));
 
     setValueMapping(forOp.getInductionVar(), newlastIter, i);
 
     // increment to next iterI
-    iterI = rewriter.create<arith::AddIOp>(loc, iterI, one);
+    iterI = arith::AddIOp::create(rewriter, loc, iterI, one);
 
     if (dynamicLoop) {
       // Disable stages when `i` is greater than total_iters.
       // pred = total_iters >= i
-      predicates[i] = rewriter.create<arith::CmpIOp>(
+      predicates[i] = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sge, totalIterations, createConst(i));
     }
   }
@@ -758,7 +758,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
           unsigned nextVersion = currentVersion + 1;
           Value pred = predicates[currentVersion];
           Value prevValue = valueMapping[mapVal][currentVersion];
-          auto selOp = rewriter.create<arith::SelectOp>(loc, pred, pair.value(),
+          auto selOp = arith::SelectOp::create(rewriter, loc, pred, pair.value(),
                                                         prevValue);
           returnValues[ri] = selOp;
           if (nextVersion <= maxStage)
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index b71ec985fa6a1..b24e276272c2c 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -64,13 +64,13 @@ static void specializeParallelLoopForUnrolling(ParallelOp op) {
   Value cond;
   for (auto bound : llvm::zip(op.getUpperBound(), constantIndices)) {
     Value constant =
-        b.create<arith::ConstantIndexOp>(op.getLoc(), std::get<1>(bound));
-    Value cmp = b.create<arith::CmpIOp>(op.getLoc(), arith::CmpIPredicate::eq,
+        arith::ConstantIndexOp::create(b, op.getLoc(), std::get<1>(bound));
+    Value cmp = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
                                         std::get<0>(bound), constant);
-    cond = cond ? b.create<arith::AndIOp>(op.getLoc(), cond, cmp) : cmp;
+    cond = cond ? arith::AndIOp::create(b, op.getLoc(), cond, cmp) : cmp;
     map.map(std::get<0>(bound), constant);
   }
-  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
   ifOp.getElseBodyBuilder().clone(*op.getOperation());
   op.erase();
@@ -95,11 +95,11 @@ static void specializeForLoopForUnrolling(ForOp op) {
 
   OpBuilder b(op);
   IRMapping map;
-  Value constant = b.create<arith::ConstantIndexOp>(op.getLoc(), minConstant);
-  Value cond = b.create<arith::CmpIOp>(op.getLoc(), arith::CmpIPredicate::eq,
+  Value constant = arith::ConstantIndexOp::create(b, op.getLoc(), minConstant);
+  Value cond = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
                                        bound, constant);
   map.map(bound, constant);
-  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
   ifOp.getElseBodyBuilder().clone(*op.getOperation());
   op.erase();
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index ad1267381c4f2..013c3b69620e2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -190,7 +190,7 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
 
   IRRewriter b(builder);
   b.setInsertionPoint(secondPloop);
-  auto newSecondPloop = b.create<ParallelOp>(
+  auto newSecondPloop = ParallelOp::create(b,
       secondPloop.getLoc(), secondPloop.getLowerBound(),
       secondPloop.getUpperBound(), secondPloop.getStep(), newInitVars);
 
@@ -212,7 +212,7 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
     SmallVector<Value> newReduceArgs(reduceArgs1.begin(), reduceArgs1.end());
     newReduceArgs.append(reduceArgs2.begin(), reduceArgs2.end());
 
-    auto newReduceOp = b.create<scf::ReduceOp>(term2.getLoc(), newReduceArgs);
+    auto newReduceOp = scf::ReduceOp::create(b, term2.getLoc(), newReduceArgs);
 
     for (auto &&[i, reg] : llvm::enumerate(llvm::concat<Region>(
              term1.getReductions(), term2.getReductions()))) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
index 66f7bc27f82ff..2ca3b1e366bdf 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
@@ -58,27 +58,27 @@ std::pair<ParallelOp, ParallelOp>
 mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
                             bool noMinMaxBounds) {
   OpBuilder b(op);
-  auto zero = b.create<arith::ConstantIndexOp>(op.getLoc(), 0);
+  auto zero = arith::ConstantIndexOp::create(b, op.getLoc(), 0);
   SmallVector<Value, 2> tileSizeConstants;
   tileSizeConstants.reserve(op.getUpperBound().size());
   for (size_t i = 0, end = op.getUpperBound().size(); i != end; ++i) {
     if (i < tileSizes.size())
       tileSizeConstants.push_back(
-          b.create<arith::ConstantIndexOp>(op.getLoc(), tileSizes[i]));
+          arith::ConstantIndexOp::create(b, op.getLoc(), tileSizes[i]));
     else
       // Just pick 1 for the remaining dimensions.
       tileSizeConstants.push_back(
-          b.create<arith::ConstantIndexOp>(op.getLoc(), 1));
+          arith::ConstantIndexOp::create(b, op.getLoc(), 1));
   }
 
   // Create the outer loop with adjusted steps.
   SmallVector<Value, 2> newSteps;
   newSteps.reserve(op.getStep().size());
   for (auto step : llvm::zip(op.getStep(), tileSizeConstants)) {
-    newSteps.push_back(b.create<arith::MulIOp>(op.getLoc(), std::get<0>(step),
+    newSteps.push_back(arith::MulIOp::create(b, op.getLoc(), std::get<0>(step),
                                                std::get<1>(step)));
   }
-  auto outerLoop = b.create<ParallelOp>(op.getLoc(), op.getLowerBound(),
+  auto outerLoop = ParallelOp::create(b, op.getLoc(), op.getLowerBound(),
                                         op.getUpperBound(), newSteps);
   b.setInsertionPointToStart(outerLoop.getBody());
 
@@ -130,10 +130,10 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // Otherwise, we dynamically compute the bound for
     // each iteration of the outer loop.
     newBounds.push_back(
-        b.create<affine::AffineMinOp>(op.getLoc(), b.getIndexType(), minMap,
+        affine::AffineMinOp::create(b, op.getLoc(), b.getIndexType(), minMap,
                                       ValueRange{newStep, upperBound, iv}));
   }
-  auto innerLoop = b.create<ParallelOp>(
+  auto innerLoop = ParallelOp::create(b,
       op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
       op.getStep());
 
@@ -141,20 +141,20 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     b.setInsertionPointToStart(innerLoop.getBody());
     // Insert in-bound check
     Value inbound =
-        b.create<arith::ConstantIntOp>(op.getLoc(), b.getIntegerType(1), 1);
+        arith::ConstantIntOp::create(b, op.getLoc(), b.getIntegerType(1), 1);
     for (auto [outerUpperBound, outerIV, innerIV, innerStep] :
          llvm::zip(outerLoop.getUpperBound(), outerLoop.getInductionVars(),
                    innerLoop.getInductionVars(), innerLoop.getStep())) {
       // %in_bound = %in_bound &&
       //             (%inner_iv * %inner_step + %outer_iv < %outer_upper_bound)
-      Value index = b.create<arith::AddIOp>(
-          op.getLoc(), b.create<arith::MulIOp>(op.getLoc(), innerIV, innerStep),
+      Value index = arith::AddIOp::create(b,
+          op.getLoc(), arith::MulIOp::create(b, op.getLoc(), innerIV, innerStep),
           outerIV);
-      Value dimInbound = b.create<arith::CmpIOp>(
+      Value dimInbound = arith::CmpIOp::create(b,
           op.getLoc(), arith::CmpIPredicate::ult, index, outerUpperBound);
-      inbound = b.create<arith::AndIOp>(op.getLoc(), inbound, dimInbound);
+      inbound = arith::AndIOp::create(b, op.getLoc(), inbound, dimInbound);
     }
-    auto ifInbound = b.create<IfOp>(op.getLoc(),
+    auto ifInbound = IfOp::create(b, op.getLoc(),
                                     /*resultTypes*/ ArrayRef<Type>{}, inbound,
                                     /*hasElseRegion*/ false);
     ifInbound.getThenRegion().takeBody(op.getRegion());
@@ -162,12 +162,12 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // Replace the scf.reduce terminator with an scf.yield terminator.
     Operation *reduceOp = thenBlock.getTerminator();
     b.setInsertionPointToEnd(&thenBlock);
-    b.create<scf::YieldOp>(reduceOp->getLoc());
+    scf::YieldOp::create(b, reduceOp->getLoc());
     reduceOp->erase();
     b.setInsertionPointToStart(innerLoop.getBody());
     for (const auto &ivs : llvm::enumerate(llvm::zip(
              innerLoop.getInductionVars(), outerLoop.getInductionVars()))) {
-      auto newIndex = b.create<arith::AddIOp>(
+      auto newIndex = arith::AddIOp::create(b,
           op.getLoc(), std::get<0>(ivs.value()), std::get<1>(ivs.value()));
       thenBlock.getArgument(ivs.index())
           .replaceAllUsesExcept(newIndex, newIndex);
@@ -179,7 +179,7 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     for (auto ivs : llvm::zip(innerLoop.getInductionVars(),
                               outerLoop.getInductionVars())) {
       Value innerIndex = std::get<0>(ivs);
-      auto newIndex = b.create<arith::AddIOp>(op.getLoc(), std::get<0>(ivs),
+      auto newIndex = arith::AddIOp::create(b, op.getLoc(), std::get<0>(ivs),
                                               std::get<1>(ivs));
       innerIndex.replaceAllUsesExcept(newIndex, newIndex);
     }
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 09326242eec2a..41895e46d953d 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -112,7 +112,7 @@ class ConvertForOpTypes
 
     // We can not do clone as the number of result types after conversion
     // might be different.
-    ForOp newOp = rewriter.create<ForOp>(
+    ForOp newOp = ForOp::create(rewriter,
         op.getLoc(), llvm::getSingleElement(adaptor.getLowerBound()),
         llvm::getSingleElement(adaptor.getUpperBound()),
         llvm::getSingleElement(adaptor.getStep()),
@@ -142,7 +142,7 @@ class ConvertIfOpTypes
                                       ConversionPatternRewriter &rewriter,
                                       TypeRange dstTypes) const {
 
-    IfOp newOp = rewriter.create<IfOp>(
+    IfOp newOp = IfOp::create(rewriter,
         op.getLoc(), dstTypes, llvm::getSingleElement(adaptor.getCondition()),
         true);
     newOp->setAttrs(op->getAttrs());
@@ -171,7 +171,7 @@ class ConvertWhileOpTypes
   std::optional<WhileOp> convertSourceOp(WhileOp op, OneToNOpAdaptor adaptor,
                                          ConversionPatternRewriter &rewriter,
                                          TypeRange dstTypes) const {
-    auto newOp = rewriter.create<WhileOp>(op.getLoc(), dstTypes,
+    auto newOp = WhileOp::create(rewriter, op.getLoc(), dstTypes,
                                           flattenValues(adaptor.getOperands()));
 
     for (auto i : {0u, 1u}) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index c7d634283fd4e..3171b96d46b8a 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -450,7 +450,7 @@ static LogicalResult generateLoopNestUsingForOp(
   SmallVector<Value> ivs;
   for (auto [lb, ub, step] : llvm::zip_equal(lbVals, ubVals, stepVals)) {
     auto loop =
-        rewriter.create<scf::ForOp>(loc, lb, ub, step, destinationTensors,
+        scf::ForOp::create(rewriter, loc, lb, ub, step, destinationTensors,
                                     [](OpBuilder &bodyBuilder, Location bodyLoc,
                                        Value iv, ValueRange /*iterArgs*/) {});
     loops.push_back(loop);
@@ -479,12 +479,12 @@ static LogicalResult generateLoopNestUsingForOp(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    auto insertSlice = rewriter.create<tensor::InsertSliceOp>(
+    auto insertSlice = tensor::InsertSliceOp::create(rewriter,
         loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
     yieldedValues.push_back(insertSlice);
   }
-  rewriter.create<scf::YieldOp>(loc, yieldedValues);
+  scf::YieldOp::create(rewriter, loc, yieldedValues);
 
   // Add the scf.yield operations for all the outer loops.
   for (auto [outerLoop, innerLoop] :
@@ -492,7 +492,7 @@ static LogicalResult generateLoopNestUsingForOp(
                        MutableArrayRef(loops).drop_front())) {
     rewriter.setInsertionPointToEnd(
         cast<scf::ForOp>(outerLoop.getOperation()).getBody());
-    rewriter.create<scf::YieldOp>(outerLoop.getLoc(), innerLoop->getResults());
+    scf::YieldOp::create(rewriter, outerLoop.getLoc(), innerLoop->getResults());
   }
   return success();
 }
@@ -533,13 +533,13 @@ static LogicalResult generateLoopNestUsingForallOp(
         continue;
       nonZeroNumThreads.push_back(nt);
     }
-    forallOp = rewriter.create<scf::ForallOp>(loc, nonZeroNumThreads,
+    forallOp = scf::ForallOp::create(rewriter, loc, nonZeroNumThreads,
                                               destinationTensors, mappingAttr);
   } else {
     SmallVector<OpFoldResult> lbs, ubs, steps;
     std::tie(lbs, ubs, steps) =
         getLoopBounds(rewriter, loc, loopRanges, tileSizes);
-    forallOp = rewriter.create<scf::ForallOp>(loc, lbs, ubs, steps,
+    forallOp = scf::ForallOp::create(rewriter, loc, lbs, ubs, steps,
                                               destinationTensors, mappingAttr);
   }
   loops.push_back(forallOp);
@@ -561,7 +561,7 @@ static LogicalResult generateLoopNestUsingForallOp(
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
 
-    rewriter.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(rewriter,
         loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
   }
@@ -798,7 +798,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
 
   auto inits = llvm::to_vector(loopOp.getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = rewriter.create<scf::ForOp>(
+  auto newLoop = scf::ForOp::create(rewriter,
       loc, loopOp.getLowerBound(), loopOp.getUpperBound(), loopOp.getStep(),
       inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
@@ -829,7 +829,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    Value insert = rewriter.create<tensor::InsertSliceOp>(
+    Value insert = tensor::InsertSliceOp::create(rewriter,
         yieldOp->getLoc(), tiledValue, regionIterArg, resultOffset, resultSize,
         resultStride);
     newYieldValues.push_back(insert);
@@ -851,7 +851,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
   rewriter.setInsertionPoint(loopOp);
   auto inits = llvm::to_vector(loopOp.getOutputs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = rewriter.create<scf::ForallOp>(
+  auto newLoop = scf::ForallOp::create(rewriter,
       loc, loopOp.getMixedLowerBound(), loopOp.getMixedUpperBound(),
       loopOp.getMixedStep(), inits, loopOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
@@ -884,7 +884,7 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
            tiledValues, regionIterArgs, resultOffsets, resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    rewriter.create<tensor::ParallelInsertSliceOp>(
+    tensor::ParallelInsertSliceOp::create(rewriter,
         terminator.getLoc(), tiledValue, iterArg, resultOffset, resultSize,
         resultStride);
   }
@@ -935,7 +935,7 @@ static LogicalResult addInitOperandsToLoopNest(
     // Create a new loop with the new init values for this loop.
     SmallVector<Value> newInits = llvm::to_vector(forLoop.getInitArgs());
     newInits.append(newInitValues.begin(), newInitValues.end());
-    auto newLoop = rewriter.create<scf::ForOp>(
+    auto newLoop = scf::ForOp::create(rewriter,
         forLoop.getLoc(), forLoop.getLowerBound(), forLoop.getUpperBound(),
         forLoop.getStep(), newInits,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {});
@@ -1419,7 +1419,7 @@ FailureOr<SmallVector<Operation *>> mlir::scf::yieldReplacementForFusedProducer(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = rewriter.create<tensor::ExtractSliceOp>(
+        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
             loc, newRegionArg, offsetList[index], sizesList[index],
             SmallVector<OpFoldResult>(offsetList[index].size(),
                                       rewriter.getIndexAttr(1)));
@@ -2092,7 +2092,7 @@ cloneAsInsertSlice<tensor::InsertSliceOp>(RewriterBase &rewriter,
 template <>
 tensor::InsertSliceOp cloneAsInsertSlice<tensor::ParallelInsertSliceOp>(
     RewriterBase &rewriter, tensor::ParallelInsertSliceOp insertSliceOp) {
-  return rewriter.create<tensor::InsertSliceOp>(
+  return tensor::InsertSliceOp::create(rewriter,
       insertSliceOp->getLoc(), insertSliceOp.getSource(),
       insertSliceOp.getDest(), insertSliceOp.getMixedOffsets(),
       insertSliceOp.getMixedSizes(), insertSliceOp.getMixedStrides());
@@ -2314,7 +2314,7 @@ mlir::scf::tileAndFuseConsumerOfSlices(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = rewriter.create<tensor::ExtractSliceOp>(
+        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
             loc, newRegionArg, resultOffsets[index], resultSizes[index],
             SmallVector<OpFoldResult>(resultOffsets[index].size(),
                                       rewriter.getIndexAttr(1)));
@@ -2391,7 +2391,7 @@ mlir::scf::lowerToLoopsUsingSCFForOp(RewriterBase &rewriter,
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.size);
     Value strideVal =
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.stride);
-    auto loop = rewriter.create<scf::ForOp>(op.getLoc(), offsetVal, sizeVal,
+    auto loop = scf::ForOp::create(rewriter, op.getLoc(), offsetVal, sizeVal,
                                             strideVal, ValueRange{});
     loops.push_back(loop);
     ivs.push_back(loop.getInductionVar());
diff --git a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
index 29d6d2574a2be..14ded96516eab 100644
--- a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
@@ -191,7 +191,7 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
   // dummy builder instead.
   auto emptyBuilder = [](OpBuilder &, Location, Value, ValueRange) {};
   auto newLoop =
-      rewriter.create<scf::ForOp>(loc, lb, ub, step, newArgs, emptyBuilder);
+      scf::ForOp::create(rewriter, loc, lb, ub, step, newArgs, emptyBuilder);
 
   Block *newBody = newLoop.getBody();
 
@@ -238,18 +238,18 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
   rewriter.setInsertionPointAfter(newLoop);
   Value one;
   if (isa<IndexType>(step.getType())) {
-    one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    one = arith::ConstantIndexOp::create(rewriter, loc, 1);
   } else {
-    one = rewriter.create<arith::ConstantIntOp>(loc, step.getType(), 1);
+    one = arith::ConstantIntOp::create(rewriter, loc, step.getType(), 1);
   }
 
-  Value stepDec = rewriter.create<arith::SubIOp>(loc, step, one);
-  Value len = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  len = rewriter.create<arith::AddIOp>(loc, len, stepDec);
-  len = rewriter.create<arith::DivSIOp>(loc, len, step);
-  len = rewriter.create<arith::SubIOp>(loc, len, one);
-  Value res = rewriter.create<arith::MulIOp>(loc, len, step);
-  res = rewriter.create<arith::AddIOp>(loc, lb, res);
+  Value stepDec = arith::SubIOp::create(rewriter, loc, step, one);
+  Value len = arith::SubIOp::create(rewriter, loc, ub, lb);
+  len = arith::AddIOp::create(rewriter, loc, len, stepDec);
+  len = arith::DivSIOp::create(rewriter, loc, len, step);
+  len = arith::SubIOp::create(rewriter, loc, len, one);
+  Value res = arith::MulIOp::create(rewriter, loc, len, step);
+  res = arith::AddIOp::create(rewriter, loc, lb, res);
 
   // Reconstruct `scf.while` results, inserting final induction var value
   // into proper place.
diff --git a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
index f829208ce8798..707ed836cf63e 100644
--- a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
@@ -96,7 +96,7 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       condOp.getArgs(), [&](Value arg) { return mapper.lookupOrDefault(arg); });
 
   // Create rotated while loop.
-  auto newLoopOp = rewriter.create<scf::WhileOp>(
+  auto newLoopOp = scf::WhileOp::create(rewriter,
       whileOp.getLoc(), whileOp.getResultTypes(), clonedCondArgs,
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Rotate and move the loop body into before block.
@@ -109,21 +109,21 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       },
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Pass through values.
-        builder.create<scf::YieldOp>(loc, args);
+        scf::YieldOp::create(builder, loc, args);
       });
 
   // Create zero-trip-check and move the while loop in.
-  auto ifOp = rewriter.create<scf::IfOp>(
+  auto ifOp = scf::IfOp::create(rewriter,
       whileOp.getLoc(), clonedCondition,
       [&](OpBuilder &builder, Location loc) {
         // Then runs the while loop.
         rewriter.moveOpBefore(newLoopOp, builder.getInsertionBlock(),
                               builder.getInsertionPoint());
-        builder.create<scf::YieldOp>(loc, newLoopOp.getResults());
+        scf::YieldOp::create(builder, loc, newLoopOp.getResults());
       },
       [&](OpBuilder &builder, Location loc) {
         // Else returns the results from precondition.
-        builder.create<scf::YieldOp>(loc, clonedCondArgs);
+        scf::YieldOp::create(builder, loc, clonedCondArgs);
       });
 
   rewriter.replaceOp(whileOp, ifOp);
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index f4047be68ccf2..6a387ddf8908a 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -153,7 +153,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
       FunctionType::get(rewriter.getContext(), outlinedFuncArgTypes,
                         originalTerminator->getOperandTypes());
   auto outlinedFunc =
-      rewriter.create<func::FuncOp>(loc, funcName, outlinedFuncType);
+      func::FuncOp::create(rewriter, loc, funcName, outlinedFuncType);
   Block *outlinedFuncBody = outlinedFunc.addEntryBlock();
 
   // Merge blocks while replacing the original block operands.
@@ -168,7 +168,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
         outlinedFuncBlockArgs.take_front(numOriginalBlockArguments));
     // Explicitly set up a new ReturnOp terminator.
     rewriter.setInsertionPointToEnd(outlinedFuncBody);
-    rewriter.create<func::ReturnOp>(loc, originalTerminator->getResultTypes(),
+    func::ReturnOp::create(rewriter, loc, originalTerminator->getResultTypes(),
                                     originalTerminator->getOperands());
   }
 
@@ -185,7 +185,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
     SmallVector<Value> callValues;
     llvm::append_range(callValues, newBlock->getArguments());
     llvm::append_range(callValues, outlinedValues);
-    auto call = rewriter.create<func::CallOp>(loc, outlinedFunc, callValues);
+    auto call = func::CallOp::create(rewriter, loc, outlinedFunc, callValues);
     if (callOp)
       *callOp = call;
 
@@ -274,12 +274,12 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
 
-  Value divisorMinusOneCst = builder.create<arith::ConstantOp>(
+  Value divisorMinusOneCst = arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(dividend.getType(), divisor - 1));
-  Value divisorCst = builder.create<arith::ConstantOp>(
+  Value divisorCst = arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(dividend.getType(), divisor));
-  Value sum = builder.create<arith::AddIOp>(loc, dividend, divisorMinusOneCst);
-  return builder.create<arith::DivUIOp>(loc, sum, divisorCst);
+  Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOneCst);
+  return arith::DivUIOp::create(builder, loc, sum, divisorCst);
 }
 
 // Build the IR that performs ceil division of a positive value by another
@@ -290,11 +290,11 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
                              Value divisor) {
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
-  Value cstOne = builder.create<arith::ConstantOp>(
+  Value cstOne = arith::ConstantOp::create(builder,
       loc, builder.getOneAttr(dividend.getType()));
-  Value divisorMinusOne = builder.create<arith::SubIOp>(loc, divisor, cstOne);
-  Value sum = builder.create<arith::AddIOp>(loc, dividend, divisorMinusOne);
-  return builder.create<arith::DivUIOp>(loc, sum, divisor);
+  Value divisorMinusOne = arith::SubIOp::create(builder, loc, divisor, cstOne);
+  Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOne);
+  return arith::DivUIOp::create(builder, loc, sum, divisor);
 }
 
 /// Returns the trip count of `forOp` if its' low bound, high bound and step are
@@ -404,7 +404,7 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'upperBoundUnrolled' and set epilogue loop flag.
     generateEpilogueLoop = upperBoundUnrolledCst < ubCst;
     if (generateEpilogueLoop)
-      upperBoundUnrolled = boundsBuilder.create<arith::ConstantOp>(
+      upperBoundUnrolled = arith::ConstantOp::create(boundsBuilder,
           loc, boundsBuilder.getIntegerAttr(forOp.getUpperBound().getType(),
                                             upperBoundUnrolledCst));
     else
@@ -413,7 +413,7 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'stepUnrolled'.
     stepUnrolled = stepCst == stepUnrolledCst
                        ? step
-                       : boundsBuilder.create<arith::ConstantOp>(
+                       : arith::ConstantOp::create(boundsBuilder,
                              loc, boundsBuilder.getIntegerAttr(
                                       step.getType(), stepUnrolledCst));
   } else {
@@ -423,22 +423,22 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     auto lowerBound = forOp.getLowerBound();
     auto upperBound = forOp.getUpperBound();
     Value diff =
-        boundsBuilder.create<arith::SubIOp>(loc, upperBound, lowerBound);
+        arith::SubIOp::create(boundsBuilder, loc, upperBound, lowerBound);
     Value tripCount = ceilDivPositive(boundsBuilder, loc, diff, step);
-    Value unrollFactorCst = boundsBuilder.create<arith::ConstantOp>(
+    Value unrollFactorCst = arith::ConstantOp::create(boundsBuilder,
         loc, boundsBuilder.getIntegerAttr(tripCount.getType(), unrollFactor));
     Value tripCountRem =
-        boundsBuilder.create<arith::RemSIOp>(loc, tripCount, unrollFactorCst);
+        arith::RemSIOp::create(boundsBuilder, loc, tripCount, unrollFactorCst);
     // Compute tripCountEvenMultiple = tripCount - (tripCount % unrollFactor)
     Value tripCountEvenMultiple =
-        boundsBuilder.create<arith::SubIOp>(loc, tripCount, tripCountRem);
+        arith::SubIOp::create(boundsBuilder, loc, tripCount, tripCountRem);
     // Compute upperBoundUnrolled = lowerBound + tripCountEvenMultiple * step
-    upperBoundUnrolled = boundsBuilder.create<arith::AddIOp>(
+    upperBoundUnrolled = arith::AddIOp::create(boundsBuilder,
         loc, lowerBound,
-        boundsBuilder.create<arith::MulIOp>(loc, tripCountEvenMultiple, step));
+        arith::MulIOp::create(boundsBuilder, loc, tripCountEvenMultiple, step));
     // Scale 'step' by 'unrollFactor'.
     stepUnrolled =
-        boundsBuilder.create<arith::MulIOp>(loc, step, unrollFactorCst);
+        arith::MulIOp::create(boundsBuilder, loc, step, unrollFactorCst);
   }
 
   UnrolledLoopInfo resultLoops;
@@ -474,11 +474,11 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
       forOp.getBody(), forOp.getInductionVar(), unrollFactor,
       [&](unsigned i, Value iv, OpBuilder b) {
         // iv' = iv + step * i;
-        auto stride = b.create<arith::MulIOp>(
+        auto stride = arith::MulIOp::create(b,
             loc, step,
-            b.create<arith::ConstantOp>(loc,
+            arith::ConstantOp::create(b, loc,
                                         b.getIntegerAttr(iv.getType(), i)));
-        return b.create<arith::AddIOp>(loc, iv, stride);
+        return arith::AddIOp::create(b, loc, iv, stride);
       },
       annotateFn, iterArgs, yieldedValues);
   // Promote the loop body up if this has turned into a single iteration loop.
@@ -781,13 +781,13 @@ void mlir::denormalizeInductionVariable(RewriterBase &rewriter, Location loc,
   if (!isStepOne) {
     Value origStepValue =
         getValueOrCreateConstantIntOp(rewriter, loc, origStep);
-    scaled = rewriter.create<arith::MulIOp>(loc, normalizedIv, origStepValue);
+    scaled = arith::MulIOp::create(rewriter, loc, normalizedIv, origStepValue);
     preserve.insert(scaled.getDefiningOp());
   }
   denormalizedIv = scaled;
   if (!isZeroBased) {
     Value origLbValue = getValueOrCreateConstantIntOp(rewriter, loc, origLb);
-    denormalizedIv = rewriter.create<arith::AddIOp>(loc, scaled, origLbValue);
+    denormalizedIv = arith::AddIOp::create(rewriter, loc, scaled, origLbValue);
     preserve.insert(denormalizedIv.getDefiningOp());
   }
 
@@ -824,7 +824,7 @@ static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc,
       continue;
     if (productOf)
       productOf =
-          rewriter.create<arith::MulIOp>(loc, productOf.value(), v).getResult();
+          arith::MulIOp::create(rewriter, loc, productOf.value(), v).getResult();
     else
       productOf = v;
   }
@@ -851,7 +851,7 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
 
   if (linearizedIv.getType().isIndex()) {
     Operation *delinearizedOp =
-        rewriter.create<affine::AffineDelinearizeIndexOp>(loc, linearizedIv,
+        affine::AffineDelinearizeIndexOp::create(rewriter, loc, linearizedIv,
                                                           ubs);
     auto resultVals = llvm::map_to_vector(
         delinearizedOp->getResults(), [](OpResult r) -> Value { return r; });
@@ -874,7 +874,7 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
     if (!isUbOne.test(index)) {
       break;
     }
-    delinearizedIvs[index] = rewriter.create<arith::ConstantOp>(
+    delinearizedIvs[index] = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(ub.getType()));
     numLeadingOneUbs++;
   }
@@ -883,16 +883,16 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
   for (unsigned i = numLeadingOneUbs, e = ubs.size(); i < e; ++i) {
     unsigned idx = ubs.size() - (i - numLeadingOneUbs) - 1;
     if (i != numLeadingOneUbs && !isUbOne.test(idx + 1)) {
-      previous = rewriter.create<arith::DivSIOp>(loc, previous, ubs[idx + 1]);
+      previous = arith::DivSIOp::create(rewriter, loc, previous, ubs[idx + 1]);
       preservedUsers.insert(previous.getDefiningOp());
     }
     Value iv = previous;
     if (i != e - 1) {
       if (!isUbOne.test(idx)) {
-        iv = rewriter.create<arith::RemSIOp>(loc, previous, ubs[idx]);
+        iv = arith::RemSIOp::create(rewriter, loc, previous, ubs[idx]);
         preservedUsers.insert(iv.getDefiningOp());
       } else {
-        iv = rewriter.create<arith::ConstantOp>(
+        iv = arith::ConstantOp::create(rewriter,
             loc, rewriter.getZeroAttr(ubs[idx].getType()));
       }
     }
@@ -1093,12 +1093,12 @@ void mlir::collapseParallelLoops(
 
   // Combine iteration spaces.
   SmallVector<Value, 3> lowerBounds, upperBounds, steps;
-  auto cst0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  auto cst1 = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  auto cst0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto cst1 = arith::ConstantIndexOp::create(rewriter, loc, 1);
   for (auto &sortedDimension : sortedDimensions) {
-    Value newUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value newUpperBound = arith::ConstantIndexOp::create(rewriter, loc, 1);
     for (auto idx : sortedDimension) {
-      newUpperBound = rewriter.create<arith::MulIOp>(
+      newUpperBound = arith::MulIOp::create(rewriter,
           loc, newUpperBound, normalizedUpperBounds[idx]);
     }
     lowerBounds.push_back(cst0);
@@ -1112,7 +1112,7 @@ void mlir::collapseParallelLoops(
   // value. The remainders then determine based on that range, which iteration
   // of the original induction value this represents. This is a normalized value
   // that is un-normalized already by the previous logic.
-  auto newPloop = rewriter.create<scf::ParallelOp>(
+  auto newPloop = scf::ParallelOp::create(rewriter,
       loc, lowerBounds, upperBounds, steps,
       [&](OpBuilder &insideBuilder, Location, ValueRange ploopIVs) {
         for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) {
@@ -1123,14 +1123,14 @@ void mlir::collapseParallelLoops(
             unsigned idx = combinedDimensions[i][j];
 
             // Determine the current induction value's current loop iteration
-            Value iv = insideBuilder.create<arith::RemSIOp>(
+            Value iv = arith::RemSIOp::create(insideBuilder,
                 loc, previous, normalizedUpperBounds[idx]);
             replaceAllUsesInRegionWith(loops.getBody()->getArgument(idx), iv,
                                        loops.getRegion());
 
             // Remove the effect of the current induction value to prepare for
             // the next value.
-            previous = insideBuilder.create<arith::DivSIOp>(
+            previous = arith::DivSIOp::create(insideBuilder,
                 loc, previous, normalizedUpperBounds[idx]);
           }
 
@@ -1241,7 +1241,7 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
   auto iv = forOp.getInductionVar();
 
   OpBuilder b(forOp);
-  forOp.setStep(b.create<arith::MulIOp>(forOp.getLoc(), originalStep, factor));
+  forOp.setStep(arith::MulIOp::create(b, forOp.getLoc(), originalStep, factor));
 
   Loops innerLoops;
   for (auto t : targets) {
@@ -1251,12 +1251,12 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
 
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator((t.getBody()));
-    Value stepped = b.create<arith::AddIOp>(t.getLoc(), iv, forOp.getStep());
+    Value stepped = arith::AddIOp::create(b, t.getLoc(), iv, forOp.getStep());
     Value ub =
-        b.create<arith::MinSIOp>(t.getLoc(), forOp.getUpperBound(), stepped);
+        arith::MinSIOp::create(b, t.getLoc(), forOp.getUpperBound(), stepped);
 
     // Splice [begin, begin + nOps - 1) into `newForOp` and replace uses.
-    auto newForOp = b.create<scf::ForOp>(t.getLoc(), iv, ub, originalStep);
+    auto newForOp = scf::ForOp::create(b, t.getLoc(), iv, ub, originalStep);
     newForOp.getBody()->getOperations().splice(
         newForOp.getBody()->getOperations().begin(),
         t.getBody()->getOperations(), begin, std::next(begin, nOps - 1));
@@ -1343,7 +1343,7 @@ TileLoops mlir::extractFixedOuterLoops(scf::ForOp rootForOp,
     auto forOp = forOps[i];
     OpBuilder builder(forOp);
     auto loc = forOp.getLoc();
-    Value diff = builder.create<arith::SubIOp>(loc, forOp.getUpperBound(),
+    Value diff = arith::SubIOp::create(builder, loc, forOp.getUpperBound(),
                                                forOp.getLowerBound());
     Value numIterations = ceilDivPositive(builder, loc, diff, forOp.getStep());
     Value iterationsPerBlock =
@@ -1376,7 +1376,7 @@ scf::ForallOp mlir::fuseIndependentSiblingForallLoops(scf::ForallOp target,
 
   // Create a new scf.forall op after the source loop.
   rewriter.setInsertionPointAfter(source);
-  scf::ForallOp fusedLoop = rewriter.create<scf::ForallOp>(
+  scf::ForallOp fusedLoop = scf::ForallOp::create(rewriter,
       source.getLoc(), source.getMixedLowerBound(), source.getMixedUpperBound(),
       source.getMixedStep(), fusedOuts, source.getMapping());
 
@@ -1429,7 +1429,7 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   // Create a new scf.for op after the source loop (with scf.yield terminator
   // (without arguments) only in case its init_args is empty).
   rewriter.setInsertionPointAfter(source);
-  scf::ForOp fusedLoop = rewriter.create<scf::ForOp>(
+  scf::ForOp fusedLoop = scf::ForOp::create(rewriter,
       source.getLoc(), source.getLowerBound(), source.getUpperBound(),
       source.getStep(), fusedInitArgs);
 
@@ -1456,7 +1456,7 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   for (Value operand : source.getBody()->getTerminator()->getOperands())
     yieldResults.push_back(mapping.lookupOrDefault(operand));
   if (!yieldResults.empty())
-    rewriter.create<scf::YieldOp>(source.getLoc(), yieldResults);
+    scf::YieldOp::create(rewriter, source.getLoc(), yieldResults);
 
   // Replace old loops by substituting their uses by results of the fused loop.
   rewriter.replaceOp(target, fusedLoop.getResults().take_front(numTargetOuts));
@@ -1487,7 +1487,7 @@ FailureOr<scf::ForallOp> mlir::normalizeForallOp(RewriterBase &rewriter,
 
   // Use the normalized builder since the lower bounds are always 0 and the
   // steps are always 1.
-  auto normalizedForallOp = rewriter.create<scf::ForallOp>(
+  auto normalizedForallOp = scf::ForallOp::create(rewriter,
       loc, newUbs, forallOp.getOutputs(), forallOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
 
diff --git a/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp b/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
index 66eed861b2bb7..48c0b1ed8e14f 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTDialect.cpp
@@ -30,14 +30,14 @@ Operation *SMTDialect::materializeConstant(OpBuilder &builder, Attribute value,
     if (auto attrValue = dyn_cast<BitVectorAttr>(value)) {
       assert(bvType == attrValue.getType() &&
              "attribute and desired result types have to match");
-      return builder.create<BVConstantOp>(loc, attrValue);
+      return BVConstantOp::create(builder, loc, attrValue);
     }
   }
 
   // BoolType constants can materialize into smt.constant
   if (auto boolType = dyn_cast<BoolType>(type)) {
     if (auto attrValue = dyn_cast<BoolAttr>(value))
-      return builder.create<BoolConstantOp>(loc, attrValue);
+      return BoolConstantOp::create(builder, loc, attrValue);
   }
 
   return nullptr;
diff --git a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
index 8977a3abc125d..c517ef2b18f6f 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
@@ -405,7 +405,7 @@ static void buildQuantifier(
         SmallVector<Location>(boundVarTypes.size(), odsState.location));
     Value returnVal =
         bodyBuilder(odsBuilder, odsState.location, block->getArguments());
-    odsBuilder.create<smt::YieldOp>(odsState.location, returnVal);
+    smt::YieldOp::create(odsBuilder, odsState.location, returnVal);
   }
   if (patternBuilder) {
     Region *region = odsState.addRegion();
@@ -416,7 +416,7 @@ static void buildQuantifier(
         SmallVector<Location>(boundVarTypes.size(), odsState.location));
     ValueRange returnVals =
         patternBuilder(odsBuilder, odsState.location, block->getArguments());
-    odsBuilder.create<smt::YieldOp>(odsState.location, returnVals);
+    smt::YieldOp::create(odsBuilder, odsState.location, returnVals);
   }
 }
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
index 371456552b5b5..cacc52d0feaf0 100644
--- a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
@@ -391,7 +391,7 @@ void LoopOp::addEntryAndMergeBlock(OpBuilder &builder) {
   builder.createBlock(&getBody());
 
   // Add a spirv.mlir.merge op into the merge block.
-  builder.create<spirv::MergeOp>(getLoc());
+  spirv::MergeOp::create(builder, getLoc());
 }
 
 //===----------------------------------------------------------------------===//
@@ -543,7 +543,7 @@ void SelectionOp::addMergeBlock(OpBuilder &builder) {
   builder.createBlock(&getBody());
 
   // Add a spirv.mlir.merge op into the merge block.
-  builder.create<spirv::MergeOp>(getLoc());
+  spirv::MergeOp::create(builder, getLoc());
 }
 
 SelectionOp
@@ -551,7 +551,7 @@ SelectionOp::createIfThen(Location loc, Value condition,
                           function_ref<void(OpBuilder &builder)> thenBody,
                           OpBuilder &builder) {
   auto selectionOp =
-      builder.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
+      spirv::SelectionOp::create(builder, loc, spirv::SelectionControl::None);
 
   selectionOp.addMergeBlock(builder);
   Block *mergeBlock = selectionOp.getMergeBlock();
@@ -562,14 +562,14 @@ SelectionOp::createIfThen(Location loc, Value condition,
     OpBuilder::InsertionGuard guard(builder);
     thenBlock = builder.createBlock(mergeBlock);
     thenBody(builder);
-    builder.create<spirv::BranchOp>(loc, mergeBlock);
+    spirv::BranchOp::create(builder, loc, mergeBlock);
   }
 
   // Build the header block.
   {
     OpBuilder::InsertionGuard guard(builder);
     builder.createBlock(thenBlock);
-    builder.create<spirv::BranchConditionalOp>(
+    spirv::BranchConditionalOp::create(builder,
         loc, condition, thenBlock,
         /*trueArguments=*/ArrayRef<Value>(), mergeBlock,
         /*falseArguments=*/ArrayRef<Value>());
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index 047f8da0cc003..53c07fc88e2ab 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -178,16 +178,16 @@ struct IAddCarryFold final : OpRewritePattern<spirv::IAddCarryOp> {
       return failure();
 
     Value addsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, adds);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, adds);
 
     Value carrysVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, carrys);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, carrys);
 
     // Create empty struct
-    Value undef = rewriter.create<spirv::UndefOp>(loc, op.getType());
+    Value undef = spirv::UndefOp::create(rewriter, loc, op.getType());
     // Fill in adds at id 0
     Value intermediate =
-        rewriter.create<spirv::CompositeInsertOp>(loc, addsVal, undef, 0);
+        spirv::CompositeInsertOp::create(rewriter, loc, addsVal, undef, 0);
     // Fill in carrys at id 1
     rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(op, carrysVal,
                                                           intermediate, 1);
@@ -260,16 +260,16 @@ struct MulExtendedFold final : OpRewritePattern<MulOp> {
       return failure();
 
     Value lowBitsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, lowBits);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, lowBits);
 
     Value highBitsVal =
-        rewriter.create<spirv::ConstantOp>(loc, constituentType, highBits);
+        spirv::ConstantOp::create(rewriter, loc, constituentType, highBits);
 
     // Create empty struct
-    Value undef = rewriter.create<spirv::UndefOp>(loc, op.getType());
+    Value undef = spirv::UndefOp::create(rewriter, loc, op.getType());
     // Fill in lowBits at id 0
     Value intermediate =
-        rewriter.create<spirv::CompositeInsertOp>(loc, lowBitsVal, undef, 0);
+        spirv::CompositeInsertOp::create(rewriter, loc, lowBitsVal, undef, 0);
     // Fill in highBits at id 1
     rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(op, highBitsVal,
                                                           intermediate, 1);
@@ -1309,10 +1309,10 @@ struct ConvertSelectionOpToSelect final : OpRewritePattern<spirv::SelectionOp> {
     auto storeOpAttributes =
         cast<spirv::StoreOp>(trueBlock->front())->getAttrs();
 
-    auto selectOp = rewriter.create<spirv::SelectOp>(
+    auto selectOp = spirv::SelectOp::create(rewriter,
         selectionOp.getLoc(), trueValue.getType(),
         brConditionalOp.getCondition(), trueValue, falseValue);
-    rewriter.create<spirv::StoreOp>(selectOp.getLoc(), ptrValue,
+    spirv::StoreOp::create(rewriter, selectOp.getLoc(), ptrValue,
                                     selectOp.getResult(), storeOpAttributes);
 
     // `spirv.mlir.selection` is not needed anymore.
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
index f32c53b8f0b9e..c9a8e97bd3296 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
@@ -940,12 +940,12 @@ Operation *SPIRVDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (!spirv::ConstantOp::isBuildableWith(type))
     return nullptr;
 
-  return builder.create<spirv::ConstantOp>(loc, type, value);
+  return spirv::ConstantOp::create(builder, loc, type, value);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 7042f12ba00f8..945f95b0f1dc6 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -651,25 +651,25 @@ spirv::ConstantOp spirv::ConstantOp::getZero(Type type, Location loc,
   if (auto intType = llvm::dyn_cast<IntegerType>(type)) {
     unsigned width = intType.getWidth();
     if (width == 1)
-      return builder.create<spirv::ConstantOp>(loc, type,
+      return spirv::ConstantOp::create(builder, loc, type,
                                                builder.getBoolAttr(false));
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getFloatAttr(floatType, 0.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 0).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 0.0).getValue()));
@@ -684,25 +684,25 @@ spirv::ConstantOp spirv::ConstantOp::getOne(Type type, Location loc,
   if (auto intType = llvm::dyn_cast<IntegerType>(type)) {
     unsigned width = intType.getWidth();
     if (width == 1)
-      return builder.create<spirv::ConstantOp>(loc, type,
+      return spirv::ConstantOp::create(builder, loc, type,
                                                builder.getBoolAttr(true));
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return builder.create<spirv::ConstantOp>(
+    return spirv::ConstantOp::create(builder,
         loc, type, builder.getFloatAttr(floatType, 1.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 1).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return builder.create<spirv::ConstantOp>(
+      return spirv::ConstantOp::create(builder,
           loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 1.0).getValue()));
@@ -1884,7 +1884,7 @@ ParseResult spirv::SpecConstantOperationOp::parse(OpAsmParser &parser,
 
   OpBuilder builder(parser.getContext());
   builder.setInsertionPointToEnd(&block);
-  builder.create<spirv::YieldOp>(wrappedOp->getLoc(), wrappedOp->getResult(0));
+  spirv::YieldOp::create(builder, wrappedOp->getLoc(), wrappedOp->getResult(0));
   result.location = wrappedOp->getLoc();
 
   result.addTypes(wrappedOp->getResult(0).getType());
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 8da688806bade..0795678a9c66b 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -105,7 +105,7 @@ OwningOpRef<spirv::ModuleOp> combine(ArrayRef<spirv::ModuleOp> inputModules,
     }
   }
 
-  auto combinedModule = combinedModuleBuilder.create<spirv::ModuleOp>(
+  auto combinedModule = spirv::ModuleOp::create(combinedModuleBuilder,
       firstModule.getLoc(), addressingModel, memoryModel, vceTriple);
   combinedModuleBuilder.setInsertionPointToStart(combinedModule.getBody());
 
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 85525a5a02fa2..574f9a7c24b12 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -70,7 +70,7 @@ createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
   varType =
       spirv::PointerType::get(varPointeeType, varPtrType.getStorageClass());
 
-  return builder.create<spirv::GlobalVariableOp>(
+  return spirv::GlobalVariableOp::create(builder,
       funcOp.getLoc(), varType, varName, abiInfo.getDescriptorSet(),
       abiInfo.getBinding());
 }
@@ -146,7 +146,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     return funcOp.emitRemark("lower entry point failure: could not select "
                              "execution model based on 'spirv.target_env'");
 
-  builder.create<spirv::EntryPointOp>(funcOp.getLoc(), *executionModel, funcOp,
+  spirv::EntryPointOp::create(builder, funcOp.getLoc(), *executionModel, funcOp,
                                       interfaceVars);
 
   // Specifies the spirv.ExecutionModeOp.
@@ -154,7 +154,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::LocalSize);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
                                              spirv::ExecutionMode::LocalSize,
                                              workgroupSizeAttr.asArrayRef());
       // Erase workgroup size.
@@ -167,7 +167,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SubgroupSize);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
                                              spirv::ExecutionMode::SubgroupSize,
                                              *subgroupSize);
       // Erase subgroup size.
@@ -180,7 +180,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SignedZeroInfNanPreserve);
     if (!caps || targetEnv.allows(*caps)) {
-      builder.create<spirv::ExecutionModeOp>(
+      spirv::ExecutionModeOp::create(builder,
           funcOp.getLoc(), funcOp,
           spirv::ExecutionMode::SignedZeroInfNanPreserve, *targetWidth);
       // Erase target width.
@@ -259,7 +259,7 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
 
     // Insert spirv::AddressOf and spirv::AccessChain operations.
     Value replacement =
-        rewriter.create<spirv::AddressOfOp>(funcOp.getLoc(), var);
+        spirv::AddressOfOp::create(rewriter, funcOp.getLoc(), var);
     // Check if the arg is a scalar or vector type. In that case, the value
     // needs to be loaded into registers.
     // TODO: This is loading value of the scalar into registers
@@ -269,9 +269,9 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     if (cast<spirv::SPIRVType>(argType.value()).isScalarOrVector()) {
       auto zero =
           spirv::ConstantOp::getZero(indexType, funcOp.getLoc(), rewriter);
-      auto loadPtr = rewriter.create<spirv::AccessChainOp>(
+      auto loadPtr = spirv::AccessChainOp::create(rewriter,
           funcOp.getLoc(), replacement, zero.getConstant());
-      replacement = rewriter.create<spirv::LoadOp>(funcOp.getLoc(), loadPtr);
+      replacement = spirv::LoadOp::create(rewriter, funcOp.getLoc(), loadPtr);
     }
     signatureConverter.remapInput(argType.index(), replacement);
   }
@@ -308,7 +308,7 @@ void LowerABIAttributesPass::runOnOperation() {
                                             ValueRange inputs, Location loc) {
     if (inputs.size() != 1 || !isa<spirv::PointerType>(inputs[0].getType()))
       return Value();
-    return builder.create<spirv::BitcastOp>(loc, type, inputs[0]).getResult();
+    return spirv::BitcastOp::create(builder, loc, type, inputs[0]).getResult();
   });
 
   RewritePatternSet patterns(context);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
index ab5898d0e3925..8bb34de8988b8 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
@@ -65,7 +65,7 @@ void RewriteInsertsPass::runOnOperation() {
       operands.push_back(insertionOp.getObject());
 
     OpBuilder builder(lastCompositeInsertOp);
-    auto compositeConstructOp = builder.create<spirv::CompositeConstructOp>(
+    auto compositeConstructOp = spirv::CompositeConstructOp::create(builder,
         location, compositeType, operands);
 
     lastCompositeInsertOp.replaceAllUsesWith(
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index f70b3325f8725..d1fa4bbc208d7 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -669,21 +669,21 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
                               Location loc) {
   // We can only cast one value in SPIR-V.
   if (inputs.size() != 1) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   Value input = inputs.front();
 
   // Only support integer types for now. Floating point types to be implemented.
   if (!isa<IntegerType>(type)) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   auto inputType = cast<IntegerType>(input.getType());
 
   auto scalarType = dyn_cast<spirv::ScalarType>(type);
   if (!scalarType) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -691,14 +691,14 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // truncating to go back so we don't need to worry about the signedness.
   // For extension, we cannot have enough signal here to decide which op to use.
   if (inputType.getIntOrFloatBitWidth() < scalarType.getIntOrFloatBitWidth()) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
   // Boolean values would need to use different ops than normal integer values.
   if (type.isInteger(1)) {
     Value one = spirv::ConstantOp::getOne(inputType, loc, builder);
-    return builder.create<spirv::IEqualOp>(loc, input, one);
+    return spirv::IEqualOp::create(builder, loc, input, one);
   }
 
   // Check that the source integer type is supported by the environment.
@@ -708,7 +708,7 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   scalarType.getCapabilities(caps);
   if (failed(checkCapabilityRequirements(type, targetEnv, caps)) ||
       failed(checkExtensionRequirements(type, targetEnv, exts))) {
-    auto castOp = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -716,9 +716,9 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // care about signedness here. Still try to use a corresponding op for better
   // consistency though.
   if (type.isSignedInteger()) {
-    return builder.create<spirv::SConvertOp>(loc, type, input);
+    return spirv::SConvertOp::create(builder, loc, type, input);
   }
-  return builder.create<spirv::UConvertOp>(loc, type, input);
+  return spirv::UConvertOp::create(builder, loc, type, input);
 }
 
 //===----------------------------------------------------------------------===//
@@ -770,7 +770,7 @@ getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
                                            spirv::StorageClass::Input);
     std::string name = getBuiltinVarName(builtin, prefix, suffix);
     newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+        spirv::GlobalVariableOp::create(builder, loc, ptrType, name, builtin);
     break;
   }
   case spirv::BuiltIn::SubgroupId:
@@ -781,7 +781,7 @@ getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
         spirv::PointerType::get(integerType, spirv::StorageClass::Input);
     std::string name = getBuiltinVarName(builtin, prefix, suffix);
     newVarOp =
-        builder.create<spirv::GlobalVariableOp>(loc, ptrType, name, builtin);
+        spirv::GlobalVariableOp::create(builder, loc, ptrType, name, builtin);
     break;
   }
   default:
@@ -842,7 +842,7 @@ getOrInsertPushConstantVariable(Location loc, Block &block,
   auto builder = OpBuilder::atBlockBegin(&block, b.getListener());
   auto type = getPushConstantStorageType(elementCount, builder, indexType);
   const char *name = "__push_constant_var__";
-  return builder.create<spirv::GlobalVariableOp>(loc, type, name,
+  return spirv::GlobalVariableOp::create(builder, loc, type, name,
                                                  /*initializer=*/nullptr);
 }
 
@@ -879,7 +879,7 @@ struct FuncOpConversion final : OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted spirv.func op.
-    auto newFuncOp = rewriter.create<spirv::FuncOp>(
+    auto newFuncOp = spirv::FuncOp::create(rewriter,
         funcOp.getLoc(), funcOp.getName(),
         rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
                                  resultType ? TypeRange(resultType)
@@ -919,7 +919,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
     }
 
     // Create a new func op with the original type and copy the function body.
-    auto newFuncOp = rewriter.create<func::FuncOp>(funcOp.getLoc(),
+    auto newFuncOp = func::FuncOp::create(rewriter, funcOp.getLoc(),
                                                    funcOp.getName(), fnType);
     rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
                                 newFuncOp.end());
@@ -954,7 +954,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto origVecType = dyn_cast<VectorType>(origType);
       if (!origVecType) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = rewriter.create<arith::ConstantOp>(
+        Value result = arith::ConstantOp::create(rewriter,
             loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
@@ -967,7 +967,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto targetShape = getTargetShape(origVecType);
       if (!targetShape) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = rewriter.create<arith::ConstantOp>(
+        Value result = arith::ConstantOp::create(rewriter,
             loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
@@ -982,11 +982,11 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
           llvm::to_vector_of<int64_t, 4>(origVecType.getShape());
 
       // Prepare the result vector.
-      Value result = rewriter.create<arith::ConstantOp>(
+      Value result = arith::ConstantOp::create(rewriter,
           loc, origVecType, rewriter.getZeroAttr(origVecType));
       ++newOpCount;
       // Prepare the placeholder for the new arguments that will be added later.
-      Value dummy = rewriter.create<arith::ConstantOp>(
+      Value dummy = arith::ConstantOp::create(rewriter,
           loc, unrolledType, rewriter.getZeroAttr(unrolledType));
       ++newOpCount;
 
@@ -995,7 +995,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       SmallVector<Type> newTypes;
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        result = rewriter.create<vector::InsertStridedSliceOp>(
+        result = vector::InsertStridedSliceOp::create(rewriter,
             loc, dummy, result, offsets, strides);
         newTypes.push_back(unrolledType);
         unrolledInputNums.push_back(newInputNo);
@@ -1109,12 +1109,12 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
       Value returnValue = returnOp.getOperand(origResultNo);
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        Value result = rewriter.create<vector::ExtractStridedSliceOp>(
+        Value result = vector::ExtractStridedSliceOp::create(rewriter,
             loc, returnValue, offsets, extractShape, strides);
         if (originalShape.size() > 1) {
           SmallVector<int64_t> extractIndices(originalShape.size() - 1, 0);
           result =
-              rewriter.create<vector::ExtractOp>(loc, result, extractIndices);
+              vector::ExtractOp::create(rewriter, loc, result, extractIndices);
         }
         newOperands.push_back(result);
         newTypes.push_back(unrolledType);
@@ -1132,7 +1132,7 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
     // Replace the return op using the new operands. This will automatically
     // update the entry block as well.
     rewriter.replaceOp(returnOp,
-                       rewriter.create<func::ReturnOp>(loc, newOperands));
+                       func::ReturnOp::create(rewriter, loc, newOperands));
 
     return success();
   }
@@ -1157,8 +1157,8 @@ Value mlir::spirv::getBuiltinVariableValue(Operation *op,
   spirv::GlobalVariableOp varOp =
       getOrInsertBuiltinVariable(*parent->getRegion(0).begin(), op->getLoc(),
                                  builtin, integerType, builder, prefix, suffix);
-  Value ptr = builder.create<spirv::AddressOfOp>(op->getLoc(), varOp);
-  return builder.create<spirv::LoadOp>(op->getLoc(), ptr);
+  Value ptr = spirv::AddressOfOp::create(builder, op->getLoc(), varOp);
+  return spirv::LoadOp::create(builder, op->getLoc(), ptr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1179,12 +1179,12 @@ Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
       loc, parent->getRegion(0).front(), elementCount, builder, integerType);
 
   Value zeroOp = spirv::ConstantOp::getZero(integerType, loc, builder);
-  Value offsetOp = builder.create<spirv::ConstantOp>(
+  Value offsetOp = spirv::ConstantOp::create(builder,
       loc, integerType, builder.getI32IntegerAttr(offset));
-  auto addrOp = builder.create<spirv::AddressOfOp>(loc, varOp);
-  auto acOp = builder.create<spirv::AccessChainOp>(
+  auto addrOp = spirv::AddressOfOp::create(builder, loc, varOp);
+  auto acOp = spirv::AccessChainOp::create(builder,
       loc, addrOp, llvm::ArrayRef({zeroOp, offsetOp}));
-  return builder.create<spirv::LoadOp>(loc, acOp);
+  return spirv::LoadOp::create(builder, loc, acOp);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1244,7 +1244,7 @@ Value mlir::spirv::getVulkanElementPtr(const SPIRVTypeConverter &typeConverter,
     linearizedIndices.push_back(
         linearizeIndex(indices, strides, offset, indexType, loc, builder));
   }
-  return builder.create<spirv::AccessChainOp>(loc, basePtr, linearizedIndices);
+  return spirv::AccessChainOp::create(builder, loc, basePtr, linearizedIndices);
 }
 
 Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
@@ -1275,10 +1275,10 @@ Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
       cast<spirv::PointerType>(basePtr.getType()).getPointeeType();
   if (isa<spirv::ArrayType>(pointeeType)) {
     linearizedIndices.push_back(linearIndex);
-    return builder.create<spirv::AccessChainOp>(loc, basePtr,
+    return spirv::AccessChainOp::create(builder, loc, basePtr,
                                                 linearizedIndices);
   }
-  return builder.create<spirv::PtrAccessChainOp>(loc, basePtr, linearIndex,
+  return spirv::PtrAccessChainOp::create(builder, loc, basePtr, linearIndex,
                                                  linearizedIndices);
 }
 
@@ -1465,7 +1465,7 @@ SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr,
       });
   addTargetMaterialization([](OpBuilder &builder, Type type, ValueRange inputs,
                               Location loc) {
-    auto cast = builder.create<UnrealizedConversionCastOp>(loc, type, inputs);
+    auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return cast.getResult(0);
   });
 }
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
index af1cf2a1373e3..17f79780afa20 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
@@ -64,16 +64,16 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   //     and 4 additions after constant folding.
   //   - With sign-extended arguments, we end up emitting 8 multiplications and
   //     and 12 additions after CSE.
-  Value cstLowMask = rewriter.create<ConstantOp>(
+  Value cstLowMask = ConstantOp::create(rewriter,
       loc, lhs.getType(), getScalarOrSplatAttr(argTy, (1 << 16) - 1));
   auto getLowDigit = [&rewriter, loc, cstLowMask](Value val) {
-    return rewriter.create<BitwiseAndOp>(loc, val, cstLowMask);
+    return BitwiseAndOp::create(rewriter, loc, val, cstLowMask);
   };
 
-  Value cst16 = rewriter.create<ConstantOp>(loc, lhs.getType(),
+  Value cst16 = ConstantOp::create(rewriter, loc, lhs.getType(),
                                             getScalarOrSplatAttr(argTy, 16));
   auto getHighDigit = [&rewriter, loc, cst16](Value val) {
-    return rewriter.create<ShiftRightLogicalOp>(loc, val, cst16);
+    return ShiftRightLogicalOp::create(rewriter, loc, val, cst16);
   };
 
   auto getSignDigit = [&rewriter, loc, cst16, &getHighDigit](Value val) {
@@ -82,10 +82,10 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
     // fine. We do not have to introduce an extra constant since any
     // value in [15, 32) would do.
     return getHighDigit(
-        rewriter.create<ShiftRightArithmeticOp>(loc, val, cst16));
+        ShiftRightArithmeticOp::create(rewriter, loc, val, cst16));
   };
 
-  Value cst0 = rewriter.create<ConstantOp>(loc, lhs.getType(),
+  Value cst0 = ConstantOp::create(rewriter, loc, lhs.getType(),
                                            getScalarOrSplatAttr(argTy, 0));
 
   Value lhsLow = getLowDigit(lhs);
@@ -108,7 +108,7 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
         continue;
 
       Value &thisResDigit = resultDigits[i + j];
-      Value mul = rewriter.create<IMulOp>(loc, lhsDigit, rhsDigit);
+      Value mul = IMulOp::create(rewriter, loc, lhsDigit, rhsDigit);
       Value current = rewriter.createOrFold<IAddOp>(loc, thisResDigit, mul);
       thisResDigit = getLowDigit(current);
 
@@ -122,13 +122,13 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   }
 
   auto combineDigits = [loc, cst16, &rewriter](Value low, Value high) {
-    Value highBits = rewriter.create<ShiftLeftLogicalOp>(loc, high, cst16);
-    return rewriter.create<BitwiseOrOp>(loc, low, highBits);
+    Value highBits = ShiftLeftLogicalOp::create(rewriter, loc, high, cst16);
+    return BitwiseOrOp::create(rewriter, loc, low, highBits);
   };
   Value low = combineDigits(resultDigits[0], resultDigits[1]);
   Value high = combineDigits(resultDigits[2], resultDigits[3]);
 
-  return rewriter.create<CompositeConstructOp>(
+  return CompositeConstructOp::create(rewriter,
       loc, mulOp->getResultTypes().front(), llvm::ArrayRef({low, high}));
 }
 
@@ -185,16 +185,16 @@ struct ExpandAddCarryPattern final : OpRewritePattern<IAddCarryOp> {
           llvm::formatv("Unexpected integer type for WebGPU: '{0}'", elemTy));
 
     Value one =
-        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 1));
+        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 1));
     Value zero =
-        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 0));
+        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 0));
 
     // Calculate the carry by checking if the addition resulted in an overflow.
-    Value out = rewriter.create<IAddOp>(loc, lhs, rhs);
-    Value cmp = rewriter.create<ULessThanOp>(loc, out, lhs);
-    Value carry = rewriter.create<SelectOp>(loc, cmp, one, zero);
+    Value out = IAddOp::create(rewriter, loc, lhs, rhs);
+    Value cmp = ULessThanOp::create(rewriter, loc, out, lhs);
+    Value carry = SelectOp::create(rewriter, loc, cmp, one, zero);
 
-    Value add = rewriter.create<CompositeConstructOp>(
+    Value add = CompositeConstructOp::create(rewriter,
         loc, op->getResultTypes().front(), llvm::ArrayRef({out, carry}));
 
     rewriter.replaceOp(op, add);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
index 527d92634c196..e7bb2d76666fb 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
@@ -380,13 +380,13 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = dstNumBytes / srcNumBytes;
-      auto ratioValue = rewriter.create<spirv::ConstantOp>(
+      auto ratioValue = spirv::ConstantOp::create(rewriter,
           loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
-          rewriter.create<spirv::SDivOp>(loc, indexType, oldIndex, ratioValue);
+          spirv::SDivOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
       indices.push_back(
-          rewriter.create<spirv::SModOp>(loc, indexType, oldIndex, ratioValue));
+          spirv::SModOp::create(rewriter, loc, indexType, oldIndex, ratioValue));
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -407,11 +407,11 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = srcNumBytes / dstNumBytes;
-      auto ratioValue = rewriter.create<spirv::ConstantOp>(
+      auto ratioValue = spirv::ConstantOp::create(rewriter,
           loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
-          rewriter.create<spirv::IMulOp>(loc, indexType, oldIndex, ratioValue);
+          spirv::IMulOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -435,14 +435,14 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
     auto dstElemType = cast<spirv::SPIRVType>(dstPtrType.getPointeeType());
 
     Location loc = loadOp.getLoc();
-    auto newLoadOp = rewriter.create<spirv::LoadOp>(loc, adaptor.getPtr());
+    auto newLoadOp = spirv::LoadOp::create(rewriter, loc, adaptor.getPtr());
     if (srcElemType == dstElemType) {
       rewriter.replaceOp(loadOp, newLoadOp->getResults());
       return success();
     }
 
     if (areSameBitwidthScalarType(srcElemType, dstElemType)) {
-      auto castOp = rewriter.create<spirv::BitcastOp>(loc, srcElemType,
+      auto castOp = spirv::BitcastOp::create(rewriter, loc, srcElemType,
                                                       newLoadOp.getValue());
       rewriter.replaceOp(loadOp, castOp->getResults());
 
@@ -475,14 +475,14 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
       auto indices = llvm::to_vector<4>(acOp.getIndices());
       for (int i = 1; i < ratio; ++i) {
         // Load all subsequent components belonging to this element.
-        indices.back() = rewriter.create<spirv::IAddOp>(
+        indices.back() = spirv::IAddOp::create(rewriter,
             loc, i32Type, indices.back(), oneValue);
-        auto componentAcOp = rewriter.create<spirv::AccessChainOp>(
+        auto componentAcOp = spirv::AccessChainOp::create(rewriter,
             loc, acOp.getBasePtr(), indices);
         // Assuming little endian, this reads lower-ordered bits of the number
         // to lower-numbered components of the vector.
         components.push_back(
-            rewriter.create<spirv::LoadOp>(loc, componentAcOp));
+            spirv::LoadOp::create(rewriter, loc, componentAcOp));
       }
 
       // Create a vector of the components and then cast back to the larger
@@ -510,15 +510,15 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
               castType = VectorType::get({count}, castType);
 
             for (Value &c : components)
-              c = rewriter.create<spirv::BitcastOp>(loc, castType, c);
+              c = spirv::BitcastOp::create(rewriter, loc, castType, c);
           }
         }
-      Value vectorValue = rewriter.create<spirv::CompositeConstructOp>(
+      Value vectorValue = spirv::CompositeConstructOp::create(rewriter,
           loc, vectorType, components);
 
       if (!isa<VectorType>(srcElemType))
         vectorValue =
-            rewriter.create<spirv::BitcastOp>(loc, srcElemType, vectorValue);
+            spirv::BitcastOp::create(rewriter, loc, srcElemType, vectorValue);
       rewriter.replaceOp(loadOp, vectorValue);
       return success();
     }
@@ -546,7 +546,7 @@ struct ConvertStore : public ConvertAliasResource<spirv::StoreOp> {
     Location loc = storeOp.getLoc();
     Value value = adaptor.getValue();
     if (srcElemType != dstElemType)
-      value = rewriter.create<spirv::BitcastOp>(loc, dstElemType, value);
+      value = spirv::BitcastOp::create(rewriter, loc, dstElemType, value);
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, adaptor.getPtr(),
                                                 value, storeOp->getAttrs());
     return success();
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 5f395eef9d601..7339ba5c2379c 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -151,16 +151,16 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
   if (auto poison = dyn_cast<ub::PoisonAttr>(value))
-    return builder.create<ub::PoisonOp>(loc, type, poison);
+    return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (llvm::isa<ShapeType>(type) || isExtentTensorType(type))
-    return builder.create<ConstShapeOp>(
+    return ConstShapeOp::create(builder,
         loc, type, llvm::cast<DenseIntElementsAttr>(value));
   if (llvm::isa<SizeType>(type))
-    return builder.create<ConstSizeOp>(loc, type,
+    return ConstSizeOp::create(builder, loc, type,
                                        llvm::cast<IntegerAttr>(value));
   if (llvm::isa<WitnessType>(type))
-    return builder.create<ConstWitnessOp>(loc, type,
+    return ConstWitnessOp::create(builder, loc, type,
                                           llvm::cast<BoolAttr>(value));
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
@@ -316,7 +316,7 @@ struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
     auto newYieldOp =
         rewriter.replaceOpWithNewOp<AssumingYieldOp>(yieldOp, newYieldOperands);
     rewriter.setInsertionPoint(op);
-    auto newOp = rewriter.create<AssumingOp>(
+    auto newOp = AssumingOp::create(rewriter,
         op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
     newOp.getDoRegion().takeBody(op.getDoRegion());
 
@@ -385,7 +385,7 @@ void AssumingOp::build(
 
   // Build body.
   SmallVector<Value, 2> yieldValues = bodyBuilder(builder, result.location);
-  builder.create<AssumingYieldOp>(result.location, yieldValues);
+  AssumingYieldOp::create(builder, result.location, yieldValues);
 
   SmallVector<Type, 2> assumingTypes;
   for (Value v : yieldValues)
@@ -736,13 +736,13 @@ struct BroadcastForwardSingleOperandPattern
     if (replacement.getType() != op.getType()) {
       auto loc = op.getLoc();
       if (llvm::isa<ShapeType>(op.getType())) {
-        replacement = rewriter.create<FromExtentTensorOp>(loc, replacement);
+        replacement = FromExtentTensorOp::create(rewriter, loc, replacement);
       } else {
         assert(!llvm::isa<ShapeType>(op.getType()) &&
                !llvm::isa<ShapeType>(replacement.getType()) &&
                "expect extent tensor cast");
         replacement =
-            rewriter.create<tensor::CastOp>(loc, op.getType(), replacement);
+            tensor::CastOp::create(rewriter, loc, op.getType(), replacement);
       }
     }
 
@@ -780,7 +780,7 @@ struct BroadcastFoldConstantOperandsPattern
     auto foldedConstantOperandsTy = RankedTensorType::get(
         {static_cast<int64_t>(foldedConstantShape.size())},
         rewriter.getIndexType());
-    newShapeOperands.push_back(rewriter.create<ConstShapeOp>(
+    newShapeOperands.push_back(ConstShapeOp::create(rewriter,
         op.getLoc(), foldedConstantOperandsTy,
         rewriter.getIndexTensorAttr(foldedConstantShape)));
     rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(),
@@ -845,7 +845,7 @@ struct BroadcastConcretizeResultTypePattern
       }
     }
 
-    auto newOp = rewriter.create<BroadcastOp>(
+    auto newOp = BroadcastOp::create(rewriter,
         op.getLoc(), getExtentTensorType(getContext(), maxRank),
         op.getShapes());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
@@ -1354,11 +1354,11 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
   auto loc = result.location;
   auto dimAttr = builder.getIndexAttr(dim);
   if (llvm::isa<ShapeType>(shape.getType())) {
-    Value dim = builder.create<ConstSizeOp>(loc, dimAttr);
+    Value dim = ConstSizeOp::create(builder, loc, dimAttr);
     build(builder, result, builder.getType<SizeType>(), shape, dim);
   } else {
     Value dim =
-        builder.create<arith::ConstantOp>(loc, builder.getIndexType(), dimAttr);
+        arith::ConstantOp::create(builder, loc, builder.getIndexType(), dimAttr);
     build(builder, result, builder.getIndexType(), shape, dim);
   }
 }
@@ -1708,7 +1708,7 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
                                   rewriter.getIndexTensorAttr(type.getShape()))
             .getResult();
     if (constShape.getType() != op.getResult().getType())
-      constShape = rewriter.create<tensor::CastOp>(
+      constShape = tensor::CastOp::create(rewriter,
           loc, op.getResult().getType(), constShape);
     rewriter.replaceOp(op, constShape);
     return success();
@@ -1751,10 +1751,10 @@ struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
 
     if (opTensorTy != shapeTensorTy) {
       if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
-        shape = rewriter.create<tensor::CastOp>(op.getLoc(), opTensorTy, shape);
+        shape = tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
       else if (!isExtentTensorType(shapeTensorTy))
         shape =
-            rewriter.create<arith::IndexCastOp>(op.getLoc(), opTensorTy, shape);
+            arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
     }
 
     rewriter.replaceOp(op, shape);
diff --git a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
index 8a471c12d21e4..5533a7e914bf0 100644
--- a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -57,7 +57,7 @@ struct AssumingOpInterface
 
     // Create new op and move over region.
     TypeRange newResultTypes(yieldOp.getOperands());
-    auto newOp = rewriter.create<shape::AssumingOp>(
+    auto newOp = shape::AssumingOp::create(rewriter,
         op->getLoc(), newResultTypes, assumingOp.getWitness());
     newOp.getDoRegion().takeBody(assumingOp.getRegion());
 
@@ -66,7 +66,7 @@ struct AssumingOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(assumingOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
+        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
             assumingOp.getLoc(), it.value(), newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
diff --git a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
index ae06a34b65709..522347ff2b993 100644
--- a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
@@ -69,7 +69,7 @@ createFuncFromCluster(OpBuilder &b, const SmallVector<Operation *, 8> &cluster,
       cluster.empty()
           ? b.getFunctionType(shape.getType(), shape.getType())
           : b.getFunctionType(ValueRange(inputs).getTypes(), shape.getType());
-  shape::FuncOp fnOp = b.create<shape::FuncOp>(loc, fnName, fnType);
+  shape::FuncOp fnOp = shape::FuncOp::create(b, loc, fnName, fnType);
   Block *block = fnOp.addEntryBlock();
   b.setInsertionPointToEnd(block);
   IRMapping bvm;
@@ -85,7 +85,7 @@ createFuncFromCluster(OpBuilder &b, const SmallVector<Operation *, 8> &cluster,
   llvm::SmallVector<Value, 4> fnReturns;
   fnReturns.push_back(bvm.lookupOrDefault(shape));
 
-  b.create<shape::ReturnOp>(loc, fnReturns);
+  shape::ReturnOp::create(b, loc, fnReturns);
   fnOp.setPrivate();
   return std::make_pair(fnOp, inputs);
 }
@@ -187,7 +187,7 @@ class TensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
   LogicalResult matchAndRewrite(tensor::DimOp op,
                                 PatternRewriter &rewriter) const override {
     auto shapeOf =
-        rewriter.create<shape::ShapeOfOp>(op.getLoc(), op.getSource());
+        shape::ShapeOfOp::create(rewriter, op.getLoc(), op.getSource());
     rewriter.replaceOpWithNewOp<shape::GetExtentOp>(op, op.getType(), shapeOf,
                                                     op.getIndex());
     return success();
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index 121e0cc133e19..b5f6230bc2362 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -44,14 +44,14 @@ NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
                    ->materializeConstant(rewriter, rewriter.getIndexAttr(1),
                                          valueType, loc)
                    ->getResult(0);
-  ReduceOp reduce = rewriter.create<ReduceOp>(loc, op.getShape(), init);
+  ReduceOp reduce = ReduceOp::create(rewriter, loc, op.getShape(), init);
 
   // Generate reduce operator.
   Block *body = reduce.getBody();
   OpBuilder b = OpBuilder::atBlockEnd(body);
-  Value product = b.create<MulOp>(loc, valueType, body->getArgument(1),
+  Value product = MulOp::create(b, loc, valueType, body->getArgument(1),
                                   body->getArgument(2));
-  b.create<shape::YieldOp>(loc, product);
+  shape::YieldOp::create(b, loc, product);
 
   rewriter.replaceOp(op, reduce.getResult());
   return success();
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index 38246b96977c8..e2a6a59636a84 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -559,7 +559,7 @@ SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
   SmallVector<Type> retType(
       dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
       builder.getIndexType());
-  auto transOp = builder.create<CrdTranslateOp>(loc, retType, crds, dir, *this);
+  auto transOp = CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
   return transOp.getOutCrds();
 }
 
@@ -1481,7 +1481,7 @@ LogicalResult CrdTranslateOp::fold(FoldAdaptor adaptor,
 
 void LvlOp::build(OpBuilder &builder, OperationState &state, Value source,
                   int64_t index) {
-  Value val = builder.create<arith::ConstantIndexOp>(state.location, index);
+  Value val = arith::ConstantIndexOp::create(builder, state.location, index);
   return build(builder, state, source, val);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
index 9c84f4c25866f..74842cc1f310b 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
@@ -41,7 +41,7 @@ LogicalResult sparse_tensor::detail::stageWithSortImpl(
 
   // -> sort
   Type dstCOOTp = dstStt.getCOOType(/*ordered=*/true);
-  Value dstCOO = rewriter.create<ReorderCOOOp>(
+  Value dstCOO = ReorderCOOOp::create(rewriter,
       loc, dstCOOTp, srcCOO, SparseTensorSortKind::HybridQuickSort);
 
   // -> dest.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
index 8ee801ba46349..6545bfbcac5c7 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
@@ -88,13 +88,13 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
         } else if (directOut) {
           Value mem;
           if (kind == SparseTensorFieldKind::PosMemRef)
-            mem = builder.create<sparse_tensor::ToPositionsOp>(loc, inputs[0],
+            mem = sparse_tensor::ToPositionsOp::create(builder, loc, inputs[0],
                                                                lv);
           else if (kind == SparseTensorFieldKind::CrdMemRef)
-            mem = builder.create<sparse_tensor::ToCoordinatesOp>(loc, inputs[0],
+            mem = sparse_tensor::ToCoordinatesOp::create(builder, loc, inputs[0],
                                                                  lv);
           else
-            mem = builder.create<sparse_tensor::ToValuesOp>(loc, inputs[0]);
+            mem = sparse_tensor::ToValuesOp::create(builder, loc, inputs[0]);
           toVals.push_back(mem);
         } else {
           ShapedType rtp = cast<ShapedType>(t);
@@ -109,7 +109,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
 
     if (isIn) {
       // Assemble multiple inputs into a single sparse tensor.
-      auto a = builder.create<sparse_tensor::AssembleOp>(loc, rtp, inputs);
+      auto a = sparse_tensor::AssembleOp::create(builder, loc, rtp, inputs);
       toVals.push_back(a.getResult());
     } else if (!directOut) {
       // Disassemble a single sparse input into multiple outputs.
@@ -117,7 +117,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
       unsigned len = retTypes.size();
       retTypes.append(cntTypes);
       auto d =
-          builder.create<sparse_tensor::DisassembleOp>(loc, retTypes, inputs);
+          sparse_tensor::DisassembleOp::create(builder, loc, retTypes, inputs);
       for (unsigned i = 0; i < len; i++)
         toVals.push_back(d.getResult(i));
     }
@@ -199,7 +199,7 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     OpBuilder moduleBuilder(modOp.getBodyRegion());
     unsigned extra = inputTypes.size();
     inputTypes.append(extraTypes);
-    auto func = moduleBuilder.create<func::FuncOp>(
+    auto func = func::FuncOp::create(moduleBuilder,
         loc, orgName, FunctionType::get(context, inputTypes, outputTypes));
     func.setPublic();
 
@@ -216,14 +216,14 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     // Call the original, now private method. A subsequent inlining pass can
     // determine whether cloning the method body in place is worthwhile.
     auto org = SymbolRefAttr::get(context, wrapper);
-    auto call = rewriter.create<func::CallOp>(loc, funcOp.getResultTypes(), org,
+    auto call = func::CallOp::create(rewriter, loc, funcOp.getResultTypes(), org,
                                               inputs);
 
     // Convert outputs and return.
     SmallVector<Value> outputs;
     convVals(rewriter, loc, funcOp.getResultTypes(), call.getResults(),
              body->getArguments(), outputs, extra, /*isIn=*/false, directOut);
-    rewriter.create<func::ReturnOp>(loc, outputs);
+    func::ReturnOp::create(rewriter, loc, outputs);
 
     // Finally, migrate a potential c-interface property.
     if (funcOp->getAttrOfType<UnitAttr>(
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
index 0c5912bb73772..12507ff1c5251 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
@@ -94,7 +94,7 @@ static FlatSymbolRefAttr getMangledSortHelperFunc(
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPoint(insertPoint);
     Location loc = insertPoint.getLoc();
-    func = builder.create<func::FuncOp>(
+    func = func::FuncOp::create(builder,
         loc, nameOstream.str(),
         FunctionType::get(context, operands.getTypes(), resultTypes));
     func.setPrivate();
@@ -111,13 +111,13 @@ static void forEachIJPairInXs(
     uint64_t ny,
     function_ref<void(uint64_t, Value, Value, Value)> bodyBuilder) {
   Value cstep = constantIndex(builder, loc, xPerm.getNumResults() + ny);
-  Value iOffset = builder.create<arith::MulIOp>(loc, args[0], cstep);
-  Value jOffset = builder.create<arith::MulIOp>(loc, args[1], cstep);
+  Value iOffset = arith::MulIOp::create(builder, loc, args[0], cstep);
+  Value jOffset = arith::MulIOp::create(builder, loc, args[1], cstep);
   for (unsigned k = 0, e = xPerm.getNumResults(); k < e; k++) {
     unsigned actualK = cast<AffineDimExpr>(xPerm.getResult(k)).getPosition();
     Value ak = constantIndex(builder, loc, actualK);
-    Value i = builder.create<arith::AddIOp>(loc, ak, iOffset);
-    Value j = builder.create<arith::AddIOp>(loc, ak, jOffset);
+    Value i = arith::AddIOp::create(builder, loc, ak, iOffset);
+    Value j = arith::AddIOp::create(builder, loc, ak, jOffset);
     Value buffer = args[xStartIdx];
 
     bodyBuilder(k, i, j, buffer);
@@ -165,10 +165,10 @@ static void forEachIJPairInAllBuffers(
 static void createSwap(OpBuilder &builder, Location loc, ValueRange args,
                        AffineMap xPerm, uint64_t ny) {
   auto swapOnePair = [&](uint64_t unused, Value i, Value j, Value buffer) {
-    Value vi = builder.create<memref::LoadOp>(loc, buffer, i);
-    Value vj = builder.create<memref::LoadOp>(loc, buffer, j);
-    builder.create<memref::StoreOp>(loc, vj, buffer, i);
-    builder.create<memref::StoreOp>(loc, vi, buffer, j);
+    Value vi = memref::LoadOp::create(builder, loc, buffer, i);
+    Value vj = memref::LoadOp::create(builder, loc, buffer, j);
+    memref::StoreOp::create(builder, loc, vj, buffer, i);
+    memref::StoreOp::create(builder, loc, vi, buffer, j);
   };
 
   forEachIJPairInAllBuffers(builder, loc, args, xPerm, ny, swapOnePair);
@@ -193,7 +193,7 @@ static Value createInlinedCompareImplementation(
       OpBuilder::InsertionGuard insertionGuard(builder);
       auto ifOp = cast<scf::IfOp>(val.getDefiningOp());
       builder.setInsertionPointAfter(ifOp);
-      builder.create<scf::YieldOp>(loc, ifOp.getResult(0));
+      scf::YieldOp::create(builder, loc, ifOp.getResult(0));
     }
   };
 
@@ -207,25 +207,25 @@ static Value createInlinedCompareImplementation(
 /// result of the comparison.
 static Value createEqCompare(OpBuilder &builder, Location loc, Value i, Value j,
                              Value x, bool isFirstDim, bool isLastDim) {
-  Value vi = builder.create<memref::LoadOp>(loc, x, i);
-  Value vj = builder.create<memref::LoadOp>(loc, x, j);
+  Value vi = memref::LoadOp::create(builder, loc, x, i);
+  Value vj = memref::LoadOp::create(builder, loc, x, j);
 
   Value res;
   if (isLastDim) {
-    res = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, vi, vj);
+    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
-      builder.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(builder, loc, res);
   } else {
     Value ne =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, vi, vj);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIntegerType(1),
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
                                                ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value f = constantI1(builder, loc, false);
-    builder.create<scf::YieldOp>(loc, f);
+    scf::YieldOp::create(builder, loc, f);
 
     // If (x[i] == x[j]). Set up the insertion point for the nested if-stmt that
     // checks the remaining dimensions.
@@ -261,26 +261,26 @@ static Value createInlinedEqCompare(OpBuilder &builder, Location loc,
 static Value createLessThanCompare(OpBuilder &builder, Location loc, Value i,
                                    Value j, Value x, bool isFirstDim,
                                    bool isLastDim) {
-  Value vi = builder.create<memref::LoadOp>(loc, x, i);
-  Value vj = builder.create<memref::LoadOp>(loc, x, j);
+  Value vi = memref::LoadOp::create(builder, loc, x, i);
+  Value vj = memref::LoadOp::create(builder, loc, x, j);
 
   Value res;
   if (isLastDim) {
-    res = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, vi, vj);
+    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
-      builder.create<scf::YieldOp>(loc, res);
+      scf::YieldOp::create(builder, loc, res);
   } else {
     Value ne =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, vi, vj);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIntegerType(1),
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
                                                ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value lt =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, vi, vj);
-    builder.create<scf::YieldOp>(loc, lt);
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
+    scf::YieldOp::create(builder, loc, lt);
 
     // If (x[i] == x[j]). Set up the insertion point for the nested if-stmt that
     // checks the remaining dimensions.
@@ -337,17 +337,17 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value p = args[hiIdx];
   SmallVector<Type, 2> types(2, p.getType()); // Only two types.
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(
+  scf::WhileOp whileOp = scf::WhileOp::create(builder,
       loc, types, SmallVector<Value, 2>{args[loIdx], args[hiIdx]});
 
   // The before-region of the WhileOp.
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, types, {loc, loc});
   builder.setInsertionPointToEnd(before);
-  Value cond1 = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                               before->getArgument(0),
                                               before->getArgument(1));
-  builder.create<scf::ConditionOp>(loc, cond1, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond1, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -357,9 +357,9 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   Value hi = after->getArgument(1);
   // Compute mid = (lo + hi) >> 1.
   Value c1 = constantIndex(builder, loc, 1);
-  Value mid = builder.create<arith::ShRUIOp>(
-      loc, builder.create<arith::AddIOp>(loc, lo, hi), c1);
-  Value midp1 = builder.create<arith::AddIOp>(loc, mid, c1);
+  Value mid = arith::ShRUIOp::create(builder,
+      loc, arith::AddIOp::create(builder, loc, lo, hi), c1);
+  Value midp1 = arith::AddIOp::create(builder, loc, mid, c1);
 
   // Compare xs[p] < xs[mid].
   SmallVector<Value> compareOperands{p, mid};
@@ -372,12 +372,12 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   //     hi = mid;
   //   else
   //     lo = mid + 1;
-  Value newLo = builder.create<arith::SelectOp>(loc, cond2, lo, midp1);
-  Value newHi = builder.create<arith::SelectOp>(loc, cond2, mid, hi);
-  builder.create<scf::YieldOp>(loc, ValueRange{newLo, newHi});
+  Value newLo = arith::SelectOp::create(builder, loc, cond2, lo, midp1);
+  Value newHi = arith::SelectOp::create(builder, loc, cond2, mid, hi);
+  scf::YieldOp::create(builder, loc, ValueRange{newLo, newHi});
 
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc, whileOp.getResult(0));
+  func::ReturnOp::create(builder, loc, whileOp.getResult(0));
 }
 
 /// Creates code to advance i in a loop based on xs[p] as follows:
@@ -393,7 +393,7 @@ static std::pair<Value, Value> createScanLoop(OpBuilder &builder,
                                               uint64_t ny, int step) {
   Location loc = func.getLoc();
   scf::WhileOp whileOp =
-      builder.create<scf::WhileOp>(loc, TypeRange{i.getType()}, ValueRange{i});
+      scf::WhileOp::create(builder, loc, TypeRange{i.getType()}, ValueRange{i});
 
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, {i.getType()}, {loc});
@@ -409,14 +409,14 @@ static std::pair<Value, Value> createScanLoop(OpBuilder &builder,
   }
   compareOperands.append(xs.begin(), xs.end());
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  builder.create<scf::ConditionOp>(loc, cond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond, before->getArguments());
 
   Block *after =
       builder.createBlock(&whileOp.getAfter(), {}, {i.getType()}, {loc});
   builder.setInsertionPointToEnd(after);
   Value cs = constantIndex(builder, loc, step);
-  i = builder.create<arith::AddIOp>(loc, after->getArgument(0), cs);
-  builder.create<scf::YieldOp>(loc, ValueRange{i});
+  i = arith::AddIOp::create(builder, loc, after->getArgument(0), cs);
+  scf::YieldOp::create(builder, loc, ValueRange{i});
   i = whileOp.getResult(0);
 
   builder.setInsertionPointAfter(whileOp);
@@ -440,7 +440,7 @@ static scf::IfOp createCompareThenSwap(OpBuilder &builder, Location loc,
   compareOperands[0] = b;
   compareOperands[1] = a;
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else=*/false);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, cond, /*else=*/false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   swapOperands[0] = b;
   swapOperands[1] = a;
@@ -517,12 +517,12 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
   swapOperands.append(args.begin() + xStartIdx, args.end());
   Location loc = func.getLoc();
   Value c1 = constantIndex(builder, loc, 1);
-  Value hiP1 = builder.create<arith::AddIOp>(loc, hi, c1);
-  Value len = builder.create<arith::SubIOp>(loc, hiP1, lo);
+  Value hiP1 = arith::AddIOp::create(builder, loc, hi, c1);
+  Value len = arith::SubIOp::create(builder, loc, hiP1, lo);
   Value lenThreshold = constantIndex(builder, loc, 1000);
-  Value lenCond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value lenCond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                                 len, lenThreshold);
-  scf::IfOp lenIf = builder.create<scf::IfOp>(loc, lenCond, /*else=*/true);
+  scf::IfOp lenIf = scf::IfOp::create(builder, loc, lenCond, /*else=*/true);
 
   // When len < 1000, choose pivot from median of 3 values.
   builder.setInsertionPointToStart(&lenIf.getThenRegion().front());
@@ -531,13 +531,13 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
 
   // When len >= 1000, choose pivot from median of 5 values.
   builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
-  Value miP1 = builder.create<arith::AddIOp>(loc, hi, c1);
-  Value a = builder.create<arith::AddIOp>(loc, lo, miP1);
+  Value miP1 = arith::AddIOp::create(builder, loc, hi, c1);
+  Value a = arith::AddIOp::create(builder, loc, lo, miP1);
   // Value a is the middle between [loc, mi].
-  a = builder.create<arith::ShRUIOp>(loc, a, c1);
-  Value b = builder.create<arith::AddIOp>(loc, mi, hiP1);
+  a = arith::ShRUIOp::create(builder, loc, a, c1);
+  Value b = arith::AddIOp::create(builder, loc, mi, hiP1);
   // Value b is the middle between [mi, hi].
-  b = builder.create<arith::ShRUIOp>(loc, b, c1);
+  b = arith::ShRUIOp::create(builder, loc, b, c1);
   createSort5(builder, loc, xPerm, ny, swapOperands, compareOperands, lo, a, mi,
               b, hi);
 
@@ -589,24 +589,24 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value sum = builder.create<arith::AddIOp>(loc, lo, hi);
+  Value sum = arith::AddIOp::create(builder, loc, lo, hi);
   Value c1 = constantIndex(builder, loc, 1);
-  Value p = builder.create<arith::ShRUIOp>(loc, sum, c1);
+  Value p = arith::ShRUIOp::create(builder, loc, sum, c1);
 
   Value i = lo;
-  Value j = builder.create<arith::SubIOp>(loc, hi, c1);
+  Value j = arith::SubIOp::create(builder, loc, hi, c1);
   createChoosePivot(builder, module, func, xPerm, ny, i, j, p, args);
   Value trueVal = constantI1(builder, loc, true); // The value for while (true)
   SmallVector<Value, 4> operands{i, j, p, trueVal}; // Exactly four values.
   SmallVector<Type, 4> types{i.getType(), j.getType(), p.getType(),
                              trueVal.getType()};
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(loc, types, operands);
+  scf::WhileOp whileOp = scf::WhileOp::create(builder, loc, types, operands);
 
   // The before-region of the WhileOp.
   Block *before = builder.createBlock(&whileOp.getBefore(), {}, types,
                                       {loc, loc, loc, loc});
   builder.setInsertionPointToEnd(before);
-  builder.create<scf::ConditionOp>(loc, before->getArgument(3),
+  scf::ConditionOp::create(builder, loc, before->getArgument(3),
                                    before->getArguments());
 
   // The after-region of the WhileOp.
@@ -629,70 +629,70 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
 
   // If i < j:
   Value cond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, i, j);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, i, j);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   SmallVector<Value> swapOperands{i, j};
   swapOperands.append(args.begin() + xStartIdx, args.end());
   createSwap(builder, loc, swapOperands, xPerm, ny);
   // If the pivot is moved, update p with the new pivot.
   Value icond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, i, p);
-  scf::IfOp ifOpI = builder.create<scf::IfOp>(loc, TypeRange{p.getType()},
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, i, p);
+  scf::IfOp ifOpI = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
                                               icond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpI.getThenRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{j});
+  scf::YieldOp::create(builder, loc, ValueRange{j});
   builder.setInsertionPointToStart(&ifOpI.getElseRegion().front());
   Value jcond =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq, j, p);
-  scf::IfOp ifOpJ = builder.create<scf::IfOp>(loc, TypeRange{p.getType()},
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, j, p);
+  scf::IfOp ifOpJ = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
                                               jcond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpJ.getThenRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{i});
+  scf::YieldOp::create(builder, loc, ValueRange{i});
   builder.setInsertionPointToStart(&ifOpJ.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{p});
+  scf::YieldOp::create(builder, loc, ValueRange{p});
   builder.setInsertionPointAfter(ifOpJ);
-  builder.create<scf::YieldOp>(loc, ifOpJ.getResults());
+  scf::YieldOp::create(builder, loc, ifOpJ.getResults());
   builder.setInsertionPointAfter(ifOpI);
   Value compareEqIJ =
-      builder.create<arith::AndIOp>(loc, iCompareEq, jCompareEq);
-  scf::IfOp ifOp2 = builder.create<scf::IfOp>(
+      arith::AndIOp::create(builder, loc, iCompareEq, jCompareEq);
+  scf::IfOp ifOp2 = scf::IfOp::create(builder,
       loc, TypeRange{i.getType(), j.getType()}, compareEqIJ, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
-  Value i2 = builder.create<arith::AddIOp>(loc, i, c1);
-  Value j2 = builder.create<arith::SubIOp>(loc, j, c1);
-  builder.create<scf::YieldOp>(loc, ValueRange{i2, j2});
+  Value i2 = arith::AddIOp::create(builder, loc, i, c1);
+  Value j2 = arith::SubIOp::create(builder, loc, j, c1);
+  scf::YieldOp::create(builder, loc, ValueRange{i2, j2});
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{i, j});
+  scf::YieldOp::create(builder, loc, ValueRange{i, j});
   builder.setInsertionPointAfter(ifOp2);
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc,
       ValueRange{ifOp2.getResult(0), ifOp2.getResult(1), ifOpI.getResult(0),
                  /*cont=*/constantI1(builder, loc, true)});
 
   // False branch for if i < j (i.e., i >= j):
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  p = builder.create<arith::AddIOp>(loc, j,
+  p = arith::AddIOp::create(builder, loc, j,
                                     constantOne(builder, loc, j.getType()));
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc, ValueRange{i, j, p, /*cont=*/constantI1(builder, loc, false)});
 
   // Return for the whileOp.
   builder.setInsertionPointAfter(ifOp);
-  builder.create<scf::YieldOp>(loc, ifOp.getResults());
+  scf::YieldOp::create(builder, loc, ifOp.getResults());
 
   // Return for the function.
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc, whileOp.getResult(2));
+  func::ReturnOp::create(builder, loc, whileOp.getResult(2));
 }
 
 /// Computes (n-2)/n, assuming n has index type.
 static Value createSubTwoDividedByTwo(OpBuilder &builder, Location loc,
                                       Value n) {
   Value i2 = constantIndex(builder, loc, 2);
-  Value res = builder.create<arith::SubIOp>(loc, n, i2);
+  Value res = arith::SubIOp::create(builder, loc, n, i2);
   Value i1 = constantIndex(builder, loc, 1);
-  return builder.create<arith::ShRUIOp>(loc, res, i1);
+  return arith::ShRUIOp::create(builder, loc, res, i1);
 }
 
 /// Creates a function to heapify the subtree with root `start` within the full
@@ -743,16 +743,16 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   // If (n >= 2).
   Value c2 = constantIndex(builder, loc, 2);
   Value condN =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, n, c2);
-  scf::IfOp ifN = builder.create<scf::IfOp>(loc, condN, /*else=*/false);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, n, c2);
+  scf::IfOp ifN = scf::IfOp::create(builder, loc, condN, /*else=*/false);
   builder.setInsertionPointToStart(&ifN.getThenRegion().front());
-  Value child = builder.create<arith::SubIOp>(loc, start, first);
+  Value child = arith::SubIOp::create(builder, loc, start, first);
 
   // If ((n-2)/2 >= child).
   Value t = createSubTwoDividedByTwo(builder, loc, n);
   Value condNc =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp ifNc = builder.create<scf::IfOp>(loc, condNc, /*else=*/false);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
+  scf::IfOp ifNc = scf::IfOp::create(builder, loc, condNc, /*else=*/false);
 
   builder.setInsertionPointToStart(&ifNc.getThenRegion().front());
   Value c1 = constantIndex(builder, loc, 1);
@@ -768,32 +768,32 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   //   if (child+1 < n && data[childIndex] < data[childIndex+1])
   //     childIndex ++; child ++ // Right child is bigger.
   auto getLargerChild = [&](Value r) -> std::pair<Value, Value> {
-    Value lChild = builder.create<arith::ShLIOp>(loc, r, c1);
-    lChild = builder.create<arith::AddIOp>(loc, lChild, c1);
-    Value lChildIdx = builder.create<arith::AddIOp>(loc, lChild, first);
-    Value rChild = builder.create<arith::AddIOp>(loc, lChild, c1);
-    Value cond1 = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+    Value lChild = arith::ShLIOp::create(builder, loc, r, c1);
+    lChild = arith::AddIOp::create(builder, loc, lChild, c1);
+    Value lChildIdx = arith::AddIOp::create(builder, loc, lChild, first);
+    Value rChild = arith::AddIOp::create(builder, loc, lChild, c1);
+    Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                                 rChild, n);
     SmallVector<Type, 2> ifTypes(2, r.getType());
     scf::IfOp if1 =
-        builder.create<scf::IfOp>(loc, ifTypes, cond1, /*else=*/true);
+        scf::IfOp::create(builder, loc, ifTypes, cond1, /*else=*/true);
     builder.setInsertionPointToStart(&if1.getThenRegion().front());
-    Value rChildIdx = builder.create<arith::AddIOp>(loc, rChild, first);
+    Value rChildIdx = arith::AddIOp::create(builder, loc, rChild, first);
     // Compare data[left] < data[right].
     compareOperands[0] = lChildIdx;
     compareOperands[1] = rChildIdx;
     Value cond2 =
         createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
     scf::IfOp if2 =
-        builder.create<scf::IfOp>(loc, ifTypes, cond2, /*else=*/true);
+        scf::IfOp::create(builder, loc, ifTypes, cond2, /*else=*/true);
     builder.setInsertionPointToStart(&if2.getThenRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{rChild, rChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{rChild, rChildIdx});
     builder.setInsertionPointToStart(&if2.getElseRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{lChild, lChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{lChild, lChildIdx});
     builder.setInsertionPointAfter(if2);
-    builder.create<scf::YieldOp>(loc, if2.getResults());
+    scf::YieldOp::create(builder, loc, if2.getResults());
     builder.setInsertionPointToStart(&if1.getElseRegion().front());
-    builder.create<scf::YieldOp>(loc, ValueRange{lChild, lChildIdx});
+    scf::YieldOp::create(builder, loc, ValueRange{lChild, lChildIdx});
     builder.setInsertionPointAfter(if1);
     return std::make_pair(if1.getResult(0), if1.getResult(1));
   };
@@ -803,7 +803,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
 
   // While (data[start] < data[childIndex]).
   SmallVector<Type, 3> types(3, child.getType());
-  scf::WhileOp whileOp = builder.create<scf::WhileOp>(
+  scf::WhileOp whileOp = scf::WhileOp::create(builder,
       loc, types, SmallVector<Value, 2>{start, child, childIdx});
 
   // The before-region of the WhileOp.
@@ -815,7 +815,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   compareOperands[0] = start;
   compareOperands[1] = childIdx;
   Value cond = createInlinedLessThan(builder, loc, compareOperands, xPerm, ny);
-  builder.create<scf::ConditionOp>(loc, cond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, cond, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after = builder.createBlock(&whileOp.getAfter(), {}, types, locs);
@@ -827,20 +827,20 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   createSwap(builder, loc, swapOperands, xPerm, ny);
   start = childIdx;
   Value cond2 =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp if2 = builder.create<scf::IfOp>(
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
+  scf::IfOp if2 = scf::IfOp::create(builder,
       loc, TypeRange{child.getType(), child.getType()}, cond2, /*else=*/true);
   builder.setInsertionPointToStart(&if2.getThenRegion().front());
   auto [newChild, newChildIdx] = getLargerChild(child);
-  builder.create<scf::YieldOp>(loc, ValueRange{newChild, newChildIdx});
+  scf::YieldOp::create(builder, loc, ValueRange{newChild, newChildIdx});
   builder.setInsertionPointToStart(&if2.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, ValueRange{child, childIdx});
+  scf::YieldOp::create(builder, loc, ValueRange{child, childIdx});
   builder.setInsertionPointAfter(if2);
-  builder.create<scf::YieldOp>(
+  scf::YieldOp::create(builder,
       loc, ValueRange{start, if2.getResult(0), if2.getResult(1)});
 
   builder.setInsertionPointAfter(ifN);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Creates a function to perform heap sort on the values in the range of index
@@ -870,45 +870,45 @@ static void createHeapSortFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value n = builder.create<arith::SubIOp>(loc, hi, lo);
+  Value n = arith::SubIOp::create(builder, loc, hi, lo);
 
   // For i = (n-2)/2 downto 0.
   Value c0 = constantIndex(builder, loc, 0);
   Value c1 = constantIndex(builder, loc, 1);
   Value s = createSubTwoDividedByTwo(builder, loc, n);
-  Value up = builder.create<arith::AddIOp>(loc, s, c1);
-  scf::ForOp forI = builder.create<scf::ForOp>(loc, c0, up, c1);
+  Value up = arith::AddIOp::create(builder, loc, s, c1);
+  scf::ForOp forI = scf::ForOp::create(builder, loc, c0, up, c1);
   builder.setInsertionPointToStart(forI.getBody());
-  Value i = builder.create<arith::SubIOp>(loc, s, forI.getInductionVar());
-  Value lopi = builder.create<arith::AddIOp>(loc, lo, i);
+  Value i = arith::SubIOp::create(builder, loc, s, forI.getInductionVar());
+  Value lopi = arith::AddIOp::create(builder, loc, lo, i);
   SmallVector<Value> shiftDownOperands = {lo, lopi};
   shiftDownOperands.append(args.begin() + xStartIdx, args.end());
   shiftDownOperands.push_back(n);
   FlatSymbolRefAttr shiftDownFunc = getMangledSortHelperFunc(
       builder, func, TypeRange(), kShiftDownFuncNamePrefix, xPerm, ny,
       shiftDownOperands, createShiftDownFunc, /*nTrailingP=*/1);
-  builder.create<func::CallOp>(loc, shiftDownFunc, TypeRange(),
+  func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
                                shiftDownOperands);
 
   builder.setInsertionPointAfter(forI);
   // For l = n downto 2.
-  up = builder.create<arith::SubIOp>(loc, n, c1);
-  scf::ForOp forL = builder.create<scf::ForOp>(loc, c0, up, c1);
+  up = arith::SubIOp::create(builder, loc, n, c1);
+  scf::ForOp forL = scf::ForOp::create(builder, loc, c0, up, c1);
   builder.setInsertionPointToStart(forL.getBody());
-  Value l = builder.create<arith::SubIOp>(loc, n, forL.getInductionVar());
-  Value loplm1 = builder.create<arith::AddIOp>(loc, lo, l);
-  loplm1 = builder.create<arith::SubIOp>(loc, loplm1, c1);
+  Value l = arith::SubIOp::create(builder, loc, n, forL.getInductionVar());
+  Value loplm1 = arith::AddIOp::create(builder, loc, lo, l);
+  loplm1 = arith::SubIOp::create(builder, loc, loplm1, c1);
   SmallVector<Value> swapOperands{lo, loplm1};
   swapOperands.append(args.begin() + xStartIdx, args.end());
   createSwap(builder, loc, swapOperands, xPerm, ny);
   shiftDownOperands[1] = lo;
   shiftDownOperands[shiftDownOperands.size() - 1] =
-      builder.create<arith::SubIOp>(loc, l, c1);
-  builder.create<func::CallOp>(loc, shiftDownFunc, TypeRange(),
+      arith::SubIOp::create(builder, loc, l, c1);
+  func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
                                shiftDownOperands);
 
   builder.setInsertionPointAfter(forL);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// A helper for generating code to perform quick sort. It partitions [lo, hi),
@@ -933,35 +933,35 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
                                       args.drop_back(nTrailingP))
                 .getResult(0);
 
-  Value lenLow = builder.create<arith::SubIOp>(loc, p, lo);
-  Value lenHigh = builder.create<arith::SubIOp>(loc, hi, p);
+  Value lenLow = arith::SubIOp::create(builder, loc, p, lo);
+  Value lenHigh = arith::SubIOp::create(builder, loc, hi, p);
   // Partition already sorts array with len <= 2
   Value c2 = constantIndex(builder, loc, 2);
-  Value len = builder.create<arith::SubIOp>(loc, hi, lo);
+  Value len = arith::SubIOp::create(builder, loc, hi, lo);
   Value lenGtTwo =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ugt, len, c2);
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ugt, len, c2);
   scf::IfOp ifLenGtTwo =
-      builder.create<scf::IfOp>(loc, types, lenGtTwo, /*else=*/true);
+      scf::IfOp::create(builder, loc, types, lenGtTwo, /*else=*/true);
   builder.setInsertionPointToStart(&ifLenGtTwo.getElseRegion().front());
   // Returns an empty range to mark the entire region is fully sorted.
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
   // Else len > 2, need recursion.
   builder.setInsertionPointToStart(&ifLenGtTwo.getThenRegion().front());
-  Value cond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule,
+  Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
                                              lenLow, lenHigh);
 
   Value c0 = constantIndex(builder, loc, 0);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
 
   auto mayRecursion = [&](Value low, Value high, Value len) {
     Value cond =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, len, c0);
-    scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else=*/false);
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, len, c0);
+    scf::IfOp ifOp = scf::IfOp::create(builder, loc, cond, /*else=*/false);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     SmallVector<Value> operands{low, high};
     operands.append(args.begin() + xStartIdx, args.end());
-    builder.create<func::CallOp>(loc, func, operands);
+    func::CallOp::create(builder, loc, func, operands);
     builder.setInsertionPointAfter(ifOp);
   };
 
@@ -969,14 +969,14 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
   // the bigger partition to be processed by the enclosed while-loop.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   mayRecursion(lo, p, lenLow);
-  builder.create<scf::YieldOp>(loc, ValueRange{p, hi});
+  scf::YieldOp::create(builder, loc, ValueRange{p, hi});
 
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
   mayRecursion(p, hi, lenHigh);
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, p});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, p});
 
   builder.setInsertionPointAfter(ifOp);
-  builder.create<scf::YieldOp>(loc, ifOp.getResults());
+  scf::YieldOp::create(builder, loc, ifOp.getResults());
 
   builder.setInsertionPointAfter(ifLenGtTwo);
   return std::make_pair(ifLenGtTwo.getResult(0), ifLenGtTwo.getResult(1));
@@ -1011,10 +1011,10 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   Value c1 = constantIndex(builder, loc, 1);
   Value lo = args[loIdx];
   Value hi = args[hiIdx];
-  Value lop1 = builder.create<arith::AddIOp>(loc, lo, c1);
+  Value lop1 = arith::AddIOp::create(builder, loc, lo, c1);
 
   // Start the outer for-stmt with induction variable i.
-  scf::ForOp forOpI = builder.create<scf::ForOp>(loc, lop1, hi, c1);
+  scf::ForOp forOpI = scf::ForOp::create(builder, loc, lop1, hi, c1);
   builder.setInsertionPointToStart(forOpI.getBody());
   Value i = forOpI.getInductionVar();
 
@@ -1035,24 +1035,24 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t unused, Value i, Value unused2, Value buffer) {
-        d.push_back(builder.create<memref::LoadOp>(loc, buffer, i));
+        d.push_back(memref::LoadOp::create(builder, loc, buffer, i));
       });
 
   // Start the inner for-stmt with induction variable j, for moving data[p..i)
   // to data[p+1..i+1).
-  Value imp = builder.create<arith::SubIOp>(loc, i, p);
+  Value imp = arith::SubIOp::create(builder, loc, i, p);
   Value c0 = constantIndex(builder, loc, 0);
-  scf::ForOp forOpJ = builder.create<scf::ForOp>(loc, c0, imp, c1);
+  scf::ForOp forOpJ = scf::ForOp::create(builder, loc, c0, imp, c1);
   builder.setInsertionPointToStart(forOpJ.getBody());
   Value j = forOpJ.getInductionVar();
-  Value imj = builder.create<arith::SubIOp>(loc, i, j);
+  Value imj = arith::SubIOp::create(builder, loc, i, j);
   operands[1] = imj;
-  operands[0] = builder.create<arith::SubIOp>(loc, imj, c1);
+  operands[0] = arith::SubIOp::create(builder, loc, imj, c1);
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t unused, Value imjm1, Value imj, Value buffer) {
-        Value t = builder.create<memref::LoadOp>(loc, buffer, imjm1);
-        builder.create<memref::StoreOp>(loc, t, buffer, imj);
+        Value t = memref::LoadOp::create(builder, loc, buffer, imjm1);
+        memref::StoreOp::create(builder, loc, t, buffer, imj);
       });
 
   // Store the value at data[i] to data[p].
@@ -1061,11 +1061,11 @@ static void createSortStableFunc(OpBuilder &builder, ModuleOp module,
   forEachIJPairInAllBuffers(
       builder, loc, operands, xPerm, ny,
       [&](uint64_t k, Value p, Value usused, Value buffer) {
-        builder.create<memref::StoreOp>(loc, d[k], buffer, p);
+        memref::StoreOp::create(builder, loc, d[k], buffer, p);
       });
 
   builder.setInsertionPointAfter(forOpI);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Creates a function to perform quick sort or a hybrid quick sort on the
@@ -1127,7 +1127,7 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   Value hi = args[hiIdx];
   SmallVector<Type, 2> types(2, lo.getType()); // Only two types.
   scf::WhileOp whileOp =
-      builder.create<scf::WhileOp>(loc, types, SmallVector<Value, 2>{lo, hi});
+      scf::WhileOp::create(builder, loc, types, SmallVector<Value, 2>{lo, hi});
 
   // The before-region of the WhileOp.
   Block *before =
@@ -1136,10 +1136,10 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   lo = before->getArgument(0);
   hi = before->getArgument(1);
   Value loP1 =
-      builder.create<arith::AddIOp>(loc, lo, constantIndex(builder, loc, 1));
+      arith::AddIOp::create(builder, loc, lo, constantIndex(builder, loc, 1));
   Value needSort =
-      builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult, loP1, hi);
-  builder.create<scf::ConditionOp>(loc, needSort, before->getArguments());
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, loP1, hi);
+  scf::ConditionOp::create(builder, loc, needSort, before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -1151,53 +1151,53 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   args[1] = hi;
 
   if (isHybrid) {
-    Value len = builder.create<arith::SubIOp>(loc, hi, lo);
+    Value len = arith::SubIOp::create(builder, loc, hi, lo);
     Value lenLimit = constantIndex(builder, loc, 30);
-    Value lenCond = builder.create<arith::CmpIOp>(
+    Value lenCond = arith::CmpIOp::create(builder,
         loc, arith::CmpIPredicate::ule, len, lenLimit);
     scf::IfOp lenIf =
-        builder.create<scf::IfOp>(loc, types, lenCond, /*else=*/true);
+        scf::IfOp::create(builder, loc, types, lenCond, /*else=*/true);
 
     // When len <= limit.
     builder.setInsertionPointToStart(&lenIf.getThenRegion().front());
     FlatSymbolRefAttr insertionSortFunc = getMangledSortHelperFunc(
         builder, func, TypeRange(), kSortStableFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createSortStableFunc);
-    builder.create<func::CallOp>(loc, insertionSortFunc, TypeRange(),
+    func::CallOp::create(builder, loc, insertionSortFunc, TypeRange(),
                                  ValueRange(args).drop_back(nTrailingP));
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When len > limit.
     builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
     Value depthLimit = args.back();
-    depthLimit = builder.create<arith::SubIOp>(loc, depthLimit,
+    depthLimit = arith::SubIOp::create(builder, loc, depthLimit,
                                                constantI64(builder, loc, 1));
     Value depthCond =
-        builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ule,
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
                                       depthLimit, constantI64(builder, loc, 0));
     scf::IfOp depthIf =
-        builder.create<scf::IfOp>(loc, types, depthCond, /*else=*/true);
+        scf::IfOp::create(builder, loc, types, depthCond, /*else=*/true);
 
     // When depth exceeds limit.
     builder.setInsertionPointToStart(&depthIf.getThenRegion().front());
     FlatSymbolRefAttr heapSortFunc = getMangledSortHelperFunc(
         builder, func, TypeRange(), kHeapSortFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createHeapSortFunc);
-    builder.create<func::CallOp>(loc, heapSortFunc, TypeRange(),
+    func::CallOp::create(builder, loc, heapSortFunc, TypeRange(),
                                  ValueRange(args).drop_back(nTrailingP));
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, lo});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When depth doesn't exceed limit.
     builder.setInsertionPointToStart(&depthIf.getElseRegion().front());
     args.back() = depthLimit;
     std::tie(lo, hi) =
         createQuickSort(builder, module, func, args, xPerm, ny, nTrailingP);
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
     builder.setInsertionPointAfter(depthIf);
     lo = depthIf.getResult(0);
     hi = depthIf.getResult(1);
-    builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+    scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
     builder.setInsertionPointAfter(lenIf);
     lo = lenIf.getResult(0);
@@ -1208,11 +1208,11 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   }
 
   // New [lo, hi) for the next while-loop iteration.
-  builder.create<scf::YieldOp>(loc, ValueRange{lo, hi});
+  scf::YieldOp::create(builder, loc, ValueRange{lo, hi});
 
   // After the while-loop.
   builder.setInsertionPointAfter(whileOp);
-  builder.create<func::ReturnOp>(loc);
+  func::ReturnOp::create(builder, loc);
 }
 
 /// Implements the rewriting for operator sort and sort_coo.
@@ -1228,7 +1228,7 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     if (!mtp.isDynamicDim(0)) {
       auto newMtp =
           MemRefType::get({ShapedType::kDynamic}, mtp.getElementType());
-      v = rewriter.create<memref::CastOp>(loc, newMtp, v);
+      v = memref::CastOp::create(rewriter, loc, newMtp, v);
     }
     operands.push_back(v);
   }
@@ -1248,12 +1248,12 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     // As a heuristics, set depthLimit = 2 * log2(n).
     Value lo = operands[loIdx];
     Value hi = operands[hiIdx];
-    Value len = rewriter.create<arith::IndexCastOp>(
+    Value len = arith::IndexCastOp::create(rewriter,
         loc, rewriter.getI64Type(),
-        rewriter.create<arith::SubIOp>(loc, hi, lo));
-    Value depthLimit = rewriter.create<arith::SubIOp>(
+        arith::SubIOp::create(rewriter, loc, hi, lo));
+    Value depthLimit = arith::SubIOp::create(rewriter,
         loc, constantI64(rewriter, loc, 64),
-        rewriter.create<math::CountLeadingZerosOp>(loc, len));
+        math::CountLeadingZerosOp::create(rewriter, loc, len));
     operands.push_back(depthLimit);
     break;
   }
@@ -1307,33 +1307,33 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
     Location loc = op->getLoc();
     Value c0 = constantIndex(rewriter, loc, 0);
     Value buffer = op.getInBuffer();
-    Value capacity = rewriter.create<memref::DimOp>(loc, buffer, c0);
+    Value capacity = memref::DimOp::create(rewriter, loc, buffer, c0);
     Value size = op.getCurSize();
     Value value = op.getValue();
 
     Value n = op.getN() ? op.getN() : constantIndex(rewriter, loc, 1);
-    Value newSize = rewriter.create<arith::AddIOp>(loc, size, n);
+    Value newSize = arith::AddIOp::create(rewriter, loc, size, n);
     auto nValue = dyn_cast_or_null<arith::ConstantIndexOp>(n.getDefiningOp());
     bool nIsOne = (nValue && nValue.value() == 1);
 
     if (!op.getInbounds()) {
-      Value cond = rewriter.create<arith::CmpIOp>(
+      Value cond = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::ugt, newSize, capacity);
 
       Value c2 = constantIndex(rewriter, loc, 2);
       auto bufferType =
           MemRefType::get({ShapedType::kDynamic}, value.getType());
-      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, bufferType, cond,
+      scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, bufferType, cond,
                                                   /*else=*/true);
       // True branch.
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       if (nIsOne) {
-        capacity = rewriter.create<arith::MulIOp>(loc, capacity, c2);
+        capacity = arith::MulIOp::create(rewriter, loc, capacity, c2);
       } else {
         // Use a do-while loop to calculate the new capacity as follows:
         //   do { new_capacity *= 2 } while (size > new_capacity)
         scf::WhileOp whileOp =
-            rewriter.create<scf::WhileOp>(loc, capacity.getType(), capacity);
+            scf::WhileOp::create(rewriter, loc, capacity.getType(), capacity);
 
         // The before-region of the WhileOp.
         Block *before = rewriter.createBlock(&whileOp.getBefore(), {},
@@ -1341,36 +1341,36 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
         rewriter.setInsertionPointToEnd(before);
 
         capacity =
-            rewriter.create<arith::MulIOp>(loc, before->getArgument(0), c2);
-        cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ugt,
+            arith::MulIOp::create(rewriter, loc, before->getArgument(0), c2);
+        cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ugt,
                                               newSize, capacity);
-        rewriter.create<scf::ConditionOp>(loc, cond, ValueRange{capacity});
+        scf::ConditionOp::create(rewriter, loc, cond, ValueRange{capacity});
         // The after-region of the WhileOp.
         Block *after = rewriter.createBlock(&whileOp.getAfter(), {},
                                             {capacity.getType()}, {loc});
         rewriter.setInsertionPointToEnd(after);
-        rewriter.create<scf::YieldOp>(loc, after->getArguments());
+        scf::YieldOp::create(rewriter, loc, after->getArguments());
 
         rewriter.setInsertionPointAfter(whileOp);
         capacity = whileOp.getResult(0);
       }
 
       Value newBuffer =
-          rewriter.create<memref::ReallocOp>(loc, bufferType, buffer, capacity);
+          memref::ReallocOp::create(rewriter, loc, bufferType, buffer, capacity);
       if (enableBufferInitialization) {
-        Value fillSize = rewriter.create<arith::SubIOp>(loc, capacity, newSize);
+        Value fillSize = arith::SubIOp::create(rewriter, loc, capacity, newSize);
         Value fillValue = constantZero(rewriter, loc, value.getType());
-        Value subBuffer = rewriter.create<memref::SubViewOp>(
+        Value subBuffer = memref::SubViewOp::create(rewriter,
             loc, newBuffer, /*offset=*/ValueRange{newSize},
             /*size=*/ValueRange{fillSize},
             /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
-        rewriter.create<linalg::FillOp>(loc, fillValue, subBuffer);
+        linalg::FillOp::create(rewriter, loc, fillValue, subBuffer);
       }
-      rewriter.create<scf::YieldOp>(loc, newBuffer);
+      scf::YieldOp::create(rewriter, loc, newBuffer);
 
       // False branch.
       rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      rewriter.create<scf::YieldOp>(loc, buffer);
+      scf::YieldOp::create(rewriter, loc, buffer);
 
       // Prepare for adding the value to the end of the buffer.
       rewriter.setInsertionPointAfter(ifOp);
@@ -1379,12 +1379,12 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
 
     // Add the value to the end of the buffer.
     if (nIsOne) {
-      rewriter.create<memref::StoreOp>(loc, value, buffer, size);
+      memref::StoreOp::create(rewriter, loc, value, buffer, size);
     } else {
-      Value subBuffer = rewriter.create<memref::SubViewOp>(
+      Value subBuffer = memref::SubViewOp::create(rewriter,
           loc, buffer, /*offset=*/ValueRange{size}, /*size=*/ValueRange{n},
           /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
-      rewriter.create<linalg::FillOp>(loc, value, subBuffer);
+      linalg::FillOp::create(rewriter, loc, value, subBuffer);
     }
 
     // Update the buffer size.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index e89b34d457ff8..001754b5531be 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -59,7 +59,7 @@ static gpu::GPUModuleOp genGPUModule(OpBuilder &builder, ModuleOp topModule) {
     return op; // existing
   markAsGPUContainer(topModule);
   builder.setInsertionPointToStart(topModule.getBody());
-  return builder.create<gpu::GPUModuleOp>(topModule->getLoc(),
+  return gpu::GPUModuleOp::create(builder, topModule->getLoc(),
                                           "sparse_kernels");
 }
 
@@ -81,7 +81,7 @@ static gpu::GPUFuncOp genGPUFunc(OpBuilder &builder, gpu::GPUModuleOp gpuModule,
     argsTp.push_back(arg.getType());
   FunctionType type = FunctionType::get(gpuModule->getContext(), argsTp, {});
   auto gpuFunc =
-      builder.create<gpu::GPUFuncOp>(gpuModule->getLoc(), kernelName, type);
+      gpu::GPUFuncOp::create(builder, gpuModule->getLoc(), kernelName, type);
   gpuFunc->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                    builder.getUnitAttr());
   return gpuFunc;
@@ -115,28 +115,28 @@ static Value genHostRegisterMemref(OpBuilder &builder, Location loc,
   MemRefType memTp = cast<MemRefType>(mem.getType());
   UnrankedMemRefType resTp =
       UnrankedMemRefType::get(memTp.getElementType(), /*memorySpace=*/0);
-  Value cast = builder.create<memref::CastOp>(loc, resTp, mem);
-  builder.create<gpu::HostRegisterOp>(loc, cast);
+  Value cast = memref::CastOp::create(builder, loc, resTp, mem);
+  gpu::HostRegisterOp::create(builder, loc, cast);
   return cast;
 }
 
 /// Unmaps the provided buffer, expecting the casted buffer.
 static void genHostUnregisterMemref(OpBuilder &builder, Location loc,
                                     Value cast) {
-  builder.create<gpu::HostUnregisterOp>(loc, cast);
+  gpu::HostUnregisterOp::create(builder, loc, cast);
 }
 
 /// Generates first wait in an asynchronous chain.
 static Value genFirstWait(OpBuilder &builder, Location loc) {
   Type tokenType = builder.getType<gpu::AsyncTokenType>();
-  return builder.create<gpu::WaitOp>(loc, tokenType, ValueRange())
+  return gpu::WaitOp::create(builder, loc, tokenType, ValueRange())
       .getAsyncToken();
 }
 
 /// Generates last, blocking wait in an asynchronous chain.
 static void genBlockingWait(OpBuilder &builder, Location loc,
                             ValueRange operands) {
-  builder.create<gpu::WaitOp>(loc, Type(), operands);
+  gpu::WaitOp::create(builder, loc, Type(), operands);
 }
 
 /// Allocates memory on the device.
@@ -156,7 +156,7 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
       dynamicSizes.push_back(dimOp);
     }
   }
-  return builder.create<gpu::AllocOp>(loc, TypeRange({memTp, token.getType()}),
+  return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
                                       token, dynamicSizes, ValueRange());
 }
 
@@ -164,14 +164,14 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
 static Value genHostBuffer(OpBuilder &builder, Location loc, Type type,
                            Value size) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
-  return builder.create<memref::AllocOp>(loc, memTp, size).getResult();
+  return memref::AllocOp::create(builder, loc, memTp, size).getResult();
 }
 
 // Allocates a typed buffer on the device with given size.
 static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Type type,
                                    Value size, Value token) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
-  return builder.create<gpu::AllocOp>(loc, TypeRange({memTp, token.getType()}),
+  return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
                                       token, size, ValueRange());
 }
 
@@ -184,14 +184,14 @@ static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Value size,
 /// Deallocates memory from the device.
 static Value genDeallocMemRef(OpBuilder &builder, Location loc, Value mem,
                               Value token) {
-  return builder.create<gpu::DeallocOp>(loc, token.getType(), token, mem)
+  return gpu::DeallocOp::create(builder, loc, token.getType(), token, mem)
       .getAsyncToken();
 }
 
 /// Copies memory between host and device (direction is implicit).
 static Value genCopyMemRef(OpBuilder &builder, Location loc, Value dst,
                            Value src, Value token) {
-  return builder.create<gpu::MemcpyOp>(loc, token.getType(), token, dst, src)
+  return gpu::MemcpyOp::create(builder, loc, token.getType(), token, dst, src)
       .getAsyncToken();
 }
 
@@ -212,7 +212,7 @@ static Value genTensorToMemref(PatternRewriter &rewriter, Location loc,
   auto tensorType = llvm::cast<ShapedType>(tensor.getType());
   auto memrefType =
       MemRefType::get(tensorType.getShape(), tensorType.getElementType());
-  return rewriter.create<bufferization::ToBufferOp>(loc, memrefType, tensor);
+  return bufferization::ToBufferOp::create(rewriter, loc, memrefType, tensor);
 }
 
 /// Prepares the outlined arguments, passing scalars and buffers in. Here we
@@ -293,13 +293,13 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
   // so that:
   //   row = blockIdx.x * blockDim.x + threadIdx.x
   //   inc = blockDim.x * gridDim.x
-  Value bid = rewriter.create<gpu::BlockIdOp>(loc, gpu::Dimension::x);
-  Value bsz = rewriter.create<gpu::BlockDimOp>(loc, gpu::Dimension::x);
-  Value tid = rewriter.create<gpu::ThreadIdOp>(loc, gpu::Dimension::x);
-  Value gsz = rewriter.create<gpu::GridDimOp>(loc, gpu::Dimension::x);
-  Value mul = rewriter.create<arith::MulIOp>(loc, bid, bsz);
-  Value row = rewriter.create<arith::AddIOp>(loc, mul, tid);
-  Value inc = rewriter.create<arith::MulIOp>(loc, bsz, gsz);
+  Value bid = gpu::BlockIdOp::create(rewriter, loc, gpu::Dimension::x);
+  Value bsz = gpu::BlockDimOp::create(rewriter, loc, gpu::Dimension::x);
+  Value tid = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
+  Value gsz = gpu::GridDimOp::create(rewriter, loc, gpu::Dimension::x);
+  Value mul = arith::MulIOp::create(rewriter, loc, bid, bsz);
+  Value row = arith::AddIOp::create(rewriter, loc, mul, tid);
+  Value inc = arith::MulIOp::create(rewriter, loc, bsz, gsz);
 
   // Construct the iteration over the computational space that
   // accounts for the fact that the total number of threads and
@@ -308,7 +308,7 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
   //     <loop-body>
   //   }
   Value upper = irMap.lookup(forallOp.getUpperBound()[0]);
-  scf::ForOp forOp = rewriter.create<scf::ForOp>(loc, row, upper, inc);
+  scf::ForOp forOp = scf::ForOp::create(rewriter, loc, row, upper, inc);
   // The scf.for builder creates an empty block. scf.for does not allow multiple
   // blocks in its region, so delete the block before `cloneRegionBefore` adds
   // an additional block.
@@ -321,7 +321,7 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc,
 
   // Done.
   rewriter.setInsertionPointAfter(forOp);
-  rewriter.create<gpu::ReturnOp>(gpuFunc->getLoc());
+  gpu::ReturnOp::create(rewriter, gpuFunc->getLoc());
 }
 
 //===----------------------------------------------------------------------===//
@@ -496,11 +496,11 @@ static Value genFirstPosOrCrds(OpBuilder &builder, Location loc, Value a,
   if (format == CuSparseFormat::kCOO) {
     // Library uses SoA COO, direct IR uses AoS COO.
     if (enableRT)
-      return builder.create<ToCoordinatesOp>(loc, a, 0);
-    return builder.create<ToCoordinatesBufferOp>(loc, a);
+      return ToCoordinatesOp::create(builder, loc, a, 0);
+    return ToCoordinatesBufferOp::create(builder, loc, a);
   }
   // Formats CSR/CSC and BSR use positions at 1.
-  return builder.create<ToPositionsOp>(loc, a, 1);
+  return ToPositionsOp::create(builder, loc, a, 1);
 }
 
 /// Generates the second coordinates of a sparse matrix.
@@ -510,7 +510,7 @@ static Value genSecondCrds(OpBuilder &builder, Location loc, Value a,
   if (isCOO && !enableRT)
     return Value(); // nothing needed
   // Formats CSR/CSC and BSR use coordinates at 1.
-  return builder.create<ToCoordinatesOp>(loc, a, 1);
+  return ToCoordinatesOp::create(builder, loc, a, 1);
 }
 
 /// Generates the sparse matrix handle.
@@ -523,12 +523,12 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
     // Library uses SoA COO, direct IR uses AoS COO.
     if (enableRT) {
       assert(colA);
-      return builder.create<gpu::CreateCooOp>(loc, handleTp, tokenTp, token,
+      return gpu::CreateCooOp::create(builder, loc, handleTp, tokenTp, token,
                                               sz1, sz2, nseA, rowA, colA, valA);
     }
 #ifdef CUSPARSE_COO_AOS
     assert(!colA);
-    return builder.create<gpu::CreateCooAoSOp>(loc, handleTp, tokenTp, token,
+    return gpu::CreateCooAoSOp::create(builder, loc, handleTp, tokenTp, token,
                                                sz1, sz2, nseA, rowA, valA);
 #else
     llvm_unreachable("gpu::CreateCooAoSOp is deprecated");
@@ -536,10 +536,10 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   }
   assert(colA);
   if (format == CuSparseFormat::kCSR)
-    return builder.create<gpu::CreateCsrOp>(loc, handleTp, tokenTp, token, sz1,
+    return gpu::CreateCsrOp::create(builder, loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
   if (format == CuSparseFormat::kCSC)
-    return builder.create<gpu::CreateCscOp>(loc, handleTp, tokenTp, token, sz1,
+    return gpu::CreateCscOp::create(builder, loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
   // BSR requires a bit more work since we need to pass in the block size
   // and all others sizes in terms of blocks (#block-rows, #block-cols,
@@ -549,11 +549,11 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   assert(dims.size() == 2 && dims[0] == dims[1]);
   uint64_t b = dims[0];
   Value bSz = constantIndex(builder, loc, b);
-  Value bRows = builder.create<arith::DivUIOp>(loc, sz1, bSz);
-  Value bCols = builder.create<arith::DivUIOp>(loc, sz2, bSz);
-  Value bNum = builder.create<arith::DivUIOp>(
+  Value bRows = arith::DivUIOp::create(builder, loc, sz1, bSz);
+  Value bCols = arith::DivUIOp::create(builder, loc, sz2, bSz);
+  Value bNum = arith::DivUIOp::create(builder,
       loc, nseA, constantIndex(builder, loc, b * b));
-  return builder.create<gpu::CreateBsrOp>(loc, handleTp, tokenTp, token, bRows,
+  return gpu::CreateBsrOp::create(builder, loc, handleTp, tokenTp, token, bRows,
                                           bCols, bNum, bSz, bSz, rowA, colA,
                                           valA);
 }
@@ -579,12 +579,12 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   //   a : memR/memC/memV -> rowA,colA,valA
   //   x : memX           -> vecX
   //   y : memY           -> vecY
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
   Value szY = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szX = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value memR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, a, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, a);
+  Value memV = ToValuesOp::create(rewriter, loc, a);
   Value rowA = genAllocCopy(rewriter, loc, memR, tokens);
   Value colA = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valA = genAllocCopy(rewriter, loc, memV, tokens);
@@ -606,18 +606,18 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dvecX = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dvecX = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, vecX, szX);
   Value dnX = dvecX.getResult(0);
   token = dvecX.getAsyncToken();
-  auto dvecY = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dvecY = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, vecY, szY);
   Value dnY = dvecY.getResult(0);
   token = dvecY.getAsyncToken();
   auto dnYType = llvm::cast<ShapedType>(y.getType()).getElementType();
 
   // Precompute buffersize for SpMV.
-  auto bufferComp = rewriter.create<gpu::SpMVBufferSizeOp>(
+  auto bufferComp = gpu::SpMVBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, spMatA, dnX, dnY,
       /*computeType=*/dnYType);
   Value bufferSz = bufferComp.getResult(0);
@@ -627,16 +627,16 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SpMV.
-  auto spmvComp = rewriter.create<gpu::SpMVOp>(
+  auto spmvComp = gpu::SpMVOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnX, dnY, /*computeType=*/dnYType, buffer);
   token = spmvComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnX)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnX)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnY)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnY)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, rowA, token);
   if (colA)
@@ -676,13 +676,13 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   //   a : memR/memC/memV -> rowA,colA,valA
   //   b : bufB           -> matB
   //   c : bufC           -> matC
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
   Value memR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, a, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, a);
+  Value memV = ToValuesOp::create(rewriter, loc, a);
   Value rowA = genAllocCopy(rewriter, loc, memR, tokens);
   Value colA = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valA = genAllocCopy(rewriter, loc, memV, tokens);
@@ -704,12 +704,12 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matB,
       SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matC,
       SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
@@ -717,7 +717,7 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dmatCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SpMM.
-  auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
   Value bufferSz = bufferComp.getResult(0);
@@ -728,16 +728,16 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Perform the SpMM.
-  auto spmmComp = rewriter.create<gpu::SpMMOp>(
+  auto spmmComp = gpu::SpMMOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType, buffer);
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnC)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, rowA, token);
   if (colA)
@@ -778,17 +778,17 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   //   b : bmemR/bmemC/bmemV -> rowB,colB,valB
   //   c : materializes
   auto dnCType = cTp.getElementType();
-  Value nseA = rewriter.create<NumberOfEntriesOp>(loc, a);
-  Value nseB = rewriter.create<NumberOfEntriesOp>(loc, b);
+  Value nseA = NumberOfEntriesOp::create(rewriter, loc, a);
+  Value nseB = NumberOfEntriesOp::create(rewriter, loc, b);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
   Value amemR = genFirstPosOrCrds(rewriter, loc, a, format, enableRT);
   Value amemC = genSecondCrds(rewriter, loc, a, format, enableRT); // not empty
-  Value amemV = rewriter.create<ToValuesOp>(loc, a);
+  Value amemV = ToValuesOp::create(rewriter, loc, a);
   Value bmemR = genFirstPosOrCrds(rewriter, loc, b, format, enableRT);
   Value bmemC = genSecondCrds(rewriter, loc, b, format, enableRT); // not empty
-  Value bmemV = rewriter.create<ToValuesOp>(loc, b);
+  Value bmemV = ToValuesOp::create(rewriter, loc, b);
   Value rowA = genAllocCopy(rewriter, loc, amemR, tokens);
   Value colA = genAllocCopy(rewriter, loc, amemC, tokens);
   Value valA = genAllocCopy(rewriter, loc, amemV, tokens);
@@ -818,7 +818,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   // Sparse matrix C materializes (also assumes beta == 0).
   Value zero = constantIndex(rewriter, loc, 0);
   Value one = constantIndex(rewriter, loc, 1);
-  Value mplus1 = rewriter.create<arith::AddIOp>(loc, szm, one);
+  Value mplus1 = arith::AddIOp::create(rewriter, loc, szm, one);
   auto e1 = genAllocBuffer(rewriter, loc, cTp.getPosType(), mplus1, token);
   Value rowC = e1.getResult(0);
   token = e1.getAsyncToken();
@@ -836,10 +836,10 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
 
   // Precompute buffersizes for SpGEMM.
   Operation *descOp =
-      rewriter.create<gpu::SpGEMMCreateDescrOp>(loc, descTp, tokenTp, token);
+      gpu::SpGEMMCreateDescrOp::create(rewriter, loc, descTp, tokenTp, token);
   Value desc = descOp->getResult(0);
   token = descOp->getResult(1);
-  Operation *work1 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
       valC, gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
@@ -848,7 +848,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   auto buf1 = genAllocBuffer(rewriter, loc, bufferSz1, token);
   Value buffer1 = buf1.getResult(0);
   token = buf1.getAsyncToken();
-  Operation *work2 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
       bufferSz1, buffer1,
@@ -856,7 +856,7 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = work2->getResult(1);
 
   // Compute step.
-  Operation *compute1 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
       valC, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
@@ -865,14 +865,14 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   auto buf2 = genAllocBuffer(rewriter, loc, bufferSz2, token);
   Value buffer2 = buf2.getResult(0);
   token = buf2.getAsyncToken();
-  Operation *compute2 = rewriter.create<gpu::SpGEMMWorkEstimationOrComputeOp>(
+  Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
       loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
       bufferSz2, buffer2, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
   token = compute2->getResult(1);
 
   // Get sizes.
-  Operation *sizes = rewriter.create<gpu::SpMatGetSizeOp>(
+  Operation *sizes = gpu::SpMatGetSizeOp::create(rewriter,
       loc, indexTp, indexTp, indexTp, tokenTp, token, spMatC);
   Value nnz = sizes->getResult(2);
   token = sizes->getResult(3);
@@ -884,10 +884,10 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = a3.getAsyncToken();
 
   // Update C with new pointers and copy final product back into C.
-  Operation *update = rewriter.create<gpu::SetCsrPointersOp>(
+  Operation *update = gpu::SetCsrPointersOp::create(rewriter,
       loc, tokenTp, token, spMatC, rowC, colC, valC);
   token = update->getResult(0);
-  Operation *copy = rewriter.create<gpu::SpGEMMCopyOp>(
+  Operation *copy = gpu::SpGEMMCopyOp::create(rewriter,
       loc, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType);
   token = copy->getResult(0);
@@ -898,13 +898,13 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   Value valH = genHostBuffer(rewriter, loc, dnCType, nnz);
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::SpGEMMDestroyDescrOp>(loc, tokenTp, token, desc)
+  token = gpu::SpGEMMDestroyDescrOp::create(rewriter, loc, tokenTp, token, desc)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatB)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatC)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatC)
               .getAsyncToken();
   token = genCopyMemRef(rewriter, loc, rowH, rowC, token);
   token = genCopyMemRef(rewriter, loc, colH, colC, token);
@@ -925,11 +925,11 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  Value vt = rewriter.create<bufferization::ToTensorOp>(
+  Value vt = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
-  Value rt = rewriter.create<bufferization::ToTensorOp>(
+  Value rt = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
-  Value ct = rewriter.create<bufferization::ToTensorOp>(
+  Value ct = bufferization::ToTensorOp::create(rewriter,
       loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
   rewriter.replaceOpWithNewOp<AssembleOp>(op, c.getType(), ValueRange{rt, ct},
                                           vt);
@@ -980,17 +980,17 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  Operation *spGenA = rewriter.create<gpu::Create2To4SpMatOp>(
+  Operation *spGenA = gpu::Create2To4SpMatOp::create(rewriter,
       loc, spMatHandleTp, tokenTp, token, szm, szk,
       gpu::Prune2To4SpMatFlag::PRUNE_AND_CHECK, matA);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matB,
       SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnTensorHandleTp, tokenTp, token, matC,
       SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
@@ -1000,7 +1000,7 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   // Precompute buffersize for SpMM.
   SmallVector<Type> bufferTypes_{indexTp, indexTp, indexTp};
   TypeRange bufferTypes(bufferTypes_);
-  auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
       loc, bufferTypes, tokenTp, token, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
@@ -1022,17 +1022,17 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
 
   // Perform the SpMM.
   auto dnCType = llvm::cast<ShapedType>(matC.getType()).getElementType();
-  auto spmmComp = rewriter.create<gpu::SpMMOp>(
+  auto spmmComp = gpu::SpMMOp::create(rewriter,
       loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType,
       SmallVector<Value>{buffer1, buffer2, buffer3});
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resources.
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatA)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnC)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, buffer1, token);
   token = genDeallocMemRef(rewriter, loc, buffer2, token);
@@ -1073,7 +1073,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   //   a : bufA           -> matA
   //   b : bufB           -> matB
   //   c : memR/memC/memV -> rowC,colC,valC
-  Value nseC = rewriter.create<NumberOfEntriesOp>(loc, c);
+  Value nseC = NumberOfEntriesOp::create(rewriter, loc, c);
   Value szm = linalg::createOrFoldDimOp(rewriter, loc, a, 0);
   Value szk = linalg::createOrFoldDimOp(rewriter, loc, a, 1);
   Value szn = linalg::createOrFoldDimOp(rewriter, loc, b, 1);
@@ -1083,7 +1083,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Value matB = genAllocCopy(rewriter, loc, bufB, tokens);
   Value memR = genFirstPosOrCrds(rewriter, loc, c, format, enableRT);
   Value memC = genSecondCrds(rewriter, loc, c, format, enableRT); // or empty
-  Value memV = rewriter.create<ToValuesOp>(loc, c);
+  Value memV = ToValuesOp::create(rewriter, loc, c);
   Value rowC = genAllocCopy(rewriter, loc, memR, tokens);
   Value colC = memC ? genAllocCopy(rewriter, loc, memC, tokens) : Value();
   Value valC = genAllocCopy(rewriter, loc, memV, tokens);
@@ -1096,11 +1096,11 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  auto dmatA = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatA = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnMatHandleTp, tokenTp, token, matA, SmallVector<Value>{szm, szk});
   Value dnA = dmatA.getResult(0);
   token = dmatA.getAsyncToken();
-  auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
+  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
       loc, dnMatHandleTp, tokenTp, token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
@@ -1112,7 +1112,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SDDMM.
-  auto bufferComp = rewriter.create<gpu::SDDMMBufferSizeOp>(
+  auto bufferComp = gpu::SDDMMBufferSizeOp::create(rewriter,
       loc, indexTp, tokenTp, token, dnA, dnB, spMatC, dnCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
@@ -1121,16 +1121,16 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SDDMM.
-  auto sddmmComp = rewriter.create<gpu::SDDMMOp>(loc, tokenTp, token, dnA, dnB,
+  auto sddmmComp = gpu::SDDMMOp::create(rewriter, loc, tokenTp, token, dnA, dnB,
                                                  spMatC, dnCType, buffer);
   token = sddmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnA)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnA)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroyDnTensorOp>(loc, tokenTp, token, dnB)
+  token = gpu::DestroyDnTensorOp::create(rewriter, loc, tokenTp, token, dnB)
               .getAsyncToken();
-  token = rewriter.create<gpu::DestroySpMatOp>(loc, tokenTp, token, spMatC)
+  token = gpu::DestroySpMatOp::create(rewriter, loc, tokenTp, token, spMatC)
               .getAsyncToken();
   token = genDeallocMemRef(rewriter, loc, buffer, token);
   token = genDeallocMemRef(rewriter, loc, matA, token);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
index 2f68008e68b5f..0986a0f2ea0f0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
@@ -67,11 +67,11 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
       op.getRegionDefinedSpace(newBlock->getParent()->getRegionNumber());
   for (unsigned i : caseBits.bits()) {
     SparseIterator *it = iters[i].get();
-    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
                                                 it->getCrd(), loopCrd);
-    casePred = rewriter.create<arith::AndIOp>(loc, casePred, pred);
+    casePred = arith::AndIOp::create(rewriter, loc, casePred, pred);
   }
-  scf::IfOp ifOp = rewriter.create<scf::IfOp>(
+  scf::IfOp ifOp = scf::IfOp::create(rewriter,
       loc, ValueRange(userReduc).getTypes(), casePred, /*else=*/true);
   rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
@@ -103,7 +103,7 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
   ValueRange yields = spY.getResults();
   rewriter.eraseOp(spY);
   rewriter.setInsertionPointToEnd(&ifOp.getThenRegion().front());
-  rewriter.create<scf::YieldOp>(loc, yields);
+  scf::YieldOp::create(rewriter, loc, yields);
 
   // Generates remaining case recursively.
   rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -111,7 +111,7 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
                                           newBlocks.drop_front(),
                                           oldBlocks.drop_front(), userReduc);
   if (!res.empty())
-    rewriter.create<scf::YieldOp>(loc, res);
+    scf::YieldOp::create(rewriter, loc, res);
 
   rewriter.setInsertionPointAfter(ifOp);
   return ifOp.getResults();
@@ -127,7 +127,7 @@ static ValueRange genLoopWithIterator(
   if (it->iteratableByFor()) {
     auto [lo, hi] = it->genForCond(rewriter, loc);
     Value step = constantIndex(rewriter, loc, 1);
-    scf::ForOp forOp = rewriter.create<scf::ForOp>(
+    scf::ForOp forOp = scf::ForOp::create(rewriter,
         loc, lo, hi, step, reduc,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
           // Empty builder function to ensure that no terminator is created.
@@ -140,7 +140,7 @@ static ValueRange genLoopWithIterator(
                                            it, forOp.getRegionIterArgs());
 
       rewriter.setInsertionPointToEnd(forOp.getBody());
-      rewriter.create<scf::YieldOp>(loc, ret);
+      scf::YieldOp::create(rewriter, loc, ret);
     }
     return forOp.getResults();
   }
@@ -149,7 +149,7 @@ static ValueRange genLoopWithIterator(
   llvm::append_range(ivs, it->getCursor());
 
   TypeRange types = ValueRange(ivs).getTypes();
-  auto whileOp = rewriter.create<scf::WhileOp>(loc, types, ivs);
+  auto whileOp = scf::WhileOp::create(rewriter, loc, types, ivs);
   {
     OpBuilder::InsertionGuard guard(rewriter);
     // Generates loop conditions.
@@ -158,7 +158,7 @@ static ValueRange genLoopWithIterator(
     rewriter.setInsertionPointToStart(before);
     ValueRange bArgs = before->getArguments();
     auto [whileCond, remArgs] = it->genWhileCond(rewriter, loc, bArgs);
-    rewriter.create<scf::ConditionOp>(loc, whileCond, before->getArguments());
+    scf::ConditionOp::create(rewriter, loc, whileCond, before->getArguments());
 
     // Delegates loop body generation.
     Region &dstRegion = whileOp.getAfter();
@@ -175,7 +175,7 @@ static ValueRange genLoopWithIterator(
     SmallVector<Value> yields;
     llvm::append_range(yields, ret);
     llvm::append_range(yields, it->forward(rewriter, loc));
-    rewriter.create<scf::YieldOp>(loc, yields);
+    scf::YieldOp::create(rewriter, loc, yields);
   }
   return whileOp.getResults().drop_front(it->getCursor().size());
 }
@@ -212,7 +212,7 @@ class ExtractValOpConverter : public OpConversionPattern<ExtractValOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value pos = adaptor.getIterator().back();
-    Value valBuf = rewriter.create<ToValuesOp>(
+    Value valBuf = ToValuesOp::create(rewriter,
         loc, llvm::getSingleElement(adaptor.getTensor()));
     rewriter.replaceOpWithNewOp<memref::LoadOp>(op, valBuf, pos);
     return success();
@@ -385,12 +385,12 @@ class SparseCoIterateOpConverter : public OpConversionPattern<CoIterateOp> {
         SmallVector<Value> nextIterYields(res);
         // 2nd. foward the loop.
         for (SparseIterator *it : validIters) {
-          Value cmp = rewriter.create<arith::CmpIOp>(
+          Value cmp = arith::CmpIOp::create(rewriter,
               loc, arith::CmpIPredicate::eq, it->getCrd(), loopCrd);
           it->forwardIf(rewriter, loc, cmp);
           llvm::append_range(nextIterYields, it->getCursor());
         }
-        rewriter.create<scf::YieldOp>(loc, nextIterYields);
+        scf::YieldOp::create(rewriter, loc, nextIterYields);
 
         // Exit the loop, relink the iterator SSA value.
         rewriter.setInsertionPointAfter(loop);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
index 4f554756b3dd2..dc6bef5387b70 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
@@ -43,7 +43,7 @@ struct DemapInsRewriter : public OpRewritePattern<SourceOp> {
     SmallVector<Value> deMappedIns(op->getOperands());
     for (Value &in : deMappedIns) {
       if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) {
-        in = rewriter.create<ReinterpretMapOp>(loc, stt->getDemappedType(), in);
+        in = ReinterpretMapOp::create(rewriter, loc, stt->getDemappedType(), in);
         changed = true;
       }
     }
@@ -337,14 +337,14 @@ translateMap(linalg::GenericOp op, PatternRewriter &rewriter) {
 // Generates a "de"mapping reinterpretation of the map.
 static Value genDemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
-  return builder.create<ReinterpretMapOp>(val.getLoc(), enc.withoutDimToLvl(),
+  return ReinterpretMapOp::create(builder, val.getLoc(), enc.withoutDimToLvl(),
                                           val);
 }
 
 // Generates a "re"mapping reinterpretation of the map.
 static Value genRemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
-  return builder.create<ReinterpretMapOp>(val.getLoc(), enc, val);
+  return ReinterpretMapOp::create(builder, val.getLoc(), enc, val);
 }
 
 static SmallVector<Value> remapValueRange(OpBuilder &rewriter, TypeRange types,
@@ -353,7 +353,7 @@ static SmallVector<Value> remapValueRange(OpBuilder &rewriter, TypeRange types,
   assert(outs.size() == types.size());
   for (auto [r, t] : llvm::zip(ret, types))
     if (r.getType() != t)
-      r = rewriter.create<ReinterpretMapOp>(r.getLoc(), t, r);
+      r = ReinterpretMapOp::create(rewriter, r.getLoc(), t, r);
   return ret;
 }
 
@@ -566,7 +566,7 @@ struct GenericOpScheduler : public OpRewritePattern<linalg::GenericOp> {
       // Inserting the transpose
       rewriter.setInsertionPoint(linalgOp);
       RankedTensorType dstTp = stt.withDimToLvl(dimToLvl).getRankedTensorType();
-      Value dst = rewriter.create<ConvertOp>(tval.getLoc(), dstTp, tval);
+      Value dst = ConvertOp::create(rewriter, tval.getLoc(), dstTp, tval);
       rewriter.modifyOpInPlace(linalgOp, [&]() {
         linalgOp->setOperand(t->getOperandNumber(), dst);
       });
@@ -574,7 +574,7 @@ struct GenericOpScheduler : public OpRewritePattern<linalg::GenericOp> {
       // Release the transposed form afterwards.
       // TODO: CSE when used in more than one following op?
       rewriter.setInsertionPointAfter(linalgOp);
-      rewriter.create<bufferization::DeallocTensorOp>(dst.getLoc(), dst);
+      bufferization::DeallocTensorOp::create(rewriter, dst.getLoc(), dst);
 
       return success();
     }
@@ -604,7 +604,7 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     ValueRange dynSz = op.getDynamicSizes();
     for (int64_t dimSz : stt.getDimShape()) {
       if (ShapedType::isDynamic(dimSz)) {
-        Value maxCrd = rewriter.create<arith::SubIOp>(
+        Value maxCrd = arith::SubIOp::create(rewriter,
             loc, dynSz.front(), constantIndex(rewriter, loc, 1));
         maxDimCrds.push_back(maxCrd);
         dynSz = dynSz.drop_front();
@@ -619,7 +619,7 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     SmallVector<Value> dynLvlSzs;
     for (unsigned i = 0, e = lvlShape.size(); i < e; i++) {
       if (ShapedType::isDynamic(lvlShape[i])) {
-        Value sz = rewriter.create<arith::AddIOp>(
+        Value sz = arith::AddIOp::create(rewriter,
             loc, maxLvlCrds[i], constantIndex(rewriter, loc, 1));
         dynLvlSzs.push_back(sz);
       }
@@ -650,7 +650,7 @@ struct TensorInsertDemapper
     auto stt = getSparseTensorType(op.getResult());
     ValueRange lvlCrd = stt.translateCrds(rewriter, loc, op.getIndices(),
                                           CrdTransDirectionKind::dim2lvl);
-    auto insertOp = rewriter.create<tensor::InsertOp>(
+    auto insertOp = tensor::InsertOp::create(rewriter,
         loc, op.getScalar(), adaptor.getDest(), lvlCrd);
 
     Value out = genRemap(rewriter, stt.getEncoding(), insertOp.getResult());
@@ -765,7 +765,7 @@ struct ForeachOpDemapper
           stt && !stt->isIdentity()) {
         Value y =
             genDemap(rewriter, stt->getEncoding(), yield.getSingleResult());
-        rewriter.create<YieldOp>(loc, y);
+        YieldOp::create(rewriter, loc, y);
         rewriter.eraseOp(yield);
       }
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
index f85c4761a8d52..7b662c2fb0db0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
@@ -126,7 +126,7 @@ void collapseSparseSpace(MutableArrayRef<CollapseSpaceInfo> toCollapse) {
   OpBuilder builder(root);
 
   // Construct the collapsed iteration space.
-  auto collapsedSpace = builder.create<ExtractIterSpaceOp>(
+  auto collapsedSpace = ExtractIterSpaceOp::create(builder,
       loc, root.getTensor(), root.getParentIter(), root.getLoLvl(),
       leaf.getHiLvl());
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
index 01028f71c20bb..a1b9035f3b6e6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
@@ -69,13 +69,13 @@ class SpecifierStructBuilder : public StructBuilder {
   Value extractField(OpBuilder &builder, Location loc,
                      ArrayRef<int64_t> indices) const {
     return genCast(builder, loc,
-                   builder.create<LLVM::ExtractValueOp>(loc, value, indices),
+                   LLVM::ExtractValueOp::create(builder, loc, value, indices),
                    builder.getIndexType());
   }
 
   void insertField(OpBuilder &builder, Location loc, ArrayRef<int64_t> indices,
                    Value v) {
-    value = builder.create<LLVM::InsertValueOp>(
+    value = LLVM::InsertValueOp::create(builder,
         loc, value, genCast(builder, loc, v, builder.getIntegerType(64)),
         indices);
   }
@@ -110,7 +110,7 @@ class SpecifierStructBuilder : public StructBuilder {
 
 Value SpecifierStructBuilder::getInitValue(OpBuilder &builder, Location loc,
                                            Type structType, Value source) {
-  Value metaData = builder.create<LLVM::PoisonOp>(loc, structType);
+  Value metaData = LLVM::PoisonOp::create(builder, loc, structType);
   SpecifierStructBuilder md(metaData);
   if (!source) {
     auto memSizeArrayType =
@@ -204,14 +204,14 @@ void SpecifierStructBuilder::setMemSize(OpBuilder &builder, Location loc,
 /// Builds IR extracting the memory size array from the descriptor.
 Value SpecifierStructBuilder::memSizeArray(OpBuilder &builder,
                                            Location loc) const {
-  return builder.create<LLVM::ExtractValueOp>(loc, value,
+  return LLVM::ExtractValueOp::create(builder, loc, value,
                                               kMemSizePosInSpecifier);
 }
 
 /// Builds IR inserting the memory size array into the descriptor.
 void SpecifierStructBuilder::setMemSizeArray(OpBuilder &builder, Location loc,
                                              Value array) {
-  value = builder.create<LLVM::InsertValueOp>(loc, value, array,
+  value = LLVM::InsertValueOp::create(builder, loc, value, array,
                                               kMemSizePosInSpecifier);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 001ea62b07360..0450e1d6e8775 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -50,7 +50,7 @@ static SmallVector<Value> flattenValues(ArrayRef<ValueRange> values) {
 /// Generates a load with proper `index` typing.
 static Value genLoad(OpBuilder &builder, Location loc, Value mem, Value idx) {
   idx = genCast(builder, loc, idx, builder.getIndexType());
-  return builder.create<memref::LoadOp>(loc, mem, idx);
+  return memref::LoadOp::create(builder, loc, mem, idx);
 }
 
 /// Generates a store with proper `index` typing and proper value.
@@ -59,7 +59,7 @@ static void genStore(OpBuilder &builder, Location loc, Value val, Value mem,
   idx = genCast(builder, loc, idx, builder.getIndexType());
   val = genCast(builder, loc, val,
                 cast<ShapedType>(mem.getType()).getElementType());
-  builder.create<memref::StoreOp>(loc, val, mem, idx);
+  memref::StoreOp::create(builder, loc, val, mem, idx);
 }
 
 /// Creates a straightforward counting for-loop.
@@ -70,7 +70,7 @@ static scf::ForOp createFor(OpBuilder &builder, Location loc, Value upper,
   if (!lower)
     lower = constantZero(builder, loc, indexType);
   Value one = constantOne(builder, loc, indexType);
-  scf::ForOp forOp = builder.create<scf::ForOp>(loc, lower, upper, one, fields);
+  scf::ForOp forOp = scf::ForOp::create(builder, loc, lower, upper, one, fields);
   for (unsigned i = 0, e = fields.size(); i < e; i++)
     fields[i] = forOp.getRegionIterArg(i);
   builder.setInsertionPointToStart(forOp.getBody());
@@ -86,7 +86,7 @@ static void createPushback(OpBuilder &builder, Location loc,
   Value field = desc.getMemRefField(kind, lvl);
   StorageSpecifierKind specFieldKind = toSpecifierKind(kind);
 
-  auto pushBackOp = builder.create<PushBackOp>(
+  auto pushBackOp = PushBackOp::create(builder,
       loc, desc.getSpecifierField(builder, loc, specFieldKind, lvl), field,
       genCast(builder, loc, value, etp), repeat);
 
@@ -112,7 +112,7 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
       Value posZero = constantZero(builder, loc, stt.getPosType());
       if (isLooseCompressedLT(lt)) {
         Value two = constantIndex(builder, loc, 2);
-        linear = builder.create<arith::MulIOp>(loc, linear, two);
+        linear = arith::MulIOp::create(builder, loc, linear, two);
       }
       createPushback(builder, loc, desc, SparseTensorFieldKind::PosMemRef, lvl,
                      /*value=*/posZero, /*repeat=*/linear);
@@ -125,7 +125,7 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
     // otherwise the values array for the from-here "all-dense" case.
     assert(isDenseLT(lt));
     Value size = desc.getLvlSize(builder, loc, lvl);
-    linear = builder.create<arith::MulIOp>(loc, linear, size);
+    linear = arith::MulIOp::create(builder, loc, linear, size);
   }
   // Reached values array so prepare for an insertion.
   Value valZero = constantZero(builder, loc, stt.getElementType());
@@ -137,11 +137,11 @@ static void allocSchemeForRank(OpBuilder &builder, Location loc,
 static Value createAllocation(OpBuilder &builder, Location loc,
                               MemRefType memRefType, Value sz,
                               bool enableInit) {
-  Value buffer = builder.create<memref::AllocOp>(loc, memRefType, sz);
+  Value buffer = memref::AllocOp::create(builder, loc, memRefType, sz);
   Type elemType = memRefType.getElementType();
   if (enableInit) {
     Value fillValue = constantZero(builder, loc, elemType);
-    builder.create<linalg::FillOp>(loc, fillValue, buffer);
+    linalg::FillOp::create(builder, loc, fillValue, buffer);
   }
   return buffer;
 }
@@ -179,14 +179,14 @@ static void createAllocFields(OpBuilder &builder, Location loc,
     valHeuristic = lvlSizesValues[0];
     for (Level lvl = 1; lvl < lvlRank; lvl++)
       valHeuristic =
-          builder.create<arith::MulIOp>(loc, valHeuristic, lvlSizesValues[lvl]);
+          arith::MulIOp::create(builder, loc, valHeuristic, lvlSizesValues[lvl]);
   } else if (sizeHint) {
     if (stt.getAoSCOOStart() == 0) {
       posHeuristic = constantIndex(builder, loc, 2);
-      crdHeuristic = builder.create<arith::MulIOp>(
+      crdHeuristic = arith::MulIOp::create(builder,
           loc, constantIndex(builder, loc, lvlRank), sizeHint); // AOS
     } else if (lvlRank == 2 && stt.isDenseLvl(0) && stt.isCompressedLvl(1)) {
-      posHeuristic = builder.create<arith::AddIOp>(
+      posHeuristic = arith::AddIOp::create(builder,
           loc, sizeHint, constantIndex(builder, loc, 1));
       crdHeuristic = sizeHint;
     } else {
@@ -280,7 +280,7 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   unsigned crdStride;
   std::tie(crdFidx, crdStride) = desc.getCrdMemRefIndexAndStride(lvl);
   const Value one = constantIndex(builder, loc, 1);
-  const Value pp1 = builder.create<arith::AddIOp>(loc, parentPos, one);
+  const Value pp1 = arith::AddIOp::create(builder, loc, parentPos, one);
   const Value positionsAtLvl = desc.getPosMemRef(lvl);
   const Value pstart = genLoad(builder, loc, positionsAtLvl, parentPos);
   const Value pstop = genLoad(builder, loc, positionsAtLvl, pp1);
@@ -288,29 +288,29 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   const Value crdStrideC =
       crdStride > 1 ? constantIndex(builder, loc, crdStride) : Value();
   const Value msz =
-      crdStrideC ? builder.create<arith::DivUIOp>(loc, crdMsz, crdStrideC)
+      crdStrideC ? arith::DivUIOp::create(builder, loc, crdMsz, crdStrideC)
                  : crdMsz;
-  const Value plast = builder.create<arith::SubIOp>(
+  const Value plast = arith::SubIOp::create(builder,
       loc, genCast(builder, loc, pstop, indexType), one);
   // Conditional expression.
-  Value lt = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+  Value lt = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
                                            pstart, pstop);
   types.push_back(boolType);
-  scf::IfOp ifOp1 = builder.create<scf::IfOp>(loc, types, lt, /*else*/ true);
+  scf::IfOp ifOp1 = scf::IfOp::create(builder, loc, types, lt, /*else*/ true);
   types.pop_back();
   builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
   Value crd =
       genLoad(builder, loc, desc.getMemRefField(crdFidx),
-              crdStrideC ? builder.create<arith::MulIOp>(loc, plast, crdStrideC)
+              crdStrideC ? arith::MulIOp::create(builder, loc, plast, crdStrideC)
                          : plast);
-  Value eq = builder.create<arith::CmpIOp>(
+  Value eq = arith::CmpIOp::create(builder,
       loc, arith::CmpIPredicate::eq, genCast(builder, loc, crd, indexType),
       lvlCoords[lvl]);
-  builder.create<scf::YieldOp>(loc, eq);
+  scf::YieldOp::create(builder, loc, eq);
   builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
   if (lvl > 0)
     genStore(builder, loc, msz, positionsAtLvl, parentPos);
-  builder.create<scf::YieldOp>(loc, constantI1(builder, loc, false));
+  scf::YieldOp::create(builder, loc, constantI1(builder, loc, false));
   builder.setInsertionPointAfter(ifOp1);
   // If present construct. Note that for a non-unique dimension level, we
   // simply set the condition to false and rely on CSE/DCE to clean up the IR.
@@ -322,19 +322,19 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   types.push_back(indexType);
   const Value p = stt.isUniqueLvl(lvl) ? ifOp1.getResult(0)
                                        : constantI1(builder, loc, false);
-  scf::IfOp ifOp2 = builder.create<scf::IfOp>(loc, types, p, /*else*/ true);
+  scf::IfOp ifOp2 = scf::IfOp::create(builder, loc, types, p, /*else*/ true);
   // If present (fields unaffected, update pnext to plast).
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
 
   // FIXME: This does not looks like a clean way, but probably the most
   // efficient way.
   desc.getFields().push_back(plast);
-  builder.create<scf::YieldOp>(loc, desc.getFields());
+  scf::YieldOp::create(builder, loc, desc.getFields());
   desc.getFields().pop_back();
 
   // If !present (changes fields, update pnext).
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
-  Value mszp1 = builder.create<arith::AddIOp>(loc, msz, one);
+  Value mszp1 = arith::AddIOp::create(builder, loc, msz, one);
   genStore(builder, loc, mszp1, positionsAtLvl, pp1);
   createPushback(builder, loc, desc, SparseTensorFieldKind::CrdMemRef, lvl,
                  /*value=*/lvlCoords[lvl]);
@@ -343,7 +343,7 @@ static Value genCompressed(OpBuilder &builder, Location loc,
     allocSchemeForRank(builder, loc, desc, lvl + 1);
 
   desc.getFields().push_back(msz);
-  builder.create<scf::YieldOp>(loc, desc.getFields());
+  scf::YieldOp::create(builder, loc, desc.getFields());
   desc.getFields().pop_back();
 
   // Update fields and return next pos.
@@ -381,17 +381,17 @@ static void genEndInsert(OpBuilder &builder, Location loc,
         Value oldv = loop.getRegionIterArg(0);
         Value newv = genLoad(builder, loc, posMemRef, i);
         Value posZero = constantZero(builder, loc, posType);
-        Value cond = builder.create<arith::CmpIOp>(
+        Value cond = arith::CmpIOp::create(builder,
             loc, arith::CmpIPredicate::eq, newv, posZero);
-        scf::IfOp ifOp = builder.create<scf::IfOp>(loc, TypeRange(posType),
+        scf::IfOp ifOp = scf::IfOp::create(builder, loc, TypeRange(posType),
                                                    cond, /*else*/ true);
         builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         genStore(builder, loc, oldv, posMemRef, i);
-        builder.create<scf::YieldOp>(loc, oldv);
+        scf::YieldOp::create(builder, loc, oldv);
         builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-        builder.create<scf::YieldOp>(loc, newv);
+        scf::YieldOp::create(builder, loc, newv);
         builder.setInsertionPointAfter(ifOp);
-        builder.create<scf::YieldOp>(loc, ifOp.getResult(0));
+        scf::YieldOp::create(builder, loc, ifOp.getResult(0));
         builder.setInsertionPointAfter(loop);
       }
     } else {
@@ -484,7 +484,7 @@ class SparseInsertGenerator
         //   <insert @ positions[lvl] at next level lvl + 1>
         if (isLooseCompressedLT(lt)) {
           Value two = constantIndex(builder, loc, 2);
-          parentPos = builder.create<arith::MulIOp>(loc, parentPos, two);
+          parentPos = arith::MulIOp::create(builder, loc, parentPos, two);
         }
         parentPos =
             genCompressed(builder, loc, desc, coords, value, parentPos, lvl);
@@ -501,8 +501,8 @@ class SparseInsertGenerator
         //   positions[lvl] = size * positions[lvl-1] + coords[lvl]
         //   <insert @ positions[lvl] at next level lvl + 1>
         Value size = desc.getLvlSize(builder, loc, lvl);
-        Value mult = builder.create<arith::MulIOp>(loc, size, parentPos);
-        parentPos = builder.create<arith::AddIOp>(loc, mult, coords[lvl]);
+        Value mult = arith::MulIOp::create(builder, loc, size, parentPos);
+        parentPos = arith::AddIOp::create(builder, loc, mult, coords[lvl]);
       }
     }
     // Reached the actual value append/insert.
@@ -582,7 +582,7 @@ class SparseCallConverter : public OpConversionPattern<func::CallOp> {
       return failure();
 
     // (1) Generates new call with flattened return value.
-    auto newCall = rewriter.create<func::CallOp>(
+    auto newCall = func::CallOp::create(rewriter,
         loc, op.getCallee(), finalRetTy, flattenValues(adaptor.getOperands()));
     // (2) Gather sparse tensor returns.
     SmallVector<SmallVector<Value>> packedResultVals;
@@ -671,7 +671,7 @@ struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
 
     auto id = AffineMap::getMultiDimIdentityMap(srcStt.getLvlRank(), ctx);
 
-    rewriter.create<SortOp>(loc, nnz, crd, ValueRange{val}, id,
+    SortOp::create(rewriter, loc, nnz, crd, ValueRange{val}, id,
                             rewriter.getIndexAttr(0), op.getAlgorithm());
 
     // Since we do in-place sorting, the destinate tensor will have the same set
@@ -757,10 +757,10 @@ class SparseTensorAllocConverter
       // Memcpy on memref fields.
       for (auto field : desc.getMemRefFields()) {
         auto memrefTp = cast<MemRefType>(field.getType());
-        auto size = rewriter.create<memref::DimOp>(loc, field, 0);
+        auto size = memref::DimOp::create(rewriter, loc, field, 0);
         auto copied =
-            rewriter.create<memref::AllocOp>(loc, memrefTp, ValueRange{size});
-        rewriter.create<memref::CopyOp>(loc, field, copied);
+            memref::AllocOp::create(rewriter, loc, memrefTp, ValueRange{size});
+        memref::CopyOp::create(rewriter, loc, field, copied);
         fields.push_back(copied);
       }
       // Reuses specifier.
@@ -863,7 +863,7 @@ class SparseTensorDeallocConverter
           cast<RankedTensorType>(op.getTensor().getType()));
       for (auto input : desc.getMemRefFields())
         // Deallocate every buffer used to store the sparse tensor handler.
-        rewriter.create<memref::DeallocOp>(loc, input);
+        memref::DeallocOp::create(rewriter, loc, input);
     }
     rewriter.eraseOp(op);
     return success();
@@ -917,7 +917,7 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // Generate a memref for `sz` elements of type `t`.
     const auto genAlloc = [&](Type t) {
       const auto memTp = MemRefType::get({ShapedType::kDynamic}, t);
-      return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
+      return memref::AllocOp::create(rewriter, loc, memTp, ValueRange{sz});
     };
     // Allocate temporary buffers for values/filled-switch and added.
     // We do not use stack buffers for this, since the expanded size may
@@ -931,10 +931,10 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, eltType)},
         ValueRange{values});
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, boolType)},
         ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
@@ -965,7 +965,7 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     // If the innermost level is ordered, we need to sort the coordinates
     // in the "added" array prior to applying the compression.
     if (dstType.isOrderedLvl(dstType.getLvlRank() - 1))
-      rewriter.create<SortOp>(
+      SortOp::create(rewriter,
           loc, count, added, ValueRange{}, rewriter.getMultiDimIdentityMap(1),
           rewriter.getIndexAttr(0), SparseTensorSortKind::HybridQuickSort);
     // While performing the insertions, we also need to reset the elements
@@ -1000,15 +1000,15 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     SmallVector<Value> insertRet = insertGen.genCallOrInline(rewriter, loc);
     genStore(rewriter, loc, constantZero(rewriter, loc, eltType), values, crd);
     genStore(rewriter, loc, constantI1(rewriter, loc, false), filled, crd);
-    rewriter.create<scf::YieldOp>(loc, insertRet);
+    scf::YieldOp::create(rewriter, loc, insertRet);
 
     rewriter.setInsertionPointAfter(loop);
     // Deallocate the buffers on exit of the full loop nest.
     Operation *parent = getTop(op);
     rewriter.setInsertionPointAfter(parent);
-    rewriter.create<memref::DeallocOp>(loc, values);
-    rewriter.create<memref::DeallocOp>(loc, filled);
-    rewriter.create<memref::DeallocOp>(loc, added);
+    memref::DeallocOp::create(rewriter, loc, values);
+    memref::DeallocOp::create(rewriter, loc, filled);
+    memref::DeallocOp::create(rewriter, loc, added);
     // Replace operation with resulting memrefs.
     rewriter.replaceOpWithMultiple(op, {loop->getResults()});
     return success();
@@ -1192,7 +1192,7 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
             // would require a subViewOp to avoid overflow when copying
             // values.
             Value sz = linalg::createOrFoldDimOp(rewriter, loc, srcMem, 0);
-            auto dstMem = rewriter.create<memref::AllocOp>(
+            auto dstMem = memref::AllocOp::create(rewriter,
                 loc, cast<MemRefType>(fTp), sz);
             if (fTp != srcMem.getType()) {
               // Converts elements type.
@@ -1201,16 +1201,16 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
                   constantIndex(rewriter, loc, 1),
                   [srcMem, &dstMem](OpBuilder &builder, Location loc,
                                     ValueRange ivs) {
-                    Value v = builder.create<memref::LoadOp>(loc, srcMem, ivs);
+                    Value v = memref::LoadOp::create(builder, loc, srcMem, ivs);
                     Value casted = genCast(builder, loc, v,
                                            dstMem.getType().getElementType());
-                    builder.create<memref::StoreOp>(loc, casted, dstMem, ivs);
+                    memref::StoreOp::create(builder, loc, casted, dstMem, ivs);
                   });
             } else {
               // TODO: We can even reuse the same memref for the new tensor,
               // but that requires a `ref-counting` based memory management
               // for shared memrefs between multiple sparse tensors.
-              rewriter.create<memref::CopyOp>(loc, srcMem, dstMem);
+              memref::CopyOp::create(rewriter, loc, srcMem, dstMem);
             }
             fields.push_back(dstMem);
           }
@@ -1242,7 +1242,7 @@ class SparseExtractSliceConverter
     auto desc = getMutDescriptorFromTensorTuple(adaptor.getSource(), fields,
                                                 op.getSource().getType());
 
-    auto newSpec = rewriter.create<StorageSpecifierInitOp>(
+    auto newSpec = StorageSpecifierInitOp::create(rewriter,
         loc, StorageSpecifierType::get(ctx, dstEnc), desc.getSpecifier());
     desc.setSpecifier(newSpec);
 
@@ -1326,11 +1326,11 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
               // Flattens the buffer to batchLvlRank.
               auto reassoc = getReassociationForFlattening(
                   mem.getType(), stt.getBatchLvlRank());
-              mem = rewriter.create<memref::CastOp>(
+              mem = memref::CastOp::create(rewriter,
                   loc, fType,
-                  rewriter.create<memref::CollapseShapeOp>(loc, mem, reassoc));
+                  memref::CollapseShapeOp::create(rewriter, loc, mem, reassoc));
             } else {
-              mem = rewriter.create<memref::CastOp>(loc, fType, mem);
+              mem = memref::CastOp::create(rewriter, loc, fType, mem);
             }
             fields.push_back(mem);
           }
@@ -1362,8 +1362,8 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       LevelType lt = stt.getLvlType(lvl);
       // Simply forwards the position index when this is a dense level.
       if (lt.isa<LevelFormat::Dense>()) {
-        memSize = rewriter.create<arith::MulIOp>(loc, lvlSize, memSize);
-        posBack = rewriter.create<arith::SubIOp>(loc, memSize, c1);
+        memSize = arith::MulIOp::create(rewriter, loc, lvlSize, memSize);
+        posBack = arith::SubIOp::create(rewriter, loc, memSize, c1);
         continue;
       }
       if (lt.isa<LevelFormat::Batch>()) {
@@ -1376,12 +1376,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       if (isWithPosLT(lt)) {
         assert(isCompressedLT(lt) || isLooseCompressedLT(lt));
         if (isLooseCompressedLT(lt)) {
-          memSize = rewriter.create<arith::MulIOp>(loc, memSize, c2);
-          posBack = rewriter.create<arith::SubIOp>(loc, memSize, c1);
+          memSize = arith::MulIOp::create(rewriter, loc, memSize, c2);
+          posBack = arith::SubIOp::create(rewriter, loc, memSize, c1);
         } else {
           assert(isCompressedLT(lt));
           posBack = memSize;
-          memSize = rewriter.create<arith::AddIOp>(loc, memSize, c1);
+          memSize = arith::AddIOp::create(rewriter, loc, memSize, c1);
         }
         desc.setPosMemSize(rewriter, loc, lvl, memSize);
         // The last value in position array is the memory size for next level.
@@ -1391,12 +1391,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
                                    constantIndex(rewriter, loc, 0));
         batched.push_back(posBack);
         memSize = genIndexLoad(rewriter, loc, desc.getPosMemRef(lvl), batched);
-        posBack = rewriter.create<arith::SubIOp>(loc, posBack, c1);
+        posBack = arith::SubIOp::create(rewriter, loc, posBack, c1);
       }
       assert(isWithCrdLT(lt) && lvl <= trailCOOStart);
       // FIXME: This seems to be unnecessarily complex, can we simplify it?
       if (lvl == trailCOOStart) {
-        Value cooSz = rewriter.create<arith::MulIOp>(
+        Value cooSz = arith::MulIOp::create(rewriter,
             loc, memSize, constantIndex(rewriter, loc, trailCOORank));
         desc.setCrdMemSize(rewriter, loc, lvl, cooSz);
       } else {
@@ -1460,18 +1460,18 @@ struct SparseDisassembleOpConverter
       if (dst.getType().getRank() > stt.getBatchLvlRank() + 1) {
         auto reassoc =
             getReassociationForFlattening(dst.getType(), stt.getBatchLvlRank());
-        flatOut = rewriter.create<memref::CollapseShapeOp>(loc, dst, reassoc);
+        flatOut = memref::CollapseShapeOp::create(rewriter, loc, dst, reassoc);
       }
       Value dstMem = genSliceToSize(rewriter, loc, flatOut, sz);
       Value srcMem = genSliceToSize(rewriter, loc, src, sz);
-      rewriter.create<memref::CopyOp>(loc, srcMem, dstMem);
+      memref::CopyOp::create(rewriter, loc, srcMem, dstMem);
       return true;
     });
 
     // Converts MemRefs back to Tensors.
     SmallVector<Value> retValues = llvm::to_vector(
         llvm::map_range(retMem, [&rewriter, loc](Value v) -> Value {
-          return rewriter.create<bufferization::ToTensorOp>(
+          return bufferization::ToTensorOp::create(rewriter,
               loc, memref::getTensorTypeFromMemRefType(v.getType()), v);
         }));
     // Appends the actual memory length used in each buffer returned.
@@ -1549,13 +1549,13 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Level lvlRank = dstTp.getLvlRank();
     if (dstTp.isOrderedLvl(lvlRank - 1)) {
       Value kFalse = constantI1(rewriter, loc, false);
-      Value notSorted = rewriter.create<arith::CmpIOp>(
+      Value notSorted = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, isSorted, kFalse);
       scf::IfOp ifOp =
-          rewriter.create<scf::IfOp>(loc, notSorted, /*else*/ false);
+          scf::IfOp::create(rewriter, loc, notSorted, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       auto xPerm = rewriter.getMultiDimIdentityMap(lvlRank);
-      rewriter.create<SortOp>(loc, nse, xs, ValueRange{ys}, xPerm,
+      SortOp::create(rewriter, loc, nse, xs, ValueRange{ys}, xPerm,
                               rewriter.getIndexAttr(0),
                               SparseTensorSortKind::HybridQuickSort);
       rewriter.setInsertionPointAfter(ifOp);
@@ -1566,10 +1566,10 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Value posMemref0 = desc.getPosMemRef(0);
     const Type posTp = dstTp.getPosType();
     const Value posNse = genCast(rewriter, loc, nse, posTp);
-    rewriter.create<memref::StoreOp>(loc, posNse, posMemref0, c1);
+    memref::StoreOp::create(rewriter, loc, posNse, posMemref0, c1);
 
     // Update storage specifier.
-    Value coordinatesSize = rewriter.create<arith::MulIOp>(
+    Value coordinatesSize = arith::MulIOp::create(rewriter,
         loc, nse, constantIndex(rewriter, loc, lvlRank));
     desc.setSpecifierField(rewriter, loc, StorageSpecifierKind::CrdMemSize, 0,
                            coordinatesSize);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 50ccb43d432b6..54ea1843b457f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -137,7 +137,7 @@ static SmallVector<Value> getDimSizes(OpBuilder &builder, Location loc,
 /// this buffer must be explicitly deallocated by client.
 static Value genAlloc(RewriterBase &rewriter, Location loc, Value sz, Type tp) {
   auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
-  return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
+  return memref::AllocOp::create(rewriter, loc, memTp, ValueRange{sz});
 }
 
 /// Generates a temporary buffer for the level-types of the given encoding.
@@ -154,7 +154,7 @@ static Value genLvlTypesBuffer(OpBuilder &builder, Location loc,
 static Value extractBarePtrFromTensor(OpBuilder &builder, Location loc,
                                       Value tensor) {
   auto buf = genToMemref(builder, loc, tensor);
-  return builder.create<memref::ExtractAlignedPointerAsIndexOp>(loc, buf);
+  return memref::ExtractAlignedPointerAsIndexOp::create(builder, loc, buf);
 }
 
 /// Generates a temporary buffer for the level-types of the given encoding.
@@ -168,11 +168,11 @@ static Value genLvlPtrsBuffers(OpBuilder &builder, Location loc,
 
   // Passing in value buffer pointers.
   lvlBarePtrs.push_back(extractBarePtrFromTensor(builder, loc, valTensor));
-  Value idxPtr = builder.create<memref::ExtractAlignedPointerAsIndexOp>(
+  Value idxPtr = memref::ExtractAlignedPointerAsIndexOp::create(builder,
       loc, allocaBuffer(builder, loc, lvlBarePtrs));
   Value idxCast =
-      builder.create<arith::IndexCastOp>(loc, builder.getI64Type(), idxPtr);
-  return builder.create<LLVM::IntToPtrOp>(loc, getOpaquePointerType(builder),
+      arith::IndexCastOp::create(builder, loc, builder.getI64Type(), idxPtr);
+  return LLVM::IntToPtrOp::create(builder, loc, getOpaquePointerType(builder),
                                           idxCast);
 }
 
@@ -227,7 +227,7 @@ class NewCallParams final {
     assert(isInitialized() && "Must initialize before genNewCall");
     StringRef name = "newSparseTensor";
     params[kParamAction] = constantAction(builder, loc, action);
-    params[kParamPtr] = ptr ? ptr : builder.create<LLVM::ZeroOp>(loc, pTp);
+    params[kParamPtr] = ptr ? ptr : LLVM::ZeroOp::create(builder, loc, pTp);
     return createFuncCall(builder, loc, name, pTp, params, EmitCInterface::On)
         .getResult(0);
   }
@@ -539,7 +539,7 @@ class SparseTensorToCoordinatesConverter
     // Cast the MemRef type to the type expected by the users, though these
     // two types should be compatible at runtime.
     if (op.getType() != crds.getType())
-      crds = rewriter.create<memref::CastOp>(loc, op.getType(), crds);
+      crds = memref::CastOp::create(rewriter, loc, op.getType(), crds);
     rewriter.replaceOp(op, crds);
     return success();
   }
@@ -560,7 +560,7 @@ class SparseToCoordinatesBufferConverter
     // Cast the MemRef type to the type expected by the users, though these
     // two types should be compatible at runtime.
     if (op.getType() != crds.getType())
-      crds = rewriter.create<memref::CastOp>(loc, op.getType(), crds);
+      crds = memref::CastOp::create(rewriter, loc, op.getType(), crds);
     rewriter.replaceOp(op, crds);
     return success();
   }
@@ -652,7 +652,7 @@ class SparseTensorInsertConverter
       vref = genAllocaScalar(rewriter, loc, elemTp);
     }
     storeAll(rewriter, loc, lvlCoords, adaptor.getIndices());
-    rewriter.create<memref::StoreOp>(loc, adaptor.getScalar(), vref);
+    memref::StoreOp::create(rewriter, loc, adaptor.getScalar(), vref);
     SmallString<12> name{"lexInsert", primaryTypeFunctionSuffix(elemTp)};
     createFuncCall(rewriter, loc, name, {},
                    {adaptor.getDest(), lvlCoords, vref}, EmitCInterface::On);
@@ -690,10 +690,10 @@ class SparseTensorExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, eltType)},
         ValueRange{values});
-    rewriter.create<linalg::FillOp>(
+    linalg::FillOp::create(rewriter,
         loc, ValueRange{constantZero(rewriter, loc, boolType)},
         ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
@@ -733,9 +733,9 @@ class SparseTensorCompressConverter : public OpConversionPattern<CompressOp> {
     rewriter.replaceOp(op, adaptor.getTensor());
     // Deallocate the buffers on exit of the loop nest.
     rewriter.setInsertionPointAfter(parent);
-    rewriter.create<memref::DeallocOp>(loc, values);
-    rewriter.create<memref::DeallocOp>(loc, filled);
-    rewriter.create<memref::DeallocOp>(loc, added);
+    memref::DeallocOp::create(rewriter, loc, values);
+    memref::DeallocOp::create(rewriter, loc, filled);
+    memref::DeallocOp::create(rewriter, loc, added);
     return success();
   }
 };
@@ -837,21 +837,21 @@ class SparseTensorDisassembleConverter
                                       cooStartLvl + 1);
       auto crdLen = linalg::createOrFoldDimOp(rewriter, loc, crds0, 0);
       auto two = constantIndex(rewriter, loc, 2);
-      auto bufLen = rewriter.create<arith::MulIOp>(loc, crdLen, two);
+      auto bufLen = arith::MulIOp::create(rewriter, loc, crdLen, two);
       Type indexType = rewriter.getIndexType();
       auto zero = constantZero(rewriter, loc, indexType);
       auto one = constantOne(rewriter, loc, indexType);
-      scf::ForOp forOp = rewriter.create<scf::ForOp>(loc, zero, crdLen, one);
+      scf::ForOp forOp = scf::ForOp::create(rewriter, loc, zero, crdLen, one);
       auto idx = forOp.getInductionVar();
       rewriter.setInsertionPointToStart(forOp.getBody());
-      auto c0 = rewriter.create<memref::LoadOp>(loc, crds0, idx);
-      auto c1 = rewriter.create<memref::LoadOp>(loc, crds1, idx);
+      auto c0 = memref::LoadOp::create(rewriter, loc, crds0, idx);
+      auto c1 = memref::LoadOp::create(rewriter, loc, crds1, idx);
       SmallVector<Value> args;
       args.push_back(idx);
       args.push_back(zero);
-      rewriter.create<memref::StoreOp>(loc, c0, buf, args);
+      memref::StoreOp::create(rewriter, loc, c0, buf, args);
       args[1] = one;
-      rewriter.create<memref::StoreOp>(loc, c1, buf, args);
+      memref::StoreOp::create(rewriter, loc, c1, buf, args);
       rewriter.setInsertionPointAfter(forOp);
       auto bufLenTp = op.getLvlLens().getTypes()[retLen.size()];
       retVal.push_back(buf);
@@ -867,11 +867,11 @@ class SparseTensorDisassembleConverter
     // Converts MemRefs back to Tensors.
     assert(retVal.size() + retLen.size() == op.getNumResults());
     for (unsigned i = 0, sz = retVal.size(); i < sz; i++) {
-      auto tensor = rewriter.create<bufferization::ToTensorOp>(
+      auto tensor = bufferization::ToTensorOp::create(rewriter,
           loc, memref::getTensorTypeFromMemRefType(retVal[i].getType()),
           retVal[i]);
       retVal[i] =
-          rewriter.create<tensor::CastOp>(loc, op.getResultTypes()[i], tensor);
+          tensor::CastOp::create(rewriter, loc, op.getResultTypes()[i], tensor);
     }
 
     // Appends the actual memory length used in each buffer returned.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index d4a02bf7a70b6..ee963ab5d1ffb 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -127,7 +127,7 @@ static void sizesForTensor(OpBuilder &builder, SmallVectorImpl<Value> &sizes,
   for (const auto &d : enumerate(stp.getShape())) {
     Value dim;
     if (d.value() == ShapedType::kDynamic)
-      dim = builder.create<tensor::DimOp>(loc, tensor, d.index());
+      dim = tensor::DimOp::create(builder, loc, tensor, d.index());
     else
       dim = constantIndex(builder, loc, d.value());
     sizes.push_back(dim);
@@ -198,7 +198,7 @@ static void concatSizesFromInputs(OpBuilder &builder,
     for (const auto &src : srcs.drop_front()) {
       Value srcSz = linalg::createOrFoldDimOp(builder, loc, src, dim);
       // Sum up all the sizes.
-      sizes[dim] = builder.create<arith::AddIOp>(loc, sizes[dim], srcSz);
+      sizes[dim] = arith::AddIOp::create(builder, loc, sizes[dim], srcSz);
     }
   }
 }
@@ -405,7 +405,7 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     inputOps.push_back(op.getDpsInputOperand(1 - other)->get());
     fusedIndexMaps.push_back(fusedIndexMaps.back()); // mimic other
     // Fuse producer and consumer into a new generic op.
-    auto fusedOp = rewriter.create<GenericOp>(
+    auto fusedOp = GenericOp::create(rewriter,
         loc, op.getResult(0).getType(), inputOps, outputOps,
         rewriter.getAffineMapArrayAttr(fusedIndexMaps), prod.getIteratorTypes(),
         /*doc=*/nullptr, /*library_call=*/nullptr);
@@ -430,7 +430,7 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     mapper.map(consBlock.getArgument(other), fusedBlock->back().getResult(0));
     mapper.map(last, rewriter.clone(*sampler, mapper)->getResult(0));
     last = rewriter.clone(*acc, mapper)->getResult(0);
-    rewriter.create<linalg::YieldOp>(loc, last);
+    linalg::YieldOp::create(rewriter, loc, last);
     // Force initial value on merged allocation for dense outputs.
     // TODO: deal with non alloc tensor here one day
     if (!getSparseTensorEncoding(op.getResult(0).getType())) {
@@ -534,7 +534,7 @@ struct GenSemiRingSelect : public OpRewritePattern<GenericOp> {
       assert(t.getType() == f.getType());
       auto selTp = t.getType();
       auto c0 = constantZero(rewriter, loc, selTp);
-      auto binOp = rewriter.create<sparse_tensor::BinaryOp>(loc, selTp, t, f);
+      auto binOp = sparse_tensor::BinaryOp::create(rewriter, loc, selTp, t, f);
       // Initializes all the blocks.
       rewriter.createBlock(&binOp.getOverlapRegion(), {}, {selTp, selTp},
                            {t.getLoc(), f.getLoc()});
@@ -564,7 +564,7 @@ struct GenSemiRingSelect : public OpRewritePattern<GenericOp> {
           irMap.map(f, b->getArgument(1));
         }
         auto y = rewriter.clone(inst, irMap)->getResult(0);
-        rewriter.create<sparse_tensor::YieldOp>(loc, y);
+        sparse_tensor::YieldOp::create(rewriter, loc, y);
       }
 
       // We successfully rewrited a operation. We can not do replacement here
@@ -674,28 +674,28 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     // Identity.
     Location loc = op.getLoc();
     Value identity =
-        rewriter.create<tensor::ExtractOp>(loc, init->get(), ValueRange());
+        tensor::ExtractOp::create(rewriter, loc, init->get(), ValueRange());
     // Unary {
     //    present -> value
     //    absent  -> zero.
     // }
     Type rtp = s0.getType();
     rewriter.setInsertionPointToStart(&op.getRegion().front());
-    auto semiring = rewriter.create<sparse_tensor::UnaryOp>(loc, rtp, s0);
+    auto semiring = sparse_tensor::UnaryOp::create(rewriter, loc, rtp, s0);
     Block *present =
         rewriter.createBlock(&semiring.getPresentRegion(), {}, rtp, loc);
     rewriter.setInsertionPointToStart(&semiring.getPresentRegion().front());
-    rewriter.create<sparse_tensor::YieldOp>(loc, present->getArgument(0));
+    sparse_tensor::YieldOp::create(rewriter, loc, present->getArgument(0));
     rewriter.createBlock(&semiring.getAbsentRegion(), {}, {}, {});
     rewriter.setInsertionPointToStart(&semiring.getAbsentRegion().front());
     auto zero =
-        rewriter.create<arith::ConstantOp>(loc, rewriter.getZeroAttr(rtp));
-    rewriter.create<sparse_tensor::YieldOp>(loc, zero);
+        arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(rtp));
+    sparse_tensor::YieldOp::create(rewriter, loc, zero);
     rewriter.setInsertionPointAfter(semiring);
     // CustomReduce {
     //    x = x REDUC y, identity
     // }
-    auto custom = rewriter.create<sparse_tensor::ReduceOp>(
+    auto custom = sparse_tensor::ReduceOp::create(rewriter,
         loc, rtp, semiring.getResult(), s1, identity);
     Block *region =
         rewriter.createBlock(&custom.getRegion(), {}, {rtp, rtp}, {loc, loc});
@@ -704,7 +704,7 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     irMap.map(red->getOperand(0), region->getArgument(0));
     irMap.map(red->getOperand(1), region->getArgument(1));
     auto *cloned = rewriter.clone(*red, irMap);
-    rewriter.create<sparse_tensor::YieldOp>(loc, cloned->getResult(0));
+    sparse_tensor::YieldOp::create(rewriter, loc, cloned->getResult(0));
     rewriter.setInsertionPointAfter(custom);
     rewriter.replaceOp(red, custom.getResult());
     return success();
@@ -723,14 +723,14 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto tensor = op.getTensor();
     auto stt = getSparseTensorType(tensor);
     // Header with NSE.
-    auto nse = rewriter.create<NumberOfEntriesOp>(loc, tensor);
-    rewriter.create<vector::PrintOp>(
+    auto nse = NumberOfEntriesOp::create(rewriter, loc, tensor);
+    vector::PrintOp::create(rewriter,
         loc, rewriter.getStringAttr("---- Sparse Tensor ----\nnse = "));
-    rewriter.create<vector::PrintOp>(loc, nse);
+    vector::PrintOp::create(rewriter, loc, nse);
     // Print run-time contents for dim/lvl sizes.
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("dim = "));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("dim = "));
     printSizes(rewriter, loc, tensor, stt.getDimRank(), /*isDim=*/true);
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("lvl = "));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("lvl = "));
     printSizes(rewriter, loc, tensor, stt.getLvlRank(), /*isDim=*/false);
     // Use the "codegen" foreach loop construct to iterate over
     // all typical sparse tensor components for printing.
@@ -744,42 +744,42 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       }
       case SparseTensorFieldKind::PosMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("pos["));
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("pos["));
+        vector::PrintOp::create(rewriter,
             loc, lvl, vector::PrintPunctuation::NoPunctuation);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("] : "));
-        auto pos = rewriter.create<ToPositionsOp>(loc, tensor, l);
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
+        auto pos = ToPositionsOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, pos);
         break;
       }
       case SparseTensorFieldKind::CrdMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("crd["));
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("crd["));
+        vector::PrintOp::create(rewriter,
             loc, lvl, vector::PrintPunctuation::NoPunctuation);
-        rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("] : "));
+        vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
         Value crd = nullptr;
         // For COO AoS storage, we want to print a single, linear view of
         // the full coordinate storage at this level. For any other storage,
         // we show the coordinate storage for every indivual level.
         if (stt.getAoSCOOStart() == l)
-          crd = rewriter.create<ToCoordinatesBufferOp>(loc, tensor);
+          crd = ToCoordinatesBufferOp::create(rewriter, loc, tensor);
         else
-          crd = rewriter.create<ToCoordinatesOp>(loc, tensor, l);
+          crd = ToCoordinatesOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, crd);
         break;
       }
       case SparseTensorFieldKind::ValMemRef: {
-        rewriter.create<vector::PrintOp>(loc,
+        vector::PrintOp::create(rewriter, loc,
                                          rewriter.getStringAttr("values : "));
-        auto val = rewriter.create<ToValuesOp>(loc, tensor);
+        auto val = ToValuesOp::create(rewriter, loc, tensor);
         printContents(rewriter, loc, val);
         break;
       }
       }
       return true;
     });
-    rewriter.create<vector::PrintOp>(loc, rewriter.getStringAttr("----\n"));
+    vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("----\n"));
     rewriter.eraseOp(op);
     return success();
   }
@@ -797,7 +797,7 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto shape = cast<ShapedType>(vec.getType()).getShape();
     SmallVector<Value> idxs;
     printContentsLevel(rewriter, loc, vec, 0, shape, idxs);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::NewLine);
   }
 
   // Helper to the helper.
@@ -805,13 +805,13 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
                                  Value vec, unsigned i, ArrayRef<int64_t> shape,
                                  SmallVectorImpl<Value> &idxs) {
     // Open bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
     // Generate for loop.
     auto zero = constantIndex(rewriter, loc, 0);
     auto index = constantIndex(rewriter, loc, i);
-    auto size = rewriter.create<memref::DimOp>(loc, vec, index);
+    auto size = memref::DimOp::create(rewriter, loc, vec, index);
     auto step = constantIndex(rewriter, loc, 1);
-    auto forOp = rewriter.create<scf::ForOp>(loc, zero, size, step);
+    auto forOp = scf::ForOp::create(rewriter, loc, zero, size, step);
     idxs.push_back(forOp.getInductionVar());
     rewriter.setInsertionPointToStart(forOp.getBody());
     if (i < shape.size() - 1) {
@@ -819,56 +819,56 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       printContentsLevel(rewriter, loc, vec, i + 1, shape, idxs);
     } else {
       // Actual contents printing.
-      auto val = rewriter.create<memref::LoadOp>(loc, vec, idxs);
+      auto val = memref::LoadOp::create(rewriter, loc, vec, idxs);
       if (llvm::isa<ComplexType>(val.getType())) {
         // Since the vector dialect does not support complex types in any op,
         // we split those into (real, imag) pairs here.
-        Value real = rewriter.create<complex::ReOp>(loc, val);
-        Value imag = rewriter.create<complex::ImOp>(loc, val);
-        rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
-        rewriter.create<vector::PrintOp>(loc, real,
+        Value real = complex::ReOp::create(rewriter, loc, val);
+        Value imag = complex::ImOp::create(rewriter, loc, val);
+        vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
+        vector::PrintOp::create(rewriter, loc, real,
                                          vector::PrintPunctuation::Comma);
-        rewriter.create<vector::PrintOp>(loc, imag,
+        vector::PrintOp::create(rewriter, loc, imag,
                                          vector::PrintPunctuation::Close);
       } else {
-        rewriter.create<vector::PrintOp>(
+        vector::PrintOp::create(rewriter,
             loc, val, vector::PrintPunctuation::NoPunctuation);
       }
       // Terminating comma (except at end).
-      auto bound = rewriter.create<arith::AddIOp>(loc, idxs.back(), step);
-      Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+      auto bound = arith::AddIOp::create(rewriter, loc, idxs.back(), step);
+      Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
                                                   bound, size);
-      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, cond, /*else*/ false);
+      scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, cond, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Comma);
+      vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Comma);
     }
     idxs.pop_back();
     rewriter.setInsertionPointAfter(forOp);
     // Close bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
   }
 
   // Helper method to print run-time lvl/dim sizes.
   static void printSizes(PatternRewriter &rewriter, Location loc, Value tensor,
                          unsigned size, bool isDim) {
     // Open bracket.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
     // Print unrolled contents (dimop requires constant value).
     for (unsigned i = 0; i < size; i++) {
       auto idx = constantIndex(rewriter, loc, i);
       Value val;
       if (isDim)
-        val = rewriter.create<tensor::DimOp>(loc, tensor, idx);
+        val = tensor::DimOp::create(rewriter, loc, tensor, idx);
       else
-        val = rewriter.create<LvlOp>(loc, tensor, idx);
-      rewriter.create<vector::PrintOp>(
+        val = LvlOp::create(rewriter, loc, tensor, idx);
+      vector::PrintOp::create(rewriter,
           loc, val,
           i != size - 1 ? vector::PrintPunctuation::Comma
                         : vector::PrintPunctuation::NoPunctuation);
     }
     // Close bracket and end of line.
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
+    vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::NewLine);
   }
 };
 
@@ -896,7 +896,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     for (Dimension d : dstTp->getDimShape())
       dstSizes.push_back(constantIndex(rewriter, loc, d));
 
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, srcTensor);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, srcTensor);
     // Only need an unordered COO buffer if input and output are not sorted
     // in the same way.
     Type bufferTp = getBufferType(
@@ -920,7 +920,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp->getEncoding();
-    ForeachOp foreachOp = rewriter.create<ForeachOp>(
+    ForeachOp foreachOp = ForeachOp::create(rewriter,
         loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
@@ -935,7 +935,7 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
           Value collapseSize = constantIndex(builder, loc, 1);
           for (Dimension d = 0; d < srcRank; d++)
             collapseSize =
-                builder.create<arith::MulIOp>(loc, collapseSize, srcSizes[d]);
+                arith::MulIOp::create(builder, loc, collapseSize, srcSizes[d]);
           SmallVector<Value, 1> collapsedSizes = {collapseSize};
 
           ReassociationIndices collapseIdx;
@@ -955,15 +955,15 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
                      dstSizes, dstDcvs);
 
           auto t =
-              builder.create<tensor::InsertOp>(loc, v, reduc.front(), dstDcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+              tensor::InsertOp::create(builder, loc, v, reduc.front(), dstDcvs);
+          sparse_tensor::YieldOp::create(builder, loc, t);
         });
 
-    Value t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
+    Value t = LoadOp::create(rewriter, loc, foreachOp.getResult(0), true);
     if (bufferTp != *dstTp) {
       auto dstRTT = dstTp->getRankedTensorType();
-      Value converted = rewriter.create<ConvertOp>(loc, dstRTT, t).getResult();
-      rewriter.create<DeallocTensorOp>(loc, t);
+      Value converted = ConvertOp::create(rewriter, loc, dstRTT, t).getResult();
+      DeallocTensorOp::create(rewriter, loc, t);
       t = converted;
     }
     rewriter.replaceOp(op, t);
@@ -1004,7 +1004,7 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           dstDynSizes.push_back(dstSizes[idx]);
       }
     }
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, srcTensor);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, srcTensor);
     // Only need a unordered COO buffer if input and output are not sorted
     // in the same way.
     Type bufferTp = getBufferType(
@@ -1025,7 +1025,7 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp.getEncoding();
-    ForeachOp foreachOp = rewriter.create<ForeachOp>(
+    ForeachOp foreachOp = ForeachOp::create(rewriter,
         loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
@@ -1040,15 +1040,15 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           reshapeCvs(builder, loc, op.getReassociationIndices(), srcSizes,
                      srcDcvs, dstSizes, dstDcvs);
           auto t =
-              builder.create<tensor::InsertOp>(loc, v, reduc.front(), dstDcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+              tensor::InsertOp::create(builder, loc, v, reduc.front(), dstDcvs);
+          sparse_tensor::YieldOp::create(builder, loc, t);
         });
 
-    Value t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
+    Value t = LoadOp::create(rewriter, loc, foreachOp.getResult(0), true);
     if (bufferTp != dstTp) {
       auto dstRTT = dstTp.getRankedTensorType();
-      Value converted = rewriter.create<ConvertOp>(loc, dstRTT, t).getResult();
-      rewriter.create<DeallocTensorOp>(loc, t);
+      Value converted = ConvertOp::create(rewriter, loc, dstRTT, t).getResult();
+      DeallocTensorOp::create(rewriter, loc, t);
       t = converted;
     }
     rewriter.replaceOp(op, t);
@@ -1079,7 +1079,7 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
       auto rtp = getRankedTensorType(op.getSrc());
       auto denseTp =
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
-      auto convert = rewriter.create<ConvertOp>(loc, denseTp, op.getSrc());
+      auto convert = ConvertOp::create(rewriter, loc, denseTp, op.getSrc());
       rewriter.modifyOpInPlace(op, [&]() { op->setOperand(0, convert); });
       return success();
     }
@@ -1089,14 +1089,14 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
       ReshapeOp reshape;
       if constexpr (std::is_same<ReshapeOp, tensor::ExpandShapeOp>::value) {
-        reshape = rewriter.create<ReshapeOp>(
+        reshape = ReshapeOp::create(rewriter,
             loc, denseTp, op.getSrc(), op.getReassociation(),
             op.getOutputShape(), op.getStaticOutputShape());
       } else {
-        reshape = rewriter.create<ReshapeOp>(loc, denseTp, op.getSrc(),
+        reshape = ReshapeOp::create(rewriter, loc, denseTp, op.getSrc(),
                                              op.getReassociation());
       }
-      Value convert = rewriter.create<ConvertOp>(loc, rtp, reshape);
+      Value convert = ConvertOp::create(rewriter, loc, rtp, reshape);
       rewriter.replaceOp(op, convert);
       return success();
     }
@@ -1112,20 +1112,20 @@ struct TensorLike {
     SmallVector<Value> dynSzs;
     getDynamicSizes(rtt, sizes, dynSzs);
 
-    val = builder.create<AllocTensorOp>(loc, rtt, dynSzs);
+    val = AllocTensorOp::create(builder, loc, rtt, dynSzs);
     if (!isSparse()) {
       Value c0 = constantZero(builder, loc, rtt.getElementType());
-      val = builder.create<linalg::FillOp>(loc, c0, val).getResult(0);
+      val = linalg::FillOp::create(builder, loc, c0, val).getResult(0);
     }
   }
 
   void insert(OpBuilder &builder, Location loc, Value v, ValueRange crds) {
-    val = builder.create<tensor::InsertOp>(loc, v, val, crds);
+    val = tensor::InsertOp::create(builder, loc, v, val, crds);
   }
 
   Value finalize(OpBuilder &builder, Location loc, RankedTensorType rtp) const {
     if (isSparse())
-      return builder.create<LoadOp>(loc, val, true);
+      return LoadOp::create(builder, loc, val, true);
     return val;
   }
 
@@ -1160,18 +1160,18 @@ struct SparseTensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
     Location loc = op.getLoc();
     SmallVector<Value> maxLvlCrds;
     for (Level l = 0; l < stt->getLvlRank(); l++) {
-      Value lvlSz = rewriter.create<LvlOp>(loc, op.getSource(), l);
-      Value maxLvlCrd = rewriter.create<arith::SubIOp>(
+      Value lvlSz = LvlOp::create(rewriter, loc, op.getSource(), l);
+      Value maxLvlCrd = arith::SubIOp::create(rewriter,
           loc, lvlSz, constantOne(rewriter, loc, rewriter.getIndexType()));
       maxLvlCrds.push_back(maxLvlCrd);
     }
 
     AffineExpr lvl2DimExp = stt->getLvlToDim().getResult(*dim);
-    Value maxDimCrd = rewriter.create<affine::AffineApplyOp>(
+    Value maxDimCrd = affine::AffineApplyOp::create(rewriter,
         op.getLoc(), AffineMap::get(stt->getLvlRank(), 0, lvl2DimExp),
         maxLvlCrds);
 
-    Value dimSz = rewriter.create<arith::AddIOp>(
+    Value dimSz = arith::AddIOp::create(rewriter,
         loc, maxDimCrd, constantOne(rewriter, loc, rewriter.getIndexType()));
     rewriter.replaceOp(op, dimSz);
     return success();
@@ -1212,26 +1212,26 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
     for (Value input : op.getInputs()) {
       // Builds a for op for each input tensor to append new values into the
       // output tensor.
-      foreachOp = rewriter.create<ForeachOp>(
+      foreachOp = ForeachOp::create(rewriter,
           loc, input, iterArg,
           [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
               ValueRange reduc) {
             SmallVector<Value> offDimCrd(dcvs);
             offDimCrd[conDim] =
-                builder.create<arith::AddIOp>(loc, offDimCrd[conDim], offset);
+                arith::AddIOp::create(builder, loc, offDimCrd[conDim], offset);
 
             // Enters foreach, updates the SSA chain.
             dstBuf.val = reduc.front();
             if (!dstTp.isAllDense()) {
               Value cond = genIsNonzero(builder, loc, v);
-              auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+              auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
                                                     /*else*/ true);
               builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-              builder.create<scf::YieldOp>(loc, dstBuf.val);
+              scf::YieldOp::create(builder, loc, dstBuf.val);
 
               builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
               dstBuf.insert(builder, loc, v, offDimCrd);
-              builder.create<scf::YieldOp>(loc, dstBuf.val);
+              scf::YieldOp::create(builder, loc, dstBuf.val);
 
               // Exits the ifOp, update the sparse tensor SSA value.
               builder.setInsertionPointAfter(ifOp);
@@ -1239,14 +1239,14 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
             } else {
               dstBuf.insert(builder, loc, v, offDimCrd);
             }
-            builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
+            sparse_tensor::YieldOp::create(builder, loc, dstBuf.val);
           });
       // Accumulates the offset. Note that only static-shaped inputs are allowed
       // by concatenate op verifier, which saves us from computing the offset
       // dynamically.
       const Size sz = getSparseTensorType(input).getDynamicDimSize(conDim);
       assert(ShapedType::isStatic(sz));
-      offset = rewriter.create<arith::AddIOp>(loc, offset,
+      offset = arith::AddIOp::create(rewriter, loc, offset,
                                               constantIndex(rewriter, loc, sz));
       iterArg = foreachOp.getResult(0);
       dstBuf.val = iterArg;
@@ -1299,7 +1299,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
     ValueRange vs;
     TensorLike dstBuf(rewriter, loc, dstStt.getRankedTensorType(), sizes);
 
-    auto foreachOp = rewriter.create<ForeachOp>(
+    auto foreachOp = ForeachOp::create(rewriter,
         loc, src, dstBuf.val, foreachOrder,
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
@@ -1307,14 +1307,14 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           dstBuf.val = reduc.front();
           if (!skipZeroCheck) {
             Value cond = genIsNonzero(builder, loc, v);
-            auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+            auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
                                                   /*else*/ true);
             builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-            builder.create<scf::YieldOp>(loc, dstBuf.val);
+            scf::YieldOp::create(builder, loc, dstBuf.val);
 
             builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
             dstBuf.insert(builder, loc, v, dcvs);
-            builder.create<scf::YieldOp>(loc, dstBuf.val);
+            scf::YieldOp::create(builder, loc, dstBuf.val);
 
             // Exits the ifOp, update the sparse tensor SSA value.
             builder.setInsertionPointAfter(ifOp);
@@ -1322,7 +1322,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           } else {
             dstBuf.insert(builder, loc, v, dcvs);
           }
-          builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
+          sparse_tensor::YieldOp::create(builder, loc, dstBuf.val);
         });
 
     rewriter.setInsertionPointAfter(foreachOp);
@@ -1349,7 +1349,7 @@ struct CrdTranslateRewriter : public OpRewritePattern<CrdTranslateOp> {
       // TODO: we should probably expand the affine map to IR using our own
       // rules, since affine.apply assume signed value, while the cooridinates
       // we provided must always be signless.
-      Value trans = rewriter.create<affine::AffineApplyOp>(
+      Value trans = affine::AffineApplyOp::create(rewriter,
           op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
           op.getInCrds());
       outCrds.push_back(trans);
@@ -1412,8 +1412,8 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
     SmallVector<Value> pos = loopEmitter.getValPosits(0);
     // Loads the value from sparse tensor using position-index;
     // loads the value from dense tensor using coords.
-    Value val = enc ? rewriter.create<memref::LoadOp>(loc, vals, pos)
-                    : rewriter.create<memref::LoadOp>(loc, vals, lcvs);
+    Value val = enc ? memref::LoadOp::create(rewriter, loc, vals, pos)
+                    : memref::LoadOp::create(rewriter, loc, vals, lcvs);
 
     // 2. Inline the block in the foreach operator.
     Block *srcBlock = op.getBody();
@@ -1472,22 +1472,22 @@ struct NewRewriter : public OpRewritePattern<NewOp> {
     // with enveloping reinterpreted_map ops for non-permutations.
     RankedTensorType dstTp = stt.getRankedTensorType();
     RankedTensorType cooTp = stt.getCOOType(/*ordered=*/true);
-    Value cooTensor = rewriter.create<NewOp>(loc, cooTp, op.getSource());
+    Value cooTensor = NewOp::create(rewriter, loc, cooTp, op.getSource());
     Value convert = cooTensor;
     auto enc = stt.getEncoding();
     if (!stt.isPermutation()) { // demap coo, demap dstTp
       auto coo = getSparseTensorType(cooTensor).getEncoding().withoutDimToLvl();
-      convert = rewriter.create<ReinterpretMapOp>(loc, coo, convert);
+      convert = ReinterpretMapOp::create(rewriter, loc, coo, convert);
       dstTp = getSparseTensorType(convert).withEncoding(enc.withoutDimToLvl());
     }
-    convert = rewriter.create<ConvertOp>(loc, dstTp, convert);
+    convert = ConvertOp::create(rewriter, loc, dstTp, convert);
     if (!stt.isPermutation()) // remap to original enc
-      convert = rewriter.create<ReinterpretMapOp>(loc, enc, convert);
+      convert = ReinterpretMapOp::create(rewriter, loc, enc, convert);
     rewriter.replaceOp(op, convert);
 
     // Release the temporary ordered COO tensor.
     rewriter.setInsertionPointAfterValue(convert);
-    rewriter.create<DeallocTensorOp>(loc, cooTensor);
+    DeallocTensorOp::create(rewriter, loc, cooTensor);
 
     return success();
   }
@@ -1501,7 +1501,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     Location loc = op.getLoc();
     // Calculate NNZ.
     Value src = op.getTensor();
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, src);
+    Value nnz = NumberOfEntriesOp::create(rewriter, loc, src);
 
     // Allocate a temporary buffer for storing dimension-sizes/coordinates.
     const auto srcTp = getSparseTensorType(src);
@@ -1514,7 +1514,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     SmallVector<Value> dims;
     sizesForTensor(rewriter, dims, loc, srcTp, src);
     for (Dimension d = 0; d < dimRank; d++) {
-      rewriter.create<memref::StoreOp>(loc, dims[d], dimSizes,
+      memref::StoreOp::create(rewriter, loc, dims[d], dimSizes,
                                        constantIndex(rewriter, loc, d));
     }
 
@@ -1536,20 +1536,20 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     ModuleOp module = op->getParentOfType<ModuleOp>();
 
     // For each element in the source tensor, output the element.
-    rewriter.create<ForeachOp>(
+    ForeachOp::create(rewriter,
         loc, src, ValueRange(),
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
           for (Dimension d = 0; d < dimRank; d++) {
-            rewriter.create<memref::StoreOp>(loc, dcvs[d], dimCoords,
+            memref::StoreOp::create(rewriter, loc, dcvs[d], dimCoords,
                                              constantIndex(builder, loc, d));
           }
-          rewriter.create<memref::StoreOp>(loc, v, value);
+          memref::StoreOp::create(rewriter, loc, v, value);
           SmallVector<Value> operands{writer, rankValue, dimCoords, value};
           FlatSymbolRefAttr fn = getFunc(module, outNextFuncName, {}, operands,
                                          EmitCInterface::On);
-          builder.create<func::CallOp>(loc, TypeRange(), fn, operands);
-          builder.create<sparse_tensor::YieldOp>(loc);
+          func::CallOp::create(builder, loc, TypeRange(), fn, operands);
+          sparse_tensor::YieldOp::create(builder, loc);
         });
 
     // Release the writer.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 52b66badef44b..7bccd9a64c9e4 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -78,7 +78,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
       matchPattern(step, m_Constant(&stepInt))) {
     if (((hiInt.getInt() - loInt.getInt()) % stepInt.getInt()) == 0) {
       Value trueVal = constantI1(rewriter, loc, true);
-      return rewriter.create<vector::BroadcastOp>(loc, mtp, trueVal);
+      return vector::BroadcastOp::create(rewriter, loc, mtp, trueVal);
     }
   }
   // Otherwise, generate a vector mask that avoids overrunning the upperbound
@@ -92,7 +92,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
       rewriter.getContext());
   Value end = rewriter.createOrFold<affine::AffineMinOp>(
       loc, min, ValueRange{hi, iv, step});
-  return rewriter.create<vector::CreateMaskOp>(loc, mtp, end);
+  return vector::CreateMaskOp::create(rewriter, loc, mtp, end);
 }
 
 /// Generates a vectorized invariant. Here we rely on subsequent loop
@@ -100,7 +100,7 @@ static Value genVectorMask(PatternRewriter &rewriter, Location loc, VL vl,
 static Value genVectorInvariantValue(PatternRewriter &rewriter, VL vl,
                                      Value val) {
   VectorType vtp = vectorType(vl, val.getType());
-  return rewriter.create<vector::BroadcastOp>(val.getLoc(), vtp, val);
+  return vector::BroadcastOp::create(rewriter, val.getLoc(), vtp, val);
 }
 
 /// Generates a vectorized load lhs = a[ind[lo:hi]] or lhs = a[lo:hi],
@@ -115,10 +115,10 @@ static Value genVectorLoad(PatternRewriter &rewriter, Location loc, VL vl,
     SmallVector<Value> scalarArgs(idxs);
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
-    return rewriter.create<vector::GatherOp>(loc, vtp, mem, scalarArgs,
+    return vector::GatherOp::create(rewriter, loc, vtp, mem, scalarArgs,
                                              indexVec, vmask, pass);
   }
-  return rewriter.create<vector::MaskedLoadOp>(loc, vtp, mem, idxs, vmask,
+  return vector::MaskedLoadOp::create(rewriter, loc, vtp, mem, idxs, vmask,
                                                pass);
 }
 
@@ -132,11 +132,11 @@ static void genVectorStore(PatternRewriter &rewriter, Location loc, Value mem,
     SmallVector<Value> scalarArgs(idxs);
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
-    rewriter.create<vector::ScatterOp>(loc, mem, scalarArgs, indexVec, vmask,
+    vector::ScatterOp::create(rewriter, loc, mem, scalarArgs, indexVec, vmask,
                                        rhs);
     return;
   }
-  rewriter.create<vector::MaskedStoreOp>(loc, mem, idxs, vmask, rhs);
+  vector::MaskedStoreOp::create(rewriter, loc, mem, idxs, vmask, rhs);
 }
 
 /// Detects a vectorizable reduction operations and returns the
@@ -197,18 +197,18 @@ static Value genVectorReducInit(PatternRewriter &rewriter, Location loc,
   case vector::CombiningKind::ADD:
   case vector::CombiningKind::XOR:
     // Initialize reduction vector to: | 0 | .. | 0 | r |
-    return rewriter.create<vector::InsertOp>(loc, r,
+    return vector::InsertOp::create(rewriter, loc, r,
                                              constantZero(rewriter, loc, vtp),
                                              constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::MUL:
     // Initialize reduction vector to: | 1 | .. | 1 | r |
-    return rewriter.create<vector::InsertOp>(loc, r,
+    return vector::InsertOp::create(rewriter, loc, r,
                                              constantOne(rewriter, loc, vtp),
                                              constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::AND:
   case vector::CombiningKind::OR:
     // Initialize reduction vector to: | r | .. | r | r |
-    return rewriter.create<vector::BroadcastOp>(loc, vtp, r);
+    return vector::BroadcastOp::create(rewriter, loc, vtp, r);
   default:
     break;
   }
@@ -300,10 +300,10 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
         Type etp = llvm::cast<VectorType>(vload.getType()).getElementType();
         if (!llvm::isa<IndexType>(etp)) {
           if (etp.getIntOrFloatBitWidth() < 32)
-            vload = rewriter.create<arith::ExtUIOp>(
+            vload = arith::ExtUIOp::create(rewriter,
                 loc, vectorType(vl, rewriter.getI32Type()), vload);
           else if (etp.getIntOrFloatBitWidth() < 64 && !vl.enableSIMDIndex32)
-            vload = rewriter.create<arith::ExtUIOp>(
+            vload = arith::ExtUIOp::create(rewriter,
                 loc, vectorType(vl, rewriter.getI64Type()), vload);
         }
         idxs.push_back(vload);
@@ -328,7 +328,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
             return false;
           if (codegen)
             idxs.push_back(
-                rewriter.create<arith::AddIOp>(forOp.getLoc(), inv, idx));
+                arith::AddIOp::create(rewriter, forOp.getLoc(), inv, idx));
           continue; // success so far
         }
       }
@@ -341,7 +341,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define UNAOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = rewriter.create<xxx>(loc, vx);                                    \
+      vexp = xxx::create(rewriter, loc, vx);                                    \
     return true;                                                               \
   }
 
@@ -349,7 +349,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
   if (auto x = dyn_cast<xxx>(def)) {                                           \
     if (codegen) {                                                             \
       VectorType vtp = vectorType(vl, x.getType());                            \
-      vexp = rewriter.create<xxx>(loc, vtp, vx);                               \
+      vexp = xxx::create(rewriter, loc, vtp, vx);                               \
     }                                                                          \
     return true;                                                               \
   }
@@ -357,7 +357,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define BINOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = rewriter.create<xxx>(loc, vx, vy);                                \
+      vexp = xxx::create(rewriter, loc, vx, vy);                                \
     return true;                                                               \
   }
 
@@ -380,9 +380,9 @@ static bool vectorizeExpr(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       // such as a[i] = i, which must convert to [i, i+1, ...].
       if (codegen) {
         VectorType vtp = vectorType(vl, arg.getType());
-        Value veci = rewriter.create<vector::BroadcastOp>(loc, vtp, arg);
-        Value incr = rewriter.create<vector::StepOp>(loc, vtp);
-        vexp = rewriter.create<arith::AddIOp>(loc, veci, incr);
+        Value veci = vector::BroadcastOp::create(rewriter, loc, vtp, arg);
+        Value incr = vector::StepOp::create(rewriter, loc, vtp);
+        vexp = arith::AddIOp::create(rewriter, loc, veci, incr);
       }
       return true;
     }
@@ -525,15 +525,15 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
     Value step = constantIndex(rewriter, loc, vl.vectorLength);
     if (vl.enableVLAVectorization) {
       Value vscale =
-          rewriter.create<vector::VectorScaleOp>(loc, rewriter.getIndexType());
-      step = rewriter.create<arith::MulIOp>(loc, vscale, step);
+          vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
+      step = arith::MulIOp::create(rewriter, loc, vscale, step);
     }
     if (!yield.getResults().empty()) {
       Value init = forOp.getInitArgs()[0];
       VectorType vtp = vectorType(vl, init.getType());
       Value vinit = genVectorReducInit(rewriter, loc, yield->getOperand(0),
                                        forOp.getRegionIterArg(0), init, vtp);
-      forOpNew = rewriter.create<scf::ForOp>(
+      forOpNew = scf::ForOp::create(rewriter,
           loc, forOp.getLowerBound(), forOp.getUpperBound(), step, vinit);
       forOpNew->setAttr(
           LoopEmitter::getLoopEmitterLoopAttrName(),
@@ -562,10 +562,10 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       if (codegen) {
         Value partial = forOpNew.getResult(0);
         Value vpass = genVectorInvariantValue(rewriter, vl, iter);
-        Value vred = rewriter.create<arith::SelectOp>(loc, vmask, vrhs, vpass);
-        rewriter.create<scf::YieldOp>(loc, vred);
+        Value vred = arith::SelectOp::create(rewriter, loc, vmask, vrhs, vpass);
+        scf::YieldOp::create(rewriter, loc, vred);
         rewriter.setInsertionPointAfter(forOpNew);
-        Value vres = rewriter.create<vector::ReductionOp>(loc, kind, partial);
+        Value vres = vector::ReductionOp::create(rewriter, loc, kind, partial);
         // Now do some relinking (last one is not completely type safe
         // but all bad ones are removed right away). This also folds away
         // nop broadcast operations.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
index d0e3e88f131d3..29a3e92fcf7fb 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
@@ -316,7 +316,7 @@ static void genBuffers(CodegenEnv &env, OpBuilder &builder) {
         if (!isInit) {
           Value zero = constantZero(builder, loc,
                                     getElementTypeOrSelf(tensor.getType()));
-          builder.create<linalg::FillOp>(loc, ValueRange{zero},
+          linalg::FillOp::create(builder, loc, ValueRange{zero},
                                          ValueRange{init});
         }
         return init;
@@ -379,7 +379,7 @@ static Value genInsertionLoad(CodegenEnv &env, OpBuilder &builder,
   }
   // Load from expanded access pattern.
   Value index = genIndex(env, t);
-  return builder.create<memref::LoadOp>(loc, env.getExpandValues(), index);
+  return memref::LoadOp::create(builder, loc, env.getExpandValues(), index);
 }
 
 /// Generates insertion code to implement dynamic tensor load for reduction.
@@ -395,22 +395,22 @@ static Value genInsertionLoadReduce(CodegenEnv &env, OpBuilder &builder,
   Value values = env.getExpandValues();
   Value filled = env.getExpandFilled();
   Value index = genIndex(env, t);
-  Value isFilled = builder.create<memref::LoadOp>(loc, filled, index);
-  Value valAtIndex = builder.create<memref::LoadOp>(loc, values, index);
-  return builder.create<arith::SelectOp>(loc, isFilled, valAtIndex, identity);
+  Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
+  Value valAtIndex = memref::LoadOp::create(builder, loc, values, index);
+  return arith::SelectOp::create(builder, loc, isFilled, valAtIndex, identity);
 }
 
 static Value genConditionalInsert(Location loc, OpBuilder &builder, Value cond,
                                   Value sparseOut, ValueRange ivs, Value v) {
   scf::IfOp condInsert =
-      builder.create<scf::IfOp>(loc, sparseOut.getType(), cond, true);
+      scf::IfOp::create(builder, loc, sparseOut.getType(), cond, true);
   // True branch.
   builder.setInsertionPointToStart(condInsert.thenBlock());
-  Value res = builder.create<tensor::InsertOp>(loc, v, sparseOut, ivs);
-  builder.create<scf::YieldOp>(loc, res);
+  Value res = tensor::InsertOp::create(builder, loc, v, sparseOut, ivs);
+  scf::YieldOp::create(builder, loc, res);
   // False branch.
   builder.setInsertionPointToStart(condInsert.elseBlock());
-  builder.create<scf::YieldOp>(loc, sparseOut);
+  scf::YieldOp::create(builder, loc, sparseOut);
   // Value assignment.
   builder.setInsertionPointAfter(condInsert);
   return condInsert.getResult(0);
@@ -447,7 +447,7 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
         Value nz = genIsNonzero(builder, loc, rhs);
         sparseOut = genConditionalInsert(loc, builder, nz, chain, ivs, rhs);
       } else {
-        sparseOut = builder.create<tensor::InsertOp>(loc, rhs, chain, ivs);
+        sparseOut = tensor::InsertOp::create(builder, loc, rhs, chain, ivs);
       }
       // Generates regular insertion chain.
       env.updateInsertionChain(sparseOut);
@@ -468,25 +468,25 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
   Value fval = constantI1(builder, loc, false);
   Value tval = constantI1(builder, loc, true);
   // If statement.
-  Value isFilled = builder.create<memref::LoadOp>(loc, filled, index);
-  Value cond = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+  Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
+  Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
                                              isFilled, fval);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, builder.getIndexType(), cond,
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIndexType(), cond,
                                              /*else=*/true);
   // True branch.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  builder.create<memref::StoreOp>(loc, tval, filled, index);
-  builder.create<memref::StoreOp>(loc, index, added, count);
+  memref::StoreOp::create(builder, loc, tval, filled, index);
+  memref::StoreOp::create(builder, loc, index, added, count);
   Value one = constantIndex(builder, loc, 1);
-  Value add = builder.create<arith::AddIOp>(loc, count, one);
-  builder.create<scf::YieldOp>(loc, add);
+  Value add = arith::AddIOp::create(builder, loc, count, one);
+  scf::YieldOp::create(builder, loc, add);
   // False branch.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, count);
+  scf::YieldOp::create(builder, loc, count);
   builder.setInsertionPointAfter(ifOp);
   // Value assignment.
   env.updateExpandCount(ifOp.getResult(0));
-  builder.create<memref::StoreOp>(loc, rhs, values, index);
+  memref::StoreOp::create(builder, loc, rhs, values, index);
 }
 
 /// Generates a load on a dense or sparse tensor.
@@ -516,9 +516,9 @@ static Value genTensorLoad(CodegenEnv &env, OpBuilder &builder, ExprId exp) {
   if (llvm::isa<TensorType>(ptr.getType())) {
     assert(env.options().sparseEmitStrategy ==
            SparseEmitStrategy::kSparseIterator);
-    return builder.create<ExtractValOp>(loc, ptr, llvm::getSingleElement(args));
+    return ExtractValOp::create(builder, loc, ptr, llvm::getSingleElement(args));
   }
-  return builder.create<memref::LoadOp>(loc, ptr, args);
+  return memref::LoadOp::create(builder, loc, ptr, args);
 }
 
 /// Generates a store on a dense or sparse tensor.
@@ -545,7 +545,7 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   if (!env.isSparseOutput(t)) {
     SmallVector<Value> args;
     Value ptr = genSubscript(env, builder, t, args);
-    builder.create<memref::StoreOp>(loc, rhs, ptr, args);
+    memref::StoreOp::create(builder, loc, rhs, ptr, args);
     return;
   }
   // Store during sparse insertion.
@@ -556,7 +556,7 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   // Select operation insertion.
   Value chain = env.getInsertionChain();
   scf::IfOp ifOp =
-      builder.create<scf::IfOp>(loc, chain.getType(), rhs, /*else=*/true);
+      scf::IfOp::create(builder, loc, chain.getType(), rhs, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   // Existing value was preserved to be used here.
   assert(env.exp(exp).val);
@@ -565,10 +565,10 @@ static void genTensorStore(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   env.merger().clearExprValue(exp);
   // Yield modified insertion chain along true branch.
   Value mchain = env.getInsertionChain();
-  builder.create<scf::YieldOp>(op.getLoc(), mchain);
+  scf::YieldOp::create(builder, op.getLoc(), mchain);
   // Yield original insertion chain along false branch.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<scf::YieldOp>(loc, chain);
+  scf::YieldOp::create(builder, loc, chain);
   // Done with if statement.
   env.updateInsertionChain(ifOp->getResult(0));
   builder.setInsertionPointAfter(ifOp);
@@ -597,7 +597,7 @@ static Value relinkBranch(CodegenEnv &env, RewriterBase &rewriter, Block *block,
       assert(!getSparseTensorType(t->get()).hasEncoding()); // dense!
       SmallVector<Value> args;
       Value ptr = genSubscript(env, rewriter, t, args);
-      return rewriter.create<memref::LoadOp>(op.getLoc(), ptr, args);
+      return memref::LoadOp::create(rewriter, op.getLoc(), ptr, args);
     }
   } else if (Operation *def = e.getDefiningOp()) {
     // Handle index computation.
@@ -768,7 +768,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Type t2 = MemRefType::get(dynShape, builder.getI1Type());
     Type t3 = MemRefType::get(dynShape, builder.getIndexType());
     Type t4 = builder.getIndexType();
-    auto r = builder.create<ExpandOp>(loc, TypeRange({t1, t2, t3, t4}), tensor);
+    auto r = ExpandOp::create(builder, loc, TypeRange({t1, t2, t3, t4}), tensor);
     assert(r.getNumResults() == 4);
     env.startExpand(r.getResult(0), r.getResult(1), r.getResult(2),
                     r.getResult(3));
@@ -781,7 +781,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Value added = env.getExpandAdded();
     Value count = env.getExpandCount();
     Value chain = env.getInsertionChain();
-    Value compress = builder.create<CompressOp>(loc, values, filled, added,
+    Value compress = CompressOp::create(builder, loc, values, filled, added,
                                                 count, chain, indices);
     env.updateInsertionChain(compress);
     env.endExpand();
@@ -889,7 +889,7 @@ static void finalizeWhileOp(CodegenEnv &env, OpBuilder &builder,
         env.updateInsertionChain(ifOp->getResult(y++));
       }
       assert(y == yields.size());
-      builder.create<scf::YieldOp>(loc, yields);
+      scf::YieldOp::create(builder, loc, yields);
       builder.setInsertionPointAfter(ifOp);
     }
   }
@@ -942,13 +942,13 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
           assert(lvl.has_value());
           const Value crd = env.emitter().getCoord(tid, *lvl);
           const Value lvar = env.getLoopVar(curr);
-          clause = builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
+          clause = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
                                                  crd, lvar);
         } else {
           assert(lt.hasDenseSemantic() || isUndefLT(lt));
           clause = constantI1(builder, loc, true);
         }
-        cond = cond ? builder.create<arith::AndIOp>(loc, cond, clause) : clause;
+        cond = cond ? arith::AndIOp::create(builder, loc, cond, clause) : clause;
       });
   if (env.isReduc()) {
     types.push_back(env.getReduc().getType());
@@ -959,7 +959,7 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     types.push_back(builder.getIndexType());
   if (env.getInsertionChain())
     types.push_back(env.getInsertionChain().getType());
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, types, cond, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   return ifOp;
 }
@@ -987,7 +987,7 @@ static void endIf(CodegenEnv &env, OpBuilder &builder, scf::IfOp ifOp,
     env.updateInsertionChain(insInput);
   }
   if (!operands.empty())
-    builder.create<scf::YieldOp>(env.op().getLoc(), operands);
+    scf::YieldOp::create(builder, env.op().getLoc(), operands);
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
 }
 
@@ -1301,7 +1301,7 @@ static void genStmt(CodegenEnv &env, RewriterBase &rewriter, ExprId exp,
           genStmt(env, rewriter, ej, curr + 1);
           // TODO: handle yield values.
           assert(reduc.empty() && "Not Implemented");
-          rewriter.create<sparse_tensor::YieldOp>(env.op().getLoc());
+          sparse_tensor::YieldOp::create(rewriter, env.op().getLoc());
           return std::nullopt;
         });
         // endIf(env, rewriter, ifOp, redInput, cntInput, insInput, validIns);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
index 7835c6c3b7797..684a2d418f66c 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
@@ -41,7 +41,7 @@ struct GuardSparseAlloc
     // operation that leaves the underlying storage in a proper state
     // before the tensor escapes across the method boundary.
     rewriter.setInsertionPointAfter(op);
-    auto load = rewriter.create<LoadOp>(op.getLoc(), op.getResult(), true);
+    auto load = LoadOp::create(rewriter, op.getLoc(), op.getResult(), true);
     rewriter.replaceAllUsesExcept(op, load, load);
     return success();
   }
@@ -60,7 +60,7 @@ struct StageUnorderedSparseOps : public OpRewritePattern<StageWithSortOp> {
     // Deallocate tmpBuf.
     // TODO: Delegate to buffer deallocation pass in the future.
     if (succeeded(stageResult) && tmpBuf)
-      rewriter.create<bufferization::DeallocTensorOp>(loc, tmpBuf);
+      bufferization::DeallocTensorOp::create(rewriter, loc, tmpBuf);
 
     return stageResult;
   }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
index 33be62d1d5e7e..27c1b106d71ba 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
@@ -153,7 +153,7 @@ Value sparse_tensor::genCast(OpBuilder &builder, Location loc, Value value,
 
   // int <=> index
   if (isa<IndexType>(srcTp) || isa<IndexType>(dstTp))
-    return builder.create<arith::IndexCastOp>(loc, dstTp, value);
+    return arith::IndexCastOp::create(builder, loc, dstTp, value);
 
   const auto srcIntTp = dyn_cast_or_null<IntegerType>(srcTp);
   const bool isUnsignedCast = srcIntTp ? srcIntTp.isUnsigned() : false;
@@ -166,19 +166,19 @@ Value sparse_tensor::genScalarToTensor(OpBuilder &builder, Location loc,
     // Scalars can only be converted to 0-ranked tensors.
     assert(rtp.getRank() == 0);
     elem = sparse_tensor::genCast(builder, loc, elem, rtp.getElementType());
-    return builder.create<tensor::FromElementsOp>(loc, rtp, elem);
+    return tensor::FromElementsOp::create(builder, loc, rtp, elem);
   }
   return sparse_tensor::genCast(builder, loc, elem, dstTp);
 }
 
 Value sparse_tensor::genIndexLoad(OpBuilder &builder, Location loc, Value mem,
                                   ValueRange s) {
-  Value load = builder.create<memref::LoadOp>(loc, mem, s);
+  Value load = memref::LoadOp::create(builder, loc, mem, s);
   if (!isa<IndexType>(load.getType())) {
     if (load.getType().getIntOrFloatBitWidth() < 64)
-      load = builder.create<arith::ExtUIOp>(loc, builder.getI64Type(), load);
+      load = arith::ExtUIOp::create(builder, loc, builder.getI64Type(), load);
     load =
-        builder.create<arith::IndexCastOp>(loc, builder.getIndexType(), load);
+        arith::IndexCastOp::create(builder, loc, builder.getIndexType(), load);
   }
   return load;
 }
@@ -203,13 +203,13 @@ Value mlir::sparse_tensor::genIsNonzero(OpBuilder &builder, mlir::Location loc,
   Type tp = v.getType();
   Value zero = constantZero(builder, loc, tp);
   if (isa<FloatType>(tp))
-    return builder.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNE, v,
+    return arith::CmpFOp::create(builder, loc, arith::CmpFPredicate::UNE, v,
                                          zero);
   if (tp.isIntOrIndex())
-    return builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, v,
+    return arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, v,
                                          zero);
   if (isa<ComplexType>(tp))
-    return builder.create<complex::NotEqualOp>(loc, v, zero);
+    return complex::NotEqualOp::create(builder, loc, v, zero);
   llvm_unreachable("Non-numeric type");
 }
 
@@ -223,7 +223,7 @@ void mlir::sparse_tensor::genReshapeDstShape(
     for (const auto &map : llvm::enumerate(reassociation)) {
       auto dstDim = constantIndex(builder, loc, 1);
       for (unsigned i = start; i < start + map.value().size(); i++) {
-        dstDim = builder.create<arith::MulIOp>(loc, dstDim, srcShape[i]);
+        dstDim = arith::MulIOp::create(builder, loc, dstDim, srcShape[i]);
       }
       dstShape.push_back(dstDim);
       start = start + map.value().size();
@@ -257,7 +257,7 @@ void mlir::sparse_tensor::genReshapeDstShape(
         // Compute the dynamic dimension size.
         Value productVal = constantIndex(builder, loc, product);
         Value dynamicSize =
-            builder.create<arith::DivUIOp>(loc, srcDim, productVal);
+            arith::DivUIOp::create(builder, loc, srcDim, productVal);
         dstShape.push_back(dynamicSize);
       } else {
         // The expanded dimension is statically known.
@@ -286,7 +286,7 @@ void mlir::sparse_tensor::reshapeCvs(
     // Prepare strides information in dimension slice.
     Value linear = constantIndex(builder, loc, 1);
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
-      linear = builder.create<arith::MulIOp>(loc, linear, sizes[j]);
+      linear = arith::MulIOp::create(builder, loc, linear, sizes[j]);
     }
     // Start expansion.
     Value val;
@@ -294,16 +294,16 @@ void mlir::sparse_tensor::reshapeCvs(
       val = srcCvs[i];
     // Iterate over dimension slice.
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
-      linear = builder.create<arith::DivUIOp>(loc, linear, sizes[j]);
+      linear = arith::DivUIOp::create(builder, loc, linear, sizes[j]);
       if (isCollapse) {
-        const Value mul = builder.create<arith::MulIOp>(loc, srcCvs[j], linear);
-        val = val ? builder.create<arith::AddIOp>(loc, val, mul) : mul;
+        const Value mul = arith::MulIOp::create(builder, loc, srcCvs[j], linear);
+        val = val ? arith::AddIOp::create(builder, loc, val, mul) : mul;
       } else {
         const Value old = val;
-        val = builder.create<arith::DivUIOp>(loc, val, linear);
+        val = arith::DivUIOp::create(builder, loc, val, linear);
         assert(dstCvs.size() == j);
         dstCvs.push_back(val);
-        val = builder.create<arith::RemUIOp>(loc, old, linear);
+        val = arith::RemUIOp::create(builder, loc, old, linear);
       }
     }
     // Finalize collapse.
@@ -326,7 +326,7 @@ FlatSymbolRefAttr mlir::sparse_tensor::getFunc(ModuleOp module, StringRef name,
   auto func = module.lookupSymbol<func::FuncOp>(result.getAttr());
   if (!func) {
     OpBuilder moduleBuilder(module.getBodyRegion());
-    func = moduleBuilder.create<func::FuncOp>(
+    func = func::FuncOp::create(moduleBuilder,
         module.getLoc(), name,
         FunctionType::get(context, operands.getTypes(), resultType));
     func.setPrivate();
@@ -343,7 +343,7 @@ func::CallOp mlir::sparse_tensor::createFuncCall(
   auto module = builder.getBlock()->getParentOp()->getParentOfType<ModuleOp>();
   FlatSymbolRefAttr fn =
       getFunc(module, name, resultType, operands, emitCInterface);
-  return builder.create<func::CallOp>(loc, resultType, fn, operands);
+  return func::CallOp::create(builder, loc, resultType, fn, operands);
 }
 
 Type mlir::sparse_tensor::getOpaquePointerType(MLIRContext *ctx) {
@@ -358,7 +358,7 @@ Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
                                      unsigned sz, Type tp, bool staticShape) {
   if (staticShape) {
     auto memTp = MemRefType::get({sz}, tp);
-    return builder.create<memref::AllocaOp>(loc, memTp);
+    return memref::AllocaOp::create(builder, loc, memTp);
   }
   return genAlloca(builder, loc, constantIndex(builder, loc, sz), tp);
 }
@@ -366,12 +366,12 @@ Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
 Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc, Value sz,
                                      Type tp) {
   auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
-  return builder.create<memref::AllocaOp>(loc, memTp, ValueRange{sz});
+  return memref::AllocaOp::create(builder, loc, memTp, ValueRange{sz});
 }
 
 Value mlir::sparse_tensor::genAllocaScalar(OpBuilder &builder, Location loc,
                                            Type tp) {
-  return builder.create<memref::AllocaOp>(loc, MemRefType::get({}, tp));
+  return memref::AllocaOp::create(builder, loc, MemRefType::get({}, tp));
 }
 
 Value mlir::sparse_tensor::allocaBuffer(OpBuilder &builder, Location loc,
@@ -381,7 +381,7 @@ Value mlir::sparse_tensor::allocaBuffer(OpBuilder &builder, Location loc,
   Value buffer = genAlloca(builder, loc, sz, values[0].getType());
   for (unsigned i = 0; i < sz; i++) {
     Value idx = constantIndex(builder, loc, i);
-    builder.create<memref::StoreOp>(loc, values[i], buffer, idx);
+    memref::StoreOp::create(builder, loc, values[i], buffer, idx);
   }
   return buffer;
 }
@@ -397,15 +397,15 @@ Value mlir::sparse_tensor::allocDenseTensor(OpBuilder &builder, Location loc,
     if (shape[i] == ShapedType::kDynamic)
       dynamicSizes.push_back(sizes[i]);
   }
-  Value mem = builder.create<memref::AllocOp>(loc, memTp, dynamicSizes);
+  Value mem = memref::AllocOp::create(builder, loc, memTp, dynamicSizes);
   Value zero = constantZero(builder, loc, elemTp);
-  builder.create<linalg::FillOp>(loc, ValueRange{zero}, ValueRange{mem});
+  linalg::FillOp::create(builder, loc, ValueRange{zero}, ValueRange{mem});
   return mem;
 }
 
 void mlir::sparse_tensor::deallocDenseTensor(OpBuilder &builder, Location loc,
                                              Value buffer) {
-  builder.create<memref::DeallocOp>(loc, buffer);
+  memref::DeallocOp::create(builder, loc, buffer);
 }
 
 void mlir::sparse_tensor::sizesFromSrc(OpBuilder &builder,
@@ -483,17 +483,17 @@ void sparse_tensor::foreachInSparseConstant(
     cvs.clear();
     for (Dimension d = 0; d < dimRank; d++) {
       auto crd = elems[i].first[d].getInt();
-      cvs.push_back(builder.create<arith::ConstantIndexOp>(loc, crd));
+      cvs.push_back(arith::ConstantIndexOp::create(builder, loc, crd));
     }
     // Remap value.
     Value val;
     if (isa<ComplexType>(attr.getElementType())) {
       auto valAttr = cast<ArrayAttr>(elems[i].second);
-      val = builder.create<complex::ConstantOp>(loc, attr.getElementType(),
+      val = complex::ConstantOp::create(builder, loc, attr.getElementType(),
                                                 valAttr);
     } else {
       auto valAttr = cast<TypedAttr>(elems[i].second);
-      val = builder.create<arith::ConstantOp>(loc, valAttr);
+      val = arith::ConstantOp::create(builder, loc, valAttr);
     }
     assert(val);
     callback(cvs, val);
@@ -513,10 +513,10 @@ SmallVector<Value> sparse_tensor::loadAll(OpBuilder &builder, Location loc,
   SmallVector<Value> vs;
   vs.reserve(size);
   for (unsigned i = 0; i < size; i++) {
-    Value v = builder.create<memref::LoadOp>(loc, mem,
+    Value v = memref::LoadOp::create(builder, loc, mem,
                                              constantIndex(builder, loc, i));
     if (i == offsetIdx && offsetVal)
-      v = builder.create<arith::AddIOp>(loc, v, offsetVal);
+      v = arith::AddIOp::create(builder, loc, v, offsetVal);
     vs.push_back(v);
   }
   return vs;
@@ -535,9 +535,9 @@ void sparse_tensor::storeAll(OpBuilder &builder, Location loc, Value mem,
   for (const auto &v : llvm::enumerate(vs)) {
     const Value w =
         (offsetIdx == v.index() && offsetVal)
-            ? builder.create<arith::AddIOp>(loc, v.value(), offsetVal)
+            ? arith::AddIOp::create(builder, loc, v.value(), offsetVal)
             : v.value();
-    builder.create<memref::StoreOp>(loc, w, mem,
+    memref::StoreOp::create(builder, loc, w, mem,
                                     constantIndex(builder, loc, v.index()));
   }
 }
@@ -547,7 +547,7 @@ sparse_tensor::genToMemref(OpBuilder &builder, Location loc, Value tensor) {
   auto tTp = llvm::cast<TensorType>(tensor.getType());
   auto mTp = MemRefType::get(tTp.getShape(), tTp.getElementType());
   return cast<TypedValue<BaseMemRefType>>(
-      builder.create<bufferization::ToBufferOp>(loc, mTp, tensor).getResult());
+      bufferization::ToBufferOp::create(builder, loc, mTp, tensor).getResult());
 }
 
 Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
@@ -557,7 +557,7 @@ Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
   std::optional<unsigned> offset = enc.getStaticDimSliceOffset(dim);
   if (offset.has_value())
     return constantIndex(builder, loc, *offset);
-  return builder.create<ToSliceOffsetOp>(loc, tensor, APInt(64, dim));
+  return ToSliceOffsetOp::create(builder, loc, tensor, APInt(64, dim));
 }
 
 Value sparse_tensor::createOrFoldSliceStrideOp(OpBuilder &builder, Location loc,
@@ -567,7 +567,7 @@ Value sparse_tensor::createOrFoldSliceStrideOp(OpBuilder &builder, Location loc,
   std::optional<unsigned> stride = enc.getStaticDimSliceStride(dim);
   if (stride.has_value())
     return constantIndex(builder, loc, *stride);
-  return builder.create<ToSliceStrideOp>(loc, tensor, APInt(64, dim));
+  return ToSliceStrideOp::create(builder, loc, tensor, APInt(64, dim));
 }
 
 Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
@@ -609,7 +609,7 @@ Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
     // subsequent clients need the values (DCE will remove unused).
     for (Dimension d = 0; d < dimRank; d++) {
       if (stt.isDynamicDim(d))
-        dimSizesValues[d] = builder.create<memref::LoadOp>(
+        dimSizesValues[d] = memref::LoadOp::create(builder,
             loc, dimSizesBuffer, constantIndex(builder, loc, d));
     }
   }
@@ -686,7 +686,7 @@ Value sparse_tensor::genMapBuffers(
     if (cm == 0) {
       lvlSz = dimSizesValues[d];
       if (cf != 0)
-        lvlSz = builder.create<arith::DivUIOp>(loc, lvlSz,
+        lvlSz = arith::DivUIOp::create(builder, loc, lvlSz,
                                                constantIndex(builder, loc, cf));
     } else {
       lvlSz = constantIndex(builder, loc, cm);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
index dc017e6baa6dc..400be54aeb86f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
@@ -97,7 +97,7 @@ class FuncCallOrInlineGenerator {
       // Create the function if not already exist.
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(getParentOpOf<func::FuncOp>(builder));
-      func = builder.create<func::FuncOp>(
+      func = func::FuncOp::create(builder,
           loc, funcName,
           FunctionType::get(context, params.getTypes(), retTypes));
       func.setPrivate();
@@ -108,10 +108,10 @@ class FuncCallOrInlineGenerator {
       // Delegates to user to generate the actually implementation.
       SmallVector<Value> result =
           genImplementation(retTypes, args, builder, loc);
-      builder.create<func::ReturnOp>(loc, result);
+      func::ReturnOp::create(builder, loc, result);
     }
     // Returns the CallOp result.
-    func::CallOp call = builder.create<func::CallOp>(loc, func, params);
+    func::CallOp call = func::CallOp::create(builder, loc, func, params);
     return call.getResults();
   }
 
@@ -310,9 +310,9 @@ inline Value constantZero(OpBuilder &builder, Location loc, Type tp) {
   if (auto ctp = dyn_cast<ComplexType>(tp)) {
     auto zeroe = builder.getZeroAttr(ctp.getElementType());
     auto zeroa = builder.getArrayAttr({zeroe, zeroe});
-    return builder.create<complex::ConstantOp>(loc, tp, zeroa);
+    return complex::ConstantOp::create(builder, loc, tp, zeroa);
   }
-  return builder.create<arith::ConstantOp>(loc, tp, builder.getZeroAttr(tp));
+  return arith::ConstantOp::create(builder, loc, tp, builder.getZeroAttr(tp));
 }
 
 /// Generates a 1-valued constant of the given type.  This supports all
@@ -322,39 +322,39 @@ inline Value constantOne(OpBuilder &builder, Location loc, Type tp) {
     auto zeroe = builder.getZeroAttr(ctp.getElementType());
     auto onee = getOneAttr(builder, ctp.getElementType());
     auto zeroa = builder.getArrayAttr({onee, zeroe});
-    return builder.create<complex::ConstantOp>(loc, tp, zeroa);
+    return complex::ConstantOp::create(builder, loc, tp, zeroa);
   }
-  return builder.create<arith::ConstantOp>(loc, tp, getOneAttr(builder, tp));
+  return arith::ConstantOp::create(builder, loc, tp, getOneAttr(builder, tp));
 }
 
 /// Generates a constant of `index` type.
 inline Value constantIndex(OpBuilder &builder, Location loc, int64_t i) {
-  return builder.create<arith::ConstantIndexOp>(loc, i);
+  return arith::ConstantIndexOp::create(builder, loc, i);
 }
 
 /// Generates a constant of `i64` type.
 inline Value constantI64(OpBuilder &builder, Location loc, int64_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 64);
+  return arith::ConstantIntOp::create(builder, loc, i, 64);
 }
 
 /// Generates a constant of `i32` type.
 inline Value constantI32(OpBuilder &builder, Location loc, int32_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 32);
+  return arith::ConstantIntOp::create(builder, loc, i, 32);
 }
 
 /// Generates a constant of `i16` type.
 inline Value constantI16(OpBuilder &builder, Location loc, int16_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 16);
+  return arith::ConstantIntOp::create(builder, loc, i, 16);
 }
 
 /// Generates a constant of `i8` type.
 inline Value constantI8(OpBuilder &builder, Location loc, int8_t i) {
-  return builder.create<arith::ConstantIntOp>(loc, i, 8);
+  return arith::ConstantIntOp::create(builder, loc, i, 8);
 }
 
 /// Generates a constant of `i1` type.
 inline Value constantI1(OpBuilder &builder, Location loc, bool b) {
-  return builder.create<arith::ConstantIntOp>(loc, b, 1);
+  return arith::ConstantIntOp::create(builder, loc, b, 1);
 }
 
 /// Generates a constant of the given `Action`.
@@ -400,12 +400,12 @@ inline Value constantLevelTypeEncoding(OpBuilder &builder, Location loc,
 inline Value genValFromAttr(OpBuilder &builder, Location loc, Attribute attr) {
   if (auto complexAttr = dyn_cast<complex::NumberAttr>(attr)) {
     Type tp = cast<ComplexType>(complexAttr.getType()).getElementType();
-    return builder.create<complex::ConstantOp>(
+    return complex::ConstantOp::create(builder,
         loc, complexAttr.getType(),
         builder.getArrayAttr({FloatAttr::get(tp, complexAttr.getReal()),
                               FloatAttr::get(tp, complexAttr.getImag())}));
   }
-  return builder.create<arith::ConstantOp>(loc, cast<TypedAttr>(attr));
+  return arith::ConstantOp::create(builder, loc, cast<TypedAttr>(attr));
 }
 
 // TODO: is this at the right place?
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
index a77e3036ac519..6eaeff91dd5c6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
@@ -25,18 +25,18 @@ using namespace mlir::sparse_tensor;
 //===----------------------------------------------------------------------===//
 
 #define CMPI(p, l, r)                                                          \
-  (builder.create<arith::CmpIOp>(loc, arith::CmpIPredicate::p, (l), (r))       \
+  (arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::p, (l), (r))       \
        .getResult())
 
 #define C_IDX(v) (constantIndex(builder, loc, (v)))
-#define YIELD(vs) (builder.create<scf::YieldOp>(loc, (vs)))
-#define ADDI(lhs, rhs) (builder.create<arith::AddIOp>(loc, (lhs), (rhs)))
-#define ANDI(lhs, rhs) (builder.create<arith::AndIOp>(loc, (lhs), (rhs)))
-#define SUBI(lhs, rhs) (builder.create<arith::SubIOp>(loc, (lhs), (rhs)))
-#define MULI(lhs, rhs) (builder.create<arith::MulIOp>(loc, (lhs), (rhs)))
-#define REMUI(lhs, rhs) (builder.create<arith::RemUIOp>(loc, (lhs), (rhs)))
-#define DIVUI(lhs, rhs) (builder.create<arith::DivUIOp>(loc, (lhs), (rhs)))
-#define SELECT(c, l, r) (builder.create<arith::SelectOp>(loc, (c), (l), (r)))
+#define YIELD(vs) (scf::YieldOp::create(builder, loc, (vs)))
+#define ADDI(lhs, rhs) (arith::AddIOp::create(builder, loc, (lhs), (rhs)))
+#define ANDI(lhs, rhs) (arith::AndIOp::create(builder, loc, (lhs), (rhs)))
+#define SUBI(lhs, rhs) (arith::SubIOp::create(builder, loc, (lhs), (rhs)))
+#define MULI(lhs, rhs) (arith::MulIOp::create(builder, loc, (lhs), (rhs)))
+#define REMUI(lhs, rhs) (arith::RemUIOp::create(builder, loc, (lhs), (rhs)))
+#define DIVUI(lhs, rhs) (arith::DivUIOp::create(builder, loc, (lhs), (rhs)))
+#define SELECT(c, l, r) (arith::SelectOp::create(builder, loc, (c), (l), (r)))
 
 //===----------------------------------------------------------------------===//
 // Debugging utils
@@ -45,7 +45,7 @@ using namespace mlir::sparse_tensor;
 #ifndef NDEBUG
 LLVM_ATTRIBUTE_UNUSED static void dumpIndexMemRef(OpBuilder &builder,
                                                   Location loc, Value memref) {
-  memref = builder.create<memref::CastOp>(
+  memref = memref::CastOp::create(builder,
       loc, UnrankedMemRefType::get(builder.getIndexType(), 0), memref);
   createFuncCall(builder, loc, "printMemrefInd", TypeRange{},
                  ValueRange{memref}, EmitCInterface::On);
@@ -261,7 +261,7 @@ void LoopEmitter::initializeLoopEmit(
         denseTp = bufferization::getMemRefTypeWithFullyDynamicLayout(rtp);
 
       Value denseVal =
-          builder.create<bufferization::ToBufferOp>(loc, denseTp, tensor);
+          bufferization::ToBufferOp::create(builder, loc, denseTp, tensor);
       // Dense outputs need special handling.
       if (isOutput && updater)
         denseVal = updater(builder, loc, denseVal, tensor);
@@ -271,7 +271,7 @@ void LoopEmitter::initializeLoopEmit(
       // Annotated sparse tensors.
       // We also need the value buffer for all-dense annotated "sparse"
       // tensors.
-      valBuffer[t] = builder.create<ToValuesOp>(loc, tensor);
+      valBuffer[t] = ToValuesOp::create(builder, loc, tensor);
     }
   }
 
@@ -479,7 +479,7 @@ std::pair<Operation *, Value> LoopEmitter::emitForLoopOverTensorAtLvl(
   Value iv;
   if (isParallel) {
     scf::ParallelOp parOp =
-        builder.create<scf::ParallelOp>(loc, lo, hi, step, reduc);
+        scf::ParallelOp::create(builder, loc, lo, hi, step, reduc);
     builder.setInsertionPointToStart(parOp.getBody());
     assert(parOp.getNumReductions() == reduc.size());
     iv = parOp.getInductionVars()[0];
@@ -495,7 +495,7 @@ std::pair<Operation *, Value> LoopEmitter::emitForLoopOverTensorAtLvl(
       reduc[i] = parOp.getInitVals()[i];
     loop = parOp;
   } else {
-    scf::ForOp forOp = builder.create<scf::ForOp>(loc, lo, hi, step, reduc);
+    scf::ForOp forOp = scf::ForOp::create(builder, loc, lo, hi, step, reduc);
     builder.setInsertionPointToStart(forOp.getBody());
     iv = forOp.getInductionVar();
 
@@ -603,11 +603,11 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
 
       // Extract and iterate over the iteration space.
       ExtractIterSpaceOp extractSpaceOp =
-          lvl == 0 ? builder.create<ExtractIterSpaceOp>(loc, t)
-                   : builder.create<ExtractIterSpaceOp>(
+          lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
+                   : ExtractIterSpaceOp::create(builder,
                          loc, t, spIterVals[tid][lvl - 1], lvl);
 
-      IterateOp iterOp = builder.create<IterateOp>(
+      IterateOp iterOp = IterateOp::create(builder,
           loc, extractSpaceOp.getExtractedSpace(), reduc);
       spIterVals[tid][lvl] = iterOp.getIterator();
 
@@ -625,12 +625,12 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
     for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
       Value t = tensors[tid];
       ExtractIterSpaceOp extractSpaceOp =
-          lvl == 0 ? builder.create<ExtractIterSpaceOp>(loc, t)
-                   : builder.create<ExtractIterSpaceOp>(
+          lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
+                   : ExtractIterSpaceOp::create(builder,
                          loc, t, spIterVals[tid][lvl - 1], lvl);
       spaces.push_back(extractSpaceOp.getExtractedSpace());
     }
-    auto coIterOp = builder.create<CoIterateOp>(loc, spaces, reduc, numCases);
+    auto coIterOp = CoIterateOp::create(builder, loc, spaces, reduc, numCases);
     // The CoIterationOp does not have insertion block nor induction variable.
     // TODO: the `struct LoopInfo` should be simplied after full migration.
     loopStack.emplace_back(tidLvls, coIterOp, /*insertion block*/ nullptr,
@@ -728,7 +728,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   if (emitStrategy == SparseEmitStrategy::kSparseIterator) {
     auto iterateOp = llvm::cast<IterateOp>(loopInfo.loop);
     assert(reduc.size() == iterateOp.getNumResults());
-    rewriter.create<sparse_tensor::YieldOp>(loc, reduc);
+    sparse_tensor::YieldOp::create(rewriter, loc, reduc);
     // Exit the loop.
     rewriter.setInsertionPointAfter(iterateOp);
     // In-place update reduction variables.
@@ -738,7 +738,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   if (auto forOp = llvm::dyn_cast<scf::ForOp>(loopInfo.loop)) {
     if (!reduc.empty()) {
       assert(reduc.size() == forOp.getNumResults());
-      rewriter.create<scf::YieldOp>(loc, reduc);
+      scf::YieldOp::create(rewriter, loc, reduc);
     }
     // Exit the loop.
     rewriter.setInsertionPointAfter(forOp);
@@ -777,7 +777,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
 #endif // NDEBUG
 
       rewriter.setInsertionPointAfter(redExp);
-      auto redOp = rewriter.create<scf::ReduceOp>(loc, curVal);
+      auto redOp = scf::ReduceOp::create(rewriter, loc, curVal);
       // Attach to the reduction op.
       Block *redBlock = &redOp.getReductions().front().front();
       rewriter.setInsertionPointToEnd(redBlock);
@@ -789,7 +789,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
       // Erases the out-dated reduction expression.
       rewriter.eraseOp(redExp);
       rewriter.setInsertionPointToEnd(redBlock);
-      rewriter.create<scf::ReduceReturnOp>(loc, newRed->getResult(0));
+      scf::ReduceReturnOp::create(rewriter, loc, newRed->getResult(0));
     }
     rewriter.setInsertionPointAfter(parOp);
     // In-place update reduction variables.
@@ -863,7 +863,7 @@ void LoopEmitter::exitCurrentLoop(RewriterBase &rewriter, Location loc,
   if (emitStrategy == SparseEmitStrategy::kSparseIterator) {
     Operation *p = loopInfo.loop;
     if (isa<IterateOp>(p))
-      rewriter.create<sparse_tensor::YieldOp>(loc, reduc);
+      sparse_tensor::YieldOp::create(rewriter, loc, reduc);
 
     // Exit the loop.
     rewriter.setInsertionPointAfter(p);
@@ -929,7 +929,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
   // Ensures all operands are valid.
   assert(!llvm::is_contained(ivs, nullptr));
   TypeRange types = ValueRange(ivs).getTypes();
-  auto whileOp = builder.create<scf::WhileOp>(loc, types, ivs);
+  auto whileOp = scf::WhileOp::create(builder, loc, types, ivs);
 
   SmallVector<Location> locs(types.size(), loc);
   Block *before = builder.createBlock(&whileOp.getBefore(), {}, types, locs);
@@ -948,7 +948,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
   // The remaining block arguments are user-provided reduction values and an
   // optional universal index. Make sure their sizes match.
   assert(bArgs.size() == reduc.size() + (uniIdx ? 1 : 0));
-  builder.create<scf::ConditionOp>(loc, whileCond, before->getArguments());
+  scf::ConditionOp::create(builder, loc, whileCond, before->getArguments());
 
   // Generates loop body.
   builder.setInsertionPointToStart(after);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
index 1c8a4789e2065..47fa3f950afd3 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
@@ -78,14 +78,14 @@ SparseTensorTypeToBufferConverter::SparseTensorTypeToBufferConverter() {
 
 Value SparseTensorSpecifier::getInitValue(OpBuilder &builder, Location loc,
                                           SparseTensorType stt) {
-  return builder.create<StorageSpecifierInitOp>(
+  return StorageSpecifierInitOp::create(builder,
       loc, StorageSpecifierType::get(stt.getEncoding()));
 }
 
 Value SparseTensorSpecifier::getSpecifierField(OpBuilder &builder, Location loc,
                                                StorageSpecifierKind kind,
                                                std::optional<Level> lvl) {
-  return builder.create<GetStorageSpecifierOp>(
+  return GetStorageSpecifierOp::create(builder,
       loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl));
 }
 
@@ -95,7 +95,7 @@ void SparseTensorSpecifier::setSpecifierField(OpBuilder &builder, Location loc,
                                               std::optional<Level> lvl) {
   // TODO: make `v` have type `TypedValue<IndexType>` instead.
   assert(v.getType().isIndex());
-  specifier = builder.create<SetStorageSpecifierOp>(
+  specifier = SetStorageSpecifierOp::create(builder,
       loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl), v);
 }
 
@@ -111,8 +111,8 @@ Value sparse_tensor::SparseTensorDescriptor::getCrdMemRefOrView(
 
   Value stride = constantIndex(builder, loc, rType.getLvlRank() - cooStart);
   Value size = getCrdMemSize(builder, loc, cooStart);
-  size = builder.create<arith::DivUIOp>(loc, size, stride);
-  return builder.create<memref::SubViewOp>(
+  size = arith::DivUIOp::create(builder, loc, size, stride);
+  return memref::SubViewOp::create(builder,
       loc, getMemRefField(SparseTensorFieldKind::CrdMemRef, cooStart),
       /*offset=*/ValueRange{constantIndex(builder, loc, lvl - cooStart)},
       /*size=*/ValueRange{size},
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
index 869c7864d7535..45d142a807c36 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
@@ -231,7 +231,7 @@ class MutSparseTensorDescriptor
 /// Packs the given values as a "tuple" value.
 inline Value genTuple(OpBuilder &builder, Location loc, Type tp,
                       ValueRange values) {
-  return builder.create<UnrealizedConversionCastOp>(loc, TypeRange(tp), values)
+  return UnrealizedConversionCastOp::create(builder, loc, TypeRange(tp), values)
       .getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
index aad5e97ed14ab..212ee99647167 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
@@ -22,23 +22,23 @@ using ValueTuple = std::tuple<Value, Value, Value>;
 // File local helper functions/macros.
 //===----------------------------------------------------------------------===//
 #define CMPI(p, lhs, rhs)                                                      \
-  (b.create<arith::CmpIOp>(l, arith::CmpIPredicate::p, (lhs), (rhs))           \
+  (arith::CmpIOp::create(b, l, arith::CmpIPredicate::p, (lhs), (rhs))           \
        .getResult())
 
 #define C_FALSE (constantI1(b, l, false))
 #define C_TRUE (constantI1(b, l, true))
 #define C_IDX(v) (constantIndex(b, l, (v)))
-#define YIELD(vs) (b.create<scf::YieldOp>(l, (vs)))
-#define ADDI(lhs, rhs) (b.create<arith::AddIOp>(l, (lhs), (rhs)).getResult())
-#define ORI(lhs, rhs) (b.create<arith::OrIOp>(l, (lhs), (rhs)).getResult())
-#define ANDI(lhs, rhs) (b.create<arith::AndIOp>(l, (lhs), (rhs)).getResult())
-#define SUBI(lhs, rhs) (b.create<arith::SubIOp>(l, (lhs), (rhs)).getResult())
-#define MULI(lhs, rhs) (b.create<arith::MulIOp>(l, (lhs), (rhs)).getResult())
-#define MINUI(lhs, rhs) (b.create<arith::MinUIOp>(l, (lhs), (rhs)).getResult())
-#define REMUI(lhs, rhs) (b.create<arith::RemUIOp>(l, (lhs), (rhs)).getResult())
-#define DIVUI(lhs, rhs) (b.create<arith::DivUIOp>(l, (lhs), (rhs)).getResult())
+#define YIELD(vs) (scf::YieldOp::create(b, l, (vs)))
+#define ADDI(lhs, rhs) (arith::AddIOp::create(b, l, (lhs), (rhs)).getResult())
+#define ORI(lhs, rhs) (arith::OrIOp::create(b, l, (lhs), (rhs)).getResult())
+#define ANDI(lhs, rhs) (arith::AndIOp::create(b, l, (lhs), (rhs)).getResult())
+#define SUBI(lhs, rhs) (arith::SubIOp::create(b, l, (lhs), (rhs)).getResult())
+#define MULI(lhs, rhs) (arith::MulIOp::create(b, l, (lhs), (rhs)).getResult())
+#define MINUI(lhs, rhs) (arith::MinUIOp::create(b, l, (lhs), (rhs)).getResult())
+#define REMUI(lhs, rhs) (arith::RemUIOp::create(b, l, (lhs), (rhs)).getResult())
+#define DIVUI(lhs, rhs) (arith::DivUIOp::create(b, l, (lhs), (rhs)).getResult())
 #define SELECT(c, lhs, rhs)                                                    \
-  (b.create<arith::SelectOp>(l, (c), (lhs), (rhs)).getResult())
+  (arith::SelectOp::create(b, l, (c), (lhs), (rhs)).getResult())
 
 //===----------------------------------------------------------------------===//
 // SparseTensorLevel derived classes.
@@ -150,19 +150,19 @@ class CompressedLevel : public SparseLevel</*hasPosBuf=*/true> {
       return loadRange();
 
     SmallVector<Type, 2> types{b.getIndexType(), b.getIndexType()};
-    scf::IfOp posRangeIf = b.create<scf::IfOp>(l, types, inPadZone, true);
+    scf::IfOp posRangeIf = scf::IfOp::create(b, l, types, inPadZone, true);
     // True branch, returns a "fake" empty range [0, 0) if parent
     // iterator is in pad zone.
     b.setInsertionPointToStart(posRangeIf.thenBlock());
 
     SmallVector<Value, 2> emptyRange{C_IDX(0), C_IDX(0)};
-    b.create<scf::YieldOp>(l, emptyRange);
+    scf::YieldOp::create(b, l, emptyRange);
 
     // False branch, returns the actual range.
     b.setInsertionPointToStart(posRangeIf.elseBlock());
     auto [pLo, pHi] = loadRange();
     SmallVector<Value, 2> loadedRange{pLo, pHi};
-    b.create<scf::YieldOp>(l, loadedRange);
+    scf::YieldOp::create(b, l, loadedRange);
 
     b.setInsertionPointAfter(posRangeIf);
     ValueRange posRange = posRangeIf.getResults();
@@ -248,7 +248,7 @@ static scf::ValueVector genWhenInBound(
     llvm::function_ref<scf::ValueVector(OpBuilder &, Location, Value)>
         builder) {
   TypeRange ifRetTypes = elseRet.getTypes();
-  auto ifOp = b.create<scf::IfOp>(l, ifRetTypes, it.genNotEnd(b, l), true);
+  auto ifOp = scf::IfOp::create(b, l, ifRetTypes, it.genNotEnd(b, l), true);
 
   b.setInsertionPointToStart(ifOp.thenBlock());
   Value crd = it.deref(b, l);
@@ -732,7 +732,7 @@ class NonEmptySubSectIterator : public SparseIterator {
   //      [itVal0, itVal1, ..., pNx0],
   //      ...]
   Value allocSubSectPosBuf(OpBuilder &b, Location l) {
-    return b.create<memref::AllocaOp>(
+    return memref::AllocaOp::create(b,
         l,
         MemRefType::get({ShapedType::kDynamic, tupleSz + 1}, b.getIndexType()),
         maxTupleCnt);
@@ -740,12 +740,12 @@ class NonEmptySubSectIterator : public SparseIterator {
 
   void storeNxLvlStart(OpBuilder &b, Location l, Value tupleId,
                        Value start) const {
-    b.create<memref::StoreOp>(l, start, subSectPosBuf,
+    memref::StoreOp::create(b, l, start, subSectPosBuf,
                               ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
   Value loadNxLvlStart(OpBuilder &b, Location l, Value tupleId) const {
-    return b.create<memref::LoadOp>(l, subSectPosBuf,
+    return memref::LoadOp::create(b, l, subSectPosBuf,
                                     ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
@@ -753,7 +753,7 @@ class NonEmptySubSectIterator : public SparseIterator {
                        ValueRange itVals) const {
     assert(itVals.size() == tupleSz);
     for (unsigned i = 0; i < tupleSz; i++) {
-      b.create<memref::StoreOp>(l, itVals[i], subSectPosBuf,
+      memref::StoreOp::create(b, l, itVals[i], subSectPosBuf,
                                 ValueRange{tupleId, C_IDX(i)});
     }
   }
@@ -762,7 +762,7 @@ class NonEmptySubSectIterator : public SparseIterator {
                                     Value tupleId) const {
     SmallVector<Value> ret;
     for (unsigned i = 0; i < tupleSz; i++) {
-      Value v = b.create<memref::LoadOp>(l, subSectPosBuf,
+      Value v = memref::LoadOp::create(b, l, subSectPosBuf,
                                          ValueRange{tupleId, C_IDX(i)});
       ret.push_back(v);
     }
@@ -1043,7 +1043,7 @@ ValueRange SparseIterator::forward(OpBuilder &b, Location l) {
 }
 
 ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
-  auto ifOp = b.create<scf::IfOp>(l, getCursor().getTypes(), cond, true);
+  auto ifOp = scf::IfOp::create(b, l, getCursor().getTypes(), cond, true);
   // Generate else branch first, otherwise iterator values will be updated by
   // `forward()`.
   b.setInsertionPointToStart(ifOp.elseBlock());
@@ -1058,12 +1058,12 @@ ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
 }
 
 Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
-  auto whileOp = b.create<scf::WhileOp>(
+  auto whileOp = scf::WhileOp::create(b,
       l, pos.getType(), pos,
       /*beforeBuilder=*/
       [this, pos](OpBuilder &b, Location l, ValueRange ivs) {
         Value inBound = CMPI(ult, ivs.front(), posHi);
-        auto ifInBound = b.create<scf::IfOp>(l, b.getI1Type(), inBound, true);
+        auto ifInBound = scf::IfOp::create(b, l, b.getI1Type(), inBound, true);
         {
           OpBuilder::InsertionGuard guard(b);
           // If in bound, load the next coordinates and check duplication.
@@ -1076,7 +1076,7 @@ Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
           b.setInsertionPointToStart(ifInBound.elseBlock());
           YIELD(constantI1(b, l, false));
         }
-        b.create<scf::ConditionOp>(l, ifInBound.getResults()[0], ivs);
+        scf::ConditionOp::create(b, l, ifInBound.getResults()[0], ivs);
       },
       /*afterBuilder=*/
       [](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1137,7 +1137,7 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
 
   SmallVector<Value> whileArgs(getCursor().begin(), getCursor().end());
   whileArgs.push_back(isFirst);
-  auto whileOp = b.create<scf::WhileOp>(
+  auto whileOp = scf::WhileOp::create(b,
       l, ValueRange(whileArgs).getTypes(), whileArgs,
       /*beforeBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1154,7 +1154,7 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
                              ret = ORI(ret, llvm::getSingleElement(isFirst));
                              return {ret};
                            });
-        b.create<scf::ConditionOp>(l, cont.front(), ivs);
+        scf::ConditionOp::create(b, l, cont.front(), ivs);
       },
       /*afterBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1219,7 +1219,7 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
     SmallVector<Value> iterArgs;
     iterArgs.push_back(C_IDX(0));
     iterArgs.append(reduc.begin(), reduc.end());
-    auto forEachLeaf = b.create<scf::ForOp>(
+    auto forEachLeaf = scf::ForOp::create(b,
         l, /*lb=*/C_IDX(0), /*ub=*/tupleCnt, /*step=*/C_IDX(1), iterArgs,
         [&helper, &builder](OpBuilder &b, Location l, Value tupleId,
                             ValueRange iterArgs) {
@@ -1235,12 +1235,12 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
           SmallVector<Value> whileArgs(helper.wrap.getCursor());
           whileArgs.append(iterArgs.begin(), iterArgs.end());
 
-          auto whileOp = b.create<scf::WhileOp>(
+          auto whileOp = scf::WhileOp::create(b,
               l, ValueRange(whileArgs).getTypes(), whileArgs,
               /*beforeBuilder=*/
               [&helper](OpBuilder &b, Location l, ValueRange ivs) {
                 helper.wrap.linkNewScope(ivs);
-                b.create<scf::ConditionOp>(l, helper.genNotEnd(b, l), ivs);
+                scf::ConditionOp::create(b, l, helper.genNotEnd(b, l), ivs);
               },
               /*afterBuilder=*/
               [&helper, &builder](OpBuilder &b, Location l, ValueRange ivs) {
@@ -1267,7 +1267,7 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
                                      ValueRange reduc) {
     assert(!parent || parent->lvl + 1 == lvl);
     delegate->genInit(b, l, parent);
-    auto forOp = b.create<scf::ForOp>(
+    auto forOp = scf::ForOp::create(b,
         l, /*lb=*/C_IDX(0), /*ub=*/subSectSz, /*step=*/C_IDX(1), reduc,
         [&](OpBuilder &b, Location l, Value crd, ValueRange iterArgs) {
           helper.locate(b, l, crd);
@@ -1411,7 +1411,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
   // if (offset + size > parents.size)
   //   isNonEmpty = false;
   Value fastPathP = CMPI(ugt, getMinCrd(), getAbsOff());
-  auto ifOp = b.create<scf::IfOp>(l, getCursor().getTypes(), fastPathP, true);
+  auto ifOp = scf::IfOp::create(b, l, getCursor().getTypes(), fastPathP, true);
   {
     OpBuilder::InsertionGuard guard(b);
     // Take the fast path
@@ -1448,7 +1448,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 Value isMin = CMPI(eq, crd, getMinCrd());
                 delegate->forwardIf(b, l, isMin);
                 // Update the forwarded iterator values if needed.
-                auto ifIsMin = b.create<scf::IfOp>(l, isMin, false);
+                auto ifIsMin = scf::IfOp::create(b, l, isMin, false);
                 b.setInsertionPointToStart(&ifIsMin.getThenRegion().front());
                 storeCursorVals(b, l, tupleId, delegate->serialize());
                 b.setInsertionPointAfter(ifIsMin);
@@ -1458,7 +1458,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 return genWhenInBound(b, l, *delegate, /*elseRet=*/iterArgs,
                                       [nxMin](OpBuilder &b, Location l,
                                               Value crd) -> scf::ValueVector {
-                                        Value nx = b.create<arith::MinUIOp>(
+                                        Value nx = arith::MinUIOp::create(b,
                                             l, crd, nxMin);
                                         return {nx, C_TRUE};
                                       });
@@ -1480,7 +1480,7 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
 
   // We should at least forward the offset by one.
   Value minAbsOff = ADDI(getAbsOff(), c1);
-  nxAbsOff = b.create<arith::MaxUIOp>(l, minAbsOff, nxAbsOff);
+  nxAbsOff = arith::MaxUIOp::create(b, l, minAbsOff, nxAbsOff);
 
   seek(ValueRange{nxMinCrd, nxAbsOff, nxNotEnd});
   // The coordinate should not exceeds the space upper bound.
@@ -1581,16 +1581,16 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &b, Location l, Value t,
   auto stt = getSparseTensorType(t);
 
   LevelType lt = stt.getLvlType(lvl);
-  Value sz = stt.hasEncoding() ? b.create<LvlOp>(l, t, lvl).getResult()
-                               : b.create<tensor::DimOp>(l, t, lvl).getResult();
+  Value sz = stt.hasEncoding() ? LvlOp::create(b, l, t, lvl).getResult()
+                               : tensor::DimOp::create(b, l, t, lvl).getResult();
 
   SmallVector<Value, 2> buffers;
   if (lt.isWithPosLT()) {
-    Value pos = b.create<ToPositionsOp>(l, t, lvl);
+    Value pos = ToPositionsOp::create(b, l, t, lvl);
     buffers.push_back(pos);
   }
   if (lt.isWithCrdLT()) {
-    Value pos = b.create<ToCoordinatesOp>(l, t, lvl);
+    Value pos = ToCoordinatesOp::create(b, l, t, lvl);
     buffers.push_back(pos);
   }
   return makeSparseTensorLevel(lt, sz, buffers, tid, lvl);
diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
index 0258f797143cb..fb68620c378e5 100644
--- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
@@ -1563,7 +1563,7 @@ static Value insertYieldOp(RewriterBase &rewriter, Location loc, Region &region,
   Block &clonedBlock = tmpRegion.front();
   YieldOp clonedYield = cast<YieldOp>(clonedBlock.getTerminator());
   // Merge cloned block and return yield value.
-  Operation *placeholder = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+  Operation *placeholder = arith::ConstantIndexOp::create(rewriter, loc, 0);
   rewriter.inlineBlockBefore(&tmpRegion.front(), placeholder, vals);
   Value val = clonedYield.getSingleResult();
   rewriter.eraseOp(clonedYield);
@@ -1603,16 +1603,16 @@ static Value buildRelu(RewriterBase &rewriter, Location loc, Value v0,
                        Attribute attr) {
   Type tp = v0.getType();
   auto zero =
-      rewriter.create<arith::ConstantOp>(loc, tp, rewriter.getZeroAttr(tp));
+      arith::ConstantOp::create(rewriter, loc, tp, rewriter.getZeroAttr(tp));
   Value cmp;
   if (isa<FloatType>(tp)) {
     auto pred = llvm::cast<arith::CmpFPredicateAttr>(attr);
-    cmp = rewriter.create<arith::CmpFOp>(loc, pred, v0, zero);
+    cmp = arith::CmpFOp::create(rewriter, loc, pred, v0, zero);
   } else {
     auto pred = llvm::cast<arith::CmpIPredicateAttr>(attr);
-    cmp = rewriter.create<arith::CmpIOp>(loc, pred, v0, zero);
+    cmp = arith::CmpIOp::create(rewriter, loc, pred, v0, zero);
   }
-  return rewriter.create<arith::SelectOp>(loc, cmp, v0, zero);
+  return arith::SelectOp::create(rewriter, loc, cmp, v0, zero);
 }
 
 Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
@@ -1627,128 +1627,128 @@ Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
     llvm_unreachable("unexpected non-op");
   // Unary operations.
   case TensorExp::Kind::kAbsF:
-    return rewriter.create<math::AbsFOp>(loc, v0);
+    return math::AbsFOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kAbsC: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::AbsOp>(loc, eltType, v0);
+    return complex::AbsOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kAbsI:
-    return rewriter.create<math::AbsIOp>(loc, v0);
+    return math::AbsIOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kCeilF:
-    return rewriter.create<math::CeilOp>(loc, v0);
+    return math::CeilOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kFloorF:
-    return rewriter.create<math::FloorOp>(loc, v0);
+    return math::FloorOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSqrtF:
-    return rewriter.create<math::SqrtOp>(loc, v0);
+    return math::SqrtOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSqrtC:
-    return rewriter.create<complex::SqrtOp>(loc, v0);
+    return complex::SqrtOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kExpm1F:
-    return rewriter.create<math::ExpM1Op>(loc, v0);
+    return math::ExpM1Op::create(rewriter, loc, v0);
   case TensorExp::Kind::kExpm1C:
-    return rewriter.create<complex::Expm1Op>(loc, v0);
+    return complex::Expm1Op::create(rewriter, loc, v0);
   case TensorExp::Kind::kLog1pF:
-    return rewriter.create<math::Log1pOp>(loc, v0);
+    return math::Log1pOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kLog1pC:
-    return rewriter.create<complex::Log1pOp>(loc, v0);
+    return complex::Log1pOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kRelu:
     return buildRelu(rewriter, loc, v0, expr.attr);
   case TensorExp::Kind::kSinF:
-    return rewriter.create<math::SinOp>(loc, v0);
+    return math::SinOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kSinC:
-    return rewriter.create<complex::SinOp>(loc, v0);
+    return complex::SinOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kTanhF:
-    return rewriter.create<math::TanhOp>(loc, v0);
+    return math::TanhOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kTanhC:
-    return rewriter.create<complex::TanhOp>(loc, v0);
+    return complex::TanhOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegF:
-    return rewriter.create<arith::NegFOp>(loc, v0);
+    return arith::NegFOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegC:
-    return rewriter.create<complex::NegOp>(loc, v0);
+    return complex::NegOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegI: // no negi in std
-    return rewriter.create<arith::SubIOp>(
+    return arith::SubIOp::create(rewriter,
         loc,
-        rewriter.create<arith::ConstantOp>(loc, v0.getType(),
+        arith::ConstantOp::create(rewriter, loc, v0.getType(),
                                            rewriter.getZeroAttr(v0.getType())),
         v0);
   case TensorExp::Kind::kTruncF:
-    return rewriter.create<arith::TruncFOp>(loc, inferType(e, v0), v0);
+    return arith::TruncFOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kExtF:
-    return rewriter.create<arith::ExtFOp>(loc, inferType(e, v0), v0);
+    return arith::ExtFOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastFS:
-    return rewriter.create<arith::FPToSIOp>(loc, inferType(e, v0), v0);
+    return arith::FPToSIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastFU:
-    return rewriter.create<arith::FPToUIOp>(loc, inferType(e, v0), v0);
+    return arith::FPToUIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastSF:
-    return rewriter.create<arith::SIToFPOp>(loc, inferType(e, v0), v0);
+    return arith::SIToFPOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastUF:
-    return rewriter.create<arith::UIToFPOp>(loc, inferType(e, v0), v0);
+    return arith::UIToFPOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastS:
-    return rewriter.create<arith::ExtSIOp>(loc, inferType(e, v0), v0);
+    return arith::ExtSIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastU:
-    return rewriter.create<arith::ExtUIOp>(loc, inferType(e, v0), v0);
+    return arith::ExtUIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCastIdx:
-    return rewriter.create<arith::IndexCastOp>(loc, inferType(e, v0), v0);
+    return arith::IndexCastOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kTruncI:
-    return rewriter.create<arith::TruncIOp>(loc, inferType(e, v0), v0);
+    return arith::TruncIOp::create(rewriter, loc, inferType(e, v0), v0);
   case TensorExp::Kind::kCIm: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::ImOp>(loc, eltType, v0);
+    return complex::ImOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kCRe: {
     auto type = cast<ComplexType>(v0.getType());
     auto eltType = cast<FloatType>(type.getElementType());
-    return rewriter.create<complex::ReOp>(loc, eltType, v0);
+    return complex::ReOp::create(rewriter, loc, eltType, v0);
   }
   case TensorExp::Kind::kBitCast:
-    return rewriter.create<arith::BitcastOp>(loc, inferType(e, v0), v0);
+    return arith::BitcastOp::create(rewriter, loc, inferType(e, v0), v0);
   // Binary operations.
   case TensorExp::Kind::kMulF:
-    return rewriter.create<arith::MulFOp>(loc, v0, v1);
+    return arith::MulFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kMulC:
-    return rewriter.create<complex::MulOp>(loc, v0, v1);
+    return complex::MulOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kMulI:
-    return rewriter.create<arith::MulIOp>(loc, v0, v1);
+    return arith::MulIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivF:
-    return rewriter.create<arith::DivFOp>(loc, v0, v1);
+    return arith::DivFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivC:
-    return rewriter.create<complex::DivOp>(loc, v0, v1);
+    return complex::DivOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivS:
-    return rewriter.create<arith::DivSIOp>(loc, v0, v1);
+    return arith::DivSIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kDivU:
-    return rewriter.create<arith::DivUIOp>(loc, v0, v1);
+    return arith::DivUIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddF:
-    return rewriter.create<arith::AddFOp>(loc, v0, v1);
+    return arith::AddFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddC:
-    return rewriter.create<complex::AddOp>(loc, v0, v1);
+    return complex::AddOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAddI:
-    return rewriter.create<arith::AddIOp>(loc, v0, v1);
+    return arith::AddIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubF:
-    return rewriter.create<arith::SubFOp>(loc, v0, v1);
+    return arith::SubFOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubC:
-    return rewriter.create<complex::SubOp>(loc, v0, v1);
+    return complex::SubOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kSubI:
-    return rewriter.create<arith::SubIOp>(loc, v0, v1);
+    return arith::SubIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kAndI:
-    return rewriter.create<arith::AndIOp>(loc, v0, v1);
+    return arith::AndIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kOrI:
-    return rewriter.create<arith::OrIOp>(loc, v0, v1);
+    return arith::OrIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kXorI:
-    return rewriter.create<arith::XOrIOp>(loc, v0, v1);
+    return arith::XOrIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShrS:
-    return rewriter.create<arith::ShRSIOp>(loc, v0, v1);
+    return arith::ShRSIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShrU:
-    return rewriter.create<arith::ShRUIOp>(loc, v0, v1);
+    return arith::ShRUIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kShlI:
-    return rewriter.create<arith::ShLIOp>(loc, v0, v1);
+    return arith::ShLIOp::create(rewriter, loc, v0, v1);
   case TensorExp::Kind::kCmpI: {
     auto predicate = llvm::cast<arith::CmpIPredicateAttr>(expr.attr);
-    return rewriter.create<arith::CmpIOp>(loc, predicate, v0, v1);
+    return arith::CmpIOp::create(rewriter, loc, predicate, v0, v1);
   }
   case TensorExp::Kind::kCmpF: {
     auto predicate = llvm::cast<arith::CmpFPredicateAttr>(expr.attr);
-    return rewriter.create<arith::CmpFOp>(loc, predicate, v0, v1);
+    return arith::CmpFOp::create(rewriter, loc, predicate, v0, v1);
   }
   case TensorExp::Kind::kBinaryBranch: // semi-ring ops with custom logic.
     return insertYieldOp(rewriter, loc, *expr.op->getBlock()->getParent(),
diff --git a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
index 26406ceef082c..78eb50cbdf540 100644
--- a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
+++ b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
@@ -74,11 +74,11 @@ struct CreatorOpShardingInterface
         if (!oldType.isDynamicDim(i) && shardType.isDynamicDim(i)) {
           if (!newSharding) {
             newSharding =
-                builder.create<ShardingOp>(op->getLoc(), resultShardings[0]);
+                ShardingOp::create(builder, op->getLoc(), resultShardings[0]);
             device =
-                builder.create<mesh::ProcessMultiIndexOp>(op->getLoc(), mesh)
+                mesh::ProcessMultiIndexOp::create(builder, op->getLoc(), mesh)
                     .getResults();
-            shapeForDevice = builder.create<mesh::ShardShapeOp>(
+            shapeForDevice = mesh::ShardShapeOp::create(builder,
                 op->getLoc(), oldType.getShape(), spmdizedOperands,
                 newSharding->getResult(0), device);
           }
@@ -88,7 +88,7 @@ struct CreatorOpShardingInterface
           newOperands.emplace_back(spmdizedOperands[++currOldOprndNum]);
         }
       }
-      newOp = builder.create<OpTy>(op->getLoc(), shardType, newOperands);
+      newOp = OpTy::create(builder, op->getLoc(), shardType, newOperands);
       spmdizationMap.map(op->getResult(0), newOp->getResult(0));
     } else {
       // `clone` will populate the mapping of old to new results.
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index b035a53692dcf..df34d9e37bd3f 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -53,7 +53,7 @@ Operation *TensorDialect::materializeConstant(OpBuilder &builder,
   if (auto op = arith::ConstantOp::materialize(builder, value, type, loc))
     return op;
   if (complex::ConstantOp::isBuildableWith(value, type))
-    return builder.create<complex::ConstantOp>(loc, type,
+    return complex::ConstantOp::create(builder, loc, type,
                                                llvm::cast<ArrayAttr>(value));
   return nullptr;
 }
@@ -107,7 +107,7 @@ FailureOr<Value> tensor::getOrCreateDestination(OpBuilder &b, Location loc,
 
   // Create empty tensor.
   Value emptyTensor =
-      b.create<tensor::EmptyOp>(loc, mixedSizes, tensorType.getElementType());
+      tensor::EmptyOp::create(b, loc, mixedSizes, tensorType.getElementType());
   return emptyTensor;
 }
 
@@ -678,7 +678,7 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
     inputShapes.emplace_back(std::move(inputShape));
   }
 
-  Value replacement = builder.create<tensor::EmptyOp>(
+  Value replacement = tensor::EmptyOp::create(builder,
       loc, outputShape, getType().getElementType());
 
   int64_t rank = getType().getRank();
@@ -687,12 +687,12 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
   SmallVector<OpFoldResult> offsets(rank, zero);
   for (auto [index, input] : llvm::enumerate(getInputs())) {
     offsets[concatDim] = concatOffsets[index];
-    auto insertSlice = builder.create<tensor::InsertSliceOp>(
+    auto insertSlice = tensor::InsertSliceOp::create(builder,
         loc, input, replacement, offsets, inputShapes[index], strides);
     replacement = insertSlice.getResult();
   }
   if (replacement.getType() != getType()) {
-    replacement = builder.create<tensor::CastOp>(loc, getType(), replacement);
+    replacement = tensor::CastOp::create(builder, loc, getType(), replacement);
   }
   return SmallVector<Value>{replacement};
 }
@@ -723,7 +723,7 @@ ConcatOp::reifyResultShapes(OpBuilder &builder,
           builder.getIndexAttr(inferredResultType.getDimSize(i)));
     } else {
       reifiedReturnShapes[0][i] =
-          builder.create<tensor::DimOp>(init.getLoc(), init, i).getResult();
+          tensor::DimOp::create(builder, init.getLoc(), init, i).getResult();
     }
   }
 
@@ -823,7 +823,7 @@ struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
 
         // Use refined operand type and create cast from original operand.
         auto castOp =
-            rewriter.create<CastOp>(concatOp->getLoc(), inferredOperandType,
+            CastOp::create(rewriter, concatOp->getLoc(), inferredOperandType,
                                     concatOp.getOperand(operandIdx));
         rewriter.modifyOpInPlace(concatOp, [=, operandIdx = operandIdx] {
           concatOp->setOperand(operandIdx, castOp->getResult(0));
@@ -864,7 +864,7 @@ struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
       return failure();
     }
 
-    auto newConcatOp = rewriter.create<ConcatOp>(
+    auto newConcatOp = ConcatOp::create(rewriter,
         concatOp->getLoc(), inferredResultType, dim, concatOp->getOperands());
     rewriter.replaceOpWithNewOp<CastOp>(concatOp, concatOp.getResultType(),
                                         newConcatOp);
@@ -892,7 +892,7 @@ void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
 void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
                   int64_t index) {
   auto loc = result.location;
-  Value indexValue = builder.create<arith::ConstantIndexOp>(loc, index);
+  Value indexValue = arith::ConstantIndexOp::create(builder, loc, index);
   build(builder, result, source, indexValue);
 }
 
@@ -1036,10 +1036,10 @@ struct DimOfReshapeOp : public OpRewritePattern<DimOp> {
     rewriter.setInsertionPointAfter(dim);
     Location loc = dim.getLoc();
     Value extract =
-        rewriter.create<ExtractOp>(loc, reshape.getShape(), dim.getIndex());
+        ExtractOp::create(rewriter, loc, reshape.getShape(), dim.getIndex());
     if (extract.getType() != dim.getType())
       extract =
-          rewriter.create<arith::IndexCastOp>(loc, dim.getType(), extract);
+          arith::IndexCastOp::create(rewriter, loc, dim.getType(), extract);
     rewriter.replaceOp(dim, extract);
     return success();
   }
@@ -1150,7 +1150,7 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
     if (foldedTensorType == op.getType())
       return failure();
 
-    auto newOp = rewriter.create<EmptyOp>(op.getLoc(), foldedTensorType,
+    auto newOp = EmptyOp::create(rewriter, op.getLoc(), foldedTensorType,
                                           foldedDynamicSizes);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
@@ -1326,7 +1326,7 @@ struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
 
       SmallVector<int64_t> basis =
           llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-      auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+      auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
           extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
       llvm::append_range(sourceIndices, delinearize.getResults());
     }
@@ -1498,7 +1498,7 @@ struct ExtractElementFromIndexCast
 
     Type elementTy = getElementTypeOrSelf(indexCast.getIn());
 
-    auto newExtract = rewriter.create<tensor::ExtractOp>(
+    auto newExtract = tensor::ExtractOp::create(rewriter,
         loc, elementTy, indexCast.getIn(), extract.getIndices());
 
     rewriter.replaceOpWithNewOp<arith::IndexCastOp>(extract, extract.getType(),
@@ -1736,7 +1736,7 @@ struct StaticTensorGenerate : public OpRewritePattern<GenerateOp> {
 
     auto loc = generateOp.getLoc();
     auto newOp =
-        rewriter.create<GenerateOp>(loc, foldedTensorType, foldedDynamicSizes);
+        GenerateOp::create(rewriter, loc, foldedTensorType, foldedDynamicSizes);
     rewriter.inlineRegionBefore(generateOp.getBody(), newOp.getBody(),
                                 newOp.getBody().begin());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(generateOp,
@@ -2161,7 +2161,7 @@ struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
         collapseShapeOp.getSrcMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = rewriter.create<CollapseShapeOp>(
+      auto newOp = CollapseShapeOp::create(rewriter,
           collapseShapeOp.getLoc(), newResultType, castOp.getSource(),
           collapseShapeOp.getReassociation());
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
@@ -2240,9 +2240,9 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
         newInputShape, expandOp.getSrcType().getElementType());
     auto outputType = RankedTensorType::get(
         newOutputShape, expandOp.getSrcType().getElementType());
-    auto inputCast = rewriter.create<CastOp>(expandOp.getLoc(), inputType,
+    auto inputCast = CastOp::create(rewriter, expandOp.getLoc(), inputType,
                                              expandOp.getSrc());
-    auto newExpand = rewriter.create<ExpandShapeOp>(
+    auto newExpand = ExpandShapeOp::create(rewriter,
         expandOp.getLoc(), outputType, inputCast.getResult(),
         expandOp.getReassociationIndices(), outputOfr);
     rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
@@ -2555,7 +2555,7 @@ class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
 
     // Create folded extract.
     Location loc = sliceOp.getLoc();
-    Value newResult = rewriter.create<ExtractSliceOp>(
+    Value newResult = ExtractSliceOp::create(rewriter,
         loc, sliceOp.getType(), castOp.getSource(), sliceOp.getOffsets(),
         sliceOp.getSizes(), sliceOp.getStrides(), sliceOp.getStaticOffsets(),
         sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
@@ -2709,7 +2709,7 @@ struct SliceCanonicalizer {
                   ExtractSliceOp newOp) {
     Value replacement = newOp.getResult();
     if (replacement.getType() != op.getType())
-      replacement = rewriter.create<tensor::CastOp>(op.getLoc(), op.getType(),
+      replacement = tensor::CastOp::create(rewriter, op.getLoc(), op.getType(),
                                                     replacement);
     rewriter.replaceOp(op, replacement);
   }
@@ -2978,7 +2978,7 @@ class InsertSliceOpConstantArgumentFolder final
       // the parallel case.
       if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      toInsert = rewriter.create<tensor::CastOp>(insertSliceOp.getLoc(),
+      toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
                                                  sourceType, toInsert);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
@@ -3075,7 +3075,7 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     if (!sliceResult.isValid)
       return failure();
 
-    Operation *replacement = rewriter.create<InsertOpTy>(
+    Operation *replacement = InsertOpTy::create(rewriter,
         insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
         mixedSizes, insertSliceOp.getMixedStrides());
 
@@ -3083,7 +3083,7 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     bool isParallelInsert =
         std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
     if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
-      replacement = rewriter.create<tensor::CastOp>(insertSliceOp.getLoc(),
+      replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
                                                     insertSliceOp.getDestType(),
                                                     replacement->getResult(0));
     }
@@ -3154,7 +3154,7 @@ struct InsertSliceOpSourceCastInserter final
     // parallel case.
     if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
       rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-    Value cast = rewriter.create<tensor::CastOp>(
+    Value cast = tensor::CastOp::create(rewriter,
         insertSliceOp.getLoc(), newSrcType, insertSliceOp.getSource());
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, cast, insertSliceOp.getDest(),
@@ -3353,7 +3353,7 @@ void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
   // a guard to reset the insertion point of the builder after it is destroyed.
   OpBuilder::InsertionGuard guard(b);
   b.createBlock(region, region->end(), blockArgTypes, blockArgLocs);
-  b.create<tensor::YieldOp>(result.location, constantPadValue);
+  tensor::YieldOp::create(b, result.location, constantPadValue);
 }
 
 llvm::SmallBitVector PadOp::getPaddedDims() {
@@ -3407,7 +3407,7 @@ struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
         padTensorOp.getSourceMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = rewriter.create<PadOp>(
+      auto newOp = PadOp::create(rewriter,
           padTensorOp->getLoc(), newResultType, padTensorOp.getSource(),
           padTensorOp.getStaticLow(), padTensorOp.getStaticHigh(),
           padTensorOp.getLow(), padTensorOp.getHigh(), padTensorOp.getNofold(),
@@ -3439,7 +3439,7 @@ struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
                                             tensorCastOp.getDest().getType()))
       return failure();
 
-    auto replacementOp = rewriter.create<PadOp>(
+    auto replacementOp = PadOp::create(rewriter,
         padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
         padTensorOp.getSource(), padTensorOp.getStaticLow(),
         padTensorOp.getStaticHigh(), padTensorOp.getLow(),
@@ -3597,10 +3597,10 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
 
     // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
     // the two paddings in one step.
-    auto newSliceOp = rewriter.create<ExtractSliceOp>(
+    auto newSliceOp = ExtractSliceOp::create(rewriter,
         padOp.getLoc(), outerSliceOp.getSource(), newOffsets, newSizes,
         innerSliceOp.getMixedStrides());
-    auto newPadOp = rewriter.create<PadOp>(
+    auto newPadOp = PadOp::create(rewriter,
         padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
         padOp.getMixedLowPad(), newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
@@ -3697,7 +3697,7 @@ struct FoldStaticPadding : public OpRewritePattern<PadOp> {
     // Rewrite the op using the new static type.
     auto newResultType = RankedTensorType::get(
         newOutDims, padTensorOp.getType().getElementType());
-    auto newOp = rewriter.create<PadOp>(
+    auto newOp = PadOp::create(rewriter,
         padTensorOp->getLoc(), newResultType, input, staticLow, staticHigh,
         newLows, newHighs, padTensorOp.getNofold(),
         getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
@@ -3777,7 +3777,7 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
     SmallVector<OpFoldResult> newLowPad =
         addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
 
-    auto newPadOp = rewriter.create<tensor::PadOp>(
+    auto newPadOp = tensor::PadOp::create(rewriter,
         padOp.getLoc(), padOp.getResultType(), producerPad.getSource(),
         newLowPad, newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, tensor::PadOp::getAttributeNames()));
@@ -3803,7 +3803,7 @@ PadOp::reifyResultShapes(OpBuilder &b,
     }
     Location loc = getLoc();
     Value dim = b.createOrFold<tensor::DimOp>(
-        loc, getSource(), b.create<arith::ConstantIndexOp>(loc, i));
+        loc, getSource(), arith::ConstantIndexOp::create(b, loc, i));
 
     AffineExpr d0, d1, d2;
     bindDims(b.getContext(), d0, d1, d2);
@@ -4108,7 +4108,7 @@ struct FoldTensorCastProducerOp
     for (auto [oldResult, newResult] :
          llvm::zip(op->getResults(), newOp->getResults())) {
       if (newResult.getType() != oldResult.getType()) {
-        replacements.push_back(rewriter.create<tensor::CastOp>(
+        replacements.push_back(tensor::CastOp::create(rewriter,
             op->getLoc(), oldResult.getType(), newResult));
       } else {
         replacements.push_back(newResult);
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 437bc5d00faa8..0a4fa7bfa007f 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -207,13 +207,13 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     if (isZeroInteger(newLength)) {
       hasZeroLen = true;
     } else if (!hasZeroLen) {
-      Value check = b.create<arith::CmpIOp>(
+      Value check = arith::CmpIOp::create(b,
           loc, arith::CmpIPredicate::eq,
           getValueOrCreateConstantIndexOp(b, loc, newLength),
           getValueOrCreateConstantIndexOp(b, loc, zero));
       dynHasZeroLenCond =
           dynHasZeroLenCond
-              ? b.create<arith::OrIOp>(loc, check, dynHasZeroLenCond)
+              ? arith::OrIOp::create(b, loc, check, dynHasZeroLenCond)
               : check;
     }
 
@@ -237,7 +237,7 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   auto castResult = [&](Value val) -> Value {
     if (resultType == val.getType())
       return val;
-    return b.create<tensor::CastOp>(loc, resultType, val);
+    return tensor::CastOp::create(b, loc, resultType, val);
   };
 
   // In cases where the original data source is unused: Emit a GenerateOp and
@@ -245,10 +245,10 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // have a dimension of size 0, the semantics of which is unclear.)
   auto createGenerateOp = [&]() {
     // Create GenerateOp.
-    auto generateOp = b.create<tensor::GenerateOp>(
+    auto generateOp = tensor::GenerateOp::create(b,
         loc, resultType, dynDims,
         [&](OpBuilder &builder, Location gLoc, ValueRange indices) {
-          builder.create<tensor::YieldOp>(gLoc, padValue);
+          tensor::YieldOp::create(builder, gLoc, padValue);
         });
     return generateOp;
   };
@@ -257,9 +257,9 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // the result shape of the new SliceOp has a zero dimension.
   auto createPadOfExtractSlice = [&]() {
     // Create pad(extract_slice(x)).
-    auto newSliceOp = b.create<tensor::ExtractSliceOp>(
+    auto newSliceOp = tensor::ExtractSliceOp::create(b,
         loc, padOp.getSource(), newOffsets, newLengths, newStrides);
-    auto newPadOp = b.create<PadOp>(
+    auto newPadOp = PadOp::create(b,
         loc, Type(), newSliceOp, newLows, newHighs,
         /*nofold=*/padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
@@ -287,17 +287,17 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     Operation *thenOp;
     Operation *elseOp;
     Operation *sliceOp;
-    auto result = b.create<scf::IfOp>(
+    auto result = scf::IfOp::create(b,
         loc, dynHasZeroLenCond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
           thenOp = createGenerateOp();
-          b.create<scf::YieldOp>(loc, castResult(thenOp->getResult(0)));
+          scf::YieldOp::create(b, loc, castResult(thenOp->getResult(0)));
         },
         /*elseBuilder=*/
         [&](OpBuilder &b, Location loc) {
           std::tie(elseOp, sliceOp) = createPadOfExtractSlice();
-          b.create<scf::YieldOp>(loc, castResult(elseOp->getResult(0)));
+          scf::YieldOp::create(b, loc, castResult(elseOp->getResult(0)));
         });
     return TilingResult{
         {elseOp}, SmallVector<Value>(result->getResults()), {sliceOp}};
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index 723731b8bed61..ba258402b61b3 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -165,7 +165,7 @@ void transform::TypeConversionCastShapeDynamicDimsOp::
     if (!tensor::CastOp::areCastCompatible(input.getType(), resultType)) {
       return Value();
     }
-    return builder.create<tensor::CastOp>(loc, resultType, input).getResult();
+    return tensor::CastOp::create(builder, loc, resultType, input).getResult();
   });
   converter.addTargetMaterialization([](OpBuilder &builder, Type resultType,
                                         ValueRange inputs,
@@ -177,7 +177,7 @@ void transform::TypeConversionCastShapeDynamicDimsOp::
     if (!tensor::CastOp::areCastCompatible(input.getType(), resultType)) {
       return Value();
     }
-    return builder.create<tensor::CastOp>(loc, resultType, input).getResult();
+    return tensor::CastOp::create(builder, loc, resultType, input).getResult();
   });
 }
 
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index 47b41efbed83b..e5b4b7a865d87 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -222,7 +222,7 @@ struct CollapseShapeOpInterface
           MemRefType::get(collapseShapeOp.getSrcType().getShape(),
                           collapseShapeOp.getSrcType().getElementType(),
                           AffineMap(), bufferType.getMemorySpace());
-      buffer = rewriter.create<bufferization::ToBufferOp>(
+      buffer = bufferization::ToBufferOp::create(rewriter,
           op->getLoc(), memrefType, *tensorAlloc);
     }
 
@@ -349,7 +349,7 @@ struct ExpandShapeOpInterface
     if (failed(buffer))
       return failure();
 
-    auto memrefExpandShape = rewriter.create<memref::ExpandShapeOp>(
+    auto memrefExpandShape = memref::ExpandShapeOp::create(rewriter,
         op->getLoc(), tensorResultType.getShape(), *buffer,
         expandShapeOp.getReassociationIndices(),
         expandShapeOp.getMixedOutputShape());
@@ -398,7 +398,7 @@ struct ExtractSliceOpInterface
         extractSliceOp.getResult(), options, state);
     if (failed(resultMemrefType))
       return failure();
-    Value subView = rewriter.create<memref::SubViewOp>(
+    Value subView = memref::SubViewOp::create(rewriter,
         loc, llvm::cast<MemRefType>(*resultMemrefType), *srcMemref,
         mixedOffsets, mixedSizes, mixedStrides);
 
@@ -469,7 +469,7 @@ static void createStores(RewriterBase &rewriter, Location loc, int dim,
   if (dim == static_cast<int>(shape.size()) - 1) {
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
-      rewriter.create<memref::StoreOp>(loc, *elementIt, buffer, indices);
+      memref::StoreOp::create(rewriter, loc, *elementIt, buffer, indices);
       ++elementIt;
     }
     return;
@@ -507,7 +507,7 @@ struct FromElementsOpInterface
         bufferization::getBufferType(*tensorAlloc, options, state);
     if (failed(memrefType))
       return failure();
-    Value buffer = rewriter.create<bufferization::ToBufferOp>(
+    Value buffer = bufferization::ToBufferOp::create(rewriter,
         op->getLoc(), *memrefType, *tensorAlloc);
 
     // Case: tensor<0xelem_type>.
@@ -518,7 +518,7 @@ struct FromElementsOpInterface
 
     // Case: tensor<elem_type>.
     if (shape.empty()) {
-      rewriter.create<memref::StoreOp>(
+      memref::StoreOp::create(rewriter,
           loc, fromElementsOp.getElements().front(), buffer);
       replaceOpWithBufferizedValues(rewriter, op, buffer);
       return success();
@@ -529,7 +529,7 @@ struct FromElementsOpInterface
     SmallVector<Value, 2> constants;
     constants.reserve(maxDim);
     for (int i = 0; i < maxDim; ++i)
-      constants.push_back(rewriter.create<arith::ConstantIndexOp>(loc, i));
+      constants.push_back(arith::ConstantIndexOp::create(rewriter, loc, i));
 
     // Traverse all `elements` and create `memref.store` ops.
     auto elementIt = fromElementsOp.getElements().begin();
@@ -576,7 +576,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   // Create linalg::MapOp.
   OpBuilder::InsertionGuard g(rewriter);
   auto linalgOp =
-      rewriter.create<linalg::MapOp>(loc, tensorType, /*inputs=*/ValueRange(),
+      linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
                                      /*init=*/tensorDestination);
   Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
@@ -584,7 +584,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   rewriter.setInsertionPointToStart(&linalgBody);
   SmallVector<Value> indices;
   for (int64_t dim = 0; dim < tensorType.getRank(); ++dim)
-    indices.push_back(rewriter.create<linalg::IndexOp>(loc, dim));
+    indices.push_back(linalg::IndexOp::create(rewriter, loc, dim));
 
   // Move over body.
   rewriter.mergeBlocks(&generateBody.front(), &linalgBody, indices);
@@ -644,7 +644,7 @@ struct InsertOpInterface
         getBuffer(rewriter, insertOp.getDest(), options, state);
     if (failed(destMemref))
       return failure();
-    rewriter.create<memref::StoreOp>(insertOp.getLoc(), insertOp.getScalar(),
+    memref::StoreOp::create(rewriter, insertOp.getLoc(), insertOp.getScalar(),
                                      *destMemref, insertOp.getIndices());
     replaceOpWithBufferizedValues(rewriter, op, *destMemref);
     return success();
@@ -713,7 +713,7 @@ struct InsertSliceOpInterface
         memref::SubViewOp::inferRankReducedResultType(
             insertSliceOp.getSourceType().getShape(), dstMemrefType,
             mixedOffsets, mixedSizes, mixedStrides);
-    Value subView = rewriter.create<memref::SubViewOp>(
+    Value subView = memref::SubViewOp::create(rewriter,
         loc, subviewMemRefType, *dstMemref, mixedOffsets, mixedSizes,
         mixedStrides);
 
@@ -796,13 +796,13 @@ struct PadOpInterface
     for (int64_t i = 0; i < resultType.getRank(); ++i) {
       if (!resultType.isDynamicDim(i))
         continue;
-      Value srcDim = rewriter.create<tensor::DimOp>(loc, padOp.getSource(), i);
+      Value srcDim = tensor::DimOp::create(rewriter, loc, padOp.getSource(), i);
       Value lowPad = toValue(mixedLowPad[i]);
       Value highPad = toValue(mixedHighPad[i]);
       AffineExpr s0, s1, s2;
       bindSymbols(op->getContext(), s0, s1, s2);
       AffineExpr sumExpr = s0 + s1 + s2;
-      Value sum = rewriter.create<affine::AffineApplyOp>(
+      Value sum = affine::AffineApplyOp::create(rewriter,
           loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
       dynamicSizes.push_back(sum);
     }
@@ -995,7 +995,7 @@ struct ParallelInsertSliceOpInterface
             parallelInsertSliceOp.getMixedOffsets(),
             parallelInsertSliceOp.getMixedSizes(),
             parallelInsertSliceOp.getMixedStrides());
-    Value subview = rewriter.create<memref::SubViewOp>(
+    Value subview = memref::SubViewOp::create(rewriter,
         parallelInsertSliceOp.getLoc(), subviewMemRefType, *destBuffer,
         parallelInsertSliceOp.getMixedOffsets(),
         parallelInsertSliceOp.getMixedSizes(),
@@ -1066,13 +1066,13 @@ struct SplatOpInterface
       return op->emitError("memory space not implemented yet");
 
     auto linalgOp =
-        rewriter.create<linalg::MapOp>(loc, tensorType, /*inputs=*/ValueRange(),
+        linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
                                        /*init=*/*tensorAlloc);
     Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
     // Create linalg::IndexOps.
     rewriter.setInsertionPointToStart(&linalgBody);
-    rewriter.create<linalg::YieldOp>(loc, splatOp.getInput());
+    linalg::YieldOp::create(rewriter, loc, splatOp.getInput());
     rewriter.replaceOp(splatOp, linalgOp.getResult()[0]);
 
     return success();
@@ -1126,7 +1126,7 @@ struct ConcatOpInterface
     MemRefType memrefType =
         MemRefType::get(concatOp.getResultType().getShape(),
                         concatOp.getResultType().getElementType(), layout);
-    Value dstBuffer = rewriter.create<bufferization::ToBufferOp>(
+    Value dstBuffer = bufferization::ToBufferOp::create(rewriter,
         op->getLoc(), memrefType, *tensorAlloc);
 
     // Extract the dimension for the concat op
@@ -1142,7 +1142,7 @@ struct ConcatOpInterface
     for (const auto &[dimIdx, dimSize] :
          llvm::enumerate(tensorType.getShape())) {
       if (dimSize == ShapedType::kDynamic) {
-        auto dimOp = rewriter.create<memref::DimOp>(loc, dstBuffer, dimIdx);
+        auto dimOp = memref::DimOp::create(rewriter, loc, dstBuffer, dimIdx);
         sizes.push_back(dimOp.getResult());
         if (dimIdx == concatDim)
           dynamicConcatDim = true;
@@ -1157,7 +1157,7 @@ struct ConcatOpInterface
     if (dynamicConcatDim) {
       // One or more operands have dynamic size, so we must accumulate the
       // offset with arith ops.
-      dynamicOffset = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+      dynamicOffset = arith::ConstantIndexOp::create(rewriter, loc, 0);
     }
 
     for (auto operand : concatOp.getInputs()) {
@@ -1174,7 +1174,7 @@ struct ConcatOpInterface
 
       if (dynamicConcatDim) {
         offsets[concatDim] = dynamicOffset.value();
-        dynamicSize = rewriter.create<memref::DimOp>(loc, *srcBuffer, concatDim)
+        dynamicSize = memref::DimOp::create(rewriter, loc, *srcBuffer, concatDim)
                           .getResult();
         sizes[concatDim] = dynamicSize.value();
       } else {
@@ -1188,7 +1188,7 @@ struct ConcatOpInterface
           memref::SubViewOp::inferRankReducedResultType(
               operandTensorType.getShape(), dstMemrefType, offsets, sizes,
               strides);
-      Value subview = rewriter.create<memref::SubViewOp>(
+      Value subview = memref::SubViewOp::create(rewriter,
           loc, subviewMemRefType, dstBuffer, offsets, sizes, strides);
 
       // Copy the source buffer into the destination subview.
@@ -1196,7 +1196,7 @@ struct ConcatOpInterface
         return failure();
 
       if (dynamicConcatDim) {
-        dynamicOffset = rewriter.create<arith::AddIOp>(
+        dynamicOffset = arith::AddIOp::create(rewriter,
             loc, dynamicOffset.value(), dynamicSize.value());
       } else {
         concatDimOffset += operandConcatDimSize;
diff --git a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
index 3c2b0ab42f7a6..24bcb8192f672 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
@@ -42,7 +42,7 @@ struct FoldEmptyTensorWithReshapeOp : public OpRewritePattern<ReshapeOp> {
 
     // Create new tensor.empty op.
     // TODO: Do not drop tensor type encoding.
-    Value emptyTensor = rewriter.create<EmptyOp>(
+    Value emptyTensor = EmptyOp::create(rewriter,
         loc, resultShapes[0], reshapeOp.getResultType().getElementType());
     if (emptyTensor.getType() != reshapeOp.getResultType()) {
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
index dd50ae54d17cc..c3801e0f05c88 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
@@ -53,7 +53,7 @@ static ValueRange invertCollapseShapeIndexing(
   for (int64_t i : reassociation[dim])
     basis.push_back(reshapeSourceShape[i]);
   auto delinearized =
-      b.create<AffineDelinearizeIndexOp>(loc, indexValue, basis);
+      AffineDelinearizeIndexOp::create(b, loc, indexValue, basis);
   return delinearized->getResults();
 }
 
@@ -142,14 +142,14 @@ tensor::ExtractSliceFromCollapseHelper::emitLoopNestBody(
   SmallVector<Range> extractParams =
       helper.getExtractSliceParams(builder.getContext(), multiIndices);
 
-  Value subTileResult = builder.create<tensor::ExtractSliceOp>(
+  Value subTileResult = tensor::ExtractSliceOp::create(builder,
       loc, collapseShapeOp.getSrc(), extractParams);
 
   SmallVector<Range> insertParams =
       helper.getInsertSliceParams(builder.getContext(), tileInductionVars);
 
   // Collapse the dimensions of the source slice back down.
-  Value collapsedResult = builder.create<tensor::CollapseShapeOp>(
+  Value collapsedResult = tensor::CollapseShapeOp::create(builder,
       loc, subTileResult, reassociationIndices);
   return std::make_pair(collapsedResult, insertParams);
 }
@@ -173,7 +173,7 @@ tensor::simplifyCollapseShapeWithRankReducingExtractSlice(
   SmallVector<OpFoldResult> sizes =
       tensor::getMixedSizes(rewriter, op.getLoc(), op.getSrc());
   SmallVector<OpFoldResult> strides(sourceType.getRank(), one);
-  auto sliceOp = rewriter.create<tensor::ExtractSliceOp>(
+  auto sliceOp = tensor::ExtractSliceOp::create(rewriter,
       op.getLoc(), info->sliceResultType, op.getSrc(), offsets, sizes, strides);
 
   if (!info->newReassociationIndices.has_value()) {
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
index 13de55b0672a5..cc32f1193a55b 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
@@ -112,7 +112,7 @@ TransferReadOfExtractSliceOpFolder::matchAndRewriteMaskableOp(
       extractSliceOp.getMixedStrides(), extractSliceOp.getDroppedDims(),
       indices, sourceIndices);
 
-  Operation *newOp = rewriter.create<vector::TransferReadOp>(
+  Operation *newOp = vector::TransferReadOp::create(rewriter,
       readOp.getLoc(), readOp.getVectorType(), extractSliceOp.getSource(),
       sourceIndices,
       AffineMapAttr::get(expandDimsToRank(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
index bad56d4111dca..a6db540ffd8a0 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
@@ -64,7 +64,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     return padOp.getResult();
 
   // Create a new tensor::PadOp.
-  auto newPadOp = b.create<PadOp>(
+  auto newPadOp = PadOp::create(b,
       loc, padOp.getResultType(), padOp.getSource(), newMixedLow, newMixedHigh,
       constantPadding, padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
 
@@ -83,7 +83,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
       offsets.push_back(b.getIndexAttr(0));
     } else {
       offsets.push_back(
-          b.create<affine::AffineApplyOp>(
+          affine::AffineApplyOp::create(b,
                loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
                std::initializer_list<Value>{cast<Value>(newMixedLow[i]),
                                             cast<Value>(prevLow)})
@@ -99,7 +99,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     strides.push_back(b.getIndexAttr(1));
   }
 
-  return b.create<ExtractSliceOp>(loc, newPadOp, offsets, sizes, strides)
+  return ExtractSliceOp::create(b, loc, newPadOp, offsets, sizes, strides)
       .getResult();
 }
 
@@ -124,7 +124,7 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b,
 
   // Create a new tensor::EmptyOp.
   Value newEmptyOp =
-      b.create<EmptyOp>(loc, newSizes, emptyOp.getType().getElementType());
+      EmptyOp::create(b, loc, newSizes, emptyOp.getType().getElementType());
 
   // Create a tensor::ExtractSliceOp.
   SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index 20bb4d1caf019..3b63c87f2c05a 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -76,7 +76,7 @@ struct FoldUnPaddingCollapseIntoExtract
       return rewriter.notifyMatchFailure(collapseShapeOp,
                                          "expected unpadding collapse");
 
-    Value unPaddedExtractSlice = rewriter.create<tensor::ExtractSliceOp>(
+    Value unPaddedExtractSlice = tensor::ExtractSliceOp::create(rewriter,
         extractSliceOp.getLoc(), collapseShapeOp.getResultType(),
         extractSliceOp.getSource(), extractSliceOp.getMixedOffsets(),
         extractSliceOp.getMixedSizes(), extractSliceOp.getMixedStrides());
@@ -270,7 +270,7 @@ struct BubbleUpExpandThroughParallelCollapse
     // matches the number of dimensions of the result, then the expand_shape
     // is a no-op.
     if (newExpandReInds.size() != newExpandSizes.size()) {
-      newCollapseSrc = rewriter.create<tensor::ExpandShapeOp>(
+      newCollapseSrc = tensor::ExpandShapeOp::create(rewriter,
           loc, expandResultType, newCollapseSrc, newExpandReInds,
           newExpandSizes);
     }
@@ -280,7 +280,7 @@ struct BubbleUpExpandThroughParallelCollapse
     // is a no-op.
     Value replacement = newCollapseSrc;
     if (newCollapseReInds.size() != newExpandSizes.size()) {
-      replacement = rewriter.create<tensor::CollapseShapeOp>(
+      replacement = tensor::CollapseShapeOp::create(rewriter,
           loc, newCollapseSrc, newCollapseReInds);
     }
     rewriter.replaceOp(expandOp, replacement);
@@ -405,7 +405,7 @@ struct BubbleUpExpandShapeThroughExtractSlice
         shape, expandShapeOp.getResultType().getElementType());
 
     // Create a new ExtractSliceOp and ExpandShapeOp.
-    Value newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
         collapsedStrides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -735,7 +735,7 @@ struct BubbleUpCollapseShapeThroughExtractSlice
                              groupExpandedOffsets.rend());
     }
 
-    Value newSliceOp = rewriter.create<tensor::ExtractSliceOp>(
+    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
         collapseShapeOp->getLoc(), collapseShapeOp.getSrc(), expandedOffsets,
         expandedSizes, expandedStrides);
     rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
index 7c9fced540adb..69e649d2eebe8 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
@@ -196,7 +196,7 @@ struct PadOpToConstant final : public OpRewritePattern<PadOp> {
                                          "tensor type not supported");
 
     if (newOp.getType() != resultType)
-      newOp = rewriter.create<tensor::CastOp>(loc, resultType, newOp);
+      newOp = tensor::CastOp::create(rewriter, loc, resultType, newOp);
 
     rewriter.replaceOp(padTensorOp, newOp);
     return success();
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
index 6e3285abffbfc..7c1dea45843d5 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
@@ -47,12 +47,12 @@ struct CastOpInterface
 
     if (isa<UnrankedTensorType>(srcType)) {
       // Check rank.
-      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
-          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
-      Value isSameRank = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
+      Value isSameRank = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameRank,
           RuntimeVerifiableOpInterface::generateErrorMessage(op,
                                                              "rank mismatch"));
@@ -70,12 +70,12 @@ struct CastOpInterface
         continue;
 
       Value srcDimSz =
-          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+          DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
-          builder.create<arith::ConstantIndexOp>(loc, it.value());
-      Value isSameSz = builder.create<arith::CmpIOp>(
+          arith::ConstantIndexOp::create(builder, loc, it.value());
+      Value isSameSz = arith::CmpIOp::create(builder,
           loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
@@ -89,9 +89,9 @@ struct DimOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto dimOp = cast<DimOp>(op);
-    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    builder.create<cf::AssertOp>(
+    Value rank = RankOp::create(builder, loc, dimOp.getSource());
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    cf::AssertOp::create(builder,
         loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
@@ -124,7 +124,7 @@ struct ExtractInsertOpInterface
     }
 
     auto indices = extractInsertOp.getIndices();
-    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
     Value assertCond;
     for (auto i : llvm::seq<int64_t>(0, rank)) {
       Value dimOp = builder.createOrFold<tensor::DimOp>(loc, tensor, i);
@@ -134,7 +134,7 @@ struct ExtractInsertOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    builder.create<cf::AssertOp>(
+    cf::AssertOp::create(builder,
         loc, assertCond,
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "out-of-bounds access"));
@@ -152,8 +152,8 @@ struct ExtractSliceOpInterface
     // For each dimension, assert that:
     // 0 <= offset < dim_size
     // 0 <= offset + (size - 1) * stride < dim_size
-    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
+    Value one = arith::ConstantIndexOp::create(builder, loc, 1);
     for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
       Value offset = getValueOrCreateConstantIndexOp(
           builder, loc, extractSliceOp.getMixedOffsets()[i]);
@@ -167,20 +167,20 @@ struct ExtractSliceOpInterface
           loc, extractSliceOp.getSource(), i);
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
       // Verify that slice does not run out-of-bounds.
-      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOne = arith::SubIOp::create(builder, loc, size, one);
       Value sizeMinusOneTimesStride =
-          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+          arith::MulIOp::create(builder, loc, sizeMinusOne, stride);
       Value lastPos =
-          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+          arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      builder.create<cf::AssertOp>(
+      cf::AssertOp::create(builder,
           loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "extract_slice runs out-of-bounds along dimension " +
diff --git a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
index d50d7c62b789c..21f70e52b8706 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
@@ -59,7 +59,7 @@ struct InsertSliceLikeOpSubsetInsertionOpInterface
   Value buildSubsetExtraction(Operation *op, OpBuilder &builder,
                               Location loc) const {
     auto insertSliceOp = cast<OpTy>(op);
-    auto extractOp = builder.create<tensor::ExtractSliceOp>(
+    auto extractOp = tensor::ExtractSliceOp::create(builder,
         loc, insertSliceOp.getSourceType(), insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
         insertSliceOp.getMixedStrides());
diff --git a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
index 3f6258b5e4d43..fd75e1090174b 100644
--- a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
@@ -56,7 +56,7 @@ PadOp mlir::tensor::createPadHighOp(RankedTensorType resType, Value source,
     high[idx] = affine::makeComposedFoldedAffineApply(b, loc, d0 - d1,
                                                       {outDim, sourceDim});
   }
-  return b.create<PadOp>(loc, resType, source, low, high, pad, nofold);
+  return PadOp::create(b, loc, resType, source, low, high, pad, nofold);
 }
 
 SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
@@ -67,7 +67,7 @@ SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
   for (const auto &en : llvm::enumerate(tensorTy.getShape())) {
     if (en.value() == ShapedType::kDynamic)
       dynamicDims.push_back(
-          b.create<tensor::DimOp>(loc, rankedTensor, en.index()));
+          tensor::DimOp::create(b, loc, rankedTensor, en.index()));
   }
   return dynamicDims;
 }
@@ -119,7 +119,7 @@ mlir::tensor::dropGivenUnitDims(OpBuilder &b, Location loc, Value src,
     reassocMaps.emplace_back(llvm::make_range(seq.begin(), seq.end()));
     nextDimToGroup = setBit + 1;
   }
-  return b.create<tensor::CollapseShapeOp>(loc, src, reassocMaps);
+  return tensor::CollapseShapeOp::create(b, loc, src, reassocMaps);
 }
 
 bool mlir::tensor::isCastLikeInsertSliceOp(InsertSliceOp op) {
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 2dd45d27157cb..7ca4f6496ccdc 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -845,7 +845,7 @@ struct PadSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
         getTosaConstShape(rewriter, sliceOp.getLoc(), newPadPaddings);
     auto newPadTy =
         RankedTensorType::get(newPadShape, inputTy.getElementType());
-    auto newPadOp = rewriter.create<tosa::PadOp>(
+    auto newPadOp = tosa::PadOp::create(rewriter,
         padOp.getLoc(), newPadTy, padOp.getInput1(), newPaddingsOp,
         padOp.getPadConst());
 
@@ -897,7 +897,7 @@ struct SliceDynamicSizeCanonicalization
     }
 
     auto size_op = getTosaConstShape(rewriter, sliceOp.getLoc(), sliceSizes);
-    auto newSliceOp = rewriter.create<tosa::SliceOp>(
+    auto newSliceOp = tosa::SliceOp::create(rewriter,
         sliceOp.getLoc(), sliceOp.getType(), sliceOp.getInput1(),
         sliceOp.getStart(), size_op);
 
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 4a952ac062cad..d110e1aeee8e4 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -180,11 +180,11 @@ Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // Tosa dialect constants only support ElementsAttr unlike standard dialect
   // constant which supports all attributes.
   if (llvm::isa<shapeType>(type) && llvm::isa<DenseIntElementsAttr>(value)) {
-    return builder.create<tosa::ConstShapeOp>(
+    return tosa::ConstShapeOp::create(builder,
         loc, type, llvm::cast<DenseIntElementsAttr>(value));
   }
   if (llvm::isa<ElementsAttr>(value))
-    return builder.create<tosa::ConstOp>(loc, type,
+    return tosa::ConstOp::create(builder, loc, type,
                                          llvm::cast<ElementsAttr>(value));
   return nullptr;
 }
@@ -323,7 +323,7 @@ Value mlir::tosa::createPadConstTensor(OpBuilder &builder, Location loc,
                                    builder.getFloatAttr(srcElemType, val))
           : DenseElementsAttr::get(padConstEType,
                                    builder.getIntegerAttr(srcElemType, val))};
-  return builder.create<tosa::ConstOp>(loc, padConstType, padConstAttr);
+  return tosa::ConstOp::create(builder, loc, padConstType, padConstAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2415,7 +2415,7 @@ LogicalResult TransposeOp::reifyResultShapes(
     int32_t dimInInput = transposePerms[dim];
     if (inputType.isDynamicDim(dimInInput))
       returnedDims[dim] =
-          builder.create<tensor::DimOp>(getLoc(), input, dimInInput)
+          tensor::DimOp::create(builder, getLoc(), input, dimInInput)
               .getResult();
     else
       returnedDims[dim] =
@@ -3946,12 +3946,12 @@ std::optional<Value> mlir::tosa::createZeroPointTensor(OpBuilder &builder,
   if (llvm::isa<FloatType>(srcElemType)) {
     auto zpAttr = DenseElementsAttr::get(
         zpType, builder.getFloatAttr(srcElemType, static_cast<double>(zp)));
-    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+    return tosa::ConstOp::create(builder, loc, zpType, zpAttr);
   }
   if (llvm::isa<IntegerType>(srcElemType)) {
     auto zpAttr =
         DenseElementsAttr::get(zpType, builder.getIntegerAttr(srcElemType, zp));
-    return builder.create<tosa::ConstOp>(loc, zpType, zpAttr);
+    return tosa::ConstOp::create(builder, loc, zpType, zpAttr);
   }
   llvm::errs() << "zero point is not allowed for unsupported data types\n";
   return std::nullopt;
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
index f6caa2a985a4d..9b77986257814 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
@@ -90,12 +90,12 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
 
     if (inputETy != resultETy) {
       inputType = inputType.clone(resultETy);
-      input = rewriter.create<tosa::CastOp>(op.getLoc(), inputType, input);
+      input = tosa::CastOp::create(rewriter, op.getLoc(), inputType, input);
     }
 
     if (weightETy != resultETy) {
       weightType = weightType.clone(resultETy);
-      weight = rewriter.create<tosa::CastOp>(op.getLoc(), weightType, weight);
+      weight = tosa::CastOp::create(rewriter, op.getLoc(), weightType, weight);
     }
 
     if (iZp != 0 || wZp != 0) {
@@ -109,8 +109,8 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         auto zpTy = RankedTensorType::get(shape, ety);
         auto zpAttr =
             DenseElementsAttr::get(zpTy, rewriter.getIntegerAttr(ety, zp));
-        auto zpVal = rewriter.create<tosa::ConstOp>(op.getLoc(), zpTy, zpAttr);
-        return rewriter.create<tosa::SubOp>(op.getLoc(), val.getType(), val,
+        auto zpVal = tosa::ConstOp::create(rewriter, op.getLoc(), zpTy, zpAttr);
+        return tosa::SubOp::create(rewriter, op.getLoc(), val.getType(), val,
                                             zpVal);
       };
 
@@ -138,9 +138,9 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
       auto padTy = RankedTensorType::get({1}, inputETy);
       auto padAttr = DenseElementsAttr::get(padTy, zeroAttr);
       Value padVal =
-          rewriter.create<tosa::ConstOp>(op->getLoc(), padTy, padAttr);
+          tosa::ConstOp::create(rewriter, op->getLoc(), padTy, padAttr);
       inputType = RankedTensorType::get(newShape, inputETy);
-      input = rewriter.create<tosa::PadOp>(op->getLoc(), inputType, input,
+      input = tosa::PadOp::create(rewriter, op->getLoc(), inputType, input,
                                            padSizeVal, padVal);
     }
 
@@ -161,7 +161,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
     auto shiftZeroAttr = DenseElementsAttr::get(
         shiftType, rewriter.getIntegerAttr(shiftElementType, 0));
     Value constZero =
-        rewriter.create<tosa::ConstOp>(op.getLoc(), shiftType, shiftZeroAttr);
+        tosa::ConstOp::create(rewriter, op.getLoc(), shiftType, shiftZeroAttr);
     Value mulValue = rewriter
                          .create<tosa::MulOp>(op.getLoc(), mulShapeType, input,
                                               weight, constZero)
@@ -174,7 +174,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         dyn_cast<RankedTensorType>(input.getType()).getElementType());
     auto outputShapeValue =
         getTosaConstShape(rewriter, op->getLoc(), outputShape);
-    Value outputValue = rewriter.create<tosa::ReshapeOp>(
+    Value outputValue = tosa::ReshapeOp::create(rewriter,
         op.getLoc(), outputShapeType, mulValue, outputShapeValue);
 
     Value bias = op.getBias();
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
index df6d52615478e..4a07220b4e181 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
@@ -62,12 +62,12 @@ class TransposeConvNonStridedConverter
     convPad[2] = kernelWidth - 1 + pad[2];
     convPad[3] = kernelWidth - 1 + pad[3];
 
-    auto reverse1 = rewriter.create<tosa::ReverseOp>(
+    auto reverse1 = tosa::ReverseOp::create(rewriter,
         loc, weightTy, weight, /* axis = */ rewriter.getI32IntegerAttr(1));
-    auto reverse2 = rewriter.create<tosa::ReverseOp>(
+    auto reverse2 = tosa::ReverseOp::create(rewriter,
         loc, weightTy, reverse1, /* axis = */ rewriter.getI32IntegerAttr(2));
 
-    Value conv2d = rewriter.create<tosa::Conv2DOp>(
+    Value conv2d = tosa::Conv2DOp::create(rewriter,
         loc, resultTy, input, reverse2, bias, op.getInputZp(), op.getWeightZp(),
         rewriter.getDenseI64ArrayAttr(convPad),
         rewriter.getDenseI64ArrayAttr(stride),
@@ -216,7 +216,7 @@ class TransposeConvStridedConverter
         inputPaddingVal, inputPadConst);
 
     // We use a zero bias as we need to broadcast the bias.
-    auto zeroBias = rewriter.create<tosa::ConstOp>(
+    auto zeroBias = tosa::ConstOp::create(rewriter,
         loc,
         RankedTensorType::get({outputChannels * stride[0] * stride[1]},
                               biasETy),
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index a9e98c8908e15..4d347c02ee16d 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -112,7 +112,7 @@ class TypeModificationState {
           OpBuilder builder{value.getContext()};
           builder.setInsertionPointAfter(value.getDefiningOp());
           castValue =
-              builder.create<tensor::CastOp>(value.getLoc(), oldType, value);
+              tensor::CastOp::create(builder, value.getLoc(), oldType, value);
         }
 
         use->set(castValue);
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index 8ebbbc94eb6a2..a281c6600fdf6 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -403,7 +403,7 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
     return std::nullopt;
   }
   ImplicitLocOpBuilder builder(reshapeOp.getLoc(), rewriter);
-  auto foldedReshape = rewriter.create<ReshapeOp>(
+  auto foldedReshape = ReshapeOp::create(rewriter,
       reshapeOp.getLoc(),
       RankedTensorType::get(applyTOSAPermutation(shape, hoistedPerms),
                             reshapeOutputType.getElementType()),
@@ -423,7 +423,7 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
   if (!maybeNewDenseAttr.has_value())
     return std::nullopt;
   auto newDenseAttr = maybeNewDenseAttr.value();
-  auto newConstOp = rewriter.create<ConstOp>(
+  auto newConstOp = ConstOp::create(rewriter,
       constOp.getLoc(), newDenseAttr.getType(), newDenseAttr);
   return newConstOp->getResult(0);
 }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
index 3b697a2ee3e47..677d8e9904a67 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaTypeConverters.cpp
@@ -37,7 +37,7 @@ void mlir::tosa::populateTosaTypeConversion(TypeConverter &converter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
   converter.addTargetMaterialization([&](OpBuilder &builder, Type resultType,
@@ -46,7 +46,7 @@ void mlir::tosa::populateTosaTypeConversion(TypeConverter &converter) {
     if (inputs.size() != 1)
       return Value();
 
-    return builder.create<UnrealizedConversionCastOp>(loc, resultType, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, resultType, inputs)
         .getResult(0);
   });
 }
diff --git a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
index 9844abcc34cb1..69ef0ba40d72e 100644
--- a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
@@ -33,18 +33,18 @@ mlir::tosa::condenseValues(const SmallVector<Value> &values) {
 
 Value mlir::tosa::clampFloatHelper(Location loc, Value arg, Value min,
                                    Value max, OpBuilder &rewriter) {
-  Value minValue = rewriter.create<arith::MinimumFOp>(loc, arg, max);
-  return rewriter.create<arith::MaximumFOp>(loc, minValue, min);
+  Value minValue = arith::MinimumFOp::create(rewriter, loc, arg, max);
+  return arith::MaximumFOp::create(rewriter, loc, minValue, min);
 }
 
 Value mlir::tosa::clampIntHelper(Location loc, Value arg, Value min, Value max,
                                  OpBuilder &rewriter, bool isUnsigned) {
   if (isUnsigned) {
-    auto minOrArg = rewriter.create<arith::MaxUIOp>(loc, min, arg);
-    return rewriter.create<arith::MinUIOp>(loc, max, minOrArg);
+    auto minOrArg = arith::MaxUIOp::create(rewriter, loc, min, arg);
+    return arith::MinUIOp::create(rewriter, loc, max, minOrArg);
   }
-  auto minOrArg = rewriter.create<arith::MaxSIOp>(loc, min, arg);
-  return rewriter.create<arith::MinSIOp>(loc, max, minOrArg);
+  auto minOrArg = arith::MaxSIOp::create(rewriter, loc, min, arg);
+  return arith::MinSIOp::create(rewriter, loc, max, minOrArg);
 }
 
 bool mlir::tosa::validIntegerRange(IntegerType ty, int64_t value) {
@@ -144,7 +144,7 @@ LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder,
       ArrayRef<int64_t>(reshapeOutputShape), reshapeInputType.getElementType());
   auto reshapeOutputShapeValue = getTosaConstShape(builder, reshapeOutputShape);
 
-  auto reshapeLower = builder.create<tosa::ReshapeOp>(
+  auto reshapeLower = tosa::ReshapeOp::create(builder,
       reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
 
   if (input1Rank > input2Rank) {
@@ -162,7 +162,7 @@ Value mlir::tosa::getTosaConstShape(ImplicitLocOpBuilder &builder,
                                     llvm::ArrayRef<int64_t> shape) {
   auto attr = builder.getIndexTensorAttr(convertFromMlirShape(shape));
   auto type = mlir::tosa::shapeType::get(builder.getContext(), shape.size());
-  mlir::Operation *mlir_op = builder.create<tosa::ConstShapeOp>(type, attr);
+  mlir::Operation *mlir_op = tosa::ConstShapeOp::create(builder, type, attr);
   return mlir_op->getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/UB/IR/UBOps.cpp b/mlir/lib/Dialect/UB/IR/UBOps.cpp
index 5b2cfe7bf4264..ee523f9522953 100644
--- a/mlir/lib/Dialect/UB/IR/UBOps.cpp
+++ b/mlir/lib/Dialect/UB/IR/UBOps.cpp
@@ -52,7 +52,7 @@ void UBDialect::initialize() {
 Operation *UBDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                           Type type, Location loc) {
   if (auto attr = dyn_cast<PoisonAttr>(value))
-    return builder.create<PoisonOp>(loc, type, attr);
+    return PoisonOp::create(builder, loc, type, attr);
 
   return nullptr;
 }
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index b58ed6b5d1144..3497b0d7a88a0 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -125,7 +125,7 @@ static MaskFormat getMaskFormat(Value mask) {
 /// Default callback to build a region with a 'vector.yield' terminator with no
 /// arguments.
 void mlir::vector::buildTerminatedBody(OpBuilder &builder, Location loc) {
-  builder.create<vector::YieldOp>(loc);
+  vector::YieldOp::create(builder, loc);
 }
 
 // Helper for verifying combining kinds in contractions and reductions.
@@ -597,16 +597,16 @@ struct ElideUnitDimsInMultiDimReduction
         VectorType newMaskType =
             VectorType::get(dstVecType.getShape(), rewriter.getI1Type(),
                             dstVecType.getScalableDims());
-        mask = rewriter.create<vector::ShapeCastOp>(loc, newMaskType, mask);
+        mask = vector::ShapeCastOp::create(rewriter, loc, newMaskType, mask);
       }
-      cast = rewriter.create<vector::ShapeCastOp>(
+      cast = vector::ShapeCastOp::create(rewriter,
           loc, reductionOp.getDestType(), reductionOp.getSource());
     } else {
       // This means we are reducing all the dimensions, and all reduction
       // dimensions are of size 1. So a simple extraction would do.
       if (mask)
-        mask = rewriter.create<vector::ExtractOp>(loc, mask);
-      cast = rewriter.create<vector::ExtractOp>(loc, reductionOp.getSource());
+        mask = vector::ExtractOp::create(rewriter, loc, mask);
+      cast = vector::ExtractOp::create(rewriter, loc, reductionOp.getSource());
     }
 
     Value result =
@@ -673,35 +673,35 @@ Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
   switch (op) {
   case arith::AtomicRMWKind::addf:
   case arith::AtomicRMWKind::addi:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::ADD, vector);
   case arith::AtomicRMWKind::mulf:
   case arith::AtomicRMWKind::muli:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MUL, vector);
   case arith::AtomicRMWKind::minimumf:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINIMUMF, vector);
   case arith::AtomicRMWKind::mins:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINSI, vector);
   case arith::AtomicRMWKind::minu:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MINUI, vector);
   case arith::AtomicRMWKind::maximumf:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXIMUMF, vector);
   case arith::AtomicRMWKind::maxs:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXSI, vector);
   case arith::AtomicRMWKind::maxu:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::MAXUI, vector);
   case arith::AtomicRMWKind::andi:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::AND, vector);
   case arith::AtomicRMWKind::ori:
-    return builder.create<vector::ReductionOp>(vector.getLoc(),
+    return vector::ReductionOp::create(builder, vector.getLoc(),
                                                CombiningKind::OR, vector);
   // TODO: Add remaining reduction operations.
   default:
@@ -741,8 +741,8 @@ struct ElideSingleElementReduction : public OpRewritePattern<ReductionOp> {
 
     Location loc = reductionOp.getLoc();
     if (mask)
-      mask = rewriter.create<ExtractOp>(loc, mask);
-    Value result = rewriter.create<ExtractOp>(loc, reductionOp.getVector());
+      mask = ExtractOp::create(rewriter, loc, mask);
+    Value result = ExtractOp::create(rewriter, loc, reductionOp.getVector());
 
     if (Value acc = reductionOp.getAcc())
       result = vector::makeArithReduction(rewriter, loc, reductionOp.getKind(),
@@ -4255,7 +4255,7 @@ class StridedSliceBroadcast final
     // just a single scalar.
     bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
     if (!lowerDimMatch && !isScalarSrc) {
-      source = rewriter.create<ExtractStridedSliceOp>(
+      source = ExtractStridedSliceOp::create(rewriter,
           op->getLoc(), source,
           getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
@@ -4350,7 +4350,7 @@ class ContiguousExtractStridedSliceToExtract final
 
     SmallVector<int64_t> offsets = getI64SubArray(op.getOffsets());
     auto extractOffsets = ArrayRef(offsets).take_front(numOffsets);
-    Value extract = rewriter.create<vector::ExtractOp>(op->getLoc(), source,
+    Value extract = vector::ExtractOp::create(rewriter, op->getLoc(), source,
                                                        extractOffsets);
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(), extract);
     return success();
@@ -4381,7 +4381,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
 
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
         *padding, /*mask=*/Value(), inBoundsAttr);
 }
@@ -4399,7 +4399,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                                 SmallVector<bool>(vectorType.getRank(), false));
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, *padding,
         permutationMapAttr, inBoundsAttr);
 }
@@ -4418,7 +4418,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                                 SmallVector<bool>(vectorType.getRank(), false));
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
   if (!padding)
-    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+    padding = ub::PoisonOp::create(builder, result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
         *padding,
         /*mask=*/Value(), inBoundsAttr);
@@ -4943,7 +4943,7 @@ struct TransferReadAfterWriteToBroadcast
     VectorType broadcastedType = VectorType::get(
         broadcastShape, defWrite.getVectorType().getElementType(),
         broadcastScalableFlags);
-    vec = rewriter.create<vector::BroadcastOp>(loc, broadcastedType, vec);
+    vec = vector::BroadcastOp::create(rewriter, loc, broadcastedType, vec);
     SmallVector<int64_t> transposePerm(permutation.begin(), permutation.end());
     rewriter.replaceOpWithNewOp<vector::TransposeOp>(readOp, vec,
                                                      transposePerm);
@@ -5421,11 +5421,11 @@ struct SwapExtractSliceOfTransferWrite
     // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
     // Set all in_bounds to false and let the folder infer them.
     SmallVector<bool> newInBounds(vectorShape.size(), false);
-    auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
+    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
         extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
         insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
         insertOp.getMixedStrides());
-    auto newTransferWriteOp = rewriter.create<TransferWriteOp>(
+    auto newTransferWriteOp = TransferWriteOp::create(rewriter,
         transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
         transferOp.getIndices(), transferOp.getPermutationMapAttr(),
         rewriter.getBoolArrayAttr(newInBounds));
@@ -6951,7 +6951,7 @@ void MaskOp::ensureTerminator(Region &region, Builder &builder, Location loc) {
   OpBuilder opBuilder(builder.getContext());
   Operation *maskedOp = &block.front();
   opBuilder.setInsertionPointToEnd(&block);
-  opBuilder.create<vector::YieldOp>(loc, maskedOp->getResults());
+  vector::YieldOp::create(opBuilder, loc, maskedOp->getResults());
 }
 
 LogicalResult MaskOp::verify() {
@@ -7286,7 +7286,7 @@ void mlir::vector::createMaskOpRegion(OpBuilder &builder,
   // Create a block and move the op to that block.
   insBlock->getOperations().splice(
       insBlock->begin(), maskableOp->getBlock()->getOperations(), maskableOp);
-  builder.create<YieldOp>(maskableOp->getLoc(), maskableOp->getResults());
+  YieldOp::create(builder, maskableOp->getLoc(), maskableOp->getResults());
 }
 
 /// Creates a vector.mask operation around a maskable operation. Returns the
@@ -7298,10 +7298,10 @@ Operation *mlir::vector::maskOperation(OpBuilder &builder,
   if (!mask)
     return maskableOp;
   if (passthru)
-    return builder.create<MaskOp>(maskableOp->getLoc(),
+    return MaskOp::create(builder, maskableOp->getLoc(),
                                   maskableOp->getResultTypes(), mask, passthru,
                                   maskableOp, createMaskOpRegion);
-  return builder.create<MaskOp>(maskableOp->getLoc(),
+  return MaskOp::create(builder, maskableOp->getLoc(),
                                 maskableOp->getResultTypes(), mask, maskableOp,
                                 createMaskOpRegion);
 }
@@ -7318,7 +7318,7 @@ Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
   if (!mask)
     return newValue;
 
-  return builder.create<arith::SelectOp>(newValue.getLoc(), newValue.getType(),
+  return arith::SelectOp::create(builder, newValue.getLoc(), newValue.getType(),
                                          mask, newValue, passthru);
 }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
index 9da051150e409..c750017a6b504 100644
--- a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -116,7 +116,7 @@ struct TransferWriteOpInterface
         getBuffer(rewriter, writeOp.getBase(), options, state);
     if (failed(resultBuffer))
       return failure();
-    rewriter.create<vector::TransferWriteOp>(
+    vector::TransferWriteOp::create(rewriter,
         writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
         writeOp.getIndices(), writeOp.getPermutationMapAttr(),
         writeOp.getMask(), writeOp.getInBoundsAttr());
@@ -241,7 +241,7 @@ struct MaskOpInterface
     // Create a new vector.mask op.
     ValueRange newYieldedValuesRange(newYieldedValues);
     TypeRange newResultTypes(newYieldedValuesRange);
-    auto newOp = rewriter.create<vector::MaskOp>(
+    auto newOp = vector::MaskOp::create(rewriter,
         op->getLoc(), newResultTypes, maskOp.getMask(), maskOp.getPassthru(),
         /*maskableOp=*/nullptr,
         /*maskRegionBuilder=*/[](OpBuilder &b, Operation *) {});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
index 89930a6bd35fa..4c3a04cfb5bfa 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBitCast.cpp
@@ -64,14 +64,14 @@ class UnrollBitCastOp final : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(shape, resultType.getElementType(), scalableDims);
 
     Location loc = op.getLoc();
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
     for (auto position : *unrollIterator) {
       Value extract =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+          vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       Value bitcast =
-          rewriter.create<vector::BitCastOp>(loc, bitcastResType, extract);
+          vector::BitCastOp::create(rewriter, loc, bitcastResType, extract);
       result =
-          rewriter.create<vector::InsertOp>(loc, bitcast, result, position);
+          vector::InsertOp::create(rewriter, loc, bitcast, result, position);
     }
 
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
index 11dcfe421e0c4..cb8e566869cfd 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorBroadcast.cpp
@@ -52,7 +52,7 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
 
     // Stretching scalar inside vector (e.g. vector<1xf32>) can use splat.
     if (srcRank <= 1 && dstRank == 1) {
-      Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource());
+      Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource());
       rewriter.replaceOpWithNewOp<vector::SplatOp>(op, dstType, ext);
       return success();
     }
@@ -70,10 +70,10 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
       // Duplication.
       VectorType resType = VectorType::Builder(dstType).dropDim(0);
       Value bcst =
-          rewriter.create<vector::BroadcastOp>(loc, resType, op.getSource());
-      Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
+          vector::BroadcastOp::create(rewriter, loc, resType, op.getSource());
+      Value result = ub::PoisonOp::create(rewriter, loc, dstType);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
       rewriter.replaceOp(op, result);
       return success();
     }
@@ -111,13 +111,13 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType =
         VectorType::get(dstType.getShape().drop_front(), eltType,
                         dstType.getScalableDims().drop_front());
-    Value result = rewriter.create<ub::PoisonOp>(loc, dstType);
+    Value result = ub::PoisonOp::create(rewriter, loc, dstType);
     if (m == 0) {
       // Stetch at start.
-      Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), 0);
-      Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
+      Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource(), 0);
+      Value bcst = vector::BroadcastOp::create(rewriter, loc, resType, ext);
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
     } else {
       // Stetch not at start.
       if (dstType.getScalableDims()[0]) {
@@ -125,9 +125,9 @@ class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
         return failure();
       }
       for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d) {
-        Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), d);
-        Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
-        result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
+        Value ext = vector::ExtractOp::create(rewriter, loc, op.getSource(), d);
+        Value bcst = vector::BroadcastOp::create(rewriter, loc, resType, ext);
+        result = vector::InsertOp::create(rewriter, loc, bcst, result, d);
       }
     }
     rewriter.replaceOp(op, result);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
index c6627b5ec0d77..b2486c5cbbcf8 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
@@ -90,17 +90,17 @@ static Value reshapeLoad(Location loc, Value val, VectorType type,
 
   // At extraction dimension?
   if (index == 0)
-    return rewriter.create<vector::ExtractOp>(loc, val, pos);
+    return vector::ExtractOp::create(rewriter, loc, val, pos);
 
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   VectorType resType = VectorType::Builder(type).dropDim(index);
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, resType, rewriter.getZeroAttr(resType));
   for (int64_t d = 0, e = resType.getDimSize(0); d < e; d++) {
-    Value ext = rewriter.create<vector::ExtractOp>(loc, val, d);
+    Value ext = vector::ExtractOp::create(rewriter, loc, val, d);
     Value load = reshapeLoad(loc, ext, vType, index - 1, pos, rewriter);
-    result = rewriter.create<vector::InsertOp>(loc, load, result, d);
+    result = vector::InsertOp::create(rewriter, loc, load, result, d);
   }
   return result;
 }
@@ -115,15 +115,15 @@ static Value reshapeStore(Location loc, Value val, Value result,
     return val;
   // At insertion dimension?
   if (index == 0)
-    return rewriter.create<vector::InsertOp>(loc, val, result, pos);
+    return vector::InsertOp::create(rewriter, loc, val, result, pos);
 
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   for (int64_t d = 0, e = type.getDimSize(0); d < e; d++) {
-    Value ext = rewriter.create<vector::ExtractOp>(loc, result, d);
-    Value ins = rewriter.create<vector::ExtractOp>(loc, val, d);
+    Value ext = vector::ExtractOp::create(rewriter, loc, result, d);
+    Value ins = vector::ExtractOp::create(rewriter, loc, val, d);
     Value sto = reshapeStore(loc, ins, ext, vType, index - 1, pos, rewriter);
-    result = rewriter.create<vector::InsertOp>(loc, sto, result, d);
+    result = vector::InsertOp::create(rewriter, loc, sto, result, d);
   }
   return result;
 }
@@ -141,7 +141,7 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
         kind == CombiningKind::MINIMUMF || kind == CombiningKind::MAXIMUMF)
       // Only valid for floating point types.
       return std::nullopt;
-    mul = rewriter.create<arith::MulIOp>(loc, x, y);
+    mul = arith::MulIOp::create(rewriter, loc, x, y);
   } else {
     // Float case.
     if (kind == CombiningKind::AND || kind == CombiningKind::MINUI ||
@@ -152,14 +152,14 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
       return std::nullopt;
     // Special case for fused multiply-add.
     if (acc && isa<VectorType>(acc.getType()) && kind == CombiningKind::ADD) {
-      Value fma = rewriter.create<vector::FMAOp>(loc, x, y, acc);
+      Value fma = vector::FMAOp::create(rewriter, loc, x, y, acc);
       if (mask)
         // The fma op doesn't need explicit masking. However, fma ops used in
         // reductions must preserve previous 'acc' values for masked-out lanes.
         fma = selectPassthru(rewriter, mask, fma, acc);
       return fma;
     }
-    mul = rewriter.create<arith::MulFOp>(loc, x, y);
+    mul = arith::MulFOp::create(rewriter, loc, x, y);
   }
 
   if (!acc)
@@ -195,8 +195,8 @@ static std::optional<unsigned> getDimPosition(AffineMap map, unsigned dim) {
 static Value createAdd(Location loc, Value x, Value y, bool isInt,
                        PatternRewriter &rewriter) {
   if (isInt)
-    return rewriter.create<arith::AddIOp>(loc, x, y);
-  return rewriter.create<arith::AddFOp>(loc, x, y);
+    return arith::AddIOp::create(rewriter, loc, x, y);
+  return arith::AddFOp::create(rewriter, loc, x, y);
 }
 
 /// Creates a MulIOp if `isInt` is true otherwise create an MulFOp using
@@ -204,8 +204,8 @@ static Value createAdd(Location loc, Value x, Value y, bool isInt,
 static Value createMul(Location loc, Value x, Value y, bool isInt,
                        PatternRewriter &rewriter) {
   if (isInt)
-    return rewriter.create<arith::MulIOp>(loc, x, y);
-  return rewriter.create<arith::MulFOp>(loc, x, y);
+    return arith::MulIOp::create(rewriter, loc, x, y);
+  return arith::MulFOp::create(rewriter, loc, x, y);
 }
 
 namespace {
@@ -411,7 +411,7 @@ struct UnrolledOuterProductGenerator
   Value t(Value v, ArrayRef<int64_t> perm = {1, 0}) {
     if (!v)
       return v;
-    return rewriter.create<vector::TransposeOp>(loc, v, perm);
+    return vector::TransposeOp::create(rewriter, loc, v, perm);
   }
 
   Value promote(Value v, Type dstElementType) {
@@ -425,8 +425,8 @@ struct UnrolledOuterProductGenerator
     if (vecType)
       promotedType = vecType.clone(promotedType);
     if (isa<FloatType>(dstElementType))
-      return rewriter.create<arith::ExtFOp>(loc, promotedType, v);
-    return rewriter.create<arith::ExtSIOp>(loc, promotedType, v);
+      return arith::ExtFOp::create(rewriter, loc, promotedType, v);
+    return arith::ExtSIOp::create(rewriter, loc, promotedType, v);
   }
 
   FailureOr<Value> outerProd(Value lhs, Value rhs, Value res,
@@ -438,16 +438,16 @@ struct UnrolledOuterProductGenerator
 
     Type resElementType = cast<VectorType>(res.getType()).getElementType();
     for (int64_t k = 0; k < reductionSize; ++k) {
-      Value extractA = rewriter.create<vector::ExtractOp>(loc, lhs, k);
-      Value extractB = rewriter.create<vector::ExtractOp>(loc, rhs, k);
+      Value extractA = vector::ExtractOp::create(rewriter, loc, lhs, k);
+      Value extractB = vector::ExtractOp::create(rewriter, loc, rhs, k);
       extractA = promote(extractA, resElementType);
       extractB = promote(extractB, resElementType);
       Value extractMask;
       if (maybeMask.has_value() && maybeMask.value())
         extractMask =
-            rewriter.create<vector::ExtractOp>(loc, maybeMask.value(), k);
+            vector::ExtractOp::create(rewriter, loc, maybeMask.value(), k);
 
-      Operation *outerProdOp = rewriter.create<vector::OuterProductOp>(
+      Operation *outerProdOp = vector::OuterProductOp::create(rewriter,
           loc, res.getType(), extractA, extractB, res, kind);
       res = maskOperation(rewriter, outerProdOp, extractMask)->getResult(0);
     }
@@ -698,28 +698,28 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
     // Two outer parallel, one inner reduction (matmat flavor).
     //
     if (maps == infer({{m, k}, {k, n}, {m, n}})) {
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{m, k}, {n, k}, {m, n}})) {
       // No need to permute anything.
     } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
-      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
     } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
       // This is the classical row-major matmul. Just permute the lhs.
       Value tmp = lhs;
-      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
       rhs = tmp;
     } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
       std::swap(lhs, rhs);
     } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
       Value tmp = lhs;
-      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-      rhs = rewriter.create<vector::TransposeOp>(loc, tmp, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, rhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, tmp, perm);
     } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
       Value tmp = rhs;
-      rhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
       lhs = tmp;
     } else {
       return failure();
@@ -732,12 +732,12 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
     if (maps == infer({{m, n}, {n}, {m}})) {
       // No need to permute anything.
     } else if (maps == infer({{n, m}, {n}, {m}})) {
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else if (maps == infer({{n}, {m, n}, {m}})) {
       std::swap(lhs, rhs);
     } else if (maps == infer({{n}, {n, m}, {m}})) {
       std::swap(lhs, rhs);
-      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = vector::TransposeOp::create(rewriter, loc, lhs, perm);
     } else {
       return failure();
     }
@@ -754,31 +754,31 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
   unsigned dstColumns = rank == 1 ? 1 : dstType.getShape()[1];
 
   // ExtractOp does not allow dynamic indexing, we must unroll explicitly.
-  Value res = rewriter.create<arith::ConstantOp>(loc, dstType,
+  Value res = arith::ConstantOp::create(rewriter, loc, dstType,
                                                  rewriter.getZeroAttr(dstType));
   bool isInt = isa<IntegerType>(dstType.getElementType());
   llvm::SmallVector<Value> extractedCols;
   extractedCols.reserve(dstColumns);
   for (unsigned r = 0; r < dstRows; ++r) {
-    Value rowLhs = rewriter.create<vector::ExtractOp>(op.getLoc(), lhs, r);
+    Value rowLhs = vector::ExtractOp::create(rewriter, op.getLoc(), lhs, r);
     for (unsigned c = 0; c < dstColumns; ++c) {
       // Extract each respective row and column of the LHS and RHS once to
       // avoid having duplicate SSA values pointing to the same rows/columns.
       if (r == 0) {
         Value colRhs =
             rank == 1 ? rhs
-                      : rewriter.create<vector::ExtractOp>(op.getLoc(), rhs, c);
+                      : vector::ExtractOp::create(rewriter, op.getLoc(), rhs, c);
         extractedCols.push_back(colRhs);
       }
       Value extractedColRhs = extractedCols[c];
       Value product =
           createMul(op.getLoc(), rowLhs, extractedColRhs, isInt, rewriter);
-      Value sum = rewriter.create<vector::ReductionOp>(
+      Value sum = vector::ReductionOp::create(rewriter,
           op.getLoc(), vector::CombiningKind::ADD, product);
 
       SmallVector<int64_t, 2> pos = rank == 1 ? SmallVector<int64_t, 2>{r}
                                               : SmallVector<int64_t, 2>{r, c};
-      res = rewriter.create<vector::InsertOp>(op.getLoc(), sum, res, pos);
+      res = vector::InsertOp::create(rewriter, op.getLoc(), sum, res, pos);
     }
   }
   if (auto acc = op.getAcc())
@@ -879,21 +879,21 @@ struct ContractOpToElementwise
       lhsDims.append(lhsShape.begin(), lhsShape.end());
       auto expandedType =
           VectorType::get(lhsDims, contractOp.getLhsType().getElementType());
-      newLhs = rewriter.create<vector::BroadcastOp>(loc, expandedType, newLhs);
+      newLhs = vector::BroadcastOp::create(rewriter, loc, expandedType, newLhs);
     }
     if (!rhsDims.empty()) {
       rhsDims.append(rhsShape.begin(), rhsShape.end());
       auto expandedType =
           VectorType::get(rhsDims, contractOp.getRhsType().getElementType());
-      newRhs = rewriter.create<vector::BroadcastOp>(loc, expandedType, newRhs);
+      newRhs = vector::BroadcastOp::create(rewriter, loc, expandedType, newRhs);
     }
     bool isInt = contractOp.getLhsType().getElementType().isIntOrIndex();
-    newLhs = rewriter.create<vector::TransposeOp>(loc, newLhs, lhsTranspose);
-    newRhs = rewriter.create<vector::TransposeOp>(loc, newRhs, rhsTranspose);
+    newLhs = vector::TransposeOp::create(rewriter, loc, newLhs, lhsTranspose);
+    newRhs = vector::TransposeOp::create(rewriter, loc, newRhs, rhsTranspose);
     SmallVector<int64_t> lhsOffsets(lhsReductionDims.size(), 0);
     SmallVector<int64_t> rhsOffsets(rhsReductionDims.size(), 0);
-    newLhs = rewriter.create<vector::ExtractOp>(loc, newLhs, lhsOffsets);
-    newRhs = rewriter.create<vector::ExtractOp>(loc, newRhs, rhsOffsets);
+    newLhs = vector::ExtractOp::create(rewriter, loc, newLhs, lhsOffsets);
+    newRhs = vector::ExtractOp::create(rewriter, loc, newRhs, rhsOffsets);
     std::optional<Value> result =
         createContractArithOp(loc, newLhs, newRhs, contractOp.getAcc(),
                               contractOp.getKind(), rewriter, isInt);
@@ -1097,7 +1097,7 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       rewriter.getArrayAttr(adjustIter(op.getIteratorTypes(), iterIndex));
   // Unroll into a series of lower dimensional vector.contract ops.
   Location loc = op.getLoc();
-  Value result = rewriter.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(rewriter,
       loc, resType, rewriter.getZeroAttr(resType));
 
   for (int64_t d = 0; d < dimSize; ++d) {
@@ -1110,7 +1110,7 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       lowMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *lowContract = rewriter.create<vector::ContractionOp>(
+    Operation *lowContract = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, acc, lowAffine, lowIter);
     lowContract = maskOperation(rewriter, lowContract, lowMask);
     result = reshapeStore(loc, lowContract->getResult(0), result, resType,
@@ -1161,8 +1161,8 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
 
     Value acc = op.getAcc();
     Operation *reductionOp =
-        acc ? rewriter.create<vector::ReductionOp>(loc, kind, m, acc)
-            : rewriter.create<vector::ReductionOp>(loc, kind, m);
+        acc ? vector::ReductionOp::create(rewriter, loc, kind, m, acc)
+            : vector::ReductionOp::create(rewriter, loc, kind, m);
     return maskOperation(rewriter, reductionOp, mask)->getResult(0);
   }
   // Construct new iterator types and affine map array attribute.
@@ -1186,7 +1186,7 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
       newMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *newContract = rewriter.create<vector::ContractionOp>(
+    Operation *newContract = vector::ContractionOp::create(rewriter,
         loc, lhs, rhs, result, lowAffine, lowIter);
     result = maskOperation(rewriter, newContract, newMask)->getResult(0);
   }
@@ -1240,7 +1240,7 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
 
     if (!rhsType) {
       // Special case: AXPY operation.
-      Value b = rewriter.create<vector::BroadcastOp>(loc, lhsType, op.getRhs());
+      Value b = vector::BroadcastOp::create(rewriter, loc, lhsType, op.getRhs());
       std::optional<Value> mult = createContractArithOp(
           loc, op.getLhs(), b, acc, kind, rewriter, isInt, mask);
       if (!mult.has_value())
@@ -1249,23 +1249,23 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
       return success();
     }
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
     for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
-      Value x = rewriter.create<vector::ExtractOp>(loc, op.getLhs(), d);
-      Value a = rewriter.create<vector::BroadcastOp>(loc, rhsType, x);
+      Value x = vector::ExtractOp::create(rewriter, loc, op.getLhs(), d);
+      Value a = vector::BroadcastOp::create(rewriter, loc, rhsType, x);
       Value r = nullptr;
       if (acc)
-        r = rewriter.create<vector::ExtractOp>(loc, acc, d);
+        r = vector::ExtractOp::create(rewriter, loc, acc, d);
       Value extrMask;
       if (mask)
-        extrMask = rewriter.create<vector::ExtractOp>(loc, mask, d);
+        extrMask = vector::ExtractOp::create(rewriter, loc, mask, d);
 
       std::optional<Value> m = createContractArithOp(
           loc, a, op.getRhs(), r, kind, rewriter, isInt, extrMask);
       if (!m.has_value())
         return failure();
-      result = rewriter.create<vector::InsertOp>(loc, *m, result, d);
+      result = vector::InsertOp::create(rewriter, loc, *m, result, d);
     }
 
     rewriter.replaceOp(rootOp, result);
@@ -1335,7 +1335,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   Value lhs = op.getLhs();
   auto lhsMap = op.getIndexingMapsArray()[0];
   if (lhsMap == AffineMap::get(3, 0, {k, m}, ctx))
-    lhs = rew.create<vector::TransposeOp>(loc, lhs, ArrayRef<int64_t>{1, 0});
+    lhs = vector::TransposeOp::create(rew, loc, lhs, ArrayRef<int64_t>{1, 0});
   else if (lhsMap != AffineMap::get(3, 0, {m, k}, ctx))
     return failure();
 
@@ -1343,7 +1343,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   Value rhs = op.getRhs();
   auto rhsMap = op.getIndexingMapsArray()[1];
   if (rhsMap == AffineMap::get(3, 0, {n, k}, ctx))
-    rhs = rew.create<vector::TransposeOp>(loc, rhs, ArrayRef<int64_t>{1, 0});
+    rhs = vector::TransposeOp::create(rew, loc, rhs, ArrayRef<int64_t>{1, 0});
   else if (rhsMap != AffineMap::get(3, 0, {k, n}, ctx))
     return failure();
 
@@ -1356,15 +1356,15 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
 
   Type flattenedLHSType =
       VectorType::get(lhsType.getNumElements(), lhsType.getElementType());
-  lhs = rew.create<vector::ShapeCastOp>(loc, flattenedLHSType, lhs);
+  lhs = vector::ShapeCastOp::create(rew, loc, flattenedLHSType, lhs);
 
   Type flattenedRHSType =
       VectorType::get(rhsType.getNumElements(), rhsType.getElementType());
-  rhs = rew.create<vector::ShapeCastOp>(loc, flattenedRHSType, rhs);
+  rhs = vector::ShapeCastOp::create(rew, loc, flattenedRHSType, rhs);
 
-  Value mul = rew.create<vector::MatmulOp>(loc, lhs, rhs, lhsRows, lhsColumns,
+  Value mul = vector::MatmulOp::create(rew, loc, lhs, rhs, lhsRows, lhsColumns,
                                            rhsColumns);
-  mul = rew.create<vector::ShapeCastOp>(
+  mul = vector::ShapeCastOp::create(rew,
       loc,
       VectorType::get({lhsRows, rhsColumns},
                       getElementTypeOrSelf(op.getAcc().getType())),
@@ -1373,15 +1373,15 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   // ACC must be C(m, n) or C(n, m).
   auto accMap = op.getIndexingMapsArray()[2];
   if (accMap == AffineMap::get(3, 0, {n, m}, ctx))
-    mul = rew.create<vector::TransposeOp>(loc, mul, ArrayRef<int64_t>{1, 0});
+    mul = vector::TransposeOp::create(rew, loc, mul, ArrayRef<int64_t>{1, 0});
   else if (accMap != AffineMap::get(3, 0, {m, n}, ctx))
     llvm_unreachable("invalid contraction semantics");
 
   Value res =
       isa<IntegerType>(elementType)
-          ? static_cast<Value>(rew.create<arith::AddIOp>(loc, op.getAcc(), mul))
+          ? static_cast<Value>(arith::AddIOp::create(rew, loc, op.getAcc(), mul))
           : static_cast<Value>(
-                rew.create<arith::AddFOp>(loc, op.getAcc(), mul));
+                arith::AddFOp::create(rew, loc, op.getAcc(), mul));
 
   return res;
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index 3000204c8ce17..c586059ca55bc 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -75,7 +75,7 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
     Value maskVec = op.getMask();
     Value passThruVec = op.getPassThru();
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultTy, rewriter.getZeroAttr(resultTy));
 
     VectorType subTy = VectorType::Builder(resultTy).dropDim(0);
@@ -84,16 +84,16 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
       int64_t thisIdx[1] = {i};
 
       Value indexSubVec =
-          rewriter.create<vector::ExtractOp>(loc, indexVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
       Value maskSubVec =
-          rewriter.create<vector::ExtractOp>(loc, maskVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, maskVec, thisIdx);
       Value passThruSubVec =
-          rewriter.create<vector::ExtractOp>(loc, passThruVec, thisIdx);
-      Value subGather = rewriter.create<vector::GatherOp>(
+          vector::ExtractOp::create(rewriter, loc, passThruVec, thisIdx);
+      Value subGather = vector::GatherOp::create(rewriter,
           loc, subTy, op.getBase(), op.getIndices(), indexSubVec, maskSubVec,
           passThruSubVec);
       result =
-          rewriter.create<vector::InsertOp>(loc, subGather, result, thisIdx);
+          vector::InsertOp::create(rewriter, loc, subGather, result, thisIdx);
     }
 
     rewriter.replaceOp(op, result);
@@ -159,22 +159,22 @@ struct RemoveStrideFromGatherSource : OpRewritePattern<vector::GatherOp> {
 
     // 1. Collapse the input memref so that it's "flat".
     SmallVector<ReassociationIndices> reassoc = {{0, 1}};
-    Value collapsed = rewriter.create<memref::CollapseShapeOp>(
+    Value collapsed = memref::CollapseShapeOp::create(rewriter,
         op.getLoc(), subview.getSource(), reassoc);
 
     // 2. Generate new gather indices that will model the
     // strided access.
     IntegerAttr stride = rewriter.getIndexAttr(srcTrailingDim);
     VectorType vType = op.getIndexVec().getType();
-    Value mulCst = rewriter.create<arith::ConstantOp>(
+    Value mulCst = arith::ConstantOp::create(rewriter,
         op.getLoc(), vType, DenseElementsAttr::get(vType, stride));
 
     Value newIdxs =
-        rewriter.create<arith::MulIOp>(op.getLoc(), op.getIndexVec(), mulCst);
+        arith::MulIOp::create(rewriter, op.getLoc(), op.getIndexVec(), mulCst);
 
     // 3. Create an updated gather op with the collapsed input memref and the
     // updated indices.
-    Value newGather = rewriter.create<vector::GatherOp>(
+    Value newGather = vector::GatherOp::create(rewriter,
         op.getLoc(), op.getResult().getType(), collapsed, op.getIndices(),
         newIdxs, op.getMask(), op.getPassThru());
     rewriter.replaceOp(op, newGather);
@@ -229,8 +229,8 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
     for (int64_t i = 0, e = resultTy.getNumElements(); i < e; ++i) {
       int64_t thisIdx[1] = {i};
       Value condition =
-          rewriter.create<vector::ExtractOp>(loc, condMask, thisIdx);
-      Value index = rewriter.create<vector::ExtractOp>(loc, indexVec, thisIdx);
+          vector::ExtractOp::create(rewriter, loc, condMask, thisIdx);
+      Value index = vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
       baseOffsets.back() =
           rewriter.createOrFold<arith::AddIOp>(loc, lastBaseOffset, index);
 
@@ -240,19 +240,19 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
           // `vector.load` does not support scalar result; emit a vector load
           // and extract the single result instead.
           Value load =
-              b.create<vector::LoadOp>(loc, elemVecTy, base, baseOffsets);
+              vector::LoadOp::create(b, loc, elemVecTy, base, baseOffsets);
           int64_t zeroIdx[1] = {0};
-          extracted = b.create<vector::ExtractOp>(loc, load, zeroIdx);
+          extracted = vector::ExtractOp::create(b, loc, load, zeroIdx);
         } else {
-          extracted = b.create<tensor::ExtractOp>(loc, base, baseOffsets);
+          extracted = tensor::ExtractOp::create(b, loc, base, baseOffsets);
         }
 
         Value newResult =
-            b.create<vector::InsertOp>(loc, extracted, result, thisIdx);
-        b.create<scf::YieldOp>(loc, newResult);
+            vector::InsertOp::create(b, loc, extracted, result, thisIdx);
+        scf::YieldOp::create(b, loc, newResult);
       };
       auto passThruBuilder = [result](OpBuilder &b, Location loc) {
-        b.create<scf::YieldOp>(loc, result);
+        scf::YieldOp::create(b, loc, result);
       };
 
       result =
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index cab0f213b14a9..b0affbb699d9e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -60,14 +60,14 @@ class UnrollInterleaveOp final : public OpRewritePattern<vector::InterleaveOp> {
       return failure();
 
     auto loc = op.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
     for (auto position : *unrollIterator) {
-      Value extractLhs = rewriter.create<ExtractOp>(loc, op.getLhs(), position);
-      Value extractRhs = rewriter.create<ExtractOp>(loc, op.getRhs(), position);
+      Value extractLhs = ExtractOp::create(rewriter, loc, op.getLhs(), position);
+      Value extractRhs = ExtractOp::create(rewriter, loc, op.getRhs(), position);
       Value interleave =
-          rewriter.create<InterleaveOp>(loc, extractLhs, extractRhs);
-      result = rewriter.create<InsertOp>(loc, interleave, result, position);
+          InterleaveOp::create(rewriter, loc, extractLhs, extractRhs);
+      result = InsertOp::create(rewriter, loc, interleave, result, position);
     }
 
     rewriter.replaceOp(op, result);
@@ -123,19 +123,19 @@ class UnrollDeinterleaveOp final
       return failure();
 
     auto loc = op.getLoc();
-    Value emptyResult = rewriter.create<arith::ConstantOp>(
+    Value emptyResult = arith::ConstantOp::create(rewriter,
         loc, resultType, rewriter.getZeroAttr(resultType));
     Value evenResult = emptyResult;
     Value oddResult = emptyResult;
 
     for (auto position : *unrollIterator) {
       auto extractSrc =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+          vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       auto deinterleave =
-          rewriter.create<vector::DeinterleaveOp>(loc, extractSrc);
-      evenResult = rewriter.create<vector::InsertOp>(
+          vector::DeinterleaveOp::create(rewriter, loc, extractSrc);
+      evenResult = vector::InsertOp::create(rewriter,
           loc, deinterleave.getRes1(), evenResult, position);
-      oddResult = rewriter.create<vector::InsertOp>(loc, deinterleave.getRes2(),
+      oddResult = vector::InsertOp::create(rewriter, loc, deinterleave.getRes2(),
                                                     oddResult, position);
     }
     rewriter.replaceOp(op, ValueRange{evenResult, oddResult});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
index ba21092d2af3c..edd840d8fbc7b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
@@ -67,19 +67,19 @@ class CreateMaskOpLowering : public OpRewritePattern<vector::CreateMaskOp> {
     Value idx = op.getOperand(0);
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = rewriter.create<vector::CreateMaskOp>(
+    Value trueVal = vector::CreateMaskOp::create(rewriter,
         loc, lowType, op.getOperands().drop_front());
-    Value falseVal = rewriter.create<arith::ConstantOp>(
+    Value falseVal = arith::ConstantOp::create(rewriter,
         loc, lowType, rewriter.getZeroAttr(lowType));
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstType, rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < dim; d++) {
       Value bnd =
-          rewriter.create<arith::ConstantOp>(loc, rewriter.getIndexAttr(d));
-      Value val = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt,
+          arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(d));
+      Value val = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
                                                  bnd, idx);
-      Value sel = rewriter.create<arith::SelectOp>(loc, val, trueVal, falseVal);
-      result = rewriter.create<vector::InsertOp>(loc, sel, result, d);
+      Value sel = arith::SelectOp::create(rewriter, loc, val, trueVal, falseVal);
+      result = vector::InsertOp::create(rewriter, loc, sel, result, d);
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -146,12 +146,12 @@ class ConstantMaskOpLowering : public OpRewritePattern<vector::ConstantMaskOp> {
           op, "Cannot unroll leading scalable dim in dstType");
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = rewriter.create<vector::ConstantMaskOp>(
+    Value trueVal = vector::ConstantMaskOp::create(rewriter,
         loc, lowType, dimSizes.drop_front());
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstType, rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < trueDimSize; d++)
-      result = rewriter.create<vector::InsertOp>(loc, trueVal, result, d);
+      result = vector::InsertOp::create(rewriter, loc, trueVal, result, d);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -261,7 +261,7 @@ struct MaskedGatherOpPattern : public MaskOpRewritePattern<GatherOp> {
                             PatternRewriter &rewriter) const override {
     Value passthru = maskingOp.hasPassthru()
                          ? maskingOp.getPassthru()
-                         : rewriter.create<arith::ConstantOp>(
+                         : arith::ConstantOp::create(rewriter,
                                gatherOp.getLoc(),
                                rewriter.getZeroAttr(gatherOp.getVectorType()));
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
index ce524b259d8d4..c5631ba00bbc6 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
@@ -103,12 +103,12 @@ class InnerOuterDimReductionConversion
     // If masked, transpose the original mask.
     Value transposedMask;
     if (maskableOp.isMasked()) {
-      transposedMask = rewriter.create<vector::TransposeOp>(
+      transposedMask = vector::TransposeOp::create(rewriter,
           loc, maskableOp.getMaskingOp().getMask(), indices);
     }
 
     // Transpose reduction source.
-    auto transposeOp = rewriter.create<vector::TransposeOp>(loc, src, indices);
+    auto transposeOp = vector::TransposeOp::create(rewriter, loc, src, indices);
     SmallVector<bool> reductionMask(srcRank, false);
     for (int i = 0; i < reductionSize; ++i) {
       if (useInnerDimsForReduction)
@@ -117,7 +117,7 @@ class InnerOuterDimReductionConversion
         reductionMask[i] = true;
     }
 
-    Operation *newMultiRedOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newMultiRedOp = vector::MultiDimReductionOp::create(rewriter,
         multiReductionOp.getLoc(), transposeOp.getResult(),
         multiReductionOp.getAcc(), reductionMask, multiReductionOp.getKind());
     newMultiRedOp =
@@ -256,13 +256,13 @@ class ReduceMultiDimReductionRank
           vectorShape,
           llvm::cast<VectorType>(vectorMask.getType()).getElementType());
       newVectorMask =
-          rewriter.create<vector::ShapeCastOp>(loc, maskCastedType, vectorMask);
+          vector::ShapeCastOp::create(rewriter, loc, maskCastedType, vectorMask);
     }
 
     auto castedType = VectorType::get(
         vectorShape, multiReductionOp.getSourceVectorType().getElementType(),
         scalableDims);
-    Value cast = rewriter.create<vector::ShapeCastOp>(
+    Value cast = vector::ShapeCastOp::create(rewriter,
         loc, castedType, multiReductionOp.getSource());
 
     Value acc = multiReductionOp.getAcc();
@@ -271,11 +271,11 @@ class ReduceMultiDimReductionRank
           {flattenedParallelDim},
           multiReductionOp.getSourceVectorType().getElementType(),
           /*scalableDims=*/{isParallelDimScalable});
-      acc = rewriter.create<vector::ShapeCastOp>(loc, accType, acc);
+      acc = vector::ShapeCastOp::create(rewriter, loc, accType, acc);
     }
     // 6. Creates the flattened form of vector.multi_reduction with inner/outer
     // most dim as reduction.
-    Operation *newMultiDimRedOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(rewriter,
         loc, cast, acc, mask, multiReductionOp.getKind());
     newMultiDimRedOp =
         mlir::vector::maskOperation(rewriter, newMultiDimRedOp, newVectorMask);
@@ -339,11 +339,11 @@ struct TwoDimMultiReductionToElementWise
 
     Value result = multiReductionOp.getAcc();
     for (int64_t i = 0; i < srcShape[0]; i++) {
-      auto operand = rewriter.create<vector::ExtractOp>(
+      auto operand = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getSource(), i);
       Value extractMask = nullptr;
       if (mask) {
-        extractMask = rewriter.create<vector::ExtractOp>(loc, mask, i);
+        extractMask = vector::ExtractOp::create(rewriter, loc, mask, i);
       }
       result =
           makeArithReduction(rewriter, loc, multiReductionOp.getKind(), operand,
@@ -383,27 +383,27 @@ struct TwoDimMultiReductionToReduction
     }
 
     auto loc = multiReductionOp.getLoc();
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, multiReductionOp.getDestType(),
         rewriter.getZeroAttr(multiReductionOp.getDestType()));
     int outerDim = multiReductionOp.getSourceVectorType().getShape()[0];
 
     for (int i = 0; i < outerDim; ++i) {
-      auto v = rewriter.create<vector::ExtractOp>(
+      auto v = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
-      auto acc = rewriter.create<vector::ExtractOp>(
+      auto acc = vector::ExtractOp::create(rewriter,
           loc, multiReductionOp.getAcc(), ArrayRef<int64_t>{i});
-      Operation *reductionOp = rewriter.create<vector::ReductionOp>(
+      Operation *reductionOp = vector::ReductionOp::create(rewriter,
           loc, multiReductionOp.getKind(), v, acc);
 
       // If masked, slice the mask and mask the new reduction operation.
       if (maskableOp.isMasked()) {
-        Value mask = rewriter.create<vector::ExtractOp>(
+        Value mask = vector::ExtractOp::create(rewriter,
             loc, maskableOp.getMaskingOp().getMask(), ArrayRef<int64_t>{i});
         reductionOp = mlir::vector::maskOperation(rewriter, reductionOp, mask);
       }
 
-      result = rewriter.create<vector::InsertOp>(loc, reductionOp->getResult(0),
+      result = vector::InsertOp::create(rewriter, loc, reductionOp->getResult(0),
                                                  result, i);
     }
 
@@ -459,9 +459,9 @@ struct OneDimMultiReductionToTwoDim
     SmallVector<bool, 2> reductionMask{false, true};
 
     /// vector.extract(vector.multi_reduce(vector.shape_cast(v, 1xk)), 0)
-    Value cast = rewriter.create<vector::ShapeCastOp>(
+    Value cast = vector::ShapeCastOp::create(rewriter,
         loc, castedType, multiReductionOp.getSource());
-    Value castAcc = rewriter.create<vector::BroadcastOp>(
+    Value castAcc = vector::BroadcastOp::create(rewriter,
         loc, accType, multiReductionOp.getAcc());
     Value castMask;
     if (maskableOp.isMasked()) {
@@ -470,10 +470,10 @@ struct OneDimMultiReductionToTwoDim
           ArrayRef<int64_t>{1, maskType.getShape().back()},
           maskType.getElementType(),
           ArrayRef<bool>{false, maskType.getScalableDims().back()});
-      castMask = rewriter.create<vector::BroadcastOp>(loc, castMaskType, mask);
+      castMask = vector::BroadcastOp::create(rewriter, loc, castMaskType, mask);
     }
 
-    Operation *newOp = rewriter.create<vector::MultiDimReductionOp>(
+    Operation *newOp = vector::MultiDimReductionOp::create(rewriter,
         loc, cast, castAcc, reductionMask, multiReductionOp.getKind());
     newOp = vector::maskOperation(rewriter, newOp, castMask);
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
index a1f67bd0e9ed3..a6985b96b59df 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
@@ -122,7 +122,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
       return failure();
 
     VectorType resType = VectorType::get(destShape, elType);
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
     int64_t reductionDim = scanOp.getReductionDim();
     bool inclusive = scanOp.getInclusive();
@@ -144,7 +144,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
     for (int i = 0; i < destShape[reductionDim]; i++) {
       offsets[reductionDim] = i;
       ArrayAttr scanOffsets = rewriter.getI64ArrayAttr(offsets);
-      Value input = rewriter.create<vector::ExtractStridedSliceOp>(
+      Value input = vector::ExtractStridedSliceOp::create(rewriter,
           loc, reductionType, scanOp.getSource(), scanOffsets, scanSizes,
           scanStrides);
       Value output;
@@ -154,10 +154,10 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         } else {
           if (initialValueRank == 0) {
             // ShapeCastOp cannot handle 0-D vectors
-            output = rewriter.create<vector::BroadcastOp>(
+            output = vector::BroadcastOp::create(rewriter,
                 loc, input.getType(), scanOp.getInitialValue());
           } else {
-            output = rewriter.create<vector::ShapeCastOp>(
+            output = vector::ShapeCastOp::create(rewriter,
                 loc, input.getType(), scanOp.getInitialValue());
           }
         }
@@ -166,7 +166,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         output = vector::makeArithReduction(rewriter, loc, scanOp.getKind(),
                                             lastOutput, y);
       }
-      result = rewriter.create<vector::InsertStridedSliceOp>(
+      result = vector::InsertStridedSliceOp::create(rewriter,
           loc, output, result, offsets, strides);
       lastOutput = output;
       lastInput = input;
@@ -174,11 +174,11 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
 
     Value reduction;
     if (initialValueRank == 0) {
-      Value v = rewriter.create<vector::ExtractOp>(loc, lastOutput, 0);
+      Value v = vector::ExtractOp::create(rewriter, loc, lastOutput, 0);
       reduction =
-          rewriter.create<vector::BroadcastOp>(loc, initialValueType, v);
+          vector::BroadcastOp::create(rewriter, loc, initialValueType, v);
     } else {
-      reduction = rewriter.create<vector::ShapeCastOp>(loc, initialValueType,
+      reduction = vector::ShapeCastOp::create(rewriter, loc, initialValueType,
                                                        lastOutput);
     }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
index 39c16fab21c4e..69d3666a13879 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
@@ -137,10 +137,10 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     const int64_t resultLeading = delta > 0 ? 0 : -delta;
 
     const Value source = shapeCast.getSource();
-    const Value poison = rewriter.create<ub::PoisonOp>(loc, resultType);
-    const Value extracted = rewriter.create<vector::ExtractOp>(
+    const Value poison = ub::PoisonOp::create(rewriter, loc, resultType);
+    const Value extracted = vector::ExtractOp::create(rewriter,
         loc, source, SmallVector<int64_t>(sourceLeading, 0));
-    const Value result = rewriter.create<vector::InsertOp>(
+    const Value result = vector::InsertOp::create(rewriter,
         loc, extracted, poison, SmallVector<int64_t>(resultLeading, 0));
 
     rewriter.replaceOp(shapeCast, result);
@@ -171,13 +171,13 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 
     SmallVector<int64_t> extractIndex(sourceDim, 0);
     SmallVector<int64_t> insertIndex(resultDim, 0);
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
 
     for (int i = 0; i < nSlices; ++i) {
       Value extracted =
-          rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+          vector::ExtractOp::create(rewriter, loc, source, extractIndex);
 
-      result = rewriter.create<vector::InsertOp>(loc, extracted, result,
+      result = vector::InsertOp::create(rewriter, loc, extracted, result,
                                                  insertIndex);
 
       inplaceAdd(1, sourceShape.take_front(sourceDim), extractIndex);
@@ -276,9 +276,9 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
     Value extracted = {};
     Value extractedStrided = {};
     Value insertedSlice = {};
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultType);
     const Value partResult =
-        rewriter.create<ub::PoisonOp>(loc, insertStridedType);
+        ub::PoisonOp::create(rewriter, loc, insertStridedType);
 
     for (size_t i = 0; i < nAtomicSlices; ++i) {
 
@@ -288,14 +288,14 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
       // vector.extract
       if (extractStridedPhase == 0) {
         extracted =
-            rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+            vector::ExtractOp::create(rewriter, loc, source, extractIndex);
         inplaceAdd(1, sourceShape.take_front(sourceSuffixStartDim),
                    extractIndex);
       }
 
       // vector.extract_strided_slice
       extractOffsets[0] = extractStridedPhase * greatestCommonDivisor;
-      extractedStrided = rewriter.create<vector::ExtractStridedSliceOp>(
+      extractedStrided = vector::ExtractStridedSliceOp::create(rewriter,
           loc, extracted, extractOffsets, atomicShape, sizes);
 
       // vector.insert_strided_slice
@@ -303,12 +303,12 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
         insertedSlice = partResult;
       }
       insertOffsets[0] = insertStridedPhase * greatestCommonDivisor;
-      insertedSlice = rewriter.create<vector::InsertStridedSliceOp>(
+      insertedSlice = vector::InsertStridedSliceOp::create(rewriter,
           loc, extractedStrided, insertedSlice, insertOffsets, sizes);
 
       // vector.insert
       if (insertStridedPhase + 1 == insertPeriod) {
-        result = rewriter.create<vector::InsertOp>(loc, insertedSlice, result,
+        result = vector::InsertOp::create(rewriter, loc, insertedSlice, result,
                                                    insertIndex);
         inplaceAdd(1, resultType.getShape().take_front(resultSuffixStartDim),
                    insertIndex);
@@ -394,7 +394,7 @@ class ScalableShapeCastOpRewritePattern
     auto extractionVectorType = VectorType::get(
         {minExtractionSize}, sourceVectorType.getElementType(), {true});
 
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resultVectorType);
     SmallVector<int64_t> srcIdx(srcRank, 0);
     SmallVector<int64_t> resIdx(resRank, 0);
 
@@ -406,7 +406,7 @@ class ScalableShapeCastOpRewritePattern
       // 1. Extract a scalable subvector from the source vector.
       if (!currentSourceScalableVector) {
         if (srcRank != 1) {
-          currentSourceScalableVector = rewriter.create<vector::ExtractOp>(
+          currentSourceScalableVector = vector::ExtractOp::create(rewriter,
               loc, op.getSource(), llvm::ArrayRef(srcIdx).drop_back());
         } else {
           currentSourceScalableVector = op.getSource();
@@ -414,7 +414,7 @@ class ScalableShapeCastOpRewritePattern
       }
       Value sourceSubVector = currentSourceScalableVector;
       if (minExtractionSize < minSourceTrailingSize) {
-        sourceSubVector = rewriter.create<vector::ScalableExtractOp>(
+        sourceSubVector = vector::ScalableExtractOp::create(rewriter,
             loc, extractionVectorType, sourceSubVector, srcIdx.back());
       }
 
@@ -423,14 +423,14 @@ class ScalableShapeCastOpRewritePattern
         if (minExtractionSize == minResultTrailingSize) {
           currentResultScalableVector = sourceSubVector;
         } else if (resRank != 1) {
-          currentResultScalableVector = rewriter.create<vector::ExtractOp>(
+          currentResultScalableVector = vector::ExtractOp::create(rewriter,
               loc, result, llvm::ArrayRef(resIdx).drop_back());
         } else {
           currentResultScalableVector = result;
         }
       }
       if (minExtractionSize < minResultTrailingSize) {
-        currentResultScalableVector = rewriter.create<vector::ScalableInsertOp>(
+        currentResultScalableVector = vector::ScalableInsertOp::create(rewriter,
             loc, sourceSubVector, currentResultScalableVector, resIdx.back());
       }
 
@@ -439,7 +439,7 @@ class ScalableShapeCastOpRewritePattern
           currentResultScalableVector != result) {
         // Finished row of result. Insert complete scalable vector into result
         // (n-D) vector.
-        result = rewriter.create<vector::InsertOp>(
+        result = vector::InsertOp::create(rewriter,
             loc, currentResultScalableVector, result,
             llvm::ArrayRef(resIdx).drop_back());
         currentResultScalableVector = {};
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
index 5b81d0d33d484..fe02cd5656571 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
@@ -47,7 +47,7 @@ static Value extendVectorRank(OpBuilder &builder, Location loc, Value vec,
                          originalVecType.getScalableDims().end());
   VectorType newVecType = VectorType::get(
       newShape, originalVecType.getElementType(), newScalableDims);
-  return builder.create<vector::BroadcastOp>(loc, newVecType, vec);
+  return vector::BroadcastOp::create(builder, loc, newVecType, vec);
 }
 
 /// Extend the rank of a vector Value by `addedRanks` by adding inner unit
@@ -62,7 +62,7 @@ static Value extendMaskRank(OpBuilder &builder, Location loc, Value vec,
     permutation.push_back(i);
   for (int64_t i = 0; i < addedRank; ++i)
     permutation.push_back(i);
-  return builder.create<vector::TransposeOp>(loc, broadcasted, permutation);
+  return vector::TransposeOp::create(builder, loc, broadcasted, permutation);
 }
 
 //===----------------------------------------------------------------------===//
@@ -138,7 +138,7 @@ struct TransferReadPermutationLowering
     // Generate new transfer_read operation.
     VectorType newReadType = VectorType::get(
         newVectorShape, op.getVectorType().getElementType(), newScalableDims);
-    Value newRead = rewriter.create<vector::TransferReadOp>(
+    Value newRead = vector::TransferReadOp::create(rewriter,
         op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
@@ -209,11 +209,11 @@ struct TransferWritePermutationLowering
         inverseTransposeInBoundsAttr(rewriter, op.getInBounds(), permutation);
 
     // Generate new transfer_write operation.
-    Value newVec = rewriter.create<vector::TransposeOp>(
+    Value newVec = vector::TransposeOp::create(rewriter,
         op.getLoc(), op.getVector(), indices);
     auto newMap = AffineMap::getMinorIdentityMap(
         map.getNumDims(), map.getNumResults(), rewriter.getContext());
-    auto newWrite = rewriter.create<vector::TransferWriteOp>(
+    auto newWrite = vector::TransferWriteOp::create(rewriter,
         op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getMask(), newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
@@ -299,7 +299,7 @@ struct TransferWriteNonPermutationLowering
       newInBoundsValues.push_back(op.isDimInBounds(i));
     }
     ArrayAttr newInBoundsAttr = rewriter.getBoolArrayAttr(newInBoundsValues);
-    auto newWrite = rewriter.create<vector::TransferWriteOp>(
+    auto newWrite = vector::TransferWriteOp::create(rewriter,
         op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), newMask, newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
@@ -370,7 +370,7 @@ struct TransferOpReduceRank
             ? rewriter.getArrayAttr(
                   op.getInBoundsAttr().getValue().take_back(reducedShapeRank))
             : ArrayAttr();
-    Value newRead = rewriter.create<vector::TransferReadOp>(
+    Value newRead = vector::TransferReadOp::create(rewriter,
         op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
@@ -471,20 +471,20 @@ struct TransferReadToVectorLoadLowering
             read, "vector type is not rank 1, can't create masked load, needs "
                   "VectorToSCF");
 
-      Value fill = rewriter.create<vector::SplatOp>(
+      Value fill = vector::SplatOp::create(rewriter,
           read.getLoc(), unbroadcastedVectorType, read.getPadding());
-      res = rewriter.create<vector::MaskedLoadOp>(
+      res = vector::MaskedLoadOp::create(rewriter,
           read.getLoc(), unbroadcastedVectorType, read.getBase(),
           read.getIndices(), read.getMask(), fill);
     } else {
-      res = rewriter.create<vector::LoadOp>(read.getLoc(),
+      res = vector::LoadOp::create(rewriter, read.getLoc(),
                                             unbroadcastedVectorType,
                                             read.getBase(), read.getIndices());
     }
 
     // Insert a broadcasting op if required.
     if (!broadcastedDims.empty())
-      res = rewriter.create<vector::BroadcastOp>(
+      res = vector::BroadcastOp::create(rewriter,
           read.getLoc(), read.getVectorType(), res->getResult(0));
     return res->getResult(0);
   }
@@ -569,11 +569,11 @@ struct TransferWriteToVectorStoreLowering
                    << write;
             });
 
-      rewriter.create<vector::MaskedStoreOp>(
+      vector::MaskedStoreOp::create(rewriter,
           write.getLoc(), write.getBase(), write.getIndices(), write.getMask(),
           write.getVector());
     } else {
-      rewriter.create<vector::StoreOp>(write.getLoc(), write.getVector(),
+      vector::StoreOp::create(rewriter, write.getLoc(), write.getVector(),
                                        write.getBase(), write.getIndices());
     }
     // There's no return value for StoreOps. Use Value() to signal success to
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index 732e316c93381..e9591bfcf6feb 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -80,7 +80,7 @@ getUnpackShufflePermFor128Lane(ArrayRef<int64_t> vals, int numBits) {
 static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({0, 1, numElem, numElem + 1}, numBits));
 }
@@ -94,7 +94,7 @@ static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({2, 3, numElem + 2, numElem + 3},
                                      numBits));
@@ -109,7 +109,7 @@ static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  auto shuffle = b.create<vector::ShuffleOp>(
+  auto shuffle = vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({0, numElem, 1, numElem + 1}, numBits));
   return shuffle;
@@ -124,7 +124,7 @@ static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2,
       getUnpackShufflePermFor128Lane({2, numElem + 2, 3, numElem + 3},
                                      numBits));
@@ -181,7 +181,7 @@ static Value create4x128BitSuffle(ImplicitLocOpBuilder &b, Value v1, Value v2,
   appendToMask(0, b23);
   appendToMask(16, b45);
   appendToMask(16, b67);
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// Lowers the value to a vector.shuffle op. The `source` is expected to be a
@@ -192,7 +192,7 @@ static Value transposeToShuffle1D(OpBuilder &b, Value source, int m, int n) {
   for (int64_t j = 0; j < n; ++j)
     for (int64_t i = 0; i < m; ++i)
       mask.push_back(i * n + j);
-  return b.create<vector::ShuffleOp>(source.getLoc(), source, source, mask);
+  return vector::ShuffleOp::create(b, source.getLoc(), source, source, mask);
 }
 
 /// Lowers the value to a sequence of vector.shuffle ops. The `source` is
@@ -284,9 +284,9 @@ static Value transposeToShuffle16x16(OpBuilder &builder, Value source, int m,
 
   auto reshInputType = VectorType::get(
       {m, n}, cast<VectorType>(source.getType()).getElementType());
-  Value res = b.create<ub::PoisonOp>(reshInputType);
+  Value res = ub::PoisonOp::create(b, reshInputType);
   for (int64_t i = 0; i < m; ++i)
-    res = b.create<vector::InsertOp>(vs[i], res, i);
+    res = vector::InsertOp::create(b, vs[i], res, i);
   return res;
 }
 
@@ -335,10 +335,10 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
       Type flattenedType =
           VectorType::get(resType.getNumElements(), resType.getElementType());
       auto matrix =
-          rewriter.create<vector::ShapeCastOp>(loc, flattenedType, input);
+          vector::ShapeCastOp::create(rewriter, loc, flattenedType, input);
       auto rows = rewriter.getI32IntegerAttr(resType.getShape()[0]);
       auto columns = rewriter.getI32IntegerAttr(resType.getShape()[1]);
-      Value trans = rewriter.create<vector::FlatTransposeOp>(
+      Value trans = vector::FlatTransposeOp::create(rewriter,
           loc, flattenedType, matrix, rows, columns);
       rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, trans);
       return success();
@@ -359,7 +359,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
     // of the leftmost transposed dimensions. We traverse every transpose
     // element using a linearized index that we delinearize to generate the
     // appropriate indices for the extract/insert operations.
-    Value result = rewriter.create<ub::PoisonOp>(loc, resType);
+    Value result = ub::PoisonOp::create(rewriter, loc, resType);
     int64_t numTransposedElements = ShapedType::getNumElements(prunedInShape);
 
     for (int64_t linearIdx = 0; linearIdx < numTransposedElements;
@@ -482,14 +482,14 @@ class TransposeOp2DToShuffleLowering
     Location loc = op.getLoc();
     auto flattenedType = VectorType::get({n * m}, srcType.getElementType());
     auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
-    auto reshInput = rewriter.create<vector::ShapeCastOp>(loc, flattenedType,
+    auto reshInput = vector::ShapeCastOp::create(rewriter, loc, flattenedType,
                                                           op.getVector());
 
     Value res;
     if (vectorTransposeLowering == VectorTransposeLowering::Shuffle16x16 &&
         m == 16 && n == 16) {
       reshInput =
-          rewriter.create<vector::ShapeCastOp>(loc, reshInputType, reshInput);
+          vector::ShapeCastOp::create(rewriter, loc, reshInputType, reshInput);
       res = transposeToShuffle16x16(rewriter, reshInput, m, n);
     } else {
       // Fallback to shuffle on 1D approach.
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index c8566b1ff83ef..2683a85ecdaeb 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -114,7 +114,7 @@ struct DistributedLoadStoreHelper {
            "preregistered sequential value.");
     // Scalar case can directly use memref.store.
     if (!isa<VectorType>(val.getType()))
-      return b.create<memref::StoreOp>(loc, val, buffer, zero);
+      return memref::StoreOp::create(b, loc, val, buffer, zero);
 
     // Vector case must use vector::TransferWriteOp which will later lower to
     //   vector.store of memref.store depending on further lowerings.
@@ -127,7 +127,7 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return b.create<vector::TransferWriteOp>(
+    return vector::TransferWriteOp::create(b,
         loc, val, buffer, indices,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
@@ -156,7 +156,7 @@ struct DistributedLoadStoreHelper {
 
     // Scalar case can directly use memref.store.
     if (!isa<VectorType>(type))
-      return b.create<memref::LoadOp>(loc, buffer, zero);
+      return memref::LoadOp::create(b, loc, buffer, zero);
 
     // Other cases must be vector atm.
     // Vector case must use vector::TransferReadOp which will later lower to
@@ -172,7 +172,7 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return b.create<vector::TransferReadOp>(
+    return vector::TransferReadOp::create(b,
         loc, cast<VectorType>(type), buffer, indices, /*padding=*/std::nullopt,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
@@ -243,10 +243,10 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
     rewriter.setInsertionPoint(warpOp);
 
     // Step 1: Create scf.if op.
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    Value isLane0 = rewriter.create<arith::CmpIOp>(
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    Value isLane0 = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, warpOp.getLaneid(), c0);
-    auto ifOp = rewriter.create<scf::IfOp>(loc, isLane0,
+    auto ifOp = scf::IfOp::create(rewriter, loc, isLane0,
                                            /*withElseRegion=*/false);
     rewriter.eraseOp(ifOp.thenBlock()->getTerminator());
 
@@ -325,7 +325,7 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
     // Step 7. Delete terminator and add empty scf.yield.
     rewriter.eraseOp(yieldOp);
     rewriter.setInsertionPointToEnd(ifOp.thenBlock());
-    rewriter.create<scf::YieldOp>(yieldLoc);
+    scf::YieldOp::create(rewriter, yieldLoc);
 
     // Compute replacements for WarpOp results.
     rewriter.replaceOp(warpOp, replacements);
@@ -512,7 +512,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
 
     // Create a second warp op that contains only writeOp.
-    auto secondWarpOp = rewriter.create<WarpExecuteOnLane0Op>(
+    auto secondWarpOp = WarpExecuteOnLane0Op::create(rewriter,
         loc, TypeRange(), newWarpOp.getLaneid(), newWarpOp.getWarpSize());
     Block &body = secondWarpOp.getBodyRegion().front();
     rewriter.setInsertionPointToStart(&body);
@@ -521,7 +521,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     newWriteOp.getValueToStoreMutable().assign(
         newWarpOp.getResult(newRetIndices[0]));
     rewriter.eraseOp(writeOp);
-    rewriter.create<gpu::YieldOp>(newWarpOp.getLoc());
+    gpu::YieldOp::create(rewriter, newWarpOp.getLoc());
     return success();
   }
 
@@ -698,7 +698,7 @@ struct WarpOpConstant : public WarpDistributionPattern {
         cast<ShapedType>(warpOp.getResult(operandIndex).getType()), scalarAttr);
     Location loc = warpOp.getLoc();
     rewriter.setInsertionPointAfter(warpOp);
-    Value distConstant = rewriter.create<arith::ConstantOp>(loc, newAttr);
+    Value distConstant = arith::ConstantOp::create(rewriter, loc, newAttr);
     rewriter.replaceAllUsesWith(warpOp.getResult(operandIndex), distConstant);
     rewriter.finalizeOpModification(warpOp);
     return success();
@@ -823,7 +823,7 @@ struct WarpOpTransferRead : public WarpDistributionPattern {
     Value newMask =
         hasMask ? newWarpOp.getResult(newRetIndices[newRetIndices.size() - 1])
                 : Value();
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         read.getLoc(), distributedVal.getType(), read.getBase(), newIndices,
         read.getPermutationMapAttr(), newPadding, newMask,
         read.getInBoundsAttr());
@@ -965,7 +965,7 @@ struct WarpOpBroadcast : public WarpDistributionPattern {
     WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
         rewriter, warpOp, {broadcastSrc}, {broadcastSrcType}, newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value broadcasted = rewriter.create<vector::BroadcastOp>(
+    Value broadcasted = vector::BroadcastOp::create(rewriter,
         loc, destVecType, newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 broadcasted);
@@ -1008,7 +1008,7 @@ struct WarpOpShapeCast : public WarpDistributionPattern {
         rewriter, warpOp, {oldCastOp.getSource()}, {castDistributedType},
         newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value newCast = rewriter.create<vector::ShapeCastOp>(
+    Value newCast = vector::ShapeCastOp::create(rewriter,
         oldCastOp.getLoc(), castResultType,
         newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newCast);
@@ -1091,7 +1091,7 @@ struct WarpOpCreateMask : public WarpDistributionPattern {
     }
 
     auto newMask =
-        rewriter.create<vector::CreateMaskOp>(loc, distType, newOperands);
+        vector::CreateMaskOp::create(rewriter, loc, distType, newOperands);
     rewriter.replaceAllUsesWith(warpOp.getResult(operandIndex), newMask);
     rewriter.finalizeOpModification(warpOp);
     return success();
@@ -1182,7 +1182,7 @@ struct WarpOpInsertStridedSlice : public WarpDistributionPattern {
     Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
     // Create a new insert strided slice op that inserts distributed source into
     // distributed dest.
-    Value newInsert = rewriter.create<vector::InsertStridedSliceOp>(
+    Value newInsert = vector::InsertStridedSliceOp::create(rewriter,
         insertOp.getLoc(), distributedDest.getType(), distributedSource,
         distributedDest, insertOp.getOffsets(), insertOp.getStrides());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newInsert);
@@ -1277,7 +1277,7 @@ struct WarpOpExtractStridedSlice : public WarpDistributionPattern {
     // Create a new extract strided slice op that extracts from the
     // distributed vector.
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
-    Value newExtract = rewriter.create<vector::ExtractStridedSliceOp>(
+    Value newExtract = vector::ExtractStridedSliceOp::create(rewriter,
         extractOp.getLoc(), distributedType, distributedVec,
         extractOp.getOffsets(),
         ArrayAttr::get(rewriter.getContext(), distributedSizes),
@@ -1323,7 +1323,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
       // Extract from distributed vector.
-      Value newExtract = rewriter.create<vector::ExtractOp>(
+      Value newExtract = vector::ExtractOp::create(rewriter,
           loc, distributedVec, extractOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
@@ -1352,7 +1352,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
     // Extract from distributed vector.
-    Value newExtract = rewriter.create<vector::ExtractOp>(
+    Value newExtract = vector::ExtractOp::create(rewriter,
         loc, distributedVec, extractOp.getMixedPosition());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 newExtract);
@@ -1422,7 +1422,7 @@ struct WarpOpExtractScalar : public WarpDistributionPattern {
       Value newExtract;
       SmallVector<int64_t> indices(extractSrcType.getRank(), 0);
       newExtract =
-          rewriter.create<vector::ExtractOp>(loc, distributedVec, indices);
+          vector::ExtractOp::create(rewriter, loc, distributedVec, indices);
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
       return success();
@@ -1442,11 +1442,11 @@ struct WarpOpExtractScalar : public WarpDistributionPattern {
     // Extract at position: pos % elementsPerLane
     Value newPos =
         elementsPerLane == 1
-            ? rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult()
+            ? arith::ConstantIndexOp::create(rewriter, loc, 0).getResult()
             : affine::makeComposedAffineApply(rewriter, loc,
                                               sym0 % elementsPerLane, pos);
     Value extracted =
-        rewriter.create<vector::ExtractOp>(loc, distributedVec, newPos);
+        vector::ExtractOp::create(rewriter, loc, distributedVec, newPos);
 
     // Shuffle the extracted value to all lanes.
     Value shuffled = warpShuffleFromIdxFn(
@@ -1514,7 +1514,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
       if (pos) {
         indices.push_back(pos);
       }
-      newInsert = rewriter.create<vector::InsertOp>(loc, newSource,
+      newInsert = vector::InsertOp::create(rewriter, loc, newSource,
                                                     distributedVec, indices);
       // Broadcast: Simply move the vector.insert op out.
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
@@ -1531,7 +1531,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
     // Insert position: pos % elementsPerLane
     OpFoldResult newPos = affine::makeComposedFoldedAffineApply(
         rewriter, loc, sym0 % elementsPerLane, pos);
-    Value isInsertingLane = rewriter.create<arith::CmpIOp>(
+    Value isInsertingLane = arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
     Value newResult =
         rewriter
@@ -1539,13 +1539,13 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
                 loc, isInsertingLane,
                 /*thenBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  Value newInsert = builder.create<vector::InsertOp>(
+                  Value newInsert = vector::InsertOp::create(builder,
                       loc, newSource, distributedVec, newPos);
-                  builder.create<scf::YieldOp>(loc, newInsert);
+                  scf::YieldOp::create(builder, loc, newInsert);
                 },
                 /*elseBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  builder.create<scf::YieldOp>(loc, distributedVec);
+                  scf::YieldOp::create(builder, loc, distributedVec);
                 })
             .getResult(0);
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newResult);
@@ -1582,7 +1582,7 @@ struct WarpOpInsert : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedSrc = newWarpOp->getResult(newRetIndices[0]);
       Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
-      Value newResult = rewriter.create<vector::InsertOp>(
+      Value newResult = vector::InsertOp::create(rewriter,
           loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newResult);
@@ -1632,7 +1632,7 @@ struct WarpOpInsert : public WarpDistributionPattern {
     Value newResult;
     if (distrSrcDim >= 0) {
       // Every lane inserts a small piece.
-      newResult = rewriter.create<vector::InsertOp>(
+      newResult = vector::InsertOp::create(rewriter,
           loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
     } else {
       // One lane inserts the entire source vector.
@@ -1640,19 +1640,19 @@ struct WarpOpInsert : public WarpDistributionPattern {
       SmallVector<OpFoldResult> pos = insertOp.getMixedPosition();
       SmallVector<int64_t> newPos = getAsIntegers(pos);
       // tid of inserting lane: pos / elementsPerLane
-      Value insertingLane = rewriter.create<arith::ConstantIndexOp>(
+      Value insertingLane = arith::ConstantIndexOp::create(rewriter,
           loc, newPos[distrDestDim] / elementsPerLane);
-      Value isInsertingLane = rewriter.create<arith::CmpIOp>(
+      Value isInsertingLane = arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
       // Insert position: pos % elementsPerLane
       newPos[distrDestDim] %= elementsPerLane;
       auto insertingBuilder = [&](OpBuilder &builder, Location loc) {
-        Value newInsert = builder.create<vector::InsertOp>(
+        Value newInsert = vector::InsertOp::create(builder,
             loc, distributedSrc, distributedDest, newPos);
-        builder.create<scf::YieldOp>(loc, newInsert);
+        scf::YieldOp::create(builder, loc, newInsert);
       };
       auto nonInsertingBuilder = [&](OpBuilder &builder, Location loc) {
-        builder.create<scf::YieldOp>(loc, distributedDest);
+        scf::YieldOp::create(builder, loc, distributedDest);
       };
       newResult = rewriter
                       .create<scf::IfOp>(loc, isInsertingLane,
@@ -1761,7 +1761,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
 
     // Create a new for op outside the region with a WarpExecuteOnLane0Op
     // region inside.
-    auto newForOp = rewriter.create<scf::ForOp>(
+    auto newForOp = scf::ForOp::create(rewriter,
         forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), newOperands);
     rewriter.setInsertionPointToStart(newForOp.getBody());
@@ -1776,7 +1776,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
       argIndexMapping[escapingValues[i]] = warpInputType.size();
       warpInputType.push_back(inputTypes[i]);
     }
-    auto innerWarp = rewriter.create<WarpExecuteOnLane0Op>(
+    auto innerWarp = WarpExecuteOnLane0Op::create(rewriter,
         newWarpOp.getLoc(), newForOp.getResultTypes(), newWarpOp.getLaneid(),
         newWarpOp.getWarpSize(), warpInput, warpInputType);
 
@@ -1792,10 +1792,10 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
     rewriter.eraseOp(forOp.getBody()->getTerminator());
     rewriter.mergeBlocks(forOp.getBody(), innerWarp.getBody(), argMapping);
     rewriter.setInsertionPointToEnd(innerWarp.getBody());
-    rewriter.create<gpu::YieldOp>(innerWarp.getLoc(), yieldOperands);
+    gpu::YieldOp::create(rewriter, innerWarp.getLoc(), yieldOperands);
     rewriter.setInsertionPointAfter(innerWarp);
     if (!innerWarp.getResults().empty())
-      rewriter.create<scf::YieldOp>(forOp.getLoc(), innerWarp.getResults());
+      scf::YieldOp::create(rewriter, forOp.getLoc(), innerWarp.getResults());
     rewriter.eraseOp(forOp);
     // Replace the warpOp result coming from the original ForOp.
     for (const auto &res : llvm::enumerate(resultIdx)) {
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
index 067d4e3491391..e9718a7795fc5 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
@@ -77,7 +77,7 @@ struct CastAwayExtractStridedSliceLeadingOneDim
 
     Location loc = extractOp.getLoc();
 
-    Value newSrcVector = rewriter.create<vector::ExtractOp>(
+    Value newSrcVector = vector::ExtractOp::create(rewriter,
         loc, extractOp.getVector(), splatZero(dropCount));
 
     // The offsets/sizes/strides attribute can have a less number of elements
@@ -89,7 +89,7 @@ struct CastAwayExtractStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         extractOp.getStrides().getValue().drop_front(dropCount));
 
-    auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
         loc, newDstType, newSrcVector, newOffsets, newSizes, newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(extractOp, oldDstType,
@@ -120,9 +120,9 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     // Trim leading one dimensions from both operands.
     Location loc = insertOp.getLoc();
 
-    Value newSrcVector = rewriter.create<vector::ExtractOp>(
+    Value newSrcVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getValueToStore(), splatZero(srcDropCount));
-    Value newDstVector = rewriter.create<vector::ExtractOp>(
+    Value newDstVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getDest(), splatZero(dstDropCount));
 
     auto newOffsets = rewriter.getArrayAttr(
@@ -130,7 +130,7 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         insertOp.getStrides().getValue().take_back(newSrcType.getRank()));
 
-    auto newInsertOp = rewriter.create<vector::InsertStridedSliceOp>(
+    auto newInsertOp = vector::InsertStridedSliceOp::create(rewriter,
         loc, newDstType, newSrcVector, newDstVector, newOffsets, newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
@@ -169,10 +169,10 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
 
     Value newSrcVector = insertOp.getValueToStore();
     if (oldSrcRank != 0) {
-      newSrcVector = rewriter.create<vector::ExtractOp>(
+      newSrcVector = vector::ExtractOp::create(rewriter,
           loc, insertOp.getValueToStore(), splatZero(srcDropCount));
     }
-    Value newDstVector = rewriter.create<vector::ExtractOp>(
+    Value newDstVector = vector::ExtractOp::create(rewriter,
         loc, insertOp.getDest(), splatZero(dstDropCount));
 
     // New position rank needs to be computed in two steps: (1) if destination
@@ -187,7 +187,7 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
     newPosition.resize(newDstType.getRank() - newSrcRank,
                        rewriter.getI64IntegerAttr(0));
 
-    auto newInsertOp = rewriter.create<vector::InsertOp>(
+    auto newInsertOp = vector::InsertOp::create(rewriter,
         loc, newSrcVector, newDstVector, newPosition);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
@@ -209,9 +209,9 @@ static Value dropUnitDimsFromMask(OpBuilder &b, Location loc, Value mask,
   if (vector::isBroadcastableTo(newMaskType, oldMaskType) ==
       BroadcastableToResult::Success) {
     int64_t dropDim = oldMaskType.getRank() - newMaskType.getRank();
-    return b.create<vector::ExtractOp>(loc, mask, splatZero(dropDim));
+    return vector::ExtractOp::create(b, loc, mask, splatZero(dropDim));
   }
-  return b.create<vector::ShapeCastOp>(loc, newMaskType, mask);
+  return vector::ShapeCastOp::create(b, loc, newMaskType, mask);
 }
 
 // Turns vector.transfer_read on vector with leading 1 dimensions into
@@ -259,7 +259,7 @@ struct CastAwayTransferReadLeadingOneDim
                                   newType, newMap, maskType);
     }
 
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         read.getLoc(), newType, read.getBase(), read.getIndices(),
         AffineMapAttr::get(newMap), read.getPadding(), mask, inBoundsAttr);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(read, oldType, newRead);
@@ -306,7 +306,7 @@ struct CastAwayTransferWriteLeadingOneDim
       inBoundsAttr = rewriter.getArrayAttr(
           write.getInBoundsAttr().getValue().take_back(newType.getRank()));
 
-    auto newVector = rewriter.create<vector::ExtractOp>(
+    auto newVector = vector::ExtractOp::create(rewriter,
         write.getLoc(), write.getVector(), splatZero(dropDim));
 
     if (write.getMask()) {
@@ -444,20 +444,20 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
     // Extract if its a valid extraction, otherwise use the operand
     // without extraction.
     newOperands.push_back(
-        validExtract ? rewriter.create<vector::ExtractOp>(
+        validExtract ? vector::ExtractOp::create(rewriter,
                            loc, operands[it.index()], splatZero(dropDim))
                      : operands[it.index()]);
   }
 
   // Depending on whether this vector.contract is masked, the replacing Op
   // should either be a new vector.contract Op or vector.mask Op.
-  Operation *newOp = rewriter.create<vector::ContractionOp>(
+  Operation *newOp = vector::ContractionOp::create(rewriter,
       loc, newOperands[0], newOperands[1], newOperands[2],
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind());
 
   if (maskingOp) {
-    auto newMask = rewriter.create<vector::ExtractOp>(loc, maskingOp.getMask(),
+    auto newMask = vector::ExtractOp::create(rewriter, loc, maskingOp.getMask(),
                                                       splatZero(dropDim));
 
     newOp = mlir::vector::maskOperation(rewriter, newOp, newMask);
@@ -519,7 +519,7 @@ class CastAwayElementwiseLeadingOneDim : public RewritePattern {
     SmallVector<Value, 4> newOperands;
     for (Value operand : op->getOperands()) {
       if (auto opVecType = dyn_cast<VectorType>(operand.getType())) {
-        newOperands.push_back(rewriter.create<vector::ExtractOp>(
+        newOperands.push_back(vector::ExtractOp::create(rewriter,
             op->getLoc(), operand, splatZero(dropDim)));
       } else {
         newOperands.push_back(operand);
@@ -559,7 +559,7 @@ struct CastAwayConstantMaskLeadingOneDim
     SmallVector<int64_t> newDimSizes = {flatLeadingSize};
     newDimSizes.append(dimSizes.begin() + dropDim + 1, dimSizes.end());
 
-    auto newMask = rewriter.create<vector::ConstantMaskOp>(
+    auto newMask = vector::ConstantMaskOp::create(rewriter,
         mask.getLoc(), newType, newDimSizes);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(mask, oldType, newMask);
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
index 8cc7008d80b3e..3411e010f3499 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
@@ -65,26 +65,26 @@ struct VectorMaskedLoadOpConverter final
     Value base = maskedLoadOp.getBase();
     Value iValue = maskedLoadOp.getPassThru();
     auto indices = llvm::to_vector_of<Value>(maskedLoadOp.getIndices());
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, indexType, IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
-      auto maskBit = rewriter.create<vector::ExtractOp>(loc, mask, i);
+      auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = rewriter.create<scf::IfOp>(
+      auto ifOp = scf::IfOp::create(rewriter,
           loc, maskBit,
           [&](OpBuilder &builder, Location loc) {
             auto loadedValue =
-                builder.create<memref::LoadOp>(loc, base, indices);
+                memref::LoadOp::create(builder, loc, base, indices);
             auto combinedValue =
-                builder.create<vector::InsertOp>(loc, loadedValue, iValue, i);
-            builder.create<scf::YieldOp>(loc, combinedValue.getResult());
+                vector::InsertOp::create(builder, loc, loadedValue, iValue, i);
+            scf::YieldOp::create(builder, loc, combinedValue.getResult());
           },
           [&](OpBuilder &builder, Location loc) {
-            builder.create<scf::YieldOp>(loc, iValue);
+            scf::YieldOp::create(builder, loc, iValue);
           });
       iValue = ifOp.getResult(0);
 
-      indices.back() = rewriter.create<arith::AddIOp>(loc, indices.back(), one);
+      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.replaceOp(maskedLoadOp, iValue);
@@ -132,18 +132,18 @@ struct VectorMaskedStoreOpConverter final
     Value base = maskedStoreOp.getBase();
     Value value = maskedStoreOp.getValueToStore();
     auto indices = llvm::to_vector_of<Value>(maskedStoreOp.getIndices());
-    Value one = rewriter.create<arith::ConstantOp>(
+    Value one = arith::ConstantOp::create(rewriter,
         loc, indexType, IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
-      auto maskBit = rewriter.create<vector::ExtractOp>(loc, mask, i);
+      auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = rewriter.create<scf::IfOp>(loc, maskBit, /*else=*/false);
+      auto ifOp = scf::IfOp::create(rewriter, loc, maskBit, /*else=*/false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      auto extractedValue = rewriter.create<vector::ExtractOp>(loc, value, i);
-      rewriter.create<memref::StoreOp>(loc, extractedValue, base, indices);
+      auto extractedValue = vector::ExtractOp::create(rewriter, loc, value, i);
+      memref::StoreOp::create(rewriter, loc, extractedValue, base, indices);
 
       rewriter.setInsertionPointAfter(ifOp);
-      indices.back() = rewriter.create<arith::AddIOp>(loc, indices.back(), one);
+      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.eraseOp(maskedStoreOp);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 0fe08417f818f..e915e85f16910 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -132,7 +132,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 SmallVector<Value> newMaskOperands(maskOperands.drop_back());
                 newMaskOperands.push_back(
                     getValueOrCreateConstantIndexOp(rewriter, loc, maskIndex));
-                return rewriter.create<vector::CreateMaskOp>(loc, newMaskType,
+                return vector::CreateMaskOp::create(rewriter, loc, newMaskType,
                                                              newMaskOperands);
               })
           .Case<vector::ConstantMaskOp>(
@@ -143,7 +143,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 int64_t &maskIndex = maskDimSizes.back();
                 maskIndex = llvm::divideCeil(numFrontPadElems + maskIndex,
                                              numSrcElemsPerDest);
-                return rewriter.create<vector::ConstantMaskOp>(loc, newMaskType,
+                return vector::ConstantMaskOp::create(rewriter, loc, newMaskType,
                                                                maskDimSizes);
               })
           .Case<arith::ConstantOp>([&](auto constantOp)
@@ -182,7 +182,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
               }
               compressedMaskValues.push_back(combinedValue);
             }
-            return rewriter.create<arith::ConstantOp>(
+            return arith::ConstantOp::create(rewriter,
                 loc, DenseElementsAttr::get(newMaskType, compressedMaskValues));
           });
 
@@ -190,7 +190,7 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
     return failure();
 
   while (!extractOps.empty()) {
-    newMask = rewriter.create<vector::ExtractOp>(
+    newMask = vector::ExtractOp::create(rewriter,
         loc, (*newMask)->getResults()[0], extractOps.back().getMixedPosition());
     extractOps.pop_back();
   }
@@ -258,7 +258,7 @@ static Value staticallyInsertSubvector(OpBuilder &rewriter, Location loc,
 
   auto offsets = rewriter.getI64ArrayAttr({offset});
   auto strides = rewriter.getI64ArrayAttr({1});
-  return rewriter.create<vector::InsertStridedSliceOp>(loc, destVecTy, src,
+  return vector::InsertStridedSliceOp::create(rewriter, loc, destVecTy, src,
                                                        dest, offsets, strides);
 }
 
@@ -301,11 +301,11 @@ static Value dynamicallyExtractSubVector(OpBuilder &rewriter, Location loc,
   for (int i = 0; i < numElemsToExtract; ++i) {
     Value extractLoc =
         (i == 0) ? dyn_cast<Value>(offset)
-                 : rewriter.create<arith::AddIOp>(
+                 : arith::AddIOp::create(rewriter,
                        loc, rewriter.getIndexType(), dyn_cast<Value>(offset),
-                       rewriter.create<arith::ConstantIndexOp>(loc, i));
-    auto extractOp = rewriter.create<vector::ExtractOp>(loc, src, extractLoc);
-    dest = rewriter.create<vector::InsertOp>(loc, extractOp, dest, i);
+                       arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto extractOp = vector::ExtractOp::create(rewriter, loc, src, extractLoc);
+    dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, i);
   }
   return dest;
 }
@@ -346,11 +346,11 @@ static Value dynamicallyInsertSubVector(RewriterBase &rewriter, Location loc,
   for (int64_t i = 0; i < numElemsToInsert; ++i) {
     auto insertLoc = i == 0
                          ? destOffsetVal
-                         : rewriter.create<arith::AddIOp>(
+                         : arith::AddIOp::create(rewriter,
                                loc, rewriter.getIndexType(), destOffsetVal,
-                               rewriter.create<arith::ConstantIndexOp>(loc, i));
-    auto extractOp = rewriter.create<vector::ExtractOp>(loc, src, i);
-    dest = rewriter.create<vector::InsertOp>(loc, extractOp, dest, insertLoc);
+                               arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto extractOp = vector::ExtractOp::create(rewriter, loc, src, i);
+    dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, insertLoc);
   }
   return dest;
 }
@@ -369,10 +369,10 @@ static VectorValue emulatedVectorLoad(OpBuilder &rewriter, Location loc,
                                       Type containerElemTy) {
   auto emulatedPerContainerElem = containerElemTy.getIntOrFloatBitWidth() /
                                   emulatedElemTy.getIntOrFloatBitWidth();
-  auto newLoad = rewriter.create<vector::LoadOp>(
+  auto newLoad = vector::LoadOp::create(rewriter,
       loc, VectorType::get(numContainerElemsToLoad, containerElemTy), base,
       getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
-  return rewriter.create<vector::BitCastOp>(
+  return vector::BitCastOp::create(rewriter,
       loc,
       VectorType::get(numContainerElemsToLoad * emulatedPerContainerElem,
                       emulatedElemTy),
@@ -390,16 +390,16 @@ static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
           upcastType.getNumElements() * upcastType.getElementTypeBitWidth() &&
       "expected input and output number of bits to match");
   if (trueValue.getType() != downcastType) {
-    trueValue = builder.create<vector::BitCastOp>(loc, downcastType, trueValue);
+    trueValue = vector::BitCastOp::create(builder, loc, downcastType, trueValue);
   }
   if (falseValue.getType() != downcastType) {
     falseValue =
-        builder.create<vector::BitCastOp>(loc, downcastType, falseValue);
+        vector::BitCastOp::create(builder, loc, downcastType, falseValue);
   }
   Value selectedType =
-      builder.create<arith::SelectOp>(loc, mask, trueValue, falseValue);
+      arith::SelectOp::create(builder, loc, mask, trueValue, falseValue);
   // Upcast the selected value to the new type.
-  return builder.create<vector::BitCastOp>(loc, upcastType, selectedType);
+  return vector::BitCastOp::create(builder, loc, upcastType, selectedType);
 }
 
 /// Emits `memref.generic_atomic_rmw` op to store a subbyte-sized value to a
@@ -422,7 +422,7 @@ static void atomicRMW(OpBuilder &builder, Location loc,
 
   // Create an atomic load-modify-write region using
   // `memref.generic_atomic_rmw`.
-  auto atomicOp = builder.create<memref::GenericAtomicRMWOp>(
+  auto atomicOp = memref::GenericAtomicRMWOp::create(builder,
       loc, linearizedMemref, ValueRange{storeIdx});
   Value origValue = atomicOp.getCurrentValue();
 
@@ -432,7 +432,7 @@ static void atomicRMW(OpBuilder &builder, Location loc,
   // Load the original value from memory, and cast it to the original element
   // type.
   auto oneElemVecType = VectorType::get({1}, origValue.getType());
-  Value origVecValue = builder.create<vector::FromElementsOp>(
+  Value origVecValue = vector::FromElementsOp::create(builder,
       loc, oneElemVecType, ValueRange{origValue});
 
   // Construct the final masked value and yield it.
@@ -440,8 +440,8 @@ static void atomicRMW(OpBuilder &builder, Location loc,
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
   auto scalarMaskedValue =
-      builder.create<vector::ExtractOp>(loc, maskedValue, 0);
-  builder.create<memref::AtomicYieldOp>(loc, scalarMaskedValue);
+      vector::ExtractOp::create(builder, loc, maskedValue, 0);
+  memref::AtomicYieldOp::create(builder, loc, scalarMaskedValue);
 }
 
 /// Generate a non-atomic read-modify-write sequence for storing to the emulated
@@ -453,15 +453,15 @@ static void nonAtomicRMW(OpBuilder &builder, Location loc,
 
   auto oneElemVecType =
       VectorType::get({1}, linearizedMemref.getType().getElementType());
-  Value origVecValue = builder.create<vector::LoadOp>(
+  Value origVecValue = vector::LoadOp::create(builder,
       loc, oneElemVecType, linearizedMemref, ValueRange{linearizedIndex});
-  origVecValue = builder.create<vector::BitCastOp>(loc, valueToStore.getType(),
+  origVecValue = vector::BitCastOp::create(builder, loc, valueToStore.getType(),
                                                    origVecValue);
 
   Value maskedValue =
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
-  builder.create<vector::StoreOp>(loc, maskedValue, linearizedMemref,
+  vector::StoreOp::create(builder, loc, maskedValue, linearizedMemref,
                                   linearizedIndex);
 }
 
@@ -489,7 +489,7 @@ static Value extractSliceIntoByte(ConversionPatternRewriter &rewriter,
   assert(8 % vectorElementType.getIntOrFloatBitWidth() == 0 &&
          "vector element must be a valid sub-byte type");
   auto emulatedPerContainerElem = 8 / vectorElementType.getIntOrFloatBitWidth();
-  auto emptyByteVector = rewriter.create<arith::ConstantOp>(
+  auto emptyByteVector = arith::ConstantOp::create(rewriter,
       loc, VectorType::get({emulatedPerContainerElem}, vectorElementType),
       rewriter.getZeroAttr(
           VectorType::get({emulatedPerContainerElem}, vectorElementType)));
@@ -602,7 +602,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         ShapedType::isDynamic(trailingDim) || trailingDim == origElements;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     // FIXME: ATM, we do not test cases where offsets, sizes, or strides are
     // non-zero. As such, this is not needed.
@@ -664,7 +664,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     if (!emulationRequiresPartialStores) {
       // Basic case: storing full bytes.
       auto numElements = origElements / emulatedPerContainerElem;
-      auto bitCast = rewriter.create<vector::BitCastOp>(
+      auto bitCast = vector::BitCastOp::create(rewriter,
           loc, VectorType::get(numElements, containerElemTy),
           op.getValueToStore());
       rewriter.replaceOpWithNewOp<vector::StoreOp>(
@@ -732,7 +732,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         std::fill_n(frontMaskValues.end() - frontSubWidthStoreElem,
                     *foldedNumFrontPadElems, true);
       }
-      auto frontMask = rewriter.create<arith::ConstantOp>(
+      auto frontMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
 
       currentSourceIndex = emulatedPerContainerElem - (*foldedNumFrontPadElems);
@@ -751,8 +751,8 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
 
     // Increment the destination index by 1 to align to the emulated width
     // boundary.
-    auto constantOne = rewriter.create<arith::ConstantIndexOp>(loc, 1);
-    currentDestIndex = rewriter.create<arith::AddIOp>(
+    auto constantOne = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    currentDestIndex = arith::AddIOp::create(rewriter,
         loc, rewriter.getIndexType(), currentDestIndex, constantOne);
 
     // 2. Full width store for the inner output bytes.
@@ -772,15 +772,15 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       auto storeType = VectorType::get(
           {originType.getNumElements() / emulatedPerContainerElem},
           memrefElemType);
-      auto bitCast = rewriter.create<vector::BitCastOp>(loc, storeType,
+      auto bitCast = vector::BitCastOp::create(rewriter, loc, storeType,
                                                         fullWidthStorePart);
-      rewriter.create<vector::StoreOp>(loc, bitCast.getResult(), memrefBase,
+      vector::StoreOp::create(rewriter, loc, bitCast.getResult(), memrefBase,
                                        currentDestIndex);
 
       currentSourceIndex += numNonFullWidthElements;
-      currentDestIndex = rewriter.create<arith::AddIOp>(
+      currentDestIndex = arith::AddIOp::create(rewriter,
           loc, rewriter.getIndexType(), currentDestIndex,
-          rewriter.create<arith::ConstantIndexOp>(loc, fullWidthStoreSize));
+          arith::ConstantIndexOp::create(rewriter, loc, fullWidthStoreSize));
     }
 
     // 3. Partial width store for the trailing output byte.
@@ -795,7 +795,7 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       // Generate back mask.
       auto maskValues = SmallVector<bool>(emulatedPerContainerElem, 0);
       std::fill_n(maskValues.begin(), remainingElements, 1);
-      auto backMask = rewriter.create<arith::ConstantOp>(
+      auto backMask = arith::ConstantOp::create(rewriter,
           loc, DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
 
       storeFunc(rewriter, loc, memrefBase, currentDestIndex,
@@ -848,7 +848,7 @@ struct ConvertVectorMaskedStore final
       return failure();
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
     OpFoldResult linearizedIndicesOfr;
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndicesOfr) =
@@ -901,21 +901,21 @@ struct ConvertVectorMaskedStore final
     auto numElements = (origElements + emulatedPerContainerElem - 1) /
                        emulatedPerContainerElem;
     auto newType = VectorType::get(numElements, containerElemTy);
-    auto passThru = rewriter.create<arith::ConstantOp>(
+    auto passThru = arith::ConstantOp::create(rewriter,
         loc, newType, rewriter.getZeroAttr(newType));
 
-    auto newLoad = rewriter.create<vector::MaskedLoadOp>(
+    auto newLoad = vector::MaskedLoadOp::create(rewriter,
         loc, newType, adaptor.getBase(), linearizedIndices,
         newMask.value()->getResult(0), passThru);
 
     auto newBitCastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
     Value valueToStore =
-        rewriter.create<vector::BitCastOp>(loc, newBitCastType, newLoad);
-    valueToStore = rewriter.create<arith::SelectOp>(
+        vector::BitCastOp::create(rewriter, loc, newBitCastType, newLoad);
+    valueToStore = arith::SelectOp::create(rewriter,
         loc, op.getMask(), op.getValueToStore(), valueToStore);
     valueToStore =
-        rewriter.create<vector::BitCastOp>(loc, newType, valueToStore);
+        vector::BitCastOp::create(rewriter, loc, newType, valueToStore);
 
     rewriter.replaceOpWithNewOp<vector::MaskedStoreOp>(
         op, adaptor.getBase(), linearizedIndices, newMask.value()->getResult(0),
@@ -990,7 +990,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
     bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
@@ -1016,7 +1016,7 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
                            numElements, emulatedElemTy, containerElemTy);
 
     if (!foldedIntraVectorOffset) {
-      auto resultVector = rewriter.create<arith::ConstantOp>(
+      auto resultVector = arith::ConstantOp::create(rewriter,
           loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(
           rewriter, loc, dyn_cast<TypedValue<VectorType>>(result), resultVector,
@@ -1111,7 +1111,7 @@ struct ConvertVectorMaskedLoad final
     bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
     std::tie(linearizedInfo, linearizedIndices) =
@@ -1142,7 +1142,7 @@ struct ConvertVectorMaskedLoad final
     auto newBitcastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
 
-    auto emptyVector = rewriter.create<arith::ConstantOp>(
+    auto emptyVector = arith::ConstantOp::create(rewriter,
         loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
     if (!foldedIntraVectorOffset) {
       passthru = dynamicallyInsertSubVector(
@@ -1153,10 +1153,10 @@ struct ConvertVectorMaskedLoad final
                                            *foldedIntraVectorOffset);
     }
     auto newPassThru =
-        rewriter.create<vector::BitCastOp>(loc, loadType, passthru);
+        vector::BitCastOp::create(rewriter, loc, loadType, passthru);
 
     // Generating the new masked load.
-    auto newLoad = rewriter.create<vector::MaskedLoadOp>(
+    auto newLoad = vector::MaskedLoadOp::create(rewriter,
         loc, loadType, adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newMask.value()->getResult(0), newPassThru);
@@ -1164,13 +1164,13 @@ struct ConvertVectorMaskedLoad final
     // Setting the part that originally was not effectively loaded from memory
     // to pass through.
     auto bitCast =
-        rewriter.create<vector::BitCastOp>(loc, newBitcastType, newLoad);
+        vector::BitCastOp::create(rewriter, loc, newBitcastType, newLoad);
 
     Value mask = op.getMask();
     auto newSelectMaskType = VectorType::get(
         numElements * emulatedPerContainerElem, rewriter.getI1Type());
     // TODO: try to fold if op's mask is constant
-    auto emptyMask = rewriter.create<arith::ConstantOp>(
+    auto emptyMask = arith::ConstantOp::create(rewriter,
         loc, newSelectMaskType, rewriter.getZeroAttr(newSelectMaskType));
     if (!foldedIntraVectorOffset) {
       mask = dynamicallyInsertSubVector(rewriter, loc, mask, emptyMask,
@@ -1182,7 +1182,7 @@ struct ConvertVectorMaskedLoad final
     }
 
     Value result =
-        rewriter.create<arith::SelectOp>(loc, mask, bitCast, passthru);
+        arith::SelectOp::create(rewriter, loc, mask, bitCast, passthru);
     if (!foldedIntraVectorOffset) {
       result = dynamicallyExtractSubVector(
           rewriter, loc, result, op.getPassThru(),
@@ -1279,10 +1279,11 @@ struct ConvertVectorTransferRead final
           padding);
     }
     auto newPadding =
-        rewriter.create<arith::ExtUIOp>(loc, containerElemTy, padding);
+        arith::ExtUIOp::create(rewriter, loc, containerElemTy,
+                                                      padding);
 
     auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, op.getBase());
+        memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
 
     OpFoldResult linearizedIndices;
     memref::LinearizedMemRefInfo linearizedInfo;
@@ -1303,19 +1304,19 @@ struct ConvertVectorTransferRead final
     auto numElements = llvm::divideCeil(maxIntraDataOffset + origElements,
                                         emulatedPerContainerElem);
 
-    auto newRead = rewriter.create<vector::TransferReadOp>(
+    auto newRead = vector::TransferReadOp::create(rewriter,
         loc, VectorType::get(numElements, containerElemTy), adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newPadding);
 
-    auto bitCast = rewriter.create<vector::BitCastOp>(
+    auto bitCast = vector::BitCastOp::create(rewriter,
         loc,
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy),
         newRead);
 
     Value result = bitCast->getResult(0);
     if (!foldedIntraVectorOffset) {
-      auto zeros = rewriter.create<arith::ConstantOp>(
+      auto zeros = arith::ConstantOp::create(rewriter,
           loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(rewriter, loc, bitCast, zeros,
                                            linearizedInfo.intraDataOffset,
@@ -1689,32 +1690,32 @@ Value BitCastRewriter::genericRewriteStep(
     PatternRewriter &rewriter, Location loc, Value initialValue,
     Value runningResult, const BitCastRewriter::Metadata &metadata) {
   // Create vector.shuffle from the metadata.
-  auto shuffleOp = rewriter.create<vector::ShuffleOp>(
+  auto shuffleOp = vector::ShuffleOp::create(rewriter,
       loc, initialValue, initialValue, metadata.shuffles);
 
   // Intersect with the mask.
   VectorType shuffledVectorType = shuffleOp.getResultVectorType();
-  auto constOp = rewriter.create<arith::ConstantOp>(
+  auto constOp = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(shuffledVectorType, metadata.masks));
-  Value andValue = rewriter.create<arith::AndIOp>(loc, shuffleOp, constOp);
+  Value andValue = arith::AndIOp::create(rewriter, loc, shuffleOp, constOp);
 
   // Align right on 0.
-  auto shiftRightConstantOp = rewriter.create<arith::ConstantOp>(
+  auto shiftRightConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftRightAmounts));
   Value shiftedRight =
-      rewriter.create<arith::ShRUIOp>(loc, andValue, shiftRightConstantOp);
+      arith::ShRUIOp::create(rewriter, loc, andValue, shiftRightConstantOp);
 
   // Shift bits left into their final position.
-  auto shiftLeftConstantOp = rewriter.create<arith::ConstantOp>(
+  auto shiftLeftConstantOp = arith::ConstantOp::create(rewriter,
       loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftLeftAmounts));
   Value shiftedLeft =
-      rewriter.create<arith::ShLIOp>(loc, shiftedRight, shiftLeftConstantOp);
+      arith::ShLIOp::create(rewriter, loc, shiftedRight, shiftLeftConstantOp);
 
   runningResult =
       runningResult
-          ? rewriter.create<arith::OrIOp>(loc, runningResult, shiftedLeft)
+          ? arith::OrIOp::create(rewriter, loc, runningResult, shiftedLeft)
           : shiftedLeft;
 
   return runningResult;
@@ -1737,7 +1738,7 @@ static Value bitcastSubByteVectorToI8(PatternRewriter &rewriter, Location loc,
   // Adjust last dimension of the vector, so the total size remains the same.
   vecShape.back() = vecShape.back() / numSrcElemsPerByte;
   auto i8VecType = VectorType::get(vecShape, rewriter.getI8Type());
-  return rewriter.create<vector::BitCastOp>(loc, i8VecType, subByteVec);
+  return vector::BitCastOp::create(rewriter, loc, i8VecType, subByteVec);
 }
 
 /// Extracts a signed N-bit sequence from each element of a vector of bytes,
@@ -1765,15 +1766,15 @@ static Value extractNBitsPerByteAndSignExtendToI8(PatternRewriter &rewriter,
   assert(bitIdx >= 0 && bitsToShiftLeft >= 0 && numBits > 0 && numBits <= 8 &&
          "Invalid bitIdx range");
   if (bitsToShiftLeft != 0) {
-    Value shiftLeftValues = rewriter.create<arith::ConstantOp>(
+    Value shiftLeftValues = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(srcType, bitsToShiftLeft));
-    shl = rewriter.create<arith::ShLIOp>(loc, src, shiftLeftValues);
+    shl = arith::ShLIOp::create(rewriter, loc, src, shiftLeftValues);
   }
 
   int8_t bitsToShiftRight = 8 - numBits;
-  Value shiftRightValues = rewriter.create<arith::ConstantOp>(
+  Value shiftRightValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
-  Value shr = rewriter.create<arith::ShRSIOp>(loc, shl, shiftRightValues);
+  Value shr = arith::ShRSIOp::create(rewriter, loc, shl, shiftRightValues);
   return shr;
 }
 
@@ -1807,17 +1808,17 @@ static Value extractNBitsPerByteAndExtendToI8(PatternRewriter &rewriter,
   int8_t bitsToShiftRight = bitIdx;
   Value shr = src;
   if (bitsToShiftRight != 0) {
-    Value shiftRightValues = rewriter.create<arith::ConstantOp>(
+    Value shiftRightValues = arith::ConstantOp::create(rewriter,
         loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
-    shr = rewriter.create<arith::ShRUIOp>(loc, src, shiftRightValues);
+    shr = arith::ShRUIOp::create(rewriter, loc, src, shiftRightValues);
   }
   if (bitIdx + numBits == 8) {
     return shr;
   }
   uint8_t lowBitsMask = (1 << numBits) - 1;
-  Value lowBitsMaskValues = rewriter.create<arith::ConstantOp>(
+  Value lowBitsMaskValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(srcType, lowBitsMask));
-  return rewriter.create<arith::AndIOp>(loc, shr, lowBitsMaskValues);
+  return arith::AndIOp::create(rewriter, loc, shr, lowBitsMaskValues);
 }
 
 using ExtractNBitsFn =
@@ -1840,7 +1841,7 @@ static Value rewriteI4ToI8Ext(PatternRewriter &rewriter, Location loc,
   Value high = extFn(rewriter, loc, i8Vector, 4, 4);
 
   // 3. Interleave low and high i8 elements.
-  return rewriter.create<vector::InterleaveOp>(loc, low, high);
+  return vector::InterleaveOp::create(rewriter, loc, low, high);
 }
 
 /// Rewrite the i2 -> i8  extension into a sequence of shuffles and
@@ -1873,9 +1874,9 @@ static Value rewriteI2ToI8Ext(PatternRewriter &rewriter, Location loc,
   // 02    = [0,2,0,2,0,2,0,2],...
   // 13    = [1,3,1,3,1,3,1,3],...
   // 0213  = [0,1,2,3,...],...
-  Value interleave02 = rewriter.create<vector::InterleaveOp>(loc, vec0, vec2);
-  Value interleave13 = rewriter.create<vector::InterleaveOp>(loc, vec1, vec3);
-  return rewriter.create<vector::InterleaveOp>(loc, interleave02, interleave13);
+  Value interleave02 = vector::InterleaveOp::create(rewriter, loc, vec0, vec2);
+  Value interleave13 = vector::InterleaveOp::create(rewriter, loc, vec1, vec3);
+  return vector::InterleaveOp::create(rewriter, loc, interleave02, interleave13);
 }
 
 /// Rewrite the i8 -> i4 truncation into a deinterleave and series of bitwise
@@ -1887,29 +1888,29 @@ static Value rewriteI8ToI4Trunc(PatternRewriter &rewriter, Location loc,
          "Expected i8 type");
 
   // 1. De-interleave low and high i8 elements.
-  auto deinterleaveOp = rewriter.create<vector::DeinterleaveOp>(loc, srcValue);
+  auto deinterleaveOp = vector::DeinterleaveOp::create(rewriter, loc, srcValue);
 
   // 2. Zero out the upper side of each low i8 element.
   constexpr int8_t i8LowBitMask = 0x0F;
   VectorType deinterI8VecType = deinterleaveOp.getResultVectorType();
-  Value zeroOutMask = rewriter.create<arith::ConstantOp>(
+  Value zeroOutMask = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(deinterI8VecType, i8LowBitMask));
-  Value zeroOutLow = rewriter.create<arith::AndIOp>(
+  Value zeroOutLow = arith::AndIOp::create(rewriter,
       loc, deinterleaveOp.getRes1(), zeroOutMask);
 
   // 3. Move high i4 values to upper side of the byte.
   constexpr int8_t bitsToShift = 4;
-  auto shiftValues = rewriter.create<arith::ConstantOp>(
+  auto shiftValues = arith::ConstantOp::create(rewriter,
       loc, DenseElementsAttr::get(deinterI8VecType, bitsToShift));
-  Value shlHigh = rewriter.create<arith::ShLIOp>(loc, deinterleaveOp.getRes2(),
+  Value shlHigh = arith::ShLIOp::create(rewriter, loc, deinterleaveOp.getRes2(),
                                                  shiftValues);
 
   // 4. Merge high and low i4 values.
-  auto mergedHiLowOp = rewriter.create<arith::OrIOp>(loc, zeroOutLow, shlHigh);
+  auto mergedHiLowOp = arith::OrIOp::create(rewriter, loc, zeroOutLow, shlHigh);
 
   // 5. Generate a bitcast vector<Xxi8> -> vector<2Xxi4>.
   auto i4VecType = srcVecType.cloneWith(std::nullopt, rewriter.getI4Type());
-  return rewriter.create<vector::BitCastOp>(loc, i4VecType, mergedHiLowOp);
+  return vector::BitCastOp::create(rewriter, loc, i4VecType, mergedHiLowOp);
 }
 
 namespace {
@@ -2151,7 +2152,7 @@ struct RewriteAlignedSubByteIntTrunc : OpRewritePattern<arith::TruncIOp> {
     Location loc = truncOp.getLoc();
     auto i8VecType = srcVecType.cloneWith(std::nullopt, rewriter.getI8Type());
     Value i8TruncVal =
-        rewriter.create<arith::TruncIOp>(loc, i8VecType, srcValue);
+        arith::TruncIOp::create(rewriter, loc, i8VecType, srcValue);
 
     // Rewrite the i8 -> i4 truncation part.
     Value subByteTrunc = rewriteI8ToI4Trunc(rewriter, loc, i8TruncVal);
@@ -2199,9 +2200,9 @@ struct RewriteVectorTranspose : OpRewritePattern<vector::TransposeOp> {
     // support is available.
     auto srcNativeVecType = srcSubByteVecType.cloneWith(
         std::nullopt, rewriter.getIntegerType(minNativeBitwidth));
-    Value extOp = rewriter.create<arith::ExtSIOp>(loc, srcNativeVecType,
+    Value extOp = arith::ExtSIOp::create(rewriter, loc, srcNativeVecType,
                                                   transposeOp.getVector());
-    Value newTranspose = rewriter.create<vector::TransposeOp>(
+    Value newTranspose = vector::TransposeOp::create(rewriter,
         loc, extOp, transposeOp.getPermutation());
     VectorType dstSubByteVecType = transposeOp.getResultVectorType();
     rewriter.replaceOpWithNewOp<arith::TruncIOp>(transposeOp, dstSubByteVecType,
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
index d834a99076834..baeca29298334 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
@@ -53,14 +53,14 @@ class DecomposeDifferentRankInsertStridedSlice
     int64_t rankRest = dstType.getRank() - rankDiff;
     // Extract / insert the subvector of matching rank and InsertStridedSlice
     // on it.
-    Value extracted = rewriter.create<ExtractOp>(
+    Value extracted = ExtractOp::create(rewriter,
         loc, op.getDest(),
         getI64SubArray(op.getOffsets(), /*dropFront=*/0,
                        /*dropBack=*/rankRest));
 
     // A different pattern will kick in for InsertStridedSlice with matching
     // ranks.
-    auto stridedSliceInnerOp = rewriter.create<InsertStridedSliceOp>(
+    auto stridedSliceInnerOp = InsertStridedSliceOp::create(rewriter,
         loc, op.getValueToStore(), extracted,
         getI64SubArray(op.getOffsets(), /*dropFront=*/rankDiff),
         getI64SubArray(op.getStrides(), /*dropFront=*/0));
@@ -131,7 +131,7 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
       SmallVector<int64_t> offsets(nDest, 0);
       for (int64_t i = 0; i < nSrc; ++i)
         offsets[i] = i;
-      Value scaledSource = rewriter.create<ShuffleOp>(
+      Value scaledSource = ShuffleOp::create(rewriter,
           loc, op.getValueToStore(), op.getValueToStore(), offsets);
 
       // 2. Create a mask where we take the value from scaledSource of dest
@@ -156,21 +156,21 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
          off += stride, ++idx) {
       // 1. extract the proper subvector (or element) from source
       Value extractedSource =
-          rewriter.create<ExtractOp>(loc, op.getValueToStore(), idx);
+          ExtractOp::create(rewriter, loc, op.getValueToStore(), idx);
       if (isa<VectorType>(extractedSource.getType())) {
         // 2. If we have a vector, extract the proper subvector from destination
         // Otherwise we are at the element level and no need to recurse.
         Value extractedDest =
-            rewriter.create<ExtractOp>(loc, op.getDest(), off);
+            ExtractOp::create(rewriter, loc, op.getDest(), off);
         // 3. Reduce the problem to lowering a new InsertStridedSlice op with
         // smaller rank.
-        extractedSource = rewriter.create<InsertStridedSliceOp>(
+        extractedSource = InsertStridedSliceOp::create(rewriter,
             loc, extractedSource, extractedDest,
             getI64SubArray(op.getOffsets(), /* dropFront=*/1),
             getI64SubArray(op.getStrides(), /* dropFront=*/1));
       }
       // 4. Insert the extractedSource into the res vector.
-      res = rewriter.create<InsertOp>(loc, extractedSource, res, off);
+      res = InsertOp::create(rewriter, loc, extractedSource, res, off);
     }
 
     rewriter.replaceOp(op, res);
@@ -250,12 +250,12 @@ class Convert1DExtractStridedSliceIntoExtractInsertChain final
     SmallVector<Value> elements;
     elements.reserve(size);
     for (int64_t i = offset, e = offset + size * stride; i < e; i += stride)
-      elements.push_back(rewriter.create<ExtractOp>(loc, op.getVector(), i));
+      elements.push_back(ExtractOp::create(rewriter, loc, op.getVector(), i));
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, rewriter.getZeroAttr(op.getType()));
     for (int64_t i = 0; i < size; ++i)
-      result = rewriter.create<InsertOp>(loc, elements[i], result, i);
+      result = InsertOp::create(rewriter, loc, elements[i], result, i);
 
     rewriter.replaceOp(op, result);
     return success();
@@ -301,17 +301,17 @@ class DecomposeNDExtractStridedSlice
       return failure();
 
     // Extract/insert on a lower ranked extract strided slice op.
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value res = rewriter.create<SplatOp>(loc, dstType, zero);
+    Value res = SplatOp::create(rewriter, loc, dstType, zero);
     for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
          off += stride, ++idx) {
-      Value one = rewriter.create<ExtractOp>(loc, op.getVector(), off);
-      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+      Value one = ExtractOp::create(rewriter, loc, op.getVector(), off);
+      Value extracted = ExtractStridedSliceOp::create(rewriter,
           loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
           getI64SubArray(op.getSizes(), /* dropFront=*/1),
           getI64SubArray(op.getStrides(), /* dropFront=*/1));
-      res = rewriter.create<InsertOp>(loc, extracted, res, idx);
+      res = InsertOp::create(rewriter, loc, extracted, res, idx);
     }
     rewriter.replaceOp(op, res);
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
index 6d3f5697dfaa0..0de9848c52461 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
@@ -658,7 +658,7 @@ struct LinearizeVectorCreateMask final
     // The result of the comparison is then multiplied with
     // the second operand of create_mask to get the 1D mask.
     auto firstOperand = adaptor.getOperands().front();
-    auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto isNonZero = rewriter.createOrFold<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::sgt, firstOperand, zero);
     auto isNonZeroIndex = rewriter.createOrFold<mlir::arith::IndexCastOp>(
@@ -668,7 +668,7 @@ struct LinearizeVectorCreateMask final
         loc, rewriter.getIndexType(), isNonZeroIndex, secondOperand);
 
     auto newMask =
-        rewriter.create<mlir::vector::CreateMaskOp>(loc, dstTy, maskSize);
+        mlir::vector::CreateMaskOp::create(rewriter, loc, dstTy, maskSize);
     rewriter.replaceOp(createMaskOp, newMask);
     return success();
   }
@@ -745,7 +745,7 @@ void mlir::vector::populateForVectorLinearize(TypeConverter &typeConverter,
     if (!isa<VectorType>(type) || !isa<VectorType>(value.getType()))
       return nullptr;
 
-    return builder.create<vector::ShapeCastOp>(loc, type, value);
+    return vector::ShapeCastOp::create(builder, loc, type, value);
   };
   typeConverter.addSourceMaterialization(materializeCast);
   typeConverter.addTargetMaterialization(materializeCast);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
index 363108238e596..70d786a516859 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
@@ -84,7 +84,7 @@ LogicalResult resolveAllTrueCreateMaskOp(IRRewriter &rewriter,
   // Replace createMaskOp with an all-true constant. This should result in the
   // mask being removed in most cases (as xfer ops + vector.mask have folds to
   // remove all-true masks).
-  auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+  auto allTrue = vector::ConstantMaskOp::create(rewriter,
       createMaskOp.getLoc(), maskType, ConstantMaskKind::AllTrue);
   rewriter.replaceAllUsesWith(createMaskOp, allTrue);
   return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index c20a1b355996c..9bbedc396664b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -286,7 +286,7 @@ static Value rankReducingSubviewDroppingUnitDims(PatternRewriter &rewriter,
   if (resultType.canonicalizeStridedLayout() ==
       inputType.canonicalizeStridedLayout())
     return input;
-  return rewriter.create<memref::SubViewOp>(loc, resultType, input, offsets,
+  return memref::SubViewOp::create(rewriter, loc, resultType, input, offsets,
                                             sizes, strides);
 }
 
@@ -395,11 +395,11 @@ class TransferReadDropUnitDimsPattern
 
     Value reducedShapeSource =
         rankReducingSubviewDroppingUnitDims(rewriter, loc, source);
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
-    Operation *newTransferReadOp = rewriter.create<vector::TransferReadOp>(
+    Operation *newTransferReadOp = vector::TransferReadOp::create(rewriter,
         loc, reducedVectorType, reducedShapeSource, zeros, identityMap,
         transferReadOp.getPadding(), maskOp,
         rewriter.getBoolArrayAttr(inBounds));
@@ -477,13 +477,13 @@ class TransferWriteDropUnitDimsPattern
     }
     Value reducedShapeSource =
         rankReducingSubviewDroppingUnitDims(rewriter, loc, source);
-    Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
     auto shapeCastSrc = rewriter.createOrFold<vector::ShapeCastOp>(
         loc, reducedVectorType, vector);
-    Operation *newXferWrite = rewriter.create<vector::TransferWriteOp>(
+    Operation *newXferWrite = vector::TransferWriteOp::create(rewriter,
         loc, Type(), shapeCastSrc, reducedShapeSource, zeros, identityMap,
         maskOp, rewriter.getBoolArrayAttr(inBounds));
 
@@ -520,7 +520,7 @@ static Value collapseInnerDims(PatternRewriter &rewriter, mlir::Location loc,
   for (int64_t i = firstDimToCollapse; i < inputType.getRank(); ++i)
     collapsedIndices.push_back(i);
   reassociation.push_back(collapsedIndices);
-  return rewriter.create<memref::CollapseShapeOp>(loc, input, reassociation);
+  return memref::CollapseShapeOp::create(rewriter, loc, input, reassociation);
 }
 
 /// Returns the new indices that collapses the inner dimensions starting from
@@ -559,7 +559,7 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   // one would get the following offset:
   //    %offset = %arg0 * 43
   OpFoldResult collapsedOffset =
-      rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult();
+      arith::ConstantIndexOp::create(rewriter, loc, 0).getResult();
 
   auto collapsedStrides = computeSuffixProduct(
       ArrayRef<int64_t>(shape.begin() + firstDimToCollapse, shape.end()));
@@ -573,7 +573,7 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   if (auto value = dyn_cast<Value>(collapsedOffset)) {
     indicesAfterCollapsing.push_back(value);
   } else {
-    indicesAfterCollapsing.push_back(rewriter.create<arith::ConstantIndexOp>(
+    indicesAfterCollapsing.push_back(arith::ConstantIndexOp::create(rewriter,
         loc, *getConstantIntValue(collapsedOffset)));
   }
 
@@ -659,7 +659,7 @@ class FlattenContiguousRowMajorTransferReadPattern
     // 3. Create new vector.transfer_read that reads from the collapsed memref
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
-    vector::TransferReadOp flatRead = rewriter.create<vector::TransferReadOp>(
+    vector::TransferReadOp flatRead = vector::TransferReadOp::create(rewriter,
         loc, flatVectorType, collapsedSource, collapsedIndices,
         transferReadOp.getPadding(), collapsedMap);
     flatRead.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
@@ -757,9 +757,9 @@ class FlattenContiguousRowMajorTransferWritePattern
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
     Value flatVector =
-        rewriter.create<vector::ShapeCastOp>(loc, flatVectorType, vector);
+        vector::ShapeCastOp::create(rewriter, loc, flatVectorType, vector);
     vector::TransferWriteOp flatWrite =
-        rewriter.create<vector::TransferWriteOp>(
+        vector::TransferWriteOp::create(rewriter,
             loc, flatVector, collapsedSource, collapsedIndices, collapsedMap);
     flatWrite.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
@@ -846,7 +846,7 @@ class RewriteScalarExtractOfTransferRead
       if (auto value = dyn_cast<Value>(composedIdx)) {
         newIndices[idx] = value;
       } else {
-        newIndices[idx] = rewriter.create<arith::ConstantIndexOp>(
+        newIndices[idx] = arith::ConstantIndexOp::create(rewriter,
             extractOp.getLoc(), *getConstantIntValue(composedIdx));
       }
     }
@@ -884,7 +884,7 @@ class RewriteScalarWrite : public OpRewritePattern<vector::TransferWriteOp> {
       return failure();
     // Only float and integer element types are supported.
     Value scalar =
-        rewriter.create<vector::ExtractOp>(xferOp.getLoc(), xferOp.getVector());
+        vector::ExtractOp::create(rewriter, xferOp.getLoc(), xferOp.getVector());
     // Construct a scalar store.
     if (isa<MemRefType>(xferOp.getBase().getType())) {
       rewriter.replaceOpWithNewOp<memref::StoreOp>(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
index 256c8cb69b1ba..a1f36a9744932 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
@@ -64,12 +64,12 @@ static Value createInBoundsCond(RewriterBase &b,
     if (maybeCstSum && maybeCstDimSz && *maybeCstSum <= *maybeCstDimSz)
       return;
     Value cond =
-        b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::sle,
+        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sle,
                                 getValueOrCreateConstantIndexOp(b, loc, sum),
                                 getValueOrCreateConstantIndexOp(b, loc, dimSz));
     // Conjunction over all dims for which we are in-bounds.
     if (inBoundsCond)
-      inBoundsCond = b.create<arith::AndIOp>(loc, inBoundsCond, cond);
+      inBoundsCond = arith::AndIOp::create(b, loc, inBoundsCond, cond);
     else
       inBoundsCond = cond;
   });
@@ -177,11 +177,11 @@ static Value castToCompatibleMemRefType(OpBuilder &b, Value memref,
     sourceType = MemRefType::get(
         sourceType.getShape(), sourceType.getElementType(),
         sourceType.getLayout(), compatibleMemRefType.getMemorySpace());
-    res = b.create<memref::MemorySpaceCastOp>(memref.getLoc(), sourceType, res);
+    res = memref::MemorySpaceCastOp::create(b, memref.getLoc(), sourceType, res);
   }
   if (sourceType == compatibleMemRefType)
     return res;
-  return b.create<memref::CastOp>(memref.getLoc(), compatibleMemRefType, res);
+  return memref::CastOp::create(b, memref.getLoc(), compatibleMemRefType, res);
 }
 
 /// Operates under a scoped context to build the intersection between the
@@ -203,15 +203,15 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
   xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
     using MapList = ArrayRef<ArrayRef<AffineExpr>>;
     Value dimMemRef =
-        b.create<memref::DimOp>(xferOp.getLoc(), xferOp.getBase(), indicesIdx);
-    Value dimAlloc = b.create<memref::DimOp>(loc, alloc, resultIdx);
+        memref::DimOp::create(b, xferOp.getLoc(), xferOp.getBase(), indicesIdx);
+    Value dimAlloc = memref::DimOp::create(b, loc, alloc, resultIdx);
     Value index = xferOp.getIndices()[indicesIdx];
     AffineExpr i, j, k;
     bindDims(xferOp.getContext(), i, j, k);
     SmallVector<AffineMap, 4> maps =
         AffineMap::inferFromExprList(MapList{{i - j, k}}, b.getContext());
     // affine_min(%dimMemRef - %index, %dimAlloc)
-    Value affineMin = b.create<affine::AffineMinOp>(
+    Value affineMin = affine::AffineMinOp::create(b,
         loc, index.getType(), maps[0], ValueRange{dimMemRef, index, dimAlloc});
     sizes.push_back(affineMin);
   });
@@ -220,9 +220,9 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
       xferOp.getIndices(), [](Value idx) -> OpFoldResult { return idx; }));
   SmallVector<OpFoldResult> destIndices(memrefRank, b.getIndexAttr(0));
   SmallVector<OpFoldResult> strides(memrefRank, b.getIndexAttr(1));
-  auto copySrc = b.create<memref::SubViewOp>(
+  auto copySrc = memref::SubViewOp::create(b,
       loc, isaWrite ? alloc : xferOp.getBase(), srcIndices, sizes, strides);
-  auto copyDest = b.create<memref::SubViewOp>(
+  auto copyDest = memref::SubViewOp::create(b,
       loc, isaWrite ? xferOp.getBase() : alloc, destIndices, sizes, strides);
   return std::make_pair(copySrc, copyDest);
 }
@@ -251,18 +251,18 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
                             TypeRange returnTypes, Value inBoundsCond,
                             MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return b.create<scf::IfOp>(
+  return scf::IfOp::create(b,
       loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
         llvm::append_range(viewAndIndices, xferOp.getIndices());
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       },
       [&](OpBuilder &b, Location loc) {
-        b.create<linalg::FillOp>(loc, ValueRange{xferOp.getPadding()},
+        linalg::FillOp::create(b, loc, ValueRange{xferOp.getPadding()},
                                  ValueRange{alloc});
         // Take partial subview of memref which guarantees no dimension
         // overflows.
@@ -270,13 +270,13 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
         std::pair<Value, Value> copyArgs = createSubViewIntersection(
             rewriter, cast<VectorTransferOpInterface>(xferOp.getOperation()),
             alloc);
-        b.create<memref::CopyOp>(loc, copyArgs.first, copyArgs.second);
+        memref::CopyOp::create(b, loc, copyArgs.first, copyArgs.second);
         Value casted =
             castToCompatibleMemRefType(b, alloc, compatibleMemRefType);
         scf::ValueVector viewAndIndices{casted};
         viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
                               zero);
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       });
 }
 
@@ -304,22 +304,22 @@ static scf::IfOp createFullPartialVectorTransferRead(
     Value inBoundsCond, MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
   scf::IfOp fullPartialIfOp;
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return b.create<scf::IfOp>(
+  return scf::IfOp::create(b,
       loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
         llvm::append_range(viewAndIndices, xferOp.getIndices());
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       },
       [&](OpBuilder &b, Location loc) {
         Operation *newXfer = b.clone(*xferOp.getOperation());
         Value vector = cast<VectorTransferOpInterface>(newXfer).getVector();
-        b.create<memref::StoreOp>(
+        memref::StoreOp::create(b,
             loc, vector,
-            b.create<vector::TypeCastOp>(
+            vector::TypeCastOp::create(b,
                 loc, MemRefType::get({}, vector.getType()), alloc));
 
         Value casted =
@@ -327,7 +327,7 @@ static scf::IfOp createFullPartialVectorTransferRead(
         scf::ValueVector viewAndIndices{casted};
         viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
                               zero);
-        b.create<scf::YieldOp>(loc, viewAndIndices);
+        scf::YieldOp::create(b, loc, viewAndIndices);
       });
 }
 
@@ -351,7 +351,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
                           TypeRange returnTypes, Value inBoundsCond,
                           MemRefType compatibleMemRefType, Value alloc) {
   Location loc = xferOp.getLoc();
-  Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
+  Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
   return b
       .create<scf::IfOp>(
@@ -361,7 +361,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
                 castToCompatibleMemRefType(b, memref, compatibleMemRefType);
             scf::ValueVector viewAndIndices{res};
             llvm::append_range(viewAndIndices, xferOp.getIndices());
-            b.create<scf::YieldOp>(loc, viewAndIndices);
+            scf::YieldOp::create(b, loc, viewAndIndices);
           },
           [&](OpBuilder &b, Location loc) {
             Value casted =
@@ -369,7 +369,7 @@ getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
             scf::ValueVector viewAndIndices{casted};
             viewAndIndices.insert(viewAndIndices.end(),
                                   xferOp.getTransferRank(), zero);
-            b.create<scf::YieldOp>(loc, viewAndIndices);
+            scf::YieldOp::create(b, loc, viewAndIndices);
           })
       ->getResults();
 }
@@ -391,15 +391,15 @@ static void createFullPartialLinalgCopy(RewriterBase &b,
                                         vector::TransferWriteOp xferOp,
                                         Value inBoundsCond, Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = b.create<arith::XOrIOp>(
-      loc, inBoundsCond, b.create<arith::ConstantIntOp>(loc, true, 1));
-  b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
+  auto notInBounds = arith::XOrIOp::create(b,
+      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRRewriter rewriter(b);
     std::pair<Value, Value> copyArgs = createSubViewIntersection(
         rewriter, cast<VectorTransferOpInterface>(xferOp.getOperation()),
         alloc);
-    b.create<memref::CopyOp>(loc, copyArgs.first, copyArgs.second);
-    b.create<scf::YieldOp>(loc, ValueRange{});
+    memref::CopyOp::create(b, loc, copyArgs.first, copyArgs.second);
+    scf::YieldOp::create(b, loc, ValueRange{});
   });
 }
 
@@ -420,18 +420,18 @@ static void createFullPartialVectorTransferWrite(RewriterBase &b,
                                                  Value inBoundsCond,
                                                  Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = b.create<arith::XOrIOp>(
-      loc, inBoundsCond, b.create<arith::ConstantIntOp>(loc, true, 1));
-  b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
+  auto notInBounds = arith::XOrIOp::create(b,
+      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRMapping mapping;
-    Value load = b.create<memref::LoadOp>(
+    Value load = memref::LoadOp::create(b,
         loc,
-        b.create<vector::TypeCastOp>(
+        vector::TypeCastOp::create(b,
             loc, MemRefType::get({}, xferOp.getVector().getType()), alloc),
         ValueRange());
     mapping.map(xferOp.getVector(), load);
     b.clone(*xferOp.getOperation(), mapping);
-    b.create<scf::YieldOp>(loc, ValueRange{});
+    scf::YieldOp::create(b, loc, ValueRange{});
   });
 }
 
@@ -561,7 +561,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
     b.setInsertionPointToStart(&scope->getRegion(0).front());
     auto shape = xferOp.getVectorType().getShape();
     Type elementType = xferOp.getVectorType().getElementType();
-    alloc = b.create<memref::AllocaOp>(scope->getLoc(),
+    alloc = memref::AllocaOp::create(b, scope->getLoc(),
                                        MemRefType::get(shape, elementType),
                                        ValueRange{}, b.getI64IntegerAttr(32));
   }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index fe2707629d82e..d318b4b16b5aa 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -381,7 +381,7 @@ FailureOr<Value> combineContractAndBroadcast(vector::ContractionOp contractOp,
   if (getUnusedDimsBitVector({maps[0], maps[1]}).any())
     return failure();
 
-  Operation *newOp = rewriter.create<vector::ContractionOp>(
+  Operation *newOp = vector::ContractionOp::create(rewriter,
       contractOp.getLoc(), lhs, rhs, contractOp.getAcc(),
       rewriter.getAffineMapArrayAttr(maps), rewriter.getArrayAttr(iterators));
 
@@ -534,7 +534,7 @@ struct ReorderElementwiseOpsOnTranspose final
         // This is a constant. Create a reverse transpose op for it.
         auto vectorType =
             srcType.clone(cast<VectorType>(operand.getType()).getElementType());
-        srcValues.push_back(rewriter.create<vector::TransposeOp>(
+        srcValues.push_back(vector::TransposeOp::create(rewriter,
             operand.getLoc(), vectorType, operand, invOrder));
       }
     }
@@ -608,12 +608,12 @@ struct BubbleDownVectorBitCastForExtract
     // Get the single scalar (as a vector) in the source value that packs the
     // desired scalar. E.g. extract vector<1xf32> from vector<4xf32>
     Location loc = extractOp.getLoc();
-    Value packedValue = rewriter.create<vector::ExtractOp>(
+    Value packedValue = vector::ExtractOp::create(rewriter,
         loc, castOp.getSource(), index / expandRatio);
     Type packedVecType = VectorType::get(/*shape=*/{1}, packedValue.getType());
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, packedVecType, rewriter.getZeroAttr(packedVecType));
-    packedValue = rewriter.create<vector::InsertOp>(loc, packedValue, zero,
+    packedValue = vector::InsertOp::create(rewriter, loc, packedValue, zero,
                                                     /*position=*/0);
 
     // Cast it to a vector with the desired scalar's type.
@@ -621,7 +621,7 @@ struct BubbleDownVectorBitCastForExtract
     VectorType packedType =
         VectorType::get({expandRatio}, castDstType.getElementType());
     Value castedValue =
-        rewriter.create<vector::BitCastOp>(loc, packedType, packedValue);
+        vector::BitCastOp::create(rewriter, loc, packedType, packedValue);
 
     // Finally extract the desired scalar.
     rewriter.replaceOpWithNewOp<vector::ExtractOp>(extractOp, castedValue,
@@ -700,7 +700,7 @@ struct BubbleDownBitCastForStridedSliceExtract
     VectorType newExtractType =
         VectorType::get(dims, castSrcType.getElementType());
 
-    auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
         extractOp.getLoc(), newExtractType, castOp.getSource(), newOffsets,
         newSizes, extractOp.getStrides());
 
@@ -761,7 +761,7 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         isNumElemsShrink ? srcDims.back() / ratio : srcDims.back() * ratio;
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
-    auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
+    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims(insertOp.getDestVectorType().getShape());
@@ -771,7 +771,7 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(dstDims, castDstType.getElementType());
 
     // Bitcast the destination.
-    auto newCastDstOp = rewriter.create<vector::BitCastOp>(
+    auto newCastDstOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     // Generate new insert.
@@ -852,7 +852,7 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
 
-    auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
+    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims =
@@ -861,7 +861,7 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastDstType =
         VectorType::get(dstDims, castDstType.getElementType());
 
-    auto newCastDstOp = rewriter.create<vector::BitCastOp>(
+    auto newCastDstOp = vector::BitCastOp::create(rewriter,
         bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     rewriter.replaceOpWithNewOp<vector::InsertStridedSliceOp>(
@@ -936,9 +936,9 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
     Type elemType = castDstType.getElementType();
     assert(elemType.isSignlessIntOrIndexOrFloat());
 
-    Value zero = rewriter.create<arith::ConstantOp>(
+    Value zero = arith::ConstantOp::create(rewriter,
         loc, elemType, rewriter.getZeroAttr(elemType));
-    Value res = rewriter.create<SplatOp>(loc, castDstType, zero);
+    Value res = SplatOp::create(rewriter, loc, castDstType, zero);
 
     SmallVector<int64_t> sliceShape = {castDstLastDim};
     SmallVector<int64_t> strides = {1};
@@ -947,12 +947,12 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
                         castDstType.getElementType());
 
     for (int i = 0, e = shrinkRatio; i < e; ++i) {
-      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+      Value extracted = ExtractStridedSliceOp::create(rewriter,
           loc, bitcastOp.getSource(), ArrayRef<int64_t>{i * castDstLastDim},
           sliceShape, strides);
       Value bitcast =
-          rewriter.create<BitCastOp>(loc, newCastDstType, extracted);
-      res = rewriter.create<InsertStridedSliceOp>(
+          BitCastOp::create(rewriter, loc, newCastDstType, extracted);
+      res = InsertStridedSliceOp::create(rewriter,
           loc, bitcast, res,
           ArrayRef<int64_t>{i * castDstLastDim / shrinkRatio}, strides);
     }
@@ -1103,7 +1103,7 @@ class ExtractOpFromElementwise final
     Location loc = eltwise->getLoc();
     SmallVector<OpFoldResult> pos = op.getMixedPosition();
     for (Value arg : eltwise->getOperands()) {
-      Value newArg = rewriter.create<vector::ExtractOp>(loc, arg, pos);
+      Value newArg = vector::ExtractOp::create(rewriter, loc, arg, pos);
       mapping.map(arg, newArg);
     }
 
@@ -1292,18 +1292,18 @@ static Value buildVectorComparison(PatternRewriter &rewriter, Operation *op,
     indicesAttr = rewriter.getI64VectorAttr(
         llvm::to_vector<4>(llvm::seq<int64_t>(0, dim)));
   }
-  Value indices = rewriter.create<arith::ConstantOp>(loc, indicesAttr);
+  Value indices = arith::ConstantOp::create(rewriter, loc, indicesAttr);
   // Add in an offset if requested.
   if (off) {
     Value o = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, *off);
-    Value ov = rewriter.create<vector::SplatOp>(loc, indices.getType(), o);
-    indices = rewriter.create<arith::AddIOp>(loc, ov, indices);
+    Value ov = vector::SplatOp::create(rewriter, loc, indices.getType(), o);
+    indices = arith::AddIOp::create(rewriter, loc, ov, indices);
   }
   // Construct the vector comparison.
   Value bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, b);
   Value bounds =
-      rewriter.create<vector::SplatOp>(loc, indices.getType(), bound);
-  return rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::slt, indices,
+      vector::SplatOp::create(rewriter, loc, indices.getType(), bound);
+  return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, indices,
                                         bounds);
 }
 
@@ -1335,15 +1335,15 @@ struct MaterializeTransferMask : public OpRewritePattern<ConcreteOp> {
     Value off = xferOp.getIndices()[lastIndex];
     Value dim =
         vector::createOrFoldDimOp(rewriter, loc, xferOp.getBase(), lastIndex);
-    Value b = rewriter.create<arith::SubIOp>(loc, dim.getType(), dim, off);
-    Value mask = rewriter.create<vector::CreateMaskOp>(
+    Value b = arith::SubIOp::create(rewriter, loc, dim.getType(), dim, off);
+    Value mask = vector::CreateMaskOp::create(rewriter,
         loc,
         VectorType::get(vtp.getShape(), rewriter.getI1Type(),
                         vtp.getScalableDims()),
         b);
     if (xferOp.getMask()) {
       // Intersect the in-bounds with the mask specified as an op parameter.
-      mask = rewriter.create<arith::AndIOp>(loc, mask, xferOp.getMask());
+      mask = arith::AndIOp::create(rewriter, loc, mask, xferOp.getMask());
     }
 
     rewriter.modifyOpInPlace(xferOp, [&]() {
@@ -1548,11 +1548,11 @@ class DropInnerMostUnitDimsTransferRead
         strides);
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         readOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
-    Value rankedReducedView = rewriter.create<memref::SubViewOp>(
+    Value rankedReducedView = memref::SubViewOp::create(rewriter,
         loc, resultMemrefType, readOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
-    Value result = rewriter.create<vector::TransferReadOp>(
+    Value result = vector::TransferReadOp::create(rewriter,
         loc, resultTargetVecType, rankedReducedView,
         readOp.getIndices().drop_back(dimsToDrop), AffineMapAttr::get(permMap),
         readOp.getPadding(),
@@ -1639,7 +1639,7 @@ class DropInnerMostUnitDimsTransferWrite
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         writeOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
 
-    Value rankedReducedView = rewriter.create<memref::SubViewOp>(
+    Value rankedReducedView = memref::SubViewOp::create(rewriter,
         loc, resultMemrefType, writeOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
@@ -1708,21 +1708,21 @@ struct CanonicalizeContractMatmulToMMT final
     auto createTranspose = [&rewriter, loc](Value mat) -> Value {
       if (auto sext = mat.getDefiningOp<arith::ExtSIOp>()) {
         Value trans =
-            rewriter.create<vector::TransposeOp>(loc, sext.getIn(), perm);
+            vector::TransposeOp::create(rewriter, loc, sext.getIn(), perm);
         VectorType newType =
             cast<VectorType>(trans.getType())
                 .clone(cast<VectorType>(mat.getType()).getElementType());
-        return rewriter.create<arith::ExtSIOp>(loc, newType, trans);
+        return arith::ExtSIOp::create(rewriter, loc, newType, trans);
       }
       if (auto zext = mat.getDefiningOp<arith::ExtUIOp>()) {
         Value trans =
-            rewriter.create<vector::TransposeOp>(loc, zext.getIn(), perm);
+            vector::TransposeOp::create(rewriter, loc, zext.getIn(), perm);
         VectorType newType =
             VectorType::get(cast<VectorType>(trans.getType()).getShape(),
                             cast<VectorType>(mat.getType()).getElementType());
-        return rewriter.create<arith::ExtUIOp>(loc, newType, trans);
+        return arith::ExtUIOp::create(rewriter, loc, newType, trans);
       }
-      return rewriter.create<vector::TransposeOp>(loc, mat, perm);
+      return vector::TransposeOp::create(rewriter, loc, mat, perm);
     };
 
     if (maps == infer({{m, k}, {k, n}, {m, n}})) {
@@ -1836,7 +1836,7 @@ struct ChainedReduction final : OpRewritePattern<vector::ReductionOp> {
       vAdd = rewriter.createOrFold<arith::AddIOp>(
           loc, parentReduction.getVector(), op.getVector());
     } else {
-      vAdd = rewriter.create<arith::AddFOp>(loc, parentReduction.getVector(),
+      vAdd = arith::AddFOp::create(rewriter, loc, parentReduction.getVector(),
                                             op.getVector());
     }
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), vAdd,
@@ -1925,7 +1925,7 @@ struct DropUnitDimFromElementwiseOps final
       if (newVType == opVectorType)
         return rewriter.notifyMatchFailure(op, "No unit dimension to remove.");
 
-      auto opSC = rewriter.create<vector::ShapeCastOp>(loc, newVType, operand);
+      auto opSC = vector::ShapeCastOp::create(rewriter, loc, newVType, operand);
       newOperands.push_back(opSC);
     }
 
@@ -2004,11 +2004,11 @@ struct DropUnitDimsFromTransposeOp final
 
     Location loc = op.getLoc();
     // Drop the unit dims via shape_cast.
-    auto dropDimsShapeCast = rewriter.create<vector::ShapeCastOp>(
+    auto dropDimsShapeCast = vector::ShapeCastOp::create(rewriter,
         loc, sourceTypeWithoutUnitDims, op.getVector());
     // Create the new transpose.
     auto transposeWithoutUnitDims =
-        rewriter.create<vector::TransposeOp>(loc, dropDimsShapeCast, newPerm);
+        vector::TransposeOp::create(rewriter, loc, dropDimsShapeCast, newPerm);
     // Restore the unit dims via shape cast.
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
         op, op.getResultVectorType(), transposeWithoutUnitDims);
@@ -2059,7 +2059,7 @@ struct DropUnitDimsFromScfForOp final : OpRewritePattern<scf::ForOp> {
 
       // Create a new ForOp with that iter operand replaced.
       auto castFn = [](OpBuilder &b, Location loc, Type type, Value source) {
-        return b.create<vector::ShapeCastOp>(loc, type, source);
+        return vector::ShapeCastOp::create(b, loc, type, source);
       };
 
       Value replacement =
@@ -2111,7 +2111,7 @@ struct ReduceRedundantZero final : OpRewritePattern<vector::ReductionOp> {
     if (!matchPattern(addLhs.getRhs(), m_AnyZeroFloat()))
       return failure();
 
-    auto newAdd = rewriter.create<arith::AddFOp>(vAdd.getLoc(), addLhs.getLhs(),
+    auto newAdd = arith::AddFOp::create(rewriter, vAdd.getLoc(), addLhs.getLhs(),
                                                  vAdd.getRhs());
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), newAdd,
                                                      op.getAcc());
@@ -2154,7 +2154,7 @@ struct BreakDownVectorReduction final : OpRewritePattern<vector::ReductionOp> {
     Location loc = op.getLoc();
     SmallVector<Value> extracted(numElems, nullptr);
     for (auto [idx, extractedElem] : llvm::enumerate(extracted))
-      extractedElem = rewriter.create<vector::ExtractOp>(
+      extractedElem = vector::ExtractOp::create(rewriter,
           loc, op.getVector(), static_cast<int64_t>(idx));
 
     Value res = extracted.front();
@@ -2234,7 +2234,7 @@ struct FoldArithToVectorOuterProduct : public OpRewritePattern<MulOpType> {
       if (!broadcastedRhs || !isValidBroadcastSource(broadcastedRhs))
         return failure();
 
-      return rewriter.create<vector::OuterProductOp>(
+      return vector::OuterProductOp::create(rewriter,
           mulOp->getLoc(), resType, broadcastedLhs.getSource(),
           broadcastedRhs.getSource(), Value(), vector::CombiningKind::ADD);
     };
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 693f4f955994d..9fa01061c1bed 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -49,7 +49,7 @@ static SmallVector<Value> sliceTransferIndices(ArrayRef<int64_t> elementOffsets,
                 getAffineConstantExpr(elementOffsets[dim.index()], ctx);
     auto map = AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, expr);
     slicedIndices[pos] =
-        builder.create<affine::AffineApplyOp>(loc, map, indices[pos]);
+        affine::AffineApplyOp::create(builder, loc, map, indices[pos]);
   }
   return slicedIndices;
 }
@@ -68,9 +68,9 @@ static SmallVector<Value> sliceLoadStoreIndices(PatternRewriter &rewriter,
   auto start = indices.size() - offsets.size();
   for (auto [i, offset] : llvm::enumerate(offsets)) {
     if (offset != 0) {
-      indices[start + i] = rewriter.create<arith::AddIOp>(
+      indices[start + i] = arith::AddIOp::create(rewriter,
           loc, originalIndices[start + i],
-          rewriter.create<arith::ConstantIndexOp>(loc, offset));
+          arith::ConstantIndexOp::create(rewriter, loc, offset));
     }
   }
   return indices;
@@ -172,7 +172,7 @@ struct UnrollTransferReadPattern
     ArrayRef<int64_t> originalSize = readOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
@@ -185,7 +185,7 @@ struct UnrollTransferReadPattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                readOp.getPermutationMap(), loc, rewriter);
-      auto slicedRead = rewriter.create<vector::TransferReadOp>(
+      auto slicedRead = vector::TransferReadOp::create(rewriter,
           loc, targetType, readOp.getBase(), indices,
           readOp.getPermutationMapAttr(), readOp.getPadding(), readOp.getMask(),
           readOp.getInBoundsAttr());
@@ -236,7 +236,7 @@ struct UnrollTransferWritePattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                writeOp.getPermutationMap(), loc, rewriter);
-      Operation *slicedWrite = rewriter.create<vector::TransferWriteOp>(
+      Operation *slicedWrite = vector::TransferWriteOp::create(rewriter,
           loc, slicedVector, resultTensor ? resultTensor : writeOp.getBase(),
           indices, writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
       // For the tensor case update the destination for the next transfer write.
@@ -348,7 +348,7 @@ struct UnrollContractionPattern
       accCache[dstOffets] = newOp->getResult(0);
     }
     // Assemble back the accumulator into a single vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
@@ -427,7 +427,7 @@ struct UnrollMultiReductionPattern
       accCache[destOffset] = result;
     }
     // Assemble back the accumulator into a single vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, reductionOp.getDestType(),
         rewriter.getZeroAttr(reductionOp.getDestType()));
     for (const auto &it : accCache) {
@@ -468,7 +468,7 @@ struct UnrollElementwisePattern : public RewritePattern {
           op, "expected input vector rank to match target shape rank");
     Location loc = op->getLoc();
     // Prepare the result vector.
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, dstVecType, rewriter.getZeroAttr(dstVecType));
     SmallVector<int64_t> strides(targetShape->size(), 1);
     VectorType newVecType =
@@ -567,7 +567,7 @@ struct UnrollTransposePattern : public OpRewritePattern<vector::TransposeOp> {
     ArrayRef<int64_t> originalSize = originalVectorType.getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, originalVectorType, rewriter.getZeroAttr(originalVectorType));
     ArrayRef<int64_t> permutation = transposeOp.getPermutation();
 
@@ -618,7 +618,7 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
     ArrayRef<int64_t> originalSize = gatherOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
@@ -638,7 +638,7 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
           rewriter.createOrFold<vector::ExtractStridedSliceOp>(
               loc, gatherOp.getPassThru(), elementOffsets, *targetShape,
               strides);
-      auto slicedGather = rewriter.create<vector::GatherOp>(
+      auto slicedGather = vector::GatherOp::create(rewriter,
           loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
           indexSubVec, maskSubVec, passThruSubVec);
 
@@ -671,7 +671,7 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
     ArrayRef<int64_t> originalShape = vecType.getShape();
     SmallVector<int64_t> strides(targetShape->size(), 1);
 
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, vecType, rewriter.getZeroAttr(vecType));
 
     SmallVector<int64_t> loopOrder =
@@ -684,7 +684,7 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
          StaticTileOffsetRange(originalShape, *targetShape, loopOrder)) {
       SmallVector<Value> indices =
           sliceLoadStoreIndices(rewriter, loc, loadOp.getIndices(), offsets);
-      Value slicedLoad = rewriter.create<vector::LoadOp>(
+      Value slicedLoad = vector::LoadOp::create(rewriter,
           loc, targetVecType, loadOp.getBase(), indices);
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, slicedLoad, result, offsets, strides);
@@ -727,7 +727,7 @@ struct UnrollStorePattern : public OpRewritePattern<vector::StoreOp> {
           sliceLoadStoreIndices(rewriter, loc, storeOp.getIndices(), offsets);
       Value slice = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
           loc, vector, offsets, *targetShape, strides);
-      rewriter.create<vector::StoreOp>(loc, slice, base, indices);
+      vector::StoreOp::create(rewriter, loc, slice, base, indices);
     }
     rewriter.eraseOp(storeOp);
     return success();
@@ -755,7 +755,7 @@ struct UnrollBroadcastPattern : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType = broadcastOp.getResultVectorType();
     VectorType targetType =
         resType.cloneWith(*targetShape, resType.getElementType());
-    Value result = rewriter.create<arith::ConstantOp>(
+    Value result = arith::ConstantOp::create(rewriter,
         loc, resType, rewriter.getZeroAttr(resType));
 
     SmallVector<int64_t> originalShape = *broadcastOp.getShapeForUnroll();
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 7e4984582b373..7c31ada1644c5 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -331,7 +331,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
   assert(padValue.getType() == sourceShapedType.getElementType() &&
          "expected same pad element type to match source element type");
   int64_t readRank = inputVectorSizes.size();
-  auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+  auto zero = arith::ConstantIndexOp::create(builder, loc, 0);
   SmallVector<bool> inBoundsVal(readRank, true);
 
   if (useInBoundsInsteadOfMasking) {
@@ -341,7 +341,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
       inBoundsVal[i] = (sourceShape[i] == inputVectorSizes[i]) &&
                        ShapedType::isStatic(sourceShape[i]);
   }
-  auto transferReadOp = builder.create<vector::TransferReadOp>(
+  auto transferReadOp = vector::TransferReadOp::create(builder,
       loc,
       /*vectorType=*/vectorType,
       /*source=*/source,
@@ -356,7 +356,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
 
   auto maskType = VectorType::get(inputVectorSizes, builder.getI1Type());
   Value mask =
-      builder.create<vector::CreateMaskOp>(loc, maskType, mixedSourceDims);
+      vector::CreateMaskOp::create(builder, loc, maskType, mixedSourceDims);
   return mlir::vector::maskOperation(builder, transferReadOp, mask)
       ->getResult(0);
 }
diff --git a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
index 179e6ee8784e6..01acade66a3c4 100644
--- a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
+++ b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
@@ -63,11 +63,11 @@ SmallVector<Value> x86vector::MaskCompressOp::getIntrinsicOperands(
   if (adaptor.getSrc()) {
     src = adaptor.getSrc();
   } else if (adaptor.getConstantSrc()) {
-    src = rewriter.create<LLVM::ConstantOp>(loc, opType,
+    src = LLVM::ConstantOp::create(rewriter, loc, opType,
                                             adaptor.getConstantSrcAttr());
   } else {
     auto zeroAttr = rewriter.getZeroAttr(opType);
-    src = rewriter.create<LLVM::ConstantOp>(loc, opType, zeroAttr);
+    src = LLVM::ConstantOp::create(rewriter, loc, opType, zeroAttr);
   }
 
   return SmallVector<Value>{adaptor.getA(), src, adaptor.getK()};
@@ -80,7 +80,7 @@ x86vector::DotOp::getIntrinsicOperands(ArrayRef<Value> operands,
   SmallVector<Value> intrinsicOperands(operands);
   // Dot product of all elements, broadcasted to all elements.
   Value scale =
-      rewriter.create<LLVM::ConstantOp>(getLoc(), rewriter.getI8Type(), 0xff);
+      LLVM::ConstantOp::create(rewriter, getLoc(), rewriter.getI8Type(), 0xff);
   intrinsicOperands.push_back(scale);
 
   return intrinsicOperands;
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
index 63f725084cabf..8f87661a68d96 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
+++ b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
@@ -38,7 +38,7 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
       "=x,x,x"; // Careful: constraint parser is very brittle: no ws!
   SmallVector<Value> asmVals{v1, v2};
   auto asmStr = llvm::formatv(asmTp, llvm::format_hex(mask, /*width=*/2)).str();
-  auto asmOp = b.create<LLVM::InlineAsmOp>(
+  auto asmOp = LLVM::InlineAsmOp::create(b,
       v1.getType(), /*operands=*/asmVals, /*asm_string=*/asmStr,
       /*constraints=*/asmCstr, /*has_side_effects=*/false,
       /*is_align_stack=*/false, LLVM::TailCallKind::None,
@@ -49,13 +49,13 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackLoPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2, ArrayRef<int64_t>{0, 8, 1, 9, 4, 12, 5, 13});
 }
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackHiPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return b.create<vector::ShuffleOp>(
+  return vector::ShuffleOp::create(b,
       v1, v2, ArrayRef<int64_t>{2, 10, 3, 11, 6, 14, 7, 15});
 }
 ///                            a  a   b   b  a  a   b   b
@@ -69,7 +69,7 @@ Value mlir::x86vector::avx2::intrin::mm256ShufflePs(ImplicitLocOpBuilder &b,
   MaskHelper::extractShuffle(mask, b01, b23, b45, b67);
   SmallVector<int64_t> shuffleMask = {
       b01, b23, b45 + 8, b67 + 8, b01 + 4, b23 + 4, b45 + 8 + 4, b67 + 8 + 4};
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 // imm[0:1] out of imm[0:3] is:
@@ -97,7 +97,7 @@ Value mlir::x86vector::avx2::intrin::mm256Permute2f128Ps(
   MaskHelper::extractPermute(mask, b03, b47);
   appendToMask(b03);
   appendToMask(b47);
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// If bit i of `mask` is zero, take f32 at i from v1 else take it from v2.
@@ -109,7 +109,7 @@ Value mlir::x86vector::avx2::intrin::mm256BlendPs(ImplicitLocOpBuilder &b,
     bool isSet = mask & (1 << i);
     shuffleMask.push_back(!isSet ? i : i + 8);
   }
-  return b.create<vector::ShuffleOp>(v1, v2, shuffleMask);
+  return vector::ShuffleOp::create(b, v1, v2, shuffleMask);
 }
 
 /// AVX2 4x8xf32-specific transpose lowering using a "C intrinsics" model.
@@ -245,13 +245,13 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
           VectorType::get({n * m}, op.getSourceVectorType().getElementType());
       auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
       auto reshInput =
-          ib.create<vector::ShapeCastOp>(flattenedType, op.getVector());
-      reshInput = ib.create<vector::ShapeCastOp>(reshInputType, reshInput);
+          vector::ShapeCastOp::create(ib, flattenedType, op.getVector());
+      reshInput = vector::ShapeCastOp::create(ib, reshInputType, reshInput);
 
       // Extract 1-D vectors from the higher-order dimension of the input
       // vector.
       for (int64_t i = 0; i < m; ++i)
-        vs.push_back(ib.create<vector::ExtractOp>(reshInput, i));
+        vs.push_back(vector::ExtractOp::create(ib, reshInput, i));
 
       // Transpose set of 1-D vectors.
       if (m == 4)
@@ -261,16 +261,16 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
 
       // Insert transposed 1-D vectors into the higher-order dimension of the
       // output vector.
-      Value res = ib.create<arith::ConstantOp>(reshInputType,
+      Value res = arith::ConstantOp::create(ib, reshInputType,
                                                ib.getZeroAttr(reshInputType));
       for (int64_t i = 0; i < m; ++i)
-        res = ib.create<vector::InsertOp>(vs[i], res, i);
+        res = vector::InsertOp::create(ib, vs[i], res, i);
 
       // The output vector still has the shape of the input vector (e.g., 4x8).
       // We have to transpose their dimensions and retrieve its original rank
       // (e.g., 1x8x1x4x1).
-      res = ib.create<vector::ShapeCastOp>(flattenedType, res);
-      res = ib.create<vector::ShapeCastOp>(op.getResultVectorType(), res);
+      res = vector::ShapeCastOp::create(ib, flattenedType, res);
+      res = vector::ShapeCastOp::create(ib, op.getResultVectorType(), res);
       rewriter.replaceOp(op, res);
       return success();
     };
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index ef7cd1424e7a4..f155b2a363834 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -414,7 +414,7 @@ void CreateDescOp::build(OpBuilder &builder, OperationState &state,
   int64_t size = static_cast<int64_t>(offsets.size());
   auto type = VectorType::get(size, builder.getIndexType());
   auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
-  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  auto offset = vector::FromElementsOp::create(builder, loc, type, values);
   build(builder, state, TensorDesc, source, offset);
 }
 
@@ -534,7 +534,7 @@ void UpdateOffsetOp::build(OpBuilder &builder, OperationState &state,
   int64_t size = static_cast<int64_t>(offsets.size());
   auto type = VectorType::get({size}, builder.getIndexType());
   auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
-  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  auto offset = vector::FromElementsOp::create(builder, loc, type, values);
   build(builder, state, tdescTy, tensorDesc, offset);
 }
 
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index c072557c2bd22..613a33fe63d87 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -120,13 +120,13 @@ static Value resolveDistributedTy(Value orig, T expected,
   // If orig is a vector type, create a shape cast op to reconcile the types.
   if (isa<VectorType>(orig.getType())) {
     auto castOp =
-        rewriter.create<vector::ShapeCastOp>(orig.getLoc(), expected, orig);
+        vector::ShapeCastOp::create(rewriter, orig.getLoc(), expected, orig);
     return castOp.getResult();
   }
   // If orig is a tensor descriptor type, create an unrealized conversion cast
   // op to reconcile the types.
   if (isa<xegpu::TensorDescType>(orig.getType())) {
-    auto castOp = rewriter.create<UnrealizedConversionCastOp>(orig.getLoc(),
+    auto castOp = UnrealizedConversionCastOp::create(rewriter, orig.getLoc(),
                                                               expected, orig);
     castOp->setAttr(resolveSIMTTypeMismatch, rewriter.getUnitAttr());
     return castOp.getResult(0);
@@ -198,16 +198,16 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         }))
       return failure();
     // Create a new function with the same signature.
-    auto newGpuFunc = rewriter.create<gpu::GPUFuncOp>(
+    auto newGpuFunc = gpu::GPUFuncOp::create(rewriter,
         gpuFuncOp.getLoc(), gpuFuncOp.getName(), gpuFuncOp.getFunctionType());
     // Create a WarpExecuteOnLane0Op with same arguments and results as the
     // original gpuFuncOp.
     rewriter.setInsertionPointToEnd(&newGpuFunc.getFunctionBody().front());
-    auto laneId = rewriter.create<gpu::LaneIdOp>(
+    auto laneId = gpu::LaneIdOp::create(rewriter,
         newGpuFunc.getLoc(), rewriter.getIndexType(),
         /** upperBound = **/ mlir::IntegerAttr());
     ArrayRef<Type> gpuFuncResultType = gpuFuncOp.getFunctionType().getResults();
-    auto warpOp = rewriter.create<gpu::WarpExecuteOnLane0Op>(
+    auto warpOp = gpu::WarpExecuteOnLane0Op::create(rewriter,
         laneId.getLoc(), gpuFuncResultType, laneId,
         xegpu::targetinfo::subgroupSize, newGpuFunc.getArguments(),
         newGpuFunc.getArgumentTypes());
@@ -216,7 +216,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
     auto origRetunOp =
         cast<gpu::ReturnOp>(gpuFuncOp.getBlocks().back().getTerminator());
     rewriter.setInsertionPointAfter(origRetunOp);
-    rewriter.create<gpu::YieldOp>(origRetunOp.getLoc(),
+    gpu::YieldOp::create(rewriter, origRetunOp.getLoc(),
                                   origRetunOp.getOperands());
     rewriter.eraseOp(origRetunOp);
     // Move the original function body to the WarpExecuteOnLane0Op body.
@@ -225,7 +225,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
     rewriter.eraseBlock(&warpBodyBlock);
     // Insert a new ReturnOp after the WarpExecuteOnLane0Op.
     rewriter.setInsertionPointAfter(warpOp);
-    rewriter.create<gpu::ReturnOp>(newGpuFunc.getLoc(), warpOp.getResults());
+    gpu::ReturnOp::create(rewriter, newGpuFunc.getLoc(), warpOp.getResults());
     rewriter.replaceOp(gpuFuncOp, newGpuFunc);
     return success();
   }
@@ -301,7 +301,7 @@ struct CreateNdDescDistribution final : public gpu::WarpDistributionPattern {
     xegpu::TensorDescType distributedTensorDescTy =
         descOp.getType().dropLayouts(); // Distributed tensor descriptor type
                                         // does not contain layout info.
-    Value newDescOp = rewriter.create<xegpu::CreateNdDescOp>(
+    Value newDescOp = xegpu::CreateNdDescOp::create(rewriter,
         newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
         descOp->getAttrs());
 
@@ -403,7 +403,7 @@ struct StoreNdDistribution final : public gpu::WarpDistributionPattern {
         resolveDistributedTy(newWarpOp.getResult(newRetIndices[1]),
                              distributedTensorDescTy, rewriter));
 
-    rewriter.create<xegpu::StoreNdOp>(
+    xegpu::StoreNdOp::create(rewriter,
         newWarpOp.getLoc(), TypeRange{}, newStoreOperands,
         removeTemporaryLayoutAttributes(storeOp->getAttrs()));
     rewriter.eraseOp(storeOp);
@@ -494,7 +494,7 @@ struct LoadNdDistribution final : public gpu::WarpDistributionPattern {
         loadOp.getTensorDescType().dropLayouts(); // Distributed tensor
                                                   // descriptor type does not
                                                   // contain layout info.
-    auto newLoadOp = rewriter.create<xegpu::LoadNdOp>(
+    auto newLoadOp = xegpu::LoadNdOp::create(rewriter,
         newWarpOp.getLoc(), loadNdDistValueTyOrFailure.value(),
         resolveDistributedTy(newWarpOp->getResult(newRetIndices[0]),
                              distributedTensorDescTy, rewriter),
@@ -630,7 +630,7 @@ struct DpasDistribution final : public gpu::WarpDistributionPattern {
           resolveDistributedTy(newWarpOp.getResult(newRetIndices[i]),
                                newDpasOperandExpectedTypes[i], rewriter));
     }
-    Value newDpasOp = rewriter.create<xegpu::DpasOp>(
+    Value newDpasOp = xegpu::DpasOp::create(rewriter,
         newWarpOp->getLoc(), distributedResultTy, newDpasOperands,
         removeTemporaryLayoutAttributes(dpasOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -717,7 +717,7 @@ struct UpdateNdOffsetDistribution final : public gpu::WarpDistributionPattern {
       }
     }
     // Create a new update op outside the warp op.
-    Value newUpdateOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+    Value newUpdateOp = xegpu::UpdateNdOffsetOp::create(rewriter,
         newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
         removeTemporaryLayoutAttributes(updateOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -783,7 +783,7 @@ struct PrefetchNdDistribution final : public gpu::WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     SmallVector<Value> newPrefetchOperands = {resolveDistributedTy(
         newWarpOp.getResult(newRetIndices[0]), newTensorDescTy, rewriter)};
-    rewriter.create<xegpu::PrefetchNdOp>(
+    xegpu::PrefetchNdOp::create(rewriter,
         newWarpOp.getLoc(), TypeRange{}, newPrefetchOperands,
         removeTemporaryLayoutAttributes(prefetchOp->getAttrs()));
     rewriter.eraseOp(prefetchOp);
@@ -806,7 +806,7 @@ struct GpuBarrierDistribution final : public gpu::WarpDistributionPattern {
       return failure();
     // Move the barrier op outside of the warp op.
     rewriter.setInsertionPointAfter(subgroupOp);
-    rewriter.create<gpu::BarrierOp>(
+    gpu::BarrierOp::create(rewriter,
         barrierOp.getLoc(), barrierOp->getResultTypes(),
         barrierOp->getOperands(), barrierOp->getAttrs());
     rewriter.eraseOp(barrierOp);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
index 13d49cb0e9d82..6266cca918d4f 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
@@ -83,7 +83,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = rewriter.create<UnrealizedConversionCastOp>(
+      auto castOp = UnrealizedConversionCastOp::create(rewriter,
           loc, destTy, srcs, ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResult(0);
     }
@@ -109,7 +109,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = rewriter.create<UnrealizedConversionCastOp>(
+      auto castOp = UnrealizedConversionCastOp::create(rewriter,
           loc, destTypes, src, ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResults();
     }
@@ -144,10 +144,10 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
     auto addi = [&](OpFoldResult a, int64_t b) -> Value {
       std::optional<int64_t> maybeInt = getConstantIntValue(a);
       if (maybeInt) {
-        return rewriter.create<arith::ConstantIndexOp>(loc, *maybeInt + b);
+        return arith::ConstantIndexOp::create(rewriter, loc, *maybeInt + b);
       } else {
         auto aV = llvm::cast<Value>(a);
-        auto bV = rewriter.create<arith::ConstantIndexOp>(loc, b);
+        auto bV = arith::ConstantIndexOp::create(rewriter, loc, b);
         return rewriter.createOrFold<arith::AddIOp>(loc, aV, bV);
       }
     };
@@ -169,7 +169,7 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
            llvm::zip(validIdxes, oldOffsets, offsets))
         mixedOffsets[idx] = addi(oldOff, offset);
 
-      auto newOp = rewriter.create<xegpu::CreateNdDescOp>(
+      auto newOp = xegpu::CreateNdDescOp::create(rewriter,
           loc, newTdescTy, op.getSource(), mixedOffsets, op.getMixedSizes(),
           op.getMixedStrides());
       newOps.push_back(newOp);
@@ -199,7 +199,7 @@ struct UnrollUpdateNdOffsetOp : public UnrollPattern<xegpu::UpdateNdOffsetOp> {
 
     SmallVector<Value> newOps;
     for (auto t : convertedTdesc) {
-      auto newOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+      auto newOp = xegpu::UpdateNdOffsetOp::create(rewriter,
           loc, t.getType(), t, op.getOffsets(), op.getConstOffsets());
       newOps.push_back(newOp);
     }
@@ -226,7 +226,7 @@ struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      rewriter.create<xegpu::PrefetchNdOp>(loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchNdOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -257,7 +257,7 @@ struct UnrollLoadNdOp : public UnrollPattern<xegpu::LoadNdOp> {
     SmallVector<Value> newOps;
     for (auto t : convertedTdescs) {
       auto newOp =
-          rewriter.create<xegpu::LoadNdOp>(loc, newValueTy, t, op->getAttrs());
+          xegpu::LoadNdOp::create(rewriter, loc, newValueTy, t, op->getAttrs());
       newOps.push_back(newOp);
     }
 
@@ -291,7 +291,7 @@ struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto [v, t] : llvm::zip(convertedValues, convertedTdescs))
-      rewriter.create<xegpu::StoreNdOp>(loc, v, t, op.getL1HintAttr(),
+      xegpu::StoreNdOp::create(rewriter, loc, v, t, op.getL1HintAttr(),
                                         op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
@@ -384,7 +384,7 @@ struct UnrollDpasOp : public UnrollPattern<xegpu::DpasOp> {
           if (tmpC)
             operands.push_back(tmpC);
 
-          tmpC = rewriter.create<xegpu::DpasOp>(loc, vecTy, operands,
+          tmpC = xegpu::DpasOp::create(rewriter, loc, vecTy, operands,
                                                 op->getAttrs());
         }
         newOps.push_back(tmpC);
@@ -436,13 +436,13 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
            llvm::zip(convertedIndiceVec, convertedIndiceTypes)) {
         for (int64_t i = 0; i < numNewChunks; ++i) {
           // Compute the offset
-          Value inc = rewriter.create<arith::ConstantIndexOp>(
+          Value inc = arith::ConstantIndexOp::create(rewriter,
               loc, i * blockedChunkSize);
-          Value incVec = rewriter.create<vector::SplatOp>(loc, indiceType, inc);
+          Value incVec = vector::SplatOp::create(rewriter, loc, indiceType, inc);
           Value offsetIndice =
-              rewriter.create<arith::AddIOp>(loc, indice, incVec);
+              arith::AddIOp::create(rewriter, loc, indice, incVec);
 
-          auto newOp = rewriter.create<xegpu::CreateDescOp>(
+          auto newOp = xegpu::CreateDescOp::create(rewriter,
               loc, newTdescTy, op.getSource(), offsetIndice);
 
           newOps.push_back(newOp);
@@ -450,7 +450,7 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
       }
     } else {
       for (auto indice : convertedIndiceVec) {
-        auto newOp = rewriter.create<xegpu::CreateDescOp>(
+        auto newOp = xegpu::CreateDescOp::create(rewriter,
             loc, newTdescTy, op.getSource(), indice);
         newOps.push_back(newOp);
       }
@@ -515,7 +515,7 @@ struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
 
     SmallVector<Value> newOps;
     for (auto [t, m] : llvm::zip(convertedTdescs, convertedMasks)) {
-      auto newOp = rewriter.create<xegpu::LoadGatherOp>(
+      auto newOp = xegpu::LoadGatherOp::create(rewriter,
           loc, newValueTy, t, m, op.getL1HintAttr(), op.getL2HintAttr(),
           op.getL3HintAttr());
       newOps.push_back(newOp);
@@ -547,7 +547,7 @@ struct UnrollPrefetchOp : public UnrollPattern<xegpu::PrefetchOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      rewriter.create<xegpu::PrefetchOp>(loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -608,7 +608,7 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
       Value v = convertedValues[i];
       Value t = convertedTdescs[i];
       Value m = op.getMask() ? convertedMasks[i] : nullptr;
-      rewriter.create<xegpu::StoreScatterOp>(loc, v, t, m, op.getL1HintAttr(),
+      xegpu::StoreScatterOp::create(rewriter, loc, v, t, m, op.getL1HintAttr(),
                                              op.getL2HintAttr(),
                                              op.getL3HintAttr());
     }
@@ -662,7 +662,7 @@ struct UnrollUpdateOffsetOp : public UnrollPattern<xegpu::UpdateOffsetOp> {
 
     for (auto [t, o] : llvm::zip(convertedTdesc, convertedOffsetVec)) {
       auto newOp =
-          rewriter.create<xegpu::UpdateOffsetOp>(loc, t.getType(), t, o);
+          xegpu::UpdateOffsetOp::create(rewriter, loc, t.getType(), t, o);
       newOps.push_back(newOp);
     }
     Value castOp = unpack(newOps, op.getType(), *targetShape, loc, rewriter);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index be7b860dd1729..aa2fc78c77f39 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -121,11 +121,11 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
     for (size_t i = 0; i < rank; ++i) {
       size_t dimIdx = originalOffsets.size() - rank + i;
       Value constOffset =
-          rewriter.create<arith::ConstantIndexOp>(loc, distUnitBaseAddr[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, distUnitBaseAddr[i]);
       Value offset =
           rewriter.createOrFold<index::AddOp>(loc, localOffset[i], constOffset);
       Value modValue =
-          rewriter.create<arith::ConstantIndexOp>(loc, distUnitShape[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, distUnitShape[i]);
       Value offsetMod =
           rewriter.createOrFold<index::RemUOp>(loc, offset, modValue);
       Value origOffset = getValueOrCreateConstantIndexOp(
@@ -162,7 +162,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
     // TODO : Handle order attribute
     // Get the subgroup ID
     auto linearSgId =
-        rewriter.create<gpu::SubgroupIdOp>(loc, /*upper_bound=*/nullptr);
+        gpu::SubgroupIdOp::create(rewriter, loc, /*upper_bound=*/nullptr);
 
     // Create constants for layout dimensions
     SmallVector<Value> sgLayoutDim(sgLayout.size());
@@ -170,8 +170,8 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
 
     for (size_t i = 0; i < sgLayout.size(); i++) {
       sgLayoutDim[i] =
-          rewriter.create<arith::ConstantIndexOp>(loc, sgLayout[i]);
-      sgDataDim[i] = rewriter.create<arith::ConstantIndexOp>(loc, sgShape[i]);
+          arith::ConstantIndexOp::create(rewriter, loc, sgLayout[i]);
+      sgDataDim[i] = arith::ConstantIndexOp::create(rewriter, loc, sgShape[i]);
     }
 
     auto deLinearizeSgId =
@@ -201,7 +201,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
           calculateGlobalOffsets(rewriter, loc, originalOffsets, localOffset,
                                  distUnitBaseAddr, distUnitShape);
 
-      auto newCreateNdOp = rewriter.create<xegpu::CreateNdDescOp>(
+      auto newCreateNdOp = xegpu::CreateNdDescOp::create(rewriter,
           loc, newTdescTy, op.getSource(), globalOffsets, op.getMixedSizes(),
           op.getMixedStrides());
       newCreateNdOps.push_back(newCreateNdOp);
@@ -224,7 +224,7 @@ struct WgToSgLoadNdOp : public OpConversionPattern<xegpu::LoadNdOp> {
           dyn_cast<xegpu::TensorDescType>(src.getType());
       ArrayRef<int64_t> srcShape = tdescTy.getShape();
       VectorType newResTy = VectorType::get(srcShape, tdescTy.getElementType());
-      auto newLoadOp = rewriter.create<xegpu::LoadNdOp>(op.getLoc(), newResTy,
+      auto newLoadOp = xegpu::LoadNdOp::create(rewriter, op.getLoc(), newResTy,
                                                         src, op->getAttrs());
       newLoadOps.push_back(newLoadOp);
     }
@@ -242,7 +242,7 @@ struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
   matchAndRewrite(xegpu::StoreNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     for (auto [v, t] : llvm::zip(adaptor.getValue(), adaptor.getTensorDesc()))
-      rewriter.create<xegpu::StoreNdOp>(op.getLoc(), v, t, op.getL1HintAttr(),
+      xegpu::StoreNdOp::create(rewriter, op.getLoc(), v, t, op.getL1HintAttr(),
                                         op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
@@ -261,7 +261,7 @@ struct WgToSgUpdateNdOffsetOp
                   ConversionPatternRewriter &rewriter) const override {
     llvm::SmallVector<Value> newUpdateTileOffsetOps;
     for (auto tDesc : adaptor.getTensorDesc()) {
-      auto newUpdateTileOffsetOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+      auto newUpdateTileOffsetOp = xegpu::UpdateNdOffsetOp::create(rewriter,
           op.getLoc(), tDesc.getType(), tDesc, op.getOffsets(),
           op.getConstOffsets());
       newUpdateTileOffsetOps.push_back(newUpdateTileOffsetOp);
@@ -305,7 +305,7 @@ struct WgToSgDpasOp : public OpConversionPattern<xegpu::DpasOp> {
             llvm::cast<VectorType>(bVec.getType()).getShape();
         VectorType resTy = VectorType::get({aVecShape[0], bVecShape[1]},
                                            resultTy.getElementType());
-        tmpC = rewriter.create<xegpu::DpasOp>(loc, resTy, operands);
+        tmpC = xegpu::DpasOp::create(rewriter, loc, resTy, operands);
         xegpu::setLayoutAttr(cast<OpResult>(tmpC),
                              originalLayout.dropSgLayoutAndData());
 
@@ -324,7 +324,7 @@ struct WgToSgPrefetchNdOp : public OpConversionPattern<xegpu::PrefetchNdOp> {
   matchAndRewrite(xegpu::PrefetchNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     for (auto src : adaptor.getTensorDesc())
-      rewriter.create<xegpu::PrefetchNdOp>(op.getLoc(), TypeRange(), src,
+      xegpu::PrefetchNdOp::create(rewriter, op.getLoc(), TypeRange(), src,
                                            op->getAttrs());
     rewriter.eraseOp(op);
     return success();
diff --git a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index 6b85a66a8bd36..9acb209d031bd 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -198,7 +198,7 @@ xegpu::extractVectorsWithShapeFromValue(OpBuilder &builder, Location loc,
   SmallVector<Value> result;
   for (SmallVector<int64_t> offsets : StaticTileOffsetRange(srcShape, shape)) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result.push_back(builder.create<vector::ExtractStridedSliceOp>(
+    result.push_back(vector::ExtractStridedSliceOp::create(builder,
         loc, value, offsets, shape, staticStrides));
   }
 
@@ -218,13 +218,13 @@ Value xegpu::createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
 
   VectorType resultTy = VectorType::get(shape, elemTy);
   auto zeroAttr = builder.getZeroAttr(elemTy);
-  Value result = builder.create<arith::ConstantOp>(
+  Value result = arith::ConstantOp::create(builder,
       loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
 
   for (auto [src, offsets] :
        llvm::zip_equal(values, StaticTileOffsetRange(shape, tileShape))) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result = builder.create<vector::InsertStridedSliceOp>(
+    result = vector::InsertStridedSliceOp::create(builder,
         loc, src, result, offsets, staticStrides);
   }
   return result;
@@ -236,7 +236,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
   auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
                             Location loc) -> Value {
-    return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+    return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
         .getResult(0);
   };
 
@@ -343,7 +343,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
         if (isa<RankedTensorType>(inputTy) && isa<VectorType>(outputTy)) {
           SmallVector<Value> values = xegpu::flattenValues(adaptor.getInputs());
-          auto newOp = rewriter.create<UnrealizedConversionCastOp>(
+          auto newOp = UnrealizedConversionCastOp::create(rewriter,
               op.getLoc(), outputTy, values);
           rewriter.replaceOp(op, newOp);
           return success();
@@ -355,7 +355,7 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
     converter.addSourceMaterialization(materializeCast);
     converter.addTargetMaterialization([&](OpBuilder &builder, TypeRange type,
                                            ValueRange inputs, Location loc) {
-      return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+      return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
           .getResults();
     });
 
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 6d7e2aa0ece7d..c88b328282275 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -132,7 +132,7 @@ void ModuleOp::build(OpBuilder &builder, OperationState &state,
 /// Construct a module from the given context.
 ModuleOp ModuleOp::create(Location loc, std::optional<StringRef> name) {
   OpBuilder builder(loc->getContext());
-  return builder.create<ModuleOp>(loc, name);
+  return ModuleOp::create(builder, loc, name);
 }
 
 DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {
diff --git a/mlir/lib/Query/Query.cpp b/mlir/lib/Query/Query.cpp
index 637e1f3cdef87..8ddb35d75734b 100644
--- a/mlir/lib/Query/Query.cpp
+++ b/mlir/lib/Query/Query.cpp
@@ -78,7 +78,7 @@ static Operation *extractFunction(std::vector<Operation *> &ops,
                       clonedOp->result_end());
   }
   // Add return operation
-  builder.create<func::ReturnOp>(loc, clonedVals);
+  func::ReturnOp::create(builder, loc, clonedVals);
 
   // Remove unused function arguments
   size_t currentIndex = 0;
diff --git a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
index 4e7f1d3185129..c0f19f6dce600 100644
--- a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
+++ b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
@@ -37,7 +37,7 @@ LogicalResult mlir::LLVMImportInterface::convertUnregisteredIntrinsic(
     return failure();
 
   Type resultType = moduleImport.convertType(inst->getType());
-  auto op = builder.create<::mlir::LLVM::CallIntrinsicOp>(
+  auto op = ::mlir::LLVM::CallIntrinsicOp::create(builder,
       moduleImport.translateLoc(inst->getDebugLoc()),
       isa<LLVMVoidType>(resultType) ? TypeRange{} : TypeRange{resultType},
       StringAttr::get(builder.getContext(), intrinName),
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index bfda223fe0f5f..6e47db6d1dc6a 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -185,7 +185,7 @@ ComdatOp ModuleImport::getGlobalComdatOp() {
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(mlirModule.getBody());
   globalComdatOp =
-      builder.create<ComdatOp>(mlirModule.getLoc(), getGlobalComdatOpName());
+      ComdatOp::create(builder, mlirModule.getLoc(), getGlobalComdatOpName());
   globalInsertionOp = globalComdatOp;
   return globalComdatOp;
 }
@@ -864,7 +864,7 @@ LogicalResult ModuleImport::convertModuleFlagsMetadata() {
   }
 
   if (!moduleFlags.empty())
-    builder.create<LLVM::ModuleFlagsOp>(mlirModule.getLoc(),
+    LLVM::ModuleFlagsOp::create(builder, mlirModule.getLoc(),
                                         builder.getArrayAttr(moduleFlags));
 
   return success();
@@ -880,7 +880,7 @@ LogicalResult ModuleImport::convertLinkerOptionsMetadata() {
       options.reserve(node->getNumOperands());
       for (const llvm::MDOperand &option : node->operands())
         options.push_back(cast<llvm::MDString>(option)->getString());
-      builder.create<LLVM::LinkerOptionsOp>(mlirModule.getLoc(),
+      LLVM::LinkerOptionsOp::create(builder, mlirModule.getLoc(),
                                             builder.getStrArrayAttr(options));
     }
   }
@@ -984,7 +984,7 @@ void ModuleImport::processComdat(const llvm::Comdat *comdat) {
   ComdatOp comdatOp = getGlobalComdatOp();
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(&comdatOp.getBody().back());
-  auto selectorOp = builder.create<ComdatSelectorOp>(
+  auto selectorOp = ComdatSelectorOp::create(builder,
       mlirModule.getLoc(), comdat->getName(),
       convertComdatFromLLVM(comdat->getSelectionKind()));
   auto symbolRef =
@@ -1346,7 +1346,7 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   Type type = convertType(alias->getValueType());
-  AliasOp aliasOp = builder.create<AliasOp>(
+  AliasOp aliasOp = AliasOp::create(builder,
       mlirModule.getLoc(), type, convertLinkageFromLLVM(alias->getLinkage()),
       alias->getName(),
       /*dso_local=*/alias->isDSOLocal(),
@@ -1360,7 +1360,7 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   FailureOr<Value> initializer = convertConstantExpr(alias->getAliasee());
   if (failed(initializer))
     return failure();
-  builder.create<ReturnOp>(aliasOp.getLoc(), *initializer);
+  ReturnOp::create(builder, aliasOp.getLoc(), *initializer);
 
   if (alias->hasAtLeastLocalUnnamedAddr())
     aliasOp.setUnnamedAddr(convertUnnamedAddrFromLLVM(alias->getUnnamedAddr()));
@@ -1403,7 +1403,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
   if (globalName.empty())
     globalName = getOrCreateNamelessSymbolName(globalVar).getValue();
 
-  GlobalOp globalOp = builder.create<GlobalOp>(
+  GlobalOp globalOp = GlobalOp::create(builder,
       mlirModule.getLoc(), type, globalVar->isConstant(),
       convertLinkageFromLLVM(globalVar->getLinkage()), StringRef(globalName),
       valueAttr, alignment, /*addr_space=*/globalVar->getAddressSpace(),
@@ -1420,7 +1420,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
         convertConstantExpr(globalVar->getInitializer());
     if (failed(initializer))
       return failure();
-    builder.create<ReturnOp>(globalOp.getLoc(), *initializer);
+    ReturnOp::create(builder, globalOp.getLoc(), *initializer);
   }
   if (globalVar->hasAtLeastLocalUnnamedAddr()) {
     globalOp.setUnnamedAddr(
@@ -1488,12 +1488,12 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   if (globalVar->getName() == getGlobalCtorsVarName()) {
-    globalInsertionOp = builder.create<LLVM::GlobalCtorsOp>(
+    globalInsertionOp = LLVM::GlobalCtorsOp::create(builder,
         mlirModule.getLoc(), builder.getArrayAttr(funcs),
         builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
     return success();
   }
-  globalInsertionOp = builder.create<LLVM::GlobalDtorsOp>(
+  globalInsertionOp = LLVM::GlobalDtorsOp::create(builder,
       mlirModule.getLoc(), builder.getArrayAttr(funcs),
       builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
   return success();
@@ -1569,33 +1569,33 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
   if (Attribute attr = getConstantAsAttr(constant)) {
     Type type = convertType(constant->getType());
     if (auto symbolRef = dyn_cast<FlatSymbolRefAttr>(attr)) {
-      return builder.create<AddressOfOp>(loc, type, symbolRef.getValue())
+      return AddressOfOp::create(builder, loc, type, symbolRef.getValue())
           .getResult();
     }
-    return builder.create<ConstantOp>(loc, type, attr).getResult();
+    return ConstantOp::create(builder, loc, type, attr).getResult();
   }
 
   // Convert null pointer constants.
   if (auto *nullPtr = dyn_cast<llvm::ConstantPointerNull>(constant)) {
     Type type = convertType(nullPtr->getType());
-    return builder.create<ZeroOp>(loc, type).getResult();
+    return ZeroOp::create(builder, loc, type).getResult();
   }
 
   // Convert none token constants.
   if (isa<llvm::ConstantTokenNone>(constant)) {
-    return builder.create<NoneTokenOp>(loc).getResult();
+    return NoneTokenOp::create(builder, loc).getResult();
   }
 
   // Convert poison.
   if (auto *poisonVal = dyn_cast<llvm::PoisonValue>(constant)) {
     Type type = convertType(poisonVal->getType());
-    return builder.create<PoisonOp>(loc, type).getResult();
+    return PoisonOp::create(builder, loc, type).getResult();
   }
 
   // Convert undef.
   if (auto *undefVal = dyn_cast<llvm::UndefValue>(constant)) {
     Type type = convertType(undefVal->getType());
-    return builder.create<UndefOp>(loc, type).getResult();
+    return UndefOp::create(builder, loc, type).getResult();
   }
 
   // Convert dso_local_equivalent.
@@ -1621,7 +1621,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
           getOrCreateNamelessSymbolName(cast<llvm::GlobalVariable>(globalObj));
     else
       symbolRef = FlatSymbolRefAttr::get(context, globalName);
-    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+    return AddressOfOp::create(builder, loc, type, symbolRef).getResult();
   }
 
   // Convert global alias accesses.
@@ -1629,7 +1629,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     Type type = convertType(globalAliasObj->getType());
     StringRef aliaseeName = globalAliasObj->getName();
     FlatSymbolRefAttr symbolRef = FlatSymbolRefAttr::get(context, aliaseeName);
-    return builder.create<AddressOfOp>(loc, type, symbolRef).getResult();
+    return AddressOfOp::create(builder, loc, type, symbolRef).getResult();
   }
 
   // Convert constant expressions.
@@ -1680,15 +1680,15 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     bool isArrayOrStruct = isa<LLVMArrayType, LLVMStructType>(rootType);
     assert((isArrayOrStruct || LLVM::isCompatibleVectorType(rootType)) &&
            "unrecognized aggregate type");
-    Value root = builder.create<UndefOp>(loc, rootType);
+    Value root = UndefOp::create(builder, loc, rootType);
     for (const auto &it : llvm::enumerate(elementValues)) {
       if (isArrayOrStruct) {
-        root = builder.create<InsertValueOp>(loc, root, it.value(), it.index());
+        root = InsertValueOp::create(builder, loc, root, it.value(), it.index());
       } else {
         Attribute indexAttr = builder.getI32IntegerAttr(it.index());
         Value indexValue =
-            builder.create<ConstantOp>(loc, builder.getI32Type(), indexAttr);
-        root = builder.create<InsertElementOp>(loc, rootType, root, it.value(),
+            ConstantOp::create(builder, loc, builder.getI32Type(), indexAttr);
+        root = InsertElementOp::create(builder, loc, rootType, root, it.value(),
                                                indexValue);
       }
     }
@@ -1702,7 +1702,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
            "target extension type does not support zero-initialization");
     // Create llvm.mlir.zero operation to represent zero-initialization of
     // target extension type.
-    return builder.create<LLVM::ZeroOp>(loc, targetExtType).getRes();
+    return LLVM::ZeroOp::create(builder, loc, targetExtType).getRes();
   }
 
   if (auto *blockAddr = dyn_cast<llvm::BlockAddress>(constant)) {
@@ -2124,7 +2124,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     if (!brInst->isConditional()) {
-      auto brOp = builder.create<LLVM::BrOp>(loc, succBlockArgs.front(),
+      auto brOp = LLVM::BrOp::create(builder, loc, succBlockArgs.front(),
                                              succBlocks.front());
       mapNoResultOp(inst, brOp);
       return success();
@@ -2132,7 +2132,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     FailureOr<Value> condition = convertValue(brInst->getCondition());
     if (failed(condition))
       return failure();
-    auto condBrOp = builder.create<LLVM::CondBrOp>(
+    auto condBrOp = LLVM::CondBrOp::create(builder,
         loc, *condition, succBlocks.front(), succBlockArgs.front(),
         succBlocks.back(), succBlockArgs.back());
     mapNoResultOp(inst, condBrOp);
@@ -2166,7 +2166,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       caseBlocks[it.index()] = lookupBlock(succBB);
     }
 
-    auto switchOp = builder.create<SwitchOp>(
+    auto switchOp = SwitchOp::create(builder,
         loc, *condition, lookupBlock(defaultBB), defaultBlockArgs, caseValues,
         caseBlocks, caseOperandRefs);
     mapNoResultOp(inst, switchOp);
@@ -2218,14 +2218,14 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
         // IR). Build the indirect call by passing an empty `callee` operand and
         // insert into `operands` to include the indirect call target.
         FlatSymbolRefAttr calleeSym = convertCalleeName(callInst);
-        Value indirectCallVal = builder.create<LLVM::AddressOfOp>(
+        Value indirectCallVal = LLVM::AddressOfOp::create(builder,
             loc, LLVM::LLVMPointerType::get(context), calleeSym);
         operands->insert(operands->begin(), indirectCallVal);
       } else {
         // Regular direct call using callee name.
         callee = convertCalleeName(callInst);
       }
-      CallOp callOp = builder.create<CallOp>(loc, *funcTy, callee, *operands);
+      CallOp callOp = CallOp::create(builder, loc, *funcTy, callee, *operands);
 
       if (failed(convertCallAttributes(callInst, callOp)))
         return failure();
@@ -2260,7 +2260,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
 
     Type type = convertType(lpInst->getType());
     auto lpOp =
-        builder.create<LandingpadOp>(loc, type, lpInst->isCleanup(), operands);
+        LandingpadOp::create(builder, loc, type, lpInst->isCleanup(), operands);
     mapValue(inst, lpOp);
     return success();
   }
@@ -2310,7 +2310,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // IR). Build the indirect invoke by passing an empty `callee` operand and
       // insert into `operands` to include the indirect invoke target.
       FlatSymbolRefAttr calleeSym = convertCalleeName(invokeInst);
-      Value indirectInvokeVal = builder.create<LLVM::AddressOfOp>(
+      Value indirectInvokeVal = LLVM::AddressOfOp::create(builder,
           loc, LLVM::LLVMPointerType::get(context), calleeSym);
       operands->insert(operands->begin(), indirectInvokeVal);
     } else {
@@ -2320,7 +2320,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     // Create the invoke operation. Normal destination block arguments will be
     // added later on to handle the case in which the operation result is
     // included in this list.
-    auto invokeOp = builder.create<InvokeOp>(
+    auto invokeOp = InvokeOp::create(builder,
         loc, *funcTy, calleeName, *operands, directNormalDest, ValueRange(),
         lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
 
@@ -2348,7 +2348,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // arguments (including the invoke operation's result).
       OpBuilder::InsertionGuard g(builder);
       builder.setInsertionPointToStart(directNormalDest);
-      builder.create<LLVM::BrOp>(loc, normalArgs, normalDest);
+      LLVM::BrOp::create(builder, loc, normalArgs, normalDest);
     } else {
       // If the invoke operation's result is not a block argument to the normal
       // destination block, just add the block arguments as usual.
@@ -2382,7 +2382,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     Type type = convertType(inst->getType());
-    auto gepOp = builder.create<GEPOp>(
+    auto gepOp = GEPOp::create(builder,
         loc, type, sourceElementType, *basePtr, indices,
         static_cast<GEPNoWrapFlags>(gepInst->getNoWrapFlags().getRaw()));
     mapValue(inst, gepOp);
@@ -2409,7 +2409,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     SmallVector<ValueRange> succBlockArgsRange =
         llvm::to_vector_of<ValueRange>(succBlockArgs);
     Location loc = translateLoc(inst->getDebugLoc());
-    auto indBrOp = builder.create<LLVM::IndirectBrOp>(
+    auto indBrOp = LLVM::IndirectBrOp::create(builder,
         loc, *basePtr, succBlockArgsRange, succBlocks);
 
     mapNoResultOp(inst, indBrOp);
@@ -2854,7 +2854,7 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) {
   builder.setInsertionPointToEnd(mlirModule.getBody());
 
   Location loc = debugImporter->translateFuncLocation(func);
-  LLVMFuncOp funcOp = builder.create<LLVMFuncOp>(
+  LLVMFuncOp funcOp = LLVMFuncOp::create(builder,
       loc, func->getName(), functionType,
       convertLinkageFromLLVM(func->getLinkage()), dsoLocal, cconv);
 
@@ -3032,11 +3032,11 @@ ModuleImport::processDebugIntrinsic(llvm::DbgVariableIntrinsic *dbgIntr,
   Operation *op =
       llvm::TypeSwitch<llvm::DbgVariableIntrinsic *, Operation *>(dbgIntr)
           .Case([&](llvm::DbgDeclareInst *) {
-            return builder.create<LLVM::DbgDeclareOp>(
+            return LLVM::DbgDeclareOp::create(builder,
                 loc, *argOperand, localVariableAttr, locationExprAttr);
           })
           .Case([&](llvm::DbgValueInst *) {
-            return builder.create<LLVM::DbgValueOp>(
+            return LLVM::DbgValueOp::create(builder,
                 loc, *argOperand, localVariableAttr, locationExprAttr);
           });
   mapNoResultOp(dbgIntr, op);
@@ -3082,7 +3082,7 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb,
   if (bb->hasAddressTaken()) {
     OpBuilder::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(block);
-    builder.create<BlockTagOp>(block->getParentOp()->getLoc(),
+    BlockTagOp::create(builder, block->getParentOp()->getLoc(),
                                BlockTagAttr::get(context, bb->getNumber()));
   }
   return success();
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index 55d6a380d0bff..6e14b94e87407 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -42,22 +42,22 @@ static inline spirv::Opcode extractOpcode(uint32_t word) {
 Value spirv::Deserializer::getValue(uint32_t id) {
   if (auto constInfo = getConstant(id)) {
     // Materialize a `spirv.Constant` op at every use site.
-    return opBuilder.create<spirv::ConstantOp>(unknownLoc, constInfo->second,
+    return spirv::ConstantOp::create(opBuilder, unknownLoc, constInfo->second,
                                                constInfo->first);
   }
   if (auto varOp = getGlobalVariable(id)) {
-    auto addressOfOp = opBuilder.create<spirv::AddressOfOp>(
+    auto addressOfOp = spirv::AddressOfOp::create(opBuilder,
         unknownLoc, varOp.getType(), SymbolRefAttr::get(varOp.getOperation()));
     return addressOfOp.getPointer();
   }
   if (auto constOp = getSpecConstant(id)) {
-    auto referenceOfOp = opBuilder.create<spirv::ReferenceOfOp>(
+    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
         unknownLoc, constOp.getDefaultValue().getType(),
         SymbolRefAttr::get(constOp.getOperation()));
     return referenceOfOp.getReference();
   }
   if (auto constCompositeOp = getSpecConstantComposite(id)) {
-    auto referenceOfOp = opBuilder.create<spirv::ReferenceOfOp>(
+    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
         unknownLoc, constCompositeOp.getType(),
         SymbolRefAttr::get(constCompositeOp.getOperation()));
     return referenceOfOp.getReference();
@@ -69,7 +69,7 @@ Value spirv::Deserializer::getValue(uint32_t id) {
         specConstOperationInfo->enclosedOpOperands);
   }
   if (auto undef = getUndefType(id)) {
-    return opBuilder.create<spirv::UndefOp>(unknownLoc, undef);
+    return spirv::UndefOp::create(opBuilder, unknownLoc, undef);
   }
   return valueMap.lookup(id);
 }
@@ -369,7 +369,7 @@ Deserializer::processOp<spirv::EntryPointOp>(ArrayRef<uint32_t> words) {
     interface.push_back(SymbolRefAttr::get(arg.getOperation()));
     wordIndex++;
   }
-  opBuilder.create<spirv::EntryPointOp>(
+  spirv::EntryPointOp::create(opBuilder,
       unknownLoc, execModel, SymbolRefAttr::get(opBuilder.getContext(), fnName),
       opBuilder.getArrayAttr(interface));
   return success();
@@ -402,7 +402,7 @@ Deserializer::processOp<spirv::ExecutionModeOp>(ArrayRef<uint32_t> words) {
     attrListElems.push_back(opBuilder.getI32IntegerAttr(words[wordIndex++]));
   }
   auto values = opBuilder.getArrayAttr(attrListElems);
-  opBuilder.create<spirv::ExecutionModeOp>(
+  spirv::ExecutionModeOp::create(opBuilder,
       unknownLoc, SymbolRefAttr::get(opBuilder.getContext(), fn.getName()),
       execMode, values);
   return success();
@@ -441,7 +441,7 @@ Deserializer::processOp<spirv::FunctionCallOp>(ArrayRef<uint32_t> operands) {
     arguments.push_back(value);
   }
 
-  auto opFunctionCall = opBuilder.create<spirv::FunctionCallOp>(
+  auto opFunctionCall = spirv::FunctionCallOp::create(opBuilder,
       unknownLoc, resultType,
       SymbolRefAttr::get(opBuilder.getContext(), functionName), arguments);
 
@@ -518,7 +518,7 @@ Deserializer::processOp<spirv::CopyMemoryOp>(ArrayRef<uint32_t> words) {
   }
 
   Location loc = createFileLineColLoc(opBuilder);
-  opBuilder.create<spirv::CopyMemoryOp>(loc, resultTypes, operands, attributes);
+  spirv::CopyMemoryOp::create(opBuilder, loc, resultTypes, operands, attributes);
 
   return success();
 }
@@ -549,7 +549,7 @@ LogicalResult Deserializer::processOp<spirv::GenericCastToPtrExplicitOp>(
   operands.push_back(arg);
 
   Location loc = createFileLineColLoc(opBuilder);
-  Operation *op = opBuilder.create<spirv::GenericCastToPtrExplicitOp>(
+  Operation *op = spirv::GenericCastToPtrExplicitOp::create(opBuilder,
       loc, resultTypes, operands);
   valueMap[valueID] = op->getResult(0);
   return success();
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index b1abd8b3dffe9..7b1f8055fadce 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -518,7 +518,7 @@ spirv::Deserializer::processFunction(ArrayRef<uint32_t> operands) {
   }
 
   std::string fnName = getFunctionSymbol(fnID);
-  auto funcOp = opBuilder.create<spirv::FuncOp>(
+  auto funcOp = spirv::FuncOp::create(opBuilder,
       unknownLoc, fnName, functionType, fnControl.value());
   // Processing other function attributes.
   if (decorations.count(fnID)) {
@@ -706,7 +706,7 @@ spirv::SpecConstantOp
 spirv::Deserializer::createSpecConstant(Location loc, uint32_t resultID,
                                         TypedAttr defaultValue) {
   auto symName = opBuilder.getStringAttr(getSpecConstantSymbol(resultID));
-  auto op = opBuilder.create<spirv::SpecConstantOp>(unknownLoc, symName,
+  auto op = spirv::SpecConstantOp::create(opBuilder, unknownLoc, symName,
                                                     defaultValue);
   if (decorations.count(resultID)) {
     for (auto attr : decorations[resultID].getAttrs())
@@ -782,7 +782,7 @@ spirv::Deserializer::processGlobalVariable(ArrayRef<uint32_t> operands) {
            << wordIndex << " of " << operands.size() << " processed";
   }
   auto loc = createFileLineColLoc(opBuilder);
-  auto varOp = opBuilder.create<spirv::GlobalVariableOp>(
+  auto varOp = spirv::GlobalVariableOp::create(opBuilder,
       loc, TypeAttr::get(type), opBuilder.getStringAttr(variableName),
       initializer);
 
@@ -1581,7 +1581,7 @@ spirv::Deserializer::processSpecConstantComposite(ArrayRef<uint32_t> operands) {
     elements.push_back(SymbolRefAttr::get(elementInfo));
   }
 
-  auto op = opBuilder.create<spirv::SpecConstantCompositeOp>(
+  auto op = spirv::SpecConstantCompositeOp::create(opBuilder,
       unknownLoc, TypeAttr::get(resultType), symName,
       opBuilder.getArrayAttr(elements));
   specConstCompositeMap[resultID] = op;
@@ -1656,7 +1656,7 @@ Value spirv::Deserializer::materializeSpecConstantOperation(
 
   auto loc = createFileLineColLoc(opBuilder);
   auto specConstOperationOp =
-      opBuilder.create<spirv::SpecConstantOperationOp>(loc, resultType);
+      spirv::SpecConstantOperationOp::create(opBuilder, loc, resultType);
 
   Region &body = specConstOperationOp.getBody();
   // Move the new block into SpecConstantOperation's body.
@@ -1669,7 +1669,7 @@ Value spirv::Deserializer::materializeSpecConstantOperation(
   OpBuilder::InsertionGuard moduleInsertionGuard(opBuilder);
   opBuilder.setInsertionPointToEnd(&block);
 
-  opBuilder.create<spirv::YieldOp>(loc, block.front().getResult(0));
+  spirv::YieldOp::create(opBuilder, loc, block.front().getResult(0));
   return specConstOperationOp.getResult();
 }
 
@@ -1733,7 +1733,7 @@ LogicalResult spirv::Deserializer::processBranch(ArrayRef<uint32_t> operands) {
   // The preceding instruction for the OpBranch instruction could be an
   // OpLoopMerge or an OpSelectionMerge instruction, in this case they will have
   // the same OpLine information.
-  opBuilder.create<spirv::BranchOp>(loc, target);
+  spirv::BranchOp::create(opBuilder, loc, target);
 
   clearDebugLine();
   return success();
@@ -1764,7 +1764,7 @@ spirv::Deserializer::processBranchConditional(ArrayRef<uint32_t> operands) {
   // an OpSelectionMerge instruction, in this case they will have the same
   // OpLine information.
   auto loc = createFileLineColLoc(opBuilder);
-  opBuilder.create<spirv::BranchConditionalOp>(
+  spirv::BranchConditionalOp::create(opBuilder,
       loc, condition, trueBlock,
       /*trueArguments=*/ArrayRef<Value>(), falseBlock,
       /*falseArguments=*/ArrayRef<Value>(), weights);
@@ -1947,7 +1947,7 @@ ControlFlowStructurizer::createSelectionOp(uint32_t selectionControl) {
   OpBuilder builder(&mergeBlock->front());
 
   auto control = static_cast<spirv::SelectionControl>(selectionControl);
-  auto selectionOp = builder.create<spirv::SelectionOp>(location, control);
+  auto selectionOp = spirv::SelectionOp::create(builder, location, control);
   selectionOp.addMergeBlock(builder);
 
   return selectionOp;
@@ -1959,7 +1959,7 @@ spirv::LoopOp ControlFlowStructurizer::createLoopOp(uint32_t loopControl) {
   OpBuilder builder(&mergeBlock->front());
 
   auto control = static_cast<spirv::LoopControl>(loopControl);
-  auto loopOp = builder.create<spirv::LoopOp>(location, control);
+  auto loopOp = spirv::LoopOp::create(builder, location, control);
   loopOp.addEntryAndMergeBlock(builder);
 
   return loopOp;
@@ -2092,7 +2092,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
     // The loop entry block should have a unconditional branch jumping to the
     // loop header block.
     builder.setInsertionPointToEnd(&body.front());
-    builder.create<spirv::BranchOp>(location, mapper.lookupOrNull(headerBlock),
+    spirv::BranchOp::create(builder, location, mapper.lookupOrNull(headerBlock),
                                     ArrayRef<Value>(blockArgs));
   }
 
@@ -2177,11 +2177,11 @@ LogicalResult ControlFlowStructurizer::structurize() {
     Operation *newOp = nullptr;
 
     if (isLoop)
-      newOp = builder.create<spirv::LoopOp>(
+      newOp = spirv::LoopOp::create(builder,
           location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::LoopControl>(control));
     else
-      newOp = builder.create<spirv::SelectionOp>(
+      newOp = spirv::SelectionOp::create(builder,
           location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::SelectionControl>(control));
 
@@ -2308,7 +2308,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
       // but replace all ops inside with a branch to the merge block.
       block->clear();
       builder.setInsertionPointToEnd(block);
-      builder.create<spirv::BranchOp>(location, mergeBlock);
+      spirv::BranchOp::create(builder, location, mergeBlock);
     } else {
       LLVM_DEBUG(logger.startLine() << "[cf] erasing block " << block << "\n");
       block->erase();
@@ -2362,7 +2362,7 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
 
     if (auto branchOp = dyn_cast<spirv::BranchOp>(op)) {
       // Replace the previous branch op with a new one with block arguments.
-      opBuilder.create<spirv::BranchOp>(branchOp.getLoc(), branchOp.getTarget(),
+      spirv::BranchOp::create(opBuilder, branchOp.getLoc(), branchOp.getTarget(),
                                         blockArgs);
       branchOp.erase();
     } else if (auto branchCondOp = dyn_cast<spirv::BranchConditionalOp>(op)) {
@@ -2370,13 +2370,13 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
               branchCondOp.getFalseBlock() == target) &&
              "expected target to be either the true or false target");
       if (target == branchCondOp.getTrueTarget())
-        opBuilder.create<spirv::BranchConditionalOp>(
+        spirv::BranchConditionalOp::create(opBuilder,
             branchCondOp.getLoc(), branchCondOp.getCondition(), blockArgs,
             branchCondOp.getFalseBlockArguments(),
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueTarget(),
             branchCondOp.getFalseTarget());
       else
-        opBuilder.create<spirv::BranchConditionalOp>(
+        spirv::BranchConditionalOp::create(opBuilder,
             branchCondOp.getLoc(), branchCondOp.getCondition(),
             branchCondOp.getTrueBlockArguments(), blockArgs,
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueBlock(),
@@ -2437,7 +2437,7 @@ LogicalResult spirv::Deserializer::splitConditionalBlocks() {
     if (!llvm::hasSingleElement(*block) || splitHeaderMergeBlock) {
       Block *newBlock = block->splitBlock(terminator);
       OpBuilder builder(block, block->end());
-      builder.create<spirv::BranchOp>(block->getParent()->getLoc(), newBlock);
+      spirv::BranchOp::create(builder, block->getParent()->getLoc(), newBlock);
 
       // After splitting we need to update the map to use the new block as a
       // header.
diff --git a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
index 824201d17b5ab..3f4e7762fed0e 100644
--- a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
+++ b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
@@ -134,7 +134,7 @@ class CodeGen {
 
 OwningOpRef<ModuleOp> CodeGen::generate(const ast::Module &module) {
   OwningOpRef<ModuleOp> mlirModule =
-      builder.create<ModuleOp>(genLoc(module.getLoc()));
+      ModuleOp::create(builder, genLoc(module.getLoc()));
   builder.setInsertionPointToStart(mlirModule->getBody());
 
   // Generate code for each of the decls within the module.
@@ -205,7 +205,7 @@ static void checkAndNestUnderRewriteOp(OpBuilder &builder, Value rootExpr,
                                        Location loc) {
   if (isa<pdl::PatternOp>(builder.getInsertionBlock()->getParentOp())) {
     pdl::RewriteOp rewrite =
-        builder.create<pdl::RewriteOp>(loc, rootExpr, /*name=*/StringAttr(),
+        pdl::RewriteOp::create(builder, loc, rootExpr, /*name=*/StringAttr(),
                                        /*externalArgs=*/ValueRange());
     builder.createBlock(&rewrite.getBodyRegion());
   }
@@ -219,7 +219,7 @@ void CodeGen::genImpl(const ast::EraseStmt *stmt) {
   // Make sure we are nested in a RewriteOp.
   OpBuilder::InsertionGuard guard(builder);
   checkAndNestUnderRewriteOp(builder, rootExpr, loc);
-  builder.create<pdl::EraseOp>(loc, rootExpr);
+  pdl::EraseOp::create(builder, loc, rootExpr);
 }
 
 void CodeGen::genImpl(const ast::LetStmt *stmt) { genVar(stmt->getVarDecl()); }
@@ -242,7 +242,7 @@ void CodeGen::genImpl(const ast::ReplaceStmt *stmt) {
   bool usesReplOperation =
       replValues.size() == 1 &&
       isa<pdl::OperationType>(replValues.front().getType());
-  builder.create<pdl::ReplaceOp>(
+  pdl::ReplaceOp::create(builder,
       loc, rootExpr, usesReplOperation ? replValues[0] : Value(),
       usesReplOperation ? ValueRange() : ValueRange(replValues));
 }
@@ -283,7 +283,7 @@ void CodeGen::genImpl(const ast::PatternDecl *decl) {
 
   // FIXME: Properly model HasBoundedRecursion in PDL so that we don't drop it
   // here.
-  pdl::PatternOp pattern = builder.create<pdl::PatternOp>(
+  pdl::PatternOp pattern = pdl::PatternOp::create(builder,
       genLoc(decl->getLoc()), decl->getBenefit(),
       name ? std::optional<StringRef>(name->getName())
            : std::optional<StringRef>());
@@ -338,19 +338,19 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   ast::Type type = varDecl->getType();
   Type mlirType = genType(type);
   if (isa<ast::ValueType>(type))
-    return builder.create<pdl::OperandOp>(loc, mlirType, getTypeConstraint());
+    return pdl::OperandOp::create(builder, loc, mlirType, getTypeConstraint());
   if (isa<ast::TypeType>(type))
-    return builder.create<pdl::TypeOp>(loc, mlirType, /*type=*/TypeAttr());
+    return pdl::TypeOp::create(builder, loc, mlirType, /*type=*/TypeAttr());
   if (isa<ast::AttributeType>(type))
-    return builder.create<pdl::AttributeOp>(loc, getTypeConstraint());
+    return pdl::AttributeOp::create(builder, loc, getTypeConstraint());
   if (ast::OperationType opType = dyn_cast<ast::OperationType>(type)) {
-    Value operands = builder.create<pdl::OperandsOp>(
+    Value operands = pdl::OperandsOp::create(builder,
         loc, pdl::RangeType::get(builder.getType<pdl::ValueType>()),
         /*type=*/Value());
-    Value results = builder.create<pdl::TypesOp>(
+    Value results = pdl::TypesOp::create(builder,
         loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
         /*types=*/ArrayAttr());
-    return builder.create<pdl::OperationOp>(loc, opType.getName(), operands,
+    return pdl::OperationOp::create(builder, loc, opType.getName(), operands,
                                             ArrayRef<StringRef>(), ValueRange(),
                                             results);
   }
@@ -358,10 +358,10 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   if (ast::RangeType rangeTy = dyn_cast<ast::RangeType>(type)) {
     ast::Type eleTy = rangeTy.getElementType();
     if (isa<ast::ValueType>(eleTy))
-      return builder.create<pdl::OperandsOp>(loc, mlirType,
+      return pdl::OperandsOp::create(builder, loc, mlirType,
                                              getTypeConstraint());
     if (isa<ast::TypeType>(eleTy))
-      return builder.create<pdl::TypesOp>(loc, mlirType, /*types=*/ArrayAttr());
+      return pdl::TypesOp::create(builder, loc, mlirType, /*types=*/ArrayAttr());
   }
 
   llvm_unreachable("invalid non-initialized variable type");
@@ -404,7 +404,7 @@ SmallVector<Value> CodeGen::genExpr(const ast::Expr *expr) {
 Value CodeGen::genExprImpl(const ast::AttributeExpr *expr) {
   Attribute attr = parseAttribute(expr->getValue(), builder.getContext());
   assert(attr && "invalid MLIR attribute data");
-  return builder.create<pdl::AttributeOp>(genLoc(expr->getLoc()), attr);
+  return pdl::AttributeOp::create(builder, genLoc(expr->getLoc()), attr);
 }
 
 SmallVector<Value> CodeGen::genExprImpl(const ast::CallExpr *expr) {
@@ -443,9 +443,9 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
     if (isa<ast::AllResultsMemberAccessExpr>(expr)) {
       Type mlirType = genType(expr->getType());
       if (isa<pdl::ValueType>(mlirType))
-        return builder.create<pdl::ResultOp>(loc, mlirType, parentExprs[0],
+        return pdl::ResultOp::create(builder, loc, mlirType, parentExprs[0],
                                              builder.getI32IntegerAttr(0));
-      return builder.create<pdl::ResultsOp>(loc, mlirType, parentExprs[0]);
+      return pdl::ResultsOp::create(builder, loc, mlirType, parentExprs[0]);
     }
 
     const ods::Operation *odsOp = opType.getODSOperation();
@@ -455,7 +455,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
       unsigned resultIndex;
       name.getAsInteger(/*Radix=*/10, resultIndex);
       IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
-      return builder.create<pdl::ResultOp>(loc, genType(expr->getType()),
+      return pdl::ResultOp::create(builder, loc, genType(expr->getType()),
                                            parentExprs[0], index);
     }
 
@@ -474,7 +474,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
 
     // Generate the result access.
     IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
-    return builder.create<pdl::ResultsOp>(loc, genType(expr->getType()),
+    return pdl::ResultsOp::create(builder, loc, genType(expr->getType()),
                                           parentExprs[0], index);
   }
 
@@ -518,7 +518,7 @@ Value CodeGen::genExprImpl(const ast::OperationExpr *expr) {
   for (const ast::Expr *result : expr->getResultTypes())
     results.push_back(genSingleExpr(result));
 
-  return builder.create<pdl::OperationOp>(loc, opName, operands, attrNames,
+  return pdl::OperationOp::create(builder, loc, opName, operands, attrNames,
                                           attrValues, results);
 }
 
@@ -527,7 +527,7 @@ Value CodeGen::genExprImpl(const ast::RangeExpr *expr) {
   for (const ast::Expr *element : expr->getElements())
     llvm::append_range(elements, genExpr(element));
 
-  return builder.create<pdl::RangeOp>(genLoc(expr->getLoc()),
+  return pdl::RangeOp::create(builder, genLoc(expr->getLoc()),
                                       genType(expr->getType()), elements);
 }
 
@@ -541,7 +541,7 @@ SmallVector<Value> CodeGen::genExprImpl(const ast::TupleExpr *expr) {
 Value CodeGen::genExprImpl(const ast::TypeExpr *expr) {
   Type type = parseType(expr->getValue(), builder.getContext());
   assert(type && "invalid MLIR type data");
-  return builder.create<pdl::TypeOp>(genLoc(expr->getLoc()),
+  return pdl::TypeOp::create(builder, genLoc(expr->getLoc()),
                                      builder.getType<pdl::TypeType>(),
                                      TypeAttr::get(type));
 }
@@ -586,7 +586,7 @@ CodeGen::genConstraintOrRewriteCall(const T *decl, Location loc,
     } else {
       resultTypes.push_back(genType(declResultType));
     }
-    PDLOpT pdlOp = builder.create<PDLOpT>(loc, resultTypes,
+    PDLOpT pdlOp = PDLOpT::create(builder, loc, resultTypes,
                                           decl->getName().getName(), inputs);
     if (isNegated && std::is_same_v<PDLOpT, pdl::ApplyNativeConstraintOp>)
       cast<pdl::ApplyNativeConstraintOp>(pdlOp).setIsNegated(true);
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 4d01a83d37165..f83184abe5308 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -1505,7 +1505,7 @@ ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
   OpBuilder builder(outputTypes.front().getContext());
   builder.setInsertionPoint(ip.getBlock(), ip.getPoint());
   auto convertOp =
-      builder.create<UnrealizedConversionCastOp>(loc, outputTypes, inputs);
+      UnrealizedConversionCastOp::create(builder, loc, outputTypes, inputs);
   if (!valuesToMap.empty())
     mapping.map(valuesToMap, convertOp.getResults());
   if (castOp)

>From 46e79c40e547e8a56aa0b9b4e46b3f39741eab64 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 12:56:56 -0400
Subject: [PATCH 2/5] fix cpp custom builders

---
 mlir/include/mlir/Dialect/Arith/IR/Arith.h | 10 +++
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp     | 79 ++++++++++++++++++----
 mlir/lib/IR/BuiltinDialect.cpp             |  2 +-
 3 files changed, 76 insertions(+), 15 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 7c50c2036ffdc..79b09b5fcaccf 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -59,15 +59,21 @@ class ConstantIntOp : public arith::ConstantOp {
   /// Build a constant int op that produces an integer of the specified width.
   static void build(OpBuilder &builder, OperationState &result, int64_t value,
                     unsigned width);
+  static ConstantIntOp create(OpBuilder &builder, Location location,
+                              int64_t value, unsigned width);
 
   /// Build a constant int op that produces an integer of the specified type,
   /// which must be an integer type.
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     int64_t value);
+  static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
+                              int64_t value);
 
   /// Build a constant int op that produces an integer from an APInt
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     const APInt &value);
+  static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
+                              const APInt &value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
@@ -85,6 +91,8 @@ class ConstantFloatOp : public arith::ConstantOp {
   /// Build a constant float op that produces a float of the specified type.
   static void build(OpBuilder &builder, OperationState &result, FloatType type,
                     const APFloat &value);
+  static ConstantFloatOp create(OpBuilder &builder, Location location,
+                                FloatType type, const APFloat &value);
 
   inline APFloat value() {
     return cast<FloatAttr>(arith::ConstantOp::getValue()).getValue();
@@ -100,6 +108,8 @@ class ConstantIndexOp : public arith::ConstantOp {
   static ::mlir::TypeID resolveTypeID() { return TypeID::get<ConstantOp>(); }
   /// Build a constant int op that produces an index.
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
+  static ConstantIndexOp create(OpBuilder &builder, Location location,
+                                int64_t value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 60c1fc6adacff..fdc2d3ac737e7 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -255,18 +255,49 @@ void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location,
+                                                  int64_t value,
+                                                  unsigned width) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, value, width);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, int64_t value) {
   arith::ConstantOp::build(builder, result, type,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location, Type type,
+                                                  int64_t value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, const APInt &value) {
   arith::ConstantOp::build(builder, result, type,
                            builder.getIntegerAttr(type, value));
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
+                                                  Location location, Type type,
+                                                  const APInt &value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantIntOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -279,6 +310,17 @@ void arith::ConstantFloatOp::build(OpBuilder &builder, OperationState &result,
                            builder.getFloatAttr(type, value));
 }
 
+arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
+                                                      Location location,
+                                                      FloatType type,
+                                                      const APFloat &value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, type, value);
+  auto result = llvm::dyn_cast<ConstantFloatOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantFloatOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return llvm::isa<FloatType>(constOp.getType());
@@ -291,6 +333,16 @@ void arith::ConstantIndexOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIndexAttr(value));
 }
 
+arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
+                                                      Location location,
+                                                      int64_t value) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, value);
+  auto result = llvm::dyn_cast<ConstantIndexOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
 bool arith::ConstantIndexOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isIndex();
@@ -2334,9 +2386,8 @@ class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
     // comparison.
     rewriter.replaceOpWithNewOp<CmpIOp>(
         op, pred, intVal,
-        ConstantOp::create(rewriter,
-            op.getLoc(), intVal.getType(),
-            rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
+        ConstantOp::create(rewriter, op.getLoc(), intVal.getType(),
+                           rewriter.getIntegerAttr(intVal.getType(), rhsInt)));
     return success();
   }
 };
@@ -2373,10 +2424,10 @@ struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
         matchPattern(op.getFalseValue(), m_One())) {
       rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
           op, op.getType(),
-          arith::XOrIOp::create(rewriter,
-              op.getLoc(), op.getCondition(),
-              arith::ConstantIntOp::create(rewriter,
-                  op.getLoc(), op.getCondition().getType(), 1)));
+          arith::XOrIOp::create(
+              rewriter, op.getLoc(), op.getCondition(),
+              arith::ConstantIntOp::create(rewriter, op.getLoc(),
+                                           op.getCondition().getType(), 1)));
       return success();
     }
 
@@ -2439,12 +2490,12 @@ OpFoldResult arith::SelectOp::fold(FoldAdaptor adaptor) {
 
   // Constant-fold constant operands over non-splat constant condition.
   // select %cst_vec, %cst0, %cst1 => %cst2
-  if (auto cond =
-          llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getCondition())) {
-    if (auto lhs =
-            llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getTrueValue())) {
-      if (auto rhs =
-              llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getFalseValue())) {
+  if (auto cond = llvm::dyn_cast_if_present<DenseElementsAttr>(
+          adaptor.getCondition())) {
+    if (auto lhs = llvm::dyn_cast_if_present<DenseElementsAttr>(
+            adaptor.getTrueValue())) {
+      if (auto rhs = llvm::dyn_cast_if_present<DenseElementsAttr>(
+              adaptor.getFalseValue())) {
         SmallVector<Attribute> results;
         results.reserve(static_cast<size_t>(cond.getNumElements()));
         auto condVals = llvm::make_range(cond.value_begin<BoolAttr>(),
@@ -2712,7 +2763,7 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
     return arith::MaximumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minimumf:
     return arith::MinimumFOp::create(builder, loc, lhs, rhs);
-   case AtomicRMWKind::maxnumf:
+  case AtomicRMWKind::maxnumf:
     return arith::MaxNumFOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::minnumf:
     return arith::MinNumFOp::create(builder, loc, lhs, rhs);
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index c88b328282275..6d7e2aa0ece7d 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -132,7 +132,7 @@ void ModuleOp::build(OpBuilder &builder, OperationState &state,
 /// Construct a module from the given context.
 ModuleOp ModuleOp::create(Location loc, std::optional<StringRef> name) {
   OpBuilder builder(loc->getContext());
-  return ModuleOp::create(builder, loc, name);
+  return builder.create<ModuleOp>(loc, name);
 }
 
 DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {

>From 08544213cbd4968a54c8720a4e868ab7a4b81485 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 17:06:46 -0400
Subject: [PATCH 3/5] fix arith+smt+create impmlicit

---
 .../mlir/Dialect/Affine/IR/AffineOps.h        |  21 ++++
 mlir/include/mlir/Dialect/Arith/IR/Arith.h    |   9 ++
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    | 116 +++++++++---------
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    |   2 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      |   2 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  98 ++++++++++-----
 .../Affine/Transforms/SuperVectorize.cpp      |  49 ++++----
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |  28 +++++
 .../Async/Transforms/AsyncParallelFor.cpp     | 105 ++++++++--------
 .../GPU/Transforms/MemoryPromotion.cpp        |   6 +-
 10 files changed, 269 insertions(+), 167 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
index 2091faa6b0b02..333de6bbd8a05 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
@@ -114,6 +114,21 @@ class AffineDmaStartOp
                     AffineMap tagMap, ValueRange tagIndices, Value numElements,
                     Value stride = nullptr, Value elementsPerStride = nullptr);
 
+  static AffineDmaStartOp
+  create(OpBuilder &builder, Location location, Value srcMemRef,
+         AffineMap srcMap, ValueRange srcIndices, Value destMemRef,
+         AffineMap dstMap, ValueRange destIndices, Value tagMemRef,
+         AffineMap tagMap, ValueRange tagIndices, Value numElements,
+         Value stride = nullptr, Value elementsPerStride = nullptr);
+
+  static AffineDmaStartOp create(ImplicitLocOpBuilder &builder, Value srcMemRef,
+                                 AffineMap srcMap, ValueRange srcIndices,
+                                 Value destMemRef, AffineMap dstMap,
+                                 ValueRange destIndices, Value tagMemRef,
+                                 AffineMap tagMap, ValueRange tagIndices,
+                                 Value numElements, Value stride = nullptr,
+                                 Value elementsPerStride = nullptr);
+
   /// Returns the operand index of the source memref.
   unsigned getSrcMemRefOperandIndex() { return 0; }
 
@@ -319,6 +334,12 @@ class AffineDmaWaitOp
 
   static void build(OpBuilder &builder, OperationState &result, Value tagMemRef,
                     AffineMap tagMap, ValueRange tagIndices, Value numElements);
+  static AffineDmaWaitOp create(OpBuilder &builder, Location location,
+                                Value tagMemRef, AffineMap tagMap,
+                                ValueRange tagIndices, Value numElements);
+  static AffineDmaWaitOp create(ImplicitLocOpBuilder &builder, Value tagMemRef,
+                                AffineMap tagMap, ValueRange tagIndices,
+                                Value numElements);
 
   static StringRef getOperationName() { return "affine.dma_wait"; }
 
diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 79b09b5fcaccf..0fc3db8e993d8 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -61,6 +61,8 @@ class ConstantIntOp : public arith::ConstantOp {
                     unsigned width);
   static ConstantIntOp create(OpBuilder &builder, Location location,
                               int64_t value, unsigned width);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, int64_t value,
+                              unsigned width);
 
   /// Build a constant int op that produces an integer of the specified type,
   /// which must be an integer type.
@@ -68,12 +70,16 @@ class ConstantIntOp : public arith::ConstantOp {
                     int64_t value);
   static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
                               int64_t value);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, Type type,
+                              int64_t value);
 
   /// Build a constant int op that produces an integer from an APInt
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     const APInt &value);
   static ConstantIntOp create(OpBuilder &builder, Location location, Type type,
                               const APInt &value);
+  static ConstantIntOp create(ImplicitLocOpBuilder &builder, Type type,
+                              const APInt &value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
@@ -93,6 +99,8 @@ class ConstantFloatOp : public arith::ConstantOp {
                     const APFloat &value);
   static ConstantFloatOp create(OpBuilder &builder, Location location,
                                 FloatType type, const APFloat &value);
+  static ConstantFloatOp create(ImplicitLocOpBuilder &builder, FloatType type,
+                                const APFloat &value);
 
   inline APFloat value() {
     return cast<FloatAttr>(arith::ConstantOp::getValue()).getValue();
@@ -110,6 +118,7 @@ class ConstantIndexOp : public arith::ConstantOp {
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
   static ConstantIndexOp create(OpBuilder &builder, Location location,
                                 int64_t value);
+  static ConstantIndexOp create(ImplicitLocOpBuilder &builder, int64_t value);
 
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 3f82342001ea5..69eec9d9bbf3e 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -20,8 +20,8 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
+#include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
@@ -114,7 +114,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
 
   auto makeConst = [&](int32_t index) -> Value {
     return LLVM::ConstantOp::create(rewriter, loc, IntegerType::get(ctx, 32),
-                                             rewriter.getI32IntegerAttr(index));
+                                    rewriter.getI32IntegerAttr(index));
   };
 
   if (arrayType) {
@@ -147,11 +147,11 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
         Value x1 =
             LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult, i * 2);
         Value x2 = LLVM::ExtractValueOp::create(rewriter, loc, intrinsicResult,
-                                                         i * 2 + 1);
+                                                i * 2 + 1);
         vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
-                                                     x1, makeConst(0));
+                                            x1, makeConst(0));
         vec = LLVM::InsertElementOp::create(rewriter, loc, vec.getType(), vec,
-                                                     x2, makeConst(1));
+                                            x2, makeConst(1));
         elements.push_back(vec);
       }
     }
@@ -160,7 +160,7 @@ static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
     Value result = LLVM::PoisonOp::create(rewriter, loc, arrayType);
     for (const auto &el : llvm::enumerate(elements)) {
       result = LLVM::InsertValueOp::create(rewriter, loc, result, el.value(),
-                                                    el.index());
+                                           el.index());
     }
     return result;
   }
@@ -208,8 +208,8 @@ static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
                          innerArrayTy.getElementType() == f32Ty)) {
       for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
            idx < innerSize; idx++) {
-        result.push_back(LLVM::ExtractElementOp::create(b,
-            toUse,
+        result.push_back(LLVM::ExtractElementOp::create(
+            b, toUse,
             LLVM::ConstantOp::create(b, i64Ty, b.getI64IntegerAttr(idx))));
       }
       continue;
@@ -285,8 +285,8 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
-    Value ldMatrixResult = NVVM::LdMatrixOp::create(b,
-        ldMatrixResultType, srcPtr,
+    Value ldMatrixResult = NVVM::LdMatrixOp::create(
+        b, ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
                                      : NVVM::MMALayout::row);
@@ -375,16 +375,16 @@ struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
     Type desiredRetTy = typeConverter->convertType(op->getResultTypes()[0]);
     Type intrinsicResTy = inferIntrinsicResultType(
         typeConverter->convertType(op->getResultTypes()[0]));
-    Value intrinsicResult = NVVM::MmaOp::create(b,
-        intrinsicResTy, matA, matB, matC,
-        /*shape=*/gemmShape,
-        /*b1Op=*/std::nullopt,
-        /*intOverflow=*/overflow,
-        /*multiplicandPtxTypes=*/
-        std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
-        /*multiplicandLayouts=*/
-        std::array<NVVM::MMALayout, 2>{NVVM::MMALayout::row,
-                                       NVVM::MMALayout::col});
+    Value intrinsicResult =
+        NVVM::MmaOp::create(b, intrinsicResTy, matA, matB, matC,
+                            /*shape=*/gemmShape,
+                            /*b1Op=*/std::nullopt,
+                            /*intOverflow=*/overflow,
+                            /*multiplicandPtxTypes=*/
+                            std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
+                            /*multiplicandLayouts=*/
+                            std::array<NVVM::MMALayout, 2>{
+                                NVVM::MMALayout::row, NVVM::MMALayout::col});
     rewriter.replaceOp(op, convertIntrinsicResult(op.getLoc(), intrinsicResTy,
                                                   desiredRetTy, intrinsicResult,
                                                   rewriter));
@@ -566,14 +566,15 @@ static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
   asmVals.push_back(indexData);
 
   return LLVM::InlineAsmOp::create(b,
-      /*resultTypes=*/intrinsicResultType,
-      /*operands=*/asmVals,
-      /*asm_string=*/asmStr,
-      /*constraints=*/constraintStr,
-      /*has_side_effects=*/true,
-      /*is_align_stack=*/false, LLVM::TailCallKind::None,
-      /*asm_dialect=*/asmDialectAttr,
-      /*operand_attrs=*/ArrayAttr());
+                                   /*resultTypes=*/intrinsicResultType,
+                                   /*operands=*/asmVals,
+                                   /*asm_string=*/asmStr,
+                                   /*constraints=*/constraintStr,
+                                   /*has_side_effects=*/true,
+                                   /*is_align_stack=*/false,
+                                   LLVM::TailCallKind::None,
+                                   /*asm_dialect=*/asmDialectAttr,
+                                   /*operand_attrs=*/ArrayAttr());
 }
 
 /// Lowers `nvgpu.mma.sp.sync` to inline assembly.
@@ -698,12 +699,12 @@ struct NVGPUAsyncCopyLowering
       // filled with zeros.
       Value c3I32 =
           LLVM::ConstantOp::create(b, b.getI32Type(), b.getI32IntegerAttr(3));
-      Value bitwidth = LLVM::ConstantOp::create(b,
-          b.getI32Type(),
+      Value bitwidth = LLVM::ConstantOp::create(
+          b, b.getI32Type(),
           b.getI32IntegerAttr(srcMemrefType.getElementTypeBitWidth()));
       Value srcElementsI32 = LLVM::TruncOp::create(b, b.getI32Type(), srcBytes);
-      srcBytes = LLVM::LShrOp::create(b,
-          LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
+      srcBytes = LLVM::LShrOp::create(
+          b, LLVM::MulOp::create(b, bitwidth, srcElementsI32), c3I32);
     }
     // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
     // 16 dst bytes.
@@ -712,14 +713,15 @@ struct NVGPUAsyncCopyLowering
             ? NVVM::LoadCacheModifierKind::CG
             : NVVM::LoadCacheModifierKind::CA;
 
-    NVVM::CpAsyncOp::create(b,
-        dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
+    NVVM::CpAsyncOp::create(
+        b, dstPtr, scrPtr, rewriter.getI32IntegerAttr(sizeInBytes),
         NVVM::LoadCacheModifierKindAttr::get(op->getContext(), cacheModifier),
         srcBytes);
 
     // Drop the result token.
-    Value zero = LLVM::ConstantOp::create(b,
-        IntegerType::get(op.getContext(), 32), rewriter.getI32IntegerAttr(0));
+    Value zero =
+        LLVM::ConstantOp::create(b, IntegerType::get(op.getContext(), 32),
+                                 rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
   }
@@ -735,9 +737,9 @@ struct NVGPUAsyncCreateGroupLowering
                   ConversionPatternRewriter &rewriter) const override {
     NVVM::CpAsyncCommitGroupOp::create(rewriter, op.getLoc());
     // Drop the result token.
-    Value zero = LLVM::ConstantOp::create(rewriter,
-        op->getLoc(), IntegerType::get(op.getContext(), 32),
-        rewriter.getI32IntegerAttr(0));
+    Value zero = LLVM::ConstantOp::create(rewriter, op->getLoc(),
+                                          IntegerType::get(op.getContext(), 32),
+                                          rewriter.getI32IntegerAttr(0));
     rewriter.replaceOp(op, zero);
     return success();
   }
@@ -771,8 +773,8 @@ struct NVGPUMBarrierCreateLowering
     SymbolTable symbolTable(moduleOp);
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(&moduleOp.front());
-    auto global = memref::GlobalOp::create(rewriter,
-        funcOp->getLoc(), "__mbarrier",
+    auto global = memref::GlobalOp::create(
+        rewriter, funcOp->getLoc(), "__mbarrier",
         /*sym_visibility=*/rewriter.getStringAttr("private"),
         /*type=*/barrierType,
         /*initial_value=*/ElementsAttr(),
@@ -1119,7 +1121,7 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
 
 static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
   return LLVM::ConstantOp::create(b, b.getIntegerType(64),
-                                    b.getI32IntegerAttr(index));
+                                  b.getI32IntegerAttr(index));
 }
 
 /// Returns a Value that holds data type enum that is expected by CUDA driver.
@@ -1182,11 +1184,11 @@ struct NVGPUTmaCreateDescriptorOpLowering
     auto promotedOperands = getTypeConverter()->promoteOperands(
         b.getLoc(), op->getOperands(), adaptor.getOperands(), b);
 
-    Value boxArrayPtr = LLVM::AllocaOp::create(b, llvmPointerType, llvmInt64Type,
-                                                 makeI64Const(b, 5));
+    Value boxArrayPtr = LLVM::AllocaOp::create(
+        b, llvmPointerType, llvmInt64Type, makeI64Const(b, 5));
     for (auto [index, value] : llvm::enumerate(adaptor.getBoxDimensions())) {
       Value gep = LLVM::GEPOp::create(b, llvmPointerType, llvmPointerType,
-                                        boxArrayPtr, makeI64Const(b, index));
+                                      boxArrayPtr, makeI64Const(b, index));
       LLVM::StoreOp::create(b, value, gep);
     }
 
@@ -1430,9 +1432,9 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      return NVVM::WgmmaMmaAsyncOp::create(b,
-          matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
-          itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
+      return NVVM::WgmmaMmaAsyncOp::create(
+          b, matrixC.getType(), matrixC, descriptorA, descriptorB, shape,
+          itypeA, itypeB, itypeD, scaleOut, scaleIn, scaleIn, layoutA, layoutB,
           overflow);
     }
 
@@ -1444,7 +1446,8 @@ struct NVGPUWarpgroupMmaOpLowering
       // Perform GEMM
       SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
+        Value matrixC =
+            LLVM::ExtractValueOp::create(b, adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
             matrixC = generateWgmma(i, j, k, matrixC);
@@ -1452,7 +1455,7 @@ struct NVGPUWarpgroupMmaOpLowering
       }
       for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
         wgmmaResult = LLVM::InsertValueOp::create(b, wgmmaResult.getType(),
-                                                    wgmmaResult, matrix, idx);
+                                                  wgmmaResult, matrix, idx);
       }
       return wgmmaResult;
     }
@@ -1486,9 +1489,9 @@ struct NVGPUWarpgroupMmaOpLowering
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
     Value generateWarpgroupMma() {
-      NVVM::WgmmaFenceAlignedOp::create(b, );
+      NVVM::WgmmaFenceAlignedOp::create(b);
       Value wgmmaResult = generateWgmmaGroup();
-      NVVM::WgmmaGroupSyncAlignedOp::create(b, );
+      NVVM::WgmmaGroupSyncAlignedOp::create(b);
       NVVM::WgmmaWaitGroupSyncOp::create(b, op.getWaitGroup());
       return wgmmaResult;
     }
@@ -1626,7 +1629,8 @@ struct NVGPUWarpgroupMmaStoreOpLowering
     auto stype = cast<LLVM::LLVMStructType>(matriDValue.getType());
     for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
       auto structType = cast<LLVM::LLVMStructType>(matrixD);
-      Value innerStructValue = LLVM::ExtractValueOp::create(b, matriDValue, idx);
+      Value innerStructValue =
+          LLVM::ExtractValueOp::create(b, matriDValue, idx);
       storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
@@ -1656,15 +1660,15 @@ struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
       auto structType = cast<LLVM::LLVMStructType>(s);
       Value structValue = LLVM::ExtractValueOp::create(b, packStruct, idx);
       for (unsigned i = 0; i < structType.getBody().size(); ++i) {
-        structValue = LLVM::InsertValueOp::create(b,
-            structType, structValue, zero, ArrayRef<int64_t>({i}));
+        structValue = LLVM::InsertValueOp::create(b, structType, structValue,
+                                                  zero, ArrayRef<int64_t>({i}));
       }
       innerStructs.push_back(structValue);
     }
     // Pack the inner structs into a single struct
     for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
       packStruct = LLVM::InsertValueOp::create(b, packStruct.getType(),
-                                                 packStruct, matrix, idx);
+                                               packStruct, matrix, idx);
     }
     rewriter.replaceOp(op, packStruct);
     return success();
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 3eee444725f74..f490e065143e7 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -709,7 +709,7 @@ class ExecutionModePattern
         loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
-    structValue = LLVM::InsertValueOp::create(rewriter, loc, structValue,
+    structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
                                                        executionMode, 0);
 
     // Insert extra operands if they exist into execution mode info struct.
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 1b31d391cb0bc..d21833736a718 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -670,7 +670,7 @@ static Value createFPReductionComparisonOpLowering(
 
   if (accumulator) {
     result =
-        typename VectorToScalarMapper<LLVMRedIntrinOp>::Type::create(rewriter,
+        rewriter.create<typename VectorToScalarMapper<LLVMRedIntrinOp>::Type>(
             loc, result, accumulator);
   }
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 3113d5896741b..bd1be7c087d05 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -1747,6 +1747,32 @@ void AffineDmaStartOp::build(OpBuilder &builder, OperationState &result,
   }
 }
 
+AffineDmaStartOp AffineDmaStartOp::create(
+    OpBuilder &builder, Location location, Value srcMemRef, AffineMap srcMap,
+    ValueRange srcIndices, Value destMemRef, AffineMap dstMap,
+    ValueRange destIndices, Value tagMemRef, AffineMap tagMap,
+    ValueRange tagIndices, Value numElements, Value stride,
+    Value elementsPerStride) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, srcMemRef, srcMap, srcIndices, destMemRef, dstMap,
+        destIndices, tagMemRef, tagMap, tagIndices, numElements, stride,
+        elementsPerStride);
+  auto result = llvm::dyn_cast<AffineDmaStartOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
+AffineDmaStartOp AffineDmaStartOp::create(
+    ImplicitLocOpBuilder &builder, Value srcMemRef, AffineMap srcMap,
+    ValueRange srcIndices, Value destMemRef, AffineMap dstMap,
+    ValueRange destIndices, Value tagMemRef, AffineMap tagMap,
+    ValueRange tagIndices, Value numElements, Value stride,
+    Value elementsPerStride) {
+  return create(builder, builder.getLoc(), srcMemRef, srcMap, srcIndices,
+                destMemRef, dstMap, destIndices, tagMemRef, tagMap, tagIndices,
+                numElements, stride, elementsPerStride);
+}
+
 void AffineDmaStartOp::print(OpAsmPrinter &p) {
   p << " " << getSrcMemRef() << '[';
   p.printAffineMapOfSSAIds(getSrcMapAttr(), getSrcIndices());
@@ -1917,6 +1943,25 @@ void AffineDmaWaitOp::build(OpBuilder &builder, OperationState &result,
   result.addOperands(numElements);
 }
 
+AffineDmaWaitOp AffineDmaWaitOp::create(OpBuilder &builder, Location location,
+                                        Value tagMemRef, AffineMap tagMap,
+                                        ValueRange tagIndices,
+                                        Value numElements) {
+  mlir::OperationState state(location, getOperationName());
+  build(builder, state, tagMemRef, tagMap, tagIndices, numElements);
+  auto result = llvm::dyn_cast<AffineDmaWaitOp>(builder.create(state));
+  assert(result && "builder didn't return the right type");
+  return result;
+}
+
+AffineDmaWaitOp AffineDmaWaitOp::create(ImplicitLocOpBuilder &builder,
+                                        Value tagMemRef, AffineMap tagMap,
+                                        ValueRange tagIndices,
+                                        Value numElements) {
+  return create(builder, builder.getLoc(), tagMemRef, tagMap, tagIndices,
+                numElements);
+}
+
 void AffineDmaWaitOp::print(OpAsmPrinter &p) {
   p << " " << getTagMemRef() << '[';
   SmallVector<Value, 2> operands(getTagIndices());
@@ -2688,8 +2733,8 @@ FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInits());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  AffineForOp newLoop = AffineForOp::create(rewriter,
-      getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
+  AffineForOp newLoop = AffineForOp::create(
+      rewriter, getLoc(), getLowerBoundOperands(), getLowerBoundMap(),
       getUpperBoundOperands(), getUpperBoundMap(), getStepAsInt(), inits);
 
   // Generate the new yield values and append them to the scf.yield operation.
@@ -2847,7 +2892,7 @@ buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb,
                              int64_t ub, int64_t step,
                              AffineForOp::BodyBuilderFn bodyBuilderFn) {
   return AffineForOp::create(builder, loc, lb, ub, step,
-                                     /*iterArgs=*/ValueRange(), bodyBuilderFn);
+                             /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
 /// Creates an affine loop from the bounds that may or may not be constants.
@@ -2861,8 +2906,8 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
     return buildAffineLoopFromConstants(builder, loc, lbConst.value(),
                                         ubConst.value(), step, bodyBuilderFn);
   return AffineForOp::create(builder, loc, lb, builder.getDimIdentityMap(), ub,
-                                     builder.getDimIdentityMap(), step,
-                                     /*iterArgs=*/ValueRange(), bodyBuilderFn);
+                             builder.getDimIdentityMap(), step,
+                             /*iterArgs=*/ValueRange(), bodyBuilderFn);
 }
 
 void mlir::affine::buildAffineLoopNest(
@@ -4906,8 +4951,8 @@ struct DropUnitExtentBasis
 
     if (!newBasis.empty()) {
       // Will drop the leading nullptr from `basis` if there was no outer bound.
-      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(rewriter,
-          loc, delinearizeOp.getLinearIndex(), newBasis);
+      auto newDelinearizeOp = affine::AffineDelinearizeIndexOp::create(
+          rewriter, loc, delinearizeOp.getLinearIndex(), newBasis);
       int newIndex = 0;
       // Map back the new delinearized indices to the values they replace.
       for (auto &replacement : replacements) {
@@ -4971,12 +5016,12 @@ struct CancelDelinearizeOfLinearizeDisjointExactTail
       return success();
     }
 
-    Value newLinearize = affine::AffineLinearizeIndexOp::create(rewriter,
-        linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
+    Value newLinearize = affine::AffineLinearizeIndexOp::create(
+        rewriter, linearizeOp.getLoc(), linearizeIns.drop_back(numMatches),
         ArrayRef<OpFoldResult>{linearizeBasis}.drop_back(numMatches),
         linearizeOp.getDisjoint());
-    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-        delinearizeOp.getLoc(), newLinearize,
+    auto newDelinearize = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), newLinearize,
         ArrayRef<OpFoldResult>{delinearizeBasis}.drop_back(numMatches),
         delinearizeOp.hasOuterBound());
     SmallVector<Value> mergedResults(newDelinearize.getResults());
@@ -5048,19 +5093,16 @@ struct SplitDelinearizeSpanningLastLinearizeArg final
           delinearizeOp,
           "need at least two elements to form the basis product");
 
-    Value linearizeWithoutBack =
-        affine::AffineLinearizeIndexOp::create(rewriter,
-            linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
-            linearizeOp.getDynamicBasis(),
-            linearizeOp.getStaticBasis().drop_back(),
-            linearizeOp.getDisjoint());
-    auto delinearizeWithoutSplitPart =
-        affine::AffineDelinearizeIndexOp::create(rewriter,
-            delinearizeOp.getLoc(), linearizeWithoutBack,
-            delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
-            delinearizeOp.hasOuterBound());
-    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(rewriter,
-        delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
+    Value linearizeWithoutBack = affine::AffineLinearizeIndexOp::create(
+        rewriter, linearizeOp.getLoc(), linearizeOp.getMultiIndex().drop_back(),
+        linearizeOp.getDynamicBasis(), linearizeOp.getStaticBasis().drop_back(),
+        linearizeOp.getDisjoint());
+    auto delinearizeWithoutSplitPart = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), linearizeWithoutBack,
+        delinearizeOp.getDynamicBasis(), basis.drop_back(elemsToSplit),
+        delinearizeOp.hasOuterBound());
+    auto delinearizeBack = affine::AffineDelinearizeIndexOp::create(
+        rewriter, delinearizeOp.getLoc(), linearizeOp.getMultiIndex().back(),
         basis.take_back(elemsToSplit), /*hasOuterBound=*/true);
     SmallVector<Value> results = llvm::to_vector(
         llvm::concat<Value>(delinearizeWithoutSplitPart.getResults(),
@@ -5437,16 +5479,16 @@ struct CancelLinearizeOfDelinearizePortion final
       newDelinBasis.erase(newDelinBasis.begin() + m.delinStart,
                           newDelinBasis.begin() + m.delinStart + m.length);
       newDelinBasis.insert(newDelinBasis.begin() + m.delinStart, newSize);
-      auto newDelinearize = AffineDelinearizeIndexOp::create(rewriter,
-          m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
+      auto newDelinearize = AffineDelinearizeIndexOp::create(
+          rewriter, m.delinearize.getLoc(), m.delinearize.getLinearIndex(),
           newDelinBasis);
 
       // Since there may be other uses of the indices we just merged together,
       // create a residual affine.delinearize_index that delinearizes the
       // merged output into its component parts.
       Value combinedElem = newDelinearize.getResult(m.delinStart);
-      auto residualDelinearize = AffineDelinearizeIndexOp::create(rewriter,
-          m.delinearize.getLoc(), combinedElem, basisToMerge);
+      auto residualDelinearize = AffineDelinearizeIndexOp::create(
+          rewriter, m.delinearize.getLoc(), combinedElem, basisToMerge);
 
       // Swap all the uses of the unaffected delinearize outputs to the new
       // delinearization so that the old code can be removed if this
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index b078514cbf978..827e1dff9f319 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -905,8 +905,8 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   for (auto resultExpr : map.getResults()) {
     auto singleResMap =
         AffineMap::get(map.getNumDims(), map.getNumSymbols(), resultExpr);
-    auto afOp = state.AffineApplyOp::create(builder, op->getLoc(), singleResMap,
-                                                    mapOperands);
+    auto afOp = AffineApplyOp::create(state.builder, op->getLoc(), singleResMap,
+                                      mapOperands);
     results.push_back(afOp);
   }
 }
@@ -961,7 +961,7 @@ static arith::ConstantOp vectorizeConstant(arith::ConstantOp constOp,
   auto vecForOp = cast<AffineForOp>(parentOp);
   state.builder.setInsertionPointToStart(vecForOp.getBody());
   auto newConstOp =
-      state.arith::ConstantOp::create(builder, constOp.getLoc(), vecAttr);
+      arith::ConstantOp::create(state.builder, constOp.getLoc(), vecAttr);
 
   // Register vector replacement for future uses in the scope.
   state.registerOpVectorReplacement(constOp, newConstOp);
@@ -986,8 +986,8 @@ static Operation *vectorizeAffineApplyOp(AffineApplyOp applyOp,
     }
   }
 
-  auto newApplyOp = state.AffineApplyOp::create(builder,
-      applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
+  auto newApplyOp = AffineApplyOp::create(
+      state.builder, applyOp.getLoc(), applyOp.getAffineMap(), updatedOperands);
 
   // Register the new affine.apply result.
   state.registerValueScalarReplacement(applyOp.getResult(),
@@ -1010,7 +1010,7 @@ static arith::ConstantOp createInitialVector(arith::AtomicRMWKind reductionKind,
   auto vecTy = getVectorType(scalarTy, state.strategy);
   auto vecAttr = DenseElementsAttr::get(vecTy, valueAttr);
   auto newConstOp =
-      state.arith::ConstantOp::create(builder, oldOperand.getLoc(), vecAttr);
+      arith::ConstantOp::create(state.builder, oldOperand.getLoc(), vecAttr);
 
   return newConstOp;
 }
@@ -1062,11 +1062,11 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   AffineMap ubMap = vecForOp.getUpperBoundMap();
   Value ub;
   if (ubMap.getNumResults() == 1)
-    ub = state.AffineApplyOp::create(builder, loc, vecForOp.getUpperBoundMap(),
-                                             vecForOp.getUpperBoundOperands());
+    ub = AffineApplyOp::create(state.builder, loc, vecForOp.getUpperBoundMap(),
+                               vecForOp.getUpperBoundOperands());
   else
-    ub = state.AffineMinOp::create(builder, loc, vecForOp.getUpperBoundMap(),
-                                           vecForOp.getUpperBoundOperands());
+    ub = AffineMinOp::create(state.builder, loc, vecForOp.getUpperBoundMap(),
+                             vecForOp.getUpperBoundOperands());
   // Then we compute the number of (original) iterations left in the loop.
   AffineExpr subExpr =
       state.builder.getAffineDimExpr(0) - state.builder.getAffineDimExpr(1);
@@ -1080,7 +1080,7 @@ static Value createMask(AffineForOp vecForOp, VectorizationState &state) {
   Type maskTy = VectorType::get(state.strategy->vectorSizes,
                                 state.builder.getIntegerType(1));
   Value mask =
-      state.vector::CreateMaskOp::create(builder, loc, maskTy, itersLeft);
+      vector::CreateMaskOp::create(state.builder, loc, maskTy, itersLeft);
 
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ creating a mask:\n"
                     << itersLeft << "\n"
@@ -1123,8 +1123,8 @@ static Operation *vectorizeUniform(Value uniformVal,
   state.builder.setInsertionPointAfterValue(uniformScalarRepl);
 
   auto vectorTy = getVectorType(uniformVal.getType(), state.strategy);
-  auto bcastOp = state.BroadcastOp::create(builder, uniformVal.getLoc(),
-                                                   vectorTy, uniformScalarRepl);
+  auto bcastOp = BroadcastOp::create(state.builder, uniformVal.getLoc(),
+                                     vectorTy, uniformScalarRepl);
   state.registerValueVectorReplacement(uniformVal, bcastOp);
   return bcastOp;
 }
@@ -1256,8 +1256,8 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.vector::TransferReadOp::create(builder,
-      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
+  auto transfer = vector::TransferReadOp::create(
+      state.builder, loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
       /*padding=*/std::nullopt, permutationMap);
 
   // Register replacement for future uses in the scope.
@@ -1303,9 +1303,9 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  auto transfer = state.vector::TransferWriteOp::create(builder,
-      storeOp.getLoc(), vectorValue, storeOp.getMemRef(), indices,
-      permutationMap);
+  auto transfer = vector::TransferWriteOp::create(
+      state.builder, storeOp.getLoc(), vectorValue, storeOp.getMemRef(),
+      indices, permutationMap);
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
 
   // Register replacement for future uses in the scope.
@@ -1387,10 +1387,10 @@ static Operation *vectorizeAffineForOp(AffineForOp forOp,
     }
   }
 
-  auto vecForOp = state.AffineForOp::create(builder,
-      forOp.getLoc(), forOp.getLowerBoundOperands(), forOp.getLowerBoundMap(),
-      forOp.getUpperBoundOperands(), forOp.getUpperBoundMap(), newStep,
-      vecIterOperands,
+  auto vecForOp = AffineForOp::create(
+      state.builder, forOp.getLoc(), forOp.getLowerBoundOperands(),
+      forOp.getLowerBoundMap(), forOp.getUpperBoundOperands(),
+      forOp.getUpperBoundMap(), newStep, vecIterOperands,
       /*bodyBuilder=*/[](OpBuilder &, Location, Value, ValueRange) {
         // Make sure we don't create a default terminator in the loop body as
         // the proper terminator will be added during vectorization.
@@ -1512,8 +1512,8 @@ static Operation *vectorizeAffineYieldOp(AffineYieldOp yieldOp,
       // IterOperands are neutral element vectors.
       Value neutralVal = cast<AffineForOp>(newParentOp).getInits()[i];
       state.builder.setInsertionPoint(combinerOps.back());
-      Value maskedReducedVal = state.arith::SelectOp::create(builder,
-          reducedVal.getLoc(), mask, reducedVal, neutralVal);
+      Value maskedReducedVal = arith::SelectOp::create(
+          state.builder, reducedVal.getLoc(), mask, reducedVal, neutralVal);
       LLVM_DEBUG(
           dbgs() << "\n[early-vect]+++++ masking an input to a binary op that"
                     "produces value for a yield Op: "
@@ -1865,7 +1865,6 @@ verifyLoopNesting(const std::vector<SmallVector<AffineForOp, 2>> &loops) {
   return success();
 }
 
-
 /// External utility to vectorize affine loops in 'loops' using the n-D
 /// vectorization factors in 'vectorSizes'. By default, each vectorization
 /// factor is applied inner-to-outer to the loops of each loop nest.
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index fdc2d3ac737e7..ac00917d33a85 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -266,6 +266,12 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  int64_t value,
+                                                  unsigned width) {
+  return create(builder, builder.getLoc(), value, width);
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, int64_t value) {
   arith::ConstantOp::build(builder, result, type,
@@ -282,6 +288,11 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  Type type, int64_t value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                                  Type type, const APInt &value) {
   arith::ConstantOp::build(builder, result, type,
@@ -298,6 +309,12 @@ arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
+                                                  Type type,
+                                                  const APInt &value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -321,6 +338,12 @@ arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantFloatOp
+arith::ConstantFloatOp::create(ImplicitLocOpBuilder &builder, FloatType type,
+                               const APFloat &value) {
+  return create(builder, builder.getLoc(), type, value);
+}
+
 bool arith::ConstantFloatOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return llvm::isa<FloatType>(constOp.getType());
@@ -343,6 +366,11 @@ arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
   return result;
 }
 
+arith::ConstantIndexOp
+arith::ConstantIndexOp::create(ImplicitLocOpBuilder &builder, int64_t value) {
+  return create(builder, builder.getLoc(), value);
+}
+
 bool arith::ConstantIndexOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isIndex();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index 447411ad989ba..09367ab544ff8 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -369,21 +369,22 @@ static ParallelComputeFunction createParallelComputeFunction(
       ImplicitLocOpBuilder b(loc, nestedBuilder);
 
       // Compute induction variable for `loopIdx`.
-      computeBlockInductionVars[loopIdx] = arith::AddIOp::create(b,
-          lowerBounds[loopIdx], arith::MulIOp::create(b, iv, steps[loopIdx]));
+      computeBlockInductionVars[loopIdx] =
+          arith::AddIOp::create(b, lowerBounds[loopIdx],
+                                arith::MulIOp::create(b, iv, steps[loopIdx]));
 
       // Check if we are inside first or last iteration of the loop.
-      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
-      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
+      isBlockFirstCoord[loopIdx] = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::eq, iv, blockFirstCoord[loopIdx]);
+      isBlockLastCoord[loopIdx] = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::eq, iv, blockLastCoord[loopIdx]);
 
       // Check if the previous loop is in its first or last iteration.
       if (loopIdx > 0) {
-        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(b,
-            isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
-        isBlockLastCoord[loopIdx] = arith::AndIOp::create(b,
-            isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
+        isBlockFirstCoord[loopIdx] = arith::AndIOp::create(
+            b, isBlockFirstCoord[loopIdx], isBlockFirstCoord[loopIdx - 1]);
+        isBlockLastCoord[loopIdx] = arith::AndIOp::create(
+            b, isBlockLastCoord[loopIdx], isBlockLastCoord[loopIdx - 1]);
       }
 
       // Keep building loop nest.
@@ -392,20 +393,20 @@ static ParallelComputeFunction createParallelComputeFunction(
           // For block aligned loops we always iterate starting from 0 up to
           // the loop trip counts.
           scf::ForOp::create(b, c0, tripCounts[loopIdx + 1], c1, ValueRange(),
-                               workLoopBuilder(loopIdx + 1));
+                             workLoopBuilder(loopIdx + 1));
 
         } else {
           // Select nested loop lower/upper bounds depending on our position in
           // the multi-dimensional iteration space.
           auto lb = arith::SelectOp::create(b, isBlockFirstCoord[loopIdx],
-                                              blockFirstCoord[loopIdx + 1], c0);
+                                            blockFirstCoord[loopIdx + 1], c0);
 
           auto ub = arith::SelectOp::create(b, isBlockLastCoord[loopIdx],
-                                              blockEndCoord[loopIdx + 1],
-                                              tripCounts[loopIdx + 1]);
+                                            blockEndCoord[loopIdx + 1],
+                                            tripCounts[loopIdx + 1]);
 
           scf::ForOp::create(b, lb, ub, c1, ValueRange(),
-                               workLoopBuilder(loopIdx + 1));
+                             workLoopBuilder(loopIdx + 1));
         }
 
         scf::YieldOp::create(b, loc);
@@ -424,7 +425,7 @@ static ParallelComputeFunction createParallelComputeFunction(
   };
 
   scf::ForOp::create(b, blockFirstCoord[0], blockEndCoord[0], c1, ValueRange(),
-                       workLoopBuilder(0));
+                     workLoopBuilder(0));
   func::ReturnOp::create(b, ValueRange());
 
   return {op.getNumLoops(), func, std::move(computeFuncType.captures)};
@@ -537,13 +538,13 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
       operands[2] = end;
 
       func::CallOp::create(executeBuilder, executeLoc, func.getSymName(),
-                                          func.getResultTypes(), operands);
+                           func.getResultTypes(), operands);
       async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to dispatch half of the block range.
     auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
-                                       executeBodyBuilder);
+                                     executeBodyBuilder);
     AddToGroupOp::create(b, indexTy, execute.getToken(), group);
     scf::YieldOp::create(b, ValueRange({start, midIndex}));
   }
@@ -558,8 +559,7 @@ createAsyncDispatchFunction(ParallelComputeFunction &computeFunc,
   computeFuncOperands.append(forwardedInputs.begin(), forwardedInputs.end());
 
   func::CallOp::create(b, computeFunc.func.getSymName(),
-                         computeFunc.func.getResultTypes(),
-                         computeFuncOperands);
+                       computeFunc.func.getResultTypes(), computeFuncOperands);
   func::ReturnOp::create(b, ValueRange());
 
   return func;
@@ -605,9 +605,9 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     appendBlockComputeOperands(operands);
 
     func::CallOp::create(b, parallelComputeFunction.func.getSymName(),
-                           parallelComputeFunction.func.getResultTypes(),
-                           operands);
-    scf::YieldOp::create(b, );
+                         parallelComputeFunction.func.getResultTypes(),
+                         operands);
+    scf::YieldOp::create(b);
   };
 
   auto asyncDispatch = [&](OpBuilder &nestedBuilder, Location loc) {
@@ -624,12 +624,12 @@ static void doAsyncDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     appendBlockComputeOperands(operands);
 
     func::CallOp::create(b, asyncDispatchFunction.getSymName(),
-                           asyncDispatchFunction.getResultTypes(), operands);
+                         asyncDispatchFunction.getResultTypes(), operands);
 
     // Wait for the completion of all parallel compute operations.
     AwaitAllOp::create(b, group);
 
-    scf::YieldOp::create(b, );
+    scf::YieldOp::create(b);
   };
 
   // Dispatch either single block compute function, or launch async dispatch.
@@ -682,16 +682,15 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
     auto executeBodyBuilder = [&](OpBuilder &executeBuilder,
                                   Location executeLoc, ValueRange executeArgs) {
       func::CallOp::create(executeBuilder, executeLoc, compute.getSymName(),
-                                          compute.getResultTypes(),
-                                          computeFuncOperands(iv));
+                           compute.getResultTypes(), computeFuncOperands(iv));
       async::YieldOp::create(executeBuilder, executeLoc, ValueRange());
     };
 
     // Create async.execute operation to launch parallel computate function.
     auto execute = ExecuteOp::create(b, TypeRange(), ValueRange(), ValueRange(),
-                                       executeBodyBuilder);
+                                     executeBodyBuilder);
     AddToGroupOp::create(b, rewriter.getIndexType(), execute.getToken(), group);
-    scf::YieldOp::create(b, );
+    scf::YieldOp::create(b);
   };
 
   // Iterate over all compute blocks and launch parallel compute operations.
@@ -699,7 +698,7 @@ doSequentialDispatch(ImplicitLocOpBuilder &b, PatternRewriter &rewriter,
 
   // Call parallel compute function for the first block in the caller thread.
   func::CallOp::create(b, compute.getSymName(), compute.getResultTypes(),
-                         computeFuncOperands(c0));
+                       computeFuncOperands(c0));
 
   // Wait for the completion of all async compute operations.
   AwaitAllOp::create(b, group);
@@ -800,7 +799,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     if (numWorkerThreads >= 0)
       numWorkerThreadsVal = arith::ConstantIndexOp::create(b, numWorkerThreads);
     else
-      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b, );
+      numWorkerThreadsVal = async::RuntimeNumWorkerThreadsOp::create(b);
 
     // With large number of threads the value of creating many compute blocks
     // is reduced because the problem typically becomes memory bound. For this
@@ -819,16 +818,16 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
         {4, 4.0f}, {8, 2.0f}, {16, 1.0f}, {32, 0.8f}, {64, 0.6f}};
     const float initialOvershardingFactor = 8.0f;
 
-    Value scalingFactor = arith::ConstantFloatOp::create(b,
-        b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
+    Value scalingFactor = arith::ConstantFloatOp::create(
+        b, b.getF32Type(), llvm::APFloat(initialOvershardingFactor));
     for (const std::pair<int, float> &p : overshardingBrackets) {
       Value bracketBegin = arith::ConstantIndexOp::create(b, p.first);
-      Value inBracket = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
-      Value bracketScalingFactor = arith::ConstantFloatOp::create(b,
-          b.getF32Type(), llvm::APFloat(p.second));
-      scalingFactor = arith::SelectOp::create(b, inBracket, bracketScalingFactor,
-                                                scalingFactor);
+      Value inBracket = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::sgt, numWorkerThreadsVal, bracketBegin);
+      Value bracketScalingFactor = arith::ConstantFloatOp::create(
+          b, b.getF32Type(), llvm::APFloat(p.second));
+      scalingFactor = arith::SelectOp::create(
+          b, inBracket, bracketScalingFactor, scalingFactor);
     }
     Value numWorkersIndex =
         arith::IndexCastOp::create(b, b.getI32Type(), numWorkerThreadsVal);
@@ -841,8 +840,8 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
     Value scaledWorkers =
         arith::IndexCastOp::create(b, b.getIndexType(), scaledNumInt);
 
-    Value maxComputeBlocks = arith::MaxSIOp::create(b,
-        arith::ConstantIndexOp::create(b, 1), scaledWorkers);
+    Value maxComputeBlocks = arith::MaxSIOp::create(
+        b, arith::ConstantIndexOp::create(b, 1), scaledWorkers);
 
     // Compute parallel block size from the parallel problem size:
     //   blockSize = min(tripCount,
@@ -869,7 +868,7 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       doDispatch(b, rewriter, compute, op, blockSize, blockCount, tripCounts);
-      scf::YieldOp::create(b, );
+      scf::YieldOp::create(b);
     };
 
     // Dispatch parallel compute function with hints for unrolling inner loops.
@@ -880,27 +879,27 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
       ImplicitLocOpBuilder b(loc, nestedBuilder);
       // Align the block size to be a multiple of the statically known
       // number of iterations in the inner loops.
-      Value numIters = arith::ConstantIndexOp::create(b,
-          numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value alignedBlockSize = arith::MulIOp::create(b,
-          arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
+      Value numIters = arith::ConstantIndexOp::create(
+          b, numIterations[op.getNumLoops() - numUnrollableLoops]);
+      Value alignedBlockSize = arith::MulIOp::create(
+          b, arith::CeilDivSIOp::create(b, blockSize, numIters), numIters);
       doDispatch(b, rewriter, compute, op, alignedBlockSize, blockCount,
                  tripCounts);
-      scf::YieldOp::create(b, );
+      scf::YieldOp::create(b);
     };
 
     // Dispatch to block aligned compute function only if the computed block
     // size is larger than the number of iterations in the unrollable inner
     // loops, because otherwise it can reduce the available parallelism.
     if (numUnrollableLoops > 0) {
-      Value numIters = arith::ConstantIndexOp::create(b,
-          numIterations[op.getNumLoops() - numUnrollableLoops]);
-      Value useBlockAlignedComputeFn = arith::CmpIOp::create(b,
-          arith::CmpIPredicate::sge, blockSize, numIters);
+      Value numIters = arith::ConstantIndexOp::create(
+          b, numIterations[op.getNumLoops() - numUnrollableLoops]);
+      Value useBlockAlignedComputeFn = arith::CmpIOp::create(
+          b, arith::CmpIPredicate::sge, blockSize, numIters);
 
       scf::IfOp::create(b, useBlockAlignedComputeFn, dispatchBlockAligned,
-                          dispatchDefault);
-      scf::YieldOp::create(b, );
+                        dispatchDefault);
+      scf::YieldOp::create(b);
     } else {
       dispatchDefault(b, loc);
     }
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 2bee1a1619665..4af08dad08462 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -18,7 +18,7 @@
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
+#include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
@@ -131,10 +131,10 @@ static void insertCopies(Region &region, Location loc, Value from, Value to) {
 
   auto b = ImplicitLocOpBuilder::atBlockBegin(loc, &region.front());
   insertCopyLoops(b, from, to);
-  gpu::BarrierOp::create(b, );
+  gpu::BarrierOp::create(b);
 
   b.setInsertionPoint(&region.front().back());
-  gpu::BarrierOp::create(b, );
+  gpu::BarrierOp::create(b);
   insertCopyLoops(b, to, from);
 }
 

>From 5b7efd80ce8d610d735421d82e5a6befe89532f8 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 7 Jul 2025 18:26:17 -0400
Subject: [PATCH 4/5] implicitbuilder

---
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    |   1 +
 flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp   |   1 +
 flang/lib/Optimizer/Dialect/FIROps.cpp        |   1 +
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp     |   1 +
 .../lib/Standalone/StandaloneDialect.cpp      |   1 +
 .../lib/Standalone/StandaloneOps.cpp          |   1 +
 mlir/examples/toy/Ch2/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch3/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch4/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch5/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch6/mlir/Dialect.cpp        |   7 +-
 mlir/examples/toy/Ch7/mlir/Dialect.cpp        |   4 +-
 .../transform/Ch2/lib/MyExtension.cpp         |   1 +
 .../transform/Ch3/lib/MyExtension.cpp         |   1 +
 .../transform/Ch4/lib/MyExtension.cpp         |   1 +
 mlir/lib/CAPI/Dialect/PDL.cpp                 |   1 +
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    |   1 +
 mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp  |   1 +
 mlir/lib/Dialect/AMX/IR/AMXDialect.cpp        |   1 +
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |   1 +
 .../TransformOps/AffineTransformOps.cpp       |   1 +
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |   1 +
 .../lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp |   1 +
 .../ArmNeonVectorTransformOps.cpp             |   1 +
 mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp         |   1 +
 .../TransformOps/ArmSVEVectorTransformOps.cpp |   1 +
 mlir/lib/Dialect/Async/IR/Async.cpp           |   1 +
 .../Bufferization/IR/BufferizationOps.cpp     |  20 ++--
 mlir/lib/Dialect/Complex/IR/ComplexOps.cpp    |   5 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.cpp |  16 +--
 .../DLTI/TransformOps/DLTITransformOps.cpp    |   1 +
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |   7 +-
 .../Func/TransformOps/FuncTransformOps.cpp    |   3 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |   9 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |   5 +-
 .../GPU/Transforms/MemoryPromotion.cpp        |   1 +
 mlir/lib/Dialect/IRDL/IR/IRDL.cpp             |   1 +
 mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp          |   1 +
 mlir/lib/Dialect/Index/IR/IndexOps.cpp        |  16 ++-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |   7 +-
 mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp    |   1 +
 mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp   |   1 +
 mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp    |   1 +
 mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp    |   1 +
 .../lib/Dialect/MLProgram/IR/MLProgramOps.cpp |   1 +
 mlir/lib/Dialect/MPI/IR/MPIOps.cpp            |   5 +-
 mlir/lib/Dialect/Math/IR/MathOps.cpp          |   1 +
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |  27 +++--
 .../TransformOps/MemRefTransformOps.cpp       |   8 +-
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |  15 +--
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       |   1 +
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |   1 +
 mlir/lib/Dialect/PDL/IR/PDL.cpp               |   1 +
 mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp   |   1 +
 mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp        |   1 +
 mlir/lib/Dialect/Quant/IR/QuantOps.cpp        |   1 +
 mlir/lib/Dialect/SCF/IR/SCF.cpp               | 106 ++++++++---------
 .../SCF/TransformOps/SCFTransformOps.cpp      |   1 +
 mlir/lib/Dialect/SMT/IR/SMTOps.cpp            |   1 +
 .../Dialect/SPIRV/IR/SPIRVOpDefinition.cpp    |   1 +
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  64 ++++++-----
 .../SparseTensor/IR/SparseTensorDialect.cpp   |   3 +-
 .../TransformOps/SparseTensorTransformOps.cpp |   1 +
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      | 107 +++++++++---------
 .../TransformOps/TensorTransformOps.cpp       |   1 +
 .../DebugExtension/DebugExtension.cpp         |   1 +
 .../DebugExtension/DebugExtensionOps.cpp      |   1 +
 .../lib/Dialect/Transform/IR/TransformOps.cpp |   1 +
 .../IRDLExtension/IRDLExtensionOps.cpp        |   1 +
 .../LoopExtension/LoopExtensionOps.cpp        |   1 +
 mlir/lib/Dialect/UB/IR/UBOps.cpp              |   1 +
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  56 ++++-----
 .../TransformOps/VectorTransformOps.cpp       |   1 +
 .../Dialect/X86Vector/IR/X86VectorDialect.cpp |   3 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   1 +
 mlir/lib/IR/BuiltinDialect.cpp                |   1 +
 .../TestTransformDialectExtension.cpp         |   1 +
 .../TestTilingInterfaceTransformOps.cpp       |   1 +
 .../test/lib/Transforms/TestTransformsOps.cpp |   1 +
 mlir/test/python/lib/PythonTestDialect.cpp    |   1 +
 80 files changed, 342 insertions(+), 234 deletions(-)

diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index 687007d957225..cef95a13c5a50 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
index 19ad6bed512c7..d7d294ab621d8 100644
--- a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Optimizer/Dialect/FIRCG/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index b6bf2753b80ce..db1981b40a6ec 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -26,6 +26,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index df6ce12215d26..7f2ec418af40a 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/APInt.h"
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
index 1ea69f9059321..3da966ef9cc85 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
@@ -8,6 +8,7 @@
 
 #include "Standalone/StandaloneDialect.h"
 #include "Standalone/StandaloneOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "Standalone/StandaloneTypes.h"
 
 using namespace mlir;
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
index 55b66b51232f2..ef20049f1b76b 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
@@ -8,6 +8,7 @@
 
 #include "Standalone/StandaloneOps.h"
 #include "Standalone/StandaloneDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 #define GET_OP_CLASSES
 #include "Standalone/StandaloneOps.cpp.inc"
diff --git a/mlir/examples/toy/Ch2/mlir/Dialect.cpp b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
index 489f348c8be52..fc4c3c75dcfda 100644
--- a/mlir/examples/toy/Ch2/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -144,7 +145,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -280,7 +282,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch3/mlir/Dialect.cpp b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
index 708855f18cf45..af3cbb385d332 100644
--- a/mlir/examples/toy/Ch3/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -144,7 +145,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -280,7 +282,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch4/mlir/Dialect.cpp b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
index 076a75a26619b..e3ab656c193a3 100644
--- a/mlir/examples/toy/Ch4/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch5/mlir/Dialect.cpp b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
index fb7c742a01802..81c9e8f9a143f 100644
--- a/mlir/examples/toy/Ch5/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch6/mlir/Dialect.cpp b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
index fb7c742a01802..81c9e8f9a143f 100644
--- a/mlir/examples/toy/Ch6/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
@@ -206,7 +207,8 @@ void ConstantOp::print(mlir::OpAsmPrinter &printer) {
 llvm::LogicalResult ConstantOp::verify() {
   // If the return type of the constant is not an unranked tensor, the shape
   // must match the shape of the attribute holding the data.
-  auto resultType = llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
+  auto resultType =
+      llvm::dyn_cast<mlir::RankedTensorType>(getResult().getType());
   if (!resultType)
     return success();
 
@@ -395,7 +397,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/toy/Ch7/mlir/Dialect.cpp b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
index 52881db87d86b..64aa4d8e8995d 100644
--- a/mlir/examples/toy/Ch7/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpImplementation.h"
@@ -429,7 +430,8 @@ llvm::LogicalResult ReturnOp::verify() {
   auto resultType = results.front();
 
   // Check that the result type of the function matches the operand type.
-  if (inputType == resultType || llvm::isa<mlir::UnrankedTensorType>(inputType) ||
+  if (inputType == resultType ||
+      llvm::isa<mlir::UnrankedTensorType>(inputType) ||
       llvm::isa<mlir::UnrankedTensorType>(resultType))
     return mlir::success();
 
diff --git a/mlir/examples/transform/Ch2/lib/MyExtension.cpp b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
index b4b27e97d266e..e24714b408e29 100644
--- a/mlir/examples/transform/Ch2/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/examples/transform/Ch3/lib/MyExtension.cpp b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
index 4b2123fa71d31..5af5cc8ac5052 100644
--- a/mlir/examples/transform/Ch3/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
@@ -17,6 +17,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/examples/transform/Ch4/lib/MyExtension.cpp b/mlir/examples/transform/Ch4/lib/MyExtension.cpp
index fa0ffc9dc2e8a..86884d3bee7a0 100644
--- a/mlir/examples/transform/Ch4/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch4/lib/MyExtension.cpp
@@ -13,6 +13,7 @@
 
 #include "MyExtension.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE_MATCHER "transform-matcher"
diff --git a/mlir/lib/CAPI/Dialect/PDL.cpp b/mlir/lib/CAPI/Dialect/PDL.cpp
index bd8b13c6516e2..d2a83da643a89 100644
--- a/mlir/lib/CAPI/Dialect/PDL.cpp
+++ b/mlir/lib/CAPI/Dialect/PDL.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 69eec9d9bbf3e..cd92e1dbba090 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -22,6 +22,7 @@
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index acaf6a2f8792a..ece11fe1a8fb2 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 748ff1edbfeb2..111d93e39c15e 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index bd1be7c087d05..cf21156085772 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
diff --git a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
index b1e40d9b289ec..267179f26c52e 100644
--- a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
+++ b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index ac00917d33a85..3eb1580e2a1d2 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
index 4bcad5f4ece25..022e8bb45107b 100644
--- a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
+++ b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/Builders.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
index d07e6a52d8b5f..f0b7a8a4b0954 100644
--- a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/ArmNeon/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
index bc505f936d6c5..12018912f6f66 100644
--- a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/Builders.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp b/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
index b2ca4fc1eaa8c..b50679239a840 100644
--- a/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index dc7b07d911c17..7dd1b856fa2b0 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -10,6 +10,7 @@
 
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
+#include "mlir/IR/Builders.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index e64fee235e50b..58bae9fcdc4d3 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include <optional>
 
@@ -321,8 +322,8 @@ struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
         newShape, op.getType().getElementType(), op.getType().getEncoding());
     if (newType == op.getType())
       return failure();
-    auto newOp = AllocTensorOp::create(rewriter,
-        op.getLoc(), newType, newDynamicSizes, /*copy=*/Value());
+    auto newOp = AllocTensorOp::create(rewriter, op.getLoc(), newType,
+                                       newDynamicSizes, /*copy=*/Value());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -514,7 +515,7 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
 
     if (source.getType() != cloneOp.getType())
       source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
-                                               cloneOp.getType(), source);
+                                      cloneOp.getType(), source);
     rewriter.replaceOp(cloneOp, source);
     rewriter.eraseOp(redundantDealloc);
     return success();
@@ -643,8 +644,9 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return ToTensorOp::create(builder,
-      loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
+  return ToTensorOp::create(
+      builder, loc, memref::getTensorTypeFromMemRefType(getDest().getType()),
+      getDest(),
       /*restrict=*/true, getWritable());
 }
 
@@ -807,7 +809,7 @@ struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
     auto memrefType = MemRefType::get(srcTensorType.getShape(),
                                       srcTensorType.getElementType());
     Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
-                                               tensorCastOperand.getOperand());
+                                      tensorCastOperand.getOperand());
     rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                 memref);
     return success();
@@ -1015,7 +1017,7 @@ struct DeallocRemoveDuplicateRetainedMemrefs
     // same as the number of condition operands.
     auto newDeallocOp =
         DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
-                                   deallocOp.getConditions(), newRetained);
+                          deallocOp.getConditions(), newRetained);
     SmallVector<Value> replacements(
         llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
           return newDeallocOp.getUpdatedConditions()[idx];
@@ -1036,8 +1038,8 @@ struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     if (deallocOp.getMemrefs().empty()) {
-      Value constFalse = arith::ConstantOp::create(rewriter,
-          deallocOp.getLoc(), rewriter.getBoolAttr(false));
+      Value constFalse = arith::ConstantOp::create(rewriter, deallocOp.getLoc(),
+                                                   rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
                                         constFalse));
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
index fb97045687d65..34008aa5c4d6a 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 
@@ -20,9 +21,7 @@ using namespace mlir::complex;
 // ConstantOp
 //===----------------------------------------------------------------------===//
 
-OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) {
-  return getValue();
-}
+OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValue(); }
 
 void ConstantOp::getAsmResultNames(
     function_ref<void(Value, StringRef)> setNameFn) {
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 38e47cc5e3714..66593783a14f1 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -89,7 +90,7 @@ LogicalResult AssertOp::canonicalize(AssertOp op, PatternRewriter &rewriter) {
   return failure();
 }
 
-// This side effect models "program termination". 
+// This side effect models "program termination".
 void AssertOp::getEffects(
     SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
         &effects) {
@@ -312,8 +313,9 @@ struct SimplifyCondBranchIdenticalSuccessors
       if (std::get<0>(it) == std::get<1>(it))
         mergedOperands.push_back(std::get<0>(it));
       else
-        mergedOperands.push_back(arith::SelectOp::create(rewriter,
-            condbr.getLoc(), condition, std::get<0>(it), std::get<1>(it)));
+        mergedOperands.push_back(
+            arith::SelectOp::create(rewriter, condbr.getLoc(), condition,
+                                    std::get<0>(it), std::get<1>(it)));
     }
 
     rewriter.replaceOpWithNewOp<BranchOp>(condbr, trueDest, mergedOperands);
@@ -412,8 +414,8 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantTrue)
-            constantTrue = arith::ConstantOp::create(rewriter,
-                condbr.getLoc(), ty, rewriter.getBoolAttr(true));
+            constantTrue = arith::ConstantOp::create(
+                rewriter, condbr.getLoc(), ty, rewriter.getBoolAttr(true));
 
           rewriter.modifyOpInPlace(use.getOwner(),
                                    [&] { use.set(constantTrue); });
@@ -427,8 +429,8 @@ struct CondBranchTruthPropagation : public OpRewritePattern<CondBranchOp> {
           replaced = true;
 
           if (!constantFalse)
-            constantFalse = arith::ConstantOp::create(rewriter,
-                condbr.getLoc(), ty, rewriter.getBoolAttr(false));
+            constantFalse = arith::ConstantOp::create(
+                rewriter, condbr.getLoc(), ty, rewriter.getBoolAttr(false));
 
           rewriter.modifyOpInPlace(use.getOwner(),
                                    [&] { use.set(constantFalse); });
diff --git a/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp b/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
index 02c41b4fe8113..4dbd1659e24dc 100644
--- a/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
+++ b/mlir/lib/Dialect/DLTI/TransformOps/DLTITransformOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/Utils/Utils.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index 9a33fc8e07dde..f40e1b625694a 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -51,7 +52,7 @@ Operation *FuncDialect::materializeConstant(OpBuilder &builder, Attribute value,
                                             Type type, Location loc) {
   if (ConstantOp::isBuildableWith(value, type))
     return ConstantOp::create(builder, loc, type,
-                                      llvm::cast<FlatSymbolRefAttr>(value));
+                              llvm::cast<FlatSymbolRefAttr>(value));
   return nullptr;
 }
 
@@ -139,9 +140,7 @@ LogicalResult ConstantOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   return success();
 }
 
-OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) {
-  return getValueAttr();
-}
+OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValueAttr(); }
 
 void ConstantOp::getAsmResultNames(
     function_ref<void(Value, StringRef)> setNameFn) {
diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
index 10515251fef7d..d869309cdd63b 100644
--- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
+++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
@@ -171,7 +172,7 @@ transform::CastAndCallOp::apply(transform::TransformRewriter &rewriter,
   }
 
   auto callOp = func::CallOp::create(rewriter, insertionPoint->getLoc(),
-                                              targetFunction, inputs);
+                                     targetFunction, inputs);
 
   // Cast the call results back to the expected types. If any conversions fail
   // this is a definite failure as the call has been constructed at this point.
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index f7eb41be287e4..8d6962d2d2be8 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -23,6 +23,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -1382,9 +1383,9 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
                       int32_t offset, int32_t width, ShuffleMode mode) {
   build(builder, result, value,
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(offset)),
+                                  builder.getI32IntegerAttr(offset)),
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(width)),
+                                  builder.getI32IntegerAttr(width)),
         mode);
 }
 
@@ -1396,9 +1397,9 @@ void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
                      int32_t offset, int32_t width) {
   build(builder, result, value,
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(offset)),
+                                  builder.getI32IntegerAttr(offset)),
         arith::ConstantOp::create(builder, result.location,
-                                          builder.getI32IntegerAttr(width)));
+                                  builder.getI32IntegerAttr(width)));
 }
 
 LogicalResult RotateOp::verify() {
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index 2035f371694a7..26adfade879d8 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -31,6 +31,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/Visitors.h"
@@ -561,7 +562,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   if (originalBasisWasProvided) {
     for (Value tmpPredicate : builderResult.predicateOps) {
       predicate = predicate ? arith::AndIOp::create(rewriter, loc, predicate,
-                                                             tmpPredicate)
+                                                    tmpPredicate)
                             : tmpPredicate;
     }
   }
@@ -574,7 +575,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   if (predicate) {
     // Step 6.a. If predicated, move at the beginning.
     auto ifOp = scf::IfOp::create(rewriter, loc, predicate,
-                                           /*withElseRegion=*/false);
+                                  /*withElseRegion=*/false);
     targetBlock = ifOp.thenBlock();
     insertionPoint = ifOp.thenBlock()->begin();
   } else {
diff --git a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
index 4af08dad08462..4a670373272a7 100644
--- a/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
index 197a5907fc4cb..25df6f876e101 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
index 7ec3aa2741023..c62e5c9c50183 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/IRDL/IR/IRDL.h"
 #include "mlir/Dialect/IRDL/IRDLSymbols.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/ValueRange.h"
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Index/IR/IndexOps.cpp b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
index 7cab93709b9c6..b7d00c553d70a 100644
--- a/mlir/lib/Dialect/Index/IR/IndexOps.cpp
+++ b/mlir/lib/Dialect/Index/IR/IndexOps.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Index/IR/IndexAttrs.h"
 #include "mlir/Dialect/Index/IR/IndexDialect.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/Utils/InferIntRangeCommon.h"
@@ -129,15 +130,18 @@ canonicalizeAssociativeCommutativeBinaryOp(BinaryOp op,
 
   auto lhsOp = op.getLhs().template getDefiningOp<BinaryOp>();
   if (!lhsOp)
-    return rewriter.notifyMatchFailure(op.getLoc(), "LHS is not the same BinaryOp");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "LHS is not the same BinaryOp");
 
   if (!mlir::matchPattern(lhsOp.getRhs(), mlir::m_Constant()))
-    return rewriter.notifyMatchFailure(op.getLoc(), "RHS of LHS op is not a constant");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "RHS of LHS op is not a constant");
 
   Value c = rewriter.createOrFold<BinaryOp>(op->getLoc(), op.getRhs(),
-                                           lhsOp.getRhs());
+                                            lhsOp.getRhs());
   if (c.getDefiningOp<BinaryOp>())
-    return rewriter.notifyMatchFailure(op.getLoc(), "new BinaryOp was not folded");
+    return rewriter.notifyMatchFailure(op.getLoc(),
+                                       "new BinaryOp was not folded");
 
   rewriter.replaceOpWithNewOp<BinaryOp>(op, lhsOp.getLhs(), c);
   return success();
@@ -716,10 +720,10 @@ LogicalResult CmpOp::canonicalize(CmpOp op, PatternRewriter &rewriter) {
   index::CmpOp newCmp;
   if (rhsIsZero)
     newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
-                                           subOp.getLhs(), subOp.getRhs());
+                                  subOp.getLhs(), subOp.getRhs());
   else
     newCmp = index::CmpOp::create(rewriter, op.getLoc(), op.getPred(),
-                                           subOp.getRhs(), subOp.getLhs());
+                                  subOp.getRhs(), subOp.getLhs());
   rewriter.replaceOp(op, newCmp);
   return success();
 }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 884ae4a2eac64..fbe8468548d61 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
@@ -4304,8 +4305,8 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
   MLIRContext *ctx = builder.getContext();
   auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
-  auto global = LLVM::GlobalOp::create(moduleBuilder,
-      loc, type, /*isConstant=*/true, linkage, name,
+  auto global = LLVM::GlobalOp::create(
+      moduleBuilder, loc, type, /*isConstant=*/true, linkage, name,
       builder.getStringAttr(value), /*alignment=*/0);
 
   LLVMPointerType ptrType = LLVMPointerType::get(ctx);
@@ -4313,7 +4314,7 @@ Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
   Value globalPtr =
       LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
   return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
-                                     ArrayRef<GEPArg>{0, 0});
+                             ArrayRef<GEPArg>{0, 0});
 }
 
 bool mlir::LLVM::satisfiesLLVMModule(Operation *op) {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 6e29b129e8835..d5713e045fa71 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -24,6 +24,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
index 1a9ccf56008b8..a39457068f319 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
index bd9d3528ceb74..b1676b4e58c39 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/VCIXDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
index 9e497829ba723..fbedc714c200b 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp b/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
index 37ddca101f64b..877f8e72caa31 100644
--- a/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
+++ b/mlir/lib/Dialect/MLProgram/IR/MLProgramOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/MLProgram/IR/MLProgram.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index 749cca0575581..da2317cd4eb97 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 
 using namespace mlir;
@@ -60,8 +61,8 @@ struct FoldRank final : public mlir::OpRewritePattern<mlir::mpi::CommRankOp> {
     if (!isa<IntegerAttr>(dltiAttr.value()))
       return op->emitError()
              << "Expected an integer attribute for MPI:comm_world_rank";
-    Value res = arith::ConstantIndexOp::create(b,
-        op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
+    Value res = arith::ConstantIndexOp::create(
+        b, op.getLoc(), cast<IntegerAttr>(dltiAttr.value()).getInt());
     if (Value retVal = op.getRetval())
       b.replaceOp(op, {retVal, res});
     else
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index a21631cbf8510..b09a67818fff6 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 706ac4b4867f4..cb5fe132494b3 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -213,9 +214,9 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
 
     // Create and insert the alloc op for the new memref.
-    auto newAlloc = AllocLikeOp::create(rewriter,
-        alloc.getLoc(), newMemRefType, dynamicSizes, alloc.getSymbolOperands(),
-        alloc.getAlignmentAttr());
+    auto newAlloc = AllocLikeOp::create(rewriter, alloc.getLoc(), newMemRefType,
+                                        dynamicSizes, alloc.getSymbolOperands(),
+                                        alloc.getAlignmentAttr());
     // Insert a cast so we have the same type as the old alloc.
     rewriter.replaceOpWithNewOp<CastOp>(alloc, alloc.getType(), newAlloc);
     return success();
@@ -1358,8 +1359,9 @@ static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
     assert(isa<Attribute>(maybeConstant) &&
            "The constified value should be either unchanged (i.e., == result) "
            "or a constant");
-    Value constantVal = arith::ConstantIndexOp::create(rewriter,
-        loc, llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
+    Value constantVal = arith::ConstantIndexOp::create(
+        rewriter, loc,
+        llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
     for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
       // modifyOpInPlace: lambda cannot capture structured bindings in C++17
       // yet.
@@ -2587,8 +2589,9 @@ struct CollapseShapeOpMemRefCastFolder
       rewriter.modifyOpInPlace(
           op, [&]() { op.getSrcMutable().assign(cast.getSource()); });
     } else {
-      Value newOp = CollapseShapeOp::create(rewriter,
-          op->getLoc(), cast.getSource(), op.getReassociationIndices());
+      Value newOp =
+          CollapseShapeOp::create(rewriter, op->getLoc(), cast.getSource(),
+                                  op.getReassociationIndices());
       rewriter.replaceOpWithNewOp<CastOp>(op, op.getType(), newOp);
     }
     return success();
@@ -3212,8 +3215,8 @@ class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
     if (!resultType)
       return failure();
 
-    Value newSubView = SubViewOp::create(rewriter,
-        subViewOp.getLoc(), resultType, castOp.getSource(),
+    Value newSubView = SubViewOp::create(
+        rewriter, subViewOp.getLoc(), resultType, castOp.getSource(),
         subViewOp.getOffsets(), subViewOp.getSizes(), subViewOp.getStrides(),
         subViewOp.getStaticOffsets(), subViewOp.getStaticSizes(),
         subViewOp.getStaticStrides());
@@ -3534,9 +3537,9 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
       return failure();
 
     // Create new ViewOp.
-    auto newViewOp = ViewOp::create(rewriter,
-        viewOp.getLoc(), newMemRefType, viewOp.getOperand(0),
-        viewOp.getByteShift(), newOperands);
+    auto newViewOp = ViewOp::create(rewriter, viewOp.getLoc(), newMemRefType,
+                                    viewOp.getOperand(0), viewOp.getByteShift(),
+                                    newOperands);
     // Insert a cast so we have the same type as the old memref type.
     rewriter.replaceOpWithNewOp<CastOp>(viewOp, viewOp.getType(), newViewOp);
     return success();
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 5c556308b1e9f..c583c64f23bc0 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -23,6 +23,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "llvm/Support/Debug.h"
 
@@ -155,9 +156,10 @@ transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
       Type resultType = alloca.getResult().getType();
       OpBuilder builder(rewriter.getContext());
       // TODO: Add a better builder for this.
-      globalOp = memref::GlobalOp::create(builder,
-          loc, StringAttr::get(ctx, "alloca"), StringAttr::get(ctx, "private"),
-          TypeAttr::get(resultType), Attribute{}, UnitAttr{}, IntegerAttr{});
+      globalOp = memref::GlobalOp::create(
+          builder, loc, StringAttr::get(ctx, "alloca"),
+          StringAttr::get(ctx, "private"), TypeAttr::get(resultType),
+          Attribute{}, UnitAttr{}, IntegerAttr{});
       symbolTable.insert(globalOp);
     }
 
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index a440749dbd703..f468e9f6f4b4d 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -317,9 +318,9 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
   if (!newShardOp) {
     auto shardingOp =
         ShardingOp::create(builder, operandValue.getLoc(), sharding);
-    newShardOp =
-        ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                                /*annotate_for_users*/ false);
+    newShardOp = ShardOp::create(builder, operandValue.getLoc(), operandValue,
+                                 shardingOp,
+                                 /*annotate_for_users*/ false);
   }
   operandValue.replaceUsesWithIf(
       newShardOp, [operandOp, operandValue](OpOperand &use) {
@@ -331,8 +332,8 @@ static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
   }
 
   auto newShardOp2 = ShardOp::create(builder, operandValue.getLoc(), newShardOp,
-                                             newShardOp.getSharding(),
-                                             /*annotate_for_users*/ true);
+                                     newShardOp.getSharding(),
+                                     /*annotate_for_users*/ true);
   newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
 }
 
@@ -381,7 +382,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
       ShardingOp::create(builder, operand.get().getLoc(), sharding);
   auto newShardOp =
       ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                              /*annotate_for_users*/ true);
+                      /*annotate_for_users*/ true);
   IRRewriter rewriter(builder);
   rewriter.replaceUsesWithIf(
       operandValue, newShardOp, [operandOp, operandValue](OpOperand &use) {
@@ -396,7 +397,7 @@ void mlir::mesh::maybeInsertSourceShardingAnnotation(MeshSharding sharding,
   builder.setInsertionPoint(newShardOp);
   auto newPreceedingShardOp =
       ShardOp::create(builder, operandValue.getLoc(), operandValue, shardingOp,
-                              /*annotate_for_users*/ false);
+                      /*annotate_for_users*/ false);
   rewriter.replaceUsesWithIf(
       newShardOp.getSrc(), newPreceedingShardOp, [&newShardOp](OpOperand &use) {
         return use.getOwner() == newShardOp.getOperation();
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..69a318f4af79b 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 9bea8b7a732a8..056b73f52cde8 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/SymbolTable.h"
diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp
index d3f7c9798b9b8..ffe53bbe80607 100644
--- a/mlir/lib/Dialect/PDL/IR/PDL.cpp
+++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
index 9b1f11d835282..5cc84ab29e246 100644
--- a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
+++ b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
index c488144508128..3695262155387 100644
--- a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
+++ b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/Dialect/Ptr/IR/PtrOps.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
diff --git a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
index e23a0d6aba825..cd4198b431669 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Quant/IR/Quant.h"
 #include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 82fa50a6ac6e0..8ead566703ab4 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
@@ -247,7 +248,7 @@ struct MultiBlockExecuteInliner : public OpRewritePattern<ExecuteRegionOp> {
       if (YieldOp yieldOp = dyn_cast<YieldOp>(blk.getTerminator())) {
         rewriter.setInsertionPoint(yieldOp);
         cf::BranchOp::create(rewriter, yieldOp.getLoc(), postBlock,
-                                      yieldOp.getResults());
+                             yieldOp.getResults());
         rewriter.eraseOp(yieldOp);
       }
     }
@@ -557,8 +558,8 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   rewriter.setInsertionPoint(getOperation());
   auto inits = llvm::to_vector(getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  scf::ForOp newLoop = scf::ForOp::create(rewriter,
-      getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
+  scf::ForOp newLoop = scf::ForOp::create(
+      rewriter, getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
       [](OpBuilder &, Location, Value, ValueRange) {});
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
 
@@ -673,8 +674,8 @@ void mlir::scf::promote(RewriterBase &rewriter, scf::ForallOp forallOp) {
     Value dst = parallelInsertSliceOp.getDest();
     Value src = parallelInsertSliceOp.getSource();
     if (llvm::isa<TensorType>(src.getType())) {
-      results.push_back(tensor::InsertSliceOp::create(rewriter,
-          forallOp.getLoc(), dst.getType(), src, dst,
+      results.push_back(tensor::InsertSliceOp::create(
+          rewriter, forallOp.getLoc(), dst.getType(), src, dst,
           parallelInsertSliceOp.getOffsets(), parallelInsertSliceOp.getSizes(),
           parallelInsertSliceOp.getStrides(),
           parallelInsertSliceOp.getStaticOffsets(),
@@ -722,8 +723,8 @@ LoopNest mlir::scf::buildLoopNest(
   ValueRange currentIterArgs = iterArgs;
   Location currentLoc = loc;
   for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
-    auto loop = scf::ForOp::create(builder,
-        currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
+    auto loop = scf::ForOp::create(
+        builder, currentLoc, lbs[i], ubs[i], steps[i], currentIterArgs,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv,
             ValueRange args) {
           ivs.push_back(iv);
@@ -801,8 +802,8 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   }
 
   // 2. Create the new forOp shell.
-  scf::ForOp newForOp = scf::ForOp::create(rewriter,
-      forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
+  scf::ForOp newForOp = scf::ForOp::create(
+      rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
       forOp.getStep(), newIterOperands);
   newForOp->setAttrs(forOp->getAttrs());
   Block &newBlock = newForOp.getRegion().front();
@@ -926,9 +927,9 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     if (!canonicalize)
       return failure();
 
-    scf::ForOp newForOp = scf::ForOp::create(rewriter,
-        forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-        forOp.getStep(), newIterArgs);
+    scf::ForOp newForOp =
+        scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(),
+                           forOp.getUpperBound(), forOp.getStep(), newIterArgs);
     newForOp->setAttrs(forOp->getAttrs());
     Block &newBlock = newForOp.getRegion().front();
 
@@ -971,7 +972,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
         if (keepMask[idx])
           filteredOperands.push_back(mergedTerminator.getOperand(idx));
       scf::YieldOp::create(rewriter, mergedTerminator.getLoc(),
-                                    filteredOperands);
+                           filteredOperands);
     };
 
     rewriter.mergeBlocks(&oldBlock, &newBlock, newBlockTransferArgs);
@@ -1685,8 +1686,8 @@ struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
 
     // Step 3. Create a new scf.forall op with the new shared_outs' operands
     //         fetched earlier
-    auto newForallOp = scf::ForallOp::create(rewriter,
-        forallOp.getLoc(), forallOp.getMixedLowerBound(),
+    auto newForallOp = scf::ForallOp::create(
+        rewriter, forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
         forallOp.getMapping(),
         /*bodyBuilderFn =*/[](OpBuilder &, Location, ValueRange) {});
@@ -1783,8 +1784,8 @@ struct ForallOpSingleOrZeroIterationDimsFolder
     // Replace the loop by a lower-dimensional loop.
     ForallOp newOp;
     newOp = ForallOp::create(rewriter, loc, newMixedLowerBounds,
-                                      newMixedUpperBounds, newMixedSteps,
-                                      op.getOutputs(), std::nullopt, nullptr);
+                             newMixedUpperBounds, newMixedSteps,
+                             op.getOutputs(), std::nullopt, nullptr);
     newOp.getBodyRegion().getBlocks().clear();
     // The new loop needs to keep all attributes from the old one, except for
     // "operandSegmentSizes" and static loop bound attributes which capture
@@ -1867,16 +1868,17 @@ struct FoldTensorCastOfOutputIntoForallOp
 
     // Create new loop.
     Location loc = forallOp.getLoc();
-    auto newForallOp = ForallOp::create(rewriter,
-        loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
-        forallOp.getMixedStep(), newOutputTensors, forallOp.getMapping(),
+    auto newForallOp = ForallOp::create(
+        rewriter, loc, forallOp.getMixedLowerBound(),
+        forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
+        newOutputTensors, forallOp.getMapping(),
         [&](OpBuilder nestedBuilder, Location nestedLoc, ValueRange bbArgs) {
           auto castBlockArgs =
               llvm::to_vector(bbArgs.take_back(forallOp->getNumResults()));
           for (auto [index, cast] : tensorCastProducers) {
             Value &oldTypeBBArg = castBlockArgs[index];
-            oldTypeBBArg = tensor::CastOp::create(nestedBuilder,
-                nestedLoc, cast.dstType, oldTypeBBArg);
+            oldTypeBBArg = tensor::CastOp::create(nestedBuilder, nestedLoc,
+                                                  cast.dstType, oldTypeBBArg);
           }
 
           // Move old body into new parallel loop.
@@ -1903,7 +1905,7 @@ struct FoldTensorCastOfOutputIntoForallOp
     for (auto &item : tensorCastProducers) {
       Value &oldTypeResult = castResults[item.first];
       oldTypeResult = tensor::CastOp::create(rewriter, loc, item.second.dstType,
-                                                      oldTypeResult);
+                                             oldTypeResult);
     }
     rewriter.replaceOp(forallOp, castResults);
     return success();
@@ -2373,7 +2375,7 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       return failure();
 
     IfOp replacement = IfOp::create(rewriter, op.getLoc(), nonHoistable, cond,
-                                             /*withElseRegion=*/false);
+                                    /*withElseRegion=*/false);
     if (replacement.thenBlock())
       rewriter.eraseBlock(replacement.thenBlock());
     replacement.getThenRegion().takeBody(op.getThenRegion());
@@ -2398,8 +2400,8 @@ struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
       } else if (trueVal == falseVal)
         results[it.index()] = trueVal;
       else
-        results[it.index()] = arith::SelectOp::create(rewriter,
-            op.getLoc(), cond, trueVal, falseVal);
+        results[it.index()] = arith::SelectOp::create(rewriter, op.getLoc(),
+                                                      cond, trueVal, falseVal);
     }
 
     rewriter.setInsertionPointToEnd(replacement.thenBlock());
@@ -2450,8 +2452,8 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantTrue)
-          constantTrue = arith::ConstantOp::create(rewriter,
-              op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
+          constantTrue = arith::ConstantOp::create(
+              rewriter, op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 1));
 
         rewriter.modifyOpInPlace(use.getOwner(),
                                  [&]() { use.set(constantTrue); });
@@ -2460,8 +2462,8 @@ struct ConditionPropagation : public OpRewritePattern<IfOp> {
         changed = true;
 
         if (!constantFalse)
-          constantFalse = arith::ConstantOp::create(rewriter,
-              op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
+          constantFalse = arith::ConstantOp::create(
+              rewriter, op.getLoc(), i1Ty, rewriter.getIntegerAttr(i1Ty, 0));
 
         rewriter.modifyOpInPlace(use.getOwner(),
                                  [&]() { use.set(constantFalse); });
@@ -2547,8 +2549,8 @@ struct ReplaceIfYieldWithConditionOrValue : public OpRewritePattern<IfOp> {
       if (!trueVal && falseVal) {
         if (!opResult.use_empty()) {
           Dialect *constDialect = trueResult.getDefiningOp()->getDialect();
-          Value notCond = arith::XOrIOp::create(rewriter,
-              op.getLoc(), op.getCondition(),
+          Value notCond = arith::XOrIOp::create(
+              rewriter, op.getLoc(), op.getCondition(),
               constDialect
                   ->materializeConstant(rewriter,
                                         rewriter.getIntegerAttr(i1Ty, 1), i1Ty,
@@ -2661,8 +2663,8 @@ struct CombineIfs : public OpRewritePattern<IfOp> {
     SmallVector<Type> mergedTypes(prevIf.getResultTypes());
     llvm::append_range(mergedTypes, nextIf.getResultTypes());
 
-    IfOp combinedIf = IfOp::create(rewriter,
-        nextIf.getLoc(), mergedTypes, prevIf.getCondition(), /*hasElse=*/false);
+    IfOp combinedIf = IfOp::create(rewriter, nextIf.getLoc(), mergedTypes,
+                                   prevIf.getCondition(), /*hasElse=*/false);
     rewriter.eraseBlock(&combinedIf.getThenRegion().back());
 
     rewriter.inlineRegionBefore(prevIf.getThenRegion(),
@@ -2823,8 +2825,8 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     }
 
     Location loc = op.getLoc();
-    Value newCondition = arith::AndIOp::create(rewriter,
-        loc, op.getCondition(), nestedIf.getCondition());
+    Value newCondition = arith::AndIOp::create(rewriter, loc, op.getCondition(),
+                                               nestedIf.getCondition());
     auto newIf = IfOp::create(rewriter, loc, op.getResultTypes(), newCondition);
     Block *newIfBlock = rewriter.createBlock(&newIf.getThenRegion());
 
@@ -2833,8 +2835,9 @@ struct CombineNestedIfs : public OpRewritePattern<IfOp> {
     rewriter.setInsertionPoint(newIf);
 
     for (auto idx : elseYieldsToUpgradeToSelect)
-      results[idx] = arith::SelectOp::create(rewriter,
-          op.getLoc(), op.getCondition(), thenYield[idx], elseYield[idx]);
+      results[idx] =
+          arith::SelectOp::create(rewriter, op.getLoc(), op.getCondition(),
+                                  thenYield[idx], elseYield[idx]);
 
     rewriter.mergeBlocks(nestedIf.thenBlock(), newIfBlock);
     rewriter.setInsertionPointToEnd(newIf.thenBlock());
@@ -3160,8 +3163,8 @@ struct ParallelOpSingleOrZeroIterationDimsFolder
     }
     // Replace the parallel loop by lower-dimensional parallel loop.
     auto newOp =
-        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds, newUpperBounds,
-                                    newSteps, op.getInitVals(), nullptr);
+        ParallelOp::create(rewriter, op.getLoc(), newLowerBounds,
+                           newUpperBounds, newSteps, op.getInitVals(), nullptr);
     // Erase the empty block that was inserted by the builder.
     rewriter.eraseBlock(newOp.getBody());
     // Clone the loop body and remap the block arguments of the collapsed loops
@@ -3541,8 +3544,8 @@ struct WhileConditionTruth : public OpRewritePattern<WhileOp> {
       if (std::get<0>(yieldedAndBlockArgs) == term.getCondition()) {
         if (!std::get<1>(yieldedAndBlockArgs).use_empty()) {
           if (!constantTrue)
-            constantTrue = arith::ConstantOp::create(rewriter,
-                op.getLoc(), term.getCondition().getType(),
+            constantTrue = arith::ConstantOp::create(
+                rewriter, op.getLoc(), term.getCondition().getType(),
                 rewriter.getBoolAttr(true));
 
           rewriter.replaceAllUsesWith(std::get<1>(yieldedAndBlockArgs),
@@ -3684,8 +3687,8 @@ struct RemoveLoopInvariantArgsFromBeforeBlock
       rewriter.replaceOpWithNewOp<YieldOp>(yieldOp, newYieldOpArgs);
     }
 
-    auto newWhile =
-        WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(), newInitArgs);
+    auto newWhile = WhileOp::create(rewriter, op.getLoc(), op.getResultTypes(),
+                                    newInitArgs);
 
     Block &newBeforeBlock = *rewriter.createBlock(
         &newWhile.getBefore(), /*insertPt*/ {},
@@ -3808,7 +3811,7 @@ struct RemoveLoopInvariantValueYielded : public OpRewritePattern<WhileOp> {
     }
 
     auto newWhile = WhileOp::create(rewriter, op.getLoc(), newAfterBlockType,
-                                             op.getOperands());
+                                    op.getOperands());
 
     Block &newAfterBlock =
         *rewriter.createBlock(&newWhile.getAfter(), /*insertPt*/ {},
@@ -4044,7 +4047,7 @@ struct WhileRemoveUnusedArgs : public OpRewritePattern<WhileOp> {
     Location loc = op.getLoc();
     auto newWhileOp =
         WhileOp::create(rewriter, loc, op.getResultTypes(), newInits,
-                                 /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
+                        /*beforeBody*/ nullptr, /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
 
@@ -4091,9 +4094,10 @@ struct WhileRemoveDuplicatedResults : public OpRewritePattern<WhileOp> {
     ValueRange argsRange(newArgs);
 
     Location loc = op.getLoc();
-    auto newWhileOp = scf::WhileOp::create(rewriter,
-        loc, argsRange.getTypes(), op.getInits(), /*beforeBody*/ nullptr,
-        /*afterBody*/ nullptr);
+    auto newWhileOp =
+        scf::WhileOp::create(rewriter, loc, argsRange.getTypes(), op.getInits(),
+                             /*beforeBody*/ nullptr,
+                             /*afterBody*/ nullptr);
     Block &newBeforeBlock = *newWhileOp.getBeforeBody();
     Block &newAfterBlock = *newWhileOp.getAfterBody();
 
@@ -4187,8 +4191,8 @@ struct WhileOpAlignBeforeArgs : public OpRewritePattern<WhileOp> {
     for (auto &&[i, j] : llvm::enumerate(*mapping))
       newResultTypes[j] = loop.getResult(i).getType();
 
-    auto newLoop = WhileOp::create(rewriter,
-        loop.getLoc(), newResultTypes, loop.getInits(),
+    auto newLoop = WhileOp::create(
+        rewriter, loop.getLoc(), newResultTypes, loop.getInits(),
         /*beforeBuilder=*/nullptr, /*afterBuilder=*/nullptr);
     auto newBefore = newLoop.getBeforeBody();
     auto newAfter = newLoop.getAfterBody();
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index e9943a3489d46..145afac6df15e 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -25,6 +25,7 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
index c517ef2b18f6f..6af87afd8b83e 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTOps.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/SMT/IR/SMTOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/APSInt.h"
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
index d8dfe164458e2..8ba2af581e00d 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOpDefinition.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 #include "SPIRVParsingUtils.h"
 
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 7339ba5c2379c..19dba20f43fa9 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -154,14 +155,14 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
     return ub::PoisonOp::create(builder, loc, type, poison);
 
   if (llvm::isa<ShapeType>(type) || isExtentTensorType(type))
-    return ConstShapeOp::create(builder,
-        loc, type, llvm::cast<DenseIntElementsAttr>(value));
+    return ConstShapeOp::create(builder, loc, type,
+                                llvm::cast<DenseIntElementsAttr>(value));
   if (llvm::isa<SizeType>(type))
     return ConstSizeOp::create(builder, loc, type,
-                                       llvm::cast<IntegerAttr>(value));
+                               llvm::cast<IntegerAttr>(value));
   if (llvm::isa<WitnessType>(type))
     return ConstWitnessOp::create(builder, loc, type,
-                                          llvm::cast<BoolAttr>(value));
+                                  llvm::cast<BoolAttr>(value));
 
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
@@ -316,8 +317,8 @@ struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
     auto newYieldOp =
         rewriter.replaceOpWithNewOp<AssumingYieldOp>(yieldOp, newYieldOperands);
     rewriter.setInsertionPoint(op);
-    auto newOp = AssumingOp::create(rewriter,
-        op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
+    auto newOp = AssumingOp::create(
+        rewriter, op.getLoc(), newYieldOp->getOperandTypes(), op.getWitness());
     newOp.getDoRegion().takeBody(op.getDoRegion());
 
     // Use the new results to replace the previously used ones.
@@ -780,9 +781,9 @@ struct BroadcastFoldConstantOperandsPattern
     auto foldedConstantOperandsTy = RankedTensorType::get(
         {static_cast<int64_t>(foldedConstantShape.size())},
         rewriter.getIndexType());
-    newShapeOperands.push_back(ConstShapeOp::create(rewriter,
-        op.getLoc(), foldedConstantOperandsTy,
-        rewriter.getIndexTensorAttr(foldedConstantShape)));
+    newShapeOperands.push_back(
+        ConstShapeOp::create(rewriter, op.getLoc(), foldedConstantOperandsTy,
+                             rewriter.getIndexTensorAttr(foldedConstantShape)));
     rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(),
                                              newShapeOperands);
     return success();
@@ -845,9 +846,9 @@ struct BroadcastConcretizeResultTypePattern
       }
     }
 
-    auto newOp = BroadcastOp::create(rewriter,
-        op.getLoc(), getExtentTensorType(getContext(), maxRank),
-        op.getShapes());
+    auto newOp = BroadcastOp::create(rewriter, op.getLoc(),
+                                     getExtentTensorType(getContext(), maxRank),
+                                     op.getShapes());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -1338,7 +1339,8 @@ std::optional<int64_t> GetExtentOp::getConstantDim() {
 }
 
 OpFoldResult GetExtentOp::fold(FoldAdaptor adaptor) {
-  auto elements = llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
+  auto elements =
+      llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
   if (!elements)
     return nullptr;
   std::optional<int64_t> dim = getConstantDim();
@@ -1357,8 +1359,8 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
     Value dim = ConstSizeOp::create(builder, loc, dimAttr);
     build(builder, result, builder.getType<SizeType>(), shape, dim);
   } else {
-    Value dim =
-        arith::ConstantOp::create(builder, loc, builder.getIndexType(), dimAttr);
+    Value dim = arith::ConstantOp::create(builder, loc, builder.getIndexType(),
+                                          dimAttr);
     build(builder, result, builder.getIndexType(), shape, dim);
   }
 }
@@ -1480,7 +1482,8 @@ bool mlir::shape::MeetOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
 //===----------------------------------------------------------------------===//
 
 OpFoldResult shape::RankOp::fold(FoldAdaptor adaptor) {
-  auto shape = llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
+  auto shape =
+      llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getShape());
   if (!shape)
     return {};
   int64_t rank = shape.getNumElements();
@@ -1708,8 +1711,8 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
                                   rewriter.getIndexTensorAttr(type.getShape()))
             .getResult();
     if (constShape.getType() != op.getResult().getType())
-      constShape = tensor::CastOp::create(rewriter,
-          loc, op.getResult().getType(), constShape);
+      constShape = tensor::CastOp::create(rewriter, loc,
+                                          op.getResult().getType(), constShape);
     rewriter.replaceOp(op, constShape);
     return success();
   }
@@ -1717,13 +1720,13 @@ struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
 
 // Canonicalize
 //
-// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
-// %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
+// tensor<*xf32> %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
 //
 // to
 //
-// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) -> tensor<*xf32>
-// %1 = %shape
+// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
+// tensor<*xf32> %1 = %shape
 //
 struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
   using OpRewritePattern<shape::ShapeOfOp>::OpRewritePattern;
@@ -1751,10 +1754,11 @@ struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
 
     if (opTensorTy != shapeTensorTy) {
       if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
-        shape = tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
-      else if (!isExtentTensorType(shapeTensorTy))
         shape =
-            arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
+            tensor::CastOp::create(rewriter, op.getLoc(), opTensorTy, shape);
+      else if (!isExtentTensorType(shapeTensorTy))
+        shape = arith::IndexCastOp::create(rewriter, op.getLoc(), opTensorTy,
+                                           shape);
     }
 
     rewriter.replaceOp(op, shape);
@@ -1895,8 +1899,9 @@ LogicalResult SplitAtOp::fold(FoldAdaptor adaptor,
                               SmallVectorImpl<OpFoldResult> &results) {
   if (!adaptor.getOperand() || !adaptor.getIndex())
     return failure();
-  auto shapeVec = llvm::to_vector<6>(
-      llvm::cast<DenseIntElementsAttr>(adaptor.getOperand()).getValues<int64_t>());
+  auto shapeVec =
+      llvm::to_vector<6>(llvm::cast<DenseIntElementsAttr>(adaptor.getOperand())
+                             .getValues<int64_t>());
   auto shape = llvm::ArrayRef(shapeVec);
   auto splitPoint = llvm::cast<IntegerAttr>(adaptor.getIndex()).getInt();
   // Verify that the split point is in the correct range.
@@ -1920,8 +1925,9 @@ OpFoldResult ToExtentTensorOp::fold(FoldAdaptor adaptor) {
   if (!adaptor.getInput())
     return OpFoldResult();
   Builder builder(getContext());
-  auto shape = llvm::to_vector<6>(
-      llvm::cast<DenseIntElementsAttr>(adaptor.getInput()).getValues<int64_t>());
+  auto shape =
+      llvm::to_vector<6>(llvm::cast<DenseIntElementsAttr>(adaptor.getInput())
+                             .getValues<int64_t>());
   auto type = RankedTensorType::get({static_cast<int64_t>(shape.size())},
                                     builder.getIndexType());
   return DenseIntElementsAttr::get(type, shape);
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index e2a6a59636a84..1a9d9e158ee75 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -559,7 +559,8 @@ SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
   SmallVector<Type> retType(
       dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
       builder.getIndexType());
-  auto transOp = CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
+  auto transOp =
+      CrdTranslateOp::create(builder, loc, retType, crds, dir, *this);
   return transOp.getOutCrds();
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp b/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
index bdec43825ddc2..33249368f7ed3 100644
--- a/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
+++ b/mlir/lib/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h"
 #include "mlir/Dialect/Linalg/TransformOps/Syntax.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 using namespace mlir::sparse_tensor;
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index df34d9e37bd3f..1884d51de5411 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/BuiltinTypeInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -54,7 +55,7 @@ Operation *TensorDialect::materializeConstant(OpBuilder &builder,
     return op;
   if (complex::ConstantOp::isBuildableWith(value, type))
     return complex::ConstantOp::create(builder, loc, type,
-                                               llvm::cast<ArrayAttr>(value));
+                                       llvm::cast<ArrayAttr>(value));
   return nullptr;
 }
 
@@ -678,8 +679,8 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
     inputShapes.emplace_back(std::move(inputShape));
   }
 
-  Value replacement = tensor::EmptyOp::create(builder,
-      loc, outputShape, getType().getElementType());
+  Value replacement = tensor::EmptyOp::create(builder, loc, outputShape,
+                                              getType().getElementType());
 
   int64_t rank = getType().getRank();
   OpFoldResult one = builder.getIndexAttr(1);
@@ -687,8 +688,8 @@ FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
   SmallVector<OpFoldResult> offsets(rank, zero);
   for (auto [index, input] : llvm::enumerate(getInputs())) {
     offsets[concatDim] = concatOffsets[index];
-    auto insertSlice = tensor::InsertSliceOp::create(builder,
-        loc, input, replacement, offsets, inputShapes[index], strides);
+    auto insertSlice = tensor::InsertSliceOp::create(
+        builder, loc, input, replacement, offsets, inputShapes[index], strides);
     replacement = insertSlice.getResult();
   }
   if (replacement.getType() != getType()) {
@@ -824,7 +825,7 @@ struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
         // Use refined operand type and create cast from original operand.
         auto castOp =
             CastOp::create(rewriter, concatOp->getLoc(), inferredOperandType,
-                                    concatOp.getOperand(operandIdx));
+                           concatOp.getOperand(operandIdx));
         rewriter.modifyOpInPlace(concatOp, [=, operandIdx = operandIdx] {
           concatOp->setOperand(operandIdx, castOp->getResult(0));
         });
@@ -864,8 +865,9 @@ struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
       return failure();
     }
 
-    auto newConcatOp = ConcatOp::create(rewriter,
-        concatOp->getLoc(), inferredResultType, dim, concatOp->getOperands());
+    auto newConcatOp =
+        ConcatOp::create(rewriter, concatOp->getLoc(), inferredResultType, dim,
+                         concatOp->getOperands());
     rewriter.replaceOpWithNewOp<CastOp>(concatOp, concatOp.getResultType(),
                                         newConcatOp);
 
@@ -1151,7 +1153,7 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
       return failure();
 
     auto newOp = EmptyOp::create(rewriter, op.getLoc(), foldedTensorType,
-                                          foldedDynamicSizes);
+                                 foldedDynamicSizes);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(), newOp);
     return success();
   }
@@ -1326,8 +1328,8 @@ struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
 
       SmallVector<int64_t> basis =
           llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-      auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-          extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
+      auto delinearize = affine::AffineDelinearizeIndexOp::create(
+          rewriter, extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
       llvm::append_range(sourceIndices, delinearize.getResults());
     }
     if (collapseOp.getReassociationIndices().empty()) {
@@ -1498,8 +1500,8 @@ struct ExtractElementFromIndexCast
 
     Type elementTy = getElementTypeOrSelf(indexCast.getIn());
 
-    auto newExtract = tensor::ExtractOp::create(rewriter,
-        loc, elementTy, indexCast.getIn(), extract.getIndices());
+    auto newExtract = tensor::ExtractOp::create(
+        rewriter, loc, elementTy, indexCast.getIn(), extract.getIndices());
 
     rewriter.replaceOpWithNewOp<arith::IndexCastOp>(extract, extract.getType(),
                                                     newExtract);
@@ -2161,9 +2163,9 @@ struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
         collapseShapeOp.getSrcMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = CollapseShapeOp::create(rewriter,
-          collapseShapeOp.getLoc(), newResultType, castOp.getSource(),
-          collapseShapeOp.getReassociation());
+      auto newOp = CollapseShapeOp::create(rewriter, collapseShapeOp.getLoc(),
+                                           newResultType, castOp.getSource(),
+                                           collapseShapeOp.getReassociation());
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
           collapseShapeOp, collapseShapeOp.getResultType(), newOp);
     }
@@ -2241,9 +2243,9 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
     auto outputType = RankedTensorType::get(
         newOutputShape, expandOp.getSrcType().getElementType());
     auto inputCast = CastOp::create(rewriter, expandOp.getLoc(), inputType,
-                                             expandOp.getSrc());
-    auto newExpand = ExpandShapeOp::create(rewriter,
-        expandOp.getLoc(), outputType, inputCast.getResult(),
+                                    expandOp.getSrc());
+    auto newExpand = ExpandShapeOp::create(
+        rewriter, expandOp.getLoc(), outputType, inputCast.getResult(),
         expandOp.getReassociationIndices(), outputOfr);
     rewriter.replaceOpWithNewOp<CastOp>(expandOp, expandOp.getType(),
                                         newExpand.getResult());
@@ -2555,10 +2557,11 @@ class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
 
     // Create folded extract.
     Location loc = sliceOp.getLoc();
-    Value newResult = ExtractSliceOp::create(rewriter,
-        loc, sliceOp.getType(), castOp.getSource(), sliceOp.getOffsets(),
-        sliceOp.getSizes(), sliceOp.getStrides(), sliceOp.getStaticOffsets(),
-        sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
+    Value newResult = ExtractSliceOp::create(
+        rewriter, loc, sliceOp.getType(), castOp.getSource(),
+        sliceOp.getOffsets(), sliceOp.getSizes(), sliceOp.getStrides(),
+        sliceOp.getStaticOffsets(), sliceOp.getStaticSizes(),
+        sliceOp.getStaticStrides());
     rewriter.replaceOp(sliceOp, newResult);
     return success();
   }
@@ -2710,7 +2713,7 @@ struct SliceCanonicalizer {
     Value replacement = newOp.getResult();
     if (replacement.getType() != op.getType())
       replacement = tensor::CastOp::create(rewriter, op.getLoc(), op.getType(),
-                                                    replacement);
+                                           replacement);
     rewriter.replaceOp(op, replacement);
   }
 };
@@ -2979,7 +2982,7 @@ class InsertSliceOpConstantArgumentFolder final
       if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
       toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
-                                                 sourceType, toInsert);
+                                        sourceType, toInsert);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, toInsert, insertSliceOp.getDest(), mixedOffsets,
@@ -3075,17 +3078,18 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     if (!sliceResult.isValid)
       return failure();
 
-    Operation *replacement = InsertOpTy::create(rewriter,
-        insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
-        mixedSizes, insertSliceOp.getMixedStrides());
+    Operation *replacement =
+        InsertOpTy::create(rewriter, insertSliceOp.getLoc(), src, dst,
+                           insertSliceOp.getMixedOffsets(), mixedSizes,
+                           insertSliceOp.getMixedStrides());
 
     // In the parallel case there is no result and so nothing to cast.
     bool isParallelInsert =
         std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
     if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
       replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
-                                                    insertSliceOp.getDestType(),
-                                                    replacement->getResult(0));
+                                           insertSliceOp.getDestType(),
+                                           replacement->getResult(0));
     }
     rewriter.replaceOp(insertSliceOp, replacement->getResults());
     return success();
@@ -3154,8 +3158,8 @@ struct InsertSliceOpSourceCastInserter final
     // parallel case.
     if (std::is_same<InsertOpTy, ParallelInsertSliceOp>::value)
       rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-    Value cast = tensor::CastOp::create(rewriter,
-        insertSliceOp.getLoc(), newSrcType, insertSliceOp.getSource());
+    Value cast = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
+                                        newSrcType, insertSliceOp.getSource());
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, cast, insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
@@ -3407,10 +3411,11 @@ struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
         padTensorOp.getSourceMutable().assign(castOp.getSource());
       });
     } else {
-      auto newOp = PadOp::create(rewriter,
-          padTensorOp->getLoc(), newResultType, padTensorOp.getSource(),
-          padTensorOp.getStaticLow(), padTensorOp.getStaticHigh(),
-          padTensorOp.getLow(), padTensorOp.getHigh(), padTensorOp.getNofold(),
+      auto newOp = PadOp::create(
+          rewriter, padTensorOp->getLoc(), newResultType,
+          padTensorOp.getSource(), padTensorOp.getStaticLow(),
+          padTensorOp.getStaticHigh(), padTensorOp.getLow(),
+          padTensorOp.getHigh(), padTensorOp.getNofold(),
           getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
       IRMapping mapper;
       padTensorOp.getRegion().cloneInto(&newOp.getRegion(), mapper);
@@ -3439,8 +3444,8 @@ struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
                                             tensorCastOp.getDest().getType()))
       return failure();
 
-    auto replacementOp = PadOp::create(rewriter,
-        padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
+    auto replacementOp = PadOp::create(
+        rewriter, padTensorOp.getLoc(), tensorCastOp.getDest().getType(),
         padTensorOp.getSource(), padTensorOp.getStaticLow(),
         padTensorOp.getStaticHigh(), padTensorOp.getLow(),
         padTensorOp.getHigh(), padTensorOp.getNofold(),
@@ -3597,11 +3602,11 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
 
     // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
     // the two paddings in one step.
-    auto newSliceOp = ExtractSliceOp::create(rewriter,
-        padOp.getLoc(), outerSliceOp.getSource(), newOffsets, newSizes,
-        innerSliceOp.getMixedStrides());
-    auto newPadOp = PadOp::create(rewriter,
-        padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
+    auto newSliceOp = ExtractSliceOp::create(
+        rewriter, padOp.getLoc(), outerSliceOp.getSource(), newOffsets,
+        newSizes, innerSliceOp.getMixedStrides());
+    auto newPadOp = PadOp::create(
+        rewriter, padOp.getLoc(), padOp.getResultType(), newSliceOp.getResult(),
         padOp.getMixedLowPad(), newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
     rewriter.inlineRegionBefore(padOp.getRegion(), newPadOp.getRegion(),
@@ -3697,9 +3702,9 @@ struct FoldStaticPadding : public OpRewritePattern<PadOp> {
     // Rewrite the op using the new static type.
     auto newResultType = RankedTensorType::get(
         newOutDims, padTensorOp.getType().getElementType());
-    auto newOp = PadOp::create(rewriter,
-        padTensorOp->getLoc(), newResultType, input, staticLow, staticHigh,
-        newLows, newHighs, padTensorOp.getNofold(),
+    auto newOp = PadOp::create(
+        rewriter, padTensorOp->getLoc(), newResultType, input, staticLow,
+        staticHigh, newLows, newHighs, padTensorOp.getNofold(),
         getPrunedAttributeList(padTensorOp, PadOp::getAttributeNames()));
 
     IRMapping mapper;
@@ -3777,9 +3782,9 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
     SmallVector<OpFoldResult> newLowPad =
         addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        padOp.getLoc(), padOp.getResultType(), producerPad.getSource(),
-        newLowPad, newHighPad, padOp.getNofold(),
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, padOp.getLoc(), padOp.getResultType(),
+        producerPad.getSource(), newLowPad, newHighPad, padOp.getNofold(),
         getPrunedAttributeList(padOp, tensor::PadOp::getAttributeNames()));
     rewriter.inlineRegionBefore(padOp.getRegion(), newPadOp.getRegion(),
                                 newPadOp.getRegion().begin());
@@ -4108,8 +4113,8 @@ struct FoldTensorCastProducerOp
     for (auto [oldResult, newResult] :
          llvm::zip(op->getResults(), newOp->getResults())) {
       if (newResult.getType() != oldResult.getType()) {
-        replacements.push_back(tensor::CastOp::create(rewriter,
-            op->getLoc(), oldResult.getType(), newResult));
+        replacements.push_back(tensor::CastOp::create(
+            rewriter, op->getLoc(), oldResult.getType(), newResult));
       } else {
         replacements.push_back(newResult);
       }
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index ba258402b61b3..b8b1a592fdca2 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
index d69535169f956..003d81fbc7771 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtension.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
index a963b3f063a8a..fa64e0fe016a2 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h"
 
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
+#include "mlir/IR/Builders.h"
 #include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 9266a63a0038a..adf1e4c46ea8f 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp b/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
index 93b0bc591ca02..15c2ae22f56ea 100644
--- a/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/IRDLExtension/IRDLExtensionOps.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/ADT/STLExtras.h"
 
diff --git a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
index 95870e8ef87be..696dbe3d1892a 100644
--- a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h"
 
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
+#include "mlir/IR/Builders.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/UB/IR/UBOps.cpp b/mlir/lib/Dialect/UB/IR/UBOps.cpp
index ee523f9522953..442a53ca8162a 100644
--- a/mlir/lib/Dialect/UB/IR/UBOps.cpp
+++ b/mlir/lib/Dialect/UB/IR/UBOps.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "mlir/Dialect/UB/IR/UBOpsDialect.cpp.inc"
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 3497b0d7a88a0..4dc3bebef8cfb 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -30,6 +30,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -599,8 +600,8 @@ struct ElideUnitDimsInMultiDimReduction
                             dstVecType.getScalableDims());
         mask = vector::ShapeCastOp::create(rewriter, loc, newMaskType, mask);
       }
-      cast = vector::ShapeCastOp::create(rewriter,
-          loc, reductionOp.getDestType(), reductionOp.getSource());
+      cast = vector::ShapeCastOp::create(
+          rewriter, loc, reductionOp.getDestType(), reductionOp.getSource());
     } else {
       // This means we are reducing all the dimensions, and all reduction
       // dimensions are of size 1. So a simple extraction would do.
@@ -674,35 +675,35 @@ Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
   case arith::AtomicRMWKind::addf:
   case arith::AtomicRMWKind::addi:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::ADD, vector);
+                                       CombiningKind::ADD, vector);
   case arith::AtomicRMWKind::mulf:
   case arith::AtomicRMWKind::muli:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MUL, vector);
+                                       CombiningKind::MUL, vector);
   case arith::AtomicRMWKind::minimumf:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINIMUMF, vector);
+                                       CombiningKind::MINIMUMF, vector);
   case arith::AtomicRMWKind::mins:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINSI, vector);
+                                       CombiningKind::MINSI, vector);
   case arith::AtomicRMWKind::minu:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MINUI, vector);
+                                       CombiningKind::MINUI, vector);
   case arith::AtomicRMWKind::maximumf:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXIMUMF, vector);
+                                       CombiningKind::MAXIMUMF, vector);
   case arith::AtomicRMWKind::maxs:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXSI, vector);
+                                       CombiningKind::MAXSI, vector);
   case arith::AtomicRMWKind::maxu:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::MAXUI, vector);
+                                       CombiningKind::MAXUI, vector);
   case arith::AtomicRMWKind::andi:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::AND, vector);
+                                       CombiningKind::AND, vector);
   case arith::AtomicRMWKind::ori:
     return vector::ReductionOp::create(builder, vector.getLoc(),
-                                               CombiningKind::OR, vector);
+                                       CombiningKind::OR, vector);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
@@ -4255,8 +4256,8 @@ class StridedSliceBroadcast final
     // just a single scalar.
     bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
     if (!lowerDimMatch && !isScalarSrc) {
-      source = ExtractStridedSliceOp::create(rewriter,
-          op->getLoc(), source,
+      source = ExtractStridedSliceOp::create(
+          rewriter, op->getLoc(), source,
           getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
           getI64SubArray(op.getStrides(), /* dropFront=*/rankDiff));
@@ -4351,7 +4352,7 @@ class ContiguousExtractStridedSliceToExtract final
     SmallVector<int64_t> offsets = getI64SubArray(op.getOffsets());
     auto extractOffsets = ArrayRef(offsets).take_front(numOffsets);
     Value extract = vector::ExtractOp::create(rewriter, op->getLoc(), source,
-                                                       extractOffsets);
+                                              extractOffsets);
     rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(), extract);
     return success();
   }
@@ -5421,13 +5422,14 @@ struct SwapExtractSliceOfTransferWrite
     // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
     // Set all in_bounds to false and let the folder infer them.
     SmallVector<bool> newInBounds(vectorShape.size(), false);
-    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
-        extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
-        insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
-        insertOp.getMixedStrides());
-    auto newTransferWriteOp = TransferWriteOp::create(rewriter,
-        transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
-        transferOp.getIndices(), transferOp.getPermutationMapAttr(),
+    auto newExtractOp = tensor::ExtractSliceOp::create(
+        rewriter, extractOp.getLoc(), insertOp.getSourceType(),
+        insertOp.getDest(), insertOp.getMixedOffsets(),
+        insertOp.getMixedSizes(), insertOp.getMixedStrides());
+    auto newTransferWriteOp = TransferWriteOp::create(
+        rewriter, transferOp.getLoc(), transferOp.getVector(),
+        newExtractOp.getResult(), transferOp.getIndices(),
+        transferOp.getPermutationMapAttr(),
         rewriter.getBoolArrayAttr(newInBounds));
     rewriter.modifyOpInPlace(insertOp, [&]() {
       insertOp.getSourceMutable().assign(newTransferWriteOp.getResult());
@@ -7299,11 +7301,11 @@ Operation *mlir::vector::maskOperation(OpBuilder &builder,
     return maskableOp;
   if (passthru)
     return MaskOp::create(builder, maskableOp->getLoc(),
-                                  maskableOp->getResultTypes(), mask, passthru,
-                                  maskableOp, createMaskOpRegion);
+                          maskableOp->getResultTypes(), mask, passthru,
+                          maskableOp, createMaskOpRegion);
   return MaskOp::create(builder, maskableOp->getLoc(),
-                                maskableOp->getResultTypes(), mask, maskableOp,
-                                createMaskOpRegion);
+                        maskableOp->getResultTypes(), mask, maskableOp,
+                        createMaskOpRegion);
 }
 
 /// Creates a vector select operation that picks values from `newValue` or
@@ -7319,7 +7321,7 @@ Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
     return newValue;
 
   return arith::SelectOp::create(builder, newValue.getLoc(), newValue.getType(),
-                                         mask, newValue, passthru);
+                                 mask, newValue, passthru);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 125c3d918284c..bb051650897b4 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
 #include "mlir/Dialect/X86Vector/Transforms.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
index 01acade66a3c4..99654f79ead24 100644
--- a/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
+++ b/mlir/lib/Dialect/X86Vector/IR/X86VectorDialect.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
@@ -64,7 +65,7 @@ SmallVector<Value> x86vector::MaskCompressOp::getIntrinsicOperands(
     src = adaptor.getSrc();
   } else if (adaptor.getConstantSrc()) {
     src = LLVM::ConstantOp::create(rewriter, loc, opType,
-                                            adaptor.getConstantSrcAttr());
+                                   adaptor.getConstantSrcAttr());
   } else {
     auto zeroAttr = rewriter.getZeroAttr(opType);
     src = LLVM::ConstantOp::create(rewriter, loc, opType, zeroAttr);
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index f155b2a363834..4622f8fc67e94 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
 
 #include "llvm/Support/Debug.h"
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 6d7e2aa0ece7d..23988a00c7a2c 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectResourceBlobManager.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeRange.h"
 
diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
index cdf44c2959d50..9753ac00af88a 100644
--- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
+++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
index 3d24d4ecc4d0d..477219b79d937 100644
--- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
+++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/IR/Dominance.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "llvm/Support/Debug.h"
diff --git a/mlir/test/lib/Transforms/TestTransformsOps.cpp b/mlir/test/lib/Transforms/TestTransformsOps.cpp
index 9a5632bb99c06..3d7b6be8a78e9 100644
--- a/mlir/test/lib/Transforms/TestTransformsOps.cpp
+++ b/mlir/test/lib/Transforms/TestTransformsOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Transforms/RegionUtils.h"
 
diff --git a/mlir/test/python/lib/PythonTestDialect.cpp b/mlir/test/python/lib/PythonTestDialect.cpp
index a0ff31504c691..e0a76f1b75d38 100644
--- a/mlir/test/python/lib/PythonTestDialect.cpp
+++ b/mlir/test/python/lib/PythonTestDialect.cpp
@@ -9,6 +9,7 @@
 #include "PythonTestDialect.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "PythonTestDialect.cpp.inc"

>From b69c5b8179dab663906dcc466b275db9fb3479e7 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Fri, 11 Jul 2025 19:41:00 -0400
Subject: [PATCH 5/5] format

---
 flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp   |   2 +-
 .../lib/Standalone/StandaloneDialect.cpp      |   2 +-
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           | 172 +++---
 .../AffineToStandard/AffineToStandard.cpp     |  22 +-
 .../ArithToAMDGPU/ArithToAMDGPU.cpp           |  62 +-
 .../ArithToArmSME/ArithToArmSME.cpp           |   4 +-
 .../Conversion/ArithToEmitC/ArithToEmitC.cpp  |  78 +--
 .../Conversion/ArithToLLVM/ArithToLLVM.cpp    |  16 +-
 .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp  |  45 +-
 .../ArmNeon2dToIntr/ArmNeon2dToIntr.cpp       |   4 +-
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  | 175 +++---
 .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp    |  76 +--
 .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp    |  66 +--
 .../BufferizationToMemRef.cpp                 |   3 +-
 .../ComplexCommon/DivisionConverter.cpp       | 442 +++++++-------
 .../ComplexToLLVM/ComplexToLLVM.cpp           |  28 +-
 .../ComplexToLibm/ComplexToLibm.cpp           |   2 +-
 .../ComplexToStandard/ComplexToStandard.cpp   | 234 ++++----
 .../ControlFlowToLLVM/ControlFlowToLLVM.cpp   |   2 +-
 .../ControlFlowToSCF/ControlFlowToSCF.cpp     |  26 +-
 .../Conversion/FuncToEmitC/FuncToEmitC.cpp    |   4 +-
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp |  45 +-
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |  79 +--
 .../GPUCommon/GPUToLLVMConversion.cpp         |  93 +--
 .../GPUCommon/IndexIntrinsicsOpLowering.h     |  10 +-
 .../GPUCommon/OpToFuncCallLowering.h          |  20 +-
 .../Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp  |  10 +-
 .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp        |  37 +-
 .../Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp    |  32 +-
 .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp      |  38 +-
 mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp |  68 +--
 .../Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp  |   5 +-
 .../Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp  |   8 +-
 .../Conversion/IndexToLLVM/IndexToLLVM.cpp    |   8 +-
 .../Conversion/IndexToSPIRV/IndexToSPIRV.cpp  |  33 +-
 .../Conversion/LLVMCommon/MemRefBuilder.cpp   |  54 +-
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    |  42 +-
 .../Conversion/LLVMCommon/PrintCallHelper.cpp |   6 +-
 .../Conversion/LLVMCommon/TypeConverter.cpp   |   2 +-
 .../Conversion/LLVMCommon/VectorPattern.cpp   |   4 +-
 .../LinalgToStandard/LinalgToStandard.cpp     |   8 +-
 mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp   |  70 +--
 .../Conversion/MathToFuncs/MathToFuncs.cpp    | 143 ++---
 mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp |  70 +--
 mlir/lib/Conversion/MathToLibm/MathToLibm.cpp |   9 +-
 .../Conversion/MathToSPIRV/MathToSPIRV.cpp    |  50 +-
 .../MemRefToEmitC/MemRefToEmitC.cpp           |   8 +-
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  | 128 ++--
 .../MemRefToSPIRV/MemRefToSPIRV.cpp           |  29 +-
 mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp   | 175 +++---
 .../Conversion/OpenACCToSCF/OpenACCToSCF.cpp  |   2 +-
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |   2 +-
 .../PDLToPDLInterp/PDLToPDLInterp.cpp         | 181 +++---
 .../SCFToControlFlow/SCFToControlFlow.cpp     |  24 +-
 mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp |  10 +-
 mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp     |  48 +-
 .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp    |  35 +-
 mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp |  41 +-
 .../ConvertLaunchFuncToLLVMCalls.cpp          |  12 +-
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    | 111 ++--
 .../ShapeToStandard/ShapeToStandard.cpp       | 157 ++---
 .../TensorToSPIRV/TensorToSPIRV.cpp           |   4 +-
 .../Conversion/TosaToArith/TosaToArith.cpp    |  39 +-
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  | 545 +++++++++---------
 .../TosaToLinalg/TosaToLinalgNamed.cpp        | 143 ++---
 .../TosaToMLProgram/TosaToMLProgram.cpp       |  12 +-
 mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp   |  24 +-
 .../Conversion/TosaToTensor/TosaToTensor.cpp  |  27 +-
 .../VectorToArmSME/VectorToArmSME.cpp         |  67 +--
 .../Conversion/VectorToGPU/VectorToGPU.cpp    |  74 +--
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      | 191 +++---
 .../Conversion/VectorToSCF/VectorToSCF.cpp    | 124 ++--
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  79 +--
 .../VectorToXeGPU/VectorToXeGPU.cpp           |  31 +-
 .../AMDGPU/Transforms/EmulateAtomics.cpp      |  15 +-
 .../AMDGPU/Transforms/MaskedloadToLoad.cpp    |  20 +-
 .../Transforms/ResolveStridedMetadata.cpp     |   8 +-
 .../Transforms/AffineExpandIndexOps.cpp       |  21 +-
 .../Affine/Transforms/DecomposeAffineOps.cpp  |   4 +-
 .../Transforms/PipelineDataTransfer.cpp       |   6 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |  76 +--
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |  80 +--
 .../BufferDeallocationOpInterfaceImpl.cpp     |   4 +-
 .../Arith/Transforms/EmulateWideInt.cpp       |  73 +--
 .../Dialect/Arith/Transforms/ExpandOps.cpp    |  69 +--
 .../Arith/Transforms/ReifyValueBounds.cpp     |  12 +-
 mlir/lib/Dialect/Arith/Utils/Utils.cpp        |  14 +-
 .../LowerContractionToNeonI8MMPattern.cpp     |  18 +-
 mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp         |   2 +-
 mlir/lib/Dialect/ArmSME/IR/Utils.cpp          |   8 +-
 .../ArmSME/Transforms/OuterProductFusion.cpp  |   9 +-
 .../ArmSME/Transforms/VectorLegalization.cpp  | 125 ++--
 .../Transforms/LegalizeForLLVMExport.cpp      |  18 +-
 .../Transforms/LegalizeVectorStorage.cpp      |  22 +-
 .../LowerContractionToSVEI8MMPattern.cpp      |  42 +-
 mlir/lib/Dialect/Async/IR/Async.cpp           |   2 +-
 .../Transforms/AsyncRuntimeRefCounting.cpp    |   2 +-
 .../Async/Transforms/AsyncToAsyncRuntime.cpp  |  36 +-
 .../IR/BufferDeallocationOpInterface.cpp      |   8 +-
 .../IR/BufferizableOpInterface.cpp            |  10 +-
 .../BufferDeallocationSimplification.cpp      |  25 +-
 .../Transforms/BufferResultsToOutParams.cpp   |   4 +-
 .../Bufferization/Transforms/BufferUtils.cpp  |   5 +-
 .../Bufferization/Transforms/Bufferize.cpp    |  12 +-
 .../DropEquivalentBufferResults.cpp           |   6 +-
 .../Transforms/EmptyTensorElimination.cpp     |   2 +-
 .../FuncBufferizableOpInterfaceImpl.cpp       |   9 +-
 .../Transforms/LowerDeallocations.cpp         | 202 +++----
 .../OwnershipBasedBufferDeallocation.cpp      |  38 +-
 .../lib/Dialect/Complex/IR/ComplexDialect.cpp |   2 +-
 .../BufferDeallocationOpInterfaceImpl.cpp     |  23 +-
 .../EmitC/Transforms/WrapFuncInClass.cpp      |   2 +-
 .../Func/Extensions/InlinerExtension.cpp      |   3 +-
 .../Func/Transforms/FuncConversions.cpp       |   6 +-
 mlir/lib/Dialect/Func/Utils/Utils.cpp         |   9 +-
 mlir/lib/Dialect/GPU/TransformOps/Utils.cpp   |   9 +-
 .../GPU/Transforms/AsyncRegionRewriter.cpp    |   5 +-
 .../GPU/Transforms/DecomposeMemRefs.cpp       |   4 +-
 .../GPU/Transforms/KernelOutlining.cpp        |  12 +-
 .../Dialect/GPU/Transforms/ModuleToBinary.cpp |   2 +-
 .../GPU/Transforms/PromoteShuffleToAMDGPU.cpp |   4 +-
 .../GPU/Transforms/ShuffleRewriter.cpp        |   4 +-
 .../GPU/Transforms/SubgroupIdRewriter.cpp     |  16 +-
 .../GPU/Transforms/SubgroupReduceLowering.cpp | 103 ++--
 .../Dialect/GPU/Utils/DistributionUtils.cpp   |   7 +-
 .../LLVMIR/IR/BasicPtxBuilderInterface.cpp    |   8 +-
 .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp   |   4 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp |  63 +-
 .../Dialect/LLVMIR/Transforms/AddComdats.cpp  |   5 +-
 .../Transforms/InlinerInterfaceImpl.cpp       |  35 +-
 .../LLVMIR/Transforms/LegalizeForExport.cpp   |   2 +-
 .../LLVMIR/Transforms/OptimizeForNVVM.cpp     |  16 +-
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 152 ++---
 .../TransformOps/LinalgTransformOps.cpp       |  24 +-
 .../BufferizableOpInterfaceImpl.cpp           |   4 +-
 .../Transforms/ConvertConv2DToImg2Col.cpp     | 123 ++--
 .../Transforms/ConvertToDestinationStyle.cpp  |  51 +-
 .../Transforms/DataLayoutPropagation.cpp      |  79 +--
 ...DecomposeGenericByUnfoldingPermutation.cpp |  13 +-
 .../Linalg/Transforms/DecomposeLinalgOps.cpp  |  20 +-
 .../Dialect/Linalg/Transforms/Detensorize.cpp |   8 +-
 .../Linalg/Transforms/DropUnitDims.cpp        |  45 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  74 +--
 .../Linalg/Transforms/ElementwiseToLinalg.cpp |   4 +-
 .../EraseUnusedOperandsAndResults.cpp         |   4 +-
 .../FusePadOpWithLinalgProducer.cpp           |  12 +-
 .../Linalg/Transforms/Generalization.cpp      |   5 +-
 .../Linalg/Transforms/HoistPadding.cpp        |  40 +-
 .../Dialect/Linalg/Transforms/Hoisting.cpp    |   6 +-
 .../Transforms/InlineScalarOperands.cpp       |  10 +-
 .../Dialect/Linalg/Transforms/Interchange.cpp |   3 +-
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |   5 +-
 .../Transforms/MeshShardingInterfaceImpl.cpp  |  14 +-
 .../Linalg/Transforms/NamedOpConversions.cpp  |  17 +-
 .../Transforms/PackAndUnpackPatterns.cpp      |  10 +-
 .../Linalg/Transforms/PadTilingInterface.cpp  |  10 +-
 .../lib/Dialect/Linalg/Transforms/Padding.cpp |  10 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |   6 +-
 mlir/lib/Dialect/Linalg/Transforms/Split.cpp  |   4 +-
 .../Linalg/Transforms/SplitReduction.cpp      |  32 +-
 .../SwapExtractSliceWithFillPatterns.cpp      |   8 +-
 mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp |  27 +-
 .../Linalg/Transforms/TilingInterfaceImpl.cpp |  74 +--
 .../Dialect/Linalg/Transforms/Transforms.cpp  | 115 ++--
 .../Linalg/Transforms/TransposeConv2D.cpp     |   2 +-
 .../Linalg/Transforms/TransposeMatmul.cpp     |  34 +-
 .../Linalg/Transforms/Vectorization.cpp       | 255 ++++----
 .../Linalg/Transforms/WinogradConv2D.cpp      | 155 ++---
 mlir/lib/Dialect/Linalg/Utils/Utils.cpp       |  24 +-
 .../BufferizableOpInterfaceImpl.cpp           |   5 +-
 .../Transforms/AlgebraicSimplification.cpp    |  13 +-
 .../Math/Transforms/ExpandPatterns.cpp        |  63 +-
 .../Transforms/PolynomialApproximation.cpp    | 143 ++---
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    |   2 +-
 .../Transforms/AllocationOpInterfaceImpl.cpp  |   5 +-
 .../MemRef/Transforms/ComposeSubView.cpp      |   4 +-
 .../MemRef/Transforms/EmulateNarrowType.cpp   | 128 ++--
 .../Dialect/MemRef/Transforms/ExpandOps.cpp   |   9 +-
 .../MemRef/Transforms/ExpandRealloc.cpp       |  22 +-
 .../Transforms/ExpandStridedMetadata.cpp      |  27 +-
 .../Transforms/ExtractAddressComputations.cpp |  26 +-
 .../MemRef/Transforms/FlattenMemRefs.cpp      |  55 +-
 .../MemRef/Transforms/FoldMemRefAliasOps.cpp  |  12 +-
 .../Transforms/IndependenceTransforms.cpp     |  12 +-
 .../Dialect/MemRef/Transforms/MultiBuffer.cpp |  13 +-
 .../MemRef/Transforms/NormalizeMemRefs.cpp    |   2 +-
 .../MemRef/Transforms/ReifyResultShapes.cpp   |   6 +-
 .../Transforms/RuntimeOpVerification.cpp      | 110 ++--
 .../Mesh/Transforms/Simplifications.cpp       |   4 +-
 .../Dialect/Mesh/Transforms/Spmdization.cpp   |  24 +-
 .../Dialect/Mesh/Transforms/Transforms.cpp    |  25 +-
 .../NVGPU/TransformOps/NVGPUTransformOps.cpp  |  51 +-
 .../NVGPU/Transforms/CreateAsyncGroups.cpp    |  19 +-
 .../NVGPU/Transforms/OptimizeSharedMemory.cpp |   3 +-
 .../Quant/Transforms/LowerQuantOps.cpp        |  30 +-
 .../Quant/Transforms/StripFuncQuantTypes.cpp  |   2 +-
 .../BufferizableOpInterfaceImpl.cpp           |  32 +-
 .../lib/Dialect/SCF/Transforms/ForToWhile.cpp |  13 +-
 .../Dialect/SCF/Transforms/LoopPipelining.cpp | 101 ++--
 .../SCF/Transforms/LoopSpecialization.cpp     |   4 +-
 .../SCF/Transforms/ParallelLoopFusion.cpp     |   4 +-
 .../SCF/Transforms/ParallelLoopTiling.cpp     |  30 +-
 .../Transforms/StructuralTypeConversions.cpp  |  18 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  61 +-
 .../SCF/Transforms/WrapInZeroTripCheck.cpp    |   8 +-
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          |  97 ++--
 mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp  |   8 +-
 .../SPIRV/IR/SPIRVCanonicalization.cpp        |   6 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp        |  36 +-
 .../Linking/ModuleCombiner/ModuleCombiner.cpp |   5 +-
 .../Transforms/LowerABIAttributesPass.cpp     |  24 +-
 .../SPIRV/Transforms/RewriteInsertsPass.cpp   |   4 +-
 .../SPIRV/Transforms/SPIRVConversion.cpp      |  59 +-
 .../Transforms/SPIRVWebGPUTransforms.cpp      |  26 +-
 .../Transforms/UnifyAliasedResourcePass.cpp   |  26 +-
 .../BufferizableOpInterfaceImpl.cpp           |   9 +-
 .../Shape/Transforms/ShapeToShapeLowering.cpp |   2 +-
 .../IR/SparseTensorInterfaces.cpp             |   4 +-
 .../Transforms/SparseAssembler.cpp            |  15 +-
 .../Transforms/SparseBufferRewriting.cpp      | 118 ++--
 .../Transforms/SparseGPUCodegen.cpp           | 161 +++---
 .../Transforms/SparseIterationToScf.cpp       |  18 +-
 .../Transforms/SparseReinterpretMap.cpp       |  17 +-
 .../Transforms/SparseSpaceCollapse.cpp        |   4 +-
 .../SparseStorageSpecifierToLLVM.cpp          |  10 +-
 .../Transforms/SparseTensorCodegen.cpp        | 107 ++--
 .../Transforms/SparseTensorConversion.cpp     |  24 +-
 .../Transforms/SparseTensorRewriting.cpp      | 102 ++--
 .../Transforms/SparseVectorization.cpp        |  32 +-
 .../Transforms/Sparsification.cpp             |  19 +-
 .../Transforms/Utils/CodegenUtils.cpp         |  23 +-
 .../Transforms/Utils/CodegenUtils.h           |   8 +-
 .../Transforms/Utils/LoopEmitter.cpp          |  18 +-
 .../Utils/SparseTensorDescriptor.cpp          |  18 +-
 .../Transforms/Utils/SparseTensorIterator.cpp |  43 +-
 .../lib/Dialect/SparseTensor/Utils/Merger.cpp |   6 +-
 .../Extensions/MeshShardingExtensions.cpp     |   4 +-
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp   |  20 +-
 .../BufferizableOpInterfaceImpl.cpp           |  63 +-
 .../Tensor/Transforms/EmptyOpPatterns.cpp     |   5 +-
 .../ExtractSliceFromReshapeUtils.cpp          |  13 +-
 .../Tensor/Transforms/FoldTensorSubsetOps.cpp |   6 +-
 .../Transforms/IndependenceTransforms.cpp     |  15 +-
 .../Tensor/Transforms/ReshapePatterns.cpp     |  22 +-
 .../Transforms/RuntimeOpVerification.cpp      |  39 +-
 .../SubsetInsertionOpInterfaceImpl.cpp        |   4 +-
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |  12 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |   6 +-
 .../Transforms/TosaDecomposeDepthwise.cpp     |   8 +-
 .../Transforms/TosaDecomposeTransposeConv.cpp |  22 +-
 .../Tosa/Transforms/TosaReduceTransposes.cpp  |   8 +-
 .../Dialect/Tosa/Utils/ConversionUtils.cpp    |   4 +-
 .../LoopExtension/LoopExtensionOps.cpp        |   2 +-
 .../BufferizableOpInterfaceImpl.cpp           |   9 +-
 .../Vector/Transforms/LowerVectorContract.cpp |  54 +-
 .../Vector/Transforms/LowerVectorGather.cpp   |  24 +-
 .../Transforms/LowerVectorInterleave.cpp      |  22 +-
 .../Vector/Transforms/LowerVectorMask.cpp     |  31 +-
 .../Transforms/LowerVectorMultiReduction.cpp  |  62 +-
 .../Vector/Transforms/LowerVectorScan.cpp     |  24 +-
 .../Transforms/LowerVectorShapeCast.cpp       |  46 +-
 .../Vector/Transforms/LowerVectorTransfer.cpp |  44 +-
 .../Transforms/LowerVectorTranspose.cpp       |  22 +-
 .../Vector/Transforms/VectorDistribute.cpp    | 100 ++--
 .../Transforms/VectorDropLeadUnitDim.cpp      |  65 ++-
 .../VectorEmulateMaskedLoadStore.cpp          |  18 +-
 .../Transforms/VectorEmulateNarrowType.cpp    | 212 +++----
 ...sertExtractStridedSliceRewritePatterns.cpp |  32 +-
 .../Transforms/VectorMaskElimination.cpp      |   4 +-
 .../Transforms/VectorTransferOpTransforms.cpp |  36 +-
 .../VectorTransferSplitRewritePatterns.cpp    |  58 +-
 .../Vector/Transforms/VectorTransforms.cpp    |  96 +--
 .../Vector/Transforms/VectorUnroll.cpp        |  58 +-
 mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp |   4 +-
 .../X86Vector/Transforms/AVXTranspose.cpp     |  14 +-
 .../Transforms/XeGPUSubgroupDistribute.cpp    |  47 +-
 .../Dialect/XeGPU/Transforms/XeGPUUnroll.cpp  |  51 +-
 .../Transforms/XeGPUWgToSgDistribute.cpp      |  16 +-
 mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp   |  16 +-
 .../lib/Target/LLVMIR/LLVMImportInterface.cpp |   4 +-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |  85 +--
 .../SPIRV/Deserialization/DeserializeOps.cpp  |  38 +-
 .../SPIRV/Deserialization/Deserializer.cpp    |  46 +-
 mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp       |  43 +-
 mlir/test/python/lib/PythonTestDialect.cpp    |   2 +-
 285 files changed, 5866 insertions(+), 5467 deletions(-)

diff --git a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
index d7d294ab621d8..4cd71f6e1f3c7 100644
--- a/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRCG/CGOps.cpp
@@ -12,9 +12,9 @@
 
 #include "flang/Optimizer/Dialect/FIRCG/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 /// FIR codegen dialect constructor.
 fir::FIRCodeGenDialect::FIRCodeGenDialect(mlir::MLIRContext *ctx)
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
index 3da966ef9cc85..4861300c8b546 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp
@@ -8,8 +8,8 @@
 
 #include "Standalone/StandaloneDialect.h"
 #include "Standalone/StandaloneOps.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "Standalone/StandaloneTypes.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 
 using namespace mlir;
 using namespace mlir::standalone;
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index e7e0a4ebd2092..fe3dc91328879 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -81,8 +81,8 @@ static Value getLinearIndexI32(ConversionPatternRewriter &rewriter,
               : LLVM::ConstantOp::create(rewriter, loc, i32, stride);
       increment = LLVM::MulOp::create(rewriter, loc, increment, strideValue);
     }
-    index =
-        index ? LLVM::AddOp::create(rewriter, loc, index, increment) : increment;
+    index = index ? LLVM::AddOp::create(rewriter, loc, index, increment)
+                  : increment;
   }
   return index ? index : createI32Constant(rewriter, loc, 0);
 }
@@ -133,13 +133,13 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
   if (chipset.majorVersion == 9 && chipset >= kGfx942 && cacheSwizzleStride) {
     Value cacheStrideZext =
         LLVM::ZExtOp::create(rewriter, loc, i16, cacheSwizzleStride);
-    Value swizzleBit = LLVM::ConstantOp::create(rewriter,
-        loc, i16, rewriter.getI16IntegerAttr(1 << 14));
+    Value swizzleBit = LLVM::ConstantOp::create(
+        rewriter, loc, i16, rewriter.getI16IntegerAttr(1 << 14));
     stride = LLVM::OrOp::create(rewriter, loc, cacheStrideZext, swizzleBit,
-                                         /*isDisjoint=*/true);
+                                /*isDisjoint=*/true);
   } else {
     stride = LLVM::ConstantOp::create(rewriter, loc, i16,
-                                               rewriter.getI16IntegerAttr(0));
+                                      rewriter.getI16IntegerAttr(0));
   }
   // Get the number of elements.
   // Flag word:
@@ -209,20 +209,21 @@ struct FatRawBufferCastLowering
             : descriptor.alignedPtr(rewriter, loc);
 
     Value offset = adaptor.getResetOffset()
-                       ? LLVM::ConstantOp::create(rewriter,
-                             loc, getIndexType(), rewriter.getIndexAttr(0))
+                       ? LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                                  rewriter.getIndexAttr(0))
                        : descriptor.offset(rewriter, loc);
 
     bool hasSizes = memrefType.getRank() > 0;
     // No need to unpack() and pack() all the individual sizes and strides,
     // so we'll just extract the arrays.
-    Value sizes = hasSizes ? LLVM::ExtractValueOp::create(rewriter,
-                                 loc, descriptor, kSizePosInMemRefDescriptor)
-                           : Value{};
-    Value strides = hasSizes
-                        ? LLVM::ExtractValueOp::create(rewriter,
-                              loc, descriptor, kStridePosInMemRefDescriptor)
-                        : Value{};
+    Value sizes = hasSizes
+                      ? LLVM::ExtractValueOp::create(rewriter, loc, descriptor,
+                                                     kSizePosInMemRefDescriptor)
+                      : Value{};
+    Value strides =
+        hasSizes ? LLVM::ExtractValueOp::create(rewriter, loc, descriptor,
+                                                kStridePosInMemRefDescriptor)
+                 : Value{};
 
     Value fatPtr = makeBufferRsrc(
         rewriter, loc, basePointer, numRecords, adaptor.getBoundsCheck(),
@@ -231,17 +232,17 @@ struct FatRawBufferCastLowering
     Value result = MemRefDescriptor::poison(
         rewriter, loc,
         getTypeConverter()->convertType(op.getResult().getType()));
-    result = LLVM::InsertValueOp::create(rewriter,
-        loc, result, fatPtr, kAllocatedPtrPosInMemRefDescriptor);
-    result = LLVM::InsertValueOp::create(rewriter,
-        loc, result, fatPtr, kAlignedPtrPosInMemRefDescriptor);
+    result = LLVM::InsertValueOp::create(rewriter, loc, result, fatPtr,
+                                         kAllocatedPtrPosInMemRefDescriptor);
+    result = LLVM::InsertValueOp::create(rewriter, loc, result, fatPtr,
+                                         kAlignedPtrPosInMemRefDescriptor);
     result = LLVM::InsertValueOp::create(rewriter, loc, result, offset,
-                                                  kOffsetPosInMemRefDescriptor);
+                                         kOffsetPosInMemRefDescriptor);
     if (hasSizes) {
       result = LLVM::InsertValueOp::create(rewriter, loc, result, sizes,
-                                                    kSizePosInMemRefDescriptor);
-      result = LLVM::InsertValueOp::create(rewriter,
-          loc, result, strides, kStridePosInMemRefDescriptor);
+                                           kSizePosInMemRefDescriptor);
+      result = LLVM::InsertValueOp::create(rewriter, loc, result, strides,
+                                           kStridePosInMemRefDescriptor);
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -342,8 +343,8 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     SmallVector<Value, 6> args;
     if (storeData) {
       if (llvmBufferValType != llvmWantedDataType) {
-        Value castForStore =
-            LLVM::BitcastOp::create(rewriter, loc, llvmBufferValType, storeData);
+        Value castForStore = LLVM::BitcastOp::create(
+            rewriter, loc, llvmBufferValType, storeData);
         args.push_back(castForStore);
       } else {
         args.push_back(storeData);
@@ -352,8 +353,8 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
 
     if (atomicCmpData) {
       if (llvmBufferValType != llvmWantedDataType) {
-        Value castForCmp = LLVM::BitcastOp::create(rewriter,
-            loc, llvmBufferValType, atomicCmpData);
+        Value castForCmp = LLVM::BitcastOp::create(
+            rewriter, loc, llvmBufferValType, atomicCmpData);
         args.push_back(castForCmp);
       } else {
         args.push_back(atomicCmpData);
@@ -382,9 +383,9 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     if (std::optional<int32_t> indexOffset = adaptor.getIndexOffset();
         indexOffset && *indexOffset > 0) {
       Value extraOffsetConst = createI32Constant(rewriter, loc, *indexOffset);
-      voffset =
-          voffset ? LLVM::AddOp::create(rewriter, loc, voffset, extraOffsetConst)
-                  : extraOffsetConst;
+      voffset = voffset ? LLVM::AddOp::create(rewriter, loc, voffset,
+                                              extraOffsetConst)
+                        : extraOffsetConst;
     }
     voffset = LLVM::MulOp::create(rewriter, loc, voffset, byteWidthConst);
     args.push_back(voffset);
@@ -404,12 +405,12 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
                                            llvmBufferValType);
     Operation *lowered = Intrinsic::create(rewriter, loc, resultTypes, args,
-                                                    ArrayRef<NamedAttribute>());
+                                           ArrayRef<NamedAttribute>());
     if (lowered->getNumResults() == 1) {
       Value replacement = lowered->getResult(0);
       if (llvmBufferValType != llvmWantedDataType) {
         replacement = LLVM::BitcastOp::create(rewriter, loc, llvmWantedDataType,
-                                                       replacement);
+                                              replacement);
       }
       rewriter.replaceOp(gpuOp, replacement);
     } else {
@@ -516,19 +517,21 @@ static Value convertMFMAVectorOperand(ConversionPatternRewriter &rewriter,
   Type inputType = input.getType();
   if (auto vectorType = dyn_cast<VectorType>(inputType)) {
     if (vectorType.getElementType().isBF16() && !allowBf16)
-      return LLVM::BitcastOp::create(rewriter,
-          loc, vectorType.clone(rewriter.getI16Type()), input);
+      return LLVM::BitcastOp::create(
+          rewriter, loc, vectorType.clone(rewriter.getI16Type()), input);
     if (vectorType.getElementType().isInteger(8) &&
         vectorType.getNumElements() <= 8)
-      return LLVM::BitcastOp::create(rewriter,
-          loc, rewriter.getIntegerType(vectorType.getNumElements() * 8), input);
+      return LLVM::BitcastOp::create(
+          rewriter, loc,
+          rewriter.getIntegerType(vectorType.getNumElements() * 8), input);
     if (isa<IntegerType>(vectorType.getElementType()) &&
         vectorType.getElementTypeBitWidth() <= 8) {
       int64_t numWords = llvm::divideCeil(
           vectorType.getNumElements() * vectorType.getElementTypeBitWidth(),
           32);
-      return LLVM::BitcastOp::create(rewriter,
-          loc, VectorType::get(numWords, rewriter.getI32Type()), input);
+      return LLVM::BitcastOp::create(
+          rewriter, loc, VectorType::get(numWords, rewriter.getI32Type()),
+          input);
     }
   }
   return input;
@@ -576,8 +579,8 @@ static void wmmaPushInputOperand(ConversionPatternRewriter &rewriter,
   Type elemType = vectorType.getElementType();
 
   if (elemType.isBF16())
-    llvmInput = LLVM::BitcastOp::create(rewriter,
-        loc, vectorType.clone(rewriter.getI16Type()), llvmInput);
+    llvmInput = LLVM::BitcastOp::create(
+        rewriter, loc, vectorType.clone(rewriter.getI16Type()), llvmInput);
   if (elemType.getIntOrFloatBitWidth() > 8) {
     operands.push_back(llvmInput);
     return;
@@ -633,8 +636,8 @@ static void wmmaPushOutputOperand(ConversionPatternRewriter &rewriter,
   auto vectorType = dyn_cast<VectorType>(inputType);
   Type elemType = vectorType.getElementType();
   if (elemType.isBF16())
-    output = LLVM::BitcastOp::create(rewriter,
-        loc, vectorType.clone(rewriter.getI16Type()), output);
+    output = LLVM::BitcastOp::create(
+        rewriter, loc, vectorType.clone(rewriter.getI16Type()), output);
   operands.push_back(output);
   if (elemType.isF16() || elemType.isBF16() || elemType.isInteger(16)) {
     operands.push_back(createI1Constant(rewriter, loc, subwordOffset));
@@ -1092,8 +1095,8 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
 
     Operation *maybeCastBack = lowered;
     if (rawOutType != outType)
-      maybeCastBack =
-          LLVM::BitcastOp::create(rewriter, loc, outType, lowered->getResult(0));
+      maybeCastBack = LLVM::BitcastOp::create(rewriter, loc, outType,
+                                              lowered->getResult(0));
     rewriter.replaceOp(op, maybeCastBack->getResults());
 
     return success();
@@ -1143,22 +1146,22 @@ struct TransposeLoadOpLowering
     switch (elementTypeSize) {
     case 4: {
       assert(numElements == 16);
-      auto rocdlOp =
-          ROCDL::ds_read_tr4_b64::create(rewriter, loc, rocdlResultType, srcPtr);
+      auto rocdlOp = ROCDL::ds_read_tr4_b64::create(rewriter, loc,
+                                                    rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 6: {
       assert(numElements == 16);
-      auto rocdlOp =
-          ROCDL::ds_read_tr6_b96::create(rewriter, loc, rocdlResultType, srcPtr);
+      auto rocdlOp = ROCDL::ds_read_tr6_b96::create(rewriter, loc,
+                                                    rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
     case 8: {
       assert(numElements == 8);
-      auto rocdlOp =
-          ROCDL::ds_read_tr8_b64::create(rewriter, loc, rocdlResultType, srcPtr);
+      auto rocdlOp = ROCDL::ds_read_tr8_b64::create(rewriter, loc,
+                                                    rocdlResultType, srcPtr);
       rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
       break;
     }
@@ -1318,8 +1321,8 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
   if (!sourceVecType || sourceVecType.getNumElements() < 4) {
     Value longVec = LLVM::UndefOp::create(rewriter, loc, v4i8);
     if (!sourceVecType) {
-      longVec = LLVM::InsertElementOp::create(rewriter,
-          loc, longVec, source, createI32Constant(rewriter, loc, 0));
+      longVec = LLVM::InsertElementOp::create(
+          rewriter, loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
@@ -1384,8 +1387,8 @@ LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
   if (sourceVecType.getNumElements() < packedVecType.getNumElements()) {
     Value longVec = LLVM::ZeroOp::create(rewriter, loc, packedVecType);
     if (!sourceVecType) {
-      longVec = LLVM::InsertElementOp::create(rewriter,
-          loc, longVec, source, createI32Constant(rewriter, loc, 0));
+      longVec = LLVM::InsertElementOp::create(
+          rewriter, loc, longVec, source, createI32Constant(rewriter, loc, 0));
     } else {
       for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
         Value idx = createI32Constant(rewriter, loc, i);
@@ -1476,32 +1479,35 @@ LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
 
   Value result;
   if (sourceElemType.isF32() && isa<Float8E5M2Type>(resultElemType))
-    result = ROCDL::CvtScaleF32PkBf8F32Op::create(rewriter,
-        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkBf8F32Op::create(rewriter, loc, intResultType,
+                                                  existing, sourceA, sourceB,
+                                                  scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E5M2Type>(resultElemType))
-    result = ROCDL::CvtScaleF32PkBf8F16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkBf8F16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E5M2Type>(resultElemType))
-    result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float8E4M3FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp8F32Op::create(rewriter,
-        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp8F32Op::create(rewriter, loc, intResultType,
+                                                  existing, sourceA, sourceB,
+                                                  scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp8F16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp8F16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float8E4M3FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isF32() && isa<Float4E2M1FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp4F32Op::create(rewriter,
-        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp4F32Op::create(rewriter, loc, intResultType,
+                                                  existing, sourceA, sourceB,
+                                                  scale, op.getIndex());
   else if (sourceElemType.isF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp4F16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp4F16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else if (sourceElemType.isBF16() && isa<Float4E2M1FNType>(resultElemType))
-    result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(rewriter,
-        loc, intResultType, existing, source, scale, op.getIndex());
+    result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(
+        rewriter, loc, intResultType, existing, source, scale, op.getIndex());
   else
     return failure();
 
@@ -1536,10 +1542,10 @@ LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
     result = ROCDL::CvtPkBf8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
-                                                   existing, op.getWordIndex());
+                                          existing, op.getWordIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
     result = ROCDL::CvtPkFp8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
-                                                   existing, op.getWordIndex());
+                                          existing, op.getWordIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
       op, getTypeConverter()->convertType(resultType), result);
@@ -1569,11 +1575,11 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
 
   Value result;
   if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
-    result = ROCDL::CvtSrBf8F32Op::create(rewriter,
-        loc, i32, source, stoch, existing, op.getStoreIndex());
+    result = ROCDL::CvtSrBf8F32Op::create(rewriter, loc, i32, source, stoch,
+                                          existing, op.getStoreIndex());
   else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
-    result = ROCDL::CvtSrFp8F32Op::create(rewriter,
-        loc, i32, source, stoch, existing, op.getStoreIndex());
+    result = ROCDL::CvtSrFp8F32Op::create(rewriter, loc, i32, source, stoch,
+                                          existing, op.getStoreIndex());
 
   result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
       op, getTypeConverter()->convertType(resultType), result);
@@ -1622,8 +1628,9 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
         auto llvmVecType = typeConverter->convertType(mlir::VectorType::get(
             32 / operandType.getIntOrFloatBitWidth(), llvmSrcIntType));
         Value undefVec = LLVM::UndefOp::create(rewriter, loc, llvmVecType);
-        operand = LLVM::InsertElementOp::create(rewriter,
-            loc, undefVec, operand, createI32Constant(rewriter, loc, 0));
+        operand =
+            LLVM::InsertElementOp::create(rewriter, loc, undefVec, operand,
+                                          createI32Constant(rewriter, loc, 0));
         operand = LLVM::BitcastOp::create(rewriter, loc, llvmType, operand);
       }
       return operand;
@@ -1711,8 +1718,9 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
     bool boundCtrl = DppOp->getAttrOfType<BoolAttr>("bound_ctrl").getValue();
 
     // create a ROCDL_DPPMovOp instruction with the appropriate attributes
-    auto dppMovOp = ROCDL::DPPUpdateOp::create(rewriter,
-        loc, llvmType, old, src, DppCtrl, rowMask, bankMask, boundCtrl);
+    auto dppMovOp =
+        ROCDL::DPPUpdateOp::create(rewriter, loc, llvmType, old, src, DppCtrl,
+                                   rowMask, bankMask, boundCtrl);
 
     Value result = dppMovOp.getRes();
     if (srcType.getIntOrFloatBitWidth() < 32) {
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 5a9d946c2fe4e..3b148f9021666 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -156,7 +156,7 @@ class AffineForLowering : public OpRewritePattern<AffineForOp> {
     Value step =
         arith::ConstantIndexOp::create(rewriter, loc, op.getStepAsInt());
     auto scfForOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound,
-                                                step, op.getInits());
+                                       step, op.getInits());
     rewriter.eraseBlock(scfForOp.getBody());
     rewriter.inlineRegionBefore(op.getRegion(), scfForOp.getRegion(),
                                 scfForOp.getRegion().end());
@@ -206,8 +206,8 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
     if (op.getResults().empty()) {
       // Case with no reduction operations/return values.
       parOp = scf::ParallelOp::create(rewriter, loc, lowerBoundTuple,
-                                               upperBoundTuple, steps,
-                                               /*bodyBuilderFn=*/nullptr);
+                                      upperBoundTuple, steps,
+                                      /*bodyBuilderFn=*/nullptr);
       rewriter.eraseBlock(parOp.getBody());
       rewriter.inlineRegionBefore(op.getRegion(), parOp.getRegion(),
                                   parOp.getRegion().end());
@@ -233,9 +233,9 @@ class AffineParallelLowering : public OpRewritePattern<AffineParallelOp> {
       identityVals.push_back(
           arith::getIdentityValue(reductionOpValue, resultType, rewriter, loc));
     }
-    parOp = scf::ParallelOp::create(rewriter,
-        loc, lowerBoundTuple, upperBoundTuple, steps, identityVals,
-        /*bodyBuilderFn=*/nullptr);
+    parOp = scf::ParallelOp::create(rewriter, loc, lowerBoundTuple,
+                                    upperBoundTuple, steps, identityVals,
+                                    /*bodyBuilderFn=*/nullptr);
 
     //  Copy the body of the affine.parallel op.
     rewriter.eraseBlock(parOp.getBody());
@@ -299,17 +299,17 @@ class AffineIfLowering : public OpRewritePattern<AffineIfOp> {
           isEquality ? arith::CmpIPredicate::eq : arith::CmpIPredicate::sge;
       Value cmpVal =
           arith::CmpIOp::create(rewriter, loc, pred, affResult, zeroConstant);
-      cond = cond
-                 ? arith::AndIOp::create(rewriter, loc, cond, cmpVal).getResult()
-                 : cmpVal;
+      cond =
+          cond ? arith::AndIOp::create(rewriter, loc, cond, cmpVal).getResult()
+               : cmpVal;
     }
     cond = cond ? cond
                 : arith::ConstantIntOp::create(rewriter, loc, /*value=*/1,
-                                                        /*width=*/1);
+                                               /*width=*/1);
 
     bool hasElseRegion = !op.getElseRegion().empty();
     auto ifOp = scf::IfOp::create(rewriter, loc, op.getResultTypes(), cond,
-                                           hasElseRegion);
+                                  hasElseRegion);
     rewriter.inlineRegionBefore(op.getThenRegion(),
                                 &ifOp.getThenRegion().back());
     rewriter.eraseBlock(&ifOp.getThenRegion().back());
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index df3f2c4926614..898a8e558dd3c 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -139,16 +139,16 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
   VectorType extResType = VectorType::get(2, rewriter.getF32Type());
   if (!inVecType) {
-    Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
-        loc, rewriter.getF32Type(), in, 0);
+    Value asFloat = amdgpu::ExtPackedFp8Op::create(
+        rewriter, loc, rewriter.getF32Type(), in, 0);
     Value result = castF32To(outElemType, asFloat, loc, rewriter);
     rewriter.replaceOp(op, result);
     return success();
   }
   int64_t numElements = inVecType.getNumElements();
 
-  Value zero = arith::ConstantOp::create(rewriter,
-      loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
+  Value zero = arith::ConstantOp::create(
+      rewriter, loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   VectorType outType = cast<VectorType>(op.getOut().getType());
 
   if (inVecType.getShape().empty()) {
@@ -158,8 +158,8 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
         vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
     Value scalarExt =
         arith::ExtFOp::create(rewriter, loc, outElemType, scalarIn);
-    Value result = vector::InsertOp::create(rewriter, loc, scalarExt, zerodSplat,
-                                                     ArrayRef<int64_t>{});
+    Value result = vector::InsertOp::create(rewriter, loc, scalarExt,
+                                            zerodSplat, ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -176,19 +176,19 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
 
   for (int64_t i = 0; i < numElements; i += 4) {
     int64_t elemsThisOp = std::min(numElements, i + 4) - i;
-    Value inSlice = vector::ExtractStridedSliceOp::create(rewriter,
-        loc, in, i, elemsThisOp, 1);
+    Value inSlice = vector::ExtractStridedSliceOp::create(rewriter, loc, in, i,
+                                                          elemsThisOp, 1);
     for (int64_t j = 0; j < elemsThisOp; j += 2) {
       if (i + j + 1 < numElements) { // Convert two 8-bit elements
-        Value asFloats = amdgpu::ExtPackedFp8Op::create(rewriter,
-            loc, extResType, inSlice, j / 2);
+        Value asFloats = amdgpu::ExtPackedFp8Op::create(
+            rewriter, loc, extResType, inSlice, j / 2);
         Type desType = VectorType::get(2, outElemType);
         Value asType = castF32To(desType, asFloats, loc, rewriter);
-        result = vector::InsertStridedSliceOp::create(rewriter,
-            loc, asType, result, i + j, 1);
+        result = vector::InsertStridedSliceOp::create(rewriter, loc, asType,
+                                                      result, i + j, 1);
       } else { // Convert a 8-bit element
-        Value asFloat = amdgpu::ExtPackedFp8Op::create(rewriter,
-            loc, rewriter.getF32Type(), inSlice, j / 2 * 2);
+        Value asFloat = amdgpu::ExtPackedFp8Op::create(
+            rewriter, loc, rewriter.getF32Type(), inSlice, j / 2 * 2);
         Value asType = castF32To(outElemType, asFloat, loc, rewriter);
         result = vector::InsertOp::create(rewriter, loc, asType, result, i + j);
       }
@@ -250,11 +250,13 @@ static Value clampInput(PatternRewriter &rewriter, Location loc,
       loc, arith::CmpFPredicate::OEQ, source, negInf);
   Value isNan = rewriter.createOrFold<arith::CmpFOp>(
       loc, arith::CmpFPredicate::UNO, source, source);
-  Value isNonFinite = arith::OrIOp::create(rewriter,
-      loc, arith::OrIOp::create(rewriter, loc, isInf, isNegInf), isNan);
+  Value isNonFinite = arith::OrIOp::create(
+      rewriter, loc, arith::OrIOp::create(rewriter, loc, isInf, isNegInf),
+      isNan);
 
   Value clampedBelow = arith::MaximumFOp::create(rewriter, loc, source, minCst);
-  Value clamped = arith::MinimumFOp::create(rewriter, loc, clampedBelow, maxCst);
+  Value clamped =
+      arith::MinimumFOp::create(rewriter, loc, clampedBelow, maxCst);
   Value res =
       arith::SelectOp::create(rewriter, loc, isNonFinite, source, clamped);
   return res;
@@ -290,8 +292,8 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   VectorType truncResType = VectorType::get(4, outElemType);
   if (!inVectorTy) {
     Value asFloat = castToF32(in, loc, rewriter);
-    Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
-        loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
+    Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(
+        rewriter, loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
         /*existing=*/nullptr);
     Value result = vector::ExtractOp::create(rewriter, loc, asF8s, 0);
     rewriter.replaceOp(op, result);
@@ -299,8 +301,8 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
   }
 
   int64_t numElements = outVecType.getNumElements();
-  Value zero = arith::ConstantOp::create(rewriter,
-      loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
+  Value zero = arith::ConstantOp::create(
+      rewriter, loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   if (outVecType.getShape().empty()) {
     Value scalarIn =
         vector::ExtractOp::create(rewriter, loc, in, ArrayRef<int64_t>{});
@@ -308,7 +310,7 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
     Value scalarTrunc =
         arith::TruncFOp::create(rewriter, loc, outElemType, scalarIn);
     Value result = vector::InsertOp::create(rewriter, loc, scalarTrunc, zero,
-                                                     ArrayRef<int64_t>{});
+                                            ArrayRef<int64_t>{});
     rewriter.replaceOp(op, result);
     return success();
   }
@@ -334,14 +336,14 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
         Value elemB = vector::ExtractOp::create(rewriter, loc, in, i + j + 1);
         asFloatB = castToF32(elemB, loc, rewriter);
       }
-      thisResult = amdgpu::PackedTrunc2xFp8Op::create(rewriter,
-          loc, truncResType, asFloatA, asFloatB, j / 2, thisResult);
+      thisResult = amdgpu::PackedTrunc2xFp8Op::create(
+          rewriter, loc, truncResType, asFloatA, asFloatB, j / 2, thisResult);
     }
     if (elemsThisOp < 4)
-      thisResult = vector::ExtractStridedSliceOp::create(rewriter,
-          loc, thisResult, 0, elemsThisOp, 1);
+      thisResult = vector::ExtractStridedSliceOp::create(
+          rewriter, loc, thisResult, 0, elemsThisOp, 1);
     result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
-                                                           result, i, 1);
+                                                  result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
@@ -407,10 +409,10 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
         ROCDL::CvtPkRtz::create(rewriter, loc, truncResType, elemA, elemB);
     // Place back the truncated result into the possibly larger vector. If we
     // are operating on a size 2 vector, these operations should be folded away
-    thisResult = vector::ExtractStridedSliceOp::create(rewriter,
-        loc, thisResult, 0, elemsThisOp, 1);
+    thisResult = vector::ExtractStridedSliceOp::create(
+        rewriter, loc, thisResult, 0, elemsThisOp, 1);
     result = vector::InsertStridedSliceOp::create(rewriter, loc, thisResult,
-                                                           result, i, 1);
+                                                  result, i, 1);
   }
 
   if (inVectorTy.getRank() != outVecType.getRank()) {
diff --git a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
index 40bf8f1cffb04..ba489436a1a4d 100644
--- a/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
+++ b/mlir/lib/Conversion/ArithToArmSME/ArithToArmSME.cpp
@@ -81,8 +81,8 @@ struct ConstantOpToArmSMELowering : public OpRewritePattern<arith::ConstantOp> {
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to write vector to tile
       // slice.
-      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
-          loc, tileType, constantOp1D, currentTile, tileSliceIndex);
+      auto nextTile = arm_sme::InsertTileSliceOp::create(
+          b, loc, tileType, constantOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
     auto forOp = mlir::arm_sme::createLoopOverTileSlices(
diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
index c5b3fdc536f72..59b3fe2e4eaed 100644
--- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
+++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
@@ -110,9 +110,9 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     emitc::CmpPredicate predicate;
     switch (op.getPredicate()) {
     case arith::CmpFPredicate::AlwaysFalse: {
-      auto constant = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), rewriter.getI1Type(),
-          rewriter.getBoolAttr(/*value=*/false));
+      auto constant =
+          emitc::ConstantOp::create(rewriter, op.getLoc(), rewriter.getI1Type(),
+                                    rewriter.getBoolAttr(/*value=*/false));
       rewriter.replaceOp(op, constant);
       return success();
     }
@@ -179,9 +179,9 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
       return success();
     }
     case arith::CmpFPredicate::AlwaysTrue: {
-      auto constant = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), rewriter.getI1Type(),
-          rewriter.getBoolAttr(/*value=*/true));
+      auto constant =
+          emitc::ConstantOp::create(rewriter, op.getLoc(), rewriter.getI1Type(),
+                                    rewriter.getBoolAttr(/*value=*/true));
       rewriter.replaceOp(op, constant);
       return success();
     }
@@ -190,7 +190,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     // Compare the values naively
     auto cmpResult =
         emitc::CmpOp::create(rewriter, op.getLoc(), op.getType(), predicate,
-                                      adaptor.getLhs(), adaptor.getRhs());
+                             adaptor.getLhs(), adaptor.getRhs());
 
     // Adjust the results for unordered/ordered semantics
     if (unordered) {
@@ -213,16 +213,16 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
   Value isNaN(ConversionPatternRewriter &rewriter, Location loc,
               Value operand) const {
     // A value is NaN exactly when it compares unequal to itself.
-    return emitc::CmpOp::create(rewriter,
-        loc, rewriter.getI1Type(), emitc::CmpPredicate::ne, operand, operand);
+    return emitc::CmpOp::create(rewriter, loc, rewriter.getI1Type(),
+                                emitc::CmpPredicate::ne, operand, operand);
   }
 
   /// Return a value that is true if \p operand is not NaN.
   Value isNotNaN(ConversionPatternRewriter &rewriter, Location loc,
                  Value operand) const {
     // A value is not NaN exactly when it compares equal to itself.
-    return emitc::CmpOp::create(rewriter,
-        loc, rewriter.getI1Type(), emitc::CmpPredicate::eq, operand, operand);
+    return emitc::CmpOp::create(rewriter, loc, rewriter.getI1Type(),
+                                emitc::CmpPredicate::eq, operand, operand);
   }
 
   /// Return a value that is true if the operands \p first and \p second are
@@ -232,7 +232,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     auto firstIsNaN = isNaN(rewriter, loc, first);
     auto secondIsNaN = isNaN(rewriter, loc, second);
     return emitc::LogicalOrOp::create(rewriter, loc, rewriter.getI1Type(),
-                                               firstIsNaN, secondIsNaN);
+                                      firstIsNaN, secondIsNaN);
   }
 
   /// Return a value that is true if the operands \p first and \p second are
@@ -242,7 +242,7 @@ class CmpFOpConversion : public OpConversionPattern<arith::CmpFOp> {
     auto firstIsNotNaN = isNotNaN(rewriter, loc, first);
     auto secondIsNotNaN = isNotNaN(rewriter, loc, second);
     return emitc::LogicalAndOp::create(rewriter, loc, rewriter.getI1Type(),
-                                                firstIsNotNaN, secondIsNotNaN);
+                                       firstIsNotNaN, secondIsNotNaN);
   }
 };
 
@@ -378,10 +378,10 @@ class CastConversion : public OpConversionPattern<ArithOp> {
       Type attrType = (emitc::isPointerWideType(operandType))
                           ? rewriter.getIndexType()
                           : operandType;
-      auto constOne = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), operandType, rewriter.getOneAttr(attrType));
-      auto oneAndOperand = emitc::BitwiseAndOp::create(rewriter,
-          op.getLoc(), operandType, adaptor.getIn(), constOne);
+      auto constOne = emitc::ConstantOp::create(
+          rewriter, op.getLoc(), operandType, rewriter.getOneAttr(attrType));
+      auto oneAndOperand = emitc::BitwiseAndOp::create(
+          rewriter, op.getLoc(), operandType, adaptor.getIn(), constOne);
       rewriter.replaceOpWithNewOp<emitc::CastOp>(op, opReturnType,
                                                  oneAndOperand);
       return success();
@@ -466,9 +466,8 @@ class BinaryUIOpConversion final : public OpConversionPattern<ArithOp> {
     Value lhsAdapted = adaptValueType(uiBinOp.getLhs(), rewriter, unsignedType);
     Value rhsAdapted = adaptValueType(uiBinOp.getRhs(), rewriter, unsignedType);
 
-    auto newDivOp =
-        EmitCOp::create(rewriter, uiBinOp.getLoc(), unsignedType,
-                                 ArrayRef<Value>{lhsAdapted, rhsAdapted});
+    auto newDivOp = EmitCOp::create(rewriter, uiBinOp.getLoc(), unsignedType,
+                                    ArrayRef<Value>{lhsAdapted, rhsAdapted});
     Value resultAdapted = adaptValueType(newDivOp, rewriter, newRetTy);
     rewriter.replaceOp(uiBinOp, resultAdapted);
     return success();
@@ -588,37 +587,39 @@ class ShiftOpConversion : public OpConversionPattern<ArithOp> {
     // Add a runtime check for overflow
     Value width;
     if (emitc::isPointerWideType(type)) {
-      Value eight = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), rhsType, rewriter.getIndexAttr(8));
-      emitc::CallOpaqueOp sizeOfCall = emitc::CallOpaqueOp::create(rewriter,
-          op.getLoc(), rhsType, "sizeof", ArrayRef<Value>{eight});
+      Value eight = emitc::ConstantOp::create(rewriter, op.getLoc(), rhsType,
+                                              rewriter.getIndexAttr(8));
+      emitc::CallOpaqueOp sizeOfCall = emitc::CallOpaqueOp::create(
+          rewriter, op.getLoc(), rhsType, "sizeof", ArrayRef<Value>{eight});
       width = emitc::MulOp::create(rewriter, op.getLoc(), rhsType, eight,
-                                            sizeOfCall.getResult(0));
+                                   sizeOfCall.getResult(0));
     } else {
-      width = emitc::ConstantOp::create(rewriter,
-          op.getLoc(), rhsType,
+      width = emitc::ConstantOp::create(
+          rewriter, op.getLoc(), rhsType,
           rewriter.getIntegerAttr(rhsType, type.getIntOrFloatBitWidth()));
     }
 
-    Value excessCheck = emitc::CmpOp::create(rewriter,
-        op.getLoc(), rewriter.getI1Type(), emitc::CmpPredicate::lt, rhs, width);
+    Value excessCheck =
+        emitc::CmpOp::create(rewriter, op.getLoc(), rewriter.getI1Type(),
+                             emitc::CmpPredicate::lt, rhs, width);
 
     // Any concrete value is a valid refinement of poison.
-    Value poison = emitc::ConstantOp::create(rewriter,
-        op.getLoc(), arithmeticType,
+    Value poison = emitc::ConstantOp::create(
+        rewriter, op.getLoc(), arithmeticType,
         (isa<IntegerType>(arithmeticType)
              ? rewriter.getIntegerAttr(arithmeticType, 0)
              : rewriter.getIndexAttr(0)));
 
-    emitc::ExpressionOp ternary = emitc::ExpressionOp::create(rewriter,
-        op.getLoc(), arithmeticType, /*do_not_inline=*/false);
+    emitc::ExpressionOp ternary = emitc::ExpressionOp::create(
+        rewriter, op.getLoc(), arithmeticType, /*do_not_inline=*/false);
     Block &bodyBlock = ternary.getBodyRegion().emplaceBlock();
     auto currentPoint = rewriter.getInsertionPoint();
     rewriter.setInsertionPointToStart(&bodyBlock);
     Value arithmeticResult =
         EmitCOp::create(rewriter, op.getLoc(), arithmeticType, lhs, rhs);
-    Value resultOrPoison = emitc::ConditionalOp::create(rewriter,
-        op.getLoc(), arithmeticType, excessCheck, arithmeticResult, poison);
+    Value resultOrPoison =
+        emitc::ConditionalOp::create(rewriter, op.getLoc(), arithmeticType,
+                                     excessCheck, arithmeticResult, poison);
     emitc::YieldOp::create(rewriter, op.getLoc(), resultOrPoison);
     rewriter.setInsertionPoint(op->getBlock(), currentPoint);
 
@@ -700,11 +701,12 @@ class FtoICastOpConversion : public OpConversionPattern<CastOp> {
                                   /*isSigned=*/false);
     }
 
-    Value result = emitc::CastOp::create(rewriter,
-        castOp.getLoc(), actualResultType, adaptor.getOperands());
+    Value result = emitc::CastOp::create(
+        rewriter, castOp.getLoc(), actualResultType, adaptor.getOperands());
 
     if (isa<arith::FPToUIOp>(castOp)) {
-      result = emitc::CastOp::create(rewriter, castOp.getLoc(), dstType, result);
+      result =
+          emitc::CastOp::create(rewriter, castOp.getLoc(), dstType, result);
     }
     rewriter.replaceOp(castOp, result);
 
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index f7d4963b5e293..18e857c81af8d 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -294,10 +294,10 @@ LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
         typename OpTy::Adaptor adaptor(operands);
         if (targetBits < sourceBits) {
           return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
-                                                adaptor.getIn());
+                                       adaptor.getIn());
         }
         return ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
-                                          adaptor.getIn());
+                                 adaptor.getIn());
       },
       rewriter);
 }
@@ -324,8 +324,8 @@ LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
     Type newOverflowType = typeConverter->convertType(overflowResultType);
     Type structType =
         LLVM::LLVMStructType::getLiteral(ctx, {sumResultType, newOverflowType});
-    Value addOverflow = LLVM::UAddWithOverflowOp::create(rewriter,
-        loc, structType, adaptor.getLhs(), adaptor.getRhs());
+    Value addOverflow = LLVM::UAddWithOverflowOp::create(
+        rewriter, loc, structType, adaptor.getLhs(), adaptor.getRhs());
     Value sumExtracted =
         LLVM::ExtractValueOp::create(rewriter, loc, addOverflow, 0);
     Value overflowExtracted =
@@ -435,8 +435,8 @@ CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return LLVM::ICmpOp::create(rewriter,
-            op.getLoc(), llvm1DVectorTy,
+        return LLVM::ICmpOp::create(
+            rewriter, op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::ICmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs());
       },
@@ -471,8 +471,8 @@ CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
       op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
       [&](Type llvm1DVectorTy, ValueRange operands) {
         OpAdaptor adaptor(operands);
-        return LLVM::FCmpOp::create(rewriter,
-            op.getLoc(), llvm1DVectorTy,
+        return LLVM::FCmpOp::create(
+            rewriter, op.getLoc(), llvm1DVectorTy,
             convertCmpPredicate<LLVM::FCmpPredicate>(op.getPredicate()),
             adaptor.getLhs(), adaptor.getRhs(), fmf);
       },
diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index c93cf13f831ef..d43e6816641cb 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -121,8 +121,8 @@ static Value getScalarOrVectorConstInt(Type type, uint64_t value,
   }
 
   if (auto intType = dyn_cast<IntegerType>(type))
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getIntegerAttr(type, value));
+    return spirv::ConstantOp::create(builder, loc, type,
+                                     builder.getIntegerAttr(type, value));
 
   return nullptr;
 }
@@ -429,7 +429,8 @@ static Value emulateSignedRemainder(Location loc, Value lhs, Value rhs,
   else
     isPositive = spirv::IEqualOp::create(builder, loc, rhs, rhsAbs);
   Value absNegate = spirv::SNegateOp::create(builder, loc, type, abs);
-  return spirv::SelectOp::create(builder, loc, type, isPositive, abs, absNegate);
+  return spirv::SelectOp::create(builder, loc, type, isPositive, abs,
+                                 absNegate);
 }
 
 /// Converts arith.remsi to GLSL SPIR-V ops.
@@ -601,13 +602,13 @@ struct ExtSII1Pattern final : public OpConversionPattern<arith::ExtSIOp> {
     Value allOnes;
     if (auto intTy = dyn_cast<IntegerType>(dstType)) {
       unsigned componentBitwidth = intTy.getWidth();
-      allOnes = spirv::ConstantOp::create(rewriter,
-          loc, intTy,
+      allOnes = spirv::ConstantOp::create(
+          rewriter, loc, intTy,
           rewriter.getIntegerAttr(intTy, APInt::getAllOnes(componentBitwidth)));
     } else if (auto vectorTy = dyn_cast<VectorType>(dstType)) {
       unsigned componentBitwidth = vectorTy.getElementTypeBitWidth();
-      allOnes = spirv::ConstantOp::create(rewriter,
-          loc, vectorTy,
+      allOnes = spirv::ConstantOp::create(
+          rewriter, loc, vectorTy,
           SplatElementsAttr::get(vectorTy,
                                  APInt::getAllOnes(componentBitwidth)));
     } else {
@@ -653,8 +654,8 @@ struct ExtSIPattern final : public OpConversionPattern<arith::ExtSIOp> {
       // First shift left to sequeeze out all leading bits beyond the original
       // bitwidth. Here we need to use the original source and result type's
       // bitwidth.
-      auto shiftLOp = spirv::ShiftLeftLogicalOp::create(rewriter,
-          op.getLoc(), dstType, adaptor.getIn(), shiftSize);
+      auto shiftLOp = spirv::ShiftLeftLogicalOp::create(
+          rewriter, op.getLoc(), dstType, adaptor.getIn(), shiftSize);
 
       // Then we perform arithmetic right shift to make sure we have the right
       // sign bits for negative values.
@@ -757,8 +758,8 @@ struct TruncII1Pattern final : public OpConversionPattern<arith::TruncIOp> {
     auto srcType = adaptor.getOperands().front().getType();
     // Check if (x & 1) == 1.
     Value mask = spirv::ConstantOp::getOne(srcType, loc, rewriter);
-    Value maskedSrc = spirv::BitwiseAndOp::create(rewriter,
-        loc, srcType, adaptor.getOperands()[0], mask);
+    Value maskedSrc = spirv::BitwiseAndOp::create(
+        rewriter, loc, srcType, adaptor.getOperands()[0], mask);
     Value isOne = spirv::IEqualOp::create(rewriter, loc, maskedSrc, mask);
 
     Value zero = spirv::ConstantOp::getZero(dstType, loc, rewriter);
@@ -1095,12 +1096,12 @@ class AddUIExtendedOpPattern final
     Type dstElemTy = adaptor.getLhs().getType();
     Location loc = op->getLoc();
     Value result = spirv::IAddCarryOp::create(rewriter, loc, adaptor.getLhs(),
-                                                       adaptor.getRhs());
+                                              adaptor.getRhs());
 
-    Value sumResult = spirv::CompositeExtractOp::create(rewriter,
-        loc, result, llvm::ArrayRef(0));
-    Value carryValue = spirv::CompositeExtractOp::create(rewriter,
-        loc, result, llvm::ArrayRef(1));
+    Value sumResult = spirv::CompositeExtractOp::create(rewriter, loc, result,
+                                                        llvm::ArrayRef(0));
+    Value carryValue = spirv::CompositeExtractOp::create(rewriter, loc, result,
+                                                         llvm::ArrayRef(1));
 
     // Convert the carry value to boolean.
     Value one = spirv::ConstantOp::getOne(dstElemTy, loc, rewriter);
@@ -1128,9 +1129,9 @@ class MulIExtendedOpPattern final : public OpConversionPattern<ArithMulOp> {
         SPIRVMulOp::create(rewriter, loc, adaptor.getLhs(), adaptor.getRhs());
 
     Value low = spirv::CompositeExtractOp::create(rewriter, loc, result,
-                                                           llvm::ArrayRef(0));
+                                                  llvm::ArrayRef(0));
     Value high = spirv::CompositeExtractOp::create(rewriter, loc, result,
-                                                            llvm::ArrayRef(1));
+                                                   llvm::ArrayRef(1));
 
     rewriter.replaceOp(op, {low, high});
     return success();
@@ -1194,9 +1195,9 @@ class MinimumMaximumFOpPattern final : public OpConversionPattern<Op> {
     Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
     Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
-                                                     adaptor.getLhs(), spirvOp);
+                                            adaptor.getLhs(), spirvOp);
     Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
-                                                     adaptor.getRhs(), select1);
+                                            adaptor.getRhs(), select1);
 
     rewriter.replaceOp(op, select2);
     return success();
@@ -1249,9 +1250,9 @@ class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
     Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
 
     Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
-                                                     adaptor.getRhs(), spirvOp);
+                                            adaptor.getRhs(), spirvOp);
     Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
-                                                     adaptor.getLhs(), select1);
+                                            adaptor.getLhs(), select1);
 
     rewriter.replaceOp(op, select2);
     return success();
diff --git a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
index 2295d661d44a5..1510b0b16b07d 100644
--- a/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
+++ b/mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
@@ -44,8 +44,8 @@ class Sdot2dLoweringPattern : public OpRewritePattern<Sdot2dOp> {
         vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, b2d);
     Value c1d =
         vector::ShapeCastOp::create(rewriter, loc, flattenedVectorType, c2d);
-    Value newOp = SdotOp::create(rewriter, loc, op.getRes().getType(), op.getA(),
-                                          b1d, c1d);
+    Value newOp = SdotOp::create(rewriter, loc, op.getRes().getType(),
+                                 op.getA(), b1d, c1d);
     rewriter.replaceOp(op, {newOp});
     return success();
   }
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index cf46e0c10df09..9bc3fa3473398 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -45,38 +45,38 @@ static Operation *createLoadTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return arm_sme::aarch64_sme_ld1b_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1b_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return arm_sme::aarch64_sme_ld1h_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1h_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return arm_sme::aarch64_sme_ld1w_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1w_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return arm_sme::aarch64_sme_ld1d_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1d_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return arm_sme::aarch64_sme_ld1q_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1q_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return arm_sme::aarch64_sme_ld1b_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1b_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return arm_sme::aarch64_sme_ld1h_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1h_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return arm_sme::aarch64_sme_ld1w_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1w_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return arm_sme::aarch64_sme_ld1d_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1d_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return arm_sme::aarch64_sme_ld1q_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_ld1q_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
       break;
     }
   }
@@ -91,38 +91,38 @@ static Operation *createStoreTileSliceIntrinsic(
   if (layout == arm_sme::TileSliceLayout::Horizontal) {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return arm_sme::aarch64_sme_st1b_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1b_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return arm_sme::aarch64_sme_st1h_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1h_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return arm_sme::aarch64_sme_st1w_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1w_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return arm_sme::aarch64_sme_st1d_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1d_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return arm_sme::aarch64_sme_st1q_horiz::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1q_horiz::create(rewriter, loc, maskOp, ptr,
+                                                     tileId, tileSliceI32);
     }
   } else {
     switch (type) {
     case arm_sme::ArmSMETileType::ZAB:
-      return arm_sme::aarch64_sme_st1b_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1b_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAH:
-      return arm_sme::aarch64_sme_st1h_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1h_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAS:
-      return arm_sme::aarch64_sme_st1w_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1w_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAD:
-      return arm_sme::aarch64_sme_st1d_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1d_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     case arm_sme::ArmSMETileType::ZAQ:
-      return arm_sme::aarch64_sme_st1q_vert::create(rewriter,
-          loc, maskOp, ptr, tileId, tileSliceI32);
+      return arm_sme::aarch64_sme_st1q_vert::create(rewriter, loc, maskOp, ptr,
+                                                    tileId, tileSliceI32);
     }
   }
   llvm_unreachable("unknown type in createStoreTileSliceIntrinsic");
@@ -154,8 +154,8 @@ createAllocaForTile(RewriterBase &rewriter, Location loc,
   auto minElementsOp =
       arith::ConstantIndexOp::create(rewriter, loc, minElements);
   auto vectorLen = arith::MulIOp::create(rewriter, loc, vscale, minElementsOp);
-  auto alloca = memref::AllocaOp::create(rewriter,
-      loc, memrefType, ValueRange{vectorLen, vectorLen});
+  auto alloca = memref::AllocaOp::create(rewriter, loc, memrefType,
+                                         ValueRange{vectorLen, vectorLen});
   return alloca;
 }
 
@@ -295,8 +295,8 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     auto descriptor =
         UnrealizedConversionCastOp::create(rewriter, loc, llvmType, tileMemory);
     auto zero = arith::ConstantIntOp::create(rewriter, loc, 0, /*width=*/64);
-    auto sliceIndexI64 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI64Type(), sliceIndex);
+    auto sliceIndexI64 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI64Type(), sliceIndex);
     return getStridedElementPtr(
         static_cast<ConversionPatternRewriter &>(rewriter), loc,
         llvm::cast<MemRefType>(tileMemory.getType()), descriptor.getResult(0),
@@ -309,20 +309,21 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
                      arm_sme::ArmSMETileType tileType, VectorType sliceType,
                      IntegerAttr tileId, Value sliceIndex) const {
     // Cast the slice index to an i32.
-    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI32Type(), sliceIndex);
+    auto sliceIndexI32 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI32Type(), sliceIndex);
     // Create an all-true predicate for the slice.
     auto predicateType = sliceType.clone(rewriter.getI1Type());
-    auto allTruePredicate = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(predicateType, true));
+    auto allTruePredicate = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(predicateType, true));
     // Create padding vector (never used due to all-true predicate).
     auto padVector = LLVM::PoisonOp::create(rewriter, loc, sliceType);
     // Get a pointer to the current slice.
     auto slicePtr =
         getInMemoryTileSlicePtr(rewriter, loc, tileAlloca, sliceIndex);
     // Read the value of the current slice from ZA.
-    auto currentTileSlice = arm_sme::aarch64_sme_read_horiz::create(rewriter,
-        loc, sliceType, padVector, allTruePredicate, tileId, sliceIndexI32);
+    auto currentTileSlice = arm_sme::aarch64_sme_read_horiz::create(
+        rewriter, loc, sliceType, padVector, allTruePredicate, tileId,
+        sliceIndexI32);
     // Load the new tile slice back from memory into ZA.
     createLoadTileSliceIntrinsic(
         rewriter, loc, tileType, arm_sme::TileSliceLayout::Horizontal,
@@ -330,7 +331,7 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     // Store the current tile slice to memory.
     auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
     vector::StoreOp::create(rewriter, loc, currentTileSlice, tileAlloca,
-                                     ValueRange{sliceIndex, zero});
+                            ValueRange{sliceIndex, zero});
   }
 
   /// Emits a full in-place swap of the contents of a tile in ZA and a
@@ -343,10 +344,12 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     auto minNumElts =
         arith::ConstantIndexOp::create(rewriter, loc, sliceType.getDimSize(0));
     auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
-    auto upperBound = arith::MulIOp::create(rewriter,
-        loc, minNumElts, vector::VectorScaleOp::create(rewriter, loc));
+    auto upperBound =
+        arith::MulIOp::create(rewriter, loc, minNumElts,
+                              vector::VectorScaleOp::create(rewriter, loc));
     auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
+    auto forOp =
+        scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     // Emit a swap for each tile slice.
     rewriter.setInsertionPointToStart(forOp.getBody());
     auto sliceIndex = forOp.getInductionVar();
@@ -479,8 +482,8 @@ struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
     //
     // This holds for all tile sizes.
     int32_t zeroMask = baseMaskForSize << int32_t(tileId.getInt());
-    arm_sme::aarch64_sme_zero::create(rewriter,
-        loc, rewriter.getI32IntegerAttr(zeroMask));
+    arm_sme::aarch64_sme_zero::create(rewriter, loc,
+                                      rewriter.getI32IntegerAttr(zeroMask));
 
     // Create a placeholder op to preserve dataflow.
     // Note: Place the `get_tile` op at the start of the block. This ensures
@@ -513,8 +516,8 @@ struct LoadTileSliceConversion
     auto tileSlice = loadTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
-        loc, rewriter.getI32Type(), tileSlice);
+    auto tileSliceI32 = arith::IndexCastUIOp::create(
+        rewriter, loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
     auto maskOp = loadTileSliceOp.getMask();
@@ -559,8 +562,8 @@ struct StoreTileSliceConversion
     auto tileSlice = storeTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice to i32 for intrinsic.
-    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
-        loc, rewriter.getI32Type(), tileSlice);
+    auto tileSliceI32 = arith::IndexCastUIOp::create(
+        rewriter, loc, rewriter.getI32Type(), tileSlice);
 
     auto maskOp = storeTileSliceOp.getMask();
 
@@ -595,12 +598,12 @@ struct InsertTileSliceConversion
     auto tileSlice = insertTileSliceOp.getTileSliceIndex();
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto tileSliceI32 = arith::IndexCastUIOp::create(rewriter,
-        loc, rewriter.getI32Type(), tileSlice);
+    auto tileSliceI32 = arith::IndexCastUIOp::create(
+        rewriter, loc, rewriter.getI32Type(), tileSlice);
 
     // Create all active predicate mask.
-    auto one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getI1Type(),
+    auto one = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getI1Type(),
         rewriter.getIntegerAttr(rewriter.getI1Type(), 1));
     auto predTy = VectorType::get(tileType.getShape()[0], rewriter.getI1Type(),
                                   /*scalableDims=*/{true});
@@ -609,14 +612,14 @@ struct InsertTileSliceConversion
     // Create 'arm_sme.intr.write.(horiz|vert)' to write vector to tile slice.
     switch (insertTileSliceOp.getLayout()) {
     case arm_sme::TileSliceLayout::Horizontal:
-      arm_sme::aarch64_sme_write_horiz::create(rewriter,
-          loc, tileId, tileSliceI32, allActiveMask,
-          insertTileSliceOp.getVector());
+      arm_sme::aarch64_sme_write_horiz::create(rewriter, loc, tileId,
+                                               tileSliceI32, allActiveMask,
+                                               insertTileSliceOp.getVector());
       break;
     case arm_sme::TileSliceLayout::Vertical:
-      arm_sme::aarch64_sme_write_vert::create(rewriter,
-          loc, tileId, tileSliceI32, allActiveMask,
-          insertTileSliceOp.getVector());
+      arm_sme::aarch64_sme_write_vert::create(rewriter, loc, tileId,
+                                              tileSliceI32, allActiveMask,
+                                              insertTileSliceOp.getVector());
       break;
     }
 
@@ -646,16 +649,16 @@ struct ExtractTileSliceConversion
 
     // Create an 'all true' predicate for the tile slice.
     auto predicateType = sliceType.cloneWith({}, rewriter.getI1Type());
-    auto allTruePredicate = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(predicateType, true));
+    auto allTruePredicate = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(predicateType, true));
 
     // Zero destination/fallback for tile slice extraction.
-    auto zeroVector = arith::ConstantOp::create(rewriter,
-        loc, sliceType, rewriter.getZeroAttr(sliceType));
+    auto zeroVector = arith::ConstantOp::create(
+        rewriter, loc, sliceType, rewriter.getZeroAttr(sliceType));
 
     // Cast tile slice from index to i32 for intrinsic.
-    auto sliceIndexI32 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI32Type(), sliceIndex);
+    auto sliceIndexI32 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI32Type(), sliceIndex);
 
     // Create 'arm_sme.intr.read.(horiz|vert)' to extract the tile slice.
     switch (extractTileSlice.getLayout()) {
@@ -754,16 +757,16 @@ struct OuterProductOpConversion
     if (!lhsMask || !rhsMask) {
       auto predTy =
           outerProductOp.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = arith::ConstantOp::create(rewriter,
-          loc, DenseElementsAttr::get(predTy, true));
+      Value allActiveMask = arith::ConstantOp::create(
+          rewriter, loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
     // Create 'arm_sme.intr.mopa' outer product intrinsic.
     arm_sme::aarch64_sme_mopa::create(rewriter, loc, tileId, lhsMask, rhsMask,
-                                               outerProductOp.getLhs(),
-                                               outerProductOp.getRhs());
+                                      outerProductOp.getLhs(),
+                                      outerProductOp.getRhs());
 
     // The outerproduct intrinsics have no result, replace
     // 'arm_sme.outerproduct' with the input tile to preserve dataflow.
@@ -801,14 +804,14 @@ struct OuterProductWideningOpConversion
     Value rhsMask = op.getRhsMask();
     if (!lhsMask || !rhsMask) {
       auto predTy = op.getLhsType().cloneWith({}, rewriter.getI1Type());
-      Value allActiveMask = arith::ConstantOp::create(rewriter,
-          loc, DenseElementsAttr::get(predTy, true));
+      Value allActiveMask = arith::ConstantOp::create(
+          rewriter, loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
-    OuterProductWideningIntrOp::create(rewriter,
-        loc, tileId, lhsMask, rhsMask, adaptor.getLhs(), adaptor.getRhs());
+    OuterProductWideningIntrOp::create(rewriter, loc, tileId, lhsMask, rhsMask,
+                                       adaptor.getLhs(), adaptor.getRhs());
 
     // The outerproduct intrinsics have no result, replace
     // 'arm_sme.outerproduct' with the input tile to preserve dataflow.
@@ -872,8 +875,8 @@ static void mergeConsecutiveTileZerosInBlock(Block *block) {
     if (zeroOpsToMerge.size() <= 1)
       return;
     IRRewriter rewriter(zeroOpsToMerge.front());
-    arm_sme::aarch64_sme_zero::create(rewriter,
-        zeroOpsToMerge.front().getLoc(),
+    arm_sme::aarch64_sme_zero::create(
+        rewriter, zeroOpsToMerge.front().getLoc(),
         rewriter.getI32IntegerAttr(mergedZeroMask));
     for (auto zeroOp : zeroOpsToMerge)
       rewriter.eraseOp(zeroOp);
diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 3591ed65f8d84..9a37b30c14813 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -59,8 +59,9 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   if (memrefIndices.size() != 2)
     return rewriter.notifyMatchFailure(loc, "invalid number of indices");
 
-  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
-      loc, arm_sme::getSMETileSliceMinNumElts(tileType.getElementType()));
+  auto minTileSlices = arith::ConstantIndexOp::create(
+      rewriter, loc,
+      arm_sme::getSMETileSliceMinNumElts(tileType.getElementType()));
   auto vscale =
       vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
   auto predicateType =
@@ -82,30 +83,30 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
     // The upper bound of the loop must be clamped at `numTileSlices` as
     // `vector.create_mask` allows operands to be greater than the size of a
     // dimension.
-    auto numRowI64 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI64Type(), maskDim0);
-    auto numTileSlicesI64 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI64Type(), numTileSlices);
+    auto numRowI64 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI64Type(), maskDim0);
+    auto numTileSlicesI64 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI64Type(), numTileSlices);
     auto upperBoundI64 =
         arith::MinSIOp::create(rewriter, loc, numRowI64, numTileSlicesI64);
-    upperBound = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getIndexType(), upperBoundI64);
+    upperBound = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getIndexType(), upperBoundI64);
 
     predicate =
         vector::CreateMaskOp::create(rewriter, loc, predicateType, maskDim1);
   } else {
     upperBound = numTileSlices;
     // No mask. Create an 'all true' predicate for the tile slice.
-    predicate = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(predicateType, true));
+    predicate = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(predicateType, true));
   }
 
   bool hasCarriedArgs = bool(initTile);
   auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
-                                           hasCarriedArgs ? ValueRange{initTile}
-                                                          : ValueRange{});
+  auto forOp =
+      scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
+                         hasCarriedArgs ? ValueRange{initTile} : ValueRange{});
 
   rewriter.setInsertionPointToStart(forOp.getBody());
   Value tileSliceIndex = forOp.getInductionVar();
@@ -207,9 +208,10 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
             Value currentTile) -> Value {
           // Create 'arm_sme.load_tile_slice' to load tile slice from memory
           // into tile.
-          return arm_sme::LoadTileSliceOp::create(rewriter,
-              loc, tileType, tileLoadOp.getBase(), predicate, currentTile,
-              memrefIndices, tileSliceIndex, tileLoadOp.getLayout());
+          return arm_sme::LoadTileSliceOp::create(
+              rewriter, loc, tileType, tileLoadOp.getBase(), predicate,
+              currentTile, memrefIndices, tileSliceIndex,
+              tileLoadOp.getLayout());
         });
 
     if (failed(forOp))
@@ -283,22 +285,22 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto numRows = createMaskOp.getOperands()[0];
     auto numCols = createMaskOp.getOperands()[1];
 
-    auto numColsI32 = arith::IndexCastUIOp::create(rewriter,
-        loc, rewriter.getI32Type(), numCols);
+    auto numColsI32 = arith::IndexCastUIOp::create(
+        rewriter, loc, rewriter.getI32Type(), numCols);
 
     auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     // Create a loop that loads each ZA tile slice from memory.
     auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
-        loc, arm_sme::getSMETileSliceMinNumElts(tileElementType));
+    auto minTileSlices = arith::ConstantIndexOp::create(
+        rewriter, loc, arm_sme::getSMETileSliceMinNumElts(tileElementType));
     auto vscale =
         vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
     auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto numTileSlices =
         arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
     auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices,
-                                             step, ValueRange{initTile});
+                                    step, ValueRange{initTile});
 
     rewriter.setInsertionPointToStart(forOp.getBody());
 
@@ -306,17 +308,18 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto currentTile = forOp.getRegionIterArg(0);
 
     // Combine masks.
-    auto rowIsActive = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ult, tileSliceIndex, numRows);
-    auto rowIsActiveI32 = arith::ExtSIOp::create(rewriter,
-        loc, rewriter.getI32Type(), rowIsActive);
-    auto mask = arith::AndIOp::create(rewriter, loc, rowIsActiveI32, numColsI32);
-    auto maskIndex =
-        arith::IndexCastOp::create(rewriter, loc, rewriter.getIndexType(), mask);
+    auto rowIsActive = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ult, tileSliceIndex, numRows);
+    auto rowIsActiveI32 = arith::ExtSIOp::create(
+        rewriter, loc, rewriter.getI32Type(), rowIsActive);
+    auto mask =
+        arith::AndIOp::create(rewriter, loc, rowIsActiveI32, numColsI32);
+    auto maskIndex = arith::IndexCastOp::create(rewriter, loc,
+                                                rewriter.getIndexType(), mask);
     auto predicateType =
         VectorType::get(tileType.getDimSize(1), rewriter.getI1Type(), true);
-    auto maskOp1D = vector::CreateMaskOp::create(rewriter,
-        loc, predicateType, maskIndex.getResult());
+    auto maskOp1D = vector::CreateMaskOp::create(rewriter, loc, predicateType,
+                                                 maskIndex.getResult());
 
     auto memrefIndices = getMemrefIndices(
         tileLoadOp.getIndices(), tileLoadOp.getMemRefType().getRank(),
@@ -326,14 +329,15 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
     auto pad1DOp = vector::SplatOp::create(rewriter, loc, tileSliceType, padOp);
 
-    auto loadSlice = vector::MaskedLoadOp::create(rewriter,
-        loc, tileSliceType, tileLoadOp.getBase(), memrefIndices, maskOp1D,
-        /*passthru=*/pad1DOp);
+    auto loadSlice = vector::MaskedLoadOp::create(rewriter, loc, tileSliceType,
+                                                  tileLoadOp.getBase(),
+                                                  memrefIndices, maskOp1D,
+                                                  /*passthru=*/pad1DOp);
 
     // Create 'arm_sme.insert_tile_slice' to insert slice into tile.
-    auto insertSlice = arm_sme::InsertTileSliceOp::create(rewriter,
-        loc, tileType, loadSlice->getResult(0), currentTile, tileSliceIndex,
-        tileLoadOp.getLayout());
+    auto insertSlice = arm_sme::InsertTileSliceOp::create(
+        rewriter, loc, tileType, loadSlice->getResult(0), currentTile,
+        tileSliceIndex, tileLoadOp.getLayout());
     scf::YieldOp::create(rewriter, loc, insertSlice.getResult());
 
     rewriter.setInsertionPointAfter(forOp);
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index e1d28f19ee236..79e1683b4e2cf 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -254,8 +254,8 @@ static void addResumeFunction(ModuleOp module) {
   auto voidTy = LLVM::LLVMVoidType::get(ctx);
   Type ptrType = AsyncAPI::opaquePointerType(ctx);
 
-  auto resumeOp = LLVM::LLVMFuncOp::create(moduleBuilder,
-      kResume, LLVM::LLVMFunctionType::get(voidTy, {ptrType}));
+  auto resumeOp = LLVM::LLVMFuncOp::create(
+      moduleBuilder, kResume, LLVM::LLVMFunctionType::get(voidTy, {ptrType}));
   resumeOp.setPrivate();
 
   auto *block = resumeOp.addEntryBlock(moduleBuilder);
@@ -282,7 +282,8 @@ class AsyncRuntimeTypeConverter : public TypeConverter {
     // in patterns for other dialects.
     auto addUnrealizedCast = [](OpBuilder &builder, Type type,
                                 ValueRange inputs, Location loc) -> Value {
-      auto cast = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+      auto cast =
+          UnrealizedConversionCastOp::create(builder, loc, type, inputs);
       return cast.getResult(0);
     };
 
@@ -382,7 +383,7 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
     // parameter.
     auto makeConstant = [&](uint64_t c) {
       return LLVM::ConstantOp::create(rewriter, op->getLoc(),
-                                               rewriter.getI64Type(), c);
+                                      rewriter.getI64Type(), c);
     };
     coroSize = LLVM::AddOp::create(rewriter, op->getLoc(), coroSize, coroAlign);
     coroSize =
@@ -397,8 +398,8 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
         rewriter, op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
     if (failed(allocFuncOp))
       return failure();
-    auto coroAlloc = LLVM::CallOp::create(rewriter,
-        loc, allocFuncOp.value(), ValueRange{coroAlign, coroSize});
+    auto coroAlloc = LLVM::CallOp::create(rewriter, loc, allocFuncOp.value(),
+                                          ValueRange{coroAlign, coroSize});
 
     // Begin a coroutine: @llvm.coro.begin.
     auto coroId = CoroBeginOpAdaptor(adaptor.getOperands()).getId();
@@ -455,15 +456,15 @@ class CoroEndOpConversion : public OpConversionPattern<CoroEndOp> {
   matchAndRewrite(CoroEndOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // We are not in the block that is part of the unwind sequence.
-    auto constFalse = LLVM::ConstantOp::create(rewriter,
-        op->getLoc(), rewriter.getI1Type(), rewriter.getBoolAttr(false));
+    auto constFalse =
+        LLVM::ConstantOp::create(rewriter, op->getLoc(), rewriter.getI1Type(),
+                                 rewriter.getBoolAttr(false));
     auto noneToken = LLVM::NoneTokenOp::create(rewriter, op->getLoc());
 
     // Mark the end of a coroutine: @llvm.coro.end.
     auto coroHdl = adaptor.getHandle();
-    LLVM::CoroEndOp::create(rewriter,
-        op->getLoc(), rewriter.getI1Type(),
-        ValueRange({coroHdl, constFalse, noneToken}));
+    LLVM::CoroEndOp::create(rewriter, op->getLoc(), rewriter.getI1Type(),
+                            ValueRange({coroHdl, constFalse, noneToken}));
     rewriter.eraseOp(op);
 
     return success();
@@ -534,13 +535,13 @@ class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
     auto loc = op->getLoc();
 
     // This is not a final suspension point.
-    auto constFalse = LLVM::ConstantOp::create(rewriter,
-        loc, rewriter.getI1Type(), rewriter.getBoolAttr(false));
+    auto constFalse = LLVM::ConstantOp::create(
+        rewriter, loc, rewriter.getI1Type(), rewriter.getBoolAttr(false));
 
     // Suspend a coroutine: @llvm.coro.suspend
     auto coroState = adaptor.getState();
-    auto coroSuspend = LLVM::CoroSuspendOp::create(rewriter,
-        loc, i8, ValueRange({coroState, constFalse}));
+    auto coroSuspend = LLVM::CoroSuspendOp::create(
+        rewriter, loc, i8, ValueRange({coroState, constFalse}));
 
     // Cast return code to i32.
 
@@ -605,7 +606,7 @@ class RuntimeCreateOpLowering : public ConvertOpToLLVMPattern<RuntimeCreateOp> {
         auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, storagePtrType);
         auto gep =
             LLVM::GEPOp::create(rewriter, loc, storagePtrType, storedType,
-                                         nullPtr, ArrayRef<LLVM::GEPArg>{1});
+                                nullPtr, ArrayRef<LLVM::GEPArg>{1});
         return LLVM::PtrToIntOp::create(rewriter, loc, i64, gep);
       };
 
@@ -740,7 +741,7 @@ class RuntimeAwaitOpLowering : public OpConversionPattern<RuntimeAwaitOp> {
             .Case<GroupType>([](Type) { return kAwaitGroup; });
 
     func::CallOp::create(rewriter, op->getLoc(), apiFuncName, TypeRange(),
-                                  adaptor.getOperands());
+                         adaptor.getOperands());
     rewriter.eraseOp(op);
 
     return success();
@@ -772,13 +773,12 @@ class RuntimeAwaitAndResumeOpLowering
 
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
-        op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
-        kResume);
+    auto resumePtr = LLVM::AddressOfOp::create(
+        rewriter, op->getLoc(),
+        AsyncAPI::opaquePointerType(rewriter.getContext()), kResume);
 
-    func::CallOp::create(rewriter,
-        op->getLoc(), apiFuncName, TypeRange(),
-        ValueRange({operand, handle, resumePtr.getRes()}));
+    func::CallOp::create(rewriter, op->getLoc(), apiFuncName, TypeRange(),
+                         ValueRange({operand, handle, resumePtr.getRes()}));
     rewriter.eraseOp(op);
 
     return success();
@@ -801,9 +801,9 @@ class RuntimeResumeOpLowering
                   ConversionPatternRewriter &rewriter) const override {
     // A pointer to coroutine resume intrinsic wrapper.
     addResumeFunction(op->getParentOfType<ModuleOp>());
-    auto resumePtr = LLVM::AddressOfOp::create(rewriter,
-        op->getLoc(), AsyncAPI::opaquePointerType(rewriter.getContext()),
-        kResume);
+    auto resumePtr = LLVM::AddressOfOp::create(
+        rewriter, op->getLoc(),
+        AsyncAPI::opaquePointerType(rewriter.getContext()), kResume);
 
     // Call async runtime API to execute a coroutine in the managed thread.
     auto coroHdl = adaptor.getHandle();
@@ -832,8 +832,8 @@ class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = func::CallOp::create(rewriter,
-        loc, kGetValueStorage, TypeRange(ptrType), storage);
+    auto storagePtr = func::CallOp::create(rewriter, loc, kGetValueStorage,
+                                           TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
     auto valueType = op.getValue().getType();
@@ -872,8 +872,8 @@ class RuntimeLoadOpLowering : public ConvertOpToLLVMPattern<RuntimeLoadOp> {
     // Get a pointer to the async value storage from the runtime.
     auto ptrType = AsyncAPI::opaquePointerType(rewriter.getContext());
     auto storage = adaptor.getStorage();
-    auto storagePtr = func::CallOp::create(rewriter,
-        loc, kGetValueStorage, TypeRange(ptrType), storage);
+    auto storagePtr = func::CallOp::create(rewriter, loc, kGetValueStorage,
+                                           TypeRange(ptrType), storage);
 
     // Cast from i8* to the LLVM pointer type.
     auto valueType = op.getResult().getType();
@@ -960,9 +960,9 @@ class RefCountingOpLowering : public OpConversionPattern<RefCountingOp> {
   LogicalResult
   matchAndRewrite(RefCountingOp op, typename RefCountingOp::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto count = arith::ConstantOp::create(rewriter,
-        op->getLoc(), rewriter.getI64Type(),
-        rewriter.getI64IntegerAttr(op.getCount()));
+    auto count =
+        arith::ConstantOp::create(rewriter, op->getLoc(), rewriter.getI64Type(),
+                                  rewriter.getI64IntegerAttr(op.getCount()));
 
     auto operand = adaptor.getOperand();
     rewriter.replaceOpWithNewOp<func::CallOp>(op, TypeRange(), apiFunctionName,
diff --git a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
index a63a1ceb00ebb..30a7170cf5c6a 100644
--- a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
+++ b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
@@ -103,7 +103,8 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
       }
 
       // Allocate a memref with identity layout.
-      alloc = memref::AllocOp::create(rewriter, loc, allocType, dynamicOperands);
+      alloc =
+          memref::AllocOp::create(rewriter, loc, allocType, dynamicOperands);
       // Cast the allocation to the specified type if needed.
       if (memrefType != allocType)
         alloc =
diff --git a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
index 1961e1004ba8e..14fbb9bf09545 100644
--- a/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
+++ b/mlir/lib/Conversion/ComplexCommon/DivisionConverter.cpp
@@ -23,35 +23,37 @@ void mlir::complex::convertDivToLLVMUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, LLVM::FastmathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = LLVM::FAddOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+  Value rhsSqNorm = LLVM::FAddOp::create(
+      rewriter, loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
       LLVM::FMulOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = LLVM::FAddOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+  Value realNumerator = LLVM::FAddOp::create(
+      rewriter, loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
       LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
 
-  Value imagNumerator = LLVM::FSubOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+  Value imagNumerator = LLVM::FSubOp::create(
+      rewriter, loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
       LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
-  *resultRe = LLVM::FDivOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
-  *resultIm = LLVM::FDivOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
+  *resultRe =
+      LLVM::FDivOp::create(rewriter, loc, realNumerator, rhsSqNorm, fmf);
+  *resultIm =
+      LLVM::FDivOp::create(rewriter, loc, imagNumerator, rhsSqNorm, fmf);
 }
 
 void mlir::complex::convertDivToStandardUsingAlgebraic(
     ConversionPatternRewriter &rewriter, Location loc, Value lhsRe, Value lhsIm,
     Value rhsRe, Value rhsIm, arith::FastMathFlagsAttr fmf, Value *resultRe,
     Value *resultIm) {
-  Value rhsSqNorm = arith::AddFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
+  Value rhsSqNorm = arith::AddFOp::create(
+      rewriter, loc, arith::MulFOp::create(rewriter, loc, rhsRe, rhsRe, fmf),
       arith::MulFOp::create(rewriter, loc, rhsIm, rhsIm, fmf), fmf);
 
-  Value realNumerator = arith::AddFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
+  Value realNumerator = arith::AddFOp::create(
+      rewriter, loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRe, fmf),
       arith::MulFOp::create(rewriter, loc, lhsIm, rhsIm, fmf), fmf);
-  Value imagNumerator = arith::SubFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+  Value imagNumerator = arith::SubFOp::create(
+      rewriter, loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
       arith::MulFOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
   *resultRe =
@@ -95,57 +97,59 @@ void mlir::complex::convertDivToLLVMUsingRangeReduction(
 
   Value rhsRealImagRatio =
       LLVM::FDivOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = LLVM::FAddOp::create(rewriter,
-      loc, rhsIm,
+  Value rhsRealImagDenom = LLVM::FAddOp::create(
+      rewriter, loc, rhsIm,
       LLVM::FMulOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = LLVM::FAddOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
-      lhsIm, fmf);
-  Value resultReal1 =
-      LLVM::FDivOp::create(rewriter, loc, realNumerator1, rhsRealImagDenom, fmf);
-  Value imagNumerator1 = LLVM::FSubOp::create(rewriter,
-      loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
-      lhsRe, fmf);
-  Value resultImag1 =
-      LLVM::FDivOp::create(rewriter, loc, imagNumerator1, rhsRealImagDenom, fmf);
+  Value realNumerator1 = LLVM::FAddOp::create(
+      rewriter, loc,
+      LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf), lhsIm,
+      fmf);
+  Value resultReal1 = LLVM::FDivOp::create(rewriter, loc, realNumerator1,
+                                           rhsRealImagDenom, fmf);
+  Value imagNumerator1 = LLVM::FSubOp::create(
+      rewriter, loc,
+      LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf), lhsRe,
+      fmf);
+  Value resultImag1 = LLVM::FDivOp::create(rewriter, loc, imagNumerator1,
+                                           rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
       LLVM::FDivOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = LLVM::FAddOp::create(rewriter,
-      loc, rhsRe,
+  Value rhsImagRealDenom = LLVM::FAddOp::create(
+      rewriter, loc, rhsRe,
       LLVM::FMulOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = LLVM::FAddOp::create(rewriter,
-      loc, lhsRe,
+  Value realNumerator2 = LLVM::FAddOp::create(
+      rewriter, loc, lhsRe,
       LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
-  Value resultReal2 =
-      LLVM::FDivOp::create(rewriter, loc, realNumerator2, rhsImagRealDenom, fmf);
-  Value imagNumerator2 = LLVM::FSubOp::create(rewriter,
-      loc, lhsIm,
+  Value resultReal2 = LLVM::FDivOp::create(rewriter, loc, realNumerator2,
+                                           rhsImagRealDenom, fmf);
+  Value imagNumerator2 = LLVM::FSubOp::create(
+      rewriter, loc, lhsIm,
       LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
-  Value resultImag2 =
-      LLVM::FDivOp::create(rewriter, loc, imagNumerator2, rhsImagRealDenom, fmf);
+  Value resultImag2 = LLVM::FDivOp::create(rewriter, loc, imagNumerator2,
+                                           rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = LLVM::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getZeroAttr(elementType));
+  Value zero = LLVM::ConstantOp::create(rewriter, loc, elementType,
+                                        rewriter.getZeroAttr(elementType));
   Value rhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, rhsRe, fmf);
-  Value rhsRealIsZero = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, zero);
+  Value rhsRealIsZero = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, zero);
   Value rhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, rhsIm, fmf);
-  Value rhsImagIsZero = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, zero);
-  Value lhsRealIsNotNaN =
-      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
-  Value lhsImagIsNotNaN =
-      LLVM::FCmpOp::create(rewriter, loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
+  Value rhsImagIsZero = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, zero);
+  Value lhsRealIsNotNaN = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::ord, lhsRe, zero);
+  Value lhsImagIsNotNaN = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::ord, lhsIm, zero);
   Value lhsContainsNotNaNValue =
       LLVM::OrOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = LLVM::AndOp::create(rewriter,
-      loc, lhsContainsNotNaNValue,
+  Value resultIsInfinity = LLVM::AndOp::create(
+      rewriter, loc, lhsContainsNotNaNValue,
       LLVM::AndOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = LLVM::ConstantOp::create(rewriter,
-      loc, elementType,
+  Value inf = LLVM::ConstantOp::create(
+      rewriter, loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfrhsReal =
@@ -156,119 +160,121 @@ void mlir::complex::convertDivToLLVMUsingRangeReduction(
       LLVM::FMulOp::create(rewriter, loc, infWithSignOfrhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::one, rhsRealAbs, inf);
-  Value rhsImagFinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::one, rhsImagAbs, inf);
+  Value rhsRealFinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::one, rhsRealAbs, inf);
+  Value rhsImagFinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::one, rhsImagAbs, inf);
   Value rhsFinite =
       LLVM::AndOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
   Value lhsRealAbs = LLVM::FAbsOp::create(rewriter, loc, lhsRe, fmf);
-  Value lhsRealInfinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, lhsRealAbs, inf);
+  Value lhsRealInfinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, lhsRealAbs, inf);
   Value lhsImagAbs = LLVM::FAbsOp::create(rewriter, loc, lhsIm, fmf);
-  Value lhsImagInfinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, lhsImagAbs, inf);
+  Value lhsImagInfinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, lhsImagAbs, inf);
   Value lhsInfinite =
       LLVM::OrOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
       LLVM::AndOp::create(rewriter, loc, lhsInfinite, rhsFinite);
-  Value one = LLVM::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
-      loc, LLVM::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
-      lhsRe);
-  Value lhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
-      loc, LLVM::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
-      lhsIm);
+  Value one = LLVM::ConstantOp::create(rewriter, loc, elementType,
+                                       rewriter.getFloatAttr(elementType, 1));
+  Value lhsRealIsInfWithSign = LLVM::CopySignOp::create(
+      rewriter, loc,
+      LLVM::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero), lhsRe);
+  Value lhsImagIsInfWithSign = LLVM::CopySignOp::create(
+      rewriter, loc,
+      LLVM::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero), lhsIm);
   Value lhsRealIsInfWithSignTimesrhsReal =
       LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesrhsImag =
       LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = LLVM::FMulOp::create(rewriter,
-      loc, inf,
+  Value resultReal3 = LLVM::FMulOp::create(
+      rewriter, loc, inf,
       LLVM::FAddOp::create(rewriter, loc, lhsRealIsInfWithSignTimesrhsReal,
-                                    lhsImagIsInfWithSignTimesrhsImag, fmf),
+                           lhsImagIsInfWithSignTimesrhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesrhsImag =
       LLVM::FMulOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesrhsReal =
       LLVM::FMulOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = LLVM::FMulOp::create(rewriter,
-      loc, inf,
+  Value resultImag3 = LLVM::FMulOp::create(
+      rewriter, loc, inf,
       LLVM::FSubOp::create(rewriter, loc, lhsImagIsInfWithSignTimesrhsReal,
-                                    lhsRealIsInfWithSignTimesrhsImag, fmf),
+                           lhsRealIsInfWithSignTimesrhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::one, lhsRealAbs, inf);
-  Value lhsImagFinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::one, lhsImagAbs, inf);
+  Value lhsRealFinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::one, lhsRealAbs, inf);
+  Value lhsImagFinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::one, lhsImagAbs, inf);
   Value lhsFinite =
       LLVM::AndOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, inf);
-  Value rhsImagInfinite = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, inf);
+  Value rhsRealInfinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, rhsRealAbs, inf);
+  Value rhsImagInfinite = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::oeq, rhsImagAbs, inf);
   Value rhsInfinite =
       LLVM::OrOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
       LLVM::AndOp::create(rewriter, loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = LLVM::CopySignOp::create(rewriter,
-      loc, LLVM::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
-      rhsRe);
-  Value rhsImagIsInfWithSign = LLVM::CopySignOp::create(rewriter,
-      loc, LLVM::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
-      rhsIm);
+  Value rhsRealIsInfWithSign = LLVM::CopySignOp::create(
+      rewriter, loc,
+      LLVM::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero), rhsRe);
+  Value rhsImagIsInfWithSign = LLVM::CopySignOp::create(
+      rewriter, loc,
+      LLVM::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero), rhsIm);
   Value rhsRealIsInfWithSignTimeslhsReal =
       LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsImag =
       LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = LLVM::FMulOp::create(rewriter,
-      loc, zero,
+  Value resultReal4 = LLVM::FMulOp::create(
+      rewriter, loc, zero,
       LLVM::FAddOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsReal,
-                                    rhsImagIsInfWithSignTimeslhsImag, fmf),
+                           rhsImagIsInfWithSignTimeslhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimeslhsImag =
       LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimeslhsReal =
       LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = LLVM::FMulOp::create(rewriter,
-      loc, zero,
+  Value resultImag4 = LLVM::FMulOp::create(
+      rewriter, loc, zero,
       LLVM::FSubOp::create(rewriter, loc, rhsRealIsInfWithSignTimeslhsImag,
-                                    rhsImagIsInfWithSignTimeslhsReal, fmf),
+                           rhsImagIsInfWithSignTimeslhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::olt, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = LLVM::SelectOp::create(rewriter,
-      loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = LLVM::SelectOp::create(rewriter,
-      loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = LLVM::SelectOp::create(rewriter,
-      loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = LLVM::SelectOp::create(rewriter,
-      loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = LLVM::SelectOp::create(rewriter,
-      loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = LLVM::SelectOp::create(rewriter,
-      loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = LLVM::SelectOp::create(rewriter,
-      loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = LLVM::SelectOp::create(rewriter,
-      loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
+  Value realAbsSmallerThanImagAbs = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::olt, rhsRealAbs, rhsImagAbs);
+  Value resultReal5 = LLVM::SelectOp::create(
+      rewriter, loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
+  Value resultImag5 = LLVM::SelectOp::create(
+      rewriter, loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
+  Value resultRealSpecialCase3 = LLVM::SelectOp::create(
+      rewriter, loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
+  Value resultImagSpecialCase3 = LLVM::SelectOp::create(
+      rewriter, loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
+  Value resultRealSpecialCase2 = LLVM::SelectOp::create(
+      rewriter, loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
+  Value resultImagSpecialCase2 = LLVM::SelectOp::create(
+      rewriter, loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
+  Value resultRealSpecialCase1 =
+      LLVM::SelectOp::create(rewriter, loc, resultIsInfinity,
+                             infinityResultReal, resultRealSpecialCase2);
+  Value resultImagSpecialCase1 =
+      LLVM::SelectOp::create(rewriter, loc, resultIsInfinity,
+                             infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::uno, resultReal5, zero);
-  Value resultImagIsNaN = LLVM::FCmpOp::create(rewriter,
-      loc, LLVM::FCmpPredicate::uno, resultImag5, zero);
+  Value resultRealIsNaN = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::uno, resultReal5, zero);
+  Value resultImagIsNaN = LLVM::FCmpOp::create(
+      rewriter, loc, LLVM::FCmpPredicate::uno, resultImag5, zero);
   Value resultIsNaN =
       LLVM::AndOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = LLVM::SelectOp::create(rewriter,
-      loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = LLVM::SelectOp::create(rewriter,
-      loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
+  *resultRe = LLVM::SelectOp::create(rewriter, loc, resultIsNaN,
+                                     resultRealSpecialCase1, resultReal5);
+  *resultIm = LLVM::SelectOp::create(rewriter, loc, resultIsNaN,
+                                     resultImagSpecialCase1, resultImag5);
 }
 
 void mlir::complex::convertDivToStandardUsingRangeReduction(
@@ -279,57 +285,59 @@ void mlir::complex::convertDivToStandardUsingRangeReduction(
 
   Value rhsRealImagRatio =
       arith::DivFOp::create(rewriter, loc, rhsRe, rhsIm, fmf);
-  Value rhsRealImagDenom = arith::AddFOp::create(rewriter,
-      loc, rhsIm,
+  Value rhsRealImagDenom = arith::AddFOp::create(
+      rewriter, loc, rhsIm,
       arith::MulFOp::create(rewriter, loc, rhsRealImagRatio, rhsRe, fmf), fmf);
-  Value realNumerator1 = arith::AddFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf),
-      lhsIm, fmf);
+  Value realNumerator1 = arith::AddFOp::create(
+      rewriter, loc,
+      arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealImagRatio, fmf), lhsIm,
+      fmf);
   Value resultReal1 = arith::DivFOp::create(rewriter, loc, realNumerator1,
-                                                     rhsRealImagDenom, fmf);
-  Value imagNumerator1 = arith::SubFOp::create(rewriter,
-      loc, arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf),
-      lhsRe, fmf);
+                                            rhsRealImagDenom, fmf);
+  Value imagNumerator1 = arith::SubFOp::create(
+      rewriter, loc,
+      arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealImagRatio, fmf), lhsRe,
+      fmf);
   Value resultImag1 = arith::DivFOp::create(rewriter, loc, imagNumerator1,
-                                                     rhsRealImagDenom, fmf);
+                                            rhsRealImagDenom, fmf);
 
   Value rhsImagRealRatio =
       arith::DivFOp::create(rewriter, loc, rhsIm, rhsRe, fmf);
-  Value rhsImagRealDenom = arith::AddFOp::create(rewriter,
-      loc, rhsRe,
+  Value rhsImagRealDenom = arith::AddFOp::create(
+      rewriter, loc, rhsRe,
       arith::MulFOp::create(rewriter, loc, rhsImagRealRatio, rhsIm, fmf), fmf);
-  Value realNumerator2 = arith::AddFOp::create(rewriter,
-      loc, lhsRe,
+  Value realNumerator2 = arith::AddFOp::create(
+      rewriter, loc, lhsRe,
       arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagRealRatio, fmf), fmf);
   Value resultReal2 = arith::DivFOp::create(rewriter, loc, realNumerator2,
-                                                     rhsImagRealDenom, fmf);
-  Value imagNumerator2 = arith::SubFOp::create(rewriter,
-      loc, lhsIm,
+                                            rhsImagRealDenom, fmf);
+  Value imagNumerator2 = arith::SubFOp::create(
+      rewriter, loc, lhsIm,
       arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagRealRatio, fmf), fmf);
   Value resultImag2 = arith::DivFOp::create(rewriter, loc, imagNumerator2,
-                                                     rhsImagRealDenom, fmf);
+                                            rhsImagRealDenom, fmf);
 
   // Consider corner cases.
   // Case 1. Zero denominator, numerator contains at most one NaN value.
-  Value zero = arith::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getZeroAttr(elementType));
+  Value zero = arith::ConstantOp::create(rewriter, loc, elementType,
+                                         rewriter.getZeroAttr(elementType));
   Value rhsRealAbs = math::AbsFOp::create(rewriter, loc, rhsRe, fmf);
-  Value rhsRealIsZero = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, rhsRealAbs, zero);
+  Value rhsRealIsZero = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, rhsRealAbs, zero);
   Value rhsImagAbs = math::AbsFOp::create(rewriter, loc, rhsIm, fmf);
-  Value rhsImagIsZero = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, rhsImagAbs, zero);
-  Value lhsRealIsNotNaN = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ORD, lhsRe, zero);
-  Value lhsImagIsNotNaN = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ORD, lhsIm, zero);
+  Value rhsImagIsZero = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, rhsImagAbs, zero);
+  Value lhsRealIsNotNaN = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ORD, lhsRe, zero);
+  Value lhsImagIsNotNaN = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ORD, lhsIm, zero);
   Value lhsContainsNotNaNValue =
       arith::OrIOp::create(rewriter, loc, lhsRealIsNotNaN, lhsImagIsNotNaN);
-  Value resultIsInfinity = arith::AndIOp::create(rewriter,
-      loc, lhsContainsNotNaNValue,
+  Value resultIsInfinity = arith::AndIOp::create(
+      rewriter, loc, lhsContainsNotNaNValue,
       arith::AndIOp::create(rewriter, loc, rhsRealIsZero, rhsImagIsZero));
-  Value inf = arith::ConstantOp::create(rewriter,
-      loc, elementType,
+  Value inf = arith::ConstantOp::create(
+      rewriter, loc, elementType,
       rewriter.getFloatAttr(elementType,
                             APFloat::getInf(elementType.getFloatSemantics())));
   Value infWithSignOfRhsReal =
@@ -340,117 +348,123 @@ void mlir::complex::convertDivToStandardUsingRangeReduction(
       arith::MulFOp::create(rewriter, loc, infWithSignOfRhsReal, lhsIm, fmf);
 
   // Case 2. Infinite numerator, finite denominator.
-  Value rhsRealFinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ONE, rhsRealAbs, inf);
-  Value rhsImagFinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ONE, rhsImagAbs, inf);
+  Value rhsRealFinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ONE, rhsRealAbs, inf);
+  Value rhsImagFinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ONE, rhsImagAbs, inf);
   Value rhsFinite =
       arith::AndIOp::create(rewriter, loc, rhsRealFinite, rhsImagFinite);
   Value lhsRealAbs = math::AbsFOp::create(rewriter, loc, lhsRe, fmf);
-  Value lhsRealInfinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, lhsRealAbs, inf);
+  Value lhsRealInfinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, lhsRealAbs, inf);
   Value lhsImagAbs = math::AbsFOp::create(rewriter, loc, lhsIm, fmf);
-  Value lhsImagInfinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, lhsImagAbs, inf);
+  Value lhsImagInfinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, lhsImagAbs, inf);
   Value lhsInfinite =
       arith::OrIOp::create(rewriter, loc, lhsRealInfinite, lhsImagInfinite);
   Value infNumFiniteDenom =
       arith::AndIOp::create(rewriter, loc, lhsInfinite, rhsFinite);
-  Value one = arith::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getFloatAttr(elementType, 1));
-  Value lhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
-      loc, arith::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
+  Value one = arith::ConstantOp::create(rewriter, loc, elementType,
+                                        rewriter.getFloatAttr(elementType, 1));
+  Value lhsRealIsInfWithSign = math::CopySignOp::create(
+      rewriter, loc,
+      arith::SelectOp::create(rewriter, loc, lhsRealInfinite, one, zero),
       lhsRe);
-  Value lhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
-      loc, arith::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
+  Value lhsImagIsInfWithSign = math::CopySignOp::create(
+      rewriter, loc,
+      arith::SelectOp::create(rewriter, loc, lhsImagInfinite, one, zero),
       lhsIm);
   Value lhsRealIsInfWithSignTimesRhsReal =
       arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsRe, fmf);
   Value lhsImagIsInfWithSignTimesRhsImag =
       arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsIm, fmf);
-  Value resultReal3 = arith::MulFOp::create(rewriter,
-      loc, inf,
+  Value resultReal3 = arith::MulFOp::create(
+      rewriter, loc, inf,
       arith::AddFOp::create(rewriter, loc, lhsRealIsInfWithSignTimesRhsReal,
-                                     lhsImagIsInfWithSignTimesRhsImag, fmf),
+                            lhsImagIsInfWithSignTimesRhsImag, fmf),
       fmf);
   Value lhsRealIsInfWithSignTimesRhsImag =
       arith::MulFOp::create(rewriter, loc, lhsRealIsInfWithSign, rhsIm, fmf);
   Value lhsImagIsInfWithSignTimesRhsReal =
       arith::MulFOp::create(rewriter, loc, lhsImagIsInfWithSign, rhsRe, fmf);
-  Value resultImag3 = arith::MulFOp::create(rewriter,
-      loc, inf,
+  Value resultImag3 = arith::MulFOp::create(
+      rewriter, loc, inf,
       arith::SubFOp::create(rewriter, loc, lhsImagIsInfWithSignTimesRhsReal,
-                                     lhsRealIsInfWithSignTimesRhsImag, fmf),
+                            lhsRealIsInfWithSignTimesRhsImag, fmf),
       fmf);
 
   // Case 3: Finite numerator, infinite denominator.
-  Value lhsRealFinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ONE, lhsRealAbs, inf);
-  Value lhsImagFinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::ONE, lhsImagAbs, inf);
+  Value lhsRealFinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ONE, lhsRealAbs, inf);
+  Value lhsImagFinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::ONE, lhsImagAbs, inf);
   Value lhsFinite =
       arith::AndIOp::create(rewriter, loc, lhsRealFinite, lhsImagFinite);
-  Value rhsRealInfinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, rhsRealAbs, inf);
-  Value rhsImagInfinite = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OEQ, rhsImagAbs, inf);
+  Value rhsRealInfinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, rhsRealAbs, inf);
+  Value rhsImagInfinite = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OEQ, rhsImagAbs, inf);
   Value rhsInfinite =
       arith::OrIOp::create(rewriter, loc, rhsRealInfinite, rhsImagInfinite);
   Value finiteNumInfiniteDenom =
       arith::AndIOp::create(rewriter, loc, lhsFinite, rhsInfinite);
-  Value rhsRealIsInfWithSign = math::CopySignOp::create(rewriter,
-      loc, arith::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
+  Value rhsRealIsInfWithSign = math::CopySignOp::create(
+      rewriter, loc,
+      arith::SelectOp::create(rewriter, loc, rhsRealInfinite, one, zero),
       rhsRe);
-  Value rhsImagIsInfWithSign = math::CopySignOp::create(rewriter,
-      loc, arith::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
+  Value rhsImagIsInfWithSign = math::CopySignOp::create(
+      rewriter, loc,
+      arith::SelectOp::create(rewriter, loc, rhsImagInfinite, one, zero),
       rhsIm);
   Value rhsRealIsInfWithSignTimesLhsReal =
       arith::MulFOp::create(rewriter, loc, lhsRe, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsImag =
       arith::MulFOp::create(rewriter, loc, lhsIm, rhsImagIsInfWithSign, fmf);
-  Value resultReal4 = arith::MulFOp::create(rewriter,
-      loc, zero,
+  Value resultReal4 = arith::MulFOp::create(
+      rewriter, loc, zero,
       arith::AddFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsReal,
-                                     rhsImagIsInfWithSignTimesLhsImag, fmf),
+                            rhsImagIsInfWithSignTimesLhsImag, fmf),
       fmf);
   Value rhsRealIsInfWithSignTimesLhsImag =
       arith::MulFOp::create(rewriter, loc, lhsIm, rhsRealIsInfWithSign, fmf);
   Value rhsImagIsInfWithSignTimesLhsReal =
       arith::MulFOp::create(rewriter, loc, lhsRe, rhsImagIsInfWithSign, fmf);
-  Value resultImag4 = arith::MulFOp::create(rewriter,
-      loc, zero,
+  Value resultImag4 = arith::MulFOp::create(
+      rewriter, loc, zero,
       arith::SubFOp::create(rewriter, loc, rhsRealIsInfWithSignTimesLhsImag,
-                                     rhsImagIsInfWithSignTimesLhsReal, fmf),
+                            rhsImagIsInfWithSignTimesLhsReal, fmf),
       fmf);
 
-  Value realAbsSmallerThanImagAbs = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OLT, rhsRealAbs, rhsImagAbs);
-  Value resultReal5 = arith::SelectOp::create(rewriter,
-      loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
-  Value resultImag5 = arith::SelectOp::create(rewriter,
-      loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
-  Value resultRealSpecialCase3 = arith::SelectOp::create(rewriter,
-      loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
-  Value resultImagSpecialCase3 = arith::SelectOp::create(rewriter,
-      loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
-  Value resultRealSpecialCase2 = arith::SelectOp::create(rewriter,
-      loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
-  Value resultImagSpecialCase2 = arith::SelectOp::create(rewriter,
-      loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
-  Value resultRealSpecialCase1 = arith::SelectOp::create(rewriter,
-      loc, resultIsInfinity, infinityResultReal, resultRealSpecialCase2);
-  Value resultImagSpecialCase1 = arith::SelectOp::create(rewriter,
-      loc, resultIsInfinity, infinityResultImag, resultImagSpecialCase2);
+  Value realAbsSmallerThanImagAbs = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OLT, rhsRealAbs, rhsImagAbs);
+  Value resultReal5 = arith::SelectOp::create(
+      rewriter, loc, realAbsSmallerThanImagAbs, resultReal1, resultReal2);
+  Value resultImag5 = arith::SelectOp::create(
+      rewriter, loc, realAbsSmallerThanImagAbs, resultImag1, resultImag2);
+  Value resultRealSpecialCase3 = arith::SelectOp::create(
+      rewriter, loc, finiteNumInfiniteDenom, resultReal4, resultReal5);
+  Value resultImagSpecialCase3 = arith::SelectOp::create(
+      rewriter, loc, finiteNumInfiniteDenom, resultImag4, resultImag5);
+  Value resultRealSpecialCase2 = arith::SelectOp::create(
+      rewriter, loc, infNumFiniteDenom, resultReal3, resultRealSpecialCase3);
+  Value resultImagSpecialCase2 = arith::SelectOp::create(
+      rewriter, loc, infNumFiniteDenom, resultImag3, resultImagSpecialCase3);
+  Value resultRealSpecialCase1 =
+      arith::SelectOp::create(rewriter, loc, resultIsInfinity,
+                              infinityResultReal, resultRealSpecialCase2);
+  Value resultImagSpecialCase1 =
+      arith::SelectOp::create(rewriter, loc, resultIsInfinity,
+                              infinityResultImag, resultImagSpecialCase2);
 
-  Value resultRealIsNaN = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::UNO, resultReal5, zero);
-  Value resultImagIsNaN = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::UNO, resultImag5, zero);
+  Value resultRealIsNaN = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::UNO, resultReal5, zero);
+  Value resultImagIsNaN = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::UNO, resultImag5, zero);
   Value resultIsNaN =
       arith::AndIOp::create(rewriter, loc, resultRealIsNaN, resultImagIsNaN);
 
-  *resultRe = arith::SelectOp::create(rewriter,
-      loc, resultIsNaN, resultRealSpecialCase1, resultReal5);
-  *resultIm = arith::SelectOp::create(rewriter,
-      loc, resultIsNaN, resultImagSpecialCase1, resultImag5);
+  *resultRe = arith::SelectOp::create(rewriter, loc, resultIsNaN,
+                                      resultRealSpecialCase1, resultReal5);
+  *resultIm = arith::SelectOp::create(rewriter, loc, resultIsNaN,
+                                      resultImagSpecialCase1, resultImag5);
 }
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index a06935c4387c6..86d02e6c6209f 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -79,8 +79,8 @@ struct AbsOpConversion : public ConvertOpToLLVMPattern<complex::AbsOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value sqNorm = LLVM::FAddOp::create(rewriter,
-        loc, LLVM::FMulOp::create(rewriter, loc, real, real, fmf),
+    Value sqNorm = LLVM::FAddOp::create(
+        rewriter, loc, LLVM::FMulOp::create(rewriter, loc, real, real, fmf),
         LLVM::FMulOp::create(rewriter, loc, imag, imag, fmf), fmf);
 
     rewriter.replaceOpWithNewOp<LLVM::SqrtOp>(op, sqNorm);
@@ -191,10 +191,10 @@ struct AddOpConversion : public ConvertOpToLLVMPattern<complex::AddOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value real =
-        LLVM::FAddOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
-    Value imag =
-        LLVM::FAddOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+    Value real = LLVM::FAddOp::create(rewriter, loc, arg.lhs.real(),
+                                      arg.rhs.real(), fmf);
+    Value imag = LLVM::FAddOp::create(rewriter, loc, arg.lhs.imag(),
+                                      arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
@@ -278,12 +278,12 @@ struct MulOpConversion : public ConvertOpToLLVMPattern<complex::MulOp> {
     Value lhsRe = arg.lhs.real();
     Value lhsIm = arg.lhs.imag();
 
-    Value real = LLVM::FSubOp::create(rewriter,
-        loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, lhsRe, fmf),
+    Value real = LLVM::FSubOp::create(
+        rewriter, loc, LLVM::FMulOp::create(rewriter, loc, rhsRe, lhsRe, fmf),
         LLVM::FMulOp::create(rewriter, loc, rhsIm, lhsIm, fmf), fmf);
 
-    Value imag = LLVM::FAddOp::create(rewriter,
-        loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
+    Value imag = LLVM::FAddOp::create(
+        rewriter, loc, LLVM::FMulOp::create(rewriter, loc, lhsIm, rhsRe, fmf),
         LLVM::FMulOp::create(rewriter, loc, lhsRe, rhsIm, fmf), fmf);
 
     result.setReal(rewriter, loc, real);
@@ -313,10 +313,10 @@ struct SubOpConversion : public ConvertOpToLLVMPattern<complex::SubOp> {
     LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
         op.getContext(),
         convertArithFastMathFlagsToLLVM(complexFMFAttr.getValue()));
-    Value real =
-        LLVM::FSubOp::create(rewriter, loc, arg.lhs.real(), arg.rhs.real(), fmf);
-    Value imag =
-        LLVM::FSubOp::create(rewriter, loc, arg.lhs.imag(), arg.rhs.imag(), fmf);
+    Value real = LLVM::FSubOp::create(rewriter, loc, arg.lhs.real(),
+                                      arg.rhs.real(), fmf);
+    Value imag = LLVM::FSubOp::create(rewriter, loc, arg.lhs.imag(),
+                                      arg.rhs.imag(), fmf);
     result.setReal(rewriter, loc, real);
     result.setImaginary(rewriter, loc, imag);
 
diff --git a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
index 4800f65c675e6..f83cac751ff05 100644
--- a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
+++ b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
@@ -85,7 +85,7 @@ LogicalResult ScalarOpToLibmCall<Op, TypeResolver>::matchAndRewrite(
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
     opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
-                                           opFunctionTy);
+                                  opFunctionTy);
     opFunc.setPrivate();
   }
   assert(isa<FunctionOpInterface>(SymbolTable::lookupSymbolIn(module, name)));
diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index d14e3888b118c..eeff8a93e7a72 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -32,7 +32,7 @@ enum class AbsFn { abs, sqrt, rsqrt };
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
                  ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
   Value one = arith::ConstantOp::create(b, real.getType(),
-                                          b.getFloatAttr(real.getType(), 1.0));
+                                        b.getFloatAttr(real.getType(), 1.0));
 
   Value absReal = math::AbsFOp::create(b, real, fmf);
   Value absImag = math::AbsFOp::create(b, imag, fmf);
@@ -55,11 +55,12 @@ Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
   }
 
   if (fn == AbsFn::sqrt) {
-    Value quarter = arith::ConstantOp::create(b,
-        real.getType(), b.getFloatAttr(real.getType(), 0.25));
+    Value quarter = arith::ConstantOp::create(
+        b, real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
     Value sqrt = math::SqrtOp::create(b, max, fmfWithNaNInf);
-    Value p025 = math::PowFOp::create(b, ratioSqPlusOne, quarter, fmfWithNaNInf);
+    Value p025 =
+        math::PowFOp::create(b, ratioSqPlusOne, quarter, fmfWithNaNInf);
     result = arith::MulFOp::create(b, sqrt, p025, fmfWithNaNInf);
   } else {
     Value sqrt = math::SqrtOp::create(b, ratioSqPlusOne, fmfWithNaNInf);
@@ -67,7 +68,7 @@ Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
   }
 
   Value isNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, result,
-                                        result, fmfWithNaNInf);
+                                      result, fmfWithNaNInf);
   return arith::SelectOp::create(b, isNaN, min, result);
 }
 
@@ -115,17 +116,17 @@ struct Atan2OpConversion : public OpConversionPattern<complex::Atan2Op> {
     Value zero =
         arith::ConstantOp::create(b, elementType, b.getZeroAttr(elementType));
     Value one = arith::ConstantOp::create(b, elementType,
-                                            b.getFloatAttr(elementType, 1));
+                                          b.getFloatAttr(elementType, 1));
     Value i = complex::CreateOp::create(b, type, zero, one);
     Value iTimesLhs = complex::MulOp::create(b, i, lhs, fmf);
     Value rhsPlusILhs = complex::AddOp::create(b, rhs, iTimesLhs, fmf);
 
-    Value divResult = complex::DivOp::create(b,
-        rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf);
+    Value divResult = complex::DivOp::create(
+        b, rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf);
     Value logResult = complex::LogOp::create(b, divResult, fmf);
 
-    Value negativeOne = arith::ConstantOp::create(b,
-        elementType, b.getFloatAttr(elementType, -1));
+    Value negativeOne = arith::ConstantOp::create(
+        b, elementType, b.getFloatAttr(elementType, -1));
     Value negativeI = complex::CreateOp::create(b, type, zero, negativeOne);
 
     rewriter.replaceOpWithNewOp<complex::MulOp>(op, negativeI, logResult, fmf);
@@ -146,10 +147,14 @@ struct ComparisonOpConversion : public OpConversionPattern<ComparisonOp> {
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getLhs().getType()).getElementType();
 
-    Value realLhs = complex::ReOp::create(rewriter, loc, type, adaptor.getLhs());
-    Value imagLhs = complex::ImOp::create(rewriter, loc, type, adaptor.getLhs());
-    Value realRhs = complex::ReOp::create(rewriter, loc, type, adaptor.getRhs());
-    Value imagRhs = complex::ImOp::create(rewriter, loc, type, adaptor.getRhs());
+    Value realLhs =
+        complex::ReOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value imagLhs =
+        complex::ImOp::create(rewriter, loc, type, adaptor.getLhs());
+    Value realRhs =
+        complex::ReOp::create(rewriter, loc, type, adaptor.getRhs());
+    Value imagRhs =
+        complex::ImOp::create(rewriter, loc, type, adaptor.getRhs());
     Value realComparison =
         arith::CmpFOp::create(rewriter, loc, p, realLhs, realRhs);
     Value imagComparison =
@@ -178,12 +183,12 @@ struct BinaryComplexOpConversion : public OpConversionPattern<BinaryComplexOp> {
 
     Value realLhs = complex::ReOp::create(b, elementType, adaptor.getLhs());
     Value realRhs = complex::ReOp::create(b, elementType, adaptor.getRhs());
-    Value resultReal = BinaryStandardOp::create(b, elementType, realLhs, realRhs,
-                                                  fmf.getValue());
+    Value resultReal = BinaryStandardOp::create(b, elementType, realLhs,
+                                                realRhs, fmf.getValue());
     Value imagLhs = complex::ImOp::create(b, elementType, adaptor.getLhs());
     Value imagRhs = complex::ImOp::create(b, elementType, adaptor.getRhs());
-    Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs, imagRhs,
-                                                  fmf.getValue());
+    Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs,
+                                                imagRhs, fmf.getValue());
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -212,8 +217,8 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
     // Trigonometric ops use a set of common building blocks to convert to real
     // ops. Here we create these building blocks and call into an op-specific
     // implementation in the subclass to combine them.
-    Value half = arith::ConstantOp::create(rewriter,
-        loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
+    Value half = arith::ConstantOp::create(
+        rewriter, loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
     Value exp = math::ExpOp::create(rewriter, loc, imag, fmf);
     Value scaledExp = arith::MulFOp::create(rewriter, loc, half, exp, fmf);
     Value reciprocalExp = arith::DivFOp::create(rewriter, loc, half, exp, fmf);
@@ -342,8 +347,8 @@ Value evaluatePolynomial(ImplicitLocOpBuilder &b, Value arg,
   Value poly =
       arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[0]));
   for (unsigned i = 1; i < coefficients.size(); ++i) {
-    poly = math::FmaOp::create(b,
-        poly, arg,
+    poly = math::FmaOp::create(
+        b, poly, arg,
         arith::ConstantOp::create(b, b.getFloatAttr(argType, coefficients[i])),
         fmf);
   }
@@ -378,13 +383,13 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
     Value cosm1Imag = emitCosm1(imag, fmf, b);
     Value cosImag = arith::AddFOp::create(b, cosm1Imag, one, fmf);
 
-    Value realResult = arith::AddFOp::create(b,
-        arith::MulFOp::create(b, expm1Real, cosImag, fmf), cosm1Imag, fmf);
+    Value realResult = arith::AddFOp::create(
+        b, arith::MulFOp::create(b, expm1Real, cosImag, fmf), cosm1Imag, fmf);
 
     Value imagIsZero = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag,
-                                               zero, fmf.getValue());
-    Value imagResult = arith::SelectOp::create(b,
-        imagIsZero, zero, arith::MulFOp::create(b, expReal, sinImag, fmf));
+                                             zero, fmf.getValue());
+    Value imagResult = arith::SelectOp::create(
+        b, imagIsZero, zero, arith::MulFOp::create(b, expReal, sinImag, fmf));
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, realResult,
                                                    imagResult);
@@ -414,13 +419,13 @@ struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> {
 
     auto forSmallArg =
         arith::AddFOp::create(b, arith::MulFOp::create(b, argPow4, poly, fmf),
-                                arith::MulFOp::create(b, negHalf, argPow2, fmf));
+                              arith::MulFOp::create(b, negHalf, argPow2, fmf));
 
     // (pi/4)^2 is approximately 0.61685
     Value piOver4Pow2 =
         arith::ConstantOp::create(b, b.getFloatAttr(argType, 0.61685));
     Value cond = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, argPow2,
-                                         piOver4Pow2, fmf.getValue());
+                                       piOver4Pow2, fmf.getValue());
     return arith::SelectOp::create(b, cond, forLargeArg, forSmallArg);
   }
 };
@@ -437,7 +442,7 @@ struct LogOpConversion : public OpConversionPattern<complex::LogOp> {
     mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
     Value abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(),
-                                         fmf.getValue());
+                                       fmf.getValue());
     Value resultReal = math::LogOp::create(b, elementType, abs, fmf.getValue());
     Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
     Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
@@ -464,9 +469,9 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     Value imag = complex::ImOp::create(b, adaptor.getComplex());
 
     Value half = arith::ConstantOp::create(b, elementType,
-                                             b.getFloatAttr(elementType, 0.5));
+                                           b.getFloatAttr(elementType, 0.5));
     Value one = arith::ConstantOp::create(b, elementType,
-                                            b.getFloatAttr(elementType, 1));
+                                          b.getFloatAttr(elementType, 1));
     Value realPlusOne = arith::AddFOp::create(b, real, one, fmf);
     Value absRealPlusOne = math::AbsFOp::create(b, realPlusOne, fmf);
     Value absImag = math::AbsFOp::create(b, imag, fmf);
@@ -475,7 +480,7 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     Value minAbs = arith::MinimumFOp::create(b, absRealPlusOne, absImag, fmf);
 
     Value useReal = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT,
-                                            realPlusOne, absImag, fmf);
+                                          realPlusOne, absImag, fmf);
     Value maxMinusOne = arith::SubFOp::create(b, maxAbs, one, fmf);
     Value maxAbsOfRealPlusOneAndImagMinusOne =
         arith::SelectOp::create(b, useReal, real, maxMinusOne);
@@ -484,14 +489,16 @@ struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> {
     Value minMaxRatio = arith::DivFOp::create(b, minAbs, maxAbs, fmfWithNaNInf);
     Value logOfMaxAbsOfRealPlusOneAndImag =
         math::Log1pOp::create(b, maxAbsOfRealPlusOneAndImagMinusOne, fmf);
-    Value logOfSqrtPart = math::Log1pOp::create(b,
-        arith::MulFOp::create(b, minMaxRatio, minMaxRatio, fmfWithNaNInf),
+    Value logOfSqrtPart = math::Log1pOp::create(
+        b, arith::MulFOp::create(b, minMaxRatio, minMaxRatio, fmfWithNaNInf),
         fmfWithNaNInf);
-    Value r = arith::AddFOp::create(b,
-        arith::MulFOp::create(b, half, logOfSqrtPart, fmfWithNaNInf),
+    Value r = arith::AddFOp::create(
+        b, arith::MulFOp::create(b, half, logOfSqrtPart, fmfWithNaNInf),
         logOfMaxAbsOfRealPlusOneAndImag, fmfWithNaNInf);
-    Value resultReal = arith::SelectOp::create(b,
-        arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, r, r, fmfWithNaNInf),
+    Value resultReal = arith::SelectOp::create(
+        b,
+        arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, r, r,
+                              fmfWithNaNInf),
         minAbs, r);
     Value resultImag = math::Atan2Op::create(b, imag, realPlusOne, fmf);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
@@ -520,13 +527,13 @@ struct MulOpConversion : public OpConversionPattern<complex::MulOp> {
     Value lhsImagTimesRhsImag =
         arith::MulFOp::create(b, lhsImag, rhsImag, fmfValue);
     Value real = arith::SubFOp::create(b, lhsRealTimesRhsReal,
-                                         lhsImagTimesRhsImag, fmfValue);
+                                       lhsImagTimesRhsImag, fmfValue);
     Value lhsImagTimesRhsReal =
         arith::MulFOp::create(b, lhsImag, rhsReal, fmfValue);
     Value lhsRealTimesRhsImag =
         arith::MulFOp::create(b, lhsReal, rhsImag, fmfValue);
     Value imag = arith::AddFOp::create(b, lhsImagTimesRhsReal,
-                                         lhsRealTimesRhsImag, fmfValue);
+                                       lhsRealTimesRhsImag, fmfValue);
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, real, imag);
     return success();
   }
@@ -594,12 +601,12 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
 
     auto cst = [&](APFloat v) {
       return arith::ConstantOp::create(b, elementType,
-                                         b.getFloatAttr(elementType, v));
+                                       b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
     Value half = arith::ConstantOp::create(b, elementType,
-                                             b.getFloatAttr(elementType, 0.5));
+                                           b.getFloatAttr(elementType, 0.5));
 
     Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
     Value imag = complex::ImOp::create(b, elementType, adaptor.getComplex());
@@ -614,8 +621,8 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
         arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, sin, zero, fmf);
 
     Value resultReal = arith::MulFOp::create(b, absSqrt, cos, fmf);
-    Value resultImag = arith::SelectOp::create(b,
-        sinIsZero, zero, arith::MulFOp::create(b, absSqrt, sin, fmf));
+    Value resultImag = arith::SelectOp::create(
+        b, sinIsZero, zero, arith::MulFOp::create(b, absSqrt, sin, fmf));
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
       Value inf = cst(APFloat::getInf(floatSemantics));
@@ -623,27 +630,28 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
       Value nan = cst(APFloat::getNaN(floatSemantics));
       Value absImag = math::AbsFOp::create(b, elementType, imag, fmf);
 
-      Value absImagIsInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
-      Value absImagIsNotInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::ONE, absImag, inf, fmf);
+      Value absImagIsInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                                 absImag, inf, fmf);
+      Value absImagIsNotInf = arith::CmpFOp::create(
+          b, arith::CmpFPredicate::ONE, absImag, inf, fmf);
       Value realIsInf =
           arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, inf, fmf);
-      Value realIsNegInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, real, negInf, fmf);
+      Value realIsNegInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                                 real, negInf, fmf);
 
-      resultReal = arith::SelectOp::create(b,
-          arith::AndIOp::create(b, realIsNegInf, absImagIsNotInf), zero,
+      resultReal = arith::SelectOp::create(
+          b, arith::AndIOp::create(b, realIsNegInf, absImagIsNotInf), zero,
           resultReal);
-      resultReal = arith::SelectOp::create(b,
-          arith::OrIOp::create(b, absImagIsInf, realIsInf), inf, resultReal);
+      resultReal = arith::SelectOp::create(
+          b, arith::OrIOp::create(b, absImagIsInf, realIsInf), inf, resultReal);
 
       Value imagSignInf = math::CopySignOp::create(b, inf, imag, fmf);
-      resultImag = arith::SelectOp::create(b,
+      resultImag = arith::SelectOp::create(
+          b,
           arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, absSqrt, absSqrt),
           nan, resultImag);
-      resultImag = arith::SelectOp::create(b,
-          arith::OrIOp::create(b, absImagIsInf, realIsNegInf), imagSignInf,
+      resultImag = arith::SelectOp::create(
+          b, arith::OrIOp::create(b, absImagIsInf, realIsNegInf), imagSignInf,
           resultImag);
     }
 
@@ -678,7 +686,8 @@ struct SignOpConversion : public OpConversionPattern<complex::SignOp> {
     Value imagIsZero =
         arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero);
     Value isZero = arith::AndIOp::create(b, realIsZero, imagIsZero);
-    auto abs = complex::AbsOp::create(b, elementType, adaptor.getComplex(), fmf);
+    auto abs =
+        complex::AbsOp::create(b, elementType, adaptor.getComplex(), fmf);
     Value realSign = arith::DivFOp::create(b, real, abs, fmf);
     Value imagSign = arith::DivFOp::create(b, imag, abs, fmf);
     Value sign = complex::CreateOp::create(b, type, realSign, imagSign);
@@ -706,8 +715,8 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
         complex::ReOp::create(b, loc, elementType, adaptor.getComplex());
     Value imag =
         complex::ImOp::create(b, loc, elementType, adaptor.getComplex());
-    Value negOne = arith::ConstantOp::create(b,
-        elementType, b.getFloatAttr(elementType, -1.0));
+    Value negOne = arith::ConstantOp::create(b, elementType,
+                                             b.getFloatAttr(elementType, -1.0));
 
     if constexpr (std::is_same_v<Op, complex::TanOp>) {
       // tan(x+yi) = -i*tanh(-y + xi)
@@ -717,60 +726,60 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
 
     auto cst = [&](APFloat v) {
       return arith::ConstantOp::create(b, elementType,
-                                         b.getFloatAttr(elementType, v));
+                                       b.getFloatAttr(elementType, v));
     };
     Value inf = cst(APFloat::getInf(floatSemantics));
     Value four = arith::ConstantOp::create(b, elementType,
-                                             b.getFloatAttr(elementType, 4.0));
+                                           b.getFloatAttr(elementType, 4.0));
     Value twoReal = arith::AddFOp::create(b, real, real, fmf);
     Value negTwoReal = arith::MulFOp::create(b, negOne, twoReal, fmf);
 
     Value expTwoRealMinusOne = math::ExpM1Op::create(b, twoReal, fmf);
     Value expNegTwoRealMinusOne = math::ExpM1Op::create(b, negTwoReal, fmf);
-    Value realNum =
-        arith::SubFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+    Value realNum = arith::SubFOp::create(b, expTwoRealMinusOne,
+                                          expNegTwoRealMinusOne, fmf);
 
     Value cosImag = math::CosOp::create(b, imag, fmf);
     Value cosImagSq = arith::MulFOp::create(b, cosImag, cosImag, fmf);
     Value twoCosTwoImagPlusOne = arith::MulFOp::create(b, cosImagSq, four, fmf);
     Value sinImag = math::SinOp::create(b, imag, fmf);
 
-    Value imagNum = arith::MulFOp::create(b,
-        four, arith::MulFOp::create(b, cosImag, sinImag, fmf), fmf);
+    Value imagNum = arith::MulFOp::create(
+        b, four, arith::MulFOp::create(b, cosImag, sinImag, fmf), fmf);
 
-    Value expSumMinusTwo =
-        arith::AddFOp::create(b, expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+    Value expSumMinusTwo = arith::AddFOp::create(b, expTwoRealMinusOne,
+                                                 expNegTwoRealMinusOne, fmf);
     Value denom =
         arith::AddFOp::create(b, expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
 
     Value isInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
-                                          expSumMinusTwo, inf, fmf);
+                                        expSumMinusTwo, inf, fmf);
     Value realLimit = math::CopySignOp::create(b, negOne, real, fmf);
 
-    Value resultReal = arith::SelectOp::create(b,
-        isInf, realLimit, arith::DivFOp::create(b, realNum, denom, fmf));
+    Value resultReal = arith::SelectOp::create(
+        b, isInf, realLimit, arith::DivFOp::create(b, realNum, denom, fmf));
     Value resultImag = arith::DivFOp::create(b, imagNum, denom, fmf);
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
       Value absReal = math::AbsFOp::create(b, real, fmf);
-      Value zero = arith::ConstantOp::create(b,
-          elementType, b.getFloatAttr(elementType, 0.0));
+      Value zero = arith::ConstantOp::create(b, elementType,
+                                             b.getFloatAttr(elementType, 0.0));
       Value nan = cst(APFloat::getNaN(floatSemantics));
 
-      Value absRealIsInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value absRealIsInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                                 absReal, inf, fmf);
       Value imagIsZero =
           arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, imag, zero, fmf);
-      Value absRealIsNotInf = arith::XOrIOp::create(b,
-          absRealIsInf, arith::ConstantIntOp::create(b, true, /*width=*/1));
+      Value absRealIsNotInf = arith::XOrIOp::create(
+          b, absRealIsInf, arith::ConstantIntOp::create(b, true, /*width=*/1));
 
       Value imagNumIsNaN = arith::CmpFOp::create(b, arith::CmpFPredicate::UNO,
-                                                   imagNum, imagNum, fmf);
+                                                 imagNum, imagNum, fmf);
       Value resultRealIsNaN =
           arith::AndIOp::create(b, imagNumIsNaN, absRealIsNotInf);
-      Value resultImagIsZero = arith::OrIOp::create(b,
-          imagIsZero, arith::AndIOp::create(b, absRealIsInf, imagNumIsNaN));
+      Value resultImagIsZero = arith::OrIOp::create(
+          b, imagIsZero, arith::AndIOp::create(b, absRealIsInf, imagNumIsNaN));
 
       resultReal = arith::SelectOp::create(b, resultRealIsNaN, nan, resultReal);
       resultImag =
@@ -837,14 +846,14 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
   Value cosQ = math::CosOp::create(builder, q, fmf);
   Value sinQ = math::SinOp::create(builder, q, fmf);
 
-  Value inf = arith::ConstantOp::create(builder,
-      elementType,
+  Value inf = arith::ConstantOp::create(
+      builder, elementType,
       builder.getFloatAttr(elementType,
                            APFloat::getInf(elementType.getFloatSemantics())));
-  Value zero = arith::ConstantOp::create(builder,
-      elementType, builder.getFloatAttr(elementType, 0.0));
-  Value one = arith::ConstantOp::create(builder,
-      elementType, builder.getFloatAttr(elementType, 1.0));
+  Value zero = arith::ConstantOp::create(
+      builder, elementType, builder.getFloatAttr(elementType, 0.0));
+  Value one = arith::ConstantOp::create(builder, elementType,
+                                        builder.getFloatAttr(elementType, 1.0));
   Value complexOne = complex::CreateOp::create(builder, type, one, zero);
   Value complexZero = complex::CreateOp::create(builder, type, zero, zero);
   Value complexInf = complex::CreateOp::create(builder, type, inf, zero);
@@ -870,9 +879,10 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
       arith::SelectOp::create(builder, cEqZero, complexOne, complexZero);
   Value coeffCosSin =
       complex::CreateOp::create(builder, type, coeffCosQ, coeffSinQ);
-  Value cutoff0 = arith::SelectOp::create(builder,
-      arith::AndIOp::create(builder,
-          arith::AndIOp::create(builder, absEqZero, dEqZero), zeroLeC),
+  Value cutoff0 = arith::SelectOp::create(
+      builder,
+      arith::AndIOp::create(
+          builder, arith::AndIOp::create(builder, absEqZero, dEqZero), zeroLeC),
       complexOneOrZero, coeffCosSin);
 
   // Case 1:
@@ -885,7 +895,8 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
 
   // Case 2:
   // 1^(c + d*i) = 1 + 0*i
-  Value lhsEqOne = arith::AndIOp::create(builder,
+  Value lhsEqOne = arith::AndIOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, one, fmf),
       bEqZero);
   Value cutoff2 =
@@ -893,22 +904,25 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
 
   // Case 3:
   // inf^(c + 0*i) = inf + 0*i, c > 0
-  Value lhsEqInf = arith::AndIOp::create(builder,
+  Value lhsEqInf = arith::AndIOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, a, inf, fmf),
       bEqZero);
-  Value rhsGt0 = arith::AndIOp::create(builder,
-      dEqZero,
+  Value rhsGt0 = arith::AndIOp::create(
+      builder, dEqZero,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, c, zero, fmf));
-  Value cutoff3 = arith::SelectOp::create(builder,
-      arith::AndIOp::create(builder, lhsEqInf, rhsGt0), complexInf, cutoff2);
+  Value cutoff3 = arith::SelectOp::create(
+      builder, arith::AndIOp::create(builder, lhsEqInf, rhsGt0), complexInf,
+      cutoff2);
 
   // Case 4:
   // inf^(c + 0*i) = 0 + 0*i, c < 0
-  Value rhsLt0 = arith::AndIOp::create(builder,
-      dEqZero,
+  Value rhsLt0 = arith::AndIOp::create(
+      builder, dEqZero,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, c, zero, fmf));
-  Value cutoff4 = arith::SelectOp::create(builder,
-      arith::AndIOp::create(builder, lhsEqInf, rhsLt0), complexZero, cutoff3);
+  Value cutoff4 = arith::SelectOp::create(
+      builder, arith::AndIOp::create(builder, lhsEqInf, rhsLt0), complexZero,
+      cutoff3);
 
   return cutoff4;
 }
@@ -946,13 +960,13 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
 
     auto cst = [&](APFloat v) {
       return arith::ConstantOp::create(b, elementType,
-                                         b.getFloatAttr(elementType, v));
+                                       b.getFloatAttr(elementType, v));
     };
     const auto &floatSemantics = elementType.getFloatSemantics();
     Value zero = cst(APFloat::getZero(floatSemantics));
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value negHalf = arith::ConstantOp::create(b,
-        elementType, b.getFloatAttr(elementType, -0.5));
+    Value negHalf = arith::ConstantOp::create(
+        b, elementType, b.getFloatAttr(elementType, -0.5));
     Value nan = cst(APFloat::getNaN(floatSemantics));
 
     Value real = complex::ReOp::create(b, elementType, adaptor.getComplex());
@@ -968,8 +982,8 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
 
     if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
                                             arith::FastMathFlags::ninf)) {
-      Value negOne = arith::ConstantOp::create(b,
-          elementType, b.getFloatAttr(elementType, -1));
+      Value negOne = arith::ConstantOp::create(b, elementType,
+                                               b.getFloatAttr(elementType, -1));
 
       Value realSignedZero = math::CopySignOp::create(b, zero, real, fmf);
       Value imagSignedZero = math::CopySignOp::create(b, zero, imag, fmf);
@@ -979,12 +993,12 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
       Value absReal = math::AbsFOp::create(b, real, fmf);
       Value absImag = math::AbsFOp::create(b, imag, fmf);
 
-      Value absImagIsInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+      Value absImagIsInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                                 absImag, inf, fmf);
       Value realIsNan =
           arith::CmpFOp::create(b, arith::CmpFPredicate::UNO, real, real, fmf);
-      Value realIsInf =
-          arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value realIsInf = arith::CmpFOp::create(b, arith::CmpFPredicate::OEQ,
+                                              absReal, inf, fmf);
       Value inIsNanInf = arith::AndIOp::create(b, absImagIsInf, realIsNan);
 
       Value resultIsZero = arith::OrIOp::create(b, inIsNanInf, realIsInf);
@@ -992,7 +1006,7 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
       resultReal =
           arith::SelectOp::create(b, resultIsZero, realSignedZero, resultReal);
       resultImag = arith::SelectOp::create(b, resultIsZero, negImagSignedZero,
-                                             resultImag);
+                                           resultImag);
     }
 
     Value isRealZero =
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index 5c3d26874faed..ff6d369176393 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -74,7 +74,7 @@ struct AssertOpLowering : public ConvertOpToLLVMPattern<cf::AssertOp> {
         rewriter.setInsertionPointToStart(module.getBody());
         auto abortFuncTy = LLVM::LLVMFunctionType::get(getVoidType(), {});
         abortFunc = LLVM::LLVMFuncOp::create(rewriter, rewriter.getUnknownLoc(),
-                                                      "abort", abortFuncTy);
+                                             "abort", abortFuncTy);
       }
       LLVM::CallOp::create(rewriter, loc, abortFunc, ValueRange());
       LLVM::UnreachableOp::create(rewriter, loc);
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index aa468c29536c7..c8311eb5a6433 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -34,7 +34,7 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
   if (auto condBrOp = dyn_cast<cf::CondBranchOp>(controlFlowCondOp)) {
     assert(regions.size() == 2);
     auto ifOp = scf::IfOp::create(builder, controlFlowCondOp->getLoc(),
-                                          resultTypes, condBrOp.getCondition());
+                                  resultTypes, condBrOp.getCondition());
     ifOp.getThenRegion().takeBody(regions[0]);
     ifOp.getElseRegion().takeBody(regions[1]);
     return ifOp.getOperation();
@@ -43,8 +43,8 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
   if (auto switchOp = dyn_cast<cf::SwitchOp>(controlFlowCondOp)) {
     // `getCFGSwitchValue` returns an i32 that we need to convert to index
     // fist.
-    auto cast = arith::IndexCastUIOp::create(builder,
-        controlFlowCondOp->getLoc(), builder.getIndexType(),
+    auto cast = arith::IndexCastUIOp::create(
+        builder, controlFlowCondOp->getLoc(), builder.getIndexType(),
         switchOp.getFlag());
     SmallVector<int64_t> cases;
     if (auto caseValues = switchOp.getCaseValues())
@@ -55,8 +55,9 @@ ControlFlowToSCFTransformation::createStructuredBranchRegionOp(
 
     assert(regions.size() == cases.size() + 1);
 
-    auto indexSwitchOp = scf::IndexSwitchOp::create(builder,
-        controlFlowCondOp->getLoc(), resultTypes, cast, cases, cases.size());
+    auto indexSwitchOp =
+        scf::IndexSwitchOp::create(builder, controlFlowCondOp->getLoc(),
+                                   resultTypes, cast, cases, cases.size());
 
     indexSwitchOp.getDefaultRegion().takeBody(regions[0]);
     for (auto &&[targetRegion, sourceRegion] :
@@ -84,16 +85,17 @@ ControlFlowToSCFTransformation::createStructuredDoWhileLoopOp(
     OpBuilder &builder, Operation *replacedOp, ValueRange loopVariablesInit,
     Value condition, ValueRange loopVariablesNextIter, Region &&loopBody) {
   Location loc = replacedOp->getLoc();
-  auto whileOp = scf::WhileOp::create(builder, loc, loopVariablesInit.getTypes(),
-                                              loopVariablesInit);
+  auto whileOp = scf::WhileOp::create(
+      builder, loc, loopVariablesInit.getTypes(), loopVariablesInit);
 
   whileOp.getBefore().takeBody(loopBody);
 
   builder.setInsertionPointToEnd(&whileOp.getBefore().back());
   // `getCFGSwitchValue` returns a i32. We therefore need to truncate the
   // condition to i1 first. It is guaranteed to be either 0 or 1 already.
-  scf::ConditionOp::create(builder,
-      loc, arith::TruncIOp::create(builder, loc, builder.getI1Type(), condition),
+  scf::ConditionOp::create(
+      builder, loc,
+      arith::TruncIOp::create(builder, loc, builder.getI1Type(), condition),
       loopVariablesNextIter);
 
   Block *afterBlock = builder.createBlock(&whileOp.getAfter());
@@ -109,7 +111,7 @@ Value ControlFlowToSCFTransformation::getCFGSwitchValue(Location loc,
                                                         OpBuilder &builder,
                                                         unsigned int value) {
   return arith::ConstantOp::create(builder, loc,
-                                           builder.getI32IntegerAttr(value));
+                                   builder.getI32IntegerAttr(value));
 }
 
 void ControlFlowToSCFTransformation::createCFGSwitchOp(
@@ -118,8 +120,8 @@ void ControlFlowToSCFTransformation::createCFGSwitchOp(
     ArrayRef<ValueRange> caseArguments, Block *defaultDest,
     ValueRange defaultArgs) {
   cf::SwitchOp::create(builder, loc, flag, defaultDest, defaultArgs,
-                               llvm::to_vector_of<int32_t>(caseValues),
-                               caseDestinations, caseArguments);
+                       llvm::to_vector_of<int32_t>(caseValues),
+                       caseDestinations, caseArguments);
 }
 
 Value ControlFlowToSCFTransformation::getUndefValue(Location loc,
diff --git a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
index 20167f168185a..197caeb4ffbfa 100644
--- a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
+++ b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
@@ -99,8 +99,8 @@ class FuncOpConversion final : public OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted `emitc.func` op.
-    emitc::FuncOp newFuncOp = emitc::FuncOp::create(rewriter,
-        funcOp.getLoc(), funcOp.getName(),
+    emitc::FuncOp newFuncOp = emitc::FuncOp::create(
+        rewriter, funcOp.getLoc(), funcOp.getName(),
         FunctionType::get(rewriter.getContext(),
                           signatureConverter.getConvertedTypes(),
                           resultType ? TypeRange(resultType) : TypeRange()));
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 51dbe6cbb7de3..67bb1c14c99a2 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -115,8 +115,8 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   SmallVector<NamedAttribute> attributes;
   filterFuncAttributes(funcOp, attributes);
 
-  auto wrapperFuncOp = LLVM::LLVMFuncOp::create(rewriter,
-      loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
+  auto wrapperFuncOp = LLVM::LLVMFuncOp::create(
+      rewriter, loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperFuncType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
   propagateArgResAttrs(rewriter, !!resultStructType, funcOp, wrapperFuncOp);
@@ -129,14 +129,14 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   for (auto [index, argType] : llvm::enumerate(type.getInputs())) {
     Value arg = wrapperFuncOp.getArgument(index + argOffset);
     if (auto memrefType = dyn_cast<MemRefType>(argType)) {
-      Value loaded = LLVM::LoadOp::create(rewriter,
-          loc, typeConverter.convertType(memrefType), arg);
+      Value loaded = LLVM::LoadOp::create(
+          rewriter, loc, typeConverter.convertType(memrefType), arg);
       MemRefDescriptor::unpack(rewriter, loc, loaded, memrefType, args);
       continue;
     }
     if (isa<UnrankedMemRefType>(argType)) {
-      Value loaded = LLVM::LoadOp::create(rewriter,
-          loc, typeConverter.convertType(argType), arg);
+      Value loaded = LLVM::LoadOp::create(
+          rewriter, loc, typeConverter.convertType(argType), arg);
       UnrankedMemRefDescriptor::unpack(rewriter, loc, loaded, args);
       continue;
     }
@@ -148,7 +148,7 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
 
   if (resultStructType) {
     LLVM::StoreOp::create(rewriter, loc, call.getResult(),
-                                   wrapperFuncOp.getArgument(0));
+                          wrapperFuncOp.getArgument(0));
     LLVM::ReturnOp::create(rewriter, loc, ValueRange{});
   } else {
     LLVM::ReturnOp::create(rewriter, loc, call.getResults());
@@ -182,8 +182,8 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   filterFuncAttributes(funcOp, attributes);
 
   // Create the auxiliary function.
-  auto wrapperFunc = LLVM::LLVMFuncOp::create(builder,
-      loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
+  auto wrapperFunc = LLVM::LLVMFuncOp::create(
+      builder, loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
       wrapperType, LLVM::Linkage::External, /*dsoLocal=*/false,
       /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attributes);
   propagateArgResAttrs(builder, !!resultStructType, funcOp, wrapperFunc);
@@ -201,8 +201,8 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   if (resultStructType) {
     // Allocate the struct on the stack and pass the pointer.
     Type resultType = cast<LLVM::LLVMFunctionType>(wrapperType).getParamType(0);
-    Value one = LLVM::ConstantOp::create(builder,
-        loc, typeConverter.convertType(builder.getIndexType()),
+    Value one = LLVM::ConstantOp::create(
+        builder, loc, typeConverter.convertType(builder.getIndexType()),
         builder.getIntegerAttr(builder.getIndexType(), 1));
     Value result =
         LLVM::AllocaOp::create(builder, loc, resultType, resultStructType, one);
@@ -229,11 +229,11 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
                     wrapperArgsRange.take_front(numToDrop));
 
       auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext());
-      Value one = LLVM::ConstantOp::create(builder,
-          loc, typeConverter.convertType(builder.getIndexType()),
+      Value one = LLVM::ConstantOp::create(
+          builder, loc, typeConverter.convertType(builder.getIndexType()),
           builder.getIntegerAttr(builder.getIndexType(), 1));
-      Value allocated = LLVM::AllocaOp::create(builder,
-          loc, ptrTy, packed.getType(), one, /*alignment=*/0);
+      Value allocated = LLVM::AllocaOp::create(
+          builder, loc, ptrTy, packed.getType(), one, /*alignment=*/0);
       LLVM::StoreOp::create(builder, loc, packed, allocated);
       arg = allocated;
     } else {
@@ -357,8 +357,8 @@ FailureOr<LLVM::LLVMFuncOp> mlir::convertFuncOpToLLVMFuncOp(
     symbolTable.remove(funcOp);
   }
 
-  auto newFuncOp = LLVM::LLVMFuncOp::create(rewriter,
-      funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
+  auto newFuncOp = LLVM::LLVMFuncOp::create(
+      rewriter, funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
       /*dsoLocal=*/false, /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr,
       attributes);
 
@@ -556,9 +556,10 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
     auto promoted = this->getTypeConverter()->promoteOperands(
         callOp.getLoc(), /*opOperands=*/callOp->getOperands(),
         adaptor.getOperands(), rewriter, useBarePtrCallConv);
-    auto newOp = LLVM::CallOp::create(rewriter,
-        callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
-        promoted, callOp->getAttrs());
+    auto newOp = LLVM::CallOp::create(rewriter, callOp.getLoc(),
+                                      packedResult ? TypeRange(packedResult)
+                                                   : TypeRange(),
+                                      promoted, callOp->getAttrs());
 
     newOp.getProperties().operandSegmentSizes = {
         static_cast<int32_t>(promoted.size()), 0};
@@ -573,8 +574,8 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
       // Extract individual results from the structure and return them as list.
       results.reserve(numResults);
       for (unsigned i = 0; i < numResults; ++i) {
-        results.push_back(LLVM::ExtractValueOp::create(rewriter,
-            callOp.getLoc(), newOp->getResult(0), i));
+        results.push_back(LLVM::ExtractValueOp::create(
+            rewriter, callOp.getLoc(), newOp->getResult(0), i));
       }
     }
 
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index f5d5460412da2..1037e296c8128 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -69,8 +69,8 @@ mlir::getOrCreateStringConstant(OpBuilder &b, Location loc,
   b.setInsertionPointToStart(moduleOp.getBody());
   SmallString<16> name = getUniqueSymbolName(moduleOp, namePrefix);
   return LLVM::GlobalOp::create(b, loc, globalType,
-                                  /*isConstant=*/true, LLVM::Linkage::Internal,
-                                  name, attr, alignment, addrSpace);
+                                /*isConstant=*/true, LLVM::Linkage::Internal,
+                                name, attr, alignment, addrSpace);
 }
 
 LogicalResult
@@ -151,8 +151,8 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
               gpuFuncOp.getWorkgroupAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      auto globalOp = LLVM::GlobalOp::create(rewriter,
-          gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
+      auto globalOp = LLVM::GlobalOp::create(
+          rewriter, gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
           LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment,
           workgroupAddrSpace);
       workgroupBuffers.push_back(globalOp);
@@ -220,8 +220,8 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
   LLVM::CConv callingConvention = gpuFuncOp.isKernel()
                                       ? kernelCallingConvention
                                       : nonKernelCallingConvention;
-  auto llvmFuncOp = LLVM::LLVMFuncOp::create(rewriter,
-      gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
+  auto llvmFuncOp = LLVM::LLVMFuncOp::create(
+      rewriter, gpuFuncOp.getLoc(), gpuFuncOp.getName(), funcType,
       LLVM::Linkage::External, /*dsoLocal=*/false, callingConvention,
       /*comdat=*/nullptr, attributes);
 
@@ -266,11 +266,11 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       for (const auto [idx, global] : llvm::enumerate(workgroupBuffers)) {
         auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext(),
                                                   global.getAddrSpace());
-        Value address = LLVM::AddressOfOp::create(rewriter,
-            loc, ptrType, global.getSymNameAttr());
+        Value address = LLVM::AddressOfOp::create(rewriter, loc, ptrType,
+                                                  global.getSymNameAttr());
         Value memory =
             LLVM::GEPOp::create(rewriter, loc, ptrType, global.getType(),
-                                         address, ArrayRef<LLVM::GEPArg>{0, 0});
+                                address, ArrayRef<LLVM::GEPArg>{0, 0});
 
         // Build a memref descriptor pointing to the buffer to plug with the
         // existing memref infrastructure. This may use more registers than
@@ -298,15 +298,16 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
       Type elementType = typeConverter->convertType(type.getElementType());
       auto ptrType =
           LLVM::LLVMPointerType::get(rewriter.getContext(), allocaAddrSpace);
-      Value numElements = LLVM::ConstantOp::create(rewriter,
-          gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
+      Value numElements = LLVM::ConstantOp::create(
+          rewriter, gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
       uint64_t alignment = 0;
       if (auto alignAttr =
               dyn_cast_or_null<IntegerAttr>(gpuFuncOp.getPrivateAttributionAttr(
                   idx, LLVM::LLVMDialect::getAlignAttrName())))
         alignment = alignAttr.getInt();
-      Value allocated = LLVM::AllocaOp::create(rewriter,
-          gpuFuncOp.getLoc(), ptrType, elementType, numElements, alignment);
+      Value allocated =
+          LLVM::AllocaOp::create(rewriter, gpuFuncOp.getLoc(), ptrType,
+                                 elementType, numElements, alignment);
       Value descr = MemRefDescriptor::fromStaticShape(
           rewriter, loc, *getTypeConverter(), type, allocated);
       signatureConversion.remapInput(
@@ -419,7 +420,8 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
 
   /// Start the printf hostcall
   Value zeroI64 = LLVM::ConstantOp::create(rewriter, loc, llvmI64, 0);
-  auto printfBeginCall = LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
+  auto printfBeginCall =
+      LLVM::CallOp::create(rewriter, loc, ocklBegin, zeroI64);
   Value printfDesc = printfBeginCall.getResult();
 
   // Create the global op or find an existing one.
@@ -427,21 +429,21 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
       rewriter, loc, moduleOp, llvmI8, "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element and pass it to printf()
-  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
-      loc,
+  Value globalPtr = LLVM::AddressOfOp::create(
+      rewriter, loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
       LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
-                                   globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
-  Value stringLen = LLVM::ConstantOp::create(rewriter,
-      loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
+                          globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+  Value stringLen = LLVM::ConstantOp::create(
+      rewriter, loc, llvmI64, cast<StringAttr>(global.getValueAttr()).size());
 
   Value oneI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 1);
   Value zeroI32 = LLVM::ConstantOp::create(rewriter, loc, llvmI32, 0);
 
-  auto appendFormatCall = LLVM::CallOp::create(rewriter,
-      loc, ocklAppendStringN,
+  auto appendFormatCall = LLVM::CallOp::create(
+      rewriter, loc, ocklAppendStringN,
       ValueRange{printfDesc, stringStart, stringLen,
                  adaptor.getArgs().empty() ? oneI32 : zeroI32});
   printfDesc = appendFormatCall.getResult();
@@ -461,8 +463,9 @@ LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
       Value arg = adaptor.getArgs()[i];
       if (auto floatType = dyn_cast<FloatType>(arg.getType())) {
         if (!floatType.isF64())
-          arg = LLVM::FPExtOp::create(rewriter,
-              loc, typeConverter->convertType(rewriter.getF64Type()), arg);
+          arg = LLVM::FPExtOp::create(
+              rewriter, loc, typeConverter->convertType(rewriter.getF64Type()),
+              arg);
         arg = LLVM::BitcastOp::create(rewriter, loc, llvmI64, arg);
       }
       if (arg.getType().getIntOrFloatBitWidth() != 64)
@@ -510,13 +513,13 @@ LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
       /*alignment=*/0, addressSpace);
 
   // Get a pointer to the format string's first element
-  Value globalPtr = LLVM::AddressOfOp::create(rewriter,
-      loc,
+  Value globalPtr = LLVM::AddressOfOp::create(
+      rewriter, loc,
       LLVM::LLVMPointerType::get(rewriter.getContext(), global.getAddrSpace()),
       global.getSymNameAttr());
   Value stringStart =
       LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
-                                   globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+                          globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
 
   // Construct arguments and function call
   auto argsRange = adaptor.getArgs();
@@ -562,7 +565,7 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   Value globalPtr = LLVM::AddressOfOp::create(rewriter, loc, global);
   Value stringStart =
       LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
-                                   globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+                          globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
   SmallVector<Type> types;
   SmallVector<Value> args;
   // Promote and pack the arguments into a stack allocation.
@@ -580,13 +583,13 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   Type structType =
       LLVM::LLVMStructType::getLiteral(gpuPrintfOp.getContext(), types);
   Value one = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
-                                                rewriter.getIndexAttr(1));
+                                       rewriter.getIndexAttr(1));
   Value tempAlloc =
       LLVM::AllocaOp::create(rewriter, loc, ptrType, structType, one,
-                                      /*alignment=*/0);
+                             /*alignment=*/0);
   for (auto [index, arg] : llvm::enumerate(args)) {
-    Value ptr = LLVM::GEPOp::create(rewriter,
-        loc, ptrType, structType, tempAlloc,
+    Value ptr = LLVM::GEPOp::create(
+        rewriter, loc, ptrType, structType, tempAlloc,
         ArrayRef<LLVM::GEPArg>{0, static_cast<int32_t>(index)});
     LLVM::StoreOp::create(rewriter, loc, arg, ptr);
   }
@@ -622,8 +625,8 @@ static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands,
     auto scalarOperands = llvm::map_to_vector(operands, extractElement);
     Operation *scalarOp =
         rewriter.create(loc, name, scalarOperands, elementType, op->getAttrs());
-    result = LLVM::InsertElementOp::create(rewriter,
-        loc, result, scalarOp->getResult(0), index);
+    result = LLVM::InsertElementOp::create(rewriter, loc, result,
+                                           scalarOp->getResult(0), index);
   }
   return result;
 }
@@ -705,10 +708,10 @@ LLVM::GlobalOp getDynamicSharedMemorySymbol(
   auto zeroSizedArrayType = LLVM::LLVMArrayType::get(
       typeConverter->convertType(memrefType.getElementType()), 0);
 
-  return LLVM::GlobalOp::create(rewriter,
-      op->getLoc(), zeroSizedArrayType, /*isConstant=*/false,
-      LLVM::Linkage::Internal, symName, /*value=*/Attribute(), alignmentByte,
-      addressSpace.value());
+  return LLVM::GlobalOp::create(rewriter, op->getLoc(), zeroSizedArrayType,
+                                /*isConstant=*/false, LLVM::Linkage::Internal,
+                                symName, /*value=*/Attribute(), alignmentByte,
+                                addressSpace.value());
 }
 
 LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
@@ -738,7 +741,7 @@ LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
   // Step 4. Generate GEP using offsets
   SmallVector<LLVM::GEPArg> gepArgs = {0};
   Value shmemPtr = LLVM::GEPOp::create(rewriter, loc, baseType, elementType,
-                                                basePtr, gepArgs);
+                                       basePtr, gepArgs);
   // Step 5. Create a memref descriptor
   SmallVector<Value> shape, strides;
   Value sizeBytes;
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 5e8254a0b6035..63eb6c58e87a7 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -79,8 +79,8 @@ class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern<OpTy> {
     uint64_t rank = type.getRank();
     Value numElements = desc.size(rewriter, loc, /*pos=*/0);
     for (unsigned i = 1; i < rank; i++)
-      numElements = LLVM::MulOp::create(rewriter,
-          loc, numElements, desc.size(rewriter, loc, /*pos=*/i));
+      numElements = LLVM::MulOp::create(rewriter, loc, numElements,
+                                        desc.size(rewriter, loc, /*pos=*/i));
     return numElements;
   }
 
@@ -785,8 +785,8 @@ LogicalResult ConvertAllocOpToGpuRuntimeCallPattern::matchAndRewrite(
                      ? nullPtr
                      : adaptor.getAsyncDependencies().front();
 
-  auto isHostShared = mlir::LLVM::ConstantOp::create(rewriter,
-      loc, llvmInt8Type, rewriter.getI8IntegerAttr(isShared));
+  auto isHostShared = mlir::LLVM::ConstantOp::create(
+      rewriter, loc, llvmInt8Type, rewriter.getI8IntegerAttr(isShared));
 
   Value allocatedPtr =
       allocCallBuilder.create(loc, rewriter, {sizeBytes, stream, isHostShared})
@@ -1012,8 +1012,8 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
       uint64_t staticSize = static_cast<uint64_t>(bitwidth / 8) *
                             static_cast<uint64_t>(memrefTy.getNumElements());
 
-      Value sizeArg = LLVM::ConstantOp::create(rewriter,
-          loc, getIndexType(), rewriter.getIndexAttr(staticSize));
+      Value sizeArg = LLVM::ConstantOp::create(
+          rewriter, loc, getIndexType(), rewriter.getIndexAttr(staticSize));
       llvmArgumentsWithSizes.push_back(llvmArg); // Presumably a bare pointer.
       llvmArgumentsWithSizes.push_back(sizeArg);
     }
@@ -1025,8 +1025,8 @@ LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
         gpu::KernelDim3{adaptor.getClusterSizeX(), adaptor.getClusterSizeY(),
                         adaptor.getClusterSizeZ()};
   }
-  gpu::LaunchFuncOp::create(rewriter,
-      launchOp.getLoc(), launchOp.getKernelAttr(),
+  gpu::LaunchFuncOp::create(
+      rewriter, launchOp.getLoc(), launchOp.getKernelAttr(),
       gpu::KernelDim3{adaptor.getGridSizeX(), adaptor.getGridSizeY(),
                       adaptor.getGridSizeZ()},
       gpu::KernelDim3{adaptor.getBlockSizeX(), adaptor.getBlockSizeY(),
@@ -1048,8 +1048,8 @@ static Value bitAndAddrspaceCast(Location loc,
                                  const LLVMTypeConverter &typeConverter) {
   auto sourceTy = cast<LLVM::LLVMPointerType>(sourcePtr.getType());
   if (destinationType.getAddressSpace() != sourceTy.getAddressSpace())
-    sourcePtr = LLVM::AddrSpaceCastOp::create(rewriter,
-        loc,
+    sourcePtr = LLVM::AddrSpaceCastOp::create(
+        rewriter, loc,
         LLVM::LLVMPointerType::get(rewriter.getContext(),
                                    destinationType.getAddressSpace()),
         sourcePtr);
@@ -1073,8 +1073,8 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite(
 
   Type elementPtrType = getElementPtrType(memRefType);
   Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
-  Value gepPtr = LLVM::GEPOp::create(rewriter,
-      loc, elementPtrType,
+  Value gepPtr = LLVM::GEPOp::create(
+      rewriter, loc, elementPtrType,
       typeConverter->convertType(memRefType.getElementType()), nullPtr,
       numElements);
   auto sizeBytes =
@@ -1151,14 +1151,14 @@ template <typename T>
 static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmInt32Type = builder.getIntegerType(32);
   return LLVM::ConstantOp::create(builder, loc, llvmInt32Type,
-                                          static_cast<int32_t>(tValue));
+                                  static_cast<int32_t>(tValue));
 }
 
 template <typename T>
 static Value genConstFloat32From(OpBuilder &builder, Location loc, T tValue) {
   Type llvmFloat32Type = builder.getF32Type();
-  return LLVM::ConstantOp::create(builder,
-      loc, llvmFloat32Type,
+  return LLVM::ConstantOp::create(
+      builder, loc, llvmFloat32Type,
       builder.getF32FloatAttr(static_cast<float>(tValue)));
 }
 
@@ -1189,10 +1189,10 @@ LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite(
   // the dnmat is used with spmat with 2:4 sparsity
   if (dims.size() == 2) {
     if (isSpMMCusparseLtOp(op.getDnTensor())) {
-      auto handleSz = LLVM::ConstantOp::create(rewriter,
-          loc, getIndexType(), rewriter.getIndexAttr(11032));
-      handle = LLVM::AllocaOp::create(rewriter,
-          loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
+      auto handleSz = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                               rewriter.getIndexAttr(11032));
+      handle = LLVM::AllocaOp::create(rewriter, loc, llvmPointerType,
+                                      llvmInt8Type, handleSz, /*alignment=*/16);
       handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
       createLtDnMatCallBuilder
@@ -1351,10 +1351,10 @@ LogicalResult ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto dtp = genConstInt32From(rewriter, loc, getCuSparseDataTypeFrom(dType));
 
   // CUDA runner asserts the size is 44104 bytes.
-  auto handleSz = LLVM::ConstantOp::create(rewriter,
-      loc, getIndexType(), rewriter.getIndexAttr(44104));
-  Value handle = LLVM::AllocaOp::create(rewriter,
-      loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
+  auto handleSz = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                           rewriter.getIndexAttr(44104));
+  Value handle = LLVM::AllocaOp::create(
+      rewriter, loc, llvmPointerType, llvmInt8Type, handleSz, /*alignment=*/16);
   handle = LLVM::BitcastOp::create(rewriter, loc, llvmPointerType, handle);
 
   create2To4SpMatCallBuilder
@@ -1442,9 +1442,10 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
     auto computeType = genConstInt32From(
         rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType()));
     auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                   rewriter.getIndexAttr(3));
-    auto bufferSize = LLVM::AllocaOp::create(rewriter,
-        loc, llvmPointerType, llvmPointerType, three, /*alignment=*/16);
+                                          rewriter.getIndexAttr(3));
+    auto bufferSize =
+        LLVM::AllocaOp::create(rewriter, loc, llvmPointerType, llvmPointerType,
+                               three, /*alignment=*/16);
     createCuSparseLtSpMMBufferSizeBuilder
         .create(loc, rewriter,
                 {bufferSize, modeA, modeB, adaptor.getSpmatA(),
@@ -1452,14 +1453,14 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
                  pruneFlag, stream})
         .getResult();
 
-    auto bufferSizePtr1 = LLVM::GEPOp::create(rewriter,
-        loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{LLVM::ConstantOp::create(rewriter,
-            loc, getIndexType(), rewriter.getIndexAttr(1))});
-    auto bufferSizePtr2 = LLVM::GEPOp::create(rewriter,
-        loc, llvmPointerType, llvmPointerType, bufferSize,
-        ValueRange{LLVM::ConstantOp::create(rewriter,
-            loc, getIndexType(), rewriter.getIndexAttr(2))});
+    auto bufferSizePtr1 = LLVM::GEPOp::create(
+        rewriter, loc, llvmPointerType, llvmPointerType, bufferSize,
+        ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                            rewriter.getIndexAttr(1))});
+    auto bufferSizePtr2 = LLVM::GEPOp::create(
+        rewriter, loc, llvmPointerType, llvmPointerType, bufferSize,
+        ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                            rewriter.getIndexAttr(2))});
     auto bufferSize0 =
         LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, bufferSize);
     auto bufferSize1 =
@@ -1670,22 +1671,22 @@ LogicalResult ConvertSpMatGetSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto stream = adaptor.getAsyncDependencies().front();
 
   auto three = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                 rewriter.getIndexAttr(3));
-  auto buffer = LLVM::AllocaOp::create(rewriter,
-      loc, llvmPointerType, llvmInt64Type, three, /*alignment=*/16);
+                                        rewriter.getIndexAttr(3));
+  auto buffer = LLVM::AllocaOp::create(rewriter, loc, llvmPointerType,
+                                       llvmInt64Type, three, /*alignment=*/16);
 
-  auto rowsPtr = LLVM::GEPOp::create(rewriter,
-      loc, llvmPointerType, llvmPointerType, buffer,
+  auto rowsPtr = LLVM::GEPOp::create(
+      rewriter, loc, llvmPointerType, llvmPointerType, buffer,
       ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                   rewriter.getIndexAttr(0))});
-  auto colsPtr = LLVM::GEPOp::create(rewriter,
-      loc, llvmPointerType, llvmPointerType, buffer,
+                                          rewriter.getIndexAttr(0))});
+  auto colsPtr = LLVM::GEPOp::create(
+      rewriter, loc, llvmPointerType, llvmPointerType, buffer,
       ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                   rewriter.getIndexAttr(1))});
-  auto nnzsPtr = LLVM::GEPOp::create(rewriter,
-      loc, llvmPointerType, llvmPointerType, buffer,
+                                          rewriter.getIndexAttr(1))});
+  auto nnzsPtr = LLVM::GEPOp::create(
+      rewriter, loc, llvmPointerType, llvmPointerType, buffer,
       ValueRange{LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                   rewriter.getIndexAttr(2))});
+                                          rewriter.getIndexAttr(2))});
   createSpMatGetSizeBuilder.create(
       loc, rewriter, {adaptor.getSpmat(), rowsPtr, colsPtr, nnzsPtr, stream});
   auto rows = LLVM::LoadOp::create(rewriter, loc, llvmInt64Type, rowsPtr);
diff --git a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
index 70964727351a9..91c43e8bd1117 100644
--- a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
@@ -124,11 +124,13 @@ struct OpLowering : public ConvertOpToLLVMPattern<Op> {
                                   rewriter.getContext(), 32, min, max));
     }
     if (indexBitwidth > 32) {
-      newOp = LLVM::SExtOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
+      newOp = LLVM::SExtOp::create(rewriter, loc,
+                                   IntegerType::get(context, indexBitwidth),
+                                   newOp->getResult(0));
     } else if (indexBitwidth < 32) {
-      newOp = LLVM::TruncOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), newOp->getResult(0));
+      newOp = LLVM::TruncOp::create(rewriter, loc,
+                                    IntegerType::get(context, indexBitwidth),
+                                    newOp->getResult(0));
     }
 
     rewriter.replaceOp(op, newOp->getResults());
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index 33b09b06649b9..9f36e5c369d06 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -115,19 +115,20 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     // there is no guarantee of a specific value being used to indicate true,
     // compare for inequality with zero (rather than truncate or shift).
     if (isResultBool) {
-      Value zero = LLVM::ConstantOp::create(rewriter,
-          op->getLoc(), rewriter.getIntegerType(32),
-          rewriter.getI32IntegerAttr(0));
-      Value truncated = LLVM::ICmpOp::create(rewriter,
-          op->getLoc(), LLVM::ICmpPredicate::ne, callOp.getResult(), zero);
+      Value zero = LLVM::ConstantOp::create(rewriter, op->getLoc(),
+                                            rewriter.getIntegerType(32),
+                                            rewriter.getI32IntegerAttr(0));
+      Value truncated =
+          LLVM::ICmpOp::create(rewriter, op->getLoc(), LLVM::ICmpPredicate::ne,
+                               callOp.getResult(), zero);
       rewriter.replaceOp(op, {truncated});
       return success();
     }
 
     assert(callOp.getResult().getType().isF32() &&
            "only f32 types are supposed to be truncated back");
-    Value truncated = LLVM::FPTruncOp::create(rewriter,
-        op->getLoc(), adaptor.getOperands().front().getType(),
+    Value truncated = LLVM::FPTruncOp::create(
+        rewriter, op->getLoc(), adaptor.getOperands().front().getType(),
         callOp.getResult());
     rewriter.replaceOp(op, {truncated});
     return success();
@@ -142,8 +143,9 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
     if (!f16Func.empty() && isa<Float16Type>(type))
       return operand;
 
-    return LLVM::FPExtOp::create(rewriter,
-        operand.getLoc(), Float32Type::get(rewriter.getContext()), operand);
+    return LLVM::FPExtOp::create(rewriter, operand.getLoc(),
+                                 Float32Type::get(rewriter.getContext()),
+                                 operand);
   }
 
   Type getFunctionType(Type resultType, ValueRange operands) const {
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index fc469102a54d2..c2363a1a40294 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -54,8 +54,8 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
       SymbolTable::lookupSymbolIn(symbolTable, name));
   if (!func) {
     OpBuilder b(symbolTable->getRegion(0));
-    func = LLVM::LLVMFuncOp::create(b,
-        symbolTable->getLoc(), name,
+    func = LLVM::LLVMFuncOp::create(
+        b, symbolTable->getLoc(), name,
         LLVM::LLVMFunctionType::get(resultType, paramTypes));
     func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
     func.setNoUnwind(true);
@@ -163,7 +163,7 @@ struct LaunchConfigConversion : ConvertToLLVMPattern {
     Location loc = op->getLoc();
     gpu::Dimension dim = getDimension(op);
     Value dimVal = LLVM::ConstantOp::create(rewriter, loc, dimTy,
-                                                     static_cast<int64_t>(dim));
+                                            static_cast<int64_t>(dim));
     rewriter.replaceOp(op, createSPIRVBuiltinCall(loc, rewriter, func, dimVal));
     return success();
   }
@@ -292,12 +292,12 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     return TypeSwitch<Type, Value>(oldVal.getType())
         .Case([&](BFloat16Type) {
           return LLVM::BitcastOp::create(rewriter, loc, rewriter.getI16Type(),
-                                                  oldVal);
+                                         oldVal);
         })
         .Case([&](IntegerType intTy) -> Value {
           if (intTy.getWidth() == 1)
             return LLVM::ZExtOp::create(rewriter, loc, rewriter.getI8Type(),
-                                                 oldVal);
+                                        oldVal);
           return oldVal;
         })
         .Default(oldVal);
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 7a6601ea754e9..317bfc2970cf5 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -120,8 +120,8 @@ struct GPUSubgroupReduceOpLowering
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     Value offset = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
 
-    auto reduxOp = NVVM::ReduxOp::create(rewriter, loc, int32Type, op.getValue(),
-                                                  mode.value(), offset);
+    auto reduxOp = NVVM::ReduxOp::create(rewriter, loc, int32Type,
+                                         op.getValue(), mode.value(), offset);
 
     rewriter.replaceOp(op, reduxOp->getResult(0));
     return success();
@@ -161,19 +161,19 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     Value one = LLVM::ConstantOp::create(rewriter, loc, int32Type, 1);
     Value minusOne = LLVM::ConstantOp::create(rewriter, loc, int32Type, -1);
     Value thirtyTwo = LLVM::ConstantOp::create(rewriter, loc, int32Type, 32);
-    Value numLeadInactiveLane = LLVM::SubOp::create(rewriter,
-        loc, int32Type, thirtyTwo, adaptor.getWidth());
+    Value numLeadInactiveLane = LLVM::SubOp::create(
+        rewriter, loc, int32Type, thirtyTwo, adaptor.getWidth());
     // Bit mask of active lanes: `(-1) >> (32 - activeWidth)`.
     Value activeMask = LLVM::LShrOp::create(rewriter, loc, int32Type, minusOne,
-                                                     numLeadInactiveLane);
+                                            numLeadInactiveLane);
     Value maskAndClamp;
     if (op.getMode() == gpu::ShuffleMode::UP) {
       // Clamp lane: `32 - activeWidth`
       maskAndClamp = numLeadInactiveLane;
     } else {
       // Clamp lane: `activeWidth - 1`
-      maskAndClamp =
-          LLVM::SubOp::create(rewriter, loc, int32Type, adaptor.getWidth(), one);
+      maskAndClamp = LLVM::SubOp::create(rewriter, loc, int32Type,
+                                         adaptor.getWidth(), one);
     }
 
     bool predIsUsed = !op->getResult(1).use_empty();
@@ -184,9 +184,10 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
       resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(),
                                                   {valueTy, predTy});
     }
-    Value shfl = NVVM::ShflOp::create(rewriter,
-        loc, resultTy, activeMask, adaptor.getValue(), adaptor.getOffset(),
-        maskAndClamp, convertShflKind(op.getMode()), returnValueAndIsValidAttr);
+    Value shfl = NVVM::ShflOp::create(
+        rewriter, loc, resultTy, activeMask, adaptor.getValue(),
+        adaptor.getOffset(), maskAndClamp, convertShflKind(op.getMode()),
+        returnValueAndIsValidAttr);
     if (predIsUsed) {
       Value shflValue = LLVM::ExtractValueOp::create(rewriter, loc, shfl, 0);
       Value isActiveSrcLane =
@@ -220,11 +221,11 @@ struct GPULaneIdOpToNVVM : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      newOp = LLVM::SExtOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), newOp);
+      newOp = LLVM::SExtOp::create(
+          rewriter, loc, IntegerType::get(context, indexBitwidth), newOp);
     } else if (indexBitwidth < 32) {
-      newOp = LLVM::TruncOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), newOp);
+      newOp = LLVM::TruncOp::create(
+          rewriter, loc, IntegerType::get(context, indexBitwidth), newOp);
     }
     rewriter.replaceOp(op, {newOp});
     return success();
@@ -272,7 +273,7 @@ struct AssertOpToAssertfailLowering
         rewriter.splitBlock(assertBlock, ++assertOp->getIterator());
     rewriter.setInsertionPointToEnd(beforeBlock);
     cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(), afterBlock,
-                                      assertBlock);
+                             assertBlock);
     rewriter.setInsertionPointToEnd(assertBlock);
     cf::BranchOp::create(rewriter, loc, afterBlock);
 
@@ -301,12 +302,12 @@ struct AssertOpToAssertfailLowering
     // Create constants.
     auto getGlobal = [&](LLVM::GlobalOp global) {
       // Get a pointer to the format string's first element.
-      Value globalPtr = LLVM::AddressOfOp::create(rewriter,
-          loc, LLVM::LLVMPointerType::get(ctx, global.getAddrSpace()),
+      Value globalPtr = LLVM::AddressOfOp::create(
+          rewriter, loc, LLVM::LLVMPointerType::get(ctx, global.getAddrSpace()),
           global.getSymNameAttr());
       Value start =
           LLVM::GEPOp::create(rewriter, loc, ptrType, global.getGlobalType(),
-                                       globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+                              globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
       return start;
     };
     Value assertMessage = getGlobal(getOrCreateStringConstant(
diff --git a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
index 7d5f65c470386..99c059cb03299 100644
--- a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
@@ -126,8 +126,8 @@ struct WmmaLoadOpToNVVMLowering
         cast<MemRefType>(subgroupMmaLoadMatrixOp.getSrcMemref().getType()),
         adaptor.getSrcMemref(), adaptor.getIndices());
 
-    Value leadingDim = LLVM::ConstantOp::create(rewriter,
-        loc, rewriter.getI32Type(),
+    Value leadingDim = LLVM::ConstantOp::create(
+        rewriter, loc, rewriter.getI32Type(),
         subgroupMmaLoadMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMALoadOp>(
         op, resType, dataPtr, leadingDim, m, n, k, layout, eltype, frag);
@@ -181,8 +181,8 @@ struct WmmaStoreOpToNVVMLowering
         rewriter, loc,
         cast<MemRefType>(subgroupMmaStoreMatrixOp.getDstMemref().getType()),
         adaptor.getDstMemref(), adaptor.getIndices());
-    Value leadingDim = LLVM::ConstantOp::create(rewriter,
-        loc, rewriter.getI32Type(),
+    Value leadingDim = LLVM::ConstantOp::create(
+        rewriter, loc, rewriter.getI32Type(),
         subgroupMmaStoreMatrixOp.getLeadDimensionAttr());
     rewriter.replaceOpWithNewOp<NVVM::WMMAStoreOp>(
         op, dataPtr, m, n, k, layout, eltype, storeOpOperands, leadingDim);
@@ -282,10 +282,10 @@ struct WmmaConstantOpToNVVMLowering
     if (auto vecType = dyn_cast<VectorType>(type.getBody()[0])) {
       Value vecCst = LLVM::PoisonOp::create(rewriter, loc, vecType);
       for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) {
-        Value idx = LLVM::ConstantOp::create(rewriter,
-            loc, rewriter.getI32Type(), vecEl);
+        Value idx = LLVM::ConstantOp::create(rewriter, loc,
+                                             rewriter.getI32Type(), vecEl);
         vecCst = LLVM::InsertElementOp::create(rewriter, loc, vecType, vecCst,
-                                                        cst, idx);
+                                               cst, idx);
       }
       cst = vecCst;
     }
@@ -305,14 +305,14 @@ static Value createMinMaxF(OpBuilder &builder, Location loc, Value lhs,
   Type i1Type = builder.getI1Type();
   if (auto vecType = dyn_cast<VectorType>(lhs.getType()))
     i1Type = VectorType::get(vecType.getShape(), i1Type);
-  Value cmp = LLVM::FCmpOp::create(builder,
-      loc, i1Type, isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt,
-      lhs, rhs);
+  Value cmp = LLVM::FCmpOp::create(
+      builder, loc, i1Type,
+      isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt, lhs, rhs);
   Value sel = LLVM::SelectOp::create(builder, loc, cmp, lhs, rhs);
-  Value isNan = LLVM::FCmpOp::create(builder,
-      loc, i1Type, LLVM::FCmpPredicate::uno, lhs, rhs);
-  Value nan = LLVM::ConstantOp::create(builder,
-      loc, lhs.getType(),
+  Value isNan = LLVM::FCmpOp::create(builder, loc, i1Type,
+                                     LLVM::FCmpPredicate::uno, lhs, rhs);
+  Value nan = LLVM::ConstantOp::create(
+      builder, loc, lhs.getType(),
       builder.getFloatAttr(floatType,
                            APFloat::getQNaN(floatType.getFloatSemantics())));
   return LLVM::SelectOp::create(builder, loc, isNan, nan, sel);
@@ -360,8 +360,8 @@ struct WmmaElementwiseOpToNVVMLowering
     for (size_t i = 0, e = destType.getBody().size(); i < e; ++i) {
       SmallVector<Value> extractedOperands;
       for (size_t opIdx = 0; opIdx < numOperands; opIdx++) {
-        extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
-            loc, adaptor.getOperands()[opIdx], i));
+        extractedOperands.push_back(LLVM::ExtractValueOp::create(
+            rewriter, loc, adaptor.getOperands()[opIdx], i));
       }
       Value element =
           createScalarOp(rewriter, loc, subgroupMmaElementwiseOp.getOpType(),
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 9b43d3ef602d1..d22364e1ef441 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -85,9 +85,9 @@ static Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
   Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
   Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
   Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, int32Type,
-                                                    ValueRange{minus1, zero});
+                                           ValueRange{minus1, zero});
   Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, int32Type,
-                                                   ValueRange{minus1, mbcntLo});
+                                          ValueRange{minus1, mbcntLo});
   return laneId;
 }
 static constexpr StringLiteral amdgcnDataLayout =
@@ -112,19 +112,19 @@ struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
     Type intTy = IntegerType::get(context, 32);
     Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
     Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
-    Value mbcntLo =
-        ROCDL::MbcntLoOp::create(rewriter, loc, intTy, ValueRange{minus1, zero});
-    Value laneId = ROCDL::MbcntHiOp::create(rewriter,
-        loc, intTy, ValueRange{minus1, mbcntLo});
+    Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, intTy,
+                                             ValueRange{minus1, zero});
+    Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, intTy,
+                                            ValueRange{minus1, mbcntLo});
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     if (indexBitwidth > 32) {
-      laneId = LLVM::SExtOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), laneId);
+      laneId = LLVM::SExtOp::create(
+          rewriter, loc, IntegerType::get(context, indexBitwidth), laneId);
     } else if (indexBitwidth < 32) {
-      laneId = LLVM::TruncOp::create(rewriter,
-          loc, IntegerType::get(context, indexBitwidth), laneId);
+      laneId = LLVM::TruncOp::create(
+          rewriter, loc, IntegerType::get(context, indexBitwidth), laneId);
     }
     rewriter.replaceOp(op, {laneId});
     return success();
@@ -149,8 +149,8 @@ struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
           /*bitWidth=*/32, /*lower=*/isBeforeGfx10 ? 64 : 32,
           /*upper=*/op.getUpperBoundAttr().getInt() + 1);
     }
-    Value wavefrontOp = ROCDL::WavefrontSizeOp::create(rewriter,
-        op.getLoc(), rewriter.getI32Type(), bounds);
+    Value wavefrontOp = ROCDL::WavefrontSizeOp::create(
+        rewriter, op.getLoc(), rewriter.getI32Type(), bounds);
     wavefrontOp = truncOrExtToLLVMType(rewriter, op.getLoc(), wavefrontOp,
                                        *getTypeConverter());
     rewriter.replaceOp(op, {wavefrontOp});
@@ -200,24 +200,24 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     switch (op.getMode()) {
     case gpu::ShuffleMode::UP:
       dstLane = LLVM::SubOp::create(rewriter, loc, int32Type, srcLaneId,
-                                             adaptor.getOffset());
+                                    adaptor.getOffset());
       break;
     case gpu::ShuffleMode::DOWN:
       dstLane = LLVM::AddOp::create(rewriter, loc, int32Type, srcLaneId,
-                                             adaptor.getOffset());
+                                    adaptor.getOffset());
       break;
     case gpu::ShuffleMode::XOR:
       dstLane = LLVM::XOrOp::create(rewriter, loc, int32Type, srcLaneId,
-                                             adaptor.getOffset());
+                                    adaptor.getOffset());
       break;
     case gpu::ShuffleMode::IDX:
       dstLane = adaptor.getOffset();
       break;
     }
-    Value isActiveSrcLane = LLVM::ICmpOp::create(rewriter,
-        loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
+    Value isActiveSrcLane = LLVM::ICmpOp::create(
+        rewriter, loc, LLVM::ICmpPredicate::slt, dstLane, widthOrZeroIfOutside);
     Value selectDstLane = LLVM::SelectOp::create(rewriter, loc, isActiveSrcLane,
-                                                          dstLane, srcLaneId);
+                                                 dstLane, srcLaneId);
     Value two = LLVM::ConstantOp::create(rewriter, loc, int32Type, 2);
     Value dwordAlignedDstLane =
         LLVM::ShlOp::create(rewriter, loc, int32Type, selectDstLane, two);
@@ -227,7 +227,7 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     SmallVector<Value> swizzled;
     for (Value v : decomposed) {
       Value res = ROCDL::DsBpermuteOp::create(rewriter, loc, int32Type,
-                                                       dwordAlignedDstLane, v);
+                                              dwordAlignedDstLane, v);
       swizzled.emplace_back(res);
     }
     Value shflValue =
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index 71618cbf8a2f2..a19194eb181fb 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -169,8 +169,8 @@ LogicalResult LaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
 
   Value vector =
       spirv::getBuiltinVariableValue(op, builtin, builtinType, rewriter);
-  Value dim = spirv::CompositeExtractOp::create(rewriter,
-      op.getLoc(), builtinType, vector,
+  Value dim = spirv::CompositeExtractOp::create(
+      rewriter, op.getLoc(), builtinType, vector,
       rewriter.getI32ArrayAttr({static_cast<int32_t>(op.getDimension())}));
   if (forShader && builtinType != indexType)
     dim = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType, dim);
@@ -199,7 +199,7 @@ SingleDimLaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
       spirv::getBuiltinVariableValue(op, builtin, i32Type, rewriter);
   if (i32Type != indexType)
     builtinValue = spirv::UConvertOp::create(rewriter, op.getLoc(), indexType,
-                                                      builtinValue);
+                                             builtinValue);
   rewriter.replaceOp(op, builtinValue);
   return success();
 }
@@ -257,8 +257,8 @@ lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter,
       signatureConverter.addInputs(argType.index(), convertedType);
     }
   }
-  auto newFuncOp = spirv::FuncOp::create(rewriter,
-      funcOp.getLoc(), funcOp.getName(),
+  auto newFuncOp = spirv::FuncOp::create(
+      rewriter, funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(signatureConverter.getConvertedTypes(), {}));
   for (const auto &namedAttr : funcOp->getAttrs()) {
     if (namedAttr.getName() == funcOp.getFunctionTypeAttrName() ||
@@ -367,8 +367,8 @@ LogicalResult GPUModuleConversion::matchAndRewrite(
 
   // Add a keyword to the module name to avoid symbolic conflict.
   std::string spvModuleName = (kSPIRVModule + moduleOp.getName()).str();
-  auto spvModule = spirv::ModuleOp::create(rewriter,
-      moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
+  auto spvModule = spirv::ModuleOp::create(
+      rewriter, moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
       StringRef(spvModuleName));
 
   // Move the region from the module op into the SPIR-V module.
@@ -452,42 +452,42 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
 
   switch (shuffleOp.getMode()) {
   case gpu::ShuffleMode::XOR: {
-    result = spirv::GroupNonUniformShuffleXorOp::create(rewriter,
-        loc, scope, adaptor.getValue(), adaptor.getOffset());
+    result = spirv::GroupNonUniformShuffleXorOp::create(
+        rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::IDX: {
-    result = spirv::GroupNonUniformShuffleOp::create(rewriter,
-        loc, scope, adaptor.getValue(), adaptor.getOffset());
+    result = spirv::GroupNonUniformShuffleOp::create(
+        rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset());
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
                                          shuffleOp.getLoc(), rewriter);
     break;
   }
   case gpu::ShuffleMode::DOWN: {
-    result = spirv::GroupNonUniformShuffleDownOp::create(rewriter,
-        loc, scope, adaptor.getValue(), adaptor.getOffset());
+    result = spirv::GroupNonUniformShuffleDownOp::create(
+        rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset());
 
     Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
         arith::AddIOp::create(rewriter, loc, laneId, adaptor.getOffset());
     validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
-                                              resultLaneId, adaptor.getWidth());
+                                     resultLaneId, adaptor.getWidth());
     break;
   }
   case gpu::ShuffleMode::UP: {
-    result = spirv::GroupNonUniformShuffleUpOp::create(rewriter,
-        loc, scope, adaptor.getValue(), adaptor.getOffset());
+    result = spirv::GroupNonUniformShuffleUpOp::create(
+        rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset());
 
     Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     Value resultLaneId =
         arith::SubIOp::create(rewriter, loc, laneId, adaptor.getOffset());
     auto i32Type = rewriter.getIntegerType(32);
-    validVal = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sge, resultLaneId,
-        arith::ConstantOp::create(rewriter,
-            loc, i32Type, rewriter.getIntegerAttr(i32Type, 0)));
+    validVal = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sge, resultLaneId,
+        arith::ConstantOp::create(rewriter, loc, i32Type,
+                                  rewriter.getIntegerAttr(i32Type, 0)));
     break;
   }
   }
@@ -516,15 +516,16 @@ LogicalResult GPURotateConversion::matchAndRewrite(
 
   Location loc = rotateOp.getLoc();
   auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
-  Value rotateResult = spirv::GroupNonUniformRotateKHROp::create(rewriter,
-      loc, scope, adaptor.getValue(), adaptor.getOffset(), adaptor.getWidth());
+  Value rotateResult = spirv::GroupNonUniformRotateKHROp::create(
+      rewriter, loc, scope, adaptor.getValue(), adaptor.getOffset(),
+      adaptor.getWidth());
   Value validVal;
   if (widthAttr.getValue().getZExtValue() == subgroupSize) {
     validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(), loc, rewriter);
   } else {
     Value laneId = gpu::LaneIdOp::create(rewriter, loc, widthAttr);
     validVal = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
-                                              laneId, adaptor.getWidth());
+                                     laneId, adaptor.getWidth());
   }
 
   rewriter.replaceOp(rotateOp, {rotateResult, validVal});
@@ -554,8 +555,8 @@ static Value createGroupReduceOpImpl(OpBuilder &builder, Location loc,
 
   Value clusterSizeValue;
   if (clusterSize.has_value())
-    clusterSizeValue = spirv::ConstantOp::create(builder,
-        loc, builder.getI32Type(),
+    clusterSizeValue = spirv::ConstantOp::create(
+        builder, loc, builder.getI32Type(),
         builder.getIntegerAttr(builder.getI32Type(), *clusterSize));
 
   return builder
@@ -740,8 +741,8 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstName =
         makeVarName(moduleOp, llvm::Twine(globalVarName) + "_sc");
 
-    return spirv::SpecConstantOp::create(rewriter,
-        loc, rewriter.getStringAttr(specCstName), attr);
+    return spirv::SpecConstantOp::create(
+        rewriter, loc, rewriter.getStringAttr(specCstName), attr);
   };
   {
     Operation *parent =
@@ -774,8 +775,8 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     std::string specCstCompositeName =
         (llvm::Twine(globalVarName) + "_scc").str();
 
-    specCstComposite = spirv::SpecConstantCompositeOp::create(rewriter,
-        loc, TypeAttr::get(globalType),
+    specCstComposite = spirv::SpecConstantCompositeOp::create(
+        rewriter, loc, TypeAttr::get(globalType),
         rewriter.getStringAttr(specCstCompositeName),
         rewriter.getArrayAttr(constituents));
 
@@ -785,16 +786,17 @@ LogicalResult GPUPrintfConversion::matchAndRewrite(
     // Define a GlobalVarOp initialized using specialized constants
     // that is used to specify the printf format string
     // to be passed to the SPIRV CLPrintfOp.
-    globalVar = spirv::GlobalVariableOp::create(rewriter,
-        loc, ptrType, globalVarName, FlatSymbolRefAttr::get(specCstComposite));
+    globalVar = spirv::GlobalVariableOp::create(
+        rewriter, loc, ptrType, globalVarName,
+        FlatSymbolRefAttr::get(specCstComposite));
 
     globalVar->setAttr("Constant", rewriter.getUnitAttr());
   }
   // Get SSA value of Global variable and create pointer to i8 to point to
   // the format string.
   Value globalPtr = spirv::AddressOfOp::create(rewriter, loc, globalVar);
-  Value fmtStr = spirv::BitcastOp::create(rewriter,
-      loc,
+  Value fmtStr = spirv::BitcastOp::create(
+      rewriter, loc,
       spirv::PointerType::get(i8Type, spirv::StorageClass::UniformConstant),
       globalPtr);
 
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index da8ffe47a7d33..a344f88326089 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -144,8 +144,9 @@ void GPUToSPIRVPass::runOnOperation() {
     if (targetEnvSupportsKernelCapability(moduleOp)) {
       moduleOp.walk([&](gpu::GPUFuncOp funcOp) {
         builder.setInsertionPoint(funcOp);
-        auto newFuncOp = func::FuncOp::create(builder,
-            funcOp.getLoc(), funcOp.getName(), funcOp.getFunctionType());
+        auto newFuncOp =
+            func::FuncOp::create(builder, funcOp.getLoc(), funcOp.getName(),
+                                 funcOp.getFunctionType());
         auto entryBlock = newFuncOp.addEntryBlock();
         builder.setInsertionPointToEnd(entryBlock);
         func::ReturnOp::create(builder, funcOp.getLoc());
diff --git a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
index 70646aa5f2e74..51dc50048024f 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
@@ -283,8 +283,8 @@ struct WmmaLoadOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = spirv::ConstantOp::create(rewriter,
-        loc, i32Type, IntegerAttr::get(i32Type, stride));
+    auto strideValue = spirv::ConstantOp::create(
+        rewriter, loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
     auto layout = isColMajor ? spirv::CooperativeMatrixLayoutKHR::ColumnMajor
@@ -315,8 +315,8 @@ struct WmmaStoreOpToSPIRVLowering final
 
     int64_t stride = op.getLeadDimension().getSExtValue();
     IntegerType i32Type = rewriter.getI32Type();
-    auto strideValue = spirv::ConstantOp::create(rewriter,
-        loc, i32Type, IntegerAttr::get(i32Type, stride));
+    auto strideValue = spirv::ConstantOp::create(
+        rewriter, loc, i32Type, IntegerAttr::get(i32Type, stride));
 
     bool isColMajor = op.getTranspose().value_or(false);
     auto layout = isColMajor ? spirv::CooperativeMatrixLayoutKHR::ColumnMajor
diff --git a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
index 0d5b318f739cc..99d2f6ca78c38 100644
--- a/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
+++ b/mlir/lib/Conversion/IndexToLLVM/IndexToLLVM.cpp
@@ -59,8 +59,8 @@ struct ConvertIndexCeilDivS : mlir::ConvertOpToLLVMPattern<CeilDivSOp> {
     // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
     Value nPos =
         LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::sgt, n, zero);
-    Value sameSign =
-        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, nPos, mPos);
+    Value sameSign = LLVM::ICmpOp::create(rewriter, loc,
+                                          LLVM::ICmpPredicate::eq, nPos, mPos);
     Value nNonZero =
         LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
     Value cmp = LLVM::AndOp::create(rewriter, loc, sameSign, nNonZero);
@@ -135,8 +135,8 @@ struct ConvertIndexFloorDivS : mlir::ConvertOpToLLVMPattern<FloorDivSOp> {
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
     Value nNeg =
         LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt, n, zero);
-    Value diffSign =
-        LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, nNeg, mNeg);
+    Value diffSign = LLVM::ICmpOp::create(rewriter, loc,
+                                          LLVM::ICmpPredicate::ne, nNeg, mNeg);
     Value nNonZero =
         LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::ne, n, zero);
     Value cmp = LLVM::AndOp::create(rewriter, loc, diffSign, nNonZero);
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index 6fc097059c5e5..36cfe9dd6e2db 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -111,12 +111,12 @@ struct ConvertIndexCeilDivSPattern final : OpConversionPattern<CeilDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, -1));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                           IntegerAttr::get(n_type, 0));
+    Value posOne = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                             IntegerAttr::get(n_type, 1));
+    Value negOne = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                             IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
     Value mPos = spirv::SGreaterThanOp::create(rewriter, loc, m, zero);
@@ -161,10 +161,10 @@ struct ConvertIndexCeilDivUPattern final : OpConversionPattern<CeilDivUOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 0));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                           IntegerAttr::get(n_type, 0));
     Value one = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                                   IntegerAttr::get(n_type, 1));
+                                          IntegerAttr::get(n_type, 1));
 
     // Compute the non-zero result.
     Value minusOne = spirv::ISubOp::create(rewriter, loc, n, one);
@@ -197,12 +197,12 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 0));
-    Value posOne = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, 1));
-    Value negOne = spirv::ConstantOp::create(rewriter,
-        loc, n_type, IntegerAttr::get(n_type, -1));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                           IntegerAttr::get(n_type, 0));
+    Value posOne = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                             IntegerAttr::get(n_type, 1));
+    Value negOne = spirv::ConstantOp::create(rewriter, loc, n_type,
+                                             IntegerAttr::get(n_type, -1));
 
     // Compute `x`.
     Value mNeg = spirv::SLessThanOp::create(rewriter, loc, m, zero);
@@ -219,7 +219,8 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
     // Pick the negative result if `n` and `m` have different signs and `n` is
     // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
     Value nNeg = spirv::SLessThanOp::create(rewriter, loc, n, zero);
-    Value diffSign = spirv::LogicalNotEqualOp::create(rewriter, loc, nNeg, mNeg);
+    Value diffSign =
+        spirv::LogicalNotEqualOp::create(rewriter, loc, nNeg, mNeg);
     Value nNonZero = spirv::INotEqualOp::create(rewriter, loc, n, zero);
 
     Value cmp = spirv::LogicalAndOp::create(rewriter, loc, diffSign, nNonZero);
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index e8ad0d4644389..fce7a3f324b86 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -100,20 +100,20 @@ void MemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc,
 static Value createIndexAttrConstant(OpBuilder &builder, Location loc,
                                      Type resultType, int64_t value) {
   return LLVM::ConstantOp::create(builder, loc, resultType,
-                                          builder.getIndexAttr(value));
+                                  builder.getIndexAttr(value));
 }
 
 /// Builds IR extracting the offset from the descriptor.
 Value MemRefDescriptor::offset(OpBuilder &builder, Location loc) {
   return LLVM::ExtractValueOp::create(builder, loc, value,
-                                              kOffsetPosInMemRefDescriptor);
+                                      kOffsetPosInMemRefDescriptor);
 }
 
 /// Builds IR inserting the offset into the descriptor.
 void MemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
                                  Value offset) {
   value = LLVM::InsertValueOp::create(builder, loc, value, offset,
-                                              kOffsetPosInMemRefDescriptor);
+                                      kOffsetPosInMemRefDescriptor);
 }
 
 /// Builds IR inserting the offset into the descriptor.
@@ -125,8 +125,9 @@ void MemRefDescriptor::setConstantOffset(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th size from the descriptor.
 Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) {
-  return LLVM::ExtractValueOp::create(builder,
-      loc, value, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
+  return LLVM::ExtractValueOp::create(
+      builder, loc, value,
+      ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
 Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
@@ -137,23 +138,25 @@ Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
 
   // Copy size values to stack-allocated memory.
   auto one = createIndexAttrConstant(builder, loc, indexType, 1);
-  auto sizes = LLVM::ExtractValueOp::create(builder,
-      loc, value, llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
+  auto sizes = LLVM::ExtractValueOp::create(
+      builder, loc, value,
+      llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
   auto sizesPtr = LLVM::AllocaOp::create(builder, loc, ptrTy, arrayTy, one,
-                                                 /*alignment=*/0);
+                                         /*alignment=*/0);
   LLVM::StoreOp::create(builder, loc, sizes, sizesPtr);
 
   // Load an return size value of interest.
   auto resultPtr = LLVM::GEPOp::create(builder, loc, ptrTy, arrayTy, sizesPtr,
-                                               ArrayRef<LLVM::GEPArg>{0, pos});
+                                       ArrayRef<LLVM::GEPArg>{0, pos});
   return LLVM::LoadOp::create(builder, loc, indexType, resultPtr);
 }
 
 /// Builds IR inserting the pos-th size into the descriptor
 void MemRefDescriptor::setSize(OpBuilder &builder, Location loc, unsigned pos,
                                Value size) {
-  value = LLVM::InsertValueOp::create(builder,
-      loc, value, size, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
+  value = LLVM::InsertValueOp::create(
+      builder, loc, value, size,
+      ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
 }
 
 void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
@@ -164,15 +167,16 @@ void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
 
 /// Builds IR extracting the pos-th stride from the descriptor.
 Value MemRefDescriptor::stride(OpBuilder &builder, Location loc, unsigned pos) {
-  return LLVM::ExtractValueOp::create(builder,
-      loc, value, ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
+  return LLVM::ExtractValueOp::create(
+      builder, loc, value,
+      ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
 
 /// Builds IR inserting the pos-th stride into the descriptor
 void MemRefDescriptor::setStride(OpBuilder &builder, Location loc, unsigned pos,
                                  Value stride) {
-  value = LLVM::InsertValueOp::create(builder,
-      loc, value, stride,
+  value = LLVM::InsertValueOp::create(
+      builder, loc, value, stride,
       ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
 }
 
@@ -208,7 +212,7 @@ Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
           : createIndexAttrConstant(builder, loc, indexType, offsetCst);
   Type elementType = converter.convertType(type.getElementType());
   ptr = LLVM::GEPOp::create(builder, loc, ptr.getType(), elementType, ptr,
-                                    offsetVal);
+                            offsetVal);
   return ptr;
 }
 
@@ -387,12 +391,12 @@ void UnrankedMemRefDescriptor::computeSizes(
     Value doubleRank = LLVM::MulOp::create(builder, loc, indexType, two, rank);
     Value doubleRankIncremented =
         LLVM::AddOp::create(builder, loc, indexType, doubleRank, one);
-    Value rankIndexSize = LLVM::MulOp::create(builder,
-        loc, indexType, doubleRankIncremented, indexSize);
+    Value rankIndexSize = LLVM::MulOp::create(builder, loc, indexType,
+                                              doubleRankIncremented, indexSize);
 
     // Total allocation size.
-    Value allocationSize = LLVM::AddOp::create(builder,
-        loc, indexType, doublePointerSize, rankIndexSize);
+    Value allocationSize = LLVM::AddOp::create(
+        builder, loc, indexType, doublePointerSize, rankIndexSize);
     sizes.push_back(allocationSize);
   }
 }
@@ -424,7 +428,7 @@ Value UnrankedMemRefDescriptor::alignedPtr(
 
   Value alignedGep =
       LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
-                                  elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
+                          elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
   return LLVM::LoadOp::create(builder, loc, elemPtrType, alignedGep);
 }
 
@@ -436,7 +440,7 @@ void UnrankedMemRefDescriptor::setAlignedPtr(
 
   Value alignedGep =
       LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
-                                  elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
+                          elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
   LLVM::StoreOp::create(builder, loc, alignedPtr, alignedGep);
 }
 
@@ -447,7 +451,7 @@ Value UnrankedMemRefDescriptor::offsetBasePtr(
       castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
 
   return LLVM::GEPOp::create(builder, loc, elemPtrPtrType, elemPtrType,
-                                     elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
+                             elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
 }
 
 Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
@@ -457,7 +461,7 @@ Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
   Value offsetPtr =
       offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
   return LLVM::LoadOp::create(builder, loc, typeConverter.getIndexType(),
-                                      offsetPtr);
+                              offsetPtr);
 }
 
 void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
@@ -478,7 +482,7 @@ Value UnrankedMemRefDescriptor::sizeBasePtr(
       indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy});
   auto resultType = LLVM::LLVMPointerType::get(builder.getContext());
   return LLVM::GEPOp::create(builder, loc, resultType, structTy, memRefDescPtr,
-                                     ArrayRef<LLVM::GEPArg>{0, 3});
+                             ArrayRef<LLVM::GEPArg>{0, 3});
 }
 
 Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc,
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index 57aed79874ad0..ecd5b6367fba4 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -58,7 +58,7 @@ Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder,
                                                     Type resultType,
                                                     int64_t value) {
   return LLVM::ConstantOp::create(builder, loc, resultType,
-                                          builder.getIndexAttr(value));
+                                  builder.getIndexAttr(value));
 }
 
 Value ConvertToLLVMPattern::getStridedElementPtr(
@@ -132,8 +132,8 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
     Type elementType = typeConverter->convertType(memRefType.getElementType());
     auto elementPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
     Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, elementPtrType);
-    Value gepPtr = LLVM::GEPOp::create(rewriter,
-        loc, elementPtrType, elementType, nullPtr, runningStride);
+    Value gepPtr = LLVM::GEPOp::create(rewriter, loc, elementPtrType,
+                                       elementType, nullPtr, runningStride);
     size = LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gepPtr);
   } else {
     size = runningStride;
@@ -151,7 +151,7 @@ Value ConvertToLLVMPattern::getSizeInBytes(
   auto convertedPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
   auto nullPtr = LLVM::ZeroOp::create(rewriter, loc, convertedPtrType);
   auto gep = LLVM::GEPOp::create(rewriter, loc, convertedPtrType, llvmType,
-                                          nullPtr, ArrayRef<LLVM::GEPArg>{1});
+                                 nullPtr, ArrayRef<LLVM::GEPArg>{1});
   return LLVM::PtrToIntOp::create(rewriter, loc, getIndexType(), gep);
 }
 
@@ -277,9 +277,9 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
                   .create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
                   .getResult()
             : LLVM::AllocaOp::create(builder, loc, getPtrType(),
-                                             IntegerType::get(getContext(), 8),
-                                             allocationSize,
-                                             /*alignment=*/0);
+                                     IntegerType::get(getContext(), 8),
+                                     allocationSize,
+                                     /*alignment=*/0);
     Value source = desc.memRefDescPtr(builder, loc);
     LLVM::MemcpyOp::create(builder, loc, memory, source, allocationSize, false);
     if (!toDynamic)
@@ -349,8 +349,8 @@ LogicalResult LLVM::detail::oneToOneRewrite(
   SmallVector<Value, 4> results;
   results.reserve(numResults);
   for (unsigned i = 0; i < numResults; ++i) {
-    results.push_back(LLVM::ExtractValueOp::create(rewriter,
-        op->getLoc(), newOp->getResult(0), i));
+    results.push_back(LLVM::ExtractValueOp::create(rewriter, op->getLoc(),
+                                                   newOp->getResult(0), i));
   }
   rewriter.replaceOp(op, results);
   return success();
@@ -371,8 +371,8 @@ LogicalResult LLVM::detail::intrinsicRewrite(
   if (numResults != 0)
     resType = typeConverter.packOperationResults(op->getResultTypes());
 
-  auto callIntrOp = LLVM::CallIntrinsicOp::create(rewriter,
-      loc, resType, rewriter.getStringAttr(intrinsic), operands);
+  auto callIntrOp = LLVM::CallIntrinsicOp::create(
+      rewriter, loc, resType, rewriter.getStringAttr(intrinsic), operands);
   // Propagate attributes.
   callIntrOp->setAttrs(op->getAttrDictionary());
 
@@ -518,20 +518,20 @@ Value mlir::LLVM::getStridedElementPtr(OpBuilder &builder, Location loc,
       Value stride =
           ShapedType::isDynamic(strides[i])
               ? memRefDescriptor.stride(builder, loc, i)
-              : LLVM::ConstantOp::create(builder,
-                    loc, indexType, builder.getIndexAttr(strides[i]));
-      increment =
-          LLVM::MulOp::create(builder, loc, increment, stride, intOverflowFlags);
+              : LLVM::ConstantOp::create(builder, loc, indexType,
+                                         builder.getIndexAttr(strides[i]));
+      increment = LLVM::MulOp::create(builder, loc, increment, stride,
+                                      intOverflowFlags);
     }
     index = index ? LLVM::AddOp::create(builder, loc, index, increment,
-                                                intOverflowFlags)
+                                        intOverflowFlags)
                   : increment;
   }
 
   Type elementPtrType = memRefDescriptor.getElementPtrType();
-  return index ? LLVM::GEPOp::create(builder,
-                     loc, elementPtrType,
-                     converter.convertType(type.getElementType()), base, index,
-                     noWrapFlags)
-               : base;
+  return index
+             ? LLVM::GEPOp::create(builder, loc, elementPtrType,
+                                   converter.convertType(type.getElementType()),
+                                   base, index, noWrapFlags)
+             : base;
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
index 091fb7e4c2afb..d95aeba8a4488 100644
--- a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
@@ -66,8 +66,8 @@ LogicalResult mlir::LLVM::createPrintStrCall(
       DenseElementsAttr::get(dataAttrType, llvm::ArrayRef(elementVals));
   auto arrayTy =
       LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), elementVals.size());
-  auto globalOp = LLVM::GlobalOp::create(builder,
-      loc, arrayTy, /*constant=*/true, LLVM::Linkage::Private,
+  auto globalOp = LLVM::GlobalOp::create(
+      builder, loc, arrayTy, /*constant=*/true, LLVM::Linkage::Private,
       ensureSymbolNameIsUnique(moduleOp, symbolName, symbolTables), dataAttr);
 
   auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext());
@@ -83,6 +83,6 @@ LogicalResult mlir::LLVM::createPrintStrCall(
   if (failed(printer))
     return failure();
   LLVM::CallOp::create(builder, loc, TypeRange(),
-                               SymbolRefAttr::get(printer.value()), gep);
+                       SymbolRefAttr::get(printer.value()), gep);
   return success();
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 77135188dabcf..1a9bf569086da 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -732,7 +732,7 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand,
   // alloca op and so we omit allocating at the entry block.
   auto ptrType = LLVM::LLVMPointerType::get(builder.getContext());
   Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
-                                               builder.getIndexAttr(1));
+                                       builder.getIndexAttr(1));
   Value allocated =
       LLVM::AllocaOp::create(builder, loc, ptrType, operand.getType(), one);
   // Store into the alloca'ed descriptor.
diff --git a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
index 4fc208e05eb9d..e7dd0b506e12d 100644
--- a/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/VectorPattern.cpp
@@ -93,8 +93,8 @@ LogicalResult LLVM::detail::handleMultidimensionalVectors(
     // element, extract operand vectors
     SmallVector<Value, 4> extractedOperands;
     for (const auto &operand : llvm::enumerate(operands)) {
-      extractedOperands.push_back(LLVM::ExtractValueOp::create(rewriter,
-          loc, operand.value(), position));
+      extractedOperands.push_back(LLVM::ExtractValueOp::create(
+          rewriter, loc, operand.value(), position));
     }
     Value newVal = createOperand(result1DVectorTy, extractedOperands);
     desc = LLVM::InsertValueOp::create(rewriter, loc, desc, newVal, position);
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 498a5cfc40204..3f4b4d6cbc8ab 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -78,8 +78,8 @@ getLibraryCallSymbolRef(Operation *op, PatternRewriter &rewriter) {
   // Insert before module terminator.
   rewriter.setInsertionPoint(module.getBody(),
                              std::prev(module.getBody()->end()));
-  func::FuncOp funcOp = func::FuncOp::create(rewriter,
-      op->getLoc(), fnNameAttr.getValue(), libFnType);
+  func::FuncOp funcOp = func::FuncOp::create(rewriter, op->getLoc(),
+                                             fnNameAttr.getValue(), libFnType);
   // Insert a function attribute that will trigger the emission of the
   // corresponding `_mlir_ciface_xxx` interface so that external libraries see
   // a normalized ABI. This interface is added during std to llvm conversion.
@@ -100,8 +100,8 @@ createTypeCanonicalizedMemRefOperands(OpBuilder &b, Location loc,
       res.push_back(op);
       continue;
     }
-    Value cast =
-        memref::CastOp::create(b, loc, makeStridedLayoutDynamic(memrefType), op);
+    Value cast = memref::CastOp::create(
+        b, loc, makeStridedLayoutDynamic(memrefType), op);
     res.push_back(cast);
   }
   return res;
diff --git a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
index 30521d66a2e4c..5b68eb8188996 100644
--- a/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
+++ b/mlir/lib/Conversion/MPIToLLVM/MPIToLLVM.cpp
@@ -55,14 +55,14 @@ std::pair<Value, Value> getRawPtrAndSize(const Location loc,
   Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
   Value dataPtr =
       LLVM::ExtractValueOp::create(rewriter, loc, ptrType, memRef, 1);
-  Value offset = LLVM::ExtractValueOp::create(rewriter,
-      loc, rewriter.getI64Type(), memRef, 2);
+  Value offset = LLVM::ExtractValueOp::create(rewriter, loc,
+                                              rewriter.getI64Type(), memRef, 2);
   Value resPtr =
       LLVM::GEPOp::create(rewriter, loc, ptrType, elType, dataPtr, offset);
   Value size;
   if (cast<LLVM::LLVMStructType>(memRef.getType()).getBody().size() > 3) {
     size = LLVM::ExtractValueOp::create(rewriter, loc, memRef,
-                                                 ArrayRef<int64_t>{3, 0});
+                                        ArrayRef<int64_t>{3, 0});
     size = LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(), size);
   } else {
     size = arith::ConstantIntOp::create(rewriter, loc, 1, 32);
@@ -158,7 +158,7 @@ class MPICHImplTraits : public MPIImplTraits {
                      ConversionPatternRewriter &rewriter) override {
     static constexpr int MPI_COMM_WORLD = 0x44000000;
     return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
-                                             MPI_COMM_WORLD);
+                                    MPI_COMM_WORLD);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
@@ -195,7 +195,8 @@ class MPICHImplTraits : public MPIImplTraits {
       mtype = MPI_UINT8_T;
     else
       assert(false && "unsupported type");
-    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), mtype);
+    return LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
+                                    mtype);
   }
 
   Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter,
@@ -281,16 +282,16 @@ class OMPIImplTraits : public MPIImplTraits {
     getOrDefineExternalStruct(loc, rewriter, name, commStructT);
 
     // get address of symbol
-    auto comm = LLVM::AddressOfOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(context),
-        SymbolRefAttr::get(context, name));
+    auto comm = LLVM::AddressOfOp::create(rewriter, loc,
+                                          LLVM::LLVMPointerType::get(context),
+                                          SymbolRefAttr::get(context, name));
     return LLVM::PtrToIntOp::create(rewriter, loc, rewriter.getI64Type(), comm);
   }
 
   Value castComm(const Location loc, ConversionPatternRewriter &rewriter,
                  Value comm) override {
-    return LLVM::IntToPtrOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(rewriter.getContext()), comm);
+    return LLVM::IntToPtrOp::create(
+        rewriter, loc, LLVM::LLVMPointerType::get(rewriter.getContext()), comm);
   }
 
   intptr_t getStatusIgnore() override { return 0; }
@@ -330,9 +331,9 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, mtype, typeStructT);
     // get address of symbol
-    return LLVM::AddressOfOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(context),
-        SymbolRefAttr::get(context, mtype));
+    return LLVM::AddressOfOp::create(rewriter, loc,
+                                     LLVM::LLVMPointerType::get(context),
+                                     SymbolRefAttr::get(context, mtype));
   }
 
   Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter,
@@ -389,9 +390,9 @@ class OMPIImplTraits : public MPIImplTraits {
     // make sure global op definition exists
     getOrDefineExternalStruct(loc, rewriter, op, opStructT);
     // get address of symbol
-    return LLVM::AddressOfOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(context),
-        SymbolRefAttr::get(context, op));
+    return LLVM::AddressOfOp::create(rewriter, loc,
+                                     LLVM::LLVMPointerType::get(context),
+                                     SymbolRefAttr::get(context, op));
   }
 };
 
@@ -524,10 +525,10 @@ struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> {
     LLVM::LLVMFuncOp funcDecl = getOrDefineFunction(moduleOp, loc, rewriter,
                                                     "MPI_Comm_split", funcType);
 
-    auto callOp = LLVM::CallOp::create(rewriter,
-        loc, funcDecl,
-        ValueRange{comm, adaptor.getColor(), adaptor.getKey(),
-                   outPtr.getRes()});
+    auto callOp =
+        LLVM::CallOp::create(rewriter, loc, funcDecl,
+                             ValueRange{comm, adaptor.getColor(),
+                                        adaptor.getKey(), outPtr.getRes()});
 
     // load the communicator into a register
     Value res = LLVM::LoadOp::create(rewriter, loc, i32, outPtr.getResult());
@@ -582,8 +583,8 @@ struct CommRankOpLowering : public ConvertOpToLLVMPattern<mpi::CommRankOp> {
     // replace with function call
     auto one = LLVM::ConstantOp::create(rewriter, loc, i32, 1);
     auto rankptr = LLVM::AllocaOp::create(rewriter, loc, ptrType, i32, one);
-    auto callOp = LLVM::CallOp::create(rewriter,
-        loc, initDecl, ValueRange{comm, rankptr.getRes()});
+    auto callOp = LLVM::CallOp::create(rewriter, loc, initDecl,
+                                       ValueRange{comm, rankptr.getRes()});
 
     // load the rank into a register
     auto loadedRank =
@@ -641,10 +642,10 @@ struct SendOpLowering : public ConvertOpToLLVMPattern<mpi::SendOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Send", funcType);
 
     // replace op with function call
-    auto funcCall = LLVM::CallOp::create(rewriter,
-        loc, funcDecl,
-        ValueRange{dataPtr, size, dataType, adaptor.getDest(), adaptor.getTag(),
-                   comm});
+    auto funcCall = LLVM::CallOp::create(rewriter, loc, funcDecl,
+                                         ValueRange{dataPtr, size, dataType,
+                                                    adaptor.getDest(),
+                                                    adaptor.getTag(), comm});
     if (op.getRetval())
       rewriter.replaceOp(op, funcCall.getResult());
     else
@@ -683,8 +684,8 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
     auto mpiTraits = MPIImplTraits::get(moduleOp);
     Value dataType = mpiTraits->getDataType(loc, rewriter, elemType);
     Value comm = mpiTraits->castComm(loc, rewriter, adaptor.getComm());
-    Value statusIgnore = LLVM::ConstantOp::create(rewriter,
-        loc, i64, mpiTraits->getStatusIgnore());
+    Value statusIgnore = LLVM::ConstantOp::create(rewriter, loc, i64,
+                                                  mpiTraits->getStatusIgnore());
     statusIgnore =
         LLVM::IntToPtrOp::create(rewriter, loc, ptrType, statusIgnore);
 
@@ -698,8 +699,8 @@ struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Recv", funcType);
 
     // replace op with function call
-    auto funcCall = LLVM::CallOp::create(rewriter,
-        loc, funcDecl,
+    auto funcCall = LLVM::CallOp::create(
+        rewriter, loc, funcDecl,
         ValueRange{dataPtr, size, dataType, adaptor.getSource(),
                    adaptor.getTag(), comm, statusIgnore});
     if (op.getRetval())
@@ -738,8 +739,9 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
 
     // If input and output are the same, request in-place operation.
     if (adaptor.getSendbuf() == adaptor.getRecvbuf()) {
-      sendPtr = LLVM::ConstantOp::create(rewriter,
-          loc, i64, reinterpret_cast<int64_t>(mpiTraits->getInPlace()));
+      sendPtr = LLVM::ConstantOp::create(
+          rewriter, loc, i64,
+          reinterpret_cast<int64_t>(mpiTraits->getInPlace()));
       sendPtr = LLVM::IntToPtrOp::create(rewriter, loc, ptrType, sendPtr);
     }
 
@@ -757,8 +759,8 @@ struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> {
         getOrDefineFunction(moduleOp, loc, rewriter, "MPI_Allreduce", funcType);
 
     // replace op with function call
-    auto funcCall = LLVM::CallOp::create(rewriter,
-        loc, funcDecl,
+    auto funcCall = LLVM::CallOp::create(
+        rewriter, loc, funcDecl,
         ValueRange{sendPtr, recvPtr, sendSize, dataType, mpiOp, commWorld});
 
     if (op.getRetval())
diff --git a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
index bb3511440e041..08a456691880c 100644
--- a/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
+++ b/mlir/lib/Conversion/MathToFuncs/MathToFuncs.cpp
@@ -121,8 +121,8 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
     initValueAttr = FloatAttr::get(resultElementType, 0.0);
   else
     initValueAttr = IntegerAttr::get(resultElementType, 0);
-  Value result = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(vecType, initValueAttr));
+  Value result = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(vecType, initValueAttr));
   SmallVector<int64_t> strides = computeStrides(shape);
   for (int64_t linearIndex = 0; linearIndex < numElements; ++linearIndex) {
     SmallVector<int64_t> positions = delinearize(linearIndex, strides);
@@ -208,12 +208,12 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value zeroValue = arith::ConstantOp::create(builder,
-      elementType, builder.getIntegerAttr(elementType, 0));
-  Value oneValue = arith::ConstantOp::create(builder,
-      elementType, builder.getIntegerAttr(elementType, 1));
-  Value minusOneValue = arith::ConstantOp::create(builder,
-      elementType,
+  Value zeroValue = arith::ConstantOp::create(
+      builder, elementType, builder.getIntegerAttr(elementType, 0));
+  Value oneValue = arith::ConstantOp::create(
+      builder, elementType, builder.getIntegerAttr(elementType, 1));
+  Value minusOneValue = arith::ConstantOp::create(
+      builder, elementType,
       builder.getIntegerAttr(elementType,
                              APInt(elementType.getIntOrFloatBitWidth(), -1ULL,
                                    /*isSigned=*/true)));
@@ -231,15 +231,16 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
 
   // if (p < T(0)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
-  auto pIsNeg =
-      arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg, zeroValue);
+  auto pIsNeg = arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg,
+                                      zeroValue);
   //   if (b == T(0))
   builder.createBlock(funcBody);
   auto bIsZero =
       arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, bArg, zeroValue);
   //     return T(1) / T(0);
   thenBlock = builder.createBlock(funcBody);
-  func::ReturnOp::create(builder,
+  func::ReturnOp::create(
+      builder,
       arith::DivSIOp::create(builder, oneValue, zeroValue).getResult());
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (b == T(0)).
@@ -261,12 +262,12 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   //   if (b == T(-1)) {
   builder.setInsertionPointToEnd(fallthroughBlock);
   auto bIsMinusOne = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
-                                                   bArg, minusOneValue);
+                                           bArg, minusOneValue);
   //     if (p & T(1))
   builder.createBlock(funcBody);
-  auto pIsOdd = arith::CmpIOp::create(builder,
-      arith::CmpIPredicate::ne, arith::AndIOp::create(builder, pArg, oneValue),
-      zeroValue);
+  auto pIsOdd = arith::CmpIOp::create(
+      builder, arith::CmpIPredicate::ne,
+      arith::AndIOp::create(builder, pArg, oneValue), zeroValue);
   //       return T(-1);
   thenBlock = builder.createBlock(funcBody);
   func::ReturnOp::create(builder, minusOneValue);
@@ -283,7 +284,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   // Set up conditional branch for (b == T(-1)).
   builder.setInsertionPointToEnd(bIsMinusOne->getBlock());
   cf::CondBranchOp::create(builder, bIsMinusOne, pIsOdd->getBlock(),
-                                   fallthroughBlock);
+                           fallthroughBlock);
 
   //   return T(0);
   // } // (p < T(0))
@@ -296,7 +297,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   builder.setInsertionPointToEnd(pIsNeg->getBlock());
   // Set initial values of 'result', 'b' and 'p' for the loop.
   cf::CondBranchOp::create(builder, pIsNeg, bIsZero->getBlock(), loopHeader,
-                                   ValueRange{oneValue, bArg, pArg});
+                           ValueRange{oneValue, bArg, pArg});
 
   // T result = T(1);
   // while (true) {
@@ -313,8 +314,8 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & T(1))
-  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
-      arith::CmpIPredicate::ne,
+  auto powerTmpIsOdd = arith::CmpIOp::create(
+      builder, arith::CmpIPredicate::ne,
       arith::AndIOp::create(builder, powerTmp, oneValue), zeroValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
@@ -326,7 +327,7 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
   // Set up conditional branch for (p & T(1)).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
   cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
-                                   resultTmp);
+                           resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
@@ -336,22 +337,23 @@ static func::FuncOp createElementIPowIFunc(ModuleOp *module, Type elementType) {
 
   //   if (p == T(0))
   auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
-                                                      newPowerTmp, zeroValue);
+                                              newPowerTmp, zeroValue);
   //     return result;
   thenBlock = builder.createBlock(funcBody);
   func::ReturnOp::create(builder, newResultTmp);
   fallthroughBlock = builder.createBlock(funcBody);
   // Set up conditional branch for (p == T(0)).
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
-  cf::CondBranchOp::create(builder, newPowerIsZero, thenBlock, fallthroughBlock);
+  cf::CondBranchOp::create(builder, newPowerIsZero, thenBlock,
+                           fallthroughBlock);
 
   //   b *= b;
   // }
   builder.setInsertionPointToEnd(fallthroughBlock);
   Value newBaseTmp = arith::MulIOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  cf::BranchOp::create(builder,
-      ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
+  cf::BranchOp::create(
+      builder, ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
   return funcOp;
 }
 
@@ -433,23 +435,25 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   Value bArg = funcOp.getArgument(0);
   Value pArg = funcOp.getArgument(1);
   builder.setInsertionPointToEnd(entryBlock);
-  Value oneBValue = arith::ConstantOp::create(builder,
-      baseType, builder.getFloatAttr(baseType, 1.0));
-  Value zeroPValue = arith::ConstantOp::create(builder,
-      powType, builder.getIntegerAttr(powType, 0));
-  Value onePValue = arith::ConstantOp::create(builder,
-      powType, builder.getIntegerAttr(powType, 1));
-  Value minPValue = arith::ConstantOp::create(builder,
-      powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMinValue(
-                                                   powType.getWidth())));
-  Value maxPValue = arith::ConstantOp::create(builder,
-      powType, builder.getIntegerAttr(powType, llvm::APInt::getSignedMaxValue(
-                                                   powType.getWidth())));
+  Value oneBValue = arith::ConstantOp::create(
+      builder, baseType, builder.getFloatAttr(baseType, 1.0));
+  Value zeroPValue = arith::ConstantOp::create(
+      builder, powType, builder.getIntegerAttr(powType, 0));
+  Value onePValue = arith::ConstantOp::create(
+      builder, powType, builder.getIntegerAttr(powType, 1));
+  Value minPValue = arith::ConstantOp::create(
+      builder, powType,
+      builder.getIntegerAttr(
+          powType, llvm::APInt::getSignedMinValue(powType.getWidth())));
+  Value maxPValue = arith::ConstantOp::create(
+      builder, powType,
+      builder.getIntegerAttr(
+          powType, llvm::APInt::getSignedMaxValue(powType.getWidth())));
 
   // if (p == Tp{0})
   //   return Tb{1};
-  auto pIsZero =
-      arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, zeroPValue);
+  auto pIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg,
+                                       zeroPValue);
   Block *thenBlock = builder.createBlock(funcBody);
   func::ReturnOp::create(builder, oneBValue);
   Block *fallthroughBlock = builder.createBlock(funcBody);
@@ -460,7 +464,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(fallthroughBlock);
   // bool isNegativePower{p < Tp{0}}
   auto pIsNeg = arith::CmpIOp::create(builder, arith::CmpIPredicate::sle, pArg,
-                                              zeroPValue);
+                                      zeroPValue);
   // bool isMin{p == std::numeric_limits<Tp>::min()};
   auto pIsMin =
       arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, pArg, minPValue);
@@ -498,8 +502,8 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(loopHeader);
 
   //   if (p & Tp{1})
-  auto powerTmpIsOdd = arith::CmpIOp::create(builder,
-      arith::CmpIPredicate::ne,
+  auto powerTmpIsOdd = arith::CmpIOp::create(
+      builder, arith::CmpIPredicate::ne,
       arith::AndIOp::create(builder, powerTmp, onePValue), zeroPValue);
   thenBlock = builder.createBlock(funcBody);
   //     result *= b;
@@ -511,7 +515,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   // Set up conditional branch for (p & Tp{1}).
   builder.setInsertionPointToEnd(powerTmpIsOdd->getBlock());
   cf::CondBranchOp::create(builder, powerTmpIsOdd, thenBlock, fallthroughBlock,
-                                   resultTmp);
+                           resultTmp);
   // Merged 'result'.
   newResultTmp = fallthroughBlock->getArgument(0);
 
@@ -521,7 +525,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
 
   //   if (p == Tp{0})
   auto newPowerIsZero = arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
-                                                      newPowerTmp, zeroPValue);
+                                              newPowerTmp, zeroPValue);
   //     break;
   //
   // The conditional branch is finalized below with a jump to
@@ -533,8 +537,8 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
   builder.setInsertionPointToEnd(fallthroughBlock);
   Value newBaseTmp = arith::MulFOp::create(builder, baseTmp, baseTmp);
   // Pass new values for 'result', 'b' and 'p' to the loop header.
-  cf::BranchOp::create(builder,
-      ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
+  cf::BranchOp::create(
+      builder, ValueRange{newResultTmp, newBaseTmp, newPowerTmp}, loopHeader);
 
   // Set up conditional branch for early loop exit:
   //   if (p == Tp{0})
@@ -543,7 +547,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
                                         builder.getLoc());
   builder.setInsertionPointToEnd(newPowerIsZero->getBlock());
   cf::CondBranchOp::create(builder, newPowerIsZero, loopExit, newResultTmp,
-                                   fallthroughBlock, ValueRange{});
+                           fallthroughBlock, ValueRange{});
 
   // if (isMin) {
   //   result *= origBase;
@@ -554,7 +558,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
                                          builder.getLoc());
   builder.setInsertionPointToEnd(loopExit);
   cf::CondBranchOp::create(builder, pIsMin, thenBlock, fallthroughBlock,
-                                   newResultTmp);
+                           newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
   newResultTmp = arith::MulFOp::create(builder, newResultTmp, bArg);
   cf::BranchOp::create(builder, newResultTmp, fallthroughBlock);
@@ -568,7 +572,7 @@ static func::FuncOp createElementFPowIFunc(ModuleOp *module,
                                            builder.getLoc());
   builder.setInsertionPointToEnd(fallthroughBlock);
   cf::CondBranchOp::create(builder, pIsNeg, thenBlock, returnBlock,
-                                   newResultTmp);
+                           newResultTmp);
   builder.setInsertionPointToEnd(thenBlock);
   newResultTmp = arith::DivFOp::create(builder, oneBValue, newResultTmp);
   cf::BranchOp::create(builder, newResultTmp, returnBlock);
@@ -683,33 +687,34 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
 
   Value arg = funcOp.getArgument(0);
   Type indexType = builder.getIndexType();
-  Value bitWidthValue = arith::ConstantOp::create(builder,
-      elementType, builder.getIntegerAttr(elementType, bitWidth));
-  Value zeroValue = arith::ConstantOp::create(builder,
-      elementType, builder.getIntegerAttr(elementType, 0));
+  Value bitWidthValue = arith::ConstantOp::create(
+      builder, elementType, builder.getIntegerAttr(elementType, bitWidth));
+  Value zeroValue = arith::ConstantOp::create(
+      builder, elementType, builder.getIntegerAttr(elementType, 0));
 
   Value inputEqZero =
       arith::CmpIOp::create(builder, arith::CmpIPredicate::eq, arg, zeroValue);
 
   // if input == 0, return bit width, else enter loop.
-  scf::IfOp ifOp = scf::IfOp::create(builder,
-      elementType, inputEqZero, /*addThenBlock=*/true, /*addElseBlock=*/true);
+  scf::IfOp ifOp =
+      scf::IfOp::create(builder, elementType, inputEqZero,
+                        /*addThenBlock=*/true, /*addElseBlock=*/true);
   ifOp.getThenBodyBuilder().create<scf::YieldOp>(loc, bitWidthValue);
 
   auto elseBuilder =
       ImplicitLocOpBuilder::atBlockEnd(loc, &ifOp.getElseRegion().front());
 
-  Value oneIndex = arith::ConstantOp::create(elseBuilder,
-      indexType, elseBuilder.getIndexAttr(1));
-  Value oneValue = arith::ConstantOp::create(elseBuilder,
-      elementType, elseBuilder.getIntegerAttr(elementType, 1));
-  Value bitWidthIndex = arith::ConstantOp::create(elseBuilder,
-      indexType, elseBuilder.getIndexAttr(bitWidth));
-  Value nValue = arith::ConstantOp::create(elseBuilder,
-      elementType, elseBuilder.getIntegerAttr(elementType, 0));
-
-  auto loop = scf::ForOp::create(elseBuilder,
-      oneIndex, bitWidthIndex, oneIndex,
+  Value oneIndex = arith::ConstantOp::create(elseBuilder, indexType,
+                                             elseBuilder.getIndexAttr(1));
+  Value oneValue = arith::ConstantOp::create(
+      elseBuilder, elementType, elseBuilder.getIntegerAttr(elementType, 1));
+  Value bitWidthIndex = arith::ConstantOp::create(
+      elseBuilder, indexType, elseBuilder.getIndexAttr(bitWidth));
+  Value nValue = arith::ConstantOp::create(
+      elseBuilder, elementType, elseBuilder.getIntegerAttr(elementType, 0));
+
+  auto loop = scf::ForOp::create(
+      elseBuilder, oneIndex, bitWidthIndex, oneIndex,
       // Initial values for two loop induction variables, the arg which is being
       // shifted left in each iteration, and the n value which tracks the count
       // of leading zeros.
@@ -725,10 +730,10 @@ static func::FuncOp createCtlzFunc(ModuleOp *module, Type elementType) {
         Value argIter = args[0];
         Value nIter = args[1];
 
-        Value argIsNonNegative = arith::CmpIOp::create(b,
-            loc, arith::CmpIPredicate::slt, argIter, zeroValue);
-        scf::IfOp ifOp = scf::IfOp::create(b,
-            loc, argIsNonNegative,
+        Value argIsNonNegative = arith::CmpIOp::create(
+            b, loc, arith::CmpIPredicate::slt, argIter, zeroValue);
+        scf::IfOp ifOp = scf::IfOp::create(
+            b, loc, argIsNonNegative,
             [&](OpBuilder &b, Location loc) {
               // If arg is negative, continue (effectively, break)
               scf::YieldOp::create(b, loc, ValueRange{argIter, nIter});
diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
index 26559f394c221..853f45498ac52 100644
--- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
+++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
@@ -108,7 +108,7 @@ struct IntOpWithFlagLowering : public ConvertOpToLLVMPattern<MathOp> {
         op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
           return LLVMOp::create(rewriter, loc, llvm1DVectorTy, operands[0],
-                                         false);
+                                false);
         },
         rewriter);
   }
@@ -145,15 +145,16 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (LLVM::isCompatibleVectorType(llvmOperandType)) {
-        one = LLVM::ConstantOp::create(rewriter,
-            loc, llvmOperandType,
+        one = LLVM::ConstantOp::create(
+            rewriter, loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
+        one =
+            LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
       auto exp = LLVM::ExpOp::create(rewriter, loc, adaptor.getOperand(),
-                                              expAttrs.getAttrs());
+                                     expAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FSubOp>(
           op, llvmOperandType, ValueRange{exp, one}, subAttrs.getAttrs());
       return success();
@@ -170,12 +171,13 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
               mlir::VectorType::get({numElements.getKnownMinValue()}, floatType,
                                     {numElements.isScalable()}),
               floatOne);
-          auto one =
-              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
-          auto exp = LLVM::ExpOp::create(rewriter,
-              loc, llvm1DVectorTy, operands[0], expAttrs.getAttrs());
-          return LLVM::FSubOp::create(rewriter,
-              loc, llvm1DVectorTy, ValueRange{exp, one}, subAttrs.getAttrs());
+          auto one = LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy,
+                                              splatAttr);
+          auto exp = LLVM::ExpOp::create(rewriter, loc, llvm1DVectorTy,
+                                         operands[0], expAttrs.getAttrs());
+          return LLVM::FSubOp::create(rewriter, loc, llvm1DVectorTy,
+                                      ValueRange{exp, one},
+                                      subAttrs.getAttrs());
         },
         rewriter);
   }
@@ -205,16 +207,16 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one =
           isa<VectorType>(llvmOperandType)
-              ? LLVM::ConstantOp::create(rewriter,
-                    loc, llvmOperandType,
+              ? LLVM::ConstantOp::create(
+                    rewriter, loc, llvmOperandType,
                     SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                            floatOne))
               : LLVM::ConstantOp::create(rewriter, loc, llvmOperandType,
-                                                  floatOne);
+                                         floatOne);
 
-      auto add = LLVM::FAddOp::create(rewriter,
-          loc, llvmOperandType, ValueRange{one, adaptor.getOperand()},
-          addAttrs.getAttrs());
+      auto add = LLVM::FAddOp::create(rewriter, loc, llvmOperandType,
+                                      ValueRange{one, adaptor.getOperand()},
+                                      addAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::LogOp>(
           op, llvmOperandType, ValueRange{add}, logAttrs.getAttrs());
       return success();
@@ -231,13 +233,13 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
               mlir::VectorType::get({numElements.getKnownMinValue()}, floatType,
                                     {numElements.isScalable()}),
               floatOne);
-          auto one =
-              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
+          auto one = LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy,
+                                              splatAttr);
           auto add = LLVM::FAddOp::create(rewriter, loc, llvm1DVectorTy,
-                                                   ValueRange{one, operands[0]},
-                                                   addAttrs.getAttrs());
-          return LLVM::LogOp::create(rewriter,
-              loc, llvm1DVectorTy, ValueRange{add}, logAttrs.getAttrs());
+                                          ValueRange{one, operands[0]},
+                                          addAttrs.getAttrs());
+          return LLVM::LogOp::create(rewriter, loc, llvm1DVectorTy,
+                                     ValueRange{add}, logAttrs.getAttrs());
         },
         rewriter);
   }
@@ -267,15 +269,16 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
     if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
       if (isa<VectorType>(llvmOperandType)) {
-        one = LLVM::ConstantOp::create(rewriter,
-            loc, llvmOperandType,
+        one = LLVM::ConstantOp::create(
+            rewriter, loc, llvmOperandType,
             SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                    floatOne));
       } else {
-        one = LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
+        one =
+            LLVM::ConstantOp::create(rewriter, loc, llvmOperandType, floatOne);
       }
       auto sqrt = LLVM::SqrtOp::create(rewriter, loc, adaptor.getOperand(),
-                                                sqrtAttrs.getAttrs());
+                                       sqrtAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FDivOp>(
           op, llvmOperandType, ValueRange{one, sqrt}, divAttrs.getAttrs());
       return success();
@@ -292,12 +295,13 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
               mlir::VectorType::get({numElements.getKnownMinValue()}, floatType,
                                     {numElements.isScalable()}),
               floatOne);
-          auto one =
-              LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy, splatAttr);
-          auto sqrt = LLVM::SqrtOp::create(rewriter,
-              loc, llvm1DVectorTy, operands[0], sqrtAttrs.getAttrs());
-          return LLVM::FDivOp::create(rewriter,
-              loc, llvm1DVectorTy, ValueRange{one, sqrt}, divAttrs.getAttrs());
+          auto one = LLVM::ConstantOp::create(rewriter, loc, llvm1DVectorTy,
+                                              splatAttr);
+          auto sqrt = LLVM::SqrtOp::create(rewriter, loc, llvm1DVectorTy,
+                                           operands[0], sqrtAttrs.getAttrs());
+          return LLVM::FDivOp::create(rewriter, loc, llvm1DVectorTy,
+                                      ValueRange{one, sqrt},
+                                      divAttrs.getAttrs());
         },
         rewriter);
   }
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index 19303dc3e8b55..f7c0d4fe3a799 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -84,9 +84,10 @@ VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
   auto shape = vecType.getShape();
   int64_t numElements = vecType.getNumElements();
 
-  Value result = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(
-               vecType, FloatAttr::get(vecType.getElementType(), 0.0)));
+  Value result = arith::ConstantOp::create(
+      rewriter, loc,
+      DenseElementsAttr::get(vecType,
+                             FloatAttr::get(vecType.getElementType(), 0.0)));
   SmallVector<int64_t> strides = computeStrides(shape);
   for (auto linearIndex = 0; linearIndex < numElements; ++linearIndex) {
     SmallVector<int64_t> positions = delinearize(linearIndex, strides);
@@ -140,7 +141,7 @@ ScalarOpToLibmCall<Op>::matchAndRewrite(Op op,
     auto opFunctionTy = FunctionType::get(
         rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
     opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
-                                           opFunctionTy);
+                                  opFunctionTy);
     opFunc.setPrivate();
 
     // By definition Math dialect operations imply LLVM's "readnone"
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 26891aa7c2025..a877ad21734a2 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -37,11 +37,11 @@ static Value getScalarOrVectorI32Constant(Type type, int value,
       return nullptr;
     SmallVector<int> values(vectorType.getNumElements(), value);
     return spirv::ConstantOp::create(builder, loc, type,
-                                             builder.getI32VectorAttr(values));
+                                     builder.getI32VectorAttr(values));
   }
   if (type.isInteger(32))
     return spirv::ConstantOp::create(builder, loc, type,
-                                             builder.getI32IntegerAttr(value));
+                                     builder.getI32IntegerAttr(value));
 
   return nullptr;
 }
@@ -144,10 +144,11 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
     Type intType = rewriter.getIntegerType(bitwidth);
     uint64_t intValue = uint64_t(1) << (bitwidth - 1);
 
-    Value signMask = spirv::ConstantOp::create(rewriter,
-        loc, intType, rewriter.getIntegerAttr(intType, intValue));
-    Value valueMask = spirv::ConstantOp::create(rewriter,
-        loc, intType, rewriter.getIntegerAttr(intType, intValue - 1u));
+    Value signMask = spirv::ConstantOp::create(
+        rewriter, loc, intType, rewriter.getIntegerAttr(intType, intValue));
+    Value valueMask = spirv::ConstantOp::create(
+        rewriter, loc, intType,
+        rewriter.getIntegerAttr(intType, intValue - 1u));
 
     if (auto vectorType = dyn_cast<VectorType>(type)) {
       assert(vectorType.getRank() == 1);
@@ -155,12 +156,12 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
       intType = VectorType::get(count, intType);
 
       SmallVector<Value> signSplat(count, signMask);
-      signMask =
-          spirv::CompositeConstructOp::create(rewriter, loc, intType, signSplat);
+      signMask = spirv::CompositeConstructOp::create(rewriter, loc, intType,
+                                                     signSplat);
 
       SmallVector<Value> valueSplat(count, valueMask);
       valueMask = spirv::CompositeConstructOp::create(rewriter, loc, intType,
-                                                               valueSplat);
+                                                      valueSplat);
     }
 
     Value lhsCast =
@@ -168,13 +169,13 @@ struct CopySignPattern final : public OpConversionPattern<math::CopySignOp> {
     Value rhsCast =
         spirv::BitcastOp::create(rewriter, loc, intType, adaptor.getRhs());
 
-    Value value = spirv::BitwiseAndOp::create(rewriter,
-        loc, intType, ValueRange{lhsCast, valueMask});
-    Value sign = spirv::BitwiseAndOp::create(rewriter,
-        loc, intType, ValueRange{rhsCast, signMask});
+    Value value = spirv::BitwiseAndOp::create(rewriter, loc, intType,
+                                              ValueRange{lhsCast, valueMask});
+    Value sign = spirv::BitwiseAndOp::create(rewriter, loc, intType,
+                                             ValueRange{rhsCast, signMask});
 
     Value result = spirv::BitwiseOrOp::create(rewriter, loc, intType,
-                                                       ValueRange{value, sign});
+                                              ValueRange{value, sign});
     rewriter.replaceOpWithNewOp<spirv::BitcastOp>(copySignOp, type, result);
     return success();
   }
@@ -321,15 +322,15 @@ struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
 
     auto getConstantValue = [&](double value) {
       if (auto floatType = dyn_cast<FloatType>(type)) {
-        return spirv::ConstantOp::create(rewriter,
-            loc, type, rewriter.getFloatAttr(floatType, value));
+        return spirv::ConstantOp::create(
+            rewriter, loc, type, rewriter.getFloatAttr(floatType, value));
       }
       if (auto vectorType = dyn_cast<VectorType>(type)) {
         Type elemType = vectorType.getElementType();
 
         if (isa<FloatType>(elemType)) {
-          return spirv::ConstantOp::create(rewriter,
-              loc, type,
+          return spirv::ConstantOp::create(
+              rewriter, loc, type,
               DenseFPElementsAttr::get(
                   vectorType, FloatAttr::get(elemType, value).getValue()));
         }
@@ -408,8 +409,9 @@ struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
 
     Value NanValue =
         spirv::ConstantOp::create(rewriter, loc, operandType, nanAttr);
-    Value lhs = spirv::SelectOp::create(rewriter,
-        loc, cmpNegativeWithFractionalExp, NanValue, adaptor.getLhs());
+    Value lhs =
+        spirv::SelectOp::create(rewriter, loc, cmpNegativeWithFractionalExp,
+                                NanValue, adaptor.getLhs());
     Value abs = spirv::GLFAbsOp::create(rewriter, loc, lhs);
 
     // TODO: The following just forcefully casts y into an integer value in
@@ -455,13 +457,13 @@ struct RoundOpPattern final : public OpConversionPattern<math::RoundOp> {
     auto one = spirv::ConstantOp::getOne(ty, loc, rewriter);
     Value half;
     if (VectorType vty = dyn_cast<VectorType>(ty)) {
-      half = spirv::ConstantOp::create(rewriter,
-          loc, vty,
+      half = spirv::ConstantOp::create(
+          rewriter, loc, vty,
           DenseElementsAttr::get(vty,
                                  rewriter.getFloatAttr(ety, 0.5).getValue()));
     } else {
-      half = spirv::ConstantOp::create(rewriter,
-          loc, ty, rewriter.getFloatAttr(ety, 0.5));
+      half = spirv::ConstantOp::create(rewriter, loc, ty,
+                                       rewriter.getFloatAttr(ety, 0.5));
     }
 
     auto abs = spirv::GLFAbsOp::create(rewriter, loc, operand);
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
index db6ce600af76a..bd6136db4376a 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
@@ -161,8 +161,8 @@ struct ConvertLoad final : public OpConversionPattern<memref::LoadOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = emitc::SubscriptOp::create(rewriter,
-        op.getLoc(), arrayValue, operands.getIndices());
+    auto subscript = emitc::SubscriptOp::create(
+        rewriter, op.getLoc(), arrayValue, operands.getIndices());
 
     rewriter.replaceOpWithNewOp<emitc::LoadOp>(op, resultTy, subscript);
     return success();
@@ -181,8 +181,8 @@ struct ConvertStore final : public OpConversionPattern<memref::StoreOp> {
       return rewriter.notifyMatchFailure(op.getLoc(), "expected array type");
     }
 
-    auto subscript = emitc::SubscriptOp::create(rewriter,
-        op.getLoc(), arrayValue, operands.getIndices());
+    auto subscript = emitc::SubscriptOp::create(
+        rewriter, op.getLoc(), arrayValue, operands.getIndices());
     rewriter.replaceOpWithNewOp<emitc::AssignOp>(op, subscript,
                                                  operands.getValue());
     return success();
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 4ef2168380620..53a19129103a3 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -88,7 +88,7 @@ getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
 static Value createAligned(ConversionPatternRewriter &rewriter, Location loc,
                            Value input, Value alignment) {
   Value one = LLVM::ConstantOp::create(rewriter, loc, alignment.getType(),
-                                                rewriter.getIndexAttr(1));
+                                       rewriter.getIndexAttr(1));
   Value bump = LLVM::SubOp::create(rewriter, loc, alignment, one);
   Value bumped = LLVM::AddOp::create(rewriter, loc, input, bump);
   Value mod = LLVM::URemOp::create(rewriter, loc, bumped, alignment);
@@ -123,8 +123,9 @@ static Value castAllocFuncResult(ConversionPatternRewriter &rewriter,
   assert(succeeded(maybeMemrefAddrSpace) && "unsupported address space");
   unsigned memrefAddrSpace = *maybeMemrefAddrSpace;
   if (allocatedPtrTy.getAddressSpace() != memrefAddrSpace)
-    allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
-        loc, LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
+    allocatedPtr = LLVM::AddrSpaceCastOp::create(
+        rewriter, loc,
+        LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
         allocatedPtr);
   return allocatedPtr;
 }
@@ -268,8 +269,9 @@ class AlignedAllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
       sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
 
     Type elementPtrType = this->getElementPtrType(memRefType);
-    auto results = LLVM::CallOp::create(rewriter,
-        loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
+    auto results =
+        LLVM::CallOp::create(rewriter, loc, allocFuncOp.value(),
+                             ValueRange({allocAlignment, sizeBytes}));
 
     Value ptr =
         castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
@@ -360,8 +362,9 @@ struct AllocaOpLowering : public ConvertOpToLLVMPattern<memref::AllocaOp> {
     auto elementPtrType =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addrSpace);
 
-    auto allocatedElementPtr = LLVM::AllocaOp::create(rewriter,
-        loc, elementPtrType, elementType, size, op.getAlignment().value_or(0));
+    auto allocatedElementPtr =
+        LLVM::AllocaOp::create(rewriter, loc, elementPtrType, elementType, size,
+                               op.getAlignment().value_or(0));
 
     // Create the MemRef descriptor.
     auto memRefDescriptor = this->createMemRefDescriptor(
@@ -455,7 +458,7 @@ struct AssumeAlignmentOpLowering
     Value alignmentConst =
         createIndexAttrConstant(rewriter, loc, getIndexType(), alignment);
     LLVM::AssumeOp::create(rewriter, loc, trueCond, LLVM::AssumeAlignTag(), ptr,
-                                    alignmentConst);
+                           alignmentConst);
     rewriter.replaceOp(op, memref);
     return success();
   }
@@ -559,18 +562,19 @@ struct DimOpLowering : public ConvertOpToLLVMPattern<memref::DimOp> {
     // Get pointer to offset field of memref<element_type> descriptor.
     auto indexPtrTy =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addressSpace);
-    Value offsetPtr = LLVM::GEPOp::create(rewriter,
-        loc, indexPtrTy, elementType, underlyingRankedDesc,
-        ArrayRef<LLVM::GEPArg>{0, 2});
+    Value offsetPtr =
+        LLVM::GEPOp::create(rewriter, loc, indexPtrTy, elementType,
+                            underlyingRankedDesc, ArrayRef<LLVM::GEPArg>{0, 2});
 
     // The size value that we have to extract can be obtained using GEPop with
     // `dimOp.index() + 1` index argument.
-    Value idxPlusOne = LLVM::AddOp::create(rewriter,
-        loc, createIndexAttrConstant(rewriter, loc, getIndexType(), 1),
+    Value idxPlusOne = LLVM::AddOp::create(
+        rewriter, loc,
+        createIndexAttrConstant(rewriter, loc, getIndexType(), 1),
         adaptor.getIndex());
-    Value sizePtr = LLVM::GEPOp::create(rewriter,
-        loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr,
-        idxPlusOne);
+    Value sizePtr = LLVM::GEPOp::create(rewriter, loc, indexPtrTy,
+                                        getTypeConverter()->getIndexType(),
+                                        offsetPtr, idxPlusOne);
     return rewriter
         .create<LLVM::LoadOp>(loc, getTypeConverter()->getIndexType(), sizePtr)
         .getResult();
@@ -674,8 +678,9 @@ struct GenericAtomicRMWOpLowering
     auto memRefType = cast<MemRefType>(atomicOp.getMemref().getType());
     auto dataPtr = getStridedElementPtr(
         rewriter, loc, memRefType, adaptor.getMemref(), adaptor.getIndices());
-    Value init = LLVM::LoadOp::create(rewriter,
-        loc, typeConverter->convertType(memRefType.getElementType()), dataPtr);
+    Value init = LLVM::LoadOp::create(
+        rewriter, loc, typeConverter->convertType(memRefType.getElementType()),
+        dataPtr);
     LLVM::BrOp::create(rewriter, loc, init, loopBlock);
 
     // Prepare the body of the loop block.
@@ -696,15 +701,16 @@ struct GenericAtomicRMWOpLowering
     // Append the cmpxchg op to the end of the loop block.
     auto successOrdering = LLVM::AtomicOrdering::acq_rel;
     auto failureOrdering = LLVM::AtomicOrdering::monotonic;
-    auto cmpxchg = LLVM::AtomicCmpXchgOp::create(rewriter,
-        loc, dataPtr, loopArgument, result, successOrdering, failureOrdering);
+    auto cmpxchg =
+        LLVM::AtomicCmpXchgOp::create(rewriter, loc, dataPtr, loopArgument,
+                                      result, successOrdering, failureOrdering);
     // Extract the %new_loaded and %ok values from the pair.
     Value newLoaded = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 0);
     Value ok = LLVM::ExtractValueOp::create(rewriter, loc, cmpxchg, 1);
 
     // Conditionally branch to the end or back to the loop depending on %ok.
     LLVM::CondBrOp::create(rewriter, loc, ok, endBlock, ArrayRef<Value>(),
-                                    loopBlock, newLoaded);
+                           loopBlock, newLoaded);
 
     rewriter.setInsertionPointToEnd(endBlock);
 
@@ -846,9 +852,9 @@ struct GetGlobalMemrefOpLowering
 
     // Get the address of the first element in the array by creating a GEP with
     // the address of the GV as the base, and (rank + 1) number of 0 indices.
-    auto gep = LLVM::GEPOp::create(rewriter,
-        loc, ptrTy, arrayTy, addressOf,
-        SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
+    auto gep =
+        LLVM::GEPOp::create(rewriter, loc, ptrTy, arrayTy, addressOf,
+                            SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
 
     // We do not expect the memref obtained using `memref.get_global` to be
     // ever deallocated. Set the allocated pointer to be known bad value to
@@ -1009,8 +1015,8 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
           loc, adaptor.getSource(), rewriter);
 
       // rank = ConstantOp srcRank
-      auto rankVal = LLVM::ConstantOp::create(rewriter,
-          loc, getIndexType(), rewriter.getIndexAttr(rank));
+      auto rankVal = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                              rewriter.getIndexAttr(rank));
       // poison = PoisonOp
       UnrankedMemRefDescriptor memRefDesc =
           UnrankedMemRefDescriptor::poison(rewriter, loc, targetStructType);
@@ -1063,8 +1069,8 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     MemRefDescriptor srcDesc(adaptor.getSource());
 
     // Compute number of elements.
-    Value numElements = LLVM::ConstantOp::create(rewriter,
-        loc, getIndexType(), rewriter.getIndexAttr(1));
+    Value numElements = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
+                                                 rewriter.getIndexAttr(1));
     for (int pos = 0; pos < srcType.getRank(); ++pos) {
       auto size = srcDesc.size(rewriter, loc, pos);
       numElements = LLVM::MulOp::create(rewriter, loc, numElements, size);
@@ -1080,15 +1086,16 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
 
     Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc);
     Value srcOffset = srcDesc.offset(rewriter, loc);
-    Value srcPtr = LLVM::GEPOp::create(rewriter,
-        loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset);
+    Value srcPtr = LLVM::GEPOp::create(rewriter, loc, srcBasePtr.getType(),
+                                       elementType, srcBasePtr, srcOffset);
     MemRefDescriptor targetDesc(adaptor.getTarget());
     Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc);
     Value targetOffset = targetDesc.offset(rewriter, loc);
-    Value targetPtr = LLVM::GEPOp::create(rewriter,
-        loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset);
+    Value targetPtr =
+        LLVM::GEPOp::create(rewriter, loc, targetBasePtr.getType(), elementType,
+                            targetBasePtr, targetOffset);
     LLVM::MemcpyOp::create(rewriter, loc, targetPtr, srcPtr, totalSize,
-                                    /*isVolatile=*/false);
+                           /*isVolatile=*/false);
     rewriter.eraseOp(op);
 
     return success();
@@ -1104,7 +1111,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     // First make sure we have an unranked memref descriptor representation.
     auto makeUnranked = [&, this](Value ranked, MemRefType type) {
       auto rank = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                    type.getRank());
+                                           type.getRank());
       auto *typeConverter = getTypeConverter();
       auto ptr =
           typeConverter->promoteOneMemRefDescriptor(loc, ranked, rewriter);
@@ -1129,7 +1136,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
 
     // Now promote the unranked descriptors to the stack.
     auto one = LLVM::ConstantOp::create(rewriter, loc, getIndexType(),
-                                                 rewriter.getIndexAttr(1));
+                                        rewriter.getIndexAttr(1));
     auto promote = [&](Value desc) {
       auto ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
       auto allocated =
@@ -1150,7 +1157,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     if (failed(copyFn))
       return failure();
     LLVM::CallOp::create(rewriter, loc, copyFn.value(),
-                                  ValueRange{elemSize, sourcePtr, targetPtr});
+                         ValueRange{elemSize, sourcePtr, targetPtr});
 
     // Restore stack used for descriptors
     LLVM::StackRestoreOp::create(rewriter, loc, stackSaveOp);
@@ -1241,8 +1248,9 @@ struct MemorySpaceCastOpLowering
       UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                              result, resultAddrSpace, sizes);
       Value resultUnderlyingSize = sizes.front();
-      Value resultUnderlyingDesc = LLVM::AllocaOp::create(rewriter,
-          loc, getPtrType(), rewriter.getI8Type(), resultUnderlyingSize);
+      Value resultUnderlyingDesc =
+          LLVM::AllocaOp::create(rewriter, loc, getPtrType(),
+                                 rewriter.getI8Type(), resultUnderlyingSize);
       result.setMemRefDescPtr(rewriter, loc, resultUnderlyingDesc);
 
       // Copy pointers, performing address space casts.
@@ -1256,10 +1264,10 @@ struct MemorySpaceCastOpLowering
       Value alignedPtr =
           sourceDesc.alignedPtr(rewriter, loc, *getTypeConverter(),
                                 sourceUnderlyingDesc, sourceElemPtrType);
-      allocatedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
-          loc, resultElemPtrType, allocatedPtr);
-      alignedPtr = LLVM::AddrSpaceCastOp::create(rewriter,
-          loc, resultElemPtrType, alignedPtr);
+      allocatedPtr = LLVM::AddrSpaceCastOp::create(
+          rewriter, loc, resultElemPtrType, allocatedPtr);
+      alignedPtr = LLVM::AddrSpaceCastOp::create(rewriter, loc,
+                                                 resultElemPtrType, alignedPtr);
 
       result.setAllocatedPtr(rewriter, loc, resultUnderlyingDesc,
                              resultElemPtrType, allocatedPtr);
@@ -1277,12 +1285,13 @@ struct MemorySpaceCastOpLowering
       int64_t bytesToSkip =
           2 * llvm::divideCeil(
                   getTypeConverter()->getPointerBitwidth(resultAddrSpace), 8);
-      Value bytesToSkipConst = LLVM::ConstantOp::create(rewriter,
-          loc, getIndexType(), rewriter.getIndexAttr(bytesToSkip));
-      Value copySize = LLVM::SubOp::create(rewriter,
-          loc, getIndexType(), resultUnderlyingSize, bytesToSkipConst);
+      Value bytesToSkipConst = LLVM::ConstantOp::create(
+          rewriter, loc, getIndexType(), rewriter.getIndexAttr(bytesToSkip));
+      Value copySize =
+          LLVM::SubOp::create(rewriter, loc, getIndexType(),
+                              resultUnderlyingSize, bytesToSkipConst);
       LLVM::MemcpyOp::create(rewriter, loc, resultIndexVals, sourceIndexVals,
-                                      copySize, /*isVolatile=*/false);
+                             copySize, /*isVolatile=*/false);
 
       rewriter.replaceOp(op, ValueRange{result});
       return success();
@@ -1522,8 +1531,9 @@ struct MemRefReshapeOpLowering
     SmallVector<Value, 4> sizes;
     UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                            targetDesc, addressSpace, sizes);
-    Value underlyingDescPtr = LLVM::AllocaOp::create(rewriter,
-        loc, getPtrType(), IntegerType::get(getContext(), 8), sizes.front());
+    Value underlyingDescPtr = LLVM::AllocaOp::create(
+        rewriter, loc, getPtrType(), IntegerType::get(getContext(), 8),
+        sizes.front());
     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
 
     // Extract pointers and offset from the source memref.
@@ -1568,15 +1578,15 @@ struct MemRefReshapeOpLowering
     rewriter.mergeBlocks(remainingBlock, condBlock, ValueRange());
 
     rewriter.setInsertionPointToEnd(initBlock);
-    LLVM::BrOp::create(rewriter, loc, ValueRange({resultRankMinusOne, oneIndex}),
-                                condBlock);
+    LLVM::BrOp::create(rewriter, loc,
+                       ValueRange({resultRankMinusOne, oneIndex}), condBlock);
     rewriter.setInsertionPointToStart(condBlock);
     Value indexArg = condBlock->getArgument(0);
     Value strideArg = condBlock->getArgument(1);
 
     Value zeroIndex = createIndexAttrConstant(rewriter, loc, indexType, 0);
-    Value pred = LLVM::ICmpOp::create(rewriter,
-        loc, IntegerType::get(rewriter.getContext(), 1),
+    Value pred = LLVM::ICmpOp::create(
+        rewriter, loc, IntegerType::get(rewriter.getContext(), 1),
         LLVM::ICmpPredicate::sge, indexArg, zeroIndex);
 
     Block *bodyBlock =
@@ -1585,8 +1595,8 @@ struct MemRefReshapeOpLowering
 
     // Copy size from shape to descriptor.
     auto llvmIndexPtrType = LLVM::LLVMPointerType::get(rewriter.getContext());
-    Value sizeLoadGep = LLVM::GEPOp::create(rewriter,
-        loc, llvmIndexPtrType,
+    Value sizeLoadGep = LLVM::GEPOp::create(
+        rewriter, loc, llvmIndexPtrType,
         typeConverter->convertType(shapeMemRefType.getElementType()),
         shapeOperandPtr, indexArg);
     Value size = LLVM::LoadOp::create(rewriter, loc, indexType, sizeLoadGep);
@@ -1601,7 +1611,7 @@ struct MemRefReshapeOpLowering
     // Decrement loop counter and branch back.
     Value decrement = LLVM::SubOp::create(rewriter, loc, indexArg, oneIndex);
     LLVM::BrOp::create(rewriter, loc, ValueRange({decrement, nextStride}),
-                                condBlock);
+                       condBlock);
 
     Block *remainder =
         rewriter.splitBlock(bodyBlock, rewriter.getInsertionPoint());
@@ -1609,7 +1619,7 @@ struct MemRefReshapeOpLowering
     // Hook up the cond exit to the remainder.
     rewriter.setInsertionPointToEnd(condBlock);
     LLVM::CondBrOp::create(rewriter, loc, pred, bodyBlock, ValueRange(),
-                                    remainder, ValueRange());
+                           remainder, ValueRange());
 
     // Reset position to beginning of new remainder block.
     rewriter.setInsertionPointToStart(remainder);
@@ -1783,8 +1793,8 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
 
     // Field 2: Copy the actual aligned pointer to payload.
     Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc);
-    alignedPtr = LLVM::GEPOp::create(rewriter,
-        loc, alignedPtr.getType(),
+    alignedPtr = LLVM::GEPOp::create(
+        rewriter, loc, alignedPtr.getType(),
         typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr,
         adaptor.getByteShift());
 
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index 7b7ad9b95b0ec..7a705336bf11c 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -79,7 +79,8 @@ adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter,
   assert(indices.size() == 2);
   indices.back() = builder.createOrFold<spirv::SDivOp>(loc, lastDim, idx);
   Type t = typeConverter.convertType(op.getComponentPtr().getType());
-  return spirv::AccessChainOp::create(builder, loc, t, op.getBasePtr(), indices);
+  return spirv::AccessChainOp::create(builder, loc, t, op.getBasePtr(),
+                                      indices);
 }
 
 /// Casts the given `srcBool` into an integer of `dstType`.
@@ -107,8 +108,8 @@ static Value shiftValue(Location loc, Value value, Value offset, Value mask,
     value = castBoolToIntN(loc, value, dstType, builder);
   } else {
     if (valueBits < targetBits) {
-      value = spirv::UConvertOp::create(builder,
-          loc, builder.getIntegerType(targetBits), value);
+      value = spirv::UConvertOp::create(
+          builder, loc, builder.getIntegerType(targetBits), value);
     }
 
     value = builder.createOrFold<spirv::BitwiseAndOp>(loc, value, mask);
@@ -373,7 +374,7 @@ AllocOpPattern::matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor,
         std::string("__workgroup_mem__") +
         std::to_string(std::distance(varOps.begin(), varOps.end()));
     varOp = spirv::GlobalVariableOp::create(rewriter, loc, spirvType, varName,
-                                                     /*initializer=*/nullptr);
+                                            /*initializer=*/nullptr);
   }
 
   // Get pointer to global variable at the current scope.
@@ -573,7 +574,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
 
     auto [memoryAccess, alignment] = *memoryRequirements;
     Value loadVal = spirv::LoadOp::create(rewriter, loc, accessChain,
-                                                   memoryAccess, alignment);
+                                          memoryAccess, alignment);
     if (isBool)
       loadVal = castIntNToBool(loc, loadVal, rewriter);
     rewriter.replaceOp(loadOp, loadVal);
@@ -602,7 +603,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
 
   auto [memoryAccess, alignment] = *memoryRequirements;
   Value spvLoadOp = spirv::LoadOp::create(rewriter, loc, dstType, adjustedPtr,
-                                                   memoryAccess, alignment);
+                                          memoryAccess, alignment);
 
   // Shift the bits to the rightmost.
   // ____XXXX________ -> ____________XXXX
@@ -770,12 +771,12 @@ IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor,
   if (!scope)
     return rewriter.notifyMatchFailure(storeOp, "atomic scope not available");
 
-  Value result = spirv::AtomicAndOp::create(rewriter,
-      loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
-      clearBitsMask);
-  result = spirv::AtomicOrOp::create(rewriter,
-      loc, dstType, adjustedPtr, *scope, spirv::MemorySemantics::AcquireRelease,
-      storeVal);
+  Value result = spirv::AtomicAndOp::create(
+      rewriter, loc, dstType, adjustedPtr, *scope,
+      spirv::MemorySemantics::AcquireRelease, clearBitsMask);
+  result = spirv::AtomicOrOp::create(
+      rewriter, loc, dstType, adjustedPtr, *scope,
+      spirv::MemorySemantics::AcquireRelease, storeVal);
 
   // The AtomicOrOp has no side effect. Since it is already inserted, we can
   // just remove the original StoreOp. Note that rewriter.replaceOp()
@@ -850,8 +851,8 @@ LogicalResult MemorySpaceCastOpPattern::matchAndRewrite(
     genericPtrType = typeConverter.convertType(intermediateType);
   }
   if (sourceSc != spirv::StorageClass::Generic) {
-    result =
-        spirv::PtrCastToGenericOp::create(rewriter, loc, genericPtrType, result);
+    result = spirv::PtrCastToGenericOp::create(rewriter, loc, genericPtrType,
+                                               result);
   }
   if (resultSc != spirv::StorageClass::Generic) {
     result =
diff --git a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
index f16b92a287d0a..63b1fdabaf407 100644
--- a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
+++ b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
@@ -147,8 +147,9 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     Value resSplitAxes = tensor::EmptyOp::create(rewriter, loc, shape, i16);
     auto attr = IntegerAttr::get(i16, -1);
     Value fillValue = arith::ConstantOp::create(rewriter, loc, i16, attr);
-    resSplitAxes = linalg::FillOp::create(rewriter, loc, fillValue, resSplitAxes)
-                       .getResult(0);
+    resSplitAxes =
+        linalg::FillOp::create(rewriter, loc, fillValue, resSplitAxes)
+            .getResult(0);
 
     // explicitly write values into tensor row by row
     std::array<int64_t, 2> strides = {1, 1};
@@ -163,8 +164,9 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       auto tensorType = RankedTensorType::get({size}, i16);
       auto attrs = DenseIntElementsAttr::get(tensorType, axes.asArrayRef());
       auto vals = arith::ConstantOp::create(rewriter, loc, tensorType, attrs);
-      resSplitAxes = tensor::InsertSliceOp::create(rewriter,
-          loc, vals, resSplitAxes, empty, empty, empty, offs, sizes, strides);
+      resSplitAxes = tensor::InsertSliceOp::create(rewriter, loc, vals,
+                                                   resSplitAxes, empty, empty,
+                                                   empty, offs, sizes, strides);
     }
 
     // To hold halos sizes, create 2d Tensor with shape {nSplits, 2}.
@@ -189,8 +191,8 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
     // MeshOp)
     Value resOffsets;
     if (adaptor.getStaticShardedDimsOffsets().empty()) {
-      resOffsets = tensor::EmptyOp::create(rewriter,
-          loc, std::array<int64_t, 2>{0, 0}, i64);
+      resOffsets = tensor::EmptyOp::create(rewriter, loc,
+                                           std::array<int64_t, 2>{0, 0}, i64);
     } else {
       SymbolTableCollection symbolTableCollection;
       auto meshOp = getMesh(op, symbolTableCollection);
@@ -204,10 +206,10 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
       assert(maxSplitSize);
       ++maxSplitSize; // add one for the total size
 
-      resOffsets = tensor::EmptyOp::create(rewriter,
-          loc, std::array<int64_t, 2>{nSplits, maxSplitSize}, i64);
-      Value zero = arith::ConstantOp::create(rewriter,
-          loc, i64, rewriter.getI64IntegerAttr(ShapedType::kDynamic));
+      resOffsets = tensor::EmptyOp::create(
+          rewriter, loc, std::array<int64_t, 2>{nSplits, maxSplitSize}, i64);
+      Value zero = arith::ConstantOp::create(
+          rewriter, loc, i64, rewriter.getI64IntegerAttr(ShapedType::kDynamic));
       resOffsets =
           linalg::FillOp::create(rewriter, loc, zero, resOffsets).getResult(0);
       SmallVector<Value> offsets =
@@ -223,8 +225,9 @@ struct ConvertShardingOp : public OpConversionPattern<ShardingOp> {
         Value vals = tensor::FromElementsOp::create(rewriter, loc, values);
         std::array<int64_t, 2> offs = {static_cast<int64_t>(i), 0};
         std::array<int64_t, 2> sizes = {1, splitSize};
-        resOffsets = tensor::InsertSliceOp::create(rewriter,
-            loc, vals, resOffsets, empty, empty, empty, offs, sizes, strides);
+        resOffsets = tensor::InsertSliceOp::create(rewriter, loc, vals,
+                                                   resOffsets, empty, empty,
+                                                   empty, offs, sizes, strides);
         curr += splitSize;
       }
     }
@@ -346,11 +349,11 @@ struct ConvertNeighborsLinearIndicesOp
     Value dimSz = dims[axes[0]];
     Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
     Value minus1 = arith::ConstantIndexOp::create(rewriter, loc, -1);
-    Value atBorder = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sle, orgIdx,
-        arith::ConstantIndexOp::create(rewriter, loc, 0));
-    auto down = scf::IfOp::create(rewriter,
-        loc, atBorder,
+    Value atBorder =
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sle, orgIdx,
+                              arith::ConstantIndexOp::create(rewriter, loc, 0));
+    auto down = scf::IfOp::create(
+        rewriter, loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
           scf::YieldOp::create(builder, loc, minus1);
         },
@@ -359,14 +362,14 @@ struct ConvertNeighborsLinearIndicesOp
           tmp[axes[0]] =
               arith::SubIOp::create(rewriter, op.getLoc(), orgIdx, one)
                   .getResult();
-          scf::YieldOp::create(builder,
-              loc, multiToLinearIndex(loc, rewriter, tmp, dims));
+          scf::YieldOp::create(builder, loc,
+                               multiToLinearIndex(loc, rewriter, tmp, dims));
         });
-    atBorder = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sge, orgIdx,
+    atBorder = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sge, orgIdx,
         arith::SubIOp::create(rewriter, loc, dimSz, one).getResult());
-    auto up = scf::IfOp::create(rewriter,
-        loc, atBorder,
+    auto up = scf::IfOp::create(
+        rewriter, loc, atBorder,
         [&](OpBuilder &builder, Location loc) {
           scf::YieldOp::create(builder, loc, minus1);
         },
@@ -374,8 +377,8 @@ struct ConvertNeighborsLinearIndicesOp
           SmallVector<Value> tmp = mIdx;
           tmp[axes[0]] =
               arith::AddIOp::create(rewriter, op.getLoc(), orgIdx, one);
-          scf::YieldOp::create(builder,
-              loc, multiToLinearIndex(loc, rewriter, tmp, dims));
+          scf::YieldOp::create(builder, loc,
+                               multiToLinearIndex(loc, rewriter, tmp, dims));
         });
     rewriter.replaceOp(op, ValueRange{down.getResult(0), up.getResult(0)});
     return success();
@@ -447,8 +450,9 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           rewriter, loc, sharding.getStaticShardedDimsOffsets(),
           sharding.getDynamicShardedDimsOffsets(), index);
       if (!tmp.empty())
-        shardedDimsOffs = tensor::FromElementsOp::create(rewriter,
-            loc, RankedTensorType::get({(int64_t)tmp.size()}, index), tmp);
+        shardedDimsOffs = tensor::FromElementsOp::create(
+            rewriter, loc, RankedTensorType::get({(int64_t)tmp.size()}, index),
+            tmp);
     }
 
     // With static mesh shape the sizes of the split axes are known.
@@ -469,8 +473,8 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
         auto axes = splitAxes[i];
         // The current dimension might not be sharded.
         // Create a value from the static position in shardDimsOffsets.
-        Value posVal =
-            arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(pos));
+        Value posVal = arith::ConstantOp::create(rewriter, loc,
+                                                 rewriter.getIndexAttr(pos));
         // Get the index of the local shard in the mesh axis.
         Value idx = multiIdx[axes[0]];
         auto numShards =
@@ -492,8 +496,8 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           Value sz = arith::SubIOp::create(rewriter, loc, nextOff, off);
           shardShape.emplace_back(sz);
         } else {
-          Value numShardsVal = arith::ConstantOp::create(rewriter,
-              loc, rewriter.getIndexAttr(numShards));
+          Value numShardsVal = arith::ConstantOp::create(
+              rewriter, loc, rewriter.getIndexAttr(numShards));
           // Compute shard dim size by distributing odd elements to trailing
           // shards:
           // sz = dim / numShards
@@ -501,8 +505,8 @@ struct ConvertShardShapeOp : public OpConversionPattern<ShardShapeOp> {
           Value sz = arith::DivSIOp::create(rewriter, loc, dim, numShardsVal);
           Value sz1 = arith::RemSIOp::create(rewriter, loc, dim, numShardsVal);
           sz1 = arith::SubIOp::create(rewriter, loc, numShardsVal, sz1);
-          auto cond = arith::CmpIOp::create(rewriter,
-              loc, arith::CmpIPredicate::sge, idx, sz1);
+          auto cond = arith::CmpIOp::create(
+              rewriter, loc, arith::CmpIPredicate::sge, idx, sz1);
           Value odd = arith::SelectOp::create(rewriter, loc, cond, one, zero);
           sz = arith::AddIOp::create(rewriter, loc, sz, odd);
           shardShape.emplace_back(sz);
@@ -583,8 +587,8 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     }
 
     // Allocate buffer and copy input to buffer.
-    Value buffer = memref::AllocOp::create(iBuilder,
-        shape, cast<ShapedType>(op.getType()).getElementType());
+    Value buffer = memref::AllocOp::create(
+        iBuilder, shape, cast<ShapedType>(op.getType()).getElementType());
     linalg::CopyOp::create(iBuilder, input, buffer);
 
     // Get an MPI_Comm_split for the AllReduce operation.
@@ -623,19 +627,19 @@ struct ConvertAllReduceOp : public OpConversionPattern<AllReduceOp> {
     if (inType.getRank() > 1) {
       ReassociationIndices reassociation(inType.getRank());
       std::iota(reassociation.begin(), reassociation.end(), 0);
-      buffer1d = memref::CollapseShapeOp::create(iBuilder,
-          buffer, ArrayRef<ReassociationIndices>(reassociation));
+      buffer1d = memref::CollapseShapeOp::create(
+          iBuilder, buffer, ArrayRef<ReassociationIndices>(reassociation));
     }
 
     // Create the MPI AllReduce operation.
-    mpi::AllReduceOp::create(iBuilder,
-        TypeRange(), buffer1d, buffer1d,
-        getMPIReductionOp(adaptor.getReductionAttr()), comm);
+    mpi::AllReduceOp::create(iBuilder, TypeRange(), buffer1d, buffer1d,
+                             getMPIReductionOp(adaptor.getReductionAttr()),
+                             comm);
 
     // If the destination is a memref, cast it to a tensor
     if (isa<RankedTensorType>(op.getType()))
       buffer = bufferization::ToTensorOp::create(iBuilder, op.getType(), buffer,
-                                                          true);
+                                                 true);
 
     rewriter.replaceOp(op, buffer);
     return success();
@@ -676,9 +680,10 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
     auto toValue = [&rewriter, &loc](OpFoldResult &v) -> Value {
       if (auto value = dyn_cast<Value>(v))
         return value;
-      return arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexAttr(
-                   cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
+      return arith::ConstantOp::create(
+          rewriter, loc,
+          rewriter.getIndexAttr(
+              cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
     };
 
     auto dest = adaptor.getDestination();
@@ -723,8 +728,8 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         offsets[i] = haloSizes[currHaloDim * 2];
 
         // prepare shape and offsets of highest dim's halo exchange
-        Value _haloSz = arith::AddIOp::create(rewriter,
-            loc, toValue(haloSizes[currHaloDim * 2]),
+        Value _haloSz = arith::AddIOp::create(
+            rewriter, loc, toValue(haloSizes[currHaloDim * 2]),
             toValue(haloSizes[currHaloDim * 2 + 1]));
         // the halo shape of lower dims exlude the halos
         dimSizes[i] =
@@ -758,20 +763,22 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
                                                        splitAxes)
                      .getResults();
       // MPI operates on i32...
-      Value neighbourIDs[2] = {arith::IndexCastOp::create(rewriter,
-                                   loc, rewriter.getI32Type(), tmp[0]),
-                               arith::IndexCastOp::create(rewriter,
-                                   loc, rewriter.getI32Type(), tmp[1])};
+      Value neighbourIDs[2] = {
+          arith::IndexCastOp::create(rewriter, loc, rewriter.getI32Type(),
+                                     tmp[0]),
+          arith::IndexCastOp::create(rewriter, loc, rewriter.getI32Type(),
+                                     tmp[1])};
 
       auto lowerRecvOffset = rewriter.getIndexAttr(0);
       auto lowerSendOffset = toValue(haloSizes[currHaloDim * 2]);
-      auto upperRecvOffset = arith::SubIOp::create(rewriter,
-          loc, toValue(shape[dim]), toValue(haloSizes[currHaloDim * 2 + 1]));
-      auto upperSendOffset = arith::SubIOp::create(rewriter,
-          loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
+      auto upperRecvOffset =
+          arith::SubIOp::create(rewriter, loc, toValue(shape[dim]),
+                                toValue(haloSizes[currHaloDim * 2 + 1]));
+      auto upperSendOffset = arith::SubIOp::create(
+          rewriter, loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
 
-      Value commWorld = mpi::CommWorldOp::create(rewriter,
-          loc, mpi::CommType::get(op->getContext()));
+      Value commWorld = mpi::CommWorldOp::create(
+          rewriter, loc, mpi::CommType::get(op->getContext()));
 
       // Make sure we send/recv in a way that does not lead to a dead-lock.
       // The current approach is by far not optimal, this should be at least
@@ -787,33 +794,34 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         // Processes on the mesh borders have only one neighbor
         auto to = upperHalo ? neighbourIDs[0] : neighbourIDs[1];
         auto from = upperHalo ? neighbourIDs[1] : neighbourIDs[0];
-        auto hasFrom = arith::CmpIOp::create(rewriter,
-            loc, arith::CmpIPredicate::sge, from, zero);
-        auto hasTo = arith::CmpIOp::create(rewriter,
-            loc, arith::CmpIPredicate::sge, to, zero);
-        auto buffer = memref::AllocOp::create(rewriter,
-            loc, dimSizes, cast<ShapedType>(array.getType()).getElementType());
+        auto hasFrom = arith::CmpIOp::create(
+            rewriter, loc, arith::CmpIPredicate::sge, from, zero);
+        auto hasTo = arith::CmpIOp::create(rewriter, loc,
+                                           arith::CmpIPredicate::sge, to, zero);
+        auto buffer = memref::AllocOp::create(
+            rewriter, loc, dimSizes,
+            cast<ShapedType>(array.getType()).getElementType());
         // if has neighbor: copy halo data from array to buffer and send
-        scf::IfOp::create(rewriter,
-            loc, hasTo, [&](OpBuilder &builder, Location loc) {
+        scf::IfOp::create(
+            rewriter, loc, hasTo, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(lowerSendOffset)
                                        : OpFoldResult(upperSendOffset);
-              auto subview = memref::SubViewOp::create(builder,
-                  loc, array, offsets, dimSizes, strides);
+              auto subview = memref::SubViewOp::create(
+                  builder, loc, array, offsets, dimSizes, strides);
               memref::CopyOp::create(builder, loc, subview, buffer);
               mpi::SendOp::create(builder, loc, TypeRange{}, buffer, tag, to,
-                                          commWorld);
+                                  commWorld);
               scf::YieldOp::create(builder, loc);
             });
         // if has neighbor: receive halo data into buffer and copy to array
-        scf::IfOp::create(rewriter,
-            loc, hasFrom, [&](OpBuilder &builder, Location loc) {
+        scf::IfOp::create(
+            rewriter, loc, hasFrom, [&](OpBuilder &builder, Location loc) {
               offsets[dim] = upperHalo ? OpFoldResult(upperRecvOffset)
                                        : OpFoldResult(lowerRecvOffset);
               mpi::RecvOp::create(builder, loc, TypeRange{}, buffer, tag, from,
-                                          commWorld);
-              auto subview = memref::SubViewOp::create(builder,
-                  loc, array, offsets, dimSizes, strides);
+                                  commWorld);
+              auto subview = memref::SubViewOp::create(
+                  builder, loc, array, offsets, dimSizes, strides);
               memref::CopyOp::create(builder, loc, buffer, subview);
               scf::YieldOp::create(builder, loc);
             });
@@ -825,16 +833,17 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
         OpFoldResult &v = haloSizes[currHaloDim * 2 + upOrDown];
         Value haloSz = dyn_cast<Value>(v);
         if (!haloSz)
-          haloSz = arith::ConstantOp::create(rewriter,
-              loc, rewriter.getI32IntegerAttr(
-                       cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
-        auto hasSize = arith::CmpIOp::create(rewriter,
-            loc, arith::CmpIPredicate::sgt, haloSz, zero);
+          haloSz = arith::ConstantOp::create(
+              rewriter, loc,
+              rewriter.getI32IntegerAttr(
+                  cast<IntegerAttr>(cast<Attribute>(v)).getInt()));
+        auto hasSize = arith::CmpIOp::create(
+            rewriter, loc, arith::CmpIPredicate::sgt, haloSz, zero);
         scf::IfOp::create(rewriter, loc, hasSize,
-                                   [&](OpBuilder &builder, Location loc) {
-                                     genSendRecv(upOrDown > 0);
-                                     scf::YieldOp::create(builder, loc);
-                                   });
+                          [&](OpBuilder &builder, Location loc) {
+                            genSendRecv(upOrDown > 0);
+                            scf::YieldOp::create(builder, loc);
+                          });
       };
 
       doSendRecv(0);
@@ -852,8 +861,8 @@ struct ConvertUpdateHaloOp : public OpConversionPattern<UpdateHaloOp> {
       rewriter.replaceOp(op, array);
     } else {
       assert(isa<RankedTensorType>(op.getResult().getType()));
-      rewriter.replaceOp(op, bufferization::ToTensorOp::create(rewriter,
-                                 loc, op.getResult().getType(), array,
+      rewriter.replaceOp(op, bufferization::ToTensorOp::create(
+                                 rewriter, loc, op.getResult().getType(), array,
                                  /*restrict=*/true, /*writable=*/true));
     }
     return success();
diff --git a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
index a6125fc07dcd5..f5b3689c88d26 100644
--- a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
+++ b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
@@ -40,7 +40,7 @@ class ExpandIfCondition : public OpRewritePattern<OpTy> {
     IntegerAttr constAttr;
     if (!matchPattern(op.getIfCond(), m_Constant(&constAttr))) {
       auto ifOp = scf::IfOp::create(rewriter, op.getLoc(), TypeRange(),
-                                             op.getIfCond(), false);
+                                    op.getIfCond(), false);
       rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });
       auto thenBodyBuilder = ifOp.getThenBodyBuilder(rewriter.getListener());
       thenBodyBuilder.clone(*op.getOperation());
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 4bf59124a9bdb..021e31a8ecd97 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -96,7 +96,7 @@ struct OpenMPOpConversion : public ConvertOpToLLVMPattern<T> {
 
     // Create new operation.
     auto newOp = T::create(rewriter, op.getLoc(), resTypes, convertedOperands,
-                                    convertedAttrs);
+                           convertedAttrs);
 
     // Translate regions.
     for (auto [originalRegion, convertedRegion] :
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 75cd35c0d7876..b711e33cfc0d6 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -272,8 +272,8 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     auto *operationPos = cast<OperationPosition>(pos);
     if (operationPos->isOperandDefiningOp())
       // Standard (downward) traversal which directly follows the defining op.
-      value = pdl_interp::GetDefiningOpOp::create(builder,
-          loc, builder.getType<pdl::OperationType>(), parentVal);
+      value = pdl_interp::GetDefiningOpOp::create(
+          builder, loc, builder.getType<pdl::OperationType>(), parentVal);
     else
       // A passthrough operation position.
       value = parentVal;
@@ -297,8 +297,8 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::ForEachPos: {
     assert(!failureBlockStack.empty() && "expected valid failure block");
-    auto foreach = pdl_interp::ForEachOp::create(builder,
-        loc, parentVal, failureBlockStack.back(), /*initLoop=*/true);
+    auto foreach = pdl_interp::ForEachOp::create(
+        builder, loc, parentVal, failureBlockStack.back(), /*initLoop=*/true);
     value = foreach.getLoopVariable();
 
     // Create the continuation block.
@@ -311,23 +311,24 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::OperandPos: {
     auto *operandPos = cast<OperandPosition>(pos);
-    value = pdl_interp::GetOperandOp::create(builder,
-        loc, builder.getType<pdl::ValueType>(), parentVal,
+    value = pdl_interp::GetOperandOp::create(
+        builder, loc, builder.getType<pdl::ValueType>(), parentVal,
         operandPos->getOperandNumber());
     break;
   }
   case Predicates::OperandGroupPos: {
     auto *operandPos = cast<OperandGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = pdl_interp::GetOperandsOp::create(builder,
-        loc, operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
+    value = pdl_interp::GetOperandsOp::create(
+        builder, loc,
+        operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, operandPos->getOperandGroupNumber());
     break;
   }
   case Predicates::AttributePos: {
     auto *attrPos = cast<AttributePosition>(pos);
-    value = pdl_interp::GetAttributeOp::create(builder,
-        loc, builder.getType<pdl::AttributeType>(), parentVal,
+    value = pdl_interp::GetAttributeOp::create(
+        builder, loc, builder.getType<pdl::AttributeType>(), parentVal,
         attrPos->getName().strref());
     break;
   }
@@ -340,23 +341,24 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
   }
   case Predicates::ResultPos: {
     auto *resPos = cast<ResultPosition>(pos);
-    value = pdl_interp::GetResultOp::create(builder,
-        loc, builder.getType<pdl::ValueType>(), parentVal,
+    value = pdl_interp::GetResultOp::create(
+        builder, loc, builder.getType<pdl::ValueType>(), parentVal,
         resPos->getResultNumber());
     break;
   }
   case Predicates::ResultGroupPos: {
     auto *resPos = cast<ResultGroupPosition>(pos);
     Type valueTy = builder.getType<pdl::ValueType>();
-    value = pdl_interp::GetResultsOp::create(builder,
-        loc, resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
+    value = pdl_interp::GetResultsOp::create(
+        builder, loc,
+        resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
         parentVal, resPos->getResultGroupNumber());
     break;
   }
   case Predicates::AttributeLiteralPos: {
     auto *attrPos = cast<AttributeLiteralPosition>(pos);
-    value =
-        pdl_interp::CreateAttributeOp::create(builder, loc, attrPos->getValue());
+    value = pdl_interp::CreateAttributeOp::create(builder, loc,
+                                                  attrPos->getValue());
     break;
   }
   case Predicates::TypeLiteralPos: {
@@ -365,8 +367,8 @@ Value PatternLowering::getValueAt(Block *&currentBlock, Position *pos) {
     if (TypeAttr typeAttr = dyn_cast<TypeAttr>(rawTypeAttr))
       value = pdl_interp::CreateTypeOp::create(builder, loc, typeAttr);
     else
-      value = pdl_interp::CreateTypesOp::create(builder,
-          loc, cast<ArrayAttr>(rawTypeAttr));
+      value = pdl_interp::CreateTypesOp::create(builder, loc,
+                                                cast<ArrayAttr>(rawTypeAttr));
     break;
   }
   case Predicates::ConstraintResultPos: {
@@ -417,52 +419,55 @@ void PatternLowering::generate(BoolNode *boolNode, Block *&currentBlock,
     break;
   case Predicates::OperationNameQuestion: {
     auto *opNameAnswer = cast<OperationNameAnswer>(answer);
-    pdl_interp::CheckOperationNameOp::create(builder,
-        loc, val, opNameAnswer->getValue().getStringRef(), success, failure);
+    pdl_interp::CheckOperationNameOp::create(
+        builder, loc, val, opNameAnswer->getValue().getStringRef(), success,
+        failure);
     break;
   }
   case Predicates::TypeQuestion: {
     auto *ans = cast<TypeAnswer>(answer);
     if (isa<pdl::RangeType>(val.getType()))
-      pdl_interp::CheckTypesOp::create(builder,
-          loc, val, llvm::cast<ArrayAttr>(ans->getValue()), success, failure);
+      pdl_interp::CheckTypesOp::create(builder, loc, val,
+                                       llvm::cast<ArrayAttr>(ans->getValue()),
+                                       success, failure);
     else
-      pdl_interp::CheckTypeOp::create(builder,
-          loc, val, llvm::cast<TypeAttr>(ans->getValue()), success, failure);
+      pdl_interp::CheckTypeOp::create(builder, loc, val,
+                                      llvm::cast<TypeAttr>(ans->getValue()),
+                                      success, failure);
     break;
   }
   case Predicates::AttributeQuestion: {
     auto *ans = cast<AttributeAnswer>(answer);
     pdl_interp::CheckAttributeOp::create(builder, loc, val, ans->getValue(),
-                                                 success, failure);
+                                         success, failure);
     break;
   }
   case Predicates::OperandCountAtLeastQuestion:
   case Predicates::OperandCountQuestion:
-    pdl_interp::CheckOperandCountOp::create(builder,
-        loc, val, cast<UnsignedAnswer>(answer)->getValue(),
+    pdl_interp::CheckOperandCountOp::create(
+        builder, loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::OperandCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::ResultCountAtLeastQuestion:
   case Predicates::ResultCountQuestion:
-    pdl_interp::CheckResultCountOp::create(builder,
-        loc, val, cast<UnsignedAnswer>(answer)->getValue(),
+    pdl_interp::CheckResultCountOp::create(
+        builder, loc, val, cast<UnsignedAnswer>(answer)->getValue(),
         /*compareAtLeast=*/kind == Predicates::ResultCountAtLeastQuestion,
         success, failure);
     break;
   case Predicates::EqualToQuestion: {
     bool trueAnswer = isa<TrueAnswer>(answer);
     pdl_interp::AreEqualOp::create(builder, loc, val, args.front(),
-                                           trueAnswer ? success : failure,
-                                           trueAnswer ? failure : success);
+                                   trueAnswer ? success : failure,
+                                   trueAnswer ? failure : success);
     break;
   }
   case Predicates::ConstraintQuestion: {
     auto *cstQuestion = cast<ConstraintQuestion>(question);
-    auto applyConstraintOp = pdl_interp::ApplyConstraintOp::create(builder,
-        loc, cstQuestion->getResultTypes(), cstQuestion->getName(), args,
-        cstQuestion->getIsNegated(), success, failure);
+    auto applyConstraintOp = pdl_interp::ApplyConstraintOp::create(
+        builder, loc, cstQuestion->getResultTypes(), cstQuestion->getName(),
+        args, cstQuestion->getIsNegated(), success, failure);
 
     constraintOpMap.insert({cstQuestion, applyConstraintOp});
     break;
@@ -536,12 +541,14 @@ void PatternLowering::generate(SwitchNode *switchNode, Block *currentBlock,
       unsigned ans = cast<UnsignedAnswer>(child.first)->getValue();
       switch (kind) {
       case Predicates::OperandCountAtLeastQuestion:
-        pdl_interp::CheckOperandCountOp::create(builder,
-            loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
+        pdl_interp::CheckOperandCountOp::create(builder, loc, val, ans,
+                                                /*compareAtLeast=*/true,
+                                                childBlock, defaultDest);
         break;
       case Predicates::ResultCountAtLeastQuestion:
-        pdl_interp::CheckResultCountOp::create(builder,
-            loc, val, ans, /*compareAtLeast=*/true, childBlock, defaultDest);
+        pdl_interp::CheckResultCountOp::create(builder, loc, val, ans,
+                                               /*compareAtLeast=*/true,
+                                               childBlock, defaultDest);
         break;
       default:
         llvm_unreachable("Generating invalid AtLeast operation");
@@ -619,8 +626,8 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
       rootKindAttr = builder.getStringAttr(*rootKind);
 
   builder.setInsertionPointToEnd(currentBlock);
-  auto matchOp = pdl_interp::RecordMatchOp::create(builder,
-      pattern.getLoc(), mappedMatchValues, locOps.getArrayRef(),
+  auto matchOp = pdl_interp::RecordMatchOp::create(
+      builder, pattern.getLoc(), mappedMatchValues, locOps.getArrayRef(),
       rewriterFuncRef, rootKindAttr, generatedOpsAttr, pattern.getBenefitAttr(),
       failureBlockStack.back());
 
@@ -632,8 +639,8 @@ void PatternLowering::generate(SuccessNode *successNode, Block *&currentBlock) {
 SymbolRefAttr PatternLowering::generateRewriter(
     pdl::PatternOp pattern, SmallVectorImpl<Position *> &usedMatchValues) {
   builder.setInsertionPointToEnd(rewriterModule.getBody());
-  auto rewriterFunc = pdl_interp::FuncOp::create(builder,
-      pattern.getLoc(), "pdl_generated_rewriter",
+  auto rewriterFunc = pdl_interp::FuncOp::create(
+      builder, pattern.getLoc(), "pdl_generated_rewriter",
       builder.getFunctionType({}, {}));
   rewriterSymbolTable.insert(rewriterFunc);
 
@@ -651,18 +658,18 @@ SymbolRefAttr PatternLowering::generateRewriter(
     Operation *oldOp = oldValue.getDefiningOp();
     if (pdl::AttributeOp attrOp = dyn_cast<pdl::AttributeOp>(oldOp)) {
       if (Attribute value = attrOp.getValueAttr()) {
-        return newValue = pdl_interp::CreateAttributeOp::create(builder,
-                   attrOp.getLoc(), value);
+        return newValue = pdl_interp::CreateAttributeOp::create(
+                   builder, attrOp.getLoc(), value);
       }
     } else if (pdl::TypeOp typeOp = dyn_cast<pdl::TypeOp>(oldOp)) {
       if (TypeAttr type = typeOp.getConstantTypeAttr()) {
-        return newValue = pdl_interp::CreateTypeOp::create(builder,
-                   typeOp.getLoc(), type);
+        return newValue = pdl_interp::CreateTypeOp::create(
+                   builder, typeOp.getLoc(), type);
       }
     } else if (pdl::TypesOp typeOp = dyn_cast<pdl::TypesOp>(oldOp)) {
       if (ArrayAttr type = typeOp.getConstantTypesAttr()) {
-        return newValue = pdl_interp::CreateTypesOp::create(builder,
-                   typeOp.getLoc(), typeOp.getType(), type);
+        return newValue = pdl_interp::CreateTypesOp::create(
+                   builder, typeOp.getLoc(), typeOp.getType(), type);
       }
     }
 
@@ -684,8 +691,9 @@ SymbolRefAttr PatternLowering::generateRewriter(
     auto mappedArgs =
         llvm::map_range(rewriter.getExternalArgs(), mapRewriteValue);
     args.append(mappedArgs.begin(), mappedArgs.end());
-    pdl_interp::ApplyRewriteOp::create(builder,
-        rewriter.getLoc(), /*resultTypes=*/TypeRange(), rewriteName, args);
+    pdl_interp::ApplyRewriteOp::create(builder, rewriter.getLoc(),
+                                       /*resultTypes=*/TypeRange(), rewriteName,
+                                       args);
   } else {
     // Otherwise this is a dag rewriter defined using PDL operations.
     for (Operation &rewriteOp : *rewriter.getBody()) {
@@ -716,9 +724,9 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 2> arguments;
   for (Value argument : rewriteOp.getArgs())
     arguments.push_back(mapRewriteValue(argument));
-  auto interpOp = pdl_interp::ApplyRewriteOp::create(builder,
-      rewriteOp.getLoc(), rewriteOp.getResultTypes(), rewriteOp.getNameAttr(),
-      arguments);
+  auto interpOp = pdl_interp::ApplyRewriteOp::create(
+      builder, rewriteOp.getLoc(), rewriteOp.getResultTypes(),
+      rewriteOp.getNameAttr(), arguments);
   for (auto it : llvm::zip(rewriteOp.getResults(), interpOp.getResults()))
     rewriteValues[std::get<0>(it)] = std::get<1>(it);
 }
@@ -726,8 +734,8 @@ void PatternLowering::generateRewriter(
 void PatternLowering::generateRewriter(
     pdl::AttributeOp attrOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  Value newAttr = pdl_interp::CreateAttributeOp::create(builder,
-      attrOp.getLoc(), attrOp.getValueAttr());
+  Value newAttr = pdl_interp::CreateAttributeOp::create(
+      builder, attrOp.getLoc(), attrOp.getValueAttr());
   rewriteValues[attrOp] = newAttr;
 }
 
@@ -735,7 +743,7 @@ void PatternLowering::generateRewriter(
     pdl::EraseOp eraseOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
   pdl_interp::EraseOp::create(builder, eraseOp.getLoc(),
-                                      mapRewriteValue(eraseOp.getOpValue()));
+                              mapRewriteValue(eraseOp.getOpValue()));
 }
 
 void PatternLowering::generateRewriter(
@@ -756,9 +764,9 @@ void PatternLowering::generateRewriter(
 
   // Create the new operation.
   Location loc = operationOp.getLoc();
-  Value createdOp = pdl_interp::CreateOperationOp::create(builder,
-      loc, *operationOp.getOpName(), types, hasInferredResultTypes, operands,
-      attributes, operationOp.getAttributeValueNames());
+  Value createdOp = pdl_interp::CreateOperationOp::create(
+      builder, loc, *operationOp.getOpName(), types, hasInferredResultTypes,
+      operands, attributes, operationOp.getAttributeValueNames());
   rewriteValues[operationOp.getOp()] = createdOp;
 
   // Generate accesses for any results that have their types constrained.
@@ -789,11 +797,12 @@ void PatternLowering::generateRewriter(
     // groups because the exact index of the result is not statically known.
     Value resultVal;
     if (seenVariableLength)
-      resultVal = pdl_interp::GetResultsOp::create(builder,
-          loc, isVariadic ? valueRangeTy : valueTy, createdOp, it.index());
+      resultVal = pdl_interp::GetResultsOp::create(
+          builder, loc, isVariadic ? valueRangeTy : valueTy, createdOp,
+          it.index());
     else
-      resultVal = pdl_interp::GetResultOp::create(builder,
-          loc, valueTy, createdOp, it.index());
+      resultVal = pdl_interp::GetResultOp::create(builder, loc, valueTy,
+                                                  createdOp, it.index());
     type = pdl_interp::GetValueTypeOp::create(builder, loc, resultVal);
   }
 }
@@ -804,8 +813,8 @@ void PatternLowering::generateRewriter(
   SmallVector<Value, 4> replOperands;
   for (Value operand : rangeOp.getArguments())
     replOperands.push_back(mapRewriteValue(operand));
-  rewriteValues[rangeOp] = pdl_interp::CreateRangeOp::create(builder,
-      rangeOp.getLoc(), rangeOp.getType(), replOperands);
+  rewriteValues[rangeOp] = pdl_interp::CreateRangeOp::create(
+      builder, rangeOp.getLoc(), rangeOp.getType(), replOperands);
 }
 
 void PatternLowering::generateRewriter(
@@ -820,8 +829,8 @@ void PatternLowering::generateRewriter(
     // Don't use replace if we know the replaced operation has no results.
     auto opOp = replaceOp.getOpValue().getDefiningOp<pdl::OperationOp>();
     if (!opOp || !opOp.getTypeValues().empty()) {
-      replOperands.push_back(pdl_interp::GetResultsOp::create(builder,
-          replOp.getLoc(), mapRewriteValue(replOp)));
+      replOperands.push_back(pdl_interp::GetResultsOp::create(
+          builder, replOp.getLoc(), mapRewriteValue(replOp)));
     }
   } else {
     for (Value operand : replaceOp.getReplValues())
@@ -830,29 +839,29 @@ void PatternLowering::generateRewriter(
 
   // If there are no replacement values, just create an erase instead.
   if (replOperands.empty()) {
-    pdl_interp::EraseOp::create(builder,
-        replaceOp.getLoc(), mapRewriteValue(replaceOp.getOpValue()));
+    pdl_interp::EraseOp::create(builder, replaceOp.getLoc(),
+                                mapRewriteValue(replaceOp.getOpValue()));
     return;
   }
 
   pdl_interp::ReplaceOp::create(builder, replaceOp.getLoc(),
-                                        mapRewriteValue(replaceOp.getOpValue()),
-                                        replOperands);
+                                mapRewriteValue(replaceOp.getOpValue()),
+                                replOperands);
 }
 
 void PatternLowering::generateRewriter(
     pdl::ResultOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = pdl_interp::GetResultOp::create(builder,
-      resultOp.getLoc(), builder.getType<pdl::ValueType>(),
+  rewriteValues[resultOp] = pdl_interp::GetResultOp::create(
+      builder, resultOp.getLoc(), builder.getType<pdl::ValueType>(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
 
 void PatternLowering::generateRewriter(
     pdl::ResultsOp resultOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  rewriteValues[resultOp] = pdl_interp::GetResultsOp::create(builder,
-      resultOp.getLoc(), resultOp.getType(),
+  rewriteValues[resultOp] = pdl_interp::GetResultsOp::create(
+      builder, resultOp.getLoc(), resultOp.getType(),
       mapRewriteValue(resultOp.getParent()), resultOp.getIndex());
 }
 
@@ -873,8 +882,8 @@ void PatternLowering::generateRewriter(
   // If the type isn't constant, the users (e.g. OperationOp) will resolve this
   // type.
   if (ArrayAttr typeAttr = typeOp.getConstantTypesAttr()) {
-    rewriteValues[typeOp] = pdl_interp::CreateTypesOp::create(builder,
-        typeOp.getLoc(), typeOp.getType(), typeAttr);
+    rewriteValues[typeOp] = pdl_interp::CreateTypesOp::create(
+        builder, typeOp.getLoc(), typeOp.getType(), typeAttr);
   }
 }
 
@@ -939,10 +948,10 @@ void PatternLowering::generateOperationResultTypeRewriter(
         !replacedOp->isBeforeInBlock(op))
       continue;
 
-    Value replacedOpResults = pdl_interp::GetResultsOp::create(builder,
-        replacedOp->getLoc(), mapRewriteValue(replOpVal));
-    types.push_back(pdl_interp::GetValueTypeOp::create(builder,
-        replacedOp->getLoc(), replacedOpResults));
+    Value replacedOpResults = pdl_interp::GetResultsOp::create(
+        builder, replacedOp->getLoc(), mapRewriteValue(replOpVal));
+    types.push_back(pdl_interp::GetValueTypeOp::create(
+        builder, replacedOp->getLoc(), replacedOpResults));
     return;
   }
 
@@ -985,16 +994,18 @@ void PDLToPDLInterpPass::runOnOperation() {
   // Create the main matcher function This function contains all of the match
   // related functionality from patterns in the module.
   OpBuilder builder = OpBuilder::atBlockBegin(module.getBody());
-  auto matcherFunc = pdl_interp::FuncOp::create(builder,
-      module.getLoc(), pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
+  auto matcherFunc = pdl_interp::FuncOp::create(
+      builder, module.getLoc(),
+      pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
       builder.getFunctionType(builder.getType<pdl::OperationType>(),
                               /*results=*/{}),
       /*attrs=*/ArrayRef<NamedAttribute>());
 
   // Create a nested module to hold the functions invoked for rewriting the IR
   // after a successful match.
-  ModuleOp rewriterModule = ModuleOp::create(builder,
-      module.getLoc(), pdl_interp::PDLInterpDialect::getRewriterModuleName());
+  ModuleOp rewriterModule =
+      ModuleOp::create(builder, module.getLoc(),
+                       pdl_interp::PDLInterpDialect::getRewriterModuleName());
 
   // Generate the code for the patterns within the module.
   PatternLowering generator(matcherFunc, rewriterModule, configMap);
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index 68527da73e1c7..afc25211ad193 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -379,12 +379,11 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
 
   // With the body block done, we can fill in the condition block.
   rewriter.setInsertionPointToEnd(conditionBlock);
-  auto comparison = arith::CmpIOp::create(rewriter,
-      loc, arith::CmpIPredicate::slt, iv, upperBound);
+  auto comparison = arith::CmpIOp::create(
+      rewriter, loc, arith::CmpIPredicate::slt, iv, upperBound);
 
   cf::CondBranchOp::create(rewriter, loc, comparison, firstBodyBlock,
-                                    ArrayRef<Value>(), endBlock,
-                                    ArrayRef<Value>());
+                           ArrayRef<Value>(), endBlock, ArrayRef<Value>());
 
   // The result of the loop operation is the values of the condition block
   // arguments except the induction variable on the last iteration.
@@ -440,8 +439,8 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
 
   rewriter.setInsertionPointToEnd(condBlock);
   cf::CondBranchOp::create(rewriter, loc, ifOp.getCondition(), thenBlock,
-                                    /*trueArgs=*/ArrayRef<Value>(), elseBlock,
-                                    /*falseArgs=*/ArrayRef<Value>());
+                           /*trueArgs=*/ArrayRef<Value>(), elseBlock,
+                           /*falseArgs=*/ArrayRef<Value>());
 
   // Ok, we're done!
   rewriter.replaceOp(ifOp, continueBlock->getArguments());
@@ -465,7 +464,8 @@ ExecuteRegionLowering::matchAndRewrite(ExecuteRegionOp op,
     if (auto terminator = dyn_cast<scf::YieldOp>(block.getTerminator())) {
       ValueRange terminatorOperands = terminator->getOperands();
       rewriter.setInsertionPointToEnd(&block);
-      cf::BranchOp::create(rewriter, loc, remainingOpsBlock, terminatorOperands);
+      cf::BranchOp::create(rewriter, loc, remainingOpsBlock,
+                           terminatorOperands);
       rewriter.eraseOp(terminator);
     }
   }
@@ -695,12 +695,12 @@ IndexSwitchLowering::matchAndRewrite(IndexSwitchOp op,
   SmallVector<ValueRange> caseOperands(caseSuccessors.size(), {});
 
   // Cast switch index to integer case value.
-  Value caseValue = arith::IndexCastOp::create(rewriter,
-      op.getLoc(), rewriter.getI32Type(), op.getArg());
+  Value caseValue = arith::IndexCastOp::create(
+      rewriter, op.getLoc(), rewriter.getI32Type(), op.getArg());
 
-  cf::SwitchOp::create(rewriter,
-      op.getLoc(), caseValue, *defaultBlock, ValueRange(),
-      rewriter.getDenseI32ArrayAttr(caseValues), caseSuccessors, caseOperands);
+  cf::SwitchOp::create(rewriter, op.getLoc(), caseValue, *defaultBlock,
+                       ValueRange(), rewriter.getDenseI32ArrayAttr(caseValues),
+                       caseSuccessors, caseOperands);
   rewriter.replaceOp(op, continueBlock->getArguments());
   return success();
 }
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index 37134fe9d992a..84cbd869c78ef 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -164,8 +164,9 @@ ForLowering::matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
 
   assignValues(adaptor.getInitArgs(), resultVariables, rewriter, loc);
 
-  emitc::ForOp loweredFor = emitc::ForOp::create(rewriter,
-      loc, adaptor.getLowerBound(), adaptor.getUpperBound(), adaptor.getStep());
+  emitc::ForOp loweredFor =
+      emitc::ForOp::create(rewriter, loc, adaptor.getLowerBound(),
+                           adaptor.getUpperBound(), adaptor.getStep());
 
   Block *loweredBody = loweredFor.getBody();
 
@@ -304,8 +305,9 @@ LogicalResult IndexSwitchOpLowering::matchAndRewrite(
                                        "create variables for results failed");
   }
 
-  auto loweredSwitch = emitc::SwitchOp::create(rewriter,
-      loc, adaptor.getArg(), adaptor.getCases(), indexSwitchOp.getNumCases());
+  auto loweredSwitch =
+      emitc::SwitchOp::create(rewriter, loc, adaptor.getArg(),
+                              adaptor.getCases(), indexSwitchOp.getNumCases());
 
   // Lowering all case regions.
   for (auto pair :
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index bfdb5a688b286..f191f3502cf5a 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -85,7 +85,7 @@ static Operation::operand_range getUpperBoundOperands(AffineForOp forOp) {
 // materialize a corresponding constant using builder.
 static Value getOrCreateStep(AffineForOp forOp, OpBuilder &builder) {
   return arith::ConstantIndexOp::create(builder, forOp.getLoc(),
-                                                forOp.getStepAsInt());
+                                        forOp.getStepAsInt());
 }
 
 // Get a Value for the loop lower bound.  If the value requires computation,
@@ -191,11 +191,11 @@ AffineLoopToGpuConverter::collectBounds(AffineForOp forOp, unsigned numLoops) {
     }
 
     Value range = arith::SubIOp::create(builder, currentLoop.getLoc(),
-                                                upperBound, lowerBound);
+                                        upperBound, lowerBound);
     Value step = getOrCreateStep(currentLoop, builder);
     if (getConstantIntValue(step) != static_cast<int64_t>(1))
-      range =
-          arith::CeilDivSIOp::create(builder, currentLoop.getLoc(), range, step);
+      range = arith::CeilDivSIOp::create(builder, currentLoop.getLoc(), range,
+                                         step);
     dims.push_back(range);
 
     lbs.push_back(lowerBound);
@@ -232,9 +232,9 @@ void AffineLoopToGpuConverter::createLaunch(AffineForOp rootForOp,
 
   // Create a launch op and move the body region of the innermost loop to the
   // launch op.
-  auto launchOp = gpu::LaunchOp::create(builder,
-      rootForOp.getLoc(), gridSizeX, gridSizeY, gridSizeZ, blockSizeX,
-      blockSizeY, blockSizeZ);
+  auto launchOp =
+      gpu::LaunchOp::create(builder, rootForOp.getLoc(), gridSizeX, gridSizeY,
+                            gridSizeZ, blockSizeX, blockSizeY, blockSizeZ);
 
   // Replace the loop terminator (loops contain only a single block) with the
   // gpu terminator and move the operations from the loop body block to the gpu
@@ -320,7 +320,7 @@ static Value deriveStaticUpperBound(Value upperBound,
     for (const AffineExpr &result : minOp.getMap().getResults()) {
       if (auto constExpr = dyn_cast<AffineConstantExpr>(result)) {
         return arith::ConstantIndexOp::create(rewriter, minOp.getLoc(),
-                                                       constExpr.getValue());
+                                              constExpr.getValue());
       }
     }
   }
@@ -344,8 +344,8 @@ static Value deriveStaticUpperBound(Value upperBound,
         if ((lhs.value() < 0) != (rhs.value() < 0))
           return {};
 
-        return arith::ConstantIndexOp::create(rewriter,
-            multiplyOp.getLoc(), lhs.value() * rhs.value());
+        return arith::ConstantIndexOp::create(rewriter, multiplyOp.getLoc(),
+                                              lhs.value() * rhs.value());
       }
   }
 
@@ -423,7 +423,7 @@ static LogicalResult processParallelLoop(
       return val;
     if (auto constOp = val.getDefiningOp<arith::ConstantOp>())
       return arith::ConstantOp::create(rewriter, constOp.getLoc(),
-                                                constOp.getValue());
+                                       constOp.getValue());
     return {};
   };
 
@@ -453,8 +453,8 @@ static LogicalResult processParallelLoop(
           1, 2,
           rewriter.getAffineDimExpr(0) * rewriter.getAffineSymbolExpr(0) +
               rewriter.getAffineSymbolExpr(1));
-      newIndex = AffineApplyOp::create(rewriter,
-          loc, annotation.getMap().compose(lowerAndStep),
+      newIndex = AffineApplyOp::create(
+          rewriter, loc, annotation.getMap().compose(lowerAndStep),
           ValueRange{operand, ensureLaunchIndependent(step),
                      ensureLaunchIndependent(lowerBound)});
       // If there was also a bound, insert that, too.
@@ -498,8 +498,8 @@ static LogicalResult processParallelLoop(
               1, 2,
               ((rewriter.getAffineDimExpr(0) - rewriter.getAffineSymbolExpr(0))
                    .ceilDiv(rewriter.getAffineSymbolExpr(1))));
-          Value launchBound = AffineApplyOp::create(rewriter,
-              loc, annotation.getBound().compose(stepMap),
+          Value launchBound = AffineApplyOp::create(
+              rewriter, loc, annotation.getBound().compose(stepMap),
               ValueRange{
                   ensureLaunchIndependent(
                       cloningMap.lookupOrDefault(upperBound)),
@@ -517,8 +517,8 @@ static LogicalResult processParallelLoop(
         if (!boundIsPrecise) {
           // We are using an approximation, create a surrounding conditional.
           Value originalBound = std::get<3>(config);
-          arith::CmpIOp pred = arith::CmpIOp::create(rewriter,
-              loc, arith::CmpIPredicate::slt, newIndex,
+          arith::CmpIOp pred = arith::CmpIOp::create(
+              rewriter, loc, arith::CmpIPredicate::slt, newIndex,
               cloningMap.lookupOrDefault(originalBound));
           scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, pred, false);
           rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
@@ -530,10 +530,10 @@ static LogicalResult processParallelLoop(
       }
     } else {
       // Create a sequential for loop.
-      auto loopOp = scf::ForOp::create(rewriter,
-          loc, cloningMap.lookupOrDefault(lowerBound),
-          cloningMap.lookupOrDefault(upperBound),
-          cloningMap.lookupOrDefault(step));
+      auto loopOp = scf::ForOp::create(rewriter, loc,
+                                       cloningMap.lookupOrDefault(lowerBound),
+                                       cloningMap.lookupOrDefault(upperBound),
+                                       cloningMap.lookupOrDefault(step));
       newIndex = loopOp.getInductionVar();
       rewriter.setInsertionPointToStart(loopOp.getBody());
       // Put a sentinel into the worklist so we know when to pop out of the loop
@@ -609,9 +609,9 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
   Location loc = parallelOp.getLoc();
   Value constantOne =
       arith::ConstantIndexOp::create(rewriter, parallelOp.getLoc(), 1);
-  gpu::LaunchOp launchOp = gpu::LaunchOp::create(rewriter,
-      parallelOp.getLoc(), constantOne, constantOne, constantOne, constantOne,
-      constantOne, constantOne);
+  gpu::LaunchOp launchOp = gpu::LaunchOp::create(
+      rewriter, parallelOp.getLoc(), constantOne, constantOne, constantOne,
+      constantOne, constantOne, constantOne);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
   gpu::TerminatorOp::create(rewriter, loc);
   rewriter.setInsertionPointToStart(&launchOp.getBody().front());
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 7b4c37d44824d..34f372af1e4b5 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -188,7 +188,7 @@ createDecl(PatternRewriter &builder, SymbolTable &symbolTable,
   OpBuilder::InsertionGuard guard(builder);
   Type type = reduce.getOperands()[reductionIndex].getType();
   auto decl = omp::DeclareReductionOp::create(builder, reduce.getLoc(),
-                                                      "__scf_reduction", type);
+                                              "__scf_reduction", type);
   symbolTable.insert(decl);
 
   builder.createBlock(&decl.getInitializerRegion(),
@@ -228,10 +228,10 @@ static omp::DeclareReductionOp addAtomicRMW(OpBuilder &builder,
   Block *atomicBlock = &decl.getAtomicReductionRegion().back();
   builder.setInsertionPointToEnd(atomicBlock);
   Value loaded = LLVM::LoadOp::create(builder, reduce.getLoc(), decl.getType(),
-                                              atomicBlock->getArgument(1));
+                                      atomicBlock->getArgument(1));
   LLVM::AtomicRMWOp::create(builder, reduce.getLoc(), atomicKind,
-                                    atomicBlock->getArgument(0), loaded,
-                                    LLVM::AtomicOrdering::monotonic);
+                            atomicBlock->getArgument(0), loaded,
+                            LLVM::AtomicOrdering::monotonic);
   omp::YieldOp::create(builder, reduce.getLoc(), ArrayRef<Value>());
   return decl;
 }
@@ -380,8 +380,9 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
     // Allocate reduction variables. Make sure the we don't overflow the stack
     // with local `alloca`s by saving and restoring the stack pointer.
     Location loc = parallelOp.getLoc();
-    Value one = LLVM::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerType(64), rewriter.getI64IntegerAttr(1));
+    Value one =
+        LLVM::ConstantOp::create(rewriter, loc, rewriter.getIntegerType(64),
+                                 rewriter.getI64IntegerAttr(1));
     SmallVector<Value> reductionVariables;
     reductionVariables.reserve(parallelOp.getNumReductions());
     auto ptrType = LLVM::LLVMPointerType::get(parallelOp.getContext());
@@ -390,8 +391,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
               isa<LLVM::PointerElementTypeInterface>(init.getType())) &&
              "cannot create a reduction variable if the type is not an LLVM "
              "pointer element");
-      Value storage =
-          LLVM::AllocaOp::create(rewriter, loc, ptrType, init.getType(), one, 0);
+      Value storage = LLVM::AllocaOp::create(rewriter, loc, ptrType,
+                                             init.getType(), one, 0);
       LLVM::StoreOp::create(rewriter, loc, init, storage);
       reductionVariables.push_back(storage);
     }
@@ -412,7 +413,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
              "expect reduction region to have one block");
       Value pvtRedVar = parallelOp.getRegion().addArgument(x.getType(), loc);
       Value pvtRedVal = LLVM::LoadOp::create(rewriter, reduce.getLoc(),
-                                                      rD.getType(), pvtRedVar);
+                                             rD.getType(), pvtRedVar);
       // Make a copy of the reduction combiner region in the body
       mlir::OpBuilder builder(rewriter.getContext());
       builder.setInsertionPoint(reduce);
@@ -437,12 +438,12 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
 
     Value numThreadsVar;
     if (numThreads > 0) {
-      numThreadsVar = LLVM::ConstantOp::create(rewriter,
-          loc, rewriter.getI32IntegerAttr(numThreads));
+      numThreadsVar = LLVM::ConstantOp::create(
+          rewriter, loc, rewriter.getI32IntegerAttr(numThreads));
     }
     // Create the parallel wrapper.
-    auto ompParallel = omp::ParallelOp::create(rewriter,
-        loc,
+    auto ompParallel = omp::ParallelOp::create(
+        rewriter, loc,
         /* allocate_vars = */ llvm::SmallVector<Value>{},
         /* allocator_vars = */ llvm::SmallVector<Value>{},
         /* if_expr = */ Value{},
@@ -490,8 +491,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
                                               parallelOp.getLoc()));
 
         // Create loop nest and populate region with contents of scf.parallel.
-        auto loopOp = omp::LoopNestOp::create(rewriter,
-            parallelOp.getLoc(), parallelOp.getLowerBound(),
+        auto loopOp = omp::LoopNestOp::create(
+            rewriter, parallelOp.getLoc(), parallelOp.getLowerBound(),
             parallelOp.getUpperBound(), parallelOp.getStep());
 
         rewriter.inlineRegionBefore(parallelOp.getRegion(), loopOp.getRegion(),
@@ -511,8 +512,8 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
             rewriter.splitBlock(&loopOpEntryBlock, loopOpEntryBlock.begin());
         rewriter.setInsertionPointToStart(&loopOpEntryBlock);
 
-        auto scope = memref::AllocaScopeOp::create(rewriter, parallelOp.getLoc(),
-                                                            TypeRange());
+        auto scope = memref::AllocaScopeOp::create(
+            rewriter, parallelOp.getLoc(), TypeRange());
         omp::YieldOp::create(rewriter, loc, ValueRange());
         Block *scopeBlock = rewriter.createBlock(&scope.getBodyRegion());
         rewriter.mergeBlocks(ops, scopeBlock);
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index b494fa98f1362..dc92367fc58cd 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -71,9 +71,9 @@ void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp,
     auto pointerType =
         spirv::PointerType::get(convertedType, spirv::StorageClass::Function);
     rewriter.setInsertionPoint(newOp);
-    auto alloc = spirv::VariableOp::create(rewriter,
-        loc, pointerType, spirv::StorageClass::Function,
-        /*initializer=*/nullptr);
+    auto alloc = spirv::VariableOp::create(rewriter, loc, pointerType,
+                                           spirv::StorageClass::Function,
+                                           /*initializer=*/nullptr);
     allocas.push_back(alloc);
     rewriter.setInsertionPointAfter(newOp);
     Value loadResult = spirv::LoadOp::create(rewriter, loc, alloc);
@@ -135,7 +135,8 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // a single back edge from the continue to header block, and a single exit
     // from header to merge.
     auto loc = forOp.getLoc();
-    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
+    auto loopOp =
+        spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     OpBuilder::InsertionGuard guard(rewriter);
@@ -177,11 +178,12 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // Generate the rest of the loop header.
     rewriter.setInsertionPointToEnd(header);
     auto *mergeBlock = loopOp.getMergeBlock();
-    auto cmpOp = spirv::SLessThanOp::create(rewriter,
-        loc, rewriter.getI1Type(), newIndVar, adaptor.getUpperBound());
+    auto cmpOp = spirv::SLessThanOp::create(rewriter, loc, rewriter.getI1Type(),
+                                            newIndVar, adaptor.getUpperBound());
 
-    spirv::BranchConditionalOp::create(rewriter,
-        loc, cmpOp, body, ArrayRef<Value>(), mergeBlock, ArrayRef<Value>());
+    spirv::BranchConditionalOp::create(rewriter, loc, cmpOp, body,
+                                       ArrayRef<Value>(), mergeBlock,
+                                       ArrayRef<Value>());
 
     // Generate instructions to increment the step of the induction variable and
     // branch to the header.
@@ -189,8 +191,8 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     rewriter.setInsertionPointToEnd(continueBlock);
 
     // Add the step to the induction variable and branch to the header.
-    Value updatedIndVar = spirv::IAddOp::create(rewriter,
-        loc, newIndVar.getType(), newIndVar, adaptor.getStep());
+    Value updatedIndVar = spirv::IAddOp::create(
+        rewriter, loc, newIndVar.getType(), newIndVar, adaptor.getStep());
     spirv::BranchOp::create(rewriter, loc, header, updatedIndVar);
 
     // Infer the return types from the init operands. Vector type may get
@@ -237,8 +239,8 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
 
     // Create `spirv.selection` operation, selection header block and merge
     // block.
-    auto selectionOp =
-        spirv::SelectionOp::create(rewriter, loc, spirv::SelectionControl::None);
+    auto selectionOp = spirv::SelectionOp::create(
+        rewriter, loc, spirv::SelectionControl::None);
     auto *mergeBlock = rewriter.createBlock(&selectionOp.getBody(),
                                             selectionOp.getBody().end());
     spirv::MergeOp::create(rewriter, loc);
@@ -268,8 +270,8 @@ struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
     // Create a `spirv.BranchConditional` operation for selection header block.
     rewriter.setInsertionPointToEnd(selectionHeaderBlock);
     spirv::BranchConditionalOp::create(rewriter, loc, adaptor.getCondition(),
-                                                thenBlock, ArrayRef<Value>(),
-                                                elseBlock, ArrayRef<Value>());
+                                       thenBlock, ArrayRef<Value>(), elseBlock,
+                                       ArrayRef<Value>());
 
     replaceSCFOutputValue(ifOp, selectionOp, rewriter, scfToSPIRVContext,
                           returnTypes);
@@ -320,7 +322,7 @@ struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
         args.append(operands.begin(), operands.end());
         rewriter.setInsertionPoint(br);
         spirv::BranchOp::create(rewriter, terminatorOp.getLoc(), br.getTarget(),
-                                         args);
+                                args);
         rewriter.eraseOp(br);
       }
     }
@@ -340,7 +342,8 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
   matchAndRewrite(scf::WhileOp whileOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto loc = whileOp.getLoc();
-    auto loopOp = spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
+    auto loopOp =
+        spirv::LoopOp::create(rewriter, loc, spirv::LoopControl::None);
     loopOp.addEntryAndMergeBlock(rewriter);
 
     Region &beforeRegion = whileOp.getBefore();
@@ -403,9 +406,9 @@ struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
 
       // Create local variables before the scf.while op.
       rewriter.setInsertionPoint(loopOp);
-      auto alloc = spirv::VariableOp::create(rewriter,
-          condLoc, pointerType, spirv::StorageClass::Function,
-          /*initializer=*/nullptr);
+      auto alloc = spirv::VariableOp::create(rewriter, condLoc, pointerType,
+                                             spirv::StorageClass::Function,
+                                             /*initializer=*/nullptr);
 
       // Load the final result values after the scf.while op.
       rewriter.setInsertionPointAfter(loopOp);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index 934bb0d1c37fd..035f197b1eac2 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -194,8 +194,8 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
     if (!kernelFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(module.getBody());
-      kernelFunc = LLVM::LLVMFuncOp::create(rewriter,
-          rewriter.getUnknownLoc(), newKernelFuncName,
+      kernelFunc = LLVM::LLVMFuncOp::create(
+          rewriter, rewriter.getUnknownLoc(), newKernelFuncName,
           LLVM::LLVMFunctionType::get(LLVM::LLVMVoidType::get(context),
                                       ArrayRef<Type>()));
       rewriter.setInsertionPoint(launchOp);
@@ -245,8 +245,8 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       if (!dstGlobal) {
         OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointToStart(module.getBody());
-        dstGlobal = LLVM::GlobalOp::create(rewriter,
-            loc, dstGlobalType,
+        dstGlobal = LLVM::GlobalOp::create(
+            rewriter, loc, dstGlobalType,
             /*isConstant=*/false, LLVM::Linkage::Linkonce, name, Attribute(),
             /*alignment=*/0);
         rewriter.setInsertionPoint(launchOp);
@@ -255,8 +255,8 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       // Copy the data from src operand pointer to dst global variable. Save
       // src, dst and size so that we can copy data back after emulating the
       // kernel call.
-      Value dst = LLVM::AddressOfOp::create(rewriter,
-          loc, typeConverter->convertType(spirvGlobal.getType()),
+      Value dst = LLVM::AddressOfOp::create(
+          rewriter, loc, typeConverter->convertType(spirvGlobal.getType()),
           dstGlobal.getSymName());
       copy(loc, dst, src, sizeBytes, rewriter);
 
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index f490e065143e7..788d9f95ec331 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -94,13 +94,13 @@ static IntegerAttr minusOneIntegerAttribute(Type type, Builder builder) {
 static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
                                       PatternRewriter &rewriter) {
   if (isa<VectorType>(srcType)) {
-    return LLVM::ConstantOp::create(rewriter,
-        loc, dstType,
+    return LLVM::ConstantOp::create(
+        rewriter, loc, dstType,
         SplatElementsAttr::get(cast<ShapedType>(srcType),
                                minusOneIntegerAttribute(srcType, rewriter)));
   }
-  return LLVM::ConstantOp::create(rewriter,
-      loc, dstType, minusOneIntegerAttribute(srcType, rewriter));
+  return LLVM::ConstantOp::create(rewriter, loc, dstType,
+                                  minusOneIntegerAttribute(srcType, rewriter));
 }
 
 /// Creates `llvm.mlir.constant` with a floating-point scalar or vector value.
@@ -108,14 +108,14 @@ static Value createFPConstant(Location loc, Type srcType, Type dstType,
                               PatternRewriter &rewriter, double value) {
   if (auto vecType = dyn_cast<VectorType>(srcType)) {
     auto floatType = cast<FloatType>(vecType.getElementType());
-    return LLVM::ConstantOp::create(rewriter,
-        loc, dstType,
+    return LLVM::ConstantOp::create(
+        rewriter, loc, dstType,
         SplatElementsAttr::get(vecType,
                                rewriter.getFloatAttr(floatType, value)));
   }
   auto floatType = cast<FloatType>(srcType);
-  return LLVM::ConstantOp::create(rewriter,
-      loc, dstType, rewriter.getFloatAttr(floatType, value));
+  return LLVM::ConstantOp::create(rewriter, loc, dstType,
+                                  rewriter.getFloatAttr(floatType, value));
 }
 
 /// Utility function for bitfield ops:
@@ -153,10 +153,10 @@ static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
   auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
   Value broadcasted = LLVM::PoisonOp::create(rewriter, loc, llvmVectorType);
   for (unsigned i = 0; i < numElements; ++i) {
-    auto index = LLVM::ConstantOp::create(rewriter,
-        loc, llvmI32Type, rewriter.getI32IntegerAttr(i));
-    broadcasted = LLVM::InsertElementOp::create(rewriter,
-        loc, llvmVectorType, broadcasted, toBroadcast, index);
+    auto index = LLVM::ConstantOp::create(rewriter, loc, llvmI32Type,
+                                          rewriter.getI32IntegerAttr(i));
+    broadcasted = LLVM::InsertElementOp::create(
+        rewriter, loc, llvmVectorType, broadcasted, toBroadcast, index);
   }
   return broadcasted;
 }
@@ -217,8 +217,8 @@ static Type convertStructTypePacked(spirv::StructType type,
 /// Creates LLVM dialect constant with the given value.
 static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
                                  unsigned value) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, IntegerType::get(rewriter.getContext(), 32),
+  return LLVM::ConstantOp::create(
+      rewriter, loc, IntegerType::get(rewriter.getContext(), 32),
       rewriter.getIntegerAttr(rewriter.getI32Type(), value));
 }
 
@@ -322,8 +322,9 @@ class AccessChainPattern : public SPIRVToLLVMConversion<spirv::AccessChainOp> {
     auto llvmIndexType = getTypeConverter()->convertType(indexType);
     if (!llvmIndexType)
       return rewriter.notifyMatchFailure(op, "type conversion failed");
-    Value zero = LLVM::ConstantOp::create(rewriter,
-        op.getLoc(), llvmIndexType, rewriter.getIntegerAttr(indexType, 0));
+    Value zero =
+        LLVM::ConstantOp::create(rewriter, op.getLoc(), llvmIndexType,
+                                 rewriter.getIntegerAttr(indexType, 0));
     indices.insert(indices.begin(), zero);
 
     auto elementType = getTypeConverter()->convertType(
@@ -377,11 +378,11 @@ class BitFieldInsertPattern
     Value maskShiftedByCount =
         LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
     Value negated = LLVM::XOrOp::create(rewriter, loc, dstType,
-                                                 maskShiftedByCount, minusOne);
+                                        maskShiftedByCount, minusOne);
     Value maskShiftedByCountAndOffset =
         LLVM::ShlOp::create(rewriter, loc, dstType, negated, offset);
-    Value mask = LLVM::XOrOp::create(rewriter,
-        loc, dstType, maskShiftedByCountAndOffset, minusOne);
+    Value mask = LLVM::XOrOp::create(rewriter, loc, dstType,
+                                     maskShiftedByCountAndOffset, minusOne);
 
     // Extract unchanged bits from the `Base`  that are outside of
     // [Offset, Offset + Count - 1]. Then `or` with shifted `Insert`.
@@ -470,8 +471,8 @@ class BitFieldSExtractPattern
     auto baseSize = rewriter.getIntegerAttr(integerType, getBitWidth(srcType));
     Value size =
         isa<VectorType>(srcType)
-            ? LLVM::ConstantOp::create(rewriter,
-                  loc, dstType,
+            ? LLVM::ConstantOp::create(
+                  rewriter, loc, dstType,
                   SplatElementsAttr::get(cast<ShapedType>(srcType), baseSize))
             : LLVM::ConstantOp::create(rewriter, loc, dstType, baseSize);
 
@@ -481,8 +482,8 @@ class BitFieldSExtractPattern
         LLVM::AddOp::create(rewriter, loc, dstType, count, offset);
     Value amountToShiftLeft =
         LLVM::SubOp::create(rewriter, loc, dstType, size, countPlusOffset);
-    Value baseShiftedLeft = LLVM::ShlOp::create(rewriter,
-        loc, dstType, op.getBase(), amountToShiftLeft);
+    Value baseShiftedLeft = LLVM::ShlOp::create(
+        rewriter, loc, dstType, op.getBase(), amountToShiftLeft);
 
     // Shift the result right, filling the bits with the sign bit.
     Value amountToShiftRight =
@@ -518,7 +519,7 @@ class BitFieldUExtractPattern
     Value maskShiftedByCount =
         LLVM::ShlOp::create(rewriter, loc, dstType, minusOne, count);
     Value mask = LLVM::XOrOp::create(rewriter, loc, dstType, maskShiftedByCount,
-                                              minusOne);
+                                     minusOne);
 
     // Shift `Base` by `Offset` and apply the mask on it.
     Value shiftedBase =
@@ -694,8 +695,8 @@ class ExecutionModePattern
     auto structType = LLVM::LLVMStructType::getLiteral(context, fields);
 
     // Create `llvm.mlir.global` with initializer region containing one block.
-    auto global = LLVM::GlobalOp::create(rewriter,
-        UnknownLoc::get(context), structType, /*isConstant=*/true,
+    auto global = LLVM::GlobalOp::create(
+        rewriter, UnknownLoc::get(context), structType, /*isConstant=*/true,
         LLVM::Linkage::External, executionModeInfoName, Attribute(),
         /*alignment=*/0);
     Location loc = global.getLoc();
@@ -705,8 +706,8 @@ class ExecutionModePattern
     // Initialize the struct and set the execution mode value.
     rewriter.setInsertionPointToStart(block);
     Value structValue = LLVM::PoisonOp::create(rewriter, loc, structType);
-    Value executionMode = LLVM::ConstantOp::create(rewriter,
-        loc, llvmI32Type,
+    Value executionMode = LLVM::ConstantOp::create(
+        rewriter, loc, llvmI32Type,
         rewriter.getI32IntegerAttr(
             static_cast<uint32_t>(executionModeAttr.getValue())));
     structValue = rewriter.create<LLVM::InsertValueOp>(loc, structValue,
@@ -716,8 +717,8 @@ class ExecutionModePattern
     for (unsigned i = 0, e = values.size(); i < e; ++i) {
       auto attr = values.getValue()[i];
       Value entry = LLVM::ConstantOp::create(rewriter, loc, llvmI32Type, attr);
-      structValue = LLVM::InsertValueOp::create(rewriter,
-          loc, structValue, entry, ArrayRef<int64_t>({1, i}));
+      structValue = LLVM::InsertValueOp::create(
+          rewriter, loc, structValue, entry, ArrayRef<int64_t>({1, i}));
     }
     LLVM::ReturnOp::create(rewriter, loc, ArrayRef<Value>({structValue}));
     rewriter.eraseOp(op);
@@ -973,8 +974,8 @@ class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
     IntegerAttr minusOne = minusOneIntegerAttribute(srcType, rewriter);
     auto mask =
         isa<VectorType>(srcType)
-            ? LLVM::ConstantOp::create(rewriter,
-                  loc, dstType,
+            ? LLVM::ConstantOp::create(
+                  rewriter, loc, dstType,
                   SplatElementsAttr::get(cast<VectorType>(srcType), minusOne))
             : LLVM::ConstantOp::create(rewriter, loc, dstType, minusOne);
     rewriter.template replaceOpWithNewOp<LLVM::XOrOp>(notOp, dstType,
@@ -1034,8 +1035,8 @@ static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
     return func;
 
   OpBuilder b(symbolTable->getRegion(0));
-  func = LLVM::LLVMFuncOp::create(b,
-      symbolTable->getLoc(), name,
+  func = LLVM::LLVMFuncOp::create(
+      b, symbolTable->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes));
   func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
   func.setConvergent(convergent);
@@ -1078,12 +1079,12 @@ class ControlBarrierPattern : public SPIRVToLLVMConversion<BarrierOpTy> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, {i32, i32, i32}, voidTy);
 
     Location loc = controlBarrierOp->getLoc();
-    Value execution = LLVM::ConstantOp::create(rewriter,
-        loc, i32, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value memory = LLVM::ConstantOp::create(rewriter,
-        loc, i32, static_cast<int32_t>(adaptor.getMemoryScope()));
-    Value semantics = LLVM::ConstantOp::create(rewriter,
-        loc, i32, static_cast<int32_t>(adaptor.getMemorySemantics()));
+    Value execution = LLVM::ConstantOp::create(
+        rewriter, loc, i32, static_cast<int32_t>(adaptor.getExecutionScope()));
+    Value memory = LLVM::ConstantOp::create(
+        rewriter, loc, i32, static_cast<int32_t>(adaptor.getMemoryScope()));
+    Value semantics = LLVM::ConstantOp::create(
+        rewriter, loc, i32, static_cast<int32_t>(adaptor.getMemorySemantics()));
 
     auto call = createSPIRVBuiltinCall(loc, rewriter, func,
                                        {execution, memory, semantics});
@@ -1255,10 +1256,12 @@ class GroupReducePattern : public SPIRVToLLVMConversion<ReduceOp> {
         lookupOrCreateSPIRVFn(symbolTable, funcName, paramTypes, retTy);
 
     Location loc = op.getLoc();
-    Value scope = LLVM::ConstantOp::create(rewriter,
-        loc, i32Ty, static_cast<int32_t>(adaptor.getExecutionScope()));
-    Value groupOp = LLVM::ConstantOp::create(rewriter,
-        loc, i32Ty, static_cast<int32_t>(adaptor.getGroupOperation()));
+    Value scope = LLVM::ConstantOp::create(
+        rewriter, loc, i32Ty,
+        static_cast<int32_t>(adaptor.getExecutionScope()));
+    Value groupOp = LLVM::ConstantOp::create(
+        rewriter, loc, i32Ty,
+        static_cast<int32_t>(adaptor.getGroupOperation()));
     SmallVector<Value> operands{scope, groupOp};
     operands.append(adaptor.getOperands().begin(), adaptor.getOperands().end());
 
@@ -1441,9 +1444,8 @@ class SelectionPattern : public SPIRVToLLVMConversion<spirv::SelectionOp> {
     Block *falseBlock = condBrOp.getFalseBlock();
     rewriter.setInsertionPointToEnd(currentBlock);
     LLVM::CondBrOp::create(rewriter, loc, condBrOp.getCondition(), trueBlock,
-                                    condBrOp.getTrueTargetOperands(),
-                                    falseBlock,
-                                    condBrOp.getFalseTargetOperands());
+                           condBrOp.getTrueTargetOperands(), falseBlock,
+                           condBrOp.getFalseTargetOperands());
 
     rewriter.eraseBlock(headerBlock);
     rewriter.inlineRegionBefore(op.getBody(), continueBlock);
@@ -1767,16 +1769,17 @@ class VectorShufflePattern
         baseVector = vector2;
       }
 
-      Value dstIndex = LLVM::ConstantOp::create(rewriter,
-          loc, llvmI32Type, rewriter.getIntegerAttr(rewriter.getI32Type(), i));
-      Value index = LLVM::ConstantOp::create(rewriter,
-          loc, llvmI32Type,
+      Value dstIndex = LLVM::ConstantOp::create(
+          rewriter, loc, llvmI32Type,
+          rewriter.getIntegerAttr(rewriter.getI32Type(), i));
+      Value index = LLVM::ConstantOp::create(
+          rewriter, loc, llvmI32Type,
           rewriter.getIntegerAttr(rewriter.getI32Type(), indexVal - offsetVal));
 
-      auto extractOp = LLVM::ExtractElementOp::create(rewriter,
-          loc, scalarType, baseVector, index);
+      auto extractOp = LLVM::ExtractElementOp::create(rewriter, loc, scalarType,
+                                                      baseVector, index);
       targetOp = LLVM::InsertElementOp::create(rewriter, loc, dstType, targetOp,
-                                                        extractOp, dstIndex);
+                                               extractOp, dstIndex);
     }
     rewriter.replaceOp(op, targetOp);
     return success();
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index 58cffa6119d93..7025c5a7daf93 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -88,34 +88,34 @@ Value getBroadcastedDim(ImplicitLocOpBuilder lb, ValueRange extentTensors,
     Value shape = std::get<0>(tup);
     Value rankDiff = std::get<1>(tup);
     Value outOfBounds = arith::CmpIOp::create(lb, arith::CmpIPredicate::ult,
-                                                 outputDimension, rankDiff);
+                                              outputDimension, rankDiff);
     Type indexTy = lb.getIndexType();
     broadcastedDim =
-        IfOp::create(lb,
-              outOfBounds,
-              [&](OpBuilder &b, Location loc) {
-                scf::YieldOp::create(b, loc, broadcastedDim);
-              },
-              [&](OpBuilder &b, Location loc) {
-                // The broadcasting logic is:
-                // - if one extent (here we arbitrarily choose the
-                // extent from the greater-rank operand) is equal to 1,
-                // then take the extent from the other operand
-                // - otherwise, take the extent as-is.
-                // Note that this logic remains correct in the presence
-                // of dimensions of zero extent.
-                Value lesserRankOperandDimension = arith::SubIOp::create(b,
-                    loc, indexTy, outputDimension, rankDiff);
-                Value lesserRankOperandExtent = tensor::ExtractOp::create(b,
-                    loc, shape, ValueRange{lesserRankOperandDimension});
-
-                Value dimIsOne =
-                    arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
-                                            lesserRankOperandExtent, one);
-                Value dim = arith::SelectOp::create(b,
-                    loc, dimIsOne, broadcastedDim, lesserRankOperandExtent);
-                scf::YieldOp::create(b, loc, dim);
-              })
+        IfOp::create(
+            lb, outOfBounds,
+            [&](OpBuilder &b, Location loc) {
+              scf::YieldOp::create(b, loc, broadcastedDim);
+            },
+            [&](OpBuilder &b, Location loc) {
+              // The broadcasting logic is:
+              // - if one extent (here we arbitrarily choose the
+              // extent from the greater-rank operand) is equal to 1,
+              // then take the extent from the other operand
+              // - otherwise, take the extent as-is.
+              // Note that this logic remains correct in the presence
+              // of dimensions of zero extent.
+              Value lesserRankOperandDimension = arith::SubIOp::create(
+                  b, loc, indexTy, outputDimension, rankDiff);
+              Value lesserRankOperandExtent = tensor::ExtractOp::create(
+                  b, loc, shape, ValueRange{lesserRankOperandDimension});
+
+              Value dimIsOne =
+                  arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
+                                        lesserRankOperandExtent, one);
+              Value dim = arith::SelectOp::create(
+                  b, loc, dimIsOne, broadcastedDim, lesserRankOperandExtent);
+              scf::YieldOp::create(b, loc, dim);
+            })
             .getResult(0);
   }
   return broadcastedDim;
@@ -155,8 +155,8 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
                        return arith::SubIOp::create(lb, indexTy, maxRank, v);
                      }));
 
-  Value replacement = tensor::GenerateOp::create(lb,
-      getExtentTensorType(lb.getContext()), ValueRange{maxRank},
+  Value replacement = tensor::GenerateOp::create(
+      lb, getExtentTensorType(lb.getContext()), ValueRange{maxRank},
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value broadcastedDim =
             getBroadcastedDim(ImplicitLocOpBuilder(loc, b), adaptor.getShapes(),
@@ -193,8 +193,8 @@ LogicalResult ConstShapeOpConverter::matchAndRewrite(
   auto loc = op.getLoc();
   SmallVector<Value, 4> extentOperands;
   for (auto extent : op.getShape()) {
-    extentOperands.push_back(
-        arith::ConstantIndexOp::create(rewriter, loc, extent.getLimitedValue()));
+    extentOperands.push_back(arith::ConstantIndexOp::create(
+        rewriter, loc, extent.getLimitedValue()));
   }
   Type resultTy =
       RankedTensorType::get({op.getShape().size()}, rewriter.getIndexType());
@@ -269,11 +269,11 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
                      }));
 
   Type i1Ty = rewriter.getI1Type();
-  Value trueVal =
-      arith::ConstantOp::create(rewriter, loc, i1Ty, rewriter.getBoolAttr(true));
+  Value trueVal = arith::ConstantOp::create(rewriter, loc, i1Ty,
+                                            rewriter.getBoolAttr(true));
 
-  auto reduceResult = ForOp::create(lb,
-      loc, zero, maxRank, one, ValueRange{trueVal},
+  auto reduceResult = ForOp::create(
+      lb, loc, zero, maxRank, one, ValueRange{trueVal},
       [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
         // Find a non-1 dim, if it exists. Note that the first part of this
         // could reuse the Broadcast lowering entirely, but we redo the work
@@ -285,34 +285,34 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
         for (auto tup : llvm::zip(adaptor.getShapes(), rankDiffs)) {
           Value shape, rankDiff;
           std::tie(shape, rankDiff) = tup;
-          Value outOfBounds = arith::CmpIOp::create(b,
-              loc, arith::CmpIPredicate::ult, iv, rankDiff);
+          Value outOfBounds = arith::CmpIOp::create(
+              b, loc, arith::CmpIPredicate::ult, iv, rankDiff);
           broadcastable =
-              IfOp::create(b,
-                   loc, outOfBounds,
-                   [&](OpBuilder &b, Location loc) {
-                     // Non existent dimensions are always broadcastable
-                     scf::YieldOp::create(b, loc, broadcastable);
-                   },
-                   [&](OpBuilder &b, Location loc) {
-                     // Every value needs to be either 1, or the same non-1
-                     // value to be broadcastable in this dim.
-                     Value operandDimension =
-                         arith::SubIOp::create(b, loc, indexTy, iv, rankDiff);
-                     Value dimensionExtent = tensor::ExtractOp::create(b,
-                         loc, shape, ValueRange{operandDimension});
-
-                     Value equalOne = arith::CmpIOp::create(b,
-                         loc, arith::CmpIPredicate::eq, dimensionExtent, one);
-                     Value equalBroadcasted = arith::CmpIOp::create(b,
-                         loc, arith::CmpIPredicate::eq, dimensionExtent,
-                         broadcastedDim);
-                     Value result = arith::AndIOp::create(b,
-                         loc, broadcastable,
-                         arith::OrIOp::create(b, loc, equalOne,
-                                                equalBroadcasted));
-                     scf::YieldOp::create(b, loc, result);
-                   })
+              IfOp::create(
+                  b, loc, outOfBounds,
+                  [&](OpBuilder &b, Location loc) {
+                    // Non existent dimensions are always broadcastable
+                    scf::YieldOp::create(b, loc, broadcastable);
+                  },
+                  [&](OpBuilder &b, Location loc) {
+                    // Every value needs to be either 1, or the same non-1
+                    // value to be broadcastable in this dim.
+                    Value operandDimension =
+                        arith::SubIOp::create(b, loc, indexTy, iv, rankDiff);
+                    Value dimensionExtent = tensor::ExtractOp::create(
+                        b, loc, shape, ValueRange{operandDimension});
+
+                    Value equalOne = arith::CmpIOp::create(
+                        b, loc, arith::CmpIPredicate::eq, dimensionExtent, one);
+                    Value equalBroadcasted =
+                        arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq,
+                                              dimensionExtent, broadcastedDim);
+                    Value result = arith::AndIOp::create(
+                        b, loc, broadcastable,
+                        arith::OrIOp::create(b, loc, equalOne,
+                                             equalBroadcasted));
+                    scf::YieldOp::create(b, loc, result);
+                  })
                   .getResult(0);
         }
 
@@ -427,10 +427,11 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, OpAdaptor adaptor,
   Value rank =
       tensor::DimOp::create(rewriter, loc, indexTy, adaptor.getShape(), zero);
 
-  auto loop = scf::ForOp::create(rewriter,
-      loc, zero, rank, one, op.getInitVals(),
+  auto loop = scf::ForOp::create(
+      rewriter, loc, zero, rank, one, op.getInitVals(),
       [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
-        Value extent = tensor::ExtractOp::create(b, loc, adaptor.getShape(), iv);
+        Value extent =
+            tensor::ExtractOp::create(b, loc, adaptor.getShape(), iv);
 
         SmallVector<Value, 2> mappedValues{iv, extent};
         mappedValues.append(args.begin(), args.end());
@@ -515,23 +516,23 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, OpAdaptor adaptor,
   // Generate a linear sequence of compares, all with firstShape as lhs.
   for (Value shape : adaptor.getShapes().drop_front(1)) {
     Value rank = tensor::DimOp::create(rewriter, loc, indexTy, shape, zero);
-    Value eqRank = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                                  firstRank, rank);
-    auto same = IfOp::create(rewriter,
-        loc, eqRank,
+    Value eqRank = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, firstRank, rank);
+    auto same = IfOp::create(
+        rewriter, loc, eqRank,
         [&](OpBuilder &b, Location loc) {
           Value one = arith::ConstantIndexOp::create(b, loc, 1);
           Value init =
               arith::ConstantOp::create(b, loc, i1Ty, b.getBoolAttr(true));
-          auto loop = scf::ForOp::create(b,
-              loc, zero, firstRank, one, ValueRange{init},
+          auto loop = scf::ForOp::create(
+              b, loc, zero, firstRank, one, ValueRange{init},
               [&](OpBuilder &b, Location nestedLoc, Value iv, ValueRange args) {
                 Value conj = args[0];
                 Value lhsExtent =
                     tensor::ExtractOp::create(b, loc, firstShape, iv);
                 Value rhsExtent = tensor::ExtractOp::create(b, loc, shape, iv);
-                Value eqExtent = arith::CmpIOp::create(b,
-                    loc, arith::CmpIPredicate::eq, lhsExtent, rhsExtent);
+                Value eqExtent = arith::CmpIOp::create(
+                    b, loc, arith::CmpIPredicate::eq, lhsExtent, rhsExtent);
                 Value conjNext = arith::AndIOp::create(b, loc, conj, eqExtent);
                 scf::YieldOp::create(b, loc, ValueRange({conjNext}));
               });
@@ -544,7 +545,7 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, OpAdaptor adaptor,
         });
     result = !result ? same.getResult(0)
                      : arith::AndIOp::create(rewriter, loc, result,
-                                                      same.getResult(0));
+                                             same.getResult(0));
   }
   rewriter.replaceOp(op, result);
   return success();
@@ -584,15 +585,15 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
         Value extent = tensor::DimOp::create(rewriter, loc, tensor, i);
         extentValues.push_back(extent);
       } else {
-        Value extent = arith::ConstantIndexOp::create(rewriter,
-            loc, rankedTensorTy.getDimSize(i));
+        Value extent = arith::ConstantIndexOp::create(
+            rewriter, loc, rankedTensorTy.getDimSize(i));
         extentValues.push_back(extent);
       }
     }
 
     // Materialize extent tensor.
-    Value staticExtentTensor = tensor::FromElementsOp::create(rewriter,
-        loc, RankedTensorType::get({rank}, rewriter.getIndexType()),
+    Value staticExtentTensor = tensor::FromElementsOp::create(
+        rewriter, loc, RankedTensorType::get({rank}, rewriter.getIndexType()),
         extentValues);
     rewriter.replaceOpWithNewOp<tensor::CastOp>(op, op.getType(),
                                                 staticExtentTensor);
@@ -649,7 +650,7 @@ LogicalResult SplitAtOpConversion::matchAndRewrite(
       tensor::ExtractSliceOp::create(b, adaptor.getOperand(), zero, index, one);
   Value tailSize = arith::SubIOp::create(b, rank, index);
   Value tail = tensor::ExtractSliceOp::create(b, adaptor.getOperand(), index,
-                                                tailSize, one);
+                                              tailSize, one);
   rewriter.replaceOp(op, {head, tail});
   return success();
 }
diff --git a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
index f06c3e1f7cb18..f24972f6b6ee1 100644
--- a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
+++ b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
@@ -69,8 +69,8 @@ class TensorExtractPattern final
       // have bugs dealing with that. So for now, use spirv.Store for
       // initialization.
       varOp = spirv::VariableOp::create(rewriter, loc, varType,
-                                                 spirv::StorageClass::Function,
-                                                 /*initializer=*/nullptr);
+                                        spirv::StorageClass::Function,
+                                        /*initializer=*/nullptr);
       spirv::StoreOp::create(rewriter, loc, varOp, adaptor.getTensor());
     } else {
       // Need to store the value to the local variable. It's questionable
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 44872c8d7540d..044b725c7d805 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -51,8 +51,8 @@ TypedAttr getConstantAttr(Type type, int64_t value, PatternRewriter &rewriter) {
 
 Value getConstantValue(Location loc, Type type, int64_t value,
                        PatternRewriter &rewriter) {
-  return arith::ConstantOp::create(rewriter,
-      loc, getConstantAttr(type, value, rewriter));
+  return arith::ConstantOp::create(rewriter, loc,
+                                   getConstantAttr(type, value, rewriter));
 }
 
 // This converts the TOSA ApplyScale operator to a set of arithmetic ops,
@@ -104,13 +104,13 @@ class ApplyScaleGenericOpConverter
       int64_t roundInt = 1 << 30;
       Value roundUp = getConstantValue(loc, i64Ty, roundInt, rewriter);
       Value roundDown = getConstantValue(loc, i64Ty, -roundInt, rewriter);
-      Value positive = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::sge, value, zero);
+      Value positive = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::sge, value, zero);
       Value dir =
           arith::SelectOp::create(rewriter, loc, positive, roundUp, roundDown);
       Value val = arith::AddIOp::create(rewriter, loc, dir, multiply64);
-      Value valid = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::sgt, shift32, thirtyOne32);
+      Value valid = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::sgt, shift32, thirtyOne32);
       multiply64 =
           arith::SelectOp::create(rewriter, loc, valid, val, multiply64);
     }
@@ -163,10 +163,10 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
     Value high32 = value64.getHigh();
 
     // Determine the direction and amount to shift the high bits.
-    Value shiftOver32 = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sge, shift32, thirtyTwo32);
-    Value roundHighBits = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sgt, shift32, thirtyTwo32);
+    Value shiftOver32 = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sge, shift32, thirtyTwo32);
+    Value roundHighBits = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sgt, shift32, thirtyTwo32);
 
     Value shiftHighL =
         arith::SubIOp::create(rewriter, loc, thirtyTwo32, shift32);
@@ -181,11 +181,11 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
     // Conditionally perform our double round.
     if (op.getRoundingMode() == "DOUBLE_ROUND") {
       Value negOne32 = getConstantValue(loc, i32Ty, -1, rewriter);
-      Value valuePositive = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::sge, value32, zero32);
+      Value valuePositive = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::sge, value32, zero32);
 
-      Value roundDir =
-          arith::SelectOp::create(rewriter, loc, valuePositive, one32, negOne32);
+      Value roundDir = arith::SelectOp::create(rewriter, loc, valuePositive,
+                                               one32, negOne32);
       roundDir =
           arith::SelectOp::create(rewriter, loc, shiftOver32, roundDir, zero32);
 
@@ -205,14 +205,15 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
       Value shiftSubOne = arith::SubIOp::create(rewriter, loc, shift32, one32);
       Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
       roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, zero32,
-                                                  roundBit);
+                                         roundBit);
 
       Value newLow32 = arith::AddIOp::create(rewriter, loc, low32, roundBit);
-      Value wasRounded = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::ugt, low32, newLow32);
+      Value wasRounded = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::ugt, low32, newLow32);
       low32 = newLow32;
 
-      Value rounded32 = arith::ExtUIOp::create(rewriter, loc, i32Ty, wasRounded);
+      Value rounded32 =
+          arith::ExtUIOp::create(rewriter, loc, i32Ty, wasRounded);
       high32 = arith::AddIOp::create(rewriter, loc, high32, rounded32);
     }
 
@@ -222,7 +223,7 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
           arith::SubIOp::create(rewriter, loc, shiftHighR, one32);
       Value roundBit = arith::ShLIOp::create(rewriter, loc, one32, shiftSubOne);
       roundBit = arith::SelectOp::create(rewriter, loc, roundHighBits, roundBit,
-                                                  zero32);
+                                         zero32);
       high32 = arith::AddIOp::create(rewriter, loc, high32, roundBit);
     }
 
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 2aec35ed9d8ca..ec55091cd7eb8 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -70,14 +70,14 @@ materializeBinaryNanCheckIfRequired(OpTy op, PatternRewriter &rewriter,
     return result;
 
   // Unordered comparison of NaN against itself will always return true.
-  Value lhsIsNaN = arith::CmpFOp::create(rewriter,
-      op.getLoc(), arith::CmpFPredicate::UNO, lhs, lhs);
-  Value rhsIsNaN = arith::CmpFOp::create(rewriter,
-      op.getLoc(), arith::CmpFPredicate::UNO, rhs, rhs);
+  Value lhsIsNaN = arith::CmpFOp::create(rewriter, op.getLoc(),
+                                         arith::CmpFPredicate::UNO, lhs, lhs);
+  Value rhsIsNaN = arith::CmpFOp::create(rewriter, op.getLoc(),
+                                         arith::CmpFPredicate::UNO, rhs, rhs);
   Value rhsOrResult =
       arith::SelectOp::create(rewriter, op.getLoc(), lhsIsNaN, rhs, result);
   return arith::SelectOp::create(rewriter, op.getLoc(), rhsIsNaN, lhs,
-                                          rhsOrResult);
+                                 rhsOrResult);
 }
 
 static Value createLinalgBodyCalculationForElementwiseOp(
@@ -92,8 +92,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     return math::AbsFOp::create(rewriter, loc, resultTypes, args);
 
   if (isa<tosa::AbsOp>(op) && isa<IntegerType>(elementTy)) {
-    auto zero = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(elementTy));
+    auto zero = arith::ConstantOp::create(rewriter, loc,
+                                          rewriter.getZeroAttr(elementTy));
     auto neg = arith::SubIOp::create(rewriter, loc, zero, args[0]);
     return arith::MaxSIOp::create(rewriter, loc, args[0], neg);
   }
@@ -140,7 +140,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                                           "Cannot have shift value for float");
         return nullptr;
       }
-      return arith::MulFOp::create(rewriter, loc, resultTypes, args[0], args[1]);
+      return arith::MulFOp::create(rewriter, loc, resultTypes, args[0],
+                                   args[1]);
     }
 
     if (isa<IntegerType>(elementTy)) {
@@ -156,8 +157,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         if (!b.getType().isInteger(32))
           b = arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), b);
 
-        auto result = tosa::ApplyScaleOp::create(rewriter,
-            loc, rewriter.getI32Type(), a, b, shiftConst,
+        auto result = tosa::ApplyScaleOp::create(
+            rewriter, loc, rewriter.getI32Type(), a, b, shiftConst,
             rewriter.getStringAttr("SINGLE_ROUND"));
 
         if (elementTy.isInteger(32))
@@ -205,10 +206,10 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     if (isa<IntegerType>(elementTy)) {
       if (!inZp && !outZp) {
-        auto constant = arith::ConstantOp::create(rewriter,
-            loc, IntegerAttr::get(elementTy, 0));
+        auto constant = arith::ConstantOp::create(
+            rewriter, loc, IntegerAttr::get(elementTy, 0));
         return arith::SubIOp::create(rewriter, loc, resultTypes, constant,
-                                              args[0]);
+                                     args[0]);
       }
 
       // Compute the maximum value that can occur in the intermediate buffer.
@@ -231,8 +232,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       }
 
       Type intermediateType = rewriter.getIntegerType(intermediateBitWidth);
-      Value zpAddValue = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
+      Value zpAddValue = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIntegerAttr(intermediateType, zpAdd));
 
       // The negation can be applied by doing:
       //  outputValue = inZp + outZp - inputValue
@@ -241,11 +242,11 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       auto sub = arith::SubIOp::create(rewriter, loc, zpAddValue, ext);
 
       // Clamp to the negation range.
-      Value min = arith::ConstantIntOp::create(rewriter,
-          loc, intermediateType,
+      Value min = arith::ConstantIntOp::create(
+          rewriter, loc, intermediateType,
           APInt::getSignedMinValue(inputBitWidth).getSExtValue());
-      Value max = arith::ConstantIntOp::create(rewriter,
-          loc, intermediateType,
+      Value max = arith::ConstantIntOp::create(
+          rewriter, loc, intermediateType,
           APInt::getSignedMaxValue(inputBitWidth).getSExtValue());
       auto clamp = clampIntHelper(loc, sub, min, max, rewriter, false);
 
@@ -291,16 +292,16 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     }
 
     Type i1Ty = IntegerType::get(rewriter.getContext(), /*width=*/1);
-    auto one =
-        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 1));
-    auto zero =
-        arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(elementTy, 0));
+    auto one = arith::ConstantOp::create(rewriter, loc,
+                                         IntegerAttr::get(elementTy, 1));
+    auto zero = arith::ConstantOp::create(rewriter, loc,
+                                          IntegerAttr::get(elementTy, 0));
     auto i1one =
         arith::ConstantOp::create(rewriter, loc, IntegerAttr::get(i1Ty, 1));
 
     // Checking that input2 != 0
-    auto shiftValueGreaterThanZero = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::sgt, args[1], zero);
+    auto shiftValueGreaterThanZero = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sgt, args[1], zero);
 
     // Checking for the last bit of input1 to be 1
     auto subtract =
@@ -308,13 +309,13 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     auto shifted =
         arith::ShRSIOp::create(rewriter, loc, resultTypes, args[0], subtract)
             ->getResults();
-    auto truncated = arith::TruncIOp::create(rewriter,
-        loc, i1Ty, shifted, ArrayRef<NamedAttribute>());
+    auto truncated = arith::TruncIOp::create(rewriter, loc, i1Ty, shifted,
+                                             ArrayRef<NamedAttribute>());
     auto isInputOdd =
         arith::AndIOp::create(rewriter, loc, i1Ty, truncated, i1one);
 
-    auto shouldRound = arith::AndIOp::create(rewriter,
-        loc, i1Ty, shiftValueGreaterThanZero, isInputOdd);
+    auto shouldRound = arith::AndIOp::create(
+        rewriter, loc, i1Ty, shiftValueGreaterThanZero, isInputOdd);
     auto extended =
         arith::ExtUIOp::create(rewriter, loc, resultTypes, shouldRound);
     return arith::AddIOp::create(rewriter, loc, resultTypes, result, extended);
@@ -331,8 +332,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
   // tosa::LogicalNot
   if (isa<tosa::LogicalNotOp>(op) && elementTy.isInteger(1)) {
-    auto one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerAttr(elementTy, 1));
+    auto one = arith::ConstantOp::create(rewriter, loc,
+                                         rewriter.getIntegerAttr(elementTy, 1));
     return arith::XOrIOp::create(rewriter, loc, resultTypes, args[0], one);
   }
 
@@ -379,29 +380,29 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   // tosa::GreaterOp
   if (isa<tosa::GreaterOp>(op) && isa<FloatType>(elementTy))
     return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGT,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   if (isa<tosa::GreaterOp>(op) && elementTy.isSignlessInteger())
     return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sgt,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   // tosa::GreaterEqualOp
   if (isa<tosa::GreaterEqualOp>(op) && isa<FloatType>(elementTy))
     return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OGE,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   if (isa<tosa::GreaterEqualOp>(op) && elementTy.isSignlessInteger())
     return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sge,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   // tosa::EqualOp
   if (isa<tosa::EqualOp>(op) && isa<FloatType>(elementTy))
     return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OEQ,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   if (isa<tosa::EqualOp>(op) && elementTy.isSignlessInteger())
     return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                          args[0], args[1]);
+                                 args[0], args[1]);
 
   // tosa::SelectOp
   if (isa<tosa::SelectOp>(op)) {
@@ -449,10 +450,10 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                    APFloat::rmNearestTiesToEven, &losesInfo);
     maxApf.convert(cast<FloatType>(elementTy).getFloatSemantics(),
                    APFloat::rmNearestTiesToEven, &losesInfo);
-    auto min = arith::ConstantOp::create(rewriter,
-        loc, elementTy, rewriter.getFloatAttr(elementTy, minApf));
-    auto max = arith::ConstantOp::create(rewriter,
-        loc, elementTy, rewriter.getFloatAttr(elementTy, maxApf));
+    auto min = arith::ConstantOp::create(
+        rewriter, loc, elementTy, rewriter.getFloatAttr(elementTy, minApf));
+    auto max = arith::ConstantOp::create(
+        rewriter, loc, elementTy, rewriter.getFloatAttr(elementTy, maxApf));
     auto result = clampFloatHelper(loc, args[0], min, max, rewriter);
 
     auto clampOp = llvm::cast<tosa::ClampOp>(op);
@@ -478,8 +479,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     //   return init if x == NaN else result
 
     // Unordered comparison of NaN against itself will always return true.
-    Value isNaN = arith::CmpFOp::create(rewriter,
-        op->getLoc(), arith::CmpFPredicate::UNO, args[0], args[0]);
+    Value isNaN = arith::CmpFOp::create(
+        rewriter, op->getLoc(), arith::CmpFPredicate::UNO, args[0], args[0]);
     // TOSA specifies that in "ignore" NaN mode the result is "min" if the input
     // is NaN.
     return arith::SelectOp::create(rewriter, op->getLoc(), isNaN, min, result);
@@ -515,10 +516,10 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     min = std::min(min, maxRepresentable);
     max = std::min(max, maxRepresentable);
 
-    auto minVal = arith::ConstantIntOp::create(rewriter,
-        loc, min, intTy.getIntOrFloatBitWidth());
-    auto maxVal = arith::ConstantIntOp::create(rewriter,
-        loc, max, intTy.getIntOrFloatBitWidth());
+    auto minVal = arith::ConstantIntOp::create(rewriter, loc, min,
+                                               intTy.getIntOrFloatBitWidth());
+    auto maxVal = arith::ConstantIntOp::create(rewriter, loc, max,
+                                               intTy.getIntOrFloatBitWidth());
     return clampIntHelper(loc, args[0], minVal, maxVal, rewriter,
                           intTy.isUnsignedInteger());
   }
@@ -550,20 +551,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && bitExtend)
       return arith::ExtFOp::create(rewriter, loc, resultTypes, args,
-                                            ArrayRef<NamedAttribute>());
+                                   ArrayRef<NamedAttribute>());
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && !bitExtend)
       return arith::TruncFOp::create(rewriter, loc, resultTypes, args,
-                                              ArrayRef<NamedAttribute>());
+                                     ArrayRef<NamedAttribute>());
 
     // 1-bit integers need to be treated as signless.
     if (srcTy.isInteger(1) && arith::UIToFPOp::areCastCompatible(srcTy, dstTy))
       return arith::UIToFPOp::create(rewriter, loc, resultTypes, args,
-                                              ArrayRef<NamedAttribute>());
+                                     ArrayRef<NamedAttribute>());
 
     if (srcTy.isInteger(1) && isa<IntegerType>(dstTy) && bitExtend)
       return arith::ExtUIOp::create(rewriter, loc, resultTypes, args,
-                                             ArrayRef<NamedAttribute>());
+                                    ArrayRef<NamedAttribute>());
 
     // Unsigned integers need an unrealized cast so that they can be passed
     // to UIToFP.
@@ -575,20 +576,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
                   args[0])
               .getResult(0);
       return arith::UIToFPOp::create(rewriter, loc, resultTypes[0],
-                                              unrealizedCast);
+                                     unrealizedCast);
     }
 
     // All other si-to-fp conversions should be handled by SIToFP.
     if (arith::SIToFPOp::areCastCompatible(srcTy, dstTy))
       return arith::SIToFPOp::create(rewriter, loc, resultTypes, args,
-                                              ArrayRef<NamedAttribute>());
+                                     ArrayRef<NamedAttribute>());
 
     // Casting to boolean, floats need to only be checked as not-equal to zero.
     if (isa<FloatType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getFloatAttr(srcTy, 0.0));
+      Value zero = arith::ConstantOp::create(rewriter, loc,
+                                             rewriter.getFloatAttr(srcTy, 0.0));
       return arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNE,
-                                            args.front(), zero);
+                                   args.front(), zero);
     }
 
     if (arith::FPToSIOp::areCastCompatible(srcTy, dstTy)) {
@@ -602,36 +603,41 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         // Use cmp + select to replace infinites by int min / int max. Other
         // integral values can be represented in the integer space.
         auto conv = arith::FPToSIOp::create(rewriter, loc, dstTy, rounded);
-        auto posInf = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getFloatAttr(getElementTypeOrSelf(srcTy),
-                                       APFloat::getInf(fltSemantics)));
-        auto negInf = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getFloatAttr(
-                     getElementTypeOrSelf(srcTy),
-                     APFloat::getInf(fltSemantics, /*Negative=*/true)));
-        auto overflow = arith::CmpFOp::create(rewriter,
-            loc, arith::CmpFPredicate::UEQ, rounded, posInf);
-        auto underflow = arith::CmpFOp::create(rewriter,
-            loc, arith::CmpFPredicate::UEQ, rounded, negInf);
-        auto intMin = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getIntegerAttr(
-                     getElementTypeOrSelf(dstTy),
-                     APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())));
-        auto intMax = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getIntegerAttr(
-                     getElementTypeOrSelf(dstTy),
-                     APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
+        auto posInf = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getFloatAttr(getElementTypeOrSelf(srcTy),
+                                  APFloat::getInf(fltSemantics)));
+        auto negInf = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getFloatAttr(
+                getElementTypeOrSelf(srcTy),
+                APFloat::getInf(fltSemantics, /*Negative=*/true)));
+        auto overflow = arith::CmpFOp::create(
+            rewriter, loc, arith::CmpFPredicate::UEQ, rounded, posInf);
+        auto underflow = arith::CmpFOp::create(
+            rewriter, loc, arith::CmpFPredicate::UEQ, rounded, negInf);
+        auto intMin = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getIntegerAttr(
+                getElementTypeOrSelf(dstTy),
+                APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())));
+        auto intMax = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getIntegerAttr(
+                getElementTypeOrSelf(dstTy),
+                APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
         auto maxClamped =
             arith::SelectOp::create(rewriter, loc, overflow, intMax, conv);
         return arith::SelectOp::create(rewriter, loc, underflow, intMin,
-                                                maxClamped);
+                                       maxClamped);
       }
 
-      auto intMinFP = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getFloatAttr(
-                   getElementTypeOrSelf(srcTy),
-                   APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())
-                       .getSExtValue()));
+      auto intMinFP = arith::ConstantOp::create(
+          rewriter, loc,
+          rewriter.getFloatAttr(
+              getElementTypeOrSelf(srcTy),
+              APInt::getSignedMinValue(dstTy.getIntOrFloatBitWidth())
+                  .getSExtValue()));
 
       // Check whether the mantissa has enough bits to represent int max.
       if (cast<FloatType>(srcTy).getFPMantissaWidth() >=
@@ -640,11 +646,12 @@ static Value createLinalgBodyCalculationForElementwiseOp(
         // consists of a single leading bit. Therefore we can clamp the input
         // in the floating-point domain.
 
-        auto intMaxFP = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getFloatAttr(
-                     getElementTypeOrSelf(srcTy),
-                     APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
-                         .getSExtValue()));
+        auto intMaxFP = arith::ConstantOp::create(
+            rewriter, loc,
+            rewriter.getFloatAttr(
+                getElementTypeOrSelf(srcTy),
+                APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
+                    .getSExtValue()));
 
         Value clamped =
             clampFloatHelper(loc, rounded, intMinFP, intMaxFP, rewriter);
@@ -655,40 +662,42 @@ static Value createLinalgBodyCalculationForElementwiseOp(
       // int min. We can therefore rely on int max + 1 being representable as
       // well because it's just int min with a positive sign. So clamp the min
       // value and compare against that to select the max int value if needed.
-      auto intMaxPlusOneFP = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getFloatAttr(
-                   getElementTypeOrSelf(srcTy),
-                   static_cast<double>(
-                       APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
-                           .getSExtValue()) +
-                       1.0f));
-
-      auto intMax = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIntegerAttr(
-                   getElementTypeOrSelf(dstTy),
-                   APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
+      auto intMaxPlusOneFP = arith::ConstantOp::create(
+          rewriter, loc,
+          rewriter.getFloatAttr(
+              getElementTypeOrSelf(srcTy),
+              static_cast<double>(
+                  APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())
+                      .getSExtValue()) +
+                  1.0f));
+
+      auto intMax = arith::ConstantOp::create(
+          rewriter, loc,
+          rewriter.getIntegerAttr(
+              getElementTypeOrSelf(dstTy),
+              APInt::getSignedMaxValue(dstTy.getIntOrFloatBitWidth())));
       auto minClampedFP =
           arith::MaximumFOp::create(rewriter, loc, rounded, intMinFP);
       auto minClamped =
           arith::FPToSIOp::create(rewriter, loc, dstTy, minClampedFP);
-      auto overflow = arith::CmpFOp::create(rewriter,
-          loc, arith::CmpFPredicate::UGE, rounded, intMaxPlusOneFP);
+      auto overflow = arith::CmpFOp::create(
+          rewriter, loc, arith::CmpFPredicate::UGE, rounded, intMaxPlusOneFP);
       return arith::SelectOp::create(rewriter, loc, overflow, intMax,
-                                              minClamped);
+                                     minClamped);
     }
 
     // Casting to boolean, integers need to only be checked as not-equal to
     // zero.
     if (isa<IntegerType>(srcTy) && dstTy.isInteger(1)) {
-      Value zero = arith::ConstantIntOp::create(rewriter,
-          loc, 0, srcTy.getIntOrFloatBitWidth());
+      Value zero = arith::ConstantIntOp::create(rewriter, loc, 0,
+                                                srcTy.getIntOrFloatBitWidth());
       return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
-                                            args.front(), zero);
+                                   args.front(), zero);
     }
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && bitExtend)
       return arith::ExtSIOp::create(rewriter, loc, resultTypes, args,
-                                             ArrayRef<NamedAttribute>());
+                                    ArrayRef<NamedAttribute>());
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && !bitExtend) {
       return arith::TruncIOp::create(rewriter, loc, dstTy, args[0]);
@@ -838,8 +847,8 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
   // Check if broadcast is necessary
   auto one = createIndex(rewriter, loc, indexPool, 1);
   auto runtimeSize = getTensorDim(rewriter, loc, indexPool, operand, dim);
-  auto broadcastNecessary = arith::CmpIOp::create(rewriter,
-      loc, arith::CmpIPredicate::eq, runtimeSize, one);
+  auto broadcastNecessary = arith::CmpIOp::create(
+      rewriter, loc, arith::CmpIPredicate::eq, runtimeSize, one);
 
   // Emit 'then' region of 'scf.if'
   auto emitThenRegion = [&](OpBuilder &opBuilder, Location loc) {
@@ -855,8 +864,8 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
                                                     operand, index);
       outputTensorShape.push_back(size);
     }
-    Value outputTensor = tensor::EmptyOp::create(opBuilder,
-        loc, outputTensorShape, rankedTensorType.getElementType());
+    Value outputTensor = tensor::EmptyOp::create(
+        opBuilder, loc, outputTensorShape, rankedTensorType.getElementType());
 
     // Emit 'linalg.generic' op
     auto resultTensor =
@@ -885,7 +894,7 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
 
   // Emit 'scf.if' op
   auto ifOp = scf::IfOp::create(rewriter, loc, broadcastNecessary,
-                                         emitThenRegion, emitElseRegion);
+                                emitThenRegion, emitElseRegion);
   return ifOp.getResult(0);
 }
 
@@ -930,8 +939,8 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
   if (!resultType) {
     return rewriter.notifyMatchFailure(operation, "failed to convert type");
   }
-  Value outputTensor = tensor::EmptyOp::create(rewriter,
-      loc, targetShape, resultType.getElementType());
+  Value outputTensor = tensor::EmptyOp::create(rewriter, loc, targetShape,
+                                               resultType.getElementType());
 
   // Create affine maps. Input affine maps broadcast static dimensions of size
   // 1. The output affine map is an identity map.
@@ -957,8 +966,8 @@ emitElementwiseComputation(ConversionPatternRewriter &rewriter, Location loc,
 
   // Emit 'linalg.generic' op
   bool encounteredError = false;
-  auto linalgOp = linalg::GenericOp::create(rewriter,
-      loc, outputTensor.getType(), operands, outputTensor, affineMaps,
+  auto linalgOp = linalg::GenericOp::create(
+      rewriter, loc, outputTensor.getType(), operands, outputTensor, affineMaps,
       getNParallelLoopsAttrs(rank),
       [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
         Value opResult = createLinalgBodyCalculationForElementwiseOp(
@@ -1202,8 +1211,8 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
   }
 
   bool didEncounterError = false;
-  linalg::LinalgOp linalgOp = linalg::ReduceOp::create(rewriter,
-      loc, inputs, outputs, axis,
+  linalg::LinalgOp linalgOp = linalg::ReduceOp::create(
+      rewriter, loc, inputs, outputs, axis,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange blockArgs) {
         std::array<Value, 2> binaryArgs{
             blockArgs[0], isNanIgnoreMode ? blockArgs[2] : blockArgs[1]};
@@ -1219,15 +1228,16 @@ static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
           auto oldAllResultsNanFlagValue = blockArgs[3];
 
           // Unordered comparison of NaN against itself will always return true.
-          Value isNaN = arith::CmpFOp::create(nestedBuilder,
-              op->getLoc(), arith::CmpFPredicate::UNO, inputValue, inputValue);
+          Value isNaN = arith::CmpFOp::create(nestedBuilder, op->getLoc(),
+                                              arith::CmpFPredicate::UNO,
+                                              inputValue, inputValue);
           // If we've encountered a NaN, take the non-NaN value.
-          auto selectOp = arith::SelectOp::create(nestedBuilder,
-              op->getLoc(), isNaN, initialValue, result);
+          auto selectOp = arith::SelectOp::create(nestedBuilder, op->getLoc(),
+                                                  isNaN, initialValue, result);
           // Update the flag which keeps track of whether we have seen a non-NaN
           // value.
-          auto newAllResultsNanFlagValue = arith::AndIOp::create(nestedBuilder,
-              op->getLoc(), oldAllResultsNanFlagValue, isNaN);
+          auto newAllResultsNanFlagValue = arith::AndIOp::create(
+              nestedBuilder, op->getLoc(), oldAllResultsNanFlagValue, isNaN);
           resultsToYield.push_back(selectOp);
           resultsToYield.push_back(newAllResultsNanFlagValue);
         } else {
@@ -1401,16 +1411,17 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value multiplierConstant;
     int64_t multiplierArg = 0;
     if (multiplierValues.size() == 1) {
-      multiplierConstant = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32IntegerAttr(multiplierValues.front()));
+      multiplierConstant = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32IntegerAttr(multiplierValues.front()));
     } else {
       SmallVector<AffineExpr, 2> multiplierExprs{
           rewriter.getAffineDimExpr(rank - 1)};
       auto multiplierType =
           RankedTensorType::get({static_cast<int64_t>(multiplierValues.size())},
                                 rewriter.getI32Type());
-      genericInputs.push_back(arith::ConstantOp::create(rewriter,
-          loc, DenseIntElementsAttr::get(multiplierType, multiplierValues)));
+      genericInputs.push_back(arith::ConstantOp::create(
+          rewriter, loc,
+          DenseIntElementsAttr::get(multiplierType, multiplierValues)));
 
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
                                             /*symbolCount=*/0, multiplierExprs,
@@ -1424,16 +1435,16 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     Value shiftConstant;
     int64_t shiftArg = 0;
     if (shiftValues.size() == 1) {
-      shiftConstant = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI8IntegerAttr(shiftValues.front()));
+      shiftConstant = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI8IntegerAttr(shiftValues.front()));
     } else {
       SmallVector<AffineExpr, 2> shiftExprs = {
           rewriter.getAffineDimExpr(rank - 1)};
       auto shiftType =
           RankedTensorType::get({static_cast<int64_t>(shiftValues.size())},
                                 rewriter.getIntegerType(8));
-      genericInputs.push_back(arith::ConstantOp::create(rewriter,
-          loc, DenseIntElementsAttr::get(shiftType, shiftValues)));
+      genericInputs.push_back(arith::ConstantOp::create(
+          rewriter, loc, DenseIntElementsAttr::get(shiftType, shiftValues)));
       indexingMaps.push_back(AffineMap::get(/*dimCount=*/rank,
                                             /*symbolCount=*/0, shiftExprs,
                                             rewriter.getContext()));
@@ -1444,13 +1455,13 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(rank));
 
     // Construct the indexing maps needed for linalg.generic ops.
-    Value emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, outputTy.getShape(), outputTy.getElementType(),
+    Value emptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, outputTy.getShape(), outputTy.getElementType(),
         ArrayRef<Value>({dynDims}));
 
-    auto linalgOp = linalg::GenericOp::create(rewriter,
-        loc, outputTy, genericInputs, ValueRange{emptyTensor}, indexingMaps,
-        getNParallelLoopsAttrs(rank),
+    auto linalgOp = linalg::GenericOp::create(
+        rewriter, loc, outputTy, genericInputs, ValueRange{emptyTensor},
+        indexingMaps, getNParallelLoopsAttrs(rank),
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
             ValueRange blockArgs) {
           Value value = blockArgs[0];
@@ -1466,9 +1477,10 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           const int32_t inBitwidth = valueTy.getIntOrFloatBitWidth();
           // Extend zeropoint for sub-32bits widths.
           const int32_t inAttrBitwidth = inBitwidth > 32 ? inBitwidth : 32;
-          auto inputZp = arith::ConstantOp::create(nestedBuilder,
-              loc, IntegerAttr::get(rewriter.getIntegerType(inAttrBitwidth),
-                                    *maybeIZp));
+          auto inputZp = arith::ConstantOp::create(
+              nestedBuilder, loc,
+              IntegerAttr::get(rewriter.getIntegerType(inAttrBitwidth),
+                               *maybeIZp));
 
           FailureOr<int64_t> maybeOZp = op.getOutputZeroPoint();
           if (failed(maybeOZp)) {
@@ -1482,9 +1494,10 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           unsigned outBitWidth = outIntType.getWidth();
           const int32_t outAttrBitwidth = 32;
           assert(outBitWidth <= 32 && "Unexpected output zeropoint bitwidth");
-          auto outputZp = arith::ConstantOp::create(nestedBuilder,
-              loc, IntegerAttr::get(rewriter.getIntegerType(outAttrBitwidth),
-                                    *maybeOZp));
+          auto outputZp = arith::ConstantOp::create(
+              nestedBuilder, loc,
+              IntegerAttr::get(rewriter.getIntegerType(outAttrBitwidth),
+                               *maybeOZp));
 
           Value multiplier = multiplierConstant ? multiplierConstant
                                                 : blockArgs[multiplierArg];
@@ -1501,20 +1514,20 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           }
           if (valueTy.getIntOrFloatBitWidth() < 32) {
             if (op.getInputUnsigned()) {
-              value = arith::ExtUIOp::create(nestedBuilder,
-                  nestedLoc, nestedBuilder.getI32Type(), value);
+              value = arith::ExtUIOp::create(nestedBuilder, nestedLoc,
+                                             nestedBuilder.getI32Type(), value);
             } else {
-              value = arith::ExtSIOp::create(nestedBuilder,
-                  nestedLoc, nestedBuilder.getI32Type(), value);
+              value = arith::ExtSIOp::create(nestedBuilder, nestedLoc,
+                                             nestedBuilder.getI32Type(), value);
             }
           }
 
           value =
               arith::SubIOp::create(nestedBuilder, nestedLoc, value, inputZp);
 
-          value = tosa::ApplyScaleOp::create(nestedBuilder,
-              loc, nestedBuilder.getI32Type(), value, multiplier, shift,
-              roundingMode);
+          value = tosa::ApplyScaleOp::create(nestedBuilder, loc,
+                                             nestedBuilder.getI32Type(), value,
+                                             multiplier, shift, roundingMode);
 
           // Move to the new zero-point.
           value =
@@ -1530,18 +1543,18 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
             intMax = APInt::getMaxValue(outBitWidth).getZExtValue();
           }
 
-          auto intMinVal = arith::ConstantOp::create(nestedBuilder,
-              loc, nestedBuilder.getI32IntegerAttr(intMin));
-          auto intMaxVal = arith::ConstantOp::create(nestedBuilder,
-              loc, nestedBuilder.getI32IntegerAttr(intMax));
+          auto intMinVal = arith::ConstantOp::create(
+              nestedBuilder, loc, nestedBuilder.getI32IntegerAttr(intMin));
+          auto intMaxVal = arith::ConstantOp::create(
+              nestedBuilder, loc, nestedBuilder.getI32IntegerAttr(intMax));
 
           value = clampIntHelper(nestedLoc, value, intMinVal, intMaxVal,
                                  nestedBuilder, /*isUnsigned=*/false);
 
           if (outIntType.getWidth() < 32) {
-            value = arith::TruncIOp::create(nestedBuilder,
-                nestedLoc, rewriter.getIntegerType(outIntType.getWidth()),
-                value);
+            value = arith::TruncIOp::create(
+                nestedBuilder, nestedLoc,
+                rewriter.getIntegerType(outIntType.getWidth()), value);
           }
 
           if (outIntType.isUnsignedInteger()) {
@@ -1609,7 +1622,7 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
         RankedTensorType::get({inputTy.getDimSize(0), inputTy.getDimSize(3)},
                               inputTy.getElementType());
     Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, input,
-                                                             reassociationMap);
+                                                     reassociationMap);
 
     // Get any dynamic shapes that appear in the input format.
     llvm::SmallVector<Value> outputDynSize;
@@ -1620,31 +1633,32 @@ class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
 
     // Generate the elementwise operation for casting scaling the input value.
     auto genericTy = collapseTy.clone(resultTy.getElementType());
-    Value empty = tensor::EmptyOp::create(builder,
-        genericTy.getShape(), resultTy.getElementType(), outputDynSize);
+    Value empty =
+        tensor::EmptyOp::create(builder, genericTy.getShape(),
+                                resultTy.getElementType(), outputDynSize);
     auto genericMap = rewriter.getMultiDimIdentityMap(genericTy.getRank());
     SmallVector<utils::IteratorType> iterators(genericTy.getRank(),
                                                utils::IteratorType::parallel);
 
-    auto generic = linalg::GenericOp::create(builder,
-        genericTy, ValueRange{collapse}, ValueRange{empty},
+    auto generic = linalg::GenericOp::create(
+        builder, genericTy, ValueRange{collapse}, ValueRange{empty},
         ArrayRef<AffineMap>{genericMap, genericMap}, iterators,
         [=](OpBuilder &b, Location loc, ValueRange args) {
           Value value = args[0];
           // This is the quantized case.
           if (inputTy.getElementType() != resultTy.getElementType()) {
-            value =
-                arith::ExtSIOp::create(b, loc, resultTy.getElementType(), value);
+            value = arith::ExtSIOp::create(b, loc, resultTy.getElementType(),
+                                           value);
 
             if (isBilinear && scale[0] != 0) {
-              Value scaleY = arith::ConstantOp::create(b,
-                  loc, b.getI32IntegerAttr(scale[0]));
+              Value scaleY = arith::ConstantOp::create(
+                  b, loc, b.getI32IntegerAttr(scale[0]));
               value = arith::MulIOp::create(b, loc, value, scaleY);
             }
 
             if (isBilinear && scale[2] != 0) {
-              Value scaleX = arith::ConstantOp::create(b,
-                  loc, b.getI32IntegerAttr(scale[2]));
+              Value scaleX = arith::ConstantOp::create(
+                  b, loc, b.getI32IntegerAttr(scale[2]));
               value = arith::MulIOp::create(b, loc, value, scaleX);
             }
           }
@@ -1697,9 +1711,9 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     resizeShape.push_back(channels);
 
     auto resizeTy = resultTy.clone(resizeShape);
-    auto resize = tosa::ResizeOp::create(builder, resizeTy, input, op.getScale(),
-                                                 op.getOffset(), op.getBorder(),
-                                                 op.getMode());
+    auto resize =
+        tosa::ResizeOp::create(builder, resizeTy, input, op.getScale(),
+                               op.getOffset(), op.getBorder(), op.getMode());
 
     // Collapse an unit result dims.
     SmallVector<ReassociationExprs, 4> reassociationMap(2);
@@ -1720,8 +1734,8 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
     collapseShape.push_back(channels);
 
     auto collapseTy = resultTy.clone(collapseShape);
-    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy, resize,
-                                                             reassociationMap);
+    Value collapse = tensor::CollapseShapeOp::create(builder, collapseTy,
+                                                     resize, reassociationMap);
 
     // Broadcast the collapsed shape to the output result.
     llvm::SmallVector<Value> outputDynSize;
@@ -1732,8 +1746,8 @@ class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
 
     SmallVector<utils::IteratorType> iterators(resultTy.getRank(),
                                                utils::IteratorType::parallel);
-    Value empty = tensor::EmptyOp::create(builder,
-        resultTy.getShape(), resultTy.getElementType(), outputDynSize);
+    Value empty = tensor::EmptyOp::create(
+        builder, resultTy.getShape(), resultTy.getElementType(), outputDynSize);
 
     SmallVector<AffineExpr, 4> inputExprs{rewriter.getAffineDimExpr(0)};
     if (inputH != 1)
@@ -1789,10 +1803,10 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
 
     SmallVector<AffineMap, 2> affineMaps = {
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
-    auto emptyTensor = tensor::EmptyOp::create(b, resultTy.getShape(), resultETy,
-                                                 *dynamicDimsOr);
-    auto genericOp = linalg::GenericOp::create(b,
-        resultTy, ValueRange({}), ValueRange{emptyTensor}, affineMaps,
+    auto emptyTensor = tensor::EmptyOp::create(b, resultTy.getShape(),
+                                               resultETy, *dynamicDimsOr);
+    auto genericOp = linalg::GenericOp::create(
+        b, resultTy, ValueRange({}), ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()));
     Value resize = genericOp.getResult(0);
 
@@ -1808,8 +1822,10 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
       Value zeroI32 =
           arith::ConstantOp::create(b, b.getZeroAttr(b.getI32Type()));
       Value zeroFp = arith::ConstantOp::create(b, b.getZeroAttr(floatTy));
-      Value hMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageH - 1));
-      Value wMax = arith::ConstantOp::create(b, b.getI32IntegerAttr(imageW - 1));
+      Value hMax =
+          arith::ConstantOp::create(b, b.getI32IntegerAttr(imageH - 1));
+      Value wMax =
+          arith::ConstantOp::create(b, b.getI32IntegerAttr(imageW - 1));
 
       Value inY = arith::IndexCastOp::create(b, b.getI32Type(), y);
       Value inX = arith::IndexCastOp::create(b, b.getI32Type(), x);
@@ -1898,12 +1914,13 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
 
           Value pred;
           if (floatingPointMode) {
-            auto h = arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 0.5f));
+            auto h =
+                arith::ConstantOp::create(b, b.getFloatAttr(floatTy, 0.5f));
             pred = arith::CmpFOp::create(b, arith::CmpFPredicate::OGE, dval, h);
           } else {
             Value dvalDouble = arith::ShLIOp::create(b, dval, one);
             pred = arith::CmpIOp::create(b, arith::CmpIPredicate::sge,
-                                           dvalDouble, scale);
+                                         dvalDouble, scale);
           }
 
           auto offset = arith::SelectOp::create(b, pred, one, zeroI32);
@@ -1915,8 +1932,8 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         iy = getNearestIndexAndClamp(iy, dy, yScaleN, hMax, imageH, b);
         ix = getNearestIndexAndClamp(ix, dx, xScaleN, wMax, imageW, b);
 
-        Value result = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, iy, ix, channel});
+        Value result = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, iy, ix, channel});
 
         linalg::YieldOp::create(b, result);
       } else {
@@ -1946,14 +1963,14 @@ class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
         getClampedIdxs(y0, y1, imageH, iy, hMax, b);
         getClampedIdxs(x0, x1, imageW, ix, wMax, b);
 
-        Value y0x0 = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, y0, x0, channel});
-        Value y0x1 = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, y0, x1, channel});
-        Value y1x0 = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, y1, x0, channel});
-        Value y1x1 = tensor::ExtractOp::create(b,
-            input, ValueRange{batch, y1, x1, channel});
+        Value y0x0 = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, y0, x0, channel});
+        Value y0x1 = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, y0, x1, channel});
+        Value y1x0 = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, y1, x0, channel});
+        Value y1x1 = tensor::ExtractOp::create(
+            b, input, ValueRange{batch, y1, x1, channel});
 
         if (floatingPointMode) {
           auto oneVal =
@@ -2100,16 +2117,16 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
               auto sizeMinusOne =
                   arith::SubIOp::create(rewriter, nestedLoc, axisDimSize, one);
               index = arith::SubIOp::create(rewriter, nestedLoc, sizeMinusOne,
-                                                     index);
+                                            index);
             }
 
             indices.push_back(index);
           }
 
-          auto extract = tensor::ExtractOp::create(nestedBuilder,
-              nestedLoc, input, indices);
+          auto extract = tensor::ExtractOp::create(nestedBuilder, nestedLoc,
+                                                   input, indices);
           linalg::YieldOp::create(nestedBuilder, op.getLoc(),
-                                                extract.getResult());
+                                  extract.getResult());
         });
     return success();
   }
@@ -2152,8 +2169,8 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
       }
     }
 
-    auto emptyTensor = tensor::EmptyOp::create(rewriter,
-        op.getLoc(), genericShape, elementTy, dynDims);
+    auto emptyTensor = tensor::EmptyOp::create(
+        rewriter, op.getLoc(), genericShape, elementTy, dynDims);
 
     // We needs to map the input shape to the non-broadcasted dimensions.
     SmallVector<AffineExpr, 4> dimExprs;
@@ -2168,8 +2185,8 @@ struct TileConverter : public OpConversionPattern<tosa::TileOp> {
     SmallVector<AffineMap, 2> affineMaps = {
         readAffineMap, rewriter.getMultiDimIdentityMap(genericShape.size())};
 
-    auto genericOp = linalg::GenericOp::create(rewriter,
-        loc, RankedTensorType::get(genericShape, elementTy), input,
+    auto genericOp = linalg::GenericOp::create(
+        rewriter, loc, RankedTensorType::get(genericShape, elementTy), input,
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(genericShape.size()),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -2229,8 +2246,8 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
                               .create<tensor::EmptyOp>(loc, resultTy.getShape(),
                                                        outElementTy, dynDims)
                               .getResult();
-    auto fillValueIdx = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerAttr(outElementTy, 0));
+    auto fillValueIdx = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getIntegerAttr(outElementTy, 0));
     auto filledTensorIdx =
         rewriter
             .create<linalg::FillOp>(loc, ValueRange{fillValueIdx},
@@ -2274,8 +2291,8 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
     bool didEncounterError = false;
     auto maps = AffineMap::inferFromExprList({srcExprs, dstExprs, dstExprs},
                                              rewriter.getContext());
-    auto linalgOp = linalg::GenericOp::create(rewriter,
-        loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
+    auto linalgOp = linalg::GenericOp::create(
+        rewriter, loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
         ValueRange({filledTensorIdx, filledTensorMax}), maps, iteratorTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc,
             ValueRange blockArgs) {
@@ -2283,8 +2300,8 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
           auto oldIndex = blockArgs[1];
           auto oldValue = blockArgs[2];
 
-          Value newIndex = arith::IndexCastOp::create(rewriter,
-              nestedLoc, oldIndex.getType(),
+          Value newIndex = arith::IndexCastOp::create(
+              rewriter, nestedLoc, oldIndex.getType(),
               linalg::IndexOp::create(rewriter, loc, axis));
 
           Value predicate;
@@ -2292,33 +2309,37 @@ class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
             if (argmaxOp.getNanMode() == "IGNORE") {
               // Only update index & max value for non NaN values. If all
               // values are NaNs, the initial index will be return which is 0.
-              predicate = arith::CmpFOp::create(rewriter,
-                  nestedLoc, arith::CmpFPredicate::OGT, newValue, oldValue);
+              predicate = arith::CmpFOp::create(rewriter, nestedLoc,
+                                                arith::CmpFPredicate::OGT,
+                                                newValue, oldValue);
             } else {
               // Update max value if either of the following is true:
               // - new value is bigger
               // - cur max is not NaN and new value is NaN
-              Value gt = arith::CmpFOp::create(rewriter,
-                  nestedLoc, arith::CmpFPredicate::UGT, newValue, oldValue);
-              Value oldNonNaN = arith::CmpFOp::create(rewriter,
-                  nestedLoc, arith::CmpFPredicate::ORD, oldValue, oldValue);
-              predicate = arith::AndIOp::create(rewriter,
-                  nestedLoc, rewriter.getI1Type(), gt, oldNonNaN);
+              Value gt = arith::CmpFOp::create(rewriter, nestedLoc,
+                                               arith::CmpFPredicate::UGT,
+                                               newValue, oldValue);
+              Value oldNonNaN = arith::CmpFOp::create(rewriter, nestedLoc,
+                                                      arith::CmpFPredicate::ORD,
+                                                      oldValue, oldValue);
+              predicate = arith::AndIOp::create(
+                  rewriter, nestedLoc, rewriter.getI1Type(), gt, oldNonNaN);
             }
           } else if (isa<IntegerType>(inElementTy)) {
-            predicate = arith::CmpIOp::create(rewriter,
-                nestedLoc, arith::CmpIPredicate::sgt, newValue, oldValue);
+            predicate = arith::CmpIOp::create(rewriter, nestedLoc,
+                                              arith::CmpIPredicate::sgt,
+                                              newValue, oldValue);
           } else {
             didEncounterError = true;
             return;
           }
 
-          auto resultMax = arith::SelectOp::create(rewriter,
-              nestedLoc, predicate, newValue, oldValue);
-          auto resultIndex = arith::SelectOp::create(rewriter,
-              nestedLoc, predicate, newIndex, oldIndex);
-          linalg::YieldOp::create(nestedBuilder,
-              nestedLoc, ValueRange({resultIndex, resultMax}));
+          auto resultMax = arith::SelectOp::create(
+              rewriter, nestedLoc, predicate, newValue, oldValue);
+          auto resultIndex = arith::SelectOp::create(
+              rewriter, nestedLoc, predicate, newIndex, oldIndex);
+          linalg::YieldOp::create(nestedBuilder, nestedLoc,
+                                  ValueRange({resultIndex, resultMax}));
         });
 
     if (didEncounterError)
@@ -2363,18 +2384,18 @@ class GatherConverter : public OpConversionPattern<tosa::GatherOp> {
             rewriter.getContext()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = linalg::GenericOp::create(rewriter,
-        loc, ArrayRef<Type>({resultTy}), ValueRange{indices},
+    auto genericOp = linalg::GenericOp::create(
+        rewriter, loc, ArrayRef<Type>({resultTy}), ValueRange{indices},
         ValueRange{emptyTensor}, affineMaps,
         getNParallelLoopsAttrs(resultTy.getRank()),
         [&](OpBuilder &b, Location loc, ValueRange args) {
           auto indexValue = args[0];
           auto index0 = linalg::IndexOp::create(rewriter, loc, 0);
-          Value index1 = arith::IndexCastOp::create(rewriter,
-              loc, rewriter.getIndexType(), indexValue);
+          Value index1 = arith::IndexCastOp::create(
+              rewriter, loc, rewriter.getIndexType(), indexValue);
           auto index2 = linalg::IndexOp::create(rewriter, loc, 2);
-          Value extract = tensor::ExtractOp::create(rewriter,
-              loc, input, ValueRange{index0, index1, index2});
+          Value extract = tensor::ExtractOp::create(
+              rewriter, loc, input, ValueRange{index0, index1, index2});
           linalg::YieldOp::create(rewriter, loc, extract);
         });
     rewriter.replaceOp(op, genericOp.getResult(0));
@@ -2437,9 +2458,9 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
         rewriter.getMultiDimIdentityMap(resultTy.getRank()),
         rewriter.getMultiDimIdentityMap(resultTy.getRank())};
 
-    auto genericOp = linalg::GenericOp::create(rewriter,
-        loc, resultTy, ValueRange({input}), ValueRange{emptyTensor}, affineMaps,
-        getNParallelLoopsAttrs(resultTy.getRank()));
+    auto genericOp = linalg::GenericOp::create(
+        rewriter, loc, resultTy, ValueRange({input}), ValueRange{emptyTensor},
+        affineMaps, getNParallelLoopsAttrs(resultTy.getRank()));
     rewriter.replaceOp(op, genericOp.getResult(0));
 
     {
@@ -2452,11 +2473,11 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
       rewriter.setInsertionPointToStart(block);
       if (inputElementTy.isInteger(8) && tableElementTy.isInteger(8) &&
           resultElementTy.isInteger(8)) {
-        Value index = arith::IndexCastOp::create(rewriter,
-            loc, rewriter.getIndexType(), inputValue);
+        Value index = arith::IndexCastOp::create(
+            rewriter, loc, rewriter.getIndexType(), inputValue);
         Value offset = arith::ConstantIndexOp::create(rewriter, loc, 128);
         index = arith::AddIOp::create(rewriter, loc, rewriter.getIndexType(),
-                                               index, offset);
+                                      index, offset);
         Value extract =
             tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
         linalg::YieldOp::create(rewriter, loc, extract);
@@ -2465,17 +2486,17 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
 
       if (inputElementTy.isInteger(16) && tableElementTy.isInteger(16) &&
           resultElementTy.isInteger(32)) {
-        Value extend = arith::ExtSIOp::create(rewriter,
-            loc, rewriter.getI32Type(), inputValue);
-
-        auto offset = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getI32IntegerAttr(32768));
-        auto seven = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getI32IntegerAttr(7));
-        auto one = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getI32IntegerAttr(1));
-        auto b1111111 = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getI32IntegerAttr(127));
+        Value extend = arith::ExtSIOp::create(
+            rewriter, loc, rewriter.getI32Type(), inputValue);
+
+        auto offset = arith::ConstantOp::create(
+            rewriter, loc, rewriter.getI32IntegerAttr(32768));
+        auto seven = arith::ConstantOp::create(rewriter, loc,
+                                               rewriter.getI32IntegerAttr(7));
+        auto one = arith::ConstantOp::create(rewriter, loc,
+                                             rewriter.getI32IntegerAttr(1));
+        auto b1111111 = arith::ConstantOp::create(
+            rewriter, loc, rewriter.getI32IntegerAttr(127));
 
         // Compute the index and fractional part from the input value:
         // value = value + 32768
@@ -2491,15 +2512,15 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
         // next = (int32_t) table[index + 1];
         Value indexPlusOne = arith::AddIOp::create(rewriter, loc, index, one);
 
-        index = arith::IndexCastOp::create(rewriter,
-            loc, rewriter.getIndexType(), index);
-        indexPlusOne = arith::IndexCastOp::create(rewriter,
-            loc, rewriter.getIndexType(), indexPlusOne);
+        index = arith::IndexCastOp::create(rewriter, loc,
+                                           rewriter.getIndexType(), index);
+        indexPlusOne = arith::IndexCastOp::create(
+            rewriter, loc, rewriter.getIndexType(), indexPlusOne);
 
         Value base =
             tensor::ExtractOp::create(rewriter, loc, table, ValueRange{index});
-        Value next = tensor::ExtractOp::create(rewriter,
-            loc, table, ValueRange{indexPlusOne});
+        Value next = tensor::ExtractOp::create(rewriter, loc, table,
+                                               ValueRange{indexPlusOne});
 
         base =
             arith::ExtSIOp::create(rewriter, loc, rewriter.getI32Type(), base);
@@ -2574,8 +2595,8 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
   static Value castIndexToFloat(OpBuilder &builder, Location loc,
                                 FloatType type, Value value) {
-    auto integerVal = arith::IndexCastUIOp::create(builder,
-        loc,
+    auto integerVal = arith::IndexCastUIOp::create(
+        builder, loc,
         type.getIntOrFloatBitWidth() > 32 ? builder.getI64Type()
                                           : builder.getI32Type(),
         value);
@@ -2683,8 +2704,10 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
-      auto outImag = arith::SubFOp::create(builder, loc, sumImag, imagComponent);
+      auto outReal =
+          arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag =
+          arith::SubFOp::create(builder, loc, sumImag, imagComponent);
 
       linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
@@ -2798,10 +2821,10 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
       auto angle = arith::MulFOp::create(builder, loc, twoPi, sumXY);
 
       if (inverse.getValue()) {
-        angle = arith::MulFOp::create(builder,
-            loc, angle,
-            arith::ConstantOp::create(rewriter,
-                loc, rewriter.getFloatAttr(real_el_ty, -1.0)));
+        angle = arith::MulFOp::create(
+            builder, loc, angle,
+            arith::ConstantOp::create(rewriter, loc,
+                                      rewriter.getFloatAttr(real_el_ty, -1.0)));
       }
 
       // realComponent = val_real * cos(a) + val_imag * sin(a);
@@ -2820,8 +2843,10 @@ struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
 
       // outReal = sumReal + realComponent
       // outImag = sumImag - imagComponent
-      auto outReal = arith::AddFOp::create(builder, loc, sumReal, realComponent);
-      auto outImag = arith::AddFOp::create(builder, loc, sumImag, imagComponent);
+      auto outReal =
+          arith::AddFOp::create(builder, loc, sumReal, realComponent);
+      auto outImag =
+          arith::AddFOp::create(builder, loc, sumImag, imagComponent);
 
       linalg::YieldOp::create(builder, loc, ValueRange{outReal, outImag});
     };
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index bfe43e8dcc274..3a205246ddd9e 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -54,9 +54,9 @@ static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
 
   Value padValue = arith::ConstantOp::create(rewriter, loc, padAttr);
 
-  return tensor::PadOp::create(rewriter,
-      loc, RankedTensorType::get(paddedShape, inputETy), input, lowIndices,
-      highIndices, padValue);
+  return tensor::PadOp::create(rewriter, loc,
+                               RankedTensorType::get(paddedShape, inputETy),
+                               input, lowIndices, highIndices, padValue);
 }
 
 static mlir::Value
@@ -160,8 +160,8 @@ static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim,
                                           int64_t dilationAttr,
                                           OpBuilder &rewriter) {
   ImplicitLocOpBuilder builder(loc, rewriter);
-  auto one = arith::ConstantOp::create(rewriter,
-      loc, IntegerAttr::get(inputDim.getType(), 1));
+  auto one = arith::ConstantOp::create(rewriter, loc,
+                                       IntegerAttr::get(inputDim.getType(), 1));
   Value padBefore = reifyConstantDim(padBeforeAttr, builder);
   Value paddedBefore = arith::AddIOp::create(builder, inputDim, padBefore);
   Value padAfter = reifyConstantDim(padAfterAttr, builder);
@@ -351,7 +351,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
         Type newWeightTy =
             RankedTensorType::get(newWeightShape, weightTy.getElementType());
         weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
-                                                    weightPermAttr);
+                                           weightPermAttr);
       }
     }
 
@@ -373,7 +373,7 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       Type newWeightTy =
           RankedTensorType::get(newWeightShape, weightTy.getElementType());
       weight = tosa::TransposeOp::create(rewriter, loc, newWeightTy, weight,
-                                                  weightPermAttr);
+                                         weightPermAttr);
     }
 
     // Extract the attributes for convolution.
@@ -384,8 +384,8 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
     auto strideAttr = rewriter.getI64TensorAttr(stride);
     auto dilationAttr = rewriter.getI64TensorAttr(dilation);
 
-    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultTy.getShape(), accETy, filteredDims);
+    Value biasEmptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, resultTy.getShape(), accETy, filteredDims);
 
     Value broadcastBias =
         linalgBroadcastAndMaybeExt(rewriter, loc, bias, biasEmptyTensor);
@@ -526,16 +526,16 @@ class DepthwiseConvConverter
                               accETy);
 
     auto resultZeroAttr = rewriter.getZeroAttr(accETy);
-    Value emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, linalgConvTy.getShape(), accETy, filteredDims);
+    Value emptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, linalgConvTy.getShape(), accETy, filteredDims);
     Value zero = arith::ConstantOp::create(rewriter, loc, resultZeroAttr);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
                            .result();
 
-    Value biasEmptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultTy.getShape(), resultETy, filteredDims);
+    Value biasEmptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, resultTy.getShape(), resultETy, filteredDims);
 
     // Broadcast the initial value to the output tensor before convolving.
     SmallVector<AffineMap, 4> indexingMaps;
@@ -553,16 +553,16 @@ class DepthwiseConvConverter
       // We may need to truncate back to the result type if the accumulator was
       // wider than the result.
       if (accETy != resultETy)
-        conv = tosa::CastOp::create(rewriter,
-            loc,
+        conv = tosa::CastOp::create(
+            rewriter, loc,
             RankedTensorType::get(cast<ShapedType>(conv.getType()).getShape(),
                                   resultETy),
             conv);
 
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
-          loc, resultTy, conv, reassociationMap);
+      Value convReshape = tensor::CollapseShapeOp::create(
+          rewriter, loc, resultTy, conv, reassociationMap);
 
       Value result =
           rewriter
@@ -575,10 +575,10 @@ class DepthwiseConvConverter
                     Value added;
                     if (llvm::isa<FloatType>(inputETy))
                       added = arith::AddFOp::create(nestedBuilder, loc, args[0],
-                                                                  args[1]);
+                                                    args[1]);
                     else
                       added = arith::AddIOp::create(nestedBuilder, loc, args[0],
-                                                                  args[1]);
+                                                    args[1]);
                     linalg::YieldOp::create(nestedBuilder, nestedLoc, added);
                   })
               .getResult(0);
@@ -596,8 +596,8 @@ class DepthwiseConvConverter
               .getResult(0);
       SmallVector<ReassociationExprs, 4> reassociationMap;
       createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
-      Value convReshape = tensor::CollapseShapeOp::create(rewriter,
-          loc, resultTy, conv, reassociationMap);
+      Value convReshape = tensor::CollapseShapeOp::create(
+          rewriter, loc, resultTy, conv, reassociationMap);
       Value result = linalgIntBroadcastExtSIAdd(
           rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
       rewriter.replaceOp(op, result);
@@ -636,8 +636,9 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
 
     auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
     Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
-    auto emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
+    auto emptyTensor =
+        tensor::EmptyOp::create(rewriter, loc, outputTy.getShape(),
+                                outputTy.getElementType(), filteredDims);
     Value zeroTensor = rewriter
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
@@ -670,10 +671,10 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
       return success();
     }
 
-    auto aZp = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getI32IntegerAttr(aZpVal));
-    auto bZp = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getI32IntegerAttr(bZpVal));
+    auto aZp = arith::ConstantOp::create(rewriter, loc,
+                                         rewriter.getI32IntegerAttr(aZpVal));
+    auto bZp = arith::ConstantOp::create(rewriter, loc,
+                                         rewriter.getI32IntegerAttr(bZpVal));
     rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
         op, TypeRange{op.getType()},
         ValueRange{adaptor.getA(), adaptor.getB(), aZp, bZp}, zeroTensor);
@@ -785,8 +786,9 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
+    Value emptyTensor =
+        tensor::EmptyOp::create(rewriter, loc, resultTy.getShape(),
+                                resultTy.getElementType(), dynamicDims);
 
     Value filledEmptyTensor =
         linalg::FillOp::create(rewriter, loc, initialValue, emptyTensor)
@@ -802,8 +804,8 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
       return llvm::success();
     }
 
-    auto resultOp = linalg::PoolingNhwcMaxOp::create(rewriter,
-        op->getLoc(), ArrayRef<Type>{resultTy},
+    auto resultOp = linalg::PoolingNhwcMaxOp::create(
+        rewriter, op->getLoc(), ArrayRef<Type>{resultTy},
         ValueRange{paddedInput, fakeWindowDims}, filledEmptyTensor, strideAttr,
         dilationAttr);
 
@@ -823,9 +825,10 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
     // it to include the appropriate checks. If the current value is NaN the
     // old value of pool will be taken otherwise we use the result.
     if (nanMode == "IGNORE") {
-      auto genericOp = linalg::GenericOp::create(rewriter,
-          loc, resultOp.getType(0), resultOp.getInputs(), resultOp.getOutputs(),
-          resultOp.getIndexingMapsArray(), resultOp.getIteratorTypesArray(),
+      auto genericOp = linalg::GenericOp::create(
+          rewriter, loc, resultOp.getType(0), resultOp.getInputs(),
+          resultOp.getOutputs(), resultOp.getIndexingMapsArray(),
+          resultOp.getIteratorTypesArray(),
           [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
             IRMapping map;
             auto oldBlock = resultOp.getRegion().begin();
@@ -833,11 +836,11 @@ class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
             auto &oldMaxOp = *resultOp.getBlock()->begin();
             map.map(oldArgs, blockArgs);
             auto *newOp = opBuilder.clone(oldMaxOp, map);
-            Value isNaN = arith::CmpFOp::create(opBuilder,
-                loc, arith::CmpFPredicate::UNO, blockArgs.front(),
-                blockArgs.front());
-            auto selectOp = arith::SelectOp::create(opBuilder,
-                loc, isNaN, blockArgs.back(), newOp->getResult(0));
+            Value isNaN =
+                arith::CmpFOp::create(opBuilder, loc, arith::CmpFPredicate::UNO,
+                                      blockArgs.front(), blockArgs.front());
+            auto selectOp = arith::SelectOp::create(
+                opBuilder, loc, isNaN, blockArgs.back(), newOp->getResult(0));
             linalg::YieldOp::create(opBuilder, loc, selectOp.getResult());
           });
       rewriter.replaceOp(resultOp, genericOp);
@@ -903,8 +906,8 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
 
     // Create the linalg op that performs pooling.
-    Value poolEmptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, accTy.getShape(), accETy, dynamicDims);
+    Value poolEmptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, accTy.getShape(), accETy, dynamicDims);
 
     Value filledEmptyTensor =
         rewriter
@@ -932,12 +935,12 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
     iH = arith::SubIOp::create(rewriter, loc, iH, one);
     iW = arith::SubIOp::create(rewriter, loc, iW, one);
 
-    Value genericEmptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultTy.getShape(), resultETy, dynamicDims);
+    Value genericEmptyTensor = tensor::EmptyOp::create(
+        rewriter, loc, resultTy.getShape(), resultETy, dynamicDims);
 
     auto affineMap = rewriter.getMultiDimIdentityMap(resultTy.getRank());
-    auto genericOp = linalg::GenericOp::create(rewriter,
-        loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
+    auto genericOp = linalg::GenericOp::create(
+        rewriter, loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
         ValueRange{genericEmptyTensor},
         ArrayRef<AffineMap>({affineMap, affineMap}),
         getNParallelLoopsAttrs(resultTy.getRank()),
@@ -981,8 +984,8 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
           Value kW3 = coverageFn(2, iW);
 
           // Compute the total number of elements and normalize.
-          auto count = arith::IndexCastOp::create(rewriter,
-              loc, rewriter.getI32Type(),
+          auto count = arith::IndexCastOp::create(
+              rewriter, loc, rewriter.getI32Type(),
               arith::MulIOp::create(rewriter, loc, kH3, kW3));
 
           // Divide by the number of summed values. For floats this is just
@@ -1002,8 +1005,8 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // If we have quantization information we need to apply an offset
             // for the input zp value.
             if (inputZpVal != 0) {
-              auto inputZp = arith::ConstantOp::create(rewriter,
-                  loc, b.getIntegerAttr(accETy, inputZpVal));
+              auto inputZp = arith::ConstantOp::create(
+                  rewriter, loc, b.getIntegerAttr(accETy, inputZpVal));
               Value offset =
                   arith::MulIOp::create(rewriter, loc, accETy, count, inputZp);
               poolVal =
@@ -1011,10 +1014,10 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             }
 
             // Compute: k = 32 - count_leading_zeros(value - 1)
-            Value one32 = arith::ConstantOp::create(rewriter,
-                loc, rewriter.getI32IntegerAttr(1));
-            Value thirtyTwo32 = arith::ConstantOp::create(rewriter,
-                loc, rewriter.getI32IntegerAttr(32));
+            Value one32 = arith::ConstantOp::create(
+                rewriter, loc, rewriter.getI32IntegerAttr(1));
+            Value thirtyTwo32 = arith::ConstantOp::create(
+                rewriter, loc, rewriter.getI32IntegerAttr(32));
 
             Value countSubOne =
                 arith::SubIOp::create(rewriter, loc, count, one32);
@@ -1026,24 +1029,24 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // Compute: numerator = ((1 << 30) + 1) << k
             Value k64 =
                 arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), k);
-            Value thirtyShiftPlusOne = arith::ConstantOp::create(rewriter,
-                loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
+            Value thirtyShiftPlusOne = arith::ConstantOp::create(
+                rewriter, loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
             Value numerator =
                 arith::ShLIOp::create(rewriter, loc, thirtyShiftPlusOne, k64);
 
             // Compute: scale.multiplier = numerator / value;
-            Value count64 = arith::ExtUIOp::create(rewriter,
-                loc, rewriter.getI64Type(), count);
+            Value count64 = arith::ExtUIOp::create(
+                rewriter, loc, rewriter.getI64Type(), count);
             Value multiplier =
                 arith::DivUIOp::create(rewriter, loc, numerator, count64);
-            multiplier = arith::TruncIOp::create(rewriter,
-                loc, rewriter.getI32Type(), multiplier);
+            multiplier = arith::TruncIOp::create(
+                rewriter, loc, rewriter.getI32Type(), multiplier);
 
             // Compute: scale.shift = 30 + k
             Value k8 =
                 arith::TruncIOp::create(rewriter, loc, rewriter.getI8Type(), k);
-            Value thirty8 = arith::ConstantOp::create(rewriter,
-                loc, rewriter.getI8IntegerAttr(30));
+            Value thirty8 = arith::ConstantOp::create(
+                rewriter, loc, rewriter.getI8IntegerAttr(30));
             Value shift = arith::AddIOp::create(rewriter, loc, k8, thirty8);
 
             auto scaled =
@@ -1056,8 +1059,9 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // If we have quantization information we need to apply output
             // zeropoint.
             if (outputZpVal != 0) {
-              auto outputZp = arith::ConstantOp::create(rewriter,
-                  loc, b.getIntegerAttr(scaled.getType(), outputZpVal));
+              auto outputZp = arith::ConstantOp::create(
+                  rewriter, loc,
+                  b.getIntegerAttr(scaled.getType(), outputZpVal));
               scaled = arith::AddIOp::create(rewriter, loc, scaled, outputZp)
                            .getResult();
             }
@@ -1065,11 +1069,11 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
             // Apply Clip.
             int64_t outBitwidth = resultETy.getIntOrFloatBitWidth();
 
-            auto min = arith::ConstantIntOp::create(rewriter,
-                loc, accETy,
+            auto min = arith::ConstantIntOp::create(
+                rewriter, loc, accETy,
                 APInt::getSignedMinValue(outBitwidth).getSExtValue());
-            auto max = arith::ConstantIntOp::create(rewriter,
-                loc, accETy,
+            auto max = arith::ConstantIntOp::create(
+                rewriter, loc, accETy,
                 APInt::getSignedMaxValue(outBitwidth).getSExtValue());
             auto clamp = clampIntHelper(loc, scaled, min, max, rewriter,
                                         /*isUnsigned=*/false);
@@ -1107,8 +1111,9 @@ class TransposeConverter : public OpRewritePattern<tosa::TransposeOp> {
     auto permutedSizes =
         applyTOSAPermutation<OpFoldResult>(inputSizes, constantPerms);
 
-    auto permutedInit = tensor::EmptyOp::create(rewriter,
-        loc, permutedSizes, op.getInput1().getType().getElementType());
+    auto permutedInit =
+        tensor::EmptyOp::create(rewriter, loc, permutedSizes,
+                                op.getInput1().getType().getElementType());
     rewriter.replaceOpWithNewOp<linalg::TransposeOp>(
         op, op.getInput1(), permutedInit,
         llvm::to_vector(llvm::map_range(
diff --git a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
index cd36aadce410a..b83f5ec9b0283 100644
--- a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
+++ b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
@@ -27,8 +27,8 @@ class VariableOpConverter : public OpRewritePattern<tosa::VariableOp> {
   LogicalResult matchAndRewrite(tosa::VariableOp op,
                                 PatternRewriter &rewriter) const final {
     auto variableType = tosa::getVariableType(op);
-    auto newVariable = mlir::ml_program::GlobalOp::create(rewriter,
-        op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
+    auto newVariable = mlir::ml_program::GlobalOp::create(
+        rewriter, op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
         op.getInitialValueAttr(), /*sym_visibility=*/nullptr);
     newVariable.setPrivate();
     rewriter.replaceOp(op, newVariable);
@@ -45,8 +45,8 @@ class VariableWriteOpConverter
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableWrite = ml_program::GlobalStoreOp::create(rewriter,
-        op.getLoc(), globalSymbolRef, op.getInput1());
+    auto newVariableWrite = ml_program::GlobalStoreOp::create(
+        rewriter, op.getLoc(), globalSymbolRef, op.getInput1());
     rewriter.replaceOp(op, newVariableWrite);
     return success();
   }
@@ -60,8 +60,8 @@ class VariableReadOpConverter : public OpRewritePattern<tosa::VariableReadOp> {
                                 PatternRewriter &rewriter) const final {
     auto globalSymbolRef =
         SymbolRefAttr::get(rewriter.getContext(), op.getName());
-    auto newVariableRead = ml_program::GlobalLoadOp::create(rewriter,
-        op.getLoc(), op.getType(), globalSymbolRef);
+    auto newVariableRead = ml_program::GlobalLoadOp::create(
+        rewriter, op.getLoc(), op.getType(), globalSymbolRef);
     rewriter.replaceOp(op, newVariableRead);
 
     return success();
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index 9188b8d1368da..aa6b4164e9876 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -46,10 +46,10 @@ static void inlineWhileCase(Region &srcRegion, Region &dstRegion,
   auto yield = cast<YieldOp>(headBlock->getTerminator());
   rewriter.setInsertionPoint(yield);
   if (isCond) {
-    auto condition =
-        tensor::ExtractOp::create(rewriter, yield.getLoc(), yield.getOperand(0));
+    auto condition = tensor::ExtractOp::create(rewriter, yield.getLoc(),
+                                               yield.getOperand(0));
     scf::ConditionOp::create(rewriter, yield.getLoc(), condition,
-                                      headBlock->getArguments());
+                             headBlock->getArguments());
   } else {
     rewriter.setInsertionPoint(yield);
     scf::YieldOp::create(rewriter, yield.getLoc(), yield.getInputs());
@@ -68,7 +68,7 @@ class IfOpConverter : public OpRewritePattern<tosa::IfOp> {
     auto condition =
         tensor::ExtractOp::create(rewriter, op.getLoc(), op.getCondition());
     auto newIf = scf::IfOp::create(rewriter, op.getLoc(), op.getResultTypes(),
-                                            condition, true);
+                                   condition, true);
 
     inlineIfCase(op.getThenGraph(), newIf.getThenRegion(), op.getInputList(),
                  rewriter);
@@ -120,8 +120,8 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
 
       // Read the index and cast it to index type
       auto index = tensor::ExtractOp::create(builder, loc, indices, ivs);
-      auto castIndex = arith::IndexCastOp::create(builder,
-          loc, builder.getIndexType(), index);
+      auto castIndex = arith::IndexCastOp::create(
+          builder, loc, builder.getIndexType(), index);
 
       // Offset, sizes, and strides for the input tensor
       auto inputOffset = llvm::to_vector(ivs);
@@ -130,13 +130,13 @@ class ScatterOpConverter : public OpRewritePattern<tosa::ScatterOp> {
       llvm::SmallVector<Value> sizes = {one, one, dimC};
       llvm::SmallVector<Value> strides = {one, one, one};
 
-      auto slice = tensor::ExtractSliceOp::create(builder,
-          loc, input, inputOffset, sizes, strides);
+      auto slice = tensor::ExtractSliceOp::create(builder, loc, input,
+                                                  inputOffset, sizes, strides);
 
       // Insert the slice into the output accumulator tensor.
       llvm::SmallVector<Value> outputOffset = {n, castIndex, zero};
-      auto updated = tensor::InsertSliceOp::create(builder,
-          loc, slice, args[0], outputOffset, sizes, strides);
+      auto updated = tensor::InsertSliceOp::create(
+          builder, loc, slice, args[0], outputOffset, sizes, strides);
 
       return {updated};
     };
@@ -155,8 +155,8 @@ class WhileOpConverter : public OpRewritePattern<tosa::WhileOp> {
 
   LogicalResult matchAndRewrite(tosa::WhileOp op,
                                 PatternRewriter &rewriter) const final {
-    auto newWhile = scf::WhileOp::create(rewriter,
-        op.getLoc(), op.getResultTypes(), op.getInputList());
+    auto newWhile = scf::WhileOp::create(
+        rewriter, op.getLoc(), op.getResultTypes(), op.getInputList());
     rewriter.createBlock(&newWhile.getBefore());
     rewriter.createBlock(&newWhile.getAfter());
 
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index a7219d423a77a..2945ae3b49f1f 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -309,14 +309,14 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
         continue;
 
       auto dim = tensor::DimOp::create(rewriter, loc, input, index);
-      auto offset = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexAttr(sliceStarts[index]));
+      auto offset = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIndexAttr(sliceStarts[index]));
       dynSizes.push_back(arith::SubIOp::create(rewriter, loc, dim, offset));
     }
 
-    auto newSliceOp = tensor::ExtractSliceOp::create(rewriter,
-        sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}), dynSizes,
-        ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
+    auto newSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, sliceOp.getLoc(), sliceOp.getType(), input, ValueRange({}),
+        dynSizes, ValueRange({}), rewriter.getDenseI64ArrayAttr(sliceStarts),
         rewriter.getDenseI64ArrayAttr(sizes),
         rewriter.getDenseI64ArrayAttr(strides));
 
@@ -375,16 +375,16 @@ class PadConverter : public OpConversionPattern<tosa::PadOp> {
     highValues.reserve(rank);
 
     for (int i = 0; i < rank; i++) {
-      Value lowVal = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexAttr(paddingVals[2 * i]));
-      Value highVal = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexAttr(paddingVals[2 * i + 1]));
+      Value lowVal = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIndexAttr(paddingVals[2 * i]));
+      Value highVal = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIndexAttr(paddingVals[2 * i + 1]));
       lowValues.push_back(lowVal);
       highValues.push_back(highVal);
     }
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, padOp.getType(), input, lowValues, highValues, padConstant);
+    auto newPadOp = tensor::PadOp::create(rewriter, loc, padOp.getType(), input,
+                                          lowValues, highValues, padConstant);
 
     rewriter.replaceOp(padOp, newPadOp.getResult());
     return success();
@@ -439,8 +439,9 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
       }
     }
 
-    Value result = tensor::EmptyOp::create(rewriter,
-        loc, resultType.getShape(), resultType.getElementType(), dynDims);
+    Value result =
+        tensor::EmptyOp::create(rewriter, loc, resultType.getShape(),
+                                resultType.getElementType(), dynDims);
 
     for (auto [arg, offset] : llvm::zip(adaptor.getOperands(), axisOffsets)) {
       auto sizes = tensor::getMixedSizes(rewriter, op.getLoc(), arg);
diff --git a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
index 9e4eb347b3689..125ea1eb60ed6 100644
--- a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
+++ b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
@@ -226,8 +226,8 @@ struct BroadcastOpToArmSMELowering
         (srcVectorType && (srcVectorType.getRank() == 0))) {
       // Broadcast scalar or 0-d vector to 1-d vector.
       VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-      broadcastOp1D = vector::BroadcastOp::create(rewriter,
-          loc, tileSliceType, broadcastOp.getSource());
+      broadcastOp1D = vector::BroadcastOp::create(rewriter, loc, tileSliceType,
+                                                  broadcastOp.getSource());
     } else if (srcVectorType && (srcVectorType.getRank() == 1))
       // Value to broadcast is already a 1-d vector, nothing to do.
       broadcastOp1D = broadcastOp.getSource();
@@ -240,8 +240,8 @@ struct BroadcastOpToArmSMELowering
                             Value currentTile) {
       // Create 'arm_sme.insert_tile_slice' to broadcast the value
       // to each tile slice.
-      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
-          loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
+      auto nextTile = arm_sme::InsertTileSliceOp::create(
+          b, loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
 
@@ -292,15 +292,15 @@ struct SplatOpToArmSMELowering : public OpRewritePattern<vector::SplatOp> {
 
     // First, broadcast the scalar to a 1-d vector.
     VectorType tileSliceType = VectorType::Builder(tileType).dropDim(0);
-    Value broadcastOp1D = vector::BroadcastOp::create(rewriter,
-        loc, tileSliceType, splatOp.getInput());
+    Value broadcastOp1D = vector::BroadcastOp::create(
+        rewriter, loc, tileSliceType, splatOp.getInput());
 
     auto initTile = arm_sme::GetTileOp::create(rewriter, loc, tileType);
 
     auto makeLoopBody = [&](OpBuilder &b, Location loc, Value tileSliceIndex,
                             Value currentTile) {
-      auto nextTile = arm_sme::InsertTileSliceOp::create(b,
-          loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
+      auto nextTile = arm_sme::InsertTileSliceOp::create(
+          b, loc, tileType, broadcastOp1D, currentTile, tileSliceIndex);
       return nextTile.getResult();
     };
 
@@ -371,8 +371,8 @@ struct TransposeOpToArmSMELowering
     // Allocate buffer to store input tile to.
     Value vscale =
         vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
-    Value minTileSlices = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIndexAttr(tileType.getDimSize(0)));
+    Value minTileSlices = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getIndexAttr(tileType.getDimSize(0)));
     Value c0 =
         arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(0));
     Value numTileSlices =
@@ -380,12 +380,12 @@ struct TransposeOpToArmSMELowering
     auto bufferType =
         MemRefType::get({ShapedType::kDynamic, ShapedType::kDynamic},
                         tileType.getElementType());
-    auto buffer = memref::AllocaOp::create(rewriter,
-        loc, bufferType, ValueRange{numTileSlices, numTileSlices});
+    auto buffer = memref::AllocaOp::create(
+        rewriter, loc, bufferType, ValueRange{numTileSlices, numTileSlices});
 
     // Store input tile.
-    auto tileStoreOp = arm_sme::TileStoreOp::create(rewriter,
-        loc, input, buffer, ValueRange{c0, c0});
+    auto tileStoreOp = arm_sme::TileStoreOp::create(rewriter, loc, input,
+                                                    buffer, ValueRange{c0, c0});
 
     // Reload input tile vertically.
     rewriter.replaceOpWithNewOp<arm_sme::TileLoadOp>(
@@ -488,10 +488,10 @@ struct VectorOuterProductToArmSMELowering
     Value rhsMaskDim = createMaskOp.getOperand(1);
 
     VectorType operandMaskType = VectorType::Builder(maskType).dropDim(0);
-    Value lhsMask =
-        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, lhsMaskDim);
-    Value rhsMask =
-        vector::CreateMaskOp::create(rewriter, loc, operandMaskType, rhsMaskDim);
+    Value lhsMask = vector::CreateMaskOp::create(rewriter, loc, operandMaskType,
+                                                 lhsMaskDim);
+    Value rhsMask = vector::CreateMaskOp::create(rewriter, loc, operandMaskType,
+                                                 rhsMaskDim);
 
     return std::make_pair(lhsMask, rhsMask);
   }
@@ -531,8 +531,8 @@ struct VectorExtractToArmSMELowering
     }
 
     Value sliceIndex = vector::getAsValues(rewriter, loc, position[0]).front();
-    auto extractTileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
-        loc, sourceVector, sliceIndex);
+    auto extractTileSlice = arm_sme::ExtractTileSliceOp::create(
+        rewriter, loc, sourceVector, sliceIndex);
 
     if (position.size() == 1) {
       // Single index case: Extracts a 1D slice.
@@ -593,10 +593,10 @@ struct VectorInsertToArmSMELowering
     if (position.size() == 2) {
       // Two indices case: Insert single element into tile.
       // We need to first extract the existing slice and update the element.
-      tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
-          loc, insertOp.getDest(), sliceIndex);
+      tileSlice = arm_sme::ExtractTileSliceOp::create(
+          rewriter, loc, insertOp.getDest(), sliceIndex);
       tileSlice = vector::InsertOp::create(rewriter, loc, source, tileSlice,
-                                                    position[1]);
+                                           position[1]);
     }
 
     // Insert the slice into the destination tile.
@@ -648,17 +648,18 @@ struct VectorPrintToArmSMELowering : public OpRewritePattern<vector::PrintOp> {
     auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto upperBound = arith::MulIOp::create(rewriter, loc, minTileRows, vscale);
     auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
+    auto forOp =
+        scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
     {
       // Loop body.
       rewriter.setInsertionPointToStart(forOp.getBody());
       // Extract the current row from the tile.
       Value rowIndex = forOp.getInductionVar();
-      auto tileSlice = arm_sme::ExtractTileSliceOp::create(rewriter,
-          loc, printOp.getSource(), rowIndex);
+      auto tileSlice = arm_sme::ExtractTileSliceOp::create(
+          rewriter, loc, printOp.getSource(), rowIndex);
       // Print the row with a 1D vector.print.
       vector::PrintOp::create(rewriter, loc, tileSlice,
-                                       printOp.getPunctuation());
+                              printOp.getPunctuation());
     }
 
     rewriter.eraseOp(printOp);
@@ -707,8 +708,8 @@ struct FoldTransferWriteOfExtractTileSlice
     Value mask = writeOp.getMask();
     if (!mask) {
       auto maskType = writeOp.getVectorType().clone(rewriter.getI1Type());
-      mask = arith::ConstantOp::create(rewriter,
-          writeOp.getLoc(), maskType, DenseElementsAttr::get(maskType, true));
+      mask = arith::ConstantOp::create(rewriter, writeOp.getLoc(), maskType,
+                                       DenseElementsAttr::get(maskType, true));
     }
 
     rewriter.replaceOpWithNewOp<arm_sme::StoreTileSliceOp>(
@@ -776,10 +777,10 @@ struct ExtractFromCreateMaskToPselLowering
     // Create the two 1-D masks at the location of the 2-D create_mask (which is
     // usually outside a loop). This prevents the need for later hoisting.
     rewriter.setInsertionPoint(createMaskOp);
-    auto rowMask = vector::CreateMaskOp::create(rewriter,
-        loc, rowMaskType, createMaskOp.getOperand(0));
-    auto colMask = vector::CreateMaskOp::create(rewriter,
-        loc, colMaskType, createMaskOp.getOperand(1));
+    auto rowMask = vector::CreateMaskOp::create(rewriter, loc, rowMaskType,
+                                                createMaskOp.getOperand(0));
+    auto colMask = vector::CreateMaskOp::create(rewriter, loc, colMaskType,
+                                                createMaskOp.getOperand(1));
 
     rewriter.setInsertionPoint(extractOp);
     auto position =
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 5b3e67509672a..d09d76a63b5ec 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -579,8 +579,8 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op,
   }
   gpu::MMAMatrixType type =
       gpu::MMAMatrixType::get(op.getVectorType().getShape(), elType, fragType);
-  Value load = gpu::SubgroupMmaLoadMatrixOp::create(rewriter,
-      op.getLoc(), type, op.getBase(), op.getIndices(),
+  Value load = gpu::SubgroupMmaLoadMatrixOp::create(
+      rewriter, op.getLoc(), type, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride),
       isTranspose ? rewriter.getUnitAttr() : UnitAttr());
   valueMapping[mappingResult] = load;
@@ -610,8 +610,8 @@ convertTransferWriteOp(RewriterBase &rewriter, vector::TransferWriteOp op,
   }
 
   Value matrix = it->second;
-  auto store = gpu::SubgroupMmaStoreMatrixOp::create(rewriter,
-      op.getLoc(), matrix, op.getBase(), op.getIndices(),
+  auto store = gpu::SubgroupMmaStoreMatrixOp::create(
+      rewriter, op.getLoc(), matrix, op.getBase(), op.getIndices(),
       rewriter.getIndexAttr(*stride), /*transpose=*/UnitAttr());
   (void)store;
 
@@ -661,8 +661,8 @@ convertConstantOpMmaSync(RewriterBase &rewriter, arith::ConstantOp op,
     return rewriter.notifyMatchFailure(op, "not a splat");
   }
 
-  Value result = arith::ConstantOp::create(rewriter,
-      op.getLoc(), vectorType,
+  Value result = arith::ConstantOp::create(
+      rewriter, op.getLoc(), vectorType,
       DenseElementsAttr::get(vectorType, dense.getSplatValue<Attribute>()));
   valueMapping[op.getResult()] = result;
   return success();
@@ -757,8 +757,9 @@ creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   getXferIndices<vector::TransferReadOp>(rewriter, op, *offsets, {laneId},
                                          indices);
 
-  nvgpu::LdMatrixOp newOp = nvgpu::LdMatrixOp::create(rewriter,
-      loc, vectorType, op.getBase(), indices, *transpose, params->numTiles);
+  nvgpu::LdMatrixOp newOp =
+      nvgpu::LdMatrixOp::create(rewriter, loc, vectorType, op.getBase(),
+                                indices, *transpose, params->numTiles);
   valueMapping[op] = newOp->getResult(0);
   return success();
 }
@@ -788,8 +789,8 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
   Type loadedElType = regInfo->registerLLVMType;
   VectorType vectorType = getMmaSyncVectorOperandType(*regInfo);
 
-  Value fill = arith::ConstantOp::create(rewriter,
-      op.getLoc(), vectorType.getElementType(),
+  Value fill = arith::ConstantOp::create(
+      rewriter, op.getLoc(), vectorType.getElementType(),
       rewriter.getZeroAttr(vectorType.getElementType()));
   Value result =
       vector::SplatOp::create(rewriter, op.getLoc(), fill, vectorType);
@@ -809,15 +810,15 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       if (failed(coords))
         return rewriter.notifyMatchFailure(op, "no coords");
 
-      Value logicalValueId = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getIndexType(),
+      Value logicalValueId = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getIndexType(),
           rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
       SmallVector<Value, 4> newIndices;
       getXferIndices<vector::TransferReadOp>(
           rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
 
       Value el = vector::LoadOp::create(rewriter, loc, loadedElType,
-                                                 op.getBase(), newIndices);
+                                        op.getBase(), newIndices);
       result = vector::InsertOp::create(rewriter, loc, el, result, i);
     }
   } else {
@@ -828,8 +829,8 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
       for (unsigned innerIdx = 0; innerIdx < vectorType.getShape()[1];
            innerIdx++) {
 
-        Value logicalValueId = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getIndexType(),
+        Value logicalValueId = arith::ConstantOp::create(
+            rewriter, loc, rewriter.getIndexType(),
             rewriter.getIndexAttr(i * regInfo->elementsPerRegister + innerIdx));
         FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
             rewriter, op.getLoc(), *warpMatrixInfo);
@@ -840,9 +841,9 @@ createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
         getXferIndices<vector::TransferReadOp>(
             rewriter, op, *coords, {laneId, logicalValueId}, newIndices);
         Value el = memref::LoadOp::create(rewriter, op.getLoc(), loadedElType,
-                                                   op.getBase(), newIndices);
-        result = vector::InsertOp::create(rewriter,
-            op.getLoc(), el, result, ArrayRef<int64_t>{i, innerIdx});
+                                          op.getBase(), newIndices);
+        result = vector::InsertOp::create(rewriter, op.getLoc(), el, result,
+                                          ArrayRef<int64_t>{i, innerIdx});
       }
     }
   }
@@ -919,8 +920,8 @@ convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
   Value laneId = gpu::LaneIdOp::create(rewriter, loc, /*upperBound=*/nullptr);
 
   for (unsigned i = 0; i < vectorType.getShape()[0]; i++) {
-    Value logicalValueId = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIndexType(),
+    Value logicalValueId = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getIndexType(),
         rewriter.getIndexAttr(i * regInfo->elementsPerRegister));
     FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
         rewriter, op.getLoc(), *warpMatrixInfo);
@@ -1015,8 +1016,8 @@ convertExtractStridedSlice(RewriterBase &rewriter,
   else if (offsets[1])
     sliceOffset[0] = (warpVectorShape[1] / offsets[1]);
 
-  Value newOp = vector::ExtractStridedSliceOp::create(rewriter,
-      loc, sourceVector, sliceOffset, sliceShape, strides);
+  Value newOp = vector::ExtractStridedSliceOp::create(
+      rewriter, loc, sourceVector, sliceOffset, sliceShape, strides);
 
   valueMapping[op] = newOp;
   return success();
@@ -1035,9 +1036,10 @@ convertContractOp(RewriterBase &rewriter, vector::ContractionOp op,
       itC == valueMapping.end())
     return rewriter.notifyMatchFailure(op, "no mapping");
   Value opA = itA->second, opB = itB->second, opC = itC->second;
-  Value matmul = gpu::SubgroupMmaComputeOp::create(rewriter,
-      op.getLoc(), opC.getType(), opA, opB, opC, /*a_transpose=*/UnitAttr(),
-      /*b_transpose=*/UnitAttr());
+  Value matmul = gpu::SubgroupMmaComputeOp::create(rewriter, op.getLoc(),
+                                                   opC.getType(), opA, opB, opC,
+                                                   /*a_transpose=*/UnitAttr(),
+                                                   /*b_transpose=*/UnitAttr());
   valueMapping[op.getResult()] = matmul;
   return success();
 }
@@ -1058,8 +1060,8 @@ convertContractOpToMmaSync(RewriterBase &rewriter, vector::ContractionOp op,
   int64_t m = cast<VectorType>(op.getLhs().getType()).getShape()[0];
   int64_t n = cast<VectorType>(op.getRhs().getType()).getShape()[0];
   int64_t k = cast<VectorType>(op.getLhs().getType()).getShape()[1];
-  Value matmul = nvgpu::MmaSyncOp::create(rewriter,
-      op.getLoc(), opA, opB, opC, rewriter.getI64ArrayAttr({m, n, k}));
+  Value matmul = nvgpu::MmaSyncOp::create(rewriter, op.getLoc(), opA, opB, opC,
+                                          rewriter.getI64ArrayAttr({m, n, k}));
   valueMapping[op.getResult()] = matmul;
   return success();
 }
@@ -1081,8 +1083,8 @@ convertConstantOp(RewriterBase &rewriter, arith::ConstantOp op,
   auto vecType = cast<VectorType>(op.getType());
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
-      op.getLoc(), type, scalarConstant);
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter, op.getLoc(),
+                                                         type, scalarConstant);
   valueMapping[op.getResult()] = matrix;
   return success();
 }
@@ -1100,8 +1102,8 @@ convertBroadcastOp(RewriterBase &rewriter, vector::BroadcastOp op,
   auto vecType = op.getResultVectorType();
   gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
       vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
-  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter,
-      op.getLoc(), type, op.getSource());
+  auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(rewriter, op.getLoc(),
+                                                         type, op.getSource());
   valueMapping[op.getResult()] = matrix;
   return success();
 }
@@ -1118,9 +1120,9 @@ static scf::ForOp replaceForOpWithNewSignature(RewriterBase &rewriter,
   rewriter.setInsertionPoint(loop);
   auto operands = llvm::to_vector<4>(loop.getInitArgs());
   llvm::append_range(operands, newInitArgs);
-  scf::ForOp newLoop = scf::ForOp::create(rewriter,
-      loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
-      operands);
+  scf::ForOp newLoop =
+      scf::ForOp::create(rewriter, loop.getLoc(), loop.getLowerBound(),
+                         loop.getUpperBound(), loop.getStep(), operands);
   rewriter.eraseBlock(newLoop.getBody());
 
   newLoop.getRegion().getBlocks().splice(
@@ -1220,8 +1222,8 @@ convertElementwiseOp(RewriterBase &rewriter, Operation *op,
                                          resultType.getOperand());
   }
 
-  Value newOp = gpu::SubgroupMmaElementwiseOp::create(rewriter,
-      op->getLoc(), resultType, matrixOperands, opType);
+  Value newOp = gpu::SubgroupMmaElementwiseOp::create(
+      rewriter, op->getLoc(), resultType, matrixOperands, opType);
   valueMapping[op->getResult(0)] = newOp;
   return success();
 }
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index d21833736a718..c43466b25af6e 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -43,11 +43,11 @@ static Value insertOne(ConversionPatternRewriter &rewriter,
   assert(rank > 0 && "0-D vector corner case should have been handled already");
   if (rank == 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = LLVM::ConstantOp::create(rewriter,
-        loc, typeConverter.convertType(idxType),
+    auto constant = LLVM::ConstantOp::create(
+        rewriter, loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
     return LLVM::InsertElementOp::create(rewriter, loc, llvmType, val1, val2,
-                                                  constant);
+                                         constant);
   }
   return LLVM::InsertValueOp::create(rewriter, loc, val1, val2, pos);
 }
@@ -58,11 +58,11 @@ static Value extractOne(ConversionPatternRewriter &rewriter,
                         Value val, Type llvmType, int64_t rank, int64_t pos) {
   if (rank <= 1) {
     auto idxType = rewriter.getIndexType();
-    auto constant = LLVM::ConstantOp::create(rewriter,
-        loc, typeConverter.convertType(idxType),
+    auto constant = LLVM::ConstantOp::create(
+        rewriter, loc, typeConverter.convertType(idxType),
         rewriter.getIntegerAttr(idxType, pos));
     return LLVM::ExtractElementOp::create(rewriter, loc, llvmType, val,
-                                                   constant);
+                                          constant);
   }
   return LLVM::ExtractValueOp::create(rewriter, loc, val, pos);
 }
@@ -141,9 +141,9 @@ static Value getIndexedPtrs(ConversionPatternRewriter &rewriter, Location loc,
   auto ptrsType =
       LLVM::getVectorType(pType, vectorType.getDimSize(0),
                           /*isScalable=*/vectorType.getScalableDims()[0]);
-  return LLVM::GEPOp::create(rewriter,
-      loc, ptrsType, typeConverter.convertType(memRefType.getElementType()),
-      base, index);
+  return LLVM::GEPOp::create(
+      rewriter, loc, ptrsType,
+      typeConverter.convertType(memRefType.getElementType()), base, index);
 }
 
 /// Convert `foldResult` into a Value. Integer attribute is converted to
@@ -476,31 +476,31 @@ static Value createReductionNeutralValue(ReductionNeutralZero neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   return LLVM::ConstantOp::create(rewriter, loc, llvmType,
-                                           rewriter.getZeroAttr(llvmType));
+                                  rewriter.getZeroAttr(llvmType));
 }
 
 /// Create the reduction neutral integer one value.
 static Value createReductionNeutralValue(ReductionNeutralIntOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType, rewriter.getIntegerAttr(llvmType, 1));
+  return LLVM::ConstantOp::create(rewriter, loc, llvmType,
+                                  rewriter.getIntegerAttr(llvmType, 1));
 }
 
 /// Create the reduction neutral fp one value.
 static Value createReductionNeutralValue(ReductionNeutralFPOne neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
+  return LLVM::ConstantOp::create(rewriter, loc, llvmType,
+                                  rewriter.getFloatAttr(llvmType, 1.0));
 }
 
 /// Create the reduction neutral all-ones value.
 static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(
           llvmType, llvm::APInt::getAllOnes(llvmType.getIntOrFloatBitWidth())));
 }
@@ -509,8 +509,8 @@ static Value createReductionNeutralValue(ReductionNeutralAllOnes neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
 }
@@ -519,8 +519,8 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMinValue(
                                             llvmType.getIntOrFloatBitWidth())));
 }
@@ -529,8 +529,8 @@ static Value createReductionNeutralValue(ReductionNeutralUIntMin neutral,
 static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getSignedMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
 }
@@ -539,8 +539,8 @@ static Value createReductionNeutralValue(ReductionNeutralSIntMax neutral,
 static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getIntegerAttr(llvmType, llvm::APInt::getMaxValue(
                                             llvmType.getIntOrFloatBitWidth())));
 }
@@ -550,8 +550,8 @@ static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
                                            /*Negative=*/false)));
@@ -562,8 +562,8 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
                                          ConversionPatternRewriter &rewriter,
                                          Location loc, Type llvmType) {
   auto floatType = cast<FloatType>(llvmType);
-  return LLVM::ConstantOp::create(rewriter,
-      loc, llvmType,
+  return LLVM::ConstantOp::create(
+      rewriter, loc, llvmType,
       rewriter.getFloatAttr(
           llvmType, llvm::APFloat::getQNaN(floatType.getFloatSemantics(),
                                            /*Negative=*/true)));
@@ -591,8 +591,8 @@ static Value createVectorLengthValue(ConversionPatternRewriter &rewriter,
   auto vShape = vType.getShape();
   assert(vShape.size() == 1 && "Unexpected multi-dim vector type");
 
-  Value baseVecLength = LLVM::ConstantOp::create(rewriter,
-      loc, rewriter.getI32Type(),
+  Value baseVecLength = LLVM::ConstantOp::create(
+      rewriter, loc, rewriter.getI32Type(),
       rewriter.getIntegerAttr(rewriter.getI32Type(), vShape[0]));
 
   if (!vType.getScalableDims()[0])
@@ -616,7 +616,8 @@ static Value createIntegerReductionArithmeticOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator) {
 
-  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
+  Value result =
+      LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
 
   if (accumulator)
     result = ScalarOp::create(rewriter, loc, accumulator, result);
@@ -631,7 +632,8 @@ template <class LLVMRedIntrinOp>
 static Value createIntegerReductionComparisonOpLowering(
     ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
     Value vectorOperand, Value accumulator, LLVM::ICmpPredicate predicate) {
-  Value result = LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
+  Value result =
+      LLVMRedIntrinOp::create(rewriter, loc, llvmType, vectorOperand);
   if (accumulator) {
     Value cmp =
         LLVM::ICmpOp::create(rewriter, loc, predicate, accumulator, result);
@@ -717,8 +719,8 @@ lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
                                 Value mask, LLVM::FastmathFlagsAttr fmf) {
   const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
       rewriter, loc, llvmType, vectorOperand.getType());
-  const Value selectedVectorByMask = LLVM::SelectOp::create(rewriter,
-      loc, mask, vectorOperand, vectorMaskNeutral);
+  const Value selectedVectorByMask = LLVM::SelectOp::create(
+      rewriter, loc, mask, vectorOperand, vectorMaskNeutral);
   return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
       rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
 }
@@ -731,8 +733,8 @@ lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
   return LLVMRedIntrinOp::create(rewriter, loc, llvmType,
-                                          /*startValue=*/accumulator,
-                                          vectorOperand, fmf);
+                                 /*startValue=*/accumulator, vectorOperand,
+                                 fmf);
 }
 
 /// Overloaded methods to lower a *predicated* reduction to an llvm intrinsic
@@ -746,8 +748,7 @@ lowerPredicatedReductionWithStartValue(ConversionPatternRewriter &rewriter,
   accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
                                                          llvmType, accumulator);
   return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
-                                            /*startValue=*/accumulator,
-                                            vectorOperand);
+                                   /*startValue=*/accumulator, vectorOperand);
 }
 
 template <class LLVMVPRedIntrinOp, class ReductionNeutral>
@@ -759,8 +760,8 @@ static Value lowerPredicatedReductionWithStartValue(
   Value vectorLength =
       createVectorLengthValue(rewriter, loc, vectorOperand.getType());
   return LLVMVPRedIntrinOp::create(rewriter, loc, llvmType,
-                                            /*startValue=*/accumulator,
-                                            vectorOperand, mask, vectorLength);
+                                   /*startValue=*/accumulator, vectorOperand,
+                                   mask, vectorLength);
 }
 
 template <class LLVMIntVPRedIntrinOp, class IntReductionNeutral,
@@ -1071,8 +1072,8 @@ class VectorShuffleOpConversion
     // For rank 0 and 1, where both operands have *exactly* the same vector
     // type, there is direct shuffle support in LLVM. Use it!
     if (rank <= 1 && v1Type == v2Type) {
-      Value llvmShuffleOp = LLVM::ShuffleVectorOp::create(rewriter,
-          loc, adaptor.getV1(), adaptor.getV2(),
+      Value llvmShuffleOp = LLVM::ShuffleVectorOp::create(
+          rewriter, loc, adaptor.getV1(), adaptor.getV2(),
           llvm::to_vector_of<int32_t>(mask));
       rewriter.replaceOp(shuffleOp, llvmShuffleOp);
       return success();
@@ -1122,9 +1123,9 @@ class VectorExtractElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = extractEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = LLVM::ConstantOp::create(rewriter,
-          loc, typeConverter->convertType(idxType),
-          rewriter.getIntegerAttr(idxType, 0));
+      auto zero = LLVM::ConstantOp::create(rewriter, loc,
+                                           typeConverter->convertType(idxType),
+                                           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
           extractEltOp, llvmType, adaptor.getVector(), zero);
       return success();
@@ -1193,13 +1194,14 @@ class VectorExtractOpConversion
       if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) {
         return failure();
       }
-      extracted = LLVM::ExtractValueOp::create(rewriter,
-          loc, extracted, getAsIntegers(position));
+      extracted = LLVM::ExtractValueOp::create(rewriter, loc, extracted,
+                                               getAsIntegers(position));
     }
 
     if (extractsScalar) {
-      extracted = LLVM::ExtractElementOp::create(rewriter,
-          loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back()));
+      extracted = LLVM::ExtractElementOp::create(
+          rewriter, loc, extracted,
+          getAsLLVMValue(rewriter, loc, positionVec.back()));
     }
 
     rewriter.replaceOp(extractOp, extracted);
@@ -1256,9 +1258,9 @@ class VectorInsertElementOpConversion
     if (vectorType.getRank() == 0) {
       Location loc = insertEltOp.getLoc();
       auto idxType = rewriter.getIndexType();
-      auto zero = LLVM::ConstantOp::create(rewriter,
-          loc, typeConverter->convertType(idxType),
-          rewriter.getIntegerAttr(idxType, 0));
+      auto zero = LLVM::ConstantOp::create(rewriter, loc,
+                                           typeConverter->convertType(idxType),
+                                           rewriter.getIntegerAttr(idxType, 0));
       rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
           insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(), zero);
       return success();
@@ -1342,8 +1344,8 @@ class VectorInsertOpConversion
           // llvm.extractvalue does not support dynamic dimensions.
           return failure();
         }
-        sourceAggregate = LLVM::ExtractValueOp::create(rewriter,
-            loc, adaptor.getDest(),
+        sourceAggregate = LLVM::ExtractValueOp::create(
+            rewriter, loc, adaptor.getDest(),
             getAsIntegers(positionOf1DVectorWithinAggregate));
       } else {
         // No-aggregate case. The destination for the InsertElementOp is just
@@ -1351,16 +1353,16 @@ class VectorInsertOpConversion
         sourceAggregate = adaptor.getDest();
       }
       // Insert the scalar into the 1D vector.
-      sourceAggregate = LLVM::InsertElementOp::create(rewriter,
-          loc, sourceAggregate.getType(), sourceAggregate,
+      sourceAggregate = LLVM::InsertElementOp::create(
+          rewriter, loc, sourceAggregate.getType(), sourceAggregate,
           adaptor.getValueToStore(),
           getAsLLVMValue(rewriter, loc, positionOfScalarWithin1DVector));
     }
 
     Value result = sourceAggregate;
     if (isNestedAggregate) {
-      result = LLVM::InsertValueOp::create(rewriter,
-          loc, adaptor.getDest(), sourceAggregate,
+      result = LLVM::InsertValueOp::create(
+          rewriter, loc, adaptor.getDest(), sourceAggregate,
           getAsIntegers(positionOf1DVectorWithinAggregate));
     }
 
@@ -1439,8 +1441,8 @@ class VectorFMAOpNDRewritePattern : public OpRewritePattern<FMAOp> {
 
     auto loc = op.getLoc();
     auto elemType = vType.getElementType();
-    Value zero = arith::ConstantOp::create(rewriter,
-        loc, elemType, rewriter.getZeroAttr(elemType));
+    Value zero = arith::ConstantOp::create(rewriter, loc, elemType,
+                                           rewriter.getZeroAttr(elemType));
     Value desc = vector::SplatOp::create(rewriter, loc, vType, zero);
     for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
       Value extrLHS = ExtractOp::create(rewriter, loc, op.getLhs(), i);
@@ -1550,7 +1552,8 @@ class VectorTypeCastOpConversion
       desc.setSize(rewriter, loc, index, size);
       auto strideAttr = rewriter.getIntegerAttr(rewriter.getIndexType(),
                                                 (*targetStrides)[index]);
-      auto stride = LLVM::ConstantOp::create(rewriter, loc, int64Ty, strideAttr);
+      auto stride =
+          LLVM::ConstantOp::create(rewriter, loc, int64Ty, strideAttr);
       desc.setStride(rewriter, loc, index, stride);
     }
 
@@ -1578,14 +1581,15 @@ class VectorCreateMaskOpConversion
     IntegerType idxType =
         force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
     auto loc = op->getLoc();
-    Value indices = LLVM::StepVectorOp::create(rewriter,
-        loc, LLVM::getVectorType(idxType, dstType.getShape()[0],
-                                 /*isScalable=*/true));
+    Value indices = LLVM::StepVectorOp::create(
+        rewriter, loc,
+        LLVM::getVectorType(idxType, dstType.getShape()[0],
+                            /*isScalable=*/true));
     auto bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType,
                                                  adaptor.getOperands()[0]);
     Value bounds = SplatOp::create(rewriter, loc, indices.getType(), bound);
     Value comp = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
-                                                indices, bounds);
+                                       indices, bounds);
     rewriter.replaceOp(op, comp);
     return success();
   }
@@ -1741,16 +1745,16 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 
     switch (conversion) {
     case PrintConversion::ZeroExt64:
-      value = arith::ExtUIOp::create(rewriter,
-          loc, IntegerType::get(rewriter.getContext(), 64), value);
+      value = arith::ExtUIOp::create(
+          rewriter, loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::SignExt64:
-      value = arith::ExtSIOp::create(rewriter,
-          loc, IntegerType::get(rewriter.getContext(), 64), value);
+      value = arith::ExtSIOp::create(
+          rewriter, loc, IntegerType::get(rewriter.getContext(), 64), value);
       break;
     case PrintConversion::Bitcast16:
-      value = LLVM::BitcastOp::create(rewriter,
-          loc, IntegerType::get(rewriter.getContext(), 16), value);
+      value = LLVM::BitcastOp::create(
+          rewriter, loc, IntegerType::get(rewriter.getContext(), 16), value);
       break;
     case PrintConversion::None:
       break;
@@ -1763,7 +1767,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
   static void emitCall(ConversionPatternRewriter &rewriter, Location loc,
                        Operation *ref, ValueRange params = ValueRange()) {
     LLVM::CallOp::create(rewriter, loc, TypeRange(), SymbolRefAttr::get(ref),
-                                  params);
+                         params);
   }
 };
 
@@ -1783,8 +1787,8 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     auto vectorType = typeConverter->convertType(splatOp.getType());
     Value poison =
         LLVM::PoisonOp::create(rewriter, splatOp.getLoc(), vectorType);
-    auto zero = LLVM::ConstantOp::create(rewriter,
-        splatOp.getLoc(),
+    auto zero = LLVM::ConstantOp::create(
+        rewriter, splatOp.getLoc(),
         typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
 
@@ -1796,8 +1800,9 @@ struct VectorSplatOpLowering : public ConvertOpToLLVMPattern<vector::SplatOp> {
     }
 
     // For 1-d vector, we additionally do a `vectorshuffle`.
-    auto v = LLVM::InsertElementOp::create(rewriter,
-        splatOp.getLoc(), vectorType, poison, adaptor.getInput(), zero);
+    auto v =
+        LLVM::InsertElementOp::create(rewriter, splatOp.getLoc(), vectorType,
+                                      poison, adaptor.getInput(), zero);
 
     int64_t width = cast<VectorType>(splatOp.getType()).getDimSize(0);
     SmallVector<int32_t> zeroValues(width, 0);
@@ -1837,11 +1842,11 @@ struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
     // Construct a 1-D vector with the splatted value that we insert in all the
     // places within the returned descriptor.
     Value vdesc = LLVM::PoisonOp::create(rewriter, loc, llvm1DVectorTy);
-    auto zero = LLVM::ConstantOp::create(rewriter,
-        loc, typeConverter->convertType(rewriter.getIntegerType(32)),
+    auto zero = LLVM::ConstantOp::create(
+        rewriter, loc, typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
-    Value v = LLVM::InsertElementOp::create(rewriter, loc, llvm1DVectorTy, vdesc,
-                                                     adaptor.getInput(), zero);
+    Value v = LLVM::InsertElementOp::create(rewriter, loc, llvm1DVectorTy,
+                                            vdesc, adaptor.getInput(), zero);
 
     // Shuffle the value across the desired number of elements.
     int64_t width = resultType.getDimSize(resultType.getRank() - 1);
@@ -1921,13 +1926,13 @@ struct VectorDeinterleaveOpLowering
       auto deinterleaveResults = deinterleaveOp.getResultTypes();
       auto packedOpResults =
           llvmTypeConverter->packOperationResults(deinterleaveResults);
-      auto intrinsic = LLVM::vector_deinterleave2::create(rewriter,
-          loc, packedOpResults, adaptor.getSource());
+      auto intrinsic = LLVM::vector_deinterleave2::create(
+          rewriter, loc, packedOpResults, adaptor.getSource());
 
-      auto evenResult = LLVM::ExtractValueOp::create(rewriter,
-          loc, intrinsic->getResult(0), 0);
-      auto oddResult = LLVM::ExtractValueOp::create(rewriter,
-          loc, intrinsic->getResult(0), 1);
+      auto evenResult = LLVM::ExtractValueOp::create(
+          rewriter, loc, intrinsic->getResult(0), 0);
+      auto oddResult = LLVM::ExtractValueOp::create(rewriter, loc,
+                                                    intrinsic->getResult(0), 1);
 
       rewriter.replaceOp(deinterleaveOp, ValueRange{evenResult, oddResult});
       return success();
@@ -1951,10 +1956,10 @@ struct VectorDeinterleaveOpLowering
     }
 
     auto poison = LLVM::PoisonOp::create(rewriter, loc, sourceType);
-    auto evenShuffle = LLVM::ShuffleVectorOp::create(rewriter,
-        loc, adaptor.getSource(), poison, evenShuffleMask);
-    auto oddShuffle = LLVM::ShuffleVectorOp::create(rewriter,
-        loc, adaptor.getSource(), poison, oddShuffleMask);
+    auto evenShuffle = LLVM::ShuffleVectorOp::create(
+        rewriter, loc, adaptor.getSource(), poison, evenShuffleMask);
+    auto oddShuffle = LLVM::ShuffleVectorOp::create(
+        rewriter, loc, adaptor.getSource(), poison, oddShuffleMask);
 
     rewriter.replaceOp(deinterleaveOp, ValueRange{evenShuffle, oddShuffle});
     return success();
@@ -2003,12 +2008,12 @@ struct VectorToElementsLowering
       if (element.use_empty())
         continue;
 
-      auto constIdx = LLVM::ConstantOp::create(rewriter,
-          loc, idxType, rewriter.getIntegerAttr(idxType, idx));
+      auto constIdx = LLVM::ConstantOp::create(
+          rewriter, loc, idxType, rewriter.getIntegerAttr(idxType, idx));
       auto llvmType = typeConverter->convertType(element.getType());
 
       Value result = LLVM::ExtractElementOp::create(rewriter, loc, llvmType,
-                                                             source, constIdx);
+                                                    source, constIdx);
       results[idx] = result;
     }
 
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index 7385e28aeb04a..f11fca77bdbe4 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -202,7 +202,7 @@ static Value generateInBoundsCheck(
     Value memrefIdx =
         affine::makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
     cond = arith::CmpIOp::create(lb, arith::CmpIPredicate::sgt, memrefDim,
-                                    memrefIdx);
+                                 memrefIdx);
   }
 
   // Condition check 2: Masked in?
@@ -215,8 +215,8 @@ static Value generateInBoundsCheck(
 
   // If the condition is non-empty, generate an SCF::IfOp.
   if (cond) {
-    auto check = scf::IfOp::create(lb,
-        cond,
+    auto check = scf::IfOp::create(
+        lb, cond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
           maybeYieldValue(b, loc, hasRetVal, inBoundsCase(b, loc));
@@ -310,7 +310,8 @@ static BufferAllocs allocBuffers(OpBuilder &b, OpTy xferOp) {
     auto maskBuffer = memref::AllocaOp::create(b, loc, maskType);
     b.setInsertionPoint(xferOp);
     memref::StoreOp::create(b, loc, xferOp.getMask(), maskBuffer);
-    result.maskBuffer = memref::LoadOp::create(b, loc, maskBuffer, ValueRange());
+    result.maskBuffer =
+        memref::LoadOp::create(b, loc, maskBuffer, ValueRange());
   }
 
   return result;
@@ -421,14 +422,15 @@ struct Strategy<TransferReadOp> {
     auto bufferType = dyn_cast<ShapedType>(buffer.getType());
     auto vecType = dyn_cast<VectorType>(bufferType.getElementType());
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
-    auto newXferOp = vector::TransferReadOp::create(b,
-        loc, vecType, xferOp.getBase(), xferIndices,
+    auto newXferOp = vector::TransferReadOp::create(
+        b, loc, vecType, xferOp.getBase(), xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
         xferOp.getPadding(), Value(), inBoundsAttr);
 
     maybeApplyPassLabel(b, newXferOp, options.targetRank);
 
-    memref::StoreOp::create(b, loc, newXferOp.getVector(), buffer, storeIndices);
+    memref::StoreOp::create(b, loc, newXferOp.getVector(), buffer,
+                            storeIndices);
     return newXferOp;
   }
 
@@ -510,8 +512,8 @@ struct Strategy<TransferWriteOp> {
     auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
     auto source = loopState.empty() ? xferOp.getBase() : loopState[0];
     Type type = isTensorOp(xferOp) ? xferOp.getShapedType() : Type();
-    auto newXferOp = vector::TransferWriteOp::create(b,
-        loc, type, vec, source, xferIndices,
+    auto newXferOp = vector::TransferWriteOp::create(
+        b, loc, type, vec, source, xferIndices,
         AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
         inBoundsAttr);
 
@@ -611,7 +613,7 @@ struct PrepareTransferReadConversion
 
     Location loc = xferOp.getLoc();
     memref::StoreOp::create(rewriter, loc, newXfer->getResult(0),
-                                     buffers.dataBuffer);
+                            buffers.dataBuffer);
     rewriter.replaceOpWithNewOp<memref::LoadOp>(xferOp, buffers.dataBuffer);
 
     return success();
@@ -654,7 +656,7 @@ struct PrepareTransferWriteConversion
     Location loc = xferOp.getLoc();
     auto buffers = allocBuffers(rewriter, xferOp);
     memref::StoreOp::create(rewriter, loc, xferOp.getVector(),
-                                     buffers.dataBuffer);
+                            buffers.dataBuffer);
     auto loadedVec = memref::LoadOp::create(rewriter, loc, buffers.dataBuffer);
     rewriter.modifyOpInPlace(xferOp, [&]() {
       xferOp.getValueToStoreMutable().assign(loadedVec);
@@ -736,14 +738,14 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
           vectorType.cloneWith({}, getIntTypeWithSignlessSemantics(legalIntTy));
       auto targetVectorType = vectorType.cloneWith({}, legalIntTy);
       value = vector::BitCastOp::create(rewriter, loc, signlessSourceVectorType,
-                                                 value);
+                                        value);
       if (value.getType() != signlessTargetVectorType) {
         if (width == 1 || intTy.isUnsigned())
-          value = arith::ExtUIOp::create(rewriter, loc, signlessTargetVectorType,
-                                                  value);
+          value = arith::ExtUIOp::create(rewriter, loc,
+                                         signlessTargetVectorType, value);
         else
-          value = arith::ExtSIOp::create(rewriter, loc, signlessTargetVectorType,
-                                                  value);
+          value = arith::ExtSIOp::create(rewriter, loc,
+                                         signlessTargetVectorType, value);
       }
       value = vector::BitCastOp::create(rewriter, loc, targetVectorType, value);
       vectorType = targetVectorType;
@@ -770,11 +772,12 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     for (unsigned d = 0; d < shape.size(); d++) {
       // Setup loop bounds and step.
       Value lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
-      Value upperBound = arith::ConstantIndexOp::create(rewriter, loc, shape[d]);
+      Value upperBound =
+          arith::ConstantIndexOp::create(rewriter, loc, shape[d]);
       Value step = arith::ConstantIndexOp::create(rewriter, loc, 1);
       if (!scalableDimensions.empty() && scalableDimensions[d]) {
-        auto vscale = vector::VectorScaleOp::create(rewriter,
-            loc, rewriter.getIndexType());
+        auto vscale = vector::VectorScaleOp::create(rewriter, loc,
+                                                    rewriter.getIndexType());
         upperBound = arith::MulIOp::create(rewriter, loc, upperBound, vscale);
       }
       auto lastIndex = arith::SubIOp::create(rewriter, loc, upperBound, step);
@@ -783,8 +786,8 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
       vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
       auto loop =
           scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step);
-      auto printClose = vector::PrintOp::create(rewriter,
-          loc, vector::PrintPunctuation::Close);
+      auto printClose = vector::PrintOp::create(
+          rewriter, loc, vector::PrintPunctuation::Close);
       if (!firstClose)
         firstClose = printClose;
 
@@ -793,14 +796,14 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
 
       // Print a comma after all but the last element.
       rewriter.setInsertionPointToStart(loop.getBody());
-      auto notLastIndex = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::ult, loopIdx, lastIndex);
+      auto notLastIndex = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::ult, loopIdx, lastIndex);
       scf::IfOp::create(rewriter, loc, notLastIndex,
-                                 [&](OpBuilder &builder, Location loc) {
-                                   vector::PrintOp::create(builder,
-                                       loc, vector::PrintPunctuation::Comma);
-                                   scf::YieldOp::create(builder, loc);
-                                 });
+                        [&](OpBuilder &builder, Location loc) {
+                          vector::PrintOp::create(
+                              builder, loc, vector::PrintPunctuation::Comma);
+                          scf::YieldOp::create(builder, loc);
+                        });
 
       rewriter.setInsertionPointToStart(loop.getBody());
     }
@@ -810,7 +813,8 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     Value flatIndex;
     auto currentStride = 1;
     for (int d = shape.size() - 1; d >= 0; d--) {
-      auto stride = arith::ConstantIndexOp::create(rewriter, loc, currentStride);
+      auto stride =
+          arith::ConstantIndexOp::create(rewriter, loc, currentStride);
       auto index = arith::MulIOp::create(rewriter, loc, stride, loopIndices[d]);
       if (flatIndex)
         flatIndex = arith::AddIOp::create(rewriter, loc, flatIndex, index);
@@ -820,10 +824,9 @@ struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> {
     }
 
     // Print the scalar elements in the inner most loop.
-    auto element =
-        vector::ExtractOp::create(rewriter, loc, value, flatIndex);
+    auto element = vector::ExtractOp::create(rewriter, loc, value, flatIndex);
     vector::PrintOp::create(rewriter, loc, element,
-                                     vector::PrintPunctuation::NoPunctuation);
+                            vector::PrintPunctuation::NoPunctuation);
 
     rewriter.setInsertionPointAfter(firstClose);
     vector::PrintOp::create(rewriter, loc, printOp.getPunctuation());
@@ -942,16 +945,16 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
 
     // Loop bounds and step.
     auto lb = arith::ConstantIndexOp::create(locB, 0);
-    auto ub = arith::ConstantIndexOp::create(locB,
-        castedDataType->getDimSize(castedDataType->getRank() - 1));
+    auto ub = arith::ConstantIndexOp::create(
+        locB, castedDataType->getDimSize(castedDataType->getRank() - 1));
     auto step = arith::ConstantIndexOp::create(locB, 1);
     // TransferWriteOps that operate on tensors return the modified tensor and
     // require a loop state.
     auto loopState = Strategy<OpTy>::initialLoopState(xferOp);
 
     // Generate for loop.
-    auto result = scf::ForOp::create(locB,
-        lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(),
+    auto result = scf::ForOp::create(
+        locB, lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(),
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopState) {
           Type stateType = loopState.empty() ? Type() : loopState[0].getType();
 
@@ -977,7 +980,7 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
                   getMaskBufferLoadIndices(xferOp, castedMaskBuffer,
                                            loadIndices, iv);
                   auto mask = memref::LoadOp::create(b, loc, castedMaskBuffer,
-                                                       loadIndices);
+                                                     loadIndices);
                   rewriter.modifyOpInPlace(newXfer, [&]() {
                     newXfer.getMaskMutable().assign(mask);
                   });
@@ -1135,15 +1138,15 @@ struct ScalableTransposeTransferWriteConversion
     VectorType sliceType = VectorType::Builder(vectorType).dropDim(0);
     Value sliceMask = nullptr;
     if (!maskDims->empty()) {
-      sliceMask = vector::CreateMaskOp::create(rewriter,
-          loc, sliceType.clone(rewriter.getI1Type()),
+      sliceMask = vector::CreateMaskOp::create(
+          rewriter, loc, sliceType.clone(rewriter.getI1Type()),
           ArrayRef<OpFoldResult>(*maskDims).drop_front());
     }
 
     Value initDest = isTensorOp(writeOp) ? writeOp.getBase() : Value{};
     ValueRange initLoopArgs = initDest ? initDest : ValueRange{};
-    auto result = scf::ForOp::create(rewriter,
-        loc, lb, ub, step, initLoopArgs,
+    auto result = scf::ForOp::create(
+        rewriter, loc, lb, ub, step, initLoopArgs,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange loopIterArgs) {
           // Indices for the new transfer op.
           SmallVector<Value, 8> xferIndices;
@@ -1152,25 +1155,25 @@ struct ScalableTransposeTransferWriteConversion
           // Extract a transposed slice from the source vector.
           SmallVector<Value> transposeElements =
               llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
-                return vector::ExtractOp::create(b,
-                    loc, transposeSourceSlices[idx], iv);
+                return vector::ExtractOp::create(
+                    b, loc, transposeSourceSlices[idx], iv);
               });
           auto sliceVec = vector::FromElementsOp::create(b, loc, sliceType,
-                                                           transposeElements);
+                                                         transposeElements);
 
           // Create the transfer_write for the slice.
           Value dest =
               loopIterArgs.empty() ? writeOp.getBase() : loopIterArgs.front();
-          auto newWriteOp = vector::TransferWriteOp::create(b,
-              loc, sliceVec, dest, xferIndices,
+          auto newWriteOp = vector::TransferWriteOp::create(
+              b, loc, sliceVec, dest, xferIndices,
               ArrayRef<bool>(writeOp.getInBoundsValues()).drop_front());
           if (sliceMask)
             newWriteOp.getMaskMutable().assign(sliceMask);
 
           // Yield from the loop.
-          scf::YieldOp::create(b, loc, loopIterArgs.empty()
-                                          ? ValueRange{}
-                                          : newWriteOp.getResult());
+          scf::YieldOp::create(b, loc,
+                               loopIterArgs.empty() ? ValueRange{}
+                                                    : newWriteOp.getResult());
         });
 
     if (isTensorOp(writeOp))
@@ -1263,7 +1266,7 @@ struct UnrollTransferReadConversion
       return insertOp.getDest();
     Location loc = xferOp.getLoc();
     return vector::SplatOp::create(rewriter, loc, xferOp.getVectorType(),
-                                            xferOp.getPadding());
+                                   xferOp.getPadding());
   }
 
   /// If the result of the TransferReadOp has exactly one user, which is a
@@ -1337,8 +1340,8 @@ struct UnrollTransferReadConversion
 
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
 
-            auto newXferOp = vector::TransferReadOp::create(b,
-                loc, newXferVecType, xferOp.getBase(), xferIndices,
+            auto newXferOp = vector::TransferReadOp::create(
+                b, loc, newXferVecType, xferOp.getBase(), xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
                 xferOp.getPadding(), Value(), inBoundsAttr);
             maybeAssignMask(b, xferOp, newXferOp, i);
@@ -1348,10 +1351,10 @@ struct UnrollTransferReadConversion
               // vector.insert does not accept rank-0 as the non-indexed
               // argument. Extract the scalar before inserting.
               valToInser = vector::ExtractOp::create(b, loc, valToInser,
-                                                       SmallVector<int64_t>());
+                                                     SmallVector<int64_t>());
             }
             return vector::InsertOp::create(b, loc, valToInser, vec,
-                                              insertionIndices);
+                                            insertionIndices);
           },
           /*outOfBoundsCase=*/
           [&](OpBuilder &b, Location loc) {
@@ -1485,13 +1488,13 @@ struct UnrollTransferWriteConversion
               // When target-rank=0, unrolling would causes the vector input
               // argument into `transfer_write` to become a scalar. We solve
               // this by broadcasting the scalar to a 0D vector.
-              xferVec = vector::BroadcastOp::create(b,
-                  loc, VectorType::get({}, extracted.getType()), extracted);
+              xferVec = vector::BroadcastOp::create(
+                  b, loc, VectorType::get({}, extracted.getType()), extracted);
             } else {
               xferVec = extracted;
             }
-            auto newXferOp = vector::TransferWriteOp::create(b,
-                loc, sourceType, xferVec, source, xferIndices,
+            auto newXferOp = vector::TransferWriteOp::create(
+                b, loc, sourceType, xferVec, source, xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
                 inBoundsAttr);
 
@@ -1585,7 +1588,7 @@ struct Strategy1d<TransferReadOp> {
     // Inititalize vector with padding value.
     Location loc = xferOp.getLoc();
     return vector::SplatOp::create(b, loc, xferOp.getVectorType(),
-                                     xferOp.getPadding());
+                                   xferOp.getPadding());
   }
 };
 
@@ -1602,8 +1605,7 @@ struct Strategy1d<TransferWriteOp> {
     generateInBoundsCheck(
         b, xferOp, iv, dim,
         /*inBoundsCase=*/[&](OpBuilder &b, Location loc) {
-          auto val =
-              vector::ExtractOp::create(b, loc, xferOp.getVector(), iv);
+          auto val = vector::ExtractOp::create(b, loc, xferOp.getVector(), iv);
           memref::StoreOp::create(b, loc, val, xferOp.getBase(), indices);
         });
     scf::YieldOp::create(b, loc);
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 2466e72277441..ea1b9c1c4203e 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -147,19 +147,19 @@ static Value sanitizeDynamicIndex(ConversionPatternRewriter &rewriter,
                                   Location loc, Value dynamicIndex,
                                   int64_t kPoisonIndex, unsigned vectorSize) {
   if (llvm::isPowerOf2_32(vectorSize)) {
-    Value inBoundsMask = spirv::ConstantOp::create(rewriter,
-        loc, dynamicIndex.getType(),
+    Value inBoundsMask = spirv::ConstantOp::create(
+        rewriter, loc, dynamicIndex.getType(),
         rewriter.getIntegerAttr(dynamicIndex.getType(), vectorSize - 1));
     return spirv::BitwiseAndOp::create(rewriter, loc, dynamicIndex,
-                                                inBoundsMask);
+                                       inBoundsMask);
   }
-  Value poisonIndex = spirv::ConstantOp::create(rewriter,
-      loc, dynamicIndex.getType(),
+  Value poisonIndex = spirv::ConstantOp::create(
+      rewriter, loc, dynamicIndex.getType(),
       rewriter.getIntegerAttr(dynamicIndex.getType(), kPoisonIndex));
   Value cmpResult =
       spirv::IEqualOp::create(rewriter, loc, dynamicIndex, poisonIndex);
-  return spirv::SelectOp::create(rewriter,
-      loc, cmpResult,
+  return spirv::SelectOp::create(
+      rewriter, loc, cmpResult,
       spirv::ConstantOp::getZero(dynamicIndex.getType(), loc, rewriter),
       dynamicIndex);
 }
@@ -427,8 +427,8 @@ static SmallVector<Value> extractAllElements(
   Location loc = reduceOp.getLoc();
 
   for (int i = 0; i < numElements; ++i) {
-    values.push_back(spirv::CompositeExtractOp::create(rewriter,
-        loc, srcVectorType.getElementType(), adaptor.getVector(),
+    values.push_back(spirv::CompositeExtractOp::create(
+        rewriter, loc, srcVectorType.getElementType(), adaptor.getVector(),
         rewriter.getI32ArrayAttr({i})));
   }
   if (Value acc = adaptor.getAcc())
@@ -481,16 +481,16 @@ struct VectorReductionPattern final : OpConversionPattern<vector::ReductionOp> {
 #define INT_AND_FLOAT_CASE(kind, iop, fop)                                     \
   case vector::CombiningKind::kind:                                            \
     if (llvm::isa<IntegerType>(resultType)) {                                  \
-      result = spirv::iop::create(rewriter, loc, resultType, result, next);     \
+      result = spirv::iop::create(rewriter, loc, resultType, result, next);    \
     } else {                                                                   \
       assert(llvm::isa<FloatType>(resultType));                                \
-      result = spirv::fop::create(rewriter, loc, resultType, result, next);     \
+      result = spirv::fop::create(rewriter, loc, resultType, result, next);    \
     }                                                                          \
     break
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = fop::create(rewriter, loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);             \
     break
 
         INT_AND_FLOAT_CASE(ADD, IAddOp, FAddOp);
@@ -537,7 +537,7 @@ struct VectorReductionFloatMinMax final
 
 #define INT_OR_FLOAT_CASE(kind, fop)                                           \
   case vector::CombiningKind::kind:                                            \
-    result = fop::create(rewriter, loc, resultType, result, next);              \
+    result = fop::create(rewriter, loc, resultType, result, next);             \
     break
 
         INT_OR_FLOAT_CASE(MAXIMUMF, SPIRVFMaxOp);
@@ -614,7 +614,7 @@ struct VectorShuffleOpConvert final
                                Value scalarOrVec, int32_t idx) -> Value {
       if (auto vecTy = dyn_cast<VectorType>(scalarOrVec.getType()))
         return spirv::CompositeExtractOp::create(rewriter, loc, scalarOrVec,
-                                                          idx);
+                                                 idx);
 
       assert(idx == 0 && "Invalid scalar element index");
       return scalarOrVec;
@@ -712,11 +712,13 @@ struct VectorDeinterleaveOpConvert final
     // We cannot use `spirv::VectorShuffleOp` directly in this case, and need to
     // use `spirv::CompositeExtractOp`.
     if (n == 2) {
-      auto elem0 = spirv::CompositeExtractOp::create(rewriter,
-          loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({0}));
+      auto elem0 = spirv::CompositeExtractOp::create(
+          rewriter, loc, newResultType, sourceVector,
+          rewriter.getI32ArrayAttr({0}));
 
-      auto elem1 = spirv::CompositeExtractOp::create(rewriter,
-          loc, newResultType, sourceVector, rewriter.getI32ArrayAttr({1}));
+      auto elem1 = spirv::CompositeExtractOp::create(
+          rewriter, loc, newResultType, sourceVector,
+          rewriter.getI32ArrayAttr({1}));
 
       rewriter.replaceOp(deinterleaveOp, {elem0, elem1});
       return success();
@@ -733,12 +735,12 @@ struct VectorDeinterleaveOpConvert final
         llvm::map_to_vector(seqOdd, [](int i) { return i * 2 + 1; });
 
     // Create two SPIR-V shuffles.
-    auto shuffleEven = spirv::VectorShuffleOp::create(rewriter,
-        loc, newResultType, sourceVector, sourceVector,
+    auto shuffleEven = spirv::VectorShuffleOp::create(
+        rewriter, loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesEven));
 
-    auto shuffleOdd = spirv::VectorShuffleOp::create(rewriter,
-        loc, newResultType, sourceVector, sourceVector,
+    auto shuffleOdd = spirv::VectorShuffleOp::create(
+        rewriter, loc, newResultType, sourceVector, sourceVector,
         rewriter.getI32ArrayAttr(indicesOdd));
 
     rewriter.replaceOp(deinterleaveOp, {shuffleEven, shuffleOdd});
@@ -779,10 +781,11 @@ struct VectorLoadOpConverter final
     // For single element vectors, we don't need to bitcast the access chain to
     // the original vector type. Both is going to be the same, a pointer
     // to a scalar.
-    Value castedAccessChain = (vectorType.getNumElements() == 1)
-                                  ? accessChain
-                                  : spirv::BitcastOp::create(rewriter,
-                                        loc, vectorPtrType, accessChain);
+    Value castedAccessChain =
+        (vectorType.getNumElements() == 1)
+            ? accessChain
+            : spirv::BitcastOp::create(rewriter, loc, vectorPtrType,
+                                       accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::LoadOp>(loadOp, spirvVectorType,
                                                castedAccessChain);
@@ -821,10 +824,11 @@ struct VectorStoreOpConverter final
     // For single element vectors, we don't need to bitcast the access chain to
     // the original vector type. Both is going to be the same, a pointer
     // to a scalar.
-    Value castedAccessChain = (vectorType.getNumElements() == 1)
-                                  ? accessChain
-                                  : spirv::BitcastOp::create(rewriter,
-                                        loc, vectorPtrType, accessChain);
+    Value castedAccessChain =
+        (vectorType.getNumElements() == 1)
+            ? accessChain
+            : spirv::BitcastOp::create(rewriter, loc, vectorPtrType,
+                                       accessChain);
 
     rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, castedAccessChain,
                                                 adaptor.getValueToStore());
@@ -905,10 +909,10 @@ struct VectorReductionToIntDotProd final
       auto v4i8Type = VectorType::get({4}, i8Type);
       Location loc = op.getLoc();
       Value zero = spirv::ConstantOp::getZero(i8Type, loc, rewriter);
-      lhsIn = spirv::CompositeConstructOp::create(rewriter,
-          loc, v4i8Type, ValueRange{lhsIn, zero});
-      rhsIn = spirv::CompositeConstructOp::create(rewriter,
-          loc, v4i8Type, ValueRange{rhsIn, zero});
+      lhsIn = spirv::CompositeConstructOp::create(rewriter, loc, v4i8Type,
+                                                  ValueRange{lhsIn, zero});
+      rhsIn = spirv::CompositeConstructOp::create(rewriter, loc, v4i8Type,
+                                                  ValueRange{rhsIn, zero});
     }
 
     // There's no variant of dot prod ops for unsigned LHS and signed RHS, so
@@ -1013,7 +1017,8 @@ struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
     source.reserve(numElements);
     for (int64_t i = 0; i < numElements; ++i) {
       Attribute intAttr = rewriter.getIntegerAttr(intType, i);
-      Value constOp = spirv::ConstantOp::create(rewriter, loc, intType, intAttr);
+      Value constOp =
+          spirv::ConstantOp::create(rewriter, loc, intType, intAttr);
       source.push_back(constOp);
     }
     rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(stepOp, dstType,
@@ -1056,8 +1061,8 @@ struct VectorToElementOpConvert final
       if (element.use_empty())
         continue;
 
-      Value result = spirv::CompositeExtractOp::create(rewriter,
-          loc, elementType, adaptor.getSource(),
+      Value result = spirv::CompositeExtractOp::create(
+          rewriter, loc, elementType, adaptor.getSource(),
           rewriter.getI32ArrayAttr({static_cast<int32_t>(idx)}));
       results[idx] = result;
     }
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index 8e074f91744e8..80107554144cf 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -109,7 +109,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
   xegpu::CreateNdDescOp ndDesc;
   if (srcTy.hasStaticShape()) {
     ndDesc = xegpu::CreateNdDescOp::create(rewriter, loc, descType, src,
-                                                    getAsOpFoldResult(offsets));
+                                           getAsOpFoldResult(offsets));
   } else {
     // In case of any dynamic shapes, source's shape and strides have to be
     // explicitly provided.
@@ -145,8 +145,8 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
     }
     std::reverse(dynStrides.begin(), dynStrides.end());
 
-    ndDesc = xegpu::CreateNdDescOp::create(rewriter,
-        loc, descType, src, dynOffsets, dynShapes, dynStrides,
+    ndDesc = xegpu::CreateNdDescOp::create(
+        rewriter, loc, descType, src, dynOffsets, dynShapes, dynStrides,
         DenseI64ArrayAttr::get(rewriter.getContext(), constOffsets),
         DenseI64ArrayAttr::get(rewriter.getContext(), srcTy.getShape()),
         DenseI64ArrayAttr::get(rewriter.getContext(), strides));
@@ -200,10 +200,10 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
                                                   ArrayRef<int64_t>{1, 0});
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadOp = xegpu::LoadNdOp::create(rewriter,
-        loc, vecTy, ndDesc, /*packed=*/nullptr, transposeAttr,
-        /*l1_hint=*/hint,
-        /*l2_hint=*/hint, /*l3_hint=*/hint);
+    auto loadOp = xegpu::LoadNdOp::create(rewriter, loc, vecTy, ndDesc,
+                                          /*packed=*/nullptr, transposeAttr,
+                                          /*l1_hint=*/hint,
+                                          /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(readOp, loadOp);
 
     return success();
@@ -239,8 +239,8 @@ struct TransferWriteLowering
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeOp =
         xegpu::StoreNdOp::create(rewriter, loc, writeOp.getVector(), ndDesc,
-                                          /*l1_hint=*/hint,
-                                          /*l2_hint=*/hint, /*l3_hint=*/hint);
+                                 /*l1_hint=*/hint,
+                                 /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(writeOp, storeOp);
 
     return success();
@@ -269,8 +269,8 @@ struct LoadLowering : public OpRewritePattern<vector::LoadOp> {
 
     // By default, no specific caching policy is assigned.
     xegpu::CachePolicyAttr hint = nullptr;
-    auto loadNdOp = xegpu::LoadNdOp::create(rewriter,
-        loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
+    auto loadNdOp = xegpu::LoadNdOp::create(
+        rewriter, loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
         /*l1_hint=*/hint,
         /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(loadOp, loadNdOp);
@@ -304,8 +304,8 @@ struct StoreLowering : public OpRewritePattern<vector::StoreOp> {
     xegpu::CachePolicyAttr hint = nullptr;
     auto storeNdOp =
         xegpu::StoreNdOp::create(rewriter, loc, vector, ndDesc,
-                                          /*l1_hint=*/hint,
-                                          /*l2_hint=*/hint, /*l3_hint=*/hint);
+                                 /*l1_hint=*/hint,
+                                 /*l2_hint=*/hint, /*l3_hint=*/hint);
     rewriter.replaceOp(storeOp, storeNdOp);
 
     return success();
@@ -339,8 +339,9 @@ struct ContractionLowering : public OpRewritePattern<vector::ContractionOp> {
     if (!isRowMajorMatmul(contractOp.getIndexingMapsAttr()))
       return rewriter.notifyMatchFailure(contractOp, "Invalid indexing maps");
 
-    auto dpasOp = xegpu::DpasOp::create(rewriter,
-        loc, TypeRange{contractOp.getResultType()}, ValueRange{lhs, rhs, acc});
+    auto dpasOp = xegpu::DpasOp::create(rewriter, loc,
+                                        TypeRange{contractOp.getResultType()},
+                                        ValueRange{lhs, rhs, acc});
     rewriter.replaceOp(contractOp, dpasOp);
 
     return success();
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
index 0230810f260ca..6d1f64e94df15 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -118,8 +118,8 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
 
   SmallVector<NamedAttribute> loadAttrs;
   patchOperandSegmentSizes(origAttrs, loadAttrs, DataArgAction::Drop);
-  Value initialLoad =
-      RawBufferLoadOp::create(rewriter, loc, dataType, invariantArgs, loadAttrs);
+  Value initialLoad = RawBufferLoadOp::create(rewriter, loc, dataType,
+                                              invariantArgs, loadAttrs);
   Block *currentBlock = rewriter.getInsertionBlock();
   Block *afterAtomic =
       rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
@@ -137,8 +137,8 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
   patchOperandSegmentSizes(origAttrs, cmpswapAttrs, DataArgAction::Duplicate);
   SmallVector<Value> cmpswapArgs = {operated, prevLoad};
   cmpswapArgs.append(invariantArgs.begin(), invariantArgs.end());
-  Value atomicRes = RawBufferAtomicCmpswapOp::create(rewriter,
-      loc, dataType, cmpswapArgs, cmpswapAttrs);
+  Value atomicRes = RawBufferAtomicCmpswapOp::create(rewriter, loc, dataType,
+                                                     cmpswapArgs, cmpswapAttrs);
 
   // We care about exact bitwise equality here, so do some bitcasts.
   // These will fold away during lowering to the ROCDL dialect, where
@@ -154,10 +154,11 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
     atomicResForCompare =
         arith::BitcastOp::create(rewriter, loc, equivInt, atomicRes);
   }
-  Value canLeave = arith::CmpIOp::create(rewriter,
-      loc, arith::CmpIPredicate::eq, atomicResForCompare, prevLoadForCompare);
+  Value canLeave =
+      arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
+                            atomicResForCompare, prevLoadForCompare);
   cf::CondBranchOp::create(rewriter, loc, canLeave, afterAtomic, ValueRange{},
-                                    loopBlock, atomicRes);
+                           loopBlock, atomicRes);
   rewriter.eraseOp(atomicOp);
   return success();
 }
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
index a11804c1d7741..86b2025e2471b 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -56,11 +56,11 @@ static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
                                            vector::MaskedLoadOp maskedOp,
                                            bool passthru) {
   VectorType vectorType = maskedOp.getVectorType();
-  Value load = vector::LoadOp::create(builder,
-      loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
+  Value load = vector::LoadOp::create(
+      builder, loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
   if (passthru)
-    load = arith::SelectOp::create(builder,loc, vectorType, maskedOp.getMask(),
-                                           load, maskedOp.getPassThru());
+    load = arith::SelectOp::create(builder, loc, vectorType, maskedOp.getMask(),
+                                   load, maskedOp.getPassThru());
   return load;
 }
 
@@ -132,14 +132,14 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
     Value delta = arith::SubIOp::create(rewriter, loc, totalSize, linearIndex);
 
     // 1) check if delta < vectorSize
-    Value isOutofBounds = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
+    Value isOutofBounds = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
 
     // 2) check if (detla % elements_per_word != 0)
-    Value elementsPerWord = arith::ConstantIndexOp::create(rewriter,
-        loc, llvm::divideCeil(32, elementBitWidth));
-    Value isNotWordAligned = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ne,
+    Value elementsPerWord = arith::ConstantIndexOp::create(
+        rewriter, loc, llvm::divideCeil(32, elementBitWidth));
+    Value isNotWordAligned = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ne,
         arith::RemUIOp::create(rewriter, loc, delta, elementsPerWord),
         arith::ConstantIndexOp::create(rewriter, loc, 0));
 
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
index 4771d655e36c2..f8bab8289cbc6 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
@@ -37,8 +37,8 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       return rewriter.notifyMatchFailure(metadataOp,
                                          "not a fat raw buffer cast");
     Location loc = castOp.getLoc();
-    auto sourceMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
-        loc, castOp.getSource());
+    auto sourceMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, castOp.getSource());
     SmallVector<Value> results;
     if (metadataOp.getBaseBuffer().use_empty()) {
       results.push_back(nullptr);
@@ -48,8 +48,8 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
       if (baseBufferType == castOp.getResult().getType()) {
         results.push_back(castOp.getResult());
       } else {
-        results.push_back(memref::ReinterpretCastOp::create(rewriter,
-            loc, baseBufferType, castOp.getResult(), /*offset=*/0,
+        results.push_back(memref::ReinterpretCastOp::create(
+            rewriter, loc, baseBufferType, castOp.getResult(), /*offset=*/0,
             /*sizes=*/ArrayRef<int64_t>{}, /*strides=*/ArrayRef<int64_t>{}));
       }
     }
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
index fb0d0d5027085..c0d174a04abf9 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
@@ -58,8 +58,9 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
       // Note: basis elements and their products are, definitionally,
       // non-negative, so `nuw` is justified.
       if (dynamicPart)
-        dynamicPart = arith::MulIOp::create(rewriter,
-            loc, dynamicPart, dynamicBasis[dynamicIndex - 1], ovflags);
+        dynamicPart =
+            arith::MulIOp::create(rewriter, loc, dynamicPart,
+                                  dynamicBasis[dynamicIndex - 1], ovflags);
       else
         dynamicPart = dynamicBasis[dynamicIndex - 1];
       --dynamicIndex;
@@ -111,15 +112,15 @@ affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
 
   auto emitModTerm = [&](Value stride) -> Value {
     Value remainder = arith::RemSIOp::create(rewriter, loc, linearIdx, stride);
-    Value remainderNegative = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::slt, remainder, zero);
+    Value remainderNegative = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::slt, remainder, zero);
     // If the correction is relevant, this term is <= stride, which is known
     // to be positive in `index`. Otherwise, while 2 * stride might overflow,
     // this branch won't be taken, so the risk of `poison` is fine.
-    Value corrected = arith::AddIOp::create(rewriter,
-        loc, remainder, stride, arith::IntegerOverflowFlags::nsw);
+    Value corrected = arith::AddIOp::create(rewriter, loc, remainder, stride,
+                                            arith::IntegerOverflowFlags::nsw);
     Value mod = arith::SelectOp::create(rewriter, loc, remainderNegative,
-                                                 corrected, remainder);
+                                        corrected, remainder);
     return mod;
   };
 
@@ -167,8 +168,8 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   // our hands on an `OpOperand&` for the loop invariant counting function.
   for (auto [stride, idxOp] :
        llvm::zip_equal(strides, llvm::drop_end(op.getMultiIndexMutable()))) {
-    Value scaledIdx = arith::MulIOp::create(rewriter,
-        loc, idxOp.get(), stride, arith::IntegerOverflowFlags::nsw);
+    Value scaledIdx = arith::MulIOp::create(rewriter, loc, idxOp.get(), stride,
+                                            arith::IntegerOverflowFlags::nsw);
     int64_t numHoistableLoops = numEnclosingInvariantLoops(idxOp);
     scaledValues.emplace_back(scaledIdx, numHoistableLoops);
   }
@@ -185,7 +186,7 @@ LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
   for (auto [scaledValue, numHoistableLoops] : llvm::drop_begin(scaledValues)) {
     std::ignore = numHoistableLoops;
     result = arith::AddIOp::create(rewriter, loc, result, scaledValue,
-                                            arith::IntegerOverflowFlags::nsw);
+                                   arith::IntegerOverflowFlags::nsw);
   }
   rewriter.replaceOp(op, result);
   return success();
diff --git a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
index a78cea5bff6dd..3c00b323473d2 100644
--- a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
@@ -89,7 +89,7 @@ static AffineApplyOp createSubApply(RewriterBase &rewriter,
   SmallVector<Value> rhsOperands = originalOp->getOperands();
   canonicalizeMapAndOperands(&rhsMap, &rhsOperands);
   return AffineApplyOp::create(rewriter, originalOp.getLoc(), rhsMap,
-                                        rhsOperands);
+                               rhsOperands);
 }
 
 FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
@@ -161,7 +161,7 @@ FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
   for (int64_t i = 1, e = subExpressions.size(); i < e; ++i) {
     Value tmp = createSubApply(rewriter, op, subExpressions[i]);
     current = AffineApplyOp::create(rewriter, op.getLoc(), binMap,
-                                             ValueRange{current, tmp});
+                                    ValueRange{current, tmp});
     LLVM_DEBUG(DBGS() << "--reassociate into: " << current << "\n");
   }
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
index 17384654586c4..c942c0248fefd 100644
--- a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
@@ -100,8 +100,8 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   }
 
   // Create and place the alloc right before the 'affine.for' operation.
-  Value newMemRef = memref::AllocOp::create(bOuter,
-      forOp.getLoc(), newMemRefType, allocOperands);
+  Value newMemRef = memref::AllocOp::create(bOuter, forOp.getLoc(),
+                                            newMemRefType, allocOperands);
 
   // Create 'iv mod 2' value to index the leading dimension.
   auto d0 = bInner.getAffineDimExpr(0);
@@ -109,7 +109,7 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
   auto modTwoMap =
       AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, d0.floorDiv(step) % 2);
   auto ivModTwoOp = AffineApplyOp::create(bInner, forOp.getLoc(), modTwoMap,
-                                                 forOp.getInductionVar());
+                                          forOp.getInductionVar());
 
   // replaceAllMemRefUsesWith will succeed unless the forOp body has
   // non-dereferencing uses of the memref (dealloc's are fine though).
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 0f16e84263ec1..f90aa3a68a1ea 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -56,7 +56,7 @@ getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
   OpBuilder b(forOp);
   auto lbMap = forOp.getLowerBoundMap();
   auto lb = AffineApplyOp::create(b, forOp.getLoc(), lbMap,
-                                    forOp.getLowerBoundOperands());
+                                  forOp.getLowerBoundOperands());
 
   // For each upper bound expr, get the range.
   // Eg: affine.for %i = lb to min (ub1, ub2),
@@ -135,8 +135,8 @@ LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
         builder.setInsertionPointToStart(&func.getFunctionBody().front());
       else
         builder.setInsertionPoint(forOp);
-      auto constOp = arith::ConstantIndexOp::create(builder,
-          forOp.getLoc(), forOp.getConstantLowerBound());
+      auto constOp = arith::ConstantIndexOp::create(
+          builder, forOp.getLoc(), forOp.getConstantLowerBound());
       iv.replaceAllUsesWith(constOp);
     } else {
       auto lbOperands = forOp.getLowerBoundOperands();
@@ -183,7 +183,7 @@ static AffineForOp generateShiftedLoop(
 
   auto loopChunk =
       AffineForOp::create(b, srcForOp.getLoc(), lbOperands, lbMap, ubOperands,
-                            ubMap, srcForOp.getStepAsInt());
+                          ubMap, srcForOp.getStepAsInt());
   auto loopChunkIV = loopChunk.getInductionVar();
   auto srcIV = srcForOp.getInductionVar();
 
@@ -198,8 +198,8 @@ static AffineForOp generateShiftedLoop(
     // Generate the remapping if the shift is not zero: remappedIV = newIV -
     // shift.
     if (!srcIV.use_empty() && shift != 0) {
-      auto ivRemap = AffineApplyOp::create(bodyBuilder,
-          srcForOp.getLoc(),
+      auto ivRemap = AffineApplyOp::create(
+          bodyBuilder, srcForOp.getLoc(),
           bodyBuilder.getSingleDimShiftAffineMap(
               -static_cast<int64_t>(srcForOp.getStepAsInt() * shift)),
           loopChunkIV);
@@ -1557,7 +1557,7 @@ stripmineSink(AffineForOp forOp, uint64_t factor,
     // Insert newForOp before the terminator of `t`.
     auto b = OpBuilder::atBlockTerminator(t.getBody());
     auto newForOp = AffineForOp::create(b, t.getLoc(), lbOperands, lbMap,
-                                          ubOperands, ubMap, originalStep);
+                                        ubOperands, ubMap, originalStep);
     auto begin = t.getBody()->begin();
     // Skip terminator and `newForOp` which is just before the terminator.
     auto nOps = t.getBody()->getOperations().size() - 2;
@@ -1653,8 +1653,8 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
     operands.push_back(prev);
     operands.push_back(upperBound);
     // Maintain running product of loop upper bounds.
-    prev = AffineApplyOp::create(builder,
-        loc,
+    prev = AffineApplyOp::create(
+        builder, loc,
         AffineMap::get(/*dimCount=*/1,
                        /*symbolCount=*/1,
                        builder.getAffineDimExpr(0) *
@@ -1683,13 +1683,12 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> operands;
       operands.push_back(previous);
       operands.push_back(upperBoundSymbols[idx]);
-      previous = AffineApplyOp::create(builder,
-          loc,
-          AffineMap::get(
-              /*dimCount=*/1, /*symbolCount=*/1,
-              builder.getAffineDimExpr(0).floorDiv(
-                  builder.getAffineSymbolExpr(0))),
-          operands);
+      previous = AffineApplyOp::create(builder, loc,
+                                       AffineMap::get(
+                                           /*dimCount=*/1, /*symbolCount=*/1,
+                                           builder.getAffineDimExpr(0).floorDiv(
+                                               builder.getAffineSymbolExpr(0))),
+                                       operands);
     }
     // Modified value of the induction variables of the nested loops after
     // coalescing.
@@ -1700,8 +1699,8 @@ LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
       SmallVector<Value, 4> applyOperands;
       applyOperands.push_back(previous);
       applyOperands.push_back(upperBoundSymbols[idx - 1]);
-      inductionVariable = AffineApplyOp::create(builder,
-          loc,
+      inductionVariable = AffineApplyOp::create(
+          builder, loc,
           AffineMap::get(
               /*dimCount=*/1, /*symbolCount=*/1,
               builder.getAffineDimExpr(0) % builder.getAffineSymbolExpr(0)),
@@ -1738,16 +1737,16 @@ void mlir::affine::mapLoopToProcessorIds(scf::ForOp forOp,
 
   Value linearIndex = processorId.front();
   for (unsigned i = 1, e = processorId.size(); i < e; ++i) {
-    auto mulApplyOp = AffineApplyOp::create(b,
-        loc, mulMap, ValueRange{linearIndex, numProcessors[i]});
-    linearIndex = AffineApplyOp::create(b,
-        loc, addMap, ValueRange{mulApplyOp, processorId[i]});
+    auto mulApplyOp = AffineApplyOp::create(
+        b, loc, mulMap, ValueRange{linearIndex, numProcessors[i]});
+    linearIndex = AffineApplyOp::create(b, loc, addMap,
+                                        ValueRange{mulApplyOp, processorId[i]});
   }
 
-  auto mulApplyOp = AffineApplyOp::create(b,
-      loc, mulMap, ValueRange{linearIndex, forOp.getStep()});
-  Value lb = AffineApplyOp::create(b,
-      loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()});
+  auto mulApplyOp = AffineApplyOp::create(
+      b, loc, mulMap, ValueRange{linearIndex, forOp.getStep()});
+  Value lb = AffineApplyOp::create(
+      b, loc, addMap, ValueRange{mulApplyOp, forOp.getLowerBound()});
   forOp.setLowerBound(lb);
 
   Value step = forOp.getStep();
@@ -1918,7 +1917,7 @@ generatePointWiseCopy(Location loc, Value memref, Value fastMemRef,
     // Copy in.
     auto load = AffineLoadOp::create(b, loc, memref, memIndices);
     AffineStoreOp::create(b, loc, load, fastMemRef, fastBufMap,
-                            fastBufMapOperands);
+                          fastBufMapOperands);
     return copyNestRoot;
   }
 
@@ -2085,7 +2084,8 @@ static LogicalResult generateCopy(
       lbs[d] = lbs[d].replaceDimsAndSymbols(
           /*dimReplacements=*/{}, symReplacements, lbs[d].getNumSymbols(),
           /*numResultSyms=*/0);
-      memIndices.push_back(AffineApplyOp::create(b, loc, lbs[d], regionSymbols));
+      memIndices.push_back(
+          AffineApplyOp::create(b, loc, lbs[d], regionSymbols));
     }
     // The fast buffer is copied into at location zero; addressing is relative.
     bufIndices.push_back(zeroIndex);
@@ -2144,8 +2144,8 @@ static LogicalResult generateCopy(
     if (!dmaStrideInfos.empty()) {
       dmaStride =
           arith::ConstantIndexOp::create(top, loc, dmaStrideInfos[0].stride);
-      numEltPerDmaStride = arith::ConstantIndexOp::create(top,
-          loc, dmaStrideInfos[0].numEltPerStride);
+      numEltPerDmaStride = arith::ConstantIndexOp::create(
+          top, loc, dmaStrideInfos[0].numEltPerStride);
     }
   }
 
@@ -2183,13 +2183,13 @@ static LogicalResult generateCopy(
     if (!region.isWrite()) {
       // DMA non-blocking read from original buffer to fast buffer.
       AffineDmaStartOp::create(b, loc, memref, memAffineMap, memIndices,
-                                 fastMemRef, bufAffineMap, bufIndices,
-                                 tagMemRef, tagAffineMap, tagIndices,
-                                 numElementsSSA, dmaStride, numEltPerDmaStride);
+                               fastMemRef, bufAffineMap, bufIndices, tagMemRef,
+                               tagAffineMap, tagIndices, numElementsSSA,
+                               dmaStride, numEltPerDmaStride);
     } else {
       // DMA non-blocking write from fast buffer to the original memref.
-      auto op = AffineDmaStartOp::create(b,
-          loc, fastMemRef, bufAffineMap, bufIndices, memref, memAffineMap,
+      auto op = AffineDmaStartOp::create(
+          b, loc, fastMemRef, bufAffineMap, bufIndices, memref, memAffineMap,
           memIndices, tagMemRef, tagAffineMap, tagIndices, numElementsSSA,
           dmaStride, numEltPerDmaStride);
       // Since new ops may be appended at 'end' (for outgoing DMAs), adjust the
@@ -2200,7 +2200,7 @@ static LogicalResult generateCopy(
 
     // Matching DMA wait to block on completion; tag always has a 0 index.
     AffineDmaWaitOp::create(b, loc, tagMemRef, tagAffineMap, zeroIndex,
-                              numElementsSSA);
+                            numElementsSSA);
 
     // Generate dealloc for the tag.
     auto tagDeallocOp = memref::DeallocOp::create(epilogue, loc, tagMemRef);
@@ -2583,7 +2583,7 @@ AffineForOp mlir::affine::createCanonicalizedAffineForOp(
   ubMap = removeDuplicateExprs(ubMap);
 
   return AffineForOp::create(b, loc, lowerOperands, lbMap, upperOperands, ubMap,
-                               step);
+                             step);
 }
 
 /// Creates an AffineIfOp that encodes the conditional to choose between
@@ -2667,7 +2667,7 @@ static AffineIfOp createSeparationCondition(MutableArrayRef<AffineForOp> loops,
   cst.getValues(0, cst.getNumDimAndSymbolVars(), &setOperands);
   canonicalizeSetAndOperands(&ifCondSet, &setOperands);
   return AffineIfOp::create(b, loops[0].getLoc(), ifCondSet, setOperands,
-                              /*withElseRegion=*/true);
+                            /*withElseRegion=*/true);
 }
 
 /// Create the full tile loop nest (along with its body).
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 0b0950968ecba..048d197722d33 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -93,12 +93,12 @@ class AffineApplyExpander
 
     Value remainder = arith::RemSIOp::create(builder, loc, lhs, rhs);
     Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
-    Value isRemainderNegative = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::slt, remainder, zeroCst);
+    Value isRemainderNegative = arith::CmpIOp::create(
+        builder, loc, arith::CmpIPredicate::slt, remainder, zeroCst);
     Value correctedRemainder =
         arith::AddIOp::create(builder, loc, remainder, rhs);
-    Value result = arith::SelectOp::create(builder,
-        loc, isRemainderNegative, correctedRemainder, remainder);
+    Value result = arith::SelectOp::create(builder, loc, isRemainderNegative,
+                                           correctedRemainder, remainder);
     return result;
   }
 
@@ -132,16 +132,17 @@ class AffineApplyExpander
 
     Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
     Value noneCst = arith::ConstantIndexOp::create(builder, loc, -1);
-    Value negative = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::slt, lhs, zeroCst);
-    Value negatedDecremented = arith::SubIOp::create(builder, loc, noneCst, lhs);
-    Value dividend =
-        arith::SelectOp::create(builder, loc, negative, negatedDecremented, lhs);
+    Value negative = arith::CmpIOp::create(
+        builder, loc, arith::CmpIPredicate::slt, lhs, zeroCst);
+    Value negatedDecremented =
+        arith::SubIOp::create(builder, loc, noneCst, lhs);
+    Value dividend = arith::SelectOp::create(builder, loc, negative,
+                                             negatedDecremented, lhs);
     Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value correctedQuotient =
         arith::SubIOp::create(builder, loc, noneCst, quotient);
     Value result = arith::SelectOp::create(builder, loc, negative,
-                                                   correctedQuotient, quotient);
+                                           correctedQuotient, quotient);
     return result;
   }
 
@@ -171,19 +172,19 @@ class AffineApplyExpander
 
     Value zeroCst = arith::ConstantIndexOp::create(builder, loc, 0);
     Value oneCst = arith::ConstantIndexOp::create(builder, loc, 1);
-    Value nonPositive = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::sle, lhs, zeroCst);
+    Value nonPositive = arith::CmpIOp::create(
+        builder, loc, arith::CmpIPredicate::sle, lhs, zeroCst);
     Value negated = arith::SubIOp::create(builder, loc, zeroCst, lhs);
     Value decremented = arith::SubIOp::create(builder, loc, lhs, oneCst);
-    Value dividend =
-        arith::SelectOp::create(builder, loc, nonPositive, negated, decremented);
+    Value dividend = arith::SelectOp::create(builder, loc, nonPositive, negated,
+                                             decremented);
     Value quotient = arith::DivSIOp::create(builder, loc, dividend, rhs);
     Value negatedQuotient =
         arith::SubIOp::create(builder, loc, zeroCst, quotient);
     Value incrementedQuotient =
         arith::AddIOp::create(builder, loc, quotient, oneCst);
-    Value result = arith::SelectOp::create(builder,
-        loc, nonPositive, negatedQuotient, incrementedQuotient);
+    Value result = arith::SelectOp::create(
+        builder, loc, nonPositive, negatedQuotient, incrementedQuotient);
     return result;
   }
 
@@ -299,8 +300,8 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   IRMapping operandMap;
   OpBuilder b(hoistOverOp);
   auto hoistedIfOp = AffineIfOp::create(b, ifOp.getLoc(), ifOp.getIntegerSet(),
-                                          ifOp.getOperands(),
-                                          /*elseBlock=*/true);
+                                        ifOp.getOperands(),
+                                        /*elseBlock=*/true);
 
   // Create a clone of hoistOverOp to use for the else branch of the hoisted
   // conditional. The else block may get optimized away if empty.
@@ -369,8 +370,8 @@ mlir::affine::affineParallelize(AffineForOp forOp,
       parallelReductions, [](const LoopReduction &red) { return red.value; }));
   auto reductionKinds = llvm::to_vector<4>(llvm::map_range(
       parallelReductions, [](const LoopReduction &red) { return red.kind; }));
-  AffineParallelOp newPloop = AffineParallelOp::create(outsideBuilder,
-      loc, ValueRange(reducedValues).getTypes(), reductionKinds,
+  AffineParallelOp newPloop = AffineParallelOp::create(
+      outsideBuilder, loc, ValueRange(reducedValues).getTypes(), reductionKinds,
       llvm::ArrayRef(lowerBoundMap), lowerBoundOperands,
       llvm::ArrayRef(upperBoundMap), upperBoundOperands,
       llvm::ArrayRef(forOp.getStepAsInt()));
@@ -541,7 +542,8 @@ void mlir::affine::normalizeAffineParallel(AffineParallelOp op) {
     SmallVector<Value, 8> applyOperands{dimOperands};
     applyOperands.push_back(iv);
     applyOperands.append(symbolOperands.begin(), symbolOperands.end());
-    auto apply = AffineApplyOp::create(builder, op.getLoc(), map, applyOperands);
+    auto apply =
+        AffineApplyOp::create(builder, op.getLoc(), map, applyOperands);
     iv.replaceAllUsesExcept(apply, apply);
   }
 
@@ -622,8 +624,9 @@ LogicalResult mlir::affine::normalizeAffineFor(AffineForOp op,
   AffineValueMap newIvToOldIvMap;
   AffineValueMap::difference(lbMap, scaleIvValueMap, &newIvToOldIvMap);
   (void)newIvToOldIvMap.canonicalize();
-  auto newIV = AffineApplyOp::create(opBuilder,
-      loc, newIvToOldIvMap.getAffineMap(), newIvToOldIvMap.getOperands());
+  auto newIV =
+      AffineApplyOp::create(opBuilder, loc, newIvToOldIvMap.getAffineMap(),
+                            newIvToOldIvMap.getOperands());
   op.getInductionVar().replaceAllUsesExcept(newIV->getResult(0), newIV);
   return success();
 }
@@ -1188,7 +1191,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
       auto singleResMap = AffineMap::get(oldMap.getNumDims(),
                                          oldMap.getNumSymbols(), resultExpr);
       auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
-                                                oldMapOperands);
+                                        oldMapOperands);
       oldMemRefOperands.push_back(afOp);
       affineApplyOps.push_back(afOp);
     }
@@ -1215,7 +1218,7 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
       auto singleResMap = AffineMap::get(
           indexRemap.getNumDims(), indexRemap.getNumSymbols(), resultExpr);
       auto afOp = AffineApplyOp::create(builder, op->getLoc(), singleResMap,
-                                                remapOperands);
+                                        remapOperands);
       remapOutputs.push_back(afOp);
       affineApplyOps.push_back(afOp);
     }
@@ -1264,8 +1267,8 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     // AffineMapAccessInterface, we need to apply the values of `newMapOperands`
     // to the `newMap` to get the correct indices.
     for (unsigned i = 0; i < newMemRefRank; i++) {
-      state.operands.push_back(AffineApplyOp::create(builder,
-          op->getLoc(),
+      state.operands.push_back(AffineApplyOp::create(
+          builder, op->getLoc(),
           AffineMap::get(newMap.getNumDims(), newMap.getNumSymbols(),
                          newMap.getResult(i)),
           newMapOperands));
@@ -1450,8 +1453,8 @@ void mlir::affine::createAffineComputationSlice(
   for (auto resultExpr : composedMap.getResults()) {
     auto singleResMap = AffineMap::get(composedMap.getNumDims(),
                                        composedMap.getNumSymbols(), resultExpr);
-    sliceOps->push_back(AffineApplyOp::create(builder,
-        opInst->getLoc(), singleResMap, composedOpOperands));
+    sliceOps->push_back(AffineApplyOp::create(
+        builder, opInst->getLoc(), singleResMap, composedOpOperands));
   }
 
   // Construct the new operands that include the results from the composed
@@ -1740,12 +1743,11 @@ LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp allocOp) {
     createNewDynamicSizes(oldMemRefType, newMemRefType, layoutMap, allocOp, b,
                           newDynamicSizes);
     // Add the new dynamic sizes in new AllocOp.
-    newAlloc =
-        AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType, newDynamicSizes,
-                              allocOp.getAlignmentAttr());
+    newAlloc = AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType,
+                                   newDynamicSizes, allocOp.getAlignmentAttr());
   } else {
     newAlloc = AllocLikeOp::create(b, allocOp.getLoc(), newMemRefType,
-                                     allocOp.getAlignmentAttr());
+                                   allocOp.getAlignmentAttr());
   }
   // Replace all uses of the old memref.
   if (failed(replaceAllMemRefUsesWith(oldMemRef, /*newMemRef=*/newAlloc,
@@ -1804,7 +1806,7 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
     if (memrefType.isDynamicDim(i))
       mapOperands[i] =
           arith::SubIOp::create(b, loc, oldSizes[0].getType(), oldSizes[idx++],
-                                  arith::ConstantIndexOp::create(b, loc, 1));
+                                arith::ConstantIndexOp::create(b, loc, 1));
     else
       mapOperands[i] = arith::ConstantIndexOp::create(b, loc, oldShape[i] - 1);
   }
@@ -1816,8 +1818,8 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0; i < newRank; i++) {
     if (!newMemRefType.isDynamicDim(i))
       continue;
-    newSizes.push_back(AffineApplyOp::create(b,
-        loc,
+    newSizes.push_back(AffineApplyOp::create(
+        b, loc,
         AffineMap::get(oldLayoutMap.getNumDims(), oldLayoutMap.getNumSymbols(),
                        oldLayoutMap.getResult(i)),
         mapOperands));
@@ -1825,11 +1827,11 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
   for (unsigned i = 0, e = newSizes.size(); i < e; i++) {
     newSizes[i] =
         arith::AddIOp::create(b, loc, newSizes[i].getType(), newSizes[i],
-                                arith::ConstantIndexOp::create(b, loc, 1));
+                              arith::ConstantIndexOp::create(b, loc, 1));
   }
   // Create the new reinterpret_cast op.
-  auto newReinterpretCast = memref::ReinterpretCastOp::create(b,
-      loc, newMemRefType, reinterpretCastOp.getSource(),
+  auto newReinterpretCast = memref::ReinterpretCastOp::create(
+      b, loc, newMemRefType, reinterpretCastOp.getSource(),
       /*offsets=*/ValueRange(), newSizes,
       /*strides=*/ValueRange(),
       /*static_offsets=*/newStaticOffsets,
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index d21da23903544..9199dccdcaff3 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -67,8 +67,8 @@ struct SelectOpInterface
       return state.getMemrefWithUniqueOwnership(builder, value,
                                                 value.getParentBlock());
 
-    Value ownership = arith::SelectOp::create(builder,
-        op->getLoc(), selectOp.getCondition(),
+    Value ownership = arith::SelectOp::create(
+        builder, op->getLoc(), selectOp.getCondition(),
         state.getOwnership(selectOp.getTrueValue(), block).getIndicator(),
         state.getOwnership(selectOp.getFalseValue(), block).getIndicator());
     return {selectOp.getResult(), ownership};
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
index 44e896a97cb4b..efe6ad2579055 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
@@ -81,7 +81,7 @@ static Value extractLastDimSlice(ConversionPatternRewriter &rewriter,
   SmallVector<int64_t> strides(shape.size(), 1);
 
   return vector::ExtractStridedSliceOp::create(rewriter, loc, input, offsets,
-                                                        sizes, strides);
+                                               sizes, strides);
 }
 
 /// Extracts two vector slices from the `input` whose type is `vector<...x2T>`,
@@ -142,7 +142,7 @@ static Value insertLastDimSlice(ConversionPatternRewriter &rewriter,
   offsets.back() = lastOffset;
   SmallVector<int64_t> strides(shape.size(), 1);
   return vector::InsertStridedSliceOp::create(rewriter, loc, source, dest,
-                                                       offsets, strides);
+                                              offsets, strides);
 }
 
 /// Constructs a new vector of type `resultType` by creating a series of
@@ -362,8 +362,8 @@ struct ConvertCmpI final : OpConversionPattern<arith::CmpIOp> {
     }
     default: {
       // Handle inequality checks.
-      Value highEq = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::eq, lhsElem1, rhsElem1);
+      Value highEq = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::eq, lhsElem1, rhsElem1);
       cmpResult =
           arith::SelectOp::create(rewriter, loc, highEq, lowCmp, highCmp);
       break;
@@ -443,8 +443,8 @@ struct ConvertExtSI final : OpConversionPattern<arith::ExtSIOp> {
         loc, newResultComponentTy, newOperand);
     Value operandZeroCst =
         createScalarOrSplatConstant(rewriter, loc, newResultComponentTy, 0);
-    Value signBit = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::slt, extended, operandZeroCst);
+    Value signBit = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::slt, extended, operandZeroCst);
     Value signValue =
         arith::ExtSIOp::create(rewriter, loc, newResultComponentTy, signBit);
 
@@ -680,16 +680,16 @@ struct ConvertShLI final : OpConversionPattern<arith::ShLIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
+    Value illegalElemShift = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
         arith::ShLIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     Value resElem0 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
-                                                      zeroCst, shiftedElem0);
+                                             zeroCst, shiftedElem0);
 
-    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, elemBitWidth, rhsElem0);
+    Value cappedShiftAmount = arith::SelectOp::create(
+        rewriter, loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value rightShiftAmount =
         arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedRight =
@@ -701,10 +701,10 @@ struct ConvertShLI final : OpConversionPattern<arith::ShLIOp> {
 
     Value shiftedElem1 =
         arith::ShLIOp::create(rewriter, loc, lhsElem1, rhsElem0);
-    Value resElem1High = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, zeroCst, shiftedElem1);
-    Value resElem1Low = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, shiftedLeft, shiftedRight);
+    Value resElem1High = arith::SelectOp::create(
+        rewriter, loc, illegalElemShift, zeroCst, shiftedElem1);
+    Value resElem1Low = arith::SelectOp::create(rewriter, loc, illegalElemShift,
+                                                shiftedLeft, shiftedRight);
     Value resElem1 =
         arith::OrIOp::create(rewriter, loc, resElem1Low, resElem1High);
 
@@ -769,20 +769,20 @@ struct ConvertShRUI final : OpConversionPattern<arith::ShRUIOp> {
     Value elemBitWidth =
         createScalarOrSplatConstant(rewriter, loc, newOperandTy, newBitWidth);
 
-    Value illegalElemShift = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
+    Value illegalElemShift = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::uge, rhsElem0, elemBitWidth);
 
     Value shiftedElem0 =
         arith::ShRUIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     Value resElem0Low = arith::SelectOp::create(rewriter, loc, illegalElemShift,
-                                                         zeroCst, shiftedElem0);
+                                                zeroCst, shiftedElem0);
     Value shiftedElem1 =
         arith::ShRUIOp::create(rewriter, loc, lhsElem1, rhsElem0);
     Value resElem1 = arith::SelectOp::create(rewriter, loc, illegalElemShift,
-                                                      zeroCst, shiftedElem1);
+                                             zeroCst, shiftedElem1);
 
-    Value cappedShiftAmount = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, elemBitWidth, rhsElem0);
+    Value cappedShiftAmount = arith::SelectOp::create(
+        rewriter, loc, illegalElemShift, elemBitWidth, rhsElem0);
     Value leftShiftAmount =
         arith::SubIOp::create(rewriter, loc, elemBitWidth, cappedShiftAmount);
     Value shiftedLeft =
@@ -792,8 +792,8 @@ struct ConvertShRUI final : OpConversionPattern<arith::ShRUIOp> {
     Value shiftedRight =
         arith::ShRUIOp::create(rewriter, loc, lhsElem1, overshotShiftAmount);
 
-    Value resElem0High = arith::SelectOp::create(rewriter,
-        loc, illegalElemShift, shiftedRight, shiftedLeft);
+    Value resElem0High = arith::SelectOp::create(
+        rewriter, loc, illegalElemShift, shiftedRight, shiftedLeft);
     Value resElem0 =
         arith::OrIOp::create(rewriter, loc, resElem0Low, resElem0High);
 
@@ -832,8 +832,8 @@ struct ConvertShRSI final : OpConversionPattern<arith::ShRSIOp> {
     // Perform as many ops over the narrow integer type as possible and let the
     // other emulation patterns convert the rest.
     Value elemZero = createScalarOrSplatConstant(rewriter, loc, narrowTy, 0);
-    Value signBit = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::slt, lhsElem1, elemZero);
+    Value signBit = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::slt, lhsElem1, elemZero);
     signBit = dropTrailingX1Dim(rewriter, loc, signBit);
 
     // Create a bit pattern of either all ones or all zeros. Then shift it left
@@ -857,8 +857,8 @@ struct ConvertShRSI final : OpConversionPattern<arith::ShRSIOp> {
 
     // Handle shifting by zero. This is necessary when the `signBits` shift is
     // invalid.
-    Value isNoop = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                                  rhsElem0, elemZero);
+    Value isNoop = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, rhsElem0, elemZero);
     isNoop = dropTrailingX1Dim(rewriter, loc, isNoop);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNoop, op.getLhs(),
                                                  shrsi);
@@ -894,8 +894,8 @@ struct ConvertSubI final : OpConversionPattern<arith::SubIOp> {
     // CARRY is 1 or 0.
     Value low = arith::SubIOp::create(rewriter, loc, lhsElem0, rhsElem0);
     // We have a carry if lhsElem0 < rhsElem0.
-    Value carry0 = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ult, lhsElem0, rhsElem0);
+    Value carry0 = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ult, lhsElem0, rhsElem0);
     Value carryVal = arith::ExtUIOp::create(rewriter, loc, newElemTy, carry0);
 
     Value high0 = arith::SubIOp::create(rewriter, loc, lhsElem1, carryVal);
@@ -933,8 +933,8 @@ struct ConvertSIToFP final : OpConversionPattern<arith::SIToFPOp> {
     // result or not based on that sign bit. We implement negation by
     // subtracting from zero. Note that this relies on the the other conversion
     // patterns to legalize created ops and narrow the bit widths.
-    Value isNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
-                                                 in, zeroCst);
+    Value isNeg = arith::CmpIOp::create(rewriter, loc,
+                                        arith::CmpIPredicate::slt, in, zeroCst);
     Value neg = arith::SubIOp::create(rewriter, loc, zeroCst, in);
     Value abs = arith::SelectOp::create(rewriter, loc, isNeg, neg, in);
 
@@ -985,8 +985,8 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     //
     // Note 2: We do not strictly need the `hi == 0`, case, but it makes
     // constant folding easier.
-    Value hiEqZero = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::eq, hiInt, zeroCst);
+    Value hiEqZero = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, hiInt, zeroCst);
 
     Type resultTy = op.getType();
     Type resultElemTy = getElementTypeOrSelf(resultTy);
@@ -999,7 +999,8 @@ struct ConvertUIToFP final : OpConversionPattern<arith::UIToFPOp> {
     if (auto vecTy = dyn_cast<VectorType>(resultTy))
       pow2Attr = SplatElementsAttr::get(vecTy, pow2Attr);
 
-    Value pow2Val = arith::ConstantOp::create(rewriter, loc, resultTy, pow2Attr);
+    Value pow2Val =
+        arith::ConstantOp::create(rewriter, loc, resultTy, pow2Attr);
 
     Value hiVal = arith::MulFOp::create(rewriter, loc, hiFp, pow2Val);
     Value result = arith::AddFOp::create(rewriter, loc, lowFp, hiVal);
@@ -1042,8 +1043,8 @@ struct ConvertFPToSI final : OpConversionPattern<arith::FPToSIOp> {
 
     // Get the absolute value. One could have used math.absf here, but that
     // introduces an extra dependency.
-    Value isNeg = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::OLT,
-                                                 inFp, zeroCst);
+    Value isNeg = arith::CmpFOp::create(
+        rewriter, loc, arith::CmpFPredicate::OLT, inFp, zeroCst);
     Value negInFp = arith::NegFOp::create(rewriter, loc, inFp);
 
     Value absVal = arith::SelectOp::create(rewriter, loc, isNeg, negInFp, inFp);
diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
index dcdb0d1687504..f304354e6f45d 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
@@ -29,8 +29,8 @@ static Value createConst(Location loc, Type type, int value,
                          PatternRewriter &rewriter) {
   auto attr = rewriter.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(shapedTy, attr));
+    return arith::ConstantOp::create(rewriter, loc,
+                                     DenseElementsAttr::get(shapedTy, attr));
   }
   return arith::ConstantOp::create(rewriter, loc, attr);
 }
@@ -40,8 +40,8 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
                               PatternRewriter &rewriter) {
   auto attr = rewriter.getFloatAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
-    return arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(shapedTy, attr));
+    return arith::ConstantOp::create(rewriter, loc,
+                                     DenseElementsAttr::get(shapedTy, attr));
   }
 
   return arith::ConstantOp::create(rewriter, loc, attr);
@@ -99,16 +99,16 @@ struct CeilDivSIOpConverter : public OpRewritePattern<arith::CeilDivSIOp> {
 
     Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
     Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
-    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ne, a, product);
+    Value notEqualDivisor = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ne, a, product);
 
-    Value aNeg =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
-    Value bNeg =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
+    Value aNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                                       a, zero);
+    Value bNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                                       b, zero);
 
-    Value signEqual = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::eq, aNeg, bNeg);
+    Value signEqual = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, aNeg, bNeg);
     Value cond =
         arith::AndIOp::create(rewriter, loc, notEqualDivisor, signEqual);
 
@@ -138,17 +138,17 @@ struct FloorDivSIOpConverter : public OpRewritePattern<arith::FloorDivSIOp> {
 
     Value quotient = arith::DivSIOp::create(rewriter, loc, a, b);
     Value product = arith::MulIOp::create(rewriter, loc, quotient, b);
-    Value notEqualDivisor = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ne, a, product);
+    Value notEqualDivisor = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ne, a, product);
     Value zero = createConst(loc, type, 0, rewriter);
 
-    Value aNeg =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, a, zero);
-    Value bNeg =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, b, zero);
+    Value aNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                                       a, zero);
+    Value bNeg = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                                       b, zero);
 
-    Value signOpposite = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::ne, aNeg, bNeg);
+    Value signOpposite = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::ne, aNeg, bNeg);
     Value cond =
         arith::AndIOp::create(rewriter, loc, notEqualDivisor, signOpposite);
 
@@ -197,8 +197,8 @@ struct MaximumMinimumFOpConverter : public OpRewritePattern<OpTy> {
     Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'.
-    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
-                                                 rhs, rhs);
+    Value isNaN = arith::CmpFOp::create(rewriter, loc,
+                                        arith::CmpFPredicate::UNO, rhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
   }
@@ -223,8 +223,8 @@ struct MaxNumMinNumFOpConverter : public OpRewritePattern<OpTy> {
     Value select = arith::SelectOp::create(rewriter, loc, cmp, lhs, rhs);
 
     // Handle the case where lhs is NaN: 'isNaN(lhs) ? rhs : select'.
-    Value isNaN = arith::CmpFOp::create(rewriter, loc, arith::CmpFPredicate::UNO,
-                                                 lhs, lhs);
+    Value isNaN = arith::CmpFOp::create(rewriter, loc,
+                                        arith::CmpFPredicate::UNO, lhs, lhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
   }
@@ -415,7 +415,8 @@ struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Value c0x8 = createConst(loc, i4Ty, 0x8, rewriter);
     Value negative =
         arith::CmpIOp::create(b, arith::CmpIPredicate::uge, i4Bits, c0x8);
-    Value bit32 = arith::SelectOp::create(b, negative, c0x80000000, zeroExpBits);
+    Value bit32 =
+        arith::SelectOp::create(b, negative, c0x80000000, zeroExpBits);
 
     // Add segments together.
     Value bits1To31 = arith::AddIOp::create(b, bits1To24, bits25To31);
@@ -464,7 +465,7 @@ struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     Value result = arith::BitcastOp::create(b, f32Ty, f32Bits);
     if (resultETy.getIntOrFloatBitWidth() < 32) {
       result = arith::TruncFOp::create(b, resultTy, result, nullptr,
-                                         op.getFastmathAttr());
+                                       op.getFastmathAttr());
     } else if (resultETy.getIntOrFloatBitWidth() > 32) {
       result = arith::ExtFOp::create(b, resultTy, result, op.getFastmathAttr());
     }
@@ -570,7 +571,7 @@ struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
         arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0xff);
     Value man23Bits = arith::AndIOp::create(b, f32Bits, cF32MantissaMask);
     Value isNonZeroMan = arith::CmpIOp::create(b, arith::CmpIPredicate::ugt,
-                                                 man23Bits, zeroExpBits);
+                                               man23Bits, zeroExpBits);
     Value roundToHalf = arith::AndIOp::create(b, isNegOneExp, isNonZeroMan);
     Value isZeroExp =
         arith::CmpIOp::create(b, arith::CmpIPredicate::eq, f8Exp, c0x00);
@@ -628,8 +629,8 @@ struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     if (operandETy.getIntOrFloatBitWidth() < 32) {
       operand = arith::ExtFOp::create(b, f32Ty, operand, op.getFastmathAttr());
     } else if (operandETy.getIntOrFloatBitWidth() > 32) {
-      operand = arith::TruncFOp::create(b,
-          f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
+      operand = arith::TruncFOp::create(
+          b, f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
     }
     Value f32Bits = arith::BitcastOp::create(b, i32Ty, operand);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
@@ -655,7 +656,7 @@ struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
       scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
-                                               op.getFastmathAttr());
+                                             op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
       return rewriter.notifyMatchFailure(
@@ -696,7 +697,7 @@ struct ScalingTruncFOpConverter
       scaleETy = b.getF8E8M0Type();
       scaleTy = cloneToShapedType(scaleTy, scaleETy);
       scaleOperand = arith::TruncFOp::create(b, scaleTy, scaleOperand, nullptr,
-                                               op.getFastmathAttr());
+                                             op.getFastmathAttr());
     }
     if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
       return rewriter.notifyMatchFailure(
@@ -710,9 +711,9 @@ struct ScalingTruncFOpConverter
     scaleOperand =
         arith::ExtFOp::create(b, inputTy, scaleOperand, op.getFastmathAttr());
     Value result = arith::DivFOp::create(b, inputOperand, scaleOperand,
-                                           op.getFastmathAttr());
-    Value resultCast = arith::TruncFOp::create(b,
-        resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
+                                         op.getFastmathAttr());
+    Value resultCast = arith::TruncFOp::create(
+        b, resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
     rewriter.replaceOp(op, resultCast);
     return success();
   }
diff --git a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
index 904aab33345f0..4bdd1e6a54d69 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
@@ -24,7 +24,7 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
     switch (e.getKind()) {
     case AffineExprKind::Constant:
       return ConstantIndexOp::create(b, loc,
-                                       cast<AffineConstantExpr>(e).getValue());
+                                     cast<AffineConstantExpr>(e).getValue());
     case AffineExprKind::DimId:
       return operands[cast<AffineDimExpr>(e).getPosition()];
     case AffineExprKind::SymbolId:
@@ -33,27 +33,27 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
     case AffineExprKind::Add: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return AddIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                              buildExpr(binaryExpr.getRHS()));
+                            buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mul: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return MulIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                              buildExpr(binaryExpr.getRHS()));
+                            buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::FloorDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return DivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                               buildExpr(binaryExpr.getRHS()));
+                             buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::CeilDiv: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return CeilDivSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                                   buildExpr(binaryExpr.getRHS()));
+                                 buildExpr(binaryExpr.getRHS()));
     }
     case AffineExprKind::Mod: {
       auto binaryExpr = cast<AffineBinaryOpExpr>(e);
       return RemSIOp::create(b, loc, buildExpr(binaryExpr.getLHS()),
-                               buildExpr(binaryExpr.getRHS()));
+                             buildExpr(binaryExpr.getRHS()));
     }
     }
     llvm_unreachable("unsupported AffineExpr kind");
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index 6e9cc14c6ada5..208e8f0d3bca3 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -105,8 +105,8 @@ Value mlir::getValueOrCreateConstantIntOp(OpBuilder &b, Location loc,
   if (auto value = dyn_cast_if_present<Value>(ofr))
     return value;
   auto attr = cast<IntegerAttr>(cast<Attribute>(ofr));
-  return arith::ConstantOp::create(b,
-      loc, b.getIntegerAttr(attr.getType(), attr.getValue().getSExtValue()));
+  return arith::ConstantOp::create(
+      b, loc, b.getIntegerAttr(attr.getType(), attr.getValue().getSExtValue()));
 }
 
 Value mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc,
@@ -215,8 +215,8 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
     if (from.getType().getIntOrFloatBitWidth() > toBitwidth) {
       from = arith::TruncFOp::create(b, toFpTy, from);
     }
-    Value zero = mlir::arith::ConstantFloatOp::create(b,
-        toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
+    Value zero = mlir::arith::ConstantFloatOp::create(
+        b, toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
     return complex::CreateOp::create(b, targetType, from, zero);
   }
 
@@ -228,8 +228,8 @@ static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
     } else {
       from = arith::SIToFPOp::create(b, toFpTy, from);
     }
-    Value zero = mlir::arith::ConstantFloatOp::create(b,
-        toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
+    Value zero = mlir::arith::ConstantFloatOp::create(
+        b, toFpTy, mlir::APFloat(toFpTy.getFloatSemantics(), 0));
     return complex::CreateOp::create(b, targetType, from, zero);
   }
 
@@ -350,7 +350,7 @@ Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values) {
 Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
                     Type resultType) {
   Value one = ConstantOp::create(builder, loc, resultType,
-                                         builder.getOneAttr(resultType));
+                                 builder.getOneAttr(resultType));
   ArithBuilder arithBuilder(builder, loc);
   return std::accumulate(
       values.begin(), values.end(), one,
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
index b9a81b71fa301..2b80441b33309 100644
--- a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
@@ -229,8 +229,9 @@ class LowerContractionToNeonI8MMPattern
 
     // Initial accumulator for the final result. This is the un-tiled result if
     // tiling is done.
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, op.getResultType(), rewriter.getZeroAttr(op.getResultType()));
+    Value result =
+        arith::ConstantOp::create(rewriter, loc, op.getResultType(),
+                                  rewriter.getZeroAttr(op.getResultType()));
 
     SmallVector<int64_t> unrolledSize = *op.getShapeForUnroll();
     SmallVector<int64_t> smmlaShape = {2, 8};
@@ -281,8 +282,9 @@ class LowerContractionToNeonI8MMPattern
       if (isVecmat) {
         auto expandForSMMLA = [&](Value tiledOperand,
                                   VectorType expandedTypeType) {
-          auto emptyOperand = arith::ConstantOp::create(rewriter,
-              loc, expandedTypeType, rewriter.getZeroAttr(expandedTypeType));
+          auto emptyOperand =
+              arith::ConstantOp::create(rewriter, loc, expandedTypeType,
+                                        rewriter.getZeroAttr(expandedTypeType));
           SmallVector<int64_t> offsets(
               cast<ShapedType>(emptyOperand.getType()).getRank(), 0);
           SmallVector<int64_t> strides(
@@ -298,8 +300,8 @@ class LowerContractionToNeonI8MMPattern
       // using the instruction for unsigned by signed multiplication with
       // reversed operands.
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledAcc = vector::TransposeOp::create(rewriter,
-            loc, tiledAcc, ArrayRef<int64_t>({1, 0}));
+        tiledAcc = vector::TransposeOp::create(rewriter, loc, tiledAcc,
+                                               ArrayRef<int64_t>({1, 0}));
 
       // Collapse tiled operands to 1D vectors required by smmla intrinsic
       auto collapsedInputType =
@@ -331,8 +333,8 @@ class LowerContractionToNeonI8MMPattern
       // Because of the reversed operands the result is obtained transposed.
       // Transpose it back,
       if (mmlaOp == MMLA::MixedSwapped)
-        tiledRes = vector::TransposeOp::create(rewriter,
-            loc, tiledRes, ArrayRef<int64_t>({1, 0}));
+        tiledRes = vector::TransposeOp::create(rewriter, loc, tiledRes,
+                                               ArrayRef<int64_t>({1, 0}));
 
       // With vecmat, only one row of tiled ACC can be inserted into the final
       // result
diff --git a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
index 12018912f6f66..dcac0ab698992 100644
--- a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
@@ -12,8 +12,8 @@
 
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
index 23ec00196f714..e5e1312f0eb04 100644
--- a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
@@ -76,15 +76,15 @@ scf::ForOp createLoopOverTileSlices(
     std::function<Value(OpBuilder &, Location, Value, Value)> makeLoopBody) {
   OpBuilder::InsertionGuard g(rewriter);
   auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-  auto minTileSlices = arith::ConstantIndexOp::create(rewriter,
-      loc, llvm::cast<VectorType>(initTile.getType()).getDimSize(0));
+  auto minTileSlices = arith::ConstantIndexOp::create(
+      rewriter, loc, llvm::cast<VectorType>(initTile.getType()).getDimSize(0));
   auto vscale =
       vector::VectorScaleOp::create(rewriter, loc, rewriter.getIndexType());
   auto lowerBound = arith::ConstantIndexOp::create(rewriter, loc, 0);
   auto numTileSlices =
       arith::MulIOp::create(rewriter, loc, minTileSlices, vscale);
-  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices, step,
-                                           ValueRange{initTile});
+  auto forOp = scf::ForOp::create(rewriter, loc, lowerBound, numTileSlices,
+                                  step, ValueRange{initTile});
   rewriter.setInsertionPointToStart(forOp.getBody());
   Value nextTile =
       makeLoopBody(rewriter, loc, /*tileSliceIndex=*/forOp.getInductionVar(),
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
index 4d03bbdabdf09..9bf026563c255 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/OuterProductFusion.cpp
@@ -456,8 +456,8 @@ struct SwapVectorExtractOfArithExtend
     Value extendSource = extendOp->getOperand(0);
 
     // Create new extract from source of extend.
-    Value newExtract = vector::ExtractOp::create(rewriter,
-        loc, extendSource, extractOp.getMixedPosition());
+    Value newExtract = vector::ExtractOp::create(rewriter, loc, extendSource,
+                                                 extractOp.getMixedPosition());
 
     // Extend new extract to original result type.
     Operation *newExtend =
@@ -503,8 +503,9 @@ struct SwapVectorScalableExtractOfArithExtend
     // Create new extract from source of extend.
     VectorType extractResultVectorType =
         resultType.clone(extendSourceVectorType.getElementType());
-    Value newExtract = vector::ScalableExtractOp::create(rewriter,
-        loc, extractResultVectorType, extendSource, extractOp.getPos());
+    Value newExtract = vector::ScalableExtractOp::create(
+        rewriter, loc, extractResultVectorType, extendSource,
+        extractOp.getPos());
 
     // Extend new extract to original result type.
     Operation *newExtend =
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 3362d0ce13533..1c0eced43dc00 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -86,8 +86,9 @@ SmallVector<Value, 2> addConstantScalableOffset(OpBuilder &builder,
   return llvm::map_to_vector(
       llvm::zip_equal(indices, scalableOffsets), [&](auto pair) -> Value {
         auto [index, base] = pair;
-        auto offset = arith::MulIOp::create(builder,
-            loc, arith::ConstantIndexOp::create(builder, loc, base), vscale);
+        auto offset = arith::MulIOp::create(
+            builder, loc, arith::ConstantIndexOp::create(builder, loc, base),
+            vscale);
         return arith::AddIOp::create(builder, loc, index, offset);
       });
 }
@@ -132,8 +133,8 @@ Value extractSMEMask(OpBuilder &builder, Location loc, Value mask,
   // from the mask operands to get the parameters for this sub-tile.
   auto smeTileMaskDims = addConstantScalableOffset(
       builder, loc, createMask.getOperands(), {-smeTile.row, -smeTile.col});
-  auto smeTileCreateMask = vector::CreateMaskOp::create(builder,
-      loc, smeTile.type.clone(builder.getI1Type()), smeTileMaskDims);
+  auto smeTileCreateMask = vector::CreateMaskOp::create(
+      builder, loc, smeTile.type.clone(builder.getI1Type()), smeTileMaskDims);
   return smeTileCreateMask.getResult();
 }
 
@@ -190,8 +191,8 @@ struct LegalizeArithConstantOpsByDecomposition
 
     auto smeTileType = getSMETileTypeForElement(vectorType.getElementType());
     auto tileCount = getNumberOfSMETilesForVectorType(vectorType);
-    auto tileSplat = arith::ConstantOp::create(rewriter,
-        constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
+    auto tileSplat = arith::ConstantOp::create(
+        rewriter, constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
     SmallVector<Value> repl(tileCount, tileSplat);
     rewriter.replaceOpWithMultiple(constantOp, {repl});
 
@@ -237,12 +238,12 @@ struct LegalizeVectorOuterProductOpsByDecomposition
              decomposeToSMETiles(rewriter, vectorType, smeTileType))) {
 
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto lhs = vector::ScalableExtractOp::create(rewriter,
-          loc, sliceType, outerProductOp.getLhs(), smeTile.row);
-      auto rhs = vector::ScalableExtractOp::create(rewriter,
-          loc, sliceType, outerProductOp.getRhs(), smeTile.col);
-      auto smeOuterProduct = vector::OuterProductOp::create(rewriter,
-          loc, smeTileType, lhs, rhs,
+      auto lhs = vector::ScalableExtractOp::create(
+          rewriter, loc, sliceType, outerProductOp.getLhs(), smeTile.row);
+      auto rhs = vector::ScalableExtractOp::create(
+          rewriter, loc, sliceType, outerProductOp.getRhs(), smeTile.col);
+      auto smeOuterProduct = vector::OuterProductOp::create(
+          rewriter, loc, smeTileType, lhs, rhs,
           !accSMETiles.empty() ? accSMETiles[index] : Value{},
           outerProductOp.getKind());
 
@@ -314,8 +315,8 @@ struct LegalizeTransferReadOpsByDecomposition
     for (SMESubTile smeTile :
          decomposeToSMETiles(rewriter, vectorType, smeTileType, transposed)) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeRead = vector::TransferReadOp::create(rewriter,
-          loc, smeTileType, readOp.getBase(),
+      auto smeRead = vector::TransferReadOp::create(
+          rewriter, loc, smeTileType, readOp.getBase(),
           getSMESubTileIndices(rewriter, loc, readOp.getIndices(), smeTile),
           readOp.getPermutationMapAttr(), readOp.getPadding(), smeMask,
           readOp.getInBoundsAttr());
@@ -363,8 +364,8 @@ struct LegalizeTransferWriteOpsByDecomposition
     for (auto [index, smeTile] : llvm::enumerate(decomposeToSMETiles(
              rewriter, vectorType, smeTileType, transposed))) {
       auto smeMask = extractSMEMask(rewriter, loc, mask, smeTile);
-      auto smeWrite = vector::TransferWriteOp::create(rewriter,
-          loc, inputSMETiles[index], destTensorOrMemref,
+      auto smeWrite = vector::TransferWriteOp::create(
+          rewriter, loc, inputSMETiles[index], destTensorOrMemref,
           getSMESubTileIndices(rewriter, loc, writeOp.getIndices(), smeTile),
           writeOp.getPermutationMapAttr(), smeMask, writeOp.getInBoundsAttr());
       if (writeOp.hasPureTensorSemantics())
@@ -478,26 +479,27 @@ struct LegalizeMultiTileTransferWriteAsStoreLoop
 
       // Where in the destination memref the current slice will be stored.
       auto storeRow = arith::AddIOp::create(rewriter, loc, sliceIndex,
-                                                     writeOp.getIndices()[0]);
-      auto storeCol =
-          arith::AddIOp::create(rewriter, loc, tileCol, writeOp.getIndices()[1]);
+                                            writeOp.getIndices()[0]);
+      auto storeCol = arith::AddIOp::create(rewriter, loc, tileCol,
+                                            writeOp.getIndices()[1]);
 
       // Extract the mask for the current slice.
       Value sliceMask = nullptr;
       if (mask) {
-        sliceMask = vector::ExtractOp::create(rewriter,
-            loc, mask, OpFoldResult(sliceIndex));
+        sliceMask = vector::ExtractOp::create(rewriter, loc, mask,
+                                              OpFoldResult(sliceIndex));
         if (sliceMaskType != sliceMask.getType())
-          sliceMask = vector::ScalableExtractOp::create(rewriter,
-              loc, sliceMaskType, sliceMask, smeTile.col);
+          sliceMask = vector::ScalableExtractOp::create(
+              rewriter, loc, sliceMaskType, sliceMask, smeTile.col);
       }
 
       // Extract and store the current slice.
       Value tile = inputSMETiles[index];
       auto slice =
           vector::ExtractOp::create(rewriter, loc, tile, tileSliceIndex);
-      vector::TransferWriteOp::create(rewriter,
-          loc, slice, writeOp.getBase(), ValueRange{storeRow, storeCol},
+      vector::TransferWriteOp::create(
+          rewriter, loc, slice, writeOp.getBase(),
+          ValueRange{storeRow, storeCol},
           AffineMapAttr::get(writeOp.getPermutationMap().dropResult(0)),
           sliceMask,
           rewriter.getBoolArrayAttr(
@@ -570,11 +572,12 @@ struct FoldExtractFromVectorOfSMELikeCreateMasks
     auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
     auto extractionIndex = getValueOrCreateConstantIndexOp(
         rewriter, loc, extractOp.getMixedPosition()[0]);
-    auto extractionInTrueRegion = arith::CmpIOp::create(rewriter,
-        loc, rewriter.getI1Type(), arith::CmpIPredicate::slt, extractionIndex,
-        frontMaskDim);
-    auto newMaskFrontDim = arith::SelectOp::create(rewriter,
-        loc, extractionInTrueRegion, createMaskOp.getOperand(1), zero);
+    auto extractionInTrueRegion = arith::CmpIOp::create(
+        rewriter, loc, rewriter.getI1Type(), arith::CmpIPredicate::slt,
+        extractionIndex, frontMaskDim);
+    auto newMaskFrontDim =
+        arith::SelectOp::create(rewriter, loc, extractionInTrueRegion,
+                                createMaskOp.getOperand(1), zero);
 
     rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(
         extractOp, extractedMaskType,
@@ -676,9 +679,9 @@ struct LiftIllegalVectorTransposeToMemory
           return arith::MulIOp::create(rewriter, loc, vscale, dimSize);
         });
     SmallVector<Value> strides(readType.getRank(), Value(one));
-    auto readSubview = memref::SubViewOp::create(rewriter,
-        loc, illegalRead.getBase(), illegalRead.getIndices(), readSizes,
-        strides);
+    auto readSubview =
+        memref::SubViewOp::create(rewriter, loc, illegalRead.getBase(),
+                                  illegalRead.getIndices(), readSizes, strides);
 
     // Apply the transpose to all values/attributes of the transfer_read:
     // - The mask
@@ -687,13 +690,13 @@ struct LiftIllegalVectorTransposeToMemory
       // Note: The transpose for the mask should fold into the
       // vector.create_mask/constant_mask op, which will then become legal.
       mask = vector::TransposeOp::create(rewriter, loc, mask,
-                                                  transposeOp.getPermutation());
+                                         transposeOp.getPermutation());
     }
     // - The source memref
     mlir::AffineMap transposeMap = AffineMap::getPermutationMap(
         transposeOp.getPermutation(), getContext());
-    auto transposedSubview = memref::TransposeOp::create(rewriter,
-        loc, readSubview, AffineMapAttr::get(transposeMap));
+    auto transposedSubview = memref::TransposeOp::create(
+        rewriter, loc, readSubview, AffineMapAttr::get(transposeMap));
     ArrayAttr inBoundsAttr = illegalRead.getInBoundsAttr();
     // - The `in_bounds` attribute
     if (inBoundsAttr) {
@@ -706,8 +709,8 @@ struct LiftIllegalVectorTransposeToMemory
     VectorType legalReadType = resultType.clone(readType.getElementType());
     // Note: The indices are all zero as the subview is already offset.
     SmallVector<Value> readIndices(illegalRead.getIndices().size(), zero);
-    auto legalRead = vector::TransferReadOp::create(rewriter,
-        loc, legalReadType, transposedSubview, readIndices,
+    auto legalRead = vector::TransferReadOp::create(
+        rewriter, loc, legalReadType, transposedSubview, readIndices,
         illegalRead.getPermutationMapAttr(), illegalRead.getPadding(), mask,
         inBoundsAttr);
 
@@ -811,12 +814,12 @@ struct LowerIllegalTransposeStoreViaZA
       // rows of the tile after 1*vscale rows.
       Value tile = undefTile;
       for (int d = 0; d < numSlicesPerTile; ++d) {
-        Value vector = vector::ExtractOp::create(rewriter,
-            loc, transposeOp.getVector(),
-            rewriter.getIndexAttr(d + smeTile.row));
+        Value vector =
+            vector::ExtractOp::create(rewriter, loc, transposeOp.getVector(),
+                                      rewriter.getIndexAttr(d + smeTile.row));
         if (vector.getType() != smeSliceType) {
-          vector = vector::ScalableExtractOp::create(rewriter,
-              loc, smeSliceType, vector, smeTile.col);
+          vector = vector::ScalableExtractOp::create(
+              rewriter, loc, smeSliceType, vector, smeTile.col);
         }
         tile = vector::InsertOp::create(rewriter, loc, vector, tile, d);
       }
@@ -831,17 +834,17 @@ struct LowerIllegalTransposeStoreViaZA
       Value maskCols;
       if (auto mask = writeOp.getMask()) {
         auto createMask = mask.getDefiningOp<vector::CreateMaskOp>();
-        maskRows = arith::SubIOp::create(rewriter, loc, createMask.getOperand(0),
-                                                  transposedRow);
-        maskCols = arith::SubIOp::create(rewriter, loc, createMask.getOperand(1),
-                                                  transposedCol);
+        maskRows = arith::SubIOp::create(
+            rewriter, loc, createMask.getOperand(0), transposedRow);
+        maskCols = arith::SubIOp::create(
+            rewriter, loc, createMask.getOperand(1), transposedCol);
         maskCols = index::MinSOp::create(rewriter, loc, maskCols, numSlices);
       } else {
         maskRows = createVscaleMultiple(smeTileType.getDimSize(0));
         maskCols = numSlices;
       }
-      auto subMask = vector::CreateMaskOp::create(rewriter,
-          loc, smeTileType.clone(rewriter.getI1Type()),
+      auto subMask = vector::CreateMaskOp::create(
+          rewriter, loc, smeTileType.clone(rewriter.getI1Type()),
           ValueRange{maskRows, maskCols});
 
       // 4. Emit a transposed tile write.
@@ -850,8 +853,8 @@ struct LowerIllegalTransposeStoreViaZA
           arith::AddIOp::create(rewriter, loc, transposedRow, writeIndices[0]);
       Value destCol =
           arith::AddIOp::create(rewriter, loc, transposedCol, writeIndices[1]);
-      auto smeWrite = vector::TransferWriteOp::create(rewriter,
-          loc, tile, destTensorOrMemref, ValueRange{destRow, destCol},
+      auto smeWrite = vector::TransferWriteOp::create(
+          rewriter, loc, tile, destTensorOrMemref, ValueRange{destRow, destCol},
           transposeMap, subMask, writeOp.getInBounds());
 
       if (writeOp.hasPureTensorSemantics())
@@ -939,27 +942,27 @@ struct LowerColumnTransferReadToLoops
         vector::makeVscaleConstantBuilder(rewriter, loc);
     auto upperBound = createVscaleMultiple(numRows);
     auto step = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    Value init = arith::ConstantOp::create(rewriter,
-        loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
+    Value init = arith::ConstantOp::create(
+        rewriter, loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
 
     scf::ForOp loadLoop;
     {
       OpBuilder::InsertionGuard g(rewriter);
       loadLoop = scf::ForOp::create(rewriter, loc, lowerBound, upperBound, step,
-                                             ValueRange{init});
+                                    ValueRange{init});
       rewriter.setInsertionPointToStart(loadLoop.getBody());
 
       auto tileSliceIndex = loadLoop.getInductionVar();
 
       auto idx0 = arith::AddIOp::create(rewriter, loc, tileSliceIndex,
-                                                 readOp.getIndices()[0]);
+                                        readOp.getIndices()[0]);
       auto idx1 = readOp.getIndices()[1];
 
-      Value scalar = memref::LoadOp::create(rewriter,
-          loc, readOp.getBase(), SmallVector<Value>({idx0, idx1}));
+      Value scalar = memref::LoadOp::create(rewriter, loc, readOp.getBase(),
+                                            SmallVector<Value>({idx0, idx1}));
 
-      Operation *updateInit = vector::InsertOp::create(rewriter,
-          loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
+      Operation *updateInit = vector::InsertOp::create(
+          rewriter, loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
 
       scf::YieldOp::create(rewriter, loc, updateInit->getResult(0));
     }
@@ -968,8 +971,8 @@ struct LowerColumnTransferReadToLoops
     // type was a 2D vector, we need to cast before returning the result. This
     // ShapeCast should cancel-out with some other ShapeCast (i.e. it's a
     // no-op).
-    auto sc = vector::ShapeCastOp::create(rewriter,
-        loc, readOp.getResult().getType(), loadLoop.getResult(0));
+    auto sc = vector::ShapeCastOp::create(
+        rewriter, loc, readOp.getResult().getType(), loadLoop.getResult(0));
 
     rewriter.replaceOp(readOp, sc);
 
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index 3ee211fc09bee..a7c69811fb7bd 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -87,8 +87,8 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     VectorType sourceType = source.getType();
     VectorType resultType = convertOp.getResult().getType();
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resultType, rewriter.getZeroAttr(resultType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resultType,
+                                             rewriter.getZeroAttr(resultType));
 
     // We want to iterate over the input vector in steps of the trailing
     // dimension. So this creates tile shape where all leading dimensions are 1,
@@ -100,15 +100,15 @@ struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
     for (SmallVector<int64_t> index :
          StaticTileOffsetRange(sourceType.getShape(), tileShape)) {
       auto extractOrInsertPosition = ArrayRef(index).drop_back();
-      auto sourceVector = vector::ExtractOp::create(rewriter,
-          loc, source, extractOrInsertPosition);
+      auto sourceVector = vector::ExtractOp::create(rewriter, loc, source,
+                                                    extractOrInsertPosition);
       VectorType convertedType =
           VectorType::Builder(llvm::cast<VectorType>(sourceVector.getType()))
               .setDim(0, resultType.getShape().back());
       auto convertedVector =
           IntrOp::create(rewriter, loc, TypeRange{convertedType}, sourceVector);
       result = vector::InsertOp::create(rewriter, loc, convertedVector, result,
-                                                 extractOrInsertPosition);
+                                        extractOrInsertPosition);
     }
 
     rewriter.replaceOp(convertOp, result);
@@ -136,11 +136,11 @@ struct PselOpLowering : public ConvertOpToLLVMPattern<PselOp> {
     auto svboolType = VectorType::get(16, rewriter.getI1Type(), true);
     auto loc = pselOp.getLoc();
     auto svboolP1 = ConvertToSvboolIntrOp::create(rewriter, loc, svboolType,
-                                                           adaptor.getP1());
-    auto indexI32 = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI32Type(), pselOp.getIndex());
+                                                  adaptor.getP1());
+    auto indexI32 = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI32Type(), pselOp.getIndex());
     auto pselIntr = PselIntrOp::create(rewriter, loc, svboolType, svboolP1,
-                                                pselOp.getP2(), indexI32);
+                                       pselOp.getP2(), indexI32);
     rewriter.replaceOpWithNewOp<ConvertFromSvboolIntrOp>(
         pselOp, adaptor.getP1().getType(), pselIntr);
     return success();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
index 33de12ca17cfc..3a409ad9ed9d6 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
@@ -71,8 +71,8 @@ void replaceOpWithUnrealizedConversion(PatternRewriter &rewriter, TOp op,
                                        TLegalizerCallback callback) {
   replaceOpWithLegalizedOp(rewriter, op, [&](TOp newOp) {
     // Mark our `unrealized_conversion_casts` with a pass label.
-    return UnrealizedConversionCastOp::create(rewriter,
-        op.getLoc(), TypeRange{op.getResult().getType()},
+    return UnrealizedConversionCastOp::create(
+        rewriter, op.getLoc(), TypeRange{op.getResult().getType()},
         ValueRange{callback(newOp)},
         NamedAttribute(rewriter.getStringAttr(kSVELegalizerTag),
                        rewriter.getUnitAttr()));
@@ -239,8 +239,8 @@ struct LegalizeSVEMaskStoreConversion
 
     auto legalMaskType = widenScalableMaskTypeToSvbool(
         llvm::cast<VectorType>(valueToStore.getType()));
-    auto convertToSvbool = arm_sve::ConvertToSvboolOp::create(rewriter,
-        loc, legalMaskType, valueToStore);
+    auto convertToSvbool = arm_sve::ConvertToSvboolOp::create(
+        rewriter, loc, legalMaskType, valueToStore);
     // Replace this store with a conversion to a storable svbool mask [1],
     // followed by a wider store.
     replaceOpWithLegalizedOp(rewriter, storeOp,
@@ -290,8 +290,8 @@ struct LegalizeSVEMaskLoadConversion : public OpRewritePattern<memref::LoadOp> {
     replaceOpWithLegalizedOp(rewriter, loadOp, [&](memref::LoadOp newLoadOp) {
       newLoadOp.setMemRef(*legalMemref);
       newLoadOp.getResult().setType(legalMaskType);
-      return arm_sve::ConvertFromSvboolOp::create(rewriter,
-          loc, loadedMask.getType(), newLoadOp);
+      return arm_sve::ConvertFromSvboolOp::create(
+          rewriter, loc, loadedMask.getType(), newLoadOp);
     });
 
     return success();
@@ -408,8 +408,8 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
       reassoc.back().push_back(i);
     if (!memref::CollapseShapeOp::isGuaranteedCollapsible(memTy, reassoc))
       return failure();
-    Value collapsedMem = memref::CollapseShapeOp::create(rewriter,
-        readOp.getLoc(), readOp.getBase(), reassoc);
+    Value collapsedMem = memref::CollapseShapeOp::create(
+        rewriter, readOp.getLoc(), readOp.getBase(), reassoc);
 
     // Get a vector type with collapsed trailing dimensions.
     SmallVector<int64_t> shape(origVT.getShape());
@@ -424,14 +424,14 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
     auto indices = readOp.getIndices().drop_back(numCollapseDims - 1);
 
     // Create the new `transfer_read`.
-    auto newReadOp = vector::TransferReadOp::create(rewriter,
-        readOp.getLoc(), collapsedVT, collapsedMem, indices,
+    auto newReadOp = vector::TransferReadOp::create(
+        rewriter, readOp.getLoc(), collapsedVT, collapsedMem, indices,
         readOp.getPadding(),
         ArrayRef<bool>(origInBounds).drop_back(numCollapseDims - 1));
 
     // Cast back to the original vector type.
     auto toOrigShape = vector::ShapeCastOp::create(rewriter, readOp.getLoc(),
-                                                            origVT, newReadOp);
+                                                   origVT, newReadOp);
 
     rewriter.replaceOp(readOp, toOrigShape);
     return success();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
index 0dfd58c582dfd..26113693536df 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
@@ -234,25 +234,25 @@ class LowerContractionToSVEI8MMPattern
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the LHS tile.
       auto r0 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
-                                                   ArrayRef<int64_t>{i});
+                                          ArrayRef<int64_t>{i});
       auto r1 = vector::ExtractOp::create(rewriter, loc, *maybeLhs,
-                                                   ArrayRef<int64_t>{i + 1});
+                                          ArrayRef<int64_t>{i + 1});
       // Concatenate to obtain a 16 x i8 flattened sub-tile.
-      auto t = vector::ShuffleOp::create(rewriter,
-          loc, r0, r1,
+      auto t = vector::ShuffleOp::create(
+          rewriter, loc, r0, r1,
           llvm::ArrayRef<int64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
                                   14, 15});
       // Turn it into a scalable vector.
-      auto s = vector::ScalableInsertOp::create(rewriter,
-          loc, t, ub::PoisonOp::create(rewriter, loc, nxv16i8), 0);
+      auto s = vector::ScalableInsertOp::create(
+          rewriter, loc, t, ub::PoisonOp::create(rewriter, loc, nxv16i8), 0);
       // Replicate the sub-tile VSCALE times to fill the entire vector.
       auto r = arm_sve::DupQLaneOp::create(rewriter, loc, s, 0);
       lhsTile.push_back(r);
     }
 
     // "Flatten" the RHS tile from <[N]x8> to <[8*N]>.
-    auto rhs = vector::ShapeCastOp::create(rewriter,
-        maybeRhs->getLoc(),
+    auto rhs = vector::ShapeCastOp::create(
+        rewriter, maybeRhs->getLoc(),
         VectorType::get(/*shape=*/8 * N, rewriter.getI8Type(),
                         /*scalableDims=*/{true}),
         *maybeRhs);
@@ -260,8 +260,8 @@ class LowerContractionToSVEI8MMPattern
     // Extract the RHS sub-tiles with logical shape <8x[2]>.
     SmallVector<Value> rhsTile;
     for (int64_t j = 0; j < N; j += 2)
-      rhsTile.push_back(
-          vector::ScalableExtractOp::create(rewriter, loc, nxv16i8, rhs, j * 8));
+      rhsTile.push_back(vector::ScalableExtractOp::create(rewriter, loc,
+                                                          nxv16i8, rhs, j * 8));
 
     // Handy types for packing/unpacking of the accumulator tile.
     auto accRowTy = VectorType::get(/*shape=*/N, rewriter.getI32Type(),
@@ -278,9 +278,9 @@ class LowerContractionToSVEI8MMPattern
     for (int64_t i = 0; i < M; i += 2) {
       // Extract two consecutive rows of the accumulator tile.
       auto r0 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
-                                                   ArrayRef<int64_t>{i});
+                                          ArrayRef<int64_t>{i});
       auto r1 = vector::ExtractOp::create(rewriter, loc, op.getAcc(),
-                                                   ArrayRef<int64_t>{i + 1});
+                                          ArrayRef<int64_t>{i + 1});
       Value accTileVec;
       if (mmlaOp == MMLA::MixedSwapped) {
         // We need to swap the positions of the LHS and RHS (since we don't have
@@ -295,7 +295,8 @@ class LowerContractionToSVEI8MMPattern
 
         // Interleave the rows, effectively flattening each 2x2 tile into 4
         // consecutive elements.
-        auto intrI64 = vector::InterleaveOp::create(rewriter, loc, r0I64, r1I64);
+        auto intrI64 =
+            vector::InterleaveOp::create(rewriter, loc, r0I64, r1I64);
 
         // Bitcast back to 32-bit elements.
         accTileVec =
@@ -303,8 +304,8 @@ class LowerContractionToSVEI8MMPattern
       }
       // Extract ACC sub-tiles.
       for (int64_t j = 0; j < N; j += 2)
-        accTile.push_back(vector::ScalableExtractOp::create(rewriter,
-            loc, nxv4i32, accTileVec, j * 2));
+        accTile.push_back(vector::ScalableExtractOp::create(
+            rewriter, loc, nxv4i32, accTileVec, j * 2));
     }
 
     // Emit sub-tile matrix multiplications.
@@ -322,8 +323,8 @@ class LowerContractionToSVEI8MMPattern
       // Collect a number of sub-tiles in a row.
       Value row = ub::PoisonOp::create(rewriter, loc, accRowX2Ty);
       for (int64_t j = 0; j < N / 2; ++j)
-        row = vector::ScalableInsertOp::create(rewriter,
-            loc, outTile[i * N / 2 + j], row, j * 4);
+        row = vector::ScalableInsertOp::create(
+            rewriter, loc, outTile[i * N / 2 + j], row, j * 4);
 
       // Unpack the row to obtain two rows of the output. If we have the out
       // sub-tiles transposed we obtain two consecutive output rows by
@@ -336,14 +337,15 @@ class LowerContractionToSVEI8MMPattern
         out1 = tmp.getRes2();
       } else {
         // Deinterleave by pairs.
-        auto row64 = vector::BitCastOp::create(rewriter, loc, accRowX264Ty, row);
+        auto row64 =
+            vector::BitCastOp::create(rewriter, loc, accRowX264Ty, row);
         auto deintr64 = vector::DeinterleaveOp::create(rewriter, loc, row64);
 
         // Bitcast back into 32-bit elements and insert into the result.
         out0 = vector::BitCastOp::create(rewriter, loc, accRowTy,
-                                                  deintr64.getRes1());
+                                         deintr64.getRes1());
         out1 = vector::BitCastOp::create(rewriter, loc, accRowTy,
-                                                  deintr64.getRes2());
+                                         deintr64.getRes2());
       }
       result = vector::InsertOp::create(rewriter, loc, out0, result, i * 2);
       result = vector::InsertOp::create(rewriter, loc, out1, result, i * 2 + 1);
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index 7dd1b856fa2b0..28ea795c173e6 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -8,9 +8,9 @@
 
 #include "mlir/Dialect/Async/IR/Async.h"
 
+#include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
-#include "mlir/IR/Builders.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
index 25ce0fea50581..ddc64ea7ebf9a 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
@@ -416,7 +416,7 @@ AsyncRuntimeRefCountingPass::addDropRefInDivergentLivenessSuccessor(
 
       OpBuilder builder = OpBuilder::atBlockBegin(refCountingBlock);
       RuntimeDropRefOp::create(builder, value.getLoc(), value,
-                                       builder.getI64IntegerAttr(1));
+                               builder.getI64IntegerAttr(1));
 
       // No need to update the terminator operation.
       if (successor == refCountingBlock)
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index 0485ad99cf7d2..a01303be3e0d8 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -375,7 +375,7 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
 
     // Add async.coro.suspend as a suspended block terminator.
     CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend,
-                                  branch.getDest(), coro.cleanupForDestroy);
+                          branch.getDest(), coro.cleanupForDestroy);
 
     branch.erase();
   }
@@ -383,8 +383,9 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
   // Replace the original `async.execute` with a call to outlined function.
   {
     ImplicitLocOpBuilder callBuilder(loc, execute);
-    auto callOutlinedFunc = func::CallOp::create(callBuilder,
-        func.getName(), execute.getResultTypes(), functionInputs.getArrayRef());
+    auto callOutlinedFunc = func::CallOp::create(callBuilder, func.getName(),
+                                                 execute.getResultTypes(),
+                                                 functionInputs.getArrayRef());
     execute.replaceAllUsesWith(callOutlinedFunc.getResults());
     execute.erase();
   }
@@ -586,12 +587,13 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
 
       // Assert that the awaited operands is not in the error state.
       Value isError = RuntimeIsErrorOp::create(builder, i1, operand);
-      Value notError = arith::XOrIOp::create(builder,
-          isError, arith::ConstantOp::create(builder,
-                       loc, i1, builder.getIntegerAttr(i1, 1)));
+      Value notError = arith::XOrIOp::create(
+          builder, isError,
+          arith::ConstantOp::create(builder, loc, i1,
+                                    builder.getIntegerAttr(i1, 1)));
 
       cf::AssertOp::create(builder, notError,
-                                   "Awaited async operand is in error state");
+                           "Awaited async operand is in error state");
     }
 
     // Inside the coroutine we convert await operation into coroutine suspension
@@ -614,8 +616,8 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
 
       // Add async.coro.suspend as a suspended block terminator.
       builder.setInsertionPointToEnd(suspended);
-      CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend, resume,
-                                    coro.cleanupForDestroy);
+      CoroSuspendOp::create(builder, coroSaveOp.getState(), coro.suspend,
+                            resume, coro.cleanupForDestroy);
 
       // Split the resume block into error checking and continuation.
       Block *continuation = rewriter.splitBlock(resume, Block::iterator(op));
@@ -624,10 +626,10 @@ class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
       builder.setInsertionPointToStart(resume);
       auto isError = RuntimeIsErrorOp::create(builder, loc, i1, operand);
       cf::CondBranchOp::create(builder, isError,
-                                       /*trueDest=*/setupSetErrorBlock(coro),
-                                       /*trueArgs=*/ArrayRef<Value>(),
-                                       /*falseDest=*/continuation,
-                                       /*falseArgs=*/ArrayRef<Value>());
+                               /*trueDest=*/setupSetErrorBlock(coro),
+                               /*trueArgs=*/ArrayRef<Value>(),
+                               /*falseDest=*/continuation,
+                               /*falseArgs=*/ArrayRef<Value>());
 
       // Make sure that replacement value will be constructed in the
       // continuation block.
@@ -757,10 +759,10 @@ class AssertOpLowering : public OpConversionPattern<cf::AssertOp> {
     Block *cont = rewriter.splitBlock(op->getBlock(), Block::iterator(op));
     rewriter.setInsertionPointToEnd(cont->getPrevNode());
     cf::CondBranchOp::create(rewriter, loc, adaptor.getArg(),
-                                      /*trueDest=*/cont,
-                                      /*trueArgs=*/ArrayRef<Value>(),
-                                      /*falseDest=*/setupSetErrorBlock(coro),
-                                      /*falseArgs=*/ArrayRef<Value>());
+                             /*trueDest=*/cont,
+                             /*trueArgs=*/ArrayRef<Value>(),
+                             /*falseDest=*/setupSetErrorBlock(coro),
+                             /*falseArgs=*/ArrayRef<Value>());
     rewriter.eraseOp(op);
 
     return success();
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index a1deab67f6bfa..4dfba74a6dd3c 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -196,8 +196,8 @@ LogicalResult DeallocationState::getMemrefsAndConditionsToDeallocate(
     // Simply cast unranked MemRefs to ranked memrefs with 0 dimensions such
     // that we can call extract_strided_metadata on it.
     if (auto unrankedMemRefTy = dyn_cast<UnrankedMemRefType>(memref.getType()))
-      memref = memref::ReinterpretCastOp::create(builder,
-          loc, memref,
+      memref = memref::ReinterpretCastOp::create(
+          builder, loc, memref,
           /*offset=*/builder.getIndexAttr(0),
           /*sizes=*/ArrayRef<OpFoldResult>{},
           /*strides=*/ArrayRef<OpFoldResult>{});
@@ -296,8 +296,8 @@ FailureOr<Operation *> deallocation_impl::insertDeallocOpForReturnLike(
   if (memrefs.empty() && toRetain.empty())
     return op;
 
-  auto deallocOp = bufferization::DeallocOp::create(builder,
-      op->getLoc(), memrefs, conditions, toRetain);
+  auto deallocOp = bufferization::DeallocOp::create(
+      builder, op->getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
   // result values of the dealloc operation as they are always unique.
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 05a1367a6b126..825f63e7147c8 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -170,8 +170,8 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   if (llvm::isa<RankedTensorType>(shapedValue.getType())) {
     tensor = shapedValue;
   } else if (llvm::isa<MemRefType>(shapedValue.getType())) {
-    tensor = ToTensorOp::create(b,
-        loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
+    tensor = ToTensorOp::create(
+        b, loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
         shapedValue);
   } else if (llvm::isa<UnrankedTensorType>(shapedValue.getType()) ||
              llvm::isa<UnrankedMemRefType>(shapedValue.getType())) {
@@ -210,7 +210,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
 
   // Create AllocTensorOp.
   auto allocTensorOp = AllocTensorOp::create(b, loc, tensorType, dynamicSizes,
-                                               copy ? tensor : Value());
+                                             copy ? tensor : Value());
 
   // Add 'memory_space' attribute. Not needed if 'copy' operand is specified.
   if (copy)
@@ -753,8 +753,8 @@ void bufferization::replaceOpWithBufferizedValues(RewriterBase &rewriter,
       // ToTensorOp. Throughout bufferization, this ToTensorOp will gradually
       // loose all of its users and eventually DCE away.
       rewriter.setInsertionPointAfter(op);
-      replacement = bufferization::ToTensorOp::create(rewriter,
-          replacement.getLoc(), opResult.getType(), replacement);
+      replacement = bufferization::ToTensorOp::create(
+          rewriter, replacement.getLoc(), opResult.getType(), replacement);
     }
     replacements.push_back(replacement);
   }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index 835ad61f54da2..891652670dd5b 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -167,8 +167,8 @@ struct RemoveDeallocMemrefsContainedInRetained
       std::optional<bool> analysisResult =
           analysis.isSameAllocation(retained, memref);
       if (analysisResult == true) {
-        auto disjunction = arith::OrIOp::create(rewriter,
-            deallocOp.getLoc(), updatedCondition, cond);
+        auto disjunction = arith::OrIOp::create(rewriter, deallocOp.getLoc(),
+                                                updatedCondition, cond);
         rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(),
                                       disjunction);
       }
@@ -247,16 +247,16 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias
         continue;
       }
 
-      replacements.push_back(arith::ConstantOp::create(rewriter,
-          deallocOp.getLoc(), rewriter.getBoolAttr(false)));
+      replacements.push_back(arith::ConstantOp::create(
+          rewriter, deallocOp.getLoc(), rewriter.getBoolAttr(false)));
     }
 
     if (newRetainedMemrefs.size() == deallocOp.getRetained().size())
       return failure();
 
-    auto newDeallocOp = DeallocOp::create(rewriter,
-        deallocOp.getLoc(), deallocOp.getMemrefs(), deallocOp.getConditions(),
-        newRetainedMemrefs);
+    auto newDeallocOp =
+        DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
+                          deallocOp.getConditions(), newRetainedMemrefs);
     int i = 0;
     for (auto &repl : replacements) {
       if (!repl)
@@ -327,7 +327,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
 
       // Create new bufferization.dealloc op for `memref`.
       auto newDeallocOp = DeallocOp::create(rewriter, loc, memref, cond,
-                                                     deallocOp.getRetained());
+                                            deallocOp.getRetained());
       updatedConditions.push_back(
           llvm::to_vector(ValueRange(newDeallocOp.getUpdatedConditions())));
     }
@@ -337,8 +337,9 @@ struct SplitDeallocWhenNotAliasingAnyOther
       return failure();
 
     // Create bufferization.dealloc op for all remaining memrefs.
-    auto newDeallocOp = DeallocOp::create(rewriter,
-        loc, remainingMemrefs, remainingConditions, deallocOp.getRetained());
+    auto newDeallocOp =
+        DeallocOp::create(rewriter, loc, remainingMemrefs, remainingConditions,
+                          deallocOp.getRetained());
 
     // Bit-or all conditions.
     SmallVector<Value> replacements =
@@ -347,8 +348,8 @@ struct SplitDeallocWhenNotAliasingAnyOther
       assert(replacements.size() == additionalConditions.size() &&
              "expected same number of updated conditions");
       for (int64_t i = 0, e = replacements.size(); i < e; ++i) {
-        replacements[i] = arith::OrIOp::create(rewriter,
-            loc, replacements[i], additionalConditions[i]);
+        replacements[i] = arith::OrIOp::create(rewriter, loc, replacements[i],
+                                               additionalConditions[i]);
       }
     }
     rewriter.replaceOp(deallocOp, replacements);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index 52490a0b2293b..e30e094c28467 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -200,8 +200,8 @@ updateCalls(ModuleOp module,
     newOperands.append(outParams.begin(), outParams.end());
     auto newResultTypes = llvm::to_vector<6>(llvm::map_range(
         replaceWithNewCallResults, [](Value v) { return v.getType(); }));
-    auto newCall = func::CallOp::create(builder, op.getLoc(), op.getCalleeAttr(),
-                                                newResultTypes, newOperands);
+    auto newCall = func::CallOp::create(
+        builder, op.getLoc(), op.getCalleeAttr(), newResultTypes, newOperands);
     for (auto t : llvm::zip(replaceWithNewCallResults, newCall.getResults()))
       std::get<0>(t).replaceAllUsesWith(std::get<1>(t));
     op.erase();
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
index 80984f21df551..c0e0809fa4111 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
@@ -141,8 +141,9 @@ bufferization::getGlobalFor(arith::ConstantOp constantOp,
       cast<MemRefType>(getMemRefTypeWithStaticIdentityLayout(type));
   if (memorySpace)
     memrefType = MemRefType::Builder(memrefType).setMemorySpace(memorySpace);
-  auto global = memref::GlobalOp::create(globalBuilder,
-      constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
+  auto global = memref::GlobalOp::create(
+      globalBuilder, constantOp.getLoc(),
+      (Twine("__constant_") + os.str()).str(),
       /*sym_visibility=*/globalBuilder.getStringAttr("private"),
       /*type=*/memrefType,
       /*initial_value=*/cast<ElementsAttr>(constantOp.getValue()),
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 4a68ecba79fb7..91f6f25e3d096 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -434,8 +434,8 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
     // Replace all uses of the original tensor bbArg.
     rewriter.setInsertionPointToStart(block);
     if (!bbArgUses.empty()) {
-      Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
-          bbArg.getLoc(), tensorType, bbArg);
+      Value toTensorOp = bufferization::ToTensorOp::create(
+          rewriter, bbArg.getLoc(), tensorType, bbArg);
       for (OpOperand *use : bbArgUses)
         use->set(toTensorOp);
     }
@@ -466,13 +466,13 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
       if (failed(operandBufferType))
         return failure();
       rewriter.setInsertionPointAfterValue(operand);
-      Value bufferizedOperand = bufferization::ToBufferOp::create(rewriter,
-          operand.getLoc(), *operandBufferType, operand);
+      Value bufferizedOperand = bufferization::ToBufferOp::create(
+          rewriter, operand.getLoc(), *operandBufferType, operand);
       // A cast is needed if the operand and the block argument have different
       // bufferized types.
       if (type != *operandBufferType)
-        bufferizedOperand = memref::CastOp::create(rewriter,
-            operand.getLoc(), type, bufferizedOperand);
+        bufferizedOperand = memref::CastOp::create(rewriter, operand.getLoc(),
+                                                   type, bufferizedOperand);
       newOperands.push_back(bufferizedOperand);
     }
     operands.getMutableForwardedOperands().assign(newOperands);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index 42d63b66485d5..a50ddbe3a8ee8 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -119,7 +119,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
     for (func::CallOp callOp : callerMap[funcOp]) {
       rewriter.setInsertionPoint(callOp);
       auto newCallOp = func::CallOp::create(rewriter, callOp.getLoc(), funcOp,
-                                                     callOp.getOperands());
+                                            callOp.getOperands());
       SmallVector<Value> newResults;
       int64_t nextResult = 0;
       for (int64_t i = 0; i < callOp.getNumResults(); ++i) {
@@ -134,8 +134,8 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
         Type expectedType = callOp.getResult(i).getType();
         if (replacement.getType() != expectedType) {
           // A cast must be inserted at the call site.
-          replacement = memref::CastOp::create(rewriter,
-              callOp.getLoc(), expectedType, replacement);
+          replacement = memref::CastOp::create(rewriter, callOp.getLoc(),
+                                               expectedType, replacement);
         }
         newResults.push_back(replacement);
       }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
index 923cbc0e2259d..1784964cf9b95 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
@@ -169,7 +169,7 @@ LogicalResult mlir::bufferization::eliminateEmptyTensors(
           continue;
         rewriter.setInsertionPointAfterValue(replacement);
         replacement = tensor::CastOp::create(rewriter, v.getLoc(), v.getType(),
-                                                      replacement);
+                                             replacement);
       }
       // Replace the specific use of the tensor::EmptyOp.
       rewriter.modifyOpInPlace(user, [&]() {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index 025d5764c6d9e..f69efd1b3fa8c 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -319,8 +319,9 @@ struct CallOpInterface
     }
 
     // 3. Create the new CallOp.
-    Operation *newCallOp = func::CallOp::create(rewriter,
-        callOp.getLoc(), funcOp.getSymName(), resultTypes, newOperands);
+    Operation *newCallOp =
+        func::CallOp::create(rewriter, callOp.getLoc(), funcOp.getSymName(),
+                             resultTypes, newOperands);
     newCallOp->setAttrs(callOp->getAttrs());
 
     // 4. Replace the old op with the new op.
@@ -483,8 +484,8 @@ struct FuncOpInterface
 
         // Note: If `inferFunctionResultLayout = true`, casts are later folded
         // away.
-        Value toBufferOp = bufferization::ToBufferOp::create(rewriter,
-            returnOp.getLoc(), bufferizedType, returnVal);
+        Value toBufferOp = bufferization::ToBufferOp::create(
+            rewriter, returnOp.getLoc(), bufferizedType, returnVal);
         returnValues.push_back(toBufferOp);
       }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
index 83ccd573a2734..f0d65b04ee447 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
@@ -108,14 +108,14 @@ class DeallocOpConversion
     // Compute the base pointer indices, compare all retained indices to the
     // memref index to check if they alias.
     SmallVector<Value> doesNotAliasList;
-    Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(rewriter,
-        op->getLoc(), adaptor.getMemrefs()[0]);
+    Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
+        rewriter, op->getLoc(), adaptor.getMemrefs()[0]);
     for (Value retained : adaptor.getRetained()) {
-      Value retainedAsIdx =
-          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op->getLoc(),
-                                                                  retained);
-      Value doesNotAlias = arith::CmpIOp::create(rewriter,
-          op->getLoc(), arith::CmpIPredicate::ne, memrefAsIdx, retainedAsIdx);
+      Value retainedAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
+          rewriter, op->getLoc(), retained);
+      Value doesNotAlias = arith::CmpIOp::create(rewriter, op->getLoc(),
+                                                 arith::CmpIPredicate::ne,
+                                                 memrefAsIdx, retainedAsIdx);
       doesNotAliasList.push_back(doesNotAlias);
     }
 
@@ -126,27 +126,28 @@ class DeallocOpConversion
 
     // Also consider the condition given by the dealloc operation and perform a
     // conditional deallocation guarded by that value.
-    Value shouldDealloc = arith::AndIOp::create(rewriter,
-        op->getLoc(), prev, adaptor.getConditions()[0]);
+    Value shouldDealloc = arith::AndIOp::create(rewriter, op->getLoc(), prev,
+                                                adaptor.getConditions()[0]);
 
-    scf::IfOp::create(rewriter,
-        op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-          memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[0]);
-          scf::YieldOp::create(builder, loc);
-        });
+    scf::IfOp::create(rewriter, op.getLoc(), shouldDealloc,
+                      [&](OpBuilder &builder, Location loc) {
+                        memref::DeallocOp::create(builder, loc,
+                                                  adaptor.getMemrefs()[0]);
+                        scf::YieldOp::create(builder, loc);
+                      });
 
     // Compute the replacement values for the dealloc operation results. This
     // inserts an already canonicalized form of
     // `select(does_alias_with_memref(r), memref_cond, false)` for each retained
     // value r.
     SmallVector<Value> replacements;
-    Value trueVal = arith::ConstantOp::create(rewriter,
-        op->getLoc(), rewriter.getBoolAttr(true));
+    Value trueVal = arith::ConstantOp::create(rewriter, op->getLoc(),
+                                              rewriter.getBoolAttr(true));
     for (Value doesNotAlias : doesNotAliasList) {
       Value aliases =
           arith::XOrIOp::create(rewriter, op->getLoc(), doesNotAlias, trueVal);
       Value result = arith::AndIOp::create(rewriter, op->getLoc(), aliases,
-                                                    adaptor.getConditions()[0]);
+                                           adaptor.getConditions()[0]);
       replacements.push_back(result);
     }
 
@@ -230,98 +231,102 @@ class DeallocOpConversion
     // Without storing them to memrefs, we could not use for-loops but only a
     // completely unrolled version of it, potentially leading to code-size
     // blow-up.
-    Value toDeallocMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
-                                     rewriter.getIndexType()));
-    Value conditionMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getConditions().size()},
-                                     rewriter.getI1Type()));
-    Value toRetainMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
-                                     rewriter.getIndexType()));
+    Value toDeallocMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
+                        rewriter.getIndexType()));
+    Value conditionMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getConditions().size()},
+                        rewriter.getI1Type()));
+    Value toRetainMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getRetained().size()},
+                        rewriter.getIndexType()));
 
     auto getConstValue = [&](uint64_t value) -> Value {
       return arith::ConstantOp::create(rewriter, op.getLoc(),
-                                                rewriter.getIndexAttr(value));
+                                       rewriter.getIndexAttr(value));
     };
 
     // Extract the base pointers of the memrefs as indices to check for aliasing
     // at runtime.
     for (auto [i, toDealloc] : llvm::enumerate(adaptor.getMemrefs())) {
-      Value memrefAsIdx =
-          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
-                                                                  toDealloc);
+      Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
+          rewriter, op.getLoc(), toDealloc);
       memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx,
-                                       toDeallocMemref, getConstValue(i));
+                              toDeallocMemref, getConstValue(i));
     }
 
     for (auto [i, cond] : llvm::enumerate(adaptor.getConditions()))
       memref::StoreOp::create(rewriter, op.getLoc(), cond, conditionMemref,
-                                       getConstValue(i));
+                              getConstValue(i));
 
     for (auto [i, toRetain] : llvm::enumerate(adaptor.getRetained())) {
-      Value memrefAsIdx =
-          memref::ExtractAlignedPointerAsIndexOp::create(rewriter, op.getLoc(),
-                                                                  toRetain);
-      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx, toRetainMemref,
-                                       getConstValue(i));
+      Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
+          rewriter, op.getLoc(), toRetain);
+      memref::StoreOp::create(rewriter, op.getLoc(), memrefAsIdx,
+                              toRetainMemref, getConstValue(i));
     }
 
     // Cast the allocated memrefs to dynamic shape because we want only one
     // helper function no matter how many operands the bufferization.dealloc
     // has.
-    Value castedDeallocMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value castedDeallocMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toDeallocMemref);
-    Value castedCondsMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value castedCondsMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         conditionMemref);
-    Value castedRetainMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value castedRetainMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getIndexType()),
         toRetainMemref);
 
-    Value deallocCondsMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
-                                     rewriter.getI1Type()));
-    Value retainCondsMemref = memref::AllocOp::create(rewriter,
-        op.getLoc(), MemRefType::get({(int64_t)adaptor.getRetained().size()},
-                                     rewriter.getI1Type()));
-
-    Value castedDeallocCondsMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value deallocCondsMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getMemrefs().size()},
+                        rewriter.getI1Type()));
+    Value retainCondsMemref = memref::AllocOp::create(
+        rewriter, op.getLoc(),
+        MemRefType::get({(int64_t)adaptor.getRetained().size()},
+                        rewriter.getI1Type()));
+
+    Value castedDeallocCondsMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         deallocCondsMemref);
-    Value castedRetainCondsMemref = memref::CastOp::create(rewriter,
-        op->getLoc(),
+    Value castedRetainCondsMemref = memref::CastOp::create(
+        rewriter, op->getLoc(),
         MemRefType::get({ShapedType::kDynamic}, rewriter.getI1Type()),
         retainCondsMemref);
 
     Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>();
-    func::CallOp::create(rewriter,
-        op.getLoc(), deallocHelperFuncMap.lookup(symtableOp),
+    func::CallOp::create(
+        rewriter, op.getLoc(), deallocHelperFuncMap.lookup(symtableOp),
         SmallVector<Value>{castedDeallocMemref, castedRetainMemref,
                            castedCondsMemref, castedDeallocCondsMemref,
                            castedRetainCondsMemref});
 
     for (unsigned i = 0, e = adaptor.getMemrefs().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value shouldDealloc = memref::LoadOp::create(rewriter,
-          op.getLoc(), deallocCondsMemref, idxValue);
-      scf::IfOp::create(rewriter,
-          op.getLoc(), shouldDealloc, [&](OpBuilder &builder, Location loc) {
-            memref::DeallocOp::create(builder, loc, adaptor.getMemrefs()[i]);
-            scf::YieldOp::create(builder, loc);
-          });
+      Value shouldDealloc = memref::LoadOp::create(
+          rewriter, op.getLoc(), deallocCondsMemref, idxValue);
+      scf::IfOp::create(rewriter, op.getLoc(), shouldDealloc,
+                        [&](OpBuilder &builder, Location loc) {
+                          memref::DeallocOp::create(builder, loc,
+                                                    adaptor.getMemrefs()[i]);
+                          scf::YieldOp::create(builder, loc);
+                        });
     }
 
     SmallVector<Value> replacements;
     for (unsigned i = 0, e = adaptor.getRetained().size(); i < e; ++i) {
       Value idxValue = getConstValue(i);
-      Value ownership = memref::LoadOp::create(rewriter,
-          op.getLoc(), retainCondsMemref, idxValue);
+      Value ownership = memref::LoadOp::create(rewriter, op.getLoc(),
+                                               retainCondsMemref, idxValue);
       replacements.push_back(ownership);
     }
 
@@ -349,8 +354,8 @@ class DeallocOpConversion
                   ConversionPatternRewriter &rewriter) const override {
     // Lower the trivial case.
     if (adaptor.getMemrefs().empty()) {
-      Value falseVal = arith::ConstantOp::create(rewriter,
-          op.getLoc(), rewriter.getBoolAttr(false));
+      Value falseVal = arith::ConstantOp::create(rewriter, op.getLoc(),
+                                                 rewriter.getBoolAttr(false));
       rewriter.replaceOp(
           op, SmallVector<Value>(adaptor.getRetained().size(), falseVal));
       return success();
@@ -455,18 +460,19 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
       arith::ConstantOp::create(builder, loc, builder.getBoolAttr(true));
   Value falseValue =
       arith::ConstantOp::create(builder, loc, builder.getBoolAttr(false));
-  Value toDeallocSize = memref::DimOp::create(builder, loc, toDeallocMemref, c0);
+  Value toDeallocSize =
+      memref::DimOp::create(builder, loc, toDeallocMemref, c0);
   Value toRetainSize = memref::DimOp::create(builder, loc, toRetainMemref, c0);
 
-  scf::ForOp::create(builder,
-      loc, c0, toRetainSize, c1, ValueRange(),
+  scf::ForOp::create(
+      builder, loc, c0, toRetainSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value i, ValueRange iterArgs) {
         memref::StoreOp::create(builder, loc, falseValue, retainCondsMemref, i);
         scf::YieldOp::create(builder, loc);
       });
 
-  scf::ForOp::create(builder,
-      loc, c0, toDeallocSize, c1, ValueRange(),
+  scf::ForOp::create(
+      builder, loc, c0, toDeallocSize, c1, ValueRange(),
       [&](OpBuilder &builder, Location loc, Value outerIter,
           ValueRange iterArgs) {
         Value toDealloc =
@@ -482,30 +488,28 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, toRetainSize, c1, trueValue,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value retainValue = memref::LoadOp::create(builder,
-                          loc, toRetainMemref, i);
-                      Value doesAlias = arith::CmpIOp::create(builder,
-                          loc, arith::CmpIPredicate::eq, retainValue,
+                      Value retainValue = memref::LoadOp::create(
+                          builder, loc, toRetainMemref, i);
+                      Value doesAlias = arith::CmpIOp::create(
+                          builder, loc, arith::CmpIPredicate::eq, retainValue,
                           toDealloc);
-                      scf::IfOp::create(builder,
-                          loc, doesAlias,
+                      scf::IfOp::create(
+                          builder, loc, doesAlias,
                           [&](OpBuilder &builder, Location loc) {
-                            Value retainCondValue =
-                                memref::LoadOp::create(builder,
-                                    loc, retainCondsMemref, i);
-                            Value aggregatedRetainCond =
-                                arith::OrIOp::create(builder,
-                                    loc, retainCondValue, cond);
-                            memref::StoreOp::create(builder,
-                                loc, aggregatedRetainCond, retainCondsMemref,
-                                i);
+                            Value retainCondValue = memref::LoadOp::create(
+                                builder, loc, retainCondsMemref, i);
+                            Value aggregatedRetainCond = arith::OrIOp::create(
+                                builder, loc, retainCondValue, cond);
+                            memref::StoreOp::create(builder, loc,
+                                                    aggregatedRetainCond,
+                                                    retainCondsMemref, i);
                             scf::YieldOp::create(builder, loc);
                           });
-                      Value doesntAlias = arith::CmpIOp::create(builder,
-                          loc, arith::CmpIPredicate::ne, retainValue,
+                      Value doesntAlias = arith::CmpIOp::create(
+                          builder, loc, arith::CmpIPredicate::ne, retainValue,
                           toDealloc);
-                      Value yieldValue = arith::AndIOp::create(builder,
-                          loc, iterArgs[0], doesntAlias);
+                      Value yieldValue = arith::AndIOp::create(
+                          builder, loc, iterArgs[0], doesntAlias);
                       scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
@@ -518,20 +522,20 @@ func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
                     loc, c0, outerIter, c1, noRetainAlias,
                     [&](OpBuilder &builder, Location loc, Value i,
                         ValueRange iterArgs) {
-                      Value prevDeallocValue = memref::LoadOp::create(builder,
-                          loc, toDeallocMemref, i);
-                      Value doesntAlias = arith::CmpIOp::create(builder,
-                          loc, arith::CmpIPredicate::ne, prevDeallocValue,
-                          toDealloc);
-                      Value yieldValue = arith::AndIOp::create(builder,
-                          loc, iterArgs[0], doesntAlias);
+                      Value prevDeallocValue = memref::LoadOp::create(
+                          builder, loc, toDeallocMemref, i);
+                      Value doesntAlias = arith::CmpIOp::create(
+                          builder, loc, arith::CmpIPredicate::ne,
+                          prevDeallocValue, toDealloc);
+                      Value yieldValue = arith::AndIOp::create(
+                          builder, loc, iterArgs[0], doesntAlias);
                       scf::YieldOp::create(builder, loc, yieldValue);
                     })
                 .getResult(0);
 
         Value shouldDealoc = arith::AndIOp::create(builder, loc, noAlias, cond);
         memref::StoreOp::create(builder, loc, shouldDealoc, deallocCondsMemref,
-                                        outerIter);
+                                outerIter);
         scf::YieldOp::create(builder, loc);
       });
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index 71890bfea4e5d..64c178dfe76d8 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -750,19 +750,18 @@ Value BufferDeallocation::materializeMemrefWithGuaranteedOwnership(
 
   // Insert a runtime check and only clone if we still don't have ownership at
   // runtime.
-  Value maybeClone =
-      builder
-          .create<scf::IfOp>(
-              memref.getLoc(), condition,
-              [&](OpBuilder &builder, Location loc) {
-                scf::YieldOp::create(builder, loc, newMemref);
-              },
-              [&](OpBuilder &builder, Location loc) {
-                Value clone =
-                    bufferization::CloneOp::create(builder, loc, newMemref);
-                scf::YieldOp::create(builder, loc, clone);
-              })
-          .getResult(0);
+  Value maybeClone = builder
+                         .create<scf::IfOp>(
+                             memref.getLoc(), condition,
+                             [&](OpBuilder &builder, Location loc) {
+                               scf::YieldOp::create(builder, loc, newMemref);
+                             },
+                             [&](OpBuilder &builder, Location loc) {
+                               Value clone = bufferization::CloneOp::create(
+                                   builder, loc, newMemref);
+                               scf::YieldOp::create(builder, loc, clone);
+                             })
+                         .getResult(0);
   Value trueVal = buildBoolValue(builder, memref.getLoc(), true);
   state.updateOwnership(maybeClone, trueVal);
   state.addMemrefToDeallocate(maybeClone, maybeClone.getParentBlock());
@@ -797,8 +796,8 @@ BufferDeallocation::handleInterface(BranchOpInterface op) {
   state.getMemrefsToRetain(block, op->getSuccessor(0), forwardedOperands,
                            toRetain);
 
-  auto deallocOp = bufferization::DeallocOp::create(builder,
-      op.getLoc(), memrefs, conditions, toRetain);
+  auto deallocOp = bufferization::DeallocOp::create(
+      builder, op.getLoc(), memrefs, conditions, toRetain);
 
   // We want to replace the current ownership of the retained values with the
   // result values of the dealloc operation as they are always unique.
@@ -885,12 +884,11 @@ BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
       builder.setInsertionPoint(op);
       Ownership ownership = state.getOwnership(operand, block);
       if (ownership.isUnique()) {
-        Value ownershipInverted = arith::XOrIOp::create(builder,
-            op.getLoc(), ownership.getIndicator(),
+        Value ownershipInverted = arith::XOrIOp::create(
+            builder, op.getLoc(), ownership.getIndicator(),
             buildBoolValue(builder, op.getLoc(), true));
-        cf::AssertOp::create(builder,
-            op.getLoc(), ownershipInverted,
-            "expected that the block does not have ownership");
+        cf::AssertOp::create(builder, op.getLoc(), ownershipInverted,
+                             "expected that the block does not have ownership");
       }
     }
   }
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
index 925c72959bd3a..0adfb51a228bb 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp
@@ -49,7 +49,7 @@ Operation *complex::ComplexDialect::materializeConstant(OpBuilder &builder,
                                                         Location loc) {
   if (complex::ConstantOp::isBuildableWith(value, type)) {
     return complex::ConstantOp::create(builder, loc, type,
-                                               llvm::cast<ArrayAttr>(value));
+                                       llvm::cast<ArrayAttr>(value));
   }
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
diff --git a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index 23f50930cb3b3..80dc0c597562d 100644
--- a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -87,8 +87,8 @@ struct CondBranchOpInterface
                                destOperands.getAsOperandRange(), toRetain);
       SmallVector<Value> adaptedConditions(
           llvm::map_range(conditions, conditionModifier));
-      auto deallocOp = bufferization::DeallocOp::create(builder,
-          condBr.getLoc(), memrefs, adaptedConditions, toRetain);
+      auto deallocOp = bufferization::DeallocOp::create(
+          builder, condBr.getLoc(), memrefs, adaptedConditions, toRetain);
       state.resetOwnerships(deallocOp.getRetained(), condBr->getBlock());
       for (auto [retained, ownership] : llvm::zip(
                deallocOp.getRetained(), deallocOp.getUpdatedConditions())) {
@@ -116,17 +116,18 @@ struct CondBranchOpInterface
         condBr.getTrueDest(), condBr.getTrueDestOperandsMutable(),
         [&](Value cond) {
           return arith::AndIOp::create(builder, condBr.getLoc(), cond,
-                                               condBr.getCondition());
+                                       condBr.getCondition());
         },
         thenMapping);
     DeallocOp elseTakenDeallocOp = insertDeallocForBranch(
         condBr.getFalseDest(), condBr.getFalseDestOperandsMutable(),
         [&](Value cond) {
-          Value trueVal = arith::ConstantOp::create(builder,
-              condBr.getLoc(), builder.getBoolAttr(true));
-          Value negation = arith::XOrIOp::create(builder,
-              condBr.getLoc(), trueVal, condBr.getCondition());
-          return arith::AndIOp::create(builder, condBr.getLoc(), cond, negation);
+          Value trueVal = arith::ConstantOp::create(builder, condBr.getLoc(),
+                                                    builder.getBoolAttr(true));
+          Value negation = arith::XOrIOp::create(
+              builder, condBr.getLoc(), trueVal, condBr.getCondition());
+          return arith::AndIOp::create(builder, condBr.getLoc(), cond,
+                                       negation);
         },
         elseMapping);
 
@@ -143,9 +144,9 @@ struct CondBranchOpInterface
 
     for (Value retained : commonValues) {
       state.resetOwnerships(retained, condBr->getBlock());
-      Value combinedOwnership = arith::SelectOp::create(builder,
-          condBr.getLoc(), condBr.getCondition(), thenMapping[retained],
-          elseMapping[retained]);
+      Value combinedOwnership = arith::SelectOp::create(
+          builder, condBr.getLoc(), condBr.getCondition(),
+          thenMapping[retained], elseMapping[retained]);
       state.updateOwnership(retained, combinedOwnership, condBr->getBlock());
     }
 
diff --git a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
index 72c331ad96607..612e8099eaf35 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
@@ -68,7 +68,7 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
       TypeAttr typeAttr = TypeAttr::get(val.getType());
       fields.push_back({fieldName, typeAttr});
       emitc::FieldOp::create(rewriter, funcOp.getLoc(), fieldName, typeAttr,
-                                      argAttr);
+                             argAttr);
     }
 
     rewriter.setInsertionPointToEnd(&newClassOp.getBody().front());
diff --git a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
index 7485ba40b4344..c39e77d823b78 100644
--- a/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
+++ b/mlir/lib/Dialect/Func/Extensions/InlinerExtension.cpp
@@ -61,7 +61,8 @@ struct FuncInlinerInterface : public DialectInlinerInterface {
 
     // Replace the return with a branch to the dest.
     OpBuilder builder(op);
-    cf::BranchOp::create(builder, op->getLoc(), newDest, returnOp.getOperands());
+    cf::BranchOp::create(builder, op->getLoc(), newDest,
+                         returnOp.getOperands());
     op->erase();
   }
 
diff --git a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
index 4b70cf172b407..b6c8cdf2f495a 100644
--- a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
@@ -46,9 +46,9 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 
     // Substitute with the new result types from the corresponding FuncType
     // conversion.
-    auto newCallOp = CallOp::create(rewriter,
-        callOp.getLoc(), callOp.getCallee(), convertedResults,
-        flattenValues(adaptor.getOperands()));
+    auto newCallOp =
+        CallOp::create(rewriter, callOp.getLoc(), callOp.getCallee(),
+                       convertedResults, flattenValues(adaptor.getOperands()));
     SmallVector<ValueRange> replacements;
     size_t offset = 0;
     for (int i = 0, e = callOp->getNumResults(); i < e; ++i) {
diff --git a/mlir/lib/Dialect/Func/Utils/Utils.cpp b/mlir/lib/Dialect/Func/Utils/Utils.cpp
index 1eba777921828..f781ed2d591b4 100644
--- a/mlir/lib/Dialect/Func/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Func/Utils/Utils.cpp
@@ -44,8 +44,8 @@ func::replaceFuncWithNewOrder(RewriterBase &rewriter, func::FuncOp funcOp,
   for (unsigned int idx : newResultsOrder)
     newOutputTypes.push_back(origOutputTypes[idx]);
   rewriter.setInsertionPoint(funcOp);
-  auto newFuncOp = func::FuncOp::create(rewriter,
-      funcOp.getLoc(), funcOp.getName(),
+  auto newFuncOp = func::FuncOp::create(
+      rewriter, funcOp.getLoc(), funcOp.getName(),
       rewriter.getFunctionType(newInputTypes, newOutputTypes));
 
   Region &newRegion = newFuncOp.getBody();
@@ -109,8 +109,9 @@ func::replaceCallOpWithNewOrder(RewriterBase &rewriter, func::CallOp callOp,
   // Replace the kernel call operation with a new one that has the
   // reordered arguments.
   rewriter.setInsertionPoint(callOp);
-  auto newCallOp = func::CallOp::create(rewriter,
-      callOp.getLoc(), callOp.getCallee(), newResultTypes, newArgsOrderValues);
+  auto newCallOp =
+      func::CallOp::create(rewriter, callOp.getLoc(), callOp.getCallee(),
+                           newResultTypes, newArgsOrderValues);
   newCallOp.setNoInlineAttr(callOp.getNoInlineAttr());
   for (auto &&[newIndex, origIndex] : llvm::enumerate(newResultsOrder))
     rewriter.replaceAllUsesWith(callOp.getResult(origIndex),
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index 6f0510138da20..224130736c945 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -76,9 +76,10 @@ buildPredicates(RewriterBase &rewriter, Location loc, ArrayRef<Value> activeIds,
     }
     if (activeMappingSize == availableMappingSize)
       continue;
-    Value idx = arith::ConstantIndexOp::create(rewriter, loc, activeMappingSize);
+    Value idx =
+        arith::ConstantIndexOp::create(rewriter, loc, activeMappingSize);
     Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
-                                                activeId, idx);
+                                       activeId, idx);
     predicateOps.push_back(pred);
   }
   return predicateOps;
@@ -422,7 +423,7 @@ DiagnosedSilenceableFailure createGpuLaunch(
   Value blkSizeY = blockDimY.has_value() ? createConst(blockDimY.value()) : one;
   Value blkSizeZ = blockDimZ.has_value() ? createConst(blockDimZ.value()) : one;
   launchOp = LaunchOp::create(rewriter, loc, gridSizeX, gridSizeY, gridSizeZ,
-                                       blkSizeX, blkSizeY, blkSizeZ);
+                              blkSizeX, blkSizeY, blkSizeZ);
   rewriter.setInsertionPointToEnd(&launchOp.getBody().front());
   TerminatorOp::create(rewriter, loc);
   return DiagnosedSilenceableFailure::success();
@@ -446,7 +447,7 @@ DiagnosedSilenceableFailure alterGpuLaunch(
   rewriter.setInsertionPointAfterValue(currentBlockdim.x);
   auto createConstValue = [&](int dim) {
     return arith::ConstantIndexOp::create(rewriter, currentBlockdim.x.getLoc(),
-                                                   dim);
+                                          dim);
   };
 
   if (gridDimX.has_value())
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index ccb5245442206..cd138401e3177 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -165,8 +165,9 @@ async::ExecuteOp addExecuteResults(async::ExecuteOp executeOp,
 
   // Clone executeOp with the extra results.
   OpBuilder builder(executeOp);
-  auto newOp = async::ExecuteOp::create(builder,
-      executeOp.getLoc(), TypeRange{resultTypes}.drop_front() /*drop token*/,
+  auto newOp = async::ExecuteOp::create(
+      builder, executeOp.getLoc(),
+      TypeRange{resultTypes}.drop_front() /*drop token*/,
       executeOp.getDependencies(), executeOp.getBodyOperands());
   IRMapping mapper;
   newOp.getRegion().getBlocks().clear();
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index 5c84a340f0e08..7b30906abc2fd 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -109,8 +109,8 @@ static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
       getFlatOffsetAndStrides(rewriter, loc, source, offsetsTemp);
   MemRefType retType = inferCastResultType(base, offset);
   return memref::ReinterpretCastOp::create(rewriter, loc, retType, base, offset,
-                                                    ArrayRef<OpFoldResult>(),
-                                                    ArrayRef<OpFoldResult>());
+                                           ArrayRef<OpFoldResult>(),
+                                           ArrayRef<OpFoldResult>());
 }
 
 static bool needFlatten(Value val) {
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index b98c060ac1da5..99f5c5b0cf139 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -195,8 +195,8 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   }
   FunctionType type =
       FunctionType::get(launchOp.getContext(), kernelOperandTypes, {});
-  auto outlinedFunc = gpu::GPUFuncOp::create(builder,
-      loc, kernelFnName, type,
+  auto outlinedFunc = gpu::GPUFuncOp::create(
+      builder, loc, kernelFnName, type,
       TypeRange(ValueRange(launchOp.getWorkgroupAttributions())),
       TypeRange(ValueRange(launchOp.getPrivateAttributions())));
   outlinedFunc->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
@@ -287,9 +287,9 @@ static void convertToLaunchFuncOp(gpu::LaunchOp launchOp,
   Value asyncToken = launchOp.getAsyncToken();
   std::optional<gpu::KernelDim3> clusterSize =
       launchOp.getClusterSizeOperandValues();
-  auto launchFunc = gpu::LaunchFuncOp::create(builder,
-      launchOp.getLoc(), kernelFunc, launchOp.getGridSizeOperandValues(),
-      launchOp.getBlockSizeOperandValues(),
+  auto launchFunc = gpu::LaunchFuncOp::create(
+      builder, launchOp.getLoc(), kernelFunc,
+      launchOp.getGridSizeOperandValues(), launchOp.getBlockSizeOperandValues(),
       launchOp.getDynamicSharedMemorySize(), operands,
       asyncToken ? asyncToken.getType() : nullptr,
       launchOp.getAsyncDependencies(), clusterSize);
@@ -416,7 +416,7 @@ class GpuKernelOutliningPass
     if (!kernelModule) {
       // If not found, create a new GPU module
       kernelModule = gpu::GPUModuleOp::create(builder, kernelFunc.getLoc(),
-                                                      kernelModuleName);
+                                              kernelModuleName);
     }
 
     // If a valid data layout spec was provided, attach it to the kernel module.
diff --git a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
index dd164e5aaa7c4..3c447337d821f 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
@@ -109,7 +109,7 @@ LogicalResult moduleSerializer(GPUModuleOp op,
     handler = moduleHandler;
   builder.setInsertionPointAfter(op);
   gpu::BinaryOp::create(builder, op.getLoc(), op.getName(), handler,
-                                builder.getArrayAttr(objects));
+                        builder.getArrayAttr(objects));
   op->erase();
   return success();
 }
diff --git a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
index 5a55a39a7748e..18c69f5f30e5d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
@@ -48,8 +48,8 @@ struct PromoteShuffleToSwizzlePattern
                                          "offset must be in the range [0, 31]");
 
     Location loc = op.getLoc();
-    Value res = amdgpu::SwizzleBitModeOp::create(rewriter,
-        loc, op.getResult(0).getType(), op.getValue(), /*andMask=*/31,
+    Value res = amdgpu::SwizzleBitModeOp::create(
+        rewriter, loc, op.getResult(0).getType(), op.getValue(), /*andMask=*/31,
         /*orMask=*/0, /*xorMask=*/offsetValue);
     Value valid = arith::ConstantIntOp::create(rewriter, loc, 1, /*width*/ 1);
     rewriter.replaceOp(op, {res, valid});
diff --git a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
index 7aa5288d8260d..d88f4d56d9009 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
@@ -53,8 +53,8 @@ struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> {
     lo = arith::TruncIOp::create(rewriter, valueLoc, i32, value);
 
     // Get the high bits by trunc(value >> 32).
-    auto c32 = arith::ConstantOp::create(rewriter,
-        valueLoc, rewriter.getIntegerAttr(i64, 32));
+    auto c32 = arith::ConstantOp::create(rewriter, valueLoc,
+                                         rewriter.getIntegerAttr(i64, 32));
     hi = arith::ShRUIOp::create(rewriter, valueLoc, value, c32);
     hi = arith::TruncIOp::create(rewriter, valueLoc, i32, hi);
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
index f401e6a267eef..79be247c2a6b5 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
@@ -60,17 +60,19 @@ struct GpuSubgroupIdRewriter final : OpRewritePattern<gpu::SubgroupIdOp> {
     Value tidY = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::y);
     Value tidZ = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::z);
 
-    Value dimYxIdZ = arith::MulIOp::create(rewriter, loc, indexType, dimY, tidZ);
+    Value dimYxIdZ =
+        arith::MulIOp::create(rewriter, loc, indexType, dimY, tidZ);
     Value dimYxIdZPlusIdY =
         arith::AddIOp::create(rewriter, loc, indexType, dimYxIdZ, tidY);
     Value dimYxIdZPlusIdYTimesDimX =
         arith::MulIOp::create(rewriter, loc, indexType, dimX, dimYxIdZPlusIdY);
-    Value IdXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(rewriter,
-        loc, indexType, tidX, dimYxIdZPlusIdYTimesDimX);
-    Value subgroupSize = gpu::SubgroupSizeOp::create(rewriter,
-        loc, rewriter.getIndexType(), /*upper_bound = */ nullptr);
-    Value subgroupIdOp = arith::DivUIOp::create(rewriter,
-        loc, indexType, IdXPlusDimYxIdZPlusIdYTimesDimX, subgroupSize);
+    Value IdXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(
+        rewriter, loc, indexType, tidX, dimYxIdZPlusIdYTimesDimX);
+    Value subgroupSize = gpu::SubgroupSizeOp::create(
+        rewriter, loc, rewriter.getIndexType(), /*upper_bound = */ nullptr);
+    Value subgroupIdOp =
+        arith::DivUIOp::create(rewriter, loc, indexType,
+                               IdXPlusDimYxIdZPlusIdYTimesDimX, subgroupSize);
     rewriter.replaceOp(op, {subgroupIdOp});
     return success();
   }
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index 7290f92f6fe08..b9e2dd5b19a6f 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -92,21 +92,22 @@ struct BreakDownSubgroupReduce final : OpRewritePattern<gpu::SubgroupReduceOp> {
         extracted =
             vector::ExtractOp::create(rewriter, loc, op.getValue(), startIdx);
       } else {
-        extracted = vector::ExtractStridedSliceOp::create(rewriter,
-            loc, op.getValue(), /*offsets=*/startIdx, /*sizes=*/numElems,
+        extracted = vector::ExtractStridedSliceOp::create(
+            rewriter, loc, op.getValue(), /*offsets=*/startIdx,
+            /*sizes=*/numElems,
             /*strides=*/1);
       }
 
-      Value reduce = gpu::SubgroupReduceOp::create(rewriter,
-          loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
-          op.getClusterStride());
+      Value reduce = gpu::SubgroupReduceOp::create(
+          rewriter, loc, extracted, op.getOp(), op.getUniform(),
+          op.getClusterSize(), op.getClusterStride());
       if (numElems == 1) {
         res = vector::InsertOp::create(rewriter, loc, reduce, res, startIdx);
         continue;
       }
 
-      res = vector::InsertStridedSliceOp::create(rewriter,
-          loc, reduce, res, /*offsets=*/startIdx, /*strides=*/1);
+      res = vector::InsertStridedSliceOp::create(
+          rewriter, loc, reduce, res, /*offsets=*/startIdx, /*strides=*/1);
     }
 
     rewriter.replaceOp(op, res);
@@ -138,10 +139,11 @@ struct ScalarizeSingleElementReduce final
     assert(vecTy.getRank() == 1 && "Unexpected vector type");
     assert(!vecTy.isScalable() && "Unexpected vector type");
     Location loc = op.getLoc();
-    Value extracted = vector::ExtractOp::create(rewriter, loc, op.getValue(), 0);
-    Value reduce = gpu::SubgroupReduceOp::create(rewriter,
-        loc, extracted, op.getOp(), op.getUniform(), op.getClusterSize(),
-        op.getClusterStride());
+    Value extracted =
+        vector::ExtractOp::create(rewriter, loc, op.getValue(), 0);
+    Value reduce = gpu::SubgroupReduceOp::create(
+        rewriter, loc, extracted, op.getOp(), op.getUniform(),
+        op.getClusterSize(), op.getClusterStride());
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, vecTy, reduce);
     return success();
   }
@@ -326,10 +328,10 @@ struct VectorSubgroupReduceToShuffles final
         static_cast<int64_t>(elementsPerShuffle), vecTy.getElementType());
     Value extendedInput = op.getValue();
     if (vecBitwidth < shuffleBitwidth) {
-      auto zero = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getZeroAttr(extendedVecTy));
-      extendedInput = vector::InsertStridedSliceOp::create(rewriter,
-          loc, extendedInput, zero, /*offsets=*/0, /*strides=*/1);
+      auto zero = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getZeroAttr(extendedVecTy));
+      extendedInput = vector::InsertStridedSliceOp::create(
+          rewriter, loc, extendedInput, zero, /*offsets=*/0, /*strides=*/1);
     }
 
     auto shuffleIntType = rewriter.getIntegerType(shuffleBitwidth);
@@ -351,8 +353,8 @@ struct VectorSubgroupReduceToShuffles final
         rewriter, loc, extendedInput, op.getOp(), *ci, packFn, unpackFn);
 
     if (vecBitwidth < shuffleBitwidth) {
-      res = vector::ExtractStridedSliceOp::create(rewriter,
-          loc, res, /*offsets=*/0, /*sizes=*/vecTy.getNumElements(),
+      res = vector::ExtractStridedSliceOp::create(
+          rewriter, loc, res, /*offsets=*/0, /*sizes=*/vecTy.getNumElements(),
           /*strides=*/1);
     }
 
@@ -378,8 +380,8 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   const bool boundCtrl = true;
   if (ci.clusterSize >= 2) {
     // Perform reduction between all lanes N <-> N+1.
-    dpp = amdgpu::DPPOp::create(rewriter,
-        loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
+    dpp = amdgpu::DPPOp::create(
+        rewriter, loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({1, 0, 3, 2}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
                                      gpu::convertReductionKind(mode), res, dpp);
@@ -387,8 +389,8 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
 
   if (ci.clusterSize >= 4) {
     // Perform reduction between all lanes N <-> N+2.
-    dpp = amdgpu::DPPOp::create(rewriter,
-        loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
+    dpp = amdgpu::DPPOp::create(
+        rewriter, loc, res.getType(), res, res, amdgpu::DPPPerm::quad_perm,
         rewriter.getI32ArrayAttr({2, 3, 0, 1}), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
                                      gpu::convertReductionKind(mode), res, dpp);
@@ -396,17 +398,18 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
   if (ci.clusterSize >= 8) {
     // Perform reduction between all lanes N <-> 7-N,
     // e.g lane[0] <-> lane[7], lane[1] <-> lane[6]..., lane[3] <-> lane[4].
-    dpp = amdgpu::DPPOp::create(rewriter,
-        loc, res.getType(), res, res, amdgpu::DPPPerm::row_half_mirror,
-        rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
+    dpp = amdgpu::DPPOp::create(rewriter, loc, res.getType(), res, res,
+                                amdgpu::DPPPerm::row_half_mirror,
+                                rewriter.getUnitAttr(), allRows, allBanks,
+                                boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
                                      gpu::convertReductionKind(mode), res, dpp);
   }
   if (ci.clusterSize >= 16) {
     // Perform reduction between all lanes N <-> 15-N,
     // e.g lane[0] <-> lane[15], lane[1] <-> lane[14]..., lane[7] <-> lane[8].
-    dpp = amdgpu::DPPOp::create(rewriter,
-        loc, res.getType(), res, res, amdgpu::DPPPerm::row_mirror,
+    dpp = amdgpu::DPPOp::create(
+        rewriter, loc, res.getType(), res, res, amdgpu::DPPPerm::row_mirror,
         rewriter.getUnitAttr(), allRows, allBanks, boundCtrl);
     res = vector::makeArithReduction(rewriter, loc,
                                      gpu::convertReductionKind(mode), res, dpp);
@@ -415,20 +418,20 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
     if (chipset.majorVersion <= 9) {
       // Broadcast last value from each row to next row.
       // Use row mask to avoid polluting rows 1 and 3.
-      dpp = amdgpu::DPPOp::create(rewriter,
-          loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_15,
-          rewriter.getUnitAttr(), 0xa, allBanks,
-          /*bound_ctrl*/ false);
+      dpp = amdgpu::DPPOp::create(rewriter, loc, res.getType(), res, res,
+                                  amdgpu::DPPPerm::row_bcast_15,
+                                  rewriter.getUnitAttr(), 0xa, allBanks,
+                                  /*bound_ctrl*/ false);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), res, dpp);
     } else if (chipset.majorVersion <= 12) {
       // Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2).
-      Value uint32Max = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
+      Value uint32Max = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
       dpp = ROCDL::PermlaneX16Op::create(rewriter, loc, res.getType(), res, res,
-                                                  uint32Max, uint32Max,
-                                                  /*fi=*/true,
-                                                  /*bound_ctrl=*/false);
+                                         uint32Max, uint32Max,
+                                         /*fi=*/true,
+                                         /*bound_ctrl=*/false);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), res, dpp);
     } else {
@@ -437,33 +440,35 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
               "this device.");
     }
     if (ci.subgroupSize == 32) {
-      Value lane31 = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
+      Value lane31 = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
+      res =
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
     }
   }
   if (ci.clusterSize >= 64) {
     if (chipset.majorVersion <= 9) {
       // Broadcast 31st lane value to rows 2 and 3.
-      dpp = amdgpu::DPPOp::create(rewriter,
-          loc, res.getType(), res, res, amdgpu::DPPPerm::row_bcast_31,
-          rewriter.getUnitAttr(), 0xf, allBanks,
-          /*bound_ctrl*/ true);
+      dpp = amdgpu::DPPOp::create(rewriter, loc, res.getType(), res, res,
+                                  amdgpu::DPPPerm::row_bcast_31,
+                                  rewriter.getUnitAttr(), 0xf, allBanks,
+                                  /*bound_ctrl*/ true);
       res = vector::makeArithReduction(
           rewriter, loc, gpu::convertReductionKind(mode), dpp, res);
       // Obtain reduction from last rows, the previous rows are polluted.
-      Value lane63 = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
-      res = ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
+      Value lane63 = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
+      res =
+          ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
 
     } else if (chipset.majorVersion <= 12) {
       // Assume reduction across 32 lanes has been done.
       // Perform final reduction manually by summing values in lane 0 and
       // lane 32.
-      Value lane31 = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
-      Value lane63 = arith::ConstantOp::create(rewriter,
-          loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
+      Value lane31 = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(31));
+      Value lane63 = arith::ConstantOp::create(
+          rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(63));
       lane31 =
           ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane31);
       lane63 =
diff --git a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
index 473a2e515c12b..384d1a0ddccd2 100644
--- a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
+++ b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
@@ -27,9 +27,10 @@ WarpDistributionPattern::moveRegionToNewWarpOpAndReplaceReturns(
   // Create a new op before the existing one, with the extra operands.
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(warpOp);
-  auto newWarpOp = WarpExecuteOnLane0Op::create(rewriter,
-      warpOp.getLoc(), newReturnTypes, warpOp.getLaneid(), warpOp.getWarpSize(),
-      warpOp.getArgs(), warpOp.getBody()->getArgumentTypes());
+  auto newWarpOp = WarpExecuteOnLane0Op::create(
+      rewriter, warpOp.getLoc(), newReturnTypes, warpOp.getLaneid(),
+      warpOp.getWarpSize(), warpOp.getArgs(),
+      warpOp.getBody()->getArgumentTypes());
 
   Region &opBody = warpOp.getBodyRegion();
   Region &newOpBody = newWarpOp.getBodyRegion();
diff --git a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
index b1d376c7c03db..894de4408c375 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
@@ -90,8 +90,8 @@ void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
     }
     for (auto [idx, t] : llvm::enumerate(stype.getBody())) {
       if (itype != PTXRegisterMod::Write) {
-        Value extractValue = LLVM::ExtractValueOp::create(rewriter,
-            interfaceOp->getLoc(), v, idx);
+        Value extractValue = LLVM::ExtractValueOp::create(
+            rewriter, interfaceOp->getLoc(), v, idx);
         addValue(extractValue);
       }
       if (itype == PTXRegisterMod::ReadWrite) {
@@ -132,8 +132,8 @@ LLVM::InlineAsmOp PtxBuilder::build() {
   // Replace all % with $
   llvm::replace(ptxInstruction, '%', '$');
 
-  return LLVM::InlineAsmOp::create(rewriter,
-      interfaceOp->getLoc(),
+  return LLVM::InlineAsmOp::create(
+      rewriter, interfaceOp->getLoc(),
       /*result types=*/resultTypes,
       /*operands=*/ptxOperands,
       /*asm_string=*/llvm::StringRef(ptxInstruction),
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 7d3f59d128367..feaffa34897b6 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -89,8 +89,8 @@ mlir::LLVM::lookupOrCreateFn(OpBuilder &b, Operation *moduleOp, StringRef name,
   OpBuilder::InsertionGuard g(b);
   assert(!moduleOp->getRegion(0).empty() && "expected non-empty region");
   b.setInsertionPointToStart(&moduleOp->getRegion(0).front());
-  auto funcOp = LLVM::LLVMFuncOp::create(b,
-      moduleOp->getLoc(), name,
+  auto funcOp = LLVM::LLVMFuncOp::create(
+      b, moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 
   if (symbolTables) {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index 98fb9b09f0cd5..d09c59ca3f24d 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -46,8 +46,8 @@ void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
   for (Operation *user : getOperation()->getUsers())
     if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
       LLVM::DbgValueOp::create(builder, declareOp.getLoc(), argument,
-                                       declareOp.getVarInfo(),
-                                       declareOp.getLocationExpr());
+                               declareOp.getVarInfo(),
+                               declareOp.getLocationExpr());
 }
 
 std::optional<PromotableAllocationOpInterface>
@@ -89,8 +89,8 @@ DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
   for (Attribute index : usedIndices) {
     Type elemType = destructurableType.getTypeAtIndex(index);
     assert(elemType && "used index must exist");
-    auto subAlloca = LLVM::AllocaOp::create(builder,
-        getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
+    auto subAlloca = LLVM::AllocaOp::create(
+        builder, getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
         getArraySize());
     newAllocators.push_back(subAlloca);
     slotMap.try_emplace<MemorySlot>(index, {subAlloca.getResult(), elemType});
@@ -260,14 +260,14 @@ static Value createExtractAndCast(OpBuilder &builder, Location loc,
   // Truncate the integer if the size of the target is less than the value.
   if (isBigEndian(dataLayout)) {
     uint64_t shiftAmount = srcTypeSize - targetTypeSize;
-    auto shiftConstant = LLVM::ConstantOp::create(builder,
-        loc, builder.getIntegerAttr(srcType, shiftAmount));
+    auto shiftConstant = LLVM::ConstantOp::create(
+        builder, loc, builder.getIntegerAttr(srcType, shiftAmount));
     replacement =
         builder.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
   }
 
-  replacement = LLVM::TruncOp::create(builder,
-      loc, builder.getIntegerType(targetTypeSize), replacement);
+  replacement = LLVM::TruncOp::create(
+      builder, loc, builder.getIntegerType(targetTypeSize), replacement);
 
   // Now cast the integer to the actual target type if required.
   return castIntValueToSameSizedType(builder, loc, replacement, targetType);
@@ -304,8 +304,9 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
     // On big endian systems, a store to the base pointer overwrites the most
     // significant bits. To accomodate for this, the stored value needs to be
     // shifted into the according position.
-    Value bigEndianShift = LLVM::ConstantOp::create(builder,
-        loc, builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
+    Value bigEndianShift = LLVM::ConstantOp::create(
+        builder, loc,
+        builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
     valueAsInt =
         builder.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
   }
@@ -325,8 +326,8 @@ static Value createInsertAndCast(OpBuilder &builder, Location loc,
   }
 
   // Mask out the affected bits ...
-  Value mask = LLVM::ConstantOp::create(builder,
-      loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
+  Value mask = LLVM::ConstantOp::create(
+      builder, loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
   Value masked = builder.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
 
   // ... and combine the result with the new value.
@@ -656,7 +657,7 @@ void LLVM::DbgDeclareOp::visitReplacedValues(
   for (auto [op, value] : definitions) {
     builder.setInsertionPointAfter(op);
     LLVM::DbgValueOp::create(builder, getLoc(), value, getVarInfo(),
-                                     getLocationExpr());
+                             getLocationExpr());
   }
 }
 
@@ -979,8 +980,8 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetOp toReplace,
           .getResult();
 
   LLVM::MemsetOp::create(builder, toReplace.getLoc(), subslots.at(index).ptr,
-                                 toReplace.getVal(), newMemsetSizeValue,
-                                 toReplace.getIsVolatile());
+                         toReplace.getVal(), newMemsetSizeValue,
+                         toReplace.getIsVolatile());
 }
 
 template <>
@@ -991,9 +992,9 @@ void createMemsetIntr(OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
   auto newMemsetSizeValue =
       IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize);
 
-  LLVM::MemsetInlineOp::create(builder,
-      toReplace.getLoc(), subslots.at(index).ptr, toReplace.getVal(),
-      newMemsetSizeValue, toReplace.getIsVolatile());
+  LLVM::MemsetInlineOp::create(builder, toReplace.getLoc(),
+                               subslots.at(index).ptr, toReplace.getVal(),
+                               newMemsetSizeValue, toReplace.getIsVolatile());
 }
 
 } // namespace
@@ -1063,8 +1064,8 @@ static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
       APInt memsetVal(/*numBits=*/width, /*val=*/0);
       for (unsigned loBit = 0; loBit < width; loBit += 8)
         memsetVal.insertBits(constantPattern.getValue(), loBit);
-      return LLVM::ConstantOp::create(builder,
-          op.getLoc(), IntegerAttr::get(intType, memsetVal));
+      return LLVM::ConstantOp::create(builder, op.getLoc(),
+                                      IntegerAttr::get(intType, memsetVal));
     }
 
     // If the output is a single byte, we can return the pattern directly.
@@ -1309,7 +1310,8 @@ memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot,
                          const SmallPtrSetImpl<OpOperand *> &blockingUses,
                          OpBuilder &builder, Value reachingDefinition) {
   if (op.loadsFrom(slot))
-    LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition, op.getDst());
+    LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition,
+                          op.getDst());
   return DeletionKind::Delete;
 }
 
@@ -1354,11 +1356,12 @@ template <class MemcpyLike>
 void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                MemcpyLike toReplace, Value dst, Value src,
                                Type toCpy, bool isVolatile) {
-  Value memcpySize = LLVM::ConstantOp::create(builder,
-      toReplace.getLoc(), IntegerAttr::get(toReplace.getLen().getType(),
-                                           layout.getTypeSize(toCpy)));
+  Value memcpySize =
+      LLVM::ConstantOp::create(builder, toReplace.getLoc(),
+                               IntegerAttr::get(toReplace.getLen().getType(),
+                                                layout.getTypeSize(toCpy)));
   MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize,
-                             isVolatile);
+                     isVolatile);
 }
 
 template <>
@@ -1367,8 +1370,8 @@ void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                Value src, Type toCpy, bool isVolatile) {
   Type lenType = IntegerType::get(toReplace->getContext(),
                                   toReplace.getLen().getBitWidth());
-  LLVM::MemcpyInlineOp::create(builder,
-      toReplace.getLoc(), dst, src,
+  LLVM::MemcpyInlineOp::create(
+      builder, toReplace.getLoc(), dst, src,
       IntegerAttr::get(lenType, layout.getTypeSize(toCpy)), isVolatile);
 }
 
@@ -1409,9 +1412,9 @@ memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
     SmallVector<LLVM::GEPArg> gepIndices{
         0, static_cast<int32_t>(
                cast<IntegerAttr>(index).getValue().getZExtValue())};
-    Value subslotPtrInOther = LLVM::GEPOp::create(builder,
-        op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()), slot.elemType,
-        isDst ? op.getSrc() : op.getDst(), gepIndices);
+    Value subslotPtrInOther = LLVM::GEPOp::create(
+        builder, op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()),
+        slot.elemType, isDst ? op.getSrc() : op.getDst(), gepIndices);
 
     // Then create a new memcpy out of this source pointer.
     createMemcpyLikeToReplace(builder, dataLayout, op,
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
index 1b1268da15646..1fb482b63670f 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/AddComdats.cpp
@@ -34,8 +34,9 @@ static void addComdat(LLVM::LLVMFuncOp &op, OpBuilder &builder,
 
   PatternRewriter::InsertionGuard guard(builder);
   builder.setInsertionPointToStart(&comdatOp.getBody().back());
-  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(builder,
-      comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
+  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(
+      builder, comdatOp.getLoc(), op.getSymName(),
+      mlir::LLVM::comdat::Comdat::Any);
   op.setComdatAttr(mlir::SymbolRefAttr::get(
       builder.getContext(), comdatName,
       mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr())));
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index 2ecef4b4c8a51..47c2441100cf9 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -113,20 +113,22 @@ handleInlinedAllocas(Operation *call,
     // scope if some are already present in the body of the caller. This is not
     // invalid IR, but LLVM cleans these up in InstCombineCalls.cpp, along with
     // other cases where the stacksave/stackrestore is redundant.
-    stackPtr = LLVM::StackSaveOp::create(builder,
-        call->getLoc(), LLVM::LLVMPointerType::get(call->getContext()));
+    stackPtr = LLVM::StackSaveOp::create(
+        builder, call->getLoc(),
+        LLVM::LLVMPointerType::get(call->getContext()));
   }
   builder.setInsertionPointToStart(callerEntryBlock);
   for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
-    auto newConstant = LLVM::ConstantOp::create(builder,
-        allocaOp->getLoc(), allocaOp.getArraySize().getType(), arraySize);
+    auto newConstant =
+        LLVM::ConstantOp::create(builder, allocaOp->getLoc(),
+                                 allocaOp.getArraySize().getType(), arraySize);
     // Insert a lifetime start intrinsic where the alloca was before moving it.
     if (shouldInsertLifetime) {
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(allocaOp);
-      LLVM::LifetimeStartOp::create(builder,
-          allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
-          allocaOp.getResult());
+      LLVM::LifetimeStartOp::create(builder, allocaOp.getLoc(),
+                                    arraySize.getValue().getLimitedValue(),
+                                    allocaOp.getResult());
     }
     allocaOp->moveAfter(newConstant);
     allocaOp.getArraySizeMutable().assign(newConstant.getResult());
@@ -142,9 +144,9 @@ handleInlinedAllocas(Operation *call,
       LLVM::StackRestoreOp::create(builder, call->getLoc(), stackPtr);
     for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
       if (shouldInsertLifetime)
-        LLVM::LifetimeEndOp::create(builder,
-            allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
-            allocaOp.getResult());
+        LLVM::LifetimeEndOp::create(builder, allocaOp.getLoc(),
+                                    arraySize.getValue().getLimitedValue(),
+                                    allocaOp.getResult());
     }
   }
 }
@@ -604,15 +606,16 @@ static Value handleByValArgumentInit(OpBuilder &builder, Location loc,
     Block *entryBlock = &(*argument.getParentRegion()->begin());
     builder.setInsertionPointToStart(entryBlock);
     Value one = LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
-                                                 builder.getI64IntegerAttr(1));
-    allocaOp = LLVM::AllocaOp::create(builder,
-        loc, argument.getType(), elementType, one, targetAlignment);
+                                         builder.getI64IntegerAttr(1));
+    allocaOp = LLVM::AllocaOp::create(builder, loc, argument.getType(),
+                                      elementType, one, targetAlignment);
   }
   // Copy the pointee to the newly allocated value.
-  Value copySize = LLVM::ConstantOp::create(builder,
-      loc, builder.getI64Type(), builder.getI64IntegerAttr(elementTypeSize));
+  Value copySize =
+      LLVM::ConstantOp::create(builder, loc, builder.getI64Type(),
+                               builder.getI64IntegerAttr(elementTypeSize));
   LLVM::MemcpyOp::create(builder, loc, allocaOp, argument, copySize,
-                                 /*isVolatile=*/false);
+                         /*isVolatile=*/false);
   return allocaOp;
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
index 6fa4f6935f11c..38a4bc844ea89 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
@@ -59,7 +59,7 @@ static void ensureDistinctSuccessors(Block &bb) {
       for (BlockArgument arg : successor.first->getArguments())
         dummyBlock->addArgument(arg.getType(), arg.getLoc());
       LLVM::BrOp::create(builder, terminator->getLoc(),
-                                 dummyBlock->getArguments(), successor.first);
+                         dummyBlock->getArguments(), successor.first);
     }
   }
 }
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
index de6c3043bbcdd..7f34f7d142d73 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/OptimizeForNVVM.cpp
@@ -68,19 +68,19 @@ LogicalResult ExpandDivF16::matchAndRewrite(LLVM::FDivOp op,
 
   // Refine the approximation with one Newton iteration:
   // float refined = approx + (lhs - approx * rhs) * rcp;
-  Value err = LLVM::FMAOp::create(rewriter,
-      loc, approx, LLVM::FNegOp::create(rewriter, loc, rhs), lhs);
+  Value err = LLVM::FMAOp::create(
+      rewriter, loc, approx, LLVM::FNegOp::create(rewriter, loc, rhs), lhs);
   Value refined = LLVM::FMAOp::create(rewriter, loc, err, rcp, approx);
 
   // Use refined value if approx is normal (exponent neither all 0 or all 1).
-  Value mask = LLVM::ConstantOp::create(rewriter,
-      loc, i32Type, rewriter.getUI32IntegerAttr(0x7f800000));
+  Value mask = LLVM::ConstantOp::create(
+      rewriter, loc, i32Type, rewriter.getUI32IntegerAttr(0x7f800000));
   Value cast = LLVM::BitcastOp::create(rewriter, loc, i32Type, approx);
   Value exp = LLVM::AndOp::create(rewriter, loc, i32Type, cast, mask);
-  Value zero = LLVM::ConstantOp::create(rewriter,
-      loc, i32Type, rewriter.getUI32IntegerAttr(0));
-  Value pred = LLVM::OrOp::create(rewriter,
-      loc,
+  Value zero = LLVM::ConstantOp::create(rewriter, loc, i32Type,
+                                        rewriter.getUI32IntegerAttr(0));
+  Value pred = LLVM::OrOp::create(
+      rewriter, loc,
       LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, zero),
       LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::eq, exp, mask));
   Value result =
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 51cc1232df891..d5e2ed6bad7b1 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -78,11 +78,11 @@ static Operation *getSlice(OpBuilder &b, Location loc, Value source,
   return TypeSwitch<Type, Operation *>(source.getType())
       .Case<RankedTensorType>([&](RankedTensorType t) -> Operation * {
         return tensor::ExtractSliceOp::create(b, loc, source, offsets, sizes,
-                                                strides);
+                                              strides);
       })
       .Case<MemRefType>([&](MemRefType type) -> Operation * {
         return memref::SubViewOp::create(b, loc, source, offsets, sizes,
-                                           strides);
+                                         strides);
       })
       .Default([&](Type t) -> Operation * { return nullptr; });
 }
@@ -467,7 +467,7 @@ class RegionBuilderHelper {
     case UnaryFn::reciprocal: {
       Attribute oneAttr = builder.getOneAttr(arg.getType());
       auto one = arith::ConstantOp::create(builder, arg.getLoc(),
-                                                   ::cast<TypedAttr>(oneAttr));
+                                           ::cast<TypedAttr>(oneAttr));
       return arith::DivFOp::create(builder, arg.getLoc(), one, arg);
     }
     case UnaryFn::round:
@@ -647,7 +647,8 @@ class RegionBuilderHelper {
     builder.setInsertionPointToEnd(&block);
     Location loc = builder.getUnknownLoc();
     Attribute valueAttr = parseAttribute(value, builder.getContext());
-    return arith::ConstantOp::create(builder, loc, ::cast<TypedAttr>(valueAttr));
+    return arith::ConstantOp::create(builder, loc,
+                                     ::cast<TypedAttr>(valueAttr));
   }
 
   Value index(int64_t dim) {
@@ -749,14 +750,14 @@ struct FoldFillWithTensorReshape : OpRewritePattern<TensorReshapeOp> {
     TensorReshapeOp newInit;
     if constexpr (std::is_same<TensorReshapeOp, tensor::ExpandShapeOp>::value) {
 
-      newInit = TensorReshapeOp::create(rewriter,
-          loc, reshapeOp.getResultType(), oldFill.output(),
+      newInit = TensorReshapeOp::create(
+          rewriter, loc, reshapeOp.getResultType(), oldFill.output(),
           reshapeOp.getReassociation(), reshapeOp.getOutputShape(),
           reshapeOp.getStaticOutputShape());
     } else {
-      newInit = TensorReshapeOp::create(rewriter, loc, reshapeOp.getResultType(),
-                                                 oldFill.output(),
-                                                 reshapeOp.getReassociation());
+      newInit = TensorReshapeOp::create(
+          rewriter, loc, reshapeOp.getResultType(), oldFill.output(),
+          reshapeOp.getReassociation());
     }
     rewriter.replaceOpWithNewOp<FillOp>(reshapeOp, ValueRange{oldFill.value()},
                                         ValueRange{newInit});
@@ -786,17 +787,17 @@ struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
       return rewriter.notifyMatchFailure(
           padOp, "failed to reify tensor.pad op result shape");
 
-    auto emptyTensor = tensor::EmptyOp::create(rewriter,
-        padOp.getLoc(), reifiedShape.front(),
-        padOp.getResultType().getElementType());
+    auto emptyTensor =
+        tensor::EmptyOp::create(rewriter, padOp.getLoc(), reifiedShape.front(),
+                                padOp.getResultType().getElementType());
     Value replacement =
         rewriter
             .create<FillOp>(fillOp.getLoc(), ValueRange{padValue},
                             ValueRange{emptyTensor})
             .getResult(0);
     if (replacement.getType() != padOp.getResultType()) {
-      replacement = tensor::CastOp::create(rewriter,
-          fillOp.getLoc(), padOp.getResultType(), replacement);
+      replacement = tensor::CastOp::create(rewriter, fillOp.getLoc(),
+                                           padOp.getResultType(), replacement);
     }
     rewriter.replaceOp(padOp, replacement);
     return success();
@@ -943,7 +944,7 @@ static FailureOr<FillOp> foldFillPackIntoFillOp(RewriterBase &rewriter,
     return failure();
 
   return linalg::FillOp::create(rewriter, packOp.getLoc(), fillOp.getInputs(),
-                                         packOp.getDest());
+                                packOp.getDest());
 }
 
 /// Wrapper pattern that applies foldFillPackIntoFillOp method.
@@ -1042,8 +1043,8 @@ struct FoldConcatsOfFill : public OpRewritePattern<tensor::ConcatOp> {
           concatOp, "not all operands are defined by a compatible fill op");
     }
 
-    Value outsConcat = tensor::ConcatOp::create(rewriter,
-        concatOp.getLoc(), concatOp.getDim(), allOuts);
+    Value outsConcat = tensor::ConcatOp::create(rewriter, concatOp.getLoc(),
+                                                concatOp.getDim(), allOuts);
     rewriter.replaceOpWithNewOp<linalg::FillOp>(
         concatOp, firstFillOp.getDpsInputOperand(0)->get(), outsConcat);
     return success();
@@ -1407,14 +1408,14 @@ struct EraseIdentityLinalgOp : public OpRewritePattern<OpTy> {
         // TODO: unify the two ops?
         if (sparse_tensor::getSparseTensorEncoding(returnType) ||
             sparse_tensor::getSparseTensorEncoding(resultType))
-          returnedArg = sparse_tensor::ConvertOp::create(rewriter,
-              linalgOp.getLoc(), resultType, returnedArg);
+          returnedArg = sparse_tensor::ConvertOp::create(
+              rewriter, linalgOp.getLoc(), resultType, returnedArg);
         else {
           if (!tensor::CastOp::areCastCompatible(returnedArg.getType(),
                                                  resultType))
             return failure();
-          returnedArg = tensor::CastOp::create(rewriter,
-              linalgOp.getLoc(), resultType, returnedArg);
+          returnedArg = tensor::CastOp::create(rewriter, linalgOp.getLoc(),
+                                               resultType, returnedArg);
         }
       }
       returnedArgs.push_back(returnedArg);
@@ -2147,8 +2148,8 @@ struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
     }
     SmallVector<OpFoldResult> transposeResultShapes =
         applyPermutation(dims, resultPerms);
-    Value transposeInit = tensor::EmptyOp::create(rewriter,
-        transposeOp.getLoc(), transposeResultShapes,
+    Value transposeInit = tensor::EmptyOp::create(
+        rewriter, transposeOp.getLoc(), transposeResultShapes,
         broadcastInputTy.getElementType());
 
     // Create broadcast(transpose(input)).
@@ -2560,8 +2561,8 @@ struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
     Operation *newOp = clone(rewriter, linalgOp, resultTypes, newOperands);
 
     // Create a tensor.cast operation back to the original type.
-    Value castBack = tensor::CastOp::create(rewriter,
-        loc, resultValue.getType(), newOp->getResult(resultNumber));
+    Value castBack = tensor::CastOp::create(
+        rewriter, loc, resultValue.getType(), newOp->getResult(resultNumber));
 
     SmallVector<Value> results(newOp->result_begin(), newOp->result_end());
     results[resultNumber] = castBack;
@@ -2924,9 +2925,9 @@ static Value reduce(OpBuilder &builder, Location loc, Value input, Value output,
          "We should have two maps: 1 for the input, 1 for the output");
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
 
-  auto genericOp = linalg::GenericOp::create(builder,
-      loc, output.getType(), input, output, indexingMaps, iteratorTypes,
-      [&](OpBuilder &b, Location loc, ValueRange args) {
+  auto genericOp = linalg::GenericOp::create(
+      builder, loc, output.getType(), input, output, indexingMaps,
+      iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
         Value result = T::create(b, loc, args[0], args[1]);
         linalg::YieldOp::create(b, loc, result);
       });
@@ -2947,9 +2948,10 @@ static Value buildSubAndExpOp(OpBuilder &builder, Location loc, Value input,
   assert(indexingMaps[0].isIdentity() && "input map should be identity");
   // Add the affine map for the output argument.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = linalg::GenericOp::create(builder,
-      loc, input.getType(), ValueRange{input, max}, output, indexingMaps,
-      iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
+  auto genericOp = linalg::GenericOp::create(
+      builder, loc, input.getType(), ValueRange{input, max}, output,
+      indexingMaps, iteratorTypes,
+      [&](OpBuilder &b, Location loc, ValueRange args) {
         Value diff = arith::SubFOp::create(b, loc, args[0], args[1]);
         Value result = math::ExpOp::create(b, loc, diff);
         linalg::YieldOp::create(b, loc, result);
@@ -2974,9 +2976,9 @@ static Value buildDivOp(OpBuilder &builder, Location loc, Value numerator,
   assert(indexingMaps[0].isIdentity() && "Numerator map should be identity");
   // Add the affine map for the output tensor.
   indexingMaps.push_back(indexingMaps[0]);
-  auto genericOp = linalg::GenericOp::create(builder,
-      loc, numerator.getType(), ValueRange{numerator, denominator}, output,
-      indexingMaps, iteratorTypes,
+  auto genericOp = linalg::GenericOp::create(
+      builder, loc, numerator.getType(), ValueRange{numerator, denominator},
+      output, indexingMaps, iteratorTypes,
       [&](OpBuilder &b, Location loc, ValueRange args) {
         Value result = arith::DivFOp::create(b, loc, args[0], args[1]);
         linalg::YieldOp::create(b, loc, result);
@@ -3153,8 +3155,8 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
   int64_t filterRank = getFilterOperandRank();
   SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
   Location loc = getLoc();
-  auto filterSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getFilter(), sliceOffsets, sliceSizes, filterStrides);
+  auto filterSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getFilter(), sliceOffsets, sliceSizes, filterStrides);
   tiledOperands.emplace_back(filterSlice);
 
   SmallVector<OpFoldResult> resultOffsets, resultSizes;
@@ -3164,8 +3166,8 @@ FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getOutput(), resultOffsets, resultSizes, outputStrides);
+  auto outputSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
   SmallVector<Type> resultTypes;
@@ -3333,8 +3335,8 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
       {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
   int64_t inputRank = getInputOperandRank();
   SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
-  auto inputSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getInput(), sliceOffsets, sliceSizes, inputStrides);
+  auto inputSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getInput(), sliceOffsets, sliceSizes, inputStrides);
   tiledOperands.emplace_back(inputSlice);
 
   SmallVector<OpFoldResult> resultOffsets, resultSizes;
@@ -3344,8 +3346,8 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
-  auto outputSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getOutput(), resultOffsets, resultSizes, outputStrides);
+  auto outputSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getOutput(), resultOffsets, resultSizes, outputStrides);
   tiledOperands.emplace_back(outputSlice);
 
   SmallVector<Type> resultTypes;
@@ -3504,8 +3506,8 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
                      sizes[getValueFDim()]});
   int64_t valueRank = getValueOperandRank();
   SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
-  auto valueSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getValue(), sliceOffsets, sliceSizes, sliceStrides);
+  auto valueSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getValue(), sliceOffsets, sliceSizes, sliceStrides);
   tiledOperands.emplace_back(valueSlice);
 
   SmallVector<OpFoldResult> resultOffsets, resultSizes;
@@ -3515,8 +3517,8 @@ FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
 
   int64_t outputRank = getOutputOperandRank();
   SmallVector<OpFoldResult> strides(outputRank, oneAttr);
-  auto outputSlice = tensor::ExtractSliceOp::create(builder,
-      loc, getOutput(), resultOffsets, resultSizes, strides);
+  auto outputSlice = tensor::ExtractSliceOp::create(
+      builder, loc, getOutput(), resultOffsets, resultSizes, strides);
   tiledOperands.emplace_back(outputSlice);
 
   SmallVector<Type> resultTypes;
@@ -4997,8 +4999,8 @@ PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
       createDestinationTensor(b, loc, getSource(), metadata.innerTiles,
                               metadata.innerDimsPos, metadata.outerDimsPerm);
   return PackOp::create(b, loc, getSource(), transposedDest,
-                          metadata.innerDimsPos, metadata.innerTiles,
-                          getPaddingValue(), metadata.outerDimsPerm);
+                        metadata.innerDimsPos, metadata.innerTiles,
+                        getPaddingValue(), metadata.outerDimsPerm);
 }
 
 /// Returns true if the tiles and the tiled dims are constant.
@@ -5250,18 +5252,20 @@ struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    PackOp newOp = PackOp::create(rewriter,
-        op.getLoc(), newOperands[0], newOperands[1], op.getInnerDimsPos(),
-        newMixedTileSizes, op.getPaddingValue(), op.getOuterDimsPerm());
+    PackOp newOp =
+        PackOp::create(rewriter, op.getLoc(), newOperands[0], newOperands[1],
+                       op.getInnerDimsPos(), newMixedTileSizes,
+                       op.getPaddingValue(), op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
 
     // Replace op.
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
-    Value replacement = (newResult.getType() != oldResult.getType())
-                            ? tensor::CastOp::create(rewriter,
-                                  op->getLoc(), oldResult.getType(), newResult)
-                            : newResult;
+    Value replacement =
+        (newResult.getType() != oldResult.getType())
+            ? tensor::CastOp::create(rewriter, op->getLoc(),
+                                     oldResult.getType(), newResult)
+            : newResult;
 
     rewriter.replaceOp(op, {replacement});
 
@@ -5358,7 +5362,8 @@ Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
   for (auto i :
        llvm::seq<unsigned>(0, srcType.getRank() - innerTileSizes.size())) {
     if (srcType.isDynamicDim(i))
-      mixedSizes.push_back(tensor::DimOp::create(b, loc, source, i).getResult());
+      mixedSizes.push_back(
+          tensor::DimOp::create(b, loc, source, i).getResult());
     else
       mixedSizes.push_back(b.getIndexAttr(srcType.getDimSize(i)));
   }
@@ -5381,8 +5386,8 @@ UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
   PackOrUnPackTransposeResult metadata = commonPermutationOfPackAndUnPackOp(
       *this, innerPermutation, outerPermutation);
   return UnPackOp::create(b, loc, transposedSource, getDest(),
-                            metadata.innerDimsPos, metadata.innerTiles,
-                            metadata.outerDimsPerm);
+                          metadata.innerDimsPos, metadata.innerTiles,
+                          metadata.outerDimsPerm);
 }
 
 /// Returns true if the `srcShape` or `destShape` is different from the one in
@@ -5454,8 +5459,8 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
             extractSliceUser.getResultType().getRank()) {
       OpBuilder::InsertionGuard g(rewriter);
       rewriter.setInsertionPoint(unPackOp);
-      auto newDest = tensor::ExtractSliceOp::create(rewriter,
-          unPackOp->getLoc(), unPackOp.getDest(),
+      auto newDest = tensor::ExtractSliceOp::create(
+          rewriter, unPackOp->getLoc(), unPackOp.getDest(),
           extractSliceUser.getMixedOffsets(), extractSliceUser.getMixedSizes(),
           extractSliceUser.getMixedStrides());
       rewriter.modifyOpInPlace(unPackOp, [&]() {
@@ -5475,17 +5480,17 @@ LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
     if (srcShape != unPackOp.getSourceType().getShape()) {
       auto newSrcType = unPackOp.getSourceType().clone(srcShape);
       source = tensor::CastOp::create(rewriter, loc, newSrcType,
-                                               unPackOp.getSource());
+                                      unPackOp.getSource());
     }
     Value dest = unPackOp.getDest();
     if (destShape != unPackOp.getDestType().getShape()) {
       auto newDestType = unPackOp.getDestType().clone(destShape);
-      dest =
-          tensor::CastOp::create(rewriter, loc, newDestType, unPackOp.getDest());
+      dest = tensor::CastOp::create(rewriter, loc, newDestType,
+                                    unPackOp.getDest());
     }
-    Value newOp = UnPackOp::create(rewriter,
-        loc, source, dest, unPackOp.getInnerDimsPos(), unPackOp.getMixedTiles(),
-        unPackOp.getOuterDimsPerm());
+    Value newOp = UnPackOp::create(
+        rewriter, loc, source, dest, unPackOp.getInnerDimsPos(),
+        unPackOp.getMixedTiles(), unPackOp.getOuterDimsPerm());
     rewriter.replaceOpWithNewOp<tensor::CastOp>(
         unPackOp, unPackOp.getResult().getType(), newOp);
     return success();
@@ -5542,18 +5547,19 @@ struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
     // TODO: Strictly speaking, discardable attributes should be _discarded_ at
     // this point. However, in practice, we use them for things that we'd like
     // to preserve. Implement a better abstraction.
-    UnPackOp newOp = UnPackOp::create(rewriter,
-        op.getLoc(), sourceTensor, newOperands[1], op.getInnerDimsPos(),
-        newMixedTileSizes, op.getOuterDimsPerm());
+    UnPackOp newOp = UnPackOp::create(rewriter, op.getLoc(), sourceTensor,
+                                      newOperands[1], op.getInnerDimsPos(),
+                                      newMixedTileSizes, op.getOuterDimsPerm());
     newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
 
     // Replace op.
     Value oldResult = op.getResult();
     Value newResult = newOp.getResult();
-    Value replacement = (newResult.getType() != oldResult.getType())
-                            ? tensor::CastOp::create(rewriter,
-                                  op->getLoc(), oldResult.getType(), newResult)
-                            : newResult;
+    Value replacement =
+        (newResult.getType() != oldResult.getType())
+            ? tensor::CastOp::create(rewriter, op->getLoc(),
+                                     oldResult.getType(), newResult)
+            : newResult;
 
     rewriter.replaceOp(op, {replacement});
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index ba03d09e5072e..9c7099404deef 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -672,9 +672,10 @@ static Operation *replaceForAllWithNewSignature(
   newOuts.push_back(outputs[resultNumber]);
 
   // Create new scf.forall op
-  auto newforallOp = scf::ForallOp::create(rewriter,
-      loc, forallOp.getMixedLowerBound(), forallOp.getMixedUpperBound(),
-      forallOp.getMixedStep(), newOuts, forallOp.getMapping());
+  auto newforallOp = scf::ForallOp::create(
+      rewriter, loc, forallOp.getMixedLowerBound(),
+      forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
+      forallOp.getMapping());
   rewriter.eraseBlock(newforallOp.getBody());
   newforallOp.getRegion().takeBody(forallOp.getRegion());
 
@@ -700,7 +701,7 @@ static Operation *replaceForAllWithNewSignature(
   Value dst = newforallOp.getRegionIterArgs().back();
   SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
   tensor::ParallelInsertSliceOp::create(rewriter, firstYieldOp->getLoc(), src,
-                                                 dst, offsets, sizes, strides);
+                                        dst, offsets, sizes, strides);
 
   for (auto result : llvm::enumerate(forallOp.getResults())) {
     rewriter.replaceAllUsesWith(result.value(),
@@ -3410,8 +3411,8 @@ transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
         for (auto [ofrIdx, ofr] : llvm::enumerate(getMixedSizes())) {
           if (auto attr = llvm::dyn_cast_if_present<Attribute>(ofr)) {
             if (scalableSizes[ofrIdx]) {
-              auto val = arith::ConstantIndexOp::create(b,
-                  getLoc(), cast<IntegerAttr>(attr).getInt());
+              auto val = arith::ConstantIndexOp::create(
+                  b, getLoc(), cast<IntegerAttr>(attr).getInt());
               Value vscale =
                   vector::VectorScaleOp::create(b, getLoc(), b.getIndexType());
               sizes.push_back(
@@ -3626,9 +3627,10 @@ static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter,
   SmallVector<OpFoldResult> normalizedSteps(normalizedUbs.size(),
                                             rewriter.getIndexAttr(1));
 
-  auto normalizedForallOp = scf::ForallOp::create(rewriter,
-      loc, normalizedLbs, normalizedUbs, normalizedSteps, loop.getOutputs(),
-      loop.getMapping(), [](OpBuilder &, Location, ValueRange) {});
+  auto normalizedForallOp = scf::ForallOp::create(
+      rewriter, loc, normalizedLbs, normalizedUbs, normalizedSteps,
+      loop.getOutputs(), loop.getMapping(),
+      [](OpBuilder &, Location, ValueRange) {});
 
   auto normalizedLoopIvs = normalizedForallOp.getInductionVars();
   OpBuilder::InsertionGuard g(rewriter);
@@ -4128,8 +4130,8 @@ DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target,
         target->template getParentOfType<scf::InParallelOp>());
   }
 
-  Value extracted = tensor::ExtractSliceOp::create(rewriter,
-      target.getLoc(), target.getDest(), target.getMixedOffsets(),
+  Value extracted = tensor::ExtractSliceOp::create(
+      rewriter, target.getLoc(), target.getDest(), target.getMixedOffsets(),
       target.getMixedSizes(), target.getMixedStrides());
   Value copied = rewriter
                      .create<linalg::CopyOp>(target.getLoc(),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
index 2cb8ad367f93c..3512ecd9d2eb2 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -185,8 +185,8 @@ struct SoftmaxOpInterface
     if (failed(outputBuffer))
       return failure();
     linalg::SoftmaxOp::create(rewriter, softmaxOp.getLoc(),
-                                       /*result=*/TypeRange(), *inputBuffer,
-                                       *outputBuffer, softmaxOp.getDimension());
+                              /*result=*/TypeRange(), *inputBuffer,
+                              *outputBuffer, softmaxOp.getDimension());
     replaceOpWithBufferizedValues(rewriter, op, *outputBuffer);
     return success();
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index 61c779850366f..d1eb27099db61 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -115,18 +115,18 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0, 1, 2}, {3}};
   auto reshapedFilterType =
       RankedTensorType::get({fh * fw * ic, oc}, filterType.getElementType());
-  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedFilterType, filter, filterReassocIndices);
+  Value reshapedFilter = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedOutputType, output, outputReassocIndices);
+  Value reshapedOutput = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = tensor::EmptyOp::create(rewriter,
-      loc, colTensorShape, inputType.getElementType());
+  Value colTensor = tensor::EmptyOp::create(rewriter, loc, colTensorShape,
+                                            inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
   auto nloops = colTensorShape.size();
@@ -138,8 +138,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = linalg::GenericOp::create(rewriter,
-      loc, colTensor.getType(),
+  auto img2ColTensor = linalg::GenericOp::create(
+      rewriter, loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -170,8 +170,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
-            loc, input, extractionIndices);
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder, loc, input,
+                                                   extractionIndices);
         linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
@@ -187,8 +187,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, reshapedOutputType,
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
       ArrayRef<AffineMap>{lhsMap, rhsMap, resultMap}, genericIterators,
@@ -200,8 +200,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp) {
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
-      loc, outputType, result, outputReassocIndices);
+  auto reshapedResult = tensor::ExpandShapeOp::create(
+      rewriter, loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
 
@@ -244,8 +244,8 @@ rewriteInIm2Col(RewriterBase &rewriter,
     SmallVector<int64_t> targetShape = llvm::to_vector<4>(llvm::map_range(
         indices, [&](int64_t index) -> int64_t { return inputShape[index]; }));
 
-    Value outputTensor = tensor::EmptyOp::create(rewriter,
-        loc, targetShape, operandTensorType.getElementType());
+    Value outputTensor = tensor::EmptyOp::create(
+        rewriter, loc, targetShape, operandTensorType.getElementType());
 
     SmallVector<utils::IteratorType> loopAttributeTypes(
         nloops, utils::IteratorType::parallel);
@@ -255,8 +255,8 @@ rewriteInIm2Col(RewriterBase &rewriter,
             AffineMap::get(nloops, 0, exprs, rewriter.getContext())),
         AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-    auto transposedOp = linalg::GenericOp::create(rewriter,
-        loc, outputTensor.getType(),
+    auto transposedOp = linalg::GenericOp::create(
+        rewriter, loc, outputTensor.getType(),
         /*inputs=*/operand, /*outputs=*/outputTensor, indexingMaps,
         loopAttributeTypes,
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -307,11 +307,11 @@ rewriteInIm2Col(RewriterBase &rewriter,
       AffineMap::get(nloops, 0, inputExprs, rewriter.getContext()),
       AffineMap::getMultiDimIdentityMap(nloops, rewriter.getContext())};
 
-  Value colTensor = tensor::EmptyOp::create(rewriter,
-      loc, colTensorShape, inputType.getElementType());
+  Value colTensor = tensor::EmptyOp::create(rewriter, loc, colTensorShape,
+                                            inputType.getElementType());
 
-  auto img2ColTensor = linalg::GenericOp::create(rewriter,
-      loc, colTensor.getType(),
+  auto img2ColTensor = linalg::GenericOp::create(
+      rewriter, loc, colTensor.getType(),
       /*inputs=*/inputT, /*outputs=*/colTensor, indexingMaps,
       loopAttributeTypes,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -331,26 +331,27 @@ rewriteInIm2Col(RewriterBase &rewriter,
   auto reshapedOutputTensorType =
       RankedTensorType::get({n * c, oh * ow}, outputType.getElementType());
 
-  Value reshapedImg2ColTensor = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedImg2ColTensorType, img2ColTensor.getResult(0),
+  Value reshapedImg2ColTensor = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedImg2ColTensorType, img2ColTensor.getResult(0),
       img2ColTensorReassocIndices);
-  Value reshapedFilterTensor = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedFilterTensorType, filterT, filterReassociationIndice);
-  Value reshapedoutputTensor = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedOutputTensorType, transposedOutputTensor,
+  Value reshapedFilterTensor =
+      tensor::CollapseShapeOp::create(rewriter, loc, reshapedFilterTensorType,
+                                      filterT, filterReassociationIndice);
+  Value reshapedoutputTensor = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedOutputTensorType, transposedOutputTensor,
       outputReassociationIndice);
 
-  auto batchMatVecResult = linalg::BatchMatvecOp::create(rewriter,
-      loc, TypeRange{reshapedoutputTensor.getType()},
+  auto batchMatVecResult = linalg::BatchMatvecOp::create(
+      rewriter, loc, TypeRange{reshapedoutputTensor.getType()},
       ValueRange{reshapedImg2ColTensor, reshapedFilterTensor},
       ValueRange{reshapedoutputTensor});
 
   SmallVector<ReassociationIndices> batchMatVecReassociationIndice = {{0, 1},
                                                                       {2, 3}};
 
-  auto batchMatVecResultReshaped = tensor::ExpandShapeOp::create(rewriter,
-      loc, transposedOutputTensor.getType(), batchMatVecResult.getResult(0),
-      batchMatVecReassociationIndice);
+  auto batchMatVecResultReshaped = tensor::ExpandShapeOp::create(
+      rewriter, loc, transposedOutputTensor.getType(),
+      batchMatVecResult.getResult(0), batchMatVecReassociationIndice);
 
   Value transposedResult =
       transposeOperand(batchMatVecResultReshaped, {0, 2, 3, 1});
@@ -400,19 +401,19 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, ic * fh * fw}, inputType.getElementType());
-  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedFilterType, filter, filterReassocIndices);
+  Value reshapedFilter = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1}, {2, 3}};
   auto reshapedOutputType =
       RankedTensorType::get({n, oc, oh * ow}, outputType.getElementType());
-  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedOutputType, output, outputReassocIndices);
+  Value reshapedOutput = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedOutputType, output, outputReassocIndices);
 
   // Convert the input to a (BKN) tensor.
   SmallVector<int64_t, 4> colTensorShape = {n, ic * fh * fw, oh * ow};
-  Value colTensor = tensor::EmptyOp::create(rewriter,
-      loc, colTensorShape, inputType.getElementType());
+  Value colTensor = tensor::EmptyOp::create(rewriter, loc, colTensorShape,
+                                            inputType.getElementType());
 
   auto nloops = colTensorShape.size();
 
@@ -423,8 +424,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   SmallVector<AffineMap, 4> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = linalg::GenericOp::create(rewriter,
-      loc, colTensor.getType(),
+  auto img2ColTensor = linalg::GenericOp::create(
+      rewriter, loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -455,8 +456,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
 
         // im2col[n, ic*fh*fw, oh*ow] = input[n, ic, sh*oh + fh, sw*ow + fw]
         SmallVector<Value> extractionIndices{bIndex, icIndex, hIndex, wIndex};
-        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
-            loc, input, extractionIndices);
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder, loc, input,
+                                                   extractionIndices);
         linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
@@ -471,8 +472,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
   auto resultMap = AffineMap::get(4, 0, {bDim, mDim, nDim}, context);
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, reshapedOutputType,
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, reshapedOutputType,
       /*inputs=*/ValueRange{reshapedFilter, img2ColTensor.getResult(0)},
       /*outputs=*/ValueRange{reshapedOutput},
       ArrayRef<AffineMap>{lhsMap, rhsMap, resultMap}, genericIterators,
@@ -484,8 +485,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
-      loc, outputType, result, outputReassocIndices);
+  auto reshapedResult = tensor::ExpandShapeOp::create(
+      rewriter, loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
 
@@ -535,18 +536,18 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
   auto reshapedFilterType =
       RankedTensorType::get({oc, fh * fw * ic}, filterType.getElementType());
-  Value reshapedFilter = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedFilterType, filter, filterReassocIndices);
+  Value reshapedFilter = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedFilterType, filter, filterReassocIndices);
 
   SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
   RankedTensorType reshapedOutputType =
       RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
-  Value reshapedOutput = tensor::CollapseShapeOp::create(rewriter,
-      loc, reshapedOutputType, output, outputReassocIndices);
+  Value reshapedOutput = tensor::CollapseShapeOp::create(
+      rewriter, loc, reshapedOutputType, output, outputReassocIndices);
 
   SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
-  Value colTensor = tensor::EmptyOp::create(rewriter,
-      loc, colTensorShape, inputType.getElementType());
+  Value colTensor = tensor::EmptyOp::create(rewriter, loc, colTensorShape,
+                                            inputType.getElementType());
 
   // Convert the input to a (BMK) column tensor.
   auto nloops = colTensorShape.size();
@@ -558,8 +559,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<AffineMap> img2colIndexingMaps = {
       AffineMap::getMultiDimIdentityMap(nloops, context)};
 
-  auto img2ColTensor = linalg::GenericOp::create(rewriter,
-      loc, colTensor.getType(),
+  auto img2ColTensor = linalg::GenericOp::create(
+      rewriter, loc, colTensor.getType(),
       /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
       img2colIterators,
       [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
@@ -590,8 +591,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
 
         // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
         SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
-        Value inputVal = tensor::ExtractOp::create(nestedBuilder,
-            loc, input, extractionIndices);
+        Value inputVal = tensor::ExtractOp::create(nestedBuilder, loc, input,
+                                                   extractionIndices);
         linalg::YieldOp::create(nestedBuilder, nestedLoc, inputVal);
       });
 
@@ -606,8 +607,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
   SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
                                                        parallel, reduction};
 
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, reshapedOutputType,
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, reshapedOutputType,
       /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
       /*outputs=*/ValueRange{reshapedOutput},
       ArrayRef<AffineMap>{lhsMap, rhsMap, resultMap}, genericIterators,
@@ -619,8 +620,8 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
       });
   Value result = genericOp.getResults().front();
 
-  auto reshapedResult = tensor::ExpandShapeOp::create(rewriter,
-      loc, outputType, result, outputReassocIndices);
+  auto reshapedResult = tensor::ExpandShapeOp::create(
+      rewriter, loc, outputType, result, outputReassocIndices);
 
   rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index f21563c323c1b..76ddee4f0e9cf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -38,7 +38,7 @@ static Value createInserts(RewriterBase &rewriter, Location loc, int dim,
     for (int i = 0; i < shape.back(); ++i) {
       indices.back() = constants[i];
       destination = tensor::InsertOp::create(rewriter, loc, *elementIt,
-                                                      destination, indices);
+                                             destination, indices);
       ++elementIt;
     }
     return destination;
@@ -65,16 +65,16 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
       MaterializeInDestination: {
     // Note: This is the preferred way of memcpy'ing because no layout map
     // and/or memory space must be specified for the source.
-    auto materializeOp = bufferization::MaterializeInDestinationOp::create(b,
-        loc, tensorSource, memrefDest);
+    auto materializeOp = bufferization::MaterializeInDestinationOp::create(
+        b, loc, tensorSource, memrefDest);
     materializeOp.setWritable(true);
   } break;
   case linalg::BufferizeToAllocationOptions::MemcpyOp::MemrefCopy: {
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = bufferization::ToBufferOp::create(b,
-        loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
+    Value toBuffer = bufferization::ToBufferOp::create(
+        b, loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
     memref::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
@@ -82,8 +82,8 @@ static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
     // TODO: Support custom memory space on source.
     // We do not know the layout map of the source yet, so use a fully dynamic
     // layout for best compatibility.
-    Value toBuffer = bufferization::ToBufferOp::create(b,
-        loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
+    Value toBuffer = bufferization::ToBufferOp::create(
+        b, loc, bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
         tensorSource, /*readOnly=*/true);
     linalg::CopyOp::create(b, loc, toBuffer, memrefDest);
   } break;
@@ -121,14 +121,14 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
                                   yieldedValue.getType(), yieldedValue.getLoc())
             ->getResult(0);
     auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(fillValue),
-                                                  ValueRange(dest));
+                                         ValueRange(dest));
     return fillOp;
   }
 
   if (invariantYieldedValue) {
     // Padding with an invariant value.
-    auto fillOp = linalg::FillOp::create(rewriter, loc, ValueRange(yieldedValue),
-                                                  ValueRange(dest));
+    auto fillOp = linalg::FillOp::create(
+        rewriter, loc, ValueRange(yieldedValue), ValueRange(dest));
     return fillOp;
   }
 
@@ -137,8 +137,8 @@ static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(resultType.getRank()));
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, resultType, /*inputs=*/ValueRange(),
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, resultType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{dest}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
   Block *body = rewriter.createBlock(&genericOp->getRegion(0), {},
@@ -180,7 +180,7 @@ static SmallVector<Value> reifyOrComputeDynamicSizes(OpBuilder &b,
     if (tensorType.isDynamicDim(i))
       dynSizes.push_back(
           DimOp::create(b, value.getLoc(), value,
-                          arith::ConstantIndexOp::create(b, value.getLoc(), i)));
+                        arith::ConstantIndexOp::create(b, value.getLoc(), i)));
   }
   return dynSizes;
 }
@@ -243,14 +243,14 @@ Value linalg::bufferizeToAllocation(
       getMixedSizes(rewriter, loc, padOp.getSource());
   SmallVector<OpFoldResult> strides(padOp.getResultType().getRank(),
                                     rewriter.getIndexAttr(1));
-  Value subview = memref::SubViewOp::create(rewriter,
-      loc, alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
+  Value subview = memref::SubViewOp::create(
+      rewriter, loc, alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
   createMemcpy(rewriter, loc, padOp.getSource(), subview, options);
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
-      loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
+  Value toTensorOp = bufferization::ToTensorOp::create(
+      rewriter, loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(padOp, toTensorOp);
   return alloc;
@@ -338,8 +338,9 @@ Value linalg::bufferizeToAllocation(
 
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
-  Value toTensorOp = bufferization::ToTensorOp::create(rewriter,
-      loc, allocTensorOp.getResult().getType(), alloc, /*restrict=*/true,
+  Value toTensorOp = bufferization::ToTensorOp::create(
+      rewriter, loc, allocTensorOp.getResult().getType(), alloc,
+      /*restrict=*/true,
       /*writable=*/true);
   rewriter.replaceOp(allocTensorOp, toTensorOp);
   return alloc;
@@ -394,16 +395,16 @@ mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
   RankedTensorType tensorType = cast<RankedTensorType>(generateOp.getType());
 
   // Create tensor.empty.
-  auto emptyOp =
-      EmptyOp::create(rewriter, loc, tensorType, generateOp.getDynamicExtents());
+  auto emptyOp = EmptyOp::create(rewriter, loc, tensorType,
+                                 generateOp.getDynamicExtents());
 
   // Create linalg.generic.
   SmallVector<utils::IteratorType> iteratorTypes(tensorType.getRank(),
                                                  utils::IteratorType::parallel);
   SmallVector<AffineMap> indexingMaps(
       1, rewriter.getMultiDimIdentityMap(tensorType.getRank()));
-  auto genericOp = linalg::GenericOp::create(rewriter,
-      loc, tensorType, /*inputs=*/ValueRange(),
+  auto genericOp = linalg::GenericOp::create(
+      rewriter, loc, tensorType, /*inputs=*/ValueRange(),
       /*outputs=*/ValueRange{emptyOp.getResult()}, /*indexingMaps=*/
       indexingMaps, iteratorTypes);
   Block *body = rewriter.createBlock(&genericOp->getRegion(0), {},
@@ -567,8 +568,8 @@ Value linalg::bufferizeToAllocation(
       createMemcpy(rewriter, op->getLoc(), operand->get(), alloc, options);
     }
     rewriter.modifyOpInPlace(op, [&]() {
-      auto toTensorOp = ToTensorOp::create(rewriter,
-          op->getLoc(), operand->get().getType(), alloc);
+      auto toTensorOp = ToTensorOp::create(rewriter, op->getLoc(),
+                                           operand->get().getType(), alloc);
       operand->set(toTensorOp);
       if (options.bufferizeDestinationOnly) {
         rewriter.modifyOpInPlace(toTensorOp, [&]() {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index eb2c7756dc6ae..91a297f7b9db7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -287,8 +287,8 @@ getOrCreatePackedViewOfOperand(OpBuilder &b, Location loc, PackInfo packInfo,
 
   auto empty = linalg::PackOp::createDestinationTensor(
       b, loc, opOperand->get(), innerTileSizes, innerDimsPos, outerDimsPerm);
-  auto packedOperand = linalg::PackOp::create(b,
-      loc, opOperand->get(), empty, innerDimsPos, innerTileSizes,
+  auto packedOperand = linalg::PackOp::create(
+      b, loc, opOperand->get(), empty, innerDimsPos, innerTileSizes,
       /*padding=*/std::nullopt, outerDimsPerm);
   return std::make_tuple(packedOperand, indexingMap);
 }
@@ -345,8 +345,9 @@ static GenericOp packGenericOp(RewriterBase &rewriter, GenericOp genericOp,
 
   indexingMaps.push_back(packedOutIndexingMap);
 
-  auto newGenericOp = linalg::GenericOp::create(rewriter,
-      loc, dest.getType(), inputOperands, dest, indexingMaps, iterTypes,
+  auto newGenericOp = linalg::GenericOp::create(
+      rewriter, loc, dest.getType(), inputOperands, dest, indexingMaps,
+      iterTypes,
       /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
   rewriter.cloneRegionBefore(genericOp.getRegion(), newGenericOp.getRegion(),
                              newGenericOp.getRegion().begin());
@@ -457,9 +458,9 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, linalg::PackOp packOp,
   if (!packOpDest.hasOneUse())
     return failure();
   if (auto emptyOp = packOpDest.getDefiningOp<tensor::EmptyOp>()) {
-    packOpDest = tensor::EmptyOp::create(rewriter,
-        genericOp->getLoc(), emptyOp.getMixedSizes(),
-        emptyOp.getType().getElementType());
+    packOpDest = tensor::EmptyOp::create(rewriter, genericOp->getLoc(),
+                                         emptyOp.getMixedSizes(),
+                                         emptyOp.getType().getElementType());
   } else {
     DominanceInfo dom(genericOp);
     if (!dom.properlyDominates(packOpDest, genericOp))
@@ -562,8 +563,8 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     auto empty = linalg::PackOp::createDestinationTensor(
         rewriter, loc, padOp.getSource(), mixedTiles, innerDimsPos,
         outerDimsPerm);
-    auto sourcePack = linalg::PackOp::create(rewriter,
-        loc, padOp.getSource(), empty, innerDimsPos, mixedTiles,
+    auto sourcePack = linalg::PackOp::create(
+        rewriter, loc, padOp.getSource(), empty, innerDimsPos, mixedTiles,
         /*padding=*/std::nullopt, outerDimsPerm);
 
     // If we have `outer_dims_perms` we need to adjust the padded dimensions.
@@ -579,17 +580,18 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern<linalg::PackOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, /*result=*/Type(), sourcePack, lowPad, highPad, paddingVal,
-        padOp.getNofold());
+    auto newPadOp =
+        tensor::PadOp::create(rewriter, loc, /*result=*/Type(), sourcePack,
+                              lowPad, highPad, paddingVal, padOp.getNofold());
 
     // If the pad has more than one user, create an unpack on the new pad to
     // replace the other uses.
     if (!padOp->hasOneUse()) {
       auto unpackEmpty = linalg::UnPackOp::createDestinationTensor(
           rewriter, loc, newPadOp, mixedTiles, innerDimsPos, outerDimsPerm);
-      Value unpackedPad = linalg::UnPackOp::create(rewriter,
-          loc, newPadOp, unpackEmpty, innerDimsPos, mixedTiles, outerDimsPerm);
+      Value unpackedPad =
+          linalg::UnPackOp::create(rewriter, loc, newPadOp, unpackEmpty,
+                                   innerDimsPos, mixedTiles, outerDimsPerm);
       rewriter.replaceAllUsesExcept(padOp, unpackedPad, sourcePack);
     }
 
@@ -719,9 +721,10 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
   auto emptyOp = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), collapseOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newPackOp = linalg::PackOp::create(rewriter,
-      packOp.getLoc(), collapseOp.getSrc(), emptyOp, projectedInnerDimsPos,
-      packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPerm);
+  auto newPackOp = linalg::PackOp::create(
+      rewriter, packOp.getLoc(), collapseOp.getSrc(), emptyOp,
+      projectedInnerDimsPos, packOp.getMixedTiles(), packOp.getPaddingValue(),
+      newOuterDimsPerm);
 
   SmallVector<ReassociationIndices> newReassocIndices = reassocIndices;
   // First apply the permutation on the reassociations of the outer dims.
@@ -735,8 +738,9 @@ bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp,
     nextPos += 1;
   }
 
-  auto newCollapseOp = tensor::CollapseShapeOp::create(rewriter,
-      collapseOp.getLoc(), packOp.getType(), newPackOp, newReassocIndices);
+  auto newCollapseOp = tensor::CollapseShapeOp::create(
+      rewriter, collapseOp.getLoc(), packOp.getType(), newPackOp,
+      newReassocIndices);
   rewriter.replaceOp(packOp, newCollapseOp);
 
   return success();
@@ -853,13 +857,14 @@ bubbleUpPackOpThroughExpandShape(tensor::ExpandShapeOp expandOp,
   Value destTensor = linalg::PackOp::createDestinationTensor(
       rewriter, packOp.getLoc(), expandOp.getSrc(), packOp.getMixedTiles(),
       projectedInnerDimsPos, /*outerDimsPerm=*/SmallVector<int64_t>{});
-  Value packedVal = linalg::PackOp::create(rewriter,
-      packOp.getLoc(), expandOp.getSrc(), destTensor, projectedInnerDimsPos,
-      packOp.getMixedTiles(), packOp.getPaddingValue(),
+  Value packedVal = linalg::PackOp::create(
+      rewriter, packOp.getLoc(), expandOp.getSrc(), destTensor,
+      projectedInnerDimsPos, packOp.getMixedTiles(), packOp.getPaddingValue(),
       /*outerDimsPerm=*/SmallVector<int64_t>{});
 
-  Value newExpandOp = tensor::ExpandShapeOp::create(rewriter,
-      packOp.getLoc(), packOp.getDestType(), packedVal, *reassocExpand);
+  Value newExpandOp = tensor::ExpandShapeOp::create(rewriter, packOp.getLoc(),
+                                                    packOp.getDestType(),
+                                                    packedVal, *reassocExpand);
   rewriter.replaceOp(packOp, newExpandOp);
 
   return success();
@@ -972,15 +977,15 @@ static LogicalResult pushDownUnPackOpThroughExpandShape(
 
   RankedTensorType newExpandType = linalg::PackOp::inferPackedType(
       expandTy, innerTileSizes, projectedInnerDimsPos, newOuterDimsPerm);
-  auto newExpandOp = tensor::ExpandShapeOp::create(rewriter,
-      expandOp.getLoc(), newExpandType, unPackOp.getSource(),
-      newReassocIndices);
+  auto newExpandOp =
+      tensor::ExpandShapeOp::create(rewriter, expandOp.getLoc(), newExpandType,
+                                    unPackOp.getSource(), newReassocIndices);
 
   auto emptyOp = linalg::UnPackOp::createDestinationTensor(
       rewriter, unPackOp.getLoc(), newExpandOp, unPackOp.getMixedTiles(),
       projectedInnerDimsPos, newOuterDimsPerm);
-  auto newUnPackOp = linalg::UnPackOp::create(rewriter,
-      unPackOp.getLoc(), newExpandOp.getResult(), emptyOp,
+  auto newUnPackOp = linalg::UnPackOp::create(
+      rewriter, unPackOp.getLoc(), newExpandOp.getResult(), emptyOp,
       projectedInnerDimsPos, unPackOp.getMixedTiles(), newOuterDimsPerm);
   rewriter.replaceOp(expandOp, newUnPackOp);
 
@@ -1212,17 +1217,17 @@ struct PushDownUnPackThroughPadOp : public OpRewritePattern<tensor::PadOp> {
     lowPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
     highPad.append(pointLoopsSize, rewriter.getIndexAttr(0));
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, /*result=*/Type(), unpackOp.getSource(), lowPad, highPad,
-        paddingVal, padOp.getNofold());
+    auto newPadOp = tensor::PadOp::create(rewriter, loc, /*result=*/Type(),
+                                          unpackOp.getSource(), lowPad, highPad,
+                                          paddingVal, padOp.getNofold());
 
     // Inject the linalg.unpack right after the packed padOp.
-    Value outputUnPack = tensor::EmptyOp::create(rewriter,
-        loc, padOp.getResultType().getShape(),
-        padOp.getResultType().getElementType());
+    Value outputUnPack =
+        tensor::EmptyOp::create(rewriter, loc, padOp.getResultType().getShape(),
+                                padOp.getResultType().getElementType());
 
-    Value replacement = linalg::UnPackOp::create(rewriter,
-        loc, newPadOp.getResult(), outputUnPack, innerDimsPos,
+    Value replacement = linalg::UnPackOp::create(
+        rewriter, loc, newPadOp.getResult(), outputUnPack, innerDimsPos,
         unpackOp.getMixedTiles(), outerDimsPerm);
     rewriter.replaceOp(padOp, replacement);
     return success();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
index 82d386b3b9099..b7da20c108f3e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
@@ -201,7 +201,7 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
           tensor::EmptyOp::create(rewriter, loc, transposedShape, elType);
 
       auto transposeOp = TransposeOp::create(rewriter, loc, newInitValues[i],
-                                                      emptyTensor, permutation);
+                                             emptyTensor, permutation);
       newInitValues[i] = transposeOp->getResult(0);
       isChanged = true;
     }
@@ -209,11 +209,11 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
     // Does it require broadcast?
     if (!broadcastedDims.empty()) {
       assert(broadcastedDims.size() && "should have non size broadcast");
-      Value emptyTensor = tensor::EmptyOp::create(rewriter,
-          loc, outputShape, inputRTType.getElementType());
+      Value emptyTensor = tensor::EmptyOp::create(rewriter, loc, outputShape,
+                                                  inputRTType.getElementType());
 
-      auto broadcastOp = linalg::BroadcastOp::create(rewriter,
-          loc, newInitValues[i], emptyTensor, broadcastedDims);
+      auto broadcastOp = linalg::BroadcastOp::create(
+          rewriter, loc, newInitValues[i], emptyTensor, broadcastedDims);
 
       newInitValues[i] = broadcastOp->getResult(0);
       isChanged = true;
@@ -227,7 +227,8 @@ LogicalResult DecomposeProjectedPermutation::matchAndRewrite(
   SmallVector<Value> operands = op->getOperands();
   ValueRange operandsRef(operands);
 
-  auto newOp = linalg::GenericOp::create(rewriter,
+  auto newOp = linalg::GenericOp::create(
+      rewriter,
       /*location=*/op.getLoc(),
       /*resultTensorTypes=*/op->getResultTypes(),
       /*inputs=*/newInitValues,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
index bd3fad01aa48b..c92a27f0af858 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp
@@ -138,8 +138,8 @@ static Value getZero(OpBuilder &b, Location loc, Type elementType) {
     return arith::ConstantIndexOp::create(b, loc, 0);
   // Assume float.
   auto floatType = cast<FloatType>(elementType);
-  return arith::ConstantFloatOp::create(b,
-      loc, floatType, APFloat::getZero(floatType.getFloatSemantics()));
+  return arith::ConstantFloatOp::create(
+      b, loc, floatType, APFloat::getZero(floatType.getFloatSemantics()));
 }
 
 GenericOp
@@ -188,8 +188,8 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
 
     // Fall back path, use an `init_tensor` and identity indexing map.
     AffineMap indexingMap = rewriter.getMultiDimIdentityMap(domain.size());
-    Value emptyTensor =
-        tensor::EmptyOp::create(rewriter, loc, domain, scalarOpResult.getType());
+    Value emptyTensor = tensor::EmptyOp::create(rewriter, loc, domain,
+                                                scalarOpResult.getType());
     newInitValues.push_back(emptyTensor);
     newResultTypes.push_back(emptyTensor.getType());
     peeledGenericOpIndexingMaps.push_back(indexingMap);
@@ -202,10 +202,10 @@ DecomposeLinalgOp::createPeeledGenericOp(GenericOp genericOp,
   resultTypes.append(newResultTypes.begin(), newResultTypes.end());
   auto indexingMapAttr =
       rewriter.getAffineMapArrayAttr(peeledGenericOpIndexingMaps);
-  return GenericOp::create(rewriter,
-      loc, resultTypes, genericOp.getInputs(), outsOperands, indexingMapAttr,
-      genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
-      [](OpBuilder, Location, ValueRange) {});
+  return GenericOp::create(
+      rewriter, loc, resultTypes, genericOp.getInputs(), outsOperands,
+      indexingMapAttr, genericOp.getIteratorTypes(), /*doc=*/nullptr,
+      /*libraryCall=*/nullptr, [](OpBuilder, Location, ValueRange) {});
 }
 
 GenericOp
@@ -239,8 +239,8 @@ DecomposeLinalgOp::createResidualGenericOp(GenericOp genericOp,
     indexingMaps.push_back(genericOp.getMatchingIndexingMap(&outOperand));
 
   auto indexingMapAttr = rewriter.getAffineMapArrayAttr(indexingMaps);
-  return GenericOp::create(rewriter,
-      genericOp->getLoc(), genericOp->getResultTypes(),
+  return GenericOp::create(
+      rewriter, genericOp->getLoc(), genericOp->getResultTypes(),
       residualGenericOpOperands, genericOp.getOutputs(), indexingMapAttr,
       genericOp.getIteratorTypes(), /*doc=*/nullptr, /*libraryCall=*/nullptr,
       [](OpBuilder, Location, ValueRange) {});
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index b8e73bd5929dd..830905495e759 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -34,8 +34,8 @@ static Value sourceMaterializationCallback(OpBuilder &builder, Type type,
 
   // A detensored value is converted back by creating a new tensor from its
   // element(s).
-  return tensor::FromElementsOp::create(builder,
-      loc, RankedTensorType::get({}, inputType), inputs[0]);
+  return tensor::FromElementsOp::create(
+      builder, loc, RankedTensorType::get({}, inputType), inputs[0]);
 }
 
 namespace {
@@ -480,8 +480,8 @@ struct LinalgDetensorize
     Block *postEntryBlock =
         rewriter.splitBlock(entryBlock, entryBlock->begin());
     rewriter.setInsertionPointToStart(entryBlock);
-    auto branch =
-        cf::BranchOp::create(rewriter, rewriter.getUnknownLoc(), postEntryBlock);
+    auto branch = cf::BranchOp::create(rewriter, rewriter.getUnknownLoc(),
+                                       postEntryBlock);
 
     if (aggressiveMode.getValue()) {
       AggressiveDetensoringModel costModel;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index fd8c177cc7247..7dbae83e993e4 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -118,16 +118,17 @@ struct MoveInitOperandsToInput : public OpRewritePattern<GenericOp> {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointAfterValue(op->get());
       auto elemType = cast<ShapedType>(op->get().getType()).getElementType();
-      auto empty = tensor::EmptyOp::create(rewriter,
-          loc, tensor::getMixedSizes(rewriter, loc, op->get()), elemType);
+      auto empty = tensor::EmptyOp::create(
+          rewriter, loc, tensor::getMixedSizes(rewriter, loc, op->get()),
+          elemType);
 
       unsigned start = genericOp.getDpsInits().getBeginOperandIndex();
       newOutputOperands[op->getOperandNumber() - start] = empty.getResult();
     }
 
-    auto newOp = GenericOp::create(rewriter,
-        loc, genericOp.getResultTypes(), newInputOperands, newOutputOperands,
-        newIndexingMaps, genericOp.getIteratorTypesArray(),
+    auto newOp = GenericOp::create(
+        rewriter, loc, genericOp.getResultTypes(), newInputOperands,
+        newOutputOperands, newIndexingMaps, genericOp.getIteratorTypesArray(),
         /*bodyBuild=*/nullptr, linalg::getPrunedAttributeList(genericOp));
 
     OpBuilder::InsertionGuard guard(rewriter);
@@ -296,7 +297,7 @@ static Value collapseValue(
     auto targetType = MemRefType::get(targetShape, memrefType.getElementType(),
                                       layout, memrefType.getMemorySpace());
     return memref::CollapseShapeOp::create(rewriter, loc, targetType, operand,
-                                                    reassociation);
+                                           reassociation);
   }
   if (auto tensorType = dyn_cast<RankedTensorType>(operand.getType())) {
     if (rankReductionStrategy ==
@@ -315,7 +316,7 @@ static Value collapseValue(
     auto targetType =
         RankedTensorType::get(targetShape, tensorType.getElementType());
     return tensor::CollapseShapeOp::create(rewriter, loc, targetType, operand,
-                                                    reassociation);
+                                           reassociation);
   }
   llvm_unreachable("unsupported operand type");
 }
@@ -520,7 +521,7 @@ linalg::dropUnitDims(RewriterBase &rewriter, GenericOp genericOp,
     resultTypes.push_back(newOutputs[i].getType());
   GenericOp replacementOp =
       GenericOp::create(rewriter, loc, resultTypes, newInputs, newOutputs,
-                                 newIndexingMaps, newIteratorTypes);
+                        newIndexingMaps, newIteratorTypes);
   rewriter.inlineRegionBefore(genericOp.getRegion(), replacementOp.getRegion(),
                               replacementOp.getRegion().begin());
   // 5a. Replace `linalg.index` operations that refer to the dropped unit
@@ -652,8 +653,8 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         collapseValue(rewriter, padOp.getLoc(), padOp.getSource(), newShape,
                       reassociationMap, options.rankReductionStrategy);
 
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        padOp.getLoc(), /*result=*/Type(), collapsedSource, newLowPad,
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, padOp.getLoc(), /*result=*/Type(), collapsedSource, newLowPad,
         newHighPad, paddingVal, padOp.getNofold());
 
     Value dest = padOp.getResult();
@@ -670,9 +671,8 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
         expandedSizes.push_back(tensor::getMixedSize(
             rewriter, padOp.getLoc(), newPadOp, dim - numUnitDims));
       }
-      dest = tensor::EmptyOp::create(rewriter,
-          padOp.getLoc(), expandedSizes,
-          padOp.getResultType().getElementType());
+      dest = tensor::EmptyOp::create(rewriter, padOp.getLoc(), expandedSizes,
+                                     padOp.getResultType().getElementType());
     }
 
     Value expandedValue =
@@ -713,8 +713,9 @@ struct RankReducedExtractSliceOp
             strides));
 
     Location loc = sliceOp.getLoc();
-    Value newSlice = tensor::ExtractSliceOp::create(rewriter,
-        loc, rankReducedType, sliceOp.getSource(), offsets, sizes, strides);
+    Value newSlice = tensor::ExtractSliceOp::create(
+        rewriter, loc, rankReducedType, sliceOp.getSource(), offsets, sizes,
+        strides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSlice, *reassociation);
     return success();
@@ -747,8 +748,8 @@ struct RankReducedInsertSliceOp : public OpRewritePattern<InsertOpTy> {
       // parallel case.
       if (std::is_same<InsertOpTy, tensor::ParallelInsertSliceOp>::value)
         rewriter.setInsertionPoint(insertSliceOp->getParentOp());
-      reshapedSource = tensor::CollapseShapeOp::create(rewriter,
-          loc, insertSliceOp.getSource(), *reassociation);
+      reshapedSource = tensor::CollapseShapeOp::create(
+          rewriter, loc, insertSliceOp.getSource(), *reassociation);
     }
     rewriter.replaceOpWithNewOp<InsertOpTy>(
         insertSliceOp, reshapedSource, insertSliceOp.getDest(),
@@ -898,8 +899,8 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
   /// Expand result tensor.
   Value expandResult(PatternRewriter &rewriter, Value result,
                      RankedTensorType expandedType, int64_t dim) const {
-    return tensor::ExpandShapeOp::create(rewriter,
-        result.getLoc(), expandedType, result,
+    return tensor::ExpandShapeOp::create(
+        rewriter, result.getLoc(), expandedType, result,
         getReassociationForReshapeAtDim(expandedType.getRank(), dim));
   }
 
@@ -934,9 +935,9 @@ struct RankReduceContractionOps : OpRewritePattern<FromOpTy> {
     SmallVector<Type, 1> collapsedResultTy;
     if (isa<RankedTensorType>(collapsedInit.getType()))
       collapsedResultTy.push_back(collapsedInit.getType());
-    auto collapsedOp = ToOpTy::create(rewriter,
-        loc, collapsedResultTy, ValueRange{collapsedLhs, collapsedRhs},
-        ValueRange{collapsedInit});
+    auto collapsedOp = ToOpTy::create(rewriter, loc, collapsedResultTy,
+                                      ValueRange{collapsedLhs, collapsedRhs},
+                                      ValueRange{collapsedInit});
     for (auto attr : contractionOp->getAttrs()) {
       if (attr.getName() == LinalgDialect::kMemoizedIndexingMapsAttrName ||
           attr.getName() == "indexing_maps")
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index 754f7652a776b..4ec4deeeabe66 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -240,8 +240,8 @@ static void generateFusedElementwiseOpRegion(
                     });
     for (IndexOp indexOp :
          llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
-      Value newIndex = affine::AffineApplyOp::create(rewriter,
-          producer.getLoc(),
+      Value newIndex = affine::AffineApplyOp::create(
+          rewriter, producer.getLoc(),
           consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
       mapper.map(indexOp.getResult(), newIndex);
     }
@@ -416,8 +416,8 @@ mlir::linalg::fuseElementwiseOps(RewriterBase &rewriter,
   }
 
   // Generate the fused op.
-  auto fusedOp = GenericOp::create(rewriter,
-      consumer.getLoc(), fusedResultTypes, fusedInputOperands,
+  auto fusedOp = GenericOp::create(
+      rewriter, consumer.getLoc(), fusedResultTypes, fusedInputOperands,
       fusedOutputOperands, rewriter.getAffineMapArrayAttr(fusedIndexMaps),
       consumer.getIteratorTypes(),
       /*doc=*/nullptr,
@@ -797,7 +797,7 @@ static Operation *createExpandedTransposeOp(PatternRewriter &rewriter,
     }
   }
   return TransposeOp::create(rewriter, transposeOp.getLoc(), expandedInput,
-                                      output, invertPermutationVector(newPerm));
+                             output, invertPermutationVector(newPerm));
 }
 
 // Create an expanded generic op.
@@ -813,9 +813,9 @@ static Operation *createExpandedGenericOp(
     for (auto j : expansionInfo.getExpandedDims(i))
       iteratorTypes[j] = type;
 
-  Operation *fused = GenericOp::create(rewriter,
-      linalgOp.getLoc(), resultTypes, expandedOpOperands, outputs,
-      expandedOpIndexingMaps, iteratorTypes);
+  Operation *fused = GenericOp::create(rewriter, linalgOp.getLoc(), resultTypes,
+                                       expandedOpOperands, outputs,
+                                       expandedOpIndexingMaps, iteratorTypes);
 
   Region &fusedRegion = fused->getRegion(0);
   Region &originalRegion = linalgOp->getRegion(0);
@@ -933,8 +933,8 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
                 reassociation,
                 /*isExpandingReshape=*/true)))
           return std::nullopt;
-        expandedOpOperands.push_back(tensor::ExpandShapeOp::create(rewriter,
-            loc, expandedOperandType, opOperand->get(), reassociation,
+        expandedOpOperands.push_back(tensor::ExpandShapeOp::create(
+            rewriter, loc, expandedOperandType, opOperand->get(), reassociation,
             expandedOperandShape));
         continue;
       }
@@ -961,8 +961,8 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               reassociation,
               /*isExpandingReshape=*/true)))
         return std::nullopt;
-      outputs.push_back(tensor::ExpandShapeOp::create(rewriter,
-          loc, expandedOutputType, opOperand.get(), reassociation,
+      outputs.push_back(tensor::ExpandShapeOp::create(
+          rewriter, loc, expandedOutputType, opOperand.get(), reassociation,
           expandedOutputShape));
     } else {
       outputs.push_back(opOperand.get());
@@ -984,8 +984,8 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
               linalgOp.getMatchingIndexingMap(
                   linalgOp.getDpsInitOperand(resultNumber)),
               expansionInfo);
-      resultVals.push_back(tensor::CollapseShapeOp::create(rewriter,
-          linalgOp.getLoc(), opResult.getType(),
+      resultVals.push_back(tensor::CollapseShapeOp::create(
+          rewriter, linalgOp.getLoc(), opResult.getType(),
           fusedOp->getResult(resultNumber), reassociation));
     } else {
       resultVals.push_back(fusedOp->getResult(resultNumber));
@@ -1086,8 +1086,8 @@ class FoldPadWithProducerReshapeOpByExpansion
 
     Location loc = padOp->getLoc();
     RankedTensorType expandedPaddedType = paddedType.clone(expandedPaddedShape);
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
     rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
@@ -1687,9 +1687,10 @@ GenericOp cloneToCollapsedOp<GenericOp>(RewriterBase &rewriter,
   SmallVector<utils::IteratorType> iteratorTypes(getCollapsedOpIteratorTypes(
       origOp.getIteratorTypesArray(), collapsingInfo));
 
-  GenericOp collapsedOp = linalg::GenericOp::create(rewriter,
-      origOp.getLoc(), resultTypes, inputOperands, outputOperands, indexingMaps,
-      iteratorTypes, [](OpBuilder &builder, Location loc, ValueRange args) {});
+  GenericOp collapsedOp = linalg::GenericOp::create(
+      rewriter, origOp.getLoc(), resultTypes, inputOperands, outputOperands,
+      indexingMaps, iteratorTypes,
+      [](OpBuilder &builder, Location loc, ValueRange args) {});
   Block *origOpBlock = &origOp->getRegion(0).front();
   Block *collapsedOpBlock = &collapsedOp->getRegion(0).front();
   rewriter.mergeBlocks(origOpBlock, collapsedOpBlock,
@@ -1794,12 +1795,12 @@ FailureOr<CollapseResult> mlir::linalg::collapseOpIterationDims(
       if (isa<MemRefType>(collapsedOpResult.getType())) {
         MemRefType expandShapeResultType = MemRefType::get(
             originalResultType.getShape(), originalResultType.getElementType());
-        result = memref::ExpandShapeOp::create(rewriter,
-            loc, expandShapeResultType, collapsedOpResult, reassociation,
-            resultShape);
+        result = memref::ExpandShapeOp::create(
+            rewriter, loc, expandShapeResultType, collapsedOpResult,
+            reassociation, resultShape);
       } else {
-        result = tensor::ExpandShapeOp::create(rewriter,
-            loc, originalResultType, collapsedOpResult, reassociation,
+        result = tensor::ExpandShapeOp::create(
+            rewriter, loc, originalResultType, collapsedOpResult, reassociation,
             resultShape);
       }
       results.push_back(result);
@@ -1982,8 +1983,8 @@ class FoldPadWithProducerReshapeOpByCollapsing
 
     RankedTensorType collapsedPaddedType =
         paddedType.clone(collapsedPaddedShape);
-    auto newPadOp = tensor::PadOp::create(rewriter,
-        loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
         padOp.getConstantPaddingValue(), padOp.getNofold());
 
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
@@ -2120,14 +2121,15 @@ class FoldScalarOrSplatConstant : public OpRewritePattern<GenericOp> {
           arith::ConstantOp::create(rewriter, def->getLoc(), constantAttr);
 
       SmallVector<Value> outputOperands = genericOp.getOutputs();
-      auto fusedOp = GenericOp::create(rewriter,
-          rewriter.getFusedLoc(fusedLocs), genericOp->getResultTypes(),
-          /*inputs=*/fusedOperands,
-          /*outputs=*/outputOperands,
-          rewriter.getAffineMapArrayAttr(fusedIndexMaps),
-          genericOp.getIteratorTypes(),
-          /*doc=*/nullptr,
-          /*library_call=*/nullptr);
+      auto fusedOp =
+          GenericOp::create(rewriter, rewriter.getFusedLoc(fusedLocs),
+                            genericOp->getResultTypes(),
+                            /*inputs=*/fusedOperands,
+                            /*outputs=*/outputOperands,
+                            rewriter.getAffineMapArrayAttr(fusedIndexMaps),
+                            genericOp.getIteratorTypes(),
+                            /*doc=*/nullptr,
+                            /*library_call=*/nullptr);
 
       // Map the block argument corresponding to the replaced argument with the
       // scalar constant.
@@ -2183,8 +2185,8 @@ struct RemoveOutsDependency : public OpRewritePattern<GenericOp> {
         modifiedOutput = true;
         SmallVector<OpFoldResult> mixedSizes =
             tensor::getMixedSizes(rewriter, loc, operandVal);
-        Value emptyTensor = tensor::EmptyOp::create(rewriter,
-            loc, mixedSizes, operandType.getElementType());
+        Value emptyTensor = tensor::EmptyOp::create(
+            rewriter, loc, mixedSizes, operandType.getElementType());
         op->setOperand(opOperand.getOperandNumber(), emptyTensor);
       }
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index d93f8c26a9f0a..c52315333c5b3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -64,8 +64,8 @@ getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) {
       continue;
 
     // Extract static / dynamic shape mix from the first operand.
-    res.push_back(tensor::EmptyOp::create(b,
-        loc, tensor::getMixedSizes(b, loc, operands.front()),
+    res.push_back(tensor::EmptyOp::create(
+        b, loc, tensor::getMixedSizes(b, loc, operands.front()),
         cast<RankedTensorType>(t).getElementType()));
   }
   return res;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
index 4e95aae86e670..9974ccd8ae79e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EraseUnusedOperandsAndResults.cpp
@@ -259,8 +259,8 @@ mlir::linalg::deduplicateOperandsAndRemoveDeadResults(
   for (Value v : newOutputOperands)
     if (isa<TensorType>(v.getType()))
       newResultTypes.push_back(v.getType());
-  auto newOp = GenericOp::create(rewriter,
-      loc, newResultTypes, newInputOperands, newOutputOperands,
+  auto newOp = GenericOp::create(
+      rewriter, loc, newResultTypes, newInputOperands, newOutputOperands,
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       genericOp.getIteratorTypes(), genericOp.getDocAttr(),
       genericOp.getLibraryCallAttr(),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
index e19e224c25028..0ca89040f3453 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
@@ -72,14 +72,14 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
     // Create the tensor of same size as output of the pad op.
     RankedTensorType padResultType = padOp.getResultType();
     auto resultSizes = resultShape[0];
-    auto emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, resultSizes, padResultType.getElementType());
+    auto emptyTensor = tensor::EmptyOp::create(rewriter, loc, resultSizes,
+                                               padResultType.getElementType());
 
     // Fill the tensor with the pad value.
     // TODO: There is an option to fill only the boundaries. For now just
     // filling the whole tensor.
-    auto fillTensor =
-        linalg::FillOp::create(rewriter, loc, padValue, emptyTensor.getResult());
+    auto fillTensor = linalg::FillOp::create(rewriter, loc, padValue,
+                                             emptyTensor.getResult());
 
     // Construct a slice of the fill result that is to be replaced with the
     // result of the generic op. The low pad values are the offsets, the size of
@@ -100,8 +100,8 @@ struct FusePadOp : OpRewritePattern<tensor::PadOp> {
       }
     }
     SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(1));
-    auto slice = tensor::ExtractSliceOp::create(rewriter,
-        loc, fillTensor.getResult(0), offsets, sizes, strides);
+    auto slice = tensor::ExtractSliceOp::create(
+        rewriter, loc, fillTensor.getResult(0), offsets, sizes, strides);
 
     // Clone the generic op.
     auto clonedOp =
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 7994af8834990..3e31393fd51ed 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -61,8 +61,9 @@ FailureOr<GenericOp> mlir::linalg::generalizeNamedOp(RewriterBase &rewriter,
   // All named ops have a region attached that can be inlined.
   assert(linalgOp->getNumRegions() == 1 &&
          "expect named op to have one region attached");
-  GenericOp genericOp = GenericOp::create(rewriter,
-      linalgOp.getLoc(), resultTypes, inputs, outputs, indexingMaps, iterators);
+  GenericOp genericOp =
+      GenericOp::create(rewriter, linalgOp.getLoc(), resultTypes, inputs,
+                        outputs, indexingMaps, iterators);
   rewriter.inlineRegionBefore(linalgOp->getRegion(0), genericOp.getRegion(),
                               genericOp.getRegion().begin());
   rewriter.replaceOp(linalgOp, genericOp->getResults());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index 86f24578376c1..19729af1bc9fa 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -591,8 +591,8 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
 
     // Create a packing loop that takes `hoistedPackedTensor` as iteration
     // argument.
-    auto clonedForOp = scf::ForOp::create(rewriter,
-        loc, bvm.lookupOrDefault(forOp.getLowerBound()),
+    auto clonedForOp = scf::ForOp::create(
+        rewriter, loc, bvm.lookupOrDefault(forOp.getLowerBound()),
         bvm.lookupOrDefault(forOp.getUpperBound()),
         bvm.lookupOrDefault(forOp.getStep()), hoistedPackedTensor);
 
@@ -640,11 +640,11 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   TransposeOp maybeTransposeOp;
   Value paddedTensor = bvm.lookup(opToHoist.getResult());
   if (!transposeVector.empty()) {
-    Value outputTensor = tensor::ExtractSliceOp::create(rewriter,
-        loc, transposedTensorType, hoistedPackedTensor, offsets, sizes,
-        strides);
-    maybeTransposeOp = linalg::TransposeOp::create(rewriter,
-        loc, paddedTensor, outputTensor, transposeVector);
+    Value outputTensor = tensor::ExtractSliceOp::create(
+        rewriter, loc, transposedTensorType, hoistedPackedTensor, offsets,
+        sizes, strides);
+    maybeTransposeOp = linalg::TransposeOp::create(
+        rewriter, loc, paddedTensor, outputTensor, transposeVector);
     paddedTensor = maybeTransposeOp.getResult()[0];
   }
 
@@ -652,8 +652,9 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   if (nPackedLoops > 0) {
     // Step 4. Create InsertSliceOp at the innermost loop level, inserting an
     // optionally transposed padded slice into the packed tensor.
-    Value inserted = tensor::InsertSliceOp::create(rewriter,
-        loc, paddedTensor, hoistedPackedTensor, offsets, sizes, strides);
+    Value inserted = tensor::InsertSliceOp::create(rewriter, loc, paddedTensor,
+                                                   hoistedPackedTensor, offsets,
+                                                   sizes, strides);
 
     // Step 5. Iteratively pop the stack and propagate the yield.
     Value valueToYield = inserted;
@@ -712,8 +713,8 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
   rewriter.setInsertionPoint(outerLoop);
   SmallVector<Value> dynamicTensorSizes =
       analysis.getHoistedPackedTensorSizes(rewriter, loc);
-  auto emptyOp = tensor::EmptyOp::create(rewriter,
-      loc, hoistedPackedTensorType.getShape(),
+  auto emptyOp = tensor::EmptyOp::create(
+      rewriter, loc, hoistedPackedTensorType.getShape(),
       hoistedPackedTensorType.getElementType(), dynamicTensorSizes);
 
   return buildPackingLoopNestImpl(rewriter, bvm, opToHoist, transposeVector,
@@ -840,8 +841,8 @@ padThroughLoopIterArg(RewriterBase &rewriter, Value paddedValueBeforeHoisting,
   {
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(forOp);
-    extracted = tensor::ExtractSliceOp::create(rewriter,
-        hoistedPackedTensor.getLoc(), hoistedPackedTensor,
+    extracted = tensor::ExtractSliceOp::create(
+        rewriter, hoistedPackedTensor.getLoc(), hoistedPackedTensor,
         outerSliceOp.getMixedOffsets(), outerSliceOp.getMixedSizes(),
         outerSliceOp.getMixedStrides());
     rewriter.replaceAllUsesWith(forOp.getResult(iterArgNumber), extracted);
@@ -934,8 +935,8 @@ static Value replaceByPackingResult(RewriterBase &rewriter,
   // offsets = [maybe_leading_ivs, 0 .. 0].
   // sizes = [1 .. 1, transposedShape] (defined above).
   // strides = [1 .. 1] (defined above)
-  return tensor::ExtractSliceOp::create(rewriter,
-      loc, transposedTensorType, hoistedPackedTensor, offsets,
+  return tensor::ExtractSliceOp::create(
+      rewriter, loc, transposedTensorType, hoistedPackedTensor, offsets,
       packingResult.sizes, packingResult.strides);
 }
 
@@ -982,10 +983,11 @@ FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
     OpBuilder::InsertionGuard g(rewriter);
     rewriter.setInsertionPointAfter(newResult.getDefiningOp());
     // Transpose the packed tensor back to the original storage order.
-    Value emptyTensor = tensor::EmptyOp::create(rewriter,
-        loc, paddedTensorType.getShape(), paddedTensorType.getElementType());
-    TransposeOp unTransposeOp = linalg::TransposeOp::create(rewriter,
-        loc, newResult, emptyTensor, transposeVector);
+    Value emptyTensor =
+        tensor::EmptyOp::create(rewriter, loc, paddedTensorType.getShape(),
+                                paddedTensorType.getElementType());
+    TransposeOp unTransposeOp = linalg::TransposeOp::create(
+        rewriter, loc, newResult, emptyTensor, transposeVector);
     newResult = unTransposeOp.getResult()[0];
     transposeOps.push_back(unTransposeOp);
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 5397fc572e558..70bc7b6360ceb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -53,9 +53,9 @@ static scf::ForOp replaceWithDifferentYield(RewriterBase &rewriter,
   assert(index < inits.size());
   inits[index] = newInitOperand;
 
-  scf::ForOp newLoop = scf::ForOp::create(rewriter,
-      loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(), loop.getStep(),
-      inits, [](OpBuilder &, Location, Value, ValueRange) {});
+  scf::ForOp newLoop = scf::ForOp::create(
+      rewriter, loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(),
+      loop.getStep(), inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
   // Generate the new yield with the replaced operand.
   auto yieldOp = cast<scf::YieldOp>(loop.getBody()->getTerminator());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
index a900215ccf014..39cc21df40583 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
@@ -60,9 +60,9 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
 
     Location loc = genericOp->getLoc();
     SmallVector<Value> outputOperands = genericOp.getOutputs();
-    auto newOp = GenericOp::create(rewriter,
-        loc, genericOp->getResultTypes(), newOperands, outputOperands,
-        newIndexingMaps, genericOp.getIteratorTypesArray());
+    auto newOp = GenericOp::create(rewriter, loc, genericOp->getResultTypes(),
+                                   newOperands, outputOperands, newIndexingMaps,
+                                   genericOp.getIteratorTypesArray());
     rewriter.cloneRegionBefore(genericOp.getRegion(), newOp.getRegion(),
                                newOp.getRegion().begin());
 
@@ -80,8 +80,8 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
             arith::ConstantIndexOp::create(rewriter, loc, idx));
       Value scalarValue = opOperand->get();
       if (isa<RankedTensorType>(scalarValue.getType())) {
-        scalarValue =
-            tensor::ExtractOp::create(rewriter, loc, scalarValue, indicesValues);
+        scalarValue = tensor::ExtractOp::create(rewriter, loc, scalarValue,
+                                                indicesValues);
       }
       body->getArgument(idx).replaceAllUsesWith(scalarValue);
       body->eraseArgument(idx);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index 2372313e29c58..96e6eee6d7a0d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -88,7 +88,8 @@ mlir::linalg::interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp,
       allIndices.reserve(genericOp.getNumLoops());
       llvm::transform(llvm::seq<uint64_t>(0, genericOp.getNumLoops()),
                       std::back_inserter(allIndices), [&](uint64_t dim) {
-                        return IndexOp::create(rewriter, indexOp->getLoc(), dim);
+                        return IndexOp::create(rewriter, indexOp->getLoc(),
+                                               dim);
                       });
       rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
           indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 0c08c1d8b997d..38f1a8b7247eb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -70,8 +70,9 @@ static void inlineRegionAndEmitStore(OpBuilder &b, Location loc, OpType op,
   Operation *terminator = block.getTerminator();
   for (OpOperand &operand : terminator->getOpOperands()) {
     Value toStore = map.lookupOrDefault(operand.get());
-    StoreOpTy::create(b, loc, toStore, outputBuffers[operand.getOperandNumber()],
-                        indexing[operand.getOperandNumber()]);
+    StoreOpTy::create(b, loc, toStore,
+                      outputBuffers[operand.getOperandNumber()],
+                      indexing[operand.getOperandNumber()]);
   }
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
index 28b246714df53..f1c9d79a80804 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
@@ -132,11 +132,11 @@ static Value createDestinationPassingStyleInitOperand(
   Value processLinearIndexInReductionGroup = mesh::createProcessLinearIndex(
       meshOp.getSymName(), reductionMeshAxes, builder);
   Value zero = arith::ConstantIndexOp::create(builder, 0);
-  Value isLeadProcess = arith::CmpIOp::create(builder,
-      builder.getI1Type(), arith::CmpIPredicate::eq,
+  Value isLeadProcess = arith::CmpIOp::create(
+      builder, builder.getI1Type(), arith::CmpIPredicate::eq,
       processLinearIndexInReductionGroup, zero);
   scf::IfOp ifOp = scf::IfOp::create(builder, spmdizedOperand.getType(),
-                                             isLeadProcess, true, true);
+                                     isLeadProcess, true, true);
   // Then block.
   {
     OpBuilder::InsertionGuard insertionGuard(builder);
@@ -158,7 +158,7 @@ static Value createDestinationPassingStyleInitOperand(
         arith::getNeutralElement(combinerOps[0]);
 
     Value init = tensor::EmptyOp::create(builder, op.getLoc(), shape,
-                                                 neutralEl.value().getType());
+                                         neutralEl.value().getType());
     Value constant =
         arith::ConstantOp::create(builder, op.getLoc(), neutralEl.value());
     Value fill = linalg::FillOp::create(builder, op.getLoc(), constant, init)
@@ -202,9 +202,9 @@ static void createAllReduceForResultWithoutPartialSharding(
   }
 
   Value spmdizedLinalgOpResult = spmdizationMap.lookup(unshardedLinalgOpResult);
-  Value reducedValue = mesh::AllReduceOp::create(builder,
-      spmdizedLinalgOpResult, resultSharding.getMesh(), allReduceMeshAxes,
-      reductionKind);
+  Value reducedValue = mesh::AllReduceOp::create(
+      builder, spmdizedLinalgOpResult, resultSharding.getMesh(),
+      allReduceMeshAxes, reductionKind);
   spmdizationMap.map(unshardedLinalgOpResult, reducedValue);
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
index 019c845b4e0e7..a2bd9d92815a0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
@@ -59,8 +59,8 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
   auto newKernelTy = RankedTensorType::get(
       {kernelTy.getDimSize(0), kernelTy.getDimSize(1), kernelTy.getDimSize(2)},
       kernelTy.getElementType());
-  auto collapsedKernel = tensor::CollapseShapeOp::create(rewriter,
-      loc, newKernelTy, kernel, collapsedKernelDims);
+  auto collapsedKernel = tensor::CollapseShapeOp::create(
+      rewriter, loc, newKernelTy, kernel, collapsedKernelDims);
 
   // Collapse init dims.
   SmallVector<ReassociationIndices, 4> collapsedInitDims = {
@@ -70,22 +70,23 @@ matchAndReplaceDepthwiseConv(Operation *operation, Value input, Value kernel,
       RankedTensorType::get({initTy.getDimSize(0), initTy.getDimSize(1),
                              initTy.getDimSize(2), initTy.getDimSize(3)},
                             initTy.getElementType());
-  auto collapsedInit = tensor::CollapseShapeOp::create(rewriter,
-      loc, newInitTy, init, collapsedInitDims);
+  auto collapsedInit = tensor::CollapseShapeOp::create(rewriter, loc, newInitTy,
+                                                       init, collapsedInitDims);
 
   SmallVector<NamedAttribute> preservedAttrs;
   Operation *newConv =
       TypeSwitch<Operation *, Operation *>(operation)
           .Case<DepthwiseConv2DNhwcHwcmOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return DepthwiseConv2DNhwcHwcOp::create(rewriter,
-                loc, newInitTy, ValueRange{input, collapsedKernel},
+            return DepthwiseConv2DNhwcHwcOp::create(
+                rewriter, loc, newInitTy, ValueRange{input, collapsedKernel},
                 ValueRange{collapsedInit}, stride, dilation);
           })
           .Case<DepthwiseConv2DNhwcHwcmQOp>([&](auto op) {
             preservedAttrs = getPrunedAttributeList(op);
-            return DepthwiseConv2DNhwcHwcQOp::create(rewriter,
-                loc, newInitTy, ValueRange{input, collapsedKernel, iZp, kZp},
+            return DepthwiseConv2DNhwcHwcQOp::create(
+                rewriter, loc, newInitTy,
+                ValueRange{input, collapsedKernel, iZp, kZp},
                 ValueRange{collapsedInit}, stride, dilation);
           })
           .Default([](Operation *op) { return nullptr; });
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
index 1368bb7e77db0..0415057eda86b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
@@ -144,7 +144,7 @@ struct SimplifyUnPackToCollapseShape : public OpRewritePattern<UnPackOp> {
     if (operand.getType() == newOperandType)
       return operand;
     return tensor::CollapseShapeOp::create(rewriter, loc, newOperandType,
-                                                    operand, reassociation);
+                                           operand, reassociation);
   }
 
   /// Returns success() if it is unpacking on the innermost dimension.
@@ -265,8 +265,8 @@ struct FoldUnpackWithExtractSliceOp
 
     // Create a new empty output tensor.
     Type elementType = unpackOp.getDestType().getElementType();
-    Value output = tensor::EmptyOp::create(rewriter,
-        sliceOp.getLoc(), sliceOp.getMixedSizes(), elementType);
+    Value output = tensor::EmptyOp::create(
+        rewriter, sliceOp.getLoc(), sliceOp.getMixedSizes(), elementType);
     rewriter.replaceOpWithNewOp<UnPackOp>(
         sliceOp, unpackOp.getSource(), output, unpackOp.getInnerDimsPos(),
         unpackOp.getMixedTiles(), unpackOp.getOuterDimsPerm());
@@ -529,8 +529,8 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
     auto elemType =
         cast<ShapedType>(unPackOp->getResultTypes()[0]).getElementType();
-    Value output = tensor::EmptyOp::create(rewriter,
-        unPackOp->getLoc(), unpackOpResultDims[0], elemType);
+    Value output = tensor::EmptyOp::create(rewriter, unPackOp->getLoc(),
+                                           unpackOpResultDims[0], elemType);
 
     rewriter.replaceOpWithNewOp<UnPackOp>(
         unPackOp, linalgOp->getOperand(0), output, newInnerDimsPosVec,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index fb43c1f92e5b3..2c62cb628a7dd 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -193,10 +193,10 @@ static Value padOperand(RewriterBase &rewriter, TilingInterface opToPad,
           dyn_cast<ComplexType>(getElementTypeOrSelf(v.getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingValueAttr);
     paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
-                                                        complexTy, complexAttr);
+                                               complexTy, complexAttr);
   } else {
-    paddingValue = arith::ConstantOp::create(rewriter,
-        opToPad.getLoc(), cast<TypedAttr>(paddingValueAttr));
+    paddingValue = arith::ConstantOp::create(rewriter, opToPad.getLoc(),
+                                             cast<TypedAttr>(paddingValueAttr));
   }
 
   // Pad the operand to the bounding box defined by `paddedShape`.
@@ -323,8 +323,8 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
-        loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(
+        rewriter, loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index af614e88fb3d5..b5c5aea56a998 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -220,10 +220,10 @@ static FailureOr<Value> padOperandToSmallestStaticBoundingBox(
           getElementTypeOrSelf(opOperand->get().getType()))) {
     auto complexAttr = cast<ArrayAttr>(paddingAttr);
     paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
-                                                        complexTy, complexAttr);
+                                               complexTy, complexAttr);
   } else {
-    paddingValue = arith::ConstantOp::create(rewriter,
-        opToPad.getLoc(), cast<TypedAttr>(paddingAttr));
+    paddingValue = arith::ConstantOp::create(rewriter, opToPad.getLoc(),
+                                             cast<TypedAttr>(paddingAttr));
   }
 
   // Computes the padded shape.
@@ -313,8 +313,8 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad,
     int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
     SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
     SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
-    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(rewriter,
-        loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
+    paddedSubtensorResults.push_back(tensor::ExtractSliceOp::create(
+        rewriter, loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
         strides));
   }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index 4042a43b04091..7014d02a70377 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -64,10 +64,10 @@ static Value allocBuffer(ImplicitLocOpBuilder &b,
         MemRefType::Builder(staticBufferType).setMemorySpace(memorySpaceAttr);
     if (options.useAlloca) {
       return memref::AllocaOp::create(b, staticBufferType, ValueRange{},
-                                        alignmentAttr);
+                                      alignmentAttr);
     }
     return memref::AllocOp::create(b, staticBufferType, ValueRange{},
-                                     alignmentAttr);
+                                   alignmentAttr);
   }
 
   // Fallback dynamic buffer.
@@ -124,7 +124,7 @@ defaultDeallocBufferCallBack(const LinalgPromotionOptions &options,
   if (!options.useAlloca) {
     auto viewOp = cast<memref::ViewOp>(fullLocalView.getDefiningOp());
     memref::DeallocOp::create(b, viewOp.getSource().getLoc(),
-                                viewOp.getSource());
+                              viewOp.getSource());
   }
   return success();
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
index 1fac80142dddf..76d0ba97be4db 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Split.cpp
@@ -52,8 +52,8 @@ createSplitPart(RewriterBase &b, Location loc, TilingInterface op,
       return nullptr;
     SmallVector<OpFoldResult> resultStrides(resultOffsets.size(),
                                             b.getIndexAttr(1));
-    Value inserted = tensor::InsertSliceOp::create(b,
-        loc, result, resultOperands[index], resultOffsets, resultSizes,
+    Value inserted = tensor::InsertSliceOp::create(
+        b, loc, result, resultOperands[index], resultOffsets, resultSizes,
         resultStrides);
     results.push_back(inserted);
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
index 1b0bfbbec8624..b8f862001bccf 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SplitReduction.cpp
@@ -115,8 +115,8 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
         newShape,
         cast<RankedTensorType>(operand->get().getType()).getElementType());
 
-    Value newInput = tensor::ExpandShapeOp::create(b,
-        loc, newType, operand->get(), reassociation);
+    Value newInput = tensor::ExpandShapeOp::create(
+        b, loc, newType, operand->get(), reassociation);
     newInputs.push_back(newInput);
   }
 
@@ -140,14 +140,14 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   Value emptyOrAllocTensor;
   if (useAlloc) {
-    emptyOrAllocTensor = bufferization::AllocTensorOp::create(b,
-        loc,
+    emptyOrAllocTensor = bufferization::AllocTensorOp::create(
+        b, loc,
         RankedTensorType::get(newOutputShape,
                               op.getRegionOutputArgs()[0].getType()),
         ValueRange{});
   } else {
-    emptyOrAllocTensor = tensor::EmptyOp::create(b,
-        loc, newOutputShape, op.getRegionOutputArgs()[0].getType());
+    emptyOrAllocTensor = tensor::EmptyOp::create(
+        b, loc, newOutputShape, op.getRegionOutputArgs()[0].getType());
   }
   Value constantOp = arith::ConstantOp::create(b, loc, *identity);
   Value identityTensor =
@@ -168,8 +168,8 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   }
   // Create the new op matching the original op with an extra parallel
   // dimension.
-  GenericOp genericOp = GenericOp::create(b,
-      loc, TypeRange({emptyOrAllocTensor.getType()}), newInputs,
+  GenericOp genericOp = GenericOp::create(
+      b, loc, TypeRange({emptyOrAllocTensor.getType()}), newInputs,
       ValueRange({identityTensor}), newMaps, newIteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
                        genericOp.getRegion().begin());
@@ -191,8 +191,8 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReduction(
   AffineMap outputMap = AffineMap::get(intermRank, 0, exprs, op.getContext());
   SmallVector<AffineMap> reductionMaps = {inputMap, outputMap};
 
-  auto reduction = GenericOp::create(b,
-      loc, op->getResultTypes(), ValueRange({genericOp.getResult(0)}),
+  auto reduction = GenericOp::create(
+      b, loc, op->getResultTypes(), ValueRange({genericOp.getResult(0)}),
       op.getDpsInits(), reductionMaps, reductionIteratorTypes,
       [reductionOp](OpBuilder &b, Location loc, ValueRange inputs) {
         Operation *clonedReductionOp = b.clone(*reductionOp);
@@ -321,11 +321,11 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
           bufferization::AllocTensorOp::create(b, loc, newT, dims);
     } else {
       emptyOrAllocTensor = tensor::EmptyOp::create(b, loc, newT.getShape(),
-                                                     t.getElementType(), dims);
+                                                   t.getElementType(), dims);
     }
     Value constantOp = arith::ConstantOp::create(b, loc, std::get<1>(it));
-    fillOps.push_back(
-        linalg::FillOp::create(b, op->getLoc(), constantOp, emptyOrAllocTensor));
+    fillOps.push_back(linalg::FillOp::create(b, op->getLoc(), constantOp,
+                                             emptyOrAllocTensor));
     newOutputs.push_back(fillOps.back().getResult(0));
     emptyOrAllocTensorOps.push_back(emptyOrAllocTensor.getDefiningOp());
   }
@@ -354,8 +354,8 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
   SmallVector<Value> newInputs = op.getDpsInputs();
   // Add a single shape-only tensor to carry the dimensions without resorting to
   // more complex inversions.
-  newInputs.push_back(tensor::EmptyOp::create(b,
-      loc, ArrayRef<int64_t>{reductionDimSize / splitFactor, splitFactor},
+  newInputs.push_back(tensor::EmptyOp::create(
+      b, loc, ArrayRef<int64_t>{reductionDimSize / splitFactor, splitFactor},
       b.getIntegerType(1)));
   // Output tensors are already good to go.
 
@@ -366,7 +366,7 @@ FailureOr<SplitReductionResult> mlir::linalg::splitReductionByScaling(
                        utils::IteratorType::parallel);
   GenericOp genericOp =
       GenericOp::create(b, loc, ValueRange(newOutputs).getTypes(), newInputs,
-                          newOutputs, newMaps, iteratorTypes);
+                        newOutputs, newMaps, iteratorTypes);
   b.inlineRegionBefore(op->getRegion(0), genericOp.getRegion(),
                        genericOp.getRegion().begin());
   genericOp.getRegion().front().insertArgument(reductionDimPos,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
index 854d3855d654d..792ca3e5123ef 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SwapExtractSliceWithFillPatterns.cpp
@@ -29,10 +29,10 @@ struct SwapExtractSliceOfFill final
     if (!fillOp || !fillOp->hasOneUse())
       return failure();
 
-    auto newExtractOp = tensor::ExtractSliceOp::create(rewriter,
-        extractOp.getLoc(), extractOp.getType(), fillOp.getOutputs()[0],
-        extractOp.getMixedOffsets(), extractOp.getMixedSizes(),
-        extractOp.getMixedStrides());
+    auto newExtractOp = tensor::ExtractSliceOp::create(
+        rewriter, extractOp.getLoc(), extractOp.getType(),
+        fillOp.getOutputs()[0], extractOp.getMixedOffsets(),
+        extractOp.getMixedSizes(), extractOp.getMixedStrides());
     rewriter.replaceOpWithNewOp<FillOp>(extractOp, fillOp.getInputs(),
                                         ValueRange{newExtractOp.getResult()});
     return success();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index c2803423fafd7..705d6f26efd29 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -96,9 +96,9 @@ static void emitIsPositiveIndexAssertion(ImplicitLocOpBuilder &b,
 
   Value zero = arith::ConstantIndexOp::create(b, 0);
   Value condition = arith::CmpIOp::create(b, arith::CmpIPredicate::sgt,
-                                            cast<Value>(value), zero);
-  cf::AssertOp::create(b,
-      condition,
+                                          cast<Value>(value), zero);
+  cf::AssertOp::create(
+      b, condition,
       b.getStringAttr("expected strictly positive tile size and divisor"));
 }
 
@@ -318,10 +318,11 @@ mlir::linalg::computeMultiTileSizes(OpBuilder &builder, LinalgOp op,
         apply(s0 * s1 + s2 * s3, {spec.lowTileSize, spec.lowTripCount,
                                   spec.highTileSize, spec.highTripCount});
     Value equals = arith::CmpIOp::create(b, arith::CmpIPredicate::eq,
-                                           coveredSize, tripCount);
-    cf::AssertOp::create(b,
-        equals, builder.getStringAttr(
-                    "could not compute dynamic multi-size tile shapes"));
+                                         coveredSize, tripCount);
+    cf::AssertOp::create(
+        b, equals,
+        builder.getStringAttr(
+            "could not compute dynamic multi-size tile shapes"));
   }
 
   return spec;
@@ -656,8 +657,8 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       getValueOrCreateConstantIndexOp(b, loc, nonZeroNumThreads);
 
   // 2. Create the ForallOp with an empty region.
-  scf::ForallOp forallOp = scf::ForallOp::create(b,
-      loc, getAsOpFoldResult(materializedNonZeroNumThreads), initTensors,
+  scf::ForallOp forallOp = scf::ForallOp::create(
+      b, loc, getAsOpFoldResult(materializedNonZeroNumThreads), initTensors,
       mapping);
 
   // 3. Calculate the tile offsets and sizes for the subsequent loop that will
@@ -689,8 +690,8 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
       sizes[reductionDim] = b.getIndexAttr(1);
       outOffsets[reductionDim] = forallOp.getInductionVars()[0];
       // TODO: use SubsetExtractOpInterface once it is available.
-      tiledDpsInitOperands.push_back(tensor::ExtractSliceOp::create(b,
-          loc, cast<RankedTensorType>(initOperand.getType()),
+      tiledDpsInitOperands.push_back(tensor::ExtractSliceOp::create(
+          b, loc, cast<RankedTensorType>(initOperand.getType()),
           destBbArgs[destNum], outOffsets, sizes, strides));
     }
 
@@ -768,8 +769,8 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
     // 6.b. Parallel insertions are inserted at the end of the combining
     // terminator.
     b.setInsertionPointToEnd(forallOp.getTerminator().getBody());
-    tensor::ParallelInsertSliceOp::create(b,
-        loc, result, bbArg, resultOffsetsRank, resultSizesRank, strides);
+    tensor::ParallelInsertSliceOp::create(
+        b, loc, result, bbArg, resultOffsetsRank, resultSizesRank, strides);
   }
 
   // 7. Merge the partial reductions.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
index 94449fae260ee..b65f9d9fb9f7f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
@@ -72,9 +72,9 @@ static LogicalResult inlinePayload(OpBuilder &b, LinalgOp linalgOp,
     OpOperand *storeInto = linalgOp.getDpsInitOperand(operand.index());
     auto indices = getIndicesForAccess(
         b, loc, linalgOp.getMatchingIndexingMap(storeInto), ivs);
-    memref::StoreOp::create(b,
-        loc, toStore, linalgOp.getDpsInitOperand(operand.index())->get(),
-        indices);
+    memref::StoreOp::create(b, loc, toStore,
+                            linalgOp.getDpsInitOperand(operand.index())->get(),
+                            indices);
   }
   return success();
 }
@@ -574,9 +574,9 @@ struct LinalgOpPartialReductionInterface
       RankedTensorType sliceResultType = RankedTensorType::get(
           sliceInfo.resultShape, valueToTileType.getElementType(),
           valueToTileType.getEncoding());
-      auto sliceOp = tensor::ExtractSliceOp::create(b,
-          loc, sliceResultType, valueToTile, sliceInfo.offsets, sliceInfo.sizes,
-          sliceInfo.strides);
+      auto sliceOp = tensor::ExtractSliceOp::create(
+          b, loc, sliceResultType, valueToTile, sliceInfo.offsets,
+          sliceInfo.sizes, sliceInfo.strides);
       tiledInits.push_back(sliceOp.getResult());
       generatedSlices.push_back(sliceOp);
     }
@@ -603,8 +603,8 @@ struct LinalgOpPartialReductionInterface
     auto resultTypes = ValueRange(tiledInits).getTypes();
     if (tilingStrategy ==
         ReductionTilingStrategy::PartialReductionOuterReduction) {
-      auto genericOp = GenericOp::create(b,
-          loc, resultTypes, tiledInputs, tiledInits, newMaps, newIteratorTypes);
+      auto genericOp = GenericOp::create(b, loc, resultTypes, tiledInputs,
+                                         tiledInits, newMaps, newIteratorTypes);
       IRMapping mapping;
       op->getRegion(0).cloneInto(&genericOp.getRegion(),
                                  genericOp.getRegion().begin(), mapping);
@@ -648,8 +648,8 @@ struct LinalgOpPartialReductionInterface
         }
       }
 
-      auto reduction = linalg::ReduceOp::create(b,
-          loc, partialResult, init, partialReductionDims,
+      auto reduction = linalg::ReduceOp::create(
+          b, loc, partialResult, init, partialReductionDims,
           [&linalgOp, &initIdx](OpBuilder &b, Location loc, ValueRange inputs) {
             // Get the combiner op.
             SmallVector<Operation *, 4> combinerOps;
@@ -790,8 +790,8 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = tensor::ExtractSliceOp::create(b,
-        loc, packOp.getSource(), inputIndices, inputSizes, strides);
+    auto sourceSlice = tensor::ExtractSliceOp::create(
+        b, loc, packOp.getSource(), inputIndices, inputSizes, strides);
     tiledOperands.push_back(sourceSlice);
 
     SmallVector<OpFoldResult> outputOffsets, outputSizes;
@@ -800,8 +800,8 @@ struct PackOpTiling
       return {};
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = tensor::ExtractSliceOp::create(b,
-        loc, packOp.getDest(), outputOffsets, outputSizes, strides);
+    auto outSlice = tensor::ExtractSliceOp::create(
+        b, loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
     if (auto val = packOp.getPaddingValue())
@@ -809,8 +809,8 @@ struct PackOpTiling
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = PackOp::create(b,
-        loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
+    Operation *tiledPackOp = PackOp::create(
+        b, loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
         {tiledPackOp},
@@ -968,8 +968,8 @@ struct PackOpTiling
     SmallVector<OpFoldResult> strides(inputRank, oneAttr);
 
     SmallVector<Value> tiledOperands;
-    auto sourceSlice = tensor::ExtractSliceOp::create(b,
-        loc, packOp.getSource(), offsets, sizes, strides);
+    auto sourceSlice = tensor::ExtractSliceOp::create(
+        b, loc, packOp.getSource(), offsets, sizes, strides);
     tiledOperands.push_back(sourceSlice);
 
     SmallVector<OpFoldResult> outerDimOffsets, outerDimSizes;
@@ -984,16 +984,16 @@ struct PackOpTiling
       return failure();
 
     strides.append(packOp.getDestRank() - inputRank, oneAttr);
-    auto outSlice = tensor::ExtractSliceOp::create(b,
-        loc, packOp.getDest(), outputOffsets, outputSizes, strides);
+    auto outSlice = tensor::ExtractSliceOp::create(
+        b, loc, packOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(outSlice);
 
     assert(!packOp.getPaddingValue() && "Expect no padding semantic");
     for (auto tile : packOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledPackOp = PackOp::create(b,
-        loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
+    Operation *tiledPackOp = PackOp::create(
+        b, loc, TypeRange{outSlice.getType()}, tiledOperands, op->getAttrs());
 
     return TilingResult{
         {tiledPackOp},
@@ -1172,37 +1172,37 @@ struct UnPackOpTiling
     sliceSrcSizes.append(unpackOp.getMixedTiles());
     sliceSrcStrides.append(numInnerTiles, oneAttr);
     SmallVector<Operation *> generatedSlices;
-    tensor::ExtractSliceOp sliceSource = tensor::ExtractSliceOp::create(b,
-        loc, unpackOp.getSource(), sliceSrcIndices, sliceSrcSizes,
+    tensor::ExtractSliceOp sliceSource = tensor::ExtractSliceOp::create(
+        b, loc, unpackOp.getSource(), sliceSrcIndices, sliceSrcSizes,
         sliceSrcStrides);
     generatedSlices.push_back(sliceSource);
 
     SmallVector<OpFoldResult> destStrides(destRank, oneAttr);
     Value sliceDest;
     if (isPerfectTilingCase) {
-      auto destSliceOp = tensor::ExtractSliceOp::create(b,
-          loc, unpackOp.getDest(), offsets, sizes, destStrides);
+      auto destSliceOp = tensor::ExtractSliceOp::create(
+          b, loc, unpackOp.getDest(), offsets, sizes, destStrides);
       sliceDest = destSliceOp;
       generatedSlices.push_back(destSliceOp);
     } else {
-      sliceDest = tensor::EmptyOp::create(b,
-          loc, destExpandedSizes, unpackOp.getDestType().getElementType());
+      sliceDest = tensor::EmptyOp::create(
+          b, loc, destExpandedSizes, unpackOp.getDestType().getElementType());
     }
 
     SmallVector<Value> tiledOperands = {sliceSource.getResult(), sliceDest};
     for (auto tile : unpackOp.getInnerTiles())
       tiledOperands.push_back(tile);
 
-    Operation *tiledUnpackOp = UnPackOp::create(b,
-        loc, TypeRange{sliceDest.getType()}, tiledOperands, op->getAttrs());
+    Operation *tiledUnpackOp = UnPackOp::create(
+        b, loc, TypeRange{sliceDest.getType()}, tiledOperands, op->getAttrs());
 
     if (isPerfectTilingCase)
       return TilingResult{{tiledUnpackOp},
                           SmallVector<Value>(tiledUnpackOp->getResults()),
                           generatedSlices};
 
-    auto extractSlice = tensor::ExtractSliceOp::create(b,
-        loc, tiledUnpackOp->getResult(0), resultOffsetsFromDest, sizes,
+    auto extractSlice = tensor::ExtractSliceOp::create(
+        b, loc, tiledUnpackOp->getResult(0), resultOffsetsFromDest, sizes,
         destStrides);
     return TilingResult{
         {tiledUnpackOp}, {extractSlice.getResult()}, generatedSlices};
@@ -1337,14 +1337,14 @@ struct UnPackOpTiling
 
     SmallVector<Value> tiledOperands;
     // Create slice of the dest operand.
-    auto extractDestSlice = tensor::ExtractSliceOp::create(b,
-        loc, unPackOp.getDest(), outputOffsets, outputSizes, strides);
+    auto extractDestSlice = tensor::ExtractSliceOp::create(
+        b, loc, unPackOp.getDest(), outputOffsets, outputSizes, strides);
     tiledOperands.push_back(extractDestSlice);
 
     strides.append(unPackOp.getSourceRank() - outputRank, oneAttr);
     // Create slice of the source operand.
-    auto extractSourceSlice = tensor::ExtractSliceOp::create(b,
-        loc, unPackOp.getSource(), offsets, sizes, strides);
+    auto extractSourceSlice = tensor::ExtractSliceOp::create(
+        b, loc, unPackOp.getSource(), offsets, sizes, strides);
     tiledOperands.insert(tiledOperands.begin(), extractSourceSlice);
     for (auto tile : unPackOp.getInnerTiles())
       tiledOperands.push_back(tile);
@@ -1352,7 +1352,7 @@ struct UnPackOpTiling
     // Create tiled unpack op.
     Operation *tiledUnPackOp =
         UnPackOp::create(b, loc, TypeRange{extractDestSlice.getType()},
-                           tiledOperands, op->getAttrs());
+                         tiledOperands, op->getAttrs());
 
     return TilingResult{{tiledUnPackOp},
                         SmallVector<Value>(tiledUnPackOp->getResults()),
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 3526f62f770dc..1f1e617738981 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -269,12 +269,12 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       packingMetadata.reassociations);
   Value paddingValue = packOp.getPaddingValue();
   if (!paddingValue) {
-    paddingValue = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(getElementTypeOrSelf(collapsed)));
+    paddingValue = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getZeroAttr(getElementTypeOrSelf(collapsed)));
   }
   auto padOp =
       tensor::PadOp::create(rewriter, loc, collapsed, packOp.getSource(), lows,
-                                     highs, paddingValue, /*nofold=*/false);
+                            highs, paddingValue, /*nofold=*/false);
 
   LLVM_DEBUG(
       DBGSNL(); DBGSNL();
@@ -313,8 +313,8 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       SmallVector<OpFoldResult> sizes =
           tensor::getMixedSizes(rewriter, loc, packOp.getDest());
 
-      auto insertSliceOp = tensor::InsertSliceOp::create(rewriter,
-          loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
+      auto insertSliceOp = tensor::InsertSliceOp::create(
+          rewriter, loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
           /*offsets=*/zeros, sizes, /*strides=*/ones);
 
       LLVM_DEBUG(DBGS() << "insert_slice op: " << insertSliceOp; DBGSNL(););
@@ -329,15 +329,15 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   // 5. Expand from the padded result to the stripMinedShape.
   auto expandShapeResultType =
       RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape);
-  auto reshapeOp = tensor::ExpandShapeOp::create(rewriter,
-      loc, expandShapeResultType, padOp.getResult(),
+  auto reshapeOp = tensor::ExpandShapeOp::create(
+      rewriter, loc, expandShapeResultType, padOp.getResult(),
       packingMetadata.reassociations);
 
   // 6. Transpose stripMinedShape to packedShape.
   SmallVector<int64_t> transpPerm =
       invertPermutationVector(packedToStripMinedShapePerm);
-  auto transposeOp = linalg::TransposeOp::create(rewriter,
-      loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
+  auto transposeOp = linalg::TransposeOp::create(
+      rewriter, loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
 
   LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
              DBGS() << "reshape op: " << reshapeOp; DBGSNL();
@@ -371,8 +371,8 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
     SmallVector<OpFoldResult> sizes(packedRank - destShape.size(), one);
     sizes.append(tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()));
 
-    auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
-        loc, destTensorType, unPackOp.getSource(),
+    auto extractSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, loc, destTensorType, unPackOp.getSource(),
         SmallVector<OpFoldResult>(packedRank, zero), sizes,
         SmallVector<OpFoldResult>(packedRank, one));
 
@@ -404,10 +404,11 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
   SmallVector<OpFoldResult, 4> dims =
       tensor::getMixedSizes(rewriter, loc, unPackOp.getSource());
   applyPermutationToVector(dims, packedToStripMinedShapePerm);
-  auto emptyOp = tensor::EmptyOp::create(rewriter,
-      loc, dims, stripMinedTensorType.getElementType());
-  auto transposeOp = linalg::TransposeOp::create(rewriter,
-      loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm);
+  auto emptyOp = tensor::EmptyOp::create(rewriter, loc, dims,
+                                         stripMinedTensorType.getElementType());
+  auto transposeOp =
+      linalg::TransposeOp::create(rewriter, loc, unPackOp.getSource(), emptyOp,
+                                  packedToStripMinedShapePerm);
 
   LLVM_DEBUG(
       DBGSNL(); DBGSNL();
@@ -426,21 +427,21 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
       DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
 
   // 4. Collapse from the stripMinedShape to the padded result.
-  auto reshapeOp = tensor::CollapseShapeOp::create(rewriter,
-      loc, collapsedType, transposeOp->getResult(0),
+  auto reshapeOp = tensor::CollapseShapeOp::create(
+      rewriter, loc, collapsedType, transposeOp->getResult(0),
       packingMetadata.reassociations);
 
   // 5. ExtractSlice.
   int64_t destRank = destTensorType.getRank();
-  auto extractSliceOp = tensor::ExtractSliceOp::create(rewriter,
-      loc, destTensorType, reshapeOp->getResult(0),
+  auto extractSliceOp = tensor::ExtractSliceOp::create(
+      rewriter, loc, destTensorType, reshapeOp->getResult(0),
       SmallVector<OpFoldResult>(destRank, zero),
       tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()),
       SmallVector<OpFoldResult>(destRank, one));
 
   // 6. Inject a copy to preserve DPS.
-  auto copyOp = linalg::CopyOp::create(rewriter,
-      loc, extractSliceOp->getResult(0), unPackOp.getDest());
+  auto copyOp = linalg::CopyOp::create(
+      rewriter, loc, extractSliceOp->getResult(0), unPackOp.getDest());
 
   // 7. Replace unPackOp by copyOp.
   rewriter.replaceOp(unPackOp, copyOp->getResults());
@@ -554,16 +555,16 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
               operandType.getShape(), innerPos,
               cast<ShapedType>(dest.getType()).getShape(), {},
               innerPackSizes)) {
-        packOps.push_back(linalg::PackOp::create(rewriter,
-            loc, operand, dest, innerPos, innerPackSizes));
+        packOps.push_back(linalg::PackOp::create(rewriter, loc, operand, dest,
+                                                 innerPos, innerPackSizes));
       } else {
         // TODO: value of the padding attribute should be determined by
         // consumers.
         auto zeroAttr =
             rewriter.getZeroAttr(getElementTypeOrSelf(dest.getType()));
         Value zero = arith::ConstantOp::create(rewriter, loc, zeroAttr);
-        packOps.push_back(linalg::PackOp::create(rewriter,
-            loc, operand, dest, innerPos, innerPackSizes, zero));
+        packOps.push_back(linalg::PackOp::create(
+            rewriter, loc, operand, dest, innerPos, innerPackSizes, zero));
       }
       inputsAndInits.push_back(packOps.back());
     }
@@ -574,9 +575,9 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       ValueRange{inputsAndInits}.take_front(linalgOp.getNumDpsInputs());
   ValueRange inits =
       ValueRange{inputsAndInits}.take_back(linalgOp.getNumDpsInits());
-  auto packedLinalgOp = linalg::GenericOp::create(rewriter,
-      linalgOp.getLoc(), inits.getTypes(), inputs, inits, indexingMaps,
-      iteratorTypes);
+  auto packedLinalgOp =
+      linalg::GenericOp::create(rewriter, linalgOp.getLoc(), inits.getTypes(),
+                                inputs, inits, indexingMaps, iteratorTypes);
   packedLinalgOp.getRegion().takeBody(linalgOp->getRegion(0));
 
   // Step 4. Propagate packing to all the op results.
@@ -589,8 +590,8 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
       continue;
     }
     // Build the symmetrical UnPackOp to the existing PackOp.
-    unPackOps.push_back(linalg::UnPackOp::create(rewriter,
-        packedLinalgOp->getLoc(), result, maybePackedInit.getSource(),
+    unPackOps.push_back(linalg::UnPackOp::create(
+        rewriter, packedLinalgOp->getLoc(), result, maybePackedInit.getSource(),
         maybePackedInit.getInnerDimsPos(), maybePackedInit.getMixedTiles()));
     results.push_back(unPackOps.back());
   }
@@ -655,7 +656,8 @@ static LinalgOp transposeOneLinalgOperandAndReplace(
   operands[opOperand.getOperandNumber()] = transposedValue;
 
   ValueRange operandsRef(operands);
-  auto transposedGenericOp = linalg::GenericOp::create(rewriter,
+  auto transposedGenericOp = linalg::GenericOp::create(
+      rewriter,
       /*location=*/linalgOp->getLoc(),
       /*resultTensorTypes=*/
       operandsRef.drop_front(linalgOp.getNumDpsInputs()).getTypes(),
@@ -930,8 +932,8 @@ Value DecomposePadOpPattern::createFillOrGenerateOp(
   }
 
   // Fill could not be optimized: Lower to tensor::GenerateOp with region.
-  auto generateOp = tensor::GenerateOp::create(rewriter,
-      padOp.getLoc(), padOp.getResultType(), dynSizes);
+  auto generateOp = tensor::GenerateOp::create(rewriter, padOp.getLoc(),
+                                               padOp.getResultType(), dynSizes);
   // Copy region to new op.
   IRMapping bvm;
   padOp.getRegion().cloneInto(&generateOp.getRegion(), bvm);
@@ -970,8 +972,9 @@ DecomposePadOpPattern::matchAndRewrite(tensor::PadOp padOp,
   }
 
   // Init tensor and fill it with padding.
-  Value emptyTensor = tensor::EmptyOp::create(rewriter,
-      padOp.getLoc(), staticSizes, resultType.getElementType(), dynSizes);
+  Value emptyTensor =
+      tensor::EmptyOp::create(rewriter, padOp.getLoc(), staticSizes,
+                              resultType.getElementType(), dynSizes);
   Value fill = createFillOrGenerateOp(rewriter, padOp, emptyTensor, dynSizes);
 
   // Generate a InsertSliceOp for copying the PadOp source.
@@ -1222,12 +1225,13 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
 
   applyPermutationToVector<OpFoldResult>(transShapeForEmptyOp,
                                          srcPermForTranspose);
-  Value empty = tensor::EmptyOp::create(rewriter,
-      loc, transShapeForEmptyOp, packOp.getSourceType().getElementType());
+  Value empty =
+      tensor::EmptyOp::create(rewriter, loc, transShapeForEmptyOp,
+                              packOp.getSourceType().getElementType());
 
   // 2.2 Create linalg.transpose
   auto transposedOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
-                                                           srcPermForTranspose);
+                                                  srcPermForTranspose);
 
   // 3. Insert the inner tile to the destination:
   //  %inserted_tile = tensor.insert_slice(%transposed_tile)
@@ -1246,9 +1250,9 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   }
 
   // 4. Replace tensor.packOp with tensor.insert_slice created above
-  auto insert = tensor::InsertSliceOp::create(rewriter,
-      loc, transposedOp.getResult()[0], packOp.getDest(), writeOffsets,
-      writeSizes, writeStrides);
+  auto insert = tensor::InsertSliceOp::create(
+      rewriter, loc, transposedOp.getResult()[0], packOp.getDest(),
+      writeOffsets, writeSizes, writeStrides);
   rewriter.replaceOp(packOp, insert.getResult());
 
   return success();
@@ -1340,8 +1344,8 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   readShapeForExtractSlice.append(tileShape.begin(), tileShape.end());
   Type elemType = unpackOp.getSourceType().getElementType();
   auto readType = RankedTensorType::get(readShapeForExtractSlice, elemType);
-  Value innerTile = tensor::ExtractSliceOp::create(rewriter,
-      loc, readType, unpackOp.getSource(), extractSliceOffsets,
+  Value innerTile = tensor::ExtractSliceOp::create(
+      rewriter, loc, readType, unpackOp.getSource(), extractSliceOffsets,
       extractSliceSizes, extractSliceStrides);
 
   // 2. Transpose the tile to match the outer corresponding tile order.
@@ -1369,8 +1373,9 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
           tensor::getMixedSize(rewriter, loc, unpackOp.getDest(), i));
   }
 
-  auto partialTile = tensor::ExtractSliceOp::create(rewriter,
-      loc, transposedOp.getResult()[0], tileOffsets, tileSizes, tileStrides);
+  auto partialTile =
+      tensor::ExtractSliceOp::create(rewriter, loc, transposedOp.getResult()[0],
+                                     tileOffsets, tileSizes, tileStrides);
 
   // 4. Insert the result to the destination tensor.
   SmallVector<OpFoldResult> writeSizes;
@@ -1382,9 +1387,9 @@ LogicalResult DecomposeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
     else
       writeSizes.push_back(oneIdxAttr);
   }
-  auto insert = tensor::InsertSliceOp::create(rewriter,
-      loc, partialTile, unpackOp.getDest(), writeOffsets, writeSizes,
-      writeStrides);
+  auto insert = tensor::InsertSliceOp::create(rewriter, loc, partialTile,
+                                              unpackOp.getDest(), writeOffsets,
+                                              writeSizes, writeStrides);
   rewriter.replaceOp(unpackOp, insert.getResult());
 
   return success();
@@ -1491,8 +1496,8 @@ FailureOr<Conv1DOp> DownscaleSizeOneWindowed2DConvolution<Conv2DOp, Conv1DOp>::
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = Conv1DOp::create(rewriter,
-      loc, newOutputType, ValueRange{newInput, newKernel},
+  auto conv1DOp = Conv1DOp::create(
+      rewriter, loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
   // Insert back.
@@ -1578,8 +1583,8 @@ DownscaleDepthwiseConv2DNhwcHwcOp::returningMatchAndRewrite(
   dilations.erase(dilations.begin() + (removeH ? 0 : 1));
   auto dilationsAttr = rewriter.getI64VectorAttr(dilations);
 
-  auto conv1DOp = DepthwiseConv1DNwcWcOp::create(rewriter,
-      loc, newOutputType, ValueRange{newInput, newKernel},
+  auto conv1DOp = DepthwiseConv1DNwcWcOp::create(
+      rewriter, loc, newOutputType, ValueRange{newInput, newKernel},
       ValueRange{newOutput}, stridesAttr, dilationsAttr);
 
   // Insert back.
@@ -1635,9 +1640,9 @@ DownscaleConv2DOp::returningMatchAndRewrite(Conv2DOp convOp,
   Value newOutput = tensor::createCanonicalRankReducingExtractSliceOp(
       rewriter, loc, output, newOutputType);
 
-  auto conv1DOp = Conv1DOp::create(rewriter, loc, newOutputType,
-                                            ValueRange{newInput, newKernel},
-                                            ValueRange{newOutput});
+  auto conv1DOp =
+      Conv1DOp::create(rewriter, loc, newOutputType,
+                       ValueRange{newInput, newKernel}, ValueRange{newOutput});
 
   // Insert back.
   Value inserted = tensor::createCanonicalRankReducingInsertSliceOp(
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index b933a2d143925..99fb8c796cf06 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -99,7 +99,7 @@ FailureOr<Operation *> transposeConv2DHelper(RewriterBase &rewriter,
   }
   auto newConv =
       HWCFConvOp::create(rewriter, loc, resultTy, newInputs, op.getOutputs(),
-                                  op.getStrides(), op.getDilations());
+                         op.getStrides(), op.getDilations());
   rewriter.replaceOp(op, newConv);
   return newConv.getOperation();
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index 6b1e89c1594d1..a2a4335ef8e22 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -52,20 +52,20 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
     dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 0));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = tensor::EmptyOp::create(rewriter,
-      loc, ArrayRef<int64_t>{shape[1], shape[0]}, type.getElementType(),
-      dynamicDims);
-  auto transposeOp = linalg::TransposeOp::create(rewriter,
-      loc, input, empty, ArrayRef<int64_t>{1, 0});
+  Value empty = tensor::EmptyOp::create(rewriter, loc,
+                                        ArrayRef<int64_t>{shape[1], shape[0]},
+                                        type.getElementType(), dynamicDims);
+  auto transposeOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
+                                                 ArrayRef<int64_t>{1, 0});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = linalg::MatmulTransposeAOp::create(rewriter,
-        loc, matmulOp.getResultTypes(),
+    newMatmulOp = linalg::MatmulTransposeAOp::create(
+        rewriter, loc, matmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), matmulOp.getInputs()[1]},
         matmulOp.getOutputs());
   } else {
-    newMatmulOp = linalg::MatmulTransposeBOp::create(rewriter,
-        loc, matmulOp.getResultTypes(),
+    newMatmulOp = linalg::MatmulTransposeBOp::create(
+        rewriter, loc, matmulOp.getResultTypes(),
         ValueRange{matmulOp.getInputs()[0], transposeOp->getResult(0)},
         matmulOp.getOutputs());
   }
@@ -109,20 +109,20 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
     dynamicDims.push_back(tensor::DimOp::create(rewriter, loc, input, 1));
 
   ArrayRef<int64_t> shape = type.getShape();
-  Value empty = tensor::EmptyOp::create(rewriter,
-      loc, ArrayRef<int64_t>{shape[0], shape[2], shape[1]},
+  Value empty = tensor::EmptyOp::create(
+      rewriter, loc, ArrayRef<int64_t>{shape[0], shape[2], shape[1]},
       type.getElementType(), dynamicDims);
-  auto transposeOp = linalg::TransposeOp::create(rewriter,
-      loc, input, empty, ArrayRef<int64_t>{0, 2, 1});
+  auto transposeOp = linalg::TransposeOp::create(rewriter, loc, input, empty,
+                                                 ArrayRef<int64_t>{0, 2, 1});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(rewriter,
-        loc, batchMatmulOp.getResultTypes(),
+    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(
+        rewriter, loc, batchMatmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), batchMatmulOp.getInputs()[1]},
         batchMatmulOp.getOutputs());
   } else {
-    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(rewriter,
-        loc, batchMatmulOp.getResultTypes(),
+    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(
+        rewriter, loc, batchMatmulOp.getResultTypes(),
         ValueRange{batchMatmulOp.getInputs()[0], transposeOp->getResult(0)},
         batchMatmulOp.getOutputs());
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 2fff28ffc7474..272537f22deb3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -119,8 +119,9 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-            loc, input, /*offsets=*/ArrayRef<int64_t>{w + kw}, sizes, strides));
+        result.push_back(vector::ExtractStridedSliceOp::create(
+            rewriter, loc, input, /*offsets=*/ArrayRef<int64_t>{w + kw}, sizes,
+            strides));
       }
     }
   } else {
@@ -130,8 +131,8 @@ extractConvInputSlices(RewriterBase &rewriter, Location loc, Value input,
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-            loc, input,
+        result.push_back(vector::ExtractStridedSliceOp::create(
+            rewriter, loc, input,
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             sizes, strides));
       }
@@ -149,8 +150,8 @@ static SmallVector<Value> extractConvFilterSlices(RewriterBase &rewriter,
   // Extract rhs slice of size [{c, f} for channeled convolutions and {1} for
   // non-chanelled convolution] @ [kw].
   for (int64_t kw = 0; kw < kwSize; ++kw) {
-    result.push_back(vector::ExtractOp::create(rewriter,
-        loc, filter, /*offsets=*/ArrayRef<int64_t>{kw}));
+    result.push_back(vector::ExtractOp::create(
+        rewriter, loc, filter, /*offsets=*/ArrayRef<int64_t>{kw}));
   }
   return result;
 }
@@ -167,8 +168,9 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {wSizeStep};
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-          loc, res, /*offsets=*/ArrayRef<int64_t>{w}, sizes, strides));
+      result.push_back(vector::ExtractStridedSliceOp::create(
+          rewriter, loc, res, /*offsets=*/ArrayRef<int64_t>{w}, sizes,
+          strides));
     }
   } else {
     // Extract res slice: {n, wSizeStep, f} @ [0, w, 0] for channeled
@@ -176,8 +178,9 @@ extractConvResultSlices(RewriterBase &rewriter, Location loc, Value res,
     SmallVector<int64_t> sizes = {nSize, wSizeStep, fSize};
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      result.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-          loc, res, /*offsets=*/ArrayRef<int64_t>{0, w, 0}, sizes, strides));
+      result.push_back(vector::ExtractStridedSliceOp::create(
+          rewriter, loc, res, /*offsets=*/ArrayRef<int64_t>{0, w, 0}, sizes,
+          strides));
     }
   }
   return result;
@@ -194,17 +197,18 @@ static Value insertConvResultSlices(RewriterBase &rewriter, Location loc,
     // This does not depend on kw.
     SmallVector<int64_t> strides = {1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = vector::InsertStridedSliceOp::create(rewriter,
-          loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{w}, strides);
+      res = vector::InsertStridedSliceOp::create(
+          rewriter, loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{w},
+          strides);
     }
   } else {
     // Write back res slice: {n, wSizeStep, f} @ [0, w, 0] for channeled
     // convolution. This does not depend on kw.
     SmallVector<int64_t> strides = {1, 1, 1};
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      res = vector::InsertStridedSliceOp::create(rewriter,
-          loc, resVals[w], res, /*offsets=*/ArrayRef<int64_t>{0, w, 0},
-          strides);
+      res = vector::InsertStridedSliceOp::create(
+          rewriter, loc, resVals[w], res,
+          /*offsets=*/ArrayRef<int64_t>{0, w, 0}, strides);
     }
   }
   return res;
@@ -337,8 +341,8 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
   for (int vecDim = 0, end = canonicalVecShape.size(); vecDim < end; ++vecDim) {
     if (ShapedType::isStatic(iterSpaceStaticSizes[vecDim])) {
       // Create constant index op for static dimensions.
-      iterSpaceValueSizes.push_back(arith::ConstantIndexOp::create(rewriter,
-          linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
+      iterSpaceValueSizes.push_back(arith::ConstantIndexOp::create(
+          rewriter, linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
       continue;
     }
 
@@ -350,11 +354,12 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
                                                          operandDimPos)))
       return failure();
 
-    Value dynamicDim = linalgOp.hasPureTensorSemantics()
-                           ? (Value)tensor::DimOp::create(rewriter,
-                                 linalgOp.getLoc(), operand, operandDimPos)
-                           : (Value)memref::DimOp::create(rewriter,
-                                 linalgOp.getLoc(), operand, operandDimPos);
+    Value dynamicDim =
+        linalgOp.hasPureTensorSemantics()
+            ? (Value)tensor::DimOp::create(rewriter, linalgOp.getLoc(), operand,
+                                           operandDimPos)
+            : (Value)memref::DimOp::create(rewriter, linalgOp.getLoc(), operand,
+                                           operandDimPos);
     iterSpaceValueSizes.push_back(dynamicDim);
   }
 
@@ -475,7 +480,7 @@ Value VectorizationState::getOrCreateMaskFor(
 
   // Create the mask based on the dimension values.
   Value mask = vector::CreateMaskOp::create(rewriter, linalgOp.getLoc(),
-                                                     maskType, upperBounds);
+                                            maskType, upperBounds);
   LDBG("Creating new mask: " << mask << "\n");
   activeMaskCache[maskingMap] = mask;
   return mask;
@@ -643,8 +648,8 @@ static Operation *buildMultiDimReduce(OpBuilder &b, Operation *reduceOp,
                                       ArrayRef<bool> dimsToMask) {
   auto maybeKind = getCombinerOpKind(reduceOp);
   assert(maybeKind && "Failed precondition: could not get reduction kind");
-  return vector::MultiDimReductionOp::create(b,
-      reduceOp->getLoc(), valueToReduce, acc, dimsToMask, *maybeKind);
+  return vector::MultiDimReductionOp::create(
+      b, reduceOp->getLoc(), valueToReduce, acc, dimsToMask, *maybeKind);
 }
 
 static SmallVector<bool> getDimsToReduce(LinalgOp linalgOp) {
@@ -688,19 +693,20 @@ static Value buildVectorWrite(RewriterBase &rewriter, Value value,
   Operation *write;
   if (vectorType.getRank() > 0) {
     AffineMap writeMap = inversePermutation(reindexIndexingMap(opOperandMap));
-    SmallVector<Value> indices(linalgOp.getRank(outputOperand),
-                               arith::ConstantIndexOp::create(rewriter, loc, 0));
+    SmallVector<Value> indices(
+        linalgOp.getRank(outputOperand),
+        arith::ConstantIndexOp::create(rewriter, loc, 0));
     value = broadcastIfNeeded(rewriter, value, vectorType);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = vector::TransferWriteOp::create(rewriter,
-        loc, value, outputOperand->get(), indices, writeMap);
+    write = vector::TransferWriteOp::create(
+        rewriter, loc, value, outputOperand->get(), indices, writeMap);
   } else {
     // 0-d case is still special: do not invert the reindexing writeMap.
     if (!isa<VectorType>(value.getType()))
       value = vector::BroadcastOp::create(rewriter, loc, vectorType, value);
     assert(value.getType() == vectorType && "Incorrect type");
-    write = vector::TransferWriteOp::create(rewriter,
-        loc, value, outputOperand->get(), ValueRange{});
+    write = vector::TransferWriteOp::create(rewriter, loc, value,
+                                            outputOperand->get(), ValueRange{});
   }
 
   write = state.maskOperation(rewriter, write, linalgOp, opOperandMap);
@@ -793,9 +799,9 @@ static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
   auto permMap =
       AffineMap::getPermutationMap(permPattern, linalgOp.getContext());
 
-  auto broadCastOp = vector::BroadcastOp::create(rewriter,
-      loc, state.getCanonicalVecType(rewriter.getIndexType(), permMap),
-      indexSteps);
+  auto broadCastOp = vector::BroadcastOp::create(
+      rewriter, loc,
+      state.getCanonicalVecType(rewriter.getIndexType(), permMap), indexSteps);
   SmallVector<int64_t> transposition =
       llvm::to_vector<16>(llvm::seq<int64_t>(0, linalgOp.getNumLoops()));
   std::swap(transposition.back(), transposition[dim]);
@@ -1110,12 +1116,12 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
 
   // Compute the static loop sizes of the extract op.
   auto resultType = state.getCanonicalVecType(extractOp.getResult().getType());
-  auto maskConstantOp = arith::ConstantOp::create(rewriter,
-      loc,
+  auto maskConstantOp = arith::ConstantOp::create(
+      rewriter, loc,
       DenseIntElementsAttr::get(state.getCanonicalVecType(rewriter.getI1Type()),
                                 /*value=*/true));
-  auto passThruConstantOp =
-      arith::ConstantOp::create(rewriter, loc, rewriter.getZeroAttr(resultType));
+  auto passThruConstantOp = arith::ConstantOp::create(
+      rewriter, loc, rewriter.getZeroAttr(resultType));
 
   // Base indices are currently set to 0. We will need to re-visit if more
   // generic scenarios are to be supported.
@@ -1131,8 +1137,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     Value offset = calculateGatherOffset(rewriter, state, extractOp, bvm);
 
     // Generate the gather load
-    Operation *gatherOp = vector::GatherOp::create(rewriter,
-        loc, resultType, extractOp.getTensor(), baseIndices, offset,
+    Operation *gatherOp = vector::GatherOp::create(
+        rewriter, loc, resultType, extractOp.getTensor(), baseIndices, offset,
         maskConstantOp, passThruConstantOp);
     gatherOp = state.maskOperation(rewriter, gatherOp, linalgOp);
 
@@ -1166,8 +1172,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
       continue;
     }
 
-    auto indexAs1dVector = vector::ShapeCastOp::create(rewriter,
-        loc,
+    auto indexAs1dVector = vector::ShapeCastOp::create(
+        rewriter, loc,
         VectorType::get(resultType.getShape().back(), rewriter.getIndexType(),
                         resultType.getScalableDims().back()),
         idx);
@@ -1186,8 +1192,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     SmallVector<AffineExpr> exprs(dstRank, getAffineConstantExpr(0, ctx));
     auto permutationMap = AffineMap::get(srcRank, 0, exprs, ctx);
 
-    auto transferReadOp = vector::TransferReadOp::create(rewriter,
-        loc, resultType, extractOp.getTensor(), transferReadIdxs,
+    auto transferReadOp = vector::TransferReadOp::create(
+        rewriter, loc, resultType, extractOp.getTensor(), transferReadIdxs,
         /*padding=*/std::nullopt, permutationMap, inBounds);
 
     // Mask this broadcasting xfer_read here rather than relying on the generic
@@ -1195,8 +1201,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     // valid here).
     SmallVector<int64_t> readMaskShape = {1};
     auto readMaskType = VectorType::get(readMaskShape, rewriter.getI1Type());
-    auto allTrue = vector::ConstantMaskOp::create(rewriter,
-        loc, readMaskType, vector::ConstantMaskKind::AllTrue);
+    auto allTrue = vector::ConstantMaskOp::create(
+        rewriter, loc, readMaskType, vector::ConstantMaskKind::AllTrue);
     auto *maskedReadOp =
         mlir::vector::maskOperation(rewriter, transferReadOp, allTrue);
 
@@ -1223,8 +1229,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     rankDiff--;
   }
 
-  auto transferReadOp = vector::TransferReadOp::create(rewriter,
-      loc, resultType, extractOp.getTensor(), transferReadIdxs,
+  auto transferReadOp = vector::TransferReadOp::create(
+      rewriter, loc, resultType, extractOp.getTensor(), transferReadIdxs,
       /*padding=*/std::nullopt, permutationMap, inBounds);
 
   LDBG("Vectorised as contiguous load: " << extractOp);
@@ -1435,8 +1441,8 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
-    Operation *read = vector::TransferReadOp::create(rewriter,
-        loc, readType, opOperand->get(), indices,
+    Operation *read = vector::TransferReadOp::create(
+        rewriter, loc, readType, opOperand->get(), indices,
         /*padding=*/std::nullopt, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
     Value readValue = read->getResult(0);
@@ -1453,7 +1459,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     // TODO: remove this.
     if (readType.getRank() == 0)
       readValue = vector::ExtractOp::create(rewriter, loc, readValue,
-                                                     ArrayRef<int64_t>());
+                                            ArrayRef<int64_t>());
 
     LDBG("New vectorized bbarg(" << bbarg.getArgNumber() << "): " << readValue
                                  << "\n");
@@ -1665,12 +1671,11 @@ createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
   }
 
   // Generate the xfer_write Op
-  Operation *write =
-      vector::TransferWriteOp::create(builder, loc,
-                                              /*vector=*/vecToStore,
-                                              /*source=*/dest,
-                                              /*indices=*/writeIndices,
-                                              /*inBounds=*/inBoundsVal);
+  Operation *write = vector::TransferWriteOp::create(builder, loc,
+                                                     /*vector=*/vecToStore,
+                                                     /*source=*/dest,
+                                                     /*indices=*/writeIndices,
+                                                     /*inBounds=*/inBoundsVal);
 
   // If masking is disabled, exit.
   if (useInBoundsInsteadOfMasking)
@@ -1742,8 +1747,9 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   Location loc = packOp.getLoc();
   auto padValue = packOp.getPaddingValue();
   if (!padValue) {
-    padValue = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(packOp.getSourceType().getElementType()));
+    padValue = arith::ConstantOp::create(
+        rewriter, loc,
+        rewriter.getZeroAttr(packOp.getSourceType().getElementType()));
   }
   ReifiedRankedShapedTypeDims reifiedReturnShapes;
   LogicalResult status =
@@ -1787,12 +1793,12 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
   // Create TransposeOp.
   auto destPermutation =
       invertPermutationVector(getPackInverseDestPerm(packOp));
-  auto transposeOp = vector::TransposeOp::create(rewriter,
-      loc, shapeCastOp.getResult(), destPermutation);
+  auto transposeOp = vector::TransposeOp::create(
+      rewriter, loc, shapeCastOp.getResult(), destPermutation);
 
   // Create TransferWriteOp.
-  Value dest = tensor::EmptyOp::create(rewriter,
-      loc, reifiedReturnShapes[0],
+  Value dest = tensor::EmptyOp::create(
+      rewriter, loc, reifiedReturnShapes[0],
       transposeOp.getResult().getType().getElementType());
   Operation *write =
       createWriteOrMaskedWrite(rewriter, loc, transposeOp.getResult(), dest);
@@ -1892,8 +1898,9 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   }
   Location loc = unpackOp->getLoc();
 
-  auto padValue = arith::ConstantOp::create(rewriter,
-      loc, rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
+  auto padValue = arith::ConstantOp::create(
+      rewriter, loc,
+      rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
 
   // Read result, mask if necessary. If transferReadOp shape is not equal
   // to shape of source, then a mask is necessary.
@@ -1911,16 +1918,16 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   RankedTensorType stripMineTensorType =
       RankedTensorType::get(stripMineShape, stripMineElemType);
   // Transpose the appropriate rows to match output.
-  vector::TransposeOp transposeOp = vector::TransposeOp::create(rewriter,
-      loc, readResult, lastDimToInsertPosPerm);
+  vector::TransposeOp transposeOp = vector::TransposeOp::create(
+      rewriter, loc, readResult, lastDimToInsertPosPerm);
 
   // Collapse the vector to the size required by result.
   RankedTensorType collapsedType = tensor::CollapseShapeOp::inferCollapsedType(
       stripMineTensorType, packMetadata.reassociations);
   mlir::VectorType vecCollapsedType =
       VectorType::get(collapsedType.getShape(), collapsedType.getElementType());
-  vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(rewriter,
-      loc, vecCollapsedType, transposeOp->getResult(0));
+  vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(
+      rewriter, loc, vecCollapsedType, transposeOp->getResult(0));
 
   // writeVectorSizes had to match the shapecast shape for dynamic sizes,
   // otherwise the validator complains that the mask size is invalid.
@@ -1928,8 +1935,8 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
       unpackOp.getDestType().hasStaticShape()
           ? vectorSizes
           : shapeCastOp.getResultVectorType().getShape());
-  Value dest = tensor::EmptyOp::create(rewriter,
-      loc, reifiedRetShapes[0],
+  Value dest = tensor::EmptyOp::create(
+      rewriter, loc, reifiedRetShapes[0],
       shapeCastOp.getResult().getType().getElementType());
   Operation *write = createWriteOrMaskedWrite(
       rewriter, loc, shapeCastOp.getResult(), dest,
@@ -1963,8 +1970,8 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
       /*useInBoundsInsteadOfMasking=*/false);
 
   // Create Xfer write Op
-  Value dest = tensor::EmptyOp::create(rewriter,
-      loc, reifiedReturnShapes[0], padOp.getResultType().getElementType());
+  Value dest = tensor::EmptyOp::create(rewriter, loc, reifiedReturnShapes[0],
+                                       padOp.getResultType().getElementType());
   Operation *write = createWriteOrMaskedWrite(rewriter, loc, maskedRead, dest);
   newResults.push_back(write->getResult(0));
   return success();
@@ -2637,17 +2644,18 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   SmallVector<Value> indices(srcType.getRank(), zero);
 
-  Value readValue = vector::TransferReadOp::create(rewriter,
-      loc, readType, copyOp.getSource(), indices,
+  Value readValue = vector::TransferReadOp::create(
+      rewriter, loc, readType, copyOp.getSource(), indices,
       /*padding=*/std::nullopt,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   if (cast<VectorType>(readValue.getType()).getRank() == 0) {
+    readValue = vector::ExtractOp::create(rewriter, loc, readValue,
+                                          ArrayRef<int64_t>());
     readValue =
-        vector::ExtractOp::create(rewriter, loc, readValue, ArrayRef<int64_t>());
-    readValue = vector::BroadcastOp::create(rewriter, loc, writeType, readValue);
+        vector::BroadcastOp::create(rewriter, loc, writeType, readValue);
   }
-  Operation *writeValue = vector::TransferWriteOp::create(rewriter,
-      loc, readValue, copyOp.getTarget(), indices,
+  Operation *writeValue = vector::TransferWriteOp::create(
+      rewriter, loc, readValue, copyOp.getTarget(), indices,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   rewriter.replaceOp(copyOp, writeValue->getResults());
   return success();
@@ -2957,8 +2965,8 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
 
   if (!padValue) {
     auto elemType = sourceType.getElementType();
-    padValue = arith::ConstantOp::create(rewriter,
-        sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
+    padValue = arith::ConstantOp::create(rewriter, sliceOp.getLoc(), elemType,
+                                         rewriter.getZeroAttr(elemType));
   }
 
   // 2. Get the vector shape
@@ -3077,8 +3085,9 @@ struct PadOpVectorizationWithInsertSlicePattern
     // padding.
     SmallVector<Value> readIndices(
         vecRank, arith::ConstantIndexOp::create(rewriter, padOp.getLoc(), 0));
-    auto read = vector::TransferReadOp::create(rewriter,
-        padOp.getLoc(), vecType, padOp.getSource(), readIndices, padValue);
+    auto read = vector::TransferReadOp::create(rewriter, padOp.getLoc(),
+                                               vecType, padOp.getSource(),
+                                               readIndices, padValue);
 
     // Generate TransferWriteOp: Write to InsertSliceOp's dest tensor at
     // specified offsets. Write is fully in-bounds because a InsertSliceOp's
@@ -3212,8 +3221,8 @@ LogicalResult LinalgCopyVTRForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_read, the attribute must be reset
   // conservatively.
   auto vectorType = xferOp.getVectorType();
-  Value res = vector::TransferReadOp::create(rewriter,
-      xferOp.getLoc(), vectorType, in, xferOp.getIndices(),
+  Value res = vector::TransferReadOp::create(
+      rewriter, xferOp.getLoc(), vectorType, in, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getPadding(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(
           SmallVector<bool>(vectorType.getRank(), false)));
@@ -3271,8 +3280,8 @@ LogicalResult LinalgCopyVTWForwardingPattern::matchAndRewrite(
   // When forwarding to vector.transfer_write, the attribute must be reset
   // conservatively.
   auto vector = xferOp.getVector();
-  vector::TransferWriteOp::create(rewriter,
-      xferOp.getLoc(), vector, out, xferOp.getIndices(),
+  vector::TransferWriteOp::create(
+      rewriter, xferOp.getLoc(), vector, out, xferOp.getIndices(),
       xferOp.getPermutationMapAttr(), xferOp.getMask(),
       rewriter.getBoolArrayAttr(SmallVector<bool>(
           dyn_cast<VectorType>(vector.getType()).getRank(), false)));
@@ -3486,17 +3495,17 @@ struct Conv1DGenerator
     SmallVector<Value> resPadding(resShape.size(), zero);
 
     // Read the whole lhs, rhs and res in one shot (with zero padding).
-    Value lhs = vector::TransferReadOp::create(rewriter,
-        loc, lhsType, lhsShaped, lhsPadding,
+    Value lhs = vector::TransferReadOp::create(
+        rewriter, loc, lhsType, lhsShaped, lhsPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     // This is needed only for Conv.
     Value rhs = nullptr;
     if (oper == ConvOperationKind::Conv)
-      rhs = vector::TransferReadOp::create(rewriter,
-          loc, rhsType, rhsShaped, rhsPadding,
+      rhs = vector::TransferReadOp::create(
+          rewriter, loc, rhsType, rhsShaped, rhsPadding,
           /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
-    Value res = vector::TransferReadOp::create(rewriter,
-        loc, resType, resShaped, resPadding,
+    Value res = vector::TransferReadOp::create(
+        rewriter, loc, resType, resShaped, resPadding,
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
 
     // The base vectorization case for channeled convolution is input:
@@ -3633,8 +3642,8 @@ struct Conv1DGenerator
     bindDims(ctx, n, w, f, c);
     lhs = promote(rewriter, loc, lhs, res.getType());
     rhs = promote(rewriter, loc, rhs, res.getType());
-    auto contrationOp = vector::ContractionOp::create(rewriter,
-        loc, lhs, rhs, res,
+    auto contrationOp = vector::ContractionOp::create(
+        rewriter, loc, lhs, rhs, res,
         /*indexingMaps=*/MapList{{n, w, c}, {c, f}, {n, w, f}},
         /*iteratorTypes=*/ArrayRef<vector::IteratorType>{par, par, par, red});
     contrationOp.setKind(reductionKind);
@@ -3645,8 +3654,8 @@ struct Conv1DGenerator
   // convolution.
   Value conv1dSliceAsOuterProduct(RewriterBase &rewriter, Location loc,
                                   Value lhs, Value rhs, Value res) {
-    return vector::OuterProductOp::create(rewriter,
-        loc, res.getType(), lhs, rhs, res, vector::CombiningKind::ADD);
+    return vector::OuterProductOp::create(rewriter, loc, res.getType(), lhs,
+                                          rhs, res, vector::CombiningKind::ADD);
   }
 
   // Create a reduction: lhs{n, w, c} -> res{n, w, c}
@@ -3743,22 +3752,22 @@ struct Conv1DGenerator
 
     // Read lhs slice of size {n, w * strideW + kw * dilationW, c} @ [0, 0,
     // 0].
-    Value lhs = vector::TransferReadOp::create(rewriter,
-        loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
+    Value lhs = vector::TransferReadOp::create(
+        rewriter, loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     auto maybeMaskedLhs = maybeMaskXferOp(
         lhsType.getShape(), lhsType.getScalableDims(), lhs.getDefiningOp());
 
     // Read rhs slice of size {kw, c} @ [0, 0].
-    Value rhs = vector::TransferReadOp::create(rewriter,
-        loc, rhsType, rhsShaped, ValueRange{zero, zero},
+    Value rhs = vector::TransferReadOp::create(
+        rewriter, loc, rhsType, rhsShaped, ValueRange{zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
     auto maybeMaskedRhs = maybeMaskXferOp(
         rhsType.getShape(), rhsType.getScalableDims(), rhs.getDefiningOp());
 
     // Read res slice of size {n, w, c} @ [0, 0, 0].
-    Value res = vector::TransferReadOp::create(rewriter,
-        loc, resType, resShaped, ValueRange{zero, zero, zero},
+    Value res = vector::TransferReadOp::create(
+        rewriter, loc, resType, resShaped, ValueRange{zero, zero, zero},
         /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
     auto maybeMaskedRes = maybeMaskXferOp(
         resType.getShape(), resType.getScalableDims(), res.getDefiningOp());
@@ -3775,22 +3784,22 @@ struct Conv1DGenerator
     //   @ [0, sw * w + dw * kw, 0].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
       for (int64_t w = 0; w < wSize; w += wSizeStep) {
-        lhsVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-            loc, maybeMaskedLhs->getResult(0),
+        lhsVals.push_back(vector::ExtractStridedSliceOp::create(
+            rewriter, loc, maybeMaskedLhs->getResult(0),
             /*offsets=*/ArrayRef<int64_t>{0, w * strideW + kw * dilationW, 0},
             inOutSliceSizes, inOutStrides));
       }
     }
     // Extract rhs slice of size {c} @ [kw].
     for (int64_t kw = 0; kw < kwSize; ++kw) {
-      rhsVals.push_back(vector::ExtractOp::create(rewriter,
-          loc, maybeMaskedRhs->getResult(0),
-          /*offsets=*/ArrayRef<int64_t>{kw}));
+      rhsVals.push_back(
+          vector::ExtractOp::create(rewriter, loc, maybeMaskedRhs->getResult(0),
+                                    /*offsets=*/ArrayRef<int64_t>{kw}));
     }
     // Extract res slice: {n, wSizeStep, c} @ [0, w, 0].
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      resVals.push_back(vector::ExtractStridedSliceOp::create(rewriter,
-          loc, maybeMaskedRes->getResult(0),
+      resVals.push_back(vector::ExtractStridedSliceOp::create(
+          rewriter, loc, maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0}, inOutSliceSizes,
           inOutStrides));
     }
@@ -3815,17 +3824,19 @@ struct Conv1DGenerator
         if (flatten) {
           // Flatten the input and output vectors (collapse the channel
           // dimension)
-          lhsVal = vector::ShapeCastOp::create(rewriter,
-              loc, lhsTypeAfterFlattening, lhsVals[linearIndex(kw, w)]);
-          resVal = vector::ShapeCastOp::create(rewriter,
-              loc, resTypeAfterFlattening, resVals[w]);
+          lhsVal =
+              vector::ShapeCastOp::create(rewriter, loc, lhsTypeAfterFlattening,
+                                          lhsVals[linearIndex(kw, w)]);
+          resVal = vector::ShapeCastOp::create(
+              rewriter, loc, resTypeAfterFlattening, resVals[w]);
         }
         resVals[w] = depthwiseConv1dSliceAsMulAcc(rewriter, loc, lhsVal,
                                                   rhsVals[kw], resVal, flatten);
         if (flatten) {
           // Un-flatten the output vector (restore the channel dimension)
-          resVals[w] = vector::ShapeCastOp::create(rewriter,
-              loc, VectorType::get(inOutSliceSizes, resEltType), resVals[w]);
+          resVals[w] = vector::ShapeCastOp::create(
+              rewriter, loc, VectorType::get(inOutSliceSizes, resEltType),
+              resVals[w]);
         }
       }
     }
@@ -3843,8 +3854,8 @@ struct Conv1DGenerator
     // Write back res slice: {n, wSizeStep, c} @ [0, w, 0].
     // This does not depend on kw.
     for (int64_t w = 0; w < wSize; w += wSizeStep) {
-      maybeMaskedRes = vector::InsertStridedSliceOp::create(rewriter,
-          loc, resVals[w], maybeMaskedRes->getResult(0),
+      maybeMaskedRes = vector::InsertStridedSliceOp::create(
+          rewriter, loc, resVals[w], maybeMaskedRes->getResult(0),
           /*offsets=*/ArrayRef<int64_t>{0, w, 0},
           /*strides=*/ArrayRef<int64_t>{1, 1, 1});
     }
@@ -3853,8 +3864,8 @@ struct Conv1DGenerator
     //===------------------------------------------------------------------===//
 
     // Write back res slice of size {n, w, c} @ [0, 0, 0].
-    Operation *resOut = vector::TransferWriteOp::create(rewriter,
-        loc, maybeMaskedRes->getResult(0), resShaped,
+    Operation *resOut = vector::TransferWriteOp::create(
+        rewriter, loc, maybeMaskedRes->getResult(0), resShaped,
         ValueRange{zero, zero, zero});
     return maybeMaskXferOp(resType.getShape(), resType.getScalableDims(),
                            resOut);
@@ -3894,8 +3905,8 @@ struct Conv1DGenerator
       rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, indices);
     }
     // Broadcast the filter to match the output vector
-    rhs = vector::BroadcastOp::create(rewriter,
-        loc, resTy.clone(rhsTy.getElementType()), rhs);
+    rhs = vector::BroadcastOp::create(rewriter, loc,
+                                      resTy.clone(rhsTy.getElementType()), rhs);
 
     rhs = promote(rewriter, loc, rhs, resTy);
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index f9c6aae9374a8..669fefcd86de1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -201,11 +201,12 @@ Value create2DTransformMatrix(OpBuilder &builder, Location loc,
                               TransformMatrix transform, Type type) {
   ArrayRef<float> constVec(transform.table, transform.rows * transform.cols);
 
-  return arith::ConstantOp::create(builder,
-      loc, DenseFPElementsAttr::get(
-               RankedTensorType::get(
-                   SmallVector<int64_t>{transform.rows, transform.cols}, type),
-               constVec));
+  return arith::ConstantOp::create(
+      builder, loc,
+      DenseFPElementsAttr::get(
+          RankedTensorType::get(
+              SmallVector<int64_t>{transform.rows, transform.cols}, type),
+          constVec));
 }
 
 /// Extract height x width data from 4D tensors.
@@ -233,8 +234,8 @@ Value extract2DDataFrom4D(OpBuilder &builder, Location loc, Value source,
 
   auto extractFilterType =
       RankedTensorType::get({extractHeight, extractWidth}, elementType);
-  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
-      loc, extractFilterType, source, offsets, sizes, strides);
+  auto extractFilterOp = tensor::ExtractSliceOp::create(
+      builder, loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
 }
@@ -267,8 +268,8 @@ Value extract2DDataFrom6D(OpBuilder &builder, Location loc, Value source,
   SmallVector<OpFoldResult> strides(srcSize, oneIndex);
 
   auto extractFilterType = RankedTensorType::get({height, width}, elementType);
-  auto extractFilterOp = tensor::ExtractSliceOp::create(builder,
-      loc, extractFilterType, source, offsets, sizes, strides);
+  auto extractFilterOp = tensor::ExtractSliceOp::create(
+      builder, loc, extractFilterType, source, offsets, sizes, strides);
 
   return extractFilterOp;
 }
@@ -293,8 +294,8 @@ Value insert2DDataTo4D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
-      loc, source, dest, retOffsets, retSizes, strides);
+  auto insertSliceOp = tensor::InsertSliceOp::create(
+      builder, loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
 }
@@ -321,8 +322,8 @@ Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
   retSizes[widthIdx] = builder.getIndexAttr(width);
   SmallVector<OpFoldResult> strides(destSize, oneIndex);
 
-  auto insertSliceOp = tensor::InsertSliceOp::create(builder,
-      loc, source, dest, retOffsets, retSizes, strides);
+  auto insertSliceOp = tensor::InsertSliceOp::create(
+      builder, loc, source, dest, retOffsets, retSizes, strides);
 
   return insertSliceOp;
 }
@@ -386,8 +387,8 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
 
     int64_t retRows = 1;
     Value matmulRetValue = extractFilter;
-    Value zero = arith::ConstantOp::create(builder,
-        loc, rewriter.getZeroAttr(elementType));
+    Value zero = arith::ConstantOp::create(builder, loc,
+                                           rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix G.
       auto it = GMatrices.find(fmr);
@@ -401,12 +402,14 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
+      auto init =
+          linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value G = create2DTransformMatrix(builder, loc, GMatrix, elementType);
       // Multiply G x g.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{G, extractFilter}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{G, extractFilter},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -423,12 +426,14 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
+      auto init =
+          linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value GT = create2DTransformMatrix(builder, loc, GTMatrix, elementType);
       // Multiply u = (G x g) x GT.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{matmulRetValue, GT}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{matmulRetValue, GT},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -516,10 +521,11 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = affine::AffineApplyOp::create(builder,
-        loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = affine::AffineApplyOp::create(builder,
-        loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
+    Value heightOffset = affine::AffineApplyOp::create(
+        builder, loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
+    Value widthOffset = affine::AffineApplyOp::create(
+        builder, loc, rightTransform ? affineMap : identityAffineMap,
+        tileWIter);
 
     // Extract (H, W) from (N, H, W, C).
     auto extractInput =
@@ -530,8 +536,8 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
     int64_t retRows = 1;
     int64_t retCols = 1;
     Value matmulRetValue = extractInput;
-    Value zero = arith::ConstantOp::create(builder,
-        loc, rewriter.getZeroAttr(elementType));
+    Value zero = arith::ConstantOp::create(builder, loc,
+                                           rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix BT.
       auto it = BTMatrices.find(fmr);
@@ -545,13 +551,15 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
+      auto init =
+          linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
 
       Value BT =
           create2DTransformMatrix(builder, loc, BTMatrix, builder.getF32Type());
       // Multiply BT x d.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{BT, matmulRetValue}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{BT, matmulRetValue},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -568,12 +576,14 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
           builder
               .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
               .getResult();
-      auto init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
+      auto init =
+          linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
       Value B =
           create2DTransformMatrix(builder, loc, BMatrix, builder.getF32Type());
       // Multiply v = (BT x d) x B.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{matmulRetValue, B}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{matmulRetValue, B},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -629,8 +639,8 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       {filterShape[0] * filterShape[1], filterShape[2], filterShape[3]},
       filterElementType);
   SmallVector<ReassociationIndices> filterReassoc = {{0, 1}, {2}, {3}};
-  Value collapseFilter = tensor::CollapseShapeOp::create(rewriter,
-      loc, filterReassocType, transformedFilter, filterReassoc);
+  Value collapseFilter = tensor::CollapseShapeOp::create(
+      rewriter, loc, filterReassocType, transformedFilter, filterReassoc);
 
   // Convert (alphaH, alphaW, tileH, tileW, N, C) to
   // (alphaH x alphaW, tileH x tileW x N, C) for input.
@@ -643,8 +653,8 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
        inputShape[2] * inputShape[3] * inputShape[4], inputShape[5]},
       inputElementType);
   SmallVector<ReassociationIndices> inputReassoc = {{0, 1}, {2, 3, 4}, {5}};
-  Value collapseInput = tensor::CollapseShapeOp::create(rewriter,
-      loc, inputReassocType, transformedInput, inputReassoc);
+  Value collapseInput = tensor::CollapseShapeOp::create(
+      rewriter, loc, inputReassocType, transformedInput, inputReassoc);
 
   // Batched matrix multiply.
   auto matmulType = RankedTensorType::get(
@@ -655,12 +665,12 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
                     .create<tensor::EmptyOp>(loc, matmulType.getShape(),
                                              outputElementType)
                     .getResult();
-  Value zero = arith::ConstantOp::create(rewriter,
-      loc, rewriter.getZeroAttr(outputElementType));
+  Value zero = arith::ConstantOp::create(
+      rewriter, loc, rewriter.getZeroAttr(outputElementType));
   Value init = linalg::FillOp::create(rewriter, loc, zero, empty).getResult(0);
 
-  auto matmulOp = linalg::BatchMatmulOp::create(rewriter,
-      loc, matmulType, ValueRange({collapseInput, collapseFilter}),
+  auto matmulOp = linalg::BatchMatmulOp::create(
+      rewriter, loc, matmulType, ValueRange({collapseInput, collapseFilter}),
       ValueRange{init});
 
   // The result shape of batch matmul is (alphaH x alphaW, tileH x tileW x N, F)
@@ -670,8 +680,8 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
       RankedTensorType::get({inputShape[0], inputShape[1], inputShape[2],
                              inputShape[3], inputShape[4], filterShape[3]},
                             outputElementType);
-  auto expandOutput = tensor::ExpandShapeOp::create(rewriter,
-      loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
+  auto expandOutput = tensor::ExpandShapeOp::create(
+      rewriter, loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
   return expandOutput;
 }
 
@@ -750,16 +760,17 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
     int64_t retRows = leftTransform ? ATMatrix.rows : 1;
 
     Value matmulRetValue = extractValue;
-    Value zero = arith::ConstantOp::create(builder,
-        loc, rewriter.getZeroAttr(elementType));
+    Value zero = arith::ConstantOp::create(builder, loc,
+                                           rewriter.getZeroAttr(elementType));
 
     auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
     auto affineMap =
         AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset = affine::AffineApplyOp::create(builder,
-        loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
-    Value widthOffset = affine::AffineApplyOp::create(builder,
-        loc, rightTransform ? affineMap : identityAffineMap, tileWIter);
+    Value heightOffset = affine::AffineApplyOp::create(
+        builder, loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
+    Value widthOffset = affine::AffineApplyOp::create(
+        builder, loc, rightTransform ? affineMap : identityAffineMap,
+        tileWIter);
 
     Value outInitVal =
         extract2DDataFrom4D(builder, loc, args[0], NIter, FIter, heightOffset,
@@ -780,8 +791,9 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
       Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
       // Multiply AT x m.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{AT, matmulRetValue}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{AT, matmulRetValue},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
@@ -799,15 +811,16 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
       Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
       // Multiply y = (AT x m) x A.
-      auto matmulOp = linalg::MatmulOp::create(builder,
-          loc, matmulType, ValueRange{matmulRetValue, A}, ValueRange{init});
+      auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
+                                               ValueRange{matmulRetValue, A},
+                                               ValueRange{init});
       matmulRetValue = matmulOp.getResult(0);
     }
 
     if (scalarFactor != 1) {
       // Multiply by scalar factor and add outInitVal.
-      Value scalarFactorValue = arith::ConstantOp::create(builder,
-          loc, FloatAttr::get(elementType, scalarFactor));
+      Value scalarFactorValue = arith::ConstantOp::create(
+          builder, loc, FloatAttr::get(elementType, scalarFactor));
       auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
       auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
       SmallVector<AffineMap> affineMaps = {
@@ -824,12 +837,12 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                       utils::IteratorType::parallel},
                   [&](OpBuilder &nestedBuilder, Location nestedLoc,
                       ValueRange args) {
-                    auto mulf = arith::MulFOp::create(nestedBuilder,
-                        nestedLoc, args[0], args[1]);
-                    auto addf = arith::AddFOp::create(nestedBuilder,
-                        nestedLoc, mulf.getResult(), args[2]);
+                    auto mulf = arith::MulFOp::create(nestedBuilder, nestedLoc,
+                                                      args[0], args[1]);
+                    auto addf = arith::AddFOp::create(
+                        nestedBuilder, nestedLoc, mulf.getResult(), args[2]);
                     linalg::YieldOp::create(nestedBuilder, nestedLoc,
-                                                          addf.getResult());
+                                            addf.getResult());
                   })
               .getResult(0);
     }
@@ -867,8 +880,8 @@ static Value padToAlignedTensor(RewriterBase &rewriter, Location loc,
   auto valueType = cast<ShapedType>(value.getType());
   Type elementType = valueType.getElementType();
   auto alignedType = RankedTensorType::get(alignedShape, elementType);
-  Value padValue = arith::ConstantOp::create(rewriter,
-      loc, elementType, rewriter.getZeroAttr(elementType));
+  Value padValue = arith::ConstantOp::create(rewriter, loc, elementType,
+                                             rewriter.getZeroAttr(elementType));
 
   return linalg::makeComposedPadHighOp(rewriter, loc, alignedType, value,
                                        padValue, false);
@@ -888,7 +901,7 @@ static Value extractFromAlignedTensor(RewriterBase &rewriter, Location loc,
       getAsOpFoldResult(rewriter.getI64ArrayAttr(extractedShape));
 
   return tensor::ExtractSliceOp::create(rewriter, loc, extractedType, value,
-                                                 offsets, sizes, strides);
+                                        offsets, sizes, strides);
 }
 
 /// Utility function to check all values in the attribute are 1.
@@ -980,9 +993,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   auto retType = RankedTensorType::get({alphaH, alphaW, filterC, filterF},
                                        filterElementType);
   Value retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
-                                                    filterElementType);
-  auto transformedFilter = linalg::WinogradFilterTransformOp::create(rewriter,
-      loc, retType, filter, retValue, fmr);
+                                           filterElementType);
+  auto transformedFilter = linalg::WinogradFilterTransformOp::create(
+      rewriter, loc, retType, filter, retValue, fmr);
 
   // --- Create operation for input transform ---
 
@@ -999,9 +1012,9 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   retType = RankedTensorType::get(
       {alphaH, alphaW, tileH, tileW, inputN, inputC}, inputElementType);
   retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
-                                              inputElementType);
-  auto transformedInput = linalg::WinogradInputTransformOp::create(rewriter,
-      loc, retType, input, retValue, fmr);
+                                     inputElementType);
+  auto transformedInput = linalg::WinogradInputTransformOp::create(
+      rewriter, loc, retType, input, retValue, fmr);
 
   Type outputElementType = outputType.getElementType();
   Value matmulRet = matrixMultiply(rewriter, loc, transformedFilter,
@@ -1023,8 +1036,8 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
     outputType = alignedOutputType;
   }
 
-  Value transformedOutput = linalg::WinogradOutputTransformOp::create(rewriter,
-      loc, outputType, matmulRet, output, fmr);
+  Value transformedOutput = linalg::WinogradOutputTransformOp::create(
+      rewriter, loc, outputType, matmulRet, output, fmr);
 
   // When output size is not aligned with output tile size, extract the
   // value from the padded buffer.
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index d964aa5cb4e81..999a6381cd2e0 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -320,8 +320,8 @@ GenericOp makeMemRefCopyOp(OpBuilder &b, Location loc, Value from, Value to) {
       AffineMap::getMultiDimIdentityMap(memrefTypeTo.getRank(), b.getContext());
   SmallVector<utils::IteratorType> iteratorTypes(memrefTypeTo.getRank(),
                                                  utils::IteratorType::parallel);
-  return linalg::GenericOp::create(b,
-      loc,
+  return linalg::GenericOp::create(
+      b, loc,
       /*inputs=*/from,
       /*outputs=*/to,
       /*indexingMaps=*/llvm::ArrayRef({id, id}),
@@ -483,8 +483,8 @@ static void generateParallelLoopNest(
   case DistributionMethod::None: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    scf::ParallelOp::create(b,
-        loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
+    scf::ParallelOp::create(
+        b, loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
           ivStorage.append(localIvs.begin(), localIvs.end());
@@ -499,8 +499,8 @@ static void generateParallelLoopNest(
   case DistributionMethod::Cyclic: {
     // Generate a single parallel loop-nest operation for all outermost
     // parallel loops and recurse.
-    scf::ParallelOp::create(b,
-        loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
+    scf::ParallelOp::create(
+        b, loc, lbs.take_front(numProcessed), ubs.take_front(numProcessed),
         steps.take_front(numProcessed),
         [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange localIvs) {
           ivStorage.append(localIvs.begin(), localIvs.end());
@@ -595,13 +595,13 @@ static Operation *materializeTiledShape(OpBuilder &builder, Location loc,
   auto shapedType = dyn_cast<ShapedType>(valueToTile.getType());
   auto *sliceOp = TypeSwitch<ShapedType, Operation *>(shapedType)
                       .Case([&](MemRefType) {
-                        return memref::SubViewOp::create(builder,
-                            loc, valueToTile, sliceParams.offsets,
+                        return memref::SubViewOp::create(
+                            builder, loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
                       .Case([&](RankedTensorType) {
-                        return tensor::ExtractSliceOp::create(builder,
-                            loc, valueToTile, sliceParams.offsets,
+                        return tensor::ExtractSliceOp::create(
+                            builder, loc, valueToTile, sliceParams.offsets,
                             sliceParams.sizes, sliceParams.strides);
                       })
                       .Default([](ShapedType) -> Operation * {
@@ -793,8 +793,8 @@ SmallVector<Value> insertSlicesBack(OpBuilder &builder, Location loc,
     // `tiledOperands`.
     Value outputTensor = operands[opOperand.getOperandNumber()];
     if (auto sliceOp = outputTensor.getDefiningOp<tensor::ExtractSliceOp>()) {
-      Value inserted = tensor::InsertSliceOp::create(builder,
-          loc, sliceOp.getSource().getType(), results[resultIdx],
+      Value inserted = tensor::InsertSliceOp::create(
+          builder, loc, sliceOp.getSource().getType(), results[resultIdx],
           sliceOp.getSource(), sliceOp.getOffsets(), sliceOp.getSizes(),
           sliceOp.getStrides(), sliceOp.getStaticOffsets(),
           sliceOp.getStaticSizes(), sliceOp.getStaticStrides());
diff --git a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
index 9dadd9bea2f67..364e4d385fd62 100644
--- a/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -135,8 +135,9 @@ struct GlobalStoreOpInterface
     auto memrefType = getMemRefTypeWithStaticIdentityLayout(tensorType);
 
     auto loc = globalStoreOp.getLoc();
-    auto targetMemref = memref::GetGlobalOp::create(rewriter,
-        loc, memrefType, globalStoreOp.getGlobalAttr().getLeafReference());
+    auto targetMemref = memref::GetGlobalOp::create(
+        rewriter, loc, memrefType,
+        globalStoreOp.getGlobalAttr().getLeafReference());
 
     auto sourceMemref =
         getBuffer(rewriter, globalStoreOp.getValue(), options, state);
diff --git a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
index 770f32841c0f6..31785eb20a642 100644
--- a/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
@@ -91,8 +91,9 @@ PowFStrengthReduction::matchAndRewrite(math::PowFOp op,
 
   // Replace `pow(x, -1.0)` with `1.0 / x`.
   if (isExponentValue(-1.0)) {
-    Value one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getFloatAttr(getElementTypeOrSelf(op.getType()), 1.0));
+    Value one = arith::ConstantOp::create(
+        rewriter, loc,
+        rewriter.getFloatAttr(getElementTypeOrSelf(op.getType()), 1.0));
     rewriter.replaceOpWithNewOp<arith::DivFOp>(op, ValueRange({bcast(one), x}));
     return success();
   }
@@ -175,11 +176,11 @@ PowIStrengthReduction<PowIOpTy, DivOpTy, MulOpTy>::matchAndRewrite(
   Value one;
   Type opType = getElementTypeOrSelf(op.getType());
   if constexpr (std::is_same_v<PowIOpTy, math::FPowIOp>)
-    one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getFloatAttr(opType, 1.0));
+    one = arith::ConstantOp::create(rewriter, loc,
+                                    rewriter.getFloatAttr(opType, 1.0));
   else
-    one = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerAttr(opType, 1));
+    one = arith::ConstantOp::create(rewriter, loc,
+                                    rewriter.getIntegerAttr(opType, 1));
 
   // Replace `[fi]powi(x, 0)` with `1`.
   if (exponentValue == 0) {
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index b5c0ddd617805..5edb6e28fb018 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -33,7 +33,7 @@ static Value createFloatConst(Location loc, Type type, APFloat value,
   auto attr = b.getFloatAttr(eltType, value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
     return arith::ConstantOp::create(b, loc,
-                                       DenseElementsAttr::get(shapedTy, attr));
+                                     DenseElementsAttr::get(shapedTy, attr));
   }
 
   return arith::ConstantOp::create(b, loc, attr);
@@ -50,7 +50,7 @@ static Value createIntConst(Location loc, Type type, int64_t value,
   auto attr = b.getIntegerAttr(getElementTypeOrSelf(type), value);
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
     return arith::ConstantOp::create(b, loc,
-                                       DenseElementsAttr::get(shapedTy, attr));
+                                     DenseElementsAttr::get(shapedTy, attr));
   }
 
   return arith::ConstantOp::create(b, loc, attr);
@@ -116,8 +116,8 @@ static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) {
   Value negTwo = createFloatConst(loc, floatType, -2.0, rewriter);
 
   // Compute sign(x) = cast<float_type>(x < 0) * (-2) + 1
-  Value isNegative = arith::CmpFOp::create(rewriter,
-      loc, arith::CmpFPredicate::OLT, op.getOperand(), zero);
+  Value isNegative = arith::CmpFOp::create(
+      rewriter, loc, arith::CmpFPredicate::OLT, op.getOperand(), zero);
   Value isNegativeFloat =
       arith::UIToFPOp::create(rewriter, loc, floatType, isNegative);
   Value isNegativeTimesNegTwo =
@@ -236,8 +236,9 @@ static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) {
   Value one = createFloatConst(op->getLoc(), opType, 1.00, rewriter);
 
   Value gtCheck = arith::CmpFOp::create(b, arith::CmpFPredicate::OGT, operand,
-                                          fpFixedConvert);
-  Value incrValue = arith::SelectOp::create(b, op->getLoc(), gtCheck, one, zero);
+                                        fpFixedConvert);
+  Value incrValue =
+      arith::SelectOp::create(b, op->getLoc(), gtCheck, one, zero);
 
   Value ret = arith::AddFOp::create(b, opType, fpFixedConvert, incrValue);
   rewriter.replaceOp(op, ret);
@@ -259,7 +260,7 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
     Value castPowerToFp =
         arith::SIToFPOp::create(rewriter, op.getLoc(), baseType, power);
     Value res = math::PowFOp::create(rewriter, op.getLoc(), baseType, base,
-                                              castPowerToFp);
+                                     castPowerToFp);
     rewriter.replaceOp(op, res);
     return success();
   };
@@ -309,7 +310,7 @@ static LogicalResult convertFPowIOp(math::FPowIOp op,
     res =
         arith::SelectOp::create(b, op->getLoc(), zeroEqCheck, posInfinity, res);
     res = arith::SelectOp::create(b, op->getLoc(), negZeroEqCheck, negInfinity,
-                                    res);
+                                  res);
   }
 
   rewriter.replaceOp(op, res);
@@ -451,14 +452,14 @@ static LogicalResult convertRoundOp(math::RoundOp op,
   //
   // All three cases satisfy the property `biasedExp >= 23`.
   Value operandBitcast = arith::BitcastOp::create(b, i32Ty, operand);
-  Value operandExp = arith::AndIOp::create(b,
-      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandExp = arith::AndIOp::create(
+      b, arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
   Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
-  Value isSpecialValOrLargeVal =
-      arith::CmpIOp::create(b, arith::CmpIPredicate::sge, operandBiasedExp, c23);
+  Value isSpecialValOrLargeVal = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::sge, operandBiasedExp, c23);
 
   Value result = arith::SelectOp::create(b, isSpecialValOrLargeVal, operand,
-                                           fpFixedConvert);
+                                         fpFixedConvert);
   rewriter.replaceOp(op, result);
   return success();
 }
@@ -488,8 +489,8 @@ static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
     auto bits = createIntConst(loc, operandTy, half, rewriter);
     auto mask = createIntConst(loc, operandTy, allbits >> half, rewriter);
 
-    Value pred =
-        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ule, x, mask);
+    Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ule,
+                                       x, mask);
     Value add = arith::AddIOp::create(rewriter, loc, count, bits);
     Value shift = arith::ShLIOp::create(rewriter, loc, x, bits);
 
@@ -499,7 +500,7 @@ static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
 
   Value zero = createIntConst(loc, operandTy, 0, rewriter);
   Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                              operand, zero);
+                                     operand, zero);
 
   Value bwval = createIntConst(loc, operandTy, bitwidth, rewriter);
   Value sel = arith::SelectOp::create(rewriter, loc, pred, bwval, count);
@@ -554,11 +555,11 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   Value roundBitcast = arith::BitcastOp::create(b, iTy, round);
 
   // Get biased exponents for operand and round(operand)
-  Value operandExp = arith::AndIOp::create(b,
-      arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
+  Value operandExp = arith::AndIOp::create(
+      b, arith::ShRUIOp::create(b, operandBitcast, c23), expMask);
   Value operandBiasedExp = arith::SubIOp::create(b, operandExp, c127);
-  Value roundExp = arith::AndIOp::create(b,
-      arith::ShRUIOp::create(b, roundBitcast, c23), expMask);
+  Value roundExp = arith::AndIOp::create(
+      b, arith::ShRUIOp::create(b, roundBitcast, c23), expMask);
   Value roundBiasedExp = arith::SubIOp::create(b, roundExp, c127);
 
   auto safeShiftRight = [&](Value x, Value shift) -> Value {
@@ -592,8 +593,8 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
       arith::CmpIOp::create(b, arith::CmpIPredicate::eq, roundBiasedExp, c0);
   Value roundBiasedExpMinus1 = arith::SubIOp::create(b, roundBiasedExp, c1);
   Value roundMaskedMantissa = maskMantissa(roundBitcast, roundBiasedExpMinus1);
-  Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(b,
-      arith::CmpIPredicate::ne, roundMaskedMantissa, c0);
+  Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::ne, roundMaskedMantissa, c0);
   roundIsNotEvenOrSpecialVal =
       arith::OrIOp::create(b, roundIsNotEvenOrSpecialVal, roundBiasedExpEq0);
 
@@ -604,19 +605,19 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   // values +-0.5 are the only halfway values that have `biasedExp == -1 < 0`,
   // so these are handled separately. In particular, if `biasedExp == -1`, the
   // value is halfway if the entire mantissa is zero.
-  Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(b,
-      arith::CmpIPredicate::eq, operandBiasedExp, cNeg1);
-  Value expectedOperandMaskedMantissa = arith::SelectOp::create(b,
-      operandBiasedExpEqNeg1, c0, safeShiftRight(c2To22, operandBiasedExp));
+  Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::eq, operandBiasedExp, cNeg1);
+  Value expectedOperandMaskedMantissa = arith::SelectOp::create(
+      b, operandBiasedExpEqNeg1, c0, safeShiftRight(c2To22, operandBiasedExp));
   Value operandMaskedMantissa = maskMantissa(operandBitcast, operandBiasedExp);
   Value operandIsHalfway =
       arith::CmpIOp::create(b, arith::CmpIPredicate::eq, operandMaskedMantissa,
-                              expectedOperandMaskedMantissa);
+                            expectedOperandMaskedMantissa);
   // Ensure `biasedExp` is in the valid range for half values.
-  Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(b,
-      arith::CmpIPredicate::sge, operandBiasedExp, cNeg1);
-  Value operandBiasedExpLt23 =
-      arith::CmpIOp::create(b, arith::CmpIPredicate::slt, operandBiasedExp, c23);
+  Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::sge, operandBiasedExp, cNeg1);
+  Value operandBiasedExpLt23 = arith::CmpIOp::create(
+      b, arith::CmpIPredicate::slt, operandBiasedExp, c23);
   operandIsHalfway =
       arith::AndIOp::create(b, operandIsHalfway, operandBiasedExpLt23);
   operandIsHalfway =
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index baa35fb5a2c5d..76720cfd4a98c 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -156,16 +156,16 @@ handleMultidimensionalVectors(ImplicitLocOpBuilder &builder,
   // Stitch results together into one large vector.
   Type resultEltType = cast<VectorType>(results[0].getType()).getElementType();
   Type resultExpandedType = VectorType::get(expandedShape, resultEltType);
-  Value result = arith::ConstantOp::create(builder,
-      resultExpandedType, builder.getZeroAttr(resultExpandedType));
+  Value result = arith::ConstantOp::create(
+      builder, resultExpandedType, builder.getZeroAttr(resultExpandedType));
 
   for (int64_t i = 0; i < maxIndex; ++i)
     result = vector::InsertOp::create(builder, results[i], result,
-                                              delinearize(i, strides));
+                                      delinearize(i, strides));
 
   // Reshape back to the original vector shape.
-  return vector::ShapeCastOp::create(builder,
-      VectorType::get(inputShape, resultEltType), result);
+  return vector::ShapeCastOp::create(
+      builder, VectorType::get(inputShape, resultEltType), result);
 }
 
 //----------------------------------------------------------------------------//
@@ -181,7 +181,7 @@ static Value floatCst(ImplicitLocOpBuilder &builder, float value,
   assert((elementType.isF16() || elementType.isF32()) &&
          "x must be f16 or f32 type.");
   return arith::ConstantOp::create(builder,
-      builder.getFloatAttr(elementType, value));
+                                   builder.getFloatAttr(elementType, value));
 }
 
 static Value f32Cst(ImplicitLocOpBuilder &builder, double value) {
@@ -203,14 +203,16 @@ static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) {
 
 // Return the minimum of the two values or NaN if value is NaN
 static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return arith::SelectOp::create(builder,
+  return arith::SelectOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT, value, bound),
       value, bound);
 }
 
 // Return the maximum of the two values or NaN if value is NaN
 static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound) {
-  return arith::SelectOp::create(builder,
+  return arith::SelectOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::UGT, value, bound),
       value, bound);
 }
@@ -252,8 +254,8 @@ static std::pair<Value, Value> frexp(ImplicitLocOpBuilder &builder, Value arg,
 
   // Compute exponent.
   Value arg0 = isPositive ? arg : math::AbsFOp::create(builder, arg);
-  Value biasedExponentBits = arith::ShRUIOp::create(builder,
-      arith::BitcastOp::create(builder, i32Vec, arg0),
+  Value biasedExponentBits = arith::ShRUIOp::create(
+      builder, arith::BitcastOp::create(builder, i32Vec, arg0),
       bcast(i32Cst(builder, 23)));
   Value biasedExponent =
       arith::SIToFPOp::create(builder, f32Vec, biasedExponentBits);
@@ -301,7 +303,7 @@ Value makePolynomialCalculation(ImplicitLocOpBuilder &builder,
     return coeffs[0];
 
   Value res = math::FmaOp::create(builder, x, coeffs[coeffs.size() - 1],
-                                          coeffs[coeffs.size() - 2]);
+                                  coeffs[coeffs.size() - 2]);
   for (auto i = ptrdiff_t(coeffs.size()) - 3; i >= 0; --i) {
     res = math::FmaOp::create(builder, x, res, coeffs[i]);
   }
@@ -511,18 +513,20 @@ Atan2Approximation::matchAndRewrite(math::Atan2Op op,
   // Handle x = 0, y > 0
   Value xZero =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, x, zero);
-  Value yGt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, y, zero);
+  Value yGt =
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, y, zero);
   Value isHalfPi = arith::AndIOp::create(builder, xZero, yGt);
   auto halfPi = broadcast(builder, f32Cst(builder, 1.57079632679f), shape);
   result = arith::SelectOp::create(builder, isHalfPi, halfPi, result);
 
   // Handle x = 0, y < 0
-  Value yLt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, y, zero);
+  Value yLt =
+      arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, y, zero);
   Value isNegativeHalfPiPi = arith::AndIOp::create(builder, xZero, yLt);
   auto negativeHalfPiPi =
       broadcast(builder, f32Cst(builder, -1.57079632679f), shape);
-  result = arith::SelectOp::create(builder, isNegativeHalfPiPi, negativeHalfPiPi,
-                                           result);
+  result = arith::SelectOp::create(builder, isNegativeHalfPiPi,
+                                   negativeHalfPiPi, result);
 
   // Handle x = 0, y = 0;
   Value yZero =
@@ -569,9 +573,9 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
 
   // Mask for tiny values that are approximated with `operand`.
   Value tiny = bcast(f32Cst(builder, 0.0004f));
-  Value tinyMask = arith::CmpFOp::create(builder,
-      arith::CmpFPredicate::OLT, math::AbsFOp::create(builder, op.getOperand()),
-      tiny);
+  Value tinyMask = arith::CmpFOp::create(
+      builder, arith::CmpFPredicate::OLT,
+      math::AbsFOp::create(builder, op.getOperand()), tiny);
 
   // The monomial coefficients of the numerator polynomial (odd).
   Value alpha1 = bcast(f32Cst(builder, 4.89352455891786e-03f));
@@ -606,8 +610,8 @@ TanhApproximation::matchAndRewrite(math::TanhOp op,
   q = math::FmaOp::create(builder, x2, q, beta0);
 
   // Divide the numerator by the denominator.
-  Value res = arith::SelectOp::create(builder,
-      tinyMask, x, arith::DivFOp::create(builder, p, q));
+  Value res = arith::SelectOp::create(builder, tinyMask, x,
+                                      arith::DivFOp::create(builder, p, q));
 
   rewriter.replaceOp(op, res);
 
@@ -691,12 +695,12 @@ LogApproximationBase<Op>::logMatchAndRewrite(Op op, PatternRewriter &rewriter,
   //     x = x + x - 1.0;
   //   } else { x = x - 1.0; }
   Value mask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x,
-                                             cstCephesSQRTHF);
+                                     cstCephesSQRTHF);
   Value tmp = arith::SelectOp::create(builder, mask, x, cstZero);
 
   x = arith::SubFOp::create(builder, x, cstOne);
-  e = arith::SubFOp::create(builder,
-      e, arith::SelectOp::create(builder, mask, cstOne, cstZero));
+  e = arith::SubFOp::create(
+      builder, e, arith::SelectOp::create(builder, mask, cstOne, cstZero));
   x = arith::AddFOp::create(builder, x, tmp);
 
   Value x2 = arith::MulFOp::create(builder, x, x);
@@ -726,20 +730,20 @@ LogApproximationBase<Op>::logMatchAndRewrite(Op op, PatternRewriter &rewriter,
   }
 
   Value invalidMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::ULT,
-                                                    op.getOperand(), cstZero);
+                                            op.getOperand(), cstZero);
   Value zeroMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
-                                                 op.getOperand(), cstZero);
+                                         op.getOperand(), cstZero);
   Value posInfMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
-                                                   op.getOperand(), cstPosInf);
+                                           op.getOperand(), cstPosInf);
 
   // Filter out invalid values:
   //  • x == 0     -> -INF
   //  • x < 0      ->  NAN
   //  • x == +INF  -> +INF
-  Value aproximation = arith::SelectOp::create(builder,
-      zeroMask, cstMinusInf,
-      arith::SelectOp::create(builder,
-          invalidMask, cstNan,
+  Value aproximation = arith::SelectOp::create(
+      builder, zeroMask, cstMinusInf,
+      arith::SelectOp::create(
+          builder, invalidMask, cstNan,
           arith::SelectOp::create(builder, posInfMask, cstPosInf, x)));
 
   rewriter.replaceOp(op, aproximation);
@@ -811,11 +815,12 @@ Log1pApproximation::matchAndRewrite(math::Log1pOp op,
   Value logU = math::LogOp::create(builder, u);
   Value uInf =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, u, logU);
-  Value logLarge = arith::MulFOp::create(builder,
-      x, arith::DivFOp::create(builder,
-             logU, arith::SubFOp::create(builder, u, cstOne)));
-  Value approximation = arith::SelectOp::create(builder,
-      arith::OrIOp::create(builder, uSmall, uInf), x, logLarge);
+  Value logLarge = arith::MulFOp::create(
+      builder, x,
+      arith::DivFOp::create(builder, logU,
+                            arith::SubFOp::create(builder, u, cstOne)));
+  Value approximation = arith::SelectOp::create(
+      builder, arith::OrIOp::create(builder, uSmall, uInf), x, logLarge);
   rewriter.replaceOp(op, approximation);
   return success();
 }
@@ -866,7 +871,9 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
 
   auto abs = [&](Value a) -> Value { return math::AbsFOp::create(builder, a); };
 
-  auto sqrt = [&](Value a) -> Value { return math::SqrtOp::create(builder, a); };
+  auto sqrt = [&](Value a) -> Value {
+    return math::SqrtOp::create(builder, a);
+  };
 
   auto scopy = [&](Value a, Value b) -> Value {
     return math::CopySignOp::create(builder, a, b);
@@ -880,9 +887,8 @@ AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
   Value aa = mul(operand, operand);
   Value opp = sqrt(sub(bcast(floatCst(builder, 1.0, elementType)), aa));
 
-  Value gt =
-      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, aa,
-                                    bcast(floatCst(builder, 0.5, elementType)));
+  Value gt = arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, aa,
+                                   bcast(floatCst(builder, 0.5, elementType)));
 
   Value x = sel(gt, opp, abso);
 
@@ -978,8 +984,8 @@ AcosPolynomialApproximation::matchAndRewrite(math::AcosOp op,
   r = arith::SelectOp::create(builder, firstPred, trueVal, falseVal);
 
   // Check whether the operand lies in between [-1.0, 0.0).
-  Value greaterThanNegOne =
-      arith::CmpFOp::create(builder, arith::CmpFPredicate::OGE, operand, negOne);
+  Value greaterThanNegOne = arith::CmpFOp::create(
+      builder, arith::CmpFPredicate::OGE, operand, negOne);
 
   Value lessThanZero =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, operand, zero);
@@ -1094,22 +1100,22 @@ ErfPolynomialApproximation::matchAndRewrite(math::ErfOp op,
         arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, bounds[j]);
     for (int i = 0; i <= polyDegree; ++i) {
       p[i] = arith::SelectOp::create(builder, isLessThanBound[j], p[i],
-                                             pp[j + 1][i]);
+                                     pp[j + 1][i]);
       q[i] = arith::SelectOp::create(builder, isLessThanBound[j], q[i],
-                                             qq[j + 1][i]);
+                                     qq[j + 1][i]);
     }
     offset = arith::SelectOp::create(builder, isLessThanBound[j], offset,
-                                             offsets[j + 1]);
+                                     offsets[j + 1]);
   }
-  isLessThanBound[intervalsCount - 1] = arith::CmpFOp::create(builder,
-      arith::CmpFPredicate::ULT, x, bounds[intervalsCount - 1]);
+  isLessThanBound[intervalsCount - 1] = arith::CmpFOp::create(
+      builder, arith::CmpFPredicate::ULT, x, bounds[intervalsCount - 1]);
 
   Value pPoly = makePolynomialCalculation(builder, p, x);
   Value qPoly = makePolynomialCalculation(builder, q, x);
   Value rationalPoly = arith::DivFOp::create(builder, pPoly, qPoly);
   Value formula = arith::AddFOp::create(builder, offset, rationalPoly);
-  formula = arith::SelectOp::create(builder, isLessThanBound[intervalsCount - 1],
-                                            formula, one);
+  formula = arith::SelectOp::create(
+      builder, isLessThanBound[intervalsCount - 1], formula, one);
 
   // erf is odd function: erf(x) = -erf(-x).
   Value negFormula = arith::NegFOp::create(builder, formula);
@@ -1160,8 +1166,9 @@ ErfcPolynomialApproximation::matchAndRewrite(math::ErfcOp op,
   Value r = arith::DivFOp::create(builder, one, p);
   Value q = math::FmaOp::create(builder, neg4, r, one);
   Value t = math::FmaOp::create(builder, arith::AddFOp::create(builder, q, one),
-                                        neg2, a);
-  Value e = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), q, t);
+                                neg2, a);
+  Value e =
+      math::FmaOp::create(builder, arith::NegFOp::create(builder, a), q, t);
   q = math::FmaOp::create(builder, r, e, q);
 
   p = bcast(floatCst(builder, -0x1.a4a000p-12f, et));        // -4.01139259e-4
@@ -1197,23 +1204,24 @@ ErfcPolynomialApproximation::matchAndRewrite(math::ErfcOp op,
   e = math::ExpOp::create(builder, arith::NegFOp::create(builder, s));
 
   t = math::FmaOp::create(builder, arith::NegFOp::create(builder, a), a, s);
-  r = math::FmaOp::create(builder,
-      r, e,
+  r = math::FmaOp::create(
+      builder, r, e,
       arith::MulFOp::create(builder, arith::MulFOp::create(builder, r, e), t));
 
-  Value isNotLessThanInf = arith::XOrIOp::create(builder,
+  Value isNotLessThanInf = arith::XOrIOp::create(
+      builder,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, a, posInf),
       trueValue);
   r = arith::SelectOp::create(builder, isNotLessThanInf,
-                                      arith::AddFOp::create(builder, x, x), r);
+                              arith::AddFOp::create(builder, x, x), r);
   Value isGreaterThanClamp =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OGT, a, clampVal);
   r = arith::SelectOp::create(builder, isGreaterThanClamp, zero, r);
 
   Value isNegative =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT, x, zero);
-  r = arith::SelectOp::create(builder,
-      isNegative, arith::SubFOp::create(builder, pos2, r), r);
+  r = arith::SelectOp::create(builder, isNegative,
+                              arith::SubFOp::create(builder, pos2, r), r);
 
   rewriter.replaceOp(op, r);
   return success();
@@ -1235,8 +1243,9 @@ Value clampWithNormals(ImplicitLocOpBuilder &builder,
   };
 
   auto selectCmp = [&builder](auto pred, Value value, Value bound) {
-    return arith::SelectOp::create(builder,
-        arith::CmpFOp::create(builder, pred, value, bound), value, bound);
+    return arith::SelectOp::create(
+        builder, arith::CmpFOp::create(builder, pred, value, bound), value,
+        bound);
   };
 
   // Note: prefer UGE/ULE vs. UGT/ULT, since they generate vmaxps/vminps vs.
@@ -1434,8 +1443,8 @@ ExpM1Approximation::matchAndRewrite(math::ExpM1Op op,
   Value uEqOneOrNaN =
       arith::CmpFOp::create(builder, arith::CmpFPredicate::UEQ, u, cstOne);
   Value uMinusOne = arith::SubFOp::create(builder, u, cstOne);
-  Value uMinusOneEqNegOne = arith::CmpFOp::create(builder,
-      arith::CmpFPredicate::OEQ, uMinusOne, cstNegOne);
+  Value uMinusOneEqNegOne = arith::CmpFOp::create(
+      builder, arith::CmpFPredicate::OEQ, uMinusOne, cstNegOne);
   // logU = log(u) ~= x
   Value logU = math::LogOp::create(builder, u);
 
@@ -1444,11 +1453,11 @@ ExpM1Approximation::matchAndRewrite(math::ExpM1Op op,
       arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ, logU, u);
 
   // (u - 1) * (x / ~x)
-  Value expm1 = arith::MulFOp::create(builder,
-      uMinusOne, arith::DivFOp::create(builder, x, logU));
+  Value expm1 = arith::MulFOp::create(builder, uMinusOne,
+                                      arith::DivFOp::create(builder, x, logU));
   expm1 = arith::SelectOp::create(builder, isInf, u, expm1);
-  Value approximation = arith::SelectOp::create(builder,
-      uEqOneOrNaN, x,
+  Value approximation = arith::SelectOp::create(
+      builder, uEqOneOrNaN, x,
       arith::SelectOp::create(builder, uMinusOneEqNegOne, cstNegOne, expm1));
   rewriter.replaceOp(op, approximation);
   return success();
@@ -1723,10 +1732,10 @@ RsqrtApproximation::matchAndRewrite(math::RsqrtOp op,
 
   // Select only the inverse sqrt of positive normals (denormals are
   // flushed to zero).
-  Value ltMinMask = arith::CmpFOp::create(builder,
-      arith::CmpFPredicate::OLT, op.getOperand(), cstMinNormPos);
+  Value ltMinMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OLT,
+                                          op.getOperand(), cstMinNormPos);
   Value infMask = arith::CmpFOp::create(builder, arith::CmpFPredicate::OEQ,
-                                                op.getOperand(), cstPosInf);
+                                        op.getOperand(), cstPosInf);
   Value notNormalFiniteMask = arith::OrIOp::create(builder, ltMinMask, infMask);
 
   // Compute an approximate result.
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index 46298fb189e41..dfa2e4e0376ed 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -89,7 +89,7 @@ Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
       })
       .Default([&](Type t) {
         return arith::ConstantOp::create(builder, getLoc(), t,
-                                                 builder.getZeroAttr(t));
+                                         builder.getZeroAttr(t));
       });
 }
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
index a2b689ed13cbf..75cc39e61656a 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.cpp
@@ -35,8 +35,9 @@ struct DefaultAllocationInterface
   static ::std::optional<::mlir::Operation *>
   buildPromotedAlloc(OpBuilder &builder, Value alloc) {
     Operation *definingOp = alloc.getDefiningOp();
-    return memref::AllocaOp::create(builder,
-        definingOp->getLoc(), cast<MemRefType>(definingOp->getResultTypes()[0]),
+    return memref::AllocaOp::create(
+        builder, definingOp->getLoc(),
+        cast<MemRefType>(definingOp->getResultTypes()[0]),
         definingOp->getOperands(), definingOp->getAttrs());
   }
 };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
index 9d9923bfd0541..106c3b458dbac 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ComposeSubView.cpp
@@ -124,8 +124,8 @@ struct ComposeSubViewOpPattern : public OpRewritePattern<memref::SubViewOp> {
         }
 
         AffineMap map = AffineMap::get(0, affineApplyOperands.size(), expr);
-        Value result = affine::AffineApplyOp::create(rewriter,
-            op.getLoc(), map, affineApplyOperands);
+        Value result = affine::AffineApplyOp::create(rewriter, op.getLoc(), map,
+                                                     affineApplyOperands);
         offsets.push_back(result);
       }
     }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
index c35036d7631c8..5fe0d995f2c41 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp
@@ -112,8 +112,8 @@ static Value getSubByteWriteMask(Location loc, OpFoldResult linearizedIndices,
   auto dstIntegerType = builder.getIntegerType(dstBits);
   auto maskRightAlignedAttr =
       builder.getIntegerAttr(dstIntegerType, (1 << srcBits) - 1);
-  Value maskRightAligned = arith::ConstantOp::create(builder,
-      loc, dstIntegerType, maskRightAlignedAttr);
+  Value maskRightAligned = arith::ConstantOp::create(
+      builder, loc, dstIntegerType, maskRightAlignedAttr);
   Value writeMaskInverse =
       arith::ShLIOp::create(builder, loc, maskRightAligned, bitwidthOffset);
   auto flipValAttr = builder.getIntegerAttr(dstIntegerType, -1);
@@ -299,15 +299,15 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
     Value bitsLoad;
     if (convertedType.getRank() == 0) {
       bitsLoad = memref::LoadOp::create(rewriter, loc, adaptor.getMemref(),
-                                                 ValueRange{});
+                                        ValueRange{});
     } else {
       // Linearize the indices of the original load instruction. Do not account
       // for the scaling yet. This will be accounted for later.
       OpFoldResult linearizedIndices = getLinearizedSrcIndices(
           rewriter, loc, srcBits, adaptor.getIndices(), op.getMemRef());
 
-      Value newLoad = memref::LoadOp::create(rewriter,
-          loc, adaptor.getMemref(),
+      Value newLoad = memref::LoadOp::create(
+          rewriter, loc, adaptor.getMemref(),
           getIndicesForLoadOrStore(rewriter, loc, linearizedIndices, srcBits,
                                    dstBits));
 
@@ -331,8 +331,8 @@ struct ConvertMemRefLoad final : OpConversionPattern<memref::LoadOp> {
             : IntegerType::get(rewriter.getContext(),
                                resultTy.getIntOrFloatBitWidth());
     if (conversionTy == convertedElementType) {
-      auto mask = arith::ConstantOp::create(rewriter,
-          loc, convertedElementType,
+      auto mask = arith::ConstantOp::create(
+          rewriter, loc, convertedElementType,
           rewriter.getIntegerAttr(convertedElementType, (1 << srcBits) - 1));
 
       result = arith::AndIOp::create(rewriter, loc, bitsLoad, mask);
@@ -435,14 +435,13 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
           input);
     }
     Value extendedInput =
-        arith::ExtUIOp::create(rewriter, loc, dstIntegerType,
-                                                          input);
+        arith::ExtUIOp::create(rewriter, loc, dstIntegerType, input);
 
     // Special case 0-rank memref stores. No need for masking.
     if (convertedType.getRank() == 0) {
       memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::assign,
-                                           extendedInput, adaptor.getMemref(),
-                                           ValueRange{});
+                                  extendedInput, adaptor.getMemref(),
+                                  ValueRange{});
       rewriter.eraseOp(op);
       return success();
     }
@@ -461,12 +460,10 @@ struct ConvertMemrefStore final : OpConversionPattern<memref::StoreOp> {
 
     // Clear destination bits
     memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::andi,
-                                         writeMask, adaptor.getMemref(),
-                                         storeIndices);
+                                writeMask, adaptor.getMemref(), storeIndices);
     // Write srcs bits to destination
     memref::AtomicRMWOp::create(rewriter, loc, arith::AtomicRMWKind::ori,
-                                         alignedVal, adaptor.getMemref(),
-                                         storeIndices);
+                                alignedVal, adaptor.getMemref(), storeIndices);
     rewriter.eraseOp(op);
     return success();
   }
@@ -526,8 +523,8 @@ struct ConvertMemRefSubview final : OpConversionPattern<memref::SubViewOp> {
     }
 
     // Transform the offsets, sizes and strides according to the emulation.
-    auto stridedMetadata = memref::ExtractStridedMetadataOp::create(rewriter,
-        loc, subViewOp.getViewSource());
+    auto stridedMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, subViewOp.getViewSource());
 
     OpFoldResult linearizedIndices;
     auto strides = stridedMetadata.getConstifiedMixedStrides();
@@ -637,53 +634,52 @@ static SmallVector<int64_t> getLinearizedShape(MemRefType ty, int srcBits,
 
 void memref::populateMemRefNarrowTypeEmulationConversions(
     arith::NarrowTypeEmulationConverter &typeConverter) {
-  typeConverter.addConversion(
-      [&typeConverter](MemRefType ty) -> std::optional<Type> {
-        Type elementType = ty.getElementType();
-        if (!elementType.isIntOrFloat())
-          return ty;
-
-        unsigned width = elementType.getIntOrFloatBitWidth();
-        unsigned loadStoreWidth = typeConverter.getLoadStoreBitwidth();
-        if (width >= loadStoreWidth)
-          return ty;
-
-        // Currently only handle innermost stride being 1, checking
-        SmallVector<int64_t> strides;
-        int64_t offset;
-        if (failed(ty.getStridesAndOffset(strides, offset)))
-          return nullptr;
-        if (!strides.empty() && strides.back() != 1)
-          return nullptr;
-
-        auto newElemTy = IntegerType::get(
-            ty.getContext(), loadStoreWidth,
-            elementType.isInteger()
-                ? cast<IntegerType>(elementType).getSignedness()
-                : IntegerType::SignednessSemantics::Signless);
-        if (!newElemTy)
-          return nullptr;
-
-        StridedLayoutAttr layoutAttr;
-        // If the offset is 0, we do not need a strided layout as the stride is
-        // 1, so we only use the strided layout if the offset is not 0.
-        if (offset != 0) {
-          if (offset == ShapedType::kDynamic) {
-            layoutAttr = StridedLayoutAttr::get(ty.getContext(), offset,
-                                                ArrayRef<int64_t>{1});
-          } else {
-            // Check if the number of bytes are a multiple of the loadStoreWidth
-            // and if so, divide it by the loadStoreWidth to get the offset.
-            if ((offset * width) % loadStoreWidth != 0)
-              return std::nullopt;
-            offset = (offset * width) / loadStoreWidth;
-
-            layoutAttr = StridedLayoutAttr::get(ty.getContext(), offset,
-                                                ArrayRef<int64_t>{1});
-          }
-        }
-
-        return MemRefType::get(getLinearizedShape(ty, width, loadStoreWidth),
-                               newElemTy, layoutAttr, ty.getMemorySpace());
-      });
+  typeConverter.addConversion([&typeConverter](
+                                  MemRefType ty) -> std::optional<Type> {
+    Type elementType = ty.getElementType();
+    if (!elementType.isIntOrFloat())
+      return ty;
+
+    unsigned width = elementType.getIntOrFloatBitWidth();
+    unsigned loadStoreWidth = typeConverter.getLoadStoreBitwidth();
+    if (width >= loadStoreWidth)
+      return ty;
+
+    // Currently only handle innermost stride being 1, checking
+    SmallVector<int64_t> strides;
+    int64_t offset;
+    if (failed(ty.getStridesAndOffset(strides, offset)))
+      return nullptr;
+    if (!strides.empty() && strides.back() != 1)
+      return nullptr;
+
+    auto newElemTy = IntegerType::get(
+        ty.getContext(), loadStoreWidth,
+        elementType.isInteger() ? cast<IntegerType>(elementType).getSignedness()
+                                : IntegerType::SignednessSemantics::Signless);
+    if (!newElemTy)
+      return nullptr;
+
+    StridedLayoutAttr layoutAttr;
+    // If the offset is 0, we do not need a strided layout as the stride is
+    // 1, so we only use the strided layout if the offset is not 0.
+    if (offset != 0) {
+      if (offset == ShapedType::kDynamic) {
+        layoutAttr = StridedLayoutAttr::get(ty.getContext(), offset,
+                                            ArrayRef<int64_t>{1});
+      } else {
+        // Check if the number of bytes are a multiple of the loadStoreWidth
+        // and if so, divide it by the loadStoreWidth to get the offset.
+        if ((offset * width) % loadStoreWidth != 0)
+          return std::nullopt;
+        offset = (offset * width) / loadStoreWidth;
+
+        layoutAttr = StridedLayoutAttr::get(ty.getContext(), offset,
+                                            ArrayRef<int64_t>{1});
+      }
+    }
+
+    return MemRefType::get(getLinearizedShape(ty, width, loadStoreWidth),
+                           newElemTy, layoutAttr, ty.getMemorySpace());
+  });
 }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
index 34f9afae9d2ad..17a148cc31dc0 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
@@ -51,8 +51,8 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
         Value index = arith::ConstantIndexOp::create(rewriter, loc, i);
         size = memref::LoadOp::create(rewriter, loc, op.getShape(), index);
         if (!isa<IndexType>(size.getType()))
-          size = arith::IndexCastOp::create(rewriter,
-              loc, rewriter.getIndexType(), size);
+          size = arith::IndexCastOp::create(rewriter, loc,
+                                            rewriter.getIndexType(), size);
         sizes[i] = size;
       } else {
         auto sizeAttr = rewriter.getIndexAttr(op.getType().getDimSize(i));
@@ -68,8 +68,9 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
         if (stride) {
           stride = arith::MulIOp::create(rewriter, loc, stride, size);
         } else if (op.getType().isDynamicDim(i)) {
-          stride = arith::MulIOp::create(rewriter,
-              loc, arith::ConstantIndexOp::create(rewriter, loc, staticStride),
+          stride = arith::MulIOp::create(
+              rewriter, loc,
+              arith::ConstantIndexOp::create(rewriter, loc, staticStride),
               size);
         } else {
           staticStride *= op.getType().getDimSize(i);
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
index b54fa61714323..01d32621b2055 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandRealloc.cpp
@@ -89,9 +89,9 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
     Value lhs = getValueOrCreateConstantIndexOp(rewriter, loc, currSize);
     Value rhs = getValueOrCreateConstantIndexOp(rewriter, loc, targetSize);
     Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ult,
-                                                lhs, rhs);
-    auto ifOp = scf::IfOp::create(rewriter,
-        loc, cond,
+                                       lhs, rhs);
+    auto ifOp = scf::IfOp::create(
+        rewriter, loc, cond,
         [&](OpBuilder &builder, Location loc) {
           // Allocate the new buffer. If it is a dynamic memref we need to pass
           // an additional operand for the size at runtime, otherwise the static
@@ -100,15 +100,16 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           if (op.getDynamicResultSize())
             dynamicSizeOperands.push_back(op.getDynamicResultSize());
 
-          Value newAlloc = memref::AllocOp::create(builder,
-              loc, op.getResult().getType(), dynamicSizeOperands,
+          Value newAlloc = memref::AllocOp::create(
+              builder, loc, op.getResult().getType(), dynamicSizeOperands,
               op.getAlignmentAttr());
 
           // Take a subview of the new (bigger) buffer such that we can copy the
           // old values over (the copy operation requires both operands to have
           // the same shape).
-          Value subview = memref::SubViewOp::create(builder,
-              loc, newAlloc, ArrayRef<OpFoldResult>{rewriter.getIndexAttr(0)},
+          Value subview = memref::SubViewOp::create(
+              builder, loc, newAlloc,
+              ArrayRef<OpFoldResult>{rewriter.getIndexAttr(0)},
               ArrayRef<OpFoldResult>{currSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
           memref::CopyOp::create(builder, loc, op.getSource(), subview);
@@ -126,9 +127,10 @@ struct ExpandReallocOpPattern : public OpRewritePattern<memref::ReallocOp> {
           // dynamic or vice-versa. If both are static and the original buffer
           // is already bigger than the requested size, the cast represents a
           // subview operation.
-          Value casted = memref::ReinterpretCastOp::create(builder,
-              loc, cast<MemRefType>(op.getResult().getType()), op.getSource(),
-              rewriter.getIndexAttr(0), ArrayRef<OpFoldResult>{targetSize},
+          Value casted = memref::ReinterpretCastOp::create(
+              builder, loc, cast<MemRefType>(op.getResult().getType()),
+              op.getSource(), rewriter.getIndexAttr(0),
+              ArrayRef<OpFoldResult>{targetSize},
               ArrayRef<OpFoldResult>{rewriter.getIndexAttr(1)});
           scf::YieldOp::create(builder, loc, casted);
         });
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index f581b6490fadb..9771bd2aaa143 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -828,8 +828,8 @@ struct ExtractStridedMetadataOpAllocFolder
       if (allocLikeOp.getType() == baseBufferType)
         results.push_back(allocLikeOp);
       else
-        results.push_back(memref::ReinterpretCastOp::create(rewriter,
-            loc, baseBufferType, allocLikeOp, offset,
+        results.push_back(memref::ReinterpretCastOp::create(
+            rewriter, loc, baseBufferType, allocLikeOp, offset,
             /*sizes=*/ArrayRef<int64_t>(),
             /*strides=*/ArrayRef<int64_t>()));
     }
@@ -900,8 +900,8 @@ struct ExtractStridedMetadataOpGetGlobalFolder
     if (getGlobalOp.getType() == baseBufferType)
       results.push_back(getGlobalOp);
     else
-      results.push_back(memref::ReinterpretCastOp::create(rewriter,
-          loc, baseBufferType, getGlobalOp, offset,
+      results.push_back(memref::ReinterpretCastOp::create(
+          rewriter, loc, baseBufferType, getGlobalOp, offset,
           /*sizes=*/ArrayRef<int64_t>(),
           /*strides=*/ArrayRef<int64_t>()));
 
@@ -1008,9 +1008,8 @@ class ExtractStridedMetadataOpReinterpretCastFolder
     SmallVector<OpFoldResult> results;
     results.resize_for_overwrite(rank * 2 + 2);
 
-    auto newExtractStridedMetadata =
-        memref::ExtractStridedMetadataOp::create(rewriter,
-            loc, reinterpretCastOp.getSource());
+    auto newExtractStridedMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, reinterpretCastOp.getSource());
 
     // Register the base_buffer.
     results[0] = newExtractStridedMetadata.getBaseBuffer();
@@ -1082,9 +1081,8 @@ class ExtractStridedMetadataOpCastFolder
     SmallVector<OpFoldResult> results;
     results.resize_for_overwrite(rank * 2 + 2);
 
-    auto newExtractStridedMetadata =
-        memref::ExtractStridedMetadataOp::create(rewriter, loc,
-                                                          castOp.getSource());
+    auto newExtractStridedMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, castOp.getSource());
 
     // Register the base_buffer.
     results[0] = newExtractStridedMetadata.getBaseBuffer();
@@ -1142,9 +1140,8 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
     auto memSpaceCastOp = source.getDefiningOp<memref::MemorySpaceCastOp>();
     if (!memSpaceCastOp)
       return failure();
-    auto newExtractStridedMetadata =
-        memref::ExtractStridedMetadataOp::create(rewriter,
-            loc, memSpaceCastOp.getSource());
+    auto newExtractStridedMetadata = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, memSpaceCastOp.getSource());
     SmallVector<Value> results(newExtractStridedMetadata.getResults());
     // As with most other strided metadata rewrite patterns, don't introduce
     // a use of the base pointer where non existed. This needs to happen here,
@@ -1158,8 +1155,8 @@ class ExtractStridedMetadataOpMemorySpaceCastFolder
       MemRefType::Builder newTypeBuilder(baseBufferType);
       newTypeBuilder.setMemorySpace(
           memSpaceCastOp.getResult().getType().getMemorySpace());
-      results[0] = memref::MemorySpaceCastOp::create(rewriter,
-          loc, Type{newTypeBuilder}, baseBuffer);
+      results[0] = memref::MemorySpaceCastOp::create(
+          rewriter, loc, Type{newTypeBuilder}, baseBuffer);
     } else {
       results[0] = nullptr;
     }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
index 65f398f6dc5a8..0946da8e4e919 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExtractAddressComputations.cpp
@@ -43,7 +43,7 @@ static memref::LoadOp rebuildLoadOp(RewriterBase &rewriter,
                                     ArrayRef<Value> indices) {
   Location loc = loadOp.getLoc();
   return memref::LoadOp::create(rewriter, loc, srcMemRef, indices,
-                                         loadOp.getNontemporal());
+                                loadOp.getNontemporal());
 }
 
 // Matches getViewSizeForEachDim specs for LoadOp.
@@ -73,8 +73,7 @@ static memref::StoreOp rebuildStoreOp(RewriterBase &rewriter,
                                       ArrayRef<Value> indices) {
   Location loc = storeOp.getLoc();
   return memref::StoreOp::create(rewriter, loc, storeOp.getValueToStore(),
-                                          srcMemRef, indices,
-                                          storeOp.getNontemporal());
+                                 srcMemRef, indices, storeOp.getNontemporal());
 }
 
 // Matches getViewSizeForEachDim specs for StoreOp.
@@ -104,8 +103,8 @@ static nvgpu::LdMatrixOp rebuildLdMatrixOp(RewriterBase &rewriter,
                                            Value srcMemRef,
                                            ArrayRef<Value> indices) {
   Location loc = ldMatrixOp.getLoc();
-  return nvgpu::LdMatrixOp::create(rewriter,
-      loc, ldMatrixOp.getResult().getType(), srcMemRef, indices,
+  return nvgpu::LdMatrixOp::create(
+      rewriter, loc, ldMatrixOp.getResult().getType(), srcMemRef, indices,
       ldMatrixOp.getTranspose(), ldMatrixOp.getNumTiles());
 }
 
@@ -132,8 +131,8 @@ rebuildTransferReadOp(RewriterBase &rewriter,
                       vector::TransferReadOp transferReadOp, Value srcMemRef,
                       ArrayRef<Value> indices) {
   Location loc = transferReadOp.getLoc();
-  return vector::TransferReadOp::create(rewriter,
-      loc, transferReadOp.getResult().getType(), srcMemRef, indices,
+  return vector::TransferReadOp::create(
+      rewriter, loc, transferReadOp.getResult().getType(), srcMemRef, indices,
       transferReadOp.getPermutationMap(), transferReadOp.getPadding(),
       transferReadOp.getMask(), transferReadOp.getInBoundsAttr());
 }
@@ -150,8 +149,8 @@ rebuildTransferWriteOp(RewriterBase &rewriter,
                        vector::TransferWriteOp transferWriteOp, Value srcMemRef,
                        ArrayRef<Value> indices) {
   Location loc = transferWriteOp.getLoc();
-  return vector::TransferWriteOp::create(rewriter,
-      loc, transferWriteOp.getValue(), srcMemRef, indices,
+  return vector::TransferWriteOp::create(
+      rewriter, loc, transferWriteOp.getValue(), srcMemRef, indices,
       transferWriteOp.getPermutationMapAttr(), transferWriteOp.getMask(),
       transferWriteOp.getInBoundsAttr());
 }
@@ -182,9 +181,8 @@ static SmallVector<OpFoldResult>
 getGenericOpViewSizeForEachDim(RewriterBase &rewriter,
                                LoadStoreLikeOp loadStoreLikeOp) {
   Location loc = loadStoreLikeOp.getLoc();
-  auto extractStridedMetadataOp =
-      memref::ExtractStridedMetadataOp::create(rewriter,
-          loc, getSrcMemRef(loadStoreLikeOp));
+  auto extractStridedMetadataOp = memref::ExtractStridedMetadataOp::create(
+      rewriter, loc, getSrcMemRef(loadStoreLikeOp));
   SmallVector<OpFoldResult> srcSizes =
       extractStridedMetadataOp.getConstifiedMixedSizes();
   SmallVector<OpFoldResult> indices =
@@ -268,8 +266,8 @@ struct LoadStoreLikeOpRewriter : public OpRewritePattern<LoadStoreLikeOp> {
     // Therefore the strides multipliers are simply ones.
     auto subview =
         memref::SubViewOp::create(rewriter, loc, /*source=*/srcMemRef,
-                                           /*offsets=*/indices,
-                                           /*sizes=*/sizes, /*strides=*/ones);
+                                  /*offsets=*/indices,
+                                  /*sizes=*/sizes, /*strides=*/ones);
     // Rewrite the load/store with the subview as the base pointer.
     SmallVector<Value> zeros(loadStoreRank,
                              arith::ConstantIndexOp::create(rewriter, loc, 0));
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
index 48dbabfdf890a..42be847811d52 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FlattenMemRefs.cpp
@@ -40,8 +40,8 @@ using namespace mlir;
 static Value getValueFromOpFoldResult(OpBuilder &rewriter, Location loc,
                                       OpFoldResult in) {
   if (Attribute offsetAttr = dyn_cast<Attribute>(in)) {
-    return arith::ConstantIndexOp::create(rewriter,
-        loc, cast<IntegerAttr>(offsetAttr).getInt());
+    return arith::ConstantIndexOp::create(
+        rewriter, loc, cast<IntegerAttr>(offsetAttr).getInt());
   }
   return cast<Value>(in);
 }
@@ -74,8 +74,8 @@ static std::pair<Value, Value> getFlattenMemrefAndOffset(OpBuilder &rewriter,
           getAsOpFoldResult(indices));
 
   return std::make_pair(
-      memref::ReinterpretCastOp::create(rewriter,
-          loc, source,
+      memref::ReinterpretCastOp::create(
+          rewriter, loc, source,
           /* offset = */ linearizedInfo.linearizedOffset,
           /* shapes = */
           ArrayRef<OpFoldResult>{linearizedInfo.linearizedSize},
@@ -125,63 +125,68 @@ static void replaceOp(T op, PatternRewriter &rewriter, Value flatMemref,
   Location loc = op->getLoc();
   llvm::TypeSwitch<Operation *>(op.getOperation())
       .template Case<memref::AllocOp>([&](auto oper) {
-        auto newAlloc = memref::AllocOp::create(rewriter,
-            loc, cast<MemRefType>(flatMemref.getType()),
+        auto newAlloc = memref::AllocOp::create(
+            rewriter, loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloc, loc, rewriter);
       })
       .template Case<memref::AllocaOp>([&](auto oper) {
-        auto newAlloca = memref::AllocaOp::create(rewriter,
-            loc, cast<MemRefType>(flatMemref.getType()),
+        auto newAlloca = memref::AllocaOp::create(
+            rewriter, loc, cast<MemRefType>(flatMemref.getType()),
             oper.getAlignmentAttr());
         castAllocResult(oper, newAlloca, loc, rewriter);
       })
       .template Case<memref::LoadOp>([&](auto op) {
-        auto newLoad = memref::LoadOp::create(rewriter,
-            loc, op->getResultTypes(), flatMemref, ValueRange{offset});
+        auto newLoad =
+            memref::LoadOp::create(rewriter, loc, op->getResultTypes(),
+                                   flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<memref::StoreOp>([&](auto op) {
-        auto newStore = memref::StoreOp::create(rewriter,
-            loc, op->getOperands().front(), flatMemref, ValueRange{offset});
+        auto newStore =
+            memref::StoreOp::create(rewriter, loc, op->getOperands().front(),
+                                    flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::LoadOp>([&](auto op) {
-        auto newLoad = vector::LoadOp::create(rewriter,
-            loc, op->getResultTypes(), flatMemref, ValueRange{offset});
+        auto newLoad =
+            vector::LoadOp::create(rewriter, loc, op->getResultTypes(),
+                                   flatMemref, ValueRange{offset});
         newLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newLoad.getResult());
       })
       .template Case<vector::StoreOp>([&](auto op) {
-        auto newStore = vector::StoreOp::create(rewriter,
-            loc, op->getOperands().front(), flatMemref, ValueRange{offset});
+        auto newStore =
+            vector::StoreOp::create(rewriter, loc, op->getOperands().front(),
+                                    flatMemref, ValueRange{offset});
         newStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newStore);
       })
       .template Case<vector::MaskedLoadOp>([&](auto op) {
-        auto newMaskedLoad = vector::MaskedLoadOp::create(rewriter,
-            loc, op.getType(), flatMemref, ValueRange{offset}, op.getMask(),
-            op.getPassThru());
+        auto newMaskedLoad = vector::MaskedLoadOp::create(
+            rewriter, loc, op.getType(), flatMemref, ValueRange{offset},
+            op.getMask(), op.getPassThru());
         newMaskedLoad->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedLoad.getResult());
       })
       .template Case<vector::MaskedStoreOp>([&](auto op) {
-        auto newMaskedStore = vector::MaskedStoreOp::create(rewriter,
-            loc, flatMemref, ValueRange{offset}, op.getMask(),
+        auto newMaskedStore = vector::MaskedStoreOp::create(
+            rewriter, loc, flatMemref, ValueRange{offset}, op.getMask(),
             op.getValueToStore());
         newMaskedStore->setAttrs(op->getAttrs());
         rewriter.replaceOp(op, newMaskedStore);
       })
       .template Case<vector::TransferReadOp>([&](auto op) {
-        auto newTransferRead = vector::TransferReadOp::create(rewriter,
-            loc, op.getType(), flatMemref, ValueRange{offset}, op.getPadding());
+        auto newTransferRead = vector::TransferReadOp::create(
+            rewriter, loc, op.getType(), flatMemref, ValueRange{offset},
+            op.getPadding());
         rewriter.replaceOp(op, newTransferRead.getResult());
       })
       .template Case<vector::TransferWriteOp>([&](auto op) {
-        auto newTransferWrite = vector::TransferWriteOp::create(rewriter,
-            loc, op.getVector(), flatMemref, ValueRange{offset});
+        auto newTransferWrite = vector::TransferWriteOp::create(
+            rewriter, loc, op.getVector(), flatMemref, ValueRange{offset});
         rewriter.replaceOp(op, newTransferWrite);
       })
       .Default([&](auto op) {
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index 8e99a101f28a6..991065c9506a1 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -76,8 +76,8 @@ static LogicalResult resolveSourceIndicesExpandShape(
         llvm::map_to_vector(group, [&](int64_t d) { return destShape[d]; });
     SmallVector<Value> groupIndices =
         llvm::map_to_vector(group, [&](int64_t d) { return indices[d]; });
-    Value collapsedIndex = affine::AffineLinearizeIndexOp::create(rewriter,
-        loc, groupIndices, groupBasis, /*disjoint=*/startsInbounds);
+    Value collapsedIndex = affine::AffineLinearizeIndexOp::create(
+        rewriter, loc, groupIndices, groupBasis, /*disjoint=*/startsInbounds);
     sourceIndices.push_back(collapsedIndex);
   }
   return success();
@@ -102,8 +102,8 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
                                   ValueRange indices,
                                   SmallVectorImpl<Value> &sourceIndices) {
   // Note: collapse_shape requires a strided memref, we can do this.
-  auto metadata = memref::ExtractStridedMetadataOp::create(rewriter,
-      loc, collapseShapeOp.getSrc());
+  auto metadata = memref::ExtractStridedMetadataOp::create(
+      rewriter, loc, collapseShapeOp.getSrc());
   SmallVector<OpFoldResult> sourceSizes = metadata.getConstifiedMixedSizes();
   for (auto [index, group] :
        llvm::zip(indices, collapseShapeOp.getReassociationIndices())) {
@@ -117,8 +117,8 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
 
     SmallVector<OpFoldResult> basis =
         llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
-    auto delinearize = affine::AffineDelinearizeIndexOp::create(rewriter,
-        loc, index, basis, /*hasOuterBound=*/true);
+    auto delinearize = affine::AffineDelinearizeIndexOp::create(
+        rewriter, loc, index, basis, /*hasOuterBound=*/true);
     llvm::append_range(sourceIndices, delinearize.getResults());
   }
   if (collapseShapeOp.getReassociationIndices().empty()) {
diff --git a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
index 953024d70d0aa..66c1aa6bf3fe1 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
@@ -71,11 +71,11 @@ propagateSubViewOp(RewriterBase &rewriter,
   MemRefType newResultType = SubViewOp::inferRankReducedResultType(
       op.getType().getShape(), op.getSourceType(), op.getMixedOffsets(),
       op.getMixedSizes(), op.getMixedStrides());
-  Value newSubview = SubViewOp::create(rewriter,
-      op.getLoc(), newResultType, conversionOp.getOperand(0),
+  Value newSubview = SubViewOp::create(
+      rewriter, op.getLoc(), newResultType, conversionOp.getOperand(0),
       op.getMixedOffsets(), op.getMixedSizes(), op.getMixedStrides());
-  auto newConversionOp = UnrealizedConversionCastOp::create(rewriter,
-      op.getLoc(), op.getType(), newSubview);
+  auto newConversionOp = UnrealizedConversionCastOp::create(
+      rewriter, op.getLoc(), op.getType(), newSubview);
   rewriter.replaceAllUsesWith(op.getResult(), newConversionOp->getResult(0));
   return newConversionOp;
 }
@@ -106,8 +106,8 @@ static void replaceAndPropagateMemRefType(RewriterBase &rewriter,
   SmallVector<UnrealizedConversionCastOp> unrealizedConversions;
   for (const auto &it :
        llvm::enumerate(llvm::zip(from->getResults(), to->getResults()))) {
-    unrealizedConversions.push_back(UnrealizedConversionCastOp::create(rewriter,
-        to->getLoc(), std::get<0>(it.value()).getType(),
+    unrealizedConversions.push_back(UnrealizedConversionCastOp::create(
+        rewriter, to->getLoc(), std::get<0>(it.value()).getType(),
         std::get<1>(it.value())));
     rewriter.replaceAllUsesWith(from->getResult(it.index()),
                                 unrealizedConversions.back()->getResult(0));
diff --git a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
index b7e4388193a06..5d3cec402cab1 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/MultiBuffer.cpp
@@ -63,9 +63,10 @@ static void replaceUsesAndPropagateType(RewriterBase &rewriter,
         subviewUse.getType().getShape(), cast<MemRefType>(val.getType()),
         subviewUse.getStaticOffsets(), subviewUse.getStaticSizes(),
         subviewUse.getStaticStrides());
-    Value newSubview = memref::SubViewOp::create(rewriter,
-        subviewUse->getLoc(), newType, val, subviewUse.getMixedOffsets(),
-        subviewUse.getMixedSizes(), subviewUse.getMixedStrides());
+    Value newSubview = memref::SubViewOp::create(
+        rewriter, subviewUse->getLoc(), newType, val,
+        subviewUse.getMixedOffsets(), subviewUse.getMixedSizes(),
+        subviewUse.getMixedStrides());
 
     // Ouch recursion ... is this really necessary?
     replaceUsesAndPropagateType(rewriter, subviewUse, newSubview);
@@ -177,8 +178,8 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   Location loc = allocOp->getLoc();
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(allocOp);
-  auto mbAlloc = memref::AllocOp::create(rewriter,
-      loc, mbMemRefType, ValueRange{}, allocOp->getAttrs());
+  auto mbAlloc = memref::AllocOp::create(rewriter, loc, mbMemRefType,
+                                         ValueRange{}, allocOp->getAttrs());
   LLVM_DEBUG(DBGS() << "--multi-buffered alloc: " << mbAlloc << "\n");
 
   // 3. Within the loop, build the modular leading index (i.e. each loop
@@ -212,7 +213,7 @@ mlir::memref::multiBuffer(RewriterBase &rewriter, memref::AllocOp allocOp,
   MemRefType dstMemref = memref::SubViewOp::inferRankReducedResultType(
       originalShape, mbMemRefType, offsets, sizes, strides);
   Value subview = memref::SubViewOp::create(rewriter, loc, dstMemref, mbAlloc,
-                                                     offsets, sizes, strides);
+                                            offsets, sizes, strides);
   LLVM_DEBUG(DBGS() << "--multi-buffered slice: " << subview << "\n");
 
   // 5. Due to the recursive nature of replaceUsesAndPropagateType , we need to
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index c522b2bd5ddec..fa7991e6c6a80 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -277,7 +277,7 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
       continue;
     Operation *newCallOp =
         func::CallOp::create(builder, userOp->getLoc(), callOp.getCalleeAttr(),
-                                     resultTypes, userOp->getOperands());
+                             resultTypes, userOp->getOperands());
     bool replacingMemRefUsesFailed = false;
     bool returnTypeChanged = false;
     for (unsigned resIndex : llvm::seq<unsigned>(0, userOp->getNumResults())) {
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
index ea7f394f48ba4..40d45b57257c6 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
@@ -116,10 +116,12 @@ static LogicalResult reifyOpResultShapes(RewriterBase &rewriter,
     // Update the type.
     newRes.setType(reifiedTy);
     if (isa<RankedTensorType>(reifiedTy)) {
-      newResults.push_back(tensor::CastOp::create(rewriter, loc, oldTy, newRes));
+      newResults.push_back(
+          tensor::CastOp::create(rewriter, loc, oldTy, newRes));
     } else {
       assert(isa<MemRefType>(reifiedTy) && "expected a memref type");
-      newResults.push_back(memref::CastOp::create(rewriter, loc, oldTy, newRes));
+      newResults.push_back(
+          memref::CastOp::create(rewriter, loc, oldTy, newRes));
     }
   }
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
index fe9555a7a16fd..1f03e9ae8d6a1 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
@@ -40,19 +40,18 @@ struct AssumeAlignmentOpInterface
   void generateRuntimeVerification(Operation *op, OpBuilder &builder,
                                    Location loc) const {
     auto assumeOp = cast<AssumeAlignmentOp>(op);
-    Value ptr = ExtractAlignedPointerAsIndexOp::create(builder,
-        loc, assumeOp.getMemref());
-    Value rest = arith::RemUIOp::create(builder,
-        loc, ptr,
+    Value ptr = ExtractAlignedPointerAsIndexOp::create(builder, loc,
+                                                       assumeOp.getMemref());
+    Value rest = arith::RemUIOp::create(
+        builder, loc, ptr,
         arith::ConstantIndexOp::create(builder, loc, assumeOp.getAlignment()));
-    Value isAligned = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::eq, rest,
-        arith::ConstantIndexOp::create(builder, loc, 0));
-    cf::AssertOp::create(builder,
-        loc, isAligned,
-        RuntimeVerifiableOpInterface::generateErrorMessage(
-            op, "memref is not aligned to " +
-                    std::to_string(assumeOp.getAlignment())));
+    Value isAligned =
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, rest,
+                              arith::ConstantIndexOp::create(builder, loc, 0));
+    cf::AssertOp::create(builder, loc, isAligned,
+                         RuntimeVerifiableOpInterface::generateErrorMessage(
+                             op, "memref is not aligned to " +
+                                     std::to_string(assumeOp.getAlignment())));
   }
 };
 
@@ -74,12 +73,11 @@ struct CastOpInterface
       Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
           arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
-      Value isSameRank = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      cf::AssertOp::create(builder,
-          loc, isSameRank,
-          RuntimeVerifiableOpInterface::generateErrorMessage(op,
-                                                             "rank mismatch"));
+      Value isSameRank = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcRank, resultRank);
+      cf::AssertOp::create(builder, loc, isSameRank,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "rank mismatch"));
     }
 
     // Get source offset and strides. We do not have an op to get offsets and
@@ -96,7 +94,8 @@ struct CastOpInterface
                         stridedLayout, resultType.getMemorySpace());
     Value helperCast =
         CastOp::create(builder, loc, dynStridesType, castOp.getSource());
-    auto metadataOp = ExtractStridedMetadataOp::create(builder, loc, helperCast);
+    auto metadataOp =
+        ExtractStridedMetadataOp::create(builder, loc, helperCast);
 
     // Check dimension sizes.
     for (const auto &it : llvm::enumerate(resultType.getShape())) {
@@ -113,10 +112,10 @@ struct CastOpInterface
           DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
           arith::ConstantIndexOp::create(builder, loc, it.value());
-      Value isSameSz = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      cf::AssertOp::create(builder,
-          loc, isSameSz,
+      Value isSameSz = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
+      cf::AssertOp::create(
+          builder, loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
     }
@@ -133,12 +132,11 @@ struct CastOpInterface
       Value srcOffset = metadataOp.getResult(1);
       Value resultOffsetVal =
           arith::ConstantIndexOp::create(builder, loc, resultOffset);
-      Value isSameOffset = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcOffset, resultOffsetVal);
-      cf::AssertOp::create(builder,
-          loc, isSameOffset,
-          RuntimeVerifiableOpInterface::generateErrorMessage(
-              op, "offset mismatch"));
+      Value isSameOffset = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcOffset, resultOffsetVal);
+      cf::AssertOp::create(builder, loc, isSameOffset,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "offset mismatch"));
     }
 
     // Check strides.
@@ -151,10 +149,10 @@ struct CastOpInterface
           metadataOp.getResult(2 + resultType.getRank() + it.index());
       Value resultStrideVal =
           arith::ConstantIndexOp::create(builder, loc, it.value());
-      Value isSameStride = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcStride, resultStrideVal);
-      cf::AssertOp::create(builder,
-          loc, isSameStride,
+      Value isSameStride = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcStride, resultStrideVal);
+      cf::AssertOp::create(
+          builder, loc, isSameStride,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "stride mismatch of dim " + std::to_string(it.index())));
     }
@@ -194,13 +192,12 @@ struct CopyOpInterface
       };
       Value sourceDim = getDimSize(copyOp.getSource(), rankedSourceType, i);
       Value targetDim = getDimSize(copyOp.getTarget(), rankedTargetType, i);
-      Value sameDimSize = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
-      cf::AssertOp::create(builder,
-          loc, sameDimSize,
-          RuntimeVerifiableOpInterface::generateErrorMessage(
-              op, "size of " + std::to_string(i) +
-                      "-th source/target dim does not match"));
+      Value sameDimSize = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
+      cf::AssertOp::create(builder, loc, sameDimSize,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "size of " + std::to_string(i) +
+                                       "-th source/target dim does not match"));
     }
   }
 };
@@ -213,8 +210,9 @@ struct DimOpInterface
     auto dimOp = cast<DimOp>(op);
     Value rank = RankOp::create(builder, loc, dimOp.getSource());
     Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
-    cf::AssertOp::create(builder,
-        loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
+    cf::AssertOp::create(
+        builder, loc,
+        generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
   }
@@ -247,10 +245,9 @@ struct LoadStoreOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    cf::AssertOp::create(builder,
-        loc, assertCond,
-        RuntimeVerifiableOpInterface::generateErrorMessage(
-            op, "out-of-bounds access"));
+    cf::AssertOp::create(builder, loc, assertCond,
+                         RuntimeVerifiableOpInterface::generateErrorMessage(
+                             op, "out-of-bounds access"));
   }
 };
 
@@ -281,8 +278,8 @@ struct SubViewOpInterface
       Value dimSize = metadataOp.getSizes()[i];
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      cf::AssertOp::create(builder,
-          loc, offsetInBounds,
+      cf::AssertOp::create(
+          builder, loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
@@ -294,8 +291,8 @@ struct SubViewOpInterface
           arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      cf::AssertOp::create(builder,
-          loc, lastPosInBounds,
+      cf::AssertOp::create(
+          builder, loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "subview runs out-of-bounds along dimension " +
                       std::to_string(i)));
@@ -334,14 +331,13 @@ struct ExpandShapeOpInterface
       // staticResultDimSz must divide srcDimSz evenly.
       Value mod =
           arith::RemSIOp::create(builder, loc, srcDimSz, staticResultDimSz);
-      Value isModZero = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, mod,
+      Value isModZero = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, mod,
           arith::ConstantIndexOp::create(builder, loc, 0));
-      cf::AssertOp::create(builder,
-          loc, isModZero,
-          RuntimeVerifiableOpInterface::generateErrorMessage(
-              op, "static result dims in reassoc group do not "
-                  "divide src dim evenly"));
+      cf::AssertOp::create(builder, loc, isModZero,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "static result dims in reassoc group do not "
+                                   "divide src dim evenly"));
     }
   }
 };
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
index 65446142aabcc..db5fd6e494da1 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Simplifications.cpp
@@ -91,8 +91,8 @@ struct MeshShapeFolder
         newShapeOpMeshAxes.push_back(opMeshAxes[i]);
       } else {
         // Fold static mesh axes.
-        newResults[i] = arith::ConstantOp::create(builder,
-            builder.getIndexAttr(meshAxisSize));
+        newResults[i] = arith::ConstantOp::create(
+            builder, builder.getIndexAttr(meshAxisSize));
       }
     }
 
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index 9229bfaa1dffb..1e54affa8198f 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -265,7 +265,8 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
       targetShardingInUnsplitLastAxis(ctx, sourceSharding, splitTensorAxis);
   ShapedType allGatherResultShape = allGatherResultShapeInUnsplitLastAxis(
       sourceShard.getType(), mesh.getShape()[splitMeshAxis], splitTensorAxis);
-  Value allGatherResult = AllGatherOp::create(builder,
+  Value allGatherResult = AllGatherOp::create(
+      builder,
       RankedTensorType::get(allGatherResultShape.getShape(),
                             allGatherResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({splitMeshAxis}), sourceShard,
@@ -273,7 +274,8 @@ unsplitLastAxisInResharding(ImplicitLocOpBuilder &builder,
   ShapedType targetShape =
       shardShapedType(sourceUnshardedShape, mesh, targetSharding);
   TypedValue<ShapedType> targetShard = cast<TypedValue<ShapedType>>(
-      tensor::CastOp::create(builder, targetShape, allGatherResult).getResult());
+      tensor::CastOp::create(builder, targetShape, allGatherResult)
+          .getResult());
   return {targetShard, targetSharding};
 }
 
@@ -398,7 +400,8 @@ moveLastSplitAxisInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   ShapedType allToAllResultShape = allToAllResultShapeInMoveLastAxis(
       sourceShard.getType(), mesh.getShape()[meshAxis], sourceTensorAxis,
       targetTensorAxis);
-  Value allToAllResult = AllToAllOp::create(builder,
+  Value allToAllResult = AllToAllOp::create(
+      builder,
       RankedTensorType::get(allToAllResultShape.getShape(),
                             allToAllResultShape.getElementType()),
       mesh.getSymName(), SmallVector<MeshAxis>({meshAxis}), sourceShard,
@@ -477,15 +480,16 @@ tryUpdateHaloInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
 
   // Extract core from source and copy into destination core.
   auto noVals = ValueRange{};
-  auto initVal = tensor::EmptyOp::create(builder,
-      sourceShard.getLoc(), outShape, sourceShard.getType().getElementType());
-  auto core = tensor::ExtractSliceOp::create(builder,
-      sourceShard.getLoc(),
+  auto initVal =
+      tensor::EmptyOp::create(builder, sourceShard.getLoc(), outShape,
+                              sourceShard.getType().getElementType());
+  auto core = tensor::ExtractSliceOp::create(
+      builder, sourceShard.getLoc(),
       RankedTensorType::get(coreShape, sourceShard.getType().getElementType()),
       sourceShard, noVals, noVals, noVals, srcCoreOffs, coreShape, strides);
-  auto initOprnd = tensor::InsertSliceOp::create(builder,
-      sourceShard.getLoc(), core, initVal, noVals, noVals, noVals, tgtCoreOffs,
-      coreShape, strides);
+  auto initOprnd = tensor::InsertSliceOp::create(
+      builder, sourceShard.getLoc(), core, initVal, noVals, noVals, noVals,
+      tgtCoreOffs, coreShape, strides);
 
   // Finally update the halo.
   auto updateHaloResult =
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
index c85394924608d..6ae95ae1f8a49 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp
@@ -52,7 +52,8 @@ struct ProcessMultiIndexOpLowering
     Value linearIndex = ProcessLinearIndexOp::create(builder, mesh);
     ValueRange meshShape = MeshShapeOp::create(builder, mesh).getResults();
     SmallVector<Value> completeMultiIndex =
-        affine::AffineDelinearizeIndexOp::create(builder, linearIndex, meshShape)
+        affine::AffineDelinearizeIndexOp::create(builder, linearIndex,
+                                                 meshShape)
             .getMultiIndex();
     SmallVector<Value> multiIndex;
     ArrayRef<MeshAxis> opMeshAxes = op.getAxes();
@@ -104,7 +105,8 @@ struct AllSliceOpLowering
     Value zero = arith::ConstantOp::create(builder, builder.getIndexAttr(0));
 
     Operation::result_range processInGroupMultiIndex =
-        ProcessMultiIndexOp::create(builder, mesh.getSymName(), op.getMeshAxes())
+        ProcessMultiIndexOp::create(builder, mesh.getSymName(),
+                                    op.getMeshAxes())
             .getResults();
 
     Operation::result_range processGroupShape =
@@ -118,13 +120,13 @@ struct AllSliceOpLowering
         tensor::DimOp::create(builder, op.getOperand(), sliceAxis);
     Value operandSliceAxisSizeModProcessGroupSize =
         arith::RemUIOp::create(builder, operandSliceAxisSize, processGroupSize);
-    Value isTargetShapeExactlyDivisible = arith::CmpIOp::create(builder,
-        arith::CmpIPredicate::eq, operandSliceAxisSizeModProcessGroupSize,
-        zero);
+    Value isTargetShapeExactlyDivisible =
+        arith::CmpIOp::create(builder, arith::CmpIPredicate::eq,
+                              operandSliceAxisSizeModProcessGroupSize, zero);
     cf::AssertOp::create(builder, isTargetShapeExactlyDivisible,
-                                 "Slicing a tensor with axis size that is "
-                                 "not exactly divisible by the "
-                                 "mesh process group size is not supported.");
+                         "Slicing a tensor with axis size that is "
+                         "not exactly divisible by the "
+                         "mesh process group size is not supported.");
     Value resultSliceAxisSize =
         arith::DivUIOp::create(builder, operandSliceAxisSize, processGroupSize);
     OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
@@ -152,8 +154,8 @@ struct AllSliceOpLowering
                  resultSliceAxisSize);
     SmallVector<OpFoldResult> strides(
         operandType.getRank(), getAsIndexOpFoldResult(builder.getContext(), 1));
-    Value slice = tensor::ExtractSliceOp::create(builder,
-        op.getOperand(), offsets, sizes, strides);
+    Value slice = tensor::ExtractSliceOp::create(builder, op.getOperand(),
+                                                 offsets, sizes, strides);
     Value newResult =
         tensor::CastOp::create(builder, op.getResult().getType(), slice);
     rewriter.replaceAllUsesWith(op.getResult(), newResult);
@@ -218,7 +220,8 @@ createProcessLinearIndex(StringRef mesh, ValueRange processInGroupMultiIndex,
       llvm::to_vector_of<OpFoldResult>(processGroupShape), builder);
   auto res = dyn_cast<Value>(processInGroupLinearIndex);
   if (!res)
-    res = arith::ConstantIndexOp::create(builder,
+    res = arith::ConstantIndexOp::create(
+        builder,
         cast<IntegerAttr>(cast<Attribute>(processInGroupLinearIndex)).getInt());
   return cast<TypedValue<IndexType>>(res);
 }
diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
index 85317a68c9f00..5d253c1199dc0 100644
--- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
+++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
@@ -333,15 +333,15 @@ static Operation *replaceOpWithPredicatedOp(RewriterBase &rewriter,
   //   srcElement = (pred) ?  prevSrcElements : 0;
   //
   Location loc = asyncCopyOp->getLoc();
-  Value dstElements =
-      arith::ConstantOp::create(rewriter, loc, asyncCopyOp.getDstElementsAttr());
+  Value dstElements = arith::ConstantOp::create(
+      rewriter, loc, asyncCopyOp.getDstElementsAttr());
   Value originalSrcElement =
       asyncCopyOp.getSrcElements() ? asyncCopyOp.getSrcElements() : dstElements;
   Value c0Index = arith::ConstantIndexOp::create(rewriter, loc, 0);
-  auto srcElements = arith::SelectOp::create(rewriter,
-      loc, predicate, originalSrcElement, c0Index);
-  auto asyncCopyZeroFillOp = nvgpu::DeviceAsyncCopyOp::create(rewriter,
-      loc, nvgpu::DeviceAsyncTokenType::get(asyncCopyOp.getContext()),
+  auto srcElements = arith::SelectOp::create(rewriter, loc, predicate,
+                                             originalSrcElement, c0Index);
+  auto asyncCopyZeroFillOp = nvgpu::DeviceAsyncCopyOp::create(
+      rewriter, loc, nvgpu::DeviceAsyncTokenType::get(asyncCopyOp.getContext()),
       asyncCopyOp.getDst(), asyncCopyOp.getDstIndices(), asyncCopyOp.getSrc(),
       asyncCopyOp.getSrcIndices(), asyncCopyOp.getDstElements(), srcElements,
       UnitAttr());
@@ -811,7 +811,7 @@ FailureOr<Operation *> MmaSyncBuilder::buildMmaSync(LinalgOp linalgOp) {
   Value res = buildMmaSyncMemRefLoadOperand(b, loc, laneId, resMemRef,
                                             resIndexFn, resShape);
   res = nvgpu::MmaSyncOp::create(b, loc, lhs, rhs, res, info.mmaShape,
-                                   info.tf32Enabled);
+                                 info.tf32Enabled);
   buildMmaSyncMemRefStoreOperand(b, loc, res, laneId, resMemRef, resIndexFn,
                                  resShape);
   return res.getDefiningOp();
@@ -832,8 +832,8 @@ DiagnosedSilenceableFailure transform::RewriteMatmulAsMmaSyncOp::applyToOne(
     }
     Location loc = linalgOp.getLoc();
     // TODO: more robust computation of laneId, for now assume a single warp.
-    Value laneId = gpu::ThreadIdOp::create(rewriter,
-        loc, rewriter.getIndexType(), gpu::Dimension::x);
+    Value laneId = gpu::ThreadIdOp::create(
+        rewriter, loc, rewriter.getIndexType(), gpu::Dimension::x);
     if (succeeded(MmaSyncBuilder(rewriter, loc, laneId).buildMmaSync(linalgOp)))
       fail = false;
   }
@@ -899,8 +899,8 @@ SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
   SmallVector<Operation *> loadOps;
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   Value tidx = gpu::ThreadIdOp::create(rewriter, loc, gpu::Dimension::x);
-  Value cond =
-      arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq, tidx, zero);
+  Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
+                                     tidx, zero);
   // clang-format off
   scf::IfOp::create(rewriter,
     /*location=*/loc,
@@ -939,13 +939,14 @@ static Attribute getSharedAddressSpaceAttribute(OpBuilder &b) {
 TypedValue<nvgpu::MBarrierGroupType>
 HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) {
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value barrier = nvgpu::MBarrierCreateOp::create(rewriter,
-      loc,
+  Value barrier = nvgpu::MBarrierCreateOp::create(
+      rewriter, loc,
       nvgpu::MBarrierGroupType::get(rewriter.getContext(), sharedMemorySpace));
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
-  nvgpu::MBarrierInitOp::create(rewriter,
-      loc, barrier, getValueOrCreateConstantIndexOp(rewriter, loc, numThreads),
-      zero, Value());
+  nvgpu::MBarrierInitOp::create(
+      rewriter, loc, barrier,
+      getValueOrCreateConstantIndexOp(rewriter, loc, numThreads), zero,
+      Value());
   gpu::BarrierOp::create(rewriter, loc);
   return cast<TypedValue<nvgpu::MBarrierGroupType>>(barrier);
 }
@@ -955,8 +956,8 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
                                            gpu::LaunchOp launchOp) {
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPoint(launchOp);
-  Value unrankedMemRef = memref::CastOp::create(rewriter,
-      loc,
+  Value unrankedMemRef = memref::CastOp::create(
+      rewriter, loc,
       UnrankedMemRefType::get(memref.getType().getElementType(),
                               memref.getType().getMemorySpace()),
       memref);
@@ -966,8 +967,8 @@ HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
       getValueOrCreateConstantIndexOp(rewriter, loc, mixedSizes);
 
   auto sharedMemorySpace = getSharedAddressSpaceAttribute(rewriter);
-  Value desc = nvgpu::TmaCreateDescriptorOp::create(rewriter,
-      loc,
+  Value desc = nvgpu::TmaCreateDescriptorOp::create(
+      rewriter, loc,
       nvgpu::TensorMapDescriptorType::get(
           rewriter.getContext(),
           MemRefType::Builder(memref.getType())
@@ -986,9 +987,9 @@ OpFoldResult HopperBuilder::buildTmaAsyncLoad(
     SmallVectorImpl<Operation *> &loadOps) {
   MLIRContext *ctx = rewriter.getContext();
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
-  Operation *loadOp = nvgpu::TmaAsyncLoadOp::create(rewriter,
-      loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero,
-      Value(), Value());
+  Operation *loadOp = nvgpu::TmaAsyncLoadOp::create(
+      rewriter, loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero},
+      zero, Value(), Value());
   loadOps.push_back(loadOp);
   auto mixedSizes = memref::getMixedSizes(rewriter, loc, sharedMemref);
   SmallVector<AffineExpr> symbols(mixedSizes.size());
@@ -1014,7 +1015,7 @@ void HopperBuilder::buildBarrierArriveTx(
   Value sizeVal = getValueOrCreateConstantIndexOp(rewriter, loc, size);
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   nvgpu::MBarrierArriveExpectTxOp::create(rewriter, loc, barrier, sizeVal, zero,
-                                                   Value());
+                                          Value());
 }
 
 void HopperBuilder::buildTryWaitParity(
@@ -1028,7 +1029,7 @@ void HopperBuilder::buildTryWaitParity(
       arith::ConstantIndexOp::create(rewriter, loc, 10000000);
   Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
   nvgpu::MBarrierTryWaitParityOp::create(rewriter, loc, barrier, parity,
-                                                  ticksBeforeRetry, zero);
+                                         ticksBeforeRetry, zero);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
index b8baadb4ef241..b392ffeb13de6 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/CreateAsyncGroups.cpp
@@ -110,15 +110,15 @@ static Value buildNumReadElements(OpBuilder &b, Location loc,
                                   transferMask->createMaskOp->getOperands())) {
     Value cmp =
         arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt,
-                                arith::ConstantIndexOp::create(b, loc, pos), sz);
+                              arith::ConstantIndexOp::create(b, loc, pos), sz);
     if (!cond) {
       cond = cmp;
       continue;
     }
     cond = arith::AndIOp::create(b, loc, cmp, cond);
   }
-  return arith::SelectOp::create(b,
-      loc, cond, transferMask->createMaskOp->getOperands().back(),
+  return arith::SelectOp::create(
+      b, loc, cond, transferMask->createMaskOp->getOperands().back(),
       arith::ConstantIndexOp::create(b, loc, 0));
 }
 
@@ -251,8 +251,9 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
       int64_t sizeInBytes =
           (dstMemref.getElementTypeBitWidth() * numElements) / 8;
       // bypass_l1 only possible with 16 byte transfer.
-      Value token = nvgpu::DeviceAsyncCopyOp::create(rewriter,
-          writeOp->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
+      Value token = nvgpu::DeviceAsyncCopyOp::create(
+          rewriter, writeOp->getLoc(),
+          nvgpu::DeviceAsyncTokenType::get(op->getContext()),
           /*dst=*/storeBase, /*dstIndices=*/nvgpu::getIndices(writeOp),
           /*src=*/loadBase,
           /*srcIndices=*/nvgpu::getIndices(readOp),
@@ -264,11 +265,11 @@ void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
     }
 
     // Create the group and wait for it right after.
-    Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(rewriter,
-        op->getLoc(), nvgpu::DeviceAsyncTokenType::get(op->getContext()),
-        tokens);
+    Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(
+        rewriter, op->getLoc(),
+        nvgpu::DeviceAsyncTokenType::get(op->getContext()), tokens);
     nvgpu::DeviceAsyncWaitOp::create(rewriter, op->getLoc(), groupToken,
-                                              nullptr);
+                                     nullptr);
     // Clean up old stores.
     for (Operation *writeOp : group)
       rewriter.eraseOp(writeOp);
diff --git a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
index 2d2ff94a25f92..957b9632422a6 100644
--- a/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
+++ b/mlir/lib/Dialect/NVGPU/Transforms/OptimizeSharedMemory.cpp
@@ -85,7 +85,8 @@ static Value permuteVectorOffset(OpBuilder &b, Location loc,
       Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, shlBits);
       srcBits = b.createOrFold<arith::ShLIOp>(loc, srcBits, finalShiftVal);
     } else if (shlBits < 0) {
-      Value finalShiftVal = arith::ConstantIndexOp::create(b, loc, -1 * shlBits);
+      Value finalShiftVal =
+          arith::ConstantIndexOp::create(b, loc, -1 * shlBits);
       srcBits = b.createOrFold<arith::ShRUIOp>(loc, srcBits, finalShiftVal);
     }
   } else {
diff --git a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
index a378b09c27c41..58cd160948f7f 100644
--- a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
@@ -95,8 +95,8 @@ std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
   auto *context = builder.getContext();
   auto shapeType = shape::getExtentTensorType(context);
   auto inputShape = shape::ShapeOfOp::create(builder, loc, shapeType, input);
-  Value inputSize = shape::NumElementsOp::create(builder,
-      loc, builder.getIndexType(), inputShape);
+  Value inputSize = shape::NumElementsOp::create(
+      builder, loc, builder.getIndexType(), inputShape);
 
   // Turn input size into 1D tensor
   auto flatShapeType = shape::getExtentTensorType(context, 1);
@@ -109,7 +109,7 @@ std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
   auto flatInputType =
       RankedTensorType::get({ShapedType::kDynamic}, elementType);
   auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
-                                                     flatInputShape);
+                                             flatInputShape);
   return std::make_pair(flatInput, inputShape);
 }
 
@@ -165,8 +165,9 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   // Compute flat input shape as a 3-element 1D tensor
   auto axisSizeValue = arith::ConstantIndexOp::create(builder, loc, axisSize);
   auto flatShapeType = shape::getExtentTensorType(context, 3);
-  auto flatInputShape = tensor::FromElementsOp::create(builder,
-      loc, flatShapeType, ValueRange{sizeLeft, axisSizeValue, sizeRight});
+  auto flatInputShape = tensor::FromElementsOp::create(
+      builder, loc, flatShapeType,
+      ValueRange{sizeLeft, axisSizeValue, sizeRight});
 
   // Reshape input to 3D tensor
   auto inputType = cast<UnrankedTensorType>(input.getType());
@@ -174,7 +175,7 @@ flattenUnrankedTensorAroundAxis(OpBuilder &builder, Location loc, Value input,
   auto flatInputType = RankedTensorType::get(
       {ShapedType::kDynamic, axisSize, ShapedType::kDynamic}, elementType);
   auto flatInput = tensor::ReshapeOp::create(builder, loc, flatInputType, input,
-                                                     flatInputShape);
+                                             flatInputShape);
 
   return std::make_pair(flatInput, inputShape);
 }
@@ -193,7 +194,7 @@ Value restoreUnrankedTensorShape(OpBuilder &builder, Location loc, Value input,
   auto elementType = inputType.getElementType();
   auto unrankedType = UnrankedTensorType::get(elementType);
   return tensor::ReshapeOp::create(builder, loc, unrankedType, input,
-                                           inputShape);
+                                   inputShape);
 }
 
 // Create a tensor constant containing all scales in a per-channel quantized
@@ -314,10 +315,10 @@ Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
   // Materialize bounds
   auto inputType = input.getType();
   auto storageType = quantizedType.getStorageType();
-  auto storageMinScalar = arith::ConstantIntOp::create(builder,
-      loc, storageType, quantizedType.getStorageTypeMin());
-  auto storageMaxScalar = arith::ConstantIntOp::create(builder,
-      loc, storageType, quantizedType.getStorageTypeMax());
+  auto storageMinScalar = arith::ConstantIntOp::create(
+      builder, loc, storageType, quantizedType.getStorageTypeMin());
+  auto storageMaxScalar = arith::ConstantIntOp::create(
+      builder, loc, storageType, quantizedType.getStorageTypeMax());
   auto storageMin = getScalarOrTensorConstant(builder, loc, storageMinScalar,
                                               inputType, inputShape);
   auto storageMax = getScalarOrTensorConstant(builder, loc, storageMaxScalar,
@@ -477,7 +478,8 @@ Value convertPerLayerRanked(OpBuilder &builder, Location loc, Operation *op,
   auto storageType = quantizedType.getStorageType();
   auto scaleAttr =
       builder.getFloatAttr(expressedType, quantizedType.getScale());
-  auto scale = arith::ConstantOp::create(builder, loc, expressedType, scaleAttr);
+  auto scale =
+      arith::ConstantOp::create(builder, loc, expressedType, scaleAttr);
   auto zeroPointAttr =
       builder.getIntegerAttr(storageType, quantizedType.getZeroPoint());
   auto zeroPoint =
@@ -729,8 +731,8 @@ struct DequantizeCastOpConversion
     // Convert quantized input to storage type
     auto storageScalarOrTensorType =
         getScalarOrTensorType(quantizedType.getStorageType(), input.getType());
-    input = quant::StorageCastOp::create(rewriter,
-        loc, storageScalarOrTensorType, input);
+    input = quant::StorageCastOp::create(rewriter, loc,
+                                         storageScalarOrTensorType, input);
 
     auto result = convertQuantized(rewriter, loc, op, input, quantizedType);
 
diff --git a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
index f5e0aef365ff7..e84cca5b0c1f6 100644
--- a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
@@ -47,7 +47,7 @@ class QuantizedTypeConverter : public TypeConverter {
   static Value materializeConversion(OpBuilder &builder, Type type,
                                      ValueRange inputs, Location loc) {
     return quant::StorageCastOp::create(builder, loc, type,
-                                                llvm::getSingleElement(inputs));
+                                        llvm::getSingleElement(inputs));
   }
 
 public:
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index 0214ce3090656..0325597c39a02 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -205,8 +205,8 @@ struct ExecuteRegionOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(executeRegionOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
-            executeRegionOp.getLoc(), it.value(),
+        newResults.push_back(bufferization::ToTensorOp::create(
+            rewriter, executeRegionOp.getLoc(), it.value(),
             newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
@@ -260,9 +260,9 @@ struct IfOpInterface
 
     // Create new op.
     rewriter.setInsertionPoint(ifOp);
-    auto newIfOp =
-        scf::IfOp::create(rewriter, ifOp.getLoc(), newTypes, ifOp.getCondition(),
-                                   /*withElseRegion=*/true);
+    auto newIfOp = scf::IfOp::create(rewriter, ifOp.getLoc(), newTypes,
+                                     ifOp.getCondition(),
+                                     /*withElseRegion=*/true);
 
     // Move over then/else blocks.
     rewriter.mergeBlocks(ifOp.thenBlock(), newIfOp.thenBlock());
@@ -374,9 +374,9 @@ struct IndexSwitchOpInterface
 
     // Create new op.
     rewriter.setInsertionPoint(switchOp);
-    auto newSwitchOp = scf::IndexSwitchOp::create(rewriter,
-        switchOp.getLoc(), newTypes, switchOp.getArg(), switchOp.getCases(),
-        switchOp.getCases().size());
+    auto newSwitchOp = scf::IndexSwitchOp::create(
+        rewriter, switchOp.getLoc(), newTypes, switchOp.getArg(),
+        switchOp.getCases(), switchOp.getCases().size());
 
     // Move over blocks.
     for (auto [src, dest] :
@@ -769,8 +769,8 @@ struct ForOpInterface
     }
 
     // Construct a new scf.for op with memref instead of tensor values.
-    auto newForOp = scf::ForOp::create(rewriter,
-        forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
+    auto newForOp = scf::ForOp::create(
+        rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
         forOp.getStep(), castedInitArgs);
     newForOp->setAttrs(forOp->getAttrs());
     Block *loopBody = newForOp.getBody();
@@ -1005,8 +1005,8 @@ struct WhileOpInterface
     // Construct a new scf.while op with memref instead of tensor values.
     ValueRange argsRangeBefore(castedInitArgs);
     TypeRange argsTypesBefore(argsRangeBefore);
-    auto newWhileOp = scf::WhileOp::create(rewriter,
-        whileOp.getLoc(), argsTypesAfter, castedInitArgs);
+    auto newWhileOp = scf::WhileOp::create(rewriter, whileOp.getLoc(),
+                                           argsTypesAfter, castedInitArgs);
 
     // Add before/after regions to the new op.
     SmallVector<Location> bbArgLocsBefore(castedInitArgs.size(),
@@ -1265,8 +1265,8 @@ struct ForallOpInterface
              forallOp.getBody()->getArguments().drop_front(rank), buffers)) {
       BlockArgument bbArg = std::get<0>(it);
       Value buffer = std::get<1>(it);
-      Value bufferAsTensor = ToTensorOp::create(rewriter,
-          forallOp.getLoc(), bbArg.getType(), buffer);
+      Value bufferAsTensor = ToTensorOp::create(rewriter, forallOp.getLoc(),
+                                                bbArg.getType(), buffer);
       bbArg.replaceAllUsesWith(bufferAsTensor);
     }
 
@@ -1274,8 +1274,8 @@ struct ForallOpInterface
     // introduced terminator.
     rewriter.setInsertionPoint(forallOp);
     ForallOp newForallOp;
-    newForallOp = ForallOp::create(rewriter,
-        forallOp.getLoc(), forallOp.getMixedLowerBound(),
+    newForallOp = ForallOp::create(
+        rewriter, forallOp.getLoc(), forallOp.getMixedLowerBound(),
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
         /*outputs=*/ValueRange(), forallOp.getMapping());
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
index bf4eba3319a4d..bee77800e9d44 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
@@ -51,18 +51,18 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     initArgs.push_back(forOp.getLowerBound());
     llvm::append_range(initArgs, forOp.getInitArgs());
     auto whileOp = WhileOp::create(rewriter, forOp.getLoc(), lcvTypes, initArgs,
-                                            forOp->getAttrs());
+                                   forOp->getAttrs());
 
     // 'before' region contains the loop condition and forwarding of iteration
     // arguments to the 'after' region.
     auto *beforeBlock = rewriter.createBlock(
         &whileOp.getBefore(), whileOp.getBefore().begin(), lcvTypes, lcvLocs);
     rewriter.setInsertionPointToStart(whileOp.getBeforeBody());
-    auto cmpOp = arith::CmpIOp::create(rewriter,
-        whileOp.getLoc(), arith::CmpIPredicate::slt,
+    auto cmpOp = arith::CmpIOp::create(
+        rewriter, whileOp.getLoc(), arith::CmpIPredicate::slt,
         beforeBlock->getArgument(0), forOp.getUpperBound());
     scf::ConditionOp::create(rewriter, whileOp.getLoc(), cmpOp.getResult(),
-                                      beforeBlock->getArguments());
+                             beforeBlock->getArguments());
 
     // Inline for-loop body into an executeRegion operation in the "after"
     // region. The return type of the execRegionOp does not contain the
@@ -72,8 +72,9 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
 
     // Add induction variable incrementation
     rewriter.setInsertionPointToEnd(afterBlock);
-    auto ivIncOp = arith::AddIOp::create(rewriter,
-        whileOp.getLoc(), afterBlock->getArgument(0), forOp.getStep());
+    auto ivIncOp =
+        arith::AddIOp::create(rewriter, whileOp.getLoc(),
+                              afterBlock->getArgument(0), forOp.getStep());
 
     // Rewrite uses of the for-loop block arguments to the new while-loop
     // "after" arguments
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index f1ede64d01e0f..59828563a8196 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -279,25 +279,25 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
     if (dynamicLoop) {
       Type t = ub.getType();
       // pred = ub > lb + (i * step)
-      Value iv = arith::AddIOp::create(rewriter,
-          loc, lb,
-          arith::MulIOp::create(rewriter,
-              loc, step,
-              arith::ConstantOp::create(rewriter,
-                  loc, rewriter.getIntegerAttr(t, i))));
-      predicates[i] = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::slt, iv, ub);
+      Value iv = arith::AddIOp::create(
+          rewriter, loc, lb,
+          arith::MulIOp::create(
+              rewriter, loc, step,
+              arith::ConstantOp::create(rewriter, loc,
+                                        rewriter.getIntegerAttr(t, i))));
+      predicates[i] = arith::CmpIOp::create(rewriter, loc,
+                                            arith::CmpIPredicate::slt, iv, ub);
     }
 
     // special handling for induction variable as the increment is implicit.
     // iv = lb + i * step
     Type t = lb.getType();
-    Value iv = arith::AddIOp::create(rewriter,
-        loc, lb,
-        arith::MulIOp::create(rewriter,
-            loc, step,
+    Value iv = arith::AddIOp::create(
+        rewriter, loc, lb,
+        arith::MulIOp::create(
+            rewriter, loc, step,
             arith::ConstantOp::create(rewriter, loc,
-                                               rewriter.getIntegerAttr(t, i))));
+                                      rewriter.getIntegerAttr(t, i))));
     setValueMapping(forOp.getInductionVar(), iv, i);
     for (Operation *op : opOrder) {
       if (stages[op] > i)
@@ -332,8 +332,8 @@ LogicalResult LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) {
             Value prevValue = valueMapping
                 [forOp.getRegionIterArgs()[operand.getOperandNumber()]]
                 [i - stages[op]];
-            source = arith::SelectOp::create(rewriter,
-                loc, predicates[predicateIdx], source, prevValue);
+            source = arith::SelectOp::create(
+                rewriter, loc, predicates[predicateIdx], source, prevValue);
           }
           setValueMapping(forOp.getRegionIterArgs()[operand.getOperandNumber()],
                           source, i - stages[op] + 1);
@@ -444,15 +444,15 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop(
     Type t = ub.getType();
     Location loc = forOp.getLoc();
     // newUb = ub - maxStage * step
-    Value maxStageValue = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getIntegerAttr(t, maxStage));
+    Value maxStageValue = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getIntegerAttr(t, maxStage));
     Value maxStageByStep =
         arith::MulIOp::create(rewriter, loc, step, maxStageValue);
     newUb = arith::SubIOp::create(rewriter, loc, ub, maxStageByStep);
   }
   auto newForOp =
       scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(), newUb,
-                                  forOp.getStep(), newLoopArg);
+                         forOp.getStep(), newLoopArg);
   // When there are no iter args, the loop body terminator will be created.
   // Since we always create it below, remove the terminator if it was created.
   if (!newForOp.getBody()->empty())
@@ -483,16 +483,17 @@ LogicalResult LoopPipelinerInternal::createKernel(
     Type t = ub.getType();
     for (unsigned i = 0; i < maxStage; i++) {
       // c = ub - (maxStage - i) * step
-      Value c = arith::SubIOp::create(rewriter,
-          loc, ub,
-          arith::MulIOp::create(rewriter,
-              loc, step,
-              arith::ConstantOp::create(rewriter,
-                  loc, rewriter.getIntegerAttr(t, int64_t(maxStage - i)))));
-
-      Value pred = arith::CmpIOp::create(rewriter,
-          newForOp.getLoc(), arith::CmpIPredicate::slt,
-          newForOp.getInductionVar(), c);
+      Value c = arith::SubIOp::create(
+          rewriter, loc, ub,
+          arith::MulIOp::create(
+              rewriter, loc, step,
+              arith::ConstantOp::create(
+                  rewriter, loc,
+                  rewriter.getIntegerAttr(t, int64_t(maxStage - i)))));
+
+      Value pred = arith::CmpIOp::create(rewriter, newForOp.getLoc(),
+                                         arith::CmpIPredicate::slt,
+                                         newForOp.getInductionVar(), c);
       predicates[i] = pred;
     }
   }
@@ -515,13 +516,13 @@ LogicalResult LoopPipelinerInternal::createKernel(
 
         // offset = (maxStage - stages[op]) * step
         Type t = step.getType();
-        Value offset = arith::MulIOp::create(rewriter,
-            forOp.getLoc(), step,
-            arith::ConstantOp::create(rewriter,
-                forOp.getLoc(),
+        Value offset = arith::MulIOp::create(
+            rewriter, forOp.getLoc(), step,
+            arith::ConstantOp::create(
+                rewriter, forOp.getLoc(),
                 rewriter.getIntegerAttr(t, maxStage - stages[op])));
-        Value iv = arith::AddIOp::create(rewriter,
-            forOp.getLoc(), newForOp.getInductionVar(), offset);
+        Value iv = arith::AddIOp::create(rewriter, forOp.getLoc(),
+                                         newForOp.getInductionVar(), offset);
         nestedNewOp->setOperand(operand->getOperandNumber(), iv);
         rewriter.setInsertionPointAfter(newOp);
         continue;
@@ -594,8 +595,8 @@ LogicalResult LoopPipelinerInternal::createKernel(
         auto defStage = stages.find(def);
         if (defStage != stages.end() && defStage->second < maxStage) {
           Value pred = predicates[defStage->second];
-          source = arith::SelectOp::create(rewriter,
-              pred.getLoc(), pred, source,
+          source = arith::SelectOp::create(
+              rewriter, pred.getLoc(), pred, source,
               newForOp.getBody()
                   ->getArguments()[yieldOperand.getOperandNumber() + 1]);
         }
@@ -653,7 +654,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
 
   auto createConst = [&](int v) {
     return arith::ConstantOp::create(rewriter, loc,
-                                              rewriter.getIntegerAttr(t, v));
+                                     rewriter.getIntegerAttr(t, v));
   };
 
   // total_iterations = cdiv(range_diff, step);
@@ -661,22 +662,23 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // - total_iterations = (range_diff + step + (step < 0 ? 1 : -1)) / step
   Value zero = createConst(0);
   Value one = createConst(1);
-  Value stepLessZero = arith::CmpIOp::create(rewriter,
-      loc, arith::CmpIPredicate::slt, step, zero);
-  Value stepDecr =
-      arith::SelectOp::create(rewriter, loc, stepLessZero, one, createConst(-1));
+  Value stepLessZero = arith::CmpIOp::create(
+      rewriter, loc, arith::CmpIPredicate::slt, step, zero);
+  Value stepDecr = arith::SelectOp::create(rewriter, loc, stepLessZero, one,
+                                           createConst(-1));
 
   Value rangeDiff = arith::SubIOp::create(rewriter, loc, ub, lb);
   Value rangeIncrStep = arith::AddIOp::create(rewriter, loc, rangeDiff, step);
   Value rangeDecr =
       arith::AddIOp::create(rewriter, loc, rangeIncrStep, stepDecr);
-  Value totalIterations = arith::DivSIOp::create(rewriter, loc, rangeDecr, step);
+  Value totalIterations =
+      arith::DivSIOp::create(rewriter, loc, rangeDecr, step);
 
   // If total_iters < max_stage, start the epilogue at zero to match the
   // ramp-up in the prologue.
   // start_iter = max(0, total_iters - max_stage)
   Value iterI = arith::SubIOp::create(rewriter, loc, totalIterations,
-                                               createConst(maxStage));
+                                      createConst(maxStage));
   iterI = arith::MaxSIOp::create(rewriter, loc, zero, iterI);
 
   // Capture predicates for dynamic loops.
@@ -684,8 +686,8 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
 
   for (int64_t i = 1; i <= maxStage; i++) {
     // newLastIter = lb + step * iterI
-    Value newlastIter = arith::AddIOp::create(rewriter,
-        loc, lb, arith::MulIOp::create(rewriter, loc, step, iterI));
+    Value newlastIter = arith::AddIOp::create(
+        rewriter, loc, lb, arith::MulIOp::create(rewriter, loc, step, iterI));
 
     setValueMapping(forOp.getInductionVar(), newlastIter, i);
 
@@ -695,8 +697,9 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
     if (dynamicLoop) {
       // Disable stages when `i` is greater than total_iters.
       // pred = total_iters >= i
-      predicates[i] = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::sge, totalIterations, createConst(i));
+      predicates[i] =
+          arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::sge,
+                                totalIterations, createConst(i));
     }
   }
 
@@ -758,8 +761,8 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
           unsigned nextVersion = currentVersion + 1;
           Value pred = predicates[currentVersion];
           Value prevValue = valueMapping[mapVal][currentVersion];
-          auto selOp = arith::SelectOp::create(rewriter, loc, pred, pair.value(),
-                                                        prevValue);
+          auto selOp = arith::SelectOp::create(rewriter, loc, pred,
+                                               pair.value(), prevValue);
           returnValues[ri] = selOp;
           if (nextVersion <= maxStage)
             setValueMapping(mapVal, selOp, nextVersion);
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index b24e276272c2c..15476d5e2bfc2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -66,7 +66,7 @@ static void specializeParallelLoopForUnrolling(ParallelOp op) {
     Value constant =
         arith::ConstantIndexOp::create(b, op.getLoc(), std::get<1>(bound));
     Value cmp = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
-                                        std::get<0>(bound), constant);
+                                      std::get<0>(bound), constant);
     cond = cond ? arith::AndIOp::create(b, op.getLoc(), cond, cmp) : cmp;
     map.map(std::get<0>(bound), constant);
   }
@@ -97,7 +97,7 @@ static void specializeForLoopForUnrolling(ForOp op) {
   IRMapping map;
   Value constant = arith::ConstantIndexOp::create(b, op.getLoc(), minConstant);
   Value cond = arith::CmpIOp::create(b, op.getLoc(), arith::CmpIPredicate::eq,
-                                       bound, constant);
+                                     bound, constant);
   map.map(bound, constant);
   auto ifOp = scf::IfOp::create(b, op.getLoc(), cond, /*withElseRegion=*/true);
   ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index 013c3b69620e2..694cd85435f63 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -190,8 +190,8 @@ static void fuseIfLegal(ParallelOp firstPloop, ParallelOp &secondPloop,
 
   IRRewriter b(builder);
   b.setInsertionPoint(secondPloop);
-  auto newSecondPloop = ParallelOp::create(b,
-      secondPloop.getLoc(), secondPloop.getLowerBound(),
+  auto newSecondPloop = ParallelOp::create(
+      b, secondPloop.getLoc(), secondPloop.getLowerBound(),
       secondPloop.getUpperBound(), secondPloop.getStep(), newInitVars);
 
   Block *newBlock = newSecondPloop.getBody();
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
index 2ca3b1e366bdf..84a779b90f6c2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
@@ -76,10 +76,10 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
   newSteps.reserve(op.getStep().size());
   for (auto step : llvm::zip(op.getStep(), tileSizeConstants)) {
     newSteps.push_back(arith::MulIOp::create(b, op.getLoc(), std::get<0>(step),
-                                               std::get<1>(step)));
+                                             std::get<1>(step)));
   }
   auto outerLoop = ParallelOp::create(b, op.getLoc(), op.getLowerBound(),
-                                        op.getUpperBound(), newSteps);
+                                      op.getUpperBound(), newSteps);
   b.setInsertionPointToStart(outerLoop.getBody());
 
   // Compute min(size, dim - offset) to avoid out-of-bounds accesses.
@@ -131,10 +131,10 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     // each iteration of the outer loop.
     newBounds.push_back(
         affine::AffineMinOp::create(b, op.getLoc(), b.getIndexType(), minMap,
-                                      ValueRange{newStep, upperBound, iv}));
+                                    ValueRange{newStep, upperBound, iv}));
   }
-  auto innerLoop = ParallelOp::create(b,
-      op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
+  auto innerLoop = ParallelOp::create(
+      b, op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
       op.getStep());
 
   if (noMinMaxBounds && needInboundCheck) {
@@ -147,16 +147,16 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
                    innerLoop.getInductionVars(), innerLoop.getStep())) {
       // %in_bound = %in_bound &&
       //             (%inner_iv * %inner_step + %outer_iv < %outer_upper_bound)
-      Value index = arith::AddIOp::create(b,
-          op.getLoc(), arith::MulIOp::create(b, op.getLoc(), innerIV, innerStep),
-          outerIV);
-      Value dimInbound = arith::CmpIOp::create(b,
-          op.getLoc(), arith::CmpIPredicate::ult, index, outerUpperBound);
+      Value index = arith::AddIOp::create(
+          b, op.getLoc(),
+          arith::MulIOp::create(b, op.getLoc(), innerIV, innerStep), outerIV);
+      Value dimInbound = arith::CmpIOp::create(
+          b, op.getLoc(), arith::CmpIPredicate::ult, index, outerUpperBound);
       inbound = arith::AndIOp::create(b, op.getLoc(), inbound, dimInbound);
     }
     auto ifInbound = IfOp::create(b, op.getLoc(),
-                                    /*resultTypes*/ ArrayRef<Type>{}, inbound,
-                                    /*hasElseRegion*/ false);
+                                  /*resultTypes*/ ArrayRef<Type>{}, inbound,
+                                  /*hasElseRegion*/ false);
     ifInbound.getThenRegion().takeBody(op.getRegion());
     Block &thenBlock = ifInbound.getThenRegion().front();
     // Replace the scf.reduce terminator with an scf.yield terminator.
@@ -167,8 +167,8 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
     b.setInsertionPointToStart(innerLoop.getBody());
     for (const auto &ivs : llvm::enumerate(llvm::zip(
              innerLoop.getInductionVars(), outerLoop.getInductionVars()))) {
-      auto newIndex = arith::AddIOp::create(b,
-          op.getLoc(), std::get<0>(ivs.value()), std::get<1>(ivs.value()));
+      auto newIndex = arith::AddIOp::create(
+          b, op.getLoc(), std::get<0>(ivs.value()), std::get<1>(ivs.value()));
       thenBlock.getArgument(ivs.index())
           .replaceAllUsesExcept(newIndex, newIndex);
     }
@@ -180,7 +180,7 @@ mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes,
                               outerLoop.getInductionVars())) {
       Value innerIndex = std::get<0>(ivs);
       auto newIndex = arith::AddIOp::create(b, op.getLoc(), std::get<0>(ivs),
-                                              std::get<1>(ivs));
+                                            std::get<1>(ivs));
       innerIndex.replaceAllUsesExcept(newIndex, newIndex);
     }
   }
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 41895e46d953d..1b07b77546030 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -112,11 +112,11 @@ class ConvertForOpTypes
 
     // We can not do clone as the number of result types after conversion
     // might be different.
-    ForOp newOp = ForOp::create(rewriter,
-        op.getLoc(), llvm::getSingleElement(adaptor.getLowerBound()),
-        llvm::getSingleElement(adaptor.getUpperBound()),
-        llvm::getSingleElement(adaptor.getStep()),
-        flattenValues(adaptor.getInitArgs()));
+    ForOp newOp = ForOp::create(rewriter, op.getLoc(),
+                                llvm::getSingleElement(adaptor.getLowerBound()),
+                                llvm::getSingleElement(adaptor.getUpperBound()),
+                                llvm::getSingleElement(adaptor.getStep()),
+                                flattenValues(adaptor.getInitArgs()));
 
     // Reserve whatever attributes in the original op.
     newOp->setAttrs(op->getAttrs());
@@ -142,9 +142,9 @@ class ConvertIfOpTypes
                                       ConversionPatternRewriter &rewriter,
                                       TypeRange dstTypes) const {
 
-    IfOp newOp = IfOp::create(rewriter,
-        op.getLoc(), dstTypes, llvm::getSingleElement(adaptor.getCondition()),
-        true);
+    IfOp newOp =
+        IfOp::create(rewriter, op.getLoc(), dstTypes,
+                     llvm::getSingleElement(adaptor.getCondition()), true);
     newOp->setAttrs(op->getAttrs());
 
     // We do not need the empty blocks created by rewriter.
@@ -172,7 +172,7 @@ class ConvertWhileOpTypes
                                          ConversionPatternRewriter &rewriter,
                                          TypeRange dstTypes) const {
     auto newOp = WhileOp::create(rewriter, op.getLoc(), dstTypes,
-                                          flattenValues(adaptor.getOperands()));
+                                 flattenValues(adaptor.getOperands()));
 
     for (auto i : {0u, 1u}) {
       if (failed(rewriter.convertRegionTypes(&op.getRegion(i), *typeConverter)))
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index 3171b96d46b8a..6a9e02d844e94 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -451,8 +451,8 @@ static LogicalResult generateLoopNestUsingForOp(
   for (auto [lb, ub, step] : llvm::zip_equal(lbVals, ubVals, stepVals)) {
     auto loop =
         scf::ForOp::create(rewriter, loc, lb, ub, step, destinationTensors,
-                                    [](OpBuilder &bodyBuilder, Location bodyLoc,
-                                       Value iv, ValueRange /*iterArgs*/) {});
+                           [](OpBuilder &bodyBuilder, Location bodyLoc,
+                              Value iv, ValueRange /*iterArgs*/) {});
     loops.push_back(loop);
     ivs.push_back(loop.getInductionVar());
     rewriter.setInsertionPointToEnd(loop.getBody());
@@ -479,8 +479,8 @@ static LogicalResult generateLoopNestUsingForOp(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    auto insertSlice = tensor::InsertSliceOp::create(rewriter,
-        loc, tiledValue, destinationTensor, resultOffset, resultSize,
+    auto insertSlice = tensor::InsertSliceOp::create(
+        rewriter, loc, tiledValue, destinationTensor, resultOffset, resultSize,
         resultStride);
     yieldedValues.push_back(insertSlice);
   }
@@ -534,13 +534,13 @@ static LogicalResult generateLoopNestUsingForallOp(
       nonZeroNumThreads.push_back(nt);
     }
     forallOp = scf::ForallOp::create(rewriter, loc, nonZeroNumThreads,
-                                              destinationTensors, mappingAttr);
+                                     destinationTensors, mappingAttr);
   } else {
     SmallVector<OpFoldResult> lbs, ubs, steps;
     std::tie(lbs, ubs, steps) =
         getLoopBounds(rewriter, loc, loopRanges, tileSizes);
     forallOp = scf::ForallOp::create(rewriter, loc, lbs, ubs, steps,
-                                              destinationTensors, mappingAttr);
+                                     destinationTensors, mappingAttr);
   }
   loops.push_back(forallOp);
 
@@ -561,9 +561,9 @@ static LogicalResult generateLoopNestUsingForallOp(
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
 
-    tensor::ParallelInsertSliceOp::create(rewriter,
-        loc, tiledValue, destinationTensor, resultOffset, resultSize,
-        resultStride);
+    tensor::ParallelInsertSliceOp::create(rewriter, loc, tiledValue,
+                                          destinationTensor, resultOffset,
+                                          resultSize, resultStride);
   }
   return success();
 }
@@ -798,9 +798,9 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
 
   auto inits = llvm::to_vector(loopOp.getInitArgs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = scf::ForOp::create(rewriter,
-      loc, loopOp.getLowerBound(), loopOp.getUpperBound(), loopOp.getStep(),
-      inits, [](OpBuilder &, Location, Value, ValueRange) {});
+  auto newLoop = scf::ForOp::create(
+      rewriter, loc, loopOp.getLowerBound(), loopOp.getUpperBound(),
+      loopOp.getStep(), inits, [](OpBuilder &, Location, Value, ValueRange) {});
 
   // Move the loop body to the new op.
   Block *loopBody = loopOp.getBody();
@@ -829,9 +829,9 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
                        resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    Value insert = tensor::InsertSliceOp::create(rewriter,
-        yieldOp->getLoc(), tiledValue, regionIterArg, resultOffset, resultSize,
-        resultStride);
+    Value insert = tensor::InsertSliceOp::create(
+        rewriter, yieldOp->getLoc(), tiledValue, regionIterArg, resultOffset,
+        resultSize, resultStride);
     newYieldValues.push_back(insert);
   }
 
@@ -851,8 +851,8 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
   rewriter.setInsertionPoint(loopOp);
   auto inits = llvm::to_vector(loopOp.getOutputs());
   inits.append(newInitOperands.begin(), newInitOperands.end());
-  auto newLoop = scf::ForallOp::create(rewriter,
-      loc, loopOp.getMixedLowerBound(), loopOp.getMixedUpperBound(),
+  auto newLoop = scf::ForallOp::create(
+      rewriter, loc, loopOp.getMixedLowerBound(), loopOp.getMixedUpperBound(),
       loopOp.getMixedStep(), inits, loopOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
 
@@ -884,9 +884,9 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForallOp>(
            tiledValues, regionIterArgs, resultOffsets, resultSizes)) {
     SmallVector<OpFoldResult> resultStride(resultOffset.size(),
                                            rewriter.getIndexAttr(1));
-    tensor::ParallelInsertSliceOp::create(rewriter,
-        terminator.getLoc(), tiledValue, iterArg, resultOffset, resultSize,
-        resultStride);
+    tensor::ParallelInsertSliceOp::create(rewriter, terminator.getLoc(),
+                                          tiledValue, iterArg, resultOffset,
+                                          resultSize, resultStride);
   }
 
   rewriter.replaceOp(loopOp,
@@ -935,9 +935,9 @@ static LogicalResult addInitOperandsToLoopNest(
     // Create a new loop with the new init values for this loop.
     SmallVector<Value> newInits = llvm::to_vector(forLoop.getInitArgs());
     newInits.append(newInitValues.begin(), newInitValues.end());
-    auto newLoop = scf::ForOp::create(rewriter,
-        forLoop.getLoc(), forLoop.getLowerBound(), forLoop.getUpperBound(),
-        forLoop.getStep(), newInits,
+    auto newLoop = scf::ForOp::create(
+        rewriter, forLoop.getLoc(), forLoop.getLowerBound(),
+        forLoop.getUpperBound(), forLoop.getStep(), newInits,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {});
 
     // Merge the body of the new loop with the body of the old loops.
@@ -1419,8 +1419,8 @@ FailureOr<SmallVector<Operation *>> mlir::scf::yieldReplacementForFusedProducer(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
-            loc, newRegionArg, offsetList[index], sizesList[index],
+        auto destSlice = tensor::ExtractSliceOp::create(
+            rewriter, loc, newRegionArg, offsetList[index], sizesList[index],
             SmallVector<OpFoldResult>(offsetList[index].size(),
                                       rewriter.getIndexAttr(1)));
         generatedSlices.push_back(destSlice);
@@ -2092,8 +2092,8 @@ cloneAsInsertSlice<tensor::InsertSliceOp>(RewriterBase &rewriter,
 template <>
 tensor::InsertSliceOp cloneAsInsertSlice<tensor::ParallelInsertSliceOp>(
     RewriterBase &rewriter, tensor::ParallelInsertSliceOp insertSliceOp) {
-  return tensor::InsertSliceOp::create(rewriter,
-      insertSliceOp->getLoc(), insertSliceOp.getSource(),
+  return tensor::InsertSliceOp::create(
+      rewriter, insertSliceOp->getLoc(), insertSliceOp.getSource(),
       insertSliceOp.getDest(), insertSliceOp.getMixedOffsets(),
       insertSliceOp.getMixedSizes(), insertSliceOp.getMixedStrides());
 }
@@ -2314,8 +2314,9 @@ mlir::scf::tileAndFuseConsumerOfSlices(
       rewriter.setInsertionPoint(tiledDestStyleOp);
       for (const auto &&[index, newRegionArg] :
            llvm::enumerate(newRegionIterArgs)) {
-        auto destSlice = tensor::ExtractSliceOp::create(rewriter,
-            loc, newRegionArg, resultOffsets[index], resultSizes[index],
+        auto destSlice = tensor::ExtractSliceOp::create(
+            rewriter, loc, newRegionArg, resultOffsets[index],
+            resultSizes[index],
             SmallVector<OpFoldResult>(resultOffsets[index].size(),
                                       rewriter.getIndexAttr(1)));
         // Make a copy of index to avoid a capturing structured binding, which
@@ -2392,7 +2393,7 @@ mlir::scf::lowerToLoopsUsingSCFForOp(RewriterBase &rewriter,
     Value strideVal =
         getValueOrCreateConstantIndexOp(rewriter, loc, loopRange.stride);
     auto loop = scf::ForOp::create(rewriter, op.getLoc(), offsetVal, sizeVal,
-                                            strideVal, ValueRange{});
+                                   strideVal, ValueRange{});
     loops.push_back(loop);
     ivs.push_back(loop.getInductionVar());
     rewriter.setInsertionPoint(loop.getBody()->getTerminator());
diff --git a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
index 707ed836cf63e..db504fe157dc9 100644
--- a/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/WrapInZeroTripCheck.cpp
@@ -96,8 +96,8 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       condOp.getArgs(), [&](Value arg) { return mapper.lookupOrDefault(arg); });
 
   // Create rotated while loop.
-  auto newLoopOp = scf::WhileOp::create(rewriter,
-      whileOp.getLoc(), whileOp.getResultTypes(), clonedCondArgs,
+  auto newLoopOp = scf::WhileOp::create(
+      rewriter, whileOp.getLoc(), whileOp.getResultTypes(), clonedCondArgs,
       [&](OpBuilder &builder, Location loc, ValueRange args) {
         // Rotate and move the loop body into before block.
         auto newBlock = builder.getBlock();
@@ -113,8 +113,8 @@ FailureOr<scf::WhileOp> mlir::scf::wrapWhileLoopInZeroTripCheck(
       });
 
   // Create zero-trip-check and move the while loop in.
-  auto ifOp = scf::IfOp::create(rewriter,
-      whileOp.getLoc(), clonedCondition,
+  auto ifOp = scf::IfOp::create(
+      rewriter, whileOp.getLoc(), clonedCondition,
       [&](OpBuilder &builder, Location loc) {
         // Then runs the while loop.
         rewriter.moveOpBefore(newLoopOp, builder.getInsertionBlock(),
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index 6a387ddf8908a..2b7459b7addec 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -169,7 +169,7 @@ FailureOr<func::FuncOp> mlir::outlineSingleBlockRegion(RewriterBase &rewriter,
     // Explicitly set up a new ReturnOp terminator.
     rewriter.setInsertionPointToEnd(outlinedFuncBody);
     func::ReturnOp::create(rewriter, loc, originalTerminator->getResultTypes(),
-                                    originalTerminator->getOperands());
+                           originalTerminator->getOperands());
   }
 
   // Reconstruct the block that was deleted and add a
@@ -274,10 +274,10 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
 
-  Value divisorMinusOneCst = arith::ConstantOp::create(builder,
-      loc, builder.getIntegerAttr(dividend.getType(), divisor - 1));
-  Value divisorCst = arith::ConstantOp::create(builder,
-      loc, builder.getIntegerAttr(dividend.getType(), divisor));
+  Value divisorMinusOneCst = arith::ConstantOp::create(
+      builder, loc, builder.getIntegerAttr(dividend.getType(), divisor - 1));
+  Value divisorCst = arith::ConstantOp::create(
+      builder, loc, builder.getIntegerAttr(dividend.getType(), divisor));
   Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOneCst);
   return arith::DivUIOp::create(builder, loc, sum, divisorCst);
 }
@@ -290,8 +290,8 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
                              Value divisor) {
   assert(dividend.getType().isIntOrIndex() &&
          "expected integer or index-typed value");
-  Value cstOne = arith::ConstantOp::create(builder,
-      loc, builder.getOneAttr(dividend.getType()));
+  Value cstOne = arith::ConstantOp::create(
+      builder, loc, builder.getOneAttr(dividend.getType()));
   Value divisorMinusOne = arith::SubIOp::create(builder, loc, divisor, cstOne);
   Value sum = arith::AddIOp::create(builder, loc, dividend, divisorMinusOne);
   return arith::DivUIOp::create(builder, loc, sum, divisor);
@@ -404,18 +404,20 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     // Create constant for 'upperBoundUnrolled' and set epilogue loop flag.
     generateEpilogueLoop = upperBoundUnrolledCst < ubCst;
     if (generateEpilogueLoop)
-      upperBoundUnrolled = arith::ConstantOp::create(boundsBuilder,
-          loc, boundsBuilder.getIntegerAttr(forOp.getUpperBound().getType(),
-                                            upperBoundUnrolledCst));
+      upperBoundUnrolled = arith::ConstantOp::create(
+          boundsBuilder, loc,
+          boundsBuilder.getIntegerAttr(forOp.getUpperBound().getType(),
+                                       upperBoundUnrolledCst));
     else
       upperBoundUnrolled = forOp.getUpperBound();
 
     // Create constant for 'stepUnrolled'.
-    stepUnrolled = stepCst == stepUnrolledCst
-                       ? step
-                       : arith::ConstantOp::create(boundsBuilder,
-                             loc, boundsBuilder.getIntegerAttr(
-                                      step.getType(), stepUnrolledCst));
+    stepUnrolled =
+        stepCst == stepUnrolledCst
+            ? step
+            : arith::ConstantOp::create(boundsBuilder, loc,
+                                        boundsBuilder.getIntegerAttr(
+                                            step.getType(), stepUnrolledCst));
   } else {
     // Dynamic loop bounds computation.
     // TODO: Add dynamic asserts for negative lb/ub/step, or
@@ -425,16 +427,17 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
     Value diff =
         arith::SubIOp::create(boundsBuilder, loc, upperBound, lowerBound);
     Value tripCount = ceilDivPositive(boundsBuilder, loc, diff, step);
-    Value unrollFactorCst = arith::ConstantOp::create(boundsBuilder,
-        loc, boundsBuilder.getIntegerAttr(tripCount.getType(), unrollFactor));
+    Value unrollFactorCst = arith::ConstantOp::create(
+        boundsBuilder, loc,
+        boundsBuilder.getIntegerAttr(tripCount.getType(), unrollFactor));
     Value tripCountRem =
         arith::RemSIOp::create(boundsBuilder, loc, tripCount, unrollFactorCst);
     // Compute tripCountEvenMultiple = tripCount - (tripCount % unrollFactor)
     Value tripCountEvenMultiple =
         arith::SubIOp::create(boundsBuilder, loc, tripCount, tripCountRem);
     // Compute upperBoundUnrolled = lowerBound + tripCountEvenMultiple * step
-    upperBoundUnrolled = arith::AddIOp::create(boundsBuilder,
-        loc, lowerBound,
+    upperBoundUnrolled = arith::AddIOp::create(
+        boundsBuilder, loc, lowerBound,
         arith::MulIOp::create(boundsBuilder, loc, tripCountEvenMultiple, step));
     // Scale 'step' by 'unrollFactor'.
     stepUnrolled =
@@ -474,10 +477,10 @@ FailureOr<UnrolledLoopInfo> mlir::loopUnrollByFactor(
       forOp.getBody(), forOp.getInductionVar(), unrollFactor,
       [&](unsigned i, Value iv, OpBuilder b) {
         // iv' = iv + step * i;
-        auto stride = arith::MulIOp::create(b,
-            loc, step,
+        auto stride = arith::MulIOp::create(
+            b, loc, step,
             arith::ConstantOp::create(b, loc,
-                                        b.getIntegerAttr(iv.getType(), i)));
+                                      b.getIntegerAttr(iv.getType(), i)));
         return arith::AddIOp::create(b, loc, iv, stride);
       },
       annotateFn, iterArgs, yieldedValues);
@@ -823,8 +826,8 @@ static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc,
     if (vOne && vOne.value() == 1)
       continue;
     if (productOf)
-      productOf =
-          arith::MulIOp::create(rewriter, loc, productOf.value(), v).getResult();
+      productOf = arith::MulIOp::create(rewriter, loc, productOf.value(), v)
+                      .getResult();
     else
       productOf = v;
   }
@@ -850,9 +853,8 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
                              Value linearizedIv, ArrayRef<Value> ubs) {
 
   if (linearizedIv.getType().isIndex()) {
-    Operation *delinearizedOp =
-        affine::AffineDelinearizeIndexOp::create(rewriter, loc, linearizedIv,
-                                                          ubs);
+    Operation *delinearizedOp = affine::AffineDelinearizeIndexOp::create(
+        rewriter, loc, linearizedIv, ubs);
     auto resultVals = llvm::map_to_vector(
         delinearizedOp->getResults(), [](OpResult r) -> Value { return r; });
     return {resultVals, SmallPtrSet<Operation *, 2>{delinearizedOp}};
@@ -874,8 +876,8 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
     if (!isUbOne.test(index)) {
       break;
     }
-    delinearizedIvs[index] = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(ub.getType()));
+    delinearizedIvs[index] = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getZeroAttr(ub.getType()));
     numLeadingOneUbs++;
   }
 
@@ -892,8 +894,8 @@ delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
         iv = arith::RemSIOp::create(rewriter, loc, previous, ubs[idx]);
         preservedUsers.insert(iv.getDefiningOp());
       } else {
-        iv = arith::ConstantOp::create(rewriter,
-            loc, rewriter.getZeroAttr(ubs[idx].getType()));
+        iv = arith::ConstantOp::create(
+            rewriter, loc, rewriter.getZeroAttr(ubs[idx].getType()));
       }
     }
     delinearizedIvs[idx] = iv;
@@ -1098,8 +1100,8 @@ void mlir::collapseParallelLoops(
   for (auto &sortedDimension : sortedDimensions) {
     Value newUpperBound = arith::ConstantIndexOp::create(rewriter, loc, 1);
     for (auto idx : sortedDimension) {
-      newUpperBound = arith::MulIOp::create(rewriter,
-          loc, newUpperBound, normalizedUpperBounds[idx]);
+      newUpperBound = arith::MulIOp::create(rewriter, loc, newUpperBound,
+                                            normalizedUpperBounds[idx]);
     }
     lowerBounds.push_back(cst0);
     steps.push_back(cst1);
@@ -1112,8 +1114,8 @@ void mlir::collapseParallelLoops(
   // value. The remainders then determine based on that range, which iteration
   // of the original induction value this represents. This is a normalized value
   // that is un-normalized already by the previous logic.
-  auto newPloop = scf::ParallelOp::create(rewriter,
-      loc, lowerBounds, upperBounds, steps,
+  auto newPloop = scf::ParallelOp::create(
+      rewriter, loc, lowerBounds, upperBounds, steps,
       [&](OpBuilder &insideBuilder, Location, ValueRange ploopIVs) {
         for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) {
           Value previous = ploopIVs[i];
@@ -1123,15 +1125,15 @@ void mlir::collapseParallelLoops(
             unsigned idx = combinedDimensions[i][j];
 
             // Determine the current induction value's current loop iteration
-            Value iv = arith::RemSIOp::create(insideBuilder,
-                loc, previous, normalizedUpperBounds[idx]);
+            Value iv = arith::RemSIOp::create(insideBuilder, loc, previous,
+                                              normalizedUpperBounds[idx]);
             replaceAllUsesInRegionWith(loops.getBody()->getArgument(idx), iv,
                                        loops.getRegion());
 
             // Remove the effect of the current induction value to prepare for
             // the next value.
-            previous = arith::DivSIOp::create(insideBuilder,
-                loc, previous, normalizedUpperBounds[idx]);
+            previous = arith::DivSIOp::create(insideBuilder, loc, previous,
+                                              normalizedUpperBounds[idx]);
           }
 
           // The final induction value is just the remaining value.
@@ -1344,7 +1346,7 @@ TileLoops mlir::extractFixedOuterLoops(scf::ForOp rootForOp,
     OpBuilder builder(forOp);
     auto loc = forOp.getLoc();
     Value diff = arith::SubIOp::create(builder, loc, forOp.getUpperBound(),
-                                               forOp.getLowerBound());
+                                       forOp.getLowerBound());
     Value numIterations = ceilDivPositive(builder, loc, diff, forOp.getStep());
     Value iterationsPerBlock =
         ceilDivPositive(builder, loc, numIterations, sizes[i]);
@@ -1376,9 +1378,10 @@ scf::ForallOp mlir::fuseIndependentSiblingForallLoops(scf::ForallOp target,
 
   // Create a new scf.forall op after the source loop.
   rewriter.setInsertionPointAfter(source);
-  scf::ForallOp fusedLoop = scf::ForallOp::create(rewriter,
-      source.getLoc(), source.getMixedLowerBound(), source.getMixedUpperBound(),
-      source.getMixedStep(), fusedOuts, source.getMapping());
+  scf::ForallOp fusedLoop = scf::ForallOp::create(
+      rewriter, source.getLoc(), source.getMixedLowerBound(),
+      source.getMixedUpperBound(), source.getMixedStep(), fusedOuts,
+      source.getMapping());
 
   // Map control operands.
   IRMapping mapping;
@@ -1429,8 +1432,8 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   // Create a new scf.for op after the source loop (with scf.yield terminator
   // (without arguments) only in case its init_args is empty).
   rewriter.setInsertionPointAfter(source);
-  scf::ForOp fusedLoop = scf::ForOp::create(rewriter,
-      source.getLoc(), source.getLowerBound(), source.getUpperBound(),
+  scf::ForOp fusedLoop = scf::ForOp::create(
+      rewriter, source.getLoc(), source.getLowerBound(), source.getUpperBound(),
       source.getStep(), fusedInitArgs);
 
   // Map original induction variables and operands to those of the fused loop.
@@ -1487,8 +1490,8 @@ FailureOr<scf::ForallOp> mlir::normalizeForallOp(RewriterBase &rewriter,
 
   // Use the normalized builder since the lower bounds are always 0 and the
   // steps are always 1.
-  auto normalizedForallOp = scf::ForallOp::create(rewriter,
-      loc, newUbs, forallOp.getOutputs(), forallOp.getMapping(),
+  auto normalizedForallOp = scf::ForallOp::create(
+      rewriter, loc, newUbs, forallOp.getOutputs(), forallOp.getMapping(),
       [](OpBuilder &, Location, ValueRange) {});
 
   rewriter.inlineRegionBefore(forallOp.getBodyRegion(),
diff --git a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
index cacc52d0feaf0..890406df74e72 100644
--- a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
@@ -569,10 +569,10 @@ SelectionOp::createIfThen(Location loc, Value condition,
   {
     OpBuilder::InsertionGuard guard(builder);
     builder.createBlock(thenBlock);
-    spirv::BranchConditionalOp::create(builder,
-        loc, condition, thenBlock,
-        /*trueArguments=*/ArrayRef<Value>(), mergeBlock,
-        /*falseArguments=*/ArrayRef<Value>());
+    spirv::BranchConditionalOp::create(builder, loc, condition, thenBlock,
+                                       /*trueArguments=*/ArrayRef<Value>(),
+                                       mergeBlock,
+                                       /*falseArguments=*/ArrayRef<Value>());
   }
 
   return selectionOp;
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index 53c07fc88e2ab..2bde44baf961e 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -1309,11 +1309,11 @@ struct ConvertSelectionOpToSelect final : OpRewritePattern<spirv::SelectionOp> {
     auto storeOpAttributes =
         cast<spirv::StoreOp>(trueBlock->front())->getAttrs();
 
-    auto selectOp = spirv::SelectOp::create(rewriter,
-        selectionOp.getLoc(), trueValue.getType(),
+    auto selectOp = spirv::SelectOp::create(
+        rewriter, selectionOp.getLoc(), trueValue.getType(),
         brConditionalOp.getCondition(), trueValue, falseValue);
     spirv::StoreOp::create(rewriter, selectOp.getLoc(), ptrValue,
-                                    selectOp.getResult(), storeOpAttributes);
+                           selectOp.getResult(), storeOpAttributes);
 
     // `spirv.mlir.selection` is not needed anymore.
     rewriter.eraseOp(op);
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 945f95b0f1dc6..5d011b237c0ab 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -652,25 +652,25 @@ spirv::ConstantOp spirv::ConstantOp::getZero(Type type, Location loc,
     unsigned width = intType.getWidth();
     if (width == 1)
       return spirv::ConstantOp::create(builder, loc, type,
-                                               builder.getBoolAttr(false));
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
+                                       builder.getBoolAttr(false));
+    return spirv::ConstantOp::create(
+        builder, loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getFloatAttr(floatType, 0.0));
+    return spirv::ConstantOp::create(builder, loc, type,
+                                     builder.getFloatAttr(floatType, 0.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return spirv::ConstantOp::create(builder,
-          loc, type,
+      return spirv::ConstantOp::create(
+          builder, loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 0).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return spirv::ConstantOp::create(builder,
-          loc, type,
+      return spirv::ConstantOp::create(
+          builder, loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 0.0).getValue()));
     }
@@ -685,25 +685,25 @@ spirv::ConstantOp spirv::ConstantOp::getOne(Type type, Location loc,
     unsigned width = intType.getWidth();
     if (width == 1)
       return spirv::ConstantOp::create(builder, loc, type,
-                                               builder.getBoolAttr(true));
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
+                                       builder.getBoolAttr(true));
+    return spirv::ConstantOp::create(
+        builder, loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
   }
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
-    return spirv::ConstantOp::create(builder,
-        loc, type, builder.getFloatAttr(floatType, 1.0));
+    return spirv::ConstantOp::create(builder, loc, type,
+                                     builder.getFloatAttr(floatType, 1.0));
   }
   if (auto vectorType = llvm::dyn_cast<VectorType>(type)) {
     Type elemType = vectorType.getElementType();
     if (llvm::isa<IntegerType>(elemType)) {
-      return spirv::ConstantOp::create(builder,
-          loc, type,
+      return spirv::ConstantOp::create(
+          builder, loc, type,
           DenseElementsAttr::get(vectorType,
                                  IntegerAttr::get(elemType, 1).getValue()));
     }
     if (llvm::isa<FloatType>(elemType)) {
-      return spirv::ConstantOp::create(builder,
-          loc, type,
+      return spirv::ConstantOp::create(
+          builder, loc, type,
           DenseFPElementsAttr::get(vectorType,
                                    FloatAttr::get(elemType, 1.0).getValue()));
     }
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 0795678a9c66b..2b9c7296830dc 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -105,8 +105,9 @@ OwningOpRef<spirv::ModuleOp> combine(ArrayRef<spirv::ModuleOp> inputModules,
     }
   }
 
-  auto combinedModule = spirv::ModuleOp::create(combinedModuleBuilder,
-      firstModule.getLoc(), addressingModel, memoryModel, vceTriple);
+  auto combinedModule =
+      spirv::ModuleOp::create(combinedModuleBuilder, firstModule.getLoc(),
+                              addressingModel, memoryModel, vceTriple);
   combinedModuleBuilder.setInsertionPointToStart(combinedModule.getBody());
 
   // In some cases, a symbol in the (current state of the) combined module is
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 574f9a7c24b12..81365b44a3aad 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -70,9 +70,9 @@ createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
   varType =
       spirv::PointerType::get(varPointeeType, varPtrType.getStorageClass());
 
-  return spirv::GlobalVariableOp::create(builder,
-      funcOp.getLoc(), varType, varName, abiInfo.getDescriptorSet(),
-      abiInfo.getBinding());
+  return spirv::GlobalVariableOp::create(builder, funcOp.getLoc(), varType,
+                                         varName, abiInfo.getDescriptorSet(),
+                                         abiInfo.getBinding());
 }
 
 /// Gets the global variables that need to be specified as interface variable
@@ -147,7 +147,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
                              "execution model based on 'spirv.target_env'");
 
   spirv::EntryPointOp::create(builder, funcOp.getLoc(), *executionModel, funcOp,
-                                      interfaceVars);
+                              interfaceVars);
 
   // Specifies the spirv.ExecutionModeOp.
   if (DenseI32ArrayAttr workgroupSizeAttr = entryPointAttr.getWorkgroupSize()) {
@@ -155,8 +155,8 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
         spirv::getCapabilities(spirv::ExecutionMode::LocalSize);
     if (!caps || targetEnv.allows(*caps)) {
       spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
-                                             spirv::ExecutionMode::LocalSize,
-                                             workgroupSizeAttr.asArrayRef());
+                                     spirv::ExecutionMode::LocalSize,
+                                     workgroupSizeAttr.asArrayRef());
       // Erase workgroup size.
       entryPointAttr = spirv::EntryPointABIAttr::get(
           entryPointAttr.getContext(), DenseI32ArrayAttr(),
@@ -168,8 +168,8 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
         spirv::getCapabilities(spirv::ExecutionMode::SubgroupSize);
     if (!caps || targetEnv.allows(*caps)) {
       spirv::ExecutionModeOp::create(builder, funcOp.getLoc(), funcOp,
-                                             spirv::ExecutionMode::SubgroupSize,
-                                             *subgroupSize);
+                                     spirv::ExecutionMode::SubgroupSize,
+                                     *subgroupSize);
       // Erase subgroup size.
       entryPointAttr = spirv::EntryPointABIAttr::get(
           entryPointAttr.getContext(), entryPointAttr.getWorkgroupSize(),
@@ -180,8 +180,8 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
     std::optional<ArrayRef<spirv::Capability>> caps =
         spirv::getCapabilities(spirv::ExecutionMode::SignedZeroInfNanPreserve);
     if (!caps || targetEnv.allows(*caps)) {
-      spirv::ExecutionModeOp::create(builder,
-          funcOp.getLoc(), funcOp,
+      spirv::ExecutionModeOp::create(
+          builder, funcOp.getLoc(), funcOp,
           spirv::ExecutionMode::SignedZeroInfNanPreserve, *targetWidth);
       // Erase target width.
       entryPointAttr = spirv::EntryPointABIAttr::get(
@@ -269,8 +269,8 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     if (cast<spirv::SPIRVType>(argType.value()).isScalarOrVector()) {
       auto zero =
           spirv::ConstantOp::getZero(indexType, funcOp.getLoc(), rewriter);
-      auto loadPtr = spirv::AccessChainOp::create(rewriter,
-          funcOp.getLoc(), replacement, zero.getConstant());
+      auto loadPtr = spirv::AccessChainOp::create(
+          rewriter, funcOp.getLoc(), replacement, zero.getConstant());
       replacement = spirv::LoadOp::create(rewriter, funcOp.getLoc(), loadPtr);
     }
     signatureConverter.remapInput(argType.index(), replacement);
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
index 8bb34de8988b8..38ef547f0769f 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
@@ -65,8 +65,8 @@ void RewriteInsertsPass::runOnOperation() {
       operands.push_back(insertionOp.getObject());
 
     OpBuilder builder(lastCompositeInsertOp);
-    auto compositeConstructOp = spirv::CompositeConstructOp::create(builder,
-        location, compositeType, operands);
+    auto compositeConstructOp = spirv::CompositeConstructOp::create(
+        builder, location, compositeType, operands);
 
     lastCompositeInsertOp.replaceAllUsesWith(
         compositeConstructOp->getResult(0));
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index d1fa4bbc208d7..35ec0190b5a61 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -669,21 +669,24 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
                               Location loc) {
   // We can only cast one value in SPIR-V.
   if (inputs.size() != 1) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   Value input = inputs.front();
 
   // Only support integer types for now. Floating point types to be implemented.
   if (!isa<IntegerType>(type)) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
   auto inputType = cast<IntegerType>(input.getType());
 
   auto scalarType = dyn_cast<spirv::ScalarType>(type);
   if (!scalarType) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -691,7 +694,8 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   // truncating to go back so we don't need to worry about the signedness.
   // For extension, we cannot have enough signal here to decide which op to use.
   if (inputType.getIntOrFloatBitWidth() < scalarType.getIntOrFloatBitWidth()) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -708,7 +712,8 @@ static Value castToSourceType(const spirv::TargetEnv &targetEnv,
   scalarType.getCapabilities(caps);
   if (failed(checkCapabilityRequirements(type, targetEnv, caps)) ||
       failed(checkExtensionRequirements(type, targetEnv, exts))) {
-    auto castOp = UnrealizedConversionCastOp::create(builder, loc, type, inputs);
+    auto castOp =
+        UnrealizedConversionCastOp::create(builder, loc, type, inputs);
     return castOp.getResult(0);
   }
 
@@ -843,7 +848,7 @@ getOrInsertPushConstantVariable(Location loc, Block &block,
   auto type = getPushConstantStorageType(elementCount, builder, indexType);
   const char *name = "__push_constant_var__";
   return spirv::GlobalVariableOp::create(builder, loc, type, name,
-                                                 /*initializer=*/nullptr);
+                                         /*initializer=*/nullptr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -879,8 +884,8 @@ struct FuncOpConversion final : OpConversionPattern<func::FuncOp> {
     }
 
     // Create the converted spirv.func op.
-    auto newFuncOp = spirv::FuncOp::create(rewriter,
-        funcOp.getLoc(), funcOp.getName(),
+    auto newFuncOp = spirv::FuncOp::create(
+        rewriter, funcOp.getLoc(), funcOp.getName(),
         rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
                                  resultType ? TypeRange(resultType)
                                             : TypeRange()));
@@ -920,7 +925,7 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
 
     // Create a new func op with the original type and copy the function body.
     auto newFuncOp = func::FuncOp::create(rewriter, funcOp.getLoc(),
-                                                   funcOp.getName(), fnType);
+                                          funcOp.getName(), fnType);
     rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
                                 newFuncOp.end());
 
@@ -954,8 +959,8 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto origVecType = dyn_cast<VectorType>(origType);
       if (!origVecType) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = arith::ConstantOp::create(rewriter,
-            loc, origType, rewriter.getZeroAttr(origType));
+        Value result = arith::ConstantOp::create(
+            rewriter, loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
         oneToNTypeMapping.addInputs(origInputNo, origType);
@@ -967,8 +972,8 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       auto targetShape = getTargetShape(origVecType);
       if (!targetShape) {
         // We need a placeholder for the old argument that will be erased later.
-        Value result = arith::ConstantOp::create(rewriter,
-            loc, origType, rewriter.getZeroAttr(origType));
+        Value result = arith::ConstantOp::create(
+            rewriter, loc, origType, rewriter.getZeroAttr(origType));
         rewriter.replaceAllUsesWith(newFuncOp.getArgument(origInputNo), result);
         tmpOps.insert({result.getDefiningOp(), newInputNo});
         oneToNTypeMapping.addInputs(origInputNo, origType);
@@ -982,12 +987,12 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
           llvm::to_vector_of<int64_t, 4>(origVecType.getShape());
 
       // Prepare the result vector.
-      Value result = arith::ConstantOp::create(rewriter,
-          loc, origVecType, rewriter.getZeroAttr(origVecType));
+      Value result = arith::ConstantOp::create(
+          rewriter, loc, origVecType, rewriter.getZeroAttr(origVecType));
       ++newOpCount;
       // Prepare the placeholder for the new arguments that will be added later.
-      Value dummy = arith::ConstantOp::create(rewriter,
-          loc, unrolledType, rewriter.getZeroAttr(unrolledType));
+      Value dummy = arith::ConstantOp::create(
+          rewriter, loc, unrolledType, rewriter.getZeroAttr(unrolledType));
       ++newOpCount;
 
       // Create the `vector.insert_strided_slice` ops.
@@ -995,8 +1000,8 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
       SmallVector<Type> newTypes;
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        result = vector::InsertStridedSliceOp::create(rewriter,
-            loc, dummy, result, offsets, strides);
+        result = vector::InsertStridedSliceOp::create(rewriter, loc, dummy,
+                                                      result, offsets, strides);
         newTypes.push_back(unrolledType);
         unrolledInputNums.push_back(newInputNo);
         ++newInputNo;
@@ -1109,8 +1114,8 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
       Value returnValue = returnOp.getOperand(origResultNo);
       for (SmallVector<int64_t> offsets :
            StaticTileOffsetRange(originalShape, *targetShape)) {
-        Value result = vector::ExtractStridedSliceOp::create(rewriter,
-            loc, returnValue, offsets, extractShape, strides);
+        Value result = vector::ExtractStridedSliceOp::create(
+            rewriter, loc, returnValue, offsets, extractShape, strides);
         if (originalShape.size() > 1) {
           SmallVector<int64_t> extractIndices(originalShape.size() - 1, 0);
           result =
@@ -1179,11 +1184,11 @@ Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
       loc, parent->getRegion(0).front(), elementCount, builder, integerType);
 
   Value zeroOp = spirv::ConstantOp::getZero(integerType, loc, builder);
-  Value offsetOp = spirv::ConstantOp::create(builder,
-      loc, integerType, builder.getI32IntegerAttr(offset));
+  Value offsetOp = spirv::ConstantOp::create(builder, loc, integerType,
+                                             builder.getI32IntegerAttr(offset));
   auto addrOp = spirv::AddressOfOp::create(builder, loc, varOp);
-  auto acOp = spirv::AccessChainOp::create(builder,
-      loc, addrOp, llvm::ArrayRef({zeroOp, offsetOp}));
+  auto acOp = spirv::AccessChainOp::create(builder, loc, addrOp,
+                                           llvm::ArrayRef({zeroOp, offsetOp}));
   return spirv::LoadOp::create(builder, loc, acOp);
 }
 
@@ -1276,10 +1281,10 @@ Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
   if (isa<spirv::ArrayType>(pointeeType)) {
     linearizedIndices.push_back(linearIndex);
     return spirv::AccessChainOp::create(builder, loc, basePtr,
-                                                linearizedIndices);
+                                        linearizedIndices);
   }
   return spirv::PtrAccessChainOp::create(builder, loc, basePtr, linearIndex,
-                                                 linearizedIndices);
+                                         linearizedIndices);
 }
 
 Value mlir::spirv::getElementPtr(const SPIRVTypeConverter &typeConverter,
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
index 17f79780afa20..e0900005ea1bb 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
@@ -64,14 +64,14 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   //     and 4 additions after constant folding.
   //   - With sign-extended arguments, we end up emitting 8 multiplications and
   //     and 12 additions after CSE.
-  Value cstLowMask = ConstantOp::create(rewriter,
-      loc, lhs.getType(), getScalarOrSplatAttr(argTy, (1 << 16) - 1));
+  Value cstLowMask = ConstantOp::create(
+      rewriter, loc, lhs.getType(), getScalarOrSplatAttr(argTy, (1 << 16) - 1));
   auto getLowDigit = [&rewriter, loc, cstLowMask](Value val) {
     return BitwiseAndOp::create(rewriter, loc, val, cstLowMask);
   };
 
   Value cst16 = ConstantOp::create(rewriter, loc, lhs.getType(),
-                                            getScalarOrSplatAttr(argTy, 16));
+                                   getScalarOrSplatAttr(argTy, 16));
   auto getHighDigit = [&rewriter, loc, cst16](Value val) {
     return ShiftRightLogicalOp::create(rewriter, loc, val, cst16);
   };
@@ -86,7 +86,7 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   };
 
   Value cst0 = ConstantOp::create(rewriter, loc, lhs.getType(),
-                                           getScalarOrSplatAttr(argTy, 0));
+                                  getScalarOrSplatAttr(argTy, 0));
 
   Value lhsLow = getLowDigit(lhs);
   Value lhsHigh = getHighDigit(lhs);
@@ -128,8 +128,9 @@ static Value lowerExtendedMultiplication(Operation *mulOp,
   Value low = combineDigits(resultDigits[0], resultDigits[1]);
   Value high = combineDigits(resultDigits[2], resultDigits[3]);
 
-  return CompositeConstructOp::create(rewriter,
-      loc, mulOp->getResultTypes().front(), llvm::ArrayRef({low, high}));
+  return CompositeConstructOp::create(rewriter, loc,
+                                      mulOp->getResultTypes().front(),
+                                      llvm::ArrayRef({low, high}));
 }
 
 //===----------------------------------------------------------------------===//
@@ -184,18 +185,19 @@ struct ExpandAddCarryPattern final : OpRewritePattern<IAddCarryOp> {
           loc,
           llvm::formatv("Unexpected integer type for WebGPU: '{0}'", elemTy));
 
-    Value one =
-        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 1));
-    Value zero =
-        ConstantOp::create(rewriter, loc, argTy, getScalarOrSplatAttr(argTy, 0));
+    Value one = ConstantOp::create(rewriter, loc, argTy,
+                                   getScalarOrSplatAttr(argTy, 1));
+    Value zero = ConstantOp::create(rewriter, loc, argTy,
+                                    getScalarOrSplatAttr(argTy, 0));
 
     // Calculate the carry by checking if the addition resulted in an overflow.
     Value out = IAddOp::create(rewriter, loc, lhs, rhs);
     Value cmp = ULessThanOp::create(rewriter, loc, out, lhs);
     Value carry = SelectOp::create(rewriter, loc, cmp, one, zero);
 
-    Value add = CompositeConstructOp::create(rewriter,
-        loc, op->getResultTypes().front(), llvm::ArrayRef({out, carry}));
+    Value add = CompositeConstructOp::create(rewriter, loc,
+                                             op->getResultTypes().front(),
+                                             llvm::ArrayRef({out, carry}));
 
     rewriter.replaceOp(op, add);
     return success();
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
index e7bb2d76666fb..692f2e7616e5a 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
@@ -380,13 +380,13 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = dstNumBytes / srcNumBytes;
-      auto ratioValue = spirv::ConstantOp::create(rewriter,
-          loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
+      auto ratioValue = spirv::ConstantOp::create(
+          rewriter, loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
           spirv::SDivOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
-      indices.push_back(
-          spirv::SModOp::create(rewriter, loc, indexType, oldIndex, ratioValue));
+      indices.push_back(spirv::SModOp::create(rewriter, loc, indexType,
+                                              oldIndex, ratioValue));
 
       rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
           acOp, adaptor.getBasePtr(), indices);
@@ -407,8 +407,8 @@ struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
       Type indexType = oldIndex.getType();
 
       int ratio = srcNumBytes / dstNumBytes;
-      auto ratioValue = spirv::ConstantOp::create(rewriter,
-          loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
+      auto ratioValue = spirv::ConstantOp::create(
+          rewriter, loc, indexType, rewriter.getIntegerAttr(indexType, ratio));
 
       indices.back() =
           spirv::IMulOp::create(rewriter, loc, indexType, oldIndex, ratioValue);
@@ -443,7 +443,7 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
 
     if (areSameBitwidthScalarType(srcElemType, dstElemType)) {
       auto castOp = spirv::BitcastOp::create(rewriter, loc, srcElemType,
-                                                      newLoadOp.getValue());
+                                             newLoadOp.getValue());
       rewriter.replaceOp(loadOp, castOp->getResults());
 
       return success();
@@ -475,10 +475,10 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
       auto indices = llvm::to_vector<4>(acOp.getIndices());
       for (int i = 1; i < ratio; ++i) {
         // Load all subsequent components belonging to this element.
-        indices.back() = spirv::IAddOp::create(rewriter,
-            loc, i32Type, indices.back(), oneValue);
-        auto componentAcOp = spirv::AccessChainOp::create(rewriter,
-            loc, acOp.getBasePtr(), indices);
+        indices.back() = spirv::IAddOp::create(rewriter, loc, i32Type,
+                                               indices.back(), oneValue);
+        auto componentAcOp = spirv::AccessChainOp::create(
+            rewriter, loc, acOp.getBasePtr(), indices);
         // Assuming little endian, this reads lower-ordered bits of the number
         // to lower-numbered components of the vector.
         components.push_back(
@@ -513,8 +513,8 @@ struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
               c = spirv::BitcastOp::create(rewriter, loc, castType, c);
           }
         }
-      Value vectorValue = spirv::CompositeConstructOp::create(rewriter,
-          loc, vectorType, components);
+      Value vectorValue = spirv::CompositeConstructOp::create(
+          rewriter, loc, vectorType, components);
 
       if (!isa<VectorType>(srcElemType))
         vectorValue =
diff --git a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
index 5533a7e914bf0..301f943559409 100644
--- a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -57,8 +57,8 @@ struct AssumingOpInterface
 
     // Create new op and move over region.
     TypeRange newResultTypes(yieldOp.getOperands());
-    auto newOp = shape::AssumingOp::create(rewriter,
-        op->getLoc(), newResultTypes, assumingOp.getWitness());
+    auto newOp = shape::AssumingOp::create(
+        rewriter, op->getLoc(), newResultTypes, assumingOp.getWitness());
     newOp.getDoRegion().takeBody(assumingOp.getRegion());
 
     // Update all uses of the old op.
@@ -66,8 +66,9 @@ struct AssumingOpInterface
     SmallVector<Value> newResults;
     for (const auto &it : llvm::enumerate(assumingOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
-        newResults.push_back(bufferization::ToTensorOp::create(rewriter,
-            assumingOp.getLoc(), it.value(), newOp->getResult(it.index())));
+        newResults.push_back(bufferization::ToTensorOp::create(
+            rewriter, assumingOp.getLoc(), it.value(),
+            newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
       }
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index b5f6230bc2362..9e9cf62c57fa9 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -50,7 +50,7 @@ NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
   Block *body = reduce.getBody();
   OpBuilder b = OpBuilder::atBlockEnd(body);
   Value product = MulOp::create(b, loc, valueType, body->getArgument(1),
-                                  body->getArgument(2));
+                                body->getArgument(2));
   shape::YieldOp::create(b, loc, product);
 
   rewriter.replaceOp(op, reduce.getResult());
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
index 74842cc1f310b..abb37a5e10b9a 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
@@ -41,8 +41,8 @@ LogicalResult sparse_tensor::detail::stageWithSortImpl(
 
   // -> sort
   Type dstCOOTp = dstStt.getCOOType(/*ordered=*/true);
-  Value dstCOO = ReorderCOOOp::create(rewriter,
-      loc, dstCOOTp, srcCOO, SparseTensorSortKind::HybridQuickSort);
+  Value dstCOO = ReorderCOOOp::create(rewriter, loc, dstCOOTp, srcCOO,
+                                      SparseTensorSortKind::HybridQuickSort);
 
   // -> dest.
   if (dstCOO.getType() == finalTp) {
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
index 6545bfbcac5c7..40c182f9dbb37 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp
@@ -89,10 +89,10 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types,
           Value mem;
           if (kind == SparseTensorFieldKind::PosMemRef)
             mem = sparse_tensor::ToPositionsOp::create(builder, loc, inputs[0],
-                                                               lv);
+                                                       lv);
           else if (kind == SparseTensorFieldKind::CrdMemRef)
-            mem = sparse_tensor::ToCoordinatesOp::create(builder, loc, inputs[0],
-                                                                 lv);
+            mem = sparse_tensor::ToCoordinatesOp::create(builder, loc,
+                                                         inputs[0], lv);
           else
             mem = sparse_tensor::ToValuesOp::create(builder, loc, inputs[0]);
           toVals.push_back(mem);
@@ -199,8 +199,9 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     OpBuilder moduleBuilder(modOp.getBodyRegion());
     unsigned extra = inputTypes.size();
     inputTypes.append(extraTypes);
-    auto func = func::FuncOp::create(moduleBuilder,
-        loc, orgName, FunctionType::get(context, inputTypes, outputTypes));
+    auto func = func::FuncOp::create(
+        moduleBuilder, loc, orgName,
+        FunctionType::get(context, inputTypes, outputTypes));
     func.setPublic();
 
     // Construct new wrapper method body.
@@ -216,8 +217,8 @@ struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
     // Call the original, now private method. A subsequent inlining pass can
     // determine whether cloning the method body in place is worthwhile.
     auto org = SymbolRefAttr::get(context, wrapper);
-    auto call = func::CallOp::create(rewriter, loc, funcOp.getResultTypes(), org,
-                                              inputs);
+    auto call = func::CallOp::create(rewriter, loc, funcOp.getResultTypes(),
+                                     org, inputs);
 
     // Convert outputs and return.
     SmallVector<Value> outputs;
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
index 12507ff1c5251..02623198c25b5 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
@@ -94,8 +94,8 @@ static FlatSymbolRefAttr getMangledSortHelperFunc(
     OpBuilder::InsertionGuard insertionGuard(builder);
     builder.setInsertionPoint(insertPoint);
     Location loc = insertPoint.getLoc();
-    func = func::FuncOp::create(builder,
-        loc, nameOstream.str(),
+    func = func::FuncOp::create(
+        builder, loc, nameOstream.str(),
         FunctionType::get(context, operands.getTypes(), resultTypes));
     func.setPrivate();
     createFunc(builder, module, func, xPerm, ny, nTrailingP);
@@ -221,7 +221,7 @@ static Value createEqCompare(OpBuilder &builder, Location loc, Value i, Value j,
     Value ne =
         arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
     scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
-                                               ne, /*else=*/true);
+                                       ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value f = constantI1(builder, loc, false);
@@ -266,7 +266,8 @@ static Value createLessThanCompare(OpBuilder &builder, Location loc, Value i,
 
   Value res;
   if (isLastDim) {
-    res = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
+    res =
+        arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult, vi, vj);
     // For 1D, we create a compare without any control flow. Otherwise, we
     // create YieldOp to return the result in the nested if-stmt.
     if (!isFirstDim)
@@ -275,7 +276,7 @@ static Value createLessThanCompare(OpBuilder &builder, Location loc, Value i,
     Value ne =
         arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, vi, vj);
     scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIntegerType(1),
-                                               ne, /*else=*/true);
+                                       ne, /*else=*/true);
     // If (x[i] != x[j]).
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     Value lt =
@@ -337,16 +338,16 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   ValueRange args = entryBlock->getArguments();
   Value p = args[hiIdx];
   SmallVector<Type, 2> types(2, p.getType()); // Only two types.
-  scf::WhileOp whileOp = scf::WhileOp::create(builder,
-      loc, types, SmallVector<Value, 2>{args[loIdx], args[hiIdx]});
+  scf::WhileOp whileOp = scf::WhileOp::create(
+      builder, loc, types, SmallVector<Value, 2>{args[loIdx], args[hiIdx]});
 
   // The before-region of the WhileOp.
   Block *before =
       builder.createBlock(&whileOp.getBefore(), {}, types, {loc, loc});
   builder.setInsertionPointToEnd(before);
-  Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
-                                              before->getArgument(0),
-                                              before->getArgument(1));
+  Value cond1 =
+      arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
+                            before->getArgument(0), before->getArgument(1));
   scf::ConditionOp::create(builder, loc, cond1, before->getArguments());
 
   // The after-region of the WhileOp.
@@ -357,8 +358,8 @@ static void createBinarySearchFunc(OpBuilder &builder, ModuleOp module,
   Value hi = after->getArgument(1);
   // Compute mid = (lo + hi) >> 1.
   Value c1 = constantIndex(builder, loc, 1);
-  Value mid = arith::ShRUIOp::create(builder,
-      loc, arith::AddIOp::create(builder, loc, lo, hi), c1);
+  Value mid = arith::ShRUIOp::create(
+      builder, loc, arith::AddIOp::create(builder, loc, lo, hi), c1);
   Value midp1 = arith::AddIOp::create(builder, loc, mid, c1);
 
   // Compare xs[p] < xs[mid].
@@ -521,7 +522,7 @@ static void createChoosePivot(OpBuilder &builder, ModuleOp module,
   Value len = arith::SubIOp::create(builder, loc, hiP1, lo);
   Value lenThreshold = constantIndex(builder, loc, 1000);
   Value lenCond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
-                                                len, lenThreshold);
+                                        len, lenThreshold);
   scf::IfOp lenIf = scf::IfOp::create(builder, loc, lenCond, /*else=*/true);
 
   // When len < 1000, choose pivot from median of 3 values.
@@ -607,7 +608,7 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
                                       {loc, loc, loc, loc});
   builder.setInsertionPointToEnd(before);
   scf::ConditionOp::create(builder, loc, before->getArgument(3),
-                                   before->getArguments());
+                           before->getArguments());
 
   // The after-region of the WhileOp.
   Block *after =
@@ -639,14 +640,14 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   Value icond =
       arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, i, p);
   scf::IfOp ifOpI = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
-                                              icond, /*else=*/true);
+                                      icond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpI.getThenRegion().front());
   scf::YieldOp::create(builder, loc, ValueRange{j});
   builder.setInsertionPointToStart(&ifOpI.getElseRegion().front());
   Value jcond =
       arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, j, p);
   scf::IfOp ifOpJ = scf::IfOp::create(builder, loc, TypeRange{p.getType()},
-                                              jcond, /*else=*/true);
+                                      jcond, /*else=*/true);
   builder.setInsertionPointToStart(&ifOpJ.getThenRegion().front());
   scf::YieldOp::create(builder, loc, ValueRange{i});
   builder.setInsertionPointToStart(&ifOpJ.getElseRegion().front());
@@ -656,8 +657,9 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   builder.setInsertionPointAfter(ifOpI);
   Value compareEqIJ =
       arith::AndIOp::create(builder, loc, iCompareEq, jCompareEq);
-  scf::IfOp ifOp2 = scf::IfOp::create(builder,
-      loc, TypeRange{i.getType(), j.getType()}, compareEqIJ, /*else=*/true);
+  scf::IfOp ifOp2 =
+      scf::IfOp::create(builder, loc, TypeRange{i.getType(), j.getType()},
+                        compareEqIJ, /*else=*/true);
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
   Value i2 = arith::AddIOp::create(builder, loc, i, c1);
   Value j2 = arith::SubIOp::create(builder, loc, j, c1);
@@ -665,17 +667,18 @@ static void createPartitionFunc(OpBuilder &builder, ModuleOp module,
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
   scf::YieldOp::create(builder, loc, ValueRange{i, j});
   builder.setInsertionPointAfter(ifOp2);
-  scf::YieldOp::create(builder,
-      loc,
-      ValueRange{ifOp2.getResult(0), ifOp2.getResult(1), ifOpI.getResult(0),
-                 /*cont=*/constantI1(builder, loc, true)});
+  scf::YieldOp::create(builder, loc,
+                       ValueRange{ifOp2.getResult(0), ifOp2.getResult(1),
+                                  ifOpI.getResult(0),
+                                  /*cont=*/constantI1(builder, loc, true)});
 
   // False branch for if i < j (i.e., i >= j):
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
   p = arith::AddIOp::create(builder, loc, j,
-                                    constantOne(builder, loc, j.getType()));
-  scf::YieldOp::create(builder,
-      loc, ValueRange{i, j, p, /*cont=*/constantI1(builder, loc, false)});
+                            constantOne(builder, loc, j.getType()));
+  scf::YieldOp::create(
+      builder, loc,
+      ValueRange{i, j, p, /*cont=*/constantI1(builder, loc, false)});
 
   // Return for the whileOp.
   builder.setInsertionPointAfter(ifOp);
@@ -773,7 +776,7 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
     Value lChildIdx = arith::AddIOp::create(builder, loc, lChild, first);
     Value rChild = arith::AddIOp::create(builder, loc, lChild, c1);
     Value cond1 = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
-                                                rChild, n);
+                                        rChild, n);
     SmallVector<Type, 2> ifTypes(2, r.getType());
     scf::IfOp if1 =
         scf::IfOp::create(builder, loc, ifTypes, cond1, /*else=*/true);
@@ -803,8 +806,8 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
 
   // While (data[start] < data[childIndex]).
   SmallVector<Type, 3> types(3, child.getType());
-  scf::WhileOp whileOp = scf::WhileOp::create(builder,
-      loc, types, SmallVector<Value, 2>{start, child, childIdx});
+  scf::WhileOp whileOp = scf::WhileOp::create(
+      builder, loc, types, SmallVector<Value, 2>{start, child, childIdx});
 
   // The before-region of the WhileOp.
   SmallVector<Location, 3> locs(3, loc);
@@ -828,16 +831,17 @@ static void createShiftDownFunc(OpBuilder &builder, ModuleOp module,
   start = childIdx;
   Value cond2 =
       arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::uge, t, child);
-  scf::IfOp if2 = scf::IfOp::create(builder,
-      loc, TypeRange{child.getType(), child.getType()}, cond2, /*else=*/true);
+  scf::IfOp if2 = scf::IfOp::create(builder, loc,
+                                    TypeRange{child.getType(), child.getType()},
+                                    cond2, /*else=*/true);
   builder.setInsertionPointToStart(&if2.getThenRegion().front());
   auto [newChild, newChildIdx] = getLargerChild(child);
   scf::YieldOp::create(builder, loc, ValueRange{newChild, newChildIdx});
   builder.setInsertionPointToStart(&if2.getElseRegion().front());
   scf::YieldOp::create(builder, loc, ValueRange{child, childIdx});
   builder.setInsertionPointAfter(if2);
-  scf::YieldOp::create(builder,
-      loc, ValueRange{start, if2.getResult(0), if2.getResult(1)});
+  scf::YieldOp::create(builder, loc,
+                       ValueRange{start, if2.getResult(0), if2.getResult(1)});
 
   builder.setInsertionPointAfter(ifN);
   func::ReturnOp::create(builder, loc);
@@ -888,7 +892,7 @@ static void createHeapSortFunc(OpBuilder &builder, ModuleOp module,
       builder, func, TypeRange(), kShiftDownFuncNamePrefix, xPerm, ny,
       shiftDownOperands, createShiftDownFunc, /*nTrailingP=*/1);
   func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
-                               shiftDownOperands);
+                       shiftDownOperands);
 
   builder.setInsertionPointAfter(forI);
   // For l = n downto 2.
@@ -905,7 +909,7 @@ static void createHeapSortFunc(OpBuilder &builder, ModuleOp module,
   shiftDownOperands[shiftDownOperands.size() - 1] =
       arith::SubIOp::create(builder, loc, l, c1);
   func::CallOp::create(builder, loc, shiftDownFunc, TypeRange(),
-                               shiftDownOperands);
+                       shiftDownOperands);
 
   builder.setInsertionPointAfter(forL);
   func::ReturnOp::create(builder, loc);
@@ -949,7 +953,7 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func,
   // Else len > 2, need recursion.
   builder.setInsertionPointToStart(&ifLenGtTwo.getThenRegion().front());
   Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
-                                             lenLow, lenHigh);
+                                     lenLow, lenHigh);
 
   Value c0 = constantIndex(builder, loc, 0);
   scf::IfOp ifOp = scf::IfOp::create(builder, loc, types, cond, /*else=*/true);
@@ -1153,8 +1157,8 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
   if (isHybrid) {
     Value len = arith::SubIOp::create(builder, loc, hi, lo);
     Value lenLimit = constantIndex(builder, loc, 30);
-    Value lenCond = arith::CmpIOp::create(builder,
-        loc, arith::CmpIPredicate::ule, len, lenLimit);
+    Value lenCond = arith::CmpIOp::create(
+        builder, loc, arith::CmpIPredicate::ule, len, lenLimit);
     scf::IfOp lenIf =
         scf::IfOp::create(builder, loc, types, lenCond, /*else=*/true);
 
@@ -1164,17 +1168,17 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
         builder, func, TypeRange(), kSortStableFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createSortStableFunc);
     func::CallOp::create(builder, loc, insertionSortFunc, TypeRange(),
-                                 ValueRange(args).drop_back(nTrailingP));
+                         ValueRange(args).drop_back(nTrailingP));
     scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When len > limit.
     builder.setInsertionPointToStart(&lenIf.getElseRegion().front());
     Value depthLimit = args.back();
     depthLimit = arith::SubIOp::create(builder, loc, depthLimit,
-                                               constantI64(builder, loc, 1));
+                                       constantI64(builder, loc, 1));
     Value depthCond =
         arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ule,
-                                      depthLimit, constantI64(builder, loc, 0));
+                              depthLimit, constantI64(builder, loc, 0));
     scf::IfOp depthIf =
         scf::IfOp::create(builder, loc, types, depthCond, /*else=*/true);
 
@@ -1184,7 +1188,7 @@ static void createQuickSortFunc(OpBuilder &builder, ModuleOp module,
         builder, func, TypeRange(), kHeapSortFuncNamePrefix, xPerm, ny,
         ValueRange(args).drop_back(nTrailingP), createHeapSortFunc);
     func::CallOp::create(builder, loc, heapSortFunc, TypeRange(),
-                                 ValueRange(args).drop_back(nTrailingP));
+                         ValueRange(args).drop_back(nTrailingP));
     scf::YieldOp::create(builder, loc, ValueRange{lo, lo});
 
     // When depth doesn't exceed limit.
@@ -1248,11 +1252,11 @@ LogicalResult matchAndRewriteSortOp(OpTy op, ValueRange xys, AffineMap xPerm,
     // As a heuristics, set depthLimit = 2 * log2(n).
     Value lo = operands[loIdx];
     Value hi = operands[hiIdx];
-    Value len = arith::IndexCastOp::create(rewriter,
-        loc, rewriter.getI64Type(),
+    Value len = arith::IndexCastOp::create(
+        rewriter, loc, rewriter.getI64Type(),
         arith::SubIOp::create(rewriter, loc, hi, lo));
-    Value depthLimit = arith::SubIOp::create(rewriter,
-        loc, constantI64(rewriter, loc, 64),
+    Value depthLimit = arith::SubIOp::create(
+        rewriter, loc, constantI64(rewriter, loc, 64),
         math::CountLeadingZerosOp::create(rewriter, loc, len));
     operands.push_back(depthLimit);
     break;
@@ -1317,14 +1321,14 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
     bool nIsOne = (nValue && nValue.value() == 1);
 
     if (!op.getInbounds()) {
-      Value cond = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::ugt, newSize, capacity);
+      Value cond = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::ugt, newSize, capacity);
 
       Value c2 = constantIndex(rewriter, loc, 2);
       auto bufferType =
           MemRefType::get({ShapedType::kDynamic}, value.getType());
       scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, bufferType, cond,
-                                                  /*else=*/true);
+                                         /*else=*/true);
       // True branch.
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       if (nIsOne) {
@@ -1343,7 +1347,7 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
         capacity =
             arith::MulIOp::create(rewriter, loc, before->getArgument(0), c2);
         cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ugt,
-                                              newSize, capacity);
+                                     newSize, capacity);
         scf::ConditionOp::create(rewriter, loc, cond, ValueRange{capacity});
         // The after-region of the WhileOp.
         Block *after = rewriter.createBlock(&whileOp.getAfter(), {},
@@ -1355,13 +1359,14 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
         capacity = whileOp.getResult(0);
       }
 
-      Value newBuffer =
-          memref::ReallocOp::create(rewriter, loc, bufferType, buffer, capacity);
+      Value newBuffer = memref::ReallocOp::create(rewriter, loc, bufferType,
+                                                  buffer, capacity);
       if (enableBufferInitialization) {
-        Value fillSize = arith::SubIOp::create(rewriter, loc, capacity, newSize);
+        Value fillSize =
+            arith::SubIOp::create(rewriter, loc, capacity, newSize);
         Value fillValue = constantZero(rewriter, loc, value.getType());
-        Value subBuffer = memref::SubViewOp::create(rewriter,
-            loc, newBuffer, /*offset=*/ValueRange{newSize},
+        Value subBuffer = memref::SubViewOp::create(
+            rewriter, loc, newBuffer, /*offset=*/ValueRange{newSize},
             /*size=*/ValueRange{fillSize},
             /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
         linalg::FillOp::create(rewriter, loc, fillValue, subBuffer);
@@ -1381,8 +1386,9 @@ struct PushBackRewriter : OpRewritePattern<PushBackOp> {
     if (nIsOne) {
       memref::StoreOp::create(rewriter, loc, value, buffer, size);
     } else {
-      Value subBuffer = memref::SubViewOp::create(rewriter,
-          loc, buffer, /*offset=*/ValueRange{size}, /*size=*/ValueRange{n},
+      Value subBuffer = memref::SubViewOp::create(
+          rewriter, loc, buffer, /*offset=*/ValueRange{size},
+          /*size=*/ValueRange{n},
           /*step=*/ValueRange{constantIndex(rewriter, loc, 1)});
       linalg::FillOp::create(rewriter, loc, value, subBuffer);
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index 001754b5531be..a317abd6c560b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -60,7 +60,7 @@ static gpu::GPUModuleOp genGPUModule(OpBuilder &builder, ModuleOp topModule) {
   markAsGPUContainer(topModule);
   builder.setInsertionPointToStart(topModule.getBody());
   return gpu::GPUModuleOp::create(builder, topModule->getLoc(),
-                                          "sparse_kernels");
+                                  "sparse_kernels");
 }
 
 /// Constructs a new GPU kernel in the given GPU module.
@@ -157,7 +157,7 @@ static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem,
     }
   }
   return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
-                                      token, dynamicSizes, ValueRange());
+                              token, dynamicSizes, ValueRange());
 }
 
 // Allocates a typed buffer on the host with given size.
@@ -172,7 +172,7 @@ static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Type type,
                                    Value size, Value token) {
   const auto memTp = MemRefType::get({ShapedType::kDynamic}, type);
   return gpu::AllocOp::create(builder, loc, TypeRange({memTp, token.getType()}),
-                                      token, size, ValueRange());
+                              token, size, ValueRange());
 }
 
 // Allocates a void buffer on the device with given size.
@@ -524,12 +524,12 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
     if (enableRT) {
       assert(colA);
       return gpu::CreateCooOp::create(builder, loc, handleTp, tokenTp, token,
-                                              sz1, sz2, nseA, rowA, colA, valA);
+                                      sz1, sz2, nseA, rowA, colA, valA);
     }
 #ifdef CUSPARSE_COO_AOS
     assert(!colA);
     return gpu::CreateCooAoSOp::create(builder, loc, handleTp, tokenTp, token,
-                                               sz1, sz2, nseA, rowA, valA);
+                                       sz1, sz2, nseA, rowA, valA);
 #else
     llvm_unreachable("gpu::CreateCooAoSOp is deprecated");
 #endif
@@ -537,10 +537,10 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   assert(colA);
   if (format == CuSparseFormat::kCSR)
     return gpu::CreateCsrOp::create(builder, loc, handleTp, tokenTp, token, sz1,
-                                            sz2, nseA, rowA, colA, valA);
+                                    sz2, nseA, rowA, colA, valA);
   if (format == CuSparseFormat::kCSC)
     return gpu::CreateCscOp::create(builder, loc, handleTp, tokenTp, token, sz1,
-                                            sz2, nseA, rowA, colA, valA);
+                                    sz2, nseA, rowA, colA, valA);
   // BSR requires a bit more work since we need to pass in the block size
   // and all others sizes in terms of blocks (#block-rows, #block-cols,
   // #nonzero-blocks).
@@ -551,11 +551,10 @@ static Operation *genSpMat(OpBuilder &builder, Location loc,
   Value bSz = constantIndex(builder, loc, b);
   Value bRows = arith::DivUIOp::create(builder, loc, sz1, bSz);
   Value bCols = arith::DivUIOp::create(builder, loc, sz2, bSz);
-  Value bNum = arith::DivUIOp::create(builder,
-      loc, nseA, constantIndex(builder, loc, b * b));
+  Value bNum = arith::DivUIOp::create(builder, loc, nseA,
+                                      constantIndex(builder, loc, b * b));
   return gpu::CreateBsrOp::create(builder, loc, handleTp, tokenTp, token, bRows,
-                                          bCols, bNum, bSz, bSz, rowA, colA,
-                                          valA);
+                                  bCols, bNum, bSz, bSz, rowA, colA, valA);
 }
 
 /// Match and rewrite SpMV kernel.
@@ -606,19 +605,19 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dvecX = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, vecX, szX);
+  auto dvecX = gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp,
+                                             tokenTp, token, vecX, szX);
   Value dnX = dvecX.getResult(0);
   token = dvecX.getAsyncToken();
-  auto dvecY = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, vecY, szY);
+  auto dvecY = gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp,
+                                             tokenTp, token, vecY, szY);
   Value dnY = dvecY.getResult(0);
   token = dvecY.getAsyncToken();
   auto dnYType = llvm::cast<ShapedType>(y.getType()).getElementType();
 
   // Precompute buffersize for SpMV.
-  auto bufferComp = gpu::SpMVBufferSizeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, spMatA, dnX, dnY,
+  auto bufferComp = gpu::SpMVBufferSizeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, spMatA, dnX, dnY,
       /*computeType=*/dnYType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
@@ -627,8 +626,9 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   token = buf.getAsyncToken();
 
   // Perform the SpMV.
-  auto spmvComp = gpu::SpMVOp::create(rewriter,
-      loc, tokenTp, token, spMatA, dnX, dnY, /*computeType=*/dnYType, buffer);
+  auto spmvComp =
+      gpu::SpMVOp::create(rewriter, loc, tokenTp, token, spMatA, dnX, dnY,
+                          /*computeType=*/dnYType, buffer);
   token = spmvComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
@@ -704,21 +704,21 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
                nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, matB,
-      SmallVector<Value>{szk, szn});
+  auto dmatB =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp, tokenTp,
+                                    token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, matC,
-      SmallVector<Value>{szm, szn});
+  auto dmatC =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp, tokenTp,
+                                    token, matC, SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
   token = dmatC.getAsyncToken();
   auto dmatCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SpMM.
-  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, spMatA, dnB, dnC,
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
@@ -728,8 +728,9 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Perform the SpMM.
-  auto spmmComp = gpu::SpMMOp::create(rewriter,
-      loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType, buffer);
+  auto spmmComp =
+      gpu::SpMMOp::create(rewriter, loc, tokenTp, token, spMatA, dnB, dnC,
+                          /*computeType=*/dnCType, buffer);
   token = spmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
@@ -839,41 +840,44 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
       gpu::SpGEMMCreateDescrOp::create(rewriter, loc, descTp, tokenTp, token);
   Value desc = descOp->getResult(0);
   token = descOp->getResult(1);
-  Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
-      valC, gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
+  Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, desc,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, spMatB, spMatC, dnCType, zero, valC,
+      gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
   Value bufferSz1 = work1->getResult(0);
   token = work1->getResult(1);
   auto buf1 = genAllocBuffer(rewriter, loc, bufferSz1, token);
   Value buffer1 = buf1.getResult(0);
   token = buf1.getAsyncToken();
-  Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
-      bufferSz1, buffer1,
+  Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, desc,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, spMatB, spMatC, dnCType, bufferSz1, buffer1,
       gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION);
   token = work2->getResult(1);
 
   // Compute step.
-  Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType, zero,
-      valC, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
+  Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, desc,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, spMatB, spMatC, dnCType, zero, valC,
+      gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
   Value bufferSz2 = compute1->getResult(0);
   token = compute1->getResult(1);
   auto buf2 = genAllocBuffer(rewriter, loc, bufferSz2, token);
   Value buffer2 = buf2.getResult(0);
   token = buf2.getAsyncToken();
-  Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType,
-      bufferSz2, buffer2, gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
+  Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, desc,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, spMatB, spMatC, dnCType, bufferSz2, buffer2,
+      gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE);
   token = compute2->getResult(1);
 
   // Get sizes.
-  Operation *sizes = gpu::SpMatGetSizeOp::create(rewriter,
-      loc, indexTp, indexTp, indexTp, tokenTp, token, spMatC);
+  Operation *sizes = gpu::SpMatGetSizeOp::create(
+      rewriter, loc, indexTp, indexTp, indexTp, tokenTp, token, spMatC);
   Value nnz = sizes->getResult(2);
   token = sizes->getResult(3);
   auto a2 = genAllocBuffer(rewriter, loc, cTp.getCrdType(), nnz, token);
@@ -884,11 +888,11 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   token = a3.getAsyncToken();
 
   // Update C with new pointers and copy final product back into C.
-  Operation *update = gpu::SetCsrPointersOp::create(rewriter,
-      loc, tokenTp, token, spMatC, rowC, colC, valC);
+  Operation *update = gpu::SetCsrPointersOp::create(
+      rewriter, loc, tokenTp, token, spMatC, rowC, colC, valC);
   token = update->getResult(0);
-  Operation *copy = gpu::SpGEMMCopyOp::create(rewriter,
-      loc, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
+  Operation *copy = gpu::SpGEMMCopyOp::create(
+      rewriter, loc, tokenTp, token, desc, gpu::TransposeMode::NON_TRANSPOSE,
       gpu::TransposeMode::NON_TRANSPOSE, spMatA, spMatB, spMatC, dnCType);
   token = copy->getResult(0);
 
@@ -925,12 +929,12 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  Value vt = bufferization::ToTensorOp::create(rewriter,
-      loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
-  Value rt = bufferization::ToTensorOp::create(rewriter,
-      loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
-  Value ct = bufferization::ToTensorOp::create(rewriter,
-      loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
+  Value vt = bufferization::ToTensorOp::create(
+      rewriter, loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
+  Value rt = bufferization::ToTensorOp::create(
+      rewriter, loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
+  Value ct = bufferization::ToTensorOp::create(
+      rewriter, loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
   rewriter.replaceOpWithNewOp<AssembleOp>(op, c.getType(), ValueRange{rt, ct},
                                           vt);
   return success();
@@ -980,19 +984,19 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  Operation *spGenA = gpu::Create2To4SpMatOp::create(rewriter,
-      loc, spMatHandleTp, tokenTp, token, szm, szk,
+  Operation *spGenA = gpu::Create2To4SpMatOp::create(
+      rewriter, loc, spMatHandleTp, tokenTp, token, szm, szk,
       gpu::Prune2To4SpMatFlag::PRUNE_AND_CHECK, matA);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, matB,
-      SmallVector<Value>{szk, szn});
+  auto dmatB =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp, tokenTp,
+                                    token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnTensorHandleTp, tokenTp, token, matC,
-      SmallVector<Value>{szm, szn});
+  auto dmatC =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnTensorHandleTp, tokenTp,
+                                    token, matC, SmallVector<Value>{szm, szn});
   Value dnC = dmatC.getResult(0);
   token = dmatC.getAsyncToken();
   auto dmatCType = llvm::cast<ShapedType>(matC.getType()).getElementType();
@@ -1000,9 +1004,10 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   // Precompute buffersize for SpMM.
   SmallVector<Type> bufferTypes_{indexTp, indexTp, indexTp};
   TypeRange bufferTypes(bufferTypes_);
-  auto bufferComp = gpu::SpMMBufferSizeOp::create(rewriter,
-      loc, bufferTypes, tokenTp, token, gpu::TransposeMode::NON_TRANSPOSE,
-      gpu::TransposeMode::NON_TRANSPOSE, spMatA, dnB, dnC,
+  auto bufferComp = gpu::SpMMBufferSizeOp::create(
+      rewriter, loc, bufferTypes, tokenTp, token,
+      gpu::TransposeMode::NON_TRANSPOSE, gpu::TransposeMode::NON_TRANSPOSE,
+      spMatA, dnB, dnC,
       /*computeType=*/dmatCType);
   token = bufferComp.getAsyncToken();
 
@@ -1022,8 +1027,8 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
 
   // Perform the SpMM.
   auto dnCType = llvm::cast<ShapedType>(matC.getType()).getElementType();
-  auto spmmComp = gpu::SpMMOp::create(rewriter,
-      loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType,
+  auto spmmComp = gpu::SpMMOp::create(
+      rewriter, loc, tokenTp, token, spMatA, dnB, dnC, /*computeType=*/dnCType,
       SmallVector<Value>{buffer1, buffer2, buffer3});
   token = spmmComp.getAsyncToken();
 
@@ -1096,12 +1101,14 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>();
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
-  auto dmatA = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnMatHandleTp, tokenTp, token, matA, SmallVector<Value>{szm, szk});
+  auto dmatA =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnMatHandleTp, tokenTp,
+                                    token, matA, SmallVector<Value>{szm, szk});
   Value dnA = dmatA.getResult(0);
   token = dmatA.getAsyncToken();
-  auto dmatB = gpu::CreateDnTensorOp::create(rewriter,
-      loc, dnMatHandleTp, tokenTp, token, matB, SmallVector<Value>{szk, szn});
+  auto dmatB =
+      gpu::CreateDnTensorOp::create(rewriter, loc, dnMatHandleTp, tokenTp,
+                                    token, matB, SmallVector<Value>{szk, szn});
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
   Operation *spGenC =
@@ -1112,8 +1119,8 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SDDMM.
-  auto bufferComp = gpu::SDDMMBufferSizeOp::create(rewriter,
-      loc, indexTp, tokenTp, token, dnA, dnB, spMatC, dnCType);
+  auto bufferComp = gpu::SDDMMBufferSizeOp::create(
+      rewriter, loc, indexTp, tokenTp, token, dnA, dnB, spMatC, dnCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
   auto buf = genAllocBuffer(rewriter, loc, bufferSz, token);
@@ -1122,7 +1129,7 @@ static LogicalResult rewriteSDDMM(PatternRewriter &rewriter,
 
   // Perform the SDDMM.
   auto sddmmComp = gpu::SDDMMOp::create(rewriter, loc, tokenTp, token, dnA, dnB,
-                                                 spMatC, dnCType, buffer);
+                                        spMatC, dnCType, buffer);
   token = sddmmComp.getAsyncToken();
 
   // Copy data back to host and free all the resoures.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
index 0986a0f2ea0f0..dfb127444e281 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseIterationToScf.cpp
@@ -68,11 +68,11 @@ genCoIterateBranchNest(PatternRewriter &rewriter, Location loc, CoIterateOp op,
   for (unsigned i : caseBits.bits()) {
     SparseIterator *it = iters[i].get();
     Value pred = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
-                                                it->getCrd(), loopCrd);
+                                       it->getCrd(), loopCrd);
     casePred = arith::AndIOp::create(rewriter, loc, casePred, pred);
   }
-  scf::IfOp ifOp = scf::IfOp::create(rewriter,
-      loc, ValueRange(userReduc).getTypes(), casePred, /*else=*/true);
+  scf::IfOp ifOp = scf::IfOp::create(
+      rewriter, loc, ValueRange(userReduc).getTypes(), casePred, /*else=*/true);
   rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
   // Erase the empty block.
@@ -127,8 +127,8 @@ static ValueRange genLoopWithIterator(
   if (it->iteratableByFor()) {
     auto [lo, hi] = it->genForCond(rewriter, loc);
     Value step = constantIndex(rewriter, loc, 1);
-    scf::ForOp forOp = scf::ForOp::create(rewriter,
-        loc, lo, hi, step, reduc,
+    scf::ForOp forOp = scf::ForOp::create(
+        rewriter, loc, lo, hi, step, reduc,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
           // Empty builder function to ensure that no terminator is created.
         });
@@ -212,8 +212,8 @@ class ExtractValOpConverter : public OpConversionPattern<ExtractValOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value pos = adaptor.getIterator().back();
-    Value valBuf = ToValuesOp::create(rewriter,
-        loc, llvm::getSingleElement(adaptor.getTensor()));
+    Value valBuf = ToValuesOp::create(
+        rewriter, loc, llvm::getSingleElement(adaptor.getTensor()));
     rewriter.replaceOpWithNewOp<memref::LoadOp>(op, valBuf, pos);
     return success();
   }
@@ -385,8 +385,8 @@ class SparseCoIterateOpConverter : public OpConversionPattern<CoIterateOp> {
         SmallVector<Value> nextIterYields(res);
         // 2nd. foward the loop.
         for (SparseIterator *it : validIters) {
-          Value cmp = arith::CmpIOp::create(rewriter,
-              loc, arith::CmpIPredicate::eq, it->getCrd(), loopCrd);
+          Value cmp = arith::CmpIOp::create(
+              rewriter, loc, arith::CmpIPredicate::eq, it->getCrd(), loopCrd);
           it->forwardIf(rewriter, loc, cmp);
           llvm::append_range(nextIterYields, it->getCursor());
         }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
index dc6bef5387b70..df9b6cf040efa 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
@@ -43,7 +43,8 @@ struct DemapInsRewriter : public OpRewritePattern<SourceOp> {
     SmallVector<Value> deMappedIns(op->getOperands());
     for (Value &in : deMappedIns) {
       if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) {
-        in = ReinterpretMapOp::create(rewriter, loc, stt->getDemappedType(), in);
+        in =
+            ReinterpretMapOp::create(rewriter, loc, stt->getDemappedType(), in);
         changed = true;
       }
     }
@@ -338,7 +339,7 @@ translateMap(linalg::GenericOp op, PatternRewriter &rewriter) {
 static Value genDemap(OpBuilder &builder, SparseTensorEncodingAttr enc,
                       Value val) {
   return ReinterpretMapOp::create(builder, val.getLoc(), enc.withoutDimToLvl(),
-                                          val);
+                                  val);
 }
 
 // Generates a "re"mapping reinterpretation of the map.
@@ -604,8 +605,8 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     ValueRange dynSz = op.getDynamicSizes();
     for (int64_t dimSz : stt.getDimShape()) {
       if (ShapedType::isDynamic(dimSz)) {
-        Value maxCrd = arith::SubIOp::create(rewriter,
-            loc, dynSz.front(), constantIndex(rewriter, loc, 1));
+        Value maxCrd = arith::SubIOp::create(rewriter, loc, dynSz.front(),
+                                             constantIndex(rewriter, loc, 1));
         maxDimCrds.push_back(maxCrd);
         dynSz = dynSz.drop_front();
       } else {
@@ -619,8 +620,8 @@ struct TensorAllocDemapper : public OpRewritePattern<AllocOp> {
     SmallVector<Value> dynLvlSzs;
     for (unsigned i = 0, e = lvlShape.size(); i < e; i++) {
       if (ShapedType::isDynamic(lvlShape[i])) {
-        Value sz = arith::AddIOp::create(rewriter,
-            loc, maxLvlCrds[i], constantIndex(rewriter, loc, 1));
+        Value sz = arith::AddIOp::create(rewriter, loc, maxLvlCrds[i],
+                                         constantIndex(rewriter, loc, 1));
         dynLvlSzs.push_back(sz);
       }
     }
@@ -650,8 +651,8 @@ struct TensorInsertDemapper
     auto stt = getSparseTensorType(op.getResult());
     ValueRange lvlCrd = stt.translateCrds(rewriter, loc, op.getIndices(),
                                           CrdTransDirectionKind::dim2lvl);
-    auto insertOp = tensor::InsertOp::create(rewriter,
-        loc, op.getScalar(), adaptor.getDest(), lvlCrd);
+    auto insertOp = tensor::InsertOp::create(rewriter, loc, op.getScalar(),
+                                             adaptor.getDest(), lvlCrd);
 
     Value out = genRemap(rewriter, stt.getEncoding(), insertOp.getResult());
     rewriter.replaceOp(op, out);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
index 7b662c2fb0db0..81cd3296de294 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseSpaceCollapse.cpp
@@ -126,8 +126,8 @@ void collapseSparseSpace(MutableArrayRef<CollapseSpaceInfo> toCollapse) {
   OpBuilder builder(root);
 
   // Construct the collapsed iteration space.
-  auto collapsedSpace = ExtractIterSpaceOp::create(builder,
-      loc, root.getTensor(), root.getParentIter(), root.getLoLvl(),
+  auto collapsedSpace = ExtractIterSpaceOp::create(
+      builder, loc, root.getTensor(), root.getParentIter(), root.getLoLvl(),
       leaf.getHiLvl());
 
   auto rItOp = llvm::cast<IterateOp>(*root->getUsers().begin());
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
index a1b9035f3b6e6..6dfffbb6e7442 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
@@ -75,9 +75,9 @@ class SpecifierStructBuilder : public StructBuilder {
 
   void insertField(OpBuilder &builder, Location loc, ArrayRef<int64_t> indices,
                    Value v) {
-    value = LLVM::InsertValueOp::create(builder,
-        loc, value, genCast(builder, loc, v, builder.getIntegerType(64)),
-        indices);
+    value = LLVM::InsertValueOp::create(
+        builder, loc, value,
+        genCast(builder, loc, v, builder.getIntegerType(64)), indices);
   }
 
 public:
@@ -205,14 +205,14 @@ void SpecifierStructBuilder::setMemSize(OpBuilder &builder, Location loc,
 Value SpecifierStructBuilder::memSizeArray(OpBuilder &builder,
                                            Location loc) const {
   return LLVM::ExtractValueOp::create(builder, loc, value,
-                                              kMemSizePosInSpecifier);
+                                      kMemSizePosInSpecifier);
 }
 
 /// Builds IR inserting the memory size array into the descriptor.
 void SpecifierStructBuilder::setMemSizeArray(OpBuilder &builder, Location loc,
                                              Value array) {
   value = LLVM::InsertValueOp::create(builder, loc, value, array,
-                                              kMemSizePosInSpecifier);
+                                      kMemSizePosInSpecifier);
 }
 
 } // namespace
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 0450e1d6e8775..70795e2eb211b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -70,7 +70,8 @@ static scf::ForOp createFor(OpBuilder &builder, Location loc, Value upper,
   if (!lower)
     lower = constantZero(builder, loc, indexType);
   Value one = constantOne(builder, loc, indexType);
-  scf::ForOp forOp = scf::ForOp::create(builder, loc, lower, upper, one, fields);
+  scf::ForOp forOp =
+      scf::ForOp::create(builder, loc, lower, upper, one, fields);
   for (unsigned i = 0, e = fields.size(); i < e; i++)
     fields[i] = forOp.getRegionIterArg(i);
   builder.setInsertionPointToStart(forOp.getBody());
@@ -86,9 +87,9 @@ static void createPushback(OpBuilder &builder, Location loc,
   Value field = desc.getMemRefField(kind, lvl);
   StorageSpecifierKind specFieldKind = toSpecifierKind(kind);
 
-  auto pushBackOp = PushBackOp::create(builder,
-      loc, desc.getSpecifierField(builder, loc, specFieldKind, lvl), field,
-      genCast(builder, loc, value, etp), repeat);
+  auto pushBackOp = PushBackOp::create(
+      builder, loc, desc.getSpecifierField(builder, loc, specFieldKind, lvl),
+      field, genCast(builder, loc, value, etp), repeat);
 
   desc.setMemRefField(kind, lvl, pushBackOp.getOutBuffer());
   desc.setSpecifierField(builder, loc, specFieldKind, lvl,
@@ -178,16 +179,16 @@ static void createAllocFields(OpBuilder &builder, Location loc,
   if (stt.isAllDense()) {
     valHeuristic = lvlSizesValues[0];
     for (Level lvl = 1; lvl < lvlRank; lvl++)
-      valHeuristic =
-          arith::MulIOp::create(builder, loc, valHeuristic, lvlSizesValues[lvl]);
+      valHeuristic = arith::MulIOp::create(builder, loc, valHeuristic,
+                                           lvlSizesValues[lvl]);
   } else if (sizeHint) {
     if (stt.getAoSCOOStart() == 0) {
       posHeuristic = constantIndex(builder, loc, 2);
-      crdHeuristic = arith::MulIOp::create(builder,
-          loc, constantIndex(builder, loc, lvlRank), sizeHint); // AOS
+      crdHeuristic = arith::MulIOp::create(
+          builder, loc, constantIndex(builder, loc, lvlRank), sizeHint); // AOS
     } else if (lvlRank == 2 && stt.isDenseLvl(0) && stt.isCompressedLvl(1)) {
-      posHeuristic = arith::AddIOp::create(builder,
-          loc, sizeHint, constantIndex(builder, loc, 1));
+      posHeuristic = arith::AddIOp::create(builder, loc, sizeHint,
+                                           constantIndex(builder, loc, 1));
       crdHeuristic = sizeHint;
     } else {
       posHeuristic = crdHeuristic = constantIndex(builder, loc, 16);
@@ -290,22 +291,22 @@ static Value genCompressed(OpBuilder &builder, Location loc,
   const Value msz =
       crdStrideC ? arith::DivUIOp::create(builder, loc, crdMsz, crdStrideC)
                  : crdMsz;
-  const Value plast = arith::SubIOp::create(builder,
-      loc, genCast(builder, loc, pstop, indexType), one);
+  const Value plast = arith::SubIOp::create(
+      builder, loc, genCast(builder, loc, pstop, indexType), one);
   // Conditional expression.
   Value lt = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ult,
-                                           pstart, pstop);
+                                   pstart, pstop);
   types.push_back(boolType);
   scf::IfOp ifOp1 = scf::IfOp::create(builder, loc, types, lt, /*else*/ true);
   types.pop_back();
   builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
-  Value crd =
-      genLoad(builder, loc, desc.getMemRefField(crdFidx),
-              crdStrideC ? arith::MulIOp::create(builder, loc, plast, crdStrideC)
-                         : plast);
-  Value eq = arith::CmpIOp::create(builder,
-      loc, arith::CmpIPredicate::eq, genCast(builder, loc, crd, indexType),
-      lvlCoords[lvl]);
+  Value crd = genLoad(
+      builder, loc, desc.getMemRefField(crdFidx),
+      crdStrideC ? arith::MulIOp::create(builder, loc, plast, crdStrideC)
+                 : plast);
+  Value eq = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
+                                   genCast(builder, loc, crd, indexType),
+                                   lvlCoords[lvl]);
   scf::YieldOp::create(builder, loc, eq);
   builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
   if (lvl > 0)
@@ -381,10 +382,10 @@ static void genEndInsert(OpBuilder &builder, Location loc,
         Value oldv = loop.getRegionIterArg(0);
         Value newv = genLoad(builder, loc, posMemRef, i);
         Value posZero = constantZero(builder, loc, posType);
-        Value cond = arith::CmpIOp::create(builder,
-            loc, arith::CmpIPredicate::eq, newv, posZero);
+        Value cond = arith::CmpIOp::create(
+            builder, loc, arith::CmpIPredicate::eq, newv, posZero);
         scf::IfOp ifOp = scf::IfOp::create(builder, loc, TypeRange(posType),
-                                                   cond, /*else*/ true);
+                                           cond, /*else*/ true);
         builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         genStore(builder, loc, oldv, posMemRef, i);
         scf::YieldOp::create(builder, loc, oldv);
@@ -582,8 +583,9 @@ class SparseCallConverter : public OpConversionPattern<func::CallOp> {
       return failure();
 
     // (1) Generates new call with flattened return value.
-    auto newCall = func::CallOp::create(rewriter,
-        loc, op.getCallee(), finalRetTy, flattenValues(adaptor.getOperands()));
+    auto newCall =
+        func::CallOp::create(rewriter, loc, op.getCallee(), finalRetTy,
+                             flattenValues(adaptor.getOperands()));
     // (2) Gather sparse tensor returns.
     SmallVector<SmallVector<Value>> packedResultVals;
     // Tracks the offset of current return value (of the original call)
@@ -672,7 +674,7 @@ struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
     auto id = AffineMap::getMultiDimIdentityMap(srcStt.getLvlRank(), ctx);
 
     SortOp::create(rewriter, loc, nnz, crd, ValueRange{val}, id,
-                            rewriter.getIndexAttr(0), op.getAlgorithm());
+                   rewriter.getIndexAttr(0), op.getAlgorithm());
 
     // Since we do in-place sorting, the destinate tensor will have the same set
     // of memrefs as the source tensor.
@@ -931,12 +933,12 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    linalg::FillOp::create(rewriter,
-        loc, ValueRange{constantZero(rewriter, loc, eltType)},
-        ValueRange{values});
-    linalg::FillOp::create(rewriter,
-        loc, ValueRange{constantZero(rewriter, loc, boolType)},
-        ValueRange{filled});
+    linalg::FillOp::create(rewriter, loc,
+                           ValueRange{constantZero(rewriter, loc, eltType)},
+                           ValueRange{values});
+    linalg::FillOp::create(rewriter, loc,
+                           ValueRange{constantZero(rewriter, loc, boolType)},
+                           ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
     assert(op.getNumResults() == 4);
     rewriter.replaceOp(op, {values, filled, added, zero});
@@ -965,9 +967,10 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     // If the innermost level is ordered, we need to sort the coordinates
     // in the "added" array prior to applying the compression.
     if (dstType.isOrderedLvl(dstType.getLvlRank() - 1))
-      SortOp::create(rewriter,
-          loc, count, added, ValueRange{}, rewriter.getMultiDimIdentityMap(1),
-          rewriter.getIndexAttr(0), SparseTensorSortKind::HybridQuickSort);
+      SortOp::create(rewriter, loc, count, added, ValueRange{},
+                     rewriter.getMultiDimIdentityMap(1),
+                     rewriter.getIndexAttr(0),
+                     SparseTensorSortKind::HybridQuickSort);
     // While performing the insertions, we also need to reset the elements
     // of the values/filled-switch by only iterating over the set elements,
     // to ensure that the runtime complexity remains proportional to the
@@ -1192,8 +1195,8 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
             // would require a subViewOp to avoid overflow when copying
             // values.
             Value sz = linalg::createOrFoldDimOp(rewriter, loc, srcMem, 0);
-            auto dstMem = memref::AllocOp::create(rewriter,
-                loc, cast<MemRefType>(fTp), sz);
+            auto dstMem = memref::AllocOp::create(rewriter, loc,
+                                                  cast<MemRefType>(fTp), sz);
             if (fTp != srcMem.getType()) {
               // Converts elements type.
               scf::buildLoopNest(
@@ -1242,8 +1245,9 @@ class SparseExtractSliceConverter
     auto desc = getMutDescriptorFromTensorTuple(adaptor.getSource(), fields,
                                                 op.getSource().getType());
 
-    auto newSpec = StorageSpecifierInitOp::create(rewriter,
-        loc, StorageSpecifierType::get(ctx, dstEnc), desc.getSpecifier());
+    auto newSpec = StorageSpecifierInitOp::create(
+        rewriter, loc, StorageSpecifierType::get(ctx, dstEnc),
+        desc.getSpecifier());
     desc.setSpecifier(newSpec);
 
     // Fills in slice information.
@@ -1326,8 +1330,8 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
               // Flattens the buffer to batchLvlRank.
               auto reassoc = getReassociationForFlattening(
                   mem.getType(), stt.getBatchLvlRank());
-              mem = memref::CastOp::create(rewriter,
-                  loc, fType,
+              mem = memref::CastOp::create(
+                  rewriter, loc, fType,
                   memref::CollapseShapeOp::create(rewriter, loc, mem, reassoc));
             } else {
               mem = memref::CastOp::create(rewriter, loc, fType, mem);
@@ -1396,8 +1400,8 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
       assert(isWithCrdLT(lt) && lvl <= trailCOOStart);
       // FIXME: This seems to be unnecessarily complex, can we simplify it?
       if (lvl == trailCOOStart) {
-        Value cooSz = arith::MulIOp::create(rewriter,
-            loc, memSize, constantIndex(rewriter, loc, trailCOORank));
+        Value cooSz = arith::MulIOp::create(
+            rewriter, loc, memSize, constantIndex(rewriter, loc, trailCOORank));
         desc.setCrdMemSize(rewriter, loc, lvl, cooSz);
       } else {
         desc.setCrdMemSize(rewriter, loc, lvl, memSize);
@@ -1471,8 +1475,9 @@ struct SparseDisassembleOpConverter
     // Converts MemRefs back to Tensors.
     SmallVector<Value> retValues = llvm::to_vector(
         llvm::map_range(retMem, [&rewriter, loc](Value v) -> Value {
-          return bufferization::ToTensorOp::create(rewriter,
-              loc, memref::getTensorTypeFromMemRefType(v.getType()), v);
+          return bufferization::ToTensorOp::create(
+              rewriter, loc, memref::getTensorTypeFromMemRefType(v.getType()),
+              v);
         }));
     // Appends the actual memory length used in each buffer returned.
     retValues.append(retLen.begin(), retLen.end());
@@ -1549,15 +1554,15 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     const Level lvlRank = dstTp.getLvlRank();
     if (dstTp.isOrderedLvl(lvlRank - 1)) {
       Value kFalse = constantI1(rewriter, loc, false);
-      Value notSorted = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::eq, isSorted, kFalse);
+      Value notSorted = arith::CmpIOp::create(
+          rewriter, loc, arith::CmpIPredicate::eq, isSorted, kFalse);
       scf::IfOp ifOp =
           scf::IfOp::create(rewriter, loc, notSorted, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       auto xPerm = rewriter.getMultiDimIdentityMap(lvlRank);
       SortOp::create(rewriter, loc, nse, xs, ValueRange{ys}, xPerm,
-                              rewriter.getIndexAttr(0),
-                              SparseTensorSortKind::HybridQuickSort);
+                     rewriter.getIndexAttr(0),
+                     SparseTensorSortKind::HybridQuickSort);
       rewriter.setInsertionPointAfter(ifOp);
     }
 
@@ -1569,8 +1574,8 @@ struct SparseNewConverter : public OpConversionPattern<NewOp> {
     memref::StoreOp::create(rewriter, loc, posNse, posMemref0, c1);
 
     // Update storage specifier.
-    Value coordinatesSize = arith::MulIOp::create(rewriter,
-        loc, nse, constantIndex(rewriter, loc, lvlRank));
+    Value coordinatesSize = arith::MulIOp::create(
+        rewriter, loc, nse, constantIndex(rewriter, loc, lvlRank));
     desc.setSpecifierField(rewriter, loc, StorageSpecifierKind::CrdMemSize, 0,
                            coordinatesSize);
     desc.setSpecifierField(rewriter, loc, StorageSpecifierKind::ValMemSize,
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 54ea1843b457f..134aef3a6c719 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -168,12 +168,12 @@ static Value genLvlPtrsBuffers(OpBuilder &builder, Location loc,
 
   // Passing in value buffer pointers.
   lvlBarePtrs.push_back(extractBarePtrFromTensor(builder, loc, valTensor));
-  Value idxPtr = memref::ExtractAlignedPointerAsIndexOp::create(builder,
-      loc, allocaBuffer(builder, loc, lvlBarePtrs));
+  Value idxPtr = memref::ExtractAlignedPointerAsIndexOp::create(
+      builder, loc, allocaBuffer(builder, loc, lvlBarePtrs));
   Value idxCast =
       arith::IndexCastOp::create(builder, loc, builder.getI64Type(), idxPtr);
   return LLVM::IntToPtrOp::create(builder, loc, getOpaquePointerType(builder),
-                                          idxCast);
+                                  idxCast);
 }
 
 /// This class abstracts over the API of `_mlir_ciface_newSparseTensor`:
@@ -690,12 +690,12 @@ class SparseTensorExpandConverter : public OpConversionPattern<ExpandOp> {
     // operation is amortized over the innermost loops for the access
     // pattern expansion. As noted in the operation doc, we would like
     // to amortize this setup cost even between kernels.
-    linalg::FillOp::create(rewriter,
-        loc, ValueRange{constantZero(rewriter, loc, eltType)},
-        ValueRange{values});
-    linalg::FillOp::create(rewriter,
-        loc, ValueRange{constantZero(rewriter, loc, boolType)},
-        ValueRange{filled});
+    linalg::FillOp::create(rewriter, loc,
+                           ValueRange{constantZero(rewriter, loc, eltType)},
+                           ValueRange{values});
+    linalg::FillOp::create(rewriter, loc,
+                           ValueRange{constantZero(rewriter, loc, boolType)},
+                           ValueRange{filled});
     // Replace expansion op with these buffers and initial coordinate.
     assert(op.getNumResults() == 4);
     rewriter.replaceOp(op, {values, filled, lastLvlCoordinates, zero});
@@ -867,9 +867,9 @@ class SparseTensorDisassembleConverter
     // Converts MemRefs back to Tensors.
     assert(retVal.size() + retLen.size() == op.getNumResults());
     for (unsigned i = 0, sz = retVal.size(); i < sz; i++) {
-      auto tensor = bufferization::ToTensorOp::create(rewriter,
-          loc, memref::getTensorTypeFromMemRefType(retVal[i].getType()),
-          retVal[i]);
+      auto tensor = bufferization::ToTensorOp::create(
+          rewriter, loc,
+          memref::getTensorTypeFromMemRefType(retVal[i].getType()), retVal[i]);
       retVal[i] =
           tensor::CastOp::create(rewriter, loc, op.getResultTypes()[i], tensor);
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index ee963ab5d1ffb..b444ac5ba1285 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -405,8 +405,8 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
     inputOps.push_back(op.getDpsInputOperand(1 - other)->get());
     fusedIndexMaps.push_back(fusedIndexMaps.back()); // mimic other
     // Fuse producer and consumer into a new generic op.
-    auto fusedOp = GenericOp::create(rewriter,
-        loc, op.getResult(0).getType(), inputOps, outputOps,
+    auto fusedOp = GenericOp::create(
+        rewriter, loc, op.getResult(0).getType(), inputOps, outputOps,
         rewriter.getAffineMapArrayAttr(fusedIndexMaps), prod.getIteratorTypes(),
         /*doc=*/nullptr, /*library_call=*/nullptr);
     Block &prodBlock = prod.getRegion().front();
@@ -695,8 +695,8 @@ struct GenSemiRingReduction : public OpRewritePattern<GenericOp> {
     // CustomReduce {
     //    x = x REDUC y, identity
     // }
-    auto custom = sparse_tensor::ReduceOp::create(rewriter,
-        loc, rtp, semiring.getResult(), s1, identity);
+    auto custom = sparse_tensor::ReduceOp::create(
+        rewriter, loc, rtp, semiring.getResult(), s1, identity);
     Block *region =
         rewriter.createBlock(&custom.getRegion(), {}, {rtp, rtp}, {loc, loc});
     rewriter.setInsertionPointToStart(&custom.getRegion().front());
@@ -724,8 +724,9 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
     auto stt = getSparseTensorType(tensor);
     // Header with NSE.
     auto nse = NumberOfEntriesOp::create(rewriter, loc, tensor);
-    vector::PrintOp::create(rewriter,
-        loc, rewriter.getStringAttr("---- Sparse Tensor ----\nnse = "));
+    vector::PrintOp::create(
+        rewriter, loc,
+        rewriter.getStringAttr("---- Sparse Tensor ----\nnse = "));
     vector::PrintOp::create(rewriter, loc, nse);
     // Print run-time contents for dim/lvl sizes.
     vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("dim = "));
@@ -745,8 +746,8 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       case SparseTensorFieldKind::PosMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
         vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("pos["));
-        vector::PrintOp::create(rewriter,
-            loc, lvl, vector::PrintPunctuation::NoPunctuation);
+        vector::PrintOp::create(rewriter, loc, lvl,
+                                vector::PrintPunctuation::NoPunctuation);
         vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
         auto pos = ToPositionsOp::create(rewriter, loc, tensor, l);
         printContents(rewriter, loc, pos);
@@ -755,8 +756,8 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       case SparseTensorFieldKind::CrdMemRef: {
         auto lvl = constantIndex(rewriter, loc, l);
         vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("crd["));
-        vector::PrintOp::create(rewriter,
-            loc, lvl, vector::PrintPunctuation::NoPunctuation);
+        vector::PrintOp::create(rewriter, loc, lvl,
+                                vector::PrintPunctuation::NoPunctuation);
         vector::PrintOp::create(rewriter, loc, rewriter.getStringAttr("] : "));
         Value crd = nullptr;
         // For COO AoS storage, we want to print a single, linear view of
@@ -771,7 +772,7 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
       }
       case SparseTensorFieldKind::ValMemRef: {
         vector::PrintOp::create(rewriter, loc,
-                                         rewriter.getStringAttr("values : "));
+                                rewriter.getStringAttr("values : "));
         auto val = ToValuesOp::create(rewriter, loc, tensor);
         printContents(rewriter, loc, val);
         break;
@@ -827,17 +828,17 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
         Value imag = complex::ImOp::create(rewriter, loc, val);
         vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Open);
         vector::PrintOp::create(rewriter, loc, real,
-                                         vector::PrintPunctuation::Comma);
+                                vector::PrintPunctuation::Comma);
         vector::PrintOp::create(rewriter, loc, imag,
-                                         vector::PrintPunctuation::Close);
+                                vector::PrintPunctuation::Close);
       } else {
-        vector::PrintOp::create(rewriter,
-            loc, val, vector::PrintPunctuation::NoPunctuation);
+        vector::PrintOp::create(rewriter, loc, val,
+                                vector::PrintPunctuation::NoPunctuation);
       }
       // Terminating comma (except at end).
       auto bound = arith::AddIOp::create(rewriter, loc, idxs.back(), step);
-      Value cond = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::ne,
-                                                  bound, size);
+      Value cond = arith::CmpIOp::create(rewriter, loc,
+                                         arith::CmpIPredicate::ne, bound, size);
       scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, cond, /*else*/ false);
       rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
       vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Comma);
@@ -861,10 +862,10 @@ struct PrintRewriter : public OpRewritePattern<PrintOp> {
         val = tensor::DimOp::create(rewriter, loc, tensor, idx);
       else
         val = LvlOp::create(rewriter, loc, tensor, idx);
-      vector::PrintOp::create(rewriter,
-          loc, val,
-          i != size - 1 ? vector::PrintPunctuation::Comma
-                        : vector::PrintPunctuation::NoPunctuation);
+      vector::PrintOp::create(rewriter, loc, val,
+                              i != size - 1
+                                  ? vector::PrintPunctuation::Comma
+                                  : vector::PrintPunctuation::NoPunctuation);
     }
     // Close bracket and end of line.
     vector::PrintOp::create(rewriter, loc, vector::PrintPunctuation::Close);
@@ -920,8 +921,8 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp->getEncoding();
-    ForeachOp foreachOp = ForeachOp::create(rewriter,
-        loc, srcTensor, buffer,
+    ForeachOp foreachOp = ForeachOp::create(
+        rewriter, loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
           const Dimension srcRank = srcTp->getDimRank();
@@ -1025,8 +1026,8 @@ struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
     //   %t = sparse_tensor.cast %tmp
     // depending on whether the input/output are sorted in the same way.
     const auto encSrc = srcTp.getEncoding();
-    ForeachOp foreachOp = ForeachOp::create(rewriter,
-        loc, srcTensor, buffer,
+    ForeachOp foreachOp = ForeachOp::create(
+        rewriter, loc, srcTensor, buffer,
         [&](OpBuilder &builder, Location loc, ValueRange srcLcvs, Value v,
             ValueRange reduc) {
           const Dimension dimRank = srcTp.getDimRank();
@@ -1089,12 +1090,12 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
           RankedTensorType::get(rtp.getShape(), rtp.getElementType());
       ReshapeOp reshape;
       if constexpr (std::is_same<ReshapeOp, tensor::ExpandShapeOp>::value) {
-        reshape = ReshapeOp::create(rewriter,
-            loc, denseTp, op.getSrc(), op.getReassociation(),
-            op.getOutputShape(), op.getStaticOutputShape());
+        reshape = ReshapeOp::create(rewriter, loc, denseTp, op.getSrc(),
+                                    op.getReassociation(), op.getOutputShape(),
+                                    op.getStaticOutputShape());
       } else {
         reshape = ReshapeOp::create(rewriter, loc, denseTp, op.getSrc(),
-                                             op.getReassociation());
+                                    op.getReassociation());
       }
       Value convert = ConvertOp::create(rewriter, loc, rtp, reshape);
       rewriter.replaceOp(op, convert);
@@ -1161,18 +1162,20 @@ struct SparseTensorDimOpRewriter : public OpRewritePattern<tensor::DimOp> {
     SmallVector<Value> maxLvlCrds;
     for (Level l = 0; l < stt->getLvlRank(); l++) {
       Value lvlSz = LvlOp::create(rewriter, loc, op.getSource(), l);
-      Value maxLvlCrd = arith::SubIOp::create(rewriter,
-          loc, lvlSz, constantOne(rewriter, loc, rewriter.getIndexType()));
+      Value maxLvlCrd = arith::SubIOp::create(
+          rewriter, loc, lvlSz,
+          constantOne(rewriter, loc, rewriter.getIndexType()));
       maxLvlCrds.push_back(maxLvlCrd);
     }
 
     AffineExpr lvl2DimExp = stt->getLvlToDim().getResult(*dim);
-    Value maxDimCrd = affine::AffineApplyOp::create(rewriter,
-        op.getLoc(), AffineMap::get(stt->getLvlRank(), 0, lvl2DimExp),
+    Value maxDimCrd = affine::AffineApplyOp::create(
+        rewriter, op.getLoc(), AffineMap::get(stt->getLvlRank(), 0, lvl2DimExp),
         maxLvlCrds);
 
-    Value dimSz = arith::AddIOp::create(rewriter,
-        loc, maxDimCrd, constantOne(rewriter, loc, rewriter.getIndexType()));
+    Value dimSz = arith::AddIOp::create(
+        rewriter, loc, maxDimCrd,
+        constantOne(rewriter, loc, rewriter.getIndexType()));
     rewriter.replaceOp(op, dimSz);
     return success();
   }
@@ -1212,8 +1215,8 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
     for (Value input : op.getInputs()) {
       // Builds a for op for each input tensor to append new values into the
       // output tensor.
-      foreachOp = ForeachOp::create(rewriter,
-          loc, input, iterArg,
+      foreachOp = ForeachOp::create(
+          rewriter, loc, input, iterArg,
           [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
               ValueRange reduc) {
             SmallVector<Value> offDimCrd(dcvs);
@@ -1224,8 +1227,9 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
             dstBuf.val = reduc.front();
             if (!dstTp.isAllDense()) {
               Value cond = genIsNonzero(builder, loc, v);
-              auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
-                                                    /*else*/ true);
+              auto ifOp =
+                  scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
+                                    /*else*/ true);
               builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
               scf::YieldOp::create(builder, loc, dstBuf.val);
 
@@ -1247,7 +1251,7 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
       const Size sz = getSparseTensorType(input).getDynamicDimSize(conDim);
       assert(ShapedType::isStatic(sz));
       offset = arith::AddIOp::create(rewriter, loc, offset,
-                                              constantIndex(rewriter, loc, sz));
+                                     constantIndex(rewriter, loc, sz));
       iterArg = foreachOp.getResult(0);
       dstBuf.val = iterArg;
     }
@@ -1299,8 +1303,8 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
     ValueRange vs;
     TensorLike dstBuf(rewriter, loc, dstStt.getRankedTensorType(), sizes);
 
-    auto foreachOp = ForeachOp::create(rewriter,
-        loc, src, dstBuf.val, foreachOrder,
+    auto foreachOp = ForeachOp::create(
+        rewriter, loc, src, dstBuf.val, foreachOrder,
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
           // Enters the loop, update the SSA value for insertion chain.
@@ -1308,7 +1312,7 @@ struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
           if (!skipZeroCheck) {
             Value cond = genIsNonzero(builder, loc, v);
             auto ifOp = scf::IfOp::create(builder, loc, reduc.getTypes(), cond,
-                                                  /*else*/ true);
+                                          /*else*/ true);
             builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
             scf::YieldOp::create(builder, loc, dstBuf.val);
 
@@ -1349,8 +1353,8 @@ struct CrdTranslateRewriter : public OpRewritePattern<CrdTranslateOp> {
       // TODO: we should probably expand the affine map to IR using our own
       // rules, since affine.apply assume signed value, while the cooridinates
       // we provided must always be signless.
-      Value trans = affine::AffineApplyOp::create(rewriter,
-          op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
+      Value trans = affine::AffineApplyOp::create(
+          rewriter, op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
           op.getInCrds());
       outCrds.push_back(trans);
     }
@@ -1515,7 +1519,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     sizesForTensor(rewriter, dims, loc, srcTp, src);
     for (Dimension d = 0; d < dimRank; d++) {
       memref::StoreOp::create(rewriter, loc, dims[d], dimSizes,
-                                       constantIndex(rewriter, loc, d));
+                              constantIndex(rewriter, loc, d));
     }
 
     // Create a sparse tensor writer and output meta data.
@@ -1536,13 +1540,13 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
     ModuleOp module = op->getParentOfType<ModuleOp>();
 
     // For each element in the source tensor, output the element.
-    ForeachOp::create(rewriter,
-        loc, src, ValueRange(),
+    ForeachOp::create(
+        rewriter, loc, src, ValueRange(),
         [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
             ValueRange reduc) {
           for (Dimension d = 0; d < dimRank; d++) {
             memref::StoreOp::create(rewriter, loc, dcvs[d], dimCoords,
-                                             constantIndex(builder, loc, d));
+                                    constantIndex(builder, loc, d));
           }
           memref::StoreOp::create(rewriter, loc, v, value);
           SmallVector<Value> operands{writer, rankValue, dimCoords, value};
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 7bccd9a64c9e4..4464450fd328f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -116,10 +116,10 @@ static Value genVectorLoad(PatternRewriter &rewriter, Location loc, VL vl,
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
     return vector::GatherOp::create(rewriter, loc, vtp, mem, scalarArgs,
-                                             indexVec, vmask, pass);
+                                    indexVec, vmask, pass);
   }
   return vector::MaskedLoadOp::create(rewriter, loc, vtp, mem, idxs, vmask,
-                                               pass);
+                                      pass);
 }
 
 /// Generates a vectorized store a[ind[lo:hi]] = rhs or a[lo:hi] = rhs
@@ -133,7 +133,7 @@ static void genVectorStore(PatternRewriter &rewriter, Location loc, Value mem,
     Value indexVec = idxs.back();
     scalarArgs.back() = constantIndex(rewriter, loc, 0);
     vector::ScatterOp::create(rewriter, loc, mem, scalarArgs, indexVec, vmask,
-                                       rhs);
+                              rhs);
     return;
   }
   vector::MaskedStoreOp::create(rewriter, loc, mem, idxs, vmask, rhs);
@@ -198,13 +198,13 @@ static Value genVectorReducInit(PatternRewriter &rewriter, Location loc,
   case vector::CombiningKind::XOR:
     // Initialize reduction vector to: | 0 | .. | 0 | r |
     return vector::InsertOp::create(rewriter, loc, r,
-                                             constantZero(rewriter, loc, vtp),
-                                             constantIndex(rewriter, loc, 0));
+                                    constantZero(rewriter, loc, vtp),
+                                    constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::MUL:
     // Initialize reduction vector to: | 1 | .. | 1 | r |
     return vector::InsertOp::create(rewriter, loc, r,
-                                             constantOne(rewriter, loc, vtp),
-                                             constantIndex(rewriter, loc, 0));
+                                    constantOne(rewriter, loc, vtp),
+                                    constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::AND:
   case vector::CombiningKind::OR:
     // Initialize reduction vector to: | r | .. | r | r |
@@ -300,11 +300,11 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
         Type etp = llvm::cast<VectorType>(vload.getType()).getElementType();
         if (!llvm::isa<IndexType>(etp)) {
           if (etp.getIntOrFloatBitWidth() < 32)
-            vload = arith::ExtUIOp::create(rewriter,
-                loc, vectorType(vl, rewriter.getI32Type()), vload);
+            vload = arith::ExtUIOp::create(
+                rewriter, loc, vectorType(vl, rewriter.getI32Type()), vload);
           else if (etp.getIntOrFloatBitWidth() < 64 && !vl.enableSIMDIndex32)
-            vload = arith::ExtUIOp::create(rewriter,
-                loc, vectorType(vl, rewriter.getI64Type()), vload);
+            vload = arith::ExtUIOp::create(
+                rewriter, loc, vectorType(vl, rewriter.getI64Type()), vload);
         }
         idxs.push_back(vload);
       }
@@ -341,7 +341,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define UNAOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = xxx::create(rewriter, loc, vx);                                    \
+      vexp = xxx::create(rewriter, loc, vx);                                   \
     return true;                                                               \
   }
 
@@ -349,7 +349,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
   if (auto x = dyn_cast<xxx>(def)) {                                           \
     if (codegen) {                                                             \
       VectorType vtp = vectorType(vl, x.getType());                            \
-      vexp = xxx::create(rewriter, loc, vtp, vx);                               \
+      vexp = xxx::create(rewriter, loc, vtp, vx);                              \
     }                                                                          \
     return true;                                                               \
   }
@@ -357,7 +357,7 @@ static bool vectorizeSubscripts(PatternRewriter &rewriter, scf::ForOp forOp,
 #define BINOP(xxx)                                                             \
   if (isa<xxx>(def)) {                                                         \
     if (codegen)                                                               \
-      vexp = xxx::create(rewriter, loc, vx, vy);                                \
+      vexp = xxx::create(rewriter, loc, vx, vy);                               \
     return true;                                                               \
   }
 
@@ -533,8 +533,8 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       VectorType vtp = vectorType(vl, init.getType());
       Value vinit = genVectorReducInit(rewriter, loc, yield->getOperand(0),
                                        forOp.getRegionIterArg(0), init, vtp);
-      forOpNew = scf::ForOp::create(rewriter,
-          loc, forOp.getLowerBound(), forOp.getUpperBound(), step, vinit);
+      forOpNew = scf::ForOp::create(rewriter, loc, forOp.getLowerBound(),
+                                    forOp.getUpperBound(), step, vinit);
       forOpNew->setAttr(
           LoopEmitter::getLoopEmitterLoopAttrName(),
           forOp->getAttr(LoopEmitter::getLoopEmitterLoopAttrName()));
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
index 29a3e92fcf7fb..0a5f5595bba56 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
@@ -317,7 +317,7 @@ static void genBuffers(CodegenEnv &env, OpBuilder &builder) {
           Value zero = constantZero(builder, loc,
                                     getElementTypeOrSelf(tensor.getType()));
           linalg::FillOp::create(builder, loc, ValueRange{zero},
-                                         ValueRange{init});
+                                 ValueRange{init});
         }
         return init;
       },
@@ -470,9 +470,9 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
   // If statement.
   Value isFilled = memref::LoadOp::create(builder, loc, filled, index);
   Value cond = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
-                                             isFilled, fval);
+                                     isFilled, fval);
   scf::IfOp ifOp = scf::IfOp::create(builder, loc, builder.getIndexType(), cond,
-                                             /*else=*/true);
+                                     /*else=*/true);
   // True branch.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   memref::StoreOp::create(builder, loc, tval, filled, index);
@@ -516,7 +516,8 @@ static Value genTensorLoad(CodegenEnv &env, OpBuilder &builder, ExprId exp) {
   if (llvm::isa<TensorType>(ptr.getType())) {
     assert(env.options().sparseEmitStrategy ==
            SparseEmitStrategy::kSparseIterator);
-    return ExtractValOp::create(builder, loc, ptr, llvm::getSingleElement(args));
+    return ExtractValOp::create(builder, loc, ptr,
+                                llvm::getSingleElement(args));
   }
   return memref::LoadOp::create(builder, loc, ptr, args);
 }
@@ -768,7 +769,8 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Type t2 = MemRefType::get(dynShape, builder.getI1Type());
     Type t3 = MemRefType::get(dynShape, builder.getIndexType());
     Type t4 = builder.getIndexType();
-    auto r = ExpandOp::create(builder, loc, TypeRange({t1, t2, t3, t4}), tensor);
+    auto r =
+        ExpandOp::create(builder, loc, TypeRange({t1, t2, t3, t4}), tensor);
     assert(r.getNumResults() == 4);
     env.startExpand(r.getResult(0), r.getResult(1), r.getResult(2),
                     r.getResult(3));
@@ -782,7 +784,7 @@ static void genExpand(CodegenEnv &env, OpBuilder &builder, LoopId curr,
     Value count = env.getExpandCount();
     Value chain = env.getInsertionChain();
     Value compress = CompressOp::create(builder, loc, values, filled, added,
-                                                count, chain, indices);
+                                        count, chain, indices);
     env.updateInsertionChain(compress);
     env.endExpand();
   }
@@ -943,12 +945,13 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
           const Value crd = env.emitter().getCoord(tid, *lvl);
           const Value lvar = env.getLoopVar(curr);
           clause = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq,
-                                                 crd, lvar);
+                                         crd, lvar);
         } else {
           assert(lt.hasDenseSemantic() || isUndefLT(lt));
           clause = constantI1(builder, loc, true);
         }
-        cond = cond ? arith::AndIOp::create(builder, loc, cond, clause) : clause;
+        cond =
+            cond ? arith::AndIOp::create(builder, loc, cond, clause) : clause;
       });
   if (env.isReduc()) {
     types.push_back(env.getReduc().getType());
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
index 27c1b106d71ba..f57f7f7fc0946 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
@@ -204,10 +204,10 @@ Value mlir::sparse_tensor::genIsNonzero(OpBuilder &builder, mlir::Location loc,
   Value zero = constantZero(builder, loc, tp);
   if (isa<FloatType>(tp))
     return arith::CmpFOp::create(builder, loc, arith::CmpFPredicate::UNE, v,
-                                         zero);
+                                 zero);
   if (tp.isIntOrIndex())
     return arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::ne, v,
-                                         zero);
+                                 zero);
   if (isa<ComplexType>(tp))
     return complex::NotEqualOp::create(builder, loc, v, zero);
   llvm_unreachable("Non-numeric type");
@@ -296,7 +296,8 @@ void mlir::sparse_tensor::reshapeCvs(
     for (unsigned j = start, end = start + map.value().size(); j < end; j++) {
       linear = arith::DivUIOp::create(builder, loc, linear, sizes[j]);
       if (isCollapse) {
-        const Value mul = arith::MulIOp::create(builder, loc, srcCvs[j], linear);
+        const Value mul =
+            arith::MulIOp::create(builder, loc, srcCvs[j], linear);
         val = val ? arith::AddIOp::create(builder, loc, val, mul) : mul;
       } else {
         const Value old = val;
@@ -326,8 +327,8 @@ FlatSymbolRefAttr mlir::sparse_tensor::getFunc(ModuleOp module, StringRef name,
   auto func = module.lookupSymbol<func::FuncOp>(result.getAttr());
   if (!func) {
     OpBuilder moduleBuilder(module.getBodyRegion());
-    func = func::FuncOp::create(moduleBuilder,
-        module.getLoc(), name,
+    func = func::FuncOp::create(
+        moduleBuilder, module.getLoc(), name,
         FunctionType::get(context, operands.getTypes(), resultType));
     func.setPrivate();
     if (static_cast<bool>(emitCInterface))
@@ -490,7 +491,7 @@ void sparse_tensor::foreachInSparseConstant(
     if (isa<ComplexType>(attr.getElementType())) {
       auto valAttr = cast<ArrayAttr>(elems[i].second);
       val = complex::ConstantOp::create(builder, loc, attr.getElementType(),
-                                                valAttr);
+                                        valAttr);
     } else {
       auto valAttr = cast<TypedAttr>(elems[i].second);
       val = arith::ConstantOp::create(builder, loc, valAttr);
@@ -514,7 +515,7 @@ SmallVector<Value> sparse_tensor::loadAll(OpBuilder &builder, Location loc,
   vs.reserve(size);
   for (unsigned i = 0; i < size; i++) {
     Value v = memref::LoadOp::create(builder, loc, mem,
-                                             constantIndex(builder, loc, i));
+                                     constantIndex(builder, loc, i));
     if (i == offsetIdx && offsetVal)
       v = arith::AddIOp::create(builder, loc, v, offsetVal);
     vs.push_back(v);
@@ -538,7 +539,7 @@ void sparse_tensor::storeAll(OpBuilder &builder, Location loc, Value mem,
             ? arith::AddIOp::create(builder, loc, v.value(), offsetVal)
             : v.value();
     memref::StoreOp::create(builder, loc, w, mem,
-                                    constantIndex(builder, loc, v.index()));
+                            constantIndex(builder, loc, v.index()));
   }
 }
 
@@ -609,8 +610,8 @@ Value sparse_tensor::genReader(OpBuilder &builder, Location loc,
     // subsequent clients need the values (DCE will remove unused).
     for (Dimension d = 0; d < dimRank; d++) {
       if (stt.isDynamicDim(d))
-        dimSizesValues[d] = memref::LoadOp::create(builder,
-            loc, dimSizesBuffer, constantIndex(builder, loc, d));
+        dimSizesValues[d] = memref::LoadOp::create(
+            builder, loc, dimSizesBuffer, constantIndex(builder, loc, d));
     }
   }
   return reader;
@@ -687,7 +688,7 @@ Value sparse_tensor::genMapBuffers(
       lvlSz = dimSizesValues[d];
       if (cf != 0)
         lvlSz = arith::DivUIOp::create(builder, loc, lvlSz,
-                                               constantIndex(builder, loc, cf));
+                                       constantIndex(builder, loc, cf));
     } else {
       lvlSz = constantIndex(builder, loc, cm);
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
index 400be54aeb86f..1c10dd5566184 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
@@ -97,8 +97,8 @@ class FuncCallOrInlineGenerator {
       // Create the function if not already exist.
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(getParentOpOf<func::FuncOp>(builder));
-      func = func::FuncOp::create(builder,
-          loc, funcName,
+      func = func::FuncOp::create(
+          builder, loc, funcName,
           FunctionType::get(context, params.getTypes(), retTypes));
       func.setPrivate();
       // Set the insertion point to the body of the function.
@@ -400,8 +400,8 @@ inline Value constantLevelTypeEncoding(OpBuilder &builder, Location loc,
 inline Value genValFromAttr(OpBuilder &builder, Location loc, Attribute attr) {
   if (auto complexAttr = dyn_cast<complex::NumberAttr>(attr)) {
     Type tp = cast<ComplexType>(complexAttr.getType()).getElementType();
-    return complex::ConstantOp::create(builder,
-        loc, complexAttr.getType(),
+    return complex::ConstantOp::create(
+        builder, loc, complexAttr.getType(),
         builder.getArrayAttr({FloatAttr::get(tp, complexAttr.getReal()),
                               FloatAttr::get(tp, complexAttr.getImag())}));
   }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
index 6eaeff91dd5c6..659282a995123 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
@@ -25,7 +25,7 @@ using namespace mlir::sparse_tensor;
 //===----------------------------------------------------------------------===//
 
 #define CMPI(p, l, r)                                                          \
-  (arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::p, (l), (r))       \
+  (arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::p, (l), (r))      \
        .getResult())
 
 #define C_IDX(v) (constantIndex(builder, loc, (v)))
@@ -45,8 +45,8 @@ using namespace mlir::sparse_tensor;
 #ifndef NDEBUG
 LLVM_ATTRIBUTE_UNUSED static void dumpIndexMemRef(OpBuilder &builder,
                                                   Location loc, Value memref) {
-  memref = memref::CastOp::create(builder,
-      loc, UnrankedMemRefType::get(builder.getIndexType(), 0), memref);
+  memref = memref::CastOp::create(
+      builder, loc, UnrankedMemRefType::get(builder.getIndexType(), 0), memref);
   createFuncCall(builder, loc, "printMemrefInd", TypeRange{},
                  ValueRange{memref}, EmitCInterface::On);
 }
@@ -604,11 +604,11 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
       // Extract and iterate over the iteration space.
       ExtractIterSpaceOp extractSpaceOp =
           lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
-                   : ExtractIterSpaceOp::create(builder,
-                         loc, t, spIterVals[tid][lvl - 1], lvl);
+                   : ExtractIterSpaceOp::create(builder, loc, t,
+                                                spIterVals[tid][lvl - 1], lvl);
 
-      IterateOp iterOp = IterateOp::create(builder,
-          loc, extractSpaceOp.getExtractedSpace(), reduc);
+      IterateOp iterOp = IterateOp::create(
+          builder, loc, extractSpaceOp.getExtractedSpace(), reduc);
       spIterVals[tid][lvl] = iterOp.getIterator();
 
       // Update the reduction varaibles.
@@ -626,8 +626,8 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
       Value t = tensors[tid];
       ExtractIterSpaceOp extractSpaceOp =
           lvl == 0 ? ExtractIterSpaceOp::create(builder, loc, t)
-                   : ExtractIterSpaceOp::create(builder,
-                         loc, t, spIterVals[tid][lvl - 1], lvl);
+                   : ExtractIterSpaceOp::create(builder, loc, t,
+                                                spIterVals[tid][lvl - 1], lvl);
       spaces.push_back(extractSpaceOp.getExtractedSpace());
     }
     auto coIterOp = CoIterateOp::create(builder, loc, spaces, reduc, numCases);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
index 47fa3f950afd3..3b3b0aadf638c 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
@@ -78,15 +78,16 @@ SparseTensorTypeToBufferConverter::SparseTensorTypeToBufferConverter() {
 
 Value SparseTensorSpecifier::getInitValue(OpBuilder &builder, Location loc,
                                           SparseTensorType stt) {
-  return StorageSpecifierInitOp::create(builder,
-      loc, StorageSpecifierType::get(stt.getEncoding()));
+  return StorageSpecifierInitOp::create(
+      builder, loc, StorageSpecifierType::get(stt.getEncoding()));
 }
 
 Value SparseTensorSpecifier::getSpecifierField(OpBuilder &builder, Location loc,
                                                StorageSpecifierKind kind,
                                                std::optional<Level> lvl) {
-  return GetStorageSpecifierOp::create(builder,
-      loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl));
+  return GetStorageSpecifierOp::create(
+      builder, loc, specifier, kind,
+      optionalLevelAttr(specifier.getContext(), lvl));
 }
 
 void SparseTensorSpecifier::setSpecifierField(OpBuilder &builder, Location loc,
@@ -95,8 +96,9 @@ void SparseTensorSpecifier::setSpecifierField(OpBuilder &builder, Location loc,
                                               std::optional<Level> lvl) {
   // TODO: make `v` have type `TypedValue<IndexType>` instead.
   assert(v.getType().isIndex());
-  specifier = SetStorageSpecifierOp::create(builder,
-      loc, specifier, kind, optionalLevelAttr(specifier.getContext(), lvl), v);
+  specifier = SetStorageSpecifierOp::create(
+      builder, loc, specifier, kind,
+      optionalLevelAttr(specifier.getContext(), lvl), v);
 }
 
 //===----------------------------------------------------------------------===//
@@ -112,8 +114,8 @@ Value sparse_tensor::SparseTensorDescriptor::getCrdMemRefOrView(
   Value stride = constantIndex(builder, loc, rType.getLvlRank() - cooStart);
   Value size = getCrdMemSize(builder, loc, cooStart);
   size = arith::DivUIOp::create(builder, loc, size, stride);
-  return memref::SubViewOp::create(builder,
-      loc, getMemRefField(SparseTensorFieldKind::CrdMemRef, cooStart),
+  return memref::SubViewOp::create(
+      builder, loc, getMemRefField(SparseTensorFieldKind::CrdMemRef, cooStart),
       /*offset=*/ValueRange{constantIndex(builder, loc, lvl - cooStart)},
       /*size=*/ValueRange{size},
       /*step=*/ValueRange{stride});
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
index 212ee99647167..46d0baac58f06 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorIterator.cpp
@@ -22,7 +22,7 @@ using ValueTuple = std::tuple<Value, Value, Value>;
 // File local helper functions/macros.
 //===----------------------------------------------------------------------===//
 #define CMPI(p, lhs, rhs)                                                      \
-  (arith::CmpIOp::create(b, l, arith::CmpIPredicate::p, (lhs), (rhs))           \
+  (arith::CmpIOp::create(b, l, arith::CmpIPredicate::p, (lhs), (rhs))          \
        .getResult())
 
 #define C_FALSE (constantI1(b, l, false))
@@ -732,8 +732,8 @@ class NonEmptySubSectIterator : public SparseIterator {
   //      [itVal0, itVal1, ..., pNx0],
   //      ...]
   Value allocSubSectPosBuf(OpBuilder &b, Location l) {
-    return memref::AllocaOp::create(b,
-        l,
+    return memref::AllocaOp::create(
+        b, l,
         MemRefType::get({ShapedType::kDynamic, tupleSz + 1}, b.getIndexType()),
         maxTupleCnt);
   }
@@ -741,12 +741,12 @@ class NonEmptySubSectIterator : public SparseIterator {
   void storeNxLvlStart(OpBuilder &b, Location l, Value tupleId,
                        Value start) const {
     memref::StoreOp::create(b, l, start, subSectPosBuf,
-                              ValueRange{tupleId, C_IDX(tupleSz)});
+                            ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
   Value loadNxLvlStart(OpBuilder &b, Location l, Value tupleId) const {
     return memref::LoadOp::create(b, l, subSectPosBuf,
-                                    ValueRange{tupleId, C_IDX(tupleSz)});
+                                  ValueRange{tupleId, C_IDX(tupleSz)});
   }
 
   void storeCursorVals(OpBuilder &b, Location l, Value tupleId,
@@ -754,7 +754,7 @@ class NonEmptySubSectIterator : public SparseIterator {
     assert(itVals.size() == tupleSz);
     for (unsigned i = 0; i < tupleSz; i++) {
       memref::StoreOp::create(b, l, itVals[i], subSectPosBuf,
-                                ValueRange{tupleId, C_IDX(i)});
+                              ValueRange{tupleId, C_IDX(i)});
     }
   }
 
@@ -763,7 +763,7 @@ class NonEmptySubSectIterator : public SparseIterator {
     SmallVector<Value> ret;
     for (unsigned i = 0; i < tupleSz; i++) {
       Value v = memref::LoadOp::create(b, l, subSectPosBuf,
-                                         ValueRange{tupleId, C_IDX(i)});
+                                       ValueRange{tupleId, C_IDX(i)});
       ret.push_back(v);
     }
     return ret;
@@ -1058,8 +1058,8 @@ ValueRange SparseIterator::forwardIf(OpBuilder &b, Location l, Value cond) {
 }
 
 Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) {
-  auto whileOp = scf::WhileOp::create(b,
-      l, pos.getType(), pos,
+  auto whileOp = scf::WhileOp::create(
+      b, l, pos.getType(), pos,
       /*beforeBuilder=*/
       [this, pos](OpBuilder &b, Location l, ValueRange ivs) {
         Value inBound = CMPI(ult, ivs.front(), posHi);
@@ -1137,8 +1137,8 @@ ValueRange FilterIterator::forwardImpl(OpBuilder &b, Location l) {
 
   SmallVector<Value> whileArgs(getCursor().begin(), getCursor().end());
   whileArgs.push_back(isFirst);
-  auto whileOp = scf::WhileOp::create(b,
-      l, ValueRange(whileArgs).getTypes(), whileArgs,
+  auto whileOp = scf::WhileOp::create(
+      b, l, ValueRange(whileArgs).getTypes(), whileArgs,
       /*beforeBuilder=*/
       [this](OpBuilder &b, Location l, ValueRange ivs) {
         ValueRange isFirst = linkNewScope(ivs);
@@ -1219,8 +1219,8 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
     SmallVector<Value> iterArgs;
     iterArgs.push_back(C_IDX(0));
     iterArgs.append(reduc.begin(), reduc.end());
-    auto forEachLeaf = scf::ForOp::create(b,
-        l, /*lb=*/C_IDX(0), /*ub=*/tupleCnt, /*step=*/C_IDX(1), iterArgs,
+    auto forEachLeaf = scf::ForOp::create(
+        b, l, /*lb=*/C_IDX(0), /*ub=*/tupleCnt, /*step=*/C_IDX(1), iterArgs,
         [&helper, &builder](OpBuilder &b, Location l, Value tupleId,
                             ValueRange iterArgs) {
           // Deserialize the iterator at the cached position (tupleId).
@@ -1235,8 +1235,8 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
           SmallVector<Value> whileArgs(helper.wrap.getCursor());
           whileArgs.append(iterArgs.begin(), iterArgs.end());
 
-          auto whileOp = scf::WhileOp::create(b,
-              l, ValueRange(whileArgs).getTypes(), whileArgs,
+          auto whileOp = scf::WhileOp::create(
+              b, l, ValueRange(whileArgs).getTypes(), whileArgs,
               /*beforeBuilder=*/
               [&helper](OpBuilder &b, Location l, ValueRange ivs) {
                 helper.wrap.linkNewScope(ivs);
@@ -1267,8 +1267,8 @@ ValueRange NonEmptySubSectIterator::inflateSubSectTree(
                                      ValueRange reduc) {
     assert(!parent || parent->lvl + 1 == lvl);
     delegate->genInit(b, l, parent);
-    auto forOp = scf::ForOp::create(b,
-        l, /*lb=*/C_IDX(0), /*ub=*/subSectSz, /*step=*/C_IDX(1), reduc,
+    auto forOp = scf::ForOp::create(
+        b, l, /*lb=*/C_IDX(0), /*ub=*/subSectSz, /*step=*/C_IDX(1), reduc,
         [&](OpBuilder &b, Location l, Value crd, ValueRange iterArgs) {
           helper.locate(b, l, crd);
           scf::ValueVector nx = builder(b, l, &helper.wrap, iterArgs);
@@ -1458,8 +1458,8 @@ ValueRange NonEmptySubSectIterator::forwardImpl(OpBuilder &b, Location l) {
                 return genWhenInBound(b, l, *delegate, /*elseRet=*/iterArgs,
                                       [nxMin](OpBuilder &b, Location l,
                                               Value crd) -> scf::ValueVector {
-                                        Value nx = arith::MinUIOp::create(b,
-                                            l, crd, nxMin);
+                                        Value nx = arith::MinUIOp::create(
+                                            b, l, crd, nxMin);
                                         return {nx, C_TRUE};
                                       });
               });
@@ -1581,8 +1581,9 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &b, Location l, Value t,
   auto stt = getSparseTensorType(t);
 
   LevelType lt = stt.getLvlType(lvl);
-  Value sz = stt.hasEncoding() ? LvlOp::create(b, l, t, lvl).getResult()
-                               : tensor::DimOp::create(b, l, t, lvl).getResult();
+  Value sz = stt.hasEncoding()
+                 ? LvlOp::create(b, l, t, lvl).getResult()
+                 : tensor::DimOp::create(b, l, t, lvl).getResult();
 
   SmallVector<Value, 2> buffers;
   if (lt.isWithPosLT()) {
diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
index fb68620c378e5..5847fecc45404 100644
--- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
@@ -1666,10 +1666,10 @@ Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
   case TensorExp::Kind::kNegC:
     return complex::NegOp::create(rewriter, loc, v0);
   case TensorExp::Kind::kNegI: // no negi in std
-    return arith::SubIOp::create(rewriter,
-        loc,
+    return arith::SubIOp::create(
+        rewriter, loc,
         arith::ConstantOp::create(rewriter, loc, v0.getType(),
-                                           rewriter.getZeroAttr(v0.getType())),
+                                  rewriter.getZeroAttr(v0.getType())),
         v0);
   case TensorExp::Kind::kTruncF:
     return arith::TruncFOp::create(rewriter, loc, inferType(e, v0), v0);
diff --git a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
index 78eb50cbdf540..7e4a5acb9867d 100644
--- a/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
+++ b/mlir/lib/Dialect/Tensor/Extensions/MeshShardingExtensions.cpp
@@ -78,8 +78,8 @@ struct CreatorOpShardingInterface
             device =
                 mesh::ProcessMultiIndexOp::create(builder, op->getLoc(), mesh)
                     .getResults();
-            shapeForDevice = mesh::ShardShapeOp::create(builder,
-                op->getLoc(), oldType.getShape(), spmdizedOperands,
+            shapeForDevice = mesh::ShardShapeOp::create(
+                builder, op->getLoc(), oldType.getShape(), spmdizedOperands,
                 newSharding->getResult(0), device);
           }
           newOperands.emplace_back(shapeForDevice.getResult()[i]);
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 0a4fa7bfa007f..124a63281a37c 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -207,8 +207,8 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     if (isZeroInteger(newLength)) {
       hasZeroLen = true;
     } else if (!hasZeroLen) {
-      Value check = arith::CmpIOp::create(b,
-          loc, arith::CmpIPredicate::eq,
+      Value check = arith::CmpIOp::create(
+          b, loc, arith::CmpIPredicate::eq,
           getValueOrCreateConstantIndexOp(b, loc, newLength),
           getValueOrCreateConstantIndexOp(b, loc, zero));
       dynHasZeroLenCond =
@@ -245,8 +245,8 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // have a dimension of size 0, the semantics of which is unclear.)
   auto createGenerateOp = [&]() {
     // Create GenerateOp.
-    auto generateOp = tensor::GenerateOp::create(b,
-        loc, resultType, dynDims,
+    auto generateOp = tensor::GenerateOp::create(
+        b, loc, resultType, dynDims,
         [&](OpBuilder &builder, Location gLoc, ValueRange indices) {
           tensor::YieldOp::create(builder, gLoc, padValue);
         });
@@ -257,10 +257,10 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   // the result shape of the new SliceOp has a zero dimension.
   auto createPadOfExtractSlice = [&]() {
     // Create pad(extract_slice(x)).
-    auto newSliceOp = tensor::ExtractSliceOp::create(b,
-        loc, padOp.getSource(), newOffsets, newLengths, newStrides);
-    auto newPadOp = PadOp::create(b,
-        loc, Type(), newSliceOp, newLows, newHighs,
+    auto newSliceOp = tensor::ExtractSliceOp::create(
+        b, loc, padOp.getSource(), newOffsets, newLengths, newStrides);
+    auto newPadOp = PadOp::create(
+        b, loc, Type(), newSliceOp, newLows, newHighs,
         /*nofold=*/padOp.getNofold(),
         getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
 
@@ -287,8 +287,8 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     Operation *thenOp;
     Operation *elseOp;
     Operation *sliceOp;
-    auto result = scf::IfOp::create(b,
-        loc, dynHasZeroLenCond,
+    auto result = scf::IfOp::create(
+        b, loc, dynHasZeroLenCond,
         /*thenBuilder=*/
         [&](OpBuilder &b, Location loc) {
           thenOp = createGenerateOp();
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index e5b4b7a865d87..bc11e567fa2d8 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -222,8 +222,8 @@ struct CollapseShapeOpInterface
           MemRefType::get(collapseShapeOp.getSrcType().getShape(),
                           collapseShapeOp.getSrcType().getElementType(),
                           AffineMap(), bufferType.getMemorySpace());
-      buffer = bufferization::ToBufferOp::create(rewriter,
-          op->getLoc(), memrefType, *tensorAlloc);
+      buffer = bufferization::ToBufferOp::create(rewriter, op->getLoc(),
+                                                 memrefType, *tensorAlloc);
     }
 
     // Result type is inferred by the builder.
@@ -349,8 +349,8 @@ struct ExpandShapeOpInterface
     if (failed(buffer))
       return failure();
 
-    auto memrefExpandShape = memref::ExpandShapeOp::create(rewriter,
-        op->getLoc(), tensorResultType.getShape(), *buffer,
+    auto memrefExpandShape = memref::ExpandShapeOp::create(
+        rewriter, op->getLoc(), tensorResultType.getShape(), *buffer,
         expandShapeOp.getReassociationIndices(),
         expandShapeOp.getMixedOutputShape());
     replaceOpWithBufferizedValues(rewriter, op,
@@ -398,8 +398,8 @@ struct ExtractSliceOpInterface
         extractSliceOp.getResult(), options, state);
     if (failed(resultMemrefType))
       return failure();
-    Value subView = memref::SubViewOp::create(rewriter,
-        loc, llvm::cast<MemRefType>(*resultMemrefType), *srcMemref,
+    Value subView = memref::SubViewOp::create(
+        rewriter, loc, llvm::cast<MemRefType>(*resultMemrefType), *srcMemref,
         mixedOffsets, mixedSizes, mixedStrides);
 
     replaceOpWithBufferizedValues(rewriter, op, subView);
@@ -507,8 +507,8 @@ struct FromElementsOpInterface
         bufferization::getBufferType(*tensorAlloc, options, state);
     if (failed(memrefType))
       return failure();
-    Value buffer = bufferization::ToBufferOp::create(rewriter,
-        op->getLoc(), *memrefType, *tensorAlloc);
+    Value buffer = bufferization::ToBufferOp::create(rewriter, op->getLoc(),
+                                                     *memrefType, *tensorAlloc);
 
     // Case: tensor<0xelem_type>.
     if (fromElementsOp.getElements().empty()) {
@@ -518,8 +518,8 @@ struct FromElementsOpInterface
 
     // Case: tensor<elem_type>.
     if (shape.empty()) {
-      memref::StoreOp::create(rewriter,
-          loc, fromElementsOp.getElements().front(), buffer);
+      memref::StoreOp::create(rewriter, loc,
+                              fromElementsOp.getElements().front(), buffer);
       replaceOpWithBufferizedValues(rewriter, op, buffer);
       return success();
     }
@@ -577,7 +577,7 @@ static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
   OpBuilder::InsertionGuard g(rewriter);
   auto linalgOp =
       linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
-                                     /*init=*/tensorDestination);
+                            /*init=*/tensorDestination);
   Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
   // Create linalg::IndexOps.
@@ -645,7 +645,7 @@ struct InsertOpInterface
     if (failed(destMemref))
       return failure();
     memref::StoreOp::create(rewriter, insertOp.getLoc(), insertOp.getScalar(),
-                                     *destMemref, insertOp.getIndices());
+                            *destMemref, insertOp.getIndices());
     replaceOpWithBufferizedValues(rewriter, op, *destMemref);
     return success();
   }
@@ -713,9 +713,9 @@ struct InsertSliceOpInterface
         memref::SubViewOp::inferRankReducedResultType(
             insertSliceOp.getSourceType().getShape(), dstMemrefType,
             mixedOffsets, mixedSizes, mixedStrides);
-    Value subView = memref::SubViewOp::create(rewriter,
-        loc, subviewMemRefType, *dstMemref, mixedOffsets, mixedSizes,
-        mixedStrides);
+    Value subView =
+        memref::SubViewOp::create(rewriter, loc, subviewMemRefType, *dstMemref,
+                                  mixedOffsets, mixedSizes, mixedStrides);
 
     // Copy tensor. If this tensor.insert_slice has a matching
     // tensor.extract_slice, the copy operation will eventually fold away.
@@ -802,8 +802,8 @@ struct PadOpInterface
       AffineExpr s0, s1, s2;
       bindSymbols(op->getContext(), s0, s1, s2);
       AffineExpr sumExpr = s0 + s1 + s2;
-      Value sum = affine::AffineApplyOp::create(rewriter,
-          loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
+      Value sum = affine::AffineApplyOp::create(
+          rewriter, loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
       dynamicSizes.push_back(sum);
     }
 
@@ -995,9 +995,9 @@ struct ParallelInsertSliceOpInterface
             parallelInsertSliceOp.getMixedOffsets(),
             parallelInsertSliceOp.getMixedSizes(),
             parallelInsertSliceOp.getMixedStrides());
-    Value subview = memref::SubViewOp::create(rewriter,
-        parallelInsertSliceOp.getLoc(), subviewMemRefType, *destBuffer,
-        parallelInsertSliceOp.getMixedOffsets(),
+    Value subview = memref::SubViewOp::create(
+        rewriter, parallelInsertSliceOp.getLoc(), subviewMemRefType,
+        *destBuffer, parallelInsertSliceOp.getMixedOffsets(),
         parallelInsertSliceOp.getMixedSizes(),
         parallelInsertSliceOp.getMixedStrides());
 
@@ -1065,9 +1065,9 @@ struct SplatOpInterface
     if (options.defaultMemorySpaceFn(tensorType) != Attribute())
       return op->emitError("memory space not implemented yet");
 
-    auto linalgOp =
-        linalg::MapOp::create(rewriter, loc, tensorType, /*inputs=*/ValueRange(),
-                                       /*init=*/*tensorAlloc);
+    auto linalgOp = linalg::MapOp::create(rewriter, loc, tensorType,
+                                          /*inputs=*/ValueRange(),
+                                          /*init=*/*tensorAlloc);
     Block &linalgBody = linalgOp.getMapper().emplaceBlock();
 
     // Create linalg::IndexOps.
@@ -1126,8 +1126,8 @@ struct ConcatOpInterface
     MemRefType memrefType =
         MemRefType::get(concatOp.getResultType().getShape(),
                         concatOp.getResultType().getElementType(), layout);
-    Value dstBuffer = bufferization::ToBufferOp::create(rewriter,
-        op->getLoc(), memrefType, *tensorAlloc);
+    Value dstBuffer = bufferization::ToBufferOp::create(
+        rewriter, op->getLoc(), memrefType, *tensorAlloc);
 
     // Extract the dimension for the concat op
     uint64_t concatDim = concatOp.getDim();
@@ -1174,8 +1174,9 @@ struct ConcatOpInterface
 
       if (dynamicConcatDim) {
         offsets[concatDim] = dynamicOffset.value();
-        dynamicSize = memref::DimOp::create(rewriter, loc, *srcBuffer, concatDim)
-                          .getResult();
+        dynamicSize =
+            memref::DimOp::create(rewriter, loc, *srcBuffer, concatDim)
+                .getResult();
         sizes[concatDim] = dynamicSize.value();
       } else {
         sizes[concatDim] = rewriter.getIndexAttr(operandConcatDimSize);
@@ -1188,16 +1189,16 @@ struct ConcatOpInterface
           memref::SubViewOp::inferRankReducedResultType(
               operandTensorType.getShape(), dstMemrefType, offsets, sizes,
               strides);
-      Value subview = memref::SubViewOp::create(rewriter,
-          loc, subviewMemRefType, dstBuffer, offsets, sizes, strides);
+      Value subview = memref::SubViewOp::create(
+          rewriter, loc, subviewMemRefType, dstBuffer, offsets, sizes, strides);
 
       // Copy the source buffer into the destination subview.
       if (failed(options.createMemCpy(rewriter, loc, *srcBuffer, subview)))
         return failure();
 
       if (dynamicConcatDim) {
-        dynamicOffset = arith::AddIOp::create(rewriter,
-            loc, dynamicOffset.value(), dynamicSize.value());
+        dynamicOffset = arith::AddIOp::create(
+            rewriter, loc, dynamicOffset.value(), dynamicSize.value());
       } else {
         concatDimOffset += operandConcatDimSize;
       }
diff --git a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
index 24bcb8192f672..670865de6031f 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/EmptyOpPatterns.cpp
@@ -42,8 +42,9 @@ struct FoldEmptyTensorWithReshapeOp : public OpRewritePattern<ReshapeOp> {
 
     // Create new tensor.empty op.
     // TODO: Do not drop tensor type encoding.
-    Value emptyTensor = EmptyOp::create(rewriter,
-        loc, resultShapes[0], reshapeOp.getResultType().getElementType());
+    Value emptyTensor =
+        EmptyOp::create(rewriter, loc, resultShapes[0],
+                        reshapeOp.getResultType().getElementType());
     if (emptyTensor.getType() != reshapeOp.getResultType()) {
       rewriter.replaceOpWithNewOp<tensor::CastOp>(
           reshapeOp, reshapeOp.getResultType(), emptyTensor);
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
index c3801e0f05c88..840b4f3121428 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ExtractSliceFromReshapeUtils.cpp
@@ -142,15 +142,15 @@ tensor::ExtractSliceFromCollapseHelper::emitLoopNestBody(
   SmallVector<Range> extractParams =
       helper.getExtractSliceParams(builder.getContext(), multiIndices);
 
-  Value subTileResult = tensor::ExtractSliceOp::create(builder,
-      loc, collapseShapeOp.getSrc(), extractParams);
+  Value subTileResult = tensor::ExtractSliceOp::create(
+      builder, loc, collapseShapeOp.getSrc(), extractParams);
 
   SmallVector<Range> insertParams =
       helper.getInsertSliceParams(builder.getContext(), tileInductionVars);
 
   // Collapse the dimensions of the source slice back down.
-  Value collapsedResult = tensor::CollapseShapeOp::create(builder,
-      loc, subTileResult, reassociationIndices);
+  Value collapsedResult = tensor::CollapseShapeOp::create(
+      builder, loc, subTileResult, reassociationIndices);
   return std::make_pair(collapsedResult, insertParams);
 }
 
@@ -173,8 +173,9 @@ tensor::simplifyCollapseShapeWithRankReducingExtractSlice(
   SmallVector<OpFoldResult> sizes =
       tensor::getMixedSizes(rewriter, op.getLoc(), op.getSrc());
   SmallVector<OpFoldResult> strides(sourceType.getRank(), one);
-  auto sliceOp = tensor::ExtractSliceOp::create(rewriter,
-      op.getLoc(), info->sliceResultType, op.getSrc(), offsets, sizes, strides);
+  auto sliceOp = tensor::ExtractSliceOp::create(
+      rewriter, op.getLoc(), info->sliceResultType, op.getSrc(), offsets, sizes,
+      strides);
 
   if (!info->newReassociationIndices.has_value()) {
     rewriter.replaceOp(op, sliceOp.getResult());
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
index cc32f1193a55b..d76c02af7ab16 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
@@ -112,9 +112,9 @@ TransferReadOfExtractSliceOpFolder::matchAndRewriteMaskableOp(
       extractSliceOp.getMixedStrides(), extractSliceOp.getDroppedDims(),
       indices, sourceIndices);
 
-  Operation *newOp = vector::TransferReadOp::create(rewriter,
-      readOp.getLoc(), readOp.getVectorType(), extractSliceOp.getSource(),
-      sourceIndices,
+  Operation *newOp = vector::TransferReadOp::create(
+      rewriter, readOp.getLoc(), readOp.getVectorType(),
+      extractSliceOp.getSource(), sourceIndices,
       AffineMapAttr::get(expandDimsToRank(
           readOp.getPermutationMap(), extractSliceOp.getSourceType().getRank(),
           extractSliceOp.getDroppedDims())),
diff --git a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
index a6db540ffd8a0..43d9d704a29ff 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
@@ -64,9 +64,10 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
     return padOp.getResult();
 
   // Create a new tensor::PadOp.
-  auto newPadOp = PadOp::create(b,
-      loc, padOp.getResultType(), padOp.getSource(), newMixedLow, newMixedHigh,
-      constantPadding, padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
+  auto newPadOp =
+      PadOp::create(b, loc, padOp.getResultType(), padOp.getSource(),
+                    newMixedLow, newMixedHigh, constantPadding,
+                    padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
 
   // Create a tensor::ExtractSliceOp.
   // Reify the result sizes of the old tensor::PadOp.
@@ -83,10 +84,10 @@ FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
       offsets.push_back(b.getIndexAttr(0));
     } else {
       offsets.push_back(
-          affine::AffineApplyOp::create(b,
-               loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
-               std::initializer_list<Value>{cast<Value>(newMixedLow[i]),
-                                            cast<Value>(prevLow)})
+          affine::AffineApplyOp::create(
+              b, loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
+              std::initializer_list<Value>{cast<Value>(newMixedLow[i]),
+                                           cast<Value>(prevLow)})
               .getResult());
     }
     // size = reified result size
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index 3b63c87f2c05a..e0af2f77d44b8 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -76,8 +76,8 @@ struct FoldUnPaddingCollapseIntoExtract
       return rewriter.notifyMatchFailure(collapseShapeOp,
                                          "expected unpadding collapse");
 
-    Value unPaddedExtractSlice = tensor::ExtractSliceOp::create(rewriter,
-        extractSliceOp.getLoc(), collapseShapeOp.getResultType(),
+    Value unPaddedExtractSlice = tensor::ExtractSliceOp::create(
+        rewriter, extractSliceOp.getLoc(), collapseShapeOp.getResultType(),
         extractSliceOp.getSource(), extractSliceOp.getMixedOffsets(),
         extractSliceOp.getMixedSizes(), extractSliceOp.getMixedStrides());
     rewriter.replaceOp(collapseShapeOp, unPaddedExtractSlice);
@@ -270,8 +270,8 @@ struct BubbleUpExpandThroughParallelCollapse
     // matches the number of dimensions of the result, then the expand_shape
     // is a no-op.
     if (newExpandReInds.size() != newExpandSizes.size()) {
-      newCollapseSrc = tensor::ExpandShapeOp::create(rewriter,
-          loc, expandResultType, newCollapseSrc, newExpandReInds,
+      newCollapseSrc = tensor::ExpandShapeOp::create(
+          rewriter, loc, expandResultType, newCollapseSrc, newExpandReInds,
           newExpandSizes);
     }
 
@@ -280,8 +280,8 @@ struct BubbleUpExpandThroughParallelCollapse
     // is a no-op.
     Value replacement = newCollapseSrc;
     if (newCollapseReInds.size() != newExpandSizes.size()) {
-      replacement = tensor::CollapseShapeOp::create(rewriter,
-          loc, newCollapseSrc, newCollapseReInds);
+      replacement = tensor::CollapseShapeOp::create(
+          rewriter, loc, newCollapseSrc, newCollapseReInds);
     }
     rewriter.replaceOp(expandOp, replacement);
     return success();
@@ -405,8 +405,8 @@ struct BubbleUpExpandShapeThroughExtractSlice
         shape, expandShapeOp.getResultType().getElementType());
 
     // Create a new ExtractSliceOp and ExpandShapeOp.
-    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
-        loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
+    Value newSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
         collapsedStrides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSliceOp,
@@ -735,9 +735,9 @@ struct BubbleUpCollapseShapeThroughExtractSlice
                              groupExpandedOffsets.rend());
     }
 
-    Value newSliceOp = tensor::ExtractSliceOp::create(rewriter,
-        collapseShapeOp->getLoc(), collapseShapeOp.getSrc(), expandedOffsets,
-        expandedSizes, expandedStrides);
+    Value newSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, collapseShapeOp->getLoc(), collapseShapeOp.getSrc(),
+        expandedOffsets, expandedSizes, expandedStrides);
     rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
         sliceOp, sliceOp.getResultType(), newSliceOp,
         collapseShapeOp.getReassociationIndices());
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
index 7c1dea45843d5..838ff1f987c63 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
@@ -50,12 +50,11 @@ struct CastOpInterface
       Value srcRank = RankOp::create(builder, loc, castOp.getSource());
       Value resultRank =
           arith::ConstantIndexOp::create(builder, loc, resultType.getRank());
-      Value isSameRank = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcRank, resultRank);
-      cf::AssertOp::create(builder,
-          loc, isSameRank,
-          RuntimeVerifiableOpInterface::generateErrorMessage(op,
-                                                             "rank mismatch"));
+      Value isSameRank = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcRank, resultRank);
+      cf::AssertOp::create(builder, loc, isSameRank,
+                           RuntimeVerifiableOpInterface::generateErrorMessage(
+                               op, "rank mismatch"));
     }
 
     // Check dimension sizes.
@@ -73,10 +72,10 @@ struct CastOpInterface
           DimOp::create(builder, loc, castOp.getSource(), it.index());
       Value resultDimSz =
           arith::ConstantIndexOp::create(builder, loc, it.value());
-      Value isSameSz = arith::CmpIOp::create(builder,
-          loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
-      cf::AssertOp::create(builder,
-          loc, isSameSz,
+      Value isSameSz = arith::CmpIOp::create(
+          builder, loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
+      cf::AssertOp::create(
+          builder, loc, isSameSz,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "size mismatch of dim " + std::to_string(it.index())));
     }
@@ -91,8 +90,9 @@ struct DimOpInterface
     auto dimOp = cast<DimOp>(op);
     Value rank = RankOp::create(builder, loc, dimOp.getSource());
     Value zero = arith::ConstantIndexOp::create(builder, loc, 0);
-    cf::AssertOp::create(builder,
-        loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
+    cf::AssertOp::create(
+        builder, loc,
+        generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
         RuntimeVerifiableOpInterface::generateErrorMessage(
             op, "index is out of bounds"));
   }
@@ -134,10 +134,9 @@ struct ExtractInsertOpInterface
           i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
                 : inBounds;
     }
-    cf::AssertOp::create(builder,
-        loc, assertCond,
-        RuntimeVerifiableOpInterface::generateErrorMessage(
-            op, "out-of-bounds access"));
+    cf::AssertOp::create(builder, loc, assertCond,
+                         RuntimeVerifiableOpInterface::generateErrorMessage(
+                             op, "out-of-bounds access"));
   }
 };
 
@@ -167,8 +166,8 @@ struct ExtractSliceOpInterface
           loc, extractSliceOp.getSource(), i);
       Value offsetInBounds =
           generateInBoundsCheck(builder, loc, offset, zero, dimSize);
-      cf::AssertOp::create(builder,
-          loc, offsetInBounds,
+      cf::AssertOp::create(
+          builder, loc, offsetInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "offset " + std::to_string(i) + " is out-of-bounds"));
 
@@ -180,8 +179,8 @@ struct ExtractSliceOpInterface
           arith::AddIOp::create(builder, loc, offset, sizeMinusOneTimesStride);
       Value lastPosInBounds =
           generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
-      cf::AssertOp::create(builder,
-          loc, lastPosInBounds,
+      cf::AssertOp::create(
+          builder, loc, lastPosInBounds,
           RuntimeVerifiableOpInterface::generateErrorMessage(
               op, "extract_slice runs out-of-bounds along dimension " +
                       std::to_string(i)));
diff --git a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
index 21f70e52b8706..b6fdba360deea 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.cpp
@@ -59,8 +59,8 @@ struct InsertSliceLikeOpSubsetInsertionOpInterface
   Value buildSubsetExtraction(Operation *op, OpBuilder &builder,
                               Location loc) const {
     auto insertSliceOp = cast<OpTy>(op);
-    auto extractOp = tensor::ExtractSliceOp::create(builder,
-        loc, insertSliceOp.getSourceType(), insertSliceOp.getDest(),
+    auto extractOp = tensor::ExtractSliceOp::create(
+        builder, loc, insertSliceOp.getSourceType(), insertSliceOp.getDest(),
         insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
         insertSliceOp.getMixedStrides());
     return extractOp.getResult();
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 7ca4f6496ccdc..967f027915e42 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -845,9 +845,9 @@ struct PadSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
         getTosaConstShape(rewriter, sliceOp.getLoc(), newPadPaddings);
     auto newPadTy =
         RankedTensorType::get(newPadShape, inputTy.getElementType());
-    auto newPadOp = tosa::PadOp::create(rewriter,
-        padOp.getLoc(), newPadTy, padOp.getInput1(), newPaddingsOp,
-        padOp.getPadConst());
+    auto newPadOp = tosa::PadOp::create(rewriter, padOp.getLoc(), newPadTy,
+                                        padOp.getInput1(), newPaddingsOp,
+                                        padOp.getPadConst());
 
     // Update SliceOp and point to new PadOp
     auto newStartOp =
@@ -897,9 +897,9 @@ struct SliceDynamicSizeCanonicalization
     }
 
     auto size_op = getTosaConstShape(rewriter, sliceOp.getLoc(), sliceSizes);
-    auto newSliceOp = tosa::SliceOp::create(rewriter,
-        sliceOp.getLoc(), sliceOp.getType(), sliceOp.getInput1(),
-        sliceOp.getStart(), size_op);
+    auto newSliceOp =
+        tosa::SliceOp::create(rewriter, sliceOp.getLoc(), sliceOp.getType(),
+                              sliceOp.getInput1(), sliceOp.getStart(), size_op);
 
     rewriter.replaceOp(sliceOp, newSliceOp.getResult());
     return success();
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index d110e1aeee8e4..2ff1e6b6d1666 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -180,12 +180,12 @@ Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
   // Tosa dialect constants only support ElementsAttr unlike standard dialect
   // constant which supports all attributes.
   if (llvm::isa<shapeType>(type) && llvm::isa<DenseIntElementsAttr>(value)) {
-    return tosa::ConstShapeOp::create(builder,
-        loc, type, llvm::cast<DenseIntElementsAttr>(value));
+    return tosa::ConstShapeOp::create(builder, loc, type,
+                                      llvm::cast<DenseIntElementsAttr>(value));
   }
   if (llvm::isa<ElementsAttr>(value))
     return tosa::ConstOp::create(builder, loc, type,
-                                         llvm::cast<ElementsAttr>(value));
+                                 llvm::cast<ElementsAttr>(value));
   return nullptr;
 }
 
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
index 9b77986257814..9474299a39582 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeDepthwise.cpp
@@ -111,7 +111,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
             DenseElementsAttr::get(zpTy, rewriter.getIntegerAttr(ety, zp));
         auto zpVal = tosa::ConstOp::create(rewriter, op.getLoc(), zpTy, zpAttr);
         return tosa::SubOp::create(rewriter, op.getLoc(), val.getType(), val,
-                                            zpVal);
+                                   zpVal);
       };
 
       input = applyZp(input, iZp);
@@ -141,7 +141,7 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
           tosa::ConstOp::create(rewriter, op->getLoc(), padTy, padAttr);
       inputType = RankedTensorType::get(newShape, inputETy);
       input = tosa::PadOp::create(rewriter, op->getLoc(), inputType, input,
-                                           padSizeVal, padVal);
+                                  padSizeVal, padVal);
     }
 
     // Perform an elementwise mul over the reshaped input and weight.
@@ -174,8 +174,8 @@ struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
         dyn_cast<RankedTensorType>(input.getType()).getElementType());
     auto outputShapeValue =
         getTosaConstShape(rewriter, op->getLoc(), outputShape);
-    Value outputValue = tosa::ReshapeOp::create(rewriter,
-        op.getLoc(), outputShapeType, mulValue, outputShapeValue);
+    Value outputValue = tosa::ReshapeOp::create(
+        rewriter, op.getLoc(), outputShapeType, mulValue, outputShapeValue);
 
     Value bias = op.getBias();
     if (EqualizeRanks(rewriter, op.getLoc(), outputValue, bias).failed()) {
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
index 4a07220b4e181..dc5c51b0abad5 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
@@ -62,14 +62,16 @@ class TransposeConvNonStridedConverter
     convPad[2] = kernelWidth - 1 + pad[2];
     convPad[3] = kernelWidth - 1 + pad[3];
 
-    auto reverse1 = tosa::ReverseOp::create(rewriter,
-        loc, weightTy, weight, /* axis = */ rewriter.getI32IntegerAttr(1));
-    auto reverse2 = tosa::ReverseOp::create(rewriter,
-        loc, weightTy, reverse1, /* axis = */ rewriter.getI32IntegerAttr(2));
-
-    Value conv2d = tosa::Conv2DOp::create(rewriter,
-        loc, resultTy, input, reverse2, bias, op.getInputZp(), op.getWeightZp(),
-        rewriter.getDenseI64ArrayAttr(convPad),
+    auto reverse1 =
+        tosa::ReverseOp::create(rewriter, loc, weightTy, weight,
+                                /* axis = */ rewriter.getI32IntegerAttr(1));
+    auto reverse2 =
+        tosa::ReverseOp::create(rewriter, loc, weightTy, reverse1,
+                                /* axis = */ rewriter.getI32IntegerAttr(2));
+
+    Value conv2d = tosa::Conv2DOp::create(
+        rewriter, loc, resultTy, input, reverse2, bias, op.getInputZp(),
+        op.getWeightZp(), rewriter.getDenseI64ArrayAttr(convPad),
         rewriter.getDenseI64ArrayAttr(stride),
         rewriter.getDenseI64ArrayAttr({1, 1}),
         /* acc_type = */ op.getAccType());
@@ -216,8 +218,8 @@ class TransposeConvStridedConverter
         inputPaddingVal, inputPadConst);
 
     // We use a zero bias as we need to broadcast the bias.
-    auto zeroBias = tosa::ConstOp::create(rewriter,
-        loc,
+    auto zeroBias = tosa::ConstOp::create(
+        rewriter, loc,
         RankedTensorType::get({outputChannels * stride[0] * stride[1]},
                               biasETy),
         DenseElementsAttr::get(
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index a281c6600fdf6..581ac0466aa7a 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -403,8 +403,8 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
     return std::nullopt;
   }
   ImplicitLocOpBuilder builder(reshapeOp.getLoc(), rewriter);
-  auto foldedReshape = ReshapeOp::create(rewriter,
-      reshapeOp.getLoc(),
+  auto foldedReshape = ReshapeOp::create(
+      rewriter, reshapeOp.getLoc(),
       RankedTensorType::get(applyTOSAPermutation(shape, hoistedPerms),
                             reshapeOutputType.getElementType()),
       reshapeOp.getInput1(),
@@ -423,8 +423,8 @@ std::optional<Value> TosaReduceTransposes::buildMappedToValue(
   if (!maybeNewDenseAttr.has_value())
     return std::nullopt;
   auto newDenseAttr = maybeNewDenseAttr.value();
-  auto newConstOp = ConstOp::create(rewriter,
-      constOp.getLoc(), newDenseAttr.getType(), newDenseAttr);
+  auto newConstOp = ConstOp::create(rewriter, constOp.getLoc(),
+                                    newDenseAttr.getType(), newDenseAttr);
   return newConstOp->getResult(0);
 }
 
diff --git a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
index 69ef0ba40d72e..69eda03e03ab3 100644
--- a/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
@@ -144,8 +144,8 @@ LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder,
       ArrayRef<int64_t>(reshapeOutputShape), reshapeInputType.getElementType());
   auto reshapeOutputShapeValue = getTosaConstShape(builder, reshapeOutputShape);
 
-  auto reshapeLower = tosa::ReshapeOp::create(builder,
-      reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
+  auto reshapeLower = tosa::ReshapeOp::create(
+      builder, reshapeOutputType, lowerTensorValue, reshapeOutputShapeValue);
 
   if (input1Rank > input2Rank) {
     input1 = higherTensorValue;
diff --git a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
index 696dbe3d1892a..2bb2ea046fc1f 100644
--- a/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/LoopExtension/LoopExtensionOps.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h"
 
-#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
index c750017a6b504..66196194b0585 100644
--- a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -116,8 +116,8 @@ struct TransferWriteOpInterface
         getBuffer(rewriter, writeOp.getBase(), options, state);
     if (failed(resultBuffer))
       return failure();
-    vector::TransferWriteOp::create(rewriter,
-        writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
+    vector::TransferWriteOp::create(
+        rewriter, writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
         writeOp.getIndices(), writeOp.getPermutationMapAttr(),
         writeOp.getMask(), writeOp.getInBoundsAttr());
     replaceOpWithBufferizedValues(rewriter, op, *resultBuffer);
@@ -241,8 +241,9 @@ struct MaskOpInterface
     // Create a new vector.mask op.
     ValueRange newYieldedValuesRange(newYieldedValues);
     TypeRange newResultTypes(newYieldedValuesRange);
-    auto newOp = vector::MaskOp::create(rewriter,
-        op->getLoc(), newResultTypes, maskOp.getMask(), maskOp.getPassthru(),
+    auto newOp = vector::MaskOp::create(
+        rewriter, op->getLoc(), newResultTypes, maskOp.getMask(),
+        maskOp.getPassthru(),
         /*maskableOp=*/nullptr,
         /*maskRegionBuilder=*/[](OpBuilder &b, Operation *) {});
     newOp.getRegion().takeBody(maskOp.getMaskRegion());
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
index b2486c5cbbcf8..ad4d12eed0d3e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
@@ -95,8 +95,8 @@ static Value reshapeLoad(Location loc, Value val, VectorType type,
   // Unroll leading dimensions.
   VectorType vType = VectorType::Builder(type).dropDim(0);
   VectorType resType = VectorType::Builder(type).dropDim(index);
-  Value result = arith::ConstantOp::create(rewriter,
-      loc, resType, rewriter.getZeroAttr(resType));
+  Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                           rewriter.getZeroAttr(resType));
   for (int64_t d = 0, e = resType.getDimSize(0); d < e; d++) {
     Value ext = vector::ExtractOp::create(rewriter, loc, val, d);
     Value load = reshapeLoad(loc, ext, vType, index - 1, pos, rewriter);
@@ -447,8 +447,8 @@ struct UnrolledOuterProductGenerator
         extractMask =
             vector::ExtractOp::create(rewriter, loc, maybeMask.value(), k);
 
-      Operation *outerProdOp = vector::OuterProductOp::create(rewriter,
-          loc, res.getType(), extractA, extractB, res, kind);
+      Operation *outerProdOp = vector::OuterProductOp::create(
+          rewriter, loc, res.getType(), extractA, extractB, res, kind);
       res = maskOperation(rewriter, outerProdOp, extractMask)->getResult(0);
     }
     return res;
@@ -755,7 +755,7 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
 
   // ExtractOp does not allow dynamic indexing, we must unroll explicitly.
   Value res = arith::ConstantOp::create(rewriter, loc, dstType,
-                                                 rewriter.getZeroAttr(dstType));
+                                        rewriter.getZeroAttr(dstType));
   bool isInt = isa<IntegerType>(dstType.getElementType());
   llvm::SmallVector<Value> extractedCols;
   extractedCols.reserve(dstColumns);
@@ -766,15 +766,16 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
       // avoid having duplicate SSA values pointing to the same rows/columns.
       if (r == 0) {
         Value colRhs =
-            rank == 1 ? rhs
-                      : vector::ExtractOp::create(rewriter, op.getLoc(), rhs, c);
+            rank == 1
+                ? rhs
+                : vector::ExtractOp::create(rewriter, op.getLoc(), rhs, c);
         extractedCols.push_back(colRhs);
       }
       Value extractedColRhs = extractedCols[c];
       Value product =
           createMul(op.getLoc(), rowLhs, extractedColRhs, isInt, rewriter);
-      Value sum = vector::ReductionOp::create(rewriter,
-          op.getLoc(), vector::CombiningKind::ADD, product);
+      Value sum = vector::ReductionOp::create(
+          rewriter, op.getLoc(), vector::CombiningKind::ADD, product);
 
       SmallVector<int64_t, 2> pos = rank == 1 ? SmallVector<int64_t, 2>{r}
                                               : SmallVector<int64_t, 2>{r, c};
@@ -1097,8 +1098,8 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       rewriter.getArrayAttr(adjustIter(op.getIteratorTypes(), iterIndex));
   // Unroll into a series of lower dimensional vector.contract ops.
   Location loc = op.getLoc();
-  Value result = arith::ConstantOp::create(rewriter,
-      loc, resType, rewriter.getZeroAttr(resType));
+  Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                           rewriter.getZeroAttr(resType));
 
   for (int64_t d = 0; d < dimSize; ++d) {
     auto lhs = reshapeLoad(loc, op.getLhs(), lhsType, lhsIndex, d, rewriter);
@@ -1110,8 +1111,8 @@ FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
       lowMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *lowContract = vector::ContractionOp::create(rewriter,
-        loc, lhs, rhs, acc, lowAffine, lowIter);
+    Operation *lowContract = vector::ContractionOp::create(
+        rewriter, loc, lhs, rhs, acc, lowAffine, lowIter);
     lowContract = maskOperation(rewriter, lowContract, lowMask);
     result = reshapeStore(loc, lowContract->getResult(0), result, resType,
                           resIndex, d, rewriter);
@@ -1186,8 +1187,8 @@ FailureOr<Value> ContractionOpLowering::lowerReduction(
       newMask = reshapeLoad(loc, mask, cast<VectorType>(mask.getType()),
                             iterIndex, d, rewriter);
 
-    Operation *newContract = vector::ContractionOp::create(rewriter,
-        loc, lhs, rhs, result, lowAffine, lowIter);
+    Operation *newContract = vector::ContractionOp::create(
+        rewriter, loc, lhs, rhs, result, lowAffine, lowIter);
     result = maskOperation(rewriter, newContract, newMask)->getResult(0);
   }
   return result;
@@ -1240,7 +1241,8 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
 
     if (!rhsType) {
       // Special case: AXPY operation.
-      Value b = vector::BroadcastOp::create(rewriter, loc, lhsType, op.getRhs());
+      Value b =
+          vector::BroadcastOp::create(rewriter, loc, lhsType, op.getRhs());
       std::optional<Value> mult = createContractArithOp(
           loc, op.getLhs(), b, acc, kind, rewriter, isInt, mask);
       if (!mult.has_value())
@@ -1249,8 +1251,8 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
       return success();
     }
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resType, rewriter.getZeroAttr(resType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                             rewriter.getZeroAttr(resType));
     for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
       Value x = vector::ExtractOp::create(rewriter, loc, op.getLhs(), d);
       Value a = vector::BroadcastOp::create(rewriter, loc, rhsType, x);
@@ -1363,9 +1365,9 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   rhs = vector::ShapeCastOp::create(rew, loc, flattenedRHSType, rhs);
 
   Value mul = vector::MatmulOp::create(rew, loc, lhs, rhs, lhsRows, lhsColumns,
-                                           rhsColumns);
-  mul = vector::ShapeCastOp::create(rew,
-      loc,
+                                       rhsColumns);
+  mul = vector::ShapeCastOp::create(
+      rew, loc,
       VectorType::get({lhsRows, rhsColumns},
                       getElementTypeOrSelf(op.getAcc().getType())),
       mul);
@@ -1377,11 +1379,11 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   else if (accMap != AffineMap::get(3, 0, {m, n}, ctx))
     llvm_unreachable("invalid contraction semantics");
 
-  Value res =
-      isa<IntegerType>(elementType)
-          ? static_cast<Value>(arith::AddIOp::create(rew, loc, op.getAcc(), mul))
-          : static_cast<Value>(
-                arith::AddFOp::create(rew, loc, op.getAcc(), mul));
+  Value res = isa<IntegerType>(elementType)
+                  ? static_cast<Value>(
+                        arith::AddIOp::create(rew, loc, op.getAcc(), mul))
+                  : static_cast<Value>(
+                        arith::AddFOp::create(rew, loc, op.getAcc(), mul));
 
   return res;
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index c586059ca55bc..b028940de1741 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -75,8 +75,8 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
     Value maskVec = op.getMask();
     Value passThruVec = op.getPassThru();
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resultTy, rewriter.getZeroAttr(resultTy));
+    Value result = arith::ConstantOp::create(rewriter, loc, resultTy,
+                                             rewriter.getZeroAttr(resultTy));
 
     VectorType subTy = VectorType::Builder(resultTy).dropDim(0);
 
@@ -89,9 +89,9 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
           vector::ExtractOp::create(rewriter, loc, maskVec, thisIdx);
       Value passThruSubVec =
           vector::ExtractOp::create(rewriter, loc, passThruVec, thisIdx);
-      Value subGather = vector::GatherOp::create(rewriter,
-          loc, subTy, op.getBase(), op.getIndices(), indexSubVec, maskSubVec,
-          passThruSubVec);
+      Value subGather = vector::GatherOp::create(
+          rewriter, loc, subTy, op.getBase(), op.getIndices(), indexSubVec,
+          maskSubVec, passThruSubVec);
       result =
           vector::InsertOp::create(rewriter, loc, subGather, result, thisIdx);
     }
@@ -159,24 +159,24 @@ struct RemoveStrideFromGatherSource : OpRewritePattern<vector::GatherOp> {
 
     // 1. Collapse the input memref so that it's "flat".
     SmallVector<ReassociationIndices> reassoc = {{0, 1}};
-    Value collapsed = memref::CollapseShapeOp::create(rewriter,
-        op.getLoc(), subview.getSource(), reassoc);
+    Value collapsed = memref::CollapseShapeOp::create(
+        rewriter, op.getLoc(), subview.getSource(), reassoc);
 
     // 2. Generate new gather indices that will model the
     // strided access.
     IntegerAttr stride = rewriter.getIndexAttr(srcTrailingDim);
     VectorType vType = op.getIndexVec().getType();
-    Value mulCst = arith::ConstantOp::create(rewriter,
-        op.getLoc(), vType, DenseElementsAttr::get(vType, stride));
+    Value mulCst = arith::ConstantOp::create(
+        rewriter, op.getLoc(), vType, DenseElementsAttr::get(vType, stride));
 
     Value newIdxs =
         arith::MulIOp::create(rewriter, op.getLoc(), op.getIndexVec(), mulCst);
 
     // 3. Create an updated gather op with the collapsed input memref and the
     // updated indices.
-    Value newGather = vector::GatherOp::create(rewriter,
-        op.getLoc(), op.getResult().getType(), collapsed, op.getIndices(),
-        newIdxs, op.getMask(), op.getPassThru());
+    Value newGather = vector::GatherOp::create(
+        rewriter, op.getLoc(), op.getResult().getType(), collapsed,
+        op.getIndices(), newIdxs, op.getMask(), op.getPassThru());
     rewriter.replaceOp(op, newGather);
 
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index b0affbb699d9e..9d6a865a9301f 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -60,11 +60,13 @@ class UnrollInterleaveOp final : public OpRewritePattern<vector::InterleaveOp> {
       return failure();
 
     auto loc = op.getLoc();
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resultType, rewriter.getZeroAttr(resultType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resultType,
+                                             rewriter.getZeroAttr(resultType));
     for (auto position : *unrollIterator) {
-      Value extractLhs = ExtractOp::create(rewriter, loc, op.getLhs(), position);
-      Value extractRhs = ExtractOp::create(rewriter, loc, op.getRhs(), position);
+      Value extractLhs =
+          ExtractOp::create(rewriter, loc, op.getLhs(), position);
+      Value extractRhs =
+          ExtractOp::create(rewriter, loc, op.getRhs(), position);
       Value interleave =
           InterleaveOp::create(rewriter, loc, extractLhs, extractRhs);
       result = InsertOp::create(rewriter, loc, interleave, result, position);
@@ -123,8 +125,8 @@ class UnrollDeinterleaveOp final
       return failure();
 
     auto loc = op.getLoc();
-    Value emptyResult = arith::ConstantOp::create(rewriter,
-        loc, resultType, rewriter.getZeroAttr(resultType));
+    Value emptyResult = arith::ConstantOp::create(
+        rewriter, loc, resultType, rewriter.getZeroAttr(resultType));
     Value evenResult = emptyResult;
     Value oddResult = emptyResult;
 
@@ -133,10 +135,10 @@ class UnrollDeinterleaveOp final
           vector::ExtractOp::create(rewriter, loc, op.getSource(), position);
       auto deinterleave =
           vector::DeinterleaveOp::create(rewriter, loc, extractSrc);
-      evenResult = vector::InsertOp::create(rewriter,
-          loc, deinterleave.getRes1(), evenResult, position);
-      oddResult = vector::InsertOp::create(rewriter, loc, deinterleave.getRes2(),
-                                                    oddResult, position);
+      evenResult = vector::InsertOp::create(
+          rewriter, loc, deinterleave.getRes1(), evenResult, position);
+      oddResult = vector::InsertOp::create(
+          rewriter, loc, deinterleave.getRes2(), oddResult, position);
     }
     rewriter.replaceOp(op, ValueRange{evenResult, oddResult});
     return success();
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
index edd840d8fbc7b..45ef7f01a85f1 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMask.cpp
@@ -67,18 +67,19 @@ class CreateMaskOpLowering : public OpRewritePattern<vector::CreateMaskOp> {
     Value idx = op.getOperand(0);
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = vector::CreateMaskOp::create(rewriter,
-        loc, lowType, op.getOperands().drop_front());
-    Value falseVal = arith::ConstantOp::create(rewriter,
-        loc, lowType, rewriter.getZeroAttr(lowType));
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, dstType, rewriter.getZeroAttr(dstType));
+    Value trueVal = vector::CreateMaskOp::create(rewriter, loc, lowType,
+                                                 op.getOperands().drop_front());
+    Value falseVal = arith::ConstantOp::create(rewriter, loc, lowType,
+                                               rewriter.getZeroAttr(lowType));
+    Value result = arith::ConstantOp::create(rewriter, loc, dstType,
+                                             rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < dim; d++) {
       Value bnd =
           arith::ConstantOp::create(rewriter, loc, rewriter.getIndexAttr(d));
-      Value val = arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
-                                                 bnd, idx);
-      Value sel = arith::SelectOp::create(rewriter, loc, val, trueVal, falseVal);
+      Value val = arith::CmpIOp::create(rewriter, loc,
+                                        arith::CmpIPredicate::slt, bnd, idx);
+      Value sel =
+          arith::SelectOp::create(rewriter, loc, val, trueVal, falseVal);
       result = vector::InsertOp::create(rewriter, loc, sel, result, d);
     }
     rewriter.replaceOp(op, result);
@@ -146,10 +147,10 @@ class ConstantMaskOpLowering : public OpRewritePattern<vector::ConstantMaskOp> {
           op, "Cannot unroll leading scalable dim in dstType");
 
     VectorType lowType = VectorType::Builder(dstType).dropDim(0);
-    Value trueVal = vector::ConstantMaskOp::create(rewriter,
-        loc, lowType, dimSizes.drop_front());
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, dstType, rewriter.getZeroAttr(dstType));
+    Value trueVal = vector::ConstantMaskOp::create(rewriter, loc, lowType,
+                                                   dimSizes.drop_front());
+    Value result = arith::ConstantOp::create(rewriter, loc, dstType,
+                                             rewriter.getZeroAttr(dstType));
     for (int64_t d = 0; d < trueDimSize; d++)
       result = vector::InsertOp::create(rewriter, loc, trueVal, result, d);
 
@@ -261,8 +262,8 @@ struct MaskedGatherOpPattern : public MaskOpRewritePattern<GatherOp> {
                             PatternRewriter &rewriter) const override {
     Value passthru = maskingOp.hasPassthru()
                          ? maskingOp.getPassthru()
-                         : arith::ConstantOp::create(rewriter,
-                               gatherOp.getLoc(),
+                         : arith::ConstantOp::create(
+                               rewriter, gatherOp.getLoc(),
                                rewriter.getZeroAttr(gatherOp.getVectorType()));
 
     // Replace the `vector.mask` operation.
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
index c5631ba00bbc6..4773732d8d9a6 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp
@@ -103,8 +103,8 @@ class InnerOuterDimReductionConversion
     // If masked, transpose the original mask.
     Value transposedMask;
     if (maskableOp.isMasked()) {
-      transposedMask = vector::TransposeOp::create(rewriter,
-          loc, maskableOp.getMaskingOp().getMask(), indices);
+      transposedMask = vector::TransposeOp::create(
+          rewriter, loc, maskableOp.getMaskingOp().getMask(), indices);
     }
 
     // Transpose reduction source.
@@ -117,8 +117,8 @@ class InnerOuterDimReductionConversion
         reductionMask[i] = true;
     }
 
-    Operation *newMultiRedOp = vector::MultiDimReductionOp::create(rewriter,
-        multiReductionOp.getLoc(), transposeOp.getResult(),
+    Operation *newMultiRedOp = vector::MultiDimReductionOp::create(
+        rewriter, multiReductionOp.getLoc(), transposeOp.getResult(),
         multiReductionOp.getAcc(), reductionMask, multiReductionOp.getKind());
     newMultiRedOp =
         mlir::vector::maskOperation(rewriter, newMultiRedOp, transposedMask);
@@ -255,15 +255,15 @@ class ReduceMultiDimReductionRank
       auto maskCastedType = VectorType::get(
           vectorShape,
           llvm::cast<VectorType>(vectorMask.getType()).getElementType());
-      newVectorMask =
-          vector::ShapeCastOp::create(rewriter, loc, maskCastedType, vectorMask);
+      newVectorMask = vector::ShapeCastOp::create(rewriter, loc, maskCastedType,
+                                                  vectorMask);
     }
 
     auto castedType = VectorType::get(
         vectorShape, multiReductionOp.getSourceVectorType().getElementType(),
         scalableDims);
-    Value cast = vector::ShapeCastOp::create(rewriter,
-        loc, castedType, multiReductionOp.getSource());
+    Value cast = vector::ShapeCastOp::create(rewriter, loc, castedType,
+                                             multiReductionOp.getSource());
 
     Value acc = multiReductionOp.getAcc();
     if (flattenedParallelDim) {
@@ -275,8 +275,8 @@ class ReduceMultiDimReductionRank
     }
     // 6. Creates the flattened form of vector.multi_reduction with inner/outer
     // most dim as reduction.
-    Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(rewriter,
-        loc, cast, acc, mask, multiReductionOp.getKind());
+    Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(
+        rewriter, loc, cast, acc, mask, multiReductionOp.getKind());
     newMultiDimRedOp =
         mlir::vector::maskOperation(rewriter, newMultiDimRedOp, newVectorMask);
 
@@ -339,8 +339,8 @@ struct TwoDimMultiReductionToElementWise
 
     Value result = multiReductionOp.getAcc();
     for (int64_t i = 0; i < srcShape[0]; i++) {
-      auto operand = vector::ExtractOp::create(rewriter,
-          loc, multiReductionOp.getSource(), i);
+      auto operand = vector::ExtractOp::create(rewriter, loc,
+                                               multiReductionOp.getSource(), i);
       Value extractMask = nullptr;
       if (mask) {
         extractMask = vector::ExtractOp::create(rewriter, loc, mask, i);
@@ -383,28 +383,29 @@ struct TwoDimMultiReductionToReduction
     }
 
     auto loc = multiReductionOp.getLoc();
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, multiReductionOp.getDestType(),
+    Value result = arith::ConstantOp::create(
+        rewriter, loc, multiReductionOp.getDestType(),
         rewriter.getZeroAttr(multiReductionOp.getDestType()));
     int outerDim = multiReductionOp.getSourceVectorType().getShape()[0];
 
     for (int i = 0; i < outerDim; ++i) {
-      auto v = vector::ExtractOp::create(rewriter,
-          loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
-      auto acc = vector::ExtractOp::create(rewriter,
-          loc, multiReductionOp.getAcc(), ArrayRef<int64_t>{i});
-      Operation *reductionOp = vector::ReductionOp::create(rewriter,
-          loc, multiReductionOp.getKind(), v, acc);
+      auto v = vector::ExtractOp::create(
+          rewriter, loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
+      auto acc = vector::ExtractOp::create(
+          rewriter, loc, multiReductionOp.getAcc(), ArrayRef<int64_t>{i});
+      Operation *reductionOp = vector::ReductionOp::create(
+          rewriter, loc, multiReductionOp.getKind(), v, acc);
 
       // If masked, slice the mask and mask the new reduction operation.
       if (maskableOp.isMasked()) {
-        Value mask = vector::ExtractOp::create(rewriter,
-            loc, maskableOp.getMaskingOp().getMask(), ArrayRef<int64_t>{i});
+        Value mask = vector::ExtractOp::create(
+            rewriter, loc, maskableOp.getMaskingOp().getMask(),
+            ArrayRef<int64_t>{i});
         reductionOp = mlir::vector::maskOperation(rewriter, reductionOp, mask);
       }
 
-      result = vector::InsertOp::create(rewriter, loc, reductionOp->getResult(0),
-                                                 result, i);
+      result = vector::InsertOp::create(rewriter, loc,
+                                        reductionOp->getResult(0), result, i);
     }
 
     rewriter.replaceOp(rootOp, result);
@@ -459,10 +460,10 @@ struct OneDimMultiReductionToTwoDim
     SmallVector<bool, 2> reductionMask{false, true};
 
     /// vector.extract(vector.multi_reduce(vector.shape_cast(v, 1xk)), 0)
-    Value cast = vector::ShapeCastOp::create(rewriter,
-        loc, castedType, multiReductionOp.getSource());
-    Value castAcc = vector::BroadcastOp::create(rewriter,
-        loc, accType, multiReductionOp.getAcc());
+    Value cast = vector::ShapeCastOp::create(rewriter, loc, castedType,
+                                             multiReductionOp.getSource());
+    Value castAcc = vector::BroadcastOp::create(rewriter, loc, accType,
+                                                multiReductionOp.getAcc());
     Value castMask;
     if (maskableOp.isMasked()) {
       auto maskType = llvm::cast<VectorType>(mask.getType());
@@ -473,8 +474,9 @@ struct OneDimMultiReductionToTwoDim
       castMask = vector::BroadcastOp::create(rewriter, loc, castMaskType, mask);
     }
 
-    Operation *newOp = vector::MultiDimReductionOp::create(rewriter,
-        loc, cast, castAcc, reductionMask, multiReductionOp.getKind());
+    Operation *newOp = vector::MultiDimReductionOp::create(
+        rewriter, loc, cast, castAcc, reductionMask,
+        multiReductionOp.getKind());
     newOp = vector::maskOperation(rewriter, newOp, castMask);
 
     rewriter.replaceOpWithNewOp<vector::ExtractOp>(rootOp, newOp->getResult(0),
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
index a6985b96b59df..4f6391b78b947 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
@@ -122,8 +122,8 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
       return failure();
 
     VectorType resType = VectorType::get(destShape, elType);
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resType, rewriter.getZeroAttr(resType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                             rewriter.getZeroAttr(resType));
     int64_t reductionDim = scanOp.getReductionDim();
     bool inclusive = scanOp.getInclusive();
     int64_t destRank = destType.getRank();
@@ -144,9 +144,9 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
     for (int i = 0; i < destShape[reductionDim]; i++) {
       offsets[reductionDim] = i;
       ArrayAttr scanOffsets = rewriter.getI64ArrayAttr(offsets);
-      Value input = vector::ExtractStridedSliceOp::create(rewriter,
-          loc, reductionType, scanOp.getSource(), scanOffsets, scanSizes,
-          scanStrides);
+      Value input = vector::ExtractStridedSliceOp::create(
+          rewriter, loc, reductionType, scanOp.getSource(), scanOffsets,
+          scanSizes, scanStrides);
       Value output;
       if (i == 0) {
         if (inclusive) {
@@ -154,11 +154,11 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         } else {
           if (initialValueRank == 0) {
             // ShapeCastOp cannot handle 0-D vectors
-            output = vector::BroadcastOp::create(rewriter,
-                loc, input.getType(), scanOp.getInitialValue());
+            output = vector::BroadcastOp::create(rewriter, loc, input.getType(),
+                                                 scanOp.getInitialValue());
           } else {
-            output = vector::ShapeCastOp::create(rewriter,
-                loc, input.getType(), scanOp.getInitialValue());
+            output = vector::ShapeCastOp::create(rewriter, loc, input.getType(),
+                                                 scanOp.getInitialValue());
           }
         }
       } else {
@@ -166,8 +166,8 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
         output = vector::makeArithReduction(rewriter, loc, scanOp.getKind(),
                                             lastOutput, y);
       }
-      result = vector::InsertStridedSliceOp::create(rewriter,
-          loc, output, result, offsets, strides);
+      result = vector::InsertStridedSliceOp::create(rewriter, loc, output,
+                                                    result, offsets, strides);
       lastOutput = output;
       lastInput = input;
     }
@@ -179,7 +179,7 @@ struct ScanToArithOps : public OpRewritePattern<vector::ScanOp> {
           vector::BroadcastOp::create(rewriter, loc, initialValueType, v);
     } else {
       reduction = vector::ShapeCastOp::create(rewriter, loc, initialValueType,
-                                                       lastOutput);
+                                              lastOutput);
     }
 
     rewriter.replaceOp(scanOp, {result, reduction});
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
index 69d3666a13879..603ea41d43360 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
@@ -138,10 +138,11 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 
     const Value source = shapeCast.getSource();
     const Value poison = ub::PoisonOp::create(rewriter, loc, resultType);
-    const Value extracted = vector::ExtractOp::create(rewriter,
-        loc, source, SmallVector<int64_t>(sourceLeading, 0));
-    const Value result = vector::InsertOp::create(rewriter,
-        loc, extracted, poison, SmallVector<int64_t>(resultLeading, 0));
+    const Value extracted = vector::ExtractOp::create(
+        rewriter, loc, source, SmallVector<int64_t>(sourceLeading, 0));
+    const Value result =
+        vector::InsertOp::create(rewriter, loc, extracted, poison,
+                                 SmallVector<int64_t>(resultLeading, 0));
 
     rewriter.replaceOp(shapeCast, result);
     return success();
@@ -178,7 +179,7 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
           vector::ExtractOp::create(rewriter, loc, source, extractIndex);
 
       result = vector::InsertOp::create(rewriter, loc, extracted, result,
-                                                 insertIndex);
+                                        insertIndex);
 
       inplaceAdd(1, sourceShape.take_front(sourceDim), extractIndex);
       inplaceAdd(1, resultShape.take_front(resultDim), insertIndex);
@@ -295,21 +296,21 @@ class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 
       // vector.extract_strided_slice
       extractOffsets[0] = extractStridedPhase * greatestCommonDivisor;
-      extractedStrided = vector::ExtractStridedSliceOp::create(rewriter,
-          loc, extracted, extractOffsets, atomicShape, sizes);
+      extractedStrided = vector::ExtractStridedSliceOp::create(
+          rewriter, loc, extracted, extractOffsets, atomicShape, sizes);
 
       // vector.insert_strided_slice
       if (insertStridedPhase == 0) {
         insertedSlice = partResult;
       }
       insertOffsets[0] = insertStridedPhase * greatestCommonDivisor;
-      insertedSlice = vector::InsertStridedSliceOp::create(rewriter,
-          loc, extractedStrided, insertedSlice, insertOffsets, sizes);
+      insertedSlice = vector::InsertStridedSliceOp::create(
+          rewriter, loc, extractedStrided, insertedSlice, insertOffsets, sizes);
 
       // vector.insert
       if (insertStridedPhase + 1 == insertPeriod) {
         result = vector::InsertOp::create(rewriter, loc, insertedSlice, result,
-                                                   insertIndex);
+                                          insertIndex);
         inplaceAdd(1, resultType.getShape().take_front(resultSuffixStartDim),
                    insertIndex);
       }
@@ -406,16 +407,18 @@ class ScalableShapeCastOpRewritePattern
       // 1. Extract a scalable subvector from the source vector.
       if (!currentSourceScalableVector) {
         if (srcRank != 1) {
-          currentSourceScalableVector = vector::ExtractOp::create(rewriter,
-              loc, op.getSource(), llvm::ArrayRef(srcIdx).drop_back());
+          currentSourceScalableVector =
+              vector::ExtractOp::create(rewriter, loc, op.getSource(),
+                                        llvm::ArrayRef(srcIdx).drop_back());
         } else {
           currentSourceScalableVector = op.getSource();
         }
       }
       Value sourceSubVector = currentSourceScalableVector;
       if (minExtractionSize < minSourceTrailingSize) {
-        sourceSubVector = vector::ScalableExtractOp::create(rewriter,
-            loc, extractionVectorType, sourceSubVector, srcIdx.back());
+        sourceSubVector = vector::ScalableExtractOp::create(
+            rewriter, loc, extractionVectorType, sourceSubVector,
+            srcIdx.back());
       }
 
       // 2. Insert the scalable subvector into the result vector.
@@ -423,15 +426,16 @@ class ScalableShapeCastOpRewritePattern
         if (minExtractionSize == minResultTrailingSize) {
           currentResultScalableVector = sourceSubVector;
         } else if (resRank != 1) {
-          currentResultScalableVector = vector::ExtractOp::create(rewriter,
-              loc, result, llvm::ArrayRef(resIdx).drop_back());
+          currentResultScalableVector = vector::ExtractOp::create(
+              rewriter, loc, result, llvm::ArrayRef(resIdx).drop_back());
         } else {
           currentResultScalableVector = result;
         }
       }
       if (minExtractionSize < minResultTrailingSize) {
-        currentResultScalableVector = vector::ScalableInsertOp::create(rewriter,
-            loc, sourceSubVector, currentResultScalableVector, resIdx.back());
+        currentResultScalableVector = vector::ScalableInsertOp::create(
+            rewriter, loc, sourceSubVector, currentResultScalableVector,
+            resIdx.back());
       }
 
       // 3. Update the source and result scalable vectors if needed.
@@ -439,9 +443,9 @@ class ScalableShapeCastOpRewritePattern
           currentResultScalableVector != result) {
         // Finished row of result. Insert complete scalable vector into result
         // (n-D) vector.
-        result = vector::InsertOp::create(rewriter,
-            loc, currentResultScalableVector, result,
-            llvm::ArrayRef(resIdx).drop_back());
+        result = vector::InsertOp::create(rewriter, loc,
+                                          currentResultScalableVector, result,
+                                          llvm::ArrayRef(resIdx).drop_back());
         currentResultScalableVector = {};
       }
       if (srcIdx.back() + minExtractionSize >= minSourceTrailingSize) {
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
index fe02cd5656571..290a131bc496d 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
@@ -138,8 +138,8 @@ struct TransferReadPermutationLowering
     // Generate new transfer_read operation.
     VectorType newReadType = VectorType::get(
         newVectorShape, op.getVectorType().getElementType(), newScalableDims);
-    Value newRead = vector::TransferReadOp::create(rewriter,
-        op.getLoc(), newReadType, op.getBase(), op.getIndices(),
+    Value newRead = vector::TransferReadOp::create(
+        rewriter, op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
 
@@ -209,12 +209,12 @@ struct TransferWritePermutationLowering
         inverseTransposeInBoundsAttr(rewriter, op.getInBounds(), permutation);
 
     // Generate new transfer_write operation.
-    Value newVec = vector::TransposeOp::create(rewriter,
-        op.getLoc(), op.getVector(), indices);
+    Value newVec = vector::TransposeOp::create(rewriter, op.getLoc(),
+                                               op.getVector(), indices);
     auto newMap = AffineMap::getMinorIdentityMap(
         map.getNumDims(), map.getNumResults(), rewriter.getContext());
-    auto newWrite = vector::TransferWriteOp::create(rewriter,
-        op.getLoc(), newVec, op.getBase(), op.getIndices(),
+    auto newWrite = vector::TransferWriteOp::create(
+        rewriter, op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getMask(), newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
       return newWrite.getResult();
@@ -299,8 +299,8 @@ struct TransferWriteNonPermutationLowering
       newInBoundsValues.push_back(op.isDimInBounds(i));
     }
     ArrayAttr newInBoundsAttr = rewriter.getBoolArrayAttr(newInBoundsValues);
-    auto newWrite = vector::TransferWriteOp::create(rewriter,
-        op.getLoc(), newVec, op.getBase(), op.getIndices(),
+    auto newWrite = vector::TransferWriteOp::create(
+        rewriter, op.getLoc(), newVec, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), newMask, newInBoundsAttr);
     if (newWrite.hasPureTensorSemantics())
       return newWrite.getResult();
@@ -370,8 +370,8 @@ struct TransferOpReduceRank
             ? rewriter.getArrayAttr(
                   op.getInBoundsAttr().getValue().take_back(reducedShapeRank))
             : ArrayAttr();
-    Value newRead = vector::TransferReadOp::create(rewriter,
-        op.getLoc(), newReadType, op.getBase(), op.getIndices(),
+    Value newRead = vector::TransferReadOp::create(
+        rewriter, op.getLoc(), newReadType, op.getBase(), op.getIndices(),
         AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
         newInBoundsAttr);
     return rewriter
@@ -471,21 +471,21 @@ struct TransferReadToVectorLoadLowering
             read, "vector type is not rank 1, can't create masked load, needs "
                   "VectorToSCF");
 
-      Value fill = vector::SplatOp::create(rewriter,
-          read.getLoc(), unbroadcastedVectorType, read.getPadding());
-      res = vector::MaskedLoadOp::create(rewriter,
-          read.getLoc(), unbroadcastedVectorType, read.getBase(),
+      Value fill = vector::SplatOp::create(
+          rewriter, read.getLoc(), unbroadcastedVectorType, read.getPadding());
+      res = vector::MaskedLoadOp::create(
+          rewriter, read.getLoc(), unbroadcastedVectorType, read.getBase(),
           read.getIndices(), read.getMask(), fill);
     } else {
       res = vector::LoadOp::create(rewriter, read.getLoc(),
-                                            unbroadcastedVectorType,
-                                            read.getBase(), read.getIndices());
+                                   unbroadcastedVectorType, read.getBase(),
+                                   read.getIndices());
     }
 
     // Insert a broadcasting op if required.
     if (!broadcastedDims.empty())
-      res = vector::BroadcastOp::create(rewriter,
-          read.getLoc(), read.getVectorType(), res->getResult(0));
+      res = vector::BroadcastOp::create(
+          rewriter, read.getLoc(), read.getVectorType(), res->getResult(0));
     return res->getResult(0);
   }
 
@@ -569,12 +569,12 @@ struct TransferWriteToVectorStoreLowering
                    << write;
             });
 
-      vector::MaskedStoreOp::create(rewriter,
-          write.getLoc(), write.getBase(), write.getIndices(), write.getMask(),
-          write.getVector());
+      vector::MaskedStoreOp::create(rewriter, write.getLoc(), write.getBase(),
+                                    write.getIndices(), write.getMask(),
+                                    write.getVector());
     } else {
       vector::StoreOp::create(rewriter, write.getLoc(), write.getVector(),
-                                       write.getBase(), write.getIndices());
+                              write.getBase(), write.getIndices());
     }
     // There's no return value for StoreOps. Use Value() to signal success to
     // matchAndRewrite.
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index e9591bfcf6feb..da2aed653a080 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -80,8 +80,8 @@ getUnpackShufflePermFor128Lane(ArrayRef<int64_t> vals, int numBits) {
 static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return vector::ShuffleOp::create(b,
-      v1, v2,
+  return vector::ShuffleOp::create(
+      b, v1, v2,
       getUnpackShufflePermFor128Lane({0, 1, numElem, numElem + 1}, numBits));
 }
 
@@ -94,8 +94,8 @@ static Value createUnpackLoPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return vector::ShuffleOp::create(b,
-      v1, v2,
+  return vector::ShuffleOp::create(
+      b, v1, v2,
       getUnpackShufflePermFor128Lane({2, 3, numElem + 2, numElem + 3},
                                      numBits));
 }
@@ -109,8 +109,8 @@ static Value createUnpackHiPd(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  auto shuffle = vector::ShuffleOp::create(b,
-      v1, v2,
+  auto shuffle = vector::ShuffleOp::create(
+      b, v1, v2,
       getUnpackShufflePermFor128Lane({0, numElem, 1, numElem + 1}, numBits));
   return shuffle;
 }
@@ -124,8 +124,8 @@ static Value createUnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
 static Value createUnpackHiPs(ImplicitLocOpBuilder &b, Value v1, Value v2,
                               int numBits) {
   int numElem = numBits / 32;
-  return vector::ShuffleOp::create(b,
-      v1, v2,
+  return vector::ShuffleOp::create(
+      b, v1, v2,
       getUnpackShufflePermFor128Lane({2, numElem + 2, 3, numElem + 3},
                                      numBits));
 }
@@ -338,8 +338,8 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
           vector::ShapeCastOp::create(rewriter, loc, flattenedType, input);
       auto rows = rewriter.getI32IntegerAttr(resType.getShape()[0]);
       auto columns = rewriter.getI32IntegerAttr(resType.getShape()[1]);
-      Value trans = vector::FlatTransposeOp::create(rewriter,
-          loc, flattenedType, matrix, rows, columns);
+      Value trans = vector::FlatTransposeOp::create(
+          rewriter, loc, flattenedType, matrix, rows, columns);
       rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, trans);
       return success();
     }
@@ -483,7 +483,7 @@ class TransposeOp2DToShuffleLowering
     auto flattenedType = VectorType::get({n * m}, srcType.getElementType());
     auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
     auto reshInput = vector::ShapeCastOp::create(rewriter, loc, flattenedType,
-                                                          op.getVector());
+                                                 op.getVector());
 
     Value res;
     if (vectorTransposeLowering == VectorTransposeLowering::Shuffle16x16 &&
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index 2683a85ecdaeb..e5d53cd87506a 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -127,8 +127,8 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return vector::TransferWriteOp::create(b,
-        loc, val, buffer, indices,
+    return vector::TransferWriteOp::create(
+        b, loc, val, buffer, indices,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
 
@@ -172,8 +172,9 @@ struct DistributedLoadStoreHelper {
       }
     }
     SmallVector<bool> inBounds(indices.size(), true);
-    return vector::TransferReadOp::create(b,
-        loc, cast<VectorType>(type), buffer, indices, /*padding=*/std::nullopt,
+    return vector::TransferReadOp::create(
+        b, loc, cast<VectorType>(type), buffer, indices,
+        /*padding=*/std::nullopt,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
 
@@ -244,10 +245,10 @@ struct WarpOpToScfIfPattern : public WarpDistributionPattern {
 
     // Step 1: Create scf.if op.
     Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
-    Value isLane0 = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::eq, warpOp.getLaneid(), c0);
+    Value isLane0 = arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::eq, warpOp.getLaneid(), c0);
     auto ifOp = scf::IfOp::create(rewriter, loc, isLane0,
-                                           /*withElseRegion=*/false);
+                                  /*withElseRegion=*/false);
     rewriter.eraseOp(ifOp.thenBlock()->getTerminator());
 
     // Step 2: insert appropriate (alloc, write)-pairs before the scf.if and
@@ -512,8 +513,9 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
 
     // Create a second warp op that contains only writeOp.
-    auto secondWarpOp = WarpExecuteOnLane0Op::create(rewriter,
-        loc, TypeRange(), newWarpOp.getLaneid(), newWarpOp.getWarpSize());
+    auto secondWarpOp = WarpExecuteOnLane0Op::create(rewriter, loc, TypeRange(),
+                                                     newWarpOp.getLaneid(),
+                                                     newWarpOp.getWarpSize());
     Block &body = secondWarpOp.getBodyRegion().front();
     rewriter.setInsertionPointToStart(&body);
     auto newWriteOp =
@@ -823,9 +825,9 @@ struct WarpOpTransferRead : public WarpDistributionPattern {
     Value newMask =
         hasMask ? newWarpOp.getResult(newRetIndices[newRetIndices.size() - 1])
                 : Value();
-    auto newRead = vector::TransferReadOp::create(rewriter,
-        read.getLoc(), distributedVal.getType(), read.getBase(), newIndices,
-        read.getPermutationMapAttr(), newPadding, newMask,
+    auto newRead = vector::TransferReadOp::create(
+        rewriter, read.getLoc(), distributedVal.getType(), read.getBase(),
+        newIndices, read.getPermutationMapAttr(), newPadding, newMask,
         read.getInBoundsAttr());
 
     rewriter.replaceAllUsesWith(distributedVal, newRead);
@@ -965,8 +967,8 @@ struct WarpOpBroadcast : public WarpDistributionPattern {
     WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
         rewriter, warpOp, {broadcastSrc}, {broadcastSrcType}, newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value broadcasted = vector::BroadcastOp::create(rewriter,
-        loc, destVecType, newWarpOp->getResult(newRetIndices[0]));
+    Value broadcasted = vector::BroadcastOp::create(
+        rewriter, loc, destVecType, newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 broadcasted);
     return success();
@@ -1008,8 +1010,8 @@ struct WarpOpShapeCast : public WarpDistributionPattern {
         rewriter, warpOp, {oldCastOp.getSource()}, {castDistributedType},
         newRetIndices);
     rewriter.setInsertionPointAfter(newWarpOp);
-    Value newCast = vector::ShapeCastOp::create(rewriter,
-        oldCastOp.getLoc(), castResultType,
+    Value newCast = vector::ShapeCastOp::create(
+        rewriter, oldCastOp.getLoc(), castResultType,
         newWarpOp->getResult(newRetIndices[0]));
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newCast);
     return success();
@@ -1182,9 +1184,10 @@ struct WarpOpInsertStridedSlice : public WarpDistributionPattern {
     Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
     // Create a new insert strided slice op that inserts distributed source into
     // distributed dest.
-    Value newInsert = vector::InsertStridedSliceOp::create(rewriter,
-        insertOp.getLoc(), distributedDest.getType(), distributedSource,
-        distributedDest, insertOp.getOffsets(), insertOp.getStrides());
+    Value newInsert = vector::InsertStridedSliceOp::create(
+        rewriter, insertOp.getLoc(), distributedDest.getType(),
+        distributedSource, distributedDest, insertOp.getOffsets(),
+        insertOp.getStrides());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newInsert);
     return success();
   }
@@ -1277,8 +1280,8 @@ struct WarpOpExtractStridedSlice : public WarpDistributionPattern {
     // Create a new extract strided slice op that extracts from the
     // distributed vector.
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
-    Value newExtract = vector::ExtractStridedSliceOp::create(rewriter,
-        extractOp.getLoc(), distributedType, distributedVec,
+    Value newExtract = vector::ExtractStridedSliceOp::create(
+        rewriter, extractOp.getLoc(), distributedType, distributedVec,
         extractOp.getOffsets(),
         ArrayAttr::get(rewriter.getContext(), distributedSizes),
         extractOp.getStrides());
@@ -1323,8 +1326,8 @@ struct WarpOpExtract : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
       // Extract from distributed vector.
-      Value newExtract = vector::ExtractOp::create(rewriter,
-          loc, distributedVec, extractOp.getMixedPosition());
+      Value newExtract = vector::ExtractOp::create(
+          rewriter, loc, distributedVec, extractOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newExtract);
       return success();
@@ -1352,8 +1355,8 @@ struct WarpOpExtract : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
     // Extract from distributed vector.
-    Value newExtract = vector::ExtractOp::create(rewriter,
-        loc, distributedVec, extractOp.getMixedPosition());
+    Value newExtract = vector::ExtractOp::create(rewriter, loc, distributedVec,
+                                                 extractOp.getMixedPosition());
     rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                 newExtract);
     return success();
@@ -1515,7 +1518,7 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
         indices.push_back(pos);
       }
       newInsert = vector::InsertOp::create(rewriter, loc, newSource,
-                                                    distributedVec, indices);
+                                           distributedVec, indices);
       // Broadcast: Simply move the vector.insert op out.
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newInsert);
@@ -1531,16 +1534,17 @@ struct WarpOpInsertScalar : public WarpDistributionPattern {
     // Insert position: pos % elementsPerLane
     OpFoldResult newPos = affine::makeComposedFoldedAffineApply(
         rewriter, loc, sym0 % elementsPerLane, pos);
-    Value isInsertingLane = arith::CmpIOp::create(rewriter,
-        loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
+    Value isInsertingLane =
+        arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
+                              newWarpOp.getLaneid(), insertingLane);
     Value newResult =
         rewriter
             .create<scf::IfOp>(
                 loc, isInsertingLane,
                 /*thenBuilder=*/
                 [&](OpBuilder &builder, Location loc) {
-                  Value newInsert = vector::InsertOp::create(builder,
-                      loc, newSource, distributedVec, newPos);
+                  Value newInsert = vector::InsertOp::create(
+                      builder, loc, newSource, distributedVec, newPos);
                   scf::YieldOp::create(builder, loc, newInsert);
                 },
                 /*elseBuilder=*/
@@ -1582,8 +1586,9 @@ struct WarpOpInsert : public WarpDistributionPattern {
       rewriter.setInsertionPointAfter(newWarpOp);
       Value distributedSrc = newWarpOp->getResult(newRetIndices[0]);
       Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
-      Value newResult = vector::InsertOp::create(rewriter,
-          loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
+      Value newResult = vector::InsertOp::create(rewriter, loc, distributedSrc,
+                                                 distributedDest,
+                                                 insertOp.getMixedPosition());
       rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
                                   newResult);
       return success();
@@ -1632,23 +1637,25 @@ struct WarpOpInsert : public WarpDistributionPattern {
     Value newResult;
     if (distrSrcDim >= 0) {
       // Every lane inserts a small piece.
-      newResult = vector::InsertOp::create(rewriter,
-          loc, distributedSrc, distributedDest, insertOp.getMixedPosition());
+      newResult = vector::InsertOp::create(rewriter, loc, distributedSrc,
+                                           distributedDest,
+                                           insertOp.getMixedPosition());
     } else {
       // One lane inserts the entire source vector.
       int64_t elementsPerLane = distrDestType.getDimSize(distrDestDim);
       SmallVector<OpFoldResult> pos = insertOp.getMixedPosition();
       SmallVector<int64_t> newPos = getAsIntegers(pos);
       // tid of inserting lane: pos / elementsPerLane
-      Value insertingLane = arith::ConstantIndexOp::create(rewriter,
-          loc, newPos[distrDestDim] / elementsPerLane);
-      Value isInsertingLane = arith::CmpIOp::create(rewriter,
-          loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
+      Value insertingLane = arith::ConstantIndexOp::create(
+          rewriter, loc, newPos[distrDestDim] / elementsPerLane);
+      Value isInsertingLane =
+          arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::eq,
+                                newWarpOp.getLaneid(), insertingLane);
       // Insert position: pos % elementsPerLane
       newPos[distrDestDim] %= elementsPerLane;
       auto insertingBuilder = [&](OpBuilder &builder, Location loc) {
-        Value newInsert = vector::InsertOp::create(builder,
-            loc, distributedSrc, distributedDest, newPos);
+        Value newInsert = vector::InsertOp::create(builder, loc, distributedSrc,
+                                                   distributedDest, newPos);
         scf::YieldOp::create(builder, loc, newInsert);
       };
       auto nonInsertingBuilder = [&](OpBuilder &builder, Location loc) {
@@ -1761,9 +1768,9 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
 
     // Create a new for op outside the region with a WarpExecuteOnLane0Op
     // region inside.
-    auto newForOp = scf::ForOp::create(rewriter,
-        forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-        forOp.getStep(), newOperands);
+    auto newForOp =
+        scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(),
+                           forOp.getUpperBound(), forOp.getStep(), newOperands);
     rewriter.setInsertionPointToStart(newForOp.getBody());
 
     SmallVector<Value> warpInput(newForOp.getRegionIterArgs().begin(),
@@ -1776,9 +1783,10 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
       argIndexMapping[escapingValues[i]] = warpInputType.size();
       warpInputType.push_back(inputTypes[i]);
     }
-    auto innerWarp = WarpExecuteOnLane0Op::create(rewriter,
-        newWarpOp.getLoc(), newForOp.getResultTypes(), newWarpOp.getLaneid(),
-        newWarpOp.getWarpSize(), warpInput, warpInputType);
+    auto innerWarp = WarpExecuteOnLane0Op::create(
+        rewriter, newWarpOp.getLoc(), newForOp.getResultTypes(),
+        newWarpOp.getLaneid(), newWarpOp.getWarpSize(), warpInput,
+        warpInputType);
 
     SmallVector<Value> argMapping;
     argMapping.push_back(newForOp.getInductionVar());
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
index e9718a7795fc5..73388a5da3e4f 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
@@ -77,8 +77,8 @@ struct CastAwayExtractStridedSliceLeadingOneDim
 
     Location loc = extractOp.getLoc();
 
-    Value newSrcVector = vector::ExtractOp::create(rewriter,
-        loc, extractOp.getVector(), splatZero(dropCount));
+    Value newSrcVector = vector::ExtractOp::create(
+        rewriter, loc, extractOp.getVector(), splatZero(dropCount));
 
     // The offsets/sizes/strides attribute can have a less number of elements
     // than the input vector's rank: it is meant for the leading dimensions.
@@ -89,8 +89,9 @@ struct CastAwayExtractStridedSliceLeadingOneDim
     auto newStrides = rewriter.getArrayAttr(
         extractOp.getStrides().getValue().drop_front(dropCount));
 
-    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
-        loc, newDstType, newSrcVector, newOffsets, newSizes, newStrides);
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(
+        rewriter, loc, newDstType, newSrcVector, newOffsets, newSizes,
+        newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(extractOp, oldDstType,
                                                      newExtractOp);
@@ -120,18 +121,19 @@ struct CastAwayInsertStridedSliceLeadingOneDim
     // Trim leading one dimensions from both operands.
     Location loc = insertOp.getLoc();
 
-    Value newSrcVector = vector::ExtractOp::create(rewriter,
-        loc, insertOp.getValueToStore(), splatZero(srcDropCount));
-    Value newDstVector = vector::ExtractOp::create(rewriter,
-        loc, insertOp.getDest(), splatZero(dstDropCount));
+    Value newSrcVector = vector::ExtractOp::create(
+        rewriter, loc, insertOp.getValueToStore(), splatZero(srcDropCount));
+    Value newDstVector = vector::ExtractOp::create(
+        rewriter, loc, insertOp.getDest(), splatZero(dstDropCount));
 
     auto newOffsets = rewriter.getArrayAttr(
         insertOp.getOffsets().getValue().take_back(newDstType.getRank()));
     auto newStrides = rewriter.getArrayAttr(
         insertOp.getStrides().getValue().take_back(newSrcType.getRank()));
 
-    auto newInsertOp = vector::InsertStridedSliceOp::create(rewriter,
-        loc, newDstType, newSrcVector, newDstVector, newOffsets, newStrides);
+    auto newInsertOp = vector::InsertStridedSliceOp::create(
+        rewriter, loc, newDstType, newSrcVector, newDstVector, newOffsets,
+        newStrides);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
                                                      newInsertOp);
@@ -169,11 +171,11 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
 
     Value newSrcVector = insertOp.getValueToStore();
     if (oldSrcRank != 0) {
-      newSrcVector = vector::ExtractOp::create(rewriter,
-          loc, insertOp.getValueToStore(), splatZero(srcDropCount));
+      newSrcVector = vector::ExtractOp::create(
+          rewriter, loc, insertOp.getValueToStore(), splatZero(srcDropCount));
     }
-    Value newDstVector = vector::ExtractOp::create(rewriter,
-        loc, insertOp.getDest(), splatZero(dstDropCount));
+    Value newDstVector = vector::ExtractOp::create(
+        rewriter, loc, insertOp.getDest(), splatZero(dstDropCount));
 
     // New position rank needs to be computed in two steps: (1) if destination
     // type has leading unit dims, we also trim the position array accordingly,
@@ -187,8 +189,8 @@ struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
     newPosition.resize(newDstType.getRank() - newSrcRank,
                        rewriter.getI64IntegerAttr(0));
 
-    auto newInsertOp = vector::InsertOp::create(rewriter,
-        loc, newSrcVector, newDstVector, newPosition);
+    auto newInsertOp = vector::InsertOp::create(rewriter, loc, newSrcVector,
+                                                newDstVector, newPosition);
 
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(insertOp, oldDstType,
                                                      newInsertOp);
@@ -259,8 +261,8 @@ struct CastAwayTransferReadLeadingOneDim
                                   newType, newMap, maskType);
     }
 
-    auto newRead = vector::TransferReadOp::create(rewriter,
-        read.getLoc(), newType, read.getBase(), read.getIndices(),
+    auto newRead = vector::TransferReadOp::create(
+        rewriter, read.getLoc(), newType, read.getBase(), read.getIndices(),
         AffineMapAttr::get(newMap), read.getPadding(), mask, inBoundsAttr);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(read, oldType, newRead);
 
@@ -306,8 +308,8 @@ struct CastAwayTransferWriteLeadingOneDim
       inBoundsAttr = rewriter.getArrayAttr(
           write.getInBoundsAttr().getValue().take_back(newType.getRank()));
 
-    auto newVector = vector::ExtractOp::create(rewriter,
-        write.getLoc(), write.getVector(), splatZero(dropDim));
+    auto newVector = vector::ExtractOp::create(
+        rewriter, write.getLoc(), write.getVector(), splatZero(dropDim));
 
     if (write.getMask()) {
       VectorType maskType = write.getMaskType();
@@ -443,22 +445,23 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
                                              contractOp.getContext()));
     // Extract if its a valid extraction, otherwise use the operand
     // without extraction.
-    newOperands.push_back(
-        validExtract ? vector::ExtractOp::create(rewriter,
-                           loc, operands[it.index()], splatZero(dropDim))
-                     : operands[it.index()]);
+    newOperands.push_back(validExtract
+                              ? vector::ExtractOp::create(rewriter, loc,
+                                                          operands[it.index()],
+                                                          splatZero(dropDim))
+                              : operands[it.index()]);
   }
 
   // Depending on whether this vector.contract is masked, the replacing Op
   // should either be a new vector.contract Op or vector.mask Op.
-  Operation *newOp = vector::ContractionOp::create(rewriter,
-      loc, newOperands[0], newOperands[1], newOperands[2],
+  Operation *newOp = vector::ContractionOp::create(
+      rewriter, loc, newOperands[0], newOperands[1], newOperands[2],
       rewriter.getAffineMapArrayAttr(newIndexingMaps),
       rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind());
 
   if (maskingOp) {
     auto newMask = vector::ExtractOp::create(rewriter, loc, maskingOp.getMask(),
-                                                      splatZero(dropDim));
+                                             splatZero(dropDim));
 
     newOp = mlir::vector::maskOperation(rewriter, newOp, newMask);
   }
@@ -519,8 +522,8 @@ class CastAwayElementwiseLeadingOneDim : public RewritePattern {
     SmallVector<Value, 4> newOperands;
     for (Value operand : op->getOperands()) {
       if (auto opVecType = dyn_cast<VectorType>(operand.getType())) {
-        newOperands.push_back(vector::ExtractOp::create(rewriter,
-            op->getLoc(), operand, splatZero(dropDim)));
+        newOperands.push_back(vector::ExtractOp::create(
+            rewriter, op->getLoc(), operand, splatZero(dropDim)));
       } else {
         newOperands.push_back(operand);
       }
@@ -559,8 +562,8 @@ struct CastAwayConstantMaskLeadingOneDim
     SmallVector<int64_t> newDimSizes = {flatLeadingSize};
     newDimSizes.append(dimSizes.begin() + dropDim + 1, dimSizes.end());
 
-    auto newMask = vector::ConstantMaskOp::create(rewriter,
-        mask.getLoc(), newType, newDimSizes);
+    auto newMask = vector::ConstantMaskOp::create(rewriter, mask.getLoc(),
+                                                  newType, newDimSizes);
     rewriter.replaceOpWithNewOp<vector::BroadcastOp>(mask, oldType, newMask);
     return success();
   }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
index 3411e010f3499..cb3e8dc67a1ae 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateMaskedLoadStore.cpp
@@ -65,13 +65,13 @@ struct VectorMaskedLoadOpConverter final
     Value base = maskedLoadOp.getBase();
     Value iValue = maskedLoadOp.getPassThru();
     auto indices = llvm::to_vector_of<Value>(maskedLoadOp.getIndices());
-    Value one = arith::ConstantOp::create(rewriter,
-        loc, indexType, IntegerAttr::get(indexType, 1));
+    Value one = arith::ConstantOp::create(rewriter, loc, indexType,
+                                          IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
       auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
-      auto ifOp = scf::IfOp::create(rewriter,
-          loc, maskBit,
+      auto ifOp = scf::IfOp::create(
+          rewriter, loc, maskBit,
           [&](OpBuilder &builder, Location loc) {
             auto loadedValue =
                 memref::LoadOp::create(builder, loc, base, indices);
@@ -84,7 +84,8 @@ struct VectorMaskedLoadOpConverter final
           });
       iValue = ifOp.getResult(0);
 
-      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
+      indices.back() =
+          arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.replaceOp(maskedLoadOp, iValue);
@@ -132,8 +133,8 @@ struct VectorMaskedStoreOpConverter final
     Value base = maskedStoreOp.getBase();
     Value value = maskedStoreOp.getValueToStore();
     auto indices = llvm::to_vector_of<Value>(maskedStoreOp.getIndices());
-    Value one = arith::ConstantOp::create(rewriter,
-        loc, indexType, IntegerAttr::get(indexType, 1));
+    Value one = arith::ConstantOp::create(rewriter, loc, indexType,
+                                          IntegerAttr::get(indexType, 1));
     for (int64_t i = 0; i < maskLength; ++i) {
       auto maskBit = vector::ExtractOp::create(rewriter, loc, mask, i);
 
@@ -143,7 +144,8 @@ struct VectorMaskedStoreOpConverter final
       memref::StoreOp::create(rewriter, loc, extractedValue, base, indices);
 
       rewriter.setInsertionPointAfter(ifOp);
-      indices.back() = arith::AddIOp::create(rewriter, loc, indices.back(), one);
+      indices.back() =
+          arith::AddIOp::create(rewriter, loc, indices.back(), one);
     }
 
     rewriter.eraseOp(maskedStoreOp);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index e915e85f16910..1c013e2617cd8 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -133,19 +133,18 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 newMaskOperands.push_back(
                     getValueOrCreateConstantIndexOp(rewriter, loc, maskIndex));
                 return vector::CreateMaskOp::create(rewriter, loc, newMaskType,
-                                                             newMaskOperands);
-              })
-          .Case<vector::ConstantMaskOp>(
-              [&](auto constantMaskOp) -> std::optional<Operation *> {
-                // Take the shape of mask, compress its trailing dimension:
-                SmallVector<int64_t> maskDimSizes(
-                    constantMaskOp.getMaskDimSizes());
-                int64_t &maskIndex = maskDimSizes.back();
-                maskIndex = llvm::divideCeil(numFrontPadElems + maskIndex,
-                                             numSrcElemsPerDest);
-                return vector::ConstantMaskOp::create(rewriter, loc, newMaskType,
-                                                               maskDimSizes);
+                                                    newMaskOperands);
               })
+          .Case<vector::ConstantMaskOp>([&](auto constantMaskOp)
+                                            -> std::optional<Operation *> {
+            // Take the shape of mask, compress its trailing dimension:
+            SmallVector<int64_t> maskDimSizes(constantMaskOp.getMaskDimSizes());
+            int64_t &maskIndex = maskDimSizes.back();
+            maskIndex = llvm::divideCeil(numFrontPadElems + maskIndex,
+                                         numSrcElemsPerDest);
+            return vector::ConstantMaskOp::create(rewriter, loc, newMaskType,
+                                                  maskDimSizes);
+          })
           .Case<arith::ConstantOp>([&](auto constantOp)
                                        -> std::optional<Operation *> {
             // TODO: Support multiple dimensions.
@@ -182,16 +181,18 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
               }
               compressedMaskValues.push_back(combinedValue);
             }
-            return arith::ConstantOp::create(rewriter,
-                loc, DenseElementsAttr::get(newMaskType, compressedMaskValues));
+            return arith::ConstantOp::create(
+                rewriter, loc,
+                DenseElementsAttr::get(newMaskType, compressedMaskValues));
           });
 
   if (!newMask)
     return failure();
 
   while (!extractOps.empty()) {
-    newMask = vector::ExtractOp::create(rewriter,
-        loc, (*newMask)->getResults()[0], extractOps.back().getMixedPosition());
+    newMask =
+        vector::ExtractOp::create(rewriter, loc, (*newMask)->getResults()[0],
+                                  extractOps.back().getMixedPosition());
     extractOps.pop_back();
   }
 
@@ -259,7 +260,7 @@ static Value staticallyInsertSubvector(OpBuilder &rewriter, Location loc,
   auto offsets = rewriter.getI64ArrayAttr({offset});
   auto strides = rewriter.getI64ArrayAttr({1});
   return vector::InsertStridedSliceOp::create(rewriter, loc, destVecTy, src,
-                                                       dest, offsets, strides);
+                                              dest, offsets, strides);
 }
 
 /// Extracts 1-D subvector from a 1-D vector.
@@ -301,8 +302,9 @@ static Value dynamicallyExtractSubVector(OpBuilder &rewriter, Location loc,
   for (int i = 0; i < numElemsToExtract; ++i) {
     Value extractLoc =
         (i == 0) ? dyn_cast<Value>(offset)
-                 : arith::AddIOp::create(rewriter,
-                       loc, rewriter.getIndexType(), dyn_cast<Value>(offset),
+                 : arith::AddIOp::create(
+                       rewriter, loc, rewriter.getIndexType(),
+                       dyn_cast<Value>(offset),
                        arith::ConstantIndexOp::create(rewriter, loc, i));
     auto extractOp = vector::ExtractOp::create(rewriter, loc, src, extractLoc);
     dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, i);
@@ -344,11 +346,11 @@ static Value dynamicallyInsertSubVector(RewriterBase &rewriter, Location loc,
 
   Value destOffsetVal = getValueOrCreateConstantIndexOp(rewriter, loc, offset);
   for (int64_t i = 0; i < numElemsToInsert; ++i) {
-    auto insertLoc = i == 0
-                         ? destOffsetVal
-                         : arith::AddIOp::create(rewriter,
-                               loc, rewriter.getIndexType(), destOffsetVal,
-                               arith::ConstantIndexOp::create(rewriter, loc, i));
+    auto insertLoc =
+        i == 0 ? destOffsetVal
+               : arith::AddIOp::create(
+                     rewriter, loc, rewriter.getIndexType(), destOffsetVal,
+                     arith::ConstantIndexOp::create(rewriter, loc, i));
     auto extractOp = vector::ExtractOp::create(rewriter, loc, src, i);
     dest = vector::InsertOp::create(rewriter, loc, extractOp, dest, insertLoc);
   }
@@ -369,11 +371,11 @@ static VectorValue emulatedVectorLoad(OpBuilder &rewriter, Location loc,
                                       Type containerElemTy) {
   auto emulatedPerContainerElem = containerElemTy.getIntOrFloatBitWidth() /
                                   emulatedElemTy.getIntOrFloatBitWidth();
-  auto newLoad = vector::LoadOp::create(rewriter,
-      loc, VectorType::get(numContainerElemsToLoad, containerElemTy), base,
-      getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
-  return vector::BitCastOp::create(rewriter,
-      loc,
+  auto newLoad = vector::LoadOp::create(
+      rewriter, loc, VectorType::get(numContainerElemsToLoad, containerElemTy),
+      base, getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices));
+  return vector::BitCastOp::create(
+      rewriter, loc,
       VectorType::get(numContainerElemsToLoad * emulatedPerContainerElem,
                       emulatedElemTy),
       newLoad);
@@ -390,7 +392,8 @@ static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
           upcastType.getNumElements() * upcastType.getElementTypeBitWidth() &&
       "expected input and output number of bits to match");
   if (trueValue.getType() != downcastType) {
-    trueValue = vector::BitCastOp::create(builder, loc, downcastType, trueValue);
+    trueValue =
+        vector::BitCastOp::create(builder, loc, downcastType, trueValue);
   }
   if (falseValue.getType() != downcastType) {
     falseValue =
@@ -422,8 +425,8 @@ static void atomicRMW(OpBuilder &builder, Location loc,
 
   // Create an atomic load-modify-write region using
   // `memref.generic_atomic_rmw`.
-  auto atomicOp = memref::GenericAtomicRMWOp::create(builder,
-      loc, linearizedMemref, ValueRange{storeIdx});
+  auto atomicOp = memref::GenericAtomicRMWOp::create(
+      builder, loc, linearizedMemref, ValueRange{storeIdx});
   Value origValue = atomicOp.getCurrentValue();
 
   OpBuilder::InsertionGuard guard(builder);
@@ -432,8 +435,8 @@ static void atomicRMW(OpBuilder &builder, Location loc,
   // Load the original value from memory, and cast it to the original element
   // type.
   auto oneElemVecType = VectorType::get({1}, origValue.getType());
-  Value origVecValue = vector::FromElementsOp::create(builder,
-      loc, oneElemVecType, ValueRange{origValue});
+  Value origVecValue = vector::FromElementsOp::create(
+      builder, loc, oneElemVecType, ValueRange{origValue});
 
   // Construct the final masked value and yield it.
   Value maskedValue =
@@ -453,16 +456,17 @@ static void nonAtomicRMW(OpBuilder &builder, Location loc,
 
   auto oneElemVecType =
       VectorType::get({1}, linearizedMemref.getType().getElementType());
-  Value origVecValue = vector::LoadOp::create(builder,
-      loc, oneElemVecType, linearizedMemref, ValueRange{linearizedIndex});
+  Value origVecValue =
+      vector::LoadOp::create(builder, loc, oneElemVecType, linearizedMemref,
+                             ValueRange{linearizedIndex});
   origVecValue = vector::BitCastOp::create(builder, loc, valueToStore.getType(),
-                                                   origVecValue);
+                                           origVecValue);
 
   Value maskedValue =
       downcastSelectAndUpcast(builder, loc, valueToStore.getType(),
                               oneElemVecType, mask, valueToStore, origVecValue);
   vector::StoreOp::create(builder, loc, maskedValue, linearizedMemref,
-                                  linearizedIndex);
+                          linearizedIndex);
 }
 
 /// Extract `sliceNumElements` from source `vector` at `extractOffset`,
@@ -489,8 +493,9 @@ static Value extractSliceIntoByte(ConversionPatternRewriter &rewriter,
   assert(8 % vectorElementType.getIntOrFloatBitWidth() == 0 &&
          "vector element must be a valid sub-byte type");
   auto emulatedPerContainerElem = 8 / vectorElementType.getIntOrFloatBitWidth();
-  auto emptyByteVector = arith::ConstantOp::create(rewriter,
-      loc, VectorType::get({emulatedPerContainerElem}, vectorElementType),
+  auto emptyByteVector = arith::ConstantOp::create(
+      rewriter, loc,
+      VectorType::get({emulatedPerContainerElem}, vectorElementType),
       rewriter.getZeroAttr(
           VectorType::get({emulatedPerContainerElem}, vectorElementType)));
   auto extracted = staticallyExtractSubvector(rewriter, loc, vector,
@@ -664,8 +669,8 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     if (!emulationRequiresPartialStores) {
       // Basic case: storing full bytes.
       auto numElements = origElements / emulatedPerContainerElem;
-      auto bitCast = vector::BitCastOp::create(rewriter,
-          loc, VectorType::get(numElements, containerElemTy),
+      auto bitCast = vector::BitCastOp::create(
+          rewriter, loc, VectorType::get(numElements, containerElemTy),
           op.getValueToStore());
       rewriter.replaceOpWithNewOp<vector::StoreOp>(
           op, bitCast.getResult(), memrefBase,
@@ -732,8 +737,9 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
         std::fill_n(frontMaskValues.end() - frontSubWidthStoreElem,
                     *foldedNumFrontPadElems, true);
       }
-      auto frontMask = arith::ConstantOp::create(rewriter,
-          loc, DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
+      auto frontMask = arith::ConstantOp::create(
+          rewriter, loc,
+          DenseElementsAttr::get(subWidthStoreMaskType, frontMaskValues));
 
       currentSourceIndex = emulatedPerContainerElem - (*foldedNumFrontPadElems);
       auto value =
@@ -752,8 +758,8 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
     // Increment the destination index by 1 to align to the emulated width
     // boundary.
     auto constantOne = arith::ConstantIndexOp::create(rewriter, loc, 1);
-    currentDestIndex = arith::AddIOp::create(rewriter,
-        loc, rewriter.getIndexType(), currentDestIndex, constantOne);
+    currentDestIndex = arith::AddIOp::create(
+        rewriter, loc, rewriter.getIndexType(), currentDestIndex, constantOne);
 
     // 2. Full width store for the inner output bytes.
     // After the previous step, the store address is aligned to the emulated
@@ -773,13 +779,13 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
           {originType.getNumElements() / emulatedPerContainerElem},
           memrefElemType);
       auto bitCast = vector::BitCastOp::create(rewriter, loc, storeType,
-                                                        fullWidthStorePart);
+                                               fullWidthStorePart);
       vector::StoreOp::create(rewriter, loc, bitCast.getResult(), memrefBase,
-                                       currentDestIndex);
+                              currentDestIndex);
 
       currentSourceIndex += numNonFullWidthElements;
-      currentDestIndex = arith::AddIOp::create(rewriter,
-          loc, rewriter.getIndexType(), currentDestIndex,
+      currentDestIndex = arith::AddIOp::create(
+          rewriter, loc, rewriter.getIndexType(), currentDestIndex,
           arith::ConstantIndexOp::create(rewriter, loc, fullWidthStoreSize));
     }
 
@@ -795,8 +801,9 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
       // Generate back mask.
       auto maskValues = SmallVector<bool>(emulatedPerContainerElem, 0);
       std::fill_n(maskValues.begin(), remainingElements, 1);
-      auto backMask = arith::ConstantOp::create(rewriter,
-          loc, DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
+      auto backMask = arith::ConstantOp::create(
+          rewriter, loc,
+          DenseElementsAttr::get(subWidthStoreMaskType, maskValues));
 
       storeFunc(rewriter, loc, memrefBase, currentDestIndex,
                 cast<VectorValue>(subWidthStorePart), backMask.getResult());
@@ -901,19 +908,19 @@ struct ConvertVectorMaskedStore final
     auto numElements = (origElements + emulatedPerContainerElem - 1) /
                        emulatedPerContainerElem;
     auto newType = VectorType::get(numElements, containerElemTy);
-    auto passThru = arith::ConstantOp::create(rewriter,
-        loc, newType, rewriter.getZeroAttr(newType));
+    auto passThru = arith::ConstantOp::create(rewriter, loc, newType,
+                                              rewriter.getZeroAttr(newType));
 
-    auto newLoad = vector::MaskedLoadOp::create(rewriter,
-        loc, newType, adaptor.getBase(), linearizedIndices,
+    auto newLoad = vector::MaskedLoadOp::create(
+        rewriter, loc, newType, adaptor.getBase(), linearizedIndices,
         newMask.value()->getResult(0), passThru);
 
     auto newBitCastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
     Value valueToStore =
         vector::BitCastOp::create(rewriter, loc, newBitCastType, newLoad);
-    valueToStore = arith::SelectOp::create(rewriter,
-        loc, op.getMask(), op.getValueToStore(), valueToStore);
+    valueToStore = arith::SelectOp::create(rewriter, loc, op.getMask(),
+                                           op.getValueToStore(), valueToStore);
     valueToStore =
         vector::BitCastOp::create(rewriter, loc, newType, valueToStore);
 
@@ -1016,8 +1023,8 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
                            numElements, emulatedElemTy, containerElemTy);
 
     if (!foldedIntraVectorOffset) {
-      auto resultVector = arith::ConstantOp::create(rewriter,
-          loc, op.getType(), rewriter.getZeroAttr(op.getType()));
+      auto resultVector = arith::ConstantOp::create(
+          rewriter, loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(
           rewriter, loc, dyn_cast<TypedValue<VectorType>>(result), resultVector,
           linearizedInfo.intraDataOffset, origElements);
@@ -1142,8 +1149,8 @@ struct ConvertVectorMaskedLoad final
     auto newBitcastType =
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy);
 
-    auto emptyVector = arith::ConstantOp::create(rewriter,
-        loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
+    auto emptyVector = arith::ConstantOp::create(
+        rewriter, loc, newBitcastType, rewriter.getZeroAttr(newBitcastType));
     if (!foldedIntraVectorOffset) {
       passthru = dynamicallyInsertSubVector(
           rewriter, loc, passthru, emptyVector, linearizedInfo.intraDataOffset,
@@ -1156,8 +1163,8 @@ struct ConvertVectorMaskedLoad final
         vector::BitCastOp::create(rewriter, loc, loadType, passthru);
 
     // Generating the new masked load.
-    auto newLoad = vector::MaskedLoadOp::create(rewriter,
-        loc, loadType, adaptor.getBase(),
+    auto newLoad = vector::MaskedLoadOp::create(
+        rewriter, loc, loadType, adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newMask.value()->getResult(0), newPassThru);
 
@@ -1170,8 +1177,9 @@ struct ConvertVectorMaskedLoad final
     auto newSelectMaskType = VectorType::get(
         numElements * emulatedPerContainerElem, rewriter.getI1Type());
     // TODO: try to fold if op's mask is constant
-    auto emptyMask = arith::ConstantOp::create(rewriter,
-        loc, newSelectMaskType, rewriter.getZeroAttr(newSelectMaskType));
+    auto emptyMask =
+        arith::ConstantOp::create(rewriter, loc, newSelectMaskType,
+                                  rewriter.getZeroAttr(newSelectMaskType));
     if (!foldedIntraVectorOffset) {
       mask = dynamicallyInsertSubVector(rewriter, loc, mask, emptyMask,
                                         linearizedInfo.intraDataOffset,
@@ -1279,8 +1287,7 @@ struct ConvertVectorTransferRead final
           padding);
     }
     auto newPadding =
-        arith::ExtUIOp::create(rewriter, loc, containerElemTy,
-                                                      padding);
+        arith::ExtUIOp::create(rewriter, loc, containerElemTy, padding);
 
     auto stridedMetadata =
         memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
@@ -1304,20 +1311,21 @@ struct ConvertVectorTransferRead final
     auto numElements = llvm::divideCeil(maxIntraDataOffset + origElements,
                                         emulatedPerContainerElem);
 
-    auto newRead = vector::TransferReadOp::create(rewriter,
-        loc, VectorType::get(numElements, containerElemTy), adaptor.getBase(),
+    auto newRead = vector::TransferReadOp::create(
+        rewriter, loc, VectorType::get(numElements, containerElemTy),
+        adaptor.getBase(),
         getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices),
         newPadding);
 
-    auto bitCast = vector::BitCastOp::create(rewriter,
-        loc,
+    auto bitCast = vector::BitCastOp::create(
+        rewriter, loc,
         VectorType::get(numElements * emulatedPerContainerElem, emulatedElemTy),
         newRead);
 
     Value result = bitCast->getResult(0);
     if (!foldedIntraVectorOffset) {
-      auto zeros = arith::ConstantOp::create(rewriter,
-          loc, op.getType(), rewriter.getZeroAttr(op.getType()));
+      auto zeros = arith::ConstantOp::create(
+          rewriter, loc, op.getType(), rewriter.getZeroAttr(op.getType()));
       result = dynamicallyExtractSubVector(rewriter, loc, bitCast, zeros,
                                            linearizedInfo.intraDataOffset,
                                            origElements);
@@ -1690,25 +1698,26 @@ Value BitCastRewriter::genericRewriteStep(
     PatternRewriter &rewriter, Location loc, Value initialValue,
     Value runningResult, const BitCastRewriter::Metadata &metadata) {
   // Create vector.shuffle from the metadata.
-  auto shuffleOp = vector::ShuffleOp::create(rewriter,
-      loc, initialValue, initialValue, metadata.shuffles);
+  auto shuffleOp = vector::ShuffleOp::create(rewriter, loc, initialValue,
+                                             initialValue, metadata.shuffles);
 
   // Intersect with the mask.
   VectorType shuffledVectorType = shuffleOp.getResultVectorType();
-  auto constOp = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(shuffledVectorType, metadata.masks));
+  auto constOp = arith::ConstantOp::create(
+      rewriter, loc,
+      DenseElementsAttr::get(shuffledVectorType, metadata.masks));
   Value andValue = arith::AndIOp::create(rewriter, loc, shuffleOp, constOp);
 
   // Align right on 0.
-  auto shiftRightConstantOp = arith::ConstantOp::create(rewriter,
-      loc,
+  auto shiftRightConstantOp = arith::ConstantOp::create(
+      rewriter, loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftRightAmounts));
   Value shiftedRight =
       arith::ShRUIOp::create(rewriter, loc, andValue, shiftRightConstantOp);
 
   // Shift bits left into their final position.
-  auto shiftLeftConstantOp = arith::ConstantOp::create(rewriter,
-      loc,
+  auto shiftLeftConstantOp = arith::ConstantOp::create(
+      rewriter, loc,
       DenseElementsAttr::get(shuffledVectorType, metadata.shiftLeftAmounts));
   Value shiftedLeft =
       arith::ShLIOp::create(rewriter, loc, shiftedRight, shiftLeftConstantOp);
@@ -1766,14 +1775,14 @@ static Value extractNBitsPerByteAndSignExtendToI8(PatternRewriter &rewriter,
   assert(bitIdx >= 0 && bitsToShiftLeft >= 0 && numBits > 0 && numBits <= 8 &&
          "Invalid bitIdx range");
   if (bitsToShiftLeft != 0) {
-    Value shiftLeftValues = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(srcType, bitsToShiftLeft));
+    Value shiftLeftValues = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(srcType, bitsToShiftLeft));
     shl = arith::ShLIOp::create(rewriter, loc, src, shiftLeftValues);
   }
 
   int8_t bitsToShiftRight = 8 - numBits;
-  Value shiftRightValues = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
+  Value shiftRightValues = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
   Value shr = arith::ShRSIOp::create(rewriter, loc, shl, shiftRightValues);
   return shr;
 }
@@ -1808,16 +1817,16 @@ static Value extractNBitsPerByteAndExtendToI8(PatternRewriter &rewriter,
   int8_t bitsToShiftRight = bitIdx;
   Value shr = src;
   if (bitsToShiftRight != 0) {
-    Value shiftRightValues = arith::ConstantOp::create(rewriter,
-        loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
+    Value shiftRightValues = arith::ConstantOp::create(
+        rewriter, loc, DenseElementsAttr::get(srcType, bitsToShiftRight));
     shr = arith::ShRUIOp::create(rewriter, loc, src, shiftRightValues);
   }
   if (bitIdx + numBits == 8) {
     return shr;
   }
   uint8_t lowBitsMask = (1 << numBits) - 1;
-  Value lowBitsMaskValues = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(srcType, lowBitsMask));
+  Value lowBitsMaskValues = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(srcType, lowBitsMask));
   return arith::AndIOp::create(rewriter, loc, shr, lowBitsMaskValues);
 }
 
@@ -1876,7 +1885,8 @@ static Value rewriteI2ToI8Ext(PatternRewriter &rewriter, Location loc,
   // 0213  = [0,1,2,3,...],...
   Value interleave02 = vector::InterleaveOp::create(rewriter, loc, vec0, vec2);
   Value interleave13 = vector::InterleaveOp::create(rewriter, loc, vec1, vec3);
-  return vector::InterleaveOp::create(rewriter, loc, interleave02, interleave13);
+  return vector::InterleaveOp::create(rewriter, loc, interleave02,
+                                      interleave13);
 }
 
 /// Rewrite the i8 -> i4 truncation into a deinterleave and series of bitwise
@@ -1893,17 +1903,17 @@ static Value rewriteI8ToI4Trunc(PatternRewriter &rewriter, Location loc,
   // 2. Zero out the upper side of each low i8 element.
   constexpr int8_t i8LowBitMask = 0x0F;
   VectorType deinterI8VecType = deinterleaveOp.getResultVectorType();
-  Value zeroOutMask = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(deinterI8VecType, i8LowBitMask));
-  Value zeroOutLow = arith::AndIOp::create(rewriter,
-      loc, deinterleaveOp.getRes1(), zeroOutMask);
+  Value zeroOutMask = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(deinterI8VecType, i8LowBitMask));
+  Value zeroOutLow = arith::AndIOp::create(
+      rewriter, loc, deinterleaveOp.getRes1(), zeroOutMask);
 
   // 3. Move high i4 values to upper side of the byte.
   constexpr int8_t bitsToShift = 4;
-  auto shiftValues = arith::ConstantOp::create(rewriter,
-      loc, DenseElementsAttr::get(deinterI8VecType, bitsToShift));
+  auto shiftValues = arith::ConstantOp::create(
+      rewriter, loc, DenseElementsAttr::get(deinterI8VecType, bitsToShift));
   Value shlHigh = arith::ShLIOp::create(rewriter, loc, deinterleaveOp.getRes2(),
-                                                 shiftValues);
+                                        shiftValues);
 
   // 4. Merge high and low i4 values.
   auto mergedHiLowOp = arith::OrIOp::create(rewriter, loc, zeroOutLow, shlHigh);
@@ -2201,9 +2211,9 @@ struct RewriteVectorTranspose : OpRewritePattern<vector::TransposeOp> {
     auto srcNativeVecType = srcSubByteVecType.cloneWith(
         std::nullopt, rewriter.getIntegerType(minNativeBitwidth));
     Value extOp = arith::ExtSIOp::create(rewriter, loc, srcNativeVecType,
-                                                  transposeOp.getVector());
-    Value newTranspose = vector::TransposeOp::create(rewriter,
-        loc, extOp, transposeOp.getPermutation());
+                                         transposeOp.getVector());
+    Value newTranspose = vector::TransposeOp::create(
+        rewriter, loc, extOp, transposeOp.getPermutation());
     VectorType dstSubByteVecType = transposeOp.getResultVectorType();
     rewriter.replaceOpWithNewOp<arith::TruncIOp>(transposeOp, dstSubByteVecType,
                                                  newTranspose);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
index baeca29298334..72352d72bfe77 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
@@ -53,15 +53,15 @@ class DecomposeDifferentRankInsertStridedSlice
     int64_t rankRest = dstType.getRank() - rankDiff;
     // Extract / insert the subvector of matching rank and InsertStridedSlice
     // on it.
-    Value extracted = ExtractOp::create(rewriter,
-        loc, op.getDest(),
-        getI64SubArray(op.getOffsets(), /*dropFront=*/0,
-                       /*dropBack=*/rankRest));
+    Value extracted =
+        ExtractOp::create(rewriter, loc, op.getDest(),
+                          getI64SubArray(op.getOffsets(), /*dropFront=*/0,
+                                         /*dropBack=*/rankRest));
 
     // A different pattern will kick in for InsertStridedSlice with matching
     // ranks.
-    auto stridedSliceInnerOp = InsertStridedSliceOp::create(rewriter,
-        loc, op.getValueToStore(), extracted,
+    auto stridedSliceInnerOp = InsertStridedSliceOp::create(
+        rewriter, loc, op.getValueToStore(), extracted,
         getI64SubArray(op.getOffsets(), /*dropFront=*/rankDiff),
         getI64SubArray(op.getStrides(), /*dropFront=*/0));
 
@@ -131,8 +131,8 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
       SmallVector<int64_t> offsets(nDest, 0);
       for (int64_t i = 0; i < nSrc; ++i)
         offsets[i] = i;
-      Value scaledSource = ShuffleOp::create(rewriter,
-          loc, op.getValueToStore(), op.getValueToStore(), offsets);
+      Value scaledSource = ShuffleOp::create(
+          rewriter, loc, op.getValueToStore(), op.getValueToStore(), offsets);
 
       // 2. Create a mask where we take the value from scaledSource of dest
       // depending on the offset.
@@ -164,8 +164,8 @@ class ConvertSameRankInsertStridedSliceIntoShuffle
             ExtractOp::create(rewriter, loc, op.getDest(), off);
         // 3. Reduce the problem to lowering a new InsertStridedSlice op with
         // smaller rank.
-        extractedSource = InsertStridedSliceOp::create(rewriter,
-            loc, extractedSource, extractedDest,
+        extractedSource = InsertStridedSliceOp::create(
+            rewriter, loc, extractedSource, extractedDest,
             getI64SubArray(op.getOffsets(), /* dropFront=*/1),
             getI64SubArray(op.getStrides(), /* dropFront=*/1));
       }
@@ -252,8 +252,8 @@ class Convert1DExtractStridedSliceIntoExtractInsertChain final
     for (int64_t i = offset, e = offset + size * stride; i < e; i += stride)
       elements.push_back(ExtractOp::create(rewriter, loc, op.getVector(), i));
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, rewriter.getZeroAttr(op.getType()));
+    Value result = arith::ConstantOp::create(
+        rewriter, loc, rewriter.getZeroAttr(op.getType()));
     for (int64_t i = 0; i < size; ++i)
       result = InsertOp::create(rewriter, loc, elements[i], result, i);
 
@@ -301,14 +301,14 @@ class DecomposeNDExtractStridedSlice
       return failure();
 
     // Extract/insert on a lower ranked extract strided slice op.
-    Value zero = arith::ConstantOp::create(rewriter,
-        loc, elemType, rewriter.getZeroAttr(elemType));
+    Value zero = arith::ConstantOp::create(rewriter, loc, elemType,
+                                           rewriter.getZeroAttr(elemType));
     Value res = SplatOp::create(rewriter, loc, dstType, zero);
     for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
          off += stride, ++idx) {
       Value one = ExtractOp::create(rewriter, loc, op.getVector(), off);
-      Value extracted = ExtractStridedSliceOp::create(rewriter,
-          loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
+      Value extracted = ExtractStridedSliceOp::create(
+          rewriter, loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
           getI64SubArray(op.getSizes(), /* dropFront=*/1),
           getI64SubArray(op.getStrides(), /* dropFront=*/1));
       res = InsertOp::create(rewriter, loc, extracted, res, idx);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
index 70d786a516859..4233212b5cbbe 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorMaskElimination.cpp
@@ -84,8 +84,8 @@ LogicalResult resolveAllTrueCreateMaskOp(IRRewriter &rewriter,
   // Replace createMaskOp with an all-true constant. This should result in the
   // mask being removed in most cases (as xfer ops + vector.mask have folds to
   // remove all-true masks).
-  auto allTrue = vector::ConstantMaskOp::create(rewriter,
-      createMaskOp.getLoc(), maskType, ConstantMaskKind::AllTrue);
+  auto allTrue = vector::ConstantMaskOp::create(
+      rewriter, createMaskOp.getLoc(), maskType, ConstantMaskKind::AllTrue);
   rewriter.replaceAllUsesWith(createMaskOp, allTrue);
   return success();
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 9bbedc396664b..2676d254c9b64 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -287,7 +287,7 @@ static Value rankReducingSubviewDroppingUnitDims(PatternRewriter &rewriter,
       inputType.canonicalizeStridedLayout())
     return input;
   return memref::SubViewOp::create(rewriter, loc, resultType, input, offsets,
-                                            sizes, strides);
+                                   sizes, strides);
 }
 
 /// Returns the number of dims that aren't unit dims.
@@ -399,9 +399,9 @@ class TransferReadDropUnitDimsPattern
     SmallVector<Value> zeros(reducedRank, c0);
     auto identityMap = rewriter.getMultiDimIdentityMap(reducedRank);
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
-    Operation *newTransferReadOp = vector::TransferReadOp::create(rewriter,
-        loc, reducedVectorType, reducedShapeSource, zeros, identityMap,
-        transferReadOp.getPadding(), maskOp,
+    Operation *newTransferReadOp = vector::TransferReadOp::create(
+        rewriter, loc, reducedVectorType, reducedShapeSource, zeros,
+        identityMap, transferReadOp.getPadding(), maskOp,
         rewriter.getBoolArrayAttr(inBounds));
 
     if (maskingOp) {
@@ -483,9 +483,9 @@ class TransferWriteDropUnitDimsPattern
     SmallVector<bool> inBounds(reducedVectorType.getRank(), true);
     auto shapeCastSrc = rewriter.createOrFold<vector::ShapeCastOp>(
         loc, reducedVectorType, vector);
-    Operation *newXferWrite = vector::TransferWriteOp::create(rewriter,
-        loc, Type(), shapeCastSrc, reducedShapeSource, zeros, identityMap,
-        maskOp, rewriter.getBoolArrayAttr(inBounds));
+    Operation *newXferWrite = vector::TransferWriteOp::create(
+        rewriter, loc, Type(), shapeCastSrc, reducedShapeSource, zeros,
+        identityMap, maskOp, rewriter.getBoolArrayAttr(inBounds));
 
     if (maskingOp) {
       auto shapeCastMask = rewriter.createOrFold<vector::ShapeCastOp>(
@@ -573,8 +573,8 @@ static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
   if (auto value = dyn_cast<Value>(collapsedOffset)) {
     indicesAfterCollapsing.push_back(value);
   } else {
-    indicesAfterCollapsing.push_back(arith::ConstantIndexOp::create(rewriter,
-        loc, *getConstantIntValue(collapsedOffset)));
+    indicesAfterCollapsing.push_back(arith::ConstantIndexOp::create(
+        rewriter, loc, *getConstantIntValue(collapsedOffset)));
   }
 
   return indicesAfterCollapsing;
@@ -659,8 +659,8 @@ class FlattenContiguousRowMajorTransferReadPattern
     // 3. Create new vector.transfer_read that reads from the collapsed memref
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
-    vector::TransferReadOp flatRead = vector::TransferReadOp::create(rewriter,
-        loc, flatVectorType, collapsedSource, collapsedIndices,
+    vector::TransferReadOp flatRead = vector::TransferReadOp::create(
+        rewriter, loc, flatVectorType, collapsedSource, collapsedIndices,
         transferReadOp.getPadding(), collapsedMap);
     flatRead.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
@@ -758,9 +758,9 @@ class FlattenContiguousRowMajorTransferWritePattern
                                                 vectorType.getElementType());
     Value flatVector =
         vector::ShapeCastOp::create(rewriter, loc, flatVectorType, vector);
-    vector::TransferWriteOp flatWrite =
-        vector::TransferWriteOp::create(rewriter,
-            loc, flatVector, collapsedSource, collapsedIndices, collapsedMap);
+    vector::TransferWriteOp flatWrite = vector::TransferWriteOp::create(
+        rewriter, loc, flatVector, collapsedSource, collapsedIndices,
+        collapsedMap);
     flatWrite.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
     // 4. Replace the old transfer_write with the new one writing the
@@ -846,8 +846,8 @@ class RewriteScalarExtractOfTransferRead
       if (auto value = dyn_cast<Value>(composedIdx)) {
         newIndices[idx] = value;
       } else {
-        newIndices[idx] = arith::ConstantIndexOp::create(rewriter,
-            extractOp.getLoc(), *getConstantIntValue(composedIdx));
+        newIndices[idx] = arith::ConstantIndexOp::create(
+            rewriter, extractOp.getLoc(), *getConstantIntValue(composedIdx));
       }
     }
     if (isa<MemRefType>(xferOp.getBase().getType())) {
@@ -883,8 +883,8 @@ class RewriteScalarWrite : public OpRewritePattern<vector::TransferWriteOp> {
     if (!xferOp.getPermutationMap().isMinorIdentity())
       return failure();
     // Only float and integer element types are supported.
-    Value scalar =
-        vector::ExtractOp::create(rewriter, xferOp.getLoc(), xferOp.getVector());
+    Value scalar = vector::ExtractOp::create(rewriter, xferOp.getLoc(),
+                                             xferOp.getVector());
     // Construct a scalar store.
     if (isa<MemRefType>(xferOp.getBase().getType())) {
       rewriter.replaceOpWithNewOp<memref::StoreOp>(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
index a1f36a9744932..71de9839f0f5c 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
@@ -65,8 +65,8 @@ static Value createInBoundsCond(RewriterBase &b,
       return;
     Value cond =
         arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sle,
-                                getValueOrCreateConstantIndexOp(b, loc, sum),
-                                getValueOrCreateConstantIndexOp(b, loc, dimSz));
+                              getValueOrCreateConstantIndexOp(b, loc, sum),
+                              getValueOrCreateConstantIndexOp(b, loc, dimSz));
     // Conjunction over all dims for which we are in-bounds.
     if (inBoundsCond)
       inBoundsCond = arith::AndIOp::create(b, loc, inBoundsCond, cond);
@@ -177,7 +177,8 @@ static Value castToCompatibleMemRefType(OpBuilder &b, Value memref,
     sourceType = MemRefType::get(
         sourceType.getShape(), sourceType.getElementType(),
         sourceType.getLayout(), compatibleMemRefType.getMemorySpace());
-    res = memref::MemorySpaceCastOp::create(b, memref.getLoc(), sourceType, res);
+    res =
+        memref::MemorySpaceCastOp::create(b, memref.getLoc(), sourceType, res);
   }
   if (sourceType == compatibleMemRefType)
     return res;
@@ -211,8 +212,9 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
     SmallVector<AffineMap, 4> maps =
         AffineMap::inferFromExprList(MapList{{i - j, k}}, b.getContext());
     // affine_min(%dimMemRef - %index, %dimAlloc)
-    Value affineMin = affine::AffineMinOp::create(b,
-        loc, index.getType(), maps[0], ValueRange{dimMemRef, index, dimAlloc});
+    Value affineMin =
+        affine::AffineMinOp::create(b, loc, index.getType(), maps[0],
+                                    ValueRange{dimMemRef, index, dimAlloc});
     sizes.push_back(affineMin);
   });
 
@@ -220,10 +222,10 @@ createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
       xferOp.getIndices(), [](Value idx) -> OpFoldResult { return idx; }));
   SmallVector<OpFoldResult> destIndices(memrefRank, b.getIndexAttr(0));
   SmallVector<OpFoldResult> strides(memrefRank, b.getIndexAttr(1));
-  auto copySrc = memref::SubViewOp::create(b,
-      loc, isaWrite ? alloc : xferOp.getBase(), srcIndices, sizes, strides);
-  auto copyDest = memref::SubViewOp::create(b,
-      loc, isaWrite ? xferOp.getBase() : alloc, destIndices, sizes, strides);
+  auto copySrc = memref::SubViewOp::create(
+      b, loc, isaWrite ? alloc : xferOp.getBase(), srcIndices, sizes, strides);
+  auto copyDest = memref::SubViewOp::create(
+      b, loc, isaWrite ? xferOp.getBase() : alloc, destIndices, sizes, strides);
   return std::make_pair(copySrc, copyDest);
 }
 
@@ -253,8 +255,8 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
   Location loc = xferOp.getLoc();
   Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return scf::IfOp::create(b,
-      loc, inBoundsCond,
+  return scf::IfOp::create(
+      b, loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
@@ -263,7 +265,7 @@ createFullPartialLinalgCopy(RewriterBase &b, vector::TransferReadOp xferOp,
       },
       [&](OpBuilder &b, Location loc) {
         linalg::FillOp::create(b, loc, ValueRange{xferOp.getPadding()},
-                                 ValueRange{alloc});
+                               ValueRange{alloc});
         // Take partial subview of memref which guarantees no dimension
         // overflows.
         IRRewriter rewriter(b);
@@ -306,8 +308,8 @@ static scf::IfOp createFullPartialVectorTransferRead(
   scf::IfOp fullPartialIfOp;
   Value zero = arith::ConstantIndexOp::create(b, loc, 0);
   Value memref = xferOp.getBase();
-  return scf::IfOp::create(b,
-      loc, inBoundsCond,
+  return scf::IfOp::create(
+      b, loc, inBoundsCond,
       [&](OpBuilder &b, Location loc) {
         Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
         scf::ValueVector viewAndIndices{res};
@@ -317,10 +319,10 @@ static scf::IfOp createFullPartialVectorTransferRead(
       [&](OpBuilder &b, Location loc) {
         Operation *newXfer = b.clone(*xferOp.getOperation());
         Value vector = cast<VectorTransferOpInterface>(newXfer).getVector();
-        memref::StoreOp::create(b,
-            loc, vector,
-            vector::TypeCastOp::create(b,
-                loc, MemRefType::get({}, vector.getType()), alloc));
+        memref::StoreOp::create(
+            b, loc, vector,
+            vector::TypeCastOp::create(
+                b, loc, MemRefType::get({}, vector.getType()), alloc));
 
         Value casted =
             castToCompatibleMemRefType(b, alloc, compatibleMemRefType);
@@ -391,8 +393,8 @@ static void createFullPartialLinalgCopy(RewriterBase &b,
                                         vector::TransferWriteOp xferOp,
                                         Value inBoundsCond, Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = arith::XOrIOp::create(b,
-      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  auto notInBounds = arith::XOrIOp::create(
+      b, loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
   scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRRewriter rewriter(b);
     std::pair<Value, Value> copyArgs = createSubViewIntersection(
@@ -420,14 +422,14 @@ static void createFullPartialVectorTransferWrite(RewriterBase &b,
                                                  Value inBoundsCond,
                                                  Value alloc) {
   Location loc = xferOp.getLoc();
-  auto notInBounds = arith::XOrIOp::create(b,
-      loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
+  auto notInBounds = arith::XOrIOp::create(
+      b, loc, inBoundsCond, arith::ConstantIntOp::create(b, loc, true, 1));
   scf::IfOp::create(b, loc, notInBounds, [&](OpBuilder &b, Location loc) {
     IRMapping mapping;
-    Value load = memref::LoadOp::create(b,
-        loc,
-        vector::TypeCastOp::create(b,
-            loc, MemRefType::get({}, xferOp.getVector().getType()), alloc),
+    Value load = memref::LoadOp::create(
+        b, loc,
+        vector::TypeCastOp::create(
+            b, loc, MemRefType::get({}, xferOp.getVector().getType()), alloc),
         ValueRange());
     mapping.map(xferOp.getVector(), load);
     b.clone(*xferOp.getOperation(), mapping);
@@ -562,8 +564,8 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
     auto shape = xferOp.getVectorType().getShape();
     Type elementType = xferOp.getVectorType().getElementType();
     alloc = memref::AllocaOp::create(b, scope->getLoc(),
-                                       MemRefType::get(shape, elementType),
-                                       ValueRange{}, b.getI64IntegerAttr(32));
+                                     MemRefType::get(shape, elementType),
+                                     ValueRange{}, b.getI64IntegerAttr(32));
   }
 
   MemRefType compatibleMemRefType =
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index d318b4b16b5aa..73ca327bb49c5 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -381,8 +381,8 @@ FailureOr<Value> combineContractAndBroadcast(vector::ContractionOp contractOp,
   if (getUnusedDimsBitVector({maps[0], maps[1]}).any())
     return failure();
 
-  Operation *newOp = vector::ContractionOp::create(rewriter,
-      contractOp.getLoc(), lhs, rhs, contractOp.getAcc(),
+  Operation *newOp = vector::ContractionOp::create(
+      rewriter, contractOp.getLoc(), lhs, rhs, contractOp.getAcc(),
       rewriter.getAffineMapArrayAttr(maps), rewriter.getArrayAttr(iterators));
 
   // Handle the mask.
@@ -534,8 +534,8 @@ struct ReorderElementwiseOpsOnTranspose final
         // This is a constant. Create a reverse transpose op for it.
         auto vectorType =
             srcType.clone(cast<VectorType>(operand.getType()).getElementType());
-        srcValues.push_back(vector::TransposeOp::create(rewriter,
-            operand.getLoc(), vectorType, operand, invOrder));
+        srcValues.push_back(vector::TransposeOp::create(
+            rewriter, operand.getLoc(), vectorType, operand, invOrder));
       }
     }
 
@@ -608,13 +608,13 @@ struct BubbleDownVectorBitCastForExtract
     // Get the single scalar (as a vector) in the source value that packs the
     // desired scalar. E.g. extract vector<1xf32> from vector<4xf32>
     Location loc = extractOp.getLoc();
-    Value packedValue = vector::ExtractOp::create(rewriter,
-        loc, castOp.getSource(), index / expandRatio);
+    Value packedValue = vector::ExtractOp::create(
+        rewriter, loc, castOp.getSource(), index / expandRatio);
     Type packedVecType = VectorType::get(/*shape=*/{1}, packedValue.getType());
-    Value zero = arith::ConstantOp::create(rewriter,
-        loc, packedVecType, rewriter.getZeroAttr(packedVecType));
+    Value zero = arith::ConstantOp::create(rewriter, loc, packedVecType,
+                                           rewriter.getZeroAttr(packedVecType));
     packedValue = vector::InsertOp::create(rewriter, loc, packedValue, zero,
-                                                    /*position=*/0);
+                                           /*position=*/0);
 
     // Cast it to a vector with the desired scalar's type.
     // E.g. f32 -> vector<2xf16>
@@ -700,9 +700,9 @@ struct BubbleDownBitCastForStridedSliceExtract
     VectorType newExtractType =
         VectorType::get(dims, castSrcType.getElementType());
 
-    auto newExtractOp = vector::ExtractStridedSliceOp::create(rewriter,
-        extractOp.getLoc(), newExtractType, castOp.getSource(), newOffsets,
-        newSizes, extractOp.getStrides());
+    auto newExtractOp = vector::ExtractStridedSliceOp::create(
+        rewriter, extractOp.getLoc(), newExtractType, castOp.getSource(),
+        newOffsets, newSizes, extractOp.getStrides());
 
     rewriter.replaceOpWithNewOp<vector::BitCastOp>(
         extractOp, extractOp.getType(), newExtractOp);
@@ -761,8 +761,9 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         isNumElemsShrink ? srcDims.back() / ratio : srcDims.back() * ratio;
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
-    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
-        bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
+    auto newCastSrcOp =
+        vector::BitCastOp::create(rewriter, bitcastOp.getLoc(), newCastSrcType,
+                                  insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims(insertOp.getDestVectorType().getShape());
     dstDims.back() =
@@ -771,8 +772,8 @@ struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
         VectorType::get(dstDims, castDstType.getElementType());
 
     // Bitcast the destination.
-    auto newCastDstOp = vector::BitCastOp::create(rewriter,
-        bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
+    auto newCastDstOp = vector::BitCastOp::create(
+        rewriter, bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     // Generate new insert.
     rewriter.replaceOpWithNewOp<vector::InsertOp>(
@@ -852,8 +853,9 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastSrcType =
         VectorType::get(srcDims, castDstType.getElementType());
 
-    auto newCastSrcOp = vector::BitCastOp::create(rewriter,
-        bitcastOp.getLoc(), newCastSrcType, insertOp.getValueToStore());
+    auto newCastSrcOp =
+        vector::BitCastOp::create(rewriter, bitcastOp.getLoc(), newCastSrcType,
+                                  insertOp.getValueToStore());
 
     SmallVector<int64_t> dstDims =
         llvm::to_vector<4>(insertOp.getDestVectorType().getShape());
@@ -861,8 +863,8 @@ struct BubbleUpBitCastForStridedSliceInsert
     VectorType newCastDstType =
         VectorType::get(dstDims, castDstType.getElementType());
 
-    auto newCastDstOp = vector::BitCastOp::create(rewriter,
-        bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
+    auto newCastDstOp = vector::BitCastOp::create(
+        rewriter, bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
 
     rewriter.replaceOpWithNewOp<vector::InsertStridedSliceOp>(
         bitcastOp, bitcastOp.getType(), newCastSrcOp, newCastDstOp, newOffsets,
@@ -936,8 +938,8 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
     Type elemType = castDstType.getElementType();
     assert(elemType.isSignlessIntOrIndexOrFloat());
 
-    Value zero = arith::ConstantOp::create(rewriter,
-        loc, elemType, rewriter.getZeroAttr(elemType));
+    Value zero = arith::ConstantOp::create(rewriter, loc, elemType,
+                                           rewriter.getZeroAttr(elemType));
     Value res = SplatOp::create(rewriter, loc, castDstType, zero);
 
     SmallVector<int64_t> sliceShape = {castDstLastDim};
@@ -947,13 +949,13 @@ struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
                         castDstType.getElementType());
 
     for (int i = 0, e = shrinkRatio; i < e; ++i) {
-      Value extracted = ExtractStridedSliceOp::create(rewriter,
-          loc, bitcastOp.getSource(), ArrayRef<int64_t>{i * castDstLastDim},
-          sliceShape, strides);
+      Value extracted = ExtractStridedSliceOp::create(
+          rewriter, loc, bitcastOp.getSource(),
+          ArrayRef<int64_t>{i * castDstLastDim}, sliceShape, strides);
       Value bitcast =
           BitCastOp::create(rewriter, loc, newCastDstType, extracted);
-      res = InsertStridedSliceOp::create(rewriter,
-          loc, bitcast, res,
+      res = InsertStridedSliceOp::create(
+          rewriter, loc, bitcast, res,
           ArrayRef<int64_t>{i * castDstLastDim / shrinkRatio}, strides);
     }
     rewriter.replaceOp(bitcastOp, res);
@@ -1303,8 +1305,8 @@ static Value buildVectorComparison(PatternRewriter &rewriter, Operation *op,
   Value bound = getValueOrCreateCastToIndexLike(rewriter, loc, idxType, b);
   Value bounds =
       vector::SplatOp::create(rewriter, loc, indices.getType(), bound);
-  return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt, indices,
-                                        bounds);
+  return arith::CmpIOp::create(rewriter, loc, arith::CmpIPredicate::slt,
+                               indices, bounds);
 }
 
 template <typename ConcreteOp>
@@ -1336,8 +1338,8 @@ struct MaterializeTransferMask : public OpRewritePattern<ConcreteOp> {
     Value dim =
         vector::createOrFoldDimOp(rewriter, loc, xferOp.getBase(), lastIndex);
     Value b = arith::SubIOp::create(rewriter, loc, dim.getType(), dim, off);
-    Value mask = vector::CreateMaskOp::create(rewriter,
-        loc,
+    Value mask = vector::CreateMaskOp::create(
+        rewriter, loc,
         VectorType::get(vtp.getShape(), rewriter.getI1Type(),
                         vtp.getScalableDims()),
         b);
@@ -1548,12 +1550,13 @@ class DropInnerMostUnitDimsTransferRead
         strides);
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         readOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
-    Value rankedReducedView = memref::SubViewOp::create(rewriter,
-        loc, resultMemrefType, readOp.getBase(), offsets, sizes, strides);
+    Value rankedReducedView =
+        memref::SubViewOp::create(rewriter, loc, resultMemrefType,
+                                  readOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
-    Value result = vector::TransferReadOp::create(rewriter,
-        loc, resultTargetVecType, rankedReducedView,
+    Value result = vector::TransferReadOp::create(
+        rewriter, loc, resultTargetVecType, rankedReducedView,
         readOp.getIndices().drop_back(dimsToDrop), AffineMapAttr::get(permMap),
         readOp.getPadding(),
         // TODO: support mask.
@@ -1639,8 +1642,9 @@ class DropInnerMostUnitDimsTransferWrite
     ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
         writeOp.getInBoundsAttr().getValue().drop_back(dimsToDrop));
 
-    Value rankedReducedView = memref::SubViewOp::create(rewriter,
-        loc, resultMemrefType, writeOp.getBase(), offsets, sizes, strides);
+    Value rankedReducedView =
+        memref::SubViewOp::create(rewriter, loc, resultMemrefType,
+                                  writeOp.getBase(), offsets, sizes, strides);
     auto permMap = getTransferMinorIdentityMap(
         cast<ShapedType>(rankedReducedView.getType()), resultTargetVecType);
 
@@ -1837,7 +1841,7 @@ struct ChainedReduction final : OpRewritePattern<vector::ReductionOp> {
           loc, parentReduction.getVector(), op.getVector());
     } else {
       vAdd = arith::AddFOp::create(rewriter, loc, parentReduction.getVector(),
-                                            op.getVector());
+                                   op.getVector());
     }
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), vAdd,
                                                      parentReduction.getAcc());
@@ -2004,8 +2008,8 @@ struct DropUnitDimsFromTransposeOp final
 
     Location loc = op.getLoc();
     // Drop the unit dims via shape_cast.
-    auto dropDimsShapeCast = vector::ShapeCastOp::create(rewriter,
-        loc, sourceTypeWithoutUnitDims, op.getVector());
+    auto dropDimsShapeCast = vector::ShapeCastOp::create(
+        rewriter, loc, sourceTypeWithoutUnitDims, op.getVector());
     // Create the new transpose.
     auto transposeWithoutUnitDims =
         vector::TransposeOp::create(rewriter, loc, dropDimsShapeCast, newPerm);
@@ -2111,8 +2115,8 @@ struct ReduceRedundantZero final : OpRewritePattern<vector::ReductionOp> {
     if (!matchPattern(addLhs.getRhs(), m_AnyZeroFloat()))
       return failure();
 
-    auto newAdd = arith::AddFOp::create(rewriter, vAdd.getLoc(), addLhs.getLhs(),
-                                                 vAdd.getRhs());
+    auto newAdd = arith::AddFOp::create(rewriter, vAdd.getLoc(),
+                                        addLhs.getLhs(), vAdd.getRhs());
     rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, op.getKind(), newAdd,
                                                      op.getAcc());
     return success();
@@ -2154,8 +2158,8 @@ struct BreakDownVectorReduction final : OpRewritePattern<vector::ReductionOp> {
     Location loc = op.getLoc();
     SmallVector<Value> extracted(numElems, nullptr);
     for (auto [idx, extractedElem] : llvm::enumerate(extracted))
-      extractedElem = vector::ExtractOp::create(rewriter,
-          loc, op.getVector(), static_cast<int64_t>(idx));
+      extractedElem = vector::ExtractOp::create(rewriter, loc, op.getVector(),
+                                                static_cast<int64_t>(idx));
 
     Value res = extracted.front();
     for (auto extractedElem : llvm::drop_begin(extracted))
@@ -2234,8 +2238,8 @@ struct FoldArithToVectorOuterProduct : public OpRewritePattern<MulOpType> {
       if (!broadcastedRhs || !isValidBroadcastSource(broadcastedRhs))
         return failure();
 
-      return vector::OuterProductOp::create(rewriter,
-          mulOp->getLoc(), resType, broadcastedLhs.getSource(),
+      return vector::OuterProductOp::create(
+          rewriter, mulOp->getLoc(), resType, broadcastedLhs.getSource(),
           broadcastedRhs.getSource(), Value(), vector::CombiningKind::ADD);
     };
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 9fa01061c1bed..fceba65fa3e3a 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -68,8 +68,8 @@ static SmallVector<Value> sliceLoadStoreIndices(PatternRewriter &rewriter,
   auto start = indices.size() - offsets.size();
   for (auto [i, offset] : llvm::enumerate(offsets)) {
     if (offset != 0) {
-      indices[start + i] = arith::AddIOp::create(rewriter,
-          loc, originalIndices[start + i],
+      indices[start + i] = arith::AddIOp::create(
+          rewriter, loc, originalIndices[start + i],
           arith::ConstantIndexOp::create(rewriter, loc, offset));
     }
   }
@@ -172,8 +172,9 @@ struct UnrollTransferReadPattern
     ArrayRef<int64_t> originalSize = readOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
+    Value result =
+        arith::ConstantOp::create(rewriter, loc, sourceVectorType,
+                                  rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
     SmallVector<Value> originalIndices(readOp.getIndices().begin(),
@@ -185,8 +186,8 @@ struct UnrollTransferReadPattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                readOp.getPermutationMap(), loc, rewriter);
-      auto slicedRead = vector::TransferReadOp::create(rewriter,
-          loc, targetType, readOp.getBase(), indices,
+      auto slicedRead = vector::TransferReadOp::create(
+          rewriter, loc, targetType, readOp.getBase(), indices,
           readOp.getPermutationMapAttr(), readOp.getPadding(), readOp.getMask(),
           readOp.getInBoundsAttr());
 
@@ -236,9 +237,10 @@ struct UnrollTransferWritePattern
       SmallVector<Value> indices =
           sliceTransferIndices(elementOffsets, originalIndices,
                                writeOp.getPermutationMap(), loc, rewriter);
-      Operation *slicedWrite = vector::TransferWriteOp::create(rewriter,
-          loc, slicedVector, resultTensor ? resultTensor : writeOp.getBase(),
-          indices, writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
+      Operation *slicedWrite = vector::TransferWriteOp::create(
+          rewriter, loc, slicedVector,
+          resultTensor ? resultTensor : writeOp.getBase(), indices,
+          writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
       // For the tensor case update the destination for the next transfer write.
       if (!slicedWrite->getResults().empty())
         resultTensor = slicedWrite->getResult(0);
@@ -348,8 +350,8 @@ struct UnrollContractionPattern
       accCache[dstOffets] = newOp->getResult(0);
     }
     // Assemble back the accumulator into a single vector.
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, dstVecType, rewriter.getZeroAttr(dstVecType));
+    Value result = arith::ConstantOp::create(rewriter, loc, dstVecType,
+                                             rewriter.getZeroAttr(dstVecType));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
@@ -427,8 +429,8 @@ struct UnrollMultiReductionPattern
       accCache[destOffset] = result;
     }
     // Assemble back the accumulator into a single vector.
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, reductionOp.getDestType(),
+    Value result = arith::ConstantOp::create(
+        rewriter, loc, reductionOp.getDestType(),
         rewriter.getZeroAttr(reductionOp.getDestType()));
     for (const auto &it : accCache) {
       SmallVector<int64_t> dstStrides(it.first.size(), 1);
@@ -468,8 +470,8 @@ struct UnrollElementwisePattern : public RewritePattern {
           op, "expected input vector rank to match target shape rank");
     Location loc = op->getLoc();
     // Prepare the result vector.
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, dstVecType, rewriter.getZeroAttr(dstVecType));
+    Value result = arith::ConstantOp::create(rewriter, loc, dstVecType,
+                                             rewriter.getZeroAttr(dstVecType));
     SmallVector<int64_t> strides(targetShape->size(), 1);
     VectorType newVecType =
         VectorType::get(*targetShape, dstVecType.getElementType());
@@ -567,8 +569,9 @@ struct UnrollTransposePattern : public OpRewritePattern<vector::TransposeOp> {
     ArrayRef<int64_t> originalSize = originalVectorType.getShape();
 
     // Prepare the result vector;
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, originalVectorType, rewriter.getZeroAttr(originalVectorType));
+    Value result =
+        arith::ConstantOp::create(rewriter, loc, originalVectorType,
+                                  rewriter.getZeroAttr(originalVectorType));
     ArrayRef<int64_t> permutation = transposeOp.getPermutation();
 
     // Unroll the computation.
@@ -618,8 +621,9 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
     ArrayRef<int64_t> originalSize = gatherOp.getVectorType().getShape();
 
     // Prepare the result vector;
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
+    Value result =
+        arith::ConstantOp::create(rewriter, loc, sourceVectorType,
+                                  rewriter.getZeroAttr(sourceVectorType));
     auto targetType =
         VectorType::get(*targetShape, sourceVectorType.getElementType());
 
@@ -638,8 +642,8 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
           rewriter.createOrFold<vector::ExtractStridedSliceOp>(
               loc, gatherOp.getPassThru(), elementOffsets, *targetShape,
               strides);
-      auto slicedGather = vector::GatherOp::create(rewriter,
-          loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
+      auto slicedGather = vector::GatherOp::create(
+          rewriter, loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
           indexSubVec, maskSubVec, passThruSubVec);
 
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
@@ -671,8 +675,8 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
     ArrayRef<int64_t> originalShape = vecType.getShape();
     SmallVector<int64_t> strides(targetShape->size(), 1);
 
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, vecType, rewriter.getZeroAttr(vecType));
+    Value result = arith::ConstantOp::create(rewriter, loc, vecType,
+                                             rewriter.getZeroAttr(vecType));
 
     SmallVector<int64_t> loopOrder =
         getUnrollOrder(originalShape.size(), loadOp, options);
@@ -684,8 +688,8 @@ struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
          StaticTileOffsetRange(originalShape, *targetShape, loopOrder)) {
       SmallVector<Value> indices =
           sliceLoadStoreIndices(rewriter, loc, loadOp.getIndices(), offsets);
-      Value slicedLoad = vector::LoadOp::create(rewriter,
-          loc, targetVecType, loadOp.getBase(), indices);
+      Value slicedLoad = vector::LoadOp::create(rewriter, loc, targetVecType,
+                                                loadOp.getBase(), indices);
       result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
           loc, slicedLoad, result, offsets, strides);
     }
@@ -755,8 +759,8 @@ struct UnrollBroadcastPattern : public OpRewritePattern<vector::BroadcastOp> {
     VectorType resType = broadcastOp.getResultVectorType();
     VectorType targetType =
         resType.cloneWith(*targetShape, resType.getElementType());
-    Value result = arith::ConstantOp::create(rewriter,
-        loc, resType, rewriter.getZeroAttr(resType));
+    Value result = arith::ConstantOp::create(rewriter, loc, resType,
+                                             rewriter.getZeroAttr(resType));
 
     SmallVector<int64_t> originalShape = *broadcastOp.getShapeForUnroll();
     SmallVector<int64_t> strides(originalShape.size(), 1);
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 7c31ada1644c5..ac542ddd7d1c4 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -341,8 +341,8 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
       inBoundsVal[i] = (sourceShape[i] == inputVectorSizes[i]) &&
                        ShapedType::isStatic(sourceShape[i]);
   }
-  auto transferReadOp = vector::TransferReadOp::create(builder,
-      loc,
+  auto transferReadOp = vector::TransferReadOp::create(
+      builder, loc,
       /*vectorType=*/vectorType,
       /*source=*/source,
       /*indices=*/SmallVector<Value>(readRank, zero),
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
index 8f87661a68d96..e11595a09cc45 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
+++ b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
@@ -38,8 +38,8 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
       "=x,x,x"; // Careful: constraint parser is very brittle: no ws!
   SmallVector<Value> asmVals{v1, v2};
   auto asmStr = llvm::formatv(asmTp, llvm::format_hex(mask, /*width=*/2)).str();
-  auto asmOp = LLVM::InlineAsmOp::create(b,
-      v1.getType(), /*operands=*/asmVals, /*asm_string=*/asmStr,
+  auto asmOp = LLVM::InlineAsmOp::create(
+      b, v1.getType(), /*operands=*/asmVals, /*asm_string=*/asmStr,
       /*constraints=*/asmCstr, /*has_side_effects=*/false,
       /*is_align_stack=*/false, LLVM::TailCallKind::None,
       /*asm_dialect=*/asmDialectAttr,
@@ -49,14 +49,14 @@ Value mlir::x86vector::avx2::inline_asm::mm256BlendPsAsm(
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackLoPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return vector::ShuffleOp::create(b,
-      v1, v2, ArrayRef<int64_t>{0, 8, 1, 9, 4, 12, 5, 13});
+  return vector::ShuffleOp::create(b, v1, v2,
+                                   ArrayRef<int64_t>{0, 8, 1, 9, 4, 12, 5, 13});
 }
 
 Value mlir::x86vector::avx2::intrin::mm256UnpackHiPs(ImplicitLocOpBuilder &b,
                                                      Value v1, Value v2) {
-  return vector::ShuffleOp::create(b,
-      v1, v2, ArrayRef<int64_t>{2, 10, 3, 11, 6, 14, 7, 15});
+  return vector::ShuffleOp::create(
+      b, v1, v2, ArrayRef<int64_t>{2, 10, 3, 11, 6, 14, 7, 15});
 }
 ///                            a  a   b   b  a  a   b   b
 /// Takes an 8 bit mask, 2 bit for each position of a[0, 3)  **and** b[0, 4):
@@ -262,7 +262,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
       // Insert transposed 1-D vectors into the higher-order dimension of the
       // output vector.
       Value res = arith::ConstantOp::create(ib, reshInputType,
-                                               ib.getZeroAttr(reshInputType));
+                                            ib.getZeroAttr(reshInputType));
       for (int64_t i = 0; i < m; ++i)
         res = vector::InsertOp::create(ib, vs[i], res, i);
 
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index 613a33fe63d87..efcafa7d33d2b 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -127,7 +127,7 @@ static Value resolveDistributedTy(Value orig, T expected,
   // op to reconcile the types.
   if (isa<xegpu::TensorDescType>(orig.getType())) {
     auto castOp = UnrealizedConversionCastOp::create(rewriter, orig.getLoc(),
-                                                              expected, orig);
+                                                     expected, orig);
     castOp->setAttr(resolveSIMTTypeMismatch, rewriter.getUnitAttr());
     return castOp.getResult(0);
   }
@@ -198,17 +198,18 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         }))
       return failure();
     // Create a new function with the same signature.
-    auto newGpuFunc = gpu::GPUFuncOp::create(rewriter,
-        gpuFuncOp.getLoc(), gpuFuncOp.getName(), gpuFuncOp.getFunctionType());
+    auto newGpuFunc = gpu::GPUFuncOp::create(rewriter, gpuFuncOp.getLoc(),
+                                             gpuFuncOp.getName(),
+                                             gpuFuncOp.getFunctionType());
     // Create a WarpExecuteOnLane0Op with same arguments and results as the
     // original gpuFuncOp.
     rewriter.setInsertionPointToEnd(&newGpuFunc.getFunctionBody().front());
-    auto laneId = gpu::LaneIdOp::create(rewriter,
-        newGpuFunc.getLoc(), rewriter.getIndexType(),
+    auto laneId = gpu::LaneIdOp::create(
+        rewriter, newGpuFunc.getLoc(), rewriter.getIndexType(),
         /** upperBound = **/ mlir::IntegerAttr());
     ArrayRef<Type> gpuFuncResultType = gpuFuncOp.getFunctionType().getResults();
-    auto warpOp = gpu::WarpExecuteOnLane0Op::create(rewriter,
-        laneId.getLoc(), gpuFuncResultType, laneId,
+    auto warpOp = gpu::WarpExecuteOnLane0Op::create(
+        rewriter, laneId.getLoc(), gpuFuncResultType, laneId,
         xegpu::targetinfo::subgroupSize, newGpuFunc.getArguments(),
         newGpuFunc.getArgumentTypes());
     Block &warpBodyBlock = warpOp.getBodyRegion().front();
@@ -217,7 +218,7 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         cast<gpu::ReturnOp>(gpuFuncOp.getBlocks().back().getTerminator());
     rewriter.setInsertionPointAfter(origRetunOp);
     gpu::YieldOp::create(rewriter, origRetunOp.getLoc(),
-                                  origRetunOp.getOperands());
+                         origRetunOp.getOperands());
     rewriter.eraseOp(origRetunOp);
     // Move the original function body to the WarpExecuteOnLane0Op body.
     rewriter.inlineRegionBefore(gpuFuncOp.getBody(), warpOp.getBodyRegion(),
@@ -301,8 +302,8 @@ struct CreateNdDescDistribution final : public gpu::WarpDistributionPattern {
     xegpu::TensorDescType distributedTensorDescTy =
         descOp.getType().dropLayouts(); // Distributed tensor descriptor type
                                         // does not contain layout info.
-    Value newDescOp = xegpu::CreateNdDescOp::create(rewriter,
-        newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
+    Value newDescOp = xegpu::CreateNdDescOp::create(
+        rewriter, newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
         descOp->getAttrs());
 
     Value distributedVal = newWarpOp.getResult(operandIdx);
@@ -403,8 +404,8 @@ struct StoreNdDistribution final : public gpu::WarpDistributionPattern {
         resolveDistributedTy(newWarpOp.getResult(newRetIndices[1]),
                              distributedTensorDescTy, rewriter));
 
-    xegpu::StoreNdOp::create(rewriter,
-        newWarpOp.getLoc(), TypeRange{}, newStoreOperands,
+    xegpu::StoreNdOp::create(
+        rewriter, newWarpOp.getLoc(), TypeRange{}, newStoreOperands,
         removeTemporaryLayoutAttributes(storeOp->getAttrs()));
     rewriter.eraseOp(storeOp);
     return success();
@@ -494,8 +495,8 @@ struct LoadNdDistribution final : public gpu::WarpDistributionPattern {
         loadOp.getTensorDescType().dropLayouts(); // Distributed tensor
                                                   // descriptor type does not
                                                   // contain layout info.
-    auto newLoadOp = xegpu::LoadNdOp::create(rewriter,
-        newWarpOp.getLoc(), loadNdDistValueTyOrFailure.value(),
+    auto newLoadOp = xegpu::LoadNdOp::create(
+        rewriter, newWarpOp.getLoc(), loadNdDistValueTyOrFailure.value(),
         resolveDistributedTy(newWarpOp->getResult(newRetIndices[0]),
                              distributedTensorDescTy, rewriter),
         removeTemporaryLayoutAttributes(loadOp->getAttrs()));
@@ -630,8 +631,8 @@ struct DpasDistribution final : public gpu::WarpDistributionPattern {
           resolveDistributedTy(newWarpOp.getResult(newRetIndices[i]),
                                newDpasOperandExpectedTypes[i], rewriter));
     }
-    Value newDpasOp = xegpu::DpasOp::create(rewriter,
-        newWarpOp->getLoc(), distributedResultTy, newDpasOperands,
+    Value newDpasOp = xegpu::DpasOp::create(
+        rewriter, newWarpOp->getLoc(), distributedResultTy, newDpasOperands,
         removeTemporaryLayoutAttributes(dpasOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
     // Resolve the output type.
@@ -717,8 +718,8 @@ struct UpdateNdOffsetDistribution final : public gpu::WarpDistributionPattern {
       }
     }
     // Create a new update op outside the warp op.
-    Value newUpdateOp = xegpu::UpdateNdOffsetOp::create(rewriter,
-        newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
+    Value newUpdateOp = xegpu::UpdateNdOffsetOp::create(
+        rewriter, newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
         removeTemporaryLayoutAttributes(updateOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
     // Resolve the distributed type with the original type.
@@ -783,8 +784,8 @@ struct PrefetchNdDistribution final : public gpu::WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     SmallVector<Value> newPrefetchOperands = {resolveDistributedTy(
         newWarpOp.getResult(newRetIndices[0]), newTensorDescTy, rewriter)};
-    xegpu::PrefetchNdOp::create(rewriter,
-        newWarpOp.getLoc(), TypeRange{}, newPrefetchOperands,
+    xegpu::PrefetchNdOp::create(
+        rewriter, newWarpOp.getLoc(), TypeRange{}, newPrefetchOperands,
         removeTemporaryLayoutAttributes(prefetchOp->getAttrs()));
     rewriter.eraseOp(prefetchOp);
     return success();
@@ -806,9 +807,9 @@ struct GpuBarrierDistribution final : public gpu::WarpDistributionPattern {
       return failure();
     // Move the barrier op outside of the warp op.
     rewriter.setInsertionPointAfter(subgroupOp);
-    gpu::BarrierOp::create(rewriter,
-        barrierOp.getLoc(), barrierOp->getResultTypes(),
-        barrierOp->getOperands(), barrierOp->getAttrs());
+    gpu::BarrierOp::create(rewriter, barrierOp.getLoc(),
+                           barrierOp->getResultTypes(),
+                           barrierOp->getOperands(), barrierOp->getAttrs());
     rewriter.eraseOp(barrierOp);
     return success();
   }
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
index 6266cca918d4f..83aeb9dc189ef 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
@@ -83,8 +83,9 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = UnrealizedConversionCastOp::create(rewriter,
-          loc, destTy, srcs, ArrayRef<NamedAttribute>({attr, blkAttr}));
+      auto castOp = UnrealizedConversionCastOp::create(
+          rewriter, loc, destTy, srcs,
+          ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResult(0);
     }
 
@@ -109,8 +110,9 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
                                  rewriter.getUnitAttr());
       auto blkAttr = NamedAttribute(rewriter.getStringAttr(blockAttrName),
                                     rewriter.getDenseI64ArrayAttr(blockSize));
-      auto castOp = UnrealizedConversionCastOp::create(rewriter,
-          loc, destTypes, src, ArrayRef<NamedAttribute>({attr, blkAttr}));
+      auto castOp = UnrealizedConversionCastOp::create(
+          rewriter, loc, destTypes, src,
+          ArrayRef<NamedAttribute>({attr, blkAttr}));
       return castOp.getResults();
     }
 
@@ -169,9 +171,9 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
            llvm::zip(validIdxes, oldOffsets, offsets))
         mixedOffsets[idx] = addi(oldOff, offset);
 
-      auto newOp = xegpu::CreateNdDescOp::create(rewriter,
-          loc, newTdescTy, op.getSource(), mixedOffsets, op.getMixedSizes(),
-          op.getMixedStrides());
+      auto newOp = xegpu::CreateNdDescOp::create(
+          rewriter, loc, newTdescTy, op.getSource(), mixedOffsets,
+          op.getMixedSizes(), op.getMixedStrides());
       newOps.push_back(newOp);
     }
     Value castOp = unpack(newOps, tdescTy, *targetShape, loc, rewriter);
@@ -199,8 +201,8 @@ struct UnrollUpdateNdOffsetOp : public UnrollPattern<xegpu::UpdateNdOffsetOp> {
 
     SmallVector<Value> newOps;
     for (auto t : convertedTdesc) {
-      auto newOp = xegpu::UpdateNdOffsetOp::create(rewriter,
-          loc, t.getType(), t, op.getOffsets(), op.getConstOffsets());
+      auto newOp = xegpu::UpdateNdOffsetOp::create(
+          rewriter, loc, t.getType(), t, op.getOffsets(), op.getConstOffsets());
       newOps.push_back(newOp);
     }
     Value castOp = unpack(newOps, op.getType(), *targetShape, loc, rewriter);
@@ -226,7 +228,8 @@ struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> {
         op.getTensorDesc(), convertedTdescTypes, *targetShape, loc, rewriter);
 
     for (auto t : convertedTdesc)
-      xegpu::PrefetchNdOp::create(rewriter, loc, TypeRange(), t, op->getAttrs());
+      xegpu::PrefetchNdOp::create(rewriter, loc, TypeRange(), t,
+                                  op->getAttrs());
 
     rewriter.eraseOp(op);
     return success();
@@ -292,7 +295,7 @@ struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> {
 
     for (auto [v, t] : llvm::zip(convertedValues, convertedTdescs))
       xegpu::StoreNdOp::create(rewriter, loc, v, t, op.getL1HintAttr(),
-                                        op.getL2HintAttr(), op.getL3HintAttr());
+                               op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
     return success();
@@ -385,7 +388,7 @@ struct UnrollDpasOp : public UnrollPattern<xegpu::DpasOp> {
             operands.push_back(tmpC);
 
           tmpC = xegpu::DpasOp::create(rewriter, loc, vecTy, operands,
-                                                op->getAttrs());
+                                       op->getAttrs());
         }
         newOps.push_back(tmpC);
       }
@@ -436,22 +439,23 @@ struct UnrollCreateDescOp : public UnrollPattern<xegpu::CreateDescOp> {
            llvm::zip(convertedIndiceVec, convertedIndiceTypes)) {
         for (int64_t i = 0; i < numNewChunks; ++i) {
           // Compute the offset
-          Value inc = arith::ConstantIndexOp::create(rewriter,
-              loc, i * blockedChunkSize);
-          Value incVec = vector::SplatOp::create(rewriter, loc, indiceType, inc);
+          Value inc = arith::ConstantIndexOp::create(rewriter, loc,
+                                                     i * blockedChunkSize);
+          Value incVec =
+              vector::SplatOp::create(rewriter, loc, indiceType, inc);
           Value offsetIndice =
               arith::AddIOp::create(rewriter, loc, indice, incVec);
 
-          auto newOp = xegpu::CreateDescOp::create(rewriter,
-              loc, newTdescTy, op.getSource(), offsetIndice);
+          auto newOp = xegpu::CreateDescOp::create(
+              rewriter, loc, newTdescTy, op.getSource(), offsetIndice);
 
           newOps.push_back(newOp);
         }
       }
     } else {
       for (auto indice : convertedIndiceVec) {
-        auto newOp = xegpu::CreateDescOp::create(rewriter,
-            loc, newTdescTy, op.getSource(), indice);
+        auto newOp = xegpu::CreateDescOp::create(rewriter, loc, newTdescTy,
+                                                 op.getSource(), indice);
         newOps.push_back(newOp);
       }
     }
@@ -515,9 +519,9 @@ struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
 
     SmallVector<Value> newOps;
     for (auto [t, m] : llvm::zip(convertedTdescs, convertedMasks)) {
-      auto newOp = xegpu::LoadGatherOp::create(rewriter,
-          loc, newValueTy, t, m, op.getL1HintAttr(), op.getL2HintAttr(),
-          op.getL3HintAttr());
+      auto newOp = xegpu::LoadGatherOp::create(
+          rewriter, loc, newValueTy, t, m, op.getL1HintAttr(),
+          op.getL2HintAttr(), op.getL3HintAttr());
       newOps.push_back(newOp);
     }
 
@@ -609,8 +613,7 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
       Value t = convertedTdescs[i];
       Value m = op.getMask() ? convertedMasks[i] : nullptr;
       xegpu::StoreScatterOp::create(rewriter, loc, v, t, m, op.getL1HintAttr(),
-                                             op.getL2HintAttr(),
-                                             op.getL3HintAttr());
+                                    op.getL2HintAttr(), op.getL3HintAttr());
     }
 
     rewriter.eraseOp(op);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index aa2fc78c77f39..80bb5e888bdc7 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -201,9 +201,9 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
           calculateGlobalOffsets(rewriter, loc, originalOffsets, localOffset,
                                  distUnitBaseAddr, distUnitShape);
 
-      auto newCreateNdOp = xegpu::CreateNdDescOp::create(rewriter,
-          loc, newTdescTy, op.getSource(), globalOffsets, op.getMixedSizes(),
-          op.getMixedStrides());
+      auto newCreateNdOp = xegpu::CreateNdDescOp::create(
+          rewriter, loc, newTdescTy, op.getSource(), globalOffsets,
+          op.getMixedSizes(), op.getMixedStrides());
       newCreateNdOps.push_back(newCreateNdOp);
     }
 
@@ -225,7 +225,7 @@ struct WgToSgLoadNdOp : public OpConversionPattern<xegpu::LoadNdOp> {
       ArrayRef<int64_t> srcShape = tdescTy.getShape();
       VectorType newResTy = VectorType::get(srcShape, tdescTy.getElementType());
       auto newLoadOp = xegpu::LoadNdOp::create(rewriter, op.getLoc(), newResTy,
-                                                        src, op->getAttrs());
+                                               src, op->getAttrs());
       newLoadOps.push_back(newLoadOp);
     }
     rewriter.replaceOpWithMultiple(op, {newLoadOps});
@@ -243,7 +243,7 @@ struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
                   ConversionPatternRewriter &rewriter) const override {
     for (auto [v, t] : llvm::zip(adaptor.getValue(), adaptor.getTensorDesc()))
       xegpu::StoreNdOp::create(rewriter, op.getLoc(), v, t, op.getL1HintAttr(),
-                                        op.getL2HintAttr(), op.getL3HintAttr());
+                               op.getL2HintAttr(), op.getL3HintAttr());
 
     rewriter.eraseOp(op);
     return success();
@@ -261,8 +261,8 @@ struct WgToSgUpdateNdOffsetOp
                   ConversionPatternRewriter &rewriter) const override {
     llvm::SmallVector<Value> newUpdateTileOffsetOps;
     for (auto tDesc : adaptor.getTensorDesc()) {
-      auto newUpdateTileOffsetOp = xegpu::UpdateNdOffsetOp::create(rewriter,
-          op.getLoc(), tDesc.getType(), tDesc, op.getOffsets(),
+      auto newUpdateTileOffsetOp = xegpu::UpdateNdOffsetOp::create(
+          rewriter, op.getLoc(), tDesc.getType(), tDesc, op.getOffsets(),
           op.getConstOffsets());
       newUpdateTileOffsetOps.push_back(newUpdateTileOffsetOp);
     }
@@ -325,7 +325,7 @@ struct WgToSgPrefetchNdOp : public OpConversionPattern<xegpu::PrefetchNdOp> {
                   ConversionPatternRewriter &rewriter) const override {
     for (auto src : adaptor.getTensorDesc())
       xegpu::PrefetchNdOp::create(rewriter, op.getLoc(), TypeRange(), src,
-                                           op->getAttrs());
+                                  op->getAttrs());
     rewriter.eraseOp(op);
     return success();
   }
diff --git a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index 9acb209d031bd..ca7c2d63b8726 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -198,8 +198,8 @@ xegpu::extractVectorsWithShapeFromValue(OpBuilder &builder, Location loc,
   SmallVector<Value> result;
   for (SmallVector<int64_t> offsets : StaticTileOffsetRange(srcShape, shape)) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result.push_back(vector::ExtractStridedSliceOp::create(builder,
-        loc, value, offsets, shape, staticStrides));
+    result.push_back(vector::ExtractStridedSliceOp::create(
+        builder, loc, value, offsets, shape, staticStrides));
   }
 
   return result;
@@ -218,14 +218,14 @@ Value xegpu::createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
 
   VectorType resultTy = VectorType::get(shape, elemTy);
   auto zeroAttr = builder.getZeroAttr(elemTy);
-  Value result = arith::ConstantOp::create(builder,
-      loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
+  Value result = arith::ConstantOp::create(
+      builder, loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
 
   for (auto [src, offsets] :
        llvm::zip_equal(values, StaticTileOffsetRange(shape, tileShape))) {
     SmallVector<int64_t> staticStrides(offsets.size(), 1);
-    result = vector::InsertStridedSliceOp::create(builder,
-        loc, src, result, offsets, staticStrides);
+    result = vector::InsertStridedSliceOp::create(builder, loc, src, result,
+                                                  offsets, staticStrides);
   }
   return result;
 }
@@ -343,8 +343,8 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
 
         if (isa<RankedTensorType>(inputTy) && isa<VectorType>(outputTy)) {
           SmallVector<Value> values = xegpu::flattenValues(adaptor.getInputs());
-          auto newOp = UnrealizedConversionCastOp::create(rewriter,
-              op.getLoc(), outputTy, values);
+          auto newOp = UnrealizedConversionCastOp::create(rewriter, op.getLoc(),
+                                                          outputTy, values);
           rewriter.replaceOp(op, newOp);
           return success();
         }
diff --git a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
index c0f19f6dce600..0fb1adbbf75aa 100644
--- a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
+++ b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
@@ -37,8 +37,8 @@ LogicalResult mlir::LLVMImportInterface::convertUnregisteredIntrinsic(
     return failure();
 
   Type resultType = moduleImport.convertType(inst->getType());
-  auto op = ::mlir::LLVM::CallIntrinsicOp::create(builder,
-      moduleImport.translateLoc(inst->getDebugLoc()),
+  auto op = ::mlir::LLVM::CallIntrinsicOp::create(
+      builder, moduleImport.translateLoc(inst->getDebugLoc()),
       isa<LLVMVoidType>(resultType) ? TypeRange{} : TypeRange{resultType},
       StringAttr::get(builder.getContext(), intrinName),
       ValueRange{mlirOperands}, FastmathFlagsAttr{});
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 6e47db6d1dc6a..51901191fe863 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -865,7 +865,7 @@ LogicalResult ModuleImport::convertModuleFlagsMetadata() {
 
   if (!moduleFlags.empty())
     LLVM::ModuleFlagsOp::create(builder, mlirModule.getLoc(),
-                                        builder.getArrayAttr(moduleFlags));
+                                builder.getArrayAttr(moduleFlags));
 
   return success();
 }
@@ -881,7 +881,7 @@ LogicalResult ModuleImport::convertLinkerOptionsMetadata() {
       for (const llvm::MDOperand &option : node->operands())
         options.push_back(cast<llvm::MDString>(option)->getString());
       LLVM::LinkerOptionsOp::create(builder, mlirModule.getLoc(),
-                                            builder.getStrArrayAttr(options));
+                                    builder.getStrArrayAttr(options));
     }
   }
   return success();
@@ -984,8 +984,8 @@ void ModuleImport::processComdat(const llvm::Comdat *comdat) {
   ComdatOp comdatOp = getGlobalComdatOp();
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(&comdatOp.getBody().back());
-  auto selectorOp = ComdatSelectorOp::create(builder,
-      mlirModule.getLoc(), comdat->getName(),
+  auto selectorOp = ComdatSelectorOp::create(
+      builder, mlirModule.getLoc(), comdat->getName(),
       convertComdatFromLLVM(comdat->getSelectionKind()));
   auto symbolRef =
       SymbolRefAttr::get(builder.getContext(), getGlobalComdatOpName(),
@@ -1346,12 +1346,12 @@ LogicalResult ModuleImport::convertAlias(llvm::GlobalAlias *alias) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   Type type = convertType(alias->getValueType());
-  AliasOp aliasOp = AliasOp::create(builder,
-      mlirModule.getLoc(), type, convertLinkageFromLLVM(alias->getLinkage()),
-      alias->getName(),
-      /*dso_local=*/alias->isDSOLocal(),
-      /*thread_local=*/alias->isThreadLocal(),
-      /*attrs=*/ArrayRef<NamedAttribute>());
+  AliasOp aliasOp = AliasOp::create(builder, mlirModule.getLoc(), type,
+                                    convertLinkageFromLLVM(alias->getLinkage()),
+                                    alias->getName(),
+                                    /*dso_local=*/alias->isDSOLocal(),
+                                    /*thread_local=*/alias->isThreadLocal(),
+                                    /*attrs=*/ArrayRef<NamedAttribute>());
   globalInsertionOp = aliasOp;
 
   clearRegionState();
@@ -1403,8 +1403,8 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
   if (globalName.empty())
     globalName = getOrCreateNamelessSymbolName(globalVar).getValue();
 
-  GlobalOp globalOp = GlobalOp::create(builder,
-      mlirModule.getLoc(), type, globalVar->isConstant(),
+  GlobalOp globalOp = GlobalOp::create(
+      builder, mlirModule.getLoc(), type, globalVar->isConstant(),
       convertLinkageFromLLVM(globalVar->getLinkage()), StringRef(globalName),
       valueAttr, alignment, /*addr_space=*/globalVar->getAddressSpace(),
       /*dso_local=*/globalVar->isDSOLocal(),
@@ -1488,13 +1488,13 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
   OpBuilder::InsertionGuard guard = setGlobalInsertionPoint();
 
   if (globalVar->getName() == getGlobalCtorsVarName()) {
-    globalInsertionOp = LLVM::GlobalCtorsOp::create(builder,
-        mlirModule.getLoc(), builder.getArrayAttr(funcs),
+    globalInsertionOp = LLVM::GlobalCtorsOp::create(
+        builder, mlirModule.getLoc(), builder.getArrayAttr(funcs),
         builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
     return success();
   }
-  globalInsertionOp = LLVM::GlobalDtorsOp::create(builder,
-      mlirModule.getLoc(), builder.getArrayAttr(funcs),
+  globalInsertionOp = LLVM::GlobalDtorsOp::create(
+      builder, mlirModule.getLoc(), builder.getArrayAttr(funcs),
       builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
   return success();
 }
@@ -1683,13 +1683,14 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
     Value root = UndefOp::create(builder, loc, rootType);
     for (const auto &it : llvm::enumerate(elementValues)) {
       if (isArrayOrStruct) {
-        root = InsertValueOp::create(builder, loc, root, it.value(), it.index());
+        root =
+            InsertValueOp::create(builder, loc, root, it.value(), it.index());
       } else {
         Attribute indexAttr = builder.getI32IntegerAttr(it.index());
         Value indexValue =
             ConstantOp::create(builder, loc, builder.getI32Type(), indexAttr);
         root = InsertElementOp::create(builder, loc, rootType, root, it.value(),
-                                               indexValue);
+                                       indexValue);
       }
     }
     return root;
@@ -2125,15 +2126,15 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
 
     if (!brInst->isConditional()) {
       auto brOp = LLVM::BrOp::create(builder, loc, succBlockArgs.front(),
-                                             succBlocks.front());
+                                     succBlocks.front());
       mapNoResultOp(inst, brOp);
       return success();
     }
     FailureOr<Value> condition = convertValue(brInst->getCondition());
     if (failed(condition))
       return failure();
-    auto condBrOp = LLVM::CondBrOp::create(builder,
-        loc, *condition, succBlocks.front(), succBlockArgs.front(),
+    auto condBrOp = LLVM::CondBrOp::create(
+        builder, loc, *condition, succBlocks.front(), succBlockArgs.front(),
         succBlocks.back(), succBlockArgs.back());
     mapNoResultOp(inst, condBrOp);
     return success();
@@ -2166,9 +2167,9 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       caseBlocks[it.index()] = lookupBlock(succBB);
     }
 
-    auto switchOp = SwitchOp::create(builder,
-        loc, *condition, lookupBlock(defaultBB), defaultBlockArgs, caseValues,
-        caseBlocks, caseOperandRefs);
+    auto switchOp = SwitchOp::create(builder, loc, *condition,
+                                     lookupBlock(defaultBB), defaultBlockArgs,
+                                     caseValues, caseBlocks, caseOperandRefs);
     mapNoResultOp(inst, switchOp);
     return success();
   }
@@ -2218,8 +2219,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
         // IR). Build the indirect call by passing an empty `callee` operand and
         // insert into `operands` to include the indirect call target.
         FlatSymbolRefAttr calleeSym = convertCalleeName(callInst);
-        Value indirectCallVal = LLVM::AddressOfOp::create(builder,
-            loc, LLVM::LLVMPointerType::get(context), calleeSym);
+        Value indirectCallVal = LLVM::AddressOfOp::create(
+            builder, loc, LLVM::LLVMPointerType::get(context), calleeSym);
         operands->insert(operands->begin(), indirectCallVal);
       } else {
         // Regular direct call using callee name.
@@ -2310,8 +2311,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
       // IR). Build the indirect invoke by passing an empty `callee` operand and
       // insert into `operands` to include the indirect invoke target.
       FlatSymbolRefAttr calleeSym = convertCalleeName(invokeInst);
-      Value indirectInvokeVal = LLVM::AddressOfOp::create(builder,
-          loc, LLVM::LLVMPointerType::get(context), calleeSym);
+      Value indirectInvokeVal = LLVM::AddressOfOp::create(
+          builder, loc, LLVM::LLVMPointerType::get(context), calleeSym);
       operands->insert(operands->begin(), indirectInvokeVal);
     } else {
       // Regular direct invoke using callee name.
@@ -2320,9 +2321,9 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     // Create the invoke operation. Normal destination block arguments will be
     // added later on to handle the case in which the operation result is
     // included in this list.
-    auto invokeOp = InvokeOp::create(builder,
-        loc, *funcTy, calleeName, *operands, directNormalDest, ValueRange(),
-        lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
+    auto invokeOp = InvokeOp::create(
+        builder, loc, *funcTy, calleeName, *operands, directNormalDest,
+        ValueRange(), lookupBlock(invokeInst->getUnwindDest()), unwindArgs);
 
     if (failed(convertInvokeAttributes(invokeInst, invokeOp)))
       return failure();
@@ -2382,8 +2383,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     }
 
     Type type = convertType(inst->getType());
-    auto gepOp = GEPOp::create(builder,
-        loc, type, sourceElementType, *basePtr, indices,
+    auto gepOp = GEPOp::create(
+        builder, loc, type, sourceElementType, *basePtr, indices,
         static_cast<GEPNoWrapFlags>(gepInst->getNoWrapFlags().getRaw()));
     mapValue(inst, gepOp);
     return success();
@@ -2409,8 +2410,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     SmallVector<ValueRange> succBlockArgsRange =
         llvm::to_vector_of<ValueRange>(succBlockArgs);
     Location loc = translateLoc(inst->getDebugLoc());
-    auto indBrOp = LLVM::IndirectBrOp::create(builder,
-        loc, *basePtr, succBlockArgsRange, succBlocks);
+    auto indBrOp = LLVM::IndirectBrOp::create(builder, loc, *basePtr,
+                                              succBlockArgsRange, succBlocks);
 
     mapNoResultOp(inst, indBrOp);
     return success();
@@ -2854,8 +2855,8 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) {
   builder.setInsertionPointToEnd(mlirModule.getBody());
 
   Location loc = debugImporter->translateFuncLocation(func);
-  LLVMFuncOp funcOp = LLVMFuncOp::create(builder,
-      loc, func->getName(), functionType,
+  LLVMFuncOp funcOp = LLVMFuncOp::create(
+      builder, loc, func->getName(), functionType,
       convertLinkageFromLLVM(func->getLinkage()), dsoLocal, cconv);
 
   convertParameterAttributes(func, funcOp, builder);
@@ -3032,12 +3033,12 @@ ModuleImport::processDebugIntrinsic(llvm::DbgVariableIntrinsic *dbgIntr,
   Operation *op =
       llvm::TypeSwitch<llvm::DbgVariableIntrinsic *, Operation *>(dbgIntr)
           .Case([&](llvm::DbgDeclareInst *) {
-            return LLVM::DbgDeclareOp::create(builder,
-                loc, *argOperand, localVariableAttr, locationExprAttr);
+            return LLVM::DbgDeclareOp::create(
+                builder, loc, *argOperand, localVariableAttr, locationExprAttr);
           })
           .Case([&](llvm::DbgValueInst *) {
-            return LLVM::DbgValueOp::create(builder,
-                loc, *argOperand, localVariableAttr, locationExprAttr);
+            return LLVM::DbgValueOp::create(
+                builder, loc, *argOperand, localVariableAttr, locationExprAttr);
           });
   mapNoResultOp(dbgIntr, op);
   setNonDebugMetadataAttrs(dbgIntr, op);
@@ -3083,7 +3084,7 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb,
     OpBuilder::InsertionGuard guard(builder);
     builder.setInsertionPointToStart(block);
     BlockTagOp::create(builder, block->getParentOp()->getLoc(),
-                               BlockTagAttr::get(context, bb->getNumber()));
+                       BlockTagAttr::get(context, bb->getNumber()));
   }
   return success();
 }
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index 6e14b94e87407..85b0e04754acf 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -43,22 +43,23 @@ Value spirv::Deserializer::getValue(uint32_t id) {
   if (auto constInfo = getConstant(id)) {
     // Materialize a `spirv.Constant` op at every use site.
     return spirv::ConstantOp::create(opBuilder, unknownLoc, constInfo->second,
-                                               constInfo->first);
+                                     constInfo->first);
   }
   if (auto varOp = getGlobalVariable(id)) {
-    auto addressOfOp = spirv::AddressOfOp::create(opBuilder,
-        unknownLoc, varOp.getType(), SymbolRefAttr::get(varOp.getOperation()));
+    auto addressOfOp =
+        spirv::AddressOfOp::create(opBuilder, unknownLoc, varOp.getType(),
+                                   SymbolRefAttr::get(varOp.getOperation()));
     return addressOfOp.getPointer();
   }
   if (auto constOp = getSpecConstant(id)) {
-    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
-        unknownLoc, constOp.getDefaultValue().getType(),
+    auto referenceOfOp = spirv::ReferenceOfOp::create(
+        opBuilder, unknownLoc, constOp.getDefaultValue().getType(),
         SymbolRefAttr::get(constOp.getOperation()));
     return referenceOfOp.getReference();
   }
   if (auto constCompositeOp = getSpecConstantComposite(id)) {
-    auto referenceOfOp = spirv::ReferenceOfOp::create(opBuilder,
-        unknownLoc, constCompositeOp.getType(),
+    auto referenceOfOp = spirv::ReferenceOfOp::create(
+        opBuilder, unknownLoc, constCompositeOp.getType(),
         SymbolRefAttr::get(constCompositeOp.getOperation()));
     return referenceOfOp.getReference();
   }
@@ -369,8 +370,9 @@ Deserializer::processOp<spirv::EntryPointOp>(ArrayRef<uint32_t> words) {
     interface.push_back(SymbolRefAttr::get(arg.getOperation()));
     wordIndex++;
   }
-  spirv::EntryPointOp::create(opBuilder,
-      unknownLoc, execModel, SymbolRefAttr::get(opBuilder.getContext(), fnName),
+  spirv::EntryPointOp::create(
+      opBuilder, unknownLoc, execModel,
+      SymbolRefAttr::get(opBuilder.getContext(), fnName),
       opBuilder.getArrayAttr(interface));
   return success();
 }
@@ -402,9 +404,10 @@ Deserializer::processOp<spirv::ExecutionModeOp>(ArrayRef<uint32_t> words) {
     attrListElems.push_back(opBuilder.getI32IntegerAttr(words[wordIndex++]));
   }
   auto values = opBuilder.getArrayAttr(attrListElems);
-  spirv::ExecutionModeOp::create(opBuilder,
-      unknownLoc, SymbolRefAttr::get(opBuilder.getContext(), fn.getName()),
-      execMode, values);
+  spirv::ExecutionModeOp::create(
+      opBuilder, unknownLoc,
+      SymbolRefAttr::get(opBuilder.getContext(), fn.getName()), execMode,
+      values);
   return success();
 }
 
@@ -441,8 +444,8 @@ Deserializer::processOp<spirv::FunctionCallOp>(ArrayRef<uint32_t> operands) {
     arguments.push_back(value);
   }
 
-  auto opFunctionCall = spirv::FunctionCallOp::create(opBuilder,
-      unknownLoc, resultType,
+  auto opFunctionCall = spirv::FunctionCallOp::create(
+      opBuilder, unknownLoc, resultType,
       SymbolRefAttr::get(opBuilder.getContext(), functionName), arguments);
 
   if (resultType)
@@ -518,7 +521,8 @@ Deserializer::processOp<spirv::CopyMemoryOp>(ArrayRef<uint32_t> words) {
   }
 
   Location loc = createFileLineColLoc(opBuilder);
-  spirv::CopyMemoryOp::create(opBuilder, loc, resultTypes, operands, attributes);
+  spirv::CopyMemoryOp::create(opBuilder, loc, resultTypes, operands,
+                              attributes);
 
   return success();
 }
@@ -549,8 +553,8 @@ LogicalResult Deserializer::processOp<spirv::GenericCastToPtrExplicitOp>(
   operands.push_back(arg);
 
   Location loc = createFileLineColLoc(opBuilder);
-  Operation *op = spirv::GenericCastToPtrExplicitOp::create(opBuilder,
-      loc, resultTypes, operands);
+  Operation *op = spirv::GenericCastToPtrExplicitOp::create(
+      opBuilder, loc, resultTypes, operands);
   valueMap[valueID] = op->getResult(0);
   return success();
 }
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index 7b1f8055fadce..2eaed616773db 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -518,8 +518,8 @@ spirv::Deserializer::processFunction(ArrayRef<uint32_t> operands) {
   }
 
   std::string fnName = getFunctionSymbol(fnID);
-  auto funcOp = spirv::FuncOp::create(opBuilder,
-      unknownLoc, fnName, functionType, fnControl.value());
+  auto funcOp = spirv::FuncOp::create(opBuilder, unknownLoc, fnName,
+                                      functionType, fnControl.value());
   // Processing other function attributes.
   if (decorations.count(fnID)) {
     for (auto attr : decorations[fnID].getAttrs()) {
@@ -707,7 +707,7 @@ spirv::Deserializer::createSpecConstant(Location loc, uint32_t resultID,
                                         TypedAttr defaultValue) {
   auto symName = opBuilder.getStringAttr(getSpecConstantSymbol(resultID));
   auto op = spirv::SpecConstantOp::create(opBuilder, unknownLoc, symName,
-                                                    defaultValue);
+                                          defaultValue);
   if (decorations.count(resultID)) {
     for (auto attr : decorations[resultID].getAttrs())
       op->setAttr(attr.getName(), attr.getValue());
@@ -782,9 +782,9 @@ spirv::Deserializer::processGlobalVariable(ArrayRef<uint32_t> operands) {
            << wordIndex << " of " << operands.size() << " processed";
   }
   auto loc = createFileLineColLoc(opBuilder);
-  auto varOp = spirv::GlobalVariableOp::create(opBuilder,
-      loc, TypeAttr::get(type), opBuilder.getStringAttr(variableName),
-      initializer);
+  auto varOp = spirv::GlobalVariableOp::create(
+      opBuilder, loc, TypeAttr::get(type),
+      opBuilder.getStringAttr(variableName), initializer);
 
   // Decorations.
   if (decorations.count(variableID)) {
@@ -1581,8 +1581,8 @@ spirv::Deserializer::processSpecConstantComposite(ArrayRef<uint32_t> operands) {
     elements.push_back(SymbolRefAttr::get(elementInfo));
   }
 
-  auto op = spirv::SpecConstantCompositeOp::create(opBuilder,
-      unknownLoc, TypeAttr::get(resultType), symName,
+  auto op = spirv::SpecConstantCompositeOp::create(
+      opBuilder, unknownLoc, TypeAttr::get(resultType), symName,
       opBuilder.getArrayAttr(elements));
   specConstCompositeMap[resultID] = op;
 
@@ -1764,8 +1764,8 @@ spirv::Deserializer::processBranchConditional(ArrayRef<uint32_t> operands) {
   // an OpSelectionMerge instruction, in this case they will have the same
   // OpLine information.
   auto loc = createFileLineColLoc(opBuilder);
-  spirv::BranchConditionalOp::create(opBuilder,
-      loc, condition, trueBlock,
+  spirv::BranchConditionalOp::create(
+      opBuilder, loc, condition, trueBlock,
       /*trueArguments=*/ArrayRef<Value>(), falseBlock,
       /*falseArguments=*/ArrayRef<Value>(), weights);
 
@@ -2093,7 +2093,7 @@ LogicalResult ControlFlowStructurizer::structurize() {
     // loop header block.
     builder.setInsertionPointToEnd(&body.front());
     spirv::BranchOp::create(builder, location, mapper.lookupOrNull(headerBlock),
-                                    ArrayRef<Value>(blockArgs));
+                            ArrayRef<Value>(blockArgs));
   }
 
   // Values defined inside the selection region that need to be yielded outside
@@ -2177,12 +2177,12 @@ LogicalResult ControlFlowStructurizer::structurize() {
     Operation *newOp = nullptr;
 
     if (isLoop)
-      newOp = spirv::LoopOp::create(builder,
-          location, TypeRange(ValueRange(outsideUses)),
-          static_cast<spirv::LoopControl>(control));
+      newOp = spirv::LoopOp::create(builder, location,
+                                    TypeRange(ValueRange(outsideUses)),
+                                    static_cast<spirv::LoopControl>(control));
     else
-      newOp = spirv::SelectionOp::create(builder,
-          location, TypeRange(ValueRange(outsideUses)),
+      newOp = spirv::SelectionOp::create(
+          builder, location, TypeRange(ValueRange(outsideUses)),
           static_cast<spirv::SelectionControl>(control));
 
     newOp->getRegion(0).takeBody(body);
@@ -2362,22 +2362,22 @@ LogicalResult spirv::Deserializer::wireUpBlockArgument() {
 
     if (auto branchOp = dyn_cast<spirv::BranchOp>(op)) {
       // Replace the previous branch op with a new one with block arguments.
-      spirv::BranchOp::create(opBuilder, branchOp.getLoc(), branchOp.getTarget(),
-                                        blockArgs);
+      spirv::BranchOp::create(opBuilder, branchOp.getLoc(),
+                              branchOp.getTarget(), blockArgs);
       branchOp.erase();
     } else if (auto branchCondOp = dyn_cast<spirv::BranchConditionalOp>(op)) {
       assert((branchCondOp.getTrueBlock() == target ||
               branchCondOp.getFalseBlock() == target) &&
              "expected target to be either the true or false target");
       if (target == branchCondOp.getTrueTarget())
-        spirv::BranchConditionalOp::create(opBuilder,
-            branchCondOp.getLoc(), branchCondOp.getCondition(), blockArgs,
-            branchCondOp.getFalseBlockArguments(),
+        spirv::BranchConditionalOp::create(
+            opBuilder, branchCondOp.getLoc(), branchCondOp.getCondition(),
+            blockArgs, branchCondOp.getFalseBlockArguments(),
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueTarget(),
             branchCondOp.getFalseTarget());
       else
-        spirv::BranchConditionalOp::create(opBuilder,
-            branchCondOp.getLoc(), branchCondOp.getCondition(),
+        spirv::BranchConditionalOp::create(
+            opBuilder, branchCondOp.getLoc(), branchCondOp.getCondition(),
             branchCondOp.getTrueBlockArguments(), blockArgs,
             branchCondOp.getBranchWeightsAttr(), branchCondOp.getTrueBlock(),
             branchCondOp.getFalseBlock());
diff --git a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
index 3f4e7762fed0e..0677828b635d4 100644
--- a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
+++ b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
@@ -206,7 +206,7 @@ static void checkAndNestUnderRewriteOp(OpBuilder &builder, Value rootExpr,
   if (isa<pdl::PatternOp>(builder.getInsertionBlock()->getParentOp())) {
     pdl::RewriteOp rewrite =
         pdl::RewriteOp::create(builder, loc, rootExpr, /*name=*/StringAttr(),
-                                       /*externalArgs=*/ValueRange());
+                               /*externalArgs=*/ValueRange());
     builder.createBlock(&rewrite.getBodyRegion());
   }
 }
@@ -242,8 +242,8 @@ void CodeGen::genImpl(const ast::ReplaceStmt *stmt) {
   bool usesReplOperation =
       replValues.size() == 1 &&
       isa<pdl::OperationType>(replValues.front().getType());
-  pdl::ReplaceOp::create(builder,
-      loc, rootExpr, usesReplOperation ? replValues[0] : Value(),
+  pdl::ReplaceOp::create(
+      builder, loc, rootExpr, usesReplOperation ? replValues[0] : Value(),
       usesReplOperation ? ValueRange() : ValueRange(replValues));
 }
 
@@ -283,8 +283,8 @@ void CodeGen::genImpl(const ast::PatternDecl *decl) {
 
   // FIXME: Properly model HasBoundedRecursion in PDL so that we don't drop it
   // here.
-  pdl::PatternOp pattern = pdl::PatternOp::create(builder,
-      genLoc(decl->getLoc()), decl->getBenefit(),
+  pdl::PatternOp pattern = pdl::PatternOp::create(
+      builder, genLoc(decl->getLoc()), decl->getBenefit(),
       name ? std::optional<StringRef>(name->getName())
            : std::optional<StringRef>());
 
@@ -344,24 +344,25 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
   if (isa<ast::AttributeType>(type))
     return pdl::AttributeOp::create(builder, loc, getTypeConstraint());
   if (ast::OperationType opType = dyn_cast<ast::OperationType>(type)) {
-    Value operands = pdl::OperandsOp::create(builder,
-        loc, pdl::RangeType::get(builder.getType<pdl::ValueType>()),
+    Value operands = pdl::OperandsOp::create(
+        builder, loc, pdl::RangeType::get(builder.getType<pdl::ValueType>()),
         /*type=*/Value());
-    Value results = pdl::TypesOp::create(builder,
-        loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
+    Value results = pdl::TypesOp::create(
+        builder, loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
         /*types=*/ArrayAttr());
     return pdl::OperationOp::create(builder, loc, opType.getName(), operands,
-                                            ArrayRef<StringRef>(), ValueRange(),
-                                            results);
+                                    ArrayRef<StringRef>(), ValueRange(),
+                                    results);
   }
 
   if (ast::RangeType rangeTy = dyn_cast<ast::RangeType>(type)) {
     ast::Type eleTy = rangeTy.getElementType();
     if (isa<ast::ValueType>(eleTy))
       return pdl::OperandsOp::create(builder, loc, mlirType,
-                                             getTypeConstraint());
+                                     getTypeConstraint());
     if (isa<ast::TypeType>(eleTy))
-      return pdl::TypesOp::create(builder, loc, mlirType, /*types=*/ArrayAttr());
+      return pdl::TypesOp::create(builder, loc, mlirType,
+                                  /*types=*/ArrayAttr());
   }
 
   llvm_unreachable("invalid non-initialized variable type");
@@ -444,7 +445,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
       Type mlirType = genType(expr->getType());
       if (isa<pdl::ValueType>(mlirType))
         return pdl::ResultOp::create(builder, loc, mlirType, parentExprs[0],
-                                             builder.getI32IntegerAttr(0));
+                                     builder.getI32IntegerAttr(0));
       return pdl::ResultsOp::create(builder, loc, mlirType, parentExprs[0]);
     }
 
@@ -456,7 +457,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
       name.getAsInteger(/*Radix=*/10, resultIndex);
       IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
       return pdl::ResultOp::create(builder, loc, genType(expr->getType()),
-                                           parentExprs[0], index);
+                                   parentExprs[0], index);
     }
 
     // Find the result with the member name or by index.
@@ -475,7 +476,7 @@ Value CodeGen::genExprImpl(const ast::MemberAccessExpr *expr) {
     // Generate the result access.
     IntegerAttr index = builder.getI32IntegerAttr(resultIndex);
     return pdl::ResultsOp::create(builder, loc, genType(expr->getType()),
-                                          parentExprs[0], index);
+                                  parentExprs[0], index);
   }
 
   // Handle tuple based member access.
@@ -519,7 +520,7 @@ Value CodeGen::genExprImpl(const ast::OperationExpr *expr) {
     results.push_back(genSingleExpr(result));
 
   return pdl::OperationOp::create(builder, loc, opName, operands, attrNames,
-                                          attrValues, results);
+                                  attrValues, results);
 }
 
 Value CodeGen::genExprImpl(const ast::RangeExpr *expr) {
@@ -528,7 +529,7 @@ Value CodeGen::genExprImpl(const ast::RangeExpr *expr) {
     llvm::append_range(elements, genExpr(element));
 
   return pdl::RangeOp::create(builder, genLoc(expr->getLoc()),
-                                      genType(expr->getType()), elements);
+                              genType(expr->getType()), elements);
 }
 
 SmallVector<Value> CodeGen::genExprImpl(const ast::TupleExpr *expr) {
@@ -542,8 +543,8 @@ Value CodeGen::genExprImpl(const ast::TypeExpr *expr) {
   Type type = parseType(expr->getValue(), builder.getContext());
   assert(type && "invalid MLIR type data");
   return pdl::TypeOp::create(builder, genLoc(expr->getLoc()),
-                                     builder.getType<pdl::TypeType>(),
-                                     TypeAttr::get(type));
+                             builder.getType<pdl::TypeType>(),
+                             TypeAttr::get(type));
 }
 
 SmallVector<Value>
@@ -587,7 +588,7 @@ CodeGen::genConstraintOrRewriteCall(const T *decl, Location loc,
       resultTypes.push_back(genType(declResultType));
     }
     PDLOpT pdlOp = PDLOpT::create(builder, loc, resultTypes,
-                                          decl->getName().getName(), inputs);
+                                  decl->getName().getName(), inputs);
     if (isNegated && std::is_same_v<PDLOpT, pdl::ApplyNativeConstraintOp>)
       cast<pdl::ApplyNativeConstraintOp>(pdlOp).setIsNegated(true);
     return pdlOp->getResults();
diff --git a/mlir/test/python/lib/PythonTestDialect.cpp b/mlir/test/python/lib/PythonTestDialect.cpp
index e0a76f1b75d38..0c29902024ca6 100644
--- a/mlir/test/python/lib/PythonTestDialect.cpp
+++ b/mlir/test/python/lib/PythonTestDialect.cpp
@@ -8,8 +8,8 @@
 
 #include "PythonTestDialect.h"
 #include "mlir/IR/DialectImplementation.h"
-#include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
+#include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "PythonTestDialect.cpp.inc"


From flang-commits at lists.llvm.org  Mon Jul 14 10:22:25 2025
From: flang-commits at lists.llvm.org (=?UTF-8?Q?Andrzej_Warzy=C5=84ski?= via flang-commits)
Date: Mon, 14 Jul 2025 10:22:25 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][Flang][NFC] Replace use of
 `vector.insertelement/extractelement` (PR #143272)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143272@github.com>
Message-ID: <68753cd1.630a0220.17e5fc.aa47@mx.google.com>


https://github.com/banach-space approved this pull request.

LGTM

It looks like you've addressed comments from @kkwli . I think that you should just land this if there are no further comments by tomorrow.

Thanks Diego!

https://github.com/llvm/llvm-project/pull/143272

From flang-commits at lists.llvm.org  Mon Jul 14 10:53:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 10:53:57 -0700 (PDT)
Subject: [flang-commits] [flang] b54cfa4 - [flang][acc] Implement
 MappableType's generatePrivateInit (#148302)
Message-ID: <68754435.050a0220.88bfa.adc8@mx.google.com>


Author: Razvan Lupusoru
Date: 2025-07-14T10:53:54-07:00
New Revision: b54cfa46a7eba293b6f2086856a36cc553444939

URL: https://github.com/llvm/llvm-project/commit/b54cfa46a7eba293b6f2086856a36cc553444939
DIFF: https://github.com/llvm/llvm-project/commit/b54cfa46a7eba293b6f2086856a36cc553444939.diff

LOG: [flang][acc] Implement MappableType's generatePrivateInit (#148302)

The recipe body generation was moved from lowering into FIR's
implementation of MappableType API. And now since all Fortran variable
types implement this type, lowering of OpenACC was updated to use this
API directly. No test changes were needed - all of the private,
firstprivate, and recipe tests get the same body as before.

Added: 
    

Modified: 
    flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
    flang/lib/Lower/CMakeLists.txt
    flang/lib/Lower/OpenACC.cpp
    flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h b/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
index 3e343f347e4ae..10853933317f7 100644
--- a/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
+++ b/flang/include/flang/Optimizer/OpenACC/FIROpenACCTypeInterfaces.h
@@ -52,6 +52,13 @@ struct OpenACCMappableModel
 
   mlir::acc::VariableTypeCategory getTypeCategory(mlir::Type type,
                                                   mlir::Value var) const;
+
+  mlir::Value generatePrivateInit(mlir::Type type, mlir::OpBuilder &builder,
+                                  mlir::Location loc,
+                                  mlir::TypedValue<mlir::acc::MappableType> var,
+                                  llvm::StringRef varName,
+                                  mlir::ValueRange extents,
+                                  mlir::Value initVal) const;
 };
 
 } // namespace fir::acc

diff  --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index cd80aaf553869..8e20abf0e9f2d 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -51,6 +51,7 @@ add_flang_library(FortranLower
   FIRDialect
   FIRDialectSupport
   FIRBuilder
+  FIROpenACCSupport
   FIRSupport
   FIRTransforms
   HLFIRDialect

diff  --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 00c9cbf0d2a8f..39e4444cde4e3 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -959,119 +959,6 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
   llvm::report_fatal_error("Unsupported OpenACC reduction type");
 }
 
-template <typename RecipeOp>
-static void genPrivateLikeInitRegion(fir::FirOpBuilder &builder,
-                                     RecipeOp recipe, mlir::Type argTy,
-                                     mlir::Location loc,
-                                     mlir::Value initValue) {
-  mlir::Value retVal = recipe.getInitRegion().front().getArgument(0);
-  mlir::Type unwrappedTy = fir::unwrapRefType(argTy);
-
-  llvm::StringRef initName;
-  if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>)
-    initName = accReductionInitName;
-  else
-    initName = accPrivateInitName;
-
-  auto getDeclareOpForType = [&](mlir::Type ty) -> hlfir::DeclareOp {
-    auto alloca = builder.create<fir::AllocaOp>(loc, ty);
-    return builder.create<hlfir::DeclareOp>(
-        loc, alloca, initName, /*shape=*/nullptr, llvm::ArrayRef<mlir::Value>{},
-        /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
-  };
-
-  if (fir::isa_trivial(unwrappedTy)) {
-    auto declareOp = getDeclareOpForType(unwrappedTy);
-    if (initValue) {
-      auto convert = builder.createConvert(loc, unwrappedTy, initValue);
-      builder.create<fir::StoreOp>(loc, convert, declareOp.getBase());
-    }
-    retVal = declareOp.getBase();
-  } else if (auto seqTy =
-                 mlir::dyn_cast_or_null<fir::SequenceType>(unwrappedTy)) {
-    if (fir::isa_trivial(seqTy.getEleTy())) {
-      mlir::Value shape;
-      llvm::SmallVector<mlir::Value> extents;
-      if (seqTy.hasDynamicExtents()) {
-        // Extents are passed as block arguments. First argument is the
-        // original value.
-        for (unsigned i = 1; i < recipe.getInitRegion().getArguments().size();
-             ++i)
-          extents.push_back(recipe.getInitRegion().getArgument(i));
-        shape = builder.create<fir::ShapeOp>(loc, extents);
-      } else {
-        shape = genShapeOp(builder, seqTy, loc);
-      }
-      auto alloca = builder.create<fir::AllocaOp>(
-          loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
-      auto declareOp = builder.create<hlfir::DeclareOp>(
-          loc, alloca, initName, shape, llvm::ArrayRef<mlir::Value>{},
-          /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
-
-      if (initValue) {
-        mlir::Type idxTy = builder.getIndexType();
-        mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
-        llvm::SmallVector<fir::DoLoopOp> loops;
-        llvm::SmallVector<mlir::Value> ivs;
-
-        if (seqTy.hasDynamicExtents()) {
-          builder.create<hlfir::AssignOp>(loc, initValue, declareOp.getBase());
-        } else {
-          for (auto ext : seqTy.getShape()) {
-            auto lb = builder.createIntegerConstant(loc, idxTy, 0);
-            auto ub = builder.createIntegerConstant(loc, idxTy, ext - 1);
-            auto step = builder.createIntegerConstant(loc, idxTy, 1);
-            auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
-                                                      /*unordered=*/false);
-            builder.setInsertionPointToStart(loop.getBody());
-            loops.push_back(loop);
-            ivs.push_back(loop.getInductionVar());
-          }
-          auto coord = builder.create<fir::CoordinateOp>(
-              loc, refTy, declareOp.getBase(), ivs);
-          builder.create<fir::StoreOp>(loc, initValue, coord);
-          builder.setInsertionPointAfter(loops[0]);
-        }
-      }
-      retVal = declareOp.getBase();
-    }
-  } else if (auto boxTy =
-                 mlir::dyn_cast_or_null<fir::BaseBoxType>(unwrappedTy)) {
-    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
-    if (fir::isa_trivial(innerTy)) {
-      retVal = getDeclareOpForType(unwrappedTy).getBase();
-    } else if (mlir::isa<fir::SequenceType>(innerTy)) {
-      fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
-      hlfir::Entity source = hlfir::Entity{retVal};
-      auto [temp, cleanup] = hlfir::createTempFromMold(loc, firBuilder, source);
-      if (fir::isa_ref_type(argTy)) {
-        // When the temp is created - it is not a reference - thus we can
-        // end up with a type inconsistency. Therefore ensure storage is created
-        // for it.
-        retVal = getDeclareOpForType(unwrappedTy).getBase();
-        mlir::Value storeDst = retVal;
-        if (fir::unwrapRefType(retVal.getType()) != temp.getType()) {
-          // `createTempFromMold` makes the unfortunate choice to lose the
-          // `fir.heap` and `fir.ptr` types when wrapping with a box. Namely,
-          // when wrapping a `fir.heap<fir.array>`, it will create instead a
-          // `fir.box<fir.array>`. Cast here to deal with this inconsistency.
-          storeDst = firBuilder.createConvert(
-              loc, firBuilder.getRefType(temp.getType()), retVal);
-        }
-        builder.create<fir::StoreOp>(loc, temp, storeDst);
-      } else {
-        retVal = temp;
-      }
-    } else {
-      TODO(loc, "Unsupported boxed type for OpenACC private-like recipe");
-    }
-    if (initValue) {
-      builder.create<hlfir::AssignOp>(loc, initValue, retVal);
-    }
-  }
-  builder.create<mlir::acc::YieldOp>(loc, retVal);
-}
-
 template <typename RecipeOp>
 static RecipeOp genRecipeOp(
     fir::FirOpBuilder &builder, mlir::ModuleOp mod, llvm::StringRef recipeName,
@@ -1100,15 +987,35 @@ static RecipeOp genRecipeOp(
       }
     }
   }
-  builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
-                      argsTy, argsLoc);
+  auto initBlock = builder.createBlock(
+      &recipe.getInitRegion(), recipe.getInitRegion().end(), argsTy, argsLoc);
   builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
   mlir::Value initValue;
   if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>) {
     assert(op != mlir::acc::ReductionOperator::AccNone);
     initValue = getReductionInitValue(builder, loc, fir::unwrapRefType(ty), op);
   }
-  genPrivateLikeInitRegion<RecipeOp>(builder, recipe, ty, loc, initValue);
+
+  // Since we reuse the same recipe for all variables of the same type - we
+  // cannot use the actual variable name. Thus use a temporary name.
+  llvm::StringRef initName;
+  if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>)
+    initName = accReductionInitName;
+  else
+    initName = accPrivateInitName;
+
+  auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(ty);
+  assert(mappableTy &&
+         "Expected that all variable types are considered mappable");
+  auto retVal = mappableTy.generatePrivateInit(
+      builder, loc,
+      mlir::cast<mlir::TypedValue<mlir::acc::MappableType>>(
+          initBlock->getArgument(0)),
+      initName,
+      initBlock->getArguments().take_back(initBlock->getArguments().size() - 1),
+      initValue);
+  builder.create<mlir::acc::YieldOp>(loc, retVal ? retVal
+                                                 : initBlock->getArgument(0));
   return recipe;
 }
 

diff  --git a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
index 0767733f53728..a87e90e8878a3 100644
--- a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
@@ -25,6 +25,7 @@
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Support/LLVM.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 namespace fir::acc {
@@ -525,4 +526,141 @@ OpenACCPointerLikeModel<fir::LLVMPointerType>::getPointeeTypeCategory(
   return categorizePointee(pointer, varPtr, varType);
 }
 
+static fir::ShapeOp genShapeOp(mlir::OpBuilder &builder,
+                               fir::SequenceType seqTy, mlir::Location loc) {
+  llvm::SmallVector<mlir::Value> extents;
+  mlir::Type idxTy = builder.getIndexType();
+  for (auto extent : seqTy.getShape())
+    extents.push_back(builder.create<mlir::arith::ConstantOp>(
+        loc, idxTy, builder.getIntegerAttr(idxTy, extent)));
+  return builder.create<fir::ShapeOp>(loc, extents);
+}
+
+template <typename Ty>
+mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const {
+  mlir::Value retVal;
+  mlir::Type unwrappedTy = fir::unwrapRefType(type);
+  mlir::ModuleOp mod = builder.getInsertionBlock()
+                           ->getParent()
+                           ->getParentOfType<mlir::ModuleOp>();
+  fir::FirOpBuilder firBuilder(builder, mod);
+
+  auto getDeclareOpForType = [&](mlir::Type ty) -> hlfir::DeclareOp {
+    auto alloca = firBuilder.create<fir::AllocaOp>(loc, ty);
+    return firBuilder.create<hlfir::DeclareOp>(
+        loc, alloca, varName, /*shape=*/nullptr, llvm::ArrayRef<mlir::Value>{},
+        /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
+  };
+
+  if (fir::isa_trivial(unwrappedTy)) {
+    auto declareOp = getDeclareOpForType(unwrappedTy);
+    if (initVal) {
+      auto convert = firBuilder.createConvert(loc, unwrappedTy, initVal);
+      firBuilder.create<fir::StoreOp>(loc, convert, declareOp.getBase());
+    }
+    retVal = declareOp.getBase();
+  } else if (auto seqTy =
+                 mlir::dyn_cast_or_null<fir::SequenceType>(unwrappedTy)) {
+    if (fir::isa_trivial(seqTy.getEleTy())) {
+      mlir::Value shape;
+      if (seqTy.hasDynamicExtents()) {
+        shape = firBuilder.create<fir::ShapeOp>(loc, llvm::to_vector(extents));
+      } else {
+        shape = genShapeOp(firBuilder, seqTy, loc);
+      }
+      auto alloca = firBuilder.create<fir::AllocaOp>(
+          loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
+      auto declareOp = firBuilder.create<hlfir::DeclareOp>(
+          loc, alloca, varName, shape, llvm::ArrayRef<mlir::Value>{},
+          /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
+
+      if (initVal) {
+        mlir::Type idxTy = firBuilder.getIndexType();
+        mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
+        llvm::SmallVector<fir::DoLoopOp> loops;
+        llvm::SmallVector<mlir::Value> ivs;
+
+        if (seqTy.hasDynamicExtents()) {
+          firBuilder.create<hlfir::AssignOp>(loc, initVal, declareOp.getBase());
+        } else {
+          for (auto ext : seqTy.getShape()) {
+            auto lb = firBuilder.createIntegerConstant(loc, idxTy, 0);
+            auto ub = firBuilder.createIntegerConstant(loc, idxTy, ext - 1);
+            auto step = firBuilder.createIntegerConstant(loc, idxTy, 1);
+            auto loop = firBuilder.create<fir::DoLoopOp>(loc, lb, ub, step,
+                                                         /*unordered=*/false);
+            firBuilder.setInsertionPointToStart(loop.getBody());
+            loops.push_back(loop);
+            ivs.push_back(loop.getInductionVar());
+          }
+          auto coord = firBuilder.create<fir::CoordinateOp>(
+              loc, refTy, declareOp.getBase(), ivs);
+          firBuilder.create<fir::StoreOp>(loc, initVal, coord);
+          firBuilder.setInsertionPointAfter(loops[0]);
+        }
+      }
+      retVal = declareOp.getBase();
+    }
+  } else if (auto boxTy =
+                 mlir::dyn_cast_or_null<fir::BaseBoxType>(unwrappedTy)) {
+    mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
+    if (fir::isa_trivial(innerTy)) {
+      retVal = getDeclareOpForType(unwrappedTy).getBase();
+    } else if (mlir::isa<fir::SequenceType>(innerTy)) {
+      hlfir::Entity source = hlfir::Entity{var};
+      auto [temp, cleanup] = hlfir::createTempFromMold(loc, firBuilder, source);
+      if (fir::isa_ref_type(type)) {
+        // When the temp is created - it is not a reference - thus we can
+        // end up with a type inconsistency. Therefore ensure storage is created
+        // for it.
+        retVal = getDeclareOpForType(unwrappedTy).getBase();
+        mlir::Value storeDst = retVal;
+        if (fir::unwrapRefType(retVal.getType()) != temp.getType()) {
+          // `createTempFromMold` makes the unfortunate choice to lose the
+          // `fir.heap` and `fir.ptr` types when wrapping with a box. Namely,
+          // when wrapping a `fir.heap<fir.array>`, it will create instead a
+          // `fir.box<fir.array>`. Cast here to deal with this inconsistency.
+          storeDst = firBuilder.createConvert(
+              loc, firBuilder.getRefType(temp.getType()), retVal);
+        }
+        builder.create<fir::StoreOp>(loc, temp, storeDst);
+      } else {
+        retVal = temp;
+      }
+    } else {
+      TODO(loc, "Unsupported boxed type for OpenACC private-like recipe");
+    }
+    if (initVal) {
+      builder.create<hlfir::AssignOp>(loc, initVal, retVal);
+    }
+  }
+  return retVal;
+}
+
+template mlir::Value
+OpenACCMappableModel<fir::BaseBoxType>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const;
+
+template mlir::Value
+OpenACCMappableModel<fir::ReferenceType>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const;
+
+template mlir::Value OpenACCMappableModel<fir::HeapType>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const;
+
+template mlir::Value
+OpenACCMappableModel<fir::PointerType>::generatePrivateInit(
+    mlir::Type type, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::MappableType> var, llvm::StringRef varName,
+    mlir::ValueRange extents, mlir::Value initVal) const;
+
 } // namespace fir::acc


        

From flang-commits at lists.llvm.org  Mon Jul 14 10:54:00 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Mon, 14 Jul 2025 10:54:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][acc] Implement MappableType's
 generatePrivateInit (PR #148302)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148302@github.com>
Message-ID: <68754438.050a0220.6cc7d.a726@mx.google.com>


https://github.com/razvanlupusoru closed https://github.com/llvm/llvm-project/pull/148302

From flang-commits at lists.llvm.org  Mon Jul 14 11:11:47 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 11:11:47 -0700 (PDT)
Subject: [flang-commits] [flang] 4dceb25 - [flang] Don't create bogus tokens
 from token pasting (##) (#147596)
Message-ID: <68754863.170a0220.100704.af27@mx.google.com>


Author: Peter Klausler
Date: 2025-07-14T11:11:43-07:00
New Revision: 4dceb25dd1b96c93a7bd302c5d81bb1aacf75f4d

URL: https://github.com/llvm/llvm-project/commit/4dceb25dd1b96c93a7bd302c5d81bb1aacf75f4d
DIFF: https://github.com/llvm/llvm-project/commit/4dceb25dd1b96c93a7bd302c5d81bb1aacf75f4d.diff

LOG: [flang] Don't create bogus tokens from token pasting (##) (#147596)

When blank tokens arise from macro replacement in token sequences with
token pasting (##), the preprocessor is producing some bogus tokens
(e.g., "name(") that can lead to subtle bugs later when macro names are
not recognized as such.

The fix is to not paste tokens together when the result would not be a
valid Fortran or C token in the preprocessing context.

Added: 
    flang/test/Preprocessing/bug1126.F90

Modified: 
    flang/include/flang/Parser/char-block.h
    flang/lib/Parser/preprocessor.cpp
    flang/lib/Parser/unparse.cpp
    flang/lib/Semantics/expression.cpp
    flang/lib/Semantics/resolve-names.cpp
    flang/lib/Semantics/tools.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/char-block.h b/flang/include/flang/Parser/char-block.h
index b3b1f04034d3c..93a7e744d8d09 100644
--- a/flang/include/flang/Parser/char-block.h
+++ b/flang/include/flang/Parser/char-block.h
@@ -46,6 +46,8 @@ class CharBlock {
   constexpr const char &operator[](std::size_t j) const {
     return interval_.start()[j];
   }
+  constexpr const char &front() const { return (*this)[0]; }
+  constexpr const char &back() const { return (*this)[size() - 1]; }
 
   bool Contains(const CharBlock &that) const {
     return interval_.Contains(that.interval_);

diff  --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index 8ef9810463d5a..0aadc41934f3c 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -156,23 +156,50 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
   }
   TokenSequence result;
   std::size_t tokens{text.SizeInTokens()};
-  bool pasting{false};
+  std::optional<CharBlock> before; // last non-blank token before ##
   for (std::size_t j{0}; j < tokens; ++j) {
-    if (IsTokenPasting(text.TokenAt(j))) {
-      if (!pasting) {
+    CharBlock after{text.TokenAt(j)};
+    if (!before) {
+      if (IsTokenPasting(after)) {
         while (!result.empty() &&
             result.TokenAt(result.SizeInTokens() - 1).IsBlank()) {
           result.pop_back();
         }
         if (!result.empty()) {
-          result.ReopenLastToken();
-          pasting = true;
+          before = result.TokenAt(result.SizeInTokens() - 1);
         }
+      } else {
+        result.AppendRange(text, j, 1);
+      }
+    } else if (after.IsBlank() || IsTokenPasting(after)) {
+      // drop it
+    } else { // pasting before ## after
+      bool doPaste{false};
+      char last{before->back()};
+      char first{after.front()};
+      // Apply basic sanity checking to pasting so avoid constructing a bogus
+      // token that might cause macro replacement to fail, like "macro(".
+      if (IsLegalInIdentifier(last) && IsLegalInIdentifier(first)) {
+        doPaste = true;
+      } else if (IsDecimalDigit(first) &&
+          (last == '.' || last == '+' || last == '-')) {
+        doPaste = true; // 1. ## 0, - ## 1
+      } else if (before->size() == 1 && after.size() == 1) {
+        if (first == last &&
+            (last == '<' || last == '>' || last == '*' || last == '/' ||
+                last == '=' || last == '&' || last == '|' || last == ':')) {
+          // Fortran **, //, ==, ::
+          // C <<, >>, &&, || for use in #if expressions
+          doPaste = true;
+        } else if (first == '=' && (last == '!' || last == '/')) {
+          doPaste = true; // != and /=
+        }
+      }
+      if (doPaste) {
+        result.ReopenLastToken();
       }
-    } else if (pasting && text.TokenAt(j).IsBlank()) {
-    } else {
       result.AppendRange(text, j, 1);
-      pasting = false;
+      before.reset();
     }
   }
   return result;

diff  --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index b66d756bdbf2c..fbe89c668fc13 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -778,7 +778,7 @@ class UnparseVisitor {
   }
   void Unparse(const SubstringInquiry &x) {
     Walk(x.v);
-    Put(x.source.end()[-1] == 'n' ? "%LEN" : "%KIND");
+    Put(x.source.back() == 'n' ? "%LEN" : "%KIND");
   }
   void Unparse(const SubstringRange &x) { // R910
     Walk(x.t, ":");

diff  --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index f4af738284ed7..53ec3827893d0 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -1269,7 +1269,7 @@ MaybeExpr ExpressionAnalyzer::Analyze(
 MaybeExpr ExpressionAnalyzer::Analyze(const parser::SubstringInquiry &x) {
   if (MaybeExpr substring{Analyze(x.v)}) {
     CHECK(x.source.size() >= 8);
-    int nameLen{x.source.end()[-1] == 'n' ? 3 /*LEN*/ : 4 /*KIND*/};
+    int nameLen{x.source.back() == 'n' ? 3 /*LEN*/ : 4 /*KIND*/};
     parser::CharBlock name{
         x.source.end() - nameLen, static_cast<std::size_t>(nameLen)};
     CHECK(name == "len" || name == "kind");

diff  --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index d0336c9cb661d..96faa5fd954cd 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -2139,7 +2139,7 @@ bool ImplicitRules::isImplicitNoneExternal() const {
 
 const DeclTypeSpec *ImplicitRules::GetType(
     SourceName name, bool respectImplicitNoneType) const {
-  char ch{name.begin()[0]};
+  char ch{name.front()};
   if (isImplicitNoneType_ && respectImplicitNoneType) {
     return nullptr;
   } else if (auto it{map_.find(ch)}; it != map_.end()) {

diff  --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index d27d250b3f11e..5e5b43f26c791 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -1740,7 +1740,7 @@ std::forward_list<std::string> GetOperatorNames(
 std::forward_list<std::string> GetAllNames(
     const SemanticsContext &context, const SourceName &name) {
   std::string str{name.ToString()};
-  if (!name.empty() && name.end()[-1] == ')' &&
+  if (!name.empty() && name.back() == ')' &&
       name.ToString().rfind("operator(", 0) == 0) {
     for (int i{0}; i != common::LogicalOperator_enumSize; ++i) {
       auto names{GetOperatorNames(context, common::LogicalOperator{i})};

diff  --git a/flang/test/Preprocessing/bug1126.F90 b/flang/test/Preprocessing/bug1126.F90
new file mode 100644
index 0000000000000..ae5bb5633581d
--- /dev/null
+++ b/flang/test/Preprocessing/bug1126.F90
@@ -0,0 +1,20 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+#define PREFIX(x) prefix ## x
+#define NAME(x) PREFIX(foo ## x)
+#define AUGMENT(x) NAME(x ## suffix)
+
+! CHECK: subroutine prefixfoosuffix()
+! CHECK: print *, "prefixfoosuffix"
+! CHECK: end subroutine prefixfoosuffix
+subroutine AUGMENT()()
+  print *, TOSTRING(AUGMENT())
+end subroutine AUGMENT()
+
+! CHECK: subroutine prefixfoobarsuffix()
+! CHECK: print *, "prefixfoobarsuffix"
+! CHECK: end subroutine prefixfoobarsuffix
+subroutine AUGMENT(bar)()
+  print *, TOSTRING(AUGMENT(bar))
+end subroutine AUGMENT(bar)


        

From flang-commits at lists.llvm.org  Mon Jul 14 11:11:50 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 14 Jul 2025 11:11:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Don't create bogus tokens from
 token pasting (##) (PR #147596)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147596@github.com>
Message-ID: <68754866.170a0220.20f3b5.9b38@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/147596

From flang-commits at lists.llvm.org  Mon Jul 14 11:12:53 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 11:12:53 -0700 (PDT)
Subject: [flang-commits] [flang] 69f3844 - [flang] Extension: TRANSFER(boz,
 integer or real scalar) (#147604)
Message-ID: <687548a5.170a0220.8f8f9.8fa7@mx.google.com>


Author: Peter Klausler
Date: 2025-07-14T11:12:50-07:00
New Revision: 69f38443e5f96913d85650b2f73fb472b54cad24

URL: https://github.com/llvm/llvm-project/commit/69f38443e5f96913d85650b2f73fb472b54cad24
DIFF: https://github.com/llvm/llvm-project/commit/69f38443e5f96913d85650b2f73fb472b54cad24.diff

LOG: [flang] Extension: TRANSFER(boz, integer or real scalar) (#147604)

Interpret TRANSFER(SOURCE=BOZ literal, MOLD=integer or real scalar) as
if it had been a reference to the standard INT or REAL intrinsic
functions, for which a BOZ literal is an acceptable argument, with a
warning about non-conformance. It's a needless extension that has
somehow crept into some other compilers and real applications.

Added: 
    flang/test/Evaluate/transfer-boz.f90

Modified: 
    flang/docs/Extensions.md
    flang/include/flang/Support/Fortran-features.h
    flang/lib/Evaluate/fold.cpp
    flang/lib/Evaluate/intrinsics.cpp
    flang/lib/Support/Fortran-features.cpp

Removed: 
    


################################################################################
diff  --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 37a49f12f9177..72d12cd92600d 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -252,6 +252,8 @@ end
   the type resolution of such BOZ literals usages is highly non portable).
 * BOZ literals can also be used as REAL values in some contexts where the
   type is unambiguous, such as initializations of REAL parameters.
+* `TRANSFER(boz, MOLD=integer or real scalar)` is accepted as an alternate
+  spelling of `INT(boz, KIND=kind(mold))` or `REAL(boz, KIND=kind(mold))`.
 * EQUIVALENCE of numeric and character sequences (a ubiquitous extension),
   as well as of sequences of non-default kinds of numeric types
   with each other.

diff  --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 857de9479e4e5..743abf606ab5f 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,8 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload,
+    TransferBOZ)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,

diff  --git a/flang/lib/Evaluate/fold.cpp b/flang/lib/Evaluate/fold.cpp
index 45e842abf589f..71ead1b3afa91 100644
--- a/flang/lib/Evaluate/fold.cpp
+++ b/flang/lib/Evaluate/fold.cpp
@@ -287,6 +287,16 @@ std::optional<Expr<SomeType>> FoldTransfer(
         CHECK(status == InitialImage::NotAConstant);
       }
     }
+  } else if (source && moldType) {
+    if (const auto *boz{std::get_if<BOZLiteralConstant>(&source->u)}) {
+      // TRANSFER(BOZ, MOLD=integer or real) extension
+      if (context.languageFeatures().ShouldWarn(
+              common::LanguageFeature::TransferBOZ)) {
+        context.messages().Say(common::LanguageFeature::TransferBOZ,
+            "TRANSFER(BOZ literal) is not standard"_port_en_US);
+      }
+      return Fold(context, ConvertToType(*moldType, Expr<SomeType>{*boz}));
+    }
   }
   return std::nullopt;
 }

diff  --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index e802915945e26..4773e136c41cb 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1015,6 +1015,15 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
             {"mold", SameType, Rank::anyOrAssumedRank},
             {"size", AnyInt, Rank::scalar}},
         SameType, Rank::vector, IntrinsicClass::transformationalFunction},
+    // TRANSFER(BOZ, MOLD=integer or real scalar) extension
+    {"transfer",
+        {{"source", AnyNumeric, Rank::elementalOrBOZ},
+            {"mold", SameInt, Rank::scalar}},
+        SameInt, Rank::scalar, IntrinsicClass::transformationalFunction},
+    {"transfer",
+        {{"source", AnyNumeric, Rank::elementalOrBOZ},
+            {"mold", SameReal, Rank::scalar}},
+        SameReal, Rank::scalar, IntrinsicClass::transformationalFunction},
     {"transpose", {{"matrix", SameType, Rank::matrix}}, SameType, Rank::matrix,
         IntrinsicClass::transformationalFunction},
     {"trim", {{"string", SameCharNoLen, Rank::scalar}}, SameCharNoLen,

diff  --git a/flang/lib/Support/Fortran-features.cpp b/flang/lib/Support/Fortran-features.cpp
index fc69fc638eda1..df51b3c577125 100644
--- a/flang/lib/Support/Fortran-features.cpp
+++ b/flang/lib/Support/Fortran-features.cpp
@@ -103,6 +103,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
   warnLanguage_.set(LanguageFeature::ListDirectedSize);
   warnLanguage_.set(LanguageFeature::IgnoreIrrelevantAttributes);
   warnLanguage_.set(LanguageFeature::AmbiguousStructureConstructor);
+  warnLanguage_.set(LanguageFeature::TransferBOZ);
   warnUsage_.set(UsageWarning::ShortArrayActual);
   warnUsage_.set(UsageWarning::FoldingException);
   warnUsage_.set(UsageWarning::FoldingAvoidsRuntimeCrash);

diff  --git a/flang/test/Evaluate/transfer-boz.f90 b/flang/test/Evaluate/transfer-boz.f90
new file mode 100644
index 0000000000000..cb02ce45cb9ef
--- /dev/null
+++ b/flang/test/Evaluate/transfer-boz.f90
@@ -0,0 +1,19 @@
+!RUN: %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
+
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: portability: TRANSFER(BOZ literal) is not standard [-Wtransfer-boz]
+!CHECK: PRINT *, -17_1
+!CHECK: PRINT *, -16657_2
+!CHECK: PRINT *, -559038737_4
+!CHECK: PRINT *, -6.259853398707798016e18_4
+!CHECK: PRINT *, 1.8457939563190925445492984919045437485528002903297587499184927815557801644025850059853181797614625558939883606558067941052215743386077465406508281130260792904117042434297523782789810991060595088402505831985608687099349932384478542748214418349497866218005870896526832696740683567118136284392505567470522678176873737470272665943917715636680452977840047387616237048162886116765806372690690621665407240253312632381089695794498069551496558727944808752442655077077609749137334421734380446229972733723471196114034171513283476953482396960122117078210644171413215132537631249069125441199719426016512325803207900875869312966829207468856964517085119314095898691593840972164432952065765882286023532502465793032567186884340163939544586513274992967126308940351009368896484375e-314_8
+
+print *, transfer(z'deadbeef', 1_1)
+print *, transfer(z'deadbeef', 1_2)
+print *, transfer(z'deadbeef', 1_4)
+print *, transfer(z'deadbeef', 1._4)
+print *, transfer(z'deadbeef', 1._8)
+end


        

From flang-commits at lists.llvm.org  Mon Jul 14 11:12:57 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 14 Jul 2025 11:12:57 -0700 (PDT)
Subject: [flang-commits]  [flang] [flang] Extension: TRANSFER(boz,
	integer or real scalar) (PR #147604)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147604@github.com>
Message-ID: <687548a9.050a0220.1b247f.c122@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/147604

From flang-commits at lists.llvm.org  Mon Jul 14 11:32:57 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Mon, 14 Jul 2025 11:32:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Optimize redundant array repacking.
	(PR #147881)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/147881@github.com>
Message-ID: <68754d59.050a0220.268424.d7d8@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `premerge-monolithic-linux` running on `premerge-linux-1` while building `flang` at step 7 "test-build-unified-tree-check-all".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/153/builds/38009

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 7 (test-build-unified-tree-check-all) failure: test (failure)
...
PASS: lld :: COFF/arm64ec-lib.test (98544 of 101555)
PASS: lld :: COFF/arm64x-import.test (98545 of 101555)
PASS: lld :: COFF/comdat-gcc-compatibility-size.test (98546 of 101555)
PASS: lld :: COFF/directives.s (98547 of 101555)
PASS: lld :: COFF/arm64x-symtab.s (98548 of 101555)
PASS: lld :: COFF/combined-resources.test (98549 of 101555)
PASS: lld :: COFF/debug-reloc.s (98550 of 101555)
PASS: lld :: COFF/duplicate.test (98551 of 101555)
PASS: lld :: COFF/delayimports-error.test (98552 of 101555)
TIMEOUT: MLIR :: Examples/standalone/test.toy (98553 of 101555)
******************** TEST 'MLIR :: Examples/standalone/test.toy' FAILED ********************
Exit Code: 1
Timeout: Reached timeout of 60 seconds

Command Output (stdout):
--
# RUN: at line 1
"/etc/cmake/bin/cmake" "/build/buildbot/premerge-monolithic-linux/llvm-project/mlir/examples/standalone" -G "Ninja"  -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_C_COMPILER=/usr/bin/clang  -DLLVM_ENABLE_LIBCXX=OFF -DMLIR_DIR=/build/buildbot/premerge-monolithic-linux/build/lib/cmake/mlir  -DLLVM_USE_LINKER=lld  -DPython3_EXECUTABLE="/usr/bin/python3.10"
# executed command: /etc/cmake/bin/cmake /build/buildbot/premerge-monolithic-linux/llvm-project/mlir/examples/standalone -G Ninja -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_C_COMPILER=/usr/bin/clang -DLLVM_ENABLE_LIBCXX=OFF -DMLIR_DIR=/build/buildbot/premerge-monolithic-linux/build/lib/cmake/mlir -DLLVM_USE_LINKER=lld -DPython3_EXECUTABLE=/usr/bin/python3.10
# .---command stdout------------
# | -- The CXX compiler identification is Clang 16.0.6
# | -- The C compiler identification is Clang 16.0.6
# | -- Detecting CXX compiler ABI info
# | -- Detecting CXX compiler ABI info - done
# | -- Check for working CXX compiler: /usr/bin/clang++ - skipped
# | -- Detecting CXX compile features
# | -- Detecting CXX compile features - done
# | -- Detecting C compiler ABI info
# | -- Detecting C compiler ABI info - done
# | -- Check for working C compiler: /usr/bin/clang - skipped
# | -- Detecting C compile features
# | -- Detecting C compile features - done
# | -- Looking for histedit.h
# | -- Looking for histedit.h - found
# | -- Found LibEdit: /usr/include (found version "2.11") 
# | -- Found ZLIB: /usr/lib/x86_64-linux-gnu/libz.so (found version "1.2.11") 
# | -- Found LibXml2: /usr/lib/x86_64-linux-gnu/libxml2.so (found version "2.9.13") 
# | -- Using MLIRConfig.cmake in: /build/buildbot/premerge-monolithic-linux/build/lib/cmake/mlir
# | -- Using LLVMConfig.cmake in: /build/buildbot/premerge-monolithic-linux/build/lib/cmake/llvm
# | -- Linker detection: unknown
# | -- Performing Test LLVM_LIBSTDCXX_MIN
# | -- Performing Test LLVM_LIBSTDCXX_MIN - Success
# | -- Performing Test LLVM_LIBSTDCXX_SOFT_ERROR
# | -- Performing Test LLVM_LIBSTDCXX_SOFT_ERROR - Success
# | -- Performing Test CXX_SUPPORTS_CUSTOM_LINKER
# | -- Performing Test CXX_SUPPORTS_CUSTOM_LINKER - Success
# | -- Performing Test C_SUPPORTS_FPIC
# | -- Performing Test C_SUPPORTS_FPIC - Success
# | -- Performing Test CXX_SUPPORTS_FPIC

```

</details>

https://github.com/llvm/llvm-project/pull/147881

From flang-commits at lists.llvm.org  Mon Jul 14 12:14:42 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Mon, 14 Jul 2025 12:14:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix crash in Semantics (PR #148706)
Message-ID: <llvm.org/llvm/llvm-project/pull/148706@github.com>


https://github.com/klausler created https://github.com/llvm/llvm-project/pull/148706

Allow for renaming in USE association of Cray pointers.

Fixes https://github.com/llvm/llvm-project/issues/148559.

>From c5ee13f2cf13778dab5450a75bc59657a4698a49 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Mon, 14 Jul 2025 12:12:20 -0700
Subject: [PATCH] [flang] Fix crash in Semantics

Allow for renaming in USE association of Cray pointers.

Fixes https://github.com/llvm/llvm-project/issues/148559.
---
 flang/lib/Semantics/tools.cpp      |  6 +++---
 flang/test/Semantics/bug148559.f90 | 12 ++++++++++++
 2 files changed, 15 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Semantics/bug148559.f90

diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 5e5b43f26c791..5a5b02e1ac3ce 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -348,9 +348,9 @@ const Symbol &BypassGeneric(const Symbol &symbol) {
 
 const Symbol &GetCrayPointer(const Symbol &crayPointee) {
   const Symbol *found{nullptr};
-  for (const auto &[pointee, pointer] :
-      crayPointee.GetUltimate().owner().crayPointers()) {
-    if (pointee == crayPointee.name()) {
+  const Symbol &ultimate{crayPointee.GetUltimate()};
+  for (const auto &[pointee, pointer] : ultimate.owner().crayPointers()) {
+    if (pointee == ultimate.name()) {
       found = &pointer.get();
       break;
     }
diff --git a/flang/test/Semantics/bug148559.f90 b/flang/test/Semantics/bug148559.f90
new file mode 100644
index 0000000000000..d7b959ac8f191
--- /dev/null
+++ b/flang/test/Semantics/bug148559.f90
@@ -0,0 +1,12 @@
+!RUN: %flang_fc1 -fsyntax-only %s
+!Regression test for crash in semantics on Cray pointers
+
+module m
+  pointer(ptr,pp)
+end module m
+
+program main
+  use m, only:renamea=>pp
+  use m, only:pp
+  print *, renamea
+end


From flang-commits at lists.llvm.org  Mon Jul 14 12:15:16 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 12:15:16 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix crash in Semantics (PR #148706)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148706@github.com>
Message-ID: <68755744.050a0220.34f215.b8aa@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Peter Klausler (klausler)

<details>
<summary>Changes</summary>

Allow for renaming in USE association of Cray pointers.

Fixes https://github.com/llvm/llvm-project/issues/148559.

---
Full diff: https://github.com/llvm/llvm-project/pull/148706.diff


2 Files Affected:

- (modified) flang/lib/Semantics/tools.cpp (+3-3) 
- (added) flang/test/Semantics/bug148559.f90 (+12) 


``````````diff
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 5e5b43f26c791..5a5b02e1ac3ce 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -348,9 +348,9 @@ const Symbol &BypassGeneric(const Symbol &symbol) {
 
 const Symbol &GetCrayPointer(const Symbol &crayPointee) {
   const Symbol *found{nullptr};
-  for (const auto &[pointee, pointer] :
-      crayPointee.GetUltimate().owner().crayPointers()) {
-    if (pointee == crayPointee.name()) {
+  const Symbol &ultimate{crayPointee.GetUltimate()};
+  for (const auto &[pointee, pointer] : ultimate.owner().crayPointers()) {
+    if (pointee == ultimate.name()) {
       found = &pointer.get();
       break;
     }
diff --git a/flang/test/Semantics/bug148559.f90 b/flang/test/Semantics/bug148559.f90
new file mode 100644
index 0000000000000..d7b959ac8f191
--- /dev/null
+++ b/flang/test/Semantics/bug148559.f90
@@ -0,0 +1,12 @@
+!RUN: %flang_fc1 -fsyntax-only %s
+!Regression test for crash in semantics on Cray pointers
+
+module m
+  pointer(ptr,pp)
+end module m
+
+program main
+  use m, only:renamea=>pp
+  use m, only:pp
+  print *, renamea
+end

``````````

</details>


https://github.com/llvm/llvm-project/pull/148706

From flang-commits at lists.llvm.org  Mon Jul 14 12:48:54 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Mon, 14 Jul 2025 12:48:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][acc] Implement MappableType's
 generatePrivateInit (PR #148302)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148302@github.com>
Message-ID: <68755f26.050a0220.22ff10.d3dd@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `premerge-monolithic-linux` running on `premerge-linux-1` while building `flang` at step 7 "test-build-unified-tree-check-all".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/153/builds/38017

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 7 (test-build-unified-tree-check-all) failure: test (failure)
...
PASS: lld :: COFF/export-armnt.yaml (98592 of 101557)
PASS: lld :: COFF/entry-inference3.test (98593 of 101557)
PASS: lld :: COFF/entry-inference-mingw.s (98594 of 101557)
PASS: lld :: COFF/filename-casing.s (98595 of 101557)
PASS: lld :: COFF/export-all-lto.ll (98596 of 101557)
PASS: lld :: COFF/entry-inference332.test (98597 of 101557)
PASS: lld :: COFF/exclude-all.s (98598 of 101557)
PASS: lld :: COFF/driver-opt.s (98599 of 101557)
PASS: lit :: shtest-external-shell-kill.py (98600 of 101557)
TIMEOUT: MLIR :: Examples/standalone/test.toy (98601 of 101557)
******************** TEST 'MLIR :: Examples/standalone/test.toy' FAILED ********************
Exit Code: 1
Timeout: Reached timeout of 60 seconds

Command Output (stdout):
--
# RUN: at line 1
"/etc/cmake/bin/cmake" "/build/buildbot/premerge-monolithic-linux/llvm-project/mlir/examples/standalone" -G "Ninja"  -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_C_COMPILER=/usr/bin/clang  -DLLVM_ENABLE_LIBCXX=OFF -DMLIR_DIR=/build/buildbot/premerge-monolithic-linux/build/lib/cmake/mlir  -DLLVM_USE_LINKER=lld  -DPython3_EXECUTABLE="/usr/bin/python3.10"
# executed command: /etc/cmake/bin/cmake /build/buildbot/premerge-monolithic-linux/llvm-project/mlir/examples/standalone -G Ninja -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_C_COMPILER=/usr/bin/clang -DLLVM_ENABLE_LIBCXX=OFF -DMLIR_DIR=/build/buildbot/premerge-monolithic-linux/build/lib/cmake/mlir -DLLVM_USE_LINKER=lld -DPython3_EXECUTABLE=/usr/bin/python3.10
# .---command stdout------------
# | -- The CXX compiler identification is Clang 16.0.6
# | -- The C compiler identification is Clang 16.0.6
# | -- Detecting CXX compiler ABI info
# | -- Detecting CXX compiler ABI info - done
# | -- Check for working CXX compiler: /usr/bin/clang++ - skipped
# | -- Detecting CXX compile features
# | -- Detecting CXX compile features - done
# | -- Detecting C compiler ABI info
# | -- Detecting C compiler ABI info - done
# | -- Check for working C compiler: /usr/bin/clang - skipped
# | -- Detecting C compile features
# | -- Detecting C compile features - done
# | -- Looking for histedit.h
# | -- Looking for histedit.h - found
# | -- Found LibEdit: /usr/include (found version "2.11") 
# | -- Found ZLIB: /usr/lib/x86_64-linux-gnu/libz.so (found version "1.2.11") 
# | -- Found LibXml2: /usr/lib/x86_64-linux-gnu/libxml2.so (found version "2.9.13") 
# | -- Using MLIRConfig.cmake in: /build/buildbot/premerge-monolithic-linux/build/lib/cmake/mlir
# | -- Using LLVMConfig.cmake in: /build/buildbot/premerge-monolithic-linux/build/lib/cmake/llvm
# | -- Linker detection: unknown
# | -- Performing Test LLVM_LIBSTDCXX_MIN
# | -- Performing Test LLVM_LIBSTDCXX_MIN - Success
# | -- Performing Test LLVM_LIBSTDCXX_SOFT_ERROR
# | -- Performing Test LLVM_LIBSTDCXX_SOFT_ERROR - Success
# | -- Performing Test CXX_SUPPORTS_CUSTOM_LINKER
# | -- Performing Test CXX_SUPPORTS_CUSTOM_LINKER - Success
# | -- Performing Test C_SUPPORTS_FPIC
# | -- Performing Test C_SUPPORTS_FPIC - Success
# | -- Performing Test CXX_SUPPORTS_FPIC

```

</details>

https://github.com/llvm/llvm-project/pull/148302

From flang-commits at lists.llvm.org  Mon Jul 14 11:04:32 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 11:04:32 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [TargetVerifier][AMDGPU] Add
 TargetVerifier. (PR #123609)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/123609@github.com>
Message-ID: <687546b0.170a0220.25e232.b89c@mx.google.com>


https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/123609

>From d9b0a5df811451d4d4a2e37fc4933c8071022317 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 20 Jan 2025 04:51:26 -0800
Subject: [PATCH 01/33] [TargetVerifier][AMDGPU] Add TargetVerifier.

This pass verifies the IR for an individual backend. This is
different than Lint because it consolidates all checks for a
given backend in a single pass. A check for Lint may be
undefined behavior across all targets, whereas a check in
TargetVerifier would only pertain to the specified target
but can check more than just undefined behavior such are IR
validity. A use case of this would be to reject programs
with invalid IR while fuzzing.
---
 llvm/include/llvm/IR/Module.h                 |   9 +
 llvm/include/llvm/Target/TargetVerifier.h     |  82 +++++++
 .../TargetVerify/AMDGPUTargetVerifier.h       |  36 +++
 llvm/lib/IR/Verifier.cpp                      |  18 +-
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 213 ++++++++++++++++++
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        |  62 +++++
 llvm/tools/llvm-tgt-verify/CMakeLists.txt     |  34 +++
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 172 ++++++++++++++
 9 files changed, 623 insertions(+), 4 deletions(-)
 create mode 100644 llvm/include/llvm/Target/TargetVerifier.h
 create mode 100644 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify.ll
 create mode 100644 llvm/tools/llvm-tgt-verify/CMakeLists.txt
 create mode 100644 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp

diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index a99937a90cbb7..3037b2c1644d9 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,6 +215,15 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
+  /// Is this Module valid as determined by one of the verification passes
+  /// i.e. Lint, Verifier, TargetVerifier.
+  bool IsValid = true;
+
+  /// Is this Module using intrinsics to record the position of debugging
+  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
+  /// \ref BasicBlock.
+  bool IsNewDbgInfoFormat;
+
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
new file mode 100644
index 0000000000000..e00c6a7b260c9
--- /dev/null
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -0,0 +1,82 @@
+//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at TargetVerifier.cpp or an
+// individual backend's TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_VERIFIER_H
+#define LLVM_TARGET_VERIFIER_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+
+class Function;
+
+class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {}
+};
+
+class TargetVerify {
+protected:
+  void WriteValues(ArrayRef<const Value *> Vs) {
+    for (const Value *V : Vs) {
+      if (!V)
+        continue;
+      if (isa<Instruction>(V)) {
+        MessagesStr << *V << '\n';
+      } else {
+        V->printAsOperand(MessagesStr, true, Mod);
+        MessagesStr << '\n';
+      }
+    }
+  }
+
+  /// A check failed, so printout out the condition and the message.
+  ///
+  /// This provides a nice place to put a breakpoint if you want to see why
+  /// something is not correct.
+  void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
+
+  /// A check failed (with values to print).
+  ///
+  /// This calls the Message-only version so that the above is easier to set
+  /// a breakpoint on.
+  template <typename T1, typename... Ts>
+  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) {
+    CheckFailed(Message);
+    WriteValues({V1, Vs...});
+  }
+public:
+  Module *Mod;
+  Triple TT;
+
+  std::string Messages;
+  raw_string_ostream MessagesStr;
+
+  TargetVerify(Module *Mod)
+      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
+        MessagesStr(Messages) {}
+
+  void run(Function &F) {};
+};
+
+} // namespace llvm
+
+#endif // LLVM_TARGET_VERIFIER_H
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
new file mode 100644
index 0000000000000..e6ff57629b141
--- /dev/null
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -0,0 +1,36 @@
+//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU ---*- C++ -*-===//
+////
+//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//// See https://llvm.org/LICENSE.txt for license information.
+//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+////
+////===----------------------------------------------------------------------===//
+////
+//// This file defines target verifier interfaces that can be used for some
+//// validation of input to the system, and for checking that transformations
+//// haven't done something bad. In contrast to the Verifier or Lint, the
+//// TargetVerifier looks for constructions invalid to a particular target
+//// machine.
+////
+//// To see what specifically is checked, look at an individual backend's
+//// TargetVerifier.
+////
+////===----------------------------------------------------------------------===//
+
+#ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
+#define LLVM_AMDGPU_TARGET_VERIFIER_H
+
+#include "llvm/Target/TargetVerifier.h"
+
+namespace llvm {
+
+class Function;
+
+class AMDGPUTargetVerifierPass : public TargetVerifierPass {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1f1041b259736..18e9760f9c617 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -135,6 +135,10 @@ static cl::opt<bool> VerifyNoAliasScopeDomination(
     cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
              "scopes are not dominating"));
 
+static cl::opt<bool>
+    VerifyAbortOnError("verifier-abort-on-error", cl::init(false),
+                       cl::desc("In the Verifier pass, abort on errors."));
+
 namespace llvm {
 
 struct VerifierSupport {
@@ -7865,16 +7869,22 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
-  if (FatalErrors && (Res.IRBroken || Res.DebugInfoBroken))
-    report_fatal_error("Broken module found, compilation aborted!");
+  if (Res.IRBroken || Res.DebugInfoBroken) {
+    M.IsValid = false;
+    if (VerifyAbortOnError && FatalErrors)
+      report_fatal_error("Broken module found, compilation aborted!");
+  }
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
-  if (res.IRBroken && FatalErrors)
-    report_fatal_error("Broken function found, compilation aborted!");
+  if (res.IRBroken) {
+    F.getParent()->IsValid = false;
+    if (VerifyAbortOnError && FatalErrors)
+      report_fatal_error("Broken function found, compilation aborted!");
+  }
 
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
new file mode 100644
index 0000000000000..585b19065c142
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -0,0 +1,213 @@
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<bool>
+MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
+
+// Check - We know that cond should be true, if not print an error message.
+#define Check(C, ...)                                                          \
+  do {                                                                         \
+    if (!(C)) {                                                                \
+      TargetVerify::CheckFailed(__VA_ARGS__);                                  \
+      return;                                                                  \
+    }                                                                          \
+  } while (false)
+
+static bool isMFMA(unsigned IID) {
+  switch (IID) {
+    case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
+    case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
+    case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
+    case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
+    case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
+    case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
+    case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
+    case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
+    case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
+    case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
+    case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
+
+    case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
+    case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
+
+    case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
+    case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
+
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
+
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
+      return true;
+    default:
+      return false;
+  }
+}
+
+namespace llvm {
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  Module *Mod;
+
+  DominatorTree *DT;
+  PostDominatorTree *PDT;
+  UniformityInfo *UA;
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  void run(Function &F);
+};
+
+static bool IsValidInt(const Type *Ty) {
+  return Ty->isIntegerTy(1) ||
+         Ty->isIntegerTy(8) ||
+         Ty->isIntegerTy(16) ||
+         Ty->isIntegerTy(32) ||
+         Ty->isIntegerTy(64) ||
+         Ty->isIntegerTy(128);
+}
+
+static bool isShader(CallingConv::ID CC) {
+  switch(CC) {
+    case CallingConv::AMDGPU_VS:
+    case CallingConv::AMDGPU_LS:
+    case CallingConv::AMDGPU_HS:
+    case CallingConv::AMDGPU_ES:
+    case CallingConv::AMDGPU_GS:
+    case CallingConv::AMDGPU_PS:
+    case CallingConv::AMDGPU_CS_Chain:
+    case CallingConv::AMDGPU_CS_ChainPreserve:
+    case CallingConv::AMDGPU_CS:
+      return true;
+    default:
+      return false;
+  }
+}
+
+void AMDGPUTargetVerify::run(Function &F) {
+  // Ensure shader calling convention returns void
+  if (isShader(F.getCallingConv()))
+    Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
+
+  for (auto &BB : F) {
+
+    for (auto &I : BB) {
+      if (MarkUniform)
+        outs() << UA->isUniform(&I) << ' ' << I << '\n';
+
+      // Ensure integral types are valid: i8, i16, i32, i64, i128
+      if (I.getType()->isIntegerTy())
+        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
+      for (unsigned i = 0; i < I.getNumOperands(); ++i)
+        if (I.getOperand(i)->getType()->isIntegerTy())
+          Check(IsValidInt(I.getOperand(i)->getType()),
+                "Int type is invalid.", I.getOperand(i));
+
+      // Ensure no store to const memory
+      if (auto *SI = dyn_cast<StoreInst>(&I))
+      {
+        unsigned AS = SI->getPointerAddressSpace();
+        Check(AS != 4, "Write to const memory", SI);
+      }
+
+      // Ensure no kernel to kernel calls.
+      if (auto *CI = dyn_cast<CallInst>(&I))
+      {
+        CallingConv::ID CalleeCC = CI->getCallingConv();
+        if (CalleeCC == CallingConv::AMDGPU_KERNEL)
+        {
+          CallingConv::ID CallerCC = CI->getParent()->getParent()->getCallingConv();
+          Check(CallerCC != CallingConv::AMDGPU_KERNEL,
+            "A kernel may not call a kernel", CI->getParent()->getParent());
+        }
+      }
+
+      // Ensure MFMA is not in control flow with diverging operands
+      if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+        if (isMFMA(II->getIntrinsicID())) {
+          bool InControlFlow = false;
+          for (const auto &P : predecessors(&BB))
+            if (!PDT->dominates(&BB, P)) {
+              InControlFlow = true;
+              break;
+            }
+          for (const auto &S : successors(&BB))
+            if (!DT->dominates(&BB, S)) {
+              InControlFlow = true;
+              break;
+            }
+          if (InControlFlow) {
+            // If operands to MFMA are not uniform, MFMA cannot be in control flow
+            bool hasUniformOperands = true;
+            for (unsigned i = 0; i < II->getNumOperands(); i++) {
+              if (!UA->isUniform(II->getOperand(i))) {
+                dbgs() << "Not uniform: " << *II->getOperand(i) << '\n';
+                hasUniformOperands = false;
+              }
+            }
+            if (!hasUniformOperands) Check(false, "MFMA in control flow", II);
+            //else Check(false, "MFMA in control flow (uniform operands)", II);
+          }
+          //else Check(false, "MFMA not in control flow", II);
+        }
+      }
+    }
+  }
+}
+
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+
+  auto *Mod = F.getParent();
+
+  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  TV.run(F);
+
+  dbgs() << TV.MessagesStr.str();
+  if (!TV.MessagesStr.str().empty()) {
+    F.getParent()->IsValid = false;
+  }
+
+  return PreservedAnalyses::all();
+}
+} // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index c6d70ee39202e..a47b5f2732f91 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -111,6 +111,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUTargetMachine.cpp
   AMDGPUTargetObjectFile.cpp
   AMDGPUTargetTransformInfo.cpp
+  AMDGPUTargetVerifier.cpp
   AMDGPUWaitSGPRHazards.cpp
   AMDGPUUnifyDivergentExitNodes.cpp
   AMDGPUUnifyMetadata.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
new file mode 100644
index 0000000000000..f56ff992a56c2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -0,0 +1,62 @@
+; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
+
+define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg) #0 {
+; CHECK: Not uniform: %in.f32 = load <32 x float>, ptr addrspace(1) %gep, align 128
+; CHECK-NEXT: MFMA in control flow
+; CHECK-NEXT:   %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
+s:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
+  %in.i32 = load <32 x i32>, ptr addrspace(1) %gep
+  %in.f32 = load <32 x float>, ptr addrspace(1) %gep
+
+  %0 = icmp eq <32 x i32> %in.i32, zeroinitializer
+  %div.br = extractelement <32 x i1> %0, i32 0
+  br i1 %div.br, label %if.3, label %else.0
+
+if.3:
+  br label %join
+
+else.0:
+  %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
+  br label %join
+
+join:
+  ret void
+}
+
+define amdgpu_cs i32 @shader() {
+; CHECK: Shaders must return void
+  ret i32 0
+}
+
+define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
+; CHECK: Undefined behavior: Write to memory in const addrspace
+; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
+; CHECK-NEXT: Write to const memory
+; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
+  %r = add i32 %a, %b
+  store i32 %r, ptr addrspace(4) %out
+  ret void
+}
+
+define amdgpu_kernel void @kernel_callee(ptr %x) {
+  ret void
+}
+
+define amdgpu_kernel void @kernel_caller(ptr %x) {
+; CHECK: A kernel may not call a kernel
+; CHECK-NEXT: ptr @kernel_caller
+  call amdgpu_kernel void @kernel_callee(ptr %x)
+  ret void
+}
+
+
+; Function Attrs: nounwind
+define i65 @invalid_type(i65 %x) #0 {
+; CHECK: Int type is invalid.
+; CHECK-NEXT: %tmp2 = ashr i65 %x, 64
+entry:
+  %tmp2 = ashr i65 %x, 64
+  ret i65 %tmp2
+}
diff --git a/llvm/tools/llvm-tgt-verify/CMakeLists.txt b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
new file mode 100644
index 0000000000000..fe47c85e6cdce
--- /dev/null
+++ b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
@@ -0,0 +1,34 @@
+set(LLVM_LINK_COMPONENTS
+  AllTargetsAsmParsers
+  AllTargetsCodeGens
+  AllTargetsDescs
+  AllTargetsInfos
+  Analysis
+  AsmPrinter
+  CodeGen
+  CodeGenTypes
+  Core
+  IRPrinter
+  IRReader
+  MC
+  MIRParser
+  Passes
+  Remarks
+  ScalarOpts
+  SelectionDAG
+  Support
+  Target
+  TargetParser
+  TransformUtils
+  Vectorize
+  )
+
+add_llvm_tool(llvm-tgt-verify
+  llvm-tgt-verify.cpp
+
+  DEPENDS
+  intrinsics_gen
+  SUPPORT_PLUGINS
+  )
+
+export_executable_symbols_for_plugins(llc)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
new file mode 100644
index 0000000000000..68422abd6f4cc
--- /dev/null
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -0,0 +1,172 @@
+//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tool to fuzz instruction selection using libFuzzer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/InitializePasses.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/Lint.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/CodeGen/CommandFlags.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetVerifier.h"
+
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#define DEBUG_TYPE "isel-fuzzer"
+
+using namespace llvm;
+
+static codegen::RegisterCodeGenFlags CGF;
+
+static cl::opt<std::string>
+InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
+
+static cl::opt<bool>
+    StacktraceAbort("stacktrace-abort",
+        cl::desc("Turn on stacktrace"), cl::init(false));
+
+static cl::opt<bool>
+    NoLint("no-lint",
+        cl::desc("Turn off Lint"), cl::init(false));
+
+static cl::opt<bool>
+    NoVerify("no-verifier",
+        cl::desc("Turn off Verifier"), cl::init(false));
+
+static cl::opt<char>
+    OptLevel("O",
+             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
+                      "(default = '-O2')"),
+             cl::Prefix, cl::init('2'));
+
+static cl::opt<std::string>
+    TargetTriple("mtriple", cl::desc("Override target triple for module"));
+
+static std::unique_ptr<TargetMachine> TM;
+
+static void handleLLVMFatalError(void *, const char *Message, bool) {
+  if (StacktraceAbort) {
+    dbgs() << "LLVM ERROR: " << Message << "\n"
+           << "Aborting.\n";
+    abort();
+  }
+}
+
+int main(int argc, char **argv) {
+  StringRef ExecName = argv[0];
+  InitLLVM X(argc, argv);
+
+  InitializeAllTargets();
+  InitializeAllTargetMCs();
+  InitializeAllAsmPrinters();
+  InitializeAllAsmParsers();
+
+  PassRegistry *Registry = PassRegistry::getPassRegistry();
+  initializeCore(*Registry);
+  initializeCodeGen(*Registry);
+  initializeAnalysis(*Registry);
+  initializeTarget(*Registry);
+
+  cl::ParseCommandLineOptions(argc, argv);
+
+  if (TargetTriple.empty()) {
+    errs() << ExecName << ": -mtriple must be specified\n";
+    exit(1);
+  }
+
+  CodeGenOptLevel OLvl;
+  if (auto Level = CodeGenOpt::parseLevel(OptLevel)) {
+    OLvl = *Level;
+  } else {
+    errs() << ExecName << ": invalid optimization level.\n";
+    return 1;
+  }
+  ExitOnError ExitOnErr(std::string(ExecName) + ": error:");
+  TM = ExitOnErr(codegen::createTargetMachineForTriple(
+      Triple::normalize(TargetTriple), OLvl));
+  assert(TM && "Could not allocate target machine!");
+
+  // Make sure we print the summary and the current unit when LLVM errors out.
+  install_fatal_error_handler(handleLLVMFatalError, nullptr);
+
+  LLVMContext Context;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
+  if (!M) {
+    errs() << "Invalid mod\n";
+    return 1;
+  }
+  auto S = Triple::normalize(TargetTriple);
+  M->setTargetTriple(S);
+
+  PassInstrumentationCallbacks PIC;
+  StandardInstrumentations SI(Context, false/*debug PM*/,
+                              false);
+  registerCodeGenCallback(PIC, *TM);
+
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+  //TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
+
+  MachineFunctionAnalysisManager MFAM;
+  LoopAnalysisManager LAM;
+  FunctionAnalysisManager FAM;
+  CGSCCAnalysisManager CGAM;
+  ModuleAnalysisManager MAM;
+  PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.registerMachineFunctionAnalyses(MFAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
+
+  SI.registerCallbacks(PIC, &MAM);
+
+  //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
+
+  Triple TT(M->getTargetTriple());
+  if (!NoLint)
+    FPM.addPass(LintPass());
+  if (!NoVerify)
+    MPM.addPass(VerifierPass());
+  if (TT.isAMDGPU())
+    FPM.addPass(AMDGPUTargetVerifierPass());
+  else if (false) {} // ...
+  else
+    FPM.addPass(TargetVerifierPass());
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
+  MPM.run(*M, MAM);
+
+  if (!M->IsValid)
+    return 1;
+
+  return 0;
+}

>From 3b69ade6d8880cadc883698b64a06bd9207d084b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 3 Feb 2025 07:15:12 -0800
Subject: [PATCH 02/33] Add hook for target verifier in llc,opt

---
 .../llvm/Passes/StandardInstrumentations.h    |  6 ++++--
 llvm/include/llvm/Target/TargetVerifier.h     |  1 +
 .../TargetVerify/AMDGPUTargetVerifier.h       | 18 ++++++++++++++++++
 llvm/lib/LTO/LTOBackend.cpp                   |  2 +-
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp         |  2 +-
 llvm/lib/Passes/CMakeLists.txt                |  1 +
 llvm/lib/Passes/PassBuilderBindings.cpp       |  2 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  | 19 +++++++++++++++----
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 12 ++++++------
 llvm/lib/Target/CMakeLists.txt                |  2 ++
 .../CodeGen/AMDGPU/tgt-verify-llc-fail.ll     |  6 ++++++
 .../CodeGen/AMDGPU/tgt-verify-llc-pass.ll     |  6 ++++++
 llvm/tools/llc/NewPMDriver.cpp                |  2 +-
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 +-
 llvm/tools/opt/NewPMDriver.cpp                |  2 +-
 llvm/unittests/IR/PassManagerTest.cpp         |  6 +++---
 16 files changed, 68 insertions(+), 21 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll

diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 4ee5ab2554868..7d7b858b447cc 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,7 +478,8 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM);
+                         ModuleAnalysisManager *MAM,
+			 FunctionAnalysisManager *FAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -624,7 +625,8 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM = nullptr);
+                         ModuleAnalysisManager *MAM,
+			 FunctionAnalysisManager *FAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index e00c6a7b260c9..ad5aeb895953d 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -75,6 +75,7 @@ class TargetVerify {
         MessagesStr(Messages) {}
 
   void run(Function &F) {};
+  void run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index e6ff57629b141..d8a3fda4f87dc 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -22,6 +22,10 @@
 
 #include "llvm/Target/TargetVerifier.h"
 
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/IR/Dominators.h"
+
 namespace llvm {
 
 class Function;
@@ -31,6 +35,20 @@ class AMDGPUTargetVerifierPass : public TargetVerifierPass {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  Module *Mod;
+
+  DominatorTree *DT;
+  PostDominatorTree *PDT;
+  UniformityInfo *UA;
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  void run(Function &F);
+};
+
 } // namespace llvm
 
 #endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 5e8cd12fe040b..7ae23ef4e7df4 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -273,7 +273,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
                               Conf.VerifyEach);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 
   RegisterPassPlugins(Conf.PassPlugins, PB);
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 5b333cda31fb8..8db92e70d5a96 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -241,7 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PipelineTuningOptions PTO;
   PTO.LoopVectorization = true;
   PTO.SLPVectorization = true;
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 91c8c4f67074d..e5912f23903b6 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,6 +30,7 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
+  TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index 933fe89e53a94..f0e1abb8cebc4 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -76,7 +76,7 @@ static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   // Run the pipeline.
   if (Fun) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 4e8abc43572d7..e2764b162fddb 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,6 +40,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
+#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -1449,9 +1450,10 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 }
 
 void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                              ModuleAnalysisManager *MAM) {
+                                              ModuleAnalysisManager *MAM,
+					      FunctionAnalysisManager *FAM) {
   PIC.registerAfterPassCallback(
-      [this, MAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+      [this, MAM, FAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
         const auto *F = unwrapIR<Function>(IR);
@@ -1468,6 +1470,15 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
             report_fatal_error(formatv("Broken function found after pass "
                                        "\"{0}\", compilation aborted!",
                                        P));
+
+          if (FAM) {
+            TargetVerify TV(const_cast<Module*>(F->getParent()));
+            TV.run(*const_cast<Function*>(F), *FAM);
+	    if (!F->getParent()->IsValid)
+              report_fatal_error(formatv("Broken function found after pass "
+                                         "\"{0}\", compilation aborted!",
+                                         P));
+          }
         } else {
           const auto *M = unwrapIR<Module>(IR);
           if (!M) {
@@ -2507,7 +2518,7 @@ void PrintCrashIRInstrumentation::registerCallbacks(
 }
 
 void StandardInstrumentations::registerCallbacks(
-    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM) {
+    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM, FunctionAnalysisManager *FAM) {
   PrintIR.registerCallbacks(PIC);
   PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
@@ -2516,7 +2527,7 @@ void StandardInstrumentations::registerCallbacks(
   PrintChangedIR.registerCallbacks(PIC);
   PseudoProbeVerification.registerCallbacks(PIC);
   if (VerifyEach)
-    Verify.registerCallbacks(PIC, MAM);
+    Verify.registerCallbacks(PIC, MAM, FAM);
   PrintChangedDiff.registerCallbacks(PIC);
   WebsiteChangeReporter.registerCallbacks(PIC);
   ChangeTester.registerCallbacks(PIC);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 585b19065c142..e6cdec7160229 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -14,8 +14,8 @@
 
 using namespace llvm;
 
-static cl::opt<bool>
-MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
+//static cl::opt<bool>
+//MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
 
 // Check - We know that cond should be true, if not print an error message.
 #define Check(C, ...)                                                          \
@@ -81,7 +81,7 @@ static bool isMFMA(unsigned IID) {
 }
 
 namespace llvm {
-class AMDGPUTargetVerify : public TargetVerify {
+/*class AMDGPUTargetVerify : public TargetVerify {
 public:
   Module *Mod;
 
@@ -93,7 +93,7 @@ class AMDGPUTargetVerify : public TargetVerify {
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   void run(Function &F);
-};
+};*/
 
 static bool IsValidInt(const Type *Ty) {
   return Ty->isIntegerTy(1) ||
@@ -129,8 +129,8 @@ void AMDGPUTargetVerify::run(Function &F) {
   for (auto &BB : F) {
 
     for (auto &I : BB) {
-      if (MarkUniform)
-        outs() << UA->isUniform(&I) << ' ' << I << '\n';
+      //if (MarkUniform)
+        //outs() << UA->isUniform(&I) << ' ' << I << '\n';
 
       // Ensure integral types are valid: i8, i16, i32, i64, i128
       if (I.getType()->isIntegerTy())
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index 9472288229cac..f2a5d545ce84f 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -7,6 +7,8 @@ add_llvm_component_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
+  TargetVerifier.cpp
+  AMDGPU/AMDGPUTargetVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Target
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
new file mode 100644
index 0000000000000..584097d7bc134
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -0,0 +1,6 @@
+; RUN: not not llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each -o - < %s 2>&1 | FileCheck %s
+
+define amdgpu_cs i32 @nonvoid_shader() {
+; CHECK: LLVM ERROR
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
new file mode 100644
index 0000000000000..0c3a5fe5ac4a5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -0,0 +1,6 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each %s -o - 2>&1 | FileCheck %s
+
+define amdgpu_cs void @void_shader() {
+; CHECK: ModuleToFunctionPassAdaptor
+  ret void
+}
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index fa82689ecf9ae..a060d16e74958 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -126,7 +126,7 @@ int llvm::compileModuleWithNewPM(
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 68422abd6f4cc..3352d07deff2f 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -147,7 +147,7 @@ int main(int argc, char **argv) {
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
 
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index 7d168a6ceb17c..a8977d80bdf44 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -423,7 +423,7 @@ bool llvm::runPassPipeline(
   PrintPassOpts.SkipAnalyses = DebugPM == DebugLogging::Quiet;
   StandardInstrumentations SI(M.getContext(), DebugPM != DebugLogging::None,
                               VK == VerifierKind::EachPass, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   DebugifyEachInstrumentation Debugify;
   DebugifyStatsMap DIStatsMap;
   DebugInfoPerPass DebugInfoBeforePass;
diff --git a/llvm/unittests/IR/PassManagerTest.cpp b/llvm/unittests/IR/PassManagerTest.cpp
index a6487169224c2..bb4db6120035f 100644
--- a/llvm/unittests/IR/PassManagerTest.cpp
+++ b/llvm/unittests/IR/PassManagerTest.cpp
@@ -828,7 +828,7 @@ TEST_F(PassManagerTest, FunctionPassCFGChecker) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -877,7 +877,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerInvalidateAnalysis) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -945,7 +945,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerWrapped) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });

>From a89ff83fbb4a39d6e91ae3bccbb6244c32433a4b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 16 Apr 2025 10:19:00 -0400
Subject: [PATCH 03/33] Run AMDGPUTargetVerifier within AMDGPU pipeline. Move
 IsValid from Module to TargetVerify.

---
 clang/lib/CodeGen/BackendUtil.cpp               | 2 +-
 llvm/include/llvm/IR/Module.h                   | 4 ----
 llvm/include/llvm/Target/TargetVerifier.h       | 2 ++
 llvm/lib/IR/Verifier.cpp                        | 4 ++--
 llvm/lib/Passes/StandardInstrumentations.cpp    | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  | 5 +++++
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 3 ++-
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp  | 6 +++---
 8 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 7e0a3cf5591ce..560129f871026 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -917,7 +917,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       TheModule->getContext(),
       (CodeGenOpts.DebugPassManager || DebugPassStructure),
       CodeGenOpts.VerifyEach, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
   // Handle the assignment tracking feature options.
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 3037b2c1644d9..34ac47a2b8be4 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,10 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module valid as determined by one of the verification passes
-  /// i.e. Lint, Verifier, TargetVerifier.
-  bool IsValid = true;
-
   /// Is this Module using intrinsics to record the position of debugging
   /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
   /// \ref BasicBlock.
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index ad5aeb895953d..2d0c039132c35 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -70,6 +70,8 @@ class TargetVerify {
   std::string Messages;
   raw_string_ostream MessagesStr;
 
+  bool IsValid = true;
+
   TargetVerify(Module *Mod)
       : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
         MessagesStr(Messages) {}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 18e9760f9c617..0054dd26327ae 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7870,7 +7870,7 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
   if (Res.IRBroken || Res.DebugInfoBroken) {
-    M.IsValid = false;
+    //M.IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken module found, compilation aborted!");
   }
@@ -7881,7 +7881,7 @@ PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
   if (res.IRBroken) {
-    F.getParent()->IsValid = false;
+    //F.getParent()->IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken function found, compilation aborted!");
   }
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index e2764b162fddb..2bc54cf7d4891 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -1474,7 +1474,7 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
           if (FAM) {
             TargetVerify TV(const_cast<Module*>(F->getParent()));
             TV.run(*const_cast<Function*>(F), *FAM);
-	    if (!F->getParent()->IsValid)
+	    if (!TV.IsValid)
               report_fatal_error(formatv("Broken function found after pass "
                                          "\"{0}\", compilation aborted!",
                                          P));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index d59087839b0e1..dc88042a66df9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,6 +90,7 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
@@ -1328,6 +1329,8 @@ void AMDGPUPassConfig::addIRPasses() {
       addPass(createLICMPass());
   }
 
+  //addPass(AMDGPUTargetVerifierPass());
+
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
@@ -2071,6 +2074,8 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   // but EarlyCSE can do neither of them.
   if (isPassEnabled(EnableScalarIRPasses))
     addEarlyCSEOrGVNPass(addPass);
+
+  addPass(AMDGPUTargetVerifierPass());
 }
 
 void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index e6cdec7160229..c70a6d1b6fa66 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -205,7 +205,8 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
 
   dbgs() << TV.MessagesStr.str();
   if (!TV.MessagesStr.str().empty()) {
-    F.getParent()->IsValid = false;
+    TV.IsValid = false;
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 3352d07deff2f..fbe7f6089ff18 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -163,9 +163,9 @@ int main(int argc, char **argv) {
     FPM.addPass(TargetVerifierPass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
-  MPM.run(*M, MAM);
-
-  if (!M->IsValid)
+  auto PA = MPM.run(*M, MAM);
+  auto PAC = PA.getChecker<VerifierAnalysis>();
+  if (!PAC.preserved())
     return 1;
 
   return 0;

>From df507f9bfc4caa5578b48be68fb6c362808ec4af Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 16 Apr 2025 11:08:20 -0400
Subject: [PATCH 04/33] Remove cmd line options that aren't required. Make
 error message explicit.

---
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll | 4 ++--
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index 584097d7bc134..c5e59d4a2369e 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,6 +1,6 @@
-; RUN: not not llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
-; CHECK: LLVM ERROR
+; CHECK: Shaders must return void
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index 0c3a5fe5ac4a5..8a503b7624a73 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each %s -o - 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
-; CHECK: ModuleToFunctionPassAdaptor
+; CHECK-NOT: Shaders must return void
   ret void
 }

>From 942312d57ea09d3dc9224d5cb3753390db13b2ba Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 19:36:34 -0400
Subject: [PATCH 05/33] Return Verifier none status through PreservedAnalyses
 on fail.

---
 llvm/lib/Analysis/Lint.cpp                     | 4 +++-
 llvm/lib/IR/Verifier.cpp                       | 2 ++
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 8 +++++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 1168005f48c0e..c8627e6db0692 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -735,9 +735,11 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) {
   Lint L(Mod, DL, AA, AC, DT, TLI);
   L.visit(F);
   dbgs() << L.MessagesStr.str();
-  if (AbortOnError && !L.MessagesStr.str().empty())
+  if (AbortOnError && !L.MessagesStr.str().empty()) {
     report_fatal_error(
         "linter found errors, aborting. (enabled by abort-on-error)", false);
+    return PreservedAnalyses::none();
+  }
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 0054dd26327ae..439823ba8b682 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7873,6 +7873,7 @@ PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
     //M.IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken module found, compilation aborted!");
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
@@ -7884,6 +7885,7 @@ PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
     //F.getParent()->IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken function found, compilation aborted!");
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index fbe7f6089ff18..042824ac37fea 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -164,9 +164,11 @@ int main(int argc, char **argv) {
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
   auto PA = MPM.run(*M, MAM);
-  auto PAC = PA.getChecker<VerifierAnalysis>();
-  if (!PAC.preserved())
-    return 1;
+  {
+    auto PAC = PA.getChecker<VerifierAnalysis>();
+    if (!PAC.preserved())
+      return 1;
+  }
 
   return 0;
 }

>From 6216a104469edfa1da128ce64be2cba2c2421b00 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 21:16:02 -0400
Subject: [PATCH 06/33] Rebase update.

---
 llvm/include/llvm/Target/TargetVerifier.h      | 2 +-
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 2d0c039132c35..fe683311b901c 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -73,7 +73,7 @@ class TargetVerify {
   bool IsValid = true;
 
   TargetVerify(Module *Mod)
-      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
+      : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
   void run(Function &F) {};
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 042824ac37fea..627bc51ef3a43 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -123,7 +123,7 @@ int main(int argc, char **argv) {
     return 1;
   }
   auto S = Triple::normalize(TargetTriple);
-  M->setTargetTriple(S);
+  M->setTargetTriple(Triple(S));
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Context, false/*debug PM*/,
@@ -153,7 +153,7 @@ int main(int argc, char **argv) {
 
   Triple TT(M->getTargetTriple());
   if (!NoLint)
-    FPM.addPass(LintPass());
+    FPM.addPass(LintPass(false));
   if (!NoVerify)
     MPM.addPass(VerifierPass());
   if (TT.isAMDGPU())

>From 4b7068f49c24f28d4fac762ea6f1f8420028d525 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 22:49:52 -0400
Subject: [PATCH 07/33] Add generic TargetVerifier.

---
 llvm/lib/Target/TargetVerifier.cpp | 32 ++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 llvm/lib/Target/TargetVerifier.cpp

diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
new file mode 100644
index 0000000000000..de3ff749e7c3c
--- /dev/null
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -0,0 +1,32 @@
+#include "llvm/Target/TargetVerifier.h"
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+
+namespace llvm {
+
+void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
+  if (TT.isAMDGPU()) {
+    auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
+    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+    }
+  }
+}
+
+} // namespace llvm

>From 86e806635cdf1f1606af654817aefa8d797b8cf5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:13:14 -0400
Subject: [PATCH 08/33] Remove store to const check since it is in Lint already

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 8 --------
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll          | 2 --
 2 files changed, 10 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index c70a6d1b6fa66..1cf2b277bee26 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -140,14 +140,6 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(IsValidInt(I.getOperand(i)->getType()),
                 "Int type is invalid.", I.getOperand(i));
 
-      // Ensure no store to const memory
-      if (auto *SI = dyn_cast<StoreInst>(&I))
-      {
-        unsigned AS = SI->getPointerAddressSpace();
-        Check(AS != 4, "Write to const memory", SI);
-      }
-
-      // Ensure no kernel to kernel calls.
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
         CallingConv::ID CalleeCC = CI->getCallingConv();
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index f56ff992a56c2..c628abbde11d1 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -32,8 +32,6 @@ define amdgpu_cs i32 @shader() {
 
 define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
 ; CHECK: Undefined behavior: Write to memory in const addrspace
-; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
-; CHECK-NEXT: Write to const memory
 ; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
   %r = add i32 %a, %b
   store i32 %r, ptr addrspace(4) %out

>From 17fc5c6be3406d4360e25b2de595d31afe7f77bc Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:14:38 -0400
Subject: [PATCH 09/33] Add chain followed by unreachable check

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp |  6 ++++++
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll          | 10 ++++++++++
 2 files changed, 16 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 1cf2b277bee26..8ea773bc0e66f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -142,6 +142,7 @@ void AMDGPUTargetVerify::run(Function &F) {
 
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
+        // Ensure no kernel to kernel calls.
         CallingConv::ID CalleeCC = CI->getCallingConv();
         if (CalleeCC == CallingConv::AMDGPU_KERNEL)
         {
@@ -149,6 +150,11 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(CallerCC != CallingConv::AMDGPU_KERNEL,
             "A kernel may not call a kernel", CI->getParent()->getParent());
         }
+
+        // Ensure chain intrinsics are followed by unreachables.
+        if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
+          Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
+            "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
 
       // Ensure MFMA is not in control flow with diverging operands
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index c628abbde11d1..e620df94ccde4 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -58,3 +58,13 @@ entry:
   %tmp2 = ashr i65 %x, 64
   ret i65 %tmp2
 }
+
+declare void @llvm.amdgcn.cs.chain.v3i32(ptr, i32, <3 x i32>, <3 x i32>, i32, ...)
+declare amdgpu_cs_chain void @chain_callee(<3 x i32> inreg, <3 x i32>)
+
+define amdgpu_cs void @no_unreachable(<3 x i32> inreg %a, <3 x i32> %b) {
+; CHECK: llvm.amdgcn.cs.chain must be followed by unreachable
+; CHECK-NEXT: call void (ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.v3i32.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  ret void
+}

>From 46e662cce2d1f3716fcf5c23be39fdb27a2d7ecb Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:26:19 -0400
Subject: [PATCH 10/33] Remove mfma check

---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 89 -------------------
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        | 25 ------
 2 files changed, 114 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 8ea773bc0e66f..684ced5bba574 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -14,9 +14,6 @@
 
 using namespace llvm;
 
-//static cl::opt<bool>
-//MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
-
 // Check - We know that cond should be true, if not print an error message.
 #define Check(C, ...)                                                          \
   do {                                                                         \
@@ -26,60 +23,6 @@ using namespace llvm;
     }                                                                          \
   } while (false)
 
-static bool isMFMA(unsigned IID) {
-  switch (IID) {
-    case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
-    case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
-    case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
-    case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
-    case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
-    case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
-    case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
-    case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
-    case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
-    case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
-    case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
-
-    case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
-    case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
-
-    case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
-    case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
-
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
-
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
-      return true;
-    default:
-      return false;
-  }
-}
-
 namespace llvm {
 /*class AMDGPUTargetVerify : public TargetVerify {
 public:
@@ -129,8 +72,6 @@ void AMDGPUTargetVerify::run(Function &F) {
   for (auto &BB : F) {
 
     for (auto &I : BB) {
-      //if (MarkUniform)
-        //outs() << UA->isUniform(&I) << ' ' << I << '\n';
 
       // Ensure integral types are valid: i8, i16, i32, i64, i128
       if (I.getType()->isIntegerTy())
@@ -156,36 +97,6 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
             "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
-
-      // Ensure MFMA is not in control flow with diverging operands
-      if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
-        if (isMFMA(II->getIntrinsicID())) {
-          bool InControlFlow = false;
-          for (const auto &P : predecessors(&BB))
-            if (!PDT->dominates(&BB, P)) {
-              InControlFlow = true;
-              break;
-            }
-          for (const auto &S : successors(&BB))
-            if (!DT->dominates(&BB, S)) {
-              InControlFlow = true;
-              break;
-            }
-          if (InControlFlow) {
-            // If operands to MFMA are not uniform, MFMA cannot be in control flow
-            bool hasUniformOperands = true;
-            for (unsigned i = 0; i < II->getNumOperands(); i++) {
-              if (!UA->isUniform(II->getOperand(i))) {
-                dbgs() << "Not uniform: " << *II->getOperand(i) << '\n';
-                hasUniformOperands = false;
-              }
-            }
-            if (!hasUniformOperands) Check(false, "MFMA in control flow", II);
-            //else Check(false, "MFMA in control flow (uniform operands)", II);
-          }
-          //else Check(false, "MFMA not in control flow", II);
-        }
-      }
     }
   }
 }
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index e620df94ccde4..62b220d7d9f49 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -1,30 +1,5 @@
 ; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
 
-define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg) #0 {
-; CHECK: Not uniform: %in.f32 = load <32 x float>, ptr addrspace(1) %gep, align 128
-; CHECK-NEXT: MFMA in control flow
-; CHECK-NEXT:   %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
-s:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
-  %in.i32 = load <32 x i32>, ptr addrspace(1) %gep
-  %in.f32 = load <32 x float>, ptr addrspace(1) %gep
-
-  %0 = icmp eq <32 x i32> %in.i32, zeroinitializer
-  %div.br = extractelement <32 x i1> %0, i32 0
-  br i1 %div.br, label %if.3, label %else.0
-
-if.3:
-  br label %join
-
-else.0:
-  %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
-  br label %join
-
-join:
-  ret void
-}
-
 define amdgpu_cs i32 @shader() {
 ; CHECK: Shaders must return void
   ret i32 0

>From 7e471c9111d6a3a33f0de6bac883353669a69b0a Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 21 Apr 2025 20:54:10 -0400
Subject: [PATCH 11/33] Add registerVerifierPasses to PassBuilder and add the
 verifier passes to PassRegistry.

---
 llvm/include/llvm/InitializePasses.h          |  1 +
 llvm/include/llvm/Passes/PassBuilder.h        | 21 +++++++
 .../llvm/Passes/TargetPassRegistry.inc        | 12 ++++
 .../TargetVerify/AMDGPUTargetVerifier.h       | 11 ++--
 llvm/lib/Passes/PassBuilder.cpp               |  7 +++
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 11 ++++
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 56 ++++++++++++++++++-
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  1 +
 8 files changed, 114 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 1b5b1d5888824..662f6b71aa46a 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,6 +333,7 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
+LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index f13b5c678a894..c134f2d0c1112 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -173,6 +173,13 @@ class PassBuilder {
   /// additional analyses.
   LLVM_ABI void registerLoopAnalyses(LoopAnalysisManager &LAM);
 
+  /// Registers all available verifier passes.
+  ///
+  /// This is an interface that can be used to populate a
+  /// \c ModuleAnalysisManager with all registered loop analyses. Callers can
+  /// still manually register any additional analyses.
+  void registerVerifierPasses(ModulePassManager &PM, FunctionPassManager &);
+
   /// Registers all available machine function analysis passes.
   ///
   /// This is an interface that can be used to populate a \c
@@ -576,6 +583,15 @@ class PassBuilder {
   }
   /// @}}
 
+  /// Register a callback for parsing an Verifier Name to populate
+  /// the given managers.
+  void registerVerifierCallback(
+      const std::function<bool(ModulePassManager &MPM)> &C,
+      const std::function<bool(FunctionPassManager &MPM)> &CF) {
+    VerifierCallbacks.push_back(C);
+    FnVerifierCallbacks.push_back(CF);
+  }
+
   /// {{@ Register pipeline parsing callbacks with this pass builder instance.
   /// Using these callbacks, callers can parse both a single pass name, as well
   /// as entire sub-pipelines, and populate the PassManager instance
@@ -851,6 +867,11 @@ class PassBuilder {
   // Callbacks to parse `filter` parameter in register allocation passes
   SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
+  // Verifier callbacks
+  SmallVector<std::function<bool(ModulePassManager &)>, 2>
+      VerifierCallbacks;
+  SmallVector<std::function<bool(FunctionPassManager &)>, 2>
+      FnVerifierCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc
index 521913cb25a4a..2d04b874cf360 100644
--- a/llvm/include/llvm/Passes/TargetPassRegistry.inc
+++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc
@@ -151,6 +151,18 @@ PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM,
   return false;
 });
 
+PB.registerVerifierCallback([](ModulePassManager &PM) {
+#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS) PM.addPass(CREATE_PASS)
+#include GET_PASS_REGISTRY
+#undef VERIFIER_MODULE_ANALYSIS
+  return false;
+}, [](FunctionPassManager &FPM) {
+#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS) FPM.addPass(CREATE_PASS)
+#include GET_PASS_REGISTRY
+#undef VERIFIER_FUNCTION_ANALYSIS
+  return false;
+});
+
 #undef ADD_PASS
 #undef ADD_PASS_WITH_PARAMS
 
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index d8a3fda4f87dc..b6a7412e8c1ef 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -39,14 +39,17 @@ class AMDGPUTargetVerify : public TargetVerify {
 public:
   Module *Mod;
 
-  DominatorTree *DT;
-  PostDominatorTree *PDT;
-  UniformityInfo *UA;
+  DominatorTree *DT = nullptr;
+  PostDominatorTree *PDT = nullptr;
+  UniformityInfo *UA = nullptr;
+
+  AMDGPUTargetVerify(Module *Mod)
+    : TargetVerify(Mod), Mod(Mod) {}
 
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
-  void run(Function &F);
+  bool run(Function &F);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 4603eaff8ade9..561996d0c8980 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -576,6 +576,13 @@ void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     C(LAM);
 }
 
+void PassBuilder::registerVerifierPasses(ModulePassManager &MPM, FunctionPassManager &FPM) {
+  for (auto &C : VerifierCallbacks)
+    C(MPM);
+  for (auto &C : FnVerifierCallbacks)
+    C(FPM);
+}
+
 static std::optional<std::pair<bool, bool>>
 parseFunctionPipelineName(StringRef Name) {
   std::pair<bool, bool> Params;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 13453963eec6d..652cd23f62443 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -82,6 +82,17 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
+#ifndef VERIFIER_MODULE_ANALYSIS
+#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS)
+#endif
+#ifndef VERIFIER_FUNCTION_ANALYSIS
+#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
+#endif
+VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
+#undef VERIFIER_MODULE_ANALYSIS
+#undef VERIFIER_FUNCTION_ANALYSIS
+
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 684ced5bba574..63a7526b9abdc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -5,6 +5,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -19,7 +20,7 @@ using namespace llvm;
   do {                                                                         \
     if (!(C)) {                                                                \
       TargetVerify::CheckFailed(__VA_ARGS__);                                  \
-      return;                                                                  \
+      return false;                                                                  \
     }                                                                          \
   } while (false)
 
@@ -64,7 +65,7 @@ static bool isShader(CallingConv::ID CC) {
   }
 }
 
-void AMDGPUTargetVerify::run(Function &F) {
+bool AMDGPUTargetVerify::run(Function &F) {
   // Ensure shader calling convention returns void
   if (isShader(F.getCallingConv()))
     Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
@@ -99,6 +100,10 @@ void AMDGPUTargetVerify::run(Function &F) {
       }
     }
   }
+
+  if (!MessagesStr.str().empty())
+    return false;
+  return true;
 }
 
 PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
@@ -120,4 +125,51 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
 
   return PreservedAnalyses::all();
 }
+
+struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<AMDGPUTargetVerify> TV;
+  bool FatalErrors = true;
+
+  AMDGPUTargetVerifierLegacyPass() : FunctionPass(ID) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
+      : FunctionPass(ID),
+        FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<AMDGPUTargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (TV->run(F) && FatalErrors) {
+      errs() << "in function " << F.getName() << '\n';
+      report_fatal_error("Broken function found, compilation aborted!");
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    //IsValid &= TV->run();
+    if (FatalErrors && !IsValid)
+      report_fatal_error("Broken module found, compilation aborted!");
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char AMDGPUTargetVerifierLegacyPass::ID = 0;
 } // namespace llvm
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverify", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 627bc51ef3a43..503db7b1f8d18 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -144,6 +144,7 @@ int main(int argc, char **argv) {
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
+  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 

>From 1538985ecdd840e68207f83ca7b159bccef98ff5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Tue, 22 Apr 2025 14:57:31 -0400
Subject: [PATCH 12/33] Remove leftovers. Add titles. Add call to
 registerVerifierCallbacks in llc.

---
 llvm/lib/Passes/CMakeLists.txt                |  2 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  4 ---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 35 +++++++++++--------
 llvm/lib/Target/TargetVerifier.cpp            | 19 ++++++++++
 llvm/tools/llc/NewPMDriver.cpp                |  6 ++--
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  7 ++--
 6 files changed, 45 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index e5912f23903b6..b8d25b15186e8 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,7 +30,7 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
-  TargetParser
+  #TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index dc88042a66df9..0c41c633dcb3d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1329,8 +1329,6 @@ void AMDGPUPassConfig::addIRPasses() {
       addPass(createLICMPass());
   }
 
-  //addPass(AMDGPUTargetVerifierPass());
-
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
@@ -2074,8 +2072,6 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   // but EarlyCSE can do neither of them.
   if (isPassEnabled(EnableScalarIRPasses))
     addEarlyCSEOrGVNPass(addPass);
-
-  addPass(AMDGPUTargetVerifierPass());
 }
 
 void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 63a7526b9abdc..0eecedaebc7ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -1,3 +1,22 @@
+//===-- AMDGPUTargetVerifier.cpp - AMDGPU -------------------------*- C++ -*-===//
+////
+//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//// See https://llvm.org/LICENSE.txt for license information.
+//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+////
+////===----------------------------------------------------------------------===//
+////
+//// This file defines target verifier interfaces that can be used for some
+//// validation of input to the system, and for checking that transformations
+//// haven't done something bad. In contrast to the Verifier or Lint, the
+//// TargetVerifier looks for constructions invalid to a particular target
+//// machine.
+////
+//// To see what specifically is checked, look at an individual backend's
+//// TargetVerifier.
+////
+////===----------------------------------------------------------------------===//
+
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
 #include "llvm/Analysis/UniformityAnalysis.h"
@@ -25,19 +44,6 @@ using namespace llvm;
   } while (false)
 
 namespace llvm {
-/*class AMDGPUTargetVerify : public TargetVerify {
-public:
-  Module *Mod;
-
-  DominatorTree *DT;
-  PostDominatorTree *PDT;
-  UniformityInfo *UA;
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
-    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
-  void run(Function &F);
-};*/
 
 static bool IsValidInt(const Type *Ty) {
   return Ty->isIntegerTy(1) ||
@@ -147,7 +153,7 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   }
 
   bool runOnFunction(Function &F) override {
-    if (TV->run(F) && FatalErrors) {
+    if (!TV->run(F) && FatalErrors) {
       errs() << "in function " << F.getName() << '\n';
       report_fatal_error("Broken function found, compilation aborted!");
     }
@@ -160,7 +166,6 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
       if (F.isDeclaration())
         IsValid &= TV->run(F);
 
-    //IsValid &= TV->run();
     if (FatalErrors && !IsValid)
       report_fatal_error("Broken module found, compilation aborted!");
     return false;
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index de3ff749e7c3c..992a0c91d93b1 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -1,3 +1,22 @@
+//===-- TargetVerifier.cpp - LLVM IR Target Verifier ----------------*- C++ -*-===//
+////
+///// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+///// See https://llvm.org/LICENSE.txt for license information.
+///// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+/////
+/////===----------------------------------------------------------------------===//
+/////
+///// This file defines target verifier interfaces that can be used for some
+///// validation of input to the system, and for checking that transformations
+///// haven't done something bad. In contrast to the Verifier or Lint, the
+///// TargetVerifier looks for constructions invalid to a particular target
+///// machine.
+/////
+///// To see what specifically is checked, look at TargetVerifier.cpp or an
+///// individual backend's TargetVerifier.
+/////
+/////===----------------------------------------------------------------------===//
+
 #include "llvm/Target/TargetVerifier.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index a060d16e74958..a8f6b999af06e 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -114,6 +114,8 @@ int llvm::compileModuleWithNewPM(
                               VK == VerifierKind::EachPass);
   registerCodeGenCallback(PIC, *Target);
 
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
   MachineFunctionAnalysisManager MFAM;
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
@@ -125,15 +127,13 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
+  PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
 
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
-
   if (!PassPipeline.empty()) {
     // Construct a custom pass pipeline that starts after instruction
     // selection.
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 503db7b1f8d18..b00bab66c6c3e 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -1,4 +1,4 @@
-//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
+//===--- llvm-tgt-verify.cpp - Target Verifier ----------------- ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Tool to fuzz instruction selection using libFuzzer.
+// Tool to verify a target.
 //
 //===----------------------------------------------------------------------===//
 
@@ -144,14 +144,11 @@ int main(int argc, char **argv) {
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
-  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
-  //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
-
   Triple TT(M->getTargetTriple());
   if (!NoLint)
     FPM.addPass(LintPass(false));

>From 32b0c0d5d74b3b78699b78736604e87369ca4217 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 24 Apr 2025 16:02:21 -0400
Subject: [PATCH 13/33] Add pass to legacy PM.

---
 llvm/include/llvm/CodeGen/Passes.h            |  2 +
 llvm/include/llvm/Target/TargetVerifier.h     |  6 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  |  4 +-
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  2 +-
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 45 ----------
 llvm/lib/Target/TargetVerifier.cpp            | 87 ++++++++++++++++++-
 llvm/tools/llc/NewPMDriver.cpp                |  6 +-
 llvm/tools/llc/llc.cpp                        |  4 +
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  1 +
 9 files changed, 105 insertions(+), 52 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 990452fa11fec..53bd70183e476 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -620,6 +620,8 @@ LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
+
+LLVM_ABI FunctionPass *createTargetVerifierLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index fe683311b901c..23ef2e0b8d4ef 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -30,7 +30,7 @@ class Function;
 
 class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 class TargetVerify {
@@ -76,8 +76,8 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  void run(Function &F) {};
-  void run(Function &F, FunctionAnalysisManager &AM);
+  bool run(Function &F);
+  bool run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 2bc54cf7d4891..539f4b24b5110 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -57,6 +57,8 @@ static cl::opt<bool> VerifyAnalysisInvalidation("verify-analysis-invalidation",
 #endif
 );
 
+static cl::opt<bool> VerifyTargetEach("verify-tgt-each");
+
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
 // of this option.  Note that this option has no effect without -print-changed.
@@ -1471,7 +1473,7 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
                                        "\"{0}\", compilation aborted!",
                                        P));
 
-          if (FAM) {
+          if (VerifyTargetEach && FAM) {
             TargetVerify TV(const_cast<Module*>(F->getParent()));
             TV.run(*const_cast<Function*>(F), *FAM);
 	    if (!TV.IsValid)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 652cd23f62443..ad4fa93d09e0c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -89,7 +89,7 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("tgtverifier", TargetVerifierPass())
 #undef VERIFIER_MODULE_ANALYSIS
 #undef VERIFIER_FUNCTION_ANALYSIS
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 0eecedaebc7ce..96bcaaf6f2ac9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -132,49 +132,4 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
   return PreservedAnalyses::all();
 }
 
-struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
-  static char ID;
-
-  std::unique_ptr<AMDGPUTargetVerify> TV;
-  bool FatalErrors = true;
-
-  AMDGPUTargetVerifierLegacyPass() : FunctionPass(ID) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
-      : FunctionPass(ID),
-        FatalErrors(FatalErrors) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool doInitialization(Module &M) override {
-    TV = std::make_unique<AMDGPUTargetVerify>(&M);
-    return false;
-  }
-
-  bool runOnFunction(Function &F) override {
-    if (!TV->run(F) && FatalErrors) {
-      errs() << "in function " << F.getName() << '\n';
-      report_fatal_error("Broken function found, compilation aborted!");
-    }
-    return false;
-  }
-
-  bool doFinalization(Module &M) override {
-    bool IsValid = true;
-    for (Function &F : M)
-      if (F.isDeclaration())
-        IsValid &= TV->run(F);
-
-    if (FatalErrors && !IsValid)
-      report_fatal_error("Broken module found, compilation aborted!");
-    return false;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-  }
-};
-char AMDGPUTargetVerifierLegacyPass::ID = 0;
 } // namespace llvm
-INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverify", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 992a0c91d93b1..170fc4769c1d8 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Target/TargetVerifier.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
+#include "llvm/InitializePasses.h"
 #include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Support/Debug.h"
@@ -32,7 +33,22 @@
 
 namespace llvm {
 
-void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
+bool TargetVerify::run(Function &F) {
+  if (TT.isAMDGPU()) {
+    AMDGPUTargetVerify TV(Mod);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+      return false;
+    }
+    return true;
+  }
+  report_fatal_error("Target has no verification method\n");
+}
+
+bool TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
   if (TT.isAMDGPU()) {
     auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
     auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
@@ -44,8 +60,77 @@ void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
     dbgs() << TV.MessagesStr.str();
     if (!TV.MessagesStr.str().empty()) {
       TV.IsValid = false;
+      return false;
+    }
+    return true;
+  }
+  report_fatal_error("Target has no verification method\n");
+}
+
+PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto TT = F.getParent()->getTargetTriple();
+
+  if (TT.isAMDGPU()) {
+    auto *Mod = F.getParent();
+
+    auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+      return PreservedAnalyses::none();
     }
+    return PreservedAnalyses::all();
   }
+  report_fatal_error("Target has no verification method\n");
 }
 
+struct TargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<TargetVerify> TV;
+
+  TargetVerifierLegacyPass() : FunctionPass(ID) {
+    initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<TargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (!TV->run(F)) {
+      errs() << "in function " << F.getName() << '\n';
+      report_fatal_error("broken function found, compilation aborted!");
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    if (!IsValid)
+      report_fatal_error("broken module found, compilation aborted!");
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char TargetVerifierLegacyPass::ID = 0;
+FunctionPass *createTargetVerifierLegacyPass() {
+  return new TargetVerifierLegacyPass();
+}
 } // namespace llvm
+using namespace llvm;
+INITIALIZE_PASS(TargetVerifierLegacyPass, "tgtverifier", "Target Verifier", false, false)
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index a8f6b999af06e..4b95977a10c5f 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -57,6 +57,9 @@ static cl::opt<bool>
     DebugPM("debug-pass-manager", cl::Hidden,
             cl::desc("Print pass management debugging information"));
 
+static cl::opt<bool> VerifyTarget("verify-tgt-new-pm",
+		                  cl::desc("Verify the target"));
+
 bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
   DiagnosticHandler::handleDiagnostics(DI);
   if (DI.getKind() == llvm::DK_SrcMgr) {
@@ -127,7 +130,8 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
-  PB.registerVerifierPasses(MPM, FPM);
+  if (VerifyTarget)
+    PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 140459ba2de21..1fd8a9f9cd9f8 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -209,6 +209,8 @@ static cl::opt<std::string> PassPipeline(
 static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
                                cl::desc("Alias for -passes"));
 
+static cl::opt<bool> VerifyTarget("verify-tgt", cl::desc("Verify the target"));
+
 namespace {
 
 std::vector<std::string> &getRunPassNames() {
@@ -658,6 +660,8 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
+  if (VerifyTarget)
+    PM.add(createTargetVerifierLegacyPass());
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index b00bab66c6c3e..b86c2318b45b7 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -141,6 +141,7 @@ int main(int argc, char **argv) {
   ModuleAnalysisManager MAM;
   PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
   PB.registerModuleAnalyses(MAM);
+  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);

>From 387b2d5936a699238ac775726b0b30860f68507f Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 00:13:42 -0400
Subject: [PATCH 14/33] Add fam in other projects.

---
 flang/lib/Frontend/FrontendActions.cpp      | 2 +-
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index d684eeb696755..c15151fc5c1b8 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -947,7 +947,7 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
 
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
-  si.registerCallbacks(pic, &mam);
+  si.registerCallbacks(pic, &mam, &fam);
   if (ci.isTimingEnabled())
     si.getTimePasses().setOutStream(ci.getTimingStreamLLVM());
   pto.LoopUnrolling = opts.UnrollLoops;
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index 0f58391c50667..f9664025f61f1 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -577,7 +577,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index 7117eaf4982b0..eae06d9f57467 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -851,7 +851,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index cb7b6cc8651c1..30ad79ef2fc58 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -970,7 +970,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 91b7191a07c6f..4a39bc33c5591 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1139,7 +1139,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Promote allocas to registers.

>From d2c9425f9e951768842ca9771148611bdba5e1bb Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 02:30:40 -0400
Subject: [PATCH 15/33] Avoid fatal errors in llc.

---
 llvm/lib/Target/TargetVerifier.cpp             | 18 +++++++++++++-----
 .../test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll |  2 +-
 .../test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll |  2 +-
 llvm/tools/llc/llc.cpp                         |  2 +-
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 170fc4769c1d8..3be50f4ef6da3 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -94,8 +94,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   static char ID;
 
   std::unique_ptr<TargetVerify> TV;
+  bool FatalErrors = false;
 
-  TargetVerifierLegacyPass() : FunctionPass(ID) {
+  TargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
+    FatalErrors(FatalErrors) {
     initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
@@ -107,7 +109,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   bool runOnFunction(Function &F) override {
     if (!TV->run(F)) {
       errs() << "in function " << F.getName() << '\n';
-      report_fatal_error("broken function found, compilation aborted!");
+      if (FatalErrors)
+        report_fatal_error("broken function found, compilation aborted!");
+      else
+        errs() << "broken function found, compilation aborted!\n";
     }
     return false;
   }
@@ -119,7 +124,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
         IsValid &= TV->run(F);
 
     if (!IsValid)
-      report_fatal_error("broken module found, compilation aborted!");
+      if (FatalErrors)
+        report_fatal_error("broken module found, compilation aborted!");
+      else
+        errs() << "broken module found, compilation aborted!\n";
     return false;
   }
 
@@ -128,8 +136,8 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   }
 };
 char TargetVerifierLegacyPass::ID = 0;
-FunctionPass *createTargetVerifierLegacyPass() {
-  return new TargetVerifierLegacyPass();
+FunctionPass *createTargetVerifierLegacyPass(bool FatalErrors) {
+  return new TargetVerifierLegacyPass(FatalErrors);
 }
 } // namespace llvm
 using namespace llvm;
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index c5e59d4a2369e..e2d9edda5d008 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
 ; CHECK: Shaders must return void
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index 8a503b7624a73..a2dab0ff47924 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm %s -o - 2>&1 | FileCheck %s --allow-empty
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
 ; CHECK-NOT: Shaders must return void
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 1fd8a9f9cd9f8..329d95826551f 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -661,7 +661,7 @@ static int compileModule(char **argv, LLVMContext &Context) {
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
   if (VerifyTarget)
-    PM.add(createTargetVerifierLegacyPass());
+    PM.add(createTargetVerifierLegacyPass(false));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {

>From 31a79a57d9d41365ffcd7b263b2a05153463b2f5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 03:40:47 -0400
Subject: [PATCH 16/33] Add tool to build/test.

---
 llvm/test/CMakeLists.txt                      |  1 +
 llvm/test/lit.cfg.py                          |  1 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |  1 +
 .../llvm/tools/llvm-tgt-verify/BUILD.gn       | 25 +++++++++++++++++++
 4 files changed, 28 insertions(+)
 create mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn

diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 2a6135da9a61e..a2bb112db8746 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -136,6 +136,7 @@ set(LLVM_TEST_DEPENDS
           llvm-strip
           llvm-symbolizer
           llvm-tblgen
+          llvm-tgt-verify
           llvm-readtapi
           llvm-tli-checker
           llvm-undname
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index bd6e37c848d8c..bf06e80dc7ae5 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -228,6 +228,7 @@ def get_asan_rtlib():
         "llvm-strings",
         "llvm-strip",
         "llvm-tblgen",
+        "llvm-tgt-verify",
         "llvm-readtapi",
         "llvm-undname",
         "llvm-windres",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 946b63f8a54fb..36cf956545caf 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -320,6 +320,7 @@ group("test") {
     "//llvm/tools/llvm-strings",
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-tli-checker",
+    "//llvm/tools/llvm-tgt-verify",
     "//llvm/tools/llvm-undname",
     "//llvm/tools/llvm-xray",
     "//llvm/tools/lto",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
new file mode 100644
index 0000000000000..b751bafc5052c
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tgtverifier("llvm-tgt-verify") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmPrinter",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGenTypes",
+    "//llvm/lib/Core",
+    "//llvm/lib/IRPrinter",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/MC",
+    "//llvm/lib/MIRParser",
+    "//llvm/lib/Passes",
+    "//llvm/lib/Remarks",
+    "//llvm/lib/ScalarOpts",
+    "//llvm/lib/SelectionDAG",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/TargetParser",
+    "//llvm/lib/TransformUtils",
+    "//llvm/lib/Vectorize",
+  ]
+  sources = [ "llvm-tgt-verify.cpp" ]
+}

>From b51b530c5672945e06a20b93b1f652d097386e61 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 28 Apr 2025 10:42:24 -0400
Subject: [PATCH 17/33] Cleanup of unrequired functions.

---
 llvm/include/llvm/Target/TargetVerifier.h     |  1 -
 .../TargetVerify/AMDGPUTargetVerifier.h       |  1 -
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 25 +++----------------
 llvm/lib/Target/TargetVerifier.cpp            | 22 ++--------------
 4 files changed, 6 insertions(+), 43 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 23ef2e0b8d4ef..427a05b2648a9 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -77,7 +77,6 @@ class TargetVerify {
         MessagesStr(Messages) {}
 
   bool run(Function &F);
-  bool run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index b6a7412e8c1ef..74e5b5f7a1efd 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -32,7 +32,6 @@ class Function;
 
 class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 class AMDGPUTargetVerify : public TargetVerify {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 96bcaaf6f2ac9..bda412f723242 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -107,29 +107,12 @@ bool AMDGPUTargetVerify::run(Function &F) {
     }
   }
 
-  if (!MessagesStr.str().empty())
+  //dbgs() << MessagesStr.str();
+  if (!MessagesStr.str().empty()) {
+    //IsValid = false;
     return false;
-  return true;
-}
-
-PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
-
-  auto *Mod = F.getParent();
-
-  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-  TV.run(F);
-
-  dbgs() << TV.MessagesStr.str();
-  if (!TV.MessagesStr.str().empty()) {
-    TV.IsValid = false;
-    return PreservedAnalyses::none();
   }
-
-  return PreservedAnalyses::all();
+  return true;
 }
 
 } // namespace llvm
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 3be50f4ef6da3..6b57c18ff9316 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -48,25 +48,6 @@ bool TargetVerify::run(Function &F) {
   report_fatal_error("Target has no verification method\n");
 }
 
-bool TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
-  if (TT.isAMDGPU()) {
-    auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
-    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return false;
-    }
-    return true;
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
 PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto TT = F.getParent()->getTargetTriple();
 
@@ -123,11 +104,12 @@ struct TargetVerifierLegacyPass : public FunctionPass {
       if (F.isDeclaration())
         IsValid &= TV->run(F);
 
-    if (!IsValid)
+    if (!IsValid) {
       if (FatalErrors)
         report_fatal_error("broken module found, compilation aborted!");
       else
         errs() << "broken module found, compilation aborted!\n";
+    }
     return false;
   }
 

>From d41d5f9e6a33b70f71a20d58642fd73c9327e1e5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 28 Apr 2025 10:43:32 -0400
Subject: [PATCH 18/33] Make virtual.

---
 llvm/include/llvm/Target/TargetVerifier.h                    | 2 +-
 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 427a05b2648a9..ade2676a64325 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -76,7 +76,7 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  bool run(Function &F);
+  virtual bool run(Function &F);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 74e5b5f7a1efd..b97fbc046e391 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -48,7 +48,7 @@ class AMDGPUTargetVerify : public TargetVerify {
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
-  bool run(Function &F);
+  bool run(Function &F) override;
 };
 
 } // namespace llvm

>From 7ece5e262a4f4aff350a511af5d33496ac2d5604 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 10:56:43 -0400
Subject: [PATCH 19/33] Remove from legacy PM. Add to target dependent
 pipeline.

---
 llvm/include/llvm/InitializePasses.h          |  2 +-
 llvm/include/llvm/Target/TargetVerifier.h     |  4 +-
 .../TargetVerify/AMDGPUTargetVerifier.h       |  1 +
 llvm/lib/Passes/StandardInstrumentations.cpp  | 10 +--
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  2 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  2 +
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 74 ++++++++++++++-
 llvm/lib/Target/TargetVerifier.cpp            | 90 -------------------
 llvm/tools/llc/llc.cpp                        |  2 -
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 -
 10 files changed, 84 insertions(+), 105 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 662f6b71aa46a..d718f92877276 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,7 +333,7 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
-LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+//LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index ade2676a64325..1d12eb55bbf0a 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -30,7 +30,7 @@ class Function;
 
 class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  virtual PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) = 0;
 };
 
 class TargetVerify {
@@ -76,7 +76,7 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  virtual bool run(Function &F);
+  virtual bool run(Function &F) = 0;
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index b97fbc046e391..49bcbc8849e3c 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -32,6 +32,7 @@ class Function;
 
 class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
 };
 
 class AMDGPUTargetVerify : public TargetVerify {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 539f4b24b5110..905ba667fac7a 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,7 +40,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
-#include "llvm/Target/TargetVerifier.h"
+//#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -1474,12 +1474,12 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
                                        P));
 
           if (VerifyTargetEach && FAM) {
-            TargetVerify TV(const_cast<Module*>(F->getParent()));
-            TV.run(*const_cast<Function*>(F), *FAM);
-	    if (!TV.IsValid)
+            //TargetVerify TV(const_cast<Module*>(F->getParent()));
+            //TV.run(*const_cast<Function*>(F), *FAM);
+	    /*if (!TV.IsValid)
               report_fatal_error(formatv("Broken function found after pass "
                                          "\"{0}\", compilation aborted!",
-                                         P));
+                                         P));*/
           }
         } else {
           const auto *M = unwrapIR<Module>(IR);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index ad4fa93d09e0c..652cd23f62443 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -89,7 +89,7 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("tgtverifier", TargetVerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
 #undef VERIFIER_MODULE_ANALYSIS
 #undef VERIFIER_FUNCTION_ANALYSIS
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 0c41c633dcb3d..718655b7ebf00 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -2010,6 +2010,8 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
 }
 
 void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
+  addPass(AMDGPUTargetVerifierPass());
+
   if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
     addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index bda412f723242..cedd9ddc78011 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -107,12 +107,82 @@ bool AMDGPUTargetVerify::run(Function &F) {
     }
   }
 
-  //dbgs() << MessagesStr.str();
+  dbgs() << MessagesStr.str();
   if (!MessagesStr.str().empty()) {
-    //IsValid = false;
+    IsValid = false;
     return false;
   }
   return true;
 }
 
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto *Mod = F.getParent();
+
+  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  TV.run(F);
+
+  dbgs() << TV.MessagesStr.str();
+  if (!TV.MessagesStr.str().empty()) {
+    TV.IsValid = false;
+    return PreservedAnalyses::none();
+  }
+  return PreservedAnalyses::all();
+}
+
+/*
+struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<AMDGPUTargetVerify> TV;
+  bool FatalErrors = false;
+
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
+    FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<AMDGPUTargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (!TV->run(F)) {
+      errs() << "in function " << F.getName() << '\n';
+      if (FatalErrors)
+        report_fatal_error("broken function found, compilation aborted!");
+      else
+        errs() << "broken function found, compilation aborted!\n";
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    if (!IsValid) {
+      if (FatalErrors)
+        report_fatal_error("broken module found, compilation aborted!");
+      else
+        errs() << "broken module found, compilation aborted!\n";
+    }
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char AMDGPUTargetVerifierLegacyPass::ID = 0;
+FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
+  return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
+}*/
 } // namespace llvm
+//INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 6b57c18ff9316..c63ae2a2c5daf 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -33,94 +33,4 @@
 
 namespace llvm {
 
-bool TargetVerify::run(Function &F) {
-  if (TT.isAMDGPU()) {
-    AMDGPUTargetVerify TV(Mod);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return false;
-    }
-    return true;
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
-PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
-  auto TT = F.getParent()->getTargetTriple();
-
-  if (TT.isAMDGPU()) {
-    auto *Mod = F.getParent();
-
-    auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return PreservedAnalyses::none();
-    }
-    return PreservedAnalyses::all();
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
-struct TargetVerifierLegacyPass : public FunctionPass {
-  static char ID;
-
-  std::unique_ptr<TargetVerify> TV;
-  bool FatalErrors = false;
-
-  TargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
-    FatalErrors(FatalErrors) {
-    initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool doInitialization(Module &M) override {
-    TV = std::make_unique<TargetVerify>(&M);
-    return false;
-  }
-
-  bool runOnFunction(Function &F) override {
-    if (!TV->run(F)) {
-      errs() << "in function " << F.getName() << '\n';
-      if (FatalErrors)
-        report_fatal_error("broken function found, compilation aborted!");
-      else
-        errs() << "broken function found, compilation aborted!\n";
-    }
-    return false;
-  }
-
-  bool doFinalization(Module &M) override {
-    bool IsValid = true;
-    for (Function &F : M)
-      if (F.isDeclaration())
-        IsValid &= TV->run(F);
-
-    if (!IsValid) {
-      if (FatalErrors)
-        report_fatal_error("broken module found, compilation aborted!");
-      else
-        errs() << "broken module found, compilation aborted!\n";
-    }
-    return false;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-  }
-};
-char TargetVerifierLegacyPass::ID = 0;
-FunctionPass *createTargetVerifierLegacyPass(bool FatalErrors) {
-  return new TargetVerifierLegacyPass(FatalErrors);
-}
 } // namespace llvm
-using namespace llvm;
-INITIALIZE_PASS(TargetVerifierLegacyPass, "tgtverifier", "Target Verifier", false, false)
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 329d95826551f..2e9e4837fe467 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -660,8 +660,6 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
-  if (VerifyTarget)
-    PM.add(createTargetVerifierLegacyPass(false));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index b86c2318b45b7..d832dcdff4ad0 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -158,8 +158,6 @@ int main(int argc, char **argv) {
   if (TT.isAMDGPU())
     FPM.addPass(AMDGPUTargetVerifierPass());
   else if (false) {} // ...
-  else
-    FPM.addPass(TargetVerifierPass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
   auto PA = MPM.run(*M, MAM);

>From 42b89826fa18b4796c9fdee25864e929a2351661 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 11:18:50 -0400
Subject: [PATCH 20/33] Add back to legacy PM.

---
 llvm/include/llvm/InitializePasses.h            | 1 -
 llvm/lib/Target/AMDGPU/AMDGPU.h                 | 3 +++
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  | 1 +
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 8 ++++----
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index d718f92877276..1b5b1d5888824 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,7 +333,6 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
-//LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 5a917734e9c74..26fb0a967fd76 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -549,6 +549,9 @@ extern char &GCNRewritePartialRegUsesID;
 void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
 extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
+FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
+void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 718655b7ebf00..8b0b8f6df740e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1411,6 +1411,7 @@ bool AMDGPUPassConfig::addGCPasses() {
 //===----------------------------------------------------------------------===//
 
 bool GCNPassConfig::addPreISel() {
+  addPass(createAMDGPUTargetVerifierLegacyPass(false));
   AMDGPUPassConfig::addPreISel();
 
   if (TM->getOptLevel() > CodeGenOptLevel::None)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index cedd9ddc78011..c4d303bee6ef8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -17,6 +17,7 @@
 ////
 ////===----------------------------------------------------------------------===//
 
+#include "AMDGPU.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
 #include "llvm/Analysis/UniformityAnalysis.h"
@@ -24,7 +25,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-#include "llvm/InitializePasses.h"
+//#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -133,7 +134,6 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
   return PreservedAnalyses::all();
 }
 
-/*
 struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   static char ID;
 
@@ -183,6 +183,6 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
 char AMDGPUTargetVerifierLegacyPass::ID = 0;
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
   return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
-}*/
+}
 } // namespace llvm
-//INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)

>From 230f33b8db0650a59c5656d2caf5c7be31f0aa49 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 14:04:10 -0400
Subject: [PATCH 21/33] Remove reference to FAM in registerCallbacks and
 VerifyEach for TargetVerify in instrumentation

---
 clang/lib/CodeGen/BackendUtil.cpp             |  2 +-
 flang/lib/Frontend/FrontendActions.cpp        |  2 +-
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp   |  2 +-
 .../llvm/Passes/StandardInstrumentations.h    |  6 ++----
 llvm/lib/LTO/LTOBackend.cpp                   |  2 +-
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp         |  2 +-
 llvm/lib/Passes/PassBuilderBindings.cpp       |  2 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  | 21 ++++---------------
 llvm/tools/llc/NewPMDriver.cpp                |  7 ++++---
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 +-
 llvm/tools/opt/NewPMDriver.cpp                |  2 +-
 llvm/unittests/IR/PassManagerTest.cpp         |  6 +++---
 15 files changed, 24 insertions(+), 38 deletions(-)

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 560129f871026..7e0a3cf5591ce 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -917,7 +917,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       TheModule->getContext(),
       (CodeGenOpts.DebugPassManager || DebugPassStructure),
       CodeGenOpts.VerifyEach, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
   // Handle the assignment tracking feature options.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index c15151fc5c1b8..d684eeb696755 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -947,7 +947,7 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
 
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
-  si.registerCallbacks(pic, &mam, &fam);
+  si.registerCallbacks(pic, &mam);
   if (ci.isTimingEnabled())
     si.getTimePasses().setOutStream(ci.getTimingStreamLLVM());
   pto.LoopUnrolling = opts.UnrollLoops;
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index f9664025f61f1..0f58391c50667 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -577,7 +577,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index eae06d9f57467..7117eaf4982b0 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -851,7 +851,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index 30ad79ef2fc58..cb7b6cc8651c1 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -970,7 +970,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 4a39bc33c5591..91b7191a07c6f 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1139,7 +1139,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Promote allocas to registers.
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 7d7b858b447cc..df9d8de8a025a 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,8 +478,7 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM,
-			 FunctionAnalysisManager *FAM);
+                         ModuleAnalysisManager *MAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -625,8 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM,
-			 FunctionAnalysisManager *FAM);
+                         ModuleAnalysisManager *MAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 7ae23ef4e7df4..5e8cd12fe040b 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -273,7 +273,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
                               Conf.VerifyEach);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 
   RegisterPassPlugins(Conf.PassPlugins, PB);
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 8db92e70d5a96..5b333cda31fb8 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -241,7 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PipelineTuningOptions PTO;
   PTO.LoopVectorization = true;
   PTO.SLPVectorization = true;
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index f0e1abb8cebc4..933fe89e53a94 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -76,7 +76,7 @@ static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   // Run the pipeline.
   if (Fun) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 905ba667fac7a..4e8abc43572d7 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,7 +40,6 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
-//#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -57,8 +56,6 @@ static cl::opt<bool> VerifyAnalysisInvalidation("verify-analysis-invalidation",
 #endif
 );
 
-static cl::opt<bool> VerifyTargetEach("verify-tgt-each");
-
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
 // of this option.  Note that this option has no effect without -print-changed.
@@ -1452,10 +1449,9 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 }
 
 void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                              ModuleAnalysisManager *MAM,
-					      FunctionAnalysisManager *FAM) {
+                                              ModuleAnalysisManager *MAM) {
   PIC.registerAfterPassCallback(
-      [this, MAM, FAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+      [this, MAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
         const auto *F = unwrapIR<Function>(IR);
@@ -1472,15 +1468,6 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
             report_fatal_error(formatv("Broken function found after pass "
                                        "\"{0}\", compilation aborted!",
                                        P));
-
-          if (VerifyTargetEach && FAM) {
-            //TargetVerify TV(const_cast<Module*>(F->getParent()));
-            //TV.run(*const_cast<Function*>(F), *FAM);
-	    /*if (!TV.IsValid)
-              report_fatal_error(formatv("Broken function found after pass "
-                                         "\"{0}\", compilation aborted!",
-                                         P));*/
-          }
         } else {
           const auto *M = unwrapIR<Module>(IR);
           if (!M) {
@@ -2520,7 +2507,7 @@ void PrintCrashIRInstrumentation::registerCallbacks(
 }
 
 void StandardInstrumentations::registerCallbacks(
-    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM, FunctionAnalysisManager *FAM) {
+    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM) {
   PrintIR.registerCallbacks(PIC);
   PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
@@ -2529,7 +2516,7 @@ void StandardInstrumentations::registerCallbacks(
   PrintChangedIR.registerCallbacks(PIC);
   PseudoProbeVerification.registerCallbacks(PIC);
   if (VerifyEach)
-    Verify.registerCallbacks(PIC, MAM, FAM);
+    Verify.registerCallbacks(PIC, MAM);
   PrintChangedDiff.registerCallbacks(PIC);
   WebsiteChangeReporter.registerCallbacks(PIC);
   ChangeTester.registerCallbacks(PIC);
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 4b95977a10c5f..863a555798dab 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -117,8 +117,6 @@ int llvm::compileModuleWithNewPM(
                               VK == VerifierKind::EachPass);
   registerCodeGenCallback(PIC, *Target);
 
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
   MachineFunctionAnalysisManager MFAM;
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
@@ -133,11 +131,14 @@ int llvm::compileModuleWithNewPM(
   if (VerifyTarget)
     PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
 
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+
   if (!PassPipeline.empty()) {
     // Construct a custom pass pipeline that starts after instruction
     // selection.
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index d832dcdff4ad0..50f4e56bb6af6 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -148,7 +148,7 @@ int main(int argc, char **argv) {
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   Triple TT(M->getTargetTriple());
   if (!NoLint)
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index a8977d80bdf44..7d168a6ceb17c 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -423,7 +423,7 @@ bool llvm::runPassPipeline(
   PrintPassOpts.SkipAnalyses = DebugPM == DebugLogging::Quiet;
   StandardInstrumentations SI(M.getContext(), DebugPM != DebugLogging::None,
                               VK == VerifierKind::EachPass, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   DebugifyEachInstrumentation Debugify;
   DebugifyStatsMap DIStatsMap;
   DebugInfoPerPass DebugInfoBeforePass;
diff --git a/llvm/unittests/IR/PassManagerTest.cpp b/llvm/unittests/IR/PassManagerTest.cpp
index bb4db6120035f..a6487169224c2 100644
--- a/llvm/unittests/IR/PassManagerTest.cpp
+++ b/llvm/unittests/IR/PassManagerTest.cpp
@@ -828,7 +828,7 @@ TEST_F(PassManagerTest, FunctionPassCFGChecker) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -877,7 +877,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerInvalidateAnalysis) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -945,7 +945,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerWrapped) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });

>From c680041d5134d6edd47e8dec65d4986feb63fb7d Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 14:44:28 -0400
Subject: [PATCH 22/33] Remove references to registry

---
 llvm/include/llvm/Passes/PassBuilder.h        | 21 -------------------
 .../llvm/Passes/StandardInstrumentations.h    |  2 +-
 .../llvm/Passes/TargetPassRegistry.inc        | 12 -----------
 llvm/lib/Passes/PassBuilder.cpp               |  7 -------
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 11 ----------
 llvm/tools/llc/NewPMDriver.cpp                |  5 -----
 llvm/tools/llc/llc.cpp                        |  2 --
 7 files changed, 1 insertion(+), 59 deletions(-)

diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index c134f2d0c1112..f13b5c678a894 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -173,13 +173,6 @@ class PassBuilder {
   /// additional analyses.
   LLVM_ABI void registerLoopAnalyses(LoopAnalysisManager &LAM);
 
-  /// Registers all available verifier passes.
-  ///
-  /// This is an interface that can be used to populate a
-  /// \c ModuleAnalysisManager with all registered loop analyses. Callers can
-  /// still manually register any additional analyses.
-  void registerVerifierPasses(ModulePassManager &PM, FunctionPassManager &);
-
   /// Registers all available machine function analysis passes.
   ///
   /// This is an interface that can be used to populate a \c
@@ -583,15 +576,6 @@ class PassBuilder {
   }
   /// @}}
 
-  /// Register a callback for parsing an Verifier Name to populate
-  /// the given managers.
-  void registerVerifierCallback(
-      const std::function<bool(ModulePassManager &MPM)> &C,
-      const std::function<bool(FunctionPassManager &MPM)> &CF) {
-    VerifierCallbacks.push_back(C);
-    FnVerifierCallbacks.push_back(CF);
-  }
-
   /// {{@ Register pipeline parsing callbacks with this pass builder instance.
   /// Using these callbacks, callers can parse both a single pass name, as well
   /// as entire sub-pipelines, and populate the PassManager instance
@@ -867,11 +851,6 @@ class PassBuilder {
   // Callbacks to parse `filter` parameter in register allocation passes
   SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
-  // Verifier callbacks
-  SmallVector<std::function<bool(ModulePassManager &)>, 2>
-      VerifierCallbacks;
-  SmallVector<std::function<bool(FunctionPassManager &)>, 2>
-      FnVerifierCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index df9d8de8a025a..d7c1bbf04c328 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -624,7 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc
index 2d04b874cf360..521913cb25a4a 100644
--- a/llvm/include/llvm/Passes/TargetPassRegistry.inc
+++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc
@@ -151,18 +151,6 @@ PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM,
   return false;
 });
 
-PB.registerVerifierCallback([](ModulePassManager &PM) {
-#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS) PM.addPass(CREATE_PASS)
-#include GET_PASS_REGISTRY
-#undef VERIFIER_MODULE_ANALYSIS
-  return false;
-}, [](FunctionPassManager &FPM) {
-#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS) FPM.addPass(CREATE_PASS)
-#include GET_PASS_REGISTRY
-#undef VERIFIER_FUNCTION_ANALYSIS
-  return false;
-});
-
 #undef ADD_PASS
 #undef ADD_PASS_WITH_PARAMS
 
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 561996d0c8980..4603eaff8ade9 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -576,13 +576,6 @@ void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     C(LAM);
 }
 
-void PassBuilder::registerVerifierPasses(ModulePassManager &MPM, FunctionPassManager &FPM) {
-  for (auto &C : VerifierCallbacks)
-    C(MPM);
-  for (auto &C : FnVerifierCallbacks)
-    C(FPM);
-}
-
 static std::optional<std::pair<bool, bool>>
 parseFunctionPipelineName(StringRef Name) {
   std::pair<bool, bool> Params;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 652cd23f62443..13453963eec6d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -82,17 +82,6 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
-#ifndef VERIFIER_MODULE_ANALYSIS
-#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS)
-#endif
-#ifndef VERIFIER_FUNCTION_ANALYSIS
-#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
-#endif
-VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
-#undef VERIFIER_MODULE_ANALYSIS
-#undef VERIFIER_FUNCTION_ANALYSIS
-
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 863a555798dab..fa82689ecf9ae 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -57,9 +57,6 @@ static cl::opt<bool>
     DebugPM("debug-pass-manager", cl::Hidden,
             cl::desc("Print pass management debugging information"));
 
-static cl::opt<bool> VerifyTarget("verify-tgt-new-pm",
-		                  cl::desc("Verify the target"));
-
 bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
   DiagnosticHandler::handleDiagnostics(DI);
   if (DI.getKind() == llvm::DK_SrcMgr) {
@@ -128,8 +125,6 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
-  if (VerifyTarget)
-    PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM);
 
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 2e9e4837fe467..140459ba2de21 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -209,8 +209,6 @@ static cl::opt<std::string> PassPipeline(
 static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
                                cl::desc("Alias for -passes"));
 
-static cl::opt<bool> VerifyTarget("verify-tgt", cl::desc("Verify the target"));
-
 namespace {
 
 std::vector<std::string> &getRunPassNames() {

>From 3f5634a532856668a42f44ea04d0ffcc862c96d4 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 19:05:53 -0400
Subject: [PATCH 23/33] Remove int check

---
 .../lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index c4d303bee6ef8..2ca0bbeb57653 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -25,7 +25,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-//#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -46,15 +45,6 @@ using namespace llvm;
 
 namespace llvm {
 
-static bool IsValidInt(const Type *Ty) {
-  return Ty->isIntegerTy(1) ||
-         Ty->isIntegerTy(8) ||
-         Ty->isIntegerTy(16) ||
-         Ty->isIntegerTy(32) ||
-         Ty->isIntegerTy(64) ||
-         Ty->isIntegerTy(128);
-}
-
 static bool isShader(CallingConv::ID CC) {
   switch(CC) {
     case CallingConv::AMDGPU_VS:
@@ -81,14 +71,6 @@ bool AMDGPUTargetVerify::run(Function &F) {
 
     for (auto &I : BB) {
 
-      // Ensure integral types are valid: i8, i16, i32, i64, i128
-      if (I.getType()->isIntegerTy())
-        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
-      for (unsigned i = 0; i < I.getNumOperands(); ++i)
-        if (I.getOperand(i)->getType()->isIntegerTy())
-          Check(IsValidInt(I.getOperand(i)->getType()),
-                "Int type is invalid.", I.getOperand(i));
-
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
         // Ensure no kernel to kernel calls.

>From 6007895007af8925e32b709009c0d6131141c282 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 21:56:28 -0400
Subject: [PATCH 24/33] Remove modifications to Lint/Verifier.

---
 llvm/lib/Analysis/Lint.cpp |  4 +---
 llvm/lib/IR/Verifier.cpp   | 20 ++++----------------
 2 files changed, 5 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index c8627e6db0692..1168005f48c0e 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -735,11 +735,9 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) {
   Lint L(Mod, DL, AA, AC, DT, TLI);
   L.visit(F);
   dbgs() << L.MessagesStr.str();
-  if (AbortOnError && !L.MessagesStr.str().empty()) {
+  if (AbortOnError && !L.MessagesStr.str().empty())
     report_fatal_error(
         "linter found errors, aborting. (enabled by abort-on-error)", false);
-    return PreservedAnalyses::none();
-  }
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 439823ba8b682..1f1041b259736 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -135,10 +135,6 @@ static cl::opt<bool> VerifyNoAliasScopeDomination(
     cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
              "scopes are not dominating"));
 
-static cl::opt<bool>
-    VerifyAbortOnError("verifier-abort-on-error", cl::init(false),
-                       cl::desc("In the Verifier pass, abort on errors."));
-
 namespace llvm {
 
 struct VerifierSupport {
@@ -7869,24 +7865,16 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
-  if (Res.IRBroken || Res.DebugInfoBroken) {
-    //M.IsValid = false;
-    if (VerifyAbortOnError && FatalErrors)
-      report_fatal_error("Broken module found, compilation aborted!");
-    return PreservedAnalyses::none();
-  }
+  if (FatalErrors && (Res.IRBroken || Res.DebugInfoBroken))
+    report_fatal_error("Broken module found, compilation aborted!");
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
-  if (res.IRBroken) {
-    //F.getParent()->IsValid = false;
-    if (VerifyAbortOnError && FatalErrors)
-      report_fatal_error("Broken function found, compilation aborted!");
-    return PreservedAnalyses::none();
-  }
+  if (res.IRBroken && FatalErrors)
+    report_fatal_error("Broken function found, compilation aborted!");
 
   return PreservedAnalyses::all();
 }

>From 756c3ebee47f59f69a267da6079edafce4f595ca Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 22:21:47 -0400
Subject: [PATCH 25/33] Remove llvm-tgt-verify tool.

---
 llvm/test/CMakeLists.txt                      |   1 -
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        |  45 -----
 llvm/test/lit.cfg.py                          |   1 -
 llvm/tools/llvm-tgt-verify/CMakeLists.txt     |  34 ----
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 171 ------------------
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |   1 -
 .../llvm/tools/llvm-tgt-verify/BUILD.gn       |  25 ---
 7 files changed, 278 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify.ll
 delete mode 100644 llvm/tools/llvm-tgt-verify/CMakeLists.txt
 delete mode 100644 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
 delete mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn

diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index a2bb112db8746..2a6135da9a61e 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -136,7 +136,6 @@ set(LLVM_TEST_DEPENDS
           llvm-strip
           llvm-symbolizer
           llvm-tblgen
-          llvm-tgt-verify
           llvm-readtapi
           llvm-tli-checker
           llvm-undname
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
deleted file mode 100644
index 62b220d7d9f49..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
-
-define amdgpu_cs i32 @shader() {
-; CHECK: Shaders must return void
-  ret i32 0
-}
-
-define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
-; CHECK: Undefined behavior: Write to memory in const addrspace
-; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
-  %r = add i32 %a, %b
-  store i32 %r, ptr addrspace(4) %out
-  ret void
-}
-
-define amdgpu_kernel void @kernel_callee(ptr %x) {
-  ret void
-}
-
-define amdgpu_kernel void @kernel_caller(ptr %x) {
-; CHECK: A kernel may not call a kernel
-; CHECK-NEXT: ptr @kernel_caller
-  call amdgpu_kernel void @kernel_callee(ptr %x)
-  ret void
-}
-
-
-; Function Attrs: nounwind
-define i65 @invalid_type(i65 %x) #0 {
-; CHECK: Int type is invalid.
-; CHECK-NEXT: %tmp2 = ashr i65 %x, 64
-entry:
-  %tmp2 = ashr i65 %x, 64
-  ret i65 %tmp2
-}
-
-declare void @llvm.amdgcn.cs.chain.v3i32(ptr, i32, <3 x i32>, <3 x i32>, i32, ...)
-declare amdgpu_cs_chain void @chain_callee(<3 x i32> inreg, <3 x i32>)
-
-define amdgpu_cs void @no_unreachable(<3 x i32> inreg %a, <3 x i32> %b) {
-; CHECK: llvm.amdgcn.cs.chain must be followed by unreachable
-; CHECK-NEXT: call void (ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.v3i32.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
-  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
-  ret void
-}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index bf06e80dc7ae5..bd6e37c848d8c 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -228,7 +228,6 @@ def get_asan_rtlib():
         "llvm-strings",
         "llvm-strip",
         "llvm-tblgen",
-        "llvm-tgt-verify",
         "llvm-readtapi",
         "llvm-undname",
         "llvm-windres",
diff --git a/llvm/tools/llvm-tgt-verify/CMakeLists.txt b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
deleted file mode 100644
index fe47c85e6cdce..0000000000000
--- a/llvm/tools/llvm-tgt-verify/CMakeLists.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  AllTargetsAsmParsers
-  AllTargetsCodeGens
-  AllTargetsDescs
-  AllTargetsInfos
-  Analysis
-  AsmPrinter
-  CodeGen
-  CodeGenTypes
-  Core
-  IRPrinter
-  IRReader
-  MC
-  MIRParser
-  Passes
-  Remarks
-  ScalarOpts
-  SelectionDAG
-  Support
-  Target
-  TargetParser
-  TransformUtils
-  Vectorize
-  )
-
-add_llvm_tool(llvm-tgt-verify
-  llvm-tgt-verify.cpp
-
-  DEPENDS
-  intrinsics_gen
-  SUPPORT_PLUGINS
-  )
-
-export_executable_symbols_for_plugins(llc)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
deleted file mode 100644
index 50f4e56bb6af6..0000000000000
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ /dev/null
@@ -1,171 +0,0 @@
-//===--- llvm-tgt-verify.cpp - Target Verifier ----------------- ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Tool to verify a target.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/InitializePasses.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/Lint.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Bitcode/BitcodeReader.h"
-#include "llvm/Bitcode/BitcodeWriter.h"
-#include "llvm/CodeGen/CommandFlags.h"
-#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Verifier.h"
-#include "llvm/IRReader/IRReader.h"
-#include "llvm/Passes/PassBuilder.h"
-#include "llvm/Passes/StandardInstrumentations.h"
-#include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/InitLLVM.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/TargetSelect.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetVerifier.h"
-
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
-
-#define DEBUG_TYPE "isel-fuzzer"
-
-using namespace llvm;
-
-static codegen::RegisterCodeGenFlags CGF;
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-static cl::opt<bool>
-    StacktraceAbort("stacktrace-abort",
-        cl::desc("Turn on stacktrace"), cl::init(false));
-
-static cl::opt<bool>
-    NoLint("no-lint",
-        cl::desc("Turn off Lint"), cl::init(false));
-
-static cl::opt<bool>
-    NoVerify("no-verifier",
-        cl::desc("Turn off Verifier"), cl::init(false));
-
-static cl::opt<char>
-    OptLevel("O",
-             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
-                      "(default = '-O2')"),
-             cl::Prefix, cl::init('2'));
-
-static cl::opt<std::string>
-    TargetTriple("mtriple", cl::desc("Override target triple for module"));
-
-static std::unique_ptr<TargetMachine> TM;
-
-static void handleLLVMFatalError(void *, const char *Message, bool) {
-  if (StacktraceAbort) {
-    dbgs() << "LLVM ERROR: " << Message << "\n"
-           << "Aborting.\n";
-    abort();
-  }
-}
-
-int main(int argc, char **argv) {
-  StringRef ExecName = argv[0];
-  InitLLVM X(argc, argv);
-
-  InitializeAllTargets();
-  InitializeAllTargetMCs();
-  InitializeAllAsmPrinters();
-  InitializeAllAsmParsers();
-
-  PassRegistry *Registry = PassRegistry::getPassRegistry();
-  initializeCore(*Registry);
-  initializeCodeGen(*Registry);
-  initializeAnalysis(*Registry);
-  initializeTarget(*Registry);
-
-  cl::ParseCommandLineOptions(argc, argv);
-
-  if (TargetTriple.empty()) {
-    errs() << ExecName << ": -mtriple must be specified\n";
-    exit(1);
-  }
-
-  CodeGenOptLevel OLvl;
-  if (auto Level = CodeGenOpt::parseLevel(OptLevel)) {
-    OLvl = *Level;
-  } else {
-    errs() << ExecName << ": invalid optimization level.\n";
-    return 1;
-  }
-  ExitOnError ExitOnErr(std::string(ExecName) + ": error:");
-  TM = ExitOnErr(codegen::createTargetMachineForTriple(
-      Triple::normalize(TargetTriple), OLvl));
-  assert(TM && "Could not allocate target machine!");
-
-  // Make sure we print the summary and the current unit when LLVM errors out.
-  install_fatal_error_handler(handleLLVMFatalError, nullptr);
-
-  LLVMContext Context;
-  SMDiagnostic Err;
-  std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
-  if (!M) {
-    errs() << "Invalid mod\n";
-    return 1;
-  }
-  auto S = Triple::normalize(TargetTriple);
-  M->setTargetTriple(Triple(S));
-
-  PassInstrumentationCallbacks PIC;
-  StandardInstrumentations SI(Context, false/*debug PM*/,
-                              false);
-  registerCodeGenCallback(PIC, *TM);
-
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
-  //TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
-
-  MachineFunctionAnalysisManager MFAM;
-  LoopAnalysisManager LAM;
-  FunctionAnalysisManager FAM;
-  CGSCCAnalysisManager CGAM;
-  ModuleAnalysisManager MAM;
-  PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
-  PB.registerModuleAnalyses(MAM);
-  //PB.registerVerifierPasses(MPM, FPM);
-  PB.registerCGSCCAnalyses(CGAM);
-  PB.registerFunctionAnalyses(FAM);
-  PB.registerLoopAnalyses(LAM);
-  PB.registerMachineFunctionAnalyses(MFAM);
-  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-
-  SI.registerCallbacks(PIC, &MAM);
-
-  Triple TT(M->getTargetTriple());
-  if (!NoLint)
-    FPM.addPass(LintPass(false));
-  if (!NoVerify)
-    MPM.addPass(VerifierPass());
-  if (TT.isAMDGPU())
-    FPM.addPass(AMDGPUTargetVerifierPass());
-  else if (false) {} // ...
-  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
-
-  auto PA = MPM.run(*M, MAM);
-  {
-    auto PAC = PA.getChecker<VerifierAnalysis>();
-    if (!PAC.preserved())
-      return 1;
-  }
-
-  return 0;
-}
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 36cf956545caf..946b63f8a54fb 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -320,7 +320,6 @@ group("test") {
     "//llvm/tools/llvm-strings",
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-tli-checker",
-    "//llvm/tools/llvm-tgt-verify",
     "//llvm/tools/llvm-undname",
     "//llvm/tools/llvm-xray",
     "//llvm/tools/lto",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
deleted file mode 100644
index b751bafc5052c..0000000000000
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
+++ /dev/null
@@ -1,25 +0,0 @@
-import("//llvm/utils/TableGen/tablegen.gni")
-
-tgtverifier("llvm-tgt-verify") {
-  deps = [
-    "//llvm/lib/Analysis",
-    "//llvm/lib/AsmPrinter",
-    "//llvm/lib/CodeGen",
-    "//llvm/lib/CodeGenTypes",
-    "//llvm/lib/Core",
-    "//llvm/lib/IRPrinter",
-    "//llvm/lib/IRReader",
-    "//llvm/lib/MC",
-    "//llvm/lib/MIRParser",
-    "//llvm/lib/Passes",
-    "//llvm/lib/Remarks",
-    "//llvm/lib/ScalarOpts",
-    "//llvm/lib/SelectionDAG",
-    "//llvm/lib/Support",
-    "//llvm/lib/Target",
-    "//llvm/lib/TargetParser",
-    "//llvm/lib/TransformUtils",
-    "//llvm/lib/Vectorize",
-  ]
-  sources = [ "llvm-tgt-verify.cpp" ]
-}

>From 3f1d72ff01d0303c0fc17d9a6c40e46661bf1fb0 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 22:23:15 -0400
Subject: [PATCH 26/33] Remove TargetVerifier.cpp

---
 llvm/lib/Passes/CMakeLists.txt     |  1 -
 llvm/lib/Target/CMakeLists.txt     |  2 --
 llvm/lib/Target/TargetVerifier.cpp | 36 ------------------------------
 3 files changed, 39 deletions(-)
 delete mode 100644 llvm/lib/Target/TargetVerifier.cpp

diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index b8d25b15186e8..91c8c4f67074d 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,7 +30,6 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
-  #TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index f2a5d545ce84f..9472288229cac 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -7,8 +7,6 @@ add_llvm_component_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
-  TargetVerifier.cpp
-  AMDGPU/AMDGPUTargetVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Target
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
deleted file mode 100644
index c63ae2a2c5daf..0000000000000
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-//===-- TargetVerifier.cpp - LLVM IR Target Verifier ----------------*- C++ -*-===//
-////
-///// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-///// See https://llvm.org/LICENSE.txt for license information.
-///// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-/////
-/////===----------------------------------------------------------------------===//
-/////
-///// This file defines target verifier interfaces that can be used for some
-///// validation of input to the system, and for checking that transformations
-///// haven't done something bad. In contrast to the Verifier or Lint, the
-///// TargetVerifier looks for constructions invalid to a particular target
-///// machine.
-/////
-///// To see what specifically is checked, look at TargetVerifier.cpp or an
-///// individual backend's TargetVerifier.
-/////
-/////===----------------------------------------------------------------------===//
-
-#include "llvm/Target/TargetVerifier.h"
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
-
-#include "llvm/InitializePasses.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/IR/Dominators.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Value.h"
-
-namespace llvm {
-
-} // namespace llvm

>From 1c721cf05616091cf3c42e4c244400a393abe755 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 1 May 2025 03:02:30 -0400
Subject: [PATCH 27/33] clang-format

---
 llvm/include/llvm/Target/TargetVerifier.h     | 10 +-
 .../TargetVerify/AMDGPUTargetVerifier.h       | 44 ++++-----
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 98 ++++++++++---------
 3 files changed, 77 insertions(+), 75 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 1d12eb55bbf0a..3f8c710a88768 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -1,4 +1,4 @@
-//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier ---*- C++ -*-===//
+//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier --*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -20,8 +20,8 @@
 #ifndef LLVM_TARGET_VERIFIER_H
 #define LLVM_TARGET_VERIFIER_H
 
-#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
@@ -59,10 +59,11 @@ class TargetVerify {
   /// This calls the Message-only version so that the above is easier to set
   /// a breakpoint on.
   template <typename T1, typename... Ts>
-  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) {
+  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
     CheckFailed(Message);
     WriteValues({V1, Vs...});
   }
+
 public:
   Module *Mod;
   Triple TT;
@@ -73,8 +74,7 @@ class TargetVerify {
   bool IsValid = true;
 
   TargetVerify(Module *Mod)
-      : Mod(Mod), TT(Mod->getTargetTriple()),
-        MessagesStr(Messages) {}
+      : Mod(Mod), TT(Mod->getTargetTriple()), MessagesStr(Messages) {}
 
   virtual bool run(Function &F) = 0;
 };
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 49bcbc8849e3c..5b8d9ec259b63 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -1,29 +1,29 @@
-//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU ---*- C++ -*-===//
-////
-//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//// See https://llvm.org/LICENSE.txt for license information.
-//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-////
-////===----------------------------------------------------------------------===//
-////
-//// This file defines target verifier interfaces that can be used for some
-//// validation of input to the system, and for checking that transformations
-//// haven't done something bad. In contrast to the Verifier or Lint, the
-//// TargetVerifier looks for constructions invalid to a particular target
-//// machine.
-////
-//// To see what specifically is checked, look at an individual backend's
-//// TargetVerifier.
-////
-////===----------------------------------------------------------------------===//
+//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU -- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
 
 #ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
 #define LLVM_AMDGPU_TARGET_VERIFIER_H
 
 #include "llvm/Target/TargetVerifier.h"
 
-#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/IR/Dominators.h"
 
 namespace llvm {
@@ -43,10 +43,10 @@ class AMDGPUTargetVerify : public TargetVerify {
   PostDominatorTree *PDT = nullptr;
   UniformityInfo *UA = nullptr;
 
-  AMDGPUTargetVerify(Module *Mod)
-    : TargetVerify(Mod), Mod(Mod) {}
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod), Mod(Mod) {}
 
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
+                     UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   bool run(Function &F) override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 2ca0bbeb57653..eb22eb2177f7f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -1,34 +1,34 @@
-//===-- AMDGPUTargetVerifier.cpp - AMDGPU -------------------------*- C++ -*-===//
-////
-//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//// See https://llvm.org/LICENSE.txt for license information.
-//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-////
-////===----------------------------------------------------------------------===//
-////
-//// This file defines target verifier interfaces that can be used for some
-//// validation of input to the system, and for checking that transformations
-//// haven't done something bad. In contrast to the Verifier or Lint, the
-//// TargetVerifier looks for constructions invalid to a particular target
-//// machine.
-////
-//// To see what specifically is checked, look at an individual backend's
-//// TargetVerifier.
-////
-////===----------------------------------------------------------------------===//
+//===-- AMDGPUTargetVerifier.cpp - AMDGPU -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
 
-#include "AMDGPU.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+#include "AMDGPU.h"
 
-#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Debug.h"
 
 #include "llvm/Support/raw_ostream.h"
 
@@ -39,53 +39,52 @@ using namespace llvm;
   do {                                                                         \
     if (!(C)) {                                                                \
       TargetVerify::CheckFailed(__VA_ARGS__);                                  \
-      return false;                                                                  \
     }                                                                          \
   } while (false)
 
 namespace llvm {
 
 static bool isShader(CallingConv::ID CC) {
-  switch(CC) {
-    case CallingConv::AMDGPU_VS:
-    case CallingConv::AMDGPU_LS:
-    case CallingConv::AMDGPU_HS:
-    case CallingConv::AMDGPU_ES:
-    case CallingConv::AMDGPU_GS:
-    case CallingConv::AMDGPU_PS:
-    case CallingConv::AMDGPU_CS_Chain:
-    case CallingConv::AMDGPU_CS_ChainPreserve:
-    case CallingConv::AMDGPU_CS:
-      return true;
-    default:
-      return false;
+  switch (CC) {
+  case CallingConv::AMDGPU_VS:
+  case CallingConv::AMDGPU_LS:
+  case CallingConv::AMDGPU_HS:
+  case CallingConv::AMDGPU_ES:
+  case CallingConv::AMDGPU_GS:
+  case CallingConv::AMDGPU_PS:
+  case CallingConv::AMDGPU_CS_Chain:
+  case CallingConv::AMDGPU_CS_ChainPreserve:
+  case CallingConv::AMDGPU_CS:
+    return true;
+  default:
+    return false;
   }
 }
 
 bool AMDGPUTargetVerify::run(Function &F) {
   // Ensure shader calling convention returns void
   if (isShader(F.getCallingConv()))
-    Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
+    Check(F.getReturnType() == Type::getVoidTy(F.getContext()),
+          "Shaders must return void");
 
   for (auto &BB : F) {
 
     for (auto &I : BB) {
 
-      if (auto *CI = dyn_cast<CallInst>(&I))
-      {
+      if (auto *CI = dyn_cast<CallInst>(&I)) {
         // Ensure no kernel to kernel calls.
         CallingConv::ID CalleeCC = CI->getCallingConv();
-        if (CalleeCC == CallingConv::AMDGPU_KERNEL)
-        {
-          CallingConv::ID CallerCC = CI->getParent()->getParent()->getCallingConv();
+        if (CalleeCC == CallingConv::AMDGPU_KERNEL) {
+          CallingConv::ID CallerCC =
+              CI->getParent()->getParent()->getCallingConv();
           Check(CallerCC != CallingConv::AMDGPU_KERNEL,
-            "A kernel may not call a kernel", CI->getParent()->getParent());
+                "A kernel may not call a kernel", CI->getParent()->getParent());
         }
 
         // Ensure chain intrinsics are followed by unreachables.
         if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
           Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
-            "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
+                "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
     }
   }
@@ -98,7 +97,8 @@ bool AMDGPUTargetVerify::run(Function &F) {
   return true;
 }
 
-PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F,
+                                                FunctionAnalysisManager &AM) {
   auto *Mod = F.getParent();
 
   auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
@@ -122,9 +122,10 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   std::unique_ptr<AMDGPUTargetVerify> TV;
   bool FatalErrors = false;
 
-  AMDGPUTargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
-    FatalErrors(FatalErrors) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
+      : FunctionPass(ID), FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(
+        *PassRegistry::getPassRegistry());
   }
 
   bool doInitialization(Module &M) override {
@@ -167,4 +168,5 @@ FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
   return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
 }
 } // namespace llvm
-INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier",
+                "AMDGPU Target Verifier", false, false)

>From f59b0f27beb766516186dcab8bde8463734ee64b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 1 May 2025 03:49:20 -0400
Subject: [PATCH 28/33] Add VerifyTarget option

---
 .../llvm/Target/TargetVerify/AMDGPUTargetVerifier.h      | 6 ++----
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp           | 9 +++++++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 5b8d9ec259b63..8ed7dd7ea2f69 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -37,17 +37,15 @@ class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 
 class AMDGPUTargetVerify : public TargetVerify {
 public:
-  Module *Mod;
-
   DominatorTree *DT = nullptr;
   PostDominatorTree *PDT = nullptr;
   UniformityInfo *UA = nullptr;
 
-  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod), Mod(Mod) {}
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
 
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
                      UniformityInfo *UA)
-    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   bool run(Function &F) override;
 };
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8b0b8f6df740e..be8bde1c1c0e0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -482,6 +482,9 @@ static cl::opt<bool> HasClosedWorldAssumption(
     cl::desc("Whether has closed-world assumption at link time"),
     cl::init(false), cl::Hidden);
 
+static cl::opt<bool> VerifyTarget("verify-tgt",
+                                  cl::desc("Enable the target verifier"));
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
@@ -1411,7 +1414,8 @@ bool AMDGPUPassConfig::addGCPasses() {
 //===----------------------------------------------------------------------===//
 
 bool GCNPassConfig::addPreISel() {
-  addPass(createAMDGPUTargetVerifierLegacyPass(false));
+  if (VerifyTarget)
+    addPass(createAMDGPUTargetVerifierLegacyPass(false));
   AMDGPUPassConfig::addPreISel();
 
   if (TM->getOptLevel() > CodeGenOptLevel::None)
@@ -2011,7 +2015,8 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
 }
 
 void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
-  addPass(AMDGPUTargetVerifierPass());
+  if (VerifyTarget)
+    addPass(AMDGPUTargetVerifierPass());
 
   if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
     addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));

>From f0f12f552a0a48eda3e1dc0b7db88e605067f945 Mon Sep 17 00:00:00 2001
From: Joey F <jofernau at amd.com>
Date: Wed, 7 May 2025 21:59:43 -0400
Subject: [PATCH 29/33] Remove AMDGPUTargetVerifier.h

---
 .../TargetVerify/AMDGPUTargetVerifier.h       | 55 -------------------
 llvm/lib/Target/AMDGPU/AMDGPU.h               |  5 ++
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  1 -
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 16 +++++-
 4 files changed, 20 insertions(+), 57 deletions(-)
 delete mode 100644 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h

diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
deleted file mode 100644
index 8ed7dd7ea2f69..0000000000000
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU -- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines target verifier interfaces that can be used for some
-// validation of input to the system, and for checking that transformations
-// haven't done something bad. In contrast to the Verifier or Lint, the
-// TargetVerifier looks for constructions invalid to a particular target
-// machine.
-//
-// To see what specifically is checked, look at an individual backend's
-// TargetVerifier.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
-#define LLVM_AMDGPU_TARGET_VERIFIER_H
-
-#include "llvm/Target/TargetVerifier.h"
-
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/IR/Dominators.h"
-
-namespace llvm {
-
-class Function;
-
-class AMDGPUTargetVerifierPass : public TargetVerifierPass {
-public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
-};
-
-class AMDGPUTargetVerify : public TargetVerify {
-public:
-  DominatorTree *DT = nullptr;
-  PostDominatorTree *PDT = nullptr;
-  UniformityInfo *UA = nullptr;
-
-  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
-                     UniformityInfo *UA)
-      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
-  bool run(Function &F) override;
-};
-
-} // namespace llvm
-
-#endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 26fb0a967fd76..f5f083ca066a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -15,6 +15,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Target/TargetVerifier.h"
 
 namespace llvm {
 
@@ -551,6 +552,10 @@ extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
 void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+class AMDGPUTargetVerifierPass : public TargetVerifierPass {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
+};
 
 namespace AMDGPU {
 enum TargetIndex {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index be8bde1c1c0e0..2009459aac329 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,7 +90,6 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/FormatVariadic.h"
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index eb22eb2177f7f..cc96164a1a558 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -17,7 +17,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "AMDGPU.h"
 
 #include "llvm/Analysis/PostDominators.h"
@@ -44,6 +43,21 @@ using namespace llvm;
 
 namespace llvm {
 
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  DominatorTree *DT = nullptr;
+  PostDominatorTree *PDT = nullptr;
+  UniformityInfo *UA = nullptr;
+
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
+                     UniformityInfo *UA)
+      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  bool run(Function &F) override;
+};
+
 static bool isShader(CallingConv::ID CC) {
   switch (CC) {
   case CallingConv::AMDGPU_VS:

>From 88a6d6b0faeff37cd98be19fd88632f9546720be Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 7 May 2025 22:47:55 -0400
Subject: [PATCH 30/33] Remove analyses.

---
 .../lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index cc96164a1a558..d07b5fb668c14 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -19,9 +19,6 @@
 
 #include "AMDGPU.h"
 
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -45,16 +42,7 @@ namespace llvm {
 
 class AMDGPUTargetVerify : public TargetVerify {
 public:
-  DominatorTree *DT = nullptr;
-  PostDominatorTree *PDT = nullptr;
-  UniformityInfo *UA = nullptr;
-
   AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
-                     UniformityInfo *UA)
-      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
   bool run(Function &F) override;
 };
 
@@ -115,11 +103,7 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F,
                                                 FunctionAnalysisManager &AM) {
   auto *Mod = F.getParent();
 
-  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  AMDGPUTargetVerify TV(Mod);
   TV.run(F);
 
   dbgs() << TV.MessagesStr.str();

>From 0d99a659957a043ec7dfa4762a1f1c5bd34e50b0 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 8 May 2025 12:59:01 -0400
Subject: [PATCH 31/33] function name style, TargetVerifier comment, option
 name generality

---
 llvm/include/llvm/Target/TargetVerifier.h       | 14 +++++++-------
 llvm/lib/Target/AMDGPU/AMDGPU.h                 |  3 +--
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp |  2 +-
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll |  2 +-
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll |  2 +-
 6 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 3f8c710a88768..eff846694ac10 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -12,8 +12,8 @@
 // TargetVerifier looks for constructions invalid to a particular target
 // machine.
 //
-// To see what specifically is checked, look at TargetVerifier.cpp or an
-// individual backend's TargetVerifier.
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
 //
 //===----------------------------------------------------------------------===//
 
@@ -35,7 +35,7 @@ class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 
 class TargetVerify {
 protected:
-  void WriteValues(ArrayRef<const Value *> Vs) {
+  void writeValues(ArrayRef<const Value *> Vs) {
     for (const Value *V : Vs) {
       if (!V)
         continue;
@@ -52,16 +52,16 @@ class TargetVerify {
   ///
   /// This provides a nice place to put a breakpoint if you want to see why
   /// something is not correct.
-  void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
+  void checkFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
 
   /// A check failed (with values to print).
   ///
   /// This calls the Message-only version so that the above is easier to set
   /// a breakpoint on.
   template <typename T1, typename... Ts>
-  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
-    CheckFailed(Message);
-    WriteValues({V1, Vs...});
+  void checkFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
+    checkFailed(Message);
+    writeValues({V1, Vs...});
   }
 
 public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index f5f083ca066a2..af09a7e4547f3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -552,8 +552,7 @@ extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
 void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
-class AMDGPUTargetVerifierPass : public TargetVerifierPass {
-public:
+struct AMDGPUTargetVerifierPass : public TargetVerifierPass {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
 };
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 2009459aac329..370fde6cce6ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -481,7 +481,7 @@ static cl::opt<bool> HasClosedWorldAssumption(
     cl::desc("Whether has closed-world assumption at link time"),
     cl::init(false), cl::Hidden);
 
-static cl::opt<bool> VerifyTarget("verify-tgt",
+static cl::opt<bool> VerifyTarget("amdgpu-verify-tgt",
                                   cl::desc("Enable the target verifier"));
 
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index d07b5fb668c14..3246b0ac0c624 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -34,7 +34,7 @@ using namespace llvm;
 #define Check(C, ...)                                                          \
   do {                                                                         \
     if (!(C)) {                                                                \
-      TargetVerify::CheckFailed(__VA_ARGS__);                                  \
+      TargetVerify::checkFailed(__VA_ARGS__);                                  \
     }                                                                          \
   } while (false)
 
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index e2d9edda5d008..819e66bb46b0a 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
 ; CHECK: Shaders must return void
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index a2dab0ff47924..08854df01acb2 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
 ; CHECK-NOT: Shaders must return void

>From 79215beef3f1330292ecee15b422f5cbfdeea9b5 Mon Sep 17 00:00:00 2001
From: Joey F <jofernau at amd.com>
Date: Mon, 16 Jun 2025 19:09:24 -0400
Subject: [PATCH 32/33] Fix after rebase

---
 llvm/include/llvm/CodeGen/Passes.h                  | 2 --
 llvm/include/llvm/IR/Module.h                       | 5 -----
 llvm/include/llvm/Passes/StandardInstrumentations.h | 4 ++--
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 53bd70183e476..990452fa11fec 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -620,8 +620,6 @@ LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
-
-LLVM_ABI FunctionPass *createTargetVerifierLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 34ac47a2b8be4..a99937a90cbb7 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,11 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module using intrinsics to record the position of debugging
-  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
-  /// \ref BasicBlock.
-  bool IsNewDbgInfoFormat;
-
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index d7c1bbf04c328..4ee5ab2554868 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,7 +478,7 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -624,7 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM = nullptr);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };

>From f25ecf1e14d99b797105a5d42386c9019be8d4d4 Mon Sep 17 00:00:00 2001
From: jofrn <jofernau at amd.com>
Date: Tue, 8 Jul 2025 08:02:10 -0400
Subject: [PATCH 33/33] Add back int check. These are invalid for AMDGPU.

---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 49 ++++++-------------
 .../Verifier/AMDGPU/tgt-verify-int-type.ll    |  6 +++
 2 files changed, 21 insertions(+), 34 deletions(-)
 create mode 100644 llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 3246b0ac0c624..ae35322c905f1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -46,48 +46,29 @@ class AMDGPUTargetVerify : public TargetVerify {
   bool run(Function &F) override;
 };
 
-static bool isShader(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::AMDGPU_VS:
-  case CallingConv::AMDGPU_LS:
-  case CallingConv::AMDGPU_HS:
-  case CallingConv::AMDGPU_ES:
-  case CallingConv::AMDGPU_GS:
-  case CallingConv::AMDGPU_PS:
-  case CallingConv::AMDGPU_CS_Chain:
-  case CallingConv::AMDGPU_CS_ChainPreserve:
-  case CallingConv::AMDGPU_CS:
-    return true;
-  default:
-    return false;
-  }
+static bool IsValidInt(const Type *Ty) {
+  return Ty->isIntegerTy(1) ||
+         Ty->isIntegerTy(8) ||
+         Ty->isIntegerTy(16) ||
+         Ty->isIntegerTy(32) ||
+         Ty->isIntegerTy(64) ||
+         Ty->isIntegerTy(128);
 }
 
 bool AMDGPUTargetVerify::run(Function &F) {
-  // Ensure shader calling convention returns void
-  if (isShader(F.getCallingConv()))
-    Check(F.getReturnType() == Type::getVoidTy(F.getContext()),
-          "Shaders must return void");
 
   for (auto &BB : F) {
 
     for (auto &I : BB) {
 
-      if (auto *CI = dyn_cast<CallInst>(&I)) {
-        // Ensure no kernel to kernel calls.
-        CallingConv::ID CalleeCC = CI->getCallingConv();
-        if (CalleeCC == CallingConv::AMDGPU_KERNEL) {
-          CallingConv::ID CallerCC =
-              CI->getParent()->getParent()->getCallingConv();
-          Check(CallerCC != CallingConv::AMDGPU_KERNEL,
-                "A kernel may not call a kernel", CI->getParent()->getParent());
-        }
-
-        // Ensure chain intrinsics are followed by unreachables.
-        if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
-          Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
-                "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
-      }
+      // Ensure integral types are valid: i8, i16, i32, i64, i128
+      if (I.getType()->isIntegerTy())
+        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
+      for (unsigned i = 0; i < I.getNumOperands(); ++i)
+        if (I.getOperand(i)->getType()->isIntegerTy())
+          Check(IsValidInt(I.getOperand(i)->getType()),
+                "Int type is invalid.", I.getOperand(i));
+
     }
   }
 
diff --git a/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll b/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll
new file mode 100644
index 0000000000000..bd3a7106756cd
--- /dev/null
+++ b/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll
@@ -0,0 +1,6 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt -o - < %s 2>&1 | FileCheck %s
+
+define amdgpu_cs i65 @invalid_int() {
+; CHECK: Int type is invalid.
+  ret i65 0
+}


From flang-commits at lists.llvm.org  Mon Jul 14 11:08:26 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 14 Jul 2025 11:08:26 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [TargetVerifier][AMDGPU] Add
 TargetVerifier. (PR #123609)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/123609@github.com>
Message-ID: <6875479a.050a0220.c7145.c1df@mx.google.com>


https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/123609

>From d9b0a5df811451d4d4a2e37fc4933c8071022317 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 20 Jan 2025 04:51:26 -0800
Subject: [PATCH 01/33] [TargetVerifier][AMDGPU] Add TargetVerifier.

This pass verifies the IR for an individual backend. This is
different than Lint because it consolidates all checks for a
given backend in a single pass. A check for Lint may be
undefined behavior across all targets, whereas a check in
TargetVerifier would only pertain to the specified target
but can check more than just undefined behavior such are IR
validity. A use case of this would be to reject programs
with invalid IR while fuzzing.
---
 llvm/include/llvm/IR/Module.h                 |   9 +
 llvm/include/llvm/Target/TargetVerifier.h     |  82 +++++++
 .../TargetVerify/AMDGPUTargetVerifier.h       |  36 +++
 llvm/lib/IR/Verifier.cpp                      |  18 +-
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 213 ++++++++++++++++++
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        |  62 +++++
 llvm/tools/llvm-tgt-verify/CMakeLists.txt     |  34 +++
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 172 ++++++++++++++
 9 files changed, 623 insertions(+), 4 deletions(-)
 create mode 100644 llvm/include/llvm/Target/TargetVerifier.h
 create mode 100644 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify.ll
 create mode 100644 llvm/tools/llvm-tgt-verify/CMakeLists.txt
 create mode 100644 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp

diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index a99937a90cbb7..3037b2c1644d9 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,6 +215,15 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
+  /// Is this Module valid as determined by one of the verification passes
+  /// i.e. Lint, Verifier, TargetVerifier.
+  bool IsValid = true;
+
+  /// Is this Module using intrinsics to record the position of debugging
+  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
+  /// \ref BasicBlock.
+  bool IsNewDbgInfoFormat;
+
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
new file mode 100644
index 0000000000000..e00c6a7b260c9
--- /dev/null
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -0,0 +1,82 @@
+//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at TargetVerifier.cpp or an
+// individual backend's TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_VERIFIER_H
+#define LLVM_TARGET_VERIFIER_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+
+class Function;
+
+class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {}
+};
+
+class TargetVerify {
+protected:
+  void WriteValues(ArrayRef<const Value *> Vs) {
+    for (const Value *V : Vs) {
+      if (!V)
+        continue;
+      if (isa<Instruction>(V)) {
+        MessagesStr << *V << '\n';
+      } else {
+        V->printAsOperand(MessagesStr, true, Mod);
+        MessagesStr << '\n';
+      }
+    }
+  }
+
+  /// A check failed, so printout out the condition and the message.
+  ///
+  /// This provides a nice place to put a breakpoint if you want to see why
+  /// something is not correct.
+  void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
+
+  /// A check failed (with values to print).
+  ///
+  /// This calls the Message-only version so that the above is easier to set
+  /// a breakpoint on.
+  template <typename T1, typename... Ts>
+  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) {
+    CheckFailed(Message);
+    WriteValues({V1, Vs...});
+  }
+public:
+  Module *Mod;
+  Triple TT;
+
+  std::string Messages;
+  raw_string_ostream MessagesStr;
+
+  TargetVerify(Module *Mod)
+      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
+        MessagesStr(Messages) {}
+
+  void run(Function &F) {};
+};
+
+} // namespace llvm
+
+#endif // LLVM_TARGET_VERIFIER_H
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
new file mode 100644
index 0000000000000..e6ff57629b141
--- /dev/null
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -0,0 +1,36 @@
+//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU ---*- C++ -*-===//
+////
+//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//// See https://llvm.org/LICENSE.txt for license information.
+//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+////
+////===----------------------------------------------------------------------===//
+////
+//// This file defines target verifier interfaces that can be used for some
+//// validation of input to the system, and for checking that transformations
+//// haven't done something bad. In contrast to the Verifier or Lint, the
+//// TargetVerifier looks for constructions invalid to a particular target
+//// machine.
+////
+//// To see what specifically is checked, look at an individual backend's
+//// TargetVerifier.
+////
+////===----------------------------------------------------------------------===//
+
+#ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
+#define LLVM_AMDGPU_TARGET_VERIFIER_H
+
+#include "llvm/Target/TargetVerifier.h"
+
+namespace llvm {
+
+class Function;
+
+class AMDGPUTargetVerifierPass : public TargetVerifierPass {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1f1041b259736..18e9760f9c617 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -135,6 +135,10 @@ static cl::opt<bool> VerifyNoAliasScopeDomination(
     cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
              "scopes are not dominating"));
 
+static cl::opt<bool>
+    VerifyAbortOnError("verifier-abort-on-error", cl::init(false),
+                       cl::desc("In the Verifier pass, abort on errors."));
+
 namespace llvm {
 
 struct VerifierSupport {
@@ -7865,16 +7869,22 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
-  if (FatalErrors && (Res.IRBroken || Res.DebugInfoBroken))
-    report_fatal_error("Broken module found, compilation aborted!");
+  if (Res.IRBroken || Res.DebugInfoBroken) {
+    M.IsValid = false;
+    if (VerifyAbortOnError && FatalErrors)
+      report_fatal_error("Broken module found, compilation aborted!");
+  }
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
-  if (res.IRBroken && FatalErrors)
-    report_fatal_error("Broken function found, compilation aborted!");
+  if (res.IRBroken) {
+    F.getParent()->IsValid = false;
+    if (VerifyAbortOnError && FatalErrors)
+      report_fatal_error("Broken function found, compilation aborted!");
+  }
 
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
new file mode 100644
index 0000000000000..585b19065c142
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -0,0 +1,213 @@
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<bool>
+MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
+
+// Check - We know that cond should be true, if not print an error message.
+#define Check(C, ...)                                                          \
+  do {                                                                         \
+    if (!(C)) {                                                                \
+      TargetVerify::CheckFailed(__VA_ARGS__);                                  \
+      return;                                                                  \
+    }                                                                          \
+  } while (false)
+
+static bool isMFMA(unsigned IID) {
+  switch (IID) {
+    case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
+    case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
+    case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
+    case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
+    case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
+    case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
+    case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
+    case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
+    case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
+    case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
+    case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
+
+    case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
+    case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
+
+    case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
+    case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
+
+    case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
+    case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
+    case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
+
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
+
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
+    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
+      return true;
+    default:
+      return false;
+  }
+}
+
+namespace llvm {
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  Module *Mod;
+
+  DominatorTree *DT;
+  PostDominatorTree *PDT;
+  UniformityInfo *UA;
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  void run(Function &F);
+};
+
+static bool IsValidInt(const Type *Ty) {
+  return Ty->isIntegerTy(1) ||
+         Ty->isIntegerTy(8) ||
+         Ty->isIntegerTy(16) ||
+         Ty->isIntegerTy(32) ||
+         Ty->isIntegerTy(64) ||
+         Ty->isIntegerTy(128);
+}
+
+static bool isShader(CallingConv::ID CC) {
+  switch(CC) {
+    case CallingConv::AMDGPU_VS:
+    case CallingConv::AMDGPU_LS:
+    case CallingConv::AMDGPU_HS:
+    case CallingConv::AMDGPU_ES:
+    case CallingConv::AMDGPU_GS:
+    case CallingConv::AMDGPU_PS:
+    case CallingConv::AMDGPU_CS_Chain:
+    case CallingConv::AMDGPU_CS_ChainPreserve:
+    case CallingConv::AMDGPU_CS:
+      return true;
+    default:
+      return false;
+  }
+}
+
+void AMDGPUTargetVerify::run(Function &F) {
+  // Ensure shader calling convention returns void
+  if (isShader(F.getCallingConv()))
+    Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
+
+  for (auto &BB : F) {
+
+    for (auto &I : BB) {
+      if (MarkUniform)
+        outs() << UA->isUniform(&I) << ' ' << I << '\n';
+
+      // Ensure integral types are valid: i8, i16, i32, i64, i128
+      if (I.getType()->isIntegerTy())
+        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
+      for (unsigned i = 0; i < I.getNumOperands(); ++i)
+        if (I.getOperand(i)->getType()->isIntegerTy())
+          Check(IsValidInt(I.getOperand(i)->getType()),
+                "Int type is invalid.", I.getOperand(i));
+
+      // Ensure no store to const memory
+      if (auto *SI = dyn_cast<StoreInst>(&I))
+      {
+        unsigned AS = SI->getPointerAddressSpace();
+        Check(AS != 4, "Write to const memory", SI);
+      }
+
+      // Ensure no kernel to kernel calls.
+      if (auto *CI = dyn_cast<CallInst>(&I))
+      {
+        CallingConv::ID CalleeCC = CI->getCallingConv();
+        if (CalleeCC == CallingConv::AMDGPU_KERNEL)
+        {
+          CallingConv::ID CallerCC = CI->getParent()->getParent()->getCallingConv();
+          Check(CallerCC != CallingConv::AMDGPU_KERNEL,
+            "A kernel may not call a kernel", CI->getParent()->getParent());
+        }
+      }
+
+      // Ensure MFMA is not in control flow with diverging operands
+      if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+        if (isMFMA(II->getIntrinsicID())) {
+          bool InControlFlow = false;
+          for (const auto &P : predecessors(&BB))
+            if (!PDT->dominates(&BB, P)) {
+              InControlFlow = true;
+              break;
+            }
+          for (const auto &S : successors(&BB))
+            if (!DT->dominates(&BB, S)) {
+              InControlFlow = true;
+              break;
+            }
+          if (InControlFlow) {
+            // If operands to MFMA are not uniform, MFMA cannot be in control flow
+            bool hasUniformOperands = true;
+            for (unsigned i = 0; i < II->getNumOperands(); i++) {
+              if (!UA->isUniform(II->getOperand(i))) {
+                dbgs() << "Not uniform: " << *II->getOperand(i) << '\n';
+                hasUniformOperands = false;
+              }
+            }
+            if (!hasUniformOperands) Check(false, "MFMA in control flow", II);
+            //else Check(false, "MFMA in control flow (uniform operands)", II);
+          }
+          //else Check(false, "MFMA not in control flow", II);
+        }
+      }
+    }
+  }
+}
+
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+
+  auto *Mod = F.getParent();
+
+  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  TV.run(F);
+
+  dbgs() << TV.MessagesStr.str();
+  if (!TV.MessagesStr.str().empty()) {
+    F.getParent()->IsValid = false;
+  }
+
+  return PreservedAnalyses::all();
+}
+} // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index c6d70ee39202e..a47b5f2732f91 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -111,6 +111,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUTargetMachine.cpp
   AMDGPUTargetObjectFile.cpp
   AMDGPUTargetTransformInfo.cpp
+  AMDGPUTargetVerifier.cpp
   AMDGPUWaitSGPRHazards.cpp
   AMDGPUUnifyDivergentExitNodes.cpp
   AMDGPUUnifyMetadata.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
new file mode 100644
index 0000000000000..f56ff992a56c2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -0,0 +1,62 @@
+; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
+
+define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg) #0 {
+; CHECK: Not uniform: %in.f32 = load <32 x float>, ptr addrspace(1) %gep, align 128
+; CHECK-NEXT: MFMA in control flow
+; CHECK-NEXT:   %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
+s:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
+  %in.i32 = load <32 x i32>, ptr addrspace(1) %gep
+  %in.f32 = load <32 x float>, ptr addrspace(1) %gep
+
+  %0 = icmp eq <32 x i32> %in.i32, zeroinitializer
+  %div.br = extractelement <32 x i1> %0, i32 0
+  br i1 %div.br, label %if.3, label %else.0
+
+if.3:
+  br label %join
+
+else.0:
+  %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
+  br label %join
+
+join:
+  ret void
+}
+
+define amdgpu_cs i32 @shader() {
+; CHECK: Shaders must return void
+  ret i32 0
+}
+
+define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
+; CHECK: Undefined behavior: Write to memory in const addrspace
+; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
+; CHECK-NEXT: Write to const memory
+; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
+  %r = add i32 %a, %b
+  store i32 %r, ptr addrspace(4) %out
+  ret void
+}
+
+define amdgpu_kernel void @kernel_callee(ptr %x) {
+  ret void
+}
+
+define amdgpu_kernel void @kernel_caller(ptr %x) {
+; CHECK: A kernel may not call a kernel
+; CHECK-NEXT: ptr @kernel_caller
+  call amdgpu_kernel void @kernel_callee(ptr %x)
+  ret void
+}
+
+
+; Function Attrs: nounwind
+define i65 @invalid_type(i65 %x) #0 {
+; CHECK: Int type is invalid.
+; CHECK-NEXT: %tmp2 = ashr i65 %x, 64
+entry:
+  %tmp2 = ashr i65 %x, 64
+  ret i65 %tmp2
+}
diff --git a/llvm/tools/llvm-tgt-verify/CMakeLists.txt b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
new file mode 100644
index 0000000000000..fe47c85e6cdce
--- /dev/null
+++ b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
@@ -0,0 +1,34 @@
+set(LLVM_LINK_COMPONENTS
+  AllTargetsAsmParsers
+  AllTargetsCodeGens
+  AllTargetsDescs
+  AllTargetsInfos
+  Analysis
+  AsmPrinter
+  CodeGen
+  CodeGenTypes
+  Core
+  IRPrinter
+  IRReader
+  MC
+  MIRParser
+  Passes
+  Remarks
+  ScalarOpts
+  SelectionDAG
+  Support
+  Target
+  TargetParser
+  TransformUtils
+  Vectorize
+  )
+
+add_llvm_tool(llvm-tgt-verify
+  llvm-tgt-verify.cpp
+
+  DEPENDS
+  intrinsics_gen
+  SUPPORT_PLUGINS
+  )
+
+export_executable_symbols_for_plugins(llc)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
new file mode 100644
index 0000000000000..68422abd6f4cc
--- /dev/null
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -0,0 +1,172 @@
+//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tool to fuzz instruction selection using libFuzzer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/InitializePasses.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/Lint.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/CodeGen/CommandFlags.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetVerifier.h"
+
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#define DEBUG_TYPE "isel-fuzzer"
+
+using namespace llvm;
+
+static codegen::RegisterCodeGenFlags CGF;
+
+static cl::opt<std::string>
+InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
+
+static cl::opt<bool>
+    StacktraceAbort("stacktrace-abort",
+        cl::desc("Turn on stacktrace"), cl::init(false));
+
+static cl::opt<bool>
+    NoLint("no-lint",
+        cl::desc("Turn off Lint"), cl::init(false));
+
+static cl::opt<bool>
+    NoVerify("no-verifier",
+        cl::desc("Turn off Verifier"), cl::init(false));
+
+static cl::opt<char>
+    OptLevel("O",
+             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
+                      "(default = '-O2')"),
+             cl::Prefix, cl::init('2'));
+
+static cl::opt<std::string>
+    TargetTriple("mtriple", cl::desc("Override target triple for module"));
+
+static std::unique_ptr<TargetMachine> TM;
+
+static void handleLLVMFatalError(void *, const char *Message, bool) {
+  if (StacktraceAbort) {
+    dbgs() << "LLVM ERROR: " << Message << "\n"
+           << "Aborting.\n";
+    abort();
+  }
+}
+
+int main(int argc, char **argv) {
+  StringRef ExecName = argv[0];
+  InitLLVM X(argc, argv);
+
+  InitializeAllTargets();
+  InitializeAllTargetMCs();
+  InitializeAllAsmPrinters();
+  InitializeAllAsmParsers();
+
+  PassRegistry *Registry = PassRegistry::getPassRegistry();
+  initializeCore(*Registry);
+  initializeCodeGen(*Registry);
+  initializeAnalysis(*Registry);
+  initializeTarget(*Registry);
+
+  cl::ParseCommandLineOptions(argc, argv);
+
+  if (TargetTriple.empty()) {
+    errs() << ExecName << ": -mtriple must be specified\n";
+    exit(1);
+  }
+
+  CodeGenOptLevel OLvl;
+  if (auto Level = CodeGenOpt::parseLevel(OptLevel)) {
+    OLvl = *Level;
+  } else {
+    errs() << ExecName << ": invalid optimization level.\n";
+    return 1;
+  }
+  ExitOnError ExitOnErr(std::string(ExecName) + ": error:");
+  TM = ExitOnErr(codegen::createTargetMachineForTriple(
+      Triple::normalize(TargetTriple), OLvl));
+  assert(TM && "Could not allocate target machine!");
+
+  // Make sure we print the summary and the current unit when LLVM errors out.
+  install_fatal_error_handler(handleLLVMFatalError, nullptr);
+
+  LLVMContext Context;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
+  if (!M) {
+    errs() << "Invalid mod\n";
+    return 1;
+  }
+  auto S = Triple::normalize(TargetTriple);
+  M->setTargetTriple(S);
+
+  PassInstrumentationCallbacks PIC;
+  StandardInstrumentations SI(Context, false/*debug PM*/,
+                              false);
+  registerCodeGenCallback(PIC, *TM);
+
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+  //TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
+
+  MachineFunctionAnalysisManager MFAM;
+  LoopAnalysisManager LAM;
+  FunctionAnalysisManager FAM;
+  CGSCCAnalysisManager CGAM;
+  ModuleAnalysisManager MAM;
+  PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.registerMachineFunctionAnalyses(MFAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
+
+  SI.registerCallbacks(PIC, &MAM);
+
+  //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
+
+  Triple TT(M->getTargetTriple());
+  if (!NoLint)
+    FPM.addPass(LintPass());
+  if (!NoVerify)
+    MPM.addPass(VerifierPass());
+  if (TT.isAMDGPU())
+    FPM.addPass(AMDGPUTargetVerifierPass());
+  else if (false) {} // ...
+  else
+    FPM.addPass(TargetVerifierPass());
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+
+  MPM.run(*M, MAM);
+
+  if (!M->IsValid)
+    return 1;
+
+  return 0;
+}

>From 3b69ade6d8880cadc883698b64a06bd9207d084b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 3 Feb 2025 07:15:12 -0800
Subject: [PATCH 02/33] Add hook for target verifier in llc,opt

---
 .../llvm/Passes/StandardInstrumentations.h    |  6 ++++--
 llvm/include/llvm/Target/TargetVerifier.h     |  1 +
 .../TargetVerify/AMDGPUTargetVerifier.h       | 18 ++++++++++++++++++
 llvm/lib/LTO/LTOBackend.cpp                   |  2 +-
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp         |  2 +-
 llvm/lib/Passes/CMakeLists.txt                |  1 +
 llvm/lib/Passes/PassBuilderBindings.cpp       |  2 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  | 19 +++++++++++++++----
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 12 ++++++------
 llvm/lib/Target/CMakeLists.txt                |  2 ++
 .../CodeGen/AMDGPU/tgt-verify-llc-fail.ll     |  6 ++++++
 .../CodeGen/AMDGPU/tgt-verify-llc-pass.ll     |  6 ++++++
 llvm/tools/llc/NewPMDriver.cpp                |  2 +-
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 +-
 llvm/tools/opt/NewPMDriver.cpp                |  2 +-
 llvm/unittests/IR/PassManagerTest.cpp         |  6 +++---
 16 files changed, 68 insertions(+), 21 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll

diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 4ee5ab2554868..7d7b858b447cc 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,7 +478,8 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM);
+                         ModuleAnalysisManager *MAM,
+			 FunctionAnalysisManager *FAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -624,7 +625,8 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM = nullptr);
+                         ModuleAnalysisManager *MAM,
+			 FunctionAnalysisManager *FAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index e00c6a7b260c9..ad5aeb895953d 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -75,6 +75,7 @@ class TargetVerify {
         MessagesStr(Messages) {}
 
   void run(Function &F) {};
+  void run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index e6ff57629b141..d8a3fda4f87dc 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -22,6 +22,10 @@
 
 #include "llvm/Target/TargetVerifier.h"
 
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/IR/Dominators.h"
+
 namespace llvm {
 
 class Function;
@@ -31,6 +35,20 @@ class AMDGPUTargetVerifierPass : public TargetVerifierPass {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  Module *Mod;
+
+  DominatorTree *DT;
+  PostDominatorTree *PDT;
+  UniformityInfo *UA;
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  void run(Function &F);
+};
+
 } // namespace llvm
 
 #endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 5e8cd12fe040b..7ae23ef4e7df4 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -273,7 +273,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
                               Conf.VerifyEach);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 
   RegisterPassPlugins(Conf.PassPlugins, PB);
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 5b333cda31fb8..8db92e70d5a96 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -241,7 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PipelineTuningOptions PTO;
   PTO.LoopVectorization = true;
   PTO.SLPVectorization = true;
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 91c8c4f67074d..e5912f23903b6 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,6 +30,7 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
+  TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index 933fe89e53a94..f0e1abb8cebc4 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -76,7 +76,7 @@ static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   // Run the pipeline.
   if (Fun) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 4e8abc43572d7..e2764b162fddb 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,6 +40,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
+#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -1449,9 +1450,10 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 }
 
 void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                              ModuleAnalysisManager *MAM) {
+                                              ModuleAnalysisManager *MAM,
+					      FunctionAnalysisManager *FAM) {
   PIC.registerAfterPassCallback(
-      [this, MAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+      [this, MAM, FAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
         const auto *F = unwrapIR<Function>(IR);
@@ -1468,6 +1470,15 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
             report_fatal_error(formatv("Broken function found after pass "
                                        "\"{0}\", compilation aborted!",
                                        P));
+
+          if (FAM) {
+            TargetVerify TV(const_cast<Module*>(F->getParent()));
+            TV.run(*const_cast<Function*>(F), *FAM);
+	    if (!F->getParent()->IsValid)
+              report_fatal_error(formatv("Broken function found after pass "
+                                         "\"{0}\", compilation aborted!",
+                                         P));
+          }
         } else {
           const auto *M = unwrapIR<Module>(IR);
           if (!M) {
@@ -2507,7 +2518,7 @@ void PrintCrashIRInstrumentation::registerCallbacks(
 }
 
 void StandardInstrumentations::registerCallbacks(
-    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM) {
+    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM, FunctionAnalysisManager *FAM) {
   PrintIR.registerCallbacks(PIC);
   PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
@@ -2516,7 +2527,7 @@ void StandardInstrumentations::registerCallbacks(
   PrintChangedIR.registerCallbacks(PIC);
   PseudoProbeVerification.registerCallbacks(PIC);
   if (VerifyEach)
-    Verify.registerCallbacks(PIC, MAM);
+    Verify.registerCallbacks(PIC, MAM, FAM);
   PrintChangedDiff.registerCallbacks(PIC);
   WebsiteChangeReporter.registerCallbacks(PIC);
   ChangeTester.registerCallbacks(PIC);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 585b19065c142..e6cdec7160229 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -14,8 +14,8 @@
 
 using namespace llvm;
 
-static cl::opt<bool>
-MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
+//static cl::opt<bool>
+//MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
 
 // Check - We know that cond should be true, if not print an error message.
 #define Check(C, ...)                                                          \
@@ -81,7 +81,7 @@ static bool isMFMA(unsigned IID) {
 }
 
 namespace llvm {
-class AMDGPUTargetVerify : public TargetVerify {
+/*class AMDGPUTargetVerify : public TargetVerify {
 public:
   Module *Mod;
 
@@ -93,7 +93,7 @@ class AMDGPUTargetVerify : public TargetVerify {
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   void run(Function &F);
-};
+};*/
 
 static bool IsValidInt(const Type *Ty) {
   return Ty->isIntegerTy(1) ||
@@ -129,8 +129,8 @@ void AMDGPUTargetVerify::run(Function &F) {
   for (auto &BB : F) {
 
     for (auto &I : BB) {
-      if (MarkUniform)
-        outs() << UA->isUniform(&I) << ' ' << I << '\n';
+      //if (MarkUniform)
+        //outs() << UA->isUniform(&I) << ' ' << I << '\n';
 
       // Ensure integral types are valid: i8, i16, i32, i64, i128
       if (I.getType()->isIntegerTy())
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index 9472288229cac..f2a5d545ce84f 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -7,6 +7,8 @@ add_llvm_component_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
+  TargetVerifier.cpp
+  AMDGPU/AMDGPUTargetVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Target
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
new file mode 100644
index 0000000000000..584097d7bc134
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -0,0 +1,6 @@
+; RUN: not not llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each -o - < %s 2>&1 | FileCheck %s
+
+define amdgpu_cs i32 @nonvoid_shader() {
+; CHECK: LLVM ERROR
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
new file mode 100644
index 0000000000000..0c3a5fe5ac4a5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -0,0 +1,6 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each %s -o - 2>&1 | FileCheck %s
+
+define amdgpu_cs void @void_shader() {
+; CHECK: ModuleToFunctionPassAdaptor
+  ret void
+}
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index fa82689ecf9ae..a060d16e74958 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -126,7 +126,7 @@ int llvm::compileModuleWithNewPM(
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 68422abd6f4cc..3352d07deff2f 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -147,7 +147,7 @@ int main(int argc, char **argv) {
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
 
   //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
 
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index 7d168a6ceb17c..a8977d80bdf44 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -423,7 +423,7 @@ bool llvm::runPassPipeline(
   PrintPassOpts.SkipAnalyses = DebugPM == DebugLogging::Quiet;
   StandardInstrumentations SI(M.getContext(), DebugPM != DebugLogging::None,
                               VK == VerifierKind::EachPass, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   DebugifyEachInstrumentation Debugify;
   DebugifyStatsMap DIStatsMap;
   DebugInfoPerPass DebugInfoBeforePass;
diff --git a/llvm/unittests/IR/PassManagerTest.cpp b/llvm/unittests/IR/PassManagerTest.cpp
index a6487169224c2..bb4db6120035f 100644
--- a/llvm/unittests/IR/PassManagerTest.cpp
+++ b/llvm/unittests/IR/PassManagerTest.cpp
@@ -828,7 +828,7 @@ TEST_F(PassManagerTest, FunctionPassCFGChecker) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -877,7 +877,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerInvalidateAnalysis) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -945,7 +945,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerWrapped) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });

>From a89ff83fbb4a39d6e91ae3bccbb6244c32433a4b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 16 Apr 2025 10:19:00 -0400
Subject: [PATCH 03/33] Run AMDGPUTargetVerifier within AMDGPU pipeline. Move
 IsValid from Module to TargetVerify.

---
 clang/lib/CodeGen/BackendUtil.cpp               | 2 +-
 llvm/include/llvm/IR/Module.h                   | 4 ----
 llvm/include/llvm/Target/TargetVerifier.h       | 2 ++
 llvm/lib/IR/Verifier.cpp                        | 4 ++--
 llvm/lib/Passes/StandardInstrumentations.cpp    | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  | 5 +++++
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 3 ++-
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp  | 6 +++---
 8 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 7e0a3cf5591ce..560129f871026 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -917,7 +917,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       TheModule->getContext(),
       (CodeGenOpts.DebugPassManager || DebugPassStructure),
       CodeGenOpts.VerifyEach, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM);
+  SI.registerCallbacks(PIC, &MAM, &FAM);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
   // Handle the assignment tracking feature options.
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 3037b2c1644d9..34ac47a2b8be4 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,10 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module valid as determined by one of the verification passes
-  /// i.e. Lint, Verifier, TargetVerifier.
-  bool IsValid = true;
-
   /// Is this Module using intrinsics to record the position of debugging
   /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
   /// \ref BasicBlock.
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index ad5aeb895953d..2d0c039132c35 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -70,6 +70,8 @@ class TargetVerify {
   std::string Messages;
   raw_string_ostream MessagesStr;
 
+  bool IsValid = true;
+
   TargetVerify(Module *Mod)
       : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
         MessagesStr(Messages) {}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 18e9760f9c617..0054dd26327ae 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7870,7 +7870,7 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
   if (Res.IRBroken || Res.DebugInfoBroken) {
-    M.IsValid = false;
+    //M.IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken module found, compilation aborted!");
   }
@@ -7881,7 +7881,7 @@ PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
   if (res.IRBroken) {
-    F.getParent()->IsValid = false;
+    //F.getParent()->IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken function found, compilation aborted!");
   }
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index e2764b162fddb..2bc54cf7d4891 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -1474,7 +1474,7 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
           if (FAM) {
             TargetVerify TV(const_cast<Module*>(F->getParent()));
             TV.run(*const_cast<Function*>(F), *FAM);
-	    if (!F->getParent()->IsValid)
+	    if (!TV.IsValid)
               report_fatal_error(formatv("Broken function found after pass "
                                          "\"{0}\", compilation aborted!",
                                          P));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index d59087839b0e1..dc88042a66df9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,6 +90,7 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
@@ -1328,6 +1329,8 @@ void AMDGPUPassConfig::addIRPasses() {
       addPass(createLICMPass());
   }
 
+  //addPass(AMDGPUTargetVerifierPass());
+
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
@@ -2071,6 +2074,8 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   // but EarlyCSE can do neither of them.
   if (isPassEnabled(EnableScalarIRPasses))
     addEarlyCSEOrGVNPass(addPass);
+
+  addPass(AMDGPUTargetVerifierPass());
 }
 
 void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index e6cdec7160229..c70a6d1b6fa66 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -205,7 +205,8 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
 
   dbgs() << TV.MessagesStr.str();
   if (!TV.MessagesStr.str().empty()) {
-    F.getParent()->IsValid = false;
+    TV.IsValid = false;
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 3352d07deff2f..fbe7f6089ff18 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -163,9 +163,9 @@ int main(int argc, char **argv) {
     FPM.addPass(TargetVerifierPass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
-  MPM.run(*M, MAM);
-
-  if (!M->IsValid)
+  auto PA = MPM.run(*M, MAM);
+  auto PAC = PA.getChecker<VerifierAnalysis>();
+  if (!PAC.preserved())
     return 1;
 
   return 0;

>From df507f9bfc4caa5578b48be68fb6c362808ec4af Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 16 Apr 2025 11:08:20 -0400
Subject: [PATCH 04/33] Remove cmd line options that aren't required. Make
 error message explicit.

---
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll | 4 ++--
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index 584097d7bc134..c5e59d4a2369e 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,6 +1,6 @@
-; RUN: not not llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
-; CHECK: LLVM ERROR
+; CHECK: Shaders must return void
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index 0c3a5fe5ac4a5..8a503b7624a73 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-new-pm -verify-each %s -o - 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
-; CHECK: ModuleToFunctionPassAdaptor
+; CHECK-NOT: Shaders must return void
   ret void
 }

>From 942312d57ea09d3dc9224d5cb3753390db13b2ba Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 19:36:34 -0400
Subject: [PATCH 05/33] Return Verifier none status through PreservedAnalyses
 on fail.

---
 llvm/lib/Analysis/Lint.cpp                     | 4 +++-
 llvm/lib/IR/Verifier.cpp                       | 2 ++
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 8 +++++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 1168005f48c0e..c8627e6db0692 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -735,9 +735,11 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) {
   Lint L(Mod, DL, AA, AC, DT, TLI);
   L.visit(F);
   dbgs() << L.MessagesStr.str();
-  if (AbortOnError && !L.MessagesStr.str().empty())
+  if (AbortOnError && !L.MessagesStr.str().empty()) {
     report_fatal_error(
         "linter found errors, aborting. (enabled by abort-on-error)", false);
+    return PreservedAnalyses::none();
+  }
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 0054dd26327ae..439823ba8b682 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7873,6 +7873,7 @@ PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
     //M.IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken module found, compilation aborted!");
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
@@ -7884,6 +7885,7 @@ PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
     //F.getParent()->IsValid = false;
     if (VerifyAbortOnError && FatalErrors)
       report_fatal_error("Broken function found, compilation aborted!");
+    return PreservedAnalyses::none();
   }
 
   return PreservedAnalyses::all();
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index fbe7f6089ff18..042824ac37fea 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -164,9 +164,11 @@ int main(int argc, char **argv) {
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
   auto PA = MPM.run(*M, MAM);
-  auto PAC = PA.getChecker<VerifierAnalysis>();
-  if (!PAC.preserved())
-    return 1;
+  {
+    auto PAC = PA.getChecker<VerifierAnalysis>();
+    if (!PAC.preserved())
+      return 1;
+  }
 
   return 0;
 }

>From 6216a104469edfa1da128ce64be2cba2c2421b00 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 21:16:02 -0400
Subject: [PATCH 06/33] Rebase update.

---
 llvm/include/llvm/Target/TargetVerifier.h      | 2 +-
 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 2d0c039132c35..fe683311b901c 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -73,7 +73,7 @@ class TargetVerify {
   bool IsValid = true;
 
   TargetVerify(Module *Mod)
-      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())),
+      : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
   void run(Function &F) {};
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 042824ac37fea..627bc51ef3a43 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -123,7 +123,7 @@ int main(int argc, char **argv) {
     return 1;
   }
   auto S = Triple::normalize(TargetTriple);
-  M->setTargetTriple(S);
+  M->setTargetTriple(Triple(S));
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Context, false/*debug PM*/,
@@ -153,7 +153,7 @@ int main(int argc, char **argv) {
 
   Triple TT(M->getTargetTriple());
   if (!NoLint)
-    FPM.addPass(LintPass());
+    FPM.addPass(LintPass(false));
   if (!NoVerify)
     MPM.addPass(VerifierPass());
   if (TT.isAMDGPU())

>From 4b7068f49c24f28d4fac762ea6f1f8420028d525 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 22:49:52 -0400
Subject: [PATCH 07/33] Add generic TargetVerifier.

---
 llvm/lib/Target/TargetVerifier.cpp | 32 ++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 llvm/lib/Target/TargetVerifier.cpp

diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
new file mode 100644
index 0000000000000..de3ff749e7c3c
--- /dev/null
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -0,0 +1,32 @@
+#include "llvm/Target/TargetVerifier.h"
+#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+
+namespace llvm {
+
+void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
+  if (TT.isAMDGPU()) {
+    auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
+    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+    }
+  }
+}
+
+} // namespace llvm

>From 86e806635cdf1f1606af654817aefa8d797b8cf5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:13:14 -0400
Subject: [PATCH 08/33] Remove store to const check since it is in Lint already

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 8 --------
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll          | 2 --
 2 files changed, 10 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index c70a6d1b6fa66..1cf2b277bee26 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -140,14 +140,6 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(IsValidInt(I.getOperand(i)->getType()),
                 "Int type is invalid.", I.getOperand(i));
 
-      // Ensure no store to const memory
-      if (auto *SI = dyn_cast<StoreInst>(&I))
-      {
-        unsigned AS = SI->getPointerAddressSpace();
-        Check(AS != 4, "Write to const memory", SI);
-      }
-
-      // Ensure no kernel to kernel calls.
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
         CallingConv::ID CalleeCC = CI->getCallingConv();
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index f56ff992a56c2..c628abbde11d1 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -32,8 +32,6 @@ define amdgpu_cs i32 @shader() {
 
 define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
 ; CHECK: Undefined behavior: Write to memory in const addrspace
-; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
-; CHECK-NEXT: Write to const memory
 ; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
   %r = add i32 %a, %b
   store i32 %r, ptr addrspace(4) %out

>From 17fc5c6be3406d4360e25b2de595d31afe7f77bc Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:14:38 -0400
Subject: [PATCH 09/33] Add chain followed by unreachable check

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp |  6 ++++++
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll          | 10 ++++++++++
 2 files changed, 16 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 1cf2b277bee26..8ea773bc0e66f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -142,6 +142,7 @@ void AMDGPUTargetVerify::run(Function &F) {
 
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
+        // Ensure no kernel to kernel calls.
         CallingConv::ID CalleeCC = CI->getCallingConv();
         if (CalleeCC == CallingConv::AMDGPU_KERNEL)
         {
@@ -149,6 +150,11 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(CallerCC != CallingConv::AMDGPU_KERNEL,
             "A kernel may not call a kernel", CI->getParent()->getParent());
         }
+
+        // Ensure chain intrinsics are followed by unreachables.
+        if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
+          Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
+            "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
 
       // Ensure MFMA is not in control flow with diverging operands
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index c628abbde11d1..e620df94ccde4 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -58,3 +58,13 @@ entry:
   %tmp2 = ashr i65 %x, 64
   ret i65 %tmp2
 }
+
+declare void @llvm.amdgcn.cs.chain.v3i32(ptr, i32, <3 x i32>, <3 x i32>, i32, ...)
+declare amdgpu_cs_chain void @chain_callee(<3 x i32> inreg, <3 x i32>)
+
+define amdgpu_cs void @no_unreachable(<3 x i32> inreg %a, <3 x i32> %b) {
+; CHECK: llvm.amdgcn.cs.chain must be followed by unreachable
+; CHECK-NEXT: call void (ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.v3i32.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
+  ret void
+}

>From 46e662cce2d1f3716fcf5c23be39fdb27a2d7ecb Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 19 Apr 2025 23:26:19 -0400
Subject: [PATCH 10/33] Remove mfma check

---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 89 -------------------
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        | 25 ------
 2 files changed, 114 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 8ea773bc0e66f..684ced5bba574 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -14,9 +14,6 @@
 
 using namespace llvm;
 
-//static cl::opt<bool>
-//MarkUniform("mark-uniform", cl::desc("Mark instructions as uniform"), cl::init(false));
-
 // Check - We know that cond should be true, if not print an error message.
 #define Check(C, ...)                                                          \
   do {                                                                         \
@@ -26,60 +23,6 @@ using namespace llvm;
     }                                                                          \
   } while (false)
 
-static bool isMFMA(unsigned IID) {
-  switch (IID) {
-    case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
-    case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
-    case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
-    case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
-    case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
-    case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
-    case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
-    case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
-    case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
-    case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
-    case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
-
-    case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
-    case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
-
-    case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
-    case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
-
-    case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
-    case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
-    case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
-
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
-
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
-    case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
-      return true;
-    default:
-      return false;
-  }
-}
-
 namespace llvm {
 /*class AMDGPUTargetVerify : public TargetVerify {
 public:
@@ -129,8 +72,6 @@ void AMDGPUTargetVerify::run(Function &F) {
   for (auto &BB : F) {
 
     for (auto &I : BB) {
-      //if (MarkUniform)
-        //outs() << UA->isUniform(&I) << ' ' << I << '\n';
 
       // Ensure integral types are valid: i8, i16, i32, i64, i128
       if (I.getType()->isIntegerTy())
@@ -156,36 +97,6 @@ void AMDGPUTargetVerify::run(Function &F) {
           Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
             "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
-
-      // Ensure MFMA is not in control flow with diverging operands
-      if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
-        if (isMFMA(II->getIntrinsicID())) {
-          bool InControlFlow = false;
-          for (const auto &P : predecessors(&BB))
-            if (!PDT->dominates(&BB, P)) {
-              InControlFlow = true;
-              break;
-            }
-          for (const auto &S : successors(&BB))
-            if (!DT->dominates(&BB, S)) {
-              InControlFlow = true;
-              break;
-            }
-          if (InControlFlow) {
-            // If operands to MFMA are not uniform, MFMA cannot be in control flow
-            bool hasUniformOperands = true;
-            for (unsigned i = 0; i < II->getNumOperands(); i++) {
-              if (!UA->isUniform(II->getOperand(i))) {
-                dbgs() << "Not uniform: " << *II->getOperand(i) << '\n';
-                hasUniformOperands = false;
-              }
-            }
-            if (!hasUniformOperands) Check(false, "MFMA in control flow", II);
-            //else Check(false, "MFMA in control flow (uniform operands)", II);
-          }
-          //else Check(false, "MFMA not in control flow", II);
-        }
-      }
     }
   }
 }
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
index e620df94ccde4..62b220d7d9f49 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
@@ -1,30 +1,5 @@
 ; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
 
-define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg) #0 {
-; CHECK: Not uniform: %in.f32 = load <32 x float>, ptr addrspace(1) %gep, align 128
-; CHECK-NEXT: MFMA in control flow
-; CHECK-NEXT:   %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
-s:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
-  %in.i32 = load <32 x i32>, ptr addrspace(1) %gep
-  %in.f32 = load <32 x float>, ptr addrspace(1) %gep
-
-  %0 = icmp eq <32 x i32> %in.i32, zeroinitializer
-  %div.br = extractelement <32 x i1> %0, i32 0
-  br i1 %div.br, label %if.3, label %else.0
-
-if.3:
-  br label %join
-
-else.0:
-  %mfma = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.f32, i32 1, i32 2, i32 3)
-  br label %join
-
-join:
-  ret void
-}
-
 define amdgpu_cs i32 @shader() {
 ; CHECK: Shaders must return void
   ret i32 0

>From 7e471c9111d6a3a33f0de6bac883353669a69b0a Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 21 Apr 2025 20:54:10 -0400
Subject: [PATCH 11/33] Add registerVerifierPasses to PassBuilder and add the
 verifier passes to PassRegistry.

---
 llvm/include/llvm/InitializePasses.h          |  1 +
 llvm/include/llvm/Passes/PassBuilder.h        | 21 +++++++
 .../llvm/Passes/TargetPassRegistry.inc        | 12 ++++
 .../TargetVerify/AMDGPUTargetVerifier.h       | 11 ++--
 llvm/lib/Passes/PassBuilder.cpp               |  7 +++
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 11 ++++
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 56 ++++++++++++++++++-
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  1 +
 8 files changed, 114 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 1b5b1d5888824..662f6b71aa46a 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,6 +333,7 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
+LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index f13b5c678a894..c134f2d0c1112 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -173,6 +173,13 @@ class PassBuilder {
   /// additional analyses.
   LLVM_ABI void registerLoopAnalyses(LoopAnalysisManager &LAM);
 
+  /// Registers all available verifier passes.
+  ///
+  /// This is an interface that can be used to populate a
+  /// \c ModuleAnalysisManager with all registered loop analyses. Callers can
+  /// still manually register any additional analyses.
+  void registerVerifierPasses(ModulePassManager &PM, FunctionPassManager &);
+
   /// Registers all available machine function analysis passes.
   ///
   /// This is an interface that can be used to populate a \c
@@ -576,6 +583,15 @@ class PassBuilder {
   }
   /// @}}
 
+  /// Register a callback for parsing an Verifier Name to populate
+  /// the given managers.
+  void registerVerifierCallback(
+      const std::function<bool(ModulePassManager &MPM)> &C,
+      const std::function<bool(FunctionPassManager &MPM)> &CF) {
+    VerifierCallbacks.push_back(C);
+    FnVerifierCallbacks.push_back(CF);
+  }
+
   /// {{@ Register pipeline parsing callbacks with this pass builder instance.
   /// Using these callbacks, callers can parse both a single pass name, as well
   /// as entire sub-pipelines, and populate the PassManager instance
@@ -851,6 +867,11 @@ class PassBuilder {
   // Callbacks to parse `filter` parameter in register allocation passes
   SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
+  // Verifier callbacks
+  SmallVector<std::function<bool(ModulePassManager &)>, 2>
+      VerifierCallbacks;
+  SmallVector<std::function<bool(FunctionPassManager &)>, 2>
+      FnVerifierCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc
index 521913cb25a4a..2d04b874cf360 100644
--- a/llvm/include/llvm/Passes/TargetPassRegistry.inc
+++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc
@@ -151,6 +151,18 @@ PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM,
   return false;
 });
 
+PB.registerVerifierCallback([](ModulePassManager &PM) {
+#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS) PM.addPass(CREATE_PASS)
+#include GET_PASS_REGISTRY
+#undef VERIFIER_MODULE_ANALYSIS
+  return false;
+}, [](FunctionPassManager &FPM) {
+#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS) FPM.addPass(CREATE_PASS)
+#include GET_PASS_REGISTRY
+#undef VERIFIER_FUNCTION_ANALYSIS
+  return false;
+});
+
 #undef ADD_PASS
 #undef ADD_PASS_WITH_PARAMS
 
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index d8a3fda4f87dc..b6a7412e8c1ef 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -39,14 +39,17 @@ class AMDGPUTargetVerify : public TargetVerify {
 public:
   Module *Mod;
 
-  DominatorTree *DT;
-  PostDominatorTree *PDT;
-  UniformityInfo *UA;
+  DominatorTree *DT = nullptr;
+  PostDominatorTree *PDT = nullptr;
+  UniformityInfo *UA = nullptr;
+
+  AMDGPUTargetVerify(Module *Mod)
+    : TargetVerify(Mod), Mod(Mod) {}
 
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
-  void run(Function &F);
+  bool run(Function &F);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 4603eaff8ade9..561996d0c8980 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -576,6 +576,13 @@ void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     C(LAM);
 }
 
+void PassBuilder::registerVerifierPasses(ModulePassManager &MPM, FunctionPassManager &FPM) {
+  for (auto &C : VerifierCallbacks)
+    C(MPM);
+  for (auto &C : FnVerifierCallbacks)
+    C(FPM);
+}
+
 static std::optional<std::pair<bool, bool>>
 parseFunctionPipelineName(StringRef Name) {
   std::pair<bool, bool> Params;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 13453963eec6d..652cd23f62443 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -82,6 +82,17 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
+#ifndef VERIFIER_MODULE_ANALYSIS
+#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS)
+#endif
+#ifndef VERIFIER_FUNCTION_ANALYSIS
+#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
+#endif
+VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
+#undef VERIFIER_MODULE_ANALYSIS
+#undef VERIFIER_FUNCTION_ANALYSIS
+
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 684ced5bba574..63a7526b9abdc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -5,6 +5,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -19,7 +20,7 @@ using namespace llvm;
   do {                                                                         \
     if (!(C)) {                                                                \
       TargetVerify::CheckFailed(__VA_ARGS__);                                  \
-      return;                                                                  \
+      return false;                                                                  \
     }                                                                          \
   } while (false)
 
@@ -64,7 +65,7 @@ static bool isShader(CallingConv::ID CC) {
   }
 }
 
-void AMDGPUTargetVerify::run(Function &F) {
+bool AMDGPUTargetVerify::run(Function &F) {
   // Ensure shader calling convention returns void
   if (isShader(F.getCallingConv()))
     Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
@@ -99,6 +100,10 @@ void AMDGPUTargetVerify::run(Function &F) {
       }
     }
   }
+
+  if (!MessagesStr.str().empty())
+    return false;
+  return true;
 }
 
 PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
@@ -120,4 +125,51 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
 
   return PreservedAnalyses::all();
 }
+
+struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<AMDGPUTargetVerify> TV;
+  bool FatalErrors = true;
+
+  AMDGPUTargetVerifierLegacyPass() : FunctionPass(ID) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
+      : FunctionPass(ID),
+        FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<AMDGPUTargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (TV->run(F) && FatalErrors) {
+      errs() << "in function " << F.getName() << '\n';
+      report_fatal_error("Broken function found, compilation aborted!");
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    //IsValid &= TV->run();
+    if (FatalErrors && !IsValid)
+      report_fatal_error("Broken module found, compilation aborted!");
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char AMDGPUTargetVerifierLegacyPass::ID = 0;
 } // namespace llvm
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverify", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 627bc51ef3a43..503db7b1f8d18 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -144,6 +144,7 @@ int main(int argc, char **argv) {
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
+  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 

>From 1538985ecdd840e68207f83ca7b159bccef98ff5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Tue, 22 Apr 2025 14:57:31 -0400
Subject: [PATCH 12/33] Remove leftovers. Add titles. Add call to
 registerVerifierCallbacks in llc.

---
 llvm/lib/Passes/CMakeLists.txt                |  2 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  4 ---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 35 +++++++++++--------
 llvm/lib/Target/TargetVerifier.cpp            | 19 ++++++++++
 llvm/tools/llc/NewPMDriver.cpp                |  6 ++--
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  7 ++--
 6 files changed, 45 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index e5912f23903b6..b8d25b15186e8 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,7 +30,7 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
-  TargetParser
+  #TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index dc88042a66df9..0c41c633dcb3d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1329,8 +1329,6 @@ void AMDGPUPassConfig::addIRPasses() {
       addPass(createLICMPass());
   }
 
-  //addPass(AMDGPUTargetVerifierPass());
-
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
@@ -2074,8 +2072,6 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   // but EarlyCSE can do neither of them.
   if (isPassEnabled(EnableScalarIRPasses))
     addEarlyCSEOrGVNPass(addPass);
-
-  addPass(AMDGPUTargetVerifierPass());
 }
 
 void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 63a7526b9abdc..0eecedaebc7ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -1,3 +1,22 @@
+//===-- AMDGPUTargetVerifier.cpp - AMDGPU -------------------------*- C++ -*-===//
+////
+//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//// See https://llvm.org/LICENSE.txt for license information.
+//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+////
+////===----------------------------------------------------------------------===//
+////
+//// This file defines target verifier interfaces that can be used for some
+//// validation of input to the system, and for checking that transformations
+//// haven't done something bad. In contrast to the Verifier or Lint, the
+//// TargetVerifier looks for constructions invalid to a particular target
+//// machine.
+////
+//// To see what specifically is checked, look at an individual backend's
+//// TargetVerifier.
+////
+////===----------------------------------------------------------------------===//
+
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
 #include "llvm/Analysis/UniformityAnalysis.h"
@@ -25,19 +44,6 @@ using namespace llvm;
   } while (false)
 
 namespace llvm {
-/*class AMDGPUTargetVerify : public TargetVerify {
-public:
-  Module *Mod;
-
-  DominatorTree *DT;
-  PostDominatorTree *PDT;
-  UniformityInfo *UA;
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
-    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
-  void run(Function &F);
-};*/
 
 static bool IsValidInt(const Type *Ty) {
   return Ty->isIntegerTy(1) ||
@@ -147,7 +153,7 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   }
 
   bool runOnFunction(Function &F) override {
-    if (TV->run(F) && FatalErrors) {
+    if (!TV->run(F) && FatalErrors) {
       errs() << "in function " << F.getName() << '\n';
       report_fatal_error("Broken function found, compilation aborted!");
     }
@@ -160,7 +166,6 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
       if (F.isDeclaration())
         IsValid &= TV->run(F);
 
-    //IsValid &= TV->run();
     if (FatalErrors && !IsValid)
       report_fatal_error("Broken module found, compilation aborted!");
     return false;
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index de3ff749e7c3c..992a0c91d93b1 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -1,3 +1,22 @@
+//===-- TargetVerifier.cpp - LLVM IR Target Verifier ----------------*- C++ -*-===//
+////
+///// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+///// See https://llvm.org/LICENSE.txt for license information.
+///// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+/////
+/////===----------------------------------------------------------------------===//
+/////
+///// This file defines target verifier interfaces that can be used for some
+///// validation of input to the system, and for checking that transformations
+///// haven't done something bad. In contrast to the Verifier or Lint, the
+///// TargetVerifier looks for constructions invalid to a particular target
+///// machine.
+/////
+///// To see what specifically is checked, look at TargetVerifier.cpp or an
+///// individual backend's TargetVerifier.
+/////
+/////===----------------------------------------------------------------------===//
+
 #include "llvm/Target/TargetVerifier.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index a060d16e74958..a8f6b999af06e 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -114,6 +114,8 @@ int llvm::compileModuleWithNewPM(
                               VK == VerifierKind::EachPass);
   registerCodeGenCallback(PIC, *Target);
 
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
   MachineFunctionAnalysisManager MFAM;
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
@@ -125,15 +127,13 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
+  PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
 
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
-
   if (!PassPipeline.empty()) {
     // Construct a custom pass pipeline that starts after instruction
     // selection.
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index 503db7b1f8d18..b00bab66c6c3e 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -1,4 +1,4 @@
-//===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
+//===--- llvm-tgt-verify.cpp - Target Verifier ----------------- ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Tool to fuzz instruction selection using libFuzzer.
+// Tool to verify a target.
 //
 //===----------------------------------------------------------------------===//
 
@@ -144,14 +144,11 @@ int main(int argc, char **argv) {
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
-  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
-  //FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
-
   Triple TT(M->getTargetTriple());
   if (!NoLint)
     FPM.addPass(LintPass(false));

>From 32b0c0d5d74b3b78699b78736604e87369ca4217 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 24 Apr 2025 16:02:21 -0400
Subject: [PATCH 13/33] Add pass to legacy PM.

---
 llvm/include/llvm/CodeGen/Passes.h            |  2 +
 llvm/include/llvm/Target/TargetVerifier.h     |  6 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  |  4 +-
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  2 +-
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 45 ----------
 llvm/lib/Target/TargetVerifier.cpp            | 87 ++++++++++++++++++-
 llvm/tools/llc/NewPMDriver.cpp                |  6 +-
 llvm/tools/llc/llc.cpp                        |  4 +
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  1 +
 9 files changed, 105 insertions(+), 52 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 990452fa11fec..53bd70183e476 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -620,6 +620,8 @@ LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
+
+LLVM_ABI FunctionPass *createTargetVerifierLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index fe683311b901c..23ef2e0b8d4ef 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -30,7 +30,7 @@ class Function;
 
 class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 class TargetVerify {
@@ -76,8 +76,8 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  void run(Function &F) {};
-  void run(Function &F, FunctionAnalysisManager &AM);
+  bool run(Function &F);
+  bool run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 2bc54cf7d4891..539f4b24b5110 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -57,6 +57,8 @@ static cl::opt<bool> VerifyAnalysisInvalidation("verify-analysis-invalidation",
 #endif
 );
 
+static cl::opt<bool> VerifyTargetEach("verify-tgt-each");
+
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
 // of this option.  Note that this option has no effect without -print-changed.
@@ -1471,7 +1473,7 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
                                        "\"{0}\", compilation aborted!",
                                        P));
 
-          if (FAM) {
+          if (VerifyTargetEach && FAM) {
             TargetVerify TV(const_cast<Module*>(F->getParent()));
             TV.run(*const_cast<Function*>(F), *FAM);
 	    if (!TV.IsValid)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 652cd23f62443..ad4fa93d09e0c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -89,7 +89,7 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("tgtverifier", TargetVerifierPass())
 #undef VERIFIER_MODULE_ANALYSIS
 #undef VERIFIER_FUNCTION_ANALYSIS
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 0eecedaebc7ce..96bcaaf6f2ac9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -132,49 +132,4 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
   return PreservedAnalyses::all();
 }
 
-struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
-  static char ID;
-
-  std::unique_ptr<AMDGPUTargetVerify> TV;
-  bool FatalErrors = true;
-
-  AMDGPUTargetVerifierLegacyPass() : FunctionPass(ID) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
-      : FunctionPass(ID),
-        FatalErrors(FatalErrors) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool doInitialization(Module &M) override {
-    TV = std::make_unique<AMDGPUTargetVerify>(&M);
-    return false;
-  }
-
-  bool runOnFunction(Function &F) override {
-    if (!TV->run(F) && FatalErrors) {
-      errs() << "in function " << F.getName() << '\n';
-      report_fatal_error("Broken function found, compilation aborted!");
-    }
-    return false;
-  }
-
-  bool doFinalization(Module &M) override {
-    bool IsValid = true;
-    for (Function &F : M)
-      if (F.isDeclaration())
-        IsValid &= TV->run(F);
-
-    if (FatalErrors && !IsValid)
-      report_fatal_error("Broken module found, compilation aborted!");
-    return false;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-  }
-};
-char AMDGPUTargetVerifierLegacyPass::ID = 0;
 } // namespace llvm
-INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverify", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 992a0c91d93b1..170fc4769c1d8 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Target/TargetVerifier.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
+#include "llvm/InitializePasses.h"
 #include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Support/Debug.h"
@@ -32,7 +33,22 @@
 
 namespace llvm {
 
-void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
+bool TargetVerify::run(Function &F) {
+  if (TT.isAMDGPU()) {
+    AMDGPUTargetVerify TV(Mod);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+      return false;
+    }
+    return true;
+  }
+  report_fatal_error("Target has no verification method\n");
+}
+
+bool TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
   if (TT.isAMDGPU()) {
     auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
     auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
@@ -44,8 +60,77 @@ void TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
     dbgs() << TV.MessagesStr.str();
     if (!TV.MessagesStr.str().empty()) {
       TV.IsValid = false;
+      return false;
+    }
+    return true;
+  }
+  report_fatal_error("Target has no verification method\n");
+}
+
+PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto TT = F.getParent()->getTargetTriple();
+
+  if (TT.isAMDGPU()) {
+    auto *Mod = F.getParent();
+
+    auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+    TV.run(F);
+
+    dbgs() << TV.MessagesStr.str();
+    if (!TV.MessagesStr.str().empty()) {
+      TV.IsValid = false;
+      return PreservedAnalyses::none();
     }
+    return PreservedAnalyses::all();
   }
+  report_fatal_error("Target has no verification method\n");
 }
 
+struct TargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<TargetVerify> TV;
+
+  TargetVerifierLegacyPass() : FunctionPass(ID) {
+    initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<TargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (!TV->run(F)) {
+      errs() << "in function " << F.getName() << '\n';
+      report_fatal_error("broken function found, compilation aborted!");
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    if (!IsValid)
+      report_fatal_error("broken module found, compilation aborted!");
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char TargetVerifierLegacyPass::ID = 0;
+FunctionPass *createTargetVerifierLegacyPass() {
+  return new TargetVerifierLegacyPass();
+}
 } // namespace llvm
+using namespace llvm;
+INITIALIZE_PASS(TargetVerifierLegacyPass, "tgtverifier", "Target Verifier", false, false)
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index a8f6b999af06e..4b95977a10c5f 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -57,6 +57,9 @@ static cl::opt<bool>
     DebugPM("debug-pass-manager", cl::Hidden,
             cl::desc("Print pass management debugging information"));
 
+static cl::opt<bool> VerifyTarget("verify-tgt-new-pm",
+		                  cl::desc("Verify the target"));
+
 bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
   DiagnosticHandler::handleDiagnostics(DI);
   if (DI.getKind() == llvm::DK_SrcMgr) {
@@ -127,7 +130,8 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
-  PB.registerVerifierPasses(MPM, FPM);
+  if (VerifyTarget)
+    PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM, &FAM);
 
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 140459ba2de21..1fd8a9f9cd9f8 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -209,6 +209,8 @@ static cl::opt<std::string> PassPipeline(
 static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
                                cl::desc("Alias for -passes"));
 
+static cl::opt<bool> VerifyTarget("verify-tgt", cl::desc("Verify the target"));
+
 namespace {
 
 std::vector<std::string> &getRunPassNames() {
@@ -658,6 +660,8 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
+  if (VerifyTarget)
+    PM.add(createTargetVerifierLegacyPass());
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index b00bab66c6c3e..b86c2318b45b7 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -141,6 +141,7 @@ int main(int argc, char **argv) {
   ModuleAnalysisManager MAM;
   PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
   PB.registerModuleAnalyses(MAM);
+  //PB.registerVerifierPasses(MPM, FPM);
   PB.registerCGSCCAnalyses(CGAM);
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);

>From 387b2d5936a699238ac775726b0b30860f68507f Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 00:13:42 -0400
Subject: [PATCH 14/33] Add fam in other projects.

---
 flang/lib/Frontend/FrontendActions.cpp      | 2 +-
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp | 2 +-
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index d684eeb696755..c15151fc5c1b8 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -947,7 +947,7 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
 
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
-  si.registerCallbacks(pic, &mam);
+  si.registerCallbacks(pic, &mam, &fam);
   if (ci.isTimingEnabled())
     si.getTimePasses().setOutStream(ci.getTimingStreamLLVM());
   pto.LoopUnrolling = opts.UnrollLoops;
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index 0f58391c50667..f9664025f61f1 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -577,7 +577,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index 7117eaf4982b0..eae06d9f57467 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -851,7 +851,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index cb7b6cc8651c1..30ad79ef2fc58 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -970,7 +970,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 91b7191a07c6f..4a39bc33c5591 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1139,7 +1139,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
 
   // Add transform passes.
   // Promote allocas to registers.

>From d2c9425f9e951768842ca9771148611bdba5e1bb Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 02:30:40 -0400
Subject: [PATCH 15/33] Avoid fatal errors in llc.

---
 llvm/lib/Target/TargetVerifier.cpp             | 18 +++++++++++++-----
 .../test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll |  2 +-
 .../test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll |  2 +-
 llvm/tools/llc/llc.cpp                         |  2 +-
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 170fc4769c1d8..3be50f4ef6da3 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -94,8 +94,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   static char ID;
 
   std::unique_ptr<TargetVerify> TV;
+  bool FatalErrors = false;
 
-  TargetVerifierLegacyPass() : FunctionPass(ID) {
+  TargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
+    FatalErrors(FatalErrors) {
     initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
@@ -107,7 +109,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   bool runOnFunction(Function &F) override {
     if (!TV->run(F)) {
       errs() << "in function " << F.getName() << '\n';
-      report_fatal_error("broken function found, compilation aborted!");
+      if (FatalErrors)
+        report_fatal_error("broken function found, compilation aborted!");
+      else
+        errs() << "broken function found, compilation aborted!\n";
     }
     return false;
   }
@@ -119,7 +124,10 @@ struct TargetVerifierLegacyPass : public FunctionPass {
         IsValid &= TV->run(F);
 
     if (!IsValid)
-      report_fatal_error("broken module found, compilation aborted!");
+      if (FatalErrors)
+        report_fatal_error("broken module found, compilation aborted!");
+      else
+        errs() << "broken module found, compilation aborted!\n";
     return false;
   }
 
@@ -128,8 +136,8 @@ struct TargetVerifierLegacyPass : public FunctionPass {
   }
 };
 char TargetVerifierLegacyPass::ID = 0;
-FunctionPass *createTargetVerifierLegacyPass() {
-  return new TargetVerifierLegacyPass();
+FunctionPass *createTargetVerifierLegacyPass(bool FatalErrors) {
+  return new TargetVerifierLegacyPass(FatalErrors);
 }
 } // namespace llvm
 using namespace llvm;
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index c5e59d4a2369e..e2d9edda5d008 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
 ; CHECK: Shaders must return void
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index 8a503b7624a73..a2dab0ff47924 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -enable-new-pm %s -o - 2>&1 | FileCheck %s --allow-empty
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
 ; CHECK-NOT: Shaders must return void
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 1fd8a9f9cd9f8..329d95826551f 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -661,7 +661,7 @@ static int compileModule(char **argv, LLVMContext &Context) {
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
   if (VerifyTarget)
-    PM.add(createTargetVerifierLegacyPass());
+    PM.add(createTargetVerifierLegacyPass(false));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {

>From 31a79a57d9d41365ffcd7b263b2a05153463b2f5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Sat, 26 Apr 2025 03:40:47 -0400
Subject: [PATCH 16/33] Add tool to build/test.

---
 llvm/test/CMakeLists.txt                      |  1 +
 llvm/test/lit.cfg.py                          |  1 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |  1 +
 .../llvm/tools/llvm-tgt-verify/BUILD.gn       | 25 +++++++++++++++++++
 4 files changed, 28 insertions(+)
 create mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn

diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 2a6135da9a61e..a2bb112db8746 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -136,6 +136,7 @@ set(LLVM_TEST_DEPENDS
           llvm-strip
           llvm-symbolizer
           llvm-tblgen
+          llvm-tgt-verify
           llvm-readtapi
           llvm-tli-checker
           llvm-undname
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index bd6e37c848d8c..bf06e80dc7ae5 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -228,6 +228,7 @@ def get_asan_rtlib():
         "llvm-strings",
         "llvm-strip",
         "llvm-tblgen",
+        "llvm-tgt-verify",
         "llvm-readtapi",
         "llvm-undname",
         "llvm-windres",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 946b63f8a54fb..36cf956545caf 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -320,6 +320,7 @@ group("test") {
     "//llvm/tools/llvm-strings",
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-tli-checker",
+    "//llvm/tools/llvm-tgt-verify",
     "//llvm/tools/llvm-undname",
     "//llvm/tools/llvm-xray",
     "//llvm/tools/lto",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
new file mode 100644
index 0000000000000..b751bafc5052c
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tgtverifier("llvm-tgt-verify") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmPrinter",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGenTypes",
+    "//llvm/lib/Core",
+    "//llvm/lib/IRPrinter",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/MC",
+    "//llvm/lib/MIRParser",
+    "//llvm/lib/Passes",
+    "//llvm/lib/Remarks",
+    "//llvm/lib/ScalarOpts",
+    "//llvm/lib/SelectionDAG",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/TargetParser",
+    "//llvm/lib/TransformUtils",
+    "//llvm/lib/Vectorize",
+  ]
+  sources = [ "llvm-tgt-verify.cpp" ]
+}

>From b51b530c5672945e06a20b93b1f652d097386e61 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 28 Apr 2025 10:42:24 -0400
Subject: [PATCH 17/33] Cleanup of unrequired functions.

---
 llvm/include/llvm/Target/TargetVerifier.h     |  1 -
 .../TargetVerify/AMDGPUTargetVerifier.h       |  1 -
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 25 +++----------------
 llvm/lib/Target/TargetVerifier.cpp            | 22 ++--------------
 4 files changed, 6 insertions(+), 43 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 23ef2e0b8d4ef..427a05b2648a9 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -77,7 +77,6 @@ class TargetVerify {
         MessagesStr(Messages) {}
 
   bool run(Function &F);
-  bool run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index b6a7412e8c1ef..74e5b5f7a1efd 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -32,7 +32,6 @@ class Function;
 
 class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 class AMDGPUTargetVerify : public TargetVerify {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 96bcaaf6f2ac9..bda412f723242 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -107,29 +107,12 @@ bool AMDGPUTargetVerify::run(Function &F) {
     }
   }
 
-  if (!MessagesStr.str().empty())
+  //dbgs() << MessagesStr.str();
+  if (!MessagesStr.str().empty()) {
+    //IsValid = false;
     return false;
-  return true;
-}
-
-PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
-
-  auto *Mod = F.getParent();
-
-  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-  TV.run(F);
-
-  dbgs() << TV.MessagesStr.str();
-  if (!TV.MessagesStr.str().empty()) {
-    TV.IsValid = false;
-    return PreservedAnalyses::none();
   }
-
-  return PreservedAnalyses::all();
+  return true;
 }
 
 } // namespace llvm
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 3be50f4ef6da3..6b57c18ff9316 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -48,25 +48,6 @@ bool TargetVerify::run(Function &F) {
   report_fatal_error("Target has no verification method\n");
 }
 
-bool TargetVerify::run(Function &F, FunctionAnalysisManager &AM) {
-  if (TT.isAMDGPU()) {
-    auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
-    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return false;
-    }
-    return true;
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
 PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto TT = F.getParent()->getTargetTriple();
 
@@ -123,11 +104,12 @@ struct TargetVerifierLegacyPass : public FunctionPass {
       if (F.isDeclaration())
         IsValid &= TV->run(F);
 
-    if (!IsValid)
+    if (!IsValid) {
       if (FatalErrors)
         report_fatal_error("broken module found, compilation aborted!");
       else
         errs() << "broken module found, compilation aborted!\n";
+    }
     return false;
   }
 

>From d41d5f9e6a33b70f71a20d58642fd73c9327e1e5 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Mon, 28 Apr 2025 10:43:32 -0400
Subject: [PATCH 18/33] Make virtual.

---
 llvm/include/llvm/Target/TargetVerifier.h                    | 2 +-
 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 427a05b2648a9..ade2676a64325 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -76,7 +76,7 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  bool run(Function &F);
+  virtual bool run(Function &F);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 74e5b5f7a1efd..b97fbc046e391 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -48,7 +48,7 @@ class AMDGPUTargetVerify : public TargetVerify {
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
-  bool run(Function &F);
+  bool run(Function &F) override;
 };
 
 } // namespace llvm

>From 7ece5e262a4f4aff350a511af5d33496ac2d5604 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 10:56:43 -0400
Subject: [PATCH 19/33] Remove from legacy PM. Add to target dependent
 pipeline.

---
 llvm/include/llvm/InitializePasses.h          |  2 +-
 llvm/include/llvm/Target/TargetVerifier.h     |  4 +-
 .../TargetVerify/AMDGPUTargetVerifier.h       |  1 +
 llvm/lib/Passes/StandardInstrumentations.cpp  | 10 +--
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  2 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  2 +
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 74 ++++++++++++++-
 llvm/lib/Target/TargetVerifier.cpp            | 90 -------------------
 llvm/tools/llc/llc.cpp                        |  2 -
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 -
 10 files changed, 84 insertions(+), 105 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 662f6b71aa46a..d718f92877276 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,7 +333,7 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
-LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+//LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index ade2676a64325..1d12eb55bbf0a 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -30,7 +30,7 @@ class Function;
 
 class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  virtual PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) = 0;
 };
 
 class TargetVerify {
@@ -76,7 +76,7 @@ class TargetVerify {
       : Mod(Mod), TT(Mod->getTargetTriple()),
         MessagesStr(Messages) {}
 
-  virtual bool run(Function &F);
+  virtual bool run(Function &F) = 0;
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index b97fbc046e391..49bcbc8849e3c 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -32,6 +32,7 @@ class Function;
 
 class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
 };
 
 class AMDGPUTargetVerify : public TargetVerify {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 539f4b24b5110..905ba667fac7a 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,7 +40,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
-#include "llvm/Target/TargetVerifier.h"
+//#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -1474,12 +1474,12 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
                                        P));
 
           if (VerifyTargetEach && FAM) {
-            TargetVerify TV(const_cast<Module*>(F->getParent()));
-            TV.run(*const_cast<Function*>(F), *FAM);
-	    if (!TV.IsValid)
+            //TargetVerify TV(const_cast<Module*>(F->getParent()));
+            //TV.run(*const_cast<Function*>(F), *FAM);
+	    /*if (!TV.IsValid)
               report_fatal_error(formatv("Broken function found after pass "
                                          "\"{0}\", compilation aborted!",
-                                         P));
+                                         P));*/
           }
         } else {
           const auto *M = unwrapIR<Module>(IR);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index ad4fa93d09e0c..652cd23f62443 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -89,7 +89,7 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("tgtverifier", TargetVerifierPass())
+VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
 #undef VERIFIER_MODULE_ANALYSIS
 #undef VERIFIER_FUNCTION_ANALYSIS
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 0c41c633dcb3d..718655b7ebf00 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -2010,6 +2010,8 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
 }
 
 void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
+  addPass(AMDGPUTargetVerifierPass());
+
   if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
     addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index bda412f723242..cedd9ddc78011 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -107,12 +107,82 @@ bool AMDGPUTargetVerify::run(Function &F) {
     }
   }
 
-  //dbgs() << MessagesStr.str();
+  dbgs() << MessagesStr.str();
   if (!MessagesStr.str().empty()) {
-    //IsValid = false;
+    IsValid = false;
     return false;
   }
   return true;
 }
 
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto *Mod = F.getParent();
+
+  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
+  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
+
+  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  TV.run(F);
+
+  dbgs() << TV.MessagesStr.str();
+  if (!TV.MessagesStr.str().empty()) {
+    TV.IsValid = false;
+    return PreservedAnalyses::none();
+  }
+  return PreservedAnalyses::all();
+}
+
+/*
+struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
+  static char ID;
+
+  std::unique_ptr<AMDGPUTargetVerify> TV;
+  bool FatalErrors = false;
+
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
+    FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override {
+    TV = std::make_unique<AMDGPUTargetVerify>(&M);
+    return false;
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (!TV->run(F)) {
+      errs() << "in function " << F.getName() << '\n';
+      if (FatalErrors)
+        report_fatal_error("broken function found, compilation aborted!");
+      else
+        errs() << "broken function found, compilation aborted!\n";
+    }
+    return false;
+  }
+
+  bool doFinalization(Module &M) override {
+    bool IsValid = true;
+    for (Function &F : M)
+      if (F.isDeclaration())
+        IsValid &= TV->run(F);
+
+    if (!IsValid) {
+      if (FatalErrors)
+        report_fatal_error("broken module found, compilation aborted!");
+      else
+        errs() << "broken module found, compilation aborted!\n";
+    }
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+};
+char AMDGPUTargetVerifierLegacyPass::ID = 0;
+FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
+  return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
+}*/
 } // namespace llvm
+//INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
index 6b57c18ff9316..c63ae2a2c5daf 100644
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ b/llvm/lib/Target/TargetVerifier.cpp
@@ -33,94 +33,4 @@
 
 namespace llvm {
 
-bool TargetVerify::run(Function &F) {
-  if (TT.isAMDGPU()) {
-    AMDGPUTargetVerify TV(Mod);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return false;
-    }
-    return true;
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
-PreservedAnalyses TargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
-  auto TT = F.getParent()->getTargetTriple();
-
-  if (TT.isAMDGPU()) {
-    auto *Mod = F.getParent();
-
-    auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-    auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
-    TV.run(F);
-
-    dbgs() << TV.MessagesStr.str();
-    if (!TV.MessagesStr.str().empty()) {
-      TV.IsValid = false;
-      return PreservedAnalyses::none();
-    }
-    return PreservedAnalyses::all();
-  }
-  report_fatal_error("Target has no verification method\n");
-}
-
-struct TargetVerifierLegacyPass : public FunctionPass {
-  static char ID;
-
-  std::unique_ptr<TargetVerify> TV;
-  bool FatalErrors = false;
-
-  TargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
-    FatalErrors(FatalErrors) {
-    initializeTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool doInitialization(Module &M) override {
-    TV = std::make_unique<TargetVerify>(&M);
-    return false;
-  }
-
-  bool runOnFunction(Function &F) override {
-    if (!TV->run(F)) {
-      errs() << "in function " << F.getName() << '\n';
-      if (FatalErrors)
-        report_fatal_error("broken function found, compilation aborted!");
-      else
-        errs() << "broken function found, compilation aborted!\n";
-    }
-    return false;
-  }
-
-  bool doFinalization(Module &M) override {
-    bool IsValid = true;
-    for (Function &F : M)
-      if (F.isDeclaration())
-        IsValid &= TV->run(F);
-
-    if (!IsValid) {
-      if (FatalErrors)
-        report_fatal_error("broken module found, compilation aborted!");
-      else
-        errs() << "broken module found, compilation aborted!\n";
-    }
-    return false;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-  }
-};
-char TargetVerifierLegacyPass::ID = 0;
-FunctionPass *createTargetVerifierLegacyPass(bool FatalErrors) {
-  return new TargetVerifierLegacyPass(FatalErrors);
-}
 } // namespace llvm
-using namespace llvm;
-INITIALIZE_PASS(TargetVerifierLegacyPass, "tgtverifier", "Target Verifier", false, false)
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 329d95826551f..2e9e4837fe467 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -660,8 +660,6 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // Build up all of the passes that we want to do to the module.
   legacy::PassManager PM;
-  if (VerifyTarget)
-    PM.add(createTargetVerifierLegacyPass(false));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   {
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index b86c2318b45b7..d832dcdff4ad0 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -158,8 +158,6 @@ int main(int argc, char **argv) {
   if (TT.isAMDGPU())
     FPM.addPass(AMDGPUTargetVerifierPass());
   else if (false) {} // ...
-  else
-    FPM.addPass(TargetVerifierPass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 
   auto PA = MPM.run(*M, MAM);

>From 42b89826fa18b4796c9fdee25864e929a2351661 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 11:18:50 -0400
Subject: [PATCH 20/33] Add back to legacy PM.

---
 llvm/include/llvm/InitializePasses.h            | 1 -
 llvm/lib/Target/AMDGPU/AMDGPU.h                 | 3 +++
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  | 1 +
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 8 ++++----
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index d718f92877276..1b5b1d5888824 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -333,7 +333,6 @@ LLVM_ABI void initializeUnpackMachineBundlesPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
-//LLVM_ABI void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 5a917734e9c74..26fb0a967fd76 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -549,6 +549,9 @@ extern char &GCNRewritePartialRegUsesID;
 void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
 extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
+FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
+void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 718655b7ebf00..8b0b8f6df740e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1411,6 +1411,7 @@ bool AMDGPUPassConfig::addGCPasses() {
 //===----------------------------------------------------------------------===//
 
 bool GCNPassConfig::addPreISel() {
+  addPass(createAMDGPUTargetVerifierLegacyPass(false));
   AMDGPUPassConfig::addPreISel();
 
   if (TM->getOptLevel() > CodeGenOptLevel::None)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index cedd9ddc78011..c4d303bee6ef8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -17,6 +17,7 @@
 ////
 ////===----------------------------------------------------------------------===//
 
+#include "AMDGPU.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 
 #include "llvm/Analysis/UniformityAnalysis.h"
@@ -24,7 +25,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-#include "llvm/InitializePasses.h"
+//#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -133,7 +134,6 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisMan
   return PreservedAnalyses::all();
 }
 
-/*
 struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   static char ID;
 
@@ -183,6 +183,6 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
 char AMDGPUTargetVerifierLegacyPass::ID = 0;
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
   return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
-}*/
+}
 } // namespace llvm
-//INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)

>From 230f33b8db0650a59c5656d2caf5c7be31f0aa49 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 14:04:10 -0400
Subject: [PATCH 21/33] Remove reference to FAM in registerCallbacks and
 VerifyEach for TargetVerify in instrumentation

---
 clang/lib/CodeGen/BackendUtil.cpp             |  2 +-
 flang/lib/Frontend/FrontendActions.cpp        |  2 +-
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp   |  2 +-
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp   |  2 +-
 .../llvm/Passes/StandardInstrumentations.h    |  6 ++----
 llvm/lib/LTO/LTOBackend.cpp                   |  2 +-
 llvm/lib/LTO/ThinLTOCodeGenerator.cpp         |  2 +-
 llvm/lib/Passes/PassBuilderBindings.cpp       |  2 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  | 21 ++++---------------
 llvm/tools/llc/NewPMDriver.cpp                |  7 ++++---
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp |  2 +-
 llvm/tools/opt/NewPMDriver.cpp                |  2 +-
 llvm/unittests/IR/PassManagerTest.cpp         |  6 +++---
 15 files changed, 24 insertions(+), 38 deletions(-)

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 560129f871026..7e0a3cf5591ce 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -917,7 +917,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       TheModule->getContext(),
       (CodeGenOpts.DebugPassManager || DebugPassStructure),
       CodeGenOpts.VerifyEach, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
   // Handle the assignment tracking feature options.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index c15151fc5c1b8..d684eeb696755 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -947,7 +947,7 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
 
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
-  si.registerCallbacks(pic, &mam, &fam);
+  si.registerCallbacks(pic, &mam);
   if (ci.isTimingEnabled())
     si.getTimePasses().setOutStream(ci.getTimingStreamLLVM());
   pto.LoopUnrolling = opts.UnrollLoops;
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index f9664025f61f1..0f58391c50667 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -577,7 +577,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index eae06d9f57467..7117eaf4982b0 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -851,7 +851,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index 30ad79ef2fc58..cb7b6cc8651c1 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -970,7 +970,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 4a39bc33c5591..91b7191a07c6f 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1139,7 +1139,7 @@ static void InitializeModuleAndManagers() {
   ThePIC = std::make_unique<PassInstrumentationCallbacks>();
   TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
                                                      /*DebugLogging*/ true);
-  TheSI->registerCallbacks(*ThePIC, TheMAM.get(), TheFAM.get());
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
   // Add transform passes.
   // Promote allocas to registers.
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 7d7b858b447cc..df9d8de8a025a 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,8 +478,7 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM,
-			 FunctionAnalysisManager *FAM);
+                         ModuleAnalysisManager *MAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -625,8 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM,
-			 FunctionAnalysisManager *FAM);
+                         ModuleAnalysisManager *MAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 7ae23ef4e7df4..5e8cd12fe040b 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -273,7 +273,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,
                               Conf.VerifyEach);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
 
   RegisterPassPlugins(Conf.PassPlugins, PB);
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 8db92e70d5a96..5b333cda31fb8 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -241,7 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   PipelineTuningOptions PTO;
   PTO.LoopVectorization = true;
   PTO.SLPVectorization = true;
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index f0e1abb8cebc4..933fe89e53a94 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -76,7 +76,7 @@ static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   // Run the pipeline.
   if (Fun) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 905ba667fac7a..4e8abc43572d7 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -40,7 +40,6 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
-//#include "llvm/Target/TargetVerifier.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -57,8 +56,6 @@ static cl::opt<bool> VerifyAnalysisInvalidation("verify-analysis-invalidation",
 #endif
 );
 
-static cl::opt<bool> VerifyTargetEach("verify-tgt-each");
-
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
 // of this option.  Note that this option has no effect without -print-changed.
@@ -1452,10 +1449,9 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 }
 
 void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                              ModuleAnalysisManager *MAM,
-					      FunctionAnalysisManager *FAM) {
+                                              ModuleAnalysisManager *MAM) {
   PIC.registerAfterPassCallback(
-      [this, MAM, FAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+      [this, MAM](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
         const auto *F = unwrapIR<Function>(IR);
@@ -1472,15 +1468,6 @@ void VerifyInstrumentation::registerCallbacks(PassInstrumentationCallbacks &PIC,
             report_fatal_error(formatv("Broken function found after pass "
                                        "\"{0}\", compilation aborted!",
                                        P));
-
-          if (VerifyTargetEach && FAM) {
-            //TargetVerify TV(const_cast<Module*>(F->getParent()));
-            //TV.run(*const_cast<Function*>(F), *FAM);
-	    /*if (!TV.IsValid)
-              report_fatal_error(formatv("Broken function found after pass "
-                                         "\"{0}\", compilation aborted!",
-                                         P));*/
-          }
         } else {
           const auto *M = unwrapIR<Module>(IR);
           if (!M) {
@@ -2520,7 +2507,7 @@ void PrintCrashIRInstrumentation::registerCallbacks(
 }
 
 void StandardInstrumentations::registerCallbacks(
-    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM, FunctionAnalysisManager *FAM) {
+    PassInstrumentationCallbacks &PIC, ModuleAnalysisManager *MAM) {
   PrintIR.registerCallbacks(PIC);
   PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
@@ -2529,7 +2516,7 @@ void StandardInstrumentations::registerCallbacks(
   PrintChangedIR.registerCallbacks(PIC);
   PseudoProbeVerification.registerCallbacks(PIC);
   if (VerifyEach)
-    Verify.registerCallbacks(PIC, MAM, FAM);
+    Verify.registerCallbacks(PIC, MAM);
   PrintChangedDiff.registerCallbacks(PIC);
   WebsiteChangeReporter.registerCallbacks(PIC);
   ChangeTester.registerCallbacks(PIC);
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 4b95977a10c5f..863a555798dab 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -117,8 +117,6 @@ int llvm::compileModuleWithNewPM(
                               VK == VerifierKind::EachPass);
   registerCodeGenCallback(PIC, *Target);
 
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
   MachineFunctionAnalysisManager MFAM;
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
@@ -133,11 +131,14 @@ int llvm::compileModuleWithNewPM(
   if (VerifyTarget)
     PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); });
   MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
 
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+
   if (!PassPipeline.empty()) {
     // Construct a custom pass pipeline that starts after instruction
     // selection.
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
index d832dcdff4ad0..50f4e56bb6af6 100644
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
@@ -148,7 +148,7 @@ int main(int argc, char **argv) {
   PB.registerMachineFunctionAnalyses(MFAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
 
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
 
   Triple TT(M->getTargetTriple());
   if (!NoLint)
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index a8977d80bdf44..7d168a6ceb17c 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -423,7 +423,7 @@ bool llvm::runPassPipeline(
   PrintPassOpts.SkipAnalyses = DebugPM == DebugLogging::Quiet;
   StandardInstrumentations SI(M.getContext(), DebugPM != DebugLogging::None,
                               VK == VerifierKind::EachPass, PrintPassOpts);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   DebugifyEachInstrumentation Debugify;
   DebugifyStatsMap DIStatsMap;
   DebugInfoPerPass DebugInfoBeforePass;
diff --git a/llvm/unittests/IR/PassManagerTest.cpp b/llvm/unittests/IR/PassManagerTest.cpp
index bb4db6120035f..a6487169224c2 100644
--- a/llvm/unittests/IR/PassManagerTest.cpp
+++ b/llvm/unittests/IR/PassManagerTest.cpp
@@ -828,7 +828,7 @@ TEST_F(PassManagerTest, FunctionPassCFGChecker) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -877,7 +877,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerInvalidateAnalysis) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
@@ -945,7 +945,7 @@ TEST_F(PassManagerTest, FunctionPassCFGCheckerWrapped) {
   FunctionPassManager FPM;
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI(M->getContext(), /*DebugLogging*/ true);
-  SI.registerCallbacks(PIC, &MAM, &FAM);
+  SI.registerCallbacks(PIC, &MAM);
   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
   MAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });
   FAM.registerPass([&] { return PassInstrumentationAnalysis(&PIC); });

>From c680041d5134d6edd47e8dec65d4986feb63fb7d Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 14:44:28 -0400
Subject: [PATCH 22/33] Remove references to registry

---
 llvm/include/llvm/Passes/PassBuilder.h        | 21 -------------------
 .../llvm/Passes/StandardInstrumentations.h    |  2 +-
 .../llvm/Passes/TargetPassRegistry.inc        | 12 -----------
 llvm/lib/Passes/PassBuilder.cpp               |  7 -------
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 11 ----------
 llvm/tools/llc/NewPMDriver.cpp                |  5 -----
 llvm/tools/llc/llc.cpp                        |  2 --
 7 files changed, 1 insertion(+), 59 deletions(-)

diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index c134f2d0c1112..f13b5c678a894 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -173,13 +173,6 @@ class PassBuilder {
   /// additional analyses.
   LLVM_ABI void registerLoopAnalyses(LoopAnalysisManager &LAM);
 
-  /// Registers all available verifier passes.
-  ///
-  /// This is an interface that can be used to populate a
-  /// \c ModuleAnalysisManager with all registered loop analyses. Callers can
-  /// still manually register any additional analyses.
-  void registerVerifierPasses(ModulePassManager &PM, FunctionPassManager &);
-
   /// Registers all available machine function analysis passes.
   ///
   /// This is an interface that can be used to populate a \c
@@ -583,15 +576,6 @@ class PassBuilder {
   }
   /// @}}
 
-  /// Register a callback for parsing an Verifier Name to populate
-  /// the given managers.
-  void registerVerifierCallback(
-      const std::function<bool(ModulePassManager &MPM)> &C,
-      const std::function<bool(FunctionPassManager &MPM)> &CF) {
-    VerifierCallbacks.push_back(C);
-    FnVerifierCallbacks.push_back(CF);
-  }
-
   /// {{@ Register pipeline parsing callbacks with this pass builder instance.
   /// Using these callbacks, callers can parse both a single pass name, as well
   /// as entire sub-pipelines, and populate the PassManager instance
@@ -867,11 +851,6 @@ class PassBuilder {
   // Callbacks to parse `filter` parameter in register allocation passes
   SmallVector<std::function<RegAllocFilterFunc(StringRef)>, 2>
       RegClassFilterParsingCallbacks;
-  // Verifier callbacks
-  SmallVector<std::function<bool(ModulePassManager &)>, 2>
-      VerifierCallbacks;
-  SmallVector<std::function<bool(FunctionPassManager &)>, 2>
-      FnVerifierCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index df9d8de8a025a..d7c1bbf04c328 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -624,7 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };
diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc
index 2d04b874cf360..521913cb25a4a 100644
--- a/llvm/include/llvm/Passes/TargetPassRegistry.inc
+++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc
@@ -151,18 +151,6 @@ PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM,
   return false;
 });
 
-PB.registerVerifierCallback([](ModulePassManager &PM) {
-#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS) PM.addPass(CREATE_PASS)
-#include GET_PASS_REGISTRY
-#undef VERIFIER_MODULE_ANALYSIS
-  return false;
-}, [](FunctionPassManager &FPM) {
-#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS) FPM.addPass(CREATE_PASS)
-#include GET_PASS_REGISTRY
-#undef VERIFIER_FUNCTION_ANALYSIS
-  return false;
-});
-
 #undef ADD_PASS
 #undef ADD_PASS_WITH_PARAMS
 
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 561996d0c8980..4603eaff8ade9 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -576,13 +576,6 @@ void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     C(LAM);
 }
 
-void PassBuilder::registerVerifierPasses(ModulePassManager &MPM, FunctionPassManager &FPM) {
-  for (auto &C : VerifierCallbacks)
-    C(MPM);
-  for (auto &C : FnVerifierCallbacks)
-    C(FPM);
-}
-
 static std::optional<std::pair<bool, bool>>
 parseFunctionPipelineName(StringRef Name) {
   std::pair<bool, bool> Params;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 652cd23f62443..13453963eec6d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -82,17 +82,6 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
-#ifndef VERIFIER_MODULE_ANALYSIS
-#define VERIFIER_MODULE_ANALYSIS(NAME, CREATE_PASS)
-#endif
-#ifndef VERIFIER_FUNCTION_ANALYSIS
-#define VERIFIER_FUNCTION_ANALYSIS(NAME, CREATE_PASS)
-#endif
-VERIFIER_MODULE_ANALYSIS("verifier", VerifierPass())
-VERIFIER_FUNCTION_ANALYSIS("amdgpu-tgtverifier", AMDGPUTargetVerifierPass())
-#undef VERIFIER_MODULE_ANALYSIS
-#undef VERIFIER_FUNCTION_ANALYSIS
-
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 863a555798dab..fa82689ecf9ae 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -57,9 +57,6 @@ static cl::opt<bool>
     DebugPM("debug-pass-manager", cl::Hidden,
             cl::desc("Print pass management debugging information"));
 
-static cl::opt<bool> VerifyTarget("verify-tgt-new-pm",
-		                  cl::desc("Verify the target"));
-
 bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
   DiagnosticHandler::handleDiagnostics(DI);
   if (DI.getKind() == llvm::DK_SrcMgr) {
@@ -128,8 +125,6 @@ int llvm::compileModuleWithNewPM(
   PB.registerFunctionAnalyses(FAM);
   PB.registerLoopAnalyses(LAM);
   PB.registerMachineFunctionAnalyses(MFAM);
-  if (VerifyTarget)
-    PB.registerVerifierPasses(MPM, FPM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
   SI.registerCallbacks(PIC, &MAM);
 
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 2e9e4837fe467..140459ba2de21 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -209,8 +209,6 @@ static cl::opt<std::string> PassPipeline(
 static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
                                cl::desc("Alias for -passes"));
 
-static cl::opt<bool> VerifyTarget("verify-tgt", cl::desc("Verify the target"));
-
 namespace {
 
 std::vector<std::string> &getRunPassNames() {

>From 3f5634a532856668a42f44ea04d0ffcc862c96d4 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 19:05:53 -0400
Subject: [PATCH 23/33] Remove int check

---
 .../lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index c4d303bee6ef8..2ca0bbeb57653 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -25,7 +25,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-//#include "llvm/InitializePasses.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
@@ -46,15 +45,6 @@ using namespace llvm;
 
 namespace llvm {
 
-static bool IsValidInt(const Type *Ty) {
-  return Ty->isIntegerTy(1) ||
-         Ty->isIntegerTy(8) ||
-         Ty->isIntegerTy(16) ||
-         Ty->isIntegerTy(32) ||
-         Ty->isIntegerTy(64) ||
-         Ty->isIntegerTy(128);
-}
-
 static bool isShader(CallingConv::ID CC) {
   switch(CC) {
     case CallingConv::AMDGPU_VS:
@@ -81,14 +71,6 @@ bool AMDGPUTargetVerify::run(Function &F) {
 
     for (auto &I : BB) {
 
-      // Ensure integral types are valid: i8, i16, i32, i64, i128
-      if (I.getType()->isIntegerTy())
-        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
-      for (unsigned i = 0; i < I.getNumOperands(); ++i)
-        if (I.getOperand(i)->getType()->isIntegerTy())
-          Check(IsValidInt(I.getOperand(i)->getType()),
-                "Int type is invalid.", I.getOperand(i));
-
       if (auto *CI = dyn_cast<CallInst>(&I))
       {
         // Ensure no kernel to kernel calls.

>From 6007895007af8925e32b709009c0d6131141c282 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 21:56:28 -0400
Subject: [PATCH 24/33] Remove modifications to Lint/Verifier.

---
 llvm/lib/Analysis/Lint.cpp |  4 +---
 llvm/lib/IR/Verifier.cpp   | 20 ++++----------------
 2 files changed, 5 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index c8627e6db0692..1168005f48c0e 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -735,11 +735,9 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) {
   Lint L(Mod, DL, AA, AC, DT, TLI);
   L.visit(F);
   dbgs() << L.MessagesStr.str();
-  if (AbortOnError && !L.MessagesStr.str().empty()) {
+  if (AbortOnError && !L.MessagesStr.str().empty())
     report_fatal_error(
         "linter found errors, aborting. (enabled by abort-on-error)", false);
-    return PreservedAnalyses::none();
-  }
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 439823ba8b682..1f1041b259736 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -135,10 +135,6 @@ static cl::opt<bool> VerifyNoAliasScopeDomination(
     cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
              "scopes are not dominating"));
 
-static cl::opt<bool>
-    VerifyAbortOnError("verifier-abort-on-error", cl::init(false),
-                       cl::desc("In the Verifier pass, abort on errors."));
-
 namespace llvm {
 
 struct VerifierSupport {
@@ -7869,24 +7865,16 @@ VerifierAnalysis::Result VerifierAnalysis::run(Function &F,
 
 PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto Res = AM.getResult<VerifierAnalysis>(M);
-  if (Res.IRBroken || Res.DebugInfoBroken) {
-    //M.IsValid = false;
-    if (VerifyAbortOnError && FatalErrors)
-      report_fatal_error("Broken module found, compilation aborted!");
-    return PreservedAnalyses::none();
-  }
+  if (FatalErrors && (Res.IRBroken || Res.DebugInfoBroken))
+    report_fatal_error("Broken module found, compilation aborted!");
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto res = AM.getResult<VerifierAnalysis>(F);
-  if (res.IRBroken) {
-    //F.getParent()->IsValid = false;
-    if (VerifyAbortOnError && FatalErrors)
-      report_fatal_error("Broken function found, compilation aborted!");
-    return PreservedAnalyses::none();
-  }
+  if (res.IRBroken && FatalErrors)
+    report_fatal_error("Broken function found, compilation aborted!");
 
   return PreservedAnalyses::all();
 }

>From 756c3ebee47f59f69a267da6079edafce4f595ca Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 22:21:47 -0400
Subject: [PATCH 25/33] Remove llvm-tgt-verify tool.

---
 llvm/test/CMakeLists.txt                      |   1 -
 llvm/test/CodeGen/AMDGPU/tgt-verify.ll        |  45 -----
 llvm/test/lit.cfg.py                          |   1 -
 llvm/tools/llvm-tgt-verify/CMakeLists.txt     |  34 ----
 .../tools/llvm-tgt-verify/llvm-tgt-verify.cpp | 171 ------------------
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |   1 -
 .../llvm/tools/llvm-tgt-verify/BUILD.gn       |  25 ---
 7 files changed, 278 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AMDGPU/tgt-verify.ll
 delete mode 100644 llvm/tools/llvm-tgt-verify/CMakeLists.txt
 delete mode 100644 llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
 delete mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn

diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index a2bb112db8746..2a6135da9a61e 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -136,7 +136,6 @@ set(LLVM_TEST_DEPENDS
           llvm-strip
           llvm-symbolizer
           llvm-tblgen
-          llvm-tgt-verify
           llvm-readtapi
           llvm-tli-checker
           llvm-undname
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
deleted file mode 100644
index 62b220d7d9f49..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; RUN: not llvm-tgt-verify %s -mtriple=amdgcn |& FileCheck %s
-
-define amdgpu_cs i32 @shader() {
-; CHECK: Shaders must return void
-  ret i32 0
-}
-
-define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
-; CHECK: Undefined behavior: Write to memory in const addrspace
-; CHECK-NEXT:   store i32 %r, ptr addrspace(4) %out, align 4
-  %r = add i32 %a, %b
-  store i32 %r, ptr addrspace(4) %out
-  ret void
-}
-
-define amdgpu_kernel void @kernel_callee(ptr %x) {
-  ret void
-}
-
-define amdgpu_kernel void @kernel_caller(ptr %x) {
-; CHECK: A kernel may not call a kernel
-; CHECK-NEXT: ptr @kernel_caller
-  call amdgpu_kernel void @kernel_callee(ptr %x)
-  ret void
-}
-
-
-; Function Attrs: nounwind
-define i65 @invalid_type(i65 %x) #0 {
-; CHECK: Int type is invalid.
-; CHECK-NEXT: %tmp2 = ashr i65 %x, 64
-entry:
-  %tmp2 = ashr i65 %x, 64
-  ret i65 %tmp2
-}
-
-declare void @llvm.amdgcn.cs.chain.v3i32(ptr, i32, <3 x i32>, <3 x i32>, i32, ...)
-declare amdgpu_cs_chain void @chain_callee(<3 x i32> inreg, <3 x i32>)
-
-define amdgpu_cs void @no_unreachable(<3 x i32> inreg %a, <3 x i32> %b) {
-; CHECK: llvm.amdgcn.cs.chain must be followed by unreachable
-; CHECK-NEXT: call void (ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.v3i32.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
-  call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr @chain_callee, i32 -1, <3 x i32> inreg %a, <3 x i32> %b, i32 0)
-  ret void
-}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index bf06e80dc7ae5..bd6e37c848d8c 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -228,7 +228,6 @@ def get_asan_rtlib():
         "llvm-strings",
         "llvm-strip",
         "llvm-tblgen",
-        "llvm-tgt-verify",
         "llvm-readtapi",
         "llvm-undname",
         "llvm-windres",
diff --git a/llvm/tools/llvm-tgt-verify/CMakeLists.txt b/llvm/tools/llvm-tgt-verify/CMakeLists.txt
deleted file mode 100644
index fe47c85e6cdce..0000000000000
--- a/llvm/tools/llvm-tgt-verify/CMakeLists.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  AllTargetsAsmParsers
-  AllTargetsCodeGens
-  AllTargetsDescs
-  AllTargetsInfos
-  Analysis
-  AsmPrinter
-  CodeGen
-  CodeGenTypes
-  Core
-  IRPrinter
-  IRReader
-  MC
-  MIRParser
-  Passes
-  Remarks
-  ScalarOpts
-  SelectionDAG
-  Support
-  Target
-  TargetParser
-  TransformUtils
-  Vectorize
-  )
-
-add_llvm_tool(llvm-tgt-verify
-  llvm-tgt-verify.cpp
-
-  DEPENDS
-  intrinsics_gen
-  SUPPORT_PLUGINS
-  )
-
-export_executable_symbols_for_plugins(llc)
diff --git a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp b/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
deleted file mode 100644
index 50f4e56bb6af6..0000000000000
--- a/llvm/tools/llvm-tgt-verify/llvm-tgt-verify.cpp
+++ /dev/null
@@ -1,171 +0,0 @@
-//===--- llvm-tgt-verify.cpp - Target Verifier ----------------- ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Tool to verify a target.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/InitializePasses.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/Lint.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Bitcode/BitcodeReader.h"
-#include "llvm/Bitcode/BitcodeWriter.h"
-#include "llvm/CodeGen/CommandFlags.h"
-#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Verifier.h"
-#include "llvm/IRReader/IRReader.h"
-#include "llvm/Passes/PassBuilder.h"
-#include "llvm/Passes/StandardInstrumentations.h"
-#include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/InitLLVM.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/TargetSelect.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetVerifier.h"
-
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
-
-#define DEBUG_TYPE "isel-fuzzer"
-
-using namespace llvm;
-
-static codegen::RegisterCodeGenFlags CGF;
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-static cl::opt<bool>
-    StacktraceAbort("stacktrace-abort",
-        cl::desc("Turn on stacktrace"), cl::init(false));
-
-static cl::opt<bool>
-    NoLint("no-lint",
-        cl::desc("Turn off Lint"), cl::init(false));
-
-static cl::opt<bool>
-    NoVerify("no-verifier",
-        cl::desc("Turn off Verifier"), cl::init(false));
-
-static cl::opt<char>
-    OptLevel("O",
-             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
-                      "(default = '-O2')"),
-             cl::Prefix, cl::init('2'));
-
-static cl::opt<std::string>
-    TargetTriple("mtriple", cl::desc("Override target triple for module"));
-
-static std::unique_ptr<TargetMachine> TM;
-
-static void handleLLVMFatalError(void *, const char *Message, bool) {
-  if (StacktraceAbort) {
-    dbgs() << "LLVM ERROR: " << Message << "\n"
-           << "Aborting.\n";
-    abort();
-  }
-}
-
-int main(int argc, char **argv) {
-  StringRef ExecName = argv[0];
-  InitLLVM X(argc, argv);
-
-  InitializeAllTargets();
-  InitializeAllTargetMCs();
-  InitializeAllAsmPrinters();
-  InitializeAllAsmParsers();
-
-  PassRegistry *Registry = PassRegistry::getPassRegistry();
-  initializeCore(*Registry);
-  initializeCodeGen(*Registry);
-  initializeAnalysis(*Registry);
-  initializeTarget(*Registry);
-
-  cl::ParseCommandLineOptions(argc, argv);
-
-  if (TargetTriple.empty()) {
-    errs() << ExecName << ": -mtriple must be specified\n";
-    exit(1);
-  }
-
-  CodeGenOptLevel OLvl;
-  if (auto Level = CodeGenOpt::parseLevel(OptLevel)) {
-    OLvl = *Level;
-  } else {
-    errs() << ExecName << ": invalid optimization level.\n";
-    return 1;
-  }
-  ExitOnError ExitOnErr(std::string(ExecName) + ": error:");
-  TM = ExitOnErr(codegen::createTargetMachineForTriple(
-      Triple::normalize(TargetTriple), OLvl));
-  assert(TM && "Could not allocate target machine!");
-
-  // Make sure we print the summary and the current unit when LLVM errors out.
-  install_fatal_error_handler(handleLLVMFatalError, nullptr);
-
-  LLVMContext Context;
-  SMDiagnostic Err;
-  std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
-  if (!M) {
-    errs() << "Invalid mod\n";
-    return 1;
-  }
-  auto S = Triple::normalize(TargetTriple);
-  M->setTargetTriple(Triple(S));
-
-  PassInstrumentationCallbacks PIC;
-  StandardInstrumentations SI(Context, false/*debug PM*/,
-                              false);
-  registerCodeGenCallback(PIC, *TM);
-
-  ModulePassManager MPM;
-  FunctionPassManager FPM;
-  //TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
-
-  MachineFunctionAnalysisManager MFAM;
-  LoopAnalysisManager LAM;
-  FunctionAnalysisManager FAM;
-  CGSCCAnalysisManager CGAM;
-  ModuleAnalysisManager MAM;
-  PassBuilder PB(TM.get(), PipelineTuningOptions(), std::nullopt, &PIC);
-  PB.registerModuleAnalyses(MAM);
-  //PB.registerVerifierPasses(MPM, FPM);
-  PB.registerCGSCCAnalyses(CGAM);
-  PB.registerFunctionAnalyses(FAM);
-  PB.registerLoopAnalyses(LAM);
-  PB.registerMachineFunctionAnalyses(MFAM);
-  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-
-  SI.registerCallbacks(PIC, &MAM);
-
-  Triple TT(M->getTargetTriple());
-  if (!NoLint)
-    FPM.addPass(LintPass(false));
-  if (!NoVerify)
-    MPM.addPass(VerifierPass());
-  if (TT.isAMDGPU())
-    FPM.addPass(AMDGPUTargetVerifierPass());
-  else if (false) {} // ...
-  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
-
-  auto PA = MPM.run(*M, MAM);
-  {
-    auto PAC = PA.getChecker<VerifierAnalysis>();
-    if (!PAC.preserved())
-      return 1;
-  }
-
-  return 0;
-}
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 36cf956545caf..946b63f8a54fb 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -320,7 +320,6 @@ group("test") {
     "//llvm/tools/llvm-strings",
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-tli-checker",
-    "//llvm/tools/llvm-tgt-verify",
     "//llvm/tools/llvm-undname",
     "//llvm/tools/llvm-xray",
     "//llvm/tools/lto",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
deleted file mode 100644
index b751bafc5052c..0000000000000
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-tgt-verify/BUILD.gn
+++ /dev/null
@@ -1,25 +0,0 @@
-import("//llvm/utils/TableGen/tablegen.gni")
-
-tgtverifier("llvm-tgt-verify") {
-  deps = [
-    "//llvm/lib/Analysis",
-    "//llvm/lib/AsmPrinter",
-    "//llvm/lib/CodeGen",
-    "//llvm/lib/CodeGenTypes",
-    "//llvm/lib/Core",
-    "//llvm/lib/IRPrinter",
-    "//llvm/lib/IRReader",
-    "//llvm/lib/MC",
-    "//llvm/lib/MIRParser",
-    "//llvm/lib/Passes",
-    "//llvm/lib/Remarks",
-    "//llvm/lib/ScalarOpts",
-    "//llvm/lib/SelectionDAG",
-    "//llvm/lib/Support",
-    "//llvm/lib/Target",
-    "//llvm/lib/TargetParser",
-    "//llvm/lib/TransformUtils",
-    "//llvm/lib/Vectorize",
-  ]
-  sources = [ "llvm-tgt-verify.cpp" ]
-}

>From 3f1d72ff01d0303c0fc17d9a6c40e46661bf1fb0 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 30 Apr 2025 22:23:15 -0400
Subject: [PATCH 26/33] Remove TargetVerifier.cpp

---
 llvm/lib/Passes/CMakeLists.txt     |  1 -
 llvm/lib/Target/CMakeLists.txt     |  2 --
 llvm/lib/Target/TargetVerifier.cpp | 36 ------------------------------
 3 files changed, 39 deletions(-)
 delete mode 100644 llvm/lib/Target/TargetVerifier.cpp

diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index b8d25b15186e8..91c8c4f67074d 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -30,7 +30,6 @@ add_llvm_component_library(LLVMPasses
   Scalar
   Support
   Target
-  #TargetParser
   TransformUtils
   Vectorize
   Instrumentation
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index f2a5d545ce84f..9472288229cac 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -7,8 +7,6 @@ add_llvm_component_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
-  TargetVerifier.cpp
-  AMDGPU/AMDGPUTargetVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Target
diff --git a/llvm/lib/Target/TargetVerifier.cpp b/llvm/lib/Target/TargetVerifier.cpp
deleted file mode 100644
index c63ae2a2c5daf..0000000000000
--- a/llvm/lib/Target/TargetVerifier.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-//===-- TargetVerifier.cpp - LLVM IR Target Verifier ----------------*- C++ -*-===//
-////
-///// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-///// See https://llvm.org/LICENSE.txt for license information.
-///// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-/////
-/////===----------------------------------------------------------------------===//
-/////
-///// This file defines target verifier interfaces that can be used for some
-///// validation of input to the system, and for checking that transformations
-///// haven't done something bad. In contrast to the Verifier or Lint, the
-///// TargetVerifier looks for constructions invalid to a particular target
-///// machine.
-/////
-///// To see what specifically is checked, look at TargetVerifier.cpp or an
-///// individual backend's TargetVerifier.
-/////
-/////===----------------------------------------------------------------------===//
-
-#include "llvm/Target/TargetVerifier.h"
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
-
-#include "llvm/InitializePasses.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/IR/Dominators.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Value.h"
-
-namespace llvm {
-
-} // namespace llvm

>From 1c721cf05616091cf3c42e4c244400a393abe755 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 1 May 2025 03:02:30 -0400
Subject: [PATCH 27/33] clang-format

---
 llvm/include/llvm/Target/TargetVerifier.h     | 10 +-
 .../TargetVerify/AMDGPUTargetVerifier.h       | 44 ++++-----
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 98 ++++++++++---------
 3 files changed, 77 insertions(+), 75 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 1d12eb55bbf0a..3f8c710a88768 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -1,4 +1,4 @@
-//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier ---*- C++ -*-===//
+//===-- llvm/Target/TargetVerifier.h - LLVM IR Target Verifier --*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -20,8 +20,8 @@
 #ifndef LLVM_TARGET_VERIFIER_H
 #define LLVM_TARGET_VERIFIER_H
 
-#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
@@ -59,10 +59,11 @@ class TargetVerify {
   /// This calls the Message-only version so that the above is easier to set
   /// a breakpoint on.
   template <typename T1, typename... Ts>
-  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) {
+  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
     CheckFailed(Message);
     WriteValues({V1, Vs...});
   }
+
 public:
   Module *Mod;
   Triple TT;
@@ -73,8 +74,7 @@ class TargetVerify {
   bool IsValid = true;
 
   TargetVerify(Module *Mod)
-      : Mod(Mod), TT(Mod->getTargetTriple()),
-        MessagesStr(Messages) {}
+      : Mod(Mod), TT(Mod->getTargetTriple()), MessagesStr(Messages) {}
 
   virtual bool run(Function &F) = 0;
 };
diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 49bcbc8849e3c..5b8d9ec259b63 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -1,29 +1,29 @@
-//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU ---*- C++ -*-===//
-////
-//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//// See https://llvm.org/LICENSE.txt for license information.
-//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-////
-////===----------------------------------------------------------------------===//
-////
-//// This file defines target verifier interfaces that can be used for some
-//// validation of input to the system, and for checking that transformations
-//// haven't done something bad. In contrast to the Verifier or Lint, the
-//// TargetVerifier looks for constructions invalid to a particular target
-//// machine.
-////
-//// To see what specifically is checked, look at an individual backend's
-//// TargetVerifier.
-////
-////===----------------------------------------------------------------------===//
+//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU -- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
 
 #ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
 #define LLVM_AMDGPU_TARGET_VERIFIER_H
 
 #include "llvm/Target/TargetVerifier.h"
 
-#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/IR/Dominators.h"
 
 namespace llvm {
@@ -43,10 +43,10 @@ class AMDGPUTargetVerify : public TargetVerify {
   PostDominatorTree *PDT = nullptr;
   UniformityInfo *UA = nullptr;
 
-  AMDGPUTargetVerify(Module *Mod)
-    : TargetVerify(Mod), Mod(Mod) {}
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod), Mod(Mod) {}
 
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT, UniformityInfo *UA)
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
+                     UniformityInfo *UA)
     : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   bool run(Function &F) override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 2ca0bbeb57653..eb22eb2177f7f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -1,34 +1,34 @@
-//===-- AMDGPUTargetVerifier.cpp - AMDGPU -------------------------*- C++ -*-===//
-////
-//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//// See https://llvm.org/LICENSE.txt for license information.
-//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-////
-////===----------------------------------------------------------------------===//
-////
-//// This file defines target verifier interfaces that can be used for some
-//// validation of input to the system, and for checking that transformations
-//// haven't done something bad. In contrast to the Verifier or Lint, the
-//// TargetVerifier looks for constructions invalid to a particular target
-//// machine.
-////
-//// To see what specifically is checked, look at an individual backend's
-//// TargetVerifier.
-////
-////===----------------------------------------------------------------------===//
+//===-- AMDGPUTargetVerifier.cpp - AMDGPU -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines target verifier interfaces that can be used for some
+// validation of input to the system, and for checking that transformations
+// haven't done something bad. In contrast to the Verifier or Lint, the
+// TargetVerifier looks for constructions invalid to a particular target
+// machine.
+//
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
+//
+//===----------------------------------------------------------------------===//
 
-#include "AMDGPU.h"
 #include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
+#include "AMDGPU.h"
 
-#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/PostDominators.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Debug.h"
 
 #include "llvm/Support/raw_ostream.h"
 
@@ -39,53 +39,52 @@ using namespace llvm;
   do {                                                                         \
     if (!(C)) {                                                                \
       TargetVerify::CheckFailed(__VA_ARGS__);                                  \
-      return false;                                                                  \
     }                                                                          \
   } while (false)
 
 namespace llvm {
 
 static bool isShader(CallingConv::ID CC) {
-  switch(CC) {
-    case CallingConv::AMDGPU_VS:
-    case CallingConv::AMDGPU_LS:
-    case CallingConv::AMDGPU_HS:
-    case CallingConv::AMDGPU_ES:
-    case CallingConv::AMDGPU_GS:
-    case CallingConv::AMDGPU_PS:
-    case CallingConv::AMDGPU_CS_Chain:
-    case CallingConv::AMDGPU_CS_ChainPreserve:
-    case CallingConv::AMDGPU_CS:
-      return true;
-    default:
-      return false;
+  switch (CC) {
+  case CallingConv::AMDGPU_VS:
+  case CallingConv::AMDGPU_LS:
+  case CallingConv::AMDGPU_HS:
+  case CallingConv::AMDGPU_ES:
+  case CallingConv::AMDGPU_GS:
+  case CallingConv::AMDGPU_PS:
+  case CallingConv::AMDGPU_CS_Chain:
+  case CallingConv::AMDGPU_CS_ChainPreserve:
+  case CallingConv::AMDGPU_CS:
+    return true;
+  default:
+    return false;
   }
 }
 
 bool AMDGPUTargetVerify::run(Function &F) {
   // Ensure shader calling convention returns void
   if (isShader(F.getCallingConv()))
-    Check(F.getReturnType() == Type::getVoidTy(F.getContext()), "Shaders must return void");
+    Check(F.getReturnType() == Type::getVoidTy(F.getContext()),
+          "Shaders must return void");
 
   for (auto &BB : F) {
 
     for (auto &I : BB) {
 
-      if (auto *CI = dyn_cast<CallInst>(&I))
-      {
+      if (auto *CI = dyn_cast<CallInst>(&I)) {
         // Ensure no kernel to kernel calls.
         CallingConv::ID CalleeCC = CI->getCallingConv();
-        if (CalleeCC == CallingConv::AMDGPU_KERNEL)
-        {
-          CallingConv::ID CallerCC = CI->getParent()->getParent()->getCallingConv();
+        if (CalleeCC == CallingConv::AMDGPU_KERNEL) {
+          CallingConv::ID CallerCC =
+              CI->getParent()->getParent()->getCallingConv();
           Check(CallerCC != CallingConv::AMDGPU_KERNEL,
-            "A kernel may not call a kernel", CI->getParent()->getParent());
+                "A kernel may not call a kernel", CI->getParent()->getParent());
         }
 
         // Ensure chain intrinsics are followed by unreachables.
         if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
           Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
-            "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
+                "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
       }
     }
   }
@@ -98,7 +97,8 @@ bool AMDGPUTargetVerify::run(Function &F) {
   return true;
 }
 
-PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
+PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F,
+                                                FunctionAnalysisManager &AM) {
   auto *Mod = F.getParent();
 
   auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
@@ -122,9 +122,10 @@ struct AMDGPUTargetVerifierLegacyPass : public FunctionPass {
   std::unique_ptr<AMDGPUTargetVerify> TV;
   bool FatalErrors = false;
 
-  AMDGPUTargetVerifierLegacyPass(bool FatalErrors) : FunctionPass(ID),
-    FatalErrors(FatalErrors) {
-    initializeAMDGPUTargetVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
+  AMDGPUTargetVerifierLegacyPass(bool FatalErrors)
+      : FunctionPass(ID), FatalErrors(FatalErrors) {
+    initializeAMDGPUTargetVerifierLegacyPassPass(
+        *PassRegistry::getPassRegistry());
   }
 
   bool doInitialization(Module &M) override {
@@ -167,4 +168,5 @@ FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors) {
   return new AMDGPUTargetVerifierLegacyPass(FatalErrors);
 }
 } // namespace llvm
-INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier", "AMDGPU Target Verifier", false, false)
+INITIALIZE_PASS(AMDGPUTargetVerifierLegacyPass, "amdgpu-tgtverifier",
+                "AMDGPU Target Verifier", false, false)

>From f59b0f27beb766516186dcab8bde8463734ee64b Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 1 May 2025 03:49:20 -0400
Subject: [PATCH 28/33] Add VerifyTarget option

---
 .../llvm/Target/TargetVerify/AMDGPUTargetVerifier.h      | 6 ++----
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp           | 9 +++++++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
index 5b8d9ec259b63..8ed7dd7ea2f69 100644
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
@@ -37,17 +37,15 @@ class AMDGPUTargetVerifierPass : public TargetVerifierPass {
 
 class AMDGPUTargetVerify : public TargetVerify {
 public:
-  Module *Mod;
-
   DominatorTree *DT = nullptr;
   PostDominatorTree *PDT = nullptr;
   UniformityInfo *UA = nullptr;
 
-  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod), Mod(Mod) {}
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
 
   AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
                      UniformityInfo *UA)
-    : TargetVerify(Mod), Mod(Mod), DT(DT), PDT(PDT), UA(UA) {}
+      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
 
   bool run(Function &F) override;
 };
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8b0b8f6df740e..be8bde1c1c0e0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -482,6 +482,9 @@ static cl::opt<bool> HasClosedWorldAssumption(
     cl::desc("Whether has closed-world assumption at link time"),
     cl::init(false), cl::Hidden);
 
+static cl::opt<bool> VerifyTarget("verify-tgt",
+                                  cl::desc("Enable the target verifier"));
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
@@ -1411,7 +1414,8 @@ bool AMDGPUPassConfig::addGCPasses() {
 //===----------------------------------------------------------------------===//
 
 bool GCNPassConfig::addPreISel() {
-  addPass(createAMDGPUTargetVerifierLegacyPass(false));
+  if (VerifyTarget)
+    addPass(createAMDGPUTargetVerifierLegacyPass(false));
   AMDGPUPassConfig::addPreISel();
 
   if (TM->getOptLevel() > CodeGenOptLevel::None)
@@ -2011,7 +2015,8 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
 }
 
 void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
-  addPass(AMDGPUTargetVerifierPass());
+  if (VerifyTarget)
+    addPass(AMDGPUTargetVerifierPass());
 
   if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
     addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));

>From f0f12f552a0a48eda3e1dc0b7db88e605067f945 Mon Sep 17 00:00:00 2001
From: Joey F <jofernau at amd.com>
Date: Wed, 7 May 2025 21:59:43 -0400
Subject: [PATCH 29/33] Remove AMDGPUTargetVerifier.h

---
 .../TargetVerify/AMDGPUTargetVerifier.h       | 55 -------------------
 llvm/lib/Target/AMDGPU/AMDGPU.h               |  5 ++
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  1 -
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 16 +++++-
 4 files changed, 20 insertions(+), 57 deletions(-)
 delete mode 100644 llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h

diff --git a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h b/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
deleted file mode 100644
index 8ed7dd7ea2f69..0000000000000
--- a/llvm/include/llvm/Target/TargetVerify/AMDGPUTargetVerifier.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- llvm/Target/TargetVerify/AMDGPUTargetVerifier.h - AMDGPU -- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines target verifier interfaces that can be used for some
-// validation of input to the system, and for checking that transformations
-// haven't done something bad. In contrast to the Verifier or Lint, the
-// TargetVerifier looks for constructions invalid to a particular target
-// machine.
-//
-// To see what specifically is checked, look at an individual backend's
-// TargetVerifier.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_AMDGPU_TARGET_VERIFIER_H
-#define LLVM_AMDGPU_TARGET_VERIFIER_H
-
-#include "llvm/Target/TargetVerifier.h"
-
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/IR/Dominators.h"
-
-namespace llvm {
-
-class Function;
-
-class AMDGPUTargetVerifierPass : public TargetVerifierPass {
-public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
-};
-
-class AMDGPUTargetVerify : public TargetVerify {
-public:
-  DominatorTree *DT = nullptr;
-  PostDominatorTree *PDT = nullptr;
-  UniformityInfo *UA = nullptr;
-
-  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
-                     UniformityInfo *UA)
-      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
-  bool run(Function &F) override;
-};
-
-} // namespace llvm
-
-#endif // LLVM_AMDGPU_TARGET_VERIFIER_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 26fb0a967fd76..f5f083ca066a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -15,6 +15,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Target/TargetVerifier.h"
 
 namespace llvm {
 
@@ -551,6 +552,10 @@ extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
 void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
+class AMDGPUTargetVerifierPass : public TargetVerifierPass {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
+};
 
 namespace AMDGPU {
 enum TargetIndex {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index be8bde1c1c0e0..2009459aac329 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,7 +90,6 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/FormatVariadic.h"
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index eb22eb2177f7f..cc96164a1a558 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -17,7 +17,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetVerify/AMDGPUTargetVerifier.h"
 #include "AMDGPU.h"
 
 #include "llvm/Analysis/PostDominators.h"
@@ -44,6 +43,21 @@ using namespace llvm;
 
 namespace llvm {
 
+class AMDGPUTargetVerify : public TargetVerify {
+public:
+  DominatorTree *DT = nullptr;
+  PostDominatorTree *PDT = nullptr;
+  UniformityInfo *UA = nullptr;
+
+  AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
+
+  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
+                     UniformityInfo *UA)
+      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
+
+  bool run(Function &F) override;
+};
+
 static bool isShader(CallingConv::ID CC) {
   switch (CC) {
   case CallingConv::AMDGPU_VS:

>From 88a6d6b0faeff37cd98be19fd88632f9546720be Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Wed, 7 May 2025 22:47:55 -0400
Subject: [PATCH 30/33] Remove analyses.

---
 .../lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index cc96164a1a558..d07b5fb668c14 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -19,9 +19,6 @@
 
 #include "AMDGPU.h"
 
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -45,16 +42,7 @@ namespace llvm {
 
 class AMDGPUTargetVerify : public TargetVerify {
 public:
-  DominatorTree *DT = nullptr;
-  PostDominatorTree *PDT = nullptr;
-  UniformityInfo *UA = nullptr;
-
   AMDGPUTargetVerify(Module *Mod) : TargetVerify(Mod) {}
-
-  AMDGPUTargetVerify(Module *Mod, DominatorTree *DT, PostDominatorTree *PDT,
-                     UniformityInfo *UA)
-      : TargetVerify(Mod), DT(DT), PDT(PDT), UA(UA) {}
-
   bool run(Function &F) override;
 };
 
@@ -115,11 +103,7 @@ PreservedAnalyses AMDGPUTargetVerifierPass::run(Function &F,
                                                 FunctionAnalysisManager &AM) {
   auto *Mod = F.getParent();
 
-  auto UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *PDT = &AM.getResult<PostDominatorTreeAnalysis>(F);
-
-  AMDGPUTargetVerify TV(Mod, DT, PDT, UA);
+  AMDGPUTargetVerify TV(Mod);
   TV.run(F);
 
   dbgs() << TV.MessagesStr.str();

>From 0d99a659957a043ec7dfa4762a1f1c5bd34e50b0 Mon Sep 17 00:00:00 2001
From: jofernau <Joe.Fernau at amd.com>
Date: Thu, 8 May 2025 12:59:01 -0400
Subject: [PATCH 31/33] function name style, TargetVerifier comment, option
 name generality

---
 llvm/include/llvm/Target/TargetVerifier.h       | 14 +++++++-------
 llvm/lib/Target/AMDGPU/AMDGPU.h                 |  3 +--
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp  |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp |  2 +-
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll |  2 +-
 llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll |  2 +-
 6 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/llvm/include/llvm/Target/TargetVerifier.h b/llvm/include/llvm/Target/TargetVerifier.h
index 3f8c710a88768..eff846694ac10 100644
--- a/llvm/include/llvm/Target/TargetVerifier.h
+++ b/llvm/include/llvm/Target/TargetVerifier.h
@@ -12,8 +12,8 @@
 // TargetVerifier looks for constructions invalid to a particular target
 // machine.
 //
-// To see what specifically is checked, look at TargetVerifier.cpp or an
-// individual backend's TargetVerifier.
+// To see what specifically is checked, look at an individual backend's
+// TargetVerifier.
 //
 //===----------------------------------------------------------------------===//
 
@@ -35,7 +35,7 @@ class TargetVerifierPass : public PassInfoMixin<TargetVerifierPass> {
 
 class TargetVerify {
 protected:
-  void WriteValues(ArrayRef<const Value *> Vs) {
+  void writeValues(ArrayRef<const Value *> Vs) {
     for (const Value *V : Vs) {
       if (!V)
         continue;
@@ -52,16 +52,16 @@ class TargetVerify {
   ///
   /// This provides a nice place to put a breakpoint if you want to see why
   /// something is not correct.
-  void CheckFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
+  void checkFailed(const Twine &Message) { MessagesStr << Message << '\n'; }
 
   /// A check failed (with values to print).
   ///
   /// This calls the Message-only version so that the above is easier to set
   /// a breakpoint on.
   template <typename T1, typename... Ts>
-  void CheckFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
-    CheckFailed(Message);
-    WriteValues({V1, Vs...});
+  void checkFailed(const Twine &Message, const T1 &V1, const Ts &...Vs) {
+    checkFailed(Message);
+    writeValues({V1, Vs...});
   }
 
 public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index f5f083ca066a2..af09a7e4547f3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -552,8 +552,7 @@ extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
 FunctionPass *createAMDGPUTargetVerifierLegacyPass(bool FatalErrors);
 void initializeAMDGPUTargetVerifierLegacyPassPass(PassRegistry &);
-class AMDGPUTargetVerifierPass : public TargetVerifierPass {
-public:
+struct AMDGPUTargetVerifierPass : public TargetVerifierPass {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) override;
 };
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 2009459aac329..370fde6cce6ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -481,7 +481,7 @@ static cl::opt<bool> HasClosedWorldAssumption(
     cl::desc("Whether has closed-world assumption at link time"),
     cl::init(false), cl::Hidden);
 
-static cl::opt<bool> VerifyTarget("verify-tgt",
+static cl::opt<bool> VerifyTarget("amdgpu-verify-tgt",
                                   cl::desc("Enable the target verifier"));
 
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index d07b5fb668c14..3246b0ac0c624 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -34,7 +34,7 @@ using namespace llvm;
 #define Check(C, ...)                                                          \
   do {                                                                         \
     if (!(C)) {                                                                \
-      TargetVerify::CheckFailed(__VA_ARGS__);                                  \
+      TargetVerify::checkFailed(__VA_ARGS__);                                  \
     }                                                                          \
   } while (false)
 
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
index e2d9edda5d008..819e66bb46b0a 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-fail.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt -o - < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt -o - < %s 2>&1 | FileCheck %s
 
 define amdgpu_cs i32 @nonvoid_shader() {
 ; CHECK: Shaders must return void
diff --git a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
index a2dab0ff47924..08854df01acb2 100644
--- a/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
+++ b/llvm/test/CodeGen/AMDGPU/tgt-verify-llc-pass.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt %s -o - 2>&1 | FileCheck %s --allow-empty
 
 define amdgpu_cs void @void_shader() {
 ; CHECK-NOT: Shaders must return void

>From 79215beef3f1330292ecee15b422f5cbfdeea9b5 Mon Sep 17 00:00:00 2001
From: Joey F <jofernau at amd.com>
Date: Mon, 16 Jun 2025 19:09:24 -0400
Subject: [PATCH 32/33] Fix after rebase

---
 llvm/include/llvm/CodeGen/Passes.h                  | 2 --
 llvm/include/llvm/IR/Module.h                       | 5 -----
 llvm/include/llvm/Passes/StandardInstrumentations.h | 4 ++--
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 53bd70183e476..990452fa11fec 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -620,8 +620,6 @@ LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
-
-LLVM_ABI FunctionPass *createTargetVerifierLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 34ac47a2b8be4..a99937a90cbb7 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -215,11 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module using intrinsics to record the position of debugging
-  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
-  /// \ref BasicBlock.
-  bool IsNewDbgInfoFormat;
-
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index d7c1bbf04c328..4ee5ab2554868 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -478,7 +478,7 @@ class VerifyInstrumentation {
 public:
   VerifyInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM);
 };
 
 /// This class implements --time-trace functionality for new pass manager.
@@ -624,7 +624,7 @@ class StandardInstrumentations {
   // Register all the standard instrumentation callbacks. If \p FAM is nullptr
   // then PreservedCFGChecker is not enabled.
   LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                                  ModuleAnalysisManager *MAM);
+                                  ModuleAnalysisManager *MAM = nullptr);
 
   TimePassesHandler &getTimePasses() { return TimePasses; }
 };

>From f25ecf1e14d99b797105a5d42386c9019be8d4d4 Mon Sep 17 00:00:00 2001
From: jofrn <jofernau at amd.com>
Date: Tue, 8 Jul 2025 08:02:10 -0400
Subject: [PATCH 33/33] Add back int check. These are invalid for AMDGPU.

---
 .../Target/AMDGPU/AMDGPUTargetVerifier.cpp    | 49 ++++++-------------
 .../Verifier/AMDGPU/tgt-verify-int-type.ll    |  6 +++
 2 files changed, 21 insertions(+), 34 deletions(-)
 create mode 100644 llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
index 3246b0ac0c624..ae35322c905f1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetVerifier.cpp
@@ -46,48 +46,29 @@ class AMDGPUTargetVerify : public TargetVerify {
   bool run(Function &F) override;
 };
 
-static bool isShader(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::AMDGPU_VS:
-  case CallingConv::AMDGPU_LS:
-  case CallingConv::AMDGPU_HS:
-  case CallingConv::AMDGPU_ES:
-  case CallingConv::AMDGPU_GS:
-  case CallingConv::AMDGPU_PS:
-  case CallingConv::AMDGPU_CS_Chain:
-  case CallingConv::AMDGPU_CS_ChainPreserve:
-  case CallingConv::AMDGPU_CS:
-    return true;
-  default:
-    return false;
-  }
+static bool IsValidInt(const Type *Ty) {
+  return Ty->isIntegerTy(1) ||
+         Ty->isIntegerTy(8) ||
+         Ty->isIntegerTy(16) ||
+         Ty->isIntegerTy(32) ||
+         Ty->isIntegerTy(64) ||
+         Ty->isIntegerTy(128);
 }
 
 bool AMDGPUTargetVerify::run(Function &F) {
-  // Ensure shader calling convention returns void
-  if (isShader(F.getCallingConv()))
-    Check(F.getReturnType() == Type::getVoidTy(F.getContext()),
-          "Shaders must return void");
 
   for (auto &BB : F) {
 
     for (auto &I : BB) {
 
-      if (auto *CI = dyn_cast<CallInst>(&I)) {
-        // Ensure no kernel to kernel calls.
-        CallingConv::ID CalleeCC = CI->getCallingConv();
-        if (CalleeCC == CallingConv::AMDGPU_KERNEL) {
-          CallingConv::ID CallerCC =
-              CI->getParent()->getParent()->getCallingConv();
-          Check(CallerCC != CallingConv::AMDGPU_KERNEL,
-                "A kernel may not call a kernel", CI->getParent()->getParent());
-        }
-
-        // Ensure chain intrinsics are followed by unreachables.
-        if (CI->getIntrinsicID() == Intrinsic::amdgcn_cs_chain)
-          Check(isa_and_present<UnreachableInst>(CI->getNextNode()),
-                "llvm.amdgcn.cs.chain must be followed by unreachable", CI);
-      }
+      // Ensure integral types are valid: i8, i16, i32, i64, i128
+      if (I.getType()->isIntegerTy())
+        Check(IsValidInt(I.getType()), "Int type is invalid.", &I);
+      for (unsigned i = 0; i < I.getNumOperands(); ++i)
+        if (I.getOperand(i)->getType()->isIntegerTy())
+          Check(IsValidInt(I.getOperand(i)->getType()),
+                "Int type is invalid.", I.getOperand(i));
+
     }
   }
 
diff --git a/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll b/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll
new file mode 100644
index 0000000000000..bd3a7106756cd
--- /dev/null
+++ b/llvm/test/Verifier/AMDGPU/tgt-verify-int-type.ll
@@ -0,0 +1,6 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-verify-tgt -o - < %s 2>&1 | FileCheck %s
+
+define amdgpu_cs i65 @invalid_int() {
+; CHECK: Int type is invalid.
+  ret i65 0
+}


From flang-commits at lists.llvm.org  Mon Jul 14 21:52:29 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Mon, 14 Jul 2025 21:52:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Handle SEQUENCE derived types for
 array repacking. (PR #148777)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148777@github.com>
Message-ID: <6875de8d.170a0220.2a14bb.b42c@mx.google.com>


https://github.com/clementval approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/148777

From flang-commits at lists.llvm.org  Mon Jul 14 22:17:19 2025
From: flang-commits at lists.llvm.org (Kazu Hirata via flang-commits)
Date: Mon, 14 Jul 2025 22:17:19 -0700 (PDT)
Subject: [flang-commits] [flang] 769bd90 - [flang] Fix a warning
Message-ID: <6875e45f.170a0220.c9c34.ad20@mx.google.com>


Author: Kazu Hirata
Date: 2025-07-14T22:17:12-07:00
New Revision: 769bd90f8b2fbb345b115586e81edf3a70ec8cc1

URL: https://github.com/llvm/llvm-project/commit/769bd90f8b2fbb345b115586e81edf3a70ec8cc1
DIFF: https://github.com/llvm/llvm-project/commit/769bd90f8b2fbb345b115586e81edf3a70ec8cc1.diff

LOG: [flang] Fix a warning

This patch fixes:

  flang/lib/Lower/ConvertVariable.cpp:787:22: error: unused variable
  'fieldTy' [-Werror,-Wunused-variable]

Added: 
    

Modified: 
    flang/lib/Lower/ConvertVariable.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 4ccaa59f503a7..ffe456de56630 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -784,7 +784,6 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
           Fortran::semantics::UltimateComponentIterator components{*derived};
           auto recTy = mlir::dyn_cast<fir::RecordType>(ty);
 
-          mlir::Type fieldTy;
           llvm::SmallVector<mlir::Value> coordinates;
           for (const auto &sym : components) {
             if (Fortran::semantics::IsDeviceAllocatable(sym)) {


        

From flang-commits at lists.llvm.org  Tue Jul 15 01:30:57 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Tue, 15 Jul 2025 01:30:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Add cuf.set_allocator_idx for
 device component (PR #148750)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148750@github.com>
Message-ID: <687611c1.050a0220.37c087.e1d9@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `flang` at step 5 "build-unified-tree".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/33785

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 5 (build-unified-tree) failure: build (failure)
...
294.835 [53/49/6835] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-call.cpp.o
300.658 [53/48/6836] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold.cpp.o
302.960 [53/47/6837] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/ReductionProcessor.cpp.o
305.181 [53/46/6838] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-loop.cpp.o
305.673 [53/45/6839] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IterationSpace.cpp.o
309.476 [53/44/6840] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Mangler.cpp.o
309.514 [53/43/6841] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HlfirIntrinsics.cpp.o
324.221 [53/42/6842] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CallInterface.cpp.o
342.832 [53/41/6843] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/VectorSubscripts.cpp.o
345.222 [53/40/6844] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o
FAILED: tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o 
ccache /home/buildbots/llvm-external-buildbots/clang.19.1.7/bin/clang++ -DEXPERIMENTAL_KEY_INSTRUCTIONS -DFLANG_INCLUDE_TESTS=1 -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Lower -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Lower -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/llvm/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/../mlir/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/mlir/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/clang/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/llvm/../clang/include -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wc++98-compat-extra-semi -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Werror -Wno-deprecated-copy -Wno-string-conversion -Wno-ctad-maybe-unsupported -Wno-unused-command-line-argument -Wstring-conversion           -Wcovered-switch-default -Wno-nested-anon-types -Xclang -fno-pch-timestamp -O3 -DNDEBUG -std=c++17  -fno-exceptions -funwind-tables -fno-rtti -UNDEBUG -Winvalid-pch -Xclang -include-pch -Xclang /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/cmake_pch.hxx.pch -Xclang -include -Xclang /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/cmake_pch.hxx -MD -MT tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o -MF tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o.d -o tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertVariable.cpp.o -c /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Lower/ConvertVariable.cpp
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Lower/ConvertVariable.cpp:787:22: error: unused variable 'fieldTy' [-Werror,-Wunused-variable]
  787 |           mlir::Type fieldTy;
      |                      ^~~~~~~
1 error generated.
345.561 [53/39/6845] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertType.cpp.o
351.382 [53/38/6846] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertProcedureDesignator.cpp.o
360.322 [53/37/6847] Building CXX object tools/flang/tools/bbc/CMakeFiles/bbc.dir/bbc.cpp.o
360.642 [53/36/6848] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HostAssociations.cpp.o
361.591 [53/35/6849] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertConstant.cpp.o
364.720 [53/34/6850] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-cuda.cpp.o
370.899 [53/33/6851] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CustomIntrinsicCall.cpp.o
372.938 [53/32/6852] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IO.cpp.o
385.000 [53/31/6853] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Allocatable.cpp.o
391.425 [53/30/6854] Building CXX object tools/flang/lib/Frontend/CMakeFiles/flangFrontend.dir/FrontendActions.cpp.o
392.052 [53/29/6855] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-coarray.cpp.o
392.274 [53/28/6856] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertArrayConstructor.cpp.o
393.979 [53/27/6857] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-do-forall.cpp.o
394.686 [53/26/6858] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertCall.cpp.o
401.682 [53/25/6859] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/check-expression.cpp.o
409.533 [53/24/6860] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Utils.cpp.o
413.079 [53/23/6861] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-complex.cpp.o
414.226 [53/22/6862] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-logical.cpp.o
421.238 [53/21/6863] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Atomic.cpp.o
423.472 [53/20/6864] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o
429.639 [53/19/6865] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Decomposer.cpp.o
430.023 [53/18/6866] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/ClauseProcessor.cpp.o
431.926 [53/17/6867] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertExprToHLFIR.cpp.o
436.813 [53/16/6868] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/OpenMP.cpp.o
441.876 [53/15/6869] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/semantics.cpp.o
442.837 [53/14/6870] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-real.cpp.o
443.024 [53/13/6871] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/Utils.cpp.o
447.459 [53/12/6872] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/expression.cpp.o
448.634 [53/11/6873] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-structure.cpp.o
452.028 [53/10/6874] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Bridge.cpp.o
452.213 [53/9/6875] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/DataSharingProcessor.cpp.o
454.642 [53/8/6876] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenACC.cpp.o
470.552 [53/7/6877] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/OpenMP/Clauses.cpp.o

```

</details>

https://github.com/llvm/llvm-project/pull/148750

From flang-commits at lists.llvm.org  Tue Jul 15 02:27:07 2025
From: flang-commits at lists.llvm.org (Jean-Didier PAILLEUX via flang-commits)
Date: Tue, 15 Jul 2025 02:27:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Adding NOTIFY specifier in image
 selector and add notify type checks (PR #148810)
Message-ID: <llvm.org/llvm/llvm-project/pull/148810@github.com>


https://github.com/JDPailleux created https://github.com/llvm/llvm-project/pull/148810

This PR adds support for the NOTIFY specifier in the image selector as described in the 2023 standard, and add checks for the NOTIFY_TYPE type.

>From d35b114290d52cab44f5aec549d4d5cffa2cd196 Mon Sep 17 00:00:00 2001
From: Jean-Didier Pailleux <jean-didier.pailleux at sipearl.com>
Date: Tue, 15 Jul 2025 09:31:50 +0200
Subject: [PATCH] [flang] Adding NOTIFY specifier in image selector and
 reinforce notify type checks

---
 flang/examples/FeatureList/FeatureList.cpp   |  1 +
 flang/include/flang/Evaluate/traverse.h      |  2 +-
 flang/include/flang/Evaluate/variable.h      |  4 +++
 flang/include/flang/Parser/dump-parse-tree.h |  1 +
 flang/include/flang/Parser/parse-tree.h      |  6 ++--
 flang/include/flang/Semantics/tools.h        |  3 ++
 flang/lib/Evaluate/variable.cpp              | 13 +++++++
 flang/lib/Lower/Support/Utils.cpp            |  5 +--
 flang/lib/Parser/Fortran-parsers.cpp         |  7 ++--
 flang/lib/Parser/unparse.cpp                 |  1 +
 flang/lib/Semantics/check-declarations.cpp   | 13 +++++++
 flang/lib/Semantics/dump-expr.cpp            |  1 +
 flang/lib/Semantics/expression.cpp           | 13 +++++++
 flang/lib/Semantics/tools.cpp                | 38 ++++++++++++++++++++
 flang/test/Semantics/coarrays02.f90          | 17 +++++++++
 flang/test/Semantics/notifywait03.f90        |  1 +
 16 files changed, 119 insertions(+), 7 deletions(-)

diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index a36b8719e365d..eff4eb6a537cb 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -350,6 +350,7 @@ struct NodeVisitor {
   READ_FEATURE(ImageSelectorSpec)
   READ_FEATURE(ImageSelectorSpec::Stat)
   READ_FEATURE(ImageSelectorSpec::Team_Number)
+  READ_FEATURE(ImageSelectorSpec::Notify)
   READ_FEATURE(ImplicitPart)
   READ_FEATURE(ImplicitPartStmt)
   READ_FEATURE(ImplicitSpec)
diff --git a/flang/include/flang/Evaluate/traverse.h b/flang/include/flang/Evaluate/traverse.h
index 48aafa8982559..9a6978b6be6f7 100644
--- a/flang/include/flang/Evaluate/traverse.h
+++ b/flang/include/flang/Evaluate/traverse.h
@@ -146,7 +146,7 @@ class Traverse {
     return Combine(x.base(), x.subscript());
   }
   Result operator()(const CoarrayRef &x) const {
-    return Combine(x.base(), x.cosubscript(), x.stat(), x.team());
+    return Combine(x.base(), x.cosubscript(), x.stat(), x.team(), x.notify());
   }
   Result operator()(const DataRef &x) const { return visitor_(x.u); }
   Result operator()(const Substring &x) const {
diff --git a/flang/include/flang/Evaluate/variable.h b/flang/include/flang/Evaluate/variable.h
index 5c14421fd3a1b..6f1716f6e44b5 100644
--- a/flang/include/flang/Evaluate/variable.h
+++ b/flang/include/flang/Evaluate/variable.h
@@ -260,6 +260,9 @@ class CoarrayRef {
   // it's TEAM=.
   std::optional<Expr<SomeType>> team() const;
   CoarrayRef &set_team(Expr<SomeType> &&);
+  // When notify() is Expr<Some>, it's NOTIFY=.
+  std::optional<Expr<SomeType>> notify() const;
+  CoarrayRef &set_notify(Expr<SomeType> &&);
 
   int Rank() const;
   int Corank() const { return 0; }
@@ -274,6 +277,7 @@ class CoarrayRef {
   std::vector<Expr<SubscriptInteger>> cosubscript_;
   std::optional<common::CopyableIndirection<Expr<SomeInteger>>> stat_;
   std::optional<common::CopyableIndirection<Expr<SomeType>>> team_;
+  std::optional<common::CopyableIndirection<Expr<SomeType>>> notify_;
 };
 
 // R911 data-ref is defined syntactically as a series of part-refs, which
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..0186181cb43d6 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -385,6 +385,7 @@ class ParseTreeDumper {
   NODE(parser, ImageSelectorSpec)
   NODE(ImageSelectorSpec, Stat)
   NODE(ImageSelectorSpec, Team_Number)
+  NODE(ImageSelectorSpec, Notify)
   NODE(parser, ImplicitPart)
   NODE(parser, ImplicitPartStmt)
   NODE(parser, ImplicitSpec)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 7e752eeb4dfe4..518be9b33a493 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -1682,12 +1682,14 @@ WRAPPER_CLASS(TeamValue, Scalar<common::Indirection<Expr>>);
 
 // R926 image-selector-spec ->
 //        STAT = stat-variable | TEAM = team-value |
-//        TEAM_NUMBER = scalar-int-expr
+//        TEAM_NUMBER = scalar-int-expr |
+//        NOTIFY = notify-variable
 struct ImageSelectorSpec {
   WRAPPER_CLASS(Stat, Scalar<Integer<common::Indirection<Variable>>>);
   WRAPPER_CLASS(Team_Number, ScalarIntExpr);
+  WRAPPER_CLASS(Notify, Scalar<common::Indirection<Variable>>);
   UNION_CLASS_BOILERPLATE(ImageSelectorSpec);
-  std::variant<Stat, TeamValue, Team_Number> u;
+  std::variant<Stat, TeamValue, Team_Number, Notify> u;
 };
 
 // R924 image-selector ->
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index fb670528f3ce4..e230620f5fcd3 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -107,6 +107,7 @@ bool IsBindCProcedure(const Scope &);
 // Returns a pointer to the function's symbol when true, else null
 const Symbol *IsFunctionResultWithSameNameAsFunction(const Symbol &);
 bool IsOrContainsEventOrLockComponent(const Symbol &);
+bool IsOrContainsNotifyComponent(const Symbol &);
 bool CanBeTypeBoundProc(const Symbol &);
 // Does a non-PARAMETER symbol have explicit initialization with =value or
 // =>target in its declaration (but not in a DATA statement)? (Being
@@ -642,6 +643,8 @@ using PotentialAndPointerComponentIterator =
 // dereferenced.
 PotentialComponentIterator::const_iterator FindEventOrLockPotentialComponent(
     const DerivedTypeSpec &, bool ignoreCoarrays = false);
+PotentialComponentIterator::const_iterator FindNotifyPotentialComponent(
+    const DerivedTypeSpec &, bool ignoreCoarrays = false);
 PotentialComponentIterator::const_iterator FindCoarrayPotentialComponent(
     const DerivedTypeSpec &);
 PotentialAndPointerComponentIterator::const_iterator
diff --git a/flang/lib/Evaluate/variable.cpp b/flang/lib/Evaluate/variable.cpp
index d1bff03a6ea5f..41003a2e53a8d 100644
--- a/flang/lib/Evaluate/variable.cpp
+++ b/flang/lib/Evaluate/variable.cpp
@@ -89,6 +89,14 @@ std::optional<Expr<SomeType>> CoarrayRef::team() const {
   }
 }
 
+std::optional<Expr<SomeType>> CoarrayRef::notify() const {
+  if (notify_) {
+    return notify_.value().value();
+  } else {
+    return std::nullopt;
+  }
+}
+
 CoarrayRef &CoarrayRef::set_stat(Expr<SomeInteger> &&v) {
   CHECK(IsVariable(v));
   stat_.emplace(std::move(v));
@@ -100,6 +108,11 @@ CoarrayRef &CoarrayRef::set_team(Expr<SomeType> &&v) {
   return *this;
 }
 
+CoarrayRef &CoarrayRef::set_notify(Expr<SomeType> &&v) {
+  notify_.emplace(std::move(v));
+  return *this;
+}
+
 const Symbol &CoarrayRef::GetFirstSymbol() const {
   return base().GetFirstSymbol();
 }
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index c65f51ce6cacd..30635b66bbd39 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -82,7 +82,7 @@ class HashEvaluateExpr {
          x.cosubscript())
       cosubs -= getHashValue(v);
     return getHashValue(x.base()) * 97u - cosubs + getHashValue(x.stat()) +
-           257u + getHashValue(x.team());
+           257u + getHashValue(x.team()) + getHashValue(x.notify());
   }
   static unsigned getHashValue(const Fortran::evaluate::NamedEntity &x) {
     if (x.IsSymbol())
@@ -341,7 +341,8 @@ class IsEqualEvaluateExpr {
                       const Fortran::evaluate::CoarrayRef &y) {
     return isEqual(x.base(), y.base()) &&
            isEqual(x.cosubscript(), y.cosubscript()) &&
-           isEqual(x.stat(), y.stat()) && isEqual(x.team(), y.team());
+           isEqual(x.stat(), y.stat()) && isEqual(x.team(), y.team()) &&
+           isEqual(x.notify(), y.notify());
   }
   static bool isEqual(const Fortran::evaluate::NamedEntity &x,
                       const Fortran::evaluate::NamedEntity &y) {
diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp
index fbe629ab52935..30bd807d16a46 100644
--- a/flang/lib/Parser/Fortran-parsers.cpp
+++ b/flang/lib/Parser/Fortran-parsers.cpp
@@ -1212,12 +1212,15 @@ TYPE_CONTEXT_PARSER("image selector"_en_US,
 
 // R926 image-selector-spec ->
 //        STAT = stat-variable | TEAM = team-value |
-//        TEAM_NUMBER = scalar-int-expr
+//        TEAM_NUMBER = scalar-int-expr |
+//        NOTIFY = notify-variable
 TYPE_PARSER(construct<ImageSelectorSpec>(construct<ImageSelectorSpec::Stat>(
                 "STAT =" >> scalar(integer(indirect(variable))))) ||
     construct<ImageSelectorSpec>(construct<TeamValue>("TEAM =" >> teamValue)) ||
     construct<ImageSelectorSpec>(construct<ImageSelectorSpec::Team_Number>(
-        "TEAM_NUMBER =" >> scalarIntExpr)))
+        "TEAM_NUMBER =" >> scalarIntExpr)) ||
+    construct<ImageSelectorSpec>(construct<ImageSelectorSpec::Notify>(
+        "NOTIFY =" >> scalar(indirect(variable)))))
 
 // R927 allocate-stmt ->
 //        ALLOCATE ( [type-spec ::] allocation-list [, alloc-opt-list] )
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index dcd1ac165adc1..942aacd777388 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -819,6 +819,7 @@ class UnparseVisitor {
       Word("TEAM=");
     }
   }
+  void Before(const ImageSelectorSpec::Notify &) { Word("NOTIFY="); }
   void Unparse(const AllocateStmt &x) { // R927
     Word("ALLOCATE(");
     Walk(std::get<std::optional<TypeSpec>>(x.t), "::");
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index f9d64485f1407..644c3e1effc21 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -762,6 +762,15 @@ void CheckHelper::CheckObjectEntity(
         messages_.Say(
             "Variable '%s' with EVENT_TYPE or LOCK_TYPE potential component '%s' must be a coarray"_err_en_US,
             symbol.name(), component.BuildResultDesignatorName());
+      } else if (IsNotifyType(derived)) { // C1612
+        messages_.Say(
+            "Variable '%s' with NOTIFY_TYPE must be a coarray"_err_en_US,
+            symbol.name());
+      } else if (auto component{FindNotifyPotentialComponent( // C1611
+                     *derived, /*ignoreCoarrays=*/true)}) {
+        messages_.Say(
+            "Variable '%s' with NOTIFY_TYPE potential component '%s' must be a coarray"_err_en_US,
+            symbol.name(), component.BuildResultDesignatorName());
       }
     }
   }
@@ -780,6 +789,10 @@ void CheckHelper::CheckObjectEntity(
         messages_.Say(
             "An INTENT(OUT) dummy argument may not be, or contain, EVENT_TYPE or LOCK_TYPE"_err_en_US);
       }
+      if (IsOrContainsNotifyComponent(symbol)) { // C1613
+        messages_.Say(
+            "An INTENT(OUT) dummy argument may not be, or contain, NOTIFY_TYPE"_err_en_US);
+      }
       if (IsAssumedSizeArray(symbol)) { // C834
         if (type && type->IsPolymorphic()) {
           messages_.Say(
diff --git a/flang/lib/Semantics/dump-expr.cpp b/flang/lib/Semantics/dump-expr.cpp
index 66cedab94bfb4..bd7373abffb31 100644
--- a/flang/lib/Semantics/dump-expr.cpp
+++ b/flang/lib/Semantics/dump-expr.cpp
@@ -25,6 +25,7 @@ void DumpEvaluateExpr::Show(const evaluate::CoarrayRef &x) {
   Show(x.cosubscript());
   Show(x.stat());
   Show(x.team());
+  Show(x.notify());
   Outdent();
 }
 
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index f4af738284ed7..b96dd77ac964c 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -1596,6 +1596,19 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::CoindexedNamedObject &x) {
                     coarrayRef.set_team(Expr<SomeType>{*expr});
                   }
                 }
+              },
+              [&](const parser::ImageSelectorSpec::Notify &x) {
+                Analyze(x.v);
+                if (const auto *expr{GetExpr(context_, x.v)}) {
+                  if (coarrayRef.notify()) {
+                    Say("coindexed reference has multiple NOTIFY= specifiers"_err_en_US);
+                  } else if (auto dyType{expr->GetType()};
+                      dyType && IsNotifyType(GetDerivedTypeSpec(*dyType))) {
+                    coarrayRef.set_notify(Expr<SomeType>{*expr});
+                  } else {
+                    Say("NOTIFY= specifier must have type NOTIFY_TYPE from ISO_FORTRAN_ENV"_err_en_US);
+                  }
+                }
               }},
           imageSelSpec.u);
     }
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index d27d250b3f11e..bc7a754cfc0d9 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -582,6 +582,18 @@ bool IsOrContainsEventOrLockComponent(const Symbol &original) {
   return false;
 }
 
+bool IsOrContainsNotifyComponent(const Symbol &original) {
+  const Symbol &symbol{ResolveAssociations(original, /*stopAtTypeGuard=*/true)};
+  if (evaluate::IsVariable(symbol)) {
+    if (const DeclTypeSpec *type{symbol.GetType()}) {
+      if (const DerivedTypeSpec *derived{type->AsDerived()}) {
+        return IsNotifyType(derived) || FindNotifyPotentialComponent(*derived);
+      }
+    }
+  }
+  return false;
+}
+
 // Check this symbol suitable as a type-bound procedure - C769
 bool CanBeTypeBoundProc(const Symbol &symbol) {
   if (IsDummy(symbol) || IsProcedurePointer(symbol)) {
@@ -1464,6 +1476,32 @@ PotentialComponentIterator::const_iterator FindEventOrLockPotentialComponent(
   return iter;
 }
 
+PotentialComponentIterator::const_iterator FindNotifyPotentialComponent(
+    const DerivedTypeSpec &derived, bool ignoreCoarrays) {
+  PotentialComponentIterator potentials{derived};
+  auto iter{potentials.begin()};
+  for (auto end{potentials.end()}; iter != end; ++iter) {
+    const Symbol &component{*iter};
+    if (const auto *object{component.detailsIf<ObjectEntityDetails>()}) {
+      if (const DeclTypeSpec *type{object->type()}) {
+        if (IsNotifyType(type->AsDerived())) {
+          if (!ignoreCoarrays) {
+            break; // found one
+          }
+          auto path{iter.GetComponentPath()};
+          path.pop_back();
+          if (std::find_if(path.begin(), path.end(), [](const Symbol &sym) {
+                return evaluate::IsCoarray(sym);
+              }) == path.end()) {
+            break; // found one not in a coarray
+          }
+        }
+      }
+    }
+  }
+  return iter;
+}
+
 UltimateComponentIterator::const_iterator FindAllocatableUltimateComponent(
     const DerivedTypeSpec &derived) {
   UltimateComponentIterator ultimates{derived};
diff --git a/flang/test/Semantics/coarrays02.f90 b/flang/test/Semantics/coarrays02.f90
index b16e0ccb58797..e866dd89c07ab 100644
--- a/flang/test/Semantics/coarrays02.f90
+++ b/flang/test/Semantics/coarrays02.f90
@@ -16,6 +16,8 @@ program main
   type(event_type) event
   !ERROR: Variable 'lock' with EVENT_TYPE or LOCK_TYPE must be a coarray
   type(lock_type) lock
+  !ERROR: Variable 'notify' with NOTIFY_TYPE must be a coarray
+  type(notify_type) notify
   integer :: local[*] ! ok in main
 end
 
@@ -120,3 +122,18 @@ subroutine s4
   !ERROR: Subscripts must appear in a coindexed reference when its base is an array
   print *, ta(1)%a[1]
 end
+
+subroutine s5(a, notify, res)
+  use iso_fortran_env
+  type t
+    type(notify_type) :: a
+  end type
+  real, intent(in) :: a[*]
+  type(event_type), intent(in) :: notify[*]
+  !ERROR: An INTENT(OUT) dummy argument may not be, or contain, NOTIFY_TYPE
+  type(notify_type), intent(out) :: res[*]
+  !ERROR: Variable 'bad' with NOTIFY_TYPE potential component '%a' must be a coarray
+  type(t) :: bad
+  !ERROR: NOTIFY= specifier must have type NOTIFY_TYPE from ISO_FORTRAN_ENV
+  print *, a[1, NOTIFY=notify]
+end
diff --git a/flang/test/Semantics/notifywait03.f90 b/flang/test/Semantics/notifywait03.f90
index 0fc56f66ad32d..a336a7a67669a 100644
--- a/flang/test/Semantics/notifywait03.f90
+++ b/flang/test/Semantics/notifywait03.f90
@@ -10,6 +10,7 @@ program test_notify_wait
   implicit none
 
   ! notify_type variables must be coarrays
+  !ERROR: Variable 'non_coarray' with NOTIFY_TYPE must be a coarray
   type(notify_type) :: non_coarray
 
   type(notify_type) :: notify_var[*], notify_array(2)[*]


From flang-commits at lists.llvm.org  Tue Jul 15 02:27:41 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 02:27:41 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Adding NOTIFY specifier in image
 selector and add notify type checks (PR #148810)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148810@github.com>
Message-ID: <68761f0d.170a0220.2cb820.deb5@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Jean-Didier PAILLEUX (JDPailleux)

<details>
<summary>Changes</summary>

This PR adds support for the NOTIFY specifier in the image selector as described in the 2023 standard, and add checks for the NOTIFY_TYPE type.

---
Full diff: https://github.com/llvm/llvm-project/pull/148810.diff


16 Files Affected:

- (modified) flang/examples/FeatureList/FeatureList.cpp (+1) 
- (modified) flang/include/flang/Evaluate/traverse.h (+1-1) 
- (modified) flang/include/flang/Evaluate/variable.h (+4) 
- (modified) flang/include/flang/Parser/dump-parse-tree.h (+1) 
- (modified) flang/include/flang/Parser/parse-tree.h (+4-2) 
- (modified) flang/include/flang/Semantics/tools.h (+3) 
- (modified) flang/lib/Evaluate/variable.cpp (+13) 
- (modified) flang/lib/Lower/Support/Utils.cpp (+3-2) 
- (modified) flang/lib/Parser/Fortran-parsers.cpp (+5-2) 
- (modified) flang/lib/Parser/unparse.cpp (+1) 
- (modified) flang/lib/Semantics/check-declarations.cpp (+13) 
- (modified) flang/lib/Semantics/dump-expr.cpp (+1) 
- (modified) flang/lib/Semantics/expression.cpp (+13) 
- (modified) flang/lib/Semantics/tools.cpp (+38) 
- (modified) flang/test/Semantics/coarrays02.f90 (+17) 
- (modified) flang/test/Semantics/notifywait03.f90 (+1) 


``````````diff
diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index a36b8719e365d..eff4eb6a537cb 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -350,6 +350,7 @@ struct NodeVisitor {
   READ_FEATURE(ImageSelectorSpec)
   READ_FEATURE(ImageSelectorSpec::Stat)
   READ_FEATURE(ImageSelectorSpec::Team_Number)
+  READ_FEATURE(ImageSelectorSpec::Notify)
   READ_FEATURE(ImplicitPart)
   READ_FEATURE(ImplicitPartStmt)
   READ_FEATURE(ImplicitSpec)
diff --git a/flang/include/flang/Evaluate/traverse.h b/flang/include/flang/Evaluate/traverse.h
index 48aafa8982559..9a6978b6be6f7 100644
--- a/flang/include/flang/Evaluate/traverse.h
+++ b/flang/include/flang/Evaluate/traverse.h
@@ -146,7 +146,7 @@ class Traverse {
     return Combine(x.base(), x.subscript());
   }
   Result operator()(const CoarrayRef &x) const {
-    return Combine(x.base(), x.cosubscript(), x.stat(), x.team());
+    return Combine(x.base(), x.cosubscript(), x.stat(), x.team(), x.notify());
   }
   Result operator()(const DataRef &x) const { return visitor_(x.u); }
   Result operator()(const Substring &x) const {
diff --git a/flang/include/flang/Evaluate/variable.h b/flang/include/flang/Evaluate/variable.h
index 5c14421fd3a1b..6f1716f6e44b5 100644
--- a/flang/include/flang/Evaluate/variable.h
+++ b/flang/include/flang/Evaluate/variable.h
@@ -260,6 +260,9 @@ class CoarrayRef {
   // it's TEAM=.
   std::optional<Expr<SomeType>> team() const;
   CoarrayRef &set_team(Expr<SomeType> &&);
+  // When notify() is Expr<Some>, it's NOTIFY=.
+  std::optional<Expr<SomeType>> notify() const;
+  CoarrayRef &set_notify(Expr<SomeType> &&);
 
   int Rank() const;
   int Corank() const { return 0; }
@@ -274,6 +277,7 @@ class CoarrayRef {
   std::vector<Expr<SubscriptInteger>> cosubscript_;
   std::optional<common::CopyableIndirection<Expr<SomeInteger>>> stat_;
   std::optional<common::CopyableIndirection<Expr<SomeType>>> team_;
+  std::optional<common::CopyableIndirection<Expr<SomeType>>> notify_;
 };
 
 // R911 data-ref is defined syntactically as a series of part-refs, which
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e3eed6aed8079..0186181cb43d6 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -385,6 +385,7 @@ class ParseTreeDumper {
   NODE(parser, ImageSelectorSpec)
   NODE(ImageSelectorSpec, Stat)
   NODE(ImageSelectorSpec, Team_Number)
+  NODE(ImageSelectorSpec, Notify)
   NODE(parser, ImplicitPart)
   NODE(parser, ImplicitPartStmt)
   NODE(parser, ImplicitSpec)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 7e752eeb4dfe4..518be9b33a493 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -1682,12 +1682,14 @@ WRAPPER_CLASS(TeamValue, Scalar<common::Indirection<Expr>>);
 
 // R926 image-selector-spec ->
 //        STAT = stat-variable | TEAM = team-value |
-//        TEAM_NUMBER = scalar-int-expr
+//        TEAM_NUMBER = scalar-int-expr |
+//        NOTIFY = notify-variable
 struct ImageSelectorSpec {
   WRAPPER_CLASS(Stat, Scalar<Integer<common::Indirection<Variable>>>);
   WRAPPER_CLASS(Team_Number, ScalarIntExpr);
+  WRAPPER_CLASS(Notify, Scalar<common::Indirection<Variable>>);
   UNION_CLASS_BOILERPLATE(ImageSelectorSpec);
-  std::variant<Stat, TeamValue, Team_Number> u;
+  std::variant<Stat, TeamValue, Team_Number, Notify> u;
 };
 
 // R924 image-selector ->
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index fb670528f3ce4..e230620f5fcd3 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -107,6 +107,7 @@ bool IsBindCProcedure(const Scope &);
 // Returns a pointer to the function's symbol when true, else null
 const Symbol *IsFunctionResultWithSameNameAsFunction(const Symbol &);
 bool IsOrContainsEventOrLockComponent(const Symbol &);
+bool IsOrContainsNotifyComponent(const Symbol &);
 bool CanBeTypeBoundProc(const Symbol &);
 // Does a non-PARAMETER symbol have explicit initialization with =value or
 // =>target in its declaration (but not in a DATA statement)? (Being
@@ -642,6 +643,8 @@ using PotentialAndPointerComponentIterator =
 // dereferenced.
 PotentialComponentIterator::const_iterator FindEventOrLockPotentialComponent(
     const DerivedTypeSpec &, bool ignoreCoarrays = false);
+PotentialComponentIterator::const_iterator FindNotifyPotentialComponent(
+    const DerivedTypeSpec &, bool ignoreCoarrays = false);
 PotentialComponentIterator::const_iterator FindCoarrayPotentialComponent(
     const DerivedTypeSpec &);
 PotentialAndPointerComponentIterator::const_iterator
diff --git a/flang/lib/Evaluate/variable.cpp b/flang/lib/Evaluate/variable.cpp
index d1bff03a6ea5f..41003a2e53a8d 100644
--- a/flang/lib/Evaluate/variable.cpp
+++ b/flang/lib/Evaluate/variable.cpp
@@ -89,6 +89,14 @@ std::optional<Expr<SomeType>> CoarrayRef::team() const {
   }
 }
 
+std::optional<Expr<SomeType>> CoarrayRef::notify() const {
+  if (notify_) {
+    return notify_.value().value();
+  } else {
+    return std::nullopt;
+  }
+}
+
 CoarrayRef &CoarrayRef::set_stat(Expr<SomeInteger> &&v) {
   CHECK(IsVariable(v));
   stat_.emplace(std::move(v));
@@ -100,6 +108,11 @@ CoarrayRef &CoarrayRef::set_team(Expr<SomeType> &&v) {
   return *this;
 }
 
+CoarrayRef &CoarrayRef::set_notify(Expr<SomeType> &&v) {
+  notify_.emplace(std::move(v));
+  return *this;
+}
+
 const Symbol &CoarrayRef::GetFirstSymbol() const {
   return base().GetFirstSymbol();
 }
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index c65f51ce6cacd..30635b66bbd39 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -82,7 +82,7 @@ class HashEvaluateExpr {
          x.cosubscript())
       cosubs -= getHashValue(v);
     return getHashValue(x.base()) * 97u - cosubs + getHashValue(x.stat()) +
-           257u + getHashValue(x.team());
+           257u + getHashValue(x.team()) + getHashValue(x.notify());
   }
   static unsigned getHashValue(const Fortran::evaluate::NamedEntity &x) {
     if (x.IsSymbol())
@@ -341,7 +341,8 @@ class IsEqualEvaluateExpr {
                       const Fortran::evaluate::CoarrayRef &y) {
     return isEqual(x.base(), y.base()) &&
            isEqual(x.cosubscript(), y.cosubscript()) &&
-           isEqual(x.stat(), y.stat()) && isEqual(x.team(), y.team());
+           isEqual(x.stat(), y.stat()) && isEqual(x.team(), y.team()) &&
+           isEqual(x.notify(), y.notify());
   }
   static bool isEqual(const Fortran::evaluate::NamedEntity &x,
                       const Fortran::evaluate::NamedEntity &y) {
diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp
index fbe629ab52935..30bd807d16a46 100644
--- a/flang/lib/Parser/Fortran-parsers.cpp
+++ b/flang/lib/Parser/Fortran-parsers.cpp
@@ -1212,12 +1212,15 @@ TYPE_CONTEXT_PARSER("image selector"_en_US,
 
 // R926 image-selector-spec ->
 //        STAT = stat-variable | TEAM = team-value |
-//        TEAM_NUMBER = scalar-int-expr
+//        TEAM_NUMBER = scalar-int-expr |
+//        NOTIFY = notify-variable
 TYPE_PARSER(construct<ImageSelectorSpec>(construct<ImageSelectorSpec::Stat>(
                 "STAT =" >> scalar(integer(indirect(variable))))) ||
     construct<ImageSelectorSpec>(construct<TeamValue>("TEAM =" >> teamValue)) ||
     construct<ImageSelectorSpec>(construct<ImageSelectorSpec::Team_Number>(
-        "TEAM_NUMBER =" >> scalarIntExpr)))
+        "TEAM_NUMBER =" >> scalarIntExpr)) ||
+    construct<ImageSelectorSpec>(construct<ImageSelectorSpec::Notify>(
+        "NOTIFY =" >> scalar(indirect(variable)))))
 
 // R927 allocate-stmt ->
 //        ALLOCATE ( [type-spec ::] allocation-list [, alloc-opt-list] )
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index dcd1ac165adc1..942aacd777388 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -819,6 +819,7 @@ class UnparseVisitor {
       Word("TEAM=");
     }
   }
+  void Before(const ImageSelectorSpec::Notify &) { Word("NOTIFY="); }
   void Unparse(const AllocateStmt &x) { // R927
     Word("ALLOCATE(");
     Walk(std::get<std::optional<TypeSpec>>(x.t), "::");
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index f9d64485f1407..644c3e1effc21 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -762,6 +762,15 @@ void CheckHelper::CheckObjectEntity(
         messages_.Say(
             "Variable '%s' with EVENT_TYPE or LOCK_TYPE potential component '%s' must be a coarray"_err_en_US,
             symbol.name(), component.BuildResultDesignatorName());
+      } else if (IsNotifyType(derived)) { // C1612
+        messages_.Say(
+            "Variable '%s' with NOTIFY_TYPE must be a coarray"_err_en_US,
+            symbol.name());
+      } else if (auto component{FindNotifyPotentialComponent( // C1611
+                     *derived, /*ignoreCoarrays=*/true)}) {
+        messages_.Say(
+            "Variable '%s' with NOTIFY_TYPE potential component '%s' must be a coarray"_err_en_US,
+            symbol.name(), component.BuildResultDesignatorName());
       }
     }
   }
@@ -780,6 +789,10 @@ void CheckHelper::CheckObjectEntity(
         messages_.Say(
             "An INTENT(OUT) dummy argument may not be, or contain, EVENT_TYPE or LOCK_TYPE"_err_en_US);
       }
+      if (IsOrContainsNotifyComponent(symbol)) { // C1613
+        messages_.Say(
+            "An INTENT(OUT) dummy argument may not be, or contain, NOTIFY_TYPE"_err_en_US);
+      }
       if (IsAssumedSizeArray(symbol)) { // C834
         if (type && type->IsPolymorphic()) {
           messages_.Say(
diff --git a/flang/lib/Semantics/dump-expr.cpp b/flang/lib/Semantics/dump-expr.cpp
index 66cedab94bfb4..bd7373abffb31 100644
--- a/flang/lib/Semantics/dump-expr.cpp
+++ b/flang/lib/Semantics/dump-expr.cpp
@@ -25,6 +25,7 @@ void DumpEvaluateExpr::Show(const evaluate::CoarrayRef &x) {
   Show(x.cosubscript());
   Show(x.stat());
   Show(x.team());
+  Show(x.notify());
   Outdent();
 }
 
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index f4af738284ed7..b96dd77ac964c 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -1596,6 +1596,19 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::CoindexedNamedObject &x) {
                     coarrayRef.set_team(Expr<SomeType>{*expr});
                   }
                 }
+              },
+              [&](const parser::ImageSelectorSpec::Notify &x) {
+                Analyze(x.v);
+                if (const auto *expr{GetExpr(context_, x.v)}) {
+                  if (coarrayRef.notify()) {
+                    Say("coindexed reference has multiple NOTIFY= specifiers"_err_en_US);
+                  } else if (auto dyType{expr->GetType()};
+                      dyType && IsNotifyType(GetDerivedTypeSpec(*dyType))) {
+                    coarrayRef.set_notify(Expr<SomeType>{*expr});
+                  } else {
+                    Say("NOTIFY= specifier must have type NOTIFY_TYPE from ISO_FORTRAN_ENV"_err_en_US);
+                  }
+                }
               }},
           imageSelSpec.u);
     }
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index d27d250b3f11e..bc7a754cfc0d9 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -582,6 +582,18 @@ bool IsOrContainsEventOrLockComponent(const Symbol &original) {
   return false;
 }
 
+bool IsOrContainsNotifyComponent(const Symbol &original) {
+  const Symbol &symbol{ResolveAssociations(original, /*stopAtTypeGuard=*/true)};
+  if (evaluate::IsVariable(symbol)) {
+    if (const DeclTypeSpec *type{symbol.GetType()}) {
+      if (const DerivedTypeSpec *derived{type->AsDerived()}) {
+        return IsNotifyType(derived) || FindNotifyPotentialComponent(*derived);
+      }
+    }
+  }
+  return false;
+}
+
 // Check this symbol suitable as a type-bound procedure - C769
 bool CanBeTypeBoundProc(const Symbol &symbol) {
   if (IsDummy(symbol) || IsProcedurePointer(symbol)) {
@@ -1464,6 +1476,32 @@ PotentialComponentIterator::const_iterator FindEventOrLockPotentialComponent(
   return iter;
 }
 
+PotentialComponentIterator::const_iterator FindNotifyPotentialComponent(
+    const DerivedTypeSpec &derived, bool ignoreCoarrays) {
+  PotentialComponentIterator potentials{derived};
+  auto iter{potentials.begin()};
+  for (auto end{potentials.end()}; iter != end; ++iter) {
+    const Symbol &component{*iter};
+    if (const auto *object{component.detailsIf<ObjectEntityDetails>()}) {
+      if (const DeclTypeSpec *type{object->type()}) {
+        if (IsNotifyType(type->AsDerived())) {
+          if (!ignoreCoarrays) {
+            break; // found one
+          }
+          auto path{iter.GetComponentPath()};
+          path.pop_back();
+          if (std::find_if(path.begin(), path.end(), [](const Symbol &sym) {
+                return evaluate::IsCoarray(sym);
+              }) == path.end()) {
+            break; // found one not in a coarray
+          }
+        }
+      }
+    }
+  }
+  return iter;
+}
+
 UltimateComponentIterator::const_iterator FindAllocatableUltimateComponent(
     const DerivedTypeSpec &derived) {
   UltimateComponentIterator ultimates{derived};
diff --git a/flang/test/Semantics/coarrays02.f90 b/flang/test/Semantics/coarrays02.f90
index b16e0ccb58797..e866dd89c07ab 100644
--- a/flang/test/Semantics/coarrays02.f90
+++ b/flang/test/Semantics/coarrays02.f90
@@ -16,6 +16,8 @@ program main
   type(event_type) event
   !ERROR: Variable 'lock' with EVENT_TYPE or LOCK_TYPE must be a coarray
   type(lock_type) lock
+  !ERROR: Variable 'notify' with NOTIFY_TYPE must be a coarray
+  type(notify_type) notify
   integer :: local[*] ! ok in main
 end
 
@@ -120,3 +122,18 @@ subroutine s4
   !ERROR: Subscripts must appear in a coindexed reference when its base is an array
   print *, ta(1)%a[1]
 end
+
+subroutine s5(a, notify, res)
+  use iso_fortran_env
+  type t
+    type(notify_type) :: a
+  end type
+  real, intent(in) :: a[*]
+  type(event_type), intent(in) :: notify[*]
+  !ERROR: An INTENT(OUT) dummy argument may not be, or contain, NOTIFY_TYPE
+  type(notify_type), intent(out) :: res[*]
+  !ERROR: Variable 'bad' with NOTIFY_TYPE potential component '%a' must be a coarray
+  type(t) :: bad
+  !ERROR: NOTIFY= specifier must have type NOTIFY_TYPE from ISO_FORTRAN_ENV
+  print *, a[1, NOTIFY=notify]
+end
diff --git a/flang/test/Semantics/notifywait03.f90 b/flang/test/Semantics/notifywait03.f90
index 0fc56f66ad32d..a336a7a67669a 100644
--- a/flang/test/Semantics/notifywait03.f90
+++ b/flang/test/Semantics/notifywait03.f90
@@ -10,6 +10,7 @@ program test_notify_wait
   implicit none
 
   ! notify_type variables must be coarrays
+  !ERROR: Variable 'non_coarray' with NOTIFY_TYPE must be a coarray
   type(notify_type) :: non_coarray
 
   type(notify_type) :: notify_var[*], notify_array(2)[*]

``````````

</details>


https://github.com/llvm/llvm-project/pull/148810

From flang-commits at lists.llvm.org  Tue Jul 15 06:57:09 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 15 Jul 2025 06:57:09 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Avoid unnecessary parsing
 of OpenMP constructs (PR #148629)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148629@github.com>
Message-ID: <68765e35.050a0220.cb249.2f4d@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM. Does this lead to any measurable speedup?

https://github.com/llvm/llvm-project/pull/148629

From flang-commits at lists.llvm.org  Tue Jul 15 04:06:00 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 15 Jul 2025 04:06:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Parser] Add whitespace token after
 the sentinel in fixed form (PR #148825)
Message-ID: <llvm.org/llvm/llvm-project/pull/148825@github.com>


https://github.com/tblah created https://github.com/llvm/llvm-project/pull/148825

Fixes #148386

The first time the line was classified (using
`Prescanner::ClassifyLine(const char *)`) the line was correctly classified as a compiler directive. But then later on the token form is invoked (`Prescanner::ClassifyLine(TokenSequence, Provenance)`). This one incorrectly classified the line as a comment because there was no whitespace token right after the sentinel. This fixes the issue by ensuring this whitespace is added.

>From a377e70dcc587b554041cc02f542246bb6fc143a Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Tue, 15 Jul 2025 10:59:32 +0000
Subject: [PATCH] [flang][Parser] Add whitespace token after the sentinel in
 fixed form

Fixes #148386

The first time the line was classified (using
`Prescanner::ClassifyLine(const char *)`) the line was correctly
classified as a compiler directive. But then later on the token form is
invoked (`Prescanner::ClassifyLine(TokenSequence, Provenance)`). This
one incorrectly classified the line as a comment because there was no
whitespace token right after the sentinel. This fixes the issue by
ensuring this whitespace is added.
---
 flang/lib/Parser/prescan.cpp                  |  4 +++-
 .../Preprocessing/omp-sentinel-fixed-form.F   | 21 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Preprocessing/omp-sentinel-fixed-form.F

diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index ec894ab8513d2..3a9a475c365ee 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -179,7 +179,9 @@ void Prescanner::Statement() {
         EmitChar(tokens, *sp);
       }
       if (inFixedForm_) {
-        while (column_ < 6) {
+        // We need to add the whitespace after the sentinel because otherwise
+        // the line cannot be re-categorised as a compiler directive.
+        while (column_ <= 6) {
           if (*at_ == '\t') {
             tabInCurrentLine_ = true;
             ++at_;
diff --git a/flang/test/Preprocessing/omp-sentinel-fixed-form.F b/flang/test/Preprocessing/omp-sentinel-fixed-form.F
new file mode 100644
index 0000000000000..c8271682a5b92
--- /dev/null
+++ b/flang/test/Preprocessing/omp-sentinel-fixed-form.F
@@ -0,0 +1,21 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+#define OMP_TARGET        .true.
+#define OMP_SIMD    .false.
+      program test
+      implicit none
+      integer     i,j,n
+      n = 100
+! CHECK: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: TARGET TEAMS DISTRIBUTE PARALLEL&
+! CHECK: !$OMP& DO) DEFAULT(TARGET TEAMS LOOP)
+!$omp  metadirective
+!$omp& when(user={condition(OMP_TARGET.or.OMP_SIMD)}:
+!$omp&      target teams distribute parallel do )
+!$omp& default(target teams loop)
+      do i=0,n
+        do j=0,n
+          write(*,*) "Test"
+        enddo
+      enddo
+      return
+      end program


From flang-commits at lists.llvm.org  Tue Jul 15 04:06:31 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 04:06:31 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Parser] Add whitespace token after
 the sentinel in fixed form (PR #148825)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148825@github.com>
Message-ID: <68763637.170a0220.34a87f.de5a@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: Tom Eccles (tblah)

<details>
<summary>Changes</summary>

Fixes #<!-- -->148386

The first time the line was classified (using
`Prescanner::ClassifyLine(const char *)`) the line was correctly classified as a compiler directive. But then later on the token form is invoked (`Prescanner::ClassifyLine(TokenSequence, Provenance)`). This one incorrectly classified the line as a comment because there was no whitespace token right after the sentinel. This fixes the issue by ensuring this whitespace is added.

---
Full diff: https://github.com/llvm/llvm-project/pull/148825.diff


2 Files Affected:

- (modified) flang/lib/Parser/prescan.cpp (+3-1) 
- (added) flang/test/Preprocessing/omp-sentinel-fixed-form.F (+21) 


``````````diff
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index ec894ab8513d2..3a9a475c365ee 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -179,7 +179,9 @@ void Prescanner::Statement() {
         EmitChar(tokens, *sp);
       }
       if (inFixedForm_) {
-        while (column_ < 6) {
+        // We need to add the whitespace after the sentinel because otherwise
+        // the line cannot be re-categorised as a compiler directive.
+        while (column_ <= 6) {
           if (*at_ == '\t') {
             tabInCurrentLine_ = true;
             ++at_;
diff --git a/flang/test/Preprocessing/omp-sentinel-fixed-form.F b/flang/test/Preprocessing/omp-sentinel-fixed-form.F
new file mode 100644
index 0000000000000..c8271682a5b92
--- /dev/null
+++ b/flang/test/Preprocessing/omp-sentinel-fixed-form.F
@@ -0,0 +1,21 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+#define OMP_TARGET        .true.
+#define OMP_SIMD    .false.
+      program test
+      implicit none
+      integer     i,j,n
+      n = 100
+! CHECK: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: TARGET TEAMS DISTRIBUTE PARALLEL&
+! CHECK: !$OMP& DO) DEFAULT(TARGET TEAMS LOOP)
+!$omp  metadirective
+!$omp& when(user={condition(OMP_TARGET.or.OMP_SIMD)}:
+!$omp&      target teams distribute parallel do )
+!$omp& default(target teams loop)
+      do i=0,n
+        do j=0,n
+          write(*,*) "Test"
+        enddo
+      enddo
+      return
+      end program

``````````

</details>


https://github.com/llvm/llvm-project/pull/148825

From flang-commits at lists.llvm.org  Tue Jul 15 07:05:45 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 15 Jul 2025 07:05:45 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Avoid unnecessary parsing
 of OpenMP constructs (PR #148629)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148629@github.com>
Message-ID: <68766039.170a0220.20f3b5.edbd@mx.google.com>


kparzysz wrote:

I don't know.  If I come across an example of where this could make a difference, I'll measure it.

https://github.com/llvm/llvm-project/pull/148629

From flang-commits at lists.llvm.org  Tue Jul 15 07:05:59 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 07:05:59 -0700 (PDT)
Subject: [flang-commits] [flang] 51b6f64 - [flang][OpenMP] Avoid unnecessary
 parsing of OpenMP constructs (#148629)
Message-ID: <68766047.170a0220.132a56.e4ee@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-15T09:05:56-05:00
New Revision: 51b6f64b892bf57879389e7031051a3922aa5061

URL: https://github.com/llvm/llvm-project/commit/51b6f64b892bf57879389e7031051a3922aa5061
DIFF: https://github.com/llvm/llvm-project/commit/51b6f64b892bf57879389e7031051a3922aa5061.diff

LOG: [flang][OpenMP] Avoid unnecessary parsing of OpenMP constructs (#148629)

When parsing a specification part, the parser will look ahead to see if
the next construct is an executable construct. In doing so it will
invoke OpenMPConstruct parser, whereas the only necessary thing to check
would be the directive alone.

Added: 
    

Modified: 
    flang/include/flang/Parser/parse-tree.h
    flang/lib/Parser/openmp-parsers.cpp
    flang/lib/Parser/parse-tree.cpp
    flang/lib/Parser/program-parsers.cpp
    flang/lib/Parser/type-parsers.h
    flang/test/Parser/OpenMP/fail-construct1.f90

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index ab2dde7d5dfbe..cc1d032f94d4a 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3481,6 +3481,9 @@ struct OmpDirectiveName {
   // This allows "construct<OmpDirectiveName>(Verbatim("<name>"))".
   OmpDirectiveName(const Verbatim &name);
   using WrapperTrait = std::true_type;
+
+  bool IsExecutionPart() const; // Is allowed in the execution part
+
   CharBlock source;
   llvm::omp::Directive v{llvm::omp::Directive::OMPD_unknown};
 };
@@ -5028,6 +5031,13 @@ struct OpenMPLoopConstruct {
       t;
 };
 
+// Lookahead class to identify execution-part OpenMP constructs without
+// parsing the entire OpenMP construct.
+struct OpenMPExecDirective {
+  WRAPPER_CLASS_BOILERPLATE(OpenMPExecDirective, OmpDirectiveName);
+  CharBlock source;
+};
+
 struct OpenMPConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPConstruct);
   std::variant<OpenMPStandaloneConstruct, OpenMPSectionsConstruct,

diff  --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index d70aaab82cbab..76c9499410017 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -912,7 +912,7 @@ TYPE_PARSER(construct<OmpNumTasksClause>(
     scalarIntExpr))
 
 TYPE_PARSER(
-    construct<OmpObject>(designator) || construct<OmpObject>("/" >> name / "/"))
+    construct<OmpObject>(designator) || "/" >> construct<OmpObject>(name) / "/")
 
 // OMP 5.0 2.19.4.5 LASTPRIVATE ([lastprivate-modifier :] list)
 TYPE_PARSER(construct<OmpLastprivateClause>(
@@ -1757,6 +1757,13 @@ TYPE_PARSER(construct<OpenMPSectionsConstruct>(
     Parser<OmpBeginSectionsDirective>{} / endOmpLine,
     Parser<OmpSectionBlocks>{}, Parser<OmpEndSectionsDirective>{} / endOmpLine))
 
+static bool IsExecutionPart(const OmpDirectiveName &name) {
+  return name.IsExecutionPart();
+}
+
+TYPE_PARSER(construct<OpenMPExecDirective>(
+    startOmpLine >> predicated(Parser<OmpDirectiveName>{}, IsExecutionPart)))
+
 TYPE_CONTEXT_PARSER("OpenMP construct"_en_US,
     startOmpLine >>
         withMessage("expected OpenMP construct"_err_en_US,

diff  --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index 824612e49293f..7b46c9f469b06 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -368,6 +368,53 @@ llvm::omp::Clause OmpClause::Id() const {
   return std::visit([](auto &&s) { return getClauseIdForClass(s); }, u);
 }
 
+bool OmpDirectiveName::IsExecutionPart() const {
+  // Can the directive appear in the execution part of the program.
+  llvm::omp::Directive id{v};
+  switch (llvm::omp::getDirectiveCategory(id)) {
+  case llvm::omp::Category::Executable:
+    return true;
+  case llvm::omp::Category::Declarative:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_allocate:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Informational:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_assume:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Meta:
+    return true;
+  case llvm::omp::Category::Subsidiary:
+    switch (id) {
+    // TODO: case llvm::omp::Directive::OMPD_task_iteration:
+    case llvm::omp::Directive::OMPD_section:
+    case llvm::omp::Directive::OMPD_scan:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  case llvm::omp::Category::Utility:
+    switch (id) {
+    case llvm::omp::Directive::OMPD_error:
+    case llvm::omp::Directive::OMPD_nothing:
+      return true;
+    default:
+      return false;
+    }
+    break;
+  }
+  return false;
+}
+
 const OmpArgumentList &OmpDirectiveSpecification::Arguments() const {
   static OmpArgumentList empty{decltype(OmpArgumentList::v){}};
   if (auto &arguments = std::get<std::optional<OmpArgumentList>>(t)) {

diff  --git a/flang/lib/Parser/program-parsers.cpp b/flang/lib/Parser/program-parsers.cpp
index d6e7f11297187..5f4e62ffdbbf2 100644
--- a/flang/lib/Parser/program-parsers.cpp
+++ b/flang/lib/Parser/program-parsers.cpp
@@ -108,7 +108,7 @@ constexpr auto actionStmtLookAhead{first(actionStmt >> ok,
     "ALLOCATE ("_tok, "CALL" >> name >> "("_tok, "GO TO"_tok, "OPEN ("_tok,
     "PRINT"_tok / space / !"("_tok, "READ ("_tok, "WRITE ("_tok)};
 constexpr auto execPartLookAhead{first(actionStmtLookAhead >> ok,
-    openaccConstruct >> ok, openmpConstruct >> ok, "ASSOCIATE ("_tok,
+    openaccConstruct >> ok, openmpExecDirective >> ok, "ASSOCIATE ("_tok,
     "BLOCK"_tok, "SELECT"_tok, "CHANGE TEAM"_sptok, "CRITICAL"_tok, "DO"_tok,
     "IF ("_tok, "WHERE ("_tok, "FORALL ("_tok, "!$CUF"_tok)};
 constexpr auto declErrorRecovery{

diff  --git a/flang/lib/Parser/type-parsers.h b/flang/lib/Parser/type-parsers.h
index 8e91ede093fc4..3900c5a86c874 100644
--- a/flang/lib/Parser/type-parsers.h
+++ b/flang/lib/Parser/type-parsers.h
@@ -137,6 +137,7 @@ constexpr Parser<CompilerDirective> compilerDirective;
 constexpr Parser<OpenACCConstruct> openaccConstruct;
 constexpr Parser<OpenACCDeclarativeConstruct> openaccDeclarativeConstruct;
 constexpr Parser<OpenMPConstruct> openmpConstruct;
+constexpr Parser<OpenMPExecDirective> openmpExecDirective;
 constexpr Parser<OpenMPDeclarativeConstruct> openmpDeclarativeConstruct;
 constexpr Parser<OmpEndLoopDirective> ompEndLoopDirective;
 constexpr Parser<IntrinsicVectorTypeSpec> intrinsicVectorTypeSpec; // Extension

diff  --git a/flang/test/Parser/OpenMP/fail-construct1.f90 b/flang/test/Parser/OpenMP/fail-construct1.f90
index f0ee22125cee0..f0b3f7438ae58 100644
--- a/flang/test/Parser/OpenMP/fail-construct1.f90
+++ b/flang/test/Parser/OpenMP/fail-construct1.f90
@@ -1,5 +1,5 @@
 ! RUN: not %flang_fc1 -fsyntax-only -fopenmp %s 2>&1 | FileCheck %s
 
-! CHECK: error: expected OpenMP construct
 !$omp  parallel
+! CHECK: error: expected '!$OMP '
 end


        

From flang-commits at lists.llvm.org  Tue Jul 15 07:06:02 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 15 Jul 2025 07:06:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Avoid unnecessary parsing
 of OpenMP constructs (PR #148629)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148629@github.com>
Message-ID: <6876604a.170a0220.28b5e3.eddb@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/148629

From flang-commits at lists.llvm.org  Tue Jul 15 08:13:32 2025
From: flang-commits at lists.llvm.org (Krzysztof Drewniak via flang-commits)
Date: Tue, 15 Jul 2025 08:13:32 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add
	ComplexTOROCDLLibraryCalls pass (PR #144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <6876701c.a70a0220.287c38.5c39@mx.google.com>


https://github.com/krzysz00 approved this pull request.

Was on vacation, lgtm, approved

https://github.com/llvm/llvm-project/pull/144926

From flang-commits at lists.llvm.org  Tue Jul 15 08:15:46 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 15 Jul 2025 08:15:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Handle SEQUENCE derived types for
 array repacking. (PR #148777)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148777@github.com>
Message-ID: <687670a2.170a0220.1bfefe.355d@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM, thanks

https://github.com/llvm/llvm-project/pull/148777

From flang-commits at lists.llvm.org  Tue Jul 15 08:20:35 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 08:20:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Parser] Add whitespace token after
 the sentinel in fixed form (PR #148825)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148825@github.com>
Message-ID: <687671c3.170a0220.1e4ac5.5dd6@mx.google.com>


https://github.com/klausler approved this pull request.

Looks great; thanks for figuring this out.

https://github.com/llvm/llvm-project/pull/148825

From flang-commits at lists.llvm.org  Tue Jul 15 08:39:37 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 08:39:37 -0700 (PDT)
Subject: [flang-commits] [flang] c2e41be - [flang][Parser] Add whitespace
 token after the sentinel in fixed form (#148825)
Message-ID: <68767639.170a0220.1768e9.325f@mx.google.com>


Author: Tom Eccles
Date: 2025-07-15T16:39:34+01:00
New Revision: c2e41be50f2cc3d1e99c30b1996b8873b05c4b0e

URL: https://github.com/llvm/llvm-project/commit/c2e41be50f2cc3d1e99c30b1996b8873b05c4b0e
DIFF: https://github.com/llvm/llvm-project/commit/c2e41be50f2cc3d1e99c30b1996b8873b05c4b0e.diff

LOG: [flang][Parser] Add whitespace token after the sentinel in fixed form (#148825)

Fixes #148386

The first time the line was classified (using
`Prescanner::ClassifyLine(const char *)`) the line was correctly
classified as a compiler directive. But then later on the token form is
invoked (`Prescanner::ClassifyLine(TokenSequence, Provenance)`). This
one incorrectly classified the line as a comment because there was no
whitespace token right after the sentinel. This fixes the issue by
ensuring this whitespace is added.

Added: 
    flang/test/Preprocessing/omp-sentinel-fixed-form.F

Modified: 
    flang/lib/Parser/prescan.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index ec894ab8513d2..3a9a475c365ee 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -179,7 +179,9 @@ void Prescanner::Statement() {
         EmitChar(tokens, *sp);
       }
       if (inFixedForm_) {
-        while (column_ < 6) {
+        // We need to add the whitespace after the sentinel because otherwise
+        // the line cannot be re-categorised as a compiler directive.
+        while (column_ <= 6) {
           if (*at_ == '\t') {
             tabInCurrentLine_ = true;
             ++at_;

diff  --git a/flang/test/Preprocessing/omp-sentinel-fixed-form.F b/flang/test/Preprocessing/omp-sentinel-fixed-form.F
new file mode 100644
index 0000000000000..c8271682a5b92
--- /dev/null
+++ b/flang/test/Preprocessing/omp-sentinel-fixed-form.F
@@ -0,0 +1,21 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+#define OMP_TARGET        .true.
+#define OMP_SIMD    .false.
+      program test
+      implicit none
+      integer     i,j,n
+      n = 100
+! CHECK: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: TARGET TEAMS DISTRIBUTE PARALLEL&
+! CHECK: !$OMP& DO) DEFAULT(TARGET TEAMS LOOP)
+!$omp  metadirective
+!$omp& when(user={condition(OMP_TARGET.or.OMP_SIMD)}:
+!$omp&      target teams distribute parallel do )
+!$omp& default(target teams loop)
+      do i=0,n
+        do j=0,n
+          write(*,*) "Test"
+        enddo
+      enddo
+      return
+      end program


        

From flang-commits at lists.llvm.org  Tue Jul 15 08:39:40 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 15 Jul 2025 08:39:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Parser] Add whitespace token after
 the sentinel in fixed form (PR #148825)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148825@github.com>
Message-ID: <6876763c.050a0220.2b6cc.8635@mx.google.com>


https://github.com/tblah closed https://github.com/llvm/llvm-project/pull/148825

From flang-commits at lists.llvm.org  Tue Jul 15 08:40:14 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 15 Jul 2025 08:40:14 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Parser] Add whitespace token after
 the sentinel in fixed form (PR #148825)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148825@github.com>
Message-ID: <6876765e.170a0220.14075.338f@mx.google.com>


https://github.com/tblah milestoned https://github.com/llvm/llvm-project/pull/148825

From flang-commits at lists.llvm.org  Tue Jul 15 08:41:17 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 15 Jul 2025 08:41:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Parser] Add whitespace token after
 the sentinel in fixed form (PR #148825)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148825@github.com>
Message-ID: <6876769d.170a0220.2c907c.31c3@mx.google.com>


tblah wrote:

/cherry-pick c2e41be

https://github.com/llvm/llvm-project/pull/148825

From flang-commits at lists.llvm.org  Tue Jul 15 09:12:51 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Tue, 15 Jul 2025 09:12:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Better error message for ambiguous
	ASSIGNMENT(=) (PR #148720)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148720@github.com>
Message-ID: <68767e03.050a0220.6edb2.7287@mx.google.com>


https://github.com/DanielCChen approved this pull request.

LGTM. 
Thanks!

https://github.com/llvm/llvm-project/pull/148720

From flang-commits at lists.llvm.org  Tue Jul 15 09:16:04 2025
From: flang-commits at lists.llvm.org (=?UTF-8?B?Um9nZXIgRmVycmVyIEliw6HDsWV6?= via flang-commits)
Date: Tue, 15 Jul 2025 09:16:04 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <68767ec4.170a0220.34c7f3.4cc6@mx.google.com>


rofirrim wrote:

I'm a bit uncertain with what we want to do with `NumGeneratedLoopNests` and `NumGeneratedLoops`.

I understand that, outside of dependent contexts, this is some sort of synthesised attribute (in the base case from analysing the loop nests / canonical loop sequences) that can be used by an enclosing loop transformation to check it is still valid.

I wonder if an alternative approach is using a list of integers, one per loop representing the depth of the canonical loop contained in there. In lack of a better name, let's call this the GeneratedLoopSequence (`gls` in the examples, read the examples bottom-up)

```cpp
// after unroll gls = [], because it is not partial and there may not be loop anymore
#pragma omp unroll 
// after fuse gls = [ 1 ]
#pragma omp fuse
// from syntax gls = [ 1, 1 ]
{
   for (...) { }
   for (...) { }
}
```

```cpp
// after fuse gls = [ 6, 1 ]
#pragma omp fuse looprange(2, 2)
// from syntax gls = [ 6, 1, 1 ]
{
   // after tile gls = [ 6 ]
   #pragma omp tile sizes(x, y, z)
   // from syntax gls = [ 3 ]
   for (...) {  
      for (...) { 
        for (...) { 
        } 
      }
   }
   // from syntax gls = [ 1 ]
   for (...) { }
   // from syntax gls = [ 1 ]
   for (...) { }
}
```

```cpp
// after split gls = [ 1, 1]
#pragma omp split counts(a, b)
// from syntax, gls = [ 1 ] 
for (...) { }
```

(For dependent contexts I was thinking on making the GeneratedLoopSequence an `std::optional`, so it is explicitly absent and can be told apart from `[]`)

But I wonder if this approach is enough. I was considering the `apply` clause, when we get to implement it.  And maybe a list of integers is not enough?

```cpp
// after apply(unroll) gls = []
// after split gls = [ 1, 1 ]
#pragma omp split counts(a, b) apply(unroll)
// from syntax, gls = [ 1 ] 
for (...) { }
```

```cpp
// after apply(unroll(2)), non-partial unroll the second loop, gls = [1, ?not a loop anymore? ] 
// after split gls = [ 1, 1 ]
#pragma omp split counts(a, b) apply(unroll(2))
// from syntax, gls = [ 1 ] 
for (...) { }
```

```cpp
// after apply(split(2) counts(c, d)), gls = [1, [1, 1] ] (?)
// after split gls = [ 1, 1 ]
#pragma omp split counts(a, b) apply(split(2) counts(c, d))
// from syntax, gls = [ 1 ] 
for (...) { }
```


```cpp
// after apply(split counts(c, d)), gls = [[1, 1], [1, 1]] (???)
// after split gls = [ 1, 1 ]
#pragma omp split counts(a, b) apply(split counts(c, d))
// from syntax, gls = [ 1 ] 
for (...) { }
```
Maybe there is no need to recursively represent all the nested transformation?

Other examples, from OpenMP, seem OK:

```cpp
void span_apply(double A[128][128])
{
  // this is not a loop transformation but this is fine because gls is a singleton
  // and collapse is 2 ≤ 4
  #pragma omp for collapse(2)
  // from apply(grid: reverse, interchange) (this affects the first two loops) gls = [ 4 ] 
  // from tile gls = [ 4 ]
  #pragma omp tile sizes(16,16) apply(grid: interchange,reverse)
  // from syntax gls = [ 2 ]
  for (int i = 0; i < 128; ++i)
    for (int j = 0; j < 128; ++j)
       A[i][j] = A[i][j] + 1;
}
```

```cpp
void nested_apply(double A[100])
{
  // after apply(reverse), gls = [ 2 ]
  // after applyt(intratile: unroll partial(2)), gls = [ 2 ]
  // after tile: gls = [ 2 ]
  #pragma omp tile sizes(10) apply(intratile: unroll partial(2) apply(reverse))
  // from syntax, gls = [ 1 ]
  for (int i = 0; i < 100; ++i)
     A[i] = A[i] + 1;
}
```

Thoughts?

https://github.com/llvm/llvm-project/pull/139293

From flang-commits at lists.llvm.org  Tue Jul 15 09:22:15 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Tue, 15 Jul 2025 09:22:15 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <68768037.050a0220.3182a2.695b@mx.google.com>


https://github.com/akuhlens updated https://github.com/llvm/llvm-project/pull/148748

>From b9cdd2e72c9c9e6b10098c4144b3e840122b56d6 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Mon, 14 Jul 2025 15:56:39 -0700
Subject: [PATCH 1/2] initial commit

---
 flang/include/flang/Parser/message.h        | 37 ++++++++++++++-------
 flang/lib/Frontend/FrontendAction.cpp       |  6 ++--
 flang/lib/Parser/message.cpp                | 37 +++++++++++++--------
 flang/lib/Semantics/resolve-names.cpp       |  3 +-
 flang/lib/Semantics/semantics.cpp           |  5 ++-
 flang/test/Driver/color-diagnostics-scan.f  | 16 ++++-----
 flang/test/Driver/fatal-errors-warnings.f90 | 32 ++++++++++++++++++
 7 files changed, 96 insertions(+), 40 deletions(-)
 create mode 100644 flang/test/Driver/fatal-errors-warnings.f90

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index db1a0a65157e3..d1d313a70a016 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -56,13 +56,19 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity() const { return severity_; }
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    }
+    return severity_;
+  }
   MessageFixedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
-  bool IsFatal() const {
-    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  bool IsFatal(bool warningsAreErrors = false) const {
+    Severity sev{severity(warningsAreErrors)};
+    return sev == Severity::Error || sev == Severity::Todo;
   }
 
 private:
@@ -105,7 +111,7 @@ class MessageFormattedText {
 public:
   template <typename... A>
   MessageFormattedText(const MessageFixedText &text, A &&...x)
-      : severity_{text.severity()} {
+      : severity_{text.severity(false)} {
     Format(&text, Convert(std::forward<A>(x))...);
   }
   MessageFormattedText(const MessageFormattedText &) = default;
@@ -113,14 +119,20 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
-  bool IsFatal() const {
-    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    }
+    return severity_;
   }
-  Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  bool IsFatal(bool warningsAreErrors = false) const {
+    Severity sev{severity(warningsAreErrors)};
+    return sev == Severity::Error || sev == Severity::Todo;
+  }
   std::string MoveString() { return std::move(string_); }
   bool operator==(const MessageFormattedText &that) const {
     return severity_ == that.severity_ && string_ == that.string_;
@@ -281,8 +293,8 @@ class Message : public common::ReferenceCounted<Message> {
   }
 
   bool SortBefore(const Message &that) const;
-  bool IsFatal() const;
-  Severity severity() const;
+  bool IsFatal(bool warningsAreErrors = false) const;
+  Severity severity(bool warningsAreErrors = false) const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
   Message &set_languageFeature(common::LanguageFeature);
@@ -293,7 +305,8 @@ class Message : public common::ReferenceCounted<Message> {
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLine = true,
-      const common::LanguageFeatureControl *hintFlags = nullptr) const;
+      const common::LanguageFeatureControl *hintFlags = nullptr,
+      bool warningsAreErrors = false) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
@@ -355,9 +368,9 @@ class Messages {
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLines = true,
       const common::LanguageFeatureControl *hintFlags = nullptr,
-      std::size_t maxErrorsToEmit = 0) const;
+      std::size_t maxErrorsToEmit = 0, bool warningsAreErrors = false) const;
   void AttachTo(Message &, std::optional<Severity> = std::nullopt);
-  bool AnyFatalError() const;
+  bool AnyFatalError(bool warningsAreErrors = false) const;
 
 private:
   std::list<Message> messages_;
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index 2429e07e5b8c4..d32f477445f93 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -238,7 +238,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLines=*/true, &features, maxErrors);
+        /*echoSourceLines=*/true, &features, maxErrors,
+        instance->getInvocation().getWarnAsErr());
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -249,7 +250,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLine=*/true, &features, maxErrors);
+        /*echoSourceLine=*/true, &features, maxErrors,
+        instance->getInvocation().getWarnAsErr());
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 909fba948a45a..574e9f5d424f8 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,16 +161,21 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-bool Message::IsFatal() const {
-  return severity() == Severity::Error || severity() == Severity::Todo;
+bool Message::IsFatal(bool warningsAreErrors) const {
+  Severity sev{severity(warningsAreErrors)};
+  return sev == Severity::Error || sev == Severity::Todo;
 }
 
-Severity Message::severity() const {
+Severity Message::severity(bool warningsAreErrors) const {
   return common::visit(
       common::visitors{
           [](const MessageExpectedText &) { return Severity::Error; },
-          [](const MessageFixedText &x) { return x.severity(); },
-          [](const MessageFormattedText &x) { return x.severity(); },
+          [=](const MessageFixedText &x) {
+            return x.severity(warningsAreErrors);
+          },
+          [=](const MessageFormattedText &x) {
+            return x.severity(warningsAreErrors);
+          },
       },
       text_);
 }
@@ -295,15 +300,16 @@ static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine,
-    const common::LanguageFeatureControl *hintFlagPtr) const {
+    bool echoSourceLine, const common::LanguageFeatureControl *hintFlagPtr,
+    bool warningsAreErrors) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
   const std::string text{ToString()};
+  Severity sev{severity(warningsAreErrors)};
   const std::string hint{
       HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
-  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
-      PrefixColor(severity()), echoSourceLine);
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(sev),
+      PrefixColor(sev), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
   // attachment->attachmentIsContext_ indicates this.
@@ -453,7 +459,7 @@ void Messages::ResolveProvenances(const AllCookedSources &allCooked) {
 
 void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     bool echoSourceLines, const common::LanguageFeatureControl *hintFlagPtr,
-    std::size_t maxErrorsToEmit) const {
+    std::size_t maxErrorsToEmit, bool warningsAreErrors) const {
   std::vector<const Message *> sorted;
   for (const auto &msg : messages_) {
     sorted.push_back(&msg);
@@ -467,9 +473,9 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr, warningsAreErrors);
     lastMsg = msg;
-    if (msg->IsFatal()) {
+    if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
     }
     // If maxErrorsToEmit is 0, emit all errors, otherwise break after
@@ -491,9 +497,12 @@ void Messages::AttachTo(Message &msg, std::optional<Severity> severity) {
   messages_.clear();
 }
 
-bool Messages::AnyFatalError() const {
+bool Messages::AnyFatalError(bool warningsAreErrors) const {
+  if (messages_.empty()) {
+    return false;
+  }
   for (const auto &msg : messages_) {
-    if (msg.IsFatal()) {
+    if (msg.IsFatal(warningsAreErrors)) {
       return true;
     }
   }
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 96faa5fd954cd..df898f6e5651e 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7575,7 +7575,8 @@ bool DeclarationVisitor::OkToAddComponent(
       if (msg) {
         auto &said{Say2(name, std::move(*msg), *prev,
             "Previous declaration of '%s'"_en_US)};
-        if (msg->severity() == parser::Severity::Error) {
+        if (msg->severity(/*warningsAreErrors=*/false) ==
+            parser::Severity::Error) {
           Resolve(name, *prev);
           return false;
         }
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -376,8 +376,7 @@ const DeclTypeSpec &SemanticsContext::MakeLogicalType(int kind) {
 }
 
 bool SemanticsContext::AnyFatalError() const {
-  return !messages_.empty() &&
-      (warningsAreErrors_ || messages_.AnyFatalError());
+  return messages_.AnyFatalError(warningsAreErrors_);
 }
 bool SemanticsContext::HasError(const Symbol &symbol) {
   return errorSymbols_.count(symbol) > 0;
@@ -658,7 +657,7 @@ void Semantics::EmitMessages(llvm::raw_ostream &os) {
   context_.messages().ResolveProvenances(context_.allCookedSources());
   context_.messages().Emit(os, context_.allCookedSources(),
       /*echoSourceLine=*/true, &context_.languageFeatures(),
-      /*maxErrorsToEmit=*/context_.maxErrors());
+      context_.maxErrors(), context_.warningsAreErrors());
 }
 
 void SemanticsContext::DumpSymbols(llvm::raw_ostream &os) {
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index 29d4635b4fb03..2f647c923c2e6 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -3,24 +3,24 @@
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
-! RUN: not %flang %s -E -Werror -fcolor-diagnostics 2>&1 \
+! RUN: %flang %s -E -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fno-color-diagnostics 2>&1 \
+! RUN: %flang %s -E -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
-! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
+! RUN: %flang_fc1 -E %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 
-! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
+! RUN: %flang %s -E -fno-diagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color=always 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color=never 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
 
diff --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
new file mode 100644
index 0000000000000..b4b23230587a4
--- /dev/null
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -0,0 +1,32 @@
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
+
+module m
+    contains
+    subroutine foo(a)
+        real, intent(in), target :: a(:)
+    end subroutine
+end module
+
+program test
+    use m
+    real, target :: a(1)
+    real :: b(1)
+    call foo(a) ! ok
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(b)
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK3-NOT: error:
+    !CHECK3-NOT: warning:
+    call foo((a))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(a([1]))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(a(1))
+end
\ No newline at end of file

>From 31d79c710a3a0a727bcf2d94e73280a9f62bb0ca Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 09:21:19 -0700
Subject: [PATCH 2/2] don't display warnings as errors

---
 flang/lib/Parser/message.cpp                |  6 +++++-
 flang/test/Driver/color-diagnostics-scan.f  | 16 ++++++++--------
 flang/test/Driver/fatal-errors-warnings.f90 | 15 +++++++--------
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 574e9f5d424f8..6644fafffc1bc 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -473,7 +473,11 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr, warningsAreErrors);
+    // We think it is confusing to users to display warnings and other
+    // diagnostics as errors, instead we just treat them as errors for the
+    // purpose of failing.
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
+        /*warningsAreErrors=*/false);
     lastMsg = msg;
     if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index 2f647c923c2e6..29d4635b4fb03 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -3,24 +3,24 @@
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
-! RUN: %flang %s -E -fcolor-diagnostics 2>&1 \
+! RUN: not %flang %s -E -Werror -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fno-color-diagnostics 2>&1 \
+! RUN: not %flang %s -E -Werror -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
-! RUN: %flang_fc1 -E %s -fcolor-diagnostics 2>&1 \
+! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 
-! RUN: %flang %s -E -fdiagnostics-color 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fno-diagnostics-color 2>&1 \
+! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: %flang %s -E -fdiagnostics-color=always 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fdiagnostics-color=never 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
 
diff --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
index b4b23230587a4..2de09c3ed0778 100644
--- a/flang/test/Driver/fatal-errors-warnings.f90
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -1,4 +1,4 @@
-! RUN: not %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
 ! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
 ! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
 
@@ -15,18 +15,17 @@ program test
     real :: b(1)
     call foo(a) ! ok
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
-    !CHECK3: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} warning:
     call foo(b)
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
     !CHECK3-NOT: error:
     !CHECK3-NOT: warning:
     call foo((a))
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
     call foo(a([1]))
-    !CHECK1: fatal-errors-warnings.f90:{{.*}} error:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
-    call foo(a(1))
+    !! Hard error instead of warning if uncommented.
+    !call foo(a(1))
 end
\ No newline at end of file


From flang-commits at lists.llvm.org  Tue Jul 15 09:25:20 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Tue, 15 Jul 2025 09:25:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <687680f0.050a0220.2fb0cf.913c@mx.google.com>


https://github.com/akuhlens edited https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Tue Jul 15 09:33:44 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 09:33:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <687682e7.170a0220.17ef7d.71ce@mx.google.com>


================
@@ -56,13 +56,19 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity() const { return severity_; }
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
----------------
klausler wrote:

This member function is now no longer an accessor.  Maybe leave the accessor alone and create a new member function EffectiveSeverity or something.

Are you sure that all severity levels other than Error are warnings?

https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Tue Jul 15 09:33:43 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 09:33:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <687682e7.170a0220.19d7b5.46c7@mx.google.com>


================
@@ -467,9 +473,13 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
+    // We think it is confusing to users to display warnings and other
+    // diagnostics as errors, instead we just treat them as errors for the
+    // purpose of failing.
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
+        /*warningsAreErrors=*/false);
----------------
klausler wrote:

is this new argument just passing the. default value?

https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Tue Jul 15 09:33:44 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 09:33:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <687682e8.170a0220.3acb89.6825@mx.google.com>


================
@@ -105,22 +111,28 @@ class MessageFormattedText {
 public:
   template <typename... A>
   MessageFormattedText(const MessageFixedText &text, A &&...x)
-      : severity_{text.severity()} {
+      : severity_{text.severity(false)} {
----------------
klausler wrote:

argument not needed here, it's the default.  but `severity()` should remain just an accessor anyway.

https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Tue Jul 15 09:50:37 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 15 Jul 2025 09:50:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add missing symbol names to the
	error message (PR #148888)
Message-ID: <llvm.org/llvm/llvm-project/pull/148888@github.com>


https://github.com/eugeneepshteyn created https://github.com/llvm/llvm-project/pull/148888

Fixes #140485

>From bc9dbac2f7adf031873f627060be11c557c0b881 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 12:41:34 -0400
Subject: [PATCH] [flang] Add missing symbol names to the error message

Fixes #140485
---
 flang/lib/Semantics/resolve-names.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 96faa5fd954cd..b3268605e7c0c 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8574,8 +8574,10 @@ bool ResolveNamesVisitor::Pre(const parser::ImportStmt &x) {
         } else {
           Say(name,
               "A distinct '%s' is already present in this scope"_err_en_US)
-              .Attach(symbol->name(), "Previous declaration of '%s'"_en_US)
-              .Attach(outer->name(), "Declaration of '%s' in host scope"_en_US);
+              .Attach(symbol->name(), "Previous declaration of '%s'"_en_US,
+                  symbol->name().ToString())
+              .Attach(outer->name(), "Declaration of '%s' in host scope"_en_US,
+                  outer->name().ToString());
         }
       }
     } else {


From flang-commits at lists.llvm.org  Tue Jul 15 09:52:20 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 15 Jul 2025 09:52:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add missing symbol names to the
	error message (PR #148888)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148888@github.com>
Message-ID: <68768744.170a0220.23ff9d.56f5@mx.google.com>


eugeneepshteyn wrote:

New output:
```
$ flang -c snfm_import_err02_2.f90
error: Semantic errors in snfm_import_err02_2.f90
./snfm_import_err02_2.f90:20:15: error: A distinct 'i' is already present in this scope
         import i
                ^
./snfm_import_err02_2.f90:7:14: Previous declaration of 'i'
    integer :: i
               ^
./snfm_import_err02_2.f90:11:14: Declaration of 'i' in host scope
    real    :: i
               ^
```
There are already semantic tests that check for this error, they are just checking the first error line.

https://github.com/llvm/llvm-project/pull/148888

From flang-commits at lists.llvm.org  Tue Jul 15 09:52:26 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 15 Jul 2025 09:52:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add missing symbol names to the
	error message (PR #148888)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148888@github.com>
Message-ID: <6876874a.050a0220.1f1f39.75dd@mx.google.com>


https://github.com/eugeneepshteyn ready_for_review https://github.com/llvm/llvm-project/pull/148888

From flang-commits at lists.llvm.org  Tue Jul 15 09:54:56 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 09:54:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add missing symbol names to the
	error message (PR #148888)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148888@github.com>
Message-ID: <687687e0.050a0220.3c9e39.9572@mx.google.com>


https://github.com/klausler approved this pull request.


https://github.com/llvm/llvm-project/pull/148888

From flang-commits at lists.llvm.org  Tue Jul 15 09:59:43 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Tue, 15 Jul 2025 09:59:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <687688ff.630a0220.116441.74bd@mx.google.com>


================
@@ -56,13 +56,19 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity() const { return severity_; }
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
----------------
akuhlens wrote:

That is how the original code worked (which I didn't implement). I was essentially trying to reproduce the previous behavior only taking into account the new flag `-Wfatal-errros`. I am happy to revisit how the `-Werror` flags works, maybe in a different PR.

https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Tue Jul 15 10:01:37 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Tue, 15 Jul 2025 10:01:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <68768971.050a0220.2e3ccb.988a@mx.google.com>


================
@@ -467,9 +473,13 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
+    // We think it is confusing to users to display warnings and other
+    // diagnostics as errors, instead we just treat them as errors for the
+    // purpose of failing.
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
+        /*warningsAreErrors=*/false);
----------------
akuhlens wrote:

Yes, I was drawing attention to it by passing it explicitly. Since it wasn't the behavior that I expected.

https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Tue Jul 15 10:08:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 10:08:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add missing symbol names to the
	error message (PR #148888)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148888@github.com>
Message-ID: <68768b1e.630a0220.1e8c50.7532@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Eugene Epshteyn (eugeneepshteyn)

<details>
<summary>Changes</summary>

Fixes #<!-- -->140485

---
Full diff: https://github.com/llvm/llvm-project/pull/148888.diff


1 Files Affected:

- (modified) flang/lib/Semantics/resolve-names.cpp (+4-2) 


``````````diff
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 96faa5fd954cd..b3268605e7c0c 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8574,8 +8574,10 @@ bool ResolveNamesVisitor::Pre(const parser::ImportStmt &x) {
         } else {
           Say(name,
               "A distinct '%s' is already present in this scope"_err_en_US)
-              .Attach(symbol->name(), "Previous declaration of '%s'"_en_US)
-              .Attach(outer->name(), "Declaration of '%s' in host scope"_en_US);
+              .Attach(symbol->name(), "Previous declaration of '%s'"_en_US,
+                  symbol->name().ToString())
+              .Attach(outer->name(), "Declaration of '%s' in host scope"_en_US,
+                  outer->name().ToString());
         }
       }
     } else {

``````````

</details>


https://github.com/llvm/llvm-project/pull/148888

From flang-commits at lists.llvm.org  Tue Jul 15 10:18:54 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Tue, 15 Jul 2025 10:18:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add missing symbol names to the
	error message (PR #148888)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148888@github.com>
Message-ID: <68768d7e.170a0220.2c1076.48e1@mx.google.com>


https://github.com/akuhlens approved this pull request.

LGTM, one improvement might be to add a test that checks the error message.

https://github.com/llvm/llvm-project/pull/148888

From flang-commits at lists.llvm.org  Tue Jul 15 11:07:58 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Tue, 15 Jul 2025 11:07:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <687698fe.170a0220.1b377c.5284@mx.google.com>


https://github.com/akuhlens updated https://github.com/llvm/llvm-project/pull/148748

>From b9cdd2e72c9c9e6b10098c4144b3e840122b56d6 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Mon, 14 Jul 2025 15:56:39 -0700
Subject: [PATCH 1/3] initial commit

---
 flang/include/flang/Parser/message.h        | 37 ++++++++++++++-------
 flang/lib/Frontend/FrontendAction.cpp       |  6 ++--
 flang/lib/Parser/message.cpp                | 37 +++++++++++++--------
 flang/lib/Semantics/resolve-names.cpp       |  3 +-
 flang/lib/Semantics/semantics.cpp           |  5 ++-
 flang/test/Driver/color-diagnostics-scan.f  | 16 ++++-----
 flang/test/Driver/fatal-errors-warnings.f90 | 32 ++++++++++++++++++
 7 files changed, 96 insertions(+), 40 deletions(-)
 create mode 100644 flang/test/Driver/fatal-errors-warnings.f90

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index db1a0a65157e3..d1d313a70a016 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -56,13 +56,19 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity() const { return severity_; }
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    }
+    return severity_;
+  }
   MessageFixedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
-  bool IsFatal() const {
-    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  bool IsFatal(bool warningsAreErrors = false) const {
+    Severity sev{severity(warningsAreErrors)};
+    return sev == Severity::Error || sev == Severity::Todo;
   }
 
 private:
@@ -105,7 +111,7 @@ class MessageFormattedText {
 public:
   template <typename... A>
   MessageFormattedText(const MessageFixedText &text, A &&...x)
-      : severity_{text.severity()} {
+      : severity_{text.severity(false)} {
     Format(&text, Convert(std::forward<A>(x))...);
   }
   MessageFormattedText(const MessageFormattedText &) = default;
@@ -113,14 +119,20 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
-  bool IsFatal() const {
-    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    }
+    return severity_;
   }
-  Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  bool IsFatal(bool warningsAreErrors = false) const {
+    Severity sev{severity(warningsAreErrors)};
+    return sev == Severity::Error || sev == Severity::Todo;
+  }
   std::string MoveString() { return std::move(string_); }
   bool operator==(const MessageFormattedText &that) const {
     return severity_ == that.severity_ && string_ == that.string_;
@@ -281,8 +293,8 @@ class Message : public common::ReferenceCounted<Message> {
   }
 
   bool SortBefore(const Message &that) const;
-  bool IsFatal() const;
-  Severity severity() const;
+  bool IsFatal(bool warningsAreErrors = false) const;
+  Severity severity(bool warningsAreErrors = false) const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
   Message &set_languageFeature(common::LanguageFeature);
@@ -293,7 +305,8 @@ class Message : public common::ReferenceCounted<Message> {
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLine = true,
-      const common::LanguageFeatureControl *hintFlags = nullptr) const;
+      const common::LanguageFeatureControl *hintFlags = nullptr,
+      bool warningsAreErrors = false) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
@@ -355,9 +368,9 @@ class Messages {
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLines = true,
       const common::LanguageFeatureControl *hintFlags = nullptr,
-      std::size_t maxErrorsToEmit = 0) const;
+      std::size_t maxErrorsToEmit = 0, bool warningsAreErrors = false) const;
   void AttachTo(Message &, std::optional<Severity> = std::nullopt);
-  bool AnyFatalError() const;
+  bool AnyFatalError(bool warningsAreErrors = false) const;
 
 private:
   std::list<Message> messages_;
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index 2429e07e5b8c4..d32f477445f93 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -238,7 +238,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLines=*/true, &features, maxErrors);
+        /*echoSourceLines=*/true, &features, maxErrors,
+        instance->getInvocation().getWarnAsErr());
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -249,7 +250,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLine=*/true, &features, maxErrors);
+        /*echoSourceLine=*/true, &features, maxErrors,
+        instance->getInvocation().getWarnAsErr());
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 909fba948a45a..574e9f5d424f8 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,16 +161,21 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-bool Message::IsFatal() const {
-  return severity() == Severity::Error || severity() == Severity::Todo;
+bool Message::IsFatal(bool warningsAreErrors) const {
+  Severity sev{severity(warningsAreErrors)};
+  return sev == Severity::Error || sev == Severity::Todo;
 }
 
-Severity Message::severity() const {
+Severity Message::severity(bool warningsAreErrors) const {
   return common::visit(
       common::visitors{
           [](const MessageExpectedText &) { return Severity::Error; },
-          [](const MessageFixedText &x) { return x.severity(); },
-          [](const MessageFormattedText &x) { return x.severity(); },
+          [=](const MessageFixedText &x) {
+            return x.severity(warningsAreErrors);
+          },
+          [=](const MessageFormattedText &x) {
+            return x.severity(warningsAreErrors);
+          },
       },
       text_);
 }
@@ -295,15 +300,16 @@ static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine,
-    const common::LanguageFeatureControl *hintFlagPtr) const {
+    bool echoSourceLine, const common::LanguageFeatureControl *hintFlagPtr,
+    bool warningsAreErrors) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
   const std::string text{ToString()};
+  Severity sev{severity(warningsAreErrors)};
   const std::string hint{
       HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
-  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
-      PrefixColor(severity()), echoSourceLine);
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(sev),
+      PrefixColor(sev), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
   // attachment->attachmentIsContext_ indicates this.
@@ -453,7 +459,7 @@ void Messages::ResolveProvenances(const AllCookedSources &allCooked) {
 
 void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     bool echoSourceLines, const common::LanguageFeatureControl *hintFlagPtr,
-    std::size_t maxErrorsToEmit) const {
+    std::size_t maxErrorsToEmit, bool warningsAreErrors) const {
   std::vector<const Message *> sorted;
   for (const auto &msg : messages_) {
     sorted.push_back(&msg);
@@ -467,9 +473,9 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr, warningsAreErrors);
     lastMsg = msg;
-    if (msg->IsFatal()) {
+    if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
     }
     // If maxErrorsToEmit is 0, emit all errors, otherwise break after
@@ -491,9 +497,12 @@ void Messages::AttachTo(Message &msg, std::optional<Severity> severity) {
   messages_.clear();
 }
 
-bool Messages::AnyFatalError() const {
+bool Messages::AnyFatalError(bool warningsAreErrors) const {
+  if (messages_.empty()) {
+    return false;
+  }
   for (const auto &msg : messages_) {
-    if (msg.IsFatal()) {
+    if (msg.IsFatal(warningsAreErrors)) {
       return true;
     }
   }
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 96faa5fd954cd..df898f6e5651e 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7575,7 +7575,8 @@ bool DeclarationVisitor::OkToAddComponent(
       if (msg) {
         auto &said{Say2(name, std::move(*msg), *prev,
             "Previous declaration of '%s'"_en_US)};
-        if (msg->severity() == parser::Severity::Error) {
+        if (msg->severity(/*warningsAreErrors=*/false) ==
+            parser::Severity::Error) {
           Resolve(name, *prev);
           return false;
         }
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -376,8 +376,7 @@ const DeclTypeSpec &SemanticsContext::MakeLogicalType(int kind) {
 }
 
 bool SemanticsContext::AnyFatalError() const {
-  return !messages_.empty() &&
-      (warningsAreErrors_ || messages_.AnyFatalError());
+  return messages_.AnyFatalError(warningsAreErrors_);
 }
 bool SemanticsContext::HasError(const Symbol &symbol) {
   return errorSymbols_.count(symbol) > 0;
@@ -658,7 +657,7 @@ void Semantics::EmitMessages(llvm::raw_ostream &os) {
   context_.messages().ResolveProvenances(context_.allCookedSources());
   context_.messages().Emit(os, context_.allCookedSources(),
       /*echoSourceLine=*/true, &context_.languageFeatures(),
-      /*maxErrorsToEmit=*/context_.maxErrors());
+      context_.maxErrors(), context_.warningsAreErrors());
 }
 
 void SemanticsContext::DumpSymbols(llvm::raw_ostream &os) {
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index 29d4635b4fb03..2f647c923c2e6 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -3,24 +3,24 @@
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
-! RUN: not %flang %s -E -Werror -fcolor-diagnostics 2>&1 \
+! RUN: %flang %s -E -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fno-color-diagnostics 2>&1 \
+! RUN: %flang %s -E -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
-! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
+! RUN: %flang_fc1 -E %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 
-! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
+! RUN: %flang %s -E -fno-diagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color=always 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color=never 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
 
diff --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
new file mode 100644
index 0000000000000..b4b23230587a4
--- /dev/null
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -0,0 +1,32 @@
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
+
+module m
+    contains
+    subroutine foo(a)
+        real, intent(in), target :: a(:)
+    end subroutine
+end module
+
+program test
+    use m
+    real, target :: a(1)
+    real :: b(1)
+    call foo(a) ! ok
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(b)
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK3-NOT: error:
+    !CHECK3-NOT: warning:
+    call foo((a))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(a([1]))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(a(1))
+end
\ No newline at end of file

>From 31d79c710a3a0a727bcf2d94e73280a9f62bb0ca Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 09:21:19 -0700
Subject: [PATCH 2/3] don't display warnings as errors

---
 flang/lib/Parser/message.cpp                |  6 +++++-
 flang/test/Driver/color-diagnostics-scan.f  | 16 ++++++++--------
 flang/test/Driver/fatal-errors-warnings.f90 | 15 +++++++--------
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 574e9f5d424f8..6644fafffc1bc 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -473,7 +473,11 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr, warningsAreErrors);
+    // We think it is confusing to users to display warnings and other
+    // diagnostics as errors, instead we just treat them as errors for the
+    // purpose of failing.
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
+        /*warningsAreErrors=*/false);
     lastMsg = msg;
     if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index 2f647c923c2e6..29d4635b4fb03 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -3,24 +3,24 @@
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
-! RUN: %flang %s -E -fcolor-diagnostics 2>&1 \
+! RUN: not %flang %s -E -Werror -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fno-color-diagnostics 2>&1 \
+! RUN: not %flang %s -E -Werror -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
-! RUN: %flang_fc1 -E %s -fcolor-diagnostics 2>&1 \
+! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 
-! RUN: %flang %s -E -fdiagnostics-color 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fno-diagnostics-color 2>&1 \
+! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: %flang %s -E -fdiagnostics-color=always 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fdiagnostics-color=never 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
 
diff --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
index b4b23230587a4..2de09c3ed0778 100644
--- a/flang/test/Driver/fatal-errors-warnings.f90
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -1,4 +1,4 @@
-! RUN: not %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
 ! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
 ! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
 
@@ -15,18 +15,17 @@ program test
     real :: b(1)
     call foo(a) ! ok
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
-    !CHECK3: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} warning:
     call foo(b)
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
     !CHECK3-NOT: error:
     !CHECK3-NOT: warning:
     call foo((a))
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
     call foo(a([1]))
-    !CHECK1: fatal-errors-warnings.f90:{{.*}} error:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
-    call foo(a(1))
+    !! Hard error instead of warning if uncommented.
+    !call foo(a(1))
 end
\ No newline at end of file

>From 892e255be088b117b94203b66d75688fcd3ebc28 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 11:07:22 -0700
Subject: [PATCH 3/3] add EffectiveSeverity(...)

---
 flang/include/flang/Parser/message.h  | 42 +++++++++++++++-----------
 flang/lib/Frontend/FrontendAction.cpp | 11 +++----
 flang/lib/Parser/message.cpp          | 43 ++++++++++++++++-----------
 flang/lib/Semantics/resolve-names.cpp |  3 +-
 4 files changed, 54 insertions(+), 45 deletions(-)

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index d1d313a70a016..423856b084b92 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -56,18 +56,22 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity(bool warningsAreErrors = false) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    }
-    return severity_;
-  }
+  Severity severity() const { return severity_; }
   MessageFixedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  // This is currently emulating how the frontend is currently handling other
+  // diagnostic messages. We may want to be more nuanced in the future.
+  Severity EffectiveSeverity(bool warningsAreErrors) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    } else {
+      return severity_;
+    }
+  }
   bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{severity(warningsAreErrors)};
+    Severity sev{EffectiveSeverity(warningsAreErrors)};
     return sev == Severity::Error || sev == Severity::Todo;
   }
 
@@ -111,7 +115,7 @@ class MessageFormattedText {
 public:
   template <typename... A>
   MessageFormattedText(const MessageFixedText &text, A &&...x)
-      : severity_{text.severity(false)} {
+      : severity_{text.severity()} {
     Format(&text, Convert(std::forward<A>(x))...);
   }
   MessageFormattedText(const MessageFormattedText &) = default;
@@ -119,18 +123,20 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
-  Severity severity(bool warningsAreErrors = false) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    }
-    return severity_;
-  }
+  Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  Severity EffectiveSeverity(bool warningsAreErrors) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    } else {
+      return severity_;
+    }
+  }
   bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{severity(warningsAreErrors)};
+    Severity sev{EffectiveSeverity(warningsAreErrors)};
     return sev == Severity::Error || sev == Severity::Todo;
   }
   std::string MoveString() { return std::move(string_); }
@@ -294,7 +300,8 @@ class Message : public common::ReferenceCounted<Message> {
 
   bool SortBefore(const Message &that) const;
   bool IsFatal(bool warningsAreErrors = false) const;
-  Severity severity(bool warningsAreErrors = false) const;
+  Severity EffectiveSeverity(bool warningsAreErrors) const;
+  Severity severity() const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
   Message &set_languageFeature(common::LanguageFeature);
@@ -305,8 +312,7 @@ class Message : public common::ReferenceCounted<Message> {
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLine = true,
-      const common::LanguageFeatureControl *hintFlags = nullptr,
-      bool warningsAreErrors = false) const;
+      const common::LanguageFeatureControl *hintFlags = nullptr) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index d32f477445f93..58901c6000380 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -230,16 +230,14 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
   const common::LanguageFeatureControl &features{
       instance->getInvocation().getFortranOpts().features};
   const size_t maxErrors{instance->getInvocation().getMaxErrors()};
-  if (!instance->getParsing().messages().empty() &&
-      (instance->getInvocation().getWarnAsErr() ||
-       instance->getParsing().messages().AnyFatalError())) {
+  const bool warningsAreErrors{instance->getInvocation().getWarnAsErr()};
+  if (instance->getParsing().messages().AnyFatalError(warningsAreErrors)) {
     const unsigned diagID = instance->getDiagnostics().getCustomDiagID(
         clang::DiagnosticsEngine::Error, message);
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLines=*/true, &features, maxErrors,
-        instance->getInvocation().getWarnAsErr());
+        /*echoSourceLines=*/true, &features, maxErrors, warningsAreErrors);
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -250,8 +248,7 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLine=*/true, &features, maxErrors,
-        instance->getInvocation().getWarnAsErr());
+        /*echoSourceLine=*/true, &features, maxErrors, warningsAreErrors);
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 6644fafffc1bc..80fe5b2a14ad2 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,25 +161,35 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-bool Message::IsFatal(bool warningsAreErrors) const {
-  Severity sev{severity(warningsAreErrors)};
-  return sev == Severity::Error || sev == Severity::Todo;
-}
-
-Severity Message::severity(bool warningsAreErrors) const {
+Severity Message::EffectiveSeverity(bool warningsAreErrors) const {
   return common::visit(
       common::visitors{
           [](const MessageExpectedText &) { return Severity::Error; },
           [=](const MessageFixedText &x) {
-            return x.severity(warningsAreErrors);
+            return x.EffectiveSeverity(warningsAreErrors);
           },
           [=](const MessageFormattedText &x) {
-            return x.severity(warningsAreErrors);
+            return x.EffectiveSeverity(warningsAreErrors);
           },
       },
       text_);
 }
 
+bool Message::IsFatal(bool warningsAreErrors) const {
+  Severity sev{EffectiveSeverity(warningsAreErrors)};
+  return sev == Severity::Error || sev == Severity::Todo;
+}
+
+Severity Message::severity() const {
+  return common::visit(
+      common::visitors{
+          [](const MessageExpectedText &) { return Severity::Error; },
+          [](const MessageFixedText &x) { return x.severity(); },
+          [](const MessageFormattedText &x) { return x.severity(); },
+      },
+      text_);
+}
+
 Message &Message::set_severity(Severity severity) {
   common::visit(
       common::visitors{
@@ -299,17 +309,18 @@ static std::string HintLanguageControlFlag(
 static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
+// We think it is confusing to users to display warnings and other
+// diagnostics as errors. Don't use EffectiveSeverity() here.
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine, const common::LanguageFeatureControl *hintFlagPtr,
-    bool warningsAreErrors) const {
+    bool echoSourceLine,
+    const common::LanguageFeatureControl *hintFlagPtr) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
   const std::string text{ToString()};
-  Severity sev{severity(warningsAreErrors)};
   const std::string hint{
       HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
-  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(sev),
-      PrefixColor(sev), echoSourceLine);
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
+      PrefixColor(severity()), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
   // attachment->attachmentIsContext_ indicates this.
@@ -473,11 +484,7 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    // We think it is confusing to users to display warnings and other
-    // diagnostics as errors, instead we just treat them as errors for the
-    // purpose of failing.
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
-        /*warningsAreErrors=*/false);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
     if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index df898f6e5651e..96faa5fd954cd 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7575,8 +7575,7 @@ bool DeclarationVisitor::OkToAddComponent(
       if (msg) {
         auto &said{Say2(name, std::move(*msg), *prev,
             "Previous declaration of '%s'"_en_US)};
-        if (msg->severity(/*warningsAreErrors=*/false) ==
-            parser::Severity::Error) {
+        if (msg->severity() == parser::Severity::Error) {
           Resolve(name, *prev);
           return false;
         }


From flang-commits at lists.llvm.org  Tue Jul 15 11:10:21 2025
From: flang-commits at lists.llvm.org (Mark Danial via flang-commits)
Date: Tue, 15 Jul 2025 11:10:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] update ppc lit tests after using
 vector.insert and vector.extract (NFC) (PR #148775)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148775@github.com>
Message-ID: <6876998d.050a0220.25a564.983f@mx.google.com>


https://github.com/madanial0 approved this pull request.

LGTM!

https://github.com/llvm/llvm-project/pull/148775

From flang-commits at lists.llvm.org  Tue Jul 15 11:51:12 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 11:51:12 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148927@github.com>
Message-ID: <6876a320.170a0220.93f1f.5765@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Peter Klausler (klausler)

<details>
<summary>Changes</summary>

Defined I/O subroutines have UNIT= and IOSTAT= dummy arguments that are required to have type INTEGER with its default kind.  When that default kind is modified via -fdefault-integer-8, calls to defined I/O subroutines from the runtime don't work.

Add a flag to the two data structures shared between the compiler and the runtime support library to indicate that a defined I/O subroutine was compiled under -fdefault-integer-8.  This has been done in a compatible manner, so that existing binaries are compatible with the new library and new binaries are compatible with the old library, unless of course -fdefault-integer-8 is used.

Fixes https://github.com/llvm/llvm-project/issues/148638.

---

Patch is 82.72 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/148927.diff


17 Files Affected:

- (modified) flang-rt/include/flang-rt/runtime/non-tbp-dio.h (+6-1) 
- (modified) flang-rt/include/flang-rt/runtime/type-info.h (+7-9) 
- (modified) flang-rt/lib/runtime/derived.cpp (+1-1) 
- (modified) flang-rt/lib/runtime/descriptor-io.cpp (+47-16) 
- (modified) flang-rt/lib/runtime/non-tbp-dio.cpp (+1-1) 
- (modified) flang-rt/lib/runtime/type-info.cpp (+1-1) 
- (modified) flang/include/flang/Semantics/runtime-type-info.h (+6-1) 
- (modified) flang/lib/Lower/IO.cpp (+5-3) 
- (modified) flang/lib/Semantics/runtime-type-info.cpp (+19-9) 
- (modified) flang/module/__fortran_type_info.f90 (+1-1) 
- (modified) flang/test/Lower/io-derived-type.f90 (+29-29) 
- (modified) flang/test/Lower/namelist.f90 (+4-4) 
- (modified) flang/test/Lower/volatile-openmp.f90 (+4-4) 
- (modified) flang/test/Semantics/typeinfo01.f90 (+9-9) 
- (modified) flang/test/Semantics/typeinfo02.f90 (+2-2) 
- (modified) flang/test/Semantics/typeinfo09.f90 (+1-1) 
- (modified) flang/test/Semantics/typeinfo13.f90 (+1-1) 


``````````diff
diff --git a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
index 99d4113b6c7a8..26849298ec959 100644
--- a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
+++ b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
@@ -34,11 +34,16 @@ namespace Fortran::runtime::io {
 
 RT_OFFLOAD_API_GROUP_BEGIN
 
+enum NonTbpDefinedIoFlags {
+  IsDtvArgPolymorphic = 1 << 0, // first dummy arg is CLASS(T)
+  DefinedIoInteger8 = 1 << 1, // -fdefault-integer-8 affected UNIT= & IOSTAT=
+};
+
 struct NonTbpDefinedIo {
   const typeInfo::DerivedType &derivedType;
   void (*subroutine)(); // null means no non-TBP defined I/O here
   common::DefinedIo definedIo;
-  bool isDtvArgPolymorphic; // first dummy arg is CLASS(T)
+  std::uint8_t flags;
 };
 
 struct NonTbpDefinedIoTable {
diff --git a/flang-rt/include/flang-rt/runtime/type-info.h b/flang-rt/include/flang-rt/runtime/type-info.h
index a8d39f4f8a1a3..93bca24a602b4 100644
--- a/flang-rt/include/flang-rt/runtime/type-info.h
+++ b/flang-rt/include/flang-rt/runtime/type-info.h
@@ -143,9 +143,9 @@ class SpecialBinding {
   // I/O procedures that are not type-bound.
   RT_API_ATTRS SpecialBinding(Which which, ProcedurePointer proc,
       std::uint8_t isArgDescSet, std::uint8_t isTypeBound,
-      std::uint8_t isArgContiguousSet)
+      std::uint8_t specialCaseFlag)
       : which_{which}, isArgDescriptorSet_{isArgDescSet},
-        isTypeBound_{isTypeBound}, isArgContiguousSet_{isArgContiguousSet},
+        isTypeBound_{isTypeBound}, specialCaseFlag_{specialCaseFlag},
         proc_{proc} {}
 
   static constexpr RT_API_ATTRS Which RankFinal(int rank) {
@@ -153,13 +153,11 @@ class SpecialBinding {
   }
 
   RT_API_ATTRS Which which() const { return which_; }
+  RT_API_ATTRS bool specialCaseFlag() const { return specialCaseFlag_; }
   RT_API_ATTRS bool IsArgDescriptor(int zeroBasedArg) const {
     return (isArgDescriptorSet_ >> zeroBasedArg) & 1;
   }
   RT_API_ATTRS bool IsTypeBound() const { return isTypeBound_ != 0; }
-  RT_API_ATTRS bool IsArgContiguous(int zeroBasedArg) const {
-    return (isArgContiguousSet_ >> zeroBasedArg) & 1;
-  }
   template <typename PROC>
   RT_API_ATTRS PROC GetProc(const Binding *bindings = nullptr) const {
     if (bindings && isTypeBound_ > 0) {
@@ -203,10 +201,10 @@ class SpecialBinding {
   // When a special binding is type-bound, this is its binding's index (plus 1,
   // so that 0 signifies that it's not type-bound).
   std::uint8_t isTypeBound_{0};
-  // True when a FINAL subroutine has a dummy argument that is an array that
-  // is CONTIGUOUS or neither assumed-rank nor assumed-shape.
-  std::uint8_t isArgContiguousSet_{0};
-
+  // For a FINAL subroutine, set when it has a dummy argument that is an array
+  // that is CONTIGUOUS or neither assumed-rank nor assumed-shape.
+  // For a defined I/O subroutine, set when UNIT= and IOSTAT= are INTEGER(8).
+  std::uint8_t specialCaseFlag_{0};
   ProcedurePointer proc_{nullptr};
 };
 
diff --git a/flang-rt/lib/runtime/derived.cpp b/flang-rt/lib/runtime/derived.cpp
index bb9a68abef2a7..4ed0baaa3d108 100644
--- a/flang-rt/lib/runtime/derived.cpp
+++ b/flang-rt/lib/runtime/derived.cpp
@@ -270,7 +270,7 @@ static RT_API_ATTRS void CallFinalSubroutine(const Descriptor &descriptor,
       StaticDescriptor<maxRank, true, 10> statDesc;
       Descriptor &copy{statDesc.descriptor()};
       const Descriptor *argDescriptor{&descriptor};
-      if (descriptor.rank() > 0 && special->IsArgContiguous(0) &&
+      if (descriptor.rank() > 0 && special->specialCaseFlag() &&
           !descriptor.IsContiguous()) {
         // The FINAL subroutine demands a contiguous array argument, but
         // this INTENT(OUT) or intrinsic assignment LHS isn't contiguous.
diff --git a/flang-rt/lib/runtime/descriptor-io.cpp b/flang-rt/lib/runtime/descriptor-io.cpp
index b208cb2c397b3..50fb24c14c95d 100644
--- a/flang-rt/lib/runtime/descriptor-io.cpp
+++ b/flang-rt/lib/runtime/descriptor-io.cpp
@@ -84,8 +84,8 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
     ChildIo &child{external->PushChildIo(io)};
     // Child formatted I/O is nonadvancing by definition (F'2018 12.6.2.4).
     auto restorer{common::ScopedSet(io.mutableModes().nonAdvancing, true)};
-    int unit{external->unitNumber()};
-    int ioStat{IostatOk};
+    std::int32_t unit{external->unitNumber()};
+    std::int32_t ioStat{IostatOk};
     char ioMsg[100];
     Fortran::common::optional<std::int64_t> startPos;
     if (edit.descriptor == DataEdit::DefinedDerivedType &&
@@ -96,25 +96,48 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
     }
     const auto *bindings{
         derived.binding().OffsetElement<const typeInfo::Binding>()};
+    bool integer8{special.specialCaseFlag()};
     if (special.IsArgDescriptor(0)) {
       // "dtv" argument is "class(t)", pass a descriptor
-      auto *p{special.GetProc<void (*)(const Descriptor &, int &, char *,
-          const Descriptor &, int &, char *, std::size_t, std::size_t)>(
-          bindings)};
       StaticDescriptor<1, true, 10 /*?*/> elementStatDesc;
       Descriptor &elementDesc{elementStatDesc.descriptor()};
       elementDesc.Establish(
           derived, nullptr, 0, nullptr, CFI_attribute_pointer);
       elementDesc.set_base_addr(descriptor.Element<char>(subscripts));
-      p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen,
-          sizeof ioMsg);
+      if (integer8) { // 64-bit UNIT=/IOSTAT=
+        std::int64_t unit64{unit};
+        std::int64_t ioStat64{ioStat};
+        auto *p{special.GetProc<void (*)(const Descriptor &, std::int64_t &,
+            char *, const Descriptor &, std::int64_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(elementDesc, unit64, ioType, vListDesc, ioStat64, ioMsg, ioTypeLen,
+            sizeof ioMsg);
+        ioStat = ioStat64;
+      } else { // 32-bit UNIT=/IOSTAT=
+        auto *p{special.GetProc<void (*)(const Descriptor &, std::int32_t &,
+            char *, const Descriptor &, std::int32_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen,
+            sizeof ioMsg);
+      }
     } else {
       // "dtv" argument is "type(t)", pass a raw pointer
-      auto *p{special.GetProc<void (*)(const void *, int &, char *,
-          const Descriptor &, int &, char *, std::size_t, std::size_t)>(
-          bindings)};
-      p(descriptor.Element<char>(subscripts), unit, ioType, vListDesc, ioStat,
-          ioMsg, ioTypeLen, sizeof ioMsg);
+      if (integer8) { // 64-bit UNIT= and IOSTAT=
+        std::int64_t unit64{unit};
+        std::int64_t ioStat64{ioStat};
+        auto *p{special.GetProc<void (*)(const void *, std::int64_t &, char *,
+            const Descriptor &, std::int64_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(descriptor.Element<char>(subscripts), unit64, ioType, vListDesc,
+            ioStat64, ioMsg, ioTypeLen, sizeof ioMsg);
+        ioStat = ioStat64;
+      } else { // 32-bit UNIT= and IOSTAT=
+        auto *p{special.GetProc<void (*)(const void *, std::int32_t &, char *,
+            const Descriptor &, std::int32_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(descriptor.Element<char>(subscripts), unit, ioType, vListDesc, ioStat,
+            ioMsg, ioTypeLen, sizeof ioMsg);
+      }
     }
     handler.Forward(ioStat, ioMsg, sizeof ioMsg);
     external->PopChildIo(child);
@@ -458,11 +481,16 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
                       ? common::DefinedIo::ReadUnformatted
                       : common::DefinedIo::WriteUnformatted)}) {
             if (definedIo->subroutine) {
+              std::uint8_t isArgDescriptorSet{0};
+              if (definedIo->flags & IsDtvArgPolymorphic) {
+                isArgDescriptorSet = 1;
+              }
               typeInfo::SpecialBinding special{DIR == Direction::Input
                       ? typeInfo::SpecialBinding::Which::ReadUnformatted
                       : typeInfo::SpecialBinding::Which::WriteUnformatted,
-                  definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                  false};
+                  definedIo->subroutine, isArgDescriptorSet,
+                  /*IsTypeBound=*/false,
+                  /*specialCaseFlag=*/!!(definedIo->flags & DefinedIoInteger8)};
               if (DefinedUnformattedIo(io_, instance_, *type, special)) {
                 anyIoTookPlace_ = true;
                 return StatOk;
@@ -719,8 +747,11 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
             nonTbpSpecial_.emplace(DIR == Direction::Input
                     ? typeInfo::SpecialBinding::Which::ReadFormatted
                     : typeInfo::SpecialBinding::Which::WriteFormatted,
-                definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                false);
+                definedIo->subroutine,
+                /*isArgDescriptorSet=*/
+                (definedIo->flags & IsDtvArgPolymorphic) ? 1 : 0,
+                /*isTypeBound=*/false,
+                /*specialCaseFlag=*/!!(definedIo->flags & DefinedIoInteger8));
             special_ = &*nonTbpSpecial_;
           }
         }
diff --git a/flang-rt/lib/runtime/non-tbp-dio.cpp b/flang-rt/lib/runtime/non-tbp-dio.cpp
index 72101b06e0c6e..d516526033c27 100644
--- a/flang-rt/lib/runtime/non-tbp-dio.cpp
+++ b/flang-rt/lib/runtime/non-tbp-dio.cpp
@@ -17,7 +17,7 @@ const NonTbpDefinedIo *NonTbpDefinedIoTable::Find(
   for (const auto *p{item}; j-- > 0; ++p) {
     if (&p->derivedType == &type && p->definedIo == definedIo) {
       return p;
-    } else if (p->isDtvArgPolymorphic) {
+    } else if (p->flags & IsDtvArgPolymorphic) {
       for (const typeInfo::DerivedType *t{type.GetParentType()}; t;
            t = t->GetParentType()) {
         if (&p->derivedType == t && p->definedIo == definedIo) {
diff --git a/flang-rt/lib/runtime/type-info.cpp b/flang-rt/lib/runtime/type-info.cpp
index 3e1d7c9c3c788..50123f4cf321c 100644
--- a/flang-rt/lib/runtime/type-info.cpp
+++ b/flang-rt/lib/runtime/type-info.cpp
@@ -330,7 +330,7 @@ FILE *SpecialBinding::Dump(FILE *f) const {
   }
   std::fprintf(f, "    isArgDescriptorSet: 0x%x\n", isArgDescriptorSet_);
   std::fprintf(f, "    isTypeBound: %d\n", isTypeBound_);
-  std::fprintf(f, "    isArgContiguousSet: 0x%x\n", isArgContiguousSet_);
+  std::fprintf(f, "    specialCaseFlag 0x%x\n", specialCaseFlag_);
   std::fprintf(f, "    proc: %p\n", reinterpret_cast<void *>(proc_));
   return f;
 }
diff --git a/flang/include/flang/Semantics/runtime-type-info.h b/flang/include/flang/Semantics/runtime-type-info.h
index 6c5a061d1c1a2..94e8293b14643 100644
--- a/flang/include/flang/Semantics/runtime-type-info.h
+++ b/flang/include/flang/Semantics/runtime-type-info.h
@@ -52,10 +52,15 @@ constexpr char procCompName[]{"proc"};
 
 SymbolVector CollectBindings(const Scope &dtScope);
 
+enum NonTbpDefinedIoFlags {
+  IsDtvArgPolymorphic = 1 << 0,
+  DefinedIoInteger8 = 1 << 1,
+};
+
 struct NonTbpDefinedIo {
   const Symbol *subroutine;
   common::DefinedIo definedIo;
-  bool isDtvArgPolymorphic;
+  std::uint8_t flags;
 };
 
 std::multimap<const Symbol *, NonTbpDefinedIo>
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 63a612d7ead61..69d72d9d63b68 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -269,10 +269,12 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   mlir::Type sizeTy =
       fir::runtime::getModel<std::size_t>()(builder.getContext());
   mlir::Type intTy = fir::runtime::getModel<int>()(builder.getContext());
+  mlir::Type byteTy =
+      fir::runtime::getModel<std::uint8_t>()(builder.getContext());
   mlir::Type boolTy = fir::runtime::getModel<bool>()(builder.getContext());
   mlir::Type listTy = fir::SequenceType::get(
       definedIoProcMap.size(),
-      mlir::TupleType::get(context, {refTy, refTy, intTy, boolTy}));
+      mlir::TupleType::get(context, {refTy, refTy, intTy, byteTy}));
   mlir::Type tableTy = mlir::TupleType::get(
       context, {sizeTy, fir::ReferenceType::get(listTy), boolTy});
 
@@ -339,9 +341,9 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       insert(builder.createIntegerConstant(
           loc, intTy, static_cast<int>(iface.second.definedIo)));
       // polymorphic flag is set if first defined IO dummy arg is CLASS(T)
+      // defaultInt8 flag is set if -fdefined-integer-8
       // [bool isDtvArgPolymorphic]
-      insert(builder.createIntegerConstant(loc, boolTy,
-                                           iface.second.isDtvArgPolymorphic));
+      insert(builder.createIntegerConstant(loc, byteTy, iface.second.flags));
     }
     if (tableIsLocal)
       builder.create<fir::StoreOp>(loc, list, listAddr);
diff --git a/flang/lib/Semantics/runtime-type-info.cpp b/flang/lib/Semantics/runtime-type-info.cpp
index 51ba21a9e5edf..5916a07df7744 100644
--- a/flang/lib/Semantics/runtime-type-info.cpp
+++ b/flang/lib/Semantics/runtime-type-info.cpp
@@ -1131,7 +1131,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
   if (auto proc{evaluate::characteristics::Procedure::Characterize(
           specific, context_.foldingContext())}) {
     std::uint8_t isArgDescriptorSet{0};
-    std::uint8_t isArgContiguousSet{0};
+    bool specialCaseFlag{0};
     int argThatMightBeDescriptor{0};
     MaybeExpr which;
     if (isAssignment) {
@@ -1197,7 +1197,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
                         TypeAndShape::Attr::AssumedShape) ||
                 dummyData.attrs.test(evaluate::characteristics::
                         DummyDataObject::Attr::Contiguous)) {
-              isArgContiguousSet |= 1;
+              specialCaseFlag = true;
             }
           }
         }
@@ -1216,7 +1216,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
         return;
       }
       if (ddo->type.type().IsPolymorphic()) {
-        isArgDescriptorSet |= 1;
+        argThatMightBeDescriptor = 1;
       }
       switch (io.value()) {
       case common::DefinedIo::ReadFormatted:
@@ -1232,6 +1232,9 @@ void RuntimeTableBuilder::DescribeSpecialProc(
         which = writeUnformattedEnum_;
         break;
       }
+      if (context_.defaultKinds().GetDefaultKind(TypeCategory::Integer) == 8) {
+        specialCaseFlag = true; // UNIT= & IOSTAT= INTEGER(8)
+      }
     }
     if (argThatMightBeDescriptor != 0) {
       if (const auto *dummyData{
@@ -1262,8 +1265,8 @@ void RuntimeTableBuilder::DescribeSpecialProc(
     }
     CHECK(bindingIndex <= 255);
     AddValue(values, specialSchema_, "istypebound"s, IntExpr<1>(bindingIndex));
-    AddValue(values, specialSchema_, "isargcontiguousset"s,
-        IntExpr<1>(isArgContiguousSet));
+    AddValue(values, specialSchema_, "specialcaseflag"s,
+        IntExpr<1>(specialCaseFlag));
     AddValue(values, specialSchema_, procCompName,
         SomeExpr{evaluate::ProcedureDesignator{specific}});
     // index might already be present in the case of an override
@@ -1383,19 +1386,26 @@ CollectNonTbpDefinedIoGenericInterfaces(
               } else {
                 // Local scope's specific overrides host's for this type
                 bool updated{false};
+                std::uint8_t flags{0};
+                if (declType->IsPolymorphic()) {
+                  flags |= IsDtvArgPolymorphic;
+                }
+                if (scope.context().GetDefaultKind(TypeCategory::Integer) ==
+                    8) {
+                  flags |= DefinedIoInteger8;
+                }
                 for (auto [iter, end]{result.equal_range(dtDesc)}; iter != end;
                      ++iter) {
                   NonTbpDefinedIo &nonTbp{iter->second};
                   if (nonTbp.definedIo == which) {
                     nonTbp.subroutine = &*specific;
-                    nonTbp.isDtvArgPolymorphic = declType->IsPolymorphic();
+                    nonTbp.flags = flags;
                     updated = true;
                   }
                 }
                 if (!updated) {
-                  result.emplace(dtDesc,
-                      NonTbpDefinedIo{
-                          &*specific, which, declType->IsPolymorphic()});
+                  result.emplace(
+                      dtDesc, NonTbpDefinedIo{&*specific, which, flags});
                 }
               }
             }
diff --git a/flang/module/__fortran_type_info.f90 b/flang/module/__fortran_type_info.f90
index 8dd27d6e4c01b..6af2a5a5e30ff 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang/module/__fortran_type_info.f90
@@ -118,7 +118,7 @@
     integer(1) :: which ! SpecialBinding::Which
     integer(1) :: isArgDescriptorSet
     integer(1) :: isTypeBound ! binding index + 1, if any
-    integer(1) :: isArgContiguousSet
+    integer(1) :: specialCaseFlag
     integer(1) :: __padding0(4)
     type(__builtin_c_funptr) :: proc
   end type
diff --git a/flang/test/Lower/io-derived-type.f90 b/flang/test/Lower/io-derived-type.f90
index 7d2fef3faa2b7..7c289ce261678 100644
--- a/flang/test/Lower/io-derived-type.f90
+++ b/flang/test/Lower/io-derived-type.f90
@@ -37,16 +37,16 @@ subroutine test1
     import, all
     ! CHECK:   %[[V_16:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:   %[[V_17:[0-9]+]] = fir.convert %[[V_16]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:   %[[V_18:[0-9]+]] = fir.address_of(@_QQMmFtest1.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:   %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:   %[[V_18:[0-9]+]] = fir.address_of(@_QQMmFtest1.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:   %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:   %[[V_20:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_17]], %[[V_19]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
     print *, 'test1 outer, should call wft: ', t(1)
     block
       import, only: t
       ! CHECK:   %[[V_37:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
       ! CHECK:   %[[V_38:[0-9]+]] = fir.convert %[[V_37]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-      ! CHECK:   %[[V_39:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-      ! CHECK:   %[[V_40:[0-9]+]] = fir.convert %[[V_39]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+      ! CHECK:   %[[V_39:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+      ! CHECK:   %[[V_40:[0-9]+]] = fir.convert %[[V_39]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
       ! CHECK:   %[[V_41:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_38]], %[[V_40]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
       print *, 'test1 block, should not call wft: ', t(2)
     end block
@@ -56,8 +56,8 @@ subroutine test1
   subroutine test2
     ! CHECK:   %[[V_15:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:   %[[V_16:[0-9]+]] = fir.convert %[[V_15]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:   %[[V_17:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:   %[[V_18:[0-9]+]] = fir.convert %[[V_17]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.re...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/148927

From flang-commits at lists.llvm.org  Tue Jul 15 12:21:27 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Tue, 15 Jul 2025 12:21:27 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Move cuf.set_allocator_idx
 after derived-type init (PR #148936)
Message-ID: <llvm.org/llvm/llvm-project/pull/148936@github.com>


https://github.com/clementval created https://github.com/llvm/llvm-project/pull/148936

Derived type initialization overwrite the component descriptor. Place the `cuf.set_allocator_idx` after the initialization is performed. 

>From 054a5a37a98acb09852018e4443b1d624e577f6b Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Tue, 15 Jul 2025 12:19:50 -0700
Subject: [PATCH] [flang][cuda] Move cuf.set_allocator_idx after derived-type
 init

---
 flang/lib/Lower/ConvertVariable.cpp          | 167 +++++++++++--------
 flang/test/Lower/CUDA/cuda-set-allocator.cuf |  14 +-
 2 files changed, 105 insertions(+), 76 deletions(-)

diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index ffe456de56630..bfd4782c764fc 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -771,79 +771,9 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
       return builder.create<cuf::SharedMemoryOp>(loc, ty, nm, symNm, lenParams,
                                                  indices);
 
-    if (!cuf::isCUDADeviceContext(builder.getRegion())) {
-      mlir::Value alloc = builder.create<cuf::AllocOp>(
-          loc, ty, nm, symNm, dataAttr, lenParams, indices);
-      if (const auto *details{
-              ultimateSymbol
-                  .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
-        const Fortran::semantics::DeclTypeSpec *type{details->type()};
-        const Fortran::semantics::DerivedTypeSpec *derived{
-            type ? type->AsDerived() : nullptr};
-        if (derived) {
-          Fortran::semantics::UltimateComponentIterator components{*derived};
-          auto recTy = mlir::dyn_cast<fir::RecordType>(ty);
-
-          llvm::SmallVector<mlir::Value> coordinates;
-          for (const auto &sym : components) {
-            if (Fortran::semantics::IsDeviceAllocatable(sym)) {
-              unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
-              mlir::Type fieldTy;
-              std::vector<mlir::Value> coordinates;
-
-              if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                // Field found in the base record type.
-                auto fieldName = recTy.getTypeList()[fieldIdx].first;
-                fieldTy = recTy.getTypeList()[fieldIdx].second;
-                mlir::Value fieldIndex = builder.create<fir::FieldIndexOp>(
-                    loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
-                    recTy,
-                    /*typeParams=*/mlir::ValueRange{});
-                coordinates.push_back(fieldIndex);
-              } else {
-                // Field not found in base record type, search in potential
-                // record type components.
-                for (auto component : recTy.getTypeList()) {
-                  if (auto childRecTy =
-                          mlir::dyn_cast<fir::RecordType>(component.second)) {
-                    fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
-                    if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                      mlir::Value parentFieldIndex =
-                          builder.create<fir::FieldIndexOp>(
-                              loc, fir::FieldType::get(childRecTy.getContext()),
-                              component.first, recTy,
-                              /*typeParams=*/mlir::ValueRange{});
-                      coordinates.push_back(parentFieldIndex);
-                      auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
-                      fieldTy = childRecTy.getTypeList()[fieldIdx].second;
-                      mlir::Value childFieldIndex =
-                          builder.create<fir::FieldIndexOp>(
-                              loc, fir::FieldType::get(fieldTy.getContext()),
-                              fieldName, childRecTy,
-                              /*typeParams=*/mlir::ValueRange{});
-                      coordinates.push_back(childFieldIndex);
-                      break;
-                    }
-                  }
-                }
-              }
-
-              if (coordinates.empty())
-                TODO(loc, "device resident component in complex derived-type "
-                          "hierarchy");
-
-              mlir::Value comp = builder.create<fir::CoordinateOp>(
-                  loc, builder.getRefType(fieldTy), alloc, coordinates);
-              cuf::DataAttributeAttr dataAttr =
-                  Fortran::lower::translateSymbolCUFDataAttribute(
-                      builder.getContext(), sym);
-              builder.create<cuf::SetAllocatorIndexOp>(loc, comp, dataAttr);
-            }
-          }
-        }
-      }
-      return alloc;
-    }
+    if (!cuf::isCUDADeviceContext(builder.getRegion()))
+      return builder.create<cuf::AllocOp>(loc, ty, nm, symNm, dataAttr,
+                                          lenParams, indices);
   }
 
   // Let the builder do all the heavy lifting.
@@ -857,6 +787,91 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   return res;
 }
 
+/// Device allocatable component in a derived-type don't have the correct
+/// allocator index in their descriptor when they are created. After
+/// initialization, cuf.set_allocator_idx operations are inserted to set the
+/// correct allocator index for each device component.
+static void
+initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
+                                   const Fortran::semantics::Symbol &symbol,
+                                   Fortran::lower::SymMap &symMap) {
+  if (const auto *details{
+          symbol.GetUltimate()
+              .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
+    const Fortran::semantics::DeclTypeSpec *type{details->type()};
+    const Fortran::semantics::DerivedTypeSpec *derived{type ? type->AsDerived()
+                                                            : nullptr};
+    if (derived) {
+      fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+      mlir::Location loc = converter.getCurrentLocation();
+
+      fir::ExtendedValue exv =
+          converter.getSymbolExtendedValue(symbol.GetUltimate(), &symMap);
+      auto recTy = mlir::dyn_cast<fir::RecordType>(
+          fir::unwrapRefType(fir::getBase(exv).getType()));
+      assert(recTy && "expected fir::RecordType");
+
+      llvm::SmallVector<mlir::Value> coordinates;
+      Fortran::semantics::UltimateComponentIterator components{*derived};
+      for (const auto &sym : components) {
+        if (Fortran::semantics::IsDeviceAllocatable(sym)) {
+          unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
+          mlir::Type fieldTy;
+          std::vector<mlir::Value> coordinates;
+
+          if (fieldIdx != std::numeric_limits<unsigned>::max()) {
+            // Field found in the base record type.
+            auto fieldName = recTy.getTypeList()[fieldIdx].first;
+            fieldTy = recTy.getTypeList()[fieldIdx].second;
+            mlir::Value fieldIndex = builder.create<fir::FieldIndexOp>(
+                loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
+                recTy,
+                /*typeParams=*/mlir::ValueRange{});
+            coordinates.push_back(fieldIndex);
+          } else {
+            // Field not found in base record type, search in potential
+            // record type components.
+            for (auto component : recTy.getTypeList()) {
+              if (auto childRecTy =
+                      mlir::dyn_cast<fir::RecordType>(component.second)) {
+                fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
+                if (fieldIdx != std::numeric_limits<unsigned>::max()) {
+                  mlir::Value parentFieldIndex =
+                      builder.create<fir::FieldIndexOp>(
+                          loc, fir::FieldType::get(childRecTy.getContext()),
+                          component.first, recTy,
+                          /*typeParams=*/mlir::ValueRange{});
+                  coordinates.push_back(parentFieldIndex);
+                  auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
+                  fieldTy = childRecTy.getTypeList()[fieldIdx].second;
+                  mlir::Value childFieldIndex =
+                      builder.create<fir::FieldIndexOp>(
+                          loc, fir::FieldType::get(fieldTy.getContext()),
+                          fieldName, childRecTy,
+                          /*typeParams=*/mlir::ValueRange{});
+                  coordinates.push_back(childFieldIndex);
+                  break;
+                }
+              }
+            }
+          }
+
+          if (coordinates.empty())
+            TODO(loc, "device resident component in complex derived-type "
+                      "hierarchy");
+
+          mlir::Value comp = builder.create<fir::CoordinateOp>(
+              loc, builder.getRefType(fieldTy), fir::getBase(exv), coordinates);
+          cuf::DataAttributeAttr dataAttr =
+              Fortran::lower::translateSymbolCUFDataAttribute(
+                  builder.getContext(), sym);
+          builder.create<cuf::SetAllocatorIndexOp>(loc, comp, dataAttr);
+        }
+      }
+    }
+  }
+}
+
 /// Must \p var be default initialized at runtime when entering its scope.
 static bool
 mustBeDefaultInitializedAtRuntime(const Fortran::lower::pft::Variable &var) {
@@ -1179,6 +1194,9 @@ static void instantiateLocal(Fortran::lower::AbstractConverter &converter,
   if (mustBeDefaultInitializedAtRuntime(var))
     Fortran::lower::defaultInitializeAtRuntime(converter, var.getSymbol(),
                                                symMap);
+  if (converter.getFoldingContext().languageFeatures().IsEnabled(
+          Fortran::common::LanguageFeature::CUDA))
+    initializeDeviceComponentAllocator(converter, var.getSymbol(), symMap);
   auto *builder = &converter.getFirOpBuilder();
   if (needCUDAAlloc(var.getSymbol()) &&
       !cuf::isCUDADeviceContext(builder->getRegion())) {
@@ -1437,6 +1455,9 @@ static void instantiateAlias(Fortran::lower::AbstractConverter &converter,
   if (mustBeDefaultInitializedAtRuntime(var))
     Fortran::lower::defaultInitializeAtRuntime(converter, var.getSymbol(),
                                                symMap);
+  if (converter.getFoldingContext().languageFeatures().IsEnabled(
+          Fortran::common::LanguageFeature::CUDA))
+    initializeDeviceComponentAllocator(converter, var.getSymbol(), symMap);
 }
 
 //===--------------------------------------------------------------===//
diff --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index bf74e012a639d..85715edf26942 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -12,10 +12,18 @@ contains
   end subroutine
 
 ! CHECK-LABEL: func.func @_QMm1Psub1()
-! CHECK: %[[DT:.*]] = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} -> !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
-! CHECK: %[[X:.*]] = fir.coordinate_of %[[DT]], x : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} -> !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DT:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>)
+! CHECK: fir.address_of(@_QQ_QMm1Tty_device.DerivedInit)
+! CHECK: fir.copy 
+! CHECK: %[[X:.*]] = fir.coordinate_of %[[DT]]#0, x : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[X]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
-! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]], z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]]#0, z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[Z]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
 
 end module
+
+
+
+    
+  


From flang-commits at lists.llvm.org  Tue Jul 15 12:21:56 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 12:21:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Move cuf.set_allocator_idx
 after derived-type init (PR #148936)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148936@github.com>
Message-ID: <6876aa54.170a0220.34c7f3.63a9@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Valentin Clement (バレンタイン クレメン) (clementval)

<details>
<summary>Changes</summary>

Derived type initialization overwrite the component descriptor. Place the `cuf.set_allocator_idx` after the initialization is performed. 

---
Full diff: https://github.com/llvm/llvm-project/pull/148936.diff


2 Files Affected:

- (modified) flang/lib/Lower/ConvertVariable.cpp (+94-73) 
- (modified) flang/test/Lower/CUDA/cuda-set-allocator.cuf (+11-3) 


``````````diff
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index ffe456de56630..bfd4782c764fc 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -771,79 +771,9 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
       return builder.create<cuf::SharedMemoryOp>(loc, ty, nm, symNm, lenParams,
                                                  indices);
 
-    if (!cuf::isCUDADeviceContext(builder.getRegion())) {
-      mlir::Value alloc = builder.create<cuf::AllocOp>(
-          loc, ty, nm, symNm, dataAttr, lenParams, indices);
-      if (const auto *details{
-              ultimateSymbol
-                  .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
-        const Fortran::semantics::DeclTypeSpec *type{details->type()};
-        const Fortran::semantics::DerivedTypeSpec *derived{
-            type ? type->AsDerived() : nullptr};
-        if (derived) {
-          Fortran::semantics::UltimateComponentIterator components{*derived};
-          auto recTy = mlir::dyn_cast<fir::RecordType>(ty);
-
-          llvm::SmallVector<mlir::Value> coordinates;
-          for (const auto &sym : components) {
-            if (Fortran::semantics::IsDeviceAllocatable(sym)) {
-              unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
-              mlir::Type fieldTy;
-              std::vector<mlir::Value> coordinates;
-
-              if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                // Field found in the base record type.
-                auto fieldName = recTy.getTypeList()[fieldIdx].first;
-                fieldTy = recTy.getTypeList()[fieldIdx].second;
-                mlir::Value fieldIndex = builder.create<fir::FieldIndexOp>(
-                    loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
-                    recTy,
-                    /*typeParams=*/mlir::ValueRange{});
-                coordinates.push_back(fieldIndex);
-              } else {
-                // Field not found in base record type, search in potential
-                // record type components.
-                for (auto component : recTy.getTypeList()) {
-                  if (auto childRecTy =
-                          mlir::dyn_cast<fir::RecordType>(component.second)) {
-                    fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
-                    if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                      mlir::Value parentFieldIndex =
-                          builder.create<fir::FieldIndexOp>(
-                              loc, fir::FieldType::get(childRecTy.getContext()),
-                              component.first, recTy,
-                              /*typeParams=*/mlir::ValueRange{});
-                      coordinates.push_back(parentFieldIndex);
-                      auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
-                      fieldTy = childRecTy.getTypeList()[fieldIdx].second;
-                      mlir::Value childFieldIndex =
-                          builder.create<fir::FieldIndexOp>(
-                              loc, fir::FieldType::get(fieldTy.getContext()),
-                              fieldName, childRecTy,
-                              /*typeParams=*/mlir::ValueRange{});
-                      coordinates.push_back(childFieldIndex);
-                      break;
-                    }
-                  }
-                }
-              }
-
-              if (coordinates.empty())
-                TODO(loc, "device resident component in complex derived-type "
-                          "hierarchy");
-
-              mlir::Value comp = builder.create<fir::CoordinateOp>(
-                  loc, builder.getRefType(fieldTy), alloc, coordinates);
-              cuf::DataAttributeAttr dataAttr =
-                  Fortran::lower::translateSymbolCUFDataAttribute(
-                      builder.getContext(), sym);
-              builder.create<cuf::SetAllocatorIndexOp>(loc, comp, dataAttr);
-            }
-          }
-        }
-      }
-      return alloc;
-    }
+    if (!cuf::isCUDADeviceContext(builder.getRegion()))
+      return builder.create<cuf::AllocOp>(loc, ty, nm, symNm, dataAttr,
+                                          lenParams, indices);
   }
 
   // Let the builder do all the heavy lifting.
@@ -857,6 +787,91 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   return res;
 }
 
+/// Device allocatable component in a derived-type don't have the correct
+/// allocator index in their descriptor when they are created. After
+/// initialization, cuf.set_allocator_idx operations are inserted to set the
+/// correct allocator index for each device component.
+static void
+initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
+                                   const Fortran::semantics::Symbol &symbol,
+                                   Fortran::lower::SymMap &symMap) {
+  if (const auto *details{
+          symbol.GetUltimate()
+              .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
+    const Fortran::semantics::DeclTypeSpec *type{details->type()};
+    const Fortran::semantics::DerivedTypeSpec *derived{type ? type->AsDerived()
+                                                            : nullptr};
+    if (derived) {
+      fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+      mlir::Location loc = converter.getCurrentLocation();
+
+      fir::ExtendedValue exv =
+          converter.getSymbolExtendedValue(symbol.GetUltimate(), &symMap);
+      auto recTy = mlir::dyn_cast<fir::RecordType>(
+          fir::unwrapRefType(fir::getBase(exv).getType()));
+      assert(recTy && "expected fir::RecordType");
+
+      llvm::SmallVector<mlir::Value> coordinates;
+      Fortran::semantics::UltimateComponentIterator components{*derived};
+      for (const auto &sym : components) {
+        if (Fortran::semantics::IsDeviceAllocatable(sym)) {
+          unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
+          mlir::Type fieldTy;
+          std::vector<mlir::Value> coordinates;
+
+          if (fieldIdx != std::numeric_limits<unsigned>::max()) {
+            // Field found in the base record type.
+            auto fieldName = recTy.getTypeList()[fieldIdx].first;
+            fieldTy = recTy.getTypeList()[fieldIdx].second;
+            mlir::Value fieldIndex = builder.create<fir::FieldIndexOp>(
+                loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
+                recTy,
+                /*typeParams=*/mlir::ValueRange{});
+            coordinates.push_back(fieldIndex);
+          } else {
+            // Field not found in base record type, search in potential
+            // record type components.
+            for (auto component : recTy.getTypeList()) {
+              if (auto childRecTy =
+                      mlir::dyn_cast<fir::RecordType>(component.second)) {
+                fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
+                if (fieldIdx != std::numeric_limits<unsigned>::max()) {
+                  mlir::Value parentFieldIndex =
+                      builder.create<fir::FieldIndexOp>(
+                          loc, fir::FieldType::get(childRecTy.getContext()),
+                          component.first, recTy,
+                          /*typeParams=*/mlir::ValueRange{});
+                  coordinates.push_back(parentFieldIndex);
+                  auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
+                  fieldTy = childRecTy.getTypeList()[fieldIdx].second;
+                  mlir::Value childFieldIndex =
+                      builder.create<fir::FieldIndexOp>(
+                          loc, fir::FieldType::get(fieldTy.getContext()),
+                          fieldName, childRecTy,
+                          /*typeParams=*/mlir::ValueRange{});
+                  coordinates.push_back(childFieldIndex);
+                  break;
+                }
+              }
+            }
+          }
+
+          if (coordinates.empty())
+            TODO(loc, "device resident component in complex derived-type "
+                      "hierarchy");
+
+          mlir::Value comp = builder.create<fir::CoordinateOp>(
+              loc, builder.getRefType(fieldTy), fir::getBase(exv), coordinates);
+          cuf::DataAttributeAttr dataAttr =
+              Fortran::lower::translateSymbolCUFDataAttribute(
+                  builder.getContext(), sym);
+          builder.create<cuf::SetAllocatorIndexOp>(loc, comp, dataAttr);
+        }
+      }
+    }
+  }
+}
+
 /// Must \p var be default initialized at runtime when entering its scope.
 static bool
 mustBeDefaultInitializedAtRuntime(const Fortran::lower::pft::Variable &var) {
@@ -1179,6 +1194,9 @@ static void instantiateLocal(Fortran::lower::AbstractConverter &converter,
   if (mustBeDefaultInitializedAtRuntime(var))
     Fortran::lower::defaultInitializeAtRuntime(converter, var.getSymbol(),
                                                symMap);
+  if (converter.getFoldingContext().languageFeatures().IsEnabled(
+          Fortran::common::LanguageFeature::CUDA))
+    initializeDeviceComponentAllocator(converter, var.getSymbol(), symMap);
   auto *builder = &converter.getFirOpBuilder();
   if (needCUDAAlloc(var.getSymbol()) &&
       !cuf::isCUDADeviceContext(builder->getRegion())) {
@@ -1437,6 +1455,9 @@ static void instantiateAlias(Fortran::lower::AbstractConverter &converter,
   if (mustBeDefaultInitializedAtRuntime(var))
     Fortran::lower::defaultInitializeAtRuntime(converter, var.getSymbol(),
                                                symMap);
+  if (converter.getFoldingContext().languageFeatures().IsEnabled(
+          Fortran::common::LanguageFeature::CUDA))
+    initializeDeviceComponentAllocator(converter, var.getSymbol(), symMap);
 }
 
 //===--------------------------------------------------------------===//
diff --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index bf74e012a639d..85715edf26942 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -12,10 +12,18 @@ contains
   end subroutine
 
 ! CHECK-LABEL: func.func @_QMm1Psub1()
-! CHECK: %[[DT:.*]] = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} -> !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
-! CHECK: %[[X:.*]] = fir.coordinate_of %[[DT]], x : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} -> !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DT:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>)
+! CHECK: fir.address_of(@_QQ_QMm1Tty_device.DerivedInit)
+! CHECK: fir.copy 
+! CHECK: %[[X:.*]] = fir.coordinate_of %[[DT]]#0, x : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[X]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
-! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]], z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]]#0, z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[Z]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
 
 end module
+
+
+
+    
+  

``````````

</details>


https://github.com/llvm/llvm-project/pull/148936

From flang-commits at lists.llvm.org  Tue Jul 15 11:50:38 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 11:50:38 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
Message-ID: <llvm.org/llvm/llvm-project/pull/148927@github.com>


https://github.com/klausler created https://github.com/llvm/llvm-project/pull/148927

Defined I/O subroutines have UNIT= and IOSTAT= dummy arguments that are required to have type INTEGER with its default kind.  When that default kind is modified via -fdefault-integer-8, calls to defined I/O subroutines from the runtime don't work.

Add a flag to the two data structures shared between the compiler and the runtime support library to indicate that a defined I/O subroutine was compiled under -fdefault-integer-8.  This has been done in a compatible manner, so that existing binaries are compatible with the new library and new binaries are compatible with the old library, unless of course -fdefault-integer-8 is used.

Fixes https://github.com/llvm/llvm-project/issues/148638.

>From 898549bf01f94eaeb8ec118d9074d02c46d696f7 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 15 Jul 2025 11:28:30 -0700
Subject: [PATCH] [flang] Allow -fdefault-integer-8 with defined I/O

Defined I/O subroutines have UNIT= and IOSTAT= dummy arguments that
are required to have type INTEGER with its default kind.  When that
default kind is modified via -fdefault-integer-8, calls to defined
I/O subroutines from the runtime don't work.

Add a flag to the two data structures shared between the compiler and
the runtime support library to indicate that a defined I/O subroutine
was compiled under -fdefault-integer-8.  This has been done in a
compatible manner, so that existing binaries are compatible with the
new library and new binaries are compatible with the old library,
unless of course -fdefault-integer-8 is used.

Fixes https://github.com/llvm/llvm-project/issues/148638.
---
 .../include/flang-rt/runtime/non-tbp-dio.h    |  7 ++-
 flang-rt/include/flang-rt/runtime/type-info.h | 16 +++--
 flang-rt/lib/runtime/derived.cpp              |  2 +-
 flang-rt/lib/runtime/descriptor-io.cpp        | 63 ++++++++++++++-----
 flang-rt/lib/runtime/non-tbp-dio.cpp          |  2 +-
 flang-rt/lib/runtime/type-info.cpp            |  2 +-
 .../flang/Semantics/runtime-type-info.h       |  7 ++-
 flang/lib/Lower/IO.cpp                        |  8 ++-
 flang/lib/Semantics/runtime-type-info.cpp     | 28 ++++++---
 flang/module/__fortran_type_info.f90          |  2 +-
 flang/test/Lower/io-derived-type.f90          | 58 ++++++++---------
 flang/test/Lower/namelist.f90                 |  8 +--
 flang/test/Lower/volatile-openmp.f90          |  8 +--
 flang/test/Semantics/typeinfo01.f90           | 18 +++---
 flang/test/Semantics/typeinfo02.f90           |  4 +-
 flang/test/Semantics/typeinfo09.f90           |  2 +-
 flang/test/Semantics/typeinfo13.f90           |  2 +-
 17 files changed, 144 insertions(+), 93 deletions(-)

diff --git a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
index 99d4113b6c7a8..26849298ec959 100644
--- a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
+++ b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
@@ -34,11 +34,16 @@ namespace Fortran::runtime::io {
 
 RT_OFFLOAD_API_GROUP_BEGIN
 
+enum NonTbpDefinedIoFlags {
+  IsDtvArgPolymorphic = 1 << 0, // first dummy arg is CLASS(T)
+  DefinedIoInteger8 = 1 << 1, // -fdefault-integer-8 affected UNIT= & IOSTAT=
+};
+
 struct NonTbpDefinedIo {
   const typeInfo::DerivedType &derivedType;
   void (*subroutine)(); // null means no non-TBP defined I/O here
   common::DefinedIo definedIo;
-  bool isDtvArgPolymorphic; // first dummy arg is CLASS(T)
+  std::uint8_t flags;
 };
 
 struct NonTbpDefinedIoTable {
diff --git a/flang-rt/include/flang-rt/runtime/type-info.h b/flang-rt/include/flang-rt/runtime/type-info.h
index a8d39f4f8a1a3..93bca24a602b4 100644
--- a/flang-rt/include/flang-rt/runtime/type-info.h
+++ b/flang-rt/include/flang-rt/runtime/type-info.h
@@ -143,9 +143,9 @@ class SpecialBinding {
   // I/O procedures that are not type-bound.
   RT_API_ATTRS SpecialBinding(Which which, ProcedurePointer proc,
       std::uint8_t isArgDescSet, std::uint8_t isTypeBound,
-      std::uint8_t isArgContiguousSet)
+      std::uint8_t specialCaseFlag)
       : which_{which}, isArgDescriptorSet_{isArgDescSet},
-        isTypeBound_{isTypeBound}, isArgContiguousSet_{isArgContiguousSet},
+        isTypeBound_{isTypeBound}, specialCaseFlag_{specialCaseFlag},
         proc_{proc} {}
 
   static constexpr RT_API_ATTRS Which RankFinal(int rank) {
@@ -153,13 +153,11 @@ class SpecialBinding {
   }
 
   RT_API_ATTRS Which which() const { return which_; }
+  RT_API_ATTRS bool specialCaseFlag() const { return specialCaseFlag_; }
   RT_API_ATTRS bool IsArgDescriptor(int zeroBasedArg) const {
     return (isArgDescriptorSet_ >> zeroBasedArg) & 1;
   }
   RT_API_ATTRS bool IsTypeBound() const { return isTypeBound_ != 0; }
-  RT_API_ATTRS bool IsArgContiguous(int zeroBasedArg) const {
-    return (isArgContiguousSet_ >> zeroBasedArg) & 1;
-  }
   template <typename PROC>
   RT_API_ATTRS PROC GetProc(const Binding *bindings = nullptr) const {
     if (bindings && isTypeBound_ > 0) {
@@ -203,10 +201,10 @@ class SpecialBinding {
   // When a special binding is type-bound, this is its binding's index (plus 1,
   // so that 0 signifies that it's not type-bound).
   std::uint8_t isTypeBound_{0};
-  // True when a FINAL subroutine has a dummy argument that is an array that
-  // is CONTIGUOUS or neither assumed-rank nor assumed-shape.
-  std::uint8_t isArgContiguousSet_{0};
-
+  // For a FINAL subroutine, set when it has a dummy argument that is an array
+  // that is CONTIGUOUS or neither assumed-rank nor assumed-shape.
+  // For a defined I/O subroutine, set when UNIT= and IOSTAT= are INTEGER(8).
+  std::uint8_t specialCaseFlag_{0};
   ProcedurePointer proc_{nullptr};
 };
 
diff --git a/flang-rt/lib/runtime/derived.cpp b/flang-rt/lib/runtime/derived.cpp
index bb9a68abef2a7..4ed0baaa3d108 100644
--- a/flang-rt/lib/runtime/derived.cpp
+++ b/flang-rt/lib/runtime/derived.cpp
@@ -270,7 +270,7 @@ static RT_API_ATTRS void CallFinalSubroutine(const Descriptor &descriptor,
       StaticDescriptor<maxRank, true, 10> statDesc;
       Descriptor &copy{statDesc.descriptor()};
       const Descriptor *argDescriptor{&descriptor};
-      if (descriptor.rank() > 0 && special->IsArgContiguous(0) &&
+      if (descriptor.rank() > 0 && special->specialCaseFlag() &&
           !descriptor.IsContiguous()) {
         // The FINAL subroutine demands a contiguous array argument, but
         // this INTENT(OUT) or intrinsic assignment LHS isn't contiguous.
diff --git a/flang-rt/lib/runtime/descriptor-io.cpp b/flang-rt/lib/runtime/descriptor-io.cpp
index b208cb2c397b3..50fb24c14c95d 100644
--- a/flang-rt/lib/runtime/descriptor-io.cpp
+++ b/flang-rt/lib/runtime/descriptor-io.cpp
@@ -84,8 +84,8 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
     ChildIo &child{external->PushChildIo(io)};
     // Child formatted I/O is nonadvancing by definition (F'2018 12.6.2.4).
     auto restorer{common::ScopedSet(io.mutableModes().nonAdvancing, true)};
-    int unit{external->unitNumber()};
-    int ioStat{IostatOk};
+    std::int32_t unit{external->unitNumber()};
+    std::int32_t ioStat{IostatOk};
     char ioMsg[100];
     Fortran::common::optional<std::int64_t> startPos;
     if (edit.descriptor == DataEdit::DefinedDerivedType &&
@@ -96,25 +96,48 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
     }
     const auto *bindings{
         derived.binding().OffsetElement<const typeInfo::Binding>()};
+    bool integer8{special.specialCaseFlag()};
     if (special.IsArgDescriptor(0)) {
       // "dtv" argument is "class(t)", pass a descriptor
-      auto *p{special.GetProc<void (*)(const Descriptor &, int &, char *,
-          const Descriptor &, int &, char *, std::size_t, std::size_t)>(
-          bindings)};
       StaticDescriptor<1, true, 10 /*?*/> elementStatDesc;
       Descriptor &elementDesc{elementStatDesc.descriptor()};
       elementDesc.Establish(
           derived, nullptr, 0, nullptr, CFI_attribute_pointer);
       elementDesc.set_base_addr(descriptor.Element<char>(subscripts));
-      p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen,
-          sizeof ioMsg);
+      if (integer8) { // 64-bit UNIT=/IOSTAT=
+        std::int64_t unit64{unit};
+        std::int64_t ioStat64{ioStat};
+        auto *p{special.GetProc<void (*)(const Descriptor &, std::int64_t &,
+            char *, const Descriptor &, std::int64_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(elementDesc, unit64, ioType, vListDesc, ioStat64, ioMsg, ioTypeLen,
+            sizeof ioMsg);
+        ioStat = ioStat64;
+      } else { // 32-bit UNIT=/IOSTAT=
+        auto *p{special.GetProc<void (*)(const Descriptor &, std::int32_t &,
+            char *, const Descriptor &, std::int32_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen,
+            sizeof ioMsg);
+      }
     } else {
       // "dtv" argument is "type(t)", pass a raw pointer
-      auto *p{special.GetProc<void (*)(const void *, int &, char *,
-          const Descriptor &, int &, char *, std::size_t, std::size_t)>(
-          bindings)};
-      p(descriptor.Element<char>(subscripts), unit, ioType, vListDesc, ioStat,
-          ioMsg, ioTypeLen, sizeof ioMsg);
+      if (integer8) { // 64-bit UNIT= and IOSTAT=
+        std::int64_t unit64{unit};
+        std::int64_t ioStat64{ioStat};
+        auto *p{special.GetProc<void (*)(const void *, std::int64_t &, char *,
+            const Descriptor &, std::int64_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(descriptor.Element<char>(subscripts), unit64, ioType, vListDesc,
+            ioStat64, ioMsg, ioTypeLen, sizeof ioMsg);
+        ioStat = ioStat64;
+      } else { // 32-bit UNIT= and IOSTAT=
+        auto *p{special.GetProc<void (*)(const void *, std::int32_t &, char *,
+            const Descriptor &, std::int32_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(descriptor.Element<char>(subscripts), unit, ioType, vListDesc, ioStat,
+            ioMsg, ioTypeLen, sizeof ioMsg);
+      }
     }
     handler.Forward(ioStat, ioMsg, sizeof ioMsg);
     external->PopChildIo(child);
@@ -458,11 +481,16 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
                       ? common::DefinedIo::ReadUnformatted
                       : common::DefinedIo::WriteUnformatted)}) {
             if (definedIo->subroutine) {
+              std::uint8_t isArgDescriptorSet{0};
+              if (definedIo->flags & IsDtvArgPolymorphic) {
+                isArgDescriptorSet = 1;
+              }
               typeInfo::SpecialBinding special{DIR == Direction::Input
                       ? typeInfo::SpecialBinding::Which::ReadUnformatted
                       : typeInfo::SpecialBinding::Which::WriteUnformatted,
-                  definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                  false};
+                  definedIo->subroutine, isArgDescriptorSet,
+                  /*IsTypeBound=*/false,
+                  /*specialCaseFlag=*/!!(definedIo->flags & DefinedIoInteger8)};
               if (DefinedUnformattedIo(io_, instance_, *type, special)) {
                 anyIoTookPlace_ = true;
                 return StatOk;
@@ -719,8 +747,11 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
             nonTbpSpecial_.emplace(DIR == Direction::Input
                     ? typeInfo::SpecialBinding::Which::ReadFormatted
                     : typeInfo::SpecialBinding::Which::WriteFormatted,
-                definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                false);
+                definedIo->subroutine,
+                /*isArgDescriptorSet=*/
+                (definedIo->flags & IsDtvArgPolymorphic) ? 1 : 0,
+                /*isTypeBound=*/false,
+                /*specialCaseFlag=*/!!(definedIo->flags & DefinedIoInteger8));
             special_ = &*nonTbpSpecial_;
           }
         }
diff --git a/flang-rt/lib/runtime/non-tbp-dio.cpp b/flang-rt/lib/runtime/non-tbp-dio.cpp
index 72101b06e0c6e..d516526033c27 100644
--- a/flang-rt/lib/runtime/non-tbp-dio.cpp
+++ b/flang-rt/lib/runtime/non-tbp-dio.cpp
@@ -17,7 +17,7 @@ const NonTbpDefinedIo *NonTbpDefinedIoTable::Find(
   for (const auto *p{item}; j-- > 0; ++p) {
     if (&p->derivedType == &type && p->definedIo == definedIo) {
       return p;
-    } else if (p->isDtvArgPolymorphic) {
+    } else if (p->flags & IsDtvArgPolymorphic) {
       for (const typeInfo::DerivedType *t{type.GetParentType()}; t;
            t = t->GetParentType()) {
         if (&p->derivedType == t && p->definedIo == definedIo) {
diff --git a/flang-rt/lib/runtime/type-info.cpp b/flang-rt/lib/runtime/type-info.cpp
index 3e1d7c9c3c788..50123f4cf321c 100644
--- a/flang-rt/lib/runtime/type-info.cpp
+++ b/flang-rt/lib/runtime/type-info.cpp
@@ -330,7 +330,7 @@ FILE *SpecialBinding::Dump(FILE *f) const {
   }
   std::fprintf(f, "    isArgDescriptorSet: 0x%x\n", isArgDescriptorSet_);
   std::fprintf(f, "    isTypeBound: %d\n", isTypeBound_);
-  std::fprintf(f, "    isArgContiguousSet: 0x%x\n", isArgContiguousSet_);
+  std::fprintf(f, "    specialCaseFlag 0x%x\n", specialCaseFlag_);
   std::fprintf(f, "    proc: %p\n", reinterpret_cast<void *>(proc_));
   return f;
 }
diff --git a/flang/include/flang/Semantics/runtime-type-info.h b/flang/include/flang/Semantics/runtime-type-info.h
index 6c5a061d1c1a2..94e8293b14643 100644
--- a/flang/include/flang/Semantics/runtime-type-info.h
+++ b/flang/include/flang/Semantics/runtime-type-info.h
@@ -52,10 +52,15 @@ constexpr char procCompName[]{"proc"};
 
 SymbolVector CollectBindings(const Scope &dtScope);
 
+enum NonTbpDefinedIoFlags {
+  IsDtvArgPolymorphic = 1 << 0,
+  DefinedIoInteger8 = 1 << 1,
+};
+
 struct NonTbpDefinedIo {
   const Symbol *subroutine;
   common::DefinedIo definedIo;
-  bool isDtvArgPolymorphic;
+  std::uint8_t flags;
 };
 
 std::multimap<const Symbol *, NonTbpDefinedIo>
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 63a612d7ead61..69d72d9d63b68 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -269,10 +269,12 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   mlir::Type sizeTy =
       fir::runtime::getModel<std::size_t>()(builder.getContext());
   mlir::Type intTy = fir::runtime::getModel<int>()(builder.getContext());
+  mlir::Type byteTy =
+      fir::runtime::getModel<std::uint8_t>()(builder.getContext());
   mlir::Type boolTy = fir::runtime::getModel<bool>()(builder.getContext());
   mlir::Type listTy = fir::SequenceType::get(
       definedIoProcMap.size(),
-      mlir::TupleType::get(context, {refTy, refTy, intTy, boolTy}));
+      mlir::TupleType::get(context, {refTy, refTy, intTy, byteTy}));
   mlir::Type tableTy = mlir::TupleType::get(
       context, {sizeTy, fir::ReferenceType::get(listTy), boolTy});
 
@@ -339,9 +341,9 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       insert(builder.createIntegerConstant(
           loc, intTy, static_cast<int>(iface.second.definedIo)));
       // polymorphic flag is set if first defined IO dummy arg is CLASS(T)
+      // defaultInt8 flag is set if -fdefined-integer-8
       // [bool isDtvArgPolymorphic]
-      insert(builder.createIntegerConstant(loc, boolTy,
-                                           iface.second.isDtvArgPolymorphic));
+      insert(builder.createIntegerConstant(loc, byteTy, iface.second.flags));
     }
     if (tableIsLocal)
       builder.create<fir::StoreOp>(loc, list, listAddr);
diff --git a/flang/lib/Semantics/runtime-type-info.cpp b/flang/lib/Semantics/runtime-type-info.cpp
index 51ba21a9e5edf..5916a07df7744 100644
--- a/flang/lib/Semantics/runtime-type-info.cpp
+++ b/flang/lib/Semantics/runtime-type-info.cpp
@@ -1131,7 +1131,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
   if (auto proc{evaluate::characteristics::Procedure::Characterize(
           specific, context_.foldingContext())}) {
     std::uint8_t isArgDescriptorSet{0};
-    std::uint8_t isArgContiguousSet{0};
+    bool specialCaseFlag{0};
     int argThatMightBeDescriptor{0};
     MaybeExpr which;
     if (isAssignment) {
@@ -1197,7 +1197,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
                         TypeAndShape::Attr::AssumedShape) ||
                 dummyData.attrs.test(evaluate::characteristics::
                         DummyDataObject::Attr::Contiguous)) {
-              isArgContiguousSet |= 1;
+              specialCaseFlag = true;
             }
           }
         }
@@ -1216,7 +1216,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
         return;
       }
       if (ddo->type.type().IsPolymorphic()) {
-        isArgDescriptorSet |= 1;
+        argThatMightBeDescriptor = 1;
       }
       switch (io.value()) {
       case common::DefinedIo::ReadFormatted:
@@ -1232,6 +1232,9 @@ void RuntimeTableBuilder::DescribeSpecialProc(
         which = writeUnformattedEnum_;
         break;
       }
+      if (context_.defaultKinds().GetDefaultKind(TypeCategory::Integer) == 8) {
+        specialCaseFlag = true; // UNIT= & IOSTAT= INTEGER(8)
+      }
     }
     if (argThatMightBeDescriptor != 0) {
       if (const auto *dummyData{
@@ -1262,8 +1265,8 @@ void RuntimeTableBuilder::DescribeSpecialProc(
     }
     CHECK(bindingIndex <= 255);
     AddValue(values, specialSchema_, "istypebound"s, IntExpr<1>(bindingIndex));
-    AddValue(values, specialSchema_, "isargcontiguousset"s,
-        IntExpr<1>(isArgContiguousSet));
+    AddValue(values, specialSchema_, "specialcaseflag"s,
+        IntExpr<1>(specialCaseFlag));
     AddValue(values, specialSchema_, procCompName,
         SomeExpr{evaluate::ProcedureDesignator{specific}});
     // index might already be present in the case of an override
@@ -1383,19 +1386,26 @@ CollectNonTbpDefinedIoGenericInterfaces(
               } else {
                 // Local scope's specific overrides host's for this type
                 bool updated{false};
+                std::uint8_t flags{0};
+                if (declType->IsPolymorphic()) {
+                  flags |= IsDtvArgPolymorphic;
+                }
+                if (scope.context().GetDefaultKind(TypeCategory::Integer) ==
+                    8) {
+                  flags |= DefinedIoInteger8;
+                }
                 for (auto [iter, end]{result.equal_range(dtDesc)}; iter != end;
                      ++iter) {
                   NonTbpDefinedIo &nonTbp{iter->second};
                   if (nonTbp.definedIo == which) {
                     nonTbp.subroutine = &*specific;
-                    nonTbp.isDtvArgPolymorphic = declType->IsPolymorphic();
+                    nonTbp.flags = flags;
                     updated = true;
                   }
                 }
                 if (!updated) {
-                  result.emplace(dtDesc,
-                      NonTbpDefinedIo{
-                          &*specific, which, declType->IsPolymorphic()});
+                  result.emplace(
+                      dtDesc, NonTbpDefinedIo{&*specific, which, flags});
                 }
               }
             }
diff --git a/flang/module/__fortran_type_info.f90 b/flang/module/__fortran_type_info.f90
index 8dd27d6e4c01b..6af2a5a5e30ff 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang/module/__fortran_type_info.f90
@@ -118,7 +118,7 @@
     integer(1) :: which ! SpecialBinding::Which
     integer(1) :: isArgDescriptorSet
     integer(1) :: isTypeBound ! binding index + 1, if any
-    integer(1) :: isArgContiguousSet
+    integer(1) :: specialCaseFlag
     integer(1) :: __padding0(4)
     type(__builtin_c_funptr) :: proc
   end type
diff --git a/flang/test/Lower/io-derived-type.f90 b/flang/test/Lower/io-derived-type.f90
index 7d2fef3faa2b7..7c289ce261678 100644
--- a/flang/test/Lower/io-derived-type.f90
+++ b/flang/test/Lower/io-derived-type.f90
@@ -37,16 +37,16 @@ subroutine test1
     import, all
     ! CHECK:   %[[V_16:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:   %[[V_17:[0-9]+]] = fir.convert %[[V_16]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:   %[[V_18:[0-9]+]] = fir.address_of(@_QQMmFtest1.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:   %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:   %[[V_18:[0-9]+]] = fir.address_of(@_QQMmFtest1.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:   %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:   %[[V_20:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_17]], %[[V_19]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
     print *, 'test1 outer, should call wft: ', t(1)
     block
       import, only: t
       ! CHECK:   %[[V_37:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
       ! CHECK:   %[[V_38:[0-9]+]] = fir.convert %[[V_37]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-      ! CHECK:   %[[V_39:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-      ! CHECK:   %[[V_40:[0-9]+]] = fir.convert %[[V_39]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+      ! CHECK:   %[[V_39:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+      ! CHECK:   %[[V_40:[0-9]+]] = fir.convert %[[V_39]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
       ! CHECK:   %[[V_41:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_38]], %[[V_40]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
       print *, 'test1 block, should not call wft: ', t(2)
     end block
@@ -56,8 +56,8 @@ subroutine test1
   subroutine test2
     ! CHECK:   %[[V_15:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:   %[[V_16:[0-9]+]] = fir.convert %[[V_15]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:   %[[V_17:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:   %[[V_18:[0-9]+]] = fir.convert %[[V_17]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:   %[[V_17:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:   %[[V_18:[0-9]+]] = fir.convert %[[V_17]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:   %[[V_19:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_16]], %[[V_18]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
 
     import, only: t
@@ -74,23 +74,23 @@ subroutine test3(p, x)
 
     ! CHECK:     %[[V_3:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:     %[[V_4:[0-9]+]] = fir.convert %[[V_3]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:     %[[V_5:[0-9]+]] = fir.alloca !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_6:[0-9]+]] = fir.undefined !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
+    ! CHECK:     %[[V_5:[0-9]+]] = fir.alloca !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_6:[0-9]+]] = fir.undefined !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
     ! CHECK:     %[[V_7:[0-9]+]] = fir.address_of(@_QMmE.dt.t)
     ! CHECK:     %[[V_8:[0-9]+]] = fir.convert %[[V_7]] : {{.*}} -> !fir.ref<none>
-    ! CHECK:     %[[V_9:[0-9]+]] = fir.insert_value %[[V_6]], %[[V_8]], [0 : index, 0 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
+    ! CHECK:     %[[V_9:[0-9]+]] = fir.insert_value %[[V_6]], %[[V_8]], [0 : index, 0 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
     ! CHECK:     %[[V_10:[0-9]+]] = fir.box_addr %arg0 : (!fir.boxproc<() -> ()>) -> !fir.ref<none>
-    ! CHECK:     %[[V_11:[0-9]+]] = fir.insert_value %[[V_9]], %[[V_10]], [0 : index, 1 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_12:[0-9]+]] = fir.insert_value %[[V_11]], %c2{{.*}}, [0 : index, 2 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, i32) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_13:[0-9]+]] = fir.insert_value %[[V_12]], %true, [0 : index, 3 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, i1) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     fir.store %[[V_13]] to %[[V_5]] : !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>
-    ! CHECK:     %[[V_14:[0-9]+]] = fir.alloca tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_15:[0-9]+]] = fir.undefined tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_16:[0-9]+]] = fir.insert_value %[[V_15]], %c1{{.*}}, [0 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, i64) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_17:[0-9]+]] = fir.insert_value %[[V_16]], %[[V_5]], [1 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_18:[0-9]+]] = fir.insert_value %[[V_17]], %true_0, [2 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, i1) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     fir.store %[[V_18]] to %[[V_14]] : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:     %[[V_19:[0-9]+]] = fir.convert %[[V_14]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:     %[[V_11:[0-9]+]] = fir.insert_value %[[V_9]], %[[V_10]], [0 : index, 1 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_12:[0-9]+]] = fir.insert_value %[[V_11]], %c2{{.*}}, [0 : index, 2 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, i32) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_13:[0-9]+]] = fir.insert_value %[[V_12]], %c1_i8, [0 : index, 3 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, i8) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     fir.store %[[V_13]] to %[[V_5]] : !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>
+    ! CHECK:     %[[V_14:[0-9]+]] = fir.alloca tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_15:[0-9]+]] = fir.undefined tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_16:[0-9]+]] = fir.insert_value %[[V_15]], %c1{{.*}}, [0 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, i64) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_17:[0-9]+]] = fir.insert_value %[[V_16]], %[[V_5]], [1 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_18:[0-9]+]] = fir.insert_value %[[V_17]], %true, [2 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, i1) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     fir.store %[[V_18]] to %[[V_14]] : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:     %[[V_19:[0-9]+]] = fir.convert %[[V_14]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:     %[[V_20:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_4]], %[[V_19]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
     print *, x
   end subroutine
@@ -112,8 +112,8 @@ program p
 
   ! CHECK:   %[[V_97:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
   ! CHECK:   %[[V_98:[0-9]+]] = fir.convert %[[V_97]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-  ! CHECK:   %[[V_99:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:   %[[V_100:[0-9]+]] = fir.convert %[[V_99]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:   %[[V_99:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:   %[[V_100:[0-9]+]] = fir.convert %[[V_99]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:   %[[V_101:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_98]], %[[V_100]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
   print *, 'main, should call wft: ', t(4)
 
@@ -122,14 +122,14 @@ program p
   ! CHECK:   %[[V_35:[0-9]+]] = fir.shape %c2{{.*}} : (index) -> !fir.shape<1>
   ! CHECK:   %[[V_36:[0-9]+]] = fir.embox %[[V_34]](%[[V_35]]) : (!fir.ref<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>
   ! CHECK:   %[[V_37:[0-9]+]] = fir.convert %[[V_36]] : (!fir.box<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>) -> !fir.box<none>
-  ! CHECK:   %[[V_38:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:   %[[V_39:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:   %[[V_38:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:   %[[V_39:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:   %[[V_40:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_37]], %[[V_39]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
   print *, y(2:3)
 end
 
-! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-! CHECK: fir.global linkonce @_QQdefault.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
+! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+! CHECK: fir.global linkonce @_QQdefault.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
diff --git a/flang/test/Lower/namelist.f90 b/flang/test/Lower/namelist.f90
index 94b0ef11cb102..770af46eea744 100644
--- a/flang/test/Lower/namelist.f90
+++ b/flang/test/Lower/namelist.f90
@@ -42,8 +42,8 @@ program p
   ! CHECK:     %[[V_42:[0-9]+]] = fir.insert_value %[[V_39]], %[[V_41]], [0 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<i8>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_43:[0-9]+]] = fir.insert_value %[[V_42]], %c2{{.*}}, [1 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, i64) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_44:[0-9]+]] = fir.insert_value %[[V_43]], %[[V_24]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
-  ! CHECK:     %[[V_45:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:     %[[V_46:[0-9]+]] = fir.convert %[[V_45]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:     %[[V_45:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:     %[[V_46:[0-9]+]] = fir.convert %[[V_45]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:     %[[V_47:[0-9]+]] = fir.insert_value %[[V_44]], %[[V_46]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     fir.store %[[V_47]] to %[[V_38]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
   ! CHECK:     %[[V_48:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
@@ -100,8 +100,8 @@ subroutine sss
   ! CHECK:     %[[V_20:[0-9]+]] = fir.insert_value %[[V_17]], %[[V_19]], [0 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<i8>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_21:[0-9]+]] = fir.insert_value %[[V_20]], %c1{{.*}}, [1 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, i64) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_22:[0-9]+]] = fir.insert_value %[[V_21]], %[[V_8]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
-  ! CHECK:     %[[V_23:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:     %[[V_24:[0-9]+]] = fir.convert %[[V_23]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:     %[[V_23:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:     %[[V_24:[0-9]+]] = fir.convert %[[V_23]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:     %[[V_25:[0-9]+]] = fir.insert_value %[[V_22]], %[[V_24]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     fir.store %[[V_25]] to %[[V_16]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
   ! CHECK:     %[[V_26:[0-9]+]] = fir.convert %[[V_16]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
diff --git a/flang/test/Lower/volatile-openmp.f90 b/flang/test/Lower/volatile-openmp.f90
index 2e05b652822b5..d1a844eddd106 100644
--- a/flang/test/Lower/volatile-openmp.f90
+++ b/flang/test/Lower/volatile-openmp.f90
@@ -23,11 +23,11 @@
 ! CHECK:           %[[VAL_11:.*]] = fir.address_of(@_QFEcontainer) : !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>
 ! CHECK:           %[[VAL_12:.*]] = fir.volatile_cast %[[VAL_11]] : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>
 ! CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFEcontainer"} : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>) -> (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>, !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>)
-! CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QFE.c.t) : !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>
+! CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QFE.c.t) : !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>
 ! CHECK:           %[[VAL_15:.*]] = fir.shape_shift %[[VAL_0]], %[[VAL_1]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]](%[[VAL_15]]) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.c.t"} : (!fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>)
-! CHECK:           %[[VAL_17:.*]] = fir.address_of(@_QFE.dt.t) : !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>
-! CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.dt.t"} : (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>) -> (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>, !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>)
+! CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]](%[[VAL_15]]) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.c.t"} : (!fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>)
+! CHECK:           %[[VAL_17:.*]] = fir.address_of(@_QFE.dt.t) : !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>
+! CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.dt.t"} : (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>) -> (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>, !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>)
 ! CHECK:           %[[VAL_19:.*]] = hlfir.designate %[[VAL_13]]#0{"array"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>, volatile>
 ! CHECK:           %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>, volatile>
 ! CHECK:           %[[VAL_21:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_0]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
diff --git a/flang/test/Semantics/typeinfo01.f90 b/flang/test/Semantics/typeinfo01.f90
index bb20c546e0261..aeec336ea58ea 100644
--- a/flang/test/Semantics/typeinfo01.f90
+++ b/flang/test/Semantics/typeinfo01.f90
@@ -87,8 +87,8 @@ subroutine s2(x, y)
 !CHECK: .c.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=6_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=2_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
 !CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=.s.t2,specialbitset=2_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
-!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s2)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
+!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s2)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 !CHECK: .v.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
 end module
@@ -115,8 +115,8 @@ subroutine s2(x, y)
 !CHECK: .c.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=6_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=2_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
 !CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=.s.t2,specialbitset=2_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
-!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s2)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
+!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s2)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 !CHECK: .v.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
 end module
@@ -133,7 +133,7 @@ impure elemental subroutine s1(x, y)
     class(t), intent(in) :: y
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=4_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 end module
 
@@ -156,7 +156,7 @@ subroutine s4(x)
     type(t), contiguous :: x(:,:,:)
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=7296_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=0_1,nofinalizationneeded=0_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=7_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=s3),specialbinding(which=10_1,isargdescriptorset=1_1,istypebound=0_1,isargcontiguousset=0_1,proc=s1),specialbinding(which=11_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=1_1,proc=s2),specialbinding(which=12_1,isargdescriptorset=1_1,istypebound=0_1,isargcontiguousset=1_1,proc=s4)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=7_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=s3),specialbinding(which=10_1,isargdescriptorset=1_1,istypebound=0_1,specialcaseflag=0_1,proc=s1),specialbinding(which=11_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=1_1,proc=s2),specialbinding(which=12_1,isargdescriptorset=1_1,istypebound=0_1,specialcaseflag=1_1,proc=s4)]
 end module
 
 module m09
@@ -198,7 +198,7 @@ subroutine wu(x,u,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=120_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=1_1,istypebound=2_1,isargcontiguousset=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=3_1,isargcontiguousset=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=1_1,istypebound=4_1,isargcontiguousset=0_1,proc=wu)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=1_1,istypebound=2_1,specialcaseflag=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=3_1,specialcaseflag=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=1_1,istypebound=4_1,specialcaseflag=0_1,proc=wu)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:3_8 init:[binding::binding(proc=rf,name=.n.rf),binding(proc=ru,name=.n.ru),binding(proc=wf,name=.n.wf),binding(proc=wu,name=.n.wu)]
 end module
 
@@ -247,7 +247,7 @@ subroutine wu(x,u,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=120_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=wu)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=wu)]
 end module
 
 module m11
@@ -290,7 +290,7 @@ module m13
    contains
     procedure :: assign1, assign2
     generic :: assignment(=) => assign1, assign2
-    ! CHECK: .s.t1, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=assign1)]
+    ! CHECK: .s.t1, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=assign1)]
   end type
  contains
   impure elemental subroutine assign1(to, from)
diff --git a/flang/test/Semantics/typeinfo02.f90 b/flang/test/Semantics/typeinfo02.f90
index 29d14c7a0f196..07293627ab492 100644
--- a/flang/test/Semantics/typeinfo02.f90
+++ b/flang/test/Semantics/typeinfo02.f90
@@ -29,5 +29,5 @@ subroutine wf2(x,u,iot,v,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 end module
-!CHECK: .s.base, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=wf1)]
-!CHECK: .s.extended, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=wf2)]
+!CHECK: .s.base, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=wf1)]
+!CHECK: .s.extended, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=wf2)]
diff --git a/flang/test/Semantics/typeinfo09.f90 b/flang/test/Semantics/typeinfo09.f90
index 3527ee6058ad8..8daa6a5f420d7 100644
--- a/flang/test/Semantics/typeinfo09.f90
+++ b/flang/test/Semantics/typeinfo09.f90
@@ -17,4 +17,4 @@ subroutine copy_impl(this, x)
 end interface
 end module
 
-!CHECK: .s.sometype, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=copy_impl)]
+!CHECK: .s.sometype, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=copy_impl)]
diff --git a/flang/test/Semantics/typeinfo13.f90 b/flang/test/Semantics/typeinfo13.f90
index ad824ad3590a2..facc280815722 100644
--- a/flang/test/Semantics/typeinfo13.f90
+++ b/flang/test/Semantics/typeinfo13.f90
@@ -22,5 +22,5 @@ impure elemental subroutine override(to, from)
   end
 end
 
-!CHECK: .s.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=1_1,istypebound=2_1,isargcontiguousset=0_1,proc=override)]
+!CHECK: .s.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=1_1,istypebound=2_1,specialcaseflag=0_1,proc=override)]
 !CHECK: .v.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:1_8 init:[binding::binding(proc=baseassign,name=.n.baseassign),binding(proc=override,name=.n.override)]


From flang-commits at lists.llvm.org  Tue Jul 15 13:46:38 2025
From: flang-commits at lists.llvm.org (Eli Friedman via flang-commits)
Date: Tue, 15 Jul 2025 13:46:38 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [Clang] Remove ARCMigrate
	(PR #119269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/119269@github.com>
Message-ID: <6876be2e.170a0220.a88da.021d@mx.google.com>


efriedma-quic wrote:

I don't know that we've had to deal with the underling functionality for anything in libclang getting removed before; we're usually pretty conservative with what we add in the first place.

Maybe we should stub out the methods so they always fail?

https://github.com/llvm/llvm-project/pull/119269

From flang-commits at lists.llvm.org  Tue Jul 15 13:48:52 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 13:48:52 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [Clang] Remove ARCMigrate
	(PR #119269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/119269@github.com>
Message-ID: <6876beb4.170a0220.329d37.58a0@mx.google.com>


Sirraide wrote:

> Maybe we should stub out the methods so they always fail?

As in make them do nothing essentially (i.e. return `NULL` etc.) or as in report a fatal error and exit?

https://github.com/llvm/llvm-project/pull/119269

From flang-commits at lists.llvm.org  Tue Jul 15 13:50:38 2025
From: flang-commits at lists.llvm.org (Eli Friedman via flang-commits)
Date: Tue, 15 Jul 2025 13:50:38 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [Clang] Remove ARCMigrate
	(PR #119269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/119269@github.com>
Message-ID: <6876bf1e.170a0220.2fd26a.660f@mx.google.com>


efriedma-quic wrote:

Return NULL, I guess; that's how the error handling for those methods used to work.

https://github.com/llvm/llvm-project/pull/119269

From flang-commits at lists.llvm.org  Tue Jul 15 13:51:24 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 13:51:24 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [Clang] Remove ARCMigrate
	(PR #119269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/119269@github.com>
Message-ID: <6876bf4c.050a0220.2a59c4.94ea@mx.google.com>


Sirraide wrote:

> Return NULL, I guess; that's how the error handling for those methods used to work.

Yeah, I was just looking at what used to be there, and we’d just print that it isn’t enabled and return `NULL`; I guess we can add them back and print ‘ARCMigrate has been removed from Clang’ or sth like that.

https://github.com/llvm/llvm-project/pull/119269

From flang-commits at lists.llvm.org  Tue Jul 15 13:51:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 13:51:45 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [Clang] Remove ARCMigrate
	(PR #119269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/119269@github.com>
Message-ID: <6876bf61.170a0220.3bc0df.6874@mx.google.com>


Sirraide wrote:

> > Return NULL, I guess; that's how the error handling for those methods used to work.
> 
> Yeah, I was just looking at what used to be there, and we’d just print that it isn’t enabled and return `NULL`; I guess we can add them back and print ‘ARCMigrate has been removed from Clang’ or sth like that.

I’ll open a pr for that

https://github.com/llvm/llvm-project/pull/119269

From flang-commits at lists.llvm.org  Tue Jul 15 13:52:04 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 13:52:04 -0700 (PDT)
Subject: [flang-commits] [flang] b4e2272 - [flang][cuda] Move
 cuf.set_allocator_idx after derived-type init (#148936)
Message-ID: <6876bf74.050a0220.39628.a8f1@mx.google.com>


Author: Valentin Clement (バレンタイン クレメン)
Date: 2025-07-15T13:52:00-07:00
New Revision: b4e2272271ee85273ca871abac2f6e9342da143d

URL: https://github.com/llvm/llvm-project/commit/b4e2272271ee85273ca871abac2f6e9342da143d
DIFF: https://github.com/llvm/llvm-project/commit/b4e2272271ee85273ca871abac2f6e9342da143d.diff

LOG: [flang][cuda] Move cuf.set_allocator_idx after derived-type init (#148936)

Derived type initialization overwrite the component descriptor. Place
the `cuf.set_allocator_idx` after the initialization is performed.

Added: 
    

Modified: 
    flang/lib/Lower/ConvertVariable.cpp
    flang/test/Lower/CUDA/cuda-set-allocator.cuf

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index ffe456de56630..2bfa9618aa4b9 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -771,79 +771,9 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
       return builder.create<cuf::SharedMemoryOp>(loc, ty, nm, symNm, lenParams,
                                                  indices);
 
-    if (!cuf::isCUDADeviceContext(builder.getRegion())) {
-      mlir::Value alloc = builder.create<cuf::AllocOp>(
-          loc, ty, nm, symNm, dataAttr, lenParams, indices);
-      if (const auto *details{
-              ultimateSymbol
-                  .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
-        const Fortran::semantics::DeclTypeSpec *type{details->type()};
-        const Fortran::semantics::DerivedTypeSpec *derived{
-            type ? type->AsDerived() : nullptr};
-        if (derived) {
-          Fortran::semantics::UltimateComponentIterator components{*derived};
-          auto recTy = mlir::dyn_cast<fir::RecordType>(ty);
-
-          llvm::SmallVector<mlir::Value> coordinates;
-          for (const auto &sym : components) {
-            if (Fortran::semantics::IsDeviceAllocatable(sym)) {
-              unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
-              mlir::Type fieldTy;
-              std::vector<mlir::Value> coordinates;
-
-              if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                // Field found in the base record type.
-                auto fieldName = recTy.getTypeList()[fieldIdx].first;
-                fieldTy = recTy.getTypeList()[fieldIdx].second;
-                mlir::Value fieldIndex = builder.create<fir::FieldIndexOp>(
-                    loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
-                    recTy,
-                    /*typeParams=*/mlir::ValueRange{});
-                coordinates.push_back(fieldIndex);
-              } else {
-                // Field not found in base record type, search in potential
-                // record type components.
-                for (auto component : recTy.getTypeList()) {
-                  if (auto childRecTy =
-                          mlir::dyn_cast<fir::RecordType>(component.second)) {
-                    fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
-                    if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                      mlir::Value parentFieldIndex =
-                          builder.create<fir::FieldIndexOp>(
-                              loc, fir::FieldType::get(childRecTy.getContext()),
-                              component.first, recTy,
-                              /*typeParams=*/mlir::ValueRange{});
-                      coordinates.push_back(parentFieldIndex);
-                      auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
-                      fieldTy = childRecTy.getTypeList()[fieldIdx].second;
-                      mlir::Value childFieldIndex =
-                          builder.create<fir::FieldIndexOp>(
-                              loc, fir::FieldType::get(fieldTy.getContext()),
-                              fieldName, childRecTy,
-                              /*typeParams=*/mlir::ValueRange{});
-                      coordinates.push_back(childFieldIndex);
-                      break;
-                    }
-                  }
-                }
-              }
-
-              if (coordinates.empty())
-                TODO(loc, "device resident component in complex derived-type "
-                          "hierarchy");
-
-              mlir::Value comp = builder.create<fir::CoordinateOp>(
-                  loc, builder.getRefType(fieldTy), alloc, coordinates);
-              cuf::DataAttributeAttr dataAttr =
-                  Fortran::lower::translateSymbolCUFDataAttribute(
-                      builder.getContext(), sym);
-              builder.create<cuf::SetAllocatorIndexOp>(loc, comp, dataAttr);
-            }
-          }
-        }
-      }
-      return alloc;
-    }
+    if (!cuf::isCUDADeviceContext(builder.getRegion()))
+      return builder.create<cuf::AllocOp>(loc, ty, nm, symNm, dataAttr,
+                                          lenParams, indices);
   }
 
   // Let the builder do all the heavy lifting.
@@ -857,6 +787,91 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   return res;
 }
 
+/// Device allocatable components in a derived-type don't have the correct
+/// allocator index in their descriptor when they are created. After
+/// initialization, cuf.set_allocator_idx operations are inserted to set the
+/// correct allocator index for each device component.
+static void
+initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
+                                   const Fortran::semantics::Symbol &symbol,
+                                   Fortran::lower::SymMap &symMap) {
+  if (const auto *details{
+          symbol.GetUltimate()
+              .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
+    const Fortran::semantics::DeclTypeSpec *type{details->type()};
+    const Fortran::semantics::DerivedTypeSpec *derived{type ? type->AsDerived()
+                                                            : nullptr};
+    if (derived) {
+      fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+      mlir::Location loc = converter.getCurrentLocation();
+
+      fir::ExtendedValue exv =
+          converter.getSymbolExtendedValue(symbol.GetUltimate(), &symMap);
+      auto recTy = mlir::dyn_cast<fir::RecordType>(
+          fir::unwrapRefType(fir::getBase(exv).getType()));
+      assert(recTy && "expected fir::RecordType");
+
+      llvm::SmallVector<mlir::Value> coordinates;
+      Fortran::semantics::UltimateComponentIterator components{*derived};
+      for (const auto &sym : components) {
+        if (Fortran::semantics::IsDeviceAllocatable(sym)) {
+          unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
+          mlir::Type fieldTy;
+          std::vector<mlir::Value> coordinates;
+
+          if (fieldIdx != std::numeric_limits<unsigned>::max()) {
+            // Field found in the base record type.
+            auto fieldName = recTy.getTypeList()[fieldIdx].first;
+            fieldTy = recTy.getTypeList()[fieldIdx].second;
+            mlir::Value fieldIndex = builder.create<fir::FieldIndexOp>(
+                loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
+                recTy,
+                /*typeParams=*/mlir::ValueRange{});
+            coordinates.push_back(fieldIndex);
+          } else {
+            // Field not found in base record type, search in potential
+            // record type components.
+            for (auto component : recTy.getTypeList()) {
+              if (auto childRecTy =
+                      mlir::dyn_cast<fir::RecordType>(component.second)) {
+                fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
+                if (fieldIdx != std::numeric_limits<unsigned>::max()) {
+                  mlir::Value parentFieldIndex =
+                      builder.create<fir::FieldIndexOp>(
+                          loc, fir::FieldType::get(childRecTy.getContext()),
+                          component.first, recTy,
+                          /*typeParams=*/mlir::ValueRange{});
+                  coordinates.push_back(parentFieldIndex);
+                  auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
+                  fieldTy = childRecTy.getTypeList()[fieldIdx].second;
+                  mlir::Value childFieldIndex =
+                      builder.create<fir::FieldIndexOp>(
+                          loc, fir::FieldType::get(fieldTy.getContext()),
+                          fieldName, childRecTy,
+                          /*typeParams=*/mlir::ValueRange{});
+                  coordinates.push_back(childFieldIndex);
+                  break;
+                }
+              }
+            }
+          }
+
+          if (coordinates.empty())
+            TODO(loc, "device resident component in complex derived-type "
+                      "hierarchy");
+
+          mlir::Value comp = builder.create<fir::CoordinateOp>(
+              loc, builder.getRefType(fieldTy), fir::getBase(exv), coordinates);
+          cuf::DataAttributeAttr dataAttr =
+              Fortran::lower::translateSymbolCUFDataAttribute(
+                  builder.getContext(), sym);
+          builder.create<cuf::SetAllocatorIndexOp>(loc, comp, dataAttr);
+        }
+      }
+    }
+  }
+}
+
 /// Must \p var be default initialized at runtime when entering its scope.
 static bool
 mustBeDefaultInitializedAtRuntime(const Fortran::lower::pft::Variable &var) {
@@ -1179,6 +1194,9 @@ static void instantiateLocal(Fortran::lower::AbstractConverter &converter,
   if (mustBeDefaultInitializedAtRuntime(var))
     Fortran::lower::defaultInitializeAtRuntime(converter, var.getSymbol(),
                                                symMap);
+  if (converter.getFoldingContext().languageFeatures().IsEnabled(
+          Fortran::common::LanguageFeature::CUDA))
+    initializeDeviceComponentAllocator(converter, var.getSymbol(), symMap);
   auto *builder = &converter.getFirOpBuilder();
   if (needCUDAAlloc(var.getSymbol()) &&
       !cuf::isCUDADeviceContext(builder->getRegion())) {
@@ -1437,6 +1455,9 @@ static void instantiateAlias(Fortran::lower::AbstractConverter &converter,
   if (mustBeDefaultInitializedAtRuntime(var))
     Fortran::lower::defaultInitializeAtRuntime(converter, var.getSymbol(),
                                                symMap);
+  if (converter.getFoldingContext().languageFeatures().IsEnabled(
+          Fortran::common::LanguageFeature::CUDA))
+    initializeDeviceComponentAllocator(converter, var.getSymbol(), symMap);
 }
 
 //===--------------------------------------------------------------===//

diff  --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index bf74e012a639d..ee89ea38a3fc7 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -12,10 +12,13 @@ contains
   end subroutine
 
 ! CHECK-LABEL: func.func @_QMm1Psub1()
-! CHECK: %[[DT:.*]] = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} -> !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
-! CHECK: %[[X:.*]] = fir.coordinate_of %[[DT]], x : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} -> !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DT:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>)
+! CHECK: fir.address_of(@_QQ_QMm1Tty_device.DerivedInit)
+! CHECK: fir.copy 
+! CHECK: %[[X:.*]] = fir.coordinate_of %[[DT]]#0, x : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[X]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
-! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]], z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]]#0, z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[Z]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
 
 end module


        

From flang-commits at lists.llvm.org  Tue Jul 15 13:52:07 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Tue, 15 Jul 2025 13:52:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Move cuf.set_allocator_idx
 after derived-type init (PR #148936)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148936@github.com>
Message-ID: <6876bf77.170a0220.1ffb09.5e00@mx.google.com>


https://github.com/clementval closed https://github.com/llvm/llvm-project/pull/148936

From flang-commits at lists.llvm.org  Tue Jul 15 14:53:22 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 14:53:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch bad members of BIND(C) COMMON
	block (PR #148971)
Message-ID: <llvm.org/llvm/llvm-project/pull/148971@github.com>


https://github.com/klausler created https://github.com/llvm/llvm-project/pull/148971

Variables that can't be BIND(C), like pointers, can't be in a BIND(C) common block, either.

Fixes https://github.com/llvm/llvm-project/issues/148922.

>From 861dd9beb753d374dec6cf09143f25a11b8e7cb7 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 15 Jul 2025 14:50:52 -0700
Subject: [PATCH] [flang] Catch bad members of BIND(C) COMMON block

Variables that can't be BIND(C), like pointers, can't be
in a BIND(C) common block, either.

Fixes https://github.com/llvm/llvm-project/issues/148922.
---
 flang/lib/Semantics/check-declarations.cpp | 39 ++++++++++++++++------
 flang/test/Semantics/bind-c18.f90          |  7 ++++
 2 files changed, 36 insertions(+), 10 deletions(-)
 create mode 100644 flang/test/Semantics/bind-c18.f90

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index f9d64485f1407..04fedcee20f12 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -151,8 +151,8 @@ class CheckHelper {
   void CheckProcedureAssemblyName(const Symbol &symbol);
   void CheckExplicitSave(const Symbol &);
   parser::Messages WhyNotInteroperableDerivedType(const Symbol &);
-  parser::Messages WhyNotInteroperableObject(
-      const Symbol &, bool allowNonInteroperableType = false);
+  parser::Messages WhyNotInteroperableObject(const Symbol &,
+      bool allowNonInteroperableType = false, bool forCommonBlock = false);
   parser::Messages WhyNotInteroperableFunctionResult(const Symbol &);
   parser::Messages WhyNotInteroperableProcedure(const Symbol &, bool isError);
   void CheckBindC(const Symbol &);
@@ -519,11 +519,28 @@ void CheckHelper::Check(const Symbol &symbol) {
 }
 
 void CheckHelper::CheckCommonBlock(const Symbol &symbol) {
+  auto restorer{messages_.SetLocation(symbol.name())};
   CheckGlobalName(symbol);
   if (symbol.attrs().test(Attr::BIND_C)) {
     CheckBindC(symbol);
+    for (auto ref : symbol.get<CommonBlockDetails>().objects()) {
+      if (ref->has<ObjectEntityDetails>()) {
+        if (auto msgs{WhyNotInteroperableObject(*ref,
+                /*allowInteroperableType=*/false, /*forCommonBlock=*/true)};
+            !msgs.empty()) {
+          if (auto *msg{messages_.Say(symbol.name(),
+                  "'%s' may not be a member of BIND(C) COMMON block /%s/"_err_en_US,
+                  ref->name(), symbol.name())}) {
+            for (parser::Message &reason : msgs.messages()) {
+              reason.set_severity(parser::Severity::Because);
+              msg->Attach(std::move(reason));
+            }
+          }
+        }
+      }
+    }
   }
-  for (MutableSymbolRef ref : symbol.get<CommonBlockDetails>().objects()) {
+  for (auto ref : symbol.get<CommonBlockDetails>().objects()) {
     if (ref->test(Symbol::Flag::CrayPointee)) {
       messages_.Say(ref->name(),
           "Cray pointee '%s' may not be a member of a COMMON block"_err_en_US,
@@ -3154,14 +3171,16 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
 }
 
 parser::Messages CheckHelper::WhyNotInteroperableObject(
-    const Symbol &symbol, bool allowNonInteroperableType) {
+    const Symbol &symbol, bool allowNonInteroperableType, bool forCommonBlock) {
   parser::Messages msgs;
-  if (examinedByWhyNotInteroperable_.find(symbol) !=
-      examinedByWhyNotInteroperable_.end()) {
-    return msgs;
+  if (!forCommonBlock) {
+    if (examinedByWhyNotInteroperable_.find(symbol) !=
+        examinedByWhyNotInteroperable_.end()) {
+      return msgs;
+    }
+    examinedByWhyNotInteroperable_.insert(symbol);
   }
   bool isExplicitBindC{symbol.attrs().test(Attr::BIND_C)};
-  examinedByWhyNotInteroperable_.insert(symbol);
   CHECK(symbol.has<ObjectEntityDetails>());
   if (isExplicitBindC && !symbol.owner().IsModule()) {
     msgs.Say(symbol.name(),
@@ -3338,8 +3357,8 @@ parser::Messages CheckHelper::WhyNotInteroperableProcedure(
           // on the C side by either a cdesc_t * or a void *.  F'2023 18.3.7 (5)
           bool allowNonInteroperableType{!dummy->attrs().test(Attr::VALUE) &&
               (IsDescriptor(*dummy) || IsAssumedType(*dummy))};
-          dummyMsgs =
-              WhyNotInteroperableObject(*dummy, allowNonInteroperableType);
+          dummyMsgs = WhyNotInteroperableObject(
+              *dummy, allowNonInteroperableType, /*forCommonBlock=*/false);
         } else {
           CheckBindC(*dummy);
         }
diff --git a/flang/test/Semantics/bind-c18.f90 b/flang/test/Semantics/bind-c18.f90
new file mode 100644
index 0000000000000..98a90796c514d
--- /dev/null
+++ b/flang/test/Semantics/bind-c18.f90
@@ -0,0 +1,7 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+bind(c) :: /blk/
+!ERROR: 'x' may not be a member of BIND(C) COMMON block /blk/
+!BECAUSE: A scalar interoperable variable may not be ALLOCATABLE or POINTER
+common /blk/ x
+integer, pointer :: x
+end


From flang-commits at lists.llvm.org  Tue Jul 15 14:53:59 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 15 Jul 2025 14:53:59 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch bad members of BIND(C) COMMON
	block (PR #148971)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148971@github.com>
Message-ID: <6876cdf7.170a0220.435b6.5c58@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Peter Klausler (klausler)

<details>
<summary>Changes</summary>

Variables that can't be BIND(C), like pointers, can't be in a BIND(C) common block, either.

Fixes https://github.com/llvm/llvm-project/issues/148922.

---
Full diff: https://github.com/llvm/llvm-project/pull/148971.diff


2 Files Affected:

- (modified) flang/lib/Semantics/check-declarations.cpp (+29-10) 
- (added) flang/test/Semantics/bind-c18.f90 (+7) 


``````````diff
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index f9d64485f1407..04fedcee20f12 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -151,8 +151,8 @@ class CheckHelper {
   void CheckProcedureAssemblyName(const Symbol &symbol);
   void CheckExplicitSave(const Symbol &);
   parser::Messages WhyNotInteroperableDerivedType(const Symbol &);
-  parser::Messages WhyNotInteroperableObject(
-      const Symbol &, bool allowNonInteroperableType = false);
+  parser::Messages WhyNotInteroperableObject(const Symbol &,
+      bool allowNonInteroperableType = false, bool forCommonBlock = false);
   parser::Messages WhyNotInteroperableFunctionResult(const Symbol &);
   parser::Messages WhyNotInteroperableProcedure(const Symbol &, bool isError);
   void CheckBindC(const Symbol &);
@@ -519,11 +519,28 @@ void CheckHelper::Check(const Symbol &symbol) {
 }
 
 void CheckHelper::CheckCommonBlock(const Symbol &symbol) {
+  auto restorer{messages_.SetLocation(symbol.name())};
   CheckGlobalName(symbol);
   if (symbol.attrs().test(Attr::BIND_C)) {
     CheckBindC(symbol);
+    for (auto ref : symbol.get<CommonBlockDetails>().objects()) {
+      if (ref->has<ObjectEntityDetails>()) {
+        if (auto msgs{WhyNotInteroperableObject(*ref,
+                /*allowInteroperableType=*/false, /*forCommonBlock=*/true)};
+            !msgs.empty()) {
+          if (auto *msg{messages_.Say(symbol.name(),
+                  "'%s' may not be a member of BIND(C) COMMON block /%s/"_err_en_US,
+                  ref->name(), symbol.name())}) {
+            for (parser::Message &reason : msgs.messages()) {
+              reason.set_severity(parser::Severity::Because);
+              msg->Attach(std::move(reason));
+            }
+          }
+        }
+      }
+    }
   }
-  for (MutableSymbolRef ref : symbol.get<CommonBlockDetails>().objects()) {
+  for (auto ref : symbol.get<CommonBlockDetails>().objects()) {
     if (ref->test(Symbol::Flag::CrayPointee)) {
       messages_.Say(ref->name(),
           "Cray pointee '%s' may not be a member of a COMMON block"_err_en_US,
@@ -3154,14 +3171,16 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
 }
 
 parser::Messages CheckHelper::WhyNotInteroperableObject(
-    const Symbol &symbol, bool allowNonInteroperableType) {
+    const Symbol &symbol, bool allowNonInteroperableType, bool forCommonBlock) {
   parser::Messages msgs;
-  if (examinedByWhyNotInteroperable_.find(symbol) !=
-      examinedByWhyNotInteroperable_.end()) {
-    return msgs;
+  if (!forCommonBlock) {
+    if (examinedByWhyNotInteroperable_.find(symbol) !=
+        examinedByWhyNotInteroperable_.end()) {
+      return msgs;
+    }
+    examinedByWhyNotInteroperable_.insert(symbol);
   }
   bool isExplicitBindC{symbol.attrs().test(Attr::BIND_C)};
-  examinedByWhyNotInteroperable_.insert(symbol);
   CHECK(symbol.has<ObjectEntityDetails>());
   if (isExplicitBindC && !symbol.owner().IsModule()) {
     msgs.Say(symbol.name(),
@@ -3338,8 +3357,8 @@ parser::Messages CheckHelper::WhyNotInteroperableProcedure(
           // on the C side by either a cdesc_t * or a void *.  F'2023 18.3.7 (5)
           bool allowNonInteroperableType{!dummy->attrs().test(Attr::VALUE) &&
               (IsDescriptor(*dummy) || IsAssumedType(*dummy))};
-          dummyMsgs =
-              WhyNotInteroperableObject(*dummy, allowNonInteroperableType);
+          dummyMsgs = WhyNotInteroperableObject(
+              *dummy, allowNonInteroperableType, /*forCommonBlock=*/false);
         } else {
           CheckBindC(*dummy);
         }
diff --git a/flang/test/Semantics/bind-c18.f90 b/flang/test/Semantics/bind-c18.f90
new file mode 100644
index 0000000000000..98a90796c514d
--- /dev/null
+++ b/flang/test/Semantics/bind-c18.f90
@@ -0,0 +1,7 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+bind(c) :: /blk/
+!ERROR: 'x' may not be a member of BIND(C) COMMON block /blk/
+!BECAUSE: A scalar interoperable variable may not be ALLOCATABLE or POINTER
+common /blk/ x
+integer, pointer :: x
+end

``````````

</details>


https://github.com/llvm/llvm-project/pull/148971

From flang-commits at lists.llvm.org  Tue Jul 15 15:48:05 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 15:48:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <6876daa5.170a0220.a88da.05b0@mx.google.com>


https://github.com/klausler approved this pull request.


https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Tue Jul 15 15:48:05 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 15 Jul 2025 15:48:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <6876daa5.a70a0220.1695c2.8c21@mx.google.com>


================
@@ -491,7 +491,13 @@ void Messages::AttachTo(Message &msg, std::optional<Severity> severity) {
   messages_.clear();
 }
 
-bool Messages::AnyFatalError() const {
+bool Messages::AnyFatalError(bool warningsAreErrors) const {
+  if (messages_.empty()) {
----------------
klausler wrote:

these six lines could be more clear as `if (warningsAreErrors{ ) return !messages_.empty(); }`

https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Tue Jul 15 18:55:50 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Tue, 15 Jul 2025 18:55:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Exit early when there is no
 device components (PR #149005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149005@github.com>
Message-ID: <687706a6.170a0220.1d61e9.5c87@mx.google.com>


https://github.com/vzakhari approved this pull request.


https://github.com/llvm/llvm-project/pull/149005

From flang-commits at lists.llvm.org  Wed Jul 16 03:14:15 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 16 Jul 2025 03:14:15 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add missing symbol names to the
	error message (PR #148888)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148888@github.com>
Message-ID: <68777b77.170a0220.2c91e2.7ed1@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/148888

>From bc9dbac2f7adf031873f627060be11c557c0b881 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 12:41:34 -0400
Subject: [PATCH] [flang] Add missing symbol names to the error message

Fixes #140485
---
 flang/lib/Semantics/resolve-names.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 96faa5fd954cd..b3268605e7c0c 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8574,8 +8574,10 @@ bool ResolveNamesVisitor::Pre(const parser::ImportStmt &x) {
         } else {
           Say(name,
               "A distinct '%s' is already present in this scope"_err_en_US)
-              .Attach(symbol->name(), "Previous declaration of '%s'"_en_US)
-              .Attach(outer->name(), "Declaration of '%s' in host scope"_en_US);
+              .Attach(symbol->name(), "Previous declaration of '%s'"_en_US,
+                  symbol->name().ToString())
+              .Attach(outer->name(), "Declaration of '%s' in host scope"_en_US,
+                  outer->name().ToString());
         }
       }
     } else {


From flang-commits at lists.llvm.org  Wed Jul 16 04:27:23 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 16 Jul 2025 04:27:23 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <68778c9b.170a0220.2cea77.8052@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul 16 04:27:23 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 16 Jul 2025 04:27:23 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <68778c9b.170a0220.276f3d.8638@mx.google.com>


================
@@ -224,6 +226,67 @@ std::optional<llvm::ArrayRef<int64_t>> getComponentLowerBoundsIfNonDefault(
     fir::RecordType recordType, llvm::StringRef component,
     mlir::ModuleOp module, const mlir::SymbolTable *symbolTable = nullptr);
 
+// Convert FIR type to LLVM without turning fir.box<T> into memory
+// reference.
+mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                             mlir::Type firType);
+
+/// Generate a LLVM constant value of type `ity`, using the provided offset.
+mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset);
+
+/// Helper function for generating the LLVM IR that computes the distance
+/// in bytes between adjacent elements pointed to by a pointer
+/// of type \p ptrTy. The result is returned as a value of \p idxTy integer
+/// type.
+mlir::Value computeElementDistance(mlir::Location loc,
+                                   mlir::Type llvmObjectType, mlir::Type idxTy,
+                                   mlir::ConversionPatternRewriter &rewriter,
+                                   const mlir::DataLayout &dataLayout);
+
+// Compute the alloc scale size (constant factors encoded in the array type).
+// We do this for arrays without a constant interior or arrays of character with
+// dynamic length arrays, since those are the only ones that get decayed to a
+// pointer to the element type.
+template <typename OP>
+inline mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
----------------
ergawy wrote:

Can you please de-templetize this and move the definiton to the `.cpp` file? >From `op`, we only need `InType` and `Loc`; both of which can be easily passed to the function on call sites.

Adding templates in headers files (specially the ones who are used in a lot places like `Uilts` files) can slow down compilation times.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul 16 04:27:24 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 16 Jul 2025 04:27:24 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <68778c9c.630a0220.37d6a9.aa47@mx.google.com>


================
@@ -1887,4 +1887,68 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Allocates memory on the specified OpenMP device for an object of the given type.
+    Returns an integer value representing the device pointer to the allocated memory.
+    The memory is uninitialized after allocation. Operations must be paired with 
+    `omp.target_freemem` to avoid memory leaks.
+
+    ```mlir
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyInteger>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<IntLikeType>:$typeparams,
+    Variadic<IntLikeType>:$shape
----------------
ergawy wrote:

Do we need both fields or is one sufficient? Also, please document the op fields in the description.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul 16 04:27:24 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 16 Jul 2025 04:27:24 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <68778c9c.170a0220.1e3a58.2f1d@mx.google.com>


https://github.com/ergawy commented:

Thanks @skc7 for taking care of the comments and apologies fir the delay. I just have a few more comments and this should be good to go.

However, it would be nice to get a review from some one else as well.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul 16 04:33:48 2025
From: flang-commits at lists.llvm.org (Aaron Ballman via flang-commits)
Date: Wed, 16 Jul 2025 04:33:48 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [Clang] Remove ARCMigrate
	(PR #119269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/119269@github.com>
Message-ID: <68778e1c.170a0220.a3385.8a24@mx.google.com>


AaronBallman wrote:

> @AaronBallman Do you happen to know if there’s precedent for this?

We recently started [documenting our guarantees](https://clang.llvm.org/docs/LibClang.html#abi-and-api-stability), and I think this falls under "Removing an explicitly deprecated API after a suitably long deprecation period." So I think the right path forward is to add stub APIs which fail gracefully, mark the APIs as deprecated and explain that the underlying functionality has been removed so these APIs only exist for ABI compatibility, and then we can remove them in a few years. Do others think that's a reasonable plan?



https://github.com/llvm/llvm-project/pull/119269

From flang-commits at lists.llvm.org  Wed Jul 16 04:48:26 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 04:48:26 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [Clang] Remove ARCMigrate
	(PR #119269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/119269@github.com>
Message-ID: <6877918a.170a0220.13e724.82be@mx.google.com>


Sirraide wrote:

I’ved opened #149079 for this.

https://github.com/llvm/llvm-project/pull/119269

From flang-commits at lists.llvm.org  Wed Jul 16 05:59:23 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 05:59:23 -0700 (PDT)
Subject: [flang-commits] [flang] 1742966 - [Flang] Force lowering to Complex
	for AMDGPU (#144927)
Message-ID: <6877a22b.050a0220.3d472f.df3e@mx.google.com>


Author: Akash Banerjee
Date: 2025-07-16T13:59:20+01:00
New Revision: 1742966c0df03d1f00a177a8d8a2a2afaec6938d

URL: https://github.com/llvm/llvm-project/commit/1742966c0df03d1f00a177a8d8a2a2afaec6938d
DIFF: https://github.com/llvm/llvm-project/commit/1742966c0df03d1f00a177a8d8a2a2afaec6938d.diff

LOG: [Flang] Force lowering to Complex for AMDGPU (#144927)

Added: 
    flang/test/Lower/amdgcn-complex.f90

Modified: 
    flang/lib/Optimizer/Builder/IntrinsicCall.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d32c1fde59f27..8d0a511744e25 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -1231,8 +1231,11 @@ mlir::Value genComplexMathOp(fir::FirOpBuilder &builder, mlir::Location loc,
   llvm::StringRef mathLibFuncName = mathOp.runtimeFunc;
   if (!mathLibFuncName.empty()) {
     // If we enabled MLIR complex or can use approximate operations, we should
-    // NOT use libm.
-    if (!forceMlirComplex && !canUseApprox) {
+    // NOT use libm. Avoid libm when targeting AMDGPU as those symbols are not
+    // available on the device and we rely on MLIR complex operations to
+    // later map to OCML calls.
+    bool isAMDGPU = fir::getTargetTriple(builder.getModule()).isAMDGCN();
+    if (!forceMlirComplex && !canUseApprox && !isAMDGPU) {
       result = genLibCall(builder, loc, mathOp, mathLibFuncType, args);
       LLVM_DEBUG(result.dump(); llvm::dbgs() << "\n");
       return result;

diff  --git a/flang/test/Lower/amdgcn-complex.f90 b/flang/test/Lower/amdgcn-complex.f90
new file mode 100644
index 0000000000000..f15c7db2b7316
--- /dev/null
+++ b/flang/test/Lower/amdgcn-complex.f90
@@ -0,0 +1,21 @@
+! REQUIRES: amdgpu-registered-target
+! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+
+subroutine cabsf_test(a, b)
+   complex :: a
+   real :: b
+   b = abs(a)
+end subroutine
+
+! CHECK-LABEL: func @_QPcabsf_test(
+! CHECK: complex.abs
+! CHECK-NOT: fir.call @cabsf
+
+subroutine cexpf_test(a, b)
+   complex :: a, b
+   b = exp(a)
+end subroutine
+
+! CHECK-LABEL: func @_QPcexpf_test(
+! CHECK: complex.exp
+! CHECK-NOT: fir.call @cexpf


        

From flang-commits at lists.llvm.org  Wed Jul 16 05:59:26 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 16 Jul 2025 05:59:26 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Force lowering to Complex for
	AMDGPU (PR #144927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144927@github.com>
Message-ID: <6877a22e.170a0220.28848c.2eb9@mx.google.com>


https://github.com/TIFitis closed https://github.com/llvm/llvm-project/pull/144927

From flang-commits at lists.llvm.org  Wed Jul 16 05:59:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 05:59:45 -0700 (PDT)
Subject: [flang-commits] [flang] fc114e4 - [MLIR] Add
	ComplexTOROCDLLibraryCalls pass (#144926)
Message-ID: <6877a241.170a0220.31b408.88f2@mx.google.com>


Author: Akash Banerjee
Date: 2025-07-16T13:59:41+01:00
New Revision: fc114e4d931ae25f74a15e42371dbead1387ad51

URL: https://github.com/llvm/llvm-project/commit/fc114e4d931ae25f74a15e42371dbead1387ad51
DIFF: https://github.com/llvm/llvm-project/commit/fc114e4d931ae25f74a15e42371dbead1387ad51.diff

LOG: [MLIR] Add ComplexTOROCDLLibraryCalls pass (#144926)

Added: 
    mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
    mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
    mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
    mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir

Modified: 
    flang/lib/Optimizer/CodeGen/CMakeLists.txt
    flang/lib/Optimizer/CodeGen/CodeGen.cpp
    mlir/include/mlir/Conversion/Passes.h
    mlir/include/mlir/Conversion/Passes.td
    mlir/lib/Conversion/CMakeLists.txt

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 980307db315d9..16c7944a885a1 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -34,6 +34,7 @@ add_flang_library(FIRCodeGen
 
   MLIR_LIBS
   MLIRComplexToLLVM
+  MLIRComplexToROCDLLibraryCalls
   MLIRComplexToStandard
   MLIRGPUDialect
   MLIRMathToFuncs

diff  --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index ecc04a6c9a2be..5ca53ee48955e 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -33,6 +33,7 @@
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
@@ -4145,22 +4146,24 @@ class FIRToLLVMLowering
     // conversions that affect the ModuleOp, e.g. create new
     // function operations in it. We have to run such conversions
     // as passes here.
-    mlir::OpPassManager mathConvertionPM("builtin.module");
+    mlir::OpPassManager mathConversionPM("builtin.module");
 
     bool isAMDGCN = fir::getTargetTriple(mod).isAMDGCN();
     // If compiling for AMD target some math operations must be lowered to AMD
     // GPU library calls, the rest can be converted to LLVM intrinsics, which
     // is handled in the mathToLLVM conversion. The lowering to libm calls is
     // not needed since all math operations are handled this way.
-    if (isAMDGCN)
-      mathConvertionPM.addPass(mlir::createConvertMathToROCDL());
+    if (isAMDGCN) {
+      mathConversionPM.addPass(mlir::createConvertMathToROCDL());
+      mathConversionPM.addPass(mlir::createConvertComplexToROCDLLibraryCalls());
+    }
 
     // Convert math::FPowI operations to inline implementation
     // only if the exponent's width is greater than 32, otherwise,
     // it will be lowered to LLVM intrinsic operation by a later conversion.
     mlir::ConvertMathToFuncsOptions mathToFuncsOptions{};
     mathToFuncsOptions.minWidthOfFPowIExponent = 33;
-    mathConvertionPM.addPass(
+    mathConversionPM.addPass(
         mlir::createConvertMathToFuncs(mathToFuncsOptions));
 
     mlir::ConvertComplexToStandardPassOptions complexToStandardOptions{};
@@ -4173,15 +4176,15 @@ class FIRToLLVMLowering
       complexToStandardOptions.complexRange =
           mlir::complex::ComplexRangeFlags::improved;
     }
-    mathConvertionPM.addPass(
+    mathConversionPM.addPass(
         mlir::createConvertComplexToStandardPass(complexToStandardOptions));
 
     // Convert Math dialect operations into LLVM dialect operations.
     // There is no way to prefer MathToLLVM patterns over MathToLibm
     // patterns (applied below), so we have to run MathToLLVM conversion here.
-    mathConvertionPM.addNestedPass<mlir::func::FuncOp>(
+    mathConversionPM.addNestedPass<mlir::func::FuncOp>(
         mlir::createConvertMathToLLVMPass());
-    if (mlir::failed(runPipeline(mathConvertionPM, mod)))
+    if (mlir::failed(runPipeline(mathConversionPM, mod)))
       return signalPassFailure();
 
     std::optional<mlir::DataLayout> dl =

diff  --git a/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
new file mode 100644
index 0000000000000..daac2a99ed80f
--- /dev/null
+++ b/mlir/include/mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h
@@ -0,0 +1,27 @@
+//===- ComplexToROCDLLibraryCalls.h - convert from Complex to ROCDL calls -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
+#define MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_
+
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+class RewritePatternSet;
+
+#define GEN_PASS_DECL_CONVERTCOMPLEXTOROCDLLIBRARYCALLS
+#include "mlir/Conversion/Passes.h.inc"
+
+/// Populate the given list with patterns that convert from Complex to ROCDL
+/// calls.
+void populateComplexToROCDLLibraryCallsConversionPatterns(
+    RewritePatternSet &patterns);
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_COMPLEXTOROCDLLIBRARYCALLS_COMPLEXTOROCDLLIBRARYCALLS_H_

diff  --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 8a5976e547169..d93fbefab74aa 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -23,6 +23,7 @@
 #include "mlir/Conversion/BufferizationToMemRef/BufferizationToMemRef.h"
 #include "mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h"
 #include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h"
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRVPass.h"
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"

diff  --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 50c67da91a4af..76e751243a12c 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -312,6 +312,18 @@ def ConvertComplexToLibm : Pass<"convert-complex-to-libm", "ModuleOp"> {
   let dependentDialects = ["func::FuncDialect"];
 }
 
+//===----------------------------------------------------------------------===//
+// ComplexToROCDLLibraryCalls
+//===----------------------------------------------------------------------===//
+
+def ConvertComplexToROCDLLibraryCalls : Pass<"convert-complex-to-rocdl-library-calls", "ModuleOp"> {
+  let summary = "Convert Complex dialect to ROCDL library calls";
+  let description = [{
+    This pass converts supported Complex ops to calls to the AMD device library.
+  }];
+  let dependentDialects = ["func::FuncDialect"];
+}
+
 //===----------------------------------------------------------------------===//
 // ComplexToSPIRV
 //===----------------------------------------------------------------------===//

diff  --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index 24a48993ad80c..f84375b6b8d6a 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(AsyncToLLVM)
 add_subdirectory(BufferizationToMemRef)
 add_subdirectory(ComplexCommon)
 add_subdirectory(ComplexToLibm)
+add_subdirectory(ComplexToROCDLLibraryCalls)
 add_subdirectory(ComplexToLLVM)
 add_subdirectory(ComplexToSPIRV)
 add_subdirectory(ComplexToStandard)

diff  --git a/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
new file mode 100644
index 0000000000000..695bb2dd0a82c
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_conversion_library(MLIRComplexToROCDLLibraryCalls
+  ComplexToROCDLLibraryCalls.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ComplexToROCDLLibraryCalls
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRComplexDialect
+  MLIRFuncDialect
+  MLIRPass
+  MLIRTransformUtils
+  )

diff  --git a/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
new file mode 100644
index 0000000000000..99d5424aef79a
--- /dev/null
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
@@ -0,0 +1,92 @@
+//=== ComplexToROCDLLibraryCalls.cpp - convert from Complex to ROCDL calls ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.h"
+#include "mlir/Dialect/Complex/IR/Complex.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTCOMPLEXTOROCDLLIBRARYCALLS
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+
+namespace {
+
+template <typename Op, typename FloatTy>
+// Pattern to convert Complex ops to ROCDL function calls.
+struct ComplexOpToROCDLLibraryCalls : public OpRewritePattern<Op> {
+  using OpRewritePattern<Op>::OpRewritePattern;
+  ComplexOpToROCDLLibraryCalls(MLIRContext *context, StringRef funcName,
+                               PatternBenefit benefit = 1)
+      : OpRewritePattern<Op>(context, benefit), funcName(funcName) {}
+
+  LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final {
+    Operation *symTable = SymbolTable::getNearestSymbolTable(op);
+    Type resType = op.getType();
+    if (auto complexType = dyn_cast<ComplexType>(resType))
+      resType = complexType.getElementType();
+    if (!isa<FloatTy>(resType))
+      return failure();
+
+    auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
+        SymbolTable::lookupSymbolIn(symTable, funcName));
+    if (!opFunc) {
+      OpBuilder::InsertionGuard guard(rewriter);
+      rewriter.setInsertionPointToStart(&symTable->getRegion(0).front());
+      auto funcTy = FunctionType::get(
+          rewriter.getContext(), op->getOperandTypes(), op->getResultTypes());
+      opFunc = rewriter.create<func::FuncOp>(rewriter.getUnknownLoc(), funcName,
+                                             funcTy);
+      opFunc.setPrivate();
+    }
+    rewriter.replaceOpWithNewOp<func::CallOp>(op, funcName, op.getType(),
+                                              op->getOperands());
+    return success();
+  }
+
+private:
+  std::string funcName;
+};
+} // namespace
+
+void mlir::populateComplexToROCDLLibraryCallsConversionPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float32Type>>(
+      patterns.getContext(), "__ocml_cabs_f32");
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float64Type>>(
+      patterns.getContext(), "__ocml_cabs_f64");
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ExpOp, Float32Type>>(
+      patterns.getContext(), "__ocml_cexp_f32");
+  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ExpOp, Float64Type>>(
+      patterns.getContext(), "__ocml_cexp_f64");
+}
+
+namespace {
+struct ConvertComplexToROCDLLibraryCallsPass
+    : public impl::ConvertComplexToROCDLLibraryCallsBase<
+          ConvertComplexToROCDLLibraryCallsPass> {
+  void runOnOperation() override;
+};
+} // namespace
+
+void ConvertComplexToROCDLLibraryCallsPass::runOnOperation() {
+  Operation *op = getOperation();
+
+  RewritePatternSet patterns(&getContext());
+  populateComplexToROCDLLibraryCallsConversionPatterns(patterns);
+
+  ConversionTarget target(getContext());
+  target.addLegalDialect<func::FuncDialect>();
+  target.addIllegalOp<complex::AbsOp, complex::ExpOp>();
+  if (failed(applyPartialConversion(op, target, std::move(patterns))))
+    signalPassFailure();
+}

diff  --git a/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
new file mode 100644
index 0000000000000..bae7c5986ef9e
--- /dev/null
+++ b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
@@ -0,0 +1,26 @@
+// RUN: mlir-opt %s -convert-complex-to-rocdl-library-calls | FileCheck %s
+
+// CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
+// CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64
+// CHECK-DAG: @__ocml_cexp_f32(complex<f32>) -> complex<f32>
+// CHECK-DAG: @__ocml_cexp_f64(complex<f64>) -> complex<f64>
+
+//CHECK-LABEL: @abs_caller
+func.func @abs_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
+  // CHECK: %[[RF:.*]] = call @__ocml_cabs_f32(%{{.*}})
+  %rf = complex.abs %f : complex<f32>
+  // CHECK: %[[RD:.*]] = call @__ocml_cabs_f64(%{{.*}})
+  %rd = complex.abs %d : complex<f64>
+  // CHECK: return %[[RF]], %[[RD]]
+  return %rf, %rd : f32, f64
+}
+
+//CHECK-LABEL: @exp_caller
+func.func @exp_caller(%f: complex<f32>, %d: complex<f64>) -> (complex<f32>, complex<f64>) {
+  // CHECK: %[[EF:.*]] = call @__ocml_cexp_f32(%{{.*}})
+  %ef = complex.exp %f : complex<f32>
+  // CHECK: %[[ED:.*]] = call @__ocml_cexp_f64(%{{.*}})
+  %ed = complex.exp %d : complex<f64>
+  // CHECK: return %[[EF]], %[[ED]]
+  return %ef, %ed : complex<f32>, complex<f64>
+}


        

From flang-commits at lists.llvm.org  Wed Jul 16 05:59:51 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 16 Jul 2025 05:59:51 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR] Add
	ComplexTOROCDLLibraryCalls pass (PR #144926)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144926@github.com>
Message-ID: <6877a247.050a0220.1764ea.00d3@mx.google.com>


https://github.com/TIFitis closed https://github.com/llvm/llvm-project/pull/144926

From flang-commits at lists.llvm.org  Wed Jul 16 06:00:09 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 06:00:09 -0700 (PDT)
Subject: [flang-commits] [flang] dbb1210 - [OpenMP] Add TargetAMDGPU support
 for Complex argument and return types (#144924)
Message-ID: <6877a259.170a0220.26c522.8949@mx.google.com>


Author: Akash Banerjee
Date: 2025-07-16T14:00:06+01:00
New Revision: dbb12109b947995d5882ca521f7b7716d6ac9ae4

URL: https://github.com/llvm/llvm-project/commit/dbb12109b947995d5882ca521f7b7716d6ac9ae4
DIFF: https://github.com/llvm/llvm-project/commit/dbb12109b947995d5882ca521f7b7716d6ac9ae4.diff

LOG: [OpenMP] Add TargetAMDGPU support for Complex argument and return types (#144924)

Added: 
    

Modified: 
    flang/lib/Optimizer/CodeGen/Target.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp
index 7dbf21ce0c125..b60a72e4340b9 100644
--- a/flang/lib/Optimizer/CodeGen/Target.cpp
+++ b/flang/lib/Optimizer/CodeGen/Target.cpp
@@ -1443,14 +1443,35 @@ struct TargetAMDGPU : public GenericTarget<TargetAMDGPU> {
   CodeGenSpecifics::Marshalling
   complexArgumentType(mlir::Location loc, mlir::Type eleTy) const override {
     CodeGenSpecifics::Marshalling marshal;
-    TODO(loc, "handle complex argument types");
+    const auto *sem = &floatToSemantics(kindMap, eleTy);
+    if (sem == &llvm::APFloat::IEEEsingle()) {
+      // Lower COMPLEX(KIND=4) as an array of two element values.
+      marshal.emplace_back(fir::SequenceType::get({2}, eleTy), AT{});
+    } else if (sem == &llvm::APFloat::IEEEdouble()) {
+      // Pass COMPLEX(KIND=8) as two separate arguments.
+      marshal.emplace_back(eleTy, AT{});
+      marshal.emplace_back(eleTy, AT{});
+    } else {
+      typeTodo(sem, loc, "argument");
+    }
     return marshal;
   }
 
   CodeGenSpecifics::Marshalling
   complexReturnType(mlir::Location loc, mlir::Type eleTy) const override {
     CodeGenSpecifics::Marshalling marshal;
-    TODO(loc, "handle complex return types");
+    const auto *sem = &floatToSemantics(kindMap, eleTy);
+    if (sem == &llvm::APFloat::IEEEsingle()) {
+      // Return COMPLEX(KIND=4) as an array of two elements.
+      marshal.emplace_back(fir::SequenceType::get({2}, eleTy), AT{});
+    } else if (sem == &llvm::APFloat::IEEEdouble()) {
+      // Return COMPLEX(KIND=8) via an aggregate with two fields.
+      marshal.emplace_back(mlir::TupleType::get(eleTy.getContext(),
+                                                mlir::TypeRange{eleTy, eleTy}),
+                           AT{});
+    } else {
+      typeTodo(sem, loc, "return");
+    }
     return marshal;
   }
 };


        

From flang-commits at lists.llvm.org  Wed Jul 16 06:00:13 2025
From: flang-commits at lists.llvm.org (Akash Banerjee via flang-commits)
Date: Wed, 16 Jul 2025 06:00:13 -0700 (PDT)
Subject: [flang-commits] [flang] [OpenMP] Add TargetAMDGPU support for
 Complex argument and return types (PR #144924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144924@github.com>
Message-ID: <6877a25d.170a0220.22b33f.9654@mx.google.com>


https://github.com/TIFitis closed https://github.com/llvm/llvm-project/pull/144924

From flang-commits at lists.llvm.org  Wed Jul 16 07:01:49 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 07:01:49 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [mlir] [flang] Add support for
 workdistribute construct in flang frontend (PR #146029)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/146029@github.com>
Message-ID: <6877b0cd.170a0220.a88da.4a3a@mx.google.com>


https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/146029

>From cb89a7f7d8edf63ef6d054737cd8615bec86ce27 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 27 Jun 2025 11:05:35 +0530
Subject: [PATCH 1/2] [flang] Add support for workdistribute construct in flang
 frontend

---
 .../flang/Semantics/openmp-directive-sets.h   | 14 +++++
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 26 +++++++-
 flang/lib/Parser/openmp-parsers.cpp           |  7 ++-
 flang/lib/Semantics/resolve-directives.cpp    |  8 ++-
 flang/test/Lower/OpenMP/workdistribute.f90    | 59 +++++++++++++++++++
 llvm/include/llvm/Frontend/OpenMP/OMP.td      | 55 +++++++++++++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 22 +++++++
 7 files changed, 188 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/workdistribute.f90

diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index dd610c9702c28..7ced6ed9b44d6 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,6 +143,7 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
+    Directive::OMPD_target_teams_workdistribute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -172,6 +173,7 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
+    Directive::OMPD_teams_workdistribute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
@@ -187,9 +189,16 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
+        Directive::OMPD_target_teams_workdistribute,
     } | topTeamsSet,
 };
 
+static const OmpDirectiveSet allWorkdistributeSet{
+    Directive::OMPD_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_target_teams_workdistribute,
+};
+
 //===----------------------------------------------------------------------===//
 // Directive sets for groups of multiple directives
 //===----------------------------------------------------------------------===//
@@ -230,6 +239,9 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
+    Directive::OMPD_target_teams_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_workdistribute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -294,6 +306,7 @@ static const OmpDirectiveSet workShareSet{
         Directive::OMPD_scope,
         Directive::OMPD_sections,
         Directive::OMPD_single,
+        Directive::OMPD_workdistribute,
     } | allDoSet,
 };
 
@@ -376,6 +389,7 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
+    Directive::OMPD_workdistribute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4458f62eea95a..fcee143a19ef9 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -618,6 +618,16 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
       cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->iv);
       break;
 
+    case OMPD_teams_workdistribute:
+      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      [[fallthrough]];
+    case OMPD_target_teams_workdistribute:
+      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      processSingleNestedIf([](Directive nestedDir) {
+        return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
+      });
+      break;
+
     // Standalone 'target' case.
     case OMPD_target: {
       processSingleNestedIf(
@@ -2719,6 +2729,17 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
+static mlir::omp::WorkdistributeOp genWorkdistributeOp(
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::Location loc, const ConstructQueue &queue,
+    ConstructQueue::const_iterator item) {
+  return genOpWithBody<mlir::omp::WorkdistributeOp>(
+      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
+                        llvm::omp::Directive::OMPD_workdistribute),
+      queue, item);
+}
+
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3339,7 +3360,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     TODO(loc, "Unhandled loop directive (" +
                   llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
   }
-  // case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_workdistribute:
+    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue,
+                                item);
+    break;
   case llvm::omp::Directive::OMPD_workshare:
     newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
                            queue, item);
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..9a08df6e4a889 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1567,12 +1567,17 @@ TYPE_PARSER(
         "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET_DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
         "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
+        "TARGET TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_target_teams_workdistribute),
         "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
         "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
         "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
         "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
+        "TEAMS WORKDISTRIBUTE" >>
+            pure(llvm::omp::Directive::OMPD_teams_workdistribute),
         "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
-        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare))))
+        "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare),
+        "WORKDISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_workdistribute))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 151f4ccae634e..42d24e703889e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1680,10 +1680,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_teams_workdistribute:
     PushContext(beginDir.source, beginDir.v);
     break;
   default:
@@ -1713,9 +1716,12 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
-  case llvm::omp::Directive::OMPD_target_parallel: {
+  case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
+  case llvm::omp::Directive::OMPD_teams_workdistribute: {
     bool hasPrivate;
     for (const auto *allocName : allocateNames_) {
       hasPrivate = false;
diff --git a/flang/test/Lower/OpenMP/workdistribute.f90 b/flang/test/Lower/OpenMP/workdistribute.f90
new file mode 100644
index 0000000000000..924205bb72e5e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/workdistribute.f90
@@ -0,0 +1,59 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute
+subroutine target_teams_workdistribute()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end target teams workdistribute
+end subroutine target_teams_workdistribute
+
+! CHECK-LABEL: func @_QPteams_workdistribute
+subroutine teams_workdistribute()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end teams workdistribute
+end subroutine teams_workdistribute
+
+! CHECK-LABEL: func @_QPtarget_teams_workdistribute_m
+subroutine target_teams_workdistribute_m()
+  ! CHECK: omp.target
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp target
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+  !$omp end target
+end subroutine target_teams_workdistribute_m
+
+! CHECK-LABEL: func @_QPteams_workdistribute_m
+subroutine teams_workdistribute_m()
+  ! CHECK: omp.teams
+  ! CHECK: omp.workdistribute
+  !$omp teams
+  !$omp workdistribute
+  ! CHECK: fir.call
+  call f1()
+  ! CHECK: omp.terminator
+  ! CHECK: omp.terminator
+  !$omp end workdistribute
+  !$omp end teams
+end subroutine teams_workdistribute_m
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 1b94657dfae1e..64a9b29a3f1d0 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -1309,6 +1309,15 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
+def OMP_Workdistribute : Directive<[Spelling<"workdistribute">]> {
+  let association = AS_Block;
+  let category = CA_Executable;
+}
+def OMP_EndWorkdistribute : Directive<[Spelling<"end workdistribute">]> {
+  let leafConstructs = OMP_Workdistribute.leafConstructs;
+  let association = OMP_Workdistribute.association;
+  let category = OMP_Workdistribute.category;
+}
 
 //===----------------------------------------------------------------------===//
 // Definitions of OpenMP compound directives
@@ -2452,6 +2461,34 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_OMPX_DynCGroupMem>,
+    VersionedClause<OMPC_OMPX_Bare>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
@@ -2682,6 +2719,24 @@ def OMP_TeamsDistributeSimd : Directive<[Spelling<"teams distribute simd">]> {
   let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TeamsWorkdistribute : Directive<[Spelling<"teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_If, 52>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+}
 def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 8cf18b43450ab..18a68ee637dd3 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -2113,4 +2113,26 @@ def AllocateDirOp : OpenMP_Op<"allocate_dir", clauses = [
   let hasVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// workdistribute Construct
+//===----------------------------------------------------------------------===//
+
+def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
+  let summary = "workdistribute directive";
+  let description = [{
+    workdistribute divides execution of the enclosed structured block into
+    separate units of work, each executed only once by each
+    initial thread in the league.
+    ```
+    !$omp target teams
+        !$omp workdistribute
+        y = a * x + y 
+        !$omp end workdistribute
+    !$omp end target teams
+    ```
+  }];
+  let regions = (region AnyRegion:$region);
+  let assemblyFormat = "$region attr-dict";
+}
+
 #endif // OPENMP_OPS

>From fe2a447e45038d1b8251b707e8f14c7ff7bf79ab Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Fri, 27 Jun 2025 16:33:58 +0530
Subject: [PATCH 2/2] [OpenMP] Add verifier and tests for workdistribute mlir
 op.

---
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  4 ++--
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  1 +
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 15 ++++++++++++
 mlir/test/Dialect/OpenMP/invalid.mlir         | 23 +++++++++++++++++++
 mlir/test/Dialect/OpenMP/ops.mlir             | 12 ++++++++++
 5 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index fcee143a19ef9..37af546657644 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -619,10 +619,10 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
       break;
 
     case OMPD_teams_workdistribute:
-      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      cp.processThreadLimit(stmtCtx, hostInfo->ops);
       [[fallthrough]];
     case OMPD_target_teams_workdistribute:
-      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      cp.processNumTeams(stmtCtx, hostInfo->ops);
       processSingleNestedIf([](Directive nestedDir) {
         return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
       });
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 18a68ee637dd3..189b96510ce99 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -2132,6 +2132,7 @@ def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
     ```
   }];
   let regions = (region AnyRegion:$region);
+  let hasVerifier = 1;
   let assemblyFormat = "$region attr-dict";
 }
 
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 769aee64e1695..fd4f1891703f1 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3878,6 +3878,21 @@ LogicalResult AllocateDirOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// WorkdistributeOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult WorkdistributeOp::verify() {
+  Region &region = getRegion();
+  if (!region.hasOneBlock())
+    return emitOpError("region must contain exactly one block");
+
+  Operation *parentOp = (*this)->getParentOp();
+  if (!llvm::dyn_cast<TeamsOp>(parentOp))
+    return emitOpError("workdistribute must be nested under teams");
+  return success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 5088f2dfa7d7a..9723a263fca61 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -3017,3 +3017,26 @@ func.func @invalid_allocate_allocator(%arg0 : memref<i32>) -> () {
 
   return
 }
+
+// -----
+func.func @invalid_workdistribute() -> () {
+// expected-error @below {{workdistribute must be nested under teams}}
+  omp.workdistribute {
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @invalid_workdistribute_with_multiple_blocks() -> () {
+  omp.teams {
+  // expected-error @below {{region must contain exactly one block}}
+  omp.workdistribute {
+    cf.br ^bb1
+  ^bb1:
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 4c50ed3230976..f77035753ddb7 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -3230,3 +3230,15 @@ func.func @omp_allocate_dir(%arg0 : memref<i32>, %arg1 : memref<i32>) -> () {
   return
 }
 
+// CHECK-LABEL: func.func @omp_workdistribute
+func.func @omp_workdistribute() {
+  // CHECK: omp.teams
+  omp.teams {
+  // CHECK: omp.workdistribute
+  omp.workdistribute {
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}


From flang-commits at lists.llvm.org  Wed Jul 16 07:22:49 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Wed, 16 Jul 2025 07:22:49 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148927@github.com>
Message-ID: <6877b5b9.170a0220.3299af.c70f@mx.google.com>


https://github.com/DanielCChen commented:

Thanks for the fix!
It fixed the `UNIT`. However, the option  `-fdefault-integer-8` seems affect the `v_list` as well as the full test case still fails with an incorrect`v_list`. 
Should I open a issue?

https://github.com/llvm/llvm-project/pull/148927

From flang-commits at lists.llvm.org  Wed Jul 16 07:23:51 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Wed, 16 Jul 2025 07:23:51 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148927@github.com>
Message-ID: <6877b5f7.170a0220.1f226.bb3f@mx.google.com>


https://github.com/DanielCChen edited https://github.com/llvm/llvm-project/pull/148927

From flang-commits at lists.llvm.org  Wed Jul 16 07:25:23 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 07:25:23 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148927@github.com>
Message-ID: <6877b653.630a0220.b84e5.548c@mx.google.com>


klausler wrote:

> Thanks for the fix! It fixed the `UNIT`. However, the option `-fdefault-integer-8` seems affect the `v_list` as well as the full test case still fails with an incorrect`v_list`. Should I open a new issue?

No, but please ensure that the original issue demonstrates the problem with `v_list`.

https://github.com/llvm/llvm-project/pull/148927

From flang-commits at lists.llvm.org  Wed Jul 16 07:27:14 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Wed, 16 Jul 2025 07:27:14 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148927@github.com>
Message-ID: <6877b6c2.050a0220.2cc13.054b@mx.google.com>


DanielCChen wrote:

> > Thanks for the fix! It fixed the `UNIT`. However, the option `-fdefault-integer-8` seems affect the `v_list` as well as the full test case still fails with an incorrect`v_list`. Should I open a new issue?
> 
> No, but please ensure that the original issue demonstrates the problem with `v_list`.

Ok. I will update the original test case in issue #148638 with the `v_list` problem as well.

https://github.com/llvm/llvm-project/pull/148927

From flang-commits at lists.llvm.org  Wed Jul 16 07:55:52 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 07:55:52 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][runtime] MCLOCK library
	routine (PR #148960)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148960@github.com>
Message-ID: <6877bd78.170a0220.16217b.e699@mx.google.com>


https://github.com/klausler updated https://github.com/llvm/llvm-project/pull/148960

>From 3184421138ce98463ce82ab792929c0a3edc8917 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 15 Jul 2025 13:54:58 -0700
Subject: [PATCH] [flang][runtime] MCLOCK library routine

Add MCLOCK as an interface to std::clock().
---
 flang-rt/lib/runtime/extensions.cpp      | 8 ++++----
 flang/docs/Intrinsics.md                 | 3 ++-
 flang/include/flang/Runtime/extensions.h | 9 +++++----
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/flang-rt/lib/runtime/extensions.cpp b/flang-rt/lib/runtime/extensions.cpp
index e70dff3997233..f6c39468d5655 100644
--- a/flang-rt/lib/runtime/extensions.cpp
+++ b/flang-rt/lib/runtime/extensions.cpp
@@ -27,10 +27,7 @@
 #include <thread>
 
 #ifdef _WIN32
-#define WIN32_LEAN_AND_MEAN
-#define NOMINMAX
-#include <windows.h>
-
+#include "flang/Common/windows-include.h"
 #include <synchapi.h>
 
 inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time,
@@ -309,6 +306,9 @@ void RTNAME(Perror)(const char *str) { perror(str); }
 // GNU extension function TIME()
 std::int64_t RTNAME(time)() { return time(nullptr); }
 
+// MCLOCK: returns accumulated CPU time in ticks
+std::int32_t FORTRAN_PROCEDURE_NAME(mclock)() { return std::clock(); }
+
 // Extension procedures related to I/O
 
 namespace io {
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index 0118f8eb7d913..f7da6c889d413 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -709,8 +709,9 @@ CACHESIZE, EOF, FP_CLASS, INT_PTR_KIND, ISNAN, LOC
 MALLOC, FREE
 ```
 
-### Library subroutine
+### Library subroutines and functions
 ```
+ticks = MCLOCK()
 CALL BACKTRACE()
 CALL FDATE(TIME)
 CALL GETLOG(USRNAME)
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index 06ae7f35d9b5b..b350204714431 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -12,14 +12,12 @@
 #ifndef FORTRAN_RUNTIME_EXTENSIONS_H_
 #define FORTRAN_RUNTIME_EXTENSIONS_H_
 
-#include "flang/Runtime/entry-names.h"
-
-#define FORTRAN_PROCEDURE_NAME(name) name##_
-
 #include "flang/Runtime/entry-names.h"
 #include <cstddef>
 #include <cstdint>
 
+#define FORTRAN_PROCEDURE_NAME(name) name##_
+
 #ifdef _WIN32
 // UID and GID don't exist on Windows, these exist to avoid errors.
 typedef std::uint32_t uid_t;
@@ -89,5 +87,8 @@ int FORTRAN_PROCEDURE_NAME(ierrno)();
 // GNU extension subroutine PERROR(STRING)
 void RTNAME(Perror)(const char *str);
 
+// MCLOCK -- returns accumulated time in ticks
+int FORTRAN_PROCEDURE_NAME(mclock)();
+
 } // extern "C"
 #endif // FORTRAN_RUNTIME_EXTENSIONS_H_


From flang-commits at lists.llvm.org  Wed Jul 16 01:35:45 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 01:35:45 -0700 (PDT)
Subject: [flang-commits] [flang] Add Flang-Tidy: A Fortran Static Analysis
	Tool (PR #143883)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143883@github.com>
Message-ID: <68776461.170a0220.37450f.7ada@mx.google.com>


https://github.com/Originns updated https://github.com/llvm/llvm-project/pull/143883

>From 29e41425289ffd99f5b554fdd62b8531b7737ecc Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 24 Jun 2025 11:43:59 +0200
Subject: [PATCH 01/31] [flang-tidy] basic executable infrastructure

---
 flang/tools/CMakeLists.txt                    |  1 +
 flang/tools/flang-tidy/.clang-format          |  2 ++
 flang/tools/flang-tidy/CMakeLists.txt         |  1 +
 flang/tools/flang-tidy/tool/CMakeLists.txt    | 27 +++++++++++++++++++
 .../flang-tidy/tool/FlangTidyToolMain.cpp     |  9 +++++++
 5 files changed, 40 insertions(+)
 create mode 100644 flang/tools/flang-tidy/.clang-format
 create mode 100644 flang/tools/flang-tidy/CMakeLists.txt
 create mode 100644 flang/tools/flang-tidy/tool/CMakeLists.txt
 create mode 100644 flang/tools/flang-tidy/tool/FlangTidyToolMain.cpp

diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..644fa7fe9c715 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -13,3 +13,4 @@ add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
 add_subdirectory(fir-opt)
 add_subdirectory(fir-lsp-server)
+add_subdirectory(flang-tidy)
diff --git a/flang/tools/flang-tidy/.clang-format b/flang/tools/flang-tidy/.clang-format
new file mode 100644
index 0000000000000..a74fda4b67345
--- /dev/null
+++ b/flang/tools/flang-tidy/.clang-format
@@ -0,0 +1,2 @@
+BasedOnStyle: LLVM
+AlwaysBreakTemplateDeclarations: Yes
diff --git a/flang/tools/flang-tidy/CMakeLists.txt b/flang/tools/flang-tidy/CMakeLists.txt
new file mode 100644
index 0000000000000..0362281046be1
--- /dev/null
+++ b/flang/tools/flang-tidy/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(tool)
diff --git a/flang/tools/flang-tidy/tool/CMakeLists.txt b/flang/tools/flang-tidy/tool/CMakeLists.txt
new file mode 100644
index 0000000000000..6233d354973eb
--- /dev/null
+++ b/flang/tools/flang-tidy/tool/CMakeLists.txt
@@ -0,0 +1,27 @@
+set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  FrontendOpenACC
+  FrontendOpenMP
+  Support
+  Option
+  TargetParser
+  )
+
+if(FLANG_PLUGIN_SUPPORT)
+  set(support_plugins SUPPORT_PLUGINS)
+endif()
+
+add_flang_tool(flang-tidy
+  FlangTidyToolMain.cpp
+
+  ${support_plugins}
+  )
+
+target_link_libraries(flang-tidy
+  PRIVATE
+  ${ALL_FLANG_TIDY_CHECKS}
+  )
+
+if(FLANG_PLUGIN_SUPPORT)
+  export_executable_symbols_for_plugins(flang-tidy)
+endif()
diff --git a/flang/tools/flang-tidy/tool/FlangTidyToolMain.cpp b/flang/tools/flang-tidy/tool/FlangTidyToolMain.cpp
new file mode 100644
index 0000000000000..8f3481de6df69
--- /dev/null
+++ b/flang/tools/flang-tidy/tool/FlangTidyToolMain.cpp
@@ -0,0 +1,9 @@
+//===--- FlangTidyToolMain.cpp - flang-tidy -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+int main(int argc, const char **argv) {}

>From 4b956962b6e3d76cf3cbcb0f2d8abe680b960837 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 24 Jun 2025 13:52:19 +0200
Subject: [PATCH 02/31] [flang-tidy] add main tool

---
 flang/tools/flang-tidy/tool/CMakeLists.txt      | 17 +++++++++++++++++
 flang/tools/flang-tidy/tool/FlangTidyMain.cpp   | 13 +++++++++++++
 flang/tools/flang-tidy/tool/FlangTidyMain.h     | 11 +++++++++++
 .../tools/flang-tidy/tool/FlangTidyToolMain.cpp |  6 +++++-
 4 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 flang/tools/flang-tidy/tool/FlangTidyMain.cpp
 create mode 100644 flang/tools/flang-tidy/tool/FlangTidyMain.h

diff --git a/flang/tools/flang-tidy/tool/CMakeLists.txt b/flang/tools/flang-tidy/tool/CMakeLists.txt
index 6233d354973eb..69db1da4723de 100644
--- a/flang/tools/flang-tidy/tool/CMakeLists.txt
+++ b/flang/tools/flang-tidy/tool/CMakeLists.txt
@@ -7,6 +7,21 @@ set(LLVM_LINK_COMPONENTS
   TargetParser
   )
 
+set(LLVM_OPTIONAL_SOURCES FlangTidyMain.cpp FlangTidyToolMain.cpp)
+
+add_flang_library(flangTidyMain STATIC
+  FlangTidyMain.cpp
+
+  LINK_LIBS
+  FortranSupport
+  FortranEvaluate
+  FortranParser
+  FortranSemantics
+  LLVMSupport
+  flangTidy
+  ${ALL_FLANG_TIDY_CHECKS}
+  )
+
 if(FLANG_PLUGIN_SUPPORT)
   set(support_plugins SUPPORT_PLUGINS)
 endif()
@@ -19,6 +34,8 @@ add_flang_tool(flang-tidy
 
 target_link_libraries(flang-tidy
   PRIVATE
+  flangTidy
+  flangTidyMain
   ${ALL_FLANG_TIDY_CHECKS}
   )
 
diff --git a/flang/tools/flang-tidy/tool/FlangTidyMain.cpp b/flang/tools/flang-tidy/tool/FlangTidyMain.cpp
new file mode 100644
index 0000000000000..7492dfda36d8f
--- /dev/null
+++ b/flang/tools/flang-tidy/tool/FlangTidyMain.cpp
@@ -0,0 +1,13 @@
+//===--- FlangTidyMain.cpp - flang-tidy -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+namespace Fortran::tidy {
+
+extern int flangTidyMain(int &argc, const char **argv) { return 0; }
+
+} // namespace Fortran::tidy
diff --git a/flang/tools/flang-tidy/tool/FlangTidyMain.h b/flang/tools/flang-tidy/tool/FlangTidyMain.h
new file mode 100644
index 0000000000000..8dcff94135cc0
--- /dev/null
+++ b/flang/tools/flang-tidy/tool/FlangTidyMain.h
@@ -0,0 +1,11 @@
+//===--- FlangTidyMain.h - flang-tidy ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+namespace Fortran::tidy {
+int flangTidyMain(int &argc, const char **argv);
+} // namespace Fortran::tidy
diff --git a/flang/tools/flang-tidy/tool/FlangTidyToolMain.cpp b/flang/tools/flang-tidy/tool/FlangTidyToolMain.cpp
index 8f3481de6df69..503f037241e1c 100644
--- a/flang/tools/flang-tidy/tool/FlangTidyToolMain.cpp
+++ b/flang/tools/flang-tidy/tool/FlangTidyToolMain.cpp
@@ -6,4 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int argc, const char **argv) {}
+#include "FlangTidyMain.h"
+
+int main(int argc, const char **argv) {
+  return Fortran::tidy::flangTidyMain(argc, argv);
+}

>From f6cea61d979042a6e6c029a84d2984d9aef16287 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Wed, 25 Jun 2025 11:38:51 +0200
Subject: [PATCH 03/31] [flang-tidy] add visitor, options and module & check
 architecture

---
 flang/tools/flang-tidy/CMakeLists.txt         |  27 +
 flang/tools/flang-tidy/FlangTidy.cpp          | 297 ++++++++++
 flang/tools/flang-tidy/FlangTidy.h            |  20 +
 flang/tools/flang-tidy/FlangTidyCheck.cpp     | 215 +++++++
 flang/tools/flang-tidy/FlangTidyCheck.h       | 542 ++++++++++++++++++
 flang/tools/flang-tidy/FlangTidyContext.h     |  78 +++
 flang/tools/flang-tidy/FlangTidyForceLinker.h |  18 +
 flang/tools/flang-tidy/FlangTidyModule.cpp    |  49 ++
 flang/tools/flang-tidy/FlangTidyModule.h      |  99 ++++
 .../flang-tidy/FlangTidyModuleRegistry.h      |  25 +
 flang/tools/flang-tidy/FlangTidyOptions.cpp   | 420 ++++++++++++++
 flang/tools/flang-tidy/FlangTidyOptions.h     | 223 +++++++
 flang/tools/flang-tidy/MultiplexVisitor.h     | 103 ++++
 flang/tools/flang-tidy/tool/FlangTidyMain.cpp | 297 +++++++++-
 .../flang-tidy/utils/CollectActualArguments.h |  45 ++
 flang/tools/flang-tidy/utils/SymbolUtils.h    |  25 +
 16 files changed, 2482 insertions(+), 1 deletion(-)
 create mode 100644 flang/tools/flang-tidy/FlangTidy.cpp
 create mode 100644 flang/tools/flang-tidy/FlangTidy.h
 create mode 100644 flang/tools/flang-tidy/FlangTidyCheck.cpp
 create mode 100644 flang/tools/flang-tidy/FlangTidyCheck.h
 create mode 100644 flang/tools/flang-tidy/FlangTidyContext.h
 create mode 100644 flang/tools/flang-tidy/FlangTidyForceLinker.h
 create mode 100644 flang/tools/flang-tidy/FlangTidyModule.cpp
 create mode 100644 flang/tools/flang-tidy/FlangTidyModule.h
 create mode 100644 flang/tools/flang-tidy/FlangTidyModuleRegistry.h
 create mode 100644 flang/tools/flang-tidy/FlangTidyOptions.cpp
 create mode 100644 flang/tools/flang-tidy/FlangTidyOptions.h
 create mode 100644 flang/tools/flang-tidy/MultiplexVisitor.h
 create mode 100644 flang/tools/flang-tidy/utils/CollectActualArguments.h
 create mode 100644 flang/tools/flang-tidy/utils/SymbolUtils.h

diff --git a/flang/tools/flang-tidy/CMakeLists.txt b/flang/tools/flang-tidy/CMakeLists.txt
index 0362281046be1..4e485cd25c46e 100644
--- a/flang/tools/flang-tidy/CMakeLists.txt
+++ b/flang/tools/flang-tidy/CMakeLists.txt
@@ -1 +1,28 @@
+set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  Support
+)
+
+add_flang_library(flangTidy STATIC
+  FlangTidy.cpp
+  FlangTidyCheck.cpp
+  FlangTidyModule.cpp
+  FlangTidyOptions.cpp
+)
+
+target_link_libraries(flangTidy
+  PRIVATE
+  FortranSupport
+  FortranEvaluate
+  FortranParser
+  FortranSemantics
+  flangFrontend
+  flangFrontendTool
+  )
+
+set(ALL_FLANG_TIDY_CHECKS
+  )
+
+set(ALL_FLANG_TIDY_CHECKS ${ALL_FLANG_TIDY_CHECKS} PARENT_SCOPE)
+
 add_subdirectory(tool)
diff --git a/flang/tools/flang-tidy/FlangTidy.cpp b/flang/tools/flang-tidy/FlangTidy.cpp
new file mode 100644
index 0000000000000..d81314538c6e4
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidy.cpp
@@ -0,0 +1,297 @@
+//===--- FlangTidy.cpp - flang-tidy ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FlangTidy.h"
+#include "FlangTidyContext.h"
+#include "FlangTidyModule.h"
+#include "FlangTidyModuleRegistry.h"
+#include "MultiplexVisitor.h"
+#include "flang/Frontend/CompilerInstance.h"
+#include "flang/Frontend/CompilerInvocation.h"
+#include "flang/Frontend/TextDiagnosticPrinter.h"
+#include "flang/FrontendTool/Utils.h"
+#include "flang/Parser/parsing.h"
+#include "flang/Parser/provenance.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Option/Arg.h"
+#include "llvm/Option/ArgList.h"
+#include "llvm/Option/OptTable.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
+#include <clang/Basic/DiagnosticOptions.h>
+#include <memory>
+
+LLVM_INSTANTIATE_REGISTRY(Fortran::tidy::FlangTidyModuleRegistry)
+
+namespace Fortran::tidy {
+
+// Factory to populate MultiplexVisitor with all registered checks
+class MultiplexVisitorFactory {
+public:
+  MultiplexVisitorFactory();
+
+public:
+  std::unique_ptr<FlangTidyCheckFactories> CheckFactories;
+};
+
+MultiplexVisitorFactory::MultiplexVisitorFactory()
+    : CheckFactories(new FlangTidyCheckFactories) {
+  // Traverse the FlangTidyModuleRegistry to register all checks
+  for (auto &entry : FlangTidyModuleRegistry::entries()) {
+    // Instantiate the module
+    std::unique_ptr<FlangTidyModule> module = entry.instantiate();
+    module->addCheckFactories(*CheckFactories);
+  }
+}
+
+static std::string extractCheckName(const std::string &message) {
+  size_t openBracket = message.rfind('[');
+  size_t closeBracket = message.rfind(']');
+
+  if (openBracket != std::string::npos && closeBracket != std::string::npos &&
+      openBracket < closeBracket && closeBracket == message.length() - 1) {
+    return message.substr(openBracket + 1, closeBracket - openBracket - 1);
+  }
+
+  return "";
+}
+
+static bool shouldSuppressWarning(const parser::ProvenanceRange &source,
+                                  const std::string &checkName,
+                                  FlangTidyContext *context) {
+  if (source.empty()) {
+    return false;
+  }
+
+  const auto &cookedSources = context->getSemanticsContext().allCookedSources();
+  const auto &allSources = cookedSources.allSources();
+
+  auto srcPosition = allSources.GetSourcePosition(source.start());
+  if (!srcPosition) {
+    return false;
+  }
+
+  int lineNum = srcPosition->line;
+
+  // Get the source file and line
+  std::size_t offset;
+  const auto *sourceFile = allSources.GetSourceFile(source.start(), &offset);
+  if (!sourceFile) {
+    return false;
+  }
+
+  // Extract the line content from the source file
+  auto checkForNoLint = [&](llvm::StringRef line) -> bool {
+    // Look for ! NOLINT or !NOLINT
+    auto commentPos = line.find('!');
+    if (commentPos == llvm::StringRef::npos) {
+      return false;
+    }
+
+    auto comment = line.substr(commentPos);
+
+    // Check for NOLINT
+    if (comment.contains("NOLINT") || comment.contains("nolint")) {
+      // If there's no specific check mentioned, it applies to all checks
+      if (!comment.contains("(")) {
+        return true;
+      }
+
+      // Check for specific check name in parentheses
+      if (comment.contains("(" + checkName + ")")) {
+        return true;
+      }
+    }
+    return false;
+  };
+
+  // Try to get the current line
+  const auto &content = sourceFile->content();
+  std::size_t lineStart = 0;
+  std::size_t lineEnd = 0;
+  int currentLine = 1;
+
+  // Find the start of the current line
+  for (std::size_t i = 0; i < content.size(); i++) {
+    if (currentLine == lineNum) {
+      lineStart = i;
+
+      // Find the end of the line
+      while (i < content.size() && content[i] != '\n') {
+        i++;
+      }
+      lineEnd = i;
+
+      // Check if this line has a NOLINT comment
+      llvm::StringRef line(content.data() + lineStart, lineEnd - lineStart);
+      if (checkForNoLint(line)) {
+        return true;
+      }
+
+      // Check previous line if we can
+      if (lineNum > 1 && lineStart > 0) {
+        // Find the start of the previous line
+        std::size_t prevLineEnd = lineStart - 1;
+        std::size_t prevLineStart = prevLineEnd;
+
+        // Go back to find the start of the previous line
+        while (prevLineStart > 0 && content[prevLineStart - 1] != '\n') {
+          prevLineStart--;
+        }
+
+        llvm::StringRef prevLine(content.data() + prevLineStart,
+                                 prevLineEnd - prevLineStart);
+        if (checkForNoLint(prevLine)) {
+          return true;
+        }
+      }
+
+      // We've checked the current and previous lines, so we're done
+      break;
+    }
+
+    if (content[i] == '\n') {
+      currentLine++;
+    }
+  }
+
+  return false;
+}
+
+static void filterMessagesWithNoLint(parser::Messages &messages,
+                                     FlangTidyContext *context) {
+  std::vector<parser::Message> filteredMessages;
+
+  const auto &cookedSources = context->getSemanticsContext().allCookedSources();
+
+  for (const auto &message : messages.messages()) {
+    // Extract the check name from the message text
+    std::string checkName = extractCheckName(message.ToString());
+
+    // Skip messages that should be suppressed
+    const auto &provenanceRange = message.GetProvenanceRange(cookedSources);
+
+    // get the charBlock
+    if (!checkName.empty() && provenanceRange &&
+        shouldSuppressWarning(provenanceRange.value(), checkName, context)) {
+      continue;
+    }
+
+    // Keep messages that aren't suppressed
+    filteredMessages.push_back(message);
+  }
+
+  // Replace the original messages with the filtered ones
+  messages.clear();
+  for (const auto &msg : filteredMessages) {
+    messages.Say(msg);
+  }
+}
+
+int runFlangTidy(const FlangTidyOptions &options) {
+  auto flang = std::make_unique<Fortran::frontend::CompilerInstance>();
+
+  // create diagnostics engine
+  flang->createDiagnostics();
+  if (!flang->hasDiagnostics()) {
+    llvm::errs() << "Failed to create diagnostics engine\n";
+    return 1;
+  }
+
+  // create diagnostics buffer
+  auto diagsPrinter =
+      std::make_unique<Fortran::frontend::TextDiagnosticPrinter>(
+          llvm::outs(), flang->getDiagnostics().getDiagnosticOptions());
+
+  flang->getDiagnostics().setClient(diagsPrinter.get(), false);
+
+  // convert the options to a format that can be passed to the compiler
+  // invocation
+  std::vector<const char *> cstrArgs;
+
+  // add input files
+  for (const auto &sourcePath : options.sourcePaths) {
+    cstrArgs.push_back(sourcePath.c_str());
+  }
+
+  // add extra args before (if present)
+  if (options.ExtraArgsBefore) {
+    for (const auto &arg : *options.ExtraArgsBefore) {
+      cstrArgs.push_back(arg.c_str());
+    }
+  }
+
+  // add extra args (if present)
+  if (options.ExtraArgs) {
+    for (const auto &arg : *options.ExtraArgs) {
+      cstrArgs.push_back(arg.c_str());
+    }
+  }
+
+  llvm::ArrayRef<const char *> argv(cstrArgs);
+  flang->getFrontendOpts().programAction = frontend::ParseSyntaxOnly;
+
+  // parse arguments
+  bool success = Fortran::frontend::CompilerInvocation::createFromArgs(
+      flang->getInvocation(), argv, flang->getDiagnostics(), options.argv0);
+
+  // initialize targets
+  llvm::InitializeAllTargets();
+  llvm::InitializeAllTargetMCs();
+  llvm::InitializeAllAsmPrinters();
+
+  // emit diagnostics
+  if (!success) {
+    llvm::errs() << "Failed to parse arguments\n";
+    return 1;
+  }
+
+  // run the compiler instance
+  if (!Fortran::frontend::executeCompilerInvocation(flang.get())) {
+    return 1;
+  }
+
+  // handle help and version flags
+  if (flang->getFrontendOpts().showHelp ||
+      flang->getFrontendOpts().showVersion) {
+    return 0;
+  }
+
+  // get the parse tree
+  auto &parsing = flang->getParsing();
+  auto &parseTree = parsing.parseTree();
+  if (!parseTree) {
+    return 1;
+  }
+
+  auto &semantics = flang->getSemantics();
+  auto &semanticsContext = semantics.context();
+
+  FlangTidyContext context{options, &semanticsContext};
+
+  MultiplexVisitorFactory visitorFactory{};
+  MultiplexVisitor visitor{semanticsContext};
+  auto checks = visitorFactory.CheckFactories->createChecks(&context);
+
+  for (auto &&check : checks) {
+    visitor.AddChecker(std::move(check));
+  }
+
+  visitor.Walk(*parseTree);
+
+  auto &messages = context.Context->messages();
+  filterMessagesWithNoLint(messages, &context);
+
+  semantics.EmitMessages(llvm::outs());
+
+  return 0;
+}
+
+} // namespace Fortran::tidy
diff --git a/flang/tools/flang-tidy/FlangTidy.h b/flang/tools/flang-tidy/FlangTidy.h
new file mode 100644
index 0000000000000..db65d430f45b6
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidy.h
@@ -0,0 +1,20 @@
+//===--- FlangTidy.h - flang-tidy -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDY_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDY_H
+
+#include "FlangTidyOptions.h"
+
+namespace Fortran::tidy {
+
+int runFlangTidy(const FlangTidyOptions &options);
+
+} // namespace Fortran::tidy
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDY_H
diff --git a/flang/tools/flang-tidy/FlangTidyCheck.cpp b/flang/tools/flang-tidy/FlangTidyCheck.cpp
new file mode 100644
index 0000000000000..a3354b4cc1fcd
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyCheck.cpp
@@ -0,0 +1,215 @@
+//===--- FlangTidyCheck.cpp - flang-tidy ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FlangTidyCheck.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/YAMLParser.h"
+#include <optional>
+
+namespace Fortran::tidy {
+
+FlangTidyCheck::FlangTidyCheck(llvm::StringRef CheckName,
+                               FlangTidyContext *Context)
+    : Options(CheckName, Context->getOptions().CheckOptions, Context),
+      name_(CheckName), context_(Context) {}
+
+FlangTidyCheck::OptionsView::OptionsView(
+    llvm::StringRef CheckName, const FlangTidyOptions::OptionMap &CheckOptions,
+    FlangTidyContext *Context)
+    : NamePrefix((CheckName + ".").str()), CheckOptions(CheckOptions),
+      Context(Context) {}
+
+std::optional<llvm::StringRef>
+FlangTidyCheck::OptionsView::get(llvm::StringRef LocalName) const {
+  const auto &Iter = CheckOptions.find((NamePrefix + LocalName).str());
+  if (Iter != CheckOptions.end())
+    return llvm::StringRef(Iter->getValue().Value);
+  return std::nullopt;
+}
+
+static const llvm::StringSet<> DeprecatedGlobalOptions{
+    "StrictMode",
+    "IgnoreMacros",
+};
+
+static FlangTidyOptions::OptionMap::const_iterator
+findPriorityOption(const FlangTidyOptions::OptionMap &Options,
+                   llvm::StringRef NamePrefix, llvm::StringRef LocalName,
+                   FlangTidyContext *Context) {
+  auto IterLocal = Options.find((NamePrefix + LocalName).str());
+  auto IterGlobal = Options.find(LocalName);
+
+  // FIXME: temporary solution for deprecation warnings, should be removed
+  // after migration. Warn configuration deps on deprecation global options.
+  if (IterLocal == Options.end() && IterGlobal != Options.end() &&
+      DeprecatedGlobalOptions.contains(LocalName)) {
+    // Could add deprecation warning here if needed
+  }
+
+  if (IterLocal == Options.end())
+    return IterGlobal;
+  if (IterGlobal == Options.end())
+    return IterLocal;
+  if (IterLocal->getValue().Priority >= IterGlobal->getValue().Priority)
+    return IterLocal;
+  return IterGlobal;
+}
+
+std::optional<llvm::StringRef>
+FlangTidyCheck::OptionsView::getLocalOrGlobal(llvm::StringRef LocalName) const {
+  auto Iter = findPriorityOption(CheckOptions, NamePrefix, LocalName, Context);
+  if (Iter != CheckOptions.end())
+    return llvm::StringRef(Iter->getValue().Value);
+  return std::nullopt;
+}
+
+static std::optional<bool> getAsBool(llvm::StringRef Value,
+                                     const llvm::Twine &LookupName) {
+  if (std::optional<bool> Parsed = llvm::yaml::parseBool(Value))
+    return Parsed;
+  // To maintain backwards compatability, we support parsing numbers as
+  // booleans, even though its not supported in YAML.
+  long long Number = 0;
+  if (!Value.getAsInteger(10, Number))
+    return Number != 0;
+  return std::nullopt;
+}
+
+template <>
+std::optional<bool>
+FlangTidyCheck::OptionsView::get<bool>(llvm::StringRef LocalName) const {
+  if (std::optional<llvm::StringRef> ValueOr = get(LocalName)) {
+    if (auto Result = getAsBool(*ValueOr, NamePrefix + LocalName))
+      return Result;
+    diagnoseBadBooleanOption(NamePrefix + LocalName, *ValueOr);
+  }
+  return std::nullopt;
+}
+
+template <>
+std::optional<bool> FlangTidyCheck::OptionsView::getLocalOrGlobal<bool>(
+    llvm::StringRef LocalName) const {
+  auto Iter = findPriorityOption(CheckOptions, NamePrefix, LocalName, Context);
+  if (Iter != CheckOptions.end()) {
+    if (auto Result = getAsBool(Iter->getValue().Value, Iter->getKey()))
+      return Result;
+    diagnoseBadBooleanOption(Iter->getKey(), Iter->getValue().Value);
+  }
+  return std::nullopt;
+}
+
+void FlangTidyCheck::OptionsView::store(FlangTidyOptions::OptionMap &Options,
+                                        llvm::StringRef LocalName,
+                                        llvm::StringRef Value) const {
+  Options[(NamePrefix + LocalName).str()] =
+      FlangTidyOptions::FlangTidyValue(Value);
+}
+
+void FlangTidyCheck::OptionsView::storeInt(FlangTidyOptions::OptionMap &Options,
+                                           llvm::StringRef LocalName,
+                                           int64_t Value) const {
+  store(Options, LocalName, llvm::itostr(Value));
+}
+
+void FlangTidyCheck::OptionsView::storeUnsigned(
+    FlangTidyOptions::OptionMap &Options, llvm::StringRef LocalName,
+    uint64_t Value) const {
+  store(Options, LocalName, llvm::utostr(Value));
+}
+
+template <>
+void FlangTidyCheck::OptionsView::store<bool>(
+    FlangTidyOptions::OptionMap &Options, llvm::StringRef LocalName,
+    bool Value) const {
+  store(Options, LocalName,
+        Value ? llvm::StringRef("true") : llvm::StringRef("false"));
+}
+
+std::optional<int64_t>
+FlangTidyCheck::OptionsView::getEnumInt(llvm::StringRef LocalName,
+                                        llvm::ArrayRef<NameAndValue> Mapping,
+                                        bool CheckGlobal) const {
+  auto Iter = CheckGlobal ? findPriorityOption(CheckOptions, NamePrefix,
+                                               LocalName, Context)
+                          : CheckOptions.find((NamePrefix + LocalName).str());
+  if (Iter == CheckOptions.end())
+    return std::nullopt;
+
+  llvm::StringRef Value = Iter->getValue().Value;
+  llvm::StringRef Closest;
+  unsigned EditDistance = 3;
+  for (const auto &NameAndEnum : Mapping) {
+    if (Value == NameAndEnum.second) {
+      return NameAndEnum.first;
+    }
+    if (Value.equals_insensitive(NameAndEnum.second)) {
+      Closest = NameAndEnum.second;
+      EditDistance = 0;
+      continue;
+    }
+    unsigned Distance =
+        Value.edit_distance(NameAndEnum.second, true, EditDistance);
+    if (Distance < EditDistance) {
+      EditDistance = Distance;
+      Closest = NameAndEnum.second;
+    }
+  }
+  if (EditDistance < 3)
+    diagnoseBadEnumOption(Iter->getKey(), Iter->getValue().Value, Closest);
+  else
+    diagnoseBadEnumOption(Iter->getKey(), Iter->getValue().Value);
+  return std::nullopt;
+}
+
+/*
+static constexpr llvm::StringLiteral ConfigWarning(
+    "invalid configuration value '%0' for option '%1'%select{|; expected a "
+    "bool|; expected an integer|; did you mean '%3'?}2");
+ */
+void FlangTidyCheck::OptionsView::diagnoseBadBooleanOption(
+    const llvm::Twine &Lookup, llvm::StringRef Unparsed) const {
+  llvm::SmallString<64> Buffer;
+  // For now, just use a simple error message since we don't have the diagnostic
+  // infrastructure
+  llvm::errs() << "Error: invalid boolean value '" << Unparsed
+               << "' for option '" << Lookup.toStringRef(Buffer) << "'\n";
+}
+
+void FlangTidyCheck::OptionsView::diagnoseBadIntegerOption(
+    const llvm::Twine &Lookup, llvm::StringRef Unparsed) const {
+  llvm::SmallString<64> Buffer;
+  llvm::errs() << "Error: invalid integer value '" << Unparsed
+               << "' for option '" << Lookup.toStringRef(Buffer) << "'\n";
+}
+
+void FlangTidyCheck::OptionsView::diagnoseBadEnumOption(
+    const llvm::Twine &Lookup, llvm::StringRef Unparsed,
+    llvm::StringRef Suggestion) const {
+  llvm::SmallString<64> Buffer;
+  llvm::errs() << "Error: invalid enum value '" << Unparsed << "' for option '"
+               << Lookup.toStringRef(Buffer) << "'";
+  if (!Suggestion.empty())
+    llvm::errs() << "; did you mean '" << Suggestion << "'?";
+  llvm::errs() << "\n";
+}
+
+llvm::StringRef
+FlangTidyCheck::OptionsView::get(llvm::StringRef LocalName,
+                                 llvm::StringRef Default) const {
+  return get(LocalName).value_or(Default);
+}
+
+llvm::StringRef
+FlangTidyCheck::OptionsView::getLocalOrGlobal(llvm::StringRef LocalName,
+                                              llvm::StringRef Default) const {
+  return getLocalOrGlobal(LocalName).value_or(Default);
+}
+
+} // namespace Fortran::tidy
diff --git a/flang/tools/flang-tidy/FlangTidyCheck.h b/flang/tools/flang-tidy/FlangTidyCheck.h
new file mode 100644
index 0000000000000..6d5e248b6c07f
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyCheck.h
@@ -0,0 +1,542 @@
+//===--- FlangTidyCheck.h - flang-tidy --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYCHECK_H
+
+#include "FlangTidyContext.h"
+#include "FlangTidyOptions.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/semantics.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include <optional>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace Fortran::tidy {
+
+/// This class should be specialized by any enum type that needs to be converted
+/// to and from an \ref llvm::StringRef.
+template <class T>
+struct OptionEnumMapping {
+  // Specializations of this struct must implement this function.
+  static llvm::ArrayRef<std::pair<T, llvm::StringRef>>
+  getEnumMapping() = delete;
+};
+
+/// This is the base class for all Flang Tidy checks. It provides a
+/// common interface for all checks and allows them to interact with
+/// the Flang Tidy context. Each check should inherit from this
+/// class and implement the necessary methods to perform its
+/// specific analysis.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/flang-tidy-checks.html
+class FlangTidyCheck : public semantics::BaseChecker {
+public:
+  FlangTidyCheck(llvm::StringRef name, FlangTidyContext *context);
+  virtual ~FlangTidyCheck() = default;
+  llvm::StringRef name() const { return name_; }
+  FlangTidyContext *context() { return context_; }
+  bool fixAvailable() const { return fixAvailable_; }
+
+  using semantics::BaseChecker::Enter;
+  using semantics::BaseChecker::Leave;
+
+  /// Should store all options supported by this check with their
+  /// current values or default values for options that haven't been overridden.
+  ///
+  /// The check should use ``Options.store()`` to store each option it supports
+  /// whether it has the default value or it has been overridden.
+  virtual void storeOptions(FlangTidyOptions::OptionMap &Options) {}
+
+  /// Provides access to the ``FlangTidyCheck`` options via check-local
+  /// names.
+  ///
+  /// Methods of this class prepend ``CheckName + "."`` to translate check-local
+  /// option names to global option names.
+  class OptionsView {
+    void diagnoseBadIntegerOption(const llvm::Twine &Lookup,
+                                  llvm::StringRef Unparsed) const;
+    void diagnoseBadBooleanOption(const llvm::Twine &Lookup,
+                                  llvm::StringRef Unparsed) const;
+    void
+    diagnoseBadEnumOption(const llvm::Twine &Lookup, llvm::StringRef Unparsed,
+                          llvm::StringRef Suggestion = llvm::StringRef()) const;
+
+  public:
+    /// Initializes the instance using \p CheckName + "." as a prefix.
+    OptionsView(llvm::StringRef CheckName,
+                const FlangTidyOptions::OptionMap &CheckOptions,
+                FlangTidyContext *Context);
+
+    /// Read a named option from the ``Context``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from the
+    /// ``CheckOptions``. If the corresponding key is not present, return
+    /// ``std::nullopt``.
+    std::optional<llvm::StringRef> get(llvm::StringRef LocalName) const;
+
+    /// Read a named option from the ``Context``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from the
+    /// ``CheckOptions``. If the corresponding key is not present, returns
+    /// \p Default.
+    llvm::StringRef get(llvm::StringRef LocalName,
+                        llvm::StringRef Default) const;
+
+    /// Read a named option from the ``Context``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from local or
+    /// global ``CheckOptions``. Gets local option first. If local is not
+    /// present, falls back to get global option. If global option is not
+    /// present either, return ``std::nullopt``.
+    std::optional<llvm::StringRef>
+    getLocalOrGlobal(llvm::StringRef LocalName) const;
+
+    /// Read a named option from the ``Context``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from local or
+    /// global ``CheckOptions``. Gets local option first. If local is not
+    /// present, falls back to get global option. If global option is not
+    /// present either, returns \p Default.
+    llvm::StringRef getLocalOrGlobal(llvm::StringRef LocalName,
+                                     llvm::StringRef Default) const;
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// integral type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from the
+    /// ``CheckOptions``. If the corresponding key is not present,
+    ///  return ``std::nullopt``.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return ``std::nullopt``.
+    template <typename T>
+    std::enable_if_t<std::is_integral_v<T>, std::optional<T>>
+    get(llvm::StringRef LocalName) const {
+      if (std::optional<llvm::StringRef> Value = get(LocalName)) {
+        T Result{};
+        if (!llvm::StringRef(*Value).getAsInteger(10, Result))
+          return Result;
+        diagnoseBadIntegerOption(NamePrefix + LocalName, *Value);
+      }
+      return std::nullopt;
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// integral type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from the
+    /// ``CheckOptions``. If the corresponding key is `none`, `null`,
+    /// `-1` or empty, return ``std::nullopt``. If the corresponding
+    /// key is not present, return \p Default.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return \p Default.
+    template <typename T>
+    std::enable_if_t<std::is_integral_v<T>, std::optional<T>>
+    get(llvm::StringRef LocalName, std::optional<T> Default) const {
+      if (std::optional<llvm::StringRef> Value = get(LocalName)) {
+        if (Value == "" || Value == "none" || Value == "null" ||
+            (std::is_unsigned_v<T> && Value == "-1"))
+          return std::nullopt;
+        T Result{};
+        if (!llvm::StringRef(*Value).getAsInteger(10, Result))
+          return Result;
+        diagnoseBadIntegerOption(NamePrefix + LocalName, *Value);
+      }
+      return Default;
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// integral type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from the
+    /// ``CheckOptions``. If the corresponding key is not present, return
+    /// \p Default.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return \p Default.
+    template <typename T>
+    std::enable_if_t<std::is_integral_v<T>, T> get(llvm::StringRef LocalName,
+                                                   T Default) const {
+      return get<T>(LocalName).value_or(Default);
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// integral type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from local or
+    /// global ``CheckOptions``. Gets local option first. If local is not
+    /// present, falls back to get global option. If global option is not
+    /// present either, return ``std::nullopt``.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return ``std::nullopt``.
+    template <typename T>
+    std::enable_if_t<std::is_integral_v<T>, std::optional<T>>
+    getLocalOrGlobal(llvm::StringRef LocalName) const {
+      std::optional<llvm::StringRef> ValueOr = get(LocalName);
+      bool IsGlobal = false;
+      if (!ValueOr) {
+        IsGlobal = true;
+        ValueOr = getLocalOrGlobal(LocalName);
+        if (!ValueOr)
+          return std::nullopt;
+      }
+      T Result{};
+      if (!llvm::StringRef(*ValueOr).getAsInteger(10, Result))
+        return Result;
+      diagnoseBadIntegerOption(
+          IsGlobal ? llvm::Twine(LocalName) : NamePrefix + LocalName, *ValueOr);
+      return std::nullopt;
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// integral type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from local or
+    /// global ``CheckOptions``. Gets local option first. If local is not
+    /// present, falls back to get global option. If global option is not
+    /// present either, return \p Default. If the value value was found
+    /// and equals ``none``, ``null``, ``-1`` or empty, return ``std::nullopt``.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return \p Default.
+    template <typename T>
+    std::enable_if_t<std::is_integral_v<T>, std::optional<T>>
+    getLocalOrGlobal(llvm::StringRef LocalName,
+                     std::optional<T> Default) const {
+      std::optional<llvm::StringRef> ValueOr = get(LocalName);
+      bool IsGlobal = false;
+      if (!ValueOr) {
+        IsGlobal = true;
+        ValueOr = getLocalOrGlobal(LocalName);
+        if (!ValueOr)
+          return Default;
+      }
+      T Result{};
+      if (ValueOr == "" || ValueOr == "none" || ValueOr == "null" ||
+          (std::is_unsigned_v<T> && ValueOr == "-1"))
+        return std::nullopt;
+      if (!llvm::StringRef(*ValueOr).getAsInteger(10, Result))
+        return Result;
+      diagnoseBadIntegerOption(
+          IsGlobal ? llvm::Twine(LocalName) : NamePrefix + LocalName, *ValueOr);
+      return Default;
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// integral type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from local or
+    /// global ``CheckOptions``. Gets local option first. If local is not
+    /// present, falls back to get global option. If global option is not
+    /// present either, return \p Default.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return \p Default.
+    template <typename T>
+    std::enable_if_t<std::is_integral_v<T>, T>
+    getLocalOrGlobal(llvm::StringRef LocalName, T Default) const {
+      return getLocalOrGlobal<T>(LocalName).value_or(Default);
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// enum type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from the
+    /// ``CheckOptions``. If the corresponding key is not present, return
+    /// ``std::nullopt``.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return ``std::nullopt``.
+    ///
+    /// \ref Fortran::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
+    template <typename T>
+    std::enable_if_t<std::is_enum_v<T>, std::optional<T>>
+    get(llvm::StringRef LocalName) const {
+      if (std::optional<int64_t> ValueOr =
+              getEnumInt(LocalName, typeEraseMapping<T>(), false))
+        return static_cast<T>(*ValueOr);
+      return std::nullopt;
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// enum type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from the
+    /// ``CheckOptions``. If the corresponding key is not present,
+    /// return \p Default.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return \p Default.
+    ///
+    /// \ref Fortran::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
+    template <typename T>
+    std::enable_if_t<std::is_enum_v<T>, T> get(llvm::StringRef LocalName,
+                                               T Default) const {
+      return get<T>(LocalName).value_or(Default);
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// enum type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from local or
+    /// global ``CheckOptions``. Gets local option first. If local is not
+    /// present, falls back to get global option. If global option is not
+    /// present either, returns ``std::nullopt``.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return ``std::nullopt``.
+    ///
+    /// \ref Fortran::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
+    template <typename T>
+    std::enable_if_t<std::is_enum_v<T>, std::optional<T>>
+    getLocalOrGlobal(llvm::StringRef LocalName) const {
+      if (std::optional<int64_t> ValueOr =
+              getEnumInt(LocalName, typeEraseMapping<T>(), true))
+        return static_cast<T>(*ValueOr);
+      return std::nullopt;
+    }
+
+    /// Read a named option from the ``Context`` and parse it as an
+    /// enum type ``T``.
+    ///
+    /// Reads the option with the check-local name \p LocalName from local or
+    /// global ``CheckOptions``. Gets local option first. If local is not
+    /// present, falls back to get global option. If global option is not
+    /// present either return \p Default.
+    ///
+    /// If the corresponding key can't be parsed as a ``T``, emit a
+    /// diagnostic and return \p Default.
+    ///
+    /// \ref Fortran::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
+    template <typename T>
+    std::enable_if_t<std::is_enum_v<T>, T>
+    getLocalOrGlobal(llvm::StringRef LocalName, T Default) const {
+      return getLocalOrGlobal<T>(LocalName).value_or(Default);
+    }
+
+    /// Stores an option with the check-local name \p LocalName with
+    /// string value \p Value to \p Options.
+    void store(FlangTidyOptions::OptionMap &Options, llvm::StringRef LocalName,
+               llvm::StringRef Value) const;
+
+    /// Stores an option with the check-local name \p LocalName with
+    /// integer value \p Value to \p Options.
+    template <typename T>
+    std::enable_if_t<std::is_integral_v<T>>
+    store(FlangTidyOptions::OptionMap &Options, llvm::StringRef LocalName,
+          T Value) const {
+      if constexpr (std::is_signed_v<T>)
+        storeInt(Options, LocalName, Value);
+      else
+        storeUnsigned(Options, LocalName, Value);
+    }
+
+    /// Stores an option with the check-local name \p LocalName with
+    /// integer value \p Value to \p Options. If the value is empty
+    /// stores "none"
+    template <typename T>
+    std::enable_if_t<std::is_integral_v<T>>
+    store(FlangTidyOptions::OptionMap &Options, llvm::StringRef LocalName,
+          std::optional<T> Value) const {
+      if (Value)
+        store(Options, LocalName, *Value);
+      else
+        store(Options, LocalName, "none");
+    }
+
+    /// Stores an option with the check-local name \p LocalName as the string
+    /// representation of the Enum \p Value to \p Options.
+    ///
+    /// \ref Fortran::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
+    template <typename T>
+    std::enable_if_t<std::is_enum_v<T>>
+    store(FlangTidyOptions::OptionMap &Options, llvm::StringRef LocalName,
+          T Value) const {
+      llvm::ArrayRef<std::pair<T, llvm::StringRef>> Mapping =
+          OptionEnumMapping<T>::getEnumMapping();
+      auto Iter = llvm::find_if(
+          Mapping, [&](const std::pair<T, llvm::StringRef> &NameAndEnum) {
+            return NameAndEnum.first == Value;
+          });
+      assert(Iter != Mapping.end() && "Unknown Case Value");
+      store(Options, LocalName, Iter->second);
+    }
+
+  private:
+    using NameAndValue = std::pair<int64_t, llvm::StringRef>;
+
+    std::optional<int64_t> getEnumInt(llvm::StringRef LocalName,
+                                      llvm::ArrayRef<NameAndValue> Mapping,
+                                      bool CheckGlobal) const;
+
+    template <typename T>
+    std::enable_if_t<std::is_enum_v<T>, std::vector<NameAndValue>>
+    typeEraseMapping() const {
+      llvm::ArrayRef<std::pair<T, llvm::StringRef>> Mapping =
+          OptionEnumMapping<T>::getEnumMapping();
+      std::vector<NameAndValue> Result;
+      Result.reserve(Mapping.size());
+      for (auto &MappedItem : Mapping) {
+        Result.emplace_back(static_cast<int64_t>(MappedItem.first),
+                            MappedItem.second);
+      }
+      return Result;
+    }
+
+    void storeInt(FlangTidyOptions::OptionMap &Options,
+                  llvm::StringRef LocalName, int64_t Value) const;
+
+    void storeUnsigned(FlangTidyOptions::OptionMap &Options,
+                       llvm::StringRef LocalName, uint64_t Value) const;
+
+    std::string NamePrefix;
+    const FlangTidyOptions::OptionMap &CheckOptions;
+    FlangTidyContext *Context;
+  };
+
+  virtual void Enter(const parser::AcImpliedDo &) {}
+  virtual void Enter(const parser::ArithmeticIfStmt &) {}
+  virtual void Enter(const parser::AssignedGotoStmt &) {}
+  virtual void Enter(const parser::AssignmentStmt &) {}
+  virtual void Enter(const parser::AssignStmt &) {}
+  virtual void Enter(const parser::AssociateConstruct &) {}
+  virtual void Enter(const parser::BackspaceStmt &) {}
+  virtual void Enter(const parser::Block &) {}
+  virtual void Enter(const parser::CallStmt &) {}
+  virtual void Enter(const parser::CaseConstruct &) {}
+  virtual void Enter(const parser::CommonStmt &) {}
+  virtual void Enter(const parser::ComputedGotoStmt &) {}
+  virtual void Enter(const parser::DataImpliedDo &) {}
+  virtual void Enter(const parser::DataStmt &) {}
+  virtual void Enter(const parser::DoConstruct &) {}
+  virtual void Enter(const parser::EntityDecl &) {}
+  virtual void Enter(const parser::ExecutableConstruct &) {}
+  virtual void Enter(const parser::Expr &) {}
+  virtual void Enter(const parser::Expr::AND &) {}
+  virtual void Enter(const parser::Expr::OR &) {}
+  virtual void Enter(const parser::ForallConstruct &) {}
+  virtual void Enter(const parser::ForallStmt &) {}
+  virtual void Enter(const parser::FunctionSubprogram &) {}
+  virtual void Enter(const parser::GotoStmt &) {}
+  virtual void Enter(const parser::IfConstruct &) {}
+  virtual void Enter(const parser::InputImpliedDo &) {}
+  virtual void Enter(const parser::OmpAtomicUpdate &) {}
+  virtual void Enter(const parser::OpenMPBlockConstruct &) {}
+  virtual void Enter(const parser::OpenMPCriticalConstruct &) {}
+  virtual void Enter(const parser::OpenMPLoopConstruct &) {}
+  virtual void Enter(const parser::OutputImpliedDo &) {}
+  virtual void Enter(const parser::PauseStmt &) {}
+  virtual void Enter(const parser::SelectRankConstruct &) {}
+  virtual void Enter(const parser::SelectTypeConstruct &) {}
+  virtual void Enter(const parser::SubroutineSubprogram &) {}
+  virtual void Leave(const parser::AllocateStmt &) {}
+  virtual void Leave(const parser::AssignmentStmt &) {}
+  virtual void Leave(const parser::BackspaceStmt &) {}
+  virtual void Leave(const parser::Block &) {}
+  virtual void Leave(const parser::CloseStmt &) {}
+  virtual void Leave(const parser::CommonStmt &) {}
+  virtual void Leave(const parser::DeallocateStmt &) {}
+  virtual void Leave(const parser::EndfileStmt &) {}
+  virtual void Leave(const parser::EventPostStmt &) {}
+  virtual void Leave(const parser::EventWaitStmt &) {}
+  virtual void Leave(const parser::FileUnitNumber &) {}
+  virtual void Leave(const parser::FlushStmt &) {}
+  virtual void Leave(const parser::FormTeamStmt &) {}
+  virtual void Leave(const parser::FunctionSubprogram &) {}
+  virtual void Leave(const parser::InquireStmt &) {}
+  virtual void Leave(const parser::LockStmt &) {}
+  virtual void Leave(const parser::Name &) {}
+  virtual void Leave(const parser::OmpAtomicUpdate &) {}
+  virtual void Leave(const parser::OpenMPBlockConstruct &) {}
+  virtual void Leave(const parser::OpenMPCriticalConstruct &) {}
+  virtual void Leave(const parser::OpenMPLoopConstruct &) {}
+  virtual void Leave(const parser::OpenStmt &) {}
+  virtual void Leave(const parser::PointerAssignmentStmt &) {}
+  virtual void Leave(const parser::PrintStmt &) {}
+  virtual void Leave(const parser::Program &) {}
+  virtual void Leave(const parser::ReadStmt &) {}
+  virtual void Leave(const parser::RewindStmt &) {}
+  virtual void Leave(const parser::SubroutineSubprogram &) {}
+  virtual void Leave(const parser::SyncAllStmt &) {}
+  virtual void Leave(const parser::SyncImagesStmt &) {}
+  virtual void Leave(const parser::SyncMemoryStmt &) {}
+  virtual void Leave(const parser::SyncTeamStmt &) {}
+  virtual void Leave(const parser::UnlockStmt &) {}
+  virtual void Leave(const parser::UseStmt &) {}
+  virtual void Leave(const parser::WaitStmt &) {}
+  virtual void Leave(const parser::WriteStmt &) {}
+
+  template <typename... Args>
+  parser::Message &Say(parser::CharBlock at, parser::MessageFixedText &&message,
+                       Args &&...args) {
+    // construct a new fixedTextMessage
+    std::string str{message.text().ToString()};
+    str.append(" [%s]");
+    parser::MessageFixedText newMessage{str.c_str(), str.length(),
+                                        message.severity()};
+    return context_->getSemanticsContext().Say(
+        at, std::move(newMessage), std::forward<Args>(args)..., name_.str());
+  }
+
+protected:
+  OptionsView Options;
+
+private:
+  bool fixAvailable_{false};
+  llvm::StringRef name_;
+  FlangTidyContext *context_;
+};
+
+/// Read a named option from the ``Context`` and parse it as a bool.
+///
+/// Reads the option with the check-local name \p LocalName from the
+/// ``CheckOptions``. If the corresponding key is not present, return
+/// ``std::nullopt``.
+///
+/// If the corresponding key can't be parsed as a bool, emit a
+/// diagnostic and return ``std::nullopt``.
+template <>
+std::optional<bool>
+FlangTidyCheck::OptionsView::get<bool>(llvm::StringRef LocalName) const;
+
+/// Read a named option from the ``Context`` and parse it as a bool.
+///
+/// Reads the option with the check-local name \p LocalName from the
+/// ``CheckOptions``. If the corresponding key is not present, return
+/// \p Default.
+///
+/// If the corresponding key can't be parsed as a bool, emit a
+/// diagnostic and return \p Default.
+template <>
+std::optional<bool> FlangTidyCheck::OptionsView::getLocalOrGlobal<bool>(
+    llvm::StringRef LocalName) const;
+
+/// Stores an option with the check-local name \p LocalName with
+/// bool value \p Value to \p Options.
+template <>
+void FlangTidyCheck::OptionsView::store<bool>(
+    FlangTidyOptions::OptionMap &Options, llvm::StringRef LocalName,
+    bool Value) const;
+
+} // namespace Fortran::tidy
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYCHECK_H
diff --git a/flang/tools/flang-tidy/FlangTidyContext.h b/flang/tools/flang-tidy/FlangTidyContext.h
new file mode 100644
index 0000000000000..8158ccdce6d7f
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyContext.h
@@ -0,0 +1,78 @@
+//===--- FlangTidyContext.h - flang-tidy ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYCONTEXT_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYCONTEXT_H
+
+#include "FlangTidyOptions.h"
+#include "flang/Semantics/semantics.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/StringRef.h"
+#include <clang/Basic/Diagnostic.h>
+
+namespace Fortran::tidy {
+
+/// This class is used to manage the context for Flang Tidy checks.
+/// It contains the enabled checks and the semantics context.
+/// It provides methods to check if a specific check is enabled and to access
+/// the semantics context.
+///
+/// For user-facing documentation, see:
+/// https://flang.llvm.org/@PLACEHOLDER@/flang-tidy.html
+class FlangTidyContext {
+public:
+  FlangTidyContext(const FlangTidyOptions &options,
+                   semantics::SemanticsContext *ctx) {
+    Options = options;
+    for (const auto &CheckName : options.enabledChecks) {
+      Checks.insert(CheckName);
+    }
+    Context = ctx;
+  }
+
+  bool isCheckEnabled(const llvm::StringRef &CheckName) const {
+    bool enabled = false;
+    for (const auto &Pattern : Checks) {
+      if (Pattern.starts_with("-")) {
+        llvm::StringRef DisablePrefix = Pattern.drop_front(1);
+        if (DisablePrefix.ends_with("*")) {
+          DisablePrefix = DisablePrefix.drop_back(1);
+          if (CheckName.starts_with(DisablePrefix)) {
+            enabled = false;
+          }
+        } else if (DisablePrefix == CheckName) {
+          enabled = false;
+        }
+      } else if (Pattern.ends_with("*")) {
+        llvm::StringRef EnablePrefix = Pattern.drop_back(1);
+        if (CheckName.starts_with(EnablePrefix)) {
+          enabled = true;
+        }
+      } else if (Pattern == CheckName) {
+        enabled = true;
+      }
+    }
+    return enabled;
+  }
+
+  semantics::SemanticsContext &getSemanticsContext() const { return *Context; }
+
+  /// Get the FlangTidy options
+  const FlangTidyOptions &getOptions() const { return Options; }
+
+public:
+  /// List of enabled checks.
+  llvm::SmallSet<llvm::StringRef, 16> Checks;
+  /// The semantics context used for the checks.
+  semantics::SemanticsContext *Context;
+  FlangTidyOptions Options;
+};
+
+} // namespace Fortran::tidy
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYCONTEXT_H
diff --git a/flang/tools/flang-tidy/FlangTidyForceLinker.h b/flang/tools/flang-tidy/FlangTidyForceLinker.h
new file mode 100644
index 0000000000000..3e9467013ea1c
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyForceLinker.h
@@ -0,0 +1,18 @@
+//===--- FlangTidyForceLinker.h - flang-tidy --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYFORCELINKER_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYFORCELINKER_H
+
+#include "llvm/Support/Compiler.h"
+
+namespace Fortran::tidy {
+
+} // namespace Fortran::tidy
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYFORCELINKER_H
diff --git a/flang/tools/flang-tidy/FlangTidyModule.cpp b/flang/tools/flang-tidy/FlangTidyModule.cpp
new file mode 100644
index 0000000000000..1cdb2a20f7dfb
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyModule.cpp
@@ -0,0 +1,49 @@
+//===--- FlangTidyModule.cpp - flang-tidy ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FlangTidyModule.h"
+#include "FlangTidyCheck.h"
+#include "FlangTidyContext.h"
+
+namespace Fortran::tidy {
+
+void FlangTidyCheckFactories::registerCheckFactory(llvm::StringRef Name,
+                                                   CheckFactory Factory) {
+  Factories.insert_or_assign(Name, std::move(Factory));
+}
+
+std::vector<std::unique_ptr<FlangTidyCheck>>
+FlangTidyCheckFactories::createChecks(FlangTidyContext *Context) const {
+  std::vector<std::unique_ptr<FlangTidyCheck>> Checks;
+  for (const auto &Factory : Factories) {
+    if (Context->isCheckEnabled(Factory.getKey())) {
+      Checks.emplace_back(Factory.getValue()(Factory.getKey(), Context));
+    }
+  }
+  return Checks;
+}
+
+std::vector<std::unique_ptr<FlangTidyCheck>>
+FlangTidyCheckFactories::createChecksForLanguage(
+    FlangTidyContext *Context) const {
+  std::vector<std::unique_ptr<FlangTidyCheck>> Checks;
+  // const LangOptions &LO = Context->getLangOpts();
+  for (const auto &Factory : Factories) {
+    if (!Context->isCheckEnabled(Factory.getKey()))
+      continue;
+    std::unique_ptr<FlangTidyCheck> Check =
+        Factory.getValue()(Factory.getKey(), Context);
+    // if (Check->isLanguageVersionSupported(LO))
+    //   Checks.push_back(std::move(Check));
+  }
+  return Checks;
+}
+
+FlangTidyOptions FlangTidyModule::getModuleOptions() { return {}; }
+
+} // namespace Fortran::tidy
diff --git a/flang/tools/flang-tidy/FlangTidyModule.h b/flang/tools/flang-tidy/FlangTidyModule.h
new file mode 100644
index 0000000000000..b3658767da2b2
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyModule.h
@@ -0,0 +1,99 @@
+//===--- FlangTidyModule.h - flang-tidy -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYMODULE_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYMODULE_H
+
+#include "FlangTidyOptions.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include <functional>
+#include <memory>
+
+namespace Fortran::tidy {
+
+class FlangTidyCheck;
+class FlangTidyContext;
+
+/// A collection of \c TidyCheckFactory instances.
+///
+/// All Flang-tidy modules register their check factories with an instance of
+/// this object.
+class FlangTidyCheckFactories {
+public:
+  using CheckFactory = std::function<std::unique_ptr<FlangTidyCheck>(
+      llvm::StringRef Name, FlangTidyContext *Context)>;
+
+  /// Registers check \p Factory with name \p Name.
+  ///
+  /// For all checks that have default constructors, use \c registerCheck.
+  void registerCheckFactory(llvm::StringRef Name, CheckFactory Factory);
+
+  /// Registers the \c CheckType with the name \p Name.
+  ///
+  /// This method should be used for all \c FlangTidyChecks that don't require
+  /// constructor parameters.
+  ///
+  /// For example, if have a Flang-tidy check like:
+  /// \code
+  /// class MyTidyCheck : public FlangTidyCheck {
+  ///   void registerMatchers(ast_matchers::MatchFinder *Finder) override {
+  ///     ..
+  ///   }
+  /// };
+  /// \endcode
+  /// you can register it with:
+  /// \code
+  /// class MyModule : public FlangTidyModule {
+  ///   void addCheckFactories(FlangTidyCheckFactories &Factories) override {
+  ///     Factories.registerCheck<MyTidyCheck>("myproject-my-check");
+  ///   }
+  /// };
+  /// \endcode
+  template <typename CheckType>
+  void registerCheck(llvm::StringRef CheckName) {
+    registerCheckFactory(CheckName,
+                         [](llvm::StringRef Name, FlangTidyContext *Context) {
+                           return std::make_unique<CheckType>(Name, Context);
+                         });
+  }
+
+  /// Create instances of checks that are enabled.
+  std::vector<std::unique_ptr<FlangTidyCheck>>
+  createChecks(FlangTidyContext *Context) const;
+
+  /// Create instances of checks that are enabled for the current Language.
+  std::vector<std::unique_ptr<FlangTidyCheck>>
+  createChecksForLanguage(FlangTidyContext *Context) const;
+
+  using FactoryMap = llvm::StringMap<CheckFactory>;
+  FactoryMap::const_iterator begin() const { return Factories.begin(); }
+  FactoryMap::const_iterator end() const { return Factories.end(); }
+  bool empty() const { return Factories.empty(); }
+
+private:
+  FactoryMap Factories;
+};
+
+/// A flang-tidy module groups a number of \c FlangTidyChecks and gives
+/// them a prefixed name.
+class FlangTidyModule {
+public:
+  virtual ~FlangTidyModule() {}
+
+  /// Implement this function in order to register all \c CheckFactories
+  /// belonging to this module.
+  virtual void addCheckFactories(FlangTidyCheckFactories &CheckFactories) = 0;
+
+  /// Gets default options for checks defined in this module.
+  virtual FlangTidyOptions getModuleOptions();
+};
+
+} // namespace Fortran::tidy
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYMODULE_H
diff --git a/flang/tools/flang-tidy/FlangTidyModuleRegistry.h b/flang/tools/flang-tidy/FlangTidyModuleRegistry.h
new file mode 100644
index 0000000000000..18ebf5c30ef28
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyModuleRegistry.h
@@ -0,0 +1,25 @@
+//===--- FlangTidyModuleRegistry.h - flang-tidy -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYMODULEREGISTRY_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYMODULEREGISTRY_H
+
+#include "FlangTidyModule.h"
+#include "llvm/Support/Registry.h"
+
+namespace Fortran::tidy {
+
+using FlangTidyModuleRegistry = llvm::Registry<FlangTidyModule>;
+
+} // namespace Fortran::tidy
+
+namespace llvm {
+extern template class Registry<Fortran::tidy::FlangTidyModule>;
+} // namespace llvm
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYMODULEREGISTRY_H
diff --git a/flang/tools/flang-tidy/FlangTidyOptions.cpp b/flang/tools/flang-tidy/FlangTidyOptions.cpp
new file mode 100644
index 0000000000000..aaf7dac0f19be
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyOptions.cpp
@@ -0,0 +1,420 @@
+//===--- FlangTidyOptions.cpp - flang-tidy --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FlangTidyOptions.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/MemoryBufferRef.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <algorithm>
+#include <sstream>
+
+#define DEBUG_TYPE "flang-tidy-options"
+
+using Fortran::tidy::FlangTidyOptions;
+using OptionsSource = Fortran::tidy::FlangTidyOptionsProvider::OptionsSource;
+
+namespace llvm::yaml {
+
+template <>
+struct MappingTraits<FlangTidyOptions::StringPair> {
+  static void mapping(IO &IO, FlangTidyOptions::StringPair &KeyValue) {
+    IO.mapRequired("key", KeyValue.first);
+    IO.mapRequired("value", KeyValue.second);
+  }
+};
+
+struct NOptionMap {
+  NOptionMap(IO &) {}
+  NOptionMap(IO &, const FlangTidyOptions::OptionMap &OptionMap) {
+    Options.reserve(OptionMap.size());
+    for (const auto &KeyValue : OptionMap)
+      Options.emplace_back(std::string(KeyValue.getKey()),
+                           KeyValue.getValue().Value);
+  }
+  FlangTidyOptions::OptionMap denormalize(IO &) {
+    FlangTidyOptions::OptionMap Map;
+    for (const auto &KeyValue : Options)
+      Map[KeyValue.first] = FlangTidyOptions::FlangTidyValue(KeyValue.second);
+    return Map;
+  }
+  std::vector<FlangTidyOptions::StringPair> Options;
+};
+
+template <>
+void yamlize(IO &IO, FlangTidyOptions::OptionMap &Val, bool,
+             EmptyContext &Ctx) {
+  if (IO.outputting()) {
+    // Ensure check options are sorted
+    std::vector<std::pair<llvm::StringRef, llvm::StringRef>> SortedOptions;
+    SortedOptions.reserve(Val.size());
+    for (auto &Key : Val) {
+      SortedOptions.emplace_back(Key.getKey(), Key.getValue().Value);
+    }
+    std::sort(SortedOptions.begin(), SortedOptions.end());
+
+    IO.beginMapping();
+    // Only output as a map
+    for (auto &Option : SortedOptions) {
+      bool UseDefault = false;
+      void *SaveInfo = nullptr;
+      IO.preflightKey(Option.first.data(), true, false, UseDefault, SaveInfo);
+      IO.scalarString(Option.second, needsQuotes(Option.second));
+      IO.postflightKey(SaveInfo);
+    }
+    IO.endMapping();
+  } else {
+    // Support both old list format and new map format for reading
+    auto &I = reinterpret_cast<Input &>(IO);
+    if (isa<SequenceNode>(I.getCurrentNode())) {
+      MappingNormalization<NOptionMap, FlangTidyOptions::OptionMap> NOpts(IO,
+                                                                          Val);
+      EmptyContext Ctx;
+      yamlize(IO, NOpts->Options, true, Ctx);
+    } else if (isa<MappingNode>(I.getCurrentNode())) {
+      IO.beginMapping();
+      for (llvm::StringRef Key : IO.keys()) {
+        IO.mapRequired(Key.data(), Val[Key].Value);
+      }
+      IO.endMapping();
+    } else {
+      IO.setError("expected a sequence or map");
+    }
+  }
+}
+
+template <>
+struct MappingTraits<FlangTidyOptions> {
+  static void mapping(IO &IO, FlangTidyOptions &Options) {
+    IO.mapOptional("Checks", Options.Checks);
+    IO.mapOptional("CheckOptions", Options.CheckOptions);
+    IO.mapOptional("ExtraArgs", Options.ExtraArgs);
+    IO.mapOptional("ExtraArgsBefore", Options.ExtraArgsBefore);
+    IO.mapOptional("InheritParentConfig", Options.InheritParentConfig);
+  }
+};
+
+} // namespace llvm::yaml
+
+namespace Fortran::tidy {
+
+const char FlangTidyOptionsProvider::OptionsSourceTypeDefaultBinary[] =
+    "flang-tidy binary";
+const char FlangTidyOptionsProvider::OptionsSourceTypeCheckCommandLineOption[] =
+    "command-line option '-checks'";
+const char
+    FlangTidyOptionsProvider::OptionsSourceTypeConfigCommandLineOption[] =
+        "command-line option '-config'";
+
+FlangTidyOptions FlangTidyOptions::getDefaults() {
+  FlangTidyOptions Options;
+  Options.Checks = "*";
+  return Options;
+}
+
+template <typename T>
+static void mergeVectors(std::optional<T> &Dest, const std::optional<T> &Src) {
+  if (Src) {
+    if (Dest)
+      Dest->insert(Dest->end(), Src->begin(), Src->end());
+    else
+      Dest = Src;
+  }
+}
+
+template <typename T>
+static void overrideValue(std::optional<T> &Dest, const std::optional<T> &Src) {
+  if (Src)
+    Dest = Src;
+}
+
+FlangTidyOptions &FlangTidyOptions::mergeWith(const FlangTidyOptions &Other,
+                                              unsigned Order) {
+  // For checks: if Other has checks defined, override completely (don't merge)
+  // This ensures config file checks override defaults instead of appending
+  if (Other.Checks) {
+    Checks = Other.Checks;
+  }
+
+  mergeVectors(ExtraArgs, Other.ExtraArgs);
+  mergeVectors(ExtraArgsBefore, Other.ExtraArgsBefore);
+  overrideValue(InheritParentConfig, Other.InheritParentConfig);
+
+  for (const auto &KeyValue : Other.CheckOptions) {
+    CheckOptions.insert_or_assign(
+        KeyValue.getKey(),
+        FlangTidyValue(KeyValue.getValue().Value,
+                       KeyValue.getValue().Priority + Order));
+  }
+  return *this;
+}
+
+FlangTidyOptions FlangTidyOptions::merge(const FlangTidyOptions &Other,
+                                         unsigned Order) const {
+  FlangTidyOptions Result = *this;
+  Result.mergeWith(Other, Order);
+  return Result;
+}
+
+void FlangTidyOptions::parseChecksString() {
+  enabledChecks.clear();
+  if (!Checks || Checks->empty())
+    return;
+
+  std::stringstream ss(*Checks);
+  std::string check;
+  while (std::getline(ss, check, ',')) {
+    // Trim whitespace
+    check.erase(0, check.find_first_not_of(" \t"));
+    check.erase(check.find_last_not_of(" \t") + 1);
+    if (!check.empty()) {
+      enabledChecks.push_back(check);
+    }
+  }
+}
+
+FlangTidyOptions
+FlangTidyOptionsProvider::getOptions(llvm::StringRef FileName) {
+  FlangTidyOptions Result;
+  unsigned Priority = 0;
+  for (auto &Source : getRawOptions(FileName))
+    Result.mergeWith(Source.first, ++Priority);
+  return Result;
+}
+
+std::vector<OptionsSource>
+DefaultOptionsProvider::getRawOptions(llvm::StringRef FileName) {
+  std::vector<OptionsSource> Result;
+  Result.emplace_back(DefaultOptions, OptionsSourceTypeDefaultBinary);
+  return Result;
+}
+
+ConfigOptionsProvider::ConfigOptionsProvider(
+    FlangTidyGlobalOptions GlobalOptions, FlangTidyOptions DefaultOptions,
+    FlangTidyOptions ConfigOptions, FlangTidyOptions OverrideOptions,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
+    : FileOptionsBaseProvider(std::move(GlobalOptions),
+                              std::move(DefaultOptions),
+                              std::move(OverrideOptions), std::move(FS)),
+      ConfigOptions(std::move(ConfigOptions)) {}
+
+std::vector<OptionsSource>
+ConfigOptionsProvider::getRawOptions(llvm::StringRef FileName) {
+  std::vector<OptionsSource> RawOptions =
+      DefaultOptionsProvider::getRawOptions(FileName);
+  if (ConfigOptions.InheritParentConfig.value_or(false)) {
+    LLVM_DEBUG(llvm::dbgs()
+               << "Getting options for file " << FileName << "...\n");
+
+    llvm::ErrorOr<llvm::SmallString<128>> AbsoluteFilePath =
+        getNormalizedAbsolutePath(FileName);
+    if (AbsoluteFilePath) {
+      addRawFileOptions(AbsoluteFilePath->str(), RawOptions);
+    }
+  }
+  RawOptions.emplace_back(ConfigOptions,
+                          OptionsSourceTypeConfigCommandLineOption);
+  // Give command-line options very high priority so they override everything
+  RawOptions.emplace_back(OverrideOptions,
+                          OptionsSourceTypeCheckCommandLineOption);
+  return RawOptions;
+}
+
+FileOptionsBaseProvider::FileOptionsBaseProvider(
+    FlangTidyGlobalOptions GlobalOptions, FlangTidyOptions DefaultOptions,
+    FlangTidyOptions OverrideOptions,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS)
+    : DefaultOptionsProvider(std::move(GlobalOptions),
+                             std::move(DefaultOptions)),
+      OverrideOptions(std::move(OverrideOptions)), FS(std::move(VFS)) {
+  if (!FS)
+    FS = llvm::vfs::getRealFileSystem();
+  ConfigHandlers.emplace_back(".flang-tidy", parseConfiguration);
+}
+
+FileOptionsBaseProvider::FileOptionsBaseProvider(
+    FlangTidyGlobalOptions GlobalOptions, FlangTidyOptions DefaultOptions,
+    FlangTidyOptions OverrideOptions,
+    FileOptionsBaseProvider::ConfigFileHandlers ConfigHandlers)
+    : DefaultOptionsProvider(std::move(GlobalOptions),
+                             std::move(DefaultOptions)),
+      OverrideOptions(std::move(OverrideOptions)),
+      ConfigHandlers(std::move(ConfigHandlers)) {}
+
+llvm::ErrorOr<llvm::SmallString<128>>
+FileOptionsBaseProvider::getNormalizedAbsolutePath(llvm::StringRef Path) {
+  assert(FS && "FS must be set.");
+  llvm::SmallString<128> NormalizedAbsolutePath = {Path};
+  std::error_code Err = FS->makeAbsolute(NormalizedAbsolutePath);
+  if (Err)
+    return Err;
+  llvm::sys::path::remove_dots(NormalizedAbsolutePath, /*remove_dot_dot=*/true);
+  return NormalizedAbsolutePath;
+}
+
+void FileOptionsBaseProvider::addRawFileOptions(
+    llvm::StringRef AbsolutePath, std::vector<OptionsSource> &CurOptions) {
+  auto CurSize = CurOptions.size();
+  llvm::StringRef RootPath = llvm::sys::path::parent_path(AbsolutePath);
+  auto MemorizedConfigFile =
+      [this,
+       &RootPath](llvm::StringRef CurrentPath) -> std::optional<OptionsSource> {
+    const auto Iter = CachedOptions.Memorized.find(CurrentPath);
+    if (Iter != CachedOptions.Memorized.end())
+      return CachedOptions.Storage[Iter->second];
+    std::optional<OptionsSource> OptionsSource = tryReadConfigFile(CurrentPath);
+    if (OptionsSource) {
+      const size_t Index = CachedOptions.Storage.size();
+      CachedOptions.Storage.emplace_back(OptionsSource.value());
+      while (RootPath != CurrentPath) {
+        LLVM_DEBUG(llvm::dbgs()
+                   << "Caching configuration for path " << RootPath << ".\n");
+        CachedOptions.Memorized[RootPath] = Index;
+        RootPath = llvm::sys::path::parent_path(RootPath);
+      }
+      CachedOptions.Memorized[CurrentPath] = Index;
+      RootPath = llvm::sys::path::parent_path(CurrentPath);
+    }
+    return OptionsSource;
+  };
+  for (llvm::StringRef CurrentPath = RootPath; !CurrentPath.empty();
+       CurrentPath = llvm::sys::path::parent_path(CurrentPath)) {
+    if (std::optional<OptionsSource> Result =
+            MemorizedConfigFile(CurrentPath)) {
+      CurOptions.emplace_back(Result.value());
+      if (!Result->first.InheritParentConfig.value_or(false))
+        break;
+    }
+  }
+  // Reverse order of file configs because closer configs should have higher
+  // priority.
+  std::reverse(CurOptions.begin() + CurSize, CurOptions.end());
+}
+
+FileOptionsProvider::FileOptionsProvider(
+    FlangTidyGlobalOptions GlobalOptions, FlangTidyOptions DefaultOptions,
+    FlangTidyOptions OverrideOptions,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS)
+    : FileOptionsBaseProvider(std::move(GlobalOptions),
+                              std::move(DefaultOptions),
+                              std::move(OverrideOptions), std::move(VFS)) {}
+
+FileOptionsProvider::FileOptionsProvider(
+    FlangTidyGlobalOptions GlobalOptions, FlangTidyOptions DefaultOptions,
+    FlangTidyOptions OverrideOptions,
+    FileOptionsBaseProvider::ConfigFileHandlers ConfigHandlers)
+    : FileOptionsBaseProvider(
+          std::move(GlobalOptions), std::move(DefaultOptions),
+          std::move(OverrideOptions), std::move(ConfigHandlers)) {}
+
+std::vector<OptionsSource>
+FileOptionsProvider::getRawOptions(llvm::StringRef FileName) {
+  LLVM_DEBUG(llvm::dbgs() << "Getting options for file " << FileName
+                          << "...\n");
+
+  llvm::ErrorOr<llvm::SmallString<128>> AbsoluteFilePath =
+      getNormalizedAbsolutePath(FileName);
+  if (!AbsoluteFilePath)
+    return {};
+
+  std::vector<OptionsSource> RawOptions =
+      DefaultOptionsProvider::getRawOptions(AbsoluteFilePath->str());
+  addRawFileOptions(AbsoluteFilePath->str(), RawOptions);
+
+  // Give command-line options very high priority so they override everything
+  OptionsSource CommandLineOptions(OverrideOptions,
+                                   OptionsSourceTypeCheckCommandLineOption);
+  RawOptions.push_back(CommandLineOptions);
+  return RawOptions;
+}
+
+std::optional<OptionsSource>
+FileOptionsBaseProvider::tryReadConfigFile(llvm::StringRef Directory) {
+  assert(!Directory.empty());
+
+  llvm::ErrorOr<llvm::vfs::Status> DirectoryStatus = FS->status(Directory);
+
+  if (!DirectoryStatus || !DirectoryStatus->isDirectory()) {
+    llvm::errs() << "Error reading configuration from " << Directory
+                 << ": directory doesn't exist.\n";
+    return std::nullopt;
+  }
+
+  for (const ConfigFileHandler &ConfigHandler : ConfigHandlers) {
+    llvm::SmallString<128> ConfigFile(Directory);
+    llvm::sys::path::append(ConfigFile, ConfigHandler.first);
+    LLVM_DEBUG(llvm::dbgs() << "Trying " << ConfigFile << "...\n");
+
+    llvm::ErrorOr<llvm::vfs::Status> FileStatus = FS->status(ConfigFile);
+
+    if (!FileStatus || !FileStatus->isRegularFile())
+      continue;
+
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
+        FS->getBufferForFile(ConfigFile);
+    if (std::error_code EC = Text.getError()) {
+      llvm::errs() << "Can't read " << ConfigFile << ": " << EC.message()
+                   << "\n";
+      continue;
+    }
+
+    // Skip empty files
+    if ((*Text)->getBuffer().empty())
+      continue;
+    llvm::ErrorOr<FlangTidyOptions> ParsedOptions =
+        ConfigHandler.second({(*Text)->getBuffer(), ConfigFile});
+    if (!ParsedOptions) {
+      if (ParsedOptions.getError())
+        llvm::errs() << "Error parsing " << ConfigFile << ": "
+                     << ParsedOptions.getError().message() << "\n";
+      continue;
+    }
+    return OptionsSource(*ParsedOptions, std::string(ConfigFile));
+  }
+  return std::nullopt;
+}
+
+llvm::ErrorOr<FlangTidyOptions>
+parseConfiguration(llvm::MemoryBufferRef Config) {
+  llvm::yaml::Input Input(Config);
+  FlangTidyOptions Options;
+  Input >> Options;
+  if (Input.error())
+    return Input.error();
+
+  // Parse the checks string into the enabledChecks vector
+  Options.parseChecksString();
+
+  return Options;
+}
+
+std::string configurationAsText(const FlangTidyOptions &Options) {
+  std::string Text;
+  llvm::raw_string_ostream Stream(Text);
+  llvm::yaml::Output Output(Stream);
+  // We use the same mapping method for input and output, so we need a non-const
+  // reference here.
+  FlangTidyOptions NonConstValue = Options;
+  Output << NonConstValue;
+  return Stream.str();
+}
+
+// Simple function to get options for a file (for backward compatibility)
+FlangTidyOptions getOptionsForFile(llvm::StringRef FileName) {
+  FlangTidyGlobalOptions GlobalOptions;
+  FlangTidyOptions DefaultOptions = FlangTidyOptions::getDefaults();
+  FlangTidyOptions OverrideOptions;
+
+  FileOptionsProvider Provider(std::move(GlobalOptions),
+                               std::move(DefaultOptions),
+                               std::move(OverrideOptions));
+  return Provider.getOptions(FileName);
+}
+
+} // namespace Fortran::tidy
diff --git a/flang/tools/flang-tidy/FlangTidyOptions.h b/flang/tools/flang-tidy/FlangTidyOptions.h
new file mode 100644
index 0000000000000..55bc2a0780515
--- /dev/null
+++ b/flang/tools/flang-tidy/FlangTidyOptions.h
@@ -0,0 +1,223 @@
+//===--- FlangTidyOptions.h - flang-tidy ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYOPTIONS_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYOPTIONS_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/MemoryBufferRef.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <functional>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace Fortran::tidy {
+
+/// Global options for FlangTidy. These options are neither stored nor read from
+/// configuration files.
+struct FlangTidyGlobalOptions {
+  // Reserved for future use (line filters, etc.)
+};
+
+/// Contains options for flang-tidy. These options may be read from
+/// configuration files, and may be different for different translation units.
+struct FlangTidyOptions {
+  /// These options are used for all settings that haven't been
+  /// overridden by the \c OptionsProvider.
+  static FlangTidyOptions getDefaults();
+
+  /// Overwrites all fields in here by the fields of \p Other that have a value.
+  /// \p Order specifies precedence of \p Other option.
+  FlangTidyOptions &mergeWith(const FlangTidyOptions &Other, unsigned Order);
+
+  /// Creates a new \c FlangTidyOptions instance combined from all fields
+  /// of this instance overridden by the fields of \p Other that have a value.
+  /// \p Order specifies precedence of \p Other option.
+  [[nodiscard]] FlangTidyOptions merge(const FlangTidyOptions &Other,
+                                       unsigned Order) const;
+
+  /// Checks filter.
+  std::optional<std::string> Checks;
+
+  /// Helper structure for storing option value with priority of the value.
+  struct FlangTidyValue {
+    FlangTidyValue() = default;
+    FlangTidyValue(const char *Value) : Value(Value) {}
+    FlangTidyValue(llvm::StringRef Value, unsigned Priority = 0)
+        : Value(Value), Priority(Priority) {}
+
+    std::string Value;
+    /// Priority stores relative precedence of the value loaded from config
+    /// files to disambiguate local vs global value from different levels.
+    unsigned Priority = 0;
+  };
+  using StringPair = std::pair<std::string, std::string>;
+  using OptionMap = llvm::StringMap<FlangTidyValue>;
+
+  /// Key-value mapping used to store check-specific options.
+  OptionMap CheckOptions;
+
+  using ArgList = std::vector<std::string>;
+
+  /// Add extra compilation arguments to the end of the list.
+  std::optional<ArgList> ExtraArgs;
+
+  /// Add extra compilation arguments to the start of the list.
+  std::optional<ArgList> ExtraArgsBefore;
+
+  /// Only used in the FileOptionsProvider and ConfigOptionsProvider. If true
+  /// and using a FileOptionsProvider, it will take a configuration file in the
+  /// parent directory (if any exists) and apply this config file on top of the
+  /// parent one. If false or missing, only this configuration file will be
+  /// used.
+  std::optional<bool> InheritParentConfig;
+
+  // Runtime-only options (not serialized to/from YAML)
+  std::vector<std::string> sourcePaths;   // Set by command line
+  std::vector<std::string> enabledChecks; // Parsed from Checks string
+  const char *argv0 = nullptr;            // Set by command line
+
+  /// Parse the Checks string into enabledChecks vector
+  void parseChecksString();
+};
+
+/// Abstract interface for retrieving various FlangTidy options.
+class FlangTidyOptionsProvider {
+public:
+  static const char OptionsSourceTypeDefaultBinary[];
+  static const char OptionsSourceTypeCheckCommandLineOption[];
+  static const char OptionsSourceTypeConfigCommandLineOption[];
+
+  virtual ~FlangTidyOptionsProvider() {}
+
+  /// Returns global options, which are independent of the file.
+  virtual const FlangTidyGlobalOptions &getGlobalOptions() = 0;
+
+  /// FlangTidyOptions and its source.
+  using OptionsSource = std::pair<FlangTidyOptions, std::string>;
+
+  /// Returns an ordered vector of OptionsSources, in order of increasing
+  /// priority.
+  virtual std::vector<OptionsSource>
+  getRawOptions(llvm::StringRef FileName) = 0;
+
+  /// Returns options applying to a specific translation unit with the
+  /// specified \p FileName.
+  FlangTidyOptions getOptions(llvm::StringRef FileName);
+};
+
+/// Implementation of the \c FlangTidyOptionsProvider interface, which
+/// returns the same options for all files.
+class DefaultOptionsProvider : public FlangTidyOptionsProvider {
+public:
+  DefaultOptionsProvider(FlangTidyGlobalOptions GlobalOptions,
+                         FlangTidyOptions Options)
+      : GlobalOptions(std::move(GlobalOptions)),
+        DefaultOptions(std::move(Options)) {}
+  const FlangTidyGlobalOptions &getGlobalOptions() override {
+    return GlobalOptions;
+  }
+  std::vector<OptionsSource> getRawOptions(llvm::StringRef FileName) override;
+
+protected:
+  FlangTidyGlobalOptions GlobalOptions;
+  FlangTidyOptions DefaultOptions;
+};
+
+class FileOptionsBaseProvider : public DefaultOptionsProvider {
+protected:
+  // A pair of configuration file base name and a function parsing
+  // configuration from text in the corresponding format.
+  using ConfigFileHandler =
+      std::pair<std::string, std::function<llvm::ErrorOr<FlangTidyOptions>(
+                                 llvm::MemoryBufferRef)>>;
+
+  /// Configuration file handlers listed in the order of priority.
+  using ConfigFileHandlers = std::vector<ConfigFileHandler>;
+
+  FileOptionsBaseProvider(FlangTidyGlobalOptions GlobalOptions,
+                          FlangTidyOptions DefaultOptions,
+                          FlangTidyOptions OverrideOptions,
+                          llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
+
+  FileOptionsBaseProvider(FlangTidyGlobalOptions GlobalOptions,
+                          FlangTidyOptions DefaultOptions,
+                          FlangTidyOptions OverrideOptions,
+                          ConfigFileHandlers ConfigHandlers);
+
+  void addRawFileOptions(llvm::StringRef AbsolutePath,
+                         std::vector<OptionsSource> &CurOptions);
+
+  llvm::ErrorOr<llvm::SmallString<128>>
+  getNormalizedAbsolutePath(llvm::StringRef AbsolutePath);
+
+  /// Try to read configuration files from \p Directory using registered
+  /// \c ConfigHandlers.
+  std::optional<OptionsSource> tryReadConfigFile(llvm::StringRef Directory);
+
+  struct OptionsCache {
+    llvm::StringMap<size_t> Memorized;
+    llvm::SmallVector<OptionsSource, 4U> Storage;
+  } CachedOptions;
+  FlangTidyOptions OverrideOptions;
+  ConfigFileHandlers ConfigHandlers;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS;
+};
+
+/// Implementation of FlangTidyOptions interface, which is used for
+/// '-config' command-line option.
+class ConfigOptionsProvider : public FileOptionsBaseProvider {
+public:
+  ConfigOptionsProvider(
+      FlangTidyGlobalOptions GlobalOptions, FlangTidyOptions DefaultOptions,
+      FlangTidyOptions ConfigOptions, FlangTidyOptions OverrideOptions,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr);
+  std::vector<OptionsSource> getRawOptions(llvm::StringRef FileName) override;
+
+private:
+  FlangTidyOptions ConfigOptions;
+};
+
+/// Implementation of the \c FlangTidyOptionsProvider interface, which
+/// tries to find a configuration file in the closest parent directory of each
+/// source file.
+class FileOptionsProvider : public FileOptionsBaseProvider {
+public:
+  FileOptionsProvider(
+      FlangTidyGlobalOptions GlobalOptions, FlangTidyOptions DefaultOptions,
+      FlangTidyOptions OverrideOptions,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr);
+
+  FileOptionsProvider(FlangTidyGlobalOptions GlobalOptions,
+                      FlangTidyOptions DefaultOptions,
+                      FlangTidyOptions OverrideOptions,
+                      ConfigFileHandlers ConfigHandlers);
+
+  std::vector<OptionsSource> getRawOptions(llvm::StringRef FileName) override;
+};
+
+/// Parse FlangTidy configuration from YAML and returns \c FlangTidyOptions or
+/// an error.
+llvm::ErrorOr<FlangTidyOptions>
+parseConfiguration(llvm::MemoryBufferRef Config);
+
+/// Serializes configuration to a YAML-encoded string.
+std::string configurationAsText(const FlangTidyOptions &Options);
+
+/// Simple function to get options for a file (for backward compatibility)
+FlangTidyOptions getOptionsForFile(llvm::StringRef FileName);
+
+} // namespace Fortran::tidy
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYOPTIONS_H
diff --git a/flang/tools/flang-tidy/MultiplexVisitor.h b/flang/tools/flang-tidy/MultiplexVisitor.h
new file mode 100644
index 0000000000000..1ba2df297fea9
--- /dev/null
+++ b/flang/tools/flang-tidy/MultiplexVisitor.h
@@ -0,0 +1,103 @@
+//===--- MultiplexVisitor.h - flang-tidy ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_MULTIPLEXVISITOR_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_MULTIPLEXVISITOR_H
+
+#include "FlangTidyCheck.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Semantics/semantics.h"
+#include <memory>
+#include <vector>
+
+namespace Fortran::tidy {
+
+class MultiplexVisitor {
+public:
+  MultiplexVisitor(semantics::SemanticsContext &context) : context_{context} {}
+
+  void AddChecker(std::unique_ptr<FlangTidyCheck> checker) {
+    checkers_.emplace_back(std::move(checker));
+  }
+
+  template <typename N>
+  bool Pre(const N &node) {
+    if constexpr (common::HasMember<const N *, semantics::ConstructNode>) {
+      context_.PushConstruct(node);
+    }
+    for (auto &checker : checkers_) {
+      checker->Enter(node);
+    }
+    return true;
+  }
+
+  template <typename N>
+  void Post(const N &node) {
+    for (auto &checker : checkers_) {
+      checker->Leave(node);
+    }
+    if constexpr (common::HasMember<const N *, semantics::ConstructNode>) {
+      context_.PopConstruct();
+    }
+  }
+
+  template <typename T>
+  bool Pre(const parser::Statement<T> &node) {
+    if (context_.IsInModuleFile(node.source))
+      return true;
+    context_.set_location(node.source);
+    for (auto &checker : checkers_) {
+      checker->Enter(node);
+    }
+    return true;
+  }
+
+  template <typename T>
+  bool Pre(const parser::UnlabeledStatement<T> &node) {
+    if (context_.IsInModuleFile(node.source))
+      return true;
+    context_.set_location(node.source);
+    for (auto &checker : checkers_) {
+      checker->Enter(node);
+    }
+    return true;
+  }
+
+  template <typename T>
+  void Post(const parser::Statement<T> &node) {
+    if (context_.IsInModuleFile(node.source))
+      return;
+    for (auto &checker : checkers_) {
+      checker->Leave(node);
+    }
+    context_.set_location(std::nullopt);
+  }
+
+  template <typename T>
+  void Post(const parser::UnlabeledStatement<T> &node) {
+    if (context_.IsInModuleFile(node.source))
+      return;
+    for (auto &checker : checkers_) {
+      checker->Leave(node);
+    }
+    context_.set_location(std::nullopt);
+  }
+
+  bool Walk(const parser::Program &program) {
+    parser::Walk(program, *this);
+    return !context_.AnyFatalError();
+  }
+
+public:
+  semantics::SemanticsContext &context_;
+  std::vector<std::unique_ptr<FlangTidyCheck>> checkers_;
+};
+
+} // namespace Fortran::tidy
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_MULTIPLEXVISITOR_H
diff --git a/flang/tools/flang-tidy/tool/FlangTidyMain.cpp b/flang/tools/flang-tidy/tool/FlangTidyMain.cpp
index 7492dfda36d8f..50229930a2600 100644
--- a/flang/tools/flang-tidy/tool/FlangTidyMain.cpp
+++ b/flang/tools/flang-tidy/tool/FlangTidyMain.cpp
@@ -6,8 +6,303 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "../FlangTidy.h"
+#include "../FlangTidyForceLinker.h"
+#include "../FlangTidyOptions.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <sstream>
+#include <vector>
+
+// Frontend driver
+#include "flang/Frontend/CompilerInstance.h"
+#include "flang/Frontend/CompilerInvocation.h"
+#include "flang/Frontend/TextDiagnosticBuffer.h"
+
 namespace Fortran::tidy {
 
-extern int flangTidyMain(int &argc, const char **argv) { return 0; }
+static llvm::cl::list<std::string>
+    SourcePaths(llvm::cl::Positional,
+                llvm::cl::desc("<source0> [... <sourceN>]"),
+                llvm::cl::OneOrMore, llvm::cl::value_desc("source files"),
+                llvm::cl::sub(llvm::cl::SubCommand::getAll()));
+
+static llvm::cl::opt<std::string>
+    CheckOption("checks",
+                llvm::cl::desc("Comma-separated list of checks to enable. "
+                               "Overrides configuration file settings."),
+                llvm::cl::init(""), llvm::cl::value_desc("check list"));
+
+static llvm::cl::opt<std::string> ConfigOption(
+    "config",
+    llvm::cl::desc(
+        "Specify configuration in YAML format: "
+        "-config=\"{Checks: '*', CognitiveComplexityThreshold: 30}\" "
+        "When empty, flang-tidy will look for .flang-tidy files."),
+    llvm::cl::init(""), llvm::cl::value_desc("yaml config"));
+
+static llvm::cl::opt<std::string> ConfigFile(
+    "config-file",
+    llvm::cl::desc("Specify the path of .flang-tidy or custom config file"),
+    llvm::cl::init(""), llvm::cl::value_desc("filename"));
+
+static llvm::cl::opt<bool>
+    DumpConfig("dump-config",
+               llvm::cl::desc("Dump configuration in YAML format to stdout"),
+               llvm::cl::init(false));
+
+static llvm::cl::list<std::string> ArgsBefore(
+    "extra-arg-before",
+    llvm::cl::desc(
+        "Additional argument to prepend to the compiler command line"),
+    llvm::cl::ZeroOrMore, llvm::cl::sub(llvm::cl::SubCommand::getAll()));
+
+static llvm::cl::list<std::string>
+    ArgsAfter("extra-arg",
+              llvm::cl::desc(
+                  "Additional argument to append to the compiler command line"),
+              llvm::cl::ZeroOrMore,
+              llvm::cl::sub(llvm::cl::SubCommand::getAll()));
+
+static std::string GetFlangToolCommand() {
+  static int Dummy;
+  std::string FlangExecutable =
+      llvm::sys::fs::getMainExecutable("flang", (void *)&Dummy);
+  llvm::SmallString<128> FlangToolPath;
+  FlangToolPath = llvm::sys::path::parent_path(FlangExecutable);
+  llvm::sys::path::append(FlangToolPath, "flang-tool");
+  return std::string(FlangToolPath);
+}
+
+static bool stripPositionalArgs(std::vector<const char *> Args,
+                                std::vector<std::string> &Result,
+                                std::string &ErrorMsg) {
+  auto flang = std::make_unique<Fortran::frontend::CompilerInstance>();
+
+  // Create diagnostics engine
+  flang->createDiagnostics();
+  if (!flang->hasDiagnostics()) {
+    llvm::errs() << "Failed to create diagnostics engine\n";
+    return false;
+  }
+
+  // Capture diagnostics
+  frontend::TextDiagnosticBuffer *diagsBuffer =
+      new frontend::TextDiagnosticBuffer;
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs> diagID(
+      new clang::DiagnosticIDs());
+  clang::DiagnosticOptions diagOpts;
+  clang::DiagnosticsEngine diags(diagID, diagOpts, diagsBuffer);
+
+  // Insert Flang tool command
+  std::string Argv0 = GetFlangToolCommand();
+  Args.insert(Args.begin(), Argv0.c_str());
+
+  // Add a dummy file to ensure at least one compilation job
+  Args.push_back("placeholder.f90");
+
+  // Remove -c flags if present
+  Args.erase(std::remove_if(
+                 Args.begin(), Args.end(),
+                 [](const char *arg) { return llvm::StringRef(arg) == "-c"; }),
+             Args.end());
+
+  // Create compiler invocation
+  bool success = Fortran::frontend::CompilerInvocation::createFromArgs(
+      flang->getInvocation(), Args, diags, Argv0.c_str());
+
+  if (!success) {
+    ErrorMsg = "Failed to create compiler invocation\n";
+    // Flush diagnostic
+    diagsBuffer->flushDiagnostics(flang->getDiagnostics());
+    return false;
+  }
+
+  // Get the list of input files from Flang's frontend options
+  std::vector<std::string> inputs;
+  for (const auto &input : flang->getFrontendOpts().inputs) {
+    inputs.push_back(input.getFile().str());
+  }
+
+  if (inputs.empty()) {
+    ErrorMsg = "warning: no compile jobs found\n";
+    return false;
+  }
+
+  // Remove input files from Args
+  std::vector<const char *>::iterator End = llvm::remove_if(
+      Args, [&](llvm::StringRef S) { return llvm::is_contained(inputs, S); });
+
+  // Store the filtered arguments
+  Result = std::vector<std::string>(Args.begin(), End);
+  return true;
+}
+
+static std::vector<std::string>
+loadFromCommandLine(int &Argc, const char *const *Argv, std::string &ErrorMsg) {
+  ErrorMsg.clear();
+  if (Argc == 0)
+    return {};
+  const char *const *DoubleDash =
+      std::find(Argv, Argv + Argc, llvm::StringRef("--"));
+  if (DoubleDash == Argv + Argc)
+    return {};
+  std::vector<const char *> CommandLine(DoubleDash + 1, Argv + Argc);
+  Argc = DoubleDash - Argv;
+
+  std::vector<std::string> StrippedArgs;
+  if (!stripPositionalArgs(CommandLine, StrippedArgs, ErrorMsg))
+    return {};
+  return StrippedArgs;
+}
+
+static std::unique_ptr<FlangTidyOptionsProvider>
+createOptionsProvider(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
+  FlangTidyGlobalOptions GlobalOptions;
+
+  FlangTidyOptions DefaultOptions = FlangTidyOptions::getDefaults();
+
+  FlangTidyOptions OverrideOptions;
+  if (!CheckOption.empty())
+    OverrideOptions.Checks =
+        CheckOption; // This completely overrides, not merges
+
+  auto LoadConfig =
+      [&](llvm::StringRef Configuration,
+          llvm::StringRef Source) -> std::unique_ptr<FlangTidyOptionsProvider> {
+    llvm::ErrorOr<FlangTidyOptions> ParsedConfig =
+        parseConfiguration(llvm::MemoryBufferRef(Configuration, Source));
+    if (ParsedConfig)
+      return std::make_unique<ConfigOptionsProvider>(
+          std::move(GlobalOptions),
+          FlangTidyOptions::getDefaults().merge(DefaultOptions, 0),
+          std::move(*ParsedConfig), std::move(OverrideOptions), std::move(FS));
+    llvm::errs() << "Error: invalid configuration specified.\n"
+                 << ParsedConfig.getError().message() << "\n";
+    return nullptr;
+  };
+
+  if (!ConfigFile.empty()) {
+    if (!ConfigOption.empty()) {
+      llvm::errs() << "Error: --config-file and --config are "
+                      "mutually exclusive. Specify only one.\n";
+      return nullptr;
+    }
+
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
+        llvm::MemoryBuffer::getFile(ConfigFile);
+    if (std::error_code EC = Text.getError()) {
+      llvm::errs() << "Error: can't read config-file '" << ConfigFile
+                   << "': " << EC.message() << "\n";
+      return nullptr;
+    }
+
+    return LoadConfig((*Text)->getBuffer(), ConfigFile);
+  }
+
+  if (!ConfigOption.empty())
+    return LoadConfig(ConfigOption, "<command-line-config>");
+
+  return std::make_unique<FileOptionsProvider>(
+      std::move(GlobalOptions), std::move(DefaultOptions),
+      std::move(OverrideOptions), std::move(FS));
+}
+
+extern int flangTidyMain(int &argc, const char **argv) {
+  llvm::InitLLVM X(argc, argv);
+
+  std::string ErrorMessage;
+  auto Compilations = loadFromCommandLine(argc, argv, ErrorMessage);
+
+  if (!ErrorMessage.empty()) {
+    llvm::outs() << ErrorMessage << "\n";
+    return 1;
+  }
+
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "flang-tidy: A Fortran source analysis tool\n");
+
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS =
+      llvm::vfs::getRealFileSystem();
+
+  auto OwningOptionsProvider = createOptionsProvider(BaseFS);
+  auto *OptionsProvider = OwningOptionsProvider.get();
+  if (!OptionsProvider)
+    return 1;
+
+  llvm::StringRef FileName = "dummy.f90";
+  if (!SourcePaths.empty())
+    FileName = SourcePaths[0];
+
+  FlangTidyOptions EffectiveOptions = OptionsProvider->getOptions(FileName);
+
+  if (DumpConfig) {
+    llvm::outs() << configurationAsText(EffectiveOptions) << "\n";
+    return 0;
+  }
+
+  EffectiveOptions.sourcePaths.assign(SourcePaths.begin(), SourcePaths.end());
+  EffectiveOptions.argv0 = argv[0];
+
+  EffectiveOptions.parseChecksString();
+
+  for (const auto &sourcePath : EffectiveOptions.sourcePaths) {
+    if (!llvm::sys::fs::exists(sourcePath)) {
+      llvm::errs() << "Error: File not found: " << sourcePath << "\n";
+      return 1;
+    }
+  }
+
+  // Add extra args from command line (these override config file settings)
+  for (const auto &arg : ArgsBefore) {
+    std::istringstream stream(arg);
+    std::string subArg;
+    while (stream >> subArg) {
+      if (!EffectiveOptions.ExtraArgsBefore)
+        EffectiveOptions.ExtraArgsBefore = std::vector<std::string>();
+      EffectiveOptions.ExtraArgsBefore->push_back(subArg);
+    }
+  }
+
+  for (const auto &arg : ArgsAfter) {
+    std::istringstream stream(arg);
+    std::string subArg;
+    while (stream >> subArg) {
+      if (!EffectiveOptions.ExtraArgs)
+        EffectiveOptions.ExtraArgs = std::vector<std::string>();
+      EffectiveOptions.ExtraArgs->push_back(subArg);
+    }
+  }
+
+  // Add compilation args if present
+  if (!Compilations.empty()) {
+    assert(EffectiveOptions.sourcePaths.size() == 1);
+    if (!EffectiveOptions.ExtraArgs)
+      EffectiveOptions.ExtraArgs = std::vector<std::string>();
+    EffectiveOptions.ExtraArgs->insert(EffectiveOptions.ExtraArgs->end(),
+                                       Compilations.begin(),
+                                       Compilations.end());
+  }
+
+  // Remove anything starting with --driver-mode
+  if (EffectiveOptions.ExtraArgs) {
+    EffectiveOptions.ExtraArgs->erase(
+        std::remove_if(EffectiveOptions.ExtraArgs->begin(),
+                       EffectiveOptions.ExtraArgs->end(),
+                       [](std::string const &arg) {
+                         return llvm::StringRef(arg).starts_with(
+                             "--driver-mode");
+                       }),
+        EffectiveOptions.ExtraArgs->end());
+  }
+
+  return runFlangTidy(EffectiveOptions);
+}
 
 } // namespace Fortran::tidy
diff --git a/flang/tools/flang-tidy/utils/CollectActualArguments.h b/flang/tools/flang-tidy/utils/CollectActualArguments.h
new file mode 100644
index 0000000000000..f445ec59097ea
--- /dev/null
+++ b/flang/tools/flang-tidy/utils/CollectActualArguments.h
@@ -0,0 +1,45 @@
+//===--- CollectActualArguments.h - flang-tidy ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_TIDY_UTILS_COLLECT_ACTUAL_ARGUMENTS
+#define FORTRAN_TIDY_UTILS_COLLECT_ACTUAL_ARGUMENTS
+
+#include "flang/Evaluate/call.h"
+#include "flang/Evaluate/traverse.h"
+
+namespace Fortran::evaluate {
+using ActualArgumentRef = common::Reference<const ActualArgument>;
+
+inline bool operator<(ActualArgumentRef x, ActualArgumentRef y) {
+  return &*x < &*y;
+}
+
+using ActualArgumentSet = std::set<evaluate::ActualArgumentRef>;
+
+struct CollectActualArgumentsHelper
+    : public evaluate::SetTraverse<CollectActualArgumentsHelper,
+                                   ActualArgumentSet> {
+  using Base = SetTraverse<CollectActualArgumentsHelper, ActualArgumentSet>;
+  CollectActualArgumentsHelper() : Base{*this} {}
+  using Base::operator();
+  ActualArgumentSet operator()(const evaluate::ActualArgument &arg) const {
+    return Combine(ActualArgumentSet{arg},
+                   CollectActualArgumentsHelper{}(arg.UnwrapExpr()));
+  }
+};
+
+template <typename A>
+ActualArgumentSet CollectActualArguments(const A &x) {
+  return CollectActualArgumentsHelper{}(x);
+}
+
+template ActualArgumentSet CollectActualArguments(const semantics::SomeExpr &);
+
+} // namespace Fortran::evaluate
+
+#endif // FORTRAN_TIDY_UTILS_COLLECT_ACTUAL_ARGUMENTS
diff --git a/flang/tools/flang-tidy/utils/SymbolUtils.h b/flang/tools/flang-tidy/utils/SymbolUtils.h
new file mode 100644
index 0000000000000..588a91d0e992b
--- /dev/null
+++ b/flang/tools/flang-tidy/utils/SymbolUtils.h
@@ -0,0 +1,25 @@
+//===--- SymbolUtils.h - flang-tidy -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_TIDY_UTILS_SYMBOL
+#define FORTRAN_TIDY_UTILS_SYMBOL
+
+#include "flang/Semantics/scope.h"
+#include "flang/Semantics/symbol.h"
+
+namespace Fortran::tidy::utils {
+
+inline bool IsFromModFileSafe(const semantics::Symbol &sym) {
+  return sym.test(semantics::Symbol::Flag::ModFile) ||
+         (!sym.owner().IsTopLevel() &&
+          (sym.owner().symbol() && IsFromModFileSafe(*sym.owner().symbol())));
+}
+
+} // namespace Fortran::tidy::utils
+
+#endif // FORTRAN_TIDY_UTILS_SYMBOL

>From 87e6d9015258971f2a307840067713b5bb1026db Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 11:30:09 +0200
Subject: [PATCH 04/31] [flang-tidy] add test infra

---
 flang/tools/CMakeLists.txt                    |   4 +
 flang/tools/test/CMakeLists.txt               |  39 ++++
 .../tools/test/flang-tidy/check_flang_tidy.py | 200 ++++++++++++++++++
 flang/tools/test/lit.cfg.py                   |  54 +++++
 flang/tools/test/lit.site.cfg.py.in           |  24 +++
 5 files changed, 321 insertions(+)
 create mode 100644 flang/tools/test/CMakeLists.txt
 create mode 100644 flang/tools/test/flang-tidy/check_flang_tidy.py
 create mode 100644 flang/tools/test/lit.cfg.py
 create mode 100644 flang/tools/test/lit.site.cfg.py.in

diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 644fa7fe9c715..5524c1ef365a1 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -14,3 +14,7 @@ add_subdirectory(f18-parse-demo)
 add_subdirectory(fir-opt)
 add_subdirectory(fir-lsp-server)
 add_subdirectory(flang-tidy)
+
+if(FLANG_INCLUDE_TESTS)
+  add_subdirectory(test)
+endif()
\ No newline at end of file
diff --git a/flang/tools/test/CMakeLists.txt b/flang/tools/test/CMakeLists.txt
new file mode 100644
index 0000000000000..47cc32561d26b
--- /dev/null
+++ b/flang/tools/test/CMakeLists.txt
@@ -0,0 +1,39 @@
+set(FLANG_TOOLS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/..")
+set(FLANG_TOOLS_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/..")
+
+llvm_canonicalize_cmake_booleans(
+  FLANG_PLUGIN_SUPPORT
+  LLVM_INSTALL_TOOLCHAIN_ONLY
+)
+
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+)
+
+set(FLANG_TOOLS_TEST_DEPS
+  # Individual tools we test.
+  flang-tidy
+  module_files
+)
+
+# Add lit test dependencies.
+set(LLVM_UTILS_DEPS
+  FileCheck count not
+)
+foreach(dep ${LLVM_UTILS_DEPS})
+  if(TARGET ${dep})
+    list(APPEND FLANG_TOOLS_TEST_DEPS ${dep})
+  endif()
+endforeach()
+
+add_lit_testsuite(check-flang-tools "Running flang-tools/test"
+   ${CMAKE_CURRENT_BINARY_DIR}
+   DEPENDS ${FLANG_TOOLS_TEST_DEPS}
+)
+
+add_lit_testsuites(FLANG-EXTRA ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS ${FLANG_TOOLS_TEST_DEPS}
+)
diff --git a/flang/tools/test/flang-tidy/check_flang_tidy.py b/flang/tools/test/flang-tidy/check_flang_tidy.py
new file mode 100644
index 0000000000000..5557038c83b8f
--- /dev/null
+++ b/flang/tools/test/flang-tidy/check_flang_tidy.py
@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+#
+# ===- check_flang_tidy.py - FlangTidy Test Helper ------------*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===------------------------------------------------------------------------===#
+
+"""
+FlangTidy Test Helper
+=====================
+
+This script helps run flang-tidy checks with llvm-lit. By default, it runs flang-tidy
+without applying fixes and uses FileCheck to verify warnings.
+"""
+
+import argparse
+import os
+import pathlib
+import re
+import subprocess
+import sys
+
+
+def write_file(file_name, text):
+    with open(file_name, "w", encoding="utf-8") as f:
+        f.write(text)
+        f.truncate()
+
+
+def try_run(args, raise_error=True):
+    try:
+        process_output = subprocess.check_output(args, stderr=subprocess.STDOUT).decode(
+            errors="ignore"
+        )
+    except subprocess.CalledProcessError as e:
+        process_output = e.output.decode(errors="ignore")
+        print("%s failed:\n%s" % (" ".join(args), process_output))
+        if raise_error:
+            raise
+    return process_output
+
+
+class MessagePrefix:
+    def __init__(self, label):
+        self.has_message = False
+        self.prefixes = []
+        self.label = label
+
+    def check(self, file_check_suffix, input_text):
+        self.prefix = self.label + file_check_suffix
+        self.has_message = self.prefix in input_text
+        if self.has_message:
+            self.prefixes.append(self.prefix)
+        return self.has_message
+
+
+class CheckRunner:
+    def __init__(self, args, extra_args):
+        self.resource_dir = args.resource_dir
+        self.input_file_name = args.input_file_name
+        self.check_name = args.check_name
+        self.temp_file_name = args.temp_file_name
+        self.expect_flang_tidy_error = args.expect_flang_tidy_error
+        self.std = args.std
+        self.check_suffix = args.check_suffix
+        self.input_text = ""
+        self.has_check_messages = False
+        self.expect_no_diagnosis = False
+        self.messages = MessagePrefix("CHECK-MESSAGES")
+
+        file_name_with_extension = self.input_file_name
+        _, extension = os.path.splitext(file_name_with_extension)
+        if extension not in [".f", ".f90", ".f95"]:
+            extension = ".f90"
+        self.temp_file_name = self.temp_file_name + extension
+
+        self.extra_args = extra_args
+        self.flang_extra_args = []
+
+        if "--" in extra_args:
+            i = self.extra_args.index("--")
+            self.flang_extra_args = self.extra_args[i + 1 :]
+            self.extra_args = self.extra_args[:i]
+
+    def read_input(self):
+        with open(self.input_file_name, "r", encoding="utf-8") as input_file:
+            self.input_text = input_file.read()
+
+    def get_prefixes(self):
+        for suffix in self.check_suffix:
+            if suffix and not re.match("^[A-Z0-9\\-]+$", suffix):
+                sys.exit(
+                    'Only A..Z, 0..9 and "-" are allowed in check suffixes list,'
+                    + ' but "%s" was given' % suffix
+                )
+
+            file_check_suffix = ("-" + suffix) if suffix else ""
+
+            has_check_message = self.messages.check(file_check_suffix, self.input_text)
+            self.has_check_messages = self.has_check_messages or has_check_message
+
+            if not has_check_message:
+                self.expect_no_diagnosis = True
+
+        if self.expect_no_diagnosis and self.has_check_messages:
+            sys.exit(
+                "%s not found in the input" % self.messages.prefix
+            )
+        assert self.has_check_messages or self.expect_no_diagnosis
+
+    def prepare_test_inputs(self):
+        cleaned_test = re.sub("// *CHECK-[A-Z0-9\\-]*:[^\r\n]*", "//", self.input_text)
+        write_file(self.temp_file_name, cleaned_test)
+
+    def run_flang_tidy(self):
+        args = (
+            [
+                "flang-tidy",
+                self.temp_file_name,
+            ]
+            + ["--checks=" + self.check_name]
+            + self.extra_args
+            #+ ["--"]
+            #+ self.flang_extra_args
+        )
+        if self.expect_flang_tidy_error:
+            args.insert(0, "not")
+        print("Running " + repr(args) + "...")
+        flang_tidy_output = try_run(args)
+        print("------------------------ flang-tidy output -----------------------")
+        print(
+            flang_tidy_output.encode(sys.stdout.encoding, errors="replace").decode(
+                sys.stdout.encoding
+            )
+        )
+        print("------------------------------------------------------------------")
+        return flang_tidy_output
+
+    def check_messages(self, flang_tidy_output):
+        if self.has_check_messages:
+            messages_file = self.temp_file_name + ".msg"
+            write_file(messages_file, flang_tidy_output)
+            try_run(
+                [
+                    "FileCheck",
+                    "-input-file=" + messages_file,
+                    self.input_file_name,
+                    "-check-prefixes=" + ",".join(self.messages.prefixes),
+                    "-implicit-check-not={{warning|error}}:",
+                ]
+            )
+
+    def run(self):
+        self.read_input()
+        self.get_prefixes()
+        self.prepare_test_inputs()
+        flang_tidy_output = self.run_flang_tidy()
+        if self.expect_no_diagnosis:
+            if flang_tidy_output != "":
+                sys.exit("No diagnostics were expected, but found the ones above")
+        else:
+            self.check_messages(flang_tidy_output)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(
+        prog=pathlib.Path(__file__).stem,
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("-expect-flang-tidy-error", action="store_true")
+    parser.add_argument("-resource-dir")
+    parser.add_argument("input_file_name")
+    parser.add_argument("check_name")
+    parser.add_argument("temp_file_name")
+    parser.add_argument(
+        "-check-suffix",
+        "-check-suffixes",
+        default=[""],
+        type=lambda x: x.split(","),
+        help="comma-separated list of FileCheck suffixes",
+    )
+    parser.add_argument(
+        "-std",
+        default="f2003",
+        help="Fortran standard to pass to flang.",
+    )
+    return parser.parse_known_args()
+
+
+def main():
+    args, extra_args = parse_arguments()
+    CheckRunner(args, extra_args).run()
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/flang/tools/test/lit.cfg.py b/flang/tools/test/lit.cfg.py
new file mode 100644
index 0000000000000..01b737179d594
--- /dev/null
+++ b/flang/tools/test/lit.cfg.py
@@ -0,0 +1,54 @@
+# -*- Python -*-
+
+import os
+import shlex
+
+import lit.formats
+
+from lit.llvm import llvm_config
+
+# Configuration file for the 'lit' test runner.
+
+# name: The name of this test suite.
+config.name = "Flang Tools"
+
+# testFormat: The test format to use to interpret tests.
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = [
+    ".f",
+    ".f90",
+    ".f95",
+    ".f03",
+    ".f08",
+    ".mod",
+    ".test"
+]
+
+# Exclude 'Inputs' directories as they are for test dependencies.
+config.excludes = ["Inputs"]
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = os.path.join(config.flang_tools_binary_dir, "test")
+
+# Tools need the same environment setup as clang (we don't need clang itself).
+llvm_config.use_clang(required=False)
+
+python_exec = shlex.quote(config.python_executable)
+check_flang_tidy = os.path.join(
+    config.test_source_root, "flang-tidy", "check_flang_tidy.py"
+)
+config.substitutions.append(
+    ("%check_flang_tidy", "%s %s" % (python_exec, check_flang_tidy))
+)
+
+# Plugins (loadable modules)
+if config.has_plugins and config.llvm_plugin_ext:
+    config.available_features.add("plugins")
+
+# Disable default configuration files to avoid interference with test runs.
+config.environment["FLANG_NO_DEFAULT_CONFIG"] = "1"
\ No newline at end of file
diff --git a/flang/tools/test/lit.site.cfg.py.in b/flang/tools/test/lit.site.cfg.py.in
new file mode 100644
index 0000000000000..0f6c901adcce0
--- /dev/null
+++ b/flang/tools/test/lit.site.cfg.py.in
@@ -0,0 +1,24 @@
+ at LIT_SITE_CFG_IN_HEADER@
+
+import sys
+
+config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.flang_tools_binary_dir = "@FLANG_TOOLS_BINARY_DIR@"
+config.llvm_shlib_dir = "@SHLIBDIR@"
+config.python_executable = "@Python3_EXECUTABLE@"
+config.target_triple = "@LLVM_TARGET_TRIPLE@"
+config.host_triple = "@LLVM_HOST_TRIPLE@"
+config.has_plugins = @FLANG_PLUGIN_SUPPORT@
+
+# Support substitution of the tools and libs dirs with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@")
+config.llvm_libs_dir = lit_config.substitute("@LLVM_LIBS_DIR@")
+config.flang_tools_dir = lit_config.substitute("@CURRENT_TOOLS_DIR@")
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@FLANG_TOOLS_SOURCE_DIR@/test/lit.cfg.py")
\ No newline at end of file

>From 50a0f81dec5a995e5cf82380d12607841bce3d8f Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 11:33:50 +0200
Subject: [PATCH 05/31] [flang-tidy] add bugprone module

---
 flang/tools/flang-tidy/CMakeLists.txt         |  3 ++
 flang/tools/flang-tidy/FlangTidyForceLinker.h |  5 +++
 .../bugprone/BugproneTidyModule.cpp           | 32 +++++++++++++++++++
 .../tools/flang-tidy/bugprone/CMakeLists.txt  | 17 ++++++++++
 4 files changed, 57 insertions(+)
 create mode 100644 flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/CMakeLists.txt

diff --git a/flang/tools/flang-tidy/CMakeLists.txt b/flang/tools/flang-tidy/CMakeLists.txt
index 4e485cd25c46e..2c18eb741e22e 100644
--- a/flang/tools/flang-tidy/CMakeLists.txt
+++ b/flang/tools/flang-tidy/CMakeLists.txt
@@ -20,7 +20,10 @@ target_link_libraries(flangTidy
   flangFrontendTool
   )
 
+add_subdirectory(bugprone)
+
 set(ALL_FLANG_TIDY_CHECKS
+  flangTidyBugproneModule
   )
 
 set(ALL_FLANG_TIDY_CHECKS ${ALL_FLANG_TIDY_CHECKS} PARENT_SCOPE)
diff --git a/flang/tools/flang-tidy/FlangTidyForceLinker.h b/flang/tools/flang-tidy/FlangTidyForceLinker.h
index 3e9467013ea1c..b612329037f89 100644
--- a/flang/tools/flang-tidy/FlangTidyForceLinker.h
+++ b/flang/tools/flang-tidy/FlangTidyForceLinker.h
@@ -13,6 +13,11 @@
 
 namespace Fortran::tidy {
 
+// This anchor is used to force the linker to link the BugproneModule.
+extern volatile int BugproneModuleAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED BugproneModuleAnchorDestination =
+    BugproneModuleAnchorSource;
+
 } // namespace Fortran::tidy
 
 #endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYFORCELINKER_H
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
new file mode 100644
index 0000000000000..9db2db341fdd6
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -0,0 +1,32 @@
+//===--- BugproneTidyModule.cpp - flang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../FlangTidyModule.h"
+#include "../FlangTidyModuleRegistry.h"
+
+namespace Fortran::tidy {
+namespace bugprone {
+
+class BugproneModule : public FlangTidyModule {
+public:
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+};
+
+} // namespace bugprone
+
+// Register the BugproneTidyModule using this statically initialized variable.
+static FlangTidyModuleRegistry::Add<bugprone::BugproneModule>
+    X("bugprone-module", "Adds checks for bugprone code constructs.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the BugproneModule.
+
+// NOLINTNEXTLINE
+volatile int BugproneModuleAnchorSource = 0;
+
+} // namespace Fortran::tidy
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
new file mode 100644
index 0000000000000..e248b365cccfb
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -0,0 +1,17 @@
+add_flang_library(flangTidyBugproneModule STATIC
+  BugproneTidyModule.cpp
+
+  LINK_LIBS
+  FortranSupport
+  FortranParser
+  FortranEvaluate
+  FortranSemantics
+  LLVMSupport
+  flangTidy
+
+  LINK_COMPONENTS
+  Support
+  FrontendOpenMP
+  FrontendOpenACC
+  TargetParser
+  )

>From d2b5e75e27996b5f0816204d0af4a02e04257e8e Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 11:39:27 +0200
Subject: [PATCH 06/31] [flang-tidy] add modernize module

---
 flang/tools/flang-tidy/CMakeLists.txt         |  2 ++
 flang/tools/flang-tidy/FlangTidyForceLinker.h |  5 +++
 .../tools/flang-tidy/modernize/CMakeLists.txt | 18 +++++++++++
 .../modernize/ModernizeTidyModule.cpp         | 32 +++++++++++++++++++
 4 files changed, 57 insertions(+)
 create mode 100644 flang/tools/flang-tidy/modernize/CMakeLists.txt
 create mode 100644 flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp

diff --git a/flang/tools/flang-tidy/CMakeLists.txt b/flang/tools/flang-tidy/CMakeLists.txt
index 2c18eb741e22e..5af3bf4fe1d17 100644
--- a/flang/tools/flang-tidy/CMakeLists.txt
+++ b/flang/tools/flang-tidy/CMakeLists.txt
@@ -21,9 +21,11 @@ target_link_libraries(flangTidy
   )
 
 add_subdirectory(bugprone)
+add_subdirectory(modernize)
 
 set(ALL_FLANG_TIDY_CHECKS
   flangTidyBugproneModule
+  flangTidyModernizeModule
   )
 
 set(ALL_FLANG_TIDY_CHECKS ${ALL_FLANG_TIDY_CHECKS} PARENT_SCOPE)
diff --git a/flang/tools/flang-tidy/FlangTidyForceLinker.h b/flang/tools/flang-tidy/FlangTidyForceLinker.h
index b612329037f89..18194dd57f727 100644
--- a/flang/tools/flang-tidy/FlangTidyForceLinker.h
+++ b/flang/tools/flang-tidy/FlangTidyForceLinker.h
@@ -18,6 +18,11 @@ extern volatile int BugproneModuleAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED BugproneModuleAnchorDestination =
     BugproneModuleAnchorSource;
 
+// This anchor is used to force the linker to link the ModernizeModule.
+extern volatile int ModernizeModuleAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED ModernizeModuleAnchorDestination =
+    ModernizeModuleAnchorSource;
+
 } // namespace Fortran::tidy
 
 #endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYFORCELINKER_H
diff --git a/flang/tools/flang-tidy/modernize/CMakeLists.txt b/flang/tools/flang-tidy/modernize/CMakeLists.txt
new file mode 100644
index 0000000000000..bbb3098f76d9b
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_flang_library(flangTidyModernizeModule STATIC
+  ModernizeTidyModule.cpp
+
+  LINK_LIBS
+  FortranSupport
+  FortranParser
+  FortranEvaluate
+  FortranSemantics
+  LLVMSupport
+  flangTidy
+
+  LINK_COMPONENTS
+  Support
+  FrontendOpenMP
+  FrontendOpenACC
+  TargetParser
+  )
+
diff --git a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
new file mode 100644
index 0000000000000..f2717adac43ed
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
@@ -0,0 +1,32 @@
+//===--- ModernizeTidyModule.cpp - flang-tidy -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../FlangTidyModule.h"
+#include "../FlangTidyModuleRegistry.h"
+
+namespace Fortran::tidy {
+namespace modernize {
+
+class ModernizeModule : public FlangTidyModule {
+public:
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+};
+
+} // namespace modernize
+
+// Register the BugproneTidyModule using this statically initialized variable.
+static FlangTidyModuleRegistry::Add<modernize::ModernizeModule>
+    X("modernize-module", "Adds checks to enforce modern code style.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the ModernizeModule.
+
+// NOLINTNEXTLINE
+volatile int ModernizeModuleAnchorSource = 0;
+
+} // namespace Fortran::tidy

>From 590611f5f4081db63c30b748a02ad84e512478e3 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 11:46:32 +0200
Subject: [PATCH 07/31] [flang-tidy] add openmp module

---
 flang/tools/flang-tidy/CMakeLists.txt         |  2 ++
 flang/tools/flang-tidy/FlangTidyForceLinker.h |  5 +++
 flang/tools/flang-tidy/openmp/CMakeLists.txt  | 17 ++++++++++
 .../flang-tidy/openmp/OpenMPTidyModule.cpp    | 32 +++++++++++++++++++
 4 files changed, 56 insertions(+)
 create mode 100644 flang/tools/flang-tidy/openmp/CMakeLists.txt
 create mode 100644 flang/tools/flang-tidy/openmp/OpenMPTidyModule.cpp

diff --git a/flang/tools/flang-tidy/CMakeLists.txt b/flang/tools/flang-tidy/CMakeLists.txt
index 5af3bf4fe1d17..78d0564f411a7 100644
--- a/flang/tools/flang-tidy/CMakeLists.txt
+++ b/flang/tools/flang-tidy/CMakeLists.txt
@@ -22,10 +22,12 @@ target_link_libraries(flangTidy
 
 add_subdirectory(bugprone)
 add_subdirectory(modernize)
+add_subdirectory(openmp)
 
 set(ALL_FLANG_TIDY_CHECKS
   flangTidyBugproneModule
   flangTidyModernizeModule
+  flangTidyOpenMPModule
   )
 
 set(ALL_FLANG_TIDY_CHECKS ${ALL_FLANG_TIDY_CHECKS} PARENT_SCOPE)
diff --git a/flang/tools/flang-tidy/FlangTidyForceLinker.h b/flang/tools/flang-tidy/FlangTidyForceLinker.h
index 18194dd57f727..74a0f2c6f68b1 100644
--- a/flang/tools/flang-tidy/FlangTidyForceLinker.h
+++ b/flang/tools/flang-tidy/FlangTidyForceLinker.h
@@ -23,6 +23,11 @@ extern volatile int ModernizeModuleAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED ModernizeModuleAnchorDestination =
     ModernizeModuleAnchorSource;
 
+// This anchor is used to force the linker to link the OpenMPModule.
+extern volatile int OpenMPModuleAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED OpenMPModuleAnchorDestination =
+    OpenMPModuleAnchorSource;
+
 } // namespace Fortran::tidy
 
 #endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYFORCELINKER_H
diff --git a/flang/tools/flang-tidy/openmp/CMakeLists.txt b/flang/tools/flang-tidy/openmp/CMakeLists.txt
new file mode 100644
index 0000000000000..69e07c16a3c88
--- /dev/null
+++ b/flang/tools/flang-tidy/openmp/CMakeLists.txt
@@ -0,0 +1,17 @@
+add_flang_library(flangTidyOpenMPModule STATIC
+  OpenMPTidyModule.cpp
+
+  LINK_LIBS
+  FortranSupport
+  FortranParser
+  FortranEvaluate
+  FortranSemantics
+  LLVMSupport
+  flangTidy
+
+  LINK_COMPONENTS
+  Support
+  FrontendOpenMP
+  FrontendOpenACC
+  TargetParser
+  )
diff --git a/flang/tools/flang-tidy/openmp/OpenMPTidyModule.cpp b/flang/tools/flang-tidy/openmp/OpenMPTidyModule.cpp
new file mode 100644
index 0000000000000..456bb3dbc6fbf
--- /dev/null
+++ b/flang/tools/flang-tidy/openmp/OpenMPTidyModule.cpp
@@ -0,0 +1,32 @@
+//===--- OpenMPTidyModule.cpp - flang-tidy --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../FlangTidyModule.h"
+#include "../FlangTidyModuleRegistry.h"
+
+namespace Fortran::tidy {
+namespace openmp {
+
+class OpenMPModule : public FlangTidyModule {
+public:
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+};
+
+} // namespace openmp
+
+// Register the OpenMPTidyModule using this statically initialized variable.
+static FlangTidyModuleRegistry::Add<openmp::OpenMPModule>
+    X("openmp-module", "Adds checks for OpenMP parallelization.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the OpenMPTidyModule.
+
+// NOLINTNEXTLINE
+volatile int OpenMPModuleAnchorSource = 0;
+
+} // namespace Fortran::tidy

>From 503dac0e103699b8006261d6cb1a221867ea9400 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 11:50:27 +0200
Subject: [PATCH 08/31] [flang-tidy] add performance module

---
 flang/tools/flang-tidy/CMakeLists.txt         |  2 ++
 flang/tools/flang-tidy/FlangTidyForceLinker.h |  5 +++
 .../flang-tidy/performance/CMakeLists.txt     | 17 ++++++++++
 .../performance/PerformanceTidyModule.cpp     | 32 +++++++++++++++++++
 4 files changed, 56 insertions(+)
 create mode 100644 flang/tools/flang-tidy/performance/CMakeLists.txt
 create mode 100644 flang/tools/flang-tidy/performance/PerformanceTidyModule.cpp

diff --git a/flang/tools/flang-tidy/CMakeLists.txt b/flang/tools/flang-tidy/CMakeLists.txt
index 78d0564f411a7..8bb639403e14f 100644
--- a/flang/tools/flang-tidy/CMakeLists.txt
+++ b/flang/tools/flang-tidy/CMakeLists.txt
@@ -23,11 +23,13 @@ target_link_libraries(flangTidy
 add_subdirectory(bugprone)
 add_subdirectory(modernize)
 add_subdirectory(openmp)
+add_subdirectory(performance)
 
 set(ALL_FLANG_TIDY_CHECKS
   flangTidyBugproneModule
   flangTidyModernizeModule
   flangTidyOpenMPModule
+  flangTidyPerformanceModule
   )
 
 set(ALL_FLANG_TIDY_CHECKS ${ALL_FLANG_TIDY_CHECKS} PARENT_SCOPE)
diff --git a/flang/tools/flang-tidy/FlangTidyForceLinker.h b/flang/tools/flang-tidy/FlangTidyForceLinker.h
index 74a0f2c6f68b1..4d90045d69471 100644
--- a/flang/tools/flang-tidy/FlangTidyForceLinker.h
+++ b/flang/tools/flang-tidy/FlangTidyForceLinker.h
@@ -28,6 +28,11 @@ extern volatile int OpenMPModuleAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED OpenMPModuleAnchorDestination =
     OpenMPModuleAnchorSource;
 
+// This anchor is used to force the linker to link the PerformanceModule.
+extern volatile int PerformanceModuleAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED PerformanceModuleAnchorDestination =
+    PerformanceModuleAnchorSource;
+
 } // namespace Fortran::tidy
 
 #endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYFORCELINKER_H
diff --git a/flang/tools/flang-tidy/performance/CMakeLists.txt b/flang/tools/flang-tidy/performance/CMakeLists.txt
new file mode 100644
index 0000000000000..0a2690d24ea3a
--- /dev/null
+++ b/flang/tools/flang-tidy/performance/CMakeLists.txt
@@ -0,0 +1,17 @@
+add_flang_library(flangTidyPerformanceModule STATIC
+  PerformanceTidyModule.cpp
+
+  LINK_LIBS
+  FortranSupport
+  FortranParser
+  FortranEvaluate
+  FortranSemantics
+  LLVMSupport
+  flangTidy
+
+  LINK_COMPONENTS
+  Support
+  FrontendOpenMP
+  FrontendOpenACC
+  TargetParser
+  )
diff --git a/flang/tools/flang-tidy/performance/PerformanceTidyModule.cpp b/flang/tools/flang-tidy/performance/PerformanceTidyModule.cpp
new file mode 100644
index 0000000000000..70eda2ecf838e
--- /dev/null
+++ b/flang/tools/flang-tidy/performance/PerformanceTidyModule.cpp
@@ -0,0 +1,32 @@
+//===--- PerformanceTidyModule.cpp - flang-tidy ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../FlangTidyModule.h"
+#include "../FlangTidyModuleRegistry.h"
+
+namespace Fortran::tidy {
+namespace performance {
+
+class PerformanceTidyModule : public FlangTidyModule {
+public:
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+};
+
+} // namespace performance
+
+// Register the PerformanceTidyModule using this statically initialized
+// variable.
+static FlangTidyModuleRegistry::Add<performance::PerformanceTidyModule>
+    X("performance-module", "Performance Tidy Module");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the PerformanceModule.
+// NOLINTNEXTLINE
+volatile int PerformanceModuleAnchorSource = 0;
+
+} // namespace Fortran::tidy

>From 02bc28e15d1b104e4d777e8abc21a39af48b773e Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 11:53:57 +0200
Subject: [PATCH 09/31] [flang-tidy] add readability module

---
 flang/tools/flang-tidy/CMakeLists.txt         |  2 ++
 flang/tools/flang-tidy/FlangTidyForceLinker.h |  5 +++
 .../flang-tidy/readability/CMakeLists.txt     | 17 ++++++++++
 .../readability/ReadabilityTidyModule.cpp     | 33 +++++++++++++++++++
 4 files changed, 57 insertions(+)
 create mode 100644 flang/tools/flang-tidy/readability/CMakeLists.txt
 create mode 100644 flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp

diff --git a/flang/tools/flang-tidy/CMakeLists.txt b/flang/tools/flang-tidy/CMakeLists.txt
index 8bb639403e14f..080d43033b1f3 100644
--- a/flang/tools/flang-tidy/CMakeLists.txt
+++ b/flang/tools/flang-tidy/CMakeLists.txt
@@ -24,12 +24,14 @@ add_subdirectory(bugprone)
 add_subdirectory(modernize)
 add_subdirectory(openmp)
 add_subdirectory(performance)
+add_subdirectory(readability)
 
 set(ALL_FLANG_TIDY_CHECKS
   flangTidyBugproneModule
   flangTidyModernizeModule
   flangTidyOpenMPModule
   flangTidyPerformanceModule
+  flangTidyReadabilityModule
   )
 
 set(ALL_FLANG_TIDY_CHECKS ${ALL_FLANG_TIDY_CHECKS} PARENT_SCOPE)
diff --git a/flang/tools/flang-tidy/FlangTidyForceLinker.h b/flang/tools/flang-tidy/FlangTidyForceLinker.h
index 4d90045d69471..59c6754d6afcf 100644
--- a/flang/tools/flang-tidy/FlangTidyForceLinker.h
+++ b/flang/tools/flang-tidy/FlangTidyForceLinker.h
@@ -33,6 +33,11 @@ extern volatile int PerformanceModuleAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED PerformanceModuleAnchorDestination =
     PerformanceModuleAnchorSource;
 
+// This anchor is used to force the linker to link the ReadabilityModule.
+extern volatile int ReadabilityModuleAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED ReadabilityModuleAnchorDestination =
+    ReadabilityModuleAnchorSource;
+
 } // namespace Fortran::tidy
 
 #endif // LLVM_FLANG_TOOLS_FLANG_TIDY_FLANGTIDYFORCELINKER_H
diff --git a/flang/tools/flang-tidy/readability/CMakeLists.txt b/flang/tools/flang-tidy/readability/CMakeLists.txt
new file mode 100644
index 0000000000000..6f8e5df4127f2
--- /dev/null
+++ b/flang/tools/flang-tidy/readability/CMakeLists.txt
@@ -0,0 +1,17 @@
+add_flang_library(flangTidyReadabilityModule STATIC
+  ReadabilityTidyModule.cpp
+
+  LINK_LIBS
+  FortranSupport
+  FortranParser
+  FortranEvaluate
+  FortranSemantics
+  LLVMSupport
+  flangTidy
+
+  LINK_COMPONENTS
+  Support
+  FrontendOpenMP
+  FrontendOpenACC
+  TargetParser
+  )
diff --git a/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp b/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp
new file mode 100644
index 0000000000000..1f217dd5eea4f
--- /dev/null
+++ b/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp
@@ -0,0 +1,33 @@
+//===--- ReadabilityTidyModule.cpp - flang-tidy ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../FlangTidyModule.h"
+#include "../FlangTidyModuleRegistry.h"
+
+namespace Fortran::tidy {
+namespace readability {
+
+class ReadabilityTidyModule : public FlangTidyModule {
+public:
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+};
+
+} // namespace readability
+
+// Register the ReadabilityTidyModule using this statically initialized
+// variable.
+static FlangTidyModuleRegistry::Add<readability::ReadabilityTidyModule>
+    X("readability-module", "Adds checks for readability.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the ReadabilityTidyModule.
+
+// NOLINTNEXTLINE
+volatile int ReadabilityModuleAnchorSource = 0;
+
+} // namespace Fortran::tidy

>From db0bee30d07ae720ec4b4cf5520ec84ff4bff373 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 12:03:10 +0200
Subject: [PATCH 10/31] [flang-tidy] add documentation skeleton

---
 flang/tools/CMakeLists.txt                  |    6 +
 flang/tools/docs/CMakeLists.txt             |  103 +
 flang/tools/docs/ReleaseNotes.rst           |   64 +
 flang/tools/docs/conf.py                    |  253 ++
 flang/tools/docs/doxygen-mainpage.dox       |    9 +
 flang/tools/docs/doxygen.cfg.in             | 2294 +++++++++++++++++++
 flang/tools/docs/flang-tidy/checks/list.rst |   17 +
 flang/tools/docs/flang-tidy/index.rst       |   79 +
 flang/tools/docs/index.rst                  |   43 +
 9 files changed, 2868 insertions(+)
 create mode 100644 flang/tools/docs/CMakeLists.txt
 create mode 100644 flang/tools/docs/ReleaseNotes.rst
 create mode 100644 flang/tools/docs/conf.py
 create mode 100644 flang/tools/docs/doxygen-mainpage.dox
 create mode 100644 flang/tools/docs/doxygen.cfg.in
 create mode 100644 flang/tools/docs/flang-tidy/checks/list.rst
 create mode 100644 flang/tools/docs/flang-tidy/index.rst
 create mode 100644 flang/tools/docs/index.rst

diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 5524c1ef365a1..0b5b545638926 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -17,4 +17,10 @@ add_subdirectory(flang-tidy)
 
 if(FLANG_INCLUDE_TESTS)
   add_subdirectory(test)
+endif()
+
+option(FLANG_TOOLS_INCLUDE_DOCS "Generate build targets for the Flang Tools docs."
+  ${LLVM_INCLUDE_DOCS})
+if( FLANG_TOOLS_INCLUDE_DOCS )
+  add_subdirectory(docs)
 endif()
\ No newline at end of file
diff --git a/flang/tools/docs/CMakeLists.txt b/flang/tools/docs/CMakeLists.txt
new file mode 100644
index 0000000000000..b56a24c4fa6ec
--- /dev/null
+++ b/flang/tools/docs/CMakeLists.txt
@@ -0,0 +1,103 @@
+if (DOXYGEN_FOUND)
+  if (LLVM_ENABLE_DOXYGEN)
+    set(abs_srcdir ${CMAKE_CURRENT_SOURCE_DIR})
+    set(abs_builddir ${CMAKE_CURRENT_BINARY_DIR})
+
+    if (HAVE_DOT)
+      set(DOT ${LLVM_PATH_DOT})
+    endif()
+
+    if (LLVM_DOXYGEN_EXTERNAL_SEARCH)
+      set(enable_searchengine "YES")
+      set(searchengine_url "${LLVM_DOXYGEN_SEARCHENGINE_URL}")
+      set(enable_server_based_search "YES")
+      set(enable_external_search "YES")
+      set(extra_search_mappings "${LLVM_DOXYGEN_SEARCH_MAPPINGS}")
+    else()
+      set(enable_searchengine "NO")
+      set(searchengine_url "")
+      set(enable_server_based_search "NO")
+      set(enable_external_search "NO")
+      set(extra_search_mappings "")
+    endif()
+
+    # If asked, configure doxygen for the creation of a Qt Compressed Help file.
+    if (LLVM_ENABLE_DOXYGEN_QT_HELP)
+      set(FLANG_TOOLS_DOXYGEN_QCH_FILENAME "org.llvm.flang.qch" CACHE STRING
+        "Filename of the Qt Compressed help file")
+      set(FLANG_TOOLS_DOXYGEN_QHP_NAMESPACE "org.llvm.flang" CACHE STRING
+        "Namespace under which the intermediate Qt Help Project file lives")
+      set(FLANG_TOOLS_DOXYGEN_QHP_CUST_FILTER_NAME "Flang ${FLANG_VERSION}" CACHE STRING
+        "See http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-filters")
+      set(FLANG_TOOLS_DOXYGEN_QHP_CUST_FILTER_ATTRS "Flang,${FLANG_VERSION}" CACHE STRING
+        "See http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes")
+      set(flang_tools_doxygen_generate_qhp "YES")
+      set(flang_tools_doxygen_qch_filename "${FLANG_DOXYGEN_QCH_FILENAME}")
+      set(flang_tools_doxygen_qhp_namespace "${FLANG_DOXYGEN_QHP_NAMESPACE}")
+      set(flang_tools_doxygen_qhelpgenerator_path "${LLVM_DOXYGEN_QHELPGENERATOR_PATH}")
+      set(flang_tools_doxygen_qhp_cust_filter_name "${FLANG_DOXYGEN_QHP_CUST_FILTER_NAME}")
+      set(flang_tools_doxygen_qhp_cust_filter_attrs "${FLANG_DOXYGEN_QHP_CUST_FILTER_ATTRS}")
+    else()
+      set(flang_tools_doxygen_generate_qhp "NO")
+      set(flang_tools_doxygen_qch_filename "")
+      set(flang_tools_doxygen_qhp_namespace "")
+      set(flang_tools_doxygen_qhelpgenerator_path "")
+      set(flang_tools_doxygen_qhp_cust_filter_name "")
+      set(flang_tools_doxygen_qhp_cust_filter_attrs "")
+    endif()
+
+    option(LLVM_DOXYGEN_SVG
+      "Use svg instead of png files for doxygen graphs." OFF)
+    if (LLVM_DOXYGEN_SVG)
+      set(DOT_IMAGE_FORMAT "svg")
+    else()
+      set(DOT_IMAGE_FORMAT "png")
+    endif()
+
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/doxygen.cfg.in
+      ${CMAKE_CURRENT_BINARY_DIR}/doxygen.cfg @ONLY)
+
+    set(abs_top_srcdir)
+    set(abs_top_builddir)
+    set(DOT)
+    set(enable_searchengine)
+    set(searchengine_url)
+    set(enable_server_based_search)
+    set(enable_external_search)
+    set(extra_search_mappings)
+    set(flang_tools_doxygen_generate_qhp)
+    set(flang_tools_doxygen_qch_filename)
+    set(flang_tools_doxygen_qhp_namespace)
+    set(flang_tools_doxygen_qhelpgenerator_path)
+    set(flang_tools_doxygen_qhp_cust_filter_name)
+    set(flang_tools_doxygen_qhp_cust_filter_attrs)
+    set(DOT_IMAGE_FORMAT)
+
+    add_custom_target(doxygen-flang-tools
+      COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/doxygen.cfg
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      COMMENT "Generating flang doxygen documentation." VERBATIM)
+    set_target_properties(doxygen-flang-tools PROPERTIES FOLDER "Flang Tools/Docs")
+
+    if (LLVM_BUILD_DOCS)
+      add_dependencies(doxygen doxygen-flang-tools)
+    endif()
+
+    if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
+      install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/doxygen/html
+        DESTINATION docs/html)
+    endif()
+  endif()
+endif()
+
+if (LLVM_ENABLE_SPHINX)
+  include(AddSphinxTarget)
+  if (SPHINX_FOUND)
+    if (${SPHINX_OUTPUT_HTML})
+      add_sphinx_target(html flang-tools)
+    endif()
+    if (${SPHINX_OUTPUT_MAN})
+      add_sphinx_target(man flang-tools)
+    endif()
+  endif()
+endif()
diff --git a/flang/tools/docs/ReleaseNotes.rst b/flang/tools/docs/ReleaseNotes.rst
new file mode 100644
index 0000000000000..a4a69412881ef
--- /dev/null
+++ b/flang/tools/docs/ReleaseNotes.rst
@@ -0,0 +1,64 @@
+====================================================
+Flang Tools |release| |ReleaseNotesTitle|
+====================================================
+
+.. contents::
+   :local:
+   :depth: 3
+
+Written by the `LLVM Team <https://llvm.org/>`_
+
+.. only:: PreRelease
+
+  .. warning::
+     These are in-progress notes for the upcoming Flang Tools |version| release.
+     Release notes for previous releases can be found on
+     `the Download Page <https://releases.llvm.org/download.html>`_.
+
+Introduction
+============
+
+This document contains the release notes for the Flang Tools, part of the Flang
+release |release|. Here we describe the status of the Flang Tools in some
+detail, including major improvements from the previous release and new feature
+work. All LLVM releases may be downloaded from the `LLVM releases web site
+<https://llvm.org/releases/>`_.
+
+For more information about Flang or LLVM, including information about
+the latest release, please see the `Flang Web Site <https://flang.llvm.org>`_ or
+the `LLVM Web Site <https://llvm.org>`_.
+
+Note that if you are reading this file from a Git checkout or the
+main Flang web page, this document applies to the *next* release, not
+the current one. To see the release notes for a specific release, please
+see the `releases page <https://llvm.org/releases/>`_.
+
+What's New in Flang Tools |release|?
+====================================
+
+Some of the major new features and improvements to Extra Flang Tools are listed
+here. Generic improvements to Extra Flang Tools as a whole or to its underlying
+infrastructure are described first, followed by tool-specific sections.
+
+Major New Features
+------------------
+
+Improvements to flang-tidy
+--------------------------
+
+- New tool :program:`flang-tidy`.
+
+New checks
+^^^^^^^^^^
+
+New check aliases
+^^^^^^^^^^^^^^^^^
+
+Changes in existing checks
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Removed checks
+^^^^^^^^^^^^^^
+
+Miscellaneous
+^^^^^^^^^^^^^
diff --git a/flang/tools/docs/conf.py b/flang/tools/docs/conf.py
new file mode 100644
index 0000000000000..d0554262d314e
--- /dev/null
+++ b/flang/tools/docs/conf.py
@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+#
+# Flang Tools documentation build configuration file, created by
+# sphinx-quickstart on Wed Feb 13 10:00:18 2013.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+from datetime import date
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+# sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ["sphinx.ext.todo", "sphinx.ext.mathjax"]
+
+# Add any paths that contain templates here, relative to this directory.
+# templates_path = ["_templates"]
+
+# The suffix of source filenames.
+source_suffix = ".rst"
+
+# The encoding of source files.
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = "index"
+
+# General information about the project.
+project = "Flang Tools"
+copyright = "2007-%d, The Flang Team" % date.today().year
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ["_build"]
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "friendly"
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+in_progress_title = "(In-Progress) " if tags.has("PreRelease") else ""
+
+rst_epilog = f"""
+.. |ReleaseNotesTitle| replace:: {in_progress_title} Release Notes
+"""
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = "haiku"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+# html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ["_static"]
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+# html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_domain_indices = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "FlangToolsdoc"
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+    (
+        "index",
+        "FlangTools.tex",
+        "Flang Tools Documentation",
+        "The Flang Team",
+        "manual",
+    ),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (
+        "index",
+        "flangtools",
+        "Flang Tools Documentation",
+        ["The Flang Team"],
+        1,
+    )
+]
+
+# If true, show URL addresses after external links.
+# man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        "index",
+        "FlangTools",
+        "Flang Tools Documentation",
+        "The Flang Team",
+        "FlangTools",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
+]
+
+# Documents to append as an appendix to all manuals.
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+# texinfo_show_urls = 'footnote'
diff --git a/flang/tools/docs/doxygen-mainpage.dox b/flang/tools/docs/doxygen-mainpage.dox
new file mode 100644
index 0000000000000..275e7ed4ae903
--- /dev/null
+++ b/flang/tools/docs/doxygen-mainpage.dox
@@ -0,0 +1,9 @@
+/// \mainpage flang-tools
+///
+/// \section main_intro Introduction
+/// Welcome to flang tools.
+///
+/// This documentation describes the **internal** software that makes
+/// up flang tools, not the **external** use of flang tools. For
+/// usage instructions, please see the programmer's guide or reference
+/// manual.
diff --git a/flang/tools/docs/doxygen.cfg.in b/flang/tools/docs/doxygen.cfg.in
new file mode 100644
index 0000000000000..b5e7542d732cd
--- /dev/null
+++ b/flang/tools/docs/doxygen.cfg.in
@@ -0,0 +1,2294 @@
+# Doxyfile 1.8.6
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all text
+# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
+# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
+# for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING      = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = flang-tools
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PACKAGE_VERSION@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          =
+
+# With the PROJECT_LOGO tag one can specify a logo or icon that is included in
+# the documentation. The maximum height of the logo should not exceed 55 pixels
+# and the maximum width should not exceed 200 pixels. Doxygen will copy the logo
+# to the output directory.
+
+PROJECT_LOGO           =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       = @abs_builddir@/doxygen
+
+# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS         = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE        = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF           = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF       =
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC    = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB  = NO
+
+# If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES        = NO
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH        = ../..
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH    =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES            = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF      = YES
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF           = YES
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS           = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a
+# new page for each member. If set to NO, the documentation of a member will be
+# part of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES  = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE               = 2
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines.
+
+ALIASES                =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C  = NO
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA   = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN   = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL   = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
+# C#, C, C++, D, PHP, Objective-C, Python, Fortran, VHDL. For instance to make
+# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C
+# (default is Fortran), use: inc=Fortran f=C.
+#
+# Note For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING      =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT       = YES
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by by putting a % sign in front of the word
+# or globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT       = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT    = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT        = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT            = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT   = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES, then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC   = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING            = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS  = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT   = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE      = 2
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL            = YES
+
+# If the EXTRACT_PRIVATE tag is set to YES all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE        = NO
+
+# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE        = NO
+
+# If the EXTRACT_STATIC tag is set to YES all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC         = YES
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES  = YES
+
+# This flag is only useful for Objective-C code. When set to YES local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS  = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES   = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO these classes will be included in the various overviews. This option has
+# no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES     = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# (class|struct|union) declarations. If set to NO these declarations will be
+# included in the documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS  = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS      = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS          = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# and Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES       = YES
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES       = NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES     = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC  = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES   = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO            = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS       = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS        = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES       = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME     = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING  = NO
+
+# The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the
+# todo list. This list is created by putting \todo commands in the
+# documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST      = YES
+
+# The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the
+# test list. This list is created by putting \test commands in the
+# documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST      = YES
+
+# The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST       = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS       =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES  = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES the list
+# will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES        = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES             = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES        = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER    =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE            =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. Do not use file names with spaces, bibtex cannot handle them. See
+# also \cite for info how to create references.
+
+CITE_BIB_FILES         =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET                  = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS               = NO
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED   = NO
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR      = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO doxygen will only warn about wrong or incomplete parameter
+# documentation, but not about the absence of documentation.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC       = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT            = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE           =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces.
+# Note: If this tag is empty the current directory is searched.
+
+INPUT                  =  \
+                          @abs_srcdir@/../flang-tidy \
+                          @abs_srcdir@/doxygen-mainpage.dox
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank the
+# following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii,
+# *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp,
+# *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown,
+# *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,
+# *.qsf, *.as and *.js.
+
+FILE_PATTERNS          =
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       =
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS        =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH           =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS       =
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE      = YES
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH             =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+
+INPUT_FILTER           =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+
+FILTER_PATTERNS        =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER ) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES    = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER         = YES
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES         = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS    = NO
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# function all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = YES
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION    = YES
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES, then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS        = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see http://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS              = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS       = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX     = YES
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX    = 4
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX          = flang::
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML          = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT            = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION    = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER            =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER            =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET        =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify an additional user-
+# defined cascading style sheet that is included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefor more robust against future updates.
+# Doxygen will copy the style sheet file to the output directory. For an example
+# see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET  =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES       =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the stylesheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE    = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT    = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA  = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP         = YES
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS  = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: http://developer.apple.com/tools/xcode/), introduced with
+# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
+# for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET        = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME        = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID       = org.doxygen.Project
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID    = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME  = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP      = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE               =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler ( hhc.exe). If non-empty
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION           =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated (
+# YES) or that it should be included in the main .chm file ( NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI           = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING     =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated (
+# YES) or a normal table of contents ( NO) in the .chm file.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC             = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND             = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP           = @flang_tools_doxygen_generate_qhp@
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE               = @flang_tools_doxygen_qch_filename@
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE          = @flang_tools_doxygen_qhp_namespace@
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER     = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME   = @flang_tools_doxygen_qhp_cust_filter_name@
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS  = @flang_tools_doxygen_qhp_cust_filter_attrs@
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS  =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION           = @flang_tools_doxygen_qhelpgenerator_path@
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP   = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID         = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX          = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW      = NO
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH         = 250
+
+# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW    = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE       = 10
+
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT    = YES
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# http://www.mathjax.org) which uses client side Javascript for the rendering
+# instead of using prerendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX            = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT         = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from http://www.mathjax.org before deployment.
+# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS     =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE       =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE           = @enable_searchengine@
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using Javascript. There
+# are two flavours of web server based searching depending on the
+# EXTERNAL_SEARCH setting. When disabled, doxygen will generate a PHP script for
+# searching and an index file used by the script. When EXTERNAL_SEARCH is
+# enabled the indexing and searching needs to be provided by external tools. See
+# the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH    = @enable_server_based_search@
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer ( doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH        = @enable_external_search@
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer ( doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL       = @searchengine_url@
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE        = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID     = flang-tools
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS  = @extra_search_mappings@
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT           = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when enabling USE_PDFLATEX this option is only used for generating
+# bitmaps for formulas in the HTML output, but not in the Makefile that is
+# written to the output directory.
+# The default file is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME         = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME     = makeindex
+
+# If the COMPACT_LATEX tag is set to YES doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX          = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE             = a4
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. To get the times font for
+# instance you can specify
+# EXTRA_PACKAGES=times
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES         =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber. Doxygen will
+# replace them by respectively the title of the page, the current date and time,
+# only the current date, the version number of doxygen, the project name (see
+# PROJECT_NAME), or the project number (see PROJECT_NUMBER).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER           =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER           =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES      =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS         = YES
+
+# If the LATEX_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX           = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE        = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES     = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE      = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE        = plain
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF           = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT             = rtf
+
+# If the COMPACT_RTF tag is set to YES doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF            = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS         = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's config
+# file, i.e. a series of assignments. You only have to provide replacements,
+# missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE    =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's config file. A template extensions file can be generated
+# using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE    =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN           = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT             = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION          = .3
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS              = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML           = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT             = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING     = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK       = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT         = docbook
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES doxygen will generate an AutoGen
+# Definitions (see http://autogen.sf.net) file that captures the structure of
+# the code including all documentation. Note that this feature is still
+# experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF   = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD       = NO
+
+# If the PERLMOD_LATEX tag is set to YES doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX          = NO
+
+# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY         = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES doxygen will expand all macro names
+# in the source code. If set to NO only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = YES
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = YES
+
+# If the SEARCH_INCLUDES tag is set to YES the includes files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES        = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           = @abs_srcdir@/../../../include \
+                         @abs_srcdir@/../../../../../include
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS  =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED             =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have an
+# all uppercase name, and do not end with a semicolon. Such function macros are
+# typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS   = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have an unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES               =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE       =
+
+# If the ALLEXTERNALS tag is set to YES all external class will be listed in the
+# class index. If set to NO only the inherited external classes will be listed.
+# The default value is: NO.
+
+ALLEXTERNALS           = YES
+
+# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed in
+# the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS        = YES
+
+# If the EXTERNAL_PAGES tag is set to YES all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES         = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script
+# interpreter (i.e. the result of 'which perl').
+# The default file (with absolute path) is: /usr/bin/perl.
+
+PERL_PATH              = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS         = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see:
+# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
+# default search path.
+
+MSCGEN_PATH            =
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH               =
+
+# If set to YES, the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS   = NO
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: NO.
+
+HAVE_DOT               = YES
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS        = 0
+
+# When you want a differently looking font n the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME           = Helvetica
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE           = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH           =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH            = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH    = NO
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS           = YES
+
+# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK               = NO
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS   = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS     = YES
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH          = NO
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH      = NO
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH             = NO
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH           = NO
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY    = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH        = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot.
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, jpg, gif and svg.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT       = @DOT_IMAGE_FORMAT@
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG        = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH               = @DOT@
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS           =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS           =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS           =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES    = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH    = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT        = YES
+
+# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS      = YES
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND        = YES
+
+# If the DOT_CLEANUP tag is set to YES doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP            = YES
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
new file mode 100644
index 0000000000000..5190645af9f31
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -0,0 +1,17 @@
+.. title:: flang-tidy - Flang-Tidy Checks
+
+Flang-Tidy Checks
+=================
+
+.. toctree::
+   :glob:
+   :hidden:
+
+   bugprone/*
+   modernize/*
+   openmp/*
+   performance/*
+   readability/*
+
+.. csv-table::
+   :header: "Name", "Offers fixes"
diff --git a/flang/tools/docs/flang-tidy/index.rst b/flang/tools/docs/flang-tidy/index.rst
new file mode 100644
index 0000000000000..29145f4050f7d
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/index.rst
@@ -0,0 +1,79 @@
+==========
+Flang-Tidy
+==========
+
+.. contents::
+
+See also:
+
+.. toctree::
+   :maxdepth: 1
+
+   The list of flang-tidy checks <checks/list>
+
+:program:`flang-tidy` is a flang-based Fortran "linter" tool. Its purpose is to
+provide an extensible framework for diagnosing and fixing typical programming
+errors, like style violations, interface misuse, or bugs that can be deduced via
+static analysis. :program:`flang-tidy` is modular and provides a convenient
+interface for writing new checks.
+
+
+Using flang-tidy
+================
+
+:program:`flang-tidy` is a flang-based tool, and it's easier to work with if you
+set up a compile command database for your project. You can also specify
+compilation options on the command line after ``--``:
+
+.. code-block:: console
+
+  $ flang-tidy test.cpp -- -Imy_project/include -DMY_DEFINES ...
+
+:program:`flang-tidy` has its own checks. Each check has a name and the checks
+to run can be chosen using the ``-checks=`` option, which specifies a
+comma-separated list of positive and negative (prefixed with ``-``)
+globs. Positive globs add subsets of checks, and negative globs remove them. For
+example,
+
+.. code-block:: console
+
+  $ flang-tidy test.cpp -checks=-*,bugprone-*,-bugprone-arithmetic-*
+
+will disable all default checks (``-*``) and enable all ``bugprone-*``
+checks except for ``bugprone-arithmetic-*`` ones.
+
+The ``-list-checks`` option lists all the enabled checks. When used without
+``-checks=``, it shows checks enabled by default. Use ``-checks=*`` to see all
+available checks or with any other value of ``-checks=`` to see which checks are
+enabled by this value.
+
+There are currently the following groups of checks:
+
+====================== =========================================================
+Name prefix            Description
+====================== =========================================================
+``bugprone-``          Checks that target bug-prone code constructs.
+``modernize-``         Checks that advocate usage of modern language constructs.
+``openmp-``            Checks that target OpenMP constructs.
+``performance-``       Checks that target performance-related issues.
+``readability-``       Checks that target readability-related issues.
+====================== =========================================================
+
+Flang diagnostics are not treated by :program:`flang-tidy` and can currently not
+be filtered out using the ``-checks=`` option.
+
+
+Suppressing Undesired Diagnostics
+=================================
+
+:program:`flang-tidy` diagnostics are intended to call out code that does not
+adhere to a coding standard, or is otherwise problematic in some way. However,
+if the code is known to be correct, it may be useful to silence the warning.
+
+If a specific suppression mechanism is not available for a certain warning, or
+its use is not desired for some reason, :program:`flang-tidy` has a generic
+mechanism to suppress diagnostics using ``NOLINT``.
+
+The ``NOLINT`` comment instructs :program:`flang-tidy` to ignore warnings on the
+*same line* (it doesn't apply to a function, a block of code or any other
+language construct; it applies to the line of code it is on).
diff --git a/flang/tools/docs/index.rst b/flang/tools/docs/index.rst
new file mode 100644
index 0000000000000..300b0520fe274
--- /dev/null
+++ b/flang/tools/docs/index.rst
@@ -0,0 +1,43 @@
+.. title:: Welcome to Flang Tools's documentation!
+
+Introduction
+============
+Welcome to the flang-tools project which contains tools built using
+Flang's tooling APIs.
+
+.. toctree::
+   :maxdepth: 1
+
+   ReleaseNotes
+
+Contents
+========
+.. toctree::
+   :maxdepth: 2
+
+   flang-tidy/index
+
+
+Doxygen Documentation
+=====================
+The Doxygen documentation describes the **internal** software that makes up the
+tools of flang-tools, not the **external** use of these tools. The Doxygen
+documentation contains no instructions about how to use the tools, only the APIs
+that make up the software. For usage instructions, please see the user's guide
+or reference manual for each tool.
+
+* `Doxygen documentation`_
+
+.. _`Doxygen documentation`: doxygen/annotated.html
+
+.. note::
+    This documentation is generated directly from the source code with doxygen.
+    Since the tools of flang-tools are constantly under active
+    development, what you're about to read is out of date!
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`

>From 098be98c20d658234d20311a2cc62e807ffda1bf Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 12:58:02 +0200
Subject: [PATCH 11/31] [flang-tidy][bugprone] add arithmetic goto check

---
 .../checks/bugprone/arithmetic-goto.rst       | 19 ++++++++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  2 ++
 .../bugprone/ArithmeticGotoCheck.cpp          | 21 +++++++++++++
 .../flang-tidy/bugprone/ArithmeticGotoCheck.h | 30 +++++++++++++++++++
 .../bugprone/BugproneTidyModule.cpp           |  6 +++-
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../tools/test/flang-tidy/arithmetic-goto.f90 | 10 +++++++
 7 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/arithmetic-goto.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/ArithmeticGotoCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/ArithmeticGotoCheck.h
 create mode 100644 flang/tools/test/flang-tidy/arithmetic-goto.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/arithmetic-goto.rst b/flang/tools/docs/flang-tidy/checks/bugprone/arithmetic-goto.rst
new file mode 100644
index 0000000000000..eaf515d6dc531
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/arithmetic-goto.rst
@@ -0,0 +1,19 @@
+.. title:: flang-tidy - bugprone-arithmetic-goto
+
+bugprone-arithmetic-goto
+========================
+
+Warns about the use of computed GOTO statements, which are considered problematic for code maintainability and clarity. The arithmetic (computed) GOTO is an obsolescent feature in Fortran that allows for complex and hard-to-maintain transfer of control logic. Modern code should use more structured control statements instead.
+
+.. code-block:: fortran
+
+    program example
+      integer :: i = 2
+      go to (10, 20, 30), i  ! This will trigger a warning
+    10 print *, "Label 10"
+      stop
+    20 print *, "Label 20"
+      stop
+    30 print *, "Label 30"
+      stop
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 5190645af9f31..e942c5021bc9c 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -15,3 +15,5 @@ Flang-Tidy Checks
 
 .. csv-table::
    :header: "Name", "Offers fixes"
+
+   :doc:`bugprone-arithmetic-goto <bugprone/arithmetic-goto>`,
diff --git a/flang/tools/flang-tidy/bugprone/ArithmeticGotoCheck.cpp b/flang/tools/flang-tidy/bugprone/ArithmeticGotoCheck.cpp
new file mode 100644
index 0000000000000..3b9436e423b04
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ArithmeticGotoCheck.cpp
@@ -0,0 +1,21 @@
+//===--- ArithmeticGotoCheck.cpp - flang-tidy -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ArithmeticGotoCheck.h"
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+void ArithmeticGotoCheck::Enter(const parser::ComputedGotoStmt &gotoStmt) {
+  if (context()->getSemanticsContext().location().has_value()) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Arithmetic goto statements are not recommended"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/ArithmeticGotoCheck.h b/flang/tools/flang-tidy/bugprone/ArithmeticGotoCheck.h
new file mode 100644
index 0000000000000..eff2296290cc6
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ArithmeticGotoCheck.h
@@ -0,0 +1,30 @@
+//===--- ArithmeticGotoCheck.h - flang-tidy ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_ARITHMETICGOTOCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_ARITHMETICGOTOCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check detects the use of arithmetic GOTO statements in Fortran code.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/bugprone-arithmetic-goto.html
+class ArithmeticGotoCheck : public FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~ArithmeticGotoCheck() = default;
+  void Enter(const parser::ComputedGotoStmt &) override;
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_ARITHMETICGOTOCHECK_H
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index 9db2db341fdd6..3e9a269f59096 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -8,13 +8,17 @@
 
 #include "../FlangTidyModule.h"
 #include "../FlangTidyModuleRegistry.h"
+#include "ArithmeticGotoCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
 
 class BugproneModule : public FlangTidyModule {
 public:
-  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {
+    CheckFactories.registerCheck<ArithmeticGotoCheck>(
+        "bugprone-arithmetic-goto");
+  }
 };
 
 } // namespace bugprone
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index e248b365cccfb..db7c3316e1519 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_flang_library(flangTidyBugproneModule STATIC
+  ArithmeticGotoCheck.cpp
   BugproneTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/test/flang-tidy/arithmetic-goto.f90 b/flang/tools/test/flang-tidy/arithmetic-goto.f90
new file mode 100644
index 0000000000000..9418973318aad
--- /dev/null
+++ b/flang/tools/test/flang-tidy/arithmetic-goto.f90
@@ -0,0 +1,10 @@
+! RUN: %check_flang_tidy %s bugprone-arithmetic-goto %t
+subroutine s(i)
+  integer, intent(in) :: i
+  goto (10, 20, 30), i
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Arithmetic goto statements are not recommended
+
+  10 continue
+  20 continue
+  30 continue
+end subroutine s

>From 1b7486f049e16a0ac193b65b1c1e0bf6f34b0986 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:01:36 +0200
Subject: [PATCH 12/31] [flang-tidy][bugprone] add arithmetic if check

---
 .../checks/bugprone/arithmetic-if.rst         | 19 ++++++++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/ArithmeticIfStmtCheck.cpp        | 21 +++++++++++++
 .../bugprone/ArithmeticIfStmtCheck.h          | 30 +++++++++++++++++++
 .../bugprone/BugproneTidyModule.cpp           |  3 ++
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 flang/tools/test/flang-tidy/arithmetic-if.f90 | 12 ++++++++
 7 files changed, 87 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/arithmetic-if.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/ArithmeticIfStmtCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/ArithmeticIfStmtCheck.h
 create mode 100644 flang/tools/test/flang-tidy/arithmetic-if.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/arithmetic-if.rst b/flang/tools/docs/flang-tidy/checks/bugprone/arithmetic-if.rst
new file mode 100644
index 0000000000000..ffa6c39646fdb
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/arithmetic-if.rst
@@ -0,0 +1,19 @@
+.. title:: flang-tidy - bugprone-arithmetic-if
+
+bugprone-arithmetic-if
+======================
+
+Detects the use of arithmetic IF statements, which are obsolescent in Fortran and should be replaced with more structured IF constructs for better readability and maintainability.
+
+.. code-block:: fortran
+
+    program example
+      real :: x = -1.0
+      if (x) 10, 20, 30  ! This will trigger a warning
+    10 print *, "X is negative"
+      stop
+    20 print *, "X is zero"
+      stop
+    30 print *, "X is positive"
+      stop
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index e942c5021bc9c..395499eb445cf 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -17,3 +17,4 @@ Flang-Tidy Checks
    :header: "Name", "Offers fixes"
 
    :doc:`bugprone-arithmetic-goto <bugprone/arithmetic-goto>`,
+    :doc:`bugprone-arithmetic-if <bugprone/arithmetic-if>`,
diff --git a/flang/tools/flang-tidy/bugprone/ArithmeticIfStmtCheck.cpp b/flang/tools/flang-tidy/bugprone/ArithmeticIfStmtCheck.cpp
new file mode 100644
index 0000000000000..b9d876a796fdf
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ArithmeticIfStmtCheck.cpp
@@ -0,0 +1,21 @@
+//===--- ArithmeticIfStmtCheck.cpp - flang-tidy ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ArithmeticIfStmtCheck.h"
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+void ArithmeticIfStmtCheck::Enter(const parser::ArithmeticIfStmt &ifStmt) {
+  if (context()->getSemanticsContext().location().has_value()) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Arithmetic if statements are not recommended"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/ArithmeticIfStmtCheck.h b/flang/tools/flang-tidy/bugprone/ArithmeticIfStmtCheck.h
new file mode 100644
index 0000000000000..cb45a69520547
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ArithmeticIfStmtCheck.h
@@ -0,0 +1,30 @@
+//===--- ArithmeticIfStmtCheck.h - flang-tidy -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_ARITHMETICIFSTMTCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_ARITHMETICIFSTMTCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check detects the use of arithmetic IF statements.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/arithmetic-if-statement.html
+class ArithmeticIfStmtCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~ArithmeticIfStmtCheck() = default;
+  void Enter(const parser::ArithmeticIfStmt &) override;
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_ARITHMETICIFSTMTCHECK_H
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index 3e9a269f59096..e1bc649b2f003 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -9,6 +9,7 @@
 #include "../FlangTidyModule.h"
 #include "../FlangTidyModuleRegistry.h"
 #include "ArithmeticGotoCheck.h"
+#include "ArithmeticIfStmtCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -18,6 +19,8 @@ class BugproneModule : public FlangTidyModule {
   void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {
     CheckFactories.registerCheck<ArithmeticGotoCheck>(
         "bugprone-arithmetic-goto");
+    CheckFactories.registerCheck<ArithmeticIfStmtCheck>(
+        "bugprone-arithmetic-if");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index db7c3316e1519..db3ca7c45eb72 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_flang_library(flangTidyBugproneModule STATIC
   ArithmeticGotoCheck.cpp
+  ArithmeticIfStmtCheck.cpp
   BugproneTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/test/flang-tidy/arithmetic-if.f90 b/flang/tools/test/flang-tidy/arithmetic-if.f90
new file mode 100644
index 0000000000000..3160abe170a2a
--- /dev/null
+++ b/flang/tools/test/flang-tidy/arithmetic-if.f90
@@ -0,0 +1,12 @@
+! RUN: %check_flang_tidy %s bugprone-arithmetic-if %t
+subroutine s
+  real :: x = -1.0
+  if (x) 10, 20, 30
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Arithmetic if statements are not recommended
+10 print *, "X is negative"
+   stop
+20 print *, "X is zero"
+   stop
+30 print *, "X is positive"
+   stop
+end subroutine s

>From 81ca658d585a5a8765f590612f3b7d3207d12dda Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:05:52 +0200
Subject: [PATCH 13/31] [flang-tidy][bugprone] add contiguous array check

---
 .../checks/bugprone/contiguous-array.rst      | 20 +++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  3 +-
 .../bugprone/BugproneTidyModule.cpp           |  3 +
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../bugprone/ContiguousArrayCheck.cpp         | 55 +++++++++++++++++++
 .../bugprone/ContiguousArrayCheck.h           | 34 ++++++++++++
 flang/tools/test/flang-tidy/contiguous01.f90  | 13 +++++
 7 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/contiguous-array.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/ContiguousArrayCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/ContiguousArrayCheck.h
 create mode 100644 flang/tools/test/flang-tidy/contiguous01.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/contiguous-array.rst b/flang/tools/docs/flang-tidy/checks/bugprone/contiguous-array.rst
new file mode 100644
index 0000000000000..fbd223e7caeb5
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/contiguous-array.rst
@@ -0,0 +1,20 @@
+.. title:: flang-tidy - bugprone-contiguous-array
+
+bugprone-contiguous-array
+=========================
+
+Verifies that assumed-shape arrays passed to interfaces are declared with the CONTIGUOUS attribute. Using contiguous arrays can lead to better performance by allowing the compiler to optimize memory access patterns.
+
+.. code-block:: fortran
+
+    module example
+      interface
+        subroutine contig(A)
+          real, contiguous, intent(in) :: A(:)
+        end subroutine contig
+
+        subroutine possibly_noncontig(A) ! This will trigger a warning
+          real, intent(in) :: A(:)
+        end subroutine possibly_noncontig
+      end interface
+    end module example
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 395499eb445cf..3ef288be8b6f3 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -17,4 +17,5 @@ Flang-Tidy Checks
    :header: "Name", "Offers fixes"
 
    :doc:`bugprone-arithmetic-goto <bugprone/arithmetic-goto>`,
-    :doc:`bugprone-arithmetic-if <bugprone/arithmetic-if>`,
+   :doc:`bugprone-arithmetic-if <bugprone/arithmetic-if>`,
+   :doc:`bugprone-contiguous-array <bugprone/contiguous-array>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index e1bc649b2f003..2e717fc957982 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -10,6 +10,7 @@
 #include "../FlangTidyModuleRegistry.h"
 #include "ArithmeticGotoCheck.h"
 #include "ArithmeticIfStmtCheck.h"
+#include "ContiguousArrayCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -21,6 +22,8 @@ class BugproneModule : public FlangTidyModule {
         "bugprone-arithmetic-goto");
     CheckFactories.registerCheck<ArithmeticIfStmtCheck>(
         "bugprone-arithmetic-if");
+    CheckFactories.registerCheck<ContiguousArrayCheck>(
+        "bugprone-contiguous-array");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index db3ca7c45eb72..8e8826b336a24 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -2,6 +2,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   ArithmeticGotoCheck.cpp
   ArithmeticIfStmtCheck.cpp
   BugproneTidyModule.cpp
+  ContiguousArrayCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/ContiguousArrayCheck.cpp b/flang/tools/flang-tidy/bugprone/ContiguousArrayCheck.cpp
new file mode 100644
index 0000000000000..d96a443458109
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ContiguousArrayCheck.cpp
@@ -0,0 +1,55 @@
+//===--- ContiguousArrayCheck.cpp - flang-tidy ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ContiguousArrayCheck.h"
+#include "flang/Semantics/symbol.h"
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+
+void ContiguousArrayCheck::CheckForContinguousArray(
+    semantics::SemanticsContext &context, const semantics::Scope &scope) {
+  if (scope.IsModuleFile())
+    return;
+
+  for (const auto &pair : scope) {
+    const semantics::Symbol &symbol{*pair.second};
+
+    if (const auto *details{symbol.detailsIf<semantics::SubprogramDetails>()};
+        details && details->isInterface()) {
+      for (const auto &dummyArg : details->dummyArgs()) {
+        if (!dummyArg)
+          continue;
+
+        if (const auto *details{
+                dummyArg->detailsIf<semantics::ObjectEntityDetails>()};
+            details && details->IsAssumedShape() &&
+            !dummyArg->attrs().test(semantics::Attr::CONTIGUOUS)) {
+          const auto dummySymbol{dummyArg->GetUltimate()};
+          Say(dummySymbol.name(),
+              "assumed-shape array '%s' should be contiguous"_warn_en_US,
+              dummySymbol.name());
+        }
+      }
+    }
+  }
+
+  for (const semantics::Scope &child : scope.children()) {
+    CheckForContinguousArray(context, child);
+  }
+}
+
+ContiguousArrayCheck::ContiguousArrayCheck(llvm::StringRef name,
+                                           FlangTidyContext *context)
+    : FlangTidyCheck(name, context) {
+  CheckForContinguousArray(context->getSemanticsContext(),
+                           context->getSemanticsContext().globalScope());
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/ContiguousArrayCheck.h b/flang/tools/flang-tidy/bugprone/ContiguousArrayCheck.h
new file mode 100644
index 0000000000000..457b48275fd0e
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ContiguousArrayCheck.h
@@ -0,0 +1,34 @@
+//===--- ContiguousArrayCheck.h - flang-tidy --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_CONTIGUOUSARRAYCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_CONTIGUOUSARRAYCHECK_H
+
+#include "../FlangTidyCheck.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check verifies that assumed-shape arrays passed to interfaces are
+/// contiguous.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/contiguous-array.html
+class ContiguousArrayCheck : public FlangTidyCheck {
+public:
+  explicit ContiguousArrayCheck(llvm::StringRef name,
+                                FlangTidyContext *context);
+  virtual ~ContiguousArrayCheck() = default;
+
+private:
+  void CheckForContinguousArray(semantics::SemanticsContext &,
+                                const semantics::Scope &);
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_CONTIGUOUSARRAYCHECK_H
diff --git a/flang/tools/test/flang-tidy/contiguous01.f90 b/flang/tools/test/flang-tidy/contiguous01.f90
new file mode 100644
index 0000000000000..139a5674defea
--- /dev/null
+++ b/flang/tools/test/flang-tidy/contiguous01.f90
@@ -0,0 +1,13 @@
+ ! RUN: %check_flang_tidy %s bugprone-contiguous-array %t
+module test
+  interface
+     subroutine contig(a)
+       real, contiguous, intent(in) :: a(:)
+     end subroutine contig
+
+     ! CHECK-MESSAGES: :[[@LINE+2]]:28: warning: assumed-shape array 'a' should be contiguous
+     subroutine possibly_noncontig(a)
+       real, intent(in) :: a(:)
+     end subroutine possibly_noncontig
+  end interface
+end module test

>From 20cf06463aee7263b2117908f2e3862dceb20b7a Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:08:40 +0200
Subject: [PATCH 14/31] [flang-tidy][bugprone] add implicit decl check

---
 .../checks/bugprone/implicit-declaration.rst  | 16 ++++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/BugproneTidyModule.cpp           |  3 ++
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../flang-tidy/bugprone/ImplicitDeclCheck.cpp | 41 +++++++++++++++++++
 .../flang-tidy/bugprone/ImplicitDeclCheck.h   | 34 +++++++++++++++
 flang/tools/test/flang-tidy/implicit01.f90    |  8 ++++
 flang/tools/test/flang-tidy/implicit02.f90    |  5 +++
 8 files changed, 109 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/implicit-declaration.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/ImplicitDeclCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/ImplicitDeclCheck.h
 create mode 100644 flang/tools/test/flang-tidy/implicit01.f90
 create mode 100644 flang/tools/test/flang-tidy/implicit02.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/implicit-declaration.rst b/flang/tools/docs/flang-tidy/checks/bugprone/implicit-declaration.rst
new file mode 100644
index 0000000000000..a9894bf6030b3
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/implicit-declaration.rst
@@ -0,0 +1,16 @@
+.. title:: flang-tidy - bugprone-implicit-declaration
+
+bugprone-implicit-declaration
+=============================
+
+Warns about implicitly declared variables, which can lead to subtle bugs. All variables should be explicitly declared to catch typos and ensure proper typing.
+
+.. code-block:: fortran
+
+    program example
+      ! Missing: implicit none
+      integer :: i
+      i = 10
+      j = 20  ! This will trigger a warning - j is implicitly declared
+      print *, i + j
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 3ef288be8b6f3..82d64e00ed6a3 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -19,3 +19,4 @@ Flang-Tidy Checks
    :doc:`bugprone-arithmetic-goto <bugprone/arithmetic-goto>`,
    :doc:`bugprone-arithmetic-if <bugprone/arithmetic-if>`,
    :doc:`bugprone-contiguous-array <bugprone/contiguous-array>`,
+   :doc:`bugprone-implicit-declaration <bugprone/implicit-declaration>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index 2e717fc957982..514a3d24fbe56 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -11,6 +11,7 @@
 #include "ArithmeticGotoCheck.h"
 #include "ArithmeticIfStmtCheck.h"
 #include "ContiguousArrayCheck.h"
+#include "ImplicitDeclCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -24,6 +25,8 @@ class BugproneModule : public FlangTidyModule {
         "bugprone-arithmetic-if");
     CheckFactories.registerCheck<ContiguousArrayCheck>(
         "bugprone-contiguous-array");
+    CheckFactories.registerCheck<ImplicitDeclCheck>(
+        "bugprone-implicit-declaration");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index 8e8826b336a24..c67ccc96baceb 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -3,6 +3,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   ArithmeticIfStmtCheck.cpp
   BugproneTidyModule.cpp
   ContiguousArrayCheck.cpp
+  ImplicitDeclCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/ImplicitDeclCheck.cpp b/flang/tools/flang-tidy/bugprone/ImplicitDeclCheck.cpp
new file mode 100644
index 0000000000000..8df1ecfafdfd8
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ImplicitDeclCheck.cpp
@@ -0,0 +1,41 @@
+//===--- ImplicitDeclCheck.cpp - flang-tidy -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ImplicitDeclCheck.h"
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+void ImplicitDeclCheck::CheckForImplicitDeclarations(
+    semantics::SemanticsContext &context, const semantics::Scope &scope) {
+  if (scope.IsModuleFile())
+    return;
+
+  for (const auto &pair : scope) {
+    const semantics::Symbol &symbol = *pair.second;
+    if (symbol.test(semantics::Symbol::Flag::Implicit) &&
+        !symbol.test(semantics::Symbol::Flag::Function) &&
+        !symbol.test(semantics::Symbol::Flag::Subroutine)) {
+      Say(symbol.name(), "Implicit declaration of symbol '%s'"_warn_en_US,
+          symbol.name());
+    }
+  }
+
+  for (const semantics::Scope &child : scope.children()) {
+    CheckForImplicitDeclarations(context, child);
+  }
+}
+
+ImplicitDeclCheck::ImplicitDeclCheck(llvm::StringRef name,
+                                     FlangTidyContext *context)
+    : FlangTidyCheck{name, context} {
+  CheckForImplicitDeclarations(context->getSemanticsContext(),
+                               context->getSemanticsContext().globalScope());
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/ImplicitDeclCheck.h b/flang/tools/flang-tidy/bugprone/ImplicitDeclCheck.h
new file mode 100644
index 0000000000000..a95c025a7977b
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ImplicitDeclCheck.h
@@ -0,0 +1,34 @@
+//===--- ImplicitDeclCheck.h - flang-tidy -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_IMPLICITDECLCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_IMPLICITDECLCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "../FlangTidyContext.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check verifies that all variables are declared before use.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/implicit-declaration.html
+class ImplicitDeclCheck : public virtual FlangTidyCheck {
+public:
+  ImplicitDeclCheck(llvm::StringRef name, FlangTidyContext *context);
+  virtual ~ImplicitDeclCheck() = default;
+
+private:
+  void CheckForImplicitDeclarations(semantics::SemanticsContext &,
+                                    const semantics::Scope &);
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_IMPLICITDECLCHECK_H
diff --git a/flang/tools/test/flang-tidy/implicit01.f90 b/flang/tools/test/flang-tidy/implicit01.f90
new file mode 100644
index 0000000000000..fd45500fb0d74
--- /dev/null
+++ b/flang/tools/test/flang-tidy/implicit01.f90
@@ -0,0 +1,8 @@
+! RUN: %check_flang_tidy %s bugprone-implicit-declaration %t
+subroutine s
+  integer :: i, j
+  i = 1
+  j = 2
+  k = i + j
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Implicit declaration of symbol 'k'
+end subroutine s
diff --git a/flang/tools/test/flang-tidy/implicit02.f90 b/flang/tools/test/flang-tidy/implicit02.f90
new file mode 100644
index 0000000000000..f793288c23174
--- /dev/null
+++ b/flang/tools/test/flang-tidy/implicit02.f90
@@ -0,0 +1,5 @@
+! RUN: %check_flang_tidy %s bugprone-implicit-declaration %t
+subroutine s(a, n)
+  real, intent(in) :: a(n)
+  ! CHECK-MESSAGES: :[[@LINE-2]]:17: warning: Implicit declaration of symbol 'n'
+end

>From cb9280d045aff4280ddcfdaafc06a7c9cec21e39 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:13:30 +0200
Subject: [PATCH 15/31] [flang-tidy][bugprone] add implied save check

---
 .../checks/bugprone/implied-save.rst          | 15 ++++++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/BugproneTidyModule.cpp           |  2 ++
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../flang-tidy/bugprone/ImpliedSaveCheck.cpp  | 30 +++++++++++++++++++
 .../flang-tidy/bugprone/ImpliedSaveCheck.h    | 30 +++++++++++++++++++
 .../tools/test/flang-tidy/implied-save01.f90  | 13 ++++++++
 7 files changed, 92 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/implied-save.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/ImpliedSaveCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/ImpliedSaveCheck.h
 create mode 100644 flang/tools/test/flang-tidy/implied-save01.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/implied-save.rst b/flang/tools/docs/flang-tidy/checks/bugprone/implied-save.rst
new file mode 100644
index 0000000000000..197967327647a
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/implied-save.rst
@@ -0,0 +1,15 @@
+.. title:: flang-tidy - bugprone-implied-save
+
+bugprone-implied-save
+=====================
+
+Identifies variables that are implicitly saved between procedure invocations. Variables with implied SAVE status can cause unexpected behavior and side effects, particularly in parallel code.
+
+.. code-block:: fortran
+
+    subroutine example
+      integer :: counter = 0  ! This will trigger a warning - implied SAVE
+      ! Should be: integer, save :: counter = 0
+      counter = counter + 1
+      print *, "Called", counter, "times"
+    end subroutine
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 82d64e00ed6a3..784bd7f3b8be2 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -20,3 +20,4 @@ Flang-Tidy Checks
    :doc:`bugprone-arithmetic-if <bugprone/arithmetic-if>`,
    :doc:`bugprone-contiguous-array <bugprone/contiguous-array>`,
    :doc:`bugprone-implicit-declaration <bugprone/implicit-declaration>`,
+   :doc:`bugprone-implied-save <bugprone/implied-save>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index 514a3d24fbe56..1e15cf4b425f5 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -12,6 +12,7 @@
 #include "ArithmeticIfStmtCheck.h"
 #include "ContiguousArrayCheck.h"
 #include "ImplicitDeclCheck.h"
+#include "ImpliedSaveCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -27,6 +28,7 @@ class BugproneModule : public FlangTidyModule {
         "bugprone-contiguous-array");
     CheckFactories.registerCheck<ImplicitDeclCheck>(
         "bugprone-implicit-declaration");
+    CheckFactories.registerCheck<ImpliedSaveCheck>("bugprone-implied-save");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index c67ccc96baceb..5d88df08c836b 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -4,6 +4,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   BugproneTidyModule.cpp
   ContiguousArrayCheck.cpp
   ImplicitDeclCheck.cpp
+  ImpliedSaveCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/ImpliedSaveCheck.cpp b/flang/tools/flang-tidy/bugprone/ImpliedSaveCheck.cpp
new file mode 100644
index 0000000000000..61c3cb266589b
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ImpliedSaveCheck.cpp
@@ -0,0 +1,30 @@
+//===--- ImpliedSaveCheck.cpp - flang-tidy --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ImpliedSaveCheck.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/attr.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+void ImpliedSaveCheck::Enter(const parser::EntityDecl &entityDecl) {
+  const auto &objectName = std::get<parser::ObjectName>(entityDecl.t);
+  const auto *symbol = objectName.symbol;
+  if (symbol && semantics::IsSaved(*symbol) &&
+      !symbol->attrs().test(semantics::Attr::SAVE)) {
+    Say(symbol->name(), "Implicit SAVE on symbol '%s'"_warn_en_US,
+        symbol->name());
+  }
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/ImpliedSaveCheck.h b/flang/tools/flang-tidy/bugprone/ImpliedSaveCheck.h
new file mode 100644
index 0000000000000..6b724dc61a04b
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/ImpliedSaveCheck.h
@@ -0,0 +1,30 @@
+//===--- ImpliedSaveCheck.h - flang-tidy ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_IMPLIEDSAVECHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_IMPLIEDSAVECHECK_H
+
+#include "../FlangTidyCheck.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check warns about variables that are implicitly saved.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/implicit-save.html
+class ImpliedSaveCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~ImpliedSaveCheck() = default;
+
+  void Enter(const parser::EntityDecl &) override;
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_IMPLIEDSAVECHECK_H
diff --git a/flang/tools/test/flang-tidy/implied-save01.f90 b/flang/tools/test/flang-tidy/implied-save01.f90
new file mode 100644
index 0000000000000..a6fbbb6877b70
--- /dev/null
+++ b/flang/tools/test/flang-tidy/implied-save01.f90
@@ -0,0 +1,13 @@
+! RUN: %check_flang_tidy %s bugprone-implied-save %t
+subroutine s
+  integer :: counter = 0
+  ! CHECK-MESSAGES: :[[@LINE-1]]:14: warning: Implicit SAVE on symbol 'counter'
+  counter = counter + 1
+  print *, "Called", counter, "times"
+end subroutine s
+
+subroutine explicit_save
+  integer, save :: counter = 0  ! No warning - explicitly saved
+  counter = counter + 1
+  print *, "Called", counter, "times"
+end subroutine explicit_save

>From 20c156de901f1b593df4d8651d3f30566506cafa Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:19:18 +0200
Subject: [PATCH 16/31] [flang-tidy][bugprone] add mismatched intent check

---
 .../checks/bugprone/mismatched-intent.rst     | 21 ++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/BugproneTidyModule.cpp           |  3 +
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../bugprone/MismatchedIntentCheck.cpp        | 95 +++++++++++++++++++
 .../bugprone/MismatchedIntentCheck.h          | 32 +++++++
 flang/tools/test/flang-tidy/mismatch01.f90    | 26 +++++
 flang/tools/test/flang-tidy/mismatch02.f90    | 18 ++++
 8 files changed, 197 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/mismatched-intent.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/MismatchedIntentCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/MismatchedIntentCheck.h
 create mode 100644 flang/tools/test/flang-tidy/mismatch01.f90
 create mode 100644 flang/tools/test/flang-tidy/mismatch02.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/mismatched-intent.rst b/flang/tools/docs/flang-tidy/checks/bugprone/mismatched-intent.rst
new file mode 100644
index 0000000000000..78d31f86a0444
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/mismatched-intent.rst
@@ -0,0 +1,21 @@
+.. title:: flang-tidy - bugprone-mismatched-intent
+
+bugprone-mismatched-intent
+==========================
+
+
+Warns when a variable is passed multiple times to a procedure with conflicting INTENT attributes or when a component of a variable is passed with less restrictive intent than its parent object.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      real :: x = 1.0
+      call update(x, x)  ! This will trigger a warning
+    contains
+      subroutine update(a, b)
+        real, intent(in) :: a
+        real, intent(out) :: b
+        b = a * 2.0
+      end subroutine
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 784bd7f3b8be2..532a45beb9f89 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -21,3 +21,4 @@ Flang-Tidy Checks
    :doc:`bugprone-contiguous-array <bugprone/contiguous-array>`,
    :doc:`bugprone-implicit-declaration <bugprone/implicit-declaration>`,
    :doc:`bugprone-implied-save <bugprone/implied-save>`,
+   :doc:`bugprone-mismatched-intent <bugprone/mismatched-intent>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index 1e15cf4b425f5..ac7b999922654 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -13,6 +13,7 @@
 #include "ContiguousArrayCheck.h"
 #include "ImplicitDeclCheck.h"
 #include "ImpliedSaveCheck.h"
+#include "MismatchedIntentCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -29,6 +30,8 @@ class BugproneModule : public FlangTidyModule {
     CheckFactories.registerCheck<ImplicitDeclCheck>(
         "bugprone-implicit-declaration");
     CheckFactories.registerCheck<ImpliedSaveCheck>("bugprone-implied-save");
+    CheckFactories.registerCheck<MismatchedIntentCheck>(
+        "bugprone-mismatched-intent");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index 5d88df08c836b..5f8441b54f6a1 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -5,6 +5,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   ContiguousArrayCheck.cpp
   ImplicitDeclCheck.cpp
   ImpliedSaveCheck.cpp
+  MismatchedIntentCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/MismatchedIntentCheck.cpp b/flang/tools/flang-tidy/bugprone/MismatchedIntentCheck.cpp
new file mode 100644
index 0000000000000..8535ab16afab1
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/MismatchedIntentCheck.cpp
@@ -0,0 +1,95 @@
+//===--- MismatchedIntentCheck.cpp - flang-tidy ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MismatchedIntentCheck.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+#include <unordered_map>
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+
+static std::string IntentToString(common::Intent intent) {
+  switch (intent) {
+  case common::Intent::In:
+    return "in";
+  case common::Intent::Out:
+    return "out";
+  case common::Intent::InOut:
+    return "inout";
+  default:
+    return "unknown";
+  }
+}
+
+static bool IsMoreRestrictive(common::Intent first, common::Intent second) {
+  if (first == common::Intent::In &&
+      (second == common::Intent::InOut || second == common::Intent::Out)) {
+    return true;
+  }
+  return false;
+}
+
+static bool AreConflictingIntents(common::Intent first, common::Intent second) {
+  if ((first == common::Intent::In && second == common::Intent::Out) ||
+      (first == common::Intent::Out && second == common::Intent::In)) {
+    return true;
+  }
+  return false;
+}
+
+void MismatchedIntentCheck::Enter(const parser::CallStmt &callStmt) {
+  const auto *procedureRef = callStmt.typedCall.get();
+  if (!procedureRef) {
+    return;
+  }
+
+  std::unordered_map<const semantics::Symbol *, common::Intent> argIntents;
+
+  for (const auto &arg : procedureRef->arguments()) {
+    if (!arg)
+      continue;
+
+    (void)IntentToString;
+    common::Intent intent{arg->dummyIntent()};
+
+    if (const auto *expr{arg->UnwrapExpr()}) {
+      if (const auto *symbol{evaluate::UnwrapWholeSymbolDataRef(*expr)}) {
+        if (argIntents.find(symbol) != argIntents.end()) {
+          common::Intent existingIntent = argIntents[symbol];
+
+          if (existingIntent != intent &&
+              (IsMoreRestrictive(existingIntent, intent) ||
+               AreConflictingIntents(existingIntent, intent))) {
+            Say(callStmt.source,
+                "argument '%s' has mismatched intent"_warn_en_US,
+                symbol->name());
+          }
+        } else {
+          argIntents[symbol] = intent;
+        }
+      } else if (const auto component{evaluate::ExtractDataRef(*expr, true)}) {
+        const semantics::Symbol &baseSymbol = component->GetFirstSymbol();
+
+        if (argIntents.find(&baseSymbol) != argIntents.end()) {
+          common::Intent baseIntent = argIntents[&baseSymbol];
+
+          if (IsMoreRestrictive(baseIntent, intent)) {
+            Say(callStmt.source,
+                "mismatched intent between class '%s' and its member '%s'"_warn_en_US,
+                baseSymbol.name(), component->GetLastSymbol().name());
+          }
+        }
+      }
+    }
+  }
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/MismatchedIntentCheck.h b/flang/tools/flang-tidy/bugprone/MismatchedIntentCheck.h
new file mode 100644
index 0000000000000..b2e24b14db07c
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/MismatchedIntentCheck.h
@@ -0,0 +1,32 @@
+//===--- MismatchedIntentCheck.h - flang-tidy -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISMATCHEDINTENTCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISMATCHEDINTENTCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check warns if a variable is passed to a procedure multiple times with
+/// mismatched intent.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/mismatched-intent.html
+class MismatchedIntentCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~MismatchedIntentCheck() = default;
+
+  void Enter(const parser::CallStmt &) override;
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISMATCHEDINTENTCHECK_H
diff --git a/flang/tools/test/flang-tidy/mismatch01.f90 b/flang/tools/test/flang-tidy/mismatch01.f90
new file mode 100644
index 0000000000000..abdb4290b292f
--- /dev/null
+++ b/flang/tools/test/flang-tidy/mismatch01.f90
@@ -0,0 +1,26 @@
+ ! RUN: %check_flang_tidy %s bugprone-mismatched-intent %t
+module test
+  type :: child_t
+     integer :: i
+  end type child_t
+
+  type :: parent_t
+     type(child_t) :: child
+  end type parent_t
+
+  interface
+     subroutine mutate_member(parent, child)
+       import parent_t
+       import child_t
+       type(parent_t), intent(in) :: parent
+       type(child_t), intent(inout) :: child
+    end subroutine mutate_member
+ end interface
+
+contains
+  subroutine check(parent)
+    type(parent_t), intent(inout) :: parent
+    call mutate_member(parent, parent%child)
+    ! CHECK-MESSAGES: :[[@LINE-1]]:5: warning: mismatched intent between class 'parent' and its member 'child'
+  end subroutine check
+end module test
diff --git a/flang/tools/test/flang-tidy/mismatch02.f90 b/flang/tools/test/flang-tidy/mismatch02.f90
new file mode 100644
index 0000000000000..d2451a5a5d655
--- /dev/null
+++ b/flang/tools/test/flang-tidy/mismatch02.f90
@@ -0,0 +1,18 @@
+ ! RUN: %check_flang_tidy %s bugprone-mismatched-intent %t
+module matrix
+
+  interface
+     subroutine inverse(src, dst)
+       real, intent(in) :: src(:,:)
+       real, intent(out) :: dst(:,:)
+     end subroutine inverse
+  end interface
+
+contains
+
+  subroutine check(A)
+    real, intent(inout) :: A(:,:)
+    call inverse(A, A)
+    ! CHECK-MESSAGES: :[[@LINE-1]]:5: warning: argument 'a' has mismatched intent
+  end subroutine check
+end module matrix

>From 6bcf06e7aeab06706173202a8d421933742ce936 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:22:52 +0200
Subject: [PATCH 17/31] [flang-tidy][bugprone] add missing action check

---
 .../checks/bugprone/missing-action.rst        | 18 +++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/BugproneTidyModule.cpp           |  2 +
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../bugprone/MissingActionCheck.cpp           | 50 +++++++++++++++++++
 .../flang-tidy/bugprone/MissingActionCheck.h  | 33 ++++++++++++
 .../test/flang-tidy/missing-action01.f90      | 14 ++++++
 7 files changed, 119 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/missing-action.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/MissingActionCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/MissingActionCheck.h
 create mode 100644 flang/tools/test/flang-tidy/missing-action01.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/missing-action.rst b/flang/tools/docs/flang-tidy/checks/bugprone/missing-action.rst
new file mode 100644
index 0000000000000..bbb64d01bed55
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/missing-action.rst
@@ -0,0 +1,18 @@
+.. title:: flang-tidy - bugprone-missing-action
+
+bugprone-missing-action
+=======================
+
+Verifies that all OPEN statements include an ACTION specifier, and that file unit numbers are not constant literals. This helps ensure proper file handling behavior and avoids hard-coded unit numbers.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      integer :: unit = 10
+      open(10, file="data.txt")  ! This will trigger two warnings:
+                                 ! Missing ACTION and constant unit number
+      ! Better: open(unit, file="data.txt", action="read")
+      write(10,*) "Hello"
+      close(10)
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 532a45beb9f89..1c5d49fcb1f49 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -22,3 +22,4 @@ Flang-Tidy Checks
    :doc:`bugprone-implicit-declaration <bugprone/implicit-declaration>`,
    :doc:`bugprone-implied-save <bugprone/implied-save>`,
    :doc:`bugprone-mismatched-intent <bugprone/mismatched-intent>`,
+   :doc:`bugprone-missing-action <bugprone/missing-action>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index ac7b999922654..28de91609d914 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -14,6 +14,7 @@
 #include "ImplicitDeclCheck.h"
 #include "ImpliedSaveCheck.h"
 #include "MismatchedIntentCheck.h"
+#include "MissingActionCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -32,6 +33,7 @@ class BugproneModule : public FlangTidyModule {
     CheckFactories.registerCheck<ImpliedSaveCheck>("bugprone-implied-save");
     CheckFactories.registerCheck<MismatchedIntentCheck>(
         "bugprone-mismatched-intent");
+    CheckFactories.registerCheck<MissingActionCheck>("bugprone-missing-action");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index 5f8441b54f6a1..8be859ce45a70 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -6,6 +6,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   ImplicitDeclCheck.cpp
   ImpliedSaveCheck.cpp
   MismatchedIntentCheck.cpp
+  MissingActionCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/MissingActionCheck.cpp b/flang/tools/flang-tidy/bugprone/MissingActionCheck.cpp
new file mode 100644
index 0000000000000..7d636a63f96e6
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/MissingActionCheck.cpp
@@ -0,0 +1,50 @@
+//===--- MissingActionCheck.cpp - flang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MissingActionCheck.h"
+#include "flang/Evaluate/check-expression.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include <algorithm>
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+
+void MissingActionCheck::Leave(const parser::FileUnitNumber &fileUnit) {
+  // warn if its a const expr
+  const auto *expr =
+      semantics::GetExpr(context()->getSemanticsContext(), fileUnit.v);
+
+  if (expr && evaluate::IsConstantExpr(*expr)) {
+    Say(context()->getSemanticsContext().location().value(),
+        "File unit number is a constant literal"_warn_en_US);
+  }
+}
+
+void MissingActionCheck::Leave(const parser::OpenStmt &openStmt) {
+  const auto &source = context()->getSemanticsContext().location().value();
+
+  const auto &connectSpec = openStmt.v;
+
+  const auto &action = std::find_if(
+      connectSpec.begin(), connectSpec.end(), [](const auto &spec) {
+        return std::holds_alternative<parser::ConnectSpec::CharExpr>(spec.u) &&
+               std::get<parser::ConnectSpec::CharExpr::Kind>(
+                   std::get<parser::ConnectSpec::CharExpr>(spec.u).t) ==
+                   parser::ConnectSpec::CharExpr::Kind::Action;
+      });
+
+  if (action == connectSpec.end()) {
+    Say(source, "ACTION specifier is missing"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/MissingActionCheck.h b/flang/tools/flang-tidy/bugprone/MissingActionCheck.h
new file mode 100644
index 0000000000000..dbebea46e0aa9
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/MissingActionCheck.h
@@ -0,0 +1,33 @@
+//===--- MissingActionCheck.h - flang-tidy ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISSINGACTIONCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISSINGACTIONCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check verifies that all OPEN statements have an ACTION clause, and that
+/// the FILE UNIT NUMBER is not a constant literal.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/missing-action.html
+class MissingActionCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~MissingActionCheck() = default;
+
+  void Leave(const parser::FileUnitNumber &) override;
+  void Leave(const parser::OpenStmt &) override;
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISSINGACTIONCHECK_H
diff --git a/flang/tools/test/flang-tidy/missing-action01.f90 b/flang/tools/test/flang-tidy/missing-action01.f90
new file mode 100644
index 0000000000000..ffea483212156
--- /dev/null
+++ b/flang/tools/test/flang-tidy/missing-action01.f90
@@ -0,0 +1,14 @@
+! RUN: %check_flang_tidy %s bugprone-missing-action %t
+program open_test
+  integer :: unit = 10
+
+  open(10, file="data.txt")  ! This will trigger a warning for missing ACTION
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: File unit number is a constant literal
+  ! CHECK-MESSAGES: :[[@LINE-2]]:3: warning: ACTION specifier is missing
+
+  open(unit, file="data2.txt")  ! This will trigger a warning for constant unit
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ACTION specifier is missing
+
+  close(10)
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: File unit number is a constant literal
+end program open_test

>From 6c1e887f6541625a73267600c968106a8380ab59 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:25:57 +0200
Subject: [PATCH 18/31] [flang-tidy][bugprone] add missing default case check

---
 .../checks/bugprone/missing-default-case.rst  | 20 ++++++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/BugproneTidyModule.cpp           |  3 ++
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../bugprone/MissingDefaultCheck.cpp          | 37 +++++++++++++++++++
 .../flang-tidy/bugprone/MissingDefaultCheck.h | 31 ++++++++++++++++
 .../test/flang-tidy/missing-default-case.f90  | 24 ++++++++++++
 7 files changed, 117 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/missing-default-case.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/MissingDefaultCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/MissingDefaultCheck.h
 create mode 100644 flang/tools/test/flang-tidy/missing-default-case.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/missing-default-case.rst b/flang/tools/docs/flang-tidy/checks/bugprone/missing-default-case.rst
new file mode 100644
index 0000000000000..c56d0fe819009
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/missing-default-case.rst
@@ -0,0 +1,20 @@
+.. title:: flang-tidy - bugprone-missing-default-case
+
+bugprone-missing-default-case
+=============================
+
+Ensures that all SELECT CASE constructs include a DEFAULT case to handle unexpected values. This makes code more robust by providing a catch-all for unforeseen values.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      integer :: i = 3
+      select case (i)  ! This will trigger a warning
+        case (1)
+          print *, "One"
+        case (2)
+          print *, "Two"
+        ! Missing: case default
+      end select
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 1c5d49fcb1f49..69dd23aba8939 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -23,3 +23,4 @@ Flang-Tidy Checks
    :doc:`bugprone-implied-save <bugprone/implied-save>`,
    :doc:`bugprone-mismatched-intent <bugprone/mismatched-intent>`,
    :doc:`bugprone-missing-action <bugprone/missing-action>`,
+   :doc:`bugprone-missing-default-case <bugprone/missing-default-case>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index 28de91609d914..3e79ceb27e173 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -15,6 +15,7 @@
 #include "ImpliedSaveCheck.h"
 #include "MismatchedIntentCheck.h"
 #include "MissingActionCheck.h"
+#include "MissingDefaultCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -34,6 +35,8 @@ class BugproneModule : public FlangTidyModule {
     CheckFactories.registerCheck<MismatchedIntentCheck>(
         "bugprone-mismatched-intent");
     CheckFactories.registerCheck<MissingActionCheck>("bugprone-missing-action");
+    CheckFactories.registerCheck<MissingDefaultCheck>(
+        "bugprone-missing-default-case");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index 8be859ce45a70..1f2ed528af5d6 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -7,6 +7,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   ImpliedSaveCheck.cpp
   MismatchedIntentCheck.cpp
   MissingActionCheck.cpp
+  MissingDefaultCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/MissingDefaultCheck.cpp b/flang/tools/flang-tidy/bugprone/MissingDefaultCheck.cpp
new file mode 100644
index 0000000000000..0ea8cfbca590a
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/MissingDefaultCheck.cpp
@@ -0,0 +1,37 @@
+//===--- MissingDefaultCheck.cpp - flang-tidy -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MissingDefaultCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+#include <algorithm>
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+void MissingDefaultCheck::Enter(const parser::CaseConstruct &caseConstruct) {
+  const auto &source =
+      std::get<parser::Statement<parser::SelectCaseStmt>>(caseConstruct.t)
+          .source;
+
+  const auto &cases =
+      std::get<std::list<parser::CaseConstruct::Case>>(caseConstruct.t);
+
+  bool hasDefault = std::any_of(cases.begin(), cases.end(), [](const auto &c) {
+    const auto &caseSelector = std::get<parser::CaseSelector>(
+        std::get<parser::Statement<parser::CaseStmt>>(c.t).statement.t);
+
+    return std::holds_alternative<parser::Default>(caseSelector.u);
+  });
+
+  if (!hasDefault) {
+    Say(source, "SELECT CASE construct has no DEFAULT case"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/MissingDefaultCheck.h b/flang/tools/flang-tidy/bugprone/MissingDefaultCheck.h
new file mode 100644
index 0000000000000..20bb735a2e1c5
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/MissingDefaultCheck.h
@@ -0,0 +1,31 @@
+//===--- MissingDefaultCheck.h - flang-tidy ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISSINGDEFAULTCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISSINGDEFAULTCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check verifies that all CASE constructs have a default case.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/missing-default.html
+class MissingDefaultCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~MissingDefaultCheck() = default;
+
+  void Enter(const parser::CaseConstruct &) override;
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_MISSINGDEFAULTCHECK_H
diff --git a/flang/tools/test/flang-tidy/missing-default-case.f90 b/flang/tools/test/flang-tidy/missing-default-case.f90
new file mode 100644
index 0000000000000..2f92e35c102e8
--- /dev/null
+++ b/flang/tools/test/flang-tidy/missing-default-case.f90
@@ -0,0 +1,24 @@
+! RUN: %check_flang_tidy %s bugprone-missing-default-case %t
+program case_test
+  integer :: i = 2
+
+  select case (i)  ! This will trigger a warning
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: SELECT CASE construct has no DEFAULT case
+    case (1)
+      print *, "One"
+    case (2)
+      print *, "Two"
+    ! Missing: case default
+  end select
+
+  ! Correct form:
+  select case (i)
+    case (1)
+      print *, "One"
+    case (2)
+      print *, "Two"
+    case default
+      print *, "Other"
+  end select
+
+end program case_test

>From 8b4426211848b9f9ffd057ccfc95b7ea7057f6bf Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:28:10 +0200
Subject: [PATCH 19/31] [flang-tidy][bugprone] add precision loss check

---
 .../checks/bugprone/precision-loss.rst        |  16 +++
 flang/tools/docs/flang-tidy/checks/list.rst   |   1 +
 .../bugprone/BugproneTidyModule.cpp           |   2 +
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |   1 +
 .../bugprone/PrecisionLossCheck.cpp           | 108 ++++++++++++++++++
 .../flang-tidy/bugprone/PrecisionLossCheck.h  |  29 +++++
 .../tools/test/flang-tidy/precisionloss01.f90 |  10 ++
 .../tools/test/flang-tidy/precisionloss02.f90 |   8 ++
 .../tools/test/flang-tidy/precisionloss03.f90 |  31 +++++
 9 files changed, 206 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/precision-loss.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/PrecisionLossCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/PrecisionLossCheck.h
 create mode 100644 flang/tools/test/flang-tidy/precisionloss01.f90
 create mode 100644 flang/tools/test/flang-tidy/precisionloss02.f90
 create mode 100644 flang/tools/test/flang-tidy/precisionloss03.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/precision-loss.rst b/flang/tools/docs/flang-tidy/checks/bugprone/precision-loss.rst
new file mode 100644
index 0000000000000..5325f881ddfb6
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/precision-loss.rst
@@ -0,0 +1,16 @@
+.. title:: flang-tidy - bugprone-precision-loss
+
+bugprone-precision-loss
+=======================
+
+Detects potential loss of precision in assignments between variables of different types or kind parameters. This helps prevent subtle numerical errors in computation.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      real(kind=8) :: x = 1.0d0
+      real(kind=4) :: y
+      y = x  ! This will trigger a warning - precision loss
+      print *, y
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 69dd23aba8939..2021cc6105959 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -24,3 +24,4 @@ Flang-Tidy Checks
    :doc:`bugprone-mismatched-intent <bugprone/mismatched-intent>`,
    :doc:`bugprone-missing-action <bugprone/missing-action>`,
    :doc:`bugprone-missing-default-case <bugprone/missing-default-case>`,
+   :doc:`bugprone-precision-loss <bugprone/precision-loss>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index 3e79ceb27e173..dc0ccdc1618e4 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -16,6 +16,7 @@
 #include "MismatchedIntentCheck.h"
 #include "MissingActionCheck.h"
 #include "MissingDefaultCheck.h"
+#include "PrecisionLossCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -37,6 +38,7 @@ class BugproneModule : public FlangTidyModule {
     CheckFactories.registerCheck<MissingActionCheck>("bugprone-missing-action");
     CheckFactories.registerCheck<MissingDefaultCheck>(
         "bugprone-missing-default-case");
+    CheckFactories.registerCheck<PrecisionLossCheck>("bugprone-precision-loss");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index 1f2ed528af5d6..aad9c9155804a 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -8,6 +8,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   MismatchedIntentCheck.cpp
   MissingActionCheck.cpp
   MissingDefaultCheck.cpp
+  PrecisionLossCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/PrecisionLossCheck.cpp b/flang/tools/flang-tidy/bugprone/PrecisionLossCheck.cpp
new file mode 100644
index 0000000000000..d465e00985ae1
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/PrecisionLossCheck.cpp
@@ -0,0 +1,108 @@
+//===--- PrecisionLossCheck.cpp - flang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PrecisionLossCheck.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+#include <clang/Basic/SourceLocation.h>
+#include <variant>
+
+namespace Fortran::tidy::bugprone {
+
+static bool IsLossOfPrecision(const semantics::SomeExpr *lhs,
+                              const semantics::SomeExpr *rhs,
+                              bool hasExplicitKind, bool isConstantReal) {
+
+  const auto &lhsType{lhs->GetType()};
+  const auto &rhsType{rhs->GetType()};
+
+  if (!lhsType || !rhsType)
+    return false;
+
+  auto lhsCat = lhsType->category();
+  auto rhsCat = lhsType->category();
+
+  // ignore derived types
+  if (lhsCat == common::TypeCategory::Derived ||
+      rhsCat == common::TypeCategory::Derived)
+    return false;
+
+  int lhsKind = lhsType->kind();
+  int rhsKind = rhsType->kind();
+
+  // integer -> integer, real, complex
+  // real -> integer, real, complex
+  // complex -> integer, real, complex
+  //
+  if (lhsCat == rhsCat && lhsKind < rhsKind)
+    return true;
+
+  if (isConstantReal && lhsKind > rhsKind && !hasExplicitKind) {
+    return true;
+  }
+
+  if (lhsCat == common::TypeCategory::Complex &&
+      rhsCat == common::TypeCategory::Real) {
+    if (lhsKind < rhsKind)
+      return true;
+    return false;
+  }
+
+  if ((lhsCat == common::TypeCategory::Real ||
+       lhsCat == common::TypeCategory::Integer) &&
+      rhsCat == common::TypeCategory::Complex) {
+    return true;
+  }
+
+  if (lhsCat == common::TypeCategory::Integer &&
+      (rhsCat == common::TypeCategory::Real ||
+       rhsCat == common::TypeCategory::Complex)) {
+    return true;
+  }
+
+  if ((lhsCat == common::TypeCategory::Real ||
+       lhsCat == common::TypeCategory::Complex) &&
+      rhsCat == common::TypeCategory::Integer && lhsKind <= rhsKind) {
+    return true;
+  }
+
+  return false;
+}
+
+using namespace parser::literals;
+void PrecisionLossCheck::Enter(const parser::AssignmentStmt &assignment) {
+  const auto &var{std::get<parser::Variable>(assignment.t)};
+  const auto &expr{std::get<parser::Expr>(assignment.t)};
+  const auto *lhs{semantics::GetExpr(context()->getSemanticsContext(), var)};
+  const auto *rhs{semantics::GetExpr(context()->getSemanticsContext(), expr)};
+
+  if (!lhs || !rhs)
+    return;
+
+  bool hasExplicitKind = false;
+  bool isConstantReal = false;
+  if (std::holds_alternative<parser::LiteralConstant>(expr.u)) {
+    const auto &literal = std::get<parser::LiteralConstant>(expr.u);
+    if (std::holds_alternative<parser::RealLiteralConstant>(literal.u)) {
+      const auto &realLiteral =
+          std::get<parser::RealLiteralConstant>(literal.u);
+      hasExplicitKind = realLiteral.kind.has_value();
+      isConstantReal = true;
+    }
+  }
+
+  if (IsLossOfPrecision(lhs, rhs, hasExplicitKind, isConstantReal)) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Possible loss of precision in implicit conversion (%s to %s) "_warn_en_US,
+        rhs->GetType()->AsFortran(), lhs->GetType()->AsFortran());
+  }
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/PrecisionLossCheck.h b/flang/tools/flang-tidy/bugprone/PrecisionLossCheck.h
new file mode 100644
index 0000000000000..07e5765352662
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/PrecisionLossCheck.h
@@ -0,0 +1,29 @@
+//===--- PrecisionLossCheck.h - flang-tidy ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_PRECISIONLOSSCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_PRECISIONLOSSCHECK_H
+
+#include "../FlangTidyCheck.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check verifies that precision loss is avoided in assignments.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/precision-loss.html
+class PrecisionLossCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~PrecisionLossCheck() = default;
+  void Enter(const parser::AssignmentStmt &) override;
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_PRECISIONLOSSCHECK_H
diff --git a/flang/tools/test/flang-tidy/precisionloss01.f90 b/flang/tools/test/flang-tidy/precisionloss01.f90
new file mode 100644
index 0000000000000..07bf3ef56db49
--- /dev/null
+++ b/flang/tools/test/flang-tidy/precisionloss01.f90
@@ -0,0 +1,10 @@
+! RUN: %check_flang_tidy %s bugprone-precision-loss %t
+subroutine s
+  real(8) :: i
+  real(4) :: j
+  i = 1.0
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Possible loss of precision in implicit conversion (REAL(4) to REAL(8))
+  j = 2.0
+  j = i
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Possible loss of precision in implicit conversion (REAL(8) to REAL(4))
+end subroutine s
diff --git a/flang/tools/test/flang-tidy/precisionloss02.f90 b/flang/tools/test/flang-tidy/precisionloss02.f90
new file mode 100644
index 0000000000000..177fb232532d0
--- /dev/null
+++ b/flang/tools/test/flang-tidy/precisionloss02.f90
@@ -0,0 +1,8 @@
+! RUN: %check_flang_tidy %s bugprone-precision-loss %t
+subroutine s
+  real(8) :: i
+  real(4) :: j
+  i = 1.0_8
+  j = 2.0
+  i = j ! no warning
+end subroutine s
diff --git a/flang/tools/test/flang-tidy/precisionloss03.f90 b/flang/tools/test/flang-tidy/precisionloss03.f90
new file mode 100644
index 0000000000000..f153ec015873a
--- /dev/null
+++ b/flang/tools/test/flang-tidy/precisionloss03.f90
@@ -0,0 +1,31 @@
+! RUN: %check_flang_tidy %s bugprone-precision-loss %t
+subroutine precision_test
+  integer :: i
+  real :: r
+  real(8) :: d
+  complex :: c
+  complex(8) :: z
+
+  ! Integer to real - no precision loss
+  r = i  ! No warning
+
+  ! Real to double - no precision loss
+  d = r  ! No warning
+
+  ! Double to real - precision loss
+  r = d
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Possible loss of precision in implicit conversion (REAL(8) to REAL(4))
+
+  ! Complex to real
+  r = c
+
+  ! Complex(8) to complex(4) - precision loss
+  c = z
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Possible loss of precision in implicit conversion (COMPLEX(8) to COMPLEX(4))
+
+  ! Real to complex - no precision loss
+  c = r  ! No warning
+
+  ! Literal with kind - no precision loss
+  d = 1.0_8  ! No warning
+end subroutine precision_test

>From 6addcb009eeb7c3839ccd84553eb9718259e2001 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:30:48 +0200
Subject: [PATCH 20/31] [flang-tidy][bugprone] add subprogram trampoline check

---
 .../checks/bugprone/subprogram-trampoline.rst | 25 +++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/BugproneTidyModule.cpp           |  3 +
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../bugprone/SubprogramTrampolineCheck.cpp    | 70 +++++++++++++++++++
 .../bugprone/SubprogramTrampolineCheck.h      | 32 +++++++++
 .../test/flang-tidy/subprogram-trampoline.f90 | 17 +++++
 7 files changed, 149 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/subprogram-trampoline.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/SubprogramTrampolineCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/SubprogramTrampolineCheck.h
 create mode 100644 flang/tools/test/flang-tidy/subprogram-trampoline.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/subprogram-trampoline.rst b/flang/tools/docs/flang-tidy/checks/bugprone/subprogram-trampoline.rst
new file mode 100644
index 0000000000000..a756bfd7f82cd
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/subprogram-trampoline.rst
@@ -0,0 +1,25 @@
+.. title:: flang-tidy - bugprone-subprogram-trampoline
+
+bugprone-subprogram-trampoline
+==============================
+
+Warns when a contained subprogram is passed as an actual argument to another procedure. This pattern can lead to unexpected behavior due to the way procedure pointers and contained subprograms interact.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      call process(inner)  ! This will trigger a warning
+    contains
+      subroutine inner()
+        print *, "Inside inner"
+      end subroutine
+
+      subroutine process(proc)
+        interface
+          subroutine proc()
+          end subroutine
+        end interface
+        call proc()
+      end subroutine
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 2021cc6105959..15dbb159a8951 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -25,3 +25,4 @@ Flang-Tidy Checks
    :doc:`bugprone-missing-action <bugprone/missing-action>`,
    :doc:`bugprone-missing-default-case <bugprone/missing-default-case>`,
    :doc:`bugprone-precision-loss <bugprone/precision-loss>`,
+   :doc:`bugprone-subprogram-trampoline <bugprone/subprogram-trampoline>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index dc0ccdc1618e4..e6ec2243f8a6c 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -17,6 +17,7 @@
 #include "MissingActionCheck.h"
 #include "MissingDefaultCheck.h"
 #include "PrecisionLossCheck.h"
+#include "SubprogramTrampolineCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -39,6 +40,8 @@ class BugproneModule : public FlangTidyModule {
     CheckFactories.registerCheck<MissingDefaultCheck>(
         "bugprone-missing-default-case");
     CheckFactories.registerCheck<PrecisionLossCheck>("bugprone-precision-loss");
+    CheckFactories.registerCheck<SubprogramTrampolineCheck>(
+        "bugprone-subprogram-trampoline");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index aad9c9155804a..68a71c0310bec 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -9,6 +9,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   MissingActionCheck.cpp
   MissingDefaultCheck.cpp
   PrecisionLossCheck.cpp
+  SubprogramTrampolineCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/SubprogramTrampolineCheck.cpp b/flang/tools/flang-tidy/bugprone/SubprogramTrampolineCheck.cpp
new file mode 100644
index 0000000000000..0ff4a41e06f91
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/SubprogramTrampolineCheck.cpp
@@ -0,0 +1,70 @@
+//===--- SubprogramTrampolineCheck.cpp - flang-tidy -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubprogramTrampolineCheck.h"
+#include "../utils/CollectActualArguments.h"
+#include "flang/Evaluate/call.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/variable.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+void SubprogramTrampolineCheck::Enter(const parser::CallStmt &callStmt) {
+  const auto *procedureRef = callStmt.typedCall.get();
+  if (procedureRef) {
+    for (const auto &arg : procedureRef->arguments()) {
+      if (!arg)
+        continue;
+      if (const semantics::SomeExpr *argExpr{arg->UnwrapExpr()}) {
+        if (!evaluate::IsProcedureDesignator(*argExpr))
+          continue;
+        const auto proc = std::get<evaluate::ProcedureDesignator>(argExpr->u);
+        if (const auto *symbol{proc.GetSymbol()}) {
+          if (symbol->has<semantics::SubprogramDetails>()) {
+            Say(callStmt.source,
+                "contained subprogram '%s' is passed as an argument"_warn_en_US,
+                symbol->name().ToString());
+          }
+        }
+      }
+    }
+  }
+}
+
+void SubprogramTrampolineCheck::Enter(const parser::Expr &e) {
+  const auto *expr = semantics::GetExpr(context()->getSemanticsContext(), e);
+  if (!expr) {
+    return;
+  }
+
+  if (std::holds_alternative<common::Indirection<parser::FunctionReference>>(
+          e.u)) {
+    evaluate::ActualArgumentSet argSet{evaluate::CollectActualArguments(*expr)};
+    for (const evaluate::ActualArgumentRef &argRef : argSet) {
+      if (const semantics::SomeExpr *argExpr{argRef->UnwrapExpr()}) {
+        if (!evaluate::IsProcedureDesignator(*argExpr))
+          continue;
+        const auto proc = std::get<evaluate::ProcedureDesignator>(argExpr->u);
+        if (const auto *symbol{proc.GetSymbol()}) {
+          if (symbol->has<semantics::SubprogramDetails>()) {
+            Say(e.source,
+                "contained subprogram '%s' is passed as an argument"_warn_en_US,
+                symbol->name().ToString());
+          }
+        }
+      }
+    }
+  }
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/SubprogramTrampolineCheck.h b/flang/tools/flang-tidy/bugprone/SubprogramTrampolineCheck.h
new file mode 100644
index 0000000000000..41322b87e052f
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/SubprogramTrampolineCheck.h
@@ -0,0 +1,32 @@
+//===--- SubprogramTrampolineCheck.h - flang-tidy ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_SUBPROGRAMTRAMPOLINECHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_SUBPROGRAMTRAMPOLINECHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check verifies that a contained subprogram is not passed as an
+/// actual argument to a procedure.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/subprogram-trampoline.html
+class SubprogramTrampolineCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~SubprogramTrampolineCheck() = default;
+  void Enter(const parser::Expr &) override;
+  void Enter(const parser::CallStmt &) override;
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_SUBPROGRAMTRAMPOLINECHECK_H
diff --git a/flang/tools/test/flang-tidy/subprogram-trampoline.f90 b/flang/tools/test/flang-tidy/subprogram-trampoline.f90
new file mode 100644
index 0000000000000..3d3974b6a6e91
--- /dev/null
+++ b/flang/tools/test/flang-tidy/subprogram-trampoline.f90
@@ -0,0 +1,17 @@
+! RUN: %check_flang_tidy %s bugprone-subprogram-trampoline %t
+program trampoline_test
+  call process(inner)  ! This will trigger a warning
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: contained subprogram 'inner' is passed as an argument
+contains
+  subroutine inner()
+    print *, "Inside inner"
+  end subroutine
+
+  subroutine process(proc)
+    interface
+      subroutine proc()
+      end subroutine
+    end interface
+    call proc()
+  end subroutine
+end program trampoline_test

>From 74000ab13dde65e5a0582d3b775a883bc27c8f56 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:33:07 +0200
Subject: [PATCH 21/31] [flang-tidy][bugprone] add undeclared proc check

---
 .../checks/bugprone/undeclared-procedure.rst  | 17 +++++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/BugproneTidyModule.cpp           |  3 ++
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../bugprone/UndeclaredProcCheck.cpp          | 50 +++++++++++++++++++
 .../flang-tidy/bugprone/UndeclaredProcCheck.h | 33 ++++++++++++
 .../tools/test/flang-tidy/implicitproc01.f90  |  5 ++
 7 files changed, 110 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/undeclared-procedure.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/UndeclaredProcCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/UndeclaredProcCheck.h
 create mode 100644 flang/tools/test/flang-tidy/implicitproc01.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/undeclared-procedure.rst b/flang/tools/docs/flang-tidy/checks/bugprone/undeclared-procedure.rst
new file mode 100644
index 0000000000000..3b1f0c3996f90
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/undeclared-procedure.rst
@@ -0,0 +1,17 @@
+.. title:: flang-tidy - bugprone-undeclared-procedure
+
+bugprone-undeclared-procedure
+=============================
+
+Warns about procedures that are implicitly declared or lack an explicit interface. Using procedures without explicit interfaces can lead to subtle bugs and prevents the compiler from performing argument checking.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      integer :: result
+      external :: no_interface_procedure
+      call no_interface_procedure(5, result)  ! This will trigger a warning (no interface)
+      call implicit_procedure(5, result)  ! This will also trigger a warning (implicit declaration)
+      print *, result
+    end program
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 15dbb159a8951..ea37e59c51328 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -26,3 +26,4 @@ Flang-Tidy Checks
    :doc:`bugprone-missing-default-case <bugprone/missing-default-case>`,
    :doc:`bugprone-precision-loss <bugprone/precision-loss>`,
    :doc:`bugprone-subprogram-trampoline <bugprone/subprogram-trampoline>`,
+   :doc:`bugprone-undeclared-procedure <bugprone/undeclared-procedure>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index e6ec2243f8a6c..36c32935783d4 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -18,6 +18,7 @@
 #include "MissingDefaultCheck.h"
 #include "PrecisionLossCheck.h"
 #include "SubprogramTrampolineCheck.h"
+#include "UndeclaredProcCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -42,6 +43,8 @@ class BugproneModule : public FlangTidyModule {
     CheckFactories.registerCheck<PrecisionLossCheck>("bugprone-precision-loss");
     CheckFactories.registerCheck<SubprogramTrampolineCheck>(
         "bugprone-subprogram-trampoline");
+    CheckFactories.registerCheck<UndeclaredProcCheck>(
+        "bugprone-undeclared-procedure");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index 68a71c0310bec..78196341c99f6 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -10,6 +10,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   MissingDefaultCheck.cpp
   PrecisionLossCheck.cpp
   SubprogramTrampolineCheck.cpp
+  UndeclaredProcCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/UndeclaredProcCheck.cpp b/flang/tools/flang-tidy/bugprone/UndeclaredProcCheck.cpp
new file mode 100644
index 0000000000000..30ca1236fd307
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/UndeclaredProcCheck.cpp
@@ -0,0 +1,50 @@
+//===--- UndeclaredProcCheck.cpp - flang-tidy -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UndeclaredProcCheck.h"
+#include "flang/Semantics/attr.h"
+#include "flang/Semantics/symbol.h"
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+void UndeclaredProcCheck::CheckForUndeclaredProcedures(
+    semantics::SemanticsContext &context, const semantics::Scope &scope) {
+  if (scope.IsModuleFile())
+    return;
+
+  for (const auto &pair : scope) {
+    const semantics::Symbol &symbol = *pair.second;
+    if (auto *details{symbol.detailsIf<semantics::ProcEntityDetails>()};
+        details) {
+      if (symbol.owner().IsGlobal()) { // unknown global procedure
+        Say(symbol.name(), "Implicit declaration of procedure '%s'"_warn_en_US,
+            symbol.name());
+      } else if (!details->HasExplicitInterface() && // no explicit interface
+                 !symbol.attrs().test(
+                     semantics::Attr::INTRINSIC)) { // not an intrinsic
+        Say(symbol.name(),
+            "Procedure '%s' has no explicit interface"_warn_en_US,
+            symbol.name());
+      }
+    }
+  }
+
+  for (const semantics::Scope &child : scope.children()) {
+    CheckForUndeclaredProcedures(context, child);
+  }
+}
+
+UndeclaredProcCheck::UndeclaredProcCheck(llvm::StringRef name,
+                                         FlangTidyContext *context)
+    : FlangTidyCheck{name, context} {
+  CheckForUndeclaredProcedures(context->getSemanticsContext(),
+                               context->getSemanticsContext().globalScope());
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/UndeclaredProcCheck.h b/flang/tools/flang-tidy/bugprone/UndeclaredProcCheck.h
new file mode 100644
index 0000000000000..0650e06d40683
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/UndeclaredProcCheck.h
@@ -0,0 +1,33 @@
+//===--- UndeclaredProcCheck.h - flang-tidy ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_UNDECLAREDPROCCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_UNDECLAREDPROCCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "../FlangTidyContext.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check verifies that all procedures are explicitly declared.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/implicit-procedure-declaration.html
+class UndeclaredProcCheck : public virtual FlangTidyCheck {
+public:
+  UndeclaredProcCheck(llvm::StringRef name, FlangTidyContext *context);
+  virtual ~UndeclaredProcCheck() = default;
+
+private:
+  void CheckForUndeclaredProcedures(semantics::SemanticsContext &,
+                                    const semantics::Scope &);
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_UNDECLAREDPROCCHECK_H
diff --git a/flang/tools/test/flang-tidy/implicitproc01.f90 b/flang/tools/test/flang-tidy/implicitproc01.f90
new file mode 100644
index 0000000000000..d044154e76789
--- /dev/null
+++ b/flang/tools/test/flang-tidy/implicitproc01.f90
@@ -0,0 +1,5 @@
+! RUN: %check_flang_tidy %s bugprone-undeclared-procedure %t
+subroutine s1
+  call s2()
+  ! CHECK-MESSAGES: :[[@LINE-1]]:8: warning: Implicit declaration of procedure 's2'
+end subroutine s1

>From 8b77899ea2d7c0f6f3101d2ba82168d8d60a4e89 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:35:34 +0200
Subject: [PATCH 22/31] [flang-tidy][bugprone] add unused intent check

---
 .../checks/bugprone/unused-intent.rst         | 17 ++++
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../bugprone/BugproneTidyModule.cpp           |  2 +
 .../tools/flang-tidy/bugprone/CMakeLists.txt  |  1 +
 .../flang-tidy/bugprone/UnusedIntentCheck.cpp | 87 +++++++++++++++++++
 .../flang-tidy/bugprone/UnusedIntentCheck.h   | 33 +++++++
 flang/tools/test/flang-tidy/intent01.f90      |  6 ++
 7 files changed, 147 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/bugprone/unused-intent.rst
 create mode 100644 flang/tools/flang-tidy/bugprone/UnusedIntentCheck.cpp
 create mode 100644 flang/tools/flang-tidy/bugprone/UnusedIntentCheck.h
 create mode 100644 flang/tools/test/flang-tidy/intent01.f90

diff --git a/flang/tools/docs/flang-tidy/checks/bugprone/unused-intent.rst b/flang/tools/docs/flang-tidy/checks/bugprone/unused-intent.rst
new file mode 100644
index 0000000000000..df9996e5a5fb3
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/bugprone/unused-intent.rst
@@ -0,0 +1,17 @@
+.. title:: flang-tidy - bugprone-unused-intent
+
+bugprone-unused-intent
+======================
+
+Identifies dummy arguments with INTENT(INOUT) that are never modified, suggesting they should be INTENT(IN) instead. It also warns about dummy arguments without explicit intent declarations, which could lead to confusion about how they're used.
+
+.. code-block:: fortran
+
+    subroutine process(a, b, c)
+      implicit none
+      real, intent(in) :: a
+      real, intent(inout) :: b  ! This will trigger a warning if b is never modified
+      real :: c               ! This will trigger a warning for missing intent
+
+      print *, a, b, c
+    end subroutine
diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index ea37e59c51328..31decff549a5d 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -27,3 +27,4 @@ Flang-Tidy Checks
    :doc:`bugprone-precision-loss <bugprone/precision-loss>`,
    :doc:`bugprone-subprogram-trampoline <bugprone/subprogram-trampoline>`,
    :doc:`bugprone-undeclared-procedure <bugprone/undeclared-procedure>`,
+   :doc:`bugprone-unused-intent <bugprone/unused-intent>`,
diff --git a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
index 36c32935783d4..6f955d65a7aa7 100644
--- a/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/flang/tools/flang-tidy/bugprone/BugproneTidyModule.cpp
@@ -19,6 +19,7 @@
 #include "PrecisionLossCheck.h"
 #include "SubprogramTrampolineCheck.h"
 #include "UndeclaredProcCheck.h"
+#include "UnusedIntentCheck.h"
 
 namespace Fortran::tidy {
 namespace bugprone {
@@ -45,6 +46,7 @@ class BugproneModule : public FlangTidyModule {
         "bugprone-subprogram-trampoline");
     CheckFactories.registerCheck<UndeclaredProcCheck>(
         "bugprone-undeclared-procedure");
+    CheckFactories.registerCheck<UnusedIntentCheck>("bugprone-unused-intent");
   }
 };
 
diff --git a/flang/tools/flang-tidy/bugprone/CMakeLists.txt b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
index 78196341c99f6..47994f914848f 100644
--- a/flang/tools/flang-tidy/bugprone/CMakeLists.txt
+++ b/flang/tools/flang-tidy/bugprone/CMakeLists.txt
@@ -11,6 +11,7 @@ add_flang_library(flangTidyBugproneModule STATIC
   PrecisionLossCheck.cpp
   SubprogramTrampolineCheck.cpp
   UndeclaredProcCheck.cpp
+  UnusedIntentCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/bugprone/UnusedIntentCheck.cpp b/flang/tools/flang-tidy/bugprone/UnusedIntentCheck.cpp
new file mode 100644
index 0000000000000..6e02028c73abd
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/UnusedIntentCheck.cpp
@@ -0,0 +1,87 @@
+//===--- UnusedIntentCheck.cpp - flang-tidy -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UnusedIntentCheck.h"
+#include "flang/Semantics/attr.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+
+namespace Fortran::tidy::bugprone {
+
+using namespace parser::literals;
+
+static std::unordered_map<const semantics::Symbol *, const semantics::Symbol *>
+    procBindingDetailsSymbolsMap;
+
+void UnusedIntentCheck::CheckUnusedIntentHelper(
+    semantics::SemanticsContext &context, const semantics::Scope &scope) {
+  if (scope.IsModuleFile())
+    return;
+
+  auto WasDefined{[&context](const semantics::Symbol &symbol) {
+    return context.IsSymbolDefined(symbol) ||
+           semantics::IsInitialized(symbol, false, false, false);
+  }};
+  for (const auto &pair : scope) {
+    const semantics::Symbol &symbol = *pair.second;
+    if (const auto *details{symbol.detailsIf<semantics::ObjectEntityDetails>()};
+        details && details->isDummy()) {
+      const auto &owningProcScope = symbol.owner();
+      const auto &owningProc = owningProcScope.symbol();
+
+      if (procBindingDetailsSymbolsMap.find(owningProc) !=
+          procBindingDetailsSymbolsMap.end()) {
+        continue;
+      }
+      if (!WasDefined(symbol) && semantics::IsIntentInOut(symbol)) {
+        Say(symbol.name(),
+            "Dummy argument '%s' with intent(inout) is never written to, consider changing to intent(in)"_warn_en_US,
+            symbol.name());
+      }
+      if (!symbol.attrs().HasAny({semantics::Attr::INTENT_IN,
+                                  semantics::Attr::INTENT_INOUT,
+                                  semantics::Attr::INTENT_OUT})) {
+        // warn about dummy arguments without explicit intent
+        Say(symbol.name(),
+            "Dummy argument '%s' has no explicit intent"_warn_en_US,
+            symbol.name());
+      }
+    }
+  }
+
+  for (const semantics::Scope &child : scope.children()) {
+    CheckUnusedIntentHelper(context, child);
+  }
+}
+
+static void MakeProcBindingSymbolSet(semantics::SemanticsContext &context,
+                                     const semantics::Scope &scope) {
+  for (const auto &pair : scope) {
+    const semantics::Symbol &symbol = *pair.second;
+    if (auto *details{symbol.detailsIf<semantics::ProcBindingDetails>()}) {
+      procBindingDetailsSymbolsMap[&details->symbol()] = &symbol;
+    }
+  }
+
+  for (const semantics::Scope &child : scope.children()) {
+    MakeProcBindingSymbolSet(context, child);
+  }
+}
+
+UnusedIntentCheck::UnusedIntentCheck(llvm::StringRef name,
+                                     FlangTidyContext *context)
+    : FlangTidyCheck{name, context} {
+
+  MakeProcBindingSymbolSet(context->getSemanticsContext(),
+                           context->getSemanticsContext().globalScope());
+
+  CheckUnusedIntentHelper(context->getSemanticsContext(),
+                          context->getSemanticsContext().globalScope());
+}
+
+} // namespace Fortran::tidy::bugprone
diff --git a/flang/tools/flang-tidy/bugprone/UnusedIntentCheck.h b/flang/tools/flang-tidy/bugprone/UnusedIntentCheck.h
new file mode 100644
index 0000000000000..147febd9f498a
--- /dev/null
+++ b/flang/tools/flang-tidy/bugprone/UnusedIntentCheck.h
@@ -0,0 +1,33 @@
+//===--- UnusedIntentCheck.h - flang-tidy -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_UNUSEDINTENTCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_UNUSEDINTENTCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "../FlangTidyContext.h"
+
+namespace Fortran::tidy::bugprone {
+
+/// This check verifies that all INTENT attributes are used.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/unused-intent.html
+class UnusedIntentCheck : public virtual FlangTidyCheck {
+public:
+  UnusedIntentCheck(llvm::StringRef name, FlangTidyContext *context);
+  virtual ~UnusedIntentCheck() = default;
+
+private:
+  void CheckUnusedIntentHelper(semantics::SemanticsContext &,
+                               const semantics::Scope &);
+};
+
+} // namespace Fortran::tidy::bugprone
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_BUGPRONE_UNUSEDINTENTCHECK_H
diff --git a/flang/tools/test/flang-tidy/intent01.f90 b/flang/tools/test/flang-tidy/intent01.f90
new file mode 100644
index 0000000000000..d62ff08015491
--- /dev/null
+++ b/flang/tools/test/flang-tidy/intent01.f90
@@ -0,0 +1,6 @@
+! RUN: %check_flang_tidy %s bugprone-unused-intent %t
+subroutine s(a, b)
+  integer, intent(inout) :: a, b
+  ! CHECK-MESSAGES: :[[@LINE-1]]:32: warning: Dummy argument 'b' with intent(inout) is never written to, consider changing to intent(in)
+  a = a + 1 - b
+end subroutine s

>From d0e5d98ab2e521f9e9946a4926272d2a08ee49dc Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:40:58 +0200
Subject: [PATCH 23/31] [flang-tidy][modernize] add avoid assign check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../checks/modernize/avoid-assign-stmt.rst    | 23 ++++++++++++++
 .../flang-tidy/modernize/AvoidAssignStmt.cpp  | 29 +++++++++++++++++
 .../flang-tidy/modernize/AvoidAssignStmt.h    | 31 +++++++++++++++++++
 .../tools/flang-tidy/modernize/CMakeLists.txt |  2 +-
 .../modernize/ModernizeTidyModule.cpp         |  6 +++-
 flang/tools/test/flang-tidy/assign.f90        |  9 ++++++
 7 files changed, 99 insertions(+), 2 deletions(-)
 create mode 100644 flang/tools/docs/flang-tidy/checks/modernize/avoid-assign-stmt.rst
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidAssignStmt.cpp
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidAssignStmt.h
 create mode 100644 flang/tools/test/flang-tidy/assign.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 31decff549a5d..91174c0276e46 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -28,3 +28,4 @@ Flang-Tidy Checks
    :doc:`bugprone-subprogram-trampoline <bugprone/subprogram-trampoline>`,
    :doc:`bugprone-undeclared-procedure <bugprone/undeclared-procedure>`,
    :doc:`bugprone-unused-intent <bugprone/unused-intent>`,
+   :doc:`modernize-avoid-assign-stmt <modernize/avoid-assign-stmt>`,
diff --git a/flang/tools/docs/flang-tidy/checks/modernize/avoid-assign-stmt.rst b/flang/tools/docs/flang-tidy/checks/modernize/avoid-assign-stmt.rst
new file mode 100644
index 0000000000000..6ea49abd11f38
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/modernize/avoid-assign-stmt.rst
@@ -0,0 +1,23 @@
+.. title:: flang-tidy - modernize-avoid-assign-stmt
+
+modernize-avoid-assign-stmt
+===========================
+
+Warns about the use of obsolete ASSIGN and assigned GOTO statements, which were deprecated in Fortran 95 and removed from the Fortran 2018 standard. Modern Fortran code should use more structured alternatives.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      integer :: i
+
+      assign 100 to i  ! This will trigger a warning
+      goto i, (100, 200, 300)  ! This will trigger a warning
+
+    100 print *, "Label 100"
+      stop
+    200 print *, "Label 200"
+      stop
+    300 print *, "Label 300"
+      stop
+    end program
diff --git a/flang/tools/flang-tidy/modernize/AvoidAssignStmt.cpp b/flang/tools/flang-tidy/modernize/AvoidAssignStmt.cpp
new file mode 100644
index 0000000000000..3010f6dcc0f5a
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidAssignStmt.cpp
@@ -0,0 +1,29 @@
+//===--- AvoidAssignStmt.cpp - flang-tidy ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AvoidAssignStmt.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::modernize {
+
+using namespace parser::literals;
+void AvoidAssignStmtCheck::Enter(const parser::AssignStmt &) {
+  if (context()->getSemanticsContext().location().has_value()) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Assign statements are not recommended"_warn_en_US);
+  }
+}
+
+void AvoidAssignStmtCheck::Enter(const parser::AssignedGotoStmt &) {
+  if (context()->getSemanticsContext().location().has_value()) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Assigned Goto statements are not recommended"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::modernize
diff --git a/flang/tools/flang-tidy/modernize/AvoidAssignStmt.h b/flang/tools/flang-tidy/modernize/AvoidAssignStmt.h
new file mode 100644
index 0000000000000..7c6366180a68c
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidAssignStmt.h
@@ -0,0 +1,31 @@
+//===--- AvoidAssignStmt.h - flang-tidy -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDASSIGNSTMT_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDASSIGNSTMT_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::modernize {
+
+/// This check verifies that ASSIGN and ASSIGNED GOTO statements are avoided.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/avoid-assign-stmt.html
+class AvoidAssignStmtCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~AvoidAssignStmtCheck() = default;
+  void Enter(const parser::AssignStmt &) override;
+  void Enter(const parser::AssignedGotoStmt &) override;
+};
+
+} // namespace Fortran::tidy::modernize
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDASSIGNSTMT_H
diff --git a/flang/tools/flang-tidy/modernize/CMakeLists.txt b/flang/tools/flang-tidy/modernize/CMakeLists.txt
index bbb3098f76d9b..074a35ed0f698 100644
--- a/flang/tools/flang-tidy/modernize/CMakeLists.txt
+++ b/flang/tools/flang-tidy/modernize/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_flang_library(flangTidyModernizeModule STATIC
+  AvoidAssignStmt.cpp
   ModernizeTidyModule.cpp
 
   LINK_LIBS
@@ -15,4 +16,3 @@ add_flang_library(flangTidyModernizeModule STATIC
   FrontendOpenACC
   TargetParser
   )
-
diff --git a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
index f2717adac43ed..3709f6eac61ac 100644
--- a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
@@ -8,13 +8,17 @@
 
 #include "../FlangTidyModule.h"
 #include "../FlangTidyModuleRegistry.h"
+#include "AvoidAssignStmt.h"
 
 namespace Fortran::tidy {
 namespace modernize {
 
 class ModernizeModule : public FlangTidyModule {
 public:
-  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {
+    CheckFactories.registerCheck<AvoidAssignStmtCheck>(
+        "modernize-avoid-assign-stmt");
+  }
 };
 
 } // namespace modernize
diff --git a/flang/tools/test/flang-tidy/assign.f90 b/flang/tools/test/flang-tidy/assign.f90
new file mode 100644
index 0000000000000..ba3f28df8167d
--- /dev/null
+++ b/flang/tools/test/flang-tidy/assign.f90
@@ -0,0 +1,9 @@
+! RUN: %check_flang_tidy %s modernize-avoid-assign-stmt %t
+subroutine s
+  integer :: i
+  assign 9 to i
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Assign statements are not recommended    
+  go to i
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Assigned Goto statements are not recommended
+9 continue
+end subroutine s

>From f212c664ee12329527eada40f7ddaf6a8253d898 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:43:03 +0200
Subject: [PATCH 24/31] [flang-tidy][modernize] add avoid backspace check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../checks/modernize/avoid-backspace-stmt.rst | 19 ++++++++++++
 .../modernize/AvoidBackspaceStmt.cpp          | 22 ++++++++++++++
 .../flang-tidy/modernize/AvoidBackspaceStmt.h | 30 +++++++++++++++++++
 .../tools/flang-tidy/modernize/CMakeLists.txt |  1 +
 .../modernize/ModernizeTidyModule.cpp         |  3 ++
 .../tools/test/flang-tidy/avoid-backspace.f90 | 13 ++++++++
 7 files changed, 89 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/modernize/avoid-backspace-stmt.rst
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidBackspaceStmt.cpp
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidBackspaceStmt.h
 create mode 100644 flang/tools/test/flang-tidy/avoid-backspace.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 91174c0276e46..ce32ead03fb8d 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -29,3 +29,4 @@ Flang-Tidy Checks
    :doc:`bugprone-undeclared-procedure <bugprone/undeclared-procedure>`,
    :doc:`bugprone-unused-intent <bugprone/unused-intent>`,
    :doc:`modernize-avoid-assign-stmt <modernize/avoid-assign-stmt>`,
+   :doc:`modernize-avoid-backspace-stmt <modernize/avoid-backspace-stmt>`,
diff --git a/flang/tools/docs/flang-tidy/checks/modernize/avoid-backspace-stmt.rst b/flang/tools/docs/flang-tidy/checks/modernize/avoid-backspace-stmt.rst
new file mode 100644
index 0000000000000..3f1919887282e
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/modernize/avoid-backspace-stmt.rst
@@ -0,0 +1,19 @@
+.. title:: flang-tidy - modernize-avoid-backspace-stmt
+
+modernize-avoid-backspace-stmt
+==============================
+
+Identifies usage of the BACKSPACE statement, which can lead to inefficient I/O operations. Modern Fortran code should use more reliable file positioning methods.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      integer :: unit = 10, x
+
+      open(unit, file="data.txt", action="read")
+      read(unit, *) x
+      backspace(unit)  ! This will trigger a warning
+      read(unit, *) x
+      close(unit)
+    end program
diff --git a/flang/tools/flang-tidy/modernize/AvoidBackspaceStmt.cpp b/flang/tools/flang-tidy/modernize/AvoidBackspaceStmt.cpp
new file mode 100644
index 0000000000000..ee4ec011e3080
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidBackspaceStmt.cpp
@@ -0,0 +1,22 @@
+//===--- AvoidBackspaceStmt.cpp - flang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AvoidBackspaceStmt.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::modernize {
+
+using namespace parser::literals;
+void AvoidBackspaceStmtCheck::Enter(const parser::BackspaceStmt &) {
+  if (context()->getSemanticsContext().location().has_value()) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Assign statements are not recommended"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::modernize
diff --git a/flang/tools/flang-tidy/modernize/AvoidBackspaceStmt.h b/flang/tools/flang-tidy/modernize/AvoidBackspaceStmt.h
new file mode 100644
index 0000000000000..10a54beb43d88
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidBackspaceStmt.h
@@ -0,0 +1,30 @@
+//===--- AvoidBackspaceStmt.h - flang-tidy ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDBACKSPACESTMT_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDBACKSPACESTMT_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::modernize {
+
+/// This check verifies that BACKSPACE statements are avoided.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/avoid-backspace-stmt.html
+class AvoidBackspaceStmtCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~AvoidBackspaceStmtCheck() = default;
+  void Enter(const parser::BackspaceStmt &) override;
+};
+
+} // namespace Fortran::tidy::modernize
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDBACKSPACESTMT_H
diff --git a/flang/tools/flang-tidy/modernize/CMakeLists.txt b/flang/tools/flang-tidy/modernize/CMakeLists.txt
index 074a35ed0f698..4d8b152b17e22 100644
--- a/flang/tools/flang-tidy/modernize/CMakeLists.txt
+++ b/flang/tools/flang-tidy/modernize/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_flang_library(flangTidyModernizeModule STATIC
   AvoidAssignStmt.cpp
+  AvoidBackspaceStmt.cpp
   ModernizeTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
index 3709f6eac61ac..92ce7ad01c4ff 100644
--- a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
@@ -9,6 +9,7 @@
 #include "../FlangTidyModule.h"
 #include "../FlangTidyModuleRegistry.h"
 #include "AvoidAssignStmt.h"
+#include "AvoidBackspaceStmt.h"
 
 namespace Fortran::tidy {
 namespace modernize {
@@ -18,6 +19,8 @@ class ModernizeModule : public FlangTidyModule {
   void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {
     CheckFactories.registerCheck<AvoidAssignStmtCheck>(
         "modernize-avoid-assign-stmt");
+    CheckFactories.registerCheck<AvoidBackspaceStmtCheck>(
+        "modernize-avoid-backspace-stmt");
   }
 };
 
diff --git a/flang/tools/test/flang-tidy/avoid-backspace.f90 b/flang/tools/test/flang-tidy/avoid-backspace.f90
new file mode 100644
index 0000000000000..ba7598a470ab1
--- /dev/null
+++ b/flang/tools/test/flang-tidy/avoid-backspace.f90
@@ -0,0 +1,13 @@
+! RUN: %check_flang_tidy %s modernize-avoid-backspace-stmt %t
+program backspace_test
+  integer :: unit = 10, x
+
+  open(unit, file="data.txt", action="readwrite")
+  write(unit, *) 1, 2, 3
+
+  backspace(unit)  ! This will trigger a warning
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Assign statements are not recommended
+
+  read(unit, *) x
+  close(unit)
+end program backspace_test

>From 52768912aecdb72195e206968607f32d270a6d67 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:46:41 +0200
Subject: [PATCH 25/31] [flang-tidy][modernize] add avoid commonblock check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../checks/modernize/avoid-common-blocks.rst  | 26 +++++++++++++++++
 .../modernize/AvoidCommonBlocks.cpp           | 21 ++++++++++++++
 .../flang-tidy/modernize/AvoidCommonBlocks.h  | 29 +++++++++++++++++++
 .../tools/flang-tidy/modernize/CMakeLists.txt |  1 +
 .../modernize/ModernizeTidyModule.cpp         |  3 ++
 flang/tools/test/flang-tidy/commonblock.f90   |  6 ++++
 7 files changed, 87 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/modernize/avoid-common-blocks.rst
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidCommonBlocks.cpp
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidCommonBlocks.h
 create mode 100644 flang/tools/test/flang-tidy/commonblock.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index ce32ead03fb8d..d68167cf05118 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -30,3 +30,4 @@ Flang-Tidy Checks
    :doc:`bugprone-unused-intent <bugprone/unused-intent>`,
    :doc:`modernize-avoid-assign-stmt <modernize/avoid-assign-stmt>`,
    :doc:`modernize-avoid-backspace-stmt <modernize/avoid-backspace-stmt>`,
+   :doc:`modernize-avoid-common-blocks <modernize/avoid-common-blocks>`,
diff --git a/flang/tools/docs/flang-tidy/checks/modernize/avoid-common-blocks.rst b/flang/tools/docs/flang-tidy/checks/modernize/avoid-common-blocks.rst
new file mode 100644
index 0000000000000..f445ea5d80677
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/modernize/avoid-common-blocks.rst
@@ -0,0 +1,26 @@
+.. title:: flang-tidy - modernize-avoid-common-blocks
+
+modernize-avoid-common-blocks
+=============================
+
+Detects the use of COMMON blocks, which were replaced by modules in Fortran 90 for data sharing. Using modules provides better encapsulation, type safety, and explicit interfaces.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      integer :: a, b
+      common /data/ a, b  ! This will trigger a warning
+
+      a = 1
+      b = 2
+      call process()
+    end program
+
+    subroutine process()
+      implicit none
+      integer :: a, b
+      common /data/ a, b
+
+      print *, a, b
+    end subroutine
diff --git a/flang/tools/flang-tidy/modernize/AvoidCommonBlocks.cpp b/flang/tools/flang-tidy/modernize/AvoidCommonBlocks.cpp
new file mode 100644
index 0000000000000..f5831a20466f8
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidCommonBlocks.cpp
@@ -0,0 +1,21 @@
+//===--- AvoidCommonBlocks.cpp - flang-tidy -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AvoidCommonBlocks.h"
+
+namespace Fortran::tidy::modernize {
+
+using namespace parser::literals;
+void AvoidCommonBlocksCheck::Enter(const parser::CommonStmt &) {
+  if (context()->getSemanticsContext().location().has_value()) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Common blocks are not recommended"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::modernize
diff --git a/flang/tools/flang-tidy/modernize/AvoidCommonBlocks.h b/flang/tools/flang-tidy/modernize/AvoidCommonBlocks.h
new file mode 100644
index 0000000000000..f75f8f46850b6
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidCommonBlocks.h
@@ -0,0 +1,29 @@
+//===--- AvoidCommonBlocks.h - flang-tidy -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDCOMMONBLOCKS_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDCOMMONBLOCKS_H
+
+#include "../FlangTidyCheck.h"
+
+namespace Fortran::tidy::modernize {
+
+/// This check verifies that COMMON blocks are avoided.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/avoid-common-blocks.html
+class AvoidCommonBlocksCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~AvoidCommonBlocksCheck() = default;
+  void Enter(const parser::CommonStmt &) override;
+};
+
+} // namespace Fortran::tidy::modernize
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDCOMMONBLOCKS_H
diff --git a/flang/tools/flang-tidy/modernize/CMakeLists.txt b/flang/tools/flang-tidy/modernize/CMakeLists.txt
index 4d8b152b17e22..d65c3931ea215 100644
--- a/flang/tools/flang-tidy/modernize/CMakeLists.txt
+++ b/flang/tools/flang-tidy/modernize/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_flang_library(flangTidyModernizeModule STATIC
   AvoidAssignStmt.cpp
   AvoidBackspaceStmt.cpp
+  AvoidCommonBlocks.cpp
   ModernizeTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
index 92ce7ad01c4ff..79d3cbaa43d45 100644
--- a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
@@ -10,6 +10,7 @@
 #include "../FlangTidyModuleRegistry.h"
 #include "AvoidAssignStmt.h"
 #include "AvoidBackspaceStmt.h"
+#include "AvoidCommonBlocks.h"
 
 namespace Fortran::tidy {
 namespace modernize {
@@ -21,6 +22,8 @@ class ModernizeModule : public FlangTidyModule {
         "modernize-avoid-assign-stmt");
     CheckFactories.registerCheck<AvoidBackspaceStmtCheck>(
         "modernize-avoid-backspace-stmt");
+    CheckFactories.registerCheck<AvoidCommonBlocksCheck>(
+        "modernize-avoid-common-blocks");
   }
 };
 
diff --git a/flang/tools/test/flang-tidy/commonblock.f90 b/flang/tools/test/flang-tidy/commonblock.f90
new file mode 100644
index 0000000000000..58f0c502d321d
--- /dev/null
+++ b/flang/tools/test/flang-tidy/commonblock.f90
@@ -0,0 +1,6 @@
+! RUN: %check_flang_tidy %s modernize-avoid-common-blocks %t
+subroutine s
+  real :: x, y, z
+  common /c/ x, y, z
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Common blocks are not recommended
+end subroutine s

>From 5684c9c5d53f0433ceef5cf6b405752a82098d60 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:48:33 +0200
Subject: [PATCH 26/31] [flang-tidy][modernize] add avoid data-construct check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../modernize/avoid-data-constructs.rst       | 18 ++++++++++++
 .../modernize/AvoidDataConstructs.cpp         | 22 ++++++++++++++
 .../modernize/AvoidDataConstructs.h           | 29 +++++++++++++++++++
 .../tools/flang-tidy/modernize/CMakeLists.txt |  1 +
 .../modernize/ModernizeTidyModule.cpp         |  3 ++
 flang/tools/test/flang-tidy/avoid-data.f90    | 11 +++++++
 7 files changed, 85 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/modernize/avoid-data-constructs.rst
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidDataConstructs.cpp
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidDataConstructs.h
 create mode 100644 flang/tools/test/flang-tidy/avoid-data.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index d68167cf05118..7b52b50ba8e39 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -31,3 +31,4 @@ Flang-Tidy Checks
    :doc:`modernize-avoid-assign-stmt <modernize/avoid-assign-stmt>`,
    :doc:`modernize-avoid-backspace-stmt <modernize/avoid-backspace-stmt>`,
    :doc:`modernize-avoid-common-blocks <modernize/avoid-common-blocks>`,
+   :doc:`modernize-avoid-data-constructs <modernize/avoid-data-constructs>`,
diff --git a/flang/tools/docs/flang-tidy/checks/modernize/avoid-data-constructs.rst b/flang/tools/docs/flang-tidy/checks/modernize/avoid-data-constructs.rst
new file mode 100644
index 0000000000000..eb2b49661f2af
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/modernize/avoid-data-constructs.rst
@@ -0,0 +1,18 @@
+.. title:: flang-tidy - modernize-avoid-data-constructs
+
+modernize-avoid-data-constructs
+===============================
+
+Warns about the use of DATA statements, which provide a less flexible and less clear way to initialize variables compared to modern initialization approaches.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      integer :: array(3)
+
+      data array /1, 2, 3/  ! This will trigger a warning
+      ! Better: array = [1, 2, 3]
+
+      print *, array
+    end program
diff --git a/flang/tools/flang-tidy/modernize/AvoidDataConstructs.cpp b/flang/tools/flang-tidy/modernize/AvoidDataConstructs.cpp
new file mode 100644
index 0000000000000..c787c67d88626
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidDataConstructs.cpp
@@ -0,0 +1,22 @@
+//===--- AvoidDataConstructs.cpp - flang-tidy -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AvoidDataConstructs.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::modernize {
+
+using namespace parser::literals;
+void AvoidDataConstructsCheck::Enter(const parser::DataStmt &) {
+  if (context()->getSemanticsContext().location().has_value()) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Data statements are not recommended"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::modernize
diff --git a/flang/tools/flang-tidy/modernize/AvoidDataConstructs.h b/flang/tools/flang-tidy/modernize/AvoidDataConstructs.h
new file mode 100644
index 0000000000000..bf301a5c4a14e
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidDataConstructs.h
@@ -0,0 +1,29 @@
+//===--- AvoidDataConstructs.h - flang-tidy ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDDATACONSTRUCTS_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDDATACONSTRUCTS_H
+
+#include "../FlangTidyCheck.h"
+
+namespace Fortran::tidy::modernize {
+
+/// This check verifies that DATA constructs are avoided.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/avoid-data-constructs.html
+class AvoidDataConstructsCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~AvoidDataConstructsCheck() = default;
+  void Enter(const parser::DataStmt &) override;
+};
+
+} // namespace Fortran::tidy::modernize
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDDATACONSTRUCTS_H
diff --git a/flang/tools/flang-tidy/modernize/CMakeLists.txt b/flang/tools/flang-tidy/modernize/CMakeLists.txt
index d65c3931ea215..a49c09f1ab831 100644
--- a/flang/tools/flang-tidy/modernize/CMakeLists.txt
+++ b/flang/tools/flang-tidy/modernize/CMakeLists.txt
@@ -2,6 +2,7 @@ add_flang_library(flangTidyModernizeModule STATIC
   AvoidAssignStmt.cpp
   AvoidBackspaceStmt.cpp
   AvoidCommonBlocks.cpp
+  AvoidDataConstructs.cpp
   ModernizeTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
index 79d3cbaa43d45..9c43f11febc1e 100644
--- a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
@@ -11,6 +11,7 @@
 #include "AvoidAssignStmt.h"
 #include "AvoidBackspaceStmt.h"
 #include "AvoidCommonBlocks.h"
+#include "AvoidDataConstructs.h"
 
 namespace Fortran::tidy {
 namespace modernize {
@@ -24,6 +25,8 @@ class ModernizeModule : public FlangTidyModule {
         "modernize-avoid-backspace-stmt");
     CheckFactories.registerCheck<AvoidCommonBlocksCheck>(
         "modernize-avoid-common-blocks");
+    CheckFactories.registerCheck<AvoidDataConstructsCheck>(
+        "modernize-avoid-data-constructs");
   }
 };
 
diff --git a/flang/tools/test/flang-tidy/avoid-data.f90 b/flang/tools/test/flang-tidy/avoid-data.f90
new file mode 100644
index 0000000000000..2b8d8541ca110
--- /dev/null
+++ b/flang/tools/test/flang-tidy/avoid-data.f90
@@ -0,0 +1,11 @@
+! RUN: %check_flang_tidy %s modernize-avoid-data-constructs %t
+program data_test
+  integer :: array(3)
+
+  data array /1, 2, 3/  ! This will trigger a warning
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Data statements are not recommended
+
+  ! Better: array = [1, 2, 3]
+
+  print *, array
+end program data_test

>From 880ff9e8be3f830b108cc8356403363f1f9f20b4 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:50:23 +0200
Subject: [PATCH 27/31] [flang-tidy][modernize] add avoid pause check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |  1 +
 .../checks/modernize/avoid-pause-stmt.rst     | 16 ++++++++++
 .../flang-tidy/modernize/AvoidPauseStmt.cpp   | 22 ++++++++++++++
 .../flang-tidy/modernize/AvoidPauseStmt.h     | 30 +++++++++++++++++++
 .../tools/flang-tidy/modernize/CMakeLists.txt |  1 +
 .../modernize/ModernizeTidyModule.cpp         |  3 ++
 flang/tools/test/flang-tidy/avoid-pause.f90   | 11 +++++++
 7 files changed, 84 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/modernize/avoid-pause-stmt.rst
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidPauseStmt.cpp
 create mode 100644 flang/tools/flang-tidy/modernize/AvoidPauseStmt.h
 create mode 100644 flang/tools/test/flang-tidy/avoid-pause.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 7b52b50ba8e39..fab8fd5bbf8ee 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -32,3 +32,4 @@ Flang-Tidy Checks
    :doc:`modernize-avoid-backspace-stmt <modernize/avoid-backspace-stmt>`,
    :doc:`modernize-avoid-common-blocks <modernize/avoid-common-blocks>`,
    :doc:`modernize-avoid-data-constructs <modernize/avoid-data-constructs>`,
+   :doc:`modernize-avoid-pause-stmt <modernize/avoid-pause-stmt>`,
diff --git a/flang/tools/docs/flang-tidy/checks/modernize/avoid-pause-stmt.rst b/flang/tools/docs/flang-tidy/checks/modernize/avoid-pause-stmt.rst
new file mode 100644
index 0000000000000..4828294a266fe
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/modernize/avoid-pause-stmt.rst
@@ -0,0 +1,16 @@
+.. title:: flang-tidy - modernize-avoid-pause-stmt
+
+modernize-avoid-pause-stmt
+==========================
+
+Identifies usage of the PAUSE statement, which was deprecated in Fortran 95 and removed from the Fortran 2018 standard. Modern code should use READ or other interactive techniques instead.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+
+      print *, "Processing data..."
+      pause  ! This will trigger a warning
+      print *, "Continuing execution..."
+    end program
diff --git a/flang/tools/flang-tidy/modernize/AvoidPauseStmt.cpp b/flang/tools/flang-tidy/modernize/AvoidPauseStmt.cpp
new file mode 100644
index 0000000000000..a9d37a1ebe282
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidPauseStmt.cpp
@@ -0,0 +1,22 @@
+//===--- AvoidPauseStmt.cpp - flang-tidy ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AvoidPauseStmt.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::modernize {
+
+using namespace parser::literals;
+void AvoidPauseStmtCheck::Enter(const parser::PauseStmt &) {
+  if (context()->getSemanticsContext().location().has_value()) {
+    Say(context()->getSemanticsContext().location().value(),
+        "Pause statements are not recommended"_warn_en_US);
+  }
+}
+
+} // namespace Fortran::tidy::modernize
diff --git a/flang/tools/flang-tidy/modernize/AvoidPauseStmt.h b/flang/tools/flang-tidy/modernize/AvoidPauseStmt.h
new file mode 100644
index 0000000000000..abb32ed56e802
--- /dev/null
+++ b/flang/tools/flang-tidy/modernize/AvoidPauseStmt.h
@@ -0,0 +1,30 @@
+//===--- AvoidPauseStmt.h - flang-tidy --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDPAUSESTMT_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDPAUSESTMT_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::modernize {
+
+/// This check verifies that PAUSE statements are avoided.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/avoid-pause-stmt.html
+class AvoidPauseStmtCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~AvoidPauseStmtCheck() = default;
+  void Enter(const parser::PauseStmt &) override;
+};
+
+} // namespace Fortran::tidy::modernize
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_MODERNIZE_AVOIDPAUSESTMT_H
diff --git a/flang/tools/flang-tidy/modernize/CMakeLists.txt b/flang/tools/flang-tidy/modernize/CMakeLists.txt
index a49c09f1ab831..834d11599db00 100644
--- a/flang/tools/flang-tidy/modernize/CMakeLists.txt
+++ b/flang/tools/flang-tidy/modernize/CMakeLists.txt
@@ -3,6 +3,7 @@ add_flang_library(flangTidyModernizeModule STATIC
   AvoidBackspaceStmt.cpp
   AvoidCommonBlocks.cpp
   AvoidDataConstructs.cpp
+  AvoidPauseStmt.cpp
   ModernizeTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
index 9c43f11febc1e..9368c2e109369 100644
--- a/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/flang/tools/flang-tidy/modernize/ModernizeTidyModule.cpp
@@ -12,6 +12,7 @@
 #include "AvoidBackspaceStmt.h"
 #include "AvoidCommonBlocks.h"
 #include "AvoidDataConstructs.h"
+#include "AvoidPauseStmt.h"
 
 namespace Fortran::tidy {
 namespace modernize {
@@ -27,6 +28,8 @@ class ModernizeModule : public FlangTidyModule {
         "modernize-avoid-common-blocks");
     CheckFactories.registerCheck<AvoidDataConstructsCheck>(
         "modernize-avoid-data-constructs");
+    CheckFactories.registerCheck<AvoidPauseStmtCheck>(
+        "modernize-avoid-pause-stmt");
   }
 };
 
diff --git a/flang/tools/test/flang-tidy/avoid-pause.f90 b/flang/tools/test/flang-tidy/avoid-pause.f90
new file mode 100644
index 0000000000000..ea74a3c409935
--- /dev/null
+++ b/flang/tools/test/flang-tidy/avoid-pause.f90
@@ -0,0 +1,11 @@
+! RUN: %check_flang_tidy %s modernize-avoid-pause-stmt %t
+program pause_test
+  print *, "Processing data..."
+
+  pause  ! This will trigger a warning
+  ! CHECK-MESSAGES: :[[@LINE-1]]:3: warning: Pause statements are not recommended
+
+  ! Better: use READ or other interactive techniques instead
+
+  print *, "Continuing execution..."
+end program pause_test

>From a60ad87d6b0624a1395cef6a7bde9dd33e6e99da Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:53:05 +0200
Subject: [PATCH 28/31] [flang-tidy][openmp] add openmp accumulator race check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |   1 +
 .../checks/openmp/accumulator-race.rst        |  29 ++
 .../openmp/AccumulatorRaceCheck.cpp           | 265 ++++++++++++++++++
 .../flang-tidy/openmp/AccumulatorRaceCheck.h  |  42 +++
 flang/tools/flang-tidy/openmp/CMakeLists.txt  |   1 +
 .../flang-tidy/openmp/OpenMPTidyModule.cpp    |   6 +-
 .../test/flang-tidy/openmp-reduction01.f90    |  71 +++++
 7 files changed, 414 insertions(+), 1 deletion(-)
 create mode 100644 flang/tools/docs/flang-tidy/checks/openmp/accumulator-race.rst
 create mode 100644 flang/tools/flang-tidy/openmp/AccumulatorRaceCheck.cpp
 create mode 100644 flang/tools/flang-tidy/openmp/AccumulatorRaceCheck.h
 create mode 100644 flang/tools/test/flang-tidy/openmp-reduction01.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index fab8fd5bbf8ee..c319c165c30bb 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -33,3 +33,4 @@ Flang-Tidy Checks
    :doc:`modernize-avoid-common-blocks <modernize/avoid-common-blocks>`,
    :doc:`modernize-avoid-data-constructs <modernize/avoid-data-constructs>`,
    :doc:`modernize-avoid-pause-stmt <modernize/avoid-pause-stmt>`,
+   :doc:`openmp-accumulator-race <openmp/accumulator-race>`,
diff --git a/flang/tools/docs/flang-tidy/checks/openmp/accumulator-race.rst b/flang/tools/docs/flang-tidy/checks/openmp/accumulator-race.rst
new file mode 100644
index 0000000000000..7ffaaca90e19e
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/openmp/accumulator-race.rst
@@ -0,0 +1,29 @@
+.. title:: flang-tidy - openmp-accumulator-race
+
+openmp-accumulator-race
+=======================
+
+Detects potential race conditions in OpenMP parallel regions where variables are modified without proper protection. This check identifies assignments to shared variables that could lead to data races and inconsistent results in parallel execution.
+
+.. code-block:: fortran
+
+    program example
+      implicit none
+      integer :: sum = 0
+      integer :: i
+
+      !$omp parallel do
+      do i = 1, 100
+        sum = sum + i  ! This will trigger a warning - race condition
+      end do
+      !$omp end parallel do
+
+      !$omp parallel do
+      do i = 1, 100
+        !$omp atomic update
+        sum = sum + i  ! This is safe
+      end do
+      !$omp end parallel do
+
+      print *, "Sum:", sum
+    end program
diff --git a/flang/tools/flang-tidy/openmp/AccumulatorRaceCheck.cpp b/flang/tools/flang-tidy/openmp/AccumulatorRaceCheck.cpp
new file mode 100644
index 0000000000000..52dab7719b776
--- /dev/null
+++ b/flang/tools/flang-tidy/openmp/AccumulatorRaceCheck.cpp
@@ -0,0 +1,265 @@
+//===--- AccumulatorRaceCheck.cpp - flang-tidy ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AccumulatorRaceCheck.h"
+#include "flang/Evaluate/check-expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "llvm/ADT/SmallVector.h"
+#include <algorithm>
+#include <variant>
+#include <vector>
+
+namespace Fortran::tidy::openmp {
+
+using namespace parser::literals;
+
+static std::vector<llvm::SmallVector<const semantics::Symbol *, 4>>
+    currentPrivate{};
+
+void AccumulatorRaceCheck::Enter(
+    const parser::OpenMPBlockConstruct &construct) {
+  ++inParallelRegion_;
+
+  currentPrivate.push_back({});
+
+  const auto &directive = std::get<parser::OmpBeginBlockDirective>(construct.t);
+  const auto &clause = std::get<parser::OmpClauseList>(directive.t);
+
+  // only handle lastprivate, firstprivate, and private for now
+  for (const auto &clause : clause.v) {
+    if (std::holds_alternative<parser::OmpClause::Lastprivate>(clause.u)) {
+      const auto &lastprivateClause =
+          std::get<parser::OmpClause::Lastprivate>(clause.u);
+      const auto &ompObjectList =
+          std::get<parser::OmpObjectList>(lastprivateClause.v.t);
+      for (const auto &ompObject : ompObjectList.v) {
+
+        // TODO: is there a better way to do this?
+        if (const auto *name{std::get_if<parser::Name>(&ompObject.u)}) {
+          const auto *symbol = name->symbol;
+          if (symbol) {
+            currentPrivate.back().emplace_back(symbol);
+          }
+        }
+        // handle Designator -> DataRef -> Name
+        if (const auto *designator{
+                std::get_if<parser::Designator>(&ompObject.u)}) {
+          const auto *name = semantics::getDesignatorNameIfDataRef(*designator);
+          if (name) {
+            const auto *symbol = name->symbol;
+            if (symbol) {
+              currentPrivate.back().emplace_back(symbol);
+            }
+          }
+        }
+      }
+
+    } else if (std::holds_alternative<parser::OmpClause::Firstprivate>(
+                   clause.u)) {
+      const auto &firstprivateClause =
+          std::get<parser::OmpClause::Firstprivate>(clause.u);
+      const auto &ompObjectList = firstprivateClause.v;
+      for (const auto &ompObject : ompObjectList.v) {
+        // get the symbol, only do names for now
+        if (const auto *name{std::get_if<parser::Name>(&ompObject.u)}) {
+          const auto *symbol = name->symbol;
+          if (symbol) {
+            currentPrivate.back().emplace_back(symbol);
+          }
+        }
+        if (const auto *designator{
+                std::get_if<parser::Designator>(&ompObject.u)}) {
+          const auto *name = semantics::getDesignatorNameIfDataRef(*designator);
+          if (name) {
+            const auto *symbol = name->symbol;
+            if (symbol) {
+              currentPrivate.back().emplace_back(symbol);
+            }
+          }
+        }
+      }
+    } else if (std::holds_alternative<parser::OmpClause::Private>(clause.u)) {
+      const auto &privateClause =
+          std::get<parser::OmpClause::Private>(clause.u);
+      const auto &ompObjectList = privateClause.v;
+      for (const auto &ompObject : ompObjectList.v) {
+        // get the symbol, only do names for now
+        if (const auto *name{std::get_if<parser::Name>(&ompObject.u)}) {
+          const auto *symbol = name->symbol;
+          if (symbol) {
+            currentPrivate.back().emplace_back(symbol);
+          }
+        }
+        if (const auto *designator{
+                std::get_if<parser::Designator>(&ompObject.u)}) {
+          const auto *name = semantics::getDesignatorNameIfDataRef(*designator);
+          if (name) {
+            const auto *symbol = name->symbol;
+            if (symbol) {
+              currentPrivate.back().emplace_back(symbol);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+void AccumulatorRaceCheck::Leave(const parser::OpenMPBlockConstruct &) {
+  --inParallelRegion_;
+  currentPrivate.pop_back();
+}
+
+void AccumulatorRaceCheck::Enter(const parser::OpenMPLoopConstruct &construct) {
+  ++inParallelRegion_;
+
+  // extract all symbols
+  currentPrivate.push_back({}); // push a new private list
+
+  const auto &directive = std::get<parser::OmpBeginLoopDirective>(construct.t);
+  const auto &clause = std::get<parser::OmpClauseList>(directive.t);
+  for (const auto &clause : clause.v) {
+    if (std::holds_alternative<parser::OmpClause::Lastprivate>(clause.u)) {
+      const auto &lastprivateClause =
+          std::get<parser::OmpClause::Lastprivate>(clause.u);
+      const auto &ompObjectList =
+          std::get<parser::OmpObjectList>(lastprivateClause.v.t);
+      for (const auto &ompObject : ompObjectList.v) {
+        // get the symbol, only do names for now
+        if (const auto *name{std::get_if<parser::Name>(&ompObject.u)}) {
+          const auto *symbol = name->symbol;
+          if (symbol) {
+            currentPrivate.back().emplace_back(symbol);
+          }
+        }
+        if (const auto *designator{
+                std::get_if<parser::Designator>(&ompObject.u)}) {
+          const auto *name = semantics::getDesignatorNameIfDataRef(*designator);
+          if (name) {
+            const auto *symbol = name->symbol;
+            if (symbol) {
+              currentPrivate.back().emplace_back(symbol);
+            }
+          }
+        }
+      }
+
+    } else if (std::holds_alternative<parser::OmpClause::Firstprivate>(
+                   clause.u)) {
+      const auto &firstprivateClause =
+          std::get<parser::OmpClause::Firstprivate>(clause.u);
+      const auto &ompObjectList = firstprivateClause.v;
+      for (const auto &ompObject : ompObjectList.v) {
+        // get the symbol, only do names for now
+        if (const auto *name{std::get_if<parser::Name>(&ompObject.u)}) {
+          const auto *symbol = name->symbol;
+          if (symbol) {
+            currentPrivate.back().emplace_back(symbol);
+          }
+        }
+        if (const auto *designator{
+                std::get_if<parser::Designator>(&ompObject.u)}) {
+          const auto *name = semantics::getDesignatorNameIfDataRef(*designator);
+          if (name) {
+            const auto *symbol = name->symbol;
+            if (symbol) {
+              currentPrivate.back().emplace_back(symbol);
+            }
+          }
+        }
+      }
+    } else if (std::holds_alternative<parser::OmpClause::Private>(clause.u)) {
+      const auto &privateClause =
+          std::get<parser::OmpClause::Private>(clause.u);
+      const auto &ompObjectList = privateClause.v;
+      for (const auto &ompObject : ompObjectList.v) {
+        // get the symbol, only do names for now
+        if (const auto *name{std::get_if<parser::Name>(&ompObject.u)}) {
+          const auto *symbol = name->symbol;
+          if (symbol) {
+            currentPrivate.back().emplace_back(symbol);
+          }
+        }
+        if (const auto *designator{
+                std::get_if<parser::Designator>(&ompObject.u)}) {
+          const auto *name = semantics::getDesignatorNameIfDataRef(*designator);
+          if (name) {
+            const auto *symbol = name->symbol;
+            if (symbol) {
+              currentPrivate.back().emplace_back(symbol);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+void AccumulatorRaceCheck::Leave(const parser::OpenMPLoopConstruct &) {
+  --inParallelRegion_;
+  currentPrivate.pop_back();
+}
+
+void AccumulatorRaceCheck::Enter(const parser::OmpAtomicUpdate &) {
+  ++inAtomicUpdate_;
+}
+
+void AccumulatorRaceCheck::Leave(const parser::OmpAtomicUpdate &) {
+  --inAtomicUpdate_;
+}
+
+void AccumulatorRaceCheck::Enter(const parser::OpenMPCriticalConstruct &) {
+  ++inCriticalSection_;
+}
+
+void AccumulatorRaceCheck::Leave(const parser::OpenMPCriticalConstruct &) {
+  --inCriticalSection_;
+}
+
+void AccumulatorRaceCheck::Enter(const parser::AssignmentStmt &stmt) {
+  if (inParallelRegion_ && !inCriticalSection_ && !inAtomicUpdate_) {
+    const auto &var = std::get<parser::Variable>(stmt.t);
+    const auto &expr = std::get<parser::Expr>(stmt.t);
+
+    const auto *lhsExpr =
+        semantics::GetExpr(context()->getSemanticsContext(), var);
+    // if the lhs isnt a whole or component data ref (NO ARRAY), ignore
+    const semantics::Symbol *lhsSymbol =
+        evaluate::UnwrapWholeSymbolOrComponentDataRef(*lhsExpr);
+
+    if (!lhsSymbol) {
+      return;
+    }
+
+    // if the lhs is in any of the private lists, ignore (stack has no begin()
+    // iterator)
+    bool isPrivate =
+        std::any_of(currentPrivate.rbegin(), currentPrivate.rend(),
+                    [&lhsSymbol](const auto &privateList) {
+                      return std::find(privateList.begin(), privateList.end(),
+                                       lhsSymbol) != privateList.end();
+                    });
+
+    if (isPrivate) {
+      return;
+    }
+
+    const auto *rhsExpr =
+        semantics::GetExpr(context()->getSemanticsContext(), expr);
+
+    if (rhsExpr && !evaluate::IsConstantExpr(*rhsExpr) && lhsSymbol) {
+      Say(var.GetSource(), "possible race condition on '%s'"_warn_en_US,
+          lhsSymbol->name());
+    }
+  }
+}
+
+} // namespace Fortran::tidy::openmp
diff --git a/flang/tools/flang-tidy/openmp/AccumulatorRaceCheck.h b/flang/tools/flang-tidy/openmp/AccumulatorRaceCheck.h
new file mode 100644
index 0000000000000..859b202847c40
--- /dev/null
+++ b/flang/tools/flang-tidy/openmp/AccumulatorRaceCheck.h
@@ -0,0 +1,42 @@
+//===--- AccumulatorRaceCheck.h - flang-tidy --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_OPENMP_ACCUMULATORRACECHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_OPENMP_ACCUMULATORRACECHECK_H
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::openmp {
+
+/// This check tries to detect potential race conditions in OpenMP accumulators.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/accumulator-race.html
+class AccumulatorRaceCheck : public virtual FlangTidyCheck {
+public:
+  using FlangTidyCheck::FlangTidyCheck;
+  virtual ~AccumulatorRaceCheck() = default;
+
+  void Enter(const parser::OpenMPBlockConstruct &) override;
+  void Leave(const parser::OpenMPBlockConstruct &) override;
+  void Enter(const parser::OpenMPLoopConstruct &) override;
+  void Leave(const parser::OpenMPLoopConstruct &) override;
+  void Enter(const parser::OmpAtomicUpdate &) override;
+  void Leave(const parser::OmpAtomicUpdate &) override;
+  void Enter(const parser::OpenMPCriticalConstruct &) override;
+  void Leave(const parser::OpenMPCriticalConstruct &) override;
+  void Enter(const parser::AssignmentStmt &) override;
+
+private:
+  int inParallelRegion_ = 0;
+  int inCriticalSection_ = 0;
+  int inAtomicUpdate_ = 0;
+};
+
+} // namespace Fortran::tidy::openmp
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_OPENMP_ACCUMULATORRACECHECK_H
diff --git a/flang/tools/flang-tidy/openmp/CMakeLists.txt b/flang/tools/flang-tidy/openmp/CMakeLists.txt
index 69e07c16a3c88..d2575dd92c2d4 100644
--- a/flang/tools/flang-tidy/openmp/CMakeLists.txt
+++ b/flang/tools/flang-tidy/openmp/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_flang_library(flangTidyOpenMPModule STATIC
+  AccumulatorRaceCheck.cpp
   OpenMPTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/flang-tidy/openmp/OpenMPTidyModule.cpp b/flang/tools/flang-tidy/openmp/OpenMPTidyModule.cpp
index 456bb3dbc6fbf..eab0a8548ee26 100644
--- a/flang/tools/flang-tidy/openmp/OpenMPTidyModule.cpp
+++ b/flang/tools/flang-tidy/openmp/OpenMPTidyModule.cpp
@@ -8,13 +8,17 @@
 
 #include "../FlangTidyModule.h"
 #include "../FlangTidyModuleRegistry.h"
+#include "AccumulatorRaceCheck.h"
 
 namespace Fortran::tidy {
 namespace openmp {
 
 class OpenMPModule : public FlangTidyModule {
 public:
-  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {
+    CheckFactories.registerCheck<AccumulatorRaceCheck>(
+        "openmp-accumulator-race");
+  }
 };
 
 } // namespace openmp
diff --git a/flang/tools/test/flang-tidy/openmp-reduction01.f90 b/flang/tools/test/flang-tidy/openmp-reduction01.f90
new file mode 100644
index 0000000000000..2b7b973f927d3
--- /dev/null
+++ b/flang/tools/test/flang-tidy/openmp-reduction01.f90
@@ -0,0 +1,71 @@
+ ! RUN: %check_flang_tidy %s openmp-accumulator-race %t -- --extra-arg=-fopenmp
+subroutine loop1(acc, N)
+  integer, intent(inout) :: acc
+  integer, intent(in) :: N
+  integer :: m
+  !$omp parallel
+  !$omp do
+  do m = 1, N
+     acc = acc + N
+     ! CHECK-MESSAGES: :[[@LINE-1]]:6: warning: possible race condition on 'acc'
+  end do
+  !$omp end parallel
+end subroutine loop1
+
+subroutine loop2(acc, N)
+  integer, intent(inout) :: acc
+  integer, intent(in) :: N
+  integer :: m
+  !$omp parallel do
+  do m = 1, N
+     !$omp critical
+     acc = acc + N
+     !$omp end critical
+  end do
+  !$omp end parallel do
+end subroutine loop2
+
+subroutine loop3(acc, N)
+  integer, intent(inout) :: acc
+  integer, intent(in) :: N
+  integer :: m
+  !$omp parallel do
+  do m = 1, N
+     !$omp atomic update
+     acc = acc + N
+  end do
+  !$omp end parallel do
+end subroutine loop3
+
+subroutine loop4(acc, N)
+  integer, intent(inout) :: acc
+  integer, intent(in) :: N
+  integer :: m
+  !$omp parallel do private(acc)
+  do m = 1, N
+     acc = acc + N
+  end do
+  !$omp end parallel do
+end subroutine loop4
+
+subroutine loop5(acc, N)
+  integer, intent(inout) :: acc
+  integer, intent(in) :: N
+  integer :: m
+  !$omp parallel do firstprivate(acc)
+  do m = 1, N
+     acc = acc + N
+  end do
+  !$omp end parallel do
+end subroutine loop5
+
+subroutine loop6(acc, N)
+  integer, intent(inout) :: acc
+  integer, intent(in) :: N
+  integer :: m
+  !$omp parallel do lastprivate(acc)
+  do m = 1, N
+     acc = acc + N
+  end do
+  !$omp end parallel do
+end subroutine loop6

>From 5384fe8c67e454b2d16dbf4fee5f748bab5f40b9 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:54:57 +0200
Subject: [PATCH 29/31] [flang-tidy][performance] add pure procedure check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |   1 +
 .../checks/performance/pure-procedure.rst     |  24 ++
 .../flang-tidy/performance/CMakeLists.txt     |   1 +
 .../performance/PerformanceTidyModule.cpp     |   6 +-
 .../performance/PureProcedureCheck.cpp        | 293 ++++++++++++++++++
 .../performance/PureProcedureCheck.h          |  61 ++++
 .../tools/test/flang-tidy/pure-procedure.f90  |  25 ++
 7 files changed, 410 insertions(+), 1 deletion(-)
 create mode 100644 flang/tools/docs/flang-tidy/checks/performance/pure-procedure.rst
 create mode 100644 flang/tools/flang-tidy/performance/PureProcedureCheck.cpp
 create mode 100644 flang/tools/flang-tidy/performance/PureProcedureCheck.h
 create mode 100644 flang/tools/test/flang-tidy/pure-procedure.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index c319c165c30bb..89f4ac10b22e9 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -34,3 +34,4 @@ Flang-Tidy Checks
    :doc:`modernize-avoid-data-constructs <modernize/avoid-data-constructs>`,
    :doc:`modernize-avoid-pause-stmt <modernize/avoid-pause-stmt>`,
    :doc:`openmp-accumulator-race <openmp/accumulator-race>`,
+   :doc:`performance-pure-procedure <performance/pure-procedure>`,
diff --git a/flang/tools/docs/flang-tidy/checks/performance/pure-procedure.rst b/flang/tools/docs/flang-tidy/checks/performance/pure-procedure.rst
new file mode 100644
index 0000000000000..9e53a1e2c9f27
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/performance/pure-procedure.rst
@@ -0,0 +1,24 @@
+.. title:: flang-tidy - performance-pure-procedure
+
+performance-pure-procedure
+==========================
+
+Identifies procedures that could be declared as PURE but are not. PURE procedures have no side effects and can enable compiler optimizations, particularly for array operations and parallel processing. This check analyzes procedures and suggests adding the PURE attribute when possible.
+
+.. code-block:: fortran
+
+    ! This will trigger a warning - could be PURE
+    subroutine add(a, b, result)
+      implicit none
+      integer, intent(in) :: a, b
+      integer, intent(out) :: result
+      result = a + b
+    end subroutine
+
+    ! Fixed version
+    pure subroutine add_pure(a, b, result)
+      implicit none
+      integer, intent(in) :: a, b
+      integer, intent(out) :: result
+      result = a + b
+    end subroutine
diff --git a/flang/tools/flang-tidy/performance/CMakeLists.txt b/flang/tools/flang-tidy/performance/CMakeLists.txt
index 0a2690d24ea3a..8bcac0b322df3 100644
--- a/flang/tools/flang-tidy/performance/CMakeLists.txt
+++ b/flang/tools/flang-tidy/performance/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_flang_library(flangTidyPerformanceModule STATIC
   PerformanceTidyModule.cpp
+  PureProcedureCheck.cpp
 
   LINK_LIBS
   FortranSupport
diff --git a/flang/tools/flang-tidy/performance/PerformanceTidyModule.cpp b/flang/tools/flang-tidy/performance/PerformanceTidyModule.cpp
index 70eda2ecf838e..e16c634d68db6 100644
--- a/flang/tools/flang-tidy/performance/PerformanceTidyModule.cpp
+++ b/flang/tools/flang-tidy/performance/PerformanceTidyModule.cpp
@@ -8,13 +8,17 @@
 
 #include "../FlangTidyModule.h"
 #include "../FlangTidyModuleRegistry.h"
+#include "PureProcedureCheck.h"
 
 namespace Fortran::tidy {
 namespace performance {
 
 class PerformanceTidyModule : public FlangTidyModule {
 public:
-  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {
+    CheckFactories.registerCheck<PureProcedureCheck>(
+        "performance-pure-procedure");
+  }
 };
 
 } // namespace performance
diff --git a/flang/tools/flang-tidy/performance/PureProcedureCheck.cpp b/flang/tools/flang-tidy/performance/PureProcedureCheck.cpp
new file mode 100644
index 0000000000000..61926baca3e8c
--- /dev/null
+++ b/flang/tools/flang-tidy/performance/PureProcedureCheck.cpp
@@ -0,0 +1,293 @@
+//===--- PureProcedureCheck.cpp - flang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PureProcedureCheck.h"
+#include "../utils/SymbolUtils.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Semantics/scope.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+namespace Fortran::tidy::performance {
+
+static void PopulateProcedures(
+    const semantics::Scope &scope,
+    std::unordered_map<const semantics::Symbol *, bool> &pureProcedures) {
+  if (scope.IsModuleFile())
+    return;
+
+  for (const auto &pair : scope) {
+    const semantics::Symbol &symbol{*pair.second};
+    const auto &ultimate = symbol.GetUltimate();
+    const auto *subprogramDetails =
+        ultimate.detailsIf<semantics::SubprogramDetails>();
+    if (semantics::IsProcedure(symbol) && !utils::IsFromModFileSafe(ultimate) &&
+        (subprogramDetails && !subprogramDetails->isInterface())) {
+      pureProcedures[&ultimate] = true;
+    }
+  }
+
+  for (const auto &child : scope.children()) {
+    PopulateProcedures(child, pureProcedures);
+  }
+}
+
+static bool CheckSymbolIsPure(const semantics::Symbol &symbol,
+                              const semantics::Scope &scope) {
+  if (scope.symbol()) {
+    if (const auto &details =
+            scope.symbol()->detailsIf<semantics::SubprogramDetails>();
+        !(details && details->isInterface()) &&
+        !semantics::FindCommonBlockContaining(symbol) && IsSaved(symbol)) {
+      return false;
+    }
+  }
+
+  if (symbol.attrs().test(semantics::Attr::VOLATILE) && IsDummy(symbol)) {
+    return false;
+  }
+
+  if (IsProcedure(symbol) && !IsPureProcedure(symbol) && IsDummy(symbol)) {
+    return false;
+  }
+
+  return true;
+}
+
+static void CheckPureSymbols(
+    const semantics::Scope &scope,
+    std::unordered_map<const semantics::Symbol *, bool> &pureProcedures) {
+
+  if (scope.IsModuleFile())
+    return;
+
+  if (!scope.IsTopLevel()) {
+    for (const auto &pair : scope) {
+      const semantics::Symbol &symbol{*pair.second};
+      const semantics::Scope &scope2{symbol.owner()};
+      const semantics::Scope &unit{GetProgramUnitContaining(scope2)};
+      // unit should not be an interface
+      if (!CheckSymbolIsPure(symbol, scope) &&
+          unit.symbol() != &symbol.GetUltimate()) {
+        pureProcedures[&unit.symbol()->GetUltimate()] = false;
+      }
+    }
+  }
+  for (const auto &child : scope.children()) {
+    CheckPureSymbols(child, pureProcedures);
+  }
+}
+
+static std::unordered_map<const semantics::Symbol *, const semantics::Symbol *>
+    procBindingDetailsSymbolsMap;
+
+static void MakeProcBindingSymbolSet(semantics::SemanticsContext &context,
+                                     const semantics::Scope &scope) {
+  for (const auto &pair : scope) {
+    const semantics::Symbol &symbol = *pair.second;
+    if (auto *details{symbol.detailsIf<semantics::ProcBindingDetails>()}) {
+      procBindingDetailsSymbolsMap[&details->symbol().GetUltimate()] = &symbol;
+    }
+  }
+
+  for (const semantics::Scope &child : scope.children()) {
+    MakeProcBindingSymbolSet(context, child);
+  }
+}
+
+PureProcedureCheck::PureProcedureCheck(llvm::StringRef name,
+                                       FlangTidyContext *context)
+    : FlangTidyCheck{name, context} {
+  MakeProcBindingSymbolSet(context->getSemanticsContext(),
+                           context->getSemanticsContext().globalScope());
+
+  PopulateProcedures(context->getSemanticsContext().globalScope(),
+                     pureProcedures_);
+  CheckPureSymbols(context->getSemanticsContext().globalScope(),
+                   pureProcedures_);
+}
+
+using namespace parser::literals;
+void PureProcedureCheck::SetImpure() {
+  const auto location{context()->getSemanticsContext().location()};
+  if (!location) {
+    return;
+  }
+
+  const semantics::Scope &scope{
+      context()->getSemanticsContext().FindScope(*location)};
+
+  if (scope.IsTopLevel())
+    return;
+
+  const semantics::Scope &unit{GetProgramUnitContaining(scope)};
+  const auto &ultimateSymbol = unit.symbol()->GetUltimate();
+
+  if (semantics::IsProcedure(unit)) {
+    pureProcedures_[&ultimateSymbol] = false;
+  }
+
+  // propagate impurity to other procedures
+  for (const auto &pair : procBindingDetailsSymbolsMap) {
+    if (pair.first == &ultimateSymbol) {
+      pureProcedures_[pair.second] = false;
+    }
+  }
+}
+
+// C1596: external I/O
+void PureProcedureCheck::Leave(const parser::BackspaceStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::CloseStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::EndfileStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::FlushStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::InquireStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::OpenStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::PrintStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::RewindStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::WaitStmt &) { SetImpure(); }
+// C1597: read/write
+void PureProcedureCheck::Leave(const parser::ReadStmt &) { SetImpure(); }
+void PureProcedureCheck::Leave(const parser::WriteStmt &) { SetImpure(); }
+
+// assignment
+static bool IsPointerDummyOfPureFunction(const semantics::Symbol &x) {
+  return IsPointerDummy(x) && FindPureProcedureContaining(x.owner()) &&
+         x.owner().symbol() && IsFunction(*x.owner().symbol());
+}
+
+static const char *WhyBaseObjectIsSuspicious(const semantics::Symbol &x,
+                                             const semantics::Scope &scope) {
+  if (IsHostAssociatedIntoSubprogram(x, scope)) {
+    return "host-associated";
+  }
+  if (IsUseAssociated(x, scope)) {
+    return "USE-associated";
+  }
+  if (IsPointerDummyOfPureFunction(x)) {
+    return "a POINTER dummy argument of a pure function";
+  }
+  if (IsIntentIn(x)) {
+    return "an INTENT(IN) dummy argument";
+  }
+  if (FindCommonBlockContaining(x)) {
+    return "in a COMMON block";
+  }
+  return nullptr;
+}
+
+static std::optional<std::string>
+GetPointerComponentDesignatorName(const semantics::SomeExpr &expr) {
+  if (const auto *derived{
+          evaluate::GetDerivedTypeSpec(evaluate::DynamicType::From(expr))}) {
+    semantics::PotentialAndPointerComponentIterator potentials{*derived};
+    if (auto pointer{std::find_if(potentials.begin(), potentials.end(),
+                                  semantics::IsPointer)}) {
+      return pointer.BuildResultDesignatorName();
+    }
+  }
+  return std::nullopt;
+}
+
+// C1593 (4)
+void PureProcedureCheck::Leave(const parser::AssignmentStmt &assignment) {
+  // get the rhs
+  const auto &rhs{std::get<parser::Expr>(assignment.t)};
+  // get the evaluate::Expr
+  const auto *expr{semantics::GetExpr(context()->getSemanticsContext(), rhs)};
+
+  const semantics::Scope &scope{
+      context()->getSemanticsContext().FindScope(rhs.source)};
+
+  if (const semantics::Symbol *base{GetFirstSymbol(expr)}) {
+    if (const char *why{
+            WhyBaseObjectIsSuspicious(base->GetUltimate(), scope)}) {
+      if (auto pointer{GetPointerComponentDesignatorName(*expr)}) {
+        // mark the procedure as impure
+        (void)why;
+        SetImpure();
+      }
+    }
+  }
+}
+
+// C1592
+void PureProcedureCheck::Leave(const parser::Name &n) {
+  if (n.symbol && n.symbol->attrs().test(semantics::Attr::VOLATILE)) {
+    SetImpure();
+  }
+}
+
+// C1598: pure procs cant have image control statements
+void PureProcedureCheck::Enter(const parser::ExecutableConstruct &exec) {
+  if (semantics::IsImageControlStmt(exec)) {
+    SetImpure();
+  }
+}
+
+// check Call Stmt
+void PureProcedureCheck::Enter(const parser::CallStmt &callStmt) {
+  const auto *procedureRef = callStmt.typedCall.get();
+  if (procedureRef) {
+    const auto *symbol{procedureRef->proc().GetSymbol()};
+    if (!symbol) {
+      return;
+    }
+
+    // if the called function isnt pure, we cant be pure
+    if (!semantics::IsPureProcedure(*symbol))
+      SetImpure();
+  }
+}
+
+void PureProcedureCheck::Leave(const parser::Program &program) {
+  // tell us about all the procedure that could be pure, but arent
+  for (const auto &pair : pureProcedures_) {
+    // it should be pure, but isnt - and its not intrinsic, and not elemental,
+    // and not an interface
+    // check if the symbol is the Ultimate symbol
+    const auto &symbol = pair.first->GetUltimate();
+    // if (symbol != *pair.first) {
+    //   continue;
+    // }
+
+    // make sure its not being mapped to from procBindingDetailsSymbolsMap
+    bool cont = false;
+    for (const auto &procBindingPair : procBindingDetailsSymbolsMap) {
+      if (procBindingPair.second == pair.first) {
+        cont = true;
+        break;
+      }
+    }
+    if (cont) {
+      continue;
+    }
+
+    if (utils::IsFromModFileSafe(symbol)) {
+      continue;
+    }
+
+    // TODO: skip interfaces
+    if (pair.second && !pair.first->attrs().test(semantics::Attr::PURE) &&
+        !pair.first->attrs().test(semantics::Attr::INTRINSIC) &&
+        !pair.first->attrs().test(semantics::Attr::ELEMENTAL) &&
+        !pair.first->attrs().test(semantics::Attr::ABSTRACT) &&
+        !pair.first->attrs().test(semantics::Attr::EXTERNAL) &&
+        !pair.first->IsFromModFile()
+        // if its been derived somewhere, we dont care
+        && procBindingDetailsSymbolsMap.find(pair.first) ==
+               procBindingDetailsSymbolsMap.end()) {
+      Say(pair.first->name(),
+          "Procedure '%s' could be PURE but is not"_warn_en_US,
+          pair.first->name());
+    }
+  }
+}
+
+} // namespace Fortran::tidy::performance
diff --git a/flang/tools/flang-tidy/performance/PureProcedureCheck.h b/flang/tools/flang-tidy/performance/PureProcedureCheck.h
new file mode 100644
index 0000000000000..826780829fa4a
--- /dev/null
+++ b/flang/tools/flang-tidy/performance/PureProcedureCheck.h
@@ -0,0 +1,61 @@
+//===--- PureProcedureCheck.h - flang-tidy ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_PERFORMANCE_PUREPROCEDURECHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_PERFORMANCE_PUREPROCEDURECHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "../FlangTidyContext.h"
+#include "flang/Parser/parse-tree.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace Fortran::tidy::performance {
+
+/// This check warns about procedures that could be pure but are not.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/pure-procedure.html
+class PureProcedureCheck : public virtual FlangTidyCheck {
+public:
+  explicit PureProcedureCheck(llvm::StringRef name, FlangTidyContext *context);
+  virtual ~PureProcedureCheck() = default;
+  void SetImpure();
+
+  // I/O
+  void Leave(const parser::BackspaceStmt &) override;
+  void Leave(const parser::CloseStmt &) override;
+  void Leave(const parser::EndfileStmt &) override;
+  void Leave(const parser::FlushStmt &) override;
+  void Leave(const parser::InquireStmt &) override;
+  void Leave(const parser::OpenStmt &) override;
+  void Leave(const parser::PrintStmt &) override;
+  void Leave(const parser::ReadStmt &) override;
+  void Leave(const parser::RewindStmt &) override;
+  void Leave(const parser::WaitStmt &) override;
+  void Leave(const parser::WriteStmt &) override;
+
+  // Assignment
+  void Leave(const parser::AssignmentStmt &) override;
+
+  void Enter(const parser::ExecutableConstruct &) override;
+  void Enter(const parser::CallStmt &) override;
+
+  // ?
+  void Leave(const parser::Name &) override;
+
+  // Leave the whole program
+  void Leave(const parser::Program &) override;
+
+private:
+  // map of all procedures and their pure status
+  std::unordered_map<const semantics::Symbol *, bool> pureProcedures_;
+};
+
+} // namespace Fortran::tidy::performance
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_PERFORMANCE_PUREPROCEDURECHECK_H
diff --git a/flang/tools/test/flang-tidy/pure-procedure.f90 b/flang/tools/test/flang-tidy/pure-procedure.f90
new file mode 100644
index 0000000000000..9604d1a7e2316
--- /dev/null
+++ b/flang/tools/test/flang-tidy/pure-procedure.f90
@@ -0,0 +1,25 @@
+! RUN: %check_flang_tidy %s performance-pure-procedure %t
+module pure_test
+  implicit none
+contains
+  ! This function could be pure but isn't
+  ! CHECK-MESSAGES: :[[@LINE+1]]:12: warning: Procedure 'add' could be PURE but is not
+  function add(a, b) result(c)
+    integer, intent(in) :: a, b
+    integer :: c
+    c = a + b
+  end function add
+
+  ! I/O statements make this procedure impossible to be pure
+  subroutine print_sum(a, b)
+    integer, intent(in) :: a, b
+    print *, "Sum:", a + b
+  end subroutine print_sum
+
+  ! A properly declared pure function
+  pure function multiply(a, b) result(c)
+    integer, intent(in) :: a, b
+    integer :: c
+    c = a * b
+  end function multiply
+end module pure_test

>From 342e79fa5690c9ec30092e93f41b18e4cf528ea9 Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:56:51 +0200
Subject: [PATCH 30/31] [flang-tidy][readability] add function cognitive
 complexity check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |   1 +
 .../function-cognitive-complexity.rst         |  45 ++++++
 .../flang-tidy/readability/CMakeLists.txt     |   1 +
 .../FunctionCognitiveComplexityCheck.cpp      | 144 ++++++++++++++++++
 .../FunctionCognitiveComplexityCheck.h        |  73 +++++++++
 .../readability/ReadabilityTidyModule.cpp     |   6 +-
 .../test/flang-tidy/function-complexity01.f90 | 128 ++++++++++++++++
 7 files changed, 397 insertions(+), 1 deletion(-)
 create mode 100644 flang/tools/docs/flang-tidy/checks/readability/function-cognitive-complexity.rst
 create mode 100644 flang/tools/flang-tidy/readability/FunctionCognitiveComplexityCheck.cpp
 create mode 100644 flang/tools/flang-tidy/readability/FunctionCognitiveComplexityCheck.h
 create mode 100644 flang/tools/test/flang-tidy/function-complexity01.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index 89f4ac10b22e9..fb1c1aac34f26 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -35,3 +35,4 @@ Flang-Tidy Checks
    :doc:`modernize-avoid-pause-stmt <modernize/avoid-pause-stmt>`,
    :doc:`openmp-accumulator-race <openmp/accumulator-race>`,
    :doc:`performance-pure-procedure <performance/pure-procedure>`,
+   :doc:`readability-function-cognitive-complexity <readability/function-cognitive-complexity>`,
diff --git a/flang/tools/docs/flang-tidy/checks/readability/function-cognitive-complexity.rst b/flang/tools/docs/flang-tidy/checks/readability/function-cognitive-complexity.rst
new file mode 100644
index 0000000000000..e4264c6fa155b
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/readability/function-cognitive-complexity.rst
@@ -0,0 +1,45 @@
+.. title:: flang-tidy - readability-function-cognitive-complexity
+
+readability-function-cognitive-complexity
+=========================================
+
+Verifies that the cognitive complexity of a function or subroutine does not exceed a threshold (default: 25). Cognitive complexity is a measure of how difficult code is to understand based on nesting, control flow, and logical operations. This check helps maintain readable and maintainable code by identifying overly complex procedures.
+
+.. code-block:: fortran
+
+    ! This will trigger a warning due to high cognitive complexity
+    subroutine complex_procedure(arr, n, threshold)
+      implicit none
+      integer, intent(in) :: n, threshold
+      real, intent(inout) :: arr(n)
+      integer :: i, j, count
+
+      count = 0
+      do i = 1, n
+        if (arr(i) > threshold) then
+          do j = i, n
+            if (arr(j) > 0) then
+              if (arr(j) > arr(i)) then
+                arr(j) = arr(j) - arr(i)
+                if (arr(j) < threshold .and. arr(j) > 0) then
+                  count = count + 1
+                  if (count > n/2) then
+                    do while (arr(j) > 0.1)
+                      arr(j) = arr(j) * 0.9
+                    end do
+                  else if (count > n/3) then
+                    arr(j) = arr(j) * 0.8
+                  end if
+                end if
+              end if
+            end if
+          end do
+        else if (arr(i) < -threshold) then
+          do j = 1, i
+            if (arr(j) < 0 .and. arr(j) < arr(i)) then
+              arr(j) = arr(j) - arr(i)
+            end if
+          end do
+        end if
+      end do
+    end subroutine
diff --git a/flang/tools/flang-tidy/readability/CMakeLists.txt b/flang/tools/flang-tidy/readability/CMakeLists.txt
index 6f8e5df4127f2..4f6603802d503 100644
--- a/flang/tools/flang-tidy/readability/CMakeLists.txt
+++ b/flang/tools/flang-tidy/readability/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_flang_library(flangTidyReadabilityModule STATIC
+  FunctionCognitiveComplexityCheck.cpp
   ReadabilityTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/flang-tidy/readability/FunctionCognitiveComplexityCheck.cpp b/flang/tools/flang-tidy/readability/FunctionCognitiveComplexityCheck.cpp
new file mode 100644
index 0000000000000..d6dedc0c9e56a
--- /dev/null
+++ b/flang/tools/flang-tidy/readability/FunctionCognitiveComplexityCheck.cpp
@@ -0,0 +1,144 @@
+//===--- FunctionCognitiveComplexityCheck.cpp - flang-tidy ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FunctionCognitiveComplexityCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::readability {
+
+using namespace parser::literals;
+
+FunctionCognitiveComplexityCheck::FunctionCognitiveComplexityCheck(
+    llvm::StringRef name, FlangTidyContext *context)
+    : FlangTidyCheck(name, context),
+      Threshold(Options.get("Threshold", DefaultThreshold)) {}
+
+void FunctionCognitiveComplexityCheck::storeOptions(
+    FlangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "Threshold", Threshold);
+}
+
+void FunctionCognitiveComplexityCheck::Enter(
+    const parser::SubroutineSubprogram &program) {
+  currentNestingLevel_ = -1; // start at -1 to account for the first block
+  maxNestingLevel_ = 0;
+  inProcedure_ = true;
+  cognitiveComplexity_ = 0;
+
+  currentProcLoc_ =
+      std::get<parser::Statement<parser::SubroutineStmt>>(program.t).source;
+}
+
+void FunctionCognitiveComplexityCheck::Leave(
+    const parser::SubroutineSubprogram &) {
+  CheckCognitiveComplexity();
+  inProcedure_ = false;
+}
+
+void FunctionCognitiveComplexityCheck::Enter(
+    const parser::FunctionSubprogram &stmt) {
+  currentNestingLevel_ = -1;
+  maxNestingLevel_ = 0;
+  inProcedure_ = true;
+  cognitiveComplexity_ = 0;
+
+  currentProcLoc_ =
+      std::get<parser::Statement<parser::FunctionStmt>>(stmt.t).source;
+}
+
+void FunctionCognitiveComplexityCheck::Leave(
+    const parser::FunctionSubprogram &) {
+  CheckCognitiveComplexity();
+  inProcedure_ = false;
+}
+
+void FunctionCognitiveComplexityCheck::Enter(const parser::DoConstruct &) {
+  cognitiveComplexity_ += (1 + currentNestingLevel_);
+}
+void FunctionCognitiveComplexityCheck::Enter(
+    const parser::IfConstruct &construct) {
+  cognitiveComplexity_ += (1 + currentNestingLevel_);
+
+  // for each if-then-else, increment the cognitive complexity
+  const auto &elseIf =
+      std::get<std::list<parser::IfConstruct::ElseIfBlock>>(construct.t);
+
+  cognitiveComplexity_ +=
+      (elseIf.size() * currentNestingLevel_ + elseIf.size());
+
+  // if there is an else, increment the cognitive complexity by 1 and the
+  // nesting level
+  if (std::get<std::optional<parser::IfConstruct::ElseBlock>>(construct.t)) {
+    cognitiveComplexity_ += (1 + currentNestingLevel_);
+  }
+}
+
+void FunctionCognitiveComplexityCheck::Enter(const parser::GotoStmt &) {
+  ++cognitiveComplexity_;
+}
+
+void FunctionCognitiveComplexityCheck::Enter(const parser::ComputedGotoStmt &) {
+  ++cognitiveComplexity_;
+}
+
+void FunctionCognitiveComplexityCheck::Enter(const parser::AssignedGotoStmt &) {
+  ++cognitiveComplexity_;
+}
+
+void FunctionCognitiveComplexityCheck::Enter(const parser::CaseConstruct &) {
+  cognitiveComplexity_ += (1 + currentNestingLevel_);
+}
+
+void FunctionCognitiveComplexityCheck::Enter(
+    const parser::SelectRankConstruct &) {
+  cognitiveComplexity_ += (1 + currentNestingLevel_);
+}
+
+void FunctionCognitiveComplexityCheck::Enter(
+    const parser::SelectTypeConstruct &) {
+  cognitiveComplexity_ += (1 + currentNestingLevel_);
+}
+
+void FunctionCognitiveComplexityCheck::Enter(const parser::Expr::AND &) {
+  cognitiveComplexity_ += (1 + currentNestingLevel_);
+}
+
+void FunctionCognitiveComplexityCheck::Enter(const parser::Expr::OR &) {
+  cognitiveComplexity_ += (1 + currentNestingLevel_);
+}
+
+void FunctionCognitiveComplexityCheck::Enter(
+    const parser::AssociateConstruct &) {
+  cognitiveComplexity_ += (1 + currentNestingLevel_);
+}
+
+void FunctionCognitiveComplexityCheck::Enter(const parser::Block &) {
+  if (inProcedure_) {
+    currentNestingLevel_++;
+    UpdateMaxNestingLevel();
+  }
+}
+void FunctionCognitiveComplexityCheck::Leave(const parser::Block &) {
+  if (inProcedure_ && currentNestingLevel_ > 0) {
+    currentNestingLevel_--;
+  }
+}
+
+void FunctionCognitiveComplexityCheck::UpdateMaxNestingLevel() {
+  maxNestingLevel_ = std::max(maxNestingLevel_, currentNestingLevel_);
+}
+
+void FunctionCognitiveComplexityCheck::CheckCognitiveComplexity() {
+  if (Threshold && cognitiveComplexity_ > Threshold) {
+    Say(currentProcLoc_,
+        "%s has a cognitive complexity of %d, which exceeds the threshold of %d"_warn_en_US,
+        currentProcLoc_, cognitiveComplexity_, Threshold.value());
+  }
+}
+
+} // namespace Fortran::tidy::readability
diff --git a/flang/tools/flang-tidy/readability/FunctionCognitiveComplexityCheck.h b/flang/tools/flang-tidy/readability/FunctionCognitiveComplexityCheck.h
new file mode 100644
index 0000000000000..4e8ecc04b7ac4
--- /dev/null
+++ b/flang/tools/flang-tidy/readability/FunctionCognitiveComplexityCheck.h
@@ -0,0 +1,73 @@
+//===--- FunctionCognitiveComplexityCheck.h - flang-tidy --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_READABILITY_FUNCTIONCOGNITIVECOMPLEXITYCHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_READABILITY_FUNCTIONCOGNITIVECOMPLEXITYCHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::readability {
+
+/// This check verifies that the cognitive complexity of a function is
+/// not too high.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/function-cognitive-complexity.html
+class FunctionCognitiveComplexityCheck : public virtual FlangTidyCheck {
+public:
+  explicit FunctionCognitiveComplexityCheck(llvm::StringRef name,
+                                            FlangTidyContext *context);
+  virtual ~FunctionCognitiveComplexityCheck() = default;
+
+  void storeOptions(FlangTidyOptions::OptionMap &Opts) override;
+
+  void Enter(const parser::SubroutineSubprogram &) override;
+  void Leave(const parser::SubroutineSubprogram &) override;
+  void Enter(const parser::FunctionSubprogram &) override;
+  void Leave(const parser::FunctionSubprogram &) override;
+
+  // DoConstruct, IfConstruct & CaseConstruct
+  void Enter(const parser::DoConstruct &) override;
+  void Enter(const parser::IfConstruct &) override;
+  void Enter(const parser::CaseConstruct &) override;
+
+  // Boolean operators &&, ||
+  void Enter(const parser::Expr::AND &) override;
+  void Enter(const parser::Expr::OR &) override;
+
+  // Select Rank? Select Type? Associate?
+  void Enter(const parser::SelectRankConstruct &) override;
+  void Enter(const parser::SelectTypeConstruct &) override;
+  void Enter(const parser::AssociateConstruct &) override;
+
+  // Goto statements
+  void Enter(const parser::GotoStmt &) override;
+  void Enter(const parser::ComputedGotoStmt &) override;
+  void Enter(const parser::AssignedGotoStmt &) override;
+
+  void Enter(const parser::Block &) override;
+  void Leave(const parser::Block &) override;
+
+private:
+  const std::optional<unsigned> Threshold;
+
+  constexpr static unsigned DefaultThreshold = 25U;
+
+  int maxNestingLevel_;
+  int currentNestingLevel_;
+  int cognitiveComplexity_;
+  parser::CharBlock currentProcLoc_;
+  bool inProcedure_;
+  void UpdateMaxNestingLevel();
+  void CheckCognitiveComplexity();
+};
+
+} // namespace Fortran::tidy::readability
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_READABILITY_FUNCTIONCOGNITIVECOMPLEXITYCHECK_H
diff --git a/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp b/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp
index 1f217dd5eea4f..5e2ffc2c9b037 100644
--- a/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp
+++ b/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp
@@ -8,13 +8,17 @@
 
 #include "../FlangTidyModule.h"
 #include "../FlangTidyModuleRegistry.h"
+#include "FunctionCognitiveComplexityCheck.h"
 
 namespace Fortran::tidy {
 namespace readability {
 
 class ReadabilityTidyModule : public FlangTidyModule {
 public:
-  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {}
+  void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {
+    CheckFactories.registerCheck<FunctionCognitiveComplexityCheck>(
+        "readability-function-cognitive-complexity");
+  }
 };
 
 } // namespace readability
diff --git a/flang/tools/test/flang-tidy/function-complexity01.f90 b/flang/tools/test/flang-tidy/function-complexity01.f90
new file mode 100644
index 0000000000000..7c68b8510ae09
--- /dev/null
+++ b/flang/tools/test/flang-tidy/function-complexity01.f90
@@ -0,0 +1,128 @@
+! RUN: %check_flang_tidy %s readability-function-cognitive-complexity %t
+MODULE function_size_test
+  IMPLICIT NONE
+
+CONTAINS
+  SUBROUTINE simple_function(n, result)
+    INTEGER, INTENT(IN) :: n
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i, temp
+
+    temp = 0
+    DO i = 1, n
+      IF (i > 5) THEN
+        temp = temp + i
+      END IF
+    END DO
+
+    result = temp
+  END SUBROUTINE simple_function
+
+  SUBROUTINE complex_function(arr, n, threshold, result)
+    INTEGER, INTENT(IN) :: arr(:,:), n, threshold
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i, j, temp, count
+
+    temp = 0
+    count = 0
+
+    DO i = 1, n                                 ! Level 1
+      DO j = 1, n                               ! Level 2
+        IF (arr(i,j) > threshold) THEN          ! Level 3
+          IF (arr(i,j) < threshold * 2) THEN    ! Level 4
+            DO WHILE (temp < arr(i,j))          ! Level 5
+              IF (temp > 0) THEN                ! Level 6
+                temp = temp + 1
+                count = count + 1
+              END IF
+            END DO
+          END IF
+        END IF
+      END DO
+    END DO
+
+    result = count
+  END SUBROUTINE complex_function
+
+  ! CHECK-MESSAGES: :[[@LINE+1]]:3: warning: subroutine borderline_function(arr, n, result) has a cognitive complexity of 39, which exceeds the threshold of 25
+  SUBROUTINE borderline_function(arr, n, result)
+    INTEGER, INTENT(IN) :: arr(:), n
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i, sum, count
+    LOGICAL :: found
+
+    sum = 0
+    count = 0
+    found = .FALSE.
+
+    DO i = 1, n                          ! Level 1
+      IF (arr(i) > 0) THEN               ! Level 2
+        DO WHILE (.NOT. found)           ! Level 3
+          IF (sum < 100) THEN            ! Level 4
+            IF (arr(i) > 10) THEN        ! Level 5
+              sum = sum + arr(i)
+              count = count + 1
+            END IF
+          END IF
+
+          found = (sum >= 100)
+        END DO
+      END IF
+    END DO
+
+    result = count
+  END SUBROUTINE borderline_function
+
+  ! CHECK-MESSAGES: :[[@LINE+1]]:3: warning: subroutine extremely_nested_function(x, result) has a cognitive complexity of 61, which exceeds the threshold of 25
+  SUBROUTINE extremely_nested_function(x, result)
+    INTEGER, INTENT(IN) :: x
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i, j, k
+
+    result = 0
+
+    outer: DO i = 1, 10                        ! Level 1
+      middle: DO j = 1, 10                     ! Level 2
+        inner: DO k = 1, 10                    ! Level 3
+          IF (i*j*k > x) THEN                  ! Level 4
+            IF (i*j*k < x*2) THEN              ! Level 5
+              BLOCK                            ! Level 6
+                IF (MOD(i*j*k, 2) == 0) THEN   ! Level 7
+                  result = result + 1
+                END IF
+              END BLOCK
+            END IF
+          END IF
+        END DO inner
+      END DO middle
+    END DO outer
+
+  END SUBROUTINE extremely_nested_function
+
+  ! CHECK-MESSAGES: :[[@LINE+1]]:3: warning: subroutine case_function(x, y, result) has a cognitive complexity of 71, which exceeds the threshold of 25
+  SUBROUTINE case_function(x, y, result)
+    INTEGER, INTENT(IN) :: x, y
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i
+
+    result = 0
+
+    DO i = 1, 10                           ! Level 1
+      SELECT CASE (i)                      ! Level 2
+        CASE (1:3)
+          IF (x > 0) THEN                  ! Level 3
+            SELECT CASE (y)                ! Level 4
+              CASE (1:5)
+                result = result + 1
+              CASE DEFAULT
+                result = result + 2
+            END SELECT
+          END IF
+        CASE DEFAULT
+          result = result + 3
+      END SELECT
+    END DO
+
+  END SUBROUTINE case_function
+
+END MODULE function_size_test

>From 0983a499f512aa0d29aae1e41b958745de5b472f Mon Sep 17 00:00:00 2001
From: Originns <68753984+Originns at users.noreply.github.com>
Date: Tue, 20 May 2025 13:58:23 +0200
Subject: [PATCH 31/31] [flang-tidy][readability] add function size check

---
 flang/tools/docs/flang-tidy/checks/list.rst   |   1 +
 .../checks/readability/function-size.rst      |  40 ++++
 .../flang-tidy/readability/CMakeLists.txt     |   1 +
 .../readability/FunctionSizeCheck.cpp         | 188 ++++++++++++++++++
 .../readability/FunctionSizeCheck.h           |  72 +++++++
 .../readability/ReadabilityTidyModule.cpp     |   3 +
 .../tools/test/flang-tidy/function-size01.f90 | 127 ++++++++++++
 7 files changed, 432 insertions(+)
 create mode 100644 flang/tools/docs/flang-tidy/checks/readability/function-size.rst
 create mode 100644 flang/tools/flang-tidy/readability/FunctionSizeCheck.cpp
 create mode 100644 flang/tools/flang-tidy/readability/FunctionSizeCheck.h
 create mode 100644 flang/tools/test/flang-tidy/function-size01.f90

diff --git a/flang/tools/docs/flang-tidy/checks/list.rst b/flang/tools/docs/flang-tidy/checks/list.rst
index fb1c1aac34f26..70008cabbeeda 100644
--- a/flang/tools/docs/flang-tidy/checks/list.rst
+++ b/flang/tools/docs/flang-tidy/checks/list.rst
@@ -36,3 +36,4 @@ Flang-Tidy Checks
    :doc:`openmp-accumulator-race <openmp/accumulator-race>`,
    :doc:`performance-pure-procedure <performance/pure-procedure>`,
    :doc:`readability-function-cognitive-complexity <readability/function-cognitive-complexity>`,
+   :doc:`readability-function-size <readability/function-size>`,
diff --git a/flang/tools/docs/flang-tidy/checks/readability/function-size.rst b/flang/tools/docs/flang-tidy/checks/readability/function-size.rst
new file mode 100644
index 0000000000000..77af584beb500
--- /dev/null
+++ b/flang/tools/docs/flang-tidy/checks/readability/function-size.rst
@@ -0,0 +1,40 @@
+.. title:: flang-tidy - readability-function-size
+
+readability-function-size
+=========================
+
+Enforces size constraints on functions and subroutines based on line count, nesting level, and number of dummy arguments. Default thresholds are 5 for nesting level and 5 for dummy arguments (line count threshold is configurable). This check helps maintain manageable and readable code by preventing procedures from growing too large or complex.
+
+.. code-block:: fortran
+
+    ! This will trigger warnings for excessive arguments and nesting
+    subroutine process_data(data1, data2, data3, data4, data5, data6, options)
+      implicit none
+      real, intent(in) :: data1(:), data2(:), data3(:)
+      real, intent(in) :: data4(:), data5(:), data6(:)
+      integer, intent(in) :: options
+      integer :: i, j, k, m, n
+
+      do i = 1, size(data1)
+        if (data1(i) > 0) then
+          do j = 1, size(data2)
+            if (data2(j) > data1(i)) then
+              do k = 1, size(data3)
+                if (data3(k) > data2(j)) then
+                  do m = 1, size(data4)
+                    if (data4(m) > data3(k)) then
+                      do n = 1, size(data5)
+                        if (data5(n) > data4(m)) then
+                          ! This is at nesting level 10, which exceeds the threshold
+                          print *, "Found match"
+                        end if
+                      end do
+                    end if
+                  end do
+                end if
+              end do
+            end if
+          end do
+        end if
+      end do
+    end subroutine
diff --git a/flang/tools/flang-tidy/readability/CMakeLists.txt b/flang/tools/flang-tidy/readability/CMakeLists.txt
index 4f6603802d503..3efce77d86f78 100644
--- a/flang/tools/flang-tidy/readability/CMakeLists.txt
+++ b/flang/tools/flang-tidy/readability/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_flang_library(flangTidyReadabilityModule STATIC
   FunctionCognitiveComplexityCheck.cpp
+  FunctionSizeCheck.cpp
   ReadabilityTidyModule.cpp
 
   LINK_LIBS
diff --git a/flang/tools/flang-tidy/readability/FunctionSizeCheck.cpp b/flang/tools/flang-tidy/readability/FunctionSizeCheck.cpp
new file mode 100644
index 0000000000000..94d0869e1e27e
--- /dev/null
+++ b/flang/tools/flang-tidy/readability/FunctionSizeCheck.cpp
@@ -0,0 +1,188 @@
+//===--- FunctionSizeCheck.cpp - flang-tidy -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FunctionSizeCheck.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::readability {
+
+using namespace parser::literals;
+
+FunctionSizeCheck::FunctionSizeCheck(llvm::StringRef name,
+                                     FlangTidyContext *context)
+    : FlangTidyCheck(name, context),
+      LineThreshold(Options.get("LineThreshold", DefaultLineThreshold)),
+      ParameterThreshold(
+          Options.get("ParameterThreshold", DefaultParameterThreshold)),
+      NestingThreshold(
+          Options.get("NestingThreshold", DefaultNestingThreshold)),
+      inProcedure_(false), currentNestingLevel_(0), maxNestingLevel_(0) {}
+
+void FunctionSizeCheck::storeOptions(FlangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "LineThreshold", LineThreshold);
+  Options.store(Opts, "ParameterThreshold", ParameterThreshold);
+  Options.store(Opts, "NestingThreshold", NestingThreshold);
+}
+
+void FunctionSizeCheck::Enter(const parser::SubroutineSubprogram &program) {
+  currentNestingLevel_ = -1; // start at -1 to account for the subroutine itself
+  maxNestingLevel_ = 0;
+  inProcedure_ = true;
+
+  currentProcLoc_ =
+      std::get<parser::Statement<parser::SubroutineStmt>>(program.t).source;
+
+  if (ParameterThreshold) {
+    const auto &subroutineStmt =
+        std::get<parser::Statement<parser::SubroutineStmt>>(program.t)
+            .statement;
+    const auto &dummyArgs =
+        std::get<std::list<parser::DummyArg>>(subroutineStmt.t);
+    if ((int)dummyArgs.size() > ParameterThreshold) {
+      Say(currentProcLoc_,
+          "%s has %d dummy arguments, which exceeds the threshold of %d"_warn_en_US,
+          currentProcLoc_, dummyArgs.size(), ParameterThreshold.value());
+    }
+  }
+
+  if (LineThreshold) {
+    // get the end of the subroutine
+    const auto &endLoc =
+        std::get<parser::Statement<parser::EndSubroutineStmt>>(program.t)
+            .source;
+
+    const auto &cookedSources =
+        context()->getSemanticsContext().allCookedSources();
+
+    // get the source position of the end location
+    auto endProvenanceRange = cookedSources.GetProvenanceRange(endLoc);
+    auto startProvenanceRange =
+        cookedSources.GetProvenanceRange(currentProcLoc_);
+
+    if (!endProvenanceRange || !startProvenanceRange) {
+      return;
+    }
+
+    // get the source position of the end location
+    auto endSourcePosition = cookedSources.allSources().GetSourcePosition(
+        endProvenanceRange->start());
+    auto startSourcePosition = cookedSources.allSources().GetSourcePosition(
+        startProvenanceRange->start());
+
+    if (!endSourcePosition || !startSourcePosition) {
+      return;
+    }
+
+    auto lineCount = endSourcePosition->line - startSourcePosition->line;
+    if (lineCount > LineThreshold) {
+      Say(currentProcLoc_,
+          "%s has a line count of %d, which exceeds the threshold of %d"_warn_en_US,
+          currentProcLoc_, lineCount, LineThreshold.value());
+    }
+  }
+}
+
+void FunctionSizeCheck::Leave(const parser::SubroutineSubprogram &) {
+  CheckNestingThreshold();
+  inProcedure_ = false;
+}
+
+void FunctionSizeCheck::Enter(
+    const parser::FunctionSubprogram &functionSubprogram) {
+  currentNestingLevel_ = -1;
+  maxNestingLevel_ = 0;
+  inProcedure_ = true;
+
+  currentProcLoc_ =
+      std::get<parser::Statement<parser::FunctionStmt>>(functionSubprogram.t)
+          .source;
+
+  if (ParameterThreshold) {
+    const auto &functionStmt =
+        std::get<parser::Statement<parser::FunctionStmt>>(functionSubprogram.t)
+            .statement;
+    const auto &args = std::get<std::list<parser::Name>>(functionStmt.t);
+    if ((int)args.size() > ParameterThreshold) {
+      Say(currentProcLoc_,
+          "%s has %d dummy arguments, which exceeds the threshold of %d"_warn_en_US,
+          currentProcLoc_, args.size(), ParameterThreshold.value());
+    }
+  }
+
+  if (LineThreshold) {
+    // get the end of the subroutine
+    const auto &endLoc = std::get<parser::Statement<parser::EndFunctionStmt>>(
+                             functionSubprogram.t)
+                             .source;
+
+    const auto &cookedSources =
+        context()->getSemanticsContext().allCookedSources();
+
+    // get the source position of the end location
+    auto endProvenanceRange = cookedSources.GetProvenanceRange(endLoc);
+    auto startProvenanceRange =
+        cookedSources.GetProvenanceRange(currentProcLoc_);
+
+    if (!endProvenanceRange || !startProvenanceRange) {
+      return;
+    }
+
+    // get the source position of the end location
+    auto endSourcePosition = cookedSources.allSources().GetSourcePosition(
+        endProvenanceRange->start());
+    auto startSourcePosition = cookedSources.allSources().GetSourcePosition(
+        startProvenanceRange->start());
+
+    if (!endSourcePosition || !startSourcePosition) {
+      return;
+    }
+
+    auto lineCount = endSourcePosition->line - startSourcePosition->line;
+    if (lineCount > LineThreshold) {
+      Say(currentProcLoc_,
+          "%s has a line count of %d, which exceeds the threshold of %d"_warn_en_US,
+          currentProcLoc_, lineCount, LineThreshold.value());
+    }
+  }
+}
+
+void FunctionSizeCheck::Leave(const parser::FunctionSubprogram &) {
+  CheckNestingThreshold();
+  inProcedure_ = false;
+}
+
+void FunctionSizeCheck::Enter(const parser::Block &) {
+  if (inProcedure_) {
+    currentNestingLevel_++;
+    UpdateMaxNestingLevel();
+  }
+}
+
+void FunctionSizeCheck::Leave(const parser::Block &) {
+  if (inProcedure_ && currentNestingLevel_ > 0) {
+    currentNestingLevel_--;
+  }
+}
+
+void FunctionSizeCheck::UpdateMaxNestingLevel() {
+  maxNestingLevel_ = std::max(maxNestingLevel_, currentNestingLevel_);
+}
+
+void FunctionSizeCheck::CheckNestingThreshold() {
+  if (!NestingThreshold) {
+    return;
+  }
+
+  if (maxNestingLevel_ > NestingThreshold) {
+    Say(currentProcLoc_,
+        "%s has a nesting level of %d, which exceeds the threshold of %d"_warn_en_US,
+        currentProcLoc_, maxNestingLevel_, NestingThreshold.value());
+  }
+}
+
+} // namespace Fortran::tidy::readability
diff --git a/flang/tools/flang-tidy/readability/FunctionSizeCheck.h b/flang/tools/flang-tidy/readability/FunctionSizeCheck.h
new file mode 100644
index 0000000000000..b9221ed977ee8
--- /dev/null
+++ b/flang/tools/flang-tidy/readability/FunctionSizeCheck.h
@@ -0,0 +1,72 @@
+//===--- FunctionSizeCheck.h - flang-tidy -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_TOOLS_FLANG_TIDY_READABILITY_FUNCTIONSIZECHECK_H
+#define LLVM_FLANG_TOOLS_FLANG_TIDY_READABILITY_FUNCTIONSIZECHECK_H
+
+#include "../FlangTidyCheck.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+
+namespace Fortran::tidy::readability {
+
+/// This check verifies that the size of functions and subroutines is within a
+/// certain threshold.
+///
+/// For the user-facing documentation see:
+/// https://flang.llvm.org/@PLACEHOLDER@/function-size.html
+class FunctionSizeCheck : public virtual FlangTidyCheck {
+public:
+  explicit FunctionSizeCheck(llvm::StringRef name, FlangTidyContext *context);
+  virtual ~FunctionSizeCheck() = default;
+
+  void storeOptions(FlangTidyOptions::OptionMap &Opts) override;
+
+  void Enter(const parser::SubroutineSubprogram &) override;
+  void Leave(const parser::SubroutineSubprogram &) override;
+  void Enter(const parser::FunctionSubprogram &) override;
+  void Leave(const parser::FunctionSubprogram &) override;
+
+  void Enter(const parser::Block &) override;
+  void Leave(const parser::Block &) override;
+
+private:
+  void UpdateMaxNestingLevel();
+  void CheckNestingThreshold();
+
+  const std::optional<unsigned> LineThreshold;
+  const std::optional<unsigned> ParameterThreshold;
+  const std::optional<unsigned> NestingThreshold;
+
+  static constexpr std::optional<unsigned> DefaultLineThreshold = std::nullopt;
+  static constexpr std::optional<unsigned> DefaultParameterThreshold =
+      std::nullopt;
+  static constexpr std::optional<unsigned> DefaultNestingThreshold =
+      std::nullopt;
+
+  bool inProcedure_ = false;
+  int currentNestingLevel_ = 0;
+  int maxNestingLevel_ = 0;
+  parser::CharBlock currentProcLoc_;
+};
+
+/*
+ * TODO: StatementThreshold, BranchThreshold, VariableThreshold
+ LineThreshold(Options.get("LineThreshold", DefaultLineThreshold)),
+ (
+     Options.get("StatementThreshold", DefaultStatementThreshold)),
+ ParameterThreshold(
+     Options.get("ParameterThreshold", DefaultParameterThreshold)),
+ NestingThreshold(
+     Options.get("NestingThreshold", DefaultNestingThreshold)),
+ *
+ */
+
+} // namespace Fortran::tidy::readability
+
+#endif // LLVM_FLANG_TOOLS_FLANG_TIDY_READABILITY_FUNCTIONSIZECHECK_H
diff --git a/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp b/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp
index 5e2ffc2c9b037..0fa3303052f30 100644
--- a/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp
+++ b/flang/tools/flang-tidy/readability/ReadabilityTidyModule.cpp
@@ -9,6 +9,7 @@
 #include "../FlangTidyModule.h"
 #include "../FlangTidyModuleRegistry.h"
 #include "FunctionCognitiveComplexityCheck.h"
+#include "FunctionSizeCheck.h"
 
 namespace Fortran::tidy {
 namespace readability {
@@ -18,6 +19,8 @@ class ReadabilityTidyModule : public FlangTidyModule {
   void addCheckFactories(FlangTidyCheckFactories &CheckFactories) override {
     CheckFactories.registerCheck<FunctionCognitiveComplexityCheck>(
         "readability-function-cognitive-complexity");
+    CheckFactories.registerCheck<FunctionSizeCheck>(
+        "readability-function-size");
   }
 };
 
diff --git a/flang/tools/test/flang-tidy/function-size01.f90 b/flang/tools/test/flang-tidy/function-size01.f90
new file mode 100644
index 0000000000000..5945b21df011f
--- /dev/null
+++ b/flang/tools/test/flang-tidy/function-size01.f90
@@ -0,0 +1,127 @@
+! RUN: %check_flang_tidy %s readability-function-size %t
+MODULE function_size_test
+  IMPLICIT NONE
+
+CONTAINS
+  SUBROUTINE simple_function(n, result)
+    INTEGER, INTENT(IN) :: n
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i, temp
+
+    temp = 0
+    DO i = 1, n
+      IF (i > 5) THEN
+        temp = temp + i
+      END IF
+    END DO
+
+    result = temp
+  END SUBROUTINE simple_function
+
+  ! CHECK-MESSAGES: :[[@LINE+1]]:3: warning: subroutine complex_function(arr, n, threshold, result) has a nesting level of 6, which exceeds the threshold of 5
+  SUBROUTINE complex_function(arr, n, threshold, result)
+    INTEGER, INTENT(IN) :: arr(:,:), n, threshold
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i, j, temp, count
+
+    temp = 0
+    count = 0
+
+    DO i = 1, n                                 ! Level 1
+      DO j = 1, n                               ! Level 2
+        IF (arr(i,j) > threshold) THEN          ! Level 3
+          IF (arr(i,j) < threshold * 2) THEN    ! Level 4
+            DO WHILE (temp < arr(i,j))          ! Level 5
+              IF (temp > 0) THEN                ! Level 6
+                temp = temp + 1
+                count = count + 1
+              END IF
+            END DO
+          END IF
+        END IF
+      END DO
+    END DO
+
+    result = count
+  END SUBROUTINE complex_function
+
+  SUBROUTINE borderline_function(arr, n, result)
+    INTEGER, INTENT(IN) :: arr(:), n
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i, sum, count
+    LOGICAL :: found
+
+    sum = 0
+    count = 0
+    found = .FALSE.
+
+    DO i = 1, n                          ! Level 1
+      IF (arr(i) > 0) THEN               ! Level 2
+        DO WHILE (.NOT. found)           ! Level 3
+          IF (sum < 100) THEN            ! Level 4
+            IF (arr(i) > 10) THEN        ! Level 5
+              sum = sum + arr(i)
+              count = count + 1
+            END IF
+          END IF
+
+          found = (sum >= 100)
+        END DO
+      END IF
+    END DO
+
+    result = count
+  END SUBROUTINE borderline_function
+
+  ! CHECK-MESSAGES: :[[@LINE+1]]:3: warning: subroutine extremely_nested_function(x, result) has a nesting level of 7, which exceeds the threshold of 5
+  SUBROUTINE extremely_nested_function(x, result)
+    INTEGER, INTENT(IN) :: x
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i, j, k
+
+    result = 0
+
+    outer: DO i = 1, 10                        ! Level 1
+      middle: DO j = 1, 10                     ! Level 2
+        inner: DO k = 1, 10                    ! Level 3
+          IF (i*j*k > x) THEN                  ! Level 4
+            IF (i*j*k < x*2) THEN              ! Level 5
+              BLOCK                            ! Level 6
+                IF (MOD(i*j*k, 2) == 0) THEN   ! Level 7
+                  result = result + 1
+                END IF
+              END BLOCK
+            END IF
+          END IF
+        END DO inner
+      END DO middle
+    END DO outer
+
+  END SUBROUTINE extremely_nested_function
+
+  SUBROUTINE case_function(x, y, result)
+    INTEGER, INTENT(IN) :: x, y
+    INTEGER, INTENT(OUT) :: result
+    INTEGER :: i
+
+    result = 0
+
+    DO i = 1, 10                           ! Level 1
+      SELECT CASE (i)                      ! Level 2
+        CASE (1:3)
+          IF (x > 0) THEN                  ! Level 3
+            SELECT CASE (y)                ! Level 4
+              CASE (1:5)
+                result = result + 1
+              CASE DEFAULT
+                result = result + 2
+            END SELECT
+          END IF
+        CASE DEFAULT
+          result = result + 3
+      END SELECT
+    END DO
+
+  END SUBROUTINE case_function
+
+END MODULE function_size_test


From flang-commits at lists.llvm.org  Wed Jul 16 07:53:35 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 07:53:35 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148927@github.com>
Message-ID: <6877bcef.170a0220.db915.90f5@mx.google.com>


https://github.com/klausler updated https://github.com/llvm/llvm-project/pull/148927

>From 1a581cc67b0cf04c4d456c1c81f78256293aac6b Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 15 Jul 2025 11:28:30 -0700
Subject: [PATCH] [flang] Allow -fdefault-integer-8 with defined I/O

Defined I/O subroutines have UNIT= and IOSTAT= dummy arguments that
are required to have type INTEGER with its default kind.  When that
default kind is modified via -fdefault-integer-8, calls to defined
I/O subroutines from the runtime don't work.

Add a flag to the two data structures shared between the compiler and
the runtime support library to indicate that a defined I/O subroutine
was compiled under -fdefault-integer-8.  This has been done in a
compatible manner, so that existing binaries are compatible with the
new library and new binaries are compatible with the old library,
unless of course -fdefault-integer-8 is used.

Fixes https://github.com/llvm/llvm-project/issues/148638.
---
 .../include/flang-rt/runtime/non-tbp-dio.h    |  7 +-
 flang-rt/include/flang-rt/runtime/type-info.h | 16 ++--
 flang-rt/lib/runtime/derived.cpp              |  2 +-
 flang-rt/lib/runtime/descriptor-io.cpp        | 88 ++++++++++++++-----
 flang-rt/lib/runtime/non-tbp-dio.cpp          |  2 +-
 flang-rt/lib/runtime/type-info.cpp            |  2 +-
 .../flang/Semantics/runtime-type-info.h       |  7 +-
 flang/lib/Lower/IO.cpp                        |  8 +-
 flang/lib/Semantics/runtime-type-info.cpp     | 28 ++++--
 flang/module/__fortran_type_info.f90          |  2 +-
 flang/test/Lower/io-derived-type.f90          | 58 ++++++------
 flang/test/Lower/namelist.f90                 |  8 +-
 flang/test/Lower/volatile-openmp.f90          |  8 +-
 flang/test/Semantics/typeinfo01.f90           | 18 ++--
 flang/test/Semantics/typeinfo02.f90           |  4 +-
 flang/test/Semantics/typeinfo09.f90           |  2 +-
 flang/test/Semantics/typeinfo13.f90           |  2 +-
 17 files changed, 164 insertions(+), 98 deletions(-)

diff --git a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
index 99d4113b6c7a8..26849298ec959 100644
--- a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
+++ b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
@@ -34,11 +34,16 @@ namespace Fortran::runtime::io {
 
 RT_OFFLOAD_API_GROUP_BEGIN
 
+enum NonTbpDefinedIoFlags {
+  IsDtvArgPolymorphic = 1 << 0, // first dummy arg is CLASS(T)
+  DefinedIoInteger8 = 1 << 1, // -fdefault-integer-8 affected UNIT= & IOSTAT=
+};
+
 struct NonTbpDefinedIo {
   const typeInfo::DerivedType &derivedType;
   void (*subroutine)(); // null means no non-TBP defined I/O here
   common::DefinedIo definedIo;
-  bool isDtvArgPolymorphic; // first dummy arg is CLASS(T)
+  std::uint8_t flags;
 };
 
 struct NonTbpDefinedIoTable {
diff --git a/flang-rt/include/flang-rt/runtime/type-info.h b/flang-rt/include/flang-rt/runtime/type-info.h
index a8d39f4f8a1a3..93bca24a602b4 100644
--- a/flang-rt/include/flang-rt/runtime/type-info.h
+++ b/flang-rt/include/flang-rt/runtime/type-info.h
@@ -143,9 +143,9 @@ class SpecialBinding {
   // I/O procedures that are not type-bound.
   RT_API_ATTRS SpecialBinding(Which which, ProcedurePointer proc,
       std::uint8_t isArgDescSet, std::uint8_t isTypeBound,
-      std::uint8_t isArgContiguousSet)
+      std::uint8_t specialCaseFlag)
       : which_{which}, isArgDescriptorSet_{isArgDescSet},
-        isTypeBound_{isTypeBound}, isArgContiguousSet_{isArgContiguousSet},
+        isTypeBound_{isTypeBound}, specialCaseFlag_{specialCaseFlag},
         proc_{proc} {}
 
   static constexpr RT_API_ATTRS Which RankFinal(int rank) {
@@ -153,13 +153,11 @@ class SpecialBinding {
   }
 
   RT_API_ATTRS Which which() const { return which_; }
+  RT_API_ATTRS bool specialCaseFlag() const { return specialCaseFlag_; }
   RT_API_ATTRS bool IsArgDescriptor(int zeroBasedArg) const {
     return (isArgDescriptorSet_ >> zeroBasedArg) & 1;
   }
   RT_API_ATTRS bool IsTypeBound() const { return isTypeBound_ != 0; }
-  RT_API_ATTRS bool IsArgContiguous(int zeroBasedArg) const {
-    return (isArgContiguousSet_ >> zeroBasedArg) & 1;
-  }
   template <typename PROC>
   RT_API_ATTRS PROC GetProc(const Binding *bindings = nullptr) const {
     if (bindings && isTypeBound_ > 0) {
@@ -203,10 +201,10 @@ class SpecialBinding {
   // When a special binding is type-bound, this is its binding's index (plus 1,
   // so that 0 signifies that it's not type-bound).
   std::uint8_t isTypeBound_{0};
-  // True when a FINAL subroutine has a dummy argument that is an array that
-  // is CONTIGUOUS or neither assumed-rank nor assumed-shape.
-  std::uint8_t isArgContiguousSet_{0};
-
+  // For a FINAL subroutine, set when it has a dummy argument that is an array
+  // that is CONTIGUOUS or neither assumed-rank nor assumed-shape.
+  // For a defined I/O subroutine, set when UNIT= and IOSTAT= are INTEGER(8).
+  std::uint8_t specialCaseFlag_{0};
   ProcedurePointer proc_{nullptr};
 };
 
diff --git a/flang-rt/lib/runtime/derived.cpp b/flang-rt/lib/runtime/derived.cpp
index bb9a68abef2a7..4ed0baaa3d108 100644
--- a/flang-rt/lib/runtime/derived.cpp
+++ b/flang-rt/lib/runtime/derived.cpp
@@ -270,7 +270,7 @@ static RT_API_ATTRS void CallFinalSubroutine(const Descriptor &descriptor,
       StaticDescriptor<maxRank, true, 10> statDesc;
       Descriptor &copy{statDesc.descriptor()};
       const Descriptor *argDescriptor{&descriptor};
-      if (descriptor.rank() > 0 && special->IsArgContiguous(0) &&
+      if (descriptor.rank() > 0 && special->specialCaseFlag() &&
           !descriptor.IsContiguous()) {
         // The FINAL subroutine demands a contiguous array argument, but
         // this INTENT(OUT) or intrinsic assignment LHS isn't contiguous.
diff --git a/flang-rt/lib/runtime/descriptor-io.cpp b/flang-rt/lib/runtime/descriptor-io.cpp
index b208cb2c397b3..3868c8ddce19f 100644
--- a/flang-rt/lib/runtime/descriptor-io.cpp
+++ b/flang-rt/lib/runtime/descriptor-io.cpp
@@ -67,13 +67,29 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
           ioType, io.mutableModes().inNamelist ? "NAMELIST" : "LISTDIRECTED");
       ioTypeLen = runtime::strlen(ioType);
     }
+    // V_LIST= argument
     StaticDescriptor<1, true> vListStatDesc;
     Descriptor &vListDesc{vListStatDesc.descriptor()};
-    vListDesc.Establish(TypeCategory::Integer, sizeof(int), nullptr, 1);
-    vListDesc.set_base_addr(edit.vList);
-    vListDesc.GetDimension(0).SetBounds(1, edit.vListEntries);
-    vListDesc.GetDimension(0).SetByteStride(
-        static_cast<SubscriptValue>(sizeof(int)));
+    bool integer8{special.specialCaseFlag()};
+    std::int64_t vList64[edit.maxVListEntries];
+    if (integer8) {
+      // Convert v_list values to INTEGER(8)
+      for (int j{0}; j < edit.vListEntries; ++j) {
+        vList64[j] = edit.vList[j];
+      }
+      vListDesc.Establish(
+          TypeCategory::Integer, sizeof(std::int64_t), nullptr, 1);
+      vListDesc.set_base_addr(vList64);
+      vListDesc.GetDimension(0).SetBounds(1, edit.vListEntries);
+      vListDesc.GetDimension(0).SetByteStride(
+          static_cast<SubscriptValue>(sizeof(std::int64_t)));
+    } else {
+      vListDesc.Establish(TypeCategory::Integer, sizeof(int), nullptr, 1);
+      vListDesc.set_base_addr(edit.vList);
+      vListDesc.GetDimension(0).SetBounds(1, edit.vListEntries);
+      vListDesc.GetDimension(0).SetByteStride(
+          static_cast<SubscriptValue>(sizeof(int)));
+    }
     ExternalFileUnit *actualExternal{io.GetExternalFileUnit()};
     ExternalFileUnit *external{actualExternal};
     if (!external) {
@@ -84,8 +100,8 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
     ChildIo &child{external->PushChildIo(io)};
     // Child formatted I/O is nonadvancing by definition (F'2018 12.6.2.4).
     auto restorer{common::ScopedSet(io.mutableModes().nonAdvancing, true)};
-    int unit{external->unitNumber()};
-    int ioStat{IostatOk};
+    std::int32_t unit{external->unitNumber()};
+    std::int32_t ioStat{IostatOk};
     char ioMsg[100];
     Fortran::common::optional<std::int64_t> startPos;
     if (edit.descriptor == DataEdit::DefinedDerivedType &&
@@ -98,23 +114,45 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
         derived.binding().OffsetElement<const typeInfo::Binding>()};
     if (special.IsArgDescriptor(0)) {
       // "dtv" argument is "class(t)", pass a descriptor
-      auto *p{special.GetProc<void (*)(const Descriptor &, int &, char *,
-          const Descriptor &, int &, char *, std::size_t, std::size_t)>(
-          bindings)};
       StaticDescriptor<1, true, 10 /*?*/> elementStatDesc;
       Descriptor &elementDesc{elementStatDesc.descriptor()};
       elementDesc.Establish(
           derived, nullptr, 0, nullptr, CFI_attribute_pointer);
       elementDesc.set_base_addr(descriptor.Element<char>(subscripts));
-      p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen,
-          sizeof ioMsg);
+      if (integer8) { // 64-bit UNIT=/IOSTAT=
+        std::int64_t unit64{unit};
+        std::int64_t ioStat64{ioStat};
+        auto *p{special.GetProc<void (*)(const Descriptor &, std::int64_t &,
+            char *, const Descriptor &, std::int64_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(elementDesc, unit64, ioType, vListDesc, ioStat64, ioMsg, ioTypeLen,
+            sizeof ioMsg);
+        ioStat = ioStat64;
+      } else { // 32-bit UNIT=/IOSTAT=
+        auto *p{special.GetProc<void (*)(const Descriptor &, std::int32_t &,
+            char *, const Descriptor &, std::int32_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen,
+            sizeof ioMsg);
+      }
     } else {
       // "dtv" argument is "type(t)", pass a raw pointer
-      auto *p{special.GetProc<void (*)(const void *, int &, char *,
-          const Descriptor &, int &, char *, std::size_t, std::size_t)>(
-          bindings)};
-      p(descriptor.Element<char>(subscripts), unit, ioType, vListDesc, ioStat,
-          ioMsg, ioTypeLen, sizeof ioMsg);
+      if (integer8) { // 64-bit UNIT= and IOSTAT=
+        std::int64_t unit64{unit};
+        std::int64_t ioStat64{ioStat};
+        auto *p{special.GetProc<void (*)(const void *, std::int64_t &, char *,
+            const Descriptor &, std::int64_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(descriptor.Element<char>(subscripts), unit64, ioType, vListDesc,
+            ioStat64, ioMsg, ioTypeLen, sizeof ioMsg);
+        ioStat = ioStat64;
+      } else { // 32-bit UNIT= and IOSTAT=
+        auto *p{special.GetProc<void (*)(const void *, std::int32_t &, char *,
+            const Descriptor &, std::int32_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(descriptor.Element<char>(subscripts), unit, ioType, vListDesc, ioStat,
+            ioMsg, ioTypeLen, sizeof ioMsg);
+      }
     }
     handler.Forward(ioStat, ioMsg, sizeof ioMsg);
     external->PopChildIo(child);
@@ -458,11 +496,16 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
                       ? common::DefinedIo::ReadUnformatted
                       : common::DefinedIo::WriteUnformatted)}) {
             if (definedIo->subroutine) {
+              std::uint8_t isArgDescriptorSet{0};
+              if (definedIo->flags & IsDtvArgPolymorphic) {
+                isArgDescriptorSet = 1;
+              }
               typeInfo::SpecialBinding special{DIR == Direction::Input
                       ? typeInfo::SpecialBinding::Which::ReadUnformatted
                       : typeInfo::SpecialBinding::Which::WriteUnformatted,
-                  definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                  false};
+                  definedIo->subroutine, isArgDescriptorSet,
+                  /*IsTypeBound=*/false,
+                  /*specialCaseFlag=*/!!(definedIo->flags & DefinedIoInteger8)};
               if (DefinedUnformattedIo(io_, instance_, *type, special)) {
                 anyIoTookPlace_ = true;
                 return StatOk;
@@ -719,8 +762,11 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
             nonTbpSpecial_.emplace(DIR == Direction::Input
                     ? typeInfo::SpecialBinding::Which::ReadFormatted
                     : typeInfo::SpecialBinding::Which::WriteFormatted,
-                definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                false);
+                definedIo->subroutine,
+                /*isArgDescriptorSet=*/
+                (definedIo->flags & IsDtvArgPolymorphic) ? 1 : 0,
+                /*isTypeBound=*/false,
+                /*specialCaseFlag=*/!!(definedIo->flags & DefinedIoInteger8));
             special_ = &*nonTbpSpecial_;
           }
         }
diff --git a/flang-rt/lib/runtime/non-tbp-dio.cpp b/flang-rt/lib/runtime/non-tbp-dio.cpp
index 72101b06e0c6e..d516526033c27 100644
--- a/flang-rt/lib/runtime/non-tbp-dio.cpp
+++ b/flang-rt/lib/runtime/non-tbp-dio.cpp
@@ -17,7 +17,7 @@ const NonTbpDefinedIo *NonTbpDefinedIoTable::Find(
   for (const auto *p{item}; j-- > 0; ++p) {
     if (&p->derivedType == &type && p->definedIo == definedIo) {
       return p;
-    } else if (p->isDtvArgPolymorphic) {
+    } else if (p->flags & IsDtvArgPolymorphic) {
       for (const typeInfo::DerivedType *t{type.GetParentType()}; t;
            t = t->GetParentType()) {
         if (&p->derivedType == t && p->definedIo == definedIo) {
diff --git a/flang-rt/lib/runtime/type-info.cpp b/flang-rt/lib/runtime/type-info.cpp
index 3e1d7c9c3c788..50123f4cf321c 100644
--- a/flang-rt/lib/runtime/type-info.cpp
+++ b/flang-rt/lib/runtime/type-info.cpp
@@ -330,7 +330,7 @@ FILE *SpecialBinding::Dump(FILE *f) const {
   }
   std::fprintf(f, "    isArgDescriptorSet: 0x%x\n", isArgDescriptorSet_);
   std::fprintf(f, "    isTypeBound: %d\n", isTypeBound_);
-  std::fprintf(f, "    isArgContiguousSet: 0x%x\n", isArgContiguousSet_);
+  std::fprintf(f, "    specialCaseFlag 0x%x\n", specialCaseFlag_);
   std::fprintf(f, "    proc: %p\n", reinterpret_cast<void *>(proc_));
   return f;
 }
diff --git a/flang/include/flang/Semantics/runtime-type-info.h b/flang/include/flang/Semantics/runtime-type-info.h
index 6c5a061d1c1a2..94e8293b14643 100644
--- a/flang/include/flang/Semantics/runtime-type-info.h
+++ b/flang/include/flang/Semantics/runtime-type-info.h
@@ -52,10 +52,15 @@ constexpr char procCompName[]{"proc"};
 
 SymbolVector CollectBindings(const Scope &dtScope);
 
+enum NonTbpDefinedIoFlags {
+  IsDtvArgPolymorphic = 1 << 0,
+  DefinedIoInteger8 = 1 << 1,
+};
+
 struct NonTbpDefinedIo {
   const Symbol *subroutine;
   common::DefinedIo definedIo;
-  bool isDtvArgPolymorphic;
+  std::uint8_t flags;
 };
 
 std::multimap<const Symbol *, NonTbpDefinedIo>
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 63a612d7ead61..69d72d9d63b68 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -269,10 +269,12 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   mlir::Type sizeTy =
       fir::runtime::getModel<std::size_t>()(builder.getContext());
   mlir::Type intTy = fir::runtime::getModel<int>()(builder.getContext());
+  mlir::Type byteTy =
+      fir::runtime::getModel<std::uint8_t>()(builder.getContext());
   mlir::Type boolTy = fir::runtime::getModel<bool>()(builder.getContext());
   mlir::Type listTy = fir::SequenceType::get(
       definedIoProcMap.size(),
-      mlir::TupleType::get(context, {refTy, refTy, intTy, boolTy}));
+      mlir::TupleType::get(context, {refTy, refTy, intTy, byteTy}));
   mlir::Type tableTy = mlir::TupleType::get(
       context, {sizeTy, fir::ReferenceType::get(listTy), boolTy});
 
@@ -339,9 +341,9 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       insert(builder.createIntegerConstant(
           loc, intTy, static_cast<int>(iface.second.definedIo)));
       // polymorphic flag is set if first defined IO dummy arg is CLASS(T)
+      // defaultInt8 flag is set if -fdefined-integer-8
       // [bool isDtvArgPolymorphic]
-      insert(builder.createIntegerConstant(loc, boolTy,
-                                           iface.second.isDtvArgPolymorphic));
+      insert(builder.createIntegerConstant(loc, byteTy, iface.second.flags));
     }
     if (tableIsLocal)
       builder.create<fir::StoreOp>(loc, list, listAddr);
diff --git a/flang/lib/Semantics/runtime-type-info.cpp b/flang/lib/Semantics/runtime-type-info.cpp
index 51ba21a9e5edf..5916a07df7744 100644
--- a/flang/lib/Semantics/runtime-type-info.cpp
+++ b/flang/lib/Semantics/runtime-type-info.cpp
@@ -1131,7 +1131,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
   if (auto proc{evaluate::characteristics::Procedure::Characterize(
           specific, context_.foldingContext())}) {
     std::uint8_t isArgDescriptorSet{0};
-    std::uint8_t isArgContiguousSet{0};
+    bool specialCaseFlag{0};
     int argThatMightBeDescriptor{0};
     MaybeExpr which;
     if (isAssignment) {
@@ -1197,7 +1197,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
                         TypeAndShape::Attr::AssumedShape) ||
                 dummyData.attrs.test(evaluate::characteristics::
                         DummyDataObject::Attr::Contiguous)) {
-              isArgContiguousSet |= 1;
+              specialCaseFlag = true;
             }
           }
         }
@@ -1216,7 +1216,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
         return;
       }
       if (ddo->type.type().IsPolymorphic()) {
-        isArgDescriptorSet |= 1;
+        argThatMightBeDescriptor = 1;
       }
       switch (io.value()) {
       case common::DefinedIo::ReadFormatted:
@@ -1232,6 +1232,9 @@ void RuntimeTableBuilder::DescribeSpecialProc(
         which = writeUnformattedEnum_;
         break;
       }
+      if (context_.defaultKinds().GetDefaultKind(TypeCategory::Integer) == 8) {
+        specialCaseFlag = true; // UNIT= & IOSTAT= INTEGER(8)
+      }
     }
     if (argThatMightBeDescriptor != 0) {
       if (const auto *dummyData{
@@ -1262,8 +1265,8 @@ void RuntimeTableBuilder::DescribeSpecialProc(
     }
     CHECK(bindingIndex <= 255);
     AddValue(values, specialSchema_, "istypebound"s, IntExpr<1>(bindingIndex));
-    AddValue(values, specialSchema_, "isargcontiguousset"s,
-        IntExpr<1>(isArgContiguousSet));
+    AddValue(values, specialSchema_, "specialcaseflag"s,
+        IntExpr<1>(specialCaseFlag));
     AddValue(values, specialSchema_, procCompName,
         SomeExpr{evaluate::ProcedureDesignator{specific}});
     // index might already be present in the case of an override
@@ -1383,19 +1386,26 @@ CollectNonTbpDefinedIoGenericInterfaces(
               } else {
                 // Local scope's specific overrides host's for this type
                 bool updated{false};
+                std::uint8_t flags{0};
+                if (declType->IsPolymorphic()) {
+                  flags |= IsDtvArgPolymorphic;
+                }
+                if (scope.context().GetDefaultKind(TypeCategory::Integer) ==
+                    8) {
+                  flags |= DefinedIoInteger8;
+                }
                 for (auto [iter, end]{result.equal_range(dtDesc)}; iter != end;
                      ++iter) {
                   NonTbpDefinedIo &nonTbp{iter->second};
                   if (nonTbp.definedIo == which) {
                     nonTbp.subroutine = &*specific;
-                    nonTbp.isDtvArgPolymorphic = declType->IsPolymorphic();
+                    nonTbp.flags = flags;
                     updated = true;
                   }
                 }
                 if (!updated) {
-                  result.emplace(dtDesc,
-                      NonTbpDefinedIo{
-                          &*specific, which, declType->IsPolymorphic()});
+                  result.emplace(
+                      dtDesc, NonTbpDefinedIo{&*specific, which, flags});
                 }
               }
             }
diff --git a/flang/module/__fortran_type_info.f90 b/flang/module/__fortran_type_info.f90
index 8dd27d6e4c01b..6af2a5a5e30ff 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang/module/__fortran_type_info.f90
@@ -118,7 +118,7 @@
     integer(1) :: which ! SpecialBinding::Which
     integer(1) :: isArgDescriptorSet
     integer(1) :: isTypeBound ! binding index + 1, if any
-    integer(1) :: isArgContiguousSet
+    integer(1) :: specialCaseFlag
     integer(1) :: __padding0(4)
     type(__builtin_c_funptr) :: proc
   end type
diff --git a/flang/test/Lower/io-derived-type.f90 b/flang/test/Lower/io-derived-type.f90
index 7d2fef3faa2b7..7c289ce261678 100644
--- a/flang/test/Lower/io-derived-type.f90
+++ b/flang/test/Lower/io-derived-type.f90
@@ -37,16 +37,16 @@ subroutine test1
     import, all
     ! CHECK:   %[[V_16:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:   %[[V_17:[0-9]+]] = fir.convert %[[V_16]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:   %[[V_18:[0-9]+]] = fir.address_of(@_QQMmFtest1.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:   %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:   %[[V_18:[0-9]+]] = fir.address_of(@_QQMmFtest1.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:   %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:   %[[V_20:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_17]], %[[V_19]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
     print *, 'test1 outer, should call wft: ', t(1)
     block
       import, only: t
       ! CHECK:   %[[V_37:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
       ! CHECK:   %[[V_38:[0-9]+]] = fir.convert %[[V_37]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-      ! CHECK:   %[[V_39:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-      ! CHECK:   %[[V_40:[0-9]+]] = fir.convert %[[V_39]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+      ! CHECK:   %[[V_39:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+      ! CHECK:   %[[V_40:[0-9]+]] = fir.convert %[[V_39]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
       ! CHECK:   %[[V_41:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_38]], %[[V_40]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
       print *, 'test1 block, should not call wft: ', t(2)
     end block
@@ -56,8 +56,8 @@ subroutine test1
   subroutine test2
     ! CHECK:   %[[V_15:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:   %[[V_16:[0-9]+]] = fir.convert %[[V_15]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:   %[[V_17:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:   %[[V_18:[0-9]+]] = fir.convert %[[V_17]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:   %[[V_17:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:   %[[V_18:[0-9]+]] = fir.convert %[[V_17]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:   %[[V_19:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_16]], %[[V_18]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
 
     import, only: t
@@ -74,23 +74,23 @@ subroutine test3(p, x)
 
     ! CHECK:     %[[V_3:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:     %[[V_4:[0-9]+]] = fir.convert %[[V_3]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:     %[[V_5:[0-9]+]] = fir.alloca !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_6:[0-9]+]] = fir.undefined !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
+    ! CHECK:     %[[V_5:[0-9]+]] = fir.alloca !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_6:[0-9]+]] = fir.undefined !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
     ! CHECK:     %[[V_7:[0-9]+]] = fir.address_of(@_QMmE.dt.t)
     ! CHECK:     %[[V_8:[0-9]+]] = fir.convert %[[V_7]] : {{.*}} -> !fir.ref<none>
-    ! CHECK:     %[[V_9:[0-9]+]] = fir.insert_value %[[V_6]], %[[V_8]], [0 : index, 0 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
+    ! CHECK:     %[[V_9:[0-9]+]] = fir.insert_value %[[V_6]], %[[V_8]], [0 : index, 0 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
     ! CHECK:     %[[V_10:[0-9]+]] = fir.box_addr %arg0 : (!fir.boxproc<() -> ()>) -> !fir.ref<none>
-    ! CHECK:     %[[V_11:[0-9]+]] = fir.insert_value %[[V_9]], %[[V_10]], [0 : index, 1 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_12:[0-9]+]] = fir.insert_value %[[V_11]], %c2{{.*}}, [0 : index, 2 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, i32) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_13:[0-9]+]] = fir.insert_value %[[V_12]], %true, [0 : index, 3 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, i1) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     fir.store %[[V_13]] to %[[V_5]] : !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>
-    ! CHECK:     %[[V_14:[0-9]+]] = fir.alloca tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_15:[0-9]+]] = fir.undefined tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_16:[0-9]+]] = fir.insert_value %[[V_15]], %c1{{.*}}, [0 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, i64) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_17:[0-9]+]] = fir.insert_value %[[V_16]], %[[V_5]], [1 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_18:[0-9]+]] = fir.insert_value %[[V_17]], %true_0, [2 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, i1) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     fir.store %[[V_18]] to %[[V_14]] : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:     %[[V_19:[0-9]+]] = fir.convert %[[V_14]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:     %[[V_11:[0-9]+]] = fir.insert_value %[[V_9]], %[[V_10]], [0 : index, 1 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_12:[0-9]+]] = fir.insert_value %[[V_11]], %c2{{.*}}, [0 : index, 2 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, i32) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_13:[0-9]+]] = fir.insert_value %[[V_12]], %c1_i8, [0 : index, 3 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, i8) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     fir.store %[[V_13]] to %[[V_5]] : !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>
+    ! CHECK:     %[[V_14:[0-9]+]] = fir.alloca tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_15:[0-9]+]] = fir.undefined tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_16:[0-9]+]] = fir.insert_value %[[V_15]], %c1{{.*}}, [0 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, i64) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_17:[0-9]+]] = fir.insert_value %[[V_16]], %[[V_5]], [1 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_18:[0-9]+]] = fir.insert_value %[[V_17]], %true, [2 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, i1) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     fir.store %[[V_18]] to %[[V_14]] : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:     %[[V_19:[0-9]+]] = fir.convert %[[V_14]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:     %[[V_20:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_4]], %[[V_19]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
     print *, x
   end subroutine
@@ -112,8 +112,8 @@ program p
 
   ! CHECK:   %[[V_97:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
   ! CHECK:   %[[V_98:[0-9]+]] = fir.convert %[[V_97]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-  ! CHECK:   %[[V_99:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:   %[[V_100:[0-9]+]] = fir.convert %[[V_99]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:   %[[V_99:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:   %[[V_100:[0-9]+]] = fir.convert %[[V_99]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:   %[[V_101:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_98]], %[[V_100]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
   print *, 'main, should call wft: ', t(4)
 
@@ -122,14 +122,14 @@ program p
   ! CHECK:   %[[V_35:[0-9]+]] = fir.shape %c2{{.*}} : (index) -> !fir.shape<1>
   ! CHECK:   %[[V_36:[0-9]+]] = fir.embox %[[V_34]](%[[V_35]]) : (!fir.ref<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>
   ! CHECK:   %[[V_37:[0-9]+]] = fir.convert %[[V_36]] : (!fir.box<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>) -> !fir.box<none>
-  ! CHECK:   %[[V_38:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:   %[[V_39:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:   %[[V_38:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:   %[[V_39:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:   %[[V_40:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_37]], %[[V_39]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
   print *, y(2:3)
 end
 
-! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-! CHECK: fir.global linkonce @_QQdefault.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
+! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+! CHECK: fir.global linkonce @_QQdefault.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
diff --git a/flang/test/Lower/namelist.f90 b/flang/test/Lower/namelist.f90
index 94b0ef11cb102..770af46eea744 100644
--- a/flang/test/Lower/namelist.f90
+++ b/flang/test/Lower/namelist.f90
@@ -42,8 +42,8 @@ program p
   ! CHECK:     %[[V_42:[0-9]+]] = fir.insert_value %[[V_39]], %[[V_41]], [0 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<i8>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_43:[0-9]+]] = fir.insert_value %[[V_42]], %c2{{.*}}, [1 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, i64) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_44:[0-9]+]] = fir.insert_value %[[V_43]], %[[V_24]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
-  ! CHECK:     %[[V_45:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:     %[[V_46:[0-9]+]] = fir.convert %[[V_45]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:     %[[V_45:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:     %[[V_46:[0-9]+]] = fir.convert %[[V_45]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:     %[[V_47:[0-9]+]] = fir.insert_value %[[V_44]], %[[V_46]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     fir.store %[[V_47]] to %[[V_38]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
   ! CHECK:     %[[V_48:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
@@ -100,8 +100,8 @@ subroutine sss
   ! CHECK:     %[[V_20:[0-9]+]] = fir.insert_value %[[V_17]], %[[V_19]], [0 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<i8>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_21:[0-9]+]] = fir.insert_value %[[V_20]], %c1{{.*}}, [1 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, i64) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_22:[0-9]+]] = fir.insert_value %[[V_21]], %[[V_8]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
-  ! CHECK:     %[[V_23:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:     %[[V_24:[0-9]+]] = fir.convert %[[V_23]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:     %[[V_23:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:     %[[V_24:[0-9]+]] = fir.convert %[[V_23]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:     %[[V_25:[0-9]+]] = fir.insert_value %[[V_22]], %[[V_24]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     fir.store %[[V_25]] to %[[V_16]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
   ! CHECK:     %[[V_26:[0-9]+]] = fir.convert %[[V_16]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
diff --git a/flang/test/Lower/volatile-openmp.f90 b/flang/test/Lower/volatile-openmp.f90
index 2e05b652822b5..d1a844eddd106 100644
--- a/flang/test/Lower/volatile-openmp.f90
+++ b/flang/test/Lower/volatile-openmp.f90
@@ -23,11 +23,11 @@
 ! CHECK:           %[[VAL_11:.*]] = fir.address_of(@_QFEcontainer) : !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>
 ! CHECK:           %[[VAL_12:.*]] = fir.volatile_cast %[[VAL_11]] : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>
 ! CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFEcontainer"} : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>) -> (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>, !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>)
-! CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QFE.c.t) : !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>
+! CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QFE.c.t) : !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>
 ! CHECK:           %[[VAL_15:.*]] = fir.shape_shift %[[VAL_0]], %[[VAL_1]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]](%[[VAL_15]]) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.c.t"} : (!fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>)
-! CHECK:           %[[VAL_17:.*]] = fir.address_of(@_QFE.dt.t) : !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>
-! CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.dt.t"} : (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>) -> (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>, !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>)
+! CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]](%[[VAL_15]]) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.c.t"} : (!fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>)
+! CHECK:           %[[VAL_17:.*]] = fir.address_of(@_QFE.dt.t) : !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>
+! CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.dt.t"} : (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>) -> (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>, !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>)
 ! CHECK:           %[[VAL_19:.*]] = hlfir.designate %[[VAL_13]]#0{"array"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>, volatile>
 ! CHECK:           %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>, volatile>
 ! CHECK:           %[[VAL_21:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_0]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
diff --git a/flang/test/Semantics/typeinfo01.f90 b/flang/test/Semantics/typeinfo01.f90
index bb20c546e0261..aeec336ea58ea 100644
--- a/flang/test/Semantics/typeinfo01.f90
+++ b/flang/test/Semantics/typeinfo01.f90
@@ -87,8 +87,8 @@ subroutine s2(x, y)
 !CHECK: .c.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=6_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=2_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
 !CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=.s.t2,specialbitset=2_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
-!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s2)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
+!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s2)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 !CHECK: .v.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
 end module
@@ -115,8 +115,8 @@ subroutine s2(x, y)
 !CHECK: .c.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=6_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=2_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
 !CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=.s.t2,specialbitset=2_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
-!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s2)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
+!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s2)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 !CHECK: .v.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
 end module
@@ -133,7 +133,7 @@ impure elemental subroutine s1(x, y)
     class(t), intent(in) :: y
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=4_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 end module
 
@@ -156,7 +156,7 @@ subroutine s4(x)
     type(t), contiguous :: x(:,:,:)
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=7296_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=0_1,nofinalizationneeded=0_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=7_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=s3),specialbinding(which=10_1,isargdescriptorset=1_1,istypebound=0_1,isargcontiguousset=0_1,proc=s1),specialbinding(which=11_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=1_1,proc=s2),specialbinding(which=12_1,isargdescriptorset=1_1,istypebound=0_1,isargcontiguousset=1_1,proc=s4)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=7_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=s3),specialbinding(which=10_1,isargdescriptorset=1_1,istypebound=0_1,specialcaseflag=0_1,proc=s1),specialbinding(which=11_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=1_1,proc=s2),specialbinding(which=12_1,isargdescriptorset=1_1,istypebound=0_1,specialcaseflag=1_1,proc=s4)]
 end module
 
 module m09
@@ -198,7 +198,7 @@ subroutine wu(x,u,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=120_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=1_1,istypebound=2_1,isargcontiguousset=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=3_1,isargcontiguousset=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=1_1,istypebound=4_1,isargcontiguousset=0_1,proc=wu)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=1_1,istypebound=2_1,specialcaseflag=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=3_1,specialcaseflag=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=1_1,istypebound=4_1,specialcaseflag=0_1,proc=wu)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:3_8 init:[binding::binding(proc=rf,name=.n.rf),binding(proc=ru,name=.n.ru),binding(proc=wf,name=.n.wf),binding(proc=wu,name=.n.wu)]
 end module
 
@@ -247,7 +247,7 @@ subroutine wu(x,u,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=120_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=wu)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=wu)]
 end module
 
 module m11
@@ -290,7 +290,7 @@ module m13
    contains
     procedure :: assign1, assign2
     generic :: assignment(=) => assign1, assign2
-    ! CHECK: .s.t1, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=assign1)]
+    ! CHECK: .s.t1, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=assign1)]
   end type
  contains
   impure elemental subroutine assign1(to, from)
diff --git a/flang/test/Semantics/typeinfo02.f90 b/flang/test/Semantics/typeinfo02.f90
index 29d14c7a0f196..07293627ab492 100644
--- a/flang/test/Semantics/typeinfo02.f90
+++ b/flang/test/Semantics/typeinfo02.f90
@@ -29,5 +29,5 @@ subroutine wf2(x,u,iot,v,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 end module
-!CHECK: .s.base, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=wf1)]
-!CHECK: .s.extended, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=wf2)]
+!CHECK: .s.base, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=wf1)]
+!CHECK: .s.extended, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=wf2)]
diff --git a/flang/test/Semantics/typeinfo09.f90 b/flang/test/Semantics/typeinfo09.f90
index 3527ee6058ad8..8daa6a5f420d7 100644
--- a/flang/test/Semantics/typeinfo09.f90
+++ b/flang/test/Semantics/typeinfo09.f90
@@ -17,4 +17,4 @@ subroutine copy_impl(this, x)
 end interface
 end module
 
-!CHECK: .s.sometype, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=copy_impl)]
+!CHECK: .s.sometype, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=copy_impl)]
diff --git a/flang/test/Semantics/typeinfo13.f90 b/flang/test/Semantics/typeinfo13.f90
index ad824ad3590a2..facc280815722 100644
--- a/flang/test/Semantics/typeinfo13.f90
+++ b/flang/test/Semantics/typeinfo13.f90
@@ -22,5 +22,5 @@ impure elemental subroutine override(to, from)
   end
 end
 
-!CHECK: .s.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=1_1,istypebound=2_1,isargcontiguousset=0_1,proc=override)]
+!CHECK: .s.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=1_1,istypebound=2_1,specialcaseflag=0_1,proc=override)]
 !CHECK: .v.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:1_8 init:[binding::binding(proc=baseassign,name=.n.baseassign),binding(proc=override,name=.n.override)]


From flang-commits at lists.llvm.org  Wed Jul 16 08:53:52 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Wed, 16 Jul 2025 08:53:52 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch bad members of BIND(C) COMMON
	block (PR #148971)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148971@github.com>
Message-ID: <6877cb10.050a0220.cda97.02d2@mx.google.com>


https://github.com/DanielCChen approved this pull request.

LGTM.
Thanks!

https://github.com/llvm/llvm-project/pull/148971

From flang-commits at lists.llvm.org  Wed Jul 16 09:09:09 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:09:09 -0700 (PDT)
Subject: [flang-commits] [flang] 6e0b0ec - [flang] Fix crash in Semantics
	(#148706)
Message-ID: <6877cea5.170a0220.36daab.1005@mx.google.com>


Author: Peter Klausler
Date: 2025-07-16T09:09:05-07:00
New Revision: 6e0b0ec66ac8de046cc95080166e2012819f7d93

URL: https://github.com/llvm/llvm-project/commit/6e0b0ec66ac8de046cc95080166e2012819f7d93
DIFF: https://github.com/llvm/llvm-project/commit/6e0b0ec66ac8de046cc95080166e2012819f7d93.diff

LOG: [flang] Fix crash in Semantics (#148706)

Allow for renaming in USE association of Cray pointers.

Fixes https://github.com/llvm/llvm-project/issues/148559.

Added: 
    flang/test/Semantics/bug148559.f90

Modified: 
    flang/lib/Semantics/tools.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 5e5b43f26c791..5a5b02e1ac3ce 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -348,9 +348,9 @@ const Symbol &BypassGeneric(const Symbol &symbol) {
 
 const Symbol &GetCrayPointer(const Symbol &crayPointee) {
   const Symbol *found{nullptr};
-  for (const auto &[pointee, pointer] :
-      crayPointee.GetUltimate().owner().crayPointers()) {
-    if (pointee == crayPointee.name()) {
+  const Symbol &ultimate{crayPointee.GetUltimate()};
+  for (const auto &[pointee, pointer] : ultimate.owner().crayPointers()) {
+    if (pointee == ultimate.name()) {
       found = &pointer.get();
       break;
     }

diff  --git a/flang/test/Semantics/bug148559.f90 b/flang/test/Semantics/bug148559.f90
new file mode 100644
index 0000000000000..d7b959ac8f191
--- /dev/null
+++ b/flang/test/Semantics/bug148559.f90
@@ -0,0 +1,12 @@
+!RUN: %flang_fc1 -fsyntax-only %s
+!Regression test for crash in semantics on Cray pointers
+
+module m
+  pointer(ptr,pp)
+end module m
+
+program main
+  use m, only:renamea=>pp
+  use m, only:pp
+  print *, renamea
+end


        

From flang-commits at lists.llvm.org  Wed Jul 16 09:09:12 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 09:09:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fix crash in Semantics (PR #148706)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148706@github.com>
Message-ID: <6877cea8.170a0220.2e825d.07cf@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/148706

From flang-commits at lists.llvm.org  Wed Jul 16 09:09:32 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:09:32 -0700 (PDT)
Subject: [flang-commits] [flang] fc7f9d7 - [flang] Better error message for
 ambiguous ASSIGNMENT(=) (#148720)
Message-ID: <6877cebc.050a0220.2a553c.3c81@mx.google.com>


Author: Peter Klausler
Date: 2025-07-16T09:09:29-07:00
New Revision: fc7f9d795d37cd119831e77e475e4690e4120bdb

URL: https://github.com/llvm/llvm-project/commit/fc7f9d795d37cd119831e77e475e4690e4120bdb
DIFF: https://github.com/llvm/llvm-project/commit/fc7f9d795d37cd119831e77e475e4690e4120bdb.diff

LOG: [flang] Better error message for ambiguous ASSIGNMENT(=) (#148720)

When a type-bound generic ASSIGNMENT(=) procedure is ambiguous for a
particular reference, say so, rather than claiming that no specific
procedure matched the types and ranks of the LHS and RHS.

Fixes https://github.com/llvm/llvm-project/issues/148675.

Added: 
    flang/test/Semantics/bug148675.f90

Modified: 
    flang/lib/Semantics/expression.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 53ec3827893d0..14473724f0f40 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -178,7 +178,7 @@ class ArgumentAnalyzer {
   }
   // Find and return a user-defined assignment
   std::optional<ProcedureRef> TryDefinedAssignment();
-  std::optional<ProcedureRef> GetDefinedAssignmentProc();
+  std::optional<ProcedureRef> GetDefinedAssignmentProc(bool &isAmbiguous);
   std::optional<DynamicType> GetType(std::size_t) const;
   void Dump(llvm::raw_ostream &);
 
@@ -191,7 +191,7 @@ class ArgumentAnalyzer {
   MaybeExpr AnalyzeExprOrWholeAssumedSizeArray(const parser::Expr &);
   bool AreConformable() const;
   const Symbol *FindBoundOp(parser::CharBlock, int passIndex,
-      const Symbol *&generic, bool isSubroutine);
+      const Symbol *&generic, bool isSubroutine, bool *isAmbiguous = nullptr);
   void AddAssignmentConversion(
       const DynamicType &lhsType, const DynamicType &rhsType);
   bool OkLogicalIntegerAssignment(TypeCategory lhs, TypeCategory rhs);
@@ -199,7 +199,8 @@ class ArgumentAnalyzer {
   bool IsBOZLiteral(std::size_t i) const {
     return evaluate::IsBOZLiteral(GetExpr(i));
   }
-  void SayNoMatch(const std::string &, bool isAssignment = false);
+  void SayNoMatch(
+      const std::string &, bool isAssignment = false, bool isAmbiguous = false);
   std::string TypeAsFortran(std::size_t);
   bool AnyUntypedOrMissingOperand();
 
@@ -4781,7 +4782,9 @@ std::optional<ProcedureRef> ArgumentAnalyzer::TryDefinedAssignment() {
     return std::nullopt; // user-defined assignment not allowed for these args
   }
   auto restorer{context_.GetContextualMessages().SetLocation(source_)};
-  if (std::optional<ProcedureRef> procRef{GetDefinedAssignmentProc()}) {
+  bool isAmbiguous{false};
+  if (std::optional<ProcedureRef> procRef{
+          GetDefinedAssignmentProc(isAmbiguous)}) {
     if (context_.inWhereBody() && !procRef->proc().IsElemental()) { // C1032
       context_.Say(
           "Defined assignment in WHERE must be elemental, but '%s' is not"_err_en_US,
@@ -4791,9 +4794,11 @@ std::optional<ProcedureRef> ArgumentAnalyzer::TryDefinedAssignment() {
     return std::move(*procRef);
   }
   if (isDefined == Tristate::Yes) {
-    if (!lhsType || !rhsType || (lhsRank != rhsRank && rhsRank != 0) ||
+    if (isAmbiguous || !lhsType || !rhsType ||
+        (lhsRank != rhsRank && rhsRank != 0) ||
         !OkLogicalIntegerAssignment(lhsType->category(), rhsType->category())) {
-      SayNoMatch("ASSIGNMENT(=)", true);
+      SayNoMatch(
+          "ASSIGNMENT(=)", /*isAssignment=*/true, /*isAmbiguous=*/isAmbiguous);
     }
   } else if (!fatalErrors_) {
     CheckAssignmentConformance();
@@ -4822,13 +4827,15 @@ bool ArgumentAnalyzer::OkLogicalIntegerAssignment(
   return true;
 }
 
-std::optional<ProcedureRef> ArgumentAnalyzer::GetDefinedAssignmentProc() {
+std::optional<ProcedureRef> ArgumentAnalyzer::GetDefinedAssignmentProc(
+    bool &isAmbiguous) {
   const Symbol *proc{nullptr};
   bool isProcElemental{false};
   std::optional<int> passedObjectIndex;
   std::string oprNameString{"assignment(=)"};
   parser::CharBlock oprName{oprNameString};
   const auto &scope{context_.context().FindScope(source_)};
+  isAmbiguous = false;
   {
     auto restorer{context_.GetContextualMessages().DiscardMessages()};
     if (const Symbol *symbol{scope.FindSymbol(oprName)}) {
@@ -4842,8 +4849,8 @@ std::optional<ProcedureRef> ArgumentAnalyzer::GetDefinedAssignmentProc() {
     for (std::size_t i{0}; (!proc || isProcElemental) && i < actuals_.size();
         ++i) {
       const Symbol *generic{nullptr};
-      if (const Symbol *
-          binding{FindBoundOp(oprName, i, generic, /*isSubroutine=*/true)}) {
+      if (const Symbol *binding{FindBoundOp(oprName, i, generic,
+              /*isSubroutine=*/true, /*isAmbiguous=*/&isAmbiguous)}) {
         // ignore inaccessible type-bound ASSIGNMENT(=) generic
         if (!CheckAccessibleSymbol(scope, DEREF(generic))) {
           const Symbol *resolution{GetBindingResolution(GetType(i), *binding)};
@@ -4967,7 +4974,8 @@ bool ArgumentAnalyzer::AreConformable() const {
 
 // Look for a type-bound operator in the type of arg number passIndex.
 const Symbol *ArgumentAnalyzer::FindBoundOp(parser::CharBlock oprName,
-    int passIndex, const Symbol *&generic, bool isSubroutine) {
+    int passIndex, const Symbol *&generic, bool isSubroutine,
+    bool *isAmbiguous) {
   const auto *type{GetDerivedTypeSpec(GetType(passIndex))};
   const semantics::Scope *scope{type ? type->scope() : nullptr};
   if (scope) {
@@ -4989,6 +4997,9 @@ const Symbol *ArgumentAnalyzer::FindBoundOp(parser::CharBlock oprName,
       // Use the most recent override of the binding, if any
       return scope->FindComponent(binding->name());
     } else {
+      if (isAmbiguous) {
+        *isAmbiguous = pair.second;
+      }
       context_.EmitGenericResolutionError(*generic, pair.second, isSubroutine);
     }
   }
@@ -5072,40 +5083,37 @@ void ArgumentAnalyzer::ConvertBOZAssignmentRHS(const DynamicType &lhsType) {
 }
 
 // Report error resolving opr when there is a user-defined one available
-void ArgumentAnalyzer::SayNoMatch(const std::string &opr, bool isAssignment) {
+void ArgumentAnalyzer::SayNoMatch(
+    const std::string &opr, bool isAssignment, bool isAmbiguous) {
   std::string type0{TypeAsFortran(0)};
   auto rank0{actuals_[0]->Rank()};
+  std::string prefix{"No intrinsic or user-defined "s + opr + " matches"};
+  if (isAmbiguous) {
+    prefix = "Multiple specific procedures for the generic "s + opr + " match";
+  }
   if (actuals_.size() == 1) {
     if (rank0 > 0) {
-      context_.Say("No intrinsic or user-defined %s matches "
-                   "rank %d array of %s"_err_en_US,
-          opr, rank0, type0);
+      context_.Say("%s rank %d array of %s"_err_en_US, prefix, rank0, type0);
     } else {
-      context_.Say("No intrinsic or user-defined %s matches "
-                   "operand type %s"_err_en_US,
-          opr, type0);
+      context_.Say("%s operand type %s"_err_en_US, prefix, type0);
     }
   } else {
     std::string type1{TypeAsFortran(1)};
     auto rank1{actuals_[1]->Rank()};
     if (rank0 > 0 && rank1 > 0 && rank0 != rank1) {
-      context_.Say("No intrinsic or user-defined %s matches "
-                   "rank %d array of %s and rank %d array of %s"_err_en_US,
-          opr, rank0, type0, rank1, type1);
+      context_.Say("%s rank %d array of %s and rank %d array of %s"_err_en_US,
+          prefix, rank0, type0, rank1, type1);
     } else if (isAssignment && rank0 != rank1) {
       if (rank0 == 0) {
-        context_.Say("No intrinsic or user-defined %s matches "
-                     "scalar %s and rank %d array of %s"_err_en_US,
-            opr, type0, rank1, type1);
+        context_.Say("%s scalar %s and rank %d array of %s"_err_en_US, prefix,
+            type0, rank1, type1);
       } else {
-        context_.Say("No intrinsic or user-defined %s matches "
-                     "rank %d array of %s and scalar %s"_err_en_US,
-            opr, rank0, type0, type1);
+        context_.Say("%s rank %d array of %s and scalar %s"_err_en_US, prefix,
+            rank0, type0, type1);
       }
     } else {
-      context_.Say("No intrinsic or user-defined %s matches "
-                   "operand types %s and %s"_err_en_US,
-          opr, type0, type1);
+      context_.Say(
+          "%s operand types %s and %s"_err_en_US, prefix, type0, type1);
     }
   }
 }

diff  --git a/flang/test/Semantics/bug148675.f90 b/flang/test/Semantics/bug148675.f90
new file mode 100644
index 0000000000000..5ce117e7bb3df
--- /dev/null
+++ b/flang/test/Semantics/bug148675.f90
@@ -0,0 +1,21 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+module m
+  type t
+    integer n
+   contains
+    procedure :: assign1 => myassign, assign2 => myassign
+    generic :: ASSIGNMENT(=) => assign1
+    generic :: ASSIGNMENT(=) => assign2
+  end type
+ contains
+  subroutine myassign(to, from)
+    class(t), intent(out) :: to
+    integer, intent(in) :: from
+    to%n = from
+  end
+  subroutine test
+    type(t) x
+    !ERROR: Multiple specific procedures for the generic ASSIGNMENT(=) match operand types TYPE(t) and INTEGER(4)
+    x = 5
+  end
+end


        

From flang-commits at lists.llvm.org  Wed Jul 16 09:09:35 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 09:09:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Better error message for ambiguous
	ASSIGNMENT(=) (PR #148720)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148720@github.com>
Message-ID: <6877cebf.050a0220.1248d3.68c2@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/148720

From flang-commits at lists.llvm.org  Wed Jul 16 09:09:55 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 09:09:55 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148927@github.com>
Message-ID: <6877ced3.170a0220.98ed.01ec@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/148927

From flang-commits at lists.llvm.org  Wed Jul 16 09:10:10 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:10:10 -0700 (PDT)
Subject: [flang-commits] [flang] bbcdad1 - [flang][runtime] MCLOCK library
	routine (#148960)
Message-ID: <6877cee2.170a0220.3bfa70.bd9a@mx.google.com>


Author: Peter Klausler
Date: 2025-07-16T09:10:07-07:00
New Revision: bbcdad1f8eab303a9d56c76a0bced7b17c6d2656

URL: https://github.com/llvm/llvm-project/commit/bbcdad1f8eab303a9d56c76a0bced7b17c6d2656
DIFF: https://github.com/llvm/llvm-project/commit/bbcdad1f8eab303a9d56c76a0bced7b17c6d2656.diff

LOG: [flang][runtime] MCLOCK library routine (#148960)

Add MCLOCK as an interface to std::clock().

Added: 
    

Modified: 
    flang-rt/lib/runtime/extensions.cpp
    flang/docs/Intrinsics.md
    flang/include/flang/Runtime/extensions.h

Removed: 
    


################################################################################
diff  --git a/flang-rt/lib/runtime/extensions.cpp b/flang-rt/lib/runtime/extensions.cpp
index e70dff3997233..f6c39468d5655 100644
--- a/flang-rt/lib/runtime/extensions.cpp
+++ b/flang-rt/lib/runtime/extensions.cpp
@@ -27,10 +27,7 @@
 #include <thread>
 
 #ifdef _WIN32
-#define WIN32_LEAN_AND_MEAN
-#define NOMINMAX
-#include <windows.h>
-
+#include "flang/Common/windows-include.h"
 #include <synchapi.h>
 
 inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time,
@@ -309,6 +306,9 @@ void RTNAME(Perror)(const char *str) { perror(str); }
 // GNU extension function TIME()
 std::int64_t RTNAME(time)() { return time(nullptr); }
 
+// MCLOCK: returns accumulated CPU time in ticks
+std::int32_t FORTRAN_PROCEDURE_NAME(mclock)() { return std::clock(); }
+
 // Extension procedures related to I/O
 
 namespace io {

diff  --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index 0118f8eb7d913..f7da6c889d413 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -709,8 +709,9 @@ CACHESIZE, EOF, FP_CLASS, INT_PTR_KIND, ISNAN, LOC
 MALLOC, FREE
 ```
 
-### Library subroutine
+### Library subroutines and functions
 ```
+ticks = MCLOCK()
 CALL BACKTRACE()
 CALL FDATE(TIME)
 CALL GETLOG(USRNAME)

diff  --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index 06ae7f35d9b5b..b350204714431 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -12,14 +12,12 @@
 #ifndef FORTRAN_RUNTIME_EXTENSIONS_H_
 #define FORTRAN_RUNTIME_EXTENSIONS_H_
 
-#include "flang/Runtime/entry-names.h"
-
-#define FORTRAN_PROCEDURE_NAME(name) name##_
-
 #include "flang/Runtime/entry-names.h"
 #include <cstddef>
 #include <cstdint>
 
+#define FORTRAN_PROCEDURE_NAME(name) name##_
+
 #ifdef _WIN32
 // UID and GID don't exist on Windows, these exist to avoid errors.
 typedef std::uint32_t uid_t;
@@ -89,5 +87,8 @@ int FORTRAN_PROCEDURE_NAME(ierrno)();
 // GNU extension subroutine PERROR(STRING)
 void RTNAME(Perror)(const char *str);
 
+// MCLOCK -- returns accumulated time in ticks
+int FORTRAN_PROCEDURE_NAME(mclock)();
+
 } // extern "C"
 #endif // FORTRAN_RUNTIME_EXTENSIONS_H_


        

From flang-commits at lists.llvm.org  Wed Jul 16 09:10:14 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 09:10:14 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][runtime] MCLOCK library
	routine (PR #148960)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148960@github.com>
Message-ID: <6877cee6.170a0220.19478f.0fe9@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/148960

From flang-commits at lists.llvm.org  Wed Jul 16 09:10:30 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:10:30 -0700 (PDT)
Subject: [flang-commits] [flang] 3de11b7 - [flang] Catch bad members of
	BIND(C) COMMON block (#148971)
Message-ID: <6877cef6.630a0220.2a28de.2eaa@mx.google.com>


Author: Peter Klausler
Date: 2025-07-16T09:10:26-07:00
New Revision: 3de11b70620d911613a48d493048cb48bb76ec19

URL: https://github.com/llvm/llvm-project/commit/3de11b70620d911613a48d493048cb48bb76ec19
DIFF: https://github.com/llvm/llvm-project/commit/3de11b70620d911613a48d493048cb48bb76ec19.diff

LOG: [flang] Catch bad members of BIND(C) COMMON block (#148971)

Variables that can't be BIND(C), like pointers, can't be in a BIND(C)
common block, either.

Fixes https://github.com/llvm/llvm-project/issues/148922.

Added: 
    flang/test/Semantics/bind-c18.f90

Modified: 
    flang/lib/Semantics/check-declarations.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index f9d64485f1407..a2f2906af10b8 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -151,8 +151,8 @@ class CheckHelper {
   void CheckProcedureAssemblyName(const Symbol &symbol);
   void CheckExplicitSave(const Symbol &);
   parser::Messages WhyNotInteroperableDerivedType(const Symbol &);
-  parser::Messages WhyNotInteroperableObject(
-      const Symbol &, bool allowNonInteroperableType = false);
+  parser::Messages WhyNotInteroperableObject(const Symbol &,
+      bool allowNonInteroperableType = false, bool forCommonBlock = false);
   parser::Messages WhyNotInteroperableFunctionResult(const Symbol &);
   parser::Messages WhyNotInteroperableProcedure(const Symbol &, bool isError);
   void CheckBindC(const Symbol &);
@@ -519,11 +519,35 @@ void CheckHelper::Check(const Symbol &symbol) {
 }
 
 void CheckHelper::CheckCommonBlock(const Symbol &symbol) {
+  auto restorer{messages_.SetLocation(symbol.name())};
   CheckGlobalName(symbol);
   if (symbol.attrs().test(Attr::BIND_C)) {
     CheckBindC(symbol);
+    for (auto ref : symbol.get<CommonBlockDetails>().objects()) {
+      if (ref->has<ObjectEntityDetails>()) {
+        if (auto msgs{WhyNotInteroperableObject(*ref,
+                /*allowInteroperableType=*/false, /*forCommonBlock=*/true)};
+            !msgs.empty()) {
+          parser::Message &reason{msgs.messages().front()};
+          parser::Message *msg{nullptr};
+          if (reason.IsFatal()) {
+            msg = messages_.Say(symbol.name(),
+                "'%s' may not be a member of BIND(C) COMMON block /%s/"_err_en_US,
+                ref->name(), symbol.name());
+          } else {
+            msg = messages_.Say(symbol.name(),
+                "'%s' should not be a member of BIND(C) COMMON block /%s/"_warn_en_US,
+                ref->name(), symbol.name());
+          }
+          if (msg) {
+            msg->Attach(
+                std::move(reason.set_severity(parser::Severity::Because)));
+          }
+        }
+      }
+    }
   }
-  for (MutableSymbolRef ref : symbol.get<CommonBlockDetails>().objects()) {
+  for (auto ref : symbol.get<CommonBlockDetails>().objects()) {
     if (ref->test(Symbol::Flag::CrayPointee)) {
       messages_.Say(ref->name(),
           "Cray pointee '%s' may not be a member of a COMMON block"_err_en_US,
@@ -3154,14 +3178,16 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
 }
 
 parser::Messages CheckHelper::WhyNotInteroperableObject(
-    const Symbol &symbol, bool allowNonInteroperableType) {
+    const Symbol &symbol, bool allowNonInteroperableType, bool forCommonBlock) {
   parser::Messages msgs;
-  if (examinedByWhyNotInteroperable_.find(symbol) !=
-      examinedByWhyNotInteroperable_.end()) {
-    return msgs;
+  if (!forCommonBlock) {
+    if (examinedByWhyNotInteroperable_.find(symbol) !=
+        examinedByWhyNotInteroperable_.end()) {
+      return msgs;
+    }
+    examinedByWhyNotInteroperable_.insert(symbol);
   }
   bool isExplicitBindC{symbol.attrs().test(Attr::BIND_C)};
-  examinedByWhyNotInteroperable_.insert(symbol);
   CHECK(symbol.has<ObjectEntityDetails>());
   if (isExplicitBindC && !symbol.owner().IsModule()) {
     msgs.Say(symbol.name(),
@@ -3258,7 +3284,7 @@ parser::Messages CheckHelper::WhyNotInteroperableObject(
     msgs.Say(symbol.name(),
         "An interoperable pointer must not be CONTIGUOUS"_err_en_US);
   }
-  if (msgs.AnyFatalError()) {
+  if (!forCommonBlock && msgs.AnyFatalError()) {
     examinedByWhyNotInteroperable_.erase(symbol);
   }
   return msgs;
@@ -3338,8 +3364,8 @@ parser::Messages CheckHelper::WhyNotInteroperableProcedure(
           // on the C side by either a cdesc_t * or a void *.  F'2023 18.3.7 (5)
           bool allowNonInteroperableType{!dummy->attrs().test(Attr::VALUE) &&
               (IsDescriptor(*dummy) || IsAssumedType(*dummy))};
-          dummyMsgs =
-              WhyNotInteroperableObject(*dummy, allowNonInteroperableType);
+          dummyMsgs = WhyNotInteroperableObject(
+              *dummy, allowNonInteroperableType, /*forCommonBlock=*/false);
         } else {
           CheckBindC(*dummy);
         }

diff  --git a/flang/test/Semantics/bind-c18.f90 b/flang/test/Semantics/bind-c18.f90
new file mode 100644
index 0000000000000..f61111458c6d9
--- /dev/null
+++ b/flang/test/Semantics/bind-c18.f90
@@ -0,0 +1,7 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+bind(c) :: /blk/
+!ERROR: 'x' may not be a member of BIND(C) COMMON block /blk/
+common /blk/ x
+!BECAUSE: A scalar interoperable variable may not be ALLOCATABLE or POINTER
+integer, pointer :: x
+end


        

From flang-commits at lists.llvm.org  Wed Jul 16 09:10:33 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 09:10:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch bad members of BIND(C) COMMON
	block (PR #148971)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148971@github.com>
Message-ID: <6877cef9.170a0220.3bc0df.188a@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/148971

From flang-commits at lists.llvm.org  Wed Jul 16 09:09:52 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:09:52 -0700 (PDT)
Subject: [flang-commits] [flang] 52a46dc - [flang] Allow -fdefault-integer-8
 with defined I/O (#148927)
Message-ID: <6877ced0.170a0220.2c3ca7.aeae@mx.google.com>


Author: Peter Klausler
Date: 2025-07-16T09:09:49-07:00
New Revision: 52a46dc57f29e87a5a298ce325562fa2e3d057c9

URL: https://github.com/llvm/llvm-project/commit/52a46dc57f29e87a5a298ce325562fa2e3d057c9
DIFF: https://github.com/llvm/llvm-project/commit/52a46dc57f29e87a5a298ce325562fa2e3d057c9.diff

LOG: [flang] Allow -fdefault-integer-8 with defined I/O (#148927)

Defined I/O subroutines have UNIT= and IOSTAT= dummy arguments that are
required to have type INTEGER with its default kind. When that default
kind is modified via -fdefault-integer-8, calls to defined I/O
subroutines from the runtime don't work.

Add a flag to the two data structures shared between the compiler and
the runtime support library to indicate that a defined I/O subroutine
was compiled under -fdefault-integer-8. This has been done in a
compatible manner, so that existing binaries are compatible with the new
library and new binaries are compatible with the old library, unless of
course -fdefault-integer-8 is used.

Fixes https://github.com/llvm/llvm-project/issues/148638.

Added: 
    

Modified: 
    flang-rt/include/flang-rt/runtime/non-tbp-dio.h
    flang-rt/include/flang-rt/runtime/type-info.h
    flang-rt/lib/runtime/derived.cpp
    flang-rt/lib/runtime/descriptor-io.cpp
    flang-rt/lib/runtime/non-tbp-dio.cpp
    flang-rt/lib/runtime/type-info.cpp
    flang/include/flang/Semantics/runtime-type-info.h
    flang/lib/Lower/IO.cpp
    flang/lib/Semantics/runtime-type-info.cpp
    flang/module/__fortran_type_info.f90
    flang/test/Lower/io-derived-type.f90
    flang/test/Lower/namelist.f90
    flang/test/Lower/volatile-openmp.f90
    flang/test/Semantics/typeinfo01.f90
    flang/test/Semantics/typeinfo02.f90
    flang/test/Semantics/typeinfo09.f90
    flang/test/Semantics/typeinfo13.f90

Removed: 
    


################################################################################
diff  --git a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
index 99d4113b6c7a8..26849298ec959 100644
--- a/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
+++ b/flang-rt/include/flang-rt/runtime/non-tbp-dio.h
@@ -34,11 +34,16 @@ namespace Fortran::runtime::io {
 
 RT_OFFLOAD_API_GROUP_BEGIN
 
+enum NonTbpDefinedIoFlags {
+  IsDtvArgPolymorphic = 1 << 0, // first dummy arg is CLASS(T)
+  DefinedIoInteger8 = 1 << 1, // -fdefault-integer-8 affected UNIT= & IOSTAT=
+};
+
 struct NonTbpDefinedIo {
   const typeInfo::DerivedType &derivedType;
   void (*subroutine)(); // null means no non-TBP defined I/O here
   common::DefinedIo definedIo;
-  bool isDtvArgPolymorphic; // first dummy arg is CLASS(T)
+  std::uint8_t flags;
 };
 
 struct NonTbpDefinedIoTable {

diff  --git a/flang-rt/include/flang-rt/runtime/type-info.h b/flang-rt/include/flang-rt/runtime/type-info.h
index a8d39f4f8a1a3..93bca24a602b4 100644
--- a/flang-rt/include/flang-rt/runtime/type-info.h
+++ b/flang-rt/include/flang-rt/runtime/type-info.h
@@ -143,9 +143,9 @@ class SpecialBinding {
   // I/O procedures that are not type-bound.
   RT_API_ATTRS SpecialBinding(Which which, ProcedurePointer proc,
       std::uint8_t isArgDescSet, std::uint8_t isTypeBound,
-      std::uint8_t isArgContiguousSet)
+      std::uint8_t specialCaseFlag)
       : which_{which}, isArgDescriptorSet_{isArgDescSet},
-        isTypeBound_{isTypeBound}, isArgContiguousSet_{isArgContiguousSet},
+        isTypeBound_{isTypeBound}, specialCaseFlag_{specialCaseFlag},
         proc_{proc} {}
 
   static constexpr RT_API_ATTRS Which RankFinal(int rank) {
@@ -153,13 +153,11 @@ class SpecialBinding {
   }
 
   RT_API_ATTRS Which which() const { return which_; }
+  RT_API_ATTRS bool specialCaseFlag() const { return specialCaseFlag_; }
   RT_API_ATTRS bool IsArgDescriptor(int zeroBasedArg) const {
     return (isArgDescriptorSet_ >> zeroBasedArg) & 1;
   }
   RT_API_ATTRS bool IsTypeBound() const { return isTypeBound_ != 0; }
-  RT_API_ATTRS bool IsArgContiguous(int zeroBasedArg) const {
-    return (isArgContiguousSet_ >> zeroBasedArg) & 1;
-  }
   template <typename PROC>
   RT_API_ATTRS PROC GetProc(const Binding *bindings = nullptr) const {
     if (bindings && isTypeBound_ > 0) {
@@ -203,10 +201,10 @@ class SpecialBinding {
   // When a special binding is type-bound, this is its binding's index (plus 1,
   // so that 0 signifies that it's not type-bound).
   std::uint8_t isTypeBound_{0};
-  // True when a FINAL subroutine has a dummy argument that is an array that
-  // is CONTIGUOUS or neither assumed-rank nor assumed-shape.
-  std::uint8_t isArgContiguousSet_{0};
-
+  // For a FINAL subroutine, set when it has a dummy argument that is an array
+  // that is CONTIGUOUS or neither assumed-rank nor assumed-shape.
+  // For a defined I/O subroutine, set when UNIT= and IOSTAT= are INTEGER(8).
+  std::uint8_t specialCaseFlag_{0};
   ProcedurePointer proc_{nullptr};
 };
 

diff  --git a/flang-rt/lib/runtime/derived.cpp b/flang-rt/lib/runtime/derived.cpp
index bb9a68abef2a7..4ed0baaa3d108 100644
--- a/flang-rt/lib/runtime/derived.cpp
+++ b/flang-rt/lib/runtime/derived.cpp
@@ -270,7 +270,7 @@ static RT_API_ATTRS void CallFinalSubroutine(const Descriptor &descriptor,
       StaticDescriptor<maxRank, true, 10> statDesc;
       Descriptor &copy{statDesc.descriptor()};
       const Descriptor *argDescriptor{&descriptor};
-      if (descriptor.rank() > 0 && special->IsArgContiguous(0) &&
+      if (descriptor.rank() > 0 && special->specialCaseFlag() &&
           !descriptor.IsContiguous()) {
         // The FINAL subroutine demands a contiguous array argument, but
         // this INTENT(OUT) or intrinsic assignment LHS isn't contiguous.

diff  --git a/flang-rt/lib/runtime/descriptor-io.cpp b/flang-rt/lib/runtime/descriptor-io.cpp
index b208cb2c397b3..3868c8ddce19f 100644
--- a/flang-rt/lib/runtime/descriptor-io.cpp
+++ b/flang-rt/lib/runtime/descriptor-io.cpp
@@ -67,13 +67,29 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
           ioType, io.mutableModes().inNamelist ? "NAMELIST" : "LISTDIRECTED");
       ioTypeLen = runtime::strlen(ioType);
     }
+    // V_LIST= argument
     StaticDescriptor<1, true> vListStatDesc;
     Descriptor &vListDesc{vListStatDesc.descriptor()};
-    vListDesc.Establish(TypeCategory::Integer, sizeof(int), nullptr, 1);
-    vListDesc.set_base_addr(edit.vList);
-    vListDesc.GetDimension(0).SetBounds(1, edit.vListEntries);
-    vListDesc.GetDimension(0).SetByteStride(
-        static_cast<SubscriptValue>(sizeof(int)));
+    bool integer8{special.specialCaseFlag()};
+    std::int64_t vList64[edit.maxVListEntries];
+    if (integer8) {
+      // Convert v_list values to INTEGER(8)
+      for (int j{0}; j < edit.vListEntries; ++j) {
+        vList64[j] = edit.vList[j];
+      }
+      vListDesc.Establish(
+          TypeCategory::Integer, sizeof(std::int64_t), nullptr, 1);
+      vListDesc.set_base_addr(vList64);
+      vListDesc.GetDimension(0).SetBounds(1, edit.vListEntries);
+      vListDesc.GetDimension(0).SetByteStride(
+          static_cast<SubscriptValue>(sizeof(std::int64_t)));
+    } else {
+      vListDesc.Establish(TypeCategory::Integer, sizeof(int), nullptr, 1);
+      vListDesc.set_base_addr(edit.vList);
+      vListDesc.GetDimension(0).SetBounds(1, edit.vListEntries);
+      vListDesc.GetDimension(0).SetByteStride(
+          static_cast<SubscriptValue>(sizeof(int)));
+    }
     ExternalFileUnit *actualExternal{io.GetExternalFileUnit()};
     ExternalFileUnit *external{actualExternal};
     if (!external) {
@@ -84,8 +100,8 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
     ChildIo &child{external->PushChildIo(io)};
     // Child formatted I/O is nonadvancing by definition (F'2018 12.6.2.4).
     auto restorer{common::ScopedSet(io.mutableModes().nonAdvancing, true)};
-    int unit{external->unitNumber()};
-    int ioStat{IostatOk};
+    std::int32_t unit{external->unitNumber()};
+    std::int32_t ioStat{IostatOk};
     char ioMsg[100];
     Fortran::common::optional<std::int64_t> startPos;
     if (edit.descriptor == DataEdit::DefinedDerivedType &&
@@ -98,23 +114,45 @@ static RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
         derived.binding().OffsetElement<const typeInfo::Binding>()};
     if (special.IsArgDescriptor(0)) {
       // "dtv" argument is "class(t)", pass a descriptor
-      auto *p{special.GetProc<void (*)(const Descriptor &, int &, char *,
-          const Descriptor &, int &, char *, std::size_t, std::size_t)>(
-          bindings)};
       StaticDescriptor<1, true, 10 /*?*/> elementStatDesc;
       Descriptor &elementDesc{elementStatDesc.descriptor()};
       elementDesc.Establish(
           derived, nullptr, 0, nullptr, CFI_attribute_pointer);
       elementDesc.set_base_addr(descriptor.Element<char>(subscripts));
-      p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen,
-          sizeof ioMsg);
+      if (integer8) { // 64-bit UNIT=/IOSTAT=
+        std::int64_t unit64{unit};
+        std::int64_t ioStat64{ioStat};
+        auto *p{special.GetProc<void (*)(const Descriptor &, std::int64_t &,
+            char *, const Descriptor &, std::int64_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(elementDesc, unit64, ioType, vListDesc, ioStat64, ioMsg, ioTypeLen,
+            sizeof ioMsg);
+        ioStat = ioStat64;
+      } else { // 32-bit UNIT=/IOSTAT=
+        auto *p{special.GetProc<void (*)(const Descriptor &, std::int32_t &,
+            char *, const Descriptor &, std::int32_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen,
+            sizeof ioMsg);
+      }
     } else {
       // "dtv" argument is "type(t)", pass a raw pointer
-      auto *p{special.GetProc<void (*)(const void *, int &, char *,
-          const Descriptor &, int &, char *, std::size_t, std::size_t)>(
-          bindings)};
-      p(descriptor.Element<char>(subscripts), unit, ioType, vListDesc, ioStat,
-          ioMsg, ioTypeLen, sizeof ioMsg);
+      if (integer8) { // 64-bit UNIT= and IOSTAT=
+        std::int64_t unit64{unit};
+        std::int64_t ioStat64{ioStat};
+        auto *p{special.GetProc<void (*)(const void *, std::int64_t &, char *,
+            const Descriptor &, std::int64_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(descriptor.Element<char>(subscripts), unit64, ioType, vListDesc,
+            ioStat64, ioMsg, ioTypeLen, sizeof ioMsg);
+        ioStat = ioStat64;
+      } else { // 32-bit UNIT= and IOSTAT=
+        auto *p{special.GetProc<void (*)(const void *, std::int32_t &, char *,
+            const Descriptor &, std::int32_t &, char *, std::size_t,
+            std::size_t)>(bindings)};
+        p(descriptor.Element<char>(subscripts), unit, ioType, vListDesc, ioStat,
+            ioMsg, ioTypeLen, sizeof ioMsg);
+      }
     }
     handler.Forward(ioStat, ioMsg, sizeof ioMsg);
     external->PopChildIo(child);
@@ -458,11 +496,16 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
                       ? common::DefinedIo::ReadUnformatted
                       : common::DefinedIo::WriteUnformatted)}) {
             if (definedIo->subroutine) {
+              std::uint8_t isArgDescriptorSet{0};
+              if (definedIo->flags & IsDtvArgPolymorphic) {
+                isArgDescriptorSet = 1;
+              }
               typeInfo::SpecialBinding special{DIR == Direction::Input
                       ? typeInfo::SpecialBinding::Which::ReadUnformatted
                       : typeInfo::SpecialBinding::Which::WriteUnformatted,
-                  definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                  false};
+                  definedIo->subroutine, isArgDescriptorSet,
+                  /*IsTypeBound=*/false,
+                  /*specialCaseFlag=*/!!(definedIo->flags & DefinedIoInteger8)};
               if (DefinedUnformattedIo(io_, instance_, *type, special)) {
                 anyIoTookPlace_ = true;
                 return StatOk;
@@ -719,8 +762,11 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
             nonTbpSpecial_.emplace(DIR == Direction::Input
                     ? typeInfo::SpecialBinding::Which::ReadFormatted
                     : typeInfo::SpecialBinding::Which::WriteFormatted,
-                definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                false);
+                definedIo->subroutine,
+                /*isArgDescriptorSet=*/
+                (definedIo->flags & IsDtvArgPolymorphic) ? 1 : 0,
+                /*isTypeBound=*/false,
+                /*specialCaseFlag=*/!!(definedIo->flags & DefinedIoInteger8));
             special_ = &*nonTbpSpecial_;
           }
         }

diff  --git a/flang-rt/lib/runtime/non-tbp-dio.cpp b/flang-rt/lib/runtime/non-tbp-dio.cpp
index 72101b06e0c6e..d516526033c27 100644
--- a/flang-rt/lib/runtime/non-tbp-dio.cpp
+++ b/flang-rt/lib/runtime/non-tbp-dio.cpp
@@ -17,7 +17,7 @@ const NonTbpDefinedIo *NonTbpDefinedIoTable::Find(
   for (const auto *p{item}; j-- > 0; ++p) {
     if (&p->derivedType == &type && p->definedIo == definedIo) {
       return p;
-    } else if (p->isDtvArgPolymorphic) {
+    } else if (p->flags & IsDtvArgPolymorphic) {
       for (const typeInfo::DerivedType *t{type.GetParentType()}; t;
            t = t->GetParentType()) {
         if (&p->derivedType == t && p->definedIo == definedIo) {

diff  --git a/flang-rt/lib/runtime/type-info.cpp b/flang-rt/lib/runtime/type-info.cpp
index 3e1d7c9c3c788..50123f4cf321c 100644
--- a/flang-rt/lib/runtime/type-info.cpp
+++ b/flang-rt/lib/runtime/type-info.cpp
@@ -330,7 +330,7 @@ FILE *SpecialBinding::Dump(FILE *f) const {
   }
   std::fprintf(f, "    isArgDescriptorSet: 0x%x\n", isArgDescriptorSet_);
   std::fprintf(f, "    isTypeBound: %d\n", isTypeBound_);
-  std::fprintf(f, "    isArgContiguousSet: 0x%x\n", isArgContiguousSet_);
+  std::fprintf(f, "    specialCaseFlag 0x%x\n", specialCaseFlag_);
   std::fprintf(f, "    proc: %p\n", reinterpret_cast<void *>(proc_));
   return f;
 }

diff  --git a/flang/include/flang/Semantics/runtime-type-info.h b/flang/include/flang/Semantics/runtime-type-info.h
index 6c5a061d1c1a2..94e8293b14643 100644
--- a/flang/include/flang/Semantics/runtime-type-info.h
+++ b/flang/include/flang/Semantics/runtime-type-info.h
@@ -52,10 +52,15 @@ constexpr char procCompName[]{"proc"};
 
 SymbolVector CollectBindings(const Scope &dtScope);
 
+enum NonTbpDefinedIoFlags {
+  IsDtvArgPolymorphic = 1 << 0,
+  DefinedIoInteger8 = 1 << 1,
+};
+
 struct NonTbpDefinedIo {
   const Symbol *subroutine;
   common::DefinedIo definedIo;
-  bool isDtvArgPolymorphic;
+  std::uint8_t flags;
 };
 
 std::multimap<const Symbol *, NonTbpDefinedIo>

diff  --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 63a612d7ead61..69d72d9d63b68 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -269,10 +269,12 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   mlir::Type sizeTy =
       fir::runtime::getModel<std::size_t>()(builder.getContext());
   mlir::Type intTy = fir::runtime::getModel<int>()(builder.getContext());
+  mlir::Type byteTy =
+      fir::runtime::getModel<std::uint8_t>()(builder.getContext());
   mlir::Type boolTy = fir::runtime::getModel<bool>()(builder.getContext());
   mlir::Type listTy = fir::SequenceType::get(
       definedIoProcMap.size(),
-      mlir::TupleType::get(context, {refTy, refTy, intTy, boolTy}));
+      mlir::TupleType::get(context, {refTy, refTy, intTy, byteTy}));
   mlir::Type tableTy = mlir::TupleType::get(
       context, {sizeTy, fir::ReferenceType::get(listTy), boolTy});
 
@@ -339,9 +341,9 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       insert(builder.createIntegerConstant(
           loc, intTy, static_cast<int>(iface.second.definedIo)));
       // polymorphic flag is set if first defined IO dummy arg is CLASS(T)
+      // defaultInt8 flag is set if -fdefined-integer-8
       // [bool isDtvArgPolymorphic]
-      insert(builder.createIntegerConstant(loc, boolTy,
-                                           iface.second.isDtvArgPolymorphic));
+      insert(builder.createIntegerConstant(loc, byteTy, iface.second.flags));
     }
     if (tableIsLocal)
       builder.create<fir::StoreOp>(loc, list, listAddr);

diff  --git a/flang/lib/Semantics/runtime-type-info.cpp b/flang/lib/Semantics/runtime-type-info.cpp
index 51ba21a9e5edf..5916a07df7744 100644
--- a/flang/lib/Semantics/runtime-type-info.cpp
+++ b/flang/lib/Semantics/runtime-type-info.cpp
@@ -1131,7 +1131,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
   if (auto proc{evaluate::characteristics::Procedure::Characterize(
           specific, context_.foldingContext())}) {
     std::uint8_t isArgDescriptorSet{0};
-    std::uint8_t isArgContiguousSet{0};
+    bool specialCaseFlag{0};
     int argThatMightBeDescriptor{0};
     MaybeExpr which;
     if (isAssignment) {
@@ -1197,7 +1197,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
                         TypeAndShape::Attr::AssumedShape) ||
                 dummyData.attrs.test(evaluate::characteristics::
                         DummyDataObject::Attr::Contiguous)) {
-              isArgContiguousSet |= 1;
+              specialCaseFlag = true;
             }
           }
         }
@@ -1216,7 +1216,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
         return;
       }
       if (ddo->type.type().IsPolymorphic()) {
-        isArgDescriptorSet |= 1;
+        argThatMightBeDescriptor = 1;
       }
       switch (io.value()) {
       case common::DefinedIo::ReadFormatted:
@@ -1232,6 +1232,9 @@ void RuntimeTableBuilder::DescribeSpecialProc(
         which = writeUnformattedEnum_;
         break;
       }
+      if (context_.defaultKinds().GetDefaultKind(TypeCategory::Integer) == 8) {
+        specialCaseFlag = true; // UNIT= & IOSTAT= INTEGER(8)
+      }
     }
     if (argThatMightBeDescriptor != 0) {
       if (const auto *dummyData{
@@ -1262,8 +1265,8 @@ void RuntimeTableBuilder::DescribeSpecialProc(
     }
     CHECK(bindingIndex <= 255);
     AddValue(values, specialSchema_, "istypebound"s, IntExpr<1>(bindingIndex));
-    AddValue(values, specialSchema_, "isargcontiguousset"s,
-        IntExpr<1>(isArgContiguousSet));
+    AddValue(values, specialSchema_, "specialcaseflag"s,
+        IntExpr<1>(specialCaseFlag));
     AddValue(values, specialSchema_, procCompName,
         SomeExpr{evaluate::ProcedureDesignator{specific}});
     // index might already be present in the case of an override
@@ -1383,19 +1386,26 @@ CollectNonTbpDefinedIoGenericInterfaces(
               } else {
                 // Local scope's specific overrides host's for this type
                 bool updated{false};
+                std::uint8_t flags{0};
+                if (declType->IsPolymorphic()) {
+                  flags |= IsDtvArgPolymorphic;
+                }
+                if (scope.context().GetDefaultKind(TypeCategory::Integer) ==
+                    8) {
+                  flags |= DefinedIoInteger8;
+                }
                 for (auto [iter, end]{result.equal_range(dtDesc)}; iter != end;
                      ++iter) {
                   NonTbpDefinedIo &nonTbp{iter->second};
                   if (nonTbp.definedIo == which) {
                     nonTbp.subroutine = &*specific;
-                    nonTbp.isDtvArgPolymorphic = declType->IsPolymorphic();
+                    nonTbp.flags = flags;
                     updated = true;
                   }
                 }
                 if (!updated) {
-                  result.emplace(dtDesc,
-                      NonTbpDefinedIo{
-                          &*specific, which, declType->IsPolymorphic()});
+                  result.emplace(
+                      dtDesc, NonTbpDefinedIo{&*specific, which, flags});
                 }
               }
             }

diff  --git a/flang/module/__fortran_type_info.f90 b/flang/module/__fortran_type_info.f90
index 8dd27d6e4c01b..6af2a5a5e30ff 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang/module/__fortran_type_info.f90
@@ -118,7 +118,7 @@
     integer(1) :: which ! SpecialBinding::Which
     integer(1) :: isArgDescriptorSet
     integer(1) :: isTypeBound ! binding index + 1, if any
-    integer(1) :: isArgContiguousSet
+    integer(1) :: specialCaseFlag
     integer(1) :: __padding0(4)
     type(__builtin_c_funptr) :: proc
   end type

diff  --git a/flang/test/Lower/io-derived-type.f90 b/flang/test/Lower/io-derived-type.f90
index 7d2fef3faa2b7..7c289ce261678 100644
--- a/flang/test/Lower/io-derived-type.f90
+++ b/flang/test/Lower/io-derived-type.f90
@@ -37,16 +37,16 @@ subroutine test1
     import, all
     ! CHECK:   %[[V_16:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:   %[[V_17:[0-9]+]] = fir.convert %[[V_16]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:   %[[V_18:[0-9]+]] = fir.address_of(@_QQMmFtest1.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:   %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:   %[[V_18:[0-9]+]] = fir.address_of(@_QQMmFtest1.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:   %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:   %[[V_20:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_17]], %[[V_19]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
     print *, 'test1 outer, should call wft: ', t(1)
     block
       import, only: t
       ! CHECK:   %[[V_37:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
       ! CHECK:   %[[V_38:[0-9]+]] = fir.convert %[[V_37]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-      ! CHECK:   %[[V_39:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-      ! CHECK:   %[[V_40:[0-9]+]] = fir.convert %[[V_39]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+      ! CHECK:   %[[V_39:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+      ! CHECK:   %[[V_40:[0-9]+]] = fir.convert %[[V_39]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
       ! CHECK:   %[[V_41:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_38]], %[[V_40]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
       print *, 'test1 block, should not call wft: ', t(2)
     end block
@@ -56,8 +56,8 @@ subroutine test1
   subroutine test2
     ! CHECK:   %[[V_15:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:   %[[V_16:[0-9]+]] = fir.convert %[[V_15]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:   %[[V_17:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:   %[[V_18:[0-9]+]] = fir.convert %[[V_17]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:   %[[V_17:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:   %[[V_18:[0-9]+]] = fir.convert %[[V_17]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:   %[[V_19:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_16]], %[[V_18]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
 
     import, only: t
@@ -74,23 +74,23 @@ subroutine test3(p, x)
 
     ! CHECK:     %[[V_3:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
     ! CHECK:     %[[V_4:[0-9]+]] = fir.convert %[[V_3]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-    ! CHECK:     %[[V_5:[0-9]+]] = fir.alloca !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_6:[0-9]+]] = fir.undefined !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
+    ! CHECK:     %[[V_5:[0-9]+]] = fir.alloca !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_6:[0-9]+]] = fir.undefined !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
     ! CHECK:     %[[V_7:[0-9]+]] = fir.address_of(@_QMmE.dt.t)
     ! CHECK:     %[[V_8:[0-9]+]] = fir.convert %[[V_7]] : {{.*}} -> !fir.ref<none>
-    ! CHECK:     %[[V_9:[0-9]+]] = fir.insert_value %[[V_6]], %[[V_8]], [0 : index, 0 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
+    ! CHECK:     %[[V_9:[0-9]+]] = fir.insert_value %[[V_6]], %[[V_8]], [0 : index, 0 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
     ! CHECK:     %[[V_10:[0-9]+]] = fir.box_addr %arg0 : (!fir.boxproc<() -> ()>) -> !fir.ref<none>
-    ! CHECK:     %[[V_11:[0-9]+]] = fir.insert_value %[[V_9]], %[[V_10]], [0 : index, 1 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_12:[0-9]+]] = fir.insert_value %[[V_11]], %c2{{.*}}, [0 : index, 2 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, i32) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     %[[V_13:[0-9]+]] = fir.insert_value %[[V_12]], %true, [0 : index, 3 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>, i1) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-    ! CHECK:     fir.store %[[V_13]] to %[[V_5]] : !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>
-    ! CHECK:     %[[V_14:[0-9]+]] = fir.alloca tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_15:[0-9]+]] = fir.undefined tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_16:[0-9]+]] = fir.insert_value %[[V_15]], %c1{{.*}}, [0 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, i64) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_17:[0-9]+]] = fir.insert_value %[[V_16]], %[[V_5]], [1 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     %[[V_18:[0-9]+]] = fir.insert_value %[[V_17]], %true_0, [2 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>, i1) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-    ! CHECK:     fir.store %[[V_18]] to %[[V_14]] : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-    ! CHECK:     %[[V_19:[0-9]+]] = fir.convert %[[V_14]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+    ! CHECK:     %[[V_11:[0-9]+]] = fir.insert_value %[[V_9]], %[[V_10]], [0 : index, 1 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, !fir.ref<none>) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_12:[0-9]+]] = fir.insert_value %[[V_11]], %c2{{.*}}, [0 : index, 2 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, i32) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     %[[V_13:[0-9]+]] = fir.insert_value %[[V_12]], %c1_i8, [0 : index, 3 : index] : (!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>, i8) -> !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+    ! CHECK:     fir.store %[[V_13]] to %[[V_5]] : !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>
+    ! CHECK:     %[[V_14:[0-9]+]] = fir.alloca tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_15:[0-9]+]] = fir.undefined tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_16:[0-9]+]] = fir.insert_value %[[V_15]], %c1{{.*}}, [0 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, i64) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_17:[0-9]+]] = fir.insert_value %[[V_16]], %[[V_5]], [1 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     %[[V_18:[0-9]+]] = fir.insert_value %[[V_17]], %true, [2 : index] : (tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>, i1) -> tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+    ! CHECK:     fir.store %[[V_18]] to %[[V_14]] : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+    ! CHECK:     %[[V_19:[0-9]+]] = fir.convert %[[V_14]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
     ! CHECK:     %[[V_20:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_4]], %[[V_19]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
     print *, x
   end subroutine
@@ -112,8 +112,8 @@ program p
 
   ! CHECK:   %[[V_97:[0-9]+]] = fir.embox %{{.*}} : (!fir.ref<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<!fir.type<_QMmTt{n:i32}>>
   ! CHECK:   %[[V_98:[0-9]+]] = fir.convert %[[V_97]] : (!fir.box<!fir.type<_QMmTt{n:i32}>>) -> !fir.box<none>
-  ! CHECK:   %[[V_99:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:   %[[V_100:[0-9]+]] = fir.convert %[[V_99]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:   %[[V_99:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:   %[[V_100:[0-9]+]] = fir.convert %[[V_99]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:   %[[V_101:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_98]], %[[V_100]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
   print *, 'main, should call wft: ', t(4)
 
@@ -122,14 +122,14 @@ program p
   ! CHECK:   %[[V_35:[0-9]+]] = fir.shape %c2{{.*}} : (index) -> !fir.shape<1>
   ! CHECK:   %[[V_36:[0-9]+]] = fir.embox %[[V_34]](%[[V_35]]) : (!fir.ref<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>
   ! CHECK:   %[[V_37:[0-9]+]] = fir.convert %[[V_36]] : (!fir.box<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>) -> !fir.box<none>
-  ! CHECK:   %[[V_38:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:   %[[V_39:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:   %[[V_38:[0-9]+]] = fir.address_of(@_QQF.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:   %[[V_39:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:   %[[V_40:[0-9]+]] = fir.call @_FortranAioOutputDerivedType(%{{.*}}, %[[V_37]], %[[V_39]]) fastmath<contract> : (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
   print *, y(2:3)
 end
 
-! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-! CHECK: fir.global linkonce @_QQdefault.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
-! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>
-! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>
+! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+! CHECK: fir.global linkonce @_QQMmFtest1.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+! CHECK: fir.global linkonce @_QQdefault.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>
+! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable.list constant : !fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>
+! CHECK: fir.global linkonce @_QQF.nonTbpDefinedIoTable constant : tuple<i64, !fir.ref<!fir.array<1xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>

diff  --git a/flang/test/Lower/namelist.f90 b/flang/test/Lower/namelist.f90
index 94b0ef11cb102..770af46eea744 100644
--- a/flang/test/Lower/namelist.f90
+++ b/flang/test/Lower/namelist.f90
@@ -42,8 +42,8 @@ program p
   ! CHECK:     %[[V_42:[0-9]+]] = fir.insert_value %[[V_39]], %[[V_41]], [0 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<i8>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_43:[0-9]+]] = fir.insert_value %[[V_42]], %c2{{.*}}, [1 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, i64) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_44:[0-9]+]] = fir.insert_value %[[V_43]], %[[V_24]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
-  ! CHECK:     %[[V_45:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:     %[[V_46:[0-9]+]] = fir.convert %[[V_45]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:     %[[V_45:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:     %[[V_46:[0-9]+]] = fir.convert %[[V_45]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:     %[[V_47:[0-9]+]] = fir.insert_value %[[V_44]], %[[V_46]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     fir.store %[[V_47]] to %[[V_38]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
   ! CHECK:     %[[V_48:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
@@ -100,8 +100,8 @@ subroutine sss
   ! CHECK:     %[[V_20:[0-9]+]] = fir.insert_value %[[V_17]], %[[V_19]], [0 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<i8>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_21:[0-9]+]] = fir.insert_value %[[V_20]], %c1{{.*}}, [1 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, i64) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     %[[V_22:[0-9]+]] = fir.insert_value %[[V_21]], %[[V_8]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
-  ! CHECK:     %[[V_23:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
-  ! CHECK:     %[[V_24:[0-9]+]] = fir.convert %[[V_23]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:     %[[V_23:[0-9]+]] = fir.address_of(@_QQdefault.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>
+  ! CHECK:     %[[V_24:[0-9]+]] = fir.convert %[[V_23]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i8>>>, i1>>) -> !fir.ref<none>
   ! CHECK:     %[[V_25:[0-9]+]] = fir.insert_value %[[V_22]], %[[V_24]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
   ! CHECK:     fir.store %[[V_25]] to %[[V_16]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
   ! CHECK:     %[[V_26:[0-9]+]] = fir.convert %[[V_16]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>

diff  --git a/flang/test/Lower/volatile-openmp.f90 b/flang/test/Lower/volatile-openmp.f90
index 2e05b652822b5..d1a844eddd106 100644
--- a/flang/test/Lower/volatile-openmp.f90
+++ b/flang/test/Lower/volatile-openmp.f90
@@ -23,11 +23,11 @@
 ! CHECK:           %[[VAL_11:.*]] = fir.address_of(@_QFEcontainer) : !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>
 ! CHECK:           %[[VAL_12:.*]] = fir.volatile_cast %[[VAL_11]] : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>
 ! CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFEcontainer"} : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>) -> (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>, !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>)
-! CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QFE.c.t) : !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>
+! CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QFE.c.t) : !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>
 ! CHECK:           %[[VAL_15:.*]] = fir.shape_shift %[[VAL_0]], %[[VAL_1]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]](%[[VAL_15]]) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.c.t"} : (!fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>)
-! CHECK:           %[[VAL_17:.*]] = fir.address_of(@_QFE.dt.t) : !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>
-! CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.dt.t"} : (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>) -> (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>, !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>)
+! CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]](%[[VAL_15]]) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.c.t"} : (!fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>, !fir.ref<!fir.array<1x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>)
+! CHECK:           %[[VAL_17:.*]] = fir.address_of(@_QFE.dt.t) : !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>
+! CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.dt.t"} : (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>) -> (!fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>, !fir.ref<!fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{{[<]?}}{genre:i8,__padding0:!fir.array<7xi8>,value:i64}{{[>]?}}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{{[<]?}}{which:i8,isargdescriptorset:i8,istypebound:i8,specialcaseflag:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}{{[>]?}}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>>)
 ! CHECK:           %[[VAL_19:.*]] = hlfir.designate %[[VAL_13]]#0{"array"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>, volatile>
 ! CHECK:           %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>, volatile>
 ! CHECK:           %[[VAL_21:.*]]:3 = fir.box_dims %[[VAL_20]], %[[VAL_0]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)

diff  --git a/flang/test/Semantics/typeinfo01.f90 b/flang/test/Semantics/typeinfo01.f90
index bb20c546e0261..aeec336ea58ea 100644
--- a/flang/test/Semantics/typeinfo01.f90
+++ b/flang/test/Semantics/typeinfo01.f90
@@ -87,8 +87,8 @@ subroutine s2(x, y)
 !CHECK: .c.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=6_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=2_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
 !CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=.s.t2,specialbitset=2_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
-!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s2)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
+!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s2)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 !CHECK: .v.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
 end module
@@ -115,8 +115,8 @@ subroutine s2(x, y)
 !CHECK: .c.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=6_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=2_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
 !CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=.s.t2,specialbitset=2_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
-!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s2)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
+!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s2)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 !CHECK: .v.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
 end module
@@ -133,7 +133,7 @@ impure elemental subroutine s1(x, y)
     class(t), intent(in) :: y
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=4_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=0_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=s1)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 end module
 
@@ -156,7 +156,7 @@ subroutine s4(x)
     type(t), contiguous :: x(:,:,:)
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=7296_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=0_1,nofinalizationneeded=0_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=7_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=s3),specialbinding(which=10_1,isargdescriptorset=1_1,istypebound=0_1,isargcontiguousset=0_1,proc=s1),specialbinding(which=11_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=1_1,proc=s2),specialbinding(which=12_1,isargdescriptorset=1_1,istypebound=0_1,isargcontiguousset=1_1,proc=s4)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=7_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=s3),specialbinding(which=10_1,isargdescriptorset=1_1,istypebound=0_1,specialcaseflag=0_1,proc=s1),specialbinding(which=11_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=1_1,proc=s2),specialbinding(which=12_1,isargdescriptorset=1_1,istypebound=0_1,specialcaseflag=1_1,proc=s4)]
 end module
 
 module m09
@@ -198,7 +198,7 @@ subroutine wu(x,u,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=120_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=1_1,istypebound=2_1,isargcontiguousset=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=3_1,isargcontiguousset=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=1_1,istypebound=4_1,isargcontiguousset=0_1,proc=wu)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=1_1,istypebound=2_1,specialcaseflag=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=3_1,specialcaseflag=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=1_1,istypebound=4_1,specialcaseflag=0_1,proc=wu)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:3_8 init:[binding::binding(proc=rf,name=.n.rf),binding(proc=ru,name=.n.ru),binding(proc=wf,name=.n.wf),binding(proc=wu,name=.n.wu)]
 end module
 
@@ -247,7 +247,7 @@ subroutine wu(x,u,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=120_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1,nodefinedassignment=1_1)
-!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=0_1,istypebound=0_1,isargcontiguousset=0_1,proc=wu)]
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=3_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=rf),specialbinding(which=4_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=ru),specialbinding(which=5_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=wf),specialbinding(which=6_1,isargdescriptorset=0_1,istypebound=0_1,specialcaseflag=0_1,proc=wu)]
 end module
 
 module m11
@@ -290,7 +290,7 @@ module m13
    contains
     procedure :: assign1, assign2
     generic :: assignment(=) => assign1, assign2
-    ! CHECK: .s.t1, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=assign1)]
+    ! CHECK: .s.t1, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=3_1,istypebound=1_1,specialcaseflag=0_1,proc=assign1)]
   end type
  contains
   impure elemental subroutine assign1(to, from)

diff  --git a/flang/test/Semantics/typeinfo02.f90 b/flang/test/Semantics/typeinfo02.f90
index 29d14c7a0f196..07293627ab492 100644
--- a/flang/test/Semantics/typeinfo02.f90
+++ b/flang/test/Semantics/typeinfo02.f90
@@ -29,5 +29,5 @@ subroutine wf2(x,u,iot,v,iostat,iomsg)
     character(len=*), intent(inout) :: iomsg
   end subroutine
 end module
-!CHECK: .s.base, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=wf1)]
-!CHECK: .s.extended, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=wf2)]
+!CHECK: .s.base, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=wf1)]
+!CHECK: .s.extended, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=wf2)]

diff  --git a/flang/test/Semantics/typeinfo09.f90 b/flang/test/Semantics/typeinfo09.f90
index 3527ee6058ad8..8daa6a5f420d7 100644
--- a/flang/test/Semantics/typeinfo09.f90
+++ b/flang/test/Semantics/typeinfo09.f90
@@ -17,4 +17,4 @@ subroutine copy_impl(this, x)
 end interface
 end module
 
-!CHECK: .s.sometype, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=copy_impl)]
+!CHECK: .s.sometype, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=1_1,istypebound=1_1,specialcaseflag=0_1,proc=copy_impl)]

diff  --git a/flang/test/Semantics/typeinfo13.f90 b/flang/test/Semantics/typeinfo13.f90
index ad824ad3590a2..facc280815722 100644
--- a/flang/test/Semantics/typeinfo13.f90
+++ b/flang/test/Semantics/typeinfo13.f90
@@ -22,5 +22,5 @@ impure elemental subroutine override(to, from)
   end
 end
 
-!CHECK: .s.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=1_1,istypebound=2_1,isargcontiguousset=0_1,proc=override)]
+!CHECK: .s.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=1_1,istypebound=2_1,specialcaseflag=0_1,proc=override)]
 !CHECK: .v.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:1_8 init:[binding::binding(proc=baseassign,name=.n.baseassign),binding(proc=override,name=.n.override)]


        

From flang-commits at lists.llvm.org  Wed Jul 16 09:25:34 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:25:34 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6877d27e.050a0220.31cc34.4760@mx.google.com>


================
@@ -1887,4 +1887,68 @@ def MaskedOp : OpenMP_Op<"masked", clauses = [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Allocates memory on the specified OpenMP device for an object of the given type.
+    Returns an integer value representing the device pointer to the allocated memory.
+    The memory is uninitialized after allocation. Operations must be paired with 
+    `omp.target_freemem` to avoid memory leaks.
+
+    ```mlir
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyInteger>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<IntLikeType>:$typeparams,
+    Variadic<IntLikeType>:$shape
----------------
skc7 wrote:

Reference for this op is fir.allocmem and when fir.allocmem is hoisted out of omp.target region, it will be replaced by omp.target_allocmem in the workdistribute lowering pass in #140523

Have used the similar arguments for this op like in fir.allocmem. 
$shape is required as it provide the runtime extents of dynamic dimensions when allocating Fortran arrays.

Updated the description in the latest patch.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul 16 09:25:59 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:25:59 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6877d297.050a0220.16280b.6f17@mx.google.com>


================
@@ -224,6 +226,67 @@ std::optional<llvm::ArrayRef<int64_t>> getComponentLowerBoundsIfNonDefault(
     fir::RecordType recordType, llvm::StringRef component,
     mlir::ModuleOp module, const mlir::SymbolTable *symbolTable = nullptr);
 
+// Convert FIR type to LLVM without turning fir.box<T> into memory
+// reference.
+mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                             mlir::Type firType);
+
+/// Generate a LLVM constant value of type `ity`, using the provided offset.
+mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset);
+
+/// Helper function for generating the LLVM IR that computes the distance
+/// in bytes between adjacent elements pointed to by a pointer
+/// of type \p ptrTy. The result is returned as a value of \p idxTy integer
+/// type.
+mlir::Value computeElementDistance(mlir::Location loc,
+                                   mlir::Type llvmObjectType, mlir::Type idxTy,
+                                   mlir::ConversionPatternRewriter &rewriter,
+                                   const mlir::DataLayout &dataLayout);
+
+// Compute the alloc scale size (constant factors encoded in the array type).
+// We do this for arrays without a constant interior or arrays of character with
+// dynamic length arrays, since those are the only ones that get decayed to a
+// pointer to the element type.
+template <typename OP>
+inline mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
----------------
skc7 wrote:

Thanks for feedback. Have updated this in latest patch.

https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Wed Jul 16 09:29:14 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Wed, 16 Jul 2025 09:29:14 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
Message-ID: <llvm.org/llvm/llvm-project/pull/149134@github.com>


https://github.com/kparzysz created https://github.com/llvm/llvm-project/pull/149134

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers. The old "map-type-modifier" is retained, but is no longer created in parsing. It will remain to take advantage of the preexisting modifier validation for older versions: when the OpenMP version is < 6.0, the modifiers will be rewritten back as map-type-modifiers (or map- type in case of "delete").

In this patch the modifiers will always be rewritten in the older format to isolate these changes to parsing as much as possible.

>From 0929234481b4d6d015381fda490c05bbeff1e22f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 08:04:17 -0500
Subject: [PATCH] [flang][OpenMP] Parse OpenMP 6.0 map modifiers

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers.
The old "map-type-modifier" is retained, but is no longer created in
parsing. It will remain to take advantage of the preexisting modifier
validation for older versions: when the OpenMP version is < 6.0,
the modifiers will be rewritten back as map-type-modifiers (or map-
type in case of "delete").

In this patch the modifiers will always be rewritten in the older
format to isolate these changes to parsing as much as possible.
---
 flang/include/flang/Parser/dump-parse-tree.h  |  14 ++
 flang/include/flang/Parser/parse-tree.h       | 138 +++++++++++++++---
 .../flang/Semantics/openmp-modifiers.h        |   7 +
 flang/lib/Parser/openmp-parsers.cpp           |  53 +++++--
 flang/lib/Parser/unparse.cpp                  |   7 +
 flang/lib/Semantics/canonicalize-omp.cpp      |  48 +++++-
 flang/lib/Semantics/canonicalize-omp.h        |   9 +-
 flang/lib/Semantics/openmp-modifiers.cpp      | 115 +++++++++++++++
 flang/lib/Semantics/resolve-directives.cpp    |   2 +
 flang/lib/Semantics/semantics.cpp             |   3 +-
 .../test/Parser/OpenMP/map-modifiers-v60.f90  | 113 ++++++++++++++
 11 files changed, 466 insertions(+), 43 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/map-modifiers-v60.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 32b6ca45609b6..23e35d106c077 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -529,6 +529,8 @@ class ParseTreeDumper {
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlwaysModifier)
+  NODE_ENUM(OmpAlwaysModifier, Value)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
@@ -546,6 +548,8 @@ class ParseTreeDumper {
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
   NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpCloseModifier)
+  NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -561,6 +565,8 @@ class ParseTreeDumper {
   NODE(parser, OmpDefaultmapClause)
   NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE(parser, OmpDeleteModifier)
+  NODE_ENUM(OmpDeleteModifier, Value)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
   NODE(parser, OmpTaskDependenceType)
@@ -628,6 +634,8 @@ class ParseTreeDumper {
   NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
   NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpPresentModifier)
+  NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
@@ -637,6 +645,10 @@ class ParseTreeDumper {
   NODE(parser, OmpInReductionClause)
   NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpRefModifier)
+  NODE_ENUM(OmpRefModifier, Value)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpInitializerProc)
@@ -673,6 +685,8 @@ class ParseTreeDumper {
   NODE(parser, OmpSectionsDirective)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpxHoldModifier)
+  NODE_ENUM(OmpxHoldModifier, Value)
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cc1d032f94d4a..92a449d2f9d49 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3756,6 +3756,19 @@ struct OmpAllocatorComplexModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// always-modifier ->
+//    ALWAYS                                        // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpAlwaysModifier {
+  ENUM_CLASS(Value, Always)
+  WRAPPER_CLASS_BOILERPLATE(OmpAlwaysModifier, Value);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3767,17 +3780,29 @@ struct OmpChunkModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpChunkModifier, Value);
 };
 
-// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
-// iterator-specifier ->
-//    [iterator-type] iterator-identifier
-//        = range-specification |                   // since 5.0
-//    [iterator-type ::] iterator-identifier
-//        = range-specification                     // since 5.2
-struct OmpIteratorSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
-  CharBlock source;
-  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+// close-modifier ->
+//    CLOSE                                         // since 5.0
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpCloseModifier {
+  ENUM_CLASS(Value, Close)
+  WRAPPER_CLASS_BOILERPLATE(OmpCloseModifier, Value);
+};
+
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// delete-modifier ->
+//    DELETE                                        // since 6.0
+//
+// Until 5.2, it was a part of map-type.
+struct OmpDeleteModifier {
+  ENUM_CLASS(Value, Delete)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeleteModifier, Value);
 };
 
 // Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289]
@@ -3867,6 +3892,19 @@ struct OmpInteropType {
   WRAPPER_CLASS_BOILERPLATE(OmpInteropType, Value);
 };
 
+// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+//
+// iterator-specifier ->
+//    [iterator-type] iterator-identifier
+//        = range-specification |                   // since 5.0
+//    [iterator-type ::] iterator-identifier
+//        = range-specification                     // since 5.2
+struct OmpIteratorSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
+  CharBlock source;
+  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+};
+
 // Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
 //
 // iterator-modifier ->
@@ -3901,21 +3939,28 @@ struct OmpMapper {
   WRAPPER_CLASS_BOILERPLATE(OmpMapper, Name);
 };
 
-// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
 // map-type ->
-//    ALLOC | DELETE | FROM | RELEASE | TO | TOFROM // since 4.5
+//    ALLOC | DELETE | RELEASE |                    // since 4.5, until 5.2
+//    FROM | TO | TOFROM |                          // since 4.5
+//    STORAGE                                       // since 6.0
+//
+// Since 6.0 DELETE is a separate delete-modifier.
 struct OmpMapType {
-  ENUM_CLASS(Value, Alloc, Delete, From, Release, To, Tofrom);
+  ENUM_CLASS(Value, Alloc, Delete, From, Release, Storage, To, Tofrom);
   WRAPPER_CLASS_BOILERPLATE(OmpMapType, Value);
 };
 
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-type-modifier ->
-//    ALWAYS |                                      // since 4.5
-//    CLOSE |                                       // since 5.0
-//    PRESENT                                       // since 5.1
+//    ALWAYS |                                      // since 4.5, until 5.2
+//    CLOSE |                                       // since 5.0, until 5.2
+//    PRESENT                                       // since 5.1, until 5.2
+// Since 6.0 the map-type-modifier has been split into individual modifiers.
+//
 struct OmpMapTypeModifier {
   ENUM_CLASS(Value, Always, Close, Present, Ompx_Hold)
   WRAPPER_CLASS_BOILERPLATE(OmpMapTypeModifier, Value);
@@ -3954,6 +3999,19 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// present-modifier ->
+//    PRESENT                                       // since 5.1
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpPresentModifier {
+  ENUM_CLASS(Value, Present)
+  WRAPPER_CLASS_BOILERPLATE(OmpPresentModifier, Value);
+};
+
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3963,6 +4021,26 @@ struct OmpReductionModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpReductionModifier, Value);
 };
 
+// Ref: [6.0:279-288]
+//
+// ref-modifier ->
+//    REF_PTEE | REF_PTR | REF_PTR_PTEE             // since 6.0
+//
+struct OmpRefModifier {
+  ENUM_CLASS(Value, Ref_Ptee, Ref_Ptr, Ref_Ptr_Ptee)
+  WRAPPER_CLASS_BOILERPLATE(OmpRefModifier, Value);
+};
+
+// Ref: [6.0:279-288]
+//
+// self-modifier ->
+//    SELF                                          // since 6.0
+//
+struct OmpSelfModifier {
+  ENUM_CLASS(Value, Self)
+  WRAPPER_CLASS_BOILERPLATE(OmpSelfModifier, Value);
+};
+
 // Ref: [5.2:117-120]
 //
 // step-complex-modifier ->
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
+//
+// ompx-hold-modifier ->
+//    OMPX_HOLD                                     // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpxHoldModifier {
+  ENUM_CLASS(Value, Ompx_Hold)
+  WRAPPER_CLASS_BOILERPLATE(OmpxHoldModifier, Value);
+};
+
 // context-selector
 using OmpContextSelector = traits::OmpContextSelectorSpecification;
 } // namespace modifier
@@ -4376,13 +4466,25 @@ struct OmpLinearClause {
 // map-clause ->
 //    MAP([modifier...:] locator-list)              // since 4.5
 // modifier ->
-//    map-type-modifier |                           // since 4.5
+//    map-type-modifier [replaced] |                // since 4.5, until 5.2
+//    always-modifier |                             // since 6.0
+//    close-modifier |                              // since 6.0
+//    delete-modifier |                             // since 6.0
+//    present-modifier |                            // since 6.0
+//    ref-modifier |                                // since 6.0
+//    self-modifier |                               // since 6.0
 //    mapper |                                      // since 5.0
 //    iterator |                                    // since 5.1
 //    map-type                                      // since 4.5
+//    ompx-hold-modifier |                          // since 6.0
+//
+// Since 6.0 the map-type-modifier has been split into individual modifiers,
+// and delete-modifier has been split from map-type.
 struct OmpMapClause {
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  MODIFIER_BOILERPLATE(OmpMapTypeModifier, OmpMapper, OmpIterator, OmpMapType);
+  MODIFIER_BOILERPLATE(OmpAlwaysModifier, OmpCloseModifier, OmpDeleteModifier,
+      OmpMapTypeModifier, OmpPresentModifier, OmpRefModifier, OmpSelfModifier,
+      OmpMapper, OmpIterator, OmpMapType, OmpxHoldModifier);
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 4fbd80f989e72..a9fe911ef8807 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -71,8 +71,11 @@ DECLARE_DESCRIPTOR(parser::OmpAlignment);
 DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
+DECLARE_DESCRIPTOR(parser::OmpAlwaysModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpCloseModifier);
 DECLARE_DESCRIPTOR(parser::OmpContextSelector);
+DECLARE_DESCRIPTOR(parser::OmpDeleteModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
@@ -88,12 +91,16 @@ DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
 DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
+DECLARE_DESCRIPTOR(parser::OmpPresentModifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
+DECLARE_DESCRIPTOR(parser::OmpRefModifier);
+DECLARE_DESCRIPTOR(parser::OmpSelfModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpVariableCategory);
+DECLARE_DESCRIPTOR(parser::OmpxHoldModifier);
 
 #undef DECLARE_DESCRIPTOR
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..d349d8ceb0bb5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -274,6 +274,10 @@ TYPE_PARSER( //
     construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}) ||
     construct<OmpTypeSpecifier>(Parser<TypeSpec>{}))
 
+// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
+TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
+    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
+
 TYPE_PARSER(construct<OmpReductionSpecifier>( //
     Parser<OmpReductionIdentifier>{},
     ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
@@ -442,9 +446,18 @@ TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
 
 TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
 
+TYPE_PARSER(construct<OmpAlwaysModifier>( //
+    "ALWAYS" >> pure(OmpAlwaysModifier::Value::Always)))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpCloseModifier>( //
+    "CLOSE" >> pure(OmpCloseModifier::Value::Close)))
+
+TYPE_PARSER(construct<OmpDeleteModifier>( //
+    "DELETE" >> pure(OmpDeleteModifier::Value::Delete)))
+
 TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
@@ -502,26 +515,16 @@ TYPE_PARSER(construct<OmpLinearModifier>( //
 TYPE_PARSER(construct<OmpMapper>( //
     "MAPPER"_tok >> parenthesized(Parser<ObjectName>{})))
 
-// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+// map-type -> ALLOC | DELETE | FROM | RELEASE | STORAGE | TO | TOFROM
 TYPE_PARSER(construct<OmpMapType>( //
     "ALLOC" >> pure(OmpMapType::Value::Alloc) ||
-    "DELETE" >> pure(OmpMapType::Value::Delete) ||
+    // Parse "DELETE" as OmpDeleteModifier
     "FROM" >> pure(OmpMapType::Value::From) ||
     "RELEASE" >> pure(OmpMapType::Value::Release) ||
+    "STORAGE" >> pure(OmpMapType::Value::Storage) ||
     "TO"_id >> pure(OmpMapType::Value::To) ||
     "TOFROM" >> pure(OmpMapType::Value::Tofrom)))
 
-// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
-TYPE_PARSER(construct<OmpMapTypeModifier>(
-    "ALWAYS" >> pure(OmpMapTypeModifier::Value::Always) ||
-    "CLOSE" >> pure(OmpMapTypeModifier::Value::Close) ||
-    "OMPX_HOLD" >> pure(OmpMapTypeModifier::Value::Ompx_Hold) ||
-    "PRESENT" >> pure(OmpMapTypeModifier::Value::Present)))
-
-// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
-TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
-    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
-
 TYPE_PARSER(construct<OmpOrderModifier>(
     "REPRODUCIBLE" >> pure(OmpOrderModifier::Value::Reproducible) ||
     "UNCONSTRAINED" >> pure(OmpOrderModifier::Value::Unconstrained)))
@@ -534,11 +537,22 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
 TYPE_PARSER(construct<OmpPrescriptiveness>(
     "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
 
+TYPE_PARSER(construct<OmpPresentModifier>( //
+    "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
     "DEFAULT" >> pure(OmpReductionModifier::Value::Default)))
 
+TYPE_PARSER(construct<OmpRefModifier>( //
+    "REF_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptee) ||
+    "REF_PTR"_id >> pure(OmpRefModifier::Value::Ref_Ptr) ||
+    "REF_PTR_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptr_Ptee)))
+
+TYPE_PARSER(construct<OmpSelfModifier>( //
+    "SELF" >> pure(OmpSelfModifier::Value::Self)))
+
 TYPE_PARSER(construct<OmpStepComplexModifier>( //
     "STEP" >> parenthesized(scalarIntExpr)))
 
@@ -559,6 +573,9 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "POINTER" >> pure(OmpVariableCategory::Value::Pointer) ||
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
+TYPE_PARSER(construct<OmpxHoldModifier>( //
+    "OMPX_HOLD" >> pure(OmpxHoldModifier::Value::Ompx_Hold)))
+
 // This could be auto-generated.
 TYPE_PARSER(
     sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
@@ -611,10 +628,16 @@ TYPE_PARSER(sourced(
     construct<OmpLinearClause::Modifier>(Parser<OmpStepSimpleModifier>{})))
 
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
-    sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
+    sourced(construct<OmpMapClause::Modifier>(Parser<OmpAlwaysModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpCloseModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpDeleteModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpPresentModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpRefModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpSelfModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpIterator>{}) ||
-        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{})))))
+        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpxHoldModifier>{})))))
 
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index fbe89c668fc13..8ed16905b5099 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -3007,8 +3007,15 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
+  WALK_NESTED_ENUM(OmpAlwaysModifier, Value)
+  WALK_NESTED_ENUM(OmpCloseModifier, Value)
+  WALK_NESTED_ENUM(OmpDeleteModifier, Value)
+  WALK_NESTED_ENUM(OmpPresentModifier, Value)
+  WALK_NESTED_ENUM(OmpRefModifier, Value)
+  WALK_NESTED_ENUM(OmpSelfModifier, Value)
   WALK_NESTED_ENUM(OmpTraitSelectorName, Value)
   WALK_NESTED_ENUM(OmpTraitSetSelectorName, Value)
+  WALK_NESTED_ENUM(OmpxHoldModifier, Value)
 
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index cf05d8463277f..46aaab19ded0a 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -88,6 +89,8 @@ class CanonicalizationOfOmp {
     CanonicalizeUtilityConstructs(spec);
   }
 
+  void Post(parser::OmpMapClause &map) { CanonicalizeMapModifiers(map); }
+
 private:
   template <typename T> T *GetConstructIf(parser::ExecutionPartConstruct &x) {
     if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index c45d6bbbf9062..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,11 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
-}
+class SemanticsContext;
+
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+} // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index c84e832ee52a1..336ce4beb24ba 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -140,6 +140,22 @@ OmpGetDescriptor<parser::OmpAllocatorSimpleModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlwaysModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"always-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   static const OmpModifierDescriptor desc{
@@ -156,6 +172,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpCloseModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"close-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   static const OmpModifierDescriptor desc{
@@ -173,6 +205,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDeleteModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"delete-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique, OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   static const OmpModifierDescriptor desc{
@@ -347,6 +396,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapType>() {
       /*props=*/
       {
           {45, {OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
       },
       /*clauses=*/
       {
@@ -367,6 +417,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapTypeModifier>() {
       /*clauses=*/
       {
           {45, {Clause::OMPC_map}},
+          {60, {}},
       },
   };
   return desc;
@@ -420,6 +471,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPrescriptiveness>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPresentModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"present-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpReductionIdentifier>() {
@@ -456,6 +523,38 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpReductionModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpRefModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ref-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpSelfModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"self-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpStepComplexModifier>() {
@@ -522,4 +621,20 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpVariableCategory>() {
   };
   return desc;
 }
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpxHoldModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ompx-hold-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 151f4ccae634e..0b860314b4a1f 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -734,6 +734,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
+      default:
+        break;
       }
     }
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..d0a5b200d0d8e 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -643,8 +643,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}
diff --git a/flang/test/Parser/OpenMP/map-modifiers-v60.f90 b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..bc80886780d46
--- /dev/null
+++ b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,113 @@
+!RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x
+  !$omp target map(always, close, delete, present, ompx_hold: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(ALWAYS, CLOSE, DELETE, PRESENT, OMPX_HOLD: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpAlwaysModifier -> Value = Always
+!PARSE-TREE: | | Modifier -> OmpCloseModifier -> Value = Close
+!PARSE-TREE: | | Modifier -> OmpDeleteModifier -> Value = Delete
+!PARSE-TREE: | | Modifier -> OmpPresentModifier -> Value = Present
+!PARSE-TREE: | | Modifier -> OmpxHoldModifier -> Value = Ompx_Hold
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f01(x)
+  integer :: x
+  !$omp target map(self, storage: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: !$OMP TARGET  MAP(SELF, STORAGE: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpSelfModifier -> Value = Self
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = Storage
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f03(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f04(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f04 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'


From flang-commits at lists.llvm.org  Wed Jul 16 09:29:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:29:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <6877d37a.630a0220.2a28de.348b@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: Krzysztof Parzyszek (kparzysz)

<details>
<summary>Changes</summary>

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers. The old "map-type-modifier" is retained, but is no longer created in parsing. It will remain to take advantage of the preexisting modifier validation for older versions: when the OpenMP version is &lt; 6.0, the modifiers will be rewritten back as map-type-modifiers (or map- type in case of "delete").

In this patch the modifiers will always be rewritten in the older format to isolate these changes to parsing as much as possible.

---

Patch is 31.82 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/149134.diff


11 Files Affected:

- (modified) flang/include/flang/Parser/dump-parse-tree.h (+14) 
- (modified) flang/include/flang/Parser/parse-tree.h (+120-18) 
- (modified) flang/include/flang/Semantics/openmp-modifiers.h (+7) 
- (modified) flang/lib/Parser/openmp-parsers.cpp (+38-15) 
- (modified) flang/lib/Parser/unparse.cpp (+7) 
- (modified) flang/lib/Semantics/canonicalize-omp.cpp (+44-4) 
- (modified) flang/lib/Semantics/canonicalize-omp.h (+5-4) 
- (modified) flang/lib/Semantics/openmp-modifiers.cpp (+115) 
- (modified) flang/lib/Semantics/resolve-directives.cpp (+2) 
- (modified) flang/lib/Semantics/semantics.cpp (+1-2) 
- (added) flang/test/Parser/OpenMP/map-modifiers-v60.f90 (+113) 


``````````diff
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 32b6ca45609b6..23e35d106c077 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -529,6 +529,8 @@ class ParseTreeDumper {
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlwaysModifier)
+  NODE_ENUM(OmpAlwaysModifier, Value)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
@@ -546,6 +548,8 @@ class ParseTreeDumper {
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
   NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpCloseModifier)
+  NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -561,6 +565,8 @@ class ParseTreeDumper {
   NODE(parser, OmpDefaultmapClause)
   NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE(parser, OmpDeleteModifier)
+  NODE_ENUM(OmpDeleteModifier, Value)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
   NODE(parser, OmpTaskDependenceType)
@@ -628,6 +634,8 @@ class ParseTreeDumper {
   NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
   NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpPresentModifier)
+  NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
@@ -637,6 +645,10 @@ class ParseTreeDumper {
   NODE(parser, OmpInReductionClause)
   NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpRefModifier)
+  NODE_ENUM(OmpRefModifier, Value)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpInitializerProc)
@@ -673,6 +685,8 @@ class ParseTreeDumper {
   NODE(parser, OmpSectionsDirective)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpxHoldModifier)
+  NODE_ENUM(OmpxHoldModifier, Value)
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cc1d032f94d4a..92a449d2f9d49 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3756,6 +3756,19 @@ struct OmpAllocatorComplexModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// always-modifier ->
+//    ALWAYS                                        // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpAlwaysModifier {
+  ENUM_CLASS(Value, Always)
+  WRAPPER_CLASS_BOILERPLATE(OmpAlwaysModifier, Value);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3767,17 +3780,29 @@ struct OmpChunkModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpChunkModifier, Value);
 };
 
-// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
-// iterator-specifier ->
-//    [iterator-type] iterator-identifier
-//        = range-specification |                   // since 5.0
-//    [iterator-type ::] iterator-identifier
-//        = range-specification                     // since 5.2
-struct OmpIteratorSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
-  CharBlock source;
-  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+// close-modifier ->
+//    CLOSE                                         // since 5.0
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpCloseModifier {
+  ENUM_CLASS(Value, Close)
+  WRAPPER_CLASS_BOILERPLATE(OmpCloseModifier, Value);
+};
+
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// delete-modifier ->
+//    DELETE                                        // since 6.0
+//
+// Until 5.2, it was a part of map-type.
+struct OmpDeleteModifier {
+  ENUM_CLASS(Value, Delete)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeleteModifier, Value);
 };
 
 // Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289]
@@ -3867,6 +3892,19 @@ struct OmpInteropType {
   WRAPPER_CLASS_BOILERPLATE(OmpInteropType, Value);
 };
 
+// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+//
+// iterator-specifier ->
+//    [iterator-type] iterator-identifier
+//        = range-specification |                   // since 5.0
+//    [iterator-type ::] iterator-identifier
+//        = range-specification                     // since 5.2
+struct OmpIteratorSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
+  CharBlock source;
+  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+};
+
 // Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
 //
 // iterator-modifier ->
@@ -3901,21 +3939,28 @@ struct OmpMapper {
   WRAPPER_CLASS_BOILERPLATE(OmpMapper, Name);
 };
 
-// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
 // map-type ->
-//    ALLOC | DELETE | FROM | RELEASE | TO | TOFROM // since 4.5
+//    ALLOC | DELETE | RELEASE |                    // since 4.5, until 5.2
+//    FROM | TO | TOFROM |                          // since 4.5
+//    STORAGE                                       // since 6.0
+//
+// Since 6.0 DELETE is a separate delete-modifier.
 struct OmpMapType {
-  ENUM_CLASS(Value, Alloc, Delete, From, Release, To, Tofrom);
+  ENUM_CLASS(Value, Alloc, Delete, From, Release, Storage, To, Tofrom);
   WRAPPER_CLASS_BOILERPLATE(OmpMapType, Value);
 };
 
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-type-modifier ->
-//    ALWAYS |                                      // since 4.5
-//    CLOSE |                                       // since 5.0
-//    PRESENT                                       // since 5.1
+//    ALWAYS |                                      // since 4.5, until 5.2
+//    CLOSE |                                       // since 5.0, until 5.2
+//    PRESENT                                       // since 5.1, until 5.2
+// Since 6.0 the map-type-modifier has been split into individual modifiers.
+//
 struct OmpMapTypeModifier {
   ENUM_CLASS(Value, Always, Close, Present, Ompx_Hold)
   WRAPPER_CLASS_BOILERPLATE(OmpMapTypeModifier, Value);
@@ -3954,6 +3999,19 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// present-modifier ->
+//    PRESENT                                       // since 5.1
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpPresentModifier {
+  ENUM_CLASS(Value, Present)
+  WRAPPER_CLASS_BOILERPLATE(OmpPresentModifier, Value);
+};
+
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3963,6 +4021,26 @@ struct OmpReductionModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpReductionModifier, Value);
 };
 
+// Ref: [6.0:279-288]
+//
+// ref-modifier ->
+//    REF_PTEE | REF_PTR | REF_PTR_PTEE             // since 6.0
+//
+struct OmpRefModifier {
+  ENUM_CLASS(Value, Ref_Ptee, Ref_Ptr, Ref_Ptr_Ptee)
+  WRAPPER_CLASS_BOILERPLATE(OmpRefModifier, Value);
+};
+
+// Ref: [6.0:279-288]
+//
+// self-modifier ->
+//    SELF                                          // since 6.0
+//
+struct OmpSelfModifier {
+  ENUM_CLASS(Value, Self)
+  WRAPPER_CLASS_BOILERPLATE(OmpSelfModifier, Value);
+};
+
 // Ref: [5.2:117-120]
 //
 // step-complex-modifier ->
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
+//
+// ompx-hold-modifier ->
+//    OMPX_HOLD                                     // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpxHoldModifier {
+  ENUM_CLASS(Value, Ompx_Hold)
+  WRAPPER_CLASS_BOILERPLATE(OmpxHoldModifier, Value);
+};
+
 // context-selector
 using OmpContextSelector = traits::OmpContextSelectorSpecification;
 } // namespace modifier
@@ -4376,13 +4466,25 @@ struct OmpLinearClause {
 // map-clause ->
 //    MAP([modifier...:] locator-list)              // since 4.5
 // modifier ->
-//    map-type-modifier |                           // since 4.5
+//    map-type-modifier [replaced] |                // since 4.5, until 5.2
+//    always-modifier |                             // since 6.0
+//    close-modifier |                              // since 6.0
+//    delete-modifier |                             // since 6.0
+//    present-modifier |                            // since 6.0
+//    ref-modifier |                                // since 6.0
+//    self-modifier |                               // since 6.0
 //    mapper |                                      // since 5.0
 //    iterator |                                    // since 5.1
 //    map-type                                      // since 4.5
+//    ompx-hold-modifier |                          // since 6.0
+//
+// Since 6.0 the map-type-modifier has been split into individual modifiers,
+// and delete-modifier has been split from map-type.
 struct OmpMapClause {
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  MODIFIER_BOILERPLATE(OmpMapTypeModifier, OmpMapper, OmpIterator, OmpMapType);
+  MODIFIER_BOILERPLATE(OmpAlwaysModifier, OmpCloseModifier, OmpDeleteModifier,
+      OmpMapTypeModifier, OmpPresentModifier, OmpRefModifier, OmpSelfModifier,
+      OmpMapper, OmpIterator, OmpMapType, OmpxHoldModifier);
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 4fbd80f989e72..a9fe911ef8807 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -71,8 +71,11 @@ DECLARE_DESCRIPTOR(parser::OmpAlignment);
 DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
+DECLARE_DESCRIPTOR(parser::OmpAlwaysModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpCloseModifier);
 DECLARE_DESCRIPTOR(parser::OmpContextSelector);
+DECLARE_DESCRIPTOR(parser::OmpDeleteModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
@@ -88,12 +91,16 @@ DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
 DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
+DECLARE_DESCRIPTOR(parser::OmpPresentModifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
+DECLARE_DESCRIPTOR(parser::OmpRefModifier);
+DECLARE_DESCRIPTOR(parser::OmpSelfModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpVariableCategory);
+DECLARE_DESCRIPTOR(parser::OmpxHoldModifier);
 
 #undef DECLARE_DESCRIPTOR
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..d349d8ceb0bb5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -274,6 +274,10 @@ TYPE_PARSER( //
     construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}) ||
     construct<OmpTypeSpecifier>(Parser<TypeSpec>{}))
 
+// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
+TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
+    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
+
 TYPE_PARSER(construct<OmpReductionSpecifier>( //
     Parser<OmpReductionIdentifier>{},
     ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
@@ -442,9 +446,18 @@ TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
 
 TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
 
+TYPE_PARSER(construct<OmpAlwaysModifier>( //
+    "ALWAYS" >> pure(OmpAlwaysModifier::Value::Always)))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpCloseModifier>( //
+    "CLOSE" >> pure(OmpCloseModifier::Value::Close)))
+
+TYPE_PARSER(construct<OmpDeleteModifier>( //
+    "DELETE" >> pure(OmpDeleteModifier::Value::Delete)))
+
 TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
@@ -502,26 +515,16 @@ TYPE_PARSER(construct<OmpLinearModifier>( //
 TYPE_PARSER(construct<OmpMapper>( //
     "MAPPER"_tok >> parenthesized(Parser<ObjectName>{})))
 
-// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+// map-type -> ALLOC | DELETE | FROM | RELEASE | STORAGE | TO | TOFROM
 TYPE_PARSER(construct<OmpMapType>( //
     "ALLOC" >> pure(OmpMapType::Value::Alloc) ||
-    "DELETE" >> pure(OmpMapType::Value::Delete) ||
+    // Parse "DELETE" as OmpDeleteModifier
     "FROM" >> pure(OmpMapType::Value::From) ||
     "RELEASE" >> pure(OmpMapType::Value::Release) ||
+    "STORAGE" >> pure(OmpMapType::Value::Storage) ||
     "TO"_id >> pure(OmpMapType::Value::To) ||
     "TOFROM" >> pure(OmpMapType::Value::Tofrom)))
 
-// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
-TYPE_PARSER(construct<OmpMapTypeModifier>(
-    "ALWAYS" >> pure(OmpMapTypeModifier::Value::Always) ||
-    "CLOSE" >> pure(OmpMapTypeModifier::Value::Close) ||
-    "OMPX_HOLD" >> pure(OmpMapTypeModifier::Value::Ompx_Hold) ||
-    "PRESENT" >> pure(OmpMapTypeModifier::Value::Present)))
-
-// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
-TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
-    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
-
 TYPE_PARSER(construct<OmpOrderModifier>(
     "REPRODUCIBLE" >> pure(OmpOrderModifier::Value::Reproducible) ||
     "UNCONSTRAINED" >> pure(OmpOrderModifier::Value::Unconstrained)))
@@ -534,11 +537,22 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
 TYPE_PARSER(construct<OmpPrescriptiveness>(
     "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
 
+TYPE_PARSER(construct<OmpPresentModifier>( //
+    "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
     "DEFAULT" >> pure(OmpReductionModifier::Value::Default)))
 
+TYPE_PARSER(construct<OmpRefModifier>( //
+    "REF_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptee) ||
+    "REF_PTR"_id >> pure(OmpRefModifier::Value::Ref_Ptr) ||
+    "REF_PTR_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptr_Ptee)))
+
+TYPE_PARSER(construct<OmpSelfModifier>( //
+    "SELF" >> pure(OmpSelfModifier::Value::Self)))
+
 TYPE_PARSER(construct<OmpStepComplexModifier>( //
     "STEP" >> parenthesized(scalarIntExpr)))
 
@@ -559,6 +573,9 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "POINTER" >> pure(OmpVariableCategory::Value::Pointer) ||
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
+TYPE_PARSER(construct<OmpxHoldModifier>( //
+    "OMPX_HOLD" >> pure(OmpxHoldModifier::Value::Ompx_Hold)))
+
 // This could be auto-generated.
 TYPE_PARSER(
     sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
@@ -611,10 +628,16 @@ TYPE_PARSER(sourced(
     construct<OmpLinearClause::Modifier>(Parser<OmpStepSimpleModifier>{})))
 
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
-    sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
+    sourced(construct<OmpMapClause::Modifier>(Parser<OmpAlwaysModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpCloseModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpDeleteModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpPresentModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpRefModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpSelfModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpIterator>{}) ||
-        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{})))))
+        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpxHoldModifier>{})))))
 
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index fbe89c668fc13..8ed16905b5099 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -3007,8 +3007,15 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
+  WALK_NESTED_ENUM(OmpAlwaysModifier, Value)
+  WALK_NESTED_ENUM(OmpCloseModifier, Value)
+  WALK_NESTED_ENUM(OmpDeleteModifier, Value)
+  WALK_NESTED_ENUM(OmpPresentModifier, Value)
+  WALK_NESTED_ENUM(OmpRefModifier, Value)
+  WALK_NESTED_ENUM(OmpSelfModifier, Value)
   WALK_NESTED_ENUM(OmpTraitSelectorName, Value)
   WALK_NESTED_ENUM(OmpTraitSetSelectorName, Value)
+  WALK_NESTED_ENUM(OmpxHoldModifier, Value)
 
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index cf05d8463277f..46aaab19ded0a 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -88,6 +89,8 @@ class CanonicalizationOfOmp {
     CanonicalizeUtilityConstructs(spec);
   }
 
+  void Post(parser::OmpMapClause &map) { CanonicalizeMapModifiers(map); }
+
 private:
   template <typename T> T *GetConstructIf(parser::ExecutionPartConstruct &x) {
     if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+ ...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Wed Jul 16 09:21:47 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 09:21:47 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <6877d19b.170a0220.2fd26a.13d3@mx.google.com>


https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/145464

>From 91d2c54266ba33fb01b30d81b6694914180ad982 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 23 Jun 2025 16:39:55 +0530
Subject: [PATCH 1/8] [flang] Introduce omp_target_allocmem and
 omp_target_freemem fir ops.

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  58 ++++++++
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 130 +++++++++++++++---
 2 files changed, 172 insertions(+), 16 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 99b5105ab365e..de10334ed7b18 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,6 +517,64 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
+def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Creates a heap memory reference suitable for storing a value of the
+    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
+    The memory object is in an undefined state.  `omp_target_allocmem` operations must
+    be paired with `omp_target_freemem` operations to avoid memory leaks.
+
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyIntegerType>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyIntegerType>:$typeparams,
+    Variadic<AnyIntegerType>:$shape
+  );
+  let results = (outs fir_HeapType);
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+    bool hasLenParams() { return !getTypeparams().empty(); }
+    bool hasShapeOperands() { return !getShape().empty(); }
+    unsigned numLenParams() { return getTypeparams().size(); }
+    operand_range getLenParams() { return getTypeparams(); }
+    unsigned numShapeOperands() { return getShape().size(); }
+    operand_range getShapeOperands() { return getShape(); }
+    static mlir::Type getRefTy(mlir::Type ty);
+  }];
+}
+
+def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free a heap object";
+
+  let description = [{
+    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
+    The memory object that is deallocated is placed in an undefined state
+    after `fir.omp_target_freemem`.
+    ```
+      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      ...
+      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyIntegerType, "", [MemFree]>:$device,
+  Arg<fir_HeapType, "", [MemFree]>:$heapref
+  );
+}
+
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index ecc04a6c9a2be..fc7a21bcbe69f 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1206,6 +1206,105 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetAllocMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = heap.getType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
+    mlir::Location loc = heap.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(dataTy);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
+    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(loc, rewriter, mallocTy, size);
+    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+        heap, ::getLlvmPtrType(heap.getContext()),
+        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
+    return mlir::success();
+  }
+
+  /// Compute the allocation size in bytes of the element type of
+  /// \p llTy pointer type. The result is returned as a value of \p idxTy
+  /// integer type.
+  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy) const {
+    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+  }
+};
+} // namespace
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp freeFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
+    return freeFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_free",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMVoidType::get(module->getContext()),
+          {getLlvmPtrType(module->getContext()), i32Ty},
+          /*isVarArg=*/false));
+}
+
+namespace {
+struct OmpTargetFreeMemOpConversion
+    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
+  using FIROpConversion::FIROpConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
+    mlir::Location loc = freemem.getLoc();
+    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
+    rewriter.create<mlir::LLVM::CallOp>(
+        loc, mlir::TypeRange{},
+        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
+        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
+    rewriter.eraseOp(freemem);
+    return mlir::success();
+  }
+};
+} // namespace
+
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4328,22 +4427,21 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, VolatileCastOpConversion, ConvertOpConversion,
       CoordinateOpConversion, CopyOpConversion, DTEntryOpConversion,
-      DeclareOpConversion,
-      DoConcurrentSpecifierOpConversion<fir::LocalitySpecifierOp>,
-      DoConcurrentSpecifierOpConversion<fir::DeclareReductionOp>,
-      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
-      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
-      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
-      GlobalOpConversion, InsertOnRangeOpConversion, IsPresentOpConversion,
-      LenParamIndexOpConversion, LoadOpConversion, MulcOpConversion,
-      NegcOpConversion, NoReassocOpConversion, SelectCaseOpConversion,
-      SelectOpConversion, SelectRankOpConversion, SelectTypeOpConversion,
-      ShapeOpConversion, ShapeShiftOpConversion, ShiftOpConversion,
-      SliceOpConversion, StoreOpConversion, StringLitOpConversion,
-      SubcOpConversion, TypeDescOpConversion, TypeInfoOpConversion,
-      UnboxCharOpConversion, UnboxProcOpConversion, UndefOpConversion,
-      UnreachableOpConversion, XArrayCoorOpConversion, XEmboxOpConversion,
-      XReboxOpConversion, ZeroOpConversion>(converter, options);
+      DeclareOpConversion, DivcOpConversion, EmboxOpConversion,
+      EmboxCharOpConversion, EmboxProcOpConversion, ExtractValueOpConversion,
+      FieldIndexOpConversion, FirEndOpConversion, FreeMemOpConversion,
+      GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
+      IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
+      LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
+      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
+      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
+      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
+      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
+      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
+      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
+      XArrayCoorOpConversion, XEmboxOpConversion, XReboxOpConversion,
+      ZeroOpConversion>(converter, options);
 
   // Patterns that are populated without a type converter do not trigger
   // target materializations for the operands of the root op.

>From 72947c8592bf6c5300790ff8aff349e139017407 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:20:36 +0530
Subject: [PATCH 2/8] [flang] Fix parsing and printing.

---
 .../include/flang/Optimizer/Dialect/FIROps.td | 13 ++-
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 90 ++++++++++++++++---
 flang/test/Fir/omp_target_allocmem.fir        | 28 ++++++
 flang/test/Fir/omp_target_freemem.fir         | 28 ++++++
 4 files changed, 145 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Fir/omp_target_allocmem.fir
 create mode 100644 flang/test/Fir/omp_target_freemem.fir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index de10334ed7b18..8fbce5a8a745d 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -528,7 +528,8 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
     be paired with `omp_target_freemem` operations to avoid memory leaks.
 
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
     ```
   }];
 
@@ -542,6 +543,9 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
   );
   let results = (outs fir_HeapType);
 
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
   let extraClassDeclaration = [{
     mlir::Type getAllocatedType();
     bool hasLenParams() { return !getTypeparams().empty(); }
@@ -563,9 +567,9 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     The memory object that is deallocated is placed in an undefined state
     after `fir.omp_target_freemem`.
     ```
-      %0 = "fir.omp_target_allocmem"(%device, %type) : (i32, index) -> !fir.heap<!fir.array<?xf32>>
-      ...
-      "fir.omp_target_freemem"(%device, %0) : (i32, !fir.heap<!fir.array<?xf32>>) -> ()
+      %device = arith.constant 0 : i32
+      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
     ```
   }];
 
@@ -573,6 +577,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   Arg<AnyIntegerType, "", [MemFree]>:$device,
   Arg<fir_HeapType, "", [MemFree]>:$heapref
   );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index b6bf2753b80ce..38862953b8001 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,24 +106,38 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca and Allocmem
-///
+/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
+/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
-                                            mlir::OpAsmParser &parser,
-                                            mlir::OperationState &result) {
+static mlir::ParseResult
+parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
+                   mlir::OperationState &result, bool isTargetOp = false) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+  // Parse device number as a new operand
+  if (isTargetOp) {
+    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+    mlir::Type deviceType;
+    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+      return mlir::failure();
+    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+      return mlir::failure();
+    if (parser.parseComma())
+      return mlir::failure();
+  }
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
-  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -147,13 +161,19 @@ static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
+
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
-                                                 {typeparamsSize, shapeSize}));
+  llvm::SmallVector<std::int32_t> segmentSizes;
+  if (isTargetOp)
+    segmentSizes.push_back(1);
+  segmentSizes.push_back(typeparamsSize);
+  segmentSizes.push_back(shapeSize);
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -385,6 +405,56 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// OmpTargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
+  return mlir::cast<fir::HeapType>(getType()).getEleTy();
+}
+
+mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
+  return fir::HeapType::get(ty);
+}
+
+mlir::ParseResult
+fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                mlir::OperationState &result) {
+  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
+}
+
+void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
+  llvm::SmallVector<llvm::StringRef> visited;
+  if (verifyInType(getInType(), visited, numShapeOperands()))
+    return emitOpError("invalid type for allocation");
+  if (verifyTypeParamCount(getInType(), numLenParams()))
+    return emitOpError("LEN params do not correspond to type");
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<fir::HeapType>(outType))
+    return emitOpError("must be a !fir.heap type");
+  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
+    return emitOpError("cannot allocate !fir.box of unknown rank or type");
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
new file mode 100644
index 0000000000000..5140c91c9510c
--- /dev/null
+++ b/flang/test/Fir/omp_target_allocmem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  return %1 : !fir.heap<!fir.array<3x3xi32>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+}
+
+// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_freemem.fir
new file mode 100644
index 0000000000000..02e136076a9cf
--- /dev/null
+++ b/flang/test/Fir/omp_target_freemem.fir
@@ -0,0 +1,28 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
+// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  return
+}

>From d929a4cf5877284e03c1bf0269ab9e2fa69aebc6 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 26 Jun 2025 10:32:21 +0530
Subject: [PATCH 3/8] [flang] Fix doc in td

---
 flang/include/flang/Optimizer/Dialect/FIROps.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 8fbce5a8a745d..1587ddd244976 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -560,7 +560,7 @@ def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
 
 def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
   [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object";
+  let summary = "free a heap object on an openmp device";
 
   let description = [{
     Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
@@ -569,7 +569,7 @@ def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
     ```
       %device = arith.constant 0 : i32
       %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<?xf32>>
+      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
     ```
   }];
 

>From 293c3fac9c0d37bff4f12631a6dc14672ec78242 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 12:22:55 +0530
Subject: [PATCH 4/8] [omp][mlir] Introduce TargetAllocMem and TargetFreeMem
 ops in openMP mlir dialect

---
 .../include/flang/Optimizer/Dialect/FIROps.td |  63 -------
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 102 +----------
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 161 ++++++++++++++++++
 flang/lib/Optimizer/Dialect/FIROps.cpp        |  88 +---------
 flang/test/Fir/omp_target_allocmem.fir        |  28 ---
 ...em.fir => omp_target_allocmem_freemem.fir} |  19 ++-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  64 +++++++
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 102 +++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  82 +++++++++
 .../ompenmp-target-allocmem-freemem.mlir      |  42 +++++
 10 files changed, 473 insertions(+), 278 deletions(-)
 delete mode 100644 flang/test/Fir/omp_target_allocmem.fir
 rename flang/test/Fir/{omp_target_freemem.fir => omp_target_allocmem_freemem.fir} (51%)
 create mode 100644 mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 1587ddd244976..99b5105ab365e 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -517,69 +517,6 @@ def fir_ZeroOp : fir_OneResultOp<"zero_bits", [NoMemoryEffect]> {
   let assemblyFormat = "type($intype) attr-dict";
 }
 
-def fir_OmpTargetAllocMemOp : fir_Op<"omp_target_allocmem",
-    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
-  let summary = "allocate storage on an openmp device for an object of a given type";
-
-  let description = [{
-    Creates a heap memory reference suitable for storing a value of the
-    given type, T.  The heap refernce returned has type `!fir.heap<T>`.
-    The memory object is in an undefined state.  `omp_target_allocmem` operations must
-    be paired with `omp_target_freemem` operations to avoid memory leaks.
-
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-    ```
-  }];
-
-  let arguments = (ins
-    Arg<AnyIntegerType>:$device,
-    TypeAttr:$in_type,
-    OptionalAttr<StrAttr>:$uniq_name,
-    OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyIntegerType>:$typeparams,
-    Variadic<AnyIntegerType>:$shape
-  );
-  let results = (outs fir_HeapType);
-
-  let hasCustomAssemblyFormat = 1;
-  let hasVerifier = 1;
-
-  let extraClassDeclaration = [{
-    mlir::Type getAllocatedType();
-    bool hasLenParams() { return !getTypeparams().empty(); }
-    bool hasShapeOperands() { return !getShape().empty(); }
-    unsigned numLenParams() { return getTypeparams().size(); }
-    operand_range getLenParams() { return getTypeparams(); }
-    unsigned numShapeOperands() { return getShape().size(); }
-    operand_range getShapeOperands() { return getShape(); }
-    static mlir::Type getRefTy(mlir::Type ty);
-  }];
-}
-
-def fir_OmpTargetFreeMemOp : fir_Op<"omp_target_freemem",
-  [MemoryEffects<[MemFree]>]> {
-  let summary = "free a heap object on an openmp device";
-
-  let description = [{
-    Deallocates a heap memory reference that was allocated by an `omp_target_allocmem`.
-    The memory object that is deallocated is placed in an undefined state
-    after `fir.omp_target_freemem`.
-    ```
-      %device = arith.constant 0 : i32
-      %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-      fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
-    ```
-  }];
-
-  let arguments = (ins
-  Arg<AnyIntegerType, "", [MemFree]>:$device,
-  Arg<fir_HeapType, "", [MemFree]>:$heapref
-  );
-  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
-}
-
 //===----------------------------------------------------------------------===//
 // Terminator operations
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index fc7a21bcbe69f..7940743763c5c 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1206,105 +1206,6 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
 };
 } // namespace
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetAllocMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetAllocMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetAllocMemOp heap, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::Type heapTy = heap.getType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(heap);
-    mlir::Location loc = heap.getLoc();
-    auto ity = lowerTy().indexType();
-    mlir::Type dataTy = fir::unwrapRefType(heapTy);
-    mlir::Type llvmObjectTy = convertObjectType(dataTy);
-    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
-      TODO(loc, "fir.omp_target_allocmem codegen of derived type with length "
-                "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
-      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands().drop_front())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
-    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
-    auto mallocTy =
-        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
-    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
-      size = integerCast(loc, rewriter, mallocTy, size);
-    heap->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        heap, ::getLlvmPtrType(heap.getContext()),
-        mlir::SmallVector<mlir::Value, 2>({size, heap.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 2));
-    return mlir::success();
-  }
-
-  /// Compute the allocation size in bytes of the element type of
-  /// \p llTy pointer type. The result is returned as a value of \p idxTy
-  /// integer type.
-  mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
-  }
-};
-} // namespace
-
-static mlir::LLVM::LLVMFuncOp getOmpTargetFree(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp freeFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_free"))
-    return freeFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_free",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMVoidType::get(module->getContext()),
-          {getLlvmPtrType(module->getContext()), i32Ty},
-          /*isVarArg=*/false));
-}
-
-namespace {
-struct OmpTargetFreeMemOpConversion
-    : public fir::FIROpConversion<fir::OmpTargetFreeMemOp> {
-  using FIROpConversion::FIROpConversion;
-
-  mlir::LogicalResult
-  matchAndRewrite(fir::OmpTargetFreeMemOp freemem, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::LLVM::LLVMFuncOp freeFunc = getOmpTargetFree(freemem);
-    mlir::Location loc = freemem.getLoc();
-    freemem->setAttr("callee", mlir::SymbolRefAttr::get(freeFunc));
-    rewriter.create<mlir::LLVM::CallOp>(
-        loc, mlir::TypeRange{},
-        mlir::ValueRange{adaptor.getHeapref(), freemem.getDevice()},
-        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 2));
-    rewriter.eraseOp(freemem);
-    return mlir::success();
-  }
-};
-} // namespace
-
 // Convert subcomponent array indices from column-major to row-major ordering.
 static llvm::SmallVector<mlir::Value>
 convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
@@ -4433,8 +4334,7 @@ void fir::populateFIRToLLVMConversionPatterns(
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
       LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, OmpTargetAllocMemOpConversion,
-      OmpTargetFreeMemOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
       SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
       ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
       StoreOpConversion, StringLitOpConversion, SubcOpConversion,
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 37f1c9f97e1ce..a04c5d7eb7ee7 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -125,10 +125,171 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  if (mlir::LLVM::LLVMFuncOp mallocFunc =
+          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
+    return mallocFunc;
+  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
+  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
+  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
+  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
+      mlir::LLVM::LLVMFunctionType::get(
+          mlir::LLVM::LLVMPointerType::get(module->getContext()),
+          {i64Ty, i32Ty},
+          /*isVarArg=*/false));
+}
+
+static mlir::Type
+convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
+static mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+static mlir::Value
+computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                       mlir::Type idxTy,
+                       mlir::ConversionPatternRewriter &rewriter,
+                       const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
+                                 mlir::ConversionPatternRewriter &rewriter,
+                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+}
+
+template <typename OP>
+static mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
+    mlir::Type ty, mlir::Value val, bool fold = false) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "omp.target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(lowerTy(), loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
+    allocmemOp->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
+    auto callOp = rewriter.create<mlir::LLVM::CallOp>(
+        loc, llvmPtrTy,
+        mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
+        addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    return mlir::success();
+  }
+};
 } // namespace
 
 void fir::populateOpenMPFIRToLLVMConversionPatterns(
     const LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
   patterns.add<MapInfoOpConversion>(converter);
   patterns.add<PrivateClauseOpConversion>(converter);
+  patterns.add<TargetAllocMemOpConversion>(converter);
 }
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 38862953b8001..4558057bec321 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -106,38 +106,24 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
   return false;
 }
 
-/// Parser shared by Alloca, Allocmem and OmpTargetAllocmem
+/// Parser shared by Alloca and Allocmem
 /// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
-/// operation ::= %res = (`fir.omp_target_alloca`) $device : devicetype,
-///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
-///                      attr-dict-without-keyword
 template <typename FN>
-static mlir::ParseResult
-parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
-                   mlir::OperationState &result, bool isTargetOp = false) {
-  auto &builder = parser.getBuilder();
-  bool hasOperands = false;
-  std::int32_t typeparamsSize = 0;
-  // Parse device number as a new operand
-  if (isTargetOp) {
-    mlir::OpAsmParser::UnresolvedOperand deviceOperand;
-    mlir::Type deviceType;
-    if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
-      return mlir::failure();
-    if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
-      return mlir::failure();
-    if (parser.parseComma())
-      return mlir::failure();
-  }
+static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
+                                            mlir::OpAsmParser &parser,
+                                            mlir::OperationState &result) {
   mlir::Type intype;
   if (parser.parseType(intype))
     return mlir::failure();
+  auto &builder = parser.getBuilder();
   result.addAttribute("in_type", mlir::TypeAttr::get(intype));
   llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
   llvm::SmallVector<mlir::Type> typeVec;
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
   if (!parser.parseOptionalLParen()) {
     // parse the LEN params of the derived type. (<params> : <types>)
     if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
@@ -161,19 +147,13 @@ parseAllocatableOp(FN wrapResultType, mlir::OpAsmParser &parser,
       parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
                              result.operands))
     return mlir::failure();
-
   mlir::Type restype = wrapResultType(intype);
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
   }
-  llvm::SmallVector<std::int32_t> segmentSizes;
-  if (isTargetOp)
-    segmentSizes.push_back(1);
-  segmentSizes.push_back(typeparamsSize);
-  segmentSizes.push_back(shapeSize);
-  result.addAttribute("operandSegmentSizes",
-                      builder.getDenseI32ArrayAttr(segmentSizes));
+  result.addAttribute("operandSegmentSizes", builder.getDenseI32ArrayAttr(
+                                                 {typeparamsSize, shapeSize}));
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.addTypeToList(restype, result.types))
     return mlir::failure();
@@ -405,56 +385,6 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
   return mlir::success();
 }
 
-//===----------------------------------------------------------------------===//
-// OmpTargetAllocMemOp
-//===----------------------------------------------------------------------===//
-
-mlir::Type fir::OmpTargetAllocMemOp::getAllocatedType() {
-  return mlir::cast<fir::HeapType>(getType()).getEleTy();
-}
-
-mlir::Type fir::OmpTargetAllocMemOp::getRefTy(mlir::Type ty) {
-  return fir::HeapType::get(ty);
-}
-
-mlir::ParseResult
-fir::OmpTargetAllocMemOp::parse(mlir::OpAsmParser &parser,
-                                mlir::OperationState &result) {
-  return parseAllocatableOp(wrapAllocMemResultType, parser, result, true);
-}
-
-void fir::OmpTargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
-  p << " ";
-  p.printOperand(getDevice());
-  p << " : ";
-  p << getDevice().getType();
-  p << ", ";
-  p << getInType();
-  if (!getTypeparams().empty()) {
-    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
-  }
-  for (auto sh : getShape()) {
-    p << ", ";
-    p.printOperand(sh);
-  }
-  p.printOptionalAttrDict((*this)->getAttrs(),
-                          {"in_type", "operandSegmentSizes"});
-}
-
-llvm::LogicalResult fir::OmpTargetAllocMemOp::verify() {
-  llvm::SmallVector<llvm::StringRef> visited;
-  if (verifyInType(getInType(), visited, numShapeOperands()))
-    return emitOpError("invalid type for allocation");
-  if (verifyTypeParamCount(getInType(), numLenParams()))
-    return emitOpError("LEN params do not correspond to type");
-  mlir::Type outType = getType();
-  if (!mlir::dyn_cast<fir::HeapType>(outType))
-    return emitOpError("must be a !fir.heap type");
-  if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
-    return emitOpError("cannot allocate !fir.box of unknown rank or type");
-  return mlir::success();
-}
-
 //===----------------------------------------------------------------------===//
 // ArrayCoorOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/omp_target_allocmem.fir b/flang/test/Fir/omp_target_allocmem.fir
deleted file mode 100644
index 5140c91c9510c..0000000000000
--- a/flang/test/Fir/omp_target_allocmem.fir
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_nonchar(
-// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
-func.func @omp_target_allocmem_array_of_nonchar() -> !fir.heap<!fir.array<3x3xi32>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  return %1 : !fir.heap<!fir.array<3x3xi32>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_char(
-// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
-func.func @omp_target_allocmem_array_of_char() -> !fir.heap<!fir.array<3x3x!fir.char<1,10>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
-}
-
-// CHECK-LABEL: define ptr @omp_target_allocmem_array_of_dynchar(
-// CHECK-SAME: i32 %[[len:.*]])
-// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
-// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
-// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
-func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> !fir.heap<!fir.array<3x3x!fir.char<1,?>>> {
-  %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  return %1 : !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
-}
diff --git a/flang/test/Fir/omp_target_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
similarity index 51%
rename from flang/test/Fir/omp_target_freemem.fir
rename to flang/test/Fir/omp_target_allocmem_freemem.fir
index 02e136076a9cf..9202202728454 100644
--- a/flang/test/Fir/omp_target_freemem.fir
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -1,28 +1,33 @@
 // RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
+// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_nonchar() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3xi32>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3xi32>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3xi32>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
+// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
 // CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
 func.func @omp_target_allocmem_array_of_char() -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,10>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
+// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
 func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
   %device = arith.constant 0 : i32
-  %1 = fir.omp_target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
-  fir.omp_target_freemem %device, %1 : i32, !fir.heap<!fir.array<3x3x!fir.char<1,?>>>
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
   return
 }
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 8cf18b43450ab..e81f57f404ea0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -2113,4 +2113,68 @@ def AllocateDirOp : OpenMP_Op<"allocate_dir", clauses = [
   let hasVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Allocates memory on the specified OpenMP device for an object of the given type.
+    Returns an integer value representing the device pointer to the allocated memory.
+    The memory is uninitialized after allocation. Operations must be paired with 
+    `omp.target_freemem` to avoid memory leaks.
+
+    ```mlir
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyInteger>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyInteger>:$typeparams,
+    Variadic<AnyInteger>:$shape
+  );
+  let results = (outs I64);
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// TargetFreeMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetFreeMemOp : OpenMP_Op<"target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free memory on an openmp device";
+
+  let description = [{
+    Deallocates memory on the specified OpenMP device that was previously 
+    allocated by an `omp.target_allocmem` operation. The memory is placed 
+    in an undefined state after deallocation.
+    ```
+      %device = arith.constant 0 : i32
+      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
+      omp.target_freemem %device, %ptr : i32, i64
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyInteger, "", [MemFree]>:$device,
+  Arg<I64, "", [MemFree]>:$heapref
+  );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 769aee64e1695..dfd2f5f275a07 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3878,6 +3878,108 @@ LogicalResult AllocateDirOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type omp::TargetAllocMemOp::getAllocatedType() {
+  return getInTypeAttr().getValue();
+}
+
+/// operation ::= %res = (`omp.target_alloc_mem`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
+static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+
+  // Parse device number as a new operand
+  mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+  mlir::Type deviceType;
+  if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+    return mlir::failure();
+  if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+    return mlir::failure();
+  if (parser.parseComma())
+    return mlir::failure();
+
+  mlir::Type intype;
+  if (parser.parseType(intype))
+    return mlir::failure();
+  result.addAttribute("in_type", mlir::TypeAttr::get(intype));
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
+  llvm::SmallVector<mlir::Type> typeVec;
+  if (!parser.parseOptionalLParen()) {
+    // parse the LEN params of the derived type. (<params> : <types>)
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
+        parser.parseColonTypeList(typeVec) || parser.parseRParen())
+      return mlir::failure();
+    typeparamsSize = operands.size();
+    hasOperands = true;
+  }
+  std::int32_t shapeSize = 0;
+  if (!parser.parseOptionalComma()) {
+    // parse size to scale by, vector of n dimensions of type index
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None))
+      return mlir::failure();
+    shapeSize = operands.size() - typeparamsSize;
+    auto idxTy = builder.getIndexType();
+    for (std::int32_t i = typeparamsSize, end = operands.size(); i != end; ++i)
+      typeVec.push_back(idxTy);
+    hasOperands = true;
+  }
+  if (hasOperands &&
+      parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
+                             result.operands))
+    return mlir::failure();
+
+  mlir::Type restype = builder.getIntegerType(64);
+  ;
+  if (!restype) {
+    parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
+    return mlir::failure();
+  }
+  llvm::SmallVector<std::int32_t> segmentSizes{1, typeparamsSize, shapeSize};
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
+  if (parser.parseOptionalAttrDict(result.attributes) ||
+      parser.addTypeToList(restype, result.types))
+    return mlir::failure();
+  return mlir::success();
+}
+
+mlir::ParseResult omp::TargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  return parseTargetAllocMemOp(parser, result);
+}
+
+void omp::TargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult omp::TargetAllocMemOp::verify() {
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<IntegerType>(outType))
+    return emitOpError("must be a integer type");
+  return mlir::success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 3185f28fe6681..f805c286ec4f8 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5858,6 +5858,82 @@ static bool isTargetDeviceOp(Operation *op) {
   return false;
 }
 
+static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder,
+                                         llvm::Module *llvmModule) {
+  llvm::Type *i64Ty = builder.getInt64Ty();
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *returnType = builder.getPtrTy(0);
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(returnType, {i64Ty, i32Ty}, false);
+  llvm::Function *func = cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_alloc", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                        LLVM::ModuleTranslation &moduleTranslation) {
+  auto allocMemOp = cast<omp::TargetAllocMemOp>(opInst);
+  if (!allocMemOp)
+    return failure();
+
+  // Get "omp_target_alloc" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = allocMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the allocation size.
+  llvm::DataLayout dataLayout = llvmModule->getDataLayout();
+  mlir::Type heapTy = allocMemOp.getAllocatedType();
+  llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
+  llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
+  llvm::ConstantInt *allocSize = builder.getInt64(typeSize.getFixedValue());
+  // Create call to "omp_target_alloc" with the args as translated llvm values.
+  llvm::CallInst *call =
+      builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});
+  llvm::Value *resultI64 = builder.CreatePtrToInt(call, builder.getInt64Ty());
+
+  // Map the result
+  moduleTranslation.mapValue(allocMemOp.getResult(), resultI64);
+  return success();
+}
+
+static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder,
+                                        llvm::Module *llvmModule) {
+  llvm::Type *ptrTy = builder.getPtrTy(0);
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *voidTy = builder.getVoidTy();
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(voidTy, {ptrTy, i32Ty}, false);
+  llvm::Function *func = dyn_cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_free", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation) {
+  auto freeMemOp = cast<omp::TargetFreeMemOp>(opInst);
+  if (!freeMemOp)
+    return failure();
+
+  // Get "omp_target_free" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = freeMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the corresponding heapref value in llvm
+  mlir::Value heapref = freeMemOp.getHeapref();
+  llvm::Value *llvmHeapref = moduleTranslation.lookupValue(heapref);
+  // Convert heapref int to ptr and call "omp_target_free"
+  llvm::Value *intToPtr =
+      builder.CreateIntToPtr(llvmHeapref, builder.getPtrTy(0));
+  builder.CreateCall(ompTragetFreeFunc, {intToPtr, llvmDeviceNum});
+  return success();
+}
+
 /// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including
 /// OpenMP runtime calls).
 static LogicalResult
@@ -6032,6 +6108,12 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
             // the omp.canonical_loop.
             return applyUnrollHeuristic(op, builder, moduleTranslation);
           })
+          .Case([&](omp::TargetAllocMemOp) {
+            return convertTargetAllocMemOp(*op, builder, moduleTranslation);
+          })
+          .Case([&](omp::TargetFreeMemOp) {
+            return convertTargetFreeMemOp(*op, builder, moduleTranslation);
+          })
           .Default([&](Operation *inst) {
             return inst->emitError()
                    << "not yet implemented: " << inst->getName();
diff --git a/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
new file mode 100644
index 0000000000000..1bc97609ccff4
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt %s -convert-openmp-to-llvm | mlir-translate -mlir-to-llvmir | FileCheck %s
+
+// This file contains MLIR test cases for omp.target_allocmem and omp.target_freemem
+
+// CHECK-LABEL: test_alloc_free_i64
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 8, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_i64() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, i64
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_1d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 64, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_1d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_2d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 1024, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_2d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16x16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}

>From 43fcbe5680a14df8add324303ab8a8a0ef6f15cd Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Thu, 3 Jul 2025 21:05:35 +0530
Subject: [PATCH 5/8] Fix comments

---
 flang/lib/Optimizer/Dialect/FIROps.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 4558057bec321..958fc46c9e41c 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -107,7 +107,6 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
 }
 
 /// Parser shared by Alloca and Allocmem
-/// boolean flag isTargetOp is used to identify omp_target_allocmem
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword

>From 2d461fee214d5aa4d0cba9dcdea2ddf9c283ff2a Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 9 Jul 2025 15:49:06 +0530
Subject: [PATCH 6/8] clang format

---
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 24 ++++++++++++-------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index a04c5d7eb7ee7..14cc7bb511f0f 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -142,8 +142,8 @@ static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
           /*isVarArg=*/false));
 }
 
-static mlir::Type
-convertObjectType(const fir::LLVMTypeConverter &converter, mlir::Type firType) {
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
     return converter.convertBoxTypeAsStruct(boxTy);
   return converter.convertType(firType);
@@ -189,8 +189,9 @@ computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
 }
 
 static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                 mlir::ConversionPatternRewriter &rewriter,
-                                 mlir::Type llTy, const mlir::DataLayout &dataLayout) {
+                                      mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Type llTy,
+                                      const mlir::DataLayout &dataLayout) {
   return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
 }
 
@@ -224,8 +225,10 @@ genAllocationScaleSize(OP op, mlir::Type ity,
 }
 
 static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
-    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
-    mlir::Type ty, mlir::Value val, bool fold = false) {
+                               mlir::Location loc,
+                               mlir::ConversionPatternRewriter &rewriter,
+                               mlir::Type ty, mlir::Value val,
+                               bool fold = false) {
   auto valTy = val.getType();
   // If the value was not yet lowered, lower its type so that it can
   // be used in getPrimitiveTypeSizeInBits.
@@ -261,11 +264,13 @@ struct TargetAllocMemOpConversion
     auto ity = lowerTy().indexType();
     mlir::Type dataTy = fir::unwrapRefType(heapTy);
     mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
-    mlir::Type llvmPtrTy = mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    mlir::Type llvmPtrTy =
+        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "omp.target_allocmem codegen of derived type with length "
                 "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy, lowerTy().getDataLayout());
+    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy,
+                                          lowerTy().getDataLayout());
     if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands().drop_front())
@@ -281,7 +286,8 @@ struct TargetAllocMemOpConversion
         loc, llvmPtrTy,
         mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
         addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(
+        allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
     return mlir::success();
   }
 };

>From ca569ebe850a92c704ae4b75469ce23708c3ae3d Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Mon, 14 Jul 2025 19:12:23 +0530
Subject: [PATCH 7/8] Update TargetAllocMemOpConversion. Move utility functions
 to utils

Co-authored by @ergawy
---
 flang/include/flang/Optimizer/Support/Utils.h |  63 ++++
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 106 ++-----
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp | 148 +--------
 flang/lib/Optimizer/Support/Utils.cpp         |  51 ++++
 .../test/Fir/omp_target_allocmem_freemem.fir  | 281 +++++++++++++++++-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   4 +-
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |   1 -
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |   5 +-
 8 files changed, 427 insertions(+), 232 deletions(-)

diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index 83c936b7dcada..002261e61adb0 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -27,6 +27,8 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
 
+#include "flang/Optimizer/CodeGen/TypeConverter.h"
+
 namespace fir {
 /// Return the integer value of a arith::ConstantOp.
 inline std::int64_t toInt(mlir::arith::ConstantOp cop) {
@@ -198,6 +200,67 @@ std::optional<llvm::ArrayRef<int64_t>> getComponentLowerBoundsIfNonDefault(
     fir::RecordType recordType, llvm::StringRef component,
     mlir::ModuleOp module, const mlir::SymbolTable *symbolTable = nullptr);
 
+// Convert FIR type to LLVM without turning fir.box<T> into memory
+// reference.
+mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                             mlir::Type firType);
+
+/// Generate a LLVM constant value of type `ity`, using the provided offset.
+mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset);
+
+/// Helper function for generating the LLVM IR that computes the distance
+/// in bytes between adjacent elements pointed to by a pointer
+/// of type \p ptrTy. The result is returned as a value of \p idxTy integer
+/// type.
+mlir::Value computeElementDistance(mlir::Location loc,
+                                   mlir::Type llvmObjectType, mlir::Type idxTy,
+                                   mlir::ConversionPatternRewriter &rewriter,
+                                   const mlir::DataLayout &dataLayout);
+
+// Compute the alloc scale size (constant factors encoded in the array type).
+// We do this for arrays without a constant interior or arrays of character with
+// dynamic length arrays, since those are the only ones that get decayed to a
+// pointer to the element type.
+template <typename OP>
+inline mlir::Value
+genAllocationScaleSize(OP op, mlir::Type ity,
+                       mlir::ConversionPatternRewriter &rewriter) {
+  mlir::Location loc = op.getLoc();
+  mlir::Type dataTy = op.getInType();
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        fir::genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+/// Perform an extension or truncation as needed on an integer value. Lowering
+/// to the specific target may involve some sign-extending or truncation of
+/// values, particularly to fit them from abstract box types to the
+/// appropriate reified structures.
+mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                        mlir::Location loc,
+                        mlir::ConversionPatternRewriter &rewriter,
+                        mlir::Type ty, mlir::Value val, bool fold = false);
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_SUPPORT_UTILS_H
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 7940743763c5c..4cc1084aa56f8 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -85,14 +85,6 @@ static inline mlir::Type getI8Type(mlir::MLIRContext *context) {
   return mlir::IntegerType::get(context, 8);
 }
 
-static mlir::LLVM::ConstantOp
-genConstantIndex(mlir::Location loc, mlir::Type ity,
-                 mlir::ConversionPatternRewriter &rewriter,
-                 std::int64_t offset) {
-  auto cattr = rewriter.getI64IntegerAttr(offset);
-  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
-}
-
 static mlir::Block *createBlock(mlir::ConversionPatternRewriter &rewriter,
                                 mlir::Block *insertBefore) {
   assert(insertBefore && "expected valid insertion block");
@@ -203,39 +195,6 @@ getDependentTypeMemSizeFn(fir::RecordType recTy, fir::AllocaOp op,
   TODO(op.getLoc(), "did not find allocation function");
 }
 
-// Compute the alloc scale size (constant factors encoded in the array type).
-// We do this for arrays without a constant interior or arrays of character with
-// dynamic length arrays, since those are the only ones that get decayed to a
-// pointer to the element type.
-template <typename OP>
-static mlir::Value
-genAllocationScaleSize(OP op, mlir::Type ity,
-                       mlir::ConversionPatternRewriter &rewriter) {
-  mlir::Location loc = op.getLoc();
-  mlir::Type dataTy = op.getInType();
-  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
-  fir::SequenceType::Extent constSize = 1;
-  if (seqTy) {
-    int constRows = seqTy.getConstantRows();
-    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
-    if (constRows != static_cast<int>(shape.size())) {
-      for (auto extent : shape) {
-        if (constRows-- > 0)
-          continue;
-        if (extent != fir::SequenceType::getUnknownExtent())
-          constSize *= extent;
-      }
-    }
-  }
-
-  if (constSize != 1) {
-    mlir::Value constVal{
-        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
-    return constVal;
-  }
-  return nullptr;
-}
-
 namespace {
 struct DeclareOpConversion : public fir::FIROpConversion<fir::cg::XDeclareOp> {
 public:
@@ -270,7 +229,7 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
     auto loc = alloc.getLoc();
     mlir::Type ity = lowerTy().indexType();
     unsigned i = 0;
-    mlir::Value size = genConstantIndex(loc, ity, rewriter, 1).getResult();
+    mlir::Value size = fir::genConstantIndex(loc, ity, rewriter, 1).getResult();
     mlir::Type firObjType = fir::unwrapRefType(alloc.getType());
     mlir::Type llvmObjectType = convertObjectType(firObjType);
     if (alloc.hasLenParams()) {
@@ -302,7 +261,7 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
                << scalarType << " with type parameters";
       }
     }
-    if (auto scaleSize = genAllocationScaleSize(alloc, ity, rewriter))
+    if (auto scaleSize = fir::genAllocationScaleSize(alloc, ity, rewriter))
       size =
           rewriter.createOrFold<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     if (alloc.hasShapeOperands()) {
@@ -479,7 +438,7 @@ struct BoxIsArrayOpConversion : public fir::FIROpConversion<fir::BoxIsArrayOp> {
     auto loc = boxisarray.getLoc();
     TypePair boxTyPair = getBoxTypePair(boxisarray.getVal().getType());
     mlir::Value rank = getRankFromBox(loc, boxTyPair, a, rewriter);
-    mlir::Value c0 = genConstantIndex(loc, rank.getType(), rewriter, 0);
+    mlir::Value c0 = fir::genConstantIndex(loc, rank.getType(), rewriter, 0);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         boxisarray, mlir::LLVM::ICmpPredicate::ne, rank, c0);
     return mlir::success();
@@ -815,7 +774,7 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
       // Do folding for constant inputs.
       if (auto constVal = fir::getIntIfConstant(op0)) {
         mlir::Value normVal =
-            genConstantIndex(loc, toTy, rewriter, *constVal ? 1 : 0);
+            fir::genConstantIndex(loc, toTy, rewriter, *constVal ? 1 : 0);
         rewriter.replaceOp(convert, normVal);
         return mlir::success();
       }
@@ -828,7 +787,7 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
       }
 
       // Compare the input with zero.
-      mlir::Value zero = genConstantIndex(loc, fromTy, rewriter, 0);
+      mlir::Value zero = fir::genConstantIndex(loc, fromTy, rewriter, 0);
       auto isTrue = rewriter.create<mlir::LLVM::ICmpOp>(
           loc, mlir::LLVM::ICmpPredicate::ne, op0, zero);
 
@@ -1075,21 +1034,6 @@ static mlir::SymbolRefAttr getMalloc(fir::AllocMemOp op,
   return getMallocInModule(mod, op, rewriter, indexType);
 }
 
-/// Helper function for generating the LLVM IR that computes the distance
-/// in bytes between adjacent elements pointed to by a pointer
-/// of type \p ptrTy. The result is returned as a value of \p idxTy integer
-/// type.
-static mlir::Value
-computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
-                       mlir::Type idxTy,
-                       mlir::ConversionPatternRewriter &rewriter,
-                       const mlir::DataLayout &dataLayout) {
-  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
-  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
-  std::int64_t distance = llvm::alignTo(size, alignment);
-  return genConstantIndex(loc, idxTy, rewriter, distance);
-}
-
 /// Return value of the stride in bytes between adjacent elements
 /// of LLVM type \p llTy. The result is returned as a value of
 /// \p idxTy integer type.
@@ -1098,7 +1042,7 @@ genTypeStrideInBytes(mlir::Location loc, mlir::Type idxTy,
                      mlir::ConversionPatternRewriter &rewriter, mlir::Type llTy,
                      const mlir::DataLayout &dataLayout) {
   // Create a pointer type and use computeElementDistance().
-  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+  return fir::computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
 }
 
 namespace {
@@ -1117,7 +1061,7 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "fir.allocmem codegen of derived type with length parameters");
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
+    if (auto scaleSize = fir::genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands())
       size = rewriter.create<mlir::LLVM::MulOp>(
@@ -1140,7 +1084,8 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
   mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
                                  mlir::ConversionPatternRewriter &rewriter,
                                  mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+    return fir::computeElementDistance(loc, llTy, idxTy, rewriter,
+                                       getDataLayout());
   }
 };
 } // namespace
@@ -1324,7 +1269,7 @@ genCUFAllocDescriptor(mlir::Location loc,
   mlir::Type structTy = typeConverter.convertBoxTypeAsStruct(boxTy);
   std::size_t boxSize = dl->getTypeSizeInBits(structTy) / 8;
   mlir::Value sizeInBytes =
-      genConstantIndex(loc, llvmIntPtrType, rewriter, boxSize);
+      fir::genConstantIndex(loc, llvmIntPtrType, rewriter, boxSize);
   llvm::SmallVector args = {sizeInBytes, sourceFile, sourceLine};
   return rewriter
       .create<mlir::LLVM::CallOp>(loc, fctTy, RTNAME_STRING(CUFAllocDescriptor),
@@ -1580,7 +1525,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
       // representation of derived types with pointer/allocatable components.
       // This has been seen in hashing algorithms using TRANSFER.
       mlir::Value zero =
-          genConstantIndex(loc, rewriter.getI64Type(), rewriter, 0);
+          fir::genConstantIndex(loc, rewriter.getI64Type(), rewriter, 0);
       descriptor = insertField(rewriter, loc, descriptor,
                                {getLenParamFieldId(boxTy), 0}, zero);
     }
@@ -1923,8 +1868,8 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
     bool hasSlice = !xbox.getSlice().empty();
     unsigned sliceOffset = xbox.getSliceOperandIndex();
     mlir::Location loc = xbox.getLoc();
-    mlir::Value zero = genConstantIndex(loc, i64Ty, rewriter, 0);
-    mlir::Value one = genConstantIndex(loc, i64Ty, rewriter, 1);
+    mlir::Value zero = fir::genConstantIndex(loc, i64Ty, rewriter, 0);
+    mlir::Value one = fir::genConstantIndex(loc, i64Ty, rewriter, 1);
     mlir::Value prevPtrOff = one;
     mlir::Type eleTy = boxTy.getEleTy();
     const unsigned rank = xbox.getRank();
@@ -1973,7 +1918,7 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
         prevDimByteStride =
             getCharacterByteSize(loc, rewriter, charTy, adaptor.getLenParams());
       } else {
-        prevDimByteStride = genConstantIndex(
+        prevDimByteStride = fir::genConstantIndex(
             loc, i64Ty, rewriter,
             charTy.getLen() * lowerTy().characterBitsize(charTy) / 8);
       }
@@ -2131,7 +2076,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     if (auto charTy = mlir::dyn_cast<fir::CharacterType>(inputEleTy)) {
       if (charTy.hasConstantLen()) {
         mlir::Value len =
-            genConstantIndex(loc, idxTy, rewriter, charTy.getLen());
+            fir::genConstantIndex(loc, idxTy, rewriter, charTy.getLen());
         lenParams.emplace_back(len);
       } else {
         mlir::Value len = getElementSizeFromBox(loc, idxTy, inputBoxTyPair,
@@ -2140,7 +2085,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
           assert(!isInGlobalOp(rewriter) &&
                  "character target in global op must have constant length");
           mlir::Value width =
-              genConstantIndex(loc, idxTy, rewriter, charTy.getFKind());
+              fir::genConstantIndex(loc, idxTy, rewriter, charTy.getFKind());
           len = rewriter.create<mlir::LLVM::SDivOp>(loc, idxTy, len, width);
         }
         lenParams.emplace_back(len);
@@ -2194,8 +2139,9 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
                 mlir::ConversionPatternRewriter &rewriter) const {
     mlir::Location loc = rebox.getLoc();
     mlir::Value zero =
-        genConstantIndex(loc, lowerTy().indexType(), rewriter, 0);
-    mlir::Value one = genConstantIndex(loc, lowerTy().indexType(), rewriter, 1);
+        fir::genConstantIndex(loc, lowerTy().indexType(), rewriter, 0);
+    mlir::Value one =
+        fir::genConstantIndex(loc, lowerTy().indexType(), rewriter, 1);
     for (auto iter : llvm::enumerate(llvm::zip(extents, strides))) {
       mlir::Value extent = std::get<0>(iter.value());
       unsigned dim = iter.index();
@@ -2227,7 +2173,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     mlir::Location loc = rebox.getLoc();
     mlir::Type byteTy = ::getI8Type(rebox.getContext());
     mlir::Type idxTy = lowerTy().indexType();
-    mlir::Value zero = genConstantIndex(loc, idxTy, rewriter, 0);
+    mlir::Value zero = fir::genConstantIndex(loc, idxTy, rewriter, 0);
     // Apply subcomponent and substring shift on base address.
     if (!rebox.getSubcomponent().empty() || !rebox.getSubstr().empty()) {
       // Cast to inputEleTy* so that a GEP can be used.
@@ -2255,7 +2201,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     // and strides.
     llvm::SmallVector<mlir::Value> slicedExtents;
     llvm::SmallVector<mlir::Value> slicedStrides;
-    mlir::Value one = genConstantIndex(loc, idxTy, rewriter, 1);
+    mlir::Value one = fir::genConstantIndex(loc, idxTy, rewriter, 1);
     const bool sliceHasOrigins = !rebox.getShift().empty();
     unsigned sliceOps = rebox.getSliceOperandIndex();
     unsigned shiftOps = rebox.getShiftOperandIndex();
@@ -2328,7 +2274,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     // which may be OK if all new extents are ones, the stride does not
     // matter, use one.
     mlir::Value stride = inputStrides.empty()
-                             ? genConstantIndex(loc, idxTy, rewriter, 1)
+                             ? fir::genConstantIndex(loc, idxTy, rewriter, 1)
                              : inputStrides[0];
     for (unsigned i = 0; i < rebox.getShape().size(); ++i) {
       mlir::Value rawExtent = operands[rebox.getShapeOperandIndex() + i];
@@ -2563,9 +2509,9 @@ struct XArrayCoorOpConversion
     unsigned shiftOffset = coor.getShiftOperandIndex();
     unsigned sliceOffset = coor.getSliceOperandIndex();
     auto sliceOps = coor.getSlice().begin();
-    mlir::Value one = genConstantIndex(loc, idxTy, rewriter, 1);
+    mlir::Value one = fir::genConstantIndex(loc, idxTy, rewriter, 1);
     mlir::Value prevExt = one;
-    mlir::Value offset = genConstantIndex(loc, idxTy, rewriter, 0);
+    mlir::Value offset = fir::genConstantIndex(loc, idxTy, rewriter, 0);
     const bool isShifted = !coor.getShift().empty();
     const bool isSliced = !coor.getSlice().empty();
     const bool baseIsBoxed =
@@ -2895,7 +2841,7 @@ struct CoordinateOpConversion
         // of lower bound aspects. This both accounts for dynamically sized
         // types and non contiguous arrays.
         auto idxTy = lowerTy().indexType();
-        mlir::Value off = genConstantIndex(loc, idxTy, rewriter, 0);
+        mlir::Value off = fir::genConstantIndex(loc, idxTy, rewriter, 0);
         unsigned arrayDim = arrTy.getDimension();
         for (unsigned dim = 0; dim < arrayDim && it != end; ++dim, ++it) {
           mlir::Value stride =
@@ -3808,7 +3754,7 @@ struct IsPresentOpConversion : public fir::FIROpConversion<fir::IsPresentOp> {
       ptr = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, ptr, 0);
     }
     mlir::LLVM::ConstantOp c0 =
-        genConstantIndex(isPresent.getLoc(), idxTy, rewriter, 0);
+        fir::genConstantIndex(isPresent.getLoc(), idxTy, rewriter, 0);
     auto addr = rewriter.create<mlir::LLVM::PtrToIntOp>(loc, idxTy, ptr);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         isPresent, mlir::LLVM::ICmpPredicate::ne, addr, c0);
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 14cc7bb511f0f..c8f9c2c154b95 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -21,6 +21,7 @@
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/FatalError.h"
 #include "flang/Optimizer/Support/InternalNames.h"
+#include "flang/Optimizer/Support/Utils.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -126,130 +127,6 @@ struct PrivateClauseOpConversion
   }
 };
 
-static mlir::LLVM::LLVMFuncOp getOmpTargetAlloc(mlir::Operation *op) {
-  auto module = op->getParentOfType<mlir::ModuleOp>();
-  if (mlir::LLVM::LLVMFuncOp mallocFunc =
-          module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("omp_target_alloc"))
-    return mallocFunc;
-  mlir::OpBuilder moduleBuilder(module.getBodyRegion());
-  auto i64Ty = mlir::IntegerType::get(module->getContext(), 64);
-  auto i32Ty = mlir::IntegerType::get(module->getContext(), 32);
-  return moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
-      moduleBuilder.getUnknownLoc(), "omp_target_alloc",
-      mlir::LLVM::LLVMFunctionType::get(
-          mlir::LLVM::LLVMPointerType::get(module->getContext()),
-          {i64Ty, i32Ty},
-          /*isVarArg=*/false));
-}
-
-static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
-                                    mlir::Type firType) {
-  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
-    return converter.convertBoxTypeAsStruct(boxTy);
-  return converter.convertType(firType);
-}
-
-static llvm::SmallVector<mlir::NamedAttribute>
-addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
-                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
-                     int32_t numCallOperands) {
-  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
-  newAttrs.reserve(attrs.size() + 2);
-
-  for (mlir::NamedAttribute attr : attrs) {
-    if (attr.getName() != "operandSegmentSizes")
-      newAttrs.push_back(attr);
-  }
-
-  newAttrs.push_back(rewriter.getNamedAttr(
-      "operandSegmentSizes",
-      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
-  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
-                                           rewriter.getDenseI32ArrayAttr({})));
-  return newAttrs;
-}
-
-static mlir::LLVM::ConstantOp
-genConstantIndex(mlir::Location loc, mlir::Type ity,
-                 mlir::ConversionPatternRewriter &rewriter,
-                 std::int64_t offset) {
-  auto cattr = rewriter.getI64IntegerAttr(offset);
-  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
-}
-
-static mlir::Value
-computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
-                       mlir::Type idxTy,
-                       mlir::ConversionPatternRewriter &rewriter,
-                       const mlir::DataLayout &dataLayout) {
-  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
-  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
-  std::int64_t distance = llvm::alignTo(size, alignment);
-  return genConstantIndex(loc, idxTy, rewriter, distance);
-}
-
-static mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
-                                      mlir::ConversionPatternRewriter &rewriter,
-                                      mlir::Type llTy,
-                                      const mlir::DataLayout &dataLayout) {
-  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
-}
-
-template <typename OP>
-static mlir::Value
-genAllocationScaleSize(OP op, mlir::Type ity,
-                       mlir::ConversionPatternRewriter &rewriter) {
-  mlir::Location loc = op.getLoc();
-  mlir::Type dataTy = op.getInType();
-  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
-  fir::SequenceType::Extent constSize = 1;
-  if (seqTy) {
-    int constRows = seqTy.getConstantRows();
-    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
-    if (constRows != static_cast<int>(shape.size())) {
-      for (auto extent : shape) {
-        if (constRows-- > 0)
-          continue;
-        if (extent != fir::SequenceType::getUnknownExtent())
-          constSize *= extent;
-      }
-    }
-  }
-
-  if (constSize != 1) {
-    mlir::Value constVal{
-        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
-    return constVal;
-  }
-  return nullptr;
-}
-
-static mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
-                               mlir::Location loc,
-                               mlir::ConversionPatternRewriter &rewriter,
-                               mlir::Type ty, mlir::Value val,
-                               bool fold = false) {
-  auto valTy = val.getType();
-  // If the value was not yet lowered, lower its type so that it can
-  // be used in getPrimitiveTypeSizeInBits.
-  if (!mlir::isa<mlir::IntegerType>(valTy))
-    valTy = converter.convertType(valTy);
-  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
-  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
-  if (fold) {
-    if (toSize < fromSize)
-      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
-    if (toSize > fromSize)
-      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
-  } else {
-    if (toSize < fromSize)
-      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
-    if (toSize > fromSize)
-      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
-  }
-  return val;
-}
-
 // FIR Op specific conversion for TargetAllocMemOp
 struct TargetAllocMemOpConversion
     : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
@@ -259,19 +136,16 @@ struct TargetAllocMemOpConversion
   matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
     mlir::Type heapTy = allocmemOp.getAllocatedType();
-    mlir::LLVM::LLVMFuncOp mallocFunc = getOmpTargetAlloc(allocmemOp);
     mlir::Location loc = allocmemOp.getLoc();
     auto ity = lowerTy().indexType();
     mlir::Type dataTy = fir::unwrapRefType(heapTy);
-    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
-    mlir::Type llvmPtrTy =
-        mlir::LLVM::LLVMPointerType::get(allocmemOp.getContext(), 0);
+    mlir::Type llvmObjectTy = fir::convertObjectType(lowerTy(), dataTy);
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "omp.target_allocmem codegen of derived type with length "
                 "parameters");
-    mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy,
-                                          lowerTy().getDataLayout());
-    if (auto scaleSize = genAllocationScaleSize(allocmemOp, ity, rewriter))
+    mlir::Value size = fir::computeElementDistance(
+        loc, llvmObjectTy, ity, rewriter, lowerTy().getDataLayout());
+    if (auto scaleSize = fir::genAllocationScaleSize(allocmemOp, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands().drop_front())
       size = rewriter.create<mlir::LLVM::MulOp>(
@@ -281,13 +155,11 @@ struct TargetAllocMemOpConversion
         mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
     if (mallocTyWidth != ity.getIntOrFloatBitWidth())
       size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
-    allocmemOp->setAttr("callee", mlir::SymbolRefAttr::get(mallocFunc));
-    auto callOp = rewriter.create<mlir::LLVM::CallOp>(
-        loc, llvmPtrTy,
-        mlir::SmallVector<mlir::Value, 2>({size, allocmemOp.getDevice()}),
-        addLLVMOpBundleAttrs(rewriter, allocmemOp->getAttrs(), 2));
-    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(
-        allocmemOp, rewriter.getIntegerType(64), callOp.getResult());
+    rewriter.modifyOpInPlace(allocmemOp, [&]() {
+      allocmemOp.setInType(rewriter.getI8Type());
+      allocmemOp.getTypeparamsMutable().clear();
+      allocmemOp.getTypeparamsMutable().append(size);
+    });
     return mlir::success();
   }
 };
diff --git a/flang/lib/Optimizer/Support/Utils.cpp b/flang/lib/Optimizer/Support/Utils.cpp
index 5d663e28336c0..dbd42285e8fae 100644
--- a/flang/lib/Optimizer/Support/Utils.cpp
+++ b/flang/lib/Optimizer/Support/Utils.cpp
@@ -50,3 +50,54 @@ std::optional<llvm::ArrayRef<int64_t>> fir::getComponentLowerBoundsIfNonDefault(
       return componentInfo.getLowerBounds();
   return std::nullopt;
 }
+
+mlir::Type fir::convertObjectType(const fir::LLVMTypeConverter &converter,
+                                  mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+mlir::LLVM::ConstantOp
+fir::genConstantIndex(mlir::Location loc, mlir::Type ity,
+                      mlir::ConversionPatternRewriter &rewriter,
+                      std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+mlir::Value
+fir::computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                            mlir::Type idxTy,
+                            mlir::ConversionPatternRewriter &rewriter,
+                            const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return fir::genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+mlir::Value fir::integerCast(const fir::LLVMTypeConverter &converter,
+                             mlir::Location loc,
+                             mlir::ConversionPatternRewriter &rewriter,
+                             mlir::Type ty, mlir::Value val, bool fold) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
diff --git a/flang/test/Fir/omp_target_allocmem_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
index 9202202728454..03eb94acb1ac7 100644
--- a/flang/test/Fir/omp_target_allocmem_freemem.fir
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -1,8 +1,106 @@
 // RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
 
-// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar(
-// CHECK: call ptr @omp_target_alloc(i64 36, i32 0)
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// UNSUPPORTED: system-windows
+// Disabled on 32-bit targets due to the additional `trunc` opcodes required
+// UNSUPPORTED: target-x86
+// UNSUPPORTED: target=sparc-{{.*}}
+// UNSUPPORTED: target=sparcel-{{.*}}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 4, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, i32
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalars_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 400, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalars_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %0 = arith.constant 100 : index
+  %1 = omp.target_allocmem %device : i32, i32, %0
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_char() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 10, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<1,10>
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_char_kind() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 20, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_char_kind() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<2,10>
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_dynchar(
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_dynchar(%l : i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<1,?>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_dynchar_kind(
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 2, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_dynchar_kind(%l : i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<2,?>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 36, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
 func.func @omp_target_allocmem_array_of_nonchar() -> () {
   %device = arith.constant 0 : i32
   %1 = omp.target_allocmem %device : i32, !fir.array<3x3xi32>
@@ -10,9 +108,12 @@ func.func @omp_target_allocmem_array_of_nonchar() -> () {
   return
 }
 
-// CHECK-LABEL: define void @omp_target_allocmem_array_of_char(
-// CHECK: call ptr @omp_target_alloc(i64 90, i32 0)
-// CHECK: call void @omp_target_free(ptr {{.*}}, i32 0)
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_char() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 90, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
 func.func @omp_target_allocmem_array_of_char() -> () {
   %device = arith.constant 0 : i32
   %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
@@ -21,13 +122,173 @@ func.func @omp_target_allocmem_array_of_char() -> () {
 }
 
 // CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
-// CHECK-SAME: i32 %[[len:.*]])
-// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
-// CHECK: %[[mul2:.*]] = mul i64 9, %[[mul1]]
-// CHECK: call ptr @omp_target_alloc(i64 %[[mul2]], i32 0)
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 9, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
 func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
   %device = arith.constant 0 : i32
   %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
   omp.target_freemem %device, %1 : i32, i64
   return
 }
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_nonchar(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 12, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_nonchar(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?xi32>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_nonchar2(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 4, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_nonchar2(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?xi32>, %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_char(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 60, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_char(%e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x!fir.char<2,10>>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_char2(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 20, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_char2(%e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?x!fir.char<2,10>>, %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_dynchar(
+// CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 6, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP1]]
+// CHECK-NEXT:    [[TMP6:%.*]] = mul i64 1, [[TMP5]]
+// CHECK-NEXT:    [[TMP7:%.*]] = call ptr @omp_target_alloc(i64 [[TMP6]], i32 0)
+// CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[TMP7]] to i64
+// CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP9]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_dynchar(%l: i32, %e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x!fir.char<2,?>>(%l : i32), %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_dynchar2(
+// CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 2, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP1]]
+// CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP1]]
+// CHECK-NEXT:    [[TMP7:%.*]] = mul i64 1, [[TMP6]]
+// CHECK-NEXT:    [[TMP8:%.*]] = call ptr @omp_target_alloc(i64 [[TMP7]], i32 0)
+// CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[TMP8]] to i64
+// CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP10]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_dynchar2(%l: i32, %e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?x!fir.char<2,?>>(%l : i32), %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_nonchar(
+// CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 240, [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 1, [[TMP4]]
+// CHECK-NEXT:    [[TMP6:%.*]] = call ptr @omp_target_alloc(i64 [[TMP5]], i32 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+// CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP8]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_nonchar(%0 : index, %1 : index) -> () {
+  %device = arith.constant 0 : i32
+  %2 = omp.target_allocmem %device : i32, !fir.array<4x?x3x?x5xi32>, %0, %1
+  omp.target_freemem %device, %2 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_char(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 240, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_char(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x4x!fir.char<2,10>>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_dynchar(
+// CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 24, [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 1, [[TMP4]]
+// CHECK-NEXT:    [[TMP6:%.*]] = call ptr @omp_target_alloc(i64 [[TMP5]], i32 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+// CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP8]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_dynchar(%arg0: index, %arg1: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x4x!fir.char<2,?>>(%arg0 : index), %arg1
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index e81f57f404ea0..f81e8eac97234 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -2138,8 +2138,8 @@ def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
     TypeAttr:$in_type,
     OptionalAttr<StrAttr>:$uniq_name,
     OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyInteger>:$typeparams,
-    Variadic<AnyInteger>:$shape
+    Variadic<IntLikeType>:$typeparams,
+    Variadic<IntLikeType>:$shape
   );
   let results = (outs I64);
 
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index dfd2f5f275a07..49a26d8cd156f 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3936,7 +3936,6 @@ static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,
     return mlir::failure();
 
   mlir::Type restype = builder.getIntegerType(64);
-  ;
   if (!restype) {
     parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
     return mlir::failure();
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index f805c286ec4f8..145433faaf5c9 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5888,7 +5888,10 @@ convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
   mlir::Type heapTy = allocMemOp.getAllocatedType();
   llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
   llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
-  llvm::ConstantInt *allocSize = builder.getInt64(typeSize.getFixedValue());
+  llvm::Value *allocSize = builder.getInt64(typeSize.getFixedValue());
+  for (auto typeParam : allocMemOp.getTypeparams())
+    allocSize =
+        builder.CreateMul(allocSize, moduleTranslation.lookupValue(typeParam));
   // Create call to "omp_target_alloc" with the args as translated llvm values.
   llvm::CallInst *call =
       builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});

>From 6ef5914b24fcb43c1a5cacad65a766309f2f7020 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 16 Jul 2025 21:50:32 +0530
Subject: [PATCH 8/8] [OpenMP] Update description for target_allocmem and
 target_freemem.

---
 flang/include/flang/Optimizer/Support/Utils.h | 31 ++---------
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 36 +++++++------
 flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp |  3 +-
 flang/lib/Optimizer/Support/Utils.cpp         | 27 ++++++++++
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 52 +++++++++++++++----
 5 files changed, 93 insertions(+), 56 deletions(-)

diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index 002261e61adb0..ae1efb9833649 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -224,34 +224,9 @@ mlir::Value computeElementDistance(mlir::Location loc,
 // We do this for arrays without a constant interior or arrays of character with
 // dynamic length arrays, since those are the only ones that get decayed to a
 // pointer to the element type.
-template <typename OP>
-inline mlir::Value
-genAllocationScaleSize(OP op, mlir::Type ity,
-                       mlir::ConversionPatternRewriter &rewriter) {
-  mlir::Location loc = op.getLoc();
-  mlir::Type dataTy = op.getInType();
-  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
-  fir::SequenceType::Extent constSize = 1;
-  if (seqTy) {
-    int constRows = seqTy.getConstantRows();
-    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
-    if (constRows != static_cast<int>(shape.size())) {
-      for (auto extent : shape) {
-        if (constRows-- > 0)
-          continue;
-        if (extent != fir::SequenceType::getUnknownExtent())
-          constSize *= extent;
-      }
-    }
-  }
-
-  if (constSize != 1) {
-    mlir::Value constVal{
-        fir::genConstantIndex(loc, ity, rewriter, constSize).getResult()};
-    return constVal;
-  }
-  return nullptr;
-}
+mlir::Value genAllocationScaleSize(mlir::Location loc, mlir::Type dataTy,
+                                   mlir::Type ity,
+                                   mlir::ConversionPatternRewriter &rewriter);
 
 /// Perform an extension or truncation as needed on an integer value. Lowering
 /// to the specific target may involve some sign-extending or truncation of
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 4cc1084aa56f8..50c1765b12409 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -261,7 +261,8 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
                << scalarType << " with type parameters";
       }
     }
-    if (auto scaleSize = fir::genAllocationScaleSize(alloc, ity, rewriter))
+    if (auto scaleSize = fir::genAllocationScaleSize(
+            alloc.getLoc(), alloc.getInType(), ity, rewriter))
       size =
           rewriter.createOrFold<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     if (alloc.hasShapeOperands()) {
@@ -1061,7 +1062,8 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "fir.allocmem codegen of derived type with length parameters");
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = fir::genAllocationScaleSize(heap, ity, rewriter))
+    if (auto scaleSize =
+            fir::genAllocationScaleSize(loc, heap.getInType(), ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands())
       size = rewriter.create<mlir::LLVM::MulOp>(
@@ -4274,20 +4276,22 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, VolatileCastOpConversion, ConvertOpConversion,
       CoordinateOpConversion, CopyOpConversion, DTEntryOpConversion,
-      DeclareOpConversion, DivcOpConversion, EmboxOpConversion,
-      EmboxCharOpConversion, EmboxProcOpConversion, ExtractValueOpConversion,
-      FieldIndexOpConversion, FirEndOpConversion, FreeMemOpConversion,
-      GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
-      IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
-      LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
-      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
-      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
-      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
-      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
-      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
-      XArrayCoorOpConversion, XEmboxOpConversion, XReboxOpConversion,
-      ZeroOpConversion>(converter, options);
+      DeclareOpConversion,
+      DoConcurrentSpecifierOpConversion<fir::LocalitySpecifierOp>,
+      DoConcurrentSpecifierOpConversion<fir::DeclareReductionOp>,
+      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
+      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
+      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
+      GlobalOpConversion, InsertOnRangeOpConversion, IsPresentOpConversion,
+      LenParamIndexOpConversion, LoadOpConversion, MulcOpConversion,
+      NegcOpConversion, NoReassocOpConversion, SelectCaseOpConversion,
+      SelectOpConversion, SelectRankOpConversion, SelectTypeOpConversion,
+      ShapeOpConversion, ShapeShiftOpConversion, ShiftOpConversion,
+      SliceOpConversion, StoreOpConversion, StringLitOpConversion,
+      SubcOpConversion, TypeDescOpConversion, TypeInfoOpConversion,
+      UnboxCharOpConversion, UnboxProcOpConversion, UndefOpConversion,
+      UnreachableOpConversion, XArrayCoorOpConversion, XEmboxOpConversion,
+      XReboxOpConversion, ZeroOpConversion>(converter, options);
 
   // Patterns that are populated without a type converter do not trigger
   // target materializations for the operands of the root op.
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index c8f9c2c154b95..b2c6b880c6f52 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -145,7 +145,8 @@ struct TargetAllocMemOpConversion
                 "parameters");
     mlir::Value size = fir::computeElementDistance(
         loc, llvmObjectTy, ity, rewriter, lowerTy().getDataLayout());
-    if (auto scaleSize = fir::genAllocationScaleSize(allocmemOp, ity, rewriter))
+    if (auto scaleSize = fir::genAllocationScaleSize(
+            loc, allocmemOp.getInType(), ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands().drop_front())
       size = rewriter.create<mlir::LLVM::MulOp>(
diff --git a/flang/lib/Optimizer/Support/Utils.cpp b/flang/lib/Optimizer/Support/Utils.cpp
index dbd42285e8fae..6dc80ff8d18a6 100644
--- a/flang/lib/Optimizer/Support/Utils.cpp
+++ b/flang/lib/Optimizer/Support/Utils.cpp
@@ -77,6 +77,33 @@ fir::computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
   return fir::genConstantIndex(loc, idxTy, rewriter, distance);
 }
 
+mlir::Value
+fir::genAllocationScaleSize(mlir::Location loc, mlir::Type dataTy,
+                            mlir::Type ity,
+                            mlir::ConversionPatternRewriter &rewriter) {
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        fir::genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
 mlir::Value fir::integerCast(const fir::LLVMTypeConverter &converter,
                              mlir::Location loc,
                              mlir::ConversionPatternRewriter &rewriter,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index f81e8eac97234..57ddc41e4ed9b 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -2127,10 +2127,31 @@ def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
     The memory is uninitialized after allocation. Operations must be paired with 
     `omp.target_freemem` to avoid memory leaks.
 
-    ```mlir
-      %device = arith.constant 0 : i32
-      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
-    ```
+    * `$device`: The integer ID of the OpenMP device where the memory will be allocated.
+    * `$in_type`: The type of the object for which memory is being allocated. 
+      For arrays, this can be a static or dynamic array type.
+    * `$uniq_name`: An optional unique name for the allocated memory.
+    * `$bindc_name`: An optional name used for C interoperability.
+    * `$typeparams`: Runtime type parameters for polymorphic or parameterized types. 
+      These are typically integer values that define aspects of a type not fixed at compile time.
+    * `$shape`: Runtime shape operands for dynamic arrays. 
+      Each operand is an integer value representing the extent of a specific dimension. 
+
+  ```mlir
+    // Allocate a static 3x3 integer vector on device 0
+    %device_0 = arith.constant 0 : i32
+    %ptr_static = omp.target_allocmem %device_0 : i32, vector<3x3xi32>
+    // ... use %ptr_static ...
+    omp.target_freemem %device_0, %ptr_static : i32, i64
+
+    // Allocate a dynamic 2D Fortran array (fir.array) on device 1
+    %device_1 = arith.constant 1 : i32
+    %rows = arith.constant 10 : index
+    %cols = arith.constant 20 : index
+    %ptr_dynamic = omp.target_allocmem %device_1 : i32, !fir.array<?x?xf32>, %rows, %cols : index, index
+    // ... use %ptr_dynamic ...
+    omp.target_freemem %device_1, %ptr_dynamic : i32, i64
+  ```
   }];
 
   let arguments = (ins
@@ -2160,13 +2181,22 @@ def TargetFreeMemOp : OpenMP_Op<"target_freemem",
   let summary = "free memory on an openmp device";
 
   let description = [{
-    Deallocates memory on the specified OpenMP device that was previously 
-    allocated by an `omp.target_allocmem` operation. The memory is placed 
-    in an undefined state after deallocation.
-    ```
-      %device = arith.constant 0 : i32
-      %ptr = omp.target_allocmem %device : i32, vector<3x3xi32>
-      omp.target_freemem %device, %ptr : i32, i64
+    Deallocates memory on the specified OpenMP device that was previously
+    allocated by an `omp.target_allocmem` operation. After this operation, the
+    deallocated memory is in an undefined state and should not be accessed.
+    It is crucial to ensure that all accesses to the memory region are completed
+    before `omp.target_freemem` is called to avoid undefined behavior.
+
+    * `$device`: The integer ID of the OpenMP device from which the memory will be freed.
+    * `$heapref`: The integer value representing the device pointer to the memory
+      to be deallocated, which was previously returned by `omp.target_allocmem`.
+
+    ```mlir
+      // Example of allocating and freeing memory on an OpenMP device
+      %device_id = arith.constant 0 : i32
+      %allocated_ptr = omp.target_allocmem %device_id : i32, vector<3x3xi32>
+      // ... operations using %allocated_ptr on the device ...
+      omp.target_freemem %device_id, %allocated_ptr : i32, i64
     ```
   }];
 


From flang-commits at lists.llvm.org  Wed Jul 16 10:08:15 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 10:08:15 -0700 (PDT)
Subject: [flang-commits] [flang] 8349bbd - [flang][cuda] Exit early when
 there is no device components (#149005)
Message-ID: <6877dc7f.050a0220.16280b.7c90@mx.google.com>


Author: Valentin Clement (バレンタイン クレメン)
Date: 2025-07-16T10:08:11-07:00
New Revision: 8349bbd0b98c84836d55593c7eb035c2b0f4e87a

URL: https://github.com/llvm/llvm-project/commit/8349bbd0b98c84836d55593c7eb035c2b0f4e87a
DIFF: https://github.com/llvm/llvm-project/commit/8349bbd0b98c84836d55593c7eb035c2b0f4e87a.diff

LOG: [flang][cuda] Exit early when there is no device components (#149005)

- Exit early when there is no device components
- Make the retrieval of the record type more robust

Added: 
    

Modified: 
    flang/lib/Lower/ConvertVariable.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 2bfa9618aa4b9..6c4516686f9d0 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -802,13 +802,20 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
     const Fortran::semantics::DerivedTypeSpec *derived{type ? type->AsDerived()
                                                             : nullptr};
     if (derived) {
+      if (!FindCUDADeviceAllocatableUltimateComponent(*derived))
+        return; // No device components.
+
       fir::FirOpBuilder &builder = converter.getFirOpBuilder();
       mlir::Location loc = converter.getCurrentLocation();
 
       fir::ExtendedValue exv =
           converter.getSymbolExtendedValue(symbol.GetUltimate(), &symMap);
-      auto recTy = mlir::dyn_cast<fir::RecordType>(
-          fir::unwrapRefType(fir::getBase(exv).getType()));
+      mlir::Type baseTy = fir::unwrapRefType(fir::getBase(exv).getType());
+      if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(baseTy))
+        baseTy = boxTy.getEleTy();
+      baseTy = fir::unwrapRefType(baseTy);
+      auto recTy =
+          mlir::dyn_cast<fir::RecordType>(fir::unwrapSequenceType(baseTy));
       assert(recTy && "expected fir::RecordType");
 
       llvm::SmallVector<mlir::Value> coordinates;


        

From flang-commits at lists.llvm.org  Wed Jul 16 10:08:18 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Wed, 16 Jul 2025 10:08:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Exit early when there is no
 device components (PR #149005)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149005@github.com>
Message-ID: <6877dc82.170a0220.32d753.bd77@mx.google.com>


https://github.com/clementval closed https://github.com/llvm/llvm-project/pull/149005

From flang-commits at lists.llvm.org  Wed Jul 16 10:14:29 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 10:14:29 -0700 (PDT)
Subject: [flang-commits] [flang] 8d21025 - [flang] Parallel runtime library
	design doc (PRIF) (#76088)
Message-ID: <6877ddf5.630a0220.282ce6.5a9e@mx.google.com>


Author: Brad Richardson
Date: 2025-07-16T10:14:26-07:00
New Revision: 8d21025c3aeb1c98caef08f8446ec138c62288d1

URL: https://github.com/llvm/llvm-project/commit/8d21025c3aeb1c98caef08f8446ec138c62288d1
DIFF: https://github.com/llvm/llvm-project/commit/8d21025c3aeb1c98caef08f8446ec138c62288d1.diff

LOG: [flang] Parallel runtime library design doc (PRIF) (#76088)

The Parallel Runtime Interface for Fortran (PRIF) specifies the interface design for 
supporting Fortran's multi-image parallel features in flang.

---------

Co-authored-by: Katherine Rasmussen <krasmussen at lbl.gov>
Co-authored-by: Damian Rouson <rouson at lbl.gov>
Co-authored-by: Dan Bonachea <dobonachea at lbl.gov>

Added: 
    flang/docs/ParallelMultiImageFortranRuntime.md

Modified: 
    flang/docs/index.md

Removed: 
    


################################################################################
diff  --git a/flang/docs/ParallelMultiImageFortranRuntime.md b/flang/docs/ParallelMultiImageFortranRuntime.md
new file mode 100644
index 0000000000000..8cf0055e5817b
--- /dev/null
+++ b/flang/docs/ParallelMultiImageFortranRuntime.md
@@ -0,0 +1,18 @@
+<!--===- docs/ParallelMultiImageFortranRuntime.md
+  
+   Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+   See https://llvm.org/LICENSE.txt for license information.
+   SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+  
+-->
+
+# Multi-Image Parallel Fortran Runtime
+
+
+The Parallel Runtime Interface for Fortran (PRIF) defines an 
+interface designed for LLVM Flang to target implementations of 
+Fortran's multi-image parallel features.
+
+The current revision of the PRIF specification is here:
+<https://doi.org/10.25344/S4CG6G>
+

diff  --git a/flang/docs/index.md b/flang/docs/index.md
index 2568ad70c5d09..016577bcb1e98 100644
--- a/flang/docs/index.md
+++ b/flang/docs/index.md
@@ -78,6 +78,7 @@ on how to get in touch with us and to learn more about the current status.
    OpenMP-semantics
    OptionComparison
    Overview
+   ParallelMultiImageFortranRuntime
    ParameterizedDerivedTypes
    ParserCombinators
    Parsing


        

From flang-commits at lists.llvm.org  Wed Jul 16 10:14:33 2025
From: flang-commits at lists.llvm.org (Katherine Rasmussen via flang-commits)
Date: Wed, 16 Jul 2025 10:14:33 -0700 (PDT)
Subject: [flang-commits] [flang] Parallel runtime library design doc (PRIF)
	(PR #76088)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/76088@github.com>
Message-ID: <6877ddf9.050a0220.39d324.7973@mx.google.com>


https://github.com/ktras closed https://github.com/llvm/llvm-project/pull/76088

From flang-commits at lists.llvm.org  Wed Jul 16 10:22:27 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 10:22:27 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
Message-ID: <llvm.org/llvm/llvm-project/pull/149147@github.com>


https://github.com/delaram-talaashrafi created https://github.com/llvm/llvm-project/pull/149147

Based on the OpenACC specification — which states that if the bind name is given as an identifier it should be resolved according to the compiled language, and if given as a string it should be used unmodified — we introduce two distinct `bindName` representations for `acc routine` to handle each case appropriately: one as an array of `SymbolRefAttr` for identifiers and another as an array of `StringAttr` for strings.

To ensure correct correspondence between bind names and devices, this patch also introduces two separate sets of device attributes. The routine operation is extended accordingly, along with the necessary updates to the OpenACC dialect and its lowering.

>From f61dd65b3cb04195edee58f2f6f3f0fc57cbfc45 Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Wed, 16 Jul 2025 10:12:26 -0700
Subject: [PATCH] [openacc][flang] Support SymbolRef and String bindName
 representations and device attribute in acc.routine
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Based on the OpenACC specification — which states that if the bind name is given as an identifier it should
be resolved according to the compiled language, and if given as a string it should be used unmodified — we
introduce two distinct `bindName` representations for `acc routine` to handle each case appropriately:
one as an array of `SymbolRefAttr` for identifiers and another as an array of `StringAttr` for strings.

To ensure correct correspondence between bind names and devices, this patch also introduces two separate
sets of device attributes. The routine operation is extended accordingly, along with the necessary updates
to the OpenACC dialect and its lowering.
---
 flang/lib/Lower/OpenACC.cpp                   | 103 +++++++++++-----
 flang/test/Lower/OpenACC/acc-routine.f90      |   7 +-
 flang/test/Lower/OpenACC/acc-routine03.f90    |   2 +-
 mlir/include/mlir/Dialect/OpenACC/OpenACC.h   |   1 +
 .../mlir/Dialect/OpenACC/OpenACCOps.td        |  12 +-
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       | 113 ++++++++++++++----
 .../Dialect/OpenACC/OpenACCOpsTest.cpp        |  44 +++++--
 7 files changed, 210 insertions(+), 72 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 39e4444cde4e3..93664dc5d8c2d 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4396,10 +4396,35 @@ getAttributeValueByDeviceType(llvm::SmallVector<mlir::Attribute> &attributes,
   return std::nullopt;
 }
 
+// Helper function to extract string value from bind name variant
+static std::optional<llvm::StringRef> getBindNameStringValue(
+    const std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+        &bindNameValue) {
+  if (!bindNameValue.has_value()) {
+    return std::nullopt;
+  }
+
+  return std::visit(
+      [](const auto &attr) -> std::optional<llvm::StringRef> {
+        if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                     mlir::StringAttr>) {
+          return attr.getValue();
+        } else if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                            mlir::SymbolRefAttr>) {
+          return attr.getLeafReference();
+        } else {
+          return std::nullopt;
+        }
+      },
+      bindNameValue.value());
+}
+
 static bool compareDeviceTypeInfo(
     mlir::acc::RoutineOp op,
-    llvm::SmallVector<mlir::Attribute> &bindNameArrayAttr,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypeArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypeArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypeArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangDimArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypeArrayAttr,
@@ -4409,9 +4434,13 @@ static bool compareDeviceTypeInfo(
   for (uint32_t dtypeInt = 0;
        dtypeInt != mlir::acc::getMaxEnumValForDeviceType(); ++dtypeInt) {
     auto dtype = static_cast<mlir::acc::DeviceType>(dtypeInt);
-    if (op.getBindNameValue(dtype) !=
-        getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
-            bindNameArrayAttr, bindNameDeviceTypeArrayAttr, dtype))
+    auto bindNameValue = getBindNameStringValue(op.getBindNameValue(dtype));
+    if (bindNameValue !=
+            getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
+                bindIdNameArrayAttr, bindIdNameDeviceTypeArrayAttr, dtype) &&
+        bindNameValue !=
+            getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
+                bindStrNameArrayAttr, bindStrNameDeviceTypeArrayAttr, dtype))
       return false;
     if (op.hasGang(dtype) != hasDeviceType(gangArrayAttr, dtype))
       return false;
@@ -4458,8 +4487,10 @@ getArrayAttrOrNull(fir::FirOpBuilder &builder,
 void createOpenACCRoutineConstruct(
     Fortran::lower::AbstractConverter &converter, mlir::Location loc,
     mlir::ModuleOp mod, mlir::func::FuncOp funcOp, std::string funcName,
-    bool hasNohost, llvm::SmallVector<mlir::Attribute> &bindNames,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypes,
+    bool hasNohost, llvm::SmallVector<mlir::Attribute> &bindIdNames,
+    llvm::SmallVector<mlir::Attribute> &bindStrNames,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDimValues,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypes,
@@ -4472,7 +4503,8 @@ void createOpenACCRoutineConstruct(
         0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
-      if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
+      if (compareDeviceTypeInfo(routineOp, bindIdNames, bindStrNames,
+                                bindIdNameDeviceTypes, bindStrNameDeviceTypes,
                                 gangDeviceTypes, gangDimValues,
                                 gangDimDeviceTypes, seqDeviceTypes,
                                 workerDeviceTypes, vectorDeviceTypes) &&
@@ -4489,8 +4521,10 @@ void createOpenACCRoutineConstruct(
   modBuilder.create<mlir::acc::RoutineOp>(
       loc, routineOpStr,
       mlir::SymbolRefAttr::get(builder.getContext(), funcName),
-      getArrayAttrOrNull(builder, bindNames),
-      getArrayAttrOrNull(builder, bindNameDeviceTypes),
+      getArrayAttrOrNull(builder, bindIdNames),
+      getArrayAttrOrNull(builder, bindStrNames),
+      getArrayAttrOrNull(builder, bindIdNameDeviceTypes),
+      getArrayAttrOrNull(builder, bindStrNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),
       getArrayAttrOrNull(builder, seqDeviceTypes), hasNohost,
@@ -4507,8 +4541,10 @@ static void interpretRoutineDeviceInfo(
     llvm::SmallVector<mlir::Attribute> &seqDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &workerDeviceTypes,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypes,
-    llvm::SmallVector<mlir::Attribute> &bindNames,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindIdNames,
+    llvm::SmallVector<mlir::Attribute> &bindStrNames,
     llvm::SmallVector<mlir::Attribute> &gangDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDimValues,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypes) {
@@ -4541,16 +4577,18 @@ static void interpretRoutineDeviceInfo(
   if (dinfo.bindNameOpt().has_value()) {
     const auto &bindName = dinfo.bindNameOpt().value();
     mlir::Attribute bindNameAttr;
-    if (const auto &bindStr{std::get_if<std::string>(&bindName)}) {
+    if (const auto &bindSym{
+            std::get_if<Fortran::semantics::SymbolRef>(&bindName)}) {
+      bindNameAttr = builder.getSymbolRefAttr(converter.mangleName(*bindSym));
+      bindIdNames.push_back(bindNameAttr);
+      bindIdNameDeviceTypes.push_back(getDeviceTypeAttr());
+    } else if (const auto &bindStr{std::get_if<std::string>(&bindName)}) {
       bindNameAttr = builder.getStringAttr(*bindStr);
-    } else if (const auto &bindSym{
-                   std::get_if<Fortran::semantics::SymbolRef>(&bindName)}) {
-      bindNameAttr = builder.getStringAttr(converter.mangleName(*bindSym));
+      bindStrNames.push_back(bindNameAttr);
+      bindStrNameDeviceTypes.push_back(getDeviceTypeAttr());
     } else {
       llvm_unreachable("Unsupported bind name type");
     }
-    bindNames.push_back(bindNameAttr);
-    bindNameDeviceTypes.push_back(getDeviceTypeAttr());
   }
 }
 
@@ -4566,8 +4604,9 @@ void Fortran::lower::genOpenACCRoutineConstruct(
   bool hasNohost{false};
 
   llvm::SmallVector<mlir::Attribute> seqDeviceTypes, vectorDeviceTypes,
-      workerDeviceTypes, bindNameDeviceTypes, bindNames, gangDeviceTypes,
-      gangDimDeviceTypes, gangDimValues;
+      workerDeviceTypes, bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+      bindIdNames, bindStrNames, gangDeviceTypes, gangDimDeviceTypes,
+      gangDimValues;
 
   for (const Fortran::semantics::OpenACCRoutineInfo &info : routineInfos) {
     // Device Independent Attributes
@@ -4576,24 +4615,26 @@ void Fortran::lower::genOpenACCRoutineConstruct(
     }
     // Note: Device Independent Attributes are set to the
     // none device type in `info`.
-    interpretRoutineDeviceInfo(converter, info, seqDeviceTypes,
-                               vectorDeviceTypes, workerDeviceTypes,
-                               bindNameDeviceTypes, bindNames, gangDeviceTypes,
-                               gangDimValues, gangDimDeviceTypes);
+    interpretRoutineDeviceInfo(
+        converter, info, seqDeviceTypes, vectorDeviceTypes, workerDeviceTypes,
+        bindIdNameDeviceTypes, bindStrNameDeviceTypes, bindIdNames,
+        bindStrNames, gangDeviceTypes, gangDimValues, gangDimDeviceTypes);
 
     // Device Dependent Attributes
     for (const Fortran::semantics::OpenACCRoutineDeviceTypeInfo &dinfo :
          info.deviceTypeInfos()) {
-      interpretRoutineDeviceInfo(
-          converter, dinfo, seqDeviceTypes, vectorDeviceTypes,
-          workerDeviceTypes, bindNameDeviceTypes, bindNames, gangDeviceTypes,
-          gangDimValues, gangDimDeviceTypes);
+      interpretRoutineDeviceInfo(converter, dinfo, seqDeviceTypes,
+                                 vectorDeviceTypes, workerDeviceTypes,
+                                 bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+                                 bindIdNames, bindStrNames, gangDeviceTypes,
+                                 gangDimValues, gangDimDeviceTypes);
     }
   }
   createOpenACCRoutineConstruct(
-      converter, loc, mod, funcOp, funcName, hasNohost, bindNames,
-      bindNameDeviceTypes, gangDeviceTypes, gangDimValues, gangDimDeviceTypes,
-      seqDeviceTypes, workerDeviceTypes, vectorDeviceTypes);
+      converter, loc, mod, funcOp, funcName, hasNohost, bindIdNames,
+      bindStrNames, bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+      gangDeviceTypes, gangDimValues, gangDimDeviceTypes, seqDeviceTypes,
+      workerDeviceTypes, vectorDeviceTypes);
 }
 
 static void
diff --git a/flang/test/Lower/OpenACC/acc-routine.f90 b/flang/test/Lower/OpenACC/acc-routine.f90
index 789f3a57e1f79..1a63b4120235c 100644
--- a/flang/test/Lower/OpenACC/acc-routine.f90
+++ b/flang/test/Lower/OpenACC/acc-routine.f90
@@ -2,13 +2,14 @@
 
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
 
-! CHECK: acc.routine @[[r14:.*]] func(@_QPacc_routine19) bind("_QPacc_routine17" [#acc.device_type<host>], "_QPacc_routine17" [#acc.device_type<default>], "_QPacc_routine16" [#acc.device_type<multicore>])
-! CHECK: acc.routine @[[r13:.*]] func(@_QPacc_routine18) bind("_QPacc_routine17" [#acc.device_type<host>], "_QPacc_routine16" [#acc.device_type<multicore>])
+! CHECK: acc.routine @[[r14:.*]] func(@_QPacc_routine19) bind(@_QPacc_routine17 [#acc.device_type<host>], @_QPacc_routine17
+! [#acc.device_type<default>], @_QPacc_routine16 [#acc.device_type<multicore>])
+! CHECK: acc.routine @[[r13:.*]] func(@_QPacc_routine18) bind(@_QPacc_routine17 [#acc.device_type<host>], @_QPacc_routine16 [#acc.device_type<multicore>])
 ! CHECK: acc.routine @[[r12:.*]] func(@_QPacc_routine17) worker ([#acc.device_type<host>]) vector ([#acc.device_type<multicore>])
 ! CHECK: acc.routine @[[r11:.*]] func(@_QPacc_routine16) gang([#acc.device_type<nvidia>]) seq ([#acc.device_type<host>])
 ! CHECK: acc.routine @[[r10:.*]] func(@_QPacc_routine11) seq
 ! CHECK: acc.routine @[[r09:.*]] func(@_QPacc_routine10) seq
-! CHECK: acc.routine @[[r08:.*]] func(@_QPacc_routine9) bind("_QPacc_routine9a")
+! CHECK: acc.routine @[[r08:.*]] func(@_QPacc_routine9) bind(@_QPacc_routine9a)
 ! CHECK: acc.routine @[[r07:.*]] func(@_QPacc_routine8) bind("routine8_")
 ! CHECK: acc.routine @[[r06:.*]] func(@_QPacc_routine7) gang(dim: 1 : i64)
 ! CHECK: acc.routine @[[r05:.*]] func(@_QPacc_routine6) nohost
diff --git a/flang/test/Lower/OpenACC/acc-routine03.f90 b/flang/test/Lower/OpenACC/acc-routine03.f90
index 85e4ef580f983..ddd6bda0367e4 100644
--- a/flang/test/Lower/OpenACC/acc-routine03.f90
+++ b/flang/test/Lower/OpenACC/acc-routine03.f90
@@ -30,6 +30,6 @@ subroutine sub2(a)
 end subroutine
 
 ! CHECK: acc.routine @acc_routine_1 func(@_QPsub2) worker nohost
-! CHECK: acc.routine @acc_routine_0 func(@_QPsub1) bind("_QPsub2") worker
+! CHECK: acc.routine @acc_routine_0 func(@_QPsub1) bind(@_QPsub2) worker
 ! CHECK: func.func @_QPsub1(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>}
 ! CHECK: func.func @_QPsub2(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>}
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index 4eb666239d4e4..8f87235fcd237 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -29,6 +29,7 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include <variant>
 
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/OpenACC/OpenACCOpsTypes.h.inc"
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 66378f116784e..96b9adcc53b3c 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2772,8 +2772,10 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
   }];
 
   let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
-      OptionalAttr<StrArrayAttr>:$bindName,
-      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<SymbolRefArrayAttr>:$bindIdName,
+      OptionalAttr<StrArrayAttr>:$bindStrName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindIdNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindStrNameDeviceType,
       OptionalAttr<DeviceTypeArrayAttr>:$worker,
       OptionalAttr<DeviceTypeArrayAttr>:$vector,
       OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
@@ -2815,14 +2817,14 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     std::optional<int64_t> getGangDimValue();
     std::optional<int64_t> getGangDimValue(mlir::acc::DeviceType deviceType);
 
-    std::optional<llvm::StringRef> getBindNameValue();
-    std::optional<llvm::StringRef> getBindNameValue(mlir::acc::DeviceType deviceType);
+    std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue();
+    std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue(mlir::acc::DeviceType deviceType);
   }];
 
   let assemblyFormat = [{
     $sym_name `func` `(` $func_name `)`
     oilist (
-        `bind` `(` custom<BindName>($bindName, $bindNameDeviceType) `)`
+        `bind` `(` custom<BindName>($bindIdName, $bindStrName ,$bindIdNameDeviceType, $bindStrNameDeviceType) `)`
       | `gang` `` custom<RoutineGangClause>($gang, $gangDim, $gangDimDeviceType)
       | `worker` custom<DeviceTypeArrayAttr>($worker)
       | `vector` custom<DeviceTypeArrayAttr>($vector)
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..33519ed08ee0b 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/LogicalResult.h"
+#include <variant>
 
 using namespace mlir;
 using namespace acc;
@@ -3461,40 +3462,89 @@ LogicalResult acc::RoutineOp::verify() {
   return success();
 }
 
-static ParseResult parseBindName(OpAsmParser &parser, mlir::ArrayAttr &bindName,
-                                 mlir::ArrayAttr &deviceTypes) {
-  llvm::SmallVector<mlir::Attribute> bindNameAttrs;
-  llvm::SmallVector<mlir::Attribute> deviceTypeAttrs;
+static ParseResult parseBindName(OpAsmParser &parser,
+                                 mlir::ArrayAttr &bindIdName,
+                                 mlir::ArrayAttr &bindStrName,
+                                 mlir::ArrayAttr &deviceIdTypes,
+                                 mlir::ArrayAttr &deviceStrTypes) {
+  llvm::SmallVector<mlir::Attribute> bindIdNameAttrs;
+  llvm::SmallVector<mlir::Attribute> bindStrNameAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceIdTypeAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceStrTypeAttrs;
 
   if (failed(parser.parseCommaSeparatedList([&]() {
-        if (parser.parseAttribute(bindNameAttrs.emplace_back()))
+        mlir::Attribute newAttr;
+        bool isSymbolRefAttr;
+        auto parseResult = parser.parseAttribute(newAttr);
+        if (auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(newAttr)) {
+          bindIdNameAttrs.push_back(symbolRefAttr);
+          isSymbolRefAttr = true;
+        } else if (auto stringAttr = dyn_cast<mlir::StringAttr>(newAttr)) {
+          bindStrNameAttrs.push_back(stringAttr);
+          isSymbolRefAttr = false;
+        }
+        if (parseResult)
           return failure();
         if (failed(parser.parseOptionalLSquare())) {
-          deviceTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
-              parser.getContext(), mlir::acc::DeviceType::None));
+          if (isSymbolRefAttr) {
+            deviceIdTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          } else {
+            deviceStrTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          }
         } else {
-          if (parser.parseAttribute(deviceTypeAttrs.emplace_back()) ||
-              parser.parseRSquare())
-            return failure();
+          if (isSymbolRefAttr) {
+            if (parser.parseAttribute(deviceIdTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          } else {
+            if (parser.parseAttribute(deviceStrTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          }
         }
         return success();
       })))
     return failure();
 
-  bindName = ArrayAttr::get(parser.getContext(), bindNameAttrs);
-  deviceTypes = ArrayAttr::get(parser.getContext(), deviceTypeAttrs);
+  bindIdName = ArrayAttr::get(parser.getContext(), bindIdNameAttrs);
+  bindStrName = ArrayAttr::get(parser.getContext(), bindStrNameAttrs);
+  deviceIdTypes = ArrayAttr::get(parser.getContext(), deviceIdTypeAttrs);
+  deviceStrTypes = ArrayAttr::get(parser.getContext(), deviceStrTypeAttrs);
 
   return success();
 }
 
 static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,
-                          std::optional<mlir::ArrayAttr> bindName,
-                          std::optional<mlir::ArrayAttr> deviceTypes) {
-  llvm::interleaveComma(llvm::zip(*bindName, *deviceTypes), p,
-                        [&](const auto &pair) {
-                          p << std::get<0>(pair);
-                          printSingleDeviceType(p, std::get<1>(pair));
-                        });
+                          std::optional<mlir::ArrayAttr> bindIdName,
+                          std::optional<mlir::ArrayAttr> bindStrName,
+                          std::optional<mlir::ArrayAttr> deviceIdTypes,
+                          std::optional<mlir::ArrayAttr> deviceStrTypes) {
+  // Create combined vectors for all bind names and device types
+  llvm::SmallVector<mlir::Attribute> allBindNames;
+  llvm::SmallVector<mlir::Attribute> allDeviceTypes;
+
+  // Append bindIdName and deviceIdTypes
+  if (hasDeviceTypeValues(deviceIdTypes)) {
+    allBindNames.append(bindIdName->begin(), bindIdName->end());
+    allDeviceTypes.append(deviceIdTypes->begin(), deviceIdTypes->end());
+  }
+
+  // Append bindStrName and deviceStrTypes
+  if (hasDeviceTypeValues(deviceStrTypes)) {
+    allBindNames.append(bindStrName->begin(), bindStrName->end());
+    allDeviceTypes.append(deviceStrTypes->begin(), deviceStrTypes->end());
+  }
+
+  // Print the combined sequence
+  if (!allBindNames.empty()) {
+    llvm::interleaveComma(llvm::zip(allBindNames, allDeviceTypes), p,
+                          [&](const auto &pair) {
+                            p << std::get<0>(pair);
+                            printSingleDeviceType(p, std::get<1>(pair));
+                          });
+  }
 }
 
 static ParseResult parseRoutineGangClause(OpAsmParser &parser,
@@ -3654,19 +3704,32 @@ bool RoutineOp::hasSeq(mlir::acc::DeviceType deviceType) {
   return hasDeviceType(getSeq(), deviceType);
 }
 
-std::optional<llvm::StringRef> RoutineOp::getBindNameValue() {
+std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+RoutineOp::getBindNameValue() {
   return getBindNameValue(mlir::acc::DeviceType::None);
 }
 
-std::optional<llvm::StringRef>
+std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
 RoutineOp::getBindNameValue(mlir::acc::DeviceType deviceType) {
-  if (!hasDeviceTypeValues(getBindNameDeviceType()))
+  if (!hasDeviceTypeValues(getBindIdNameDeviceType()) &&
+      !hasDeviceTypeValues(getBindStrNameDeviceType())) {
     return std::nullopt;
-  if (auto pos = findSegment(*getBindNameDeviceType(), deviceType)) {
-    auto attr = (*getBindName())[*pos];
+  }
+
+  if (auto pos = findSegment(*getBindIdNameDeviceType(), deviceType)) {
+    auto attr = (*getBindIdName())[*pos];
+    auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(attr);
+    assert(symbolRefAttr && "expected SymbolRef");
+    return symbolRefAttr;
+  }
+
+  if (auto pos = findSegment(*getBindStrNameDeviceType(), deviceType)) {
+    auto attr = (*getBindStrName())[*pos];
     auto stringAttr = dyn_cast<mlir::StringAttr>(attr);
-    return stringAttr.getValue();
+    assert(stringAttr && "expected String");
+    return stringAttr;
   }
+
   return std::nullopt;
 }
 
diff --git a/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp b/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
index aa16421cbec51..836efdb307f97 100644
--- a/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
+++ b/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
@@ -519,14 +519,44 @@ TEST_F(OpenACCOpsTest, routineOpTest) {
   op->removeGangDimDeviceTypeAttr();
   op->removeGangDimAttr();
 
-  op->setBindNameDeviceTypeAttr(b.getArrayAttr({dtypeNone}));
-  op->setBindNameAttr(b.getArrayAttr({b.getStringAttr("fname")}));
+  op->setBindIdNameDeviceTypeAttr(
+      b.getArrayAttr({DeviceTypeAttr::get(&context, DeviceType::Host)}));
+  op->setBindStrNameDeviceTypeAttr(b.getArrayAttr({dtypeNone}));
+  op->setBindIdNameAttr(
+      b.getArrayAttr({SymbolRefAttr::get(&context, "test_symbol")}));
+  op->setBindStrNameAttr(b.getArrayAttr({b.getStringAttr("fname")}));
   EXPECT_TRUE(op->getBindNameValue().has_value());
-  EXPECT_EQ(op->getBindNameValue().value(), "fname");
-  for (auto d : dtypesWithoutNone)
-    EXPECT_FALSE(op->getBindNameValue(d).has_value());
-  op->removeBindNameDeviceTypeAttr();
-  op->removeBindNameAttr();
+  EXPECT_TRUE(op->getBindNameValue(DeviceType::Host).has_value());
+  EXPECT_EQ(std::visit(
+                [](const auto &attr) -> std::string {
+                  if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                               mlir::StringAttr>) {
+                    return attr.str();
+                  } else {
+                    return attr.getLeafReference().str();
+                  }
+                },
+                op->getBindNameValue().value()),
+            "fname");
+  EXPECT_EQ(std::visit(
+                [](const auto &attr) -> std::string {
+                  if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                               mlir::StringAttr>) {
+                    return attr.str();
+                  } else {
+                    return attr.getLeafReference().str();
+                  }
+                },
+                op->getBindNameValue(DeviceType::Host).value()),
+            "test_symbol");
+  for (auto d : dtypesWithoutNone) {
+    if (d != DeviceType::Host)
+      EXPECT_FALSE(op->getBindNameValue(d).has_value());
+  }
+  op->removeBindIdNameDeviceTypeAttr();
+  op->removeBindStrNameDeviceTypeAttr();
+  op->removeBindIdNameAttr();
+  op->removeBindStrNameAttr();
 }
 
 template <typename Op>


From flang-commits at lists.llvm.org  Wed Jul 16 10:23:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 10:23:01 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <6877dff5.170a0220.c2c64.16a3@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-openacc

@llvm/pr-subscribers-flang-fir-hlfir

Author: None (delaram-talaashrafi)

<details>
<summary>Changes</summary>

Based on the OpenACC specification — which states that if the bind name is given as an identifier it should be resolved according to the compiled language, and if given as a string it should be used unmodified — we introduce two distinct `bindName` representations for `acc routine` to handle each case appropriately: one as an array of `SymbolRefAttr` for identifiers and another as an array of `StringAttr` for strings.

To ensure correct correspondence between bind names and devices, this patch also introduces two separate sets of device attributes. The routine operation is extended accordingly, along with the necessary updates to the OpenACC dialect and its lowering.

---

Patch is 23.26 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/149147.diff


7 Files Affected:

- (modified) flang/lib/Lower/OpenACC.cpp (+72-31) 
- (modified) flang/test/Lower/OpenACC/acc-routine.f90 (+4-3) 
- (modified) flang/test/Lower/OpenACC/acc-routine03.f90 (+1-1) 
- (modified) mlir/include/mlir/Dialect/OpenACC/OpenACC.h (+1) 
- (modified) mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td (+7-5) 
- (modified) mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp (+88-25) 
- (modified) mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp (+37-7) 


``````````diff
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 39e4444cde4e3..93664dc5d8c2d 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4396,10 +4396,35 @@ getAttributeValueByDeviceType(llvm::SmallVector<mlir::Attribute> &attributes,
   return std::nullopt;
 }
 
+// Helper function to extract string value from bind name variant
+static std::optional<llvm::StringRef> getBindNameStringValue(
+    const std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+        &bindNameValue) {
+  if (!bindNameValue.has_value()) {
+    return std::nullopt;
+  }
+
+  return std::visit(
+      [](const auto &attr) -> std::optional<llvm::StringRef> {
+        if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                     mlir::StringAttr>) {
+          return attr.getValue();
+        } else if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                            mlir::SymbolRefAttr>) {
+          return attr.getLeafReference();
+        } else {
+          return std::nullopt;
+        }
+      },
+      bindNameValue.value());
+}
+
 static bool compareDeviceTypeInfo(
     mlir::acc::RoutineOp op,
-    llvm::SmallVector<mlir::Attribute> &bindNameArrayAttr,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypeArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypeArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypeArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangDimArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypeArrayAttr,
@@ -4409,9 +4434,13 @@ static bool compareDeviceTypeInfo(
   for (uint32_t dtypeInt = 0;
        dtypeInt != mlir::acc::getMaxEnumValForDeviceType(); ++dtypeInt) {
     auto dtype = static_cast<mlir::acc::DeviceType>(dtypeInt);
-    if (op.getBindNameValue(dtype) !=
-        getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
-            bindNameArrayAttr, bindNameDeviceTypeArrayAttr, dtype))
+    auto bindNameValue = getBindNameStringValue(op.getBindNameValue(dtype));
+    if (bindNameValue !=
+            getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
+                bindIdNameArrayAttr, bindIdNameDeviceTypeArrayAttr, dtype) &&
+        bindNameValue !=
+            getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
+                bindStrNameArrayAttr, bindStrNameDeviceTypeArrayAttr, dtype))
       return false;
     if (op.hasGang(dtype) != hasDeviceType(gangArrayAttr, dtype))
       return false;
@@ -4458,8 +4487,10 @@ getArrayAttrOrNull(fir::FirOpBuilder &builder,
 void createOpenACCRoutineConstruct(
     Fortran::lower::AbstractConverter &converter, mlir::Location loc,
     mlir::ModuleOp mod, mlir::func::FuncOp funcOp, std::string funcName,
-    bool hasNohost, llvm::SmallVector<mlir::Attribute> &bindNames,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypes,
+    bool hasNohost, llvm::SmallVector<mlir::Attribute> &bindIdNames,
+    llvm::SmallVector<mlir::Attribute> &bindStrNames,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDimValues,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypes,
@@ -4472,7 +4503,8 @@ void createOpenACCRoutineConstruct(
         0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
-      if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
+      if (compareDeviceTypeInfo(routineOp, bindIdNames, bindStrNames,
+                                bindIdNameDeviceTypes, bindStrNameDeviceTypes,
                                 gangDeviceTypes, gangDimValues,
                                 gangDimDeviceTypes, seqDeviceTypes,
                                 workerDeviceTypes, vectorDeviceTypes) &&
@@ -4489,8 +4521,10 @@ void createOpenACCRoutineConstruct(
   modBuilder.create<mlir::acc::RoutineOp>(
       loc, routineOpStr,
       mlir::SymbolRefAttr::get(builder.getContext(), funcName),
-      getArrayAttrOrNull(builder, bindNames),
-      getArrayAttrOrNull(builder, bindNameDeviceTypes),
+      getArrayAttrOrNull(builder, bindIdNames),
+      getArrayAttrOrNull(builder, bindStrNames),
+      getArrayAttrOrNull(builder, bindIdNameDeviceTypes),
+      getArrayAttrOrNull(builder, bindStrNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),
       getArrayAttrOrNull(builder, seqDeviceTypes), hasNohost,
@@ -4507,8 +4541,10 @@ static void interpretRoutineDeviceInfo(
     llvm::SmallVector<mlir::Attribute> &seqDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &workerDeviceTypes,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypes,
-    llvm::SmallVector<mlir::Attribute> &bindNames,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindIdNames,
+    llvm::SmallVector<mlir::Attribute> &bindStrNames,
     llvm::SmallVector<mlir::Attribute> &gangDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDimValues,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypes) {
@@ -4541,16 +4577,18 @@ static void interpretRoutineDeviceInfo(
   if (dinfo.bindNameOpt().has_value()) {
     const auto &bindName = dinfo.bindNameOpt().value();
     mlir::Attribute bindNameAttr;
-    if (const auto &bindStr{std::get_if<std::string>(&bindName)}) {
+    if (const auto &bindSym{
+            std::get_if<Fortran::semantics::SymbolRef>(&bindName)}) {
+      bindNameAttr = builder.getSymbolRefAttr(converter.mangleName(*bindSym));
+      bindIdNames.push_back(bindNameAttr);
+      bindIdNameDeviceTypes.push_back(getDeviceTypeAttr());
+    } else if (const auto &bindStr{std::get_if<std::string>(&bindName)}) {
       bindNameAttr = builder.getStringAttr(*bindStr);
-    } else if (const auto &bindSym{
-                   std::get_if<Fortran::semantics::SymbolRef>(&bindName)}) {
-      bindNameAttr = builder.getStringAttr(converter.mangleName(*bindSym));
+      bindStrNames.push_back(bindNameAttr);
+      bindStrNameDeviceTypes.push_back(getDeviceTypeAttr());
     } else {
       llvm_unreachable("Unsupported bind name type");
     }
-    bindNames.push_back(bindNameAttr);
-    bindNameDeviceTypes.push_back(getDeviceTypeAttr());
   }
 }
 
@@ -4566,8 +4604,9 @@ void Fortran::lower::genOpenACCRoutineConstruct(
   bool hasNohost{false};
 
   llvm::SmallVector<mlir::Attribute> seqDeviceTypes, vectorDeviceTypes,
-      workerDeviceTypes, bindNameDeviceTypes, bindNames, gangDeviceTypes,
-      gangDimDeviceTypes, gangDimValues;
+      workerDeviceTypes, bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+      bindIdNames, bindStrNames, gangDeviceTypes, gangDimDeviceTypes,
+      gangDimValues;
 
   for (const Fortran::semantics::OpenACCRoutineInfo &info : routineInfos) {
     // Device Independent Attributes
@@ -4576,24 +4615,26 @@ void Fortran::lower::genOpenACCRoutineConstruct(
     }
     // Note: Device Independent Attributes are set to the
     // none device type in `info`.
-    interpretRoutineDeviceInfo(converter, info, seqDeviceTypes,
-                               vectorDeviceTypes, workerDeviceTypes,
-                               bindNameDeviceTypes, bindNames, gangDeviceTypes,
-                               gangDimValues, gangDimDeviceTypes);
+    interpretRoutineDeviceInfo(
+        converter, info, seqDeviceTypes, vectorDeviceTypes, workerDeviceTypes,
+        bindIdNameDeviceTypes, bindStrNameDeviceTypes, bindIdNames,
+        bindStrNames, gangDeviceTypes, gangDimValues, gangDimDeviceTypes);
 
     // Device Dependent Attributes
     for (const Fortran::semantics::OpenACCRoutineDeviceTypeInfo &dinfo :
          info.deviceTypeInfos()) {
-      interpretRoutineDeviceInfo(
-          converter, dinfo, seqDeviceTypes, vectorDeviceTypes,
-          workerDeviceTypes, bindNameDeviceTypes, bindNames, gangDeviceTypes,
-          gangDimValues, gangDimDeviceTypes);
+      interpretRoutineDeviceInfo(converter, dinfo, seqDeviceTypes,
+                                 vectorDeviceTypes, workerDeviceTypes,
+                                 bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+                                 bindIdNames, bindStrNames, gangDeviceTypes,
+                                 gangDimValues, gangDimDeviceTypes);
     }
   }
   createOpenACCRoutineConstruct(
-      converter, loc, mod, funcOp, funcName, hasNohost, bindNames,
-      bindNameDeviceTypes, gangDeviceTypes, gangDimValues, gangDimDeviceTypes,
-      seqDeviceTypes, workerDeviceTypes, vectorDeviceTypes);
+      converter, loc, mod, funcOp, funcName, hasNohost, bindIdNames,
+      bindStrNames, bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+      gangDeviceTypes, gangDimValues, gangDimDeviceTypes, seqDeviceTypes,
+      workerDeviceTypes, vectorDeviceTypes);
 }
 
 static void
diff --git a/flang/test/Lower/OpenACC/acc-routine.f90 b/flang/test/Lower/OpenACC/acc-routine.f90
index 789f3a57e1f79..1a63b4120235c 100644
--- a/flang/test/Lower/OpenACC/acc-routine.f90
+++ b/flang/test/Lower/OpenACC/acc-routine.f90
@@ -2,13 +2,14 @@
 
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
 
-! CHECK: acc.routine @[[r14:.*]] func(@_QPacc_routine19) bind("_QPacc_routine17" [#acc.device_type<host>], "_QPacc_routine17" [#acc.device_type<default>], "_QPacc_routine16" [#acc.device_type<multicore>])
-! CHECK: acc.routine @[[r13:.*]] func(@_QPacc_routine18) bind("_QPacc_routine17" [#acc.device_type<host>], "_QPacc_routine16" [#acc.device_type<multicore>])
+! CHECK: acc.routine @[[r14:.*]] func(@_QPacc_routine19) bind(@_QPacc_routine17 [#acc.device_type<host>], @_QPacc_routine17
+! [#acc.device_type<default>], @_QPacc_routine16 [#acc.device_type<multicore>])
+! CHECK: acc.routine @[[r13:.*]] func(@_QPacc_routine18) bind(@_QPacc_routine17 [#acc.device_type<host>], @_QPacc_routine16 [#acc.device_type<multicore>])
 ! CHECK: acc.routine @[[r12:.*]] func(@_QPacc_routine17) worker ([#acc.device_type<host>]) vector ([#acc.device_type<multicore>])
 ! CHECK: acc.routine @[[r11:.*]] func(@_QPacc_routine16) gang([#acc.device_type<nvidia>]) seq ([#acc.device_type<host>])
 ! CHECK: acc.routine @[[r10:.*]] func(@_QPacc_routine11) seq
 ! CHECK: acc.routine @[[r09:.*]] func(@_QPacc_routine10) seq
-! CHECK: acc.routine @[[r08:.*]] func(@_QPacc_routine9) bind("_QPacc_routine9a")
+! CHECK: acc.routine @[[r08:.*]] func(@_QPacc_routine9) bind(@_QPacc_routine9a)
 ! CHECK: acc.routine @[[r07:.*]] func(@_QPacc_routine8) bind("routine8_")
 ! CHECK: acc.routine @[[r06:.*]] func(@_QPacc_routine7) gang(dim: 1 : i64)
 ! CHECK: acc.routine @[[r05:.*]] func(@_QPacc_routine6) nohost
diff --git a/flang/test/Lower/OpenACC/acc-routine03.f90 b/flang/test/Lower/OpenACC/acc-routine03.f90
index 85e4ef580f983..ddd6bda0367e4 100644
--- a/flang/test/Lower/OpenACC/acc-routine03.f90
+++ b/flang/test/Lower/OpenACC/acc-routine03.f90
@@ -30,6 +30,6 @@ subroutine sub2(a)
 end subroutine
 
 ! CHECK: acc.routine @acc_routine_1 func(@_QPsub2) worker nohost
-! CHECK: acc.routine @acc_routine_0 func(@_QPsub1) bind("_QPsub2") worker
+! CHECK: acc.routine @acc_routine_0 func(@_QPsub1) bind(@_QPsub2) worker
 ! CHECK: func.func @_QPsub1(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>}
 ! CHECK: func.func @_QPsub2(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>}
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index 4eb666239d4e4..8f87235fcd237 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -29,6 +29,7 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include <variant>
 
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/OpenACC/OpenACCOpsTypes.h.inc"
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 66378f116784e..96b9adcc53b3c 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2772,8 +2772,10 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
   }];
 
   let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
-      OptionalAttr<StrArrayAttr>:$bindName,
-      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<SymbolRefArrayAttr>:$bindIdName,
+      OptionalAttr<StrArrayAttr>:$bindStrName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindIdNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindStrNameDeviceType,
       OptionalAttr<DeviceTypeArrayAttr>:$worker,
       OptionalAttr<DeviceTypeArrayAttr>:$vector,
       OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
@@ -2815,14 +2817,14 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     std::optional<int64_t> getGangDimValue();
     std::optional<int64_t> getGangDimValue(mlir::acc::DeviceType deviceType);
 
-    std::optional<llvm::StringRef> getBindNameValue();
-    std::optional<llvm::StringRef> getBindNameValue(mlir::acc::DeviceType deviceType);
+    std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue();
+    std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue(mlir::acc::DeviceType deviceType);
   }];
 
   let assemblyFormat = [{
     $sym_name `func` `(` $func_name `)`
     oilist (
-        `bind` `(` custom<BindName>($bindName, $bindNameDeviceType) `)`
+        `bind` `(` custom<BindName>($bindIdName, $bindStrName ,$bindIdNameDeviceType, $bindStrNameDeviceType) `)`
       | `gang` `` custom<RoutineGangClause>($gang, $gangDim, $gangDimDeviceType)
       | `worker` custom<DeviceTypeArrayAttr>($worker)
       | `vector` custom<DeviceTypeArrayAttr>($vector)
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..33519ed08ee0b 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/LogicalResult.h"
+#include <variant>
 
 using namespace mlir;
 using namespace acc;
@@ -3461,40 +3462,89 @@ LogicalResult acc::RoutineOp::verify() {
   return success();
 }
 
-static ParseResult parseBindName(OpAsmParser &parser, mlir::ArrayAttr &bindName,
-                                 mlir::ArrayAttr &deviceTypes) {
-  llvm::SmallVector<mlir::Attribute> bindNameAttrs;
-  llvm::SmallVector<mlir::Attribute> deviceTypeAttrs;
+static ParseResult parseBindName(OpAsmParser &parser,
+                                 mlir::ArrayAttr &bindIdName,
+                                 mlir::ArrayAttr &bindStrName,
+                                 mlir::ArrayAttr &deviceIdTypes,
+                                 mlir::ArrayAttr &deviceStrTypes) {
+  llvm::SmallVector<mlir::Attribute> bindIdNameAttrs;
+  llvm::SmallVector<mlir::Attribute> bindStrNameAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceIdTypeAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceStrTypeAttrs;
 
   if (failed(parser.parseCommaSeparatedList([&]() {
-        if (parser.parseAttribute(bindNameAttrs.emplace_back()))
+        mlir::Attribute newAttr;
+        bool isSymbolRefAttr;
+        auto parseResult = parser.parseAttribute(newAttr);
+        if (auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(newAttr)) {
+          bindIdNameAttrs.push_back(symbolRefAttr);
+          isSymbolRefAttr = true;
+        } else if (auto stringAttr = dyn_cast<mlir::StringAttr>(newAttr)) {
+          bindStrNameAttrs.push_back(stringAttr);
+          isSymbolRefAttr = false;
+        }
+        if (parseResult)
           return failure();
         if (failed(parser.parseOptionalLSquare())) {
-          deviceTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
-              parser.getContext(), mlir::acc::DeviceType::None));
+          if (isSymbolRefAttr) {
+            deviceIdTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          } else {
+            deviceStrTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          }
         } else {
-          if (parser.parseAttribute(deviceTypeAttrs.emplace_back()) ||
-              parser.parseRSquare())
-            return failure();
+          if (isSymbolRefAttr) {
+            if (parser.parseAttribute(deviceIdTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          } else {
+            if (parser.parseAttribute(deviceStrTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          }
         }
         return success();
       })))
     return failure();
 
-  bindName = ArrayAttr::get(parser.getContext(), bindNameAttrs);
-  deviceTypes = ArrayAttr::get(parser.getContext(), deviceTypeAttrs);
+  bindIdName = ArrayAttr::get(parser.getContext(), bindIdNameAttrs);
+  bindStrName = ArrayAttr::get(parser.getContext(), bindStrNameAttrs);
+  deviceIdTypes = ArrayAttr::get(parser.getContext(), deviceIdTypeAttrs);
+  deviceStrTypes = ArrayAttr::get(parser.getContext(), deviceStrTypeAttrs);
 
   return success();
 }
 
 static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,
-                          std::optional<mlir::ArrayAttr> bindName,
-                          std::optional<mlir::ArrayAttr> deviceTypes) {
-  llvm::interleaveComma(llvm::zip(*bindName, *deviceTypes), p,
-                        [&](const auto &pair) {
-                          p << std::get<0>(pair);
-                          printSingleDeviceType(p, std::get<1>(pair));
-                        });
+                          std::optional<mlir::ArrayAttr> bindIdName,
+                          std::optional<mlir::ArrayAttr> bindStrName,
+                          std::optional<mlir::ArrayAttr> deviceIdTypes,
+                          std::optional<mlir::ArrayAttr> deviceStrTypes) {
+  // Create combined vectors for all bind names and device types
+  llvm::SmallVector<mlir::Attribute> allBindNames;
+  llvm::SmallVector<mlir::Attribute> allDeviceTypes;
+
+  // Append bindIdName and deviceIdTypes
+  if (hasDeviceTypeValues(deviceIdTypes)) {
+    allBindNames.append(bindIdName->begin(), bindIdName->end());
+    allDeviceTypes.append(deviceIdTypes->begin(), deviceIdTypes->end());
+  }
+
+  // Append bindStrName and deviceStrTypes
+  if (hasDeviceTypeValues(deviceStrTypes)) {
+    allBindNames.append(bindStrName->begin(), bindStrName->end());
+    allDeviceTypes.append(deviceStrTypes->begin(), deviceStrTypes->end());
+  }
+
+  // Print the combined sequence
+  if (!allBindNames.empty()) {
+    llvm::interleaveComma(llvm::zip(allBindNames, allDeviceTypes), p,
+                          [&](const auto &pair) {
+                            p << std::get<0>(pair);
+                            printSingleDeviceType(p, std::get<1>(pair));
+                          });
+  }
 }
 
 static ParseResult parseRoutineGangClause(OpAsmParser &parser,
@@ -3654,19 +3704,32 @@ bool RoutineOp::hasSeq(mlir::acc::DeviceType deviceType) {
   return hasDeviceType(getSeq(), deviceType);
 }
 
-std::optional<llvm::StringRef> RoutineOp::getBindNameValue() {
+std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+RoutineOp::getBindNameValue() {
   return getBindNameValue(mlir::acc::DeviceType::None);
 }
 
-std::optional<llvm::StringRef>
+std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
 RoutineOp::getBindNameValue(mlir::acc::DeviceType deviceType) {
-  if (!hasDeviceTypeValues(getBindName...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/149147

From flang-commits at lists.llvm.org  Wed Jul 16 10:24:04 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 10:24:04 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <6877e034.050a0220.eaf6.5309@mx.google.com>


github-actions[bot] wrote:

⚠️ We detected that you are using a GitHub private e-mail address to contribute to the repo.<br/> Please turn off [Keep my email addresses private](https://github.com/settings/emails) setting in your account.<br/> See [LLVM Developer Policy](https://llvm.org/docs/DeveloperPolicy.html#email-addresses) and [LLVM Discourse](https://discourse.llvm.org/t/hidden-emails-on-github-should-we-do-something-about-it) for more information.

https://github.com/llvm/llvm-project/pull/149147

From flang-commits at lists.llvm.org  Wed Jul 16 10:29:43 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Wed, 16 Jul 2025 10:29:43 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <6877e187.050a0220.171b07.4d6f@mx.google.com>


================
@@ -4396,10 +4396,35 @@ getAttributeValueByDeviceType(llvm::SmallVector<mlir::Attribute> &attributes,
   return std::nullopt;
 }
 
+// Helper function to extract string value from bind name variant
+static std::optional<llvm::StringRef> getBindNameStringValue(
+    const std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+        &bindNameValue) {
+  if (!bindNameValue.has_value()) {
+    return std::nullopt;
+  }
----------------
clementval wrote:

no brace

https://github.com/llvm/llvm-project/pull/149147

From flang-commits at lists.llvm.org  Wed Jul 16 10:31:49 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Wed, 16 Jul 2025 10:31:49 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <6877e205.050a0220.195c87.6db9@mx.google.com>


================
@@ -3461,40 +3462,89 @@ LogicalResult acc::RoutineOp::verify() {
   return success();
 }
 
-static ParseResult parseBindName(OpAsmParser &parser, mlir::ArrayAttr &bindName,
-                                 mlir::ArrayAttr &deviceTypes) {
-  llvm::SmallVector<mlir::Attribute> bindNameAttrs;
-  llvm::SmallVector<mlir::Attribute> deviceTypeAttrs;
+static ParseResult parseBindName(OpAsmParser &parser,
+                                 mlir::ArrayAttr &bindIdName,
+                                 mlir::ArrayAttr &bindStrName,
+                                 mlir::ArrayAttr &deviceIdTypes,
+                                 mlir::ArrayAttr &deviceStrTypes) {
+  llvm::SmallVector<mlir::Attribute> bindIdNameAttrs;
+  llvm::SmallVector<mlir::Attribute> bindStrNameAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceIdTypeAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceStrTypeAttrs;
 
   if (failed(parser.parseCommaSeparatedList([&]() {
-        if (parser.parseAttribute(bindNameAttrs.emplace_back()))
+        mlir::Attribute newAttr;
+        bool isSymbolRefAttr;
+        auto parseResult = parser.parseAttribute(newAttr);
+        if (auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(newAttr)) {
+          bindIdNameAttrs.push_back(symbolRefAttr);
+          isSymbolRefAttr = true;
+        } else if (auto stringAttr = dyn_cast<mlir::StringAttr>(newAttr)) {
+          bindStrNameAttrs.push_back(stringAttr);
+          isSymbolRefAttr = false;
+        }
+        if (parseResult)
           return failure();
         if (failed(parser.parseOptionalLSquare())) {
-          deviceTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
-              parser.getContext(), mlir::acc::DeviceType::None));
+          if (isSymbolRefAttr) {
+            deviceIdTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          } else {
+            deviceStrTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          }
         } else {
-          if (parser.parseAttribute(deviceTypeAttrs.emplace_back()) ||
-              parser.parseRSquare())
-            return failure();
+          if (isSymbolRefAttr) {
+            if (parser.parseAttribute(deviceIdTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          } else {
+            if (parser.parseAttribute(deviceStrTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          }
         }
         return success();
       })))
     return failure();
 
-  bindName = ArrayAttr::get(parser.getContext(), bindNameAttrs);
-  deviceTypes = ArrayAttr::get(parser.getContext(), deviceTypeAttrs);
+  bindIdName = ArrayAttr::get(parser.getContext(), bindIdNameAttrs);
+  bindStrName = ArrayAttr::get(parser.getContext(), bindStrNameAttrs);
+  deviceIdTypes = ArrayAttr::get(parser.getContext(), deviceIdTypeAttrs);
+  deviceStrTypes = ArrayAttr::get(parser.getContext(), deviceStrTypeAttrs);
 
   return success();
 }
 
 static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,
-                          std::optional<mlir::ArrayAttr> bindName,
-                          std::optional<mlir::ArrayAttr> deviceTypes) {
-  llvm::interleaveComma(llvm::zip(*bindName, *deviceTypes), p,
-                        [&](const auto &pair) {
-                          p << std::get<0>(pair);
-                          printSingleDeviceType(p, std::get<1>(pair));
-                        });
+                          std::optional<mlir::ArrayAttr> bindIdName,
+                          std::optional<mlir::ArrayAttr> bindStrName,
+                          std::optional<mlir::ArrayAttr> deviceIdTypes,
+                          std::optional<mlir::ArrayAttr> deviceStrTypes) {
+  // Create combined vectors for all bind names and device types
+  llvm::SmallVector<mlir::Attribute> allBindNames;
+  llvm::SmallVector<mlir::Attribute> allDeviceTypes;
+
+  // Append bindIdName and deviceIdTypes
+  if (hasDeviceTypeValues(deviceIdTypes)) {
+    allBindNames.append(bindIdName->begin(), bindIdName->end());
+    allDeviceTypes.append(deviceIdTypes->begin(), deviceIdTypes->end());
+  }
+
+  // Append bindStrName and deviceStrTypes
+  if (hasDeviceTypeValues(deviceStrTypes)) {
+    allBindNames.append(bindStrName->begin(), bindStrName->end());
+    allDeviceTypes.append(deviceStrTypes->begin(), deviceStrTypes->end());
+  }
+
+  // Print the combined sequence
+  if (!allBindNames.empty()) {
+    llvm::interleaveComma(llvm::zip(allBindNames, allDeviceTypes), p,
+                          [&](const auto &pair) {
+                            p << std::get<0>(pair);
+                            printSingleDeviceType(p, std::get<1>(pair));
+                          });
+  }
----------------
clementval wrote:

No brace

https://github.com/llvm/llvm-project/pull/149147

From flang-commits at lists.llvm.org  Wed Jul 16 10:32:58 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Wed, 16 Jul 2025 10:32:58 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <6877e24a.170a0220.143317.c3f7@mx.google.com>


https://github.com/clementval approved this pull request.

Looks good to me. Couple of nits for simple braces https://llvm.org/docs/CodingStandards.html#don-t-use-braces-on-simple-single-statement-bodies-of-if-else-loop-statements

https://github.com/llvm/llvm-project/pull/149147

From flang-commits at lists.llvm.org  Wed Jul 16 10:44:10 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Wed, 16 Jul 2025 10:44:10 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <6877e4ea.050a0220.38c4b.97f9@mx.google.com>


https://github.com/razvanlupusoru approved this pull request.

Great work!

https://github.com/llvm/llvm-project/pull/149147

From flang-commits at lists.llvm.org  Wed Jul 16 10:50:51 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Wed, 16 Jul 2025 10:50:51 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang] Allow -fdefault-integer-8
 with defined I/O (PR #148927)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148927@github.com>
Message-ID: <6877e67b.630a0220.9633d.5135@mx.google.com>


DanielCChen wrote:

The full test case is passing now. Thanks for the fix!

https://github.com/llvm/llvm-project/pull/148927

From flang-commits at lists.llvm.org  Wed Jul 16 11:02:23 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 11:02:23 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <6877e92f.170a0220.98b5d.c869@mx.google.com>


https://github.com/delaram-talaashrafi updated https://github.com/llvm/llvm-project/pull/149147

>From f61dd65b3cb04195edee58f2f6f3f0fc57cbfc45 Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Wed, 16 Jul 2025 10:12:26 -0700
Subject: [PATCH 1/2] [openacc][flang] Support SymbolRef and String bindName
 representations and device attribute in acc.routine
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Based on the OpenACC specification — which states that if the bind name is given as an identifier it should
be resolved according to the compiled language, and if given as a string it should be used unmodified — we
introduce two distinct `bindName` representations for `acc routine` to handle each case appropriately:
one as an array of `SymbolRefAttr` for identifiers and another as an array of `StringAttr` for strings.

To ensure correct correspondence between bind names and devices, this patch also introduces two separate
sets of device attributes. The routine operation is extended accordingly, along with the necessary updates
to the OpenACC dialect and its lowering.
---
 flang/lib/Lower/OpenACC.cpp                   | 103 +++++++++++-----
 flang/test/Lower/OpenACC/acc-routine.f90      |   7 +-
 flang/test/Lower/OpenACC/acc-routine03.f90    |   2 +-
 mlir/include/mlir/Dialect/OpenACC/OpenACC.h   |   1 +
 .../mlir/Dialect/OpenACC/OpenACCOps.td        |  12 +-
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       | 113 ++++++++++++++----
 .../Dialect/OpenACC/OpenACCOpsTest.cpp        |  44 +++++--
 7 files changed, 210 insertions(+), 72 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 39e4444cde4e3..93664dc5d8c2d 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4396,10 +4396,35 @@ getAttributeValueByDeviceType(llvm::SmallVector<mlir::Attribute> &attributes,
   return std::nullopt;
 }
 
+// Helper function to extract string value from bind name variant
+static std::optional<llvm::StringRef> getBindNameStringValue(
+    const std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+        &bindNameValue) {
+  if (!bindNameValue.has_value()) {
+    return std::nullopt;
+  }
+
+  return std::visit(
+      [](const auto &attr) -> std::optional<llvm::StringRef> {
+        if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                     mlir::StringAttr>) {
+          return attr.getValue();
+        } else if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                            mlir::SymbolRefAttr>) {
+          return attr.getLeafReference();
+        } else {
+          return std::nullopt;
+        }
+      },
+      bindNameValue.value());
+}
+
 static bool compareDeviceTypeInfo(
     mlir::acc::RoutineOp op,
-    llvm::SmallVector<mlir::Attribute> &bindNameArrayAttr,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypeArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypeArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypeArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangDimArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypeArrayAttr,
@@ -4409,9 +4434,13 @@ static bool compareDeviceTypeInfo(
   for (uint32_t dtypeInt = 0;
        dtypeInt != mlir::acc::getMaxEnumValForDeviceType(); ++dtypeInt) {
     auto dtype = static_cast<mlir::acc::DeviceType>(dtypeInt);
-    if (op.getBindNameValue(dtype) !=
-        getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
-            bindNameArrayAttr, bindNameDeviceTypeArrayAttr, dtype))
+    auto bindNameValue = getBindNameStringValue(op.getBindNameValue(dtype));
+    if (bindNameValue !=
+            getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
+                bindIdNameArrayAttr, bindIdNameDeviceTypeArrayAttr, dtype) &&
+        bindNameValue !=
+            getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
+                bindStrNameArrayAttr, bindStrNameDeviceTypeArrayAttr, dtype))
       return false;
     if (op.hasGang(dtype) != hasDeviceType(gangArrayAttr, dtype))
       return false;
@@ -4458,8 +4487,10 @@ getArrayAttrOrNull(fir::FirOpBuilder &builder,
 void createOpenACCRoutineConstruct(
     Fortran::lower::AbstractConverter &converter, mlir::Location loc,
     mlir::ModuleOp mod, mlir::func::FuncOp funcOp, std::string funcName,
-    bool hasNohost, llvm::SmallVector<mlir::Attribute> &bindNames,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypes,
+    bool hasNohost, llvm::SmallVector<mlir::Attribute> &bindIdNames,
+    llvm::SmallVector<mlir::Attribute> &bindStrNames,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDimValues,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypes,
@@ -4472,7 +4503,8 @@ void createOpenACCRoutineConstruct(
         0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
-      if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
+      if (compareDeviceTypeInfo(routineOp, bindIdNames, bindStrNames,
+                                bindIdNameDeviceTypes, bindStrNameDeviceTypes,
                                 gangDeviceTypes, gangDimValues,
                                 gangDimDeviceTypes, seqDeviceTypes,
                                 workerDeviceTypes, vectorDeviceTypes) &&
@@ -4489,8 +4521,10 @@ void createOpenACCRoutineConstruct(
   modBuilder.create<mlir::acc::RoutineOp>(
       loc, routineOpStr,
       mlir::SymbolRefAttr::get(builder.getContext(), funcName),
-      getArrayAttrOrNull(builder, bindNames),
-      getArrayAttrOrNull(builder, bindNameDeviceTypes),
+      getArrayAttrOrNull(builder, bindIdNames),
+      getArrayAttrOrNull(builder, bindStrNames),
+      getArrayAttrOrNull(builder, bindIdNameDeviceTypes),
+      getArrayAttrOrNull(builder, bindStrNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),
       getArrayAttrOrNull(builder, seqDeviceTypes), hasNohost,
@@ -4507,8 +4541,10 @@ static void interpretRoutineDeviceInfo(
     llvm::SmallVector<mlir::Attribute> &seqDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &workerDeviceTypes,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypes,
-    llvm::SmallVector<mlir::Attribute> &bindNames,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindIdNames,
+    llvm::SmallVector<mlir::Attribute> &bindStrNames,
     llvm::SmallVector<mlir::Attribute> &gangDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDimValues,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypes) {
@@ -4541,16 +4577,18 @@ static void interpretRoutineDeviceInfo(
   if (dinfo.bindNameOpt().has_value()) {
     const auto &bindName = dinfo.bindNameOpt().value();
     mlir::Attribute bindNameAttr;
-    if (const auto &bindStr{std::get_if<std::string>(&bindName)}) {
+    if (const auto &bindSym{
+            std::get_if<Fortran::semantics::SymbolRef>(&bindName)}) {
+      bindNameAttr = builder.getSymbolRefAttr(converter.mangleName(*bindSym));
+      bindIdNames.push_back(bindNameAttr);
+      bindIdNameDeviceTypes.push_back(getDeviceTypeAttr());
+    } else if (const auto &bindStr{std::get_if<std::string>(&bindName)}) {
       bindNameAttr = builder.getStringAttr(*bindStr);
-    } else if (const auto &bindSym{
-                   std::get_if<Fortran::semantics::SymbolRef>(&bindName)}) {
-      bindNameAttr = builder.getStringAttr(converter.mangleName(*bindSym));
+      bindStrNames.push_back(bindNameAttr);
+      bindStrNameDeviceTypes.push_back(getDeviceTypeAttr());
     } else {
       llvm_unreachable("Unsupported bind name type");
     }
-    bindNames.push_back(bindNameAttr);
-    bindNameDeviceTypes.push_back(getDeviceTypeAttr());
   }
 }
 
@@ -4566,8 +4604,9 @@ void Fortran::lower::genOpenACCRoutineConstruct(
   bool hasNohost{false};
 
   llvm::SmallVector<mlir::Attribute> seqDeviceTypes, vectorDeviceTypes,
-      workerDeviceTypes, bindNameDeviceTypes, bindNames, gangDeviceTypes,
-      gangDimDeviceTypes, gangDimValues;
+      workerDeviceTypes, bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+      bindIdNames, bindStrNames, gangDeviceTypes, gangDimDeviceTypes,
+      gangDimValues;
 
   for (const Fortran::semantics::OpenACCRoutineInfo &info : routineInfos) {
     // Device Independent Attributes
@@ -4576,24 +4615,26 @@ void Fortran::lower::genOpenACCRoutineConstruct(
     }
     // Note: Device Independent Attributes are set to the
     // none device type in `info`.
-    interpretRoutineDeviceInfo(converter, info, seqDeviceTypes,
-                               vectorDeviceTypes, workerDeviceTypes,
-                               bindNameDeviceTypes, bindNames, gangDeviceTypes,
-                               gangDimValues, gangDimDeviceTypes);
+    interpretRoutineDeviceInfo(
+        converter, info, seqDeviceTypes, vectorDeviceTypes, workerDeviceTypes,
+        bindIdNameDeviceTypes, bindStrNameDeviceTypes, bindIdNames,
+        bindStrNames, gangDeviceTypes, gangDimValues, gangDimDeviceTypes);
 
     // Device Dependent Attributes
     for (const Fortran::semantics::OpenACCRoutineDeviceTypeInfo &dinfo :
          info.deviceTypeInfos()) {
-      interpretRoutineDeviceInfo(
-          converter, dinfo, seqDeviceTypes, vectorDeviceTypes,
-          workerDeviceTypes, bindNameDeviceTypes, bindNames, gangDeviceTypes,
-          gangDimValues, gangDimDeviceTypes);
+      interpretRoutineDeviceInfo(converter, dinfo, seqDeviceTypes,
+                                 vectorDeviceTypes, workerDeviceTypes,
+                                 bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+                                 bindIdNames, bindStrNames, gangDeviceTypes,
+                                 gangDimValues, gangDimDeviceTypes);
     }
   }
   createOpenACCRoutineConstruct(
-      converter, loc, mod, funcOp, funcName, hasNohost, bindNames,
-      bindNameDeviceTypes, gangDeviceTypes, gangDimValues, gangDimDeviceTypes,
-      seqDeviceTypes, workerDeviceTypes, vectorDeviceTypes);
+      converter, loc, mod, funcOp, funcName, hasNohost, bindIdNames,
+      bindStrNames, bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+      gangDeviceTypes, gangDimValues, gangDimDeviceTypes, seqDeviceTypes,
+      workerDeviceTypes, vectorDeviceTypes);
 }
 
 static void
diff --git a/flang/test/Lower/OpenACC/acc-routine.f90 b/flang/test/Lower/OpenACC/acc-routine.f90
index 789f3a57e1f79..1a63b4120235c 100644
--- a/flang/test/Lower/OpenACC/acc-routine.f90
+++ b/flang/test/Lower/OpenACC/acc-routine.f90
@@ -2,13 +2,14 @@
 
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
 
-! CHECK: acc.routine @[[r14:.*]] func(@_QPacc_routine19) bind("_QPacc_routine17" [#acc.device_type<host>], "_QPacc_routine17" [#acc.device_type<default>], "_QPacc_routine16" [#acc.device_type<multicore>])
-! CHECK: acc.routine @[[r13:.*]] func(@_QPacc_routine18) bind("_QPacc_routine17" [#acc.device_type<host>], "_QPacc_routine16" [#acc.device_type<multicore>])
+! CHECK: acc.routine @[[r14:.*]] func(@_QPacc_routine19) bind(@_QPacc_routine17 [#acc.device_type<host>], @_QPacc_routine17
+! [#acc.device_type<default>], @_QPacc_routine16 [#acc.device_type<multicore>])
+! CHECK: acc.routine @[[r13:.*]] func(@_QPacc_routine18) bind(@_QPacc_routine17 [#acc.device_type<host>], @_QPacc_routine16 [#acc.device_type<multicore>])
 ! CHECK: acc.routine @[[r12:.*]] func(@_QPacc_routine17) worker ([#acc.device_type<host>]) vector ([#acc.device_type<multicore>])
 ! CHECK: acc.routine @[[r11:.*]] func(@_QPacc_routine16) gang([#acc.device_type<nvidia>]) seq ([#acc.device_type<host>])
 ! CHECK: acc.routine @[[r10:.*]] func(@_QPacc_routine11) seq
 ! CHECK: acc.routine @[[r09:.*]] func(@_QPacc_routine10) seq
-! CHECK: acc.routine @[[r08:.*]] func(@_QPacc_routine9) bind("_QPacc_routine9a")
+! CHECK: acc.routine @[[r08:.*]] func(@_QPacc_routine9) bind(@_QPacc_routine9a)
 ! CHECK: acc.routine @[[r07:.*]] func(@_QPacc_routine8) bind("routine8_")
 ! CHECK: acc.routine @[[r06:.*]] func(@_QPacc_routine7) gang(dim: 1 : i64)
 ! CHECK: acc.routine @[[r05:.*]] func(@_QPacc_routine6) nohost
diff --git a/flang/test/Lower/OpenACC/acc-routine03.f90 b/flang/test/Lower/OpenACC/acc-routine03.f90
index 85e4ef580f983..ddd6bda0367e4 100644
--- a/flang/test/Lower/OpenACC/acc-routine03.f90
+++ b/flang/test/Lower/OpenACC/acc-routine03.f90
@@ -30,6 +30,6 @@ subroutine sub2(a)
 end subroutine
 
 ! CHECK: acc.routine @acc_routine_1 func(@_QPsub2) worker nohost
-! CHECK: acc.routine @acc_routine_0 func(@_QPsub1) bind("_QPsub2") worker
+! CHECK: acc.routine @acc_routine_0 func(@_QPsub1) bind(@_QPsub2) worker
 ! CHECK: func.func @_QPsub1(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>}
 ! CHECK: func.func @_QPsub2(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>}
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index 4eb666239d4e4..8f87235fcd237 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -29,6 +29,7 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include <variant>
 
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/OpenACC/OpenACCOpsTypes.h.inc"
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 66378f116784e..96b9adcc53b3c 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2772,8 +2772,10 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
   }];
 
   let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
-      OptionalAttr<StrArrayAttr>:$bindName,
-      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<SymbolRefArrayAttr>:$bindIdName,
+      OptionalAttr<StrArrayAttr>:$bindStrName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindIdNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindStrNameDeviceType,
       OptionalAttr<DeviceTypeArrayAttr>:$worker,
       OptionalAttr<DeviceTypeArrayAttr>:$vector,
       OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
@@ -2815,14 +2817,14 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     std::optional<int64_t> getGangDimValue();
     std::optional<int64_t> getGangDimValue(mlir::acc::DeviceType deviceType);
 
-    std::optional<llvm::StringRef> getBindNameValue();
-    std::optional<llvm::StringRef> getBindNameValue(mlir::acc::DeviceType deviceType);
+    std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue();
+    std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue(mlir::acc::DeviceType deviceType);
   }];
 
   let assemblyFormat = [{
     $sym_name `func` `(` $func_name `)`
     oilist (
-        `bind` `(` custom<BindName>($bindName, $bindNameDeviceType) `)`
+        `bind` `(` custom<BindName>($bindIdName, $bindStrName ,$bindIdNameDeviceType, $bindStrNameDeviceType) `)`
       | `gang` `` custom<RoutineGangClause>($gang, $gangDim, $gangDimDeviceType)
       | `worker` custom<DeviceTypeArrayAttr>($worker)
       | `vector` custom<DeviceTypeArrayAttr>($vector)
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..33519ed08ee0b 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/LogicalResult.h"
+#include <variant>
 
 using namespace mlir;
 using namespace acc;
@@ -3461,40 +3462,89 @@ LogicalResult acc::RoutineOp::verify() {
   return success();
 }
 
-static ParseResult parseBindName(OpAsmParser &parser, mlir::ArrayAttr &bindName,
-                                 mlir::ArrayAttr &deviceTypes) {
-  llvm::SmallVector<mlir::Attribute> bindNameAttrs;
-  llvm::SmallVector<mlir::Attribute> deviceTypeAttrs;
+static ParseResult parseBindName(OpAsmParser &parser,
+                                 mlir::ArrayAttr &bindIdName,
+                                 mlir::ArrayAttr &bindStrName,
+                                 mlir::ArrayAttr &deviceIdTypes,
+                                 mlir::ArrayAttr &deviceStrTypes) {
+  llvm::SmallVector<mlir::Attribute> bindIdNameAttrs;
+  llvm::SmallVector<mlir::Attribute> bindStrNameAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceIdTypeAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceStrTypeAttrs;
 
   if (failed(parser.parseCommaSeparatedList([&]() {
-        if (parser.parseAttribute(bindNameAttrs.emplace_back()))
+        mlir::Attribute newAttr;
+        bool isSymbolRefAttr;
+        auto parseResult = parser.parseAttribute(newAttr);
+        if (auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(newAttr)) {
+          bindIdNameAttrs.push_back(symbolRefAttr);
+          isSymbolRefAttr = true;
+        } else if (auto stringAttr = dyn_cast<mlir::StringAttr>(newAttr)) {
+          bindStrNameAttrs.push_back(stringAttr);
+          isSymbolRefAttr = false;
+        }
+        if (parseResult)
           return failure();
         if (failed(parser.parseOptionalLSquare())) {
-          deviceTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
-              parser.getContext(), mlir::acc::DeviceType::None));
+          if (isSymbolRefAttr) {
+            deviceIdTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          } else {
+            deviceStrTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          }
         } else {
-          if (parser.parseAttribute(deviceTypeAttrs.emplace_back()) ||
-              parser.parseRSquare())
-            return failure();
+          if (isSymbolRefAttr) {
+            if (parser.parseAttribute(deviceIdTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          } else {
+            if (parser.parseAttribute(deviceStrTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          }
         }
         return success();
       })))
     return failure();
 
-  bindName = ArrayAttr::get(parser.getContext(), bindNameAttrs);
-  deviceTypes = ArrayAttr::get(parser.getContext(), deviceTypeAttrs);
+  bindIdName = ArrayAttr::get(parser.getContext(), bindIdNameAttrs);
+  bindStrName = ArrayAttr::get(parser.getContext(), bindStrNameAttrs);
+  deviceIdTypes = ArrayAttr::get(parser.getContext(), deviceIdTypeAttrs);
+  deviceStrTypes = ArrayAttr::get(parser.getContext(), deviceStrTypeAttrs);
 
   return success();
 }
 
 static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,
-                          std::optional<mlir::ArrayAttr> bindName,
-                          std::optional<mlir::ArrayAttr> deviceTypes) {
-  llvm::interleaveComma(llvm::zip(*bindName, *deviceTypes), p,
-                        [&](const auto &pair) {
-                          p << std::get<0>(pair);
-                          printSingleDeviceType(p, std::get<1>(pair));
-                        });
+                          std::optional<mlir::ArrayAttr> bindIdName,
+                          std::optional<mlir::ArrayAttr> bindStrName,
+                          std::optional<mlir::ArrayAttr> deviceIdTypes,
+                          std::optional<mlir::ArrayAttr> deviceStrTypes) {
+  // Create combined vectors for all bind names and device types
+  llvm::SmallVector<mlir::Attribute> allBindNames;
+  llvm::SmallVector<mlir::Attribute> allDeviceTypes;
+
+  // Append bindIdName and deviceIdTypes
+  if (hasDeviceTypeValues(deviceIdTypes)) {
+    allBindNames.append(bindIdName->begin(), bindIdName->end());
+    allDeviceTypes.append(deviceIdTypes->begin(), deviceIdTypes->end());
+  }
+
+  // Append bindStrName and deviceStrTypes
+  if (hasDeviceTypeValues(deviceStrTypes)) {
+    allBindNames.append(bindStrName->begin(), bindStrName->end());
+    allDeviceTypes.append(deviceStrTypes->begin(), deviceStrTypes->end());
+  }
+
+  // Print the combined sequence
+  if (!allBindNames.empty()) {
+    llvm::interleaveComma(llvm::zip(allBindNames, allDeviceTypes), p,
+                          [&](const auto &pair) {
+                            p << std::get<0>(pair);
+                            printSingleDeviceType(p, std::get<1>(pair));
+                          });
+  }
 }
 
 static ParseResult parseRoutineGangClause(OpAsmParser &parser,
@@ -3654,19 +3704,32 @@ bool RoutineOp::hasSeq(mlir::acc::DeviceType deviceType) {
   return hasDeviceType(getSeq(), deviceType);
 }
 
-std::optional<llvm::StringRef> RoutineOp::getBindNameValue() {
+std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+RoutineOp::getBindNameValue() {
   return getBindNameValue(mlir::acc::DeviceType::None);
 }
 
-std::optional<llvm::StringRef>
+std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
 RoutineOp::getBindNameValue(mlir::acc::DeviceType deviceType) {
-  if (!hasDeviceTypeValues(getBindNameDeviceType()))
+  if (!hasDeviceTypeValues(getBindIdNameDeviceType()) &&
+      !hasDeviceTypeValues(getBindStrNameDeviceType())) {
     return std::nullopt;
-  if (auto pos = findSegment(*getBindNameDeviceType(), deviceType)) {
-    auto attr = (*getBindName())[*pos];
+  }
+
+  if (auto pos = findSegment(*getBindIdNameDeviceType(), deviceType)) {
+    auto attr = (*getBindIdName())[*pos];
+    auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(attr);
+    assert(symbolRefAttr && "expected SymbolRef");
+    return symbolRefAttr;
+  }
+
+  if (auto pos = findSegment(*getBindStrNameDeviceType(), deviceType)) {
+    auto attr = (*getBindStrName())[*pos];
     auto stringAttr = dyn_cast<mlir::StringAttr>(attr);
-    return stringAttr.getValue();
+    assert(stringAttr && "expected String");
+    return stringAttr;
   }
+
   return std::nullopt;
 }
 
diff --git a/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp b/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
index aa16421cbec51..836efdb307f97 100644
--- a/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
+++ b/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
@@ -519,14 +519,44 @@ TEST_F(OpenACCOpsTest, routineOpTest) {
   op->removeGangDimDeviceTypeAttr();
   op->removeGangDimAttr();
 
-  op->setBindNameDeviceTypeAttr(b.getArrayAttr({dtypeNone}));
-  op->setBindNameAttr(b.getArrayAttr({b.getStringAttr("fname")}));
+  op->setBindIdNameDeviceTypeAttr(
+      b.getArrayAttr({DeviceTypeAttr::get(&context, DeviceType::Host)}));
+  op->setBindStrNameDeviceTypeAttr(b.getArrayAttr({dtypeNone}));
+  op->setBindIdNameAttr(
+      b.getArrayAttr({SymbolRefAttr::get(&context, "test_symbol")}));
+  op->setBindStrNameAttr(b.getArrayAttr({b.getStringAttr("fname")}));
   EXPECT_TRUE(op->getBindNameValue().has_value());
-  EXPECT_EQ(op->getBindNameValue().value(), "fname");
-  for (auto d : dtypesWithoutNone)
-    EXPECT_FALSE(op->getBindNameValue(d).has_value());
-  op->removeBindNameDeviceTypeAttr();
-  op->removeBindNameAttr();
+  EXPECT_TRUE(op->getBindNameValue(DeviceType::Host).has_value());
+  EXPECT_EQ(std::visit(
+                [](const auto &attr) -> std::string {
+                  if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                               mlir::StringAttr>) {
+                    return attr.str();
+                  } else {
+                    return attr.getLeafReference().str();
+                  }
+                },
+                op->getBindNameValue().value()),
+            "fname");
+  EXPECT_EQ(std::visit(
+                [](const auto &attr) -> std::string {
+                  if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                               mlir::StringAttr>) {
+                    return attr.str();
+                  } else {
+                    return attr.getLeafReference().str();
+                  }
+                },
+                op->getBindNameValue(DeviceType::Host).value()),
+            "test_symbol");
+  for (auto d : dtypesWithoutNone) {
+    if (d != DeviceType::Host)
+      EXPECT_FALSE(op->getBindNameValue(d).has_value());
+  }
+  op->removeBindIdNameDeviceTypeAttr();
+  op->removeBindStrNameDeviceTypeAttr();
+  op->removeBindIdNameAttr();
+  op->removeBindStrNameAttr();
 }
 
 template <typename Op>

>From 878c70c1556461253f86da13fb3596fd0389f5d6 Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Wed, 16 Jul 2025 11:01:48 -0700
Subject: [PATCH 2/2] Remove extra braces

---
 flang/lib/Lower/OpenACC.cpp             | 3 +--
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 93664dc5d8c2d..a8ade004bc553 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4400,9 +4400,8 @@ getAttributeValueByDeviceType(llvm::SmallVector<mlir::Attribute> &attributes,
 static std::optional<llvm::StringRef> getBindNameStringValue(
     const std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
         &bindNameValue) {
-  if (!bindNameValue.has_value()) {
+  if (!bindNameValue.has_value())
     return std::nullopt;
-  }
 
   return std::visit(
       [](const auto &attr) -> std::optional<llvm::StringRef> {
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 33519ed08ee0b..fbc1f003ab648 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -3538,13 +3538,12 @@ static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,
   }
 
   // Print the combined sequence
-  if (!allBindNames.empty()) {
+  if (!allBindNames.empty())
     llvm::interleaveComma(llvm::zip(allBindNames, allDeviceTypes), p,
                           [&](const auto &pair) {
                             p << std::get<0>(pair);
                             printSingleDeviceType(p, std::get<1>(pair));
                           });
-  }
 }
 
 static ParseResult parseRoutineGangClause(OpAsmParser &parser,


From flang-commits at lists.llvm.org  Wed Jul 16 11:03:18 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 11:03:18 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <6877e966.050a0220.3b711a.5914@mx.google.com>


================
@@ -3461,40 +3462,89 @@ LogicalResult acc::RoutineOp::verify() {
   return success();
 }
 
-static ParseResult parseBindName(OpAsmParser &parser, mlir::ArrayAttr &bindName,
-                                 mlir::ArrayAttr &deviceTypes) {
-  llvm::SmallVector<mlir::Attribute> bindNameAttrs;
-  llvm::SmallVector<mlir::Attribute> deviceTypeAttrs;
+static ParseResult parseBindName(OpAsmParser &parser,
+                                 mlir::ArrayAttr &bindIdName,
+                                 mlir::ArrayAttr &bindStrName,
+                                 mlir::ArrayAttr &deviceIdTypes,
+                                 mlir::ArrayAttr &deviceStrTypes) {
+  llvm::SmallVector<mlir::Attribute> bindIdNameAttrs;
+  llvm::SmallVector<mlir::Attribute> bindStrNameAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceIdTypeAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceStrTypeAttrs;
 
   if (failed(parser.parseCommaSeparatedList([&]() {
-        if (parser.parseAttribute(bindNameAttrs.emplace_back()))
+        mlir::Attribute newAttr;
+        bool isSymbolRefAttr;
+        auto parseResult = parser.parseAttribute(newAttr);
+        if (auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(newAttr)) {
+          bindIdNameAttrs.push_back(symbolRefAttr);
+          isSymbolRefAttr = true;
+        } else if (auto stringAttr = dyn_cast<mlir::StringAttr>(newAttr)) {
+          bindStrNameAttrs.push_back(stringAttr);
+          isSymbolRefAttr = false;
+        }
+        if (parseResult)
           return failure();
         if (failed(parser.parseOptionalLSquare())) {
-          deviceTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
-              parser.getContext(), mlir::acc::DeviceType::None));
+          if (isSymbolRefAttr) {
+            deviceIdTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          } else {
+            deviceStrTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          }
         } else {
-          if (parser.parseAttribute(deviceTypeAttrs.emplace_back()) ||
-              parser.parseRSquare())
-            return failure();
+          if (isSymbolRefAttr) {
+            if (parser.parseAttribute(deviceIdTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          } else {
+            if (parser.parseAttribute(deviceStrTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          }
         }
         return success();
       })))
     return failure();
 
-  bindName = ArrayAttr::get(parser.getContext(), bindNameAttrs);
-  deviceTypes = ArrayAttr::get(parser.getContext(), deviceTypeAttrs);
+  bindIdName = ArrayAttr::get(parser.getContext(), bindIdNameAttrs);
+  bindStrName = ArrayAttr::get(parser.getContext(), bindStrNameAttrs);
+  deviceIdTypes = ArrayAttr::get(parser.getContext(), deviceIdTypeAttrs);
+  deviceStrTypes = ArrayAttr::get(parser.getContext(), deviceStrTypeAttrs);
 
   return success();
 }
 
 static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,
-                          std::optional<mlir::ArrayAttr> bindName,
-                          std::optional<mlir::ArrayAttr> deviceTypes) {
-  llvm::interleaveComma(llvm::zip(*bindName, *deviceTypes), p,
-                        [&](const auto &pair) {
-                          p << std::get<0>(pair);
-                          printSingleDeviceType(p, std::get<1>(pair));
-                        });
+                          std::optional<mlir::ArrayAttr> bindIdName,
+                          std::optional<mlir::ArrayAttr> bindStrName,
+                          std::optional<mlir::ArrayAttr> deviceIdTypes,
+                          std::optional<mlir::ArrayAttr> deviceStrTypes) {
+  // Create combined vectors for all bind names and device types
+  llvm::SmallVector<mlir::Attribute> allBindNames;
+  llvm::SmallVector<mlir::Attribute> allDeviceTypes;
+
+  // Append bindIdName and deviceIdTypes
+  if (hasDeviceTypeValues(deviceIdTypes)) {
+    allBindNames.append(bindIdName->begin(), bindIdName->end());
+    allDeviceTypes.append(deviceIdTypes->begin(), deviceIdTypes->end());
+  }
+
+  // Append bindStrName and deviceStrTypes
+  if (hasDeviceTypeValues(deviceStrTypes)) {
+    allBindNames.append(bindStrName->begin(), bindStrName->end());
+    allDeviceTypes.append(deviceStrTypes->begin(), deviceStrTypes->end());
+  }
+
+  // Print the combined sequence
+  if (!allBindNames.empty()) {
+    llvm::interleaveComma(llvm::zip(allBindNames, allDeviceTypes), p,
+                          [&](const auto &pair) {
+                            p << std::get<0>(pair);
+                            printSingleDeviceType(p, std::get<1>(pair));
+                          });
+  }
----------------
delaram-talaashrafi wrote:

I removed the extra braces, thanks.

https://github.com/llvm/llvm-project/pull/149147

From flang-commits at lists.llvm.org  Wed Jul 16 11:18:51 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Wed, 16 Jul 2025 11:18:51 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Import type descriptor in the
 gpu module when needed (PR #149157)
Message-ID: <llvm.org/llvm/llvm-project/pull/149157@github.com>


https://github.com/clementval created https://github.com/llvm/llvm-project/pull/149157

None

>From 557fee5038c1620c73601a57a41ccb92d5810b80 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Wed, 16 Jul 2025 10:19:45 -0700
Subject: [PATCH] [flang][cuda] Import type descriptor in the gpu module when
 needed

---
 .../Optimizer/Transforms/CUFDeviceGlobal.cpp  | 42 +++++++++++++------
 flang/test/Fir/CUDA/cuda-device-global.f90    | 22 ++++++++++
 2 files changed, 52 insertions(+), 12 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
index bfb0daeacb8c3..35badb6eadb1c 100644
--- a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
@@ -53,21 +53,26 @@ static void processAddrOfOp(fir::AddrOfOp addrOfOp,
   }
 }
 
+static void processTypeDescriptor(fir::RecordType recTy,
+                                  mlir::SymbolTable &symbolTable,
+                                  llvm::DenseSet<fir::GlobalOp> &candidates) {
+  if (auto globalOp = symbolTable.lookup<fir::GlobalOp>(
+          fir::NameUniquer::getTypeDescriptorName(recTy.getName()))) {
+    if (!candidates.contains(globalOp)) {
+      globalOp.walk([&](fir::AddrOfOp op) {
+        processAddrOfOp(op, symbolTable, candidates,
+                        /*recurseInGlobal=*/true);
+      });
+      candidates.insert(globalOp);
+    }
+  }
+}
+
 static void processEmboxOp(fir::EmboxOp emboxOp, mlir::SymbolTable &symbolTable,
                            llvm::DenseSet<fir::GlobalOp> &candidates) {
   if (auto recTy = mlir::dyn_cast<fir::RecordType>(
-          fir::unwrapRefType(emboxOp.getMemref().getType()))) {
-    if (auto globalOp = symbolTable.lookup<fir::GlobalOp>(
-            fir::NameUniquer::getTypeDescriptorName(recTy.getName()))) {
-      if (!candidates.contains(globalOp)) {
-        globalOp.walk([&](fir::AddrOfOp op) {
-          processAddrOfOp(op, symbolTable, candidates,
-                          /*recurseInGlobal=*/true);
-        });
-        candidates.insert(globalOp);
-      }
-    }
-  }
+          fir::unwrapRefType(emboxOp.getMemref().getType())))
+    processTypeDescriptor(recTy, symbolTable, candidates);
 }
 
 static void
@@ -85,6 +90,17 @@ prepareImplicitDeviceGlobals(mlir::func::FuncOp funcOp,
   }
 }
 
+static void
+processPotentialTypeDescriptor(mlir::Type candidateType,
+                               mlir::SymbolTable &symbolTable,
+                               llvm::DenseSet<fir::GlobalOp> &candidates) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(candidateType))
+    candidateType = boxTy.getEleTy();
+  candidateType = fir::unwrapSequenceType(fir::unwrapRefType(candidateType));
+  if (auto recTy = mlir::dyn_cast<fir::RecordType>(candidateType))
+    processTypeDescriptor(recTy, symbolTable, candidates);
+}
+
 class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
 public:
   void runOnOperation() override {
@@ -115,6 +131,8 @@ class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
     for (auto globalOp : mod.getOps<fir::GlobalOp>()) {
       if (cuf::isRegisteredDeviceGlobal(globalOp)) {
         candidates.insert(globalOp);
+        processPotentialTypeDescriptor(globalOp.getType(), parentSymTable,
+                                       candidates);
       } else if (globalOp.getConstant() &&
                  mlir::isa<fir::SequenceType>(
                      fir::unwrapRefType(globalOp.resultType()))) {
diff --git a/flang/test/Fir/CUDA/cuda-device-global.f90 b/flang/test/Fir/CUDA/cuda-device-global.f90
index 4c634513745fd..a2875a2dd241d 100644
--- a/flang/test/Fir/CUDA/cuda-device-global.f90
+++ b/flang/test/Fir/CUDA/cuda-device-global.f90
@@ -24,3 +24,25 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.conta
 // CHECK: gpu.module @cuda_device_mod
 // CHECK-DAG: fir.global @_QMm2ECc
 // CHECK-DAG: fir.global @_QMm1ECb
+
+// -----
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module} {
+  fir.global @_QMmEddarrays {data_attr = #cuf.cuda<managed>} : !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>> {
+    %c0 = arith.constant 0 : index
+    %0 = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>
+    %1 = fir.shape %c0 : (index) -> !fir.shape<1>
+    %2 = fir.embox %0(%1) {allocator_idx = 3 : i32} : (!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>>
+    fir.has_value %2 : !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>>
+  }
+  fir.global linkonce_odr @_QMmE.dt.devicearrays constant target : !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}> {
+    %0 = fir.undefined !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>
+    fir.has_value %0 : !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>
+  }
+}
+
+// CHECK: fir.global @_QMmEddarrays
+// CHECK: fir.global linkonce_odr @_QMmE.dt.devicearrays
+// CHECK: gpu.module @cuda_device_mod
+// CHECK: fir.global @_QMmEddarrays
+// CHECK: fir.global linkonce_odr @_QMmE.dt.devicearrays


From flang-commits at lists.llvm.org  Wed Jul 16 11:19:26 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 11:19:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Import type descriptor in the
 gpu module when needed (PR #149157)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149157@github.com>
Message-ID: <6877ed2e.050a0220.6edb2.5cb6@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Valentin Clement (バレンタイン クレメン) (clementval)

<details>
<summary>Changes</summary>



---
Full diff: https://github.com/llvm/llvm-project/pull/149157.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp (+30-12) 
- (modified) flang/test/Fir/CUDA/cuda-device-global.f90 (+22) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
index bfb0daeacb8c3..35badb6eadb1c 100644
--- a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
@@ -53,21 +53,26 @@ static void processAddrOfOp(fir::AddrOfOp addrOfOp,
   }
 }
 
+static void processTypeDescriptor(fir::RecordType recTy,
+                                  mlir::SymbolTable &symbolTable,
+                                  llvm::DenseSet<fir::GlobalOp> &candidates) {
+  if (auto globalOp = symbolTable.lookup<fir::GlobalOp>(
+          fir::NameUniquer::getTypeDescriptorName(recTy.getName()))) {
+    if (!candidates.contains(globalOp)) {
+      globalOp.walk([&](fir::AddrOfOp op) {
+        processAddrOfOp(op, symbolTable, candidates,
+                        /*recurseInGlobal=*/true);
+      });
+      candidates.insert(globalOp);
+    }
+  }
+}
+
 static void processEmboxOp(fir::EmboxOp emboxOp, mlir::SymbolTable &symbolTable,
                            llvm::DenseSet<fir::GlobalOp> &candidates) {
   if (auto recTy = mlir::dyn_cast<fir::RecordType>(
-          fir::unwrapRefType(emboxOp.getMemref().getType()))) {
-    if (auto globalOp = symbolTable.lookup<fir::GlobalOp>(
-            fir::NameUniquer::getTypeDescriptorName(recTy.getName()))) {
-      if (!candidates.contains(globalOp)) {
-        globalOp.walk([&](fir::AddrOfOp op) {
-          processAddrOfOp(op, symbolTable, candidates,
-                          /*recurseInGlobal=*/true);
-        });
-        candidates.insert(globalOp);
-      }
-    }
-  }
+          fir::unwrapRefType(emboxOp.getMemref().getType())))
+    processTypeDescriptor(recTy, symbolTable, candidates);
 }
 
 static void
@@ -85,6 +90,17 @@ prepareImplicitDeviceGlobals(mlir::func::FuncOp funcOp,
   }
 }
 
+static void
+processPotentialTypeDescriptor(mlir::Type candidateType,
+                               mlir::SymbolTable &symbolTable,
+                               llvm::DenseSet<fir::GlobalOp> &candidates) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(candidateType))
+    candidateType = boxTy.getEleTy();
+  candidateType = fir::unwrapSequenceType(fir::unwrapRefType(candidateType));
+  if (auto recTy = mlir::dyn_cast<fir::RecordType>(candidateType))
+    processTypeDescriptor(recTy, symbolTable, candidates);
+}
+
 class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
 public:
   void runOnOperation() override {
@@ -115,6 +131,8 @@ class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
     for (auto globalOp : mod.getOps<fir::GlobalOp>()) {
       if (cuf::isRegisteredDeviceGlobal(globalOp)) {
         candidates.insert(globalOp);
+        processPotentialTypeDescriptor(globalOp.getType(), parentSymTable,
+                                       candidates);
       } else if (globalOp.getConstant() &&
                  mlir::isa<fir::SequenceType>(
                      fir::unwrapRefType(globalOp.resultType()))) {
diff --git a/flang/test/Fir/CUDA/cuda-device-global.f90 b/flang/test/Fir/CUDA/cuda-device-global.f90
index 4c634513745fd..a2875a2dd241d 100644
--- a/flang/test/Fir/CUDA/cuda-device-global.f90
+++ b/flang/test/Fir/CUDA/cuda-device-global.f90
@@ -24,3 +24,25 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.conta
 // CHECK: gpu.module @cuda_device_mod
 // CHECK-DAG: fir.global @_QMm2ECc
 // CHECK-DAG: fir.global @_QMm1ECb
+
+// -----
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module} {
+  fir.global @_QMmEddarrays {data_attr = #cuf.cuda<managed>} : !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>> {
+    %c0 = arith.constant 0 : index
+    %0 = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>
+    %1 = fir.shape %c0 : (index) -> !fir.shape<1>
+    %2 = fir.embox %0(%1) {allocator_idx = 3 : i32} : (!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>>
+    fir.has_value %2 : !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>>
+  }
+  fir.global linkonce_odr @_QMmE.dt.devicearrays constant target : !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}> {
+    %0 = fir.undefined !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>
+    fir.has_value %0 : !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>
+  }
+}
+
+// CHECK: fir.global @_QMmEddarrays
+// CHECK: fir.global linkonce_odr @_QMmE.dt.devicearrays
+// CHECK: gpu.module @cuda_device_mod
+// CHECK: fir.global @_QMmEddarrays
+// CHECK: fir.global linkonce_odr @_QMmE.dt.devicearrays

``````````

</details>


https://github.com/llvm/llvm-project/pull/149157

From flang-commits at lists.llvm.org  Wed Jul 16 11:29:43 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 16 Jul 2025 11:29:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <6877ef97.a70a0220.1983ae.737a@mx.google.com>


https://github.com/akuhlens updated https://github.com/llvm/llvm-project/pull/148748

>From b9cdd2e72c9c9e6b10098c4144b3e840122b56d6 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Mon, 14 Jul 2025 15:56:39 -0700
Subject: [PATCH 1/6] initial commit

---
 flang/include/flang/Parser/message.h        | 37 ++++++++++++++-------
 flang/lib/Frontend/FrontendAction.cpp       |  6 ++--
 flang/lib/Parser/message.cpp                | 37 +++++++++++++--------
 flang/lib/Semantics/resolve-names.cpp       |  3 +-
 flang/lib/Semantics/semantics.cpp           |  5 ++-
 flang/test/Driver/color-diagnostics-scan.f  | 16 ++++-----
 flang/test/Driver/fatal-errors-warnings.f90 | 32 ++++++++++++++++++
 7 files changed, 96 insertions(+), 40 deletions(-)
 create mode 100644 flang/test/Driver/fatal-errors-warnings.f90

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index db1a0a65157e3..d1d313a70a016 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -56,13 +56,19 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity() const { return severity_; }
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    }
+    return severity_;
+  }
   MessageFixedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
-  bool IsFatal() const {
-    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  bool IsFatal(bool warningsAreErrors = false) const {
+    Severity sev{severity(warningsAreErrors)};
+    return sev == Severity::Error || sev == Severity::Todo;
   }
 
 private:
@@ -105,7 +111,7 @@ class MessageFormattedText {
 public:
   template <typename... A>
   MessageFormattedText(const MessageFixedText &text, A &&...x)
-      : severity_{text.severity()} {
+      : severity_{text.severity(false)} {
     Format(&text, Convert(std::forward<A>(x))...);
   }
   MessageFormattedText(const MessageFormattedText &) = default;
@@ -113,14 +119,20 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
-  bool IsFatal() const {
-    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    }
+    return severity_;
   }
-  Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  bool IsFatal(bool warningsAreErrors = false) const {
+    Severity sev{severity(warningsAreErrors)};
+    return sev == Severity::Error || sev == Severity::Todo;
+  }
   std::string MoveString() { return std::move(string_); }
   bool operator==(const MessageFormattedText &that) const {
     return severity_ == that.severity_ && string_ == that.string_;
@@ -281,8 +293,8 @@ class Message : public common::ReferenceCounted<Message> {
   }
 
   bool SortBefore(const Message &that) const;
-  bool IsFatal() const;
-  Severity severity() const;
+  bool IsFatal(bool warningsAreErrors = false) const;
+  Severity severity(bool warningsAreErrors = false) const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
   Message &set_languageFeature(common::LanguageFeature);
@@ -293,7 +305,8 @@ class Message : public common::ReferenceCounted<Message> {
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLine = true,
-      const common::LanguageFeatureControl *hintFlags = nullptr) const;
+      const common::LanguageFeatureControl *hintFlags = nullptr,
+      bool warningsAreErrors = false) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
@@ -355,9 +368,9 @@ class Messages {
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLines = true,
       const common::LanguageFeatureControl *hintFlags = nullptr,
-      std::size_t maxErrorsToEmit = 0) const;
+      std::size_t maxErrorsToEmit = 0, bool warningsAreErrors = false) const;
   void AttachTo(Message &, std::optional<Severity> = std::nullopt);
-  bool AnyFatalError() const;
+  bool AnyFatalError(bool warningsAreErrors = false) const;
 
 private:
   std::list<Message> messages_;
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index 2429e07e5b8c4..d32f477445f93 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -238,7 +238,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLines=*/true, &features, maxErrors);
+        /*echoSourceLines=*/true, &features, maxErrors,
+        instance->getInvocation().getWarnAsErr());
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -249,7 +250,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLine=*/true, &features, maxErrors);
+        /*echoSourceLine=*/true, &features, maxErrors,
+        instance->getInvocation().getWarnAsErr());
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 909fba948a45a..574e9f5d424f8 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,16 +161,21 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-bool Message::IsFatal() const {
-  return severity() == Severity::Error || severity() == Severity::Todo;
+bool Message::IsFatal(bool warningsAreErrors) const {
+  Severity sev{severity(warningsAreErrors)};
+  return sev == Severity::Error || sev == Severity::Todo;
 }
 
-Severity Message::severity() const {
+Severity Message::severity(bool warningsAreErrors) const {
   return common::visit(
       common::visitors{
           [](const MessageExpectedText &) { return Severity::Error; },
-          [](const MessageFixedText &x) { return x.severity(); },
-          [](const MessageFormattedText &x) { return x.severity(); },
+          [=](const MessageFixedText &x) {
+            return x.severity(warningsAreErrors);
+          },
+          [=](const MessageFormattedText &x) {
+            return x.severity(warningsAreErrors);
+          },
       },
       text_);
 }
@@ -295,15 +300,16 @@ static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine,
-    const common::LanguageFeatureControl *hintFlagPtr) const {
+    bool echoSourceLine, const common::LanguageFeatureControl *hintFlagPtr,
+    bool warningsAreErrors) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
   const std::string text{ToString()};
+  Severity sev{severity(warningsAreErrors)};
   const std::string hint{
       HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
-  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
-      PrefixColor(severity()), echoSourceLine);
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(sev),
+      PrefixColor(sev), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
   // attachment->attachmentIsContext_ indicates this.
@@ -453,7 +459,7 @@ void Messages::ResolveProvenances(const AllCookedSources &allCooked) {
 
 void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     bool echoSourceLines, const common::LanguageFeatureControl *hintFlagPtr,
-    std::size_t maxErrorsToEmit) const {
+    std::size_t maxErrorsToEmit, bool warningsAreErrors) const {
   std::vector<const Message *> sorted;
   for (const auto &msg : messages_) {
     sorted.push_back(&msg);
@@ -467,9 +473,9 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr, warningsAreErrors);
     lastMsg = msg;
-    if (msg->IsFatal()) {
+    if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
     }
     // If maxErrorsToEmit is 0, emit all errors, otherwise break after
@@ -491,9 +497,12 @@ void Messages::AttachTo(Message &msg, std::optional<Severity> severity) {
   messages_.clear();
 }
 
-bool Messages::AnyFatalError() const {
+bool Messages::AnyFatalError(bool warningsAreErrors) const {
+  if (messages_.empty()) {
+    return false;
+  }
   for (const auto &msg : messages_) {
-    if (msg.IsFatal()) {
+    if (msg.IsFatal(warningsAreErrors)) {
       return true;
     }
   }
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 96faa5fd954cd..df898f6e5651e 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7575,7 +7575,8 @@ bool DeclarationVisitor::OkToAddComponent(
       if (msg) {
         auto &said{Say2(name, std::move(*msg), *prev,
             "Previous declaration of '%s'"_en_US)};
-        if (msg->severity() == parser::Severity::Error) {
+        if (msg->severity(/*warningsAreErrors=*/false) ==
+            parser::Severity::Error) {
           Resolve(name, *prev);
           return false;
         }
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -376,8 +376,7 @@ const DeclTypeSpec &SemanticsContext::MakeLogicalType(int kind) {
 }
 
 bool SemanticsContext::AnyFatalError() const {
-  return !messages_.empty() &&
-      (warningsAreErrors_ || messages_.AnyFatalError());
+  return messages_.AnyFatalError(warningsAreErrors_);
 }
 bool SemanticsContext::HasError(const Symbol &symbol) {
   return errorSymbols_.count(symbol) > 0;
@@ -658,7 +657,7 @@ void Semantics::EmitMessages(llvm::raw_ostream &os) {
   context_.messages().ResolveProvenances(context_.allCookedSources());
   context_.messages().Emit(os, context_.allCookedSources(),
       /*echoSourceLine=*/true, &context_.languageFeatures(),
-      /*maxErrorsToEmit=*/context_.maxErrors());
+      context_.maxErrors(), context_.warningsAreErrors());
 }
 
 void SemanticsContext::DumpSymbols(llvm::raw_ostream &os) {
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index 29d4635b4fb03..2f647c923c2e6 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -3,24 +3,24 @@
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
-! RUN: not %flang %s -E -Werror -fcolor-diagnostics 2>&1 \
+! RUN: %flang %s -E -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fno-color-diagnostics 2>&1 \
+! RUN: %flang %s -E -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
-! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
+! RUN: %flang_fc1 -E %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 
-! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
+! RUN: %flang %s -E -fno-diagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color=always 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color=never 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
 
diff --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
new file mode 100644
index 0000000000000..b4b23230587a4
--- /dev/null
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -0,0 +1,32 @@
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
+
+module m
+    contains
+    subroutine foo(a)
+        real, intent(in), target :: a(:)
+    end subroutine
+end module
+
+program test
+    use m
+    real, target :: a(1)
+    real :: b(1)
+    call foo(a) ! ok
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(b)
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK3-NOT: error:
+    !CHECK3-NOT: warning:
+    call foo((a))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(a([1]))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(a(1))
+end
\ No newline at end of file

>From 31d79c710a3a0a727bcf2d94e73280a9f62bb0ca Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 09:21:19 -0700
Subject: [PATCH 2/6] don't display warnings as errors

---
 flang/lib/Parser/message.cpp                |  6 +++++-
 flang/test/Driver/color-diagnostics-scan.f  | 16 ++++++++--------
 flang/test/Driver/fatal-errors-warnings.f90 | 15 +++++++--------
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 574e9f5d424f8..6644fafffc1bc 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -473,7 +473,11 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr, warningsAreErrors);
+    // We think it is confusing to users to display warnings and other
+    // diagnostics as errors, instead we just treat them as errors for the
+    // purpose of failing.
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
+        /*warningsAreErrors=*/false);
     lastMsg = msg;
     if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index 2f647c923c2e6..29d4635b4fb03 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -3,24 +3,24 @@
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
-! RUN: %flang %s -E -fcolor-diagnostics 2>&1 \
+! RUN: not %flang %s -E -Werror -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fno-color-diagnostics 2>&1 \
+! RUN: not %flang %s -E -Werror -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
-! RUN: %flang_fc1 -E %s -fcolor-diagnostics 2>&1 \
+! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 
-! RUN: %flang %s -E -fdiagnostics-color 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fno-diagnostics-color 2>&1 \
+! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: %flang %s -E -fdiagnostics-color=always 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fdiagnostics-color=never 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
 
diff --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
index b4b23230587a4..2de09c3ed0778 100644
--- a/flang/test/Driver/fatal-errors-warnings.f90
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -1,4 +1,4 @@
-! RUN: not %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
 ! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
 ! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
 
@@ -15,18 +15,17 @@ program test
     real :: b(1)
     call foo(a) ! ok
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
-    !CHECK3: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} warning:
     call foo(b)
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
     !CHECK3-NOT: error:
     !CHECK3-NOT: warning:
     call foo((a))
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
     call foo(a([1]))
-    !CHECK1: fatal-errors-warnings.f90:{{.*}} error:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
-    call foo(a(1))
+    !! Hard error instead of warning if uncommented.
+    !call foo(a(1))
 end
\ No newline at end of file

>From 892e255be088b117b94203b66d75688fcd3ebc28 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 11:07:22 -0700
Subject: [PATCH 3/6] add EffectiveSeverity(...)

---
 flang/include/flang/Parser/message.h  | 42 +++++++++++++++-----------
 flang/lib/Frontend/FrontendAction.cpp | 11 +++----
 flang/lib/Parser/message.cpp          | 43 ++++++++++++++++-----------
 flang/lib/Semantics/resolve-names.cpp |  3 +-
 4 files changed, 54 insertions(+), 45 deletions(-)

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index d1d313a70a016..423856b084b92 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -56,18 +56,22 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity(bool warningsAreErrors = false) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    }
-    return severity_;
-  }
+  Severity severity() const { return severity_; }
   MessageFixedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  // This is currently emulating how the frontend is currently handling other
+  // diagnostic messages. We may want to be more nuanced in the future.
+  Severity EffectiveSeverity(bool warningsAreErrors) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    } else {
+      return severity_;
+    }
+  }
   bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{severity(warningsAreErrors)};
+    Severity sev{EffectiveSeverity(warningsAreErrors)};
     return sev == Severity::Error || sev == Severity::Todo;
   }
 
@@ -111,7 +115,7 @@ class MessageFormattedText {
 public:
   template <typename... A>
   MessageFormattedText(const MessageFixedText &text, A &&...x)
-      : severity_{text.severity(false)} {
+      : severity_{text.severity()} {
     Format(&text, Convert(std::forward<A>(x))...);
   }
   MessageFormattedText(const MessageFormattedText &) = default;
@@ -119,18 +123,20 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
-  Severity severity(bool warningsAreErrors = false) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    }
-    return severity_;
-  }
+  Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  Severity EffectiveSeverity(bool warningsAreErrors) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    } else {
+      return severity_;
+    }
+  }
   bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{severity(warningsAreErrors)};
+    Severity sev{EffectiveSeverity(warningsAreErrors)};
     return sev == Severity::Error || sev == Severity::Todo;
   }
   std::string MoveString() { return std::move(string_); }
@@ -294,7 +300,8 @@ class Message : public common::ReferenceCounted<Message> {
 
   bool SortBefore(const Message &that) const;
   bool IsFatal(bool warningsAreErrors = false) const;
-  Severity severity(bool warningsAreErrors = false) const;
+  Severity EffectiveSeverity(bool warningsAreErrors) const;
+  Severity severity() const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
   Message &set_languageFeature(common::LanguageFeature);
@@ -305,8 +312,7 @@ class Message : public common::ReferenceCounted<Message> {
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLine = true,
-      const common::LanguageFeatureControl *hintFlags = nullptr,
-      bool warningsAreErrors = false) const;
+      const common::LanguageFeatureControl *hintFlags = nullptr) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index d32f477445f93..58901c6000380 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -230,16 +230,14 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
   const common::LanguageFeatureControl &features{
       instance->getInvocation().getFortranOpts().features};
   const size_t maxErrors{instance->getInvocation().getMaxErrors()};
-  if (!instance->getParsing().messages().empty() &&
-      (instance->getInvocation().getWarnAsErr() ||
-       instance->getParsing().messages().AnyFatalError())) {
+  const bool warningsAreErrors{instance->getInvocation().getWarnAsErr()};
+  if (instance->getParsing().messages().AnyFatalError(warningsAreErrors)) {
     const unsigned diagID = instance->getDiagnostics().getCustomDiagID(
         clang::DiagnosticsEngine::Error, message);
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLines=*/true, &features, maxErrors,
-        instance->getInvocation().getWarnAsErr());
+        /*echoSourceLines=*/true, &features, maxErrors, warningsAreErrors);
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -250,8 +248,7 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLine=*/true, &features, maxErrors,
-        instance->getInvocation().getWarnAsErr());
+        /*echoSourceLine=*/true, &features, maxErrors, warningsAreErrors);
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 6644fafffc1bc..80fe5b2a14ad2 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,25 +161,35 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-bool Message::IsFatal(bool warningsAreErrors) const {
-  Severity sev{severity(warningsAreErrors)};
-  return sev == Severity::Error || sev == Severity::Todo;
-}
-
-Severity Message::severity(bool warningsAreErrors) const {
+Severity Message::EffectiveSeverity(bool warningsAreErrors) const {
   return common::visit(
       common::visitors{
           [](const MessageExpectedText &) { return Severity::Error; },
           [=](const MessageFixedText &x) {
-            return x.severity(warningsAreErrors);
+            return x.EffectiveSeverity(warningsAreErrors);
           },
           [=](const MessageFormattedText &x) {
-            return x.severity(warningsAreErrors);
+            return x.EffectiveSeverity(warningsAreErrors);
           },
       },
       text_);
 }
 
+bool Message::IsFatal(bool warningsAreErrors) const {
+  Severity sev{EffectiveSeverity(warningsAreErrors)};
+  return sev == Severity::Error || sev == Severity::Todo;
+}
+
+Severity Message::severity() const {
+  return common::visit(
+      common::visitors{
+          [](const MessageExpectedText &) { return Severity::Error; },
+          [](const MessageFixedText &x) { return x.severity(); },
+          [](const MessageFormattedText &x) { return x.severity(); },
+      },
+      text_);
+}
+
 Message &Message::set_severity(Severity severity) {
   common::visit(
       common::visitors{
@@ -299,17 +309,18 @@ static std::string HintLanguageControlFlag(
 static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
+// We think it is confusing to users to display warnings and other
+// diagnostics as errors. Don't use EffectiveSeverity() here.
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine, const common::LanguageFeatureControl *hintFlagPtr,
-    bool warningsAreErrors) const {
+    bool echoSourceLine,
+    const common::LanguageFeatureControl *hintFlagPtr) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
   const std::string text{ToString()};
-  Severity sev{severity(warningsAreErrors)};
   const std::string hint{
       HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
-  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(sev),
-      PrefixColor(sev), echoSourceLine);
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
+      PrefixColor(severity()), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
   // attachment->attachmentIsContext_ indicates this.
@@ -473,11 +484,7 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    // We think it is confusing to users to display warnings and other
-    // diagnostics as errors, instead we just treat them as errors for the
-    // purpose of failing.
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
-        /*warningsAreErrors=*/false);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
     if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index df898f6e5651e..96faa5fd954cd 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7575,8 +7575,7 @@ bool DeclarationVisitor::OkToAddComponent(
       if (msg) {
         auto &said{Say2(name, std::move(*msg), *prev,
             "Previous declaration of '%s'"_en_US)};
-        if (msg->severity(/*warningsAreErrors=*/false) ==
-            parser::Severity::Error) {
+        if (msg->severity() == parser::Severity::Error) {
           Resolve(name, *prev);
           return false;
         }

>From 6bfd9255636df484d88fd46fbc84c368085501b6 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 15:07:25 -0700
Subject: [PATCH 4/6] remove actual promotion

---
 flang/include/flang/Parser/message.h | 28 +++++-----------------------
 flang/lib/Parser/message.cpp         | 26 +++++++-------------------
 2 files changed, 12 insertions(+), 42 deletions(-)

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index 423856b084b92..839bca158a78a 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -61,18 +61,9 @@ class MessageFixedText {
     severity_ = severity;
     return *this;
   }
-  // This is currently emulating how the frontend is currently handling other
-  // diagnostic messages. We may want to be more nuanced in the future.
-  Severity EffectiveSeverity(bool warningsAreErrors) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    } else {
-      return severity_;
-    }
-  }
-  bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{EffectiveSeverity(warningsAreErrors)};
-    return sev == Severity::Error || sev == Severity::Todo;
+
+  bool IsFatal() const {
+    return severity() == Severity::Error || severity() == Severity::Todo;
   }
 
 private:
@@ -128,16 +119,8 @@ class MessageFormattedText {
     severity_ = severity;
     return *this;
   }
-  Severity EffectiveSeverity(bool warningsAreErrors) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    } else {
-      return severity_;
-    }
-  }
   bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{EffectiveSeverity(warningsAreErrors)};
-    return sev == Severity::Error || sev == Severity::Todo;
+    return severity() == Severity::Error || severity() == Severity::Todo;
   }
   std::string MoveString() { return std::move(string_); }
   bool operator==(const MessageFormattedText &that) const {
@@ -299,8 +282,7 @@ class Message : public common::ReferenceCounted<Message> {
   }
 
   bool SortBefore(const Message &that) const;
-  bool IsFatal(bool warningsAreErrors = false) const;
-  Severity EffectiveSeverity(bool warningsAreErrors) const;
+  bool IsFatal() const;
   Severity severity() const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 80fe5b2a14ad2..36d570408cdb9 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,23 +161,8 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-Severity Message::EffectiveSeverity(bool warningsAreErrors) const {
-  return common::visit(
-      common::visitors{
-          [](const MessageExpectedText &) { return Severity::Error; },
-          [=](const MessageFixedText &x) {
-            return x.EffectiveSeverity(warningsAreErrors);
-          },
-          [=](const MessageFormattedText &x) {
-            return x.EffectiveSeverity(warningsAreErrors);
-          },
-      },
-      text_);
-}
-
-bool Message::IsFatal(bool warningsAreErrors) const {
-  Severity sev{EffectiveSeverity(warningsAreErrors)};
-  return sev == Severity::Error || sev == Severity::Todo;
+bool Message::IsFatal() const {
+  return severity() == Severity::Error || severity() == Severity::Todo;
 }
 
 Severity Message::severity() const {
@@ -486,7 +471,7 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     }
     msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
-    if (msg->IsFatal(warningsAreErrors)) {
+    if (warningsAreErrors || msg->IsFatal()) {
       ++errorsEmitted;
     }
     // If maxErrorsToEmit is 0, emit all errors, otherwise break after
@@ -512,8 +497,11 @@ bool Messages::AnyFatalError(bool warningsAreErrors) const {
   if (messages_.empty()) {
     return false;
   }
+  if (warningsAreErrors) {
+    return true;
+  }
   for (const auto &msg : messages_) {
-    if (msg.IsFatal(warningsAreErrors)) {
+    if (msg.IsFatal()) {
       return true;
     }
   }

>From 2693db8d3188c956204db8e70c8dc4ba97801875 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 15:15:55 -0700
Subject: [PATCH 5/6] clean up removal

---
 flang/include/flang/Parser/message.h | 9 ++++-----
 flang/lib/Parser/message.cpp         | 2 --
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index 839bca158a78a..9192d23529913 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -61,9 +61,8 @@ class MessageFixedText {
     severity_ = severity;
     return *this;
   }
-
   bool IsFatal() const {
-    return severity() == Severity::Error || severity() == Severity::Todo;
+    return severity_ == Severity::Error || severity_ == Severity::Todo;
   }
 
 private:
@@ -114,14 +113,14 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
+  bool IsFatal() const {
+    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  }
   Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
-  bool IsFatal(bool warningsAreErrors = false) const {
-    return severity() == Severity::Error || severity() == Severity::Todo;
-  }
   std::string MoveString() { return std::move(string_); }
   bool operator==(const MessageFormattedText &that) const {
     return severity_ == that.severity_ && string_ == that.string_;
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 36d570408cdb9..064494de9b478 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -294,8 +294,6 @@ static std::string HintLanguageControlFlag(
 static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
-// We think it is confusing to users to display warnings and other
-// diagnostics as errors. Don't use EffectiveSeverity() here.
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     bool echoSourceLine,
     const common::LanguageFeatureControl *hintFlagPtr) const {

>From da3cddb3dc58580fb7d0ed12bb9cb608b8397dd1 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Wed, 16 Jul 2025 11:27:13 -0700
Subject: [PATCH 6/6] clarify code

---
 flang/lib/Parser/message.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 064494de9b478..2a8101dd0b810 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -492,12 +492,17 @@ void Messages::AttachTo(Message &msg, std::optional<Severity> severity) {
 }
 
 bool Messages::AnyFatalError(bool warningsAreErrors) const {
+  // Short-circuit in the most common case.
   if (messages_.empty()) {
     return false;
   }
+  // If warnings are errors and there are warnings or errors, this is fatal.
+  // This preserves the compiler's current behavior of treating any non-fatal
+  // message as a warning. We may want to refine this in the future.
   if (warningsAreErrors) {
     return true;
   }
+  // Otherwise, check the message buffer for fatal errors.
   for (const auto &msg : messages_) {
     if (msg.IsFatal()) {
       return true;


From flang-commits at lists.llvm.org  Wed Jul 16 11:30:59 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 16 Jul 2025 11:30:59 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <6877efe3.170a0220.db915.cbf2@mx.google.com>


https://github.com/akuhlens updated https://github.com/llvm/llvm-project/pull/148748

>From b9cdd2e72c9c9e6b10098c4144b3e840122b56d6 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Mon, 14 Jul 2025 15:56:39 -0700
Subject: [PATCH 1/6] initial commit

---
 flang/include/flang/Parser/message.h        | 37 ++++++++++++++-------
 flang/lib/Frontend/FrontendAction.cpp       |  6 ++--
 flang/lib/Parser/message.cpp                | 37 +++++++++++++--------
 flang/lib/Semantics/resolve-names.cpp       |  3 +-
 flang/lib/Semantics/semantics.cpp           |  5 ++-
 flang/test/Driver/color-diagnostics-scan.f  | 16 ++++-----
 flang/test/Driver/fatal-errors-warnings.f90 | 32 ++++++++++++++++++
 7 files changed, 96 insertions(+), 40 deletions(-)
 create mode 100644 flang/test/Driver/fatal-errors-warnings.f90

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index db1a0a65157e3..d1d313a70a016 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -56,13 +56,19 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity() const { return severity_; }
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    }
+    return severity_;
+  }
   MessageFixedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
-  bool IsFatal() const {
-    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  bool IsFatal(bool warningsAreErrors = false) const {
+    Severity sev{severity(warningsAreErrors)};
+    return sev == Severity::Error || sev == Severity::Todo;
   }
 
 private:
@@ -105,7 +111,7 @@ class MessageFormattedText {
 public:
   template <typename... A>
   MessageFormattedText(const MessageFixedText &text, A &&...x)
-      : severity_{text.severity()} {
+      : severity_{text.severity(false)} {
     Format(&text, Convert(std::forward<A>(x))...);
   }
   MessageFormattedText(const MessageFormattedText &) = default;
@@ -113,14 +119,20 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
-  bool IsFatal() const {
-    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  Severity severity(bool warningsAreErrors = false) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    }
+    return severity_;
   }
-  Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  bool IsFatal(bool warningsAreErrors = false) const {
+    Severity sev{severity(warningsAreErrors)};
+    return sev == Severity::Error || sev == Severity::Todo;
+  }
   std::string MoveString() { return std::move(string_); }
   bool operator==(const MessageFormattedText &that) const {
     return severity_ == that.severity_ && string_ == that.string_;
@@ -281,8 +293,8 @@ class Message : public common::ReferenceCounted<Message> {
   }
 
   bool SortBefore(const Message &that) const;
-  bool IsFatal() const;
-  Severity severity() const;
+  bool IsFatal(bool warningsAreErrors = false) const;
+  Severity severity(bool warningsAreErrors = false) const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
   Message &set_languageFeature(common::LanguageFeature);
@@ -293,7 +305,8 @@ class Message : public common::ReferenceCounted<Message> {
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLine = true,
-      const common::LanguageFeatureControl *hintFlags = nullptr) const;
+      const common::LanguageFeatureControl *hintFlags = nullptr,
+      bool warningsAreErrors = false) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
@@ -355,9 +368,9 @@ class Messages {
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLines = true,
       const common::LanguageFeatureControl *hintFlags = nullptr,
-      std::size_t maxErrorsToEmit = 0) const;
+      std::size_t maxErrorsToEmit = 0, bool warningsAreErrors = false) const;
   void AttachTo(Message &, std::optional<Severity> = std::nullopt);
-  bool AnyFatalError() const;
+  bool AnyFatalError(bool warningsAreErrors = false) const;
 
 private:
   std::list<Message> messages_;
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index 2429e07e5b8c4..d32f477445f93 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -238,7 +238,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLines=*/true, &features, maxErrors);
+        /*echoSourceLines=*/true, &features, maxErrors,
+        instance->getInvocation().getWarnAsErr());
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -249,7 +250,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLine=*/true, &features, maxErrors);
+        /*echoSourceLine=*/true, &features, maxErrors,
+        instance->getInvocation().getWarnAsErr());
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 909fba948a45a..574e9f5d424f8 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,16 +161,21 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-bool Message::IsFatal() const {
-  return severity() == Severity::Error || severity() == Severity::Todo;
+bool Message::IsFatal(bool warningsAreErrors) const {
+  Severity sev{severity(warningsAreErrors)};
+  return sev == Severity::Error || sev == Severity::Todo;
 }
 
-Severity Message::severity() const {
+Severity Message::severity(bool warningsAreErrors) const {
   return common::visit(
       common::visitors{
           [](const MessageExpectedText &) { return Severity::Error; },
-          [](const MessageFixedText &x) { return x.severity(); },
-          [](const MessageFormattedText &x) { return x.severity(); },
+          [=](const MessageFixedText &x) {
+            return x.severity(warningsAreErrors);
+          },
+          [=](const MessageFormattedText &x) {
+            return x.severity(warningsAreErrors);
+          },
       },
       text_);
 }
@@ -295,15 +300,16 @@ static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine,
-    const common::LanguageFeatureControl *hintFlagPtr) const {
+    bool echoSourceLine, const common::LanguageFeatureControl *hintFlagPtr,
+    bool warningsAreErrors) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
   const std::string text{ToString()};
+  Severity sev{severity(warningsAreErrors)};
   const std::string hint{
       HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
-  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
-      PrefixColor(severity()), echoSourceLine);
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(sev),
+      PrefixColor(sev), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
   // attachment->attachmentIsContext_ indicates this.
@@ -453,7 +459,7 @@ void Messages::ResolveProvenances(const AllCookedSources &allCooked) {
 
 void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     bool echoSourceLines, const common::LanguageFeatureControl *hintFlagPtr,
-    std::size_t maxErrorsToEmit) const {
+    std::size_t maxErrorsToEmit, bool warningsAreErrors) const {
   std::vector<const Message *> sorted;
   for (const auto &msg : messages_) {
     sorted.push_back(&msg);
@@ -467,9 +473,9 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr, warningsAreErrors);
     lastMsg = msg;
-    if (msg->IsFatal()) {
+    if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
     }
     // If maxErrorsToEmit is 0, emit all errors, otherwise break after
@@ -491,9 +497,12 @@ void Messages::AttachTo(Message &msg, std::optional<Severity> severity) {
   messages_.clear();
 }
 
-bool Messages::AnyFatalError() const {
+bool Messages::AnyFatalError(bool warningsAreErrors) const {
+  if (messages_.empty()) {
+    return false;
+  }
   for (const auto &msg : messages_) {
-    if (msg.IsFatal()) {
+    if (msg.IsFatal(warningsAreErrors)) {
       return true;
     }
   }
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 96faa5fd954cd..df898f6e5651e 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7575,7 +7575,8 @@ bool DeclarationVisitor::OkToAddComponent(
       if (msg) {
         auto &said{Say2(name, std::move(*msg), *prev,
             "Previous declaration of '%s'"_en_US)};
-        if (msg->severity() == parser::Severity::Error) {
+        if (msg->severity(/*warningsAreErrors=*/false) ==
+            parser::Severity::Error) {
           Resolve(name, *prev);
           return false;
         }
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -376,8 +376,7 @@ const DeclTypeSpec &SemanticsContext::MakeLogicalType(int kind) {
 }
 
 bool SemanticsContext::AnyFatalError() const {
-  return !messages_.empty() &&
-      (warningsAreErrors_ || messages_.AnyFatalError());
+  return messages_.AnyFatalError(warningsAreErrors_);
 }
 bool SemanticsContext::HasError(const Symbol &symbol) {
   return errorSymbols_.count(symbol) > 0;
@@ -658,7 +657,7 @@ void Semantics::EmitMessages(llvm::raw_ostream &os) {
   context_.messages().ResolveProvenances(context_.allCookedSources());
   context_.messages().Emit(os, context_.allCookedSources(),
       /*echoSourceLine=*/true, &context_.languageFeatures(),
-      /*maxErrorsToEmit=*/context_.maxErrors());
+      context_.maxErrors(), context_.warningsAreErrors());
 }
 
 void SemanticsContext::DumpSymbols(llvm::raw_ostream &os) {
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index 29d4635b4fb03..2f647c923c2e6 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -3,24 +3,24 @@
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
-! RUN: not %flang %s -E -Werror -fcolor-diagnostics 2>&1 \
+! RUN: %flang %s -E -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fno-color-diagnostics 2>&1 \
+! RUN: %flang %s -E -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
-! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
+! RUN: %flang_fc1 -E %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 
-! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
+! RUN: %flang %s -E -fno-diagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color=always 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
+! RUN: %flang %s -E -fdiagnostics-color=never 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
 
diff --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
new file mode 100644
index 0000000000000..b4b23230587a4
--- /dev/null
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -0,0 +1,32 @@
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
+
+module m
+    contains
+    subroutine foo(a)
+        real, intent(in), target :: a(:)
+    end subroutine
+end module
+
+program test
+    use m
+    real, target :: a(1)
+    real :: b(1)
+    call foo(a) ! ok
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(b)
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK3-NOT: error:
+    !CHECK3-NOT: warning:
+    call foo((a))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(a([1]))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    call foo(a(1))
+end
\ No newline at end of file

>From 31d79c710a3a0a727bcf2d94e73280a9f62bb0ca Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 09:21:19 -0700
Subject: [PATCH 2/6] don't display warnings as errors

---
 flang/lib/Parser/message.cpp                |  6 +++++-
 flang/test/Driver/color-diagnostics-scan.f  | 16 ++++++++--------
 flang/test/Driver/fatal-errors-warnings.f90 | 15 +++++++--------
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 574e9f5d424f8..6644fafffc1bc 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -473,7 +473,11 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr, warningsAreErrors);
+    // We think it is confusing to users to display warnings and other
+    // diagnostics as errors, instead we just treat them as errors for the
+    // purpose of failing.
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
+        /*warningsAreErrors=*/false);
     lastMsg = msg;
     if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index 2f647c923c2e6..29d4635b4fb03 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -3,24 +3,24 @@
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
-! RUN: %flang %s -E -fcolor-diagnostics 2>&1 \
+! RUN: not %flang %s -E -Werror -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fno-color-diagnostics 2>&1 \
+! RUN: not %flang %s -E -Werror -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
-! RUN: %flang_fc1 -E %s -fcolor-diagnostics 2>&1 \
+! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 
-! RUN: %flang %s -E -fdiagnostics-color 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fno-diagnostics-color 2>&1 \
+! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: %flang %s -E -fdiagnostics-color=always 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
-! RUN: %flang %s -E -fdiagnostics-color=never 2>&1 \
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 
-! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
 
diff --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
index b4b23230587a4..2de09c3ed0778 100644
--- a/flang/test/Driver/fatal-errors-warnings.f90
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -1,4 +1,4 @@
-! RUN: not %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
 ! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
 ! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
 
@@ -15,18 +15,17 @@ program test
     real :: b(1)
     call foo(a) ! ok
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
-    !CHECK3: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} warning:
     call foo(b)
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
     !CHECK3-NOT: error:
     !CHECK3-NOT: warning:
     call foo((a))
     !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
     call foo(a([1]))
-    !CHECK1: fatal-errors-warnings.f90:{{.*}} error:
-    !CHECK2: fatal-errors-warnings.f90:{{.*}} error:
-    call foo(a(1))
+    !! Hard error instead of warning if uncommented.
+    !call foo(a(1))
 end
\ No newline at end of file

>From 892e255be088b117b94203b66d75688fcd3ebc28 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 11:07:22 -0700
Subject: [PATCH 3/6] add EffectiveSeverity(...)

---
 flang/include/flang/Parser/message.h  | 42 +++++++++++++++-----------
 flang/lib/Frontend/FrontendAction.cpp | 11 +++----
 flang/lib/Parser/message.cpp          | 43 ++++++++++++++++-----------
 flang/lib/Semantics/resolve-names.cpp |  3 +-
 4 files changed, 54 insertions(+), 45 deletions(-)

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index d1d313a70a016..423856b084b92 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -56,18 +56,22 @@ class MessageFixedText {
 
   CharBlock text() const { return text_; }
   bool empty() const { return text_.empty(); }
-  Severity severity(bool warningsAreErrors = false) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    }
-    return severity_;
-  }
+  Severity severity() const { return severity_; }
   MessageFixedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  // This is currently emulating how the frontend is currently handling other
+  // diagnostic messages. We may want to be more nuanced in the future.
+  Severity EffectiveSeverity(bool warningsAreErrors) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    } else {
+      return severity_;
+    }
+  }
   bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{severity(warningsAreErrors)};
+    Severity sev{EffectiveSeverity(warningsAreErrors)};
     return sev == Severity::Error || sev == Severity::Todo;
   }
 
@@ -111,7 +115,7 @@ class MessageFormattedText {
 public:
   template <typename... A>
   MessageFormattedText(const MessageFixedText &text, A &&...x)
-      : severity_{text.severity(false)} {
+      : severity_{text.severity()} {
     Format(&text, Convert(std::forward<A>(x))...);
   }
   MessageFormattedText(const MessageFormattedText &) = default;
@@ -119,18 +123,20 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
-  Severity severity(bool warningsAreErrors = false) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    }
-    return severity_;
-  }
+  Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
+  Severity EffectiveSeverity(bool warningsAreErrors) const {
+    if (warningsAreErrors) {
+      return Severity::Error;
+    } else {
+      return severity_;
+    }
+  }
   bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{severity(warningsAreErrors)};
+    Severity sev{EffectiveSeverity(warningsAreErrors)};
     return sev == Severity::Error || sev == Severity::Todo;
   }
   std::string MoveString() { return std::move(string_); }
@@ -294,7 +300,8 @@ class Message : public common::ReferenceCounted<Message> {
 
   bool SortBefore(const Message &that) const;
   bool IsFatal(bool warningsAreErrors = false) const;
-  Severity severity(bool warningsAreErrors = false) const;
+  Severity EffectiveSeverity(bool warningsAreErrors) const;
+  Severity severity() const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
   Message &set_languageFeature(common::LanguageFeature);
@@ -305,8 +312,7 @@ class Message : public common::ReferenceCounted<Message> {
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLine = true,
-      const common::LanguageFeatureControl *hintFlags = nullptr,
-      bool warningsAreErrors = false) const;
+      const common::LanguageFeatureControl *hintFlags = nullptr) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index d32f477445f93..58901c6000380 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -230,16 +230,14 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
   const common::LanguageFeatureControl &features{
       instance->getInvocation().getFortranOpts().features};
   const size_t maxErrors{instance->getInvocation().getMaxErrors()};
-  if (!instance->getParsing().messages().empty() &&
-      (instance->getInvocation().getWarnAsErr() ||
-       instance->getParsing().messages().AnyFatalError())) {
+  const bool warningsAreErrors{instance->getInvocation().getWarnAsErr()};
+  if (instance->getParsing().messages().AnyFatalError(warningsAreErrors)) {
     const unsigned diagID = instance->getDiagnostics().getCustomDiagID(
         clang::DiagnosticsEngine::Error, message);
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLines=*/true, &features, maxErrors,
-        instance->getInvocation().getWarnAsErr());
+        /*echoSourceLines=*/true, &features, maxErrors, warningsAreErrors);
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -250,8 +248,7 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLine=*/true, &features, maxErrors,
-        instance->getInvocation().getWarnAsErr());
+        /*echoSourceLine=*/true, &features, maxErrors, warningsAreErrors);
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 6644fafffc1bc..80fe5b2a14ad2 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,25 +161,35 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-bool Message::IsFatal(bool warningsAreErrors) const {
-  Severity sev{severity(warningsAreErrors)};
-  return sev == Severity::Error || sev == Severity::Todo;
-}
-
-Severity Message::severity(bool warningsAreErrors) const {
+Severity Message::EffectiveSeverity(bool warningsAreErrors) const {
   return common::visit(
       common::visitors{
           [](const MessageExpectedText &) { return Severity::Error; },
           [=](const MessageFixedText &x) {
-            return x.severity(warningsAreErrors);
+            return x.EffectiveSeverity(warningsAreErrors);
           },
           [=](const MessageFormattedText &x) {
-            return x.severity(warningsAreErrors);
+            return x.EffectiveSeverity(warningsAreErrors);
           },
       },
       text_);
 }
 
+bool Message::IsFatal(bool warningsAreErrors) const {
+  Severity sev{EffectiveSeverity(warningsAreErrors)};
+  return sev == Severity::Error || sev == Severity::Todo;
+}
+
+Severity Message::severity() const {
+  return common::visit(
+      common::visitors{
+          [](const MessageExpectedText &) { return Severity::Error; },
+          [](const MessageFixedText &x) { return x.severity(); },
+          [](const MessageFormattedText &x) { return x.severity(); },
+      },
+      text_);
+}
+
 Message &Message::set_severity(Severity severity) {
   common::visit(
       common::visitors{
@@ -299,17 +309,18 @@ static std::string HintLanguageControlFlag(
 static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
+// We think it is confusing to users to display warnings and other
+// diagnostics as errors. Don't use EffectiveSeverity() here.
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine, const common::LanguageFeatureControl *hintFlagPtr,
-    bool warningsAreErrors) const {
+    bool echoSourceLine,
+    const common::LanguageFeatureControl *hintFlagPtr) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
   const std::string text{ToString()};
-  Severity sev{severity(warningsAreErrors)};
   const std::string hint{
       HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
-  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(sev),
-      PrefixColor(sev), echoSourceLine);
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
+      PrefixColor(severity()), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
   // attachment->attachmentIsContext_ indicates this.
@@ -473,11 +484,7 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    // We think it is confusing to users to display warnings and other
-    // diagnostics as errors, instead we just treat them as errors for the
-    // purpose of failing.
-    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr,
-        /*warningsAreErrors=*/false);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
     if (msg->IsFatal(warningsAreErrors)) {
       ++errorsEmitted;
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index df898f6e5651e..96faa5fd954cd 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7575,8 +7575,7 @@ bool DeclarationVisitor::OkToAddComponent(
       if (msg) {
         auto &said{Say2(name, std::move(*msg), *prev,
             "Previous declaration of '%s'"_en_US)};
-        if (msg->severity(/*warningsAreErrors=*/false) ==
-            parser::Severity::Error) {
+        if (msg->severity() == parser::Severity::Error) {
           Resolve(name, *prev);
           return false;
         }

>From 6bfd9255636df484d88fd46fbc84c368085501b6 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 15:07:25 -0700
Subject: [PATCH 4/6] remove actual promotion

---
 flang/include/flang/Parser/message.h | 28 +++++-----------------------
 flang/lib/Parser/message.cpp         | 26 +++++++-------------------
 2 files changed, 12 insertions(+), 42 deletions(-)

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index 423856b084b92..839bca158a78a 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -61,18 +61,9 @@ class MessageFixedText {
     severity_ = severity;
     return *this;
   }
-  // This is currently emulating how the frontend is currently handling other
-  // diagnostic messages. We may want to be more nuanced in the future.
-  Severity EffectiveSeverity(bool warningsAreErrors) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    } else {
-      return severity_;
-    }
-  }
-  bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{EffectiveSeverity(warningsAreErrors)};
-    return sev == Severity::Error || sev == Severity::Todo;
+
+  bool IsFatal() const {
+    return severity() == Severity::Error || severity() == Severity::Todo;
   }
 
 private:
@@ -128,16 +119,8 @@ class MessageFormattedText {
     severity_ = severity;
     return *this;
   }
-  Severity EffectiveSeverity(bool warningsAreErrors) const {
-    if (warningsAreErrors) {
-      return Severity::Error;
-    } else {
-      return severity_;
-    }
-  }
   bool IsFatal(bool warningsAreErrors = false) const {
-    Severity sev{EffectiveSeverity(warningsAreErrors)};
-    return sev == Severity::Error || sev == Severity::Todo;
+    return severity() == Severity::Error || severity() == Severity::Todo;
   }
   std::string MoveString() { return std::move(string_); }
   bool operator==(const MessageFormattedText &that) const {
@@ -299,8 +282,7 @@ class Message : public common::ReferenceCounted<Message> {
   }
 
   bool SortBefore(const Message &that) const;
-  bool IsFatal(bool warningsAreErrors = false) const;
-  Severity EffectiveSeverity(bool warningsAreErrors) const;
+  bool IsFatal() const;
   Severity severity() const;
   Message &set_severity(Severity);
   std::optional<common::LanguageFeature> languageFeature() const;
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 80fe5b2a14ad2..36d570408cdb9 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -161,23 +161,8 @@ bool Message::SortBefore(const Message &that) const {
       location_, that.location_);
 }
 
-Severity Message::EffectiveSeverity(bool warningsAreErrors) const {
-  return common::visit(
-      common::visitors{
-          [](const MessageExpectedText &) { return Severity::Error; },
-          [=](const MessageFixedText &x) {
-            return x.EffectiveSeverity(warningsAreErrors);
-          },
-          [=](const MessageFormattedText &x) {
-            return x.EffectiveSeverity(warningsAreErrors);
-          },
-      },
-      text_);
-}
-
-bool Message::IsFatal(bool warningsAreErrors) const {
-  Severity sev{EffectiveSeverity(warningsAreErrors)};
-  return sev == Severity::Error || sev == Severity::Todo;
+bool Message::IsFatal() const {
+  return severity() == Severity::Error || severity() == Severity::Todo;
 }
 
 Severity Message::severity() const {
@@ -486,7 +471,7 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     }
     msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
-    if (msg->IsFatal(warningsAreErrors)) {
+    if (warningsAreErrors || msg->IsFatal()) {
       ++errorsEmitted;
     }
     // If maxErrorsToEmit is 0, emit all errors, otherwise break after
@@ -512,8 +497,11 @@ bool Messages::AnyFatalError(bool warningsAreErrors) const {
   if (messages_.empty()) {
     return false;
   }
+  if (warningsAreErrors) {
+    return true;
+  }
   for (const auto &msg : messages_) {
-    if (msg.IsFatal(warningsAreErrors)) {
+    if (msg.IsFatal()) {
       return true;
     }
   }

>From 2693db8d3188c956204db8e70c8dc4ba97801875 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Tue, 15 Jul 2025 15:15:55 -0700
Subject: [PATCH 5/6] clean up removal

---
 flang/include/flang/Parser/message.h | 9 ++++-----
 flang/lib/Parser/message.cpp         | 2 --
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index 839bca158a78a..9192d23529913 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -61,9 +61,8 @@ class MessageFixedText {
     severity_ = severity;
     return *this;
   }
-
   bool IsFatal() const {
-    return severity() == Severity::Error || severity() == Severity::Todo;
+    return severity_ == Severity::Error || severity_ == Severity::Todo;
   }
 
 private:
@@ -114,14 +113,14 @@ class MessageFormattedText {
   MessageFormattedText &operator=(const MessageFormattedText &) = default;
   MessageFormattedText &operator=(MessageFormattedText &&) = default;
   const std::string &string() const { return string_; }
+  bool IsFatal() const {
+    return severity_ == Severity::Error || severity_ == Severity::Todo;
+  }
   Severity severity() const { return severity_; }
   MessageFormattedText &set_severity(Severity severity) {
     severity_ = severity;
     return *this;
   }
-  bool IsFatal(bool warningsAreErrors = false) const {
-    return severity() == Severity::Error || severity() == Severity::Todo;
-  }
   std::string MoveString() { return std::move(string_); }
   bool operator==(const MessageFormattedText &that) const {
     return severity_ == that.severity_ && string_ == that.string_;
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 36d570408cdb9..064494de9b478 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -294,8 +294,6 @@ static std::string HintLanguageControlFlag(
 static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
-// We think it is confusing to users to display warnings and other
-// diagnostics as errors. Don't use EffectiveSeverity() here.
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     bool echoSourceLine,
     const common::LanguageFeatureControl *hintFlagPtr) const {

>From da3cddb3dc58580fb7d0ed12bb9cb608b8397dd1 Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Wed, 16 Jul 2025 11:27:13 -0700
Subject: [PATCH 6/6] clarify code

---
 flang/lib/Parser/message.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 064494de9b478..2a8101dd0b810 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -492,12 +492,17 @@ void Messages::AttachTo(Message &msg, std::optional<Severity> severity) {
 }
 
 bool Messages::AnyFatalError(bool warningsAreErrors) const {
+  // Short-circuit in the most common case.
   if (messages_.empty()) {
     return false;
   }
+  // If warnings are errors and there are warnings or errors, this is fatal.
+  // This preserves the compiler's current behavior of treating any non-fatal
+  // message as a warning. We may want to refine this in the future.
   if (warningsAreErrors) {
     return true;
   }
+  // Otherwise, check the message buffer for fatal errors.
   for (const auto &msg : messages_) {
     if (msg.IsFatal()) {
       return true;


From flang-commits at lists.llvm.org  Wed Jul 16 11:48:35 2025
From: flang-commits at lists.llvm.org (Kelvin Li via flang-commits)
Date: Wed, 16 Jul 2025 11:48:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
Message-ID: <llvm.org/llvm/llvm-project/pull/149165@github.com>


https://github.com/kkwli created https://github.com/llvm/llvm-project/pull/149165

This PR handles the allocation of zero-sized objects for different implementations. One byte is allocated for the zero-sized objects.

>From 8ea361b26adbad56aa8c0b04d2b1113fe3e20fd7 Mon Sep 17 00:00:00 2001
From: Kelvin Li <kli at ca.ibm.com>
Date: Wed, 16 Jul 2025 14:24:43 -0400
Subject: [PATCH] [flang] handle allocation of zero-sized objects

---
 flang/lib/Optimizer/CodeGen/CodeGen.cpp | 16 +++++++++--
 flang/test/Lower/zero-size2.f90         | 37 +++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Lower/zero-size2.f90

diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index ecc04a6c9a2be..06686005bf2c9 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1119,9 +1119,19 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
     if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    for (mlir::Value opnd : adaptor.getOperands()) {
+      auto arg = integerCast(loc, rewriter, ity, opnd);
+      auto val = fir::getIntIfConstant(arg);
+      if (val && *val == 0) {
+        // As the return value of malloc(0) is implementation defined, allocate
+        // one byte to ensure the allocation status being true. This behavior
+        // aligns to what the runtime has.
+        size = genConstantIndex(loc, ity, rewriter, 1);
+        break;
+      } else {
+        size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, arg);
+      }
+    }
     auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
     auto mallocTy =
         mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
diff --git a/flang/test/Lower/zero-size2.f90 b/flang/test/Lower/zero-size2.f90
new file mode 100644
index 0000000000000..efaf57a6ac59f
--- /dev/null
+++ b/flang/test/Lower/zero-size2.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-llvm -o - %s | FileCheck %s
+
+subroutine sub1()
+  integer, allocatable :: arr(:)
+  allocate(arr(0))
+! CHECK-LABEL: @sub1_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end
+
+subroutine sub2()
+  real, allocatable :: arr(:,:)
+  allocate(arr(10,0))
+! CHECK-LABEL: @sub2_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end
+
+subroutine sub3(i)
+  integer :: i
+  real, allocatable :: arr(:,:)
+  allocate(arr(i,0))
+! CHECK-LABEL: @sub3_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end
+
+subroutine sub4()
+  character(:), allocatable :: c
+  allocate(character(0)::c)
+! CHECK-LABEL: @sub4_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end  
+
+subroutine sub5()
+  character(:), allocatable :: c(:)
+  allocate(character(5)::c(0))
+! CHECK-LABEL: @sub5_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end


From flang-commits at lists.llvm.org  Wed Jul 16 11:49:04 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 11:49:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149165@github.com>
Message-ID: <6877f420.170a0220.34474f.209d@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-codegen

@llvm/pr-subscribers-flang-fir-hlfir

Author: Kelvin Li (kkwli)

<details>
<summary>Changes</summary>

This PR handles the allocation of zero-sized objects for different implementations. One byte is allocated for the zero-sized objects.

---
Full diff: https://github.com/llvm/llvm-project/pull/149165.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/CodeGen/CodeGen.cpp (+13-3) 
- (added) flang/test/Lower/zero-size2.f90 (+37) 


``````````diff
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index ecc04a6c9a2be..06686005bf2c9 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1119,9 +1119,19 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
     if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    for (mlir::Value opnd : adaptor.getOperands()) {
+      auto arg = integerCast(loc, rewriter, ity, opnd);
+      auto val = fir::getIntIfConstant(arg);
+      if (val && *val == 0) {
+        // As the return value of malloc(0) is implementation defined, allocate
+        // one byte to ensure the allocation status being true. This behavior
+        // aligns to what the runtime has.
+        size = genConstantIndex(loc, ity, rewriter, 1);
+        break;
+      } else {
+        size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, arg);
+      }
+    }
     auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
     auto mallocTy =
         mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
diff --git a/flang/test/Lower/zero-size2.f90 b/flang/test/Lower/zero-size2.f90
new file mode 100644
index 0000000000000..efaf57a6ac59f
--- /dev/null
+++ b/flang/test/Lower/zero-size2.f90
@@ -0,0 +1,37 @@
+! RUN: %flang_fc1 -emit-llvm -o - %s | FileCheck %s
+
+subroutine sub1()
+  integer, allocatable :: arr(:)
+  allocate(arr(0))
+! CHECK-LABEL: @sub1_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end
+
+subroutine sub2()
+  real, allocatable :: arr(:,:)
+  allocate(arr(10,0))
+! CHECK-LABEL: @sub2_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end
+
+subroutine sub3(i)
+  integer :: i
+  real, allocatable :: arr(:,:)
+  allocate(arr(i,0))
+! CHECK-LABEL: @sub3_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end
+
+subroutine sub4()
+  character(:), allocatable :: c
+  allocate(character(0)::c)
+! CHECK-LABEL: @sub4_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end  
+
+subroutine sub5()
+  character(:), allocatable :: c(:)
+  allocate(character(5)::c(0))
+! CHECK-LABEL: @sub5_
+! CHECK: %[[p:.*]] = call ptr @malloc(i64 1)
+end

``````````

</details>


https://github.com/llvm/llvm-project/pull/149165

From flang-commits at lists.llvm.org  Wed Jul 16 11:50:19 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 11:50:19 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149165@github.com>
Message-ID: <6877f46b.170a0220.28c1fe.2664@mx.google.com>


github-actions[bot] wrote:

⚠️ We detected that you are using a GitHub private e-mail address to contribute to the repo.<br/> Please turn off [Keep my email addresses private](https://github.com/settings/emails) setting in your account.<br/> See [LLVM Developer Policy](https://llvm.org/docs/DeveloperPolicy.html#email-addresses) and [LLVM Discourse](https://discourse.llvm.org/t/hidden-emails-on-github-should-we-do-something-about-it) for more information.

https://github.com/llvm/llvm-project/pull/149165

From flang-commits at lists.llvm.org  Wed Jul 16 11:56:32 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Wed, 16 Jul 2025 11:56:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149165@github.com>
Message-ID: <6877f5e0.170a0220.187b9d.2723@mx.google.com>


================
@@ -1119,9 +1119,19 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
     if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    for (mlir::Value opnd : adaptor.getOperands()) {
+      auto arg = integerCast(loc, rewriter, ity, opnd);
+      auto val = fir::getIntIfConstant(arg);
+      if (val && *val == 0) {
+        // As the return value of malloc(0) is implementation defined, allocate
+        // one byte to ensure the allocation status being true. This behavior
+        // aligns to what the runtime has.
+        size = genConstantIndex(loc, ity, rewriter, 1);
+        break;
----------------
DanielCChen wrote:

Does this `break` prevent this code from handling multiple allocate objects with zero-size?

https://github.com/llvm/llvm-project/pull/149165

From flang-commits at lists.llvm.org  Wed Jul 16 11:56:33 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Wed, 16 Jul 2025 11:56:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149165@github.com>
Message-ID: <6877f5e1.170a0220.2e4a44.2105@mx.google.com>


================
@@ -1119,9 +1119,19 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
     if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    for (mlir::Value opnd : adaptor.getOperands()) {
+      auto arg = integerCast(loc, rewriter, ity, opnd);
+      auto val = fir::getIntIfConstant(arg);
+      if (val && *val == 0) {
+        // As the return value of malloc(0) is implementation defined, allocate
+        // one byte to ensure the allocation status being true. This behavior
+        // aligns to what the runtime has.
+        size = genConstantIndex(loc, ity, rewriter, 1);
+        break;
+      } else {
+        size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, arg);
----------------
DanielCChen wrote:

I may read it wrong, but should this be also called for the `if` part at the above?

https://github.com/llvm/llvm-project/pull/149165

From flang-commits at lists.llvm.org  Wed Jul 16 12:00:09 2025
From: flang-commits at lists.llvm.org (Kelvin Li via flang-commits)
Date: Wed, 16 Jul 2025 12:00:09 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149165@github.com>
Message-ID: <6877f6b9.170a0220.17728f.2f2e@mx.google.com>


================
@@ -1119,9 +1119,19 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
     if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    for (mlir::Value opnd : adaptor.getOperands()) {
+      auto arg = integerCast(loc, rewriter, ity, opnd);
+      auto val = fir::getIntIfConstant(arg);
+      if (val && *val == 0) {
+        // As the return value of malloc(0) is implementation defined, allocate
+        // one byte to ensure the allocation status being true. This behavior
+        // aligns to what the runtime has.
+        size = genConstantIndex(loc, ity, rewriter, 1);
+        break;
----------------
kkwli wrote:

No. This break is to stop computing the remaining dimensions of the same object. The other allocatable variables on the same allocate statement are handled separately.

https://github.com/llvm/llvm-project/pull/149165

From flang-commits at lists.llvm.org  Wed Jul 16 12:03:03 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Wed, 16 Jul 2025 12:03:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149165@github.com>
Message-ID: <6877f767.170a0220.69d84.33d9@mx.google.com>


================
@@ -1119,9 +1119,19 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
     if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
       size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
-    for (mlir::Value opnd : adaptor.getOperands())
-      size = rewriter.create<mlir::LLVM::MulOp>(
-          loc, ity, size, integerCast(loc, rewriter, ity, opnd));
+    for (mlir::Value opnd : adaptor.getOperands()) {
+      auto arg = integerCast(loc, rewriter, ity, opnd);
+      auto val = fir::getIntIfConstant(arg);
----------------
vzakhari wrote:

Thanks for the changes!

Is that possible to have a dynamic size value in `fir.allocmem` operation?  I think it is, so the check has to be dynamic, i.e. we need to generate a select `size <= 0 ? 1 : size` instead.

https://github.com/llvm/llvm-project/pull/149165

From flang-commits at lists.llvm.org  Wed Jul 16 12:11:21 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Wed, 16 Jul 2025 12:11:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Handle SEQUENCE derived types for
 array repacking. (PR #148777)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148777@github.com>
Message-ID: <6877f959.630a0220.8eb08.6d82@mx.google.com>


https://github.com/vzakhari closed https://github.com/llvm/llvm-project/pull/148777

From flang-commits at lists.llvm.org  Wed Jul 16 12:13:15 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 16 Jul 2025 12:13:15 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <6877f9cb.050a0220.39ccee.872e@mx.google.com>


https://github.com/eugeneepshteyn edited https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Wed Jul 16 12:13:38 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 12:13:38 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <6877f9e2.170a0220.1d41b0.21df@mx.google.com>


github-actions[bot] wrote:

⚠️ We detected that you are using a GitHub private e-mail address to contribute to the repo.<br/> Please turn off [Keep my email addresses private](https://github.com/settings/emails) setting in your account.<br/> See [LLVM Developer Policy](https://llvm.org/docs/DeveloperPolicy.html#email-addresses) and [LLVM Discourse](https://discourse.llvm.org/t/hidden-emails-on-github-should-we-do-something-about-it) for more information.

https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Wed Jul 16 12:20:40 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 16 Jul 2025 12:20:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <6877fb88.050a0220.35840a.5e58@mx.google.com>


eugeneepshteyn wrote:

> ⚠️ We detected that you are using a GitHub private e-mail address to contribute to the repo. Please turn off [Keep my email addresses private](https://github.com/settings/emails) setting in your account. See [LLVM Developer Policy](https://llvm.org/docs/DeveloperPolicy.html#email-addresses) and [LLVM Discourse](https://discourse.llvm.org/t/hidden-emails-on-github-should-we-do-something-about-it) for more information.

No I'm not!

https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Wed Jul 16 12:21:45 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 16 Jul 2025 12:21:45 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <6877fbc9.170a0220.1dcc01.257a@mx.google.com>


https://github.com/eugeneepshteyn ready_for_review https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Wed Jul 16 12:24:03 2025
From: flang-commits at lists.llvm.org (Zhen Wang via flang-commits)
Date: Wed, 16 Jul 2025 12:24:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Import type descriptor in the
 gpu module when needed (PR #149157)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149157@github.com>
Message-ID: <6877fc53.170a0220.d4b52.8a8e@mx.google.com>


https://github.com/wangzpgi approved this pull request.


https://github.com/llvm/llvm-project/pull/149157

From flang-commits at lists.llvm.org  Wed Jul 16 12:26:41 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 12:26:41 -0700 (PDT)
Subject: [flang-commits] [flang] b3c72a9 - [flang][driver] -Werror promotes
 warnings to error and interopts with -Wfatal-errors (#148748)
Message-ID: <6877fcf1.170a0220.11d4b5.2a47@mx.google.com>


Author: Andre Kuhlenschmidt
Date: 2025-07-16T12:26:38-07:00
New Revision: b3c72a97c5ac352b89c12f3cf7c3f223219f91ed

URL: https://github.com/llvm/llvm-project/commit/b3c72a97c5ac352b89c12f3cf7c3f223219f91ed
DIFF: https://github.com/llvm/llvm-project/commit/b3c72a97c5ac352b89c12f3cf7c3f223219f91ed.diff

LOG: [flang][driver] -Werror promotes warnings to error and interopts with -Wfatal-errors (#148748)

This PR changes how `-Werror` promotes warnings to errors so that it
interoperates with `-Wfatal-error`. It maintains the property that
warnings and other messages promoted to errors are displayed as there
original message.

Added: 
    flang/test/Driver/fatal-errors-warnings.f90

Modified: 
    flang/include/flang/Parser/message.h
    flang/lib/Frontend/FrontendAction.cpp
    flang/lib/Parser/message.cpp
    flang/lib/Semantics/semantics.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index db1a0a65157e3..9192d23529913 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -355,9 +355,9 @@ class Messages {
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
       bool echoSourceLines = true,
       const common::LanguageFeatureControl *hintFlags = nullptr,
-      std::size_t maxErrorsToEmit = 0) const;
+      std::size_t maxErrorsToEmit = 0, bool warningsAreErrors = false) const;
   void AttachTo(Message &, std::optional<Severity> = std::nullopt);
-  bool AnyFatalError() const;
+  bool AnyFatalError(bool warningsAreErrors = false) const;
 
 private:
   std::list<Message> messages_;

diff  --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index 2429e07e5b8c4..58901c6000380 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -230,15 +230,14 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
   const common::LanguageFeatureControl &features{
       instance->getInvocation().getFortranOpts().features};
   const size_t maxErrors{instance->getInvocation().getMaxErrors()};
-  if (!instance->getParsing().messages().empty() &&
-      (instance->getInvocation().getWarnAsErr() ||
-       instance->getParsing().messages().AnyFatalError())) {
+  const bool warningsAreErrors{instance->getInvocation().getWarnAsErr()};
+  if (instance->getParsing().messages().AnyFatalError(warningsAreErrors)) {
     const unsigned diagID = instance->getDiagnostics().getCustomDiagID(
         clang::DiagnosticsEngine::Error, message);
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLines=*/true, &features, maxErrors);
+        /*echoSourceLines=*/true, &features, maxErrors, warningsAreErrors);
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -249,7 +248,7 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(
         llvm::errs(), instance->getAllCookedSources(),
-        /*echoSourceLine=*/true, &features, maxErrors);
+        /*echoSourceLine=*/true, &features, maxErrors, warningsAreErrors);
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);

diff  --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 909fba948a45a..2a8101dd0b810 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -453,7 +453,7 @@ void Messages::ResolveProvenances(const AllCookedSources &allCooked) {
 
 void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     bool echoSourceLines, const common::LanguageFeatureControl *hintFlagPtr,
-    std::size_t maxErrorsToEmit) const {
+    std::size_t maxErrorsToEmit, bool warningsAreErrors) const {
   std::vector<const Message *> sorted;
   for (const auto &msg : messages_) {
     sorted.push_back(&msg);
@@ -469,7 +469,7 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
     }
     msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
-    if (msg->IsFatal()) {
+    if (warningsAreErrors || msg->IsFatal()) {
       ++errorsEmitted;
     }
     // If maxErrorsToEmit is 0, emit all errors, otherwise break after
@@ -491,7 +491,18 @@ void Messages::AttachTo(Message &msg, std::optional<Severity> severity) {
   messages_.clear();
 }
 
-bool Messages::AnyFatalError() const {
+bool Messages::AnyFatalError(bool warningsAreErrors) const {
+  // Short-circuit in the most common case.
+  if (messages_.empty()) {
+    return false;
+  }
+  // If warnings are errors and there are warnings or errors, this is fatal.
+  // This preserves the compiler's current behavior of treating any non-fatal
+  // message as a warning. We may want to refine this in the future.
+  if (warningsAreErrors) {
+    return true;
+  }
+  // Otherwise, check the message buffer for fatal errors.
   for (const auto &msg : messages_) {
     if (msg.IsFatal()) {
       return true;

diff  --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -376,8 +376,7 @@ const DeclTypeSpec &SemanticsContext::MakeLogicalType(int kind) {
 }
 
 bool SemanticsContext::AnyFatalError() const {
-  return !messages_.empty() &&
-      (warningsAreErrors_ || messages_.AnyFatalError());
+  return messages_.AnyFatalError(warningsAreErrors_);
 }
 bool SemanticsContext::HasError(const Symbol &symbol) {
   return errorSymbols_.count(symbol) > 0;
@@ -658,7 +657,7 @@ void Semantics::EmitMessages(llvm::raw_ostream &os) {
   context_.messages().ResolveProvenances(context_.allCookedSources());
   context_.messages().Emit(os, context_.allCookedSources(),
       /*echoSourceLine=*/true, &context_.languageFeatures(),
-      /*maxErrorsToEmit=*/context_.maxErrors());
+      context_.maxErrors(), context_.warningsAreErrors());
 }
 
 void SemanticsContext::DumpSymbols(llvm::raw_ostream &os) {

diff  --git a/flang/test/Driver/fatal-errors-warnings.f90 b/flang/test/Driver/fatal-errors-warnings.f90
new file mode 100644
index 0000000000000..2de09c3ed0778
--- /dev/null
+++ b/flang/test/Driver/fatal-errors-warnings.f90
@@ -0,0 +1,31 @@
+! RUN: %flang_fc1 -Wfatal-errors -pedantic %s 2>&1 | FileCheck %s --check-prefix=CHECK1
+! RUN: not %flang_fc1 -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK2
+! RUN: not %flang_fc1 -Wfatal-errors -pedantic -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK3
+
+module m
+    contains
+    subroutine foo(a)
+        real, intent(in), target :: a(:)
+    end subroutine
+end module
+
+program test
+    use m
+    real, target :: a(1)
+    real :: b(1)
+    call foo(a) ! ok
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK3: fatal-errors-warnings.f90:{{.*}} warning:
+    call foo(b)
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK3-NOT: error:
+    !CHECK3-NOT: warning:
+    call foo((a))
+    !CHECK1: fatal-errors-warnings.f90:{{.*}} warning:
+    !CHECK2: fatal-errors-warnings.f90:{{.*}} warning:
+    call foo(a([1]))
+    !! Hard error instead of warning if uncommented.
+    !call foo(a(1))
+end
\ No newline at end of file


        

From flang-commits at lists.llvm.org  Wed Jul 16 12:26:47 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 16 Jul 2025 12:26:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][driver] -Werror promotes warnings
 to error and interopts with -Wfatal-errors (PR #148748)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148748@github.com>
Message-ID: <6877fcf7.170a0220.195d52.0336@mx.google.com>


https://github.com/akuhlens closed https://github.com/llvm/llvm-project/pull/148748

From flang-commits at lists.llvm.org  Wed Jul 16 12:43:58 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Wed, 16 Jul 2025 12:43:58 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Import type descriptor in the
 gpu module when needed (PR #149157)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149157@github.com>
Message-ID: <687800fe.170a0220.216dea.c8d4@mx.google.com>


https://github.com/clementval updated https://github.com/llvm/llvm-project/pull/149157

>From 557fee5038c1620c73601a57a41ccb92d5810b80 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Wed, 16 Jul 2025 10:19:45 -0700
Subject: [PATCH 1/2] [flang][cuda] Import type descriptor in the gpu module
 when needed

---
 .../Optimizer/Transforms/CUFDeviceGlobal.cpp  | 42 +++++++++++++------
 flang/test/Fir/CUDA/cuda-device-global.f90    | 22 ++++++++++
 2 files changed, 52 insertions(+), 12 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
index bfb0daeacb8c3..35badb6eadb1c 100644
--- a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
@@ -53,21 +53,26 @@ static void processAddrOfOp(fir::AddrOfOp addrOfOp,
   }
 }
 
+static void processTypeDescriptor(fir::RecordType recTy,
+                                  mlir::SymbolTable &symbolTable,
+                                  llvm::DenseSet<fir::GlobalOp> &candidates) {
+  if (auto globalOp = symbolTable.lookup<fir::GlobalOp>(
+          fir::NameUniquer::getTypeDescriptorName(recTy.getName()))) {
+    if (!candidates.contains(globalOp)) {
+      globalOp.walk([&](fir::AddrOfOp op) {
+        processAddrOfOp(op, symbolTable, candidates,
+                        /*recurseInGlobal=*/true);
+      });
+      candidates.insert(globalOp);
+    }
+  }
+}
+
 static void processEmboxOp(fir::EmboxOp emboxOp, mlir::SymbolTable &symbolTable,
                            llvm::DenseSet<fir::GlobalOp> &candidates) {
   if (auto recTy = mlir::dyn_cast<fir::RecordType>(
-          fir::unwrapRefType(emboxOp.getMemref().getType()))) {
-    if (auto globalOp = symbolTable.lookup<fir::GlobalOp>(
-            fir::NameUniquer::getTypeDescriptorName(recTy.getName()))) {
-      if (!candidates.contains(globalOp)) {
-        globalOp.walk([&](fir::AddrOfOp op) {
-          processAddrOfOp(op, symbolTable, candidates,
-                          /*recurseInGlobal=*/true);
-        });
-        candidates.insert(globalOp);
-      }
-    }
-  }
+          fir::unwrapRefType(emboxOp.getMemref().getType())))
+    processTypeDescriptor(recTy, symbolTable, candidates);
 }
 
 static void
@@ -85,6 +90,17 @@ prepareImplicitDeviceGlobals(mlir::func::FuncOp funcOp,
   }
 }
 
+static void
+processPotentialTypeDescriptor(mlir::Type candidateType,
+                               mlir::SymbolTable &symbolTable,
+                               llvm::DenseSet<fir::GlobalOp> &candidates) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(candidateType))
+    candidateType = boxTy.getEleTy();
+  candidateType = fir::unwrapSequenceType(fir::unwrapRefType(candidateType));
+  if (auto recTy = mlir::dyn_cast<fir::RecordType>(candidateType))
+    processTypeDescriptor(recTy, symbolTable, candidates);
+}
+
 class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
 public:
   void runOnOperation() override {
@@ -115,6 +131,8 @@ class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
     for (auto globalOp : mod.getOps<fir::GlobalOp>()) {
       if (cuf::isRegisteredDeviceGlobal(globalOp)) {
         candidates.insert(globalOp);
+        processPotentialTypeDescriptor(globalOp.getType(), parentSymTable,
+                                       candidates);
       } else if (globalOp.getConstant() &&
                  mlir::isa<fir::SequenceType>(
                      fir::unwrapRefType(globalOp.resultType()))) {
diff --git a/flang/test/Fir/CUDA/cuda-device-global.f90 b/flang/test/Fir/CUDA/cuda-device-global.f90
index 4c634513745fd..a2875a2dd241d 100644
--- a/flang/test/Fir/CUDA/cuda-device-global.f90
+++ b/flang/test/Fir/CUDA/cuda-device-global.f90
@@ -24,3 +24,25 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.conta
 // CHECK: gpu.module @cuda_device_mod
 // CHECK-DAG: fir.global @_QMm2ECc
 // CHECK-DAG: fir.global @_QMm1ECb
+
+// -----
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module} {
+  fir.global @_QMmEddarrays {data_attr = #cuf.cuda<managed>} : !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>> {
+    %c0 = arith.constant 0 : index
+    %0 = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>
+    %1 = fir.shape %c0 : (index) -> !fir.shape<1>
+    %2 = fir.embox %0(%1) {allocator_idx = 3 : i32} : (!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>>
+    fir.has_value %2 : !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMmTdevicearrays{phi_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,phi0_i:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_r:!fir.box<!fir.heap<!fir.array<?xf64>>>,buf_i:!fir.box<!fir.heap<!fir.array<?xf64>>>}>>>>
+  }
+  fir.global linkonce_odr @_QMmE.dt.devicearrays constant target : !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}> {
+    %0 = fir.undefined !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>
+    fir.has_value %0 : !fir.type<_QM__fortran_type_infoTderivedtype{binding:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTbinding{proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>}>>>>,name:!fir.box<!fir.ptr<!fir.char<1,?>>>,sizeinbytes:i64,uninstantiated:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,kindparameter:!fir.box<!fir.ptr<!fir.array<?xi64>>>,lenparameterkind:!fir.box<!fir.ptr<!fir.array<?xi8>>>,component:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,genre:i8,category:i8,kind:i8,rank:i8,__padding0:!fir.array<4xi8>,offset:i64,characterlen:!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>,derived:!fir.box<!fir.ptr<!fir.type<_QM__fortran_type_infoTderivedtype>>>,lenvalue:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,bounds:!fir.box<!fir.ptr<!fir.array<?x?x!fir.type<_QM__fortran_type_infoTvalue{genre:i8,__padding0:!fir.array<7xi8>,value:i64}>>>>,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>>>,procptr:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTprocptrcomponent{name:!fir.box<!fir.ptr<!fir.char<1,?>>>,offset:i64,initialization:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,special:!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QM__fortran_type_infoTspecialbinding{which:i8,isargdescriptorset:i8,istypebound:i8,isargcontiguousset:i8,__padding0:!fir.array<4xi8>,proc:!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>}>>>>,specialbitset:i32,hasparent:i8,noinitializationneeded:i8,nodestructionneeded:i8,nofinalizationneeded:i8,nodefinedassignment:i8,__padding0:!fir.array<3xi8>}>
+  }
+}
+
+// CHECK: fir.global @_QMmEddarrays
+// CHECK: fir.global linkonce_odr @_QMmE.dt.devicearrays
+// CHECK: gpu.module @cuda_device_mod
+// CHECK: fir.global @_QMmEddarrays
+// CHECK: fir.global linkonce_odr @_QMmE.dt.devicearrays

>From 095a56421a0abd558b63b51215d1c797d71456dc Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval at gmail.com>
Date: Wed, 16 Jul 2025 12:43:46 -0700
Subject: [PATCH 2/2] Fix test

---
 flang/test/Fir/CUDA/cuda-device-global.f90 | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/flang/test/Fir/CUDA/cuda-device-global.f90 b/flang/test/Fir/CUDA/cuda-device-global.f90
index a2875a2dd241d..35c025dad3000 100644
--- a/flang/test/Fir/CUDA/cuda-device-global.f90
+++ b/flang/test/Fir/CUDA/cuda-device-global.f90
@@ -41,8 +41,9 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.conta
   }
 }
 
-// CHECK: fir.global @_QMmEddarrays
-// CHECK: fir.global linkonce_odr @_QMmE.dt.devicearrays
+
+// CHECK-NAG: fir.global @_QMmEddarrays
+// CHECK-NAG: fir.global linkonce_odr @_QMmE.dt.devicearrays
 // CHECK: gpu.module @cuda_device_mod
-// CHECK: fir.global @_QMmEddarrays
-// CHECK: fir.global linkonce_odr @_QMmE.dt.devicearrays
+// CHECK-NAG: fir.global @_QMmEddarrays
+// CHECK-NAG: fir.global linkonce_odr @_QMmE.dt.devicearrays


From flang-commits at lists.llvm.org  Wed Jul 16 12:49:20 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 12:49:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <68780240.170a0220.3d0691.0b31@mx.google.com>


https://github.com/klausler approved this pull request.


https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Wed Jul 16 12:49:21 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 12:49:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <68780241.050a0220.152068.930f@mx.google.com>


================
@@ -489,15 +489,32 @@ class ParseTreeAnalyzer {
 
   // C1401
   void Post(const parser::MainProgram &mainProgram) {
-    if (const parser::CharBlock *
-        endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
-            mainProgram.t))}) {
-      if (const auto &program{
-              std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
-                  mainProgram.t)}) {
-        if (*endName != program->statement.v.source) {
+    // Uppercase the name of the main program, so that its symbol name
+    // would be unique from similarly named non-main-program symbols.
+    auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
+      char *ch = const_cast<char *>(cb.begin());
----------------
klausler wrote:

Please always use braced initialization in the bits of flang-new that are modern C++.

https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Wed Jul 16 12:49:21 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 12:49:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <68780241.050a0220.38c4b.a252@mx.google.com>


================
@@ -489,15 +489,32 @@ class ParseTreeAnalyzer {
 
   // C1401
   void Post(const parser::MainProgram &mainProgram) {
-    if (const parser::CharBlock *
-        endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
-            mainProgram.t))}) {
-      if (const auto &program{
-              std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
-                  mainProgram.t)}) {
-        if (*endName != program->statement.v.source) {
+    // Uppercase the name of the main program, so that its symbol name
+    // would be unique from similarly named non-main-program symbols.
+    auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
+      char *ch = const_cast<char *>(cb.begin());
+      char *endCh = ch + cb.size();
+      while (ch != endCh) {
+        *ch++ = parser::ToUpperCaseLetter(*ch);
+      }
+    };
+    const parser::CharBlock *progName{nullptr};
+    if (const auto &program{
+            std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
+                mainProgram.t)}) {
+      progName = &program->statement.v.source;
+    }
+    if (progName) {
+      upperCaseCharBlock(*progName);
+    }
+    const parser::CharBlock *endName{GetStmtName(
+        std::get<parser::Statement<parser::EndProgramStmt>>(mainProgram.t))};
+    if (endName) {
----------------
klausler wrote:

these two `if` statements could each be combined with the preceding local variable declarations.

https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Wed Jul 16 12:49:22 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 16 Jul 2025 12:49:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <68780242.170a0220.1c5c7a.052a@mx.google.com>


================
@@ -170,6 +170,15 @@ end
   In the case of `DEFERRED` bindings in an `ABSTRACT` derived type,
   however, overrides are necessary, so they are permitted for inaccessible
   bindings with an optional warning.
+* Main program name is allowed to be the same as the other symbols used
+  in the main program, for example:
----------------
klausler wrote:

Please add a note to the effect that the program name is capitalized, which is something that people using debuggers may need to know.

https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Wed Jul 16 12:11:28 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 12:11:28 -0700 (PDT)
Subject: [flang-commits] [flang] bcee18a - [flang] Handle SEQUENCE derived
 types for array repacking. (#148777)
Message-ID: <6877f960.630a0220.3dc27d.62b5@mx.google.com>


Author: Slava Zakharin
Date: 2025-07-16T12:11:15-07:00
New Revision: bcee18a2e268dd106e4b3c2a1d083a4da21f4f23

URL: https://github.com/llvm/llvm-project/commit/bcee18a2e268dd106e4b3c2a1d083a4da21f4f23
DIFF: https://github.com/llvm/llvm-project/commit/bcee18a2e268dd106e4b3c2a1d083a4da21f4f23.diff

LOG: [flang] Handle SEQUENCE derived types for array repacking. (#148777)

It is possible that a non-polymorphic dummy argument
has a dynamic type that does not match its static type
in a valid Fortran program, e.g. when the actual and
the dummy arguments have different compatible derived
SEQUENCE types:
module mod
  type t
    sequence
    integer x
  end type
contains
  subroutine test(x)
    type t
      sequence
      integer x
    end type
    type(t) :: x(:)
  end subroutine
end module

'test' may be called with an actual argument of type 'mod::t',
which is the dynamic type of 'x' on entry to 'test'.
If we create the repacking temporary based on the static type of 'x'
('test::t'), then the runtime will report the types mismatch
as an error. Thus, we have to create the temporary using
the dynamic type of 'x'. The fact that the dummy's type
has SEQUENCE or BIND attribute is not easily computable
at this stage, so we use the dynamic type for all derived
type cases. As long as this is done only when the repacking
actually happens, the overhead should not be noticeable.

Added: 
    

Modified: 
    flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
    flang/test/Transforms/lower-repack-arrays.fir

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
index e34771c67b0c3..d2cf85bedd54c 100644
--- a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
+++ b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
@@ -63,13 +63,14 @@ class PackArrayConversion : public mlir::OpRewritePattern<fir::PackArrayOp> {
   static constexpr llvm::StringRef bufferName = ".repacked";
 
   // Return value of fir::BaseBoxType that represents a temporary
-  // array created for the original box with given extents and
-  // type parameters. The new box has the default lower bounds.
-  // If useStack is true, then the temporary will be allocated
+  // array created for the original box with given lbounds/extents and
+  // type parameters. The new box has the same shape as the original
+  // array. If useStack is true, then the temporary will be allocated
   // in stack memory (when possible).
   static mlir::Value allocateTempBuffer(fir::FirOpBuilder &builder,
                                         mlir::Location loc, bool useStack,
                                         mlir::Value origBox,
+                                        llvm::ArrayRef<mlir::Value> lbounds,
                                         llvm::ArrayRef<mlir::Value> extents,
                                         llvm::ArrayRef<mlir::Value> typeParams);
 
@@ -99,7 +100,9 @@ class UnpackArrayConversion
 // the presence of the stack attribute does not automatically
 // mean that the allocation is actually done in stack memory.
 // For example, we always do the heap allocation for polymorphic
-// types using Fortran runtime.
+// types using Fortran runtime. Currently, we allocate all
+// repack temporaries of derived types as polymorphic,
+// so that we can preserve the dynamic type of the original.
 // Adding the polymorpic mold to fir.alloca and then using
 // Fortran runtime to compute the allocation size could probably
 // resolve this limitation.
@@ -170,7 +173,8 @@ PackArrayConversion::matchAndRewrite(fir::PackArrayOp op,
 
 mlir::Value PackArrayConversion::allocateTempBuffer(
     fir::FirOpBuilder &builder, mlir::Location loc, bool useStack,
-    mlir::Value origBox, llvm::ArrayRef<mlir::Value> extents,
+    mlir::Value origBox, llvm::ArrayRef<mlir::Value> lbounds,
+    llvm::ArrayRef<mlir::Value> extents,
     llvm::ArrayRef<mlir::Value> typeParams) {
   auto tempType = mlir::cast<fir::SequenceType>(
       fir::extractSequenceType(origBox.getType()));
@@ -191,16 +195,35 @@ mlir::Value PackArrayConversion::allocateTempBuffer(
     assert(!isHeapAllocation && "temp must have been allocated on the stack");
 
   mlir::Type ptrType = base.getType();
-  if (llvm::isa<fir::BaseBoxType>(ptrType))
-    return base;
+  if (auto tempBoxType = mlir::dyn_cast<fir::BaseBoxType>(ptrType)) {
+    // We need to reset the CFI_attribute_allocatable before
+    // returning the temporary box to avoid any mishandling
+    // of the temporary box in Fortran runtime.
+    base = builder.create<fir::BoxAddrOp>(loc, fir::boxMemRefType(tempBoxType),
+                                          base);
+    ptrType = base.getType();
+  }
 
-  mlir::Type tempBoxType = fir::BoxType::get(mlir::isa<fir::HeapType>(ptrType)
-                                                 ? ptrType
-                                                 : fir::unwrapRefType(ptrType));
+  // Create the temporary using dynamic type of the original,
+  // if it is polymorphic, or it has a derived type with SEQUENCE
+  // or BIND attribute (such dummy arguments may have their dynamic
+  // type not exactly matching their static type).
+  // Note that for the latter case, the allocation can still be done
+  // without the mold, because the dynamic and static types
+  // must be storage compatible.
+  bool useDynamicType = fir::isBoxedRecordType(origBox.getType()) ||
+                        fir::isPolymorphicType(origBox.getType());
+  mlir::Type tempBoxType =
+      fir::wrapInClassOrBoxType(fir::unwrapRefType(ptrType),
+                                /*isPolymorphic=*/useDynamicType);
+  // Use the shape with proper lower bounds for the final box.
+  shape = builder.genShape(loc, lbounds, extents);
   mlir::Value newBox =
       builder.createBox(loc, tempBoxType, base, shape, /*slice=*/nullptr,
-                        typeParams, /*tdesc=*/nullptr);
-  return newBox;
+                        typeParams, useDynamicType ? origBox : nullptr);
+  // The new box might be !fir.class, while the original might be
+  // !fir.box - we have to add a conversion.
+  return builder.createConvert(loc, origBox.getType(), newBox);
 }
 
 mlir::FailureOr<mlir::Value>
@@ -280,16 +303,11 @@ PackArrayConversion::genRepackedBox(fir::FirOpBuilder &builder,
                             << op.getOperation() << '\n';
   }
 
-  mlir::Value tempBox =
-      allocateTempBuffer(builder, loc, op.getStack(), box, extents, typeParams);
+  mlir::Value tempBox = allocateTempBuffer(builder, loc, op.getStack(), box,
+                                           lbounds, extents, typeParams);
   if (!op.getNoCopy())
     fir::runtime::genShallowCopy(builder, loc, tempBox, box,
                                  /*resultIsAllocated=*/true);
-
-  // Set lower bounds after the original box.
-  mlir::Value shift = builder.genShift(loc, lbounds);
-  tempBox = builder.create<fir::ReboxOp>(loc, boxType, tempBox, shift,
-                                         /*slice=*/nullptr);
   builder.create<fir::ResultOp>(loc, tempBox);
 
   return ifOp.getResult(0);

diff  --git a/flang/test/Transforms/lower-repack-arrays.fir b/flang/test/Transforms/lower-repack-arrays.fir
index 458869cce45fd..9232a74f224d3 100644
--- a/flang/test/Transforms/lower-repack-arrays.fir
+++ b/flang/test/Transforms/lower-repack-arrays.fir
@@ -28,15 +28,14 @@ func.func @_QPtest1(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x"})
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_18:.*]] = fir.allocmem !fir.array<?x?xf32>, %[[VAL_15]]#1, %[[VAL_16]]#1 {bindc_name = ".repacked", uniq_name = ""}
 // CHECK:               %[[VAL_19:.*]] = fir.declare %[[VAL_18]](%[[VAL_17]]) {uniq_name = ".repacked"} : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.heap<!fir.array<?x?xf32>>
-// CHECK:               %[[VAL_20:.*]] = fir.embox %[[VAL_19]](%[[VAL_17]]) : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+// CHECK:               %[[VAL_25:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_20:.*]] = fir.embox %[[VAL_19]](%[[VAL_25]]) : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<?x?xf32>>
 // CHECK:               %[[VAL_21:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_25:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_26:.*]] = fir.rebox %[[VAL_20]](%[[VAL_25]]) : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?xf32>>
-// CHECK:               fir.result %[[VAL_26]] : !fir.box<!fir.array<?x?xf32>>
+// CHECK:               fir.result %[[VAL_20]] : !fir.box<!fir.array<?x?xf32>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?xf32>>
 // CHECK:             }
@@ -93,15 +92,14 @@ func.func @_QPtest1_whole(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name =
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_18:.*]] = fir.alloca !fir.array<?x?xf32>, %[[VAL_15]]#1, %[[VAL_16]]#1 {bindc_name = ".repacked"}
 // CHECK:               %[[VAL_19:.*]] = fir.declare %[[VAL_18]](%[[VAL_17]]) {uniq_name = ".repacked"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.ref<!fir.array<?x?xf32>>
-// CHECK:               %[[VAL_20:.*]] = fir.embox %[[VAL_19]](%[[VAL_17]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+// CHECK:               %[[VAL_25:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_20:.*]] = fir.embox %[[VAL_19]](%[[VAL_25]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<?x?xf32>>
 // CHECK:               %[[VAL_21:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_25:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_26:.*]] = fir.rebox %[[VAL_20]](%[[VAL_25]]) : (!fir.box<!fir.array<?x?xf32>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?xf32>>
-// CHECK:               fir.result %[[VAL_26]] : !fir.box<!fir.array<?x?xf32>>
+// CHECK:               fir.result %[[VAL_20]] : !fir.box<!fir.array<?x?xf32>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?xf32>>
 // CHECK:             }
@@ -156,15 +154,14 @@ func.func @_QPtest1_in(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x
 // CHECK:               %[[VAL_16:.*]] = fir.shape %[[VAL_14]]#1, %[[VAL_15]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_17:.*]] = fir.allocmem !fir.array<?x?xf32>, %[[VAL_14]]#1, %[[VAL_15]]#1 {bindc_name = ".repacked", uniq_name = ""}
 // CHECK:               %[[VAL_18:.*]] = fir.declare %[[VAL_17]](%[[VAL_16]]) {uniq_name = ".repacked"} : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.heap<!fir.array<?x?xf32>>
-// CHECK:               %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_16]]) : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+// CHECK:               %[[VAL_24:.*]] = fir.shape_shift %[[VAL_14]]#0, %[[VAL_14]]#1, %[[VAL_15]]#0, %[[VAL_15]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_24]]) : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<?x?xf32>>
 // CHECK:               %[[VAL_20:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_24:.*]] = fir.shift %[[VAL_14]]#0, %[[VAL_15]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_25:.*]] = fir.rebox %[[VAL_19]](%[[VAL_24]]) : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?xf32>>
-// CHECK:               fir.result %[[VAL_25]] : !fir.box<!fir.array<?x?xf32>>
+// CHECK:               fir.result %[[VAL_19]] : !fir.box<!fir.array<?x?xf32>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?xf32>>
 // CHECK:             }
@@ -215,10 +212,9 @@ func.func @_QPtest1_out(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "
 // CHECK:               %[[VAL_16:.*]] = fir.shape %[[VAL_14]]#1, %[[VAL_15]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_17:.*]] = fir.allocmem !fir.array<?x?xf32>, %[[VAL_14]]#1, %[[VAL_15]]#1 {bindc_name = ".repacked", uniq_name = ""}
 // CHECK:               %[[VAL_18:.*]] = fir.declare %[[VAL_17]](%[[VAL_16]]) {uniq_name = ".repacked"} : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.heap<!fir.array<?x?xf32>>
-// CHECK:               %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_16]]) : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xf32>>>
-// CHECK:               %[[VAL_20:.*]] = fir.shift %[[VAL_14]]#0, %[[VAL_15]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_21:.*]] = fir.rebox %[[VAL_19]](%[[VAL_20]]) : (!fir.box<!fir.heap<!fir.array<?x?xf32>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?xf32>>
-// CHECK:               fir.result %[[VAL_21]] : !fir.box<!fir.array<?x?xf32>>
+// CHECK:               %[[VAL_20:.*]] = fir.shape_shift %[[VAL_14]]#0, %[[VAL_14]]#1, %[[VAL_15]]#0, %[[VAL_15]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_20]]) : (!fir.heap<!fir.array<?x?xf32>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<?x?xf32>>
+// CHECK:               fir.result %[[VAL_19]] : !fir.box<!fir.array<?x?xf32>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?xf32>>
 // CHECK:             }
@@ -286,15 +282,14 @@ func.func @_QPtest2(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.box
 // CHECK:               %[[VAL_23:.*]] = fir.shape %[[VAL_21]]#1, %[[VAL_22]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_24:.*]] = fir.allocmem !fir.array<?x?x!fir.char<1,?>>(%[[VAL_12]] : i32), %[[VAL_21]]#1, %[[VAL_22]]#1 {bindc_name = ".repacked", uniq_name = ""}
 // CHECK:               %[[VAL_25:.*]] = fir.declare %[[VAL_24]](%[[VAL_23]]) typeparams %[[VAL_12]] {uniq_name = ".repacked"} : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, i32) -> !fir.heap<!fir.array<?x?x!fir.char<1,?>>>
-// CHECK:               %[[VAL_26:.*]] = fir.embox %[[VAL_25]](%[[VAL_23]]) typeparams %[[VAL_12]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, i32) -> !fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,?>>>>
+// CHECK:               %[[VAL_31:.*]] = fir.shape_shift %[[VAL_21]]#0, %[[VAL_21]]#1, %[[VAL_22]]#0, %[[VAL_22]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_26:.*]] = fir.embox %[[VAL_25]](%[[VAL_31]]) typeparams %[[VAL_12]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>, !fir.shapeshift<2>, i32) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:               %[[VAL_27:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_26]] : (!fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,?>>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_26]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_1]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_27]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_28]], %[[VAL_29]], %[[VAL_30]], %[[VAL_3]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_31:.*]] = fir.shift %[[VAL_21]]#0, %[[VAL_22]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_32:.*]] = fir.rebox %[[VAL_26]](%[[VAL_31]]) : (!fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,?>>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
-// CHECK:               fir.result %[[VAL_32]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:               fir.result %[[VAL_26]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_1]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:             }
@@ -362,15 +357,14 @@ func.func @_QPtest2_stack(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !f
 // CHECK:               %[[VAL_23:.*]] = fir.shape %[[VAL_21]]#1, %[[VAL_22]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_24:.*]] = fir.alloca !fir.array<?x?x!fir.char<1,?>>(%[[VAL_12]] : i32), %[[VAL_21]]#1, %[[VAL_22]]#1 {bindc_name = ".repacked"}
 // CHECK:               %[[VAL_25:.*]] = fir.declare %[[VAL_24]](%[[VAL_23]]) typeparams %[[VAL_12]] {uniq_name = ".repacked"} : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, i32) -> !fir.ref<!fir.array<?x?x!fir.char<1,?>>>
-// CHECK:               %[[VAL_26:.*]] = fir.embox %[[VAL_25]](%[[VAL_23]]) typeparams %[[VAL_12]] : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, i32) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:               %[[VAL_31:.*]] = fir.shape_shift %[[VAL_21]]#0, %[[VAL_21]]#1, %[[VAL_22]]#0, %[[VAL_22]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_26:.*]] = fir.embox %[[VAL_25]](%[[VAL_31]]) typeparams %[[VAL_12]] : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shapeshift<2>, i32) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:               %[[VAL_27:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_26]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_1]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_27]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_28]], %[[VAL_29]], %[[VAL_30]], %[[VAL_3]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_31:.*]] = fir.shift %[[VAL_21]]#0, %[[VAL_22]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_32:.*]] = fir.rebox %[[VAL_26]](%[[VAL_31]]) : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
-// CHECK:               fir.result %[[VAL_32]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:               fir.result %[[VAL_26]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_1]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:             }
@@ -427,15 +421,14 @@ func.func @_QPtest3(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,?>>> {fir.bindc_n
 // CHECK:               %[[VAL_18:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_19:.*]] = fir.allocmem !fir.array<?x?x!fir.char<1,?>>(%[[VAL_17]] : index), %[[VAL_15]]#1, %[[VAL_16]]#1 {bindc_name = ".repacked", uniq_name = ""}
 // CHECK:               %[[VAL_20:.*]] = fir.declare %[[VAL_19]](%[[VAL_18]]) typeparams %[[VAL_17]] {uniq_name = ".repacked"} : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, index) -> !fir.heap<!fir.array<?x?x!fir.char<1,?>>>
-// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_18]]) typeparams %[[VAL_17]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, index) -> !fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,?>>>>
+// CHECK:               %[[VAL_26:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_26]]) typeparams %[[VAL_17]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>, !fir.shapeshift<2>, index) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:               %[[VAL_22:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,?>>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_22]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_26:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_27:.*]] = fir.rebox %[[VAL_21]](%[[VAL_26]]) : (!fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,?>>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
-// CHECK:               fir.result %[[VAL_27]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:               fir.result %[[VAL_21]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:             }
@@ -493,15 +486,14 @@ func.func @_QPtest3_stack(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,?>>> {fir.b
 // CHECK:               %[[VAL_18:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_19:.*]] = fir.alloca !fir.array<?x?x!fir.char<1,?>>(%[[VAL_17]] : index), %[[VAL_15]]#1, %[[VAL_16]]#1 {bindc_name = ".repacked"}
 // CHECK:               %[[VAL_20:.*]] = fir.declare %[[VAL_19]](%[[VAL_18]]) typeparams %[[VAL_17]] {uniq_name = ".repacked"} : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, index) -> !fir.ref<!fir.array<?x?x!fir.char<1,?>>>
-// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_18]]) typeparams %[[VAL_17]] : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, index) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:               %[[VAL_26:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_26]]) typeparams %[[VAL_17]] : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shapeshift<2>, index) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:               %[[VAL_22:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_22]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_26:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_27:.*]] = fir.rebox %[[VAL_21]](%[[VAL_26]]) : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,?>>>
-// CHECK:               fir.result %[[VAL_27]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:               fir.result %[[VAL_21]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?x!fir.char<1,?>>>
 // CHECK:             }
@@ -559,15 +551,14 @@ func.func @_QPtest4(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,10>>> {fir.bindc_
 // CHECK:               %[[VAL_18:.*]] = fir.shape %[[VAL_16]]#1, %[[VAL_17]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_19:.*]] = fir.allocmem !fir.array<?x?x!fir.char<1,10>>, %[[VAL_16]]#1, %[[VAL_17]]#1 {bindc_name = ".repacked", uniq_name = ""}
 // CHECK:               %[[VAL_20:.*]] = fir.declare %[[VAL_19]](%[[VAL_18]]) typeparams %[[VAL_6]] {uniq_name = ".repacked"} : (!fir.heap<!fir.array<?x?x!fir.char<1,10>>>, !fir.shape<2>, index) -> !fir.heap<!fir.array<?x?x!fir.char<1,10>>>
-// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_18]]) : (!fir.heap<!fir.array<?x?x!fir.char<1,10>>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,10>>>>
+// CHECK:               %[[VAL_26:.*]] = fir.shape_shift %[[VAL_16]]#0, %[[VAL_16]]#1, %[[VAL_17]]#0, %[[VAL_17]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_26]]) : (!fir.heap<!fir.array<?x?x!fir.char<1,10>>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,10>>>
 // CHECK:               %[[VAL_22:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,10>>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_22]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_26:.*]] = fir.shift %[[VAL_16]]#0, %[[VAL_17]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_27:.*]] = fir.rebox %[[VAL_21]](%[[VAL_26]]) : (!fir.box<!fir.heap<!fir.array<?x?x!fir.char<1,10>>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,10>>>
-// CHECK:               fir.result %[[VAL_27]] : !fir.box<!fir.array<?x?x!fir.char<1,10>>>
+// CHECK:               fir.result %[[VAL_21]] : !fir.box<!fir.array<?x?x!fir.char<1,10>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?x!fir.char<1,10>>>
 // CHECK:             }
@@ -626,15 +617,14 @@ func.func @_QPtest4_stack(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,10>>> {fir.
 // CHECK:               %[[VAL_18:.*]] = fir.shape %[[VAL_16]]#1, %[[VAL_17]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_19:.*]] = fir.alloca !fir.array<?x?x!fir.char<1,10>>, %[[VAL_16]]#1, %[[VAL_17]]#1 {bindc_name = ".repacked"}
 // CHECK:               %[[VAL_20:.*]] = fir.declare %[[VAL_19]](%[[VAL_18]]) typeparams %[[VAL_6]] {uniq_name = ".repacked"} : (!fir.ref<!fir.array<?x?x!fir.char<1,10>>>, !fir.shape<2>, index) -> !fir.ref<!fir.array<?x?x!fir.char<1,10>>>
-// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_18]]) : (!fir.ref<!fir.array<?x?x!fir.char<1,10>>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,10>>>
+// CHECK:               %[[VAL_26:.*]] = fir.shape_shift %[[VAL_16]]#0, %[[VAL_16]]#1, %[[VAL_17]]#0, %[[VAL_17]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_26]]) : (!fir.ref<!fir.array<?x?x!fir.char<1,10>>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,10>>>
 // CHECK:               %[[VAL_22:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_22]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_26:.*]] = fir.shift %[[VAL_16]]#0, %[[VAL_17]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_27:.*]] = fir.rebox %[[VAL_21]](%[[VAL_26]]) : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?x!fir.char<1,10>>>
-// CHECK:               fir.result %[[VAL_27]] : !fir.box<!fir.array<?x?x!fir.char<1,10>>>
+// CHECK:               fir.result %[[VAL_21]] : !fir.box<!fir.array<?x?x!fir.char<1,10>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?x!fir.char<1,10>>>
 // CHECK:             }
@@ -690,15 +680,15 @@ func.func @_QPtest5(%arg0: !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bind
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_18:.*]] = fir.allocmem !fir.array<?x?x!fir.type<_QMmTt>>, %[[VAL_15]]#1, %[[VAL_16]]#1 {bindc_name = ".repacked", uniq_name = ""}
 // CHECK:               %[[VAL_19:.*]] = fir.declare %[[VAL_18]](%[[VAL_17]]) {uniq_name = ".repacked"} : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shape<2>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:               %[[VAL_20:.*]] = fir.embox %[[VAL_19]](%[[VAL_17]]) : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:               %[[VAL_25:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_20:.*]] = fir.embox %[[VAL_19]](%[[VAL_25]]) source_box %[[VAL_0]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shapeshift<2>, !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               %[[BOX:.*]] = fir.convert %[[VAL_20]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:               %[[VAL_21:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.box<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_22:.*]] = fir.convert %[[BOX]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_25:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_26:.*]] = fir.rebox %[[VAL_20]](%[[VAL_25]]) : (!fir.box<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:               fir.result %[[VAL_26]] : !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               fir.result %[[BOX]] : !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             }
@@ -755,15 +745,15 @@ func.func @_QPtest5_stack(%arg0: !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>> {fi
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:               %[[VAL_18:.*]] = fir.alloca !fir.array<?x?x!fir.type<_QMmTt>>, %[[VAL_15]]#1, %[[VAL_16]]#1 {bindc_name = ".repacked"}
 // CHECK:               %[[VAL_19:.*]] = fir.declare %[[VAL_18]](%[[VAL_17]]) {uniq_name = ".repacked"} : (!fir.ref<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shape<2>) -> !fir.ref<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:               %[[VAL_20:.*]] = fir.embox %[[VAL_19]](%[[VAL_17]]) : (!fir.ref<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               %[[VAL_25:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_20:.*]] = fir.embox %[[VAL_19]](%[[VAL_25]]) source_box %[[VAL_0]] : (!fir.ref<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shapeshift<2>, !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               %[[BOX:.*]] = fir.convert %[[VAL_20]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:               %[[VAL_21:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_22:.*]] = fir.convert %[[BOX]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_25:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_26:.*]] = fir.rebox %[[VAL_20]](%[[VAL_25]]) : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:               fir.result %[[VAL_26]] : !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               fir.result %[[BOX]] : !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[VAL_0]] : !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             }
@@ -830,13 +820,14 @@ func.func @_QPtest6(%arg0: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bi
 // CHECK:               %[[VAL_26:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_23]], %[[VAL_24]], %[[VAL_4]], %[[VAL_22]], %[[VAL_25]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 // CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
 // CHECK:               %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:               %[[ADDR:.*]] = fir.box_addr %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               %[[VAL_33:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_34:.*]] = fir.embox %[[ADDR]](%[[VAL_33]]) source_box %[[ARG0]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shapeshift<2>, !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
-// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_34]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_31:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_29]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_33:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_34:.*]] = fir.rebox %[[VAL_28]](%[[VAL_33]]) : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:               fir.result %[[VAL_34]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[ARG0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
@@ -906,13 +897,14 @@ func.func @_QPtest6_stack(%arg0: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {
 // CHECK:               %[[VAL_26:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_23]], %[[VAL_24]], %[[VAL_4]], %[[VAL_22]], %[[VAL_25]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 // CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
 // CHECK:               %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:               %[[ADDR:.*]] = fir.box_addr %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               %[[VAL_33:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_34:.*]] = fir.embox %[[ADDR]](%[[VAL_33]]) source_box %[[ARG0]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shapeshift<2>, !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
-// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_34]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_31:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_29]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_33:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_34:.*]] = fir.rebox %[[VAL_28]](%[[VAL_33]]) : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:               fir.result %[[VAL_34]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[ARG0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
@@ -981,13 +973,14 @@ func.func @_QPtest7(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_name = "x
 // CHECK:               %[[VAL_26:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_23]], %[[VAL_24]], %[[VAL_4]], %[[VAL_22]], %[[VAL_25]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 // CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
 // CHECK:               %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.class<!fir.heap<!fir.array<?x?xnone>>>
+// CHECK:               %[[ADDR:.*]] = fir.box_addr %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.heap<!fir.array<?x?xnone>>
+// CHECK:               %[[VAL_33:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_34:.*]] = fir.embox %[[ADDR]](%[[VAL_33]]) source_box %[[ARG0]] : (!fir.heap<!fir.array<?x?xnone>>, !fir.shapeshift<2>, !fir.class<!fir.array<?x?xnone>>) -> !fir.class<!fir.array<?x?xnone>>
 // CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
-// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_34]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
 // CHECK:               %[[VAL_31:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
 // CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_29]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_33:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_34:.*]] = fir.rebox %[[VAL_28]](%[[VAL_33]]) : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?xnone>>
 // CHECK:               fir.result %[[VAL_34]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[ARG0]] : !fir.class<!fir.array<?x?xnone>>
@@ -1057,13 +1050,14 @@ func.func @_QPtest7_stack(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_nam
 // CHECK:               %[[VAL_26:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_23]], %[[VAL_24]], %[[VAL_4]], %[[VAL_22]], %[[VAL_25]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 // CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
 // CHECK:               %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.class<!fir.heap<!fir.array<?x?xnone>>>
+// CHECK:               %[[ADDR:.*]] = fir.box_addr %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.heap<!fir.array<?x?xnone>>
+// CHECK:               %[[VAL_33:.*]] = fir.shape_shift %[[VAL_15]]#0, %[[VAL_15]]#1, %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+// CHECK:               %[[VAL_34:.*]] = fir.embox %[[ADDR]](%[[VAL_33]]) source_box %[[ARG0]] : (!fir.heap<!fir.array<?x?xnone>>, !fir.shapeshift<2>, !fir.class<!fir.array<?x?xnone>>) -> !fir.class<!fir.array<?x?xnone>>
 // CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
-// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_34]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
 // CHECK:               %[[VAL_31:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
 // CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_29]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 // CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_33:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_34:.*]] = fir.rebox %[[VAL_28]](%[[VAL_33]]) : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?xnone>>
 // CHECK:               fir.result %[[VAL_34]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:             } else {
 // CHECK:               fir.result %[[ARG0]] : !fir.class<!fir.array<?x?xnone>>


        

From flang-commits at lists.llvm.org  Wed Jul 16 12:12:00 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 16 Jul 2025 12:12:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
Message-ID: <llvm.org/llvm/llvm-project/pull/149169@github.com>


https://github.com/eugeneepshteyn created https://github.com/llvm/llvm-project/pull/149169

The following code is now accepted:
```
module m
end
program m
use m
end
```
The PROGRAM name doesn't really have an effect on the compilation result, so it shouldn't result in symbol name conflicts.

This change makes the main program symbol name all uppercase in the cooked character stream. This makes it distinct from all other symbol names that are all lowercase in cooked character stream.

>From f7cc4e803ca099d98d9711b42fc342751de67367 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 15:33:12 -0400
Subject: [PATCH 1/8] [flang] Attempt to address main program symbol conflicts

... by making main program symbol names upper case.
---
 flang/lib/Semantics/resolve-labels.cpp | 32 ++++++++++++++++++++------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index b0cbc4b56e889..0278f70b90226 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -489,15 +489,33 @@ class ParseTreeAnalyzer {
 
   // C1401
   void Post(const parser::MainProgram &mainProgram) {
-    if (const parser::CharBlock *
+    // Uppercase the name of the main program, so that its symbol name
+    // would be unique from similarly named non-main-program symbols.
+    auto upperCaseCharBlock = [](const parser::CharBlock& cb) {
+      char* ch = const_cast<char*>(cb.begin());
+      char* endCh = ch + cb.size();
+      while (ch != endCh) {
+        *ch++ = parser::ToUpperCaseLetter(*ch);
+      }
+    };
+    const parser::CharBlock *progName{nullptr};
+    if (const auto &program{
+            std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
+                mainProgram.t)}) {
+      progName = &program->statement.v.source;
+    }
+    if (progName) {
+      upperCaseCharBlock(*progName);
+    }
+    const parser::CharBlock *
         endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
-            mainProgram.t))}) {
-      if (const auto &program{
-              std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
-                  mainProgram.t)}) {
-        if (*endName != program->statement.v.source) {
+            mainProgram.t))};
+    if (endName) {
+      upperCaseCharBlock(*endName);
+      if (progName) {
+        if (*endName != *progName) {
           context_.Say(*endName, "END PROGRAM name mismatch"_err_en_US)
-              .Attach(program->statement.v.source, "should be"_en_US);
+              .Attach(*progName, "should be"_en_US);
         }
       } else {
         context_.Say(*endName,

>From 336c138bda8c1738c5845f274533b84a075c41aa Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:16:13 -0400
Subject: [PATCH 2/8] clang-format

---
 flang/lib/Semantics/resolve-labels.cpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index 0278f70b90226..d3fda03383783 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -491,9 +491,9 @@ class ParseTreeAnalyzer {
   void Post(const parser::MainProgram &mainProgram) {
     // Uppercase the name of the main program, so that its symbol name
     // would be unique from similarly named non-main-program symbols.
-    auto upperCaseCharBlock = [](const parser::CharBlock& cb) {
-      char* ch = const_cast<char*>(cb.begin());
-      char* endCh = ch + cb.size();
+    auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
+      char *ch = const_cast<char *>(cb.begin());
+      char *endCh = ch + cb.size();
       while (ch != endCh) {
         *ch++ = parser::ToUpperCaseLetter(*ch);
       }
@@ -507,9 +507,8 @@ class ParseTreeAnalyzer {
     if (progName) {
       upperCaseCharBlock(*progName);
     }
-    const parser::CharBlock *
-        endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
-            mainProgram.t))};
+    const parser::CharBlock *endName{GetStmtName(
+        std::get<parser::Statement<parser::EndProgramStmt>>(mainProgram.t))};
     if (endName) {
       upperCaseCharBlock(*endName);
       if (progName) {

>From f6507f56a22227d1b3cb5327d2a0b561a494327e Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:16:52 -0400
Subject: [PATCH 3/8] Initial test changes

---
 flang/test/Driver/cuda-option.f90             |  2 +-
 flang/test/Driver/unparse-use-analyzed.f95    |  4 +-
 flang/test/Driver/unparse-with-modules.f90    |  2 +-
 .../test/Integration/debug-common-block-1.f90 |  2 +-
 flang/test/Integration/debug-local-var-2.f90  |  2 +-
 flang/test/Lower/CUDA/cuda-derived.cuf        |  2 +-
 flang/test/Lower/CUDA/cuda-return01.cuf       |  2 +-
 flang/test/Lower/CUDA/cuda-return02.cuf       |  2 +-
 .../Lower/HLFIR/intrinsic-subroutines.f90     |  2 +-
 ...ointer-component-structure-constructor.f90 |  2 +-
 flang/test/Lower/OpenACC/acc-atomic-read.f90  |  2 +-
 flang/test/Lower/OpenACC/acc-atomic-write.f90 |  2 +-
 flang/test/Lower/OpenACC/acc-routine04.f90    |  2 +-
 flang/test/Lower/OpenMP/atomic-read.f90       |  2 +-
 flang/test/Lower/OpenMP/atomic-write.f90      |  2 +-
 .../Lower/OpenMP/common-atomic-lowering.f90   |  2 +-
 flang/test/Lower/OpenMP/cray-pointers02.f90   |  2 +-
 .../Lower/OpenMP/default-clause-byref.f90     |  2 +-
 flang/test/Lower/OpenMP/default-clause.f90    |  2 +-
 .../parallel-reduction-allocatable-array.f90  |  2 +-
 .../OpenMP/parallel-reduction-array-lb.f90    |  2 +-
 .../OpenMP/parallel-reduction-array2.f90      |  2 +-
 .../Lower/OpenMP/parallel-reduction-byref.f90 |  2 +-
 .../parallel-reduction-pointer-array.f90      |  2 +-
 .../OpenMP/parallel-reduction-rename.f90      |  2 +-
 .../test/Lower/OpenMP/parallel-reduction.f90  |  2 +-
 flang/test/Lower/OpenMP/sections.f90          |  2 +-
 .../threadprivate-host-association-2.f90      |  2 +-
 .../threadprivate-host-association-3.f90      |  2 +-
 .../OpenMP/threadprivate-host-association.f90 |  2 +-
 flang/test/Lower/OpenMP/wsloop-chunks.f90     |  2 +-
 flang/test/Lower/OpenMP/wsloop-collapse.f90   |  2 +-
 ...oop-reduction-allocatable-array-minmax.f90 |  2 +-
 .../OpenMP/wsloop-reduction-allocatable.f90   |  2 +-
 .../OpenMP/wsloop-reduction-array-lb.f90      |  2 +-
 .../OpenMP/wsloop-reduction-array-lb2.f90     |  2 +-
 .../Lower/OpenMP/wsloop-reduction-array.f90   |  2 +-
 .../Lower/OpenMP/wsloop-reduction-array2.f90  |  2 +-
 .../Lower/OpenMP/wsloop-reduction-min2.f90    |  2 +-
 .../wsloop-reduction-multiple-clauses.f90     |  2 +-
 .../Lower/OpenMP/wsloop-reduction-pointer.f90 |  2 +-
 flang/test/Lower/array-character.f90          |  2 +-
 flang/test/Lower/array-expression-slice-1.f90 |  2 +-
 flang/test/Lower/basic-program.f90            |  6 +-
 flang/test/Lower/big-integer-parameter.f90    |  2 +-
 .../test/Lower/derived-type-finalization.f90  |  2 +-
 flang/test/Lower/location.f90                 |  2 +-
 flang/test/Lower/nested-where.f90             |  2 +-
 flang/test/Lower/polymorphic.f90              |  2 +-
 flang/test/Lower/pre-fir-tree02.f90           |  2 +-
 flang/test/Lower/pre-fir-tree03.f90           |  2 +-
 flang/test/Lower/pre-fir-tree06.f90           |  4 +-
 .../test/Lower/program-units-fir-mangling.f90 |  2 +-
 flang/test/Lower/return-statement.f90         |  2 +-
 flang/test/Lower/volatile-openmp1.f90         |  2 +-
 flang/test/Lower/volatile-string.f90          |  2 +-
 flang/test/Lower/volatile3.f90                |  2 +-
 flang/test/Parser/acc-unparse.f90             |  2 +-
 .../test/Semantics/OpenACC/acc-symbols01.f90  |  2 +-
 .../OpenMP/critical_within_default.f90        |  2 +-
 .../OpenMP/declare-mapper-symbols.f90         |  2 +-
 .../OpenMP/declare-reduction-mangled.f90      |  2 +-
 .../OpenMP/declare-reduction-operators.f90    |  2 +-
 .../OpenMP/declare-reduction-renamedop.f90    |  2 +-
 .../Semantics/OpenMP/declare-reduction.f90    |  2 +-
 flang/test/Semantics/OpenMP/do-schedule03.f90 | 28 +++----
 .../Semantics/OpenMP/do01-positivecase.f90    |  6 +-
 .../Semantics/OpenMP/do04-positivecase.f90    | 14 ++--
 .../Semantics/OpenMP/do05-positivecase.f90    | 32 ++++----
 .../Semantics/OpenMP/do06-positivecases.f90   | 10 +--
 flang/test/Semantics/OpenMP/do11.f90          | 26 +++----
 flang/test/Semantics/OpenMP/do12.f90          | 74 +++++++++----------
 flang/test/Semantics/OpenMP/do14.f90          | 62 ++++++++--------
 flang/test/Semantics/OpenMP/do17.f90          | 38 +++++-----
 .../Semantics/OpenMP/map-clause-symbols.f90   |  2 +-
 flang/test/Semantics/OpenMP/reduction08.f90   | 48 ++++++------
 flang/test/Semantics/OpenMP/reduction09.f90   | 38 +++++-----
 flang/test/Semantics/OpenMP/reduction11.f90   |  2 +-
 flang/test/Semantics/OpenMP/scan2.f90         |  2 +-
 flang/test/Semantics/OpenMP/symbol01.f90      | 52 ++++++-------
 flang/test/Semantics/OpenMP/symbol05.f90      |  4 +-
 flang/test/Semantics/OpenMP/symbol07.f90      |  2 +-
 flang/test/Semantics/OpenMP/symbol09.f90      |  2 +-
 flang/test/Semantics/getsymbols03-a.f90       |  2 +-
 flang/test/Semantics/modproc01.f90            |  2 +-
 flang/test/Semantics/multi-programs04.f90     |  2 +-
 flang/test/Semantics/procinterface01.f90      | 22 +++---
 flang/test/Semantics/resolve125.f90           |  4 +-
 flang/test/Semantics/symbol03.f90             | 18 ++---
 flang/test/Semantics/symbol06.f90             | 70 +++++++++---------
 flang/test/Semantics/symbol07.f90             | 46 ++++++------
 flang/test/Semantics/symbol08.f90             | 14 ++--
 flang/test/Semantics/symbol15.f90             | 10 +--
 flang/test/Semantics/symbol16.f90             | 18 ++---
 flang/test/Semantics/symbol17.f90             | 50 ++++++-------
 flang/test/Semantics/symbol18.f90             | 20 ++---
 flang/test/Semantics/symbol20.f90             | 12 +--
 flang/test/Semantics/symbol25.f90             | 12 +--
 flang/test/Semantics/symbol26.f90             |  8 +-
 .../Transforms/DoConcurrent/basic_host.f90    |  2 +-
 100 files changed, 446 insertions(+), 446 deletions(-)

diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 0740ed509a077..f55e88dab20ce 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -8,7 +8,7 @@ program main
   integer, device :: dvar
 end program
 
-! CHECK-LABEL: PROGRAM main
+! CHECK-LABEL: PROGRAM MAIN
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
diff --git a/flang/test/Driver/unparse-use-analyzed.f95 b/flang/test/Driver/unparse-use-analyzed.f95
index eb6046aebba54..4bcd72c9a5f50 100644
--- a/flang/test/Driver/unparse-use-analyzed.f95
+++ b/flang/test/Driver/unparse-use-analyzed.f95
@@ -6,12 +6,12 @@
 ! RUN: %flang_fc1 -fdebug-unparse  %s | FileCheck %s --check-prefix=DEFAULT
 ! RUN: %flang_fc1 -fdebug-unparse -fno-analyzed-objects-for-unparse %s | FileCheck %s --check-prefix=DISABLED
 
-! DEFAULT: PROGRAM test
+! DEFAULT: PROGRAM TEST
 ! DEFAULT-NEXT:  REAL, PARAMETER :: val = 3.43e2_4
 ! DEFAULT-NEXT:  PRINT *, 3.47e2_4
 ! DEFAULT-NEXT: END PROGRAM
 
-! DISABLED: PROGRAM test
+! DISABLED: PROGRAM TEST
 ! DISABLED-NEXT:  REAL, PARAMETER :: val = 343.0
 ! DISABLED-NEXT:  PRINT *, val+4
 ! DISABLED-NEXT: END PROGRAM
diff --git a/flang/test/Driver/unparse-with-modules.f90 b/flang/test/Driver/unparse-with-modules.f90
index 53997f7804efa..f6444afbe47c1 100644
--- a/flang/test/Driver/unparse-with-modules.f90
+++ b/flang/test/Driver/unparse-with-modules.f90
@@ -25,7 +25,7 @@ program test
 !CHECK:  implicit none
 !CHECK:  real(kind=real32) x
 !CHECK: end module
-!CHECK: program test
+!CHECK: program TEST
 !CHECK:  use :: m1
 !CHECK:  use :: basictestmoduletwo
 !CHECK:  implicit none
diff --git a/flang/test/Integration/debug-common-block-1.f90 b/flang/test/Integration/debug-common-block-1.f90
index 18217637be0fa..77f47daea4a91 100644
--- a/flang/test/Integration/debug-common-block-1.f90
+++ b/flang/test/Integration/debug-common-block-1.f90
@@ -89,7 +89,7 @@ program test
 ! CHECK-DAG: ![[CBF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "__BLNK__"{{.*}})
 ! CHECK-DAG: ![[CBAF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "a"{{.*}})
 
-! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "test"{{.*}})
+! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "TEST"{{.*}})
 ! CHECK-DAG: ![[CBM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "__BLNK__"{{.*}})
 ! CHECK-DAG: ![[CBAM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "a"{{.*}})
 
diff --git a/flang/test/Integration/debug-local-var-2.f90 b/flang/test/Integration/debug-local-var-2.f90
index b97be141cc8d0..0ddac633a5b1e 100644
--- a/flang/test/Integration/debug-local-var-2.f90
+++ b/flang/test/Integration/debug-local-var-2.f90
@@ -37,7 +37,7 @@
 ! BOTH-LABEL: }
 
 program mn
-! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "mn", {{.*}})
+! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "MN", {{.*}})
 
 ! BOTH-DAG: ![[TYI32:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
 ! BOTH-DAG: ![[TYI64:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)
diff --git a/flang/test/Lower/CUDA/cuda-derived.cuf b/flang/test/Lower/CUDA/cuda-derived.cuf
index 96250d88d81c4..d419ee074f7a0 100644
--- a/flang/test/Lower/CUDA/cuda-derived.cuf
+++ b/flang/test/Lower/CUDA/cuda-derived.cuf
@@ -25,6 +25,6 @@ program main
   type(t2) :: b
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"}
 ! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEa"}
 ! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tt2{b:!fir.type<_QMm1Tt1{a:!fir.box<!fir.heap<!fir.array<?xf32>>>}>}> {bindc_name = "b", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/CUDA/cuda-return01.cuf b/flang/test/Lower/CUDA/cuda-return01.cuf
index 47e69a903efd3..ed7c640a71082 100644
--- a/flang/test/Lower/CUDA/cuda-return01.cuf
+++ b/flang/test/Lower/CUDA/cuda-return01.cuf
@@ -28,6 +28,6 @@ program main
   return
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"}
 ! CHECK: cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QFEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK-NOT: cuf.free
diff --git a/flang/test/Lower/CUDA/cuda-return02.cuf b/flang/test/Lower/CUDA/cuda-return02.cuf
index e450d7e796f22..e54818444e49c 100644
--- a/flang/test/Lower/CUDA/cuda-return02.cuf
+++ b/flang/test/Lower/CUDA/cuda-return02.cuf
@@ -13,7 +13,7 @@ program test
   return
 end
 
-! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare
 ! CHECK: cf.cond_br %{{.*}}, ^bb1, ^bb2
 ! CHECK-NEXT: ^bb1:
diff --git a/flang/test/Lower/HLFIR/intrinsic-subroutines.f90 b/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
index 07c4f012781d4..cbc56ca1e395b 100644
--- a/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
+++ b/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
@@ -24,7 +24,7 @@ program main
   call mvbits(from, 2, 2, to, 0)
   if (any(to /= 5)) STOP 1
 end program
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<3xi32> {bindc_name = "from", uniq_name = "_QFEfrom"}
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
index 7b64634d10d4b..a097b1522307e 100644
--- a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
@@ -35,7 +35,7 @@ FUNCTION BAR() RESULT(res)
     END
   END
 
-! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:    %[[VAL_0:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_1:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_2:.*]] = fir.alloca !fir.boxproc<(!fir.ref<i32>) -> i32> {bindc_name = "pp2", uniq_name = "_QFEpp2"}
diff --git a/flang/test/Lower/OpenACC/acc-atomic-read.f90 b/flang/test/Lower/OpenACC/acc-atomic-read.f90
index 639a98051e3a2..76751a0fa63a8 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-read.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-read.f90
@@ -8,7 +8,7 @@ program acc_atomic_test
      g = h
 end program acc_atomic_test
 
-! CHECK: func @_QQmain() attributes {fir.bindc_name = "acc_atomic_test"} {
+! CHECK: func @_QQmain() attributes {fir.bindc_name = "ACC_ATOMIC_TEST"} {
 ! CHECK: %[[VAR_G:.*]] = fir.alloca f32 {bindc_name = "g", uniq_name = "_QFEg"}
 ! CHECK: %[[G_DECL:.*]]:2 = hlfir.declare %[[VAR_G]] {uniq_name = "_QFEg"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK: %[[VAR_H:.*]] = fir.alloca f32 {bindc_name = "h", uniq_name = "_QFEh"}
diff --git a/flang/test/Lower/OpenACC/acc-atomic-write.f90 b/flang/test/Lower/OpenACC/acc-atomic-write.f90
index 3c55394021abf..e0116e3281820 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-write.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-write.f90
@@ -2,7 +2,7 @@
 
 ! This test checks the lowering of atomic write
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "acc_atomic_write_test"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "ACC_ATOMIC_WRITE_TEST"} {
 !CHECK: %[[VAR_X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[VAR_X]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[VAR_Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
diff --git a/flang/test/Lower/OpenACC/acc-routine04.f90 b/flang/test/Lower/OpenACC/acc-routine04.f90
index f603376163901..655e2762b9694 100644
--- a/flang/test/Lower/OpenACC/acc-routine04.f90
+++ b/flang/test/Lower/OpenACC/acc-routine04.f90
@@ -30,5 +30,5 @@ subroutine sub2()
 ! CHECK: acc.routine @acc_routine_1 func(@_QFPsub2) seq
 ! CHECK: acc.routine @acc_routine_0 func(@_QMdummy_modPsub1) seq
 ! CHECK: func.func @_QMdummy_modPsub1(%arg0: !fir.ref<i32> {fir.bindc_name = "i"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>}
-! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test_acc_routine"}
+! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "TEST_ACC_ROUTINE"}
 ! CHECK: func.func private @_QFPsub2() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>, fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>}
diff --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
index 68dcaac90eef5..30313e240efa3 100644
--- a/flang/test/Lower/OpenMP/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -4,7 +4,7 @@
 
 ! This test checks the lowering of atomic read
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomic"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "OMPATOMIC"} {
 !CHECK:    %[[A_REF:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_REF]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[B_REF:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/OpenMP/atomic-write.f90 b/flang/test/Lower/OpenMP/atomic-write.f90
index 6eded49b0b15d..742fd475c0f04 100644
--- a/flang/test/Lower/OpenMP/atomic-write.f90
+++ b/flang/test/Lower/OpenMP/atomic-write.f90
@@ -4,7 +4,7 @@
 
 ! This test checks the lowering of atomic write
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomicwrite"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "OMPATOMICWRITE"} {
 !CHECK:    %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[Y_REF:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
diff --git a/flang/test/Lower/OpenMP/common-atomic-lowering.f90 b/flang/test/Lower/OpenMP/common-atomic-lowering.f90
index a53cc101024c6..f729bbb00ac8e 100644
--- a/flang/test/Lower/OpenMP/common-atomic-lowering.f90
+++ b/flang/test/Lower/OpenMP/common-atomic-lowering.f90
@@ -1,6 +1,6 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "sample"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "SAMPLE"} {
 !CHECK: %[[val_0:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK: %[[val_1:.*]]:2 = hlfir.declare %[[val_0]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[val_2:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/OpenMP/cray-pointers02.f90 b/flang/test/Lower/OpenMP/cray-pointers02.f90
index 19e4cd09fe50a..79d838702e4b0 100644
--- a/flang/test/Lower/OpenMP/cray-pointers02.f90
+++ b/flang/test/Lower/OpenMP/cray-pointers02.f90
@@ -1,7 +1,7 @@
 ! Test lowering of Cray pointee references.
 ! RUN: flang -fc1 -emit-hlfir -fopenmp %s -o - 2>&1 | FileCheck %s
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test_cray_pointers_02"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST_CRAY_POINTERS_02"}
 program test_cray_pointers_02
     implicit none
 
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index c44c6bb966580..af51c4cc3e814 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -34,7 +34,7 @@
 !CHECK:    omp.yield(%[[PRIV_X]] : !fir.ref<i32>)
 !CHECK:  }
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "DEFAULT_CLAUSE_LOWERING"} {
 !CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"}
 !CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index ee5f579f06b91..505fa4f0f5d63 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -8,7 +8,7 @@
 ! RUN: | FileCheck %s
 
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "DEFAULT_CLAUSE_LOWERING"} {
 !CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"}
 !CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index 4bfd5d8d19261..0036670317a8e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -80,7 +80,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index ec54294c7104f..ea0aa9ec3f53b 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -68,7 +68,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
index 488ecc353af8e..eb0df2b3a17de 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
@@ -63,7 +63,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
index 596276a99cafc..2caec0384a6ab 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
@@ -18,7 +18,7 @@
 !CHECK:    fir.store %[[CR]] to %[[C0]] : !fir.ref<i32>
 !CHECK:    omp.yield(%[[C0]] : !fir.ref<i32>)
 !CHECK:  }
-!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "mn"} {
+!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "MN"} {
 !CHECK:    %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
index f638688bc2cc1..3c1daa0eb983f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
@@ -82,7 +82,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90 b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
index c06343e997bfd..2be154f4bbaf5 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
@@ -25,7 +25,7 @@ end program main
 ! CHECK:           omp.yield(%[[VAL_2]] : i32)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction.f90 b/flang/test/Lower/OpenMP/parallel-reduction.f90
index 612549fb32de5..15e8cc325916d 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction.f90
@@ -10,7 +10,7 @@
 !CHECK:    %[[CR:[_a-z0-9]+]] = arith.addi %[[C0]], %[[C1]] : i32
 !CHECK:    omp.yield(%[[CR]] : i32)
 !CHECK:  }
-!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "mn"} {
+!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "MN"} {
 !CHECK:    %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index d11925cafdc12..3d5c0326fb6b9 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -5,7 +5,7 @@
 ! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 ! RUN: bbc -hlfir -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "sample"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "SAMPLE"} {
 !CHECK:   %[[COUNT:.*]] = fir.address_of(@_QFEcount) : !fir.ref<i32>
 !CHECK:   %[[COUNT_DECL:.*]]:2 = hlfir.declare %[[COUNT]] {uniq_name = "_QFEcount"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[ETA:.*]] = fir.alloca f32 {bindc_name = "eta", uniq_name = "_QFEeta"}
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
index 5e54cef8c29db..5c90ef7d84f89 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
index 21547b47cf381..0e61261e8853e 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association.f90
index 7a27efa2f84aa..1887e8aa68fdc 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[TP_A:.*]] = omp.threadprivate %[[A_DECL]]#0 : !fir.ref<i32> -> !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-chunks.f90 b/flang/test/Lower/OpenMP/wsloop-chunks.f90
index 29c02a3b3c8d5..f3f11d8c4a6c2 100644
--- a/flang/test/Lower/OpenMP/wsloop-chunks.f90
+++ b/flang/test/Lower/OpenMP/wsloop-chunks.f90
@@ -7,7 +7,7 @@ program wsloop
         integer :: i
         integer :: chunk
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "wsloop"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "WSLOOP"} {
 ! CHECK:         %[[CHUNK_REF:.*]] = fir.alloca i32 {bindc_name = "chunk", uniq_name = "_QFEchunk"}
 ! CHECK:         %[[VAL_0:.*]]:2 = hlfir.declare %[[CHUNK_REF]] {uniq_name = "_QFEchunk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
diff --git a/flang/test/Lower/OpenMP/wsloop-collapse.f90 b/flang/test/Lower/OpenMP/wsloop-collapse.f90
index a4d5cbdc03d3e..7ec40ab4b2f43 100644
--- a/flang/test/Lower/OpenMP/wsloop-collapse.f90
+++ b/flang/test/Lower/OpenMP/wsloop-collapse.f90
@@ -2,7 +2,7 @@
 
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
 
-!CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "wsloop_collapse"} {
+!CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "WSLOOP_COLLAPSE"} {
 program wsloop_collapse
 !CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index 58b68e5ec4cfd..e2f75bc8e4481 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -156,7 +156,7 @@ program reduce15
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce15"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE15"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEarr"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index 0a536eb34e7af..663851cba46c6 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -63,7 +63,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "r", uniq_name = "_QFEr"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
index 9f0dd16002baf..2233a74600948 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
@@ -31,5 +31,5 @@ program reduce
 ! CHECK:           omp.yield(%[[ARG0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           omp.wsloop {{.*}} reduction(byref @add_reduction_byref_box_2xi32 %{{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<2xi32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
index 5ada623a0ed23..211bde19da8db 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
@@ -40,5 +40,5 @@ subroutine sub(a, lb, ub)
 ! CHECK:           omp.yield(%[[ARG0]] : !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           omp.wsloop {{.*}} reduction(byref @add_reduction_byref_box_Uxi32 %{{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index 21261da49710c..b7882bcbc0d13 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index ab8dcf1f076c0..7d90335a13a87 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
index 1e26f5a24d41e..d776bd7cfdd03 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
@@ -28,7 +28,7 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_2]] : i32)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
index e0a3b469f40c1..5133db0347034 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -93,7 +93,7 @@ program main
 ! CHECK:           omp.yield(%[[VAL_2]] : f64)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarray) : !fir.ref<!fir.array<3x3xf64>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
index 40b4302f24cd4..27b726376fbeb 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
@@ -64,7 +64,7 @@ program reduce_pointer
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce_pointer"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE_POINTER"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "v", uniq_name = "_QFEv"}
diff --git a/flang/test/Lower/array-character.f90 b/flang/test/Lower/array-character.f90
index 1bc73dae44235..e2899d967c80d 100644
--- a/flang/test/Lower/array-character.f90
+++ b/flang/test/Lower/array-character.f90
@@ -32,7 +32,7 @@ program p
   call charlit
 end program p
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.array<3x!fir.char<1,4>> {bindc_name = "c1", uniq_name = "_QFEc1"}
diff --git a/flang/test/Lower/array-expression-slice-1.f90 b/flang/test/Lower/array-expression-slice-1.f90
index b597814bc0d9f..73943137cb18d 100644
--- a/flang/test/Lower/array-expression-slice-1.f90
+++ b/flang/test/Lower/array-expression-slice-1.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -hlfir=false -fwrapv -o - --outline-intrinsics %s | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK-DAG:         %[[VAL_0:.*]] = arith.constant 10 : index
 ! CHECK-DAG:         %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK-DAG:         %[[VAL_5:.*]] = arith.constant 1 : index
diff --git a/flang/test/Lower/basic-program.f90 b/flang/test/Lower/basic-program.f90
index 5a0e4bdc7b4a1..7e5b40d9e2f0a 100644
--- a/flang/test/Lower/basic-program.f90
+++ b/flang/test/Lower/basic-program.f90
@@ -4,10 +4,10 @@
 program basic
 end program
 
-! CHECK: 1 Program basic
+! CHECK: 1 Program BASIC
 ! CHECK:   1 EndProgramStmt: end program
-! CHECK: End Program basic
+! CHECK: End Program BASIC
 
-! FIR-LABEL: func @_QQmain() attributes {fir.bindc_name = "basic"} {
+! FIR-LABEL: func @_QQmain() attributes {fir.bindc_name = "BASIC"} {
 ! FIR:         return
 ! FIR:       }
diff --git a/flang/test/Lower/big-integer-parameter.f90 b/flang/test/Lower/big-integer-parameter.f90
index a413b1224ebc2..ca90b8adfb318 100644
--- a/flang/test/Lower/big-integer-parameter.f90
+++ b/flang/test/Lower/big-integer-parameter.f90
@@ -13,7 +13,7 @@ program i128
   print*,y
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "i128"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "I128"} {
 ! CHECK-COUNT-2:  %{{.*}} = fir.call @_FortranAioOutputInteger128(%{{.*}}, %{{.*}}) {{.*}}: (!fir.ref<i8>, i128) -> i1
 
 
diff --git a/flang/test/Lower/derived-type-finalization.f90 b/flang/test/Lower/derived-type-finalization.f90
index 3ea58cd719f4a..71cef34899603 100644
--- a/flang/test/Lower/derived-type-finalization.f90
+++ b/flang/test/Lower/derived-type-finalization.f90
@@ -255,5 +255,5 @@ program p
   type(t1) :: t
 end program
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK-NOT: fir.call @_FortranADestroy
diff --git a/flang/test/Lower/location.f90 b/flang/test/Lower/location.f90
index a6ece31bbebed..95bf2260fc107 100644
--- a/flang/test/Lower/location.f90
+++ b/flang/test/Lower/location.f90
@@ -5,7 +5,7 @@ program test
 
 end 
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: fir.call @_FortranAioOutputAscii(%{{.*}}, %{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1 loc(fused<#fir<loc_kind_array[ base,  inclusion,  inclusion]>>["{{.*}}location1.inc":1:10, "{{.*}}location0.inc":1:1, "{{.*}}location.f90":4:1])
 ! CHECK: return loc("{{.*}}location.f90":6:1)
 ! CHECK: } loc("{{.*}}location.f90":3:1)
diff --git a/flang/test/Lower/nested-where.f90 b/flang/test/Lower/nested-where.f90
index ab457280b80ce..28aced2325813 100644
--- a/flang/test/Lower/nested-where.f90
+++ b/flang/test/Lower/nested-where.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "nested_where"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "NESTED_WHERE"} {
 program nested_where
 
   ! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {adapt.valuebyref, bindc_name = "i"}
diff --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90
index b7be5f685d9e3..a84b495dd09d0 100644
--- a/flang/test/Lower/polymorphic.f90
+++ b/flang/test/Lower/polymorphic.f90
@@ -1146,7 +1146,7 @@ program test
   l = i < o%inner
 end program
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: %[[ADDR_O:.*]] = fir.address_of(@_QFEo) : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMpolymorphic_testTouter{inner:!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>}>>>>
 ! CHECK: %[[BOX_NONE:.*]] = fir.convert %[[ADDR_O]] : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMpolymorphic_testTouter{inner:!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>}>>>>) -> !fir.ref<!fir.box<none>>
 ! CHECK: %{{.*}} = fir.call @_FortranAAllocatableAllocate(%[[BOX_NONE]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
diff --git a/flang/test/Lower/pre-fir-tree02.f90 b/flang/test/Lower/pre-fir-tree02.f90
index f4fa626ba6548..65c33e9b364fe 100644
--- a/flang/test/Lower/pre-fir-tree02.f90
+++ b/flang/test/Lower/pre-fir-tree02.f90
@@ -3,7 +3,7 @@
 ! Test Pre-FIR Tree captures all the intended nodes from the parse-tree
 ! Coarray and OpenMP related nodes are tested in other files.
 
-! CHECK: Program test_prog
+! CHECK: Program TEST_PROG
 program test_prog
   ! Check specification part is not part of the tree.
   interface
diff --git a/flang/test/Lower/pre-fir-tree03.f90 b/flang/test/Lower/pre-fir-tree03.f90
index 313dab4d6ec7c..1de66e3f8d016 100644
--- a/flang/test/Lower/pre-fir-tree03.f90
+++ b/flang/test/Lower/pre-fir-tree03.f90
@@ -2,7 +2,7 @@
 
 ! Test Pre-FIR Tree captures OpenMP related constructs
 
-! CHECK: Program test_omp
+! CHECK: Program TEST_OMP
 program test_omp
   ! CHECK: PrintStmt
   print *, "sequential"
diff --git a/flang/test/Lower/pre-fir-tree06.f90 b/flang/test/Lower/pre-fir-tree06.f90
index f84bcd8b58b2d..ed1e76cb375bd 100644
--- a/flang/test/Lower/pre-fir-tree06.f90
+++ b/flang/test/Lower/pre-fir-tree06.f90
@@ -25,13 +25,13 @@ subroutine sub2()
 end
 ! CHECK: End Module m2
 
-! CHECK: Program main
+! CHECK: Program MAIN
 program main
   real :: y
   ! CHECK-NEXT: OpenMPDeclarativeConstruct
   !$omp threadprivate(y)
 end
-! CHECK: End Program main
+! CHECK: End Program MAIN
 
 ! CHECK: Subroutine sub1
 subroutine sub1()
diff --git a/flang/test/Lower/program-units-fir-mangling.f90 b/flang/test/Lower/program-units-fir-mangling.f90
index e0af6f065f34d..65940b4e1ff17 100644
--- a/flang/test/Lower/program-units-fir-mangling.f90
+++ b/flang/test/Lower/program-units-fir-mangling.f90
@@ -124,7 +124,7 @@ subroutine should_not_collide()
 ! CHECK: }
 end subroutine
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 program test
 ! CHECK: }
 contains
diff --git a/flang/test/Lower/return-statement.f90 b/flang/test/Lower/return-statement.f90
index 6351a6859eb4f..8ab69e3146e2f 100644
--- a/flang/test/Lower/return-statement.f90
+++ b/flang/test/Lower/return-statement.f90
@@ -4,7 +4,7 @@ program basic
   return
 end program
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "basic"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "BASIC"} {
 ! CHECK:         return
 ! CHECK:       }
 
diff --git a/flang/test/Lower/volatile-openmp1.f90 b/flang/test/Lower/volatile-openmp1.f90
index 163db953b6b80..07d81a1aeb240 100644
--- a/flang/test/Lower/volatile-openmp1.f90
+++ b/flang/test/Lower/volatile-openmp1.f90
@@ -13,7 +13,7 @@ program main
 !$omp end parallel
 end program
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 1000 : i32
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/volatile-string.f90 b/flang/test/Lower/volatile-string.f90
index 88b21d7b245e9..f263db7abb5fc 100644
--- a/flang/test/Lower/volatile-string.f90
+++ b/flang/test/Lower/volatile-string.f90
@@ -21,7 +21,7 @@ subroutine assign_different_length(string)
   end subroutine
 end program
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 11 : i32
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant true
diff --git a/flang/test/Lower/volatile3.f90 b/flang/test/Lower/volatile3.f90
index 8825f8f3afbcb..a32f29d2bb9e7 100644
--- a/flang/test/Lower/volatile3.f90
+++ b/flang/test/Lower/volatile3.f90
@@ -70,7 +70,7 @@ subroutine sub_select_rank(arr)
 end program
 
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 10 : index
diff --git a/flang/test/Parser/acc-unparse.f90 b/flang/test/Parser/acc-unparse.f90
index 62e0d4487f3f7..12e6dec19f272 100644
--- a/flang/test/Parser/acc-unparse.f90
+++ b/flang/test/Parser/acc-unparse.f90
@@ -15,7 +15,7 @@ program bug47659
   end do label1
 end program
 
-!CHECK-LABEL: PROGRAM bug47659
+!CHECK-LABEL: PROGRAM BUG47659
 !CHECK: !$ACC PARALLEL LOOP
 
 
diff --git a/flang/test/Semantics/OpenACC/acc-symbols01.f90 b/flang/test/Semantics/OpenACC/acc-symbols01.f90
index 375445bad13a5..ea22f41197aa7 100644
--- a/flang/test/Semantics/OpenACC/acc-symbols01.f90
+++ b/flang/test/Semantics/OpenACC/acc-symbols01.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenacc
 
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !DEF: /mm/x ObjectEntity REAL(4)
   !DEF: /mm/y ObjectEntity REAL(4)
diff --git a/flang/test/Semantics/OpenMP/critical_within_default.f90 b/flang/test/Semantics/OpenMP/critical_within_default.f90
index dd972e6e52949..a5fe30eeb7de0 100644
--- a/flang/test/Semantics/OpenMP/critical_within_default.f90
+++ b/flang/test/Semantics/OpenMP/critical_within_default.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
 ! Test that we do not make a private copy of the critical name
 
-!CHECK:  MainProgram scope: mn
+!CHECK:  MainProgram scope: MN
 !CHECK-NEXT:    j size=4 offset=0: ObjectEntity type: INTEGER(4)
 !CHECK-NEXT:    OtherConstruct scope:
 !CHECK-NEXT:      j (OmpPrivate): HostAssoc
diff --git a/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90 b/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
index 06f41ab8ce76f..e57a5c0c1cea6 100644
--- a/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
+++ b/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
 
 program main
-!CHECK-LABEL: MainProgram scope: main
+!CHECK-LABEL: MainProgram scope: MAIN
    implicit none
 
    type ty
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90 b/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
index 9d0a097fb1991..fc977f2f1b839 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
@@ -17,7 +17,7 @@ end function mymax
 end module mymod
 
 program omp_examples
-!CHECK-LABEL: MainProgram scope: omp_examples
+!CHECK-LABEL: MainProgram scope: OMP_EXAMPLES
   use mymod
   implicit none
   integer, parameter :: n = 100
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-operators.f90 b/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
index d7a9f2fc0a36b..84dbe1af01877 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
@@ -49,7 +49,7 @@ function my_add(x, y)
 end module m1
 
 program test_vector
-!CHECK-LABEL: MainProgram scope: test_vector
+!CHECK-LABEL: MainProgram scope: TEST_VECTOR
   use vector_mod
 !CHECK: add_vectors (Function): Use from add_vectors in vector_mod
   implicit none
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90 b/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
index 12e80cbf7b327..9cd638d796091 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
@@ -22,7 +22,7 @@ end function my_mul
 end module module1
 
 program test_omp_reduction
-!CHECK: MainProgram scope: test_omp_reduction
+!CHECK: MainProgram scope: TEST_OMP_REDUCTION
   use module1, only: t1, operator(.modmul.) => operator(.mul.)
 
 !CHECK: .modmul. (Function): Use from .mul. in module1
diff --git a/flang/test/Semantics/OpenMP/declare-reduction.f90 b/flang/test/Semantics/OpenMP/declare-reduction.f90
index ddca38fd57812..1f39c57c54ad1 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction.f90
@@ -31,7 +31,7 @@ end subroutine initme
 end function func
 
 program main
-!CHECK-LABEL: MainProgram scope: main
+!CHECK-LABEL: MainProgram scope: MAIN
 
   !$omp declare reduction (my_add_red : integer : omp_out = omp_out + omp_in) initializer (omp_priv=0)
 
diff --git a/flang/test/Semantics/OpenMP/do-schedule03.f90 b/flang/test/Semantics/OpenMP/do-schedule03.f90
index 8787b094d581a..375f3920b57ad 100644
--- a/flang/test/Semantics/OpenMP/do-schedule03.f90
+++ b/flang/test/Semantics/OpenMP/do-schedule03.f90
@@ -2,26 +2,26 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - xx.
-  !DEF: /ompdoschedule MainProgram
+  !DEF: /OMPDOSCHEDULE MainProgram
 program ompdoschedule
-  !DEF: /ompdoschedule/a ObjectEntity REAL(4)
-  !DEF: /ompdoschedule/y ObjectEntity REAL(4)
-  !DEF: /ompdoschedule/z ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/a ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/y ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/z ObjectEntity REAL(4)
   real  a(100),y(100),z(100)
-  !DEF: /ompdoschedule/b ObjectEntity INTEGER(4)
-  !DEF: /ompdoschedule/i ObjectEntity INTEGER(4)
-  !DEF: /ompdoschedule/n ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/b ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/i ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/n ObjectEntity INTEGER(4)
   integer  b,i,n
-  !REF: /ompdoschedule/b
+  !REF: /OMPDOSCHEDULE/b
   b = 10
   !$omp do  schedule(static,b-b)
-  !DEF: /ompdoschedule/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !REF: /ompdoschedule/n
+  !DEF: /OMPDOSCHEDULE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !REF: /OMPDOSCHEDULE/n
   do i = 2,n+1
-    !REF: /ompdoschedule/y
-    !REF: /ompdoschedule/OtherConstruct1/i
-    !REF: /ompdoschedule/z
-    !REF: /ompdoschedule/a
+    !REF: /OMPDOSCHEDULE/y
+    !REF: /OMPDOSCHEDULE/OtherConstruct1/i
+    !REF: /OMPDOSCHEDULE/z
+    !REF: /OMPDOSCHEDULE/a
     y(i) = z(i-1) + a(i)
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/do01-positivecase.f90 b/flang/test/Semantics/OpenMP/do01-positivecase.f90
index 905fdbaf18476..a4a28f6530d7b 100644
--- a/flang/test/Semantics/OpenMP/do01-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do01-positivecase.f90
@@ -4,13 +4,13 @@
 ! The loop iteration variable may not appear in a firstprivate directive.
 ! A positive case
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   integer i
 
   !$omp do  firstprivate(k)
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
     print *, "Hello"
   end do
diff --git a/flang/test/Semantics/OpenMP/do04-positivecase.f90 b/flang/test/Semantics/OpenMP/do04-positivecase.f90
index eb2d67bb8ceb2..0b0c661687dc2 100644
--- a/flang/test/Semantics/OpenMP/do04-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do04-positivecase.f90
@@ -2,18 +2,18 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 
-!DEF: /omp_do1 MainProgram
+!DEF: /OMP_DO1 MainProgram
 program omp_do1
-  !DEF: /omp_do1/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/n (OmpThreadprivate) ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/n (OmpThreadprivate) ObjectEntity INTEGER(4)
   integer i, j, k, n
   !$omp threadprivate (k,n)
   !$omp do
-  !DEF: /omp_do1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do1/j
+    !REF: /OMP_DO1/j
     do j=1,10
       print *, "Hello"
     end do
diff --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index eda04610535c2..07b12dd1d4d23 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -3,13 +3,13 @@
 ! 2.7.1 Loop Construct restrictions on single directive.
 ! A positive case
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/n ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/n ObjectEntity INTEGER(4)
   integer i,n
   !$omp parallel
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
     !$omp single
     print *, "hello"
@@ -19,13 +19,13 @@ program omp_do
 
   !$omp parallel  default(shared)
   !$omp do
-  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
   do i=1,n
     !$omp parallel
     !$omp single
     !DEF: /work EXTERNAL (Subroutine) ProcEntity
-    !DEF: /omp_do/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
+    !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
     call work(i, 1)
     !$omp end single
     !$omp end parallel
@@ -34,7 +34,7 @@ program omp_do
   !$omp end parallel
 
   !$omp parallel private(i)
-  !DEF: /omp_do/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
   do i=1,10
      !$omp single
      print *, "hello"
@@ -43,32 +43,32 @@ program omp_do
   !$omp end parallel
 
   !$omp target teams distribute parallel do
-  !DEF:/omp_do/OtherConstruct4/i (OmpPrivate ,OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF:/OMP_DO/OtherConstruct4/i (OmpPrivate ,OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF:/omp_do/OtherConstruct4/i
+    !REF:/OMP_DO/OtherConstruct4/i
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do
 
   !$omp target teams distribute parallel do simd
-  !DEF:/omp_do/OtherConstruct5/i (OmpLinear,OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF:/OMP_DO/OtherConstruct5/i (OmpLinear,OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF:/omp_do/OtherConstruct5/i
+    !REF:/OMP_DO/OtherConstruct5/i
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute 
-  !DEF: /omp_do/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF: /omp_do/OtherConstruct6/i
+    !REF: /OMP_DO/OtherConstruct6/i
     if(i < 5) cycle
   end do
 
   !$omp target teams distribute simd
-  !DEF: /omp_do/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF: /omp_do/OtherConstruct7/i
+    !REF: /OMP_DO/OtherConstruct7/i
     if(i < 5) cycle
   end do
 end program omp_do
diff --git a/flang/test/Semantics/OpenMP/do06-positivecases.f90 b/flang/test/Semantics/OpenMP/do06-positivecases.f90
index 2713b55fa2ecb..4d45800298a29 100644
--- a/flang/test/Semantics/OpenMP/do06-positivecases.f90
+++ b/flang/test/Semantics/OpenMP/do06-positivecases.f90
@@ -5,14 +5,14 @@
 ! region ever binds to a loop region arising from the loop construct.
 
 ! A positive case
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
   integer i, j, k
   !$omp do  ordered
-    !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do i=1,10
       !$omp ordered
       !DEF: /my_func EXTERNAL (Subroutine) ProcEntity
diff --git a/flang/test/Semantics/OpenMP/do11.f90 b/flang/test/Semantics/OpenMP/do11.f90
index faab457efff3c..886a9f7ae6081 100644
--- a/flang/test/Semantics/OpenMP/do11.f90
+++ b/flang/test/Semantics/OpenMP/do11.f90
@@ -2,34 +2,34 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
   integer i, j, k
   !$omp do
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do/j
+    !REF: /OMP_DO/j
     do j=1,10
-      !REF: /omp_do/OtherConstruct1/i
-      !REF: /omp_do/j
+      !REF: /OMP_DO/OtherConstruct1/i
+      !REF: /OMP_DO/j
       print *, "it", i, j
     end do
   end do
   !$omp end do
 end program omp_do
 
-!DEF: /omp_do2 (Subroutine)Subprogram
+!DEF: /OMP_DO2 (Subroutine)Subprogram
 subroutine omp_do2
-  !DEF: /omp_do2/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do2/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO2/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO2/k ObjectEntity INTEGER(4)
   integer :: i = 0, k
   !$omp do
-  !DEF: /omp_do2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do2/OtherConstruct1/i
+    !REF: /OMP_DO2/OtherConstruct1/i
     print *, "it", i
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/do12.f90 b/flang/test/Semantics/OpenMP/do12.f90
index a057a246f7a99..f36a230426197 100644
--- a/flang/test/Semantics/OpenMP/do12.f90
+++ b/flang/test/Semantics/OpenMP/do12.f90
@@ -2,20 +2,20 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /omp_cycle MainProgram
+!DEF: /OMP_CYCLE MainProgram
 program omp_cycle
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/OtherConstruct1/i
+    !REF: /OMP_CYCLE/OtherConstruct1/i
     if (i<1) cycle
-    !DEF: /omp_cycle/j (Implicit) ObjectEntity INTEGER(4)
+    !DEF: /OMP_CYCLE/j (Implicit) ObjectEntity INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/k (Implicit) ObjectEntity INTEGER(4)
+      !DEF: /OMP_CYCLE/k (Implicit) ObjectEntity INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct1/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct1/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -23,17 +23,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/j
+    !REF: /OMP_CYCLE/j
     do j=0,10
-      !REF: /omp_cycle/OtherConstruct2/i
+      !REF: /OMP_CYCLE/OtherConstruct2/i
       if (i<1) cycle
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct2/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct2/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -41,17 +41,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(2)
-  !DEF: /omp_cycle/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/i
         if (i<1) cycle
-        !REF: /omp_cycle/OtherConstruct3/i
-        !REF: /omp_cycle/OtherConstruct3/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -59,17 +59,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/i
         if (i<1) cycle
-        !REF: /omp_cycle/OtherConstruct4/i
-        !REF: /omp_cycle/OtherConstruct4/j
-        !REF: /omp_cycle/OtherConstruct4/k
+        !REF: /OMP_CYCLE/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/j
+        !REF: /OMP_CYCLE/OtherConstruct4/k
         print *, i, j, k
       end do
     end do
@@ -77,20 +77,20 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo:do i=0,10
-    !DEF: /omp_cycle/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     foo1:do j=0,10
-      !DEF: /omp_cycle/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       foo2:do k=0,10
-        !REF: /omp_cycle/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/i
         if (i<1) cycle foo2
-        !REF: /omp_cycle/OtherConstruct5/i
-        !REF: /omp_cycle/OtherConstruct5/j
-        !REF: /omp_cycle/OtherConstruct5/k
+        !REF: /OMP_CYCLE/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/j
+        !REF: /OMP_CYCLE/OtherConstruct5/k
         print *, i, j, k
       end do foo2
     end do foo1
   end do foo
   !$omp end do
-end program omp_cycle
+end program OMP_CYCLE
diff --git a/flang/test/Semantics/OpenMP/do14.f90 b/flang/test/Semantics/OpenMP/do14.f90
index 5e8a5a64c2979..b508d32d90201 100644
--- a/flang/test/Semantics/OpenMP/do14.f90
+++ b/flang/test/Semantics/OpenMP/do14.f90
@@ -2,19 +2,19 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /omp_cycle MainProgram
+!DEF: /OMP_CYCLE MainProgram
 program omp_cycle
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
     cycle
-    !DEF: /omp_cycle/j (Implicit) ObjectEntity INTEGER(4)
+    !DEF: /OMP_CYCLE/j (Implicit) ObjectEntity INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/k (Implicit) ObjectEntity INTEGER(4)
+      !DEF: /OMP_CYCLE/k (Implicit) ObjectEntity INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct1/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct1/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -22,16 +22,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/j
+    !REF: /OMP_CYCLE/j
     do j=0,10
       cycle
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct2/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct2/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -39,16 +39,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(2)
-  !DEF: /omp_cycle/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
         cycle
-        !REF: /omp_cycle/OtherConstruct3/i
-        !REF: /omp_cycle/OtherConstruct3/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -56,16 +56,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       do k=0,10
         cycle
-        !REF: /omp_cycle/OtherConstruct4/i
-        !REF: /omp_cycle/OtherConstruct4/j
-        !REF: /omp_cycle/OtherConstruct4/k
+        !REF: /OMP_CYCLE/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/j
+        !REF: /OMP_CYCLE/OtherConstruct4/k
         print *, i, j, k
       end do
     end do
@@ -73,16 +73,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  ordered(3)
-  !DEF: /omp_cycle/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo:do i=0,10
-    !DEF: /omp_cycle/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     foo1:do j=0,10
-      !DEF: /omp_cycle/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       foo2:do k=0,10
         cycle foo2
-        !REF: /omp_cycle/OtherConstruct5/i
-        !REF: /omp_cycle/OtherConstruct5/j
-        !REF: /omp_cycle/OtherConstruct5/k
+        !REF: /OMP_CYCLE/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/j
+        !REF: /OMP_CYCLE/OtherConstruct5/k
         print *, i, j, k
       end do foo2
     end do foo1
diff --git a/flang/test/Semantics/OpenMP/do17.f90 b/flang/test/Semantics/OpenMP/do17.f90
index c0c59f16dee1b..0a4cf46d2d6af 100644
--- a/flang/test/Semantics/OpenMP/do17.f90
+++ b/flang/test/Semantics/OpenMP/do17.f90
@@ -2,53 +2,53 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /test MainProgram
+!DEF: /TEST MainProgram
 program test
- !DEF: /test/i ObjectEntity INTEGER(4)
- !DEF: /test/j ObjectEntity INTEGER(4)
- !DEF: /test/k ObjectEntity INTEGER(4)
+ !DEF: /TEST/i ObjectEntity INTEGER(4)
+ !DEF: /TEST/j ObjectEntity INTEGER(4)
+ !DEF: /TEST/k ObjectEntity INTEGER(4)
  integer i, j, k
  !$omp do  collapse(2)
- !DEF: /test/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /TEST/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  foo: do i=0,10
-  !DEF: /test/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /TEST/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo1: do j=0,10
-   !REF: /test/k
+   !REF: /TEST/k
    foo2: do k=0,10
-    !REF: /test/OtherConstruct1/i
+    !REF: /TEST/OtherConstruct1/i
     select case (i)
     case (5)
      cycle foo1
     case (7)
      cycle foo2
     end select
-    !REF: /test/OtherConstruct1/i
-    !REF: /test/OtherConstruct1/j
-    !REF: /test/k
+    !REF: /TEST/OtherConstruct1/i
+    !REF: /TEST/OtherConstruct1/j
+    !REF: /TEST/k
     print *, i, j, k
    end do foo2
   end do foo1
  end do foo
 
  !$omp do  collapse(2)
- !DEF: /test/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /TEST/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  foo: do i=0,10
-  !DEF: /test/OtherConstruct2/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /TEST/OtherConstruct2/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo1: do j=0,10
-   !REF: /test/k
+   !REF: /TEST/k
    foo2: do k=0,10
-    !REF: /test/OtherConstruct2/i
+    !REF: /TEST/OtherConstruct2/i
     if (i<3) then
      cycle foo1
-     !REF: /test/OtherConstruct2/i
+     !REF: /TEST/OtherConstruct2/i
     else if (i>8) then
      cycle foo1
     else
      cycle foo2
     end if
-    !REF: /test/OtherConstruct2/i
-    !REF: /test/OtherConstruct2/j
-    !REF: /test/k
+    !REF: /TEST/OtherConstruct2/i
+    !REF: /TEST/OtherConstruct2/j
+    !REF: /TEST/k
     print *, i, j, k
    end do foo2
   end do foo1
diff --git a/flang/test/Semantics/OpenMP/map-clause-symbols.f90 b/flang/test/Semantics/OpenMP/map-clause-symbols.f90
index 8f984fcd2fa7e..1d6315b4a2312 100644
--- a/flang/test/Semantics/OpenMP/map-clause-symbols.f90
+++ b/flang/test/Semantics/OpenMP/map-clause-symbols.f90
@@ -1,6 +1,6 @@
 ! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
 program main
-!CHECK-LABEL:  MainProgram scope: main
+!CHECK-LABEL:  MainProgram scope: MAIN
   integer, parameter :: n = 256
   real(8) :: a(256)
   !$omp target map(mapper(xx), from:a)
diff --git a/flang/test/Semantics/OpenMP/reduction08.f90 b/flang/test/Semantics/OpenMP/reduction08.f90
index 01a06eb7d7414..1cf2838fe765a 100644
--- a/flang/test/Semantics/OpenMP/reduction08.f90
+++ b/flang/test/Semantics/OpenMP/reduction08.f90
@@ -2,60 +2,60 @@
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases
 
-!DEF: /omp_reduction MainProgram
+!DEF: /OMP_REDUCTION MainProgram
 program omp_reduction
-  !DEF: /omp_reduction/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
-  !DEF: /omp_reduction/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
   integer :: k = 10
-  !DEF: /omp_reduction/m ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/m ObjectEntity INTEGER(4)
   integer :: m = 12
   !$omp parallel do  reduction(max:k)
-  !DEF: /omp_reduction/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
     k = max(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(min:k)
-  !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
     k = min(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(iand:k)
-  !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
     k = iand(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(ior:k)
-  !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
     k = ior(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(ieor:k)
-  !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
     k = ieor(k,m)
   end do
   !$omp end parallel do
diff --git a/flang/test/Semantics/OpenMP/reduction09.f90 b/flang/test/Semantics/OpenMP/reduction09.f90
index d6c71c30d2834..4b1ca116d675b 100644
--- a/flang/test/Semantics/OpenMP/reduction09.f90
+++ b/flang/test/Semantics/OpenMP/reduction09.f90
@@ -1,22 +1,22 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases.
-!DEF: /omp_reduction MainProgram
+!DEF: /OMP_REDUCTION MainProgram
 program omp_reduction
-  !DEF: /omp_reduction/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
-  !DEF: /omp_reduction/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
   integer :: k = 10
-  !DEF: /omp_reduction/a ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/a ObjectEntity INTEGER(4)
   integer a(10)
-  !DEF: /omp_reduction/b ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/b ObjectEntity INTEGER(4)
   integer b(10,10,10)
 
   !$omp parallel  shared(k)
   !$omp do  reduction(+:k)
-  !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
@@ -24,52 +24,52 @@ program omp_reduction
 
 
   !$omp parallel do  reduction(+:a(10))
-  !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
 
   !$omp parallel do  reduction(+:a(1:10:1))
-  !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2))
-  !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2:5:1))
-  !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel  private(i)
   !$omp do reduction(+:k) reduction(+:j)
-  !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
   !$omp end parallel
 
   !$omp do reduction(+:k) reduction(*:j) reduction(+:l)
-  !DEF: /omp_reduction/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/reduction11.f90 b/flang/test/Semantics/OpenMP/reduction11.f90
index b2ad0f6a6ee11..dfb3986d37d78 100644
--- a/flang/test/Semantics/OpenMP/reduction11.f90
+++ b/flang/test/Semantics/OpenMP/reduction11.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols -o - %s 2>&1 | FileCheck %s
 ! Check intrinsic reduction symbols (in this case "max" are marked as INTRINSIC
 
-! CHECK: MainProgram scope: omp_reduction
+! CHECK: MainProgram scope: OMP_REDUCTION
 program omp_reduction
   ! CHECK: i size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer i
diff --git a/flang/test/Semantics/OpenMP/scan2.f90 b/flang/test/Semantics/OpenMP/scan2.f90
index ffe84910f88a2..1ae5e871595c4 100644
--- a/flang/test/Semantics/OpenMP/scan2.f90
+++ b/flang/test/Semantics/OpenMP/scan2.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols -o - %s 2>&1 | FileCheck %s
 ! Check scan reduction
 
-! CHECK: MainProgram scope: omp_reduction
+! CHECK: MainProgram scope: OMP_REDUCTION
 program omp_reduction
   ! CHECK: i size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer i
diff --git a/flang/test/Semantics/OpenMP/symbol01.f90 b/flang/test/Semantics/OpenMP/symbol01.f90
index fbd9a0286c79b..c1f16d61ada1e 100644
--- a/flang/test/Semantics/OpenMP/symbol01.f90
+++ b/flang/test/Semantics/OpenMP/symbol01.f90
@@ -16,53 +16,53 @@ module md
   integer :: b
  end type myty
 end module md
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
  !REF: /md
  use :: md
- !DEF: /mm/c CommonBlockDetails
- !DEF: /mm/x (InCommonBlock) ObjectEntity REAL(4)
- !DEF: /mm/y (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /MM/c CommonBlockDetails
+ !DEF: /MM/x (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /MM/y (InCommonBlock) ObjectEntity REAL(4)
  common /c/x, y
- !REF: /mm/x
- !REF: /mm/y
+ !REF: /MM/x
+ !REF: /MM/y
  real x, y
- !DEF: /mm/myty Use
- !DEF: /mm/t ObjectEntity TYPE(myty)
+ !DEF: /MM/myty Use
+ !DEF: /MM/t ObjectEntity TYPE(myty)
  type(myty) :: t
- !DEF: /mm/b ObjectEntity INTEGER(4)
+ !DEF: /MM/b ObjectEntity INTEGER(4)
  integer b(10)
- !REF: /mm/t
+ !REF: /MM/t
  !REF: /md/myty/a
  t%a = 3.14
- !REF: /mm/t
+ !REF: /MM/t
  !REF: /md/myty/b
  t%b = 1
- !REF: /mm/b
+ !REF: /MM/b
  b = 2
- !DEF: /mm/a (Implicit) ObjectEntity REAL(4)
+ !DEF: /MM/a (Implicit) ObjectEntity REAL(4)
  a = 1.0
- !DEF: /mm/c (Implicit) ObjectEntity REAL(4)
+ !DEF: /MM/c (Implicit) ObjectEntity REAL(4)
  c = 2.0
 !$omp parallel do  private(a,t,/c/) shared(c)
- !DEF: /mm/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /MM/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  do i=1,10
-  !DEF: /mm/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
-  !DEF: /mm/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
-  !REF: /mm/OtherConstruct1/i
+  !DEF: /MM/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
+  !REF: /MM/OtherConstruct1/i
   a = a+b(i)
-  !DEF: /mm/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
+  !DEF: /MM/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
   !REF: /md/myty/a
-  !REF: /mm/OtherConstruct1/i
+  !REF: /MM/OtherConstruct1/i
   t%a = i
-  !DEF: /mm/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   y = 0.
-  !DEF: /mm/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
-  !REF: /mm/OtherConstruct1/a
-  !REF: /mm/OtherConstruct1/i
-  !REF: /mm/OtherConstruct1/y
+  !DEF: /MM/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !REF: /MM/OtherConstruct1/a
+  !REF: /MM/OtherConstruct1/i
+  !REF: /MM/OtherConstruct1/y
   x = a+i+y
-  !DEF: /mm/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
   c = 3.0
  end do
 end program
diff --git a/flang/test/Semantics/OpenMP/symbol05.f90 b/flang/test/Semantics/OpenMP/symbol05.f90
index fe01f15d20aa3..03e2c0ba3e0cc 100644
--- a/flang/test/Semantics/OpenMP/symbol05.f90
+++ b/flang/test/Semantics/OpenMP/symbol05.f90
@@ -31,10 +31,10 @@ subroutine foo
     end block
   end subroutine foo
 end module mm
-!DEF: /tt MainProgram
+!DEF: /TT MainProgram
 program tt
   !REF: /mm
   use :: mm
-  !DEF: /tt/foo (Subroutine) Use
+  !DEF: /TT/foo (Subroutine) Use
   call foo
 end program tt
diff --git a/flang/test/Semantics/OpenMP/symbol07.f90 b/flang/test/Semantics/OpenMP/symbol07.f90
index 86b7305411347..f5eb2127819bd 100644
--- a/flang/test/Semantics/OpenMP/symbol07.f90
+++ b/flang/test/Semantics/OpenMP/symbol07.f90
@@ -30,7 +30,7 @@ subroutine function_call_in_region
   !REF: /function_call_in_region/b
   print *, a, b
 end subroutine function_call_in_region
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !REF: /function_call_in_region
   call function_call_in_region
diff --git a/flang/test/Semantics/OpenMP/symbol09.f90 b/flang/test/Semantics/OpenMP/symbol09.f90
index 86b7305411347..f5eb2127819bd 100644
--- a/flang/test/Semantics/OpenMP/symbol09.f90
+++ b/flang/test/Semantics/OpenMP/symbol09.f90
@@ -30,7 +30,7 @@ subroutine function_call_in_region
   !REF: /function_call_in_region/b
   print *, a, b
 end subroutine function_call_in_region
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !REF: /function_call_in_region
   call function_call_in_region
diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index 95b7fb418367d..e3c2e0bf53f92 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -9,6 +9,6 @@ program main
 
 ! RUN: %flang_fc1 -fget-symbols-sources %s 2>&1 | FileCheck %s
 ! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
-! CHECK:main:{{.*}}getsymbols03-a.f90, 4, 9-13
+! CHECK:MAIN:{{.*}}getsymbols03-a.f90, 4, 9-13
 ! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
 ! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14
diff --git a/flang/test/Semantics/modproc01.f90 b/flang/test/Semantics/modproc01.f90
index 5f45362e95093..e565ddcfbe0b1 100644
--- a/flang/test/Semantics/modproc01.f90
+++ b/flang/test/Semantics/modproc01.f90
@@ -125,7 +125,7 @@ program test
   x = mf(3, "abc", pdt1(1,3)())
 !  call ms(mf)
 end program
-!CHECK:  MainProgram scope: test size=88 alignment=8
+!CHECK:  MainProgram scope: TEST size=88 alignment=8
 !CHECK:    mf, MODULE (Function): Use from mf in m
 !CHECK:    pdt1: Use from pdt1 in m
 !CHECK:    pdt2: Use from pdt2 in m
diff --git a/flang/test/Semantics/multi-programs04.f90 b/flang/test/Semantics/multi-programs04.f90
index 54b0235aa78f0..e69ac7325278e 100644
--- a/flang/test/Semantics/multi-programs04.f90
+++ b/flang/test/Semantics/multi-programs04.f90
@@ -4,6 +4,6 @@
 program m
 end
 !ERROR: A source file cannot contain more than one main program
-!ERROR: 'm' is already declared in this scoping unit
+!ERROR: 'M' is already declared in this scoping unit
 program m
 end
diff --git a/flang/test/Semantics/procinterface01.f90 b/flang/test/Semantics/procinterface01.f90
index 73040b0987bd0..7542bb8bb6fa4 100644
--- a/flang/test/Semantics/procinterface01.f90
+++ b/flang/test/Semantics/procinterface01.f90
@@ -159,35 +159,35 @@ end function logical
  tan = "?"
 end function tan
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /module1
  use :: module1
- !DEF: /main/derived1 Use
- !DEF: /main/instance ObjectEntity TYPE(derived1)
+ !DEF: /MAIN/derived1 Use
+ !DEF: /MAIN/instance ObjectEntity TYPE(derived1)
  type(derived1) :: instance
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p1
  if (instance%p1(1.)/=2.) print *, "p1 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p2
  if (instance%p2(1.)/=2.) print *, "p2 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p3
  if (.not.instance%p3(1.)) print *, "p3 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p4
  if (.not.instance%p4(1.)) print *, "p4 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p5
  if (instance%p5(1.)/=(5.,6.)) print *, "p5 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p6
  if (instance%p6(1.)/=2.) print *, "p6 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p7
  if (instance%p7(0.)/=1.) print *, "p7 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p8
  if (instance%p8(1.)/="a") print *, "p8 failed"
 end program main
diff --git a/flang/test/Semantics/resolve125.f90 b/flang/test/Semantics/resolve125.f90
index e040c006ec179..620c7d65578cd 100644
--- a/flang/test/Semantics/resolve125.f90
+++ b/flang/test/Semantics/resolve125.f90
@@ -43,7 +43,7 @@ subroutine reset
   end subroutine reset
 end module m2
 
-!CHECK: MainProgram scope: main
+!CHECK: MainProgram scope: MAIN
 !CHECK:       i: Use from i in m2
 !CHECK:       i2: Use from i2 in m2
 !CHECK:       init (Subroutine): Use from init in m2
@@ -61,4 +61,4 @@ program main
   else
     print *, "fail"
   end if
-end program main
\ No newline at end of file
+end program main
diff --git a/flang/test/Semantics/symbol03.f90 b/flang/test/Semantics/symbol03.f90
index a6b4b0bd15937..f50a0e91fd289 100644
--- a/flang/test/Semantics/symbol03.f90
+++ b/flang/test/Semantics/symbol03.f90
@@ -1,23 +1,23 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Test host association in internal subroutine of main program.
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/x ObjectEntity INTEGER(4)
+ !DEF: /MAIN/x ObjectEntity INTEGER(4)
  integer x
- !DEF: /main/s (Subroutine) Subprogram
+ !DEF: /MAIN/s (Subroutine) Subprogram
  call s
 contains
- !REF: /main/s
+ !REF: /MAIN/s
  subroutine s
-  !DEF: /main/s/y (Implicit) ObjectEntity REAL(4)
-  !DEF: /main/s/x HostAssoc INTEGER(4)
+  !DEF: /MAIN/s/y (Implicit) ObjectEntity REAL(4)
+  !DEF: /MAIN/s/x HostAssoc INTEGER(4)
   y = x
  contains
-  !DEF: /main/s/s2 (Subroutine) Subprogram
+  !DEF: /MAIN/s/s2 (Subroutine) Subprogram
   subroutine s2
-   !DEF: /main/s/s2/z (Implicit) ObjectEntity REAL(4)
-   !DEF: /main/s/s2/x HostAssoc INTEGER(4)
+   !DEF: /MAIN/s/s2/z (Implicit) ObjectEntity REAL(4)
+   !DEF: /MAIN/s/s2/x HostAssoc INTEGER(4)
    z = x
   end subroutine
  end subroutine
diff --git a/flang/test/Semantics/symbol06.f90 b/flang/test/Semantics/symbol06.f90
index bbd6d4d071c89..984ebcc1c651c 100644
--- a/flang/test/Semantics/symbol06.f90
+++ b/flang/test/Semantics/symbol06.f90
@@ -1,56 +1,56 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/t1 DerivedType
+ !DEF: /MAIN/t1 DerivedType
  type :: t1
-  !DEF: /main/t1/a1 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t1/a1 ObjectEntity INTEGER(4)
   integer :: a1
  end type
- !REF: /main/t1
- !DEF: /main/t2 DerivedType
+ !REF: /MAIN/t1
+ !DEF: /MAIN/t2 DerivedType
  type, extends(t1) :: t2
-  !DEF: /main/t2/a2 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t2/a2 ObjectEntity INTEGER(4)
   integer :: a2
  end type
- !REF: /main/t2
- !DEF: /main/t3 DerivedType
+ !REF: /MAIN/t2
+ !DEF: /MAIN/t3 DerivedType
  type, extends(t2) :: t3
-  !DEF: /main/t3/a3 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t3/a3 ObjectEntity INTEGER(4)
   integer :: a3
  end type
- !REF: /main/t3
- !DEF: /main/x3 ObjectEntity TYPE(t3)
+ !REF: /MAIN/t3
+ !DEF: /MAIN/x3 ObjectEntity TYPE(t3)
  type(t3) :: x3
- !DEF: /main/i ObjectEntity INTEGER(4)
+ !DEF: /MAIN/i ObjectEntity INTEGER(4)
  integer i
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t2/a2
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t2/a2
  i = x3%a2
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t1/a1
  i = x3%a1
- !REF: /main/i
- !REF: /main/x3
- !DEF: /main/t3/t2 (ParentComp) ObjectEntity TYPE(t2)
- !REF: /main/t2/a2
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !DEF: /MAIN/t3/t2 (ParentComp) ObjectEntity TYPE(t2)
+ !REF: /MAIN/t2/a2
  i = x3%t2%a2
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t3/t2
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t3/t2
+ !REF: /MAIN/t1/a1
  i = x3%t2%a1
- !REF: /main/i
- !REF: /main/x3
- !DEF: /main/t2/t1 (ParentComp) ObjectEntity TYPE(t1)
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !DEF: /MAIN/t2/t1 (ParentComp) ObjectEntity TYPE(t1)
+ !REF: /MAIN/t1/a1
  i = x3%t1%a1
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t3/t2
- !REF: /main/t2/t1
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t3/t2
+ !REF: /MAIN/t2/t1
+ !REF: /MAIN/t1/a1
  i = x3%t2%t1%a1
 end program
 
diff --git a/flang/test/Semantics/symbol07.f90 b/flang/test/Semantics/symbol07.f90
index f3cc934e51b16..b3e92f1465fa6 100644
--- a/flang/test/Semantics/symbol07.f90
+++ b/flang/test/Semantics/symbol07.f90
@@ -1,40 +1,40 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  implicit complex(z)
- !DEF: /main/t DerivedType
+ !DEF: /MAIN/t DerivedType
  type :: t
-  !DEF: /main/t/re ObjectEntity REAL(4)
+  !DEF: /MAIN/t/re ObjectEntity REAL(4)
   real :: re
-  !DEF: /main/t/im ObjectEntity REAL(4)
+  !DEF: /MAIN/t/im ObjectEntity REAL(4)
   real :: im
  end type
- !DEF: /main/z1 ObjectEntity COMPLEX(4)
+ !DEF: /MAIN/z1 ObjectEntity COMPLEX(4)
  complex z1
- !REF: /main/t
- !DEF: /main/w ObjectEntity TYPE(t)
+ !REF: /MAIN/t
+ !DEF: /MAIN/w ObjectEntity TYPE(t)
  type(t) :: w
- !DEF: /main/x ObjectEntity REAL(4)
- !DEF: /main/y ObjectEntity REAL(4)
+ !DEF: /MAIN/x ObjectEntity REAL(4)
+ !DEF: /MAIN/y ObjectEntity REAL(4)
  real x, y
- !REF: /main/x
- !REF: /main/z1
+ !REF: /MAIN/x
+ !REF: /MAIN/z1
  x = z1%re
- !REF: /main/y
- !REF: /main/z1
+ !REF: /MAIN/y
+ !REF: /MAIN/z1
  y = z1%im
- !DEF: /main/z2 (Implicit) ObjectEntity COMPLEX(4)
- !REF: /main/x
+ !DEF: /MAIN/z2 (Implicit) ObjectEntity COMPLEX(4)
+ !REF: /MAIN/x
  z2%re = x
- !REF: /main/z2
- !REF: /main/y
+ !REF: /MAIN/z2
+ !REF: /MAIN/y
  z2%im = y
- !REF: /main/x
- !REF: /main/w
- !REF: /main/t/re
+ !REF: /MAIN/x
+ !REF: /MAIN/w
+ !REF: /MAIN/t/re
  x = w%re
- !REF: /main/y
- !REF: /main/w
- !REF: /main/t/im
+ !REF: /MAIN/y
+ !REF: /MAIN/w
+ !REF: /MAIN/t/im
  y = w%im
 end program
diff --git a/flang/test/Semantics/symbol08.f90 b/flang/test/Semantics/symbol08.f90
index 61dab798955c5..e2e9959d0a591 100644
--- a/flang/test/Semantics/symbol08.f90
+++ b/flang/test/Semantics/symbol08.f90
@@ -1,15 +1,15 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/x POINTER ObjectEntity REAL(4)
+ !DEF: /MAIN/x POINTER ObjectEntity REAL(4)
  pointer :: x
- !REF: /main/x
+ !REF: /MAIN/x
  real x
- !DEF: /main/y EXTERNAL, POINTER (Function) ProcEntity REAL(4)
+ !DEF: /MAIN/y EXTERNAL, POINTER (Function) ProcEntity REAL(4)
  pointer :: y
- !REF: /main/y
+ !REF: /MAIN/y
  procedure (real) :: y
- !DEF: /main/z (Implicit) ObjectEntity REAL(4)
- !REF: /main/y
+ !DEF: /MAIN/z (Implicit) ObjectEntity REAL(4)
+ !REF: /MAIN/y
  z = y()
 end program
diff --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index df10942e6af2d..5cc1cf58b4f21 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -249,15 +249,15 @@ subroutine ext2
 !DEF: /ext3 (Subroutine) Subprogram
 subroutine ext3
 end subroutine
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /m
  use :: m
- !DEF: /main/pdt1 Use
- !DEF: /main/pdt1y ObjectEntity TYPE(pdt1(k=2_4))
+ !DEF: /MAIN/pdt1 Use
+ !DEF: /MAIN/pdt1y ObjectEntity TYPE(pdt1(k=2_4))
  type(pdt1(2)) :: pdt1y
- !DEF: /main/pdt2 Use
- !DEF: /main/pdt2y ObjectEntity TYPE(pdt2(k=2_4))
+ !DEF: /MAIN/pdt2 Use
+ !DEF: /MAIN/pdt2y ObjectEntity TYPE(pdt2(k=2_4))
  type(pdt2(2)) :: pdt2y
  print *, "compiled"
 end program
diff --git a/flang/test/Semantics/symbol16.f90 b/flang/test/Semantics/symbol16.f90
index 7a46092c36b53..c29831a5dfa90 100644
--- a/flang/test/Semantics/symbol16.f90
+++ b/flang/test/Semantics/symbol16.f90
@@ -1,18 +1,18 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Statement functions
 
-!DEF: /p1 MainProgram
+!DEF: /P1 MainProgram
 program p1
- !DEF: /p1/f (Function, StmtFunction) Subprogram INTEGER(4)
- !DEF: /p1/i ObjectEntity INTEGER(4)
- !DEF: /p1/j ObjectEntity INTEGER(4)
+ !DEF: /P1/f (Function, StmtFunction) Subprogram INTEGER(4)
+ !DEF: /P1/i ObjectEntity INTEGER(4)
+ !DEF: /P1/j ObjectEntity INTEGER(4)
  integer f, i, j
- !REF: /p1/f
- !REF: /p1/i
- !DEF: /p1/f/i ObjectEntity INTEGER(4)
+ !REF: /P1/f
+ !REF: /P1/i
+ !DEF: /P1/f/i ObjectEntity INTEGER(4)
  f(i) = i + 1
- !REF: /p1/j
- !REF: /p1/f
+ !REF: /P1/j
+ !REF: /P1/f
  j = f(2)
 end program
 
diff --git a/flang/test/Semantics/symbol17.f90 b/flang/test/Semantics/symbol17.f90
index 434f124509a32..51f3afb19692d 100644
--- a/flang/test/Semantics/symbol17.f90
+++ b/flang/test/Semantics/symbol17.f90
@@ -1,44 +1,44 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Forward references to derived types (non-error cases)
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/t1 DerivedType
+ !DEF: /MAIN/t1 DerivedType
  type :: t1
-  !DEF: /main/t2 DerivedType
-  !DEF: /main/t1/t1a ALLOCATABLE ObjectEntity TYPE(t2)
+  !DEF: /MAIN/t2 DerivedType
+  !DEF: /MAIN/t1/t1a ALLOCATABLE ObjectEntity TYPE(t2)
   type(t2), allocatable :: t1a
-  !REF: /main/t2
-  !DEF: /main/t1/t1p POINTER ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t1/t1p POINTER ObjectEntity TYPE(t2)
   type(t2), pointer :: t1p
  end type
- !REF: /main/t2
+ !REF: /MAIN/t2
  type :: t2
-  !REF: /main/t2
-  !DEF: /main/t2/t2a ALLOCATABLE ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t2/t2a ALLOCATABLE ObjectEntity TYPE(t2)
   type(t2), allocatable :: t2a
-  !REF: /main/t2
-  !DEF: /main/t2/t2p POINTER ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t2/t2p POINTER ObjectEntity TYPE(t2)
   type(t2), pointer :: t2p
  end type
- !REF: /main/t1
- !DEF: /main/t1x TARGET ObjectEntity TYPE(t1)
+ !REF: /MAIN/t1
+ !DEF: /MAIN/t1x TARGET ObjectEntity TYPE(t1)
  type(t1), target :: t1x
- !REF: /main/t1x
- !REF: /main/t1/t1a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
  allocate(t1x%t1a)
- !REF: /main/t1x
- !REF: /main/t1/t1p
- !REF: /main/t1/t1a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1p
+ !REF: /MAIN/t1/t1a
  t1x%t1p => t1x%t1a
- !REF: /main/t1x
- !REF: /main/t1/t1a
- !REF: /main/t2/t2a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
+ !REF: /MAIN/t2/t2a
  allocate(t1x%t1a%t2a)
- !REF: /main/t1x
- !REF: /main/t1/t1a
- !REF: /main/t2/t2p
- !REF: /main/t2/t2a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
+ !REF: /MAIN/t2/t2p
+ !REF: /MAIN/t2/t2a
  t1x%t1a%t2p => t1x%t1a%t2a
 end program
 !DEF: /f1/fwd DerivedType
diff --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
index a37792bce21d7..bfe25ff4df25f 100644
--- a/flang/test/Semantics/symbol18.f90
+++ b/flang/test/Semantics/symbol18.f90
@@ -2,21 +2,21 @@
 
 ! Intrinsic function in type declaration statement: type is ignored
 
-!DEF: /p1 MainProgram
+!DEF: /P1 MainProgram
 program p1
- !DEF: /p1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
+ !DEF: /P1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
  integer cos
- !DEF: /p1/y (Implicit) ObjectEntity REAL(4)
- !REF: /p1/cos
- !DEF: /p1/x (Implicit) ObjectEntity REAL(4)
+ !DEF: /P1/y (Implicit) ObjectEntity REAL(4)
+ !REF: /P1/cos
+ !DEF: /P1/x (Implicit) ObjectEntity REAL(4)
  y = cos(x)
- !REF: /p1/y
- !DEF: /p1/sin ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
- !REF: /p1/x
+ !REF: /P1/y
+ !DEF: /P1/sin ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+ !REF: /P1/x
  y = sin(x)
- !REF: /p1/y
+ !REF: /P1/y
  !DEF: /f EXTERNAL (Function, Implicit) ProcEntity REAL(4)
- !REF: /p1/x
+ !REF: /P1/x
  y = f(x)
 end program
 
diff --git a/flang/test/Semantics/symbol20.f90 b/flang/test/Semantics/symbol20.f90
index 8c82776933321..20bd1bdbf2a7c 100644
--- a/flang/test/Semantics/symbol20.f90
+++ b/flang/test/Semantics/symbol20.f90
@@ -32,16 +32,16 @@ subroutine bar
   print *, "in bar"
  end subroutine
 end module
-!DEF: /demo MainProgram
+!DEF: /DEMO MainProgram
 program demo
  !REF: /m
  use :: m
- !DEF: /demo/bar (Subroutine) Use
- !DEF: /demo/p EXTERNAL, POINTER (Subroutine) ProcEntity
+ !DEF: /DEMO/bar (Subroutine) Use
+ !DEF: /DEMO/p EXTERNAL, POINTER (Subroutine) ProcEntity
  procedure(bar), pointer :: p
- !REF: /demo/p
- !DEF: /demo/foo (Function) Use
+ !REF: /DEMO/p
+ !DEF: /DEMO/foo (Function) Use
  p => foo()
- !REF: /demo/p
+ !REF: /DEMO/p
  call p
 end program
diff --git a/flang/test/Semantics/symbol25.f90 b/flang/test/Semantics/symbol25.f90
index ac3dd37ef92eb..a131bf899340f 100644
--- a/flang/test/Semantics/symbol25.f90
+++ b/flang/test/Semantics/symbol25.f90
@@ -38,23 +38,23 @@ subroutine inner1
   end subroutine inner1
  end subroutine outer
 end module m
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /m
  use :: m
  !REF: /m/specific1
  call generic
- !DEF: /main/inner2 (Subroutine) Subprogram
+ !DEF: /MAIN/inner2 (Subroutine) Subprogram
  call inner2
 contains
- !REF: /main/inner2
+ !REF: /MAIN/inner2
  subroutine inner2
-  !DEF: /main/inner2/generic (Subroutine) Generic
+  !DEF: /MAIN/inner2/generic (Subroutine) Generic
   interface generic
-   !DEF: /main/specific2 (Subroutine) Use
+   !DEF: /MAIN/specific2 (Subroutine) Use
    module procedure :: specific2
   end interface
-  !REF: /main/specific2
+  !REF: /MAIN/specific2
   call generic
  end subroutine inner2
 end program
diff --git a/flang/test/Semantics/symbol26.f90 b/flang/test/Semantics/symbol26.f90
index f5e95853ca099..e04cd32df57c4 100644
--- a/flang/test/Semantics/symbol26.f90
+++ b/flang/test/Semantics/symbol26.f90
@@ -8,16 +8,16 @@ module m
  !DEF: /m/j PUBLIC (Implicit, InNamelist) ObjectEntity INTEGER(4)
  namelist/a/j
 end module m
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/j (Implicit) ObjectEntity INTEGER(4)
+ !DEF: /MAIN/j (Implicit) ObjectEntity INTEGER(4)
  j = 1
 contains
- !DEF: /main/inner (Subroutine) Subprogram
+ !DEF: /MAIN/inner (Subroutine) Subprogram
  subroutine inner
   !REF: /m
   use :: m
-  !DEF: /main/inner/j (Implicit, InNamelist) Use INTEGER(4)
+  !DEF: /MAIN/inner/j (Implicit, InNamelist) Use INTEGER(4)
   j = 2
  end subroutine
 end program
diff --git a/flang/test/Transforms/DoConcurrent/basic_host.f90 b/flang/test/Transforms/DoConcurrent/basic_host.f90
index 12f63031cbaee..6f24b346e3fb9 100644
--- a/flang/test/Transforms/DoConcurrent/basic_host.f90
+++ b/flang/test/Transforms/DoConcurrent/basic_host.f90
@@ -5,7 +5,7 @@
 ! RUN: bbc -emit-hlfir -fopenmp -fdo-concurrent-to-openmp=host %s -o - \
 ! RUN:   | FileCheck %s
  
-! CHECK-LABEL: do_concurrent_basic
+! CHECK-LABEL: DO_CONCURRENT_BASIC
 program do_concurrent_basic
     ! CHECK: %[[ARR:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFEa"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 

>From 632cd4b1351cbb57efbdf3cb8fe9435211004a0a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:24:17 -0400
Subject: [PATCH 4/8] More test changes

---
 .../test/Semantics/OpenACC/acc-symbols01.f90  | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/flang/test/Semantics/OpenACC/acc-symbols01.f90 b/flang/test/Semantics/OpenACC/acc-symbols01.f90
index ea22f41197aa7..51a7a3a23e8ce 100644
--- a/flang/test/Semantics/OpenACC/acc-symbols01.f90
+++ b/flang/test/Semantics/OpenACC/acc-symbols01.f90
@@ -1,24 +1,24 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenacc
 
 !DEF: /MM MainProgram
-program mm
-  !DEF: /mm/x ObjectEntity REAL(4)
-  !DEF: /mm/y ObjectEntity REAL(4)
+program MM
+  !DEF: /MM/x ObjectEntity REAL(4)
+  !DEF: /MM/y ObjectEntity REAL(4)
   real x, y
-  !DEF: /mm/a ObjectEntity INTEGER(4)
-  !DEF: /mm/b ObjectEntity INTEGER(4)
-  !DEF: /mm/c ObjectEntity INTEGER(4)
-  !DEF: /mm/i ObjectEntity INTEGER(4)
+  !DEF: /MM/a ObjectEntity INTEGER(4)
+  !DEF: /MM/b ObjectEntity INTEGER(4)
+  !DEF: /MM/c ObjectEntity INTEGER(4)
+  !DEF: /MM/i ObjectEntity INTEGER(4)
   integer a(10), b(10), c(10), i
-  !REF: /mm/b
+  !REF: /MM/b
   b = 2
  !$acc parallel present(c) firstprivate(b) private(a)
  !$acc loop
-  !REF: /mm/i
+  !REF: /MM/i
   do i=1,10
-   !REF: /mm/a
-   !REF: /mm/i
-   !REF: /mm/b
+   !REF: /MM/a
+   !REF: /MM/i
+   !REF: /MM/b
    a(i) = b(i)
   end do
  !$acc end parallel

>From ce7b837b6554c62ad459729674ddb0cb84f49747 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:37:07 -0400
Subject: [PATCH 5/8] more tests

---
 flang/test/Semantics/OpenMP/do-schedule03.f90      |  4 ++--
 flang/test/Semantics/OpenMP/do01-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do04-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do05-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do06-positivecases.f90 |  4 ++--
 flang/test/Semantics/OpenMP/do11.f90               | 14 +++++++-------
 6 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/flang/test/Semantics/OpenMP/do-schedule03.f90 b/flang/test/Semantics/OpenMP/do-schedule03.f90
index 375f3920b57ad..05602ca57e4a9 100644
--- a/flang/test/Semantics/OpenMP/do-schedule03.f90
+++ b/flang/test/Semantics/OpenMP/do-schedule03.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - xx.
   !DEF: /OMPDOSCHEDULE MainProgram
-program ompdoschedule
+program OMPDOSCHEDULE
   !DEF: /OMPDOSCHEDULE/a ObjectEntity REAL(4)
   !DEF: /OMPDOSCHEDULE/y ObjectEntity REAL(4)
   !DEF: /OMPDOSCHEDULE/z ObjectEntity REAL(4)
@@ -25,4 +25,4 @@ program ompdoschedule
     y(i) = z(i-1) + a(i)
   end do
   !$omp end do
-end program ompdoschedule
+end program OMPDOSCHEDULE
diff --git a/flang/test/Semantics/OpenMP/do01-positivecase.f90 b/flang/test/Semantics/OpenMP/do01-positivecase.f90
index a4a28f6530d7b..50a6870f43896 100644
--- a/flang/test/Semantics/OpenMP/do01-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do01-positivecase.f90
@@ -5,7 +5,7 @@
 ! A positive case
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   integer i
 
@@ -16,4 +16,4 @@ program omp_do
   end do
   !$omp end do
 
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do04-positivecase.f90 b/flang/test/Semantics/OpenMP/do04-positivecase.f90
index 0b0c661687dc2..51b69fce3c7cc 100644
--- a/flang/test/Semantics/OpenMP/do04-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do04-positivecase.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop Constructs
 
 !DEF: /OMP_DO1 MainProgram
-program omp_do1
+program OMP_DO1
   !DEF: /OMP_DO1/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO1/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
@@ -19,4 +19,4 @@ program omp_do1
     end do
   end do
   !$omp end do
-end program omp_do1
+end program OMP_DO1
diff --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index 07b12dd1d4d23..d4eb1fd6bc3da 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -4,7 +4,7 @@
 ! A positive case
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/n ObjectEntity INTEGER(4)
   integer i,n
@@ -71,4 +71,4 @@ program omp_do
     !REF: /OMP_DO/OtherConstruct7/i
     if(i < 5) cycle
   end do
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do06-positivecases.f90 b/flang/test/Semantics/OpenMP/do06-positivecases.f90
index 4d45800298a29..dfb1d999bbc53 100644
--- a/flang/test/Semantics/OpenMP/do06-positivecases.f90
+++ b/flang/test/Semantics/OpenMP/do06-positivecases.f90
@@ -6,7 +6,7 @@
 
 ! A positive case
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
@@ -20,4 +20,4 @@ program omp_do
       !$omp end ordered
     end do
   !$omp end do
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do11.f90 b/flang/test/Semantics/OpenMP/do11.f90
index 886a9f7ae6081..472048d684276 100644
--- a/flang/test/Semantics/OpenMP/do11.f90
+++ b/flang/test/Semantics/OpenMP/do11.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop Constructs
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
@@ -19,17 +19,17 @@ program omp_do
     end do
   end do
   !$omp end do
-end program omp_do
+end program OMP_DO
 
-!DEF: /OMP_DO2 (Subroutine)Subprogram
+!DEF: /omp_do2 (Subroutine)Subprogram
 subroutine omp_do2
-  !DEF: /OMP_DO2/i ObjectEntity INTEGER(4)
-  !DEF: /OMP_DO2/k ObjectEntity INTEGER(4)
+  !DEF: /omp_do2/i ObjectEntity INTEGER(4)
+  !DEF: /omp_do2/k ObjectEntity INTEGER(4)
   integer :: i = 0, k
   !$omp do
-  !DEF: /OMP_DO2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /omp_do2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /OMP_DO2/OtherConstruct1/i
+    !REF: /omp_do2/OtherConstruct1/i
     print *, "it", i
   end do
   !$omp end do

>From 3e3d5d489bd2d299cb14bdeca728856f8bd59f18 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 23:03:25 -0400
Subject: [PATCH 6/8] More test changes

---
 flang/test/Semantics/OpenMP/declare-target03.f90 | 4 ++--
 flang/test/Semantics/OpenMP/do12.f90             | 2 +-
 flang/test/Semantics/OpenMP/do14.f90             | 4 ++--
 flang/test/Semantics/OpenMP/do17.f90             | 4 ++--
 flang/test/Semantics/OpenMP/reduction08.f90      | 4 ++--
 flang/test/Semantics/OpenMP/reduction09.f90      | 4 ++--
 flang/test/Semantics/OpenMP/symbol01.f90         | 2 +-
 flang/test/Semantics/OpenMP/symbol05.f90         | 4 ++--
 flang/test/Semantics/OpenMP/symbol07.f90         | 4 ++--
 flang/test/Semantics/OpenMP/symbol09.f90         | 4 ++--
 flang/test/Semantics/OpenMP/threadprivate03.f90  | 4 ++--
 flang/test/Semantics/long-name.f90               | 2 +-
 flang/test/Semantics/pointer01.f90               | 1 -
 flang/test/Semantics/procinterface01.f90         | 4 ++--
 flang/test/Semantics/resolve05.f90               | 1 -
 flang/test/Semantics/symbol03.f90                | 2 +-
 flang/test/Semantics/symbol06.f90                | 2 +-
 flang/test/Semantics/symbol07.f90                | 2 +-
 flang/test/Semantics/symbol08.f90                | 2 +-
 flang/test/Semantics/symbol15.f90                | 2 +-
 flang/test/Semantics/symbol16.f90                | 2 +-
 flang/test/Semantics/symbol17.f90                | 2 +-
 flang/test/Semantics/symbol18.f90                | 2 +-
 flang/test/Semantics/symbol20.f90                | 2 +-
 flang/test/Semantics/symbol25.f90                | 2 +-
 flang/test/Semantics/symbol26.f90                | 2 +-
 26 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index 64a299d78224a..b3d6551ac85de 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -16,7 +16,7 @@ program main
   !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
-  !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
+  ! This is now allowed: "main" is implicitly declared symbol separate
+  ! from the main program symbol
   !$omp declare target (main)
 end
diff --git a/flang/test/Semantics/OpenMP/do12.f90 b/flang/test/Semantics/OpenMP/do12.f90
index f36a230426197..06055b7572a60 100644
--- a/flang/test/Semantics/OpenMP/do12.f90
+++ b/flang/test/Semantics/OpenMP/do12.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /OMP_CYCLE MainProgram
-program omp_cycle
+program OMP_CYCLE
   !$omp do  collapse(1)
   !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
diff --git a/flang/test/Semantics/OpenMP/do14.f90 b/flang/test/Semantics/OpenMP/do14.f90
index b508d32d90201..e17647394fff7 100644
--- a/flang/test/Semantics/OpenMP/do14.f90
+++ b/flang/test/Semantics/OpenMP/do14.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /OMP_CYCLE MainProgram
-program omp_cycle
+program OMP_CYCLE
   !$omp do  collapse(1)
   !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
@@ -88,4 +88,4 @@ program omp_cycle
     end do foo1
   end do foo
   !$omp end do
-end program omp_cycle
+end program OMP_CYCLE
diff --git a/flang/test/Semantics/OpenMP/do17.f90 b/flang/test/Semantics/OpenMP/do17.f90
index 0a4cf46d2d6af..cac11f215f074 100644
--- a/flang/test/Semantics/OpenMP/do17.f90
+++ b/flang/test/Semantics/OpenMP/do17.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /TEST MainProgram
-program test
+program TEST
  !DEF: /TEST/i ObjectEntity INTEGER(4)
  !DEF: /TEST/j ObjectEntity INTEGER(4)
  !DEF: /TEST/k ObjectEntity INTEGER(4)
@@ -54,4 +54,4 @@ program test
   end do foo1
  end do foo
 !$omp end do
-end program test
+end program TEST
diff --git a/flang/test/Semantics/OpenMP/reduction08.f90 b/flang/test/Semantics/OpenMP/reduction08.f90
index 1cf2838fe765a..b4a81e644c1e7 100644
--- a/flang/test/Semantics/OpenMP/reduction08.f90
+++ b/flang/test/Semantics/OpenMP/reduction08.f90
@@ -3,7 +3,7 @@
 ! 2.15.3.6 Reduction Clause Positive cases
 
 !DEF: /OMP_REDUCTION MainProgram
-program omp_reduction
+program OMP_REDUCTION
   !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
   !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
@@ -60,4 +60,4 @@ program omp_reduction
   end do
   !$omp end parallel do
 
-end program omp_reduction
+end program OMP_REDUCTION
diff --git a/flang/test/Semantics/OpenMP/reduction09.f90 b/flang/test/Semantics/OpenMP/reduction09.f90
index 4b1ca116d675b..ca60805e8c416 100644
--- a/flang/test/Semantics/OpenMP/reduction09.f90
+++ b/flang/test/Semantics/OpenMP/reduction09.f90
@@ -2,7 +2,7 @@
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases.
 !DEF: /OMP_REDUCTION MainProgram
-program omp_reduction
+program OMP_REDUCTION
   !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
   !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
@@ -73,4 +73,4 @@ program omp_reduction
     k = k+1
   end do
   !$omp end do
-end program omp_reduction
+end program OMP_REDUCTION
diff --git a/flang/test/Semantics/OpenMP/symbol01.f90 b/flang/test/Semantics/OpenMP/symbol01.f90
index c1f16d61ada1e..74fb420cc517e 100644
--- a/flang/test/Semantics/OpenMP/symbol01.f90
+++ b/flang/test/Semantics/OpenMP/symbol01.f90
@@ -17,7 +17,7 @@ module md
  end type myty
 end module md
 !DEF: /MM MainProgram
-program mm
+program MM
  !REF: /md
  use :: md
  !DEF: /MM/c CommonBlockDetails
diff --git a/flang/test/Semantics/OpenMP/symbol05.f90 b/flang/test/Semantics/OpenMP/symbol05.f90
index 03e2c0ba3e0cc..4f3d1926013dc 100644
--- a/flang/test/Semantics/OpenMP/symbol05.f90
+++ b/flang/test/Semantics/OpenMP/symbol05.f90
@@ -32,9 +32,9 @@ subroutine foo
   end subroutine foo
 end module mm
 !DEF: /TT MainProgram
-program tt
+program TT
   !REF: /mm
   use :: mm
   !DEF: /TT/foo (Subroutine) Use
   call foo
-end program tt
+end program TT
diff --git a/flang/test/Semantics/OpenMP/symbol07.f90 b/flang/test/Semantics/OpenMP/symbol07.f90
index f5eb2127819bd..1b0c25b7a04b0 100644
--- a/flang/test/Semantics/OpenMP/symbol07.f90
+++ b/flang/test/Semantics/OpenMP/symbol07.f90
@@ -31,7 +31,7 @@ subroutine function_call_in_region
   print *, a, b
 end subroutine function_call_in_region
 !DEF: /MM MainProgram
-program mm
+program MM
   !REF: /function_call_in_region
   call function_call_in_region
-end program mm
+end program MM
diff --git a/flang/test/Semantics/OpenMP/symbol09.f90 b/flang/test/Semantics/OpenMP/symbol09.f90
index f5eb2127819bd..1b0c25b7a04b0 100644
--- a/flang/test/Semantics/OpenMP/symbol09.f90
+++ b/flang/test/Semantics/OpenMP/symbol09.f90
@@ -31,7 +31,7 @@ subroutine function_call_in_region
   print *, a, b
 end subroutine function_call_in_region
 !DEF: /MM MainProgram
-program mm
+program MM
   !REF: /function_call_in_region
   call function_call_in_region
-end program mm
+end program MM
diff --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index 81e26ee327a9d..9f222773ad432 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -13,8 +13,8 @@ program main
   !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
-  !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
+  ! This is now allowed, since "main" is implicitly declared symbol,
+  ! separate from the main program symbol.
   !$omp threadprivate(main)
 
   !ERROR: The entity with PARAMETER attribute cannot be in a THREADPRIVATE directive
diff --git a/flang/test/Semantics/long-name.f90 b/flang/test/Semantics/long-name.f90
index 44899b13edd5a..d5a795113e204 100644
--- a/flang/test/Semantics/long-name.f90
+++ b/flang/test/Semantics/long-name.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror -pedantic
 
-!PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
+!PORTABILITY: AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEEFFFFFFFFFFGGG1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
 program aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1
 
   !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
diff --git a/flang/test/Semantics/pointer01.f90 b/flang/test/Semantics/pointer01.f90
index eaa2426dd77e3..79d6016a6af46 100644
--- a/flang/test/Semantics/pointer01.f90
+++ b/flang/test/Semantics/pointer01.f90
@@ -7,7 +7,6 @@ subroutine msubr
 end module
 program main
   use m
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   pointer main
   !ERROR: Cannot change POINTER attribute on use-associated 'mobj'
   pointer mobj
diff --git a/flang/test/Semantics/procinterface01.f90 b/flang/test/Semantics/procinterface01.f90
index 7542bb8bb6fa4..70f4a889d6809 100644
--- a/flang/test/Semantics/procinterface01.f90
+++ b/flang/test/Semantics/procinterface01.f90
@@ -160,7 +160,7 @@ end function logical
 end function tan
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /module1
  use :: module1
  !DEF: /MAIN/derived1 Use
@@ -190,4 +190,4 @@ program main
  !REF: /MAIN/instance
  !REF: /module1/derived1/p8
  if (instance%p8(1.)/="a") print *, "p8 failed"
-end program main
+end program MAIN
diff --git a/flang/test/Semantics/resolve05.f90 b/flang/test/Semantics/resolve05.f90
index 0c9877af9b4e2..7b142d2ebd613 100644
--- a/flang/test/Semantics/resolve05.f90
+++ b/flang/test/Semantics/resolve05.f90
@@ -1,6 +1,5 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 program p
-  !PORTABILITY: Name 'p' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   integer :: p
 end
 module m
diff --git a/flang/test/Semantics/symbol03.f90 b/flang/test/Semantics/symbol03.f90
index f50a0e91fd289..62472495d9736 100644
--- a/flang/test/Semantics/symbol03.f90
+++ b/flang/test/Semantics/symbol03.f90
@@ -2,7 +2,7 @@
 ! Test host association in internal subroutine of main program.
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/x ObjectEntity INTEGER(4)
  integer x
  !DEF: /MAIN/s (Subroutine) Subprogram
diff --git a/flang/test/Semantics/symbol06.f90 b/flang/test/Semantics/symbol06.f90
index 984ebcc1c651c..b45edabcd5318 100644
--- a/flang/test/Semantics/symbol06.f90
+++ b/flang/test/Semantics/symbol06.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/t1 DerivedType
  type :: t1
   !DEF: /MAIN/t1/a1 ObjectEntity INTEGER(4)
diff --git a/flang/test/Semantics/symbol07.f90 b/flang/test/Semantics/symbol07.f90
index b3e92f1465fa6..e1d8257b9e190 100644
--- a/flang/test/Semantics/symbol07.f90
+++ b/flang/test/Semantics/symbol07.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  implicit complex(z)
  !DEF: /MAIN/t DerivedType
  type :: t
diff --git a/flang/test/Semantics/symbol08.f90 b/flang/test/Semantics/symbol08.f90
index e2e9959d0a591..933ff6d0c2ba8 100644
--- a/flang/test/Semantics/symbol08.f90
+++ b/flang/test/Semantics/symbol08.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/x POINTER ObjectEntity REAL(4)
  pointer :: x
  !REF: /MAIN/x
diff --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index 5cc1cf58b4f21..79a45491306ef 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -250,7 +250,7 @@ subroutine ext2
 subroutine ext3
 end subroutine
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /m
  use :: m
  !DEF: /MAIN/pdt1 Use
diff --git a/flang/test/Semantics/symbol16.f90 b/flang/test/Semantics/symbol16.f90
index c29831a5dfa90..547c4624d4cdb 100644
--- a/flang/test/Semantics/symbol16.f90
+++ b/flang/test/Semantics/symbol16.f90
@@ -2,7 +2,7 @@
 ! Statement functions
 
 !DEF: /P1 MainProgram
-program p1
+program P1
  !DEF: /P1/f (Function, StmtFunction) Subprogram INTEGER(4)
  !DEF: /P1/i ObjectEntity INTEGER(4)
  !DEF: /P1/j ObjectEntity INTEGER(4)
diff --git a/flang/test/Semantics/symbol17.f90 b/flang/test/Semantics/symbol17.f90
index 51f3afb19692d..a0d916e55cfa4 100644
--- a/flang/test/Semantics/symbol17.f90
+++ b/flang/test/Semantics/symbol17.f90
@@ -2,7 +2,7 @@
 ! Forward references to derived types (non-error cases)
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/t1 DerivedType
  type :: t1
   !DEF: /MAIN/t2 DerivedType
diff --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
index bfe25ff4df25f..6e41bb5db91ee 100644
--- a/flang/test/Semantics/symbol18.f90
+++ b/flang/test/Semantics/symbol18.f90
@@ -3,7 +3,7 @@
 ! Intrinsic function in type declaration statement: type is ignored
 
 !DEF: /P1 MainProgram
-program p1
+program P1
  !DEF: /P1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
  integer cos
  !DEF: /P1/y (Implicit) ObjectEntity REAL(4)
diff --git a/flang/test/Semantics/symbol20.f90 b/flang/test/Semantics/symbol20.f90
index 20bd1bdbf2a7c..bf3aff489b3b9 100644
--- a/flang/test/Semantics/symbol20.f90
+++ b/flang/test/Semantics/symbol20.f90
@@ -33,7 +33,7 @@ subroutine bar
  end subroutine
 end module
 !DEF: /DEMO MainProgram
-program demo
+program DEMO
  !REF: /m
  use :: m
  !DEF: /DEMO/bar (Subroutine) Use
diff --git a/flang/test/Semantics/symbol25.f90 b/flang/test/Semantics/symbol25.f90
index a131bf899340f..ac47a19eae8cc 100644
--- a/flang/test/Semantics/symbol25.f90
+++ b/flang/test/Semantics/symbol25.f90
@@ -39,7 +39,7 @@ end subroutine inner1
  end subroutine outer
 end module m
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /m
  use :: m
  !REF: /m/specific1
diff --git a/flang/test/Semantics/symbol26.f90 b/flang/test/Semantics/symbol26.f90
index e04cd32df57c4..dded4b632c654 100644
--- a/flang/test/Semantics/symbol26.f90
+++ b/flang/test/Semantics/symbol26.f90
@@ -9,7 +9,7 @@ module m
  namelist/a/j
 end module m
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/j (Implicit) ObjectEntity INTEGER(4)
  j = 1
 contains

>From 37dbcbf9c27aa973b939dfa45a709e8307937405 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 14:28:06 -0400
Subject: [PATCH 7/8] More test changes

---
 flang/test/Semantics/getsymbols03-a.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index e3c2e0bf53f92..5c5e87575a9cb 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -8,7 +8,7 @@ program main
 end program
 
 ! RUN: %flang_fc1 -fget-symbols-sources %s 2>&1 | FileCheck %s
-! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
 ! CHECK:MAIN:{{.*}}getsymbols03-a.f90, 4, 9-13
+! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
 ! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
 ! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14

>From 8cd6fa1d597d328effcbbd1544d3e614e5ad8377 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 15:06:18 -0400
Subject: [PATCH 8/8] Tweaked the error message and updated the tests
 accordingly

---
 flang/lib/Semantics/check-omp-structure.cpp      | 3 +--
 flang/test/Semantics/OpenMP/declare-target03.f90 | 2 +-
 flang/test/Semantics/OpenMP/threadprivate03.f90  | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2425265e196c6..e4a94efcc6b55 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -1156,8 +1156,7 @@ void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar(
                         (sym->has<MainProgramDetails>() ||
                             sym->has<ModuleDetails>())) {
                       context_.Say(name->source,
-                          "The module name or main program name cannot be in a "
-                          "%s "
+                          "The module name cannot be in a %s "
                           "directive"_err_en_US,
                           ContextDirectiveAsFortran());
                     } else if (!IsSaved(*name->symbol) &&
diff --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index b3d6551ac85de..48cfc68393873 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -13,7 +13,7 @@ subroutine bar
 program main
   use mod1
 
-  !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
+  !ERROR: The module name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
   ! This is now allowed: "main" is implicitly declared symbol separate
diff --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index 9f222773ad432..fda2fe608ac3c 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -10,7 +10,7 @@ program main
   use mod1
   integer, parameter :: i = 1
 
-  !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
+  !ERROR: The module name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
   ! This is now allowed, since "main" is implicitly declared symbol,


From flang-commits at lists.llvm.org  Wed Jul 16 12:15:15 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Wed, 16 Jul 2025 12:15:15 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <6877fa43.170a0220.132a56.25b1@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/149169

>From f7cc4e803ca099d98d9711b42fc342751de67367 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 15:33:12 -0400
Subject: [PATCH 1/9] [flang] Attempt to address main program symbol conflicts

... by making main program symbol names upper case.
---
 flang/lib/Semantics/resolve-labels.cpp | 32 ++++++++++++++++++++------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index b0cbc4b56e889..0278f70b90226 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -489,15 +489,33 @@ class ParseTreeAnalyzer {
 
   // C1401
   void Post(const parser::MainProgram &mainProgram) {
-    if (const parser::CharBlock *
+    // Uppercase the name of the main program, so that its symbol name
+    // would be unique from similarly named non-main-program symbols.
+    auto upperCaseCharBlock = [](const parser::CharBlock& cb) {
+      char* ch = const_cast<char*>(cb.begin());
+      char* endCh = ch + cb.size();
+      while (ch != endCh) {
+        *ch++ = parser::ToUpperCaseLetter(*ch);
+      }
+    };
+    const parser::CharBlock *progName{nullptr};
+    if (const auto &program{
+            std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
+                mainProgram.t)}) {
+      progName = &program->statement.v.source;
+    }
+    if (progName) {
+      upperCaseCharBlock(*progName);
+    }
+    const parser::CharBlock *
         endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
-            mainProgram.t))}) {
-      if (const auto &program{
-              std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
-                  mainProgram.t)}) {
-        if (*endName != program->statement.v.source) {
+            mainProgram.t))};
+    if (endName) {
+      upperCaseCharBlock(*endName);
+      if (progName) {
+        if (*endName != *progName) {
           context_.Say(*endName, "END PROGRAM name mismatch"_err_en_US)
-              .Attach(program->statement.v.source, "should be"_en_US);
+              .Attach(*progName, "should be"_en_US);
         }
       } else {
         context_.Say(*endName,

>From 336c138bda8c1738c5845f274533b84a075c41aa Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:16:13 -0400
Subject: [PATCH 2/9] clang-format

---
 flang/lib/Semantics/resolve-labels.cpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index 0278f70b90226..d3fda03383783 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -491,9 +491,9 @@ class ParseTreeAnalyzer {
   void Post(const parser::MainProgram &mainProgram) {
     // Uppercase the name of the main program, so that its symbol name
     // would be unique from similarly named non-main-program symbols.
-    auto upperCaseCharBlock = [](const parser::CharBlock& cb) {
-      char* ch = const_cast<char*>(cb.begin());
-      char* endCh = ch + cb.size();
+    auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
+      char *ch = const_cast<char *>(cb.begin());
+      char *endCh = ch + cb.size();
       while (ch != endCh) {
         *ch++ = parser::ToUpperCaseLetter(*ch);
       }
@@ -507,9 +507,8 @@ class ParseTreeAnalyzer {
     if (progName) {
       upperCaseCharBlock(*progName);
     }
-    const parser::CharBlock *
-        endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
-            mainProgram.t))};
+    const parser::CharBlock *endName{GetStmtName(
+        std::get<parser::Statement<parser::EndProgramStmt>>(mainProgram.t))};
     if (endName) {
       upperCaseCharBlock(*endName);
       if (progName) {

>From f6507f56a22227d1b3cb5327d2a0b561a494327e Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:16:52 -0400
Subject: [PATCH 3/9] Initial test changes

---
 flang/test/Driver/cuda-option.f90             |  2 +-
 flang/test/Driver/unparse-use-analyzed.f95    |  4 +-
 flang/test/Driver/unparse-with-modules.f90    |  2 +-
 .../test/Integration/debug-common-block-1.f90 |  2 +-
 flang/test/Integration/debug-local-var-2.f90  |  2 +-
 flang/test/Lower/CUDA/cuda-derived.cuf        |  2 +-
 flang/test/Lower/CUDA/cuda-return01.cuf       |  2 +-
 flang/test/Lower/CUDA/cuda-return02.cuf       |  2 +-
 .../Lower/HLFIR/intrinsic-subroutines.f90     |  2 +-
 ...ointer-component-structure-constructor.f90 |  2 +-
 flang/test/Lower/OpenACC/acc-atomic-read.f90  |  2 +-
 flang/test/Lower/OpenACC/acc-atomic-write.f90 |  2 +-
 flang/test/Lower/OpenACC/acc-routine04.f90    |  2 +-
 flang/test/Lower/OpenMP/atomic-read.f90       |  2 +-
 flang/test/Lower/OpenMP/atomic-write.f90      |  2 +-
 .../Lower/OpenMP/common-atomic-lowering.f90   |  2 +-
 flang/test/Lower/OpenMP/cray-pointers02.f90   |  2 +-
 .../Lower/OpenMP/default-clause-byref.f90     |  2 +-
 flang/test/Lower/OpenMP/default-clause.f90    |  2 +-
 .../parallel-reduction-allocatable-array.f90  |  2 +-
 .../OpenMP/parallel-reduction-array-lb.f90    |  2 +-
 .../OpenMP/parallel-reduction-array2.f90      |  2 +-
 .../Lower/OpenMP/parallel-reduction-byref.f90 |  2 +-
 .../parallel-reduction-pointer-array.f90      |  2 +-
 .../OpenMP/parallel-reduction-rename.f90      |  2 +-
 .../test/Lower/OpenMP/parallel-reduction.f90  |  2 +-
 flang/test/Lower/OpenMP/sections.f90          |  2 +-
 .../threadprivate-host-association-2.f90      |  2 +-
 .../threadprivate-host-association-3.f90      |  2 +-
 .../OpenMP/threadprivate-host-association.f90 |  2 +-
 flang/test/Lower/OpenMP/wsloop-chunks.f90     |  2 +-
 flang/test/Lower/OpenMP/wsloop-collapse.f90   |  2 +-
 ...oop-reduction-allocatable-array-minmax.f90 |  2 +-
 .../OpenMP/wsloop-reduction-allocatable.f90   |  2 +-
 .../OpenMP/wsloop-reduction-array-lb.f90      |  2 +-
 .../OpenMP/wsloop-reduction-array-lb2.f90     |  2 +-
 .../Lower/OpenMP/wsloop-reduction-array.f90   |  2 +-
 .../Lower/OpenMP/wsloop-reduction-array2.f90  |  2 +-
 .../Lower/OpenMP/wsloop-reduction-min2.f90    |  2 +-
 .../wsloop-reduction-multiple-clauses.f90     |  2 +-
 .../Lower/OpenMP/wsloop-reduction-pointer.f90 |  2 +-
 flang/test/Lower/array-character.f90          |  2 +-
 flang/test/Lower/array-expression-slice-1.f90 |  2 +-
 flang/test/Lower/basic-program.f90            |  6 +-
 flang/test/Lower/big-integer-parameter.f90    |  2 +-
 .../test/Lower/derived-type-finalization.f90  |  2 +-
 flang/test/Lower/location.f90                 |  2 +-
 flang/test/Lower/nested-where.f90             |  2 +-
 flang/test/Lower/polymorphic.f90              |  2 +-
 flang/test/Lower/pre-fir-tree02.f90           |  2 +-
 flang/test/Lower/pre-fir-tree03.f90           |  2 +-
 flang/test/Lower/pre-fir-tree06.f90           |  4 +-
 .../test/Lower/program-units-fir-mangling.f90 |  2 +-
 flang/test/Lower/return-statement.f90         |  2 +-
 flang/test/Lower/volatile-openmp1.f90         |  2 +-
 flang/test/Lower/volatile-string.f90          |  2 +-
 flang/test/Lower/volatile3.f90                |  2 +-
 flang/test/Parser/acc-unparse.f90             |  2 +-
 .../test/Semantics/OpenACC/acc-symbols01.f90  |  2 +-
 .../OpenMP/critical_within_default.f90        |  2 +-
 .../OpenMP/declare-mapper-symbols.f90         |  2 +-
 .../OpenMP/declare-reduction-mangled.f90      |  2 +-
 .../OpenMP/declare-reduction-operators.f90    |  2 +-
 .../OpenMP/declare-reduction-renamedop.f90    |  2 +-
 .../Semantics/OpenMP/declare-reduction.f90    |  2 +-
 flang/test/Semantics/OpenMP/do-schedule03.f90 | 28 +++----
 .../Semantics/OpenMP/do01-positivecase.f90    |  6 +-
 .../Semantics/OpenMP/do04-positivecase.f90    | 14 ++--
 .../Semantics/OpenMP/do05-positivecase.f90    | 32 ++++----
 .../Semantics/OpenMP/do06-positivecases.f90   | 10 +--
 flang/test/Semantics/OpenMP/do11.f90          | 26 +++----
 flang/test/Semantics/OpenMP/do12.f90          | 74 +++++++++----------
 flang/test/Semantics/OpenMP/do14.f90          | 62 ++++++++--------
 flang/test/Semantics/OpenMP/do17.f90          | 38 +++++-----
 .../Semantics/OpenMP/map-clause-symbols.f90   |  2 +-
 flang/test/Semantics/OpenMP/reduction08.f90   | 48 ++++++------
 flang/test/Semantics/OpenMP/reduction09.f90   | 38 +++++-----
 flang/test/Semantics/OpenMP/reduction11.f90   |  2 +-
 flang/test/Semantics/OpenMP/scan2.f90         |  2 +-
 flang/test/Semantics/OpenMP/symbol01.f90      | 52 ++++++-------
 flang/test/Semantics/OpenMP/symbol05.f90      |  4 +-
 flang/test/Semantics/OpenMP/symbol07.f90      |  2 +-
 flang/test/Semantics/OpenMP/symbol09.f90      |  2 +-
 flang/test/Semantics/getsymbols03-a.f90       |  2 +-
 flang/test/Semantics/modproc01.f90            |  2 +-
 flang/test/Semantics/multi-programs04.f90     |  2 +-
 flang/test/Semantics/procinterface01.f90      | 22 +++---
 flang/test/Semantics/resolve125.f90           |  4 +-
 flang/test/Semantics/symbol03.f90             | 18 ++---
 flang/test/Semantics/symbol06.f90             | 70 +++++++++---------
 flang/test/Semantics/symbol07.f90             | 46 ++++++------
 flang/test/Semantics/symbol08.f90             | 14 ++--
 flang/test/Semantics/symbol15.f90             | 10 +--
 flang/test/Semantics/symbol16.f90             | 18 ++---
 flang/test/Semantics/symbol17.f90             | 50 ++++++-------
 flang/test/Semantics/symbol18.f90             | 20 ++---
 flang/test/Semantics/symbol20.f90             | 12 +--
 flang/test/Semantics/symbol25.f90             | 12 +--
 flang/test/Semantics/symbol26.f90             |  8 +-
 .../Transforms/DoConcurrent/basic_host.f90    |  2 +-
 100 files changed, 446 insertions(+), 446 deletions(-)

diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 0740ed509a077..f55e88dab20ce 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -8,7 +8,7 @@ program main
   integer, device :: dvar
 end program
 
-! CHECK-LABEL: PROGRAM main
+! CHECK-LABEL: PROGRAM MAIN
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
diff --git a/flang/test/Driver/unparse-use-analyzed.f95 b/flang/test/Driver/unparse-use-analyzed.f95
index eb6046aebba54..4bcd72c9a5f50 100644
--- a/flang/test/Driver/unparse-use-analyzed.f95
+++ b/flang/test/Driver/unparse-use-analyzed.f95
@@ -6,12 +6,12 @@
 ! RUN: %flang_fc1 -fdebug-unparse  %s | FileCheck %s --check-prefix=DEFAULT
 ! RUN: %flang_fc1 -fdebug-unparse -fno-analyzed-objects-for-unparse %s | FileCheck %s --check-prefix=DISABLED
 
-! DEFAULT: PROGRAM test
+! DEFAULT: PROGRAM TEST
 ! DEFAULT-NEXT:  REAL, PARAMETER :: val = 3.43e2_4
 ! DEFAULT-NEXT:  PRINT *, 3.47e2_4
 ! DEFAULT-NEXT: END PROGRAM
 
-! DISABLED: PROGRAM test
+! DISABLED: PROGRAM TEST
 ! DISABLED-NEXT:  REAL, PARAMETER :: val = 343.0
 ! DISABLED-NEXT:  PRINT *, val+4
 ! DISABLED-NEXT: END PROGRAM
diff --git a/flang/test/Driver/unparse-with-modules.f90 b/flang/test/Driver/unparse-with-modules.f90
index 53997f7804efa..f6444afbe47c1 100644
--- a/flang/test/Driver/unparse-with-modules.f90
+++ b/flang/test/Driver/unparse-with-modules.f90
@@ -25,7 +25,7 @@ program test
 !CHECK:  implicit none
 !CHECK:  real(kind=real32) x
 !CHECK: end module
-!CHECK: program test
+!CHECK: program TEST
 !CHECK:  use :: m1
 !CHECK:  use :: basictestmoduletwo
 !CHECK:  implicit none
diff --git a/flang/test/Integration/debug-common-block-1.f90 b/flang/test/Integration/debug-common-block-1.f90
index 18217637be0fa..77f47daea4a91 100644
--- a/flang/test/Integration/debug-common-block-1.f90
+++ b/flang/test/Integration/debug-common-block-1.f90
@@ -89,7 +89,7 @@ program test
 ! CHECK-DAG: ![[CBF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "__BLNK__"{{.*}})
 ! CHECK-DAG: ![[CBAF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "a"{{.*}})
 
-! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "test"{{.*}})
+! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "TEST"{{.*}})
 ! CHECK-DAG: ![[CBM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "__BLNK__"{{.*}})
 ! CHECK-DAG: ![[CBAM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "a"{{.*}})
 
diff --git a/flang/test/Integration/debug-local-var-2.f90 b/flang/test/Integration/debug-local-var-2.f90
index b97be141cc8d0..0ddac633a5b1e 100644
--- a/flang/test/Integration/debug-local-var-2.f90
+++ b/flang/test/Integration/debug-local-var-2.f90
@@ -37,7 +37,7 @@
 ! BOTH-LABEL: }
 
 program mn
-! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "mn", {{.*}})
+! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "MN", {{.*}})
 
 ! BOTH-DAG: ![[TYI32:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
 ! BOTH-DAG: ![[TYI64:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)
diff --git a/flang/test/Lower/CUDA/cuda-derived.cuf b/flang/test/Lower/CUDA/cuda-derived.cuf
index 96250d88d81c4..d419ee074f7a0 100644
--- a/flang/test/Lower/CUDA/cuda-derived.cuf
+++ b/flang/test/Lower/CUDA/cuda-derived.cuf
@@ -25,6 +25,6 @@ program main
   type(t2) :: b
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"}
 ! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEa"}
 ! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tt2{b:!fir.type<_QMm1Tt1{a:!fir.box<!fir.heap<!fir.array<?xf32>>>}>}> {bindc_name = "b", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/CUDA/cuda-return01.cuf b/flang/test/Lower/CUDA/cuda-return01.cuf
index 47e69a903efd3..ed7c640a71082 100644
--- a/flang/test/Lower/CUDA/cuda-return01.cuf
+++ b/flang/test/Lower/CUDA/cuda-return01.cuf
@@ -28,6 +28,6 @@ program main
   return
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"}
 ! CHECK: cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QFEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK-NOT: cuf.free
diff --git a/flang/test/Lower/CUDA/cuda-return02.cuf b/flang/test/Lower/CUDA/cuda-return02.cuf
index e450d7e796f22..e54818444e49c 100644
--- a/flang/test/Lower/CUDA/cuda-return02.cuf
+++ b/flang/test/Lower/CUDA/cuda-return02.cuf
@@ -13,7 +13,7 @@ program test
   return
 end
 
-! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare
 ! CHECK: cf.cond_br %{{.*}}, ^bb1, ^bb2
 ! CHECK-NEXT: ^bb1:
diff --git a/flang/test/Lower/HLFIR/intrinsic-subroutines.f90 b/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
index 07c4f012781d4..cbc56ca1e395b 100644
--- a/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
+++ b/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
@@ -24,7 +24,7 @@ program main
   call mvbits(from, 2, 2, to, 0)
   if (any(to /= 5)) STOP 1
 end program
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<3xi32> {bindc_name = "from", uniq_name = "_QFEfrom"}
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
index 7b64634d10d4b..a097b1522307e 100644
--- a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
@@ -35,7 +35,7 @@ FUNCTION BAR() RESULT(res)
     END
   END
 
-! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:    %[[VAL_0:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_1:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_2:.*]] = fir.alloca !fir.boxproc<(!fir.ref<i32>) -> i32> {bindc_name = "pp2", uniq_name = "_QFEpp2"}
diff --git a/flang/test/Lower/OpenACC/acc-atomic-read.f90 b/flang/test/Lower/OpenACC/acc-atomic-read.f90
index 639a98051e3a2..76751a0fa63a8 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-read.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-read.f90
@@ -8,7 +8,7 @@ program acc_atomic_test
      g = h
 end program acc_atomic_test
 
-! CHECK: func @_QQmain() attributes {fir.bindc_name = "acc_atomic_test"} {
+! CHECK: func @_QQmain() attributes {fir.bindc_name = "ACC_ATOMIC_TEST"} {
 ! CHECK: %[[VAR_G:.*]] = fir.alloca f32 {bindc_name = "g", uniq_name = "_QFEg"}
 ! CHECK: %[[G_DECL:.*]]:2 = hlfir.declare %[[VAR_G]] {uniq_name = "_QFEg"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK: %[[VAR_H:.*]] = fir.alloca f32 {bindc_name = "h", uniq_name = "_QFEh"}
diff --git a/flang/test/Lower/OpenACC/acc-atomic-write.f90 b/flang/test/Lower/OpenACC/acc-atomic-write.f90
index 3c55394021abf..e0116e3281820 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-write.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-write.f90
@@ -2,7 +2,7 @@
 
 ! This test checks the lowering of atomic write
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "acc_atomic_write_test"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "ACC_ATOMIC_WRITE_TEST"} {
 !CHECK: %[[VAR_X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[VAR_X]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[VAR_Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
diff --git a/flang/test/Lower/OpenACC/acc-routine04.f90 b/flang/test/Lower/OpenACC/acc-routine04.f90
index f603376163901..655e2762b9694 100644
--- a/flang/test/Lower/OpenACC/acc-routine04.f90
+++ b/flang/test/Lower/OpenACC/acc-routine04.f90
@@ -30,5 +30,5 @@ subroutine sub2()
 ! CHECK: acc.routine @acc_routine_1 func(@_QFPsub2) seq
 ! CHECK: acc.routine @acc_routine_0 func(@_QMdummy_modPsub1) seq
 ! CHECK: func.func @_QMdummy_modPsub1(%arg0: !fir.ref<i32> {fir.bindc_name = "i"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>}
-! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test_acc_routine"}
+! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "TEST_ACC_ROUTINE"}
 ! CHECK: func.func private @_QFPsub2() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>, fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>}
diff --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
index 68dcaac90eef5..30313e240efa3 100644
--- a/flang/test/Lower/OpenMP/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -4,7 +4,7 @@
 
 ! This test checks the lowering of atomic read
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomic"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "OMPATOMIC"} {
 !CHECK:    %[[A_REF:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_REF]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[B_REF:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/OpenMP/atomic-write.f90 b/flang/test/Lower/OpenMP/atomic-write.f90
index 6eded49b0b15d..742fd475c0f04 100644
--- a/flang/test/Lower/OpenMP/atomic-write.f90
+++ b/flang/test/Lower/OpenMP/atomic-write.f90
@@ -4,7 +4,7 @@
 
 ! This test checks the lowering of atomic write
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomicwrite"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "OMPATOMICWRITE"} {
 !CHECK:    %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[Y_REF:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
diff --git a/flang/test/Lower/OpenMP/common-atomic-lowering.f90 b/flang/test/Lower/OpenMP/common-atomic-lowering.f90
index a53cc101024c6..f729bbb00ac8e 100644
--- a/flang/test/Lower/OpenMP/common-atomic-lowering.f90
+++ b/flang/test/Lower/OpenMP/common-atomic-lowering.f90
@@ -1,6 +1,6 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "sample"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "SAMPLE"} {
 !CHECK: %[[val_0:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK: %[[val_1:.*]]:2 = hlfir.declare %[[val_0]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[val_2:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/OpenMP/cray-pointers02.f90 b/flang/test/Lower/OpenMP/cray-pointers02.f90
index 19e4cd09fe50a..79d838702e4b0 100644
--- a/flang/test/Lower/OpenMP/cray-pointers02.f90
+++ b/flang/test/Lower/OpenMP/cray-pointers02.f90
@@ -1,7 +1,7 @@
 ! Test lowering of Cray pointee references.
 ! RUN: flang -fc1 -emit-hlfir -fopenmp %s -o - 2>&1 | FileCheck %s
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test_cray_pointers_02"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST_CRAY_POINTERS_02"}
 program test_cray_pointers_02
     implicit none
 
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index c44c6bb966580..af51c4cc3e814 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -34,7 +34,7 @@
 !CHECK:    omp.yield(%[[PRIV_X]] : !fir.ref<i32>)
 !CHECK:  }
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "DEFAULT_CLAUSE_LOWERING"} {
 !CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"}
 !CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index ee5f579f06b91..505fa4f0f5d63 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -8,7 +8,7 @@
 ! RUN: | FileCheck %s
 
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "DEFAULT_CLAUSE_LOWERING"} {
 !CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"}
 !CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index 4bfd5d8d19261..0036670317a8e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -80,7 +80,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index ec54294c7104f..ea0aa9ec3f53b 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -68,7 +68,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
index 488ecc353af8e..eb0df2b3a17de 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
@@ -63,7 +63,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
index 596276a99cafc..2caec0384a6ab 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
@@ -18,7 +18,7 @@
 !CHECK:    fir.store %[[CR]] to %[[C0]] : !fir.ref<i32>
 !CHECK:    omp.yield(%[[C0]] : !fir.ref<i32>)
 !CHECK:  }
-!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "mn"} {
+!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "MN"} {
 !CHECK:    %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
index f638688bc2cc1..3c1daa0eb983f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
@@ -82,7 +82,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90 b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
index c06343e997bfd..2be154f4bbaf5 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
@@ -25,7 +25,7 @@ end program main
 ! CHECK:           omp.yield(%[[VAL_2]] : i32)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction.f90 b/flang/test/Lower/OpenMP/parallel-reduction.f90
index 612549fb32de5..15e8cc325916d 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction.f90
@@ -10,7 +10,7 @@
 !CHECK:    %[[CR:[_a-z0-9]+]] = arith.addi %[[C0]], %[[C1]] : i32
 !CHECK:    omp.yield(%[[CR]] : i32)
 !CHECK:  }
-!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "mn"} {
+!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "MN"} {
 !CHECK:    %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index d11925cafdc12..3d5c0326fb6b9 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -5,7 +5,7 @@
 ! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 ! RUN: bbc -hlfir -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "sample"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "SAMPLE"} {
 !CHECK:   %[[COUNT:.*]] = fir.address_of(@_QFEcount) : !fir.ref<i32>
 !CHECK:   %[[COUNT_DECL:.*]]:2 = hlfir.declare %[[COUNT]] {uniq_name = "_QFEcount"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[ETA:.*]] = fir.alloca f32 {bindc_name = "eta", uniq_name = "_QFEeta"}
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
index 5e54cef8c29db..5c90ef7d84f89 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
index 21547b47cf381..0e61261e8853e 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association.f90
index 7a27efa2f84aa..1887e8aa68fdc 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[TP_A:.*]] = omp.threadprivate %[[A_DECL]]#0 : !fir.ref<i32> -> !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-chunks.f90 b/flang/test/Lower/OpenMP/wsloop-chunks.f90
index 29c02a3b3c8d5..f3f11d8c4a6c2 100644
--- a/flang/test/Lower/OpenMP/wsloop-chunks.f90
+++ b/flang/test/Lower/OpenMP/wsloop-chunks.f90
@@ -7,7 +7,7 @@ program wsloop
         integer :: i
         integer :: chunk
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "wsloop"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "WSLOOP"} {
 ! CHECK:         %[[CHUNK_REF:.*]] = fir.alloca i32 {bindc_name = "chunk", uniq_name = "_QFEchunk"}
 ! CHECK:         %[[VAL_0:.*]]:2 = hlfir.declare %[[CHUNK_REF]] {uniq_name = "_QFEchunk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
diff --git a/flang/test/Lower/OpenMP/wsloop-collapse.f90 b/flang/test/Lower/OpenMP/wsloop-collapse.f90
index a4d5cbdc03d3e..7ec40ab4b2f43 100644
--- a/flang/test/Lower/OpenMP/wsloop-collapse.f90
+++ b/flang/test/Lower/OpenMP/wsloop-collapse.f90
@@ -2,7 +2,7 @@
 
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
 
-!CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "wsloop_collapse"} {
+!CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "WSLOOP_COLLAPSE"} {
 program wsloop_collapse
 !CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index 58b68e5ec4cfd..e2f75bc8e4481 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -156,7 +156,7 @@ program reduce15
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce15"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE15"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEarr"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index 0a536eb34e7af..663851cba46c6 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -63,7 +63,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "r", uniq_name = "_QFEr"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
index 9f0dd16002baf..2233a74600948 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
@@ -31,5 +31,5 @@ program reduce
 ! CHECK:           omp.yield(%[[ARG0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           omp.wsloop {{.*}} reduction(byref @add_reduction_byref_box_2xi32 %{{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<2xi32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
index 5ada623a0ed23..211bde19da8db 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
@@ -40,5 +40,5 @@ subroutine sub(a, lb, ub)
 ! CHECK:           omp.yield(%[[ARG0]] : !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           omp.wsloop {{.*}} reduction(byref @add_reduction_byref_box_Uxi32 %{{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index 21261da49710c..b7882bcbc0d13 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index ab8dcf1f076c0..7d90335a13a87 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
index 1e26f5a24d41e..d776bd7cfdd03 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
@@ -28,7 +28,7 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_2]] : i32)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
index e0a3b469f40c1..5133db0347034 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -93,7 +93,7 @@ program main
 ! CHECK:           omp.yield(%[[VAL_2]] : f64)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarray) : !fir.ref<!fir.array<3x3xf64>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
index 40b4302f24cd4..27b726376fbeb 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
@@ -64,7 +64,7 @@ program reduce_pointer
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce_pointer"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE_POINTER"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "v", uniq_name = "_QFEv"}
diff --git a/flang/test/Lower/array-character.f90 b/flang/test/Lower/array-character.f90
index 1bc73dae44235..e2899d967c80d 100644
--- a/flang/test/Lower/array-character.f90
+++ b/flang/test/Lower/array-character.f90
@@ -32,7 +32,7 @@ program p
   call charlit
 end program p
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.array<3x!fir.char<1,4>> {bindc_name = "c1", uniq_name = "_QFEc1"}
diff --git a/flang/test/Lower/array-expression-slice-1.f90 b/flang/test/Lower/array-expression-slice-1.f90
index b597814bc0d9f..73943137cb18d 100644
--- a/flang/test/Lower/array-expression-slice-1.f90
+++ b/flang/test/Lower/array-expression-slice-1.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -hlfir=false -fwrapv -o - --outline-intrinsics %s | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK-DAG:         %[[VAL_0:.*]] = arith.constant 10 : index
 ! CHECK-DAG:         %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK-DAG:         %[[VAL_5:.*]] = arith.constant 1 : index
diff --git a/flang/test/Lower/basic-program.f90 b/flang/test/Lower/basic-program.f90
index 5a0e4bdc7b4a1..7e5b40d9e2f0a 100644
--- a/flang/test/Lower/basic-program.f90
+++ b/flang/test/Lower/basic-program.f90
@@ -4,10 +4,10 @@
 program basic
 end program
 
-! CHECK: 1 Program basic
+! CHECK: 1 Program BASIC
 ! CHECK:   1 EndProgramStmt: end program
-! CHECK: End Program basic
+! CHECK: End Program BASIC
 
-! FIR-LABEL: func @_QQmain() attributes {fir.bindc_name = "basic"} {
+! FIR-LABEL: func @_QQmain() attributes {fir.bindc_name = "BASIC"} {
 ! FIR:         return
 ! FIR:       }
diff --git a/flang/test/Lower/big-integer-parameter.f90 b/flang/test/Lower/big-integer-parameter.f90
index a413b1224ebc2..ca90b8adfb318 100644
--- a/flang/test/Lower/big-integer-parameter.f90
+++ b/flang/test/Lower/big-integer-parameter.f90
@@ -13,7 +13,7 @@ program i128
   print*,y
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "i128"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "I128"} {
 ! CHECK-COUNT-2:  %{{.*}} = fir.call @_FortranAioOutputInteger128(%{{.*}}, %{{.*}}) {{.*}}: (!fir.ref<i8>, i128) -> i1
 
 
diff --git a/flang/test/Lower/derived-type-finalization.f90 b/flang/test/Lower/derived-type-finalization.f90
index 3ea58cd719f4a..71cef34899603 100644
--- a/flang/test/Lower/derived-type-finalization.f90
+++ b/flang/test/Lower/derived-type-finalization.f90
@@ -255,5 +255,5 @@ program p
   type(t1) :: t
 end program
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK-NOT: fir.call @_FortranADestroy
diff --git a/flang/test/Lower/location.f90 b/flang/test/Lower/location.f90
index a6ece31bbebed..95bf2260fc107 100644
--- a/flang/test/Lower/location.f90
+++ b/flang/test/Lower/location.f90
@@ -5,7 +5,7 @@ program test
 
 end 
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: fir.call @_FortranAioOutputAscii(%{{.*}}, %{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1 loc(fused<#fir<loc_kind_array[ base,  inclusion,  inclusion]>>["{{.*}}location1.inc":1:10, "{{.*}}location0.inc":1:1, "{{.*}}location.f90":4:1])
 ! CHECK: return loc("{{.*}}location.f90":6:1)
 ! CHECK: } loc("{{.*}}location.f90":3:1)
diff --git a/flang/test/Lower/nested-where.f90 b/flang/test/Lower/nested-where.f90
index ab457280b80ce..28aced2325813 100644
--- a/flang/test/Lower/nested-where.f90
+++ b/flang/test/Lower/nested-where.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "nested_where"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "NESTED_WHERE"} {
 program nested_where
 
   ! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {adapt.valuebyref, bindc_name = "i"}
diff --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90
index b7be5f685d9e3..a84b495dd09d0 100644
--- a/flang/test/Lower/polymorphic.f90
+++ b/flang/test/Lower/polymorphic.f90
@@ -1146,7 +1146,7 @@ program test
   l = i < o%inner
 end program
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: %[[ADDR_O:.*]] = fir.address_of(@_QFEo) : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMpolymorphic_testTouter{inner:!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>}>>>>
 ! CHECK: %[[BOX_NONE:.*]] = fir.convert %[[ADDR_O]] : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMpolymorphic_testTouter{inner:!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>}>>>>) -> !fir.ref<!fir.box<none>>
 ! CHECK: %{{.*}} = fir.call @_FortranAAllocatableAllocate(%[[BOX_NONE]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
diff --git a/flang/test/Lower/pre-fir-tree02.f90 b/flang/test/Lower/pre-fir-tree02.f90
index f4fa626ba6548..65c33e9b364fe 100644
--- a/flang/test/Lower/pre-fir-tree02.f90
+++ b/flang/test/Lower/pre-fir-tree02.f90
@@ -3,7 +3,7 @@
 ! Test Pre-FIR Tree captures all the intended nodes from the parse-tree
 ! Coarray and OpenMP related nodes are tested in other files.
 
-! CHECK: Program test_prog
+! CHECK: Program TEST_PROG
 program test_prog
   ! Check specification part is not part of the tree.
   interface
diff --git a/flang/test/Lower/pre-fir-tree03.f90 b/flang/test/Lower/pre-fir-tree03.f90
index 313dab4d6ec7c..1de66e3f8d016 100644
--- a/flang/test/Lower/pre-fir-tree03.f90
+++ b/flang/test/Lower/pre-fir-tree03.f90
@@ -2,7 +2,7 @@
 
 ! Test Pre-FIR Tree captures OpenMP related constructs
 
-! CHECK: Program test_omp
+! CHECK: Program TEST_OMP
 program test_omp
   ! CHECK: PrintStmt
   print *, "sequential"
diff --git a/flang/test/Lower/pre-fir-tree06.f90 b/flang/test/Lower/pre-fir-tree06.f90
index f84bcd8b58b2d..ed1e76cb375bd 100644
--- a/flang/test/Lower/pre-fir-tree06.f90
+++ b/flang/test/Lower/pre-fir-tree06.f90
@@ -25,13 +25,13 @@ subroutine sub2()
 end
 ! CHECK: End Module m2
 
-! CHECK: Program main
+! CHECK: Program MAIN
 program main
   real :: y
   ! CHECK-NEXT: OpenMPDeclarativeConstruct
   !$omp threadprivate(y)
 end
-! CHECK: End Program main
+! CHECK: End Program MAIN
 
 ! CHECK: Subroutine sub1
 subroutine sub1()
diff --git a/flang/test/Lower/program-units-fir-mangling.f90 b/flang/test/Lower/program-units-fir-mangling.f90
index e0af6f065f34d..65940b4e1ff17 100644
--- a/flang/test/Lower/program-units-fir-mangling.f90
+++ b/flang/test/Lower/program-units-fir-mangling.f90
@@ -124,7 +124,7 @@ subroutine should_not_collide()
 ! CHECK: }
 end subroutine
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 program test
 ! CHECK: }
 contains
diff --git a/flang/test/Lower/return-statement.f90 b/flang/test/Lower/return-statement.f90
index 6351a6859eb4f..8ab69e3146e2f 100644
--- a/flang/test/Lower/return-statement.f90
+++ b/flang/test/Lower/return-statement.f90
@@ -4,7 +4,7 @@ program basic
   return
 end program
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "basic"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "BASIC"} {
 ! CHECK:         return
 ! CHECK:       }
 
diff --git a/flang/test/Lower/volatile-openmp1.f90 b/flang/test/Lower/volatile-openmp1.f90
index 163db953b6b80..07d81a1aeb240 100644
--- a/flang/test/Lower/volatile-openmp1.f90
+++ b/flang/test/Lower/volatile-openmp1.f90
@@ -13,7 +13,7 @@ program main
 !$omp end parallel
 end program
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 1000 : i32
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/volatile-string.f90 b/flang/test/Lower/volatile-string.f90
index 88b21d7b245e9..f263db7abb5fc 100644
--- a/flang/test/Lower/volatile-string.f90
+++ b/flang/test/Lower/volatile-string.f90
@@ -21,7 +21,7 @@ subroutine assign_different_length(string)
   end subroutine
 end program
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 11 : i32
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant true
diff --git a/flang/test/Lower/volatile3.f90 b/flang/test/Lower/volatile3.f90
index 8825f8f3afbcb..a32f29d2bb9e7 100644
--- a/flang/test/Lower/volatile3.f90
+++ b/flang/test/Lower/volatile3.f90
@@ -70,7 +70,7 @@ subroutine sub_select_rank(arr)
 end program
 
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 10 : index
diff --git a/flang/test/Parser/acc-unparse.f90 b/flang/test/Parser/acc-unparse.f90
index 62e0d4487f3f7..12e6dec19f272 100644
--- a/flang/test/Parser/acc-unparse.f90
+++ b/flang/test/Parser/acc-unparse.f90
@@ -15,7 +15,7 @@ program bug47659
   end do label1
 end program
 
-!CHECK-LABEL: PROGRAM bug47659
+!CHECK-LABEL: PROGRAM BUG47659
 !CHECK: !$ACC PARALLEL LOOP
 
 
diff --git a/flang/test/Semantics/OpenACC/acc-symbols01.f90 b/flang/test/Semantics/OpenACC/acc-symbols01.f90
index 375445bad13a5..ea22f41197aa7 100644
--- a/flang/test/Semantics/OpenACC/acc-symbols01.f90
+++ b/flang/test/Semantics/OpenACC/acc-symbols01.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenacc
 
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !DEF: /mm/x ObjectEntity REAL(4)
   !DEF: /mm/y ObjectEntity REAL(4)
diff --git a/flang/test/Semantics/OpenMP/critical_within_default.f90 b/flang/test/Semantics/OpenMP/critical_within_default.f90
index dd972e6e52949..a5fe30eeb7de0 100644
--- a/flang/test/Semantics/OpenMP/critical_within_default.f90
+++ b/flang/test/Semantics/OpenMP/critical_within_default.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
 ! Test that we do not make a private copy of the critical name
 
-!CHECK:  MainProgram scope: mn
+!CHECK:  MainProgram scope: MN
 !CHECK-NEXT:    j size=4 offset=0: ObjectEntity type: INTEGER(4)
 !CHECK-NEXT:    OtherConstruct scope:
 !CHECK-NEXT:      j (OmpPrivate): HostAssoc
diff --git a/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90 b/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
index 06f41ab8ce76f..e57a5c0c1cea6 100644
--- a/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
+++ b/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
 
 program main
-!CHECK-LABEL: MainProgram scope: main
+!CHECK-LABEL: MainProgram scope: MAIN
    implicit none
 
    type ty
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90 b/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
index 9d0a097fb1991..fc977f2f1b839 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
@@ -17,7 +17,7 @@ end function mymax
 end module mymod
 
 program omp_examples
-!CHECK-LABEL: MainProgram scope: omp_examples
+!CHECK-LABEL: MainProgram scope: OMP_EXAMPLES
   use mymod
   implicit none
   integer, parameter :: n = 100
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-operators.f90 b/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
index d7a9f2fc0a36b..84dbe1af01877 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
@@ -49,7 +49,7 @@ function my_add(x, y)
 end module m1
 
 program test_vector
-!CHECK-LABEL: MainProgram scope: test_vector
+!CHECK-LABEL: MainProgram scope: TEST_VECTOR
   use vector_mod
 !CHECK: add_vectors (Function): Use from add_vectors in vector_mod
   implicit none
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90 b/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
index 12e80cbf7b327..9cd638d796091 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
@@ -22,7 +22,7 @@ end function my_mul
 end module module1
 
 program test_omp_reduction
-!CHECK: MainProgram scope: test_omp_reduction
+!CHECK: MainProgram scope: TEST_OMP_REDUCTION
   use module1, only: t1, operator(.modmul.) => operator(.mul.)
 
 !CHECK: .modmul. (Function): Use from .mul. in module1
diff --git a/flang/test/Semantics/OpenMP/declare-reduction.f90 b/flang/test/Semantics/OpenMP/declare-reduction.f90
index ddca38fd57812..1f39c57c54ad1 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction.f90
@@ -31,7 +31,7 @@ end subroutine initme
 end function func
 
 program main
-!CHECK-LABEL: MainProgram scope: main
+!CHECK-LABEL: MainProgram scope: MAIN
 
   !$omp declare reduction (my_add_red : integer : omp_out = omp_out + omp_in) initializer (omp_priv=0)
 
diff --git a/flang/test/Semantics/OpenMP/do-schedule03.f90 b/flang/test/Semantics/OpenMP/do-schedule03.f90
index 8787b094d581a..375f3920b57ad 100644
--- a/flang/test/Semantics/OpenMP/do-schedule03.f90
+++ b/flang/test/Semantics/OpenMP/do-schedule03.f90
@@ -2,26 +2,26 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - xx.
-  !DEF: /ompdoschedule MainProgram
+  !DEF: /OMPDOSCHEDULE MainProgram
 program ompdoschedule
-  !DEF: /ompdoschedule/a ObjectEntity REAL(4)
-  !DEF: /ompdoschedule/y ObjectEntity REAL(4)
-  !DEF: /ompdoschedule/z ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/a ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/y ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/z ObjectEntity REAL(4)
   real  a(100),y(100),z(100)
-  !DEF: /ompdoschedule/b ObjectEntity INTEGER(4)
-  !DEF: /ompdoschedule/i ObjectEntity INTEGER(4)
-  !DEF: /ompdoschedule/n ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/b ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/i ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/n ObjectEntity INTEGER(4)
   integer  b,i,n
-  !REF: /ompdoschedule/b
+  !REF: /OMPDOSCHEDULE/b
   b = 10
   !$omp do  schedule(static,b-b)
-  !DEF: /ompdoschedule/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !REF: /ompdoschedule/n
+  !DEF: /OMPDOSCHEDULE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !REF: /OMPDOSCHEDULE/n
   do i = 2,n+1
-    !REF: /ompdoschedule/y
-    !REF: /ompdoschedule/OtherConstruct1/i
-    !REF: /ompdoschedule/z
-    !REF: /ompdoschedule/a
+    !REF: /OMPDOSCHEDULE/y
+    !REF: /OMPDOSCHEDULE/OtherConstruct1/i
+    !REF: /OMPDOSCHEDULE/z
+    !REF: /OMPDOSCHEDULE/a
     y(i) = z(i-1) + a(i)
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/do01-positivecase.f90 b/flang/test/Semantics/OpenMP/do01-positivecase.f90
index 905fdbaf18476..a4a28f6530d7b 100644
--- a/flang/test/Semantics/OpenMP/do01-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do01-positivecase.f90
@@ -4,13 +4,13 @@
 ! The loop iteration variable may not appear in a firstprivate directive.
 ! A positive case
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   integer i
 
   !$omp do  firstprivate(k)
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
     print *, "Hello"
   end do
diff --git a/flang/test/Semantics/OpenMP/do04-positivecase.f90 b/flang/test/Semantics/OpenMP/do04-positivecase.f90
index eb2d67bb8ceb2..0b0c661687dc2 100644
--- a/flang/test/Semantics/OpenMP/do04-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do04-positivecase.f90
@@ -2,18 +2,18 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 
-!DEF: /omp_do1 MainProgram
+!DEF: /OMP_DO1 MainProgram
 program omp_do1
-  !DEF: /omp_do1/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/n (OmpThreadprivate) ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/n (OmpThreadprivate) ObjectEntity INTEGER(4)
   integer i, j, k, n
   !$omp threadprivate (k,n)
   !$omp do
-  !DEF: /omp_do1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do1/j
+    !REF: /OMP_DO1/j
     do j=1,10
       print *, "Hello"
     end do
diff --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index eda04610535c2..07b12dd1d4d23 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -3,13 +3,13 @@
 ! 2.7.1 Loop Construct restrictions on single directive.
 ! A positive case
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/n ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/n ObjectEntity INTEGER(4)
   integer i,n
   !$omp parallel
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
     !$omp single
     print *, "hello"
@@ -19,13 +19,13 @@ program omp_do
 
   !$omp parallel  default(shared)
   !$omp do
-  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
   do i=1,n
     !$omp parallel
     !$omp single
     !DEF: /work EXTERNAL (Subroutine) ProcEntity
-    !DEF: /omp_do/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
+    !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
     call work(i, 1)
     !$omp end single
     !$omp end parallel
@@ -34,7 +34,7 @@ program omp_do
   !$omp end parallel
 
   !$omp parallel private(i)
-  !DEF: /omp_do/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
   do i=1,10
      !$omp single
      print *, "hello"
@@ -43,32 +43,32 @@ program omp_do
   !$omp end parallel
 
   !$omp target teams distribute parallel do
-  !DEF:/omp_do/OtherConstruct4/i (OmpPrivate ,OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF:/OMP_DO/OtherConstruct4/i (OmpPrivate ,OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF:/omp_do/OtherConstruct4/i
+    !REF:/OMP_DO/OtherConstruct4/i
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do
 
   !$omp target teams distribute parallel do simd
-  !DEF:/omp_do/OtherConstruct5/i (OmpLinear,OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF:/OMP_DO/OtherConstruct5/i (OmpLinear,OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF:/omp_do/OtherConstruct5/i
+    !REF:/OMP_DO/OtherConstruct5/i
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute 
-  !DEF: /omp_do/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF: /omp_do/OtherConstruct6/i
+    !REF: /OMP_DO/OtherConstruct6/i
     if(i < 5) cycle
   end do
 
   !$omp target teams distribute simd
-  !DEF: /omp_do/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF: /omp_do/OtherConstruct7/i
+    !REF: /OMP_DO/OtherConstruct7/i
     if(i < 5) cycle
   end do
 end program omp_do
diff --git a/flang/test/Semantics/OpenMP/do06-positivecases.f90 b/flang/test/Semantics/OpenMP/do06-positivecases.f90
index 2713b55fa2ecb..4d45800298a29 100644
--- a/flang/test/Semantics/OpenMP/do06-positivecases.f90
+++ b/flang/test/Semantics/OpenMP/do06-positivecases.f90
@@ -5,14 +5,14 @@
 ! region ever binds to a loop region arising from the loop construct.
 
 ! A positive case
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
   integer i, j, k
   !$omp do  ordered
-    !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do i=1,10
       !$omp ordered
       !DEF: /my_func EXTERNAL (Subroutine) ProcEntity
diff --git a/flang/test/Semantics/OpenMP/do11.f90 b/flang/test/Semantics/OpenMP/do11.f90
index faab457efff3c..886a9f7ae6081 100644
--- a/flang/test/Semantics/OpenMP/do11.f90
+++ b/flang/test/Semantics/OpenMP/do11.f90
@@ -2,34 +2,34 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
   integer i, j, k
   !$omp do
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do/j
+    !REF: /OMP_DO/j
     do j=1,10
-      !REF: /omp_do/OtherConstruct1/i
-      !REF: /omp_do/j
+      !REF: /OMP_DO/OtherConstruct1/i
+      !REF: /OMP_DO/j
       print *, "it", i, j
     end do
   end do
   !$omp end do
 end program omp_do
 
-!DEF: /omp_do2 (Subroutine)Subprogram
+!DEF: /OMP_DO2 (Subroutine)Subprogram
 subroutine omp_do2
-  !DEF: /omp_do2/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do2/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO2/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO2/k ObjectEntity INTEGER(4)
   integer :: i = 0, k
   !$omp do
-  !DEF: /omp_do2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do2/OtherConstruct1/i
+    !REF: /OMP_DO2/OtherConstruct1/i
     print *, "it", i
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/do12.f90 b/flang/test/Semantics/OpenMP/do12.f90
index a057a246f7a99..f36a230426197 100644
--- a/flang/test/Semantics/OpenMP/do12.f90
+++ b/flang/test/Semantics/OpenMP/do12.f90
@@ -2,20 +2,20 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /omp_cycle MainProgram
+!DEF: /OMP_CYCLE MainProgram
 program omp_cycle
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/OtherConstruct1/i
+    !REF: /OMP_CYCLE/OtherConstruct1/i
     if (i<1) cycle
-    !DEF: /omp_cycle/j (Implicit) ObjectEntity INTEGER(4)
+    !DEF: /OMP_CYCLE/j (Implicit) ObjectEntity INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/k (Implicit) ObjectEntity INTEGER(4)
+      !DEF: /OMP_CYCLE/k (Implicit) ObjectEntity INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct1/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct1/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -23,17 +23,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/j
+    !REF: /OMP_CYCLE/j
     do j=0,10
-      !REF: /omp_cycle/OtherConstruct2/i
+      !REF: /OMP_CYCLE/OtherConstruct2/i
       if (i<1) cycle
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct2/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct2/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -41,17 +41,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(2)
-  !DEF: /omp_cycle/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/i
         if (i<1) cycle
-        !REF: /omp_cycle/OtherConstruct3/i
-        !REF: /omp_cycle/OtherConstruct3/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -59,17 +59,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/i
         if (i<1) cycle
-        !REF: /omp_cycle/OtherConstruct4/i
-        !REF: /omp_cycle/OtherConstruct4/j
-        !REF: /omp_cycle/OtherConstruct4/k
+        !REF: /OMP_CYCLE/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/j
+        !REF: /OMP_CYCLE/OtherConstruct4/k
         print *, i, j, k
       end do
     end do
@@ -77,20 +77,20 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo:do i=0,10
-    !DEF: /omp_cycle/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     foo1:do j=0,10
-      !DEF: /omp_cycle/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       foo2:do k=0,10
-        !REF: /omp_cycle/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/i
         if (i<1) cycle foo2
-        !REF: /omp_cycle/OtherConstruct5/i
-        !REF: /omp_cycle/OtherConstruct5/j
-        !REF: /omp_cycle/OtherConstruct5/k
+        !REF: /OMP_CYCLE/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/j
+        !REF: /OMP_CYCLE/OtherConstruct5/k
         print *, i, j, k
       end do foo2
     end do foo1
   end do foo
   !$omp end do
-end program omp_cycle
+end program OMP_CYCLE
diff --git a/flang/test/Semantics/OpenMP/do14.f90 b/flang/test/Semantics/OpenMP/do14.f90
index 5e8a5a64c2979..b508d32d90201 100644
--- a/flang/test/Semantics/OpenMP/do14.f90
+++ b/flang/test/Semantics/OpenMP/do14.f90
@@ -2,19 +2,19 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /omp_cycle MainProgram
+!DEF: /OMP_CYCLE MainProgram
 program omp_cycle
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
     cycle
-    !DEF: /omp_cycle/j (Implicit) ObjectEntity INTEGER(4)
+    !DEF: /OMP_CYCLE/j (Implicit) ObjectEntity INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/k (Implicit) ObjectEntity INTEGER(4)
+      !DEF: /OMP_CYCLE/k (Implicit) ObjectEntity INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct1/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct1/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -22,16 +22,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/j
+    !REF: /OMP_CYCLE/j
     do j=0,10
       cycle
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct2/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct2/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -39,16 +39,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(2)
-  !DEF: /omp_cycle/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
         cycle
-        !REF: /omp_cycle/OtherConstruct3/i
-        !REF: /omp_cycle/OtherConstruct3/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -56,16 +56,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       do k=0,10
         cycle
-        !REF: /omp_cycle/OtherConstruct4/i
-        !REF: /omp_cycle/OtherConstruct4/j
-        !REF: /omp_cycle/OtherConstruct4/k
+        !REF: /OMP_CYCLE/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/j
+        !REF: /OMP_CYCLE/OtherConstruct4/k
         print *, i, j, k
       end do
     end do
@@ -73,16 +73,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  ordered(3)
-  !DEF: /omp_cycle/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo:do i=0,10
-    !DEF: /omp_cycle/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     foo1:do j=0,10
-      !DEF: /omp_cycle/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       foo2:do k=0,10
         cycle foo2
-        !REF: /omp_cycle/OtherConstruct5/i
-        !REF: /omp_cycle/OtherConstruct5/j
-        !REF: /omp_cycle/OtherConstruct5/k
+        !REF: /OMP_CYCLE/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/j
+        !REF: /OMP_CYCLE/OtherConstruct5/k
         print *, i, j, k
       end do foo2
     end do foo1
diff --git a/flang/test/Semantics/OpenMP/do17.f90 b/flang/test/Semantics/OpenMP/do17.f90
index c0c59f16dee1b..0a4cf46d2d6af 100644
--- a/flang/test/Semantics/OpenMP/do17.f90
+++ b/flang/test/Semantics/OpenMP/do17.f90
@@ -2,53 +2,53 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /test MainProgram
+!DEF: /TEST MainProgram
 program test
- !DEF: /test/i ObjectEntity INTEGER(4)
- !DEF: /test/j ObjectEntity INTEGER(4)
- !DEF: /test/k ObjectEntity INTEGER(4)
+ !DEF: /TEST/i ObjectEntity INTEGER(4)
+ !DEF: /TEST/j ObjectEntity INTEGER(4)
+ !DEF: /TEST/k ObjectEntity INTEGER(4)
  integer i, j, k
  !$omp do  collapse(2)
- !DEF: /test/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /TEST/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  foo: do i=0,10
-  !DEF: /test/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /TEST/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo1: do j=0,10
-   !REF: /test/k
+   !REF: /TEST/k
    foo2: do k=0,10
-    !REF: /test/OtherConstruct1/i
+    !REF: /TEST/OtherConstruct1/i
     select case (i)
     case (5)
      cycle foo1
     case (7)
      cycle foo2
     end select
-    !REF: /test/OtherConstruct1/i
-    !REF: /test/OtherConstruct1/j
-    !REF: /test/k
+    !REF: /TEST/OtherConstruct1/i
+    !REF: /TEST/OtherConstruct1/j
+    !REF: /TEST/k
     print *, i, j, k
    end do foo2
   end do foo1
  end do foo
 
  !$omp do  collapse(2)
- !DEF: /test/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /TEST/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  foo: do i=0,10
-  !DEF: /test/OtherConstruct2/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /TEST/OtherConstruct2/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo1: do j=0,10
-   !REF: /test/k
+   !REF: /TEST/k
    foo2: do k=0,10
-    !REF: /test/OtherConstruct2/i
+    !REF: /TEST/OtherConstruct2/i
     if (i<3) then
      cycle foo1
-     !REF: /test/OtherConstruct2/i
+     !REF: /TEST/OtherConstruct2/i
     else if (i>8) then
      cycle foo1
     else
      cycle foo2
     end if
-    !REF: /test/OtherConstruct2/i
-    !REF: /test/OtherConstruct2/j
-    !REF: /test/k
+    !REF: /TEST/OtherConstruct2/i
+    !REF: /TEST/OtherConstruct2/j
+    !REF: /TEST/k
     print *, i, j, k
    end do foo2
   end do foo1
diff --git a/flang/test/Semantics/OpenMP/map-clause-symbols.f90 b/flang/test/Semantics/OpenMP/map-clause-symbols.f90
index 8f984fcd2fa7e..1d6315b4a2312 100644
--- a/flang/test/Semantics/OpenMP/map-clause-symbols.f90
+++ b/flang/test/Semantics/OpenMP/map-clause-symbols.f90
@@ -1,6 +1,6 @@
 ! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
 program main
-!CHECK-LABEL:  MainProgram scope: main
+!CHECK-LABEL:  MainProgram scope: MAIN
   integer, parameter :: n = 256
   real(8) :: a(256)
   !$omp target map(mapper(xx), from:a)
diff --git a/flang/test/Semantics/OpenMP/reduction08.f90 b/flang/test/Semantics/OpenMP/reduction08.f90
index 01a06eb7d7414..1cf2838fe765a 100644
--- a/flang/test/Semantics/OpenMP/reduction08.f90
+++ b/flang/test/Semantics/OpenMP/reduction08.f90
@@ -2,60 +2,60 @@
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases
 
-!DEF: /omp_reduction MainProgram
+!DEF: /OMP_REDUCTION MainProgram
 program omp_reduction
-  !DEF: /omp_reduction/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
-  !DEF: /omp_reduction/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
   integer :: k = 10
-  !DEF: /omp_reduction/m ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/m ObjectEntity INTEGER(4)
   integer :: m = 12
   !$omp parallel do  reduction(max:k)
-  !DEF: /omp_reduction/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
     k = max(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(min:k)
-  !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
     k = min(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(iand:k)
-  !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
     k = iand(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(ior:k)
-  !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
     k = ior(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(ieor:k)
-  !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
     k = ieor(k,m)
   end do
   !$omp end parallel do
diff --git a/flang/test/Semantics/OpenMP/reduction09.f90 b/flang/test/Semantics/OpenMP/reduction09.f90
index d6c71c30d2834..4b1ca116d675b 100644
--- a/flang/test/Semantics/OpenMP/reduction09.f90
+++ b/flang/test/Semantics/OpenMP/reduction09.f90
@@ -1,22 +1,22 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases.
-!DEF: /omp_reduction MainProgram
+!DEF: /OMP_REDUCTION MainProgram
 program omp_reduction
-  !DEF: /omp_reduction/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
-  !DEF: /omp_reduction/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
   integer :: k = 10
-  !DEF: /omp_reduction/a ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/a ObjectEntity INTEGER(4)
   integer a(10)
-  !DEF: /omp_reduction/b ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/b ObjectEntity INTEGER(4)
   integer b(10,10,10)
 
   !$omp parallel  shared(k)
   !$omp do  reduction(+:k)
-  !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
@@ -24,52 +24,52 @@ program omp_reduction
 
 
   !$omp parallel do  reduction(+:a(10))
-  !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
 
   !$omp parallel do  reduction(+:a(1:10:1))
-  !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2))
-  !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2:5:1))
-  !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel  private(i)
   !$omp do reduction(+:k) reduction(+:j)
-  !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
   !$omp end parallel
 
   !$omp do reduction(+:k) reduction(*:j) reduction(+:l)
-  !DEF: /omp_reduction/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/reduction11.f90 b/flang/test/Semantics/OpenMP/reduction11.f90
index b2ad0f6a6ee11..dfb3986d37d78 100644
--- a/flang/test/Semantics/OpenMP/reduction11.f90
+++ b/flang/test/Semantics/OpenMP/reduction11.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols -o - %s 2>&1 | FileCheck %s
 ! Check intrinsic reduction symbols (in this case "max" are marked as INTRINSIC
 
-! CHECK: MainProgram scope: omp_reduction
+! CHECK: MainProgram scope: OMP_REDUCTION
 program omp_reduction
   ! CHECK: i size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer i
diff --git a/flang/test/Semantics/OpenMP/scan2.f90 b/flang/test/Semantics/OpenMP/scan2.f90
index ffe84910f88a2..1ae5e871595c4 100644
--- a/flang/test/Semantics/OpenMP/scan2.f90
+++ b/flang/test/Semantics/OpenMP/scan2.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols -o - %s 2>&1 | FileCheck %s
 ! Check scan reduction
 
-! CHECK: MainProgram scope: omp_reduction
+! CHECK: MainProgram scope: OMP_REDUCTION
 program omp_reduction
   ! CHECK: i size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer i
diff --git a/flang/test/Semantics/OpenMP/symbol01.f90 b/flang/test/Semantics/OpenMP/symbol01.f90
index fbd9a0286c79b..c1f16d61ada1e 100644
--- a/flang/test/Semantics/OpenMP/symbol01.f90
+++ b/flang/test/Semantics/OpenMP/symbol01.f90
@@ -16,53 +16,53 @@ module md
   integer :: b
  end type myty
 end module md
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
  !REF: /md
  use :: md
- !DEF: /mm/c CommonBlockDetails
- !DEF: /mm/x (InCommonBlock) ObjectEntity REAL(4)
- !DEF: /mm/y (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /MM/c CommonBlockDetails
+ !DEF: /MM/x (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /MM/y (InCommonBlock) ObjectEntity REAL(4)
  common /c/x, y
- !REF: /mm/x
- !REF: /mm/y
+ !REF: /MM/x
+ !REF: /MM/y
  real x, y
- !DEF: /mm/myty Use
- !DEF: /mm/t ObjectEntity TYPE(myty)
+ !DEF: /MM/myty Use
+ !DEF: /MM/t ObjectEntity TYPE(myty)
  type(myty) :: t
- !DEF: /mm/b ObjectEntity INTEGER(4)
+ !DEF: /MM/b ObjectEntity INTEGER(4)
  integer b(10)
- !REF: /mm/t
+ !REF: /MM/t
  !REF: /md/myty/a
  t%a = 3.14
- !REF: /mm/t
+ !REF: /MM/t
  !REF: /md/myty/b
  t%b = 1
- !REF: /mm/b
+ !REF: /MM/b
  b = 2
- !DEF: /mm/a (Implicit) ObjectEntity REAL(4)
+ !DEF: /MM/a (Implicit) ObjectEntity REAL(4)
  a = 1.0
- !DEF: /mm/c (Implicit) ObjectEntity REAL(4)
+ !DEF: /MM/c (Implicit) ObjectEntity REAL(4)
  c = 2.0
 !$omp parallel do  private(a,t,/c/) shared(c)
- !DEF: /mm/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /MM/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  do i=1,10
-  !DEF: /mm/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
-  !DEF: /mm/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
-  !REF: /mm/OtherConstruct1/i
+  !DEF: /MM/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
+  !REF: /MM/OtherConstruct1/i
   a = a+b(i)
-  !DEF: /mm/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
+  !DEF: /MM/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
   !REF: /md/myty/a
-  !REF: /mm/OtherConstruct1/i
+  !REF: /MM/OtherConstruct1/i
   t%a = i
-  !DEF: /mm/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   y = 0.
-  !DEF: /mm/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
-  !REF: /mm/OtherConstruct1/a
-  !REF: /mm/OtherConstruct1/i
-  !REF: /mm/OtherConstruct1/y
+  !DEF: /MM/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !REF: /MM/OtherConstruct1/a
+  !REF: /MM/OtherConstruct1/i
+  !REF: /MM/OtherConstruct1/y
   x = a+i+y
-  !DEF: /mm/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
   c = 3.0
  end do
 end program
diff --git a/flang/test/Semantics/OpenMP/symbol05.f90 b/flang/test/Semantics/OpenMP/symbol05.f90
index fe01f15d20aa3..03e2c0ba3e0cc 100644
--- a/flang/test/Semantics/OpenMP/symbol05.f90
+++ b/flang/test/Semantics/OpenMP/symbol05.f90
@@ -31,10 +31,10 @@ subroutine foo
     end block
   end subroutine foo
 end module mm
-!DEF: /tt MainProgram
+!DEF: /TT MainProgram
 program tt
   !REF: /mm
   use :: mm
-  !DEF: /tt/foo (Subroutine) Use
+  !DEF: /TT/foo (Subroutine) Use
   call foo
 end program tt
diff --git a/flang/test/Semantics/OpenMP/symbol07.f90 b/flang/test/Semantics/OpenMP/symbol07.f90
index 86b7305411347..f5eb2127819bd 100644
--- a/flang/test/Semantics/OpenMP/symbol07.f90
+++ b/flang/test/Semantics/OpenMP/symbol07.f90
@@ -30,7 +30,7 @@ subroutine function_call_in_region
   !REF: /function_call_in_region/b
   print *, a, b
 end subroutine function_call_in_region
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !REF: /function_call_in_region
   call function_call_in_region
diff --git a/flang/test/Semantics/OpenMP/symbol09.f90 b/flang/test/Semantics/OpenMP/symbol09.f90
index 86b7305411347..f5eb2127819bd 100644
--- a/flang/test/Semantics/OpenMP/symbol09.f90
+++ b/flang/test/Semantics/OpenMP/symbol09.f90
@@ -30,7 +30,7 @@ subroutine function_call_in_region
   !REF: /function_call_in_region/b
   print *, a, b
 end subroutine function_call_in_region
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !REF: /function_call_in_region
   call function_call_in_region
diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index 95b7fb418367d..e3c2e0bf53f92 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -9,6 +9,6 @@ program main
 
 ! RUN: %flang_fc1 -fget-symbols-sources %s 2>&1 | FileCheck %s
 ! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
-! CHECK:main:{{.*}}getsymbols03-a.f90, 4, 9-13
+! CHECK:MAIN:{{.*}}getsymbols03-a.f90, 4, 9-13
 ! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
 ! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14
diff --git a/flang/test/Semantics/modproc01.f90 b/flang/test/Semantics/modproc01.f90
index 5f45362e95093..e565ddcfbe0b1 100644
--- a/flang/test/Semantics/modproc01.f90
+++ b/flang/test/Semantics/modproc01.f90
@@ -125,7 +125,7 @@ program test
   x = mf(3, "abc", pdt1(1,3)())
 !  call ms(mf)
 end program
-!CHECK:  MainProgram scope: test size=88 alignment=8
+!CHECK:  MainProgram scope: TEST size=88 alignment=8
 !CHECK:    mf, MODULE (Function): Use from mf in m
 !CHECK:    pdt1: Use from pdt1 in m
 !CHECK:    pdt2: Use from pdt2 in m
diff --git a/flang/test/Semantics/multi-programs04.f90 b/flang/test/Semantics/multi-programs04.f90
index 54b0235aa78f0..e69ac7325278e 100644
--- a/flang/test/Semantics/multi-programs04.f90
+++ b/flang/test/Semantics/multi-programs04.f90
@@ -4,6 +4,6 @@
 program m
 end
 !ERROR: A source file cannot contain more than one main program
-!ERROR: 'm' is already declared in this scoping unit
+!ERROR: 'M' is already declared in this scoping unit
 program m
 end
diff --git a/flang/test/Semantics/procinterface01.f90 b/flang/test/Semantics/procinterface01.f90
index 73040b0987bd0..7542bb8bb6fa4 100644
--- a/flang/test/Semantics/procinterface01.f90
+++ b/flang/test/Semantics/procinterface01.f90
@@ -159,35 +159,35 @@ end function logical
  tan = "?"
 end function tan
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /module1
  use :: module1
- !DEF: /main/derived1 Use
- !DEF: /main/instance ObjectEntity TYPE(derived1)
+ !DEF: /MAIN/derived1 Use
+ !DEF: /MAIN/instance ObjectEntity TYPE(derived1)
  type(derived1) :: instance
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p1
  if (instance%p1(1.)/=2.) print *, "p1 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p2
  if (instance%p2(1.)/=2.) print *, "p2 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p3
  if (.not.instance%p3(1.)) print *, "p3 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p4
  if (.not.instance%p4(1.)) print *, "p4 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p5
  if (instance%p5(1.)/=(5.,6.)) print *, "p5 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p6
  if (instance%p6(1.)/=2.) print *, "p6 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p7
  if (instance%p7(0.)/=1.) print *, "p7 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p8
  if (instance%p8(1.)/="a") print *, "p8 failed"
 end program main
diff --git a/flang/test/Semantics/resolve125.f90 b/flang/test/Semantics/resolve125.f90
index e040c006ec179..620c7d65578cd 100644
--- a/flang/test/Semantics/resolve125.f90
+++ b/flang/test/Semantics/resolve125.f90
@@ -43,7 +43,7 @@ subroutine reset
   end subroutine reset
 end module m2
 
-!CHECK: MainProgram scope: main
+!CHECK: MainProgram scope: MAIN
 !CHECK:       i: Use from i in m2
 !CHECK:       i2: Use from i2 in m2
 !CHECK:       init (Subroutine): Use from init in m2
@@ -61,4 +61,4 @@ program main
   else
     print *, "fail"
   end if
-end program main
\ No newline at end of file
+end program main
diff --git a/flang/test/Semantics/symbol03.f90 b/flang/test/Semantics/symbol03.f90
index a6b4b0bd15937..f50a0e91fd289 100644
--- a/flang/test/Semantics/symbol03.f90
+++ b/flang/test/Semantics/symbol03.f90
@@ -1,23 +1,23 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Test host association in internal subroutine of main program.
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/x ObjectEntity INTEGER(4)
+ !DEF: /MAIN/x ObjectEntity INTEGER(4)
  integer x
- !DEF: /main/s (Subroutine) Subprogram
+ !DEF: /MAIN/s (Subroutine) Subprogram
  call s
 contains
- !REF: /main/s
+ !REF: /MAIN/s
  subroutine s
-  !DEF: /main/s/y (Implicit) ObjectEntity REAL(4)
-  !DEF: /main/s/x HostAssoc INTEGER(4)
+  !DEF: /MAIN/s/y (Implicit) ObjectEntity REAL(4)
+  !DEF: /MAIN/s/x HostAssoc INTEGER(4)
   y = x
  contains
-  !DEF: /main/s/s2 (Subroutine) Subprogram
+  !DEF: /MAIN/s/s2 (Subroutine) Subprogram
   subroutine s2
-   !DEF: /main/s/s2/z (Implicit) ObjectEntity REAL(4)
-   !DEF: /main/s/s2/x HostAssoc INTEGER(4)
+   !DEF: /MAIN/s/s2/z (Implicit) ObjectEntity REAL(4)
+   !DEF: /MAIN/s/s2/x HostAssoc INTEGER(4)
    z = x
   end subroutine
  end subroutine
diff --git a/flang/test/Semantics/symbol06.f90 b/flang/test/Semantics/symbol06.f90
index bbd6d4d071c89..984ebcc1c651c 100644
--- a/flang/test/Semantics/symbol06.f90
+++ b/flang/test/Semantics/symbol06.f90
@@ -1,56 +1,56 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/t1 DerivedType
+ !DEF: /MAIN/t1 DerivedType
  type :: t1
-  !DEF: /main/t1/a1 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t1/a1 ObjectEntity INTEGER(4)
   integer :: a1
  end type
- !REF: /main/t1
- !DEF: /main/t2 DerivedType
+ !REF: /MAIN/t1
+ !DEF: /MAIN/t2 DerivedType
  type, extends(t1) :: t2
-  !DEF: /main/t2/a2 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t2/a2 ObjectEntity INTEGER(4)
   integer :: a2
  end type
- !REF: /main/t2
- !DEF: /main/t3 DerivedType
+ !REF: /MAIN/t2
+ !DEF: /MAIN/t3 DerivedType
  type, extends(t2) :: t3
-  !DEF: /main/t3/a3 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t3/a3 ObjectEntity INTEGER(4)
   integer :: a3
  end type
- !REF: /main/t3
- !DEF: /main/x3 ObjectEntity TYPE(t3)
+ !REF: /MAIN/t3
+ !DEF: /MAIN/x3 ObjectEntity TYPE(t3)
  type(t3) :: x3
- !DEF: /main/i ObjectEntity INTEGER(4)
+ !DEF: /MAIN/i ObjectEntity INTEGER(4)
  integer i
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t2/a2
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t2/a2
  i = x3%a2
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t1/a1
  i = x3%a1
- !REF: /main/i
- !REF: /main/x3
- !DEF: /main/t3/t2 (ParentComp) ObjectEntity TYPE(t2)
- !REF: /main/t2/a2
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !DEF: /MAIN/t3/t2 (ParentComp) ObjectEntity TYPE(t2)
+ !REF: /MAIN/t2/a2
  i = x3%t2%a2
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t3/t2
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t3/t2
+ !REF: /MAIN/t1/a1
  i = x3%t2%a1
- !REF: /main/i
- !REF: /main/x3
- !DEF: /main/t2/t1 (ParentComp) ObjectEntity TYPE(t1)
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !DEF: /MAIN/t2/t1 (ParentComp) ObjectEntity TYPE(t1)
+ !REF: /MAIN/t1/a1
  i = x3%t1%a1
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t3/t2
- !REF: /main/t2/t1
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t3/t2
+ !REF: /MAIN/t2/t1
+ !REF: /MAIN/t1/a1
  i = x3%t2%t1%a1
 end program
 
diff --git a/flang/test/Semantics/symbol07.f90 b/flang/test/Semantics/symbol07.f90
index f3cc934e51b16..b3e92f1465fa6 100644
--- a/flang/test/Semantics/symbol07.f90
+++ b/flang/test/Semantics/symbol07.f90
@@ -1,40 +1,40 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  implicit complex(z)
- !DEF: /main/t DerivedType
+ !DEF: /MAIN/t DerivedType
  type :: t
-  !DEF: /main/t/re ObjectEntity REAL(4)
+  !DEF: /MAIN/t/re ObjectEntity REAL(4)
   real :: re
-  !DEF: /main/t/im ObjectEntity REAL(4)
+  !DEF: /MAIN/t/im ObjectEntity REAL(4)
   real :: im
  end type
- !DEF: /main/z1 ObjectEntity COMPLEX(4)
+ !DEF: /MAIN/z1 ObjectEntity COMPLEX(4)
  complex z1
- !REF: /main/t
- !DEF: /main/w ObjectEntity TYPE(t)
+ !REF: /MAIN/t
+ !DEF: /MAIN/w ObjectEntity TYPE(t)
  type(t) :: w
- !DEF: /main/x ObjectEntity REAL(4)
- !DEF: /main/y ObjectEntity REAL(4)
+ !DEF: /MAIN/x ObjectEntity REAL(4)
+ !DEF: /MAIN/y ObjectEntity REAL(4)
  real x, y
- !REF: /main/x
- !REF: /main/z1
+ !REF: /MAIN/x
+ !REF: /MAIN/z1
  x = z1%re
- !REF: /main/y
- !REF: /main/z1
+ !REF: /MAIN/y
+ !REF: /MAIN/z1
  y = z1%im
- !DEF: /main/z2 (Implicit) ObjectEntity COMPLEX(4)
- !REF: /main/x
+ !DEF: /MAIN/z2 (Implicit) ObjectEntity COMPLEX(4)
+ !REF: /MAIN/x
  z2%re = x
- !REF: /main/z2
- !REF: /main/y
+ !REF: /MAIN/z2
+ !REF: /MAIN/y
  z2%im = y
- !REF: /main/x
- !REF: /main/w
- !REF: /main/t/re
+ !REF: /MAIN/x
+ !REF: /MAIN/w
+ !REF: /MAIN/t/re
  x = w%re
- !REF: /main/y
- !REF: /main/w
- !REF: /main/t/im
+ !REF: /MAIN/y
+ !REF: /MAIN/w
+ !REF: /MAIN/t/im
  y = w%im
 end program
diff --git a/flang/test/Semantics/symbol08.f90 b/flang/test/Semantics/symbol08.f90
index 61dab798955c5..e2e9959d0a591 100644
--- a/flang/test/Semantics/symbol08.f90
+++ b/flang/test/Semantics/symbol08.f90
@@ -1,15 +1,15 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/x POINTER ObjectEntity REAL(4)
+ !DEF: /MAIN/x POINTER ObjectEntity REAL(4)
  pointer :: x
- !REF: /main/x
+ !REF: /MAIN/x
  real x
- !DEF: /main/y EXTERNAL, POINTER (Function) ProcEntity REAL(4)
+ !DEF: /MAIN/y EXTERNAL, POINTER (Function) ProcEntity REAL(4)
  pointer :: y
- !REF: /main/y
+ !REF: /MAIN/y
  procedure (real) :: y
- !DEF: /main/z (Implicit) ObjectEntity REAL(4)
- !REF: /main/y
+ !DEF: /MAIN/z (Implicit) ObjectEntity REAL(4)
+ !REF: /MAIN/y
  z = y()
 end program
diff --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index df10942e6af2d..5cc1cf58b4f21 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -249,15 +249,15 @@ subroutine ext2
 !DEF: /ext3 (Subroutine) Subprogram
 subroutine ext3
 end subroutine
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /m
  use :: m
- !DEF: /main/pdt1 Use
- !DEF: /main/pdt1y ObjectEntity TYPE(pdt1(k=2_4))
+ !DEF: /MAIN/pdt1 Use
+ !DEF: /MAIN/pdt1y ObjectEntity TYPE(pdt1(k=2_4))
  type(pdt1(2)) :: pdt1y
- !DEF: /main/pdt2 Use
- !DEF: /main/pdt2y ObjectEntity TYPE(pdt2(k=2_4))
+ !DEF: /MAIN/pdt2 Use
+ !DEF: /MAIN/pdt2y ObjectEntity TYPE(pdt2(k=2_4))
  type(pdt2(2)) :: pdt2y
  print *, "compiled"
 end program
diff --git a/flang/test/Semantics/symbol16.f90 b/flang/test/Semantics/symbol16.f90
index 7a46092c36b53..c29831a5dfa90 100644
--- a/flang/test/Semantics/symbol16.f90
+++ b/flang/test/Semantics/symbol16.f90
@@ -1,18 +1,18 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Statement functions
 
-!DEF: /p1 MainProgram
+!DEF: /P1 MainProgram
 program p1
- !DEF: /p1/f (Function, StmtFunction) Subprogram INTEGER(4)
- !DEF: /p1/i ObjectEntity INTEGER(4)
- !DEF: /p1/j ObjectEntity INTEGER(4)
+ !DEF: /P1/f (Function, StmtFunction) Subprogram INTEGER(4)
+ !DEF: /P1/i ObjectEntity INTEGER(4)
+ !DEF: /P1/j ObjectEntity INTEGER(4)
  integer f, i, j
- !REF: /p1/f
- !REF: /p1/i
- !DEF: /p1/f/i ObjectEntity INTEGER(4)
+ !REF: /P1/f
+ !REF: /P1/i
+ !DEF: /P1/f/i ObjectEntity INTEGER(4)
  f(i) = i + 1
- !REF: /p1/j
- !REF: /p1/f
+ !REF: /P1/j
+ !REF: /P1/f
  j = f(2)
 end program
 
diff --git a/flang/test/Semantics/symbol17.f90 b/flang/test/Semantics/symbol17.f90
index 434f124509a32..51f3afb19692d 100644
--- a/flang/test/Semantics/symbol17.f90
+++ b/flang/test/Semantics/symbol17.f90
@@ -1,44 +1,44 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Forward references to derived types (non-error cases)
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/t1 DerivedType
+ !DEF: /MAIN/t1 DerivedType
  type :: t1
-  !DEF: /main/t2 DerivedType
-  !DEF: /main/t1/t1a ALLOCATABLE ObjectEntity TYPE(t2)
+  !DEF: /MAIN/t2 DerivedType
+  !DEF: /MAIN/t1/t1a ALLOCATABLE ObjectEntity TYPE(t2)
   type(t2), allocatable :: t1a
-  !REF: /main/t2
-  !DEF: /main/t1/t1p POINTER ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t1/t1p POINTER ObjectEntity TYPE(t2)
   type(t2), pointer :: t1p
  end type
- !REF: /main/t2
+ !REF: /MAIN/t2
  type :: t2
-  !REF: /main/t2
-  !DEF: /main/t2/t2a ALLOCATABLE ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t2/t2a ALLOCATABLE ObjectEntity TYPE(t2)
   type(t2), allocatable :: t2a
-  !REF: /main/t2
-  !DEF: /main/t2/t2p POINTER ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t2/t2p POINTER ObjectEntity TYPE(t2)
   type(t2), pointer :: t2p
  end type
- !REF: /main/t1
- !DEF: /main/t1x TARGET ObjectEntity TYPE(t1)
+ !REF: /MAIN/t1
+ !DEF: /MAIN/t1x TARGET ObjectEntity TYPE(t1)
  type(t1), target :: t1x
- !REF: /main/t1x
- !REF: /main/t1/t1a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
  allocate(t1x%t1a)
- !REF: /main/t1x
- !REF: /main/t1/t1p
- !REF: /main/t1/t1a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1p
+ !REF: /MAIN/t1/t1a
  t1x%t1p => t1x%t1a
- !REF: /main/t1x
- !REF: /main/t1/t1a
- !REF: /main/t2/t2a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
+ !REF: /MAIN/t2/t2a
  allocate(t1x%t1a%t2a)
- !REF: /main/t1x
- !REF: /main/t1/t1a
- !REF: /main/t2/t2p
- !REF: /main/t2/t2a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
+ !REF: /MAIN/t2/t2p
+ !REF: /MAIN/t2/t2a
  t1x%t1a%t2p => t1x%t1a%t2a
 end program
 !DEF: /f1/fwd DerivedType
diff --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
index a37792bce21d7..bfe25ff4df25f 100644
--- a/flang/test/Semantics/symbol18.f90
+++ b/flang/test/Semantics/symbol18.f90
@@ -2,21 +2,21 @@
 
 ! Intrinsic function in type declaration statement: type is ignored
 
-!DEF: /p1 MainProgram
+!DEF: /P1 MainProgram
 program p1
- !DEF: /p1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
+ !DEF: /P1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
  integer cos
- !DEF: /p1/y (Implicit) ObjectEntity REAL(4)
- !REF: /p1/cos
- !DEF: /p1/x (Implicit) ObjectEntity REAL(4)
+ !DEF: /P1/y (Implicit) ObjectEntity REAL(4)
+ !REF: /P1/cos
+ !DEF: /P1/x (Implicit) ObjectEntity REAL(4)
  y = cos(x)
- !REF: /p1/y
- !DEF: /p1/sin ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
- !REF: /p1/x
+ !REF: /P1/y
+ !DEF: /P1/sin ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+ !REF: /P1/x
  y = sin(x)
- !REF: /p1/y
+ !REF: /P1/y
  !DEF: /f EXTERNAL (Function, Implicit) ProcEntity REAL(4)
- !REF: /p1/x
+ !REF: /P1/x
  y = f(x)
 end program
 
diff --git a/flang/test/Semantics/symbol20.f90 b/flang/test/Semantics/symbol20.f90
index 8c82776933321..20bd1bdbf2a7c 100644
--- a/flang/test/Semantics/symbol20.f90
+++ b/flang/test/Semantics/symbol20.f90
@@ -32,16 +32,16 @@ subroutine bar
   print *, "in bar"
  end subroutine
 end module
-!DEF: /demo MainProgram
+!DEF: /DEMO MainProgram
 program demo
  !REF: /m
  use :: m
- !DEF: /demo/bar (Subroutine) Use
- !DEF: /demo/p EXTERNAL, POINTER (Subroutine) ProcEntity
+ !DEF: /DEMO/bar (Subroutine) Use
+ !DEF: /DEMO/p EXTERNAL, POINTER (Subroutine) ProcEntity
  procedure(bar), pointer :: p
- !REF: /demo/p
- !DEF: /demo/foo (Function) Use
+ !REF: /DEMO/p
+ !DEF: /DEMO/foo (Function) Use
  p => foo()
- !REF: /demo/p
+ !REF: /DEMO/p
  call p
 end program
diff --git a/flang/test/Semantics/symbol25.f90 b/flang/test/Semantics/symbol25.f90
index ac3dd37ef92eb..a131bf899340f 100644
--- a/flang/test/Semantics/symbol25.f90
+++ b/flang/test/Semantics/symbol25.f90
@@ -38,23 +38,23 @@ subroutine inner1
   end subroutine inner1
  end subroutine outer
 end module m
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /m
  use :: m
  !REF: /m/specific1
  call generic
- !DEF: /main/inner2 (Subroutine) Subprogram
+ !DEF: /MAIN/inner2 (Subroutine) Subprogram
  call inner2
 contains
- !REF: /main/inner2
+ !REF: /MAIN/inner2
  subroutine inner2
-  !DEF: /main/inner2/generic (Subroutine) Generic
+  !DEF: /MAIN/inner2/generic (Subroutine) Generic
   interface generic
-   !DEF: /main/specific2 (Subroutine) Use
+   !DEF: /MAIN/specific2 (Subroutine) Use
    module procedure :: specific2
   end interface
-  !REF: /main/specific2
+  !REF: /MAIN/specific2
   call generic
  end subroutine inner2
 end program
diff --git a/flang/test/Semantics/symbol26.f90 b/flang/test/Semantics/symbol26.f90
index f5e95853ca099..e04cd32df57c4 100644
--- a/flang/test/Semantics/symbol26.f90
+++ b/flang/test/Semantics/symbol26.f90
@@ -8,16 +8,16 @@ module m
  !DEF: /m/j PUBLIC (Implicit, InNamelist) ObjectEntity INTEGER(4)
  namelist/a/j
 end module m
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/j (Implicit) ObjectEntity INTEGER(4)
+ !DEF: /MAIN/j (Implicit) ObjectEntity INTEGER(4)
  j = 1
 contains
- !DEF: /main/inner (Subroutine) Subprogram
+ !DEF: /MAIN/inner (Subroutine) Subprogram
  subroutine inner
   !REF: /m
   use :: m
-  !DEF: /main/inner/j (Implicit, InNamelist) Use INTEGER(4)
+  !DEF: /MAIN/inner/j (Implicit, InNamelist) Use INTEGER(4)
   j = 2
  end subroutine
 end program
diff --git a/flang/test/Transforms/DoConcurrent/basic_host.f90 b/flang/test/Transforms/DoConcurrent/basic_host.f90
index 12f63031cbaee..6f24b346e3fb9 100644
--- a/flang/test/Transforms/DoConcurrent/basic_host.f90
+++ b/flang/test/Transforms/DoConcurrent/basic_host.f90
@@ -5,7 +5,7 @@
 ! RUN: bbc -emit-hlfir -fopenmp -fdo-concurrent-to-openmp=host %s -o - \
 ! RUN:   | FileCheck %s
  
-! CHECK-LABEL: do_concurrent_basic
+! CHECK-LABEL: DO_CONCURRENT_BASIC
 program do_concurrent_basic
     ! CHECK: %[[ARR:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFEa"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 

>From 632cd4b1351cbb57efbdf3cb8fe9435211004a0a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:24:17 -0400
Subject: [PATCH 4/9] More test changes

---
 .../test/Semantics/OpenACC/acc-symbols01.f90  | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/flang/test/Semantics/OpenACC/acc-symbols01.f90 b/flang/test/Semantics/OpenACC/acc-symbols01.f90
index ea22f41197aa7..51a7a3a23e8ce 100644
--- a/flang/test/Semantics/OpenACC/acc-symbols01.f90
+++ b/flang/test/Semantics/OpenACC/acc-symbols01.f90
@@ -1,24 +1,24 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenacc
 
 !DEF: /MM MainProgram
-program mm
-  !DEF: /mm/x ObjectEntity REAL(4)
-  !DEF: /mm/y ObjectEntity REAL(4)
+program MM
+  !DEF: /MM/x ObjectEntity REAL(4)
+  !DEF: /MM/y ObjectEntity REAL(4)
   real x, y
-  !DEF: /mm/a ObjectEntity INTEGER(4)
-  !DEF: /mm/b ObjectEntity INTEGER(4)
-  !DEF: /mm/c ObjectEntity INTEGER(4)
-  !DEF: /mm/i ObjectEntity INTEGER(4)
+  !DEF: /MM/a ObjectEntity INTEGER(4)
+  !DEF: /MM/b ObjectEntity INTEGER(4)
+  !DEF: /MM/c ObjectEntity INTEGER(4)
+  !DEF: /MM/i ObjectEntity INTEGER(4)
   integer a(10), b(10), c(10), i
-  !REF: /mm/b
+  !REF: /MM/b
   b = 2
  !$acc parallel present(c) firstprivate(b) private(a)
  !$acc loop
-  !REF: /mm/i
+  !REF: /MM/i
   do i=1,10
-   !REF: /mm/a
-   !REF: /mm/i
-   !REF: /mm/b
+   !REF: /MM/a
+   !REF: /MM/i
+   !REF: /MM/b
    a(i) = b(i)
   end do
  !$acc end parallel

>From ce7b837b6554c62ad459729674ddb0cb84f49747 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:37:07 -0400
Subject: [PATCH 5/9] more tests

---
 flang/test/Semantics/OpenMP/do-schedule03.f90      |  4 ++--
 flang/test/Semantics/OpenMP/do01-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do04-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do05-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do06-positivecases.f90 |  4 ++--
 flang/test/Semantics/OpenMP/do11.f90               | 14 +++++++-------
 6 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/flang/test/Semantics/OpenMP/do-schedule03.f90 b/flang/test/Semantics/OpenMP/do-schedule03.f90
index 375f3920b57ad..05602ca57e4a9 100644
--- a/flang/test/Semantics/OpenMP/do-schedule03.f90
+++ b/flang/test/Semantics/OpenMP/do-schedule03.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - xx.
   !DEF: /OMPDOSCHEDULE MainProgram
-program ompdoschedule
+program OMPDOSCHEDULE
   !DEF: /OMPDOSCHEDULE/a ObjectEntity REAL(4)
   !DEF: /OMPDOSCHEDULE/y ObjectEntity REAL(4)
   !DEF: /OMPDOSCHEDULE/z ObjectEntity REAL(4)
@@ -25,4 +25,4 @@ program ompdoschedule
     y(i) = z(i-1) + a(i)
   end do
   !$omp end do
-end program ompdoschedule
+end program OMPDOSCHEDULE
diff --git a/flang/test/Semantics/OpenMP/do01-positivecase.f90 b/flang/test/Semantics/OpenMP/do01-positivecase.f90
index a4a28f6530d7b..50a6870f43896 100644
--- a/flang/test/Semantics/OpenMP/do01-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do01-positivecase.f90
@@ -5,7 +5,7 @@
 ! A positive case
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   integer i
 
@@ -16,4 +16,4 @@ program omp_do
   end do
   !$omp end do
 
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do04-positivecase.f90 b/flang/test/Semantics/OpenMP/do04-positivecase.f90
index 0b0c661687dc2..51b69fce3c7cc 100644
--- a/flang/test/Semantics/OpenMP/do04-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do04-positivecase.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop Constructs
 
 !DEF: /OMP_DO1 MainProgram
-program omp_do1
+program OMP_DO1
   !DEF: /OMP_DO1/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO1/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
@@ -19,4 +19,4 @@ program omp_do1
     end do
   end do
   !$omp end do
-end program omp_do1
+end program OMP_DO1
diff --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index 07b12dd1d4d23..d4eb1fd6bc3da 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -4,7 +4,7 @@
 ! A positive case
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/n ObjectEntity INTEGER(4)
   integer i,n
@@ -71,4 +71,4 @@ program omp_do
     !REF: /OMP_DO/OtherConstruct7/i
     if(i < 5) cycle
   end do
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do06-positivecases.f90 b/flang/test/Semantics/OpenMP/do06-positivecases.f90
index 4d45800298a29..dfb1d999bbc53 100644
--- a/flang/test/Semantics/OpenMP/do06-positivecases.f90
+++ b/flang/test/Semantics/OpenMP/do06-positivecases.f90
@@ -6,7 +6,7 @@
 
 ! A positive case
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
@@ -20,4 +20,4 @@ program omp_do
       !$omp end ordered
     end do
   !$omp end do
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do11.f90 b/flang/test/Semantics/OpenMP/do11.f90
index 886a9f7ae6081..472048d684276 100644
--- a/flang/test/Semantics/OpenMP/do11.f90
+++ b/flang/test/Semantics/OpenMP/do11.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop Constructs
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
@@ -19,17 +19,17 @@ program omp_do
     end do
   end do
   !$omp end do
-end program omp_do
+end program OMP_DO
 
-!DEF: /OMP_DO2 (Subroutine)Subprogram
+!DEF: /omp_do2 (Subroutine)Subprogram
 subroutine omp_do2
-  !DEF: /OMP_DO2/i ObjectEntity INTEGER(4)
-  !DEF: /OMP_DO2/k ObjectEntity INTEGER(4)
+  !DEF: /omp_do2/i ObjectEntity INTEGER(4)
+  !DEF: /omp_do2/k ObjectEntity INTEGER(4)
   integer :: i = 0, k
   !$omp do
-  !DEF: /OMP_DO2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /omp_do2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /OMP_DO2/OtherConstruct1/i
+    !REF: /omp_do2/OtherConstruct1/i
     print *, "it", i
   end do
   !$omp end do

>From 3e3d5d489bd2d299cb14bdeca728856f8bd59f18 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 23:03:25 -0400
Subject: [PATCH 6/9] More test changes

---
 flang/test/Semantics/OpenMP/declare-target03.f90 | 4 ++--
 flang/test/Semantics/OpenMP/do12.f90             | 2 +-
 flang/test/Semantics/OpenMP/do14.f90             | 4 ++--
 flang/test/Semantics/OpenMP/do17.f90             | 4 ++--
 flang/test/Semantics/OpenMP/reduction08.f90      | 4 ++--
 flang/test/Semantics/OpenMP/reduction09.f90      | 4 ++--
 flang/test/Semantics/OpenMP/symbol01.f90         | 2 +-
 flang/test/Semantics/OpenMP/symbol05.f90         | 4 ++--
 flang/test/Semantics/OpenMP/symbol07.f90         | 4 ++--
 flang/test/Semantics/OpenMP/symbol09.f90         | 4 ++--
 flang/test/Semantics/OpenMP/threadprivate03.f90  | 4 ++--
 flang/test/Semantics/long-name.f90               | 2 +-
 flang/test/Semantics/pointer01.f90               | 1 -
 flang/test/Semantics/procinterface01.f90         | 4 ++--
 flang/test/Semantics/resolve05.f90               | 1 -
 flang/test/Semantics/symbol03.f90                | 2 +-
 flang/test/Semantics/symbol06.f90                | 2 +-
 flang/test/Semantics/symbol07.f90                | 2 +-
 flang/test/Semantics/symbol08.f90                | 2 +-
 flang/test/Semantics/symbol15.f90                | 2 +-
 flang/test/Semantics/symbol16.f90                | 2 +-
 flang/test/Semantics/symbol17.f90                | 2 +-
 flang/test/Semantics/symbol18.f90                | 2 +-
 flang/test/Semantics/symbol20.f90                | 2 +-
 flang/test/Semantics/symbol25.f90                | 2 +-
 flang/test/Semantics/symbol26.f90                | 2 +-
 26 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index 64a299d78224a..b3d6551ac85de 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -16,7 +16,7 @@ program main
   !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
-  !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
+  ! This is now allowed: "main" is implicitly declared symbol separate
+  ! from the main program symbol
   !$omp declare target (main)
 end
diff --git a/flang/test/Semantics/OpenMP/do12.f90 b/flang/test/Semantics/OpenMP/do12.f90
index f36a230426197..06055b7572a60 100644
--- a/flang/test/Semantics/OpenMP/do12.f90
+++ b/flang/test/Semantics/OpenMP/do12.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /OMP_CYCLE MainProgram
-program omp_cycle
+program OMP_CYCLE
   !$omp do  collapse(1)
   !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
diff --git a/flang/test/Semantics/OpenMP/do14.f90 b/flang/test/Semantics/OpenMP/do14.f90
index b508d32d90201..e17647394fff7 100644
--- a/flang/test/Semantics/OpenMP/do14.f90
+++ b/flang/test/Semantics/OpenMP/do14.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /OMP_CYCLE MainProgram
-program omp_cycle
+program OMP_CYCLE
   !$omp do  collapse(1)
   !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
@@ -88,4 +88,4 @@ program omp_cycle
     end do foo1
   end do foo
   !$omp end do
-end program omp_cycle
+end program OMP_CYCLE
diff --git a/flang/test/Semantics/OpenMP/do17.f90 b/flang/test/Semantics/OpenMP/do17.f90
index 0a4cf46d2d6af..cac11f215f074 100644
--- a/flang/test/Semantics/OpenMP/do17.f90
+++ b/flang/test/Semantics/OpenMP/do17.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /TEST MainProgram
-program test
+program TEST
  !DEF: /TEST/i ObjectEntity INTEGER(4)
  !DEF: /TEST/j ObjectEntity INTEGER(4)
  !DEF: /TEST/k ObjectEntity INTEGER(4)
@@ -54,4 +54,4 @@ program test
   end do foo1
  end do foo
 !$omp end do
-end program test
+end program TEST
diff --git a/flang/test/Semantics/OpenMP/reduction08.f90 b/flang/test/Semantics/OpenMP/reduction08.f90
index 1cf2838fe765a..b4a81e644c1e7 100644
--- a/flang/test/Semantics/OpenMP/reduction08.f90
+++ b/flang/test/Semantics/OpenMP/reduction08.f90
@@ -3,7 +3,7 @@
 ! 2.15.3.6 Reduction Clause Positive cases
 
 !DEF: /OMP_REDUCTION MainProgram
-program omp_reduction
+program OMP_REDUCTION
   !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
   !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
@@ -60,4 +60,4 @@ program omp_reduction
   end do
   !$omp end parallel do
 
-end program omp_reduction
+end program OMP_REDUCTION
diff --git a/flang/test/Semantics/OpenMP/reduction09.f90 b/flang/test/Semantics/OpenMP/reduction09.f90
index 4b1ca116d675b..ca60805e8c416 100644
--- a/flang/test/Semantics/OpenMP/reduction09.f90
+++ b/flang/test/Semantics/OpenMP/reduction09.f90
@@ -2,7 +2,7 @@
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases.
 !DEF: /OMP_REDUCTION MainProgram
-program omp_reduction
+program OMP_REDUCTION
   !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
   !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
@@ -73,4 +73,4 @@ program omp_reduction
     k = k+1
   end do
   !$omp end do
-end program omp_reduction
+end program OMP_REDUCTION
diff --git a/flang/test/Semantics/OpenMP/symbol01.f90 b/flang/test/Semantics/OpenMP/symbol01.f90
index c1f16d61ada1e..74fb420cc517e 100644
--- a/flang/test/Semantics/OpenMP/symbol01.f90
+++ b/flang/test/Semantics/OpenMP/symbol01.f90
@@ -17,7 +17,7 @@ module md
  end type myty
 end module md
 !DEF: /MM MainProgram
-program mm
+program MM
  !REF: /md
  use :: md
  !DEF: /MM/c CommonBlockDetails
diff --git a/flang/test/Semantics/OpenMP/symbol05.f90 b/flang/test/Semantics/OpenMP/symbol05.f90
index 03e2c0ba3e0cc..4f3d1926013dc 100644
--- a/flang/test/Semantics/OpenMP/symbol05.f90
+++ b/flang/test/Semantics/OpenMP/symbol05.f90
@@ -32,9 +32,9 @@ subroutine foo
   end subroutine foo
 end module mm
 !DEF: /TT MainProgram
-program tt
+program TT
   !REF: /mm
   use :: mm
   !DEF: /TT/foo (Subroutine) Use
   call foo
-end program tt
+end program TT
diff --git a/flang/test/Semantics/OpenMP/symbol07.f90 b/flang/test/Semantics/OpenMP/symbol07.f90
index f5eb2127819bd..1b0c25b7a04b0 100644
--- a/flang/test/Semantics/OpenMP/symbol07.f90
+++ b/flang/test/Semantics/OpenMP/symbol07.f90
@@ -31,7 +31,7 @@ subroutine function_call_in_region
   print *, a, b
 end subroutine function_call_in_region
 !DEF: /MM MainProgram
-program mm
+program MM
   !REF: /function_call_in_region
   call function_call_in_region
-end program mm
+end program MM
diff --git a/flang/test/Semantics/OpenMP/symbol09.f90 b/flang/test/Semantics/OpenMP/symbol09.f90
index f5eb2127819bd..1b0c25b7a04b0 100644
--- a/flang/test/Semantics/OpenMP/symbol09.f90
+++ b/flang/test/Semantics/OpenMP/symbol09.f90
@@ -31,7 +31,7 @@ subroutine function_call_in_region
   print *, a, b
 end subroutine function_call_in_region
 !DEF: /MM MainProgram
-program mm
+program MM
   !REF: /function_call_in_region
   call function_call_in_region
-end program mm
+end program MM
diff --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index 81e26ee327a9d..9f222773ad432 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -13,8 +13,8 @@ program main
   !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
-  !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
+  ! This is now allowed, since "main" is implicitly declared symbol,
+  ! separate from the main program symbol.
   !$omp threadprivate(main)
 
   !ERROR: The entity with PARAMETER attribute cannot be in a THREADPRIVATE directive
diff --git a/flang/test/Semantics/long-name.f90 b/flang/test/Semantics/long-name.f90
index 44899b13edd5a..d5a795113e204 100644
--- a/flang/test/Semantics/long-name.f90
+++ b/flang/test/Semantics/long-name.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror -pedantic
 
-!PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
+!PORTABILITY: AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEEFFFFFFFFFFGGG1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
 program aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1
 
   !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
diff --git a/flang/test/Semantics/pointer01.f90 b/flang/test/Semantics/pointer01.f90
index eaa2426dd77e3..79d6016a6af46 100644
--- a/flang/test/Semantics/pointer01.f90
+++ b/flang/test/Semantics/pointer01.f90
@@ -7,7 +7,6 @@ subroutine msubr
 end module
 program main
   use m
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   pointer main
   !ERROR: Cannot change POINTER attribute on use-associated 'mobj'
   pointer mobj
diff --git a/flang/test/Semantics/procinterface01.f90 b/flang/test/Semantics/procinterface01.f90
index 7542bb8bb6fa4..70f4a889d6809 100644
--- a/flang/test/Semantics/procinterface01.f90
+++ b/flang/test/Semantics/procinterface01.f90
@@ -160,7 +160,7 @@ end function logical
 end function tan
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /module1
  use :: module1
  !DEF: /MAIN/derived1 Use
@@ -190,4 +190,4 @@ program main
  !REF: /MAIN/instance
  !REF: /module1/derived1/p8
  if (instance%p8(1.)/="a") print *, "p8 failed"
-end program main
+end program MAIN
diff --git a/flang/test/Semantics/resolve05.f90 b/flang/test/Semantics/resolve05.f90
index 0c9877af9b4e2..7b142d2ebd613 100644
--- a/flang/test/Semantics/resolve05.f90
+++ b/flang/test/Semantics/resolve05.f90
@@ -1,6 +1,5 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 program p
-  !PORTABILITY: Name 'p' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   integer :: p
 end
 module m
diff --git a/flang/test/Semantics/symbol03.f90 b/flang/test/Semantics/symbol03.f90
index f50a0e91fd289..62472495d9736 100644
--- a/flang/test/Semantics/symbol03.f90
+++ b/flang/test/Semantics/symbol03.f90
@@ -2,7 +2,7 @@
 ! Test host association in internal subroutine of main program.
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/x ObjectEntity INTEGER(4)
  integer x
  !DEF: /MAIN/s (Subroutine) Subprogram
diff --git a/flang/test/Semantics/symbol06.f90 b/flang/test/Semantics/symbol06.f90
index 984ebcc1c651c..b45edabcd5318 100644
--- a/flang/test/Semantics/symbol06.f90
+++ b/flang/test/Semantics/symbol06.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/t1 DerivedType
  type :: t1
   !DEF: /MAIN/t1/a1 ObjectEntity INTEGER(4)
diff --git a/flang/test/Semantics/symbol07.f90 b/flang/test/Semantics/symbol07.f90
index b3e92f1465fa6..e1d8257b9e190 100644
--- a/flang/test/Semantics/symbol07.f90
+++ b/flang/test/Semantics/symbol07.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  implicit complex(z)
  !DEF: /MAIN/t DerivedType
  type :: t
diff --git a/flang/test/Semantics/symbol08.f90 b/flang/test/Semantics/symbol08.f90
index e2e9959d0a591..933ff6d0c2ba8 100644
--- a/flang/test/Semantics/symbol08.f90
+++ b/flang/test/Semantics/symbol08.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/x POINTER ObjectEntity REAL(4)
  pointer :: x
  !REF: /MAIN/x
diff --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index 5cc1cf58b4f21..79a45491306ef 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -250,7 +250,7 @@ subroutine ext2
 subroutine ext3
 end subroutine
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /m
  use :: m
  !DEF: /MAIN/pdt1 Use
diff --git a/flang/test/Semantics/symbol16.f90 b/flang/test/Semantics/symbol16.f90
index c29831a5dfa90..547c4624d4cdb 100644
--- a/flang/test/Semantics/symbol16.f90
+++ b/flang/test/Semantics/symbol16.f90
@@ -2,7 +2,7 @@
 ! Statement functions
 
 !DEF: /P1 MainProgram
-program p1
+program P1
  !DEF: /P1/f (Function, StmtFunction) Subprogram INTEGER(4)
  !DEF: /P1/i ObjectEntity INTEGER(4)
  !DEF: /P1/j ObjectEntity INTEGER(4)
diff --git a/flang/test/Semantics/symbol17.f90 b/flang/test/Semantics/symbol17.f90
index 51f3afb19692d..a0d916e55cfa4 100644
--- a/flang/test/Semantics/symbol17.f90
+++ b/flang/test/Semantics/symbol17.f90
@@ -2,7 +2,7 @@
 ! Forward references to derived types (non-error cases)
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/t1 DerivedType
  type :: t1
   !DEF: /MAIN/t2 DerivedType
diff --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
index bfe25ff4df25f..6e41bb5db91ee 100644
--- a/flang/test/Semantics/symbol18.f90
+++ b/flang/test/Semantics/symbol18.f90
@@ -3,7 +3,7 @@
 ! Intrinsic function in type declaration statement: type is ignored
 
 !DEF: /P1 MainProgram
-program p1
+program P1
  !DEF: /P1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
  integer cos
  !DEF: /P1/y (Implicit) ObjectEntity REAL(4)
diff --git a/flang/test/Semantics/symbol20.f90 b/flang/test/Semantics/symbol20.f90
index 20bd1bdbf2a7c..bf3aff489b3b9 100644
--- a/flang/test/Semantics/symbol20.f90
+++ b/flang/test/Semantics/symbol20.f90
@@ -33,7 +33,7 @@ subroutine bar
  end subroutine
 end module
 !DEF: /DEMO MainProgram
-program demo
+program DEMO
  !REF: /m
  use :: m
  !DEF: /DEMO/bar (Subroutine) Use
diff --git a/flang/test/Semantics/symbol25.f90 b/flang/test/Semantics/symbol25.f90
index a131bf899340f..ac47a19eae8cc 100644
--- a/flang/test/Semantics/symbol25.f90
+++ b/flang/test/Semantics/symbol25.f90
@@ -39,7 +39,7 @@ end subroutine inner1
  end subroutine outer
 end module m
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /m
  use :: m
  !REF: /m/specific1
diff --git a/flang/test/Semantics/symbol26.f90 b/flang/test/Semantics/symbol26.f90
index e04cd32df57c4..dded4b632c654 100644
--- a/flang/test/Semantics/symbol26.f90
+++ b/flang/test/Semantics/symbol26.f90
@@ -9,7 +9,7 @@ module m
  namelist/a/j
 end module m
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/j (Implicit) ObjectEntity INTEGER(4)
  j = 1
 contains

>From 37dbcbf9c27aa973b939dfa45a709e8307937405 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 14:28:06 -0400
Subject: [PATCH 7/9] More test changes

---
 flang/test/Semantics/getsymbols03-a.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index e3c2e0bf53f92..5c5e87575a9cb 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -8,7 +8,7 @@ program main
 end program
 
 ! RUN: %flang_fc1 -fget-symbols-sources %s 2>&1 | FileCheck %s
-! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
 ! CHECK:MAIN:{{.*}}getsymbols03-a.f90, 4, 9-13
+! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
 ! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
 ! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14

>From 8cd6fa1d597d328effcbbd1544d3e614e5ad8377 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 15:06:18 -0400
Subject: [PATCH 8/9] Tweaked the error message and updated the tests
 accordingly

---
 flang/lib/Semantics/check-omp-structure.cpp      | 3 +--
 flang/test/Semantics/OpenMP/declare-target03.f90 | 2 +-
 flang/test/Semantics/OpenMP/threadprivate03.f90  | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2425265e196c6..e4a94efcc6b55 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -1156,8 +1156,7 @@ void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar(
                         (sym->has<MainProgramDetails>() ||
                             sym->has<ModuleDetails>())) {
                       context_.Say(name->source,
-                          "The module name or main program name cannot be in a "
-                          "%s "
+                          "The module name cannot be in a %s "
                           "directive"_err_en_US,
                           ContextDirectiveAsFortran());
                     } else if (!IsSaved(*name->symbol) &&
diff --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index b3d6551ac85de..48cfc68393873 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -13,7 +13,7 @@ subroutine bar
 program main
   use mod1
 
-  !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
+  !ERROR: The module name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
   ! This is now allowed: "main" is implicitly declared symbol separate
diff --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index 9f222773ad432..fda2fe608ac3c 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -10,7 +10,7 @@ program main
   use mod1
   integer, parameter :: i = 1
 
-  !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
+  !ERROR: The module name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
   ! This is now allowed, since "main" is implicitly declared symbol,

>From 103af162b881d2f39f404d6546e11734b0feb795 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 15:15:01 -0400
Subject: [PATCH 9/9] Documentation change

---
 flang/docs/Extensions.md | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 72d12cd92600d..a7d9281be3e98 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -1,9 +1,9 @@
-<!--===- docs/Extensions.md 
-  
+<!--===- docs/Extensions.md
+
    Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
    See https://llvm.org/LICENSE.txt for license information.
    SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-  
+
 -->
 
 # Fortran Extensions supported by Flang
@@ -170,6 +170,15 @@ end
   In the case of `DEFERRED` bindings in an `ABSTRACT` derived type,
   however, overrides are necessary, so they are permitted for inaccessible
   bindings with an optional warning.
+* Main program name is allowed to be the same as the other symbols used
+  in the main program, for example:
+```
+module m
+end
+program m
+use m
+end
+```
 
 ## Extensions, deletions, and legacy features supported by default
 


From flang-commits at lists.llvm.org  Wed Jul 16 15:47:56 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Wed, 16 Jul 2025 15:47:56 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [draft][acc][flang] Add recipes to
 data entry operations (PR #149210)
Message-ID: <llvm.org/llvm/llvm-project/pull/149210@github.com>


https://github.com/razvanlupusoru created https://github.com/llvm/llvm-project/pull/149210

Currently recipes are held as part of the constructs. But the recipes are needed specifically for materializing semantics of the data clauses. There is no reason to not have this on the data clauses themselves.

TODO: Fix the IR in flang tests which do not expect new operand
TODO: Deprecate the recipe list held on the constructs

>From 4e24d9765993dcece195f59b0761a87fed540e71 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Wed, 16 Jul 2025 15:44:55 -0700
Subject: [PATCH] [draft][acc][flang] Add recipes to data entry operations

Currently recipes are held as part of the constructs. But
the recipes are needed specifically for materializing semantics
of the data clauses. There is no reason to not have this on the
data clauses themselves.

TODO: Fix the IR in flang tests which do not expect new operand
TODO: Deprecate the recipe list held on the constructs
---
 flang/lib/Lower/OpenACC.cpp                   |  8 +++++
 mlir/include/mlir/Dialect/OpenACC/OpenACC.h   |  3 ++
 .../mlir/Dialect/OpenACC/OpenACCOps.td        | 24 +++++++++++--
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       | 34 +++++++++++++++++++
 4 files changed, 66 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 39e4444cde4e3..c2a884d8e4c3a 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -1340,6 +1340,8 @@ static void genPrivatizationRecipes(
           builder, operandLocation, info.addr, asFortran, bounds, true,
           /*implicit=*/false, mlir::acc::DataClause::acc_private, retTy, async,
           asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true);
+      op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                                recipe.getSymName().str()));
       dataOperands.push_back(op.getAccVar());
     } else {
       std::string suffix =
@@ -1353,6 +1355,8 @@ static void genPrivatizationRecipes(
           /*implicit=*/false, mlir::acc::DataClause::acc_firstprivate, retTy,
           async, asyncDeviceTypes, asyncOnlyDeviceTypes,
           /*unwrapBoxAddr=*/true);
+      op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                                recipe.getSymName().str()));
       dataOperands.push_back(op.getAccVar());
     }
     privatizationRecipes.push_back(mlir::SymbolRefAttr::get(
@@ -1787,6 +1791,8 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
     mlir::acc::ReductionRecipeOp recipe =
         Fortran::lower::createOrGetReductionRecipe(
             builder, recipeName, operandLocation, ty, mlirOp, bounds);
+    op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                              recipe.getSymName().str()));
     reductionRecipes.push_back(mlir::SymbolRefAttr::get(
         builder.getContext(), recipe.getSymName().str()));
     reductionOperands.push_back(op.getAccVar());
@@ -2038,6 +2044,8 @@ privatizeIv(Fortran::lower::AbstractConverter &converter,
         builder, loc, ivValue, asFortran, {}, true, /*implicit=*/true,
         mlir::acc::DataClause::acc_private, ivValue.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
+    op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                              recipe.getSymName().str()));
     privateOp = op.getOperation();
 
     privateOperands.push_back(op.getAccVar());
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index 4eb666239d4e4..b3a29d10e10e4 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -151,6 +151,9 @@ mlir::ValueRange getDataOperands(mlir::Operation *accOp);
 /// Used to get a mutable range iterating over the data operands.
 mlir::MutableOperandRange getMutableDataOperands(mlir::Operation *accOp);
 
+/// Used to get the recipe attribute from a data clause operation.
+mlir::SymbolRefAttr getRecipe(mlir::Operation *accOp);
+
 /// Used to obtain the enclosing compute construct operation that contains
 /// the provided `region`. Returns nullptr if no compute construct operation
 /// is found. The returns operation is one of types defined by
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 66378f116784e..dd395912f9e7b 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -514,7 +514,8 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           DefaultValuedAttr<BoolAttr, "false">:$implicit,
           DefaultValuedAttr<OpenACC_DataClauseModifierAttr,
             "mlir::acc::DataClauseModifier::none">:$modifiers,
-          OptionalAttr<StrAttr>:$name));
+          OptionalAttr<StrAttr>:$name,
+          OptionalAttr<SymbolRefAttr>:$recipe));
 
   let description = !strconcat(extraDescription, [{
     Description of arguments:
@@ -623,7 +624,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
-          /*name=*/nullptr);
+          /*name=*/nullptr, /*recipe=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Value":$var,
                    "bool":$structured, "bool":$implicit,
@@ -641,7 +642,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
-          /*name=*/$_builder.getStringAttr(name));
+          /*name=*/$_builder.getStringAttr(name), /*recipe=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Type":$accVarType, "::mlir::Value":$var,
                    "::mlir::Type":$varType, "::mlir::Value":$varPtrPtr,
@@ -652,10 +653,27 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
                    "::mlir::acc::DataClause":$dataClause, "bool":$structured,
                    "bool":$implicit, "::mlir::StringAttr":$name),
       [{
+        // Builder provided to ease transition for new data clause modifiers operand.
         build($_builder, $_state, accVarType, var, varType, varPtrPtr, bounds,
           asyncOperands, asyncOperandsDeviceType, asyncOnly, dataClause,
           structured, implicit, ::mlir::acc::DataClauseModifier::none, name);
       }]>,
+    OpBuilder<(ins "::mlir::Type":$accVarType, "::mlir::Value":$var,
+                   "::mlir::Type":$varType, "::mlir::Value":$varPtrPtr,
+                   "::mlir::ValueRange":$bounds,
+                   "::mlir::ValueRange":$asyncOperands,
+                   "::mlir::ArrayAttr":$asyncOperandsDeviceType,
+                   "::mlir::ArrayAttr":$asyncOnly,
+                   "::mlir::acc::DataClause":$dataClause, "bool":$structured,
+                   "bool":$implicit,
+                   "::mlir::acc::DataClauseModifier":$modifiers,
+                   "::mlir::StringAttr":$name),
+      [{
+        // Builder provided to simplify building after recipe operand was added.
+        build($_builder, $_state, accVarType, var, varType, varPtrPtr, bounds,
+              asyncOperands, asyncOperandsDeviceType, asyncOnly, dataClause,
+              structured, implicit, modifiers, name, /*recipe=*/nullptr);
+      }]>,
     ];
 }
 
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..eb56d2d8933a0 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -332,6 +332,13 @@ checkValidModifier(Op op, acc::DataClauseModifier validModifiers) {
   return success();
 }
 
+template <typename Op>
+static LogicalResult checkNoRecipe(Op op) {
+  if (op.getRecipe().has_value())
+    return op.emitError("no recipes are allowed");
+  return success();
+}
+
 static ParseResult parseVar(mlir::OpAsmParser &parser,
                             OpAsmParser::UnresolvedOperand &var) {
   // Either `var` or `varPtr` keyword is required.
@@ -1063,6 +1070,24 @@ checkSymOperandList(Operation *op, std::optional<mlir::ArrayAttr> attributes,
       return op->emitOpError()
              << "expected as many " << symbolName << " symbol reference as "
              << operandName << " operands";
+    if (attributes) {
+      for (auto operandAndAttribute : llvm::zip(operands, *attributes)) {
+        mlir::Value operand = std::get<0>(operandAndAttribute);
+        if (auto *definingOp = operand.getDefiningOp()) {
+          mlir::SymbolRefAttr operandRecipe = getRecipe(definingOp);
+          // If the operand operation has a recipe - check that it is consistent
+          // with the one recorded in the construct.
+          if (operandRecipe) {
+            if (operandRecipe.getLeafReference().compare(
+                    llvm::cast<mlir::SymbolRefAttr>(
+                        std::get<1>(operandAndAttribute))
+                        .getLeafReference()) != 0)
+              return op->emitOpError() << "expected consistent recipe for "
+                                       << operandName << " operand";
+          }
+        }
+      }
+    }
   } else {
     if (attributes)
       return op->emitOpError()
@@ -4075,6 +4100,15 @@ mlir::acc::getMutableDataOperands(mlir::Operation *accOp) {
   return dataOperands;
 }
 
+mlir::SymbolRefAttr mlir::acc::getRecipe(mlir::Operation *accOp) {
+  auto recipe{
+      llvm::TypeSwitch<mlir::Operation *, mlir::SymbolRefAttr>(accOp)
+          .Case<ACC_DATA_ENTRY_OPS>(
+              [&](auto entry) { return entry.getRecipeAttr(); })
+          .Default([&](mlir::Operation *) { return mlir::SymbolRefAttr{}; })};
+  return recipe;
+}
+
 mlir::Operation *mlir::acc::getEnclosingComputeOp(mlir::Region &region) {
   mlir::Operation *parentOp = region.getParentOp();
   while (parentOp) {


From flang-commits at lists.llvm.org  Wed Jul 16 15:48:22 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 16 Jul 2025 15:48:22 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [draft][acc][flang] Add recipes to
 data entry operations (PR #149210)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149210@github.com>
Message-ID: <68782c36.170a0220.200a96.2fe3@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Razvan Lupusoru (razvanlupusoru)

<details>
<summary>Changes</summary>

Currently recipes are held as part of the constructs. But the recipes are needed specifically for materializing semantics of the data clauses. There is no reason to not have this on the data clauses themselves.

TODO: Fix the IR in flang tests which do not expect new operand
TODO: Deprecate the recipe list held on the constructs

---
Full diff: https://github.com/llvm/llvm-project/pull/149210.diff


4 Files Affected:

- (modified) flang/lib/Lower/OpenACC.cpp (+8) 
- (modified) mlir/include/mlir/Dialect/OpenACC/OpenACC.h (+3) 
- (modified) mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td (+21-3) 
- (modified) mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp (+34) 


``````````diff
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 39e4444cde4e3..c2a884d8e4c3a 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -1340,6 +1340,8 @@ static void genPrivatizationRecipes(
           builder, operandLocation, info.addr, asFortran, bounds, true,
           /*implicit=*/false, mlir::acc::DataClause::acc_private, retTy, async,
           asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true);
+      op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                                recipe.getSymName().str()));
       dataOperands.push_back(op.getAccVar());
     } else {
       std::string suffix =
@@ -1353,6 +1355,8 @@ static void genPrivatizationRecipes(
           /*implicit=*/false, mlir::acc::DataClause::acc_firstprivate, retTy,
           async, asyncDeviceTypes, asyncOnlyDeviceTypes,
           /*unwrapBoxAddr=*/true);
+      op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                                recipe.getSymName().str()));
       dataOperands.push_back(op.getAccVar());
     }
     privatizationRecipes.push_back(mlir::SymbolRefAttr::get(
@@ -1787,6 +1791,8 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
     mlir::acc::ReductionRecipeOp recipe =
         Fortran::lower::createOrGetReductionRecipe(
             builder, recipeName, operandLocation, ty, mlirOp, bounds);
+    op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                              recipe.getSymName().str()));
     reductionRecipes.push_back(mlir::SymbolRefAttr::get(
         builder.getContext(), recipe.getSymName().str()));
     reductionOperands.push_back(op.getAccVar());
@@ -2038,6 +2044,8 @@ privatizeIv(Fortran::lower::AbstractConverter &converter,
         builder, loc, ivValue, asFortran, {}, true, /*implicit=*/true,
         mlir::acc::DataClause::acc_private, ivValue.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
+    op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                              recipe.getSymName().str()));
     privateOp = op.getOperation();
 
     privateOperands.push_back(op.getAccVar());
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index 4eb666239d4e4..b3a29d10e10e4 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -151,6 +151,9 @@ mlir::ValueRange getDataOperands(mlir::Operation *accOp);
 /// Used to get a mutable range iterating over the data operands.
 mlir::MutableOperandRange getMutableDataOperands(mlir::Operation *accOp);
 
+/// Used to get the recipe attribute from a data clause operation.
+mlir::SymbolRefAttr getRecipe(mlir::Operation *accOp);
+
 /// Used to obtain the enclosing compute construct operation that contains
 /// the provided `region`. Returns nullptr if no compute construct operation
 /// is found. The returns operation is one of types defined by
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 66378f116784e..dd395912f9e7b 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -514,7 +514,8 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           DefaultValuedAttr<BoolAttr, "false">:$implicit,
           DefaultValuedAttr<OpenACC_DataClauseModifierAttr,
             "mlir::acc::DataClauseModifier::none">:$modifiers,
-          OptionalAttr<StrAttr>:$name));
+          OptionalAttr<StrAttr>:$name,
+          OptionalAttr<SymbolRefAttr>:$recipe));
 
   let description = !strconcat(extraDescription, [{
     Description of arguments:
@@ -623,7 +624,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
-          /*name=*/nullptr);
+          /*name=*/nullptr, /*recipe=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Value":$var,
                    "bool":$structured, "bool":$implicit,
@@ -641,7 +642,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
-          /*name=*/$_builder.getStringAttr(name));
+          /*name=*/$_builder.getStringAttr(name), /*recipe=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Type":$accVarType, "::mlir::Value":$var,
                    "::mlir::Type":$varType, "::mlir::Value":$varPtrPtr,
@@ -652,10 +653,27 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
                    "::mlir::acc::DataClause":$dataClause, "bool":$structured,
                    "bool":$implicit, "::mlir::StringAttr":$name),
       [{
+        // Builder provided to ease transition for new data clause modifiers operand.
         build($_builder, $_state, accVarType, var, varType, varPtrPtr, bounds,
           asyncOperands, asyncOperandsDeviceType, asyncOnly, dataClause,
           structured, implicit, ::mlir::acc::DataClauseModifier::none, name);
       }]>,
+    OpBuilder<(ins "::mlir::Type":$accVarType, "::mlir::Value":$var,
+                   "::mlir::Type":$varType, "::mlir::Value":$varPtrPtr,
+                   "::mlir::ValueRange":$bounds,
+                   "::mlir::ValueRange":$asyncOperands,
+                   "::mlir::ArrayAttr":$asyncOperandsDeviceType,
+                   "::mlir::ArrayAttr":$asyncOnly,
+                   "::mlir::acc::DataClause":$dataClause, "bool":$structured,
+                   "bool":$implicit,
+                   "::mlir::acc::DataClauseModifier":$modifiers,
+                   "::mlir::StringAttr":$name),
+      [{
+        // Builder provided to simplify building after recipe operand was added.
+        build($_builder, $_state, accVarType, var, varType, varPtrPtr, bounds,
+              asyncOperands, asyncOperandsDeviceType, asyncOnly, dataClause,
+              structured, implicit, modifiers, name, /*recipe=*/nullptr);
+      }]>,
     ];
 }
 
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..eb56d2d8933a0 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -332,6 +332,13 @@ checkValidModifier(Op op, acc::DataClauseModifier validModifiers) {
   return success();
 }
 
+template <typename Op>
+static LogicalResult checkNoRecipe(Op op) {
+  if (op.getRecipe().has_value())
+    return op.emitError("no recipes are allowed");
+  return success();
+}
+
 static ParseResult parseVar(mlir::OpAsmParser &parser,
                             OpAsmParser::UnresolvedOperand &var) {
   // Either `var` or `varPtr` keyword is required.
@@ -1063,6 +1070,24 @@ checkSymOperandList(Operation *op, std::optional<mlir::ArrayAttr> attributes,
       return op->emitOpError()
              << "expected as many " << symbolName << " symbol reference as "
              << operandName << " operands";
+    if (attributes) {
+      for (auto operandAndAttribute : llvm::zip(operands, *attributes)) {
+        mlir::Value operand = std::get<0>(operandAndAttribute);
+        if (auto *definingOp = operand.getDefiningOp()) {
+          mlir::SymbolRefAttr operandRecipe = getRecipe(definingOp);
+          // If the operand operation has a recipe - check that it is consistent
+          // with the one recorded in the construct.
+          if (operandRecipe) {
+            if (operandRecipe.getLeafReference().compare(
+                    llvm::cast<mlir::SymbolRefAttr>(
+                        std::get<1>(operandAndAttribute))
+                        .getLeafReference()) != 0)
+              return op->emitOpError() << "expected consistent recipe for "
+                                       << operandName << " operand";
+          }
+        }
+      }
+    }
   } else {
     if (attributes)
       return op->emitOpError()
@@ -4075,6 +4100,15 @@ mlir::acc::getMutableDataOperands(mlir::Operation *accOp) {
   return dataOperands;
 }
 
+mlir::SymbolRefAttr mlir::acc::getRecipe(mlir::Operation *accOp) {
+  auto recipe{
+      llvm::TypeSwitch<mlir::Operation *, mlir::SymbolRefAttr>(accOp)
+          .Case<ACC_DATA_ENTRY_OPS>(
+              [&](auto entry) { return entry.getRecipeAttr(); })
+          .Default([&](mlir::Operation *) { return mlir::SymbolRefAttr{}; })};
+  return recipe;
+}
+
 mlir::Operation *mlir::acc::getEnclosingComputeOp(mlir::Region &region) {
   mlir::Operation *parentOp = region.getParentOp();
   while (parentOp) {

``````````

</details>


https://github.com/llvm/llvm-project/pull/149210

From flang-commits at lists.llvm.org  Wed Jul 16 22:04:29 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Wed, 16 Jul 2025 22:04:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149165@github.com>
Message-ID: <6878845d.050a0220.317ebc.6e47@mx.google.com>


https://github.com/vzakhari approved this pull request.

Thank you for the update! LGTM

https://github.com/llvm/llvm-project/pull/149165

From flang-commits at lists.llvm.org  Thu Jul 17 04:52:39 2025
From: flang-commits at lists.llvm.org (Alexey Bataev via flang-commits)
Date: Thu, 17 Jul 2025 04:52:39 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <6878e407.170a0220.2f44aa.572c@mx.google.com>


https://github.com/alexey-bataev commented:

Some previous comments are not addressed yet

https://github.com/llvm/llvm-project/pull/139293

From flang-commits at lists.llvm.org  Thu Jul 17 04:52:39 2025
From: flang-commits at lists.llvm.org (Alexey Bataev via flang-commits)
Date: Thu, 17 Jul 2025 04:52:39 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <6878e407.630a0220.327294.83e4@mx.google.com>


https://github.com/alexey-bataev edited https://github.com/llvm/llvm-project/pull/139293

From flang-commits at lists.llvm.org  Thu Jul 17 04:52:39 2025
From: flang-commits at lists.llvm.org (Alexey Bataev via flang-commits)
Date: Thu, 17 Jul 2025 04:52:39 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <6878e407.170a0220.1d61e9.452c@mx.google.com>


================
@@ -1143,6 +1143,83 @@ class OMPFullClause final : public OMPNoChildClause<llvm::omp::OMPC_full> {
   static OMPFullClause *CreateEmpty(const ASTContext &C);
 };
 
+/// This class represents the 'looprange' clause in the
+/// '#pragma omp fuse' directive
+///
+/// \code {c}
+/// #pragma omp fuse looprange(1,2)
+/// {
+///   for(int i = 0; i < 64; ++i)
+///   for(int j = 0; j < 256; j+=2)
+///   for(int k = 127; k >= 0; --k)
+/// \endcode
+class OMPLoopRangeClause final
+    : public OMPClause {
+  friend class OMPClauseReader;
+  /// Location of '('
+  SourceLocation LParenLoc;
+
+  /// Location of first and count expressions
+  SourceLocation FirstLoc, CountLoc;
+
+  /// Number of looprange arguments (always 2: first, count)
+  static constexpr unsigned NumArgs = 2;
+  Stmt *Args[NumArgs] = {nullptr, nullptr};
+
+  /// Set looprange 'first' expression
+  void setFirst(Expr *E) { Args[0] = E; }
+
+  /// Set looprange 'count' expression
+  void setCount(Expr *E) { Args[1] = E; }
+
+  /// Build an empty clause for deserialization.
+  explicit OMPLoopRangeClause()
+      : OMPClause(llvm::omp::OMPC_looprange, {}, {}) {}
+
+public:
+  /// Build a 'looprange' clause AST node.
+  static OMPLoopRangeClause *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
+         SourceLocation FirstLoc, SourceLocation CountLoc,
+         SourceLocation EndLoc, Expr* First, Expr* Count);
----------------
alexey-bataev wrote:

Formatting?

https://github.com/llvm/llvm-project/pull/139293

From flang-commits at lists.llvm.org  Thu Jul 17 05:46:47 2025
From: flang-commits at lists.llvm.org (Carlo Bramini via flang-commits)
Date: Thu, 17 Jul 2025 05:46:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][CMake] CYGWIN: Fix undefined
 references at link time. (PR #67105)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/67105@github.com>
Message-ID: <6878f0b7.170a0220.11d4b5.56ed@mx.google.com>


carlo-bramini wrote:

Still unresolved in llvm-20.1.8.

https://github.com/llvm/llvm-project/pull/67105

From flang-commits at lists.llvm.org  Thu Jul 17 06:07:20 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 06:07:20 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <6878f588.170a0220.1a48ca.5f38@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {darker}-->


:warning: Python code formatter, darker found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
darker --check --diff -r HEAD~1...HEAD flang/test/lit.cfg.py
``````````

</details>

<details>
<summary>
View the diff from darker here.
</summary>

``````````diff
--- lit.cfg.py	2025-07-17 12:09:57.000000 +0000
+++ lit.cfg.py	2025-07-17 13:06:52.649010 +0000
@@ -126,43 +126,57 @@
 # Check for DEFAULT_SYSROOT, because when it is set -isysroot has no effect.
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
+
 
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
-
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
+
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
         extra_args=isysroot_flag + extra_intrinsics_search_args,
@@ -172,16 +186,15 @@
         "%flang_fc1",
         command=FindTool("flang"),
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
         command=FindTool("flang"),
         extra_args=isysroot_flag,
@@ -223,14 +236,14 @@
     config.environment["LIBPGMATH"] = True
 
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
-else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    config.available_features.add("flangrt-modules")
+else:
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
     config.available_features.add("openmp_runtime")

``````````

</details>


https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Thu Jul 17 06:18:09 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Thu, 17 Jul 2025 06:18:09 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] handle allocation of zero-sized
	objects (PR #149165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149165@github.com>
Message-ID: <6878f811.630a0220.37dc76.8af2@mx.google.com>


https://github.com/DanielCChen approved this pull request.

LGTM.
Thanks!

https://github.com/llvm/llvm-project/pull/149165

From flang-commits at lists.llvm.org  Thu Jul 17 06:24:32 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 06:24:32 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <6878f990.050a0220.3d472f.b96c@mx.google.com>


Meinersbur wrote:

> What's the main limitation here? If this is just a file dependency it should be identical to how all the OpenMP tests depend on `omp.h` being in the resource directory.

`omp.h` is created by `configure_file` at configure time. No dependency other than `runtimes-configure` needed.

> IMHO this is trivial if we do a runtimes build, since we can just require that `openmp;flang-rt` are in the same toolchain, 

With toolchain you mean a bootstrapping build with `LLVM_ENABLE_RUNTIMES=openmp;flang-rt` ? Don't forget the users of a Flang-standalone build (`cmake -S <llvm-project>/flang`).

> which then gives us well defined access to `openmp`'s CMake targets so long as it's listed before `flang-rt`.

`check-flang` (LLVM_ENABLE_PROJECTS=flang) needs access to `libomp.mod` (LLVM_ENABLE_RUNTIMES=openmp) and the flang modules as well to work.


https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Thu Jul 17 06:25:22 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 06:25:22 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <6878f9c2.170a0220.22c35.3d0e@mx.google.com>


================
@@ -3979,6 +3979,16 @@ def fsyntax_only : Flag<["-"], "fsyntax-only">,
   Visibility<[ClangOption, CLOption, DXCOption, CC1Option, FC1Option, FlangOption]>,
   Group<Action_Group>,
   HelpText<"Run the preprocessor, parser and semantic analysis stages">;
+
+
+def fno_builtin_modules : Flag<["-"], "fno-builtin-modules">,
+  Visibility<[FC1Option]>,
+  HelpText<"Do not implicitly use builtin modules (for internal use only)">;
----------------
Meinersbur wrote:

removed

https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Thu Jul 17 07:48:29 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 07:48:29 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <68790d3d.630a0220.11fdb3.e122@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Thu Jul 17 07:51:09 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 07:51:09 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <68790ddd.050a0220.19a51b.d2ae@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Thu Jul 17 04:11:33 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Thu, 17 Jul 2025 04:11:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <6878da65.050a0220.39628.af67@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/149169

>From f7cc4e803ca099d98d9711b42fc342751de67367 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 15:33:12 -0400
Subject: [PATCH 01/11] [flang] Attempt to address main program symbol
 conflicts

... by making main program symbol names upper case.
---
 flang/lib/Semantics/resolve-labels.cpp | 32 ++++++++++++++++++++------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index b0cbc4b56e889..0278f70b90226 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -489,15 +489,33 @@ class ParseTreeAnalyzer {
 
   // C1401
   void Post(const parser::MainProgram &mainProgram) {
-    if (const parser::CharBlock *
+    // Uppercase the name of the main program, so that its symbol name
+    // would be unique from similarly named non-main-program symbols.
+    auto upperCaseCharBlock = [](const parser::CharBlock& cb) {
+      char* ch = const_cast<char*>(cb.begin());
+      char* endCh = ch + cb.size();
+      while (ch != endCh) {
+        *ch++ = parser::ToUpperCaseLetter(*ch);
+      }
+    };
+    const parser::CharBlock *progName{nullptr};
+    if (const auto &program{
+            std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
+                mainProgram.t)}) {
+      progName = &program->statement.v.source;
+    }
+    if (progName) {
+      upperCaseCharBlock(*progName);
+    }
+    const parser::CharBlock *
         endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
-            mainProgram.t))}) {
-      if (const auto &program{
-              std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
-                  mainProgram.t)}) {
-        if (*endName != program->statement.v.source) {
+            mainProgram.t))};
+    if (endName) {
+      upperCaseCharBlock(*endName);
+      if (progName) {
+        if (*endName != *progName) {
           context_.Say(*endName, "END PROGRAM name mismatch"_err_en_US)
-              .Attach(program->statement.v.source, "should be"_en_US);
+              .Attach(*progName, "should be"_en_US);
         }
       } else {
         context_.Say(*endName,

>From 336c138bda8c1738c5845f274533b84a075c41aa Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:16:13 -0400
Subject: [PATCH 02/11] clang-format

---
 flang/lib/Semantics/resolve-labels.cpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index 0278f70b90226..d3fda03383783 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -491,9 +491,9 @@ class ParseTreeAnalyzer {
   void Post(const parser::MainProgram &mainProgram) {
     // Uppercase the name of the main program, so that its symbol name
     // would be unique from similarly named non-main-program symbols.
-    auto upperCaseCharBlock = [](const parser::CharBlock& cb) {
-      char* ch = const_cast<char*>(cb.begin());
-      char* endCh = ch + cb.size();
+    auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
+      char *ch = const_cast<char *>(cb.begin());
+      char *endCh = ch + cb.size();
       while (ch != endCh) {
         *ch++ = parser::ToUpperCaseLetter(*ch);
       }
@@ -507,9 +507,8 @@ class ParseTreeAnalyzer {
     if (progName) {
       upperCaseCharBlock(*progName);
     }
-    const parser::CharBlock *
-        endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
-            mainProgram.t))};
+    const parser::CharBlock *endName{GetStmtName(
+        std::get<parser::Statement<parser::EndProgramStmt>>(mainProgram.t))};
     if (endName) {
       upperCaseCharBlock(*endName);
       if (progName) {

>From f6507f56a22227d1b3cb5327d2a0b561a494327e Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:16:52 -0400
Subject: [PATCH 03/11] Initial test changes

---
 flang/test/Driver/cuda-option.f90             |  2 +-
 flang/test/Driver/unparse-use-analyzed.f95    |  4 +-
 flang/test/Driver/unparse-with-modules.f90    |  2 +-
 .../test/Integration/debug-common-block-1.f90 |  2 +-
 flang/test/Integration/debug-local-var-2.f90  |  2 +-
 flang/test/Lower/CUDA/cuda-derived.cuf        |  2 +-
 flang/test/Lower/CUDA/cuda-return01.cuf       |  2 +-
 flang/test/Lower/CUDA/cuda-return02.cuf       |  2 +-
 .../Lower/HLFIR/intrinsic-subroutines.f90     |  2 +-
 ...ointer-component-structure-constructor.f90 |  2 +-
 flang/test/Lower/OpenACC/acc-atomic-read.f90  |  2 +-
 flang/test/Lower/OpenACC/acc-atomic-write.f90 |  2 +-
 flang/test/Lower/OpenACC/acc-routine04.f90    |  2 +-
 flang/test/Lower/OpenMP/atomic-read.f90       |  2 +-
 flang/test/Lower/OpenMP/atomic-write.f90      |  2 +-
 .../Lower/OpenMP/common-atomic-lowering.f90   |  2 +-
 flang/test/Lower/OpenMP/cray-pointers02.f90   |  2 +-
 .../Lower/OpenMP/default-clause-byref.f90     |  2 +-
 flang/test/Lower/OpenMP/default-clause.f90    |  2 +-
 .../parallel-reduction-allocatable-array.f90  |  2 +-
 .../OpenMP/parallel-reduction-array-lb.f90    |  2 +-
 .../OpenMP/parallel-reduction-array2.f90      |  2 +-
 .../Lower/OpenMP/parallel-reduction-byref.f90 |  2 +-
 .../parallel-reduction-pointer-array.f90      |  2 +-
 .../OpenMP/parallel-reduction-rename.f90      |  2 +-
 .../test/Lower/OpenMP/parallel-reduction.f90  |  2 +-
 flang/test/Lower/OpenMP/sections.f90          |  2 +-
 .../threadprivate-host-association-2.f90      |  2 +-
 .../threadprivate-host-association-3.f90      |  2 +-
 .../OpenMP/threadprivate-host-association.f90 |  2 +-
 flang/test/Lower/OpenMP/wsloop-chunks.f90     |  2 +-
 flang/test/Lower/OpenMP/wsloop-collapse.f90   |  2 +-
 ...oop-reduction-allocatable-array-minmax.f90 |  2 +-
 .../OpenMP/wsloop-reduction-allocatable.f90   |  2 +-
 .../OpenMP/wsloop-reduction-array-lb.f90      |  2 +-
 .../OpenMP/wsloop-reduction-array-lb2.f90     |  2 +-
 .../Lower/OpenMP/wsloop-reduction-array.f90   |  2 +-
 .../Lower/OpenMP/wsloop-reduction-array2.f90  |  2 +-
 .../Lower/OpenMP/wsloop-reduction-min2.f90    |  2 +-
 .../wsloop-reduction-multiple-clauses.f90     |  2 +-
 .../Lower/OpenMP/wsloop-reduction-pointer.f90 |  2 +-
 flang/test/Lower/array-character.f90          |  2 +-
 flang/test/Lower/array-expression-slice-1.f90 |  2 +-
 flang/test/Lower/basic-program.f90            |  6 +-
 flang/test/Lower/big-integer-parameter.f90    |  2 +-
 .../test/Lower/derived-type-finalization.f90  |  2 +-
 flang/test/Lower/location.f90                 |  2 +-
 flang/test/Lower/nested-where.f90             |  2 +-
 flang/test/Lower/polymorphic.f90              |  2 +-
 flang/test/Lower/pre-fir-tree02.f90           |  2 +-
 flang/test/Lower/pre-fir-tree03.f90           |  2 +-
 flang/test/Lower/pre-fir-tree06.f90           |  4 +-
 .../test/Lower/program-units-fir-mangling.f90 |  2 +-
 flang/test/Lower/return-statement.f90         |  2 +-
 flang/test/Lower/volatile-openmp1.f90         |  2 +-
 flang/test/Lower/volatile-string.f90          |  2 +-
 flang/test/Lower/volatile3.f90                |  2 +-
 flang/test/Parser/acc-unparse.f90             |  2 +-
 .../test/Semantics/OpenACC/acc-symbols01.f90  |  2 +-
 .../OpenMP/critical_within_default.f90        |  2 +-
 .../OpenMP/declare-mapper-symbols.f90         |  2 +-
 .../OpenMP/declare-reduction-mangled.f90      |  2 +-
 .../OpenMP/declare-reduction-operators.f90    |  2 +-
 .../OpenMP/declare-reduction-renamedop.f90    |  2 +-
 .../Semantics/OpenMP/declare-reduction.f90    |  2 +-
 flang/test/Semantics/OpenMP/do-schedule03.f90 | 28 +++----
 .../Semantics/OpenMP/do01-positivecase.f90    |  6 +-
 .../Semantics/OpenMP/do04-positivecase.f90    | 14 ++--
 .../Semantics/OpenMP/do05-positivecase.f90    | 32 ++++----
 .../Semantics/OpenMP/do06-positivecases.f90   | 10 +--
 flang/test/Semantics/OpenMP/do11.f90          | 26 +++----
 flang/test/Semantics/OpenMP/do12.f90          | 74 +++++++++----------
 flang/test/Semantics/OpenMP/do14.f90          | 62 ++++++++--------
 flang/test/Semantics/OpenMP/do17.f90          | 38 +++++-----
 .../Semantics/OpenMP/map-clause-symbols.f90   |  2 +-
 flang/test/Semantics/OpenMP/reduction08.f90   | 48 ++++++------
 flang/test/Semantics/OpenMP/reduction09.f90   | 38 +++++-----
 flang/test/Semantics/OpenMP/reduction11.f90   |  2 +-
 flang/test/Semantics/OpenMP/scan2.f90         |  2 +-
 flang/test/Semantics/OpenMP/symbol01.f90      | 52 ++++++-------
 flang/test/Semantics/OpenMP/symbol05.f90      |  4 +-
 flang/test/Semantics/OpenMP/symbol07.f90      |  2 +-
 flang/test/Semantics/OpenMP/symbol09.f90      |  2 +-
 flang/test/Semantics/getsymbols03-a.f90       |  2 +-
 flang/test/Semantics/modproc01.f90            |  2 +-
 flang/test/Semantics/multi-programs04.f90     |  2 +-
 flang/test/Semantics/procinterface01.f90      | 22 +++---
 flang/test/Semantics/resolve125.f90           |  4 +-
 flang/test/Semantics/symbol03.f90             | 18 ++---
 flang/test/Semantics/symbol06.f90             | 70 +++++++++---------
 flang/test/Semantics/symbol07.f90             | 46 ++++++------
 flang/test/Semantics/symbol08.f90             | 14 ++--
 flang/test/Semantics/symbol15.f90             | 10 +--
 flang/test/Semantics/symbol16.f90             | 18 ++---
 flang/test/Semantics/symbol17.f90             | 50 ++++++-------
 flang/test/Semantics/symbol18.f90             | 20 ++---
 flang/test/Semantics/symbol20.f90             | 12 +--
 flang/test/Semantics/symbol25.f90             | 12 +--
 flang/test/Semantics/symbol26.f90             |  8 +-
 .../Transforms/DoConcurrent/basic_host.f90    |  2 +-
 100 files changed, 446 insertions(+), 446 deletions(-)

diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 0740ed509a077..f55e88dab20ce 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -8,7 +8,7 @@ program main
   integer, device :: dvar
 end program
 
-! CHECK-LABEL: PROGRAM main
+! CHECK-LABEL: PROGRAM MAIN
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
diff --git a/flang/test/Driver/unparse-use-analyzed.f95 b/flang/test/Driver/unparse-use-analyzed.f95
index eb6046aebba54..4bcd72c9a5f50 100644
--- a/flang/test/Driver/unparse-use-analyzed.f95
+++ b/flang/test/Driver/unparse-use-analyzed.f95
@@ -6,12 +6,12 @@
 ! RUN: %flang_fc1 -fdebug-unparse  %s | FileCheck %s --check-prefix=DEFAULT
 ! RUN: %flang_fc1 -fdebug-unparse -fno-analyzed-objects-for-unparse %s | FileCheck %s --check-prefix=DISABLED
 
-! DEFAULT: PROGRAM test
+! DEFAULT: PROGRAM TEST
 ! DEFAULT-NEXT:  REAL, PARAMETER :: val = 3.43e2_4
 ! DEFAULT-NEXT:  PRINT *, 3.47e2_4
 ! DEFAULT-NEXT: END PROGRAM
 
-! DISABLED: PROGRAM test
+! DISABLED: PROGRAM TEST
 ! DISABLED-NEXT:  REAL, PARAMETER :: val = 343.0
 ! DISABLED-NEXT:  PRINT *, val+4
 ! DISABLED-NEXT: END PROGRAM
diff --git a/flang/test/Driver/unparse-with-modules.f90 b/flang/test/Driver/unparse-with-modules.f90
index 53997f7804efa..f6444afbe47c1 100644
--- a/flang/test/Driver/unparse-with-modules.f90
+++ b/flang/test/Driver/unparse-with-modules.f90
@@ -25,7 +25,7 @@ program test
 !CHECK:  implicit none
 !CHECK:  real(kind=real32) x
 !CHECK: end module
-!CHECK: program test
+!CHECK: program TEST
 !CHECK:  use :: m1
 !CHECK:  use :: basictestmoduletwo
 !CHECK:  implicit none
diff --git a/flang/test/Integration/debug-common-block-1.f90 b/flang/test/Integration/debug-common-block-1.f90
index 18217637be0fa..77f47daea4a91 100644
--- a/flang/test/Integration/debug-common-block-1.f90
+++ b/flang/test/Integration/debug-common-block-1.f90
@@ -89,7 +89,7 @@ program test
 ! CHECK-DAG: ![[CBF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "__BLNK__"{{.*}})
 ! CHECK-DAG: ![[CBAF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "a"{{.*}})
 
-! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "test"{{.*}})
+! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "TEST"{{.*}})
 ! CHECK-DAG: ![[CBM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "__BLNK__"{{.*}})
 ! CHECK-DAG: ![[CBAM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "a"{{.*}})
 
diff --git a/flang/test/Integration/debug-local-var-2.f90 b/flang/test/Integration/debug-local-var-2.f90
index b97be141cc8d0..0ddac633a5b1e 100644
--- a/flang/test/Integration/debug-local-var-2.f90
+++ b/flang/test/Integration/debug-local-var-2.f90
@@ -37,7 +37,7 @@
 ! BOTH-LABEL: }
 
 program mn
-! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "mn", {{.*}})
+! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "MN", {{.*}})
 
 ! BOTH-DAG: ![[TYI32:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
 ! BOTH-DAG: ![[TYI64:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)
diff --git a/flang/test/Lower/CUDA/cuda-derived.cuf b/flang/test/Lower/CUDA/cuda-derived.cuf
index 96250d88d81c4..d419ee074f7a0 100644
--- a/flang/test/Lower/CUDA/cuda-derived.cuf
+++ b/flang/test/Lower/CUDA/cuda-derived.cuf
@@ -25,6 +25,6 @@ program main
   type(t2) :: b
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"}
 ! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEa"}
 ! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tt2{b:!fir.type<_QMm1Tt1{a:!fir.box<!fir.heap<!fir.array<?xf32>>>}>}> {bindc_name = "b", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/CUDA/cuda-return01.cuf b/flang/test/Lower/CUDA/cuda-return01.cuf
index 47e69a903efd3..ed7c640a71082 100644
--- a/flang/test/Lower/CUDA/cuda-return01.cuf
+++ b/flang/test/Lower/CUDA/cuda-return01.cuf
@@ -28,6 +28,6 @@ program main
   return
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"}
 ! CHECK: cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QFEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK-NOT: cuf.free
diff --git a/flang/test/Lower/CUDA/cuda-return02.cuf b/flang/test/Lower/CUDA/cuda-return02.cuf
index e450d7e796f22..e54818444e49c 100644
--- a/flang/test/Lower/CUDA/cuda-return02.cuf
+++ b/flang/test/Lower/CUDA/cuda-return02.cuf
@@ -13,7 +13,7 @@ program test
   return
 end
 
-! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare
 ! CHECK: cf.cond_br %{{.*}}, ^bb1, ^bb2
 ! CHECK-NEXT: ^bb1:
diff --git a/flang/test/Lower/HLFIR/intrinsic-subroutines.f90 b/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
index 07c4f012781d4..cbc56ca1e395b 100644
--- a/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
+++ b/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
@@ -24,7 +24,7 @@ program main
   call mvbits(from, 2, 2, to, 0)
   if (any(to /= 5)) STOP 1
 end program
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<3xi32> {bindc_name = "from", uniq_name = "_QFEfrom"}
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
index 7b64634d10d4b..a097b1522307e 100644
--- a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
@@ -35,7 +35,7 @@ FUNCTION BAR() RESULT(res)
     END
   END
 
-! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:    %[[VAL_0:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_1:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_2:.*]] = fir.alloca !fir.boxproc<(!fir.ref<i32>) -> i32> {bindc_name = "pp2", uniq_name = "_QFEpp2"}
diff --git a/flang/test/Lower/OpenACC/acc-atomic-read.f90 b/flang/test/Lower/OpenACC/acc-atomic-read.f90
index 639a98051e3a2..76751a0fa63a8 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-read.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-read.f90
@@ -8,7 +8,7 @@ program acc_atomic_test
      g = h
 end program acc_atomic_test
 
-! CHECK: func @_QQmain() attributes {fir.bindc_name = "acc_atomic_test"} {
+! CHECK: func @_QQmain() attributes {fir.bindc_name = "ACC_ATOMIC_TEST"} {
 ! CHECK: %[[VAR_G:.*]] = fir.alloca f32 {bindc_name = "g", uniq_name = "_QFEg"}
 ! CHECK: %[[G_DECL:.*]]:2 = hlfir.declare %[[VAR_G]] {uniq_name = "_QFEg"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK: %[[VAR_H:.*]] = fir.alloca f32 {bindc_name = "h", uniq_name = "_QFEh"}
diff --git a/flang/test/Lower/OpenACC/acc-atomic-write.f90 b/flang/test/Lower/OpenACC/acc-atomic-write.f90
index 3c55394021abf..e0116e3281820 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-write.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-write.f90
@@ -2,7 +2,7 @@
 
 ! This test checks the lowering of atomic write
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "acc_atomic_write_test"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "ACC_ATOMIC_WRITE_TEST"} {
 !CHECK: %[[VAR_X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[VAR_X]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[VAR_Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
diff --git a/flang/test/Lower/OpenACC/acc-routine04.f90 b/flang/test/Lower/OpenACC/acc-routine04.f90
index f603376163901..655e2762b9694 100644
--- a/flang/test/Lower/OpenACC/acc-routine04.f90
+++ b/flang/test/Lower/OpenACC/acc-routine04.f90
@@ -30,5 +30,5 @@ subroutine sub2()
 ! CHECK: acc.routine @acc_routine_1 func(@_QFPsub2) seq
 ! CHECK: acc.routine @acc_routine_0 func(@_QMdummy_modPsub1) seq
 ! CHECK: func.func @_QMdummy_modPsub1(%arg0: !fir.ref<i32> {fir.bindc_name = "i"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>}
-! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test_acc_routine"}
+! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "TEST_ACC_ROUTINE"}
 ! CHECK: func.func private @_QFPsub2() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>, fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>}
diff --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
index 68dcaac90eef5..30313e240efa3 100644
--- a/flang/test/Lower/OpenMP/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -4,7 +4,7 @@
 
 ! This test checks the lowering of atomic read
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomic"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "OMPATOMIC"} {
 !CHECK:    %[[A_REF:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_REF]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[B_REF:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/OpenMP/atomic-write.f90 b/flang/test/Lower/OpenMP/atomic-write.f90
index 6eded49b0b15d..742fd475c0f04 100644
--- a/flang/test/Lower/OpenMP/atomic-write.f90
+++ b/flang/test/Lower/OpenMP/atomic-write.f90
@@ -4,7 +4,7 @@
 
 ! This test checks the lowering of atomic write
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomicwrite"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "OMPATOMICWRITE"} {
 !CHECK:    %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[Y_REF:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
diff --git a/flang/test/Lower/OpenMP/common-atomic-lowering.f90 b/flang/test/Lower/OpenMP/common-atomic-lowering.f90
index a53cc101024c6..f729bbb00ac8e 100644
--- a/flang/test/Lower/OpenMP/common-atomic-lowering.f90
+++ b/flang/test/Lower/OpenMP/common-atomic-lowering.f90
@@ -1,6 +1,6 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "sample"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "SAMPLE"} {
 !CHECK: %[[val_0:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK: %[[val_1:.*]]:2 = hlfir.declare %[[val_0]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[val_2:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/OpenMP/cray-pointers02.f90 b/flang/test/Lower/OpenMP/cray-pointers02.f90
index 19e4cd09fe50a..79d838702e4b0 100644
--- a/flang/test/Lower/OpenMP/cray-pointers02.f90
+++ b/flang/test/Lower/OpenMP/cray-pointers02.f90
@@ -1,7 +1,7 @@
 ! Test lowering of Cray pointee references.
 ! RUN: flang -fc1 -emit-hlfir -fopenmp %s -o - 2>&1 | FileCheck %s
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test_cray_pointers_02"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST_CRAY_POINTERS_02"}
 program test_cray_pointers_02
     implicit none
 
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index c44c6bb966580..af51c4cc3e814 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -34,7 +34,7 @@
 !CHECK:    omp.yield(%[[PRIV_X]] : !fir.ref<i32>)
 !CHECK:  }
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "DEFAULT_CLAUSE_LOWERING"} {
 !CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"}
 !CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index ee5f579f06b91..505fa4f0f5d63 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -8,7 +8,7 @@
 ! RUN: | FileCheck %s
 
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "DEFAULT_CLAUSE_LOWERING"} {
 !CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"}
 !CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index 4bfd5d8d19261..0036670317a8e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -80,7 +80,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index ec54294c7104f..ea0aa9ec3f53b 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -68,7 +68,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
index 488ecc353af8e..eb0df2b3a17de 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
@@ -63,7 +63,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
index 596276a99cafc..2caec0384a6ab 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
@@ -18,7 +18,7 @@
 !CHECK:    fir.store %[[CR]] to %[[C0]] : !fir.ref<i32>
 !CHECK:    omp.yield(%[[C0]] : !fir.ref<i32>)
 !CHECK:  }
-!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "mn"} {
+!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "MN"} {
 !CHECK:    %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
index f638688bc2cc1..3c1daa0eb983f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
@@ -82,7 +82,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90 b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
index c06343e997bfd..2be154f4bbaf5 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
@@ -25,7 +25,7 @@ end program main
 ! CHECK:           omp.yield(%[[VAL_2]] : i32)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction.f90 b/flang/test/Lower/OpenMP/parallel-reduction.f90
index 612549fb32de5..15e8cc325916d 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction.f90
@@ -10,7 +10,7 @@
 !CHECK:    %[[CR:[_a-z0-9]+]] = arith.addi %[[C0]], %[[C1]] : i32
 !CHECK:    omp.yield(%[[CR]] : i32)
 !CHECK:  }
-!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "mn"} {
+!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "MN"} {
 !CHECK:    %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index d11925cafdc12..3d5c0326fb6b9 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -5,7 +5,7 @@
 ! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 ! RUN: bbc -hlfir -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "sample"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "SAMPLE"} {
 !CHECK:   %[[COUNT:.*]] = fir.address_of(@_QFEcount) : !fir.ref<i32>
 !CHECK:   %[[COUNT_DECL:.*]]:2 = hlfir.declare %[[COUNT]] {uniq_name = "_QFEcount"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[ETA:.*]] = fir.alloca f32 {bindc_name = "eta", uniq_name = "_QFEeta"}
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
index 5e54cef8c29db..5c90ef7d84f89 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
index 21547b47cf381..0e61261e8853e 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/threadprivate-host-association.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association.f90
index 7a27efa2f84aa..1887e8aa68fdc 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[TP_A:.*]] = omp.threadprivate %[[A_DECL]]#0 : !fir.ref<i32> -> !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-chunks.f90 b/flang/test/Lower/OpenMP/wsloop-chunks.f90
index 29c02a3b3c8d5..f3f11d8c4a6c2 100644
--- a/flang/test/Lower/OpenMP/wsloop-chunks.f90
+++ b/flang/test/Lower/OpenMP/wsloop-chunks.f90
@@ -7,7 +7,7 @@ program wsloop
         integer :: i
         integer :: chunk
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "wsloop"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "WSLOOP"} {
 ! CHECK:         %[[CHUNK_REF:.*]] = fir.alloca i32 {bindc_name = "chunk", uniq_name = "_QFEchunk"}
 ! CHECK:         %[[VAL_0:.*]]:2 = hlfir.declare %[[CHUNK_REF]] {uniq_name = "_QFEchunk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
diff --git a/flang/test/Lower/OpenMP/wsloop-collapse.f90 b/flang/test/Lower/OpenMP/wsloop-collapse.f90
index a4d5cbdc03d3e..7ec40ab4b2f43 100644
--- a/flang/test/Lower/OpenMP/wsloop-collapse.f90
+++ b/flang/test/Lower/OpenMP/wsloop-collapse.f90
@@ -2,7 +2,7 @@
 
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
 
-!CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "wsloop_collapse"} {
+!CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "WSLOOP_COLLAPSE"} {
 program wsloop_collapse
 !CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index 58b68e5ec4cfd..e2f75bc8e4481 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -156,7 +156,7 @@ program reduce15
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce15"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE15"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEarr"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index 0a536eb34e7af..663851cba46c6 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -63,7 +63,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "r", uniq_name = "_QFEr"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
index 9f0dd16002baf..2233a74600948 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
@@ -31,5 +31,5 @@ program reduce
 ! CHECK:           omp.yield(%[[ARG0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           omp.wsloop {{.*}} reduction(byref @add_reduction_byref_box_2xi32 %{{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<2xi32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
index 5ada623a0ed23..211bde19da8db 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
@@ -40,5 +40,5 @@ subroutine sub(a, lb, ub)
 ! CHECK:           omp.yield(%[[ARG0]] : !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           omp.wsloop {{.*}} reduction(byref @add_reduction_byref_box_Uxi32 %{{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index 21261da49710c..b7882bcbc0d13 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index ab8dcf1f076c0..7d90335a13a87 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
index 1e26f5a24d41e..d776bd7cfdd03 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
@@ -28,7 +28,7 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_2]] : i32)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
index e0a3b469f40c1..5133db0347034 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -93,7 +93,7 @@ program main
 ! CHECK:           omp.yield(%[[VAL_2]] : f64)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarray) : !fir.ref<!fir.array<3x3xf64>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
index 40b4302f24cd4..27b726376fbeb 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
@@ -64,7 +64,7 @@ program reduce_pointer
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce_pointer"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE_POINTER"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "v", uniq_name = "_QFEv"}
diff --git a/flang/test/Lower/array-character.f90 b/flang/test/Lower/array-character.f90
index 1bc73dae44235..e2899d967c80d 100644
--- a/flang/test/Lower/array-character.f90
+++ b/flang/test/Lower/array-character.f90
@@ -32,7 +32,7 @@ program p
   call charlit
 end program p
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.array<3x!fir.char<1,4>> {bindc_name = "c1", uniq_name = "_QFEc1"}
diff --git a/flang/test/Lower/array-expression-slice-1.f90 b/flang/test/Lower/array-expression-slice-1.f90
index b597814bc0d9f..73943137cb18d 100644
--- a/flang/test/Lower/array-expression-slice-1.f90
+++ b/flang/test/Lower/array-expression-slice-1.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -hlfir=false -fwrapv -o - --outline-intrinsics %s | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK-DAG:         %[[VAL_0:.*]] = arith.constant 10 : index
 ! CHECK-DAG:         %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK-DAG:         %[[VAL_5:.*]] = arith.constant 1 : index
diff --git a/flang/test/Lower/basic-program.f90 b/flang/test/Lower/basic-program.f90
index 5a0e4bdc7b4a1..7e5b40d9e2f0a 100644
--- a/flang/test/Lower/basic-program.f90
+++ b/flang/test/Lower/basic-program.f90
@@ -4,10 +4,10 @@
 program basic
 end program
 
-! CHECK: 1 Program basic
+! CHECK: 1 Program BASIC
 ! CHECK:   1 EndProgramStmt: end program
-! CHECK: End Program basic
+! CHECK: End Program BASIC
 
-! FIR-LABEL: func @_QQmain() attributes {fir.bindc_name = "basic"} {
+! FIR-LABEL: func @_QQmain() attributes {fir.bindc_name = "BASIC"} {
 ! FIR:         return
 ! FIR:       }
diff --git a/flang/test/Lower/big-integer-parameter.f90 b/flang/test/Lower/big-integer-parameter.f90
index a413b1224ebc2..ca90b8adfb318 100644
--- a/flang/test/Lower/big-integer-parameter.f90
+++ b/flang/test/Lower/big-integer-parameter.f90
@@ -13,7 +13,7 @@ program i128
   print*,y
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "i128"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "I128"} {
 ! CHECK-COUNT-2:  %{{.*}} = fir.call @_FortranAioOutputInteger128(%{{.*}}, %{{.*}}) {{.*}}: (!fir.ref<i8>, i128) -> i1
 
 
diff --git a/flang/test/Lower/derived-type-finalization.f90 b/flang/test/Lower/derived-type-finalization.f90
index 3ea58cd719f4a..71cef34899603 100644
--- a/flang/test/Lower/derived-type-finalization.f90
+++ b/flang/test/Lower/derived-type-finalization.f90
@@ -255,5 +255,5 @@ program p
   type(t1) :: t
 end program
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK-NOT: fir.call @_FortranADestroy
diff --git a/flang/test/Lower/location.f90 b/flang/test/Lower/location.f90
index a6ece31bbebed..95bf2260fc107 100644
--- a/flang/test/Lower/location.f90
+++ b/flang/test/Lower/location.f90
@@ -5,7 +5,7 @@ program test
 
 end 
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: fir.call @_FortranAioOutputAscii(%{{.*}}, %{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1 loc(fused<#fir<loc_kind_array[ base,  inclusion,  inclusion]>>["{{.*}}location1.inc":1:10, "{{.*}}location0.inc":1:1, "{{.*}}location.f90":4:1])
 ! CHECK: return loc("{{.*}}location.f90":6:1)
 ! CHECK: } loc("{{.*}}location.f90":3:1)
diff --git a/flang/test/Lower/nested-where.f90 b/flang/test/Lower/nested-where.f90
index ab457280b80ce..28aced2325813 100644
--- a/flang/test/Lower/nested-where.f90
+++ b/flang/test/Lower/nested-where.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "nested_where"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "NESTED_WHERE"} {
 program nested_where
 
   ! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {adapt.valuebyref, bindc_name = "i"}
diff --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90
index b7be5f685d9e3..a84b495dd09d0 100644
--- a/flang/test/Lower/polymorphic.f90
+++ b/flang/test/Lower/polymorphic.f90
@@ -1146,7 +1146,7 @@ program test
   l = i < o%inner
 end program
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: %[[ADDR_O:.*]] = fir.address_of(@_QFEo) : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMpolymorphic_testTouter{inner:!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>}>>>>
 ! CHECK: %[[BOX_NONE:.*]] = fir.convert %[[ADDR_O]] : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMpolymorphic_testTouter{inner:!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>}>>>>) -> !fir.ref<!fir.box<none>>
 ! CHECK: %{{.*}} = fir.call @_FortranAAllocatableAllocate(%[[BOX_NONE]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
diff --git a/flang/test/Lower/pre-fir-tree02.f90 b/flang/test/Lower/pre-fir-tree02.f90
index f4fa626ba6548..65c33e9b364fe 100644
--- a/flang/test/Lower/pre-fir-tree02.f90
+++ b/flang/test/Lower/pre-fir-tree02.f90
@@ -3,7 +3,7 @@
 ! Test Pre-FIR Tree captures all the intended nodes from the parse-tree
 ! Coarray and OpenMP related nodes are tested in other files.
 
-! CHECK: Program test_prog
+! CHECK: Program TEST_PROG
 program test_prog
   ! Check specification part is not part of the tree.
   interface
diff --git a/flang/test/Lower/pre-fir-tree03.f90 b/flang/test/Lower/pre-fir-tree03.f90
index 313dab4d6ec7c..1de66e3f8d016 100644
--- a/flang/test/Lower/pre-fir-tree03.f90
+++ b/flang/test/Lower/pre-fir-tree03.f90
@@ -2,7 +2,7 @@
 
 ! Test Pre-FIR Tree captures OpenMP related constructs
 
-! CHECK: Program test_omp
+! CHECK: Program TEST_OMP
 program test_omp
   ! CHECK: PrintStmt
   print *, "sequential"
diff --git a/flang/test/Lower/pre-fir-tree06.f90 b/flang/test/Lower/pre-fir-tree06.f90
index f84bcd8b58b2d..ed1e76cb375bd 100644
--- a/flang/test/Lower/pre-fir-tree06.f90
+++ b/flang/test/Lower/pre-fir-tree06.f90
@@ -25,13 +25,13 @@ subroutine sub2()
 end
 ! CHECK: End Module m2
 
-! CHECK: Program main
+! CHECK: Program MAIN
 program main
   real :: y
   ! CHECK-NEXT: OpenMPDeclarativeConstruct
   !$omp threadprivate(y)
 end
-! CHECK: End Program main
+! CHECK: End Program MAIN
 
 ! CHECK: Subroutine sub1
 subroutine sub1()
diff --git a/flang/test/Lower/program-units-fir-mangling.f90 b/flang/test/Lower/program-units-fir-mangling.f90
index e0af6f065f34d..65940b4e1ff17 100644
--- a/flang/test/Lower/program-units-fir-mangling.f90
+++ b/flang/test/Lower/program-units-fir-mangling.f90
@@ -124,7 +124,7 @@ subroutine should_not_collide()
 ! CHECK: }
 end subroutine
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 program test
 ! CHECK: }
 contains
diff --git a/flang/test/Lower/return-statement.f90 b/flang/test/Lower/return-statement.f90
index 6351a6859eb4f..8ab69e3146e2f 100644
--- a/flang/test/Lower/return-statement.f90
+++ b/flang/test/Lower/return-statement.f90
@@ -4,7 +4,7 @@ program basic
   return
 end program
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "basic"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "BASIC"} {
 ! CHECK:         return
 ! CHECK:       }
 
diff --git a/flang/test/Lower/volatile-openmp1.f90 b/flang/test/Lower/volatile-openmp1.f90
index 163db953b6b80..07d81a1aeb240 100644
--- a/flang/test/Lower/volatile-openmp1.f90
+++ b/flang/test/Lower/volatile-openmp1.f90
@@ -13,7 +13,7 @@ program main
 !$omp end parallel
 end program
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 1000 : i32
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
diff --git a/flang/test/Lower/volatile-string.f90 b/flang/test/Lower/volatile-string.f90
index 88b21d7b245e9..f263db7abb5fc 100644
--- a/flang/test/Lower/volatile-string.f90
+++ b/flang/test/Lower/volatile-string.f90
@@ -21,7 +21,7 @@ subroutine assign_different_length(string)
   end subroutine
 end program
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 11 : i32
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant true
diff --git a/flang/test/Lower/volatile3.f90 b/flang/test/Lower/volatile3.f90
index 8825f8f3afbcb..a32f29d2bb9e7 100644
--- a/flang/test/Lower/volatile3.f90
+++ b/flang/test/Lower/volatile3.f90
@@ -70,7 +70,7 @@ subroutine sub_select_rank(arr)
 end program
 
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 10 : index
diff --git a/flang/test/Parser/acc-unparse.f90 b/flang/test/Parser/acc-unparse.f90
index 62e0d4487f3f7..12e6dec19f272 100644
--- a/flang/test/Parser/acc-unparse.f90
+++ b/flang/test/Parser/acc-unparse.f90
@@ -15,7 +15,7 @@ program bug47659
   end do label1
 end program
 
-!CHECK-LABEL: PROGRAM bug47659
+!CHECK-LABEL: PROGRAM BUG47659
 !CHECK: !$ACC PARALLEL LOOP
 
 
diff --git a/flang/test/Semantics/OpenACC/acc-symbols01.f90 b/flang/test/Semantics/OpenACC/acc-symbols01.f90
index 375445bad13a5..ea22f41197aa7 100644
--- a/flang/test/Semantics/OpenACC/acc-symbols01.f90
+++ b/flang/test/Semantics/OpenACC/acc-symbols01.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenacc
 
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !DEF: /mm/x ObjectEntity REAL(4)
   !DEF: /mm/y ObjectEntity REAL(4)
diff --git a/flang/test/Semantics/OpenMP/critical_within_default.f90 b/flang/test/Semantics/OpenMP/critical_within_default.f90
index dd972e6e52949..a5fe30eeb7de0 100644
--- a/flang/test/Semantics/OpenMP/critical_within_default.f90
+++ b/flang/test/Semantics/OpenMP/critical_within_default.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
 ! Test that we do not make a private copy of the critical name
 
-!CHECK:  MainProgram scope: mn
+!CHECK:  MainProgram scope: MN
 !CHECK-NEXT:    j size=4 offset=0: ObjectEntity type: INTEGER(4)
 !CHECK-NEXT:    OtherConstruct scope:
 !CHECK-NEXT:      j (OmpPrivate): HostAssoc
diff --git a/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90 b/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
index 06f41ab8ce76f..e57a5c0c1cea6 100644
--- a/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
+++ b/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
 
 program main
-!CHECK-LABEL: MainProgram scope: main
+!CHECK-LABEL: MainProgram scope: MAIN
    implicit none
 
    type ty
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90 b/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
index 9d0a097fb1991..fc977f2f1b839 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
@@ -17,7 +17,7 @@ end function mymax
 end module mymod
 
 program omp_examples
-!CHECK-LABEL: MainProgram scope: omp_examples
+!CHECK-LABEL: MainProgram scope: OMP_EXAMPLES
   use mymod
   implicit none
   integer, parameter :: n = 100
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-operators.f90 b/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
index d7a9f2fc0a36b..84dbe1af01877 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
@@ -49,7 +49,7 @@ function my_add(x, y)
 end module m1
 
 program test_vector
-!CHECK-LABEL: MainProgram scope: test_vector
+!CHECK-LABEL: MainProgram scope: TEST_VECTOR
   use vector_mod
 !CHECK: add_vectors (Function): Use from add_vectors in vector_mod
   implicit none
diff --git a/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90 b/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
index 12e80cbf7b327..9cd638d796091 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
@@ -22,7 +22,7 @@ end function my_mul
 end module module1
 
 program test_omp_reduction
-!CHECK: MainProgram scope: test_omp_reduction
+!CHECK: MainProgram scope: TEST_OMP_REDUCTION
   use module1, only: t1, operator(.modmul.) => operator(.mul.)
 
 !CHECK: .modmul. (Function): Use from .mul. in module1
diff --git a/flang/test/Semantics/OpenMP/declare-reduction.f90 b/flang/test/Semantics/OpenMP/declare-reduction.f90
index ddca38fd57812..1f39c57c54ad1 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction.f90
@@ -31,7 +31,7 @@ end subroutine initme
 end function func
 
 program main
-!CHECK-LABEL: MainProgram scope: main
+!CHECK-LABEL: MainProgram scope: MAIN
 
   !$omp declare reduction (my_add_red : integer : omp_out = omp_out + omp_in) initializer (omp_priv=0)
 
diff --git a/flang/test/Semantics/OpenMP/do-schedule03.f90 b/flang/test/Semantics/OpenMP/do-schedule03.f90
index 8787b094d581a..375f3920b57ad 100644
--- a/flang/test/Semantics/OpenMP/do-schedule03.f90
+++ b/flang/test/Semantics/OpenMP/do-schedule03.f90
@@ -2,26 +2,26 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - xx.
-  !DEF: /ompdoschedule MainProgram
+  !DEF: /OMPDOSCHEDULE MainProgram
 program ompdoschedule
-  !DEF: /ompdoschedule/a ObjectEntity REAL(4)
-  !DEF: /ompdoschedule/y ObjectEntity REAL(4)
-  !DEF: /ompdoschedule/z ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/a ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/y ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/z ObjectEntity REAL(4)
   real  a(100),y(100),z(100)
-  !DEF: /ompdoschedule/b ObjectEntity INTEGER(4)
-  !DEF: /ompdoschedule/i ObjectEntity INTEGER(4)
-  !DEF: /ompdoschedule/n ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/b ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/i ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/n ObjectEntity INTEGER(4)
   integer  b,i,n
-  !REF: /ompdoschedule/b
+  !REF: /OMPDOSCHEDULE/b
   b = 10
   !$omp do  schedule(static,b-b)
-  !DEF: /ompdoschedule/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !REF: /ompdoschedule/n
+  !DEF: /OMPDOSCHEDULE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !REF: /OMPDOSCHEDULE/n
   do i = 2,n+1
-    !REF: /ompdoschedule/y
-    !REF: /ompdoschedule/OtherConstruct1/i
-    !REF: /ompdoschedule/z
-    !REF: /ompdoschedule/a
+    !REF: /OMPDOSCHEDULE/y
+    !REF: /OMPDOSCHEDULE/OtherConstruct1/i
+    !REF: /OMPDOSCHEDULE/z
+    !REF: /OMPDOSCHEDULE/a
     y(i) = z(i-1) + a(i)
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/do01-positivecase.f90 b/flang/test/Semantics/OpenMP/do01-positivecase.f90
index 905fdbaf18476..a4a28f6530d7b 100644
--- a/flang/test/Semantics/OpenMP/do01-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do01-positivecase.f90
@@ -4,13 +4,13 @@
 ! The loop iteration variable may not appear in a firstprivate directive.
 ! A positive case
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   integer i
 
   !$omp do  firstprivate(k)
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
     print *, "Hello"
   end do
diff --git a/flang/test/Semantics/OpenMP/do04-positivecase.f90 b/flang/test/Semantics/OpenMP/do04-positivecase.f90
index eb2d67bb8ceb2..0b0c661687dc2 100644
--- a/flang/test/Semantics/OpenMP/do04-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do04-positivecase.f90
@@ -2,18 +2,18 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 
-!DEF: /omp_do1 MainProgram
+!DEF: /OMP_DO1 MainProgram
 program omp_do1
-  !DEF: /omp_do1/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/n (OmpThreadprivate) ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/n (OmpThreadprivate) ObjectEntity INTEGER(4)
   integer i, j, k, n
   !$omp threadprivate (k,n)
   !$omp do
-  !DEF: /omp_do1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do1/j
+    !REF: /OMP_DO1/j
     do j=1,10
       print *, "Hello"
     end do
diff --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index eda04610535c2..07b12dd1d4d23 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -3,13 +3,13 @@
 ! 2.7.1 Loop Construct restrictions on single directive.
 ! A positive case
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/n ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/n ObjectEntity INTEGER(4)
   integer i,n
   !$omp parallel
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
     !$omp single
     print *, "hello"
@@ -19,13 +19,13 @@ program omp_do
 
   !$omp parallel  default(shared)
   !$omp do
-  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
   do i=1,n
     !$omp parallel
     !$omp single
     !DEF: /work EXTERNAL (Subroutine) ProcEntity
-    !DEF: /omp_do/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
+    !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
     call work(i, 1)
     !$omp end single
     !$omp end parallel
@@ -34,7 +34,7 @@ program omp_do
   !$omp end parallel
 
   !$omp parallel private(i)
-  !DEF: /omp_do/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
   do i=1,10
      !$omp single
      print *, "hello"
@@ -43,32 +43,32 @@ program omp_do
   !$omp end parallel
 
   !$omp target teams distribute parallel do
-  !DEF:/omp_do/OtherConstruct4/i (OmpPrivate ,OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF:/OMP_DO/OtherConstruct4/i (OmpPrivate ,OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF:/omp_do/OtherConstruct4/i
+    !REF:/OMP_DO/OtherConstruct4/i
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do
 
   !$omp target teams distribute parallel do simd
-  !DEF:/omp_do/OtherConstruct5/i (OmpLinear,OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF:/OMP_DO/OtherConstruct5/i (OmpLinear,OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF:/omp_do/OtherConstruct5/i
+    !REF:/OMP_DO/OtherConstruct5/i
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute 
-  !DEF: /omp_do/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF: /omp_do/OtherConstruct6/i
+    !REF: /OMP_DO/OtherConstruct6/i
     if(i < 5) cycle
   end do
 
   !$omp target teams distribute simd
-  !DEF: /omp_do/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF: /omp_do/OtherConstruct7/i
+    !REF: /OMP_DO/OtherConstruct7/i
     if(i < 5) cycle
   end do
 end program omp_do
diff --git a/flang/test/Semantics/OpenMP/do06-positivecases.f90 b/flang/test/Semantics/OpenMP/do06-positivecases.f90
index 2713b55fa2ecb..4d45800298a29 100644
--- a/flang/test/Semantics/OpenMP/do06-positivecases.f90
+++ b/flang/test/Semantics/OpenMP/do06-positivecases.f90
@@ -5,14 +5,14 @@
 ! region ever binds to a loop region arising from the loop construct.
 
 ! A positive case
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
   integer i, j, k
   !$omp do  ordered
-    !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do i=1,10
       !$omp ordered
       !DEF: /my_func EXTERNAL (Subroutine) ProcEntity
diff --git a/flang/test/Semantics/OpenMP/do11.f90 b/flang/test/Semantics/OpenMP/do11.f90
index faab457efff3c..886a9f7ae6081 100644
--- a/flang/test/Semantics/OpenMP/do11.f90
+++ b/flang/test/Semantics/OpenMP/do11.f90
@@ -2,34 +2,34 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 
-!DEF: /omp_do MainProgram
+!DEF: /OMP_DO MainProgram
 program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
   integer i, j, k
   !$omp do
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do/j
+    !REF: /OMP_DO/j
     do j=1,10
-      !REF: /omp_do/OtherConstruct1/i
-      !REF: /omp_do/j
+      !REF: /OMP_DO/OtherConstruct1/i
+      !REF: /OMP_DO/j
       print *, "it", i, j
     end do
   end do
   !$omp end do
 end program omp_do
 
-!DEF: /omp_do2 (Subroutine)Subprogram
+!DEF: /OMP_DO2 (Subroutine)Subprogram
 subroutine omp_do2
-  !DEF: /omp_do2/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do2/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO2/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO2/k ObjectEntity INTEGER(4)
   integer :: i = 0, k
   !$omp do
-  !DEF: /omp_do2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do2/OtherConstruct1/i
+    !REF: /OMP_DO2/OtherConstruct1/i
     print *, "it", i
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/do12.f90 b/flang/test/Semantics/OpenMP/do12.f90
index a057a246f7a99..f36a230426197 100644
--- a/flang/test/Semantics/OpenMP/do12.f90
+++ b/flang/test/Semantics/OpenMP/do12.f90
@@ -2,20 +2,20 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /omp_cycle MainProgram
+!DEF: /OMP_CYCLE MainProgram
 program omp_cycle
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/OtherConstruct1/i
+    !REF: /OMP_CYCLE/OtherConstruct1/i
     if (i<1) cycle
-    !DEF: /omp_cycle/j (Implicit) ObjectEntity INTEGER(4)
+    !DEF: /OMP_CYCLE/j (Implicit) ObjectEntity INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/k (Implicit) ObjectEntity INTEGER(4)
+      !DEF: /OMP_CYCLE/k (Implicit) ObjectEntity INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct1/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct1/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -23,17 +23,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/j
+    !REF: /OMP_CYCLE/j
     do j=0,10
-      !REF: /omp_cycle/OtherConstruct2/i
+      !REF: /OMP_CYCLE/OtherConstruct2/i
       if (i<1) cycle
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct2/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct2/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -41,17 +41,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(2)
-  !DEF: /omp_cycle/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/i
         if (i<1) cycle
-        !REF: /omp_cycle/OtherConstruct3/i
-        !REF: /omp_cycle/OtherConstruct3/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -59,17 +59,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/i
         if (i<1) cycle
-        !REF: /omp_cycle/OtherConstruct4/i
-        !REF: /omp_cycle/OtherConstruct4/j
-        !REF: /omp_cycle/OtherConstruct4/k
+        !REF: /OMP_CYCLE/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/j
+        !REF: /OMP_CYCLE/OtherConstruct4/k
         print *, i, j, k
       end do
     end do
@@ -77,20 +77,20 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo:do i=0,10
-    !DEF: /omp_cycle/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     foo1:do j=0,10
-      !DEF: /omp_cycle/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       foo2:do k=0,10
-        !REF: /omp_cycle/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/i
         if (i<1) cycle foo2
-        !REF: /omp_cycle/OtherConstruct5/i
-        !REF: /omp_cycle/OtherConstruct5/j
-        !REF: /omp_cycle/OtherConstruct5/k
+        !REF: /OMP_CYCLE/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/j
+        !REF: /OMP_CYCLE/OtherConstruct5/k
         print *, i, j, k
       end do foo2
     end do foo1
   end do foo
   !$omp end do
-end program omp_cycle
+end program OMP_CYCLE
diff --git a/flang/test/Semantics/OpenMP/do14.f90 b/flang/test/Semantics/OpenMP/do14.f90
index 5e8a5a64c2979..b508d32d90201 100644
--- a/flang/test/Semantics/OpenMP/do14.f90
+++ b/flang/test/Semantics/OpenMP/do14.f90
@@ -2,19 +2,19 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /omp_cycle MainProgram
+!DEF: /OMP_CYCLE MainProgram
 program omp_cycle
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
     cycle
-    !DEF: /omp_cycle/j (Implicit) ObjectEntity INTEGER(4)
+    !DEF: /OMP_CYCLE/j (Implicit) ObjectEntity INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/k (Implicit) ObjectEntity INTEGER(4)
+      !DEF: /OMP_CYCLE/k (Implicit) ObjectEntity INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct1/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct1/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -22,16 +22,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/j
+    !REF: /OMP_CYCLE/j
     do j=0,10
       cycle
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct2/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct2/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -39,16 +39,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(2)
-  !DEF: /omp_cycle/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
         cycle
-        !REF: /omp_cycle/OtherConstruct3/i
-        !REF: /omp_cycle/OtherConstruct3/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -56,16 +56,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       do k=0,10
         cycle
-        !REF: /omp_cycle/OtherConstruct4/i
-        !REF: /omp_cycle/OtherConstruct4/j
-        !REF: /omp_cycle/OtherConstruct4/k
+        !REF: /OMP_CYCLE/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/j
+        !REF: /OMP_CYCLE/OtherConstruct4/k
         print *, i, j, k
       end do
     end do
@@ -73,16 +73,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  ordered(3)
-  !DEF: /omp_cycle/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo:do i=0,10
-    !DEF: /omp_cycle/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     foo1:do j=0,10
-      !DEF: /omp_cycle/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       foo2:do k=0,10
         cycle foo2
-        !REF: /omp_cycle/OtherConstruct5/i
-        !REF: /omp_cycle/OtherConstruct5/j
-        !REF: /omp_cycle/OtherConstruct5/k
+        !REF: /OMP_CYCLE/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/j
+        !REF: /OMP_CYCLE/OtherConstruct5/k
         print *, i, j, k
       end do foo2
     end do foo1
diff --git a/flang/test/Semantics/OpenMP/do17.f90 b/flang/test/Semantics/OpenMP/do17.f90
index c0c59f16dee1b..0a4cf46d2d6af 100644
--- a/flang/test/Semantics/OpenMP/do17.f90
+++ b/flang/test/Semantics/OpenMP/do17.f90
@@ -2,53 +2,53 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /test MainProgram
+!DEF: /TEST MainProgram
 program test
- !DEF: /test/i ObjectEntity INTEGER(4)
- !DEF: /test/j ObjectEntity INTEGER(4)
- !DEF: /test/k ObjectEntity INTEGER(4)
+ !DEF: /TEST/i ObjectEntity INTEGER(4)
+ !DEF: /TEST/j ObjectEntity INTEGER(4)
+ !DEF: /TEST/k ObjectEntity INTEGER(4)
  integer i, j, k
  !$omp do  collapse(2)
- !DEF: /test/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /TEST/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  foo: do i=0,10
-  !DEF: /test/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /TEST/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo1: do j=0,10
-   !REF: /test/k
+   !REF: /TEST/k
    foo2: do k=0,10
-    !REF: /test/OtherConstruct1/i
+    !REF: /TEST/OtherConstruct1/i
     select case (i)
     case (5)
      cycle foo1
     case (7)
      cycle foo2
     end select
-    !REF: /test/OtherConstruct1/i
-    !REF: /test/OtherConstruct1/j
-    !REF: /test/k
+    !REF: /TEST/OtherConstruct1/i
+    !REF: /TEST/OtherConstruct1/j
+    !REF: /TEST/k
     print *, i, j, k
    end do foo2
   end do foo1
  end do foo
 
  !$omp do  collapse(2)
- !DEF: /test/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /TEST/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  foo: do i=0,10
-  !DEF: /test/OtherConstruct2/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /TEST/OtherConstruct2/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo1: do j=0,10
-   !REF: /test/k
+   !REF: /TEST/k
    foo2: do k=0,10
-    !REF: /test/OtherConstruct2/i
+    !REF: /TEST/OtherConstruct2/i
     if (i<3) then
      cycle foo1
-     !REF: /test/OtherConstruct2/i
+     !REF: /TEST/OtherConstruct2/i
     else if (i>8) then
      cycle foo1
     else
      cycle foo2
     end if
-    !REF: /test/OtherConstruct2/i
-    !REF: /test/OtherConstruct2/j
-    !REF: /test/k
+    !REF: /TEST/OtherConstruct2/i
+    !REF: /TEST/OtherConstruct2/j
+    !REF: /TEST/k
     print *, i, j, k
    end do foo2
   end do foo1
diff --git a/flang/test/Semantics/OpenMP/map-clause-symbols.f90 b/flang/test/Semantics/OpenMP/map-clause-symbols.f90
index 8f984fcd2fa7e..1d6315b4a2312 100644
--- a/flang/test/Semantics/OpenMP/map-clause-symbols.f90
+++ b/flang/test/Semantics/OpenMP/map-clause-symbols.f90
@@ -1,6 +1,6 @@
 ! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
 program main
-!CHECK-LABEL:  MainProgram scope: main
+!CHECK-LABEL:  MainProgram scope: MAIN
   integer, parameter :: n = 256
   real(8) :: a(256)
   !$omp target map(mapper(xx), from:a)
diff --git a/flang/test/Semantics/OpenMP/reduction08.f90 b/flang/test/Semantics/OpenMP/reduction08.f90
index 01a06eb7d7414..1cf2838fe765a 100644
--- a/flang/test/Semantics/OpenMP/reduction08.f90
+++ b/flang/test/Semantics/OpenMP/reduction08.f90
@@ -2,60 +2,60 @@
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases
 
-!DEF: /omp_reduction MainProgram
+!DEF: /OMP_REDUCTION MainProgram
 program omp_reduction
-  !DEF: /omp_reduction/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
-  !DEF: /omp_reduction/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
   integer :: k = 10
-  !DEF: /omp_reduction/m ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/m ObjectEntity INTEGER(4)
   integer :: m = 12
   !$omp parallel do  reduction(max:k)
-  !DEF: /omp_reduction/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
     k = max(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(min:k)
-  !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
     k = min(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(iand:k)
-  !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
     k = iand(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(ior:k)
-  !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
     k = ior(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(ieor:k)
-  !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
     k = ieor(k,m)
   end do
   !$omp end parallel do
diff --git a/flang/test/Semantics/OpenMP/reduction09.f90 b/flang/test/Semantics/OpenMP/reduction09.f90
index d6c71c30d2834..4b1ca116d675b 100644
--- a/flang/test/Semantics/OpenMP/reduction09.f90
+++ b/flang/test/Semantics/OpenMP/reduction09.f90
@@ -1,22 +1,22 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases.
-!DEF: /omp_reduction MainProgram
+!DEF: /OMP_REDUCTION MainProgram
 program omp_reduction
-  !DEF: /omp_reduction/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
-  !DEF: /omp_reduction/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
   integer :: k = 10
-  !DEF: /omp_reduction/a ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/a ObjectEntity INTEGER(4)
   integer a(10)
-  !DEF: /omp_reduction/b ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/b ObjectEntity INTEGER(4)
   integer b(10,10,10)
 
   !$omp parallel  shared(k)
   !$omp do  reduction(+:k)
-  !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
@@ -24,52 +24,52 @@ program omp_reduction
 
 
   !$omp parallel do  reduction(+:a(10))
-  !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
 
   !$omp parallel do  reduction(+:a(1:10:1))
-  !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2))
-  !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2:5:1))
-  !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel  private(i)
   !$omp do reduction(+:k) reduction(+:j)
-  !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
   !$omp end parallel
 
   !$omp do reduction(+:k) reduction(*:j) reduction(+:l)
-  !DEF: /omp_reduction/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
diff --git a/flang/test/Semantics/OpenMP/reduction11.f90 b/flang/test/Semantics/OpenMP/reduction11.f90
index b2ad0f6a6ee11..dfb3986d37d78 100644
--- a/flang/test/Semantics/OpenMP/reduction11.f90
+++ b/flang/test/Semantics/OpenMP/reduction11.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols -o - %s 2>&1 | FileCheck %s
 ! Check intrinsic reduction symbols (in this case "max" are marked as INTRINSIC
 
-! CHECK: MainProgram scope: omp_reduction
+! CHECK: MainProgram scope: OMP_REDUCTION
 program omp_reduction
   ! CHECK: i size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer i
diff --git a/flang/test/Semantics/OpenMP/scan2.f90 b/flang/test/Semantics/OpenMP/scan2.f90
index ffe84910f88a2..1ae5e871595c4 100644
--- a/flang/test/Semantics/OpenMP/scan2.f90
+++ b/flang/test/Semantics/OpenMP/scan2.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols -o - %s 2>&1 | FileCheck %s
 ! Check scan reduction
 
-! CHECK: MainProgram scope: omp_reduction
+! CHECK: MainProgram scope: OMP_REDUCTION
 program omp_reduction
   ! CHECK: i size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer i
diff --git a/flang/test/Semantics/OpenMP/symbol01.f90 b/flang/test/Semantics/OpenMP/symbol01.f90
index fbd9a0286c79b..c1f16d61ada1e 100644
--- a/flang/test/Semantics/OpenMP/symbol01.f90
+++ b/flang/test/Semantics/OpenMP/symbol01.f90
@@ -16,53 +16,53 @@ module md
   integer :: b
  end type myty
 end module md
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
  !REF: /md
  use :: md
- !DEF: /mm/c CommonBlockDetails
- !DEF: /mm/x (InCommonBlock) ObjectEntity REAL(4)
- !DEF: /mm/y (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /MM/c CommonBlockDetails
+ !DEF: /MM/x (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /MM/y (InCommonBlock) ObjectEntity REAL(4)
  common /c/x, y
- !REF: /mm/x
- !REF: /mm/y
+ !REF: /MM/x
+ !REF: /MM/y
  real x, y
- !DEF: /mm/myty Use
- !DEF: /mm/t ObjectEntity TYPE(myty)
+ !DEF: /MM/myty Use
+ !DEF: /MM/t ObjectEntity TYPE(myty)
  type(myty) :: t
- !DEF: /mm/b ObjectEntity INTEGER(4)
+ !DEF: /MM/b ObjectEntity INTEGER(4)
  integer b(10)
- !REF: /mm/t
+ !REF: /MM/t
  !REF: /md/myty/a
  t%a = 3.14
- !REF: /mm/t
+ !REF: /MM/t
  !REF: /md/myty/b
  t%b = 1
- !REF: /mm/b
+ !REF: /MM/b
  b = 2
- !DEF: /mm/a (Implicit) ObjectEntity REAL(4)
+ !DEF: /MM/a (Implicit) ObjectEntity REAL(4)
  a = 1.0
- !DEF: /mm/c (Implicit) ObjectEntity REAL(4)
+ !DEF: /MM/c (Implicit) ObjectEntity REAL(4)
  c = 2.0
 !$omp parallel do  private(a,t,/c/) shared(c)
- !DEF: /mm/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /MM/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  do i=1,10
-  !DEF: /mm/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
-  !DEF: /mm/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
-  !REF: /mm/OtherConstruct1/i
+  !DEF: /MM/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
+  !REF: /MM/OtherConstruct1/i
   a = a+b(i)
-  !DEF: /mm/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
+  !DEF: /MM/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
   !REF: /md/myty/a
-  !REF: /mm/OtherConstruct1/i
+  !REF: /MM/OtherConstruct1/i
   t%a = i
-  !DEF: /mm/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   y = 0.
-  !DEF: /mm/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
-  !REF: /mm/OtherConstruct1/a
-  !REF: /mm/OtherConstruct1/i
-  !REF: /mm/OtherConstruct1/y
+  !DEF: /MM/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !REF: /MM/OtherConstruct1/a
+  !REF: /MM/OtherConstruct1/i
+  !REF: /MM/OtherConstruct1/y
   x = a+i+y
-  !DEF: /mm/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
   c = 3.0
  end do
 end program
diff --git a/flang/test/Semantics/OpenMP/symbol05.f90 b/flang/test/Semantics/OpenMP/symbol05.f90
index fe01f15d20aa3..03e2c0ba3e0cc 100644
--- a/flang/test/Semantics/OpenMP/symbol05.f90
+++ b/flang/test/Semantics/OpenMP/symbol05.f90
@@ -31,10 +31,10 @@ subroutine foo
     end block
   end subroutine foo
 end module mm
-!DEF: /tt MainProgram
+!DEF: /TT MainProgram
 program tt
   !REF: /mm
   use :: mm
-  !DEF: /tt/foo (Subroutine) Use
+  !DEF: /TT/foo (Subroutine) Use
   call foo
 end program tt
diff --git a/flang/test/Semantics/OpenMP/symbol07.f90 b/flang/test/Semantics/OpenMP/symbol07.f90
index 86b7305411347..f5eb2127819bd 100644
--- a/flang/test/Semantics/OpenMP/symbol07.f90
+++ b/flang/test/Semantics/OpenMP/symbol07.f90
@@ -30,7 +30,7 @@ subroutine function_call_in_region
   !REF: /function_call_in_region/b
   print *, a, b
 end subroutine function_call_in_region
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !REF: /function_call_in_region
   call function_call_in_region
diff --git a/flang/test/Semantics/OpenMP/symbol09.f90 b/flang/test/Semantics/OpenMP/symbol09.f90
index 86b7305411347..f5eb2127819bd 100644
--- a/flang/test/Semantics/OpenMP/symbol09.f90
+++ b/flang/test/Semantics/OpenMP/symbol09.f90
@@ -30,7 +30,7 @@ subroutine function_call_in_region
   !REF: /function_call_in_region/b
   print *, a, b
 end subroutine function_call_in_region
-!DEF: /mm MainProgram
+!DEF: /MM MainProgram
 program mm
   !REF: /function_call_in_region
   call function_call_in_region
diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index 95b7fb418367d..e3c2e0bf53f92 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -9,6 +9,6 @@ program main
 
 ! RUN: %flang_fc1 -fget-symbols-sources %s 2>&1 | FileCheck %s
 ! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
-! CHECK:main:{{.*}}getsymbols03-a.f90, 4, 9-13
+! CHECK:MAIN:{{.*}}getsymbols03-a.f90, 4, 9-13
 ! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
 ! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14
diff --git a/flang/test/Semantics/modproc01.f90 b/flang/test/Semantics/modproc01.f90
index 5f45362e95093..e565ddcfbe0b1 100644
--- a/flang/test/Semantics/modproc01.f90
+++ b/flang/test/Semantics/modproc01.f90
@@ -125,7 +125,7 @@ program test
   x = mf(3, "abc", pdt1(1,3)())
 !  call ms(mf)
 end program
-!CHECK:  MainProgram scope: test size=88 alignment=8
+!CHECK:  MainProgram scope: TEST size=88 alignment=8
 !CHECK:    mf, MODULE (Function): Use from mf in m
 !CHECK:    pdt1: Use from pdt1 in m
 !CHECK:    pdt2: Use from pdt2 in m
diff --git a/flang/test/Semantics/multi-programs04.f90 b/flang/test/Semantics/multi-programs04.f90
index 54b0235aa78f0..e69ac7325278e 100644
--- a/flang/test/Semantics/multi-programs04.f90
+++ b/flang/test/Semantics/multi-programs04.f90
@@ -4,6 +4,6 @@
 program m
 end
 !ERROR: A source file cannot contain more than one main program
-!ERROR: 'm' is already declared in this scoping unit
+!ERROR: 'M' is already declared in this scoping unit
 program m
 end
diff --git a/flang/test/Semantics/procinterface01.f90 b/flang/test/Semantics/procinterface01.f90
index 73040b0987bd0..7542bb8bb6fa4 100644
--- a/flang/test/Semantics/procinterface01.f90
+++ b/flang/test/Semantics/procinterface01.f90
@@ -159,35 +159,35 @@ end function logical
  tan = "?"
 end function tan
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /module1
  use :: module1
- !DEF: /main/derived1 Use
- !DEF: /main/instance ObjectEntity TYPE(derived1)
+ !DEF: /MAIN/derived1 Use
+ !DEF: /MAIN/instance ObjectEntity TYPE(derived1)
  type(derived1) :: instance
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p1
  if (instance%p1(1.)/=2.) print *, "p1 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p2
  if (instance%p2(1.)/=2.) print *, "p2 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p3
  if (.not.instance%p3(1.)) print *, "p3 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p4
  if (.not.instance%p4(1.)) print *, "p4 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p5
  if (instance%p5(1.)/=(5.,6.)) print *, "p5 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p6
  if (instance%p6(1.)/=2.) print *, "p6 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p7
  if (instance%p7(0.)/=1.) print *, "p7 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p8
  if (instance%p8(1.)/="a") print *, "p8 failed"
 end program main
diff --git a/flang/test/Semantics/resolve125.f90 b/flang/test/Semantics/resolve125.f90
index e040c006ec179..620c7d65578cd 100644
--- a/flang/test/Semantics/resolve125.f90
+++ b/flang/test/Semantics/resolve125.f90
@@ -43,7 +43,7 @@ subroutine reset
   end subroutine reset
 end module m2
 
-!CHECK: MainProgram scope: main
+!CHECK: MainProgram scope: MAIN
 !CHECK:       i: Use from i in m2
 !CHECK:       i2: Use from i2 in m2
 !CHECK:       init (Subroutine): Use from init in m2
@@ -61,4 +61,4 @@ program main
   else
     print *, "fail"
   end if
-end program main
\ No newline at end of file
+end program main
diff --git a/flang/test/Semantics/symbol03.f90 b/flang/test/Semantics/symbol03.f90
index a6b4b0bd15937..f50a0e91fd289 100644
--- a/flang/test/Semantics/symbol03.f90
+++ b/flang/test/Semantics/symbol03.f90
@@ -1,23 +1,23 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Test host association in internal subroutine of main program.
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/x ObjectEntity INTEGER(4)
+ !DEF: /MAIN/x ObjectEntity INTEGER(4)
  integer x
- !DEF: /main/s (Subroutine) Subprogram
+ !DEF: /MAIN/s (Subroutine) Subprogram
  call s
 contains
- !REF: /main/s
+ !REF: /MAIN/s
  subroutine s
-  !DEF: /main/s/y (Implicit) ObjectEntity REAL(4)
-  !DEF: /main/s/x HostAssoc INTEGER(4)
+  !DEF: /MAIN/s/y (Implicit) ObjectEntity REAL(4)
+  !DEF: /MAIN/s/x HostAssoc INTEGER(4)
   y = x
  contains
-  !DEF: /main/s/s2 (Subroutine) Subprogram
+  !DEF: /MAIN/s/s2 (Subroutine) Subprogram
   subroutine s2
-   !DEF: /main/s/s2/z (Implicit) ObjectEntity REAL(4)
-   !DEF: /main/s/s2/x HostAssoc INTEGER(4)
+   !DEF: /MAIN/s/s2/z (Implicit) ObjectEntity REAL(4)
+   !DEF: /MAIN/s/s2/x HostAssoc INTEGER(4)
    z = x
   end subroutine
  end subroutine
diff --git a/flang/test/Semantics/symbol06.f90 b/flang/test/Semantics/symbol06.f90
index bbd6d4d071c89..984ebcc1c651c 100644
--- a/flang/test/Semantics/symbol06.f90
+++ b/flang/test/Semantics/symbol06.f90
@@ -1,56 +1,56 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/t1 DerivedType
+ !DEF: /MAIN/t1 DerivedType
  type :: t1
-  !DEF: /main/t1/a1 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t1/a1 ObjectEntity INTEGER(4)
   integer :: a1
  end type
- !REF: /main/t1
- !DEF: /main/t2 DerivedType
+ !REF: /MAIN/t1
+ !DEF: /MAIN/t2 DerivedType
  type, extends(t1) :: t2
-  !DEF: /main/t2/a2 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t2/a2 ObjectEntity INTEGER(4)
   integer :: a2
  end type
- !REF: /main/t2
- !DEF: /main/t3 DerivedType
+ !REF: /MAIN/t2
+ !DEF: /MAIN/t3 DerivedType
  type, extends(t2) :: t3
-  !DEF: /main/t3/a3 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t3/a3 ObjectEntity INTEGER(4)
   integer :: a3
  end type
- !REF: /main/t3
- !DEF: /main/x3 ObjectEntity TYPE(t3)
+ !REF: /MAIN/t3
+ !DEF: /MAIN/x3 ObjectEntity TYPE(t3)
  type(t3) :: x3
- !DEF: /main/i ObjectEntity INTEGER(4)
+ !DEF: /MAIN/i ObjectEntity INTEGER(4)
  integer i
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t2/a2
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t2/a2
  i = x3%a2
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t1/a1
  i = x3%a1
- !REF: /main/i
- !REF: /main/x3
- !DEF: /main/t3/t2 (ParentComp) ObjectEntity TYPE(t2)
- !REF: /main/t2/a2
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !DEF: /MAIN/t3/t2 (ParentComp) ObjectEntity TYPE(t2)
+ !REF: /MAIN/t2/a2
  i = x3%t2%a2
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t3/t2
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t3/t2
+ !REF: /MAIN/t1/a1
  i = x3%t2%a1
- !REF: /main/i
- !REF: /main/x3
- !DEF: /main/t2/t1 (ParentComp) ObjectEntity TYPE(t1)
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !DEF: /MAIN/t2/t1 (ParentComp) ObjectEntity TYPE(t1)
+ !REF: /MAIN/t1/a1
  i = x3%t1%a1
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t3/t2
- !REF: /main/t2/t1
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t3/t2
+ !REF: /MAIN/t2/t1
+ !REF: /MAIN/t1/a1
  i = x3%t2%t1%a1
 end program
 
diff --git a/flang/test/Semantics/symbol07.f90 b/flang/test/Semantics/symbol07.f90
index f3cc934e51b16..b3e92f1465fa6 100644
--- a/flang/test/Semantics/symbol07.f90
+++ b/flang/test/Semantics/symbol07.f90
@@ -1,40 +1,40 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  implicit complex(z)
- !DEF: /main/t DerivedType
+ !DEF: /MAIN/t DerivedType
  type :: t
-  !DEF: /main/t/re ObjectEntity REAL(4)
+  !DEF: /MAIN/t/re ObjectEntity REAL(4)
   real :: re
-  !DEF: /main/t/im ObjectEntity REAL(4)
+  !DEF: /MAIN/t/im ObjectEntity REAL(4)
   real :: im
  end type
- !DEF: /main/z1 ObjectEntity COMPLEX(4)
+ !DEF: /MAIN/z1 ObjectEntity COMPLEX(4)
  complex z1
- !REF: /main/t
- !DEF: /main/w ObjectEntity TYPE(t)
+ !REF: /MAIN/t
+ !DEF: /MAIN/w ObjectEntity TYPE(t)
  type(t) :: w
- !DEF: /main/x ObjectEntity REAL(4)
- !DEF: /main/y ObjectEntity REAL(4)
+ !DEF: /MAIN/x ObjectEntity REAL(4)
+ !DEF: /MAIN/y ObjectEntity REAL(4)
  real x, y
- !REF: /main/x
- !REF: /main/z1
+ !REF: /MAIN/x
+ !REF: /MAIN/z1
  x = z1%re
- !REF: /main/y
- !REF: /main/z1
+ !REF: /MAIN/y
+ !REF: /MAIN/z1
  y = z1%im
- !DEF: /main/z2 (Implicit) ObjectEntity COMPLEX(4)
- !REF: /main/x
+ !DEF: /MAIN/z2 (Implicit) ObjectEntity COMPLEX(4)
+ !REF: /MAIN/x
  z2%re = x
- !REF: /main/z2
- !REF: /main/y
+ !REF: /MAIN/z2
+ !REF: /MAIN/y
  z2%im = y
- !REF: /main/x
- !REF: /main/w
- !REF: /main/t/re
+ !REF: /MAIN/x
+ !REF: /MAIN/w
+ !REF: /MAIN/t/re
  x = w%re
- !REF: /main/y
- !REF: /main/w
- !REF: /main/t/im
+ !REF: /MAIN/y
+ !REF: /MAIN/w
+ !REF: /MAIN/t/im
  y = w%im
 end program
diff --git a/flang/test/Semantics/symbol08.f90 b/flang/test/Semantics/symbol08.f90
index 61dab798955c5..e2e9959d0a591 100644
--- a/flang/test/Semantics/symbol08.f90
+++ b/flang/test/Semantics/symbol08.f90
@@ -1,15 +1,15 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/x POINTER ObjectEntity REAL(4)
+ !DEF: /MAIN/x POINTER ObjectEntity REAL(4)
  pointer :: x
- !REF: /main/x
+ !REF: /MAIN/x
  real x
- !DEF: /main/y EXTERNAL, POINTER (Function) ProcEntity REAL(4)
+ !DEF: /MAIN/y EXTERNAL, POINTER (Function) ProcEntity REAL(4)
  pointer :: y
- !REF: /main/y
+ !REF: /MAIN/y
  procedure (real) :: y
- !DEF: /main/z (Implicit) ObjectEntity REAL(4)
- !REF: /main/y
+ !DEF: /MAIN/z (Implicit) ObjectEntity REAL(4)
+ !REF: /MAIN/y
  z = y()
 end program
diff --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index df10942e6af2d..5cc1cf58b4f21 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -249,15 +249,15 @@ subroutine ext2
 !DEF: /ext3 (Subroutine) Subprogram
 subroutine ext3
 end subroutine
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /m
  use :: m
- !DEF: /main/pdt1 Use
- !DEF: /main/pdt1y ObjectEntity TYPE(pdt1(k=2_4))
+ !DEF: /MAIN/pdt1 Use
+ !DEF: /MAIN/pdt1y ObjectEntity TYPE(pdt1(k=2_4))
  type(pdt1(2)) :: pdt1y
- !DEF: /main/pdt2 Use
- !DEF: /main/pdt2y ObjectEntity TYPE(pdt2(k=2_4))
+ !DEF: /MAIN/pdt2 Use
+ !DEF: /MAIN/pdt2y ObjectEntity TYPE(pdt2(k=2_4))
  type(pdt2(2)) :: pdt2y
  print *, "compiled"
 end program
diff --git a/flang/test/Semantics/symbol16.f90 b/flang/test/Semantics/symbol16.f90
index 7a46092c36b53..c29831a5dfa90 100644
--- a/flang/test/Semantics/symbol16.f90
+++ b/flang/test/Semantics/symbol16.f90
@@ -1,18 +1,18 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Statement functions
 
-!DEF: /p1 MainProgram
+!DEF: /P1 MainProgram
 program p1
- !DEF: /p1/f (Function, StmtFunction) Subprogram INTEGER(4)
- !DEF: /p1/i ObjectEntity INTEGER(4)
- !DEF: /p1/j ObjectEntity INTEGER(4)
+ !DEF: /P1/f (Function, StmtFunction) Subprogram INTEGER(4)
+ !DEF: /P1/i ObjectEntity INTEGER(4)
+ !DEF: /P1/j ObjectEntity INTEGER(4)
  integer f, i, j
- !REF: /p1/f
- !REF: /p1/i
- !DEF: /p1/f/i ObjectEntity INTEGER(4)
+ !REF: /P1/f
+ !REF: /P1/i
+ !DEF: /P1/f/i ObjectEntity INTEGER(4)
  f(i) = i + 1
- !REF: /p1/j
- !REF: /p1/f
+ !REF: /P1/j
+ !REF: /P1/f
  j = f(2)
 end program
 
diff --git a/flang/test/Semantics/symbol17.f90 b/flang/test/Semantics/symbol17.f90
index 434f124509a32..51f3afb19692d 100644
--- a/flang/test/Semantics/symbol17.f90
+++ b/flang/test/Semantics/symbol17.f90
@@ -1,44 +1,44 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Forward references to derived types (non-error cases)
 
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/t1 DerivedType
+ !DEF: /MAIN/t1 DerivedType
  type :: t1
-  !DEF: /main/t2 DerivedType
-  !DEF: /main/t1/t1a ALLOCATABLE ObjectEntity TYPE(t2)
+  !DEF: /MAIN/t2 DerivedType
+  !DEF: /MAIN/t1/t1a ALLOCATABLE ObjectEntity TYPE(t2)
   type(t2), allocatable :: t1a
-  !REF: /main/t2
-  !DEF: /main/t1/t1p POINTER ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t1/t1p POINTER ObjectEntity TYPE(t2)
   type(t2), pointer :: t1p
  end type
- !REF: /main/t2
+ !REF: /MAIN/t2
  type :: t2
-  !REF: /main/t2
-  !DEF: /main/t2/t2a ALLOCATABLE ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t2/t2a ALLOCATABLE ObjectEntity TYPE(t2)
   type(t2), allocatable :: t2a
-  !REF: /main/t2
-  !DEF: /main/t2/t2p POINTER ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t2/t2p POINTER ObjectEntity TYPE(t2)
   type(t2), pointer :: t2p
  end type
- !REF: /main/t1
- !DEF: /main/t1x TARGET ObjectEntity TYPE(t1)
+ !REF: /MAIN/t1
+ !DEF: /MAIN/t1x TARGET ObjectEntity TYPE(t1)
  type(t1), target :: t1x
- !REF: /main/t1x
- !REF: /main/t1/t1a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
  allocate(t1x%t1a)
- !REF: /main/t1x
- !REF: /main/t1/t1p
- !REF: /main/t1/t1a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1p
+ !REF: /MAIN/t1/t1a
  t1x%t1p => t1x%t1a
- !REF: /main/t1x
- !REF: /main/t1/t1a
- !REF: /main/t2/t2a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
+ !REF: /MAIN/t2/t2a
  allocate(t1x%t1a%t2a)
- !REF: /main/t1x
- !REF: /main/t1/t1a
- !REF: /main/t2/t2p
- !REF: /main/t2/t2a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
+ !REF: /MAIN/t2/t2p
+ !REF: /MAIN/t2/t2a
  t1x%t1a%t2p => t1x%t1a%t2a
 end program
 !DEF: /f1/fwd DerivedType
diff --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
index a37792bce21d7..bfe25ff4df25f 100644
--- a/flang/test/Semantics/symbol18.f90
+++ b/flang/test/Semantics/symbol18.f90
@@ -2,21 +2,21 @@
 
 ! Intrinsic function in type declaration statement: type is ignored
 
-!DEF: /p1 MainProgram
+!DEF: /P1 MainProgram
 program p1
- !DEF: /p1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
+ !DEF: /P1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
  integer cos
- !DEF: /p1/y (Implicit) ObjectEntity REAL(4)
- !REF: /p1/cos
- !DEF: /p1/x (Implicit) ObjectEntity REAL(4)
+ !DEF: /P1/y (Implicit) ObjectEntity REAL(4)
+ !REF: /P1/cos
+ !DEF: /P1/x (Implicit) ObjectEntity REAL(4)
  y = cos(x)
- !REF: /p1/y
- !DEF: /p1/sin ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
- !REF: /p1/x
+ !REF: /P1/y
+ !DEF: /P1/sin ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+ !REF: /P1/x
  y = sin(x)
- !REF: /p1/y
+ !REF: /P1/y
  !DEF: /f EXTERNAL (Function, Implicit) ProcEntity REAL(4)
- !REF: /p1/x
+ !REF: /P1/x
  y = f(x)
 end program
 
diff --git a/flang/test/Semantics/symbol20.f90 b/flang/test/Semantics/symbol20.f90
index 8c82776933321..20bd1bdbf2a7c 100644
--- a/flang/test/Semantics/symbol20.f90
+++ b/flang/test/Semantics/symbol20.f90
@@ -32,16 +32,16 @@ subroutine bar
   print *, "in bar"
  end subroutine
 end module
-!DEF: /demo MainProgram
+!DEF: /DEMO MainProgram
 program demo
  !REF: /m
  use :: m
- !DEF: /demo/bar (Subroutine) Use
- !DEF: /demo/p EXTERNAL, POINTER (Subroutine) ProcEntity
+ !DEF: /DEMO/bar (Subroutine) Use
+ !DEF: /DEMO/p EXTERNAL, POINTER (Subroutine) ProcEntity
  procedure(bar), pointer :: p
- !REF: /demo/p
- !DEF: /demo/foo (Function) Use
+ !REF: /DEMO/p
+ !DEF: /DEMO/foo (Function) Use
  p => foo()
- !REF: /demo/p
+ !REF: /DEMO/p
  call p
 end program
diff --git a/flang/test/Semantics/symbol25.f90 b/flang/test/Semantics/symbol25.f90
index ac3dd37ef92eb..a131bf899340f 100644
--- a/flang/test/Semantics/symbol25.f90
+++ b/flang/test/Semantics/symbol25.f90
@@ -38,23 +38,23 @@ subroutine inner1
   end subroutine inner1
  end subroutine outer
 end module m
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
  !REF: /m
  use :: m
  !REF: /m/specific1
  call generic
- !DEF: /main/inner2 (Subroutine) Subprogram
+ !DEF: /MAIN/inner2 (Subroutine) Subprogram
  call inner2
 contains
- !REF: /main/inner2
+ !REF: /MAIN/inner2
  subroutine inner2
-  !DEF: /main/inner2/generic (Subroutine) Generic
+  !DEF: /MAIN/inner2/generic (Subroutine) Generic
   interface generic
-   !DEF: /main/specific2 (Subroutine) Use
+   !DEF: /MAIN/specific2 (Subroutine) Use
    module procedure :: specific2
   end interface
-  !REF: /main/specific2
+  !REF: /MAIN/specific2
   call generic
  end subroutine inner2
 end program
diff --git a/flang/test/Semantics/symbol26.f90 b/flang/test/Semantics/symbol26.f90
index f5e95853ca099..e04cd32df57c4 100644
--- a/flang/test/Semantics/symbol26.f90
+++ b/flang/test/Semantics/symbol26.f90
@@ -8,16 +8,16 @@ module m
  !DEF: /m/j PUBLIC (Implicit, InNamelist) ObjectEntity INTEGER(4)
  namelist/a/j
 end module m
-!DEF: /main MainProgram
+!DEF: /MAIN MainProgram
 program main
- !DEF: /main/j (Implicit) ObjectEntity INTEGER(4)
+ !DEF: /MAIN/j (Implicit) ObjectEntity INTEGER(4)
  j = 1
 contains
- !DEF: /main/inner (Subroutine) Subprogram
+ !DEF: /MAIN/inner (Subroutine) Subprogram
  subroutine inner
   !REF: /m
   use :: m
-  !DEF: /main/inner/j (Implicit, InNamelist) Use INTEGER(4)
+  !DEF: /MAIN/inner/j (Implicit, InNamelist) Use INTEGER(4)
   j = 2
  end subroutine
 end program
diff --git a/flang/test/Transforms/DoConcurrent/basic_host.f90 b/flang/test/Transforms/DoConcurrent/basic_host.f90
index 12f63031cbaee..6f24b346e3fb9 100644
--- a/flang/test/Transforms/DoConcurrent/basic_host.f90
+++ b/flang/test/Transforms/DoConcurrent/basic_host.f90
@@ -5,7 +5,7 @@
 ! RUN: bbc -emit-hlfir -fopenmp -fdo-concurrent-to-openmp=host %s -o - \
 ! RUN:   | FileCheck %s
  
-! CHECK-LABEL: do_concurrent_basic
+! CHECK-LABEL: DO_CONCURRENT_BASIC
 program do_concurrent_basic
     ! CHECK: %[[ARR:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFEa"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 

>From 632cd4b1351cbb57efbdf3cb8fe9435211004a0a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:24:17 -0400
Subject: [PATCH 04/11] More test changes

---
 .../test/Semantics/OpenACC/acc-symbols01.f90  | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/flang/test/Semantics/OpenACC/acc-symbols01.f90 b/flang/test/Semantics/OpenACC/acc-symbols01.f90
index ea22f41197aa7..51a7a3a23e8ce 100644
--- a/flang/test/Semantics/OpenACC/acc-symbols01.f90
+++ b/flang/test/Semantics/OpenACC/acc-symbols01.f90
@@ -1,24 +1,24 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenacc
 
 !DEF: /MM MainProgram
-program mm
-  !DEF: /mm/x ObjectEntity REAL(4)
-  !DEF: /mm/y ObjectEntity REAL(4)
+program MM
+  !DEF: /MM/x ObjectEntity REAL(4)
+  !DEF: /MM/y ObjectEntity REAL(4)
   real x, y
-  !DEF: /mm/a ObjectEntity INTEGER(4)
-  !DEF: /mm/b ObjectEntity INTEGER(4)
-  !DEF: /mm/c ObjectEntity INTEGER(4)
-  !DEF: /mm/i ObjectEntity INTEGER(4)
+  !DEF: /MM/a ObjectEntity INTEGER(4)
+  !DEF: /MM/b ObjectEntity INTEGER(4)
+  !DEF: /MM/c ObjectEntity INTEGER(4)
+  !DEF: /MM/i ObjectEntity INTEGER(4)
   integer a(10), b(10), c(10), i
-  !REF: /mm/b
+  !REF: /MM/b
   b = 2
  !$acc parallel present(c) firstprivate(b) private(a)
  !$acc loop
-  !REF: /mm/i
+  !REF: /MM/i
   do i=1,10
-   !REF: /mm/a
-   !REF: /mm/i
-   !REF: /mm/b
+   !REF: /MM/a
+   !REF: /MM/i
+   !REF: /MM/b
    a(i) = b(i)
   end do
  !$acc end parallel

>From ce7b837b6554c62ad459729674ddb0cb84f49747 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 18:37:07 -0400
Subject: [PATCH 05/11] more tests

---
 flang/test/Semantics/OpenMP/do-schedule03.f90      |  4 ++--
 flang/test/Semantics/OpenMP/do01-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do04-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do05-positivecase.f90  |  4 ++--
 flang/test/Semantics/OpenMP/do06-positivecases.f90 |  4 ++--
 flang/test/Semantics/OpenMP/do11.f90               | 14 +++++++-------
 6 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/flang/test/Semantics/OpenMP/do-schedule03.f90 b/flang/test/Semantics/OpenMP/do-schedule03.f90
index 375f3920b57ad..05602ca57e4a9 100644
--- a/flang/test/Semantics/OpenMP/do-schedule03.f90
+++ b/flang/test/Semantics/OpenMP/do-schedule03.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - xx.
   !DEF: /OMPDOSCHEDULE MainProgram
-program ompdoschedule
+program OMPDOSCHEDULE
   !DEF: /OMPDOSCHEDULE/a ObjectEntity REAL(4)
   !DEF: /OMPDOSCHEDULE/y ObjectEntity REAL(4)
   !DEF: /OMPDOSCHEDULE/z ObjectEntity REAL(4)
@@ -25,4 +25,4 @@ program ompdoschedule
     y(i) = z(i-1) + a(i)
   end do
   !$omp end do
-end program ompdoschedule
+end program OMPDOSCHEDULE
diff --git a/flang/test/Semantics/OpenMP/do01-positivecase.f90 b/flang/test/Semantics/OpenMP/do01-positivecase.f90
index a4a28f6530d7b..50a6870f43896 100644
--- a/flang/test/Semantics/OpenMP/do01-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do01-positivecase.f90
@@ -5,7 +5,7 @@
 ! A positive case
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   integer i
 
@@ -16,4 +16,4 @@ program omp_do
   end do
   !$omp end do
 
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do04-positivecase.f90 b/flang/test/Semantics/OpenMP/do04-positivecase.f90
index 0b0c661687dc2..51b69fce3c7cc 100644
--- a/flang/test/Semantics/OpenMP/do04-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do04-positivecase.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop Constructs
 
 !DEF: /OMP_DO1 MainProgram
-program omp_do1
+program OMP_DO1
   !DEF: /OMP_DO1/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO1/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
@@ -19,4 +19,4 @@ program omp_do1
     end do
   end do
   !$omp end do
-end program omp_do1
+end program OMP_DO1
diff --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index 07b12dd1d4d23..d4eb1fd6bc3da 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -4,7 +4,7 @@
 ! A positive case
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/n ObjectEntity INTEGER(4)
   integer i,n
@@ -71,4 +71,4 @@ program omp_do
     !REF: /OMP_DO/OtherConstruct7/i
     if(i < 5) cycle
   end do
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do06-positivecases.f90 b/flang/test/Semantics/OpenMP/do06-positivecases.f90
index 4d45800298a29..dfb1d999bbc53 100644
--- a/flang/test/Semantics/OpenMP/do06-positivecases.f90
+++ b/flang/test/Semantics/OpenMP/do06-positivecases.f90
@@ -6,7 +6,7 @@
 
 ! A positive case
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
@@ -20,4 +20,4 @@ program omp_do
       !$omp end ordered
     end do
   !$omp end do
-end program omp_do
+end program OMP_DO
diff --git a/flang/test/Semantics/OpenMP/do11.f90 b/flang/test/Semantics/OpenMP/do11.f90
index 886a9f7ae6081..472048d684276 100644
--- a/flang/test/Semantics/OpenMP/do11.f90
+++ b/flang/test/Semantics/OpenMP/do11.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop Constructs
 
 !DEF: /OMP_DO MainProgram
-program omp_do
+program OMP_DO
   !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
   !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
@@ -19,17 +19,17 @@ program omp_do
     end do
   end do
   !$omp end do
-end program omp_do
+end program OMP_DO
 
-!DEF: /OMP_DO2 (Subroutine)Subprogram
+!DEF: /omp_do2 (Subroutine)Subprogram
 subroutine omp_do2
-  !DEF: /OMP_DO2/i ObjectEntity INTEGER(4)
-  !DEF: /OMP_DO2/k ObjectEntity INTEGER(4)
+  !DEF: /omp_do2/i ObjectEntity INTEGER(4)
+  !DEF: /omp_do2/k ObjectEntity INTEGER(4)
   integer :: i = 0, k
   !$omp do
-  !DEF: /OMP_DO2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /omp_do2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /OMP_DO2/OtherConstruct1/i
+    !REF: /omp_do2/OtherConstruct1/i
     print *, "it", i
   end do
   !$omp end do

>From 3e3d5d489bd2d299cb14bdeca728856f8bd59f18 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Jul 2025 23:03:25 -0400
Subject: [PATCH 06/11] More test changes

---
 flang/test/Semantics/OpenMP/declare-target03.f90 | 4 ++--
 flang/test/Semantics/OpenMP/do12.f90             | 2 +-
 flang/test/Semantics/OpenMP/do14.f90             | 4 ++--
 flang/test/Semantics/OpenMP/do17.f90             | 4 ++--
 flang/test/Semantics/OpenMP/reduction08.f90      | 4 ++--
 flang/test/Semantics/OpenMP/reduction09.f90      | 4 ++--
 flang/test/Semantics/OpenMP/symbol01.f90         | 2 +-
 flang/test/Semantics/OpenMP/symbol05.f90         | 4 ++--
 flang/test/Semantics/OpenMP/symbol07.f90         | 4 ++--
 flang/test/Semantics/OpenMP/symbol09.f90         | 4 ++--
 flang/test/Semantics/OpenMP/threadprivate03.f90  | 4 ++--
 flang/test/Semantics/long-name.f90               | 2 +-
 flang/test/Semantics/pointer01.f90               | 1 -
 flang/test/Semantics/procinterface01.f90         | 4 ++--
 flang/test/Semantics/resolve05.f90               | 1 -
 flang/test/Semantics/symbol03.f90                | 2 +-
 flang/test/Semantics/symbol06.f90                | 2 +-
 flang/test/Semantics/symbol07.f90                | 2 +-
 flang/test/Semantics/symbol08.f90                | 2 +-
 flang/test/Semantics/symbol15.f90                | 2 +-
 flang/test/Semantics/symbol16.f90                | 2 +-
 flang/test/Semantics/symbol17.f90                | 2 +-
 flang/test/Semantics/symbol18.f90                | 2 +-
 flang/test/Semantics/symbol20.f90                | 2 +-
 flang/test/Semantics/symbol25.f90                | 2 +-
 flang/test/Semantics/symbol26.f90                | 2 +-
 26 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index 64a299d78224a..b3d6551ac85de 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -16,7 +16,7 @@ program main
   !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
-  !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
+  ! This is now allowed: "main" is implicitly declared symbol separate
+  ! from the main program symbol
   !$omp declare target (main)
 end
diff --git a/flang/test/Semantics/OpenMP/do12.f90 b/flang/test/Semantics/OpenMP/do12.f90
index f36a230426197..06055b7572a60 100644
--- a/flang/test/Semantics/OpenMP/do12.f90
+++ b/flang/test/Semantics/OpenMP/do12.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /OMP_CYCLE MainProgram
-program omp_cycle
+program OMP_CYCLE
   !$omp do  collapse(1)
   !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
diff --git a/flang/test/Semantics/OpenMP/do14.f90 b/flang/test/Semantics/OpenMP/do14.f90
index b508d32d90201..e17647394fff7 100644
--- a/flang/test/Semantics/OpenMP/do14.f90
+++ b/flang/test/Semantics/OpenMP/do14.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /OMP_CYCLE MainProgram
-program omp_cycle
+program OMP_CYCLE
   !$omp do  collapse(1)
   !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
@@ -88,4 +88,4 @@ program omp_cycle
     end do foo1
   end do foo
   !$omp end do
-end program omp_cycle
+end program OMP_CYCLE
diff --git a/flang/test/Semantics/OpenMP/do17.f90 b/flang/test/Semantics/OpenMP/do17.f90
index 0a4cf46d2d6af..cac11f215f074 100644
--- a/flang/test/Semantics/OpenMP/do17.f90
+++ b/flang/test/Semantics/OpenMP/do17.f90
@@ -3,7 +3,7 @@
 ! 2.7.1 Do Loop constructs.
 
 !DEF: /TEST MainProgram
-program test
+program TEST
  !DEF: /TEST/i ObjectEntity INTEGER(4)
  !DEF: /TEST/j ObjectEntity INTEGER(4)
  !DEF: /TEST/k ObjectEntity INTEGER(4)
@@ -54,4 +54,4 @@ program test
   end do foo1
  end do foo
 !$omp end do
-end program test
+end program TEST
diff --git a/flang/test/Semantics/OpenMP/reduction08.f90 b/flang/test/Semantics/OpenMP/reduction08.f90
index 1cf2838fe765a..b4a81e644c1e7 100644
--- a/flang/test/Semantics/OpenMP/reduction08.f90
+++ b/flang/test/Semantics/OpenMP/reduction08.f90
@@ -3,7 +3,7 @@
 ! 2.15.3.6 Reduction Clause Positive cases
 
 !DEF: /OMP_REDUCTION MainProgram
-program omp_reduction
+program OMP_REDUCTION
   !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
   !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
@@ -60,4 +60,4 @@ program omp_reduction
   end do
   !$omp end parallel do
 
-end program omp_reduction
+end program OMP_REDUCTION
diff --git a/flang/test/Semantics/OpenMP/reduction09.f90 b/flang/test/Semantics/OpenMP/reduction09.f90
index 4b1ca116d675b..ca60805e8c416 100644
--- a/flang/test/Semantics/OpenMP/reduction09.f90
+++ b/flang/test/Semantics/OpenMP/reduction09.f90
@@ -2,7 +2,7 @@
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases.
 !DEF: /OMP_REDUCTION MainProgram
-program omp_reduction
+program OMP_REDUCTION
   !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
   !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
@@ -73,4 +73,4 @@ program omp_reduction
     k = k+1
   end do
   !$omp end do
-end program omp_reduction
+end program OMP_REDUCTION
diff --git a/flang/test/Semantics/OpenMP/symbol01.f90 b/flang/test/Semantics/OpenMP/symbol01.f90
index c1f16d61ada1e..74fb420cc517e 100644
--- a/flang/test/Semantics/OpenMP/symbol01.f90
+++ b/flang/test/Semantics/OpenMP/symbol01.f90
@@ -17,7 +17,7 @@ module md
  end type myty
 end module md
 !DEF: /MM MainProgram
-program mm
+program MM
  !REF: /md
  use :: md
  !DEF: /MM/c CommonBlockDetails
diff --git a/flang/test/Semantics/OpenMP/symbol05.f90 b/flang/test/Semantics/OpenMP/symbol05.f90
index 03e2c0ba3e0cc..4f3d1926013dc 100644
--- a/flang/test/Semantics/OpenMP/symbol05.f90
+++ b/flang/test/Semantics/OpenMP/symbol05.f90
@@ -32,9 +32,9 @@ subroutine foo
   end subroutine foo
 end module mm
 !DEF: /TT MainProgram
-program tt
+program TT
   !REF: /mm
   use :: mm
   !DEF: /TT/foo (Subroutine) Use
   call foo
-end program tt
+end program TT
diff --git a/flang/test/Semantics/OpenMP/symbol07.f90 b/flang/test/Semantics/OpenMP/symbol07.f90
index f5eb2127819bd..1b0c25b7a04b0 100644
--- a/flang/test/Semantics/OpenMP/symbol07.f90
+++ b/flang/test/Semantics/OpenMP/symbol07.f90
@@ -31,7 +31,7 @@ subroutine function_call_in_region
   print *, a, b
 end subroutine function_call_in_region
 !DEF: /MM MainProgram
-program mm
+program MM
   !REF: /function_call_in_region
   call function_call_in_region
-end program mm
+end program MM
diff --git a/flang/test/Semantics/OpenMP/symbol09.f90 b/flang/test/Semantics/OpenMP/symbol09.f90
index f5eb2127819bd..1b0c25b7a04b0 100644
--- a/flang/test/Semantics/OpenMP/symbol09.f90
+++ b/flang/test/Semantics/OpenMP/symbol09.f90
@@ -31,7 +31,7 @@ subroutine function_call_in_region
   print *, a, b
 end subroutine function_call_in_region
 !DEF: /MM MainProgram
-program mm
+program MM
   !REF: /function_call_in_region
   call function_call_in_region
-end program mm
+end program MM
diff --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index 81e26ee327a9d..9f222773ad432 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -13,8 +13,8 @@ program main
   !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
-  !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
+  ! This is now allowed, since "main" is implicitly declared symbol,
+  ! separate from the main program symbol.
   !$omp threadprivate(main)
 
   !ERROR: The entity with PARAMETER attribute cannot be in a THREADPRIVATE directive
diff --git a/flang/test/Semantics/long-name.f90 b/flang/test/Semantics/long-name.f90
index 44899b13edd5a..d5a795113e204 100644
--- a/flang/test/Semantics/long-name.f90
+++ b/flang/test/Semantics/long-name.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror -pedantic
 
-!PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
+!PORTABILITY: AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEEFFFFFFFFFFGGG1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
 program aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1
 
   !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
diff --git a/flang/test/Semantics/pointer01.f90 b/flang/test/Semantics/pointer01.f90
index eaa2426dd77e3..79d6016a6af46 100644
--- a/flang/test/Semantics/pointer01.f90
+++ b/flang/test/Semantics/pointer01.f90
@@ -7,7 +7,6 @@ subroutine msubr
 end module
 program main
   use m
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   pointer main
   !ERROR: Cannot change POINTER attribute on use-associated 'mobj'
   pointer mobj
diff --git a/flang/test/Semantics/procinterface01.f90 b/flang/test/Semantics/procinterface01.f90
index 7542bb8bb6fa4..70f4a889d6809 100644
--- a/flang/test/Semantics/procinterface01.f90
+++ b/flang/test/Semantics/procinterface01.f90
@@ -160,7 +160,7 @@ end function logical
 end function tan
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /module1
  use :: module1
  !DEF: /MAIN/derived1 Use
@@ -190,4 +190,4 @@ program main
  !REF: /MAIN/instance
  !REF: /module1/derived1/p8
  if (instance%p8(1.)/="a") print *, "p8 failed"
-end program main
+end program MAIN
diff --git a/flang/test/Semantics/resolve05.f90 b/flang/test/Semantics/resolve05.f90
index 0c9877af9b4e2..7b142d2ebd613 100644
--- a/flang/test/Semantics/resolve05.f90
+++ b/flang/test/Semantics/resolve05.f90
@@ -1,6 +1,5 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 program p
-  !PORTABILITY: Name 'p' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   integer :: p
 end
 module m
diff --git a/flang/test/Semantics/symbol03.f90 b/flang/test/Semantics/symbol03.f90
index f50a0e91fd289..62472495d9736 100644
--- a/flang/test/Semantics/symbol03.f90
+++ b/flang/test/Semantics/symbol03.f90
@@ -2,7 +2,7 @@
 ! Test host association in internal subroutine of main program.
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/x ObjectEntity INTEGER(4)
  integer x
  !DEF: /MAIN/s (Subroutine) Subprogram
diff --git a/flang/test/Semantics/symbol06.f90 b/flang/test/Semantics/symbol06.f90
index 984ebcc1c651c..b45edabcd5318 100644
--- a/flang/test/Semantics/symbol06.f90
+++ b/flang/test/Semantics/symbol06.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/t1 DerivedType
  type :: t1
   !DEF: /MAIN/t1/a1 ObjectEntity INTEGER(4)
diff --git a/flang/test/Semantics/symbol07.f90 b/flang/test/Semantics/symbol07.f90
index b3e92f1465fa6..e1d8257b9e190 100644
--- a/flang/test/Semantics/symbol07.f90
+++ b/flang/test/Semantics/symbol07.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  implicit complex(z)
  !DEF: /MAIN/t DerivedType
  type :: t
diff --git a/flang/test/Semantics/symbol08.f90 b/flang/test/Semantics/symbol08.f90
index e2e9959d0a591..933ff6d0c2ba8 100644
--- a/flang/test/Semantics/symbol08.f90
+++ b/flang/test/Semantics/symbol08.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/x POINTER ObjectEntity REAL(4)
  pointer :: x
  !REF: /MAIN/x
diff --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index 5cc1cf58b4f21..79a45491306ef 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -250,7 +250,7 @@ subroutine ext2
 subroutine ext3
 end subroutine
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /m
  use :: m
  !DEF: /MAIN/pdt1 Use
diff --git a/flang/test/Semantics/symbol16.f90 b/flang/test/Semantics/symbol16.f90
index c29831a5dfa90..547c4624d4cdb 100644
--- a/flang/test/Semantics/symbol16.f90
+++ b/flang/test/Semantics/symbol16.f90
@@ -2,7 +2,7 @@
 ! Statement functions
 
 !DEF: /P1 MainProgram
-program p1
+program P1
  !DEF: /P1/f (Function, StmtFunction) Subprogram INTEGER(4)
  !DEF: /P1/i ObjectEntity INTEGER(4)
  !DEF: /P1/j ObjectEntity INTEGER(4)
diff --git a/flang/test/Semantics/symbol17.f90 b/flang/test/Semantics/symbol17.f90
index 51f3afb19692d..a0d916e55cfa4 100644
--- a/flang/test/Semantics/symbol17.f90
+++ b/flang/test/Semantics/symbol17.f90
@@ -2,7 +2,7 @@
 ! Forward references to derived types (non-error cases)
 
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/t1 DerivedType
  type :: t1
   !DEF: /MAIN/t2 DerivedType
diff --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
index bfe25ff4df25f..6e41bb5db91ee 100644
--- a/flang/test/Semantics/symbol18.f90
+++ b/flang/test/Semantics/symbol18.f90
@@ -3,7 +3,7 @@
 ! Intrinsic function in type declaration statement: type is ignored
 
 !DEF: /P1 MainProgram
-program p1
+program P1
  !DEF: /P1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
  integer cos
  !DEF: /P1/y (Implicit) ObjectEntity REAL(4)
diff --git a/flang/test/Semantics/symbol20.f90 b/flang/test/Semantics/symbol20.f90
index 20bd1bdbf2a7c..bf3aff489b3b9 100644
--- a/flang/test/Semantics/symbol20.f90
+++ b/flang/test/Semantics/symbol20.f90
@@ -33,7 +33,7 @@ subroutine bar
  end subroutine
 end module
 !DEF: /DEMO MainProgram
-program demo
+program DEMO
  !REF: /m
  use :: m
  !DEF: /DEMO/bar (Subroutine) Use
diff --git a/flang/test/Semantics/symbol25.f90 b/flang/test/Semantics/symbol25.f90
index a131bf899340f..ac47a19eae8cc 100644
--- a/flang/test/Semantics/symbol25.f90
+++ b/flang/test/Semantics/symbol25.f90
@@ -39,7 +39,7 @@ end subroutine inner1
  end subroutine outer
 end module m
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !REF: /m
  use :: m
  !REF: /m/specific1
diff --git a/flang/test/Semantics/symbol26.f90 b/flang/test/Semantics/symbol26.f90
index e04cd32df57c4..dded4b632c654 100644
--- a/flang/test/Semantics/symbol26.f90
+++ b/flang/test/Semantics/symbol26.f90
@@ -9,7 +9,7 @@ module m
  namelist/a/j
 end module m
 !DEF: /MAIN MainProgram
-program main
+program MAIN
  !DEF: /MAIN/j (Implicit) ObjectEntity INTEGER(4)
  j = 1
 contains

>From 37dbcbf9c27aa973b939dfa45a709e8307937405 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 14:28:06 -0400
Subject: [PATCH 07/11] More test changes

---
 flang/test/Semantics/getsymbols03-a.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index e3c2e0bf53f92..5c5e87575a9cb 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -8,7 +8,7 @@ program main
 end program
 
 ! RUN: %flang_fc1 -fget-symbols-sources %s 2>&1 | FileCheck %s
-! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
 ! CHECK:MAIN:{{.*}}getsymbols03-a.f90, 4, 9-13
+! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
 ! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
 ! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14

>From 8cd6fa1d597d328effcbbd1544d3e614e5ad8377 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 15:06:18 -0400
Subject: [PATCH 08/11] Tweaked the error message and updated the tests
 accordingly

---
 flang/lib/Semantics/check-omp-structure.cpp      | 3 +--
 flang/test/Semantics/OpenMP/declare-target03.f90 | 2 +-
 flang/test/Semantics/OpenMP/threadprivate03.f90  | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2425265e196c6..e4a94efcc6b55 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -1156,8 +1156,7 @@ void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar(
                         (sym->has<MainProgramDetails>() ||
                             sym->has<ModuleDetails>())) {
                       context_.Say(name->source,
-                          "The module name or main program name cannot be in a "
-                          "%s "
+                          "The module name cannot be in a %s "
                           "directive"_err_en_US,
                           ContextDirectiveAsFortran());
                     } else if (!IsSaved(*name->symbol) &&
diff --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index b3d6551ac85de..48cfc68393873 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -13,7 +13,7 @@ subroutine bar
 program main
   use mod1
 
-  !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
+  !ERROR: The module name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
   ! This is now allowed: "main" is implicitly declared symbol separate
diff --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index 9f222773ad432..fda2fe608ac3c 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -10,7 +10,7 @@ program main
   use mod1
   integer, parameter :: i = 1
 
-  !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
+  !ERROR: The module name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
   ! This is now allowed, since "main" is implicitly declared symbol,

>From 103af162b881d2f39f404d6546e11734b0feb795 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 15:15:01 -0400
Subject: [PATCH 09/11] Documentation change

---
 flang/docs/Extensions.md | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 72d12cd92600d..a7d9281be3e98 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -1,9 +1,9 @@
-<!--===- docs/Extensions.md 
-  
+<!--===- docs/Extensions.md
+
    Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
    See https://llvm.org/LICENSE.txt for license information.
    SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-  
+
 -->
 
 # Fortran Extensions supported by Flang
@@ -170,6 +170,15 @@ end
   In the case of `DEFERRED` bindings in an `ABSTRACT` derived type,
   however, overrides are necessary, so they are permitted for inaccessible
   bindings with an optional warning.
+* Main program name is allowed to be the same as the other symbols used
+  in the main program, for example:
+```
+module m
+end
+program m
+use m
+end
+```
 
 ## Extensions, deletions, and legacy features supported by default
 

>From 13b2ec43c619383690b7a2c673d5836f4173e6d8 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 20:22:13 -0400
Subject: [PATCH 10/11] Code review feedback

---
 flang/docs/Extensions.md               |  3 +++
 flang/lib/Semantics/resolve-labels.cpp | 11 ++++-------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index a7d9281be3e98..c167a55bc486d 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -179,6 +179,9 @@ program m
 use m
 end
 ```
+  Note that internally the main program symbol name is all uppercase, unlike
+  the names of all other symbols, which are usually all lowercase. This
+  may make a difference in testing/debugging.
 
 ## Extensions, deletions, and legacy features supported by default
 
diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index d3fda03383783..2b3ddccfcb2f5 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -492,8 +492,8 @@ class ParseTreeAnalyzer {
     // Uppercase the name of the main program, so that its symbol name
     // would be unique from similarly named non-main-program symbols.
     auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
-      char *ch = const_cast<char *>(cb.begin());
-      char *endCh = ch + cb.size();
+      char *ch{const_cast<char *>(cb.begin())};
+      char *endCh{ch + cb.size()};
       while (ch != endCh) {
         *ch++ = parser::ToUpperCaseLetter(*ch);
       }
@@ -503,13 +503,10 @@ class ParseTreeAnalyzer {
             std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
                 mainProgram.t)}) {
       progName = &program->statement.v.source;
-    }
-    if (progName) {
       upperCaseCharBlock(*progName);
     }
-    const parser::CharBlock *endName{GetStmtName(
-        std::get<parser::Statement<parser::EndProgramStmt>>(mainProgram.t))};
-    if (endName) {
+    if (const parser::CharBlock *endName{GetStmtName(
+        std::get<parser::Statement<parser::EndProgramStmt>>(mainProgram.t))}) {
       upperCaseCharBlock(*endName);
       if (progName) {
         if (*endName != *progName) {

>From 82f1fc244360fe8b06c48e5229712afc0aee3215 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 16 Jul 2025 20:22:57 -0400
Subject: [PATCH 11/11] clang-format

---
 flang/lib/Semantics/resolve-labels.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index 2b3ddccfcb2f5..27e259fab3873 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -505,8 +505,9 @@ class ParseTreeAnalyzer {
       progName = &program->statement.v.source;
       upperCaseCharBlock(*progName);
     }
-    if (const parser::CharBlock *endName{GetStmtName(
-        std::get<parser::Statement<parser::EndProgramStmt>>(mainProgram.t))}) {
+    if (const parser::CharBlock *
+        endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
+            mainProgram.t))}) {
       upperCaseCharBlock(*endName);
       if (progName) {
         if (*endName != *progName) {


From flang-commits at lists.llvm.org  Thu Jul 17 06:04:54 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 06:04:54 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <6878f4f6.050a0220.2ace57.994d@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH] [Flang][Flang-RT][OpenMP] Move builtin .mod generation into
 runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file


From flang-commits at lists.llvm.org  Thu Jul 17 06:14:35 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 06:14:35 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <6878f73b.630a0220.3b688f.9c68@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 1/2] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 2/2] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"


From flang-commits at lists.llvm.org  Thu Jul 17 08:40:50 2025
From: flang-commits at lists.llvm.org (Kazu Hirata via flang-commits)
Date: Thu, 17 Jul 2025 08:40:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Migrate away from
	ArrayRef(std::nullopt_t) (PR #149337)
Message-ID: <llvm.org/llvm/llvm-project/pull/149337@github.com>


https://github.com/kazutakahirata created https://github.com/llvm/llvm-project/pull/149337

ArrayRef(std::nullopt_t) has been deprecated.  This patch replaces
std::nullopt with {}.

A subsequence patch will address those places where we need to replace
std::nullopt with mlir::TypeRange{} or mlir::ValueRange{}.


>From 7d209694b90bbb4ecc7396370f610ed04a05e94e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Wed, 16 Jul 2025 13:20:33 -0700
Subject: [PATCH] [flang] Migrate away from ArrayRef(std::nullopt_t)

ArrayRef(std::nullopt_t) has been deprecated.  This patch replaces
std::nullopt with {}.

A subsequence patch will address those places where we need to replace
std::nullopt with mlir::TypeRange{} or mlir::ValueRange{}.
---
 flang/lib/Lower/Bridge.cpp                    | 24 ++++++++--------
 flang/lib/Lower/ConvertCall.cpp               |  5 ++--
 flang/lib/Lower/ConvertConstant.cpp           |  4 +--
 flang/lib/Lower/ConvertExpr.cpp               |  7 ++---
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |  2 +-
 flang/lib/Lower/ConvertVariable.cpp           | 11 ++++----
 flang/lib/Lower/HostAssociations.cpp          |  6 ++--
 flang/lib/Lower/IO.cpp                        |  2 +-
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |  2 +-
 .../Optimizer/Builder/LowLevelIntrinsics.cpp  | 13 ++++-----
 flang/lib/Optimizer/Builder/MutableBox.cpp    | 28 +++++++++----------
 flang/lib/Optimizer/Dialect/FIROps.cpp        |  2 +-
 .../HLFIR/Transforms/BufferizeHLFIR.cpp       |  2 +-
 .../Optimizer/Builder/CharacterTest.cpp       |  2 +-
 .../Optimizer/Builder/ComplexTest.cpp         |  2 +-
 .../Optimizer/Builder/FIRBuilderTest.cpp      |  5 ++--
 .../Optimizer/Builder/HLFIRToolsTest.cpp      |  2 +-
 .../Builder/Runtime/RuntimeCallTestBase.h     |  5 ++--
 .../Optimizer/FortranVariableTest.cpp         |  5 ++--
 19 files changed, 59 insertions(+), 70 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 33c1f1e7a3c3a..4241d12601242 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -810,11 +810,11 @@ class FirConverter : public Fortran::lower::AbstractConverter {
             fir::ExtendedValue read = fir::factory::genMutableBoxRead(
                 *builder, loc, box, /*mayBePolymorphic=*/false);
             if (auto read_arr_box = read.getBoxOf<fir::ArrayBoxValue>()) {
-              fir::factory::genInlinedAllocation(
-                  *builder, loc, *new_box, read_arr_box->getLBounds(),
-                  read_arr_box->getExtents(),
-                  /*lenParams=*/std::nullopt, name,
-                  /*mustBeHeap=*/true);
+              fir::factory::genInlinedAllocation(*builder, loc, *new_box,
+                                                 read_arr_box->getLBounds(),
+                                                 read_arr_box->getExtents(),
+                                                 /*lenParams=*/{}, name,
+                                                 /*mustBeHeap=*/true);
             } else if (auto read_char_arr_box =
                            read.getBoxOf<fir::CharArrayBoxValue>()) {
               fir::factory::genInlinedAllocation(
@@ -825,8 +825,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
             } else if (auto read_char_box =
                            read.getBoxOf<fir::CharBoxValue>()) {
               fir::factory::genInlinedAllocation(*builder, loc, *new_box,
-                                                 /*lbounds=*/std::nullopt,
-                                                 /*extents=*/std::nullopt,
+                                                 /*lbounds=*/{},
+                                                 /*extents=*/{},
                                                  read_char_box->getLen(), name,
                                                  /*mustBeHeap=*/true);
             } else {
@@ -4590,8 +4590,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // the static type of the LHS.
     if (Fortran::evaluate::UnwrapExpr<Fortran::evaluate::NullPointer>(
             assign.rhs))
-      return fir::factory::createUnallocatedBox(*builder, loc, lhsBoxType,
-                                                std::nullopt);
+      return fir::factory::createUnallocatedBox(*builder, loc, lhsBoxType, {});
     hlfir::Entity rhs = Fortran::lower::convertExprToHLFIR(
         loc, *this, assign.rhs, localSymbols, rhsContext);
     // Create pointer descriptor value from the RHS.
@@ -5199,7 +5198,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                               "LEN parameters");
                   lhsRealloc = fir::factory::genReallocIfNeeded(
                       *builder, loc, *lhsMutableBox,
-                      /*shape=*/std::nullopt, lengthParams);
+                      /*shape=*/{}, lengthParams);
                   return lhsRealloc->newValue;
                 }
                 return genExprAddr(assign.lhs, stmtCtx);
@@ -5271,7 +5270,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
               if (lhsIsWholeAllocatable) {
                 assert(lhsRealloc.has_value());
                 fir::factory::finalizeRealloc(*builder, loc, *lhsMutableBox,
-                                              /*lbounds=*/std::nullopt,
+                                              /*lbounds=*/{},
                                               /*takeLboundsIfRealloc=*/false,
                                               *lhsRealloc);
               }
@@ -6059,8 +6058,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::func::FuncOp func = fir::FirOpBuilder::createFunction(
         mlir::UnknownLoc::get(context), getModuleOp(),
         fir::NameUniquer::doGenerated("Sham"),
-        mlir::FunctionType::get(context, std::nullopt, std::nullopt),
-        symbolTable);
+        mlir::FunctionType::get(context, {}, {}), symbolTable);
     func.addEntryBlock();
     CHECK(!builder && "Expected builder to be uninitialized");
     builder = new fir::FirOpBuilder(func, bridge.getKindMap(), symbolTable);
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 6ed15df0de754..071513303da25 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -496,8 +496,7 @@ Fortran::lower::genCallOpAndResult(
     auto *context = builder.getContext();
     if (mlir::isa<fir::BoxProcType>(snd) &&
         mlir::isa<mlir::FunctionType>(fst.getType())) {
-      auto funcTy =
-          mlir::FunctionType::get(context, std::nullopt, std::nullopt);
+      auto funcTy = mlir::FunctionType::get(context, {}, {});
       auto boxProcTy = builder.getBoxProcType(funcTy);
       if (mlir::Value host = argumentHostAssocs(converter, fst)) {
         cast = builder.create<fir::EmboxProcOp>(
@@ -1714,7 +1713,7 @@ void prepareUserCallArguments(
                                     /*nonDeferredParams=*/mlir::ValueRange{},
                                     /*mutableProperties=*/{});
         fir::factory::associateMutableBox(builder, loc, ptrBox, actualExv,
-                                          /*lbounds=*/std::nullopt);
+                                          /*lbounds=*/{});
         caller.placeInput(arg, irBox);
         continue;
       }
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1051d50ce8a9a..1850b67898126 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -374,8 +374,8 @@ static mlir::Value genStructureComponentInit(
                                "allocatable component value that is not NULL");
     } else {
       // Handle NULL() initialization
-      mlir::Value componentValue{fir::factory::createUnallocatedBox(
-          builder, loc, componentTy, std::nullopt)};
+      mlir::Value componentValue{
+          fir::factory::createUnallocatedBox(builder, loc, componentTy, {})};
       componentValue = builder.createConvert(loc, componentTy, componentValue);
 
       return builder.create<fir::InsertValueOp>(
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index f3430bfa3021e..0a1cd67789772 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -596,7 +596,7 @@ absentBoxToUnallocatedBox(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Type boxType = box.getType();
   assert(mlir::isa<fir::BoxType>(boxType) && "argument must be a fir.box");
   mlir::Value emptyBox =
-      fir::factory::createUnallocatedBox(builder, loc, boxType, std::nullopt);
+      fir::factory::createUnallocatedBox(builder, loc, boxType, {});
   auto safeToReadBox =
       builder.create<mlir::arith::SelectOp>(loc, isPresent, box, emptyBox);
   return fir::substBase(exv, safeToReadBox);
@@ -2663,8 +2663,7 @@ class ScalarExprLowering {
                                        /*nonDeferredParams=*/mlir::ValueRange{},
                                        /*mutableProperties=*/{});
           Fortran::lower::associateMutableBox(converter, loc, pointer, *expr,
-                                              /*lbounds=*/std::nullopt,
-                                              stmtCtx);
+                                              /*lbounds=*/{}, stmtCtx);
           caller.placeInput(arg, irBox);
           continue;
         }
@@ -6186,7 +6185,7 @@ class ArrayExprLowering {
   mlir::FunctionType memcpyType() {
     auto ptrTy = mlir::LLVM::LLVMPointerType::get(builder.getContext());
     llvm::SmallVector<mlir::Type> args = {ptrTy, ptrTy, builder.getI64Type()};
-    return mlir::FunctionType::get(builder.getContext(), args, std::nullopt);
+    return mlir::FunctionType::get(builder.getContext(), args, {});
   }
 
   /// Create a call to the LLVM memcpy intrinsic.
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 9689f920840fb..7de433d6a201a 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1945,7 +1945,7 @@ class HlfirBuilder {
           fir::emitFatalError(loc, "pointer component designator could not be "
                                    "lowered to mutable box");
         Fortran::lower::associateMutableBox(converter, loc, *toBox, expr,
-                                            /*lbounds=*/std::nullopt, stmtCtx);
+                                            /*lbounds=*/{}, stmtCtx);
         continue;
       }
 
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 6c4516686f9d0..6cda742874ccf 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -236,9 +236,8 @@ mlir::Value Fortran::lower::genInitialDataTarget(
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   if (Fortran::evaluate::UnwrapExpr<Fortran::evaluate::NullPointer>(
           initialTarget))
-    return fir::factory::createUnallocatedBox(
-        builder, loc, boxType,
-        /*nonDeferredParams=*/std::nullopt);
+    return fir::factory::createUnallocatedBox(builder, loc, boxType,
+                                              /*nonDeferredParams=*/{});
   // Pointer initial data target, and NULL(mold).
   for (const auto &sym : Fortran::evaluate::CollectSymbols(initialTarget)) {
     // Derived type component symbols should not be instantiated as objects
@@ -354,8 +353,8 @@ static mlir::Value genComponentDefaultInit(
       // From a standard point of view, pointer without initialization do not
       // need to be disassociated, but for sanity and simplicity, do it in
       // global constructor since this has no runtime cost.
-      componentValue = fir::factory::createUnallocatedBox(
-          builder, loc, componentTy, std::nullopt);
+      componentValue =
+          fir::factory::createUnallocatedBox(builder, loc, componentTy, {});
     } else if (Fortran::lower::hasDefaultInitialization(component)) {
       // Component type has default initialization.
       componentValue = genDefaultInitializerValue(converter, loc, component,
@@ -554,7 +553,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
       createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &b) {
         mlir::Value box = fir::factory::createUnallocatedBox(
             b, loc, symTy,
-            /*nonDeferredParams=*/std::nullopt,
+            /*nonDeferredParams=*/{},
             /*typeSourceBox=*/{}, getAllocatorIdxFromDataAttr(dataAttr));
         b.create<fir::HasValueOp>(loc, box);
       });
diff --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp
index 6a44be65a6cde..95ea74b791b47 100644
--- a/flang/lib/Lower/HostAssociations.cpp
+++ b/flang/lib/Lower/HostAssociations.cpp
@@ -410,15 +410,15 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
           .genThen([&]() {
             fir::factory::associateMutableBox(builder, loc, boxInTuple,
                                               args.hostValue,
-                                              /*lbounds=*/std::nullopt);
+                                              /*lbounds=*/{});
           })
           .genElse([&]() {
             fir::factory::disassociateMutableBox(builder, loc, boxInTuple);
           })
           .end();
     } else {
-      fir::factory::associateMutableBox(
-          builder, loc, boxInTuple, args.hostValue, /*lbounds=*/std::nullopt);
+      fir::factory::associateMutableBox(builder, loc, boxInTuple,
+                                        args.hostValue, /*lbounds=*/{});
     }
   }
 
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 69d72d9d63b68..53bf61922392d 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -528,7 +528,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
         descAddr = builder.createTemporary(loc, boxType);
         fir::MutableBoxValue box = fir::MutableBoxValue(descAddr, {}, {});
         fir::factory::associateMutableBox(builder, loc, box, exv,
-                                          /*lbounds=*/std::nullopt);
+                                          /*lbounds=*/{});
       }
       descAddr = builder.createConvert(loc, descRefTy, descAddr);
       list = builder.create<fir::InsertValueOp>(loc, listTy, list, descAddr,
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index e59a6bf2bf224..fb6f0dbf719fb 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -1403,7 +1403,7 @@ hlfir::Entity hlfir::createStackTempFromMold(mlir::Location loc,
         builder.createTemporary(loc, sequenceType, tmpName, extents, lenParams);
   } else {
     alloc = builder.createTemporary(loc, mold.getFortranElementType(), tmpName,
-                                    /*shape=*/std::nullopt, lenParams);
+                                    /*shape=*/{}, lenParams);
   }
   auto declareOp =
       builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, shape, lenParams,
diff --git a/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp b/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp
index 64d70d70829fb..3fb7fab099965 100644
--- a/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp
@@ -31,8 +31,7 @@ mlir::func::FuncOp fir::factory::getRealloc(fir::FirOpBuilder &builder) {
 mlir::func::FuncOp
 fir::factory::getLlvmGetRounding(fir::FirOpBuilder &builder) {
   auto int32Ty = builder.getIntegerType(32);
-  auto funcTy =
-      mlir::FunctionType::get(builder.getContext(), std::nullopt, {int32Ty});
+  auto funcTy = mlir::FunctionType::get(builder.getContext(), {}, {int32Ty});
   return builder.createFunction(builder.getUnknownLoc(), "llvm.get.rounding",
                                 funcTy);
 }
@@ -40,8 +39,7 @@ fir::factory::getLlvmGetRounding(fir::FirOpBuilder &builder) {
 mlir::func::FuncOp
 fir::factory::getLlvmSetRounding(fir::FirOpBuilder &builder) {
   auto int32Ty = builder.getIntegerType(32);
-  auto funcTy =
-      mlir::FunctionType::get(builder.getContext(), {int32Ty}, std::nullopt);
+  auto funcTy = mlir::FunctionType::get(builder.getContext(), {int32Ty}, {});
   return builder.createFunction(builder.getUnknownLoc(), "llvm.set.rounding",
                                 funcTy);
 }
@@ -49,8 +47,8 @@ fir::factory::getLlvmSetRounding(fir::FirOpBuilder &builder) {
 mlir::func::FuncOp
 fir::factory::getLlvmInitTrampoline(fir::FirOpBuilder &builder) {
   auto ptrTy = builder.getRefType(builder.getIntegerType(8));
-  auto funcTy = mlir::FunctionType::get(builder.getContext(),
-                                        {ptrTy, ptrTy, ptrTy}, std::nullopt);
+  auto funcTy =
+      mlir::FunctionType::get(builder.getContext(), {ptrTy, ptrTy, ptrTy}, {});
   return builder.createFunction(builder.getUnknownLoc(), "llvm.init.trampoline",
                                 funcTy);
 }
@@ -90,8 +88,7 @@ mlir::func::FuncOp fir::factory::getFeenableexcept(fir::FirOpBuilder &builder) {
 
 mlir::func::FuncOp fir::factory::getFegetexcept(fir::FirOpBuilder &builder) {
   auto int32Ty = builder.getIntegerType(32);
-  auto funcTy =
-      mlir::FunctionType::get(builder.getContext(), std::nullopt, {int32Ty});
+  auto funcTy = mlir::FunctionType::get(builder.getContext(), {}, {int32Ty});
   return builder.createFunction(builder.getUnknownLoc(), "fegetexcept", funcTy);
 }
 
diff --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp
index d944a4c98473e..93abedc43936d 100644
--- a/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -521,23 +521,23 @@ void fir::factory::associateMutableBox(fir::FirOpBuilder &builder,
         mlir::Value sourceBox;
         if (auto *polyBox = source.getBoxOf<fir::PolymorphicValue>())
           sourceBox = polyBox->getSourceBox();
-        writer.updateMutableBox(p.getAddr(), /*lbounds=*/std::nullopt,
-                                /*extents=*/std::nullopt,
-                                /*lengths=*/std::nullopt, sourceBox);
+        writer.updateMutableBox(p.getAddr(), /*lbounds=*/{},
+                                /*extents=*/{},
+                                /*lengths=*/{}, sourceBox);
       },
       [&](const fir::UnboxedValue &addr) {
-        writer.updateMutableBox(addr, /*lbounds=*/std::nullopt,
-                                /*extents=*/std::nullopt,
-                                /*lengths=*/std::nullopt);
+        writer.updateMutableBox(addr, /*lbounds=*/{},
+                                /*extents=*/{},
+                                /*lengths=*/{});
       },
       [&](const fir::CharBoxValue &ch) {
-        writer.updateMutableBox(ch.getAddr(), /*lbounds=*/std::nullopt,
-                                /*extents=*/std::nullopt, {ch.getLen()});
+        writer.updateMutableBox(ch.getAddr(), /*lbounds=*/{},
+                                /*extents=*/{}, {ch.getLen()});
       },
       [&](const fir::ArrayBoxValue &arr) {
         writer.updateMutableBox(arr.getAddr(),
                                 lbounds.empty() ? arr.getLBounds() : lbounds,
-                                arr.getExtents(), /*lengths=*/std::nullopt);
+                                arr.getExtents(), /*lengths=*/{});
       },
       [&](const fir::CharArrayBoxValue &arr) {
         writer.updateMutableBox(arr.getAddr(),
@@ -634,11 +634,11 @@ void fir::factory::associateMutableBoxWithRemap(
   source.match(
       [&](const fir::PolymorphicValue &p) {
         writer.updateMutableBox(cast(p.getAddr()), lbounds, extents,
-                                /*lengths=*/std::nullopt);
+                                /*lengths=*/{});
       },
       [&](const fir::UnboxedValue &addr) {
         writer.updateMutableBox(cast(addr), lbounds, extents,
-                                /*lengths=*/std::nullopt);
+                                /*lengths=*/{});
       },
       [&](const fir::CharBoxValue &ch) {
         writer.updateMutableBox(cast(ch.getAddr()), lbounds, extents,
@@ -646,7 +646,7 @@ void fir::factory::associateMutableBoxWithRemap(
       },
       [&](const fir::ArrayBoxValue &arr) {
         writer.updateMutableBox(cast(arr.getAddr()), lbounds, extents,
-                                /*lengths=*/std::nullopt);
+                                /*lengths=*/{});
       },
       [&](const fir::CharArrayBoxValue &arr) {
         writer.updateMutableBox(cast(arr.getAddr()), lbounds, extents,
@@ -755,8 +755,8 @@ static mlir::Value allocateAndInitNewStorage(fir::FirOpBuilder &builder,
     // there is no way to know here if a derived type needs it or not. But the
     // information is available at compile time and could be reflected here
     // somehow.
-    mlir::Value irBox = createNewFirBox(builder, loc, box, newStorage,
-                                        std::nullopt, extents, lengths);
+    mlir::Value irBox =
+        createNewFirBox(builder, loc, box, newStorage, {}, extents, lengths);
     fir::runtime::genDerivedTypeInitialize(builder, loc, irBox);
   }
   return newStorage;
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index b6bf2753b80ce..cf20d84cbbcdb 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -4448,7 +4448,7 @@ llvm::LogicalResult fir::UnboxProcOp::verify() {
 
 void fir::IfOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
                       mlir::Value cond, bool withElseRegion) {
-  build(builder, result, std::nullopt, cond, withElseRegion);
+  build(builder, result, {}, cond, withElseRegion);
 }
 
 void fir::IfOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 58f2b57712974..00ca6731c035b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -296,7 +296,7 @@ struct SetLengthOpConversion
     llvm::StringRef tmpName{".tmp"};
     llvm::SmallVector<mlir::Value, 1> lenParams{adaptor.getLength()};
     auto alloca = builder.createTemporary(loc, charType, tmpName,
-                                          /*shape=*/std::nullopt, lenParams);
+                                          /*shape=*/{}, lenParams);
     auto declareOp = builder.create<hlfir::DeclareOp>(
         loc, alloca, tmpName, /*shape=*/mlir::Value{}, lenParams,
         /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
diff --git a/flang/unittests/Optimizer/Builder/CharacterTest.cpp b/flang/unittests/Optimizer/Builder/CharacterTest.cpp
index 6d912b81d9541..d8d2da40ba9a6 100644
--- a/flang/unittests/Optimizer/Builder/CharacterTest.cpp
+++ b/flang/unittests/Optimizer/Builder/CharacterTest.cpp
@@ -29,7 +29,7 @@ struct CharacterTest : public testing::Test {
     moduleOp = builder.create<mlir::ModuleOp>(loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
     mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
-        loc, "func1", builder.getFunctionType(std::nullopt, std::nullopt));
+        loc, "func1", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
 
diff --git a/flang/unittests/Optimizer/Builder/ComplexTest.cpp b/flang/unittests/Optimizer/Builder/ComplexTest.cpp
index 689af4642b0b6..d5f00c9b61108 100644
--- a/flang/unittests/Optimizer/Builder/ComplexTest.cpp
+++ b/flang/unittests/Optimizer/Builder/ComplexTest.cpp
@@ -25,7 +25,7 @@ struct ComplexTest : public testing::Test {
     moduleOp = builder.create<mlir::ModuleOp>(loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
     mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
-        loc, "func1", builder.getFunctionType(std::nullopt, std::nullopt));
+        loc, "func1", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
 
diff --git a/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp b/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
index 3e2af24c47b96..e4c21f6b65a36 100644
--- a/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
+++ b/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
@@ -29,7 +29,7 @@ struct FIRBuilderTest : public testing::Test {
     moduleOp = builder.create<mlir::ModuleOp>(loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
     mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
-        loc, "func1", builder.getFunctionType(std::nullopt, std::nullopt));
+        loc, "func1", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
 
@@ -176,8 +176,7 @@ TEST_F(FIRBuilderTest, getNamedFunction) {
   auto func2 = builder.getNamedFunction("func2");
   EXPECT_EQ(nullptr, func2);
   auto loc = builder.getUnknownLoc();
-  func2 = builder.createFunction(
-      loc, "func2", builder.getFunctionType(std::nullopt, std::nullopt));
+  func2 = builder.createFunction(loc, "func2", builder.getFunctionType({}, {}));
   auto func2query = builder.getNamedFunction("func2");
   EXPECT_EQ(func2, func2query);
 }
diff --git a/flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp b/flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp
index 29700d2d3dbff..a0785198b078d 100644
--- a/flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp
+++ b/flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp
@@ -28,7 +28,7 @@ struct HLFIRToolsTest : public testing::Test {
     moduleOp = builder.create<mlir::ModuleOp>(loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
     mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
-        loc, "func1", builder.getFunctionType(std::nullopt, std::nullopt));
+        loc, "func1", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
 
diff --git a/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h b/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
index 40abf567400b3..4ecec92f42dc2 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
+++ b/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
@@ -26,9 +26,8 @@ struct RuntimeCallTest : public testing::Test {
     // Set the insertion point in the function entry block.
     moduleOp = builder.create<mlir::ModuleOp>(loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
-    mlir::func::FuncOp func =
-        builder.create<mlir::func::FuncOp>(loc, "runtime_unit_tests_func",
-            builder.getFunctionType(std::nullopt, std::nullopt));
+    mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
+        loc, "runtime_unit_tests_func", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
 
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index 30c23b63b4d56..98270adaa7c73 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -21,9 +21,8 @@ struct FortranVariableTest : public testing::Test {
     // Set the insertion point in the function entry block.
     moduleOp = builder->create<mlir::ModuleOp>(loc);
     builder->setInsertionPointToStart(moduleOp->getBody());
-    mlir::func::FuncOp func =
-        builder->create<mlir::func::FuncOp>(loc, "fortran_variable_tests",
-            builder->getFunctionType(std::nullopt, std::nullopt));
+    mlir::func::FuncOp func = builder->create<mlir::func::FuncOp>(
+        loc, "fortran_variable_tests", builder->getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder->setInsertionPointToStart(entryBlock);
   }


From flang-commits at lists.llvm.org  Thu Jul 17 08:41:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 08:41:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Migrate away from
	ArrayRef(std::nullopt_t) (PR #149337)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149337@github.com>
Message-ID: <687919ba.630a0220.347c94.1744@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Kazu Hirata (kazutakahirata)

<details>
<summary>Changes</summary>

ArrayRef(std::nullopt_t) has been deprecated.  This patch replaces
std::nullopt with {}.

A subsequence patch will address those places where we need to replace
std::nullopt with mlir::TypeRange{} or mlir::ValueRange{}.


---

Patch is 25.62 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/149337.diff


19 Files Affected:

- (modified) flang/lib/Lower/Bridge.cpp (+11-13) 
- (modified) flang/lib/Lower/ConvertCall.cpp (+2-3) 
- (modified) flang/lib/Lower/ConvertConstant.cpp (+2-2) 
- (modified) flang/lib/Lower/ConvertExpr.cpp (+3-4) 
- (modified) flang/lib/Lower/ConvertExprToHLFIR.cpp (+1-1) 
- (modified) flang/lib/Lower/ConvertVariable.cpp (+5-6) 
- (modified) flang/lib/Lower/HostAssociations.cpp (+3-3) 
- (modified) flang/lib/Lower/IO.cpp (+1-1) 
- (modified) flang/lib/Optimizer/Builder/HLFIRTools.cpp (+1-1) 
- (modified) flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp (+5-8) 
- (modified) flang/lib/Optimizer/Builder/MutableBox.cpp (+14-14) 
- (modified) flang/lib/Optimizer/Dialect/FIROps.cpp (+1-1) 
- (modified) flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp (+1-1) 
- (modified) flang/unittests/Optimizer/Builder/CharacterTest.cpp (+1-1) 
- (modified) flang/unittests/Optimizer/Builder/ComplexTest.cpp (+1-1) 
- (modified) flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp (+2-3) 
- (modified) flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp (+1-1) 
- (modified) flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h (+2-3) 
- (modified) flang/unittests/Optimizer/FortranVariableTest.cpp (+2-3) 


``````````diff
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 33c1f1e7a3c3a..4241d12601242 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -810,11 +810,11 @@ class FirConverter : public Fortran::lower::AbstractConverter {
             fir::ExtendedValue read = fir::factory::genMutableBoxRead(
                 *builder, loc, box, /*mayBePolymorphic=*/false);
             if (auto read_arr_box = read.getBoxOf<fir::ArrayBoxValue>()) {
-              fir::factory::genInlinedAllocation(
-                  *builder, loc, *new_box, read_arr_box->getLBounds(),
-                  read_arr_box->getExtents(),
-                  /*lenParams=*/std::nullopt, name,
-                  /*mustBeHeap=*/true);
+              fir::factory::genInlinedAllocation(*builder, loc, *new_box,
+                                                 read_arr_box->getLBounds(),
+                                                 read_arr_box->getExtents(),
+                                                 /*lenParams=*/{}, name,
+                                                 /*mustBeHeap=*/true);
             } else if (auto read_char_arr_box =
                            read.getBoxOf<fir::CharArrayBoxValue>()) {
               fir::factory::genInlinedAllocation(
@@ -825,8 +825,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
             } else if (auto read_char_box =
                            read.getBoxOf<fir::CharBoxValue>()) {
               fir::factory::genInlinedAllocation(*builder, loc, *new_box,
-                                                 /*lbounds=*/std::nullopt,
-                                                 /*extents=*/std::nullopt,
+                                                 /*lbounds=*/{},
+                                                 /*extents=*/{},
                                                  read_char_box->getLen(), name,
                                                  /*mustBeHeap=*/true);
             } else {
@@ -4590,8 +4590,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // the static type of the LHS.
     if (Fortran::evaluate::UnwrapExpr<Fortran::evaluate::NullPointer>(
             assign.rhs))
-      return fir::factory::createUnallocatedBox(*builder, loc, lhsBoxType,
-                                                std::nullopt);
+      return fir::factory::createUnallocatedBox(*builder, loc, lhsBoxType, {});
     hlfir::Entity rhs = Fortran::lower::convertExprToHLFIR(
         loc, *this, assign.rhs, localSymbols, rhsContext);
     // Create pointer descriptor value from the RHS.
@@ -5199,7 +5198,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                               "LEN parameters");
                   lhsRealloc = fir::factory::genReallocIfNeeded(
                       *builder, loc, *lhsMutableBox,
-                      /*shape=*/std::nullopt, lengthParams);
+                      /*shape=*/{}, lengthParams);
                   return lhsRealloc->newValue;
                 }
                 return genExprAddr(assign.lhs, stmtCtx);
@@ -5271,7 +5270,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
               if (lhsIsWholeAllocatable) {
                 assert(lhsRealloc.has_value());
                 fir::factory::finalizeRealloc(*builder, loc, *lhsMutableBox,
-                                              /*lbounds=*/std::nullopt,
+                                              /*lbounds=*/{},
                                               /*takeLboundsIfRealloc=*/false,
                                               *lhsRealloc);
               }
@@ -6059,8 +6058,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::func::FuncOp func = fir::FirOpBuilder::createFunction(
         mlir::UnknownLoc::get(context), getModuleOp(),
         fir::NameUniquer::doGenerated("Sham"),
-        mlir::FunctionType::get(context, std::nullopt, std::nullopt),
-        symbolTable);
+        mlir::FunctionType::get(context, {}, {}), symbolTable);
     func.addEntryBlock();
     CHECK(!builder && "Expected builder to be uninitialized");
     builder = new fir::FirOpBuilder(func, bridge.getKindMap(), symbolTable);
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 6ed15df0de754..071513303da25 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -496,8 +496,7 @@ Fortran::lower::genCallOpAndResult(
     auto *context = builder.getContext();
     if (mlir::isa<fir::BoxProcType>(snd) &&
         mlir::isa<mlir::FunctionType>(fst.getType())) {
-      auto funcTy =
-          mlir::FunctionType::get(context, std::nullopt, std::nullopt);
+      auto funcTy = mlir::FunctionType::get(context, {}, {});
       auto boxProcTy = builder.getBoxProcType(funcTy);
       if (mlir::Value host = argumentHostAssocs(converter, fst)) {
         cast = builder.create<fir::EmboxProcOp>(
@@ -1714,7 +1713,7 @@ void prepareUserCallArguments(
                                     /*nonDeferredParams=*/mlir::ValueRange{},
                                     /*mutableProperties=*/{});
         fir::factory::associateMutableBox(builder, loc, ptrBox, actualExv,
-                                          /*lbounds=*/std::nullopt);
+                                          /*lbounds=*/{});
         caller.placeInput(arg, irBox);
         continue;
       }
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1051d50ce8a9a..1850b67898126 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -374,8 +374,8 @@ static mlir::Value genStructureComponentInit(
                                "allocatable component value that is not NULL");
     } else {
       // Handle NULL() initialization
-      mlir::Value componentValue{fir::factory::createUnallocatedBox(
-          builder, loc, componentTy, std::nullopt)};
+      mlir::Value componentValue{
+          fir::factory::createUnallocatedBox(builder, loc, componentTy, {})};
       componentValue = builder.createConvert(loc, componentTy, componentValue);
 
       return builder.create<fir::InsertValueOp>(
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index f3430bfa3021e..0a1cd67789772 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -596,7 +596,7 @@ absentBoxToUnallocatedBox(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Type boxType = box.getType();
   assert(mlir::isa<fir::BoxType>(boxType) && "argument must be a fir.box");
   mlir::Value emptyBox =
-      fir::factory::createUnallocatedBox(builder, loc, boxType, std::nullopt);
+      fir::factory::createUnallocatedBox(builder, loc, boxType, {});
   auto safeToReadBox =
       builder.create<mlir::arith::SelectOp>(loc, isPresent, box, emptyBox);
   return fir::substBase(exv, safeToReadBox);
@@ -2663,8 +2663,7 @@ class ScalarExprLowering {
                                        /*nonDeferredParams=*/mlir::ValueRange{},
                                        /*mutableProperties=*/{});
           Fortran::lower::associateMutableBox(converter, loc, pointer, *expr,
-                                              /*lbounds=*/std::nullopt,
-                                              stmtCtx);
+                                              /*lbounds=*/{}, stmtCtx);
           caller.placeInput(arg, irBox);
           continue;
         }
@@ -6186,7 +6185,7 @@ class ArrayExprLowering {
   mlir::FunctionType memcpyType() {
     auto ptrTy = mlir::LLVM::LLVMPointerType::get(builder.getContext());
     llvm::SmallVector<mlir::Type> args = {ptrTy, ptrTy, builder.getI64Type()};
-    return mlir::FunctionType::get(builder.getContext(), args, std::nullopt);
+    return mlir::FunctionType::get(builder.getContext(), args, {});
   }
 
   /// Create a call to the LLVM memcpy intrinsic.
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 9689f920840fb..7de433d6a201a 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1945,7 +1945,7 @@ class HlfirBuilder {
           fir::emitFatalError(loc, "pointer component designator could not be "
                                    "lowered to mutable box");
         Fortran::lower::associateMutableBox(converter, loc, *toBox, expr,
-                                            /*lbounds=*/std::nullopt, stmtCtx);
+                                            /*lbounds=*/{}, stmtCtx);
         continue;
       }
 
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 6c4516686f9d0..6cda742874ccf 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -236,9 +236,8 @@ mlir::Value Fortran::lower::genInitialDataTarget(
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   if (Fortran::evaluate::UnwrapExpr<Fortran::evaluate::NullPointer>(
           initialTarget))
-    return fir::factory::createUnallocatedBox(
-        builder, loc, boxType,
-        /*nonDeferredParams=*/std::nullopt);
+    return fir::factory::createUnallocatedBox(builder, loc, boxType,
+                                              /*nonDeferredParams=*/{});
   // Pointer initial data target, and NULL(mold).
   for (const auto &sym : Fortran::evaluate::CollectSymbols(initialTarget)) {
     // Derived type component symbols should not be instantiated as objects
@@ -354,8 +353,8 @@ static mlir::Value genComponentDefaultInit(
       // From a standard point of view, pointer without initialization do not
       // need to be disassociated, but for sanity and simplicity, do it in
       // global constructor since this has no runtime cost.
-      componentValue = fir::factory::createUnallocatedBox(
-          builder, loc, componentTy, std::nullopt);
+      componentValue =
+          fir::factory::createUnallocatedBox(builder, loc, componentTy, {});
     } else if (Fortran::lower::hasDefaultInitialization(component)) {
       // Component type has default initialization.
       componentValue = genDefaultInitializerValue(converter, loc, component,
@@ -554,7 +553,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
       createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &b) {
         mlir::Value box = fir::factory::createUnallocatedBox(
             b, loc, symTy,
-            /*nonDeferredParams=*/std::nullopt,
+            /*nonDeferredParams=*/{},
             /*typeSourceBox=*/{}, getAllocatorIdxFromDataAttr(dataAttr));
         b.create<fir::HasValueOp>(loc, box);
       });
diff --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp
index 6a44be65a6cde..95ea74b791b47 100644
--- a/flang/lib/Lower/HostAssociations.cpp
+++ b/flang/lib/Lower/HostAssociations.cpp
@@ -410,15 +410,15 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
           .genThen([&]() {
             fir::factory::associateMutableBox(builder, loc, boxInTuple,
                                               args.hostValue,
-                                              /*lbounds=*/std::nullopt);
+                                              /*lbounds=*/{});
           })
           .genElse([&]() {
             fir::factory::disassociateMutableBox(builder, loc, boxInTuple);
           })
           .end();
     } else {
-      fir::factory::associateMutableBox(
-          builder, loc, boxInTuple, args.hostValue, /*lbounds=*/std::nullopt);
+      fir::factory::associateMutableBox(builder, loc, boxInTuple,
+                                        args.hostValue, /*lbounds=*/{});
     }
   }
 
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 69d72d9d63b68..53bf61922392d 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -528,7 +528,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
         descAddr = builder.createTemporary(loc, boxType);
         fir::MutableBoxValue box = fir::MutableBoxValue(descAddr, {}, {});
         fir::factory::associateMutableBox(builder, loc, box, exv,
-                                          /*lbounds=*/std::nullopt);
+                                          /*lbounds=*/{});
       }
       descAddr = builder.createConvert(loc, descRefTy, descAddr);
       list = builder.create<fir::InsertValueOp>(loc, listTy, list, descAddr,
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index e59a6bf2bf224..fb6f0dbf719fb 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -1403,7 +1403,7 @@ hlfir::Entity hlfir::createStackTempFromMold(mlir::Location loc,
         builder.createTemporary(loc, sequenceType, tmpName, extents, lenParams);
   } else {
     alloc = builder.createTemporary(loc, mold.getFortranElementType(), tmpName,
-                                    /*shape=*/std::nullopt, lenParams);
+                                    /*shape=*/{}, lenParams);
   }
   auto declareOp =
       builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, shape, lenParams,
diff --git a/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp b/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp
index 64d70d70829fb..3fb7fab099965 100644
--- a/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp
@@ -31,8 +31,7 @@ mlir::func::FuncOp fir::factory::getRealloc(fir::FirOpBuilder &builder) {
 mlir::func::FuncOp
 fir::factory::getLlvmGetRounding(fir::FirOpBuilder &builder) {
   auto int32Ty = builder.getIntegerType(32);
-  auto funcTy =
-      mlir::FunctionType::get(builder.getContext(), std::nullopt, {int32Ty});
+  auto funcTy = mlir::FunctionType::get(builder.getContext(), {}, {int32Ty});
   return builder.createFunction(builder.getUnknownLoc(), "llvm.get.rounding",
                                 funcTy);
 }
@@ -40,8 +39,7 @@ fir::factory::getLlvmGetRounding(fir::FirOpBuilder &builder) {
 mlir::func::FuncOp
 fir::factory::getLlvmSetRounding(fir::FirOpBuilder &builder) {
   auto int32Ty = builder.getIntegerType(32);
-  auto funcTy =
-      mlir::FunctionType::get(builder.getContext(), {int32Ty}, std::nullopt);
+  auto funcTy = mlir::FunctionType::get(builder.getContext(), {int32Ty}, {});
   return builder.createFunction(builder.getUnknownLoc(), "llvm.set.rounding",
                                 funcTy);
 }
@@ -49,8 +47,8 @@ fir::factory::getLlvmSetRounding(fir::FirOpBuilder &builder) {
 mlir::func::FuncOp
 fir::factory::getLlvmInitTrampoline(fir::FirOpBuilder &builder) {
   auto ptrTy = builder.getRefType(builder.getIntegerType(8));
-  auto funcTy = mlir::FunctionType::get(builder.getContext(),
-                                        {ptrTy, ptrTy, ptrTy}, std::nullopt);
+  auto funcTy =
+      mlir::FunctionType::get(builder.getContext(), {ptrTy, ptrTy, ptrTy}, {});
   return builder.createFunction(builder.getUnknownLoc(), "llvm.init.trampoline",
                                 funcTy);
 }
@@ -90,8 +88,7 @@ mlir::func::FuncOp fir::factory::getFeenableexcept(fir::FirOpBuilder &builder) {
 
 mlir::func::FuncOp fir::factory::getFegetexcept(fir::FirOpBuilder &builder) {
   auto int32Ty = builder.getIntegerType(32);
-  auto funcTy =
-      mlir::FunctionType::get(builder.getContext(), std::nullopt, {int32Ty});
+  auto funcTy = mlir::FunctionType::get(builder.getContext(), {}, {int32Ty});
   return builder.createFunction(builder.getUnknownLoc(), "fegetexcept", funcTy);
 }
 
diff --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp
index d944a4c98473e..93abedc43936d 100644
--- a/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -521,23 +521,23 @@ void fir::factory::associateMutableBox(fir::FirOpBuilder &builder,
         mlir::Value sourceBox;
         if (auto *polyBox = source.getBoxOf<fir::PolymorphicValue>())
           sourceBox = polyBox->getSourceBox();
-        writer.updateMutableBox(p.getAddr(), /*lbounds=*/std::nullopt,
-                                /*extents=*/std::nullopt,
-                                /*lengths=*/std::nullopt, sourceBox);
+        writer.updateMutableBox(p.getAddr(), /*lbounds=*/{},
+                                /*extents=*/{},
+                                /*lengths=*/{}, sourceBox);
       },
       [&](const fir::UnboxedValue &addr) {
-        writer.updateMutableBox(addr, /*lbounds=*/std::nullopt,
-                                /*extents=*/std::nullopt,
-                                /*lengths=*/std::nullopt);
+        writer.updateMutableBox(addr, /*lbounds=*/{},
+                                /*extents=*/{},
+                                /*lengths=*/{});
       },
       [&](const fir::CharBoxValue &ch) {
-        writer.updateMutableBox(ch.getAddr(), /*lbounds=*/std::nullopt,
-                                /*extents=*/std::nullopt, {ch.getLen()});
+        writer.updateMutableBox(ch.getAddr(), /*lbounds=*/{},
+                                /*extents=*/{}, {ch.getLen()});
       },
       [&](const fir::ArrayBoxValue &arr) {
         writer.updateMutableBox(arr.getAddr(),
                                 lbounds.empty() ? arr.getLBounds() : lbounds,
-                                arr.getExtents(), /*lengths=*/std::nullopt);
+                                arr.getExtents(), /*lengths=*/{});
       },
       [&](const fir::CharArrayBoxValue &arr) {
         writer.updateMutableBox(arr.getAddr(),
@@ -634,11 +634,11 @@ void fir::factory::associateMutableBoxWithRemap(
   source.match(
       [&](const fir::PolymorphicValue &p) {
         writer.updateMutableBox(cast(p.getAddr()), lbounds, extents,
-                                /*lengths=*/std::nullopt);
+                                /*lengths=*/{});
       },
       [&](const fir::UnboxedValue &addr) {
         writer.updateMutableBox(cast(addr), lbounds, extents,
-                                /*lengths=*/std::nullopt);
+                                /*lengths=*/{});
       },
       [&](const fir::CharBoxValue &ch) {
         writer.updateMutableBox(cast(ch.getAddr()), lbounds, extents,
@@ -646,7 +646,7 @@ void fir::factory::associateMutableBoxWithRemap(
       },
       [&](const fir::ArrayBoxValue &arr) {
         writer.updateMutableBox(cast(arr.getAddr()), lbounds, extents,
-                                /*lengths=*/std::nullopt);
+                                /*lengths=*/{});
       },
       [&](const fir::CharArrayBoxValue &arr) {
         writer.updateMutableBox(cast(arr.getAddr()), lbounds, extents,
@@ -755,8 +755,8 @@ static mlir::Value allocateAndInitNewStorage(fir::FirOpBuilder &builder,
     // there is no way to know here if a derived type needs it or not. But the
     // information is available at compile time and could be reflected here
     // somehow.
-    mlir::Value irBox = createNewFirBox(builder, loc, box, newStorage,
-                                        std::nullopt, extents, lengths);
+    mlir::Value irBox =
+        createNewFirBox(builder, loc, box, newStorage, {}, extents, lengths);
     fir::runtime::genDerivedTypeInitialize(builder, loc, irBox);
   }
   return newStorage;
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index b6bf2753b80ce..cf20d84cbbcdb 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -4448,7 +4448,7 @@ llvm::LogicalResult fir::UnboxProcOp::verify() {
 
 void fir::IfOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
                       mlir::Value cond, bool withElseRegion) {
-  build(builder, result, std::nullopt, cond, withElseRegion);
+  build(builder, result, {}, cond, withElseRegion);
 }
 
 void fir::IfOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 58f2b57712974..00ca6731c035b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -296,7 +296,7 @@ struct SetLengthOpConversion
     llvm::StringRef tmpName{".tmp"};
     llvm::SmallVector<mlir::Value, 1> lenParams{adaptor.getLength()};
     auto alloca = builder.createTemporary(loc, c...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/149337

From flang-commits at lists.llvm.org  Thu Jul 17 08:50:48 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Thu, 17 Jul 2025 08:50:48 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `COSPI` (PR #149343)
Message-ID: <llvm.org/llvm/llvm-project/pull/149343@github.com>


https://github.com/c8ef created https://github.com/llvm/llvm-project/pull/149343

This feature is added in the Fortran 2023 standard.

>From 6b2da588c8a286881fbcc8053f62dba749192445 Mon Sep 17 00:00:00 2001
From: c8ef <c8ef at outlook.com>
Date: Thu, 17 Jul 2025 15:49:22 +0000
Subject: [PATCH] implement cospi for flang

---
 .../flang/Optimizer/Builder/IntrinsicCall.h   |  1 +
 flang/lib/Evaluate/intrinsics.cpp             |  1 +
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 16 ++++++++++++++
 flang/test/Lower/Intrinsics/cospi.f90         | 22 +++++++++++++++++++
 4 files changed, 40 insertions(+)
 create mode 100644 flang/test/Lower/Intrinsics/cospi.f90

diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 01801dbdaffca..acdba7c49e6b3 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -245,6 +245,7 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genCPtrCompare(mlir::Type,
                                     llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genCosd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genCospi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   void genDateAndTime(llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genDim(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genDotProduct(mlir::Type,
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 4773e136c41cb..9957010684d48 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -428,6 +428,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"conjg", {{"z", SameComplex}}, SameComplex},
     {"cos", {{"x", SameFloating}}, SameFloating},
     {"cosd", {{"x", SameFloating}}, SameFloating},
+    {"cospi", {{"x", SameFloating}}, SameFloating},
     {"cosh", {{"x", SameFloating}}, SameFloating},
     {"coshape", {{"coarray", AnyData, Rank::coarray}, SizeDefaultKIND}, KINDInt,
         Rank::vector, IntrinsicClass::inquiryFunction},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 7aa5602d2bc84..d77a656158a37 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -396,6 +396,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"command_argument_count", &I::genCommandArgumentCount},
     {"conjg", &I::genConjg},
     {"cosd", &I::genCosd},
+    {"cospi", &I::genCospi},
     {"count",
      &I::genCount,
      {{{"mask", asAddr}, {"dim", asValue}, {"kind", asValue}}},
@@ -3623,6 +3624,21 @@ mlir::Value IntrinsicLibrary::genCosd(mlir::Type resultType,
   return getRuntimeCallGenerator("cos", ftype)(builder, loc, {arg});
 }
 
+// COSPI
+mlir::Value IntrinsicLibrary::genCospi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  return getRuntimeCallGenerator("cos", ftype)(builder, loc, {arg});
+}
+
 // COUNT
 fir::ExtendedValue
 IntrinsicLibrary::genCount(mlir::Type resultType,
diff --git a/flang/test/Lower/Intrinsics/cospi.f90 b/flang/test/Lower/Intrinsics/cospi.f90
new file mode 100644
index 0000000000000..894002566141b
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/cospi.f90
@@ -0,0 +1,22 @@
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = cospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f32
+! CHECK: %[[cos:.*]] = math.cos %[[mul]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = cospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[dfactor]] fastmath<contract> : f64
+! CHECK: %[[cos:.*]] = math.cos %[[mul]] fastmath<contract> : f64


From flang-commits at lists.llvm.org  Thu Jul 17 08:51:17 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 08:51:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `COSPI` (PR #149343)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149343@github.com>
Message-ID: <68791bf5.050a0220.16280b.4580@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Connector Switch (c8ef)

<details>
<summary>Changes</summary>

This feature is added in the Fortran 2023 standard.

---
Full diff: https://github.com/llvm/llvm-project/pull/149343.diff


4 Files Affected:

- (modified) flang/include/flang/Optimizer/Builder/IntrinsicCall.h (+1) 
- (modified) flang/lib/Evaluate/intrinsics.cpp (+1) 
- (modified) flang/lib/Optimizer/Builder/IntrinsicCall.cpp (+16) 
- (added) flang/test/Lower/Intrinsics/cospi.f90 (+22) 


``````````diff
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 01801dbdaffca..acdba7c49e6b3 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -245,6 +245,7 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genCPtrCompare(mlir::Type,
                                     llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genCosd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genCospi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   void genDateAndTime(llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genDim(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genDotProduct(mlir::Type,
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 4773e136c41cb..9957010684d48 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -428,6 +428,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"conjg", {{"z", SameComplex}}, SameComplex},
     {"cos", {{"x", SameFloating}}, SameFloating},
     {"cosd", {{"x", SameFloating}}, SameFloating},
+    {"cospi", {{"x", SameFloating}}, SameFloating},
     {"cosh", {{"x", SameFloating}}, SameFloating},
     {"coshape", {{"coarray", AnyData, Rank::coarray}, SizeDefaultKIND}, KINDInt,
         Rank::vector, IntrinsicClass::inquiryFunction},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 7aa5602d2bc84..d77a656158a37 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -396,6 +396,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"command_argument_count", &I::genCommandArgumentCount},
     {"conjg", &I::genConjg},
     {"cosd", &I::genCosd},
+    {"cospi", &I::genCospi},
     {"count",
      &I::genCount,
      {{{"mask", asAddr}, {"dim", asValue}, {"kind", asValue}}},
@@ -3623,6 +3624,21 @@ mlir::Value IntrinsicLibrary::genCosd(mlir::Type resultType,
   return getRuntimeCallGenerator("cos", ftype)(builder, loc, {arg});
 }
 
+// COSPI
+mlir::Value IntrinsicLibrary::genCospi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  return getRuntimeCallGenerator("cos", ftype)(builder, loc, {arg});
+}
+
 // COUNT
 fir::ExtendedValue
 IntrinsicLibrary::genCount(mlir::Type resultType,
diff --git a/flang/test/Lower/Intrinsics/cospi.f90 b/flang/test/Lower/Intrinsics/cospi.f90
new file mode 100644
index 0000000000000..894002566141b
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/cospi.f90
@@ -0,0 +1,22 @@
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = cospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f32
+! CHECK: %[[cos:.*]] = math.cos %[[mul]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = cospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[dfactor]] fastmath<contract> : f64
+! CHECK: %[[cos:.*]] = math.cos %[[mul]] fastmath<contract> : f64

``````````

</details>


https://github.com/llvm/llvm-project/pull/149343

From flang-commits at lists.llvm.org  Thu Jul 17 08:06:50 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 08:06:50 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <6879118a.170a0220.2ea743.c19c@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 1/4] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 2/4] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 3/4] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 4/4] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir


From flang-commits at lists.llvm.org  Thu Jul 17 09:04:48 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 09:04:48 -0700 (PDT)
Subject: [flang-commits] [flang] a7f595e - [flang][acc] Create UseDeviceOp
 for both results of hlfir.declare (#148017)
Message-ID: <68791f20.170a0220.66dfd.ee29@mx.google.com>


Author: nvptm
Date: 2025-07-17T09:04:44-07:00
New Revision: a7f595efd840f7ed2210f2703048fad4d0027fac

URL: https://github.com/llvm/llvm-project/commit/a7f595efd840f7ed2210f2703048fad4d0027fac
DIFF: https://github.com/llvm/llvm-project/commit/a7f595efd840f7ed2210f2703048fad4d0027fac.diff

LOG: [flang][acc] Create UseDeviceOp for both results of hlfir.declare (#148017)

A sample such as 
```
program test
  integer :: N = 100
  real*8 :: b(-1:N)
  !$acc data copy(b)
  !$acc host_data use_device(b)
  call vadd(b)
  !$acc end host_data
  !$acc end data
end

```
is lowered to
```
    %13:2 = hlfir.declare %11(%12) {uniq_name = "_QFEb"} : (!fir.ref<!fir.array<?xf64>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>)
    %14 = acc.copyin var(%13#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
    acc.data dataOperands(%14 : !fir.box<!fir.array<?xf64>>) {
      %15 = acc.use_device var(%13#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
      acc.host_data dataOperands(%15 : !fir.box<!fir.array<?xf64>>) {
        fir.call @_QPvadd(%13#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>) -> ()
        acc.terminator
      }
      acc.terminator
    }
    acc.copyout accVar(%14 : !fir.box<!fir.array<?xf64>>) to var(%13#0 : !fir.box<!fir.array<?xf64>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
```
Note that while the use_device clause is applied to %13#0, the argument
passed to vadd is %13#1. To avoid problems later in lowering, this
change additionally applies the use_device clause to %13#1, so that the
resulting MLIR is
```
   %13:2 = hlfir.declare %11(%12) {uniq_name = "_QFEb"} : (!fir.ref<!fir.array<?xf64>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>)
    %14 = acc.copyin var(%13#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
    acc.data dataOperands(%14 : !fir.box<!fir.array<?xf64>>) {
      %15 = acc.use_device var(%13#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
      %16 = acc.use_device varPtr(%13#1 : !fir.ref<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>> {name = "b"}
      acc.host_data dataOperands(%15, %16 : !fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>) {
        fir.call @_QPvadd(%13#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>) -> ()
        acc.terminator
      }
      acc.terminator
    }
    acc.copyout accVar(%14 : !fir.box<!fir.array<?xf64>>) to var(%13#0 : !fir.box<!fir.array<?xf64>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
  
```

Added: 
    flang/test/Lower/OpenACC/acc-use-device.f90

Modified: 
    flang/lib/Lower/OpenACC.cpp
    flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
    flang/test/Lower/OpenACC/acc-host-data.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 39e4444cde4e3..25682cba5620e 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -708,6 +708,7 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
                          bool setDeclareAttr = false) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   Fortran::evaluate::ExpressionAnalyzer ea{semanticsContext};
+  const bool unwrapBoxAddr = true;
   for (const auto &accObject : objectList.v) {
     llvm::SmallVector<mlir::Value> bounds;
     std::stringstream asFortran;
@@ -735,8 +736,25 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
     Op op = createDataEntryOp<Op>(
         builder, operandLocation, baseAddr, asFortran, bounds, structured,
         implicit, dataClause, baseAddr.getType(), async, asyncDeviceTypes,
-        asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true, info.isPresent);
+        asyncOnlyDeviceTypes, unwrapBoxAddr, info.isPresent);
     dataOperands.push_back(op.getAccVar());
+
+    // For UseDeviceOp, if operand is one of a pair resulting from a
+    // declare operation, create a UseDeviceOp for the other operand as well.
+    if constexpr (std::is_same_v<Op, mlir::acc::UseDeviceOp>) {
+      if (auto declareOp =
+              mlir::dyn_cast<hlfir::DeclareOp>(baseAddr.getDefiningOp())) {
+        mlir::Value otherAddr = declareOp.getResult(1);
+        if (baseAddr != otherAddr) {
+          Op op = createDataEntryOp<Op>(builder, operandLocation, otherAddr,
+                                        asFortran, bounds, structured, implicit,
+                                        dataClause, otherAddr.getType(), async,
+                                        asyncDeviceTypes, asyncOnlyDeviceTypes,
+                                        unwrapBoxAddr, info.isPresent);
+          dataOperands.push_back(op.getAccVar());
+        }
+      }
+    }
   }
 }
 

diff  --git a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
index 164eb32a8f684..2de7cc5761a2b 100644
--- a/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data-unwrap-defaultbounds.f90
@@ -15,15 +15,17 @@ subroutine acc_host_data()
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+ ! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>{{.*}}) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
@@ -33,14 +35,14 @@ subroutine acc_host_data()
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%{{.*}} : index)
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.false.)
     a = 1.0

diff  --git a/flang/test/Lower/OpenACC/acc-host-data.f90 b/flang/test/Lower/OpenACC/acc-host-data.f90
index 871eabd256ca6..4d09b25b983b9 100644
--- a/flang/test/Lower/OpenACC/acc-host-data.f90
+++ b/flang/test/Lower/OpenACC/acc-host-data.f90
@@ -14,34 +14,37 @@ subroutine acc_host_data()
   !$acc host_data use_device(a)
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
   !$acc host_data use_device(a) if_present
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: acc.host_data dataOperands(%[[DA0]], %[[DA1]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 ! CHECK: } attributes {ifPresent}
 
-  !$acc host_data use_device(a) if_present if_present
+  !$acc host_data use_device(a) if_present 
   !$acc end host_data
-! CHECK: acc.host_data dataOperands(%{{.*}} : !fir.ref<!fir.array<10xf32>>) {
+! CHECK: acc.host_data dataOperands(%{{.*}}{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}}) {
 ! CHECK: } attributes {ifPresent}
 
   !$acc host_data use_device(a) if(ifCondition)
   !$acc end host_data
 
-! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA0:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK: %[[DA1:.*]] = acc.use_device varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK: %[[LOAD_IFCOND:.*]] = fir.load %[[DECLIFCOND]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[IFCOND_I1:.*]] = fir.convert %[[LOAD_IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data if(%[[IFCOND_I1]]) dataOperands(%[[DA0]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.true.)
   !$acc end host_data
 
 ! CHECK: %[[DA:.*]] = acc.use_device varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK: acc.host_data dataOperands(%[[DA]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK: acc.host_data dataOperands(%[[DA]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 
   !$acc host_data use_device(a) if(.false.)
     a = 1.0

diff  --git a/flang/test/Lower/OpenACC/acc-use-device.f90 b/flang/test/Lower/OpenACC/acc-use-device.f90
new file mode 100644
index 0000000000000..081a6e317bfc9
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-use-device.f90
@@ -0,0 +1,61 @@
+! This test checks whether the OpenACC use_device clause is applied on both results of hlfir.declare.
+
+! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
+
+! Test for automatic variable appearing in use_device clause.
+subroutine test()
+  integer :: N = 100
+  real*8 :: b(-1:N)
+! CHECK: %[[A0:.*]] = fir.alloca !fir.array<?xf64>, %{{.*}} {bindc_name = "b", uniq_name = "_QFtestEb"}
+! CHECK: %[[A1:.*]] = fir.shape_shift {{.*}} : (index, index) -> !fir.shapeshift<1>
+! CHECK: %[[A:.*]]:2 = hlfir.declare %[[A0]](%[[A1]]) {uniq_name = "_QFtestEb"} : (!fir.ref<!fir.array<?xf64>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>)
+  
+  !$acc data copy(b)
+! CHECK: %[[B:.*]] = acc.copyin var(%[[A]]#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
+! CHECK: acc.data dataOperands(%[[B]] : !fir.box<!fir.array<?xf64>>) {
+
+  !$acc host_data use_device(b)
+  call vadd(b)
+  !$acc end host_data
+! CHECK: %[[C:.*]] = acc.use_device var(%[[A]]#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
+! CHECK: %[[D:.*]] = acc.use_device varPtr(%[[A]]#1 : !fir.ref<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>> {name = "b"}
+! CHECK: acc.host_data dataOperands(%[[C]], %[[D]] : !fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>) {
+! CHECK: fir.call @_QPvadd(%[[A]]#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>) -> ()
+  !$acc end data
+! CHECK: acc.copyout accVar(%[[B]] : !fir.box<!fir.array<?xf64>>) to var(%[[A]]#0 : !fir.box<!fir.array<?xf64>>) {dataClause = #acc<data_clause acc_copy>, name = "b"}
+end 
+
+! Test for allocatable, pointer and assumed-shape variables appearing in use_device clause.
+subroutine test2(a, b, c)
+  integer :: N = 100
+  real*8, allocatable :: a(:)
+  real*8, target, allocatable :: d(:)
+  real*8 :: b(:)
+  real*8, pointer :: c(:)
+  call allocate(a(N))
+  call allocate(d(N))
+  c => d
+! CHECK: %[[DS:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK: %[[E:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[DS]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest2Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>)
+! CHECK: %[[F:.*]]:2 = hlfir.declare %arg1 dummy_scope %[[DS]] {uniq_name = "_QFtest2Eb"} : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
+! CHECK: %[[G:.*]]:2 = hlfir.declare %arg2 dummy_scope %[[DS]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest2Ec"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>)
+
+  !$acc data copy(a,b,c,d)
+  !$acc host_data use_device(a,b,c)
+  call vadd2(a,b,c)
+  !$acc end host_data
+
+! CHECK: %[[H:.*]] = acc.use_device varPtr(%[[E]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>> {name = "a"}
+! CHECK: %[[I:.*]] = acc.use_device varPtr(%[[E]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>> {name = "a"}
+! CHECK: %[[J:.*]] = acc.use_device var(%[[F]]#0 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
+! CHECK: %[[K:.*]] = acc.use_device var(%[[F]]#1 : !fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>> {name = "b"}
+! CHECK: %[[L:.*]] = acc.use_device varPtr(%[[G]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>> {name = "c"}
+! CHECK: %[[M:.*]] = acc.use_device varPtr(%[[G]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>> {name = "c"}
+! CHECK: acc.host_data dataOperands(%[[H]], %[[I]], %[[J]], %[[K]], %[[L]], %[[M]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf64>>>>) {
+
+
+
+
+  !$acc end data
+
+end


        

From flang-commits at lists.llvm.org  Thu Jul 17 09:04:53 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Thu, 17 Jul 2025 09:04:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][acc] Create UseDeviceOp for both
 results of hlfir.declare (PR #148017)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148017@github.com>
Message-ID: <68791f25.050a0220.29430b.1fa9@mx.google.com>


https://github.com/razvanlupusoru closed https://github.com/llvm/llvm-project/pull/148017

From flang-commits at lists.llvm.org  Thu Jul 17 09:10:11 2025
From: flang-commits at lists.llvm.org (Shilei Tian via flang-commits)
Date: Thu, 17 Jul 2025 09:10:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Migrate away from
	ArrayRef(std::nullopt_t) (PR #149337)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149337@github.com>
Message-ID: <68792063.170a0220.11567f.d911@mx.google.com>


https://github.com/shiltian approved this pull request.


https://github.com/llvm/llvm-project/pull/149337

From flang-commits at lists.llvm.org  Thu Jul 17 09:15:06 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 17 Jul 2025 09:15:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Migrate away from
	ArrayRef(std::nullopt_t) (PR #149337)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149337@github.com>
Message-ID: <6879218a.170a0220.332fc.d031@mx.google.com>


https://github.com/clementval commented:

LGTM. Thanks for doing the chan

https://github.com/llvm/llvm-project/pull/149337

From flang-commits at lists.llvm.org  Thu Jul 17 09:16:27 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 17 Jul 2025 09:16:27 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `COSPI` (PR #149343)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149343@github.com>
Message-ID: <687921db.170a0220.e8974.e67f@mx.google.com>


https://github.com/clementval approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/149343

From flang-commits at lists.llvm.org  Thu Jul 17 09:21:36 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Thu, 17 Jul 2025 09:21:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `COSPI` (PR #149343)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149343@github.com>
Message-ID: <68792310.050a0220.33f881.40c4@mx.google.com>


c8ef wrote:

Thanks for the quick review!

I have a minor question: Does flang have a utility similar to `update_test_checks.py`? Editing the test file manually is a bit tedious.

https://github.com/llvm/llvm-project/pull/149343

From flang-commits at lists.llvm.org  Thu Jul 17 09:27:56 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 17 Jul 2025 09:27:56 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Move
 extractOmpDirective to Utils.cpp, NFC (PR #148653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148653@github.com>
Message-ID: <6879248c.170a0220.776bf.9a37@mx.google.com>


https://github.com/kparzysz edited https://github.com/llvm/llvm-project/pull/148653

From flang-commits at lists.llvm.org  Thu Jul 17 09:28:51 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 17 Jul 2025 09:28:51 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Move
 extractOmpDirective to Utils.cpp, NFC (PR #148653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148653@github.com>
Message-ID: <687924c3.170a0220.ba30c.d5c8@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/148653

>From 76b1c6594d1e73f7d4a7011db3247a0c37b51339 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 07:45:34 -0500
Subject: [PATCH 1/3] [Frontend][OpenMP] Move isPrivatizingClause to OMP.h, NFC

---
 .../Frontend/OpenMP/ConstructDecompositionT.h  | 18 +-----------------
 llvm/include/llvm/Frontend/OpenMP/OMP.h        | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
index cdc80c88b7425..611bfe3f8aced 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -795,25 +795,9 @@ bool ConstructDecompositionT<C, H>::applyClause(
   // assigned to which leaf constructs.
 
   // [5.2:340:33]
-  auto canMakePrivateCopy = [](llvm::omp::Clause id) {
-    switch (id) {
-    // Clauses with "privatization" property:
-    case llvm::omp::Clause::OMPC_firstprivate:
-    case llvm::omp::Clause::OMPC_in_reduction:
-    case llvm::omp::Clause::OMPC_lastprivate:
-    case llvm::omp::Clause::OMPC_linear:
-    case llvm::omp::Clause::OMPC_private:
-    case llvm::omp::Clause::OMPC_reduction:
-    case llvm::omp::Clause::OMPC_task_reduction:
-      return true;
-    default:
-      return false;
-    }
-  };
-
   bool applied = applyIf(node, [&](const auto &leaf) {
     return llvm::any_of(leaf.clauses, [&](const ClauseTy *n) {
-      return canMakePrivateCopy(n->id);
+      return llvm::omp::isPrivatizingClause(n->id);
     });
   });
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index 35dafc6d246f0..d44c33301bde7 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -48,6 +48,22 @@ static constexpr inline bool canHaveIterator(Clause C) {
   }
 }
 
+// Can clause C create a private copy of a variable.
+static constexpr inline bool isPrivatizingClause(Clause C) {
+  switch (C) {
+  case OMPC_firstprivate:
+  case OMPC_in_reduction:
+  case OMPC_lastprivate:
+  case OMPC_linear:
+  case OMPC_private:
+  case OMPC_reduction:
+  case OMPC_task_reduction:
+    return true;
+  default:
+    return false;
+  }
+}
+
 static constexpr unsigned FallbackVersion = 52;
 LLVM_ABI ArrayRef<unsigned> getOpenMPVersions();
 

>From 82829a6f51fd96f720a64255442d96133a09b3dc Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 13:28:02 -0500
Subject: [PATCH 2/3] [utils][TableGen] Make some non-bitmask enums iterable

Additionally, add sentinel values <Enum>::First_ and <Enum>::Last_
to each one of those enums.

This will allow using `enum_seq_inclusive` to generate the list of
enum-typed values of any generated scoped (non-bitmask) enum.
---
 llvm/test/TableGen/directive1.td              | 25 +++++++++++++++++++
 llvm/test/TableGen/directive2.td              | 25 +++++++++++++++++++
 .../OpenMPDirectiveNameParserTest.cpp         | 21 +++++-----------
 .../utils/TableGen/Basic/DirectiveEmitter.cpp | 20 +++++++++++++--
 4 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/llvm/test/TableGen/directive1.td b/llvm/test/TableGen/directive1.td
index 1d2bd51204e4f..3eda077eeabf7 100644
--- a/llvm/test/TableGen/directive1.td
+++ b/llvm/test/TableGen/directive1.td
@@ -53,6 +53,7 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  #include "llvm/ADT/ArrayRef.h"
 // CHECK-NEXT:  #include "llvm/ADT/BitmaskEnum.h"
+// CHECK-NEXT:  #include "llvm/ADT/Sequence.h"
 // CHECK-NEXT:  #include "llvm/ADT/StringRef.h"
 // CHECK-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // CHECK-NEXT:  #include "llvm/Support/Compiler.h"
@@ -66,22 +67,26 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Association {
 // CHECK-NEXT:    Block,
+// CHECK-NEXT:    First_ = Block,
 // CHECK-NEXT:    Declaration,
 // CHECK-NEXT:    Delimited,
 // CHECK-NEXT:    Loop,
 // CHECK-NEXT:    None,
 // CHECK-NEXT:    Separating,
+// CHECK-NEXT:    Last_ = Separating,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Association_enumSize = 6;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Category {
 // CHECK-NEXT:    Declarative,
+// CHECK-NEXT:    First_ = Declarative,
 // CHECK-NEXT:    Executable,
 // CHECK-NEXT:    Informational,
 // CHECK-NEXT:    Meta,
 // CHECK-NEXT:    Subsidiary,
 // CHECK-NEXT:    Utility,
+// CHECK-NEXT:    Last_ = Utility,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Category_enumSize = 6;
@@ -96,6 +101,8 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Directive {
 // CHECK-NEXT:    TDLD_dira,
+// CHECK-NEXT:    First_ = TDLD_dira,
+// CHECK-NEXT:    Last_ = TDLD_dira,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Directive_enumSize = 1;
@@ -104,8 +111,10 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Clause {
 // CHECK-NEXT:    TDLC_clausea,
+// CHECK-NEXT:    First_ = TDLC_clausea,
 // CHECK-NEXT:    TDLC_clauseb,
 // CHECK-NEXT:    TDLC_clausec,
+// CHECK-NEXT:    Last_ = TDLC_clausec,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 3;
@@ -151,6 +160,22 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  LLVM_ABI StringRef getTdlAKindName(AKind x);
 // CHECK-EMPTY:
 // CHECK-NEXT:  } // namespace tdl
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Association> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Category> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Directive> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Clause> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
 // CHECK-NEXT:  } // namespace llvm
 // CHECK-NEXT:  #endif // LLVM_Tdl_INC
 
diff --git a/llvm/test/TableGen/directive2.td b/llvm/test/TableGen/directive2.td
index 3a64bb3900a31..a25197c3efd93 100644
--- a/llvm/test/TableGen/directive2.td
+++ b/llvm/test/TableGen/directive2.td
@@ -46,6 +46,7 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  #define LLVM_Tdl_INC
 // CHECK-EMPTY:
 // CHECK-NEXT:  #include "llvm/ADT/ArrayRef.h"
+// CHECK-NEXT:  #include "llvm/ADT/Sequence.h"
 // CHECK-NEXT:  #include "llvm/ADT/StringRef.h"
 // CHECK-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // CHECK-NEXT:  #include "llvm/Support/Compiler.h"
@@ -57,22 +58,26 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Association {
 // CHECK-NEXT:    Block,
+// CHECK-NEXT:    First_ = Block,
 // CHECK-NEXT:    Declaration,
 // CHECK-NEXT:    Delimited,
 // CHECK-NEXT:    Loop,
 // CHECK-NEXT:    None,
 // CHECK-NEXT:    Separating,
+// CHECK-NEXT:    Last_ = Separating,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Association_enumSize = 6;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Category {
 // CHECK-NEXT:    Declarative,
+// CHECK-NEXT:    First_ = Declarative,
 // CHECK-NEXT:    Executable,
 // CHECK-NEXT:    Informational,
 // CHECK-NEXT:    Meta,
 // CHECK-NEXT:    Subsidiary,
 // CHECK-NEXT:    Utility,
+// CHECK-NEXT:    Last_ = Utility,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Category_enumSize = 6;
@@ -87,15 +92,19 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Directive {
 // CHECK-NEXT:    TDLD_dira,
+// CHECK-NEXT:    First_ = TDLD_dira,
+// CHECK-NEXT:    Last_ = TDLD_dira,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Directive_enumSize = 1;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Clause {
 // CHECK-NEXT:    TDLC_clausea,
+// CHECK-NEXT:    First_ = TDLC_clausea,
 // CHECK-NEXT:    TDLC_clauseb,
 // CHECK-NEXT:    TDLC_clausec,
 // CHECK-NEXT:    TDLC_claused,
+// CHECK-NEXT:    Last_ = TDLC_claused,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 4;
@@ -124,6 +133,22 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  LLVM_ABI Category getDirectiveCategory(Directive D);
 // CHECK-NEXT:  LLVM_ABI SourceLanguage getDirectiveLanguages(Directive D);
 // CHECK-NEXT:  } // namespace tdl
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Association> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Category> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Directive> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Clause> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
 // CHECK-NEXT:  } // namespace llvm
 // CHECK-NEXT:  #endif // LLVM_Tdl_INC
 
diff --git a/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
index 0363a08cc0f03..10329820bef76 100644
--- a/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
@@ -48,12 +48,6 @@ static std::string &prepareParamName(std::string &Name) {
   return Name;
 }
 
-namespace llvm {
-template <> struct enum_iteration_traits<omp::Directive> {
-  static constexpr bool is_iterable = true;
-};
-} // namespace llvm
-
 // Test tokenizing.
 
 class Tokenize : public testing::TestWithParam<omp::Directive> {};
@@ -87,12 +81,10 @@ getParamName1(const testing::TestParamInfo<Tokenize::ParamType> &Info) {
   return prepareParamName(Name);
 }
 
-INSTANTIATE_TEST_SUITE_P(
-    DirectiveNameParserTest, Tokenize,
-    testing::ValuesIn(
-        llvm::enum_seq(static_cast<omp::Directive>(0),
-                       static_cast<omp::Directive>(omp::Directive_enumSize))),
-    getParamName1);
+INSTANTIATE_TEST_SUITE_P(DirectiveNameParserTest, Tokenize,
+                         testing::ValuesIn(llvm::enum_seq_inclusive(
+                             omp::Directive::First_, omp::Directive::Last_)),
+                         getParamName1);
 
 // Test parsing of valid names.
 
@@ -131,9 +123,8 @@ getParamName2(const testing::TestParamInfo<ParseValid::ParamType> &Info) {
 
 INSTANTIATE_TEST_SUITE_P(
     DirectiveNameParserTest, ParseValid,
-    testing::Combine(testing::ValuesIn(llvm::enum_seq(
-                         static_cast<omp::Directive>(0),
-                         static_cast<omp::Directive>(omp::Directive_enumSize))),
+    testing::Combine(testing::ValuesIn(llvm::enum_seq_inclusive(
+                         omp::Directive::First_, omp::Directive::Last_)),
                      testing::ValuesIn(omp::getOpenMPVersions())),
     getParamName2);
 
diff --git a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
index 177eecebce9a5..b48b9f6b7c41c 100644
--- a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
@@ -106,9 +106,15 @@ static void generateEnumClass(ArrayRef<const Record *> Records, raw_ostream &OS,
                               bool ExportEnums) {
   OS << "\n";
   OS << "enum class " << Enum << " {\n";
-  for (const Record *R : Records) {
-    OS << "  " << getIdentifierName(R, Prefix) << ",\n";
+  std::string N;
+  for (auto [I, R] : llvm::enumerate(Records)) {
+    N = getIdentifierName(R, Prefix);
+    OS << "  " << N << ",\n";
+    // Make the sentinel names less likely to conflict with actual names...
+    if (I == 0)
+      OS << "  First_ = " << N << ",\n";
   }
+  OS << "  Last_ = " << N << ",\n";
   OS << "};\n";
   OS << "\n";
   OS << "static constexpr std::size_t " << Enum
@@ -282,6 +288,7 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   if (DirLang.hasEnableBitmaskEnumInNamespace())
     OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n";
 
+  OS << "#include \"llvm/ADT/Sequence.h\"\n";
   OS << "#include \"llvm/ADT/StringRef.h\"\n";
   OS << "#include \"llvm/Frontend/Directive/Spelling.h\"\n";
   OS << "#include \"llvm/Support/Compiler.h\"\n";
@@ -375,6 +382,15 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   for (auto Ns : reverse(Namespaces))
     OS << "} // namespace " << Ns << "\n";
 
+  // These specializations need to be in ::llvm.
+  for (StringRef Enum : {"Association", "Category", "Directive", "Clause"}) {
+    OS << "\n";
+    OS << "template <> struct enum_iteration_traits<"
+       << DirLang.getCppNamespace() << "::" << Enum << "> {\n";
+    OS << "  static constexpr bool is_iterable = true;\n";
+    OS << "};\n";
+  }
+
   OS << "} // namespace llvm\n";
 
   OS << "#endif // LLVM_" << Lang << "_INC\n";

>From d3ed6f948468c4d27c5ac9999436612e650f9460 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 14 Jul 2025 10:18:07 -0500
Subject: [PATCH 3/3] [flang][OpenMP] Move extractOmpDirective to Utils.cpp,
 NFC

---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 84 -------------------------------
 flang/lib/Lower/OpenMP/Utils.cpp  | 84 +++++++++++++++++++++++++++++++
 flang/lib/Lower/OpenMP/Utils.h    |  3 ++
 3 files changed, 87 insertions(+), 84 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4458f62eea95a..fcb20fdf187ff 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -372,90 +372,6 @@ extractMappedBaseValues(llvm::ArrayRef<mlir::Value> vars,
   });
 }
 
-/// Get the directive enumeration value corresponding to the given OpenMP
-/// construct PFT node.
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
-  return common::visit(
-      common::visitors{
-          [](const parser::OpenMPAllocatorsConstruct &c) {
-            return llvm::omp::OMPD_allocators;
-          },
-          [](const parser::OpenMPAssumeConstruct &c) {
-            return llvm::omp::OMPD_assume;
-          },
-          [](const parser::OpenMPAtomicConstruct &c) {
-            return llvm::omp::OMPD_atomic;
-          },
-          [](const parser::OpenMPBlockConstruct &c) {
-            return std::get<parser::OmpBlockDirective>(
-                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPCriticalConstruct &c) {
-            return llvm::omp::OMPD_critical;
-          },
-          [](const parser::OpenMPDeclarativeAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPDispatchConstruct &c) {
-            return llvm::omp::OMPD_dispatch;
-          },
-          [](const parser::OpenMPExecutableAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPLoopConstruct &c) {
-            return std::get<parser::OmpLoopDirective>(
-                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPSectionConstruct &c) {
-            return llvm::omp::OMPD_section;
-          },
-          [](const parser::OpenMPSectionsConstruct &c) {
-            return std::get<parser::OmpSectionsDirective>(
-                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPStandaloneConstruct &c) {
-            return common::visit(
-                common::visitors{
-                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
-                      return c.v.DirId();
-                    },
-                    [](const parser::OpenMPFlushConstruct &c) {
-                      return llvm::omp::OMPD_flush;
-                    },
-                    [](const parser::OpenMPCancelConstruct &c) {
-                      return llvm::omp::OMPD_cancel;
-                    },
-                    [](const parser::OpenMPCancellationPointConstruct &c) {
-                      return llvm::omp::OMPD_cancellation_point;
-                    },
-                    [](const parser::OmpMetadirectiveDirective &c) {
-                      return llvm::omp::OMPD_metadirective;
-                    },
-                    [](const parser::OpenMPDepobjConstruct &c) {
-                      return llvm::omp::OMPD_depobj;
-                    },
-                    [](const parser::OpenMPInteropConstruct &c) {
-                      return llvm::omp::OMPD_interop;
-                    }},
-                c.u);
-          },
-          [](const parser::OpenMPUtilityConstruct &c) {
-            return common::visit(
-                common::visitors{[](const parser::OmpErrorDirective &c) {
-                                   return llvm::omp::OMPD_error;
-                                 },
-                                 [](const parser::OmpNothingDirective &c) {
-                                   return llvm::omp::OMPD_nothing;
-                                 }},
-                c.u);
-          }},
-      ompConstruct.u);
-}
-
 /// Populate the global \see hostEvalInfo after processing clauses for the given
 /// \p eval OpenMP target construct, or nested constructs, if these must be
 /// evaluated outside of the target region per the spec.
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 2e53f01f1da6a..b194150c0f7f0 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -661,6 +661,90 @@ bool collectLoopRelatedInfo(
 
   return found;
 }
+
+/// Get the directive enumeration value corresponding to the given OpenMP
+/// construct PFT node.
+llvm::omp::Directive
+extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
+  return common::visit(
+      common::visitors{
+          [](const parser::OpenMPAllocatorsConstruct &c) {
+            return llvm::omp::OMPD_allocators;
+          },
+          [](const parser::OpenMPAssumeConstruct &c) {
+            return llvm::omp::OMPD_assume;
+          },
+          [](const parser::OpenMPAtomicConstruct &c) {
+            return llvm::omp::OMPD_atomic;
+          },
+          [](const parser::OpenMPBlockConstruct &c) {
+            return std::get<parser::OmpBlockDirective>(
+                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPCriticalConstruct &c) {
+            return llvm::omp::OMPD_critical;
+          },
+          [](const parser::OpenMPDeclarativeAllocate &c) {
+            return llvm::omp::OMPD_allocate;
+          },
+          [](const parser::OpenMPDispatchConstruct &c) {
+            return llvm::omp::OMPD_dispatch;
+          },
+          [](const parser::OpenMPExecutableAllocate &c) {
+            return llvm::omp::OMPD_allocate;
+          },
+          [](const parser::OpenMPLoopConstruct &c) {
+            return std::get<parser::OmpLoopDirective>(
+                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPSectionConstruct &c) {
+            return llvm::omp::OMPD_section;
+          },
+          [](const parser::OpenMPSectionsConstruct &c) {
+            return std::get<parser::OmpSectionsDirective>(
+                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPStandaloneConstruct &c) {
+            return common::visit(
+                common::visitors{
+                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
+                      return c.v.DirId();
+                    },
+                    [](const parser::OpenMPFlushConstruct &c) {
+                      return llvm::omp::OMPD_flush;
+                    },
+                    [](const parser::OpenMPCancelConstruct &c) {
+                      return llvm::omp::OMPD_cancel;
+                    },
+                    [](const parser::OpenMPCancellationPointConstruct &c) {
+                      return llvm::omp::OMPD_cancellation_point;
+                    },
+                    [](const parser::OmpMetadirectiveDirective &c) {
+                      return llvm::omp::OMPD_metadirective;
+                    },
+                    [](const parser::OpenMPDepobjConstruct &c) {
+                      return llvm::omp::OMPD_depobj;
+                    },
+                    [](const parser::OpenMPInteropConstruct &c) {
+                      return llvm::omp::OMPD_interop;
+                    }},
+                c.u);
+          },
+          [](const parser::OpenMPUtilityConstruct &c) {
+            return common::visit(
+                common::visitors{[](const parser::OmpErrorDirective &c) {
+                                   return llvm::omp::OMPD_error;
+                                 },
+                                 [](const parser::OmpNothingDirective &c) {
+                                   return llvm::omp::OMPD_nothing;
+                                 }},
+                c.u);
+          }},
+      ompConstruct.u);
+}
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 1526bd4e90233..8e3ad5c3452e2 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -166,6 +166,9 @@ bool collectLoopRelatedInfo(
     lower::pft::Evaluation &eval, const omp::List<omp::Clause> &clauses,
     mlir::omp::LoopRelatedClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &iv);
+
+llvm::omp::Directive
+extractOmpDirective(const parser::OpenMPConstruct &ompConstruct);
 } // namespace omp
 } // namespace lower
 } // namespace Fortran


From flang-commits at lists.llvm.org  Thu Jul 17 09:31:05 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 17 Jul 2025 09:31:05 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Move
 extractOmpDirective to Utils.cpp, NFC (PR #148653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148653@github.com>
Message-ID: <68792549.170a0220.66dfd.f251@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/148653

>From 76b1c6594d1e73f7d4a7011db3247a0c37b51339 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 07:45:34 -0500
Subject: [PATCH 1/4] [Frontend][OpenMP] Move isPrivatizingClause to OMP.h, NFC

---
 .../Frontend/OpenMP/ConstructDecompositionT.h  | 18 +-----------------
 llvm/include/llvm/Frontend/OpenMP/OMP.h        | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
index cdc80c88b7425..611bfe3f8aced 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -795,25 +795,9 @@ bool ConstructDecompositionT<C, H>::applyClause(
   // assigned to which leaf constructs.
 
   // [5.2:340:33]
-  auto canMakePrivateCopy = [](llvm::omp::Clause id) {
-    switch (id) {
-    // Clauses with "privatization" property:
-    case llvm::omp::Clause::OMPC_firstprivate:
-    case llvm::omp::Clause::OMPC_in_reduction:
-    case llvm::omp::Clause::OMPC_lastprivate:
-    case llvm::omp::Clause::OMPC_linear:
-    case llvm::omp::Clause::OMPC_private:
-    case llvm::omp::Clause::OMPC_reduction:
-    case llvm::omp::Clause::OMPC_task_reduction:
-      return true;
-    default:
-      return false;
-    }
-  };
-
   bool applied = applyIf(node, [&](const auto &leaf) {
     return llvm::any_of(leaf.clauses, [&](const ClauseTy *n) {
-      return canMakePrivateCopy(n->id);
+      return llvm::omp::isPrivatizingClause(n->id);
     });
   });
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index 35dafc6d246f0..d44c33301bde7 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -48,6 +48,22 @@ static constexpr inline bool canHaveIterator(Clause C) {
   }
 }
 
+// Can clause C create a private copy of a variable.
+static constexpr inline bool isPrivatizingClause(Clause C) {
+  switch (C) {
+  case OMPC_firstprivate:
+  case OMPC_in_reduction:
+  case OMPC_lastprivate:
+  case OMPC_linear:
+  case OMPC_private:
+  case OMPC_reduction:
+  case OMPC_task_reduction:
+    return true;
+  default:
+    return false;
+  }
+}
+
 static constexpr unsigned FallbackVersion = 52;
 LLVM_ABI ArrayRef<unsigned> getOpenMPVersions();
 

>From 82829a6f51fd96f720a64255442d96133a09b3dc Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 13:28:02 -0500
Subject: [PATCH 2/4] [utils][TableGen] Make some non-bitmask enums iterable

Additionally, add sentinel values <Enum>::First_ and <Enum>::Last_
to each one of those enums.

This will allow using `enum_seq_inclusive` to generate the list of
enum-typed values of any generated scoped (non-bitmask) enum.
---
 llvm/test/TableGen/directive1.td              | 25 +++++++++++++++++++
 llvm/test/TableGen/directive2.td              | 25 +++++++++++++++++++
 .../OpenMPDirectiveNameParserTest.cpp         | 21 +++++-----------
 .../utils/TableGen/Basic/DirectiveEmitter.cpp | 20 +++++++++++++--
 4 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/llvm/test/TableGen/directive1.td b/llvm/test/TableGen/directive1.td
index 1d2bd51204e4f..3eda077eeabf7 100644
--- a/llvm/test/TableGen/directive1.td
+++ b/llvm/test/TableGen/directive1.td
@@ -53,6 +53,7 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  #include "llvm/ADT/ArrayRef.h"
 // CHECK-NEXT:  #include "llvm/ADT/BitmaskEnum.h"
+// CHECK-NEXT:  #include "llvm/ADT/Sequence.h"
 // CHECK-NEXT:  #include "llvm/ADT/StringRef.h"
 // CHECK-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // CHECK-NEXT:  #include "llvm/Support/Compiler.h"
@@ -66,22 +67,26 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Association {
 // CHECK-NEXT:    Block,
+// CHECK-NEXT:    First_ = Block,
 // CHECK-NEXT:    Declaration,
 // CHECK-NEXT:    Delimited,
 // CHECK-NEXT:    Loop,
 // CHECK-NEXT:    None,
 // CHECK-NEXT:    Separating,
+// CHECK-NEXT:    Last_ = Separating,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Association_enumSize = 6;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Category {
 // CHECK-NEXT:    Declarative,
+// CHECK-NEXT:    First_ = Declarative,
 // CHECK-NEXT:    Executable,
 // CHECK-NEXT:    Informational,
 // CHECK-NEXT:    Meta,
 // CHECK-NEXT:    Subsidiary,
 // CHECK-NEXT:    Utility,
+// CHECK-NEXT:    Last_ = Utility,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Category_enumSize = 6;
@@ -96,6 +101,8 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Directive {
 // CHECK-NEXT:    TDLD_dira,
+// CHECK-NEXT:    First_ = TDLD_dira,
+// CHECK-NEXT:    Last_ = TDLD_dira,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Directive_enumSize = 1;
@@ -104,8 +111,10 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Clause {
 // CHECK-NEXT:    TDLC_clausea,
+// CHECK-NEXT:    First_ = TDLC_clausea,
 // CHECK-NEXT:    TDLC_clauseb,
 // CHECK-NEXT:    TDLC_clausec,
+// CHECK-NEXT:    Last_ = TDLC_clausec,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 3;
@@ -151,6 +160,22 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  LLVM_ABI StringRef getTdlAKindName(AKind x);
 // CHECK-EMPTY:
 // CHECK-NEXT:  } // namespace tdl
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Association> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Category> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Directive> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Clause> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
 // CHECK-NEXT:  } // namespace llvm
 // CHECK-NEXT:  #endif // LLVM_Tdl_INC
 
diff --git a/llvm/test/TableGen/directive2.td b/llvm/test/TableGen/directive2.td
index 3a64bb3900a31..a25197c3efd93 100644
--- a/llvm/test/TableGen/directive2.td
+++ b/llvm/test/TableGen/directive2.td
@@ -46,6 +46,7 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  #define LLVM_Tdl_INC
 // CHECK-EMPTY:
 // CHECK-NEXT:  #include "llvm/ADT/ArrayRef.h"
+// CHECK-NEXT:  #include "llvm/ADT/Sequence.h"
 // CHECK-NEXT:  #include "llvm/ADT/StringRef.h"
 // CHECK-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // CHECK-NEXT:  #include "llvm/Support/Compiler.h"
@@ -57,22 +58,26 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Association {
 // CHECK-NEXT:    Block,
+// CHECK-NEXT:    First_ = Block,
 // CHECK-NEXT:    Declaration,
 // CHECK-NEXT:    Delimited,
 // CHECK-NEXT:    Loop,
 // CHECK-NEXT:    None,
 // CHECK-NEXT:    Separating,
+// CHECK-NEXT:    Last_ = Separating,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Association_enumSize = 6;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Category {
 // CHECK-NEXT:    Declarative,
+// CHECK-NEXT:    First_ = Declarative,
 // CHECK-NEXT:    Executable,
 // CHECK-NEXT:    Informational,
 // CHECK-NEXT:    Meta,
 // CHECK-NEXT:    Subsidiary,
 // CHECK-NEXT:    Utility,
+// CHECK-NEXT:    Last_ = Utility,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Category_enumSize = 6;
@@ -87,15 +92,19 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Directive {
 // CHECK-NEXT:    TDLD_dira,
+// CHECK-NEXT:    First_ = TDLD_dira,
+// CHECK-NEXT:    Last_ = TDLD_dira,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Directive_enumSize = 1;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Clause {
 // CHECK-NEXT:    TDLC_clausea,
+// CHECK-NEXT:    First_ = TDLC_clausea,
 // CHECK-NEXT:    TDLC_clauseb,
 // CHECK-NEXT:    TDLC_clausec,
 // CHECK-NEXT:    TDLC_claused,
+// CHECK-NEXT:    Last_ = TDLC_claused,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 4;
@@ -124,6 +133,22 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  LLVM_ABI Category getDirectiveCategory(Directive D);
 // CHECK-NEXT:  LLVM_ABI SourceLanguage getDirectiveLanguages(Directive D);
 // CHECK-NEXT:  } // namespace tdl
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Association> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Category> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Directive> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Clause> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
 // CHECK-NEXT:  } // namespace llvm
 // CHECK-NEXT:  #endif // LLVM_Tdl_INC
 
diff --git a/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
index 0363a08cc0f03..10329820bef76 100644
--- a/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
@@ -48,12 +48,6 @@ static std::string &prepareParamName(std::string &Name) {
   return Name;
 }
 
-namespace llvm {
-template <> struct enum_iteration_traits<omp::Directive> {
-  static constexpr bool is_iterable = true;
-};
-} // namespace llvm
-
 // Test tokenizing.
 
 class Tokenize : public testing::TestWithParam<omp::Directive> {};
@@ -87,12 +81,10 @@ getParamName1(const testing::TestParamInfo<Tokenize::ParamType> &Info) {
   return prepareParamName(Name);
 }
 
-INSTANTIATE_TEST_SUITE_P(
-    DirectiveNameParserTest, Tokenize,
-    testing::ValuesIn(
-        llvm::enum_seq(static_cast<omp::Directive>(0),
-                       static_cast<omp::Directive>(omp::Directive_enumSize))),
-    getParamName1);
+INSTANTIATE_TEST_SUITE_P(DirectiveNameParserTest, Tokenize,
+                         testing::ValuesIn(llvm::enum_seq_inclusive(
+                             omp::Directive::First_, omp::Directive::Last_)),
+                         getParamName1);
 
 // Test parsing of valid names.
 
@@ -131,9 +123,8 @@ getParamName2(const testing::TestParamInfo<ParseValid::ParamType> &Info) {
 
 INSTANTIATE_TEST_SUITE_P(
     DirectiveNameParserTest, ParseValid,
-    testing::Combine(testing::ValuesIn(llvm::enum_seq(
-                         static_cast<omp::Directive>(0),
-                         static_cast<omp::Directive>(omp::Directive_enumSize))),
+    testing::Combine(testing::ValuesIn(llvm::enum_seq_inclusive(
+                         omp::Directive::First_, omp::Directive::Last_)),
                      testing::ValuesIn(omp::getOpenMPVersions())),
     getParamName2);
 
diff --git a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
index 177eecebce9a5..b48b9f6b7c41c 100644
--- a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
@@ -106,9 +106,15 @@ static void generateEnumClass(ArrayRef<const Record *> Records, raw_ostream &OS,
                               bool ExportEnums) {
   OS << "\n";
   OS << "enum class " << Enum << " {\n";
-  for (const Record *R : Records) {
-    OS << "  " << getIdentifierName(R, Prefix) << ",\n";
+  std::string N;
+  for (auto [I, R] : llvm::enumerate(Records)) {
+    N = getIdentifierName(R, Prefix);
+    OS << "  " << N << ",\n";
+    // Make the sentinel names less likely to conflict with actual names...
+    if (I == 0)
+      OS << "  First_ = " << N << ",\n";
   }
+  OS << "  Last_ = " << N << ",\n";
   OS << "};\n";
   OS << "\n";
   OS << "static constexpr std::size_t " << Enum
@@ -282,6 +288,7 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   if (DirLang.hasEnableBitmaskEnumInNamespace())
     OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n";
 
+  OS << "#include \"llvm/ADT/Sequence.h\"\n";
   OS << "#include \"llvm/ADT/StringRef.h\"\n";
   OS << "#include \"llvm/Frontend/Directive/Spelling.h\"\n";
   OS << "#include \"llvm/Support/Compiler.h\"\n";
@@ -375,6 +382,15 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   for (auto Ns : reverse(Namespaces))
     OS << "} // namespace " << Ns << "\n";
 
+  // These specializations need to be in ::llvm.
+  for (StringRef Enum : {"Association", "Category", "Directive", "Clause"}) {
+    OS << "\n";
+    OS << "template <> struct enum_iteration_traits<"
+       << DirLang.getCppNamespace() << "::" << Enum << "> {\n";
+    OS << "  static constexpr bool is_iterable = true;\n";
+    OS << "};\n";
+  }
+
   OS << "} // namespace llvm\n";
 
   OS << "#endif // LLVM_" << Lang << "_INC\n";

>From d3ed6f948468c4d27c5ac9999436612e650f9460 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 14 Jul 2025 10:18:07 -0500
Subject: [PATCH 3/4] [flang][OpenMP] Move extractOmpDirective to Utils.cpp,
 NFC

---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 84 -------------------------------
 flang/lib/Lower/OpenMP/Utils.cpp  | 84 +++++++++++++++++++++++++++++++
 flang/lib/Lower/OpenMP/Utils.h    |  3 ++
 3 files changed, 87 insertions(+), 84 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4458f62eea95a..fcb20fdf187ff 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -372,90 +372,6 @@ extractMappedBaseValues(llvm::ArrayRef<mlir::Value> vars,
   });
 }
 
-/// Get the directive enumeration value corresponding to the given OpenMP
-/// construct PFT node.
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
-  return common::visit(
-      common::visitors{
-          [](const parser::OpenMPAllocatorsConstruct &c) {
-            return llvm::omp::OMPD_allocators;
-          },
-          [](const parser::OpenMPAssumeConstruct &c) {
-            return llvm::omp::OMPD_assume;
-          },
-          [](const parser::OpenMPAtomicConstruct &c) {
-            return llvm::omp::OMPD_atomic;
-          },
-          [](const parser::OpenMPBlockConstruct &c) {
-            return std::get<parser::OmpBlockDirective>(
-                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPCriticalConstruct &c) {
-            return llvm::omp::OMPD_critical;
-          },
-          [](const parser::OpenMPDeclarativeAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPDispatchConstruct &c) {
-            return llvm::omp::OMPD_dispatch;
-          },
-          [](const parser::OpenMPExecutableAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPLoopConstruct &c) {
-            return std::get<parser::OmpLoopDirective>(
-                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPSectionConstruct &c) {
-            return llvm::omp::OMPD_section;
-          },
-          [](const parser::OpenMPSectionsConstruct &c) {
-            return std::get<parser::OmpSectionsDirective>(
-                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPStandaloneConstruct &c) {
-            return common::visit(
-                common::visitors{
-                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
-                      return c.v.DirId();
-                    },
-                    [](const parser::OpenMPFlushConstruct &c) {
-                      return llvm::omp::OMPD_flush;
-                    },
-                    [](const parser::OpenMPCancelConstruct &c) {
-                      return llvm::omp::OMPD_cancel;
-                    },
-                    [](const parser::OpenMPCancellationPointConstruct &c) {
-                      return llvm::omp::OMPD_cancellation_point;
-                    },
-                    [](const parser::OmpMetadirectiveDirective &c) {
-                      return llvm::omp::OMPD_metadirective;
-                    },
-                    [](const parser::OpenMPDepobjConstruct &c) {
-                      return llvm::omp::OMPD_depobj;
-                    },
-                    [](const parser::OpenMPInteropConstruct &c) {
-                      return llvm::omp::OMPD_interop;
-                    }},
-                c.u);
-          },
-          [](const parser::OpenMPUtilityConstruct &c) {
-            return common::visit(
-                common::visitors{[](const parser::OmpErrorDirective &c) {
-                                   return llvm::omp::OMPD_error;
-                                 },
-                                 [](const parser::OmpNothingDirective &c) {
-                                   return llvm::omp::OMPD_nothing;
-                                 }},
-                c.u);
-          }},
-      ompConstruct.u);
-}
-
 /// Populate the global \see hostEvalInfo after processing clauses for the given
 /// \p eval OpenMP target construct, or nested constructs, if these must be
 /// evaluated outside of the target region per the spec.
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 2e53f01f1da6a..b194150c0f7f0 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -661,6 +661,90 @@ bool collectLoopRelatedInfo(
 
   return found;
 }
+
+/// Get the directive enumeration value corresponding to the given OpenMP
+/// construct PFT node.
+llvm::omp::Directive
+extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
+  return common::visit(
+      common::visitors{
+          [](const parser::OpenMPAllocatorsConstruct &c) {
+            return llvm::omp::OMPD_allocators;
+          },
+          [](const parser::OpenMPAssumeConstruct &c) {
+            return llvm::omp::OMPD_assume;
+          },
+          [](const parser::OpenMPAtomicConstruct &c) {
+            return llvm::omp::OMPD_atomic;
+          },
+          [](const parser::OpenMPBlockConstruct &c) {
+            return std::get<parser::OmpBlockDirective>(
+                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPCriticalConstruct &c) {
+            return llvm::omp::OMPD_critical;
+          },
+          [](const parser::OpenMPDeclarativeAllocate &c) {
+            return llvm::omp::OMPD_allocate;
+          },
+          [](const parser::OpenMPDispatchConstruct &c) {
+            return llvm::omp::OMPD_dispatch;
+          },
+          [](const parser::OpenMPExecutableAllocate &c) {
+            return llvm::omp::OMPD_allocate;
+          },
+          [](const parser::OpenMPLoopConstruct &c) {
+            return std::get<parser::OmpLoopDirective>(
+                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPSectionConstruct &c) {
+            return llvm::omp::OMPD_section;
+          },
+          [](const parser::OpenMPSectionsConstruct &c) {
+            return std::get<parser::OmpSectionsDirective>(
+                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPStandaloneConstruct &c) {
+            return common::visit(
+                common::visitors{
+                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
+                      return c.v.DirId();
+                    },
+                    [](const parser::OpenMPFlushConstruct &c) {
+                      return llvm::omp::OMPD_flush;
+                    },
+                    [](const parser::OpenMPCancelConstruct &c) {
+                      return llvm::omp::OMPD_cancel;
+                    },
+                    [](const parser::OpenMPCancellationPointConstruct &c) {
+                      return llvm::omp::OMPD_cancellation_point;
+                    },
+                    [](const parser::OmpMetadirectiveDirective &c) {
+                      return llvm::omp::OMPD_metadirective;
+                    },
+                    [](const parser::OpenMPDepobjConstruct &c) {
+                      return llvm::omp::OMPD_depobj;
+                    },
+                    [](const parser::OpenMPInteropConstruct &c) {
+                      return llvm::omp::OMPD_interop;
+                    }},
+                c.u);
+          },
+          [](const parser::OpenMPUtilityConstruct &c) {
+            return common::visit(
+                common::visitors{[](const parser::OmpErrorDirective &c) {
+                                   return llvm::omp::OMPD_error;
+                                 },
+                                 [](const parser::OmpNothingDirective &c) {
+                                   return llvm::omp::OMPD_nothing;
+                                 }},
+                c.u);
+          }},
+      ompConstruct.u);
+}
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 1526bd4e90233..8e3ad5c3452e2 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -166,6 +166,9 @@ bool collectLoopRelatedInfo(
     lower::pft::Evaluation &eval, const omp::List<omp::Clause> &clauses,
     mlir::omp::LoopRelatedClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &iv);
+
+llvm::omp::Directive
+extractOmpDirective(const parser::OpenMPConstruct &ompConstruct);
 } // namespace omp
 } // namespace lower
 } // namespace Fortran

>From 937d723ae6dc07a931309967f072373357a2a3a7 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Thu, 17 Jul 2025 11:30:56 -0500
Subject: [PATCH 4/4] Update DirectiveEmitter.cpp

---
 llvm/utils/TableGen/Basic/DirectiveEmitter.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
index 99975a3482e18..f0e23690367db 100644
--- a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
@@ -117,7 +117,6 @@ static void generateEnumClass(ArrayRef<const Record *> Records, raw_ostream &OS,
     }
     OS << "  Last_ = " << N << ",\n";
   }
-  OS << "  Last_ = " << N << ",\n";
   OS << "};\n";
   OS << "\n";
   OS << "static constexpr std::size_t " << Enum


From flang-commits at lists.llvm.org  Thu Jul 17 09:38:05 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 09:38:05 -0700 (PDT)
Subject: [flang-commits] [flang] 0dae924 - [openacc][flang] Support two type
 bindName representation in acc routine (#149147)
Message-ID: <687926ed.050a0220.3ca644.a596@mx.google.com>


Author: delaram-talaashrafi
Date: 2025-07-17T09:38:02-07:00
New Revision: 0dae924c1f668f74370b642ba91f818b728aca40

URL: https://github.com/llvm/llvm-project/commit/0dae924c1f668f74370b642ba91f818b728aca40
DIFF: https://github.com/llvm/llvm-project/commit/0dae924c1f668f74370b642ba91f818b728aca40.diff

LOG: [openacc][flang] Support two type bindName representation in acc routine (#149147)

Based on the OpenACC specification — which states that if the bind name
is given as an identifier it should be resolved according to the
compiled language, and if given as a string it should be used unmodified
— we introduce two distinct `bindName` representations for `acc routine`
to handle each case appropriately: one as an array of `SymbolRefAttr`
for identifiers and another as an array of `StringAttr` for strings.

To ensure correct correspondence between bind names and devices, this
patch also introduces two separate sets of device attributes. The
routine operation is extended accordingly, along with the necessary
updates to the OpenACC dialect and its lowering.

Added: 
    

Modified: 
    flang/lib/Lower/OpenACC.cpp
    flang/test/Lower/OpenACC/acc-routine.f90
    flang/test/Lower/OpenACC/acc-routine03.f90
    mlir/include/mlir/Dialect/OpenACC/OpenACC.h
    mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
    mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
    mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 25682cba5620e..51eb33dec186b 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4414,10 +4414,34 @@ getAttributeValueByDeviceType(llvm::SmallVector<mlir::Attribute> &attributes,
   return std::nullopt;
 }
 
+// Helper function to extract string value from bind name variant
+static std::optional<llvm::StringRef> getBindNameStringValue(
+    const std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+        &bindNameValue) {
+  if (!bindNameValue.has_value())
+    return std::nullopt;
+
+  return std::visit(
+      [](const auto &attr) -> std::optional<llvm::StringRef> {
+        if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                     mlir::StringAttr>) {
+          return attr.getValue();
+        } else if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                            mlir::SymbolRefAttr>) {
+          return attr.getLeafReference();
+        } else {
+          return std::nullopt;
+        }
+      },
+      bindNameValue.value());
+}
+
 static bool compareDeviceTypeInfo(
     mlir::acc::RoutineOp op,
-    llvm::SmallVector<mlir::Attribute> &bindNameArrayAttr,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypeArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypeArrayAttr,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypeArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangDimArrayAttr,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypeArrayAttr,
@@ -4427,9 +4451,13 @@ static bool compareDeviceTypeInfo(
   for (uint32_t dtypeInt = 0;
        dtypeInt != mlir::acc::getMaxEnumValForDeviceType(); ++dtypeInt) {
     auto dtype = static_cast<mlir::acc::DeviceType>(dtypeInt);
-    if (op.getBindNameValue(dtype) !=
-        getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
-            bindNameArrayAttr, bindNameDeviceTypeArrayAttr, dtype))
+    auto bindNameValue = getBindNameStringValue(op.getBindNameValue(dtype));
+    if (bindNameValue !=
+            getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
+                bindIdNameArrayAttr, bindIdNameDeviceTypeArrayAttr, dtype) &&
+        bindNameValue !=
+            getAttributeValueByDeviceType<llvm::StringRef, mlir::StringAttr>(
+                bindStrNameArrayAttr, bindStrNameDeviceTypeArrayAttr, dtype))
       return false;
     if (op.hasGang(dtype) != hasDeviceType(gangArrayAttr, dtype))
       return false;
@@ -4476,8 +4504,10 @@ getArrayAttrOrNull(fir::FirOpBuilder &builder,
 void createOpenACCRoutineConstruct(
     Fortran::lower::AbstractConverter &converter, mlir::Location loc,
     mlir::ModuleOp mod, mlir::func::FuncOp funcOp, std::string funcName,
-    bool hasNohost, llvm::SmallVector<mlir::Attribute> &bindNames,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypes,
+    bool hasNohost, llvm::SmallVector<mlir::Attribute> &bindIdNames,
+    llvm::SmallVector<mlir::Attribute> &bindStrNames,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDimValues,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypes,
@@ -4490,7 +4520,8 @@ void createOpenACCRoutineConstruct(
         0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
-      if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
+      if (compareDeviceTypeInfo(routineOp, bindIdNames, bindStrNames,
+                                bindIdNameDeviceTypes, bindStrNameDeviceTypes,
                                 gangDeviceTypes, gangDimValues,
                                 gangDimDeviceTypes, seqDeviceTypes,
                                 workerDeviceTypes, vectorDeviceTypes) &&
@@ -4507,8 +4538,10 @@ void createOpenACCRoutineConstruct(
   modBuilder.create<mlir::acc::RoutineOp>(
       loc, routineOpStr,
       mlir::SymbolRefAttr::get(builder.getContext(), funcName),
-      getArrayAttrOrNull(builder, bindNames),
-      getArrayAttrOrNull(builder, bindNameDeviceTypes),
+      getArrayAttrOrNull(builder, bindIdNames),
+      getArrayAttrOrNull(builder, bindStrNames),
+      getArrayAttrOrNull(builder, bindIdNameDeviceTypes),
+      getArrayAttrOrNull(builder, bindStrNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),
       getArrayAttrOrNull(builder, seqDeviceTypes), hasNohost,
@@ -4525,8 +4558,10 @@ static void interpretRoutineDeviceInfo(
     llvm::SmallVector<mlir::Attribute> &seqDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &workerDeviceTypes,
-    llvm::SmallVector<mlir::Attribute> &bindNameDeviceTypes,
-    llvm::SmallVector<mlir::Attribute> &bindNames,
+    llvm::SmallVector<mlir::Attribute> &bindIdNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindStrNameDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &bindIdNames,
+    llvm::SmallVector<mlir::Attribute> &bindStrNames,
     llvm::SmallVector<mlir::Attribute> &gangDeviceTypes,
     llvm::SmallVector<mlir::Attribute> &gangDimValues,
     llvm::SmallVector<mlir::Attribute> &gangDimDeviceTypes) {
@@ -4559,16 +4594,18 @@ static void interpretRoutineDeviceInfo(
   if (dinfo.bindNameOpt().has_value()) {
     const auto &bindName = dinfo.bindNameOpt().value();
     mlir::Attribute bindNameAttr;
-    if (const auto &bindStr{std::get_if<std::string>(&bindName)}) {
+    if (const auto &bindSym{
+            std::get_if<Fortran::semantics::SymbolRef>(&bindName)}) {
+      bindNameAttr = builder.getSymbolRefAttr(converter.mangleName(*bindSym));
+      bindIdNames.push_back(bindNameAttr);
+      bindIdNameDeviceTypes.push_back(getDeviceTypeAttr());
+    } else if (const auto &bindStr{std::get_if<std::string>(&bindName)}) {
       bindNameAttr = builder.getStringAttr(*bindStr);
-    } else if (const auto &bindSym{
-                   std::get_if<Fortran::semantics::SymbolRef>(&bindName)}) {
-      bindNameAttr = builder.getStringAttr(converter.mangleName(*bindSym));
+      bindStrNames.push_back(bindNameAttr);
+      bindStrNameDeviceTypes.push_back(getDeviceTypeAttr());
     } else {
       llvm_unreachable("Unsupported bind name type");
     }
-    bindNames.push_back(bindNameAttr);
-    bindNameDeviceTypes.push_back(getDeviceTypeAttr());
   }
 }
 
@@ -4584,8 +4621,9 @@ void Fortran::lower::genOpenACCRoutineConstruct(
   bool hasNohost{false};
 
   llvm::SmallVector<mlir::Attribute> seqDeviceTypes, vectorDeviceTypes,
-      workerDeviceTypes, bindNameDeviceTypes, bindNames, gangDeviceTypes,
-      gangDimDeviceTypes, gangDimValues;
+      workerDeviceTypes, bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+      bindIdNames, bindStrNames, gangDeviceTypes, gangDimDeviceTypes,
+      gangDimValues;
 
   for (const Fortran::semantics::OpenACCRoutineInfo &info : routineInfos) {
     // Device Independent Attributes
@@ -4594,24 +4632,26 @@ void Fortran::lower::genOpenACCRoutineConstruct(
     }
     // Note: Device Independent Attributes are set to the
     // none device type in `info`.
-    interpretRoutineDeviceInfo(converter, info, seqDeviceTypes,
-                               vectorDeviceTypes, workerDeviceTypes,
-                               bindNameDeviceTypes, bindNames, gangDeviceTypes,
-                               gangDimValues, gangDimDeviceTypes);
+    interpretRoutineDeviceInfo(
+        converter, info, seqDeviceTypes, vectorDeviceTypes, workerDeviceTypes,
+        bindIdNameDeviceTypes, bindStrNameDeviceTypes, bindIdNames,
+        bindStrNames, gangDeviceTypes, gangDimValues, gangDimDeviceTypes);
 
     // Device Dependent Attributes
     for (const Fortran::semantics::OpenACCRoutineDeviceTypeInfo &dinfo :
          info.deviceTypeInfos()) {
-      interpretRoutineDeviceInfo(
-          converter, dinfo, seqDeviceTypes, vectorDeviceTypes,
-          workerDeviceTypes, bindNameDeviceTypes, bindNames, gangDeviceTypes,
-          gangDimValues, gangDimDeviceTypes);
+      interpretRoutineDeviceInfo(converter, dinfo, seqDeviceTypes,
+                                 vectorDeviceTypes, workerDeviceTypes,
+                                 bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+                                 bindIdNames, bindStrNames, gangDeviceTypes,
+                                 gangDimValues, gangDimDeviceTypes);
     }
   }
   createOpenACCRoutineConstruct(
-      converter, loc, mod, funcOp, funcName, hasNohost, bindNames,
-      bindNameDeviceTypes, gangDeviceTypes, gangDimValues, gangDimDeviceTypes,
-      seqDeviceTypes, workerDeviceTypes, vectorDeviceTypes);
+      converter, loc, mod, funcOp, funcName, hasNohost, bindIdNames,
+      bindStrNames, bindIdNameDeviceTypes, bindStrNameDeviceTypes,
+      gangDeviceTypes, gangDimValues, gangDimDeviceTypes, seqDeviceTypes,
+      workerDeviceTypes, vectorDeviceTypes);
 }
 
 static void

diff  --git a/flang/test/Lower/OpenACC/acc-routine.f90 b/flang/test/Lower/OpenACC/acc-routine.f90
index 789f3a57e1f79..1a63b4120235c 100644
--- a/flang/test/Lower/OpenACC/acc-routine.f90
+++ b/flang/test/Lower/OpenACC/acc-routine.f90
@@ -2,13 +2,14 @@
 
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
 
-! CHECK: acc.routine @[[r14:.*]] func(@_QPacc_routine19) bind("_QPacc_routine17" [#acc.device_type<host>], "_QPacc_routine17" [#acc.device_type<default>], "_QPacc_routine16" [#acc.device_type<multicore>])
-! CHECK: acc.routine @[[r13:.*]] func(@_QPacc_routine18) bind("_QPacc_routine17" [#acc.device_type<host>], "_QPacc_routine16" [#acc.device_type<multicore>])
+! CHECK: acc.routine @[[r14:.*]] func(@_QPacc_routine19) bind(@_QPacc_routine17 [#acc.device_type<host>], @_QPacc_routine17
+! [#acc.device_type<default>], @_QPacc_routine16 [#acc.device_type<multicore>])
+! CHECK: acc.routine @[[r13:.*]] func(@_QPacc_routine18) bind(@_QPacc_routine17 [#acc.device_type<host>], @_QPacc_routine16 [#acc.device_type<multicore>])
 ! CHECK: acc.routine @[[r12:.*]] func(@_QPacc_routine17) worker ([#acc.device_type<host>]) vector ([#acc.device_type<multicore>])
 ! CHECK: acc.routine @[[r11:.*]] func(@_QPacc_routine16) gang([#acc.device_type<nvidia>]) seq ([#acc.device_type<host>])
 ! CHECK: acc.routine @[[r10:.*]] func(@_QPacc_routine11) seq
 ! CHECK: acc.routine @[[r09:.*]] func(@_QPacc_routine10) seq
-! CHECK: acc.routine @[[r08:.*]] func(@_QPacc_routine9) bind("_QPacc_routine9a")
+! CHECK: acc.routine @[[r08:.*]] func(@_QPacc_routine9) bind(@_QPacc_routine9a)
 ! CHECK: acc.routine @[[r07:.*]] func(@_QPacc_routine8) bind("routine8_")
 ! CHECK: acc.routine @[[r06:.*]] func(@_QPacc_routine7) gang(dim: 1 : i64)
 ! CHECK: acc.routine @[[r05:.*]] func(@_QPacc_routine6) nohost

diff  --git a/flang/test/Lower/OpenACC/acc-routine03.f90 b/flang/test/Lower/OpenACC/acc-routine03.f90
index 85e4ef580f983..ddd6bda0367e4 100644
--- a/flang/test/Lower/OpenACC/acc-routine03.f90
+++ b/flang/test/Lower/OpenACC/acc-routine03.f90
@@ -30,6 +30,6 @@ subroutine sub2(a)
 end subroutine
 
 ! CHECK: acc.routine @acc_routine_1 func(@_QPsub2) worker nohost
-! CHECK: acc.routine @acc_routine_0 func(@_QPsub1) bind("_QPsub2") worker
+! CHECK: acc.routine @acc_routine_0 func(@_QPsub1) bind(@_QPsub2) worker
 ! CHECK: func.func @_QPsub1(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>}
 ! CHECK: func.func @_QPsub2(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>}

diff  --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index 4eb666239d4e4..8f87235fcd237 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -29,6 +29,7 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include <variant>
 
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/OpenACC/OpenACCOpsTypes.h.inc"

diff  --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 66378f116784e..96b9adcc53b3c 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2772,8 +2772,10 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
   }];
 
   let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
-      OptionalAttr<StrArrayAttr>:$bindName,
-      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<SymbolRefArrayAttr>:$bindIdName,
+      OptionalAttr<StrArrayAttr>:$bindStrName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindIdNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindStrNameDeviceType,
       OptionalAttr<DeviceTypeArrayAttr>:$worker,
       OptionalAttr<DeviceTypeArrayAttr>:$vector,
       OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
@@ -2815,14 +2817,14 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     std::optional<int64_t> getGangDimValue();
     std::optional<int64_t> getGangDimValue(mlir::acc::DeviceType deviceType);
 
-    std::optional<llvm::StringRef> getBindNameValue();
-    std::optional<llvm::StringRef> getBindNameValue(mlir::acc::DeviceType deviceType);
+    std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue();
+    std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue(mlir::acc::DeviceType deviceType);
   }];
 
   let assemblyFormat = [{
     $sym_name `func` `(` $func_name `)`
     oilist (
-        `bind` `(` custom<BindName>($bindName, $bindNameDeviceType) `)`
+        `bind` `(` custom<BindName>($bindIdName, $bindStrName ,$bindIdNameDeviceType, $bindStrNameDeviceType) `)`
       | `gang` `` custom<RoutineGangClause>($gang, $gangDim, $gangDimDeviceType)
       | `worker` custom<DeviceTypeArrayAttr>($worker)
       | `vector` custom<DeviceTypeArrayAttr>($vector)

diff  --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..fbc1f003ab648 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/LogicalResult.h"
+#include <variant>
 
 using namespace mlir;
 using namespace acc;
@@ -3461,40 +3462,88 @@ LogicalResult acc::RoutineOp::verify() {
   return success();
 }
 
-static ParseResult parseBindName(OpAsmParser &parser, mlir::ArrayAttr &bindName,
-                                 mlir::ArrayAttr &deviceTypes) {
-  llvm::SmallVector<mlir::Attribute> bindNameAttrs;
-  llvm::SmallVector<mlir::Attribute> deviceTypeAttrs;
+static ParseResult parseBindName(OpAsmParser &parser,
+                                 mlir::ArrayAttr &bindIdName,
+                                 mlir::ArrayAttr &bindStrName,
+                                 mlir::ArrayAttr &deviceIdTypes,
+                                 mlir::ArrayAttr &deviceStrTypes) {
+  llvm::SmallVector<mlir::Attribute> bindIdNameAttrs;
+  llvm::SmallVector<mlir::Attribute> bindStrNameAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceIdTypeAttrs;
+  llvm::SmallVector<mlir::Attribute> deviceStrTypeAttrs;
 
   if (failed(parser.parseCommaSeparatedList([&]() {
-        if (parser.parseAttribute(bindNameAttrs.emplace_back()))
+        mlir::Attribute newAttr;
+        bool isSymbolRefAttr;
+        auto parseResult = parser.parseAttribute(newAttr);
+        if (auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(newAttr)) {
+          bindIdNameAttrs.push_back(symbolRefAttr);
+          isSymbolRefAttr = true;
+        } else if (auto stringAttr = dyn_cast<mlir::StringAttr>(newAttr)) {
+          bindStrNameAttrs.push_back(stringAttr);
+          isSymbolRefAttr = false;
+        }
+        if (parseResult)
           return failure();
         if (failed(parser.parseOptionalLSquare())) {
-          deviceTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
-              parser.getContext(), mlir::acc::DeviceType::None));
+          if (isSymbolRefAttr) {
+            deviceIdTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          } else {
+            deviceStrTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(
+                parser.getContext(), mlir::acc::DeviceType::None));
+          }
         } else {
-          if (parser.parseAttribute(deviceTypeAttrs.emplace_back()) ||
-              parser.parseRSquare())
-            return failure();
+          if (isSymbolRefAttr) {
+            if (parser.parseAttribute(deviceIdTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          } else {
+            if (parser.parseAttribute(deviceStrTypeAttrs.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          }
         }
         return success();
       })))
     return failure();
 
-  bindName = ArrayAttr::get(parser.getContext(), bindNameAttrs);
-  deviceTypes = ArrayAttr::get(parser.getContext(), deviceTypeAttrs);
+  bindIdName = ArrayAttr::get(parser.getContext(), bindIdNameAttrs);
+  bindStrName = ArrayAttr::get(parser.getContext(), bindStrNameAttrs);
+  deviceIdTypes = ArrayAttr::get(parser.getContext(), deviceIdTypeAttrs);
+  deviceStrTypes = ArrayAttr::get(parser.getContext(), deviceStrTypeAttrs);
 
   return success();
 }
 
 static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,
-                          std::optional<mlir::ArrayAttr> bindName,
-                          std::optional<mlir::ArrayAttr> deviceTypes) {
-  llvm::interleaveComma(llvm::zip(*bindName, *deviceTypes), p,
-                        [&](const auto &pair) {
-                          p << std::get<0>(pair);
-                          printSingleDeviceType(p, std::get<1>(pair));
-                        });
+                          std::optional<mlir::ArrayAttr> bindIdName,
+                          std::optional<mlir::ArrayAttr> bindStrName,
+                          std::optional<mlir::ArrayAttr> deviceIdTypes,
+                          std::optional<mlir::ArrayAttr> deviceStrTypes) {
+  // Create combined vectors for all bind names and device types
+  llvm::SmallVector<mlir::Attribute> allBindNames;
+  llvm::SmallVector<mlir::Attribute> allDeviceTypes;
+
+  // Append bindIdName and deviceIdTypes
+  if (hasDeviceTypeValues(deviceIdTypes)) {
+    allBindNames.append(bindIdName->begin(), bindIdName->end());
+    allDeviceTypes.append(deviceIdTypes->begin(), deviceIdTypes->end());
+  }
+
+  // Append bindStrName and deviceStrTypes
+  if (hasDeviceTypeValues(deviceStrTypes)) {
+    allBindNames.append(bindStrName->begin(), bindStrName->end());
+    allDeviceTypes.append(deviceStrTypes->begin(), deviceStrTypes->end());
+  }
+
+  // Print the combined sequence
+  if (!allBindNames.empty())
+    llvm::interleaveComma(llvm::zip(allBindNames, allDeviceTypes), p,
+                          [&](const auto &pair) {
+                            p << std::get<0>(pair);
+                            printSingleDeviceType(p, std::get<1>(pair));
+                          });
 }
 
 static ParseResult parseRoutineGangClause(OpAsmParser &parser,
@@ -3654,19 +3703,32 @@ bool RoutineOp::hasSeq(mlir::acc::DeviceType deviceType) {
   return hasDeviceType(getSeq(), deviceType);
 }
 
-std::optional<llvm::StringRef> RoutineOp::getBindNameValue() {
+std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
+RoutineOp::getBindNameValue() {
   return getBindNameValue(mlir::acc::DeviceType::None);
 }
 
-std::optional<llvm::StringRef>
+std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
 RoutineOp::getBindNameValue(mlir::acc::DeviceType deviceType) {
-  if (!hasDeviceTypeValues(getBindNameDeviceType()))
+  if (!hasDeviceTypeValues(getBindIdNameDeviceType()) &&
+      !hasDeviceTypeValues(getBindStrNameDeviceType())) {
     return std::nullopt;
-  if (auto pos = findSegment(*getBindNameDeviceType(), deviceType)) {
-    auto attr = (*getBindName())[*pos];
+  }
+
+  if (auto pos = findSegment(*getBindIdNameDeviceType(), deviceType)) {
+    auto attr = (*getBindIdName())[*pos];
+    auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(attr);
+    assert(symbolRefAttr && "expected SymbolRef");
+    return symbolRefAttr;
+  }
+
+  if (auto pos = findSegment(*getBindStrNameDeviceType(), deviceType)) {
+    auto attr = (*getBindStrName())[*pos];
     auto stringAttr = dyn_cast<mlir::StringAttr>(attr);
-    return stringAttr.getValue();
+    assert(stringAttr && "expected String");
+    return stringAttr;
   }
+
   return std::nullopt;
 }
 

diff  --git a/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp b/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
index aa16421cbec51..836efdb307f97 100644
--- a/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
+++ b/mlir/unittests/Dialect/OpenACC/OpenACCOpsTest.cpp
@@ -519,14 +519,44 @@ TEST_F(OpenACCOpsTest, routineOpTest) {
   op->removeGangDimDeviceTypeAttr();
   op->removeGangDimAttr();
 
-  op->setBindNameDeviceTypeAttr(b.getArrayAttr({dtypeNone}));
-  op->setBindNameAttr(b.getArrayAttr({b.getStringAttr("fname")}));
+  op->setBindIdNameDeviceTypeAttr(
+      b.getArrayAttr({DeviceTypeAttr::get(&context, DeviceType::Host)}));
+  op->setBindStrNameDeviceTypeAttr(b.getArrayAttr({dtypeNone}));
+  op->setBindIdNameAttr(
+      b.getArrayAttr({SymbolRefAttr::get(&context, "test_symbol")}));
+  op->setBindStrNameAttr(b.getArrayAttr({b.getStringAttr("fname")}));
   EXPECT_TRUE(op->getBindNameValue().has_value());
-  EXPECT_EQ(op->getBindNameValue().value(), "fname");
-  for (auto d : dtypesWithoutNone)
-    EXPECT_FALSE(op->getBindNameValue(d).has_value());
-  op->removeBindNameDeviceTypeAttr();
-  op->removeBindNameAttr();
+  EXPECT_TRUE(op->getBindNameValue(DeviceType::Host).has_value());
+  EXPECT_EQ(std::visit(
+                [](const auto &attr) -> std::string {
+                  if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                               mlir::StringAttr>) {
+                    return attr.str();
+                  } else {
+                    return attr.getLeafReference().str();
+                  }
+                },
+                op->getBindNameValue().value()),
+            "fname");
+  EXPECT_EQ(std::visit(
+                [](const auto &attr) -> std::string {
+                  if constexpr (std::is_same_v<std::decay_t<decltype(attr)>,
+                                               mlir::StringAttr>) {
+                    return attr.str();
+                  } else {
+                    return attr.getLeafReference().str();
+                  }
+                },
+                op->getBindNameValue(DeviceType::Host).value()),
+            "test_symbol");
+  for (auto d : dtypesWithoutNone) {
+    if (d != DeviceType::Host)
+      EXPECT_FALSE(op->getBindNameValue(d).has_value());
+  }
+  op->removeBindIdNameDeviceTypeAttr();
+  op->removeBindStrNameDeviceTypeAttr();
+  op->removeBindIdNameAttr();
+  op->removeBindStrNameAttr();
 }
 
 template <typename Op>


        

From flang-commits at lists.llvm.org  Thu Jul 17 09:38:09 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Thu, 17 Jul 2025 09:38:09 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [openacc][flang] Support two type
 bindName representation in acc routine (PR #149147)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149147@github.com>
Message-ID: <687926f1.050a0220.232572.28fc@mx.google.com>


https://github.com/razvanlupusoru closed https://github.com/llvm/llvm-project/pull/149147

From flang-commits at lists.llvm.org  Thu Jul 17 10:05:03 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 17 Jul 2025 10:05:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `COSPI` (PR #149343)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149343@github.com>
Message-ID: <68792d3f.050a0220.2fb0cf.52f7@mx.google.com>


clementval wrote:

> I have a minor question: Does flang have a utility similar to `update_test_checks.py`? Editing the test file manually is a bit tedious.

I don't think we have such a tool. Any LLM will be happy to make the check line for you. 


https://github.com/llvm/llvm-project/pull/149343

From flang-commits at lists.llvm.org  Thu Jul 17 10:11:15 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 10:11:15 -0700 (PDT)
Subject: [flang-commits] [flang] ff5784b - [flang][OpenMP] Move
 extractOmpDirective to Utils.cpp, NFC (#148653)
Message-ID: <68792eb3.170a0220.a642e.e62d@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-17T12:11:12-05:00
New Revision: ff5784bb9094f6035851dc7abc4a5760fdc21e45

URL: https://github.com/llvm/llvm-project/commit/ff5784bb9094f6035851dc7abc4a5760fdc21e45
DIFF: https://github.com/llvm/llvm-project/commit/ff5784bb9094f6035851dc7abc4a5760fdc21e45.diff

LOG: [flang][OpenMP] Move extractOmpDirective to Utils.cpp, NFC (#148653)

Added: 
    

Modified: 
    flang/lib/Lower/OpenMP/OpenMP.cpp
    flang/lib/Lower/OpenMP/Utils.cpp
    flang/lib/Lower/OpenMP/Utils.h

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4458f62eea95a..fcb20fdf187ff 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -372,90 +372,6 @@ extractMappedBaseValues(llvm::ArrayRef<mlir::Value> vars,
   });
 }
 
-/// Get the directive enumeration value corresponding to the given OpenMP
-/// construct PFT node.
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
-  return common::visit(
-      common::visitors{
-          [](const parser::OpenMPAllocatorsConstruct &c) {
-            return llvm::omp::OMPD_allocators;
-          },
-          [](const parser::OpenMPAssumeConstruct &c) {
-            return llvm::omp::OMPD_assume;
-          },
-          [](const parser::OpenMPAtomicConstruct &c) {
-            return llvm::omp::OMPD_atomic;
-          },
-          [](const parser::OpenMPBlockConstruct &c) {
-            return std::get<parser::OmpBlockDirective>(
-                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPCriticalConstruct &c) {
-            return llvm::omp::OMPD_critical;
-          },
-          [](const parser::OpenMPDeclarativeAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPDispatchConstruct &c) {
-            return llvm::omp::OMPD_dispatch;
-          },
-          [](const parser::OpenMPExecutableAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPLoopConstruct &c) {
-            return std::get<parser::OmpLoopDirective>(
-                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPSectionConstruct &c) {
-            return llvm::omp::OMPD_section;
-          },
-          [](const parser::OpenMPSectionsConstruct &c) {
-            return std::get<parser::OmpSectionsDirective>(
-                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPStandaloneConstruct &c) {
-            return common::visit(
-                common::visitors{
-                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
-                      return c.v.DirId();
-                    },
-                    [](const parser::OpenMPFlushConstruct &c) {
-                      return llvm::omp::OMPD_flush;
-                    },
-                    [](const parser::OpenMPCancelConstruct &c) {
-                      return llvm::omp::OMPD_cancel;
-                    },
-                    [](const parser::OpenMPCancellationPointConstruct &c) {
-                      return llvm::omp::OMPD_cancellation_point;
-                    },
-                    [](const parser::OmpMetadirectiveDirective &c) {
-                      return llvm::omp::OMPD_metadirective;
-                    },
-                    [](const parser::OpenMPDepobjConstruct &c) {
-                      return llvm::omp::OMPD_depobj;
-                    },
-                    [](const parser::OpenMPInteropConstruct &c) {
-                      return llvm::omp::OMPD_interop;
-                    }},
-                c.u);
-          },
-          [](const parser::OpenMPUtilityConstruct &c) {
-            return common::visit(
-                common::visitors{[](const parser::OmpErrorDirective &c) {
-                                   return llvm::omp::OMPD_error;
-                                 },
-                                 [](const parser::OmpNothingDirective &c) {
-                                   return llvm::omp::OMPD_nothing;
-                                 }},
-                c.u);
-          }},
-      ompConstruct.u);
-}
-
 /// Populate the global \see hostEvalInfo after processing clauses for the given
 /// \p eval OpenMP target construct, or nested constructs, if these must be
 /// evaluated outside of the target region per the spec.

diff  --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 2e53f01f1da6a..b194150c0f7f0 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -661,6 +661,90 @@ bool collectLoopRelatedInfo(
 
   return found;
 }
+
+/// Get the directive enumeration value corresponding to the given OpenMP
+/// construct PFT node.
+llvm::omp::Directive
+extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
+  return common::visit(
+      common::visitors{
+          [](const parser::OpenMPAllocatorsConstruct &c) {
+            return llvm::omp::OMPD_allocators;
+          },
+          [](const parser::OpenMPAssumeConstruct &c) {
+            return llvm::omp::OMPD_assume;
+          },
+          [](const parser::OpenMPAtomicConstruct &c) {
+            return llvm::omp::OMPD_atomic;
+          },
+          [](const parser::OpenMPBlockConstruct &c) {
+            return std::get<parser::OmpBlockDirective>(
+                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPCriticalConstruct &c) {
+            return llvm::omp::OMPD_critical;
+          },
+          [](const parser::OpenMPDeclarativeAllocate &c) {
+            return llvm::omp::OMPD_allocate;
+          },
+          [](const parser::OpenMPDispatchConstruct &c) {
+            return llvm::omp::OMPD_dispatch;
+          },
+          [](const parser::OpenMPExecutableAllocate &c) {
+            return llvm::omp::OMPD_allocate;
+          },
+          [](const parser::OpenMPLoopConstruct &c) {
+            return std::get<parser::OmpLoopDirective>(
+                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPSectionConstruct &c) {
+            return llvm::omp::OMPD_section;
+          },
+          [](const parser::OpenMPSectionsConstruct &c) {
+            return std::get<parser::OmpSectionsDirective>(
+                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPStandaloneConstruct &c) {
+            return common::visit(
+                common::visitors{
+                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
+                      return c.v.DirId();
+                    },
+                    [](const parser::OpenMPFlushConstruct &c) {
+                      return llvm::omp::OMPD_flush;
+                    },
+                    [](const parser::OpenMPCancelConstruct &c) {
+                      return llvm::omp::OMPD_cancel;
+                    },
+                    [](const parser::OpenMPCancellationPointConstruct &c) {
+                      return llvm::omp::OMPD_cancellation_point;
+                    },
+                    [](const parser::OmpMetadirectiveDirective &c) {
+                      return llvm::omp::OMPD_metadirective;
+                    },
+                    [](const parser::OpenMPDepobjConstruct &c) {
+                      return llvm::omp::OMPD_depobj;
+                    },
+                    [](const parser::OpenMPInteropConstruct &c) {
+                      return llvm::omp::OMPD_interop;
+                    }},
+                c.u);
+          },
+          [](const parser::OpenMPUtilityConstruct &c) {
+            return common::visit(
+                common::visitors{[](const parser::OmpErrorDirective &c) {
+                                   return llvm::omp::OMPD_error;
+                                 },
+                                 [](const parser::OmpNothingDirective &c) {
+                                   return llvm::omp::OMPD_nothing;
+                                 }},
+                c.u);
+          }},
+      ompConstruct.u);
+}
 } // namespace omp
 } // namespace lower
 } // namespace Fortran

diff  --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 1526bd4e90233..8e3ad5c3452e2 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -166,6 +166,9 @@ bool collectLoopRelatedInfo(
     lower::pft::Evaluation &eval, const omp::List<omp::Clause> &clauses,
     mlir::omp::LoopRelatedClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &iv);
+
+llvm::omp::Directive
+extractOmpDirective(const parser::OpenMPConstruct &ompConstruct);
 } // namespace omp
 } // namespace lower
 } // namespace Fortran


        

From flang-commits at lists.llvm.org  Thu Jul 17 10:11:19 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 17 Jul 2025 10:11:19 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Move
 extractOmpDirective to Utils.cpp, NFC (PR #148653)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148653@github.com>
Message-ID: <68792eb7.050a0220.153dad.51f1@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/148653

From flang-commits at lists.llvm.org  Thu Jul 17 10:11:25 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 17 Jul 2025 10:11:25 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Generalize
 isOpenMPPrivatizingConstruct (PR #148654)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148654@github.com>
Message-ID: <68792ebd.170a0220.3c0b67.a214@mx.google.com>


https://github.com/kparzysz edited https://github.com/llvm/llvm-project/pull/148654

From flang-commits at lists.llvm.org  Thu Jul 17 10:12:51 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 17 Jul 2025 10:12:51 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Generalize
 isOpenMPPrivatizingConstruct (PR #148654)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148654@github.com>
Message-ID: <68792f13.170a0220.ad8e2.eac6@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/148654

>From 76b1c6594d1e73f7d4a7011db3247a0c37b51339 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 07:45:34 -0500
Subject: [PATCH 1/4] [Frontend][OpenMP] Move isPrivatizingClause to OMP.h, NFC

---
 .../Frontend/OpenMP/ConstructDecompositionT.h  | 18 +-----------------
 llvm/include/llvm/Frontend/OpenMP/OMP.h        | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
index cdc80c88b7425..611bfe3f8aced 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -795,25 +795,9 @@ bool ConstructDecompositionT<C, H>::applyClause(
   // assigned to which leaf constructs.
 
   // [5.2:340:33]
-  auto canMakePrivateCopy = [](llvm::omp::Clause id) {
-    switch (id) {
-    // Clauses with "privatization" property:
-    case llvm::omp::Clause::OMPC_firstprivate:
-    case llvm::omp::Clause::OMPC_in_reduction:
-    case llvm::omp::Clause::OMPC_lastprivate:
-    case llvm::omp::Clause::OMPC_linear:
-    case llvm::omp::Clause::OMPC_private:
-    case llvm::omp::Clause::OMPC_reduction:
-    case llvm::omp::Clause::OMPC_task_reduction:
-      return true;
-    default:
-      return false;
-    }
-  };
-
   bool applied = applyIf(node, [&](const auto &leaf) {
     return llvm::any_of(leaf.clauses, [&](const ClauseTy *n) {
-      return canMakePrivateCopy(n->id);
+      return llvm::omp::isPrivatizingClause(n->id);
     });
   });
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index 35dafc6d246f0..d44c33301bde7 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -48,6 +48,22 @@ static constexpr inline bool canHaveIterator(Clause C) {
   }
 }
 
+// Can clause C create a private copy of a variable.
+static constexpr inline bool isPrivatizingClause(Clause C) {
+  switch (C) {
+  case OMPC_firstprivate:
+  case OMPC_in_reduction:
+  case OMPC_lastprivate:
+  case OMPC_linear:
+  case OMPC_private:
+  case OMPC_reduction:
+  case OMPC_task_reduction:
+    return true;
+  default:
+    return false;
+  }
+}
+
 static constexpr unsigned FallbackVersion = 52;
 LLVM_ABI ArrayRef<unsigned> getOpenMPVersions();
 

>From 82829a6f51fd96f720a64255442d96133a09b3dc Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 13:28:02 -0500
Subject: [PATCH 2/4] [utils][TableGen] Make some non-bitmask enums iterable

Additionally, add sentinel values <Enum>::First_ and <Enum>::Last_
to each one of those enums.

This will allow using `enum_seq_inclusive` to generate the list of
enum-typed values of any generated scoped (non-bitmask) enum.
---
 llvm/test/TableGen/directive1.td              | 25 +++++++++++++++++++
 llvm/test/TableGen/directive2.td              | 25 +++++++++++++++++++
 .../OpenMPDirectiveNameParserTest.cpp         | 21 +++++-----------
 .../utils/TableGen/Basic/DirectiveEmitter.cpp | 20 +++++++++++++--
 4 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/llvm/test/TableGen/directive1.td b/llvm/test/TableGen/directive1.td
index 1d2bd51204e4f..3eda077eeabf7 100644
--- a/llvm/test/TableGen/directive1.td
+++ b/llvm/test/TableGen/directive1.td
@@ -53,6 +53,7 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  #include "llvm/ADT/ArrayRef.h"
 // CHECK-NEXT:  #include "llvm/ADT/BitmaskEnum.h"
+// CHECK-NEXT:  #include "llvm/ADT/Sequence.h"
 // CHECK-NEXT:  #include "llvm/ADT/StringRef.h"
 // CHECK-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // CHECK-NEXT:  #include "llvm/Support/Compiler.h"
@@ -66,22 +67,26 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Association {
 // CHECK-NEXT:    Block,
+// CHECK-NEXT:    First_ = Block,
 // CHECK-NEXT:    Declaration,
 // CHECK-NEXT:    Delimited,
 // CHECK-NEXT:    Loop,
 // CHECK-NEXT:    None,
 // CHECK-NEXT:    Separating,
+// CHECK-NEXT:    Last_ = Separating,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Association_enumSize = 6;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Category {
 // CHECK-NEXT:    Declarative,
+// CHECK-NEXT:    First_ = Declarative,
 // CHECK-NEXT:    Executable,
 // CHECK-NEXT:    Informational,
 // CHECK-NEXT:    Meta,
 // CHECK-NEXT:    Subsidiary,
 // CHECK-NEXT:    Utility,
+// CHECK-NEXT:    Last_ = Utility,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Category_enumSize = 6;
@@ -96,6 +101,8 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Directive {
 // CHECK-NEXT:    TDLD_dira,
+// CHECK-NEXT:    First_ = TDLD_dira,
+// CHECK-NEXT:    Last_ = TDLD_dira,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Directive_enumSize = 1;
@@ -104,8 +111,10 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Clause {
 // CHECK-NEXT:    TDLC_clausea,
+// CHECK-NEXT:    First_ = TDLC_clausea,
 // CHECK-NEXT:    TDLC_clauseb,
 // CHECK-NEXT:    TDLC_clausec,
+// CHECK-NEXT:    Last_ = TDLC_clausec,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 3;
@@ -151,6 +160,22 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  LLVM_ABI StringRef getTdlAKindName(AKind x);
 // CHECK-EMPTY:
 // CHECK-NEXT:  } // namespace tdl
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Association> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Category> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Directive> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Clause> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
 // CHECK-NEXT:  } // namespace llvm
 // CHECK-NEXT:  #endif // LLVM_Tdl_INC
 
diff --git a/llvm/test/TableGen/directive2.td b/llvm/test/TableGen/directive2.td
index 3a64bb3900a31..a25197c3efd93 100644
--- a/llvm/test/TableGen/directive2.td
+++ b/llvm/test/TableGen/directive2.td
@@ -46,6 +46,7 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  #define LLVM_Tdl_INC
 // CHECK-EMPTY:
 // CHECK-NEXT:  #include "llvm/ADT/ArrayRef.h"
+// CHECK-NEXT:  #include "llvm/ADT/Sequence.h"
 // CHECK-NEXT:  #include "llvm/ADT/StringRef.h"
 // CHECK-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // CHECK-NEXT:  #include "llvm/Support/Compiler.h"
@@ -57,22 +58,26 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Association {
 // CHECK-NEXT:    Block,
+// CHECK-NEXT:    First_ = Block,
 // CHECK-NEXT:    Declaration,
 // CHECK-NEXT:    Delimited,
 // CHECK-NEXT:    Loop,
 // CHECK-NEXT:    None,
 // CHECK-NEXT:    Separating,
+// CHECK-NEXT:    Last_ = Separating,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Association_enumSize = 6;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Category {
 // CHECK-NEXT:    Declarative,
+// CHECK-NEXT:    First_ = Declarative,
 // CHECK-NEXT:    Executable,
 // CHECK-NEXT:    Informational,
 // CHECK-NEXT:    Meta,
 // CHECK-NEXT:    Subsidiary,
 // CHECK-NEXT:    Utility,
+// CHECK-NEXT:    Last_ = Utility,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Category_enumSize = 6;
@@ -87,15 +92,19 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Directive {
 // CHECK-NEXT:    TDLD_dira,
+// CHECK-NEXT:    First_ = TDLD_dira,
+// CHECK-NEXT:    Last_ = TDLD_dira,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Directive_enumSize = 1;
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Clause {
 // CHECK-NEXT:    TDLC_clausea,
+// CHECK-NEXT:    First_ = TDLC_clausea,
 // CHECK-NEXT:    TDLC_clauseb,
 // CHECK-NEXT:    TDLC_clausec,
 // CHECK-NEXT:    TDLC_claused,
+// CHECK-NEXT:    Last_ = TDLC_claused,
 // CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 4;
@@ -124,6 +133,22 @@ def TDL_DirA : Directive<[Spelling<"dira">]> {
 // CHECK-NEXT:  LLVM_ABI Category getDirectiveCategory(Directive D);
 // CHECK-NEXT:  LLVM_ABI SourceLanguage getDirectiveLanguages(Directive D);
 // CHECK-NEXT:  } // namespace tdl
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Association> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Category> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Directive> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  template <> struct enum_iteration_traits<tdl::Clause> {
+// CHECK-NEXT:    static constexpr bool is_iterable = true;
+// CHECK-NEXT:  };
 // CHECK-NEXT:  } // namespace llvm
 // CHECK-NEXT:  #endif // LLVM_Tdl_INC
 
diff --git a/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
index 0363a08cc0f03..10329820bef76 100644
--- a/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
@@ -48,12 +48,6 @@ static std::string &prepareParamName(std::string &Name) {
   return Name;
 }
 
-namespace llvm {
-template <> struct enum_iteration_traits<omp::Directive> {
-  static constexpr bool is_iterable = true;
-};
-} // namespace llvm
-
 // Test tokenizing.
 
 class Tokenize : public testing::TestWithParam<omp::Directive> {};
@@ -87,12 +81,10 @@ getParamName1(const testing::TestParamInfo<Tokenize::ParamType> &Info) {
   return prepareParamName(Name);
 }
 
-INSTANTIATE_TEST_SUITE_P(
-    DirectiveNameParserTest, Tokenize,
-    testing::ValuesIn(
-        llvm::enum_seq(static_cast<omp::Directive>(0),
-                       static_cast<omp::Directive>(omp::Directive_enumSize))),
-    getParamName1);
+INSTANTIATE_TEST_SUITE_P(DirectiveNameParserTest, Tokenize,
+                         testing::ValuesIn(llvm::enum_seq_inclusive(
+                             omp::Directive::First_, omp::Directive::Last_)),
+                         getParamName1);
 
 // Test parsing of valid names.
 
@@ -131,9 +123,8 @@ getParamName2(const testing::TestParamInfo<ParseValid::ParamType> &Info) {
 
 INSTANTIATE_TEST_SUITE_P(
     DirectiveNameParserTest, ParseValid,
-    testing::Combine(testing::ValuesIn(llvm::enum_seq(
-                         static_cast<omp::Directive>(0),
-                         static_cast<omp::Directive>(omp::Directive_enumSize))),
+    testing::Combine(testing::ValuesIn(llvm::enum_seq_inclusive(
+                         omp::Directive::First_, omp::Directive::Last_)),
                      testing::ValuesIn(omp::getOpenMPVersions())),
     getParamName2);
 
diff --git a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
index 177eecebce9a5..b48b9f6b7c41c 100644
--- a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
@@ -106,9 +106,15 @@ static void generateEnumClass(ArrayRef<const Record *> Records, raw_ostream &OS,
                               bool ExportEnums) {
   OS << "\n";
   OS << "enum class " << Enum << " {\n";
-  for (const Record *R : Records) {
-    OS << "  " << getIdentifierName(R, Prefix) << ",\n";
+  std::string N;
+  for (auto [I, R] : llvm::enumerate(Records)) {
+    N = getIdentifierName(R, Prefix);
+    OS << "  " << N << ",\n";
+    // Make the sentinel names less likely to conflict with actual names...
+    if (I == 0)
+      OS << "  First_ = " << N << ",\n";
   }
+  OS << "  Last_ = " << N << ",\n";
   OS << "};\n";
   OS << "\n";
   OS << "static constexpr std::size_t " << Enum
@@ -282,6 +288,7 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   if (DirLang.hasEnableBitmaskEnumInNamespace())
     OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n";
 
+  OS << "#include \"llvm/ADT/Sequence.h\"\n";
   OS << "#include \"llvm/ADT/StringRef.h\"\n";
   OS << "#include \"llvm/Frontend/Directive/Spelling.h\"\n";
   OS << "#include \"llvm/Support/Compiler.h\"\n";
@@ -375,6 +382,15 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   for (auto Ns : reverse(Namespaces))
     OS << "} // namespace " << Ns << "\n";
 
+  // These specializations need to be in ::llvm.
+  for (StringRef Enum : {"Association", "Category", "Directive", "Clause"}) {
+    OS << "\n";
+    OS << "template <> struct enum_iteration_traits<"
+       << DirLang.getCppNamespace() << "::" << Enum << "> {\n";
+    OS << "  static constexpr bool is_iterable = true;\n";
+    OS << "};\n";
+  }
+
   OS << "} // namespace llvm\n";
 
   OS << "#endif // LLVM_" << Lang << "_INC\n";

>From d3ed6f948468c4d27c5ac9999436612e650f9460 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 14 Jul 2025 10:18:07 -0500
Subject: [PATCH 3/4] [flang][OpenMP] Move extractOmpDirective to Utils.cpp,
 NFC

---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 84 -------------------------------
 flang/lib/Lower/OpenMP/Utils.cpp  | 84 +++++++++++++++++++++++++++++++
 flang/lib/Lower/OpenMP/Utils.h    |  3 ++
 3 files changed, 87 insertions(+), 84 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4458f62eea95a..fcb20fdf187ff 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -372,90 +372,6 @@ extractMappedBaseValues(llvm::ArrayRef<mlir::Value> vars,
   });
 }
 
-/// Get the directive enumeration value corresponding to the given OpenMP
-/// construct PFT node.
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
-  return common::visit(
-      common::visitors{
-          [](const parser::OpenMPAllocatorsConstruct &c) {
-            return llvm::omp::OMPD_allocators;
-          },
-          [](const parser::OpenMPAssumeConstruct &c) {
-            return llvm::omp::OMPD_assume;
-          },
-          [](const parser::OpenMPAtomicConstruct &c) {
-            return llvm::omp::OMPD_atomic;
-          },
-          [](const parser::OpenMPBlockConstruct &c) {
-            return std::get<parser::OmpBlockDirective>(
-                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPCriticalConstruct &c) {
-            return llvm::omp::OMPD_critical;
-          },
-          [](const parser::OpenMPDeclarativeAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPDispatchConstruct &c) {
-            return llvm::omp::OMPD_dispatch;
-          },
-          [](const parser::OpenMPExecutableAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPLoopConstruct &c) {
-            return std::get<parser::OmpLoopDirective>(
-                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPSectionConstruct &c) {
-            return llvm::omp::OMPD_section;
-          },
-          [](const parser::OpenMPSectionsConstruct &c) {
-            return std::get<parser::OmpSectionsDirective>(
-                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPStandaloneConstruct &c) {
-            return common::visit(
-                common::visitors{
-                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
-                      return c.v.DirId();
-                    },
-                    [](const parser::OpenMPFlushConstruct &c) {
-                      return llvm::omp::OMPD_flush;
-                    },
-                    [](const parser::OpenMPCancelConstruct &c) {
-                      return llvm::omp::OMPD_cancel;
-                    },
-                    [](const parser::OpenMPCancellationPointConstruct &c) {
-                      return llvm::omp::OMPD_cancellation_point;
-                    },
-                    [](const parser::OmpMetadirectiveDirective &c) {
-                      return llvm::omp::OMPD_metadirective;
-                    },
-                    [](const parser::OpenMPDepobjConstruct &c) {
-                      return llvm::omp::OMPD_depobj;
-                    },
-                    [](const parser::OpenMPInteropConstruct &c) {
-                      return llvm::omp::OMPD_interop;
-                    }},
-                c.u);
-          },
-          [](const parser::OpenMPUtilityConstruct &c) {
-            return common::visit(
-                common::visitors{[](const parser::OmpErrorDirective &c) {
-                                   return llvm::omp::OMPD_error;
-                                 },
-                                 [](const parser::OmpNothingDirective &c) {
-                                   return llvm::omp::OMPD_nothing;
-                                 }},
-                c.u);
-          }},
-      ompConstruct.u);
-}
-
 /// Populate the global \see hostEvalInfo after processing clauses for the given
 /// \p eval OpenMP target construct, or nested constructs, if these must be
 /// evaluated outside of the target region per the spec.
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 2e53f01f1da6a..b194150c0f7f0 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -661,6 +661,90 @@ bool collectLoopRelatedInfo(
 
   return found;
 }
+
+/// Get the directive enumeration value corresponding to the given OpenMP
+/// construct PFT node.
+llvm::omp::Directive
+extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
+  return common::visit(
+      common::visitors{
+          [](const parser::OpenMPAllocatorsConstruct &c) {
+            return llvm::omp::OMPD_allocators;
+          },
+          [](const parser::OpenMPAssumeConstruct &c) {
+            return llvm::omp::OMPD_assume;
+          },
+          [](const parser::OpenMPAtomicConstruct &c) {
+            return llvm::omp::OMPD_atomic;
+          },
+          [](const parser::OpenMPBlockConstruct &c) {
+            return std::get<parser::OmpBlockDirective>(
+                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPCriticalConstruct &c) {
+            return llvm::omp::OMPD_critical;
+          },
+          [](const parser::OpenMPDeclarativeAllocate &c) {
+            return llvm::omp::OMPD_allocate;
+          },
+          [](const parser::OpenMPDispatchConstruct &c) {
+            return llvm::omp::OMPD_dispatch;
+          },
+          [](const parser::OpenMPExecutableAllocate &c) {
+            return llvm::omp::OMPD_allocate;
+          },
+          [](const parser::OpenMPLoopConstruct &c) {
+            return std::get<parser::OmpLoopDirective>(
+                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPSectionConstruct &c) {
+            return llvm::omp::OMPD_section;
+          },
+          [](const parser::OpenMPSectionsConstruct &c) {
+            return std::get<parser::OmpSectionsDirective>(
+                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
+                .v;
+          },
+          [](const parser::OpenMPStandaloneConstruct &c) {
+            return common::visit(
+                common::visitors{
+                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
+                      return c.v.DirId();
+                    },
+                    [](const parser::OpenMPFlushConstruct &c) {
+                      return llvm::omp::OMPD_flush;
+                    },
+                    [](const parser::OpenMPCancelConstruct &c) {
+                      return llvm::omp::OMPD_cancel;
+                    },
+                    [](const parser::OpenMPCancellationPointConstruct &c) {
+                      return llvm::omp::OMPD_cancellation_point;
+                    },
+                    [](const parser::OmpMetadirectiveDirective &c) {
+                      return llvm::omp::OMPD_metadirective;
+                    },
+                    [](const parser::OpenMPDepobjConstruct &c) {
+                      return llvm::omp::OMPD_depobj;
+                    },
+                    [](const parser::OpenMPInteropConstruct &c) {
+                      return llvm::omp::OMPD_interop;
+                    }},
+                c.u);
+          },
+          [](const parser::OpenMPUtilityConstruct &c) {
+            return common::visit(
+                common::visitors{[](const parser::OmpErrorDirective &c) {
+                                   return llvm::omp::OMPD_error;
+                                 },
+                                 [](const parser::OmpNothingDirective &c) {
+                                   return llvm::omp::OMPD_nothing;
+                                 }},
+                c.u);
+          }},
+      ompConstruct.u);
+}
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 1526bd4e90233..8e3ad5c3452e2 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -166,6 +166,9 @@ bool collectLoopRelatedInfo(
     lower::pft::Evaluation &eval, const omp::List<omp::Clause> &clauses,
     mlir::omp::LoopRelatedClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &iv);
+
+llvm::omp::Directive
+extractOmpDirective(const parser::OpenMPConstruct &ompConstruct);
 } // namespace omp
 } // namespace lower
 } // namespace Fortran

>From 4f58a01c96812fdb8086a9c9ad35f260451af8dc Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Fri, 11 Jul 2025 08:21:12 -0500
Subject: [PATCH 4/4] [flang][OpenMP] Generalize isOpenMPPrivatizingConstruct

Instead of treating all block and all loop constructs as privatizing,
actually check if the construct allows a privatizing clause.
---
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp | 57 +++++++++++++++----
 flang/lib/Lower/OpenMP/DataSharingProcessor.h | 12 +++-
 flang/test/Lower/OpenMP/taskgroup02.f90       |  5 +-
 llvm/include/llvm/Frontend/OpenMP/OMP.h       |  4 ++
 4 files changed, 61 insertions(+), 17 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 3fae3f3a0ddfd..675a58e4f35a1 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -26,6 +26,8 @@
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Semantics/attr.h"
 #include "flang/Semantics/tools.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallSet.h"
 
 namespace Fortran {
 namespace lower {
@@ -49,7 +51,7 @@ DataSharingProcessor::DataSharingProcessor(
       firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
       shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols),
       useDelayedPrivatization(useDelayedPrivatization), symTable(symTable),
-      visitor() {
+      visitor(semaCtx) {
   eval.visit([&](const auto &functionParserNode) {
     parser::Walk(functionParserNode, visitor);
   });
@@ -424,24 +426,55 @@ getSource(const semantics::SemanticsContext &semaCtx,
   return source;
 }
 
+static void collectPrivatizingConstructs(
+    llvm::SmallSet<llvm::omp::Directive, 16> &constructs, unsigned version) {
+  using Clause = llvm::omp::Clause;
+  using Directive = llvm::omp::Directive;
+
+  static const Clause privatizingClauses[] = {
+      Clause::OMPC_private,
+      Clause::OMPC_lastprivate,
+      Clause::OMPC_firstprivate,
+      Clause::OMPC_in_reduction,
+      Clause::OMPC_reduction,
+      Clause::OMPC_linear,
+      // TODO: Clause::OMPC_induction,
+      Clause::OMPC_task_reduction,
+      Clause::OMPC_detach,
+      Clause::OMPC_use_device_ptr,
+      Clause::OMPC_is_device_ptr,
+  };
+
+  for (auto dir : llvm::enum_seq_inclusive<Directive>(Directive::First_,
+                                                      Directive::Last_)) {
+    bool allowsPrivatizing = llvm::any_of(privatizingClauses, [&](Clause cls) {
+      return llvm::omp::isAllowedClauseForDirective(dir, cls, version);
+    });
+    if (allowsPrivatizing)
+      constructs.insert(dir);
+  }
+}
+
 bool DataSharingProcessor::isOpenMPPrivatizingConstruct(
-    const parser::OpenMPConstruct &omp) {
-  return common::visit(
-      [](auto &&s) {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        return std::is_same_v<BareS, parser::OpenMPBlockConstruct> ||
-               std::is_same_v<BareS, parser::OpenMPLoopConstruct> ||
-               std::is_same_v<BareS, parser::OpenMPSectionsConstruct>;
-      },
-      omp.u);
+    const parser::OpenMPConstruct &omp, unsigned version) {
+  static llvm::SmallSet<llvm::omp::Directive, 16> privatizing;
+  [[maybe_unused]] static bool init =
+      (collectPrivatizingConstructs(privatizing, version), true);
+
+  // As of OpenMP 6.0, privatizing constructs (with the test being if they
+  // allow a privatizing clause) are: dispatch, distribute, do, for, loop,
+  // parallel, scope, sections, simd, single, target, target_data, task,
+  // taskgroup, taskloop, and teams.
+  return llvm::is_contained(privatizing, extractOmpDirective(omp));
 }
 
 bool DataSharingProcessor::isOpenMPPrivatizingEvaluation(
     const pft::Evaluation &eval) const {
-  return eval.visit([](auto &&s) {
+  unsigned version = semaCtx.langOptions().OpenMPVersion;
+  return eval.visit([=](auto &&s) {
     using BareS = llvm::remove_cvref_t<decltype(s)>;
     if constexpr (std::is_same_v<BareS, parser::OpenMPConstruct>) {
-      return isOpenMPPrivatizingConstruct(s);
+      return isOpenMPPrivatizingConstruct(s, version);
     } else {
       return false;
     }
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index ee2fc70d2e673..bc422f410403a 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -36,6 +36,8 @@ class DataSharingProcessor {
   /// at any point in time. This is used to track Symbol definition scopes in
   /// order to tell which OMP scope defined vs. references a certain Symbol.
   struct OMPConstructSymbolVisitor {
+    OMPConstructSymbolVisitor(semantics::SemanticsContext &ctx)
+        : version(ctx.langOptions().OpenMPVersion) {}
     template <typename T>
     bool Pre(const T &) {
       return true;
@@ -45,13 +47,13 @@ class DataSharingProcessor {
 
     bool Pre(const parser::OpenMPConstruct &omp) {
       // Skip constructs that may not have privatizations.
-      if (isOpenMPPrivatizingConstruct(omp))
+      if (isOpenMPPrivatizingConstruct(omp, version))
         constructs.push_back(&omp);
       return true;
     }
 
     void Post(const parser::OpenMPConstruct &omp) {
-      if (isOpenMPPrivatizingConstruct(omp))
+      if (isOpenMPPrivatizingConstruct(omp, version))
         constructs.pop_back();
     }
 
@@ -68,6 +70,9 @@ class DataSharingProcessor {
     /// construct that defines symbol.
     bool isSymbolDefineBy(const semantics::Symbol *symbol,
                           lower::pft::Evaluation &eval) const;
+
+  private:
+    unsigned version;
   };
 
   mlir::OpBuilder::InsertPoint lastPrivIP;
@@ -115,7 +120,8 @@ class DataSharingProcessor {
                              mlir::OpBuilder::InsertPoint *lastPrivIP);
   void insertDeallocs();
 
-  static bool isOpenMPPrivatizingConstruct(const parser::OpenMPConstruct &omp);
+  static bool isOpenMPPrivatizingConstruct(const parser::OpenMPConstruct &omp,
+                                           unsigned version);
   bool isOpenMPPrivatizingEvaluation(const pft::Evaluation &eval) const;
 
 public:
diff --git a/flang/test/Lower/OpenMP/taskgroup02.f90 b/flang/test/Lower/OpenMP/taskgroup02.f90
index 1e996a030c23a..4c470b7aa82d1 100644
--- a/flang/test/Lower/OpenMP/taskgroup02.f90
+++ b/flang/test/Lower/OpenMP/taskgroup02.f90
@@ -3,8 +3,9 @@
 ! Check that variables are not privatized twice when TASKGROUP is used.
 
 !CHECK-LABEL: func.func @_QPsub() {
-!CHECK:         omp.parallel {
-!CHECK:           %[[PAR_I:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsubEi"}
+!CHECK:         omp.parallel private(@_QFsubEi_private_i32 %[[SUB_I:.*]]#0 -> %[[ARG:.*]] : !fir.ref<i32>)
+!CHECK:           %[[ALLOCA:.*]] = fir.alloca i32
+!CHECK:           %[[PAR_I:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFsubEi"}
 !CHECK:           omp.master {
 !CHECK:             omp.taskgroup {
 !CHECK-NEXT:          omp.task private(@_QFsubEi_firstprivate_i32 %[[PAR_I]]#0 -> %[[TASK_I:.*]] : !fir.ref<i32>) {
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index d44c33301bde7..9d0a55432e1ae 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -51,13 +51,17 @@ static constexpr inline bool canHaveIterator(Clause C) {
 // Can clause C create a private copy of a variable.
 static constexpr inline bool isPrivatizingClause(Clause C) {
   switch (C) {
+  case OMPC_detach:
   case OMPC_firstprivate:
+  // TODO case OMPC_induction:
   case OMPC_in_reduction:
+  case OMPC_is_device_ptr:
   case OMPC_lastprivate:
   case OMPC_linear:
   case OMPC_private:
   case OMPC_reduction:
   case OMPC_task_reduction:
+  case OMPC_use_device_ptr:
     return true;
   default:
     return false;


From flang-commits at lists.llvm.org  Thu Jul 17 10:39:28 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 10:39:28 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <68793550.170a0220.115a2a.e126@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 1/5] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 2/5] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 3/5] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 4/5] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 5/5] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"


From flang-commits at lists.llvm.org  Thu Jul 17 11:17:31 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Thu, 17 Jul 2025 11:17:31 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Mangle main program's symbol name
 to make it distinct from the other symbols (PR #148714)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148714@github.com>
Message-ID: <68793e3b.170a0220.1b0533.f7fa@mx.google.com>


https://github.com/eugeneepshteyn closed https://github.com/llvm/llvm-project/pull/148714

From flang-commits at lists.llvm.org  Thu Jul 17 11:17:32 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Thu, 17 Jul 2025 11:17:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Mangle main program's symbol name
 to make it distinct from the other symbols (PR #148714)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148714@github.com>
Message-ID: <68793e3c.050a0220.1b247f.54e7@mx.google.com>


eugeneepshteyn wrote:

Superseded by https://github.com/llvm/llvm-project/pull/149169

https://github.com/llvm/llvm-project/pull/148714

From flang-commits at lists.llvm.org  Thu Jul 17 11:18:32 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Thu, 17 Jul 2025 11:18:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <68793e78.170a0220.17a8c0.ff4a@mx.google.com>


https://github.com/eugeneepshteyn closed https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Thu Jul 17 11:41:07 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 11:41:07 -0700 (PDT)
Subject: [flang-commits] [flang] 73d4cea - [flang][OpenMP] Generalize
 isOpenMPPrivatizingConstruct (#148654)
Message-ID: <687943c3.170a0220.2e825d.f6ed@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-17T13:41:04-05:00
New Revision: 73d4cea68cce998b1349e3820dc5d80e1096b015

URL: https://github.com/llvm/llvm-project/commit/73d4cea68cce998b1349e3820dc5d80e1096b015
DIFF: https://github.com/llvm/llvm-project/commit/73d4cea68cce998b1349e3820dc5d80e1096b015.diff

LOG: [flang][OpenMP] Generalize isOpenMPPrivatizingConstruct (#148654)

Instead of treating all block and all loop constructs as privatizing,
actually check if the construct allows a privatizing clause.

Added: 
    

Modified: 
    flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
    flang/lib/Lower/OpenMP/DataSharingProcessor.h
    flang/test/Lower/OpenMP/taskgroup02.f90
    llvm/include/llvm/Frontend/OpenMP/OMP.h

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 3fae3f3a0ddfd..675a58e4f35a1 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -26,6 +26,8 @@
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Semantics/attr.h"
 #include "flang/Semantics/tools.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallSet.h"
 
 namespace Fortran {
 namespace lower {
@@ -49,7 +51,7 @@ DataSharingProcessor::DataSharingProcessor(
       firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
       shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols),
       useDelayedPrivatization(useDelayedPrivatization), symTable(symTable),
-      visitor() {
+      visitor(semaCtx) {
   eval.visit([&](const auto &functionParserNode) {
     parser::Walk(functionParserNode, visitor);
   });
@@ -424,24 +426,55 @@ getSource(const semantics::SemanticsContext &semaCtx,
   return source;
 }
 
+static void collectPrivatizingConstructs(
+    llvm::SmallSet<llvm::omp::Directive, 16> &constructs, unsigned version) {
+  using Clause = llvm::omp::Clause;
+  using Directive = llvm::omp::Directive;
+
+  static const Clause privatizingClauses[] = {
+      Clause::OMPC_private,
+      Clause::OMPC_lastprivate,
+      Clause::OMPC_firstprivate,
+      Clause::OMPC_in_reduction,
+      Clause::OMPC_reduction,
+      Clause::OMPC_linear,
+      // TODO: Clause::OMPC_induction,
+      Clause::OMPC_task_reduction,
+      Clause::OMPC_detach,
+      Clause::OMPC_use_device_ptr,
+      Clause::OMPC_is_device_ptr,
+  };
+
+  for (auto dir : llvm::enum_seq_inclusive<Directive>(Directive::First_,
+                                                      Directive::Last_)) {
+    bool allowsPrivatizing = llvm::any_of(privatizingClauses, [&](Clause cls) {
+      return llvm::omp::isAllowedClauseForDirective(dir, cls, version);
+    });
+    if (allowsPrivatizing)
+      constructs.insert(dir);
+  }
+}
+
 bool DataSharingProcessor::isOpenMPPrivatizingConstruct(
-    const parser::OpenMPConstruct &omp) {
-  return common::visit(
-      [](auto &&s) {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        return std::is_same_v<BareS, parser::OpenMPBlockConstruct> ||
-               std::is_same_v<BareS, parser::OpenMPLoopConstruct> ||
-               std::is_same_v<BareS, parser::OpenMPSectionsConstruct>;
-      },
-      omp.u);
+    const parser::OpenMPConstruct &omp, unsigned version) {
+  static llvm::SmallSet<llvm::omp::Directive, 16> privatizing;
+  [[maybe_unused]] static bool init =
+      (collectPrivatizingConstructs(privatizing, version), true);
+
+  // As of OpenMP 6.0, privatizing constructs (with the test being if they
+  // allow a privatizing clause) are: dispatch, distribute, do, for, loop,
+  // parallel, scope, sections, simd, single, target, target_data, task,
+  // taskgroup, taskloop, and teams.
+  return llvm::is_contained(privatizing, extractOmpDirective(omp));
 }
 
 bool DataSharingProcessor::isOpenMPPrivatizingEvaluation(
     const pft::Evaluation &eval) const {
-  return eval.visit([](auto &&s) {
+  unsigned version = semaCtx.langOptions().OpenMPVersion;
+  return eval.visit([=](auto &&s) {
     using BareS = llvm::remove_cvref_t<decltype(s)>;
     if constexpr (std::is_same_v<BareS, parser::OpenMPConstruct>) {
-      return isOpenMPPrivatizingConstruct(s);
+      return isOpenMPPrivatizingConstruct(s, version);
     } else {
       return false;
     }

diff  --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index ee2fc70d2e673..bc422f410403a 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -36,6 +36,8 @@ class DataSharingProcessor {
   /// at any point in time. This is used to track Symbol definition scopes in
   /// order to tell which OMP scope defined vs. references a certain Symbol.
   struct OMPConstructSymbolVisitor {
+    OMPConstructSymbolVisitor(semantics::SemanticsContext &ctx)
+        : version(ctx.langOptions().OpenMPVersion) {}
     template <typename T>
     bool Pre(const T &) {
       return true;
@@ -45,13 +47,13 @@ class DataSharingProcessor {
 
     bool Pre(const parser::OpenMPConstruct &omp) {
       // Skip constructs that may not have privatizations.
-      if (isOpenMPPrivatizingConstruct(omp))
+      if (isOpenMPPrivatizingConstruct(omp, version))
         constructs.push_back(&omp);
       return true;
     }
 
     void Post(const parser::OpenMPConstruct &omp) {
-      if (isOpenMPPrivatizingConstruct(omp))
+      if (isOpenMPPrivatizingConstruct(omp, version))
         constructs.pop_back();
     }
 
@@ -68,6 +70,9 @@ class DataSharingProcessor {
     /// construct that defines symbol.
     bool isSymbolDefineBy(const semantics::Symbol *symbol,
                           lower::pft::Evaluation &eval) const;
+
+  private:
+    unsigned version;
   };
 
   mlir::OpBuilder::InsertPoint lastPrivIP;
@@ -115,7 +120,8 @@ class DataSharingProcessor {
                              mlir::OpBuilder::InsertPoint *lastPrivIP);
   void insertDeallocs();
 
-  static bool isOpenMPPrivatizingConstruct(const parser::OpenMPConstruct &omp);
+  static bool isOpenMPPrivatizingConstruct(const parser::OpenMPConstruct &omp,
+                                           unsigned version);
   bool isOpenMPPrivatizingEvaluation(const pft::Evaluation &eval) const;
 
 public:

diff  --git a/flang/test/Lower/OpenMP/taskgroup02.f90 b/flang/test/Lower/OpenMP/taskgroup02.f90
index 1e996a030c23a..4c470b7aa82d1 100644
--- a/flang/test/Lower/OpenMP/taskgroup02.f90
+++ b/flang/test/Lower/OpenMP/taskgroup02.f90
@@ -3,8 +3,9 @@
 ! Check that variables are not privatized twice when TASKGROUP is used.
 
 !CHECK-LABEL: func.func @_QPsub() {
-!CHECK:         omp.parallel {
-!CHECK:           %[[PAR_I:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsubEi"}
+!CHECK:         omp.parallel private(@_QFsubEi_private_i32 %[[SUB_I:.*]]#0 -> %[[ARG:.*]] : !fir.ref<i32>)
+!CHECK:           %[[ALLOCA:.*]] = fir.alloca i32
+!CHECK:           %[[PAR_I:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFsubEi"}
 !CHECK:           omp.master {
 !CHECK:             omp.taskgroup {
 !CHECK-NEXT:          omp.task private(@_QFsubEi_firstprivate_i32 %[[PAR_I]]#0 -> %[[TASK_I:.*]] : !fir.ref<i32>) {

diff  --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index d44c33301bde7..9d0a55432e1ae 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -51,13 +51,17 @@ static constexpr inline bool canHaveIterator(Clause C) {
 // Can clause C create a private copy of a variable.
 static constexpr inline bool isPrivatizingClause(Clause C) {
   switch (C) {
+  case OMPC_detach:
   case OMPC_firstprivate:
+  // TODO case OMPC_induction:
   case OMPC_in_reduction:
+  case OMPC_is_device_ptr:
   case OMPC_lastprivate:
   case OMPC_linear:
   case OMPC_private:
   case OMPC_reduction:
   case OMPC_task_reduction:
+  case OMPC_use_device_ptr:
     return true;
   default:
     return false;


        

From flang-commits at lists.llvm.org  Thu Jul 17 11:41:12 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Thu, 17 Jul 2025 11:41:12 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Generalize
 isOpenMPPrivatizingConstruct (PR #148654)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148654@github.com>
Message-ID: <687943c8.170a0220.2f44aa.0280@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/148654

From flang-commits at lists.llvm.org  Thu Jul 17 11:18:25 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 11:18:25 -0700 (PDT)
Subject: [flang-commits] [flang] 413e71b - [flang] Main program symbol no
 longer conflicts with the other symbols (#149169)
Message-ID: <68793e71.050a0220.c7145.662e@mx.google.com>


Author: Eugene Epshteyn
Date: 2025-07-17T14:18:21-04:00
New Revision: 413e71b700562f914b369c7eab6ad41c18910bdf

URL: https://github.com/llvm/llvm-project/commit/413e71b700562f914b369c7eab6ad41c18910bdf
DIFF: https://github.com/llvm/llvm-project/commit/413e71b700562f914b369c7eab6ad41c18910bdf.diff

LOG: [flang] Main program symbol no longer conflicts with the other symbols (#149169)

The following code is now accepted:
```
module m
end
program m
use m
end
```
The PROGRAM name doesn't really have an effect on the compilation
result, so it shouldn't result in symbol name conflicts.

This change makes the main program symbol name all uppercase in the
cooked character stream. This makes it distinct from all other symbol
names that are all lowercase in cooked character stream.

Modified the tests that were checking for lower case main program name.

Added: 
    

Modified: 
    flang/docs/Extensions.md
    flang/lib/Semantics/check-omp-structure.cpp
    flang/lib/Semantics/resolve-labels.cpp
    flang/test/Driver/cuda-option.f90
    flang/test/Driver/unparse-use-analyzed.f95
    flang/test/Driver/unparse-with-modules.f90
    flang/test/Integration/debug-common-block-1.f90
    flang/test/Integration/debug-local-var-2.f90
    flang/test/Lower/CUDA/cuda-derived.cuf
    flang/test/Lower/CUDA/cuda-return01.cuf
    flang/test/Lower/CUDA/cuda-return02.cuf
    flang/test/Lower/HLFIR/intrinsic-subroutines.f90
    flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
    flang/test/Lower/OpenACC/acc-atomic-read.f90
    flang/test/Lower/OpenACC/acc-atomic-write.f90
    flang/test/Lower/OpenACC/acc-routine04.f90
    flang/test/Lower/OpenMP/atomic-read.f90
    flang/test/Lower/OpenMP/atomic-write.f90
    flang/test/Lower/OpenMP/common-atomic-lowering.f90
    flang/test/Lower/OpenMP/cray-pointers02.f90
    flang/test/Lower/OpenMP/default-clause-byref.f90
    flang/test/Lower/OpenMP/default-clause.f90
    flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
    flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
    flang/test/Lower/OpenMP/parallel-reduction-array2.f90
    flang/test/Lower/OpenMP/parallel-reduction-byref.f90
    flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
    flang/test/Lower/OpenMP/parallel-reduction-rename.f90
    flang/test/Lower/OpenMP/parallel-reduction.f90
    flang/test/Lower/OpenMP/sections.f90
    flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
    flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
    flang/test/Lower/OpenMP/threadprivate-host-association.f90
    flang/test/Lower/OpenMP/wsloop-chunks.f90
    flang/test/Lower/OpenMP/wsloop-collapse.f90
    flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
    flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
    flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
    flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
    flang/test/Lower/OpenMP/wsloop-reduction-array.f90
    flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
    flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
    flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
    flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
    flang/test/Lower/array-character.f90
    flang/test/Lower/array-expression-slice-1.f90
    flang/test/Lower/basic-program.f90
    flang/test/Lower/big-integer-parameter.f90
    flang/test/Lower/derived-type-finalization.f90
    flang/test/Lower/location.f90
    flang/test/Lower/nested-where.f90
    flang/test/Lower/polymorphic.f90
    flang/test/Lower/pre-fir-tree02.f90
    flang/test/Lower/pre-fir-tree03.f90
    flang/test/Lower/pre-fir-tree06.f90
    flang/test/Lower/program-units-fir-mangling.f90
    flang/test/Lower/return-statement.f90
    flang/test/Lower/volatile-openmp1.f90
    flang/test/Lower/volatile-string.f90
    flang/test/Lower/volatile3.f90
    flang/test/Parser/acc-unparse.f90
    flang/test/Semantics/OpenACC/acc-symbols01.f90
    flang/test/Semantics/OpenMP/critical_within_default.f90
    flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
    flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
    flang/test/Semantics/OpenMP/declare-reduction-operators.f90
    flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
    flang/test/Semantics/OpenMP/declare-reduction.f90
    flang/test/Semantics/OpenMP/declare-target03.f90
    flang/test/Semantics/OpenMP/do-schedule03.f90
    flang/test/Semantics/OpenMP/do01-positivecase.f90
    flang/test/Semantics/OpenMP/do04-positivecase.f90
    flang/test/Semantics/OpenMP/do05-positivecase.f90
    flang/test/Semantics/OpenMP/do06-positivecases.f90
    flang/test/Semantics/OpenMP/do11.f90
    flang/test/Semantics/OpenMP/do12.f90
    flang/test/Semantics/OpenMP/do14.f90
    flang/test/Semantics/OpenMP/do17.f90
    flang/test/Semantics/OpenMP/map-clause-symbols.f90
    flang/test/Semantics/OpenMP/reduction08.f90
    flang/test/Semantics/OpenMP/reduction09.f90
    flang/test/Semantics/OpenMP/reduction11.f90
    flang/test/Semantics/OpenMP/scan2.f90
    flang/test/Semantics/OpenMP/symbol01.f90
    flang/test/Semantics/OpenMP/symbol05.f90
    flang/test/Semantics/OpenMP/symbol07.f90
    flang/test/Semantics/OpenMP/symbol09.f90
    flang/test/Semantics/OpenMP/threadprivate03.f90
    flang/test/Semantics/getsymbols03-a.f90
    flang/test/Semantics/long-name.f90
    flang/test/Semantics/modproc01.f90
    flang/test/Semantics/multi-programs04.f90
    flang/test/Semantics/pointer01.f90
    flang/test/Semantics/procinterface01.f90
    flang/test/Semantics/resolve05.f90
    flang/test/Semantics/resolve125.f90
    flang/test/Semantics/symbol03.f90
    flang/test/Semantics/symbol06.f90
    flang/test/Semantics/symbol07.f90
    flang/test/Semantics/symbol08.f90
    flang/test/Semantics/symbol15.f90
    flang/test/Semantics/symbol16.f90
    flang/test/Semantics/symbol17.f90
    flang/test/Semantics/symbol18.f90
    flang/test/Semantics/symbol20.f90
    flang/test/Semantics/symbol25.f90
    flang/test/Semantics/symbol26.f90
    flang/test/Transforms/DoConcurrent/basic_host.f90

Removed: 
    


################################################################################
diff  --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 72d12cd92600d..c167a55bc486d 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -1,9 +1,9 @@
-<!--===- docs/Extensions.md 
-  
+<!--===- docs/Extensions.md
+
    Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
    See https://llvm.org/LICENSE.txt for license information.
    SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-  
+
 -->
 
 # Fortran Extensions supported by Flang
@@ -170,6 +170,18 @@ end
   In the case of `DEFERRED` bindings in an `ABSTRACT` derived type,
   however, overrides are necessary, so they are permitted for inaccessible
   bindings with an optional warning.
+* Main program name is allowed to be the same as the other symbols used
+  in the main program, for example:
+```
+module m
+end
+program m
+use m
+end
+```
+  Note that internally the main program symbol name is all uppercase, unlike
+  the names of all other symbols, which are usually all lowercase. This
+  may make a 
diff erence in testing/debugging.
 
 ## Extensions, deletions, and legacy features supported by default
 

diff  --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2425265e196c6..e4a94efcc6b55 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -1156,8 +1156,7 @@ void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar(
                         (sym->has<MainProgramDetails>() ||
                             sym->has<ModuleDetails>())) {
                       context_.Say(name->source,
-                          "The module name or main program name cannot be in a "
-                          "%s "
+                          "The module name cannot be in a %s "
                           "directive"_err_en_US,
                           ContextDirectiveAsFortran());
                     } else if (!IsSaved(*name->symbol) &&

diff  --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index b0cbc4b56e889..27e259fab3873 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -489,15 +489,30 @@ class ParseTreeAnalyzer {
 
   // C1401
   void Post(const parser::MainProgram &mainProgram) {
+    // Uppercase the name of the main program, so that its symbol name
+    // would be unique from similarly named non-main-program symbols.
+    auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
+      char *ch{const_cast<char *>(cb.begin())};
+      char *endCh{ch + cb.size()};
+      while (ch != endCh) {
+        *ch++ = parser::ToUpperCaseLetter(*ch);
+      }
+    };
+    const parser::CharBlock *progName{nullptr};
+    if (const auto &program{
+            std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
+                mainProgram.t)}) {
+      progName = &program->statement.v.source;
+      upperCaseCharBlock(*progName);
+    }
     if (const parser::CharBlock *
         endName{GetStmtName(std::get<parser::Statement<parser::EndProgramStmt>>(
             mainProgram.t))}) {
-      if (const auto &program{
-              std::get<std::optional<parser::Statement<parser::ProgramStmt>>>(
-                  mainProgram.t)}) {
-        if (*endName != program->statement.v.source) {
+      upperCaseCharBlock(*endName);
+      if (progName) {
+        if (*endName != *progName) {
           context_.Say(*endName, "END PROGRAM name mismatch"_err_en_US)
-              .Attach(program->statement.v.source, "should be"_en_US);
+              .Attach(*progName, "should be"_en_US);
         }
       } else {
         context_.Say(*endName,

diff  --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 0740ed509a077..f55e88dab20ce 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -8,7 +8,7 @@ program main
   integer, device :: dvar
 end program
 
-! CHECK-LABEL: PROGRAM main
+! CHECK-LABEL: PROGRAM MAIN
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 

diff  --git a/flang/test/Driver/unparse-use-analyzed.f95 b/flang/test/Driver/unparse-use-analyzed.f95
index eb6046aebba54..4bcd72c9a5f50 100644
--- a/flang/test/Driver/unparse-use-analyzed.f95
+++ b/flang/test/Driver/unparse-use-analyzed.f95
@@ -6,12 +6,12 @@
 ! RUN: %flang_fc1 -fdebug-unparse  %s | FileCheck %s --check-prefix=DEFAULT
 ! RUN: %flang_fc1 -fdebug-unparse -fno-analyzed-objects-for-unparse %s | FileCheck %s --check-prefix=DISABLED
 
-! DEFAULT: PROGRAM test
+! DEFAULT: PROGRAM TEST
 ! DEFAULT-NEXT:  REAL, PARAMETER :: val = 3.43e2_4
 ! DEFAULT-NEXT:  PRINT *, 3.47e2_4
 ! DEFAULT-NEXT: END PROGRAM
 
-! DISABLED: PROGRAM test
+! DISABLED: PROGRAM TEST
 ! DISABLED-NEXT:  REAL, PARAMETER :: val = 343.0
 ! DISABLED-NEXT:  PRINT *, val+4
 ! DISABLED-NEXT: END PROGRAM

diff  --git a/flang/test/Driver/unparse-with-modules.f90 b/flang/test/Driver/unparse-with-modules.f90
index 53997f7804efa..f6444afbe47c1 100644
--- a/flang/test/Driver/unparse-with-modules.f90
+++ b/flang/test/Driver/unparse-with-modules.f90
@@ -25,7 +25,7 @@ program test
 !CHECK:  implicit none
 !CHECK:  real(kind=real32) x
 !CHECK: end module
-!CHECK: program test
+!CHECK: program TEST
 !CHECK:  use :: m1
 !CHECK:  use :: basictestmoduletwo
 !CHECK:  implicit none

diff  --git a/flang/test/Integration/debug-common-block-1.f90 b/flang/test/Integration/debug-common-block-1.f90
index 18217637be0fa..77f47daea4a91 100644
--- a/flang/test/Integration/debug-common-block-1.f90
+++ b/flang/test/Integration/debug-common-block-1.f90
@@ -89,7 +89,7 @@ program test
 ! CHECK-DAG: ![[CBF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "__BLNK__"{{.*}})
 ! CHECK-DAG: ![[CBAF3]] = !DICommonBlock(scope: ![[F3]], declaration: null, name: "a"{{.*}})
 
-! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "test"{{.*}})
+! CHECK-DAG: ![[MAIN:[0-9]+]] = {{.*}}!DISubprogram(name: "TEST"{{.*}})
 ! CHECK-DAG: ![[CBM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "__BLNK__"{{.*}})
 ! CHECK-DAG: ![[CBAM]] = !DICommonBlock(scope: ![[MAIN]], declaration: null, name: "a"{{.*}})
 

diff  --git a/flang/test/Integration/debug-local-var-2.f90 b/flang/test/Integration/debug-local-var-2.f90
index b97be141cc8d0..0ddac633a5b1e 100644
--- a/flang/test/Integration/debug-local-var-2.f90
+++ b/flang/test/Integration/debug-local-var-2.f90
@@ -37,7 +37,7 @@
 ! BOTH-LABEL: }
 
 program mn
-! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "mn", {{.*}})
+! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "MN", {{.*}})
 
 ! BOTH-DAG: ![[TYI32:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
 ! BOTH-DAG: ![[TYI64:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)

diff  --git a/flang/test/Lower/CUDA/cuda-derived.cuf b/flang/test/Lower/CUDA/cuda-derived.cuf
index 96250d88d81c4..d419ee074f7a0 100644
--- a/flang/test/Lower/CUDA/cuda-derived.cuf
+++ b/flang/test/Lower/CUDA/cuda-derived.cuf
@@ -25,6 +25,6 @@ program main
   type(t2) :: b
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"}
 ! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEa"}
 ! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tt2{b:!fir.type<_QMm1Tt1{a:!fir.box<!fir.heap<!fir.array<?xf32>>>}>}> {bindc_name = "b", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEb"}

diff  --git a/flang/test/Lower/CUDA/cuda-return01.cuf b/flang/test/Lower/CUDA/cuda-return01.cuf
index 47e69a903efd3..ed7c640a71082 100644
--- a/flang/test/Lower/CUDA/cuda-return01.cuf
+++ b/flang/test/Lower/CUDA/cuda-return01.cuf
@@ -28,6 +28,6 @@ program main
   return
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"}
 ! CHECK: cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QFEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK-NOT: cuf.free

diff  --git a/flang/test/Lower/CUDA/cuda-return02.cuf b/flang/test/Lower/CUDA/cuda-return02.cuf
index e450d7e796f22..e54818444e49c 100644
--- a/flang/test/Lower/CUDA/cuda-return02.cuf
+++ b/flang/test/Lower/CUDA/cuda-return02.cuf
@@ -13,7 +13,7 @@ program test
   return
 end
 
-! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare
 ! CHECK: cf.cond_br %{{.*}}, ^bb1, ^bb2
 ! CHECK-NEXT: ^bb1:

diff  --git a/flang/test/Lower/HLFIR/intrinsic-subroutines.f90 b/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
index 07c4f012781d4..cbc56ca1e395b 100644
--- a/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
+++ b/flang/test/Lower/HLFIR/intrinsic-subroutines.f90
@@ -24,7 +24,7 @@ program main
   call mvbits(from, 2, 2, to, 0)
   if (any(to /= 5)) STOP 1
 end program
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<3xi32> {bindc_name = "from", uniq_name = "_QFEfrom"}
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>

diff  --git a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
index 7b64634d10d4b..a097b1522307e 100644
--- a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
@@ -35,7 +35,7 @@ FUNCTION BAR() RESULT(res)
     END
   END
 
-! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:    %[[VAL_0:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_1:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_2:.*]] = fir.alloca !fir.boxproc<(!fir.ref<i32>) -> i32> {bindc_name = "pp2", uniq_name = "_QFEpp2"}

diff  --git a/flang/test/Lower/OpenACC/acc-atomic-read.f90 b/flang/test/Lower/OpenACC/acc-atomic-read.f90
index 639a98051e3a2..76751a0fa63a8 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-read.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-read.f90
@@ -8,7 +8,7 @@ program acc_atomic_test
      g = h
 end program acc_atomic_test
 
-! CHECK: func @_QQmain() attributes {fir.bindc_name = "acc_atomic_test"} {
+! CHECK: func @_QQmain() attributes {fir.bindc_name = "ACC_ATOMIC_TEST"} {
 ! CHECK: %[[VAR_G:.*]] = fir.alloca f32 {bindc_name = "g", uniq_name = "_QFEg"}
 ! CHECK: %[[G_DECL:.*]]:2 = hlfir.declare %[[VAR_G]] {uniq_name = "_QFEg"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK: %[[VAR_H:.*]] = fir.alloca f32 {bindc_name = "h", uniq_name = "_QFEh"}

diff  --git a/flang/test/Lower/OpenACC/acc-atomic-write.f90 b/flang/test/Lower/OpenACC/acc-atomic-write.f90
index 3c55394021abf..e0116e3281820 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-write.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-write.f90
@@ -2,7 +2,7 @@
 
 ! This test checks the lowering of atomic write
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "acc_atomic_write_test"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "ACC_ATOMIC_WRITE_TEST"} {
 !CHECK: %[[VAR_X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[VAR_X]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[VAR_Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}

diff  --git a/flang/test/Lower/OpenACC/acc-routine04.f90 b/flang/test/Lower/OpenACC/acc-routine04.f90
index f603376163901..655e2762b9694 100644
--- a/flang/test/Lower/OpenACC/acc-routine04.f90
+++ b/flang/test/Lower/OpenACC/acc-routine04.f90
@@ -30,5 +30,5 @@ subroutine sub2()
 ! CHECK: acc.routine @acc_routine_1 func(@_QFPsub2) seq
 ! CHECK: acc.routine @acc_routine_0 func(@_QMdummy_modPsub1) seq
 ! CHECK: func.func @_QMdummy_modPsub1(%arg0: !fir.ref<i32> {fir.bindc_name = "i"}) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>}
-! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "test_acc_routine"}
+! CHECK: func.func @_QQmain() attributes {fir.bindc_name = "TEST_ACC_ROUTINE"}
 ! CHECK: func.func private @_QFPsub2() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_1]>, fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>}

diff  --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
index 68dcaac90eef5..30313e240efa3 100644
--- a/flang/test/Lower/OpenMP/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -4,7 +4,7 @@
 
 ! This test checks the lowering of atomic read
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomic"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "OMPATOMIC"} {
 !CHECK:    %[[A_REF:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_REF]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[B_REF:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}

diff  --git a/flang/test/Lower/OpenMP/atomic-write.f90 b/flang/test/Lower/OpenMP/atomic-write.f90
index 6eded49b0b15d..742fd475c0f04 100644
--- a/flang/test/Lower/OpenMP/atomic-write.f90
+++ b/flang/test/Lower/OpenMP/atomic-write.f90
@@ -4,7 +4,7 @@
 
 ! This test checks the lowering of atomic write
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomicwrite"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "OMPATOMICWRITE"} {
 !CHECK:    %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[Y_REF:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}

diff  --git a/flang/test/Lower/OpenMP/common-atomic-lowering.f90 b/flang/test/Lower/OpenMP/common-atomic-lowering.f90
index a53cc101024c6..f729bbb00ac8e 100644
--- a/flang/test/Lower/OpenMP/common-atomic-lowering.f90
+++ b/flang/test/Lower/OpenMP/common-atomic-lowering.f90
@@ -1,6 +1,6 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "sample"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "SAMPLE"} {
 !CHECK: %[[val_0:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK: %[[val_1:.*]]:2 = hlfir.declare %[[val_0]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[val_2:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}

diff  --git a/flang/test/Lower/OpenMP/cray-pointers02.f90 b/flang/test/Lower/OpenMP/cray-pointers02.f90
index 19e4cd09fe50a..79d838702e4b0 100644
--- a/flang/test/Lower/OpenMP/cray-pointers02.f90
+++ b/flang/test/Lower/OpenMP/cray-pointers02.f90
@@ -1,7 +1,7 @@
 ! Test lowering of Cray pointee references.
 ! RUN: flang -fc1 -emit-hlfir -fopenmp %s -o - 2>&1 | FileCheck %s
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test_cray_pointers_02"}
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST_CRAY_POINTERS_02"}
 program test_cray_pointers_02
     implicit none
 

diff  --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index c44c6bb966580..af51c4cc3e814 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -34,7 +34,7 @@
 !CHECK:    omp.yield(%[[PRIV_X]] : !fir.ref<i32>)
 !CHECK:  }
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "DEFAULT_CLAUSE_LOWERING"} {
 !CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"}
 !CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}

diff  --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index ee5f579f06b91..505fa4f0f5d63 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -8,7 +8,7 @@
 ! RUN: | FileCheck %s
 
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "DEFAULT_CLAUSE_LOWERING"} {
 !CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"}
 !CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}

diff  --git a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index 4bfd5d8d19261..0036670317a8e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -80,7 +80,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>

diff  --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index ec54294c7104f..ea0aa9ec3f53b 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -68,7 +68,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3x2xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index

diff  --git a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
index 488ecc353af8e..eb0df2b3a17de 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
@@ -63,7 +63,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>

diff  --git a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
index 596276a99cafc..2caec0384a6ab 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
@@ -18,7 +18,7 @@
 !CHECK:    fir.store %[[CR]] to %[[C0]] : !fir.ref<i32>
 !CHECK:    omp.yield(%[[C0]] : !fir.ref<i32>)
 !CHECK:  }
-!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "mn"} {
+!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "MN"} {
 !CHECK:    %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32

diff  --git a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
index f638688bc2cc1..3c1daa0eb983f 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
@@ -82,7 +82,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>

diff  --git a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90 b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
index c06343e997bfd..2be154f4bbaf5 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90
@@ -25,7 +25,7 @@ end program main
 ! CHECK:           omp.yield(%[[VAL_2]] : i32)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32

diff  --git a/flang/test/Lower/OpenMP/parallel-reduction.f90 b/flang/test/Lower/OpenMP/parallel-reduction.f90
index 612549fb32de5..15e8cc325916d 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction.f90
@@ -10,7 +10,7 @@
 !CHECK:    %[[CR:[_a-z0-9]+]] = arith.addi %[[C0]], %[[C1]] : i32
 !CHECK:    omp.yield(%[[CR]] : i32)
 !CHECK:  }
-!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "mn"} {
+!CHECK:  func.func @_QQmain() attributes {fir.bindc_name = "MN"} {
 !CHECK:    %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32

diff  --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index d11925cafdc12..3d5c0326fb6b9 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -5,7 +5,7 @@
 ! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 ! RUN: bbc -hlfir -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
-!CHECK: func @_QQmain() attributes {fir.bindc_name = "sample"} {
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "SAMPLE"} {
 !CHECK:   %[[COUNT:.*]] = fir.address_of(@_QFEcount) : !fir.ref<i32>
 !CHECK:   %[[COUNT_DECL:.*]]:2 = hlfir.declare %[[COUNT]] {uniq_name = "_QFEcount"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[ETA:.*]] = fir.alloca f32 {bindc_name = "eta", uniq_name = "_QFEeta"}

diff  --git a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
index 5e54cef8c29db..5c90ef7d84f89 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association-2.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>

diff  --git a/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
index 21547b47cf381..0e61261e8853e 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association-3.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[A_ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>

diff  --git a/flang/test/Lower/OpenMP/threadprivate-host-association.f90 b/flang/test/Lower/OpenMP/threadprivate-host-association.f90
index 7a27efa2f84aa..1887e8aa68fdc 100644
--- a/flang/test/Lower/OpenMP/threadprivate-host-association.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-host-association.f90
@@ -3,7 +3,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 !CHECK:   %[[A:.*]] = fir.address_of(@_QFEa) : !fir.ref<i32>
 !CHECK:   %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<internal_assoc>, uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:   %[[TP_A:.*]] = omp.threadprivate %[[A_DECL]]#0 : !fir.ref<i32> -> !fir.ref<i32>

diff  --git a/flang/test/Lower/OpenMP/wsloop-chunks.f90 b/flang/test/Lower/OpenMP/wsloop-chunks.f90
index 29c02a3b3c8d5..f3f11d8c4a6c2 100644
--- a/flang/test/Lower/OpenMP/wsloop-chunks.f90
+++ b/flang/test/Lower/OpenMP/wsloop-chunks.f90
@@ -7,7 +7,7 @@ program wsloop
         integer :: i
         integer :: chunk
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "wsloop"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "WSLOOP"} {
 ! CHECK:         %[[CHUNK_REF:.*]] = fir.alloca i32 {bindc_name = "chunk", uniq_name = "_QFEchunk"}
 ! CHECK:         %[[VAL_0:.*]]:2 = hlfir.declare %[[CHUNK_REF]] {uniq_name = "_QFEchunk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 

diff  --git a/flang/test/Lower/OpenMP/wsloop-collapse.f90 b/flang/test/Lower/OpenMP/wsloop-collapse.f90
index a4d5cbdc03d3e..7ec40ab4b2f43 100644
--- a/flang/test/Lower/OpenMP/wsloop-collapse.f90
+++ b/flang/test/Lower/OpenMP/wsloop-collapse.f90
@@ -2,7 +2,7 @@
 
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
 
-!CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "wsloop_collapse"} {
+!CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "WSLOOP_COLLAPSE"} {
 program wsloop_collapse
 !CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index 58b68e5ec4cfd..e2f75bc8e4481 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -156,7 +156,7 @@ program reduce15
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce15"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE15"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEarr"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index 0a536eb34e7af..663851cba46c6 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -63,7 +63,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "r", uniq_name = "_QFEr"}

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
index 9f0dd16002baf..2233a74600948 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb.f90
@@ -31,5 +31,5 @@ program reduce
 ! CHECK:           omp.yield(%[[ARG0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           omp.wsloop {{.*}} reduction(byref @add_reduction_byref_box_2xi32 %{{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<2xi32>>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
index 5ada623a0ed23..211bde19da8db 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-lb2.f90
@@ -40,5 +40,5 @@ subroutine sub(a, lb, ub)
 ! CHECK:           omp.yield(%[[ARG0]] : !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           omp.wsloop {{.*}} reduction(byref @add_reduction_byref_box_Uxi32 %{{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>)

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index 21261da49710c..b7882bcbc0d13 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index ab8dcf1f076c0..7d90335a13a87 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -65,7 +65,7 @@ program reduce
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<!fir.array<2xi32>>

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
index 1e26f5a24d41e..d776bd7cfdd03 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
@@ -28,7 +28,7 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_2]] : i32)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<i32>

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
index e0a3b469f40c1..5133db0347034 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -93,7 +93,7 @@ program main
 ! CHECK:           omp.yield(%[[VAL_2]] : f64)
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarray) : !fir.ref<!fir.array<3x3xf64>>
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
index 40b4302f24cd4..27b726376fbeb 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
@@ -64,7 +64,7 @@ program reduce_pointer
 ! CHECK:           omp.yield
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce_pointer"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "REDUCE_POINTER"} {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "v", uniq_name = "_QFEv"}

diff  --git a/flang/test/Lower/array-character.f90 b/flang/test/Lower/array-character.f90
index 1bc73dae44235..e2899d967c80d 100644
--- a/flang/test/Lower/array-character.f90
+++ b/flang/test/Lower/array-character.f90
@@ -32,7 +32,7 @@ program p
   call charlit
 end program p
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.array<3x!fir.char<1,4>> {bindc_name = "c1", uniq_name = "_QFEc1"}

diff  --git a/flang/test/Lower/array-expression-slice-1.f90 b/flang/test/Lower/array-expression-slice-1.f90
index b597814bc0d9f..73943137cb18d 100644
--- a/flang/test/Lower/array-expression-slice-1.f90
+++ b/flang/test/Lower/array-expression-slice-1.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -hlfir=false -fwrapv -o - --outline-intrinsics %s | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK-DAG:         %[[VAL_0:.*]] = arith.constant 10 : index
 ! CHECK-DAG:         %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK-DAG:         %[[VAL_5:.*]] = arith.constant 1 : index

diff  --git a/flang/test/Lower/basic-program.f90 b/flang/test/Lower/basic-program.f90
index 5a0e4bdc7b4a1..7e5b40d9e2f0a 100644
--- a/flang/test/Lower/basic-program.f90
+++ b/flang/test/Lower/basic-program.f90
@@ -4,10 +4,10 @@
 program basic
 end program
 
-! CHECK: 1 Program basic
+! CHECK: 1 Program BASIC
 ! CHECK:   1 EndProgramStmt: end program
-! CHECK: End Program basic
+! CHECK: End Program BASIC
 
-! FIR-LABEL: func @_QQmain() attributes {fir.bindc_name = "basic"} {
+! FIR-LABEL: func @_QQmain() attributes {fir.bindc_name = "BASIC"} {
 ! FIR:         return
 ! FIR:       }

diff  --git a/flang/test/Lower/big-integer-parameter.f90 b/flang/test/Lower/big-integer-parameter.f90
index a413b1224ebc2..ca90b8adfb318 100644
--- a/flang/test/Lower/big-integer-parameter.f90
+++ b/flang/test/Lower/big-integer-parameter.f90
@@ -13,7 +13,7 @@ program i128
   print*,y
 end
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "i128"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "I128"} {
 ! CHECK-COUNT-2:  %{{.*}} = fir.call @_FortranAioOutputInteger128(%{{.*}}, %{{.*}}) {{.*}}: (!fir.ref<i8>, i128) -> i1
 
 

diff  --git a/flang/test/Lower/derived-type-finalization.f90 b/flang/test/Lower/derived-type-finalization.f90
index 3ea58cd719f4a..71cef34899603 100644
--- a/flang/test/Lower/derived-type-finalization.f90
+++ b/flang/test/Lower/derived-type-finalization.f90
@@ -255,5 +255,5 @@ program p
   type(t1) :: t
 end program
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK-NOT: fir.call @_FortranADestroy

diff  --git a/flang/test/Lower/location.f90 b/flang/test/Lower/location.f90
index a6ece31bbebed..95bf2260fc107 100644
--- a/flang/test/Lower/location.f90
+++ b/flang/test/Lower/location.f90
@@ -5,7 +5,7 @@ program test
 
 end 
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: fir.call @_FortranAioOutputAscii(%{{.*}}, %{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1 loc(fused<#fir<loc_kind_array[ base,  inclusion,  inclusion]>>["{{.*}}location1.inc":1:10, "{{.*}}location0.inc":1:1, "{{.*}}location.f90":4:1])
 ! CHECK: return loc("{{.*}}location.f90":6:1)
 ! CHECK: } loc("{{.*}}location.f90":3:1)

diff  --git a/flang/test/Lower/nested-where.f90 b/flang/test/Lower/nested-where.f90
index ab457280b80ce..28aced2325813 100644
--- a/flang/test/Lower/nested-where.f90
+++ b/flang/test/Lower/nested-where.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "nested_where"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "NESTED_WHERE"} {
 program nested_where
 
   ! CHECK:  %[[VAL_0:.*]] = fir.alloca i32 {adapt.valuebyref, bindc_name = "i"}

diff  --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90
index b7be5f685d9e3..a84b495dd09d0 100644
--- a/flang/test/Lower/polymorphic.f90
+++ b/flang/test/Lower/polymorphic.f90
@@ -1146,7 +1146,7 @@ program test
   l = i < o%inner
 end program
 
-! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 ! CHECK: %[[ADDR_O:.*]] = fir.address_of(@_QFEo) : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMpolymorphic_testTouter{inner:!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>}>>>>
 ! CHECK: %[[BOX_NONE:.*]] = fir.convert %[[ADDR_O]] : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMpolymorphic_testTouter{inner:!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>}>>>>) -> !fir.ref<!fir.box<none>>
 ! CHECK: %{{.*}} = fir.call @_FortranAAllocatableAllocate(%[[BOX_NONE]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32

diff  --git a/flang/test/Lower/pre-fir-tree02.f90 b/flang/test/Lower/pre-fir-tree02.f90
index f4fa626ba6548..65c33e9b364fe 100644
--- a/flang/test/Lower/pre-fir-tree02.f90
+++ b/flang/test/Lower/pre-fir-tree02.f90
@@ -3,7 +3,7 @@
 ! Test Pre-FIR Tree captures all the intended nodes from the parse-tree
 ! Coarray and OpenMP related nodes are tested in other files.
 
-! CHECK: Program test_prog
+! CHECK: Program TEST_PROG
 program test_prog
   ! Check specification part is not part of the tree.
   interface

diff  --git a/flang/test/Lower/pre-fir-tree03.f90 b/flang/test/Lower/pre-fir-tree03.f90
index 313dab4d6ec7c..1de66e3f8d016 100644
--- a/flang/test/Lower/pre-fir-tree03.f90
+++ b/flang/test/Lower/pre-fir-tree03.f90
@@ -2,7 +2,7 @@
 
 ! Test Pre-FIR Tree captures OpenMP related constructs
 
-! CHECK: Program test_omp
+! CHECK: Program TEST_OMP
 program test_omp
   ! CHECK: PrintStmt
   print *, "sequential"

diff  --git a/flang/test/Lower/pre-fir-tree06.f90 b/flang/test/Lower/pre-fir-tree06.f90
index f84bcd8b58b2d..ed1e76cb375bd 100644
--- a/flang/test/Lower/pre-fir-tree06.f90
+++ b/flang/test/Lower/pre-fir-tree06.f90
@@ -25,13 +25,13 @@ subroutine sub2()
 end
 ! CHECK: End Module m2
 
-! CHECK: Program main
+! CHECK: Program MAIN
 program main
   real :: y
   ! CHECK-NEXT: OpenMPDeclarativeConstruct
   !$omp threadprivate(y)
 end
-! CHECK: End Program main
+! CHECK: End Program MAIN
 
 ! CHECK: Subroutine sub1
 subroutine sub1()

diff  --git a/flang/test/Lower/program-units-fir-mangling.f90 b/flang/test/Lower/program-units-fir-mangling.f90
index e0af6f065f34d..65940b4e1ff17 100644
--- a/flang/test/Lower/program-units-fir-mangling.f90
+++ b/flang/test/Lower/program-units-fir-mangling.f90
@@ -124,7 +124,7 @@ subroutine should_not_collide()
 ! CHECK: }
 end subroutine
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "test"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "TEST"} {
 program test
 ! CHECK: }
 contains

diff  --git a/flang/test/Lower/return-statement.f90 b/flang/test/Lower/return-statement.f90
index 6351a6859eb4f..8ab69e3146e2f 100644
--- a/flang/test/Lower/return-statement.f90
+++ b/flang/test/Lower/return-statement.f90
@@ -4,7 +4,7 @@ program basic
   return
 end program
 
-! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "basic"} {
+! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "BASIC"} {
 ! CHECK:         return
 ! CHECK:       }
 

diff  --git a/flang/test/Lower/volatile-openmp1.f90 b/flang/test/Lower/volatile-openmp1.f90
index 163db953b6b80..07d81a1aeb240 100644
--- a/flang/test/Lower/volatile-openmp1.f90
+++ b/flang/test/Lower/volatile-openmp1.f90
@@ -13,7 +13,7 @@ program main
 !$omp end parallel
 end program
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "MAIN"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 1000 : i32
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32

diff  --git a/flang/test/Lower/volatile-string.f90 b/flang/test/Lower/volatile-string.f90
index 88b21d7b245e9..f263db7abb5fc 100644
--- a/flang/test/Lower/volatile-string.f90
+++ b/flang/test/Lower/volatile-string.f90
@@ -21,7 +21,7 @@ subroutine assign_
diff erent_length(string)
   end subroutine
 end program
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 11 : i32
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant true

diff  --git a/flang/test/Lower/volatile3.f90 b/flang/test/Lower/volatile3.f90
index 8825f8f3afbcb..a32f29d2bb9e7 100644
--- a/flang/test/Lower/volatile3.f90
+++ b/flang/test/Lower/volatile3.f90
@@ -70,7 +70,7 @@ subroutine sub_select_rank(arr)
 end program
 
 
-! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "p"} {
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "P"} {
 ! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 10 : index

diff  --git a/flang/test/Parser/acc-unparse.f90 b/flang/test/Parser/acc-unparse.f90
index 62e0d4487f3f7..12e6dec19f272 100644
--- a/flang/test/Parser/acc-unparse.f90
+++ b/flang/test/Parser/acc-unparse.f90
@@ -15,7 +15,7 @@ program bug47659
   end do label1
 end program
 
-!CHECK-LABEL: PROGRAM bug47659
+!CHECK-LABEL: PROGRAM BUG47659
 !CHECK: !$ACC PARALLEL LOOP
 
 

diff  --git a/flang/test/Semantics/OpenACC/acc-symbols01.f90 b/flang/test/Semantics/OpenACC/acc-symbols01.f90
index 375445bad13a5..51a7a3a23e8ce 100644
--- a/flang/test/Semantics/OpenACC/acc-symbols01.f90
+++ b/flang/test/Semantics/OpenACC/acc-symbols01.f90
@@ -1,24 +1,24 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenacc
 
-!DEF: /mm MainProgram
-program mm
-  !DEF: /mm/x ObjectEntity REAL(4)
-  !DEF: /mm/y ObjectEntity REAL(4)
+!DEF: /MM MainProgram
+program MM
+  !DEF: /MM/x ObjectEntity REAL(4)
+  !DEF: /MM/y ObjectEntity REAL(4)
   real x, y
-  !DEF: /mm/a ObjectEntity INTEGER(4)
-  !DEF: /mm/b ObjectEntity INTEGER(4)
-  !DEF: /mm/c ObjectEntity INTEGER(4)
-  !DEF: /mm/i ObjectEntity INTEGER(4)
+  !DEF: /MM/a ObjectEntity INTEGER(4)
+  !DEF: /MM/b ObjectEntity INTEGER(4)
+  !DEF: /MM/c ObjectEntity INTEGER(4)
+  !DEF: /MM/i ObjectEntity INTEGER(4)
   integer a(10), b(10), c(10), i
-  !REF: /mm/b
+  !REF: /MM/b
   b = 2
  !$acc parallel present(c) firstprivate(b) private(a)
  !$acc loop
-  !REF: /mm/i
+  !REF: /MM/i
   do i=1,10
-   !REF: /mm/a
-   !REF: /mm/i
-   !REF: /mm/b
+   !REF: /MM/a
+   !REF: /MM/i
+   !REF: /MM/b
    a(i) = b(i)
   end do
  !$acc end parallel

diff  --git a/flang/test/Semantics/OpenMP/critical_within_default.f90 b/flang/test/Semantics/OpenMP/critical_within_default.f90
index dd972e6e52949..a5fe30eeb7de0 100644
--- a/flang/test/Semantics/OpenMP/critical_within_default.f90
+++ b/flang/test/Semantics/OpenMP/critical_within_default.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
 ! Test that we do not make a private copy of the critical name
 
-!CHECK:  MainProgram scope: mn
+!CHECK:  MainProgram scope: MN
 !CHECK-NEXT:    j size=4 offset=0: ObjectEntity type: INTEGER(4)
 !CHECK-NEXT:    OtherConstruct scope:
 !CHECK-NEXT:      j (OmpPrivate): HostAssoc

diff  --git a/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90 b/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
index 06f41ab8ce76f..e57a5c0c1cea6 100644
--- a/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
+++ b/flang/test/Semantics/OpenMP/declare-mapper-symbols.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
 
 program main
-!CHECK-LABEL: MainProgram scope: main
+!CHECK-LABEL: MainProgram scope: MAIN
    implicit none
 
    type ty

diff  --git a/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90 b/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
index 9d0a097fb1991..fc977f2f1b839 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
@@ -17,7 +17,7 @@ end function mymax
 end module mymod
 
 program omp_examples
-!CHECK-LABEL: MainProgram scope: omp_examples
+!CHECK-LABEL: MainProgram scope: OMP_EXAMPLES
   use mymod
   implicit none
   integer, parameter :: n = 100

diff  --git a/flang/test/Semantics/OpenMP/declare-reduction-operators.f90 b/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
index d7a9f2fc0a36b..84dbe1af01877 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
@@ -49,7 +49,7 @@ function my_add(x, y)
 end module m1
 
 program test_vector
-!CHECK-LABEL: MainProgram scope: test_vector
+!CHECK-LABEL: MainProgram scope: TEST_VECTOR
   use vector_mod
 !CHECK: add_vectors (Function): Use from add_vectors in vector_mod
   implicit none

diff  --git a/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90 b/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
index 12e80cbf7b327..9cd638d796091 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
@@ -22,7 +22,7 @@ end function my_mul
 end module module1
 
 program test_omp_reduction
-!CHECK: MainProgram scope: test_omp_reduction
+!CHECK: MainProgram scope: TEST_OMP_REDUCTION
   use module1, only: t1, operator(.modmul.) => operator(.mul.)
 
 !CHECK: .modmul. (Function): Use from .mul. in module1

diff  --git a/flang/test/Semantics/OpenMP/declare-reduction.f90 b/flang/test/Semantics/OpenMP/declare-reduction.f90
index ddca38fd57812..1f39c57c54ad1 100644
--- a/flang/test/Semantics/OpenMP/declare-reduction.f90
+++ b/flang/test/Semantics/OpenMP/declare-reduction.f90
@@ -31,7 +31,7 @@ end subroutine initme
 end function func
 
 program main
-!CHECK-LABEL: MainProgram scope: main
+!CHECK-LABEL: MainProgram scope: MAIN
 
   !$omp declare reduction (my_add_red : integer : omp_out = omp_out + omp_in) initializer (omp_priv=0)
 

diff  --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index 64a299d78224a..48cfc68393873 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -13,10 +13,10 @@ subroutine bar
 program main
   use mod1
 
-  !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
+  !ERROR: The module name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
-  !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
+  ! This is now allowed: "main" is implicitly declared symbol separate
+  ! from the main program symbol
   !$omp declare target (main)
 end

diff  --git a/flang/test/Semantics/OpenMP/do-schedule03.f90 b/flang/test/Semantics/OpenMP/do-schedule03.f90
index 8787b094d581a..05602ca57e4a9 100644
--- a/flang/test/Semantics/OpenMP/do-schedule03.f90
+++ b/flang/test/Semantics/OpenMP/do-schedule03.f90
@@ -2,27 +2,27 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - xx.
-  !DEF: /ompdoschedule MainProgram
-program ompdoschedule
-  !DEF: /ompdoschedule/a ObjectEntity REAL(4)
-  !DEF: /ompdoschedule/y ObjectEntity REAL(4)
-  !DEF: /ompdoschedule/z ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE MainProgram
+program OMPDOSCHEDULE
+  !DEF: /OMPDOSCHEDULE/a ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/y ObjectEntity REAL(4)
+  !DEF: /OMPDOSCHEDULE/z ObjectEntity REAL(4)
   real  a(100),y(100),z(100)
-  !DEF: /ompdoschedule/b ObjectEntity INTEGER(4)
-  !DEF: /ompdoschedule/i ObjectEntity INTEGER(4)
-  !DEF: /ompdoschedule/n ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/b ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/i ObjectEntity INTEGER(4)
+  !DEF: /OMPDOSCHEDULE/n ObjectEntity INTEGER(4)
   integer  b,i,n
-  !REF: /ompdoschedule/b
+  !REF: /OMPDOSCHEDULE/b
   b = 10
   !$omp do  schedule(static,b-b)
-  !DEF: /ompdoschedule/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !REF: /ompdoschedule/n
+  !DEF: /OMPDOSCHEDULE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !REF: /OMPDOSCHEDULE/n
   do i = 2,n+1
-    !REF: /ompdoschedule/y
-    !REF: /ompdoschedule/OtherConstruct1/i
-    !REF: /ompdoschedule/z
-    !REF: /ompdoschedule/a
+    !REF: /OMPDOSCHEDULE/y
+    !REF: /OMPDOSCHEDULE/OtherConstruct1/i
+    !REF: /OMPDOSCHEDULE/z
+    !REF: /OMPDOSCHEDULE/a
     y(i) = z(i-1) + a(i)
   end do
   !$omp end do
-end program ompdoschedule
+end program OMPDOSCHEDULE

diff  --git a/flang/test/Semantics/OpenMP/do01-positivecase.f90 b/flang/test/Semantics/OpenMP/do01-positivecase.f90
index 905fdbaf18476..50a6870f43896 100644
--- a/flang/test/Semantics/OpenMP/do01-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do01-positivecase.f90
@@ -4,16 +4,16 @@
 ! The loop iteration variable may not appear in a firstprivate directive.
 ! A positive case
 
-!DEF: /omp_do MainProgram
-program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
+!DEF: /OMP_DO MainProgram
+program OMP_DO
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
   integer i
 
   !$omp do  firstprivate(k)
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
     print *, "Hello"
   end do
   !$omp end do
 
-end program omp_do
+end program OMP_DO

diff  --git a/flang/test/Semantics/OpenMP/do04-positivecase.f90 b/flang/test/Semantics/OpenMP/do04-positivecase.f90
index eb2d67bb8ceb2..51b69fce3c7cc 100644
--- a/flang/test/Semantics/OpenMP/do04-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do04-positivecase.f90
@@ -2,21 +2,21 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 
-!DEF: /omp_do1 MainProgram
-program omp_do1
-  !DEF: /omp_do1/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
-  !DEF: /omp_do1/n (OmpThreadprivate) ObjectEntity INTEGER(4)
+!DEF: /OMP_DO1 MainProgram
+program OMP_DO1
+  !DEF: /OMP_DO1/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/k (OmpThreadprivate) ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO1/n (OmpThreadprivate) ObjectEntity INTEGER(4)
   integer i, j, k, n
   !$omp threadprivate (k,n)
   !$omp do
-  !DEF: /omp_do1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do1/j
+    !REF: /OMP_DO1/j
     do j=1,10
       print *, "Hello"
     end do
   end do
   !$omp end do
-end program omp_do1
+end program OMP_DO1

diff  --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index eda04610535c2..d4eb1fd6bc3da 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -3,13 +3,13 @@
 ! 2.7.1 Loop Construct restrictions on single directive.
 ! A positive case
 
-!DEF: /omp_do MainProgram
-program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/n ObjectEntity INTEGER(4)
+!DEF: /OMP_DO MainProgram
+program OMP_DO
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/n ObjectEntity INTEGER(4)
   integer i,n
   !$omp parallel
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
     !$omp single
     print *, "hello"
@@ -19,13 +19,13 @@ program omp_do
 
   !$omp parallel  default(shared)
   !$omp do
-  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
   do i=1,n
     !$omp parallel
     !$omp single
     !DEF: /work EXTERNAL (Subroutine) ProcEntity
-    !DEF: /omp_do/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
+    !DEF: /OMP_DO/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
     call work(i, 1)
     !$omp end single
     !$omp end parallel
@@ -34,7 +34,7 @@ program omp_do
   !$omp end parallel
 
   !$omp parallel private(i)
-  !DEF: /omp_do/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
   do i=1,10
      !$omp single
      print *, "hello"
@@ -43,32 +43,32 @@ program omp_do
   !$omp end parallel
 
   !$omp target teams distribute parallel do
-  !DEF:/omp_do/OtherConstruct4/i (OmpPrivate ,OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF:/OMP_DO/OtherConstruct4/i (OmpPrivate ,OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF:/omp_do/OtherConstruct4/i
+    !REF:/OMP_DO/OtherConstruct4/i
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do
 
   !$omp target teams distribute parallel do simd
-  !DEF:/omp_do/OtherConstruct5/i (OmpLinear,OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF:/OMP_DO/OtherConstruct5/i (OmpLinear,OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF:/omp_do/OtherConstruct5/i
+    !REF:/OMP_DO/OtherConstruct5/i
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute 
-  !DEF: /omp_do/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF: /omp_do/OtherConstruct6/i
+    !REF: /OMP_DO/OtherConstruct6/i
     if(i < 5) cycle
   end do
 
   !$omp target teams distribute simd
-  !DEF: /omp_do/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,100
-    !REF: /omp_do/OtherConstruct7/i
+    !REF: /OMP_DO/OtherConstruct7/i
     if(i < 5) cycle
   end do
-end program omp_do
+end program OMP_DO

diff  --git a/flang/test/Semantics/OpenMP/do06-positivecases.f90 b/flang/test/Semantics/OpenMP/do06-positivecases.f90
index 2713b55fa2ecb..dfb1d999bbc53 100644
--- a/flang/test/Semantics/OpenMP/do06-positivecases.f90
+++ b/flang/test/Semantics/OpenMP/do06-positivecases.f90
@@ -5,14 +5,14 @@
 ! region ever binds to a loop region arising from the loop construct.
 
 ! A positive case
-!DEF: /omp_do MainProgram
-program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do/k ObjectEntity INTEGER(4)
+!DEF: /OMP_DO MainProgram
+program OMP_DO
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
   integer i, j, k
   !$omp do  ordered
-    !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do i=1,10
       !$omp ordered
       !DEF: /my_func EXTERNAL (Subroutine) ProcEntity
@@ -20,4 +20,4 @@ program omp_do
       !$omp end ordered
     end do
   !$omp end do
-end program omp_do
+end program OMP_DO

diff  --git a/flang/test/Semantics/OpenMP/do11.f90 b/flang/test/Semantics/OpenMP/do11.f90
index faab457efff3c..472048d684276 100644
--- a/flang/test/Semantics/OpenMP/do11.f90
+++ b/flang/test/Semantics/OpenMP/do11.f90
@@ -2,24 +2,24 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 
-!DEF: /omp_do MainProgram
-program omp_do
-  !DEF: /omp_do/i ObjectEntity INTEGER(4)
-  !DEF: /omp_do/j ObjectEntity INTEGER(4)
-  !DEF: /omp_do/k ObjectEntity INTEGER(4)
+!DEF: /OMP_DO MainProgram
+program OMP_DO
+  !DEF: /OMP_DO/i ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/j ObjectEntity INTEGER(4)
+  !DEF: /OMP_DO/k ObjectEntity INTEGER(4)
   integer i, j, k
   !$omp do
-  !DEF: /omp_do/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_DO/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !REF: /omp_do/j
+    !REF: /OMP_DO/j
     do j=1,10
-      !REF: /omp_do/OtherConstruct1/i
-      !REF: /omp_do/j
+      !REF: /OMP_DO/OtherConstruct1/i
+      !REF: /OMP_DO/j
       print *, "it", i, j
     end do
   end do
   !$omp end do
-end program omp_do
+end program OMP_DO
 
 !DEF: /omp_do2 (Subroutine)Subprogram
 subroutine omp_do2

diff  --git a/flang/test/Semantics/OpenMP/do12.f90 b/flang/test/Semantics/OpenMP/do12.f90
index a057a246f7a99..06055b7572a60 100644
--- a/flang/test/Semantics/OpenMP/do12.f90
+++ b/flang/test/Semantics/OpenMP/do12.f90
@@ -2,20 +2,20 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /omp_cycle MainProgram
-program omp_cycle
+!DEF: /OMP_CYCLE MainProgram
+program OMP_CYCLE
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/OtherConstruct1/i
+    !REF: /OMP_CYCLE/OtherConstruct1/i
     if (i<1) cycle
-    !DEF: /omp_cycle/j (Implicit) ObjectEntity INTEGER(4)
+    !DEF: /OMP_CYCLE/j (Implicit) ObjectEntity INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/k (Implicit) ObjectEntity INTEGER(4)
+      !DEF: /OMP_CYCLE/k (Implicit) ObjectEntity INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct1/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct1/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -23,17 +23,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/j
+    !REF: /OMP_CYCLE/j
     do j=0,10
-      !REF: /omp_cycle/OtherConstruct2/i
+      !REF: /OMP_CYCLE/OtherConstruct2/i
       if (i<1) cycle
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct2/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct2/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -41,17 +41,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(2)
-  !DEF: /omp_cycle/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/i
         if (i<1) cycle
-        !REF: /omp_cycle/OtherConstruct3/i
-        !REF: /omp_cycle/OtherConstruct3/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -59,17 +59,17 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/i
         if (i<1) cycle
-        !REF: /omp_cycle/OtherConstruct4/i
-        !REF: /omp_cycle/OtherConstruct4/j
-        !REF: /omp_cycle/OtherConstruct4/k
+        !REF: /OMP_CYCLE/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/j
+        !REF: /OMP_CYCLE/OtherConstruct4/k
         print *, i, j, k
       end do
     end do
@@ -77,20 +77,20 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo:do i=0,10
-    !DEF: /omp_cycle/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     foo1:do j=0,10
-      !DEF: /omp_cycle/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       foo2:do k=0,10
-        !REF: /omp_cycle/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/i
         if (i<1) cycle foo2
-        !REF: /omp_cycle/OtherConstruct5/i
-        !REF: /omp_cycle/OtherConstruct5/j
-        !REF: /omp_cycle/OtherConstruct5/k
+        !REF: /OMP_CYCLE/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/j
+        !REF: /OMP_CYCLE/OtherConstruct5/k
         print *, i, j, k
       end do foo2
     end do foo1
   end do foo
   !$omp end do
-end program omp_cycle
+end program OMP_CYCLE

diff  --git a/flang/test/Semantics/OpenMP/do14.f90 b/flang/test/Semantics/OpenMP/do14.f90
index 5e8a5a64c2979..e17647394fff7 100644
--- a/flang/test/Semantics/OpenMP/do14.f90
+++ b/flang/test/Semantics/OpenMP/do14.f90
@@ -2,19 +2,19 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /omp_cycle MainProgram
-program omp_cycle
+!DEF: /OMP_CYCLE MainProgram
+program OMP_CYCLE
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
     cycle
-    !DEF: /omp_cycle/j (Implicit) ObjectEntity INTEGER(4)
+    !DEF: /OMP_CYCLE/j (Implicit) ObjectEntity INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/k (Implicit) ObjectEntity INTEGER(4)
+      !DEF: /OMP_CYCLE/k (Implicit) ObjectEntity INTEGER(4)
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct1/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct1/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -22,16 +22,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(1)
-  !DEF: /omp_cycle/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !REF: /omp_cycle/j
+    !REF: /OMP_CYCLE/j
     do j=0,10
       cycle
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
-        !REF: /omp_cycle/OtherConstruct2/i
-        !REF: /omp_cycle/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct2/i
+        !REF: /OMP_CYCLE/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -39,16 +39,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(2)
-  !DEF: /omp_cycle/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct3/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !REF: /omp_cycle/k
+      !REF: /OMP_CYCLE/k
       do k=0,10
         cycle
-        !REF: /omp_cycle/OtherConstruct3/i
-        !REF: /omp_cycle/OtherConstruct3/j
-        !REF: /omp_cycle/k
+        !REF: /OMP_CYCLE/OtherConstruct3/i
+        !REF: /OMP_CYCLE/OtherConstruct3/j
+        !REF: /OMP_CYCLE/k
         print *, i, j, k
       end do
     end do
@@ -56,16 +56,16 @@ program omp_cycle
   !$omp end do
 
   !$omp do  collapse(3)
-  !DEF: /omp_cycle/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=0,10
-    !DEF: /omp_cycle/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct4/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=0,10
-      !DEF: /omp_cycle/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct4/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       do k=0,10
         cycle
-        !REF: /omp_cycle/OtherConstruct4/i
-        !REF: /omp_cycle/OtherConstruct4/j
-        !REF: /omp_cycle/OtherConstruct4/k
+        !REF: /OMP_CYCLE/OtherConstruct4/i
+        !REF: /OMP_CYCLE/OtherConstruct4/j
+        !REF: /OMP_CYCLE/OtherConstruct4/k
         print *, i, j, k
       end do
     end do
@@ -73,19 +73,19 @@ program omp_cycle
   !$omp end do
 
   !$omp do  ordered(3)
-  !DEF: /omp_cycle/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_CYCLE/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo:do i=0,10
-    !DEF: /omp_cycle/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    !DEF: /OMP_CYCLE/OtherConstruct5/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     foo1:do j=0,10
-      !DEF: /omp_cycle/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      !DEF: /OMP_CYCLE/OtherConstruct5/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
       foo2:do k=0,10
         cycle foo2
-        !REF: /omp_cycle/OtherConstruct5/i
-        !REF: /omp_cycle/OtherConstruct5/j
-        !REF: /omp_cycle/OtherConstruct5/k
+        !REF: /OMP_CYCLE/OtherConstruct5/i
+        !REF: /OMP_CYCLE/OtherConstruct5/j
+        !REF: /OMP_CYCLE/OtherConstruct5/k
         print *, i, j, k
       end do foo2
     end do foo1
   end do foo
   !$omp end do
-end program omp_cycle
+end program OMP_CYCLE

diff  --git a/flang/test/Semantics/OpenMP/do17.f90 b/flang/test/Semantics/OpenMP/do17.f90
index c0c59f16dee1b..cac11f215f074 100644
--- a/flang/test/Semantics/OpenMP/do17.f90
+++ b/flang/test/Semantics/OpenMP/do17.f90
@@ -2,56 +2,56 @@
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 
-!DEF: /test MainProgram
-program test
- !DEF: /test/i ObjectEntity INTEGER(4)
- !DEF: /test/j ObjectEntity INTEGER(4)
- !DEF: /test/k ObjectEntity INTEGER(4)
+!DEF: /TEST MainProgram
+program TEST
+ !DEF: /TEST/i ObjectEntity INTEGER(4)
+ !DEF: /TEST/j ObjectEntity INTEGER(4)
+ !DEF: /TEST/k ObjectEntity INTEGER(4)
  integer i, j, k
  !$omp do  collapse(2)
- !DEF: /test/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /TEST/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  foo: do i=0,10
-  !DEF: /test/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /TEST/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo1: do j=0,10
-   !REF: /test/k
+   !REF: /TEST/k
    foo2: do k=0,10
-    !REF: /test/OtherConstruct1/i
+    !REF: /TEST/OtherConstruct1/i
     select case (i)
     case (5)
      cycle foo1
     case (7)
      cycle foo2
     end select
-    !REF: /test/OtherConstruct1/i
-    !REF: /test/OtherConstruct1/j
-    !REF: /test/k
+    !REF: /TEST/OtherConstruct1/i
+    !REF: /TEST/OtherConstruct1/j
+    !REF: /TEST/k
     print *, i, j, k
    end do foo2
   end do foo1
  end do foo
 
  !$omp do  collapse(2)
- !DEF: /test/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /TEST/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  foo: do i=0,10
-  !DEF: /test/OtherConstruct2/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /TEST/OtherConstruct2/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   foo1: do j=0,10
-   !REF: /test/k
+   !REF: /TEST/k
    foo2: do k=0,10
-    !REF: /test/OtherConstruct2/i
+    !REF: /TEST/OtherConstruct2/i
     if (i<3) then
      cycle foo1
-     !REF: /test/OtherConstruct2/i
+     !REF: /TEST/OtherConstruct2/i
     else if (i>8) then
      cycle foo1
     else
      cycle foo2
     end if
-    !REF: /test/OtherConstruct2/i
-    !REF: /test/OtherConstruct2/j
-    !REF: /test/k
+    !REF: /TEST/OtherConstruct2/i
+    !REF: /TEST/OtherConstruct2/j
+    !REF: /TEST/k
     print *, i, j, k
    end do foo2
   end do foo1
  end do foo
 !$omp end do
-end program test
+end program TEST

diff  --git a/flang/test/Semantics/OpenMP/map-clause-symbols.f90 b/flang/test/Semantics/OpenMP/map-clause-symbols.f90
index 8f984fcd2fa7e..1d6315b4a2312 100644
--- a/flang/test/Semantics/OpenMP/map-clause-symbols.f90
+++ b/flang/test/Semantics/OpenMP/map-clause-symbols.f90
@@ -1,6 +1,6 @@
 ! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
 program main
-!CHECK-LABEL:  MainProgram scope: main
+!CHECK-LABEL:  MainProgram scope: MAIN
   integer, parameter :: n = 256
   real(8) :: a(256)
   !$omp target map(mapper(xx), from:a)

diff  --git a/flang/test/Semantics/OpenMP/reduction08.f90 b/flang/test/Semantics/OpenMP/reduction08.f90
index 01a06eb7d7414..b4a81e644c1e7 100644
--- a/flang/test/Semantics/OpenMP/reduction08.f90
+++ b/flang/test/Semantics/OpenMP/reduction08.f90
@@ -2,62 +2,62 @@
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases
 
-!DEF: /omp_reduction MainProgram
-program omp_reduction
-  !DEF: /omp_reduction/i ObjectEntity INTEGER(4)
+!DEF: /OMP_REDUCTION MainProgram
+program OMP_REDUCTION
+  !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
-  !DEF: /omp_reduction/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
   integer :: k = 10
-  !DEF: /omp_reduction/m ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/m ObjectEntity INTEGER(4)
   integer :: m = 12
   !$omp parallel do  reduction(max:k)
-  !DEF: /omp_reduction/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
     k = max(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(min:k)
-  !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
     k = min(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(iand:k)
-  !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
     k = iand(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(ior:k)
-  !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
     k = ior(k, m)
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(ieor:k)
-  !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
-    !DEF: /omp_reduction/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+    !DEF: /OMP_REDUCTION/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
     k = ieor(k,m)
   end do
   !$omp end parallel do
 
-end program omp_reduction
+end program OMP_REDUCTION

diff  --git a/flang/test/Semantics/OpenMP/reduction09.f90 b/flang/test/Semantics/OpenMP/reduction09.f90
index d6c71c30d2834..ca60805e8c416 100644
--- a/flang/test/Semantics/OpenMP/reduction09.f90
+++ b/flang/test/Semantics/OpenMP/reduction09.f90
@@ -1,22 +1,22 @@
 ! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases.
-!DEF: /omp_reduction MainProgram
-program omp_reduction
-  !DEF: /omp_reduction/i ObjectEntity INTEGER(4)
+!DEF: /OMP_REDUCTION MainProgram
+program OMP_REDUCTION
+  !DEF: /OMP_REDUCTION/i ObjectEntity INTEGER(4)
   integer i
-  !DEF: /omp_reduction/k ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/k ObjectEntity INTEGER(4)
   integer :: k = 10
-  !DEF: /omp_reduction/a ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/a ObjectEntity INTEGER(4)
   integer a(10)
-  !DEF: /omp_reduction/b ObjectEntity INTEGER(4)
+  !DEF: /OMP_REDUCTION/b ObjectEntity INTEGER(4)
   integer b(10,10,10)
 
   !$omp parallel  shared(k)
   !$omp do  reduction(+:k)
-  !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
@@ -24,53 +24,53 @@ program omp_reduction
 
 
   !$omp parallel do  reduction(+:a(10))
-  !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
 
   !$omp parallel do  reduction(+:a(1:10:1))
-  !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2))
-  !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2:5:1))
-  !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
 
   !$omp parallel  private(i)
   !$omp do reduction(+:k) reduction(+:j)
-  !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
   !$omp end parallel
 
   !$omp do reduction(+:k) reduction(*:j) reduction(+:l)
-  !DEF: /omp_reduction/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /OMP_REDUCTION/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /OMP_REDUCTION/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
-end program omp_reduction
+end program OMP_REDUCTION

diff  --git a/flang/test/Semantics/OpenMP/reduction11.f90 b/flang/test/Semantics/OpenMP/reduction11.f90
index b2ad0f6a6ee11..dfb3986d37d78 100644
--- a/flang/test/Semantics/OpenMP/reduction11.f90
+++ b/flang/test/Semantics/OpenMP/reduction11.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols -o - %s 2>&1 | FileCheck %s
 ! Check intrinsic reduction symbols (in this case "max" are marked as INTRINSIC
 
-! CHECK: MainProgram scope: omp_reduction
+! CHECK: MainProgram scope: OMP_REDUCTION
 program omp_reduction
   ! CHECK: i size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer i

diff  --git a/flang/test/Semantics/OpenMP/scan2.f90 b/flang/test/Semantics/OpenMP/scan2.f90
index ffe84910f88a2..1ae5e871595c4 100644
--- a/flang/test/Semantics/OpenMP/scan2.f90
+++ b/flang/test/Semantics/OpenMP/scan2.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols -o - %s 2>&1 | FileCheck %s
 ! Check scan reduction
 
-! CHECK: MainProgram scope: omp_reduction
+! CHECK: MainProgram scope: OMP_REDUCTION
 program omp_reduction
   ! CHECK: i size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer i

diff  --git a/flang/test/Semantics/OpenMP/symbol01.f90 b/flang/test/Semantics/OpenMP/symbol01.f90
index fbd9a0286c79b..74fb420cc517e 100644
--- a/flang/test/Semantics/OpenMP/symbol01.f90
+++ b/flang/test/Semantics/OpenMP/symbol01.f90
@@ -16,53 +16,53 @@ module md
   integer :: b
  end type myty
 end module md
-!DEF: /mm MainProgram
-program mm
+!DEF: /MM MainProgram
+program MM
  !REF: /md
  use :: md
- !DEF: /mm/c CommonBlockDetails
- !DEF: /mm/x (InCommonBlock) ObjectEntity REAL(4)
- !DEF: /mm/y (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /MM/c CommonBlockDetails
+ !DEF: /MM/x (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /MM/y (InCommonBlock) ObjectEntity REAL(4)
  common /c/x, y
- !REF: /mm/x
- !REF: /mm/y
+ !REF: /MM/x
+ !REF: /MM/y
  real x, y
- !DEF: /mm/myty Use
- !DEF: /mm/t ObjectEntity TYPE(myty)
+ !DEF: /MM/myty Use
+ !DEF: /MM/t ObjectEntity TYPE(myty)
  type(myty) :: t
- !DEF: /mm/b ObjectEntity INTEGER(4)
+ !DEF: /MM/b ObjectEntity INTEGER(4)
  integer b(10)
- !REF: /mm/t
+ !REF: /MM/t
  !REF: /md/myty/a
  t%a = 3.14
- !REF: /mm/t
+ !REF: /MM/t
  !REF: /md/myty/b
  t%b = 1
- !REF: /mm/b
+ !REF: /MM/b
  b = 2
- !DEF: /mm/a (Implicit) ObjectEntity REAL(4)
+ !DEF: /MM/a (Implicit) ObjectEntity REAL(4)
  a = 1.0
- !DEF: /mm/c (Implicit) ObjectEntity REAL(4)
+ !DEF: /MM/c (Implicit) ObjectEntity REAL(4)
  c = 2.0
 !$omp parallel do  private(a,t,/c/) shared(c)
- !DEF: /mm/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+ !DEF: /MM/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  do i=1,10
-  !DEF: /mm/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
-  !DEF: /mm/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
-  !REF: /mm/OtherConstruct1/i
+  !DEF: /MM/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
+  !REF: /MM/OtherConstruct1/i
   a = a+b(i)
-  !DEF: /mm/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
+  !DEF: /MM/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
   !REF: /md/myty/a
-  !REF: /mm/OtherConstruct1/i
+  !REF: /MM/OtherConstruct1/i
   t%a = i
-  !DEF: /mm/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   y = 0.
-  !DEF: /mm/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
-  !REF: /mm/OtherConstruct1/a
-  !REF: /mm/OtherConstruct1/i
-  !REF: /mm/OtherConstruct1/y
+  !DEF: /MM/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !REF: /MM/OtherConstruct1/a
+  !REF: /MM/OtherConstruct1/i
+  !REF: /MM/OtherConstruct1/y
   x = a+i+y
-  !DEF: /mm/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /MM/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
   c = 3.0
  end do
 end program

diff  --git a/flang/test/Semantics/OpenMP/symbol05.f90 b/flang/test/Semantics/OpenMP/symbol05.f90
index fe01f15d20aa3..4f3d1926013dc 100644
--- a/flang/test/Semantics/OpenMP/symbol05.f90
+++ b/flang/test/Semantics/OpenMP/symbol05.f90
@@ -31,10 +31,10 @@ subroutine foo
     end block
   end subroutine foo
 end module mm
-!DEF: /tt MainProgram
-program tt
+!DEF: /TT MainProgram
+program TT
   !REF: /mm
   use :: mm
-  !DEF: /tt/foo (Subroutine) Use
+  !DEF: /TT/foo (Subroutine) Use
   call foo
-end program tt
+end program TT

diff  --git a/flang/test/Semantics/OpenMP/symbol07.f90 b/flang/test/Semantics/OpenMP/symbol07.f90
index 86b7305411347..1b0c25b7a04b0 100644
--- a/flang/test/Semantics/OpenMP/symbol07.f90
+++ b/flang/test/Semantics/OpenMP/symbol07.f90
@@ -30,8 +30,8 @@ subroutine function_call_in_region
   !REF: /function_call_in_region/b
   print *, a, b
 end subroutine function_call_in_region
-!DEF: /mm MainProgram
-program mm
+!DEF: /MM MainProgram
+program MM
   !REF: /function_call_in_region
   call function_call_in_region
-end program mm
+end program MM

diff  --git a/flang/test/Semantics/OpenMP/symbol09.f90 b/flang/test/Semantics/OpenMP/symbol09.f90
index 86b7305411347..1b0c25b7a04b0 100644
--- a/flang/test/Semantics/OpenMP/symbol09.f90
+++ b/flang/test/Semantics/OpenMP/symbol09.f90
@@ -30,8 +30,8 @@ subroutine function_call_in_region
   !REF: /function_call_in_region/b
   print *, a, b
 end subroutine function_call_in_region
-!DEF: /mm MainProgram
-program mm
+!DEF: /MM MainProgram
+program MM
   !REF: /function_call_in_region
   call function_call_in_region
-end program mm
+end program MM

diff  --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index 81e26ee327a9d..fda2fe608ac3c 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -10,11 +10,11 @@ program main
   use mod1
   integer, parameter :: i = 1
 
-  !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
+  !ERROR: The module name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
-  !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
+  ! This is now allowed, since "main" is implicitly declared symbol,
+  ! separate from the main program symbol.
   !$omp threadprivate(main)
 
   !ERROR: The entity with PARAMETER attribute cannot be in a THREADPRIVATE directive

diff  --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index 95b7fb418367d..5c5e87575a9cb 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -8,7 +8,7 @@ program main
 end program
 
 ! RUN: %flang_fc1 -fget-symbols-sources %s 2>&1 | FileCheck %s
+! CHECK:MAIN:{{.*}}getsymbols03-a.f90, 4, 9-13
 ! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
-! CHECK:main:{{.*}}getsymbols03-a.f90, 4, 9-13
 ! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
 ! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14

diff  --git a/flang/test/Semantics/long-name.f90 b/flang/test/Semantics/long-name.f90
index 44899b13edd5a..d5a795113e204 100644
--- a/flang/test/Semantics/long-name.f90
+++ b/flang/test/Semantics/long-name.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror -pedantic
 
-!PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
+!PORTABILITY: AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEEFFFFFFFFFFGGG1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
 program aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1
 
   !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2 has length 64, which is greater than the maximum name length 63 [-Wlong-names]

diff  --git a/flang/test/Semantics/modproc01.f90 b/flang/test/Semantics/modproc01.f90
index 5f45362e95093..e565ddcfbe0b1 100644
--- a/flang/test/Semantics/modproc01.f90
+++ b/flang/test/Semantics/modproc01.f90
@@ -125,7 +125,7 @@ program test
   x = mf(3, "abc", pdt1(1,3)())
 !  call ms(mf)
 end program
-!CHECK:  MainProgram scope: test size=88 alignment=8
+!CHECK:  MainProgram scope: TEST size=88 alignment=8
 !CHECK:    mf, MODULE (Function): Use from mf in m
 !CHECK:    pdt1: Use from pdt1 in m
 !CHECK:    pdt2: Use from pdt2 in m

diff  --git a/flang/test/Semantics/multi-programs04.f90 b/flang/test/Semantics/multi-programs04.f90
index 54b0235aa78f0..e69ac7325278e 100644
--- a/flang/test/Semantics/multi-programs04.f90
+++ b/flang/test/Semantics/multi-programs04.f90
@@ -4,6 +4,6 @@
 program m
 end
 !ERROR: A source file cannot contain more than one main program
-!ERROR: 'm' is already declared in this scoping unit
+!ERROR: 'M' is already declared in this scoping unit
 program m
 end

diff  --git a/flang/test/Semantics/pointer01.f90 b/flang/test/Semantics/pointer01.f90
index eaa2426dd77e3..79d6016a6af46 100644
--- a/flang/test/Semantics/pointer01.f90
+++ b/flang/test/Semantics/pointer01.f90
@@ -7,7 +7,6 @@ subroutine msubr
 end module
 program main
   use m
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   pointer main
   !ERROR: Cannot change POINTER attribute on use-associated 'mobj'
   pointer mobj

diff  --git a/flang/test/Semantics/procinterface01.f90 b/flang/test/Semantics/procinterface01.f90
index 73040b0987bd0..70f4a889d6809 100644
--- a/flang/test/Semantics/procinterface01.f90
+++ b/flang/test/Semantics/procinterface01.f90
@@ -159,35 +159,35 @@ end function logical
  tan = "?"
 end function tan
 
-!DEF: /main MainProgram
-program main
+!DEF: /MAIN MainProgram
+program MAIN
  !REF: /module1
  use :: module1
- !DEF: /main/derived1 Use
- !DEF: /main/instance ObjectEntity TYPE(derived1)
+ !DEF: /MAIN/derived1 Use
+ !DEF: /MAIN/instance ObjectEntity TYPE(derived1)
  type(derived1) :: instance
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p1
  if (instance%p1(1.)/=2.) print *, "p1 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p2
  if (instance%p2(1.)/=2.) print *, "p2 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p3
  if (.not.instance%p3(1.)) print *, "p3 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p4
  if (.not.instance%p4(1.)) print *, "p4 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p5
  if (instance%p5(1.)/=(5.,6.)) print *, "p5 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p6
  if (instance%p6(1.)/=2.) print *, "p6 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p7
  if (instance%p7(0.)/=1.) print *, "p7 failed"
- !REF: /main/instance
+ !REF: /MAIN/instance
  !REF: /module1/derived1/p8
  if (instance%p8(1.)/="a") print *, "p8 failed"
-end program main
+end program MAIN

diff  --git a/flang/test/Semantics/resolve05.f90 b/flang/test/Semantics/resolve05.f90
index 0c9877af9b4e2..7b142d2ebd613 100644
--- a/flang/test/Semantics/resolve05.f90
+++ b/flang/test/Semantics/resolve05.f90
@@ -1,6 +1,5 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 program p
-  !PORTABILITY: Name 'p' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   integer :: p
 end
 module m

diff  --git a/flang/test/Semantics/resolve125.f90 b/flang/test/Semantics/resolve125.f90
index e040c006ec179..620c7d65578cd 100644
--- a/flang/test/Semantics/resolve125.f90
+++ b/flang/test/Semantics/resolve125.f90
@@ -43,7 +43,7 @@ subroutine reset
   end subroutine reset
 end module m2
 
-!CHECK: MainProgram scope: main
+!CHECK: MainProgram scope: MAIN
 !CHECK:       i: Use from i in m2
 !CHECK:       i2: Use from i2 in m2
 !CHECK:       init (Subroutine): Use from init in m2
@@ -61,4 +61,4 @@ program main
   else
     print *, "fail"
   end if
-end program main
\ No newline at end of file
+end program main

diff  --git a/flang/test/Semantics/symbol03.f90 b/flang/test/Semantics/symbol03.f90
index a6b4b0bd15937..62472495d9736 100644
--- a/flang/test/Semantics/symbol03.f90
+++ b/flang/test/Semantics/symbol03.f90
@@ -1,23 +1,23 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Test host association in internal subroutine of main program.
 
-!DEF: /main MainProgram
-program main
- !DEF: /main/x ObjectEntity INTEGER(4)
+!DEF: /MAIN MainProgram
+program MAIN
+ !DEF: /MAIN/x ObjectEntity INTEGER(4)
  integer x
- !DEF: /main/s (Subroutine) Subprogram
+ !DEF: /MAIN/s (Subroutine) Subprogram
  call s
 contains
- !REF: /main/s
+ !REF: /MAIN/s
  subroutine s
-  !DEF: /main/s/y (Implicit) ObjectEntity REAL(4)
-  !DEF: /main/s/x HostAssoc INTEGER(4)
+  !DEF: /MAIN/s/y (Implicit) ObjectEntity REAL(4)
+  !DEF: /MAIN/s/x HostAssoc INTEGER(4)
   y = x
  contains
-  !DEF: /main/s/s2 (Subroutine) Subprogram
+  !DEF: /MAIN/s/s2 (Subroutine) Subprogram
   subroutine s2
-   !DEF: /main/s/s2/z (Implicit) ObjectEntity REAL(4)
-   !DEF: /main/s/s2/x HostAssoc INTEGER(4)
+   !DEF: /MAIN/s/s2/z (Implicit) ObjectEntity REAL(4)
+   !DEF: /MAIN/s/s2/x HostAssoc INTEGER(4)
    z = x
   end subroutine
  end subroutine

diff  --git a/flang/test/Semantics/symbol06.f90 b/flang/test/Semantics/symbol06.f90
index bbd6d4d071c89..b45edabcd5318 100644
--- a/flang/test/Semantics/symbol06.f90
+++ b/flang/test/Semantics/symbol06.f90
@@ -1,56 +1,56 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
-program main
- !DEF: /main/t1 DerivedType
+!DEF: /MAIN MainProgram
+program MAIN
+ !DEF: /MAIN/t1 DerivedType
  type :: t1
-  !DEF: /main/t1/a1 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t1/a1 ObjectEntity INTEGER(4)
   integer :: a1
  end type
- !REF: /main/t1
- !DEF: /main/t2 DerivedType
+ !REF: /MAIN/t1
+ !DEF: /MAIN/t2 DerivedType
  type, extends(t1) :: t2
-  !DEF: /main/t2/a2 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t2/a2 ObjectEntity INTEGER(4)
   integer :: a2
  end type
- !REF: /main/t2
- !DEF: /main/t3 DerivedType
+ !REF: /MAIN/t2
+ !DEF: /MAIN/t3 DerivedType
  type, extends(t2) :: t3
-  !DEF: /main/t3/a3 ObjectEntity INTEGER(4)
+  !DEF: /MAIN/t3/a3 ObjectEntity INTEGER(4)
   integer :: a3
  end type
- !REF: /main/t3
- !DEF: /main/x3 ObjectEntity TYPE(t3)
+ !REF: /MAIN/t3
+ !DEF: /MAIN/x3 ObjectEntity TYPE(t3)
  type(t3) :: x3
- !DEF: /main/i ObjectEntity INTEGER(4)
+ !DEF: /MAIN/i ObjectEntity INTEGER(4)
  integer i
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t2/a2
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t2/a2
  i = x3%a2
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t1/a1
  i = x3%a1
- !REF: /main/i
- !REF: /main/x3
- !DEF: /main/t3/t2 (ParentComp) ObjectEntity TYPE(t2)
- !REF: /main/t2/a2
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !DEF: /MAIN/t3/t2 (ParentComp) ObjectEntity TYPE(t2)
+ !REF: /MAIN/t2/a2
  i = x3%t2%a2
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t3/t2
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t3/t2
+ !REF: /MAIN/t1/a1
  i = x3%t2%a1
- !REF: /main/i
- !REF: /main/x3
- !DEF: /main/t2/t1 (ParentComp) ObjectEntity TYPE(t1)
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !DEF: /MAIN/t2/t1 (ParentComp) ObjectEntity TYPE(t1)
+ !REF: /MAIN/t1/a1
  i = x3%t1%a1
- !REF: /main/i
- !REF: /main/x3
- !REF: /main/t3/t2
- !REF: /main/t2/t1
- !REF: /main/t1/a1
+ !REF: /MAIN/i
+ !REF: /MAIN/x3
+ !REF: /MAIN/t3/t2
+ !REF: /MAIN/t2/t1
+ !REF: /MAIN/t1/a1
  i = x3%t2%t1%a1
 end program
 

diff  --git a/flang/test/Semantics/symbol07.f90 b/flang/test/Semantics/symbol07.f90
index f3cc934e51b16..e1d8257b9e190 100644
--- a/flang/test/Semantics/symbol07.f90
+++ b/flang/test/Semantics/symbol07.f90
@@ -1,40 +1,40 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
-program main
+!DEF: /MAIN MainProgram
+program MAIN
  implicit complex(z)
- !DEF: /main/t DerivedType
+ !DEF: /MAIN/t DerivedType
  type :: t
-  !DEF: /main/t/re ObjectEntity REAL(4)
+  !DEF: /MAIN/t/re ObjectEntity REAL(4)
   real :: re
-  !DEF: /main/t/im ObjectEntity REAL(4)
+  !DEF: /MAIN/t/im ObjectEntity REAL(4)
   real :: im
  end type
- !DEF: /main/z1 ObjectEntity COMPLEX(4)
+ !DEF: /MAIN/z1 ObjectEntity COMPLEX(4)
  complex z1
- !REF: /main/t
- !DEF: /main/w ObjectEntity TYPE(t)
+ !REF: /MAIN/t
+ !DEF: /MAIN/w ObjectEntity TYPE(t)
  type(t) :: w
- !DEF: /main/x ObjectEntity REAL(4)
- !DEF: /main/y ObjectEntity REAL(4)
+ !DEF: /MAIN/x ObjectEntity REAL(4)
+ !DEF: /MAIN/y ObjectEntity REAL(4)
  real x, y
- !REF: /main/x
- !REF: /main/z1
+ !REF: /MAIN/x
+ !REF: /MAIN/z1
  x = z1%re
- !REF: /main/y
- !REF: /main/z1
+ !REF: /MAIN/y
+ !REF: /MAIN/z1
  y = z1%im
- !DEF: /main/z2 (Implicit) ObjectEntity COMPLEX(4)
- !REF: /main/x
+ !DEF: /MAIN/z2 (Implicit) ObjectEntity COMPLEX(4)
+ !REF: /MAIN/x
  z2%re = x
- !REF: /main/z2
- !REF: /main/y
+ !REF: /MAIN/z2
+ !REF: /MAIN/y
  z2%im = y
- !REF: /main/x
- !REF: /main/w
- !REF: /main/t/re
+ !REF: /MAIN/x
+ !REF: /MAIN/w
+ !REF: /MAIN/t/re
  x = w%re
- !REF: /main/y
- !REF: /main/w
- !REF: /main/t/im
+ !REF: /MAIN/y
+ !REF: /MAIN/w
+ !REF: /MAIN/t/im
  y = w%im
 end program

diff  --git a/flang/test/Semantics/symbol08.f90 b/flang/test/Semantics/symbol08.f90
index 61dab798955c5..933ff6d0c2ba8 100644
--- a/flang/test/Semantics/symbol08.f90
+++ b/flang/test/Semantics/symbol08.f90
@@ -1,15 +1,15 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
-!DEF: /main MainProgram
-program main
- !DEF: /main/x POINTER ObjectEntity REAL(4)
+!DEF: /MAIN MainProgram
+program MAIN
+ !DEF: /MAIN/x POINTER ObjectEntity REAL(4)
  pointer :: x
- !REF: /main/x
+ !REF: /MAIN/x
  real x
- !DEF: /main/y EXTERNAL, POINTER (Function) ProcEntity REAL(4)
+ !DEF: /MAIN/y EXTERNAL, POINTER (Function) ProcEntity REAL(4)
  pointer :: y
- !REF: /main/y
+ !REF: /MAIN/y
  procedure (real) :: y
- !DEF: /main/z (Implicit) ObjectEntity REAL(4)
- !REF: /main/y
+ !DEF: /MAIN/z (Implicit) ObjectEntity REAL(4)
+ !REF: /MAIN/y
  z = y()
 end program

diff  --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index df10942e6af2d..79a45491306ef 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -249,15 +249,15 @@ subroutine ext2
 !DEF: /ext3 (Subroutine) Subprogram
 subroutine ext3
 end subroutine
-!DEF: /main MainProgram
-program main
+!DEF: /MAIN MainProgram
+program MAIN
  !REF: /m
  use :: m
- !DEF: /main/pdt1 Use
- !DEF: /main/pdt1y ObjectEntity TYPE(pdt1(k=2_4))
+ !DEF: /MAIN/pdt1 Use
+ !DEF: /MAIN/pdt1y ObjectEntity TYPE(pdt1(k=2_4))
  type(pdt1(2)) :: pdt1y
- !DEF: /main/pdt2 Use
- !DEF: /main/pdt2y ObjectEntity TYPE(pdt2(k=2_4))
+ !DEF: /MAIN/pdt2 Use
+ !DEF: /MAIN/pdt2y ObjectEntity TYPE(pdt2(k=2_4))
  type(pdt2(2)) :: pdt2y
  print *, "compiled"
 end program

diff  --git a/flang/test/Semantics/symbol16.f90 b/flang/test/Semantics/symbol16.f90
index 7a46092c36b53..547c4624d4cdb 100644
--- a/flang/test/Semantics/symbol16.f90
+++ b/flang/test/Semantics/symbol16.f90
@@ -1,18 +1,18 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Statement functions
 
-!DEF: /p1 MainProgram
-program p1
- !DEF: /p1/f (Function, StmtFunction) Subprogram INTEGER(4)
- !DEF: /p1/i ObjectEntity INTEGER(4)
- !DEF: /p1/j ObjectEntity INTEGER(4)
+!DEF: /P1 MainProgram
+program P1
+ !DEF: /P1/f (Function, StmtFunction) Subprogram INTEGER(4)
+ !DEF: /P1/i ObjectEntity INTEGER(4)
+ !DEF: /P1/j ObjectEntity INTEGER(4)
  integer f, i, j
- !REF: /p1/f
- !REF: /p1/i
- !DEF: /p1/f/i ObjectEntity INTEGER(4)
+ !REF: /P1/f
+ !REF: /P1/i
+ !DEF: /P1/f/i ObjectEntity INTEGER(4)
  f(i) = i + 1
- !REF: /p1/j
- !REF: /p1/f
+ !REF: /P1/j
+ !REF: /P1/f
  j = f(2)
 end program
 

diff  --git a/flang/test/Semantics/symbol17.f90 b/flang/test/Semantics/symbol17.f90
index 434f124509a32..a0d916e55cfa4 100644
--- a/flang/test/Semantics/symbol17.f90
+++ b/flang/test/Semantics/symbol17.f90
@@ -1,44 +1,44 @@
 ! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Forward references to derived types (non-error cases)
 
-!DEF: /main MainProgram
-program main
- !DEF: /main/t1 DerivedType
+!DEF: /MAIN MainProgram
+program MAIN
+ !DEF: /MAIN/t1 DerivedType
  type :: t1
-  !DEF: /main/t2 DerivedType
-  !DEF: /main/t1/t1a ALLOCATABLE ObjectEntity TYPE(t2)
+  !DEF: /MAIN/t2 DerivedType
+  !DEF: /MAIN/t1/t1a ALLOCATABLE ObjectEntity TYPE(t2)
   type(t2), allocatable :: t1a
-  !REF: /main/t2
-  !DEF: /main/t1/t1p POINTER ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t1/t1p POINTER ObjectEntity TYPE(t2)
   type(t2), pointer :: t1p
  end type
- !REF: /main/t2
+ !REF: /MAIN/t2
  type :: t2
-  !REF: /main/t2
-  !DEF: /main/t2/t2a ALLOCATABLE ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t2/t2a ALLOCATABLE ObjectEntity TYPE(t2)
   type(t2), allocatable :: t2a
-  !REF: /main/t2
-  !DEF: /main/t2/t2p POINTER ObjectEntity TYPE(t2)
+  !REF: /MAIN/t2
+  !DEF: /MAIN/t2/t2p POINTER ObjectEntity TYPE(t2)
   type(t2), pointer :: t2p
  end type
- !REF: /main/t1
- !DEF: /main/t1x TARGET ObjectEntity TYPE(t1)
+ !REF: /MAIN/t1
+ !DEF: /MAIN/t1x TARGET ObjectEntity TYPE(t1)
  type(t1), target :: t1x
- !REF: /main/t1x
- !REF: /main/t1/t1a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
  allocate(t1x%t1a)
- !REF: /main/t1x
- !REF: /main/t1/t1p
- !REF: /main/t1/t1a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1p
+ !REF: /MAIN/t1/t1a
  t1x%t1p => t1x%t1a
- !REF: /main/t1x
- !REF: /main/t1/t1a
- !REF: /main/t2/t2a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
+ !REF: /MAIN/t2/t2a
  allocate(t1x%t1a%t2a)
- !REF: /main/t1x
- !REF: /main/t1/t1a
- !REF: /main/t2/t2p
- !REF: /main/t2/t2a
+ !REF: /MAIN/t1x
+ !REF: /MAIN/t1/t1a
+ !REF: /MAIN/t2/t2p
+ !REF: /MAIN/t2/t2a
  t1x%t1a%t2p => t1x%t1a%t2a
 end program
 !DEF: /f1/fwd DerivedType

diff  --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
index a37792bce21d7..6e41bb5db91ee 100644
--- a/flang/test/Semantics/symbol18.f90
+++ b/flang/test/Semantics/symbol18.f90
@@ -2,21 +2,21 @@
 
 ! Intrinsic function in type declaration statement: type is ignored
 
-!DEF: /p1 MainProgram
-program p1
- !DEF: /p1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
+!DEF: /P1 MainProgram
+program P1
+ !DEF: /P1/cos ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity INTEGER(4)
  integer cos
- !DEF: /p1/y (Implicit) ObjectEntity REAL(4)
- !REF: /p1/cos
- !DEF: /p1/x (Implicit) ObjectEntity REAL(4)
+ !DEF: /P1/y (Implicit) ObjectEntity REAL(4)
+ !REF: /P1/cos
+ !DEF: /P1/x (Implicit) ObjectEntity REAL(4)
  y = cos(x)
- !REF: /p1/y
- !DEF: /p1/sin ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
- !REF: /p1/x
+ !REF: /P1/y
+ !DEF: /P1/sin ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
+ !REF: /P1/x
  y = sin(x)
- !REF: /p1/y
+ !REF: /P1/y
  !DEF: /f EXTERNAL (Function, Implicit) ProcEntity REAL(4)
- !REF: /p1/x
+ !REF: /P1/x
  y = f(x)
 end program
 

diff  --git a/flang/test/Semantics/symbol20.f90 b/flang/test/Semantics/symbol20.f90
index 8c82776933321..bf3aff489b3b9 100644
--- a/flang/test/Semantics/symbol20.f90
+++ b/flang/test/Semantics/symbol20.f90
@@ -32,16 +32,16 @@ subroutine bar
   print *, "in bar"
  end subroutine
 end module
-!DEF: /demo MainProgram
-program demo
+!DEF: /DEMO MainProgram
+program DEMO
  !REF: /m
  use :: m
- !DEF: /demo/bar (Subroutine) Use
- !DEF: /demo/p EXTERNAL, POINTER (Subroutine) ProcEntity
+ !DEF: /DEMO/bar (Subroutine) Use
+ !DEF: /DEMO/p EXTERNAL, POINTER (Subroutine) ProcEntity
  procedure(bar), pointer :: p
- !REF: /demo/p
- !DEF: /demo/foo (Function) Use
+ !REF: /DEMO/p
+ !DEF: /DEMO/foo (Function) Use
  p => foo()
- !REF: /demo/p
+ !REF: /DEMO/p
  call p
 end program

diff  --git a/flang/test/Semantics/symbol25.f90 b/flang/test/Semantics/symbol25.f90
index ac3dd37ef92eb..ac47a19eae8cc 100644
--- a/flang/test/Semantics/symbol25.f90
+++ b/flang/test/Semantics/symbol25.f90
@@ -38,23 +38,23 @@ subroutine inner1
   end subroutine inner1
  end subroutine outer
 end module m
-!DEF: /main MainProgram
-program main
+!DEF: /MAIN MainProgram
+program MAIN
  !REF: /m
  use :: m
  !REF: /m/specific1
  call generic
- !DEF: /main/inner2 (Subroutine) Subprogram
+ !DEF: /MAIN/inner2 (Subroutine) Subprogram
  call inner2
 contains
- !REF: /main/inner2
+ !REF: /MAIN/inner2
  subroutine inner2
-  !DEF: /main/inner2/generic (Subroutine) Generic
+  !DEF: /MAIN/inner2/generic (Subroutine) Generic
   interface generic
-   !DEF: /main/specific2 (Subroutine) Use
+   !DEF: /MAIN/specific2 (Subroutine) Use
    module procedure :: specific2
   end interface
-  !REF: /main/specific2
+  !REF: /MAIN/specific2
   call generic
  end subroutine inner2
 end program

diff  --git a/flang/test/Semantics/symbol26.f90 b/flang/test/Semantics/symbol26.f90
index f5e95853ca099..dded4b632c654 100644
--- a/flang/test/Semantics/symbol26.f90
+++ b/flang/test/Semantics/symbol26.f90
@@ -8,16 +8,16 @@ module m
  !DEF: /m/j PUBLIC (Implicit, InNamelist) ObjectEntity INTEGER(4)
  namelist/a/j
 end module m
-!DEF: /main MainProgram
-program main
- !DEF: /main/j (Implicit) ObjectEntity INTEGER(4)
+!DEF: /MAIN MainProgram
+program MAIN
+ !DEF: /MAIN/j (Implicit) ObjectEntity INTEGER(4)
  j = 1
 contains
- !DEF: /main/inner (Subroutine) Subprogram
+ !DEF: /MAIN/inner (Subroutine) Subprogram
  subroutine inner
   !REF: /m
   use :: m
-  !DEF: /main/inner/j (Implicit, InNamelist) Use INTEGER(4)
+  !DEF: /MAIN/inner/j (Implicit, InNamelist) Use INTEGER(4)
   j = 2
  end subroutine
 end program

diff  --git a/flang/test/Transforms/DoConcurrent/basic_host.f90 b/flang/test/Transforms/DoConcurrent/basic_host.f90
index 12f63031cbaee..6f24b346e3fb9 100644
--- a/flang/test/Transforms/DoConcurrent/basic_host.f90
+++ b/flang/test/Transforms/DoConcurrent/basic_host.f90
@@ -5,7 +5,7 @@
 ! RUN: bbc -emit-hlfir -fopenmp -fdo-concurrent-to-openmp=host %s -o - \
 ! RUN:   | FileCheck %s
  
-! CHECK-LABEL: do_concurrent_basic
+! CHECK-LABEL: DO_CONCURRENT_BASIC
 program do_concurrent_basic
     ! CHECK: %[[ARR:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFEa"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 


        

From flang-commits at lists.llvm.org  Thu Jul 17 12:00:23 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Thu, 17 Jul 2025 12:00:23 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Control alignment of constant
	folded reals (PR #149381)
Message-ID: <llvm.org/llvm/llvm-project/pull/149381@github.com>


https://github.com/eugeneepshteyn created https://github.com/llvm/llvm-project/pull/149381

When REAL types are constant folded, the underneath implementation uses arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the resulting code for flang compiler ended up using SSE2 aligned load instructions for REAL(16) constant folding on x86_64, and these instructions require that the values are loaded from the aligned addresses.

>From a155a355cc1cef2169c37d069ad72b4c180f0633 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 14:56:11 -0400
Subject: [PATCH] [flang] Control alignment of constant folded reals

When REAL types are constant folded, the underneath implementation uses
arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the
resulting code for flang compiler ended up using SSE2 aligned load
instructions for REAL(16) constant folding on x86_64, and these
instructions require that the values are loaded from the aligned
addresses.
---
 flang/include/flang/Evaluate/integer.h | 1 +
 flang/include/flang/Evaluate/real.h    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index fccc2ad774a8f..5953fc81cb111 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,6 +74,7 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
+  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 03294881850a1..fa74792a0377f 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,7 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  Word word_{}; // an Integer<>
+  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format


From flang-commits at lists.llvm.org  Thu Jul 17 12:16:08 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Thu, 17 Jul 2025 12:16:08 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Generalize
 isOpenMPPrivatizingConstruct (PR #148654)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148654@github.com>
Message-ID: <68794bf8.170a0220.3a0bf2.00b3@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `clang-m68k-linux-cross` running on `suse-gary-m68k-cross` while building `flang,llvm` at step 5 "ninja check 1".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/27/builds/13239

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 5 (ninja check 1) failure: stage 1 checked (failure)
...
[232/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/IntegerLiteral.cpp.o
[233/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/InitListExprPreOrder.cpp.o
[234/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/LambdaDefaultCapture.cpp.o
[235/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/LambdaTemplateParams.cpp.o
[236/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/MemberPointerTypeLoc.cpp.o
[237/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/LambdaExpr.cpp.o
[238/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/NestedNameSpecifiers.cpp.o
[239/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/ParenExpr.cpp.o
[240/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/TemplateArgumentLocTraverser.cpp.o
[241/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/AST/ASTImporterTest.cpp.o
FAILED: tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/AST/ASTImporterTest.cpp.o 
/usr/bin/c++ -DEXPERIMENTAL_KEY_INSTRUCTIONS -DGTEST_HAS_RTTI=0 -DLLVM_BUILD_STATIC -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/stage1/tools/clang/unittests -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/llvm/clang/unittests -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/llvm/clang/include -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/stage1/tools/clang/include -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/stage1/include -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/llvm/llvm/include -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/llvm/clang/unittests/Tooling -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/llvm/third-party/unittest/googletest/include -I/var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/llvm/third-party/unittest/googlemock/include -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough -Wno-uninitialized -Wno-nonnull -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wdelete-non-virtual-dtor -Wsuggest-override -Wno-comment -Wno-misleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -fno-common -Woverloaded-virtual -O3 -DNDEBUG -std=c++17  -Wno-variadic-macros -fno-exceptions -funwind-tables -fno-rtti -UNDEBUG -Wno-suggest-override -MD -MT tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/AST/ASTImporterTest.cpp.o -MF tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/AST/ASTImporterTest.cpp.o.d -o tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/AST/ASTImporterTest.cpp.o -c /var/lib/buildbot/workers/suse-gary-m68k-cross/clang-m68k-linux-cross/llvm/clang/unittests/AST/ASTImporterTest.cpp
c++: fatal error: Killed signal terminated program cc1plus
compilation terminated.
[242/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/Syntax/TreeTestBase.cpp.o
[243/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/InitListExprPostOrder.cpp.o
[244/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/InitListExprPostOrderNoQueue.cpp.o
[245/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/TraversalScope.cpp.o
[246/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTestDeclVisitor.cpp.o
[247/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTestTypeLocVisitor.cpp.o
[248/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RefactoringActionRulesTest.cpp.o
[249/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/ReplacementsYamlTest.cpp.o
[250/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/Syntax/MutationsTest.cpp.o
[251/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/Syntax/SynthesisTest.cpp.o
[252/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Frontend/CompilerInstanceTest.cpp.o
[253/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Frontend/ASTUnitTest.cpp.o
[254/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/Syntax/TreeTest.cpp.o
[255/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/SourceCodeBuildersTest.cpp.o
[256/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RefactoringTest.cpp.o
[257/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RefactoringCallbacksTest.cpp.o
[258/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/CallbacksLeaf.cpp.o
[259/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/CallbacksCompoundAssignOperator.cpp.o
[260/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Frontend/CompilerInvocationTest.cpp.o
[261/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/SourceCodeTest.cpp.o
[262/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/Syntax/TokensTest.cpp.o
[263/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/Syntax/BuildTreeTest.cpp.o
[264/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/CallbacksBinaryOperator.cpp.o
[265/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/StencilTest.cpp.o
[266/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/ToolingTest.cpp.o
[267/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTestPostOrderVisitor.cpp.o
[268/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/CallbacksUnaryOperator.cpp.o
[269/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/RecursiveASTVisitorTests/CallbacksCallExpr.cpp.o
[270/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Tooling/TransformerTest.cpp.o
[271/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/Analysis/FlowSensitive/TransferTest.cpp.o
[272/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/ASTMatchers/ASTMatchersNarrowingTest.cpp.o
[273/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/ASTMatchers/ASTMatchersNodeTest.cpp.o
[274/379] Building CXX object tools/clang/unittests/CMakeFiles/AllClangUnitTests.dir/ASTMatchers/ASTMatchersTraversalTest.cpp.o
ninja: build stopped: subcommand failed.

```

</details>

https://github.com/llvm/llvm-project/pull/148654

From flang-commits at lists.llvm.org  Thu Jul 17 12:43:24 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Thu, 17 Jul 2025 12:43:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Control alignment of constant
	folded reals (PR #149381)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149381@github.com>
Message-ID: <6879525c.170a0220.1a0372.f014@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/149381

>From a155a355cc1cef2169c37d069ad72b4c180f0633 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 14:56:11 -0400
Subject: [PATCH 1/2] [flang] Control alignment of constant folded reals

When REAL types are constant folded, the underneath implementation uses
arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the
resulting code for flang compiler ended up using SSE2 aligned load
instructions for REAL(16) constant folding on x86_64, and these
instructions require that the values are loaded from the aligned
addresses.
---
 flang/include/flang/Evaluate/integer.h | 1 +
 flang/include/flang/Evaluate/real.h    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index fccc2ad774a8f..5953fc81cb111 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,6 +74,7 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
+  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 03294881850a1..fa74792a0377f 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,7 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  Word word_{}; // an Integer<>
+  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format

>From 43102f9b97cc055124b7e0e730cecf71fecdc29a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 15:43:08 -0400
Subject: [PATCH 2/2] Added comment

---
 flang/include/flang/Evaluate/real.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index fa74792a0377f..af17adc8ad8a3 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,6 +490,9 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
+  // Require alignment, in case code generation code on x86_64 decides that
+  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 


From flang-commits at lists.llvm.org  Thu Jul 17 15:57:28 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Thu, 17 Jul 2025 15:57:28 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <68797fd8.a70a0220.8cf23.02e9@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 1/6] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 2/6] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 3/6] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 4/6] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 5/6] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 6/6] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 


From flang-commits at lists.llvm.org  Thu Jul 17 18:06:18 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Thu, 17 Jul 2025 18:06:18 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][runtime] Preserve type when
 remapping monomorphic pointers (PR #149427)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149427@github.com>
Message-ID: <68799e0a.170a0220.329538.0e0d@mx.google.com>


https://github.com/clementval approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/149427

From flang-commits at lists.llvm.org  Thu Jul 17 22:28:35 2025
From: flang-commits at lists.llvm.org (Kazu Hirata via flang-commits)
Date: Thu, 17 Jul 2025 22:28:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Migrate away from
	ArrayRef(std::nullopt_t) (PR #149454)
Message-ID: <llvm.org/llvm/llvm-project/pull/149454@github.com>


https://github.com/kazutakahirata created https://github.com/llvm/llvm-project/pull/149454

ArrayRef(std::nullopt_t) has been deprecated.  This patch replaces
std::nullopt with mlir::TypeRange{} or mlir::ValueRange{} as
appropriate.


>From 6c6d7cf43e8942e626add5fc58c3a3f63653161b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Wed, 16 Jul 2025 13:20:33 -0700
Subject: [PATCH] [flang] Migrate away from ArrayRef(std::nullopt_t)

ArrayRef(std::nullopt_t) has been deprecated.  This patch replaces
std::nullopt with mlir::TypeRange{} or mlir::ValueRange{} as
appropriate.
---
 flang/lib/Lower/Bridge.cpp                    | 15 ++++++-----
 flang/lib/Lower/ConvertConstant.cpp           |  2 +-
 flang/lib/Lower/ConvertExpr.cpp               | 26 ++++++++++---------
 flang/lib/Lower/Runtime.cpp                   |  4 +--
 flang/lib/Lower/VectorSubscripts.cpp          |  2 +-
 flang/lib/Optimizer/Builder/FIRBuilder.cpp    |  2 +-
 .../Optimizer/Builder/Runtime/Intrinsics.cpp  |  6 +++--
 flang/lib/Optimizer/Builder/Runtime/Stop.cpp  |  2 +-
 .../LowerHLFIROrderedAssignments.cpp          |  4 +--
 .../lib/Optimizer/Transforms/MemoryUtils.cpp  |  2 +-
 .../Transforms/PolymorphicOpConversion.cpp    |  2 +-
 .../Transforms/SimplifyFIROperations.cpp      |  2 +-
 .../Optimizer/FortranVariableTest.cpp         |  6 ++---
 13 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4241d12601242..5f0783f869bf6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1466,8 +1466,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     assert(falseTarget && "missing conditional branch false block");
     mlir::Location loc = toLocation();
     mlir::Value bcc = builder->createConvert(loc, builder->getI1Type(), cond);
-    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget, std::nullopt,
-                                            falseTarget, std::nullopt);
+    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget,
+                                            mlir::ValueRange{}, falseTarget,
+                                            mlir::ValueRange{});
   }
   void genConditionalBranch(mlir::Value cond,
                             Fortran::lower::pft::Evaluation *trueTarget,
@@ -2556,8 +2557,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
           loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
-          /*local_vars=*/std::nullopt,
-          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*local_vars=*/mlir::ValueRange{},
+          /*local_syms=*/nullptr, /*reduce_vars=*/mlir::ValueRange{},
           /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
           /*reduce_attrs=*/nullptr);
 
@@ -3810,9 +3811,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       mlir::Block *selectCaseBlock = insertBlock(blockList[0]);
       mlir::Block *assumedSizeBlock =
           rankStarBlock ? rankStarBlock : defaultBlock;
-      builder->create<mlir::cf::CondBranchOp>(loc, isAssumedSize,
-                                              assumedSizeBlock, std::nullopt,
-                                              selectCaseBlock, std::nullopt);
+      builder->create<mlir::cf::CondBranchOp>(
+          loc, isAssumedSize, assumedSizeBlock, mlir::ValueRange{},
+          selectCaseBlock, mlir::ValueRange{});
       startBlock(selectCaseBlock);
     }
     // Create fir.select_case for the other rank cases.
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1850b67898126..b8ab5d09e3e08 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -303,7 +303,7 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
     return builder.create<fir::StringLitOp>(
-        loc, llvm::ArrayRef<mlir::Type>{type}, std::nullopt, attrs);
+        loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
   }
 }
 
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 0a1cd67789772..281ab229d1b6a 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1003,9 +1003,9 @@ class ScalarExprLowering {
           },
           [&](const fir::MutableBoxValue &toBox) {
             if (toBox.isPointer()) {
-              Fortran::lower::associateMutableBox(converter, loc, toBox, expr,
-                                                  /*lbounds=*/std::nullopt,
-                                                  stmtCtx);
+              Fortran::lower::associateMutableBox(
+                  converter, loc, toBox, expr,
+                  /*lbounds=*/mlir::ValueRange{}, stmtCtx);
               return;
             }
             // For allocatable components, a deep copy is needed.
@@ -3604,8 +3604,9 @@ class ArrayExprLowering {
       mlir::Value castTo =
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
-      return builder.create<fir::ArrayLoadOp>(
-          loc, seqTy, castTo, shapeOp, /*slice=*/mlir::Value{}, std::nullopt);
+      return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
+                                              /*slice=*/mlir::Value{},
+                                              mlir::ValueRange{});
     };
     // Custom lowering of the element store to deal with the extra indirection
     // to the lazy allocated buffer.
@@ -4207,7 +4208,7 @@ class ArrayExprLowering {
       auto addr =
           builder->create<fir::ArrayCoorOp>(loc, eleRefTy, tmp, shape,
                                             /*slice=*/mlir::Value{}, indices,
-                                            /*typeParams=*/std::nullopt);
+                                            /*typeParams=*/mlir::ValueRange{});
       auto load = builder->create<fir::LoadOp>(loc, addr);
       return builder->createConvert(loc, i1Ty, load);
     };
@@ -4522,17 +4523,18 @@ class ArrayExprLowering {
         fir::isRecordWithAllocatableMember(eleTy))
       TODO(loc, "creating an array temp where the element type has "
                 "allocatable members");
-    mlir::Value temp = !seqTy.hasDynamicExtents()
-                           ? builder.create<fir::AllocMemOp>(loc, type)
-                           : builder.create<fir::AllocMemOp>(
-                                 loc, type, ".array.expr", std::nullopt, shape);
+    mlir::Value temp =
+        !seqTy.hasDynamicExtents()
+            ? builder.create<fir::AllocMemOp>(loc, type)
+            : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
+                                              mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
     return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
                                             /*slice=*/mlir::Value{},
-                                            std::nullopt);
+                                            mlir::ValueRange{});
   }
 
   static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -6483,7 +6485,7 @@ class ArrayExprLowering {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
         mem = builder.create<fir::AllocMemOp>(
-            loc, eleTy, /*typeparams=*/std::nullopt, initBuffSz);
+            loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
         builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
       }
     } else {
diff --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index 2be5ef76e46b8..5f73335242336 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
   genUnreachable(builder, loc);
 }
 
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
 }
 
 void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index 389a89ddcf102..c7b3e11728cea 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -122,7 +122,7 @@ class VectorSubscriptBoxBuilder {
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
-        loc, fldTy, componentName, recTy, /*typeParams*/ std::nullopt));
+        loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
 
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index acd5a88a2582d..5b1dbc4435d6c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -620,7 +620,7 @@ fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc,
   mlir::NamedAttribute sizeAttr(sizeTag, getI64IntegerAttr(data.size()));
   llvm::SmallVector<mlir::NamedAttribute> attrs{dataAttr, sizeAttr};
   return create<fir::StringLitOp>(loc, llvm::ArrayRef<mlir::Type>{type},
-                                  std::nullopt, attrs);
+                                  mlir::ValueRange{}, attrs);
 }
 
 mlir::Value fir::FirOpBuilder::genShape(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 773d6408079cc..04703f7911176 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -59,7 +59,8 @@ mlir::Value fir::runtime::genCpuTime(fir::FirOpBuilder &builder,
                                      mlir::Location loc) {
   mlir::func::FuncOp func =
       fir::runtime::getRuntimeFunc<mkRTKey(CpuTime)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
@@ -280,7 +281,8 @@ void fir::runtime::genRename(fir::FirOpBuilder &builder, mlir::Location loc,
 mlir::Value fir::runtime::genTime(fir::FirOpBuilder &builder,
                                   mlir::Location loc) {
   auto func = fir::runtime::getRuntimeFunc<mkRTKey(time)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 /// generate runtime call to transfer intrinsic with no size argument
diff --git a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
index 411181cc6dd1c..9b5e43b80b1f6 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
@@ -25,7 +25,7 @@ void fir::runtime::genExit(fir::FirOpBuilder &builder, mlir::Location loc,
 void fir::runtime::genAbort(fir::FirOpBuilder &builder, mlir::Location loc) {
   mlir::func::FuncOp abortFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(Abort)>(loc, builder);
-  builder.create<fir::CallOp>(loc, abortFunc, std::nullopt);
+  builder.create<fir::CallOp>(loc, abortFunc, mlir::ValueRange{});
 }
 
 void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 03cc92e975b19..c5cf01ed98357 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -405,7 +405,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
   mlir::Location loc = forallMaskOp.getLoc();
   mlir::Value mask = generateYieldedScalarValue(forallMaskOp.getMaskRegion(),
                                                 builder.getI1Type());
-  auto ifOp = builder.create<fir::IfOp>(loc, std::nullopt, mask, false);
+  auto ifOp = builder.create<fir::IfOp>(loc, mlir::TypeRange{}, mask, false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   constructStack.push_back(ifOp);
 }
@@ -530,7 +530,7 @@ void OrderedAssignmentRewriter::generateMaskIfOp(mlir::Value cdt) {
   mlir::Location loc = cdt.getLoc();
   cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
   cdt = builder.createConvert(loc, builder.getI1Type(), cdt);
-  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), std::nullopt, cdt,
+  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), mlir::TypeRange{}, cdt,
                                         /*withElseRegion=*/false);
   constructStack.push_back(ifOp.getOperation());
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
diff --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index 1f8edf851de9b..bc4fcd8b0112e 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -222,7 +222,7 @@ void AllocaReplaceImpl::genIndirectDeallocation(
         rewriter.create<fir::ConvertOp>(loc, intPtrTy, ptrVal);
     mlir::Value isAllocated = rewriter.create<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::ne, ptrToInt, c0);
-    auto ifOp = rewriter.create<fir::IfOp>(loc, std::nullopt, isAllocated,
+    auto ifOp = rewriter.create<fir::IfOp>(loc, mlir::TypeRange{}, isAllocated,
                                            /*withElseRegion=*/false);
     rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value cast = fir::factory::createConvert(
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 57eae1ff052a2..6e45aae4246d0 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -456,7 +456,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
   rewriter.setInsertionPointToEnd(thisBlock);
   if (destOps.has_value())
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, destOps.value(),
-                                            newBlock, std::nullopt);
+                                            newBlock, mlir::ValueRange{});
   else
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, newBlock);
   rewriter.setInsertionPointToEnd(newBlock);
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 506c8e66dbdfa..ad8464b495888 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -261,7 +261,7 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
+          /*iterArgs=*/mlir::ValueRange{}, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index 98270adaa7c73..59808779aa6ef 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -48,7 +48,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
   mlir::Value addr = builder->create<fir::AllocaOp>(loc, eleType);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      /*shape=*/mlir::Value{}, /*typeParams=*/std::nullopt,
+      /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{},
       /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
@@ -102,11 +102,11 @@ TEST_F(FortranVariableTest, SimpleArray) {
       extents.size(), fir::SequenceType::getUnknownExtent());
   mlir::Type seqTy = fir::SequenceType::get(typeShape, eleType);
   mlir::Value addr = builder->create<fir::AllocaOp>(
-      loc, seqTy, /*pinned=*/false, /*typeParams=*/std::nullopt, extents);
+      loc, seqTy, /*pinned=*/false, /*typeParams=*/mlir::ValueRange{}, extents);
   mlir::Value shape = createShape(extents);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      shape, /*typeParams*/ std::nullopt, /*dummy_scope=*/nullptr, name,
+      shape, /*typeParams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
 


From flang-commits at lists.llvm.org  Thu Jul 17 22:29:06 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Thu, 17 Jul 2025 22:29:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Migrate away from
	ArrayRef(std::nullopt_t) (PR #149454)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149454@github.com>
Message-ID: <6879dba2.170a0220.129548.18f6@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Kazu Hirata (kazutakahirata)

<details>
<summary>Changes</summary>

ArrayRef(std::nullopt_t) has been deprecated.  This patch replaces
std::nullopt with mlir::TypeRange{} or mlir::ValueRange{} as
appropriate.


---
Full diff: https://github.com/llvm/llvm-project/pull/149454.diff


13 Files Affected:

- (modified) flang/lib/Lower/Bridge.cpp (+8-7) 
- (modified) flang/lib/Lower/ConvertConstant.cpp (+1-1) 
- (modified) flang/lib/Lower/ConvertExpr.cpp (+14-12) 
- (modified) flang/lib/Lower/Runtime.cpp (+2-2) 
- (modified) flang/lib/Lower/VectorSubscripts.cpp (+1-1) 
- (modified) flang/lib/Optimizer/Builder/FIRBuilder.cpp (+1-1) 
- (modified) flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp (+4-2) 
- (modified) flang/lib/Optimizer/Builder/Runtime/Stop.cpp (+1-1) 
- (modified) flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp (+2-2) 
- (modified) flang/lib/Optimizer/Transforms/MemoryUtils.cpp (+1-1) 
- (modified) flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp (+1-1) 
- (modified) flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp (+1-1) 
- (modified) flang/unittests/Optimizer/FortranVariableTest.cpp (+3-3) 


``````````diff
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4241d12601242..5f0783f869bf6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1466,8 +1466,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     assert(falseTarget && "missing conditional branch false block");
     mlir::Location loc = toLocation();
     mlir::Value bcc = builder->createConvert(loc, builder->getI1Type(), cond);
-    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget, std::nullopt,
-                                            falseTarget, std::nullopt);
+    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget,
+                                            mlir::ValueRange{}, falseTarget,
+                                            mlir::ValueRange{});
   }
   void genConditionalBranch(mlir::Value cond,
                             Fortran::lower::pft::Evaluation *trueTarget,
@@ -2556,8 +2557,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
           loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
-          /*local_vars=*/std::nullopt,
-          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*local_vars=*/mlir::ValueRange{},
+          /*local_syms=*/nullptr, /*reduce_vars=*/mlir::ValueRange{},
           /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
           /*reduce_attrs=*/nullptr);
 
@@ -3810,9 +3811,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       mlir::Block *selectCaseBlock = insertBlock(blockList[0]);
       mlir::Block *assumedSizeBlock =
           rankStarBlock ? rankStarBlock : defaultBlock;
-      builder->create<mlir::cf::CondBranchOp>(loc, isAssumedSize,
-                                              assumedSizeBlock, std::nullopt,
-                                              selectCaseBlock, std::nullopt);
+      builder->create<mlir::cf::CondBranchOp>(
+          loc, isAssumedSize, assumedSizeBlock, mlir::ValueRange{},
+          selectCaseBlock, mlir::ValueRange{});
       startBlock(selectCaseBlock);
     }
     // Create fir.select_case for the other rank cases.
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1850b67898126..b8ab5d09e3e08 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -303,7 +303,7 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
     return builder.create<fir::StringLitOp>(
-        loc, llvm::ArrayRef<mlir::Type>{type}, std::nullopt, attrs);
+        loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
   }
 }
 
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 0a1cd67789772..281ab229d1b6a 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1003,9 +1003,9 @@ class ScalarExprLowering {
           },
           [&](const fir::MutableBoxValue &toBox) {
             if (toBox.isPointer()) {
-              Fortran::lower::associateMutableBox(converter, loc, toBox, expr,
-                                                  /*lbounds=*/std::nullopt,
-                                                  stmtCtx);
+              Fortran::lower::associateMutableBox(
+                  converter, loc, toBox, expr,
+                  /*lbounds=*/mlir::ValueRange{}, stmtCtx);
               return;
             }
             // For allocatable components, a deep copy is needed.
@@ -3604,8 +3604,9 @@ class ArrayExprLowering {
       mlir::Value castTo =
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
-      return builder.create<fir::ArrayLoadOp>(
-          loc, seqTy, castTo, shapeOp, /*slice=*/mlir::Value{}, std::nullopt);
+      return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
+                                              /*slice=*/mlir::Value{},
+                                              mlir::ValueRange{});
     };
     // Custom lowering of the element store to deal with the extra indirection
     // to the lazy allocated buffer.
@@ -4207,7 +4208,7 @@ class ArrayExprLowering {
       auto addr =
           builder->create<fir::ArrayCoorOp>(loc, eleRefTy, tmp, shape,
                                             /*slice=*/mlir::Value{}, indices,
-                                            /*typeParams=*/std::nullopt);
+                                            /*typeParams=*/mlir::ValueRange{});
       auto load = builder->create<fir::LoadOp>(loc, addr);
       return builder->createConvert(loc, i1Ty, load);
     };
@@ -4522,17 +4523,18 @@ class ArrayExprLowering {
         fir::isRecordWithAllocatableMember(eleTy))
       TODO(loc, "creating an array temp where the element type has "
                 "allocatable members");
-    mlir::Value temp = !seqTy.hasDynamicExtents()
-                           ? builder.create<fir::AllocMemOp>(loc, type)
-                           : builder.create<fir::AllocMemOp>(
-                                 loc, type, ".array.expr", std::nullopt, shape);
+    mlir::Value temp =
+        !seqTy.hasDynamicExtents()
+            ? builder.create<fir::AllocMemOp>(loc, type)
+            : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
+                                              mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
     return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
                                             /*slice=*/mlir::Value{},
-                                            std::nullopt);
+                                            mlir::ValueRange{});
   }
 
   static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -6483,7 +6485,7 @@ class ArrayExprLowering {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
         mem = builder.create<fir::AllocMemOp>(
-            loc, eleTy, /*typeparams=*/std::nullopt, initBuffSz);
+            loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
         builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
       }
     } else {
diff --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index 2be5ef76e46b8..5f73335242336 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
   genUnreachable(builder, loc);
 }
 
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
 }
 
 void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index 389a89ddcf102..c7b3e11728cea 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -122,7 +122,7 @@ class VectorSubscriptBoxBuilder {
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
-        loc, fldTy, componentName, recTy, /*typeParams*/ std::nullopt));
+        loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
 
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index acd5a88a2582d..5b1dbc4435d6c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -620,7 +620,7 @@ fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc,
   mlir::NamedAttribute sizeAttr(sizeTag, getI64IntegerAttr(data.size()));
   llvm::SmallVector<mlir::NamedAttribute> attrs{dataAttr, sizeAttr};
   return create<fir::StringLitOp>(loc, llvm::ArrayRef<mlir::Type>{type},
-                                  std::nullopt, attrs);
+                                  mlir::ValueRange{}, attrs);
 }
 
 mlir::Value fir::FirOpBuilder::genShape(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 773d6408079cc..04703f7911176 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -59,7 +59,8 @@ mlir::Value fir::runtime::genCpuTime(fir::FirOpBuilder &builder,
                                      mlir::Location loc) {
   mlir::func::FuncOp func =
       fir::runtime::getRuntimeFunc<mkRTKey(CpuTime)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
@@ -280,7 +281,8 @@ void fir::runtime::genRename(fir::FirOpBuilder &builder, mlir::Location loc,
 mlir::Value fir::runtime::genTime(fir::FirOpBuilder &builder,
                                   mlir::Location loc) {
   auto func = fir::runtime::getRuntimeFunc<mkRTKey(time)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 /// generate runtime call to transfer intrinsic with no size argument
diff --git a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
index 411181cc6dd1c..9b5e43b80b1f6 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
@@ -25,7 +25,7 @@ void fir::runtime::genExit(fir::FirOpBuilder &builder, mlir::Location loc,
 void fir::runtime::genAbort(fir::FirOpBuilder &builder, mlir::Location loc) {
   mlir::func::FuncOp abortFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(Abort)>(loc, builder);
-  builder.create<fir::CallOp>(loc, abortFunc, std::nullopt);
+  builder.create<fir::CallOp>(loc, abortFunc, mlir::ValueRange{});
 }
 
 void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 03cc92e975b19..c5cf01ed98357 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -405,7 +405,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
   mlir::Location loc = forallMaskOp.getLoc();
   mlir::Value mask = generateYieldedScalarValue(forallMaskOp.getMaskRegion(),
                                                 builder.getI1Type());
-  auto ifOp = builder.create<fir::IfOp>(loc, std::nullopt, mask, false);
+  auto ifOp = builder.create<fir::IfOp>(loc, mlir::TypeRange{}, mask, false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   constructStack.push_back(ifOp);
 }
@@ -530,7 +530,7 @@ void OrderedAssignmentRewriter::generateMaskIfOp(mlir::Value cdt) {
   mlir::Location loc = cdt.getLoc();
   cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
   cdt = builder.createConvert(loc, builder.getI1Type(), cdt);
-  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), std::nullopt, cdt,
+  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), mlir::TypeRange{}, cdt,
                                         /*withElseRegion=*/false);
   constructStack.push_back(ifOp.getOperation());
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
diff --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index 1f8edf851de9b..bc4fcd8b0112e 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -222,7 +222,7 @@ void AllocaReplaceImpl::genIndirectDeallocation(
         rewriter.create<fir::ConvertOp>(loc, intPtrTy, ptrVal);
     mlir::Value isAllocated = rewriter.create<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::ne, ptrToInt, c0);
-    auto ifOp = rewriter.create<fir::IfOp>(loc, std::nullopt, isAllocated,
+    auto ifOp = rewriter.create<fir::IfOp>(loc, mlir::TypeRange{}, isAllocated,
                                            /*withElseRegion=*/false);
     rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value cast = fir::factory::createConvert(
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 57eae1ff052a2..6e45aae4246d0 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -456,7 +456,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
   rewriter.setInsertionPointToEnd(thisBlock);
   if (destOps.has_value())
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, destOps.value(),
-                                            newBlock, std::nullopt);
+                                            newBlock, mlir::ValueRange{});
   else
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, newBlock);
   rewriter.setInsertionPointToEnd(newBlock);
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 506c8e66dbdfa..ad8464b495888 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -261,7 +261,7 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
+          /*iterArgs=*/mlir::ValueRange{}, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index 98270adaa7c73..59808779aa6ef 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -48,7 +48,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
   mlir::Value addr = builder->create<fir::AllocaOp>(loc, eleType);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      /*shape=*/mlir::Value{}, /*typeParams=*/std::nullopt,
+      /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{},
       /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
@@ -102,11 +102,11 @@ TEST_F(FortranVariableTest, SimpleArray) {
       extents.size(), fir::SequenceType::getUnknownExtent());
   mlir::Type seqTy = fir::SequenceType::get(typeShape, eleType);
   mlir::Value addr = builder->create<fir::AllocaOp>(
-      loc, seqTy, /*pinned=*/false, /*typeParams=*/std::nullopt, extents);
+      loc, seqTy, /*pinned=*/false, /*typeParams=*/mlir::ValueRange{}, extents);
   mlir::Value shape = createShape(extents);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      shape, /*typeParams*/ std::nullopt, /*dummy_scope=*/nullptr, name,
+      shape, /*typeParams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
 

``````````

</details>


https://github.com/llvm/llvm-project/pull/149454

From flang-commits at lists.llvm.org  Thu Jul 17 23:20:19 2025
From: flang-commits at lists.llvm.org (=?UTF-8?B?Um9nZXIgRmVycmVyIEliw6HDsWV6?= via flang-commits)
Date: Thu, 17 Jul 2025 23:20:19 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <6879e7a3.050a0220.3931cc.1598@mx.google.com>


================
@@ -1143,6 +1143,83 @@ class OMPFullClause final : public OMPNoChildClause<llvm::omp::OMPC_full> {
   static OMPFullClause *CreateEmpty(const ASTContext &C);
 };
 
+/// This class represents the 'looprange' clause in the
+/// '#pragma omp fuse' directive
+///
+/// \code {c}
+/// #pragma omp fuse looprange(1,2)
+/// {
+///   for(int i = 0; i < 64; ++i)
+///   for(int j = 0; j < 256; j+=2)
+///   for(int k = 127; k >= 0; --k)
+/// \endcode
+class OMPLoopRangeClause final
+    : public OMPClause {
+  friend class OMPClauseReader;
+  /// Location of '('
+  SourceLocation LParenLoc;
+
+  /// Location of first and count expressions
+  SourceLocation FirstLoc, CountLoc;
+
+  /// Number of looprange arguments (always 2: first, count)
+  static constexpr unsigned NumArgs = 2;
+  Stmt *Args[NumArgs] = {nullptr, nullptr};
+
+  /// Set looprange 'first' expression
+  void setFirst(Expr *E) { Args[0] = E; }
+
+  /// Set looprange 'count' expression
+  void setCount(Expr *E) { Args[1] = E; }
+
+  /// Build an empty clause for deserialization.
+  explicit OMPLoopRangeClause()
+      : OMPClause(llvm::omp::OMPC_looprange, {}, {}) {}
+
+public:
+  /// Build a 'looprange' clause AST node.
+  static OMPLoopRangeClause *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
+         SourceLocation FirstLoc, SourceLocation CountLoc,
+         SourceLocation EndLoc, Expr* First, Expr* Count);
----------------
rofirrim wrote:

`clang-format` from LLVM 20 seems to be OK with this :thinking: 

https://github.com/llvm/llvm-project/pull/139293

From flang-commits at lists.llvm.org  Fri Jul 18 02:38:14 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Fri, 18 Jul 2025 02:38:14 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <687a1606.a70a0220.22ca67.1f9c@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `flang` at step 6 "test-build-unified-tree-check-flang".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/34156

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (test-build-unified-tree-check-flang) failure: test (failure)
******************** TEST 'Flang :: Semantics/PowerPC/ppc-vector-types01.f90' FAILED ********************
Exit Code: 1

Command Output (stderr):
--
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -fdebug-unparse /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Semantics/PowerPC/ppc-vector-types01.f90 | /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Semantics/PowerPC/ppc-vector-types01.f90 # RUN: at line 1
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -fdebug-unparse /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Semantics/PowerPC/ppc-vector-types01.f90:4:17: error: CHECK-LABEL: expected string not found in input
 ! CHECK-LABEL: PROGRAM ppc_vec_unit
                ^
<stdin>:1:1: note: scanning from here
PROGRAM PPC_VEC_UNIT
^

Input file: <stdin>
Check file: /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Semantics/PowerPC/ppc-vector-types01.f90

-dump-input=help explains the following input dump.

Input was:
<<<<<<
         1: PROGRAM PPC_VEC_UNIT 
label:4     X~~~~~~~~~~~~~~~~~~~~ error: no match found
         2:  IMPLICIT NONE 
label:4     ~~~~~~~~~~~~~~~
         3:  VECTOR(INTEGER(KIND=4_4)) :: vi1, vi2 
label:4     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         4:  VECTOR(REAL(KIND=8_4)) :: vr1, vr2 
label:4     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         5:  VECTOR(UNSIGNED(KIND=2_4)) :: vu1, vu2 
label:4     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         6:  __VECTOR_PAIR :: vp1, vp2 
label:4     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
         .
         .
         .
>>>>>>

--

********************


```

</details>

https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Fri Jul 18 04:17:19 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 04:17:19 -0700 (PDT)
Subject: [flang-commits] [flang] 6c63316 - [flang][cuda] Support device
 component in a pointer or allocatable derived-type (#149418)
Message-ID: <687a2d3f.170a0220.75263.25a6@mx.google.com>


Author: Valentin Clement (バレンタイン クレメン)
Date: 2025-07-18T04:17:15-07:00
New Revision: 6c63316ee17462c97c722a960680b2b45d2fff4d

URL: https://github.com/llvm/llvm-project/commit/6c63316ee17462c97c722a960680b2b45d2fff4d
DIFF: https://github.com/llvm/llvm-project/commit/6c63316ee17462c97c722a960680b2b45d2fff4d.diff

LOG: [flang][cuda] Support device component in a pointer or allocatable derived-type (#149418)

Added: 
    

Modified: 
    flang/lib/Lower/ConvertVariable.cpp
    flang/test/Lower/CUDA/cuda-set-allocator.cuf

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 6cda742874ccf..cacf4e249aa28 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -813,6 +813,10 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
       if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(baseTy))
         baseTy = boxTy.getEleTy();
       baseTy = fir::unwrapRefType(baseTy);
+
+      if (mlir::isa<fir::SequenceType>(baseTy))
+        TODO(loc, "array of derived-type with device component");
+
       auto recTy =
           mlir::dyn_cast<fir::RecordType>(fir::unwrapSequenceType(baseTy));
       assert(recTy && "expected fir::RecordType");
@@ -823,7 +827,7 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
         if (Fortran::semantics::IsDeviceAllocatable(sym)) {
           unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
           mlir::Type fieldTy;
-          std::vector<mlir::Value> coordinates;
+          llvm::SmallVector<mlir::Value> coordinates;
 
           if (fieldIdx != std::numeric_limits<unsigned>::max()) {
             // Field found in the base record type.
@@ -866,8 +870,24 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
             TODO(loc, "device resident component in complex derived-type "
                       "hierarchy");
 
-          mlir::Value comp = builder.create<fir::CoordinateOp>(
-              loc, builder.getRefType(fieldTy), fir::getBase(exv), coordinates);
+          mlir::Value base = fir::getBase(exv);
+          mlir::Value comp;
+          if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(base.getType()))) {
+            mlir::Value box = builder.create<fir::LoadOp>(loc, base);
+            mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, box);
+            llvm::SmallVector<mlir::Value> lenParams;
+            assert(coordinates.size() == 1 && "expect one coordinate");
+            auto field = mlir::dyn_cast<fir::FieldIndexOp>(
+                coordinates[0].getDefiningOp());
+            comp = builder.create<hlfir::DesignateOp>(
+                loc, builder.getRefType(fieldTy), addr,
+                /*component=*/field.getFieldName(),
+                /*componentShape=*/mlir::Value{},
+                hlfir::DesignateOp::Subscripts{});
+          } else {
+            comp = builder.create<fir::CoordinateOp>(
+                loc, builder.getRefType(fieldTy), base, coordinates);
+          }
           cuf::DataAttributeAttr dataAttr =
               Fortran::lower::translateSymbolCUFDataAttribute(
                   builder.getContext(), sym);

diff  --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index ee89ea38a3fc7..e3bb181f65398 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -21,4 +21,36 @@ contains
 ! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]]#0, z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[Z]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
 
+  subroutine sub2()
+    type(ty_device), pointer :: d1
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub2()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub2Ed1"} -> !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMm1Fsub2Ed1"} : (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
+  subroutine sub3()
+    type(ty_device), allocatable :: d1
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub3()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub3Ed1"} -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMm1Fsub3Ed1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
 end module


        

From flang-commits at lists.llvm.org  Fri Jul 18 04:17:21 2025
From: flang-commits at lists.llvm.org (Valentin Clement =?utf-8?b?44OQ44Os44Oz44K/44Kk44OzIOOCr+ODrOODoeODsw==?= via flang-commits)
Date: Fri, 18 Jul 2025 04:17:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][cuda] Support device component in a
 pointer or allocatable derived-type (PR #149418)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149418@github.com>
Message-ID: <687a2d41.170a0220.3096b4.2b0f@mx.google.com>


https://github.com/clementval closed https://github.com/llvm/llvm-project/pull/149418

From flang-commits at lists.llvm.org  Fri Jul 18 05:41:55 2025
From: flang-commits at lists.llvm.org (Kelvin Li via flang-commits)
Date: Fri, 18 Jul 2025 05:41:55 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] convert program name to upper case
	(NFC) (PR #149508)
Message-ID: <llvm.org/llvm/llvm-project/pull/149508@github.com>


https://github.com/kkwli created https://github.com/llvm/llvm-project/pull/149508

See https://github.com/llvm/llvm-project/pull/149169

>From 600876fa834740376422b72f7988c8c7026142e2 Mon Sep 17 00:00:00 2001
From: Kelvin Li <kli at ca.ibm.com>
Date: Fri, 18 Jul 2025 08:37:50 -0400
Subject: [PATCH] [flang] convert program name to upper case (NFC)

---
 flang/test/Semantics/PowerPC/ppc-vector-types01.f90 | 2 +-
 flang/test/Semantics/PowerPC/ppc-vector-types02.f90 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Semantics/PowerPC/ppc-vector-types01.f90 b/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
index ad69b69a47f76..ea54a00fa4392 100644
--- a/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
+++ b/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fdebug-unparse %s | FileCheck %s
 ! REQUIRES: target=powerpc{{.*}}
 
-    ! CHECK-LABEL: PROGRAM ppc_vec_unit
+    ! CHECK-LABEL: PROGRAM PPC_VEC_UNIT
       program ppc_vec_unit
       implicit none
       ! CHECK: VECTOR(INTEGER(KIND=4_4)) :: vi1, vi2
diff --git a/flang/test/Semantics/PowerPC/ppc-vector-types02.f90 b/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
index 8c96684c50eb7..175b58680a209 100644
--- a/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
+++ b/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
@@ -2,7 +2,7 @@
 ! REQUIRES: target=powerpc{{.*}}
 
 ! C: MainProgram scope: ppc_vec_types
-! CHECK-LABEL: MainProgram scope: ppc_vec_types size={{[0-9]*}} alignment={{[0-9]*}}
+! CHECK-LABEL: MainProgram scope: PPC_VEC_TYPES size={{[0-9]*}} alignment={{[0-9]*}}
 program ppc_vec_types
   implicit none
   vector(integer(4)) :: vi


From flang-commits at lists.llvm.org  Fri Jul 18 05:42:25 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 05:42:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] convert program name to upper case
	(NFC) (PR #149508)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149508@github.com>
Message-ID: <687a4131.630a0220.33a70.2ae4@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Kelvin Li (kkwli)

<details>
<summary>Changes</summary>

See https://github.com/llvm/llvm-project/pull/149169

---
Full diff: https://github.com/llvm/llvm-project/pull/149508.diff


2 Files Affected:

- (modified) flang/test/Semantics/PowerPC/ppc-vector-types01.f90 (+1-1) 
- (modified) flang/test/Semantics/PowerPC/ppc-vector-types02.f90 (+1-1) 


``````````diff
diff --git a/flang/test/Semantics/PowerPC/ppc-vector-types01.f90 b/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
index ad69b69a47f76..ea54a00fa4392 100644
--- a/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
+++ b/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fdebug-unparse %s | FileCheck %s
 ! REQUIRES: target=powerpc{{.*}}
 
-    ! CHECK-LABEL: PROGRAM ppc_vec_unit
+    ! CHECK-LABEL: PROGRAM PPC_VEC_UNIT
       program ppc_vec_unit
       implicit none
       ! CHECK: VECTOR(INTEGER(KIND=4_4)) :: vi1, vi2
diff --git a/flang/test/Semantics/PowerPC/ppc-vector-types02.f90 b/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
index 8c96684c50eb7..175b58680a209 100644
--- a/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
+++ b/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
@@ -2,7 +2,7 @@
 ! REQUIRES: target=powerpc{{.*}}
 
 ! C: MainProgram scope: ppc_vec_types
-! CHECK-LABEL: MainProgram scope: ppc_vec_types size={{[0-9]*}} alignment={{[0-9]*}}
+! CHECK-LABEL: MainProgram scope: PPC_VEC_TYPES size={{[0-9]*}} alignment={{[0-9]*}}
 program ppc_vec_types
   implicit none
   vector(integer(4)) :: vi

``````````

</details>


https://github.com/llvm/llvm-project/pull/149508

From flang-commits at lists.llvm.org  Fri Jul 18 06:03:26 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 06:03:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Main program symbol no longer
 conflicts with the other symbols (PR #149169)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149169@github.com>
Message-ID: <687a461e.170a0220.28c5ee.2f19@mx.google.com>


eugeneepshteyn wrote:

> LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `flang` at step 6 "test-build-unified-tree-check-flang".
> 
> Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/34156
> 
> Here is the relevant piece of the build log for the reference

One PPC specific test needs to be tweaked.

https://github.com/llvm/llvm-project/pull/149169

From flang-commits at lists.llvm.org  Fri Jul 18 06:05:53 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 06:05:53 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] convert program name to upper case
	(NFC) (PR #149508)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149508@github.com>
Message-ID: <687a46b1.170a0220.327cd4.e463@mx.google.com>


https://github.com/eugeneepshteyn approved this pull request.

Sorry, I had no idea there were separate PPC tests. Thanks for modifying the tests!

https://github.com/llvm/llvm-project/pull/149508

From flang-commits at lists.llvm.org  Fri Jul 18 06:24:44 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Fri, 18 Jul 2025 06:24:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] convert program name to upper case
	(NFC) (PR #149508)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149508@github.com>
Message-ID: <687a4b1c.170a0220.2a1902.3086@mx.google.com>


https://github.com/DanielCChen approved this pull request.

LGTM.

https://github.com/llvm/llvm-project/pull/149508

From flang-commits at lists.llvm.org  Fri Jul 18 06:39:18 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 06:39:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
Message-ID: <llvm.org/llvm/llvm-project/pull/149513@github.com>


https://github.com/eugeneepshteyn created https://github.com/llvm/llvm-project/pull/149513

Fixed a crash in the following example:
```
subroutine sub()
  implicit none
  print *, (i, i = 1, 2)  ! Problem: using undefined var in implied-do loop
end subroutine sub
```
The error message was already generated, but the compiler crashed before it could display it.

>From a155a355cc1cef2169c37d069ad72b4c180f0633 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 14:56:11 -0400
Subject: [PATCH 1/5] [flang] Control alignment of constant folded reals

When REAL types are constant folded, the underneath implementation uses
arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the
resulting code for flang compiler ended up using SSE2 aligned load
instructions for REAL(16) constant folding on x86_64, and these
instructions require that the values are loaded from the aligned
addresses.
---
 flang/include/flang/Evaluate/integer.h | 1 +
 flang/include/flang/Evaluate/real.h    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index fccc2ad774a8f..5953fc81cb111 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,6 +74,7 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
+  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 03294881850a1..fa74792a0377f 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,7 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  Word word_{}; // an Integer<>
+  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format

>From 43102f9b97cc055124b7e0e730cecf71fecdc29a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 15:43:08 -0400
Subject: [PATCH 2/5] Added comment

---
 flang/include/flang/Evaluate/real.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index fa74792a0377f..af17adc8ad8a3 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,6 +490,9 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
+  // Require alignment, in case code generation code on x86_64 decides that
+  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 

>From 26432b55e826ea49030252b43714baf26afa36ac Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 19:15:09 -0400
Subject: [PATCH 3/5] Comment update

---
 flang/include/flang/Evaluate/real.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index af17adc8ad8a3..76d25d9fe2670 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,8 +490,8 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  // Require alignment, in case code generation code on x86_64 decides that
-  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // Require alignment, in case code generation on x86_64 decides that our
+  // Real object is suitable for SSE2 instructions and then gets surprised
   // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };

>From 270cdfcf328bc128f722888baa284fb5fdc67ea8 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 08:39:01 -0400
Subject: [PATCH 4/5] [flang]

First change: missing variable name in the error message.
---
 flang/lib/Semantics/resolve-names.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index b3268605e7c0c..deafdbd269c78 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8700,7 +8700,7 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     return &name;
   }
   if (isImplicitNoneType() && !deferImplicitTyping_) {
-    Say(name, "No explicit type declared for '%s'"_err_en_US);
+    Say(name, "No explicit type declared for '%s'"_err_en_US, name.source);
     return nullptr;
   }
   // Create the symbol, then ensure that it is accessible

>From fc450bc1dd1b7eddf668e3ceaafdebaa17dc1a8f Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:14:56 -0400
Subject: [PATCH 5/5] Actual fix

---
 flang/lib/Semantics/check-do-forall.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index cc1d4bf58745a..e258df86a4b1c 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -1180,7 +1180,9 @@ void DoForallChecker::Leave(const parser::IoControlSpec &ioControlSpec) {
 void DoForallChecker::Leave(const parser::OutputImpliedDo &outputImpliedDo) {
   const auto &control{std::get<parser::IoImpliedDoControl>(outputImpliedDo.t)};
   const parser::Name &name{control.name.thing.thing};
-  context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  if (name.symbol) {
+    context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  }
 }
 
 void DoForallChecker::Leave(const parser::StatVariable &statVariable) {


From flang-commits at lists.llvm.org  Fri Jul 18 06:41:26 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 06:41:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a4f06.050a0220.7e59b.36bc@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/149513

>From a155a355cc1cef2169c37d069ad72b4c180f0633 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 14:56:11 -0400
Subject: [PATCH 1/6] [flang] Control alignment of constant folded reals

When REAL types are constant folded, the underneath implementation uses
arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the
resulting code for flang compiler ended up using SSE2 aligned load
instructions for REAL(16) constant folding on x86_64, and these
instructions require that the values are loaded from the aligned
addresses.
---
 flang/include/flang/Evaluate/integer.h | 1 +
 flang/include/flang/Evaluate/real.h    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index fccc2ad774a8f..5953fc81cb111 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,6 +74,7 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
+  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 03294881850a1..fa74792a0377f 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,7 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  Word word_{}; // an Integer<>
+  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format

>From 43102f9b97cc055124b7e0e730cecf71fecdc29a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 15:43:08 -0400
Subject: [PATCH 2/6] Added comment

---
 flang/include/flang/Evaluate/real.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index fa74792a0377f..af17adc8ad8a3 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,6 +490,9 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
+  // Require alignment, in case code generation code on x86_64 decides that
+  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 

>From 26432b55e826ea49030252b43714baf26afa36ac Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 19:15:09 -0400
Subject: [PATCH 3/6] Comment update

---
 flang/include/flang/Evaluate/real.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index af17adc8ad8a3..76d25d9fe2670 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,8 +490,8 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  // Require alignment, in case code generation code on x86_64 decides that
-  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // Require alignment, in case code generation on x86_64 decides that our
+  // Real object is suitable for SSE2 instructions and then gets surprised
   // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };

>From 270cdfcf328bc128f722888baa284fb5fdc67ea8 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 08:39:01 -0400
Subject: [PATCH 4/6] [flang]

First change: missing variable name in the error message.
---
 flang/lib/Semantics/resolve-names.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index b3268605e7c0c..deafdbd269c78 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8700,7 +8700,7 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     return &name;
   }
   if (isImplicitNoneType() && !deferImplicitTyping_) {
-    Say(name, "No explicit type declared for '%s'"_err_en_US);
+    Say(name, "No explicit type declared for '%s'"_err_en_US, name.source);
     return nullptr;
   }
   // Create the symbol, then ensure that it is accessible

>From fc450bc1dd1b7eddf668e3ceaafdebaa17dc1a8f Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:14:56 -0400
Subject: [PATCH 5/6] Actual fix

---
 flang/lib/Semantics/check-do-forall.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index cc1d4bf58745a..e258df86a4b1c 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -1180,7 +1180,9 @@ void DoForallChecker::Leave(const parser::IoControlSpec &ioControlSpec) {
 void DoForallChecker::Leave(const parser::OutputImpliedDo &outputImpliedDo) {
   const auto &control{std::get<parser::IoImpliedDoControl>(outputImpliedDo.t)};
   const parser::Name &name{control.name.thing.thing};
-  context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  if (name.symbol) {
+    context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  }
 }
 
 void DoForallChecker::Leave(const parser::StatVariable &statVariable) {

>From 95dd942ae95b605eaf65cec91a6bbcf8bfc072c9 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:40:59 -0400
Subject: [PATCH 6/6] Added test

---
 flang/test/Semantics/resolve40.f90 | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flang/test/Semantics/resolve40.f90 b/flang/test/Semantics/resolve40.f90
index a91507aa62282..81bb5f989ec48 100644
--- a/flang/test/Semantics/resolve40.f90
+++ b/flang/test/Semantics/resolve40.f90
@@ -96,3 +96,10 @@ subroutine s12(x)
   !BECAUSE: 'x' is an INTENT(IN) dummy argument
   read(*,nml=nl)
 end
+
+subroutine s13()
+  implicit none
+  !ERROR: No explicit type declared for 'i'
+  !ERROR: No explicit type declared for 'i'
+  print *, (i, i = 1, 2)
+end


From flang-commits at lists.llvm.org  Fri Jul 18 06:44:56 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 06:44:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a4fd8.170a0220.37a56a.42b5@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/149513

>From a155a355cc1cef2169c37d069ad72b4c180f0633 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 14:56:11 -0400
Subject: [PATCH 1/7] [flang] Control alignment of constant folded reals

When REAL types are constant folded, the underneath implementation uses
arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the
resulting code for flang compiler ended up using SSE2 aligned load
instructions for REAL(16) constant folding on x86_64, and these
instructions require that the values are loaded from the aligned
addresses.
---
 flang/include/flang/Evaluate/integer.h | 1 +
 flang/include/flang/Evaluate/real.h    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index fccc2ad774a8f..5953fc81cb111 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,6 +74,7 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
+  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 03294881850a1..fa74792a0377f 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,7 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  Word word_{}; // an Integer<>
+  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format

>From 43102f9b97cc055124b7e0e730cecf71fecdc29a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 15:43:08 -0400
Subject: [PATCH 2/7] Added comment

---
 flang/include/flang/Evaluate/real.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index fa74792a0377f..af17adc8ad8a3 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,6 +490,9 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
+  // Require alignment, in case code generation code on x86_64 decides that
+  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 

>From 26432b55e826ea49030252b43714baf26afa36ac Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 19:15:09 -0400
Subject: [PATCH 3/7] Comment update

---
 flang/include/flang/Evaluate/real.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index af17adc8ad8a3..76d25d9fe2670 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,8 +490,8 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  // Require alignment, in case code generation code on x86_64 decides that
-  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // Require alignment, in case code generation on x86_64 decides that our
+  // Real object is suitable for SSE2 instructions and then gets surprised
   // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };

>From 270cdfcf328bc128f722888baa284fb5fdc67ea8 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 08:39:01 -0400
Subject: [PATCH 4/7] [flang]

First change: missing variable name in the error message.
---
 flang/lib/Semantics/resolve-names.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index b3268605e7c0c..deafdbd269c78 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8700,7 +8700,7 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     return &name;
   }
   if (isImplicitNoneType() && !deferImplicitTyping_) {
-    Say(name, "No explicit type declared for '%s'"_err_en_US);
+    Say(name, "No explicit type declared for '%s'"_err_en_US, name.source);
     return nullptr;
   }
   // Create the symbol, then ensure that it is accessible

>From fc450bc1dd1b7eddf668e3ceaafdebaa17dc1a8f Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:14:56 -0400
Subject: [PATCH 5/7] Actual fix

---
 flang/lib/Semantics/check-do-forall.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index cc1d4bf58745a..e258df86a4b1c 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -1180,7 +1180,9 @@ void DoForallChecker::Leave(const parser::IoControlSpec &ioControlSpec) {
 void DoForallChecker::Leave(const parser::OutputImpliedDo &outputImpliedDo) {
   const auto &control{std::get<parser::IoImpliedDoControl>(outputImpliedDo.t)};
   const parser::Name &name{control.name.thing.thing};
-  context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  if (name.symbol) {
+    context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  }
 }
 
 void DoForallChecker::Leave(const parser::StatVariable &statVariable) {

>From 95dd942ae95b605eaf65cec91a6bbcf8bfc072c9 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:40:59 -0400
Subject: [PATCH 6/7] Added test

---
 flang/test/Semantics/resolve40.f90 | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flang/test/Semantics/resolve40.f90 b/flang/test/Semantics/resolve40.f90
index a91507aa62282..81bb5f989ec48 100644
--- a/flang/test/Semantics/resolve40.f90
+++ b/flang/test/Semantics/resolve40.f90
@@ -96,3 +96,10 @@ subroutine s12(x)
   !BECAUSE: 'x' is an INTENT(IN) dummy argument
   read(*,nml=nl)
 end
+
+subroutine s13()
+  implicit none
+  !ERROR: No explicit type declared for 'i'
+  !ERROR: No explicit type declared for 'i'
+  print *, (i, i = 1, 2)
+end

>From 38133c4fa4bd1175f914230628fdf566038fa5d3 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:44:42 -0400
Subject: [PATCH 7/7] Restored the files accidentally grabbed from another
 branch

---
 flang/include/flang/Evaluate/integer.h | 1 -
 flang/include/flang/Evaluate/real.h    | 5 +----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index 5953fc81cb111..fccc2ad774a8f 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,7 +74,6 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
-  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 76d25d9fe2670..03294881850a1 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,10 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  // Require alignment, in case code generation on x86_64 decides that our
-  // Real object is suitable for SSE2 instructions and then gets surprised
-  // by unaligned address.
-  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
+  Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format


From flang-commits at lists.llvm.org  Fri Jul 18 06:45:21 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 06:45:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a4ff1.050a0220.af705.42c1@mx.google.com>


https://github.com/eugeneepshteyn ready_for_review https://github.com/llvm/llvm-project/pull/149513

From flang-commits at lists.llvm.org  Fri Jul 18 06:46:00 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 06:46:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a5018.170a0220.feb16.45ce@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Eugene Epshteyn (eugeneepshteyn)

<details>
<summary>Changes</summary>

Fixed a crash in the following example:
```
subroutine sub()
  implicit none
  print *, (i, i = 1, 2)  ! Problem: using undefined var in implied-do loop
end subroutine sub
```
The error message was already generated, but the compiler crashed before it could display it.

---
Full diff: https://github.com/llvm/llvm-project/pull/149513.diff


3 Files Affected:

- (modified) flang/lib/Semantics/check-do-forall.cpp (+3-1) 
- (modified) flang/lib/Semantics/resolve-names.cpp (+1-1) 
- (modified) flang/test/Semantics/resolve40.f90 (+7) 


``````````diff
diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index cc1d4bf58745a..e258df86a4b1c 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -1180,7 +1180,9 @@ void DoForallChecker::Leave(const parser::IoControlSpec &ioControlSpec) {
 void DoForallChecker::Leave(const parser::OutputImpliedDo &outputImpliedDo) {
   const auto &control{std::get<parser::IoImpliedDoControl>(outputImpliedDo.t)};
   const parser::Name &name{control.name.thing.thing};
-  context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  if (name.symbol) {
+    context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  }
 }
 
 void DoForallChecker::Leave(const parser::StatVariable &statVariable) {
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index b3268605e7c0c..deafdbd269c78 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8700,7 +8700,7 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     return &name;
   }
   if (isImplicitNoneType() && !deferImplicitTyping_) {
-    Say(name, "No explicit type declared for '%s'"_err_en_US);
+    Say(name, "No explicit type declared for '%s'"_err_en_US, name.source);
     return nullptr;
   }
   // Create the symbol, then ensure that it is accessible
diff --git a/flang/test/Semantics/resolve40.f90 b/flang/test/Semantics/resolve40.f90
index a91507aa62282..81bb5f989ec48 100644
--- a/flang/test/Semantics/resolve40.f90
+++ b/flang/test/Semantics/resolve40.f90
@@ -96,3 +96,10 @@ subroutine s12(x)
   !BECAUSE: 'x' is an INTENT(IN) dummy argument
   read(*,nml=nl)
 end
+
+subroutine s13()
+  implicit none
+  !ERROR: No explicit type declared for 'i'
+  !ERROR: No explicit type declared for 'i'
+  print *, (i, i = 1, 2)
+end

``````````

</details>


https://github.com/llvm/llvm-project/pull/149513

From flang-commits at lists.llvm.org  Fri Jul 18 07:05:12 2025
From: flang-commits at lists.llvm.org (Daniel Chen via flang-commits)
Date: Fri, 18 Jul 2025 07:05:12 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][runtime] Preserve type when
 remapping monomorphic pointers (PR #149427)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149427@github.com>
Message-ID: <687a5498.170a0220.29253f.7413@mx.google.com>


https://github.com/DanielCChen approved this pull request.

LGTM. 
Thanks!

https://github.com/llvm/llvm-project/pull/149427

From flang-commits at lists.llvm.org  Fri Jul 18 07:29:54 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 07:29:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a5a62.630a0220.1f7c2.930c@mx.google.com>


================
@@ -8700,7 +8700,7 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     return &name;
   }
   if (isImplicitNoneType() && !deferImplicitTyping_) {
-    Say(name, "No explicit type declared for '%s'"_err_en_US);
+    Say(name, "No explicit type declared for '%s'"_err_en_US, name.source);
----------------
klausler wrote:

Was this change necessary?  The `Say()` member function overloads in name resolution often have `name.source` as an implicit first value.

https://github.com/llvm/llvm-project/pull/149513

From flang-commits at lists.llvm.org  Fri Jul 18 07:55:18 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Fri, 18 Jul 2025 07:55:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement sinpi (PR #149525)
Message-ID: <llvm.org/llvm/llvm-project/pull/149525@github.com>


https://github.com/c8ef created https://github.com/llvm/llvm-project/pull/149525

None

>From 1e9f5c1dfe5f5795c82e49cae6526bff281842b2 Mon Sep 17 00:00:00 2001
From: c8ef <c8ef at outlook.com>
Date: Fri, 18 Jul 2025 14:54:27 +0000
Subject: [PATCH] [flang] Implement sinpi

---
 .../flang/Optimizer/Builder/IntrinsicCall.h   |  1 +
 flang/lib/Evaluate/intrinsics.cpp             |  1 +
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 16 ++++++++++++++
 flang/test/Lower/Intrinsics/sinpi.f90         | 22 +++++++++++++++++++
 4 files changed, 40 insertions(+)
 create mode 100644 flang/test/Lower/Intrinsics/sinpi.f90

diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index acdba7c49e6b3..d84d3593ebca6 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -419,6 +419,7 @@ struct IntrinsicLibrary {
   mlir::Value genShiftA(mlir::Type resultType, llvm::ArrayRef<mlir::Value>);
   mlir::Value genSign(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genSind(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genSinpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genSize(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   fir::ExtendedValue genSizeOf(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genSpacing(mlir::Type resultType,
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9957010684d48..d44239b41fa20 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -957,6 +957,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"sin", {{"x", SameFloating}}, SameFloating},
     {"sind", {{"x", SameFloating}}, SameFloating},
     {"sinh", {{"x", SameFloating}}, SameFloating},
+    {"sinpi", {{"x", SameFloating}}, SameFloating},
     {"size",
         {{"array", AnyData, Rank::arrayOrAssumedRank},
             OptionalDIM, // unless array is assumed-size
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d77a656158a37..823b1eb887992 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -901,6 +901,7 @@ static constexpr IntrinsicHandler handlers[]{
      {{{"number", asValue}, {"handler", asAddr}, {"status", asAddr}}},
      /*isElemental=*/false},
     {"sind", &I::genSind},
+    {"sinpi", &I::genSinpi},
     {"size",
      &I::genSize,
      {{{"array", asBox},
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
+}
+
 // SIZE
 fir::ExtendedValue
 IntrinsicLibrary::genSize(mlir::Type resultType,
diff --git a/flang/test/Lower/Intrinsics/sinpi.f90 b/flang/test/Lower/Intrinsics/sinpi.f90
new file mode 100644
index 0000000000000..38c2277892ec7
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/sinpi.f90
@@ -0,0 +1,22 @@
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = sinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f32
+! CHECK: %[[sin:.*]] = math.sin %[[mul]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = sinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[dfactor]] fastmath<contract> : f64
+! CHECK: %[[sin:.*]] = math.sin %[[mul]] fastmath<contract> : f64


From flang-commits at lists.llvm.org  Fri Jul 18 07:55:37 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Fri, 18 Jul 2025 07:55:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement sinpi (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687a6069.170a0220.1e6ea3.4d81@mx.google.com>


c8ef wrote:

* **#149525** <a href="https://app.graphite.dev/github/pr/llvm/llvm-project/149525?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/llvm/llvm-project/149525?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Fri Jul 18 07:56:57 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Fri, 18 Jul 2025 07:56:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687a60b9.170a0220.2d9d15.5537@mx.google.com>


https://github.com/c8ef edited https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Fri Jul 18 07:57:05 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Fri, 18 Jul 2025 07:57:05 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687a60c1.170a0220.309371.4e55@mx.google.com>


https://github.com/c8ef ready_for_review https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Fri Jul 18 07:57:34 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 07:57:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687a60de.630a0220.adcf9.a184@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Connector Switch (c8ef)

<details>
<summary>Changes</summary>



---
Full diff: https://github.com/llvm/llvm-project/pull/149525.diff


4 Files Affected:

- (modified) flang/include/flang/Optimizer/Builder/IntrinsicCall.h (+1) 
- (modified) flang/lib/Evaluate/intrinsics.cpp (+1) 
- (modified) flang/lib/Optimizer/Builder/IntrinsicCall.cpp (+16) 
- (added) flang/test/Lower/Intrinsics/sinpi.f90 (+22) 


``````````diff
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index acdba7c49e6b3..d84d3593ebca6 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -419,6 +419,7 @@ struct IntrinsicLibrary {
   mlir::Value genShiftA(mlir::Type resultType, llvm::ArrayRef<mlir::Value>);
   mlir::Value genSign(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genSind(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genSinpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genSize(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   fir::ExtendedValue genSizeOf(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genSpacing(mlir::Type resultType,
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9957010684d48..d44239b41fa20 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -957,6 +957,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"sin", {{"x", SameFloating}}, SameFloating},
     {"sind", {{"x", SameFloating}}, SameFloating},
     {"sinh", {{"x", SameFloating}}, SameFloating},
+    {"sinpi", {{"x", SameFloating}}, SameFloating},
     {"size",
         {{"array", AnyData, Rank::arrayOrAssumedRank},
             OptionalDIM, // unless array is assumed-size
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d77a656158a37..823b1eb887992 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -901,6 +901,7 @@ static constexpr IntrinsicHandler handlers[]{
      {{{"number", asValue}, {"handler", asAddr}, {"status", asAddr}}},
      /*isElemental=*/false},
     {"sind", &I::genSind},
+    {"sinpi", &I::genSinpi},
     {"size",
      &I::genSize,
      {{{"array", asBox},
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
+}
+
 // SIZE
 fir::ExtendedValue
 IntrinsicLibrary::genSize(mlir::Type resultType,
diff --git a/flang/test/Lower/Intrinsics/sinpi.f90 b/flang/test/Lower/Intrinsics/sinpi.f90
new file mode 100644
index 0000000000000..38c2277892ec7
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/sinpi.f90
@@ -0,0 +1,22 @@
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = sinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f32
+! CHECK: %[[sin:.*]] = math.sin %[[mul]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = sinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[dfactor]] fastmath<contract> : f64
+! CHECK: %[[sin:.*]] = math.sin %[[mul]] fastmath<contract> : f64

``````````

</details>


https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Fri Jul 18 07:59:06 2025
From: flang-commits at lists.llvm.org (Jakub Kuderski via flang-commits)
Date: Fri, 18 Jul 2025 07:59:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Migrate away from
	ArrayRef(std::nullopt_t) (PR #149454)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149454@github.com>
Message-ID: <687a613a.170a0220.11ae56.9fd9@mx.google.com>


https://github.com/kuhar approved this pull request.


https://github.com/llvm/llvm-project/pull/149454

From flang-commits at lists.llvm.org  Fri Jul 18 08:05:15 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 08:05:15 -0700 (PDT)
Subject: [flang-commits] [flang] 151fffc - [flang] Migrate away from
 ArrayRef(std::nullopt_t) (#149454)
Message-ID: <687a62ab.170a0220.1573d2.9e85@mx.google.com>


Author: Kazu Hirata
Date: 2025-07-18T08:05:12-07:00
New Revision: 151fffccf1340d8a2800664cbcaaa579ba772a4c

URL: https://github.com/llvm/llvm-project/commit/151fffccf1340d8a2800664cbcaaa579ba772a4c
DIFF: https://github.com/llvm/llvm-project/commit/151fffccf1340d8a2800664cbcaaa579ba772a4c.diff

LOG: [flang] Migrate away from ArrayRef(std::nullopt_t) (#149454)

ArrayRef(std::nullopt_t) has been deprecated.  This patch replaces
std::nullopt with mlir::TypeRange{} or mlir::ValueRange{} as
appropriate.

Added: 
    

Modified: 
    flang/lib/Lower/Bridge.cpp
    flang/lib/Lower/ConvertConstant.cpp
    flang/lib/Lower/ConvertExpr.cpp
    flang/lib/Lower/Runtime.cpp
    flang/lib/Lower/VectorSubscripts.cpp
    flang/lib/Optimizer/Builder/FIRBuilder.cpp
    flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
    flang/lib/Optimizer/Builder/Runtime/Stop.cpp
    flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
    flang/lib/Optimizer/Transforms/MemoryUtils.cpp
    flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
    flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
    flang/unittests/Optimizer/FortranVariableTest.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4241d12601242..5f0783f869bf6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1466,8 +1466,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     assert(falseTarget && "missing conditional branch false block");
     mlir::Location loc = toLocation();
     mlir::Value bcc = builder->createConvert(loc, builder->getI1Type(), cond);
-    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget, std::nullopt,
-                                            falseTarget, std::nullopt);
+    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget,
+                                            mlir::ValueRange{}, falseTarget,
+                                            mlir::ValueRange{});
   }
   void genConditionalBranch(mlir::Value cond,
                             Fortran::lower::pft::Evaluation *trueTarget,
@@ -2556,8 +2557,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
           loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
-          /*local_vars=*/std::nullopt,
-          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*local_vars=*/mlir::ValueRange{},
+          /*local_syms=*/nullptr, /*reduce_vars=*/mlir::ValueRange{},
           /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
           /*reduce_attrs=*/nullptr);
 
@@ -3810,9 +3811,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       mlir::Block *selectCaseBlock = insertBlock(blockList[0]);
       mlir::Block *assumedSizeBlock =
           rankStarBlock ? rankStarBlock : defaultBlock;
-      builder->create<mlir::cf::CondBranchOp>(loc, isAssumedSize,
-                                              assumedSizeBlock, std::nullopt,
-                                              selectCaseBlock, std::nullopt);
+      builder->create<mlir::cf::CondBranchOp>(
+          loc, isAssumedSize, assumedSizeBlock, mlir::ValueRange{},
+          selectCaseBlock, mlir::ValueRange{});
       startBlock(selectCaseBlock);
     }
     // Create fir.select_case for the other rank cases.

diff  --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1850b67898126..b8ab5d09e3e08 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -303,7 +303,7 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
     return builder.create<fir::StringLitOp>(
-        loc, llvm::ArrayRef<mlir::Type>{type}, std::nullopt, attrs);
+        loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
   }
 }
 

diff  --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 0a1cd67789772..281ab229d1b6a 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1003,9 +1003,9 @@ class ScalarExprLowering {
           },
           [&](const fir::MutableBoxValue &toBox) {
             if (toBox.isPointer()) {
-              Fortran::lower::associateMutableBox(converter, loc, toBox, expr,
-                                                  /*lbounds=*/std::nullopt,
-                                                  stmtCtx);
+              Fortran::lower::associateMutableBox(
+                  converter, loc, toBox, expr,
+                  /*lbounds=*/mlir::ValueRange{}, stmtCtx);
               return;
             }
             // For allocatable components, a deep copy is needed.
@@ -3604,8 +3604,9 @@ class ArrayExprLowering {
       mlir::Value castTo =
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
-      return builder.create<fir::ArrayLoadOp>(
-          loc, seqTy, castTo, shapeOp, /*slice=*/mlir::Value{}, std::nullopt);
+      return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
+                                              /*slice=*/mlir::Value{},
+                                              mlir::ValueRange{});
     };
     // Custom lowering of the element store to deal with the extra indirection
     // to the lazy allocated buffer.
@@ -4207,7 +4208,7 @@ class ArrayExprLowering {
       auto addr =
           builder->create<fir::ArrayCoorOp>(loc, eleRefTy, tmp, shape,
                                             /*slice=*/mlir::Value{}, indices,
-                                            /*typeParams=*/std::nullopt);
+                                            /*typeParams=*/mlir::ValueRange{});
       auto load = builder->create<fir::LoadOp>(loc, addr);
       return builder->createConvert(loc, i1Ty, load);
     };
@@ -4522,17 +4523,18 @@ class ArrayExprLowering {
         fir::isRecordWithAllocatableMember(eleTy))
       TODO(loc, "creating an array temp where the element type has "
                 "allocatable members");
-    mlir::Value temp = !seqTy.hasDynamicExtents()
-                           ? builder.create<fir::AllocMemOp>(loc, type)
-                           : builder.create<fir::AllocMemOp>(
-                                 loc, type, ".array.expr", std::nullopt, shape);
+    mlir::Value temp =
+        !seqTy.hasDynamicExtents()
+            ? builder.create<fir::AllocMemOp>(loc, type)
+            : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
+                                              mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
     return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
                                             /*slice=*/mlir::Value{},
-                                            std::nullopt);
+                                            mlir::ValueRange{});
   }
 
   static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -6483,7 +6485,7 @@ class ArrayExprLowering {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
         mem = builder.create<fir::AllocMemOp>(
-            loc, eleTy, /*typeparams=*/std::nullopt, initBuffSz);
+            loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
         builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
       }
     } else {

diff  --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index 2be5ef76e46b8..5f73335242336 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
   genUnreachable(builder, loc);
 }
 
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
 }
 
 void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,

diff  --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index 389a89ddcf102..c7b3e11728cea 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -122,7 +122,7 @@ class VectorSubscriptBoxBuilder {
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
-        loc, fldTy, componentName, recTy, /*typeParams*/ std::nullopt));
+        loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
 

diff  --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index acd5a88a2582d..5b1dbc4435d6c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -620,7 +620,7 @@ fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc,
   mlir::NamedAttribute sizeAttr(sizeTag, getI64IntegerAttr(data.size()));
   llvm::SmallVector<mlir::NamedAttribute> attrs{dataAttr, sizeAttr};
   return create<fir::StringLitOp>(loc, llvm::ArrayRef<mlir::Type>{type},
-                                  std::nullopt, attrs);
+                                  mlir::ValueRange{}, attrs);
 }
 
 mlir::Value fir::FirOpBuilder::genShape(mlir::Location loc,

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 773d6408079cc..04703f7911176 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -59,7 +59,8 @@ mlir::Value fir::runtime::genCpuTime(fir::FirOpBuilder &builder,
                                      mlir::Location loc) {
   mlir::func::FuncOp func =
       fir::runtime::getRuntimeFunc<mkRTKey(CpuTime)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
@@ -280,7 +281,8 @@ void fir::runtime::genRename(fir::FirOpBuilder &builder, mlir::Location loc,
 mlir::Value fir::runtime::genTime(fir::FirOpBuilder &builder,
                                   mlir::Location loc) {
   auto func = fir::runtime::getRuntimeFunc<mkRTKey(time)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 /// generate runtime call to transfer intrinsic with no size argument

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
index 411181cc6dd1c..9b5e43b80b1f6 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
@@ -25,7 +25,7 @@ void fir::runtime::genExit(fir::FirOpBuilder &builder, mlir::Location loc,
 void fir::runtime::genAbort(fir::FirOpBuilder &builder, mlir::Location loc) {
   mlir::func::FuncOp abortFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(Abort)>(loc, builder);
-  builder.create<fir::CallOp>(loc, abortFunc, std::nullopt);
+  builder.create<fir::CallOp>(loc, abortFunc, mlir::ValueRange{});
 }
 
 void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,

diff  --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 03cc92e975b19..c5cf01ed98357 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -405,7 +405,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
   mlir::Location loc = forallMaskOp.getLoc();
   mlir::Value mask = generateYieldedScalarValue(forallMaskOp.getMaskRegion(),
                                                 builder.getI1Type());
-  auto ifOp = builder.create<fir::IfOp>(loc, std::nullopt, mask, false);
+  auto ifOp = builder.create<fir::IfOp>(loc, mlir::TypeRange{}, mask, false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   constructStack.push_back(ifOp);
 }
@@ -530,7 +530,7 @@ void OrderedAssignmentRewriter::generateMaskIfOp(mlir::Value cdt) {
   mlir::Location loc = cdt.getLoc();
   cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
   cdt = builder.createConvert(loc, builder.getI1Type(), cdt);
-  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), std::nullopt, cdt,
+  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), mlir::TypeRange{}, cdt,
                                         /*withElseRegion=*/false);
   constructStack.push_back(ifOp.getOperation());
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());

diff  --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index 1f8edf851de9b..bc4fcd8b0112e 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -222,7 +222,7 @@ void AllocaReplaceImpl::genIndirectDeallocation(
         rewriter.create<fir::ConvertOp>(loc, intPtrTy, ptrVal);
     mlir::Value isAllocated = rewriter.create<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::ne, ptrToInt, c0);
-    auto ifOp = rewriter.create<fir::IfOp>(loc, std::nullopt, isAllocated,
+    auto ifOp = rewriter.create<fir::IfOp>(loc, mlir::TypeRange{}, isAllocated,
                                            /*withElseRegion=*/false);
     rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value cast = fir::factory::createConvert(

diff  --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 57eae1ff052a2..6e45aae4246d0 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -456,7 +456,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
   rewriter.setInsertionPointToEnd(thisBlock);
   if (destOps.has_value())
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, destOps.value(),
-                                            newBlock, std::nullopt);
+                                            newBlock, mlir::ValueRange{});
   else
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, newBlock);
   rewriter.setInsertionPointToEnd(newBlock);

diff  --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 506c8e66dbdfa..ad8464b495888 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -261,7 +261,7 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
+          /*iterArgs=*/mlir::ValueRange{}, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());

diff  --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index 98270adaa7c73..59808779aa6ef 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -48,7 +48,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
   mlir::Value addr = builder->create<fir::AllocaOp>(loc, eleType);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      /*shape=*/mlir::Value{}, /*typeParams=*/std::nullopt,
+      /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{},
       /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
@@ -102,11 +102,11 @@ TEST_F(FortranVariableTest, SimpleArray) {
       extents.size(), fir::SequenceType::getUnknownExtent());
   mlir::Type seqTy = fir::SequenceType::get(typeShape, eleType);
   mlir::Value addr = builder->create<fir::AllocaOp>(
-      loc, seqTy, /*pinned=*/false, /*typeParams=*/std::nullopt, extents);
+      loc, seqTy, /*pinned=*/false, /*typeParams=*/mlir::ValueRange{}, extents);
   mlir::Value shape = createShape(extents);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      shape, /*typeParams*/ std::nullopt, /*dummy_scope=*/nullptr, name,
+      shape, /*typeParams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
 


        

From flang-commits at lists.llvm.org  Fri Jul 18 08:05:18 2025
From: flang-commits at lists.llvm.org (Kazu Hirata via flang-commits)
Date: Fri, 18 Jul 2025 08:05:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Migrate away from
	ArrayRef(std::nullopt_t) (PR #149454)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149454@github.com>
Message-ID: <687a62ae.170a0220.13e2d9.9e4e@mx.google.com>


https://github.com/kazutakahirata closed https://github.com/llvm/llvm-project/pull/149454

From flang-commits at lists.llvm.org  Fri Jul 18 08:20:07 2025
From: flang-commits at lists.llvm.org (Kelvin Li via flang-commits)
Date: Fri, 18 Jul 2025 08:20:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] convert program name to upper case
	(NFC) (PR #149508)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149508@github.com>
Message-ID: <687a6627.170a0220.3150c3.9ecf@mx.google.com>


https://github.com/kkwli closed https://github.com/llvm/llvm-project/pull/149508

From flang-commits at lists.llvm.org  Fri Jul 18 08:21:29 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 08:21:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a6679.170a0220.f83ae.9f4a@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/149513

>From a155a355cc1cef2169c37d069ad72b4c180f0633 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 14:56:11 -0400
Subject: [PATCH 1/8] [flang] Control alignment of constant folded reals

When REAL types are constant folded, the underneath implementation uses
arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the
resulting code for flang compiler ended up using SSE2 aligned load
instructions for REAL(16) constant folding on x86_64, and these
instructions require that the values are loaded from the aligned
addresses.
---
 flang/include/flang/Evaluate/integer.h | 1 +
 flang/include/flang/Evaluate/real.h    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index fccc2ad774a8f..5953fc81cb111 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,6 +74,7 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
+  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 03294881850a1..fa74792a0377f 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,7 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  Word word_{}; // an Integer<>
+  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format

>From 43102f9b97cc055124b7e0e730cecf71fecdc29a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 15:43:08 -0400
Subject: [PATCH 2/8] Added comment

---
 flang/include/flang/Evaluate/real.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index fa74792a0377f..af17adc8ad8a3 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,6 +490,9 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
+  // Require alignment, in case code generation code on x86_64 decides that
+  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 

>From 26432b55e826ea49030252b43714baf26afa36ac Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 19:15:09 -0400
Subject: [PATCH 3/8] Comment update

---
 flang/include/flang/Evaluate/real.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index af17adc8ad8a3..76d25d9fe2670 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,8 +490,8 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  // Require alignment, in case code generation code on x86_64 decides that
-  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // Require alignment, in case code generation on x86_64 decides that our
+  // Real object is suitable for SSE2 instructions and then gets surprised
   // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };

>From 270cdfcf328bc128f722888baa284fb5fdc67ea8 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 08:39:01 -0400
Subject: [PATCH 4/8] [flang]

First change: missing variable name in the error message.
---
 flang/lib/Semantics/resolve-names.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index b3268605e7c0c..deafdbd269c78 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8700,7 +8700,7 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     return &name;
   }
   if (isImplicitNoneType() && !deferImplicitTyping_) {
-    Say(name, "No explicit type declared for '%s'"_err_en_US);
+    Say(name, "No explicit type declared for '%s'"_err_en_US, name.source);
     return nullptr;
   }
   // Create the symbol, then ensure that it is accessible

>From fc450bc1dd1b7eddf668e3ceaafdebaa17dc1a8f Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:14:56 -0400
Subject: [PATCH 5/8] Actual fix

---
 flang/lib/Semantics/check-do-forall.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index cc1d4bf58745a..e258df86a4b1c 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -1180,7 +1180,9 @@ void DoForallChecker::Leave(const parser::IoControlSpec &ioControlSpec) {
 void DoForallChecker::Leave(const parser::OutputImpliedDo &outputImpliedDo) {
   const auto &control{std::get<parser::IoImpliedDoControl>(outputImpliedDo.t)};
   const parser::Name &name{control.name.thing.thing};
-  context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  if (name.symbol) {
+    context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  }
 }
 
 void DoForallChecker::Leave(const parser::StatVariable &statVariable) {

>From 95dd942ae95b605eaf65cec91a6bbcf8bfc072c9 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:40:59 -0400
Subject: [PATCH 6/8] Added test

---
 flang/test/Semantics/resolve40.f90 | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flang/test/Semantics/resolve40.f90 b/flang/test/Semantics/resolve40.f90
index a91507aa62282..81bb5f989ec48 100644
--- a/flang/test/Semantics/resolve40.f90
+++ b/flang/test/Semantics/resolve40.f90
@@ -96,3 +96,10 @@ subroutine s12(x)
   !BECAUSE: 'x' is an INTENT(IN) dummy argument
   read(*,nml=nl)
 end
+
+subroutine s13()
+  implicit none
+  !ERROR: No explicit type declared for 'i'
+  !ERROR: No explicit type declared for 'i'
+  print *, (i, i = 1, 2)
+end

>From 38133c4fa4bd1175f914230628fdf566038fa5d3 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 09:44:42 -0400
Subject: [PATCH 7/8] Restored the files accidentally grabbed from another
 branch

---
 flang/include/flang/Evaluate/integer.h | 1 -
 flang/include/flang/Evaluate/real.h    | 5 +----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index 5953fc81cb111..fccc2ad774a8f 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,7 +74,6 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
-  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 76d25d9fe2670..03294881850a1 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,10 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  // Require alignment, in case code generation on x86_64 decides that our
-  // Real object is suitable for SSE2 instructions and then gets surprised
-  // by unaligned address.
-  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
+  Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format

>From 8a0cac891d3248f93320f66627de7d7b4454e2f5 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Fri, 18 Jul 2025 11:21:14 -0400
Subject: [PATCH 8/8] Undo unnecessary change

---
 flang/lib/Semantics/resolve-names.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index deafdbd269c78..b3268605e7c0c 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8700,7 +8700,7 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     return &name;
   }
   if (isImplicitNoneType() && !deferImplicitTyping_) {
-    Say(name, "No explicit type declared for '%s'"_err_en_US, name.source);
+    Say(name, "No explicit type declared for '%s'"_err_en_US);
     return nullptr;
   }
   // Create the symbol, then ensure that it is accessible


From flang-commits at lists.llvm.org  Fri Jul 18 08:22:17 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 08:22:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a66a9.170a0220.2f7960.aa9b@mx.google.com>


================
@@ -8700,7 +8700,7 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     return &name;
   }
   if (isImplicitNoneType() && !deferImplicitTyping_) {
-    Say(name, "No explicit type declared for '%s'"_err_en_US);
+    Say(name, "No explicit type declared for '%s'"_err_en_US, name.source);
----------------
eugeneepshteyn wrote:

I undid this change.

https://github.com/llvm/llvm-project/pull/149513

From flang-commits at lists.llvm.org  Fri Jul 18 00:57:40 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Fri, 18 Jul 2025 00:57:40 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <6879fe74.170a0220.8c112.ce7d@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 1/7] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 2/7] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 3/7] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 4/7] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 5/7] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 6/7] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 

>From 498d60cf39c237c654c0cfddbe444d63bc87bc6c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 09:57:18 +0200
Subject: [PATCH 7/7] CI test

---
 flang-rt/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 5ffa1bcc34a41..d987115c8e9e8 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,40 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
 enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
@@ -159,6 +193,7 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)


From flang-commits at lists.llvm.org  Fri Jul 18 05:37:07 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Fri, 18 Jul 2025 05:37:07 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687a3ff3.170a0220.35ded4.302d@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 1/8] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 2/8] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 3/8] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 4/8] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 5/8] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 6/8] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 

>From 498d60cf39c237c654c0cfddbe444d63bc87bc6c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 09:57:18 +0200
Subject: [PATCH 7/8] CI test

---
 flang-rt/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 5ffa1bcc34a41..d987115c8e9e8 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,40 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
 enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
@@ -159,6 +193,7 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)

>From 1bb0144eb21c2b1c8e1e66029d5573d8f5619c4c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 14:24:44 +0200
Subject: [PATCH 8/8] Enforce REAL(16) support

---
 flang-rt/CMakeLists.txt      | 45 ++++-------------------
 llvm/runtimes/CMakeLists.txt |  8 ++++
 runtimes/CMakeLists.txt      | 71 ++++++++++++++++++------------------
 3 files changed, 51 insertions(+), 73 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d987115c8e9e8..bfbd0af0c31dc 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,41 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
@@ -79,6 +44,11 @@ include(CMakePushCheckState)
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
+# Fortran compiler not optional for building Flang-RT
+enable_language(Fortran)
+
+flang_module_fortran_enable()
+
 
 #################
 # Build Options #
@@ -193,7 +163,9 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+# Look for support of REAL(16), if not already defined via command line.
+# NOTE: Does not work with Flang and CMake < 3.24
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
@@ -206,7 +178,6 @@ check_fortran_source_compiles([[
 )
 cmake_pop_check_state()
 
-flang_module_fortran_enable()
 
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 2f2f6db16255e..2bf2b0ee8ed50 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -528,6 +528,14 @@ if(build_runtimes)
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
+  if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    # Ensure REAL(16) support in runtimes to be consistent with compiler
+    if (FLANG_RUNTIME_F128_MATH_LIB OR HAVE_LDBL_MANT_DIG_113)
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=TRUE")
+   else ()
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=FALSE")
+   endif ()
+  endif ()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 987d173a283be..51214a46f558e 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -94,6 +94,41 @@ include(CheckCXXCompilerFlag)
 include(ExtendPath)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang, if used.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
+
 # Determine whether we are in the runtimes/runtimes-bins directory of a
 # bootstrapping build.
 set(LLVM_TREE_AVAILABLE OFF)
@@ -334,42 +369,6 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
-      _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-
-    enable_language(Fortran)
-  endif ()
-endif ()
-
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir


From flang-commits at lists.llvm.org  Fri Jul 18 08:20:04 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 08:20:04 -0700 (PDT)
Subject: [flang-commits] [flang] ff5f3ae - [flang] convert program name to
	upper case (NFC) (#149508)
Message-ID: <687a6624.170a0220.2bc316.578b@mx.google.com>


Author: Kelvin Li
Date: 2025-07-18T11:20:00-04:00
New Revision: ff5f3ae02aeac848dbb80ad9c652eae3ec107201

URL: https://github.com/llvm/llvm-project/commit/ff5f3ae02aeac848dbb80ad9c652eae3ec107201
DIFF: https://github.com/llvm/llvm-project/commit/ff5f3ae02aeac848dbb80ad9c652eae3ec107201.diff

LOG: [flang] convert program name to upper case (NFC) (#149508)

Added: 
    

Modified: 
    flang/test/Semantics/PowerPC/ppc-vector-types01.f90
    flang/test/Semantics/PowerPC/ppc-vector-types02.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Semantics/PowerPC/ppc-vector-types01.f90 b/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
index ad69b69a47f76..ea54a00fa4392 100644
--- a/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
+++ b/flang/test/Semantics/PowerPC/ppc-vector-types01.f90
@@ -1,7 +1,7 @@
 ! RUN: %flang_fc1 -fdebug-unparse %s | FileCheck %s
 ! REQUIRES: target=powerpc{{.*}}
 
-    ! CHECK-LABEL: PROGRAM ppc_vec_unit
+    ! CHECK-LABEL: PROGRAM PPC_VEC_UNIT
       program ppc_vec_unit
       implicit none
       ! CHECK: VECTOR(INTEGER(KIND=4_4)) :: vi1, vi2

diff  --git a/flang/test/Semantics/PowerPC/ppc-vector-types02.f90 b/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
index 8c96684c50eb7..175b58680a209 100644
--- a/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
+++ b/flang/test/Semantics/PowerPC/ppc-vector-types02.f90
@@ -2,7 +2,7 @@
 ! REQUIRES: target=powerpc{{.*}}
 
 ! C: MainProgram scope: ppc_vec_types
-! CHECK-LABEL: MainProgram scope: ppc_vec_types size={{[0-9]*}} alignment={{[0-9]*}}
+! CHECK-LABEL: MainProgram scope: PPC_VEC_TYPES size={{[0-9]*}} alignment={{[0-9]*}}
 program ppc_vec_types
   implicit none
   vector(integer(4)) :: vi


        

From flang-commits at lists.llvm.org  Fri Jul 18 08:24:45 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 08:24:45 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a673d.170a0220.2ccdf9.a1c7@mx.google.com>


https://github.com/klausler approved this pull request.


https://github.com/llvm/llvm-project/pull/149513

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:09 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:09 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cb9.170a0220.30a0f5.a5b7@mx.google.com>


================
@@ -941,10 +941,32 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                   m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
             match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
     };
-    if (isIntMinMaxRecurrenceKind(Kind) ||
-        (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+    if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
-    else if (isFMulAddIntrinsic(I))
+    if (isFPMinMaxRecurrenceKind(Kind)) {
+      InstDesc Res = isMinMaxPattern(I, Kind, Prev);
+      if (!Res.isRecurrence())
+        return InstDesc(false, I);
+
+      if (HasRequiredFMF())
+        return Res;
+
+      // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
+      // intrinsics with extra checks ensuring the vector loop handles only
+      // non-NaN inputs.
+      if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMax &&
+               "unexpected recurrence kind for maxnum");
+        return InstDesc(I, RecurKind::FMaxNum);
+      }
+      if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMin &&
+               "unexpected recurrence kind for minnum");
+        return InstDesc(I, RecurKind::FMinNum);
+      }
+
+      return InstDesc(false, I);
+    } else if (isFMulAddIntrinsic(I))
----------------
fhahn wrote:

Done thanks!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:10 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:10 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cba.170a0220.399259.a518@mx.google.com>


================
@@ -1096,7 +1100,8 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
                             Value *Right) {
   Type *Ty = Left->getType();
   if (Ty->isIntOrIntVectorTy() ||
-      (RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
+      (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
+       RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
        RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
     // TODO: Add float minnum/maxnum support when FMF nnan is set.
----------------
fhahn wrote:

I removed it now, as we support FMinNum/FMaxNUm now. The VPlan-transform can also be updated to check for nnan flags and avoid checks if present.

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:10 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:10 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cba.050a0220.1129a9.9fb9@mx.google.com>


================
@@ -42,18 +42,60 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
----------------
fhahn wrote:

Yep, we can also additionally check if the input already has nnan, and avoid adding the extra checks as follow-up

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:10 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:10 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cba.170a0220.31d4f2.a366@mx.google.com>


================
@@ -652,3 +652,146 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                           : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (RepR &&
+            cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+                RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
+
+    if (MinMaxR->getOperand(0) == RedPhiR)
+      return MinMaxR->getOperand(1);
+
+    assert(MinMaxR->getOperand(1) == RedPhiR &&
+           "Reduction phi operand expected");
+    return MinMaxR->getOperand(0);
+  };
+
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support fixed-order recurrence phis.
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
+    // For now, only a single reduction is supported.
+    // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
+    if (RedPhiR)
+      return false;
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNum)
+      continue;
+    RedPhiR = Cur;
+  }
+
+  if (!RedPhiR)
+    return true;
+
+  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+         "unsupported reduction");
+
+  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+  if (!MinMaxOp)
+    return false;
----------------
fhahn wrote:

Moved up, thanks!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:10 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:10 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cba.170a0220.26bd12.9f79@mx.google.com>


================
@@ -652,3 +652,146 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                           : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (RepR &&
+            cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+                RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
+
+    if (MinMaxR->getOperand(0) == RedPhiR)
+      return MinMaxR->getOperand(1);
+
+    assert(MinMaxR->getOperand(1) == RedPhiR &&
+           "Reduction phi operand expected");
+    return MinMaxR->getOperand(0);
+  };
+
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support fixed-order recurrence phis.
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
----------------
fhahn wrote:

Updated thanks!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:10 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:10 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cba.170a0220.1fbfed.9ef8@mx.google.com>


================
@@ -652,3 +652,140 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  VPValue *MinMaxOp = nullptr;
+  bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
+                                  VPReductionPHIRecipe *RedPhi) -> VPValue * {
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+    if (MinMaxOp->getOperand(0) == RedPhi)
+      return MinMaxOp->getOperand(1);
+    assert(MinMaxOp->getOperand(1) == RedPhi &&
+           "Reduction phi operand expected");
+    return MinMaxOp->getOperand(0);
+  };
+
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support first-order recurrence phis.
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
+    // For now, only a single reduction is supported.
+    // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
+    if (RedPhiR)
+      return false;
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNum)
+      continue;
+
+    RedPhiR = Cur;
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return false;
+    MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
+    if (!MinMaxOp)
+      return false;
+  }
+
+  if (!RedPhiR)
+    return true;
+
+  if (HasUnsupportedPhi || !Plan.hasScalarTail())
+    return false;
+
+  /// Check if the vector loop of \p Plan can early exit and restart
+  /// execution of last vector iteration in the scalar loop. This requires all
+  /// recipes up to early exit point be side-effect free as they are
+  /// re-executed. Currently we check that the loop is free of any recipe that
+  /// may write to memory. Expected to operate on an early VPlan w/o nested
+  /// regions.
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = cast<VPBasicBlock>(VPB);
+    for (auto &R : *VPBB) {
+      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        continue;
+      if (R.mayWriteToMemory())
+        return false;
+    }
+  }
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
+  auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
+  if (!RdxResult ||
+      RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
+      RdxResult->getOperand(0) != RedPhiR)
+    return false;
+
+  // Create a new reduction phi recipe with either FMin/FMax, replacing
+  // FMinNum/FMaxNum.
+  RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
+                        ? RecurKind::FMin
+                        : RecurKind::FMax;
+  auto *NewRedPhiR = new VPReductionPHIRecipe(
+      cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
+      *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
+  NewRedPhiR->addOperand(RedPhiR->getOperand(1));
+  NewRedPhiR->insertBefore(RedPhiR);
+  RedPhiR->replaceAllUsesWith(NewRedPhiR);
+  RedPhiR->eraseFromParent();
+
+  // Update the loop exit condition to exit if either any of the inputs is NaN
+  // or the vector trip count is reached.
+  VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
+  VPBuilder Builder(LatchVPBB->getTerminator());
+  auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+  assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
+         "Unexpected terminator");
+  auto *IsLatchExitTaken =
+      Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
+                         LatchExitingBranch->getOperand(1));
+
+  VPValue *IsNaN = Builder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
+  VPValue *AnyNaN = Builder.createNaryOp(VPInstruction::AnyOf, {IsNaN});
+  auto *AnyExitTaken =
+      Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
+  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
+  LatchExitingBranch->eraseFromParent();
+
+  // If we exit early due to NaNs, compute the final reduction result based on
+  // the reduction phi at the beginning of the last vector iteration.
+  Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
+  auto *NewSel =
+      Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+  RdxResult->setOperand(1, NewSel);
+
+  auto *ScalarPH = Plan.getScalarPreheader();
+  // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+  // true, the resume from the start of the last vector iteration via the
+  // canonical IV, otherwise from the original value.
+  for (auto &R : ScalarPH->phis()) {
+    auto *ResumeR = cast<VPPhi>(&R);
+    VPValue *VecV = ResumeR->getOperand(0);
+    if (VecV == RdxResult)
+      continue;
+    if (VecV != &Plan.getVectorTripCount())
+      return false;
----------------
fhahn wrote:

Thanks, added a debug message + comment!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:10 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:10 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cba.170a0220.43545.a18f@mx.google.com>


================
@@ -652,3 +652,146 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                           : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (RepR &&
+            cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+                RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
+
+    if (MinMaxR->getOperand(0) == RedPhiR)
+      return MinMaxR->getOperand(1);
+
+    assert(MinMaxR->getOperand(1) == RedPhiR &&
+           "Reduction phi operand expected");
+    return MinMaxR->getOperand(0);
+  };
+
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support fixed-order recurrence phis.
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
+    // For now, only a single reduction is supported.
+    // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
+    if (RedPhiR)
+      return false;
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNum)
+      continue;
----------------
fhahn wrote:

Ah yes, better to handle this earlier. Currently this will be rejected when updating the resume phis, but it seems better to bail out earlier.

Added a new test to cover this, `@fmaxnum_with_additional_add`, thanks!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:10 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:10 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cba.630a0220.232362.9d51@mx.google.com>


================
@@ -652,3 +652,146 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                           : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (RepR &&
+            cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+                RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
+
+    if (MinMaxR->getOperand(0) == RedPhiR)
+      return MinMaxR->getOperand(1);
+
+    assert(MinMaxR->getOperand(1) == RedPhiR &&
+           "Reduction phi operand expected");
+    return MinMaxR->getOperand(0);
+  };
+
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support fixed-order recurrence phis.
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
+    // For now, only a single reduction is supported.
+    // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
+    if (RedPhiR)
+      return false;
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNum)
+      continue;
+    RedPhiR = Cur;
+  }
+
+  if (!RedPhiR)
+    return true;
+
+  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+         "unsupported reduction");
+
+  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+  if (!MinMaxOp)
+    return false;
+
+  // We won't be able to resume execution in the scalar tail, if there are
+  // unsupported header phis or there is no scalar tail at all, due to
+  // tail-folding.
+  if (HasUnsupportedPhi || !Plan.hasScalarTail())
+    return false;
+
+  /// Check if the vector loop of \p Plan can early exit and restart
+  /// execution of last vector iteration in the scalar loop. This requires all
+  /// recipes up to early exit point be side-effect free as they are
+  /// re-executed. Currently we check that the loop is free of any recipe that
+  /// may write to memory. Expected to operate on an early VPlan w/o nested
+  /// regions.
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = cast<VPBasicBlock>(VPB);
+    for (auto &R : *VPBB) {
+      if (R.mayWriteToMemory() &&
+          !match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        return false;
+    }
+  }
+
+  VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
+  VPBuilder Builder(LatchVPBB->getTerminator());
+  auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+  assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
+         "Unexpected terminator");
+  auto *IsLatchExitTaken =
+      Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
+                         LatchExitingBranch->getOperand(1));
+
+  VPValue *IsNaN = Builder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
+  VPValue *AnyNaN = Builder.createNaryOp(VPInstruction::AnyOf, {IsNaN});
+  auto *AnyExitTaken =
+      Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
+  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
+  LatchExitingBranch->eraseFromParent();
+
+  // If we exit early due to NaNs, compute the final reduction result based on
+  // the reduction phi at the beginning of the last vector iteration.
+  auto *RdxResult = find_singleton<VPSingleDefRecipe>(
+      RedPhiR->users(), [](VPUser *U, bool) -> VPSingleDefRecipe * {
+        auto *VPI = dyn_cast<VPInstruction>(U);
+        if (VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult)
+          return VPI;
+        return nullptr;
+      });
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
+  Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
+  auto *NewSel =
+      Builder.createSelect(AnyNaN, RedPhiR, RdxResult->getOperand(1));
+  RdxResult->setOperand(1, NewSel);
+
+  auto *ScalarPH = Plan.getScalarPreheader();
+  // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+  // true, the resume from the start of the last vector iteration via the
----------------
fhahn wrote:

Updated, thanks

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:13 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cbd.170a0220.343223.a0dd@mx.google.com>


================
@@ -941,10 +941,32 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                   m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
             match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
     };
-    if (isIntMinMaxRecurrenceKind(Kind) ||
-        (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+    if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
-    else if (isFMulAddIntrinsic(I))
+    if (isFPMinMaxRecurrenceKind(Kind)) {
+      InstDesc Res = isMinMaxPattern(I, Kind, Prev);
+      if (!Res.isRecurrence())
+        return InstDesc(false, I);
+
+      if (HasRequiredFMF())
+        return Res;
+
----------------
fhahn wrote:

Done, thanks!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:11 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:11 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cbb.050a0220.2be3ad.a118@mx.google.com>


================
@@ -941,10 +941,32 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                   m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
             match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
     };
-    if (isIntMinMaxRecurrenceKind(Kind) ||
-        (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+    if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
-    else if (isFMulAddIntrinsic(I))
+    if (isFPMinMaxRecurrenceKind(Kind)) {
+      InstDesc Res = isMinMaxPattern(I, Kind, Prev);
+      if (!Res.isRecurrence())
+        return InstDesc(false, I);
+
----------------
fhahn wrote:

Done, thanks!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:11 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:11 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cbb.170a0220.3096b4.a57c@mx.google.com>


================
@@ -652,3 +652,146 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
----------------
fhahn wrote:

Removed, thanks

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:12 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:12 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cbc.170a0220.345077.a286@mx.google.com>


================
@@ -103,6 +103,12 @@ struct VPlanTransforms {
   /// not valid.
   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
 
+  /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
----------------
fhahn wrote:

Updated thanks!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:13 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cbd.170a0220.1358a9.b9ac@mx.google.com>


================
@@ -941,10 +941,32 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                   m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
             match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
     };
-    if (isIntMinMaxRecurrenceKind(Kind) ||
-        (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+    if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
-    else if (isFMulAddIntrinsic(I))
+    if (isFPMinMaxRecurrenceKind(Kind)) {
+      InstDesc Res = isMinMaxPattern(I, Kind, Prev);
+      if (!Res.isRecurrence())
+        return InstDesc(false, I);
+
+      if (HasRequiredFMF())
+        return Res;
+
+      // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
+      // intrinsics with extra checks ensuring the vector loop handles only
+      // non-NaN inputs.
+      if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMax &&
+               "unexpected recurrence kind for maxnum");
+        return InstDesc(I, RecurKind::FMaxNum);
+      }
+      if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMin &&
+               "unexpected recurrence kind for minnum");
+        return InstDesc(I, RecurKind::FMinNum);
+      }
+
----------------
fhahn wrote:

Done, thanks!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:16 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:16 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cc0.170a0220.202015.a6f9@mx.google.com>


================
@@ -652,3 +652,146 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                           : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (RepR &&
+            cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+                RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
+
+    if (MinMaxR->getOperand(0) == RedPhiR)
+      return MinMaxR->getOperand(1);
+
+    assert(MinMaxR->getOperand(1) == RedPhiR &&
+           "Reduction phi operand expected");
+    return MinMaxR->getOperand(0);
+  };
+
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support fixed-order recurrence phis.
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
----------------
fhahn wrote:

Updated thanks

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 08:48:14 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 08:48:14 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a6cbe.a70a0220.20cf45.a1c5@mx.google.com>


https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/148239

>From 42d11944daf17b42cc9eae6a1b14e25740f37de4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 18 Jun 2025 14:07:04 +0100
Subject: [PATCH 1/6] [LV] Vectorize maxnum/minnum w/o fast-math flags.

Update LV to vectorize maxnum/minnum reductions without fast-math flags,
by adding an extra check in the loop if any inputs to maxnum/minnum are
NaN. If any input is NaN,
 *exit the vector loop,
 *compute the reduction result up to the vector iteration that contained NaN
  inputs and
 * resume in the scalar loop

New recurrence kinds are added for reductions using maxnum/minnum
without fast-math flags. The new recurrence kinds are not supported in
the code to generate IR to perform the reductions to prevent accidential
mis-use. Users need to add the required checks ensuring no NaN inputs,
and convert to regular FMin/FMax recurrence kinds.
---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   2 +
 llvm/lib/Analysis/IVDescriptors.cpp           |  23 ++-
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   2 +
 .../Transforms/Vectorize/LoopVectorize.cpp    |  14 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |   6 +
 llvm/lib/Transforms/Vectorize/VPlan.h         |   8 +-
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   1 +
 .../Vectorize/VPlanConstruction.cpp           | 137 ++++++++++++++++++
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |   8 +-
 .../Transforms/Vectorize/VPlanTransforms.h    |   5 +
 .../AArch64/fmax-without-fast-math-flags.ll   |  54 ++++++-
 .../AArch64/fmin-without-fast-math-flags.ll   |  54 ++++++-
 ...fmax-without-fast-math-flags-interleave.ll |  54 ++++++-
 .../fmax-without-fast-math-flags.ll           |  90 ++++++++++--
 .../fmin-without-fast-math-flags.ll           |  90 ++++++++++--
 .../LoopVectorize/minmax_reduction.ll         |   6 +-
 16 files changed, 492 insertions(+), 62 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index b985292ccee40..e28901c072e29 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,6 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
+  FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
+  FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 39f74beca082f..ef095b73d5419 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -941,10 +941,27 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                   m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
             match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
     };
-    if (isIntMinMaxRecurrenceKind(Kind) ||
-        (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+    if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
-    else if (isFMulAddIntrinsic(I))
+    if (isFPMinMaxRecurrenceKind(Kind)) {
+      if (HasRequiredFMF())
+        return isMinMaxPattern(I, Kind, Prev);
+      // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
+      // intrinsics with extra checks ensuring the inputs are not NaN.
+      auto *StartV = dyn_cast<ConstantFP>(
+          OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
+      if (StartV && !StartV->getValue().isNaN() &&
+          isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
+        if (((Kind == RecurKind::FMax &&
+              match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
+             Kind == RecurKind::FMaxNumNoFMFs))
+          return InstDesc(I, RecurKind::FMaxNumNoFMFs);
+        if (((Kind == RecurKind::FMin &&
+              match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
+             Kind == RecurKind::FMinNumNoFMFs))
+          return InstDesc(I, RecurKind::FMinNumNoFMFs);
+      }
+    } else if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
     return InstDesc(false, I);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 200d1fb854155..0cf7bc7af878c 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -938,8 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
   case RecurKind::UMin:
     return Intrinsic::vector_reduce_umin;
   case RecurKind::FMax:
+  case RecurKind::FMaxNumNoFMFs:
     return Intrinsic::vector_reduce_fmax;
   case RecurKind::FMin:
+  case RecurKind::FMinNumNoFMFs:
     return Intrinsic::vector_reduce_fmin;
   case RecurKind::FMaximum:
     return Intrinsic::vector_reduce_fmaximum;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ceeabd65cced3..b070b04cc3c35 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4347,8 +4347,15 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
   // Cross iteration phis such as reductions need special handling and are
   // currently unsupported.
-  if (any_of(OrigLoop->getHeader()->phis(),
-             [&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
+  if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
+        if (Legal->isReductionVariable(&Phi)) {
+          RecurKind RK =
+              Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
+          return RK == RecurKind::FMinNumNoFMFs ||
+                 RK == RecurKind::FMaxNumNoFMFs;
+        }
+        return Legal->isFixedOrderRecurrence(&Phi);
+      }))
     return false;
 
   // Phis with uses outside of the loop require special handling and are
@@ -8769,6 +8776,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
 
   // Adjust the recipes for any inloop reductions.
   adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+  if (!VPlanTransforms::runPass(
+          VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
+    return nullptr;
 
   // Transform recipes to abstract recipes if it is legal and beneficial and
   // clamp the range for better cost estimation.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index da6af353c709f..b3040d0317c95 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23193,6 +23193,8 @@ class HorizontalReduction {
         case RecurKind::FindFirstIVUMin:
         case RecurKind::FindLastIVSMax:
         case RecurKind::FindLastIVUMax:
+        case RecurKind::FMaxNumNoFMFs:
+        case RecurKind::FMinNumNoFMFs:
         case RecurKind::FMaximumNum:
         case RecurKind::FMinimumNum:
         case RecurKind::None:
@@ -23330,6 +23332,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
+    case RecurKind::FMaxNumNoFMFs:
+    case RecurKind::FMinNumNoFMFs:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
@@ -23432,6 +23436,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
+    case RecurKind::FMaxNumNoFMFs:
+    case RecurKind::FMinNumNoFMFs:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 204268e586b43..70c34ec3c25cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1356,16 +1356,16 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
   unsigned Opcode;
 
 public:
+  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+        Opcode(I.getOpcode()) {}
+
   VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags, const VPIRMetadata &Metadata,
                 DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
         VPIRMetadata(Metadata), Opcode(Opcode) {}
 
-  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
-      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
-        Opcode(I.getOpcode()) {}
-
   ~VPWidenRecipe() override = default;
 
   VPWidenRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index b27a7ffeed208..66657b98b094b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -84,6 +84,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
     return ResTy;
   }
   case Instruction::ICmp:
+  case Instruction::FCmp:
   case VPInstruction::ActiveLaneMask:
     assert(inferScalarType(R->getOperand(0)) ==
                inferScalarType(R->getOperand(1)) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 7fb5e82f9d32b..69efb9f89b479 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -652,3 +652,140 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
+                                      VPReductionPHIRecipe *RedPhi) {
+  auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+  if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+      !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+    return nullptr;
+
+  if (MinMaxOp->getOperand(0) == RedPhi)
+    return MinMaxOp->getOperand(1);
+  return MinMaxOp->getOperand(0);
+}
+
+/// Returns true if there VPlan is read-only and execution can be resumed at the
+/// beginning of the last vector iteration in the scalar loop
+static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
+    if (!VPBB)
+      return false;
+    for (auto &R : *VPBB) {
+      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        continue;
+      if (R.mayWriteToMemory())
+        return false;
+    }
+  }
+  return true;
+}
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPValue *AnyNaN = nullptr;
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  VPRecipeWithIRFlags *MinMaxOp = nullptr;
+  bool HasUnsupportedPhi = false;
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
+    if (RedPhiR)
+      return false;
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+      continue;
+
+    RedPhiR = Cur;
+    MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxOp)
+      return false;
+    VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
+    if (!In)
+      return false;
+
+    auto *IsNaN =
+        new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
+    IsNaN->insertBefore(MinMaxOp);
+    AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
+    AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
+  }
+
+  if (!AnyNaN)
+    return true;
+
+  if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+    return false;
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
+  auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
+  if (!RdxResult ||
+      RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
+      RdxResult->getOperand(0) != RedPhiR)
+    return false;
+
+  auto *ScalarPH = Plan.getScalarPreheader();
+  // Update the resume phis in the scalar preheader. They all must either resume
+  // from the reduction result or the canonical induction. Bail out if there are
+  // other resume phis.
+  for (auto &R : ScalarPH->phis()) {
+    auto *ResumeR = cast<VPPhi>(&R);
+    VPValue *VecV = ResumeR->getOperand(0);
+    VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
+    if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
+      return false;
+    ResumeR->setOperand(
+        1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
+    ResumeR->addOperand(BypassV);
+  }
+
+  // Create a new reduction phi recipe with either FMin/FMax, replacing
+  // FMinNumNoFMFs/FMaxNumNoFMFs.
+  RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
+                        ? RecurKind::FMin
+                        : RecurKind::FMax;
+  auto *NewRedPhiR = new VPReductionPHIRecipe(
+      cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
+      *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
+  NewRedPhiR->addOperand(RedPhiR->getOperand(1));
+  NewRedPhiR->insertBefore(RedPhiR);
+  RedPhiR->replaceAllUsesWith(NewRedPhiR);
+  RedPhiR->eraseFromParent();
+
+  // Update the loop exit condition to exit if either any of the inputs is NaN
+  // or the vector trip count is reached.
+  VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
+  VPBuilder Builder(LatchVPBB->getTerminator());
+  auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+  assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
+         "Unexpected terminator");
+  auto *IsLatchExitTaken =
+      Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
+                         LatchExitingBranch->getOperand(1));
+  auto *AnyExitTaken =
+      Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
+  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
+  LatchExitingBranch->eraseFromParent();
+
+  // Split the middle block and introduce a new block, branching to the scalar
+  // preheader to resume iteration in the scalar loop if any NaNs have been
+  // encountered.
+  MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
+  Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
+  auto *NewSel =
+      Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+  RdxResult->setOperand(1, NewSel);
+  Builder.setInsertPoint(MiddleVPBB);
+  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
+  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
+  MiddleVPBB->swapSuccessors();
+  std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
+  return true;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1664bcc3881aa..57b713d3dfcb9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -587,6 +587,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     Value *Op = State.get(getOperand(0), vputils::onlyFirstLaneUsed(this));
     return Builder.CreateFreeze(Op, Name);
   }
+  case Instruction::FCmp:
   case Instruction::ICmp: {
     bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
     Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
@@ -860,7 +861,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     Value *Res = State.get(getOperand(0));
     for (VPValue *Op : drop_begin(operands()))
       Res = Builder.CreateOr(Res, State.get(Op));
-    return Builder.CreateOrReduce(Res);
+    return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res);
   }
   case VPInstruction::FirstActiveLane: {
     if (getNumOperands() == 1) {
@@ -1033,6 +1034,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   switch (getOpcode()) {
   case Instruction::ExtractElement:
   case Instruction::Freeze:
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
   case VPInstruction::AnyOf:
@@ -1068,6 +1070,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
     return Op == getOperand(1);
   case Instruction::PHI:
     return true;
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
   case Instruction::Or:
@@ -1100,6 +1103,7 @@ bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const {
   switch (getOpcode()) {
   default:
     return false;
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
     return vputils::onlyFirstPartUsed(this);
@@ -1786,7 +1790,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
     return Opcode == Instruction::ZExt;
     break;
   case OperationType::Cmp:
-    return Opcode == Instruction::ICmp;
+    return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
   case OperationType::Other:
     return true;
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 84a12470f45e4..52e33ad0d4f99 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -103,6 +103,11 @@ struct VPlanTransforms {
   /// not valid.
   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
 
+  /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
+  /// If they do, try to update the vector loop to exit early if any input is
+  /// NaN and resume executing in the scalar loop to handle the NaNs there.
+  static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
+
   /// Clear NSW/NUW flags from reduction instructions if necessary.
   static void clearReductionWrapFlags(VPlan &Plan);
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 451574a258c2b..1a9752a1ef9fd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -42,18 +42,58 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index e93ee5563b057..3deaa5f0e1f60 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -42,18 +42,58 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0x7FF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index b2e080fef2e57..8ef5b2720e4cd 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -42,18 +42,58 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 5661406b88a5a..332f3a60c5b3e 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -192,18 +192,49 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum_1(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -227,18 +258,49 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum_2(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 148beb64a3609..0e3566b18432a 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -192,18 +192,49 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum_1(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[L]], float [[MAX]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -227,18 +258,49 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum_2(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index 85a90f2e04c5e..1429dc26c5207 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -1001,8 +1001,9 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
+; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmin_intrinsic_nofast(
-; CHECK-NOT: <2 x float> @llvm.minnum.v2f32
+; CHECK: <2 x float> @llvm.minnum.v2f32
 define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body
@@ -1021,8 +1022,9 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
+; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmax_intrinsic_nofast(
-; CHECK-NOT: <2 x float> @llvm.maxnum.v2f32
+; CHECK: <2 x float> @llvm.maxnum.v2f32
 define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body

>From de2d6f2c89a3ead527a03c23eaaa4b7ce2ade11b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Jul 2025 09:53:59 +0100
Subject: [PATCH 2/6] !fixup address latest comments, thanks

---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   4 +-
 llvm/lib/Analysis/IVDescriptors.cpp           |  31 ++--
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   4 +-
 .../Vectorize/LoopVectorizationPlanner.h      |  12 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  10 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  12 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |   8 +-
 .../Vectorize/VPlanConstruction.cpp           | 144 +++++++++---------
 .../Transforms/Vectorize/VPlanTransforms.h    |   1 +
 .../AArch64/fmax-without-fast-math-flags.ll   |  26 ++--
 .../AArch64/fmin-without-fast-math-flags.ll   |  26 ++--
 ...fmax-without-fast-math-flags-interleave.ll |  26 ++--
 .../fmax-without-fast-math-flags.ll           |  44 +++---
 .../fmin-without-fast-math-flags.ll           |  44 +++---
 .../LoopVectorize/minmax_reduction.ll         |   2 +
 15 files changed, 212 insertions(+), 182 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index e28901c072e29..84a7672cd47f3 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,8 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
-  FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
-  FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
+  FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
+  FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index ef095b73d5419..accda5a9713fc 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -944,23 +944,24 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
     if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
     if (isFPMinMaxRecurrenceKind(Kind)) {
+      InstDesc Res = isMinMaxPattern(I, Kind, Prev);
       if (HasRequiredFMF())
-        return isMinMaxPattern(I, Kind, Prev);
+        return Res;
+
+      if (!Res.isRecurrence())
+        return InstDesc(false, I);
+
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
-      // intrinsics with extra checks ensuring the inputs are not NaN.
-      auto *StartV = dyn_cast<ConstantFP>(
-          OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
-      if (StartV && !StartV->getValue().isNaN() &&
-          isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
-        if (((Kind == RecurKind::FMax &&
-              match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
-             Kind == RecurKind::FMaxNumNoFMFs))
-          return InstDesc(I, RecurKind::FMaxNumNoFMFs);
-        if (((Kind == RecurKind::FMin &&
-              match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
-             Kind == RecurKind::FMinNumNoFMFs))
-          return InstDesc(I, RecurKind::FMinNumNoFMFs);
-      }
+      // intrinsics with extra checks ensuring the vector loop handles only
+      // non-NaN inputs.
+      if (Kind == RecurKind::FMax &&
+          match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+        return InstDesc(I, RecurKind::FMaxNum);
+      if (Kind == RecurKind::FMin &&
+          match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+        return InstDesc(I, RecurKind::FMinNum);
+
+      return InstDesc(false, I);
     } else if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 0cf7bc7af878c..7bb7a7b116384 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -938,10 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
   case RecurKind::UMin:
     return Intrinsic::vector_reduce_umin;
   case RecurKind::FMax:
-  case RecurKind::FMaxNumNoFMFs:
+  case RecurKind::FMaxNum:
     return Intrinsic::vector_reduce_fmax;
   case RecurKind::FMin:
-  case RecurKind::FMinNumNoFMFs:
+  case RecurKind::FMinNum:
     return Intrinsic::vector_reduce_fmin;
   case RecurKind::FMaximum:
     return Intrinsic::vector_reduce_fmaximum;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 11853859484e3..f57ce0c3ccb4d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -230,7 +230,6 @@ class VPBuilder {
 
   /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
   /// and \p B.
-  /// TODO: add createFCmp when needed.
   VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
                             DebugLoc DL = DebugLoc::getUnknown(),
                             const Twine &Name = "") {
@@ -240,6 +239,17 @@ class VPBuilder {
         new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name));
   }
 
+  /// Create a new FCmp VPInstruction with predicate \p Pred and operands \p A
+  /// and \p B.
+  VPInstruction *createFCmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
+                            DebugLoc DL = DebugLoc::getUnknown(),
+                            const Twine &Name = "") {
+    assert(Pred >= CmpInst::FIRST_FCMP_PREDICATE &&
+           Pred <= CmpInst::LAST_FCMP_PREDICATE && "invalid predicate");
+    return tryInsertInstruction(
+        new VPInstruction(Instruction::FCmp, {A, B}, Pred, DL, Name));
+  }
+
   VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
                               DebugLoc DL = DebugLoc::getUnknown(),
                               const Twine &Name = "") {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b070b04cc3c35..712c5f8eb4c56 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4345,14 +4345,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
 
 bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
-  // Cross iteration phis such as reductions need special handling and are
-  // currently unsupported.
+  // Cross iteration phis such as first-order recurrences and FMaxNum/FMinNum
+  // reductions need special handling and are currently unsupported.
   if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
         if (Legal->isReductionVariable(&Phi)) {
           RecurKind RK =
               Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
-          return RK == RecurKind::FMinNumNoFMFs ||
-                 RK == RecurKind::FMaxNumNoFMFs;
+          return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
         }
         return Legal->isFixedOrderRecurrence(&Phi);
       }))
@@ -8776,6 +8775,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
 
   // Adjust the recipes for any inloop reductions.
   adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+
+  // Apply mandatory transformation to handle FP maxnum/minnum reduction with
+  // NaNs if possible, bail out otherwise.
   if (!VPlanTransforms::runPass(
           VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
     return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b3040d0317c95..c1e4ff00a65e9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23193,8 +23193,8 @@ class HorizontalReduction {
         case RecurKind::FindFirstIVUMin:
         case RecurKind::FindLastIVSMax:
         case RecurKind::FindLastIVUMax:
-        case RecurKind::FMaxNumNoFMFs:
-        case RecurKind::FMinNumNoFMFs:
+        case RecurKind::FMaxNum:
+        case RecurKind::FMinNum:
         case RecurKind::FMaximumNum:
         case RecurKind::FMinimumNum:
         case RecurKind::None:
@@ -23332,8 +23332,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
-    case RecurKind::FMaxNumNoFMFs:
-    case RecurKind::FMinNumNoFMFs:
+    case RecurKind::FMaxNum:
+    case RecurKind::FMinNum:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
@@ -23436,8 +23436,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
-    case RecurKind::FMaxNumNoFMFs:
-    case RecurKind::FMinNumNoFMFs:
+    case RecurKind::FMaxNum:
+    case RecurKind::FMinNum:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 70c34ec3c25cd..204268e586b43 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1356,16 +1356,16 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
   unsigned Opcode;
 
 public:
-  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
-      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
-        Opcode(I.getOpcode()) {}
-
   VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags, const VPIRMetadata &Metadata,
                 DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
         VPIRMetadata(Metadata), Opcode(Opcode) {}
 
+  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+        Opcode(I.getOpcode()) {}
+
   ~VPWidenRecipe() override = default;
 
   VPWidenRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 69efb9f89b479..8d1ec5b68457e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -653,77 +653,75 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
   }
 }
 
-static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
-                                      VPReductionPHIRecipe *RedPhi) {
-  auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
-  if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
-      !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
-    return nullptr;
-
-  if (MinMaxOp->getOperand(0) == RedPhi)
-    return MinMaxOp->getOperand(1);
-  return MinMaxOp->getOperand(0);
-}
-
-/// Returns true if there VPlan is read-only and execution can be resumed at the
-/// beginning of the last vector iteration in the scalar loop
-static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
-  for (VPBlockBase *VPB : vp_depth_first_shallow(
-           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
-    auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
-    if (!VPBB)
-      return false;
-    for (auto &R : *VPBB) {
-      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
-        continue;
-      if (R.mayWriteToMemory())
-        return false;
-    }
-  }
-  return true;
-}
-
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-  VPValue *AnyNaN = nullptr;
   VPReductionPHIRecipe *RedPhiR = nullptr;
-  VPRecipeWithIRFlags *MinMaxOp = nullptr;
+  VPValue *MinMaxOp = nullptr;
   bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
+                                  VPReductionPHIRecipe *RedPhi) -> VPValue * {
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+    if (MinMaxOp->getOperand(0) == RedPhi)
+      return MinMaxOp->getOperand(1);
+    assert(MinMaxOp->getOperand(1) == RedPhi &&
+           "Reduction phi operand expected");
+    return MinMaxOp->getOperand(0);
+  };
+
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support first-order recurrence phis.
     HasUnsupportedPhi |=
         !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
              VPReductionPHIRecipe>(&R);
     auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
     if (!Cur)
       continue;
+    // For now, only a single reduction is supported.
+    // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
     if (RedPhiR)
       return false;
-    if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
-        Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNum)
       continue;
 
     RedPhiR = Cur;
-    MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
         RedPhiR->getBackedgeValue()->getDefiningRecipe());
-    if (!MinMaxOp)
+    if (!MinMaxR)
       return false;
-    VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
-    if (!In)
+    MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
+    if (!MinMaxOp)
       return false;
-
-    auto *IsNaN =
-        new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
-    IsNaN->insertBefore(MinMaxOp);
-    AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
-    AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
   }
 
-  if (!AnyNaN)
+  if (!RedPhiR)
     return true;
 
-  if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+  if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
+  /// Check if the vector loop of \p Plan can early exit and restart
+  /// execution of last vector iteration in the scalar loop. This requires all
+  /// recipes up to early exit point be side-effect free as they are
+  /// re-executed. Currently we check that the loop is free of any recipe that
+  /// may write to memory. Expected to operate on an early VPlan w/o nested
+  /// regions.
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = cast<VPBasicBlock>(VPB);
+    for (auto &R : *VPBB) {
+      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        continue;
+      if (R.mayWriteToMemory())
+        return false;
+    }
+  }
+
   auto *MiddleVPBB = Plan.getMiddleBlock();
   auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
   if (!RdxResult ||
@@ -731,24 +729,9 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
       RdxResult->getOperand(0) != RedPhiR)
     return false;
 
-  auto *ScalarPH = Plan.getScalarPreheader();
-  // Update the resume phis in the scalar preheader. They all must either resume
-  // from the reduction result or the canonical induction. Bail out if there are
-  // other resume phis.
-  for (auto &R : ScalarPH->phis()) {
-    auto *ResumeR = cast<VPPhi>(&R);
-    VPValue *VecV = ResumeR->getOperand(0);
-    VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
-    if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
-      return false;
-    ResumeR->setOperand(
-        1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
-    ResumeR->addOperand(BypassV);
-  }
-
   // Create a new reduction phi recipe with either FMin/FMax, replacing
-  // FMinNumNoFMFs/FMaxNumNoFMFs.
-  RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
+  // FMinNum/FMaxNum.
+  RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
                         ? RecurKind::FMin
                         : RecurKind::FMax;
   auto *NewRedPhiR = new VPReductionPHIRecipe(
@@ -769,23 +752,40 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   auto *IsLatchExitTaken =
       Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
                          LatchExitingBranch->getOperand(1));
+
+  VPValue *IsNaN = Builder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
+  VPValue *AnyNaN = Builder.createNaryOp(VPInstruction::AnyOf, {IsNaN});
   auto *AnyExitTaken =
       Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
   Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
   LatchExitingBranch->eraseFromParent();
 
-  // Split the middle block and introduce a new block, branching to the scalar
-  // preheader to resume iteration in the scalar loop if any NaNs have been
-  // encountered.
-  MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
+  // If we exit early due to NaNs, compute the final reduction result based on
+  // the reduction phi at the beginning of the last vector iteration.
   Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
   auto *NewSel =
       Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
   RdxResult->setOperand(1, NewSel);
-  Builder.setInsertPoint(MiddleVPBB);
-  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
-  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
-  MiddleVPBB->swapSuccessors();
-  std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
+
+  auto *ScalarPH = Plan.getScalarPreheader();
+  // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+  // true, the resume from the start of the last vector iteration via the
+  // canonical IV, otherwise from the original value.
+  for (auto &R : ScalarPH->phis()) {
+    auto *ResumeR = cast<VPPhi>(&R);
+    VPValue *VecV = ResumeR->getOperand(0);
+    if (VecV == RdxResult)
+      continue;
+    if (VecV != &Plan.getVectorTripCount())
+      return false;
+    auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
+    ResumeR->setOperand(0, NewSel);
+  }
+
+  auto *MiddleTerm = MiddleVPBB->getTerminator();
+  Builder.setInsertPoint(MiddleTerm);
+  VPValue *MiddleCond = MiddleTerm->getOperand(0);
+  VPValue *NewCond = Builder.createAnd(MiddleCond, Builder.createNot(AnyNaN));
+  MiddleTerm->setOperand(0, NewCond);
   return true;
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 52e33ad0d4f99..b1692d3e6b337 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -106,6 +106,7 @@ struct VPlanTransforms {
   /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
   /// If they do, try to update the vector loop to exit early if any input is
   /// NaN and resume executing in the scalar loop to handle the NaNs there.
+  /// Return false if this attempt was unsuccessful.
   static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
 
   /// Clear NSW/NUW flags from reduction instructions if necessary.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 1a9752a1ef9fd..6810c90f0b829 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -57,31 +57,33 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index 3deaa5f0e1f60..a18714f94343b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -57,31 +57,33 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 8ef5b2720e4cd..5e60b32e3ae6b 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -57,31 +57,33 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 332f3a60c5b3e..5199bc2bad1da 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -204,25 +204,27 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -234,7 +236,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -270,25 +272,27 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -300,7 +304,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 0e3566b18432a..f220b900d4d0f 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -204,25 +204,27 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -234,7 +236,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -270,25 +272,27 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -300,7 +304,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index 1429dc26c5207..e7ab02cd98a5e 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -1004,6 +1004,7 @@ for.body:                                         ; preds = %entry, %for.body
 ; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmin_intrinsic_nofast(
 ; CHECK: <2 x float> @llvm.minnum.v2f32
+; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
 define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body
@@ -1025,6 +1026,7 @@ for.body:                                         ; preds = %entry, %for.body
 ; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmax_intrinsic_nofast(
 ; CHECK: <2 x float> @llvm.maxnum.v2f32
+; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
 define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body

>From 8ebc7ff41e3f68c42a99492fce9a5f55e5f23bde Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Jul 2025 15:36:12 +0100
Subject: [PATCH 3/6] !fixup add assertion for intrinsic ID

---
 .../lib/Transforms/Vectorize/VPlanConstruction.cpp | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8d1ec5b68457e..21e3ec3e3f0fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -697,6 +697,20 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
     if (!MinMaxOp)
       return false;
+
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        Cur->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                       : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (isa<VPReplicateRecipe>(MinMaxR) &&
+            cast<IntrinsicInst>(
+                cast<VPReplicateRecipe>(MinMaxR)->getUnderlyingInstr())
+                    ->getIntrinsicID() == RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
   }
 
   if (!RedPhiR)

>From 914a0b4b32617cc63b1c902402fd670e4055267d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 10:27:24 +0100
Subject: [PATCH 4/6] !fixup address latest comments, thanks

---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   5 +-
 llvm/lib/Analysis/IVDescriptors.cpp           |  18 +--
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   7 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  12 +-
 .../Vectorize/VPlanConstruction.cpp           | 108 ++++++++----------
 .../AArch64/fmax-without-fast-math-flags.ll   |   7 +-
 .../AArch64/fmin-without-fast-math-flags.ll   |   7 +-
 ...fmax-without-fast-math-flags-interleave.ll |   7 +-
 .../fmax-without-fast-math-flags.ll           |   4 +-
 .../fmin-without-fast-math-flags.ll           |   4 +-
 10 files changed, 88 insertions(+), 91 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 84a7672cd47f3..1dc73205a0ebb 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,8 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
-  FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
-  FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
+  FMinNum,  ///< FP min with llvm.minnum semantics including NaNs.
+  FMaxNum,  ///< FP max with llvm.maxnum semantics including NaNs.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
@@ -252,6 +252,7 @@ class RecurrenceDescriptor {
   /// Returns true if the recurrence kind is a floating-point min/max kind.
   static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
     return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
+           Kind == RecurKind::FMinNum || Kind == RecurKind::FMaxNum ||
            Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
            Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum;
   }
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index accda5a9713fc..a88bbd7b65d53 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -945,21 +945,25 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
       return isMinMaxPattern(I, Kind, Prev);
     if (isFPMinMaxRecurrenceKind(Kind)) {
       InstDesc Res = isMinMaxPattern(I, Kind, Prev);
-      if (HasRequiredFMF())
-        return Res;
-
       if (!Res.isRecurrence())
         return InstDesc(false, I);
 
+      if (HasRequiredFMF())
+        return Res;
+
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
       // intrinsics with extra checks ensuring the vector loop handles only
       // non-NaN inputs.
-      if (Kind == RecurKind::FMax &&
-          match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+      if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMax &&
+               "unexpected recurrence kind for maxnum");
         return InstDesc(I, RecurKind::FMaxNum);
-      if (Kind == RecurKind::FMin &&
-          match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+      }
+      if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMin &&
+               "unexpected recurrence kind for minnum");
         return InstDesc(I, RecurKind::FMinNum);
+      }
 
       return InstDesc(false, I);
     } else if (isFMulAddIntrinsic(I))
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 7bb7a7b116384..154f50acf3d1d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1039,8 +1039,10 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) {
   case RecurKind::SMax:
     return Intrinsic::smax;
   case RecurKind::FMin:
+  case RecurKind::FMinNum:
     return Intrinsic::minnum;
   case RecurKind::FMax:
+  case RecurKind::FMaxNum:
     return Intrinsic::maxnum;
   case RecurKind::FMinimum:
     return Intrinsic::minimum;
@@ -1098,7 +1100,8 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
                             Value *Right) {
   Type *Ty = Left->getType();
   if (Ty->isIntOrIntVectorTy() ||
-      (RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
+      (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
+       RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
        RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
     // TODO: Add float minnum/maxnum support when FMF nnan is set.
     Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
@@ -1310,6 +1313,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
   case RecurKind::UMin:
   case RecurKind::FMax:
   case RecurKind::FMin:
+  case RecurKind::FMinNum:
+  case RecurKind::FMaxNum:
   case RecurKind::FMinimum:
   case RecurKind::FMaximum:
   case RecurKind::FMinimumNum:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 712c5f8eb4c56..db0902c589d49 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4345,15 +4345,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
 
 bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
-  // Cross iteration phis such as first-order recurrences and FMaxNum/FMinNum
+  // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
   // reductions need special handling and are currently unsupported.
   if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
-        if (Legal->isReductionVariable(&Phi)) {
-          RecurKind RK =
-              Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
-          return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
-        }
-        return Legal->isFixedOrderRecurrence(&Phi);
+        if (!Legal->isReductionVariable(&Phi))
+          return Legal->isFixedOrderRecurrence(&Phi);
+        RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
+        return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
       }))
     return false;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 21e3ec3e3f0fe..a426d06b4befe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -656,25 +656,42 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   VPReductionPHIRecipe *RedPhiR = nullptr;
-  VPValue *MinMaxOp = nullptr;
   bool HasUnsupportedPhi = false;
 
-  auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
-                                  VPReductionPHIRecipe *RedPhi) -> VPValue * {
-    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
-    if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
         !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
       return nullptr;
 
-    if (MinMaxOp->getOperand(0) == RedPhi)
-      return MinMaxOp->getOperand(1);
-    assert(MinMaxOp->getOperand(1) == RedPhi &&
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                           : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (RepR &&
+            cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+                RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
+
+    if (MinMaxR->getOperand(0) == RedPhiR)
+      return MinMaxR->getOperand(1);
+
+    assert(MinMaxR->getOperand(1) == RedPhiR &&
            "Reduction phi operand expected");
-    return MinMaxOp->getOperand(0);
+    return MinMaxR->getOperand(0);
   };
 
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
-    // TODO: Also support first-order recurrence phis.
+    // TODO: Also support fixed-order recurrence phis.
     HasUnsupportedPhi |=
         !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
              VPReductionPHIRecipe>(&R);
@@ -688,34 +705,23 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
         Cur->getRecurrenceKind() != RecurKind::FMinNum)
       continue;
-
     RedPhiR = Cur;
-    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
-        RedPhiR->getBackedgeValue()->getDefiningRecipe());
-    if (!MinMaxR)
-      return false;
-    MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
-    if (!MinMaxOp)
-      return false;
-
-#ifndef NDEBUG
-    Intrinsic::ID RdxIntrinsicId =
-        Cur->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
-                                                       : Intrinsic::minnum;
-    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
-            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
-                RdxIntrinsicId) ||
-           (isa<VPReplicateRecipe>(MinMaxR) &&
-            cast<IntrinsicInst>(
-                cast<VPReplicateRecipe>(MinMaxR)->getUnderlyingInstr())
-                    ->getIntrinsicID() == RdxIntrinsicId) &&
-               "Intrinsic did not match recurrence kind");
-#endif
   }
 
   if (!RedPhiR)
     return true;
 
+  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+         "unsupported reduction");
+
+  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+  if (!MinMaxOp)
+    return false;
+
+  // We won't be able to resume execution in the scalar tail, if there are
+  // unsupported header phis or there is no scalar tail at all, due to
+  // tail-folding.
   if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
@@ -729,35 +735,12 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
            Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
     auto *VPBB = cast<VPBasicBlock>(VPB);
     for (auto &R : *VPBB) {
-      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
-        continue;
-      if (R.mayWriteToMemory())
+      if (R.mayWriteToMemory() &&
+          !match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
         return false;
     }
   }
 
-  auto *MiddleVPBB = Plan.getMiddleBlock();
-  auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
-  if (!RdxResult ||
-      RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
-      RdxResult->getOperand(0) != RedPhiR)
-    return false;
-
-  // Create a new reduction phi recipe with either FMin/FMax, replacing
-  // FMinNum/FMaxNum.
-  RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
-                        ? RecurKind::FMin
-                        : RecurKind::FMax;
-  auto *NewRedPhiR = new VPReductionPHIRecipe(
-      cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
-      *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
-  NewRedPhiR->addOperand(RedPhiR->getOperand(1));
-  NewRedPhiR->insertBefore(RedPhiR);
-  RedPhiR->replaceAllUsesWith(NewRedPhiR);
-  RedPhiR->eraseFromParent();
-
-  // Update the loop exit condition to exit if either any of the inputs is NaN
-  // or the vector trip count is reached.
   VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
   VPBuilder Builder(LatchVPBB->getTerminator());
   auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
@@ -776,9 +759,18 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
 
   // If we exit early due to NaNs, compute the final reduction result based on
   // the reduction phi at the beginning of the last vector iteration.
+  auto *RdxResult = find_singleton<VPSingleDefRecipe>(
+      RedPhiR->users(), [](VPUser *U, bool) -> VPSingleDefRecipe * {
+        auto *VPI = dyn_cast<VPInstruction>(U);
+        if (VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult)
+          return VPI;
+        return nullptr;
+      });
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
   Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
   auto *NewSel =
-      Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+      Builder.createSelect(AnyNaN, RedPhiR, RdxResult->getOperand(1));
   RdxResult->setOperand(1, NewSel);
 
   auto *ScalarPH = Plan.getScalarPreheader();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 6810c90f0b829..427a05cc1c843 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -50,8 +50,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index a18714f94343b..1a8e5940d88e7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -50,8 +50,8 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0x7FF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 5e60b32e3ae6b..a2eddad179216 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -50,8 +50,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 5199bc2bad1da..6204d540df40c 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -200,7 +200,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
@@ -268,7 +268,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index f220b900d4d0f..68bc8d0640a3f 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -200,7 +200,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
@@ -268,7 +268,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4

>From 18312dc501beba7c0c77b50c7f6e344f44174ee3 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 16:19:36 +0100
Subject: [PATCH 5/6] Merge remote-tracking branch 'origin/main' into
 lv-vectorize-fmaxnum-without-fmfs

---
 .../workflows/build-ci-container-windows.yml  |   2 -
 .github/workflows/build-ci-container.yml      |   2 -
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/Basic/BuiltinsAMDGPU.def  |   1 +
 clang/include/clang/Sema/Overload.h           |   2 -
 .../Core/PathSensitive/ExprEngine.h           |   7 +-
 clang/lib/AST/ByteCode/Descriptor.cpp         |   2 +
 clang/lib/AST/ByteCode/Descriptor.h           |   4 +
 clang/lib/AST/ByteCode/Disasm.cpp             |   1 +
 clang/lib/AST/ByteCode/Interp.cpp             |   5 +-
 clang/lib/AST/ByteCode/InterpBlock.h          |   5 -
 clang/lib/AST/ByteCode/InterpBuiltin.cpp      |   3 +-
 clang/lib/AST/ByteCode/InterpFrame.cpp        |   5 +
 clang/lib/AST/ByteCode/Pointer.h              |   5 +
 clang/lib/CodeGen/CGCall.cpp                  |  15 +-
 clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp   |   1 +
 clang/lib/Driver/ToolChain.cpp                |  40 +-
 clang/lib/Sema/SemaConcept.cpp                |   5 +
 clang/lib/Sema/SemaOverload.cpp               |  49 +-
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp  |  46 +-
 clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp |  48 --
 clang/test/AST/ByteCode/mutable.cpp           |  56 ++-
 clang/test/AST/ByteCode/unions.cpp            |  14 +
 clang/test/Analysis/div-zero-cxx20.cpp        |  61 +++
 clang/test/Analysis/div-zero.cpp              |  60 +++
 clang/test/CodeGen/64bit-swiftcall.c          |   2 +-
 clang/test/CodeGen/AArch64/byval-temp.c       |  16 +-
 .../AArch64/pure-scalable-args-empty-union.c  |   2 +-
 .../test/CodeGen/AArch64/pure-scalable-args.c |  42 +-
 .../AArch64/struct-coerce-using-ptr.cpp       |   4 +-
 ...-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c |   6 +-
 ...cle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp |   2 +-
 clang/test/CodeGen/LoongArch/bitint.c         |   6 +-
 clang/test/CodeGen/PowerPC/ppc64-vector.c     |   2 +-
 clang/test/CodeGen/RISCV/riscv-abi.cpp        |   8 +-
 .../RISCV/riscv-vector-callingconv-llvm-ir.c  |  10 +-
 .../riscv-vector-callingconv-llvm-ir.cpp      |  10 +-
 clang/test/CodeGen/RISCV/riscv32-abi.c        |  74 +--
 clang/test/CodeGen/RISCV/riscv32-vararg.c     |   2 +-
 clang/test/CodeGen/RISCV/riscv64-abi.c        |  18 +-
 clang/test/CodeGen/RISCV/riscv64-vararg.c     |   2 +-
 .../test/CodeGen/SystemZ/systemz-abi-vector.c |  52 +-
 clang/test/CodeGen/SystemZ/systemz-abi.c      |  38 +-
 .../test/CodeGen/SystemZ/systemz-inline-asm.c |   2 +-
 clang/test/CodeGen/X86/cx-complex-range.c     |   2 +-
 .../test/CodeGen/X86/x86_32-arguments-win32.c |  14 +-
 .../test/CodeGen/X86/x86_64-arguments-win32.c |   2 +-
 clang/test/CodeGen/aapcs64-align.cpp          |   4 +-
 clang/test/CodeGen/arm-aapcs-vfp.c            |   2 +-
 clang/test/CodeGen/arm-abi-vector.c           |   6 +-
 clang/test/CodeGen/arm-swiftcall.c            |   2 +-
 clang/test/CodeGen/arm64-abi-vector.c         |  14 +-
 clang/test/CodeGen/arm64-arguments.c          |  26 +-
 .../CodeGen/arm64-microsoft-arguments.cpp     |   2 +-
 clang/test/CodeGen/armv7k-abi.c               |   2 +-
 clang/test/CodeGen/atomic-arm64.c             |   2 +-
 clang/test/CodeGen/attr-noundef.cpp           |   7 +-
 clang/test/CodeGen/cx-complex-range.c         |  36 +-
 clang/test/CodeGen/ext-int-cc.c               |  36 +-
 clang/test/CodeGen/isfpclass.c                |   2 +-
 .../math-libcalls-tbaa-indirect-args.c        |  86 ++--
 clang/test/CodeGen/mingw-long-double.c        |   6 +-
 clang/test/CodeGen/ms_abi.c                   |   4 +-
 clang/test/CodeGen/pass-by-value-noalias.c    |   4 +-
 clang/test/CodeGen/ptrauth-in-c-struct.c      |   4 +-
 clang/test/CodeGen/regcall.c                  |  10 +-
 clang/test/CodeGen/regcall2.c                 |   2 +-
 clang/test/CodeGen/regcall4.c                 |  10 +-
 clang/test/CodeGen/sparcv9-abi.c              |   4 +-
 clang/test/CodeGen/vectorcall.c               |  46 +-
 clang/test/CodeGen/win-fp128.c                |   2 +-
 clang/test/CodeGen/win64-i128.c               |   4 +-
 clang/test/CodeGen/windows-swiftcall.c        |   2 +-
 .../CodeGenCXX/aarch64-mangle-sve-vectors.cpp |   4 +-
 clang/test/CodeGenCXX/arm-cc.cpp              |   2 +-
 .../CodeGenCXX/attr-target-mv-inalloca.cpp    |   8 +-
 clang/test/CodeGenCXX/copy-initialization.cpp |   2 +-
 clang/test/CodeGenCXX/debug-info.cpp          |   2 +-
 .../empty-nontrivially-copyable.cpp           |   4 +-
 clang/test/CodeGenCXX/fastcall.cpp            |   2 +-
 .../CodeGenCXX/homogeneous-aggregates.cpp     |  14 +-
 clang/test/CodeGenCXX/inalloca-lambda.cpp     |   6 +-
 .../test/CodeGenCXX/inalloca-overaligned.cpp  |   8 +-
 clang/test/CodeGenCXX/inalloca-vector.cpp     |   4 +-
 .../CodeGenCXX/inheriting-constructor.cpp     |   8 +-
 .../member-function-pointer-calls.cpp         |   4 +-
 .../CodeGenCXX/microsoft-abi-arg-order.cpp    |   4 +-
 .../CodeGenCXX/microsoft-abi-byval-thunks.cpp |  12 +-
 .../microsoft-abi-member-pointers.cpp         |   6 +-
 .../microsoft-abi-sret-and-byval.cpp          |  26 +-
 .../CodeGenCXX/microsoft-abi-unknown-arch.cpp |   2 +-
 clang/test/CodeGenCXX/ms-property.cpp         |   4 +-
 clang/test/CodeGenCXX/nrvo.cpp                |   4 +-
 .../test/CodeGenCXX/pass-by-value-noalias.cpp |  12 +-
 .../CodeGenCXX/ptrauth-qualifier-struct.cpp   |   2 +-
 clang/test/CodeGenCXX/regparm.cpp             |   2 +-
 clang/test/CodeGenCXX/trivial_abi.cpp         |   8 +-
 clang/test/CodeGenCXX/uncopyable-args.cpp     |  32 +-
 clang/test/CodeGenCXX/wasm-args-returns.cpp   |  12 +-
 .../test/CodeGenCXX/windows-x86-swiftcall.cpp |   4 +-
 .../nontrivial-c-struct-exception.m           |   2 +-
 .../test/CodeGenObjC/pass-by-value-noalias.m  |   4 +-
 clang/test/CodeGenObjC/weak-in-c-struct.m     |   6 +-
 .../test/CodeGenObjCXX/objc-struct-cxx-abi.mm |  12 +-
 clang/test/CodeGenObjCXX/property-objects.mm  |   2 +-
 .../CodeGenObjCXX/ptrauth-struct-cxx-abi.mm   |   2 +-
 .../CodeGenOpenCL/builtins-amdgcn-gfx1250.cl  |  38 ++
 .../test/Driver/print-multi-selection-flags.c |  19 +
 clang/test/Headers/stdarg.cpp                 |   4 +-
 .../test/OpenMP/for_firstprivate_codegen.cpp  |  30 +-
 .../OpenMP/parallel_firstprivate_codegen.cpp  | 216 ++++-----
 .../OpenMP/sections_firstprivate_codegen.cpp  |  14 +-
 .../OpenMP/single_firstprivate_codegen.cpp    |  14 +-
 ..._teams_distribute_firstprivate_codegen.cpp |  82 ++--
 ...bute_parallel_for_firstprivate_codegen.cpp | 224 ++++-----
 .../teams_distribute_firstprivate_codegen.cpp |  90 ++--
 ...bute_parallel_for_firstprivate_codegen.cpp | 124 ++---
 .../OpenMP/teams_firstprivate_codegen.cpp     |  80 ++--
 clang/test/Sema/dllexport.c                   |   4 +
 ...overload-resolution-deferred-templates.cpp |  28 ++
 .../test/SemaTemplate/concepts-using-decl.cpp |  21 +
 flang/lib/Lower/Bridge.cpp                    |  15 +-
 flang/lib/Lower/ConvertConstant.cpp           |   2 +-
 flang/lib/Lower/ConvertExpr.cpp               |  26 +-
 flang/lib/Lower/ConvertVariable.cpp           |  26 +-
 flang/lib/Lower/Runtime.cpp                   |   4 +-
 flang/lib/Lower/VectorSubscripts.cpp          |   2 +-
 flang/lib/Optimizer/Builder/FIRBuilder.cpp    |   2 +-
 .../Optimizer/Builder/Runtime/Intrinsics.cpp  |   6 +-
 flang/lib/Optimizer/Builder/Runtime/Stop.cpp  |   2 +-
 .../LowerHLFIROrderedAssignments.cpp          |   4 +-
 .../lib/Optimizer/Transforms/MemoryUtils.cpp  |   2 +-
 .../Transforms/PolymorphicOpConversion.cpp    |   2 +-
 .../Transforms/SimplifyFIROperations.cpp      |   2 +-
 flang/test/Lower/CUDA/cuda-set-allocator.cuf  |  32 ++
 .../Optimizer/FortranVariableTest.cpp         |   6 +-
 libc/cmake/modules/LLVMLibCTestRules.cmake    |  17 +-
 libcxx/include/__config_site.in               |   1 -
 libcxx/test/libcxx/fuzzing/random.pass.cpp    |   7 +-
 .../std/depr/depr.c.headers/math_h.pass.cpp   |   7 +-
 .../test/std/numerics/c.math/cmath.pass.cpp   |   7 +-
 lld/ELF/BPSectionOrderer.cpp                  |   6 +-
 lld/test/ELF/bp-section-orderer.s             |  21 +-
 lldb/source/Expression/IRExecutionUnit.cpp    |  16 +-
 .../TestNSDictionarySynthetic.py              |   3 +-
 .../PDB/Inputs/UdtLayoutTest.script           |   1 +
 llvm/Maintainers.md                           |  14 +
 llvm/docs/CodingStandards.rst                 |  46 +-
 .../llvm/CodeGen/MachineBlockFrequencyInfo.h  |   4 +-
 llvm/include/llvm/IR/DebugInfo.h              |   5 +-
 llvm/include/llvm/IR/DebugInfoMetadata.h      |   3 -
 .../llvm/Transforms/Utils/SSAUpdater.h        |   4 -
 llvm/lib/CodeGen/CodeGenPrepare.cpp           |  59 +--
 .../lib/CodeGen/MachineBlockFrequencyInfo.cpp |   4 +-
 llvm/lib/CodeGen/MachineDebugify.cpp          |  18 +-
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  16 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   4 +-
 llvm/lib/IR/DebugInfo.cpp                     |  25 +-
 llvm/lib/IR/DebugInfoMetadata.cpp             |   9 -
 llvm/lib/Object/IRSymtab.cpp                  |  45 +-
 .../Target/AArch64/AArch64ISelLowering.cpp    |   4 +-
 llvm/lib/Target/AMDGPU/AMDGPU.h               |   3 +
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |   1 +
 .../Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp  | 108 +++++
 .../Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h    |  23 +
 .../AMDGPU/AMDGPUResourceUsageAnalysis.cpp    | 234 +--------
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  13 +
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h  |   2 +
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     | 146 ++++--
 llvm/lib/Target/AMDGPU/SIInstrInfo.h          |   1 -
 llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp     |   8 +-
 llvm/lib/Target/AMDGPU/SIRegisterInfo.h       |   6 +-
 llvm/lib/Target/AMDGPU/VOP1Instructions.td    |   1 +
 .../LoongArch/LoongArchFrameLowering.cpp      |   7 +-
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp  | 122 +++--
 llvm/lib/Target/X86/X86CallingConv.cpp        |   2 +-
 llvm/lib/TargetParser/X86TargetParser.cpp     |   6 +-
 llvm/lib/Transforms/Coroutines/CoroFrame.cpp  |  52 +-
 llvm/lib/Transforms/Coroutines/CoroInternal.h |   5 +-
 llvm/lib/Transforms/Coroutines/CoroSplit.cpp  |  23 +-
 llvm/lib/Transforms/Coroutines/SpillUtils.cpp |   6 +-
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |  19 +-
 .../InstCombine/InstCombineInternal.h         |   3 -
 .../InstCombine/InstructionCombining.cpp      |  78 +--
 .../Scalar/ConstraintElimination.cpp          |   1 +
 .../Scalar/DeadStoreElimination.cpp           |   1 +
 llvm/lib/Transforms/Scalar/JumpThreading.cpp  |  39 +-
 .../lib/Transforms/Scalar/LoopInterchange.cpp |  24 +-
 .../Transforms/Scalar/LoopStrengthReduce.cpp  | 184 ++++---
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |   8 +-
 llvm/lib/Transforms/Utils/Debugify.cpp        |   7 -
 llvm/lib/Transforms/Utils/InlineFunction.cpp  |   3 +-
 llvm/lib/Transforms/Utils/Local.cpp           | 103 +---
 .../Transforms/Utils/LoopRotationUtils.cpp    |  23 +-
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   1 -
 .../Utils/PromoteMemoryToRegister.cpp         |  49 +-
 llvm/lib/Transforms/Utils/SSAUpdater.cpp      |  22 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  39 +-
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   9 +-
 llvm/lib/Transforms/Vectorize/VPlanAnalysis.h |   6 +-
 .../AArch64/sve-intrinsics-ldst-ext.ll        |  24 +
 llvm/test/CodeGen/AMDGPU/agpr-remat.ll        |  18 +-
 .../AMDGPU/amdgpu-prepare-agpr-alloc.mir      |  95 ++++
 .../bad-agpr-vgpr-regalloc-priority.mir       |  74 +++
 .../branch-folding-implicit-def-subreg.ll     |  46 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll  |   4 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |   4 +
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll  |   4 +-
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll  |   4 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-mov.ll     |  10 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-mov.mir    |  28 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-read.mir   |  26 +-
 ...al-regcopy-and-spill-missed-at-regalloc.ll |  20 +-
 .../AMDGPU/ptradd-sdag-optimizations.ll       | 196 ++++----
 .../CodeGen/AMDGPU/spill-vector-superclass.ll |   6 +-
 .../CodeGen/LoongArch/calling-conv-common.ll  |  48 +-
 .../CodeGen/LoongArch/calling-conv-half.ll    |  16 +-
 .../LoongArch/can-not-realign-stack.ll        |  44 +-
 .../CodeGen/LoongArch/emergency-spill-slot.ll |   4 +-
 llvm/test/CodeGen/LoongArch/frame.ll          | 107 ++++-
 .../CodeGen/LoongArch/intrinsic-memcpy.ll     |   8 +-
 llvm/test/CodeGen/LoongArch/lasx/fpowi.ll     |  88 ++--
 .../lasx/ir-instruction/extractelement.ll     | 120 ++---
 .../lasx/ir-instruction/insertelement.ll      | 132 +++---
 llvm/test/CodeGen/LoongArch/llvm.sincos.ll    | 150 +++---
 llvm/test/CodeGen/LoongArch/lsx/pr146455.ll   | 287 +++++++++++
 ...realignment-with-variable-sized-objects.ll |  24 +-
 .../CodeGen/LoongArch/stack-realignment.ll    |  80 ++--
 .../LoongArch/unaligned-memcpy-inline.ll      |  14 +-
 llvm/test/CodeGen/LoongArch/vararg.ll         |  70 +--
 llvm/test/CodeGen/NVPTX/i8x4-instructions.ll  |  47 ++
 llvm/test/CodeGen/NVPTX/trunc-tofp.ll         |  81 ++++
 .../CodeGen/RISCV/short-forward-branch-opt.ll |  32 +-
 llvm/test/CodeGen/RISCV/zdinx-spill.ll        |  26 -
 .../AddressSanitizer/lifetime.ll              | 447 +++++++++++++++---
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s |  45 ++
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s        |  45 ++
 .../MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s |  52 ++
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s  |  52 ++
 .../MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s  |  12 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s   |  12 +
 .../gfx1250_asm_vop3_from_vop1-fake16.s       |  36 ++
 .../MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s    |  36 ++
 .../gfx1250_asm_vop3_from_vop1_dpp16-fake16.s |  44 ++
 .../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s |  44 ++
 .../gfx1250_asm_vop3_from_vop1_dpp8-fake16.s  |   4 +
 .../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s  |   4 +
 .../Disassembler/AMDGPU/gfx1250_dasm_vop1.txt |  45 ++
 .../AMDGPU/gfx1250_dasm_vop1_dpp16.txt        |  39 ++
 .../AMDGPU/gfx1250_dasm_vop1_dpp8.txt         |   9 +
 .../AMDGPU/gfx1250_dasm_vop3_from_vop1.txt    |  36 ++
 .../gfx1250_dasm_vop3_from_vop1_dpp16.txt     |  33 ++
 .../gfx1250_dasm_vop3_from_vop1_dpp8.txt      |   7 +-
 .../DeadStoreElimination/zeroed-missing.ll    |   2 +-
 .../LoopInterchange/force-interchange.ll      |  43 ++
 .../LoopStrengthReduce/X86/lifetime-use.ll    |  59 +++
 .../AArch64/maxbandwidth-regpressure.ll       |  38 ++
 .../X86/split-node-reorder-node-with-ops.ll   |   2 -
 llvm/tools/llvm-dis/llvm-dis.cpp              |  14 -
 llvm/unittests/CodeGen/LexicalScopesTest.cpp  |   2 +-
 llvm/unittests/IR/DebugInfoTest.cpp           |   2 +
 .../Transforms/Utils/DebugifyTest.cpp         |   9 +-
 llvm/unittests/Transforms/Utils/LocalTest.cpp |  56 ---
 .../gn/secondary/libcxx/include/BUILD.gn      |   1 -
 .../secondary/llvm/lib/Target/AMDGPU/BUILD.gn |   1 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |   1 +
 .../secondary/llvm/tools/llvm-ir2vec/BUILD.gn |   9 +
 mlir/docs/Tutorials/transform/Ch2.md          |  30 +-
 .../Conversion/VectorToXeGPU/VectorToXeGPU.h  |   3 -
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |  63 ++-
 .../common/include/PluginInterface.h          |  28 +-
 offload/plugins-nextgen/cuda/src/rtl.cpp      |  10 +-
 .../unittests/OffloadAPI/common/Fixtures.hpp  |   3 -
 .../OffloadAPI/event/olWaitEvent.cpp          |   3 -
 275 files changed, 4637 insertions(+), 2846 deletions(-)
 create mode 100644 clang/test/Analysis/div-zero-cxx20.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
 create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
 create mode 100644 llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/trunc-tofp.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/force-interchange.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
 create mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn

diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index 59079f057d021..f76c69f29fb30 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -11,8 +11,6 @@ on:
       - .github/workflows/build-ci-container-windows.yml
       - '.github/workflows/containers/github-action-ci-windows/**'
   pull_request:
-    branches:
-      - main
     paths:
       - .github/workflows/build-ci-container-windows.yml
       - '.github/workflows/containers/github-action-ci-windows/**'
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 3159aae32ca51..7f01264af8534 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -11,8 +11,6 @@ on:
       - .github/workflows/build-ci-container.yml
       - '.github/workflows/containers/github-action-ci/**'
   pull_request:
-    branches:
-      - main
     paths:
       - .github/workflows/build-ci-container.yml
       - '.github/workflows/containers/github-action-ci/**'
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index fcd3887ec7a09..ea16029268dba 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -808,6 +808,7 @@ Bug Fixes in This Version
   nested scopes. (#GH147495)
 - Fixed a failed assertion with an operator call expression which comes from a
   macro expansion when performing analysis for nullability attributes. (#GH138371)
+- Fixed a concept equivalent checking crash due to untransformed constraint expressions. (#GH146614)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1206,6 +1207,8 @@ Static Analyzer
 ---------------
 - Fixed a crash when C++20 parenthesized initializer lists are used. This issue
   was causing a crash in clang-tidy. (#GH136041)
+- The Clang Static Analyzer now handles parenthesized initialization.
+  (#GH148875)
 
 New features
 ^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index ed51f1d5de447..a916af7e0c2df 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -673,6 +673,7 @@ TARGET_BUILTIN(__builtin_amdgcn_tanhf, "ff", "nc", "tanh-insts")
 TARGET_BUILTIN(__builtin_amdgcn_tanhh, "hh", "nc", "tanh-insts")
 TARGET_BUILTIN(__builtin_amdgcn_tanh_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_rcp_bf16, "yy", "nc", "bf16-trans-insts")
+TARGET_BUILTIN(__builtin_amdgcn_sqrt_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_rsq_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_log_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_exp2_bf16, "yy", "nc", "bf16-trans-insts")
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index a70335bef9dd4..d34a4146ddbd6 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -1491,8 +1491,6 @@ class Sema;
     OverloadingResult
     BestViableFunctionImpl(Sema &S, SourceLocation Loc,
                            OverloadCandidateSet::iterator &Best);
-    void PerfectViableFunction(Sema &S, SourceLocation Loc,
-                               OverloadCandidateSet::iterator &Best);
   };
 
   bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index 6370586e218ef..fbb34340a5c67 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -499,9 +499,6 @@ class ExprEngine {
   void VisitGuardedExpr(const Expr *Ex, const Expr *L, const Expr *R,
                         ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
-  void VisitInitListExpr(const InitListExpr *E, ExplodedNode *Pred,
-                         ExplodedNodeSet &Dst);
-
   /// VisitAttributedStmt - Transfer function logic for AttributedStmt.
   void VisitAttributedStmt(const AttributedStmt *A, ExplodedNode *Pred,
                            ExplodedNodeSet &Dst);
@@ -591,6 +588,10 @@ class ExprEngine {
                                 ExplodedNode *Pred,
                                 ExplodedNodeSet &Dst);
 
+  void ConstructInitList(const Expr *Source, ArrayRef<Expr *> Args,
+                         bool IsTransparent, ExplodedNode *Pred,
+                         ExplodedNodeSet &Dst);
+
   /// evalEagerlyAssumeBifurcation - Given the nodes in 'Src', eagerly assume
   /// concrete boolean values for 'Ex', storing the resulting nodes in 'Dst'.
   void evalEagerlyAssumeBifurcation(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index c89eca9bef440..5b9f44518fcc2 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -162,6 +162,8 @@ static void initField(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable,
   Desc->IsConst = IsConst || D->IsConst;
   Desc->IsFieldMutable = IsMutable || D->IsMutable;
   Desc->IsVolatile = IsVolatile || D->IsVolatile;
+  // True if this field is const AND the parent is mutable.
+  Desc->IsConstInMutable = Desc->IsConst && IsMutable;
 
   if (auto Fn = D->CtorFn)
     Fn(B, Ptr + FieldOffset, Desc->IsConst, Desc->IsFieldMutable,
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 4591eabb69bb4..0227e4c0c7e38 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -101,6 +101,10 @@ struct InlineDescriptor {
   /// Flag indicating if the field is mutable (if in a record).
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsFieldMutable : 1;
+  /// Flag indicating if this field is a const field nested in
+  /// a mutable parent field.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned IsConstInMutable : 1;
   /// Flag indicating if the field is an element of a composite array.
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsArrayElement : 1;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index f64501f4a31e8..74399d177b5a2 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -445,6 +445,7 @@ LLVM_DUMP_METHOD void InlineDescriptor::dump(llvm::raw_ostream &OS) const {
   OS << "InUnion: " << InUnion << "\n";
   OS << "IsFieldMutable: " << IsFieldMutable << "\n";
   OS << "IsArrayElement: " << IsArrayElement << "\n";
+  OS << "IsConstInMutable: " << IsConstInMutable << '\n';
   OS << "Desc: ";
   if (Desc)
     Desc->dump(OS);
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index e8b519478c026..df5e3be83d741 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -566,7 +566,10 @@ bool CheckDowncast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
 bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   assert(Ptr.isLive() && "Pointer is not live");
-  if (!Ptr.isConst() || Ptr.isMutable())
+  if (!Ptr.isConst())
+    return true;
+
+  if (Ptr.isMutable() && !Ptr.isConstInMutable())
     return true;
 
   if (!Ptr.isBlockPointer())
diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
index 7798b6f886a85..51622238e275c 100644
--- a/clang/lib/AST/ByteCode/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -14,11 +14,6 @@
 #define LLVM_CLANG_AST_INTERP_BLOCK_H
 
 #include "Descriptor.h"
-#include "clang/AST/ComparisonCategories.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclCXX.h"
-#include "clang/AST/Expr.h"
-#include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index de0b97fd93c76..462b9a11e0a5c 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 #include "../ExprConstShared.h"
 #include "Boolean.h"
-#include "Compiler.h"
 #include "EvalEmitter.h"
 #include "Interp.h"
 #include "InterpBuiltinBitCast.h"
@@ -2906,6 +2905,8 @@ static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,
         if (!copyField(F, /*Activate=*/true))
           return false;
       } else {
+        if (!CheckMutable(S, OpPC, Src.atField(F.Offset)))
+          return false;
         Pointer DestField = Dest.atField(F.Offset);
         zeroAll(DestField);
       }
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index a5a4bd25fe712..d62a4f6275b50 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -128,6 +128,11 @@ static bool shouldSkipInBacktrace(const Function *F) {
   if (FD->getDeclName().getCXXOverloadedOperator() == OO_New ||
       FD->getDeclName().getCXXOverloadedOperator() == OO_Array_New)
     return true;
+
+  if (const auto *MD = dyn_cast<CXXMethodDecl>(FD);
+      MD && MD->getParent()->isAnonymousStructOrUnion())
+    return true;
+
   return false;
 }
 
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index e6a64e6658f06..da74013cf83a6 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -576,6 +576,11 @@ class Pointer {
       return true;
     return isRoot() ? getDeclDesc()->IsConst : getInlineDesc()->IsConst;
   }
+  bool isConstInMutable() const {
+    if (!isBlockPointer())
+      return false;
+    return isRoot() ? false : getInlineDesc()->IsConstInMutable;
+  }
 
   /// Checks if an object or a subfield is volatile.
   bool isVolatile() const {
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index c8c3d6b20c496..19d8ba26d44d8 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2852,8 +2852,21 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
       if (AI.getInReg())
         Attrs.addAttribute(llvm::Attribute::InReg);
 
-      if (AI.getIndirectByVal())
+      // Depending on the ABI, this may be either a byval or a dead_on_return
+      // argument.
+      if (AI.getIndirectByVal()) {
         Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
+      } else {
+        // Add dead_on_return when the object's lifetime ends in the callee.
+        // This includes trivially-destructible objects, as well as objects
+        // whose destruction / clean-up is carried out within the callee (e.g.,
+        // Obj-C ARC-managed structs, MSVC callee-destroyed objects).
+        if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
+            ParamType->castAs<RecordType>()
+                ->getDecl()
+                ->isParamDestroyedInCallee())
+          Attrs.addAttribute(llvm::Attribute::DeadOnReturn);
+      }
 
       auto *Decl = ParamType->getAsRecordDecl();
       if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index a7d796ecccc61..ee736a2816218 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -416,6 +416,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_sqrt:
   case AMDGPU::BI__builtin_amdgcn_sqrtf:
   case AMDGPU::BI__builtin_amdgcn_sqrth:
+  case AMDGPU::BI__builtin_amdgcn_sqrt_bf16:
     return emitBuiltinWithOneOverloadedType<1>(*this, E,
                                                Intrinsic::amdgcn_sqrt);
   case AMDGPU::BI__builtin_amdgcn_rsq:
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..481f575518b93 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -258,10 +258,10 @@ static void getAArch64MultilibFlags(const Driver &D,
   processMultilibCustomFlags(Result, Args);
 }
 
-static void getARMMultilibFlags(const Driver &D,
-                                      const llvm::Triple &Triple,
-                                      const llvm::opt::ArgList &Args,
-                                      Multilib::flags_list &Result) {
+static void getARMMultilibFlags(const Driver &D, const llvm::Triple &Triple,
+                                llvm::Reloc::Model RelocationModel,
+                                const llvm::opt::ArgList &Args,
+                                Multilib::flags_list &Result) {
   std::vector<StringRef> Features;
   llvm::ARM::FPUKind FPUKind = tools::arm::getARMTargetFeatures(
       D, Triple, Args, Features, false /*ForAs*/, true /*ForMultilib*/);
@@ -304,6 +304,18 @@ static void getARMMultilibFlags(const Driver &D,
     llvm_unreachable("Invalid float ABI");
   }
 
+  if (RelocationModel == llvm::Reloc::ROPI ||
+      RelocationModel == llvm::Reloc::ROPI_RWPI)
+    Result.push_back("-fropi");
+  else
+    Result.push_back("-fno-ropi");
+
+  if (RelocationModel == llvm::Reloc::RWPI ||
+      RelocationModel == llvm::Reloc::ROPI_RWPI)
+    Result.push_back("-frwpi");
+  else
+    Result.push_back("-fno-rwpi");
+
   const Arg *BranchProtectionArg =
       Args.getLastArgNoClaim(options::OPT_mbranch_protection_EQ);
   if (BranchProtectionArg) {
@@ -344,6 +356,18 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   const llvm::Triple Triple(ComputeEffectiveClangTriple(Args));
   Result.push_back("--target=" + Triple.str());
 
+  // A difference of relocation model (absolutely addressed data, PIC, Arm
+  // ROPI/RWPI) is likely to change whether a particular multilib variant is
+  // compatible with a given link. Determine the relocation model of the
+  // current link, so as to add appropriate multilib flags.
+  llvm::Reloc::Model RelocationModel;
+  unsigned PICLevel;
+  bool IsPIE;
+  {
+    RegisterEffectiveTriple TripleRAII(*this, Triple);
+    std::tie(RelocationModel, PICLevel, IsPIE) = ParsePICArgs(*this, Args);
+  }
+
   switch (Triple.getArch()) {
   case llvm::Triple::aarch64:
   case llvm::Triple::aarch64_32:
@@ -354,7 +378,7 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   case llvm::Triple::armeb:
   case llvm::Triple::thumb:
   case llvm::Triple::thumbeb:
-    getARMMultilibFlags(D, Triple, Args, Result);
+    getARMMultilibFlags(D, Triple, RelocationModel, Args, Result);
     break;
   case llvm::Triple::riscv32:
   case llvm::Triple::riscv64:
@@ -376,6 +400,12 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   else
     Result.push_back("-fexceptions");
 
+  if (RelocationModel == llvm::Reloc::PIC_)
+    Result.push_back(IsPIE ? (PICLevel > 1 ? "-fPIE" : "-fpie")
+                           : (PICLevel > 1 ? "-fPIC" : "-fpic"));
+  else
+    Result.push_back("-fno-pic");
+
   // Sort and remove duplicates.
   std::sort(Result.begin(), Result.end());
   Result.erase(llvm::unique(Result), Result.end());
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 834417f8e15ac..5205ca0bca6fa 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -925,7 +925,12 @@ static const Expr *SubstituteConstraintExpressionWithoutSatisfaction(
       ND && ND->isFunctionOrFunctionTemplate()) {
     ScopeForParameters.emplace(S, /*CombineWithOuterScope=*/true);
     const FunctionDecl *FD = ND->getAsFunction();
+    if (FunctionTemplateDecl *Template = FD->getDescribedFunctionTemplate();
+        Template && Template->getInstantiatedFromMemberTemplate())
+      FD = Template->getInstantiatedFromMemberTemplate()->getTemplatedDecl();
     for (auto *PVD : FD->parameters()) {
+      if (ScopeForParameters->getInstantiationOfIfExists(PVD))
+        continue;
       if (!PVD->isParameterPack()) {
         ScopeForParameters->InstantiatedLocal(PVD, PVD);
         continue;
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 1b54628c5e564..5dd5b495480d9 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -11354,55 +11354,18 @@ OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
       DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
 
   if (TwoPhaseResolution) {
-
-    PerfectViableFunction(S, Loc, Best);
-    if (Best != end())
-      return ResultForBestCandidate(Best);
+    OverloadingResult Res = BestViableFunctionImpl(S, Loc, Best);
+    if (Best != end() && Best->isPerfectMatch(S.Context)) {
+      if (!(HasDeferredTemplateConstructors &&
+            isa_and_nonnull<CXXConversionDecl>(Best->Function)))
+        return Res;
+    }
   }
 
   InjectNonDeducedTemplateCandidates(S);
   return BestViableFunctionImpl(S, Loc, Best);
 }
 
-void OverloadCandidateSet::PerfectViableFunction(
-    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
-
-  Best = end();
-  for (auto It = Candidates.begin(); It != Candidates.end(); ++It) {
-
-    if (!It->isPerfectMatch(S.getASTContext()))
-      continue;
-
-    // We found a suitable conversion function
-    // but if there is a template constructor in the target class
-    // we might prefer that instead.
-    if (HasDeferredTemplateConstructors &&
-        isa_and_nonnull<CXXConversionDecl>(It->Function)) {
-      Best = end();
-      break;
-    }
-
-    if (Best == end()) {
-      Best = It;
-      continue;
-    }
-    if (Best->Function && It->Function) {
-      FunctionDecl *D =
-          S.getMoreConstrainedFunction(Best->Function, It->Function);
-      if (D == nullptr) {
-        Best = end();
-        break;
-      }
-      if (D == It->Function)
-        Best = It;
-      continue;
-    }
-    // ambiguous
-    Best = end();
-    break;
-  }
-}
-
 OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
     Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
 
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index c77ef26da568d..d87484470f8b5 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1941,7 +1941,6 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::ConceptSpecializationExprClass:
     case Stmt::CXXRewrittenBinaryOperatorClass:
     case Stmt::RequiresExprClass:
-    case Expr::CXXParenListInitExprClass:
     case Stmt::EmbedExprClass:
       // Fall through.
 
@@ -2315,11 +2314,22 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
       break;
     }
 
-    case Stmt::InitListExprClass:
+    case Stmt::InitListExprClass: {
+      const InitListExpr *E = cast<InitListExpr>(S);
       Bldr.takeNodes(Pred);
-      VisitInitListExpr(cast<InitListExpr>(S), Pred, Dst);
+      ConstructInitList(E, E->inits(), E->isTransparent(), Pred, Dst);
       Bldr.addNodes(Dst);
       break;
+    }
+
+    case Expr::CXXParenListInitExprClass: {
+      const CXXParenListInitExpr *E = cast<CXXParenListInitExpr>(S);
+      Bldr.takeNodes(Pred);
+      ConstructInitList(E, E->getInitExprs(), /*IsTransparent*/ false, Pred,
+                        Dst);
+      Bldr.addNodes(Dst);
+      break;
+    }
 
     case Stmt::MemberExprClass:
       Bldr.takeNodes(Pred);
@@ -4114,3 +4124,33 @@ void *ProgramStateTrait<ReplayWithoutInlining>::GDMIndex() {
 }
 
 void ExprEngine::anchor() { }
+
+void ExprEngine::ConstructInitList(const Expr *E, ArrayRef<Expr *> Args,
+                                   bool IsTransparent, ExplodedNode *Pred,
+                                   ExplodedNodeSet &Dst) {
+  assert((isa<InitListExpr, CXXParenListInitExpr>(E)));
+
+  const LocationContext *LC = Pred->getLocationContext();
+
+  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
+  ProgramStateRef S = Pred->getState();
+  QualType T = E->getType().getCanonicalType();
+
+  bool IsCompound = T->isArrayType() || T->isRecordType() ||
+                    T->isAnyComplexType() || T->isVectorType();
+
+  if (Args.size() > 1 || (E->isPRValue() && IsCompound && !IsTransparent)) {
+    llvm::ImmutableList<SVal> ArgList = getBasicVals().getEmptySValList();
+    for (Expr *E : llvm::reverse(Args))
+      ArgList = getBasicVals().prependSVal(S->getSVal(E, LC), ArgList);
+
+    B.generateNode(E, Pred,
+                   S->BindExpr(E, LC, svalBuilder.makeCompoundVal(T, ArgList)));
+  } else {
+    B.generateNode(E, Pred,
+                   S->BindExpr(E, LC,
+                               Args.size() == 0
+                                   ? getSValBuilder().makeZeroVal(T)
+                                   : S->getSVal(Args.front(), LC)));
+  }
+}
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index fa8e669b6bb2f..f1a25a750dd0d 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -771,54 +771,6 @@ void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
   Bldr.generateNode(B, Pred, state->BindExpr(B, Pred->getLocationContext(), X));
 }
 
-void ExprEngine::VisitInitListExpr(const InitListExpr *IE,
-                                   ExplodedNode *Pred,
-                                   ExplodedNodeSet &Dst) {
-  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
-
-  ProgramStateRef state = Pred->getState();
-  const LocationContext *LCtx = Pred->getLocationContext();
-  QualType T = getContext().getCanonicalType(IE->getType());
-  unsigned NumInitElements = IE->getNumInits();
-
-  if (!IE->isGLValue() && !IE->isTransparent() &&
-      (T->isArrayType() || T->isRecordType() || T->isVectorType() ||
-       T->isAnyComplexType())) {
-    llvm::ImmutableList<SVal> vals = getBasicVals().getEmptySValList();
-
-    // Handle base case where the initializer has no elements.
-    // e.g: static int* myArray[] = {};
-    if (NumInitElements == 0) {
-      SVal V = svalBuilder.makeCompoundVal(T, vals);
-      B.generateNode(IE, Pred, state->BindExpr(IE, LCtx, V));
-      return;
-    }
-
-    for (const Stmt *S : llvm::reverse(*IE)) {
-      SVal V = state->getSVal(cast<Expr>(S), LCtx);
-      vals = getBasicVals().prependSVal(V, vals);
-    }
-
-    B.generateNode(IE, Pred,
-                   state->BindExpr(IE, LCtx,
-                                   svalBuilder.makeCompoundVal(T, vals)));
-    return;
-  }
-
-  // Handle scalars: int{5} and int{} and GLvalues.
-  // Note, if the InitListExpr is a GLvalue, it means that there is an address
-  // representing it, so it must have a single init element.
-  assert(NumInitElements <= 1);
-
-  SVal V;
-  if (NumInitElements == 0)
-    V = getSValBuilder().makeZeroVal(T);
-  else
-    V = state->getSVal(IE->getInit(0), LCtx);
-
-  B.generateNode(IE, Pred, state->BindExpr(IE, LCtx, V));
-}
-
 void ExprEngine::VisitGuardedExpr(const Expr *Ex,
                                   const Expr *L,
                                   const Expr *R,
diff --git a/clang/test/AST/ByteCode/mutable.cpp b/clang/test/AST/ByteCode/mutable.cpp
index aebbea920578c..35c5a0389921e 100644
--- a/clang/test/AST/ByteCode/mutable.cpp
+++ b/clang/test/AST/ByteCode/mutable.cpp
@@ -1,11 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++11 -verify=expected,expected11,both,both11 %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++14 -verify=expected,expected14,both %s
-// RUN: %clang_cc1 -std=c++11 -verify=ref,ref11,both,both11 %s
-// RUN: %clang_cc1 -std=c++14 -verify=ref,ref14,both %s
-
-
-
-
+// RUN: %clang_cc1 -std=c++11 -verify=expected,expected11,both,both11 %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++14 -verify=expected,expected14,both        %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++11 -verify=ref,ref11,both,both11           %s
+// RUN: %clang_cc1 -std=c++14 -verify=ref,ref14,both                  %s
 
 namespace Simple {
   struct S {
@@ -26,3 +22,47 @@ namespace Simple {
   static_assert(s2.a2 == 12, ""); // both11-error {{not an integral constant expression}} \
                                   // both11-note {{initializer of 's2' is not a constant expression}}
 }
+#if __cplusplus >= 201402L
+namespace ConstInMutable {
+  class B {
+    public:
+
+    const int f;
+    constexpr B() : f(12) {}
+  };
+  class A {
+    public:
+    mutable B b;
+    constexpr A() = default;
+  };
+  constexpr int constInMutable() {
+    A a;
+
+    int *m = (int*)&a.b.f;
+    *m = 12; // both-note {{modification of object of const-qualified type 'const int' is not allowed in a constant expression}}
+    return 1;
+  }
+  static_assert(constInMutable() == 1, ""); // both-error {{not an integral constant expression}} \
+                                            // both-note {{in call to}}
+}
+
+namespace MutableInConst {
+  class C {
+  public:
+    mutable int c;
+    constexpr C() : c(50) {}
+  };
+  class D {
+  public:
+    C c;
+    constexpr D() {}
+  };
+  constexpr int mutableInConst() {
+    const D d{};
+    int *m = (int*)&d.c.c;
+    *m = 12;
+    return 1;
+  }
+  static_assert(mutableInConst() == 1, "");
+}
+#endif
diff --git a/clang/test/AST/ByteCode/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
index 0fa44a259a4ff..7cfd0d677a7b3 100644
--- a/clang/test/AST/ByteCode/unions.cpp
+++ b/clang/test/AST/ByteCode/unions.cpp
@@ -847,6 +847,20 @@ namespace Activation2 {
   }
   static_assert(change_member_indirectly() == 4);
 }
+
+namespace CopyCtorMutable {
+  struct E {
+    union { // expected-note {{read of mutable member 'b'}}
+      int a;
+      mutable int b; // both-note {{here}}
+    };
+  };
+  constexpr E e1 = {{1}};
+  constexpr E e2 = e1; // both-error {{constant}} \
+                       // ref-note {{read of mutable member 'b'}} \
+                       // both-note {{in call}}
+}
+
 #endif
 
 namespace AddressComparison {
diff --git a/clang/test/Analysis/div-zero-cxx20.cpp b/clang/test/Analysis/div-zero-cxx20.cpp
new file mode 100644
index 0000000000000..00ea96e796777
--- /dev/null
+++ b/clang/test/Analysis/div-zero-cxx20.cpp
@@ -0,0 +1,61 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core.DivideZero -std=c++20 -verify %s
+
+namespace GH148875 {
+struct A {
+  int x;
+  A(int v) : x(v) {}
+};
+
+struct B {
+  int x;
+  B() : x(0) {}
+};
+
+struct C {
+  int x, y;
+  C(int a, int b) : x(a), y(b) {}
+};
+
+struct D {
+  int x;
+};
+
+struct E {
+  D d;
+  E(int a) : d(a) {}
+};
+
+struct F {
+  int x;
+};
+
+int t1() {
+  A a{42};
+  return 1 / (a.x - 42); // expected-warning {{Division by zero}}
+}
+
+int t2() {
+  B b{};
+  return 1 / b.x; // expected-warning {{Division by zero}}
+}
+
+int t3() {
+  C c1{1, -1};
+  return 1 / (c1.x + c1.y); // expected-warning {{Division by zero}}
+}
+
+int t4() {
+  C c2{0, 0};
+  return 1 / (c2.x + c2.y); // expected-warning {{Division by zero}}
+}
+
+int t5() {
+  E e{32};
+  return 1 / (e.d.x - 32); // expected-warning {{Division by zero}}
+}
+
+int t6() {
+  F f(32);
+  return 1 / (f.x - 32); // expected-warning {{Division by zero}}
+}
+} // namespace GH148875
diff --git a/clang/test/Analysis/div-zero.cpp b/clang/test/Analysis/div-zero.cpp
index 063450d8883b0..51ea25e828a18 100644
--- a/clang/test/Analysis/div-zero.cpp
+++ b/clang/test/Analysis/div-zero.cpp
@@ -11,3 +11,63 @@ int fooPR10616 (int qX ) {
   return (a % (qX-1)); // expected-warning {{Division by zero}}
 
 }
+
+namespace GH148875 {
+struct A {
+  int x;
+  A(int v) : x(v) {}
+};
+
+struct B {
+  int x;
+  B() : x(0) {}
+};
+
+struct C {
+  int x, y;
+  C(int a, int b) : x(a), y(b) {}
+};
+
+struct D {
+  int x;
+};
+
+struct E {
+  D d;
+  E(int a) : d{a} {}
+};
+
+struct F {
+  int x;
+};
+
+int t1() {
+  A a{42};
+  return 1 / (a.x - 42); // expected-warning {{Division by zero}}
+}
+
+int t2() {
+  B b{};
+  return 1 / b.x; // expected-warning {{Division by zero}}
+}
+
+int t3() {
+  C c1{1, -1};
+  return 1 / (c1.x + c1.y); // expected-warning {{Division by zero}}
+}
+
+int t4() {
+  C c2{0, 0};
+  return 1 / (c2.x + c2.y); // expected-warning {{Division by zero}}
+}
+
+int t5() {
+  E e{32};
+  return 1 / (e.d.x - 32); // expected-warning {{Division by zero}}
+}
+
+int t6() {
+  F f{32};
+  return 1 / (f.x - 32); // expected-warning {{Division by zero}}
+}
+}
diff --git a/clang/test/CodeGen/64bit-swiftcall.c b/clang/test/CodeGen/64bit-swiftcall.c
index 7f8aa02d97ce1..448bca7acbca3 100644
--- a/clang/test/CodeGen/64bit-swiftcall.c
+++ b/clang/test/CodeGen/64bit-swiftcall.c
@@ -239,7 +239,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define {{.*}} void @return_struct_big_1(ptr dead_on_unwind noalias writable sret
 
 // Should not be byval.
-// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr{{( %.*)?}})
+// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr dead_on_return{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGen/AArch64/byval-temp.c b/clang/test/CodeGen/AArch64/byval-temp.c
index 0ee0312b2362d..5033b6cf5ac03 100644
--- a/clang/test/CodeGen/AArch64/byval-temp.c
+++ b/clang/test/CodeGen/AArch64/byval-temp.c
@@ -30,10 +30,10 @@ void example(void) {
 // Then, memcpy `l` to the temporary stack space.
 // CHECK-O0-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
 // Finally, call using a pointer to the temporary stack space.
-// CHECK-O0-NEXT: call void @pass_large(ptr noundef %[[byvaltemp]])
+// CHECK-O0-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
 // Now, do the same for the second call, using the second temporary alloca.
 // CHECK-O0-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
-// CHECK-O0-NEXT: call void @pass_large(ptr noundef %[[byvaltemp1]])
+// CHECK-O0-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
 // CHECK-O0-NEXT: ret void
 //
 // At O3, we should have lifetime markers to help the optimizer re-use the temporary allocas.
@@ -58,7 +58,7 @@ void example(void) {
 // Then, memcpy `l` to the temporary stack space.
 // CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
 // Finally, call using a pointer to the temporary stack space.
-// CHECK-O3-NEXT: call void @pass_large(ptr noundef %[[byvaltemp]])
+// CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
 //
 // The lifetime of the temporary used to pass a pointer to the struct ends here.
 // CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp]])
@@ -66,7 +66,7 @@ void example(void) {
 // Now, do the same for the second call, using the second temporary alloca.
 // CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %[[byvaltemp1]])
 // CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
-// CHECK-O3-NEXT: call void @pass_large(ptr noundef %[[byvaltemp1]])
+// CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
 // CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp1]])
 //
 // Mark the end of the lifetime of `l`.
@@ -88,12 +88,12 @@ void example_BitInt(void) {
 // CHECK-O0-NEXT:    [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
 // CHECK-O0-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
 // CHECK-O0-NEXT:    store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16
-// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP]])
+// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
 // CHECK-O0-NEXT:    [[TMP1:%.*]] = load i256, ptr [[L]], align 16
 // CHECK-O0-NEXT:    [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
 // CHECK-O0-NEXT:    [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
 // CHECK-O0-NEXT:    store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16
-// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP1]])
+// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
 // CHECK-O0-NEXT:    ret void
 //
 // CHECK-O3-LABEL: define dso_local void @example_BitInt(
@@ -108,13 +108,13 @@ void example_BitInt(void) {
 // CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]]) 
 // CHECK-O3-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
 // CHECK-O3-NEXT:    store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16, !tbaa [[TBAA6]]
-// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP]])
+// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]]) 
 // CHECK-O3-NEXT:    [[TMP1:%.*]] = load i256, ptr [[L]], align 16, !tbaa [[TBAA6]]
 // CHECK-O3-NEXT:    [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
 // CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]]) 
 // CHECK-O3-NEXT:    [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
 // CHECK-O3-NEXT:    store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16, !tbaa [[TBAA6]]
-// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP1]])
+// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]]) 
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[L]]) 
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c b/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
index 546910068c78a..804e14a2ea34b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
@@ -19,7 +19,7 @@ void f0(S0 *p) {
   use0(*p);
 }
 // CHECK-C:   declare void @use0(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
-// CHECK-CXX: declare void @use0(ptr noundef)
+// CHECK-CXX: declare void @use0(ptr dead_on_return noundef)
 
 #ifdef __cplusplus
 
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args.c b/clang/test/CodeGen/AArch64/pure-scalable-args.c
index fecd370d09be3..48988f7a1722b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args.c
@@ -92,7 +92,7 @@ void test_argpass_simple(PST *p) {
 // CHECK-AAPCS-NEXT: ret void
 
 // CHECK-AAPCS:  declare void @argpass_simple_callee(<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_simple_callee(ptr noundef)
+// CHECK-DARWIN: declare void @argpass_simple_callee(ptr dead_on_return noundef)
 
 // Boundary case of using the last available Z-reg, PST expanded.
 //   0.0  -> d0-d3
@@ -107,7 +107,7 @@ void test_argpass_last_z(PST *p) {
     argpass_last_z_callee(.0, .0, .0, .0, *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr dead_on_return noundef)
 
 
 // Like the above, but using a tuple type to occupy some registers.
@@ -123,7 +123,7 @@ void test_argpass_last_z_tuple(PST *p, svfloat64x4_t x) {
   argpass_last_z_tuple_callee(x, *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr dead_on_return noundef)
 
 
 // Boundary case of using the last available P-reg, PST expanded.
@@ -139,7 +139,7 @@ void test_argpass_last_p(PST *p) {
     argpass_last_p_callee(svpfalse(), svpfalse_c(), *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr dead_on_return noundef)
 
 
 // Not enough Z-regs, push PST to memory and pass a pointer, Z-regs and
@@ -157,7 +157,7 @@ void test_argpass_no_z(PST *p, double dummy, svmfloat8_t u, int8x16_t v, mfloat8
     void argpass_no_z_callee(svmfloat8_t, int8x16_t, mfloat8x16_t, double, double, int, PST, int, double, svbool_t);
     argpass_no_z_callee(u, v, w, .0, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Like the above, using a tuple to occupy some registers.
@@ -173,7 +173,7 @@ void test_argpass_no_z_tuple_f64(PST *p, float dummy, svfloat64x4_t x) {
                                      double, svbool_t);
   argpass_no_z_tuple_f64_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Likewise, using a different tuple.
@@ -189,7 +189,7 @@ void test_argpass_no_z_tuple_mfp8(PST *p, float dummy, svmfloat8x4_t x) {
                                       double, svbool_t);
   argpass_no_z_tuple_mfp8_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Not enough Z-regs (consumed by a HFA), PST passed indirectly
@@ -204,8 +204,8 @@ void test_argpass_no_z_hfa(HFA *h, PST *p) {
     void argpass_no_z_hfa_callee(double, HFA, int, PST, int, svbool_t);
     argpass_no_z_hfa_callee(.0, *h, 1, *p, 2, svptrue_b64());
 }
-// CHECK-AAPCS:  declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
 
 // Not enough Z-regs (consumed by a HVA), PST passed indirectly
 //   0.0  -> d0
@@ -219,8 +219,8 @@ void test_argpass_no_z_hva(HVA *h, PST *p) {
     void argpass_no_z_hva_callee(double, HVA, int, PST, int, svbool_t);
     argpass_no_z_hva_callee(.0, *h, 1, *p, 2, svptrue_b64());
 }
-// CHECK-AAPCS:  declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
 
 // Not enough P-regs, PST passed indirectly, Z-regs and P-regs still available.
 //   true -> p0-p2
@@ -233,7 +233,7 @@ void test_argpass_no_p(PST *p) {
     void argpass_no_p_callee(svbool_t, svbool_t, svbool_t, int, PST, int, double, svbool_t);
     argpass_no_p_callee(svptrue_b8(), svptrue_b16(), svptrue_b32(), 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Like above, using a tuple to occupy some registers.
@@ -250,7 +250,7 @@ void test_argpass_no_p_tuple(PST *p, svbool_t u, svboolx2_t v) {
                                  svbool_t);
   argpass_no_p_tuple_callee(v, u, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // HFAs go back-to-back to memory, afterwards Z-regs not available, PST passed indirectly.
@@ -263,8 +263,8 @@ void test_after_hfa(HFA *h, PST *p) {
     void after_hfa_callee(double, double, double, double, double, HFA, PST, HFA, svbool_t);
     after_hfa_callee(.0, .0, .0, .0, .0, *h, *p, *h, svpfalse());
 }
-// CHECK-AAPCS:  declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr noundef, [4 x float], <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr dead_on_return noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr dead_on_return noundef, [4 x float], <vscale x 16 x i1>)
 
 // Small PST, not enough registers, passed indirectly, unlike other small
 // aggregates.
@@ -277,7 +277,7 @@ void test_small_pst(SmallPST *p, SmallAgg *s) {
     void small_pst_callee(SmallAgg, double, double, double, double, double, double, double, double, double, SmallPST, double);
     small_pst_callee(*s, .0, .0, .0, .0, .0, .0, .0, .0, 1.0, *p, 2.0);
 }
-// CHECK-AAPCS:  declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr noundef, double noundef)
+// CHECK-AAPCS:  declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr dead_on_return noundef, double noundef)
 // CHECK-DARWIN: declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, i128, double noundef)
 
 
@@ -326,12 +326,12 @@ void test_pass_variadic(PST *p, PST *q) {
     pass_variadic_callee(*p, *q);
 }
 // CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
-// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr noundef nonnull %byval-temp)
+// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr dead_on_return noundef nonnull %byval-temp)
 
 // CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %p, i64 96, i1 false)
 // CHECK-DARWIN: call void @llvm.lifetime.start.p0(i64 96, ptr nonnull %byval-temp1)
 // CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp1, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
-// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr noundef nonnull %byval-temp, ptr noundef nonnull %byval-temp1)
+// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr dead_on_return noundef nonnull %byval-temp, ptr dead_on_return noundef nonnull %byval-temp1)
 
 
 // Test passing a small PST, still passed indirectly, despite being <= 128 bits
@@ -340,7 +340,7 @@ void test_small_pst_variadic(SmallPST *p) {
     small_pst_variadic_callee(0, *p);
 }
 // CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) %byval-temp, ptr noundef nonnull align 16 dereferenceable(16) %p, i64 16, i1 false)
-// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr noundef nonnull %byval-temp)
+// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr dead_on_return noundef nonnull %byval-temp)
 
 // CHECK-DARWIN: %0 = load i128, ptr %p, align 16
 // CHECK-DARWIN: tail call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, i128 %0)
@@ -467,7 +467,7 @@ void test_tuple_reg_count(svfloat32_t x, svfloat32x2_t y) {
                                    svfloat32_t, svfloat32_t, svfloat32_t, svfloat32x2_t);
   test_tuple_reg_count_callee(x, x, x, x, x, x, x, y);
 }
-// CHECK-AAPCS: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, ptr noundef)
+// CHECK-AAPCS: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, ptr dead_on_return noundef)
 // CHECK-DARWIN: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
 
 // Regression test for incorrect passing of SVE vector tuples
@@ -476,5 +476,5 @@ void test_tuple_reg_count_bool(svboolx4_t x, svboolx4_t y) {
   void test_tuple_reg_count_bool_callee(svboolx4_t, svboolx4_t);
   test_tuple_reg_count_bool_callee(x, y);
 }
-// CHECK-AAPCS:  declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, ptr noundef)
+// CHECK-AAPCS:  declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, ptr dead_on_return noundef)
 // CHECK-DARWIN: declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
diff --git a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
index b1232921df363..f0c9ef28201a5 100644
--- a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
+++ b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
@@ -82,7 +82,7 @@ struct Sppp {
     int *x, *y, *z;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z4Tppp4Sppp(
-// CHECK-A64-SAME: ptr noundef [[S:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr dead_on_return noundef [[S:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-A64-NEXT:    store ptr [[S]], ptr [[S_INDIRECT_ADDR]], align 8
@@ -490,7 +490,7 @@ struct Spa3 {
     int* xs[3];
 };
 // CHECK-A64-LABEL: define dso_local void @_Z4Tpa34Spa3(
-// CHECK-A64-SAME: ptr noundef [[S:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr dead_on_return noundef [[S:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-A64-NEXT:    store ptr [[S]], ptr [[S_INDIRECT_ADDR]], align 8
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
index c3d0541229fac..d244a8ba88572 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
@@ -58,7 +58,7 @@ typedef int8_t vec_int8 __attribute__((vector_size(N / 8)));
 // CHECK128-NEXT:    ret <16 x i8> [[CASTFIXEDSVE]]
 
 // CHECK-LABEL: define{{.*}} void @f2(
-// CHECK-SAME:   ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr noundef readonly captures(none) %0)
+// CHECK-SAME:   ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr dead_on_return noundef readonly captures(none) %0)
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   [[X:%.*]] = load <[[#div(VBITS,8)]] x i8>, ptr [[TMP0:%.*]], align 16, [[TBAA6:!tbaa !.*]]
 // CHECK-NEXT:   [[CASTSCALABLESVE:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v[[#div(VBITS,8)]]i8(<vscale x 16 x i8> poison, <[[#div(VBITS,8)]] x i8> [[X]], i64 0)
@@ -88,13 +88,13 @@ typedef svint8_t vec2 __attribute__((arm_sve_vector_bits(N)));
 // CHECK-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS,8)]] x i8> @llvm.vector.extract.v[[#div(VBITS,8)]]i8.nxv16i8(<vscale x 16 x i8> [[X_COERCE:%.*]], i64 0)
 // CHECK-NEXT:   call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:   store <[[#div(VBITS,8)]] x i8> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6]]
-// CHECK-NEXT:   call void @f3(ptr noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
+// CHECK-NEXT:   call void @f3(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
 // CHECK-NEXT:   call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:   ret void
 
 // CHECK128-LABEL: declare void @f3(<16 x i8> noundef)
 
 // CHECK-LABEL: declare void @f3(
-// CHECK-SAME:   ptr noundef)
+// CHECK-SAME:   ptr dead_on_return noundef)
 void g(vec2 x) { f3(x); } // OK
 #endif
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
index e82069aab2486..d42ecb663050f 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
@@ -75,7 +75,7 @@ typedef svint16_t vec2 __attribute__((arm_sve_vector_bits(N)));
 // CHECKWIDE-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS, 16)]] x i16> @llvm.vector.extract.v[[#div(VBITS, 16)]]i16.nxv8i16(<vscale x 8 x i16> [[X_COERCE:%.*]], i64 0)
 // CHECKWIDE-NEXT:   call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECKWIDE-NEXT:   store <[[#div(VBITS, 16)]] x i16> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6:!tbaa !.*]]
-// CHECKWIDE-NEXT:   call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
+// CHECKWIDE-NEXT:   call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
 // CHECKWIDE-NEXT:   call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECKWIDE-NEXT:   ret void
 void g(vec2 x) { f(x); } // OK
diff --git a/clang/test/CodeGen/LoongArch/bitint.c b/clang/test/CodeGen/LoongArch/bitint.c
index f346f569d0eb0..950f5308e32cc 100644
--- a/clang/test/CodeGen/LoongArch/bitint.c
+++ b/clang/test/CodeGen/LoongArch/bitint.c
@@ -26,12 +26,12 @@ void pass_BitInt129(_BitInt(129));
 // LA32-NEXT:    [[LOADEDV1:%.*]] = trunc i128 [[TMP1]] to i65
 // LA32-NEXT:    [[STOREDV:%.*]] = sext i65 [[LOADEDV1]] to i128
 // LA32-NEXT:    store i128 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
-// LA32-NEXT:    call void @pass_BitInt65(ptr noundef [[BYVAL_TEMP]])
+// LA32-NEXT:    call void @pass_BitInt65(ptr dead_on_return noundef [[BYVAL_TEMP]])
 // LA32-NEXT:    [[TMP2:%.*]] = load i256, ptr [[L129]], align 16
 // LA32-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
 // LA32-NEXT:    [[STOREDV4:%.*]] = sext i129 [[LOADEDV2]] to i256
 // LA32-NEXT:    store i256 [[STOREDV4]], ptr [[BYVAL_TEMP3]], align 16
-// LA32-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP3]])
+// LA32-NEXT:    call void @pass_BitInt129(ptr dead_on_return noundef [[BYVAL_TEMP3]])
 // LA32-NEXT:    ret void
 //
 // LA64-LABEL: define dso_local void @example_BitInt(
@@ -54,7 +54,7 @@ void pass_BitInt129(_BitInt(129));
 // LA64-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
 // LA64-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV2]] to i256
 // LA64-NEXT:    store i256 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
-// LA64-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP]])
+// LA64-NEXT:    call void @pass_BitInt129(ptr dead_on_return noundef [[BYVAL_TEMP]])
 // LA64-NEXT:    ret void
 //
 void example_BitInt(void) {
diff --git a/clang/test/CodeGen/PowerPC/ppc64-vector.c b/clang/test/CodeGen/PowerPC/ppc64-vector.c
index 5d3dd86a009d5..2e99781f84910 100644
--- a/clang/test/CodeGen/PowerPC/ppc64-vector.c
+++ b/clang/test/CodeGen/PowerPC/ppc64-vector.c
@@ -39,7 +39,7 @@ v8i16 test_v8i16(v8i16 x)
   return x;
 }
 
-// CHECK: define{{.*}} void @test_v16i16(ptr dead_on_unwind noalias writable sret(<16 x i16>) align 32 %agg.result, ptr noundef %0)
+// CHECK: define{{.*}} void @test_v16i16(ptr dead_on_unwind noalias writable sret(<16 x i16>) align 32 %agg.result, ptr dead_on_return noundef %0)
 v16i16 test_v16i16(v16i16 x)
 {
   return x;
diff --git a/clang/test/CodeGen/RISCV/riscv-abi.cpp b/clang/test/CodeGen/RISCV/riscv-abi.cpp
index fe1a2b6d8595c..d2e080829e72f 100644
--- a/clang/test/CodeGen/RISCV/riscv-abi.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-abi.cpp
@@ -75,7 +75,7 @@ struct child3_int64_s : parent3_float_s {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @_Z30float_int64_struct_inheritance14child3_int64_s
-// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD3_INT64_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD3_INT64_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 // LP64-LABEL: define dso_local [2 x i64] @_Z30float_int64_struct_inheritance14child3_int64_s
@@ -99,7 +99,7 @@ struct child4_double_s : parent4_double_s {
 };
 
 // ILP32-ILP32F-LABEL: define dso_local void @_Z32double_double_struct_inheritance15child4_double_s
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD4_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD4_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @_Z32double_double_struct_inheritance15child4_double_s
@@ -130,11 +130,11 @@ struct child5_virtual_s : virtual parent5_virtual_s {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @_Z38int32_float_virtual_struct_inheritance16child5_virtual_s
-// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 // LP64-LP64F-LP64D-LABEL: define dso_local void @_Z38int32_float_virtual_struct_inheritance16child5_virtual_s
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct child5_virtual_s int32_float_virtual_struct_inheritance(struct child5_virtual_s a) {
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
index 82e43fff0c3aa..bc89cb532bdcc 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
@@ -138,7 +138,7 @@ struct st_i32x4x9 {
 
 typedef int __attribute__((vector_size(256))) int32x64_t;
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_too_large(ptr noundef %0)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_too_large(ptr dead_on_return noundef %0)
 void __attribute__((riscv_vls_cc)) test_too_large(int32x64_t arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_too_large_256(<vscale x 16 x i32> noundef %arg.coerce)
 void __attribute__((riscv_vls_cc(256))) test_too_large_256(int32x64_t arg) {}
@@ -173,9 +173,9 @@ void __attribute__((riscv_vls_cc)) test_st_i32x8x2(struct st_i32x8x2 arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x8x2_256(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc)) test_st_i32x64x2(struct st_i32x64x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
@@ -188,7 +188,7 @@ void __attribute__((riscv_vls_cc)) test_st_i32x4x8(struct st_i32x4x8 arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x8_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc)) test_st_i32x4x9(struct st_i32x4x9 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
index 5f6539796c20d..128610e578c26 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
@@ -118,7 +118,7 @@ struct st_i32x4x9 {
 
 typedef int __attribute__((vector_size(256))) int32x64_t;
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z14test_too_largeDv64_i(ptr noundef %0)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z14test_too_largeDv64_i(ptr dead_on_return noundef %0)
 [[riscv::vls_cc]] void test_too_large(int32x64_t arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z18test_too_large_256Dv64_i(<vscale x 16 x i32> noundef %arg.coerce)
 [[riscv::vls_cc(256)]] void test_too_large_256(int32x64_t arg) {}
@@ -153,9 +153,9 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x8x2_25610st_i32x8x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc]] void test_st_i32x64x2(struct st_i32x64x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x310st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
@@ -168,7 +168,7 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x8_25610st_i32x4x8(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc]] void test_st_i32x4x9(struct st_i32x4x9 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x9_25610st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x9_25610st_i32x4x9(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
diff --git a/clang/test/CodeGen/RISCV/riscv32-abi.c b/clang/test/CodeGen/RISCV/riscv32-abi.c
index b53f9a9169146..a9e56d40817ae 100644
--- a/clang/test/CodeGen/RISCV/riscv32-abi.c
+++ b/clang/test/CodeGen/RISCV/riscv32-abi.c
@@ -246,7 +246,7 @@ struct large {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_agg_large
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[X:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[X:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_agg_large(struct large x) {
@@ -266,7 +266,7 @@ struct large f_agg_large_ret(int32_t i, int8_t j) {
 typedef unsigned char v16i8 __attribute__((vector_size(16)));
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_vec_large_v16i8
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[TMP0:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[TMP0:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_vec_large_v16i8(v16i8 x) {
@@ -285,7 +285,7 @@ v16i8 f_vec_large_v16i8_ret(void) {
 // if they were passed in registers.
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local i32 @f_scalar_stack_1
-// ILP32-ILP32F-ILP32D-SAME: (i32 [[A_COERCE:%.*]], [2 x i32] [[B_COERCE:%.*]], i64 [[C_COERCE:%.*]], ptr noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (i32 [[A_COERCE:%.*]], [2 x i32] [[B_COERCE:%.*]], i64 [[C_COERCE:%.*]], ptr dead_on_return noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
@@ -343,7 +343,7 @@ struct large f_scalar_stack_6(float a, int64_t b, double c, long double d,
 // they would be if passed via registers.
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_agg_stack
-// ILP32-ILP32F-ILP32D-SAME: (double noundef [[A:%.*]], i64 noundef [[B:%.*]], double noundef [[C:%.*]], i64 noundef [[D:%.*]], i32 [[E_COERCE:%.*]], [2 x i32] [[F_COERCE:%.*]], i64 [[G_COERCE:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (double noundef [[A:%.*]], i64 noundef [[B:%.*]], double noundef [[C:%.*]], i64 noundef [[D:%.*]], i32 [[E_COERCE:%.*]], [2 x i32] [[F_COERCE:%.*]], i64 [[G_COERCE:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_agg_stack(double a, int64_t b, double c, int64_t d, struct tiny e,
@@ -366,7 +366,7 @@ struct double_int8_s { double d; int64_t i; };
 struct int_double_s { int a; double b; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_int_double_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_int_double_s_arg
@@ -482,7 +482,7 @@ struct zbf_double_zbf_s f_ret_zbf_double_zbf_s(void) {
 struct double_float_s { double f; float g; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_double_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_double_s_arg
@@ -504,7 +504,7 @@ struct double_double_s f_ret_double_double_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_float_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_float_s_arg
@@ -526,7 +526,7 @@ struct double_float_s f_ret_double_float_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_double_s_arg_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_double_s_arg_insufficient_fprs(float a, double b, double c, double d,
@@ -543,7 +543,7 @@ struct double_int64bf_s { double d; int64_t i : 32; };
 struct double_int8_zbf_s { double d; int8_t i; int : 0; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int8_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int8_s_arg(struct double_int8_s a) {}
@@ -557,7 +557,7 @@ struct double_int8_s f_ret_double_int8_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_uint8_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_uint8_s_arg
@@ -579,7 +579,7 @@ struct double_uint8_s f_ret_double_uint8_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int32_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int32_s_arg
@@ -601,7 +601,7 @@ struct double_int32_s f_ret_double_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int64_s_arg(struct double_int64_s a) {}
@@ -615,7 +615,7 @@ struct double_int64_s f_ret_double_int64_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int64bf_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int64bf_s_arg
@@ -640,7 +640,7 @@ struct double_int64bf_s f_ret_double_int64bf_s(void) {
 // floating point calling convention.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int8_zbf_s
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int8_zbf_s
@@ -662,14 +662,14 @@ struct double_int8_zbf_s f_ret_double_int8_zbf_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int8_s_arg_insufficient_gprs
-// ILP32-ILP32F-ILP32D-SAME: (i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], i32 noundef [[H:%.*]], ptr noundef [[I:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], i32 noundef [[H:%.*]], ptr dead_on_return noundef [[I:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int8_s_arg_insufficient_gprs(int a, int b, int c, int d, int e,
                                           int f, int g, int h, struct double_int8_s i) {}
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_struct_double_int8_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], double noundef [[H:%.*]], ptr noundef [[I:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], double noundef [[H:%.*]], ptr dead_on_return noundef [[I:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_struct_double_int8_insufficient_fprs(float a, double b, double c, double d,
@@ -679,7 +679,7 @@ void f_struct_double_int8_insufficient_fprs(float a, double b, double c, double
 // floating-point value should be passed as if it were an fp+fp struct.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublecomplex
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublecomplex
@@ -703,7 +703,7 @@ double __complex__ f_ret_doublecomplex(void) {
 struct doublecomplex_s { double __complex__ c; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublecomplex_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublecomplex_s_arg
@@ -754,7 +754,7 @@ struct doublearr1_s f_ret_doublearr1_s(void) {
 struct doublearr2_s { double a[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_s_arg
@@ -778,7 +778,7 @@ struct doublearr2_s f_ret_doublearr2_s(void) {
 struct doublearr2_tricky1_s { struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky1_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky1_s_arg
@@ -802,7 +802,7 @@ struct doublearr2_tricky1_s f_ret_doublearr2_tricky1_s(void) {
 struct doublearr2_tricky2_s { struct {}; struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky2_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky2_s_arg
@@ -826,7 +826,7 @@ struct doublearr2_tricky2_s f_ret_doublearr2_tricky2_s(void) {
 struct doublearr2_tricky3_s { union {}; struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky3_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky3_s_arg
@@ -850,7 +850,7 @@ struct doublearr2_tricky3_s f_ret_doublearr2_tricky3_s(void) {
 struct doublearr2_tricky4_s { union {}; struct { struct {}; double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky4_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky4_s_arg
@@ -877,7 +877,7 @@ struct doublearr2_tricky4_s f_ret_doublearr2_tricky4_s(void) {
 struct int_double_int_s { int a; double b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_double_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_double_int_s_arg(struct int_double_int_s a) {}
@@ -893,7 +893,7 @@ struct int_double_int_s f_ret_int_double_int_s(void) {
 struct int64_double_s { int64_t a; double b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_double_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_double_s_arg(struct int64_double_s a) {}
@@ -909,7 +909,7 @@ struct int64_double_s f_ret_int64_double_s(void) {
 struct char_char_double_s { char a; char b; double c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_char_char_double_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_char_char_double_s_arg(struct char_char_double_s a) {}
@@ -948,7 +948,7 @@ union double_u f_ret_double_u(void) {
 // double+double structs by the ABI.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_double_int32_s_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_INT32_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_INT32_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, i32 } @f_ret_double_int32_s_double_int32_s_just_sufficient_gprs
@@ -961,7 +961,7 @@ struct double_int32_s f_ret_double_int32_s_double_int32_s_just_sufficient_gprs(
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_double_double_s_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @f_ret_double_double_s_double_int32_s_just_sufficient_gprs
@@ -974,7 +974,7 @@ struct double_double_s f_ret_double_double_s_double_int32_s_just_sufficient_gprs
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_doublecomplex_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @f_ret_doublecomplex_double_int32_s_just_sufficient_gprs
@@ -1189,7 +1189,7 @@ struct float_int32_s f_ret_float_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_float_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_float_int64_s_arg(struct float_int64_s a) {}
@@ -1465,7 +1465,7 @@ struct floatarr2_tricky4_s f_ret_floatarr2_tricky4_s(void) {
 struct int_float_int_s { int a; float b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_float_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_float_int_s_arg(struct int_float_int_s a) {}
@@ -1481,7 +1481,7 @@ struct int_float_int_s f_ret_int_float_int_s(void) {
 struct int64_float_s { int64_t a; float b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_float_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_float_s_arg(struct int64_float_s a) {}
@@ -1619,7 +1619,7 @@ struct zbf_float16_zbf_s f_ret_zbf_float16_zbf_s(void) {
 struct double_float16_s { double f; _Float16 g; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_float16_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_float16_s_arg
@@ -1641,7 +1641,7 @@ struct double_float16_s f_ret_double_float16_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_float16_s_arg_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_float16_s_arg_insufficient_fprs(float a, double b, double c, double d,
@@ -1725,7 +1725,7 @@ struct float16_int32_s f_ret_float16_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_float16_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_float16_int64_s_arg(struct float16_int64_s a) {}
@@ -2001,7 +2001,7 @@ struct float16arr2_tricky4_s f_ret_float16arr2_tricky4_s(void) {
 struct int_float16_int_s { int a; _Float16 b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_float16_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_float16_int_s_arg(struct int_float16_int_s a) {}
@@ -2017,7 +2017,7 @@ struct int_float16_int_s f_ret_int_float16_int_s(void) {
 struct int64_float16_s { int64_t a; _Float16 b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_float16_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_float16_s_arg(struct int64_float16_s a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv32-vararg.c b/clang/test/CodeGen/RISCV/riscv32-vararg.c
index 2b332410f8637..ed301f9269bb8 100644
--- a/clang/test/CodeGen/RISCV/riscv32-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv32-vararg.c
@@ -64,7 +64,7 @@ int f_va_callee(int, ...);
 // CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL4]], i32 0, i32 0
 // CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[COERCE_DIVE]], align 8
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[BYVAL_TEMP]], ptr align 4 [[DOTCOMPOUNDLITERAL6]], i32 16, i1 false)
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 (i32, ...) @f_va_callee(i32 noundef 1, i32 noundef 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i32 [[TMP0]], [2 x i32] [[TMP1]], i64 [[TMP2]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 (i32, ...) @f_va_callee(i32 noundef 1, i32 noundef 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i32 [[TMP0]], [2 x i32] [[TMP1]], i64 [[TMP2]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    ret void
 //
 void f_va_caller(void) {
diff --git a/clang/test/CodeGen/RISCV/riscv64-abi.c b/clang/test/CodeGen/RISCV/riscv64-abi.c
index 021565238904e..dc01750e56970 100644
--- a/clang/test/CodeGen/RISCV/riscv64-abi.c
+++ b/clang/test/CodeGen/RISCV/riscv64-abi.c
@@ -242,7 +242,7 @@ struct large {
 };
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_agg_large
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[X:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[X:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_agg_large(struct large x) {
@@ -262,7 +262,7 @@ struct large f_agg_large_ret(int32_t i, int8_t j) {
 typedef unsigned char v32i8 __attribute__((vector_size(32)));
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_vec_large_v32i8
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[TMP0:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[TMP0:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_vec_large_v32i8(v32i8 x) {
@@ -281,7 +281,7 @@ v32i8 f_vec_large_v32i8_ret(void) {
 // if they were passed in registers.
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_1
-// LP64-LP64F-LP64D-SAME: (i64 [[A_COERCE:%.*]], [2 x i64] [[B_COERCE:%.*]], i128 [[C_COERCE:%.*]], ptr noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i64 [[A_COERCE:%.*]], [2 x i64] [[B_COERCE:%.*]], i128 [[C_COERCE:%.*]], ptr dead_on_return noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
@@ -290,7 +290,7 @@ int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_2
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], i64 noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], i64 noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_2(int32_t a, __int128_t b, int64_t c, long double d, v32i8 e,
@@ -299,7 +299,7 @@ int f_scalar_stack_2(int32_t a, __int128_t b, int64_t c, long double d, v32i8 e,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_3
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], double noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], double noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_3(int32_t a, __int128_t b, double c, long double d, v32i8 e,
@@ -312,7 +312,7 @@ int f_scalar_stack_3(int32_t a, __int128_t b, double c, long double d, v32i8 e,
 // to pass a pointer.
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_scalar_stack_4
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct large f_scalar_stack_4(uint32_t a, __int128_t b, long double c, v32i8 d,
@@ -321,7 +321,7 @@ struct large f_scalar_stack_4(uint32_t a, __int128_t b, long double c, v32i8 d,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_scalar_stack_5
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], double noundef [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], double noundef [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct large f_scalar_stack_5(double a, __int128_t b, long double c, v32i8 d,
@@ -330,7 +330,7 @@ struct large f_scalar_stack_5(double a, __int128_t b, long double c, v32i8 d,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_6
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], float noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]], half noundef [[I:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], float noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]], half noundef [[I:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_6(int32_t a, __int128_t b, float c, long double d, v32i8 e,
@@ -1440,7 +1440,7 @@ struct doublearr2_tricky4_s f_ret_doublearr2_tricky4_s(void) {
 struct int_double_int_s { int a; double b; int c; };
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_int_double_int_s_arg
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_int_double_int_s_arg(struct int_double_int_s a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv64-vararg.c b/clang/test/CodeGen/RISCV/riscv64-vararg.c
index a278f74ca4a86..17802553c795a 100644
--- a/clang/test/CodeGen/RISCV/riscv64-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv64-vararg.c
@@ -74,7 +74,7 @@ int f_va_callee(int, ...);
 // CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL4]], i32 0, i32 0
 // CHECK-NEXT:    [[TMP2:%.*]] = load i128, ptr [[COERCE_DIVE]], align 16
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[BYVAL_TEMP]], ptr align 8 [[DOTCOMPOUNDLITERAL6]], i64 32, i1 false)
-// CHECK-NEXT:    [[CALL:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i64 [[TMP0]], [2 x i64] [[TMP1]], i128 [[TMP2]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    [[CALL:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i64 [[TMP0]], [2 x i64] [[TMP1]], i128 [[TMP2]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    [[CALL11:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i32 noundef signext 3, i32 noundef signext 4, fp128 noundef 0xL00000000000000004001400000000000, i32 noundef signext 6, i32 noundef signext 7, i32 noundef signext 8, i32 noundef signext 9)
 // CHECK-NEXT:    [[A13:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL12]], i32 0, i32 0
 // CHECK-NEXT:    store i128 5, ptr [[A13]], align 16
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi-vector.c b/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
index e5704709a3a33..fab6050a0d876 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
@@ -58,91 +58,91 @@ unsigned int align = __alignof__ (v16i8);
 // CHECK-VECTOR: @align ={{.*}} global i32 8
 
 v1i8 pass_v1i8(v1i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i8(ptr dead_on_unwind noalias writable sret(<1 x i8>) align 1 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i8(ptr dead_on_unwind noalias writable sret(<1 x i8>) align 1 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i8> @pass_v1i8(<1 x i8> %{{.*}})
 
 v2i8 pass_v2i8(v2i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i8(ptr dead_on_unwind noalias writable sret(<2 x i8>) align 2 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i8(ptr dead_on_unwind noalias writable sret(<2 x i8>) align 2 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i8> @pass_v2i8(<2 x i8> %{{.*}})
 
 v4i8 pass_v4i8(v4i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i8(ptr dead_on_unwind noalias writable sret(<4 x i8>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i8(ptr dead_on_unwind noalias writable sret(<4 x i8>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i8> @pass_v4i8(<4 x i8> %{{.*}})
 
 v8i8 pass_v8i8(v8i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v8i8(ptr dead_on_unwind noalias writable sret(<8 x i8>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v8i8(ptr dead_on_unwind noalias writable sret(<8 x i8>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <8 x i8> @pass_v8i8(<8 x i8> %{{.*}})
 
 v16i8 pass_v16i8(v16i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v16i8(ptr dead_on_unwind noalias writable sret(<16 x i8>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v16i8(ptr dead_on_unwind noalias writable sret(<16 x i8>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <16 x i8> @pass_v16i8(<16 x i8> %{{.*}})
 
 v32i8 pass_v32i8(v32i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 32 %{{.*}}, ptr %0)
-// CHECK-VECTOR-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 32 %{{.*}}, ptr dead_on_return %0)
+// CHECK-VECTOR-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 8 %{{.*}}, ptr dead_on_return %0)
 
 v1i16 pass_v1i16(v1i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i16(ptr dead_on_unwind noalias writable sret(<1 x i16>) align 2 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i16(ptr dead_on_unwind noalias writable sret(<1 x i16>) align 2 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i16> @pass_v1i16(<1 x i16> %{{.*}})
 
 v2i16 pass_v2i16(v2i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i16(ptr dead_on_unwind noalias writable sret(<2 x i16>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i16(ptr dead_on_unwind noalias writable sret(<2 x i16>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i16> @pass_v2i16(<2 x i16> %{{.*}})
 
 v4i16 pass_v4i16(v4i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i16(ptr dead_on_unwind noalias writable sret(<4 x i16>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i16(ptr dead_on_unwind noalias writable sret(<4 x i16>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i16> @pass_v4i16(<4 x i16> %{{.*}})
 
 v8i16 pass_v8i16(v8i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v8i16(ptr dead_on_unwind noalias writable sret(<8 x i16>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v8i16(ptr dead_on_unwind noalias writable sret(<8 x i16>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <8 x i16> @pass_v8i16(<8 x i16> %{{.*}})
 
 v1i32 pass_v1i32(v1i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i32(ptr dead_on_unwind noalias writable sret(<1 x i32>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i32(ptr dead_on_unwind noalias writable sret(<1 x i32>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i32> @pass_v1i32(<1 x i32> %{{.*}})
 
 v2i32 pass_v2i32(v2i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i32(ptr dead_on_unwind noalias writable sret(<2 x i32>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i32(ptr dead_on_unwind noalias writable sret(<2 x i32>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i32> @pass_v2i32(<2 x i32> %{{.*}})
 
 v4i32 pass_v4i32(v4i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i32(ptr dead_on_unwind noalias writable sret(<4 x i32>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i32(ptr dead_on_unwind noalias writable sret(<4 x i32>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i32> @pass_v4i32(<4 x i32> %{{.*}})
 
 v1i64 pass_v1i64(v1i64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i64(ptr dead_on_unwind noalias writable sret(<1 x i64>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i64(ptr dead_on_unwind noalias writable sret(<1 x i64>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i64> @pass_v1i64(<1 x i64> %{{.*}})
 
 v2i64 pass_v2i64(v2i64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i64(ptr dead_on_unwind noalias writable sret(<2 x i64>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i64(ptr dead_on_unwind noalias writable sret(<2 x i64>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i64> @pass_v2i64(<2 x i64> %{{.*}})
 
 v1i128 pass_v1i128(v1i128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i128(ptr dead_on_unwind noalias writable sret(<1 x i128>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i128(ptr dead_on_unwind noalias writable sret(<1 x i128>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i128> @pass_v1i128(<1 x i128> %{{.*}})
 
 v1f32 pass_v1f32(v1f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f32(ptr dead_on_unwind noalias writable sret(<1 x float>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f32(ptr dead_on_unwind noalias writable sret(<1 x float>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x float> @pass_v1f32(<1 x float> %{{.*}})
 
 v2f32 pass_v2f32(v2f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2f32(ptr dead_on_unwind noalias writable sret(<2 x float>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2f32(ptr dead_on_unwind noalias writable sret(<2 x float>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x float> @pass_v2f32(<2 x float> %{{.*}})
 
 v4f32 pass_v4f32(v4f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4f32(ptr dead_on_unwind noalias writable sret(<4 x float>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4f32(ptr dead_on_unwind noalias writable sret(<4 x float>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x float> @pass_v4f32(<4 x float> %{{.*}})
 
 v1f64 pass_v1f64(v1f64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f64(ptr dead_on_unwind noalias writable sret(<1 x double>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f64(ptr dead_on_unwind noalias writable sret(<1 x double>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x double> @pass_v1f64(<1 x double> %{{.*}})
 
 v2f64 pass_v2f64(v2f64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2f64(ptr dead_on_unwind noalias writable sret(<2 x double>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2f64(ptr dead_on_unwind noalias writable sret(<2 x double>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x double> @pass_v2f64(<2 x double> %{{.*}})
 
 v1f128 pass_v1f128(v1f128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f128(ptr dead_on_unwind noalias writable sret(<1 x fp128>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f128(ptr dead_on_unwind noalias writable sret(<1 x fp128>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x fp128> @pass_v1f128(<1 x fp128> %{{.*}})
 
 
@@ -170,13 +170,13 @@ struct agg_v8i8 pass_agg_v8i8(struct agg_v8i8 arg) { return arg; }
 
 struct agg_v16i8 { v16i8 a; };
 struct agg_v16i8 pass_agg_v16i8(struct agg_v16i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 // CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 8 %{{.*}}, <16 x i8> %{{.*}})
 
 struct agg_v32i8 { v32i8 a; };
 struct agg_v32i8 pass_agg_v32i8(struct agg_v32i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 32 %{{.*}}, ptr %{{.*}})
-// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 8 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 32 %{{.*}}, ptr dead_on_return %{{.*}})
+// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 8 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Verify that the following are *not* vector-like aggregate types
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi.c b/clang/test/CodeGen/SystemZ/systemz-abi.c
index f26084ab44eae..83137ae6d5f82 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi.c
@@ -50,7 +50,7 @@ long long pass_longlong(long long arg) { return arg; }
 // CHECK-LABEL: define{{.*}} i64 @pass_longlong(i64 %{{.*}})
 
 __int128 pass_int128(__int128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr dead_on_return %0)
 
 _Float16 pass__Float16(_Float16 arg) { return arg; }
 // CHECK-LABEL: define{{.*}} half @pass__Float16(half %{{.*}})
@@ -62,37 +62,37 @@ double pass_double(double arg) { return arg; }
 // CHECK-LABEL: define{{.*}} double @pass_double(double %{{.*}})
 
 long double pass_longdouble(long double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_longdouble(ptr dead_on_unwind noalias writable sret(fp128) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_longdouble(ptr dead_on_unwind noalias writable sret(fp128) align 8 %{{.*}}, ptr dead_on_return %0)
 
 
 // Complex types
 
 _Complex char pass_complex_char(_Complex char arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_char(ptr dead_on_unwind noalias writable sret({ i8, i8 }) align 1 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_char(ptr dead_on_unwind noalias writable sret({ i8, i8 }) align 1 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex short pass_complex_short(_Complex short arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_short(ptr dead_on_unwind noalias writable sret({ i16, i16 }) align 2 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_short(ptr dead_on_unwind noalias writable sret({ i16, i16 }) align 2 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex int pass_complex_int(_Complex int arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_int(ptr dead_on_unwind noalias writable sret({ i32, i32 }) align 4 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_int(ptr dead_on_unwind noalias writable sret({ i32, i32 }) align 4 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long pass_complex_long(_Complex long arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_long(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_long(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long long pass_complex_longlong(_Complex long long arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex _Float16 pass_complex__Float16(_Complex _Float16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex float pass_complex_float(_Complex float arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex double pass_complex_double(_Complex double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_double(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_double(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long double pass_complex_longdouble(_Complex long double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_longdouble(ptr dead_on_unwind noalias writable sret({ fp128, fp128 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_longdouble(ptr dead_on_unwind noalias writable sret({ fp128, fp128 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 
 // Aggregate types
@@ -107,7 +107,7 @@ struct agg_2byte pass_agg_2byte(struct agg_2byte arg) { return arg; }
 
 struct agg_3byte { char a[3]; };
 struct agg_3byte pass_agg_3byte(struct agg_3byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_3byte(ptr dead_on_unwind noalias writable sret(%struct.agg_3byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_3byte(ptr dead_on_unwind noalias writable sret(%struct.agg_3byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_4byte { char a[4]; };
 struct agg_4byte pass_agg_4byte(struct agg_4byte arg) { return arg; }
@@ -115,15 +115,15 @@ struct agg_4byte pass_agg_4byte(struct agg_4byte arg) { return arg; }
 
 struct agg_5byte { char a[5]; };
 struct agg_5byte pass_agg_5byte(struct agg_5byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_5byte(ptr dead_on_unwind noalias writable sret(%struct.agg_5byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_5byte(ptr dead_on_unwind noalias writable sret(%struct.agg_5byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_6byte { char a[6]; };
 struct agg_6byte pass_agg_6byte(struct agg_6byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_6byte(ptr dead_on_unwind noalias writable sret(%struct.agg_6byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_6byte(ptr dead_on_unwind noalias writable sret(%struct.agg_6byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_7byte { char a[7]; };
 struct agg_7byte pass_agg_7byte(struct agg_7byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_7byte(ptr dead_on_unwind noalias writable sret(%struct.agg_7byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_7byte(ptr dead_on_unwind noalias writable sret(%struct.agg_7byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_8byte { char a[8]; };
 struct agg_8byte pass_agg_8byte(struct agg_8byte arg) { return arg; }
@@ -131,7 +131,7 @@ struct agg_8byte pass_agg_8byte(struct agg_8byte arg) { return arg; }
 
 struct agg_16byte { char a[16]; };
 struct agg_16byte pass_agg_16byte(struct agg_16byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_16byte(ptr dead_on_unwind noalias writable sret(%struct.agg_16byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_16byte(ptr dead_on_unwind noalias writable sret(%struct.agg_16byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Float-like aggregate types
@@ -153,7 +153,7 @@ struct agg_double pass_agg_double(struct agg_double arg) { return arg; }
 
 struct agg_longdouble { long double a; };
 struct agg_longdouble pass_agg_longdouble(struct agg_longdouble arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg__Float16_a4 { _Float16 a __attribute__((aligned (4))); };
 struct agg__Float16_a4 pass_agg__Float16_a4(struct agg__Float16_a4 arg) { return arg; }
@@ -167,7 +167,7 @@ struct agg__Float16_a8 pass_agg__Float16_a8(struct agg__Float16_a8 arg) { return
 
 struct agg__Float16_a16 { _Float16 a __attribute__((aligned (16))); };
 struct agg__Float16_a16 pass_agg__Float16_a16(struct agg__Float16_a16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_float_a8 { float a __attribute__((aligned (8))); };
 struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
@@ -176,7 +176,7 @@ struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
 
 struct agg_float_a16 { float a __attribute__((aligned (16))); };
 struct agg_float_a16 pass_agg_float_a16(struct agg_float_a16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_float_a16(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a16) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_float_a16(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a16) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Verify that the following are *not* float-like aggregate types
diff --git a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
index 434937a66389c..d76fb4bd1fda6 100644
--- a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
+++ b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
@@ -131,7 +131,7 @@ double test_f64(double f, double g) {
 long double test_f128(long double f, long double g) {
   asm("axbr %0, %2" : "=f" (f) : "0" (f), "f" (g));
   return f;
-// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr noundef readonly captures(none) %0, ptr noundef readonly captures(none) %1)
+// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr dead_on_return noundef readonly captures(none) %0, ptr dead_on_return noundef readonly captures(none) %1)
 // CHECK: %f = load fp128, ptr %0
 // CHECK: %g = load fp128, ptr %1
 // CHECK: [[RESULT:%.*]] = tail call fp128 asm "axbr $0, $2", "=f,0,f"(fp128 %f, fp128 %g)
diff --git a/clang/test/CodeGen/X86/cx-complex-range.c b/clang/test/CodeGen/X86/cx-complex-range.c
index f87091427df71..b16b10b7b8a21 100644
--- a/clang/test/CodeGen/X86/cx-complex-range.c
+++ b/clang/test/CodeGen/X86/cx-complex-range.c
@@ -1064,7 +1064,7 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // PRMTD-NEXT:    ret <2 x half> [[TMP33]]
 //
 // X86WINPRMTD-LABEL: define dso_local i32 @f1(
-// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[RETVAL:%.*]] = alloca { half, half }, align 2
 // X86WINPRMTD-NEXT:    [[A:%.*]] = alloca { half, half }, align 2
diff --git a/clang/test/CodeGen/X86/x86_32-arguments-win32.c b/clang/test/CodeGen/X86/x86_32-arguments-win32.c
index 5b81c43f4bbb8..53d040af725b0 100644
--- a/clang/test/CodeGen/X86/x86_32-arguments-win32.c
+++ b/clang/test/CodeGen/X86/x86_32-arguments-win32.c
@@ -72,10 +72,10 @@ void receive_vec_512(__m512 x, __m512 y, __m512 z, __m512 w, __m512 q) {
 void receive_vec_1024(__m1024 x, __m1024 y, __m1024 z, __m1024 w, __m1024 q) {
   gv1024 = x + y + z + w + q;
 }
-// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg noundef %x, <8 x float> inreg noundef %y, <8 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg noundef %x, <16 x float> inreg noundef %y, <16 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_1024(ptr noundef %0, ptr noundef %1, ptr noundef %2, ptr noundef %3, ptr noundef %4)
+// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg noundef %x, <8 x float> inreg noundef %y, <8 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg noundef %x, <16 x float> inreg noundef %y, <16 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_1024(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1, ptr dead_on_return noundef %2, ptr dead_on_return noundef %3, ptr dead_on_return noundef %4)
 
 void pass_vec_128(void) {
   __m128 z = {0};
@@ -83,13 +83,13 @@ void pass_vec_128(void) {
 }
 
 // CHECK-LABEL: define dso_local void @pass_vec_128()
-// CHECK: call void @receive_vec_128(<4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, ptr noundef %{{[^,)]*}}, ptr noundef %{{[^,)]*}})
+// CHECK: call void @receive_vec_128(<4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, ptr dead_on_return noundef %{{[^,)]*}}, ptr dead_on_return noundef %{{[^,)]*}})
 
 
 void __fastcall fastcall_indirect_vec(__m128 x, __m128 y, __m128 z, __m128 w, int edx, __m128 q) {
   gv128 = x + y + z + w + q;
 }
-// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01 at fastcall_indirect_vec@84"(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr inreg noundef %0, i32 inreg noundef %edx, ptr noundef %1)
+// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01 at fastcall_indirect_vec@84"(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr dead_on_return inreg noundef %0, i32 inreg noundef %edx, ptr dead_on_return noundef %1)
 
 struct __declspec(align(1)) Align1 { unsigned long long x; };
 struct __declspec(align(4)) Align4 { unsigned long long x; };
@@ -156,4 +156,4 @@ void pass_fixed_align_variadic() {
 // correctly in Clang than it is to be bug for bug compatible, so we pass such
 // arguments indirectly.
 // CHECK-LABEL: define dso_local void @pass_fixed_align_variadic()
-// CHECK: call void (ptr, ...) @receive_fixed_align_variadic(ptr noundef %{{[^)]*}}, i32 noundef 42)
+// CHECK: call void (ptr, ...) @receive_fixed_align_variadic(ptr dead_on_return noundef %{{[^)]*}}, i32 noundef 42)
diff --git a/clang/test/CodeGen/X86/x86_64-arguments-win32.c b/clang/test/CodeGen/X86/x86_64-arguments-win32.c
index 8768e73a854aa..6010e531acb00 100644
--- a/clang/test/CodeGen/X86/x86_64-arguments-win32.c
+++ b/clang/test/CodeGen/X86/x86_64-arguments-win32.c
@@ -21,7 +21,7 @@ void f4(unsigned short a) {}
 // CHECK-LABEL: define dso_local void @f5(i64 noundef %a.coerce)
 void f5(_Complex float a) {}
 
-// CHECK-LABEL: define dso_local void @f6(ptr noundef %a)
+// CHECK-LABEL: define dso_local void @f6(ptr dead_on_return noundef %a)
 void f6(_Complex double a) {}
 
 // CHECK-LABEL: define dso_local i64 @f7()
diff --git a/clang/test/CodeGen/aapcs64-align.cpp b/clang/test/CodeGen/aapcs64-align.cpp
index 1c26d68e434f4..53fc53c2f7296 100644
--- a/clang/test/CodeGen/aapcs64-align.cpp
+++ b/clang/test/CodeGen/aapcs64-align.cpp
@@ -122,8 +122,8 @@ unsigned sizeof_RidiculouslyOverSizedBitfield = sizeof(RidiculouslyOverSizedBitf
 unsigned alignof_RidiculouslyOverSizedBitfield = alignof(RidiculouslyOverSizedBitfield);
 
 // CHECK: define{{.*}} void @g9
-// CHECK: call void @f9(i32 noundef 1, ptr noundef nonnull %agg.tmp)
-// CHECK: declare void @f9(i32 noundef, ptr noundef)
+// CHECK: call void @f9(i32 noundef 1, ptr dead_on_return noundef nonnull %agg.tmp)
+// CHECK: declare void @f9(i32 noundef, ptr dead_on_return noundef)
 void f9(int a, RidiculouslyOverSizedBitfield b);
 void g9() {
   RidiculouslyOverSizedBitfield s = {42};
diff --git a/clang/test/CodeGen/arm-aapcs-vfp.c b/clang/test/CodeGen/arm-aapcs-vfp.c
index 6581929f99f14..e60ed1e52c33a 100644
--- a/clang/test/CodeGen/arm-aapcs-vfp.c
+++ b/clang/test/CodeGen/arm-aapcs-vfp.c
@@ -65,7 +65,7 @@ struct big_struct {
   float f4;
 };
 // CHECK: define{{.*}} arm_aapcs_vfpcc void @test_big([5 x i32] %{{.*}})
-// CHECK64: define{{.*}} void @test_big(ptr noundef %{{.*}})
+// CHECK64: define{{.*}} void @test_big(ptr dead_on_return noundef %{{.*}})
 // CHECK64: call void @llvm.memcpy
 // CHECK64: call void @big_callee(ptr
 extern void big_callee(struct big_struct);
diff --git a/clang/test/CodeGen/arm-abi-vector.c b/clang/test/CodeGen/arm-abi-vector.c
index c2a8902007980..93b770878c3fa 100644
--- a/clang/test/CodeGen/arm-abi-vector.c
+++ b/clang/test/CodeGen/arm-abi-vector.c
@@ -177,11 +177,11 @@ double varargs_vec_19c(int fixed, ...) {
 
 double test_19c(__char19 *in) {
 // CHECK: test_19c
-// CHECK: call arm_aapcscc double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call arm_aapcscc double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
 // APCS-GNU: test_19c
-// APCS-GNU: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// APCS-GNU: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
 // ANDROID: test_19c
-// ANDROID: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// ANDROID: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_19c(19, *in);
 }
 
diff --git a/clang/test/CodeGen/arm-swiftcall.c b/clang/test/CodeGen/arm-swiftcall.c
index 677b878c6765d..55c22a45fd8de 100644
--- a/clang/test/CodeGen/arm-swiftcall.c
+++ b/clang/test/CodeGen/arm-swiftcall.c
@@ -261,7 +261,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define{{.*}} void @return_struct_big_1({{.*}} dead_on_unwind noalias writable sret({{.*}})
 
 // Should not be byval.
-// CHECK-LABEL: define{{.*}} void @take_struct_big_1(ptr{{( %.*)?}})
+// CHECK-LABEL: define{{.*}} void @take_struct_big_1(ptr dead_on_return{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGen/arm64-abi-vector.c b/clang/test/CodeGen/arm64-abi-vector.c
index 81e42315c883b..cf50cdd2fe86e 100644
--- a/clang/test/CodeGen/arm64-abi-vector.c
+++ b/clang/test/CodeGen/arm64-abi-vector.c
@@ -128,7 +128,7 @@ double varargs_vec_19c(int fixed, ...) {
 
 double test_19c(__char19 *in) {
 // CHECK: test_19c
-// CHECK: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_19c(19, *in);
 }
 
@@ -211,7 +211,7 @@ double varargs_vec_5i(int fixed, ...) {
 
 double test_5i(__int5 *in) {
 // CHECK: test_5i
-// CHECK: call double (i32, ...) @varargs_vec_5i(i32 noundef 5, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_5i(i32 noundef 5, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_5i(5, *in);
 }
 
@@ -231,7 +231,7 @@ double varargs_vec_3d(int fixed, ...) {
 
 double test_3d(__double3 *in) {
 // CHECK: test_3d
-// CHECK: call double (i32, ...) @varargs_vec_3d(i32 noundef 3, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_3d(i32 noundef 3, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_3d(3, *in);
 }
 
@@ -291,7 +291,7 @@ double test(__char3 *c3, __char5 *c5, __char9 *c9, __char19 *c19,
             __short3 *s3, __short5 *s5, __int3 *i3, __int5 *i5,
             __double3 *d3) {
   double ret = varargs_vec(3, *c3, *c5, *c9, *c19, *s3, *s5, *i3, *i5, *d3);
-// CHECK: call double (i32, ...) @varargs_vec(i32 noundef 3, i32 {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr noundef {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr noundef {{%.*}}, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec(i32 noundef 3, i32 {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr dead_on_return noundef {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr dead_on_return noundef {{%.*}}, ptr dead_on_return noundef {{%.*}})
   return ret;
 }
 
@@ -350,7 +350,7 @@ __attribute__((noinline)) double args_vec_19c(int fixed, __char19 c19) {
 
 double fixed_19c(__char19 *in) {
 // CHECK: fixed_19c
-// CHECK: call double @args_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return args_vec_19c(19, *in);
 }
 
@@ -409,7 +409,7 @@ __attribute__((noinline)) double args_vec_5i(int fixed, __int5 c5) {
 
 double fixed_5i(__int5 *in) {
 // CHECK: fixed_5i
-// CHECK: call double @args_vec_5i(i32 noundef 5, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_5i(i32 noundef 5, ptr dead_on_return noundef {{%.*}})
   return args_vec_5i(5, *in);
 }
 
@@ -424,6 +424,6 @@ __attribute__((noinline)) double args_vec_3d(int fixed, __double3 c3) {
 
 double fixed_3d(__double3 *in) {
 // CHECK: fixed_3d
-// CHECK: call double @args_vec_3d(i32 noundef 3, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_3d(i32 noundef 3, ptr dead_on_return noundef {{%.*}})
   return args_vec_3d(3, *in);
 }
diff --git a/clang/test/CodeGen/arm64-arguments.c b/clang/test/CodeGen/arm64-arguments.c
index 4c4f85d923e78..2e3ab388432f6 100644
--- a/clang/test/CodeGen/arm64-arguments.c
+++ b/clang/test/CodeGen/arm64-arguments.c
@@ -163,7 +163,7 @@ void f32(struct s32 s) { }
 // A composite type larger than 16 bytes should be passed indirectly.
 struct s33 { char buf[32*32]; };
 void f33(struct s33 s) { }
-// CHECK: define{{.*}} void @f33(ptr noundef %s)
+// CHECK: define{{.*}} void @f33(ptr dead_on_return noundef %s)
 
 struct s34 { char c; };
 void f34(struct s34 s);
@@ -226,9 +226,9 @@ T_float32x2 f1_0(T_float32x2 a0) { return a0; }
 // CHECK: define{{.*}} <4 x float> @f1_1(<4 x float> noundef %{{.*}})
 T_float32x4 f1_1(T_float32x4 a0) { return a0; }
 // Vector with length bigger than 16-byte is illegal and is passed indirectly.
-// CHECK: define{{.*}} void @f1_2(ptr dead_on_unwind noalias writable sret(<8 x float>) align 16 %{{.*}}, ptr noundef %0)
+// CHECK: define{{.*}} void @f1_2(ptr dead_on_unwind noalias writable sret(<8 x float>) align 16 %{{.*}}, ptr dead_on_return noundef %0)
 T_float32x8 f1_2(T_float32x8 a0) { return a0; }
-// CHECK: define{{.*}} void @f1_3(ptr dead_on_unwind noalias writable sret(<16 x float>) align 16 %{{.*}}, ptr noundef %0)
+// CHECK: define{{.*}} void @f1_3(ptr dead_on_unwind noalias writable sret(<16 x float>) align 16 %{{.*}}, ptr dead_on_return noundef %0)
 T_float32x16 f1_3(T_float32x16 a0) { return a0; }
 
 // Testing alignment with aggregates: HFA, aggregates with size <= 16 bytes and
@@ -278,7 +278,7 @@ struct s37
 typedef struct s37 s37_with_align;
 
 int32x4_t f37(int i, s37_with_align s1, s37_with_align s2) {
-// CHECK: define{{.*}} <4 x i32> @f37(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} <4 x i32> @f37(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: load <4 x i32>, ptr %s1, align 16
 // CHECK: load <4 x i32>, ptr %s2, align 16
   int32x4_t v = vaddq_s32(*(int32x4_t *)&s1,
@@ -292,7 +292,7 @@ int32x4_t caller37() {
 // CHECK: %[[b:.*]] = alloca %struct.s37, align 16
 // CHECK: call void @llvm.memcpy
 // CHECK: call void @llvm.memcpy
-// CHECK: call <4 x i32> @f37(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call <4 x i32> @f37(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f37(3, g37, g37);
 }
 
@@ -530,7 +530,7 @@ typedef struct s42 s42_no_align;
 // passing structs in registers
 __attribute__ ((noinline))
 int f42(int i, s42_no_align s1, s42_no_align s2) {
-// CHECK: define{{.*}} i32 @f42(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f42(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 1
@@ -545,14 +545,14 @@ int caller42() {
 // CHECK: %[[b:.*]] = alloca %struct.s42, align 4
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f42(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f42(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f42(3, g42, g42_2);
 }
 // passing structs on stack
 __attribute__ ((noinline))
 int f42_stack(int i, int i2, int i3, int i4, int i5, int i6, int i7, int i8,
               int i9, s42_no_align s1, s42_no_align s2) {
-// CHECK: define{{.*}} i32 @f42_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f42_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 1
@@ -565,7 +565,7 @@ int caller42_stack() {
 // CHECK: %[[b:.*]] = alloca %struct.s42, align 4
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f42_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f42_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f42_stack(1, 2, 3, 4, 5, 6, 7, 8, 9, g42, g42_2);
 }
 
@@ -583,7 +583,7 @@ typedef struct s43 s43_with_align;
 // passing aligned structs in registers
 __attribute__ ((noinline))
 int f43(int i, s43_with_align s1, s43_with_align s2) {
-// CHECK: define{{.*}} i32 @f43(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f43(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 1
@@ -598,14 +598,14 @@ int caller43() {
 // CHECK: %[[b:.*]] = alloca %struct.s43, align 16
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f43(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f43(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f43(3, g43, g43_2);
 }
 // passing aligned structs on stack
 __attribute__ ((noinline))
 int f43_stack(int i, int i2, int i3, int i4, int i5, int i6, int i7, int i8,
               int i9, s43_with_align s1, s43_with_align s2) {
-// CHECK: define{{.*}} i32 @f43_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f43_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 1
@@ -618,7 +618,7 @@ int caller43_stack() {
 // CHECK: %[[b:.*]] = alloca %struct.s43, align 16
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f43_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f43_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f43_stack(1, 2, 3, 4, 5, 6, 7, 8, 9, g43, g43_2);
 }
 
diff --git a/clang/test/CodeGen/arm64-microsoft-arguments.cpp b/clang/test/CodeGen/arm64-microsoft-arguments.cpp
index 85472645acb3b..a0a81be54325f 100644
--- a/clang/test/CodeGen/arm64-microsoft-arguments.cpp
+++ b/clang/test/CodeGen/arm64-microsoft-arguments.cpp
@@ -29,7 +29,7 @@ S2 f2() {
 
 // Pass and return for type size > 16 bytes.
 // CHECK: define {{.*}} void @{{.*}}f3{{.*}}(ptr dead_on_unwind noalias writable sret(%struct.S3) align 4 %agg.result)
-// CHECK: call void {{.*}}func3{{.*}}(ptr dead_on_unwind writable sret(%struct.S3) align 4 %agg.result, ptr noundef %agg.tmp)
+// CHECK: call void {{.*}}func3{{.*}}(ptr dead_on_unwind writable sret(%struct.S3) align 4 %agg.result, ptr dead_on_return noundef %agg.tmp)
 struct S3 {
   int a[5];
 };
diff --git a/clang/test/CodeGen/armv7k-abi.c b/clang/test/CodeGen/armv7k-abi.c
index 872e6423a4a99..6a781bc04d042 100644
--- a/clang/test/CodeGen/armv7k-abi.c
+++ b/clang/test/CodeGen/armv7k-abi.c
@@ -39,7 +39,7 @@ typedef struct {
   double z;
 } BigStruct;
 
-// CHECK: define{{.*}} void @big_struct_indirect(ptr noundef %b)
+// CHECK: define{{.*}} void @big_struct_indirect(ptr dead_on_return noundef %b)
 void big_struct_indirect(BigStruct b) {}
 
 // CHECK: define{{.*}} void @return_big_struct_indirect(ptr dead_on_unwind noalias writable sret
diff --git a/clang/test/CodeGen/atomic-arm64.c b/clang/test/CodeGen/atomic-arm64.c
index d2a30a3b6e66f..d539cad0c6a7d 100644
--- a/clang/test/CodeGen/atomic-arm64.c
+++ b/clang/test/CodeGen/atomic-arm64.c
@@ -57,7 +57,7 @@ void test3(pointer_pair_t pair) {
 }
 
 // CHECK-LABEL:define{{.*}} void @test4(
-// CHECK-SAME: ptr noundef [[QUAD:%.*]])
+// CHECK-SAME: ptr dead_on_return noundef [[QUAD:%.*]])
 // CHECK:      [[QUAD_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT: [[TEMP:%.*]] = alloca [[QUAD_T:%.*]], align 8
 // CHECK-NEXT: store ptr [[QUAD]], ptr [[QUAD_INDIRECT_ADDR]]
diff --git a/clang/test/CodeGen/attr-noundef.cpp b/clang/test/CodeGen/attr-noundef.cpp
index abdf9496bd396..619dbec7678db 100644
--- a/clang/test/CodeGen/attr-noundef.cpp
+++ b/clang/test/CodeGen/attr-noundef.cpp
@@ -27,7 +27,7 @@ struct NoCopy {
 NoCopy ret_nocopy() { return {}; }
 void pass_nocopy(NoCopy e) {}
 // CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr noundef %
+// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr dead_on_return noundef %
 
 struct Huge {
   int a[1024];
@@ -35,7 +35,8 @@ struct Huge {
 Huge ret_huge() { return {}; }
 void pass_huge(Huge h) {}
 // CHECK: [[DEF]] void @{{.*}}ret_huge{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr noundef
+// CHECK-INTEL: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr noundef
+// CHECK-AARCH: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr dead_on_return noundef
 } // namespace check_structs
 
 //************ Passing unions by value
@@ -59,7 +60,7 @@ union NoCopy {
 NoCopy ret_nocopy() { return {}; }
 void pass_nocopy(NoCopy e) {}
 // CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr noundef %
+// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr dead_on_return noundef %
 } // namespace check_unions
 
 //************ Passing `this` pointers
diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c
index cf74ab2dcca3f..fba692cac4492 100644
--- a/clang/test/CodeGen/cx-complex-range.c
+++ b/clang/test/CodeGen/cx-complex-range.c
@@ -1520,7 +1520,7 @@ void mulassignf(_Complex float *a, _Complex float b) {
 // PRMTD-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @divd(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -1744,7 +1744,7 @@ void mulassignf(_Complex float *a, _Complex float b) {
 // PRMTD_FAST-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divd(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -1938,7 +1938,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @divassignd(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2180,7 +2180,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divassignd(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2325,7 +2325,7 @@ void divassignd(_Complex double *a, _Complex double b) {
 // PRMTD-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @muld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -2457,7 +2457,7 @@ void divassignd(_Complex double *a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @muld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -2594,7 +2594,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulassignd(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2744,7 +2744,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulassignd(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2922,7 +2922,7 @@ void mulassignd(_Complex double *a, _Complex double b) {
 // PRMTD-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @divld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -3190,7 +3190,7 @@ void mulassignd(_Complex double *a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -3432,7 +3432,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @divassignld(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -3702,7 +3702,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divassignld(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -3895,7 +3895,7 @@ void divassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -4059,7 +4059,7 @@ void divassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -4220,7 +4220,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulassignld(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -4386,7 +4386,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulassignld(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -4644,7 +4644,7 @@ void mulassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD-NEXT:    ret <2 x float> [[RETVAL_SROA_0_4_VEC_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local i64 @f1(
-// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE]] to i32
 // X86WINPRMTD-NEXT:    [[TMP0:%.*]] = bitcast i32 [[A_SROA_0_0_EXTRACT_TRUNC]] to float
@@ -5052,7 +5052,7 @@ void mulassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret <2 x float> [[RETVAL_SROA_0_4_VEC_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local i64 @f1(
-// X86WINPRMTD_STRICT-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: i64 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE]] to i32
 // X86WINPRMTD_STRICT-NEXT:    [[TMP0:%.*]] = bitcast i32 [[A_SROA_0_0_EXTRACT_TRUNC]] to float
diff --git a/clang/test/CodeGen/ext-int-cc.c b/clang/test/CodeGen/ext-int-cc.c
index fdca4012ee4a4..7cfd992fd48b4 100644
--- a/clang/test/CodeGen/ext-int-cc.c
+++ b/clang/test/CodeGen/ext-int-cc.c
@@ -33,9 +33,9 @@
 // Make sure 128 and 64 bit versions are passed like integers.
 void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
 // LIN64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// WIN64: define dso_local void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
-// WIN32: define dso_local void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// WIN64: define dso_local void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
+// WIN32: define dso_local void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // NVPTX64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // NVPTX: define{{.*}} void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // SPARCV9: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
@@ -50,22 +50,22 @@ void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
 // ARC: define{{.*}} void @ParamPassing(ptr byval(i128) align 4 %{{.+}}, i64 inreg %{{.+}})
 // XCORE: define{{.*}} void @ParamPassing(ptr byval(i128) align 4 %{{.+}}, i64 %{{.+}})
 // RISCV64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// RISCV32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// RISCV32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // WASM: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// SYSTEMZ: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// SYSTEMZ: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // PPC64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // PPC32: define{{.*}} void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // AARCH64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // AARCH64DARWIN: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // ARM: define{{.*}} arm_aapcscc void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // LA64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// LA32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// LA32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 
 void ParamPassing2(_BitInt(127) b, _BitInt(63) c) {}
 // LIN64: define{{.*}} void @ParamPassing2(i64 %{{.+}}, i64 %{{.+}}, i64 %{{.+}})
-// WIN64: define dso_local void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
-// WIN32: define dso_local void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// WIN64: define dso_local void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
+// WIN32: define dso_local void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 // NVPTX64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // NVPTX: define{{.*}} void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // SPARCV9: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
@@ -80,16 +80,16 @@ void ParamPassing2(_BitInt(127) b, _BitInt(63) c) {}
 // ARC: define{{.*}} void @ParamPassing2(ptr byval(i128) align 4 %{{.+}}, i63 inreg %{{.+}})
 // XCORE: define{{.*}} void @ParamPassing2(ptr byval(i128) align 4 %{{.+}}, i63 %{{.+}})
 // RISCV64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
-// RISCV32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// RISCV32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 // WASM: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
-// SYSTEMZ: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 signext %{{.+}})
+// SYSTEMZ: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 signext %{{.+}})
 // PPC64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
 // PPC32: define{{.*}} void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // AARCH64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // AARCH64DARWIN: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // ARM: define{{.*}} arm_aapcscc void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // LA64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
-// LA32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// LA32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 
 // Make sure we follow the signext rules for promotable integer types.
 void ParamPassing3(_BitInt(15) a, _BitInt(31) b) {}
@@ -129,10 +129,10 @@ void ParamPassing3(_BitInt(15) a, _BitInt(31) b) {}
 // _BitInt widths to alert us to enable the test.
 void ParamPassing4(_BitInt(129) a) {}
 // LIN64: define{{.*}} void @ParamPassing4(ptr byval([24 x i8]) align 8 %{{.+}})
-// WIN64: define dso_local void @ParamPassing4(ptr %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// WIN32: define dso_local void @ParamPassing4(ptr %{{.+}})
-// AARCH64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// WIN64: define dso_local void @ParamPassing4(ptr dead_on_return %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
+// WIN32: define dso_local void @ParamPassing4(ptr dead_on_return %{{.+}})
+// AARCH64: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
 // NVPTX64-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // NVPTX-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // SPARCV9-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
@@ -154,8 +154,8 @@ void ParamPassing4(_BitInt(129) a) {}
 // PPC32-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // AARCH64DARWIN-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // ARM-NOT: define{{.*}} arm_aapcscc void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
-// LA64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// LA32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// LA64: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
+// LA32: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
 #endif
 
 _BitInt(63) ReturnPassing(void) { return 0; }
diff --git a/clang/test/CodeGen/isfpclass.c b/clang/test/CodeGen/isfpclass.c
index 26dd846a2bf20..ee3a22b40fefd 100644
--- a/clang/test/CodeGen/isfpclass.c
+++ b/clang/test/CodeGen/isfpclass.c
@@ -160,7 +160,7 @@ int4 check_isfpclass_nan_strict_v4f32(float4 x) {
 }
 
 // CHECK-LABEL: define dso_local void @check_isfpclass_nan_v4f64
-// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X:%.*]] = load <4 x double>, ptr [[TMP0]], align 16, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <4 x double> [[X]], zeroinitializer
diff --git a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
index eb706154300a2..0124cc5c06d43 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
+++ b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
@@ -30,37 +30,37 @@ long double powl(long double a, long double b);
 //
 // CHECK-PPC-LABEL: define dso_local ppc_fp128 @test_powl(
 // CHECK-PPC-SAME: ppc_fp128 noundef [[A:%.*]], ppc_fp128 noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @powl(ppc_fp128 noundef [[A]], ppc_fp128 noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @powl(ppc_fp128 noundef [[A]], ppc_fp128 noundef [[B]]) #[[ATTR4:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-ARM-LABEL: define dso_local double @test_powl(
 // CHECK-ARM-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-ARM:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-ARM-HF-LABEL: define dso_local double @test_powl(
 // CHECK-ARM-HF-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-THUMB-LABEL: define double @test_powl(
 // CHECK-THUMB-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-THUMB:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-AARCH-LABEL: define dso_local fp128 @test_powl(
 // CHECK-AARCH-SAME: fp128 noundef [[A:%.*]], fp128 noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @powl(fp128 noundef [[A]], fp128 noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @powl(fp128 noundef [[A]], fp128 noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func double @test_powl(
 // CHECK-SPIR-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR4:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-MINGW32-LABEL: define dso_local void @test_powl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]], ptr noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3:![0-9]+]]
-// CHECK-MINGW32:    [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[B]], ptr [[BYVAL_TEMP1:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    call void @powl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr noundef nonnull [[BYVAL_TEMP]], ptr noundef nonnull [[BYVAL_TEMP1]]) #[[ATTR3:[0-9]+]]
-// CHECK-MINGW32:    [[TMP2:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[TMP2]], ptr [[AGG_RESULT]], align 16, !tbaa [[TBAA3]]
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA6:![0-9]+]]
+// CHECK-MINGW32:    [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[B]], ptr [[BYVAL_TEMP1:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    call void @powl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP1]]) #[[ATTR3:[0-9]+]]
+// CHECK-MINGW32:    [[TMP2:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[TMP2]], ptr [[AGG_RESULT]], align 16, !tbaa [[TBAA6]]
 //
 long double test_powl(long double a, long double b) {
    return powl(a, b);
@@ -93,51 +93,51 @@ long double test_powl(long double a, long double b) {
 // CHECK-I686:    store x86_fp80 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 4
 //
 // CHECK-PPC-LABEL: define dso_local void @test_cargl(
-// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-PPC:    [[CLD_REAL:%.*]] = load ppc_fp128, ptr [[CLD]], align 16
 // CHECK-PPC:    [[CLD_IMAG:%.*]] = load ppc_fp128, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-PPC:    store ppc_fp128 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
 // CHECK-PPC:    store ppc_fp128 [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 16
-// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @cargl(ptr noundef nonnull byval({ ppc_fp128, ppc_fp128 }) align 16 [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @cargl(ptr noundef nonnull byval({ ppc_fp128, ppc_fp128 }) align 16 [[BYVAL_TEMP]]) #[[ATTR4]]
 // CHECK-PPC:    store ppc_fp128 [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 16
 // CHECK-PPC:    store ppc_fp128 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 16
 //
 // CHECK-ARM-LABEL: define dso_local void @test_cargl(
-// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-ARM:    [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 // CHECK-ARM:    store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
 // CHECK-ARM:    store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
 //
 // CHECK-ARM-HF-LABEL: define dso_local { double, double } @test_cargl(
-// CHECK-ARM-HF-SAME: { double, double } noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @cargl({ double, double } noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-HF-SAME: { double, double } noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @cargl({ double, double } noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-THUMB-LABEL: define { double, double } @test_cargl(
-// CHECK-THUMB-SAME: [2 x double] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call double @cargl([2 x double] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-THUMB-SAME: [2 x double] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-THUMB:    [[CALL:%.*]] = tail call double @cargl([2 x double] noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-AARCH-LABEL: define dso_local { fp128, fp128 } @test_cargl(
-// CHECK-AARCH-SAME: [2 x fp128] noundef alignstack(16) [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA2]]
+// CHECK-AARCH-SAME: [2 x fp128] noundef alignstack(16) [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA2]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func void @test_cargl(
-// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-SPIR:    [[CLD_REAL:%.*]] = load double, ptr [[CLD]], align 8
 // CHECK-SPIR:    [[CLD_IMAG:%.*]] = load double, ptr [[CLD_IMAGP:%.*]], align 8
 // CHECK-SPIR:    store double [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 8
 // CHECK-SPIR:    store double [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 8
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @cargl(ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @cargl(ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]]) #[[ATTR4]]
 // CHECK-SPIR:    store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
 // CHECK-SPIR:    store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
 //
 // CHECK-MINGW32-LABEL: define dso_local void @test_cargl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-MINGW32:    [[CLD_REAL:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK-MINGW32:    [[CLD_IMAG:%.*]] = load x86_fp80, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 16
-// CHECK-MINGW32:    call void @cargl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
-// CHECK-MINGW32:    [[TMP0:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA3]]
+// CHECK-MINGW32:    call void @cargl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-MINGW32:    [[TMP0:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA6]]
 // CHECK-MINGW32:    [[CLD_REAL3:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK-MINGW32:    [[CLD_IMAG5:%.*]] = load x86_fp80, ptr [[CLD_IMAGP]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 16
@@ -166,33 +166,33 @@ int ilogbl(long double a);
 //
 // CHECK-PPC-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-PPC-SAME: ppc_fp128 noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-PPC:    [[CALL:%.*]] = tail call i32 @ilogbl(ppc_fp128 noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call i32 @ilogbl(ppc_fp128 noundef [[A]]) #[[ATTR4]], !tbaa [[TBAA2]]
 //
 // CHECK-ARM-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-ARM-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-ARM-HF-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-ARM-HF-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-THUMB-LABEL: define i32 @test_ilogb(
 // CHECK-THUMB-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-THUMB:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-AARCH-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-AARCH-SAME: fp128 noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call i32 @ilogbl(fp128 noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA2]]
+// CHECK-AARCH:    [[CALL:%.*]] = tail call i32 @ilogbl(fp128 noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func i32 @test_ilogb(
 // CHECK-SPIR-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR4]], !tbaa [[TBAA2]]
 //
 // CHECK-MINGW32-LABEL: define dso_local i32 @test_ilogb(
-// CHECK-MINGW32-SAME: ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    [[CALL:%.*]] = call i32 @ilogbl(ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-MINGW32-SAME: ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    [[CALL:%.*]] = call i32 @ilogbl(ptr dead_on_return noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
 //
 int test_ilogb(long double a) {
    return ilogbl(a);
@@ -243,8 +243,8 @@ int test_ilogb(long double a) {
 // CHECK-SPIR: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
 // CHECK-SPIR: [[META5]] = !{!"Simple C/C++ TBAA"}
 //.
-// CHECK-MINGW32: [[TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
-// CHECK-MINGW32: [[META4]] = !{!"long double", [[META5:![0-9]+]], i64 0}
-// CHECK-MINGW32: [[META5]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
-// CHECK-MINGW32: [[META6]] = !{!"Simple C/C++ TBAA"}
+// CHECK-MINGW32: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+// CHECK-MINGW32: [[META7]] = !{!"long double", [[META8:![0-9]+]], i64 0}
+// CHECK-MINGW32: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+// CHECK-MINGW32: [[META9]] = !{!"Simple C/C++ TBAA"}
 //.
diff --git a/clang/test/CodeGen/mingw-long-double.c b/clang/test/CodeGen/mingw-long-double.c
index 0fc8f01509682..b98929701bc62 100644
--- a/clang/test/CodeGen/mingw-long-double.c
+++ b/clang/test/CodeGen/mingw-long-double.c
@@ -29,15 +29,15 @@ long double TestLD(long double x) {
   return x * x;
 }
 // GNU32: define dso_local x86_fp80 @TestLD(x86_fp80 noundef %x)
-// GNU64: define dso_local void @TestLD(ptr dead_on_unwind noalias writable sret(x86_fp80) align 16 %agg.result, ptr noundef %0)
+// GNU64: define dso_local void @TestLD(ptr dead_on_unwind noalias writable sret(x86_fp80) align 16 %agg.result, ptr dead_on_return noundef %0)
 // MSC64: define dso_local double @TestLD(double noundef %x)
 
 long double _Complex TestLDC(long double _Complex x) {
   return x * x;
 }
 // GNU32: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 4 %agg.result, ptr noundef byval({ x86_fp80, x86_fp80 }) align 4 %x)
-// GNU64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 16 %agg.result, ptr noundef %x)
-// MSC64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %agg.result, ptr noundef %x)
+// GNU64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 16 %agg.result, ptr dead_on_return noundef %x)
+// MSC64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %agg.result, ptr dead_on_return noundef %x)
 
 // GNU32: declare dso_local void @__mulxc3
 // GNU64: declare dso_local void @__mulxc3
diff --git a/clang/test/CodeGen/ms_abi.c b/clang/test/CodeGen/ms_abi.c
index 528e546f315d5..5d58c9816da78 100644
--- a/clang/test/CodeGen/ms_abi.c
+++ b/clang/test/CodeGen/ms_abi.c
@@ -142,7 +142,7 @@ struct i128 {
 };
 
 __attribute__((ms_abi)) struct i128 f7(struct i128 a) {
-  // WIN64: define dso_local void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr noundef %a)
-  // FREEBSD: define{{.*}} win64cc void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr noundef %a)
+  // WIN64: define dso_local void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr dead_on_return noundef %a)
+  // FREEBSD: define{{.*}} win64cc void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr dead_on_return noundef %a)
   return a;
 }
diff --git a/clang/test/CodeGen/pass-by-value-noalias.c b/clang/test/CodeGen/pass-by-value-noalias.c
index bc35d13c4df6a..e673ceb80bebe 100644
--- a/clang/test/CodeGen/pass-by-value-noalias.c
+++ b/clang/test/CodeGen/pass-by-value-noalias.c
@@ -11,6 +11,6 @@ struct Foo {
   int f;
 };
 
-// WITH_NOALIAS: define{{.*}} void @take(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @take(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
 void take(struct Foo arg) {}
diff --git a/clang/test/CodeGen/ptrauth-in-c-struct.c b/clang/test/CodeGen/ptrauth-in-c-struct.c
index 2aec31ec3baf9..c74be17b4c837 100644
--- a/clang/test/CodeGen/ptrauth-in-c-struct.c
+++ b/clang/test/CodeGen/ptrauth-in-c-struct.c
@@ -115,7 +115,7 @@ void test_move_assignment_SA(SA *p) {
   *p = getSA();
 }
 
-// CHECK: define void @test_parameter_SA(ptr noundef %{{.*}})
+// CHECK: define void @test_parameter_SA(ptr dead_on_return noundef %{{.*}})
 // CHECK-NOT: call
 // CHECK: ret void
 
@@ -128,7 +128,7 @@ void test_parameter_SA(SA a) {
 // CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
 // CHECK: call void @__copy_constructor_8_8_t0w4_pa1_50_8(ptr %[[AGG_TMP]], ptr %[[V0]])
-// CHECK: call void @calleeSA(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @calleeSA(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK-NOT: call
 // CHECK: ret void
 
diff --git a/clang/test/CodeGen/regcall.c b/clang/test/CodeGen/regcall.c
index f10da87353fa1..d4b9f00d54d41 100644
--- a/clang/test/CodeGen/regcall.c
+++ b/clang/test/CodeGen/regcall.c
@@ -28,7 +28,7 @@ struct Large { int a[5]; };
 void __regcall v4(int a, struct Large b, int c) {}
 // Win32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // Lin32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
-// Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 // Lin64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
 
 void __regcall v5(long long a, int b, int c) {}
@@ -47,7 +47,7 @@ void __regcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __regcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_regcallcc void @__regcall3__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall3__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall3__hfa2(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
@@ -61,7 +61,7 @@ void __regcall hfa3(double a, double b, double c, double d, double e, struct HFA
 // handling to ensure alignment.
 void __regcall hfa4(struct HFA5 a) {}
 // X32: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr noundef byval(%struct.HFA5) align 4 %{{.*}})
-// Win64: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr noundef %a)
+// Win64: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr dead_on_return noundef %a)
 // Lin64: define dso_local x86_regcallcc void @__regcall3__hfa4(double %a.coerce0, double %a.coerce1, double %a.coerce2, double %a.coerce3, double %a.coerce4)
 
 // Return HFAs of 4 or fewer elements in registers.
@@ -79,7 +79,7 @@ void __regcall hva1(int a, struct HVA4 b, int c) {}
 // X64: define dso_local x86_regcallcc void @__regcall3__hva1(i32 noundef %a, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 noundef %c)
 
 void __regcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {}
-// X86: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> noundef %c)
 
 void __regcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {}
@@ -95,6 +95,6 @@ void __regcall odd_size_hva(struct OddSizeHVA a) {}
 
 struct HFA6 { __m128 f[4]; };
 struct HFA6 __regcall ret_reg_reused(struct HFA6 a, struct HFA6 b, struct HFA6 c, struct HFA6 d){ struct HFA6 h; return h;}
-// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %c, ptr inreg noundef %d)
+// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %c, ptr dead_on_return inreg noundef %d)
 // Win64: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, <4 x float> %c.0, <4 x float> %c.1, <4 x float> %c.2, <4 x float> %c.3, <4 x float> %d.0, <4 x float> %d.1, <4 x float> %d.2, <4 x float> %d.3)
 // Lin64: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused([4 x <4 x float>] %a.coerce, [4 x <4 x float>] %b.coerce, [4 x <4 x float>] %c.coerce, [4 x <4 x float>] %d.coerce)
diff --git a/clang/test/CodeGen/regcall2.c b/clang/test/CodeGen/regcall2.c
index c88d4e485b104..42619369677b2 100644
--- a/clang/test/CodeGen/regcall2.c
+++ b/clang/test/CodeGen/regcall2.c
@@ -20,7 +20,7 @@ double __regcall bar(__sVector a) {
 
 // FIXME: Do we need to change for Windows?
 // Win: define dso_local x86_regcallcc void @__regcall3__foo(ptr dead_on_unwind noalias writable sret(%struct.__sVector) align 64 %agg.result, i32 noundef %a) #0
-// Win: define dso_local x86_regcallcc double @__regcall3__bar(ptr noundef %a) #0
+// Win: define dso_local x86_regcallcc double @__regcall3__bar(ptr dead_on_return noundef %a) #0
 // Win: attributes #0 = { noinline nounwind optnone "min-legal-vector-width"="0" "no-builtins" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+avx,+avx2,+avx512f,+avx512vl,+crc32,+cx8,+evex512,+f16c,+fma,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" }
 
 // Lin: define dso_local x86_regcallcc %struct.__sVector @__regcall3__foo(i32 noundef %a) #0
diff --git a/clang/test/CodeGen/regcall4.c b/clang/test/CodeGen/regcall4.c
index 5fbe77fbc7d76..d5fe5d88a0e8c 100644
--- a/clang/test/CodeGen/regcall4.c
+++ b/clang/test/CodeGen/regcall4.c
@@ -28,7 +28,7 @@ struct Large { int a[5]; };
 void __regcall v4(int a, struct Large b, int c) {}
 // Win32: define dso_local x86_regcallcc void @__regcall4__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // Lin32: define dso_local x86_regcallcc void @__regcall4__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
-// Win64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// Win64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 // Lin64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
 
 void __regcall v5(long long a, int b, int c) {}
@@ -47,7 +47,7 @@ void __regcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __regcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_regcallcc void @__regcall4__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall4__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall4__hfa2(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
@@ -61,7 +61,7 @@ void __regcall hfa3(double a, double b, double c, double d, double e, struct HFA
 // handling to ensure alignment.
 void __regcall hfa4(struct HFA5 a) {}
 // X32: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr noundef byval(%struct.HFA5) align 4 %{{.*}})
-// Win64: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr noundef %a)
+// Win64: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr dead_on_return noundef %a)
 // Lin64: define dso_local x86_regcallcc void @__regcall4__hfa4(double %a.coerce0, double %a.coerce1, double %a.coerce2, double %a.coerce3, double %a.coerce4)
 
 // Return HFAs of 4 or fewer elements in registers.
@@ -79,7 +79,7 @@ void __regcall hva1(int a, struct HVA4 b, int c) {}
 // X64: define dso_local x86_regcallcc void @__regcall4__hva1(i32 noundef %a, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 noundef %c)
 
 void __regcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {}
-// X86: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> noundef %c)
 
 void __regcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {}
@@ -95,6 +95,6 @@ void __regcall odd_size_hva(struct OddSizeHVA a) {}
 
 struct HFA6 { __m128 f[4]; };
 struct HFA6 __regcall ret_reg_reused(struct HFA6 a, struct HFA6 b, struct HFA6 c, struct HFA6 d){ struct HFA6 h; return h;}
-// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %c, ptr inreg noundef %d)
+// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %c, ptr dead_on_return inreg noundef %d)
 // Win64: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, <4 x float> %c.0, <4 x float> %c.1, <4 x float> %c.2, <4 x float> %c.3, <4 x float> %d.0, <4 x float> %d.1, <4 x float> %d.2, <4 x float> %d.3)
 // Lin64: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused([4 x <4 x float>] %a.coerce, [4 x <4 x float>] %b.coerce, [4 x <4 x float>] %c.coerce, [4 x <4 x float>] %d.coerce)
diff --git a/clang/test/CodeGen/sparcv9-abi.c b/clang/test/CodeGen/sparcv9-abi.c
index 616e24e7c519d..5a3d64fd37889 100644
--- a/clang/test/CodeGen/sparcv9-abi.c
+++ b/clang/test/CodeGen/sparcv9-abi.c
@@ -80,7 +80,7 @@ struct medium {
   int *c, *d;
 };
 
-// CHECK-LABEL: define{{.*}} %struct.medium @f_medium(ptr noundef %x)
+// CHECK-LABEL: define{{.*}} %struct.medium @f_medium(ptr dead_on_return noundef %x)
 struct medium f_medium(struct medium x) {
   x.a += *x.b;
   x.b = 0;
@@ -94,7 +94,7 @@ struct large {
   int x;
 };
 
-// CHECK-LABEL: define{{.*}} void @f_large(ptr dead_on_unwind noalias writable sret(%struct.large) align 8 %agg.result, ptr noundef %x)
+// CHECK-LABEL: define{{.*}} void @f_large(ptr dead_on_unwind noalias writable sret(%struct.large) align 8 %agg.result, ptr dead_on_return noundef %x)
 struct large f_large(struct large x) {
   x.a += *x.b;
   x.b = 0;
diff --git a/clang/test/CodeGen/vectorcall.c b/clang/test/CodeGen/vectorcall.c
index cab7fc0972d7b..09b3310c7c4c8 100644
--- a/clang/test/CodeGen/vectorcall.c
+++ b/clang/test/CodeGen/vectorcall.c
@@ -17,7 +17,7 @@ void __vectorcall v3(int a, struct Small b, int c) {}
 struct Large { int a[5]; };
 void __vectorcall v4(int a, struct Large b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01v4@@28"(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 
 void __vectorcall v5(long long a, int b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01v5@@16"(i64 noundef %a, i32 inreg noundef %b, i32 inreg noundef %c)
@@ -35,21 +35,21 @@ void __vectorcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __vectorcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr inreg noundef %b, double inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr noundef %b, double noundef %c)
+// X86: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, double inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr dead_on_return noundef %b, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
 // SSE register usage.
 void __vectorcall hfa3(double a, double b, double c, double d, double e, struct HFA2 f) {}
-// X86: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double inreg noundef %a, double inreg noundef %b, double inreg noundef %c, double inreg noundef %d, double inreg noundef %e, ptr inreg noundef %f)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double noundef %a, double noundef %b, double noundef %c, double noundef %d, double noundef %e, ptr noundef %f)
+// X86: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double inreg noundef %a, double inreg noundef %b, double inreg noundef %c, double inreg noundef %d, double inreg noundef %e, ptr dead_on_return inreg noundef %f)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double noundef %a, double noundef %b, double noundef %c, double noundef %d, double noundef %e, ptr dead_on_return noundef %f)
 
 // Aggregates with more than four elements are not HFAs and are passed byval.
 // Because they are not classified as homogeneous, they don't get special
 // handling to ensure alignment.
 void __vectorcall hfa4(struct HFA5 a) {}
 // X86: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr noundef byval(%struct.HFA5) align 4 %0)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr noundef %a)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr dead_on_return noundef %a)
 
 // Return HFAs of 4 or fewer elements in registers.
 static struct HFA2 g_hfa2;
@@ -68,26 +68,26 @@ v4f32 __vectorcall hva1(int a, struct HVA4 b, int c) {return b.w;}
 // X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva1@@80"(i32 noundef %a, %struct.HVA4 inreg %b.coerce, i32 noundef %c)
 
 v4f32 __vectorcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {return c;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c)
 
 v4f32 __vectorcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {return f.x;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> inreg noundef %a, <4 x float> inreg noundef %b, <4 x float> inreg noundef %c, <4 x float> inreg noundef %d, <4 x float> inreg noundef %e, ptr inreg noundef %f)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> noundef %a, <4 x float> noundef %b, <4 x float> noundef %c, <4 x float> noundef %d, <4 x float> noundef %e, ptr noundef %f)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> inreg noundef %a, <4 x float> inreg noundef %b, <4 x float> inreg noundef %c, <4 x float> inreg noundef %d, <4 x float> inreg noundef %e, ptr dead_on_return inreg noundef %f)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> noundef %a, <4 x float> noundef %b, <4 x float> noundef %c, <4 x float> noundef %d, <4 x float> noundef %e, ptr dead_on_return noundef %f)
 
 // vector types have higher priority then HVA structures, So vector types are allocated first
 // and HVAs are allocated if enough registers are available
 v4f32 __vectorcall hva4(struct HVA4 a, struct HVA2 b, v4f32 c) {return b.y;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c)
 
 v4f32 __vectorcall hva5(struct HVA3 a, struct HVA3 b, v4f32 c, struct HVA2 d) {return d.y;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c, %struct.HVA2 inreg %d.coerce)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c, %struct.HVA2 inreg %d.coerce)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c, %struct.HVA2 inreg %d.coerce)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c, %struct.HVA2 inreg %d.coerce)
 
 struct HVA4 __vectorcall hva6(struct HVA4 a, struct HVA4 b) { return b;}
-// X86: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b)
-// X64: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr noundef %b)
+// X86: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b)
+// X64: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b)
 
 struct HVA5 __vectorcall hva7(void) {struct HVA5 a = {}; return a;}
 // X86: define dso_local x86_vectorcallcc void @"\01hva7@@0"(ptr dead_on_unwind noalias writable sret(%struct.HVA5) align 16 %agg.result)
@@ -108,8 +108,8 @@ void __vectorcall odd_size_hva(struct OddSizeHVA a) {}
 // consider 'p7' as a register.  Instead p5 gets put into the register on the second pass.
 // x86 should pass p2, p6 and p7 in registers, then p1 in the second pass.
 struct HFA2 __vectorcall AddParticles(struct HFA2 p1, float p2, struct HFA4 p3, int p4, struct HFA2 p5, float p6, float p7, int p8){ return p1;}
-// X86: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@84"(%struct.HFA2 inreg %p1.coerce, float inreg noundef %p2, ptr inreg noundef %p3, i32 inreg noundef %p4, ptr noundef %p5, float inreg noundef %p6, float inreg noundef %p7, i32 noundef %p8)
-// X64: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@104"(%struct.HFA2 inreg %p1.coerce, float noundef %p2, ptr noundef %p3, i32 noundef %p4, %struct.HFA2 inreg %p5.coerce, float noundef %p6, float noundef %p7, i32 noundef %p8)
+// X86: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@84"(%struct.HFA2 inreg %p1.coerce, float inreg noundef %p2, ptr dead_on_return inreg noundef %p3, i32 inreg noundef %p4, ptr dead_on_return noundef %p5, float inreg noundef %p6, float inreg noundef %p7, i32 noundef %p8)
+// X64: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@104"(%struct.HFA2 inreg %p1.coerce, float noundef %p2, ptr dead_on_return noundef %p3, i32 noundef %p4, %struct.HFA2 inreg %p5.coerce, float noundef %p6, float noundef %p7, i32 noundef %p8)
 
 // Vectorcall in both architectures allows passing of an HVA as long as there is room,
 // even if it is not one of the first 6 arguments.  First pass puts p4 into a
@@ -117,8 +117,8 @@ struct HFA2 __vectorcall AddParticles(struct HFA2 p1, float p2, struct HFA4 p3,
 // in a register, does NOT put p7 in a register (since there's no room), then puts
 // p8 in a register.
 void __vectorcall HVAAnywhere(struct HFA2 p1, int p2, int p3, float p4, int p5, int p6, struct HFA4 p7, struct HFA2 p8, float p9){}
-// X86: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@88"(%struct.HFA2 inreg %p1.coerce, i32 inreg noundef %p2, i32 inreg noundef %p3, float inreg noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr noundef %p7, %struct.HFA2 inreg %p8.coerce, float inreg noundef %p9)
-// X64: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@112"(%struct.HFA2 inreg %p1.coerce, i32 noundef %p2, i32 noundef %p3, float noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr noundef %p7, %struct.HFA2 inreg %p8.coerce, float noundef %p9)
+// X86: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@88"(%struct.HFA2 inreg %p1.coerce, i32 inreg noundef %p2, i32 inreg noundef %p3, float inreg noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr dead_on_return noundef %p7, %struct.HFA2 inreg %p8.coerce, float inreg noundef %p9)
+// X64: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@112"(%struct.HFA2 inreg %p1.coerce, i32 noundef %p2, i32 noundef %p3, float noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr dead_on_return noundef %p7, %struct.HFA2 inreg %p8.coerce, float noundef %p9)
 
 #ifndef __x86_64__
 // This covers the three ways XMM values can be passed on 32-bit x86:
@@ -137,9 +137,9 @@ void __vectorcall vectorcall_indirect_vec(
 // X86-SAME: double inreg noundef %xmm3,
 // X86-SAME: double inreg noundef %xmm4,
 // X86-SAME: <4 x float> inreg noundef %xmm5,
-// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: ptr dead_on_return inreg noundef %0,
 // X86-SAME: i32 inreg noundef %edx,
-// X86-SAME: ptr noundef %1)
+// X86-SAME: ptr dead_on_return noundef %1)
 
 void __vectorcall vectorcall_indirect_fp(
     double xmm0, double xmm1, double xmm2, double xmm3, double xmm4,
@@ -153,7 +153,7 @@ void __vectorcall vectorcall_indirect_fp(
 // X86-SAME: double inreg noundef %xmm3,
 // X86-SAME: double inreg noundef %xmm4,
 // X86-SAME: <4 x float> inreg noundef %xmm5,
-// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: ptr dead_on_return inreg noundef %0,
 // X86-SAME: i32 inreg noundef %edx,
 // X86-SAME: double noundef %mem)
 #endif
diff --git a/clang/test/CodeGen/win-fp128.c b/clang/test/CodeGen/win-fp128.c
index 328a7aaa7df57..8d223741bc93e 100644
--- a/clang/test/CodeGen/win-fp128.c
+++ b/clang/test/CodeGen/win-fp128.c
@@ -6,7 +6,7 @@ __float128 fp128_ret(void) { return 0; }
 // CHECK-GNU64: define dso_local <2 x i64>  @fp128_ret()
 
 __float128 fp128_args(__float128 a, __float128 b) { return a * b; }
-// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr noundef %0, ptr noundef %1)
+// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 
 void fp128_vararg(int a, ...) {
   // CHECK-GNU64-LABEL: define dso_local void @fp128_vararg
diff --git a/clang/test/CodeGen/win64-i128.c b/clang/test/CodeGen/win64-i128.c
index e10b2be0530eb..2d83889d8f89b 100644
--- a/clang/test/CodeGen/win64-i128.c
+++ b/clang/test/CodeGen/win64-i128.c
@@ -12,8 +12,8 @@ int128_t foo(void) { return 0; }
 
 int128_t bar(int128_t a, int128_t b) { return a * b; }
 
-// GNU64: define dso_local <2 x i64> @bar(ptr noundef %0, ptr noundef %1)
-// MSC64: define dso_local <2 x i64> @bar(ptr noundef %0, ptr noundef %1)
+// GNU64: define dso_local <2 x i64> @bar(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// MSC64: define dso_local <2 x i64> @bar(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 
 void vararg(int a, ...) {
   // GNU64-LABEL: define{{.*}} void @vararg
diff --git a/clang/test/CodeGen/windows-swiftcall.c b/clang/test/CodeGen/windows-swiftcall.c
index 41569c2606622..8716f25b9ddfb 100644
--- a/clang/test/CodeGen/windows-swiftcall.c
+++ b/clang/test/CodeGen/windows-swiftcall.c
@@ -219,7 +219,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define {{.*}} void @return_struct_big_1({{.*}} dead_on_unwind noalias writable sret
 
 // Should not be byval.
-// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr noundef{{( %.*)?}})
+// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr dead_on_return noundef{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
index 152be26948f28..1709c88563267 100644
--- a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
+++ b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
@@ -576,7 +576,7 @@ void f(__clang_svmfloat8x4_t, __clang_svmfloat8x4_t);
 // CHECK-NEXT:    [[COERCE74_EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 2
 // CHECK-NEXT:    [[COERCE74_EXTRACT3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 3
 // CHECK-NEXT:    store { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[BYVAL_TEMP]], align 2
-// CHECK-NEXT:    call void @_Z1f10svboolx4_tS_(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    call void @_Z1f10svboolx4_tS_(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    store { <vscale x 16 x i8>, <vscale x 16 x i8> } zeroinitializer, ptr [[COERCE75]], align 16
 // CHECK-NEXT:    [[COERCE75_TUPLE:%.*]] = load { <vscale x 16 x i8>, <vscale x 16 x i8> }, ptr [[COERCE75]], align 16
 // CHECK-NEXT:    [[COERCE75_EXTRACT0:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[COERCE75_TUPLE]], 0
@@ -1125,7 +1125,7 @@ void f(__clang_svmfloat8x4_t, __clang_svmfloat8x4_t);
 // COMPAT_17-NEXT:    [[COERCE74_EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 2
 // COMPAT_17-NEXT:    [[COERCE74_EXTRACT3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 3
 // COMPAT_17-NEXT:    store { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[BYVAL_TEMP]], align 2
-// COMPAT_17-NEXT:    call void @_Z1f10svboolx4_t10svboolx4_t(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr noundef [[BYVAL_TEMP]])
+// COMPAT_17-NEXT:    call void @_Z1f10svboolx4_t10svboolx4_t(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // COMPAT_17-NEXT:    store { <vscale x 16 x i8>, <vscale x 16 x i8> } zeroinitializer, ptr [[COERCE75]], align 16
 // COMPAT_17-NEXT:    [[COERCE75_TUPLE:%.*]] = load { <vscale x 16 x i8>, <vscale x 16 x i8> }, ptr [[COERCE75]], align 16
 // COMPAT_17-NEXT:    [[COERCE75_EXTRACT0:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[COERCE75_TUPLE]], 0
diff --git a/clang/test/CodeGenCXX/arm-cc.cpp b/clang/test/CodeGenCXX/arm-cc.cpp
index 68e1b7e4e1e46..939615fcc69e0 100644
--- a/clang/test/CodeGenCXX/arm-cc.cpp
+++ b/clang/test/CodeGenCXX/arm-cc.cpp
@@ -17,4 +17,4 @@ void baz() {
 }
 
 // CHECK: declare void @_Z3fooPv(ptr dead_on_unwind writable sret(%class.SMLoc) align 4, ptr noundef)
-// CHECK: declare void @_Z3zed5SMLoc(ptr noundef)
+// CHECK: declare void @_Z3zed5SMLoc(ptr dead_on_return noundef)
diff --git a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
index c341bd2b855ff..f2d602b3b523e 100644
--- a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
+++ b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
@@ -55,18 +55,18 @@ void usage() {
 // WINDOWS-NEXT: ret i32 %[[RET]]
 
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: ret i32 %[[LOAD]]
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 1
 // WINDOWS64: ret i32 %[[ADD]]
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 2
@@ -75,7 +75,7 @@ void usage() {
 // WINDOWS64: define dso_local void @"?usage@@YAXXZ"()
 // WINDOWS64: %[[F:[0-9a-zA-Z]+]] = alloca %struct.Foo
 // WINDOWS64: %[[ARG:[0-9a-zA-Z.]+]] = alloca %struct.Foo
-// WINDOWS64: %[[CALL:[0-9a-zA-Z]+]] = call noundef i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr noundef %[[ARG]])
+// WINDOWS64: %[[CALL:[0-9a-zA-Z]+]] = call noundef i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr dead_on_return noundef %[[ARG]])
 
 // WINDOWS64: define weak_odr dso_local i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr %0)
 // WINDOWS64: %[[RET:[0-9a-zA-Z]+]] = musttail call i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr %0)
diff --git a/clang/test/CodeGenCXX/copy-initialization.cpp b/clang/test/CodeGenCXX/copy-initialization.cpp
index aa0c6395f158d..4e6194cc040c2 100644
--- a/clang/test/CodeGenCXX/copy-initialization.cpp
+++ b/clang/test/CodeGenCXX/copy-initialization.cpp
@@ -12,7 +12,7 @@ struct Bar {
 
 void f(Foo);
 
-// CHECK-LABEL: define{{.*}} void @_Z1g3Foo(ptr noundef %foo)
+// CHECK-LABEL: define{{.*}} void @_Z1g3Foo(ptr dead_on_return noundef %foo)
 void g(Foo foo) {
   // CHECK: call void @_ZN3BarC1Ev
   // CHECK: @_ZNK3BarcvRK3FooEv
diff --git a/clang/test/CodeGenCXX/debug-info.cpp b/clang/test/CodeGenCXX/debug-info.cpp
index 8594a897ef7c0..9cf26ba83ba3e 100644
--- a/clang/test/CodeGenCXX/debug-info.cpp
+++ b/clang/test/CodeGenCXX/debug-info.cpp
@@ -4,7 +4,7 @@
 // CHECK: @_ZN6pr96081xE ={{.*}} global ptr null, align 8, !dbg [[X:![0-9]+]]
 
 // CHECK: define{{.*}} void @_ZN7pr147634funcENS_3fooE
-// CHECK-SAME: ptr noundef [[param:%.*]])
+// CHECK-SAME: ptr dead_on_return noundef [[param:%.*]])
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   alloca ptr, align 8
 // CHECK-NEXT:   [[param_addr_storage:%.*]] = alloca ptr, align 8
diff --git a/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp b/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
index c7d3a017414ef..c8f5a0f7c2ea6 100644
--- a/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
+++ b/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
@@ -13,7 +13,7 @@ struct Empty {
 };
 
 bool foo(Empty e) {
-// CHECK: @_Z3foo5Empty(ptr noundef %e)
+// CHECK: @_Z3foo5Empty(ptr dead_on_return noundef %e)
 // CHECK: call {{.*}} @_ZN5Empty5checkEv(ptr {{[^,]*}} %e)
   return e.check();
 }
@@ -21,6 +21,6 @@ bool foo(Empty e) {
 void caller(Empty &e) {
 // CHECK: @_Z6callerR5Empty(ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %e)
 // CHECK: call {{.*}} @_ZN5EmptyC1ERKS_(ptr {{[^,]*}} [[NEWTMP:%.*]], ptr
-// CHECK: call {{.*}} @_Z3foo5Empty(ptr noundef [[NEWTMP]])
+// CHECK: call {{.*}} @_Z3foo5Empty(ptr dead_on_return noundef [[NEWTMP]])
   foo(e);
 }
diff --git a/clang/test/CodeGenCXX/fastcall.cpp b/clang/test/CodeGenCXX/fastcall.cpp
index 4c94c1623ee16..405917f7a14bb 100644
--- a/clang/test/CodeGenCXX/fastcall.cpp
+++ b/clang/test/CodeGenCXX/fastcall.cpp
@@ -15,6 +15,6 @@ struct S1 {
 void __attribute__((fastcall)) foo2(S1 a, int b);
 void bar2(S1 a, int b) {
   // CHECK-LABEL: define{{.*}} void @_Z4bar22S1i
-  // CHECK: call x86_fastcallcc void @_Z4foo22S1i(ptr inreg %{{.*}}, i32 inreg %
+  // CHECK: call x86_fastcallcc void @_Z4foo22S1i(ptr dead_on_return inreg %{{.*}}, i32 inreg %
   foo2(a, b);
 }
diff --git a/clang/test/CodeGenCXX/homogeneous-aggregates.cpp b/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
index 63ffc6b5bfac8..5ebeb8aad4c18 100644
--- a/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
+++ b/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
@@ -41,8 +41,8 @@ struct D5 : I1, I2, I3 {}; // homogeneous aggregate
 
 // PPC: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, [3 x i64] %x.coerce)
 // ARM32: define{{.*}} arm_aapcs_vfpcc void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, [3 x i64] %x.coerce)
-// ARM64: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr noundef %x)
-// X64: define dso_local x86_vectorcallcc void @"\01_Z7func_D12D1@@24"(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr noundef %x)
+// ARM64: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr dead_on_return noundef %x)
+// X64: define dso_local x86_vectorcallcc void @"\01_Z7func_D12D1@@24"(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr dead_on_return noundef %x)
 D1 CC func_D1(D1 x) { return x; }
 
 // PPC: define{{.*}} [3 x double] @_Z7func_D22D2([3 x double] %x.coerce)
@@ -53,7 +53,7 @@ D2 CC func_D2(D2 x) { return x; }
 
 // PPC: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, [4 x i64] %x.coerce)
 // ARM32: define{{.*}} arm_aapcs_vfpcc void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, [4 x i64] %x.coerce)
-// ARM64: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, ptr noundef %x)
+// ARM64: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, ptr dead_on_return noundef %x)
 D3 CC func_D3(D3 x) { return x; }
 
 // PPC: define{{.*}} [4 x double] @_Z7func_D42D4([4 x double] %x.coerce)
@@ -201,7 +201,7 @@ struct NonHFA {
   virtual void f1();
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at polymorphic@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at polymorphic@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace trivial_copy_assignment {
 struct HFA {
@@ -221,7 +221,7 @@ struct NonHFA {
   NonHFA &operator=(const NonHFA&);
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at non_trivial_copy_assignment@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at non_trivial_copy_assignment@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace user_provided_ctor {
 struct HFA {
@@ -251,7 +251,7 @@ struct NonHFA {
   ~NonHFA();
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at non_trivial_dtor@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at non_trivial_dtor@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace non_empty_base {
 struct non_empty_base { double d; };
@@ -272,7 +272,7 @@ struct NonHFA {
   empty e;
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at empty_field@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at empty_field@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace non_empty_field {
 struct non_empty { double d; };
diff --git a/clang/test/CodeGenCXX/inalloca-lambda.cpp b/clang/test/CodeGenCXX/inalloca-lambda.cpp
index dc78aa2773f89..0d527e1a02e5f 100644
--- a/clang/test/CodeGenCXX/inalloca-lambda.cpp
+++ b/clang/test/CodeGenCXX/inalloca-lambda.cpp
@@ -22,7 +22,7 @@ void test() {
 // CHECK: %[[V:.*]] = getelementptr inbounds nuw <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0
 // CHECK: %call = call x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %this, ptr noundef %[[V]])
+// CHECK-SAME: (ptr noundef %this, ptr dead_on_return noundef %[[V]])
 
 // CHECK: define internal noundef i32
 // CHECK-SAME: @"?__invoke@<lambda_0>@?0??test@@YAXXZ at CA?A?<auto>@@UA@@@Z"
@@ -31,12 +31,12 @@ void test() {
 // CHECK: %[[VAR:.*]] = getelementptr inbounds nuw <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0
 // CHECK: %call = call x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %unused.capture, ptr noundef %[[VAR]])
+// CHECK-SAME: (ptr noundef %unused.capture, ptr dead_on_return noundef %[[VAR]])
 // CHECK: ret i32 %call
 
 // CHECK: define internal x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %this, ptr noundef %[[ARG:.*]])
+// CHECK-SAME: (ptr noundef %this, ptr dead_on_return noundef %[[ARG:.*]])
 // CHECK: %this.addr = alloca ptr, align 4
 // CHECK: store ptr %this, ptr %this.addr, align 4
 // CHECK: %this1 = load ptr, ptr %this.addr, align 4
diff --git a/clang/test/CodeGenCXX/inalloca-overaligned.cpp b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
index 3751751ec0bca..305b8c5dccd10 100644
--- a/clang/test/CodeGenCXX/inalloca-overaligned.cpp
+++ b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
@@ -57,7 +57,7 @@ int receive_both(Both o) {
 }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?receive_both@@Y{{.*}}"
-// CHECK-SAME: (ptr noundef %o)
+// CHECK-SAME: (ptr dead_on_return noundef %o)
 
 int pass_both() {
   gvi32 = receive_both(Both());
@@ -67,7 +67,7 @@ int pass_both() {
 // CHECK-LABEL: define dso_local noundef i32 @"?pass_both@@Y{{.*}}"
 // CHECK: [[TMP:%[^ ]*]] = alloca %struct.Both, align 8
 // CHECK: call x86_thiscallcc noundef ptr @"??0Both@@QAE at XZ"(ptr {{[^,]*}} [[TMP]])
-// CHECK: call noundef i32 @"?receive_both@@Y{{.*}}"(ptr noundef [[TMP]])
+// CHECK: call noundef i32 @"?receive_both@@Y{{.*}}"(ptr dead_on_return noundef [[TMP]])
 
 int receive_inalloca_both(NonTrivial nt, Both o) {
   return nt.x + o.x + o.y;
@@ -101,11 +101,11 @@ struct [[trivial_abi]] alignas(8) MyPtr {
 int receiveMyPtr(MyPtr o) { return *o.ptr; }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?receiveMyPtr@@Y{{.*}}"
-// CHECK-SAME: (ptr noundef %o)
+// CHECK-SAME: (ptr dead_on_return noundef %o)
 
 int passMyPtr() { return receiveMyPtr(MyPtr()); }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?passMyPtr@@Y{{.*}}"
 // CHECK: [[TMP:%[^ ]*]] = alloca %struct.MyPtr, align 8
 // CHECK: call x86_thiscallcc noundef ptr @"??0MyPtr@@QAE at XZ"(ptr {{[^,]*}} [[TMP]])
-// CHECK: call noundef i32 @"?receiveMyPtr@@Y{{.*}}"(ptr noundef [[TMP]])
+// CHECK: call noundef i32 @"?receiveMyPtr@@Y{{.*}}"(ptr dead_on_return noundef [[TMP]])
diff --git a/clang/test/CodeGenCXX/inalloca-vector.cpp b/clang/test/CodeGenCXX/inalloca-vector.cpp
index d1bacb4f0dc8c..2db4c49df116a 100644
--- a/clang/test/CodeGenCXX/inalloca-vector.cpp
+++ b/clang/test/CodeGenCXX/inalloca-vector.cpp
@@ -56,7 +56,7 @@ void __fastcall fastcall_receive_vec(__m128 x, __m128 y, __m128 z, __m128 w, int
 // CHECK-SAME: (<4 x float> inreg noundef %x,
 // CHECK-SAME: <4 x float> inreg noundef %y,
 // CHECK-SAME: <4 x float> inreg noundef %z,
-// CHECK-SAME: ptr inreg noundef %0,
+// CHECK-SAME: ptr dead_on_return inreg noundef %0,
 // CHECK-SAME: i32 inreg noundef %edx,
 // CHECK-SAME: ptr inalloca(<{ ptr, %struct.NonTrivial }>) %1)
 
@@ -73,6 +73,6 @@ void __vectorcall vectorcall_receive_vec(double xmm0, double xmm1, double xmm2,
 // CHECK-SAME: <4 x float> inreg noundef %x,
 // CHECK-SAME: <4 x float> inreg noundef %y,
 // CHECK-SAME: <4 x float> inreg noundef %z,
-// CHECK-SAME: ptr inreg noundef %0,
+// CHECK-SAME: ptr dead_on_return inreg noundef %0,
 // CHECK-SAME: i32 inreg noundef %edx,
 // CHECK-SAME: ptr inalloca(<{ ptr, %struct.NonTrivial }>) %1)
diff --git a/clang/test/CodeGenCXX/inheriting-constructor.cpp b/clang/test/CodeGenCXX/inheriting-constructor.cpp
index 100ca269d7f3c..21751bea055dc 100644
--- a/clang/test/CodeGenCXX/inheriting-constructor.cpp
+++ b/clang/test/CodeGenCXX/inheriting-constructor.cpp
@@ -166,7 +166,7 @@ namespace inalloca_nonvirt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
-  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
 
@@ -202,7 +202,7 @@ namespace inalloca_nonvirt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
-  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
 }
@@ -253,7 +253,7 @@ namespace inalloca_virt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: br i1
-  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: br
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
@@ -302,7 +302,7 @@ namespace inalloca_virt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: br i1
   // WIN64: store {{.*}} @"??_8C at inalloca_virt@@7B@"
-  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
diff --git a/clang/test/CodeGenCXX/member-function-pointer-calls.cpp b/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
index ff511c0243801..f06cda8b7684e 100644
--- a/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
+++ b/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
@@ -16,7 +16,7 @@ int f(A* a, int (A::*fp)()) {
 // CHECK-NOT: }
 // CHECK: ret i32 1
 // MINGW64-LABEL: define dso_local noundef i32 @_Z2g1v()
-// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr noundef %{{.*}})
+// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr dead_on_return noundef %{{.*}})
 int g1() {
   A a;
   return f(&a, &A::vf1);
@@ -26,7 +26,7 @@ int g1() {
 // CHECK-NOT: }
 // CHECK: ret i32 2
 // MINGW64-LABEL: define dso_local noundef i32 @_Z2g2v()
-// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr noundef %{{.*}})
+// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr dead_on_return noundef %{{.*}})
 int g2() {
   A a;
   return f(&a, &A::vf2);
diff --git a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
index b551df747c073..63a4d5525336b 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
@@ -24,7 +24,7 @@ void foo(A a, A b, A c) {
 // X86: ret void
 
 // X64-LABEL: define dso_local void @"?foo@@YAXUA@@00 at Z"
-// X64:         (ptr noundef %[[a:[^,]*]], ptr noundef %[[b:[^,]*]], ptr noundef %[[c:[^)]*]])
+// X64:         (ptr dead_on_return noundef %[[a:[^,]*]], ptr dead_on_return noundef %[[b:[^,]*]], ptr dead_on_return noundef %[[c:[^)]*]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[a]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[b]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[c]])
@@ -64,7 +64,7 @@ void call_foo() {
 // X64: invoke noundef ptr @"??0A@@QEAA at H@Z"(ptr {{[^,]*}} %[[arg2:[^,]*]], i32 noundef 2)
 // X64: invoke noundef ptr @"??0A@@QEAA at H@Z"(ptr {{[^,]*}} %[[arg1:[^,]*]], i32 noundef 1)
 // X64: call void @"?foo@@YAXUA@@00 at Z"
-// X64:       (ptr noundef %[[arg1]], ptr noundef %[[arg2]], ptr noundef %[[arg3]])
+// X64:       (ptr dead_on_return noundef %[[arg1]], ptr dead_on_return noundef %[[arg2]], ptr dead_on_return noundef %[[arg3]])
 // X64: ret void
 //
 //   lpad2:
diff --git a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
index 5654db3ba8151..813abb03a7810 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
@@ -22,10 +22,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret void
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@byval_thunk@@W7EAAXUAgg at 2@@Z"
-// CHECK64:             (ptr noundef %this, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@byval_thunk@@UEAAXUAgg at 2@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
@@ -54,10 +54,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret void
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@stdcall_thunk@@W7EAAXUAgg at 2@@Z"
-// CHECK64:             (ptr noundef %this, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@stdcall_thunk@@UEAAXUAgg at 2@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
@@ -86,10 +86,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret ptr %[[rv]]
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@sret_thunk@@W7EAA?AUAgg at 2@U32@@Z"
-// CHECK64:             (ptr noundef %this, ptr dead_on_unwind noalias writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_unwind noalias writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@sret_thunk@@UEAA?AUAgg at 2@U32@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
diff --git a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
index 806bc5b63ef02..8defb68c668b2 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
@@ -428,7 +428,7 @@ bool nullTestDataUnspecified(int Unspecified::*mp) {
 
 // Pass this large type indirectly.
 // X64-LABEL: define dso_local noundef zeroext i1 @"?nullTestDataUnspecified@@
-// X64:             (ptr noundef %0)
+// X64:             (ptr dead_on_return noundef %0)
 }
 
 bool nullTestFunctionUnspecified(void (Unspecified::*mp)()) {
@@ -590,7 +590,7 @@ bool unspecFuncMemptrEq(void (Unspecified::*l)(), void (Unspecified::*r)()) {
 // CHECK: }
 
 // X64-LABEL: define dso_local noundef zeroext i1 @"?unspecFuncMemptrEq@@
-// X64:             (ptr noundef %0, ptr noundef %1)
+// X64:             (ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 }
 
 bool unspecFuncMemptrNeq(void (Unspecified::*l)(), void (Unspecified::*r)()) {
@@ -635,7 +635,7 @@ bool unspecDataMemptrEq(int Unspecified::*l, int Unspecified::*r) {
 // CHECK: }
 
 // X64-LABEL: define dso_local noundef zeroext i1 @"?unspecDataMemptrEq@@
-// X64:             (ptr noundef %0, ptr noundef %1)
+// X64:             (ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 }
 
 void (Multiple::*convertB2FuncToMultiple(void (B2::*mp)()))() {
diff --git a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
index 6e8ba3953b2cf..767bf168633ae 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
@@ -175,7 +175,7 @@ void multibyte_arg(Multibyte s) {}
 void packed_arg(Packed s) {}
 // LINUX-LABEL: define{{.*}} void @_Z10packed_arg6Packed(ptr noundef byval(%struct.Packed) align 4 %s)
 // WIN32: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr noundef byval(%struct.Packed) align 4 %s)
-// WIN64: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr dead_on_return noundef %s)
 
 // Test that dtors are invoked in the callee.
 void small_arg_with_dtor(SmallWithDtor s) {}
@@ -190,7 +190,7 @@ void small_arg_with_dtor(SmallWithDtor s) {}
 // WOA64: }
 
 // FIXME: MSVC incompatible!
-// WOA: define dso_local arm_aapcs_vfpcc void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(ptr noundef %s) {{.*}} {
+// WOA: define dso_local arm_aapcs_vfpcc void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(ptr dead_on_return noundef %s) {{.*}} {
 // WOA:   call arm_aapcs_vfpcc void @"??1SmallWithDtor@@QAA at XZ"(ptr {{[^,]*}} %s)
 // WOA: }
 
@@ -220,7 +220,7 @@ void ref_small_arg_with_dtor(const SmallWithDtor &s) { }
 // WIN64-LABEL: define dso_local void @"?ref_small_arg_with_dtor@@YAXAEBUSmallWithDtor@@@Z"(ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %s)
 
 void big_arg_with_dtor(BigWithDtor s) {}
-// WIN64-LABEL: define dso_local void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr noundef %s)
+// WIN64-LABEL: define dso_local void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr dead_on_return noundef %s)
 // WIN64:   call void @"??1BigWithDtor@@QEAA at XZ"
 // WIN64: }
 
@@ -231,7 +231,7 @@ void call_big_arg_with_dtor() {
 // larger than 8 bytes and is passed indirectly.
 // WIN64-LABEL: define dso_local void @"?call_big_arg_with_dtor@@YAXXZ"()
 // WIN64:   call noundef ptr @"??0BigWithDtor@@QEAA at XZ"
-// WIN64:   call void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr noundef %{{.*}})
+// WIN64:   call void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr dead_on_return noundef %{{.*}})
 // WIN64-NOT: call void @"??1BigWithDtor@@QEAA at XZ"
 // WIN64:   ret void
 
@@ -259,22 +259,22 @@ void eh_cleanup_arg_with_dtor() {
 // WIN32: }
 
 void small_arg_with_vftable(SmallWithVftable s) {}
-// LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(ptr noundef %s)
+// LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(ptr dead_on_return noundef %s)
 // WIN32: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr inalloca(<{ %struct.SmallWithVftable }>) %0)
-// WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr noundef %s)
-// WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr dead_on_return noundef %s)
+// WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr dead_on_return noundef %s)
 
 void medium_arg_with_copy_ctor(MediumWithCopyCtor s) {}
-// LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(ptr noundef %s)
+// LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(ptr dead_on_return noundef %s)
 // WIN32: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr inalloca(<{ %struct.MediumWithCopyCtor }>) %0)
-// WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
-// WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
-// WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
+// WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
+// WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
 
 void big_arg(Big s) {}
 // LINUX-LABEL: define{{.*}} void @_Z7big_arg3Big(ptr noundef byval(%struct.Big) align 4 %s)
 // WIN32: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr noundef byval(%struct.Big) align 4 %s)
-// WIN64: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr dead_on_return noundef %s)
 
 // PR27607: We would attempt to load i32 value out of the reference instead of
 // just loading the pointer from the struct during argument expansion.
@@ -346,7 +346,7 @@ class Class {
   void thiscall_method_arg(Big s) {}
   // LINUX: define {{.*}} void @_ZN5Class19thiscall_method_argE3Big(ptr {{[^,]*}} %this, ptr noundef byval(%struct.Big) align 4 %s)
   // WIN32: define {{.*}} void @"?thiscall_method_arg at Class@@QAEXUBig@@@Z"(ptr {{[^,]*}} %this, ptr noundef byval(%struct.Big) align 4 %s)
-  // WIN64: define linkonce_odr dso_local void @"?thiscall_method_arg at Class@@QEAAXUBig@@@Z"(ptr {{[^,]*}} %this, ptr noundef %s)
+  // WIN64: define linkonce_odr dso_local void @"?thiscall_method_arg at Class@@QEAAXUBig@@@Z"(ptr {{[^,]*}} %this, ptr dead_on_return noundef %s)
 };
 
 void use_class() {
diff --git a/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp b/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
index 9e37e71e257fd..b7653632cf882 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
@@ -18,4 +18,4 @@ A B::foo(A x) {
   return x;
 }
 
-// CHECK-LABEL: define{{.*}} void @"?foo at B@@QEAA?AUA@@U2@@Z"(ptr {{[^,]*}} %this, ptr dead_on_unwind noalias writable sret(%struct.A) align 4 %agg.result, ptr noundef %x)
+// CHECK-LABEL: define{{.*}} void @"?foo at B@@QEAA?AUA@@U2@@Z"(ptr {{[^,]*}} %this, ptr dead_on_unwind noalias writable sret(%struct.A) align 4 %agg.result, ptr dead_on_return noundef %x)
diff --git a/clang/test/CodeGenCXX/ms-property.cpp b/clang/test/CodeGenCXX/ms-property.cpp
index 744de224b2f9a..d9fbf46dacb25 100644
--- a/clang/test/CodeGenCXX/ms-property.cpp
+++ b/clang/test/CodeGenCXX/ms-property.cpp
@@ -105,7 +105,7 @@ int main(int argc, char **argv) {
   // CHECK: [[ARGC:%.+]] = load i32, ptr %
   // CHECK: [[P1_X_ARGC_0:%.+]] = call noundef i32 @"?GetX at S@@QEAAHHH at Z"(ptr {{[^,]*}} [[P1]], i32 noundef [[ARGC]], i32 noundef 0)
   // CHECK: [[CAST:%.+]] = trunc i32 [[P1_X_ARGC_0]] to i8
-  // CHECK: [[P2_Y_p1_X_ARGC_0_T:%.+]] = call noundef i8 @"?GetY@?$St at M@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[P2_2]], i8 noundef [[CAST]], ptr noundef %{{.+}})
+  // CHECK: [[P2_Y_p1_X_ARGC_0_T:%.+]] = call noundef i8 @"?GetY@?$St at M@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[P2_2]], i8 noundef [[CAST]], ptr dead_on_return noundef %{{.+}})
   // CHECK: [[CAST:%.+]] = sitofp i8 [[P2_Y_p1_X_ARGC_0_T]] to float
   // CHECK: [[J:%.+]] = load i32, ptr %
   // CHECK: [[CAST1:%.+]] = sitofp i32 [[J]] to float
@@ -124,6 +124,6 @@ int main(int argc, char **argv) {
 // CHECK: call noundef i32 @"?GetX@?$St at H@@QEAAHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}} i32 noundef %{{.+}})
 // CHECK: call void @"?PutY@?$St at H@@QEAAXDHN at Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, i32 noundef %{{.+}}, double noundef %{{.+}}
 // CHECK: call noundef i32 @"?GetX@?$St at H@@QEAAHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}} i32 noundef %{{.+}})
-// CHECK: call noundef i8 @"?GetY@?$St at H@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, ptr noundef %{{.+}})
+// CHECK: call noundef i8 @"?GetY@?$St at H@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, ptr dead_on_return noundef %{{.+}})
 // CHECK: call noundef i32 @"?PutX@?$St at H@@QEAAHHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}}, i32 noundef %{{.+}}, i32 noundef %{{.+}})
 #endif //HEADER
diff --git a/clang/test/CodeGenCXX/nrvo.cpp b/clang/test/CodeGenCXX/nrvo.cpp
index 1141bc35de582..5b0fc914120e3 100644
--- a/clang/test/CodeGenCXX/nrvo.cpp
+++ b/clang/test/CodeGenCXX/nrvo.cpp
@@ -2197,7 +2197,7 @@ void test16() { // http://wg21.link/p2025r2#ex-9
 // CHECK-EH-11-NEXT:    br i1 [[CMP9]], label [[IF_THEN10:%.*]], label [[IF_END11:%.*]]
 // CHECK-EH-11:       if.then10:
 // CHECK-EH-11-NEXT:    store i32 3, ptr [[CLEANUP_DEST_SLOT]], align 4
-// CHECK-EH-11-NEXT:    br label [[CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK-EH-11-NEXT:    br label [[CLEANUP]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK-EH-11:       if.end11:
 // CHECK-EH-11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-EH-11-NEXT:    [[CMP12:%.*]] = icmp eq i32 [[TMP4]], 3
@@ -2239,7 +2239,7 @@ void test16() { // http://wg21.link/p2025r2#ex-9
 // CHECK-EH-11-NEXT:      i32 2, label [[IMPOSSIBLE]]
 // CHECK-EH-11-NEXT:    ]
 // CHECK-EH-11:       cleanup.cont:
-// CHECK-EH-11-NEXT:    br label [[WHILE_BODY]], !llvm.loop [[LOOP3]]
+// CHECK-EH-11-NEXT:    br label [[WHILE_BODY]], !llvm.loop [[LOOP4]]
 // CHECK-EH-11:       while.end:
 // CHECK-EH-11-NEXT:    call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]])
 // CHECK-EH-11-NEXT:    br label [[RETURN]]
diff --git a/clang/test/CodeGenCXX/pass-by-value-noalias.cpp b/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
index 773cf6b81c3b2..947379d9b8b92 100644
--- a/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
+++ b/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
@@ -14,8 +14,8 @@ struct Foo {
 // Make sure noalias is added to indirect arguments with trivially copyable types
 // if -fpass-by-value-is-noalias is provided.
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr dead_on_return noundef %arg)
 void take(Foo arg) {}
 
 int G;
@@ -38,8 +38,8 @@ struct NonTrivial {
 // Make sure noalias is not added to indirect arguments that are not trivially
 // copyable even if -fpass-by-value-is-noalias is provided.
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr noundef %arg)
-// NO_NOALIAS:   define{{.*}} void @_Z4take10NonTrivial(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr dead_on_return noundef %arg)
+// NO_NOALIAS:   define{{.*}} void @_Z4take10NonTrivial(ptr dead_on_return noundef %arg)
 void take(NonTrivial arg) {}
 
 // Escape examples. Pointers to the objects passed to take() may escape, depending on whether a temporary copy is created or not (e.g. due to NRVO).
@@ -54,8 +54,8 @@ struct A {
 };
 A *p;
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take1A(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take1A(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take1A(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take1A(ptr dead_on_return noundef %arg)
 void take(A arg) {}
 
 // WITH_NOALIAS: define{{.*}} void @_Z7CreateAPP1A(ptr dead_on_unwind noalias writable sret(%struct.A) align 1 %agg.result, ptr noundef %where)
diff --git a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
index daeea77774ec8..0310535362e3d 100644
--- a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
@@ -24,7 +24,7 @@ struct __attribute__((trivial_abi)) TrivialSA {
 // Check that TrivialSA is passed indirectly despite being annotated with
 // 'trivial_abi'.
 
-// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr noundef %{{.*}})
+// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr dead_on_return noundef %{{.*}})
 
 void testParamTrivialSA(TrivialSA a) {
 }
diff --git a/clang/test/CodeGenCXX/regparm.cpp b/clang/test/CodeGenCXX/regparm.cpp
index b9735485db8de..a31394016fb55 100644
--- a/clang/test/CodeGenCXX/regparm.cpp
+++ b/clang/test/CodeGenCXX/regparm.cpp
@@ -11,7 +11,7 @@ struct S1 {
 };
 
 void __attribute__((regparm(3))) foo2(S1 a, int b);
-// CHECK: declare void @_Z4foo22S1i(ptr inreg noundef, i32 inreg noundef)
+// CHECK: declare void @_Z4foo22S1i(ptr dead_on_return inreg noundef, i32 inreg noundef)
 void bar2(S1 a, int b) {
   foo2(a, b);
 }
diff --git a/clang/test/CodeGenCXX/trivial_abi.cpp b/clang/test/CodeGenCXX/trivial_abi.cpp
index b8cc0d1cc6528..eacbde594e517 100644
--- a/clang/test/CodeGenCXX/trivial_abi.cpp
+++ b/clang/test/CodeGenCXX/trivial_abi.cpp
@@ -140,7 +140,7 @@ void testIgnoredSmall() {
   testReturnSmall();
 }
 
-// CHECK: define{{.*}} void @_Z14testParamLarge5Large(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5LargeD1Ev(ptr {{[^,]*}} %[[A]])
 // CHECK: ret void
 // CHECK: }
@@ -163,7 +163,7 @@ Large testReturnLarge() {
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_LARGE]], align 8
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[T]])
 // CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN5LargeC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(520) %[[T]])
-// CHECK: call void @_Z14testParamLarge5Large(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: %[[CALL2:.*]] = call noundef ptr @_ZN5LargeD1Ev(ptr {{[^,]*}} %[[T]])
 // CHECK: ret void
 // CHECK: }
@@ -176,7 +176,7 @@ void testCallLarge0() {
 // CHECK: define{{.*}} void @_Z14testCallLarge1v()
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_LARGE:.*]], align 8
 // CHECK: call void @_Z15testReturnLargev(ptr dead_on_unwind writable sret(%[[STRUCT_LARGE]]) align 8 %[[AGG_TMP]])
-// CHECK: call void @_Z14testParamLarge5Large(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: ret void
 // CHECK: }
 
@@ -244,7 +244,7 @@ void testExceptionSmall() {
 // CHECK: call noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[AGG_TMP]])
 // CHECK: invoke noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[AGG_TMP1]])
 
-// CHECK: call void @_Z20calleeExceptionLarge5LargeS_(ptr noundef %[[AGG_TMP]], ptr noundef %[[AGG_TMP1]])
+// CHECK: call void @_Z20calleeExceptionLarge5LargeS_(ptr dead_on_return noundef %[[AGG_TMP]], ptr dead_on_return noundef %[[AGG_TMP1]])
 // CHECK-NEXT: ret void
 
 // CHECK: landingpad { ptr, i32 }
diff --git a/clang/test/CodeGenCXX/uncopyable-args.cpp b/clang/test/CodeGenCXX/uncopyable-args.cpp
index 31192b65cc362..2d09732f9d3c2 100644
--- a/clang/test/CodeGenCXX/uncopyable-args.cpp
+++ b/clang/test/CodeGenCXX/uncopyable-args.cpp
@@ -59,12 +59,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN9move_ctor3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN9move_ctor3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN9move_ctor3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN9move_ctor3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at move_ctor@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at move_ctor@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace all_deleted {
@@ -81,12 +81,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN11all_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN11all_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN11all_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN11all_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at all_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at all_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace implicitly_deleted {
@@ -102,14 +102,14 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN18implicitly_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr)
 
 // In MSVC 2013, the copy ctor is not deleted by a move assignment. In MSVC 2015, it is.
 // WIN64-18-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(i64
-// WIN64-19-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-19-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace one_deleted {
@@ -125,12 +125,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN11one_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN11one_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN11one_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN11one_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at one_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at one_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace copy_defaulted {
@@ -170,7 +170,7 @@ void bar() {
 // CHECK: call void @_ZN14move_defaulted3fooENS_1AE(ptr %{{.*}})
 // CHECK-LABEL: declare void @_ZN14move_defaulted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at move_defaulted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at move_defaulted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace trivial_defaulted {
@@ -207,12 +207,12 @@ void bar() {
 }
 // CHECK-LABEL: define{{.*}} void @_ZN14two_copy_ctors3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
-// NEWABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef byval
-// NEWABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef byval
 
-// WIN64-LABEL: declare dso_local void @"?foo at two_copy_ctors@@YAXUB at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at two_copy_ctors@@YAXUB at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace definition_only {
diff --git a/clang/test/CodeGenCXX/wasm-args-returns.cpp b/clang/test/CodeGenCXX/wasm-args-returns.cpp
index fbb152ac1bb3d..7b1c27bf0808d 100644
--- a/clang/test/CodeGenCXX/wasm-args-returns.cpp
+++ b/clang/test/CodeGenCXX/wasm-args-returns.cpp
@@ -46,17 +46,17 @@ struct copy_ctor {
   copy_ctor(copy_ctor const &);
 };
 test(copy_ctor);
-// CHECK: define void @_Z7forward9copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.copy_ctor) align 8 %{{.*}}, ptr nonnull %{{.*}})
+// CHECK: define void @_Z7forward9copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.copy_ctor) align 8 %{{.*}}, ptr dead_on_return nonnull %{{.*}})
 //
 // CHECK: declare ptr @_ZN9copy_ctorC1ERKS_(ptr {{[^,]*}} returned {{[^,]*}}, ptr nonnull align 8 dereferenceable(8))
 //
 // CHECK: define void @_Z14test_copy_ctorv()
 // CHECK: %[[tmp:.*]] = alloca %struct.copy_ctor, align 8
 // CHECK: call void @_Z13def_copy_ctorv(ptr dead_on_unwind nonnull writable sret(%struct.copy_ctor) align 8 %[[tmp]])
-// CHECK: call void @_Z3use9copy_ctor(ptr nonnull %[[tmp]])
+// CHECK: call void @_Z3use9copy_ctor(ptr dead_on_return nonnull %[[tmp]])
 // CHECK: ret void
 //
-// CHECK: declare void @_Z3use9copy_ctor(ptr)
+// CHECK: declare void @_Z3use9copy_ctor(ptr dead_on_return)
 // CHECK: declare void @_Z13def_copy_ctorv(ptr dead_on_unwind writable sret(%struct.copy_ctor) align 8)
 
 struct __attribute__((aligned(16))) aligned_copy_ctor {
@@ -64,17 +64,17 @@ struct __attribute__((aligned(16))) aligned_copy_ctor {
   aligned_copy_ctor(aligned_copy_ctor const &);
 };
 test(aligned_copy_ctor);
-// CHECK: define void @_Z7forward17aligned_copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.aligned_copy_ctor) align 16 %{{.*}}, ptr nonnull %{{.*}})
+// CHECK: define void @_Z7forward17aligned_copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.aligned_copy_ctor) align 16 %{{.*}}, ptr dead_on_return nonnull %{{.*}})
 //
 // CHECK: declare ptr @_ZN17aligned_copy_ctorC1ERKS_(ptr {{[^,]*}} returned {{[^,]*}}, ptr nonnull align 16 dereferenceable(16))
 //
 // CHECK: define void @_Z22test_aligned_copy_ctorv()
 // CHECK: %[[tmp:.*]] = alloca %struct.aligned_copy_ctor, align 16
 // CHECK: call void @_Z21def_aligned_copy_ctorv(ptr dead_on_unwind nonnull writable sret(%struct.aligned_copy_ctor) align 16 %[[tmp]])
-// CHECK: call void @_Z3use17aligned_copy_ctor(ptr nonnull %[[tmp]])
+// CHECK: call void @_Z3use17aligned_copy_ctor(ptr dead_on_return nonnull %[[tmp]])
 // CHECK: ret void
 //
-// CHECK: declare void @_Z3use17aligned_copy_ctor(ptr)
+// CHECK: declare void @_Z3use17aligned_copy_ctor(ptr dead_on_return)
 // CHECK: declare void @_Z21def_aligned_copy_ctorv(ptr dead_on_unwind writable sret(%struct.aligned_copy_ctor) align 16)
 
 struct empty {};
diff --git a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
index 9927d0b24799e..78c45753b5bbd 100644
--- a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
+++ b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
@@ -18,7 +18,7 @@ struct NonTrivial {
 
 SWIFTCALL int receiveNonTrivial(NonTrivial o) { return o.o; }
 
-// CHECK-LABEL: define dso_local swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr noundef %o)
+// CHECK-LABEL: define dso_local swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr dead_on_return noundef %o)
 
 int passNonTrivial() {
   return receiveNonTrivial({});
@@ -26,4 +26,4 @@ int passNonTrivial() {
 
 // CHECK-LABEL: define dso_local noundef i32 @"?passNonTrivial@@YAHXZ"()
 // CHECK-NOT: stacksave
-// CHECK: call swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr noundef %{{.*}})
+// CHECK: call swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr dead_on_return noundef %{{.*}})
diff --git a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m b/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
index d2a954ae26a04..ba8a04b52716e 100644
--- a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
+++ b/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
@@ -42,7 +42,7 @@ void testStrongException(void) {
 // CHECK: call void @genWeak(ptr dead_on_unwind writable sret(%[[STRUCT_WEAK]]) align 8 %[[AGG_TMP]])
 // CHECK: invoke void @genWeak(ptr dead_on_unwind writable sret(%[[STRUCT_WEAK]]) align 8 %[[AGG_TMP1]])
 
-// CHECK: call void @calleeWeak(ptr noundef %[[AGG_TMP]], ptr noundef %[[AGG_TMP1]])
+// CHECK: call void @calleeWeak(ptr dead_on_return noundef %[[AGG_TMP]], ptr dead_on_return noundef %[[AGG_TMP1]])
 // CHECK: ret void
 
 // CHECK: landingpad { ptr, i32 }
diff --git a/clang/test/CodeGenObjC/pass-by-value-noalias.m b/clang/test/CodeGenObjC/pass-by-value-noalias.m
index ed94d4c80b525..86a4ba36a1b8f 100644
--- a/clang/test/CodeGenObjC/pass-by-value-noalias.m
+++ b/clang/test/CodeGenObjC/pass-by-value-noalias.m
@@ -17,6 +17,6 @@ @interface Bar
   Bar *__weak f;
 };
 
-// WITH_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
-// NO_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
+// NO_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
 void take(struct Foo arg) {}
diff --git a/clang/test/CodeGenObjC/weak-in-c-struct.m b/clang/test/CodeGenObjC/weak-in-c-struct.m
index be80edd1ff11d..6809360d03da1 100644
--- a/clang/test/CodeGenObjC/weak-in-c-struct.m
+++ b/clang/test/CodeGenObjC/weak-in-c-struct.m
@@ -130,7 +130,7 @@ void test_move_assignment_Weak(Weak *p) {
   *p = getWeak();
 }
 
-// COMMON: define{{.*}} void @test_parameter_Weak(ptr noundef %[[A:.*]])
+// COMMON: define{{.*}} void @test_parameter_Weak(ptr dead_on_return noundef %[[A:.*]])
 // COMMON: call void @__destructor_{{.*}}(ptr %[[A]])
 
 void test_parameter_Weak(Weak a) {
@@ -142,7 +142,7 @@ void test_parameter_Weak(Weak a) {
 // COMMON: store ptr %[[A]], ptr %[[A_ADDR]]
 // COMMON: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]]
 // COMMON: call void @__copy_constructor_{{.*}}(ptr %[[AGG_TMP]], ptr %[[V0]])
-// COMMON: call void @calleeWeak(ptr noundef %[[AGG_TMP]])
+// COMMON: call void @calleeWeak(ptr dead_on_return noundef %[[AGG_TMP]])
 // COMMON-NEXT: ret
 
 void test_argument_Weak(Weak *a) {
@@ -164,7 +164,7 @@ Weak test_return_Weak(Weak *a) {
 // COMMON: %[[AGG_TMP:.*]] = alloca %[[STRUCT_WEAK]]
 // COMMON: br i1
 
-// COMMON: call void @objc_msgSend({{.*}}, ptr noundef %[[AGG_TMP]])
+// COMMON: call void @objc_msgSend({{.*}}, ptr dead_on_return noundef %[[AGG_TMP]])
 // COMMON: br
 
 // COMMON: call void @__destructor_{{.*}}(ptr %[[AGG_TMP]])
diff --git a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
index 4d76796d86d1f..3a043c4892981 100644
--- a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
@@ -75,7 +75,7 @@ - (void)passStrongWeak:(StrongWeak)a;
 - (void)passNonTrivial:(NonTrivial)a;
 @end
 
-// CHECK: define{{.*}} void @_Z19testParamStrongWeak10StrongWeak(ptr noundef %{{.*}})
+// CHECK: define{{.*}} void @_Z19testParamStrongWeak10StrongWeak(ptr dead_on_return noundef %{{.*}})
 // CHECK: call noundef ptr @_ZN10StrongWeakD1Ev(
 // CHECK-NEXT: ret void
 
@@ -88,7 +88,7 @@ void testParamStrongWeak(StrongWeak a) {
 // CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN10StrongWeakC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(16) %[[V0]])
-// CHECK: call void @_Z19testParamStrongWeak10StrongWeak(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z19testParamStrongWeak10StrongWeak(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK-NOT: call
 // CHECK: ret void
 
@@ -107,13 +107,13 @@ StrongWeak testReturnStrongWeak(StrongWeak *a) {
   return *a;
 }
 
-// CHECK: define{{.*}} void @_Z27testParamContainsStrongWeak18ContainsStrongWeak(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z27testParamContainsStrongWeak18ContainsStrongWeak(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN18ContainsStrongWeakD1Ev(ptr {{[^,]*}} %[[A]])
 
 void testParamContainsStrongWeak(ContainsStrongWeak a) {
 }
 
-// CHECK: define{{.*}} void @_Z26testParamDerivedStrongWeak17DerivedStrongWeak(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z26testParamDerivedStrongWeak17DerivedStrongWeak(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN17DerivedStrongWeakD1Ev(ptr {{[^,]*}} %[[A]])
 
 void testParamDerivedStrongWeak(DerivedStrongWeak a) {
@@ -163,7 +163,7 @@ Strong testReturnStrong(Strong *a) {
   return *a;
 }
 
-// CHECK: define{{.*}} void @_Z21testParamWeakTemplate1SIU6__weakP11objc_objectE(ptr noundef %{{.*}})
+// CHECK: define{{.*}} void @_Z21testParamWeakTemplate1SIU6__weakP11objc_objectE(ptr dead_on_return noundef %{{.*}})
 // CHECK: call noundef ptr @_ZN1SIU6__weakP11objc_objectED1Ev(
 // CHECK-NEXT: ret void
 
@@ -237,7 +237,7 @@ void test0(C *c) {
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_STRONGWEAK]], align 8
 // CHECK: br i1
 
-// CHECK: call void @objc_msgSend({{.*}}, ptr noundef %[[AGG_TMP]])
+// CHECK: call void @objc_msgSend({{.*}}, ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: br
 
 // CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN10StrongWeakD1Ev(ptr noundef nonnull align 8 dereferenceable(16) %[[AGG_TMP]])
diff --git a/clang/test/CodeGenObjCXX/property-objects.mm b/clang/test/CodeGenObjCXX/property-objects.mm
index 7ae20f66177bd..8354794254933 100644
--- a/clang/test/CodeGenObjCXX/property-objects.mm
+++ b/clang/test/CodeGenObjCXX/property-objects.mm
@@ -60,7 +60,7 @@ - (struct CGRect)extent {return bounds;}
 
 // CHECK-LABEL: define{{.*}} i32 @main
 // CHECK: call void @_ZN1SC1ERKS_(ptr {{[^,]*}} [[AGGTMP:%[a-zA-Z0-9\.]+]], ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) {{%[a-zA-Z0-9\.]+}})
-// CHECK: call void @objc_msgSend(ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef [[AGGTMP]])
+// CHECK: call void @objc_msgSend(ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr dead_on_return noundef [[AGGTMP]])
 // CHECK-NEXT: ret i32 0
 int main() {
   I *i;
diff --git a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
index e5cb71bad47c0..9428940d6da48 100644
--- a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
@@ -15,7 +15,7 @@
 
 // Check that AddrDiscStrong0 is destructed in the callee.
 
-// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr noundef %[[A:.*]])
+// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN15AddrDiscStrong0D1Ev(ptr noundef nonnull align {{[0-9]+}} dereferenceable(16) %[[A]])
 // CHECK: ret void
 
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
index 738b7ab7f2b75..d42e51d04ab9d 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
@@ -4,6 +4,7 @@
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
+typedef unsigned int uint;
 typedef half __attribute__((ext_vector_type(2))) half2;
 
 // CHECK-LABEL: @test_setprio_inc_wg(
@@ -42,6 +43,24 @@ void test_s_wait_tensorcnt() {
   __builtin_amdgcn_s_wait_tensorcnt(0);
 }
 
+// CHECK-LABEL: @test_prng_b32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.prng.b32(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(1) [[TMP2]], align 4
+// CHECK-NEXT:    ret void
+//
+void test_prng_b32(global uint* out, uint a) {
+  *out = __builtin_amdgcn_prng_b32(a);
+}
+
 // CHECK-LABEL: @test_tanh_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -119,6 +138,25 @@ void test_rcp_bf16(global __bf16* out, __bf16 a)
   *out = __builtin_amdgcn_rcp_bf16(a);
 }
 
+// CHECK-LABEL: @test_sqrt_bf16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = call bfloat @llvm.amdgcn.sqrt.bf16(bfloat [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[TMP1]], ptr addrspace(1) [[TMP2]], align 2
+// CHECK-NEXT:    ret void
+//
+void test_sqrt_bf16(global __bf16* out, __bf16 a)
+{
+  *out = __builtin_amdgcn_sqrt_bf16(a);
+}
+
 // CHECK-LABEL: @test_rsq_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
diff --git a/clang/test/Driver/print-multi-selection-flags.c b/clang/test/Driver/print-multi-selection-flags.c
index 5f9383fbed8f4..8cf8f04bb6b48 100644
--- a/clang/test/Driver/print-multi-selection-flags.c
+++ b/clang/test/Driver/print-multi-selection-flags.c
@@ -107,3 +107,22 @@
 // CHECK-AARCH64-MULTILIB-CUSTOM-FLAG: --target=aarch64-unknown-none-eabi
 // CHECK-MULTILIB-CUSTOM-FLAG-DAG:     -fmultilib-flag=foo
 // CHECK-MULTILIB-CUSTOM-FLAG-DAG:     -fmultilib-flag=bar
+
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fropi              | FileCheck --check-prefixes=CHECK-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -frwpi              | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fropi -frwpi       | FileCheck --check-prefixes=CHECK-ROPI,CHECK-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fno-ropi -fno-rwpi | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a                     | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fpic               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIC1 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fPIC               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIC2 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fpie               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIE1 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fPIE               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIE2 %s
+// CHECK-PIC2: -fPIC
+// CHECK-PIE2: -fPIE
+// CHECK-NO-PIC: -fno-pic
+// CHECK-NO-ROPI: -fno-ropi
+// CHECK-NO-RWPI: -fno-rwpi
+// CHECK-PIC1: -fpic
+// CHECK-PIE1: -fpie
+// CHECK-ROPI: -fropi
+// CHECK-RWPI: -frwpi
diff --git a/clang/test/Headers/stdarg.cpp b/clang/test/Headers/stdarg.cpp
index 20bf17caf15f7..bfc3af11a23b6 100644
--- a/clang/test/Headers/stdarg.cpp
+++ b/clang/test/Headers/stdarg.cpp
@@ -15,8 +15,8 @@
 
 #include <stdarg.h>
 
-// AARCH64-C: define {{.*}} @f(i32 noundef %n, ptr noundef %list)
-// AARCH64-CXX: define {{.*}} @_Z1fiSt9__va_list(i32 noundef %n, ptr noundef %list)
+// AARCH64-C: define {{.*}} @f(i32 noundef %n, ptr dead_on_return noundef %list)
+// AARCH64-CXX: define {{.*}} @_Z1fiSt9__va_list(i32 noundef %n, ptr dead_on_return noundef %list)
 // X86_64-C: define {{.*}} @f(i32 noundef %n, ptr noundef %list)
 // X86_64-CXX: define {{.*}} @_Z1fiP13__va_list_tag(i32 noundef %n, ptr noundef %list)
 // PPC64-C: define {{.*}} @f(i32 noundef signext %n, ptr noundef %list)
diff --git a/clang/test/OpenMP/for_firstprivate_codegen.cpp b/clang/test/OpenMP/for_firstprivate_codegen.cpp
index 0255e1e3d4aea..83b5939799642 100644
--- a/clang/test/OpenMP/for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/for_firstprivate_codegen.cpp
@@ -427,7 +427,7 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[TMP0]])
 // CHECK1-NEXT:    store i32 0, ptr [[RETVAL]], align 4
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
@@ -469,7 +469,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -543,12 +543,12 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store ptr [[TMP3]], ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store ptr [[TMP4]], ptr [[_TMP1]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -572,7 +572,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP6]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE6]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done6:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP8]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR7]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP7]], ptr noundef [[AGG_TMP8]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP8]]) #[[ATTR2]]
@@ -611,7 +611,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC4]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP9]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP9]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM11:%.*]] = sext i32 [[TMP19]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR5]], i64 0, i64 [[IDXPROM11]]
@@ -711,7 +711,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1044,8 +1044,8 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr @g1, align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr @g1, align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
 // CHECK4-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1086,7 +1086,7 @@ int main() {
 // CHECK4-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK4-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[G]], align 4
-// CHECK4-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK4-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store volatile i32 1, ptr [[TMP13]], align 4
 // CHECK4-NEXT:    store i32 2, ptr [[SIVAR3]], align 4
 // CHECK4-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 0
@@ -1103,7 +1103,7 @@ int main() {
 // CHECK4-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[G]], align 4
 // CHECK4-NEXT:    store volatile i32 [[TMP14]], ptr [[BLOCK_CAPTURED]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 5
-// CHECK4-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK4-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store ptr [[TMP15]], ptr [[BLOCK_CAPTURED5]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 7
 // CHECK4-NEXT:    [[TMP16:%.*]] = load i32, ptr [[SIVAR3]], align 4
@@ -1137,7 +1137,7 @@ int main() {
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
 // CHECK4-NEXT:    store i32 2, ptr [[BLOCK_CAPTURE_ADDR]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR1]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store i32 2, ptr [[TMP0]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
 // CHECK4-NEXT:    store i32 4, ptr [[BLOCK_CAPTURE_ADDR2]], align 4
diff --git a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
index 93e71b9a8312e..441e809dc59e5 100644
--- a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
@@ -350,9 +350,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i32 8, i1 false)
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i32 2
@@ -524,7 +524,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK1-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK1-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK1-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -535,22 +535,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK1-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK1-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK1-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK1-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK1-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK1-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK1-NEXT:    ret void
 //
@@ -578,25 +578,25 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK1-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK1-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK1-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK1-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP5]], -1
 // CHECK1-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP7]], 1
 // CHECK1-NEXT:    store i32 [[DIV]], ptr [[TMP6]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP8]], i32 0, i32 2
 // CHECK1-NEXT:    store i32 1111, ptr [[ARRAYIDX]], align 4
 // CHECK1-NEXT:    ret void
@@ -656,7 +656,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -738,9 +738,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i32 8, i1 false)
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i32 2
@@ -840,7 +840,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 0, ptr [[A]], align 4
 // CHECK1-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SST]], ptr [[THIS1]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_CASTED]], align 4
@@ -862,7 +862,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
@@ -896,7 +896,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -965,7 +965,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK3-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK3-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK3-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -976,22 +976,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK3-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK3-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK3-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK3-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK3-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK3-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK3-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK3-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK3-NEXT:    ret void
 //
@@ -1020,22 +1020,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK3-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK3-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP5]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK3-NEXT:    store ptr [[B_ADDR]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK3-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP8]], ptr [[TMP7]], align 4
 // CHECK3-NEXT:    call void @_ZZN2SSC1ERiENKUlvE_clEv(ptr nonnull align 4 dereferenceable(16) [[REF_TMP]])
 // CHECK3-NEXT:    ret void
@@ -1053,32 +1053,32 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0:%.*]], ptr [[THIS1]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK3-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP7]], -1
 // CHECK3-NEXT:    store i32 [[DEC]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 4
+// CHECK3-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
 // CHECK3-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP10]], 1
 // CHECK3-NEXT:    store i32 [[DIV]], ptr [[TMP9]], align 4
 // CHECK3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 4
+// CHECK3-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
 // CHECK3-NEXT:    store i32 [[TMP17]], ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 4
+// CHECK3-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK3-NEXT:    store i32 [[TMP21]], ptr [[C_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP22:%.*]] = load i32, ptr [[C_CASTED]], align 4
@@ -1106,14 +1106,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK3-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK3-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK3-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK3-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK3-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -1252,7 +1252,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK4-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK4-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
 // CHECK4-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK4-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK4-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -1263,22 +1263,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK4-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK4-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK4-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK4-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK4-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK4-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK4-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK4-NEXT:    ret void
 //
@@ -1307,11 +1307,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK4-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK4-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK4-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
 // CHECK4-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 0
@@ -1327,13 +1327,13 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BLOCK_CAPTURED_THIS_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 5
 // CHECK4-NEXT:    store ptr [[TMP0]], ptr [[BLOCK_CAPTURED_THIS_ADDR]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP3]], ptr [[BLOCK_CAPTURED]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 7
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK4-NEXT:    store i32 [[TMP4]], ptr [[BLOCK_CAPTURED5]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP5]], ptr [[BLOCK_CAPTURED6]], align 4
 // CHECK4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___BLOCK_LITERAL_GENERIC:%.*]], ptr [[BLOCK]], i32 0, i32 3
 // CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 4
@@ -1354,7 +1354,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BLOCK_CAPTURED_THIS:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
 // CHECK4-NEXT:    [[THIS:%.*]] = load ptr, ptr [[BLOCK_CAPTURED_THIS]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK4-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
 // CHECK4-NEXT:    store i32 [[INC]], ptr [[TMP0]], align 4
@@ -1363,12 +1363,12 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP2]], -1
 // CHECK4-NEXT:    store i32 [[DEC]], ptr [[BLOCK_CAPTURE_ADDR1]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 4
+// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK4-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP4]], 1
 // CHECK4-NEXT:    store i32 [[DIV]], ptr [[TMP3]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 4
+// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
 // CHECK4-NEXT:    store i32 [[TMP6]], ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_CASTED]], align 4
@@ -1377,7 +1377,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    store i32 [[TMP8]], ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 4
+// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
 // CHECK4-NEXT:    store i32 [[TMP11]], ptr [[C_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP12:%.*]] = load i32, ptr [[C_CASTED]], align 4
@@ -1405,14 +1405,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK4-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK4-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK4-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK4-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK4-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK4-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK4-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK4-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK4-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -1530,9 +1530,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i64 2
@@ -1704,7 +1704,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK9-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK9-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK9-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -1715,22 +1715,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK9-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK9-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK9-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK9-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK9-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK9-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK9-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK9-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK9-NEXT:    ret void
 //
@@ -1758,25 +1758,25 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK9-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK9-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK9-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
-// CHECK9-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK9-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK9-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK9-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK9-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP5]], -1
 // CHECK9-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK9-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK9-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP7]], 1
 // CHECK9-NEXT:    store i32 [[DIV]], ptr [[TMP6]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP8]], i64 0, i64 2
 // CHECK9-NEXT:    store i32 1111, ptr [[ARRAYIDX]], align 4
 // CHECK9-NEXT:    ret void
@@ -1836,7 +1836,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1918,9 +1918,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i64 2
@@ -2020,7 +2020,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i32 0, ptr [[A]], align 4
 // CHECK9-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SST]], ptr [[THIS1]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK9-NEXT:    store i32 [[TMP1]], ptr [[A_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP2:%.*]] = load i64, ptr [[A_CASTED]], align 8
@@ -2042,7 +2042,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[A]], ptr [[A_ADDR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK9-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK9-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
@@ -2076,7 +2076,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2145,7 +2145,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK11-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK11-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK11-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK11-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK11-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -2156,22 +2156,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK11-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK11-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK11-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK11-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK11-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK11-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK11-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK11-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK11-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK11-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK11-NEXT:    ret void
 //
@@ -2200,22 +2200,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK11-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK11-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK11-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK11-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK11-NEXT:    store ptr [[TMP0]], ptr [[TMP3]], align 8
 // CHECK11-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK11-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP5]], ptr [[TMP4]], align 8
 // CHECK11-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK11-NEXT:    store ptr [[B_ADDR]], ptr [[TMP6]], align 8
 // CHECK11-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK11-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP8]], ptr [[TMP7]], align 8
 // CHECK11-NEXT:    call void @_ZZN2SSC1ERiENKUlvE_clEv(ptr nonnull align 8 dereferenceable(32) [[REF_TMP]])
 // CHECK11-NEXT:    ret void
@@ -2233,32 +2233,32 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0:%.*]], ptr [[THIS1]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
 // CHECK11-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8
+// CHECK11-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK11-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK11-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK11-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8
+// CHECK11-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK11-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP7]], -1
 // CHECK11-NEXT:    store i32 [[DEC]], ptr [[TMP6]], align 4
 // CHECK11-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8
+// CHECK11-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
 // CHECK11-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP10]], 1
 // CHECK11-NEXT:    store i32 [[DIV]], ptr [[TMP9]], align 4
 // CHECK11-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 8
+// CHECK11-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 // CHECK11-NEXT:    store i32 [[TMP13]], ptr [[A_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP14:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8
+// CHECK11-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
 // CHECK11-NEXT:    store i32 [[TMP17]], ptr [[B_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP18:%.*]] = load i64, ptr [[B_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+// CHECK11-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK11-NEXT:    store i32 [[TMP21]], ptr [[C_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP22:%.*]] = load i64, ptr [[C_CASTED]], align 8
@@ -2286,14 +2286,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK11-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK11-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK11-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK11-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK11-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK11-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK11-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK11-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK11-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -2432,7 +2432,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK12-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK12-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK12-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK12-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK12-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -2443,22 +2443,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK12-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK12-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK12-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK12-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK12-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK12-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK12-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK12-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK12-NEXT:    ret void
 //
@@ -2487,11 +2487,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK12-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK12-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK12-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK12-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
 // CHECK12-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 0
@@ -2507,13 +2507,13 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BLOCK_CAPTURED_THIS_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 5
 // CHECK12-NEXT:    store ptr [[TMP0]], ptr [[BLOCK_CAPTURED_THIS_ADDR]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP3]], ptr [[BLOCK_CAPTURED]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 8
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK12-NEXT:    store i32 [[TMP4]], ptr [[BLOCK_CAPTURED5]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP5]], ptr [[BLOCK_CAPTURED6]], align 8
 // CHECK12-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___BLOCK_LITERAL_GENERIC:%.*]], ptr [[BLOCK]], i32 0, i32 3
 // CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 8
@@ -2534,7 +2534,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BLOCK_CAPTURED_THIS:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
 // CHECK12-NEXT:    [[THIS:%.*]] = load ptr, ptr [[BLOCK_CAPTURED_THIS]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK12-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
 // CHECK12-NEXT:    store i32 [[INC]], ptr [[TMP0]], align 4
@@ -2543,12 +2543,12 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP2]], -1
 // CHECK12-NEXT:    store i32 [[DEC]], ptr [[BLOCK_CAPTURE_ADDR1]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 8
+// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK12-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP4]], 1
 // CHECK12-NEXT:    store i32 [[DIV]], ptr [[TMP3]], align 4
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 8
+// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
 // CHECK12-NEXT:    store i32 [[TMP6]], ptr [[A_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP7:%.*]] = load i64, ptr [[A_CASTED]], align 8
@@ -2557,7 +2557,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    store i32 [[TMP8]], ptr [[B_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP9:%.*]] = load i64, ptr [[B_CASTED]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 8
+// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
 // CHECK12-NEXT:    store i32 [[TMP11]], ptr [[C_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP12:%.*]] = load i64, ptr [[C_CASTED]], align 8
@@ -2585,14 +2585,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK12-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK12-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK12-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK12-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK12-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK12-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK12-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK12-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK12-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK12-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK12-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -2660,11 +2660,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    store i64 [[VLA2]], ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    store i64 [[VLA4]], ptr [[VLA_ADDR5]], align 8
 // CHECK17-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META3]], !align [[META5:![0-9]+]]
 // CHECK17-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2751,8 +2751,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META3]], !align [[META5]]
+// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
 // CHECK17-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 // CHECK17-NEXT:    [[TMP8:%.*]] = add nuw i64 [[TMP7]], 127
diff --git a/clang/test/OpenMP/sections_firstprivate_codegen.cpp b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
index 7c6d1839fb10e..32c5826e6f75d 100644
--- a/clang/test/OpenMP/sections_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
@@ -448,7 +448,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -517,10 +517,10 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_SECTIONS_LB_]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_UB_]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_ST_]], align 4
@@ -668,7 +668,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -996,7 +996,7 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK4-NEXT:    store i32 0, ptr [[DOTOMP_SECTIONS_LB_]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_UB_]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_ST_]], align 4
diff --git a/clang/test/OpenMP/single_firstprivate_codegen.cpp b/clang/test/OpenMP/single_firstprivate_codegen.cpp
index 27cd220adf225..31ea1ca4952fc 100644
--- a/clang/test/OpenMP/single_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/single_firstprivate_codegen.cpp
@@ -403,7 +403,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -467,10 +467,10 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK1-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_single(ptr @[[GLOB1]], i32 [[TMP5]])
@@ -585,7 +585,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -908,7 +908,7 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_single(ptr @[[GLOB1]], i32 [[TMP2]])
diff --git a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
index aa50d8fb3aabd..a171827a18646 100644
--- a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
@@ -354,9 +354,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -396,9 +396,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -550,12 +550,12 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -655,7 +655,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -711,14 +711,14 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -752,9 +752,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -776,7 +776,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -815,7 +815,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i64 0, i64 [[IDXPROM9]]
@@ -912,7 +912,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1155,9 +1155,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1197,9 +1197,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1349,12 +1349,12 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1454,7 +1454,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1510,14 +1510,14 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1551,9 +1551,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1575,7 +1575,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1613,7 +1613,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP14]]
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP15]], i32 4, i1 false)
@@ -1709,7 +1709,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1877,7 +1877,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -1943,13 +1943,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP8]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP9]], align 8
 // CHECK9-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP11]], ptr [[TMP10]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP12]], align 8
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
index 78e40e54671ac..678770520f677 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -415,9 +415,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -459,9 +459,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -616,9 +616,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -741,12 +741,12 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -846,7 +846,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -902,14 +902,14 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -944,9 +944,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -968,7 +968,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1005,7 +1005,7 @@ int main() {
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
@@ -1082,9 +1082,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1112,7 +1112,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done5:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR2]]
@@ -1151,7 +1151,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -1233,7 +1233,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1476,9 +1476,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1520,9 +1520,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1675,9 +1675,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -1796,12 +1796,12 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1901,7 +1901,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1957,14 +1957,14 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1999,9 +1999,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -2023,7 +2023,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2058,7 +2058,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK3-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK3:       omp.inner.for.inc:
@@ -2135,9 +2135,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -2163,7 +2163,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2201,7 +2201,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -2282,7 +2282,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2450,7 +2450,7 @@ int main() {
 // CHECK5-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK5-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK5-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK5-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK5-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2520,7 +2520,7 @@ int main() {
 // CHECK5-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK5-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK5-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2607,13 +2607,13 @@ int main() {
 // CHECK5-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK5-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK5-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK5-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK5-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK5-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK5-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK5-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK5-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK5-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
@@ -2659,9 +2659,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -2703,9 +2703,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -2860,9 +2860,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -2987,14 +2987,14 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK13-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK13-NEXT:    ret void
 //
@@ -3029,9 +3029,9 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -3053,7 +3053,7 @@ int main() {
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done4:
-// CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3090,7 +3090,7 @@ int main() {
 // CHECK13-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK13-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK13-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK13-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK13:       omp.inner.for.inc:
@@ -3167,9 +3167,9 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -3197,7 +3197,7 @@ int main() {
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done5:
-// CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR3]]
@@ -3236,7 +3236,7 @@ int main() {
 // CHECK13-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK13-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK13-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK13-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK13-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK13-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK13-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -3321,7 +3321,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK13-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -3352,7 +3352,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK13-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -3379,9 +3379,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9:![0-9]+]], !align [[META10:![0-9]+]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -3423,9 +3423,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -3578,9 +3578,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -3701,14 +3701,14 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK15-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK15-NEXT:    ret void
 //
@@ -3743,9 +3743,9 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -3767,7 +3767,7 @@ int main() {
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
-// CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3802,7 +3802,7 @@ int main() {
 // CHECK15-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK15-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK15-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK15-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK15:       omp.inner.for.inc:
@@ -3879,9 +3879,9 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -3907,7 +3907,7 @@ int main() {
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
-// CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3945,7 +3945,7 @@ int main() {
 // CHECK15-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK15-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK15-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK15-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK15-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK15-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -4029,7 +4029,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK15-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -4060,7 +4060,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK15-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -4089,7 +4089,7 @@ int main() {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK17-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK17-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
 // CHECK17-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK17-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -4159,7 +4159,7 @@ int main() {
 // CHECK17-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK17-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK17-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -4246,13 +4246,13 @@ int main() {
 // CHECK17-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK17-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK17-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK17-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK17-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK17-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK17-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK17-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK17-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK17-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
diff --git a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
index 294fcba7872b3..f3c9565a17656 100644
--- a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
@@ -357,9 +357,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -399,9 +399,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -553,14 +553,14 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store i64 [[TMP2]], ptr [[TMP6]], align 8
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -660,7 +660,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -716,14 +716,14 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -757,9 +757,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -781,7 +781,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -820,7 +820,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i64 0, i64 [[IDXPROM9]]
@@ -917,7 +917,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1160,9 +1160,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1202,9 +1202,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1354,14 +1354,14 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store i32 [[TMP2]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1461,7 +1461,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1517,14 +1517,14 @@ int main() {
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1558,9 +1558,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1582,7 +1582,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1620,7 +1620,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP14]]
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP15]], i32 4, i1 false)
@@ -1716,7 +1716,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1884,7 +1884,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -1950,13 +1950,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP8]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP9]], align 8
 // CHECK9-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP11]], ptr [[TMP10]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP12]], align 8
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
index d742b0a85af42..037aa12d57226 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -391,9 +391,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -435,9 +435,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -592,9 +592,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -717,14 +717,14 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store i64 [[TMP2]], ptr [[TMP6]], align 8
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -824,7 +824,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -880,14 +880,14 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -922,9 +922,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -946,7 +946,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -983,7 +983,7 @@ int main() {
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
@@ -1060,9 +1060,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1090,7 +1090,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done5:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR2]]
@@ -1129,7 +1129,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -1211,7 +1211,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1454,9 +1454,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1498,9 +1498,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1653,9 +1653,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -1774,14 +1774,14 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store i32 [[TMP2]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1881,7 +1881,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1937,14 +1937,14 @@ int main() {
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1979,9 +1979,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -2003,7 +2003,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2038,7 +2038,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK3-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK3:       omp.inner.for.inc:
@@ -2115,9 +2115,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -2143,7 +2143,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2181,7 +2181,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -2262,7 +2262,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2430,7 +2430,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2500,7 +2500,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK9-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2587,13 +2587,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK9-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
diff --git a/clang/test/OpenMP/teams_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
index 6f21c9e31bd8d..fec8fcb78f91e 100644
--- a/clang/test/OpenMP/teams_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
@@ -458,9 +458,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11:![0-9]+]], !align [[META12:![0-9]+]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -493,9 +493,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i64 2
@@ -808,7 +808,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -872,9 +872,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -902,9 +902,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i64 2
@@ -1032,7 +1032,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1250,9 +1250,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12:![0-9]+]], !align [[META13:![0-9]+]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1285,9 +1285,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i32 8, i1 false)
 // CHECK11-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i32 2
@@ -1600,7 +1600,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK11-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK11-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK11-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1664,9 +1664,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1694,9 +1694,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i32 8, i1 false)
 // CHECK11-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i32 2
@@ -1824,7 +1824,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK11-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK11-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK11-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1997,7 +1997,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
 // CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[S_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[A_ADDR]], align 8
@@ -2031,11 +2031,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    store i64 [[VLA2]], ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    store i64 [[VLA4]], ptr [[VLA_ADDR5]], align 8
 // CHECK17-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META8]], !align [[META10:![0-9]+]]
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK17-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2235,7 +2235,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[S_ADDR]], align 8
@@ -2273,8 +2273,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META8]], !align [[META10]]
 // CHECK17-NEXT:    [[TMP6:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP6]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2460,7 +2460,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
 // CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[S_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[A_ADDR]], align 4
@@ -2494,11 +2494,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    store i32 [[VLA2]], ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    store i32 [[VLA4]], ptr [[VLA_ADDR5]], align 4
 // CHECK19-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 4
-// CHECK19-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP3:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK19-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = mul nuw i32 [[TMP2]], [[TMP3]]
@@ -2696,7 +2696,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[S_ADDR]], align 4
@@ -2734,8 +2734,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP3:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
-// CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP6:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK19-NEXT:    store ptr [[TMP6]], ptr [[SAVED_STACK]], align 4
 // CHECK19-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP2]], [[TMP3]]
diff --git a/clang/test/Sema/dllexport.c b/clang/test/Sema/dllexport.c
index 3f911fb095c0f..5f6ff36e290e9 100644
--- a/clang/test/Sema/dllexport.c
+++ b/clang/test/Sema/dllexport.c
@@ -2,6 +2,10 @@
 // RUN: %clang_cc1 -triple x86_64-win32   -fsyntax-only -fms-extensions -verify -std=c11 %s
 // RUN: %clang_cc1 -triple i686-mingw32   -fsyntax-only -fms-extensions -verify -std=c11 %s
 // RUN: %clang_cc1 -triple x86_64-mingw32 -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple i686-windows-itanium   -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple x86_64-windows-itanium -fsyntax-only -fms-extensions -verify -std=c11 %s
+// RUN: %clang_cc1 -triple x86_64-sie-ps5         -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple x86_64-sie-ps5         -fsyntax-only -fms-extensions -verify -std=c11 %s
 
 // Invalid usage.
 __declspec(dllexport) typedef int typedef1;
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
index 46c3670848529..135865c8450f5 100644
--- a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
+++ b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -283,3 +283,31 @@ void f() {
 }
 
 #endif
+
+namespace GH147374 {
+
+struct String {};
+template <typename T> void operator+(T, String &&) = delete;
+
+struct Bar {
+    void operator+(String) const; // expected-note {{candidate function}}
+    friend void operator+(Bar, String) {};  // expected-note {{candidate function}}
+};
+
+struct Baz {
+    void operator+(String); // expected-note {{candidate function}}
+    friend void operator+(Baz, String) {}; // expected-note {{candidate function}}
+};
+
+void test() {
+    Bar a;
+    String b;
+    a + b;
+    //expected-error at -1 {{use of overloaded operator '+' is ambiguous (with operand types 'Bar' and 'String')}}
+
+    Baz z;
+    z + b;
+    //expected-error at -1 {{use of overloaded operator '+' is ambiguous (with operand types 'Baz' and 'String')}}
+}
+
+}
diff --git a/clang/test/SemaTemplate/concepts-using-decl.cpp b/clang/test/SemaTemplate/concepts-using-decl.cpp
index fca69dea5c88f..41f7b6d2f8faa 100644
--- a/clang/test/SemaTemplate/concepts-using-decl.cpp
+++ b/clang/test/SemaTemplate/concepts-using-decl.cpp
@@ -176,3 +176,24 @@ void func() {
   f.foo<10, 10>(); // expected-error {{no matching member function for call to 'foo'}}
 }
 } // namespace heads_without_concepts.
+
+namespace GH146614 {
+
+template <typename T>
+struct base {
+    template <typename A>
+    void foo(A x)
+        requires (requires{x;})
+    {}
+};
+
+
+struct child : base<int> {
+  using base<int>::foo;
+  template <typename A>
+  void foo(A x)
+      requires (false)
+  {}
+};
+
+}
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4241d12601242..5f0783f869bf6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1466,8 +1466,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     assert(falseTarget && "missing conditional branch false block");
     mlir::Location loc = toLocation();
     mlir::Value bcc = builder->createConvert(loc, builder->getI1Type(), cond);
-    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget, std::nullopt,
-                                            falseTarget, std::nullopt);
+    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget,
+                                            mlir::ValueRange{}, falseTarget,
+                                            mlir::ValueRange{});
   }
   void genConditionalBranch(mlir::Value cond,
                             Fortran::lower::pft::Evaluation *trueTarget,
@@ -2556,8 +2557,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
           loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
-          /*local_vars=*/std::nullopt,
-          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*local_vars=*/mlir::ValueRange{},
+          /*local_syms=*/nullptr, /*reduce_vars=*/mlir::ValueRange{},
           /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
           /*reduce_attrs=*/nullptr);
 
@@ -3810,9 +3811,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       mlir::Block *selectCaseBlock = insertBlock(blockList[0]);
       mlir::Block *assumedSizeBlock =
           rankStarBlock ? rankStarBlock : defaultBlock;
-      builder->create<mlir::cf::CondBranchOp>(loc, isAssumedSize,
-                                              assumedSizeBlock, std::nullopt,
-                                              selectCaseBlock, std::nullopt);
+      builder->create<mlir::cf::CondBranchOp>(
+          loc, isAssumedSize, assumedSizeBlock, mlir::ValueRange{},
+          selectCaseBlock, mlir::ValueRange{});
       startBlock(selectCaseBlock);
     }
     // Create fir.select_case for the other rank cases.
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1850b67898126..b8ab5d09e3e08 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -303,7 +303,7 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
     return builder.create<fir::StringLitOp>(
-        loc, llvm::ArrayRef<mlir::Type>{type}, std::nullopt, attrs);
+        loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
   }
 }
 
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 0a1cd67789772..281ab229d1b6a 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1003,9 +1003,9 @@ class ScalarExprLowering {
           },
           [&](const fir::MutableBoxValue &toBox) {
             if (toBox.isPointer()) {
-              Fortran::lower::associateMutableBox(converter, loc, toBox, expr,
-                                                  /*lbounds=*/std::nullopt,
-                                                  stmtCtx);
+              Fortran::lower::associateMutableBox(
+                  converter, loc, toBox, expr,
+                  /*lbounds=*/mlir::ValueRange{}, stmtCtx);
               return;
             }
             // For allocatable components, a deep copy is needed.
@@ -3604,8 +3604,9 @@ class ArrayExprLowering {
       mlir::Value castTo =
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
-      return builder.create<fir::ArrayLoadOp>(
-          loc, seqTy, castTo, shapeOp, /*slice=*/mlir::Value{}, std::nullopt);
+      return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
+                                              /*slice=*/mlir::Value{},
+                                              mlir::ValueRange{});
     };
     // Custom lowering of the element store to deal with the extra indirection
     // to the lazy allocated buffer.
@@ -4207,7 +4208,7 @@ class ArrayExprLowering {
       auto addr =
           builder->create<fir::ArrayCoorOp>(loc, eleRefTy, tmp, shape,
                                             /*slice=*/mlir::Value{}, indices,
-                                            /*typeParams=*/std::nullopt);
+                                            /*typeParams=*/mlir::ValueRange{});
       auto load = builder->create<fir::LoadOp>(loc, addr);
       return builder->createConvert(loc, i1Ty, load);
     };
@@ -4522,17 +4523,18 @@ class ArrayExprLowering {
         fir::isRecordWithAllocatableMember(eleTy))
       TODO(loc, "creating an array temp where the element type has "
                 "allocatable members");
-    mlir::Value temp = !seqTy.hasDynamicExtents()
-                           ? builder.create<fir::AllocMemOp>(loc, type)
-                           : builder.create<fir::AllocMemOp>(
-                                 loc, type, ".array.expr", std::nullopt, shape);
+    mlir::Value temp =
+        !seqTy.hasDynamicExtents()
+            ? builder.create<fir::AllocMemOp>(loc, type)
+            : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
+                                              mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
     return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
                                             /*slice=*/mlir::Value{},
-                                            std::nullopt);
+                                            mlir::ValueRange{});
   }
 
   static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -6483,7 +6485,7 @@ class ArrayExprLowering {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
         mem = builder.create<fir::AllocMemOp>(
-            loc, eleTy, /*typeparams=*/std::nullopt, initBuffSz);
+            loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
         builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
       }
     } else {
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 6cda742874ccf..cacf4e249aa28 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -813,6 +813,10 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
       if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(baseTy))
         baseTy = boxTy.getEleTy();
       baseTy = fir::unwrapRefType(baseTy);
+
+      if (mlir::isa<fir::SequenceType>(baseTy))
+        TODO(loc, "array of derived-type with device component");
+
       auto recTy =
           mlir::dyn_cast<fir::RecordType>(fir::unwrapSequenceType(baseTy));
       assert(recTy && "expected fir::RecordType");
@@ -823,7 +827,7 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
         if (Fortran::semantics::IsDeviceAllocatable(sym)) {
           unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
           mlir::Type fieldTy;
-          std::vector<mlir::Value> coordinates;
+          llvm::SmallVector<mlir::Value> coordinates;
 
           if (fieldIdx != std::numeric_limits<unsigned>::max()) {
             // Field found in the base record type.
@@ -866,8 +870,24 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
             TODO(loc, "device resident component in complex derived-type "
                       "hierarchy");
 
-          mlir::Value comp = builder.create<fir::CoordinateOp>(
-              loc, builder.getRefType(fieldTy), fir::getBase(exv), coordinates);
+          mlir::Value base = fir::getBase(exv);
+          mlir::Value comp;
+          if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(base.getType()))) {
+            mlir::Value box = builder.create<fir::LoadOp>(loc, base);
+            mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, box);
+            llvm::SmallVector<mlir::Value> lenParams;
+            assert(coordinates.size() == 1 && "expect one coordinate");
+            auto field = mlir::dyn_cast<fir::FieldIndexOp>(
+                coordinates[0].getDefiningOp());
+            comp = builder.create<hlfir::DesignateOp>(
+                loc, builder.getRefType(fieldTy), addr,
+                /*component=*/field.getFieldName(),
+                /*componentShape=*/mlir::Value{},
+                hlfir::DesignateOp::Subscripts{});
+          } else {
+            comp = builder.create<fir::CoordinateOp>(
+                loc, builder.getRefType(fieldTy), base, coordinates);
+          }
           cuf::DataAttributeAttr dataAttr =
               Fortran::lower::translateSymbolCUFDataAttribute(
                   builder.getContext(), sym);
diff --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index 2be5ef76e46b8..5f73335242336 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
   genUnreachable(builder, loc);
 }
 
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
 }
 
 void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index 389a89ddcf102..c7b3e11728cea 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -122,7 +122,7 @@ class VectorSubscriptBoxBuilder {
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
-        loc, fldTy, componentName, recTy, /*typeParams*/ std::nullopt));
+        loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
 
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index acd5a88a2582d..5b1dbc4435d6c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -620,7 +620,7 @@ fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc,
   mlir::NamedAttribute sizeAttr(sizeTag, getI64IntegerAttr(data.size()));
   llvm::SmallVector<mlir::NamedAttribute> attrs{dataAttr, sizeAttr};
   return create<fir::StringLitOp>(loc, llvm::ArrayRef<mlir::Type>{type},
-                                  std::nullopt, attrs);
+                                  mlir::ValueRange{}, attrs);
 }
 
 mlir::Value fir::FirOpBuilder::genShape(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 773d6408079cc..04703f7911176 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -59,7 +59,8 @@ mlir::Value fir::runtime::genCpuTime(fir::FirOpBuilder &builder,
                                      mlir::Location loc) {
   mlir::func::FuncOp func =
       fir::runtime::getRuntimeFunc<mkRTKey(CpuTime)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
@@ -280,7 +281,8 @@ void fir::runtime::genRename(fir::FirOpBuilder &builder, mlir::Location loc,
 mlir::Value fir::runtime::genTime(fir::FirOpBuilder &builder,
                                   mlir::Location loc) {
   auto func = fir::runtime::getRuntimeFunc<mkRTKey(time)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 /// generate runtime call to transfer intrinsic with no size argument
diff --git a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
index 411181cc6dd1c..9b5e43b80b1f6 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
@@ -25,7 +25,7 @@ void fir::runtime::genExit(fir::FirOpBuilder &builder, mlir::Location loc,
 void fir::runtime::genAbort(fir::FirOpBuilder &builder, mlir::Location loc) {
   mlir::func::FuncOp abortFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(Abort)>(loc, builder);
-  builder.create<fir::CallOp>(loc, abortFunc, std::nullopt);
+  builder.create<fir::CallOp>(loc, abortFunc, mlir::ValueRange{});
 }
 
 void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 03cc92e975b19..c5cf01ed98357 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -405,7 +405,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
   mlir::Location loc = forallMaskOp.getLoc();
   mlir::Value mask = generateYieldedScalarValue(forallMaskOp.getMaskRegion(),
                                                 builder.getI1Type());
-  auto ifOp = builder.create<fir::IfOp>(loc, std::nullopt, mask, false);
+  auto ifOp = builder.create<fir::IfOp>(loc, mlir::TypeRange{}, mask, false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   constructStack.push_back(ifOp);
 }
@@ -530,7 +530,7 @@ void OrderedAssignmentRewriter::generateMaskIfOp(mlir::Value cdt) {
   mlir::Location loc = cdt.getLoc();
   cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
   cdt = builder.createConvert(loc, builder.getI1Type(), cdt);
-  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), std::nullopt, cdt,
+  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), mlir::TypeRange{}, cdt,
                                         /*withElseRegion=*/false);
   constructStack.push_back(ifOp.getOperation());
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
diff --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index 1f8edf851de9b..bc4fcd8b0112e 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -222,7 +222,7 @@ void AllocaReplaceImpl::genIndirectDeallocation(
         rewriter.create<fir::ConvertOp>(loc, intPtrTy, ptrVal);
     mlir::Value isAllocated = rewriter.create<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::ne, ptrToInt, c0);
-    auto ifOp = rewriter.create<fir::IfOp>(loc, std::nullopt, isAllocated,
+    auto ifOp = rewriter.create<fir::IfOp>(loc, mlir::TypeRange{}, isAllocated,
                                            /*withElseRegion=*/false);
     rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value cast = fir::factory::createConvert(
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 57eae1ff052a2..6e45aae4246d0 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -456,7 +456,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
   rewriter.setInsertionPointToEnd(thisBlock);
   if (destOps.has_value())
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, destOps.value(),
-                                            newBlock, std::nullopt);
+                                            newBlock, mlir::ValueRange{});
   else
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, newBlock);
   rewriter.setInsertionPointToEnd(newBlock);
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 506c8e66dbdfa..ad8464b495888 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -261,7 +261,7 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
+          /*iterArgs=*/mlir::ValueRange{}, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
diff --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index ee89ea38a3fc7..e3bb181f65398 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -21,4 +21,36 @@ contains
 ! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]]#0, z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[Z]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
 
+  subroutine sub2()
+    type(ty_device), pointer :: d1
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub2()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub2Ed1"} -> !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMm1Fsub2Ed1"} : (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
+  subroutine sub3()
+    type(ty_device), allocatable :: d1
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub3()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub3Ed1"} -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMm1Fsub3Ed1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
 end module
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index 98270adaa7c73..59808779aa6ef 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -48,7 +48,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
   mlir::Value addr = builder->create<fir::AllocaOp>(loc, eleType);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      /*shape=*/mlir::Value{}, /*typeParams=*/std::nullopt,
+      /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{},
       /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
@@ -102,11 +102,11 @@ TEST_F(FortranVariableTest, SimpleArray) {
       extents.size(), fir::SequenceType::getUnknownExtent());
   mlir::Type seqTy = fir::SequenceType::get(typeShape, eleType);
   mlir::Value addr = builder->create<fir::AllocaOp>(
-      loc, seqTy, /*pinned=*/false, /*typeParams=*/std::nullopt, extents);
+      loc, seqTy, /*pinned=*/false, /*typeParams=*/mlir::ValueRange{}, extents);
   mlir::Value shape = createShape(extents);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      shape, /*typeParams*/ std::nullopt, /*dummy_scope=*/nullptr, name,
+      shape, /*typeParams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
 
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index e210992c5111a..3fb62788c1168 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -571,6 +571,8 @@ function(add_integration_test test_name)
   target_compile_options(${fq_build_target_name} PRIVATE
                          ${compile_options} ${INTEGRATION_TEST_COMPILE_OPTIONS})
 
+  set(compiler_runtime "")
+
   if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU)
     target_link_options(${fq_build_target_name} PRIVATE
       ${LIBC_COMPILE_OPTIONS_DEFAULT} ${INTEGRATION_TEST_COMPILE_OPTIONS}
@@ -599,17 +601,19 @@ function(add_integration_test test_name)
     set(link_options
       -nolibc
       -nostartfiles
-      -static
+      -nostdlib
       ${LIBC_LINK_OPTIONS_DEFAULT}
       ${LIBC_TEST_LINK_OPTIONS_DEFAULT}
     )
     target_link_options(${fq_build_target_name} PRIVATE ${link_options})
+    list(APPEND compiler_runtime ${LIBGCC_S_LOCATION})
   endif()
   target_link_libraries(
     ${fq_build_target_name}
-    ${fq_target_name}.__libc__
     libc.startup.${LIBC_TARGET_OS}.crt1
     libc.test.IntegrationTest.test
+    ${fq_target_name}.__libc__
+    ${compiler_runtime}
   )
   add_dependencies(${fq_build_target_name}
                    libc.test.IntegrationTest.test
@@ -770,6 +774,7 @@ function(add_libc_hermetic test_name)
                          ${HERMETIC_TEST_COMPILE_OPTIONS})
 
   set(link_libraries "")
+  set(compiler_runtime "")
   foreach(lib IN LISTS HERMETIC_TEST_LINK_LIBRARIES)
     if(TARGET ${lib}.hermetic)
       list(APPEND link_libraries ${lib}.hermetic)
@@ -807,12 +812,12 @@ function(add_libc_hermetic test_name)
     set(link_options
       -nolibc
       -nostartfiles
-      -static
+      -nostdlib
       ${LIBC_LINK_OPTIONS_DEFAULT}
       ${LIBC_TEST_LINK_OPTIONS_DEFAULT}
     )
     target_link_options(${fq_build_target_name} PRIVATE ${link_options})
-    list(APPEND link_libraries ${LIBGCC_S_LOCATION})
+    list(APPEND compiler_runtime ${LIBGCC_S_LOCATION})
   endif()
   target_link_libraries(
     ${fq_build_target_name}
@@ -820,7 +825,9 @@ function(add_libc_hermetic test_name)
       libc.startup.${LIBC_TARGET_OS}.crt1
       ${link_libraries}
       LibcHermeticTestSupport.hermetic
-      ${fq_target_name}.__libc__)
+      ${fq_target_name}.__libc__
+      ${compiler_runtime}
+    )
   add_dependencies(${fq_build_target_name}
                    LibcTest.hermetic
                    libc.test.UnitTest.ErrnoSetterMatcher
diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in
index fc01aaf2d8746..b68c0c8258366 100644
--- a/libcxx/include/__config_site.in
+++ b/libcxx/include/__config_site.in
@@ -30,7 +30,6 @@
 #cmakedefine01 _LIBCPP_HAS_LOCALIZATION
 #cmakedefine01 _LIBCPP_HAS_UNICODE
 #cmakedefine01 _LIBCPP_HAS_WIDE_CHARACTERS
-#cmakedefine _LIBCPP_HAS_NO_STD_MODULES
 #cmakedefine01 _LIBCPP_HAS_TIME_ZONE_DATABASE
 #cmakedefine01 _LIBCPP_INSTRUMENTED_WITH_ASAN
 
diff --git a/libcxx/test/libcxx/fuzzing/random.pass.cpp b/libcxx/test/libcxx/fuzzing/random.pass.cpp
index cb074bd60fdc8..f0256a01f29ae 100644
--- a/libcxx/test/libcxx/fuzzing/random.pass.cpp
+++ b/libcxx/test/libcxx/fuzzing/random.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // UNSUPPORTED: c++03, c++11
 
diff --git a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
index 1ba0063c1dada..f9f81d22ff80e 100644
--- a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // <math.h>
 
diff --git a/libcxx/test/std/numerics/c.math/cmath.pass.cpp b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
index 48c2918802fc3..8d261e9fcbdb2 100644
--- a/libcxx/test/std/numerics/c.math/cmath.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // <cmath>
 
diff --git a/lld/ELF/BPSectionOrderer.cpp b/lld/ELF/BPSectionOrderer.cpp
index f464b1d4518a4..06152046d13d4 100644
--- a/lld/ELF/BPSectionOrderer.cpp
+++ b/lld/ELF/BPSectionOrderer.cpp
@@ -76,10 +76,10 @@ DenseMap<const InputSectionBase *, int> elf::runBalancedPartitioning(
     if (!d)
       return;
     auto *sec = dyn_cast_or_null<InputSection>(d->section);
-    // Skip empty, discarded, ICF folded sections. Skipping ICF folded sections
-    // reduces duplicate detection work in BPSectionOrderer.
+    // Skip empty, discarded, ICF folded sections, .bss. Skipping ICF folded
+    // sections reduces duplicate detection work in BPSectionOrderer.
     if (!sec || sec->size == 0 || !sec->isLive() || sec->repl != sec ||
-        !orderer.secToSym.try_emplace(sec, d).second)
+        !sec->content().data() || !orderer.secToSym.try_emplace(sec, d).second)
       return;
     rootSymbolToSectionIdxs[CachedHashStringRef(
                                 lld::utils::getRootSymbol(sym.getName()))]
diff --git a/lld/test/ELF/bp-section-orderer.s b/lld/test/ELF/bp-section-orderer.s
index 4df2e8d43022e..438d7c2da0f76 100644
--- a/lld/test/ELF/bp-section-orderer.s
+++ b/lld/test/ELF/bp-section-orderer.s
@@ -26,28 +26,28 @@
 
 # RUN: ld.lld -o out.s a.o --irpgo-profile=a.profdata --bp-startup-sort=function
 # RUN: llvm-nm -jn out.s | tr '\n' , | FileCheck %s --check-prefix=STARTUP
-# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,{{$}}
+# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.os a.o --irpgo-profile=a.profdata --bp-startup-sort=function --symbol-ordering-file a.txt
 # RUN: llvm-nm -jn out.os | tr '\n' , | FileCheck %s --check-prefix=ORDER-STARTUP
-# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,{{$}}
+# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.cf a.o --verbose-bp-section-orderer --bp-compression-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-FUNC
 # RUN: ld.lld -o out.cf.icf a.o --verbose-bp-section-orderer --bp-compression-sort=function --icf=all --gc-sections 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-ICF-FUNC
 # RUN: llvm-nm -jn out.cf | tr '\n' , | FileCheck %s --check-prefix=CFUNC
-# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,{{$}}
+# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.cd a.o --verbose-bp-section-orderer --bp-compression-sort=data 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-DATA
 # RUN: llvm-nm -jn out.cd | tr '\n' , | FileCheck %s --check-prefix=CDATA
-# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,{{$}}
+# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
 
 # RUN: ld.lld -o out.cb a.o --verbose-bp-section-orderer --bp-compression-sort=both 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
 # RUN: llvm-nm -jn out.cb | tr '\n' , | FileCheck %s --check-prefix=CBOTH
-# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,{{$}}
+# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,g1,{{$}}
 
 # RUN: ld.lld -o out.cbs a.o --verbose-bp-section-orderer --bp-compression-sort=both --irpgo-profile=a.profdata --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
 # RUN: llvm-nm -jn out.cbs | tr '\n' , | FileCheck %s --check-prefix=CBOTH-STARTUP
-# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,{{$}}
+# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
 
 # BP-COMPRESSION-FUNC: Ordered 9 sections ([[#]] bytes) using balanced partitioning
 # BP-COMPRESSION-ICF-FUNC: Ordered 8 sections ([[#]] bytes) using balanced partitioning
@@ -108,6 +108,7 @@ d3
 d2
 
 #--- a.c
+int g1;
 const char s5[] = "engineering";
 const char s4[] = "computer program";
 const char s3[] = "hardware engineer";
@@ -377,6 +378,14 @@ d1:
 	.word	6                               // 0x6
 	.size	d1, 16
 
+	.type	g1, at object                      // @g1
+	.section	.bss.g1,"aw", at nobits
+	.globl	g1
+	.p2align	2, 0x0
+g1:
+	.word	0                               // 0x0
+	.size	g1, 4
+
 	.section	".note.GNU-stack","", at progbits
 	.addrsig
 	.addrsig_sym F
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index e445fa8833022..6f812b91a8b1d 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -700,7 +700,7 @@ void IRExecutionUnit::CollectCandidateCPlusPlusNames(
 
 class LoadAddressResolver {
 public:
-  LoadAddressResolver(Target *target, bool &symbol_was_missing_weak)
+  LoadAddressResolver(Target &target, bool &symbol_was_missing_weak)
       : m_target(target), m_symbol_was_missing_weak(symbol_was_missing_weak) {}
 
   std::optional<lldb::addr_t> Resolve(SymbolContextList &sc_list) {
@@ -722,11 +722,11 @@ class LoadAddressResolver {
 
       // First try the symbol.
       if (candidate_sc.symbol) {
-        load_address = candidate_sc.symbol->ResolveCallableAddress(*m_target);
+        load_address = candidate_sc.symbol->ResolveCallableAddress(m_target);
         if (load_address == LLDB_INVALID_ADDRESS) {
           Address addr = candidate_sc.symbol->GetAddress();
-          load_address = m_target->GetProcessSP()
-                             ? addr.GetLoadAddress(m_target)
+          load_address = m_target.GetProcessSP()
+                             ? addr.GetLoadAddress(&m_target)
                              : addr.GetFileAddress();
         }
       }
@@ -734,8 +734,8 @@ class LoadAddressResolver {
       // If that didn't work, try the function.
       if (load_address == LLDB_INVALID_ADDRESS && candidate_sc.function) {
         Address addr = candidate_sc.function->GetAddress();
-        load_address = m_target->GetProcessSP() ? addr.GetLoadAddress(m_target)
-                                                : addr.GetFileAddress();
+        load_address = m_target.GetProcessSP() ? addr.GetLoadAddress(&m_target)
+                                               : addr.GetFileAddress();
       }
 
       // We found a load address.
@@ -766,7 +766,7 @@ class LoadAddressResolver {
   }
 
 private:
-  Target *m_target;
+  Target &m_target;
   bool &m_symbol_was_missing_weak;
   lldb::addr_t m_best_internal_load_address = LLDB_INVALID_ADDRESS;
 };
@@ -790,7 +790,7 @@ IRExecutionUnit::FindInSymbols(const std::vector<ConstString> &names,
   for (size_t i = 0; i < m_preferred_modules.GetSize(); ++i)
     non_local_images.Remove(m_preferred_modules.GetModuleAtIndex(i));
 
-  LoadAddressResolver resolver(target, symbol_was_missing_weak);
+  LoadAddressResolver resolver(*target, symbol_was_missing_weak);
 
   ModuleFunctionSearchOptions function_options;
   function_options.include_symbols = true;
diff --git a/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
index e1d7e42bdd1a9..cd60227572be4 100644
--- a/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
+++ b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
@@ -2,7 +2,6 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -120,6 +119,8 @@ def cleanup():
                 '@"2 elements"',
             ],
         )
+
+        self.runCmd("settings set target.max-children-depth 6")
         self.expect(
             "frame variable mutabledict --ptr-depth 3",
             substrs=[
diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script b/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
index 91de55f4ade4a..43018eacf709b 100644
--- a/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
+++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
@@ -1,3 +1,4 @@
+settings set target.max-children-depth 10
 breakpoint set --file UdtLayoutTest.cpp --line 60
 run
 target variable
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index de5f66ce1584c..87d2a9ac3bf88 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -263,6 +263,20 @@ amaclean at nvidia.com (email), [AlexMaclean](https://github.com/AlexMaclean) (GitH
 
 #### PowerPC backend
 
+Amy Kwan (esp. release issues) \
+Amy.Kwan1 at ibm.com (email), [amy-kwan](https://github.com/amy-kwan) (GitHub) \
+Lei Huang \
+lei at ca.ibm.com (email), [lei137](https://github.com/lei137) (GitHub) \
+Sean Fertile (esp. ABI/ELF/XCOFF) \
+sfertile at ca.ibm.com (email), [mandlebug](https://github.com/mandlebug) (GitHub) \
+Zhijian Lin \
+zhijian at ca.ibm.com (email), [diggerlin](https://github.com/diggerlin) (GitHub) \
+Maryam Moghadas \
+maryammo at ca.ibm.com (email), [maryammo](https://github.com/maryammo) (GitHub) \
+Roland Froese \
+froese at ca.ibm.com (email), [RolandF77](https://github.com/RolandF77) (GitHub) \
+llvmonpower \
+powerllvm at ca.ibm.com (email), [llvmonpower](https://github.com/llvmonpower) (GitHub)
 
 #### RISCV backend
 
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index c614a6d7ace9e..732227b98ab9e 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -30,7 +30,7 @@ because the naming and other conventions are dictated by the C++ standard.
 
 There are some conventions that are not uniformly followed in the code base
 (e.g. the naming convention).  This is because they are relatively new, and a
-lot of code was written before they were put in place.  Our long term goal is
+lot of code was written before they were put in place.  Our long-term goal is
 for the entire codebase to follow the convention, but we explicitly *do not*
 want patches that do large-scale reformatting of existing code.  On the other
 hand, it is reasonable to rename the methods of a class if you're about to
@@ -50,7 +50,7 @@ code imported into the tree. Generally, our preference is for standards
 conforming, modern, and portable C++ code as the implementation language of
 choice.
 
-For automation, build-systems and utility scripts Python is preferred and
+For automation, build-systems, and utility scripts, Python is preferred and
 is widely used in the LLVM repository already.
 
 C++ Standard Versions
@@ -92,7 +92,7 @@ LLVM support libraries (for example, `ADT
 <https://github.com/llvm/llvm-project/tree/main/llvm/include/llvm/ADT>`_)
 implement specialized data structures or functionality missing in the standard
 library. Such libraries are usually implemented in the ``llvm`` namespace and
-follow the expected standard interface, when there is one.
+follow the expected standard interface when there is one.
 
 When both C++ and the LLVM support libraries provide similar functionality, and
 there isn't a specific reason to favor the C++ implementation, it is generally
@@ -325,8 +325,8 @@ implementation file.  In any case, implementation files can include additional
 comments (not necessarily in Doxygen markup) to explain implementation details
 as needed.
 
-Don't duplicate function or class name at the beginning of the comment.
-For humans it is obvious which function or class is being documented;
+Don't duplicate the function or class name at the beginning of the comment.
+For humans, it is obvious which function or class is being documented;
 automatic documentation processing tools are smart enough to bind the comment
 to the correct declaration.
 
@@ -369,7 +369,7 @@ lower-case letter, and finish the last sentence without a period, if it would
 end in one otherwise. Sentences which end with different punctuation, such as
 "did you forget ';'?", should still do so.
 
-For example this is a good error message:
+For example, this is a good error message:
 
 .. code-block:: none
 
@@ -443,7 +443,7 @@ Write your code to fit within 80 columns.
 There must be some limit to the width of the code in
 order to allow developers to have multiple files side-by-side in
 windows on a modest display.  If you are going to pick a width limit, it is
-somewhat arbitrary but you might as well pick something standard.  Going with 90
+somewhat arbitrary, but you might as well pick something standard.  Going with 90
 columns (for example) instead of 80 columns wouldn't add any significant value
 and would be detrimental to printing out code.  Also many other projects have
 standardized on 80 columns, so some people have already configured their editors
@@ -520,7 +520,7 @@ within each other and within function calls in order to build up aggregates
 The historically common formatting of braced initialization of aggregate
 variables does not mix cleanly with deep nesting, general expression contexts,
 function arguments, and lambdas. We suggest new code use a simple rule for
-formatting braced initialization lists: act as-if the braces were parentheses
+formatting braced initialization lists: act as if the braces were parentheses
 in a function call. The formatting rules exactly match those already well
 understood for formatting nested function calls. Examples:
 
@@ -607,11 +607,11 @@ Static constructors and destructors (e.g., global variables whose types have a
 constructor or destructor) should not be added to the code base, and should be
 removed wherever possible.
 
-Globals in different source files are initialized in `arbitrary order
+Globals in different source files are initialized in an `arbitrary order
 <https://yosefk.com/c++fqa/ctors.html#fqa-10.12>`_, making the code more
 difficult to reason about.
 
-Static constructors have negative impact on launch time of programs that use
+Static constructors have a negative impact on the launch time of programs that use
 LLVM as a library. We would really like for there to be zero cost for linking
 in an additional LLVM target or other library into an application, but static
 constructors undermine this goal.
@@ -698,7 +698,7 @@ If you use a braced initializer list when initializing a variable, use an equals
 Use ``auto`` Type Deduction to Make Code More Readable
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Some are advocating a policy of "almost always ``auto``" in C++11, however LLVM
+Some are advocating a policy of "almost always ``auto``" in C++11; however, LLVM
 uses a more moderate stance. Use ``auto`` if and only if it makes the code more
 readable or easier to maintain. Don't "almost always" use ``auto``, but do use
 ``auto`` with initializers like ``cast<Foo>(...)`` or other places where the
@@ -783,14 +783,14 @@ guards, and might not include their prerequisites. Name such files with the
 
 In general, a header should be implemented by one or more ``.cpp`` files.  Each
 of these ``.cpp`` files should include the header that defines their interface
-first.  This ensures that all of the dependences of the header have been
+first.  This ensures that all of the dependencies of the header have been
 properly added to the header itself, and are not implicit.  System headers
 should be included after user headers for a translation unit.
 
 Library Layering
 ^^^^^^^^^^^^^^^^
 
-A directory of header files (for example ``include/llvm/Foo``) defines a
+A directory of header files (for example, ``include/llvm/Foo``) defines a
 library (``Foo``). One library (both
 its headers and implementation) should only use things from the libraries
 listed in its dependencies.
@@ -822,7 +822,7 @@ especially in header files.
 
 But wait! Sometimes you need to have the definition of a class to use it, or to
 inherit from it.  In these cases go ahead and ``#include`` that header file.  Be
-aware however that there are many cases where you don't need to have the full
+aware, however, that there are many cases where you don't need to have the full
 definition of a class.  If you are using a pointer or reference to a class, you
 don't need the header file.  If you are simply returning a class instance from a
 prototyped function or method, you don't need it.  In fact, for most cases, you
@@ -970,7 +970,7 @@ loops.  A silly example is something like this:
 When you have very, very small loops, this sort of structure is fine. But if it
 exceeds more than 10-15 lines, it becomes difficult for people to read and
 understand at a glance. The problem with this sort of code is that it gets very
-nested very quickly. Meaning that the reader of the code has to keep a lot of
+nested very quickly. This means that the reader of the code has to keep a lot of
 context in their brain to remember what is going immediately on in the loop,
 because they don't know if/when the ``if`` conditions will have ``else``\s etc.
 It is strongly preferred to structure the loop like this:
@@ -988,7 +988,7 @@ It is strongly preferred to structure the loop like this:
     ...
   }
 
-This has all the benefits of using early exits for functions: it reduces nesting
+This has all the benefits of using early exits for functions: it reduces the nesting
 of the loop, it makes it easier to describe why the conditions are true, and it
 makes it obvious to the reader that there is no ``else`` coming up that they
 have to push context into their brain for.  If a loop is large, this can be a
@@ -1149,12 +1149,12 @@ In general, names should be in camel case (e.g. ``TextFileReader`` and
   nouns and start with an upper-case letter (e.g. ``TextFileReader``).
 
 * **Variable names** should be nouns (as they represent state).  The name should
-  be camel case, and start with an upper case letter (e.g. ``Leader`` or
+  be camel case, and start with an upper-case letter (e.g. ``Leader`` or
   ``Boats``).
 
 * **Function names** should be verb phrases (as they represent actions), and
   command-like function should be imperative.  The name should be camel case,
-  and start with a lower case letter (e.g. ``openFile()`` or ``isFoo()``).
+  and start with a lower-case letter (e.g. ``openFile()`` or ``isFoo()``).
 
 * **Enum declarations** (e.g. ``enum Foo {...}``) are types, so they should
   follow the naming conventions for types.  A common use for enums is as a
@@ -1207,7 +1207,7 @@ Assert Liberally
 ^^^^^^^^^^^^^^^^
 
 Use the "``assert``" macro to its fullest.  Check all of your preconditions and
-assumptions, you never know when a bug (not necessarily even yours) might be
+assumptions.  You never know when a bug (not necessarily even yours) might be
 caught early by an assertion, which reduces debugging time dramatically.  The
 "``<cassert>``" header file is probably already included by the header files you
 are using, so it doesn't cost anything to use it.
@@ -1302,7 +1302,7 @@ preferred to write the code like this:
   assert(NewToSet && "The value shouldn't be in the set yet");
 
 In C code where ``[[maybe_unused]]`` is not supported, use ``void`` cast to
-suppress unused variable warning as follows:
+suppress an unused variable warning as follows:
 
 .. code-block:: c
 
@@ -1546,7 +1546,7 @@ whenever possible.
 The semantics of postincrement include making a copy of the value being
 incremented, returning it, and then preincrementing the "work value".  For
 primitive types, this isn't a big deal. But for iterators, it can be a huge
-issue (for example, some iterators contains stack and set objects in them...
+issue (for example, some iterators contain stack and set objects in them...
 copying an iterator could invoke the copy ctor's of these as well).  In general,
 get in the habit of always using preincrement, and you won't have a problem.
 
@@ -1663,7 +1663,7 @@ Don't Use Braces on Simple Single-Statement Bodies of if/else/loop Statements
 
 When writing the body of an ``if``, ``else``, or for/while loop statement, we
 prefer to omit the braces to avoid unnecessary line noise. However, braces
-should be used in cases where the omission of braces harm the readability and
+should be used in cases where the omission of braces harms the readability and
 maintainability of the code.
 
 We consider that readability is harmed when omitting the brace in the presence
@@ -1763,7 +1763,7 @@ would help to avoid running into a "dangling else" situation.
         handleAttrOnDecl(D, A, i);
   }
 
-  // Use braces on the outer block because of a nested `if`; otherwise the
+  // Use braces on the outer block because of a nested `if`; otherwise, the
   // compiler would warn: `add explicit braces to avoid dangling else`
   if (auto *D = dyn_cast<FunctionDecl>(D)) {
     if (shouldProcess(D))
diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index aef91909dd17b..794075201d646 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -39,8 +39,8 @@ class MachineBlockFrequencyInfo {
 public:
   LLVM_ABI MachineBlockFrequencyInfo(); // Legacy pass manager only.
   LLVM_ABI explicit MachineBlockFrequencyInfo(
-      MachineFunction &F, MachineBranchProbabilityInfo &MBPI,
-      MachineLoopInfo &MLI);
+      const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
+      const MachineLoopInfo &MLI);
   LLVM_ABI MachineBlockFrequencyInfo(MachineBlockFrequencyInfo &&);
   LLVM_ABI ~MachineBlockFrequencyInfo();
 
diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h
index 77cee875f16e7..f8241a3cdf160 100644
--- a/llvm/include/llvm/IR/DebugInfo.h
+++ b/llvm/include/llvm/IR/DebugInfo.h
@@ -115,8 +115,7 @@ class DebugInfoFinder {
   LLVM_ABI void processVariable(DILocalVariable *DVI);
   /// Process debug info location.
   LLVM_ABI void processLocation(const Module &M, const DILocation *Loc);
-  /// Process a DbgRecord (e.g, treat a DbgVariableRecord like a
-  /// DbgVariableIntrinsic).
+  /// Process a DbgRecord.
   LLVM_ABI void processDbgRecord(const Module &M, const DbgRecord &DR);
 
   /// Process subprogram.
@@ -290,8 +289,6 @@ struct VarRecord {
   DILocalVariable *Var;
   DILocation *DL;
 
-  VarRecord(DbgVariableIntrinsic *DVI)
-      : Var(DVI->getVariable()), DL(getDebugValueLoc(DVI)) {}
   VarRecord(DbgVariableRecord *DVR)
       : Var(DVR->getVariable()), DL(getDebugValueLoc(DVR)) {}
   VarRecord(DILocalVariable *Var, DILocation *DL) : Var(Var), DL(DL) {}
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 9345f95015301..f1f0c18949c35 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -66,7 +66,6 @@ namespace dwarf {
 enum Tag : uint16_t;
 }
 
-class DbgVariableIntrinsic;
 class DbgVariableRecord;
 
 LLVM_ABI extern cl::opt<bool> EnableFSDiscriminator;
@@ -4613,7 +4612,6 @@ class DebugVariable {
   LLVM_ABI static const FragmentInfo DefaultFragment;
 
 public:
-  LLVM_ABI DebugVariable(const DbgVariableIntrinsic *DII);
   LLVM_ABI DebugVariable(const DbgVariableRecord *DVR);
 
   DebugVariable(const DILocalVariable *Var,
@@ -4681,7 +4679,6 @@ template <> struct DenseMapInfo<DebugVariable> {
 /// information).
 class DebugVariableAggregate : public DebugVariable {
 public:
-  LLVM_ABI DebugVariableAggregate(const DbgVariableIntrinsic *DVI);
   DebugVariableAggregate(const DebugVariable &V)
       : DebugVariable(V.getVariable(), std::nullopt, V.getInlinedAt()) {}
 };
diff --git a/llvm/include/llvm/Transforms/Utils/SSAUpdater.h b/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
index 4e5da81a7e885..9500b1f160ea9 100644
--- a/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
+++ b/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
@@ -29,7 +29,6 @@ template <typename T> class SSAUpdaterTraits;
 class Type;
 class Use;
 class Value;
-class DbgValueInst;
 
 /// Helper class for SSA formation on a set of values defined in
 /// multiple blocks.
@@ -122,8 +121,6 @@ class SSAUpdater {
   /// the instruction. Anything outside of its block will have its
   /// value set to the new SSA value if available, and undef if not.
   void UpdateDebugValues(Instruction *I);
-  void UpdateDebugValues(Instruction *I,
-                         SmallVectorImpl<DbgValueInst *> &DbgValues);
   void UpdateDebugValues(Instruction *I,
                          SmallVectorImpl<DbgVariableRecord *> &DbgValues);
 
@@ -136,7 +133,6 @@ class SSAUpdater {
 
 private:
   Value *GetValueAtEndOfBlockInternal(BasicBlock *BB);
-  void UpdateDebugValue(Instruction *I, DbgValueInst *DbgValue);
   void UpdateDebugValue(Instruction *I, DbgVariableRecord *DbgValue);
 };
 
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index d9d41f1d72e35..dc8184394f74d 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -445,7 +445,6 @@ class CodeGenPrepare {
   bool optimizeSwitchInst(SwitchInst *SI);
   bool optimizeExtractElementInst(Instruction *Inst);
   bool dupRetToEnableTailCallOpts(BasicBlock *BB, ModifyDT &ModifiedDT);
-  bool fixupDbgValue(Instruction *I);
   bool fixupDbgVariableRecord(DbgVariableRecord &I);
   bool fixupDbgVariableRecordsOnInst(Instruction &I);
   bool placeDbgValues(Function &F);
@@ -2762,9 +2761,6 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
     case Intrinsic::fshl:
     case Intrinsic::fshr:
       return optimizeFunnelShift(II);
-    case Intrinsic::dbg_assign:
-    case Intrinsic::dbg_value:
-      return fixupDbgValue(II);
     case Intrinsic::masked_gather:
       return optimizeGatherScatterInst(II, II->getArgOperand(0));
     case Intrinsic::masked_scatter:
@@ -3554,8 +3550,6 @@ class TypePromotionTransaction {
     /// Keep track of the original uses (pair Instruction, Index).
     SmallVector<InstructionAndIdx, 4> OriginalUses;
     /// Keep track of the debug users.
-    SmallVector<DbgValueInst *, 1> DbgValues;
-    /// And non-instruction debug-users too.
     SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
 
     /// Keep track of the new value so that we can undo it by replacing
@@ -3577,7 +3571,9 @@ class TypePromotionTransaction {
       }
       // Record the debug uses separately. They are not in the instruction's
       // use list, but they are replaced by RAUW.
+      SmallVector<DbgValueInst *> DbgValues;
       findDbgValues(DbgValues, Inst, &DbgVariableRecords);
+      assert(DbgValues.empty());
 
       // Now, we can replace the uses.
       Inst->replaceAllUsesWith(New);
@@ -3591,11 +3587,7 @@ class TypePromotionTransaction {
       // RAUW has replaced all original uses with references to the new value,
       // including the debug uses. Since we are undoing the replacements,
       // the original debug uses must also be reinstated to maintain the
-      // correctness and utility of debug value instructions.
-      for (auto *DVI : DbgValues)
-        DVI->replaceVariableLocationOp(New, Inst);
-      // Similar story with DbgVariableRecords, the non-instruction
-      // representation of dbg.values.
+      // correctness and utility of debug value records.
       for (DbgVariableRecord *DVR : DbgVariableRecords)
         DVR->replaceVariableLocationOp(New, Inst);
     }
@@ -8933,32 +8925,6 @@ bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT) {
   return MadeChange;
 }
 
-// Some CGP optimizations may move or alter what's computed in a block. Check
-// whether a dbg.value intrinsic could be pointed at a more appropriate operand.
-bool CodeGenPrepare::fixupDbgValue(Instruction *I) {
-  assert(isa<DbgValueInst>(I));
-  DbgValueInst &DVI = *cast<DbgValueInst>(I);
-
-  // Does this dbg.value refer to a sunk address calculation?
-  bool AnyChange = false;
-  SmallDenseSet<Value *> LocationOps(DVI.location_ops().begin(),
-                                     DVI.location_ops().end());
-  for (Value *Location : LocationOps) {
-    WeakTrackingVH SunkAddrVH = SunkAddrs[Location];
-    Value *SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;
-    if (SunkAddr) {
-      // Point dbg.value at locally computed address, which should give the best
-      // opportunity to be accurately lowered. This update may change the type
-      // of pointer being referred to; however this makes no difference to
-      // debugging information, and we can't generate bitcasts that may affect
-      // codegen.
-      DVI.replaceVariableLocationOp(Location, SunkAddr);
-      AnyChange = true;
-    }
-  }
-  return AnyChange;
-}
-
 bool CodeGenPrepare::fixupDbgVariableRecordsOnInst(Instruction &I) {
   bool AnyChange = false;
   for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
@@ -8993,14 +8959,6 @@ bool CodeGenPrepare::fixupDbgVariableRecord(DbgVariableRecord &DVR) {
   return AnyChange;
 }
 
-static void DbgInserterHelper(DbgValueInst *DVI, BasicBlock::iterator VI) {
-  DVI->removeFromParent();
-  if (isa<PHINode>(VI))
-    DVI->insertBefore(VI->getParent()->getFirstInsertionPt());
-  else
-    DVI->insertAfter(VI);
-}
-
 static void DbgInserterHelper(DbgVariableRecord *DVR, BasicBlock::iterator VI) {
   DVR->removeFromParent();
   BasicBlock *VIBB = VI->getParent();
@@ -9065,15 +9023,8 @@ bool CodeGenPrepare::placeDbgValues(Function &F) {
 
   for (BasicBlock &BB : F) {
     for (Instruction &Insn : llvm::make_early_inc_range(BB)) {
-      // Process dbg.value intrinsics.
-      DbgValueInst *DVI = dyn_cast<DbgValueInst>(&Insn);
-      if (DVI) {
-        DbgProcessor(DVI, DVI);
-        continue;
-      }
-
-      // If this isn't a dbg.value, process any attached DbgVariableRecord
-      // records attached to this instruction.
+      // Process any DbgVariableRecord records attached to this
+      // instruction.
       for (DbgVariableRecord &DVR : llvm::make_early_inc_range(
                filterDbgVars(Insn.getDbgRecordRange()))) {
         if (DVR.Type != DbgVariableRecord::LocationType::Value)
diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index 9daacfd399787..e7fa0824fd98a 100644
--- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -202,8 +202,8 @@ MachineBlockFrequencyInfo::MachineBlockFrequencyInfo(
     MachineBlockFrequencyInfo &&) = default;
 
 MachineBlockFrequencyInfo::MachineBlockFrequencyInfo(
-    MachineFunction &F, MachineBranchProbabilityInfo &MBPI,
-    MachineLoopInfo &MLI) {
+    const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
+    const MachineLoopInfo &MLI) {
   calculate(F, MBPI, MLI);
 }
 
diff --git a/llvm/lib/CodeGen/MachineDebugify.cpp b/llvm/lib/CodeGen/MachineDebugify.cpp
index 9b9cebc74054d..1a20fe586e951 100644
--- a/llvm/lib/CodeGen/MachineDebugify.cpp
+++ b/llvm/lib/CodeGen/MachineDebugify.cpp
@@ -63,24 +63,9 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
   // which cover a wide range of lines can help stress the debug info passes:
   // if we can't do that, fall back to using the local variable which precedes
   // all the others.
-  Function *DbgValF = M.getFunction("llvm.dbg.value");
-  DbgValueInst *EarliestDVI = nullptr;
   DbgVariableRecord *EarliestDVR = nullptr;
   DenseMap<unsigned, DILocalVariable *> Line2Var;
   DIExpression *Expr = nullptr;
-  if (DbgValF) {
-    for (const Use &U : DbgValF->uses()) {
-      auto *DVI = dyn_cast<DbgValueInst>(U.getUser());
-      if (!DVI || DVI->getFunction() != &F)
-        continue;
-      unsigned Line = DVI->getDebugLoc().getLine();
-      assert(Line != 0 && "debugify should not insert line 0 locations");
-      Line2Var[Line] = DVI->getVariable();
-      if (!EarliestDVI || Line < EarliestDVI->getDebugLoc().getLine())
-        EarliestDVI = DVI;
-      Expr = DVI->getExpression();
-    }
-  }
   for (BasicBlock &BB : F) {
     for (Instruction &I : BB) {
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
@@ -125,8 +110,7 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
       unsigned Line = MI.getDebugLoc().getLine();
       auto It = Line2Var.find(Line);
       if (It == Line2Var.end()) {
-        Line = EarliestDVI ? EarliestDVI->getDebugLoc().getLine()
-                           : EarliestDVR->getDebugLoc().getLine();
+        Line = EarliestDVR->getDebugLoc().getLine();
         It = Line2Var.find(Line);
         assert(It != Line2Var.end());
       }
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 40464e91f9efc..b7e41427f2c27 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7641,7 +7641,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
 
     if (SDValue(GN0, 0).hasOneUse() &&
         isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
-        TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+        TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
       SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
                        GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
 
@@ -15745,7 +15745,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   // fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
   if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
     if (SDValue(GN0, 0).hasOneUse() && ExtVT == GN0->getMemoryVT() &&
-        TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+        TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
       SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
                        GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
 
@@ -18727,6 +18727,12 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
   if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
     return FTrunc;
 
+  // fold (sint_to_fp (trunc nsw x)) -> (sint_to_fp x)
+  if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoSignedWrap() &&
+      TLI.isTypeDesirableForOp(ISD::SINT_TO_FP,
+                               N0.getOperand(0).getValueType()))
+    return DAG.getNode(ISD::SINT_TO_FP, DL, VT, N0.getOperand(0));
+
   return SDValue();
 }
 
@@ -18764,6 +18770,12 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
   if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
     return FTrunc;
 
+  // fold (uint_to_fp (trunc nuw x)) -> (uint_to_fp x)
+  if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoUnsignedWrap() &&
+      TLI.isTypeDesirableForOp(ISD::UINT_TO_FP,
+                               N0.getOperand(0).getValueType()))
+    return DAG.getNode(ISD::UINT_TO_FP, DL, VT, N0.getOperand(0));
+
   return SDValue();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 56c8bb441ddf8..5453828177c72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6751,7 +6751,9 @@ SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
     return SDValue();
   int64_t Offset = C2->getSExtValue();
   switch (Opcode) {
-  case ISD::ADD: break;
+  case ISD::ADD:
+  case ISD::PTRADD:
+    break;
   case ISD::SUB: Offset = -uint64_t(Offset); break;
   default: return SDValue();
   }
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 84a56058de834..8fb33c30e5cac 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -2288,39 +2288,36 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
   // Collect a map of {backing storage : dbg.declares} (currently "backing
   // storage" is limited to Allocas). We'll use this to find dbg.declares to
   // delete after running `trackAssignments`.
-  DenseMap<const AllocaInst *, SmallPtrSet<DbgDeclareInst *, 2>> DbgDeclares;
   DenseMap<const AllocaInst *, SmallPtrSet<DbgVariableRecord *, 2>> DVRDeclares;
   // Create another similar map of {storage : variables} that we'll pass to
   // trackAssignments.
   StorageToVarsMap Vars;
-  auto ProcessDeclare = [&](auto *Declare, auto &DeclareList) {
+  auto ProcessDeclare = [&](DbgVariableRecord &Declare) {
     // FIXME: trackAssignments doesn't let you specify any modifiers to the
     // variable (e.g. fragment) or location (e.g. offset), so we have to
     // leave dbg.declares with non-empty expressions in place.
-    if (Declare->getExpression()->getNumElements() != 0)
+    if (Declare.getExpression()->getNumElements() != 0)
       return;
-    if (!Declare->getAddress())
+    if (!Declare.getAddress())
       return;
     if (AllocaInst *Alloca =
-            dyn_cast<AllocaInst>(Declare->getAddress()->stripPointerCasts())) {
+            dyn_cast<AllocaInst>(Declare.getAddress()->stripPointerCasts())) {
       // FIXME: Skip VLAs for now (let these variables use dbg.declares).
       if (!Alloca->isStaticAlloca())
         return;
       // Similarly, skip scalable vectors (use dbg.declares instead).
       if (auto Sz = Alloca->getAllocationSize(*DL); Sz && Sz->isScalable())
         return;
-      DeclareList[Alloca].insert(Declare);
-      Vars[Alloca].insert(VarRecord(Declare));
+      DVRDeclares[Alloca].insert(&Declare);
+      Vars[Alloca].insert(VarRecord(&Declare));
     }
   };
   for (auto &BB : F) {
     for (auto &I : BB) {
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
         if (DVR.isDbgDeclare())
-          ProcessDeclare(&DVR, DVRDeclares);
+          ProcessDeclare(DVR);
       }
-      if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(&I))
-        ProcessDeclare(DDI, DbgDeclares);
     }
   }
 
@@ -2336,8 +2333,8 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
   trackAssignments(F.begin(), F.end(), Vars, *DL);
 
   // Delete dbg.declares for variables now tracked with assignment tracking.
-  auto DeleteSubsumedDeclare = [&](const auto &Markers, auto &Declares) {
-    (void)Markers;
+  for (auto &[Insts, Declares] : DVRDeclares) {
+    auto Markers = at::getDVRAssignmentMarkers(Insts);
     for (auto *Declare : Declares) {
       // Assert that the alloca that Declare uses is now linked to a dbg.assign
       // describing the same variable (i.e. check that this dbg.declare has
@@ -2356,10 +2353,6 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
       Changed = true;
     }
   };
-  for (auto &P : DbgDeclares)
-    DeleteSubsumedDeclare(at::getAssignmentMarkers(P.first), P.second);
-  for (auto &P : DVRDeclares)
-    DeleteSubsumedDeclare(at::getDVRAssignmentMarkers(P.first), P.second);
   return Changed;
 }
 
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 2270923bd3719..f16963dce56e1 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -49,20 +49,11 @@ uint32_t DIType::getAlignInBits() const {
 const DIExpression::FragmentInfo DebugVariable::DefaultFragment = {
     std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::min()};
 
-DebugVariable::DebugVariable(const DbgVariableIntrinsic *DII)
-    : Variable(DII->getVariable()),
-      Fragment(DII->getExpression()->getFragmentInfo()),
-      InlinedAt(DII->getDebugLoc().getInlinedAt()) {}
-
 DebugVariable::DebugVariable(const DbgVariableRecord *DVR)
     : Variable(DVR->getVariable()),
       Fragment(DVR->getExpression()->getFragmentInfo()),
       InlinedAt(DVR->getDebugLoc().getInlinedAt()) {}
 
-DebugVariableAggregate::DebugVariableAggregate(const DbgVariableIntrinsic *DVI)
-    : DebugVariable(DVI->getVariable(), std::nullopt,
-                    DVI->getDebugLoc()->getInlinedAt()) {}
-
 DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
                        unsigned Column, uint64_t AtomGroup, uint8_t AtomRank,
                        ArrayRef<Metadata *> MDs, bool ImplicitCode)
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 79eeb08cddeef..2579fa37935f0 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -54,11 +54,6 @@ static const char *PreservedSymbols[] = {
     "__stack_chk_guard",
 };
 
-static bool isPreservedGlobalVarName(StringRef Name) {
-  return StringRef(PreservedSymbols[0]) == Name ||
-         StringRef(PreservedSymbols[1]) == Name;
-}
-
 namespace {
 
 const char *getExpectedProducerName() {
@@ -86,16 +81,12 @@ struct Builder {
   // The StringTableBuilder does not create a copy of any strings added to it,
   // so this provides somewhere to store any strings that we create.
   Builder(SmallVector<char, 0> &Symtab, StringTableBuilder &StrtabBuilder,
-          BumpPtrAllocator &Alloc, const Triple &TT)
-      : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc), TT(TT),
-        Libcalls(TT) {}
+          BumpPtrAllocator &Alloc)
+      : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc) {}
 
   DenseMap<const Comdat *, int> ComdatMap;
   Mangler Mang;
-  const Triple &TT;
-
-  // FIXME: This shouldn't be here.
-  RTLIB::RuntimeLibcallsInfo Libcalls;
+  Triple TT;
 
   std::vector<storage::Comdat> Comdats;
   std::vector<storage::Module> Mods;
@@ -107,10 +98,6 @@ struct Builder {
 
   std::vector<storage::Str> DependentLibraries;
 
-  bool isPreservedLibFuncName(StringRef Name) {
-    return Libcalls.getSupportedLibcallImpl(Name) != RTLIB::Unsupported;
-  }
-
   void setStr(storage::Str &S, StringRef Value) {
     S.Offset = StrtabBuilder.add(Value);
     S.Size = Value.size();
@@ -226,6 +213,18 @@ Expected<int> Builder::getComdatIndex(const Comdat *C, const Module *M) {
   return P.first->second;
 }
 
+static DenseSet<StringRef> buildPreservedSymbolsSet(const Triple &TT) {
+  DenseSet<StringRef> PreservedSymbolSet(std::begin(PreservedSymbols),
+                                         std::end(PreservedSymbols));
+  // FIXME: Do we need to pass in ABI fields from TargetOptions?
+  RTLIB::RuntimeLibcallsInfo Libcalls(TT);
+  for (RTLIB::LibcallImpl Impl : Libcalls.getLibcallImpls()) {
+    if (Impl != RTLIB::Unsupported)
+      PreservedSymbolSet.insert(Libcalls.getLibcallImplName(Impl));
+  }
+  return PreservedSymbolSet;
+}
+
 Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
                          const SmallPtrSet<GlobalValue *, 4> &Used,
                          ModuleSymbolTable::Symbol Msym) {
@@ -279,11 +278,13 @@ Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
     return Error::success();
   }
 
-  StringRef GVName = GV->getName();
-  setStr(Sym.IRName, GVName);
+  setStr(Sym.IRName, GV->getName());
+
+  static const DenseSet<StringRef> PreservedSymbolsSet =
+      buildPreservedSymbolsSet(GV->getParent()->getTargetTriple());
+  bool IsPreservedSymbol = PreservedSymbolsSet.contains(GV->getName());
 
-  if (Used.count(GV) || isPreservedLibFuncName(GVName) ||
-      isPreservedGlobalVarName(GVName))
+  if (Used.count(GV) || IsPreservedSymbol)
     Sym.Flags |= 1 << storage::Symbol::FB_used;
   if (GV->isThreadLocal())
     Sym.Flags |= 1 << storage::Symbol::FB_tls;
@@ -350,6 +351,7 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
   setStr(Hdr.Producer, kExpectedProducerName);
   setStr(Hdr.TargetTriple, IRMods[0]->getTargetTriple().str());
   setStr(Hdr.SourceFileName, IRMods[0]->getSourceFileName());
+  TT = IRMods[0]->getTargetTriple();
 
   for (auto *M : IRMods)
     if (Error Err = addModule(M))
@@ -375,8 +377,7 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
 Error irsymtab::build(ArrayRef<Module *> Mods, SmallVector<char, 0> &Symtab,
                       StringTableBuilder &StrtabBuilder,
                       BumpPtrAllocator &Alloc) {
-  const Triple &TT = Mods[0]->getTargetTriple();
-  return Builder(Symtab, StrtabBuilder, Alloc, TT).build(Mods);
+  return Builder(Symtab, StrtabBuilder, Alloc).build(Mods);
 }
 
 // Upgrade a vector of bitcode modules created by an old version of LLVM by
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d04e6c45e2103..f026726c3f484 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -6439,7 +6439,9 @@ bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
     }
   }
 
-  return true;
+  EVT PreExtScalarVT = ExtVal->getOperand(0).getValueType().getScalarType();
+  return PreExtScalarVT == MVT::i8 || PreExtScalarVT == MVT::i16 ||
+         PreExtScalarVT == MVT::i32 || PreExtScalarVT == MVT::i64;
 }
 
 unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 23f106a9c1d4d..007b481f84960 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -153,6 +153,9 @@ struct AMDGPULowerBufferFatPointersPass
   const TargetMachine &TM;
 };
 
+void initializeAMDGPUPrepareAGPRAllocLegacyPass(PassRegistry &);
+extern char &AMDGPUPrepareAGPRAllocLegacyID;
+
 void initializeAMDGPUReserveWWMRegsLegacyPass(PassRegistry &);
 extern char &AMDGPUReserveWWMRegsLegacyID;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 250547acb1ee7..b6c6d927d0e89 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -114,6 +114,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUse
 MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
 MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
 MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
+MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass())
 MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
 MACHINE_FUNCTION_PASS("amdgpu-wait-sgpr-hazards", AMDGPUWaitSGPRHazardsPass())
 MACHINE_FUNCTION_PASS("gcn-create-vopd", GCNCreateVOPDPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
new file mode 100644
index 0000000000000..3b06e9b00ac69
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
@@ -0,0 +1,108 @@
+//===-- AMDGPUPrepareAGPRAlloc.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Make simple transformations to relax register constraints for cases which can
+// allocate to AGPRs or VGPRs. Replace materialize of inline immediates into
+// AGPR or VGPR with a pseudo with an AV_* class register constraint. This
+// allows later passes to inflate the register class if necessary. The register
+// allocator does not know to replace instructions to relax constraints.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUPrepareAGPRAlloc.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-prepare-agpr-alloc"
+
+namespace {
+
+class AMDGPUPrepareAGPRAllocImpl {
+private:
+  const SIInstrInfo &TII;
+  MachineRegisterInfo &MRI;
+
+public:
+  AMDGPUPrepareAGPRAllocImpl(const GCNSubtarget &ST, MachineRegisterInfo &MRI)
+      : TII(*ST.getInstrInfo()), MRI(MRI) {}
+  bool run(MachineFunction &MF);
+};
+
+class AMDGPUPrepareAGPRAllocLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  AMDGPUPrepareAGPRAllocLegacy() : MachineFunctionPass(ID) {
+    initializeAMDGPUPrepareAGPRAllocLegacyPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override { return "AMDGPU Prepare AGPR Alloc"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(AMDGPUPrepareAGPRAllocLegacy, DEBUG_TYPE,
+                      "AMDGPU Prepare AGPR Alloc", false, false)
+INITIALIZE_PASS_END(AMDGPUPrepareAGPRAllocLegacy, DEBUG_TYPE,
+                    "AMDGPU Prepare AGPR Alloc", false, false)
+
+char AMDGPUPrepareAGPRAllocLegacy::ID = 0;
+
+char &llvm::AMDGPUPrepareAGPRAllocLegacyID = AMDGPUPrepareAGPRAllocLegacy::ID;
+
+bool AMDGPUPrepareAGPRAllocLegacy::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  return AMDGPUPrepareAGPRAllocImpl(ST, MF.getRegInfo()).run(MF);
+}
+
+PreservedAnalyses
+AMDGPUPrepareAGPRAllocPass::run(MachineFunction &MF,
+                                MachineFunctionAnalysisManager &MFAM) {
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  AMDGPUPrepareAGPRAllocImpl(ST, MF.getRegInfo()).run(MF);
+  return PreservedAnalyses::all();
+}
+
+bool AMDGPUPrepareAGPRAllocImpl::run(MachineFunction &MF) {
+  if (MRI.isReserved(AMDGPU::AGPR0))
+    return false;
+
+  const MCInstrDesc &AVImmPseudo = TII.get(AMDGPU::AV_MOV_B32_IMM_PSEUDO);
+
+  bool Changed = false;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      if ((MI.getOpcode() == AMDGPU::V_MOV_B32_e32 &&
+           TII.isInlineConstant(MI, 1)) ||
+          (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
+           MI.getOperand(1).isImm())) {
+        MI.setDesc(AVImmPseudo);
+        Changed = true;
+      }
+    }
+  }
+
+  return Changed;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
new file mode 100644
index 0000000000000..dc598c98f241b
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
@@ -0,0 +1,23 @@
+//===- AMDGPUPrepareAGPRAlloc.h ---------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class AMDGPUPrepareAGPRAllocPass
+    : public PassInfoMixin<AMDGPUPrepareAGPRAllocPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
index 46027b8890234..8101c68986241 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
@@ -167,77 +167,39 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
 
   Info.UsesVCC =
       MRI.isPhysRegUsed(AMDGPU::VCC_LO) || MRI.isPhysRegUsed(AMDGPU::VCC_HI);
+  Info.NumExplicitSGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::SGPR_32RegClass,
+                                                /*IncludeCalls=*/false);
+  if (ST.hasMAIInsts())
+    Info.NumAGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass,
+                                          /*IncludeCalls=*/false);
 
   // If there are no calls, MachineRegisterInfo can tell us the used register
   // count easily.
   // A tail call isn't considered a call for MachineFrameInfo's purposes.
   if (!FrameInfo.hasCalls() && !FrameInfo.hasTailCall()) {
-    Info.NumVGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass);
-    Info.NumExplicitSGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::SGPR_32RegClass);
-    if (ST.hasMAIInsts())
-      Info.NumAGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass);
+    Info.NumVGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass,
+                                          /*IncludeCalls=*/false);
     return Info;
   }
 
   int32_t MaxVGPR = -1;
-  int32_t MaxAGPR = -1;
-  int32_t MaxSGPR = -1;
   Info.CalleeSegmentSize = 0;
 
   for (const MachineBasicBlock &MBB : MF) {
     for (const MachineInstr &MI : MBB) {
-      // TODO: Check regmasks? Do they occur anywhere except calls?
-      for (const MachineOperand &MO : MI.operands()) {
-        unsigned Width = 0;
-        bool IsSGPR = false;
-        bool IsAGPR = false;
+      for (unsigned I = 0; I < MI.getNumOperands(); ++I) {
+        const MachineOperand &MO = MI.getOperand(I);
 
         if (!MO.isReg())
           continue;
 
         Register Reg = MO.getReg();
         switch (Reg) {
-        case AMDGPU::EXEC:
-        case AMDGPU::EXEC_LO:
-        case AMDGPU::EXEC_HI:
-        case AMDGPU::SCC:
-        case AMDGPU::M0:
-        case AMDGPU::M0_LO16:
-        case AMDGPU::M0_HI16:
-        case AMDGPU::SRC_SHARED_BASE_LO:
-        case AMDGPU::SRC_SHARED_BASE:
-        case AMDGPU::SRC_SHARED_LIMIT_LO:
-        case AMDGPU::SRC_SHARED_LIMIT:
-        case AMDGPU::SRC_PRIVATE_BASE_LO:
-        case AMDGPU::SRC_PRIVATE_BASE:
-        case AMDGPU::SRC_PRIVATE_LIMIT_LO:
-        case AMDGPU::SRC_PRIVATE_LIMIT:
-        case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
-        case AMDGPU::SGPR_NULL:
-        case AMDGPU::SGPR_NULL64:
-        case AMDGPU::MODE:
-          continue;
-
         case AMDGPU::NoRegister:
           assert(MI.isDebugInstr() &&
                  "Instruction uses invalid noreg register");
           continue;
 
-        case AMDGPU::VCC:
-        case AMDGPU::VCC_LO:
-        case AMDGPU::VCC_HI:
-        case AMDGPU::VCC_LO_LO16:
-        case AMDGPU::VCC_LO_HI16:
-        case AMDGPU::VCC_HI_LO16:
-        case AMDGPU::VCC_HI_HI16:
-          Info.UsesVCC = true;
-          continue;
-
-        case AMDGPU::FLAT_SCR:
-        case AMDGPU::FLAT_SCR_LO:
-        case AMDGPU::FLAT_SCR_HI:
-          continue;
-
         case AMDGPU::XNACK_MASK:
         case AMDGPU::XNACK_MASK_LO:
         case AMDGPU::XNACK_MASK_HI:
@@ -267,170 +229,22 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
           break;
         }
 
-        if (AMDGPU::SGPR_32RegClass.contains(Reg) ||
-            AMDGPU::SGPR_LO16RegClass.contains(Reg) ||
-            AMDGPU::SGPR_HI16RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 1;
-        } else if (AMDGPU::VGPR_32RegClass.contains(Reg) ||
-                   AMDGPU::VGPR_16RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 1;
-        } else if (AMDGPU::AGPR_32RegClass.contains(Reg) ||
-                   AMDGPU::AGPR_LO16RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 1;
-        } else if (AMDGPU::SGPR_64RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 2;
-        } else if (AMDGPU::VReg_64RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 2;
-        } else if (AMDGPU::AReg_64RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 2;
-        } else if (AMDGPU::VReg_96RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 3;
-        } else if (AMDGPU::SReg_96RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 3;
-        } else if (AMDGPU::AReg_96RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 3;
-        } else if (AMDGPU::SGPR_128RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 4;
-        } else if (AMDGPU::VReg_128RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 4;
-        } else if (AMDGPU::AReg_128RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 4;
-        } else if (AMDGPU::VReg_160RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 5;
-        } else if (AMDGPU::SReg_160RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 5;
-        } else if (AMDGPU::AReg_160RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 5;
-        } else if (AMDGPU::VReg_192RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 6;
-        } else if (AMDGPU::SReg_192RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 6;
-        } else if (AMDGPU::AReg_192RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 6;
-        } else if (AMDGPU::VReg_224RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 7;
-        } else if (AMDGPU::SReg_224RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 7;
-        } else if (AMDGPU::AReg_224RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 7;
-        } else if (AMDGPU::SReg_256RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 8;
-        } else if (AMDGPU::VReg_256RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 8;
-        } else if (AMDGPU::AReg_256RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 8;
-        } else if (AMDGPU::VReg_288RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 9;
-        } else if (AMDGPU::SReg_288RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 9;
-        } else if (AMDGPU::AReg_288RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 9;
-        } else if (AMDGPU::VReg_320RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 10;
-        } else if (AMDGPU::SReg_320RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 10;
-        } else if (AMDGPU::AReg_320RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 10;
-        } else if (AMDGPU::VReg_352RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 11;
-        } else if (AMDGPU::SReg_352RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 11;
-        } else if (AMDGPU::AReg_352RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 11;
-        } else if (AMDGPU::VReg_384RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 12;
-        } else if (AMDGPU::SReg_384RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 12;
-        } else if (AMDGPU::AReg_384RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 12;
-        } else if (AMDGPU::SReg_512RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 16;
-        } else if (AMDGPU::VReg_512RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 16;
-        } else if (AMDGPU::AReg_512RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 16;
-        } else if (AMDGPU::SReg_1024RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 32;
-        } else if (AMDGPU::VReg_1024RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 32;
-        } else if (AMDGPU::AReg_1024RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 32;
-        } else {
-          // We only expect TTMP registers or registers that do not belong to
-          // any RC.
-          assert((AMDGPU::TTMP_32RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_64RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_128RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_256RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_512RegClass.contains(Reg) ||
-                  !TRI.getPhysRegBaseClass(Reg)) &&
-                 "Unknown register class");
-        }
+        const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg);
+        assert((!RC || TRI.isVGPRClass(RC) || TRI.isSGPRClass(RC) ||
+                TRI.isAGPRClass(RC) || AMDGPU::TTMP_32RegClass.contains(Reg) ||
+                AMDGPU::TTMP_64RegClass.contains(Reg) ||
+                AMDGPU::TTMP_128RegClass.contains(Reg) ||
+                AMDGPU::TTMP_256RegClass.contains(Reg) ||
+                AMDGPU::TTMP_512RegClass.contains(Reg)) &&
+               "Unknown register class");
+
+        if (!RC || !TRI.isVGPRClass(RC))
+          continue;
+
+        unsigned Width = divideCeil(TRI.getRegSizeInBits(*RC), 32);
         unsigned HWReg = TRI.getHWRegIndex(Reg);
         int MaxUsed = HWReg + Width - 1;
-        if (IsSGPR) {
-          MaxSGPR = MaxUsed > MaxSGPR ? MaxUsed : MaxSGPR;
-        } else if (IsAGPR) {
-          MaxAGPR = MaxUsed > MaxAGPR ? MaxUsed : MaxAGPR;
-        } else {
-          MaxVGPR = MaxUsed > MaxVGPR ? MaxUsed : MaxVGPR;
-        }
+        MaxVGPR = std::max(MaxUsed, MaxVGPR);
       }
 
       if (MI.isCall()) {
@@ -492,9 +306,7 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
     }
   }
 
-  Info.NumExplicitSGPR = MaxSGPR + 1;
   Info.NumVGPR = MaxVGPR + 1;
-  Info.NumAGPR = MaxAGPR + 1;
 
   return Info;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 31a80e00edd3b..c865082a1dcea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -25,6 +25,7 @@
 #include "AMDGPUMacroFusion.h"
 #include "AMDGPUPerfHintAnalysis.h"
 #include "AMDGPUPreloadKernArgProlog.h"
+#include "AMDGPUPrepareAGPRAlloc.h"
 #include "AMDGPURemoveIncompatibleFunctions.h"
 #include "AMDGPUReserveWWMRegs.h"
 #include "AMDGPUResourceUsageAnalysis.h"
@@ -499,6 +500,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeGlobalISel(*PR);
   initializeAMDGPUAsmPrinterPass(*PR);
   initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
+  initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
   initializeGCNDPPCombineLegacyPass(*PR);
   initializeSILowerI1CopiesLegacyPass(*PR);
   initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);
@@ -1196,6 +1198,7 @@ class GCNPassConfig final : public AMDGPUPassConfig {
   bool addRegBankSelect() override;
   void addPreGlobalInstructionSelect() override;
   bool addGlobalInstructionSelect() override;
+  void addPreRegAlloc() override;
   void addFastRegAlloc() override;
   void addOptimizedRegAlloc() override;
 
@@ -1539,6 +1542,11 @@ void GCNPassConfig::addFastRegAlloc() {
   TargetPassConfig::addFastRegAlloc();
 }
 
+void GCNPassConfig::addPreRegAlloc() {
+  if (getOptLevel() != CodeGenOptLevel::None)
+    addPass(&AMDGPUPrepareAGPRAllocLegacyID);
+}
+
 void GCNPassConfig::addOptimizedRegAlloc() {
   if (EnableDCEInRA)
     insertPass(&DetectDeadLanesID, &DeadMachineInstructionElimID);
@@ -2235,6 +2243,11 @@ void AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
   Base::addOptimizedRegAlloc(addPass);
 }
 
+void AMDGPUCodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const {
+  if (getOptLevel() != CodeGenOptLevel::None)
+    addPass(AMDGPUPrepareAGPRAllocPass());
+}
+
 Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
     AddMachinePass &addPass) const {
   // TODO: Check --regalloc-npm option
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 3b2f39c14a9bc..e0f1296ddded8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -181,7 +181,9 @@ class AMDGPUCodeGenPassBuilder
   void addMachineSSAOptimization(AddMachinePass &) const;
   void addPostRegAlloc(AddMachinePass &) const;
   void addPreEmitPass(AddMachinePass &) const;
+  void addPreEmitRegAlloc(AddMachinePass &) const;
   Error addRegAssignmentOptimized(AddMachinePass &) const;
+  void addPreRegAlloc(AddMachinePass &) const;
   void addOptimizedRegAlloc(AddMachinePass &) const;
   void addPreSched2(AddMachinePass &) const;
 
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index e3519f192137c..42edec0d01493 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -74,6 +74,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPULowerKernelArguments.cpp
   AMDGPULowerKernelAttributes.cpp
   AMDGPULowerModuleLDSPass.cpp
+  AMDGPUPrepareAGPRAlloc.cpp
   AMDGPUSwLowerLDS.cpp
   AMDGPUMachineFunction.cpp
   AMDGPUMachineModuleInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 400795c29b0e4..0c76ff2ec5ea7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -33,6 +33,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -46,6 +47,7 @@
 #include <optional>
 
 using namespace llvm;
+using namespace llvm::SDPatternMatch;
 
 #define DEBUG_TYPE "si-lower"
 
@@ -14561,7 +14563,7 @@ static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
 // instead of a tree.
 SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
                                             DAGCombinerInfo &DCI) const {
-  assert(N->getOpcode() == ISD::ADD);
+  assert(N->isAnyAdd());
 
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
@@ -14594,7 +14596,7 @@ SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
     for (SDNode *User : LHS->users()) {
       // There is a use that does not feed into addition, so the multiply can't
       // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
-      if (User->getOpcode() != ISD::ADD)
+      if (!User->isAnyAdd())
         return SDValue();
 
       // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
@@ -14706,8 +14708,11 @@ SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
 
     SDValue Hi = getHiHalf64(LHS, DAG);
     SDValue ConstHi32 = DAG.getConstant(Hi_32(Val), SL, MVT::i32);
+    unsigned Opcode = N->getOpcode();
+    if (Opcode == ISD::PTRADD)
+      Opcode = ISD::ADD;
     SDValue AddHi =
-        DAG.getNode(N->getOpcode(), SL, MVT::i32, Hi, ConstHi32, N->getFlags());
+        DAG.getNode(Opcode, SL, MVT::i32, Hi, ConstHi32, N->getFlags());
 
     SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
     return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, Lo, AddHi);
@@ -15181,42 +15186,123 @@ SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
+  EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
-  if (N1.getOpcode() == ISD::ADD) {
-    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
-    //    y is not, and (add y, z) is used only once.
-    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
-    //    z is not, and (add y, z) is used only once.
-    // The goal is to move constant offsets to the outermost ptradd, to create
-    // more opportunities to fold offsets into memory instructions.
-    // Together with the generic combines in DAGCombiner.cpp, this also
-    // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
-    //
-    // This transform is here instead of in the general DAGCombiner as it can
-    // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
-    // AArch64's CPA.
-    SDValue X = N0;
-    SDValue Y = N1.getOperand(0);
-    SDValue Z = N1.getOperand(1);
-    if (N1.hasOneUse()) {
-      bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
-      bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
-      if (ZIsConstant != YIsConstant) {
-        // If both additions in the original were NUW, the new ones are as well.
-        SDNodeFlags Flags =
-            (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
-        if (YIsConstant)
-          std::swap(Y, Z);
+  // The following folds transform PTRADDs into regular arithmetic in cases
+  // where the PTRADD wouldn't be folded as an immediate offset into memory
+  // instructions anyway. They are target-specific in that other targets might
+  // prefer to not lose information about the pointer arithmetic.
+
+  // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
+  // Adapted from DAGCombiner::visitADDLikeCommutative.
+  SDValue V, K;
+  if (sd_match(N1, m_Shl(m_Neg(m_Value(V)), m_Value(K)))) {
+    SDNodeFlags ShlFlags = N1->getFlags();
+    // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
+    // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
+    // preserved.
+    SDNodeFlags NewShlFlags =
+        ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
+            ? SDNodeFlags::NoSignedWrap
+            : SDNodeFlags();
+    SDValue Inner = DAG.getNode(ISD::SHL, DL, VT, V, K, NewShlFlags);
+    DCI.AddToWorklist(Inner.getNode());
+    return DAG.getNode(ISD::SUB, DL, VT, N0, Inner);
+  }
+
+  // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
+  // performAddCombine.
+  if (N1.getOpcode() == ISD::MUL) {
+    if (Subtarget->hasMad64_32()) {
+      if (SDValue Folded = tryFoldToMad64_32(N, DCI))
+        return Folded;
+    }
+  }
 
-        SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, Flags);
+  // If the 32 low bits of the constant are all zero, there is nothing to fold
+  // into an immediate offset, so it's better to eliminate the unnecessary
+  // addition for the lower 32 bits than to preserve the PTRADD.
+  // Analogous to a fold in performAddCombine.
+  if (VT == MVT::i64) {
+    if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
+      return Folded;
+  }
+
+  if (N0.getOpcode() == ISD::PTRADD && N1.getOpcode() == ISD::Constant) {
+    // Fold (ptradd (ptradd GA, v), c) -> (ptradd (ptradd GA, c) v) with
+    // global address GA and constant c, such that c can be folded into GA.
+    SDValue GAValue = N0.getOperand(0);
+    if (const GlobalAddressSDNode *GA =
+            dyn_cast<GlobalAddressSDNode>(GAValue)) {
+      if (DCI.isBeforeLegalizeOps() && isOffsetFoldingLegal(GA)) {
+        // If both additions in the original were NUW, reassociation preserves
+        // that.
+        SDNodeFlags Flags =
+            (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+        SDValue Inner = DAG.getMemBasePlusOffset(GAValue, N1, DL, Flags);
         DCI.AddToWorklist(Inner.getNode());
-        return DAG.getMemBasePlusOffset(Inner, Z, DL, Flags);
+        return DAG.getMemBasePlusOffset(Inner, N0.getOperand(1), DL, Flags);
       }
     }
   }
 
+  if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
+    return SDValue();
+
+  // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
+  //    y is not, and (add y, z) is used only once.
+  // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
+  //    z is not, and (add y, z) is used only once.
+  // The goal is to move constant offsets to the outermost ptradd, to create
+  // more opportunities to fold offsets into memory instructions.
+  // Together with the generic combines in DAGCombiner.cpp, this also
+  // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
+  //
+  // This transform is here instead of in the general DAGCombiner as it can
+  // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
+  // AArch64's CPA.
+  SDValue X = N0;
+  SDValue Y = N1.getOperand(0);
+  SDValue Z = N1.getOperand(1);
+  bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
+  bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
+
+  // If both additions in the original were NUW, reassociation preserves that.
+  SDNodeFlags ReassocFlags =
+      (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+
+  if (ZIsConstant != YIsConstant) {
+    if (YIsConstant)
+      std::swap(Y, Z);
+    SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
+    DCI.AddToWorklist(Inner.getNode());
+    return DAG.getMemBasePlusOffset(Inner, Z, DL, ReassocFlags);
+  }
+
+  // If one of Y and Z is constant, they have been handled above. If both were
+  // constant, the addition would have been folded in SelectionDAG::getNode
+  // already. This ensures that the generic DAG combines won't undo the
+  // following reassociation.
+  assert(!YIsConstant && !ZIsConstant);
+
+  if (!X->isDivergent() && Y->isDivergent() != Z->isDivergent()) {
+    // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
+    // y are uniform and z isn't.
+    // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
+    // z are uniform and y isn't.
+    // The goal is to push uniform operands up in the computation, so that they
+    // can be handled with scalar operations. We can't use reassociateScalarOps
+    // for this since it requires two identical commutative operations to
+    // reassociate.
+    if (Y->isDivergent())
+      std::swap(Y, Z);
+    SDValue UniformInner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
+    DCI.AddToWorklist(UniformInner.getNode());
+    return DAG.getMemBasePlusOffset(UniformInner, Z, DL, ReassocFlags);
+  }
+
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 2764ed3d3f0b1..5e92921f3ea21 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1113,7 +1113,6 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   // that will not require an additional 4-bytes; this function assumes that it
   // will.
   bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
-    assert(!MO.isReg() && "isInlineConstant called on register operand!");
     if (!MO.isImm())
       return false;
     return isInlineConstant(MO.getImm(), OperandType);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 9173041a7bccd..fa2b8db6ba55a 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4052,11 +4052,11 @@ SIRegisterInfo::getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
   return 0;
 }
 
-unsigned
-SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
-                                   const TargetRegisterClass &RC) const {
+unsigned SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
+                                            const TargetRegisterClass &RC,
+                                            bool IncludeCalls) const {
   for (MCPhysReg Reg : reverse(RC.getRegisters()))
-    if (MRI.isPhysRegUsed(Reg))
+    if (MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/!IncludeCalls))
       return getHWRegIndex(Reg) + 1;
   return 0;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 06a7a17b0246b..0008e5f8cf3b4 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -486,9 +486,11 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
                                      unsigned SubReg) const;
 
   // \returns a number of registers of a given \p RC used in a function.
-  // Does not go inside function calls.
+  // Does not go inside function calls. If \p IncludeCalls is true, it will
+  // include registers that may be clobbered by calls.
   unsigned getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
-                              const TargetRegisterClass &RC) const;
+                              const TargetRegisterClass &RC,
+                              bool IncludeCalls = true) const;
 
   std::optional<uint8_t> getVRegFlagValue(StringRef Name) const override {
     return Name == "WWM_REG" ? AMDGPU::VirtRegFlag::WWM_REG
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 1bbbb610305e9..3ee90857b34b8 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -1148,6 +1148,7 @@ defm V_MOV_B64 : VOP1_Real_FULL <GFX1250Gen, 0x1d>;
 defm V_TANH_F32              : VOP1_Real_FULL<GFX1250Gen, 0x01e>;
 defm V_TANH_F16              : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x01f>;
 defm V_TANH_BF16             : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x04a>;
+defm V_PRNG_B32              : VOP1_Real_FULL<GFX1250Gen, 0x04b>;
 defm V_CVT_F32_BF16          : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x072, "v_cvt_f32_bf16", "V_CVT_F32_BF16_gfx1250">;
 defm V_CVT_PK_F16_FP8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x075>;
 defm V_CVT_PK_F16_BF8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x076>;
diff --git a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
index ac5e7f3891c72..1493bf4cba695 100644
--- a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
@@ -158,7 +158,12 @@ void LoongArchFrameLowering::processFunctionBeforeFrameFinalized(
   // estimateStackSize has been observed to under-estimate the final stack
   // size, so give ourselves wiggle-room by checking for stack size
   // representable an 11-bit signed field rather than 12-bits.
-  if (!isInt<11>(MFI.estimateStackSize(MF)))
+  // For [x]vstelm.{b/h/w/d} memory instructions with 8 imm offset, 7-bit
+  // signed field is fine.
+  unsigned EstimateStackSize = MFI.estimateStackSize(MF);
+  if (!isInt<11>(EstimateStackSize) ||
+      (MF.getSubtarget<LoongArchSubtarget>().hasExtLSX() &&
+       !isInt<7>(EstimateStackSize)))
     ScavSlotsNum = std::max(ScavSlotsNum, 1u);
 
   // For CFR spill.
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 6c8e3da80b932..23b4554349003 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -95,6 +95,11 @@ static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
     /* -21, -22, -23, -24 are reserved */
 };
 
+/// Returns true if DWARF CFI instructions ("frame moves") should be emitted.
+static bool needsDwarfCFI(const MachineFunction &MF) {
+  return MF.needsFrameMoves();
+}
+
 // For now we use x3, a.k.a gp, as pointer to shadow call stack.
 // User should not use x3 in their asm.
 static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -141,6 +146,9 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
       .addImm(-SlotSize)
       .setMIFlag(MachineInstr::FrameSetup);
 
+  if (!needsDwarfCFI(MF))
+    return;
+
   // Emit a CFI instruction that causes SlotSize to be subtracted from the value
   // of the shadow stack pointer when unwinding past this frame.
   char DwarfSCSReg = TRI->getDwarfRegNum(SCSPReg, /*IsEH*/ true);
@@ -199,8 +207,10 @@ static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
       .addReg(SCSPReg)
       .addImm(-SlotSize)
       .setMIFlag(MachineInstr::FrameDestroy);
-  // Restore the SCS pointer
-  CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
+  if (needsDwarfCFI(MF)) {
+    // Restore the SCS pointer
+    CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
+  }
 }
 
 // Insert instruction to swap mscratchsw with sp
@@ -935,6 +945,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   MBBI = std::prev(MBBI, getRVVCalleeSavedInfo(MF, CSI).size() +
                              getUnmanagedCSI(MF, CSI).size());
   CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
+  bool NeedsDwarfCFI = needsDwarfCFI(MF);
 
   // If libcalls are used to spill and restore callee-saved registers, the frame
   // has two sections; the opaque section managed by the libcalls, and the
@@ -962,10 +973,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
         alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
     RVFI->setLibCallStackSize(LibCallFrameSize);
 
-    CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
-    for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-      CFIBuilder.buildOffset(CS.getReg(),
-                             MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
+      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+        CFIBuilder.buildOffset(CS.getReg(),
+                               MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   // FIXME (note copied from Lanai): This appears to be overallocating.  Needs
@@ -996,14 +1009,17 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     // could only be the next instruction.
     ++PossiblePush;
 
-    // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
-    // could be. The PUSH will also get its own CFI metadata for its own
-    // modifications, which should come after the PUSH.
-    CFIInstBuilder PushCFIBuilder(MBB, PossiblePush, MachineInstr::FrameSetup);
-    PushCFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
-    for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
-      PushCFIBuilder.buildOffset(CS.getReg(),
-                                 MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
+      // could be. The PUSH will also get its own CFI metadata for its own
+      // modifications, which should come after the PUSH.
+      CFIInstBuilder PushCFIBuilder(MBB, PossiblePush,
+                                    MachineInstr::FrameSetup);
+      PushCFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
+      for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
+        PushCFIBuilder.buildOffset(CS.getReg(),
+                                   MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   if (RVFI->isPushable(MF) && PossiblePush != MBB.end() &&
@@ -1017,10 +1033,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     PossiblePush->getOperand(1).setImm(StackAdj);
     StackSize -= StackAdj;
 
-    CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
-    for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-      CFIBuilder.buildOffset(CS.getReg(),
-                             MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
+      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+        CFIBuilder.buildOffset(CS.getReg(),
+                               MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   // Allocate space on the stack if necessary.
@@ -1031,7 +1049,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   bool DynAllocation =
       MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
   if (StackSize != 0)
-    allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, /*EmitCFI=*/true,
+    allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, NeedsDwarfCFI,
                   NeedProbe, ProbeSize, DynAllocation,
                   MachineInstr::FrameSetup);
 
@@ -1049,8 +1067,10 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
 
   // Iterate over list of callee-saved registers and emit .cfi_offset
   // directives.
-  for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
-    CFIBuilder.buildOffset(CS.getReg(), MFI.getObjectOffset(CS.getFrameIdx()));
+  if (NeedsDwarfCFI)
+    for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+      CFIBuilder.buildOffset(CS.getReg(),
+                             MFI.getObjectOffset(CS.getFrameIdx()));
 
   // Generate new FP.
   if (hasFP(MF)) {
@@ -1069,7 +1089,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
           MachineInstr::FrameSetup, getStackAlign());
     }
 
-    CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
+    if (NeedsDwarfCFI)
+      CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
   }
 
   uint64_t SecondSPAdjustAmount = 0;
@@ -1080,15 +1101,16 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
            "SecondSPAdjustAmount should be greater than zero");
 
     allocateStack(MBB, MBBI, MF, SecondSPAdjustAmount,
-                  getStackSizeWithRVVPadding(MF), !hasFP(MF), NeedProbe,
-                  ProbeSize, DynAllocation, MachineInstr::FrameSetup);
+                  getStackSizeWithRVVPadding(MF), NeedsDwarfCFI && !hasFP(MF),
+                  NeedProbe, ProbeSize, DynAllocation,
+                  MachineInstr::FrameSetup);
   }
 
   if (RVVStackSize) {
     if (NeedProbe) {
       allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, RVVStackSize,
-                                  MachineInstr::FrameSetup, !hasFP(MF),
-                                  DynAllocation);
+                                  MachineInstr::FrameSetup,
+                                  NeedsDwarfCFI && !hasFP(MF), DynAllocation);
     } else {
       // We must keep the stack pointer aligned through any intermediate
       // updates.
@@ -1097,14 +1119,15 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
                     MachineInstr::FrameSetup, getStackAlign());
     }
 
-    if (!hasFP(MF)) {
+    if (NeedsDwarfCFI && !hasFP(MF)) {
       // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
       CFIBuilder.insertCFIInst(createDefCFAExpression(
           *RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8));
     }
 
     std::advance(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
-    emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
+    if (NeedsDwarfCFI)
+      emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
   }
 
   if (hasFP(MF)) {
@@ -1171,8 +1194,9 @@ void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
                 MachineInstr::FrameDestroy, getStackAlign());
   StackSize = 0;
 
-  CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
-      .buildDefCFAOffset(CFAOffset);
+  if (needsDwarfCFI(MF))
+    CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
+        .buildDefCFAOffset(CFAOffset);
 }
 
 void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
@@ -1212,6 +1236,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
       std::next(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
   CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
                             MachineInstr::FrameDestroy);
+  bool NeedsDwarfCFI = needsDwarfCFI(MF);
 
   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
   uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
@@ -1232,10 +1257,11 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                     StackOffset::getScalable(RVVStackSize),
                     MachineInstr::FrameDestroy, getStackAlign());
 
-    if (!hasFP(MF))
-      CFIBuilder.buildDefCFA(SPReg, RealStackSize);
-
-    emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
+    if (NeedsDwarfCFI) {
+      if (!hasFP(MF))
+        CFIBuilder.buildDefCFA(SPReg, RealStackSize);
+      emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
+    }
   }
 
   if (FirstSPAdjustAmount) {
@@ -1251,7 +1277,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                     StackOffset::getFixed(SecondSPAdjustAmount),
                     MachineInstr::FrameDestroy, getStackAlign());
 
-    if (!hasFP(MF))
+    if (NeedsDwarfCFI && !hasFP(MF))
       CFIBuilder.buildDefCFAOffset(FirstSPAdjustAmount);
   }
 
@@ -1272,7 +1298,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                   getStackAlign());
   }
 
-  if (hasFP(MF))
+  if (NeedsDwarfCFI && hasFP(MF))
     CFIBuilder.buildDefCFA(SPReg, RealStackSize);
 
   // Skip to after the restores of scalar callee-saved registers
@@ -1295,8 +1321,9 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 
   // Recover callee-saved registers.
-  for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
-    CFIBuilder.buildRestore(CS.getReg());
+  if (NeedsDwarfCFI)
+    for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+      CFIBuilder.buildRestore(CS.getReg());
 
   if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(MBBI->getOpcode())) {
     // Use available stack adjustment in pop instruction to deallocate stack
@@ -1315,15 +1342,17 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
     auto NextI = next_nodbg(MBBI, MBB.end());
     if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
       ++MBBI;
-      CFIBuilder.setInsertPoint(MBBI);
+      if (NeedsDwarfCFI) {
+        CFIBuilder.setInsertPoint(MBBI);
 
-      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-        CFIBuilder.buildRestore(CS.getReg());
+        for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+          CFIBuilder.buildRestore(CS.getReg());
 
-      // Update CFA Offset. If this is a QCI interrupt function, there will be a
-      // leftover offset which is deallocated by `QC.C.MILEAVERET`, otherwise
-      // getQCIInterruptStackSize() will be 0.
-      CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
+        // Update CFA Offset. If this is a QCI interrupt function, there will
+        // be a leftover offset which is deallocated by `QC.C.MILEAVERET`,
+        // otherwise getQCIInterruptStackSize() will be 0.
+        CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
+      }
     }
   }
 
@@ -1812,7 +1841,8 @@ MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
         // allocateStack.
         bool DynAllocation =
             MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
-        allocateStack(MBB, MI, MF, -Amount, -Amount, !hasFP(MF),
+        allocateStack(MBB, MI, MF, -Amount, -Amount,
+                      needsDwarfCFI(MF) && !hasFP(MF),
                       /*NeedProbe=*/true, ProbeSize, DynAllocation,
                       MachineInstr::NoFlags);
       } else {
diff --git a/llvm/lib/Target/X86/X86CallingConv.cpp b/llvm/lib/Target/X86/X86CallingConv.cpp
index 82e8ce4e0bd7c..5d5a705893242 100644
--- a/llvm/lib/Target/X86/X86CallingConv.cpp
+++ b/llvm/lib/Target/X86/X86CallingConv.cpp
@@ -389,7 +389,7 @@ static bool CC_X86_32_I128_FP128(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
   if (!ArgFlags.isInConsecutiveRegsLast())
     return true;
 
-  assert(PendingMembers.size() == 4 && "Should have two parts");
+  assert(PendingMembers.size() == 4 && "Should have four parts");
 
   int64_t Offset = State.AllocateStack(16, Align(16));
   PendingMembers[0].convertToMem(Offset);
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 57fbc71fa22ee..9cd35e35d4bc9 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -176,10 +176,10 @@ constexpr FeatureBitset FeaturesArrowlakeS =
     FeaturesArrowlake | FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 |
     FeatureSM4;
 constexpr FeatureBitset FeaturesPantherlake =
-    FeaturesArrowlakeS ^ FeatureWIDEKL | FeaturePREFETCHI;
+    (FeaturesArrowlakeS ^ FeatureWIDEKL) | FeaturePREFETCHI;
 constexpr FeatureBitset FeaturesClearwaterforest =
-    FeaturesSierraforest ^ FeatureWIDEKL | FeatureAVXVNNIINT16 | FeatureSHA512 |
-    FeatureSM3 | FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
+    (FeaturesSierraforest ^ FeatureWIDEKL) | FeatureAVXVNNIINT16 |
+    FeatureSHA512 | FeatureSM3 | FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
 
 // Geode Processor.
 constexpr FeatureBitset FeaturesGeode =
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index fbeb7218ba9a3..a65d0fb54c212 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1103,14 +1103,13 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
               FrameTy->getElementType(FrameData.getFieldIndex(E.first)), GEP,
               SpillAlignment, E.first->getName() + Twine(".reload"));
 
-        TinyPtrVector<DbgDeclareInst *> DIs = findDbgDeclares(Def);
         TinyPtrVector<DbgVariableRecord *> DVRs = findDVRDeclares(Def);
         // Try best to find dbg.declare. If the spill is a temp, there may not
         // be a direct dbg.declare. Walk up the load chain to find one from an
         // alias.
         if (F->getSubprogram()) {
           auto *CurDef = Def;
-          while (DIs.empty() && DVRs.empty() && isa<LoadInst>(CurDef)) {
+          while (DVRs.empty() && isa<LoadInst>(CurDef)) {
             auto *LdInst = cast<LoadInst>(CurDef);
             // Only consider ptr to ptr same type load.
             if (LdInst->getPointerOperandType() != LdInst->getType())
@@ -1118,12 +1117,11 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
             CurDef = LdInst->getPointerOperand();
             if (!isa<AllocaInst, LoadInst>(CurDef))
               break;
-            DIs = findDbgDeclares(CurDef);
             DVRs = findDVRDeclares(CurDef);
           }
         }
 
-        auto SalvageOne = [&](auto *DDI) {
+        auto SalvageOne = [&](DbgVariableRecord *DDI) {
           // This dbg.declare is preserved for all coro-split function
           // fragments. It will be unreachable in the main function, and
           // processed by coro::salvageDebugInfo() by the Cloner.
@@ -1137,7 +1135,6 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
           // will be deleted in all coro-split functions.
           coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
         };
-        for_each(DIs, SalvageOne);
         for_each(DVRs, SalvageOne);
       }
 
@@ -1225,8 +1222,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
     SmallVector<DbgVariableIntrinsic *, 4> DIs;
     SmallVector<DbgVariableRecord *> DbgVariableRecords;
     findDbgUsers(DIs, Alloca, &DbgVariableRecords);
-    for (auto *DVI : DIs)
-      DVI->replaceUsesOfWith(Alloca, G);
+    assert(DIs.empty() && "Should never see debug-intrinsics");
     for (auto *DVR : DbgVariableRecords)
       DVR->replaceVariableLocationOp(Alloca, G);
 
@@ -1920,48 +1916,6 @@ salvageDebugInfoImpl(SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
   return {{*Storage, *Expr}};
 }
 
-void coro::salvageDebugInfo(
-    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
-    DbgVariableIntrinsic &DVI, bool UseEntryValue) {
-
-  Function *F = DVI.getFunction();
-  // Follow the pointer arithmetic all the way to the incoming
-  // function argument and convert into a DIExpression.
-  bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
-  Value *OriginalStorage = DVI.getVariableLocationOp(0);
-
-  auto SalvagedInfo =
-      ::salvageDebugInfoImpl(ArgToAllocaMap, UseEntryValue, F, OriginalStorage,
-                             DVI.getExpression(), SkipOutermostLoad);
-  if (!SalvagedInfo)
-    return;
-
-  Value *Storage = &SalvagedInfo->first;
-  DIExpression *Expr = &SalvagedInfo->second;
-
-  DVI.replaceVariableLocationOp(OriginalStorage, Storage);
-  DVI.setExpression(Expr);
-  // We only hoist dbg.declare today since it doesn't make sense to hoist
-  // dbg.value since it does not have the same function wide guarantees that
-  // dbg.declare does.
-  if (isa<DbgDeclareInst>(DVI)) {
-    std::optional<BasicBlock::iterator> InsertPt;
-    if (auto *I = dyn_cast<Instruction>(Storage)) {
-      InsertPt = I->getInsertionPointAfterDef();
-      // Update DILocation only if variable was not inlined.
-      DebugLoc ILoc = I->getDebugLoc();
-      DebugLoc DVILoc = DVI.getDebugLoc();
-      if (ILoc && DVILoc &&
-          DVILoc->getScope()->getSubprogram() ==
-              ILoc->getScope()->getSubprogram())
-        DVI.setDebugLoc(I->getDebugLoc());
-    } else if (isa<Argument>(Storage))
-      InsertPt = F->getEntryBlock().begin();
-    if (InsertPt)
-      DVI.moveBefore(*(*InsertPt)->getParent(), *InsertPt);
-  }
-}
-
 void coro::salvageDebugInfo(
     SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
     DbgVariableRecord &DVR, bool UseEntryValue) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index b53c5a48eb10b..52f4ffe292dae 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -34,14 +34,11 @@ void suppressCoroAllocs(CoroIdInst *CoroId);
 void suppressCoroAllocs(LLVMContext &Context,
                         ArrayRef<CoroAllocInst *> CoroAllocs);
 
-/// Attempts to rewrite the location operand of debug intrinsics in terms of
+/// Attempts to rewrite the location operand of debug records in terms of
 /// the coroutine frame pointer, folding pointer offsets into the DIExpression
 /// of the intrinsic.
 /// If the frame pointer is an Argument, store it into an alloca to enhance the
 /// debugability.
-void salvageDebugInfo(
-    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
-    DbgVariableIntrinsic &DVI, bool IsEntryPoint);
 void salvageDebugInfo(
     SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
     DbgVariableRecord &DVR, bool UseEntryValue);
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 5a8a41f0dc432..64b33e46404f0 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -618,19 +618,15 @@ static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape,
   }
 }
 
-/// Returns all DbgVariableIntrinsic in F.
-static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,
-                 SmallVector<DbgVariableRecord *>>
-collectDbgVariableIntrinsics(Function &F) {
-  SmallVector<DbgVariableIntrinsic *, 8> Intrinsics;
+/// Returns all debug records in F.
+static SmallVector<DbgVariableRecord *>
+collectDbgVariableRecords(Function &F) {
   SmallVector<DbgVariableRecord *> DbgVariableRecords;
   for (auto &I : instructions(F)) {
     for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
       DbgVariableRecords.push_back(&DVR);
-    if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
-      Intrinsics.push_back(DVI);
   }
-  return {Intrinsics, DbgVariableRecords};
+  return DbgVariableRecords;
 }
 
 void coro::BaseCloner::replaceSwiftErrorOps() {
@@ -638,13 +634,11 @@ void coro::BaseCloner::replaceSwiftErrorOps() {
 }
 
 void coro::BaseCloner::salvageDebugInfo() {
-  auto [Worklist, DbgVariableRecords] = collectDbgVariableIntrinsics(*NewF);
+  auto DbgVariableRecords = collectDbgVariableRecords(*NewF);
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
 
   // Only 64-bit ABIs have a register we can refer to with the entry value.
   bool UseEntryValue = OrigF.getParent()->getTargetTriple().isArch64Bit();
-  for (DbgVariableIntrinsic *DVI : Worklist)
-    coro::salvageDebugInfo(ArgToAllocaMap, *DVI, UseEntryValue);
   for (DbgVariableRecord *DVR : DbgVariableRecords)
     coro::salvageDebugInfo(ArgToAllocaMap, *DVR, UseEntryValue);
 
@@ -655,7 +649,7 @@ void coro::BaseCloner::salvageDebugInfo() {
     return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
                                    &DomTree);
   };
-  auto RemoveOne = [&](auto *DVI) {
+  auto RemoveOne = [&](DbgVariableRecord *DVI) {
     if (IsUnreachableBlock(DVI->getParent()))
       DVI->eraseFromParent();
     else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
@@ -669,7 +663,6 @@ void coro::BaseCloner::salvageDebugInfo() {
         DVI->eraseFromParent();
     }
   };
-  for_each(Worklist, RemoveOne);
   for_each(DbgVariableRecords, RemoveOne);
 }
 
@@ -2022,9 +2015,7 @@ static void doSplitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
   // original function. The Cloner has already salvaged debug info in the new
   // coroutine funclets.
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
-  auto [DbgInsts, DbgVariableRecords] = collectDbgVariableIntrinsics(F);
-  for (auto *DDI : DbgInsts)
-    coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
+  auto DbgVariableRecords = collectDbgVariableRecords(F);
   for (DbgVariableRecord *DVR : DbgVariableRecords)
     coro::salvageDebugInfo(ArgToAllocaMap, *DVR, false /*UseEntryValue*/);
 
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index 8017db1cfe146..5fd5f7d9dad09 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -514,7 +514,7 @@ void collectSpillsAndAllocasFromInsts(
 void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F,
                               const SuspendCrossingInfo &Checker) {
   // We don't want the layout of coroutine frame to be affected
-  // by debug information. So we only choose to salvage DbgValueInst for
+  // by debug information. So we only choose to salvage dbg.values for
   // whose value is already in the frame.
   // We would handle the dbg.values for allocas specially
   for (auto &Iter : Spills) {
@@ -522,9 +522,7 @@ void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F,
     SmallVector<DbgValueInst *, 16> DVIs;
     SmallVector<DbgVariableRecord *, 16> DVRs;
     findDbgValues(DVIs, V, &DVRs);
-    for (DbgValueInst *DVI : DVIs)
-      if (Checker.isDefinitionAcrossSuspend(*V, DVI))
-        Spills[V].push_back(DVI);
+    assert(DVIs.empty());
     // Add the instructions which carry debug info that is in the frame.
     for (DbgVariableRecord *DVR : DVRs)
       if (Checker.isDefinitionAcrossSuspend(*V, DVR->Marker->MarkedInstr))
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index d4555e9435f1d..f5525deb0172f 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -572,7 +572,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
 
   // Work out whether a dbg.value intrinsic or an equivalent DbgVariableRecord
   // is a parameter to be preserved.
-  auto ExamineDbgValue = [](auto *DbgVal, auto &Container) {
+  auto ExamineDbgValue = [&PDVRRelated](DbgVariableRecord *DbgVal) {
     LLVM_DEBUG(dbgs() << " Deciding: ");
     LLVM_DEBUG(DbgVal->print(dbgs()));
     LLVM_DEBUG(dbgs() << "\n");
@@ -581,7 +581,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
       LLVM_DEBUG(dbgs() << "  Include (parameter): ");
       LLVM_DEBUG(DbgVal->print(dbgs()));
       LLVM_DEBUG(dbgs() << "\n");
-      Container.insert(DbgVal);
+      PDVRRelated.insert(DbgVal);
     } else {
       LLVM_DEBUG(dbgs() << "  Delete (!parameter): ");
       LLVM_DEBUG(DbgVal->print(dbgs()));
@@ -589,7 +589,8 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
     }
   };
 
-  auto ExamineDbgDeclare = [&PDIRelated](auto *DbgDecl, auto &Container) {
+  auto ExamineDbgDeclare = [&PDIRelated,
+                            &PDVRRelated](DbgVariableRecord *DbgDecl) {
     LLVM_DEBUG(dbgs() << " Deciding: ");
     LLVM_DEBUG(DbgDecl->print(dbgs()));
     LLVM_DEBUG(dbgs() << "\n");
@@ -616,7 +617,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
                 LLVM_DEBUG(dbgs() << "  Include: ");
                 LLVM_DEBUG(DbgDecl->print(dbgs()));
                 LLVM_DEBUG(dbgs() << "\n");
-                Container.insert(DbgDecl);
+                PDVRRelated.insert(DbgDecl);
               } else {
                 LLVM_DEBUG(dbgs() << "   Delete (!parameter): ");
                 LLVM_DEBUG(SI->print(dbgs()));
@@ -647,18 +648,14 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
     // they connected to parameters?
     for (DbgVariableRecord &DVR : filterDbgVars(BI->getDbgRecordRange())) {
       if (DVR.isDbgValue() || DVR.isDbgAssign()) {
-        ExamineDbgValue(&DVR, PDVRRelated);
+        ExamineDbgValue(&DVR);
       } else {
         assert(DVR.isDbgDeclare());
-        ExamineDbgDeclare(&DVR, PDVRRelated);
+        ExamineDbgDeclare(&DVR);
       }
     }
 
-    if (auto *DVI = dyn_cast<DbgValueInst>(&*BI)) {
-      ExamineDbgValue(DVI, PDIRelated);
-    } else if (auto *DDI = dyn_cast<DbgDeclareInst>(&*BI)) {
-      ExamineDbgDeclare(DDI, PDIRelated);
-    } else if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
+    if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
       LLVM_DEBUG(dbgs() << " Will Include Terminator: ");
       LLVM_DEBUG(BI->print(dbgs()));
       LLVM_DEBUG(dbgs() << "\n");
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 9d7c025ccff86..f7fbf0815df03 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -825,9 +825,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Value *EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned);
 
   bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock);
-  void tryToSinkInstructionDbgValues(
-      Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
-      BasicBlock *DestBlock, SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers);
   void tryToSinkInstructionDbgVariableRecords(
       Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
       BasicBlock *DestBlock, SmallVectorImpl<DbgVariableRecord *> &DPUsers);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 684b9a1f90161..503611a4fc32c 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1420,21 +1420,16 @@ void InstCombinerImpl::freelyInvertAllUsersOf(Value *I, Value *IgnoredUser) {
   SmallVector<DbgValueInst *, 4> DbgValues;
   SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
   llvm::findDbgValues(DbgValues, I, &DbgVariableRecords);
+  assert(DbgValues.empty());
 
-  auto InvertDbgValueUse = [&](auto *DbgVal) {
+  for (DbgVariableRecord *DbgVal : DbgVariableRecords) {
     SmallVector<uint64_t, 1> Ops = {dwarf::DW_OP_not};
     for (unsigned Idx = 0, End = DbgVal->getNumVariableLocationOps();
          Idx != End; ++Idx)
       if (DbgVal->getVariableLocationOp(Idx) == I)
         DbgVal->setExpression(
             DIExpression::appendOpsToArg(DbgVal->getExpression(), Ops, Idx));
-  };
-
-  for (DbgValueInst *DVI : DbgValues)
-    InvertDbgValueUse(DVI);
-
-  for (DbgVariableRecord *DVR : DbgVariableRecords)
-    InvertDbgValueUse(DVR);
+  }
 }
 
 /// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a
@@ -3575,6 +3570,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
   std::unique_ptr<DIBuilder> DIB;
   if (isa<AllocaInst>(MI)) {
     findDbgUsers(DVIs, &MI, &DVRs);
+    assert(DVIs.empty());
     DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
   }
 
@@ -5253,8 +5249,7 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I,
   SmallVector<DbgVariableIntrinsic *, 2> DbgUsers;
   SmallVector<DbgVariableRecord *, 2> DbgVariableRecords;
   findDbgUsers(DbgUsers, I, &DbgVariableRecords);
-  if (!DbgUsers.empty())
-    tryToSinkInstructionDbgValues(I, InsertPos, SrcBlock, DestBlock, DbgUsers);
+  assert(DbgUsers.empty());
   if (!DbgVariableRecords.empty())
     tryToSinkInstructionDbgVariableRecords(I, InsertPos, SrcBlock, DestBlock,
                                            DbgVariableRecords);
@@ -5271,71 +5266,12 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I,
   return true;
 }
 
-void InstCombinerImpl::tryToSinkInstructionDbgValues(
-    Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
-    BasicBlock *DestBlock, SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers) {
-  // For all debug values in the destination block, the sunk instruction
-  // will still be available, so they do not need to be dropped.
-  SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSalvage;
-  for (auto &DbgUser : DbgUsers)
-    if (DbgUser->getParent() != DestBlock)
-      DbgUsersToSalvage.push_back(DbgUser);
-
-  // Process the sinking DbgUsersToSalvage in reverse order, as we only want
-  // to clone the last appearing debug intrinsic for each given variable.
-  SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSink;
-  for (DbgVariableIntrinsic *DVI : DbgUsersToSalvage)
-    if (DVI->getParent() == SrcBlock)
-      DbgUsersToSink.push_back(DVI);
-  llvm::sort(DbgUsersToSink,
-             [](auto *A, auto *B) { return B->comesBefore(A); });
-
-  SmallVector<DbgVariableIntrinsic *, 2> DIIClones;
-  SmallSet<DebugVariable, 4> SunkVariables;
-  for (auto *User : DbgUsersToSink) {
-    // A dbg.declare instruction should not be cloned, since there can only be
-    // one per variable fragment. It should be left in the original place
-    // because the sunk instruction is not an alloca (otherwise we could not be
-    // here).
-    if (isa<DbgDeclareInst>(User))
-      continue;
-
-    DebugVariable DbgUserVariable =
-        DebugVariable(User->getVariable(), User->getExpression(),
-                      User->getDebugLoc()->getInlinedAt());
-
-    if (!SunkVariables.insert(DbgUserVariable).second)
-      continue;
-
-    // Leave dbg.assign intrinsics in their original positions and there should
-    // be no need to insert a clone.
-    if (isa<DbgAssignIntrinsic>(User))
-      continue;
-
-    DIIClones.emplace_back(cast<DbgVariableIntrinsic>(User->clone()));
-    if (isa<DbgDeclareInst>(User) && isa<CastInst>(I))
-      DIIClones.back()->replaceVariableLocationOp(I, I->getOperand(0));
-    LLVM_DEBUG(dbgs() << "CLONE: " << *DIIClones.back() << '\n');
-  }
-
-  // Perform salvaging without the clones, then sink the clones.
-  if (!DIIClones.empty()) {
-    salvageDebugInfoForDbgValues(*I, DbgUsersToSalvage, {});
-    // The clones are in reverse order of original appearance, reverse again to
-    // maintain the original order.
-    for (auto &DIIClone : llvm::reverse(DIIClones)) {
-      DIIClone->insertBefore(InsertPos);
-      LLVM_DEBUG(dbgs() << "SINK: " << *DIIClone << '\n');
-    }
-  }
-}
-
 void InstCombinerImpl::tryToSinkInstructionDbgVariableRecords(
     Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
     BasicBlock *DestBlock,
     SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
-  // Implementation of tryToSinkInstructionDbgValues, but for the
-  // DbgVariableRecord of variable assignments rather than dbg.values.
+  // For all debug values in the destination block, the sunk instruction
+  // will still be available, so they do not need to be dropped.
 
   // Fetch all DbgVariableRecords not already in the destination.
   SmallVector<DbgVariableRecord *, 2> DbgVariableRecordsToSalvage;
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 2786d81773ed9..df3160233c510 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1489,6 +1489,7 @@ static bool checkAndReplaceCondition(
     SmallVector<DbgVariableIntrinsic *> DbgUsers;
     SmallVector<DbgVariableRecord *> DVRUsers;
     findDbgUsers(DbgUsers, Cmp, &DVRUsers);
+    assert(DbgUsers.empty());
 
     for (auto *DVR : DVRUsers) {
       auto *DTN = DT.getNode(DVR->getParent());
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 85dd9a1bf7161..0f63ed0166cf4 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -2079,6 +2079,7 @@ struct DSEState {
       AllocFnKind AllocKind =
           Attrs.getFnAttr(Attribute::AllocKind).getAllocKind() |
           AllocFnKind::Zeroed;
+      AllocKind &= ~AllocFnKind::Uninitialized;
       Attrs =
           Attrs.addFnAttribute(Ctx, Attribute::getWithAllocKind(Ctx, AllocKind))
               .removeFnAttribute(Ctx, "alloc-variant-zeroed");
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index b5dbef13289ac..4d1f44076db7e 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1979,15 +1979,13 @@ void JumpThreadingPass::updateSSA(BasicBlock *BB, BasicBlock *NewBB,
 
     // Find debug values outside of the block
     findDbgValues(DbgValues, &I, &DbgVariableRecords);
-    llvm::erase_if(DbgValues, [&](const DbgValueInst *DbgVal) {
-      return DbgVal->getParent() == BB;
-    });
+    assert(DbgValues.empty());
     llvm::erase_if(DbgVariableRecords, [&](const DbgVariableRecord *DbgVarRec) {
       return DbgVarRec->getParent() == BB;
     });
 
     // If there are no uses outside the block, we're done with this instruction.
-    if (UsesToRename.empty() && DbgValues.empty() && DbgVariableRecords.empty())
+    if (UsesToRename.empty() && DbgVariableRecords.empty())
       continue;
     LLVM_DEBUG(dbgs() << "JT: Renaming non-local uses of: " << I << "\n");
 
@@ -2000,8 +1998,7 @@ void JumpThreadingPass::updateSSA(BasicBlock *BB, BasicBlock *NewBB,
 
     while (!UsesToRename.empty())
       SSAUpdate.RewriteUse(*UsesToRename.pop_back_val());
-    if (!DbgValues.empty() || !DbgVariableRecords.empty()) {
-      SSAUpdate.UpdateDebugValues(&I, DbgValues);
+    if (!DbgVariableRecords.empty()) {
       SSAUpdate.UpdateDebugValues(&I, DbgVariableRecords);
       DbgValues.clear();
       DbgVariableRecords.clear();
@@ -2032,32 +2029,7 @@ void JumpThreadingPass::cloneInstructions(ValueToValueMapTy &ValueMapping,
   // copy of the block 'NewBB'.  If there are PHI nodes in the source basic
   // block, evaluate them to account for entry from PredBB.
 
-  // Retargets llvm.dbg.value to any renamed variables.
-  auto RetargetDbgValueIfPossible = [&](Instruction *NewInst) -> bool {
-    auto DbgInstruction = dyn_cast<DbgValueInst>(NewInst);
-    if (!DbgInstruction)
-      return false;
-
-    SmallSet<std::pair<Value *, Value *>, 16> OperandsToRemap;
-    for (auto DbgOperand : DbgInstruction->location_ops()) {
-      auto DbgOperandInstruction = dyn_cast<Instruction>(DbgOperand);
-      if (!DbgOperandInstruction)
-        continue;
-
-      auto I = ValueMapping.find(DbgOperandInstruction);
-      if (I != ValueMapping.end()) {
-        OperandsToRemap.insert(
-            std::pair<Value *, Value *>(DbgOperand, I->second));
-      }
-    }
-
-    for (auto &[OldOp, MappedOp] : OperandsToRemap)
-      DbgInstruction->replaceVariableLocationOp(OldOp, MappedOp);
-    return true;
-  };
-
-  // Duplicate implementation of the above dbg.value code, using
-  // DbgVariableRecords instead.
+  // Retargets dbg.value to any renamed variables.
   auto RetargetDbgVariableRecordIfPossible = [&](DbgVariableRecord *DVR) {
     SmallSet<std::pair<Value *, Value *>, 16> OperandsToRemap;
     for (auto *Op : DVR->location_ops()) {
@@ -2116,9 +2088,6 @@ void JumpThreadingPass::cloneInstructions(ValueToValueMapTy &ValueMapping,
     if (const DebugLoc &DL = New->getDebugLoc())
       mapAtomInstance(DL, ValueMapping);
 
-    if (RetargetDbgValueIfPossible(New))
-      continue;
-
     // Remap operands to patch up intra-block references.
     for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)
       if (Instruction *Inst = dyn_cast<Instruction>(New->getOperand(i))) {
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index a0f9f3c4a35a5..70e9eee5339a7 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -78,6 +78,7 @@ enum class RuleTy {
   PerLoopCacheAnalysis,
   PerInstrOrderCost,
   ForVectorization,
+  Ignore
 };
 
 } // end anonymous namespace
@@ -106,14 +107,20 @@ static cl::list<RuleTy> Profitabilities(
                clEnumValN(RuleTy::PerInstrOrderCost, "instorder",
                           "Prioritize the IVs order of each instruction"),
                clEnumValN(RuleTy::ForVectorization, "vectorize",
-                          "Prioritize vectorization")));
+                          "Prioritize vectorization"),
+               clEnumValN(RuleTy::Ignore, "ignore",
+                          "Ignore profitability, force interchange (does not "
+                          "work with other options)")));
 
 #ifndef NDEBUG
-static bool noDuplicateRules(ArrayRef<RuleTy> Rules) {
+static bool noDuplicateRulesAndIgnore(ArrayRef<RuleTy> Rules) {
   SmallSet<RuleTy, 4> Set;
-  for (RuleTy Rule : Rules)
+  for (RuleTy Rule : Rules) {
     if (!Set.insert(Rule).second)
       return false;
+    if (Rule == RuleTy::Ignore)
+      return false;
+  }
   return true;
 }
 
@@ -1357,6 +1364,13 @@ std::optional<bool> LoopInterchangeProfitability::isProfitableForVectorization(
 bool LoopInterchangeProfitability::isProfitable(
     const Loop *InnerLoop, const Loop *OuterLoop, unsigned InnerLoopId,
     unsigned OuterLoopId, CharMatrix &DepMatrix, CacheCostManager &CCM) {
+
+  // Return true if interchange is forced and the cost-model ignored.
+  if (Profitabilities.size() == 1 && Profitabilities[0] == RuleTy::Ignore)
+    return true;
+  assert(noDuplicateRulesAndIgnore(Profitabilities) &&
+         "Duplicate rules and option 'ignore' are not allowed");
+
   // isProfitable() is structured to avoid endless loop interchange. If the
   // highest priority rule (isProfitablePerLoopCacheAnalysis by default) could
   // decide the profitability then, profitability check will stop and return the
@@ -1365,7 +1379,6 @@ bool LoopInterchangeProfitability::isProfitable(
   // second highest priority rule (isProfitablePerInstrOrderCost by default).
   // Likewise, if it failed to analysis the profitability then only, the last
   // rule (isProfitableForVectorization by default) will decide.
-  assert(noDuplicateRules(Profitabilities) && "Detect duplicate rules");
   std::optional<bool> shouldInterchange;
   for (RuleTy RT : Profitabilities) {
     switch (RT) {
@@ -1382,6 +1395,9 @@ bool LoopInterchangeProfitability::isProfitable(
       shouldInterchange =
           isProfitableForVectorization(InnerLoopId, OuterLoopId, DepMatrix);
       break;
+    case RuleTy::Ignore:
+      llvm_unreachable("Option 'ignore' is not supported with other options");
+      break;
     }
 
     // If this rule could determine the profitability, don't call subsequent
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index dc8fa4379752f..9e318b04c2c99 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -3790,6 +3790,11 @@ LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
             continue;
         }
 
+        // Do not consider uses inside lifetime intrinsics. These are not
+        // actually materialized.
+        if (UserInst->isLifetimeStartOrEnd())
+          continue;
+
         std::pair<size_t, Immediate> P =
             getUse(S, LSRUse::Basic, MemAccessTy());
         size_t LUIdx = P.first;
@@ -6630,13 +6635,10 @@ struct SCEVDbgValueBuilder {
 /// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
 /// and DIExpression.
 struct DVIRecoveryRec {
-  DVIRecoveryRec(DbgValueInst *DbgValue)
-      : DbgRef(DbgValue), Expr(DbgValue->getExpression()),
-        HadLocationArgList(false) {}
   DVIRecoveryRec(DbgVariableRecord *DVR)
       : DbgRef(DVR), Expr(DVR->getExpression()), HadLocationArgList(false) {}
 
-  PointerUnion<DbgValueInst *, DbgVariableRecord *> DbgRef;
+  DbgVariableRecord *DbgRef;
   DIExpression *Expr;
   bool HadLocationArgList;
   SmallVector<WeakVH, 2> LocationOps;
@@ -6695,44 +6697,38 @@ static void updateDVIWithLocations(T &DbgVal,
 }
 
 /// Write the new expression and new location ops for the dbg.value. If possible
-/// reduce the szie of the dbg.value intrinsic by omitting DIArglist. This
+/// reduce the szie of the dbg.value by omitting DIArglist. This
 /// can be omitted if:
 /// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
 /// 2. The DW_OP_LLVM_arg is the first operand in the expression.
-static void UpdateDbgValueInst(DVIRecoveryRec &DVIRec,
-                               SmallVectorImpl<Value *> &NewLocationOps,
-                               SmallVectorImpl<uint64_t> &NewExpr) {
-  auto UpdateDbgValueInstImpl = [&](auto *DbgVal) {
-    unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
-    if (NumLLVMArgs == 0) {
-      // Location assumed to be on the stack.
-      updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
-    } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
-      // There is only a single DW_OP_llvm_arg at the start of the expression,
-      // so it can be omitted along with DIArglist.
-      assert(NewExpr[1] == 0 &&
-             "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
-      llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2));
-      updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
-    } else {
-      // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
-      updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
-    }
+static void UpdateDbgValue(DVIRecoveryRec &DVIRec,
+                           SmallVectorImpl<Value *> &NewLocationOps,
+                           SmallVectorImpl<uint64_t> &NewExpr) {
+  DbgVariableRecord *DbgVal = DVIRec.DbgRef;
+  unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
+  if (NumLLVMArgs == 0) {
+    // Location assumed to be on the stack.
+    updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
+  } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
+    // There is only a single DW_OP_llvm_arg at the start of the expression,
+    // so it can be omitted along with DIArglist.
+    assert(NewExpr[1] == 0 &&
+           "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
+    llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2));
+    updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
+  } else {
+    // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
+    updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
+  }
 
-    // If the DIExpression was previously empty then add the stack terminator.
-    // Non-empty expressions have only had elements inserted into them and so
-    // the terminator should already be present e.g. stack_value or fragment.
-    DIExpression *SalvageExpr = DbgVal->getExpression();
-    if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
-      SalvageExpr =
-          DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
-      DbgVal->setExpression(SalvageExpr);
-    }
-  };
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    UpdateDbgValueInstImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
-  else
-    UpdateDbgValueInstImpl(cast<DbgVariableRecord *>(DVIRec.DbgRef));
+  // If the DIExpression was previously empty then add the stack terminator.
+  // Non-empty expressions have only had elements inserted into them and so
+  // the terminator should already be present e.g. stack_value or fragment.
+  DIExpression *SalvageExpr = DbgVal->getExpression();
+  if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
+    SalvageExpr = DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
+    DbgVal->setExpression(SalvageExpr);
+  }
 }
 
 /// Cached location ops may be erased during LSR, in which case a poison is
@@ -6746,39 +6742,34 @@ static Value *getValueOrPoison(WeakVH &VH, LLVMContext &C) {
 
 /// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
 static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
-  auto RestorePreTransformStateImpl = [&](auto *DbgVal) {
-    LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
-                      << "scev-salvage: post-LSR: " << *DbgVal << '\n');
-    assert(DVIRec.Expr && "Expected an expression");
-    DbgVal->setExpression(DVIRec.Expr);
-
-    // Even a single location-op may be inside a DIArgList and referenced with
-    // DW_OP_LLVM_arg, which is valid only with a DIArgList.
-    if (!DVIRec.HadLocationArgList) {
-      assert(DVIRec.LocationOps.size() == 1 &&
-             "Unexpected number of location ops.");
-      // LSR's unsuccessful salvage attempt may have added DIArgList, which in
-      // this case was not present before, so force the location back to a
-      // single uncontained Value.
-      Value *CachedValue =
-          getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
-      DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
-    } else {
-      SmallVector<ValueAsMetadata *, 3> MetadataLocs;
-      for (WeakVH VH : DVIRec.LocationOps) {
-        Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
-        MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
-      }
-      auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
-      DbgVal->setRawLocation(
-          llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
+  DbgVariableRecord *DbgVal = DVIRec.DbgRef;
+  LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
+                    << "scev-salvage: post-LSR: " << *DbgVal << '\n');
+  assert(DVIRec.Expr && "Expected an expression");
+  DbgVal->setExpression(DVIRec.Expr);
+
+  // Even a single location-op may be inside a DIArgList and referenced with
+  // DW_OP_LLVM_arg, which is valid only with a DIArgList.
+  if (!DVIRec.HadLocationArgList) {
+    assert(DVIRec.LocationOps.size() == 1 &&
+           "Unexpected number of location ops.");
+    // LSR's unsuccessful salvage attempt may have added DIArgList, which in
+    // this case was not present before, so force the location back to a
+    // single uncontained Value.
+    Value *CachedValue =
+        getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
+    DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
+  } else {
+    SmallVector<ValueAsMetadata *, 3> MetadataLocs;
+    for (WeakVH VH : DVIRec.LocationOps) {
+      Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
+      MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
     }
-    LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
-  };
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    RestorePreTransformStateImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
-  else
-    RestorePreTransformStateImpl(cast<DbgVariableRecord *>(DVIRec.DbgRef));
+    auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
+    DbgVal->setRawLocation(
+        llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
+  }
+  LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
 }
 
 static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
@@ -6786,9 +6777,7 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
                        const SCEV *SCEVInductionVar,
                        SCEVDbgValueBuilder IterCountExpr) {
 
-  if (isa<DbgValueInst *>(DVIRec.DbgRef)
-          ? !cast<DbgValueInst *>(DVIRec.DbgRef)->isKillLocation()
-          : !cast<DbgVariableRecord *>(DVIRec.DbgRef)->isKillLocation())
+  if (!DVIRec.DbgRef->isKillLocation())
     return false;
 
   // LSR may have caused several changes to the dbg.value in the failed salvage
@@ -6882,13 +6871,8 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
     DbgBuilder->appendToVectors(NewExpr, NewLocationOps);
   }
 
-  UpdateDbgValueInst(DVIRec, NewLocationOps, NewExpr);
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
-                      << *cast<DbgValueInst *>(DVIRec.DbgRef) << "\n");
-  else
-    LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
-                      << *cast<DbgVariableRecord *>(DVIRec.DbgRef) << "\n");
+  UpdateDbgValue(DVIRec, NewLocationOps, NewExpr);
+  LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DbgRef << "\n");
   return true;
 }
 
@@ -6934,21 +6918,23 @@ static void DbgRewriteSalvageableDVIs(
 /// cacheing and salvaging.
 static void DbgGatherSalvagableDVI(
     Loop *L, ScalarEvolution &SE,
-    SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs,
-    SmallSet<AssertingVH<DbgValueInst>, 2> &DVIHandles) {
+    SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs) {
   for (const auto &B : L->getBlocks()) {
     for (auto &I : *B) {
-      auto ProcessDbgValue = [&](auto *DbgVal) -> bool {
+      for (DbgVariableRecord &DbgVal : filterDbgVars(I.getDbgRecordRange())) {
+        if (!DbgVal.isDbgValue() && !DbgVal.isDbgAssign())
+          continue;
+
         // Ensure that if any location op is undef that the dbg.vlue is not
         // cached.
-        if (DbgVal->isKillLocation())
-          return false;
+        if (DbgVal.isKillLocation())
+          continue;
 
         // Check that the location op SCEVs are suitable for translation to
         // DIExpression.
         const auto &HasTranslatableLocationOps =
-            [&](const auto *DbgValToTranslate) -> bool {
-          for (const auto LocOp : DbgValToTranslate->location_ops()) {
+            [&](const DbgVariableRecord &DbgValToTranslate) -> bool {
+          for (const auto LocOp : DbgValToTranslate.location_ops()) {
             if (!LocOp)
               return false;
 
@@ -6963,31 +6949,21 @@ static void DbgGatherSalvagableDVI(
         };
 
         if (!HasTranslatableLocationOps(DbgVal))
-          return false;
+          continue;
 
         std::unique_ptr<DVIRecoveryRec> NewRec =
-            std::make_unique<DVIRecoveryRec>(DbgVal);
+            std::make_unique<DVIRecoveryRec>(&DbgVal);
         // Each location Op may need a SCEVDbgValueBuilder in order to recover
         // it. Pre-allocating a vector will enable quick lookups of the builder
         // later during the salvage.
-        NewRec->RecoveryExprs.resize(DbgVal->getNumVariableLocationOps());
-        for (const auto LocOp : DbgVal->location_ops()) {
+        NewRec->RecoveryExprs.resize(DbgVal.getNumVariableLocationOps());
+        for (const auto LocOp : DbgVal.location_ops()) {
           NewRec->SCEVs.push_back(SE.getSCEV(LocOp));
           NewRec->LocationOps.push_back(LocOp);
-          NewRec->HadLocationArgList = DbgVal->hasArgList();
+          NewRec->HadLocationArgList = DbgVal.hasArgList();
         }
         SalvageableDVISCEVs.push_back(std::move(NewRec));
-        return true;
-      };
-      for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
-        if (DVR.isDbgValue() || DVR.isDbgAssign())
-          ProcessDbgValue(&DVR);
       }
-      auto DVI = dyn_cast<DbgValueInst>(&I);
-      if (!DVI)
-        continue;
-      if (ProcessDbgValue(DVI))
-        DVIHandles.insert(DVI);
     }
   }
 }
@@ -7036,8 +7012,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   // Debug preservation - before we start removing anything identify which DVI
   // meet the salvageable criteria and store their DIExpression and SCEVs.
   SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> SalvageableDVIRecords;
-  SmallSet<AssertingVH<DbgValueInst>, 2> DVIHandles;
-  DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords, DVIHandles);
+  DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords);
 
   bool Changed = false;
   std::unique_ptr<MemorySSAUpdater> MSSAU;
@@ -7105,7 +7080,6 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   for (auto &Rec : SalvageableDVIRecords)
     Rec->clear();
   SalvageableDVIRecords.clear();
-  DVIHandles.clear();
   return Changed;
 }
 
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index eacaf42e4e8ba..1d1af42153325 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1222,9 +1222,7 @@ static void eraseDebugIntrinsicsWithNonLocalRefs(Function &F) {
     SmallVector<DbgVariableIntrinsic *, 4> DbgUsers;
     SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
     findDbgUsers(DbgUsers, &I, &DbgVariableRecords);
-    for (DbgVariableIntrinsic *DVI : DbgUsers)
-      if (DVI->getFunction() != &F)
-        DVI->eraseFromParent();
+    assert(DbgUsers.empty());
     for (DbgVariableRecord *DVR : DbgVariableRecords)
       if (DVR->getFunction() != &F)
         DVR->eraseFromParent();
@@ -1289,14 +1287,12 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
     SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
     SmallVector<DbgVariableRecord *, 1> DPUsers;
     findDbgUsers(DbgUsers, Input, &DPUsers);
+    assert(DbgUsers.empty());
     DIExpression *Expr = DIB.createExpression();
 
     // Iterate the debud users of the Input values. If they are in the extracted
     // function then update their location with the new value. If they are in
     // the parent function then create a similar debug record.
-    for (auto *DVI : DbgUsers)
-      UpdateOrInsertDebugRecord(DVI, Input, NewVal, Expr,
-                                isa<DbgDeclareInst>(DVI));
     for (auto *DVR : DPUsers)
       UpdateOrInsertDebugRecord(DVR, Input, NewVal, Expr, DVR->isDbgDeclare());
   }
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index a1f030a336c15..4210ce6da1eb2 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -808,9 +808,6 @@ bool checkDebugifyMetadata(Module &M,
 
     // Find missing lines.
     for (Instruction &I : instructions(F)) {
-      if (isa<DbgValueInst>(&I))
-        continue;
-
       auto DL = I.getDebugLoc();
       if (DL && DL.getLine() != 0) {
         MissingLines.reset(DL.getLine() - 1);
@@ -839,10 +836,6 @@ bool checkDebugifyMetadata(Module &M,
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
         if (DVR.isDbgValue() || DVR.isDbgAssign())
           CheckForMisSized(&DVR);
-      auto *DVI = dyn_cast<DbgValueInst>(&I);
-      if (!DVI)
-        continue;
-      CheckForMisSized(DVI);
     }
   }
 
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 6929d14bc56ea..ed3dca2f7c307 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1978,14 +1978,13 @@ static at::StorageToVarsMap collectEscapedLocals(const DataLayout &DL,
       continue;
 
     // Find all local variables associated with the backing storage.
-    auto CollectAssignsForStorage = [&](auto *DbgAssign) {
+    auto CollectAssignsForStorage = [&](DbgVariableRecord *DbgAssign) {
       // Skip variables from inlined functions - they are not local variables.
       if (DbgAssign->getDebugLoc().getInlinedAt())
         return;
       LLVM_DEBUG(errs() << " > DEF : " << *DbgAssign << "\n");
       EscapedLocals[Base].insert(at::VarRecord(DbgAssign));
     };
-    for_each(at::getAssignmentMarkers(Base), CollectAssignsForStorage);
     for_each(at::getDVRAssignmentMarkers(Base), CollectAssignsForStorage);
   }
   return EscapedLocals;
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index b14bbeac97675..d481ad9dee181 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -613,11 +613,10 @@ bool llvm::replaceDbgUsesWithUndef(Instruction *I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, I, &DPUsers);
-  for (auto *DII : DbgUsers)
-    DII->setKillLocation();
+  assert(DbgUsers.empty());
   for (auto *DVR : DPUsers)
     DVR->setKillLocation();
-  return !DbgUsers.empty() || !DPUsers.empty();
+  return !DPUsers.empty();
 }
 
 /// areAllUsesEqual - Check whether the uses of a value are all the same.
@@ -1607,12 +1606,8 @@ static bool PhiHasDebugValue(DILocalVariable *DIVar,
   SmallVector<DbgValueInst *, 1> DbgValues;
   SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
   findDbgValues(DbgValues, APN, &DbgVariableRecords);
-  for (auto *DVI : DbgValues) {
-    assert(is_contained(DVI->getValues(), APN));
-    if ((DVI->getVariable() == DIVar) && (DVI->getExpression() == DIExpr))
-      return true;
-  }
-  for (auto *DVR : DbgVariableRecords) {
+  assert(DbgValues.empty());
+  for (DbgVariableRecord *DVR : DbgVariableRecords) {
     assert(is_contained(DVR->location_ops(), APN));
     if ((DVR->getVariable() == DIVar) && (DVR->getExpression() == DIExpr))
       return true;
@@ -1971,7 +1966,6 @@ bool llvm::replaceDbgDeclare(Value *Address, Value *NewAddress,
 static void updateOneDbgValueForAlloca(const DebugLoc &Loc,
                                        DILocalVariable *DIVar,
                                        DIExpression *DIExpr, Value *NewAddress,
-                                       DbgValueInst *DVI,
                                        DbgVariableRecord *DVR,
                                        DIBuilder &Builder, int Offset) {
   assert(DIVar && "Missing variable");
@@ -1987,14 +1981,8 @@ static void updateOneDbgValueForAlloca(const DebugLoc &Loc,
   if (Offset)
     DIExpr = DIExpression::prepend(DIExpr, 0, Offset);
 
-  if (DVI) {
-    DVI->setExpression(DIExpr);
-    DVI->replaceVariableLocationOp(0u, NewAddress);
-  } else {
-    assert(DVR);
-    DVR->setExpression(DIExpr);
-    DVR->replaceVariableLocationOp(0u, NewAddress);
-  }
+  DVR->setExpression(DIExpr);
+  DVR->replaceVariableLocationOp(0u, NewAddress);
 }
 
 void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
@@ -2002,18 +1990,13 @@ void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
   SmallVector<DbgValueInst *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgValues(DbgUsers, AI, &DPUsers);
-
-  // Attempt to replace dbg.values that use this alloca.
-  for (auto *DVI : DbgUsers)
-    updateOneDbgValueForAlloca(DVI->getDebugLoc(), DVI->getVariable(),
-                               DVI->getExpression(), NewAllocaAddress, DVI,
-                               nullptr, Builder, Offset);
+  assert(DbgUsers.empty());
 
   // Replace any DbgVariableRecords that use this alloca.
   for (DbgVariableRecord *DVR : DPUsers)
     updateOneDbgValueForAlloca(DVR->getDebugLoc(), DVR->getVariable(),
-                               DVR->getExpression(), NewAllocaAddress, nullptr,
-                               DVR, Builder, Offset);
+                               DVR->getExpression(), NewAllocaAddress, DVR,
+                               Builder, Offset);
 }
 
 /// Where possible to salvage debug information for \p I do so.
@@ -2022,6 +2005,7 @@ void llvm::salvageDebugInfo(Instruction &I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, &I, &DPUsers);
+  assert(DbgUsers.empty());
   salvageDebugInfoForDbgValues(I, DbgUsers, DPUsers);
 }
 
@@ -2070,66 +2054,9 @@ void llvm::salvageDebugInfoForDbgValues(
   const unsigned MaxExpressionSize = 128;
   bool Salvaged = false;
 
-  for (auto *DII : DbgUsers) {
-    if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DII)) {
-      if (DAI->getAddress() == &I) {
-        salvageDbgAssignAddress(DAI);
-        Salvaged = true;
-      }
-      if (DAI->getValue() != &I)
-        continue;
-    }
+  // We should never see debug intrinsics nowadays.
+  assert(DbgUsers.empty());
 
-    // Do not add DW_OP_stack_value for DbgDeclare, because they are implicitly
-    // pointing out the value as a DWARF memory location description.
-    bool StackValue = isa<DbgValueInst>(DII);
-    auto DIILocation = DII->location_ops();
-    assert(
-        is_contained(DIILocation, &I) &&
-        "DbgVariableIntrinsic must use salvaged instruction as its location");
-    SmallVector<Value *, 4> AdditionalValues;
-    // `I` may appear more than once in DII's location ops, and each use of `I`
-    // must be updated in the DIExpression and potentially have additional
-    // values added; thus we call salvageDebugInfoImpl for each `I` instance in
-    // DIILocation.
-    Value *Op0 = nullptr;
-    DIExpression *SalvagedExpr = DII->getExpression();
-    auto LocItr = find(DIILocation, &I);
-    while (SalvagedExpr && LocItr != DIILocation.end()) {
-      SmallVector<uint64_t, 16> Ops;
-      unsigned LocNo = std::distance(DIILocation.begin(), LocItr);
-      uint64_t CurrentLocOps = SalvagedExpr->getNumLocationOperands();
-      Op0 = salvageDebugInfoImpl(I, CurrentLocOps, Ops, AdditionalValues);
-      if (!Op0)
-        break;
-      SalvagedExpr =
-          DIExpression::appendOpsToArg(SalvagedExpr, Ops, LocNo, StackValue);
-      LocItr = std::find(++LocItr, DIILocation.end(), &I);
-    }
-    // salvageDebugInfoImpl should fail on examining the first element of
-    // DbgUsers, or none of them.
-    if (!Op0)
-      break;
-
-    SalvagedExpr = SalvagedExpr->foldConstantMath();
-    DII->replaceVariableLocationOp(&I, Op0);
-    bool IsValidSalvageExpr = SalvagedExpr->getNumElements() <= MaxExpressionSize;
-    if (AdditionalValues.empty() && IsValidSalvageExpr) {
-      DII->setExpression(SalvagedExpr);
-    } else if (isa<DbgValueInst>(DII) && IsValidSalvageExpr &&
-               DII->getNumVariableLocationOps() + AdditionalValues.size() <=
-                   MaxDebugArgs) {
-      DII->addVariableLocationOps(AdditionalValues, SalvagedExpr);
-    } else {
-      // Do not salvage using DIArgList for dbg.declare, as it is not currently
-      // supported in those instructions. Also do not salvage if the resulting
-      // DIArgList would contain an unreasonably large number of values.
-      DII->setKillLocation();
-    }
-    LLVM_DEBUG(dbgs() << "SALVAGE: " << *DII << '\n');
-    Salvaged = true;
-  }
-  // Duplicate of above block for DbgVariableRecords.
   for (auto *DVR : DPUsers) {
     if (DVR->isDbgAssign()) {
       if (DVR->getAddress() == &I) {
@@ -2198,9 +2125,6 @@ void llvm::salvageDebugInfoForDbgValues(
   if (Salvaged)
     return;
 
-  for (auto *DII : DbgUsers)
-    DII->setKillLocation();
-
   for (auto *DVR : DPUsers)
     DVR->setKillLocation();
 }
@@ -3429,8 +3353,7 @@ void llvm::dropDebugUsers(Instruction &I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, &I, &DPUsers);
-  for (auto *DII : DbgUsers)
-    DII->eraseFromParent();
+  assert(DbgUsers.empty());
   for (auto *DVR : DPUsers)
     DVR->eraseFromParent();
 }
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 66d0573e83f65..06115e0741ade 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -161,29 +161,8 @@ static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader,
     SmallVector<DbgValueInst *, 1> DbgValues;
     SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
     llvm::findDbgValues(DbgValues, OrigHeaderVal, &DbgVariableRecords);
-    for (auto &DbgValue : DbgValues) {
-      // The original users in the OrigHeader are already using the original
-      // definitions.
-      BasicBlock *UserBB = DbgValue->getParent();
-      if (UserBB == OrigHeader)
-        continue;
-
-      // Users in the OrigPreHeader need to use the value to which the
-      // original definitions are mapped and anything else can be handled by
-      // the SSAUpdater. To avoid adding PHINodes, check if the value is
-      // available in UserBB, if not substitute poison.
-      Value *NewVal;
-      if (UserBB == OrigPreheader)
-        NewVal = OrigPreHeaderVal;
-      else if (SSA.HasValueForBlock(UserBB))
-        NewVal = SSA.GetValueInMiddleOfBlock(UserBB);
-      else
-        NewVal = PoisonValue::get(OrigHeaderVal->getType());
-      DbgValue->replaceVariableLocationOp(OrigHeaderVal, NewVal);
-    }
+    assert(DbgValues.empty());
 
-    // RemoveDIs: duplicate implementation for non-instruction debug-info
-    // storage in DbgVariableRecords.
     for (DbgVariableRecord *DVR : DbgVariableRecords) {
       // The original users in the OrigHeader are already using the original
       // definitions.
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 154f50acf3d1d..e7623aaff105d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1103,7 +1103,6 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
       (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
        RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
        RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
-    // TODO: Add float minnum/maxnum support when FMF nnan is set.
     Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
     return Builder.CreateIntrinsic(Ty, Id, {Left, Right}, nullptr,
                                    "rdx.minmax");
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index ccd7ee360e014..73b5f48796b7a 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -190,7 +190,6 @@ class AssignmentTrackingInfo {
 };
 
 struct AllocaInfo {
-  using DbgUserVec = SmallVector<DbgVariableIntrinsic *, 1>;
   using DPUserVec = SmallVector<DbgVariableRecord *, 1>;
 
   SmallVector<BasicBlock *, 32> DefiningBlocks;
@@ -201,7 +200,6 @@ struct AllocaInfo {
   bool OnlyUsedInOneBlock;
 
   /// Debug users of the alloca - does not include dbg.assign intrinsics.
-  DbgUserVec DbgUsers;
   DPUserVec DPUsers;
   /// Helper to update assignment tracking debug info.
   AssignmentTrackingInfo AssignmentTracking;
@@ -212,7 +210,6 @@ struct AllocaInfo {
     OnlyStore = nullptr;
     OnlyBlock = nullptr;
     OnlyUsedInOneBlock = true;
-    DbgUsers.clear();
     DPUsers.clear();
     AssignmentTracking.clear();
   }
@@ -246,13 +243,10 @@ struct AllocaInfo {
           OnlyUsedInOneBlock = false;
       }
     }
-    DbgUserVec AllDbgUsers;
+    SmallVector<DbgVariableIntrinsic *> AllDbgUsers;
     SmallVector<DbgVariableRecord *> AllDPUsers;
     findDbgUsers(AllDbgUsers, AI, &AllDPUsers);
-    std::copy_if(AllDbgUsers.begin(), AllDbgUsers.end(),
-                 std::back_inserter(DbgUsers), [](DbgVariableIntrinsic *DII) {
-                   return !isa<DbgAssignIntrinsic>(DII);
-                 });
+    assert(AllDbgUsers.empty());
     std::copy_if(AllDPUsers.begin(), AllDPUsers.end(),
                  std::back_inserter(DPUsers),
                  [](DbgVariableRecord *DVR) { return !DVR->isDbgAssign(); });
@@ -380,10 +374,9 @@ struct PromoteMem2Reg {
   /// to.
   DenseMap<PHINode *, unsigned> PhiToAllocaMap;
 
-  /// For each alloca, we keep track of the dbg.declare intrinsic that
+  /// For each alloca, we keep track of the dbg.declare record that
   /// describes it, if any, so that we can convert it to a dbg.value
-  /// intrinsic if the alloca gets promoted.
-  SmallVector<AllocaInfo::DbgUserVec, 8> AllocaDbgUsers;
+  /// record if the alloca gets promoted.
   SmallVector<AllocaInfo::DPUserVec, 8> AllocaDPUsers;
 
   /// For each alloca, keep an instance of a helper class that gives us an easy
@@ -741,14 +734,11 @@ promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
   AI->eraseFromParent();
 
   // The alloca's debuginfo can be removed as well.
-  auto DbgUpdateForAlloca = [&](auto &Container) {
-    for (auto *DbgItem : Container)
-      if (DbgItem->isAddressOfVariable() ||
-          DbgItem->getExpression()->startsWithDeref())
-        DbgItem->eraseFromParent();
-  };
-  DbgUpdateForAlloca(Info.DbgUsers);
-  DbgUpdateForAlloca(Info.DPUsers);
+  for (DbgVariableRecord *DbgItem : Info.DPUsers) {
+    if (DbgItem->isAddressOfVariable() ||
+        DbgItem->getExpression()->startsWithDeref())
+      DbgItem->eraseFromParent();
+  }
 
   ++NumLocalPromoted;
   return true;
@@ -757,7 +747,6 @@ promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
 void PromoteMem2Reg::run() {
   Function &F = *DT.getRoot()->getParent();
 
-  AllocaDbgUsers.resize(Allocas.size());
   AllocaATInfo.resize(Allocas.size());
   AllocaDPUsers.resize(Allocas.size());
 
@@ -816,9 +805,7 @@ void PromoteMem2Reg::run() {
     if (BBNumPreds.empty())
       BBNumPreds.resize(F.getMaxBlockNumber());
 
-    // Remember the dbg.declare intrinsic describing this alloca, if any.
-    if (!Info.DbgUsers.empty())
-      AllocaDbgUsers[AllocaNum] = Info.DbgUsers;
+    // Remember the dbg.declare record describing this alloca, if any.
     if (!Info.AssignmentTracking.empty())
       AllocaATInfo[AllocaNum] = Info.AssignmentTracking;
     if (!Info.DPUsers.empty())
@@ -894,16 +881,12 @@ void PromoteMem2Reg::run() {
   }
 
   // Remove alloca's dbg.declare intrinsics from the function.
-  auto RemoveDbgDeclares = [&](auto &Container) {
-    for (auto &DbgUsers : Container) {
-      for (auto *DbgItem : DbgUsers)
-        if (DbgItem->isAddressOfVariable() ||
-            DbgItem->getExpression()->startsWithDeref())
-          DbgItem->eraseFromParent();
-    }
-  };
-  RemoveDbgDeclares(AllocaDbgUsers);
-  RemoveDbgDeclares(AllocaDPUsers);
+  for (auto &DbgUsers : AllocaDPUsers) {
+    for (DbgVariableRecord *DbgItem : DbgUsers)
+      if (DbgItem->isAddressOfVariable() ||
+          DbgItem->getExpression()->startsWithDeref())
+        DbgItem->eraseFromParent();
+  }
 
   // Loop over all of the PHI nodes and see if there are any that we can get
   // rid of because they merge all of the same incoming values.  This can
diff --git a/llvm/lib/Transforms/Utils/SSAUpdater.cpp b/llvm/lib/Transforms/Utils/SSAUpdater.cpp
index 5db7fc956c497..561c898ec55d8 100644
--- a/llvm/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/llvm/lib/Transforms/Utils/SSAUpdater.cpp
@@ -200,11 +200,7 @@ void SSAUpdater::UpdateDebugValues(Instruction *I) {
   SmallVector<DbgValueInst *, 4> DbgValues;
   SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
   llvm::findDbgValues(DbgValues, I, &DbgVariableRecords);
-  for (auto &DbgValue : DbgValues) {
-    if (DbgValue->getParent() == I->getParent())
-      continue;
-    UpdateDebugValue(I, DbgValue);
-  }
+  assert(DbgValues.empty());
   for (auto &DVR : DbgVariableRecords) {
     if (DVR->getParent() == I->getParent())
       continue;
@@ -212,13 +208,6 @@ void SSAUpdater::UpdateDebugValues(Instruction *I) {
   }
 }
 
-void SSAUpdater::UpdateDebugValues(Instruction *I,
-                                   SmallVectorImpl<DbgValueInst *> &DbgValues) {
-  for (auto &DbgValue : DbgValues) {
-    UpdateDebugValue(I, DbgValue);
-  }
-}
-
 void SSAUpdater::UpdateDebugValues(
     Instruction *I, SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
   for (auto &DVR : DbgVariableRecords) {
@@ -226,15 +215,6 @@ void SSAUpdater::UpdateDebugValues(
   }
 }
 
-void SSAUpdater::UpdateDebugValue(Instruction *I, DbgValueInst *DbgValue) {
-  BasicBlock *UserBB = DbgValue->getParent();
-  if (HasValueForBlock(UserBB)) {
-    Value *NewVal = GetValueAtEndOfBlock(UserBB);
-    DbgValue->replaceVariableLocationOp(I, NewVal);
-  } else
-    DbgValue->setKillLocation();
-}
-
 void SSAUpdater::UpdateDebugValue(Instruction *I, DbgVariableRecord *DVR) {
   BasicBlock *UserBB = DVR->getParent();
   if (HasValueForBlock(UserBB)) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index db0902c589d49..6e420632d83e5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -947,9 +947,8 @@ class LoopVectorizationCostModel {
   /// user options, for the given register kind.
   bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind);
 
-  /// \return True if maximizing vector bandwidth is enabled by the target or
-  /// user options, for the given vector factor.
-  bool useMaxBandwidth(ElementCount VF);
+  /// \return True if register pressure should be calculated for the given VF.
+  bool shouldCalculateRegPressureForVF(ElementCount VF);
 
   /// \return The size (in bits) of the smallest and widest types in the code
   /// that needs to be vectorized. We ignore values that remain scalar such as
@@ -1736,6 +1735,9 @@ class LoopVectorizationCostModel {
   /// Whether this loop should be optimized for size based on function attribute
   /// or profile information.
   bool OptForSize;
+
+  /// The highest VF possible for this loop, without using MaxBandwidth.
+  FixedScalableVFPair MaxPermissibleVFWithoutMaxBW;
 };
 } // end namespace llvm
 
@@ -3832,10 +3834,16 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   return FixedScalableVFPair::getNone();
 }
 
-bool LoopVectorizationCostModel::useMaxBandwidth(ElementCount VF) {
-  return useMaxBandwidth(VF.isScalable()
-                             ? TargetTransformInfo::RGK_ScalableVector
-                             : TargetTransformInfo::RGK_FixedWidthVector);
+bool LoopVectorizationCostModel::shouldCalculateRegPressureForVF(
+    ElementCount VF) {
+  if (!useMaxBandwidth(VF.isScalable()
+                           ? TargetTransformInfo::RGK_ScalableVector
+                           : TargetTransformInfo::RGK_FixedWidthVector))
+    return false;
+  // Only calculate register pressure for VFs enabled by MaxBandwidth.
+  return ElementCount::isKnownGT(
+      VF, VF.isScalable() ? MaxPermissibleVFWithoutMaxBW.ScalableVF
+                          : MaxPermissibleVFWithoutMaxBW.FixedVF);
 }
 
 bool LoopVectorizationCostModel::useMaxBandwidth(
@@ -3911,6 +3919,12 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
       ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
                            : TargetTransformInfo::RGK_FixedWidthVector;
   ElementCount MaxVF = MaxVectorElementCount;
+
+  if (MaxVF.isScalable())
+    MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
+  else
+    MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
+
   if (useMaxBandwidth(RegKind)) {
     auto MaxVectorElementCountMaxBW = ElementCount::get(
         llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
@@ -4264,9 +4278,11 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
       if (VF.isScalar())
         continue;
 
-      /// Don't consider the VF if it exceeds the number of registers for the
-      /// target.
-      if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI))
+      /// If the register pressure needs to be considered for VF,
+      /// don't consider the VF as valid if it exceeds the number
+      /// of registers for the target.
+      if (CM.shouldCalculateRegPressureForVF(VF) &&
+          RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs))
         continue;
 
       InstructionCost C = CM.expectedCost(VF);
@@ -7048,7 +7064,8 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
       InstructionCost Cost = cost(*P, VF);
       VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
 
-      if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI)) {
+      if (CM.shouldCalculateRegPressureForVF(VF) &&
+          RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs)) {
         LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
                           << VF << " because it uses too many registers\n");
         continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 66657b98b094b..3499e650ae853 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -405,9 +405,12 @@ static unsigned getVFScaleFactor(VPRecipeBase *R) {
   return 1;
 }
 
-bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI) const {
-  return any_of(MaxLocalUsers, [&TTI](auto &LU) {
-    return LU.second > TTI.getNumberOfRegisters(LU.first);
+bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI,
+                                        unsigned OverrideMaxNumRegs) const {
+  return any_of(MaxLocalUsers, [&TTI, &OverrideMaxNumRegs](auto &LU) {
+    return LU.second > (OverrideMaxNumRegs > 0
+                            ? OverrideMaxNumRegs
+                            : TTI.getNumberOfRegisters(LU.first));
   });
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 7bcf9dba8c311..cd86d27cf9122 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -85,8 +85,10 @@ struct VPRegisterUsage {
   SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
 
   /// Check if any of the tracked live intervals exceeds the number of
-  /// available registers for the target.
-  bool exceedsMaxNumRegs(const TargetTransformInfo &TTI) const;
+  /// available registers for the target. If non-zero, OverrideMaxNumRegs
+  /// is used in place of the target's number of registers.
+  bool exceedsMaxNumRegs(const TargetTransformInfo &TTI,
+                         unsigned OverrideMaxNumRegs = 0) const;
 };
 
 /// Estimate the register usage for \p Plan and vectorization factors in \p VFs
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
index 4153f0be611a1..9698f1a6768fd 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
@@ -231,3 +231,27 @@ define <vscale x 8 x i64> @sload_8i8_8i64(ptr %a) {
   %aext = sext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
   ret <vscale x 8 x i64> %aext
 }
+
+; Ensure we don't try to promote a predicate load to a sign-extended load.
+define <vscale x 16 x i8> @sload_16i1_16i8(ptr %addr) {
+; CHECK-LABEL: sload_16i1_16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr p0, [x0]
+; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    ret
+  %load = load <vscale x 16 x i1>, ptr %addr
+  %zext = sext <vscale x 16 x i1> %load to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %zext
+}
+
+; Ensure we don't try to promote a predicate load to a zero-extended load.
+define <vscale x 16 x i8> @zload_16i1_16i8(ptr %addr) {
+; CHECK-LABEL: zload_16i1_16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr p0, [x0]
+; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    ret
+  %load = load <vscale x 16 x i1>, ptr %addr
+  %zext = zext <vscale x 16 x i1> %load to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %zext
+}
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
index 6742ae6c1d584..f6465de86fa4f 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
@@ -6,17 +6,17 @@
 define amdgpu_kernel void @remat_constant_voids_spill(ptr addrspace(1) %p) #1 {
 ; GFX908-LABEL: remat_constant_voids_spill:
 ; GFX908:       ; %bb.0:
-; GFX908-NEXT:    v_accvgpr_write_b32 a1, 1
-; GFX908-NEXT:    v_accvgpr_write_b32 a5, 6
-; GFX908-NEXT:    v_accvgpr_write_b32 a6, 7
-; GFX908-NEXT:    v_accvgpr_write_b32 a7, 8
-; GFX908-NEXT:    v_accvgpr_write_b32 a0, 9
-; GFX908-NEXT:    v_accvgpr_write_b32 a2, 2
-; GFX908-NEXT:    v_accvgpr_write_b32 a3, 3
-; GFX908-NEXT:    v_accvgpr_write_b32 a4, 4
+; GFX908-NEXT:    v_accvgpr_write_b32 a0, 1
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, 2
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 3
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, 4
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_accvgpr_write_b32 a1, 5
+; GFX908-NEXT:    v_accvgpr_write_b32 a0, 5
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, 6
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 7
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, 8
+; GFX908-NEXT:    v_accvgpr_write_b32 a4, 9
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ;;#ASMEND
 ; GFX908-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
new file mode 100644
index 0000000000000..69bdb1f5066f0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
@@ -0,0 +1,95 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX90A %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX908 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx906 -passes=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefix=NO-AGPR %s
+
+--- |
+  define void @func() {
+    ret void
+  }
+
+  ; Attribute is ignored for gfx90a
+  define void @no_agprs() "amdgpu-agpr-alloc"="0,0" {
+    ret void
+  }
+
+...
+---
+name: func
+tracksRegLiveness: true
+stack:
+  - { id: 0, size: 4 }
+body:             |
+  ; HAS-AGPR-LABEL: name: func
+  ; HAS-AGPR: bb.0:
+  ; HAS-AGPR-NEXT:   successors: %bb.1(0x80000000)
+  ; HAS-AGPR-NEXT:   liveins: $vgpr0
+  ; HAS-AGPR-NEXT: {{  $}}
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_2:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 6, implicit $exec
+  ; HAS-AGPR-NEXT: {{  $}}
+  ; HAS-AGPR-NEXT: bb.1:
+  ; HAS-AGPR-NEXT:   [[AV_MOV_3:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 3, implicit $exec
+  ;
+  ; NO-AGPR-LABEL: name: func
+  ; NO-AGPR: bb.0:
+  ; NO-AGPR-NEXT:   successors: %bb.1(0x80000000)
+  ; NO-AGPR-NEXT:   liveins: $vgpr0
+  ; NO-AGPR-NEXT: {{  $}}
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_2:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec
+  ; NO-AGPR-NEXT: {{  $}}
+  ; NO-AGPR-NEXT: bb.1:
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  bb.0:
+    liveins: $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+    %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+    %4:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %5:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    %6:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec
+
+  bb.1:
+    %7:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+
+...
+
+---
+name: no_agprs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+    ; GFX90A-LABEL: name: no_agprs
+    ; GFX90A: liveins: $vgpr0
+    ; GFX90A-NEXT: {{  $}}
+    ; GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX90A-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    ;
+    ; GFX908-LABEL: name: no_agprs
+    ; GFX908: liveins: $vgpr0
+    ; GFX908-NEXT: {{  $}}
+    ; GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+    ; GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+    ;
+    ; NO-AGPR-LABEL: name: no_agprs
+    ; NO-AGPR: liveins: $vgpr0
+    ; NO-AGPR-NEXT: {{  $}}
+    ; NO-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    %0:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
new file mode 100644
index 0000000000000..1a457c94778fd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
@@ -0,0 +1,74 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+
+---
+name:            bad_ra
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sgpr_64, preferred-register: '$sgpr4_sgpr5' }
+  - { id: 1, class: sgpr_128, preferred-register: '%2' }
+  - { id: 2, class: areg_128, preferred-register: '%1' }
+  - { id: 3, class: areg_128, preferred-register: '%4' }
+  - { id: 4, class: av_128, preferred-register: '%3' }
+  - { id: 5, class: areg_128, preferred-register: '%6' }
+  - { id: 6, class: vreg_128, preferred-register: '%5' }
+  - { id: 7, class: areg_128, preferred-register: '%4' }
+  - { id: 8, class: vgpr_32 }
+  - { id: 9, class: vgpr_32 }
+  - { id: 10, class: vgpr_32 }
+  - { id: 11, class: areg_128 }
+liveins:
+  - { reg: '$sgpr4_sgpr5', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  vgprForAGPRCopy: '$vgpr255'
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  bb.0:
+    liveins: $sgpr4_sgpr5
+
+    ; CHECK-LABEL: name: bad_ra
+    ; CHECK: liveins: $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1)
+    ; CHECK-NEXT: renamable $vgpr4 = V_MOV_B32_e32 1065353216, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr5 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr6 = V_MOV_B32_e32 1073741824, implicit $exec
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr1 = COPY renamable $agpr1
+    ; CHECK-NEXT: renamable $vgpr0 = COPY renamable $agpr0
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr3 = COPY renamable $agpr1
+    ; CHECK-NEXT: renamable $vgpr2 = COPY killed renamable $agpr0
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 killed $vgpr4, killed $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr5, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1)
+    %8:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+    %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %10:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec
+    %2:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
+    %3:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %2, 0, 0, 0, implicit $mode, implicit $exec
+    undef %4.sub1:av_128 = COPY %3.sub1
+    %4.sub0:av_128 = COPY %3.sub0
+    %11:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %3, 0, 0, 0, implicit $mode, implicit $exec
+    %4.sub3:av_128 = COPY %11.sub1
+    %4.sub2:av_128 = COPY %11.sub0
+    %7:areg_128 = COPY %4
+    %5:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %7, 0, 0, 0, implicit $mode, implicit $exec
+    %6:vreg_128 = COPY %5
+    GLOBAL_STORE_DWORDX4_SADDR %9, %6, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index ae90cfb631e8d..7eb7d72e6cb97 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -25,7 +25,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr17, 8, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_CSELECT_B64 -1, 0, implicit killed $scc
   ; GFX90A-NEXT:   renamable $sgpr30_sgpr31 = S_XOR_B64 killed renamable $sgpr18_sgpr19, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = DS_READ_B32_gfx9 renamable $vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3)
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr28_sgpr29, implicit-def dead $scc
@@ -56,8 +56,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr30 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $vgpr15 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr17 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr15 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr17 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_VCCZ %bb.57, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.4.bb15:
@@ -112,14 +112,14 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   successors: %bb.7(0x80000000)
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr21 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr20 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr23 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr22 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr25 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr24 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr19 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr18 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr21 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr20 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr23 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr22 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr25 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr24 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.7.Flow19:
   ; GFX90A-NEXT:   successors: %bb.62(0x40000000), %bb.8(0x40000000)
@@ -671,7 +671,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr50_sgpr51, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr30, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.54, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
@@ -759,7 +759,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr53 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $sgpr64_sgpr65, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr10 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr14_vgpr15 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr21, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from %ir.7, addrspace 3)
@@ -801,12 +801,12 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr42_vgpr43 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr40_vgpr41 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr46_vgpr47 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr52 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr16 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr53 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr14 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr52 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr16 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr53 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr13 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr12 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_BRANCH %bb.7
   ; GFX90A-NEXT: {{  $}}
@@ -814,7 +814,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   successors: %bb.3(0x80000000)
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3)
@@ -913,7 +913,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr25, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr3 = V_OR_B32_e32 killed $vgpr11, killed $vgpr19, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U32_sdwa 0, killed $vgpr53, 0, $vgpr3, 0, 0, 6, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr10 = V_OR_B32_e32 killed $vgpr52, killed $vgpr13, implicit $exec
@@ -955,7 +955,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr26, $vgpr2, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr3, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr10, killed $vgpr3, killed $vcc, 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr27 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr27 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr49 = COPY renamable $vgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr35 = COPY renamable $vgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr39 = COPY renamable $vgpr27, implicit $exec
@@ -989,7 +989,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr10 = V_OR_B32_e32 $vgpr50, killed $vgpr16, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr54 = V_OR_B32_e32 killed $vgpr10, killed $vgpr14, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr55 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr55 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 killed renamable $vgpr55, renamable $vgpr54_vgpr55, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_BRANCH %bb.69
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 50fa7ac2a19aa..4f81d3599d1de 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -10,9 +10,9 @@
 
 ; GCN-O0: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(atomic-expand,verify,gc-lowering,lower-constant-intrinsics,unreachableblockelim,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(lower-switch,lower-invoke,unreachableblockelim,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa,require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,localstackalloc))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,phi-node-elimination,two-address-instruction,regallocfast,si-fix-vgpr-copies,remove-redundant-debug-values,fixup-statepoint-caller-saved,prolog-epilog,post-ra-pseudos,si-post-ra-bundler,fentry-insert,xray-instrumentation,patchable-function,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
-; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
+; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
-; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
+; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
 define void @empty() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index af3241e95e91d..2a5c65278f7dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -329,6 +329,7 @@
 ; GCN-O1-NEXT:        Remove dead machine instructions
 ; GCN-O1-NEXT:        SI Shrink Instructions
 ; GCN-O1-NEXT:        Register Usage Information Propagation
+; GCN-O1-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O1-NEXT:        Detect Dead Lanes
 ; GCN-O1-NEXT:        Remove dead machine instructions
 ; GCN-O1-NEXT:        Init Undef Pass
@@ -640,6 +641,7 @@
 ; GCN-O1-OPTS-NEXT:        Remove dead machine instructions
 ; GCN-O1-OPTS-NEXT:        SI Shrink Instructions
 ; GCN-O1-OPTS-NEXT:        Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O1-OPTS-NEXT:        Detect Dead Lanes
 ; GCN-O1-OPTS-NEXT:        Remove dead machine instructions
 ; GCN-O1-OPTS-NEXT:        Init Undef Pass
@@ -956,6 +958,7 @@
 ; GCN-O2-NEXT:        Remove dead machine instructions
 ; GCN-O2-NEXT:        SI Shrink Instructions
 ; GCN-O2-NEXT:        Register Usage Information Propagation
+; GCN-O2-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O2-NEXT:        Detect Dead Lanes
 ; GCN-O2-NEXT:        Remove dead machine instructions
 ; GCN-O2-NEXT:        Init Undef Pass
@@ -1286,6 +1289,7 @@
 ; GCN-O3-NEXT:        Remove dead machine instructions
 ; GCN-O3-NEXT:        SI Shrink Instructions
 ; GCN-O3-NEXT:        Register Usage Information Propagation
+; GCN-O3-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O3-NEXT:        Detect Dead Lanes
 ; GCN-O3-NEXT:        Remove dead machine instructions
 ; GCN-O3-NEXT:        Init Undef Pass
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 67ae05eb6f0b8..561eaca3b77df 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -4365,8 +4365,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
 ; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
 ; NOLIT-SRCC:       ; %bb.0: ; %bb
 ; NOLIT-SRCC-NEXT:    v_mov_b32_e32 v0, 1.0
-; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a0, 1.0
+; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a2, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a3, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a4, 0
@@ -4465,8 +4465,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
 ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
 ; LIT-SRCC:       ; %bb.0: ; %bb
 ; LIT-SRCC-NEXT:    v_mov_b32_e32 v0, 1.0
-; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a0, 1.0
+; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a2, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a3, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a4, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
index 2faf375a97a86..465414c5471ee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s
 
 declare i32 @llvm.amdgcn.prng.b32(i32) #0
 
@@ -29,4 +31,4 @@ define amdgpu_kernel void @prng_b32_constant_100(ptr addrspace(1) %out) #1 {
 
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind }
\ No newline at end of file
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
index 3844d6054e130..cf244f0b1f884 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
@@ -6,16 +6,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
 ; GFX942-LABEL: matmul_kernel:
 ; GFX942:       ; %bb.0: ; %entry
 ; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942-NEXT:    v_mov_b32_e32 v0, 0
-; GFX942-NEXT:    v_accvgpr_write_b32 a2, v0
+; GFX942-NEXT:    v_accvgpr_write_b32 a2, 0
 ; GFX942-NEXT:    s_mov_b32 s2, 0
 ; GFX942-NEXT:    v_accvgpr_write_b32 a1, 0
+; GFX942-NEXT:    s_mov_b32 s3, 0
 ; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX942-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX942-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
 ; GFX942-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX942-NEXT:    s_mov_b32 s3, 0
 ; GFX942-NEXT:    s_branch .LBB0_2
 ; GFX942-NEXT:  .LBB0_1: ; %bb2
 ; GFX942-NEXT:    ; in Loop: Header=BB0_2 Depth=1
@@ -43,16 +42,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
 ; GFX908-LABEL: matmul_kernel:
 ; GFX908:       ; %bb.0: ; %entry
 ; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX908-NEXT:    v_mov_b32_e32 v0, 0
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 0
 ; GFX908-NEXT:    v_accvgpr_write_b32 a1, 0
 ; GFX908-NEXT:    s_mov_b32 s2, 0
-; GFX908-NEXT:    v_accvgpr_write_b32 a2, v0
+; GFX908-NEXT:    s_mov_b32 s3, 0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX908-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX908-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX908-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
 ; GFX908-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX908-NEXT:    s_mov_b32 s3, 0
 ; GFX908-NEXT:    s_branch .LBB0_2
 ; GFX908-NEXT:  .LBB0_1: ; %bb2
 ; GFX908-NEXT:    ; in Loop: Header=BB0_2 Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
index ee5481617cf59..01506d0af1913 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
@@ -80,16 +80,16 @@ body:             |
   ; COALESCE-NEXT:   S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
   ; COALESCE-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0
   ; COALESCE-NEXT:   [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec
   ; COALESCE-NEXT:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.1:
   ; COALESCE-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; COALESCE-NEXT: {{  $}}
-  ; COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
   ; COALESCE-NEXT:   $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; COALESCE-NEXT:   S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
@@ -103,10 +103,10 @@ body:             |
   ; COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]]
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.3:
@@ -134,16 +134,16 @@ body:             |
   ; GFX908-COALESCE-NEXT:   S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
   ; GFX908-COALESCE-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0
   ; GFX908-COALESCE-NEXT:   [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.1:
   ; GFX908-COALESCE-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
   ; GFX908-COALESCE-NEXT:   $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; GFX908-COALESCE-NEXT:   S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
@@ -157,10 +157,10 @@ body:             |
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]]
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
index 49c0aaf9fb390..a9207de317ea1 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
@@ -67,7 +67,7 @@ body:             |
   ; COALESCE-NEXT: bb.1:
   ; COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; COALESCE-NEXT: {{  $}}
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   S_BRANCH %bb.3
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.2:
@@ -78,13 +78,13 @@ body:             |
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], 0, 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+  ; COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.3:
-  ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; COALESCE-NEXT:   [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec
   ; COALESCE-NEXT:   undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec
-  ; COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+  ; COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
@@ -105,28 +105,28 @@ body:             |
   ; GFX908-COALESCE-NEXT: bb.1:
   ; GFX908-COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   S_BRANCH %bb.3
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.2:
   ; GFX908-COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[AV_MOV_1]].sub0
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub0
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]].sub0_sub1
-  ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.3:
-  ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; GFX908-COALESCE-NEXT:   [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 663fd98b46bf7..ce96766116089 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -17,9 +17,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
@@ -42,8 +42,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX908-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
-  ; PEI-GFX908-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-  ; PEI-GFX908-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+  ; PEI-GFX908-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; PEI-GFX908-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
   ; PEI-GFX908-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
   ; PEI-GFX908-NEXT:   $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
@@ -62,9 +62,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX90A-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; REGALLOC-GFX90A-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; REGALLOC-GFX90A-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; REGALLOC-GFX90A-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64_align2, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64_align2, [[V_MFMA_I32_4X4X4I8_e64_]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   S_ENDPGM 0
@@ -85,8 +85,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX90A-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
-  ; PEI-GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-  ; PEI-GFX90A-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+  ; PEI-GFX90A-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; PEI-GFX90A-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
   ; PEI-GFX90A-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
   ; PEI-GFX90A-NEXT:   $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index 1ec94162951a6..d48bfe0bb7f21 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -145,49 +145,29 @@ entry:
 
 ; Test skipping the lower-32-bit addition if it is unnecessary.
 define ptr @huge_offset_low_32_unused(ptr %p) {
-; GFX942_PTRADD-LABEL: huge_offset_low_32_unused:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    s_mov_b32 s0, 0
-; GFX942_PTRADD-NEXT:    s_mov_b32 s1, 1
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: huge_offset_low_32_unused:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+; GFX942-LABEL: huge_offset_low_32_unused:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_add_u32_e32 v1, 1, v1
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr inbounds i8, ptr %p, i64 u0x100000000
   ret ptr %gep
 }
 
 ; Reassociate address computation if it leads to more scalar operations.
 define amdgpu_kernel void @reassoc_scalar_r(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
-; GFX942_PTRADD-LABEL: reassoc_scalar_r:
-; GFX942_PTRADD:       ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_PTRADD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], v[0:1], 0, s[6:7]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: reassoc_scalar_r:
-; GFX942_LEGACY:       ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    s_add_u32 s2, s2, s6
-; GFX942_LEGACY-NEXT:    s_addc_u32 s3, s3, s7
-; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: reassoc_scalar_r:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_u32 s2, s2, s6
+; GFX942-NEXT:    s_addc_u32 s3, s3, s7
+; GFX942-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
 entry:
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
@@ -198,30 +178,18 @@ entry:
 }
 
 define amdgpu_kernel void @reassoc_scalar_l(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
-; GFX942_PTRADD-LABEL: reassoc_scalar_l:
-; GFX942_PTRADD:       ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_PTRADD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[6:7], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: reassoc_scalar_l:
-; GFX942_LEGACY:       ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    s_add_u32 s2, s2, s6
-; GFX942_LEGACY-NEXT:    s_addc_u32 s3, s3, s7
-; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: reassoc_scalar_l:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_u32 s2, s2, s6
+; GFX942-NEXT:    s_addc_u32 s3, s3, s7
+; GFX942-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
 entry:
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
@@ -233,24 +201,14 @@ entry:
 
 ; Tests the target-specific (ptradd x, shl(0 - y, k)) -> sub(x, shl(y, k)) fold
 define ptr addrspace(1) @shl_neg_offset(ptr addrspace(1) %p, i64 %noffset, i64 %shift) {
-; GFX942_PTRADD-LABEL: shl_neg_offset:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_sub_co_u32_e32 v2, vcc, 0, v2
-; GFX942_PTRADD-NEXT:    s_nop 1
-; GFX942_PTRADD-NEXT:    v_subb_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: shl_neg_offset:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX942_LEGACY-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
-; GFX942_LEGACY-NEXT:    s_nop 1
-; GFX942_LEGACY-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+; GFX942-LABEL: shl_neg_offset:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %offset = sub i64 0, %noffset
   %x = shl i64 %offset, %shift
   %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 %x
@@ -268,10 +226,9 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
 ; GFX942_PTRADD:       ; %bb.0:
 ; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942_PTRADD-NEXT:    s_getpc_b64 s[0:1]
-; GFX942_PTRADD-NEXT:    s_add_u32 s0, s0, v0 at rel32@lo+4
-; GFX942_PTRADD-NEXT:    s_addc_u32 s1, s1, v0 at rel32@hi+12
+; GFX942_PTRADD-NEXT:    s_add_u32 s0, s0, v0 at rel32@lo+14
+; GFX942_PTRADD-NEXT:    s_addc_u32 s1, s1, v0 at rel32@hi+22
 ; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 10
 ; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942_LEGACY-LABEL: complextype_global_gep:
@@ -291,30 +248,63 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
 
 ; Tests the tryFoldToMad64_32 PTRADD combine.
 define amdgpu_kernel void @fold_mad64(ptr addrspace(1) %p) {
-; GFX942_PTRADD-LABEL: fold_mad64:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    v_mul_hi_u32_u24_e32 v1, 12, v0
-; GFX942_PTRADD-NEXT:    v_mul_u32_u24_e32 v0, 12, v0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, 1.0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    global_store_dword v[0:1], v2, off
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: fold_mad64:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, 1.0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dword v[0:1], v2, off
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: fold_mad64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 1.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
+; GFX942-NEXT:    global_store_dword v[0:1], v2, off
+; GFX942-NEXT:    s_endpgm
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %p, i64 %voffset, i32 0
   store float 1.0, ptr addrspace(1) %p1
   ret void
 }
+
+; Use non-zero shift amounts in v_lshl_add_u64.
+define ptr @select_v_lshl_add_u64(ptr %base, i64 %voffset) {
+; GFX942_PTRADD-LABEL: select_v_lshl_add_u64:
+; GFX942_PTRADD:       ; %bb.0:
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], 3, v[2:3]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: select_v_lshl_add_u64:
+; GFX942_LEGACY:       ; %bb.0:
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[2:3], 3, v[0:1]
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr inbounds i64, ptr %base, i64 %voffset
+  ret ptr %gep
+}
+
+; Fold mul and add into v_mad, even if amdgpu-codegenprepare-mul24 turned the
+; mul into a mul24.
+define ptr @fold_mul24_into_mad(ptr %base, i64 %a, i64 %b) {
+; GFX942_PTRADD-LABEL: fold_mul24_into_mad:
+; GFX942_PTRADD:       ; %bb.0:
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v2, 0xfffff, v2
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v4, 0xfffff, v4
+; GFX942_PTRADD-NEXT:    v_mul_hi_u32_u24_e32 v3, v2, v4
+; GFX942_PTRADD-NEXT:    v_mul_u32_u24_e32 v2, v2, v4
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: fold_mul24_into_mad:
+; GFX942_LEGACY:       ; %bb.0:
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v2, 0xfffff, v2
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v3, 0xfffff, v4
+; GFX942_LEGACY-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v2, v3, v[0:1]
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+  %a_masked = and i64 %a, u0xfffff
+  %b_masked = and i64 %b, u0xfffff
+  %mul = mul i64 %a_masked, %b_masked
+  %gep = getelementptr inbounds i8, ptr %base, i64 %mul
+  ret ptr %gep
+}
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index bd255e88b9512..648b59f69ea79 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -9,9 +9,9 @@ define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; GCN-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; GCN-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; GCN-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GCN-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; GCN-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; GCN-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; GCN-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 2228234 /* regdef:VGPR_32 */, def undef %14.sub0
   ; GCN-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
   ; GCN-NEXT:   GLOBAL_STORE_DWORDX4 undef %24:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
index d07e2914c753a..f7653af1fa9ba 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
@@ -122,23 +122,23 @@ define i64 @callee_large_scalars(i256 %a, i256 %b) nounwind {
 define i64 @caller_large_scalars() nounwind {
 ; CHECK-LABEL: caller_large_scalars:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $zero, $sp, 24
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $zero, $sp, 40
 ; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 8
+; CHECK-NEXT:    vst $vr0, $sp, 24
 ; CHECK-NEXT:    ori $a0, $zero, 2
-; CHECK-NEXT:    st.d $a0, $sp, 0
-; CHECK-NEXT:    st.d $zero, $sp, 56
-; CHECK-NEXT:    vst $vr0, $sp, 40
+; CHECK-NEXT:    st.d $a0, $sp, 16
+; CHECK-NEXT:    st.d $zero, $sp, 72
+; CHECK-NEXT:    vst $vr0, $sp, 56
 ; CHECK-NEXT:    ori $a2, $zero, 1
-; CHECK-NEXT:    addi.d $a0, $sp, 32
-; CHECK-NEXT:    addi.d $a1, $sp, 0
-; CHECK-NEXT:    st.d $a2, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 48
+; CHECK-NEXT:    addi.d $a1, $sp, 16
+; CHECK-NEXT:    st.d $a2, $sp, 48
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(callee_large_scalars)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %1 = call i64 @callee_large_scalars(i256 1, i256 2)
   ret i64 %1
@@ -177,20 +177,20 @@ define i64 @callee_large_scalars_exhausted_regs(i64 %a, i64 %b, i64 %c, i64 %d,
 define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; CHECK-LABEL: caller_large_scalars_exhausted_regs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -96
-; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $a0, $sp, 16
+; CHECK-NEXT:    addi.d $sp, $sp, -112
+; CHECK-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    st.d $a0, $sp, 8
 ; CHECK-NEXT:    ori $a0, $zero, 9
 ; CHECK-NEXT:    st.d $a0, $sp, 0
-; CHECK-NEXT:    st.d $zero, $sp, 40
+; CHECK-NEXT:    st.d $zero, $sp, 56
 ; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 24
+; CHECK-NEXT:    vst $vr0, $sp, 40
 ; CHECK-NEXT:    ori $a0, $zero, 10
-; CHECK-NEXT:    st.d $a0, $sp, 16
-; CHECK-NEXT:    st.d $zero, $sp, 72
+; CHECK-NEXT:    st.d $a0, $sp, 32
+; CHECK-NEXT:    st.d $zero, $sp, 88
 ; CHECK-NEXT:    ori $a0, $zero, 8
-; CHECK-NEXT:    st.d $a0, $sp, 48
+; CHECK-NEXT:    st.d $a0, $sp, 64
 ; CHECK-NEXT:    ori $a0, $zero, 1
 ; CHECK-NEXT:    ori $a1, $zero, 2
 ; CHECK-NEXT:    ori $a2, $zero, 3
@@ -198,12 +198,12 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; CHECK-NEXT:    ori $a4, $zero, 5
 ; CHECK-NEXT:    ori $a5, $zero, 6
 ; CHECK-NEXT:    ori $a6, $zero, 7
-; CHECK-NEXT:    addi.d $a7, $sp, 48
-; CHECK-NEXT:    vst $vr0, $sp, 56
+; CHECK-NEXT:    addi.d $a7, $sp, 64
+; CHECK-NEXT:    vst $vr0, $sp, 72
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(callee_large_scalars_exhausted_regs)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 96
+; CHECK-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 112
 ; CHECK-NEXT:    ret
   %1 = call i64 @callee_large_scalars_exhausted_regs(
       i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i256 8, i64 9,
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-half.ll b/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
index c88b67f13d1e7..da8c3e93f6842 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
@@ -1252,8 +1252,8 @@ define i32 @caller_half_on_stack() nounwind {
 ;
 ; LA64F-LP64S-LABEL: caller_half_on_stack:
 ; LA64F-LP64S:       # %bb.0:
-; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -80
-; LA64F-LP64S-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -96
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
 ; LA64F-LP64S-NEXT:    lu12i.w $a0, -12
 ; LA64F-LP64S-NEXT:    ori $a1, $a0, 3200
 ; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
@@ -1292,8 +1292,8 @@ define i32 @caller_half_on_stack() nounwind {
 ; LA64F-LP64S-NEXT:    st.w $t0, $sp, 0
 ; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
 ; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
-; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 80
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 96
 ; LA64F-LP64S-NEXT:    ret
 ;
 ; LA64F-LP64D-LABEL: caller_half_on_stack:
@@ -1336,8 +1336,8 @@ define i32 @caller_half_on_stack() nounwind {
 ;
 ; LA64D-LP64S-LABEL: caller_half_on_stack:
 ; LA64D-LP64S:       # %bb.0:
-; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -80
-; LA64D-LP64S-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -96
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
 ; LA64D-LP64S-NEXT:    lu12i.w $a0, -12
 ; LA64D-LP64S-NEXT:    ori $a1, $a0, 3200
 ; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
@@ -1376,8 +1376,8 @@ define i32 @caller_half_on_stack() nounwind {
 ; LA64D-LP64S-NEXT:    st.w $t0, $sp, 0
 ; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
 ; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
-; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 80
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 96
 ; LA64D-LP64S-NEXT:    ret
 ;
 ; LA64D-LP64D-LABEL: caller_half_on_stack:
diff --git a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
index 52d8dd05aaa4c..1a9de3b0ef3d1 100644
--- a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
+++ b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
@@ -14,41 +14,41 @@
 define dso_local noundef signext i32 @main() nounwind {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -272
-; CHECK-NEXT:    st.d $ra, $sp, 264 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -288
+; CHECK-NEXT:    st.d $ra, $sp, 280 # 8-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
 ; CHECK-NEXT:    xvld $xr0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    xvst $xr0, $sp, 96 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 112 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_1)
 ; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI0_1)
-; CHECK-NEXT:    xvst $xr1, $sp, 64 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr1, $sp, 80 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_2)
 ; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_2)
-; CHECK-NEXT:    xvst $xr2, $sp, 32 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr2, $sp, 48 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_3)
 ; CHECK-NEXT:    xvld $xr3, $a0, %pc_lo12(.LCPI0_3)
-; CHECK-NEXT:    xvst $xr3, $sp, 0 # 32-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 136
-; CHECK-NEXT:    xvst $xr1, $sp, 168
-; CHECK-NEXT:    xvst $xr2, $sp, 200
-; CHECK-NEXT:    xvst $xr3, $sp, 232
-; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    xvst $xr3, $sp, 16 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 152
+; CHECK-NEXT:    xvst $xr1, $sp, 184
+; CHECK-NEXT:    xvst $xr2, $sp, 216
+; CHECK-NEXT:    xvst $xr3, $sp, 248
+; CHECK-NEXT:    addi.d $a0, $sp, 152
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(foo)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 96 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 136
-; CHECK-NEXT:    xvld $xr0, $sp, 64 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 168
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 200
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 232
-; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    xvld $xr0, $sp, 112 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 152
+; CHECK-NEXT:    xvld $xr0, $sp, 80 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 184
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 216
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 248
+; CHECK-NEXT:    addi.d $a0, $sp, 152
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(bar)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    move $a0, $zero
-; CHECK-NEXT:    ld.d $ra, $sp, 264 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 272
+; CHECK-NEXT:    ld.d $ra, $sp, 280 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 288
 ; CHECK-NEXT:    ret
 entry:
   %s = alloca %struct.S, align 2
diff --git a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
index ccc5c703e71ed..15ac95dfc6c55 100644
--- a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
+++ b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
@@ -28,12 +28,12 @@ define void @func() {
 ; CHECK-NEXT:    ld.w $a3, $a1, 0
 ; CHECK-NEXT:    ld.w $a2, $a1, 0
 ; CHECK-NEXT:    ld.w $a0, $a1, 0
-; CHECK-NEXT:    st.d $fp, $sp, 0
+; CHECK-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
 ; CHECK-NEXT:    lu12i.w $fp, 1
 ; CHECK-NEXT:    ori $fp, $fp, 12
 ; CHECK-NEXT:    add.d $fp, $sp, $fp
 ; CHECK-NEXT:    st.w $t8, $fp, 0
-; CHECK-NEXT:    ld.d $fp, $sp, 0
+; CHECK-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
 ; CHECK-NEXT:    st.w $t8, $a1, 0
 ; CHECK-NEXT:    st.w $t7, $a1, 0
 ; CHECK-NEXT:    st.w $t6, $a1, 0
diff --git a/llvm/test/CodeGen/LoongArch/frame.ll b/llvm/test/CodeGen/LoongArch/frame.ll
index 048703029d8c6..b29d8634854f3 100644
--- a/llvm/test/CodeGen/LoongArch/frame.ll
+++ b/llvm/test/CodeGen/LoongArch/frame.ll
@@ -1,5 +1,6 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+d,-lsx < %s | FileCheck %s --check-prefixes=CHECK,NOLSX
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lsx < %s | FileCheck %s --check-prefixes=CHECK,LSX
 
 %struct.key_t = type { i32, [16 x i8] }
 
@@ -7,20 +8,35 @@ declare void @llvm.memset.p0.i64(ptr, i8, i64, i1)
 declare void @test1(ptr)
 
 define i32 @test() nounwind {
-; CHECK-LABEL: test:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -32
-; CHECK-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; CHECK-NEXT:    st.w $zero, $sp, 16
-; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 4
-; CHECK-NEXT:    pcaddu18i $ra, %call36(test1)
-; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    move $a0, $zero
-; CHECK-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 32
-; CHECK-NEXT:    ret
+; NOLSX-LABEL: test:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -32
+; NOLSX-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; NOLSX-NEXT:    st.w $zero, $sp, 16
+; NOLSX-NEXT:    st.d $zero, $sp, 8
+; NOLSX-NEXT:    st.d $zero, $sp, 0
+; NOLSX-NEXT:    addi.d $a0, $sp, 4
+; NOLSX-NEXT:    pcaddu18i $ra, %call36(test1)
+; NOLSX-NEXT:    jirl $ra, $ra, 0
+; NOLSX-NEXT:    move $a0, $zero
+; NOLSX-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; NOLSX-NEXT:    addi.d $sp, $sp, 32
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -32
+; LSX-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LSX-NEXT:    st.w $zero, $sp, 16
+; LSX-NEXT:    vrepli.b $vr0, 0
+; LSX-NEXT:    vst $vr0, $sp, 0
+; LSX-NEXT:    addi.d $a0, $sp, 4
+; LSX-NEXT:    pcaddu18i $ra, %call36(test1)
+; LSX-NEXT:    jirl $ra, $ra, 0
+; LSX-NEXT:    move $a0, $zero
+; LSX-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LSX-NEXT:    addi.d $sp, $sp, 32
+; LSX-NEXT:    ret
   %key = alloca %struct.key_t, align 4
   call void @llvm.memset.p0.i64(ptr %key, i8 0, i64 20, i1 false)
   %1 = getelementptr inbounds %struct.key_t, ptr %key, i64 0, i32 1, i64 0
@@ -98,3 +114,62 @@ define void @test_large_frame_size_1234576() "frame-pointer"="all" {
   %1 = alloca i8, i32 1234567
   ret void
 }
+
+;; Note: will create an emergency spill slot, if (!isInt<7>(StackSize)).
+;; Should involve only one SP-adjusting addi per adjustment.
+;; LSX 112 + 16(emergency solt) = 128
+define void @test_frame_size_112() {
+; NOLSX-LABEL: test_frame_size_112:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -112
+; NOLSX-NEXT:    .cfi_def_cfa_offset 112
+; NOLSX-NEXT:    addi.d $sp, $sp, 112
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_112:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -128
+; LSX-NEXT:    .cfi_def_cfa_offset 128
+; LSX-NEXT:    addi.d $sp, $sp, 128
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 112
+  ret void
+}
+
+;; LSX 128 + 16(emergency solt) = 144
+define void @test_frame_size_128() {
+; NOLSX-LABEL: test_frame_size_128:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -128
+; NOLSX-NEXT:    .cfi_def_cfa_offset 128
+; NOLSX-NEXT:    addi.d $sp, $sp, 128
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_128:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -144
+; LSX-NEXT:    .cfi_def_cfa_offset 144
+; LSX-NEXT:    addi.d $sp, $sp, 144
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 128
+  ret void
+}
+
+;; LSX 144 + 16(emergency solt) = 160
+define void @test_frame_size_144() {
+; NOLSX-LABEL: test_frame_size_144:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -144
+; NOLSX-NEXT:    .cfi_def_cfa_offset 144
+; NOLSX-NEXT:    addi.d $sp, $sp, 144
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_144:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -160
+; LSX-NEXT:    .cfi_def_cfa_offset 160
+; LSX-NEXT:    addi.d $sp, $sp, 160
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 144
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
index 402ddb9ad941b..5a55b253c77bb 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
@@ -6,11 +6,11 @@
 define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 dereferenceable(48) %b, i64 %i) {
 ; CHECK-LABEL: box:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -96
-; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    addi.d $sp, $sp, -112
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
 ; CHECK-NEXT:    slli.d $a2, $a1, 5
 ; CHECK-NEXT:    alsl.d $a1, $a1, $a2, 4
-; CHECK-NEXT:    addi.d $a2, $sp, 0
+; CHECK-NEXT:    addi.d $a2, $sp, 16
 ; CHECK-NEXT:    add.d $a3, $a2, $a1
 ; CHECK-NEXT:    vldx $vr0, $a1, $a2
 ; CHECK-NEXT:    vld $vr1, $a3, 32
@@ -18,7 +18,7 @@ define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 der
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    vst $vr1, $a0, 32
 ; CHECK-NEXT:    vst $vr2, $a0, 16
-; CHECK-NEXT:    addi.d $sp, $sp, 96
+; CHECK-NEXT:    addi.d $sp, $sp, 112
 ; CHECK-NEXT:    ret
   %1 = alloca [2 x %Box], align 16
   %2 = getelementptr inbounds [2 x %Box], ptr %1, i64 0, i64 %i
diff --git a/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll b/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
index 789b51d9b5e5b..9528280d181a3 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
@@ -6,10 +6,10 @@ declare <8 x float> @llvm.powi.v8f32.i32(<8 x float>, i32)
 define <8 x float> @powi_v8f32(<8 x float> %va, i32 %b) nounwind {
 ; CHECK-LABEL: powi_v8f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 0 # 32-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 16 # 32-byte Folded Spill
 ; CHECK-NEXT:    addi.w $fp, $a0, 0
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 0
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
@@ -18,79 +18,79 @@ define <8 x float> @powi_v8f32(<8 x float> %va, i32 %b) nounwind {
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 1
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 2
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 2
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 3
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 3
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 4
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 4
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 5
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 5
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 6
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 6
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 7
-; CHECK-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
 entry:
   %res = call <8 x float> @llvm.powi.v8f32.i32(<8 x float> %va, i32 %b)
@@ -102,10 +102,10 @@ declare <4 x double> @llvm.powi.v4f64.i32(<4 x double>, i32)
 define <4 x double> @powi_v4f64(<4 x double> %va, i32 %b) nounwind {
 ; CHECK-LABEL: powi_v4f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 0 # 32-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 16 # 32-byte Folded Spill
 ; CHECK-NEXT:    addi.w $fp, $a0, 0
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
@@ -114,39 +114,39 @@ define <4 x double> @powi_v4f64(<4 x double> %va, i32 %b) nounwind {
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 2
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 3
-; CHECK-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
 entry:
   %res = call <4 x double> @llvm.powi.v4f64.i32(<4 x double> %va, i32 %b)
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
index 04214f5dfa9d2..2e1618748688a 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
@@ -76,21 +76,21 @@ define void @extract_4xdouble(ptr %src, ptr %dst) nounwind {
 define void @extract_32xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_32xi8_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 0
 ; CHECK-NEXT:    ld.b $a0, $a0, 0
 ; CHECK-NEXT:    st.b $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <32 x i8>, ptr %src
   %e = extractelement <32 x i8> %v, i32 %idx
@@ -101,21 +101,21 @@ define void @extract_32xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_16xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_16xi16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 1
 ; CHECK-NEXT:    ld.h $a0, $a0, 0
 ; CHECK-NEXT:    st.h $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <16 x i16>, ptr %src
   %e = extractelement <16 x i16> %v, i32 %idx
@@ -126,21 +126,21 @@ define void @extract_16xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_8xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_8xi32_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    ld.w $a0, $a0, 0
 ; CHECK-NEXT:    st.w $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x i32>, ptr %src
   %e = extractelement <8 x i32> %v, i32 %idx
@@ -151,21 +151,21 @@ define void @extract_8xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_4xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_4xi64_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    ld.d $a0, $a0, 0
 ; CHECK-NEXT:    st.d $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x i64>, ptr %src
   %e = extractelement <4 x i64> %v, i32 %idx
@@ -176,21 +176,21 @@ define void @extract_4xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_8xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_8xfloat_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    fld.s $fa0, $a0, 0
 ; CHECK-NEXT:    fst.s $fa0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x float>, ptr %src
   %e = extractelement <8 x float> %v, i32 %idx
@@ -201,21 +201,21 @@ define void @extract_8xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_4xdouble_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_4xdouble_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    fld.d $fa0, $a0, 0
 ; CHECK-NEXT:    fst.d $fa0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x double>, ptr %src
   %e = extractelement <4 x double> %v, i32 %idx
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
index 3a4f6efd2c893..b24f95e676d10 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
@@ -114,22 +114,22 @@ define void @insert_4xdouble(ptr %src, ptr %dst, double %in) nounwind {
 define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_32xi8_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 0
 ; CHECK-NEXT:    st.b $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <32 x i8>, ptr %src
   %v_new = insertelement <32 x i8> %v, i8 %in, i32 %idx
@@ -140,22 +140,22 @@ define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
 define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_16xi16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 1
 ; CHECK-NEXT:    st.h $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <16 x i16>, ptr %src
   %v_new = insertelement <16 x i16> %v, i16 %in, i32 %idx
@@ -166,22 +166,22 @@ define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
 define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_8xi32_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 2
 ; CHECK-NEXT:    st.w $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x i32>, ptr %src
   %v_new = insertelement <8 x i32> %v, i32 %in, i32 %idx
@@ -192,22 +192,22 @@ define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
 define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_4xi64_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 3
 ; CHECK-NEXT:    st.d $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x i64>, ptr %src
   %v_new = insertelement <4 x i64> %v, i64 %in, i32 %idx
@@ -218,22 +218,22 @@ define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
 define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_8xfloat_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
-; CHECK-NEXT:    xvst $xr1, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr1, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    fst.s $fa0, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x float>, ptr %src
   %v_new = insertelement <8 x float> %v, float %in, i32 %idx
@@ -244,22 +244,22 @@ define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwin
 define void @insert_4xdouble_idx(ptr %src, ptr %dst, double %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_4xdouble_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
-; CHECK-NEXT:    xvst $xr1, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr1, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    fst.d $fa0, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x double>, ptr %src
   %v_new = insertelement <4 x double> %v, double %in, i32 %idx
diff --git a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
index ffedd7f9e9438..648c19d509715 100644
--- a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
+++ b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
@@ -347,42 +347,42 @@ define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    vpackev.w $vr0, $vr0, $vr1
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vpackev.w $vr1, $vr0, $vr1
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
   %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
   ret { <2 x float>, <2 x float> } %result
@@ -439,48 +439,48 @@ define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v3f32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -96
-; LA64-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -112
+; LA64-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 2
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 72
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 88
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 68
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 84
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 64
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 80
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 56
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 72
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 52
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 68
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 48
-; LA64-NEXT:    vld $vr0, $sp, 64
-; LA64-NEXT:    vld $vr1, $sp, 48
-; LA64-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 96
+; LA64-NEXT:    fst.s $fa0, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 80
+; LA64-NEXT:    vld $vr1, $sp, 64
+; LA64-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 112
 ; LA64-NEXT:    ret
   %result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a)
   ret { <3 x float>, <3 x float> } %result
@@ -568,44 +568,44 @@ define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f64:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
-; LA64-NEXT:    vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
 ; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.d $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 1
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 0
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
-; LA64-NEXT:    vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    vinsgr2vr.d $vr1, $a0, 1
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
   %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
   ret { <2 x double>, <2 x double> } %result
@@ -801,17 +801,17 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f128:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -80
-; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s6, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s7, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -96
+; LA64-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s1, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s2, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s3, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s4, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s5, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s6, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s7, $sp, 16 # 8-byte Folded Spill
 ; LA64-NEXT:    ld.d $fp, $a1, 16
 ; LA64-NEXT:    ld.d $s0, $a1, 24
 ; LA64-NEXT:    ld.d $s1, $a1, 0
@@ -847,17 +847,17 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
 ; LA64-NEXT:    st.d $s6, $s3, 16
 ; LA64-NEXT:    st.d $s5, $s3, 8
 ; LA64-NEXT:    st.d $s4, $s3, 0
-; LA64-NEXT:    ld.d $s7, $sp, 0 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s6, $sp, 8 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 80
+; LA64-NEXT:    ld.d $s7, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s6, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s5, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s4, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s3, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s2, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s1, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 96
 ; LA64-NEXT:    ret
   %result = call { <2 x fp128>, <2 x fp128> } @llvm.sincos.v2f128(<2 x fp128> %a)
   ret { <2 x fp128>, <2 x fp128> } %result
diff --git a/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll b/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
new file mode 100644
index 0000000000000..96159e5884d3f
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lsx --verify-machineinstrs < %s | FileCheck %s
+define void @eliminate_frame_index(<16 x i8> %a) nounwind {
+; CHECK-LABEL: eliminate_frame_index:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.d $sp, $sp, -240
+; CHECK-NEXT:    st.d $ra, $sp, 232 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 224 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s0, $sp, 216 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s1, $sp, 208 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s2, $sp, 200 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s3, $sp, 192 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s4, $sp, 184 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s5, $sp, 176 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s6, $sp, 168 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s7, $sp, 160 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s8, $sp, 152 # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $zero, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $ra, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $tp, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t8, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $fp, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s8, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    st.d $a0, $sp, 0 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    vstelm.b $vr0, $a0, 0, 0
+; CHECK-NEXT:    ld.d $a0, $sp, 0 # 8-byte Folded Reload
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $zero
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $ra
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $tp
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t8
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $fp
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s8
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    ld.d $s8, $sp, 152 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s7, $sp, 160 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s6, $sp, 168 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s5, $sp, 176 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s4, $sp, 184 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s3, $sp, 192 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s2, $sp, 200 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s1, $sp, 208 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s0, $sp, 216 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $fp, $sp, 224 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 232 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 240
+; CHECK-NEXT:    ret
+  %s = alloca [16 x i8]
+  %ss = alloca [128 x i8]
+
+  %zero =  call i64 asm sideeffect "addi.d $$zero, $$zero, 1", "={r0}"()
+  %ra =  call i64 asm sideeffect "addi.d $$ra, $$zero, 1", "={r1}"()
+  %tp =  call i64 asm sideeffect "addi.d $$tp, $$zero, 1", "={r2}"()
+  %a0 =  call i64 asm sideeffect "addi.d $$a0, $$zero, 1", "={r4}"()
+  %a1 =  call i64 asm sideeffect "addi.d $$a1, $$zero, 1", "={r5}"()
+  %a2 =  call i64 asm sideeffect "addi.d $$a2, $$zero, 1", "={r6}"()
+  %a3 =  call i64 asm sideeffect "addi.d $$a3, $$zero, 1", "={r7}"()
+  %a4 =  call i64 asm sideeffect "addi.d $$a4, $$zero, 1", "={r8}"()
+  %a5 =  call i64 asm sideeffect "addi.d $$a5, $$zero, 1", "={r9}"()
+  %a6 =  call i64 asm sideeffect "addi.d $$a6, $$zero, 1", "={r10}"()
+  %a7 =  call i64 asm sideeffect "addi.d $$a7, $$zero, 1", "={r11}"()
+  %t0 =  call i64 asm sideeffect "addi.d $$t0, $$zero, 1", "={r12}"()
+  %t1 =  call i64 asm sideeffect "addi.d $$t1, $$zero, 1", "={r13}"()
+  %t2 =  call i64 asm sideeffect "addi.d $$t2, $$zero, 1", "={r14}"()
+  %t3 =  call i64 asm sideeffect "addi.d $$t3, $$zero, 1", "={r15}"()
+  %t4 =  call i64 asm sideeffect "addi.d $$t4, $$zero, 1", "={r16}"()
+  %t5 =  call i64 asm sideeffect "addi.d $$t5, $$zero, 1", "={r17}"()
+  %t6 =  call i64 asm sideeffect "addi.d $$t6, $$zero, 1", "={r18}"()
+  %t7 =  call i64 asm sideeffect "addi.d $$t7, $$zero, 1", "={r19}"()
+  %t8 =  call i64 asm sideeffect "addi.d $$t8, $$zero, 1", "={r20}"()
+  ;; r21 Reserved (Non-allocatable)
+  %s9 =  call i64 asm sideeffect "addi.d $$s9, $$zero, 1", "={r22}"()
+  %s0 =  call i64 asm sideeffect "addi.d $$s0, $$zero, 1", "={r23}"()
+  %s1 =  call i64 asm sideeffect "addi.d $$s1, $$zero, 1", "={r24}"()
+  %s2 =  call i64 asm sideeffect "addi.d $$s2, $$zero, 1", "={r25}"()
+  %s3 =  call i64 asm sideeffect "addi.d $$s3, $$zero, 1", "={r26}"()
+  %s4 =  call i64 asm sideeffect "addi.d $$s4, $$zero, 1", "={r27}"()
+  %s5 =  call i64 asm sideeffect "addi.d $$s5, $$zero, 1", "={r28}"()
+  %s6 =  call i64 asm sideeffect "addi.d $$s6, $$zero, 1", "={r29}"()
+  %s7 =  call i64 asm sideeffect "addi.d $$s7, $$zero, 1", "={r30}"()
+  %s8 =  call i64 asm sideeffect "addi.d $$s8, $$zero, 1", "={r31}"()
+
+  %e = extractelement <16 x i8> %a, i64 0
+
+  store volatile i8 %e, ptr %s
+
+  call void asm sideeffect "# reg use $0", "{r0}"(i64 %zero)
+  call void asm sideeffect "# reg use $0", "{r1}"(i64 %ra)
+  call void asm sideeffect "# reg use $0", "{r2}"(i64 %tp)
+  call void asm sideeffect "# reg use $0", "{r4}"(i64 %a0)
+  call void asm sideeffect "# reg use $0", "{r5}"(i64 %a1)
+  call void asm sideeffect "# reg use $0", "{r6}"(i64 %a2)
+  call void asm sideeffect "# reg use $0", "{r7}"(i64 %a3)
+  call void asm sideeffect "# reg use $0", "{r8}"(i64 %a4)
+  call void asm sideeffect "# reg use $0", "{r9}"(i64 %a5)
+  call void asm sideeffect "# reg use $0", "{r10}"(i64 %a6)
+  call void asm sideeffect "# reg use $0", "{r11}"(i64 %a7)
+  call void asm sideeffect "# reg use $0", "{r12}"(i64 %t0)
+  call void asm sideeffect "# reg use $0", "{r13}"(i64 %t1)
+  call void asm sideeffect "# reg use $0", "{r14}"(i64 %t2)
+  call void asm sideeffect "# reg use $0", "{r15}"(i64 %t3)
+  call void asm sideeffect "# reg use $0", "{r16}"(i64 %t4)
+  call void asm sideeffect "# reg use $0", "{r17}"(i64 %t5)
+  call void asm sideeffect "# reg use $0", "{r18}"(i64 %t6)
+  call void asm sideeffect "# reg use $0", "{r19}"(i64 %t7)
+  call void asm sideeffect "# reg use $0", "{r20}"(i64 %t8)
+  ;; r21 Reserved (Non-allocatable)
+  call void asm sideeffect "# reg use $0", "{r22}"(i64 %s9)
+  call void asm sideeffect "# reg use $0", "{r23}"(i64 %s0)
+  call void asm sideeffect "# reg use $0", "{r24}"(i64 %s1)
+  call void asm sideeffect "# reg use $0", "{r25}"(i64 %s2)
+  call void asm sideeffect "# reg use $0", "{r26}"(i64 %s3)
+  call void asm sideeffect "# reg use $0", "{r27}"(i64 %s4)
+  call void asm sideeffect "# reg use $0", "{r28}"(i64 %s5)
+  call void asm sideeffect "# reg use $0", "{r29}"(i64 %s6)
+  call void asm sideeffect "# reg use $0", "{r30}"(i64 %s7)
+  call void asm sideeffect "# reg use $0", "{r31}"(i64 %s8)
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
index 9f15604fcca6b..69995a0721f8a 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
@@ -36,15 +36,15 @@ define void @caller(i32 %n) {
 ;
 ; LA64-LABEL: caller:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s8, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -128
+; LA64-NEXT:    .cfi_def_cfa_offset 128
+; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s8, $sp, 104 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
 ; LA64-NEXT:    .cfi_offset 31, -24
-; LA64-NEXT:    addi.d $fp, $sp, 64
+; LA64-NEXT:    addi.d $fp, $sp, 128
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 5, 0
 ; LA64-NEXT:    move $s8, $sp
@@ -54,14 +54,14 @@ define void @caller(i32 %n) {
 ; LA64-NEXT:    slli.d $a0, $a0, 4
 ; LA64-NEXT:    sub.d $a0, $sp, $a0
 ; LA64-NEXT:    move $sp, $a0
-; LA64-NEXT:    addi.d $a1, $s8, 0
+; LA64-NEXT:    addi.d $a1, $s8, 64
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -64
-; LA64-NEXT:    ld.d $s8, $sp, 40 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    addi.d $sp, $fp, -128
+; LA64-NEXT:    ld.d $s8, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 128
 ; LA64-NEXT:    ret
   %1 = alloca i8, i32 %n
   %2 = alloca i32, align 64
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment.ll b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
index 0645339358b64..0188884543adb 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
@@ -28,22 +28,22 @@ define void @caller32() {
 ;
 ; LA64-LABEL: caller32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -32
-; LA64-NEXT:    .cfi_def_cfa_offset 32
-; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -64
+; LA64-NEXT:    .cfi_def_cfa_offset 64
+; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 32
+; LA64-NEXT:    addi.d $fp, $sp, 64
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 4, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 32
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -32
-; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    addi.d $sp, $fp, -64
+; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 64
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 32
   call void @callee(ptr %1)
@@ -102,22 +102,22 @@ define void @caller64() {
 ;
 ; LA64-LABEL: caller64:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -128
+; LA64-NEXT:    .cfi_def_cfa_offset 128
+; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 64
+; LA64-NEXT:    addi.d $fp, $sp, 128
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 5, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 64
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -64
-; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    addi.d $sp, $fp, -128
+; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 128
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 64
   call void @callee(ptr %1)
@@ -176,22 +176,22 @@ define void @caller128() {
 ;
 ; LA64-LABEL: caller128:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -128
-; LA64-NEXT:    .cfi_def_cfa_offset 128
-; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -256
+; LA64-NEXT:    .cfi_def_cfa_offset 256
+; LA64-NEXT:    st.d $ra, $sp, 248 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 240 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 128
+; LA64-NEXT:    addi.d $fp, $sp, 256
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 6, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 128
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -128
-; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 128
+; LA64-NEXT:    addi.d $sp, $fp, -256
+; LA64-NEXT:    ld.d $fp, $sp, 240 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 248 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 256
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 128
   call void @callee(ptr %1)
@@ -250,22 +250,22 @@ define void @caller256() {
 ;
 ; LA64-LABEL: caller256:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -256
-; LA64-NEXT:    .cfi_def_cfa_offset 256
-; LA64-NEXT:    st.d $ra, $sp, 248 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 240 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -512
+; LA64-NEXT:    .cfi_def_cfa_offset 512
+; LA64-NEXT:    st.d $ra, $sp, 504 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 496 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 256
+; LA64-NEXT:    addi.d $fp, $sp, 512
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 7, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 256
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -256
-; LA64-NEXT:    ld.d $fp, $sp, 240 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 248 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 256
+; LA64-NEXT:    addi.d $sp, $fp, -512
+; LA64-NEXT:    ld.d $fp, $sp, 496 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 504 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 512
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 256
   call void @callee(ptr %1)
diff --git a/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll b/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
index 925fdf3d60646..0d441e66a0c84 100644
--- a/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
+++ b/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
@@ -121,19 +121,19 @@ define void @t3() {
 ;
 ; LA64-LABEL: t3:
 ; LA64:       # %bb.0: # %entry
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    .cfi_def_cfa_offset 80
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(.L.str)
 ; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(.L.str)
 ; LA64-NEXT:    ld.h $a1, $a0, 20
 ; LA64-NEXT:    ld.w $a2, $a0, 16
 ; LA64-NEXT:    ld.d $a3, $a0, 8
 ; LA64-NEXT:    ld.d $a0, $a0, 0
-; LA64-NEXT:    st.h $a1, $sp, 20
-; LA64-NEXT:    st.w $a2, $sp, 16
-; LA64-NEXT:    st.d $a3, $sp, 8
-; LA64-NEXT:    st.d $a0, $sp, 0
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    st.h $a1, $sp, 36
+; LA64-NEXT:    st.w $a2, $sp, 32
+; LA64-NEXT:    st.d $a3, $sp, 24
+; LA64-NEXT:    st.d $a0, $sp, 16
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
 entry:
   %msgbuf = alloca [64 x i8], align 1
diff --git a/llvm/test/CodeGen/LoongArch/vararg.ll b/llvm/test/CodeGen/LoongArch/vararg.ll
index 939cd2015c5b1..bc4b8a77c7e15 100644
--- a/llvm/test/CodeGen/LoongArch/vararg.ll
+++ b/llvm/test/CodeGen/LoongArch/vararg.ll
@@ -47,7 +47,7 @@ define i64 @va1(ptr %fmt, ...) {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a1, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a1, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $a1, $fp, -32
 ; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
@@ -94,7 +94,7 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a1, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a1, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $a1, $fp, -32
 ; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
@@ -112,11 +112,11 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind {
 define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-LABEL: va1_va_arg_alloca:
 ; LA64-FPELIM:       # %bb.0:
-; LA64-FPELIM-NEXT:    addi.d $sp, $sp, -96
-; LA64-FPELIM-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    addi.d $fp, $sp, 32
+; LA64-FPELIM-NEXT:    addi.d $sp, $sp, -112
+; LA64-FPELIM-NEXT:    st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    addi.d $fp, $sp, 48
 ; LA64-FPELIM-NEXT:    move $s0, $a1
 ; LA64-FPELIM-NEXT:    st.d $a7, $fp, 56
 ; LA64-FPELIM-NEXT:    st.d $a6, $fp, 48
@@ -126,7 +126,7 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-NEXT:    st.d $a2, $fp, 16
 ; LA64-FPELIM-NEXT:    st.d $a1, $fp, 8
 ; LA64-FPELIM-NEXT:    addi.d $a0, $fp, 16
-; LA64-FPELIM-NEXT:    st.d $a0, $fp, -32
+; LA64-FPELIM-NEXT:    st.d $a0, $fp, -40
 ; LA64-FPELIM-NEXT:    addi.d $a0, $a1, 15
 ; LA64-FPELIM-NEXT:    bstrins.d $a0, $zero, 3, 0
 ; LA64-FPELIM-NEXT:    sub.d $a0, $sp, $a0
@@ -134,20 +134,20 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-NEXT:    pcaddu18i $ra, %call36(notdead)
 ; LA64-FPELIM-NEXT:    jirl $ra, $ra, 0
 ; LA64-FPELIM-NEXT:    move $a0, $s0
-; LA64-FPELIM-NEXT:    addi.d $sp, $fp, -32
-; LA64-FPELIM-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    addi.d $sp, $sp, 96
+; LA64-FPELIM-NEXT:    addi.d $sp, $fp, -48
+; LA64-FPELIM-NEXT:    ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    addi.d $sp, $sp, 112
 ; LA64-FPELIM-NEXT:    ret
 ;
 ; LA64-WITHFP-LABEL: va1_va_arg_alloca:
 ; LA64-WITHFP:       # %bb.0:
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -96
-; LA64-WITHFP-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 32
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -112
+; LA64-WITHFP-NEXT:    st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 48
 ; LA64-WITHFP-NEXT:    move $s0, $a1
 ; LA64-WITHFP-NEXT:    st.d $a7, $fp, 56
 ; LA64-WITHFP-NEXT:    st.d $a6, $fp, 48
@@ -157,7 +157,7 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a0, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a0, $fp, -32
+; LA64-WITHFP-NEXT:    st.d $a0, $fp, -40
 ; LA64-WITHFP-NEXT:    addi.d $a0, $a1, 15
 ; LA64-WITHFP-NEXT:    bstrins.d $a0, $zero, 3, 0
 ; LA64-WITHFP-NEXT:    sub.d $a0, $sp, $a0
@@ -165,11 +165,11 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    pcaddu18i $ra, %call36(notdead)
 ; LA64-WITHFP-NEXT:    jirl $ra, $ra, 0
 ; LA64-WITHFP-NEXT:    move $a0, $s0
-; LA64-WITHFP-NEXT:    addi.d $sp, $fp, -32
-; LA64-WITHFP-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
+; LA64-WITHFP-NEXT:    addi.d $sp, $fp, -48
+; LA64-WITHFP-NEXT:    ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 112
 ; LA64-WITHFP-NEXT:    ret
   %va = alloca ptr, align 8
   call void @llvm.va_start(ptr %va)
@@ -314,10 +314,10 @@ define void @va_aligned_stack_caller() nounwind {
 ;
 ; LA64-WITHFP-LABEL: va_aligned_stack_caller:
 ; LA64-WITHFP:       # %bb.0:
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -112
-; LA64-WITHFP-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $fp, $sp, 96 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 112
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -128
+; LA64-WITHFP-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 128
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 17
 ; LA64-WITHFP-NEXT:    st.d $a0, $sp, 48
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 16
@@ -336,23 +336,23 @@ define void @va_aligned_stack_caller() nounwind {
 ; LA64-WITHFP-NEXT:    lu32i.d $a0, 335544
 ; LA64-WITHFP-NEXT:    lu52i.d $a0, $a0, -328
 ; LA64-WITHFP-NEXT:    st.d $a0, $sp, 16
-; LA64-WITHFP-NEXT:    st.d $zero, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $zero, $fp, -40
 ; LA64-WITHFP-NEXT:    vrepli.b $vr0, 0
-; LA64-WITHFP-NEXT:    vst $vr0, $fp, -40
+; LA64-WITHFP-NEXT:    vst $vr0, $fp, -56
 ; LA64-WITHFP-NEXT:    ori $a5, $zero, 1000
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 1
 ; LA64-WITHFP-NEXT:    ori $a1, $zero, 11
-; LA64-WITHFP-NEXT:    addi.d $a2, $fp, -48
+; LA64-WITHFP-NEXT:    addi.d $a2, $fp, -64
 ; LA64-WITHFP-NEXT:    ori $a3, $zero, 12
 ; LA64-WITHFP-NEXT:    ori $a4, $zero, 13
 ; LA64-WITHFP-NEXT:    ori $a7, $zero, 1
-; LA64-WITHFP-NEXT:    st.d $a5, $fp, -48
+; LA64-WITHFP-NEXT:    st.d $a5, $fp, -64
 ; LA64-WITHFP-NEXT:    move $a6, $zero
 ; LA64-WITHFP-NEXT:    pcaddu18i $ra, %call36(va_aligned_stack_callee)
 ; LA64-WITHFP-NEXT:    jirl $ra, $ra, 0
-; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 96 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 112
+; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 128
 ; LA64-WITHFP-NEXT:    ret
   %1 = call i32 (i32, ...) @va_aligned_stack_callee(i32 1, i32 11,
     i256 1000, i32 12, i32 13, i128 18446744073709551616, i32 14,
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index cbc9f700b1f01..aba20e6b0f27f 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -2311,4 +2311,51 @@ entry:
   ret void
 }
 
+define <4 x float> @test_uitofp_v4i8(<4 x i8> %a) {
+; CHECK-LABEL: test_uitofp_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_uitofp_v4i8_param_0];
+; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 0x7773U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r3, %r2;
+; CHECK-NEXT:    prmt.b32 %r4, %r1, 0, 0x7772U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r5, %r4;
+; CHECK-NEXT:    prmt.b32 %r6, %r1, 0, 0x7771U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r7, %r6;
+; CHECK-NEXT:    prmt.b32 %r8, %r1, 0, 0x7770U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r9, %r8;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r7, %r5, %r3};
+; CHECK-NEXT:    ret;
+  %r = uitofp <4 x i8> %a to <4 x float>
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_sitofp_v4i8(<4 x i8> %a) {
+; CHECK-LABEL: test_sitofp_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sitofp_v4i8_param_0];
+; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 0xbbb3U;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r3, %rs1;
+; CHECK-NEXT:    prmt.b32 %r4, %r1, 0, 0xaaa2U;
+; CHECK-NEXT:    cvt.u16.u32 %rs2, %r4;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r5, %rs2;
+; CHECK-NEXT:    prmt.b32 %r6, %r1, 0, 0x9991U;
+; CHECK-NEXT:    cvt.u16.u32 %rs3, %r6;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r7, %rs3;
+; CHECK-NEXT:    prmt.b32 %r8, %r1, 0, 0x8880U;
+; CHECK-NEXT:    cvt.u16.u32 %rs4, %r8;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r9, %rs4;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r7, %r5, %r3};
+; CHECK-NEXT:    ret;
+  %r = sitofp <4 x i8> %a to <4 x float>
+  ret <4 x float> %r
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/NVPTX/trunc-tofp.ll b/llvm/test/CodeGen/NVPTX/trunc-tofp.ll
new file mode 100644
index 0000000000000..404c423cc026a
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/trunc-tofp.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_80 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mcpu=sm_80 | %ptxas-verify %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define float @uitofp_trunc_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: uitofp_trunc_nuw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [uitofp_trunc_nuw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [uitofp_trunc_nuw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r4, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+  %v = add i32 %x, %y
+  %t = trunc nuw i32 %v to i16
+  %f = uitofp i16 %t to float
+  ret float %f
+}
+
+define float @sitofp_trunc_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: sitofp_trunc_nsw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sitofp_trunc_nsw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [sitofp_trunc_nsw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.s32 %r4, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+  %v = add i32 %x, %y
+  %t = trunc nsw i32 %v to i16
+  %f = sitofp i16 %t to float
+  ret float %f
+}
+
+define float @uitofp_trunc_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: uitofp_trunc_nsw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [uitofp_trunc_nsw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [uitofp_trunc_nsw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
+; CHECK-NEXT:    cvt.rn.f32.u16 %r4, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+    %v = add i32 %x, %y
+    %t = trunc nsw i32 %v to i16
+    %f = uitofp i16 %t to float
+    ret float %f
+}
+
+define float @sitofp_trunc_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: sitofp_trunc_nuw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sitofp_trunc_nuw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [sitofp_trunc_nuw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r4, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+    %v = add i32 %x, %y
+    %t = trunc nuw i32 %v to i16
+    %f = sitofp i16 %t to float
+    ret float %f
+}
diff --git a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
index 061435c45ad0e..59a702ab6b17f 100644
--- a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
+++ b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
@@ -798,12 +798,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccor:
 ; RV64SFBSIFIVEU74:       # %bb.0: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
-; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s0, a3
+; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
 ; RV64SFBSIFIVEU74-NEXT:    mv s1, a2
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB15_4
 ; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    or s0, a3, a1
@@ -824,11 +824,11 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV64SFBANDESAX45-LABEL: sextw_removal_ccor:
 ; RV64SFBANDESAX45:       # %bb.0: # %bb
 ; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
-; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s0, a3
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s1, a2
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB15_2
 ; RV64SFBANDESAX45-NEXT:  # %bb.1:
 ; RV64SFBANDESAX45-NEXT:    or s0, s0, a1
@@ -848,12 +848,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; ZICOND-LABEL: sextw_removal_ccor:
 ; ZICOND:       # %bb.0: # %bb
 ; ZICOND-NEXT:    addi sp, sp, -32
-; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s0, a3
+; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    andi a0, a0, 1
 ; ZICOND-NEXT:    mv s1, a2
+; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    beqz a0, .LBB15_4
 ; ZICOND-NEXT:  # %bb.3: # %bb
 ; ZICOND-NEXT:    or s0, a3, a1
@@ -874,12 +874,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV32SFB-LABEL: sextw_removal_ccor:
 ; RV32SFB:       # %bb.0: # %bb
 ; RV32SFB-NEXT:    addi sp, sp, -16
-; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
-; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s0, a3
+; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    andi a0, a0, 1
 ; RV32SFB-NEXT:    mv s1, a2
+; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    beqz a0, .LBB15_4
 ; RV32SFB-NEXT:  # %bb.3: # %bb
 ; RV32SFB-NEXT:    or s0, a3, a1
@@ -941,11 +941,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccaddw:
 ; RV64SFBSIFIVEU74:       # %bb.0: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
-; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s1, a1
 ; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
+; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s0, a2
 ; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB16_4
 ; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
@@ -967,11 +967,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV64SFBANDESAX45-LABEL: sextw_removal_ccaddw:
 ; RV64SFBANDESAX45:       # %bb.0: # %bb
 ; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
-; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s0, a2
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s1, a1
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB16_2
 ; RV64SFBANDESAX45-NEXT:  # %bb.1:
 ; RV64SFBANDESAX45-NEXT:    addw s1, s1, a3
@@ -991,11 +991,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; ZICOND-LABEL: sextw_removal_ccaddw:
 ; ZICOND:       # %bb.0: # %bb
 ; ZICOND-NEXT:    addi sp, sp, -32
-; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s1, a1
 ; ZICOND-NEXT:    andi a0, a0, 1
+; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s0, a2
 ; ZICOND-NEXT:    beqz a0, .LBB16_4
 ; ZICOND-NEXT:  # %bb.3: # %bb
@@ -1017,11 +1017,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV32SFB-LABEL: sextw_removal_ccaddw:
 ; RV32SFB:       # %bb.0: # %bb
 ; RV32SFB-NEXT:    addi sp, sp, -16
-; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s1, a1
 ; RV32SFB-NEXT:    andi a0, a0, 1
+; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s0, a2
 ; RV32SFB-NEXT:    beqz a0, .LBB16_4
 ; RV32SFB-NEXT:  # %bb.3: # %bb
diff --git a/llvm/test/CodeGen/RISCV/zdinx-spill.ll b/llvm/test/CodeGen/RISCV/zdinx-spill.ll
index d7a700622bf8c..6f206fe571c17 100644
--- a/llvm/test/CodeGen/RISCV/zdinx-spill.ll
+++ b/llvm/test/CodeGen/RISCV/zdinx-spill.ll
@@ -9,7 +9,6 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   liveins: $x10, $x11, $x8, $x9, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $x2 = frame-setup ADDI $x2, -64
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION def_cfa_offset 64
   ; CHECK-NEXT:   frame-setup SW killed $x8, $x2, 60 :: (store (s32) into %stack.1)
   ; CHECK-NEXT:   frame-setup SW killed $x9, $x2, 56 :: (store (s32) into %stack.2)
   ; CHECK-NEXT:   frame-setup SW killed $x18, $x2, 52 :: (store (s32) into %stack.3)
@@ -22,18 +21,6 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   frame-setup SW killed $x25, $x2, 24 :: (store (s32) into %stack.10)
   ; CHECK-NEXT:   frame-setup SW killed $x26, $x2, 20 :: (store (s32) into %stack.11)
   ; CHECK-NEXT:   frame-setup SW killed $x27, $x2, 16 :: (store (s32) into %stack.12)
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x8, -4
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x9, -8
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x18, -12
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x19, -16
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x20, -20
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x21, -24
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x22, -28
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x23, -32
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x24, -36
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x25, -40
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x26, -44
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x27, -48
   ; CHECK-NEXT:   renamable $x10_x11 = nofpexcept FADD_D_IN32X killed renamable $x10_x11, renamable $x10_x11, 7, implicit $frm
   ; CHECK-NEXT:   PseudoRV32ZdinxSD killed renamable $x10_x11, $x2, 8 :: (store (s64) into %stack.0, align 4)
   ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 12 /* clobber */, implicit-def dead early-clobber $x6, 12 /* clobber */, implicit-def dead early-clobber $x7, 12 /* clobber */, implicit-def dead early-clobber $x8, 12 /* clobber */, implicit-def dead early-clobber $x9, 12 /* clobber */, implicit-def dead early-clobber $x10, 12 /* clobber */, implicit-def dead early-clobber $x11, 12 /* clobber */, implicit-def dead early-clobber $x12, 12 /* clobber */, implicit-def dead early-clobber $x13, 12 /* clobber */, implicit-def dead early-clobber $x14, 12 /* clobber */, implicit-def dead early-clobber $x15, 12 /* clobber */, implicit-def dead early-clobber $x16, 12 /* clobber */, implicit-def dead early-clobber $x17, 12 /* clobber */, implicit-def dead early-clobber $x18, 12 /* clobber */, implicit-def dead early-clobber $x19, 12 /* clobber */, implicit-def dead early-clobber $x20, 12 /* clobber */, implicit-def dead early-clobber $x21, 12 /* clobber */, implicit-def dead early-clobber $x22, 12 /* clobber */, implicit-def dead early-clobber $x23, 12 /* clobber */, implicit-def dead early-clobber $x24, 12 /* clobber */, implicit-def dead early-clobber $x25, 12 /* clobber */, implicit-def dead early-clobber $x26, 12 /* clobber */, implicit-def dead early-clobber $x27, 12 /* clobber */, implicit-def dead early-clobber $x28, 12 /* clobber */, implicit-def dead early-clobber $x29, 12 /* clobber */, implicit-def dead early-clobber $x31
@@ -50,20 +37,7 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   $x25 = frame-destroy LW $x2, 24 :: (load (s32) from %stack.10)
   ; CHECK-NEXT:   $x26 = frame-destroy LW $x2, 20 :: (load (s32) from %stack.11)
   ; CHECK-NEXT:   $x27 = frame-destroy LW $x2, 16 :: (load (s32) from %stack.12)
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x8
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x9
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x18
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x19
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x20
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x21
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x22
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x23
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x24
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x25
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x26
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x27
   ; CHECK-NEXT:   $x2 = frame-destroy ADDI $x2, 64
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION def_cfa_offset 0
   ; CHECK-NEXT:   PseudoRET implicit $x10, implicit $x11
   %a = fadd double %x, %x
   call void asm sideeffect "", "~{x6},~{x7},~{x8},~{x9},~{x10},~{x11},~{x12},~{x13},~{x14},~{x15},~{x16},~{x17},~{x18},~{x19},~{x20},~{x21},~{x22},~{x23},~{x24},~{x25},~{x26},~{x27},~{x28},~{x29},~{xr0},~{x31}"()
diff --git a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
index 1d073cdc3bdb9..a878dbe94d11d 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; Test handling of llvm.lifetime intrinsics.
 ; RUN: opt < %s -passes=asan -asan-use-after-scope -asan-use-after-return=never -asan-use-stack-safety=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT
 ; RUN: opt < %s -passes=asan -asan-use-after-scope -asan-use-after-return=never -asan-use-stack-safety=0 -asan-instrument-dynamic-allocas=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK-NO-DYNAMIC
@@ -8,109 +9,389 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
 
-; CHECK-LABEL: define void @lifetime_no_size(
 define void @lifetime_no_size(i64 %i) sanitize_address {
+; CHECK-LABEL: define void @lifetime_no_size(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @lifetime_no_size to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i64 -868083117767659023, ptr [[TMP11]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP2]])
+; CHECK-NEXT:    [[AI:%.*]] = getelementptr inbounds [2 x i32], ptr [[TMP2]], i64 0, i64 [[I]]
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[AI]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP13]], 2147450880
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, ptr [[TMP15]], align 1
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i8 [[TMP16]], 0
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[BB18:.*]], label %[[BB23:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i8
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp sge i8 [[TMP20]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB23]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    call void @__asan_report_store1(i64 [[TMP12]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    store volatile i8 0, ptr [[AI]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[TMP2]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-NEXT:    store i64 0, ptr [[TMP25]], align 1
+; CHECK-NEXT:    ret void
+;
 entry:
   %a = alloca [2 x i32], align 4
 
   ; Poison memory in prologue: 0xf3f3f300f1f1f1f1
-  ; CHECK: store i64 -868083117767659023, ptr %[[#]]
 
   call void @llvm.lifetime.start.p0(i64 -1, ptr %a)
   ; Check that lifetime with no size are ignored.
-  ; CHECK-NOT: store
-  ; CHECK: call void @llvm.lifetime.start
 
   %ai = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 %i
   store volatile i8 0, ptr %ai, align 4
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 -1, ptr %a)
   ; Check that lifetime with no size are ignored.
-  ; CHECK-NOT: store
-  ; CHECK: call void @llvm.lifetime.end
 
   ; Unpoison stack frame on exit.
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK: ret void
   ret void
 }
 
 ; Generic case of lifetime analysis.
 define void @lifetime() sanitize_address {
-  ; CHECK-LABEL: define void @lifetime()
+; CHECK-DEFAULT-LABEL: define void @lifetime(
+; CHECK-DEFAULT-SAME: ) #[[ATTR1]] {
+; CHECK-DEFAULT-NEXT:    [[TMP1:%.*]] = alloca i64, align 32
+; CHECK-DEFAULT-NEXT:    store i64 0, ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-DEFAULT-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 32
+; CHECK-DEFAULT-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 1102416563, ptr [[TMP5]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP6:%.*]] = add i64 [[TMP2]], 8
+; CHECK-DEFAULT-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP7]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP8:%.*]] = add i64 [[TMP2]], 16
+; CHECK-DEFAULT-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP9]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP10:%.*]] = lshr i64 [[TMP2]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 0
+; CHECK-DEFAULT-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 -868082052615769615, ptr [[TMP13]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 4, ptr [[TMP15]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 3, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP16]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB27:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB22]]:
+; CHECK-DEFAULT-NEXT:    [[TMP23:%.*]] = and i64 [[TMP16]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP25:%.*]] = icmp sge i8 [[TMP24]], [[TMP20]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP25]], label %[[BB26:.*]], label %[[BB27]]
+; CHECK-DEFAULT:       [[BB26]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP16]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB27]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP4]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP28:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP29]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP30:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP31:%.*]] = inttoptr i64 [[TMP30]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP31]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP32:%.*]] = alloca i8, i64 128, align 32
+; CHECK-DEFAULT-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP32]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP34:%.*]] = add i64 [[TMP33]], 32
+; CHECK-DEFAULT-NEXT:    call void @__asan_alloca_poison(i64 [[TMP34]], i64 40)
+; CHECK-DEFAULT-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP32]] to i64
+; CHECK-DEFAULT-NEXT:    store i64 [[TMP35]], ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP36:%.*]] = inttoptr i64 [[TMP34]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP37:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    call void @__asan_unpoison_stack_memory(i64 [[TMP37]], i64 40)
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[TMP36]])
+; CHECK-DEFAULT-NEXT:    [[TMP38:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP39:%.*]] = lshr i64 [[TMP38]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP40:%.*]] = add i64 [[TMP39]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP41:%.*]] = inttoptr i64 [[TMP40]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP42:%.*]] = load i8, ptr [[TMP41]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP43:%.*]] = icmp ne i8 [[TMP42]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP43]], label %[[BB44:.*]], label %[[BB49:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB44]]:
+; CHECK-DEFAULT-NEXT:    [[TMP45:%.*]] = and i64 [[TMP38]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP45]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP47:%.*]] = icmp sge i8 [[TMP46]], [[TMP42]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP47]], label %[[BB48:.*]], label %[[BB49]]
+; CHECK-DEFAULT:       [[BB48]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP38]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB49]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP36]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP50:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    call void @__asan_poison_stack_memory(i64 [[TMP50]], i64 40)
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[TMP36]])
+; CHECK-DEFAULT-NEXT:    [[TMP51:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP52:%.*]] = inttoptr i64 [[TMP51]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 4, ptr [[TMP52]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP53:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP54:%.*]] = lshr i64 [[TMP53]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP55:%.*]] = add i64 [[TMP54]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP56:%.*]] = inttoptr i64 [[TMP55]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP57:%.*]] = load i8, ptr [[TMP56]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP58:%.*]] = icmp ne i8 [[TMP57]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP58]], label %[[BB59:.*]], label %[[BB64:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB59]]:
+; CHECK-DEFAULT-NEXT:    [[TMP60:%.*]] = and i64 [[TMP53]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP62:%.*]] = icmp sge i8 [[TMP61]], [[TMP57]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP62]], label %[[BB63:.*]], label %[[BB64]]
+; CHECK-DEFAULT:       [[BB63]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP53]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB64]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP4]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP65:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP66:%.*]] = inttoptr i64 [[TMP65]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP66]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP67:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP68:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP68]], i64 [[TMP67]])
+; CHECK-DEFAULT-NEXT:    store i64 1172321806, ptr [[TMP5]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP69:%.*]] = add i64 [[TMP11]], 0
+; CHECK-DEFAULT-NEXT:    [[TMP70:%.*]] = inttoptr i64 [[TMP69]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 0, ptr [[TMP70]], align 1
+; CHECK-DEFAULT-NEXT:    ret void
+;
+; CHECK-NO-DYNAMIC-LABEL: define void @lifetime(
+; CHECK-NO-DYNAMIC-SAME: ) #[[ATTR1]] {
+; CHECK-NO-DYNAMIC-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 32
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 1102416563, ptr [[TMP4]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP6]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP7:%.*]] = add i64 [[TMP1]], 16
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP8]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 -868082052615769615, ptr [[TMP12]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP13:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 4, ptr [[TMP14]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 3, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP16:%.*]] = lshr i64 [[TMP15]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP17:%.*]] = add i64 [[TMP16]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP19]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP20]], label %[[BB21:.*]], label %[[BB26:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB21]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP22:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP22]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP24:%.*]] = icmp sge i8 [[TMP23]], [[TMP19]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP24]], label %[[BB25:.*]], label %[[BB26]]
+; CHECK-NO-DYNAMIC:       [[BB25]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP15]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB26]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[TMP3]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP27:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP28:%.*]] = inttoptr i64 [[TMP27]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP28]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP29:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP30:%.*]] = inttoptr i64 [[TMP29]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP30]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[ARR:%.*]] = alloca [10 x i32], align 16
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[ARR]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP33:%.*]] = add i64 [[TMP32]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP34:%.*]] = inttoptr i64 [[TMP33]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP36:%.*]] = icmp ne i8 [[TMP35]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP36]], label %[[BB37:.*]], label %[[BB42:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB37]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP38:%.*]] = and i64 [[TMP31]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP38]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP40:%.*]] = icmp sge i8 [[TMP39]], [[TMP35]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP40]], label %[[BB41:.*]], label %[[BB42]]
+; CHECK-NO-DYNAMIC:       [[BB41]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP31]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB42]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[ARR]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP43:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP44:%.*]] = inttoptr i64 [[TMP43]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 4, ptr [[TMP44]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP45:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP46:%.*]] = lshr i64 [[TMP45]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP47:%.*]] = add i64 [[TMP46]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP48:%.*]] = inttoptr i64 [[TMP47]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP50:%.*]] = icmp ne i8 [[TMP49]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP50]], label %[[BB51:.*]], label %[[BB56:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB51]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP52:%.*]] = and i64 [[TMP45]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP53:%.*]] = trunc i64 [[TMP52]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP54:%.*]] = icmp sge i8 [[TMP53]], [[TMP49]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP54]], label %[[BB55:.*]], label %[[BB56]]
+; CHECK-NO-DYNAMIC:       [[BB55]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP45]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB56]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[TMP3]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP57:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP58:%.*]] = inttoptr i64 [[TMP57]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP58]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    store i64 1172321806, ptr [[TMP4]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP59:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP60:%.*]] = inttoptr i64 [[TMP59]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 0, ptr [[TMP60]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    ret void
+;
 
   ; Regular variable lifetime intrinsics.
   %i = alloca i32, align 4
 
   ; Poison memory in prologue: F1F1F1F1F8F3F3F3
-  ; CHECK: store i64 -868082052615769615, ptr %{{[0-9]+}}
 
   ; Memory is unpoisoned at llvm.lifetime.start
   call void @llvm.lifetime.start.p0(i64 3, ptr %i)
-  ; CHECK: store i8 4, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 4, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ; Memory is poisoned at every call to llvm.lifetime.end
   call void @llvm.lifetime.end.p0(i64 2, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ; Lifetime intrinsics for array.
   %arr = alloca [10 x i32], align 16
 
   call void @llvm.lifetime.start.p0(i64 40, ptr %arr)
-  ; CHECK-DEFAULT: call void @__asan_unpoison_stack_memory(i64 %{{[^ ]+}}, i64 40)
-  ; CHECK-NO-DYNAMIC-NOT: call void @__asan_unpoison_stack_memory(i64 %{{[^ ]+}}, i64 40)
 
   store volatile i8 0, ptr %arr
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 40, ptr %arr)
-  ; CHECK-DEFAULT: call void @__asan_poison_stack_memory(i64 %{{[^ ]+}}, i64 40)
-  ; CHECK-NO-DYNAMIC-NOT: call void @__asan_poison_stack_memory(i64 %{{[^ ]+}}, i64 40)
 
   ; One more lifetime start/end for the same variable %i.
   call void @llvm.lifetime.start.p0(i64 2, ptr %i)
-  ; CHECK: store i8 4, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 4, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.end
 
   ; Memory is unpoisoned at function exit (only once).
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: ret void
   ret void
 }
 
 ; Check that arguments of lifetime may come from phi nodes.
 define void @phi_args(i1 %x) sanitize_address {
-  ; CHECK-LABEL: define void @phi_args(i1 %x)
+; CHECK-LABEL: define void @phi_args(
+; CHECK-SAME: i1 [[X:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.2 to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @phi_args to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i64 -868082052615769615, ptr [[TMP11]], align 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    store i8 0, ptr [[TMP13]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[TMP15]], 2147450880
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    [[TMP18:%.*]] = load i8, ptr [[TMP17]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne i8 [[TMP18]], 0
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[BB20:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp sge i8 [[TMP22]], [[TMP18]]
+; CHECK-NEXT:    br i1 [[TMP23]], label %[[BB24:.*]], label %[[BB25]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__asan_report_store1(i64 [[TMP14]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    store volatile i8 0, ptr [[TMP2]], align 1
+; CHECK-NEXT:    br i1 [[X]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB0]]:
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[I_PHI:%.*]] = phi ptr [ [[TMP2]], %[[BB25]] ], [ [[TMP2]], %[[BB0]] ]
+; CHECK-NEXT:    [[TMP26:%.*]] = add i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr
+; CHECK-NEXT:    store i8 -8, ptr [[TMP27]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[I_PHI]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-NEXT:    store i64 0, ptr [[TMP29]], align 1
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %i = alloca i64, align 4
 
   ; Poison memory in prologue: F1F1F1F1F8F3F3F3
-  ; CHECK: store i64 -868082052615769615, ptr %{{[0-9]+}}
 
   call void @llvm.lifetime.start.p0(i64 8, ptr %i)
-  ; CHECK: store i8 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   br i1 %x, label %bb0, label %bb1
 
@@ -120,49 +401,101 @@ bb0:
 bb1:
   %i.phi = phi ptr [ %i, %entry ], [ %i, %bb0 ]
   call void @llvm.lifetime.end.p0(i64 8, ptr %i.phi)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.end
 
   ret void
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: ret void
 }
 
 ; Check that arguments of lifetime may come from getelementptr nodes.
 define void @getelementptr_args(i64 %i) sanitize_address{
-  ; CHECK-LABEL: define void @getelementptr_args
+; CHECK-LABEL: define void @getelementptr_args(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 1216, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP0]], 1184
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.3 to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @getelementptr_args to i64), ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 2147450880
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    store i32 -235802127, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP14]], i64 128)
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP11]], 132
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    store i64 -940422246894996750, ptr [[TMP16]], align 1
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP11]], 140
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    store i64 -940422246894996750, ptr [[TMP18]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP11]], 150
+; CHECK-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT:    store i16 -3085, ptr [[TMP20]], align 1
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP21]], i64 128)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1024, ptr [[TMP2]])
+; CHECK-NEXT:    [[AI:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP4]], i64 0, i64 [[I]]
+; CHECK-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[AI]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[TMP23]], 2147450880
+; CHECK-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[TMP25]], align 1
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne i8 [[TMP26]], 0
+; CHECK-NEXT:    br i1 [[TMP27]], label %[[BB28:.*]], label %[[BB29:.*]]
+; CHECK:       [[BB28]]:
+; CHECK-NEXT:    call void @__asan_report_store8(i64 [[TMP22]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB29]]:
+; CHECK-NEXT:    store ptr [[TMP2]], ptr [[AI]], align 8
+; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP30]], i64 128)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1024, ptr [[TMP2]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[TMP11]], 0
+; CHECK-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP31]], i64 148)
+; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[TMP11]], 150
+; CHECK-NEXT:    [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT:    store i16 0, ptr [[TMP33]], align 1
+; CHECK-NEXT:    ret void
+;
 entry:
   %x = alloca [1024 x i8], align 16
   %a = alloca [2 x ptr], align 8
 
   ; F1F1F1F1
-  ; CHECK: store i32 -235802127, ptr %{{[0-9]+}}
-  ; CHECK: call void @__asan_set_shadow_f8(i64 %[[#]], i64 128)
   ; 0xf2f2f2f2f2f2f2f2
-  ; CHECK: store i64 -940422246894996750, ptr %[[#]]
   ; 0xf2f2f2f2f2f2f2f2
-  ; CHECK: store i64 -940422246894996750, ptr %[[#]]
 
   call void @llvm.lifetime.start.p0(i64 1024, ptr %x)
-  ; CHECK: call void @__asan_set_shadow_00(i64 %{{[0-9]+}}, i64 128)
-  ; CHECK-NEXT: call void @llvm.lifetime.start
 
   %ai = getelementptr inbounds [2 x ptr], ptr %a, i64 0, i64 %i
   store ptr %x, ptr %ai, align 8
-  ; CHECK: store ptr
 
   call void @llvm.lifetime.end.p0(i64 1024, ptr %x)
-  ; CHECK: call void @__asan_set_shadow_f8(i64 %{{[0-9]+}}, i64 128)
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ret void
-  ; CHECK: call void @__asan_set_shadow_00(i64 %{{[0-9]+}}, i64 148)
-  ; CHECK: store i16 0, ptr %[[#]], align 1
-  ; CHECK-NEXT: ret void
 }
 
 define void @zero_sized(i64 %a) #0 {
-; CHECK-LABEL: define void @zero_sized(i64 %a)
+; CHECK-LABEL: define void @zero_sized(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    [[B:%.*]] = alloca [0 x i8], align 1
+; CHECK-NEXT:    store i64 [[A]], ptr [[A_ADDR]], align 8
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 0, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 0, ptr [[B]])
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %a.addr = alloca i64, align 8
@@ -170,11 +503,13 @@ entry:
   store i64 %a, ptr %a.addr, align 8
 
   call void @llvm.lifetime.start.p0(i64 0, ptr %b) #2
-  ; CHECK: call void @llvm.lifetime.start
 
   call void @llvm.lifetime.end.p0(i64 0, ptr %b) #2
-  ; CHECK: call void @llvm.lifetime.end
 
   ret void
-  ; CHECK-NEXT: ret void
 }
+;.
+; CHECK-DEFAULT: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
+; CHECK-NO-DYNAMIC: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
index 279bb262bff04..5f310a9954ad0 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
@@ -163,6 +163,51 @@ v_tanh_bf16 v5, src_scc
 v_tanh_bf16 v127, 0x8000
 // GFX1250: v_tanh_bf16_e32 v127, 0x8000            ; encoding: [0xff,0x94,0xfe,0x7e,0x00,0x80,0x00,0x00]
 
+v_prng_b32 v5, v1
+// GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, v255
+// GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, s1
+// GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, s105
+// GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_lo
+// GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_hi
+// GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, ttmp15
+// GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, m0
+// GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_lo
+// GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_hi
+// GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, null
+// GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, -1
+// GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, 0.5
+// GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, src_scc
+// GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+v_prng_b32 v255, 0xaf123456
+// GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
 v_rcp_bf16 v5, v1
 // GFX1250: v_rcp_bf16_e32 v5, v1                   ; encoding: [0x01,0xf3,0x0a,0x7e]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
index 76272d25d92d4..aa2e028f661e1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
@@ -169,6 +169,51 @@ v_tanh_bf16 v127, 0x8000
 v_tanh_bf16 v5.h, v1.h
 // GFX1250: v_tanh_bf16_e32 v5.h, v1.h              ; encoding: [0x81,0x95,0x0a,0x7f]
 
+v_prng_b32 v5, v1
+// GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, v255
+// GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, s1
+// GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, s105
+// GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_lo
+// GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_hi
+// GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, ttmp15
+// GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, m0
+// GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_lo
+// GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_hi
+// GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, null
+// GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, -1
+// GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, 0.5
+// GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, src_scc
+// GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+v_prng_b32 v255, 0xaf123456
+// GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
 v_rcp_bf16 v5, v1
 // GFX1250: v_rcp_bf16_e32 v5, v1                   ; encoding: [0x01,0xf3,0x0a,0x7e]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
index 0a8ee84561d33..e1cd2e3043693 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
@@ -170,6 +170,58 @@ v_tanh_bf16 v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 f
 // GFX1250: v_tanh_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x94,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf2,0x0a,0x7e,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
index d4afb9d9b2d9a..c1d3238b65cbd 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
@@ -178,6 +178,58 @@ v_tanh_bf16 v5.h, v1.h quad_perm:[3,2,1,0]
 // GFX1250: v_tanh_bf16_dpp v5.h, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf2,0x0a,0x7e,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
index a7cb6bf8de69c..100e9f92ff58b 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
@@ -38,6 +38,18 @@ v_tanh_bf16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX1250: v_tanh_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x94,0xfe,0x7e,0x7f,0x00,0x00,0x00]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf2,0x0a,0x7e,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
index 6acab7edc0d49..2ae103545443c 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
@@ -46,6 +46,18 @@ v_tanh_bf16 v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_tanh_bf16_dpp v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x94,0x0a,0x7f,0x81,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf2,0x0a,0x7e,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
index 7486d849253e8..9c6a9127d82e4 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
@@ -127,6 +127,42 @@ v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp
 v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp
 // GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2 clamp ; encoding: [0x01,0x88,0xec,0xd5,0x03,0x01,0x00,0x00]
 
+v_prng_b32_e64 v5, v1
+// GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, v255
+// GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, s1
+// GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, s105
+// GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_lo
+// GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_hi
+// GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, ttmp15
+// GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, m0
+// GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_lo
+// GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_hi
+// GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, null
+// GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, -1
+// GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
 v_tanh_f32_e64 v5, v1
 // GFX1250: v_tanh_f32_e64 v5, v1                   ; encoding: [0x05,0x00,0x9e,0xd5,0x01,0x01,0x00,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
index b59b8b31e2d5f..2f57d1c331c42 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
@@ -223,6 +223,42 @@ v_tanh_f16_e64 v255, -|0x8000| clamp div:2
 v_tanh_f16 v5.l, v128.h
 // GFX1250: v_tanh_f16_e64 v5.l, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0x9f,0xd5,0x80,0x01,0x00,0x00]
 
+v_prng_b32_e64 v5, v1
+// GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, v255
+// GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, s1
+// GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, s105
+// GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_lo
+// GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_hi
+// GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, ttmp15
+// GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, m0
+// GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_lo
+// GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_hi
+// GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, null
+// GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, -1
+// GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
 v_rcp_bf16_e64 v5, v1
 // GFX1250: v_rcp_bf16_e64 v5, v1                   ; encoding: [0x05,0x00,0xf9,0xd5,0x01,0x01,0x00,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
index f7f20f46161ce..29bb842b529b7 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
@@ -170,6 +170,50 @@ v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mas
 // GFX1250: v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xca,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
index e1241b01ccae1..7df92751c38d1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
@@ -178,6 +178,50 @@ v_tanh_bf16_e64_dpp v5.h, v128.h quad_perm:[3,2,1,0]
 // GFX1250: v_tanh_bf16_e64_dpp v5.h, v128.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xca,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
index 0106175301d20..d235aeb9f3e62 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
@@ -50,6 +50,10 @@ v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX1250: v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xca,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
index 93b86f3ffb841..f25e2a5882436 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
@@ -58,6 +58,10 @@ v_tanh_bf16_e64_dpp v5.h, v128.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_tanh_bf16_e64_dpp v5.h, v128.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xca,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
index 5f37ba91e071b..aa968b2bb2bee 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
@@ -200,6 +200,51 @@
 0x81,0x95,0x0a,0x7f
 # GFX1250-REAL16: v_tanh_bf16_e32 v5.h, v1.h              ; encoding: [0x81,0x95,0x0a,0x7f]
 
+0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf
+# GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
+0xc1,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+0xf0,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+0x7f,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+0x7e,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+0x7d,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+0x7c,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+0x01,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+0x69,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+0xfd,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+0x7b,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+0x01,0x97,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+0xff,0x97,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+0x6b,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+0x6a,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
 0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_e32 v127.l, 0x8000           ; encoding: [0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_e32 v127, 0x8000             ; encoding: [0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
index 57bee2766ce44..913a2a916ff62 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
@@ -162,6 +162,45 @@
 0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff
 # GFX1250-REAL16: v_tanh_bf16_dpp v5.h, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff]
 
+0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01
+# GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13
+# GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+
 0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30
 # GFX1250-REAL16: v_rcp_bf16_dpp v127.l, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
 # GFX1250-FAKE16: v_rcp_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
index 28ec6b11b4de3..4afe44e241bf3 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
@@ -43,6 +43,15 @@
 # GFX1250-REAL16: v_tanh_bf16_dpp v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x94,0x0a,0x7f,0x81,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_add_f64_e32 v[156:157], v[129:130], v[187:188] ; encoding: [0x81,0x77,0x39,0x05]
 
+0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00
+# GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+
+0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
 0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_dpp v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
index 5004762729701..1cf3b8807d044 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
@@ -175,6 +175,42 @@
 # GFX1250-REAL16: v_tanh_bf16_e64 v5.l, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0xca,0xd5,0x80,0x01,0x00,0x00]
 # GFX1250-FAKE16: v_tanh_bf16_e64 v5, v128                ; encoding: [0x05,0x00,0xca,0xd5,0x80,0x01,0x00,0x00]
 
+0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
 0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_e64 v255.l, -|0x8000| clamp div:2 ; encoding: [0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_e64 v255, -|0x8000| clamp div:2 ; encoding: [0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
index de908b95d94f9..83a647ad7c658 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
@@ -104,6 +104,39 @@
 # GFX1250-REAL16: v_tanh_f16_e64_dpp v5.h, v128.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x9f,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 # GFX1250-FAKE16: v_tanh_f16_e64_dpp v5, v128 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x9f,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+
 0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30
 # GFX1250-REAL16: v_rsq_bf16_e64_dpp v255.l, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
 # GFX1250-FAKE16: v_rsq_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
index cfe7173c383b3..ef5ede4d1d453 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
 
 0xff,0x81,0x9e,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00
 # GFX1250: v_tanh_f32_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0x9e,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
@@ -34,6 +34,9 @@
 # GFX1250-REAL16: v_tanh_f16_e64_dpp v5.h, v128.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0x9f,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_tanh_f16_e64_dpp v5, v128 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x9f,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 
+0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
 0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00
 # GFX1250-REAL16: v_rsq_bf16_e64_dpp v255.l, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
 # GFX1250-FAKE16: v_rsq_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll b/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
index e390d4bdca632..303afc207c023 100644
--- a/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
@@ -12,6 +12,6 @@ define ptr @undeclared_customalloc(i64 %size, i64 %align) {
   ret ptr %call
 }
 
-declare ptr @customalloc2(i64, i64) allockind("alloc") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
+declare ptr @customalloc2(i64, i64) allockind("alloc,uninitialized") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
 ; CHECK-DAG: declare ptr @customalloc2_zeroed(i64, i64) #[[CA2ATTR:[0-9]+]]
 ; CHECK-DAG: attributes #[[CA2ATTR]] = { allockind("alloc,zeroed") "alloc-family"="customalloc2" }
diff --git a/llvm/test/Transforms/LoopInterchange/force-interchange.ll b/llvm/test/Transforms/LoopInterchange/force-interchange.ll
new file mode 100644
index 0000000000000..c33ecdf7d9905
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/force-interchange.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -passes=loop-interchange -pass-remarks-output=%t -disable-output -loop-interchange-profitabilities=ignore -S
+; RUN: FileCheck --input-file=%t %s
+
+; There should be no reason to interchange this, unless it is forced.
+;
+;     for (int i = 0; i<1024; i++)
+;       for (int j = 0; j<1024; j++)
+;         A[i][j] = 42;
+;
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Interchanged
+; CHECK-NEXT: Function:        f
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Loop interchanged with enclosing loop.
+; CHECK-NEXT: ...
+
+ at A = dso_local local_unnamed_addr global [1024 x [1024 x i32]] zeroinitializer, align 4
+
+define dso_local void @f() local_unnamed_addr #0 {
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %inner.header ]
+  br label %inner.body
+
+inner.header:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond20.not = icmp eq i64 %i.next, 1024
+  br i1 %exitcond20.not, label %exit, label %outer.header
+
+inner.body:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
+  %arrayidx6 = getelementptr inbounds nuw [1024 x [1024 x i32]], ptr @A, i64 0, i64 %i, i64 %j
+  store i32 42, ptr %arrayidx6, align 4
+  %j.next = add nuw nsw i64 %j, 1
+  %exitcond.not = icmp eq i64 %j.next, 1024
+  br i1 %exitcond.not, label %inner.header, label %inner.body
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
new file mode 100644
index 0000000000000..c7a0de22b200b
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=loop-reduce -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(ptr %p, i64 %idx) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4 x [4 x i32]], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[IDX]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 48
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[ALLOCA]], i64 48
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], %[[LOOP]] ], [ -8, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[LSR_IV]], 2
+; CHECK-NEXT:    [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[SCEVGEP8]], i64 32
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[SCEVGEP9]], align 4
+; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[SCEVGEP6]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[SCEVGEP7]], align 4
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[SCEVGEP5]], align 4
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[SCEVGEP2]], align 4
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], 4
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %alloca = alloca [4 x [4 x i32]], align 16
+  call void @llvm.lifetime.start.p0(i64 64, ptr %alloca)
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %gep1 = getelementptr [4 x [12 x [4 x [4 x i32]]]], ptr %p, i64 0, i64 0, i64 0, i64 %indvars.iv, i64 0
+  %0 = load i32, ptr %gep1, align 4
+  %gep2 = getelementptr [6 x [4 x [4 x i32]]], ptr %p, i64 0, i64 0, i64 0, i64 %indvars.iv
+  %1 = load i32, ptr %gep2, align 4
+  %gep3 = getelementptr [4 x [4 x i32]], ptr %alloca, i64 0, i64 3, i64 %indvars.iv
+  %2 = load i32, ptr %gep3, align 4
+  %gep4 = getelementptr [6 x [4 x [4 x i32]]], ptr %p, i64 0, i64 %idx, i64 3, i64 %indvars.iv
+  %3 = load i32, ptr %gep4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv, 1
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  call void @llvm.lifetime.end.p0(i64 64, ptr %alloca)
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll b/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
new file mode 100644
index 0000000000000..2d1543185098f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
@@ -0,0 +1,38 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-REGS-VP
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -force-target-num-vector-regs=1 -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NOREGS-VP
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @dotp(ptr %a, ptr %b) #0 {
+; CHECK-REGS-VP-NOT: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; CHECK-REGS-VP: LV: Selecting VF: vscale x 8.
+;
+; CHECK-NOREGS-VP: LV(REG): Not considering vector loop of width vscale x 8 because it uses too many registers
+; CHECK-NOREGS-VP: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; CHECK-NOREGS-VP: LV: Selecting VF: vscale x 4.
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %gep.b = getelementptr i8, ptr %b, i64 %iv
+  %load.b = load i8, ptr %gep.b, align 1
+  %ext.b = zext i8 %load.b to i32
+  %mul = mul i32 %ext.b, %ext.a
+  %sub = sub i32 0, %mul
+  %add = add i32 %accum, %sub
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
index 8e09847e9264e..cfff11758a37a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
@@ -58,7 +58,6 @@ define void @test(i32 %0, i8 %1, i64 %2, float %3) {
 ; CHECK-NEXT:    br label %[[BB54:.*]]
 ; CHECK:       [[BB54]]:
 ; CHECK-NEXT:    [[TMP54:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP17]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 0, ptr null)
 ; CHECK-NEXT:    [[TMP55:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP21]])
 ; CHECK-NEXT:    [[TMP56:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <8 x float> [[TMP56]], <8 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison>
@@ -198,7 +197,6 @@ define void @test(i32 %0, i8 %1, i64 %2, float %3) {
   %95 = or i64 %94, %91
   %96 = or i64 %95, %37
   store i64 %96, ptr null, align 1
-  call void @llvm.lifetime.start.p0(i64 0, ptr null)
   store i64 %42, ptr null, align 1
   %97 = bitcast float %3 to i32
   %98 = icmp ult i32 %97, 1325400064
diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp
index 422eb855ba2cf..2b43d27f292a0 100644
--- a/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -131,20 +131,6 @@ class CommentWriter : public AssemblyAnnotationWriter {
         printDebugLoc(DL,OS);
         OS << "]";
       }
-      if (const DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(I)) {
-        if (!Padded) {
-          OS.PadToColumn(50);
-          OS << ";";
-        }
-        OS << " [debug variable = " << DDI->getVariable()->getName() << "]";
-      }
-      else if (const DbgValueInst *DVI = dyn_cast<DbgValueInst>(I)) {
-        if (!Padded) {
-          OS.PadToColumn(50);
-          OS << ";";
-        }
-        OS << " [debug variable = " << DVI->getVariable()->getName() << "]";
-      }
     }
   }
 };
diff --git a/llvm/unittests/CodeGen/LexicalScopesTest.cpp b/llvm/unittests/CodeGen/LexicalScopesTest.cpp
index 3d707462fa615..563d496d1e600 100644
--- a/llvm/unittests/CodeGen/LexicalScopesTest.cpp
+++ b/llvm/unittests/CodeGen/LexicalScopesTest.cpp
@@ -67,7 +67,7 @@ class LexicalScopesTest : public testing::Test {
     BeanInst.Opcode = 1;
     BeanInst.Size = 1;
 
-    memset(&DbgValueInst, 0, sizeof(DbgValueInst));
+    memset(&DbgValueInst, 0, sizeof(MCInstrDesc));
     DbgValueInst.Opcode = TargetOpcode::DBG_VALUE;
     DbgValueInst.Size = 1;
     DbgValueInst.Flags = 1U << MCID::Meta;
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index 41bf863420304..baa13e1199eea 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -188,6 +188,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgRecord) {
   SmallVector<DbgValueInst *, 1> DVIs;
   SmallVector<DbgVariableRecord *, 1> DVRs;
   findDbgValues(DVIs, &I, &DVRs);
+  assert(DVIs.empty());
 
   // Delete %b. The dbg.value should now point to undef.
   I.eraseFromParent();
@@ -314,6 +315,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
   SmallVector<DbgValueInst *, 2> DVIs;
   SmallVector<DbgVariableRecord *, 2> DVRs;
   findDbgValues(DVIs, &I, &DVRs);
+  assert(DVIs.empty());
   ASSERT_EQ(DVRs.size(), 2u);
 
   // Delete %b. The DbgVariableRecord should now point to undef.
diff --git a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
index 0b00734fc4d75..1daf381ee2862 100644
--- a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
+++ b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
@@ -54,20 +54,13 @@ struct DebugInfoDrop : public FunctionPass {
 struct DebugValueDrop : public FunctionPass {
   static char ID;
   bool runOnFunction(Function &F) override {
-    SmallVector<DbgVariableIntrinsic *, 4> Dbgs;
     for (BasicBlock &BB : F) {
-      // Remove dbg var intrinsics.
       for (Instruction &I : BB) {
-        if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
-          Dbgs.push_back(DVI);
-        // If there are any non-intrinsic records (DbgRecords), drop those too.
+        // If there are any debug records, drop them.
         I.dropDbgRecords();
       }
     }
 
-    for (auto &I : Dbgs)
-      I->eraseFromParent();
-
     return true;
   }
 
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index b922216ef8893..dd2a6249c7cf9 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -633,62 +633,6 @@ TEST(Local, ChangeToUnreachable) {
   EXPECT_EQ(DLA, DLB);
 }
 
-TEST(Local, FindDbgUsers) {
-  LLVMContext Ctx;
-  std::unique_ptr<Module> M = parseIR(Ctx,
-                                      R"(
-  define dso_local void @fun(ptr %a) #0 !dbg !11 {
-  entry:
-      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
-    ret void
-  }
-
-  !llvm.dbg.cu = !{!0}
-  !llvm.module.flags = !{!2, !3, !9}
-  !llvm.ident = !{!10}
-
-  !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-  !1 = !DIFile(filename: "test.cpp", directory: "/")
-  !2 = !{i32 7, !"Dwarf Version", i32 5}
-  !3 = !{i32 2, !"Debug Info Version", i32 3}
-  !4 = !{i32 1, !"wchar_size", i32 4}
-  !9 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
-  !10 = !{!"clang version 17.0.0"}
-  !11 = distinct !DISubprogram(name: "fun", linkageName: "fun", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !14)
-  !12 = !DISubroutineType(types: !13)
-  !13 = !{null}
-  !14 = !{}
-  !15 = distinct !DIAssignID()
-  !16 = !DILocalVariable(name: "x", scope: !11, file: !1, line: 2, type: !17)
-  !17 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 64)
-  !18 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-  !19 = !DILocation(line: 0, scope: !11)
-  )");
-
-  bool BrokenDebugInfo = true;
-  verifyModule(*M, &errs(), &BrokenDebugInfo);
-  ASSERT_FALSE(BrokenDebugInfo);
-
-  // Convert to debug intrinsics as we want to test findDbgUsers and
-  // findDbgValue's debug-intrinsic-finding code here.
-  // TODO: Remove this test when debug intrinsics are removed.
-  M->convertFromNewDbgValues();
-
-  Function &Fun = *cast<Function>(M->getNamedValue("fun"));
-  Value *Arg = Fun.getArg(0);
-  SmallVector<DbgVariableIntrinsic *> Users;
-  // Arg (%a) is used twice by a single dbg.assign. Check findDbgUsers returns
-  // only 1 pointer to it rather than 2.
-  findDbgUsers(Users, Arg);
-  EXPECT_EQ(Users.size(), 1u);
-
-  SmallVector<DbgValueInst *> Vals;
-  // Arg (%a) is used twice by a single dbg.assign. Check findDbgValues returns
-  // only 1 pointer to it rather than 2.
-  findDbgValues(Vals, Arg);
-  EXPECT_EQ(Vals.size(), 1u);
-}
-
 TEST(Local, FindDbgRecords) {
   // DbgRecord copy of the FindDbgUsers test above.
   LLVMContext Ctx;
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 82ec8121548c9..5309b5d095134 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -35,7 +35,6 @@ if (current_toolchain == default_toolchain) {
       "_LIBCPP_HAS_LOCALIZATION=1",
       "_LIBCPP_HAS_UNICODE=1",
       "_LIBCPP_HAS_WIDE_CHARACTERS=1",
-      "_LIBCPP_HAS_NO_STD_MODULES=",
       "_LIBCPP_HAS_TERMINAL=1",
       "_LIBCPP_INSTRUMENTED_WITH_ASAN=",
       "_LIBCPP_ABI_DEFINES=",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index d4adeddd9b4e4..3d11ce566207a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -176,6 +176,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUPreLegalizerCombiner.cpp",
     "AMDGPUPreloadKernArgProlog.cpp",
     "AMDGPUPreloadKernelArguments.cpp",
+    "AMDGPUPrepareAGPRAlloc.cpp",
     "AMDGPUPrintfRuntimeBinding.cpp",
     "AMDGPUPromoteAlloca.cpp",
     "AMDGPUPromoteKernelArguments.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 946b63f8a54fb..7ed0d3c6824e2 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -286,6 +286,7 @@ group("test") {
     "//llvm/tools/llvm-extract",
     "//llvm/tools/llvm-gsymutil:llvm-gsymutil",
     "//llvm/tools/llvm-ifs",
+    "//llvm/tools/llvm-ir2vec",
     "//llvm/tools/llvm-isel-fuzzer",
     "//llvm/tools/llvm-jitlink",
     "//llvm/tools/llvm-jitlink/llvm-jitlink-executor",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
new file mode 100644
index 0000000000000..07a795122c76c
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
@@ -0,0 +1,9 @@
+executable("llvm-ir2vec") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+  ]
+  sources = [ "llvm-ir2vec.cpp" ]
+}
diff --git a/mlir/docs/Tutorials/transform/Ch2.md b/mlir/docs/Tutorials/transform/Ch2.md
index 0f45f5607bab9..08c4e0f9d859c 100644
--- a/mlir/docs/Tutorials/transform/Ch2.md
+++ b/mlir/docs/Tutorials/transform/Ch2.md
@@ -133,6 +133,8 @@ This will generate two files, `MyExtension.h.inc` and `MyExtension.cpp.inc`, tha
 ```c++
 // In MyExtension.cpp.
 
+#include "MyExtension.h"
+
 #define GET_OP_CLASSES
 #include "MyExtension.cpp.inc"
 
@@ -283,7 +285,7 @@ void registerMyExtension(::mlir::DialectRegistry &registry) {
 }
 ```
 
-After registering the extension, it becomes possible to use our new operation in the Transform dialect interpreter. The upstream testing pass can be used as is.
+After registering the extension, it becomes possible to use our new operation in the Transform dialect interpreter. The upstream testing pass can be used as is. It actually exists in `mlir/test/Examples/transform/Ch2/sequence.mlir`, which contains the `microkernel` implementation. 
 
 ```mlir
 module attributes {transform.with_named_sequence} {
@@ -300,7 +302,7 @@ module attributes {transform.with_named_sequence} {
 
     // The actual tiling transformation takes tile sizes as attributes. It
     // produces a handle to the loop generated during tiling.
-    %loop, %tiled = transform.structured.tile_using_forall %max
+    %tiled, %loop = transform.structured.tile_using_forall %max
                     tile_sizes [8, 32]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
@@ -311,32 +313,32 @@ module attributes {transform.with_named_sequence} {
     // a single handle to all operations and give it to
     // `fuse_into_containing_op` that would take care of the ordering in this
     // case.
-    %add_fused = transform.structured.fuse_into_containing_op %add into %loop
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
-    %matmul_fused = transform.structured.fuse_into_containing_op %arg1
-                    into %loop
+    %add_fused, %loop2 = transform.structured.fuse_into_containing_op %add into %loop
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_fused, %loop3 = transform.structured.fuse_into_containing_op %arg1
+                    into %loop2
         : (!transform.op<"linalg.matmul">, !transform.any_op)
-       -> !transform.any_op
+       -> (!transform.any_op, !transform.any_op)
 
     // Tile again to get the desired size. Note that this time this tiles the
     // "add" operation and fuses matmul into the loop, but doesn't affect the
     // "max" operation. This illustrates the precise targeting with the
     // transform dialect. Otherwise, it is difficult to differentiate "add" and
     // "max", both of which having the same kind.
-    %loop_2, %tiled_2 = transform.structured.tile_using_forall %add_fused
+    %tiled_second, %loop_second = transform.structured.tile_using_forall %add_fused
                         tile_sizes [4, 4]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
-    %matmul_fused_2 = transform.structured.fuse_into_containing_op %matmul_fused
-                      into %loop_2
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
+    %matmul_fused_2, %loop_second_2 = transform.structured.fuse_into_containing_op %matmul_fused
+                      into %loop_second
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     // Since outlining is currently only implemented for region-holding
     // operations such as loops, use tiling to size 1 to materialize the outer
     // loop that is going to be outlined.
-    %outline_target, %_ = transform.structured.tile_using_forall %tiled_2 tile_sizes [1]
+    %_0, %loop_third = transform.structured.tile_using_forall %tiled_second tile_sizes [1]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
-    transform.structured.fuse_into_containing_op %matmul_fused_2 into %outline_target
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
+    %_1, %outline_target = transform.structured.fuse_into_containing_op %matmul_fused_2 into %loop_third
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
     %func, %call = transform.loop.outline %outline_target
                    {func_name = "outlined"}
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
diff --git a/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h b/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
index ac4915901fdec..ff99d7ce96daf 100644
--- a/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
+++ b/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
@@ -21,9 +21,6 @@ class RewritePatternSet;
 /// Collect a set of patterns to convert from the vector to XeGPU ops.
 void populateVectorToXeGPUConversionPatterns(RewritePatternSet &patterns);
 
-/// Create a pass to convert ops from vector to XeGPU.
-std::unique_ptr<Pass> createConvertVectorToXeGPUPass();
-
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_VECTORTOXEGPU_VECTORTOXEGPU_H
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 12c7cc62905c9..d4400547f9568 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -1115,6 +1115,18 @@ struct AMDGPUStreamTy {
     return Plugin::success();
   }
 
+  /// Complete pending post actions until and including the event in target
+  /// slot.
+  Error completeUntil(uint32_t TargetSlot) {
+    for (uint32_t Slot = 0; Slot <= TargetSlot; ++Slot) {
+      // Take the post action of the operation if any.
+      if (auto Err = Slots[Slot].performAction())
+        return Err;
+    }
+
+    return Plugin::success();
+  }
+
   /// Make the current stream wait on a specific operation of another stream.
   /// The idea is to make the current stream waiting on two signals: 1) the last
   /// signal of the current stream, and 2) the last signal of the other stream.
@@ -1502,6 +1514,11 @@ struct AMDGPUStreamTy {
     return complete();
   }
 
+  /// Synchronize the stream until the given event. The current thread waits
+  /// until the provided event is finalized, and it performs the pending post
+  /// actions for that and prior events.
+  Error synchronizeOn(AMDGPUEventTy &Event);
+
   /// Query the stream and complete pending post actions if operations finished.
   /// Return whether all the operations completed. This operation does not block
   /// the calling thread.
@@ -1575,6 +1592,21 @@ struct AMDGPUEventTy {
     return Stream.waitEvent(*this);
   }
 
+  Error sync() {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    if (!RecordedStream)
+      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
+                           "event does not have any recorded stream");
+
+    // No need to wait on anything, the recorded stream already finished the
+    // corresponding operation.
+    if (RecordedSlot < 0)
+      return Plugin::success();
+
+    return RecordedStream->synchronizeOn(*this);
+  }
+
 protected:
   /// The stream registered in this event.
   AMDGPUStreamTy *RecordedStream;
@@ -1630,6 +1662,22 @@ Error AMDGPUStreamTy::waitEvent(const AMDGPUEventTy &Event) {
   return waitOnStreamOperation(RecordedStream, Event.RecordedSlot);
 }
 
+Error AMDGPUStreamTy::synchronizeOn(AMDGPUEventTy &Event) {
+  std::lock_guard<std::mutex> Lock(Mutex);
+
+  // Wait until the requested slot has completed
+  if (auto Err = Slots[Event.RecordedSlot].Signal->wait(
+          StreamBusyWaitMicroseconds, &Device))
+    return Err;
+
+  // If the event is the last one in the stream, just do a full finalize
+  if (Event.RecordedSlot == last())
+    return complete();
+
+  // Otherwise, only finalize until the appropriate event
+  return completeUntil(Event.RecordedSlot);
+}
+
 struct AMDGPUStreamManagerTy final
     : GenericDeviceResourceManagerTy<AMDGPUResourceRef<AMDGPUStreamTy>> {
   using ResourceRef = AMDGPUResourceRef<AMDGPUStreamTy>;
@@ -2540,8 +2588,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
   /// Synchronize the current thread with the event.
   Error syncEventImpl(void *EventPtr) override {
-    return Plugin::error(ErrorCode::UNIMPLEMENTED,
-                         "synchronize event not implemented");
+    AMDGPUEventTy *Event = reinterpret_cast<AMDGPUEventTy *>(EventPtr);
+    return Event->sync();
   }
 
   /// Print information about the device.
@@ -2562,7 +2610,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     Status2 = hsa_system_get_info(HSA_SYSTEM_INFO_VERSION_MINOR, &Minor);
     if (Status == HSA_STATUS_SUCCESS && Status2 == HSA_STATUS_SUCCESS)
       Info.add("HSA Runtime Version",
-               std::to_string(Major) + "." + std::to_string(Minor));
+               std::to_string(Major) + "." + std::to_string(Minor), "",
+               DeviceInfo::DRIVER_VERSION);
 
     Info.add("HSA OpenMP Device Number", DeviceId);
 
@@ -2572,11 +2621,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_NAME, TmpChar);
     if (Status == HSA_STATUS_SUCCESS)
-      Info.add("Device Name", TmpChar);
+      Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_VENDOR_NAME, TmpChar);
     if (Status == HSA_STATUS_SUCCESS)
-      Info.add("Vendor Name", TmpChar);
+      Info.add("Vendor Name", TmpChar, "", DeviceInfo::VENDOR);
 
     hsa_device_type_t DevType;
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_DEVICE, DevType);
@@ -2652,7 +2701,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_WORKGROUP_MAX_DIM, WorkgrpMaxDim);
     if (Status == HSA_STATUS_SUCCESS) {
-      auto &MaxSize = *Info.add("Workgroup Max Size per Dimension");
+      auto &MaxSize =
+          *Info.add("Workgroup Max Size per Dimension", std::monostate{}, "",
+                    DeviceInfo::MAX_WORK_GROUP_SIZE);
       MaxSize.add("x", WorkgrpMaxDim[0]);
       MaxSize.add("y", WorkgrpMaxDim[1]);
       MaxSize.add("z", WorkgrpMaxDim[2]);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 162b149ab483e..8c17a2ee07047 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -113,6 +113,12 @@ struct AsyncInfoWrapperTy {
   __tgt_async_info *AsyncInfoPtr;
 };
 
+enum class DeviceInfo {
+#define OFFLOAD_DEVINFO(Name, _, Value) Name = Value,
+#include "OffloadInfo.inc"
+#undef OFFLOAD_DEVINFO
+};
+
 /// Tree node for device information
 ///
 /// This information is either printed or used by liboffload to extract certain
@@ -133,6 +139,8 @@ struct InfoTreeNode {
   // * The same key can appear multiple times
   std::unique_ptr<llvm::SmallVector<InfoTreeNode, 8>> Children;
 
+  llvm::DenseMap<DeviceInfo, size_t> DeviceInfoMap;
+
   InfoTreeNode() : InfoTreeNode("", std::monostate{}, "") {}
   InfoTreeNode(std::string Key, VariantType Value, std::string Units)
       : Key(Key), Value(Value), Units(Units) {}
@@ -140,10 +148,12 @@ struct InfoTreeNode {
   /// Add a new info entry as a child of this node. The entry requires at least
   /// a key string in \p Key. The value in \p Value is optional and can be any
   /// type that is representable as a string. The units in \p Units is optional
-  /// and must be a string.
+  /// and must be a string. Providing a device info key allows liboffload to
+  /// use that value for an appropriate olGetDeviceInfo query
   template <typename T = std::monostate>
   InfoTreeNode *add(std::string Key, T Value = T(),
-                    const std::string &Units = std::string()) {
+                    const std::string &Units = std::string(),
+                    std::optional<DeviceInfo> DeviceInfoKey = std::nullopt) {
     assert(!Key.empty() && "Invalid info key");
 
     if (!Children)
@@ -157,7 +167,12 @@ struct InfoTreeNode {
     else
       ValueVariant = std::string{Value};
 
-    return &Children->emplace_back(Key, ValueVariant, Units);
+    auto Ptr = &Children->emplace_back(Key, ValueVariant, Units);
+
+    if (DeviceInfoKey)
+      DeviceInfoMap[*DeviceInfoKey] = Children->size() - 1;
+
+    return Ptr;
   }
 
   std::optional<InfoTreeNode *> get(StringRef Key) {
@@ -171,6 +186,13 @@ struct InfoTreeNode {
     return It;
   }
 
+  std::optional<InfoTreeNode *> get(DeviceInfo Info) {
+    auto Result = DeviceInfoMap.find(Info);
+    if (Result != DeviceInfoMap.end())
+      return &(*Children)[Result->second];
+    return std::nullopt;
+  }
+
   /// Print all info entries in the tree
   void print() const {
     // Fake an additional indent so that values are offset from the keys
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index b787376eb1770..728bf07c572bb 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -935,15 +935,16 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Res == CUDA_SUCCESS)
       // For consistency with other drivers, store the version as a string
       // rather than an integer
-      Info.add("CUDA Driver Version", std::to_string(TmpInt));
+      Info.add("CUDA Driver Version", std::to_string(TmpInt), "",
+               DeviceInfo::DRIVER_VERSION);
 
     Info.add("CUDA OpenMP Device Number", DeviceId);
 
     Res = cuDeviceGetName(TmpChar, 1000, Device);
     if (Res == CUDA_SUCCESS)
-      Info.add("Device Name", TmpChar);
+      Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
 
-    Info.add("Vendor Name", "NVIDIA");
+    Info.add("Vendor Name", "NVIDIA", "", DeviceInfo::VENDOR);
 
     Res = cuDeviceTotalMem(&TmpSt, Device);
     if (Res == CUDA_SUCCESS)
@@ -978,7 +979,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Res == CUDA_SUCCESS)
       Info.add("Maximum Threads per Block", TmpInt);
 
-    auto &MaxBlock = *Info.add("Maximum Block Dimensions", "");
+    auto &MaxBlock = *Info.add("Maximum Block Dimensions", std::monostate{}, "",
+                               DeviceInfo::MAX_WORK_GROUP_SIZE);
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, TmpInt);
     if (Res == CUDA_SUCCESS)
       MaxBlock.add("x", TmpInt);
diff --git a/offload/unittests/OffloadAPI/common/Fixtures.hpp b/offload/unittests/OffloadAPI/common/Fixtures.hpp
index e5d815ecda965..546921164f691 100644
--- a/offload/unittests/OffloadAPI/common/Fixtures.hpp
+++ b/offload/unittests/OffloadAPI/common/Fixtures.hpp
@@ -171,9 +171,6 @@ struct OffloadQueueTest : OffloadDeviceTest {
 struct OffloadEventTest : OffloadQueueTest {
   void SetUp() override {
     RETURN_ON_FATAL_FAILURE(OffloadQueueTest::SetUp());
-    if (getPlatformBackend() == OL_PLATFORM_BACKEND_AMDGPU)
-      GTEST_SKIP() << "AMDGPU synchronize event not implemented";
-
     // Get an event from a memcpy. We can still use it in olGetEventInfo etc
     // after it has been waited on.
     void *Alloc;
diff --git a/offload/unittests/OffloadAPI/event/olWaitEvent.cpp b/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
index 05356d4ef8d75..f80dabb4fc93f 100644
--- a/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
+++ b/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
@@ -14,9 +14,6 @@ using olWaitEventTest = OffloadQueueTest;
 OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olWaitEventTest);
 
 TEST_P(olWaitEventTest, Success) {
-  if (getPlatformBackend() == OL_PLATFORM_BACKEND_AMDGPU)
-    GTEST_SKIP() << "AMDGPU synchronize event not implemented";
-
   uint32_t Src = 42;
   void *DstPtr;
 

>From bfe68539b78d8610bd0cd315c9a3e8fe9b456a3f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 16:45:02 +0100
Subject: [PATCH 6/6] !fixup address latest comments, thanks

---
 llvm/lib/Analysis/IVDescriptors.cpp           |   6 +-
 .../Vectorize/VPlanConstruction.cpp           |  49 ++++----
 .../Transforms/Vectorize/VPlanTransforms.h    |   8 +-
 .../fmax-without-fast-math-flags.ll           | 116 ++++++++++++++++++
 4 files changed, 150 insertions(+), 29 deletions(-)

diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index a88bbd7b65d53..8be5de3bf356f 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -947,10 +947,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
       InstDesc Res = isMinMaxPattern(I, Kind, Prev);
       if (!Res.isRecurrence())
         return InstDesc(false, I);
-
       if (HasRequiredFMF())
         return Res;
-
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
       // intrinsics with extra checks ensuring the vector loop handles only
       // non-NaN inputs.
@@ -964,9 +962,9 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                "unexpected recurrence kind for minnum");
         return InstDesc(I, RecurKind::FMinNum);
       }
-
       return InstDesc(false, I);
-    } else if (isFMulAddIntrinsic(I))
+    }
+    if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
     return InstDesc(false, I);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a426d06b4befe..98c2fe2d6129d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -654,10 +654,6 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
 }
 
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
-  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-  VPReductionPHIRecipe *RedPhiR = nullptr;
-  bool HasUnsupportedPhi = false;
-
   auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
     auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
         RedPhiR->getBackedgeValue()->getDefiningRecipe());
@@ -666,7 +662,7 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
 
     auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
     if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
-        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+        !(RepR && isa<IntrinsicInst>(RepR->getUnderlyingInstr())))
       return nullptr;
 
 #ifndef NDEBUG
@@ -690,41 +686,47 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     return MinMaxR->getOperand(0);
   };
 
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
-    // TODO: Also support fixed-order recurrence phis.
-    HasUnsupportedPhi |=
-        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
-             VPReductionPHIRecipe>(&R);
+    if (isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe>(&R))
+      continue;
     auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
-    if (!Cur)
+    if (!Cur) {
+      // TODO: Also support fixed-order recurrence phis.
+      HasUnsupportedPhi = true;
       continue;
+    }
     // For now, only a single reduction is supported.
     // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
     if (RedPhiR)
       return false;
     if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
-        Cur->getRecurrenceKind() != RecurKind::FMinNum)
+        Cur->getRecurrenceKind() != RecurKind::FMinNum) {
+      HasUnsupportedPhi = true;
       continue;
+    }
     RedPhiR = Cur;
   }
 
   if (!RedPhiR)
     return true;
 
-  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
-  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
-         "unsupported reduction");
-
-  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
-  if (!MinMaxOp)
-    return false;
-
   // We won't be able to resume execution in the scalar tail, if there are
   // unsupported header phis or there is no scalar tail at all, due to
   // tail-folding.
   if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
+  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+  if (!MinMaxOp)
+    return false;
+
+  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+         "unsupported reduction");
+
   /// Check if the vector loop of \p Plan can early exit and restart
   /// execution of last vector iteration in the scalar loop. This requires all
   /// recipes up to early exit point be side-effect free as they are
@@ -774,7 +776,7 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   RdxResult->setOperand(1, NewSel);
 
   auto *ScalarPH = Plan.getScalarPreheader();
-  // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+  // Update resume phis for inductions in the scalar preheader. If AnyNaN is
   // true, the resume from the start of the last vector iteration via the
   // canonical IV, otherwise from the original value.
   for (auto &R : ScalarPH->phis()) {
@@ -782,8 +784,13 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     VPValue *VecV = ResumeR->getOperand(0);
     if (VecV == RdxResult)
       continue;
-    if (VecV != &Plan.getVectorTripCount())
+    // Bail out and abandon the current, partially modified, VPlan if we
+    // encounter resume phi that cannot be updated yet.
+    if (VecV != &Plan.getVectorTripCount()) {
+      LLVM_DEBUG(dbgs() << "Found resume phi we cannot update for VPlan with "
+                           "FMaxNum/FMinNum reduction.\n");
       return false;
+    }
     auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
     ResumeR->setOperand(0, NewSel);
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index b1692d3e6b337..04cb7a7a5c19b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -103,10 +103,10 @@ struct VPlanTransforms {
   /// not valid.
   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
 
-  /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
-  /// If they do, try to update the vector loop to exit early if any input is
-  /// NaN and resume executing in the scalar loop to handle the NaNs there.
-  /// Return false if this attempt was unsuccessful.
+  /// Check if \p Plan contains any FMaxNum or FMinNum reductions. If they do,
+  /// try to update the vector loop to exit early if any input is NaN and resume
+  /// executing in the scalar loop to handle the NaNs there. Return false if
+  /// this attempt was unsuccessful.
   static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
 
   /// Clear NSW/NUW flags from reduction instructions if necessary.
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 6204d540df40c..49c25b02ce964 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -324,6 +324,122 @@ exit:
   ret float %max.next
 }
 
+define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_induction_starts_at_10(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 10, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 10, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret float %max.next
+}
+
+define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_induction_starts_at_value(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret float %max.next
+}
+
+define float @fmaxnum_with_additional_add(ptr noalias %src, ptr noalias %src.2, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_with_additional_add(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[SRC_2:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC_2]], i64 [[IV]]
+; CHECK-NEXT:    [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4
+; CHECK-NEXT:    [[SUM_NEXT]] = add i32 [[SUM]], [[L_SRC_2]]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    store i32 [[SUM_NEXT_LCSSA]], ptr [[SRC_2]], align 4
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %sum.next, %loop ]
+  %gep.src.2 = getelementptr inbounds nuw i32, ptr %src.2, i64 %iv
+  %l.src.2 = load i32, ptr %gep.src.2, align 4
+  %sum.next = add i32 %sum, %l.src.2
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  store i32 %sum.next, ptr %src.2
+  ret float %max.next
+}
+
 define float @fmax_select_with_blend(ptr %A, ptr %B) {
 ; CHECK-LABEL: define float @fmax_select_with_blend(
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {


From flang-commits at lists.llvm.org  Fri Jul 18 09:42:29 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 09:42:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix build-time warning (PR
	#149549)
Message-ID: <llvm.org/llvm/llvm-project/pull/149549@github.com>


https://github.com/klausler created https://github.com/llvm/llvm-project/pull/149549

Don't increment the LHS variable of an assignment that also uses that variable on the RHS.

>From db0880f45cb80e98381a1da4c829a80fe31308a4 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Fri, 18 Jul 2025 09:40:15 -0700
Subject: [PATCH] [flang][NFC] Fix build-time warning

Don't increment the LHS variable of an assignment that also
uses that variable on the RHS.
---
 flang/lib/Semantics/resolve-labels.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index 27e259fab38738..9454ef9fe928ad 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -492,10 +492,9 @@ class ParseTreeAnalyzer {
     // Uppercase the name of the main program, so that its symbol name
     // would be unique from similarly named non-main-program symbols.
     auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
-      char *ch{const_cast<char *>(cb.begin())};
-      char *endCh{ch + cb.size()};
-      while (ch != endCh) {
-        *ch++ = parser::ToUpperCaseLetter(*ch);
+      auto ch{const_cast<char *>(cb.begin())};
+      for (char *endCh{ch + cb.size()}; ch != endCh; ++ch) {
+        *ch = parser::ToUpperCaseLetter(*ch);
       }
     };
     const parser::CharBlock *progName{nullptr};


From flang-commits at lists.llvm.org  Fri Jul 18 09:43:16 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 09:43:16 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix build-time warning (PR
	#149549)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149549@github.com>
Message-ID: <687a79a4.170a0220.2cb196.b7bb@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Peter Klausler (klausler)

<details>
<summary>Changes</summary>

Don't increment the LHS variable of an assignment that also uses that variable on the RHS.

---
Full diff: https://github.com/llvm/llvm-project/pull/149549.diff


1 Files Affected:

- (modified) flang/lib/Semantics/resolve-labels.cpp (+3-4) 


``````````diff
diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index 27e259fab38738..9454ef9fe928ad 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -492,10 +492,9 @@ class ParseTreeAnalyzer {
     // Uppercase the name of the main program, so that its symbol name
     // would be unique from similarly named non-main-program symbols.
     auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
-      char *ch{const_cast<char *>(cb.begin())};
-      char *endCh{ch + cb.size()};
-      while (ch != endCh) {
-        *ch++ = parser::ToUpperCaseLetter(*ch);
+      auto ch{const_cast<char *>(cb.begin())};
+      for (char *endCh{ch + cb.size()}; ch != endCh; ++ch) {
+        *ch = parser::ToUpperCaseLetter(*ch);
       }
     };
     const parser::CharBlock *progName{nullptr};

``````````

</details>


https://github.com/llvm/llvm-project/pull/149549

From flang-commits at lists.llvm.org  Fri Jul 18 09:44:32 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 09:44:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix build-time warning (PR
	#149549)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149549@github.com>
Message-ID: <687a79f0.170a0220.122c7f.bb50@mx.google.com>


https://github.com/eugeneepshteyn approved this pull request.


https://github.com/llvm/llvm-project/pull/149549

From flang-commits at lists.llvm.org  Fri Jul 18 09:47:43 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 09:47:43 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a7aaf.050a0220.3afcec.b845@mx.google.com>


fhahn wrote:

Fixed the PR the remove various changes added by accident.

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 09:46:07 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 09:46:07 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a7a4f.170a0220.3b9f3f.65c6@mx.google.com>


https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/148239

>From 42d11944daf17b42cc9eae6a1b14e25740f37de4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 18 Jun 2025 14:07:04 +0100
Subject: [PATCH 1/6] [LV] Vectorize maxnum/minnum w/o fast-math flags.

Update LV to vectorize maxnum/minnum reductions without fast-math flags,
by adding an extra check in the loop if any inputs to maxnum/minnum are
NaN. If any input is NaN,
 *exit the vector loop,
 *compute the reduction result up to the vector iteration that contained NaN
  inputs and
 * resume in the scalar loop

New recurrence kinds are added for reductions using maxnum/minnum
without fast-math flags. The new recurrence kinds are not supported in
the code to generate IR to perform the reductions to prevent accidential
mis-use. Users need to add the required checks ensuring no NaN inputs,
and convert to regular FMin/FMax recurrence kinds.
---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   2 +
 llvm/lib/Analysis/IVDescriptors.cpp           |  23 ++-
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   2 +
 .../Transforms/Vectorize/LoopVectorize.cpp    |  14 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |   6 +
 llvm/lib/Transforms/Vectorize/VPlan.h         |   8 +-
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   1 +
 .../Vectorize/VPlanConstruction.cpp           | 137 ++++++++++++++++++
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |   8 +-
 .../Transforms/Vectorize/VPlanTransforms.h    |   5 +
 .../AArch64/fmax-without-fast-math-flags.ll   |  54 ++++++-
 .../AArch64/fmin-without-fast-math-flags.ll   |  54 ++++++-
 ...fmax-without-fast-math-flags-interleave.ll |  54 ++++++-
 .../fmax-without-fast-math-flags.ll           |  90 ++++++++++--
 .../fmin-without-fast-math-flags.ll           |  90 ++++++++++--
 .../LoopVectorize/minmax_reduction.ll         |   6 +-
 16 files changed, 492 insertions(+), 62 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index b985292ccee40..e28901c072e29 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,6 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
+  FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
+  FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 39f74beca082f..ef095b73d5419 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -941,10 +941,27 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                   m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
             match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
     };
-    if (isIntMinMaxRecurrenceKind(Kind) ||
-        (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+    if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
-    else if (isFMulAddIntrinsic(I))
+    if (isFPMinMaxRecurrenceKind(Kind)) {
+      if (HasRequiredFMF())
+        return isMinMaxPattern(I, Kind, Prev);
+      // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
+      // intrinsics with extra checks ensuring the inputs are not NaN.
+      auto *StartV = dyn_cast<ConstantFP>(
+          OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
+      if (StartV && !StartV->getValue().isNaN() &&
+          isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
+        if (((Kind == RecurKind::FMax &&
+              match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
+             Kind == RecurKind::FMaxNumNoFMFs))
+          return InstDesc(I, RecurKind::FMaxNumNoFMFs);
+        if (((Kind == RecurKind::FMin &&
+              match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
+             Kind == RecurKind::FMinNumNoFMFs))
+          return InstDesc(I, RecurKind::FMinNumNoFMFs);
+      }
+    } else if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
     return InstDesc(false, I);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 200d1fb854155..0cf7bc7af878c 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -938,8 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
   case RecurKind::UMin:
     return Intrinsic::vector_reduce_umin;
   case RecurKind::FMax:
+  case RecurKind::FMaxNumNoFMFs:
     return Intrinsic::vector_reduce_fmax;
   case RecurKind::FMin:
+  case RecurKind::FMinNumNoFMFs:
     return Intrinsic::vector_reduce_fmin;
   case RecurKind::FMaximum:
     return Intrinsic::vector_reduce_fmaximum;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ceeabd65cced3..b070b04cc3c35 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4347,8 +4347,15 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
   // Cross iteration phis such as reductions need special handling and are
   // currently unsupported.
-  if (any_of(OrigLoop->getHeader()->phis(),
-             [&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
+  if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
+        if (Legal->isReductionVariable(&Phi)) {
+          RecurKind RK =
+              Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
+          return RK == RecurKind::FMinNumNoFMFs ||
+                 RK == RecurKind::FMaxNumNoFMFs;
+        }
+        return Legal->isFixedOrderRecurrence(&Phi);
+      }))
     return false;
 
   // Phis with uses outside of the loop require special handling and are
@@ -8769,6 +8776,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
 
   // Adjust the recipes for any inloop reductions.
   adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+  if (!VPlanTransforms::runPass(
+          VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
+    return nullptr;
 
   // Transform recipes to abstract recipes if it is legal and beneficial and
   // clamp the range for better cost estimation.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index da6af353c709f..b3040d0317c95 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23193,6 +23193,8 @@ class HorizontalReduction {
         case RecurKind::FindFirstIVUMin:
         case RecurKind::FindLastIVSMax:
         case RecurKind::FindLastIVUMax:
+        case RecurKind::FMaxNumNoFMFs:
+        case RecurKind::FMinNumNoFMFs:
         case RecurKind::FMaximumNum:
         case RecurKind::FMinimumNum:
         case RecurKind::None:
@@ -23330,6 +23332,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
+    case RecurKind::FMaxNumNoFMFs:
+    case RecurKind::FMinNumNoFMFs:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
@@ -23432,6 +23436,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
+    case RecurKind::FMaxNumNoFMFs:
+    case RecurKind::FMinNumNoFMFs:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 204268e586b43..70c34ec3c25cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1356,16 +1356,16 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
   unsigned Opcode;
 
 public:
+  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+        Opcode(I.getOpcode()) {}
+
   VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags, const VPIRMetadata &Metadata,
                 DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
         VPIRMetadata(Metadata), Opcode(Opcode) {}
 
-  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
-      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
-        Opcode(I.getOpcode()) {}
-
   ~VPWidenRecipe() override = default;
 
   VPWidenRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index b27a7ffeed208..66657b98b094b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -84,6 +84,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
     return ResTy;
   }
   case Instruction::ICmp:
+  case Instruction::FCmp:
   case VPInstruction::ActiveLaneMask:
     assert(inferScalarType(R->getOperand(0)) ==
                inferScalarType(R->getOperand(1)) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 7fb5e82f9d32b..69efb9f89b479 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -652,3 +652,140 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
+                                      VPReductionPHIRecipe *RedPhi) {
+  auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+  if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+      !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+    return nullptr;
+
+  if (MinMaxOp->getOperand(0) == RedPhi)
+    return MinMaxOp->getOperand(1);
+  return MinMaxOp->getOperand(0);
+}
+
+/// Returns true if there VPlan is read-only and execution can be resumed at the
+/// beginning of the last vector iteration in the scalar loop
+static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
+    if (!VPBB)
+      return false;
+    for (auto &R : *VPBB) {
+      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        continue;
+      if (R.mayWriteToMemory())
+        return false;
+    }
+  }
+  return true;
+}
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPValue *AnyNaN = nullptr;
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  VPRecipeWithIRFlags *MinMaxOp = nullptr;
+  bool HasUnsupportedPhi = false;
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
+    if (RedPhiR)
+      return false;
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+      continue;
+
+    RedPhiR = Cur;
+    MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxOp)
+      return false;
+    VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
+    if (!In)
+      return false;
+
+    auto *IsNaN =
+        new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
+    IsNaN->insertBefore(MinMaxOp);
+    AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
+    AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
+  }
+
+  if (!AnyNaN)
+    return true;
+
+  if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+    return false;
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
+  auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
+  if (!RdxResult ||
+      RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
+      RdxResult->getOperand(0) != RedPhiR)
+    return false;
+
+  auto *ScalarPH = Plan.getScalarPreheader();
+  // Update the resume phis in the scalar preheader. They all must either resume
+  // from the reduction result or the canonical induction. Bail out if there are
+  // other resume phis.
+  for (auto &R : ScalarPH->phis()) {
+    auto *ResumeR = cast<VPPhi>(&R);
+    VPValue *VecV = ResumeR->getOperand(0);
+    VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
+    if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
+      return false;
+    ResumeR->setOperand(
+        1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
+    ResumeR->addOperand(BypassV);
+  }
+
+  // Create a new reduction phi recipe with either FMin/FMax, replacing
+  // FMinNumNoFMFs/FMaxNumNoFMFs.
+  RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
+                        ? RecurKind::FMin
+                        : RecurKind::FMax;
+  auto *NewRedPhiR = new VPReductionPHIRecipe(
+      cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
+      *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
+  NewRedPhiR->addOperand(RedPhiR->getOperand(1));
+  NewRedPhiR->insertBefore(RedPhiR);
+  RedPhiR->replaceAllUsesWith(NewRedPhiR);
+  RedPhiR->eraseFromParent();
+
+  // Update the loop exit condition to exit if either any of the inputs is NaN
+  // or the vector trip count is reached.
+  VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
+  VPBuilder Builder(LatchVPBB->getTerminator());
+  auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+  assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
+         "Unexpected terminator");
+  auto *IsLatchExitTaken =
+      Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
+                         LatchExitingBranch->getOperand(1));
+  auto *AnyExitTaken =
+      Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
+  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
+  LatchExitingBranch->eraseFromParent();
+
+  // Split the middle block and introduce a new block, branching to the scalar
+  // preheader to resume iteration in the scalar loop if any NaNs have been
+  // encountered.
+  MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
+  Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
+  auto *NewSel =
+      Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+  RdxResult->setOperand(1, NewSel);
+  Builder.setInsertPoint(MiddleVPBB);
+  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
+  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
+  MiddleVPBB->swapSuccessors();
+  std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
+  return true;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1664bcc3881aa..57b713d3dfcb9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -587,6 +587,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     Value *Op = State.get(getOperand(0), vputils::onlyFirstLaneUsed(this));
     return Builder.CreateFreeze(Op, Name);
   }
+  case Instruction::FCmp:
   case Instruction::ICmp: {
     bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
     Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
@@ -860,7 +861,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     Value *Res = State.get(getOperand(0));
     for (VPValue *Op : drop_begin(operands()))
       Res = Builder.CreateOr(Res, State.get(Op));
-    return Builder.CreateOrReduce(Res);
+    return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res);
   }
   case VPInstruction::FirstActiveLane: {
     if (getNumOperands() == 1) {
@@ -1033,6 +1034,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   switch (getOpcode()) {
   case Instruction::ExtractElement:
   case Instruction::Freeze:
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
   case VPInstruction::AnyOf:
@@ -1068,6 +1070,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
     return Op == getOperand(1);
   case Instruction::PHI:
     return true;
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
   case Instruction::Or:
@@ -1100,6 +1103,7 @@ bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const {
   switch (getOpcode()) {
   default:
     return false;
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
     return vputils::onlyFirstPartUsed(this);
@@ -1786,7 +1790,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
     return Opcode == Instruction::ZExt;
     break;
   case OperationType::Cmp:
-    return Opcode == Instruction::ICmp;
+    return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
   case OperationType::Other:
     return true;
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 84a12470f45e4..52e33ad0d4f99 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -103,6 +103,11 @@ struct VPlanTransforms {
   /// not valid.
   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
 
+  /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
+  /// If they do, try to update the vector loop to exit early if any input is
+  /// NaN and resume executing in the scalar loop to handle the NaNs there.
+  static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
+
   /// Clear NSW/NUW flags from reduction instructions if necessary.
   static void clearReductionWrapFlags(VPlan &Plan);
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 451574a258c2b..1a9752a1ef9fd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -42,18 +42,58 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index e93ee5563b057..3deaa5f0e1f60 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -42,18 +42,58 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0x7FF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index b2e080fef2e57..8ef5b2720e4cd 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -42,18 +42,58 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 5661406b88a5a..332f3a60c5b3e 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -192,18 +192,49 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum_1(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -227,18 +258,49 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum_2(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 148beb64a3609..0e3566b18432a 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -192,18 +192,49 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum_1(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[L]], float [[MAX]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -227,18 +258,49 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum_2(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index 85a90f2e04c5e..1429dc26c5207 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -1001,8 +1001,9 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
+; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmin_intrinsic_nofast(
-; CHECK-NOT: <2 x float> @llvm.minnum.v2f32
+; CHECK: <2 x float> @llvm.minnum.v2f32
 define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body
@@ -1021,8 +1022,9 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
+; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmax_intrinsic_nofast(
-; CHECK-NOT: <2 x float> @llvm.maxnum.v2f32
+; CHECK: <2 x float> @llvm.maxnum.v2f32
 define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body

>From de2d6f2c89a3ead527a03c23eaaa4b7ce2ade11b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Jul 2025 09:53:59 +0100
Subject: [PATCH 2/6] !fixup address latest comments, thanks

---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   4 +-
 llvm/lib/Analysis/IVDescriptors.cpp           |  31 ++--
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   4 +-
 .../Vectorize/LoopVectorizationPlanner.h      |  12 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  10 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  12 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |   8 +-
 .../Vectorize/VPlanConstruction.cpp           | 144 +++++++++---------
 .../Transforms/Vectorize/VPlanTransforms.h    |   1 +
 .../AArch64/fmax-without-fast-math-flags.ll   |  26 ++--
 .../AArch64/fmin-without-fast-math-flags.ll   |  26 ++--
 ...fmax-without-fast-math-flags-interleave.ll |  26 ++--
 .../fmax-without-fast-math-flags.ll           |  44 +++---
 .../fmin-without-fast-math-flags.ll           |  44 +++---
 .../LoopVectorize/minmax_reduction.ll         |   2 +
 15 files changed, 212 insertions(+), 182 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index e28901c072e29..84a7672cd47f3 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,8 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
-  FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
-  FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
+  FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
+  FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index ef095b73d5419..accda5a9713fc 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -944,23 +944,24 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
     if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
     if (isFPMinMaxRecurrenceKind(Kind)) {
+      InstDesc Res = isMinMaxPattern(I, Kind, Prev);
       if (HasRequiredFMF())
-        return isMinMaxPattern(I, Kind, Prev);
+        return Res;
+
+      if (!Res.isRecurrence())
+        return InstDesc(false, I);
+
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
-      // intrinsics with extra checks ensuring the inputs are not NaN.
-      auto *StartV = dyn_cast<ConstantFP>(
-          OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
-      if (StartV && !StartV->getValue().isNaN() &&
-          isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
-        if (((Kind == RecurKind::FMax &&
-              match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
-             Kind == RecurKind::FMaxNumNoFMFs))
-          return InstDesc(I, RecurKind::FMaxNumNoFMFs);
-        if (((Kind == RecurKind::FMin &&
-              match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
-             Kind == RecurKind::FMinNumNoFMFs))
-          return InstDesc(I, RecurKind::FMinNumNoFMFs);
-      }
+      // intrinsics with extra checks ensuring the vector loop handles only
+      // non-NaN inputs.
+      if (Kind == RecurKind::FMax &&
+          match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+        return InstDesc(I, RecurKind::FMaxNum);
+      if (Kind == RecurKind::FMin &&
+          match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+        return InstDesc(I, RecurKind::FMinNum);
+
+      return InstDesc(false, I);
     } else if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 0cf7bc7af878c..7bb7a7b116384 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -938,10 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
   case RecurKind::UMin:
     return Intrinsic::vector_reduce_umin;
   case RecurKind::FMax:
-  case RecurKind::FMaxNumNoFMFs:
+  case RecurKind::FMaxNum:
     return Intrinsic::vector_reduce_fmax;
   case RecurKind::FMin:
-  case RecurKind::FMinNumNoFMFs:
+  case RecurKind::FMinNum:
     return Intrinsic::vector_reduce_fmin;
   case RecurKind::FMaximum:
     return Intrinsic::vector_reduce_fmaximum;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 11853859484e3..f57ce0c3ccb4d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -230,7 +230,6 @@ class VPBuilder {
 
   /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
   /// and \p B.
-  /// TODO: add createFCmp when needed.
   VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
                             DebugLoc DL = DebugLoc::getUnknown(),
                             const Twine &Name = "") {
@@ -240,6 +239,17 @@ class VPBuilder {
         new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name));
   }
 
+  /// Create a new FCmp VPInstruction with predicate \p Pred and operands \p A
+  /// and \p B.
+  VPInstruction *createFCmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
+                            DebugLoc DL = DebugLoc::getUnknown(),
+                            const Twine &Name = "") {
+    assert(Pred >= CmpInst::FIRST_FCMP_PREDICATE &&
+           Pred <= CmpInst::LAST_FCMP_PREDICATE && "invalid predicate");
+    return tryInsertInstruction(
+        new VPInstruction(Instruction::FCmp, {A, B}, Pred, DL, Name));
+  }
+
   VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
                               DebugLoc DL = DebugLoc::getUnknown(),
                               const Twine &Name = "") {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b070b04cc3c35..712c5f8eb4c56 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4345,14 +4345,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
 
 bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
-  // Cross iteration phis such as reductions need special handling and are
-  // currently unsupported.
+  // Cross iteration phis such as first-order recurrences and FMaxNum/FMinNum
+  // reductions need special handling and are currently unsupported.
   if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
         if (Legal->isReductionVariable(&Phi)) {
           RecurKind RK =
               Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
-          return RK == RecurKind::FMinNumNoFMFs ||
-                 RK == RecurKind::FMaxNumNoFMFs;
+          return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
         }
         return Legal->isFixedOrderRecurrence(&Phi);
       }))
@@ -8776,6 +8775,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
 
   // Adjust the recipes for any inloop reductions.
   adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+
+  // Apply mandatory transformation to handle FP maxnum/minnum reduction with
+  // NaNs if possible, bail out otherwise.
   if (!VPlanTransforms::runPass(
           VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
     return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b3040d0317c95..c1e4ff00a65e9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23193,8 +23193,8 @@ class HorizontalReduction {
         case RecurKind::FindFirstIVUMin:
         case RecurKind::FindLastIVSMax:
         case RecurKind::FindLastIVUMax:
-        case RecurKind::FMaxNumNoFMFs:
-        case RecurKind::FMinNumNoFMFs:
+        case RecurKind::FMaxNum:
+        case RecurKind::FMinNum:
         case RecurKind::FMaximumNum:
         case RecurKind::FMinimumNum:
         case RecurKind::None:
@@ -23332,8 +23332,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
-    case RecurKind::FMaxNumNoFMFs:
-    case RecurKind::FMinNumNoFMFs:
+    case RecurKind::FMaxNum:
+    case RecurKind::FMinNum:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
@@ -23436,8 +23436,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
-    case RecurKind::FMaxNumNoFMFs:
-    case RecurKind::FMinNumNoFMFs:
+    case RecurKind::FMaxNum:
+    case RecurKind::FMinNum:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 70c34ec3c25cd..204268e586b43 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1356,16 +1356,16 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
   unsigned Opcode;
 
 public:
-  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
-      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
-        Opcode(I.getOpcode()) {}
-
   VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags, const VPIRMetadata &Metadata,
                 DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
         VPIRMetadata(Metadata), Opcode(Opcode) {}
 
+  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+        Opcode(I.getOpcode()) {}
+
   ~VPWidenRecipe() override = default;
 
   VPWidenRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 69efb9f89b479..8d1ec5b68457e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -653,77 +653,75 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
   }
 }
 
-static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
-                                      VPReductionPHIRecipe *RedPhi) {
-  auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
-  if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
-      !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
-    return nullptr;
-
-  if (MinMaxOp->getOperand(0) == RedPhi)
-    return MinMaxOp->getOperand(1);
-  return MinMaxOp->getOperand(0);
-}
-
-/// Returns true if there VPlan is read-only and execution can be resumed at the
-/// beginning of the last vector iteration in the scalar loop
-static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
-  for (VPBlockBase *VPB : vp_depth_first_shallow(
-           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
-    auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
-    if (!VPBB)
-      return false;
-    for (auto &R : *VPBB) {
-      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
-        continue;
-      if (R.mayWriteToMemory())
-        return false;
-    }
-  }
-  return true;
-}
-
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-  VPValue *AnyNaN = nullptr;
   VPReductionPHIRecipe *RedPhiR = nullptr;
-  VPRecipeWithIRFlags *MinMaxOp = nullptr;
+  VPValue *MinMaxOp = nullptr;
   bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
+                                  VPReductionPHIRecipe *RedPhi) -> VPValue * {
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+    if (MinMaxOp->getOperand(0) == RedPhi)
+      return MinMaxOp->getOperand(1);
+    assert(MinMaxOp->getOperand(1) == RedPhi &&
+           "Reduction phi operand expected");
+    return MinMaxOp->getOperand(0);
+  };
+
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support first-order recurrence phis.
     HasUnsupportedPhi |=
         !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
              VPReductionPHIRecipe>(&R);
     auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
     if (!Cur)
       continue;
+    // For now, only a single reduction is supported.
+    // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
     if (RedPhiR)
       return false;
-    if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
-        Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNum)
       continue;
 
     RedPhiR = Cur;
-    MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
         RedPhiR->getBackedgeValue()->getDefiningRecipe());
-    if (!MinMaxOp)
+    if (!MinMaxR)
       return false;
-    VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
-    if (!In)
+    MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
+    if (!MinMaxOp)
       return false;
-
-    auto *IsNaN =
-        new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
-    IsNaN->insertBefore(MinMaxOp);
-    AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
-    AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
   }
 
-  if (!AnyNaN)
+  if (!RedPhiR)
     return true;
 
-  if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+  if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
+  /// Check if the vector loop of \p Plan can early exit and restart
+  /// execution of last vector iteration in the scalar loop. This requires all
+  /// recipes up to early exit point be side-effect free as they are
+  /// re-executed. Currently we check that the loop is free of any recipe that
+  /// may write to memory. Expected to operate on an early VPlan w/o nested
+  /// regions.
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = cast<VPBasicBlock>(VPB);
+    for (auto &R : *VPBB) {
+      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        continue;
+      if (R.mayWriteToMemory())
+        return false;
+    }
+  }
+
   auto *MiddleVPBB = Plan.getMiddleBlock();
   auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
   if (!RdxResult ||
@@ -731,24 +729,9 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
       RdxResult->getOperand(0) != RedPhiR)
     return false;
 
-  auto *ScalarPH = Plan.getScalarPreheader();
-  // Update the resume phis in the scalar preheader. They all must either resume
-  // from the reduction result or the canonical induction. Bail out if there are
-  // other resume phis.
-  for (auto &R : ScalarPH->phis()) {
-    auto *ResumeR = cast<VPPhi>(&R);
-    VPValue *VecV = ResumeR->getOperand(0);
-    VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
-    if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
-      return false;
-    ResumeR->setOperand(
-        1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
-    ResumeR->addOperand(BypassV);
-  }
-
   // Create a new reduction phi recipe with either FMin/FMax, replacing
-  // FMinNumNoFMFs/FMaxNumNoFMFs.
-  RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
+  // FMinNum/FMaxNum.
+  RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
                         ? RecurKind::FMin
                         : RecurKind::FMax;
   auto *NewRedPhiR = new VPReductionPHIRecipe(
@@ -769,23 +752,40 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   auto *IsLatchExitTaken =
       Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
                          LatchExitingBranch->getOperand(1));
+
+  VPValue *IsNaN = Builder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
+  VPValue *AnyNaN = Builder.createNaryOp(VPInstruction::AnyOf, {IsNaN});
   auto *AnyExitTaken =
       Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
   Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
   LatchExitingBranch->eraseFromParent();
 
-  // Split the middle block and introduce a new block, branching to the scalar
-  // preheader to resume iteration in the scalar loop if any NaNs have been
-  // encountered.
-  MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
+  // If we exit early due to NaNs, compute the final reduction result based on
+  // the reduction phi at the beginning of the last vector iteration.
   Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
   auto *NewSel =
       Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
   RdxResult->setOperand(1, NewSel);
-  Builder.setInsertPoint(MiddleVPBB);
-  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
-  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
-  MiddleVPBB->swapSuccessors();
-  std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
+
+  auto *ScalarPH = Plan.getScalarPreheader();
+  // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+  // true, the resume from the start of the last vector iteration via the
+  // canonical IV, otherwise from the original value.
+  for (auto &R : ScalarPH->phis()) {
+    auto *ResumeR = cast<VPPhi>(&R);
+    VPValue *VecV = ResumeR->getOperand(0);
+    if (VecV == RdxResult)
+      continue;
+    if (VecV != &Plan.getVectorTripCount())
+      return false;
+    auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
+    ResumeR->setOperand(0, NewSel);
+  }
+
+  auto *MiddleTerm = MiddleVPBB->getTerminator();
+  Builder.setInsertPoint(MiddleTerm);
+  VPValue *MiddleCond = MiddleTerm->getOperand(0);
+  VPValue *NewCond = Builder.createAnd(MiddleCond, Builder.createNot(AnyNaN));
+  MiddleTerm->setOperand(0, NewCond);
   return true;
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 52e33ad0d4f99..b1692d3e6b337 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -106,6 +106,7 @@ struct VPlanTransforms {
   /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
   /// If they do, try to update the vector loop to exit early if any input is
   /// NaN and resume executing in the scalar loop to handle the NaNs there.
+  /// Return false if this attempt was unsuccessful.
   static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
 
   /// Clear NSW/NUW flags from reduction instructions if necessary.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 1a9752a1ef9fd..6810c90f0b829 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -57,31 +57,33 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index 3deaa5f0e1f60..a18714f94343b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -57,31 +57,33 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 8ef5b2720e4cd..5e60b32e3ae6b 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -57,31 +57,33 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 332f3a60c5b3e..5199bc2bad1da 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -204,25 +204,27 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -234,7 +236,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -270,25 +272,27 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -300,7 +304,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 0e3566b18432a..f220b900d4d0f 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -204,25 +204,27 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -234,7 +236,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -270,25 +272,27 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -300,7 +304,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index 1429dc26c5207..e7ab02cd98a5e 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -1004,6 +1004,7 @@ for.body:                                         ; preds = %entry, %for.body
 ; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmin_intrinsic_nofast(
 ; CHECK: <2 x float> @llvm.minnum.v2f32
+; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
 define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body
@@ -1025,6 +1026,7 @@ for.body:                                         ; preds = %entry, %for.body
 ; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmax_intrinsic_nofast(
 ; CHECK: <2 x float> @llvm.maxnum.v2f32
+; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
 define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body

>From 8ebc7ff41e3f68c42a99492fce9a5f55e5f23bde Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Jul 2025 15:36:12 +0100
Subject: [PATCH 3/6] !fixup add assertion for intrinsic ID

---
 .../lib/Transforms/Vectorize/VPlanConstruction.cpp | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8d1ec5b68457e..21e3ec3e3f0fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -697,6 +697,20 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
     if (!MinMaxOp)
       return false;
+
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        Cur->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                       : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (isa<VPReplicateRecipe>(MinMaxR) &&
+            cast<IntrinsicInst>(
+                cast<VPReplicateRecipe>(MinMaxR)->getUnderlyingInstr())
+                    ->getIntrinsicID() == RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
   }
 
   if (!RedPhiR)

>From 914a0b4b32617cc63b1c902402fd670e4055267d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 10:27:24 +0100
Subject: [PATCH 4/6] !fixup address latest comments, thanks

---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   5 +-
 llvm/lib/Analysis/IVDescriptors.cpp           |  18 +--
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   7 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  12 +-
 .../Vectorize/VPlanConstruction.cpp           | 108 ++++++++----------
 .../AArch64/fmax-without-fast-math-flags.ll   |   7 +-
 .../AArch64/fmin-without-fast-math-flags.ll   |   7 +-
 ...fmax-without-fast-math-flags-interleave.ll |   7 +-
 .../fmax-without-fast-math-flags.ll           |   4 +-
 .../fmin-without-fast-math-flags.ll           |   4 +-
 10 files changed, 88 insertions(+), 91 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 84a7672cd47f3..1dc73205a0ebb 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,8 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
-  FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
-  FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
+  FMinNum,  ///< FP min with llvm.minnum semantics including NaNs.
+  FMaxNum,  ///< FP max with llvm.maxnum semantics including NaNs.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
@@ -252,6 +252,7 @@ class RecurrenceDescriptor {
   /// Returns true if the recurrence kind is a floating-point min/max kind.
   static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
     return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
+           Kind == RecurKind::FMinNum || Kind == RecurKind::FMaxNum ||
            Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
            Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum;
   }
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index accda5a9713fc..a88bbd7b65d53 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -945,21 +945,25 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
       return isMinMaxPattern(I, Kind, Prev);
     if (isFPMinMaxRecurrenceKind(Kind)) {
       InstDesc Res = isMinMaxPattern(I, Kind, Prev);
-      if (HasRequiredFMF())
-        return Res;
-
       if (!Res.isRecurrence())
         return InstDesc(false, I);
 
+      if (HasRequiredFMF())
+        return Res;
+
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
       // intrinsics with extra checks ensuring the vector loop handles only
       // non-NaN inputs.
-      if (Kind == RecurKind::FMax &&
-          match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+      if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMax &&
+               "unexpected recurrence kind for maxnum");
         return InstDesc(I, RecurKind::FMaxNum);
-      if (Kind == RecurKind::FMin &&
-          match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+      }
+      if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMin &&
+               "unexpected recurrence kind for minnum");
         return InstDesc(I, RecurKind::FMinNum);
+      }
 
       return InstDesc(false, I);
     } else if (isFMulAddIntrinsic(I))
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 7bb7a7b116384..154f50acf3d1d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1039,8 +1039,10 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) {
   case RecurKind::SMax:
     return Intrinsic::smax;
   case RecurKind::FMin:
+  case RecurKind::FMinNum:
     return Intrinsic::minnum;
   case RecurKind::FMax:
+  case RecurKind::FMaxNum:
     return Intrinsic::maxnum;
   case RecurKind::FMinimum:
     return Intrinsic::minimum;
@@ -1098,7 +1100,8 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
                             Value *Right) {
   Type *Ty = Left->getType();
   if (Ty->isIntOrIntVectorTy() ||
-      (RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
+      (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
+       RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
        RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
     // TODO: Add float minnum/maxnum support when FMF nnan is set.
     Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
@@ -1310,6 +1313,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
   case RecurKind::UMin:
   case RecurKind::FMax:
   case RecurKind::FMin:
+  case RecurKind::FMinNum:
+  case RecurKind::FMaxNum:
   case RecurKind::FMinimum:
   case RecurKind::FMaximum:
   case RecurKind::FMinimumNum:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 712c5f8eb4c56..db0902c589d49 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4345,15 +4345,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
 
 bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
-  // Cross iteration phis such as first-order recurrences and FMaxNum/FMinNum
+  // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
   // reductions need special handling and are currently unsupported.
   if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
-        if (Legal->isReductionVariable(&Phi)) {
-          RecurKind RK =
-              Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
-          return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
-        }
-        return Legal->isFixedOrderRecurrence(&Phi);
+        if (!Legal->isReductionVariable(&Phi))
+          return Legal->isFixedOrderRecurrence(&Phi);
+        RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
+        return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
       }))
     return false;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 21e3ec3e3f0fe..a426d06b4befe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -656,25 +656,42 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   VPReductionPHIRecipe *RedPhiR = nullptr;
-  VPValue *MinMaxOp = nullptr;
   bool HasUnsupportedPhi = false;
 
-  auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
-                                  VPReductionPHIRecipe *RedPhi) -> VPValue * {
-    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
-    if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
         !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
       return nullptr;
 
-    if (MinMaxOp->getOperand(0) == RedPhi)
-      return MinMaxOp->getOperand(1);
-    assert(MinMaxOp->getOperand(1) == RedPhi &&
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                           : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (RepR &&
+            cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+                RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
+
+    if (MinMaxR->getOperand(0) == RedPhiR)
+      return MinMaxR->getOperand(1);
+
+    assert(MinMaxR->getOperand(1) == RedPhiR &&
            "Reduction phi operand expected");
-    return MinMaxOp->getOperand(0);
+    return MinMaxR->getOperand(0);
   };
 
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
-    // TODO: Also support first-order recurrence phis.
+    // TODO: Also support fixed-order recurrence phis.
     HasUnsupportedPhi |=
         !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
              VPReductionPHIRecipe>(&R);
@@ -688,34 +705,23 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
         Cur->getRecurrenceKind() != RecurKind::FMinNum)
       continue;
-
     RedPhiR = Cur;
-    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
-        RedPhiR->getBackedgeValue()->getDefiningRecipe());
-    if (!MinMaxR)
-      return false;
-    MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
-    if (!MinMaxOp)
-      return false;
-
-#ifndef NDEBUG
-    Intrinsic::ID RdxIntrinsicId =
-        Cur->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
-                                                       : Intrinsic::minnum;
-    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
-            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
-                RdxIntrinsicId) ||
-           (isa<VPReplicateRecipe>(MinMaxR) &&
-            cast<IntrinsicInst>(
-                cast<VPReplicateRecipe>(MinMaxR)->getUnderlyingInstr())
-                    ->getIntrinsicID() == RdxIntrinsicId) &&
-               "Intrinsic did not match recurrence kind");
-#endif
   }
 
   if (!RedPhiR)
     return true;
 
+  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+         "unsupported reduction");
+
+  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+  if (!MinMaxOp)
+    return false;
+
+  // We won't be able to resume execution in the scalar tail, if there are
+  // unsupported header phis or there is no scalar tail at all, due to
+  // tail-folding.
   if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
@@ -729,35 +735,12 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
            Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
     auto *VPBB = cast<VPBasicBlock>(VPB);
     for (auto &R : *VPBB) {
-      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
-        continue;
-      if (R.mayWriteToMemory())
+      if (R.mayWriteToMemory() &&
+          !match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
         return false;
     }
   }
 
-  auto *MiddleVPBB = Plan.getMiddleBlock();
-  auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
-  if (!RdxResult ||
-      RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
-      RdxResult->getOperand(0) != RedPhiR)
-    return false;
-
-  // Create a new reduction phi recipe with either FMin/FMax, replacing
-  // FMinNum/FMaxNum.
-  RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
-                        ? RecurKind::FMin
-                        : RecurKind::FMax;
-  auto *NewRedPhiR = new VPReductionPHIRecipe(
-      cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
-      *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
-  NewRedPhiR->addOperand(RedPhiR->getOperand(1));
-  NewRedPhiR->insertBefore(RedPhiR);
-  RedPhiR->replaceAllUsesWith(NewRedPhiR);
-  RedPhiR->eraseFromParent();
-
-  // Update the loop exit condition to exit if either any of the inputs is NaN
-  // or the vector trip count is reached.
   VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
   VPBuilder Builder(LatchVPBB->getTerminator());
   auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
@@ -776,9 +759,18 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
 
   // If we exit early due to NaNs, compute the final reduction result based on
   // the reduction phi at the beginning of the last vector iteration.
+  auto *RdxResult = find_singleton<VPSingleDefRecipe>(
+      RedPhiR->users(), [](VPUser *U, bool) -> VPSingleDefRecipe * {
+        auto *VPI = dyn_cast<VPInstruction>(U);
+        if (VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult)
+          return VPI;
+        return nullptr;
+      });
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
   Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
   auto *NewSel =
-      Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+      Builder.createSelect(AnyNaN, RedPhiR, RdxResult->getOperand(1));
   RdxResult->setOperand(1, NewSel);
 
   auto *ScalarPH = Plan.getScalarPreheader();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 6810c90f0b829..427a05cc1c843 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -50,8 +50,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index a18714f94343b..1a8e5940d88e7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -50,8 +50,8 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0x7FF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 5e60b32e3ae6b..a2eddad179216 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -50,8 +50,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 5199bc2bad1da..6204d540df40c 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -200,7 +200,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
@@ -268,7 +268,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index f220b900d4d0f..68bc8d0640a3f 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -200,7 +200,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
@@ -268,7 +268,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4

>From 18312dc501beba7c0c77b50c7f6e344f44174ee3 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 16:19:36 +0100
Subject: [PATCH 5/6] Merge remote-tracking branch 'origin/main' into
 lv-vectorize-fmaxnum-without-fmfs

---
 .../workflows/build-ci-container-windows.yml  |   2 -
 .github/workflows/build-ci-container.yml      |   2 -
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/Basic/BuiltinsAMDGPU.def  |   1 +
 clang/include/clang/Sema/Overload.h           |   2 -
 .../Core/PathSensitive/ExprEngine.h           |   7 +-
 clang/lib/AST/ByteCode/Descriptor.cpp         |   2 +
 clang/lib/AST/ByteCode/Descriptor.h           |   4 +
 clang/lib/AST/ByteCode/Disasm.cpp             |   1 +
 clang/lib/AST/ByteCode/Interp.cpp             |   5 +-
 clang/lib/AST/ByteCode/InterpBlock.h          |   5 -
 clang/lib/AST/ByteCode/InterpBuiltin.cpp      |   3 +-
 clang/lib/AST/ByteCode/InterpFrame.cpp        |   5 +
 clang/lib/AST/ByteCode/Pointer.h              |   5 +
 clang/lib/CodeGen/CGCall.cpp                  |  15 +-
 clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp   |   1 +
 clang/lib/Driver/ToolChain.cpp                |  40 +-
 clang/lib/Sema/SemaConcept.cpp                |   5 +
 clang/lib/Sema/SemaOverload.cpp               |  49 +-
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp  |  46 +-
 clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp |  48 --
 clang/test/AST/ByteCode/mutable.cpp           |  56 ++-
 clang/test/AST/ByteCode/unions.cpp            |  14 +
 clang/test/Analysis/div-zero-cxx20.cpp        |  61 +++
 clang/test/Analysis/div-zero.cpp              |  60 +++
 clang/test/CodeGen/64bit-swiftcall.c          |   2 +-
 clang/test/CodeGen/AArch64/byval-temp.c       |  16 +-
 .../AArch64/pure-scalable-args-empty-union.c  |   2 +-
 .../test/CodeGen/AArch64/pure-scalable-args.c |  42 +-
 .../AArch64/struct-coerce-using-ptr.cpp       |   4 +-
 ...-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c |   6 +-
 ...cle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp |   2 +-
 clang/test/CodeGen/LoongArch/bitint.c         |   6 +-
 clang/test/CodeGen/PowerPC/ppc64-vector.c     |   2 +-
 clang/test/CodeGen/RISCV/riscv-abi.cpp        |   8 +-
 .../RISCV/riscv-vector-callingconv-llvm-ir.c  |  10 +-
 .../riscv-vector-callingconv-llvm-ir.cpp      |  10 +-
 clang/test/CodeGen/RISCV/riscv32-abi.c        |  74 +--
 clang/test/CodeGen/RISCV/riscv32-vararg.c     |   2 +-
 clang/test/CodeGen/RISCV/riscv64-abi.c        |  18 +-
 clang/test/CodeGen/RISCV/riscv64-vararg.c     |   2 +-
 .../test/CodeGen/SystemZ/systemz-abi-vector.c |  52 +-
 clang/test/CodeGen/SystemZ/systemz-abi.c      |  38 +-
 .../test/CodeGen/SystemZ/systemz-inline-asm.c |   2 +-
 clang/test/CodeGen/X86/cx-complex-range.c     |   2 +-
 .../test/CodeGen/X86/x86_32-arguments-win32.c |  14 +-
 .../test/CodeGen/X86/x86_64-arguments-win32.c |   2 +-
 clang/test/CodeGen/aapcs64-align.cpp          |   4 +-
 clang/test/CodeGen/arm-aapcs-vfp.c            |   2 +-
 clang/test/CodeGen/arm-abi-vector.c           |   6 +-
 clang/test/CodeGen/arm-swiftcall.c            |   2 +-
 clang/test/CodeGen/arm64-abi-vector.c         |  14 +-
 clang/test/CodeGen/arm64-arguments.c          |  26 +-
 .../CodeGen/arm64-microsoft-arguments.cpp     |   2 +-
 clang/test/CodeGen/armv7k-abi.c               |   2 +-
 clang/test/CodeGen/atomic-arm64.c             |   2 +-
 clang/test/CodeGen/attr-noundef.cpp           |   7 +-
 clang/test/CodeGen/cx-complex-range.c         |  36 +-
 clang/test/CodeGen/ext-int-cc.c               |  36 +-
 clang/test/CodeGen/isfpclass.c                |   2 +-
 .../math-libcalls-tbaa-indirect-args.c        |  86 ++--
 clang/test/CodeGen/mingw-long-double.c        |   6 +-
 clang/test/CodeGen/ms_abi.c                   |   4 +-
 clang/test/CodeGen/pass-by-value-noalias.c    |   4 +-
 clang/test/CodeGen/ptrauth-in-c-struct.c      |   4 +-
 clang/test/CodeGen/regcall.c                  |  10 +-
 clang/test/CodeGen/regcall2.c                 |   2 +-
 clang/test/CodeGen/regcall4.c                 |  10 +-
 clang/test/CodeGen/sparcv9-abi.c              |   4 +-
 clang/test/CodeGen/vectorcall.c               |  46 +-
 clang/test/CodeGen/win-fp128.c                |   2 +-
 clang/test/CodeGen/win64-i128.c               |   4 +-
 clang/test/CodeGen/windows-swiftcall.c        |   2 +-
 .../CodeGenCXX/aarch64-mangle-sve-vectors.cpp |   4 +-
 clang/test/CodeGenCXX/arm-cc.cpp              |   2 +-
 .../CodeGenCXX/attr-target-mv-inalloca.cpp    |   8 +-
 clang/test/CodeGenCXX/copy-initialization.cpp |   2 +-
 clang/test/CodeGenCXX/debug-info.cpp          |   2 +-
 .../empty-nontrivially-copyable.cpp           |   4 +-
 clang/test/CodeGenCXX/fastcall.cpp            |   2 +-
 .../CodeGenCXX/homogeneous-aggregates.cpp     |  14 +-
 clang/test/CodeGenCXX/inalloca-lambda.cpp     |   6 +-
 .../test/CodeGenCXX/inalloca-overaligned.cpp  |   8 +-
 clang/test/CodeGenCXX/inalloca-vector.cpp     |   4 +-
 .../CodeGenCXX/inheriting-constructor.cpp     |   8 +-
 .../member-function-pointer-calls.cpp         |   4 +-
 .../CodeGenCXX/microsoft-abi-arg-order.cpp    |   4 +-
 .../CodeGenCXX/microsoft-abi-byval-thunks.cpp |  12 +-
 .../microsoft-abi-member-pointers.cpp         |   6 +-
 .../microsoft-abi-sret-and-byval.cpp          |  26 +-
 .../CodeGenCXX/microsoft-abi-unknown-arch.cpp |   2 +-
 clang/test/CodeGenCXX/ms-property.cpp         |   4 +-
 clang/test/CodeGenCXX/nrvo.cpp                |   4 +-
 .../test/CodeGenCXX/pass-by-value-noalias.cpp |  12 +-
 .../CodeGenCXX/ptrauth-qualifier-struct.cpp   |   2 +-
 clang/test/CodeGenCXX/regparm.cpp             |   2 +-
 clang/test/CodeGenCXX/trivial_abi.cpp         |   8 +-
 clang/test/CodeGenCXX/uncopyable-args.cpp     |  32 +-
 clang/test/CodeGenCXX/wasm-args-returns.cpp   |  12 +-
 .../test/CodeGenCXX/windows-x86-swiftcall.cpp |   4 +-
 .../nontrivial-c-struct-exception.m           |   2 +-
 .../test/CodeGenObjC/pass-by-value-noalias.m  |   4 +-
 clang/test/CodeGenObjC/weak-in-c-struct.m     |   6 +-
 .../test/CodeGenObjCXX/objc-struct-cxx-abi.mm |  12 +-
 clang/test/CodeGenObjCXX/property-objects.mm  |   2 +-
 .../CodeGenObjCXX/ptrauth-struct-cxx-abi.mm   |   2 +-
 .../CodeGenOpenCL/builtins-amdgcn-gfx1250.cl  |  38 ++
 .../test/Driver/print-multi-selection-flags.c |  19 +
 clang/test/Headers/stdarg.cpp                 |   4 +-
 .../test/OpenMP/for_firstprivate_codegen.cpp  |  30 +-
 .../OpenMP/parallel_firstprivate_codegen.cpp  | 216 ++++-----
 .../OpenMP/sections_firstprivate_codegen.cpp  |  14 +-
 .../OpenMP/single_firstprivate_codegen.cpp    |  14 +-
 ..._teams_distribute_firstprivate_codegen.cpp |  82 ++--
 ...bute_parallel_for_firstprivate_codegen.cpp | 224 ++++-----
 .../teams_distribute_firstprivate_codegen.cpp |  90 ++--
 ...bute_parallel_for_firstprivate_codegen.cpp | 124 ++---
 .../OpenMP/teams_firstprivate_codegen.cpp     |  80 ++--
 clang/test/Sema/dllexport.c                   |   4 +
 ...overload-resolution-deferred-templates.cpp |  28 ++
 .../test/SemaTemplate/concepts-using-decl.cpp |  21 +
 flang/lib/Lower/Bridge.cpp                    |  15 +-
 flang/lib/Lower/ConvertConstant.cpp           |   2 +-
 flang/lib/Lower/ConvertExpr.cpp               |  26 +-
 flang/lib/Lower/ConvertVariable.cpp           |  26 +-
 flang/lib/Lower/Runtime.cpp                   |   4 +-
 flang/lib/Lower/VectorSubscripts.cpp          |   2 +-
 flang/lib/Optimizer/Builder/FIRBuilder.cpp    |   2 +-
 .../Optimizer/Builder/Runtime/Intrinsics.cpp  |   6 +-
 flang/lib/Optimizer/Builder/Runtime/Stop.cpp  |   2 +-
 .../LowerHLFIROrderedAssignments.cpp          |   4 +-
 .../lib/Optimizer/Transforms/MemoryUtils.cpp  |   2 +-
 .../Transforms/PolymorphicOpConversion.cpp    |   2 +-
 .../Transforms/SimplifyFIROperations.cpp      |   2 +-
 flang/test/Lower/CUDA/cuda-set-allocator.cuf  |  32 ++
 .../Optimizer/FortranVariableTest.cpp         |   6 +-
 libc/cmake/modules/LLVMLibCTestRules.cmake    |  17 +-
 libcxx/include/__config_site.in               |   1 -
 libcxx/test/libcxx/fuzzing/random.pass.cpp    |   7 +-
 .../std/depr/depr.c.headers/math_h.pass.cpp   |   7 +-
 .../test/std/numerics/c.math/cmath.pass.cpp   |   7 +-
 lld/ELF/BPSectionOrderer.cpp                  |   6 +-
 lld/test/ELF/bp-section-orderer.s             |  21 +-
 lldb/source/Expression/IRExecutionUnit.cpp    |  16 +-
 .../TestNSDictionarySynthetic.py              |   3 +-
 .../PDB/Inputs/UdtLayoutTest.script           |   1 +
 llvm/Maintainers.md                           |  14 +
 llvm/docs/CodingStandards.rst                 |  46 +-
 .../llvm/CodeGen/MachineBlockFrequencyInfo.h  |   4 +-
 llvm/include/llvm/IR/DebugInfo.h              |   5 +-
 llvm/include/llvm/IR/DebugInfoMetadata.h      |   3 -
 .../llvm/Transforms/Utils/SSAUpdater.h        |   4 -
 llvm/lib/CodeGen/CodeGenPrepare.cpp           |  59 +--
 .../lib/CodeGen/MachineBlockFrequencyInfo.cpp |   4 +-
 llvm/lib/CodeGen/MachineDebugify.cpp          |  18 +-
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  16 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   4 +-
 llvm/lib/IR/DebugInfo.cpp                     |  25 +-
 llvm/lib/IR/DebugInfoMetadata.cpp             |   9 -
 llvm/lib/Object/IRSymtab.cpp                  |  45 +-
 .../Target/AArch64/AArch64ISelLowering.cpp    |   4 +-
 llvm/lib/Target/AMDGPU/AMDGPU.h               |   3 +
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |   1 +
 .../Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp  | 108 +++++
 .../Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h    |  23 +
 .../AMDGPU/AMDGPUResourceUsageAnalysis.cpp    | 234 +--------
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  13 +
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h  |   2 +
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     | 146 ++++--
 llvm/lib/Target/AMDGPU/SIInstrInfo.h          |   1 -
 llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp     |   8 +-
 llvm/lib/Target/AMDGPU/SIRegisterInfo.h       |   6 +-
 llvm/lib/Target/AMDGPU/VOP1Instructions.td    |   1 +
 .../LoongArch/LoongArchFrameLowering.cpp      |   7 +-
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp  | 122 +++--
 llvm/lib/Target/X86/X86CallingConv.cpp        |   2 +-
 llvm/lib/TargetParser/X86TargetParser.cpp     |   6 +-
 llvm/lib/Transforms/Coroutines/CoroFrame.cpp  |  52 +-
 llvm/lib/Transforms/Coroutines/CoroInternal.h |   5 +-
 llvm/lib/Transforms/Coroutines/CoroSplit.cpp  |  23 +-
 llvm/lib/Transforms/Coroutines/SpillUtils.cpp |   6 +-
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |  19 +-
 .../InstCombine/InstCombineInternal.h         |   3 -
 .../InstCombine/InstructionCombining.cpp      |  78 +--
 .../Scalar/ConstraintElimination.cpp          |   1 +
 .../Scalar/DeadStoreElimination.cpp           |   1 +
 llvm/lib/Transforms/Scalar/JumpThreading.cpp  |  39 +-
 .../lib/Transforms/Scalar/LoopInterchange.cpp |  24 +-
 .../Transforms/Scalar/LoopStrengthReduce.cpp  | 184 ++++---
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |   8 +-
 llvm/lib/Transforms/Utils/Debugify.cpp        |   7 -
 llvm/lib/Transforms/Utils/InlineFunction.cpp  |   3 +-
 llvm/lib/Transforms/Utils/Local.cpp           | 103 +---
 .../Transforms/Utils/LoopRotationUtils.cpp    |  23 +-
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   1 -
 .../Utils/PromoteMemoryToRegister.cpp         |  49 +-
 llvm/lib/Transforms/Utils/SSAUpdater.cpp      |  22 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  39 +-
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   9 +-
 llvm/lib/Transforms/Vectorize/VPlanAnalysis.h |   6 +-
 .../AArch64/sve-intrinsics-ldst-ext.ll        |  24 +
 llvm/test/CodeGen/AMDGPU/agpr-remat.ll        |  18 +-
 .../AMDGPU/amdgpu-prepare-agpr-alloc.mir      |  95 ++++
 .../bad-agpr-vgpr-regalloc-priority.mir       |  74 +++
 .../branch-folding-implicit-def-subreg.ll     |  46 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll  |   4 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |   4 +
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll  |   4 +-
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll  |   4 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-mov.ll     |  10 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-mov.mir    |  28 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-read.mir   |  26 +-
 ...al-regcopy-and-spill-missed-at-regalloc.ll |  20 +-
 .../AMDGPU/ptradd-sdag-optimizations.ll       | 196 ++++----
 .../CodeGen/AMDGPU/spill-vector-superclass.ll |   6 +-
 .../CodeGen/LoongArch/calling-conv-common.ll  |  48 +-
 .../CodeGen/LoongArch/calling-conv-half.ll    |  16 +-
 .../LoongArch/can-not-realign-stack.ll        |  44 +-
 .../CodeGen/LoongArch/emergency-spill-slot.ll |   4 +-
 llvm/test/CodeGen/LoongArch/frame.ll          | 107 ++++-
 .../CodeGen/LoongArch/intrinsic-memcpy.ll     |   8 +-
 llvm/test/CodeGen/LoongArch/lasx/fpowi.ll     |  88 ++--
 .../lasx/ir-instruction/extractelement.ll     | 120 ++---
 .../lasx/ir-instruction/insertelement.ll      | 132 +++---
 llvm/test/CodeGen/LoongArch/llvm.sincos.ll    | 150 +++---
 llvm/test/CodeGen/LoongArch/lsx/pr146455.ll   | 287 +++++++++++
 ...realignment-with-variable-sized-objects.ll |  24 +-
 .../CodeGen/LoongArch/stack-realignment.ll    |  80 ++--
 .../LoongArch/unaligned-memcpy-inline.ll      |  14 +-
 llvm/test/CodeGen/LoongArch/vararg.ll         |  70 +--
 llvm/test/CodeGen/NVPTX/i8x4-instructions.ll  |  47 ++
 llvm/test/CodeGen/NVPTX/trunc-tofp.ll         |  81 ++++
 .../CodeGen/RISCV/short-forward-branch-opt.ll |  32 +-
 llvm/test/CodeGen/RISCV/zdinx-spill.ll        |  26 -
 .../AddressSanitizer/lifetime.ll              | 447 +++++++++++++++---
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s |  45 ++
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s        |  45 ++
 .../MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s |  52 ++
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s  |  52 ++
 .../MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s  |  12 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s   |  12 +
 .../gfx1250_asm_vop3_from_vop1-fake16.s       |  36 ++
 .../MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s    |  36 ++
 .../gfx1250_asm_vop3_from_vop1_dpp16-fake16.s |  44 ++
 .../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s |  44 ++
 .../gfx1250_asm_vop3_from_vop1_dpp8-fake16.s  |   4 +
 .../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s  |   4 +
 .../Disassembler/AMDGPU/gfx1250_dasm_vop1.txt |  45 ++
 .../AMDGPU/gfx1250_dasm_vop1_dpp16.txt        |  39 ++
 .../AMDGPU/gfx1250_dasm_vop1_dpp8.txt         |   9 +
 .../AMDGPU/gfx1250_dasm_vop3_from_vop1.txt    |  36 ++
 .../gfx1250_dasm_vop3_from_vop1_dpp16.txt     |  33 ++
 .../gfx1250_dasm_vop3_from_vop1_dpp8.txt      |   7 +-
 .../DeadStoreElimination/zeroed-missing.ll    |   2 +-
 .../LoopInterchange/force-interchange.ll      |  43 ++
 .../LoopStrengthReduce/X86/lifetime-use.ll    |  59 +++
 .../AArch64/maxbandwidth-regpressure.ll       |  38 ++
 .../X86/split-node-reorder-node-with-ops.ll   |   2 -
 llvm/tools/llvm-dis/llvm-dis.cpp              |  14 -
 llvm/unittests/CodeGen/LexicalScopesTest.cpp  |   2 +-
 llvm/unittests/IR/DebugInfoTest.cpp           |   2 +
 .../Transforms/Utils/DebugifyTest.cpp         |   9 +-
 llvm/unittests/Transforms/Utils/LocalTest.cpp |  56 ---
 .../gn/secondary/libcxx/include/BUILD.gn      |   1 -
 .../secondary/llvm/lib/Target/AMDGPU/BUILD.gn |   1 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |   1 +
 .../secondary/llvm/tools/llvm-ir2vec/BUILD.gn |   9 +
 mlir/docs/Tutorials/transform/Ch2.md          |  30 +-
 .../Conversion/VectorToXeGPU/VectorToXeGPU.h  |   3 -
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |  63 ++-
 .../common/include/PluginInterface.h          |  28 +-
 offload/plugins-nextgen/cuda/src/rtl.cpp      |  10 +-
 .../unittests/OffloadAPI/common/Fixtures.hpp  |   3 -
 .../OffloadAPI/event/olWaitEvent.cpp          |   3 -
 275 files changed, 4637 insertions(+), 2846 deletions(-)
 create mode 100644 clang/test/Analysis/div-zero-cxx20.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
 create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
 create mode 100644 llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/trunc-tofp.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/force-interchange.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
 create mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn

diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index 59079f057d021..f76c69f29fb30 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -11,8 +11,6 @@ on:
       - .github/workflows/build-ci-container-windows.yml
       - '.github/workflows/containers/github-action-ci-windows/**'
   pull_request:
-    branches:
-      - main
     paths:
       - .github/workflows/build-ci-container-windows.yml
       - '.github/workflows/containers/github-action-ci-windows/**'
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 3159aae32ca51..7f01264af8534 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -11,8 +11,6 @@ on:
       - .github/workflows/build-ci-container.yml
       - '.github/workflows/containers/github-action-ci/**'
   pull_request:
-    branches:
-      - main
     paths:
       - .github/workflows/build-ci-container.yml
       - '.github/workflows/containers/github-action-ci/**'
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index fcd3887ec7a09..ea16029268dba 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -808,6 +808,7 @@ Bug Fixes in This Version
   nested scopes. (#GH147495)
 - Fixed a failed assertion with an operator call expression which comes from a
   macro expansion when performing analysis for nullability attributes. (#GH138371)
+- Fixed a concept equivalent checking crash due to untransformed constraint expressions. (#GH146614)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1206,6 +1207,8 @@ Static Analyzer
 ---------------
 - Fixed a crash when C++20 parenthesized initializer lists are used. This issue
   was causing a crash in clang-tidy. (#GH136041)
+- The Clang Static Analyzer now handles parenthesized initialization.
+  (#GH148875)
 
 New features
 ^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index ed51f1d5de447..a916af7e0c2df 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -673,6 +673,7 @@ TARGET_BUILTIN(__builtin_amdgcn_tanhf, "ff", "nc", "tanh-insts")
 TARGET_BUILTIN(__builtin_amdgcn_tanhh, "hh", "nc", "tanh-insts")
 TARGET_BUILTIN(__builtin_amdgcn_tanh_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_rcp_bf16, "yy", "nc", "bf16-trans-insts")
+TARGET_BUILTIN(__builtin_amdgcn_sqrt_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_rsq_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_log_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_exp2_bf16, "yy", "nc", "bf16-trans-insts")
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index a70335bef9dd4..d34a4146ddbd6 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -1491,8 +1491,6 @@ class Sema;
     OverloadingResult
     BestViableFunctionImpl(Sema &S, SourceLocation Loc,
                            OverloadCandidateSet::iterator &Best);
-    void PerfectViableFunction(Sema &S, SourceLocation Loc,
-                               OverloadCandidateSet::iterator &Best);
   };
 
   bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index 6370586e218ef..fbb34340a5c67 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -499,9 +499,6 @@ class ExprEngine {
   void VisitGuardedExpr(const Expr *Ex, const Expr *L, const Expr *R,
                         ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
-  void VisitInitListExpr(const InitListExpr *E, ExplodedNode *Pred,
-                         ExplodedNodeSet &Dst);
-
   /// VisitAttributedStmt - Transfer function logic for AttributedStmt.
   void VisitAttributedStmt(const AttributedStmt *A, ExplodedNode *Pred,
                            ExplodedNodeSet &Dst);
@@ -591,6 +588,10 @@ class ExprEngine {
                                 ExplodedNode *Pred,
                                 ExplodedNodeSet &Dst);
 
+  void ConstructInitList(const Expr *Source, ArrayRef<Expr *> Args,
+                         bool IsTransparent, ExplodedNode *Pred,
+                         ExplodedNodeSet &Dst);
+
   /// evalEagerlyAssumeBifurcation - Given the nodes in 'Src', eagerly assume
   /// concrete boolean values for 'Ex', storing the resulting nodes in 'Dst'.
   void evalEagerlyAssumeBifurcation(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index c89eca9bef440..5b9f44518fcc2 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -162,6 +162,8 @@ static void initField(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable,
   Desc->IsConst = IsConst || D->IsConst;
   Desc->IsFieldMutable = IsMutable || D->IsMutable;
   Desc->IsVolatile = IsVolatile || D->IsVolatile;
+  // True if this field is const AND the parent is mutable.
+  Desc->IsConstInMutable = Desc->IsConst && IsMutable;
 
   if (auto Fn = D->CtorFn)
     Fn(B, Ptr + FieldOffset, Desc->IsConst, Desc->IsFieldMutable,
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 4591eabb69bb4..0227e4c0c7e38 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -101,6 +101,10 @@ struct InlineDescriptor {
   /// Flag indicating if the field is mutable (if in a record).
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsFieldMutable : 1;
+  /// Flag indicating if this field is a const field nested in
+  /// a mutable parent field.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned IsConstInMutable : 1;
   /// Flag indicating if the field is an element of a composite array.
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsArrayElement : 1;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index f64501f4a31e8..74399d177b5a2 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -445,6 +445,7 @@ LLVM_DUMP_METHOD void InlineDescriptor::dump(llvm::raw_ostream &OS) const {
   OS << "InUnion: " << InUnion << "\n";
   OS << "IsFieldMutable: " << IsFieldMutable << "\n";
   OS << "IsArrayElement: " << IsArrayElement << "\n";
+  OS << "IsConstInMutable: " << IsConstInMutable << '\n';
   OS << "Desc: ";
   if (Desc)
     Desc->dump(OS);
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index e8b519478c026..df5e3be83d741 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -566,7 +566,10 @@ bool CheckDowncast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
 bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   assert(Ptr.isLive() && "Pointer is not live");
-  if (!Ptr.isConst() || Ptr.isMutable())
+  if (!Ptr.isConst())
+    return true;
+
+  if (Ptr.isMutable() && !Ptr.isConstInMutable())
     return true;
 
   if (!Ptr.isBlockPointer())
diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
index 7798b6f886a85..51622238e275c 100644
--- a/clang/lib/AST/ByteCode/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -14,11 +14,6 @@
 #define LLVM_CLANG_AST_INTERP_BLOCK_H
 
 #include "Descriptor.h"
-#include "clang/AST/ComparisonCategories.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclCXX.h"
-#include "clang/AST/Expr.h"
-#include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index de0b97fd93c76..462b9a11e0a5c 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 #include "../ExprConstShared.h"
 #include "Boolean.h"
-#include "Compiler.h"
 #include "EvalEmitter.h"
 #include "Interp.h"
 #include "InterpBuiltinBitCast.h"
@@ -2906,6 +2905,8 @@ static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,
         if (!copyField(F, /*Activate=*/true))
           return false;
       } else {
+        if (!CheckMutable(S, OpPC, Src.atField(F.Offset)))
+          return false;
         Pointer DestField = Dest.atField(F.Offset);
         zeroAll(DestField);
       }
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index a5a4bd25fe712..d62a4f6275b50 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -128,6 +128,11 @@ static bool shouldSkipInBacktrace(const Function *F) {
   if (FD->getDeclName().getCXXOverloadedOperator() == OO_New ||
       FD->getDeclName().getCXXOverloadedOperator() == OO_Array_New)
     return true;
+
+  if (const auto *MD = dyn_cast<CXXMethodDecl>(FD);
+      MD && MD->getParent()->isAnonymousStructOrUnion())
+    return true;
+
   return false;
 }
 
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index e6a64e6658f06..da74013cf83a6 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -576,6 +576,11 @@ class Pointer {
       return true;
     return isRoot() ? getDeclDesc()->IsConst : getInlineDesc()->IsConst;
   }
+  bool isConstInMutable() const {
+    if (!isBlockPointer())
+      return false;
+    return isRoot() ? false : getInlineDesc()->IsConstInMutable;
+  }
 
   /// Checks if an object or a subfield is volatile.
   bool isVolatile() const {
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index c8c3d6b20c496..19d8ba26d44d8 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2852,8 +2852,21 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
       if (AI.getInReg())
         Attrs.addAttribute(llvm::Attribute::InReg);
 
-      if (AI.getIndirectByVal())
+      // Depending on the ABI, this may be either a byval or a dead_on_return
+      // argument.
+      if (AI.getIndirectByVal()) {
         Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
+      } else {
+        // Add dead_on_return when the object's lifetime ends in the callee.
+        // This includes trivially-destructible objects, as well as objects
+        // whose destruction / clean-up is carried out within the callee (e.g.,
+        // Obj-C ARC-managed structs, MSVC callee-destroyed objects).
+        if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
+            ParamType->castAs<RecordType>()
+                ->getDecl()
+                ->isParamDestroyedInCallee())
+          Attrs.addAttribute(llvm::Attribute::DeadOnReturn);
+      }
 
       auto *Decl = ParamType->getAsRecordDecl();
       if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index a7d796ecccc61..ee736a2816218 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -416,6 +416,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_sqrt:
   case AMDGPU::BI__builtin_amdgcn_sqrtf:
   case AMDGPU::BI__builtin_amdgcn_sqrth:
+  case AMDGPU::BI__builtin_amdgcn_sqrt_bf16:
     return emitBuiltinWithOneOverloadedType<1>(*this, E,
                                                Intrinsic::amdgcn_sqrt);
   case AMDGPU::BI__builtin_amdgcn_rsq:
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..481f575518b93 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -258,10 +258,10 @@ static void getAArch64MultilibFlags(const Driver &D,
   processMultilibCustomFlags(Result, Args);
 }
 
-static void getARMMultilibFlags(const Driver &D,
-                                      const llvm::Triple &Triple,
-                                      const llvm::opt::ArgList &Args,
-                                      Multilib::flags_list &Result) {
+static void getARMMultilibFlags(const Driver &D, const llvm::Triple &Triple,
+                                llvm::Reloc::Model RelocationModel,
+                                const llvm::opt::ArgList &Args,
+                                Multilib::flags_list &Result) {
   std::vector<StringRef> Features;
   llvm::ARM::FPUKind FPUKind = tools::arm::getARMTargetFeatures(
       D, Triple, Args, Features, false /*ForAs*/, true /*ForMultilib*/);
@@ -304,6 +304,18 @@ static void getARMMultilibFlags(const Driver &D,
     llvm_unreachable("Invalid float ABI");
   }
 
+  if (RelocationModel == llvm::Reloc::ROPI ||
+      RelocationModel == llvm::Reloc::ROPI_RWPI)
+    Result.push_back("-fropi");
+  else
+    Result.push_back("-fno-ropi");
+
+  if (RelocationModel == llvm::Reloc::RWPI ||
+      RelocationModel == llvm::Reloc::ROPI_RWPI)
+    Result.push_back("-frwpi");
+  else
+    Result.push_back("-fno-rwpi");
+
   const Arg *BranchProtectionArg =
       Args.getLastArgNoClaim(options::OPT_mbranch_protection_EQ);
   if (BranchProtectionArg) {
@@ -344,6 +356,18 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   const llvm::Triple Triple(ComputeEffectiveClangTriple(Args));
   Result.push_back("--target=" + Triple.str());
 
+  // A difference of relocation model (absolutely addressed data, PIC, Arm
+  // ROPI/RWPI) is likely to change whether a particular multilib variant is
+  // compatible with a given link. Determine the relocation model of the
+  // current link, so as to add appropriate multilib flags.
+  llvm::Reloc::Model RelocationModel;
+  unsigned PICLevel;
+  bool IsPIE;
+  {
+    RegisterEffectiveTriple TripleRAII(*this, Triple);
+    std::tie(RelocationModel, PICLevel, IsPIE) = ParsePICArgs(*this, Args);
+  }
+
   switch (Triple.getArch()) {
   case llvm::Triple::aarch64:
   case llvm::Triple::aarch64_32:
@@ -354,7 +378,7 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   case llvm::Triple::armeb:
   case llvm::Triple::thumb:
   case llvm::Triple::thumbeb:
-    getARMMultilibFlags(D, Triple, Args, Result);
+    getARMMultilibFlags(D, Triple, RelocationModel, Args, Result);
     break;
   case llvm::Triple::riscv32:
   case llvm::Triple::riscv64:
@@ -376,6 +400,12 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   else
     Result.push_back("-fexceptions");
 
+  if (RelocationModel == llvm::Reloc::PIC_)
+    Result.push_back(IsPIE ? (PICLevel > 1 ? "-fPIE" : "-fpie")
+                           : (PICLevel > 1 ? "-fPIC" : "-fpic"));
+  else
+    Result.push_back("-fno-pic");
+
   // Sort and remove duplicates.
   std::sort(Result.begin(), Result.end());
   Result.erase(llvm::unique(Result), Result.end());
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 834417f8e15ac..5205ca0bca6fa 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -925,7 +925,12 @@ static const Expr *SubstituteConstraintExpressionWithoutSatisfaction(
       ND && ND->isFunctionOrFunctionTemplate()) {
     ScopeForParameters.emplace(S, /*CombineWithOuterScope=*/true);
     const FunctionDecl *FD = ND->getAsFunction();
+    if (FunctionTemplateDecl *Template = FD->getDescribedFunctionTemplate();
+        Template && Template->getInstantiatedFromMemberTemplate())
+      FD = Template->getInstantiatedFromMemberTemplate()->getTemplatedDecl();
     for (auto *PVD : FD->parameters()) {
+      if (ScopeForParameters->getInstantiationOfIfExists(PVD))
+        continue;
       if (!PVD->isParameterPack()) {
         ScopeForParameters->InstantiatedLocal(PVD, PVD);
         continue;
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 1b54628c5e564..5dd5b495480d9 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -11354,55 +11354,18 @@ OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
       DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
 
   if (TwoPhaseResolution) {
-
-    PerfectViableFunction(S, Loc, Best);
-    if (Best != end())
-      return ResultForBestCandidate(Best);
+    OverloadingResult Res = BestViableFunctionImpl(S, Loc, Best);
+    if (Best != end() && Best->isPerfectMatch(S.Context)) {
+      if (!(HasDeferredTemplateConstructors &&
+            isa_and_nonnull<CXXConversionDecl>(Best->Function)))
+        return Res;
+    }
   }
 
   InjectNonDeducedTemplateCandidates(S);
   return BestViableFunctionImpl(S, Loc, Best);
 }
 
-void OverloadCandidateSet::PerfectViableFunction(
-    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
-
-  Best = end();
-  for (auto It = Candidates.begin(); It != Candidates.end(); ++It) {
-
-    if (!It->isPerfectMatch(S.getASTContext()))
-      continue;
-
-    // We found a suitable conversion function
-    // but if there is a template constructor in the target class
-    // we might prefer that instead.
-    if (HasDeferredTemplateConstructors &&
-        isa_and_nonnull<CXXConversionDecl>(It->Function)) {
-      Best = end();
-      break;
-    }
-
-    if (Best == end()) {
-      Best = It;
-      continue;
-    }
-    if (Best->Function && It->Function) {
-      FunctionDecl *D =
-          S.getMoreConstrainedFunction(Best->Function, It->Function);
-      if (D == nullptr) {
-        Best = end();
-        break;
-      }
-      if (D == It->Function)
-        Best = It;
-      continue;
-    }
-    // ambiguous
-    Best = end();
-    break;
-  }
-}
-
 OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
     Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
 
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index c77ef26da568d..d87484470f8b5 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1941,7 +1941,6 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::ConceptSpecializationExprClass:
     case Stmt::CXXRewrittenBinaryOperatorClass:
     case Stmt::RequiresExprClass:
-    case Expr::CXXParenListInitExprClass:
     case Stmt::EmbedExprClass:
       // Fall through.
 
@@ -2315,11 +2314,22 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
       break;
     }
 
-    case Stmt::InitListExprClass:
+    case Stmt::InitListExprClass: {
+      const InitListExpr *E = cast<InitListExpr>(S);
       Bldr.takeNodes(Pred);
-      VisitInitListExpr(cast<InitListExpr>(S), Pred, Dst);
+      ConstructInitList(E, E->inits(), E->isTransparent(), Pred, Dst);
       Bldr.addNodes(Dst);
       break;
+    }
+
+    case Expr::CXXParenListInitExprClass: {
+      const CXXParenListInitExpr *E = cast<CXXParenListInitExpr>(S);
+      Bldr.takeNodes(Pred);
+      ConstructInitList(E, E->getInitExprs(), /*IsTransparent*/ false, Pred,
+                        Dst);
+      Bldr.addNodes(Dst);
+      break;
+    }
 
     case Stmt::MemberExprClass:
       Bldr.takeNodes(Pred);
@@ -4114,3 +4124,33 @@ void *ProgramStateTrait<ReplayWithoutInlining>::GDMIndex() {
 }
 
 void ExprEngine::anchor() { }
+
+void ExprEngine::ConstructInitList(const Expr *E, ArrayRef<Expr *> Args,
+                                   bool IsTransparent, ExplodedNode *Pred,
+                                   ExplodedNodeSet &Dst) {
+  assert((isa<InitListExpr, CXXParenListInitExpr>(E)));
+
+  const LocationContext *LC = Pred->getLocationContext();
+
+  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
+  ProgramStateRef S = Pred->getState();
+  QualType T = E->getType().getCanonicalType();
+
+  bool IsCompound = T->isArrayType() || T->isRecordType() ||
+                    T->isAnyComplexType() || T->isVectorType();
+
+  if (Args.size() > 1 || (E->isPRValue() && IsCompound && !IsTransparent)) {
+    llvm::ImmutableList<SVal> ArgList = getBasicVals().getEmptySValList();
+    for (Expr *E : llvm::reverse(Args))
+      ArgList = getBasicVals().prependSVal(S->getSVal(E, LC), ArgList);
+
+    B.generateNode(E, Pred,
+                   S->BindExpr(E, LC, svalBuilder.makeCompoundVal(T, ArgList)));
+  } else {
+    B.generateNode(E, Pred,
+                   S->BindExpr(E, LC,
+                               Args.size() == 0
+                                   ? getSValBuilder().makeZeroVal(T)
+                                   : S->getSVal(Args.front(), LC)));
+  }
+}
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index fa8e669b6bb2f..f1a25a750dd0d 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -771,54 +771,6 @@ void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
   Bldr.generateNode(B, Pred, state->BindExpr(B, Pred->getLocationContext(), X));
 }
 
-void ExprEngine::VisitInitListExpr(const InitListExpr *IE,
-                                   ExplodedNode *Pred,
-                                   ExplodedNodeSet &Dst) {
-  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
-
-  ProgramStateRef state = Pred->getState();
-  const LocationContext *LCtx = Pred->getLocationContext();
-  QualType T = getContext().getCanonicalType(IE->getType());
-  unsigned NumInitElements = IE->getNumInits();
-
-  if (!IE->isGLValue() && !IE->isTransparent() &&
-      (T->isArrayType() || T->isRecordType() || T->isVectorType() ||
-       T->isAnyComplexType())) {
-    llvm::ImmutableList<SVal> vals = getBasicVals().getEmptySValList();
-
-    // Handle base case where the initializer has no elements.
-    // e.g: static int* myArray[] = {};
-    if (NumInitElements == 0) {
-      SVal V = svalBuilder.makeCompoundVal(T, vals);
-      B.generateNode(IE, Pred, state->BindExpr(IE, LCtx, V));
-      return;
-    }
-
-    for (const Stmt *S : llvm::reverse(*IE)) {
-      SVal V = state->getSVal(cast<Expr>(S), LCtx);
-      vals = getBasicVals().prependSVal(V, vals);
-    }
-
-    B.generateNode(IE, Pred,
-                   state->BindExpr(IE, LCtx,
-                                   svalBuilder.makeCompoundVal(T, vals)));
-    return;
-  }
-
-  // Handle scalars: int{5} and int{} and GLvalues.
-  // Note, if the InitListExpr is a GLvalue, it means that there is an address
-  // representing it, so it must have a single init element.
-  assert(NumInitElements <= 1);
-
-  SVal V;
-  if (NumInitElements == 0)
-    V = getSValBuilder().makeZeroVal(T);
-  else
-    V = state->getSVal(IE->getInit(0), LCtx);
-
-  B.generateNode(IE, Pred, state->BindExpr(IE, LCtx, V));
-}
-
 void ExprEngine::VisitGuardedExpr(const Expr *Ex,
                                   const Expr *L,
                                   const Expr *R,
diff --git a/clang/test/AST/ByteCode/mutable.cpp b/clang/test/AST/ByteCode/mutable.cpp
index aebbea920578c..35c5a0389921e 100644
--- a/clang/test/AST/ByteCode/mutable.cpp
+++ b/clang/test/AST/ByteCode/mutable.cpp
@@ -1,11 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++11 -verify=expected,expected11,both,both11 %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++14 -verify=expected,expected14,both %s
-// RUN: %clang_cc1 -std=c++11 -verify=ref,ref11,both,both11 %s
-// RUN: %clang_cc1 -std=c++14 -verify=ref,ref14,both %s
-
-
-
-
+// RUN: %clang_cc1 -std=c++11 -verify=expected,expected11,both,both11 %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++14 -verify=expected,expected14,both        %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++11 -verify=ref,ref11,both,both11           %s
+// RUN: %clang_cc1 -std=c++14 -verify=ref,ref14,both                  %s
 
 namespace Simple {
   struct S {
@@ -26,3 +22,47 @@ namespace Simple {
   static_assert(s2.a2 == 12, ""); // both11-error {{not an integral constant expression}} \
                                   // both11-note {{initializer of 's2' is not a constant expression}}
 }
+#if __cplusplus >= 201402L
+namespace ConstInMutable {
+  class B {
+    public:
+
+    const int f;
+    constexpr B() : f(12) {}
+  };
+  class A {
+    public:
+    mutable B b;
+    constexpr A() = default;
+  };
+  constexpr int constInMutable() {
+    A a;
+
+    int *m = (int*)&a.b.f;
+    *m = 12; // both-note {{modification of object of const-qualified type 'const int' is not allowed in a constant expression}}
+    return 1;
+  }
+  static_assert(constInMutable() == 1, ""); // both-error {{not an integral constant expression}} \
+                                            // both-note {{in call to}}
+}
+
+namespace MutableInConst {
+  class C {
+  public:
+    mutable int c;
+    constexpr C() : c(50) {}
+  };
+  class D {
+  public:
+    C c;
+    constexpr D() {}
+  };
+  constexpr int mutableInConst() {
+    const D d{};
+    int *m = (int*)&d.c.c;
+    *m = 12;
+    return 1;
+  }
+  static_assert(mutableInConst() == 1, "");
+}
+#endif
diff --git a/clang/test/AST/ByteCode/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
index 0fa44a259a4ff..7cfd0d677a7b3 100644
--- a/clang/test/AST/ByteCode/unions.cpp
+++ b/clang/test/AST/ByteCode/unions.cpp
@@ -847,6 +847,20 @@ namespace Activation2 {
   }
   static_assert(change_member_indirectly() == 4);
 }
+
+namespace CopyCtorMutable {
+  struct E {
+    union { // expected-note {{read of mutable member 'b'}}
+      int a;
+      mutable int b; // both-note {{here}}
+    };
+  };
+  constexpr E e1 = {{1}};
+  constexpr E e2 = e1; // both-error {{constant}} \
+                       // ref-note {{read of mutable member 'b'}} \
+                       // both-note {{in call}}
+}
+
 #endif
 
 namespace AddressComparison {
diff --git a/clang/test/Analysis/div-zero-cxx20.cpp b/clang/test/Analysis/div-zero-cxx20.cpp
new file mode 100644
index 0000000000000..00ea96e796777
--- /dev/null
+++ b/clang/test/Analysis/div-zero-cxx20.cpp
@@ -0,0 +1,61 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core.DivideZero -std=c++20 -verify %s
+
+namespace GH148875 {
+struct A {
+  int x;
+  A(int v) : x(v) {}
+};
+
+struct B {
+  int x;
+  B() : x(0) {}
+};
+
+struct C {
+  int x, y;
+  C(int a, int b) : x(a), y(b) {}
+};
+
+struct D {
+  int x;
+};
+
+struct E {
+  D d;
+  E(int a) : d(a) {}
+};
+
+struct F {
+  int x;
+};
+
+int t1() {
+  A a{42};
+  return 1 / (a.x - 42); // expected-warning {{Division by zero}}
+}
+
+int t2() {
+  B b{};
+  return 1 / b.x; // expected-warning {{Division by zero}}
+}
+
+int t3() {
+  C c1{1, -1};
+  return 1 / (c1.x + c1.y); // expected-warning {{Division by zero}}
+}
+
+int t4() {
+  C c2{0, 0};
+  return 1 / (c2.x + c2.y); // expected-warning {{Division by zero}}
+}
+
+int t5() {
+  E e{32};
+  return 1 / (e.d.x - 32); // expected-warning {{Division by zero}}
+}
+
+int t6() {
+  F f(32);
+  return 1 / (f.x - 32); // expected-warning {{Division by zero}}
+}
+} // namespace GH148875
diff --git a/clang/test/Analysis/div-zero.cpp b/clang/test/Analysis/div-zero.cpp
index 063450d8883b0..51ea25e828a18 100644
--- a/clang/test/Analysis/div-zero.cpp
+++ b/clang/test/Analysis/div-zero.cpp
@@ -11,3 +11,63 @@ int fooPR10616 (int qX ) {
   return (a % (qX-1)); // expected-warning {{Division by zero}}
 
 }
+
+namespace GH148875 {
+struct A {
+  int x;
+  A(int v) : x(v) {}
+};
+
+struct B {
+  int x;
+  B() : x(0) {}
+};
+
+struct C {
+  int x, y;
+  C(int a, int b) : x(a), y(b) {}
+};
+
+struct D {
+  int x;
+};
+
+struct E {
+  D d;
+  E(int a) : d{a} {}
+};
+
+struct F {
+  int x;
+};
+
+int t1() {
+  A a{42};
+  return 1 / (a.x - 42); // expected-warning {{Division by zero}}
+}
+
+int t2() {
+  B b{};
+  return 1 / b.x; // expected-warning {{Division by zero}}
+}
+
+int t3() {
+  C c1{1, -1};
+  return 1 / (c1.x + c1.y); // expected-warning {{Division by zero}}
+}
+
+int t4() {
+  C c2{0, 0};
+  return 1 / (c2.x + c2.y); // expected-warning {{Division by zero}}
+}
+
+int t5() {
+  E e{32};
+  return 1 / (e.d.x - 32); // expected-warning {{Division by zero}}
+}
+
+int t6() {
+  F f{32};
+  return 1 / (f.x - 32); // expected-warning {{Division by zero}}
+}
+}
diff --git a/clang/test/CodeGen/64bit-swiftcall.c b/clang/test/CodeGen/64bit-swiftcall.c
index 7f8aa02d97ce1..448bca7acbca3 100644
--- a/clang/test/CodeGen/64bit-swiftcall.c
+++ b/clang/test/CodeGen/64bit-swiftcall.c
@@ -239,7 +239,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define {{.*}} void @return_struct_big_1(ptr dead_on_unwind noalias writable sret
 
 // Should not be byval.
-// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr{{( %.*)?}})
+// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr dead_on_return{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGen/AArch64/byval-temp.c b/clang/test/CodeGen/AArch64/byval-temp.c
index 0ee0312b2362d..5033b6cf5ac03 100644
--- a/clang/test/CodeGen/AArch64/byval-temp.c
+++ b/clang/test/CodeGen/AArch64/byval-temp.c
@@ -30,10 +30,10 @@ void example(void) {
 // Then, memcpy `l` to the temporary stack space.
 // CHECK-O0-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
 // Finally, call using a pointer to the temporary stack space.
-// CHECK-O0-NEXT: call void @pass_large(ptr noundef %[[byvaltemp]])
+// CHECK-O0-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
 // Now, do the same for the second call, using the second temporary alloca.
 // CHECK-O0-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
-// CHECK-O0-NEXT: call void @pass_large(ptr noundef %[[byvaltemp1]])
+// CHECK-O0-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
 // CHECK-O0-NEXT: ret void
 //
 // At O3, we should have lifetime markers to help the optimizer re-use the temporary allocas.
@@ -58,7 +58,7 @@ void example(void) {
 // Then, memcpy `l` to the temporary stack space.
 // CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
 // Finally, call using a pointer to the temporary stack space.
-// CHECK-O3-NEXT: call void @pass_large(ptr noundef %[[byvaltemp]])
+// CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
 //
 // The lifetime of the temporary used to pass a pointer to the struct ends here.
 // CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp]])
@@ -66,7 +66,7 @@ void example(void) {
 // Now, do the same for the second call, using the second temporary alloca.
 // CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %[[byvaltemp1]])
 // CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
-// CHECK-O3-NEXT: call void @pass_large(ptr noundef %[[byvaltemp1]])
+// CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
 // CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp1]])
 //
 // Mark the end of the lifetime of `l`.
@@ -88,12 +88,12 @@ void example_BitInt(void) {
 // CHECK-O0-NEXT:    [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
 // CHECK-O0-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
 // CHECK-O0-NEXT:    store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16
-// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP]])
+// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
 // CHECK-O0-NEXT:    [[TMP1:%.*]] = load i256, ptr [[L]], align 16
 // CHECK-O0-NEXT:    [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
 // CHECK-O0-NEXT:    [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
 // CHECK-O0-NEXT:    store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16
-// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP1]])
+// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
 // CHECK-O0-NEXT:    ret void
 //
 // CHECK-O3-LABEL: define dso_local void @example_BitInt(
@@ -108,13 +108,13 @@ void example_BitInt(void) {
 // CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]]) 
 // CHECK-O3-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
 // CHECK-O3-NEXT:    store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16, !tbaa [[TBAA6]]
-// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP]])
+// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]]) 
 // CHECK-O3-NEXT:    [[TMP1:%.*]] = load i256, ptr [[L]], align 16, !tbaa [[TBAA6]]
 // CHECK-O3-NEXT:    [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
 // CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]]) 
 // CHECK-O3-NEXT:    [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
 // CHECK-O3-NEXT:    store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16, !tbaa [[TBAA6]]
-// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP1]])
+// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]]) 
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[L]]) 
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c b/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
index 546910068c78a..804e14a2ea34b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
@@ -19,7 +19,7 @@ void f0(S0 *p) {
   use0(*p);
 }
 // CHECK-C:   declare void @use0(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
-// CHECK-CXX: declare void @use0(ptr noundef)
+// CHECK-CXX: declare void @use0(ptr dead_on_return noundef)
 
 #ifdef __cplusplus
 
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args.c b/clang/test/CodeGen/AArch64/pure-scalable-args.c
index fecd370d09be3..48988f7a1722b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args.c
@@ -92,7 +92,7 @@ void test_argpass_simple(PST *p) {
 // CHECK-AAPCS-NEXT: ret void
 
 // CHECK-AAPCS:  declare void @argpass_simple_callee(<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_simple_callee(ptr noundef)
+// CHECK-DARWIN: declare void @argpass_simple_callee(ptr dead_on_return noundef)
 
 // Boundary case of using the last available Z-reg, PST expanded.
 //   0.0  -> d0-d3
@@ -107,7 +107,7 @@ void test_argpass_last_z(PST *p) {
     argpass_last_z_callee(.0, .0, .0, .0, *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr dead_on_return noundef)
 
 
 // Like the above, but using a tuple type to occupy some registers.
@@ -123,7 +123,7 @@ void test_argpass_last_z_tuple(PST *p, svfloat64x4_t x) {
   argpass_last_z_tuple_callee(x, *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr dead_on_return noundef)
 
 
 // Boundary case of using the last available P-reg, PST expanded.
@@ -139,7 +139,7 @@ void test_argpass_last_p(PST *p) {
     argpass_last_p_callee(svpfalse(), svpfalse_c(), *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr dead_on_return noundef)
 
 
 // Not enough Z-regs, push PST to memory and pass a pointer, Z-regs and
@@ -157,7 +157,7 @@ void test_argpass_no_z(PST *p, double dummy, svmfloat8_t u, int8x16_t v, mfloat8
     void argpass_no_z_callee(svmfloat8_t, int8x16_t, mfloat8x16_t, double, double, int, PST, int, double, svbool_t);
     argpass_no_z_callee(u, v, w, .0, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Like the above, using a tuple to occupy some registers.
@@ -173,7 +173,7 @@ void test_argpass_no_z_tuple_f64(PST *p, float dummy, svfloat64x4_t x) {
                                      double, svbool_t);
   argpass_no_z_tuple_f64_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Likewise, using a different tuple.
@@ -189,7 +189,7 @@ void test_argpass_no_z_tuple_mfp8(PST *p, float dummy, svmfloat8x4_t x) {
                                       double, svbool_t);
   argpass_no_z_tuple_mfp8_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Not enough Z-regs (consumed by a HFA), PST passed indirectly
@@ -204,8 +204,8 @@ void test_argpass_no_z_hfa(HFA *h, PST *p) {
     void argpass_no_z_hfa_callee(double, HFA, int, PST, int, svbool_t);
     argpass_no_z_hfa_callee(.0, *h, 1, *p, 2, svptrue_b64());
 }
-// CHECK-AAPCS:  declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
 
 // Not enough Z-regs (consumed by a HVA), PST passed indirectly
 //   0.0  -> d0
@@ -219,8 +219,8 @@ void test_argpass_no_z_hva(HVA *h, PST *p) {
     void argpass_no_z_hva_callee(double, HVA, int, PST, int, svbool_t);
     argpass_no_z_hva_callee(.0, *h, 1, *p, 2, svptrue_b64());
 }
-// CHECK-AAPCS:  declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
 
 // Not enough P-regs, PST passed indirectly, Z-regs and P-regs still available.
 //   true -> p0-p2
@@ -233,7 +233,7 @@ void test_argpass_no_p(PST *p) {
     void argpass_no_p_callee(svbool_t, svbool_t, svbool_t, int, PST, int, double, svbool_t);
     argpass_no_p_callee(svptrue_b8(), svptrue_b16(), svptrue_b32(), 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Like above, using a tuple to occupy some registers.
@@ -250,7 +250,7 @@ void test_argpass_no_p_tuple(PST *p, svbool_t u, svboolx2_t v) {
                                  svbool_t);
   argpass_no_p_tuple_callee(v, u, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // HFAs go back-to-back to memory, afterwards Z-regs not available, PST passed indirectly.
@@ -263,8 +263,8 @@ void test_after_hfa(HFA *h, PST *p) {
     void after_hfa_callee(double, double, double, double, double, HFA, PST, HFA, svbool_t);
     after_hfa_callee(.0, .0, .0, .0, .0, *h, *p, *h, svpfalse());
 }
-// CHECK-AAPCS:  declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr noundef, [4 x float], <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr dead_on_return noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr dead_on_return noundef, [4 x float], <vscale x 16 x i1>)
 
 // Small PST, not enough registers, passed indirectly, unlike other small
 // aggregates.
@@ -277,7 +277,7 @@ void test_small_pst(SmallPST *p, SmallAgg *s) {
     void small_pst_callee(SmallAgg, double, double, double, double, double, double, double, double, double, SmallPST, double);
     small_pst_callee(*s, .0, .0, .0, .0, .0, .0, .0, .0, 1.0, *p, 2.0);
 }
-// CHECK-AAPCS:  declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr noundef, double noundef)
+// CHECK-AAPCS:  declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr dead_on_return noundef, double noundef)
 // CHECK-DARWIN: declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, i128, double noundef)
 
 
@@ -326,12 +326,12 @@ void test_pass_variadic(PST *p, PST *q) {
     pass_variadic_callee(*p, *q);
 }
 // CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
-// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr noundef nonnull %byval-temp)
+// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr dead_on_return noundef nonnull %byval-temp)
 
 // CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %p, i64 96, i1 false)
 // CHECK-DARWIN: call void @llvm.lifetime.start.p0(i64 96, ptr nonnull %byval-temp1)
 // CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp1, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
-// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr noundef nonnull %byval-temp, ptr noundef nonnull %byval-temp1)
+// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr dead_on_return noundef nonnull %byval-temp, ptr dead_on_return noundef nonnull %byval-temp1)
 
 
 // Test passing a small PST, still passed indirectly, despite being <= 128 bits
@@ -340,7 +340,7 @@ void test_small_pst_variadic(SmallPST *p) {
     small_pst_variadic_callee(0, *p);
 }
 // CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) %byval-temp, ptr noundef nonnull align 16 dereferenceable(16) %p, i64 16, i1 false)
-// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr noundef nonnull %byval-temp)
+// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr dead_on_return noundef nonnull %byval-temp)
 
 // CHECK-DARWIN: %0 = load i128, ptr %p, align 16
 // CHECK-DARWIN: tail call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, i128 %0)
@@ -467,7 +467,7 @@ void test_tuple_reg_count(svfloat32_t x, svfloat32x2_t y) {
                                    svfloat32_t, svfloat32_t, svfloat32_t, svfloat32x2_t);
   test_tuple_reg_count_callee(x, x, x, x, x, x, x, y);
 }
-// CHECK-AAPCS: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, ptr noundef)
+// CHECK-AAPCS: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, ptr dead_on_return noundef)
 // CHECK-DARWIN: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
 
 // Regression test for incorrect passing of SVE vector tuples
@@ -476,5 +476,5 @@ void test_tuple_reg_count_bool(svboolx4_t x, svboolx4_t y) {
   void test_tuple_reg_count_bool_callee(svboolx4_t, svboolx4_t);
   test_tuple_reg_count_bool_callee(x, y);
 }
-// CHECK-AAPCS:  declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, ptr noundef)
+// CHECK-AAPCS:  declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, ptr dead_on_return noundef)
 // CHECK-DARWIN: declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
diff --git a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
index b1232921df363..f0c9ef28201a5 100644
--- a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
+++ b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
@@ -82,7 +82,7 @@ struct Sppp {
     int *x, *y, *z;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z4Tppp4Sppp(
-// CHECK-A64-SAME: ptr noundef [[S:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr dead_on_return noundef [[S:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-A64-NEXT:    store ptr [[S]], ptr [[S_INDIRECT_ADDR]], align 8
@@ -490,7 +490,7 @@ struct Spa3 {
     int* xs[3];
 };
 // CHECK-A64-LABEL: define dso_local void @_Z4Tpa34Spa3(
-// CHECK-A64-SAME: ptr noundef [[S:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr dead_on_return noundef [[S:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-A64-NEXT:    store ptr [[S]], ptr [[S_INDIRECT_ADDR]], align 8
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
index c3d0541229fac..d244a8ba88572 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
@@ -58,7 +58,7 @@ typedef int8_t vec_int8 __attribute__((vector_size(N / 8)));
 // CHECK128-NEXT:    ret <16 x i8> [[CASTFIXEDSVE]]
 
 // CHECK-LABEL: define{{.*}} void @f2(
-// CHECK-SAME:   ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr noundef readonly captures(none) %0)
+// CHECK-SAME:   ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr dead_on_return noundef readonly captures(none) %0)
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   [[X:%.*]] = load <[[#div(VBITS,8)]] x i8>, ptr [[TMP0:%.*]], align 16, [[TBAA6:!tbaa !.*]]
 // CHECK-NEXT:   [[CASTSCALABLESVE:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v[[#div(VBITS,8)]]i8(<vscale x 16 x i8> poison, <[[#div(VBITS,8)]] x i8> [[X]], i64 0)
@@ -88,13 +88,13 @@ typedef svint8_t vec2 __attribute__((arm_sve_vector_bits(N)));
 // CHECK-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS,8)]] x i8> @llvm.vector.extract.v[[#div(VBITS,8)]]i8.nxv16i8(<vscale x 16 x i8> [[X_COERCE:%.*]], i64 0)
 // CHECK-NEXT:   call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:   store <[[#div(VBITS,8)]] x i8> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6]]
-// CHECK-NEXT:   call void @f3(ptr noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
+// CHECK-NEXT:   call void @f3(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
 // CHECK-NEXT:   call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:   ret void
 
 // CHECK128-LABEL: declare void @f3(<16 x i8> noundef)
 
 // CHECK-LABEL: declare void @f3(
-// CHECK-SAME:   ptr noundef)
+// CHECK-SAME:   ptr dead_on_return noundef)
 void g(vec2 x) { f3(x); } // OK
 #endif
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
index e82069aab2486..d42ecb663050f 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
@@ -75,7 +75,7 @@ typedef svint16_t vec2 __attribute__((arm_sve_vector_bits(N)));
 // CHECKWIDE-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS, 16)]] x i16> @llvm.vector.extract.v[[#div(VBITS, 16)]]i16.nxv8i16(<vscale x 8 x i16> [[X_COERCE:%.*]], i64 0)
 // CHECKWIDE-NEXT:   call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECKWIDE-NEXT:   store <[[#div(VBITS, 16)]] x i16> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6:!tbaa !.*]]
-// CHECKWIDE-NEXT:   call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
+// CHECKWIDE-NEXT:   call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
 // CHECKWIDE-NEXT:   call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECKWIDE-NEXT:   ret void
 void g(vec2 x) { f(x); } // OK
diff --git a/clang/test/CodeGen/LoongArch/bitint.c b/clang/test/CodeGen/LoongArch/bitint.c
index f346f569d0eb0..950f5308e32cc 100644
--- a/clang/test/CodeGen/LoongArch/bitint.c
+++ b/clang/test/CodeGen/LoongArch/bitint.c
@@ -26,12 +26,12 @@ void pass_BitInt129(_BitInt(129));
 // LA32-NEXT:    [[LOADEDV1:%.*]] = trunc i128 [[TMP1]] to i65
 // LA32-NEXT:    [[STOREDV:%.*]] = sext i65 [[LOADEDV1]] to i128
 // LA32-NEXT:    store i128 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
-// LA32-NEXT:    call void @pass_BitInt65(ptr noundef [[BYVAL_TEMP]])
+// LA32-NEXT:    call void @pass_BitInt65(ptr dead_on_return noundef [[BYVAL_TEMP]])
 // LA32-NEXT:    [[TMP2:%.*]] = load i256, ptr [[L129]], align 16
 // LA32-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
 // LA32-NEXT:    [[STOREDV4:%.*]] = sext i129 [[LOADEDV2]] to i256
 // LA32-NEXT:    store i256 [[STOREDV4]], ptr [[BYVAL_TEMP3]], align 16
-// LA32-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP3]])
+// LA32-NEXT:    call void @pass_BitInt129(ptr dead_on_return noundef [[BYVAL_TEMP3]])
 // LA32-NEXT:    ret void
 //
 // LA64-LABEL: define dso_local void @example_BitInt(
@@ -54,7 +54,7 @@ void pass_BitInt129(_BitInt(129));
 // LA64-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
 // LA64-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV2]] to i256
 // LA64-NEXT:    store i256 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
-// LA64-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP]])
+// LA64-NEXT:    call void @pass_BitInt129(ptr dead_on_return noundef [[BYVAL_TEMP]])
 // LA64-NEXT:    ret void
 //
 void example_BitInt(void) {
diff --git a/clang/test/CodeGen/PowerPC/ppc64-vector.c b/clang/test/CodeGen/PowerPC/ppc64-vector.c
index 5d3dd86a009d5..2e99781f84910 100644
--- a/clang/test/CodeGen/PowerPC/ppc64-vector.c
+++ b/clang/test/CodeGen/PowerPC/ppc64-vector.c
@@ -39,7 +39,7 @@ v8i16 test_v8i16(v8i16 x)
   return x;
 }
 
-// CHECK: define{{.*}} void @test_v16i16(ptr dead_on_unwind noalias writable sret(<16 x i16>) align 32 %agg.result, ptr noundef %0)
+// CHECK: define{{.*}} void @test_v16i16(ptr dead_on_unwind noalias writable sret(<16 x i16>) align 32 %agg.result, ptr dead_on_return noundef %0)
 v16i16 test_v16i16(v16i16 x)
 {
   return x;
diff --git a/clang/test/CodeGen/RISCV/riscv-abi.cpp b/clang/test/CodeGen/RISCV/riscv-abi.cpp
index fe1a2b6d8595c..d2e080829e72f 100644
--- a/clang/test/CodeGen/RISCV/riscv-abi.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-abi.cpp
@@ -75,7 +75,7 @@ struct child3_int64_s : parent3_float_s {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @_Z30float_int64_struct_inheritance14child3_int64_s
-// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD3_INT64_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD3_INT64_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 // LP64-LABEL: define dso_local [2 x i64] @_Z30float_int64_struct_inheritance14child3_int64_s
@@ -99,7 +99,7 @@ struct child4_double_s : parent4_double_s {
 };
 
 // ILP32-ILP32F-LABEL: define dso_local void @_Z32double_double_struct_inheritance15child4_double_s
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD4_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD4_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @_Z32double_double_struct_inheritance15child4_double_s
@@ -130,11 +130,11 @@ struct child5_virtual_s : virtual parent5_virtual_s {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @_Z38int32_float_virtual_struct_inheritance16child5_virtual_s
-// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 // LP64-LP64F-LP64D-LABEL: define dso_local void @_Z38int32_float_virtual_struct_inheritance16child5_virtual_s
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct child5_virtual_s int32_float_virtual_struct_inheritance(struct child5_virtual_s a) {
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
index 82e43fff0c3aa..bc89cb532bdcc 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
@@ -138,7 +138,7 @@ struct st_i32x4x9 {
 
 typedef int __attribute__((vector_size(256))) int32x64_t;
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_too_large(ptr noundef %0)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_too_large(ptr dead_on_return noundef %0)
 void __attribute__((riscv_vls_cc)) test_too_large(int32x64_t arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_too_large_256(<vscale x 16 x i32> noundef %arg.coerce)
 void __attribute__((riscv_vls_cc(256))) test_too_large_256(int32x64_t arg) {}
@@ -173,9 +173,9 @@ void __attribute__((riscv_vls_cc)) test_st_i32x8x2(struct st_i32x8x2 arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x8x2_256(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc)) test_st_i32x64x2(struct st_i32x64x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
@@ -188,7 +188,7 @@ void __attribute__((riscv_vls_cc)) test_st_i32x4x8(struct st_i32x4x8 arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x8_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc)) test_st_i32x4x9(struct st_i32x4x9 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
index 5f6539796c20d..128610e578c26 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
@@ -118,7 +118,7 @@ struct st_i32x4x9 {
 
 typedef int __attribute__((vector_size(256))) int32x64_t;
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z14test_too_largeDv64_i(ptr noundef %0)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z14test_too_largeDv64_i(ptr dead_on_return noundef %0)
 [[riscv::vls_cc]] void test_too_large(int32x64_t arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z18test_too_large_256Dv64_i(<vscale x 16 x i32> noundef %arg.coerce)
 [[riscv::vls_cc(256)]] void test_too_large_256(int32x64_t arg) {}
@@ -153,9 +153,9 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x8x2_25610st_i32x8x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc]] void test_st_i32x64x2(struct st_i32x64x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x310st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
@@ -168,7 +168,7 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x8_25610st_i32x4x8(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc]] void test_st_i32x4x9(struct st_i32x4x9 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x9_25610st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x9_25610st_i32x4x9(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
diff --git a/clang/test/CodeGen/RISCV/riscv32-abi.c b/clang/test/CodeGen/RISCV/riscv32-abi.c
index b53f9a9169146..a9e56d40817ae 100644
--- a/clang/test/CodeGen/RISCV/riscv32-abi.c
+++ b/clang/test/CodeGen/RISCV/riscv32-abi.c
@@ -246,7 +246,7 @@ struct large {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_agg_large
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[X:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[X:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_agg_large(struct large x) {
@@ -266,7 +266,7 @@ struct large f_agg_large_ret(int32_t i, int8_t j) {
 typedef unsigned char v16i8 __attribute__((vector_size(16)));
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_vec_large_v16i8
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[TMP0:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[TMP0:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_vec_large_v16i8(v16i8 x) {
@@ -285,7 +285,7 @@ v16i8 f_vec_large_v16i8_ret(void) {
 // if they were passed in registers.
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local i32 @f_scalar_stack_1
-// ILP32-ILP32F-ILP32D-SAME: (i32 [[A_COERCE:%.*]], [2 x i32] [[B_COERCE:%.*]], i64 [[C_COERCE:%.*]], ptr noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (i32 [[A_COERCE:%.*]], [2 x i32] [[B_COERCE:%.*]], i64 [[C_COERCE:%.*]], ptr dead_on_return noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
@@ -343,7 +343,7 @@ struct large f_scalar_stack_6(float a, int64_t b, double c, long double d,
 // they would be if passed via registers.
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_agg_stack
-// ILP32-ILP32F-ILP32D-SAME: (double noundef [[A:%.*]], i64 noundef [[B:%.*]], double noundef [[C:%.*]], i64 noundef [[D:%.*]], i32 [[E_COERCE:%.*]], [2 x i32] [[F_COERCE:%.*]], i64 [[G_COERCE:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (double noundef [[A:%.*]], i64 noundef [[B:%.*]], double noundef [[C:%.*]], i64 noundef [[D:%.*]], i32 [[E_COERCE:%.*]], [2 x i32] [[F_COERCE:%.*]], i64 [[G_COERCE:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_agg_stack(double a, int64_t b, double c, int64_t d, struct tiny e,
@@ -366,7 +366,7 @@ struct double_int8_s { double d; int64_t i; };
 struct int_double_s { int a; double b; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_int_double_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_int_double_s_arg
@@ -482,7 +482,7 @@ struct zbf_double_zbf_s f_ret_zbf_double_zbf_s(void) {
 struct double_float_s { double f; float g; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_double_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_double_s_arg
@@ -504,7 +504,7 @@ struct double_double_s f_ret_double_double_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_float_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_float_s_arg
@@ -526,7 +526,7 @@ struct double_float_s f_ret_double_float_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_double_s_arg_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_double_s_arg_insufficient_fprs(float a, double b, double c, double d,
@@ -543,7 +543,7 @@ struct double_int64bf_s { double d; int64_t i : 32; };
 struct double_int8_zbf_s { double d; int8_t i; int : 0; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int8_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int8_s_arg(struct double_int8_s a) {}
@@ -557,7 +557,7 @@ struct double_int8_s f_ret_double_int8_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_uint8_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_uint8_s_arg
@@ -579,7 +579,7 @@ struct double_uint8_s f_ret_double_uint8_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int32_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int32_s_arg
@@ -601,7 +601,7 @@ struct double_int32_s f_ret_double_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int64_s_arg(struct double_int64_s a) {}
@@ -615,7 +615,7 @@ struct double_int64_s f_ret_double_int64_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int64bf_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int64bf_s_arg
@@ -640,7 +640,7 @@ struct double_int64bf_s f_ret_double_int64bf_s(void) {
 // floating point calling convention.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int8_zbf_s
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int8_zbf_s
@@ -662,14 +662,14 @@ struct double_int8_zbf_s f_ret_double_int8_zbf_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int8_s_arg_insufficient_gprs
-// ILP32-ILP32F-ILP32D-SAME: (i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], i32 noundef [[H:%.*]], ptr noundef [[I:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], i32 noundef [[H:%.*]], ptr dead_on_return noundef [[I:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int8_s_arg_insufficient_gprs(int a, int b, int c, int d, int e,
                                           int f, int g, int h, struct double_int8_s i) {}
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_struct_double_int8_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], double noundef [[H:%.*]], ptr noundef [[I:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], double noundef [[H:%.*]], ptr dead_on_return noundef [[I:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_struct_double_int8_insufficient_fprs(float a, double b, double c, double d,
@@ -679,7 +679,7 @@ void f_struct_double_int8_insufficient_fprs(float a, double b, double c, double
 // floating-point value should be passed as if it were an fp+fp struct.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublecomplex
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublecomplex
@@ -703,7 +703,7 @@ double __complex__ f_ret_doublecomplex(void) {
 struct doublecomplex_s { double __complex__ c; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublecomplex_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublecomplex_s_arg
@@ -754,7 +754,7 @@ struct doublearr1_s f_ret_doublearr1_s(void) {
 struct doublearr2_s { double a[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_s_arg
@@ -778,7 +778,7 @@ struct doublearr2_s f_ret_doublearr2_s(void) {
 struct doublearr2_tricky1_s { struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky1_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky1_s_arg
@@ -802,7 +802,7 @@ struct doublearr2_tricky1_s f_ret_doublearr2_tricky1_s(void) {
 struct doublearr2_tricky2_s { struct {}; struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky2_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky2_s_arg
@@ -826,7 +826,7 @@ struct doublearr2_tricky2_s f_ret_doublearr2_tricky2_s(void) {
 struct doublearr2_tricky3_s { union {}; struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky3_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky3_s_arg
@@ -850,7 +850,7 @@ struct doublearr2_tricky3_s f_ret_doublearr2_tricky3_s(void) {
 struct doublearr2_tricky4_s { union {}; struct { struct {}; double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky4_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky4_s_arg
@@ -877,7 +877,7 @@ struct doublearr2_tricky4_s f_ret_doublearr2_tricky4_s(void) {
 struct int_double_int_s { int a; double b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_double_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_double_int_s_arg(struct int_double_int_s a) {}
@@ -893,7 +893,7 @@ struct int_double_int_s f_ret_int_double_int_s(void) {
 struct int64_double_s { int64_t a; double b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_double_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_double_s_arg(struct int64_double_s a) {}
@@ -909,7 +909,7 @@ struct int64_double_s f_ret_int64_double_s(void) {
 struct char_char_double_s { char a; char b; double c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_char_char_double_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_char_char_double_s_arg(struct char_char_double_s a) {}
@@ -948,7 +948,7 @@ union double_u f_ret_double_u(void) {
 // double+double structs by the ABI.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_double_int32_s_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_INT32_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_INT32_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, i32 } @f_ret_double_int32_s_double_int32_s_just_sufficient_gprs
@@ -961,7 +961,7 @@ struct double_int32_s f_ret_double_int32_s_double_int32_s_just_sufficient_gprs(
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_double_double_s_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @f_ret_double_double_s_double_int32_s_just_sufficient_gprs
@@ -974,7 +974,7 @@ struct double_double_s f_ret_double_double_s_double_int32_s_just_sufficient_gprs
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_doublecomplex_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @f_ret_doublecomplex_double_int32_s_just_sufficient_gprs
@@ -1189,7 +1189,7 @@ struct float_int32_s f_ret_float_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_float_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_float_int64_s_arg(struct float_int64_s a) {}
@@ -1465,7 +1465,7 @@ struct floatarr2_tricky4_s f_ret_floatarr2_tricky4_s(void) {
 struct int_float_int_s { int a; float b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_float_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_float_int_s_arg(struct int_float_int_s a) {}
@@ -1481,7 +1481,7 @@ struct int_float_int_s f_ret_int_float_int_s(void) {
 struct int64_float_s { int64_t a; float b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_float_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_float_s_arg(struct int64_float_s a) {}
@@ -1619,7 +1619,7 @@ struct zbf_float16_zbf_s f_ret_zbf_float16_zbf_s(void) {
 struct double_float16_s { double f; _Float16 g; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_float16_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_float16_s_arg
@@ -1641,7 +1641,7 @@ struct double_float16_s f_ret_double_float16_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_float16_s_arg_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_float16_s_arg_insufficient_fprs(float a, double b, double c, double d,
@@ -1725,7 +1725,7 @@ struct float16_int32_s f_ret_float16_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_float16_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_float16_int64_s_arg(struct float16_int64_s a) {}
@@ -2001,7 +2001,7 @@ struct float16arr2_tricky4_s f_ret_float16arr2_tricky4_s(void) {
 struct int_float16_int_s { int a; _Float16 b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_float16_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_float16_int_s_arg(struct int_float16_int_s a) {}
@@ -2017,7 +2017,7 @@ struct int_float16_int_s f_ret_int_float16_int_s(void) {
 struct int64_float16_s { int64_t a; _Float16 b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_float16_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_float16_s_arg(struct int64_float16_s a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv32-vararg.c b/clang/test/CodeGen/RISCV/riscv32-vararg.c
index 2b332410f8637..ed301f9269bb8 100644
--- a/clang/test/CodeGen/RISCV/riscv32-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv32-vararg.c
@@ -64,7 +64,7 @@ int f_va_callee(int, ...);
 // CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL4]], i32 0, i32 0
 // CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[COERCE_DIVE]], align 8
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[BYVAL_TEMP]], ptr align 4 [[DOTCOMPOUNDLITERAL6]], i32 16, i1 false)
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 (i32, ...) @f_va_callee(i32 noundef 1, i32 noundef 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i32 [[TMP0]], [2 x i32] [[TMP1]], i64 [[TMP2]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 (i32, ...) @f_va_callee(i32 noundef 1, i32 noundef 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i32 [[TMP0]], [2 x i32] [[TMP1]], i64 [[TMP2]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    ret void
 //
 void f_va_caller(void) {
diff --git a/clang/test/CodeGen/RISCV/riscv64-abi.c b/clang/test/CodeGen/RISCV/riscv64-abi.c
index 021565238904e..dc01750e56970 100644
--- a/clang/test/CodeGen/RISCV/riscv64-abi.c
+++ b/clang/test/CodeGen/RISCV/riscv64-abi.c
@@ -242,7 +242,7 @@ struct large {
 };
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_agg_large
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[X:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[X:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_agg_large(struct large x) {
@@ -262,7 +262,7 @@ struct large f_agg_large_ret(int32_t i, int8_t j) {
 typedef unsigned char v32i8 __attribute__((vector_size(32)));
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_vec_large_v32i8
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[TMP0:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[TMP0:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_vec_large_v32i8(v32i8 x) {
@@ -281,7 +281,7 @@ v32i8 f_vec_large_v32i8_ret(void) {
 // if they were passed in registers.
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_1
-// LP64-LP64F-LP64D-SAME: (i64 [[A_COERCE:%.*]], [2 x i64] [[B_COERCE:%.*]], i128 [[C_COERCE:%.*]], ptr noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i64 [[A_COERCE:%.*]], [2 x i64] [[B_COERCE:%.*]], i128 [[C_COERCE:%.*]], ptr dead_on_return noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
@@ -290,7 +290,7 @@ int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_2
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], i64 noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], i64 noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_2(int32_t a, __int128_t b, int64_t c, long double d, v32i8 e,
@@ -299,7 +299,7 @@ int f_scalar_stack_2(int32_t a, __int128_t b, int64_t c, long double d, v32i8 e,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_3
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], double noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], double noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_3(int32_t a, __int128_t b, double c, long double d, v32i8 e,
@@ -312,7 +312,7 @@ int f_scalar_stack_3(int32_t a, __int128_t b, double c, long double d, v32i8 e,
 // to pass a pointer.
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_scalar_stack_4
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct large f_scalar_stack_4(uint32_t a, __int128_t b, long double c, v32i8 d,
@@ -321,7 +321,7 @@ struct large f_scalar_stack_4(uint32_t a, __int128_t b, long double c, v32i8 d,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_scalar_stack_5
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], double noundef [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], double noundef [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct large f_scalar_stack_5(double a, __int128_t b, long double c, v32i8 d,
@@ -330,7 +330,7 @@ struct large f_scalar_stack_5(double a, __int128_t b, long double c, v32i8 d,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_6
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], float noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]], half noundef [[I:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], float noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]], half noundef [[I:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_6(int32_t a, __int128_t b, float c, long double d, v32i8 e,
@@ -1440,7 +1440,7 @@ struct doublearr2_tricky4_s f_ret_doublearr2_tricky4_s(void) {
 struct int_double_int_s { int a; double b; int c; };
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_int_double_int_s_arg
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_int_double_int_s_arg(struct int_double_int_s a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv64-vararg.c b/clang/test/CodeGen/RISCV/riscv64-vararg.c
index a278f74ca4a86..17802553c795a 100644
--- a/clang/test/CodeGen/RISCV/riscv64-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv64-vararg.c
@@ -74,7 +74,7 @@ int f_va_callee(int, ...);
 // CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL4]], i32 0, i32 0
 // CHECK-NEXT:    [[TMP2:%.*]] = load i128, ptr [[COERCE_DIVE]], align 16
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[BYVAL_TEMP]], ptr align 8 [[DOTCOMPOUNDLITERAL6]], i64 32, i1 false)
-// CHECK-NEXT:    [[CALL:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i64 [[TMP0]], [2 x i64] [[TMP1]], i128 [[TMP2]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    [[CALL:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i64 [[TMP0]], [2 x i64] [[TMP1]], i128 [[TMP2]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    [[CALL11:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i32 noundef signext 3, i32 noundef signext 4, fp128 noundef 0xL00000000000000004001400000000000, i32 noundef signext 6, i32 noundef signext 7, i32 noundef signext 8, i32 noundef signext 9)
 // CHECK-NEXT:    [[A13:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL12]], i32 0, i32 0
 // CHECK-NEXT:    store i128 5, ptr [[A13]], align 16
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi-vector.c b/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
index e5704709a3a33..fab6050a0d876 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
@@ -58,91 +58,91 @@ unsigned int align = __alignof__ (v16i8);
 // CHECK-VECTOR: @align ={{.*}} global i32 8
 
 v1i8 pass_v1i8(v1i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i8(ptr dead_on_unwind noalias writable sret(<1 x i8>) align 1 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i8(ptr dead_on_unwind noalias writable sret(<1 x i8>) align 1 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i8> @pass_v1i8(<1 x i8> %{{.*}})
 
 v2i8 pass_v2i8(v2i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i8(ptr dead_on_unwind noalias writable sret(<2 x i8>) align 2 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i8(ptr dead_on_unwind noalias writable sret(<2 x i8>) align 2 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i8> @pass_v2i8(<2 x i8> %{{.*}})
 
 v4i8 pass_v4i8(v4i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i8(ptr dead_on_unwind noalias writable sret(<4 x i8>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i8(ptr dead_on_unwind noalias writable sret(<4 x i8>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i8> @pass_v4i8(<4 x i8> %{{.*}})
 
 v8i8 pass_v8i8(v8i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v8i8(ptr dead_on_unwind noalias writable sret(<8 x i8>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v8i8(ptr dead_on_unwind noalias writable sret(<8 x i8>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <8 x i8> @pass_v8i8(<8 x i8> %{{.*}})
 
 v16i8 pass_v16i8(v16i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v16i8(ptr dead_on_unwind noalias writable sret(<16 x i8>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v16i8(ptr dead_on_unwind noalias writable sret(<16 x i8>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <16 x i8> @pass_v16i8(<16 x i8> %{{.*}})
 
 v32i8 pass_v32i8(v32i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 32 %{{.*}}, ptr %0)
-// CHECK-VECTOR-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 32 %{{.*}}, ptr dead_on_return %0)
+// CHECK-VECTOR-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 8 %{{.*}}, ptr dead_on_return %0)
 
 v1i16 pass_v1i16(v1i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i16(ptr dead_on_unwind noalias writable sret(<1 x i16>) align 2 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i16(ptr dead_on_unwind noalias writable sret(<1 x i16>) align 2 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i16> @pass_v1i16(<1 x i16> %{{.*}})
 
 v2i16 pass_v2i16(v2i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i16(ptr dead_on_unwind noalias writable sret(<2 x i16>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i16(ptr dead_on_unwind noalias writable sret(<2 x i16>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i16> @pass_v2i16(<2 x i16> %{{.*}})
 
 v4i16 pass_v4i16(v4i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i16(ptr dead_on_unwind noalias writable sret(<4 x i16>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i16(ptr dead_on_unwind noalias writable sret(<4 x i16>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i16> @pass_v4i16(<4 x i16> %{{.*}})
 
 v8i16 pass_v8i16(v8i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v8i16(ptr dead_on_unwind noalias writable sret(<8 x i16>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v8i16(ptr dead_on_unwind noalias writable sret(<8 x i16>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <8 x i16> @pass_v8i16(<8 x i16> %{{.*}})
 
 v1i32 pass_v1i32(v1i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i32(ptr dead_on_unwind noalias writable sret(<1 x i32>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i32(ptr dead_on_unwind noalias writable sret(<1 x i32>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i32> @pass_v1i32(<1 x i32> %{{.*}})
 
 v2i32 pass_v2i32(v2i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i32(ptr dead_on_unwind noalias writable sret(<2 x i32>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i32(ptr dead_on_unwind noalias writable sret(<2 x i32>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i32> @pass_v2i32(<2 x i32> %{{.*}})
 
 v4i32 pass_v4i32(v4i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i32(ptr dead_on_unwind noalias writable sret(<4 x i32>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i32(ptr dead_on_unwind noalias writable sret(<4 x i32>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i32> @pass_v4i32(<4 x i32> %{{.*}})
 
 v1i64 pass_v1i64(v1i64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i64(ptr dead_on_unwind noalias writable sret(<1 x i64>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i64(ptr dead_on_unwind noalias writable sret(<1 x i64>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i64> @pass_v1i64(<1 x i64> %{{.*}})
 
 v2i64 pass_v2i64(v2i64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i64(ptr dead_on_unwind noalias writable sret(<2 x i64>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i64(ptr dead_on_unwind noalias writable sret(<2 x i64>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i64> @pass_v2i64(<2 x i64> %{{.*}})
 
 v1i128 pass_v1i128(v1i128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i128(ptr dead_on_unwind noalias writable sret(<1 x i128>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i128(ptr dead_on_unwind noalias writable sret(<1 x i128>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i128> @pass_v1i128(<1 x i128> %{{.*}})
 
 v1f32 pass_v1f32(v1f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f32(ptr dead_on_unwind noalias writable sret(<1 x float>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f32(ptr dead_on_unwind noalias writable sret(<1 x float>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x float> @pass_v1f32(<1 x float> %{{.*}})
 
 v2f32 pass_v2f32(v2f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2f32(ptr dead_on_unwind noalias writable sret(<2 x float>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2f32(ptr dead_on_unwind noalias writable sret(<2 x float>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x float> @pass_v2f32(<2 x float> %{{.*}})
 
 v4f32 pass_v4f32(v4f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4f32(ptr dead_on_unwind noalias writable sret(<4 x float>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4f32(ptr dead_on_unwind noalias writable sret(<4 x float>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x float> @pass_v4f32(<4 x float> %{{.*}})
 
 v1f64 pass_v1f64(v1f64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f64(ptr dead_on_unwind noalias writable sret(<1 x double>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f64(ptr dead_on_unwind noalias writable sret(<1 x double>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x double> @pass_v1f64(<1 x double> %{{.*}})
 
 v2f64 pass_v2f64(v2f64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2f64(ptr dead_on_unwind noalias writable sret(<2 x double>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2f64(ptr dead_on_unwind noalias writable sret(<2 x double>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x double> @pass_v2f64(<2 x double> %{{.*}})
 
 v1f128 pass_v1f128(v1f128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f128(ptr dead_on_unwind noalias writable sret(<1 x fp128>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f128(ptr dead_on_unwind noalias writable sret(<1 x fp128>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x fp128> @pass_v1f128(<1 x fp128> %{{.*}})
 
 
@@ -170,13 +170,13 @@ struct agg_v8i8 pass_agg_v8i8(struct agg_v8i8 arg) { return arg; }
 
 struct agg_v16i8 { v16i8 a; };
 struct agg_v16i8 pass_agg_v16i8(struct agg_v16i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 // CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 8 %{{.*}}, <16 x i8> %{{.*}})
 
 struct agg_v32i8 { v32i8 a; };
 struct agg_v32i8 pass_agg_v32i8(struct agg_v32i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 32 %{{.*}}, ptr %{{.*}})
-// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 8 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 32 %{{.*}}, ptr dead_on_return %{{.*}})
+// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 8 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Verify that the following are *not* vector-like aggregate types
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi.c b/clang/test/CodeGen/SystemZ/systemz-abi.c
index f26084ab44eae..83137ae6d5f82 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi.c
@@ -50,7 +50,7 @@ long long pass_longlong(long long arg) { return arg; }
 // CHECK-LABEL: define{{.*}} i64 @pass_longlong(i64 %{{.*}})
 
 __int128 pass_int128(__int128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr dead_on_return %0)
 
 _Float16 pass__Float16(_Float16 arg) { return arg; }
 // CHECK-LABEL: define{{.*}} half @pass__Float16(half %{{.*}})
@@ -62,37 +62,37 @@ double pass_double(double arg) { return arg; }
 // CHECK-LABEL: define{{.*}} double @pass_double(double %{{.*}})
 
 long double pass_longdouble(long double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_longdouble(ptr dead_on_unwind noalias writable sret(fp128) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_longdouble(ptr dead_on_unwind noalias writable sret(fp128) align 8 %{{.*}}, ptr dead_on_return %0)
 
 
 // Complex types
 
 _Complex char pass_complex_char(_Complex char arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_char(ptr dead_on_unwind noalias writable sret({ i8, i8 }) align 1 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_char(ptr dead_on_unwind noalias writable sret({ i8, i8 }) align 1 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex short pass_complex_short(_Complex short arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_short(ptr dead_on_unwind noalias writable sret({ i16, i16 }) align 2 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_short(ptr dead_on_unwind noalias writable sret({ i16, i16 }) align 2 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex int pass_complex_int(_Complex int arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_int(ptr dead_on_unwind noalias writable sret({ i32, i32 }) align 4 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_int(ptr dead_on_unwind noalias writable sret({ i32, i32 }) align 4 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long pass_complex_long(_Complex long arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_long(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_long(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long long pass_complex_longlong(_Complex long long arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex _Float16 pass_complex__Float16(_Complex _Float16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex float pass_complex_float(_Complex float arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex double pass_complex_double(_Complex double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_double(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_double(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long double pass_complex_longdouble(_Complex long double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_longdouble(ptr dead_on_unwind noalias writable sret({ fp128, fp128 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_longdouble(ptr dead_on_unwind noalias writable sret({ fp128, fp128 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 
 // Aggregate types
@@ -107,7 +107,7 @@ struct agg_2byte pass_agg_2byte(struct agg_2byte arg) { return arg; }
 
 struct agg_3byte { char a[3]; };
 struct agg_3byte pass_agg_3byte(struct agg_3byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_3byte(ptr dead_on_unwind noalias writable sret(%struct.agg_3byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_3byte(ptr dead_on_unwind noalias writable sret(%struct.agg_3byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_4byte { char a[4]; };
 struct agg_4byte pass_agg_4byte(struct agg_4byte arg) { return arg; }
@@ -115,15 +115,15 @@ struct agg_4byte pass_agg_4byte(struct agg_4byte arg) { return arg; }
 
 struct agg_5byte { char a[5]; };
 struct agg_5byte pass_agg_5byte(struct agg_5byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_5byte(ptr dead_on_unwind noalias writable sret(%struct.agg_5byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_5byte(ptr dead_on_unwind noalias writable sret(%struct.agg_5byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_6byte { char a[6]; };
 struct agg_6byte pass_agg_6byte(struct agg_6byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_6byte(ptr dead_on_unwind noalias writable sret(%struct.agg_6byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_6byte(ptr dead_on_unwind noalias writable sret(%struct.agg_6byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_7byte { char a[7]; };
 struct agg_7byte pass_agg_7byte(struct agg_7byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_7byte(ptr dead_on_unwind noalias writable sret(%struct.agg_7byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_7byte(ptr dead_on_unwind noalias writable sret(%struct.agg_7byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_8byte { char a[8]; };
 struct agg_8byte pass_agg_8byte(struct agg_8byte arg) { return arg; }
@@ -131,7 +131,7 @@ struct agg_8byte pass_agg_8byte(struct agg_8byte arg) { return arg; }
 
 struct agg_16byte { char a[16]; };
 struct agg_16byte pass_agg_16byte(struct agg_16byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_16byte(ptr dead_on_unwind noalias writable sret(%struct.agg_16byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_16byte(ptr dead_on_unwind noalias writable sret(%struct.agg_16byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Float-like aggregate types
@@ -153,7 +153,7 @@ struct agg_double pass_agg_double(struct agg_double arg) { return arg; }
 
 struct agg_longdouble { long double a; };
 struct agg_longdouble pass_agg_longdouble(struct agg_longdouble arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg__Float16_a4 { _Float16 a __attribute__((aligned (4))); };
 struct agg__Float16_a4 pass_agg__Float16_a4(struct agg__Float16_a4 arg) { return arg; }
@@ -167,7 +167,7 @@ struct agg__Float16_a8 pass_agg__Float16_a8(struct agg__Float16_a8 arg) { return
 
 struct agg__Float16_a16 { _Float16 a __attribute__((aligned (16))); };
 struct agg__Float16_a16 pass_agg__Float16_a16(struct agg__Float16_a16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_float_a8 { float a __attribute__((aligned (8))); };
 struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
@@ -176,7 +176,7 @@ struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
 
 struct agg_float_a16 { float a __attribute__((aligned (16))); };
 struct agg_float_a16 pass_agg_float_a16(struct agg_float_a16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_float_a16(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a16) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_float_a16(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a16) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Verify that the following are *not* float-like aggregate types
diff --git a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
index 434937a66389c..d76fb4bd1fda6 100644
--- a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
+++ b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
@@ -131,7 +131,7 @@ double test_f64(double f, double g) {
 long double test_f128(long double f, long double g) {
   asm("axbr %0, %2" : "=f" (f) : "0" (f), "f" (g));
   return f;
-// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr noundef readonly captures(none) %0, ptr noundef readonly captures(none) %1)
+// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr dead_on_return noundef readonly captures(none) %0, ptr dead_on_return noundef readonly captures(none) %1)
 // CHECK: %f = load fp128, ptr %0
 // CHECK: %g = load fp128, ptr %1
 // CHECK: [[RESULT:%.*]] = tail call fp128 asm "axbr $0, $2", "=f,0,f"(fp128 %f, fp128 %g)
diff --git a/clang/test/CodeGen/X86/cx-complex-range.c b/clang/test/CodeGen/X86/cx-complex-range.c
index f87091427df71..b16b10b7b8a21 100644
--- a/clang/test/CodeGen/X86/cx-complex-range.c
+++ b/clang/test/CodeGen/X86/cx-complex-range.c
@@ -1064,7 +1064,7 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // PRMTD-NEXT:    ret <2 x half> [[TMP33]]
 //
 // X86WINPRMTD-LABEL: define dso_local i32 @f1(
-// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[RETVAL:%.*]] = alloca { half, half }, align 2
 // X86WINPRMTD-NEXT:    [[A:%.*]] = alloca { half, half }, align 2
diff --git a/clang/test/CodeGen/X86/x86_32-arguments-win32.c b/clang/test/CodeGen/X86/x86_32-arguments-win32.c
index 5b81c43f4bbb8..53d040af725b0 100644
--- a/clang/test/CodeGen/X86/x86_32-arguments-win32.c
+++ b/clang/test/CodeGen/X86/x86_32-arguments-win32.c
@@ -72,10 +72,10 @@ void receive_vec_512(__m512 x, __m512 y, __m512 z, __m512 w, __m512 q) {
 void receive_vec_1024(__m1024 x, __m1024 y, __m1024 z, __m1024 w, __m1024 q) {
   gv1024 = x + y + z + w + q;
 }
-// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg noundef %x, <8 x float> inreg noundef %y, <8 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg noundef %x, <16 x float> inreg noundef %y, <16 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_1024(ptr noundef %0, ptr noundef %1, ptr noundef %2, ptr noundef %3, ptr noundef %4)
+// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg noundef %x, <8 x float> inreg noundef %y, <8 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg noundef %x, <16 x float> inreg noundef %y, <16 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_1024(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1, ptr dead_on_return noundef %2, ptr dead_on_return noundef %3, ptr dead_on_return noundef %4)
 
 void pass_vec_128(void) {
   __m128 z = {0};
@@ -83,13 +83,13 @@ void pass_vec_128(void) {
 }
 
 // CHECK-LABEL: define dso_local void @pass_vec_128()
-// CHECK: call void @receive_vec_128(<4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, ptr noundef %{{[^,)]*}}, ptr noundef %{{[^,)]*}})
+// CHECK: call void @receive_vec_128(<4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, ptr dead_on_return noundef %{{[^,)]*}}, ptr dead_on_return noundef %{{[^,)]*}})
 
 
 void __fastcall fastcall_indirect_vec(__m128 x, __m128 y, __m128 z, __m128 w, int edx, __m128 q) {
   gv128 = x + y + z + w + q;
 }
-// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01 at fastcall_indirect_vec@84"(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr inreg noundef %0, i32 inreg noundef %edx, ptr noundef %1)
+// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01 at fastcall_indirect_vec@84"(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr dead_on_return inreg noundef %0, i32 inreg noundef %edx, ptr dead_on_return noundef %1)
 
 struct __declspec(align(1)) Align1 { unsigned long long x; };
 struct __declspec(align(4)) Align4 { unsigned long long x; };
@@ -156,4 +156,4 @@ void pass_fixed_align_variadic() {
 // correctly in Clang than it is to be bug for bug compatible, so we pass such
 // arguments indirectly.
 // CHECK-LABEL: define dso_local void @pass_fixed_align_variadic()
-// CHECK: call void (ptr, ...) @receive_fixed_align_variadic(ptr noundef %{{[^)]*}}, i32 noundef 42)
+// CHECK: call void (ptr, ...) @receive_fixed_align_variadic(ptr dead_on_return noundef %{{[^)]*}}, i32 noundef 42)
diff --git a/clang/test/CodeGen/X86/x86_64-arguments-win32.c b/clang/test/CodeGen/X86/x86_64-arguments-win32.c
index 8768e73a854aa..6010e531acb00 100644
--- a/clang/test/CodeGen/X86/x86_64-arguments-win32.c
+++ b/clang/test/CodeGen/X86/x86_64-arguments-win32.c
@@ -21,7 +21,7 @@ void f4(unsigned short a) {}
 // CHECK-LABEL: define dso_local void @f5(i64 noundef %a.coerce)
 void f5(_Complex float a) {}
 
-// CHECK-LABEL: define dso_local void @f6(ptr noundef %a)
+// CHECK-LABEL: define dso_local void @f6(ptr dead_on_return noundef %a)
 void f6(_Complex double a) {}
 
 // CHECK-LABEL: define dso_local i64 @f7()
diff --git a/clang/test/CodeGen/aapcs64-align.cpp b/clang/test/CodeGen/aapcs64-align.cpp
index 1c26d68e434f4..53fc53c2f7296 100644
--- a/clang/test/CodeGen/aapcs64-align.cpp
+++ b/clang/test/CodeGen/aapcs64-align.cpp
@@ -122,8 +122,8 @@ unsigned sizeof_RidiculouslyOverSizedBitfield = sizeof(RidiculouslyOverSizedBitf
 unsigned alignof_RidiculouslyOverSizedBitfield = alignof(RidiculouslyOverSizedBitfield);
 
 // CHECK: define{{.*}} void @g9
-// CHECK: call void @f9(i32 noundef 1, ptr noundef nonnull %agg.tmp)
-// CHECK: declare void @f9(i32 noundef, ptr noundef)
+// CHECK: call void @f9(i32 noundef 1, ptr dead_on_return noundef nonnull %agg.tmp)
+// CHECK: declare void @f9(i32 noundef, ptr dead_on_return noundef)
 void f9(int a, RidiculouslyOverSizedBitfield b);
 void g9() {
   RidiculouslyOverSizedBitfield s = {42};
diff --git a/clang/test/CodeGen/arm-aapcs-vfp.c b/clang/test/CodeGen/arm-aapcs-vfp.c
index 6581929f99f14..e60ed1e52c33a 100644
--- a/clang/test/CodeGen/arm-aapcs-vfp.c
+++ b/clang/test/CodeGen/arm-aapcs-vfp.c
@@ -65,7 +65,7 @@ struct big_struct {
   float f4;
 };
 // CHECK: define{{.*}} arm_aapcs_vfpcc void @test_big([5 x i32] %{{.*}})
-// CHECK64: define{{.*}} void @test_big(ptr noundef %{{.*}})
+// CHECK64: define{{.*}} void @test_big(ptr dead_on_return noundef %{{.*}})
 // CHECK64: call void @llvm.memcpy
 // CHECK64: call void @big_callee(ptr
 extern void big_callee(struct big_struct);
diff --git a/clang/test/CodeGen/arm-abi-vector.c b/clang/test/CodeGen/arm-abi-vector.c
index c2a8902007980..93b770878c3fa 100644
--- a/clang/test/CodeGen/arm-abi-vector.c
+++ b/clang/test/CodeGen/arm-abi-vector.c
@@ -177,11 +177,11 @@ double varargs_vec_19c(int fixed, ...) {
 
 double test_19c(__char19 *in) {
 // CHECK: test_19c
-// CHECK: call arm_aapcscc double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call arm_aapcscc double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
 // APCS-GNU: test_19c
-// APCS-GNU: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// APCS-GNU: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
 // ANDROID: test_19c
-// ANDROID: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// ANDROID: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_19c(19, *in);
 }
 
diff --git a/clang/test/CodeGen/arm-swiftcall.c b/clang/test/CodeGen/arm-swiftcall.c
index 677b878c6765d..55c22a45fd8de 100644
--- a/clang/test/CodeGen/arm-swiftcall.c
+++ b/clang/test/CodeGen/arm-swiftcall.c
@@ -261,7 +261,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define{{.*}} void @return_struct_big_1({{.*}} dead_on_unwind noalias writable sret({{.*}})
 
 // Should not be byval.
-// CHECK-LABEL: define{{.*}} void @take_struct_big_1(ptr{{( %.*)?}})
+// CHECK-LABEL: define{{.*}} void @take_struct_big_1(ptr dead_on_return{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGen/arm64-abi-vector.c b/clang/test/CodeGen/arm64-abi-vector.c
index 81e42315c883b..cf50cdd2fe86e 100644
--- a/clang/test/CodeGen/arm64-abi-vector.c
+++ b/clang/test/CodeGen/arm64-abi-vector.c
@@ -128,7 +128,7 @@ double varargs_vec_19c(int fixed, ...) {
 
 double test_19c(__char19 *in) {
 // CHECK: test_19c
-// CHECK: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_19c(19, *in);
 }
 
@@ -211,7 +211,7 @@ double varargs_vec_5i(int fixed, ...) {
 
 double test_5i(__int5 *in) {
 // CHECK: test_5i
-// CHECK: call double (i32, ...) @varargs_vec_5i(i32 noundef 5, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_5i(i32 noundef 5, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_5i(5, *in);
 }
 
@@ -231,7 +231,7 @@ double varargs_vec_3d(int fixed, ...) {
 
 double test_3d(__double3 *in) {
 // CHECK: test_3d
-// CHECK: call double (i32, ...) @varargs_vec_3d(i32 noundef 3, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_3d(i32 noundef 3, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_3d(3, *in);
 }
 
@@ -291,7 +291,7 @@ double test(__char3 *c3, __char5 *c5, __char9 *c9, __char19 *c19,
             __short3 *s3, __short5 *s5, __int3 *i3, __int5 *i5,
             __double3 *d3) {
   double ret = varargs_vec(3, *c3, *c5, *c9, *c19, *s3, *s5, *i3, *i5, *d3);
-// CHECK: call double (i32, ...) @varargs_vec(i32 noundef 3, i32 {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr noundef {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr noundef {{%.*}}, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec(i32 noundef 3, i32 {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr dead_on_return noundef {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr dead_on_return noundef {{%.*}}, ptr dead_on_return noundef {{%.*}})
   return ret;
 }
 
@@ -350,7 +350,7 @@ __attribute__((noinline)) double args_vec_19c(int fixed, __char19 c19) {
 
 double fixed_19c(__char19 *in) {
 // CHECK: fixed_19c
-// CHECK: call double @args_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return args_vec_19c(19, *in);
 }
 
@@ -409,7 +409,7 @@ __attribute__((noinline)) double args_vec_5i(int fixed, __int5 c5) {
 
 double fixed_5i(__int5 *in) {
 // CHECK: fixed_5i
-// CHECK: call double @args_vec_5i(i32 noundef 5, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_5i(i32 noundef 5, ptr dead_on_return noundef {{%.*}})
   return args_vec_5i(5, *in);
 }
 
@@ -424,6 +424,6 @@ __attribute__((noinline)) double args_vec_3d(int fixed, __double3 c3) {
 
 double fixed_3d(__double3 *in) {
 // CHECK: fixed_3d
-// CHECK: call double @args_vec_3d(i32 noundef 3, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_3d(i32 noundef 3, ptr dead_on_return noundef {{%.*}})
   return args_vec_3d(3, *in);
 }
diff --git a/clang/test/CodeGen/arm64-arguments.c b/clang/test/CodeGen/arm64-arguments.c
index 4c4f85d923e78..2e3ab388432f6 100644
--- a/clang/test/CodeGen/arm64-arguments.c
+++ b/clang/test/CodeGen/arm64-arguments.c
@@ -163,7 +163,7 @@ void f32(struct s32 s) { }
 // A composite type larger than 16 bytes should be passed indirectly.
 struct s33 { char buf[32*32]; };
 void f33(struct s33 s) { }
-// CHECK: define{{.*}} void @f33(ptr noundef %s)
+// CHECK: define{{.*}} void @f33(ptr dead_on_return noundef %s)
 
 struct s34 { char c; };
 void f34(struct s34 s);
@@ -226,9 +226,9 @@ T_float32x2 f1_0(T_float32x2 a0) { return a0; }
 // CHECK: define{{.*}} <4 x float> @f1_1(<4 x float> noundef %{{.*}})
 T_float32x4 f1_1(T_float32x4 a0) { return a0; }
 // Vector with length bigger than 16-byte is illegal and is passed indirectly.
-// CHECK: define{{.*}} void @f1_2(ptr dead_on_unwind noalias writable sret(<8 x float>) align 16 %{{.*}}, ptr noundef %0)
+// CHECK: define{{.*}} void @f1_2(ptr dead_on_unwind noalias writable sret(<8 x float>) align 16 %{{.*}}, ptr dead_on_return noundef %0)
 T_float32x8 f1_2(T_float32x8 a0) { return a0; }
-// CHECK: define{{.*}} void @f1_3(ptr dead_on_unwind noalias writable sret(<16 x float>) align 16 %{{.*}}, ptr noundef %0)
+// CHECK: define{{.*}} void @f1_3(ptr dead_on_unwind noalias writable sret(<16 x float>) align 16 %{{.*}}, ptr dead_on_return noundef %0)
 T_float32x16 f1_3(T_float32x16 a0) { return a0; }
 
 // Testing alignment with aggregates: HFA, aggregates with size <= 16 bytes and
@@ -278,7 +278,7 @@ struct s37
 typedef struct s37 s37_with_align;
 
 int32x4_t f37(int i, s37_with_align s1, s37_with_align s2) {
-// CHECK: define{{.*}} <4 x i32> @f37(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} <4 x i32> @f37(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: load <4 x i32>, ptr %s1, align 16
 // CHECK: load <4 x i32>, ptr %s2, align 16
   int32x4_t v = vaddq_s32(*(int32x4_t *)&s1,
@@ -292,7 +292,7 @@ int32x4_t caller37() {
 // CHECK: %[[b:.*]] = alloca %struct.s37, align 16
 // CHECK: call void @llvm.memcpy
 // CHECK: call void @llvm.memcpy
-// CHECK: call <4 x i32> @f37(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call <4 x i32> @f37(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f37(3, g37, g37);
 }
 
@@ -530,7 +530,7 @@ typedef struct s42 s42_no_align;
 // passing structs in registers
 __attribute__ ((noinline))
 int f42(int i, s42_no_align s1, s42_no_align s2) {
-// CHECK: define{{.*}} i32 @f42(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f42(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 1
@@ -545,14 +545,14 @@ int caller42() {
 // CHECK: %[[b:.*]] = alloca %struct.s42, align 4
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f42(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f42(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f42(3, g42, g42_2);
 }
 // passing structs on stack
 __attribute__ ((noinline))
 int f42_stack(int i, int i2, int i3, int i4, int i5, int i6, int i7, int i8,
               int i9, s42_no_align s1, s42_no_align s2) {
-// CHECK: define{{.*}} i32 @f42_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f42_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 1
@@ -565,7 +565,7 @@ int caller42_stack() {
 // CHECK: %[[b:.*]] = alloca %struct.s42, align 4
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f42_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f42_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f42_stack(1, 2, 3, 4, 5, 6, 7, 8, 9, g42, g42_2);
 }
 
@@ -583,7 +583,7 @@ typedef struct s43 s43_with_align;
 // passing aligned structs in registers
 __attribute__ ((noinline))
 int f43(int i, s43_with_align s1, s43_with_align s2) {
-// CHECK: define{{.*}} i32 @f43(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f43(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 1
@@ -598,14 +598,14 @@ int caller43() {
 // CHECK: %[[b:.*]] = alloca %struct.s43, align 16
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f43(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f43(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f43(3, g43, g43_2);
 }
 // passing aligned structs on stack
 __attribute__ ((noinline))
 int f43_stack(int i, int i2, int i3, int i4, int i5, int i6, int i7, int i8,
               int i9, s43_with_align s1, s43_with_align s2) {
-// CHECK: define{{.*}} i32 @f43_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f43_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 1
@@ -618,7 +618,7 @@ int caller43_stack() {
 // CHECK: %[[b:.*]] = alloca %struct.s43, align 16
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f43_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f43_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f43_stack(1, 2, 3, 4, 5, 6, 7, 8, 9, g43, g43_2);
 }
 
diff --git a/clang/test/CodeGen/arm64-microsoft-arguments.cpp b/clang/test/CodeGen/arm64-microsoft-arguments.cpp
index 85472645acb3b..a0a81be54325f 100644
--- a/clang/test/CodeGen/arm64-microsoft-arguments.cpp
+++ b/clang/test/CodeGen/arm64-microsoft-arguments.cpp
@@ -29,7 +29,7 @@ S2 f2() {
 
 // Pass and return for type size > 16 bytes.
 // CHECK: define {{.*}} void @{{.*}}f3{{.*}}(ptr dead_on_unwind noalias writable sret(%struct.S3) align 4 %agg.result)
-// CHECK: call void {{.*}}func3{{.*}}(ptr dead_on_unwind writable sret(%struct.S3) align 4 %agg.result, ptr noundef %agg.tmp)
+// CHECK: call void {{.*}}func3{{.*}}(ptr dead_on_unwind writable sret(%struct.S3) align 4 %agg.result, ptr dead_on_return noundef %agg.tmp)
 struct S3 {
   int a[5];
 };
diff --git a/clang/test/CodeGen/armv7k-abi.c b/clang/test/CodeGen/armv7k-abi.c
index 872e6423a4a99..6a781bc04d042 100644
--- a/clang/test/CodeGen/armv7k-abi.c
+++ b/clang/test/CodeGen/armv7k-abi.c
@@ -39,7 +39,7 @@ typedef struct {
   double z;
 } BigStruct;
 
-// CHECK: define{{.*}} void @big_struct_indirect(ptr noundef %b)
+// CHECK: define{{.*}} void @big_struct_indirect(ptr dead_on_return noundef %b)
 void big_struct_indirect(BigStruct b) {}
 
 // CHECK: define{{.*}} void @return_big_struct_indirect(ptr dead_on_unwind noalias writable sret
diff --git a/clang/test/CodeGen/atomic-arm64.c b/clang/test/CodeGen/atomic-arm64.c
index d2a30a3b6e66f..d539cad0c6a7d 100644
--- a/clang/test/CodeGen/atomic-arm64.c
+++ b/clang/test/CodeGen/atomic-arm64.c
@@ -57,7 +57,7 @@ void test3(pointer_pair_t pair) {
 }
 
 // CHECK-LABEL:define{{.*}} void @test4(
-// CHECK-SAME: ptr noundef [[QUAD:%.*]])
+// CHECK-SAME: ptr dead_on_return noundef [[QUAD:%.*]])
 // CHECK:      [[QUAD_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT: [[TEMP:%.*]] = alloca [[QUAD_T:%.*]], align 8
 // CHECK-NEXT: store ptr [[QUAD]], ptr [[QUAD_INDIRECT_ADDR]]
diff --git a/clang/test/CodeGen/attr-noundef.cpp b/clang/test/CodeGen/attr-noundef.cpp
index abdf9496bd396..619dbec7678db 100644
--- a/clang/test/CodeGen/attr-noundef.cpp
+++ b/clang/test/CodeGen/attr-noundef.cpp
@@ -27,7 +27,7 @@ struct NoCopy {
 NoCopy ret_nocopy() { return {}; }
 void pass_nocopy(NoCopy e) {}
 // CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr noundef %
+// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr dead_on_return noundef %
 
 struct Huge {
   int a[1024];
@@ -35,7 +35,8 @@ struct Huge {
 Huge ret_huge() { return {}; }
 void pass_huge(Huge h) {}
 // CHECK: [[DEF]] void @{{.*}}ret_huge{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr noundef
+// CHECK-INTEL: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr noundef
+// CHECK-AARCH: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr dead_on_return noundef
 } // namespace check_structs
 
 //************ Passing unions by value
@@ -59,7 +60,7 @@ union NoCopy {
 NoCopy ret_nocopy() { return {}; }
 void pass_nocopy(NoCopy e) {}
 // CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr noundef %
+// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr dead_on_return noundef %
 } // namespace check_unions
 
 //************ Passing `this` pointers
diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c
index cf74ab2dcca3f..fba692cac4492 100644
--- a/clang/test/CodeGen/cx-complex-range.c
+++ b/clang/test/CodeGen/cx-complex-range.c
@@ -1520,7 +1520,7 @@ void mulassignf(_Complex float *a, _Complex float b) {
 // PRMTD-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @divd(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -1744,7 +1744,7 @@ void mulassignf(_Complex float *a, _Complex float b) {
 // PRMTD_FAST-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divd(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -1938,7 +1938,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @divassignd(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2180,7 +2180,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divassignd(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2325,7 +2325,7 @@ void divassignd(_Complex double *a, _Complex double b) {
 // PRMTD-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @muld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -2457,7 +2457,7 @@ void divassignd(_Complex double *a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @muld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -2594,7 +2594,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulassignd(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2744,7 +2744,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulassignd(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2922,7 +2922,7 @@ void mulassignd(_Complex double *a, _Complex double b) {
 // PRMTD-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @divld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -3190,7 +3190,7 @@ void mulassignd(_Complex double *a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -3432,7 +3432,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @divassignld(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -3702,7 +3702,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divassignld(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -3895,7 +3895,7 @@ void divassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -4059,7 +4059,7 @@ void divassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -4220,7 +4220,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulassignld(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -4386,7 +4386,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulassignld(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -4644,7 +4644,7 @@ void mulassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD-NEXT:    ret <2 x float> [[RETVAL_SROA_0_4_VEC_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local i64 @f1(
-// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE]] to i32
 // X86WINPRMTD-NEXT:    [[TMP0:%.*]] = bitcast i32 [[A_SROA_0_0_EXTRACT_TRUNC]] to float
@@ -5052,7 +5052,7 @@ void mulassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret <2 x float> [[RETVAL_SROA_0_4_VEC_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local i64 @f1(
-// X86WINPRMTD_STRICT-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: i64 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE]] to i32
 // X86WINPRMTD_STRICT-NEXT:    [[TMP0:%.*]] = bitcast i32 [[A_SROA_0_0_EXTRACT_TRUNC]] to float
diff --git a/clang/test/CodeGen/ext-int-cc.c b/clang/test/CodeGen/ext-int-cc.c
index fdca4012ee4a4..7cfd992fd48b4 100644
--- a/clang/test/CodeGen/ext-int-cc.c
+++ b/clang/test/CodeGen/ext-int-cc.c
@@ -33,9 +33,9 @@
 // Make sure 128 and 64 bit versions are passed like integers.
 void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
 // LIN64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// WIN64: define dso_local void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
-// WIN32: define dso_local void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// WIN64: define dso_local void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
+// WIN32: define dso_local void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // NVPTX64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // NVPTX: define{{.*}} void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // SPARCV9: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
@@ -50,22 +50,22 @@ void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
 // ARC: define{{.*}} void @ParamPassing(ptr byval(i128) align 4 %{{.+}}, i64 inreg %{{.+}})
 // XCORE: define{{.*}} void @ParamPassing(ptr byval(i128) align 4 %{{.+}}, i64 %{{.+}})
 // RISCV64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// RISCV32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// RISCV32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // WASM: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// SYSTEMZ: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// SYSTEMZ: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // PPC64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // PPC32: define{{.*}} void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // AARCH64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // AARCH64DARWIN: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // ARM: define{{.*}} arm_aapcscc void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // LA64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// LA32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// LA32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 
 void ParamPassing2(_BitInt(127) b, _BitInt(63) c) {}
 // LIN64: define{{.*}} void @ParamPassing2(i64 %{{.+}}, i64 %{{.+}}, i64 %{{.+}})
-// WIN64: define dso_local void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
-// WIN32: define dso_local void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// WIN64: define dso_local void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
+// WIN32: define dso_local void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 // NVPTX64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // NVPTX: define{{.*}} void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // SPARCV9: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
@@ -80,16 +80,16 @@ void ParamPassing2(_BitInt(127) b, _BitInt(63) c) {}
 // ARC: define{{.*}} void @ParamPassing2(ptr byval(i128) align 4 %{{.+}}, i63 inreg %{{.+}})
 // XCORE: define{{.*}} void @ParamPassing2(ptr byval(i128) align 4 %{{.+}}, i63 %{{.+}})
 // RISCV64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
-// RISCV32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// RISCV32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 // WASM: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
-// SYSTEMZ: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 signext %{{.+}})
+// SYSTEMZ: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 signext %{{.+}})
 // PPC64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
 // PPC32: define{{.*}} void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // AARCH64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // AARCH64DARWIN: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // ARM: define{{.*}} arm_aapcscc void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // LA64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
-// LA32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// LA32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 
 // Make sure we follow the signext rules for promotable integer types.
 void ParamPassing3(_BitInt(15) a, _BitInt(31) b) {}
@@ -129,10 +129,10 @@ void ParamPassing3(_BitInt(15) a, _BitInt(31) b) {}
 // _BitInt widths to alert us to enable the test.
 void ParamPassing4(_BitInt(129) a) {}
 // LIN64: define{{.*}} void @ParamPassing4(ptr byval([24 x i8]) align 8 %{{.+}})
-// WIN64: define dso_local void @ParamPassing4(ptr %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// WIN32: define dso_local void @ParamPassing4(ptr %{{.+}})
-// AARCH64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// WIN64: define dso_local void @ParamPassing4(ptr dead_on_return %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
+// WIN32: define dso_local void @ParamPassing4(ptr dead_on_return %{{.+}})
+// AARCH64: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
 // NVPTX64-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // NVPTX-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // SPARCV9-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
@@ -154,8 +154,8 @@ void ParamPassing4(_BitInt(129) a) {}
 // PPC32-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // AARCH64DARWIN-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // ARM-NOT: define{{.*}} arm_aapcscc void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
-// LA64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// LA32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// LA64: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
+// LA32: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
 #endif
 
 _BitInt(63) ReturnPassing(void) { return 0; }
diff --git a/clang/test/CodeGen/isfpclass.c b/clang/test/CodeGen/isfpclass.c
index 26dd846a2bf20..ee3a22b40fefd 100644
--- a/clang/test/CodeGen/isfpclass.c
+++ b/clang/test/CodeGen/isfpclass.c
@@ -160,7 +160,7 @@ int4 check_isfpclass_nan_strict_v4f32(float4 x) {
 }
 
 // CHECK-LABEL: define dso_local void @check_isfpclass_nan_v4f64
-// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X:%.*]] = load <4 x double>, ptr [[TMP0]], align 16, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <4 x double> [[X]], zeroinitializer
diff --git a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
index eb706154300a2..0124cc5c06d43 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
+++ b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
@@ -30,37 +30,37 @@ long double powl(long double a, long double b);
 //
 // CHECK-PPC-LABEL: define dso_local ppc_fp128 @test_powl(
 // CHECK-PPC-SAME: ppc_fp128 noundef [[A:%.*]], ppc_fp128 noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @powl(ppc_fp128 noundef [[A]], ppc_fp128 noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @powl(ppc_fp128 noundef [[A]], ppc_fp128 noundef [[B]]) #[[ATTR4:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-ARM-LABEL: define dso_local double @test_powl(
 // CHECK-ARM-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-ARM:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-ARM-HF-LABEL: define dso_local double @test_powl(
 // CHECK-ARM-HF-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-THUMB-LABEL: define double @test_powl(
 // CHECK-THUMB-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-THUMB:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-AARCH-LABEL: define dso_local fp128 @test_powl(
 // CHECK-AARCH-SAME: fp128 noundef [[A:%.*]], fp128 noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @powl(fp128 noundef [[A]], fp128 noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @powl(fp128 noundef [[A]], fp128 noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func double @test_powl(
 // CHECK-SPIR-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR4:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-MINGW32-LABEL: define dso_local void @test_powl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]], ptr noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3:![0-9]+]]
-// CHECK-MINGW32:    [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[B]], ptr [[BYVAL_TEMP1:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    call void @powl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr noundef nonnull [[BYVAL_TEMP]], ptr noundef nonnull [[BYVAL_TEMP1]]) #[[ATTR3:[0-9]+]]
-// CHECK-MINGW32:    [[TMP2:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[TMP2]], ptr [[AGG_RESULT]], align 16, !tbaa [[TBAA3]]
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA6:![0-9]+]]
+// CHECK-MINGW32:    [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[B]], ptr [[BYVAL_TEMP1:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    call void @powl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP1]]) #[[ATTR3:[0-9]+]]
+// CHECK-MINGW32:    [[TMP2:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[TMP2]], ptr [[AGG_RESULT]], align 16, !tbaa [[TBAA6]]
 //
 long double test_powl(long double a, long double b) {
    return powl(a, b);
@@ -93,51 +93,51 @@ long double test_powl(long double a, long double b) {
 // CHECK-I686:    store x86_fp80 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 4
 //
 // CHECK-PPC-LABEL: define dso_local void @test_cargl(
-// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-PPC:    [[CLD_REAL:%.*]] = load ppc_fp128, ptr [[CLD]], align 16
 // CHECK-PPC:    [[CLD_IMAG:%.*]] = load ppc_fp128, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-PPC:    store ppc_fp128 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
 // CHECK-PPC:    store ppc_fp128 [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 16
-// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @cargl(ptr noundef nonnull byval({ ppc_fp128, ppc_fp128 }) align 16 [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @cargl(ptr noundef nonnull byval({ ppc_fp128, ppc_fp128 }) align 16 [[BYVAL_TEMP]]) #[[ATTR4]]
 // CHECK-PPC:    store ppc_fp128 [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 16
 // CHECK-PPC:    store ppc_fp128 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 16
 //
 // CHECK-ARM-LABEL: define dso_local void @test_cargl(
-// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-ARM:    [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 // CHECK-ARM:    store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
 // CHECK-ARM:    store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
 //
 // CHECK-ARM-HF-LABEL: define dso_local { double, double } @test_cargl(
-// CHECK-ARM-HF-SAME: { double, double } noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @cargl({ double, double } noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-HF-SAME: { double, double } noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @cargl({ double, double } noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-THUMB-LABEL: define { double, double } @test_cargl(
-// CHECK-THUMB-SAME: [2 x double] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call double @cargl([2 x double] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-THUMB-SAME: [2 x double] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-THUMB:    [[CALL:%.*]] = tail call double @cargl([2 x double] noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-AARCH-LABEL: define dso_local { fp128, fp128 } @test_cargl(
-// CHECK-AARCH-SAME: [2 x fp128] noundef alignstack(16) [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA2]]
+// CHECK-AARCH-SAME: [2 x fp128] noundef alignstack(16) [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA2]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func void @test_cargl(
-// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-SPIR:    [[CLD_REAL:%.*]] = load double, ptr [[CLD]], align 8
 // CHECK-SPIR:    [[CLD_IMAG:%.*]] = load double, ptr [[CLD_IMAGP:%.*]], align 8
 // CHECK-SPIR:    store double [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 8
 // CHECK-SPIR:    store double [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 8
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @cargl(ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @cargl(ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]]) #[[ATTR4]]
 // CHECK-SPIR:    store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
 // CHECK-SPIR:    store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
 //
 // CHECK-MINGW32-LABEL: define dso_local void @test_cargl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-MINGW32:    [[CLD_REAL:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK-MINGW32:    [[CLD_IMAG:%.*]] = load x86_fp80, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 16
-// CHECK-MINGW32:    call void @cargl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
-// CHECK-MINGW32:    [[TMP0:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA3]]
+// CHECK-MINGW32:    call void @cargl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-MINGW32:    [[TMP0:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA6]]
 // CHECK-MINGW32:    [[CLD_REAL3:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK-MINGW32:    [[CLD_IMAG5:%.*]] = load x86_fp80, ptr [[CLD_IMAGP]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 16
@@ -166,33 +166,33 @@ int ilogbl(long double a);
 //
 // CHECK-PPC-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-PPC-SAME: ppc_fp128 noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-PPC:    [[CALL:%.*]] = tail call i32 @ilogbl(ppc_fp128 noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call i32 @ilogbl(ppc_fp128 noundef [[A]]) #[[ATTR4]], !tbaa [[TBAA2]]
 //
 // CHECK-ARM-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-ARM-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-ARM-HF-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-ARM-HF-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-THUMB-LABEL: define i32 @test_ilogb(
 // CHECK-THUMB-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-THUMB:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-AARCH-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-AARCH-SAME: fp128 noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call i32 @ilogbl(fp128 noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA2]]
+// CHECK-AARCH:    [[CALL:%.*]] = tail call i32 @ilogbl(fp128 noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func i32 @test_ilogb(
 // CHECK-SPIR-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR4]], !tbaa [[TBAA2]]
 //
 // CHECK-MINGW32-LABEL: define dso_local i32 @test_ilogb(
-// CHECK-MINGW32-SAME: ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    [[CALL:%.*]] = call i32 @ilogbl(ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-MINGW32-SAME: ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    [[CALL:%.*]] = call i32 @ilogbl(ptr dead_on_return noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
 //
 int test_ilogb(long double a) {
    return ilogbl(a);
@@ -243,8 +243,8 @@ int test_ilogb(long double a) {
 // CHECK-SPIR: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
 // CHECK-SPIR: [[META5]] = !{!"Simple C/C++ TBAA"}
 //.
-// CHECK-MINGW32: [[TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
-// CHECK-MINGW32: [[META4]] = !{!"long double", [[META5:![0-9]+]], i64 0}
-// CHECK-MINGW32: [[META5]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
-// CHECK-MINGW32: [[META6]] = !{!"Simple C/C++ TBAA"}
+// CHECK-MINGW32: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+// CHECK-MINGW32: [[META7]] = !{!"long double", [[META8:![0-9]+]], i64 0}
+// CHECK-MINGW32: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+// CHECK-MINGW32: [[META9]] = !{!"Simple C/C++ TBAA"}
 //.
diff --git a/clang/test/CodeGen/mingw-long-double.c b/clang/test/CodeGen/mingw-long-double.c
index 0fc8f01509682..b98929701bc62 100644
--- a/clang/test/CodeGen/mingw-long-double.c
+++ b/clang/test/CodeGen/mingw-long-double.c
@@ -29,15 +29,15 @@ long double TestLD(long double x) {
   return x * x;
 }
 // GNU32: define dso_local x86_fp80 @TestLD(x86_fp80 noundef %x)
-// GNU64: define dso_local void @TestLD(ptr dead_on_unwind noalias writable sret(x86_fp80) align 16 %agg.result, ptr noundef %0)
+// GNU64: define dso_local void @TestLD(ptr dead_on_unwind noalias writable sret(x86_fp80) align 16 %agg.result, ptr dead_on_return noundef %0)
 // MSC64: define dso_local double @TestLD(double noundef %x)
 
 long double _Complex TestLDC(long double _Complex x) {
   return x * x;
 }
 // GNU32: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 4 %agg.result, ptr noundef byval({ x86_fp80, x86_fp80 }) align 4 %x)
-// GNU64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 16 %agg.result, ptr noundef %x)
-// MSC64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %agg.result, ptr noundef %x)
+// GNU64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 16 %agg.result, ptr dead_on_return noundef %x)
+// MSC64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %agg.result, ptr dead_on_return noundef %x)
 
 // GNU32: declare dso_local void @__mulxc3
 // GNU64: declare dso_local void @__mulxc3
diff --git a/clang/test/CodeGen/ms_abi.c b/clang/test/CodeGen/ms_abi.c
index 528e546f315d5..5d58c9816da78 100644
--- a/clang/test/CodeGen/ms_abi.c
+++ b/clang/test/CodeGen/ms_abi.c
@@ -142,7 +142,7 @@ struct i128 {
 };
 
 __attribute__((ms_abi)) struct i128 f7(struct i128 a) {
-  // WIN64: define dso_local void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr noundef %a)
-  // FREEBSD: define{{.*}} win64cc void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr noundef %a)
+  // WIN64: define dso_local void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr dead_on_return noundef %a)
+  // FREEBSD: define{{.*}} win64cc void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr dead_on_return noundef %a)
   return a;
 }
diff --git a/clang/test/CodeGen/pass-by-value-noalias.c b/clang/test/CodeGen/pass-by-value-noalias.c
index bc35d13c4df6a..e673ceb80bebe 100644
--- a/clang/test/CodeGen/pass-by-value-noalias.c
+++ b/clang/test/CodeGen/pass-by-value-noalias.c
@@ -11,6 +11,6 @@ struct Foo {
   int f;
 };
 
-// WITH_NOALIAS: define{{.*}} void @take(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @take(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
 void take(struct Foo arg) {}
diff --git a/clang/test/CodeGen/ptrauth-in-c-struct.c b/clang/test/CodeGen/ptrauth-in-c-struct.c
index 2aec31ec3baf9..c74be17b4c837 100644
--- a/clang/test/CodeGen/ptrauth-in-c-struct.c
+++ b/clang/test/CodeGen/ptrauth-in-c-struct.c
@@ -115,7 +115,7 @@ void test_move_assignment_SA(SA *p) {
   *p = getSA();
 }
 
-// CHECK: define void @test_parameter_SA(ptr noundef %{{.*}})
+// CHECK: define void @test_parameter_SA(ptr dead_on_return noundef %{{.*}})
 // CHECK-NOT: call
 // CHECK: ret void
 
@@ -128,7 +128,7 @@ void test_parameter_SA(SA a) {
 // CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
 // CHECK: call void @__copy_constructor_8_8_t0w4_pa1_50_8(ptr %[[AGG_TMP]], ptr %[[V0]])
-// CHECK: call void @calleeSA(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @calleeSA(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK-NOT: call
 // CHECK: ret void
 
diff --git a/clang/test/CodeGen/regcall.c b/clang/test/CodeGen/regcall.c
index f10da87353fa1..d4b9f00d54d41 100644
--- a/clang/test/CodeGen/regcall.c
+++ b/clang/test/CodeGen/regcall.c
@@ -28,7 +28,7 @@ struct Large { int a[5]; };
 void __regcall v4(int a, struct Large b, int c) {}
 // Win32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // Lin32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
-// Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 // Lin64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
 
 void __regcall v5(long long a, int b, int c) {}
@@ -47,7 +47,7 @@ void __regcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __regcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_regcallcc void @__regcall3__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall3__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall3__hfa2(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
@@ -61,7 +61,7 @@ void __regcall hfa3(double a, double b, double c, double d, double e, struct HFA
 // handling to ensure alignment.
 void __regcall hfa4(struct HFA5 a) {}
 // X32: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr noundef byval(%struct.HFA5) align 4 %{{.*}})
-// Win64: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr noundef %a)
+// Win64: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr dead_on_return noundef %a)
 // Lin64: define dso_local x86_regcallcc void @__regcall3__hfa4(double %a.coerce0, double %a.coerce1, double %a.coerce2, double %a.coerce3, double %a.coerce4)
 
 // Return HFAs of 4 or fewer elements in registers.
@@ -79,7 +79,7 @@ void __regcall hva1(int a, struct HVA4 b, int c) {}
 // X64: define dso_local x86_regcallcc void @__regcall3__hva1(i32 noundef %a, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 noundef %c)
 
 void __regcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {}
-// X86: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> noundef %c)
 
 void __regcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {}
@@ -95,6 +95,6 @@ void __regcall odd_size_hva(struct OddSizeHVA a) {}
 
 struct HFA6 { __m128 f[4]; };
 struct HFA6 __regcall ret_reg_reused(struct HFA6 a, struct HFA6 b, struct HFA6 c, struct HFA6 d){ struct HFA6 h; return h;}
-// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %c, ptr inreg noundef %d)
+// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %c, ptr dead_on_return inreg noundef %d)
 // Win64: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, <4 x float> %c.0, <4 x float> %c.1, <4 x float> %c.2, <4 x float> %c.3, <4 x float> %d.0, <4 x float> %d.1, <4 x float> %d.2, <4 x float> %d.3)
 // Lin64: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused([4 x <4 x float>] %a.coerce, [4 x <4 x float>] %b.coerce, [4 x <4 x float>] %c.coerce, [4 x <4 x float>] %d.coerce)
diff --git a/clang/test/CodeGen/regcall2.c b/clang/test/CodeGen/regcall2.c
index c88d4e485b104..42619369677b2 100644
--- a/clang/test/CodeGen/regcall2.c
+++ b/clang/test/CodeGen/regcall2.c
@@ -20,7 +20,7 @@ double __regcall bar(__sVector a) {
 
 // FIXME: Do we need to change for Windows?
 // Win: define dso_local x86_regcallcc void @__regcall3__foo(ptr dead_on_unwind noalias writable sret(%struct.__sVector) align 64 %agg.result, i32 noundef %a) #0
-// Win: define dso_local x86_regcallcc double @__regcall3__bar(ptr noundef %a) #0
+// Win: define dso_local x86_regcallcc double @__regcall3__bar(ptr dead_on_return noundef %a) #0
 // Win: attributes #0 = { noinline nounwind optnone "min-legal-vector-width"="0" "no-builtins" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+avx,+avx2,+avx512f,+avx512vl,+crc32,+cx8,+evex512,+f16c,+fma,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" }
 
 // Lin: define dso_local x86_regcallcc %struct.__sVector @__regcall3__foo(i32 noundef %a) #0
diff --git a/clang/test/CodeGen/regcall4.c b/clang/test/CodeGen/regcall4.c
index 5fbe77fbc7d76..d5fe5d88a0e8c 100644
--- a/clang/test/CodeGen/regcall4.c
+++ b/clang/test/CodeGen/regcall4.c
@@ -28,7 +28,7 @@ struct Large { int a[5]; };
 void __regcall v4(int a, struct Large b, int c) {}
 // Win32: define dso_local x86_regcallcc void @__regcall4__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // Lin32: define dso_local x86_regcallcc void @__regcall4__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
-// Win64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// Win64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 // Lin64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
 
 void __regcall v5(long long a, int b, int c) {}
@@ -47,7 +47,7 @@ void __regcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __regcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_regcallcc void @__regcall4__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall4__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall4__hfa2(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
@@ -61,7 +61,7 @@ void __regcall hfa3(double a, double b, double c, double d, double e, struct HFA
 // handling to ensure alignment.
 void __regcall hfa4(struct HFA5 a) {}
 // X32: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr noundef byval(%struct.HFA5) align 4 %{{.*}})
-// Win64: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr noundef %a)
+// Win64: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr dead_on_return noundef %a)
 // Lin64: define dso_local x86_regcallcc void @__regcall4__hfa4(double %a.coerce0, double %a.coerce1, double %a.coerce2, double %a.coerce3, double %a.coerce4)
 
 // Return HFAs of 4 or fewer elements in registers.
@@ -79,7 +79,7 @@ void __regcall hva1(int a, struct HVA4 b, int c) {}
 // X64: define dso_local x86_regcallcc void @__regcall4__hva1(i32 noundef %a, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 noundef %c)
 
 void __regcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {}
-// X86: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> noundef %c)
 
 void __regcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {}
@@ -95,6 +95,6 @@ void __regcall odd_size_hva(struct OddSizeHVA a) {}
 
 struct HFA6 { __m128 f[4]; };
 struct HFA6 __regcall ret_reg_reused(struct HFA6 a, struct HFA6 b, struct HFA6 c, struct HFA6 d){ struct HFA6 h; return h;}
-// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %c, ptr inreg noundef %d)
+// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %c, ptr dead_on_return inreg noundef %d)
 // Win64: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, <4 x float> %c.0, <4 x float> %c.1, <4 x float> %c.2, <4 x float> %c.3, <4 x float> %d.0, <4 x float> %d.1, <4 x float> %d.2, <4 x float> %d.3)
 // Lin64: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused([4 x <4 x float>] %a.coerce, [4 x <4 x float>] %b.coerce, [4 x <4 x float>] %c.coerce, [4 x <4 x float>] %d.coerce)
diff --git a/clang/test/CodeGen/sparcv9-abi.c b/clang/test/CodeGen/sparcv9-abi.c
index 616e24e7c519d..5a3d64fd37889 100644
--- a/clang/test/CodeGen/sparcv9-abi.c
+++ b/clang/test/CodeGen/sparcv9-abi.c
@@ -80,7 +80,7 @@ struct medium {
   int *c, *d;
 };
 
-// CHECK-LABEL: define{{.*}} %struct.medium @f_medium(ptr noundef %x)
+// CHECK-LABEL: define{{.*}} %struct.medium @f_medium(ptr dead_on_return noundef %x)
 struct medium f_medium(struct medium x) {
   x.a += *x.b;
   x.b = 0;
@@ -94,7 +94,7 @@ struct large {
   int x;
 };
 
-// CHECK-LABEL: define{{.*}} void @f_large(ptr dead_on_unwind noalias writable sret(%struct.large) align 8 %agg.result, ptr noundef %x)
+// CHECK-LABEL: define{{.*}} void @f_large(ptr dead_on_unwind noalias writable sret(%struct.large) align 8 %agg.result, ptr dead_on_return noundef %x)
 struct large f_large(struct large x) {
   x.a += *x.b;
   x.b = 0;
diff --git a/clang/test/CodeGen/vectorcall.c b/clang/test/CodeGen/vectorcall.c
index cab7fc0972d7b..09b3310c7c4c8 100644
--- a/clang/test/CodeGen/vectorcall.c
+++ b/clang/test/CodeGen/vectorcall.c
@@ -17,7 +17,7 @@ void __vectorcall v3(int a, struct Small b, int c) {}
 struct Large { int a[5]; };
 void __vectorcall v4(int a, struct Large b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01v4@@28"(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 
 void __vectorcall v5(long long a, int b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01v5@@16"(i64 noundef %a, i32 inreg noundef %b, i32 inreg noundef %c)
@@ -35,21 +35,21 @@ void __vectorcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __vectorcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr inreg noundef %b, double inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr noundef %b, double noundef %c)
+// X86: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, double inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr dead_on_return noundef %b, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
 // SSE register usage.
 void __vectorcall hfa3(double a, double b, double c, double d, double e, struct HFA2 f) {}
-// X86: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double inreg noundef %a, double inreg noundef %b, double inreg noundef %c, double inreg noundef %d, double inreg noundef %e, ptr inreg noundef %f)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double noundef %a, double noundef %b, double noundef %c, double noundef %d, double noundef %e, ptr noundef %f)
+// X86: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double inreg noundef %a, double inreg noundef %b, double inreg noundef %c, double inreg noundef %d, double inreg noundef %e, ptr dead_on_return inreg noundef %f)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double noundef %a, double noundef %b, double noundef %c, double noundef %d, double noundef %e, ptr dead_on_return noundef %f)
 
 // Aggregates with more than four elements are not HFAs and are passed byval.
 // Because they are not classified as homogeneous, they don't get special
 // handling to ensure alignment.
 void __vectorcall hfa4(struct HFA5 a) {}
 // X86: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr noundef byval(%struct.HFA5) align 4 %0)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr noundef %a)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr dead_on_return noundef %a)
 
 // Return HFAs of 4 or fewer elements in registers.
 static struct HFA2 g_hfa2;
@@ -68,26 +68,26 @@ v4f32 __vectorcall hva1(int a, struct HVA4 b, int c) {return b.w;}
 // X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva1@@80"(i32 noundef %a, %struct.HVA4 inreg %b.coerce, i32 noundef %c)
 
 v4f32 __vectorcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {return c;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c)
 
 v4f32 __vectorcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {return f.x;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> inreg noundef %a, <4 x float> inreg noundef %b, <4 x float> inreg noundef %c, <4 x float> inreg noundef %d, <4 x float> inreg noundef %e, ptr inreg noundef %f)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> noundef %a, <4 x float> noundef %b, <4 x float> noundef %c, <4 x float> noundef %d, <4 x float> noundef %e, ptr noundef %f)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> inreg noundef %a, <4 x float> inreg noundef %b, <4 x float> inreg noundef %c, <4 x float> inreg noundef %d, <4 x float> inreg noundef %e, ptr dead_on_return inreg noundef %f)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> noundef %a, <4 x float> noundef %b, <4 x float> noundef %c, <4 x float> noundef %d, <4 x float> noundef %e, ptr dead_on_return noundef %f)
 
 // vector types have higher priority then HVA structures, So vector types are allocated first
 // and HVAs are allocated if enough registers are available
 v4f32 __vectorcall hva4(struct HVA4 a, struct HVA2 b, v4f32 c) {return b.y;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c)
 
 v4f32 __vectorcall hva5(struct HVA3 a, struct HVA3 b, v4f32 c, struct HVA2 d) {return d.y;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c, %struct.HVA2 inreg %d.coerce)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c, %struct.HVA2 inreg %d.coerce)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c, %struct.HVA2 inreg %d.coerce)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c, %struct.HVA2 inreg %d.coerce)
 
 struct HVA4 __vectorcall hva6(struct HVA4 a, struct HVA4 b) { return b;}
-// X86: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b)
-// X64: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr noundef %b)
+// X86: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b)
+// X64: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b)
 
 struct HVA5 __vectorcall hva7(void) {struct HVA5 a = {}; return a;}
 // X86: define dso_local x86_vectorcallcc void @"\01hva7@@0"(ptr dead_on_unwind noalias writable sret(%struct.HVA5) align 16 %agg.result)
@@ -108,8 +108,8 @@ void __vectorcall odd_size_hva(struct OddSizeHVA a) {}
 // consider 'p7' as a register.  Instead p5 gets put into the register on the second pass.
 // x86 should pass p2, p6 and p7 in registers, then p1 in the second pass.
 struct HFA2 __vectorcall AddParticles(struct HFA2 p1, float p2, struct HFA4 p3, int p4, struct HFA2 p5, float p6, float p7, int p8){ return p1;}
-// X86: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@84"(%struct.HFA2 inreg %p1.coerce, float inreg noundef %p2, ptr inreg noundef %p3, i32 inreg noundef %p4, ptr noundef %p5, float inreg noundef %p6, float inreg noundef %p7, i32 noundef %p8)
-// X64: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@104"(%struct.HFA2 inreg %p1.coerce, float noundef %p2, ptr noundef %p3, i32 noundef %p4, %struct.HFA2 inreg %p5.coerce, float noundef %p6, float noundef %p7, i32 noundef %p8)
+// X86: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@84"(%struct.HFA2 inreg %p1.coerce, float inreg noundef %p2, ptr dead_on_return inreg noundef %p3, i32 inreg noundef %p4, ptr dead_on_return noundef %p5, float inreg noundef %p6, float inreg noundef %p7, i32 noundef %p8)
+// X64: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@104"(%struct.HFA2 inreg %p1.coerce, float noundef %p2, ptr dead_on_return noundef %p3, i32 noundef %p4, %struct.HFA2 inreg %p5.coerce, float noundef %p6, float noundef %p7, i32 noundef %p8)
 
 // Vectorcall in both architectures allows passing of an HVA as long as there is room,
 // even if it is not one of the first 6 arguments.  First pass puts p4 into a
@@ -117,8 +117,8 @@ struct HFA2 __vectorcall AddParticles(struct HFA2 p1, float p2, struct HFA4 p3,
 // in a register, does NOT put p7 in a register (since there's no room), then puts
 // p8 in a register.
 void __vectorcall HVAAnywhere(struct HFA2 p1, int p2, int p3, float p4, int p5, int p6, struct HFA4 p7, struct HFA2 p8, float p9){}
-// X86: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@88"(%struct.HFA2 inreg %p1.coerce, i32 inreg noundef %p2, i32 inreg noundef %p3, float inreg noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr noundef %p7, %struct.HFA2 inreg %p8.coerce, float inreg noundef %p9)
-// X64: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@112"(%struct.HFA2 inreg %p1.coerce, i32 noundef %p2, i32 noundef %p3, float noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr noundef %p7, %struct.HFA2 inreg %p8.coerce, float noundef %p9)
+// X86: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@88"(%struct.HFA2 inreg %p1.coerce, i32 inreg noundef %p2, i32 inreg noundef %p3, float inreg noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr dead_on_return noundef %p7, %struct.HFA2 inreg %p8.coerce, float inreg noundef %p9)
+// X64: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@112"(%struct.HFA2 inreg %p1.coerce, i32 noundef %p2, i32 noundef %p3, float noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr dead_on_return noundef %p7, %struct.HFA2 inreg %p8.coerce, float noundef %p9)
 
 #ifndef __x86_64__
 // This covers the three ways XMM values can be passed on 32-bit x86:
@@ -137,9 +137,9 @@ void __vectorcall vectorcall_indirect_vec(
 // X86-SAME: double inreg noundef %xmm3,
 // X86-SAME: double inreg noundef %xmm4,
 // X86-SAME: <4 x float> inreg noundef %xmm5,
-// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: ptr dead_on_return inreg noundef %0,
 // X86-SAME: i32 inreg noundef %edx,
-// X86-SAME: ptr noundef %1)
+// X86-SAME: ptr dead_on_return noundef %1)
 
 void __vectorcall vectorcall_indirect_fp(
     double xmm0, double xmm1, double xmm2, double xmm3, double xmm4,
@@ -153,7 +153,7 @@ void __vectorcall vectorcall_indirect_fp(
 // X86-SAME: double inreg noundef %xmm3,
 // X86-SAME: double inreg noundef %xmm4,
 // X86-SAME: <4 x float> inreg noundef %xmm5,
-// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: ptr dead_on_return inreg noundef %0,
 // X86-SAME: i32 inreg noundef %edx,
 // X86-SAME: double noundef %mem)
 #endif
diff --git a/clang/test/CodeGen/win-fp128.c b/clang/test/CodeGen/win-fp128.c
index 328a7aaa7df57..8d223741bc93e 100644
--- a/clang/test/CodeGen/win-fp128.c
+++ b/clang/test/CodeGen/win-fp128.c
@@ -6,7 +6,7 @@ __float128 fp128_ret(void) { return 0; }
 // CHECK-GNU64: define dso_local <2 x i64>  @fp128_ret()
 
 __float128 fp128_args(__float128 a, __float128 b) { return a * b; }
-// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr noundef %0, ptr noundef %1)
+// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 
 void fp128_vararg(int a, ...) {
   // CHECK-GNU64-LABEL: define dso_local void @fp128_vararg
diff --git a/clang/test/CodeGen/win64-i128.c b/clang/test/CodeGen/win64-i128.c
index e10b2be0530eb..2d83889d8f89b 100644
--- a/clang/test/CodeGen/win64-i128.c
+++ b/clang/test/CodeGen/win64-i128.c
@@ -12,8 +12,8 @@ int128_t foo(void) { return 0; }
 
 int128_t bar(int128_t a, int128_t b) { return a * b; }
 
-// GNU64: define dso_local <2 x i64> @bar(ptr noundef %0, ptr noundef %1)
-// MSC64: define dso_local <2 x i64> @bar(ptr noundef %0, ptr noundef %1)
+// GNU64: define dso_local <2 x i64> @bar(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// MSC64: define dso_local <2 x i64> @bar(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 
 void vararg(int a, ...) {
   // GNU64-LABEL: define{{.*}} void @vararg
diff --git a/clang/test/CodeGen/windows-swiftcall.c b/clang/test/CodeGen/windows-swiftcall.c
index 41569c2606622..8716f25b9ddfb 100644
--- a/clang/test/CodeGen/windows-swiftcall.c
+++ b/clang/test/CodeGen/windows-swiftcall.c
@@ -219,7 +219,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define {{.*}} void @return_struct_big_1({{.*}} dead_on_unwind noalias writable sret
 
 // Should not be byval.
-// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr noundef{{( %.*)?}})
+// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr dead_on_return noundef{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
index 152be26948f28..1709c88563267 100644
--- a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
+++ b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
@@ -576,7 +576,7 @@ void f(__clang_svmfloat8x4_t, __clang_svmfloat8x4_t);
 // CHECK-NEXT:    [[COERCE74_EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 2
 // CHECK-NEXT:    [[COERCE74_EXTRACT3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 3
 // CHECK-NEXT:    store { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[BYVAL_TEMP]], align 2
-// CHECK-NEXT:    call void @_Z1f10svboolx4_tS_(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    call void @_Z1f10svboolx4_tS_(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    store { <vscale x 16 x i8>, <vscale x 16 x i8> } zeroinitializer, ptr [[COERCE75]], align 16
 // CHECK-NEXT:    [[COERCE75_TUPLE:%.*]] = load { <vscale x 16 x i8>, <vscale x 16 x i8> }, ptr [[COERCE75]], align 16
 // CHECK-NEXT:    [[COERCE75_EXTRACT0:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[COERCE75_TUPLE]], 0
@@ -1125,7 +1125,7 @@ void f(__clang_svmfloat8x4_t, __clang_svmfloat8x4_t);
 // COMPAT_17-NEXT:    [[COERCE74_EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 2
 // COMPAT_17-NEXT:    [[COERCE74_EXTRACT3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 3
 // COMPAT_17-NEXT:    store { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[BYVAL_TEMP]], align 2
-// COMPAT_17-NEXT:    call void @_Z1f10svboolx4_t10svboolx4_t(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr noundef [[BYVAL_TEMP]])
+// COMPAT_17-NEXT:    call void @_Z1f10svboolx4_t10svboolx4_t(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // COMPAT_17-NEXT:    store { <vscale x 16 x i8>, <vscale x 16 x i8> } zeroinitializer, ptr [[COERCE75]], align 16
 // COMPAT_17-NEXT:    [[COERCE75_TUPLE:%.*]] = load { <vscale x 16 x i8>, <vscale x 16 x i8> }, ptr [[COERCE75]], align 16
 // COMPAT_17-NEXT:    [[COERCE75_EXTRACT0:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[COERCE75_TUPLE]], 0
diff --git a/clang/test/CodeGenCXX/arm-cc.cpp b/clang/test/CodeGenCXX/arm-cc.cpp
index 68e1b7e4e1e46..939615fcc69e0 100644
--- a/clang/test/CodeGenCXX/arm-cc.cpp
+++ b/clang/test/CodeGenCXX/arm-cc.cpp
@@ -17,4 +17,4 @@ void baz() {
 }
 
 // CHECK: declare void @_Z3fooPv(ptr dead_on_unwind writable sret(%class.SMLoc) align 4, ptr noundef)
-// CHECK: declare void @_Z3zed5SMLoc(ptr noundef)
+// CHECK: declare void @_Z3zed5SMLoc(ptr dead_on_return noundef)
diff --git a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
index c341bd2b855ff..f2d602b3b523e 100644
--- a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
+++ b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
@@ -55,18 +55,18 @@ void usage() {
 // WINDOWS-NEXT: ret i32 %[[RET]]
 
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: ret i32 %[[LOAD]]
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 1
 // WINDOWS64: ret i32 %[[ADD]]
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 2
@@ -75,7 +75,7 @@ void usage() {
 // WINDOWS64: define dso_local void @"?usage@@YAXXZ"()
 // WINDOWS64: %[[F:[0-9a-zA-Z]+]] = alloca %struct.Foo
 // WINDOWS64: %[[ARG:[0-9a-zA-Z.]+]] = alloca %struct.Foo
-// WINDOWS64: %[[CALL:[0-9a-zA-Z]+]] = call noundef i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr noundef %[[ARG]])
+// WINDOWS64: %[[CALL:[0-9a-zA-Z]+]] = call noundef i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr dead_on_return noundef %[[ARG]])
 
 // WINDOWS64: define weak_odr dso_local i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr %0)
 // WINDOWS64: %[[RET:[0-9a-zA-Z]+]] = musttail call i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr %0)
diff --git a/clang/test/CodeGenCXX/copy-initialization.cpp b/clang/test/CodeGenCXX/copy-initialization.cpp
index aa0c6395f158d..4e6194cc040c2 100644
--- a/clang/test/CodeGenCXX/copy-initialization.cpp
+++ b/clang/test/CodeGenCXX/copy-initialization.cpp
@@ -12,7 +12,7 @@ struct Bar {
 
 void f(Foo);
 
-// CHECK-LABEL: define{{.*}} void @_Z1g3Foo(ptr noundef %foo)
+// CHECK-LABEL: define{{.*}} void @_Z1g3Foo(ptr dead_on_return noundef %foo)
 void g(Foo foo) {
   // CHECK: call void @_ZN3BarC1Ev
   // CHECK: @_ZNK3BarcvRK3FooEv
diff --git a/clang/test/CodeGenCXX/debug-info.cpp b/clang/test/CodeGenCXX/debug-info.cpp
index 8594a897ef7c0..9cf26ba83ba3e 100644
--- a/clang/test/CodeGenCXX/debug-info.cpp
+++ b/clang/test/CodeGenCXX/debug-info.cpp
@@ -4,7 +4,7 @@
 // CHECK: @_ZN6pr96081xE ={{.*}} global ptr null, align 8, !dbg [[X:![0-9]+]]
 
 // CHECK: define{{.*}} void @_ZN7pr147634funcENS_3fooE
-// CHECK-SAME: ptr noundef [[param:%.*]])
+// CHECK-SAME: ptr dead_on_return noundef [[param:%.*]])
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   alloca ptr, align 8
 // CHECK-NEXT:   [[param_addr_storage:%.*]] = alloca ptr, align 8
diff --git a/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp b/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
index c7d3a017414ef..c8f5a0f7c2ea6 100644
--- a/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
+++ b/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
@@ -13,7 +13,7 @@ struct Empty {
 };
 
 bool foo(Empty e) {
-// CHECK: @_Z3foo5Empty(ptr noundef %e)
+// CHECK: @_Z3foo5Empty(ptr dead_on_return noundef %e)
 // CHECK: call {{.*}} @_ZN5Empty5checkEv(ptr {{[^,]*}} %e)
   return e.check();
 }
@@ -21,6 +21,6 @@ bool foo(Empty e) {
 void caller(Empty &e) {
 // CHECK: @_Z6callerR5Empty(ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %e)
 // CHECK: call {{.*}} @_ZN5EmptyC1ERKS_(ptr {{[^,]*}} [[NEWTMP:%.*]], ptr
-// CHECK: call {{.*}} @_Z3foo5Empty(ptr noundef [[NEWTMP]])
+// CHECK: call {{.*}} @_Z3foo5Empty(ptr dead_on_return noundef [[NEWTMP]])
   foo(e);
 }
diff --git a/clang/test/CodeGenCXX/fastcall.cpp b/clang/test/CodeGenCXX/fastcall.cpp
index 4c94c1623ee16..405917f7a14bb 100644
--- a/clang/test/CodeGenCXX/fastcall.cpp
+++ b/clang/test/CodeGenCXX/fastcall.cpp
@@ -15,6 +15,6 @@ struct S1 {
 void __attribute__((fastcall)) foo2(S1 a, int b);
 void bar2(S1 a, int b) {
   // CHECK-LABEL: define{{.*}} void @_Z4bar22S1i
-  // CHECK: call x86_fastcallcc void @_Z4foo22S1i(ptr inreg %{{.*}}, i32 inreg %
+  // CHECK: call x86_fastcallcc void @_Z4foo22S1i(ptr dead_on_return inreg %{{.*}}, i32 inreg %
   foo2(a, b);
 }
diff --git a/clang/test/CodeGenCXX/homogeneous-aggregates.cpp b/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
index 63ffc6b5bfac8..5ebeb8aad4c18 100644
--- a/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
+++ b/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
@@ -41,8 +41,8 @@ struct D5 : I1, I2, I3 {}; // homogeneous aggregate
 
 // PPC: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, [3 x i64] %x.coerce)
 // ARM32: define{{.*}} arm_aapcs_vfpcc void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, [3 x i64] %x.coerce)
-// ARM64: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr noundef %x)
-// X64: define dso_local x86_vectorcallcc void @"\01_Z7func_D12D1@@24"(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr noundef %x)
+// ARM64: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr dead_on_return noundef %x)
+// X64: define dso_local x86_vectorcallcc void @"\01_Z7func_D12D1@@24"(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr dead_on_return noundef %x)
 D1 CC func_D1(D1 x) { return x; }
 
 // PPC: define{{.*}} [3 x double] @_Z7func_D22D2([3 x double] %x.coerce)
@@ -53,7 +53,7 @@ D2 CC func_D2(D2 x) { return x; }
 
 // PPC: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, [4 x i64] %x.coerce)
 // ARM32: define{{.*}} arm_aapcs_vfpcc void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, [4 x i64] %x.coerce)
-// ARM64: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, ptr noundef %x)
+// ARM64: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, ptr dead_on_return noundef %x)
 D3 CC func_D3(D3 x) { return x; }
 
 // PPC: define{{.*}} [4 x double] @_Z7func_D42D4([4 x double] %x.coerce)
@@ -201,7 +201,7 @@ struct NonHFA {
   virtual void f1();
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at polymorphic@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at polymorphic@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace trivial_copy_assignment {
 struct HFA {
@@ -221,7 +221,7 @@ struct NonHFA {
   NonHFA &operator=(const NonHFA&);
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at non_trivial_copy_assignment@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at non_trivial_copy_assignment@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace user_provided_ctor {
 struct HFA {
@@ -251,7 +251,7 @@ struct NonHFA {
   ~NonHFA();
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at non_trivial_dtor@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at non_trivial_dtor@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace non_empty_base {
 struct non_empty_base { double d; };
@@ -272,7 +272,7 @@ struct NonHFA {
   empty e;
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at empty_field@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at empty_field@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace non_empty_field {
 struct non_empty { double d; };
diff --git a/clang/test/CodeGenCXX/inalloca-lambda.cpp b/clang/test/CodeGenCXX/inalloca-lambda.cpp
index dc78aa2773f89..0d527e1a02e5f 100644
--- a/clang/test/CodeGenCXX/inalloca-lambda.cpp
+++ b/clang/test/CodeGenCXX/inalloca-lambda.cpp
@@ -22,7 +22,7 @@ void test() {
 // CHECK: %[[V:.*]] = getelementptr inbounds nuw <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0
 // CHECK: %call = call x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %this, ptr noundef %[[V]])
+// CHECK-SAME: (ptr noundef %this, ptr dead_on_return noundef %[[V]])
 
 // CHECK: define internal noundef i32
 // CHECK-SAME: @"?__invoke@<lambda_0>@?0??test@@YAXXZ at CA?A?<auto>@@UA@@@Z"
@@ -31,12 +31,12 @@ void test() {
 // CHECK: %[[VAR:.*]] = getelementptr inbounds nuw <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0
 // CHECK: %call = call x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %unused.capture, ptr noundef %[[VAR]])
+// CHECK-SAME: (ptr noundef %unused.capture, ptr dead_on_return noundef %[[VAR]])
 // CHECK: ret i32 %call
 
 // CHECK: define internal x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %this, ptr noundef %[[ARG:.*]])
+// CHECK-SAME: (ptr noundef %this, ptr dead_on_return noundef %[[ARG:.*]])
 // CHECK: %this.addr = alloca ptr, align 4
 // CHECK: store ptr %this, ptr %this.addr, align 4
 // CHECK: %this1 = load ptr, ptr %this.addr, align 4
diff --git a/clang/test/CodeGenCXX/inalloca-overaligned.cpp b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
index 3751751ec0bca..305b8c5dccd10 100644
--- a/clang/test/CodeGenCXX/inalloca-overaligned.cpp
+++ b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
@@ -57,7 +57,7 @@ int receive_both(Both o) {
 }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?receive_both@@Y{{.*}}"
-// CHECK-SAME: (ptr noundef %o)
+// CHECK-SAME: (ptr dead_on_return noundef %o)
 
 int pass_both() {
   gvi32 = receive_both(Both());
@@ -67,7 +67,7 @@ int pass_both() {
 // CHECK-LABEL: define dso_local noundef i32 @"?pass_both@@Y{{.*}}"
 // CHECK: [[TMP:%[^ ]*]] = alloca %struct.Both, align 8
 // CHECK: call x86_thiscallcc noundef ptr @"??0Both@@QAE at XZ"(ptr {{[^,]*}} [[TMP]])
-// CHECK: call noundef i32 @"?receive_both@@Y{{.*}}"(ptr noundef [[TMP]])
+// CHECK: call noundef i32 @"?receive_both@@Y{{.*}}"(ptr dead_on_return noundef [[TMP]])
 
 int receive_inalloca_both(NonTrivial nt, Both o) {
   return nt.x + o.x + o.y;
@@ -101,11 +101,11 @@ struct [[trivial_abi]] alignas(8) MyPtr {
 int receiveMyPtr(MyPtr o) { return *o.ptr; }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?receiveMyPtr@@Y{{.*}}"
-// CHECK-SAME: (ptr noundef %o)
+// CHECK-SAME: (ptr dead_on_return noundef %o)
 
 int passMyPtr() { return receiveMyPtr(MyPtr()); }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?passMyPtr@@Y{{.*}}"
 // CHECK: [[TMP:%[^ ]*]] = alloca %struct.MyPtr, align 8
 // CHECK: call x86_thiscallcc noundef ptr @"??0MyPtr@@QAE at XZ"(ptr {{[^,]*}} [[TMP]])
-// CHECK: call noundef i32 @"?receiveMyPtr@@Y{{.*}}"(ptr noundef [[TMP]])
+// CHECK: call noundef i32 @"?receiveMyPtr@@Y{{.*}}"(ptr dead_on_return noundef [[TMP]])
diff --git a/clang/test/CodeGenCXX/inalloca-vector.cpp b/clang/test/CodeGenCXX/inalloca-vector.cpp
index d1bacb4f0dc8c..2db4c49df116a 100644
--- a/clang/test/CodeGenCXX/inalloca-vector.cpp
+++ b/clang/test/CodeGenCXX/inalloca-vector.cpp
@@ -56,7 +56,7 @@ void __fastcall fastcall_receive_vec(__m128 x, __m128 y, __m128 z, __m128 w, int
 // CHECK-SAME: (<4 x float> inreg noundef %x,
 // CHECK-SAME: <4 x float> inreg noundef %y,
 // CHECK-SAME: <4 x float> inreg noundef %z,
-// CHECK-SAME: ptr inreg noundef %0,
+// CHECK-SAME: ptr dead_on_return inreg noundef %0,
 // CHECK-SAME: i32 inreg noundef %edx,
 // CHECK-SAME: ptr inalloca(<{ ptr, %struct.NonTrivial }>) %1)
 
@@ -73,6 +73,6 @@ void __vectorcall vectorcall_receive_vec(double xmm0, double xmm1, double xmm2,
 // CHECK-SAME: <4 x float> inreg noundef %x,
 // CHECK-SAME: <4 x float> inreg noundef %y,
 // CHECK-SAME: <4 x float> inreg noundef %z,
-// CHECK-SAME: ptr inreg noundef %0,
+// CHECK-SAME: ptr dead_on_return inreg noundef %0,
 // CHECK-SAME: i32 inreg noundef %edx,
 // CHECK-SAME: ptr inalloca(<{ ptr, %struct.NonTrivial }>) %1)
diff --git a/clang/test/CodeGenCXX/inheriting-constructor.cpp b/clang/test/CodeGenCXX/inheriting-constructor.cpp
index 100ca269d7f3c..21751bea055dc 100644
--- a/clang/test/CodeGenCXX/inheriting-constructor.cpp
+++ b/clang/test/CodeGenCXX/inheriting-constructor.cpp
@@ -166,7 +166,7 @@ namespace inalloca_nonvirt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
-  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
 
@@ -202,7 +202,7 @@ namespace inalloca_nonvirt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
-  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
 }
@@ -253,7 +253,7 @@ namespace inalloca_virt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: br i1
-  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: br
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
@@ -302,7 +302,7 @@ namespace inalloca_virt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: br i1
   // WIN64: store {{.*}} @"??_8C at inalloca_virt@@7B@"
-  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
diff --git a/clang/test/CodeGenCXX/member-function-pointer-calls.cpp b/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
index ff511c0243801..f06cda8b7684e 100644
--- a/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
+++ b/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
@@ -16,7 +16,7 @@ int f(A* a, int (A::*fp)()) {
 // CHECK-NOT: }
 // CHECK: ret i32 1
 // MINGW64-LABEL: define dso_local noundef i32 @_Z2g1v()
-// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr noundef %{{.*}})
+// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr dead_on_return noundef %{{.*}})
 int g1() {
   A a;
   return f(&a, &A::vf1);
@@ -26,7 +26,7 @@ int g1() {
 // CHECK-NOT: }
 // CHECK: ret i32 2
 // MINGW64-LABEL: define dso_local noundef i32 @_Z2g2v()
-// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr noundef %{{.*}})
+// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr dead_on_return noundef %{{.*}})
 int g2() {
   A a;
   return f(&a, &A::vf2);
diff --git a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
index b551df747c073..63a4d5525336b 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
@@ -24,7 +24,7 @@ void foo(A a, A b, A c) {
 // X86: ret void
 
 // X64-LABEL: define dso_local void @"?foo@@YAXUA@@00 at Z"
-// X64:         (ptr noundef %[[a:[^,]*]], ptr noundef %[[b:[^,]*]], ptr noundef %[[c:[^)]*]])
+// X64:         (ptr dead_on_return noundef %[[a:[^,]*]], ptr dead_on_return noundef %[[b:[^,]*]], ptr dead_on_return noundef %[[c:[^)]*]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[a]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[b]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[c]])
@@ -64,7 +64,7 @@ void call_foo() {
 // X64: invoke noundef ptr @"??0A@@QEAA at H@Z"(ptr {{[^,]*}} %[[arg2:[^,]*]], i32 noundef 2)
 // X64: invoke noundef ptr @"??0A@@QEAA at H@Z"(ptr {{[^,]*}} %[[arg1:[^,]*]], i32 noundef 1)
 // X64: call void @"?foo@@YAXUA@@00 at Z"
-// X64:       (ptr noundef %[[arg1]], ptr noundef %[[arg2]], ptr noundef %[[arg3]])
+// X64:       (ptr dead_on_return noundef %[[arg1]], ptr dead_on_return noundef %[[arg2]], ptr dead_on_return noundef %[[arg3]])
 // X64: ret void
 //
 //   lpad2:
diff --git a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
index 5654db3ba8151..813abb03a7810 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
@@ -22,10 +22,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret void
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@byval_thunk@@W7EAAXUAgg at 2@@Z"
-// CHECK64:             (ptr noundef %this, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@byval_thunk@@UEAAXUAgg at 2@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
@@ -54,10 +54,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret void
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@stdcall_thunk@@W7EAAXUAgg at 2@@Z"
-// CHECK64:             (ptr noundef %this, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@stdcall_thunk@@UEAAXUAgg at 2@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
@@ -86,10 +86,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret ptr %[[rv]]
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@sret_thunk@@W7EAA?AUAgg at 2@U32@@Z"
-// CHECK64:             (ptr noundef %this, ptr dead_on_unwind noalias writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_unwind noalias writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@sret_thunk@@UEAA?AUAgg at 2@U32@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
diff --git a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
index 806bc5b63ef02..8defb68c668b2 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
@@ -428,7 +428,7 @@ bool nullTestDataUnspecified(int Unspecified::*mp) {
 
 // Pass this large type indirectly.
 // X64-LABEL: define dso_local noundef zeroext i1 @"?nullTestDataUnspecified@@
-// X64:             (ptr noundef %0)
+// X64:             (ptr dead_on_return noundef %0)
 }
 
 bool nullTestFunctionUnspecified(void (Unspecified::*mp)()) {
@@ -590,7 +590,7 @@ bool unspecFuncMemptrEq(void (Unspecified::*l)(), void (Unspecified::*r)()) {
 // CHECK: }
 
 // X64-LABEL: define dso_local noundef zeroext i1 @"?unspecFuncMemptrEq@@
-// X64:             (ptr noundef %0, ptr noundef %1)
+// X64:             (ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 }
 
 bool unspecFuncMemptrNeq(void (Unspecified::*l)(), void (Unspecified::*r)()) {
@@ -635,7 +635,7 @@ bool unspecDataMemptrEq(int Unspecified::*l, int Unspecified::*r) {
 // CHECK: }
 
 // X64-LABEL: define dso_local noundef zeroext i1 @"?unspecDataMemptrEq@@
-// X64:             (ptr noundef %0, ptr noundef %1)
+// X64:             (ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 }
 
 void (Multiple::*convertB2FuncToMultiple(void (B2::*mp)()))() {
diff --git a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
index 6e8ba3953b2cf..767bf168633ae 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
@@ -175,7 +175,7 @@ void multibyte_arg(Multibyte s) {}
 void packed_arg(Packed s) {}
 // LINUX-LABEL: define{{.*}} void @_Z10packed_arg6Packed(ptr noundef byval(%struct.Packed) align 4 %s)
 // WIN32: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr noundef byval(%struct.Packed) align 4 %s)
-// WIN64: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr dead_on_return noundef %s)
 
 // Test that dtors are invoked in the callee.
 void small_arg_with_dtor(SmallWithDtor s) {}
@@ -190,7 +190,7 @@ void small_arg_with_dtor(SmallWithDtor s) {}
 // WOA64: }
 
 // FIXME: MSVC incompatible!
-// WOA: define dso_local arm_aapcs_vfpcc void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(ptr noundef %s) {{.*}} {
+// WOA: define dso_local arm_aapcs_vfpcc void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(ptr dead_on_return noundef %s) {{.*}} {
 // WOA:   call arm_aapcs_vfpcc void @"??1SmallWithDtor@@QAA at XZ"(ptr {{[^,]*}} %s)
 // WOA: }
 
@@ -220,7 +220,7 @@ void ref_small_arg_with_dtor(const SmallWithDtor &s) { }
 // WIN64-LABEL: define dso_local void @"?ref_small_arg_with_dtor@@YAXAEBUSmallWithDtor@@@Z"(ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %s)
 
 void big_arg_with_dtor(BigWithDtor s) {}
-// WIN64-LABEL: define dso_local void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr noundef %s)
+// WIN64-LABEL: define dso_local void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr dead_on_return noundef %s)
 // WIN64:   call void @"??1BigWithDtor@@QEAA at XZ"
 // WIN64: }
 
@@ -231,7 +231,7 @@ void call_big_arg_with_dtor() {
 // larger than 8 bytes and is passed indirectly.
 // WIN64-LABEL: define dso_local void @"?call_big_arg_with_dtor@@YAXXZ"()
 // WIN64:   call noundef ptr @"??0BigWithDtor@@QEAA at XZ"
-// WIN64:   call void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr noundef %{{.*}})
+// WIN64:   call void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr dead_on_return noundef %{{.*}})
 // WIN64-NOT: call void @"??1BigWithDtor@@QEAA at XZ"
 // WIN64:   ret void
 
@@ -259,22 +259,22 @@ void eh_cleanup_arg_with_dtor() {
 // WIN32: }
 
 void small_arg_with_vftable(SmallWithVftable s) {}
-// LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(ptr noundef %s)
+// LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(ptr dead_on_return noundef %s)
 // WIN32: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr inalloca(<{ %struct.SmallWithVftable }>) %0)
-// WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr noundef %s)
-// WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr dead_on_return noundef %s)
+// WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr dead_on_return noundef %s)
 
 void medium_arg_with_copy_ctor(MediumWithCopyCtor s) {}
-// LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(ptr noundef %s)
+// LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(ptr dead_on_return noundef %s)
 // WIN32: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr inalloca(<{ %struct.MediumWithCopyCtor }>) %0)
-// WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
-// WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
-// WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
+// WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
+// WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
 
 void big_arg(Big s) {}
 // LINUX-LABEL: define{{.*}} void @_Z7big_arg3Big(ptr noundef byval(%struct.Big) align 4 %s)
 // WIN32: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr noundef byval(%struct.Big) align 4 %s)
-// WIN64: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr dead_on_return noundef %s)
 
 // PR27607: We would attempt to load i32 value out of the reference instead of
 // just loading the pointer from the struct during argument expansion.
@@ -346,7 +346,7 @@ class Class {
   void thiscall_method_arg(Big s) {}
   // LINUX: define {{.*}} void @_ZN5Class19thiscall_method_argE3Big(ptr {{[^,]*}} %this, ptr noundef byval(%struct.Big) align 4 %s)
   // WIN32: define {{.*}} void @"?thiscall_method_arg at Class@@QAEXUBig@@@Z"(ptr {{[^,]*}} %this, ptr noundef byval(%struct.Big) align 4 %s)
-  // WIN64: define linkonce_odr dso_local void @"?thiscall_method_arg at Class@@QEAAXUBig@@@Z"(ptr {{[^,]*}} %this, ptr noundef %s)
+  // WIN64: define linkonce_odr dso_local void @"?thiscall_method_arg at Class@@QEAAXUBig@@@Z"(ptr {{[^,]*}} %this, ptr dead_on_return noundef %s)
 };
 
 void use_class() {
diff --git a/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp b/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
index 9e37e71e257fd..b7653632cf882 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
@@ -18,4 +18,4 @@ A B::foo(A x) {
   return x;
 }
 
-// CHECK-LABEL: define{{.*}} void @"?foo at B@@QEAA?AUA@@U2@@Z"(ptr {{[^,]*}} %this, ptr dead_on_unwind noalias writable sret(%struct.A) align 4 %agg.result, ptr noundef %x)
+// CHECK-LABEL: define{{.*}} void @"?foo at B@@QEAA?AUA@@U2@@Z"(ptr {{[^,]*}} %this, ptr dead_on_unwind noalias writable sret(%struct.A) align 4 %agg.result, ptr dead_on_return noundef %x)
diff --git a/clang/test/CodeGenCXX/ms-property.cpp b/clang/test/CodeGenCXX/ms-property.cpp
index 744de224b2f9a..d9fbf46dacb25 100644
--- a/clang/test/CodeGenCXX/ms-property.cpp
+++ b/clang/test/CodeGenCXX/ms-property.cpp
@@ -105,7 +105,7 @@ int main(int argc, char **argv) {
   // CHECK: [[ARGC:%.+]] = load i32, ptr %
   // CHECK: [[P1_X_ARGC_0:%.+]] = call noundef i32 @"?GetX at S@@QEAAHHH at Z"(ptr {{[^,]*}} [[P1]], i32 noundef [[ARGC]], i32 noundef 0)
   // CHECK: [[CAST:%.+]] = trunc i32 [[P1_X_ARGC_0]] to i8
-  // CHECK: [[P2_Y_p1_X_ARGC_0_T:%.+]] = call noundef i8 @"?GetY@?$St at M@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[P2_2]], i8 noundef [[CAST]], ptr noundef %{{.+}})
+  // CHECK: [[P2_Y_p1_X_ARGC_0_T:%.+]] = call noundef i8 @"?GetY@?$St at M@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[P2_2]], i8 noundef [[CAST]], ptr dead_on_return noundef %{{.+}})
   // CHECK: [[CAST:%.+]] = sitofp i8 [[P2_Y_p1_X_ARGC_0_T]] to float
   // CHECK: [[J:%.+]] = load i32, ptr %
   // CHECK: [[CAST1:%.+]] = sitofp i32 [[J]] to float
@@ -124,6 +124,6 @@ int main(int argc, char **argv) {
 // CHECK: call noundef i32 @"?GetX@?$St at H@@QEAAHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}} i32 noundef %{{.+}})
 // CHECK: call void @"?PutY@?$St at H@@QEAAXDHN at Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, i32 noundef %{{.+}}, double noundef %{{.+}}
 // CHECK: call noundef i32 @"?GetX@?$St at H@@QEAAHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}} i32 noundef %{{.+}})
-// CHECK: call noundef i8 @"?GetY@?$St at H@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, ptr noundef %{{.+}})
+// CHECK: call noundef i8 @"?GetY@?$St at H@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, ptr dead_on_return noundef %{{.+}})
 // CHECK: call noundef i32 @"?PutX@?$St at H@@QEAAHHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}}, i32 noundef %{{.+}}, i32 noundef %{{.+}})
 #endif //HEADER
diff --git a/clang/test/CodeGenCXX/nrvo.cpp b/clang/test/CodeGenCXX/nrvo.cpp
index 1141bc35de582..5b0fc914120e3 100644
--- a/clang/test/CodeGenCXX/nrvo.cpp
+++ b/clang/test/CodeGenCXX/nrvo.cpp
@@ -2197,7 +2197,7 @@ void test16() { // http://wg21.link/p2025r2#ex-9
 // CHECK-EH-11-NEXT:    br i1 [[CMP9]], label [[IF_THEN10:%.*]], label [[IF_END11:%.*]]
 // CHECK-EH-11:       if.then10:
 // CHECK-EH-11-NEXT:    store i32 3, ptr [[CLEANUP_DEST_SLOT]], align 4
-// CHECK-EH-11-NEXT:    br label [[CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK-EH-11-NEXT:    br label [[CLEANUP]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK-EH-11:       if.end11:
 // CHECK-EH-11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-EH-11-NEXT:    [[CMP12:%.*]] = icmp eq i32 [[TMP4]], 3
@@ -2239,7 +2239,7 @@ void test16() { // http://wg21.link/p2025r2#ex-9
 // CHECK-EH-11-NEXT:      i32 2, label [[IMPOSSIBLE]]
 // CHECK-EH-11-NEXT:    ]
 // CHECK-EH-11:       cleanup.cont:
-// CHECK-EH-11-NEXT:    br label [[WHILE_BODY]], !llvm.loop [[LOOP3]]
+// CHECK-EH-11-NEXT:    br label [[WHILE_BODY]], !llvm.loop [[LOOP4]]
 // CHECK-EH-11:       while.end:
 // CHECK-EH-11-NEXT:    call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]])
 // CHECK-EH-11-NEXT:    br label [[RETURN]]
diff --git a/clang/test/CodeGenCXX/pass-by-value-noalias.cpp b/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
index 773cf6b81c3b2..947379d9b8b92 100644
--- a/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
+++ b/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
@@ -14,8 +14,8 @@ struct Foo {
 // Make sure noalias is added to indirect arguments with trivially copyable types
 // if -fpass-by-value-is-noalias is provided.
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr dead_on_return noundef %arg)
 void take(Foo arg) {}
 
 int G;
@@ -38,8 +38,8 @@ struct NonTrivial {
 // Make sure noalias is not added to indirect arguments that are not trivially
 // copyable even if -fpass-by-value-is-noalias is provided.
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr noundef %arg)
-// NO_NOALIAS:   define{{.*}} void @_Z4take10NonTrivial(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr dead_on_return noundef %arg)
+// NO_NOALIAS:   define{{.*}} void @_Z4take10NonTrivial(ptr dead_on_return noundef %arg)
 void take(NonTrivial arg) {}
 
 // Escape examples. Pointers to the objects passed to take() may escape, depending on whether a temporary copy is created or not (e.g. due to NRVO).
@@ -54,8 +54,8 @@ struct A {
 };
 A *p;
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take1A(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take1A(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take1A(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take1A(ptr dead_on_return noundef %arg)
 void take(A arg) {}
 
 // WITH_NOALIAS: define{{.*}} void @_Z7CreateAPP1A(ptr dead_on_unwind noalias writable sret(%struct.A) align 1 %agg.result, ptr noundef %where)
diff --git a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
index daeea77774ec8..0310535362e3d 100644
--- a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
@@ -24,7 +24,7 @@ struct __attribute__((trivial_abi)) TrivialSA {
 // Check that TrivialSA is passed indirectly despite being annotated with
 // 'trivial_abi'.
 
-// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr noundef %{{.*}})
+// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr dead_on_return noundef %{{.*}})
 
 void testParamTrivialSA(TrivialSA a) {
 }
diff --git a/clang/test/CodeGenCXX/regparm.cpp b/clang/test/CodeGenCXX/regparm.cpp
index b9735485db8de..a31394016fb55 100644
--- a/clang/test/CodeGenCXX/regparm.cpp
+++ b/clang/test/CodeGenCXX/regparm.cpp
@@ -11,7 +11,7 @@ struct S1 {
 };
 
 void __attribute__((regparm(3))) foo2(S1 a, int b);
-// CHECK: declare void @_Z4foo22S1i(ptr inreg noundef, i32 inreg noundef)
+// CHECK: declare void @_Z4foo22S1i(ptr dead_on_return inreg noundef, i32 inreg noundef)
 void bar2(S1 a, int b) {
   foo2(a, b);
 }
diff --git a/clang/test/CodeGenCXX/trivial_abi.cpp b/clang/test/CodeGenCXX/trivial_abi.cpp
index b8cc0d1cc6528..eacbde594e517 100644
--- a/clang/test/CodeGenCXX/trivial_abi.cpp
+++ b/clang/test/CodeGenCXX/trivial_abi.cpp
@@ -140,7 +140,7 @@ void testIgnoredSmall() {
   testReturnSmall();
 }
 
-// CHECK: define{{.*}} void @_Z14testParamLarge5Large(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5LargeD1Ev(ptr {{[^,]*}} %[[A]])
 // CHECK: ret void
 // CHECK: }
@@ -163,7 +163,7 @@ Large testReturnLarge() {
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_LARGE]], align 8
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[T]])
 // CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN5LargeC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(520) %[[T]])
-// CHECK: call void @_Z14testParamLarge5Large(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: %[[CALL2:.*]] = call noundef ptr @_ZN5LargeD1Ev(ptr {{[^,]*}} %[[T]])
 // CHECK: ret void
 // CHECK: }
@@ -176,7 +176,7 @@ void testCallLarge0() {
 // CHECK: define{{.*}} void @_Z14testCallLarge1v()
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_LARGE:.*]], align 8
 // CHECK: call void @_Z15testReturnLargev(ptr dead_on_unwind writable sret(%[[STRUCT_LARGE]]) align 8 %[[AGG_TMP]])
-// CHECK: call void @_Z14testParamLarge5Large(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: ret void
 // CHECK: }
 
@@ -244,7 +244,7 @@ void testExceptionSmall() {
 // CHECK: call noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[AGG_TMP]])
 // CHECK: invoke noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[AGG_TMP1]])
 
-// CHECK: call void @_Z20calleeExceptionLarge5LargeS_(ptr noundef %[[AGG_TMP]], ptr noundef %[[AGG_TMP1]])
+// CHECK: call void @_Z20calleeExceptionLarge5LargeS_(ptr dead_on_return noundef %[[AGG_TMP]], ptr dead_on_return noundef %[[AGG_TMP1]])
 // CHECK-NEXT: ret void
 
 // CHECK: landingpad { ptr, i32 }
diff --git a/clang/test/CodeGenCXX/uncopyable-args.cpp b/clang/test/CodeGenCXX/uncopyable-args.cpp
index 31192b65cc362..2d09732f9d3c2 100644
--- a/clang/test/CodeGenCXX/uncopyable-args.cpp
+++ b/clang/test/CodeGenCXX/uncopyable-args.cpp
@@ -59,12 +59,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN9move_ctor3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN9move_ctor3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN9move_ctor3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN9move_ctor3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at move_ctor@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at move_ctor@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace all_deleted {
@@ -81,12 +81,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN11all_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN11all_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN11all_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN11all_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at all_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at all_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace implicitly_deleted {
@@ -102,14 +102,14 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN18implicitly_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr)
 
 // In MSVC 2013, the copy ctor is not deleted by a move assignment. In MSVC 2015, it is.
 // WIN64-18-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(i64
-// WIN64-19-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-19-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace one_deleted {
@@ -125,12 +125,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN11one_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN11one_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN11one_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN11one_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at one_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at one_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace copy_defaulted {
@@ -170,7 +170,7 @@ void bar() {
 // CHECK: call void @_ZN14move_defaulted3fooENS_1AE(ptr %{{.*}})
 // CHECK-LABEL: declare void @_ZN14move_defaulted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at move_defaulted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at move_defaulted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace trivial_defaulted {
@@ -207,12 +207,12 @@ void bar() {
 }
 // CHECK-LABEL: define{{.*}} void @_ZN14two_copy_ctors3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
-// NEWABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef byval
-// NEWABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef byval
 
-// WIN64-LABEL: declare dso_local void @"?foo at two_copy_ctors@@YAXUB at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at two_copy_ctors@@YAXUB at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace definition_only {
diff --git a/clang/test/CodeGenCXX/wasm-args-returns.cpp b/clang/test/CodeGenCXX/wasm-args-returns.cpp
index fbb152ac1bb3d..7b1c27bf0808d 100644
--- a/clang/test/CodeGenCXX/wasm-args-returns.cpp
+++ b/clang/test/CodeGenCXX/wasm-args-returns.cpp
@@ -46,17 +46,17 @@ struct copy_ctor {
   copy_ctor(copy_ctor const &);
 };
 test(copy_ctor);
-// CHECK: define void @_Z7forward9copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.copy_ctor) align 8 %{{.*}}, ptr nonnull %{{.*}})
+// CHECK: define void @_Z7forward9copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.copy_ctor) align 8 %{{.*}}, ptr dead_on_return nonnull %{{.*}})
 //
 // CHECK: declare ptr @_ZN9copy_ctorC1ERKS_(ptr {{[^,]*}} returned {{[^,]*}}, ptr nonnull align 8 dereferenceable(8))
 //
 // CHECK: define void @_Z14test_copy_ctorv()
 // CHECK: %[[tmp:.*]] = alloca %struct.copy_ctor, align 8
 // CHECK: call void @_Z13def_copy_ctorv(ptr dead_on_unwind nonnull writable sret(%struct.copy_ctor) align 8 %[[tmp]])
-// CHECK: call void @_Z3use9copy_ctor(ptr nonnull %[[tmp]])
+// CHECK: call void @_Z3use9copy_ctor(ptr dead_on_return nonnull %[[tmp]])
 // CHECK: ret void
 //
-// CHECK: declare void @_Z3use9copy_ctor(ptr)
+// CHECK: declare void @_Z3use9copy_ctor(ptr dead_on_return)
 // CHECK: declare void @_Z13def_copy_ctorv(ptr dead_on_unwind writable sret(%struct.copy_ctor) align 8)
 
 struct __attribute__((aligned(16))) aligned_copy_ctor {
@@ -64,17 +64,17 @@ struct __attribute__((aligned(16))) aligned_copy_ctor {
   aligned_copy_ctor(aligned_copy_ctor const &);
 };
 test(aligned_copy_ctor);
-// CHECK: define void @_Z7forward17aligned_copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.aligned_copy_ctor) align 16 %{{.*}}, ptr nonnull %{{.*}})
+// CHECK: define void @_Z7forward17aligned_copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.aligned_copy_ctor) align 16 %{{.*}}, ptr dead_on_return nonnull %{{.*}})
 //
 // CHECK: declare ptr @_ZN17aligned_copy_ctorC1ERKS_(ptr {{[^,]*}} returned {{[^,]*}}, ptr nonnull align 16 dereferenceable(16))
 //
 // CHECK: define void @_Z22test_aligned_copy_ctorv()
 // CHECK: %[[tmp:.*]] = alloca %struct.aligned_copy_ctor, align 16
 // CHECK: call void @_Z21def_aligned_copy_ctorv(ptr dead_on_unwind nonnull writable sret(%struct.aligned_copy_ctor) align 16 %[[tmp]])
-// CHECK: call void @_Z3use17aligned_copy_ctor(ptr nonnull %[[tmp]])
+// CHECK: call void @_Z3use17aligned_copy_ctor(ptr dead_on_return nonnull %[[tmp]])
 // CHECK: ret void
 //
-// CHECK: declare void @_Z3use17aligned_copy_ctor(ptr)
+// CHECK: declare void @_Z3use17aligned_copy_ctor(ptr dead_on_return)
 // CHECK: declare void @_Z21def_aligned_copy_ctorv(ptr dead_on_unwind writable sret(%struct.aligned_copy_ctor) align 16)
 
 struct empty {};
diff --git a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
index 9927d0b24799e..78c45753b5bbd 100644
--- a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
+++ b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
@@ -18,7 +18,7 @@ struct NonTrivial {
 
 SWIFTCALL int receiveNonTrivial(NonTrivial o) { return o.o; }
 
-// CHECK-LABEL: define dso_local swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr noundef %o)
+// CHECK-LABEL: define dso_local swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr dead_on_return noundef %o)
 
 int passNonTrivial() {
   return receiveNonTrivial({});
@@ -26,4 +26,4 @@ int passNonTrivial() {
 
 // CHECK-LABEL: define dso_local noundef i32 @"?passNonTrivial@@YAHXZ"()
 // CHECK-NOT: stacksave
-// CHECK: call swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr noundef %{{.*}})
+// CHECK: call swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr dead_on_return noundef %{{.*}})
diff --git a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m b/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
index d2a954ae26a04..ba8a04b52716e 100644
--- a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
+++ b/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
@@ -42,7 +42,7 @@ void testStrongException(void) {
 // CHECK: call void @genWeak(ptr dead_on_unwind writable sret(%[[STRUCT_WEAK]]) align 8 %[[AGG_TMP]])
 // CHECK: invoke void @genWeak(ptr dead_on_unwind writable sret(%[[STRUCT_WEAK]]) align 8 %[[AGG_TMP1]])
 
-// CHECK: call void @calleeWeak(ptr noundef %[[AGG_TMP]], ptr noundef %[[AGG_TMP1]])
+// CHECK: call void @calleeWeak(ptr dead_on_return noundef %[[AGG_TMP]], ptr dead_on_return noundef %[[AGG_TMP1]])
 // CHECK: ret void
 
 // CHECK: landingpad { ptr, i32 }
diff --git a/clang/test/CodeGenObjC/pass-by-value-noalias.m b/clang/test/CodeGenObjC/pass-by-value-noalias.m
index ed94d4c80b525..86a4ba36a1b8f 100644
--- a/clang/test/CodeGenObjC/pass-by-value-noalias.m
+++ b/clang/test/CodeGenObjC/pass-by-value-noalias.m
@@ -17,6 +17,6 @@ @interface Bar
   Bar *__weak f;
 };
 
-// WITH_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
-// NO_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
+// NO_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
 void take(struct Foo arg) {}
diff --git a/clang/test/CodeGenObjC/weak-in-c-struct.m b/clang/test/CodeGenObjC/weak-in-c-struct.m
index be80edd1ff11d..6809360d03da1 100644
--- a/clang/test/CodeGenObjC/weak-in-c-struct.m
+++ b/clang/test/CodeGenObjC/weak-in-c-struct.m
@@ -130,7 +130,7 @@ void test_move_assignment_Weak(Weak *p) {
   *p = getWeak();
 }
 
-// COMMON: define{{.*}} void @test_parameter_Weak(ptr noundef %[[A:.*]])
+// COMMON: define{{.*}} void @test_parameter_Weak(ptr dead_on_return noundef %[[A:.*]])
 // COMMON: call void @__destructor_{{.*}}(ptr %[[A]])
 
 void test_parameter_Weak(Weak a) {
@@ -142,7 +142,7 @@ void test_parameter_Weak(Weak a) {
 // COMMON: store ptr %[[A]], ptr %[[A_ADDR]]
 // COMMON: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]]
 // COMMON: call void @__copy_constructor_{{.*}}(ptr %[[AGG_TMP]], ptr %[[V0]])
-// COMMON: call void @calleeWeak(ptr noundef %[[AGG_TMP]])
+// COMMON: call void @calleeWeak(ptr dead_on_return noundef %[[AGG_TMP]])
 // COMMON-NEXT: ret
 
 void test_argument_Weak(Weak *a) {
@@ -164,7 +164,7 @@ Weak test_return_Weak(Weak *a) {
 // COMMON: %[[AGG_TMP:.*]] = alloca %[[STRUCT_WEAK]]
 // COMMON: br i1
 
-// COMMON: call void @objc_msgSend({{.*}}, ptr noundef %[[AGG_TMP]])
+// COMMON: call void @objc_msgSend({{.*}}, ptr dead_on_return noundef %[[AGG_TMP]])
 // COMMON: br
 
 // COMMON: call void @__destructor_{{.*}}(ptr %[[AGG_TMP]])
diff --git a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
index 4d76796d86d1f..3a043c4892981 100644
--- a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
@@ -75,7 +75,7 @@ - (void)passStrongWeak:(StrongWeak)a;
 - (void)passNonTrivial:(NonTrivial)a;
 @end
 
-// CHECK: define{{.*}} void @_Z19testParamStrongWeak10StrongWeak(ptr noundef %{{.*}})
+// CHECK: define{{.*}} void @_Z19testParamStrongWeak10StrongWeak(ptr dead_on_return noundef %{{.*}})
 // CHECK: call noundef ptr @_ZN10StrongWeakD1Ev(
 // CHECK-NEXT: ret void
 
@@ -88,7 +88,7 @@ void testParamStrongWeak(StrongWeak a) {
 // CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN10StrongWeakC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(16) %[[V0]])
-// CHECK: call void @_Z19testParamStrongWeak10StrongWeak(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z19testParamStrongWeak10StrongWeak(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK-NOT: call
 // CHECK: ret void
 
@@ -107,13 +107,13 @@ StrongWeak testReturnStrongWeak(StrongWeak *a) {
   return *a;
 }
 
-// CHECK: define{{.*}} void @_Z27testParamContainsStrongWeak18ContainsStrongWeak(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z27testParamContainsStrongWeak18ContainsStrongWeak(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN18ContainsStrongWeakD1Ev(ptr {{[^,]*}} %[[A]])
 
 void testParamContainsStrongWeak(ContainsStrongWeak a) {
 }
 
-// CHECK: define{{.*}} void @_Z26testParamDerivedStrongWeak17DerivedStrongWeak(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z26testParamDerivedStrongWeak17DerivedStrongWeak(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN17DerivedStrongWeakD1Ev(ptr {{[^,]*}} %[[A]])
 
 void testParamDerivedStrongWeak(DerivedStrongWeak a) {
@@ -163,7 +163,7 @@ Strong testReturnStrong(Strong *a) {
   return *a;
 }
 
-// CHECK: define{{.*}} void @_Z21testParamWeakTemplate1SIU6__weakP11objc_objectE(ptr noundef %{{.*}})
+// CHECK: define{{.*}} void @_Z21testParamWeakTemplate1SIU6__weakP11objc_objectE(ptr dead_on_return noundef %{{.*}})
 // CHECK: call noundef ptr @_ZN1SIU6__weakP11objc_objectED1Ev(
 // CHECK-NEXT: ret void
 
@@ -237,7 +237,7 @@ void test0(C *c) {
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_STRONGWEAK]], align 8
 // CHECK: br i1
 
-// CHECK: call void @objc_msgSend({{.*}}, ptr noundef %[[AGG_TMP]])
+// CHECK: call void @objc_msgSend({{.*}}, ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: br
 
 // CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN10StrongWeakD1Ev(ptr noundef nonnull align 8 dereferenceable(16) %[[AGG_TMP]])
diff --git a/clang/test/CodeGenObjCXX/property-objects.mm b/clang/test/CodeGenObjCXX/property-objects.mm
index 7ae20f66177bd..8354794254933 100644
--- a/clang/test/CodeGenObjCXX/property-objects.mm
+++ b/clang/test/CodeGenObjCXX/property-objects.mm
@@ -60,7 +60,7 @@ - (struct CGRect)extent {return bounds;}
 
 // CHECK-LABEL: define{{.*}} i32 @main
 // CHECK: call void @_ZN1SC1ERKS_(ptr {{[^,]*}} [[AGGTMP:%[a-zA-Z0-9\.]+]], ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) {{%[a-zA-Z0-9\.]+}})
-// CHECK: call void @objc_msgSend(ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef [[AGGTMP]])
+// CHECK: call void @objc_msgSend(ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr dead_on_return noundef [[AGGTMP]])
 // CHECK-NEXT: ret i32 0
 int main() {
   I *i;
diff --git a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
index e5cb71bad47c0..9428940d6da48 100644
--- a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
@@ -15,7 +15,7 @@
 
 // Check that AddrDiscStrong0 is destructed in the callee.
 
-// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr noundef %[[A:.*]])
+// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN15AddrDiscStrong0D1Ev(ptr noundef nonnull align {{[0-9]+}} dereferenceable(16) %[[A]])
 // CHECK: ret void
 
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
index 738b7ab7f2b75..d42e51d04ab9d 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
@@ -4,6 +4,7 @@
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
+typedef unsigned int uint;
 typedef half __attribute__((ext_vector_type(2))) half2;
 
 // CHECK-LABEL: @test_setprio_inc_wg(
@@ -42,6 +43,24 @@ void test_s_wait_tensorcnt() {
   __builtin_amdgcn_s_wait_tensorcnt(0);
 }
 
+// CHECK-LABEL: @test_prng_b32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.prng.b32(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(1) [[TMP2]], align 4
+// CHECK-NEXT:    ret void
+//
+void test_prng_b32(global uint* out, uint a) {
+  *out = __builtin_amdgcn_prng_b32(a);
+}
+
 // CHECK-LABEL: @test_tanh_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -119,6 +138,25 @@ void test_rcp_bf16(global __bf16* out, __bf16 a)
   *out = __builtin_amdgcn_rcp_bf16(a);
 }
 
+// CHECK-LABEL: @test_sqrt_bf16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = call bfloat @llvm.amdgcn.sqrt.bf16(bfloat [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[TMP1]], ptr addrspace(1) [[TMP2]], align 2
+// CHECK-NEXT:    ret void
+//
+void test_sqrt_bf16(global __bf16* out, __bf16 a)
+{
+  *out = __builtin_amdgcn_sqrt_bf16(a);
+}
+
 // CHECK-LABEL: @test_rsq_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
diff --git a/clang/test/Driver/print-multi-selection-flags.c b/clang/test/Driver/print-multi-selection-flags.c
index 5f9383fbed8f4..8cf8f04bb6b48 100644
--- a/clang/test/Driver/print-multi-selection-flags.c
+++ b/clang/test/Driver/print-multi-selection-flags.c
@@ -107,3 +107,22 @@
 // CHECK-AARCH64-MULTILIB-CUSTOM-FLAG: --target=aarch64-unknown-none-eabi
 // CHECK-MULTILIB-CUSTOM-FLAG-DAG:     -fmultilib-flag=foo
 // CHECK-MULTILIB-CUSTOM-FLAG-DAG:     -fmultilib-flag=bar
+
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fropi              | FileCheck --check-prefixes=CHECK-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -frwpi              | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fropi -frwpi       | FileCheck --check-prefixes=CHECK-ROPI,CHECK-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fno-ropi -fno-rwpi | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a                     | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fpic               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIC1 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fPIC               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIC2 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fpie               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIE1 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fPIE               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIE2 %s
+// CHECK-PIC2: -fPIC
+// CHECK-PIE2: -fPIE
+// CHECK-NO-PIC: -fno-pic
+// CHECK-NO-ROPI: -fno-ropi
+// CHECK-NO-RWPI: -fno-rwpi
+// CHECK-PIC1: -fpic
+// CHECK-PIE1: -fpie
+// CHECK-ROPI: -fropi
+// CHECK-RWPI: -frwpi
diff --git a/clang/test/Headers/stdarg.cpp b/clang/test/Headers/stdarg.cpp
index 20bf17caf15f7..bfc3af11a23b6 100644
--- a/clang/test/Headers/stdarg.cpp
+++ b/clang/test/Headers/stdarg.cpp
@@ -15,8 +15,8 @@
 
 #include <stdarg.h>
 
-// AARCH64-C: define {{.*}} @f(i32 noundef %n, ptr noundef %list)
-// AARCH64-CXX: define {{.*}} @_Z1fiSt9__va_list(i32 noundef %n, ptr noundef %list)
+// AARCH64-C: define {{.*}} @f(i32 noundef %n, ptr dead_on_return noundef %list)
+// AARCH64-CXX: define {{.*}} @_Z1fiSt9__va_list(i32 noundef %n, ptr dead_on_return noundef %list)
 // X86_64-C: define {{.*}} @f(i32 noundef %n, ptr noundef %list)
 // X86_64-CXX: define {{.*}} @_Z1fiP13__va_list_tag(i32 noundef %n, ptr noundef %list)
 // PPC64-C: define {{.*}} @f(i32 noundef signext %n, ptr noundef %list)
diff --git a/clang/test/OpenMP/for_firstprivate_codegen.cpp b/clang/test/OpenMP/for_firstprivate_codegen.cpp
index 0255e1e3d4aea..83b5939799642 100644
--- a/clang/test/OpenMP/for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/for_firstprivate_codegen.cpp
@@ -427,7 +427,7 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[TMP0]])
 // CHECK1-NEXT:    store i32 0, ptr [[RETVAL]], align 4
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
@@ -469,7 +469,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -543,12 +543,12 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store ptr [[TMP3]], ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store ptr [[TMP4]], ptr [[_TMP1]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -572,7 +572,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP6]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE6]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done6:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP8]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR7]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP7]], ptr noundef [[AGG_TMP8]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP8]]) #[[ATTR2]]
@@ -611,7 +611,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC4]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP9]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP9]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM11:%.*]] = sext i32 [[TMP19]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR5]], i64 0, i64 [[IDXPROM11]]
@@ -711,7 +711,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1044,8 +1044,8 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr @g1, align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr @g1, align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
 // CHECK4-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1086,7 +1086,7 @@ int main() {
 // CHECK4-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK4-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[G]], align 4
-// CHECK4-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK4-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store volatile i32 1, ptr [[TMP13]], align 4
 // CHECK4-NEXT:    store i32 2, ptr [[SIVAR3]], align 4
 // CHECK4-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 0
@@ -1103,7 +1103,7 @@ int main() {
 // CHECK4-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[G]], align 4
 // CHECK4-NEXT:    store volatile i32 [[TMP14]], ptr [[BLOCK_CAPTURED]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 5
-// CHECK4-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK4-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store ptr [[TMP15]], ptr [[BLOCK_CAPTURED5]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 7
 // CHECK4-NEXT:    [[TMP16:%.*]] = load i32, ptr [[SIVAR3]], align 4
@@ -1137,7 +1137,7 @@ int main() {
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
 // CHECK4-NEXT:    store i32 2, ptr [[BLOCK_CAPTURE_ADDR]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR1]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store i32 2, ptr [[TMP0]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
 // CHECK4-NEXT:    store i32 4, ptr [[BLOCK_CAPTURE_ADDR2]], align 4
diff --git a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
index 93e71b9a8312e..441e809dc59e5 100644
--- a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
@@ -350,9 +350,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i32 8, i1 false)
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i32 2
@@ -524,7 +524,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK1-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK1-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK1-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -535,22 +535,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK1-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK1-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK1-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK1-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK1-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK1-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK1-NEXT:    ret void
 //
@@ -578,25 +578,25 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK1-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK1-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK1-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK1-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP5]], -1
 // CHECK1-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP7]], 1
 // CHECK1-NEXT:    store i32 [[DIV]], ptr [[TMP6]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP8]], i32 0, i32 2
 // CHECK1-NEXT:    store i32 1111, ptr [[ARRAYIDX]], align 4
 // CHECK1-NEXT:    ret void
@@ -656,7 +656,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -738,9 +738,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i32 8, i1 false)
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i32 2
@@ -840,7 +840,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 0, ptr [[A]], align 4
 // CHECK1-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SST]], ptr [[THIS1]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_CASTED]], align 4
@@ -862,7 +862,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
@@ -896,7 +896,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -965,7 +965,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK3-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK3-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK3-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -976,22 +976,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK3-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK3-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK3-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK3-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK3-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK3-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK3-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK3-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK3-NEXT:    ret void
 //
@@ -1020,22 +1020,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK3-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK3-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP5]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK3-NEXT:    store ptr [[B_ADDR]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK3-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP8]], ptr [[TMP7]], align 4
 // CHECK3-NEXT:    call void @_ZZN2SSC1ERiENKUlvE_clEv(ptr nonnull align 4 dereferenceable(16) [[REF_TMP]])
 // CHECK3-NEXT:    ret void
@@ -1053,32 +1053,32 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0:%.*]], ptr [[THIS1]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK3-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP7]], -1
 // CHECK3-NEXT:    store i32 [[DEC]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 4
+// CHECK3-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
 // CHECK3-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP10]], 1
 // CHECK3-NEXT:    store i32 [[DIV]], ptr [[TMP9]], align 4
 // CHECK3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 4
+// CHECK3-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
 // CHECK3-NEXT:    store i32 [[TMP17]], ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 4
+// CHECK3-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK3-NEXT:    store i32 [[TMP21]], ptr [[C_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP22:%.*]] = load i32, ptr [[C_CASTED]], align 4
@@ -1106,14 +1106,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK3-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK3-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK3-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK3-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK3-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -1252,7 +1252,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK4-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK4-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
 // CHECK4-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK4-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK4-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -1263,22 +1263,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK4-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK4-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK4-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK4-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK4-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK4-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK4-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK4-NEXT:    ret void
 //
@@ -1307,11 +1307,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK4-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK4-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK4-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
 // CHECK4-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 0
@@ -1327,13 +1327,13 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BLOCK_CAPTURED_THIS_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 5
 // CHECK4-NEXT:    store ptr [[TMP0]], ptr [[BLOCK_CAPTURED_THIS_ADDR]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP3]], ptr [[BLOCK_CAPTURED]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 7
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK4-NEXT:    store i32 [[TMP4]], ptr [[BLOCK_CAPTURED5]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP5]], ptr [[BLOCK_CAPTURED6]], align 4
 // CHECK4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___BLOCK_LITERAL_GENERIC:%.*]], ptr [[BLOCK]], i32 0, i32 3
 // CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 4
@@ -1354,7 +1354,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BLOCK_CAPTURED_THIS:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
 // CHECK4-NEXT:    [[THIS:%.*]] = load ptr, ptr [[BLOCK_CAPTURED_THIS]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK4-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
 // CHECK4-NEXT:    store i32 [[INC]], ptr [[TMP0]], align 4
@@ -1363,12 +1363,12 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP2]], -1
 // CHECK4-NEXT:    store i32 [[DEC]], ptr [[BLOCK_CAPTURE_ADDR1]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 4
+// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK4-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP4]], 1
 // CHECK4-NEXT:    store i32 [[DIV]], ptr [[TMP3]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 4
+// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
 // CHECK4-NEXT:    store i32 [[TMP6]], ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_CASTED]], align 4
@@ -1377,7 +1377,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    store i32 [[TMP8]], ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 4
+// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
 // CHECK4-NEXT:    store i32 [[TMP11]], ptr [[C_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP12:%.*]] = load i32, ptr [[C_CASTED]], align 4
@@ -1405,14 +1405,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK4-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK4-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK4-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK4-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK4-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK4-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK4-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK4-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK4-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -1530,9 +1530,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i64 2
@@ -1704,7 +1704,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK9-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK9-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK9-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -1715,22 +1715,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK9-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK9-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK9-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK9-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK9-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK9-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK9-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK9-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK9-NEXT:    ret void
 //
@@ -1758,25 +1758,25 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK9-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK9-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK9-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
-// CHECK9-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK9-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK9-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK9-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK9-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP5]], -1
 // CHECK9-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK9-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK9-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP7]], 1
 // CHECK9-NEXT:    store i32 [[DIV]], ptr [[TMP6]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP8]], i64 0, i64 2
 // CHECK9-NEXT:    store i32 1111, ptr [[ARRAYIDX]], align 4
 // CHECK9-NEXT:    ret void
@@ -1836,7 +1836,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1918,9 +1918,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i64 2
@@ -2020,7 +2020,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i32 0, ptr [[A]], align 4
 // CHECK9-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SST]], ptr [[THIS1]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK9-NEXT:    store i32 [[TMP1]], ptr [[A_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP2:%.*]] = load i64, ptr [[A_CASTED]], align 8
@@ -2042,7 +2042,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[A]], ptr [[A_ADDR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK9-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK9-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
@@ -2076,7 +2076,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2145,7 +2145,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK11-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK11-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK11-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK11-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK11-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -2156,22 +2156,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK11-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK11-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK11-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK11-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK11-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK11-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK11-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK11-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK11-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK11-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK11-NEXT:    ret void
 //
@@ -2200,22 +2200,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK11-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK11-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK11-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK11-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK11-NEXT:    store ptr [[TMP0]], ptr [[TMP3]], align 8
 // CHECK11-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK11-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP5]], ptr [[TMP4]], align 8
 // CHECK11-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK11-NEXT:    store ptr [[B_ADDR]], ptr [[TMP6]], align 8
 // CHECK11-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK11-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP8]], ptr [[TMP7]], align 8
 // CHECK11-NEXT:    call void @_ZZN2SSC1ERiENKUlvE_clEv(ptr nonnull align 8 dereferenceable(32) [[REF_TMP]])
 // CHECK11-NEXT:    ret void
@@ -2233,32 +2233,32 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0:%.*]], ptr [[THIS1]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
 // CHECK11-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8
+// CHECK11-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK11-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK11-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK11-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8
+// CHECK11-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK11-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP7]], -1
 // CHECK11-NEXT:    store i32 [[DEC]], ptr [[TMP6]], align 4
 // CHECK11-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8
+// CHECK11-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
 // CHECK11-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP10]], 1
 // CHECK11-NEXT:    store i32 [[DIV]], ptr [[TMP9]], align 4
 // CHECK11-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 8
+// CHECK11-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 // CHECK11-NEXT:    store i32 [[TMP13]], ptr [[A_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP14:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8
+// CHECK11-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
 // CHECK11-NEXT:    store i32 [[TMP17]], ptr [[B_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP18:%.*]] = load i64, ptr [[B_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+// CHECK11-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK11-NEXT:    store i32 [[TMP21]], ptr [[C_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP22:%.*]] = load i64, ptr [[C_CASTED]], align 8
@@ -2286,14 +2286,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK11-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK11-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK11-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK11-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK11-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK11-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK11-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK11-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK11-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -2432,7 +2432,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK12-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK12-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK12-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK12-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK12-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -2443,22 +2443,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK12-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK12-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK12-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK12-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK12-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK12-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK12-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK12-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK12-NEXT:    ret void
 //
@@ -2487,11 +2487,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK12-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK12-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK12-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK12-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
 // CHECK12-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 0
@@ -2507,13 +2507,13 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BLOCK_CAPTURED_THIS_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 5
 // CHECK12-NEXT:    store ptr [[TMP0]], ptr [[BLOCK_CAPTURED_THIS_ADDR]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP3]], ptr [[BLOCK_CAPTURED]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 8
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK12-NEXT:    store i32 [[TMP4]], ptr [[BLOCK_CAPTURED5]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP5]], ptr [[BLOCK_CAPTURED6]], align 8
 // CHECK12-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___BLOCK_LITERAL_GENERIC:%.*]], ptr [[BLOCK]], i32 0, i32 3
 // CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 8
@@ -2534,7 +2534,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BLOCK_CAPTURED_THIS:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
 // CHECK12-NEXT:    [[THIS:%.*]] = load ptr, ptr [[BLOCK_CAPTURED_THIS]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK12-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
 // CHECK12-NEXT:    store i32 [[INC]], ptr [[TMP0]], align 4
@@ -2543,12 +2543,12 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP2]], -1
 // CHECK12-NEXT:    store i32 [[DEC]], ptr [[BLOCK_CAPTURE_ADDR1]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 8
+// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK12-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP4]], 1
 // CHECK12-NEXT:    store i32 [[DIV]], ptr [[TMP3]], align 4
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 8
+// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
 // CHECK12-NEXT:    store i32 [[TMP6]], ptr [[A_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP7:%.*]] = load i64, ptr [[A_CASTED]], align 8
@@ -2557,7 +2557,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    store i32 [[TMP8]], ptr [[B_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP9:%.*]] = load i64, ptr [[B_CASTED]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 8
+// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
 // CHECK12-NEXT:    store i32 [[TMP11]], ptr [[C_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP12:%.*]] = load i64, ptr [[C_CASTED]], align 8
@@ -2585,14 +2585,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK12-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK12-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK12-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK12-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK12-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK12-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK12-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK12-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK12-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK12-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK12-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -2660,11 +2660,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    store i64 [[VLA2]], ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    store i64 [[VLA4]], ptr [[VLA_ADDR5]], align 8
 // CHECK17-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META3]], !align [[META5:![0-9]+]]
 // CHECK17-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2751,8 +2751,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META3]], !align [[META5]]
+// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
 // CHECK17-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 // CHECK17-NEXT:    [[TMP8:%.*]] = add nuw i64 [[TMP7]], 127
diff --git a/clang/test/OpenMP/sections_firstprivate_codegen.cpp b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
index 7c6d1839fb10e..32c5826e6f75d 100644
--- a/clang/test/OpenMP/sections_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
@@ -448,7 +448,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -517,10 +517,10 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_SECTIONS_LB_]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_UB_]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_ST_]], align 4
@@ -668,7 +668,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -996,7 +996,7 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK4-NEXT:    store i32 0, ptr [[DOTOMP_SECTIONS_LB_]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_UB_]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_ST_]], align 4
diff --git a/clang/test/OpenMP/single_firstprivate_codegen.cpp b/clang/test/OpenMP/single_firstprivate_codegen.cpp
index 27cd220adf225..31ea1ca4952fc 100644
--- a/clang/test/OpenMP/single_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/single_firstprivate_codegen.cpp
@@ -403,7 +403,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -467,10 +467,10 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK1-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_single(ptr @[[GLOB1]], i32 [[TMP5]])
@@ -585,7 +585,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -908,7 +908,7 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_single(ptr @[[GLOB1]], i32 [[TMP2]])
diff --git a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
index aa50d8fb3aabd..a171827a18646 100644
--- a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
@@ -354,9 +354,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -396,9 +396,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -550,12 +550,12 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -655,7 +655,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -711,14 +711,14 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -752,9 +752,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -776,7 +776,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -815,7 +815,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i64 0, i64 [[IDXPROM9]]
@@ -912,7 +912,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1155,9 +1155,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1197,9 +1197,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1349,12 +1349,12 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1454,7 +1454,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1510,14 +1510,14 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1551,9 +1551,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1575,7 +1575,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1613,7 +1613,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP14]]
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP15]], i32 4, i1 false)
@@ -1709,7 +1709,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1877,7 +1877,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -1943,13 +1943,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP8]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP9]], align 8
 // CHECK9-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP11]], ptr [[TMP10]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP12]], align 8
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
index 78e40e54671ac..678770520f677 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -415,9 +415,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -459,9 +459,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -616,9 +616,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -741,12 +741,12 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -846,7 +846,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -902,14 +902,14 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -944,9 +944,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -968,7 +968,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1005,7 +1005,7 @@ int main() {
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
@@ -1082,9 +1082,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1112,7 +1112,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done5:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR2]]
@@ -1151,7 +1151,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -1233,7 +1233,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1476,9 +1476,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1520,9 +1520,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1675,9 +1675,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -1796,12 +1796,12 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1901,7 +1901,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1957,14 +1957,14 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1999,9 +1999,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -2023,7 +2023,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2058,7 +2058,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK3-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK3:       omp.inner.for.inc:
@@ -2135,9 +2135,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -2163,7 +2163,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2201,7 +2201,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -2282,7 +2282,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2450,7 +2450,7 @@ int main() {
 // CHECK5-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK5-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK5-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK5-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK5-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2520,7 +2520,7 @@ int main() {
 // CHECK5-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK5-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK5-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2607,13 +2607,13 @@ int main() {
 // CHECK5-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK5-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK5-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK5-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK5-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK5-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK5-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK5-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK5-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK5-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
@@ -2659,9 +2659,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -2703,9 +2703,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -2860,9 +2860,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -2987,14 +2987,14 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK13-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK13-NEXT:    ret void
 //
@@ -3029,9 +3029,9 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -3053,7 +3053,7 @@ int main() {
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done4:
-// CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3090,7 +3090,7 @@ int main() {
 // CHECK13-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK13-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK13-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK13-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK13:       omp.inner.for.inc:
@@ -3167,9 +3167,9 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -3197,7 +3197,7 @@ int main() {
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done5:
-// CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR3]]
@@ -3236,7 +3236,7 @@ int main() {
 // CHECK13-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK13-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK13-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK13-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK13-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK13-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK13-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -3321,7 +3321,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK13-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -3352,7 +3352,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK13-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -3379,9 +3379,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9:![0-9]+]], !align [[META10:![0-9]+]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -3423,9 +3423,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -3578,9 +3578,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -3701,14 +3701,14 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK15-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK15-NEXT:    ret void
 //
@@ -3743,9 +3743,9 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -3767,7 +3767,7 @@ int main() {
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
-// CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3802,7 +3802,7 @@ int main() {
 // CHECK15-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK15-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK15-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK15-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK15:       omp.inner.for.inc:
@@ -3879,9 +3879,9 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -3907,7 +3907,7 @@ int main() {
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
-// CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3945,7 +3945,7 @@ int main() {
 // CHECK15-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK15-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK15-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK15-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK15-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK15-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -4029,7 +4029,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK15-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -4060,7 +4060,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK15-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -4089,7 +4089,7 @@ int main() {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK17-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK17-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
 // CHECK17-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK17-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -4159,7 +4159,7 @@ int main() {
 // CHECK17-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK17-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK17-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -4246,13 +4246,13 @@ int main() {
 // CHECK17-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK17-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK17-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK17-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK17-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK17-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK17-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK17-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK17-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK17-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
diff --git a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
index 294fcba7872b3..f3c9565a17656 100644
--- a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
@@ -357,9 +357,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -399,9 +399,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -553,14 +553,14 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store i64 [[TMP2]], ptr [[TMP6]], align 8
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -660,7 +660,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -716,14 +716,14 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -757,9 +757,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -781,7 +781,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -820,7 +820,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i64 0, i64 [[IDXPROM9]]
@@ -917,7 +917,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1160,9 +1160,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1202,9 +1202,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1354,14 +1354,14 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store i32 [[TMP2]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1461,7 +1461,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1517,14 +1517,14 @@ int main() {
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1558,9 +1558,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1582,7 +1582,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1620,7 +1620,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP14]]
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP15]], i32 4, i1 false)
@@ -1716,7 +1716,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1884,7 +1884,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -1950,13 +1950,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP8]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP9]], align 8
 // CHECK9-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP11]], ptr [[TMP10]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP12]], align 8
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
index d742b0a85af42..037aa12d57226 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -391,9 +391,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -435,9 +435,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -592,9 +592,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -717,14 +717,14 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store i64 [[TMP2]], ptr [[TMP6]], align 8
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -824,7 +824,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -880,14 +880,14 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -922,9 +922,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -946,7 +946,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -983,7 +983,7 @@ int main() {
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
@@ -1060,9 +1060,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1090,7 +1090,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done5:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR2]]
@@ -1129,7 +1129,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -1211,7 +1211,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1454,9 +1454,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1498,9 +1498,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1653,9 +1653,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -1774,14 +1774,14 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store i32 [[TMP2]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1881,7 +1881,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1937,14 +1937,14 @@ int main() {
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1979,9 +1979,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -2003,7 +2003,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2038,7 +2038,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK3-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK3:       omp.inner.for.inc:
@@ -2115,9 +2115,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -2143,7 +2143,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2181,7 +2181,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -2262,7 +2262,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2430,7 +2430,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2500,7 +2500,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK9-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2587,13 +2587,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK9-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
diff --git a/clang/test/OpenMP/teams_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
index 6f21c9e31bd8d..fec8fcb78f91e 100644
--- a/clang/test/OpenMP/teams_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
@@ -458,9 +458,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11:![0-9]+]], !align [[META12:![0-9]+]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -493,9 +493,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i64 2
@@ -808,7 +808,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -872,9 +872,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -902,9 +902,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i64 2
@@ -1032,7 +1032,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1250,9 +1250,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12:![0-9]+]], !align [[META13:![0-9]+]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1285,9 +1285,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i32 8, i1 false)
 // CHECK11-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i32 2
@@ -1600,7 +1600,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK11-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK11-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK11-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1664,9 +1664,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1694,9 +1694,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i32 8, i1 false)
 // CHECK11-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i32 2
@@ -1824,7 +1824,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK11-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK11-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK11-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1997,7 +1997,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
 // CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[S_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[A_ADDR]], align 8
@@ -2031,11 +2031,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    store i64 [[VLA2]], ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    store i64 [[VLA4]], ptr [[VLA_ADDR5]], align 8
 // CHECK17-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META8]], !align [[META10:![0-9]+]]
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK17-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2235,7 +2235,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[S_ADDR]], align 8
@@ -2273,8 +2273,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META8]], !align [[META10]]
 // CHECK17-NEXT:    [[TMP6:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP6]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2460,7 +2460,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
 // CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[S_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[A_ADDR]], align 4
@@ -2494,11 +2494,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    store i32 [[VLA2]], ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    store i32 [[VLA4]], ptr [[VLA_ADDR5]], align 4
 // CHECK19-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 4
-// CHECK19-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP3:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK19-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = mul nuw i32 [[TMP2]], [[TMP3]]
@@ -2696,7 +2696,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[S_ADDR]], align 4
@@ -2734,8 +2734,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP3:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
-// CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP6:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK19-NEXT:    store ptr [[TMP6]], ptr [[SAVED_STACK]], align 4
 // CHECK19-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP2]], [[TMP3]]
diff --git a/clang/test/Sema/dllexport.c b/clang/test/Sema/dllexport.c
index 3f911fb095c0f..5f6ff36e290e9 100644
--- a/clang/test/Sema/dllexport.c
+++ b/clang/test/Sema/dllexport.c
@@ -2,6 +2,10 @@
 // RUN: %clang_cc1 -triple x86_64-win32   -fsyntax-only -fms-extensions -verify -std=c11 %s
 // RUN: %clang_cc1 -triple i686-mingw32   -fsyntax-only -fms-extensions -verify -std=c11 %s
 // RUN: %clang_cc1 -triple x86_64-mingw32 -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple i686-windows-itanium   -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple x86_64-windows-itanium -fsyntax-only -fms-extensions -verify -std=c11 %s
+// RUN: %clang_cc1 -triple x86_64-sie-ps5         -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple x86_64-sie-ps5         -fsyntax-only -fms-extensions -verify -std=c11 %s
 
 // Invalid usage.
 __declspec(dllexport) typedef int typedef1;
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
index 46c3670848529..135865c8450f5 100644
--- a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
+++ b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -283,3 +283,31 @@ void f() {
 }
 
 #endif
+
+namespace GH147374 {
+
+struct String {};
+template <typename T> void operator+(T, String &&) = delete;
+
+struct Bar {
+    void operator+(String) const; // expected-note {{candidate function}}
+    friend void operator+(Bar, String) {};  // expected-note {{candidate function}}
+};
+
+struct Baz {
+    void operator+(String); // expected-note {{candidate function}}
+    friend void operator+(Baz, String) {}; // expected-note {{candidate function}}
+};
+
+void test() {
+    Bar a;
+    String b;
+    a + b;
+    //expected-error at -1 {{use of overloaded operator '+' is ambiguous (with operand types 'Bar' and 'String')}}
+
+    Baz z;
+    z + b;
+    //expected-error at -1 {{use of overloaded operator '+' is ambiguous (with operand types 'Baz' and 'String')}}
+}
+
+}
diff --git a/clang/test/SemaTemplate/concepts-using-decl.cpp b/clang/test/SemaTemplate/concepts-using-decl.cpp
index fca69dea5c88f..41f7b6d2f8faa 100644
--- a/clang/test/SemaTemplate/concepts-using-decl.cpp
+++ b/clang/test/SemaTemplate/concepts-using-decl.cpp
@@ -176,3 +176,24 @@ void func() {
   f.foo<10, 10>(); // expected-error {{no matching member function for call to 'foo'}}
 }
 } // namespace heads_without_concepts.
+
+namespace GH146614 {
+
+template <typename T>
+struct base {
+    template <typename A>
+    void foo(A x)
+        requires (requires{x;})
+    {}
+};
+
+
+struct child : base<int> {
+  using base<int>::foo;
+  template <typename A>
+  void foo(A x)
+      requires (false)
+  {}
+};
+
+}
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4241d12601242..5f0783f869bf6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1466,8 +1466,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     assert(falseTarget && "missing conditional branch false block");
     mlir::Location loc = toLocation();
     mlir::Value bcc = builder->createConvert(loc, builder->getI1Type(), cond);
-    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget, std::nullopt,
-                                            falseTarget, std::nullopt);
+    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget,
+                                            mlir::ValueRange{}, falseTarget,
+                                            mlir::ValueRange{});
   }
   void genConditionalBranch(mlir::Value cond,
                             Fortran::lower::pft::Evaluation *trueTarget,
@@ -2556,8 +2557,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
           loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
-          /*local_vars=*/std::nullopt,
-          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*local_vars=*/mlir::ValueRange{},
+          /*local_syms=*/nullptr, /*reduce_vars=*/mlir::ValueRange{},
           /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
           /*reduce_attrs=*/nullptr);
 
@@ -3810,9 +3811,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       mlir::Block *selectCaseBlock = insertBlock(blockList[0]);
       mlir::Block *assumedSizeBlock =
           rankStarBlock ? rankStarBlock : defaultBlock;
-      builder->create<mlir::cf::CondBranchOp>(loc, isAssumedSize,
-                                              assumedSizeBlock, std::nullopt,
-                                              selectCaseBlock, std::nullopt);
+      builder->create<mlir::cf::CondBranchOp>(
+          loc, isAssumedSize, assumedSizeBlock, mlir::ValueRange{},
+          selectCaseBlock, mlir::ValueRange{});
       startBlock(selectCaseBlock);
     }
     // Create fir.select_case for the other rank cases.
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1850b67898126..b8ab5d09e3e08 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -303,7 +303,7 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
     return builder.create<fir::StringLitOp>(
-        loc, llvm::ArrayRef<mlir::Type>{type}, std::nullopt, attrs);
+        loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
   }
 }
 
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 0a1cd67789772..281ab229d1b6a 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1003,9 +1003,9 @@ class ScalarExprLowering {
           },
           [&](const fir::MutableBoxValue &toBox) {
             if (toBox.isPointer()) {
-              Fortran::lower::associateMutableBox(converter, loc, toBox, expr,
-                                                  /*lbounds=*/std::nullopt,
-                                                  stmtCtx);
+              Fortran::lower::associateMutableBox(
+                  converter, loc, toBox, expr,
+                  /*lbounds=*/mlir::ValueRange{}, stmtCtx);
               return;
             }
             // For allocatable components, a deep copy is needed.
@@ -3604,8 +3604,9 @@ class ArrayExprLowering {
       mlir::Value castTo =
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
-      return builder.create<fir::ArrayLoadOp>(
-          loc, seqTy, castTo, shapeOp, /*slice=*/mlir::Value{}, std::nullopt);
+      return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
+                                              /*slice=*/mlir::Value{},
+                                              mlir::ValueRange{});
     };
     // Custom lowering of the element store to deal with the extra indirection
     // to the lazy allocated buffer.
@@ -4207,7 +4208,7 @@ class ArrayExprLowering {
       auto addr =
           builder->create<fir::ArrayCoorOp>(loc, eleRefTy, tmp, shape,
                                             /*slice=*/mlir::Value{}, indices,
-                                            /*typeParams=*/std::nullopt);
+                                            /*typeParams=*/mlir::ValueRange{});
       auto load = builder->create<fir::LoadOp>(loc, addr);
       return builder->createConvert(loc, i1Ty, load);
     };
@@ -4522,17 +4523,18 @@ class ArrayExprLowering {
         fir::isRecordWithAllocatableMember(eleTy))
       TODO(loc, "creating an array temp where the element type has "
                 "allocatable members");
-    mlir::Value temp = !seqTy.hasDynamicExtents()
-                           ? builder.create<fir::AllocMemOp>(loc, type)
-                           : builder.create<fir::AllocMemOp>(
-                                 loc, type, ".array.expr", std::nullopt, shape);
+    mlir::Value temp =
+        !seqTy.hasDynamicExtents()
+            ? builder.create<fir::AllocMemOp>(loc, type)
+            : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
+                                              mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
     return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
                                             /*slice=*/mlir::Value{},
-                                            std::nullopt);
+                                            mlir::ValueRange{});
   }
 
   static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -6483,7 +6485,7 @@ class ArrayExprLowering {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
         mem = builder.create<fir::AllocMemOp>(
-            loc, eleTy, /*typeparams=*/std::nullopt, initBuffSz);
+            loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
         builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
       }
     } else {
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 6cda742874ccf..cacf4e249aa28 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -813,6 +813,10 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
       if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(baseTy))
         baseTy = boxTy.getEleTy();
       baseTy = fir::unwrapRefType(baseTy);
+
+      if (mlir::isa<fir::SequenceType>(baseTy))
+        TODO(loc, "array of derived-type with device component");
+
       auto recTy =
           mlir::dyn_cast<fir::RecordType>(fir::unwrapSequenceType(baseTy));
       assert(recTy && "expected fir::RecordType");
@@ -823,7 +827,7 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
         if (Fortran::semantics::IsDeviceAllocatable(sym)) {
           unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
           mlir::Type fieldTy;
-          std::vector<mlir::Value> coordinates;
+          llvm::SmallVector<mlir::Value> coordinates;
 
           if (fieldIdx != std::numeric_limits<unsigned>::max()) {
             // Field found in the base record type.
@@ -866,8 +870,24 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
             TODO(loc, "device resident component in complex derived-type "
                       "hierarchy");
 
-          mlir::Value comp = builder.create<fir::CoordinateOp>(
-              loc, builder.getRefType(fieldTy), fir::getBase(exv), coordinates);
+          mlir::Value base = fir::getBase(exv);
+          mlir::Value comp;
+          if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(base.getType()))) {
+            mlir::Value box = builder.create<fir::LoadOp>(loc, base);
+            mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, box);
+            llvm::SmallVector<mlir::Value> lenParams;
+            assert(coordinates.size() == 1 && "expect one coordinate");
+            auto field = mlir::dyn_cast<fir::FieldIndexOp>(
+                coordinates[0].getDefiningOp());
+            comp = builder.create<hlfir::DesignateOp>(
+                loc, builder.getRefType(fieldTy), addr,
+                /*component=*/field.getFieldName(),
+                /*componentShape=*/mlir::Value{},
+                hlfir::DesignateOp::Subscripts{});
+          } else {
+            comp = builder.create<fir::CoordinateOp>(
+                loc, builder.getRefType(fieldTy), base, coordinates);
+          }
           cuf::DataAttributeAttr dataAttr =
               Fortran::lower::translateSymbolCUFDataAttribute(
                   builder.getContext(), sym);
diff --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index 2be5ef76e46b8..5f73335242336 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
   genUnreachable(builder, loc);
 }
 
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
 }
 
 void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index 389a89ddcf102..c7b3e11728cea 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -122,7 +122,7 @@ class VectorSubscriptBoxBuilder {
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
-        loc, fldTy, componentName, recTy, /*typeParams*/ std::nullopt));
+        loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
 
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index acd5a88a2582d..5b1dbc4435d6c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -620,7 +620,7 @@ fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc,
   mlir::NamedAttribute sizeAttr(sizeTag, getI64IntegerAttr(data.size()));
   llvm::SmallVector<mlir::NamedAttribute> attrs{dataAttr, sizeAttr};
   return create<fir::StringLitOp>(loc, llvm::ArrayRef<mlir::Type>{type},
-                                  std::nullopt, attrs);
+                                  mlir::ValueRange{}, attrs);
 }
 
 mlir::Value fir::FirOpBuilder::genShape(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 773d6408079cc..04703f7911176 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -59,7 +59,8 @@ mlir::Value fir::runtime::genCpuTime(fir::FirOpBuilder &builder,
                                      mlir::Location loc) {
   mlir::func::FuncOp func =
       fir::runtime::getRuntimeFunc<mkRTKey(CpuTime)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
@@ -280,7 +281,8 @@ void fir::runtime::genRename(fir::FirOpBuilder &builder, mlir::Location loc,
 mlir::Value fir::runtime::genTime(fir::FirOpBuilder &builder,
                                   mlir::Location loc) {
   auto func = fir::runtime::getRuntimeFunc<mkRTKey(time)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 /// generate runtime call to transfer intrinsic with no size argument
diff --git a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
index 411181cc6dd1c..9b5e43b80b1f6 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
@@ -25,7 +25,7 @@ void fir::runtime::genExit(fir::FirOpBuilder &builder, mlir::Location loc,
 void fir::runtime::genAbort(fir::FirOpBuilder &builder, mlir::Location loc) {
   mlir::func::FuncOp abortFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(Abort)>(loc, builder);
-  builder.create<fir::CallOp>(loc, abortFunc, std::nullopt);
+  builder.create<fir::CallOp>(loc, abortFunc, mlir::ValueRange{});
 }
 
 void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 03cc92e975b19..c5cf01ed98357 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -405,7 +405,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
   mlir::Location loc = forallMaskOp.getLoc();
   mlir::Value mask = generateYieldedScalarValue(forallMaskOp.getMaskRegion(),
                                                 builder.getI1Type());
-  auto ifOp = builder.create<fir::IfOp>(loc, std::nullopt, mask, false);
+  auto ifOp = builder.create<fir::IfOp>(loc, mlir::TypeRange{}, mask, false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   constructStack.push_back(ifOp);
 }
@@ -530,7 +530,7 @@ void OrderedAssignmentRewriter::generateMaskIfOp(mlir::Value cdt) {
   mlir::Location loc = cdt.getLoc();
   cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
   cdt = builder.createConvert(loc, builder.getI1Type(), cdt);
-  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), std::nullopt, cdt,
+  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), mlir::TypeRange{}, cdt,
                                         /*withElseRegion=*/false);
   constructStack.push_back(ifOp.getOperation());
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
diff --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index 1f8edf851de9b..bc4fcd8b0112e 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -222,7 +222,7 @@ void AllocaReplaceImpl::genIndirectDeallocation(
         rewriter.create<fir::ConvertOp>(loc, intPtrTy, ptrVal);
     mlir::Value isAllocated = rewriter.create<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::ne, ptrToInt, c0);
-    auto ifOp = rewriter.create<fir::IfOp>(loc, std::nullopt, isAllocated,
+    auto ifOp = rewriter.create<fir::IfOp>(loc, mlir::TypeRange{}, isAllocated,
                                            /*withElseRegion=*/false);
     rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value cast = fir::factory::createConvert(
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 57eae1ff052a2..6e45aae4246d0 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -456,7 +456,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
   rewriter.setInsertionPointToEnd(thisBlock);
   if (destOps.has_value())
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, destOps.value(),
-                                            newBlock, std::nullopt);
+                                            newBlock, mlir::ValueRange{});
   else
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, newBlock);
   rewriter.setInsertionPointToEnd(newBlock);
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 506c8e66dbdfa..ad8464b495888 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -261,7 +261,7 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
+          /*iterArgs=*/mlir::ValueRange{}, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
diff --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index ee89ea38a3fc7..e3bb181f65398 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -21,4 +21,36 @@ contains
 ! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]]#0, z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[Z]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
 
+  subroutine sub2()
+    type(ty_device), pointer :: d1
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub2()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub2Ed1"} -> !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMm1Fsub2Ed1"} : (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
+  subroutine sub3()
+    type(ty_device), allocatable :: d1
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub3()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub3Ed1"} -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMm1Fsub3Ed1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
 end module
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index 98270adaa7c73..59808779aa6ef 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -48,7 +48,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
   mlir::Value addr = builder->create<fir::AllocaOp>(loc, eleType);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      /*shape=*/mlir::Value{}, /*typeParams=*/std::nullopt,
+      /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{},
       /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
@@ -102,11 +102,11 @@ TEST_F(FortranVariableTest, SimpleArray) {
       extents.size(), fir::SequenceType::getUnknownExtent());
   mlir::Type seqTy = fir::SequenceType::get(typeShape, eleType);
   mlir::Value addr = builder->create<fir::AllocaOp>(
-      loc, seqTy, /*pinned=*/false, /*typeParams=*/std::nullopt, extents);
+      loc, seqTy, /*pinned=*/false, /*typeParams=*/mlir::ValueRange{}, extents);
   mlir::Value shape = createShape(extents);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      shape, /*typeParams*/ std::nullopt, /*dummy_scope=*/nullptr, name,
+      shape, /*typeParams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
 
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index e210992c5111a..3fb62788c1168 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -571,6 +571,8 @@ function(add_integration_test test_name)
   target_compile_options(${fq_build_target_name} PRIVATE
                          ${compile_options} ${INTEGRATION_TEST_COMPILE_OPTIONS})
 
+  set(compiler_runtime "")
+
   if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU)
     target_link_options(${fq_build_target_name} PRIVATE
       ${LIBC_COMPILE_OPTIONS_DEFAULT} ${INTEGRATION_TEST_COMPILE_OPTIONS}
@@ -599,17 +601,19 @@ function(add_integration_test test_name)
     set(link_options
       -nolibc
       -nostartfiles
-      -static
+      -nostdlib
       ${LIBC_LINK_OPTIONS_DEFAULT}
       ${LIBC_TEST_LINK_OPTIONS_DEFAULT}
     )
     target_link_options(${fq_build_target_name} PRIVATE ${link_options})
+    list(APPEND compiler_runtime ${LIBGCC_S_LOCATION})
   endif()
   target_link_libraries(
     ${fq_build_target_name}
-    ${fq_target_name}.__libc__
     libc.startup.${LIBC_TARGET_OS}.crt1
     libc.test.IntegrationTest.test
+    ${fq_target_name}.__libc__
+    ${compiler_runtime}
   )
   add_dependencies(${fq_build_target_name}
                    libc.test.IntegrationTest.test
@@ -770,6 +774,7 @@ function(add_libc_hermetic test_name)
                          ${HERMETIC_TEST_COMPILE_OPTIONS})
 
   set(link_libraries "")
+  set(compiler_runtime "")
   foreach(lib IN LISTS HERMETIC_TEST_LINK_LIBRARIES)
     if(TARGET ${lib}.hermetic)
       list(APPEND link_libraries ${lib}.hermetic)
@@ -807,12 +812,12 @@ function(add_libc_hermetic test_name)
     set(link_options
       -nolibc
       -nostartfiles
-      -static
+      -nostdlib
       ${LIBC_LINK_OPTIONS_DEFAULT}
       ${LIBC_TEST_LINK_OPTIONS_DEFAULT}
     )
     target_link_options(${fq_build_target_name} PRIVATE ${link_options})
-    list(APPEND link_libraries ${LIBGCC_S_LOCATION})
+    list(APPEND compiler_runtime ${LIBGCC_S_LOCATION})
   endif()
   target_link_libraries(
     ${fq_build_target_name}
@@ -820,7 +825,9 @@ function(add_libc_hermetic test_name)
       libc.startup.${LIBC_TARGET_OS}.crt1
       ${link_libraries}
       LibcHermeticTestSupport.hermetic
-      ${fq_target_name}.__libc__)
+      ${fq_target_name}.__libc__
+      ${compiler_runtime}
+    )
   add_dependencies(${fq_build_target_name}
                    LibcTest.hermetic
                    libc.test.UnitTest.ErrnoSetterMatcher
diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in
index fc01aaf2d8746..b68c0c8258366 100644
--- a/libcxx/include/__config_site.in
+++ b/libcxx/include/__config_site.in
@@ -30,7 +30,6 @@
 #cmakedefine01 _LIBCPP_HAS_LOCALIZATION
 #cmakedefine01 _LIBCPP_HAS_UNICODE
 #cmakedefine01 _LIBCPP_HAS_WIDE_CHARACTERS
-#cmakedefine _LIBCPP_HAS_NO_STD_MODULES
 #cmakedefine01 _LIBCPP_HAS_TIME_ZONE_DATABASE
 #cmakedefine01 _LIBCPP_INSTRUMENTED_WITH_ASAN
 
diff --git a/libcxx/test/libcxx/fuzzing/random.pass.cpp b/libcxx/test/libcxx/fuzzing/random.pass.cpp
index cb074bd60fdc8..f0256a01f29ae 100644
--- a/libcxx/test/libcxx/fuzzing/random.pass.cpp
+++ b/libcxx/test/libcxx/fuzzing/random.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // UNSUPPORTED: c++03, c++11
 
diff --git a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
index 1ba0063c1dada..f9f81d22ff80e 100644
--- a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // <math.h>
 
diff --git a/libcxx/test/std/numerics/c.math/cmath.pass.cpp b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
index 48c2918802fc3..8d261e9fcbdb2 100644
--- a/libcxx/test/std/numerics/c.math/cmath.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // <cmath>
 
diff --git a/lld/ELF/BPSectionOrderer.cpp b/lld/ELF/BPSectionOrderer.cpp
index f464b1d4518a4..06152046d13d4 100644
--- a/lld/ELF/BPSectionOrderer.cpp
+++ b/lld/ELF/BPSectionOrderer.cpp
@@ -76,10 +76,10 @@ DenseMap<const InputSectionBase *, int> elf::runBalancedPartitioning(
     if (!d)
       return;
     auto *sec = dyn_cast_or_null<InputSection>(d->section);
-    // Skip empty, discarded, ICF folded sections. Skipping ICF folded sections
-    // reduces duplicate detection work in BPSectionOrderer.
+    // Skip empty, discarded, ICF folded sections, .bss. Skipping ICF folded
+    // sections reduces duplicate detection work in BPSectionOrderer.
     if (!sec || sec->size == 0 || !sec->isLive() || sec->repl != sec ||
-        !orderer.secToSym.try_emplace(sec, d).second)
+        !sec->content().data() || !orderer.secToSym.try_emplace(sec, d).second)
       return;
     rootSymbolToSectionIdxs[CachedHashStringRef(
                                 lld::utils::getRootSymbol(sym.getName()))]
diff --git a/lld/test/ELF/bp-section-orderer.s b/lld/test/ELF/bp-section-orderer.s
index 4df2e8d43022e..438d7c2da0f76 100644
--- a/lld/test/ELF/bp-section-orderer.s
+++ b/lld/test/ELF/bp-section-orderer.s
@@ -26,28 +26,28 @@
 
 # RUN: ld.lld -o out.s a.o --irpgo-profile=a.profdata --bp-startup-sort=function
 # RUN: llvm-nm -jn out.s | tr '\n' , | FileCheck %s --check-prefix=STARTUP
-# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,{{$}}
+# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.os a.o --irpgo-profile=a.profdata --bp-startup-sort=function --symbol-ordering-file a.txt
 # RUN: llvm-nm -jn out.os | tr '\n' , | FileCheck %s --check-prefix=ORDER-STARTUP
-# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,{{$}}
+# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.cf a.o --verbose-bp-section-orderer --bp-compression-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-FUNC
 # RUN: ld.lld -o out.cf.icf a.o --verbose-bp-section-orderer --bp-compression-sort=function --icf=all --gc-sections 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-ICF-FUNC
 # RUN: llvm-nm -jn out.cf | tr '\n' , | FileCheck %s --check-prefix=CFUNC
-# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,{{$}}
+# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.cd a.o --verbose-bp-section-orderer --bp-compression-sort=data 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-DATA
 # RUN: llvm-nm -jn out.cd | tr '\n' , | FileCheck %s --check-prefix=CDATA
-# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,{{$}}
+# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
 
 # RUN: ld.lld -o out.cb a.o --verbose-bp-section-orderer --bp-compression-sort=both 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
 # RUN: llvm-nm -jn out.cb | tr '\n' , | FileCheck %s --check-prefix=CBOTH
-# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,{{$}}
+# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,g1,{{$}}
 
 # RUN: ld.lld -o out.cbs a.o --verbose-bp-section-orderer --bp-compression-sort=both --irpgo-profile=a.profdata --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
 # RUN: llvm-nm -jn out.cbs | tr '\n' , | FileCheck %s --check-prefix=CBOTH-STARTUP
-# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,{{$}}
+# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
 
 # BP-COMPRESSION-FUNC: Ordered 9 sections ([[#]] bytes) using balanced partitioning
 # BP-COMPRESSION-ICF-FUNC: Ordered 8 sections ([[#]] bytes) using balanced partitioning
@@ -108,6 +108,7 @@ d3
 d2
 
 #--- a.c
+int g1;
 const char s5[] = "engineering";
 const char s4[] = "computer program";
 const char s3[] = "hardware engineer";
@@ -377,6 +378,14 @@ d1:
 	.word	6                               // 0x6
 	.size	d1, 16
 
+	.type	g1, at object                      // @g1
+	.section	.bss.g1,"aw", at nobits
+	.globl	g1
+	.p2align	2, 0x0
+g1:
+	.word	0                               // 0x0
+	.size	g1, 4
+
 	.section	".note.GNU-stack","", at progbits
 	.addrsig
 	.addrsig_sym F
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index e445fa8833022..6f812b91a8b1d 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -700,7 +700,7 @@ void IRExecutionUnit::CollectCandidateCPlusPlusNames(
 
 class LoadAddressResolver {
 public:
-  LoadAddressResolver(Target *target, bool &symbol_was_missing_weak)
+  LoadAddressResolver(Target &target, bool &symbol_was_missing_weak)
       : m_target(target), m_symbol_was_missing_weak(symbol_was_missing_weak) {}
 
   std::optional<lldb::addr_t> Resolve(SymbolContextList &sc_list) {
@@ -722,11 +722,11 @@ class LoadAddressResolver {
 
       // First try the symbol.
       if (candidate_sc.symbol) {
-        load_address = candidate_sc.symbol->ResolveCallableAddress(*m_target);
+        load_address = candidate_sc.symbol->ResolveCallableAddress(m_target);
         if (load_address == LLDB_INVALID_ADDRESS) {
           Address addr = candidate_sc.symbol->GetAddress();
-          load_address = m_target->GetProcessSP()
-                             ? addr.GetLoadAddress(m_target)
+          load_address = m_target.GetProcessSP()
+                             ? addr.GetLoadAddress(&m_target)
                              : addr.GetFileAddress();
         }
       }
@@ -734,8 +734,8 @@ class LoadAddressResolver {
       // If that didn't work, try the function.
       if (load_address == LLDB_INVALID_ADDRESS && candidate_sc.function) {
         Address addr = candidate_sc.function->GetAddress();
-        load_address = m_target->GetProcessSP() ? addr.GetLoadAddress(m_target)
-                                                : addr.GetFileAddress();
+        load_address = m_target.GetProcessSP() ? addr.GetLoadAddress(&m_target)
+                                               : addr.GetFileAddress();
       }
 
       // We found a load address.
@@ -766,7 +766,7 @@ class LoadAddressResolver {
   }
 
 private:
-  Target *m_target;
+  Target &m_target;
   bool &m_symbol_was_missing_weak;
   lldb::addr_t m_best_internal_load_address = LLDB_INVALID_ADDRESS;
 };
@@ -790,7 +790,7 @@ IRExecutionUnit::FindInSymbols(const std::vector<ConstString> &names,
   for (size_t i = 0; i < m_preferred_modules.GetSize(); ++i)
     non_local_images.Remove(m_preferred_modules.GetModuleAtIndex(i));
 
-  LoadAddressResolver resolver(target, symbol_was_missing_weak);
+  LoadAddressResolver resolver(*target, symbol_was_missing_weak);
 
   ModuleFunctionSearchOptions function_options;
   function_options.include_symbols = true;
diff --git a/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
index e1d7e42bdd1a9..cd60227572be4 100644
--- a/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
+++ b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
@@ -2,7 +2,6 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -120,6 +119,8 @@ def cleanup():
                 '@"2 elements"',
             ],
         )
+
+        self.runCmd("settings set target.max-children-depth 6")
         self.expect(
             "frame variable mutabledict --ptr-depth 3",
             substrs=[
diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script b/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
index 91de55f4ade4a..43018eacf709b 100644
--- a/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
+++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
@@ -1,3 +1,4 @@
+settings set target.max-children-depth 10
 breakpoint set --file UdtLayoutTest.cpp --line 60
 run
 target variable
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index de5f66ce1584c..87d2a9ac3bf88 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -263,6 +263,20 @@ amaclean at nvidia.com (email), [AlexMaclean](https://github.com/AlexMaclean) (GitH
 
 #### PowerPC backend
 
+Amy Kwan (esp. release issues) \
+Amy.Kwan1 at ibm.com (email), [amy-kwan](https://github.com/amy-kwan) (GitHub) \
+Lei Huang \
+lei at ca.ibm.com (email), [lei137](https://github.com/lei137) (GitHub) \
+Sean Fertile (esp. ABI/ELF/XCOFF) \
+sfertile at ca.ibm.com (email), [mandlebug](https://github.com/mandlebug) (GitHub) \
+Zhijian Lin \
+zhijian at ca.ibm.com (email), [diggerlin](https://github.com/diggerlin) (GitHub) \
+Maryam Moghadas \
+maryammo at ca.ibm.com (email), [maryammo](https://github.com/maryammo) (GitHub) \
+Roland Froese \
+froese at ca.ibm.com (email), [RolandF77](https://github.com/RolandF77) (GitHub) \
+llvmonpower \
+powerllvm at ca.ibm.com (email), [llvmonpower](https://github.com/llvmonpower) (GitHub)
 
 #### RISCV backend
 
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index c614a6d7ace9e..732227b98ab9e 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -30,7 +30,7 @@ because the naming and other conventions are dictated by the C++ standard.
 
 There are some conventions that are not uniformly followed in the code base
 (e.g. the naming convention).  This is because they are relatively new, and a
-lot of code was written before they were put in place.  Our long term goal is
+lot of code was written before they were put in place.  Our long-term goal is
 for the entire codebase to follow the convention, but we explicitly *do not*
 want patches that do large-scale reformatting of existing code.  On the other
 hand, it is reasonable to rename the methods of a class if you're about to
@@ -50,7 +50,7 @@ code imported into the tree. Generally, our preference is for standards
 conforming, modern, and portable C++ code as the implementation language of
 choice.
 
-For automation, build-systems and utility scripts Python is preferred and
+For automation, build-systems, and utility scripts, Python is preferred and
 is widely used in the LLVM repository already.
 
 C++ Standard Versions
@@ -92,7 +92,7 @@ LLVM support libraries (for example, `ADT
 <https://github.com/llvm/llvm-project/tree/main/llvm/include/llvm/ADT>`_)
 implement specialized data structures or functionality missing in the standard
 library. Such libraries are usually implemented in the ``llvm`` namespace and
-follow the expected standard interface, when there is one.
+follow the expected standard interface when there is one.
 
 When both C++ and the LLVM support libraries provide similar functionality, and
 there isn't a specific reason to favor the C++ implementation, it is generally
@@ -325,8 +325,8 @@ implementation file.  In any case, implementation files can include additional
 comments (not necessarily in Doxygen markup) to explain implementation details
 as needed.
 
-Don't duplicate function or class name at the beginning of the comment.
-For humans it is obvious which function or class is being documented;
+Don't duplicate the function or class name at the beginning of the comment.
+For humans, it is obvious which function or class is being documented;
 automatic documentation processing tools are smart enough to bind the comment
 to the correct declaration.
 
@@ -369,7 +369,7 @@ lower-case letter, and finish the last sentence without a period, if it would
 end in one otherwise. Sentences which end with different punctuation, such as
 "did you forget ';'?", should still do so.
 
-For example this is a good error message:
+For example, this is a good error message:
 
 .. code-block:: none
 
@@ -443,7 +443,7 @@ Write your code to fit within 80 columns.
 There must be some limit to the width of the code in
 order to allow developers to have multiple files side-by-side in
 windows on a modest display.  If you are going to pick a width limit, it is
-somewhat arbitrary but you might as well pick something standard.  Going with 90
+somewhat arbitrary, but you might as well pick something standard.  Going with 90
 columns (for example) instead of 80 columns wouldn't add any significant value
 and would be detrimental to printing out code.  Also many other projects have
 standardized on 80 columns, so some people have already configured their editors
@@ -520,7 +520,7 @@ within each other and within function calls in order to build up aggregates
 The historically common formatting of braced initialization of aggregate
 variables does not mix cleanly with deep nesting, general expression contexts,
 function arguments, and lambdas. We suggest new code use a simple rule for
-formatting braced initialization lists: act as-if the braces were parentheses
+formatting braced initialization lists: act as if the braces were parentheses
 in a function call. The formatting rules exactly match those already well
 understood for formatting nested function calls. Examples:
 
@@ -607,11 +607,11 @@ Static constructors and destructors (e.g., global variables whose types have a
 constructor or destructor) should not be added to the code base, and should be
 removed wherever possible.
 
-Globals in different source files are initialized in `arbitrary order
+Globals in different source files are initialized in an `arbitrary order
 <https://yosefk.com/c++fqa/ctors.html#fqa-10.12>`_, making the code more
 difficult to reason about.
 
-Static constructors have negative impact on launch time of programs that use
+Static constructors have a negative impact on the launch time of programs that use
 LLVM as a library. We would really like for there to be zero cost for linking
 in an additional LLVM target or other library into an application, but static
 constructors undermine this goal.
@@ -698,7 +698,7 @@ If you use a braced initializer list when initializing a variable, use an equals
 Use ``auto`` Type Deduction to Make Code More Readable
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Some are advocating a policy of "almost always ``auto``" in C++11, however LLVM
+Some are advocating a policy of "almost always ``auto``" in C++11; however, LLVM
 uses a more moderate stance. Use ``auto`` if and only if it makes the code more
 readable or easier to maintain. Don't "almost always" use ``auto``, but do use
 ``auto`` with initializers like ``cast<Foo>(...)`` or other places where the
@@ -783,14 +783,14 @@ guards, and might not include their prerequisites. Name such files with the
 
 In general, a header should be implemented by one or more ``.cpp`` files.  Each
 of these ``.cpp`` files should include the header that defines their interface
-first.  This ensures that all of the dependences of the header have been
+first.  This ensures that all of the dependencies of the header have been
 properly added to the header itself, and are not implicit.  System headers
 should be included after user headers for a translation unit.
 
 Library Layering
 ^^^^^^^^^^^^^^^^
 
-A directory of header files (for example ``include/llvm/Foo``) defines a
+A directory of header files (for example, ``include/llvm/Foo``) defines a
 library (``Foo``). One library (both
 its headers and implementation) should only use things from the libraries
 listed in its dependencies.
@@ -822,7 +822,7 @@ especially in header files.
 
 But wait! Sometimes you need to have the definition of a class to use it, or to
 inherit from it.  In these cases go ahead and ``#include`` that header file.  Be
-aware however that there are many cases where you don't need to have the full
+aware, however, that there are many cases where you don't need to have the full
 definition of a class.  If you are using a pointer or reference to a class, you
 don't need the header file.  If you are simply returning a class instance from a
 prototyped function or method, you don't need it.  In fact, for most cases, you
@@ -970,7 +970,7 @@ loops.  A silly example is something like this:
 When you have very, very small loops, this sort of structure is fine. But if it
 exceeds more than 10-15 lines, it becomes difficult for people to read and
 understand at a glance. The problem with this sort of code is that it gets very
-nested very quickly. Meaning that the reader of the code has to keep a lot of
+nested very quickly. This means that the reader of the code has to keep a lot of
 context in their brain to remember what is going immediately on in the loop,
 because they don't know if/when the ``if`` conditions will have ``else``\s etc.
 It is strongly preferred to structure the loop like this:
@@ -988,7 +988,7 @@ It is strongly preferred to structure the loop like this:
     ...
   }
 
-This has all the benefits of using early exits for functions: it reduces nesting
+This has all the benefits of using early exits for functions: it reduces the nesting
 of the loop, it makes it easier to describe why the conditions are true, and it
 makes it obvious to the reader that there is no ``else`` coming up that they
 have to push context into their brain for.  If a loop is large, this can be a
@@ -1149,12 +1149,12 @@ In general, names should be in camel case (e.g. ``TextFileReader`` and
   nouns and start with an upper-case letter (e.g. ``TextFileReader``).
 
 * **Variable names** should be nouns (as they represent state).  The name should
-  be camel case, and start with an upper case letter (e.g. ``Leader`` or
+  be camel case, and start with an upper-case letter (e.g. ``Leader`` or
   ``Boats``).
 
 * **Function names** should be verb phrases (as they represent actions), and
   command-like function should be imperative.  The name should be camel case,
-  and start with a lower case letter (e.g. ``openFile()`` or ``isFoo()``).
+  and start with a lower-case letter (e.g. ``openFile()`` or ``isFoo()``).
 
 * **Enum declarations** (e.g. ``enum Foo {...}``) are types, so they should
   follow the naming conventions for types.  A common use for enums is as a
@@ -1207,7 +1207,7 @@ Assert Liberally
 ^^^^^^^^^^^^^^^^
 
 Use the "``assert``" macro to its fullest.  Check all of your preconditions and
-assumptions, you never know when a bug (not necessarily even yours) might be
+assumptions.  You never know when a bug (not necessarily even yours) might be
 caught early by an assertion, which reduces debugging time dramatically.  The
 "``<cassert>``" header file is probably already included by the header files you
 are using, so it doesn't cost anything to use it.
@@ -1302,7 +1302,7 @@ preferred to write the code like this:
   assert(NewToSet && "The value shouldn't be in the set yet");
 
 In C code where ``[[maybe_unused]]`` is not supported, use ``void`` cast to
-suppress unused variable warning as follows:
+suppress an unused variable warning as follows:
 
 .. code-block:: c
 
@@ -1546,7 +1546,7 @@ whenever possible.
 The semantics of postincrement include making a copy of the value being
 incremented, returning it, and then preincrementing the "work value".  For
 primitive types, this isn't a big deal. But for iterators, it can be a huge
-issue (for example, some iterators contains stack and set objects in them...
+issue (for example, some iterators contain stack and set objects in them...
 copying an iterator could invoke the copy ctor's of these as well).  In general,
 get in the habit of always using preincrement, and you won't have a problem.
 
@@ -1663,7 +1663,7 @@ Don't Use Braces on Simple Single-Statement Bodies of if/else/loop Statements
 
 When writing the body of an ``if``, ``else``, or for/while loop statement, we
 prefer to omit the braces to avoid unnecessary line noise. However, braces
-should be used in cases where the omission of braces harm the readability and
+should be used in cases where the omission of braces harms the readability and
 maintainability of the code.
 
 We consider that readability is harmed when omitting the brace in the presence
@@ -1763,7 +1763,7 @@ would help to avoid running into a "dangling else" situation.
         handleAttrOnDecl(D, A, i);
   }
 
-  // Use braces on the outer block because of a nested `if`; otherwise the
+  // Use braces on the outer block because of a nested `if`; otherwise, the
   // compiler would warn: `add explicit braces to avoid dangling else`
   if (auto *D = dyn_cast<FunctionDecl>(D)) {
     if (shouldProcess(D))
diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index aef91909dd17b..794075201d646 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -39,8 +39,8 @@ class MachineBlockFrequencyInfo {
 public:
   LLVM_ABI MachineBlockFrequencyInfo(); // Legacy pass manager only.
   LLVM_ABI explicit MachineBlockFrequencyInfo(
-      MachineFunction &F, MachineBranchProbabilityInfo &MBPI,
-      MachineLoopInfo &MLI);
+      const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
+      const MachineLoopInfo &MLI);
   LLVM_ABI MachineBlockFrequencyInfo(MachineBlockFrequencyInfo &&);
   LLVM_ABI ~MachineBlockFrequencyInfo();
 
diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h
index 77cee875f16e7..f8241a3cdf160 100644
--- a/llvm/include/llvm/IR/DebugInfo.h
+++ b/llvm/include/llvm/IR/DebugInfo.h
@@ -115,8 +115,7 @@ class DebugInfoFinder {
   LLVM_ABI void processVariable(DILocalVariable *DVI);
   /// Process debug info location.
   LLVM_ABI void processLocation(const Module &M, const DILocation *Loc);
-  /// Process a DbgRecord (e.g, treat a DbgVariableRecord like a
-  /// DbgVariableIntrinsic).
+  /// Process a DbgRecord.
   LLVM_ABI void processDbgRecord(const Module &M, const DbgRecord &DR);
 
   /// Process subprogram.
@@ -290,8 +289,6 @@ struct VarRecord {
   DILocalVariable *Var;
   DILocation *DL;
 
-  VarRecord(DbgVariableIntrinsic *DVI)
-      : Var(DVI->getVariable()), DL(getDebugValueLoc(DVI)) {}
   VarRecord(DbgVariableRecord *DVR)
       : Var(DVR->getVariable()), DL(getDebugValueLoc(DVR)) {}
   VarRecord(DILocalVariable *Var, DILocation *DL) : Var(Var), DL(DL) {}
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 9345f95015301..f1f0c18949c35 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -66,7 +66,6 @@ namespace dwarf {
 enum Tag : uint16_t;
 }
 
-class DbgVariableIntrinsic;
 class DbgVariableRecord;
 
 LLVM_ABI extern cl::opt<bool> EnableFSDiscriminator;
@@ -4613,7 +4612,6 @@ class DebugVariable {
   LLVM_ABI static const FragmentInfo DefaultFragment;
 
 public:
-  LLVM_ABI DebugVariable(const DbgVariableIntrinsic *DII);
   LLVM_ABI DebugVariable(const DbgVariableRecord *DVR);
 
   DebugVariable(const DILocalVariable *Var,
@@ -4681,7 +4679,6 @@ template <> struct DenseMapInfo<DebugVariable> {
 /// information).
 class DebugVariableAggregate : public DebugVariable {
 public:
-  LLVM_ABI DebugVariableAggregate(const DbgVariableIntrinsic *DVI);
   DebugVariableAggregate(const DebugVariable &V)
       : DebugVariable(V.getVariable(), std::nullopt, V.getInlinedAt()) {}
 };
diff --git a/llvm/include/llvm/Transforms/Utils/SSAUpdater.h b/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
index 4e5da81a7e885..9500b1f160ea9 100644
--- a/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
+++ b/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
@@ -29,7 +29,6 @@ template <typename T> class SSAUpdaterTraits;
 class Type;
 class Use;
 class Value;
-class DbgValueInst;
 
 /// Helper class for SSA formation on a set of values defined in
 /// multiple blocks.
@@ -122,8 +121,6 @@ class SSAUpdater {
   /// the instruction. Anything outside of its block will have its
   /// value set to the new SSA value if available, and undef if not.
   void UpdateDebugValues(Instruction *I);
-  void UpdateDebugValues(Instruction *I,
-                         SmallVectorImpl<DbgValueInst *> &DbgValues);
   void UpdateDebugValues(Instruction *I,
                          SmallVectorImpl<DbgVariableRecord *> &DbgValues);
 
@@ -136,7 +133,6 @@ class SSAUpdater {
 
 private:
   Value *GetValueAtEndOfBlockInternal(BasicBlock *BB);
-  void UpdateDebugValue(Instruction *I, DbgValueInst *DbgValue);
   void UpdateDebugValue(Instruction *I, DbgVariableRecord *DbgValue);
 };
 
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index d9d41f1d72e35..dc8184394f74d 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -445,7 +445,6 @@ class CodeGenPrepare {
   bool optimizeSwitchInst(SwitchInst *SI);
   bool optimizeExtractElementInst(Instruction *Inst);
   bool dupRetToEnableTailCallOpts(BasicBlock *BB, ModifyDT &ModifiedDT);
-  bool fixupDbgValue(Instruction *I);
   bool fixupDbgVariableRecord(DbgVariableRecord &I);
   bool fixupDbgVariableRecordsOnInst(Instruction &I);
   bool placeDbgValues(Function &F);
@@ -2762,9 +2761,6 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
     case Intrinsic::fshl:
     case Intrinsic::fshr:
       return optimizeFunnelShift(II);
-    case Intrinsic::dbg_assign:
-    case Intrinsic::dbg_value:
-      return fixupDbgValue(II);
     case Intrinsic::masked_gather:
       return optimizeGatherScatterInst(II, II->getArgOperand(0));
     case Intrinsic::masked_scatter:
@@ -3554,8 +3550,6 @@ class TypePromotionTransaction {
     /// Keep track of the original uses (pair Instruction, Index).
     SmallVector<InstructionAndIdx, 4> OriginalUses;
     /// Keep track of the debug users.
-    SmallVector<DbgValueInst *, 1> DbgValues;
-    /// And non-instruction debug-users too.
     SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
 
     /// Keep track of the new value so that we can undo it by replacing
@@ -3577,7 +3571,9 @@ class TypePromotionTransaction {
       }
       // Record the debug uses separately. They are not in the instruction's
       // use list, but they are replaced by RAUW.
+      SmallVector<DbgValueInst *> DbgValues;
       findDbgValues(DbgValues, Inst, &DbgVariableRecords);
+      assert(DbgValues.empty());
 
       // Now, we can replace the uses.
       Inst->replaceAllUsesWith(New);
@@ -3591,11 +3587,7 @@ class TypePromotionTransaction {
       // RAUW has replaced all original uses with references to the new value,
       // including the debug uses. Since we are undoing the replacements,
       // the original debug uses must also be reinstated to maintain the
-      // correctness and utility of debug value instructions.
-      for (auto *DVI : DbgValues)
-        DVI->replaceVariableLocationOp(New, Inst);
-      // Similar story with DbgVariableRecords, the non-instruction
-      // representation of dbg.values.
+      // correctness and utility of debug value records.
       for (DbgVariableRecord *DVR : DbgVariableRecords)
         DVR->replaceVariableLocationOp(New, Inst);
     }
@@ -8933,32 +8925,6 @@ bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT) {
   return MadeChange;
 }
 
-// Some CGP optimizations may move or alter what's computed in a block. Check
-// whether a dbg.value intrinsic could be pointed at a more appropriate operand.
-bool CodeGenPrepare::fixupDbgValue(Instruction *I) {
-  assert(isa<DbgValueInst>(I));
-  DbgValueInst &DVI = *cast<DbgValueInst>(I);
-
-  // Does this dbg.value refer to a sunk address calculation?
-  bool AnyChange = false;
-  SmallDenseSet<Value *> LocationOps(DVI.location_ops().begin(),
-                                     DVI.location_ops().end());
-  for (Value *Location : LocationOps) {
-    WeakTrackingVH SunkAddrVH = SunkAddrs[Location];
-    Value *SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;
-    if (SunkAddr) {
-      // Point dbg.value at locally computed address, which should give the best
-      // opportunity to be accurately lowered. This update may change the type
-      // of pointer being referred to; however this makes no difference to
-      // debugging information, and we can't generate bitcasts that may affect
-      // codegen.
-      DVI.replaceVariableLocationOp(Location, SunkAddr);
-      AnyChange = true;
-    }
-  }
-  return AnyChange;
-}
-
 bool CodeGenPrepare::fixupDbgVariableRecordsOnInst(Instruction &I) {
   bool AnyChange = false;
   for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
@@ -8993,14 +8959,6 @@ bool CodeGenPrepare::fixupDbgVariableRecord(DbgVariableRecord &DVR) {
   return AnyChange;
 }
 
-static void DbgInserterHelper(DbgValueInst *DVI, BasicBlock::iterator VI) {
-  DVI->removeFromParent();
-  if (isa<PHINode>(VI))
-    DVI->insertBefore(VI->getParent()->getFirstInsertionPt());
-  else
-    DVI->insertAfter(VI);
-}
-
 static void DbgInserterHelper(DbgVariableRecord *DVR, BasicBlock::iterator VI) {
   DVR->removeFromParent();
   BasicBlock *VIBB = VI->getParent();
@@ -9065,15 +9023,8 @@ bool CodeGenPrepare::placeDbgValues(Function &F) {
 
   for (BasicBlock &BB : F) {
     for (Instruction &Insn : llvm::make_early_inc_range(BB)) {
-      // Process dbg.value intrinsics.
-      DbgValueInst *DVI = dyn_cast<DbgValueInst>(&Insn);
-      if (DVI) {
-        DbgProcessor(DVI, DVI);
-        continue;
-      }
-
-      // If this isn't a dbg.value, process any attached DbgVariableRecord
-      // records attached to this instruction.
+      // Process any DbgVariableRecord records attached to this
+      // instruction.
       for (DbgVariableRecord &DVR : llvm::make_early_inc_range(
                filterDbgVars(Insn.getDbgRecordRange()))) {
         if (DVR.Type != DbgVariableRecord::LocationType::Value)
diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index 9daacfd399787..e7fa0824fd98a 100644
--- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -202,8 +202,8 @@ MachineBlockFrequencyInfo::MachineBlockFrequencyInfo(
     MachineBlockFrequencyInfo &&) = default;
 
 MachineBlockFrequencyInfo::MachineBlockFrequencyInfo(
-    MachineFunction &F, MachineBranchProbabilityInfo &MBPI,
-    MachineLoopInfo &MLI) {
+    const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
+    const MachineLoopInfo &MLI) {
   calculate(F, MBPI, MLI);
 }
 
diff --git a/llvm/lib/CodeGen/MachineDebugify.cpp b/llvm/lib/CodeGen/MachineDebugify.cpp
index 9b9cebc74054d..1a20fe586e951 100644
--- a/llvm/lib/CodeGen/MachineDebugify.cpp
+++ b/llvm/lib/CodeGen/MachineDebugify.cpp
@@ -63,24 +63,9 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
   // which cover a wide range of lines can help stress the debug info passes:
   // if we can't do that, fall back to using the local variable which precedes
   // all the others.
-  Function *DbgValF = M.getFunction("llvm.dbg.value");
-  DbgValueInst *EarliestDVI = nullptr;
   DbgVariableRecord *EarliestDVR = nullptr;
   DenseMap<unsigned, DILocalVariable *> Line2Var;
   DIExpression *Expr = nullptr;
-  if (DbgValF) {
-    for (const Use &U : DbgValF->uses()) {
-      auto *DVI = dyn_cast<DbgValueInst>(U.getUser());
-      if (!DVI || DVI->getFunction() != &F)
-        continue;
-      unsigned Line = DVI->getDebugLoc().getLine();
-      assert(Line != 0 && "debugify should not insert line 0 locations");
-      Line2Var[Line] = DVI->getVariable();
-      if (!EarliestDVI || Line < EarliestDVI->getDebugLoc().getLine())
-        EarliestDVI = DVI;
-      Expr = DVI->getExpression();
-    }
-  }
   for (BasicBlock &BB : F) {
     for (Instruction &I : BB) {
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
@@ -125,8 +110,7 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
       unsigned Line = MI.getDebugLoc().getLine();
       auto It = Line2Var.find(Line);
       if (It == Line2Var.end()) {
-        Line = EarliestDVI ? EarliestDVI->getDebugLoc().getLine()
-                           : EarliestDVR->getDebugLoc().getLine();
+        Line = EarliestDVR->getDebugLoc().getLine();
         It = Line2Var.find(Line);
         assert(It != Line2Var.end());
       }
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 40464e91f9efc..b7e41427f2c27 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7641,7 +7641,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
 
     if (SDValue(GN0, 0).hasOneUse() &&
         isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
-        TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+        TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
       SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
                        GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
 
@@ -15745,7 +15745,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   // fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
   if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
     if (SDValue(GN0, 0).hasOneUse() && ExtVT == GN0->getMemoryVT() &&
-        TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+        TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
       SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
                        GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
 
@@ -18727,6 +18727,12 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
   if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
     return FTrunc;
 
+  // fold (sint_to_fp (trunc nsw x)) -> (sint_to_fp x)
+  if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoSignedWrap() &&
+      TLI.isTypeDesirableForOp(ISD::SINT_TO_FP,
+                               N0.getOperand(0).getValueType()))
+    return DAG.getNode(ISD::SINT_TO_FP, DL, VT, N0.getOperand(0));
+
   return SDValue();
 }
 
@@ -18764,6 +18770,12 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
   if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
     return FTrunc;
 
+  // fold (uint_to_fp (trunc nuw x)) -> (uint_to_fp x)
+  if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoUnsignedWrap() &&
+      TLI.isTypeDesirableForOp(ISD::UINT_TO_FP,
+                               N0.getOperand(0).getValueType()))
+    return DAG.getNode(ISD::UINT_TO_FP, DL, VT, N0.getOperand(0));
+
   return SDValue();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 56c8bb441ddf8..5453828177c72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6751,7 +6751,9 @@ SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
     return SDValue();
   int64_t Offset = C2->getSExtValue();
   switch (Opcode) {
-  case ISD::ADD: break;
+  case ISD::ADD:
+  case ISD::PTRADD:
+    break;
   case ISD::SUB: Offset = -uint64_t(Offset); break;
   default: return SDValue();
   }
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 84a56058de834..8fb33c30e5cac 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -2288,39 +2288,36 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
   // Collect a map of {backing storage : dbg.declares} (currently "backing
   // storage" is limited to Allocas). We'll use this to find dbg.declares to
   // delete after running `trackAssignments`.
-  DenseMap<const AllocaInst *, SmallPtrSet<DbgDeclareInst *, 2>> DbgDeclares;
   DenseMap<const AllocaInst *, SmallPtrSet<DbgVariableRecord *, 2>> DVRDeclares;
   // Create another similar map of {storage : variables} that we'll pass to
   // trackAssignments.
   StorageToVarsMap Vars;
-  auto ProcessDeclare = [&](auto *Declare, auto &DeclareList) {
+  auto ProcessDeclare = [&](DbgVariableRecord &Declare) {
     // FIXME: trackAssignments doesn't let you specify any modifiers to the
     // variable (e.g. fragment) or location (e.g. offset), so we have to
     // leave dbg.declares with non-empty expressions in place.
-    if (Declare->getExpression()->getNumElements() != 0)
+    if (Declare.getExpression()->getNumElements() != 0)
       return;
-    if (!Declare->getAddress())
+    if (!Declare.getAddress())
       return;
     if (AllocaInst *Alloca =
-            dyn_cast<AllocaInst>(Declare->getAddress()->stripPointerCasts())) {
+            dyn_cast<AllocaInst>(Declare.getAddress()->stripPointerCasts())) {
       // FIXME: Skip VLAs for now (let these variables use dbg.declares).
       if (!Alloca->isStaticAlloca())
         return;
       // Similarly, skip scalable vectors (use dbg.declares instead).
       if (auto Sz = Alloca->getAllocationSize(*DL); Sz && Sz->isScalable())
         return;
-      DeclareList[Alloca].insert(Declare);
-      Vars[Alloca].insert(VarRecord(Declare));
+      DVRDeclares[Alloca].insert(&Declare);
+      Vars[Alloca].insert(VarRecord(&Declare));
     }
   };
   for (auto &BB : F) {
     for (auto &I : BB) {
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
         if (DVR.isDbgDeclare())
-          ProcessDeclare(&DVR, DVRDeclares);
+          ProcessDeclare(DVR);
       }
-      if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(&I))
-        ProcessDeclare(DDI, DbgDeclares);
     }
   }
 
@@ -2336,8 +2333,8 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
   trackAssignments(F.begin(), F.end(), Vars, *DL);
 
   // Delete dbg.declares for variables now tracked with assignment tracking.
-  auto DeleteSubsumedDeclare = [&](const auto &Markers, auto &Declares) {
-    (void)Markers;
+  for (auto &[Insts, Declares] : DVRDeclares) {
+    auto Markers = at::getDVRAssignmentMarkers(Insts);
     for (auto *Declare : Declares) {
       // Assert that the alloca that Declare uses is now linked to a dbg.assign
       // describing the same variable (i.e. check that this dbg.declare has
@@ -2356,10 +2353,6 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
       Changed = true;
     }
   };
-  for (auto &P : DbgDeclares)
-    DeleteSubsumedDeclare(at::getAssignmentMarkers(P.first), P.second);
-  for (auto &P : DVRDeclares)
-    DeleteSubsumedDeclare(at::getDVRAssignmentMarkers(P.first), P.second);
   return Changed;
 }
 
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 2270923bd3719..f16963dce56e1 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -49,20 +49,11 @@ uint32_t DIType::getAlignInBits() const {
 const DIExpression::FragmentInfo DebugVariable::DefaultFragment = {
     std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::min()};
 
-DebugVariable::DebugVariable(const DbgVariableIntrinsic *DII)
-    : Variable(DII->getVariable()),
-      Fragment(DII->getExpression()->getFragmentInfo()),
-      InlinedAt(DII->getDebugLoc().getInlinedAt()) {}
-
 DebugVariable::DebugVariable(const DbgVariableRecord *DVR)
     : Variable(DVR->getVariable()),
       Fragment(DVR->getExpression()->getFragmentInfo()),
       InlinedAt(DVR->getDebugLoc().getInlinedAt()) {}
 
-DebugVariableAggregate::DebugVariableAggregate(const DbgVariableIntrinsic *DVI)
-    : DebugVariable(DVI->getVariable(), std::nullopt,
-                    DVI->getDebugLoc()->getInlinedAt()) {}
-
 DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
                        unsigned Column, uint64_t AtomGroup, uint8_t AtomRank,
                        ArrayRef<Metadata *> MDs, bool ImplicitCode)
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 79eeb08cddeef..2579fa37935f0 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -54,11 +54,6 @@ static const char *PreservedSymbols[] = {
     "__stack_chk_guard",
 };
 
-static bool isPreservedGlobalVarName(StringRef Name) {
-  return StringRef(PreservedSymbols[0]) == Name ||
-         StringRef(PreservedSymbols[1]) == Name;
-}
-
 namespace {
 
 const char *getExpectedProducerName() {
@@ -86,16 +81,12 @@ struct Builder {
   // The StringTableBuilder does not create a copy of any strings added to it,
   // so this provides somewhere to store any strings that we create.
   Builder(SmallVector<char, 0> &Symtab, StringTableBuilder &StrtabBuilder,
-          BumpPtrAllocator &Alloc, const Triple &TT)
-      : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc), TT(TT),
-        Libcalls(TT) {}
+          BumpPtrAllocator &Alloc)
+      : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc) {}
 
   DenseMap<const Comdat *, int> ComdatMap;
   Mangler Mang;
-  const Triple &TT;
-
-  // FIXME: This shouldn't be here.
-  RTLIB::RuntimeLibcallsInfo Libcalls;
+  Triple TT;
 
   std::vector<storage::Comdat> Comdats;
   std::vector<storage::Module> Mods;
@@ -107,10 +98,6 @@ struct Builder {
 
   std::vector<storage::Str> DependentLibraries;
 
-  bool isPreservedLibFuncName(StringRef Name) {
-    return Libcalls.getSupportedLibcallImpl(Name) != RTLIB::Unsupported;
-  }
-
   void setStr(storage::Str &S, StringRef Value) {
     S.Offset = StrtabBuilder.add(Value);
     S.Size = Value.size();
@@ -226,6 +213,18 @@ Expected<int> Builder::getComdatIndex(const Comdat *C, const Module *M) {
   return P.first->second;
 }
 
+static DenseSet<StringRef> buildPreservedSymbolsSet(const Triple &TT) {
+  DenseSet<StringRef> PreservedSymbolSet(std::begin(PreservedSymbols),
+                                         std::end(PreservedSymbols));
+  // FIXME: Do we need to pass in ABI fields from TargetOptions?
+  RTLIB::RuntimeLibcallsInfo Libcalls(TT);
+  for (RTLIB::LibcallImpl Impl : Libcalls.getLibcallImpls()) {
+    if (Impl != RTLIB::Unsupported)
+      PreservedSymbolSet.insert(Libcalls.getLibcallImplName(Impl));
+  }
+  return PreservedSymbolSet;
+}
+
 Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
                          const SmallPtrSet<GlobalValue *, 4> &Used,
                          ModuleSymbolTable::Symbol Msym) {
@@ -279,11 +278,13 @@ Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
     return Error::success();
   }
 
-  StringRef GVName = GV->getName();
-  setStr(Sym.IRName, GVName);
+  setStr(Sym.IRName, GV->getName());
+
+  static const DenseSet<StringRef> PreservedSymbolsSet =
+      buildPreservedSymbolsSet(GV->getParent()->getTargetTriple());
+  bool IsPreservedSymbol = PreservedSymbolsSet.contains(GV->getName());
 
-  if (Used.count(GV) || isPreservedLibFuncName(GVName) ||
-      isPreservedGlobalVarName(GVName))
+  if (Used.count(GV) || IsPreservedSymbol)
     Sym.Flags |= 1 << storage::Symbol::FB_used;
   if (GV->isThreadLocal())
     Sym.Flags |= 1 << storage::Symbol::FB_tls;
@@ -350,6 +351,7 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
   setStr(Hdr.Producer, kExpectedProducerName);
   setStr(Hdr.TargetTriple, IRMods[0]->getTargetTriple().str());
   setStr(Hdr.SourceFileName, IRMods[0]->getSourceFileName());
+  TT = IRMods[0]->getTargetTriple();
 
   for (auto *M : IRMods)
     if (Error Err = addModule(M))
@@ -375,8 +377,7 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
 Error irsymtab::build(ArrayRef<Module *> Mods, SmallVector<char, 0> &Symtab,
                       StringTableBuilder &StrtabBuilder,
                       BumpPtrAllocator &Alloc) {
-  const Triple &TT = Mods[0]->getTargetTriple();
-  return Builder(Symtab, StrtabBuilder, Alloc, TT).build(Mods);
+  return Builder(Symtab, StrtabBuilder, Alloc).build(Mods);
 }
 
 // Upgrade a vector of bitcode modules created by an old version of LLVM by
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d04e6c45e2103..f026726c3f484 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -6439,7 +6439,9 @@ bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
     }
   }
 
-  return true;
+  EVT PreExtScalarVT = ExtVal->getOperand(0).getValueType().getScalarType();
+  return PreExtScalarVT == MVT::i8 || PreExtScalarVT == MVT::i16 ||
+         PreExtScalarVT == MVT::i32 || PreExtScalarVT == MVT::i64;
 }
 
 unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 23f106a9c1d4d..007b481f84960 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -153,6 +153,9 @@ struct AMDGPULowerBufferFatPointersPass
   const TargetMachine &TM;
 };
 
+void initializeAMDGPUPrepareAGPRAllocLegacyPass(PassRegistry &);
+extern char &AMDGPUPrepareAGPRAllocLegacyID;
+
 void initializeAMDGPUReserveWWMRegsLegacyPass(PassRegistry &);
 extern char &AMDGPUReserveWWMRegsLegacyID;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 250547acb1ee7..b6c6d927d0e89 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -114,6 +114,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUse
 MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
 MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
 MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
+MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass())
 MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
 MACHINE_FUNCTION_PASS("amdgpu-wait-sgpr-hazards", AMDGPUWaitSGPRHazardsPass())
 MACHINE_FUNCTION_PASS("gcn-create-vopd", GCNCreateVOPDPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
new file mode 100644
index 0000000000000..3b06e9b00ac69
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
@@ -0,0 +1,108 @@
+//===-- AMDGPUPrepareAGPRAlloc.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Make simple transformations to relax register constraints for cases which can
+// allocate to AGPRs or VGPRs. Replace materialize of inline immediates into
+// AGPR or VGPR with a pseudo with an AV_* class register constraint. This
+// allows later passes to inflate the register class if necessary. The register
+// allocator does not know to replace instructions to relax constraints.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUPrepareAGPRAlloc.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-prepare-agpr-alloc"
+
+namespace {
+
+class AMDGPUPrepareAGPRAllocImpl {
+private:
+  const SIInstrInfo &TII;
+  MachineRegisterInfo &MRI;
+
+public:
+  AMDGPUPrepareAGPRAllocImpl(const GCNSubtarget &ST, MachineRegisterInfo &MRI)
+      : TII(*ST.getInstrInfo()), MRI(MRI) {}
+  bool run(MachineFunction &MF);
+};
+
+class AMDGPUPrepareAGPRAllocLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  AMDGPUPrepareAGPRAllocLegacy() : MachineFunctionPass(ID) {
+    initializeAMDGPUPrepareAGPRAllocLegacyPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override { return "AMDGPU Prepare AGPR Alloc"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(AMDGPUPrepareAGPRAllocLegacy, DEBUG_TYPE,
+                      "AMDGPU Prepare AGPR Alloc", false, false)
+INITIALIZE_PASS_END(AMDGPUPrepareAGPRAllocLegacy, DEBUG_TYPE,
+                    "AMDGPU Prepare AGPR Alloc", false, false)
+
+char AMDGPUPrepareAGPRAllocLegacy::ID = 0;
+
+char &llvm::AMDGPUPrepareAGPRAllocLegacyID = AMDGPUPrepareAGPRAllocLegacy::ID;
+
+bool AMDGPUPrepareAGPRAllocLegacy::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  return AMDGPUPrepareAGPRAllocImpl(ST, MF.getRegInfo()).run(MF);
+}
+
+PreservedAnalyses
+AMDGPUPrepareAGPRAllocPass::run(MachineFunction &MF,
+                                MachineFunctionAnalysisManager &MFAM) {
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  AMDGPUPrepareAGPRAllocImpl(ST, MF.getRegInfo()).run(MF);
+  return PreservedAnalyses::all();
+}
+
+bool AMDGPUPrepareAGPRAllocImpl::run(MachineFunction &MF) {
+  if (MRI.isReserved(AMDGPU::AGPR0))
+    return false;
+
+  const MCInstrDesc &AVImmPseudo = TII.get(AMDGPU::AV_MOV_B32_IMM_PSEUDO);
+
+  bool Changed = false;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      if ((MI.getOpcode() == AMDGPU::V_MOV_B32_e32 &&
+           TII.isInlineConstant(MI, 1)) ||
+          (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
+           MI.getOperand(1).isImm())) {
+        MI.setDesc(AVImmPseudo);
+        Changed = true;
+      }
+    }
+  }
+
+  return Changed;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
new file mode 100644
index 0000000000000..dc598c98f241b
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
@@ -0,0 +1,23 @@
+//===- AMDGPUPrepareAGPRAlloc.h ---------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class AMDGPUPrepareAGPRAllocPass
+    : public PassInfoMixin<AMDGPUPrepareAGPRAllocPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
index 46027b8890234..8101c68986241 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
@@ -167,77 +167,39 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
 
   Info.UsesVCC =
       MRI.isPhysRegUsed(AMDGPU::VCC_LO) || MRI.isPhysRegUsed(AMDGPU::VCC_HI);
+  Info.NumExplicitSGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::SGPR_32RegClass,
+                                                /*IncludeCalls=*/false);
+  if (ST.hasMAIInsts())
+    Info.NumAGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass,
+                                          /*IncludeCalls=*/false);
 
   // If there are no calls, MachineRegisterInfo can tell us the used register
   // count easily.
   // A tail call isn't considered a call for MachineFrameInfo's purposes.
   if (!FrameInfo.hasCalls() && !FrameInfo.hasTailCall()) {
-    Info.NumVGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass);
-    Info.NumExplicitSGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::SGPR_32RegClass);
-    if (ST.hasMAIInsts())
-      Info.NumAGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass);
+    Info.NumVGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass,
+                                          /*IncludeCalls=*/false);
     return Info;
   }
 
   int32_t MaxVGPR = -1;
-  int32_t MaxAGPR = -1;
-  int32_t MaxSGPR = -1;
   Info.CalleeSegmentSize = 0;
 
   for (const MachineBasicBlock &MBB : MF) {
     for (const MachineInstr &MI : MBB) {
-      // TODO: Check regmasks? Do they occur anywhere except calls?
-      for (const MachineOperand &MO : MI.operands()) {
-        unsigned Width = 0;
-        bool IsSGPR = false;
-        bool IsAGPR = false;
+      for (unsigned I = 0; I < MI.getNumOperands(); ++I) {
+        const MachineOperand &MO = MI.getOperand(I);
 
         if (!MO.isReg())
           continue;
 
         Register Reg = MO.getReg();
         switch (Reg) {
-        case AMDGPU::EXEC:
-        case AMDGPU::EXEC_LO:
-        case AMDGPU::EXEC_HI:
-        case AMDGPU::SCC:
-        case AMDGPU::M0:
-        case AMDGPU::M0_LO16:
-        case AMDGPU::M0_HI16:
-        case AMDGPU::SRC_SHARED_BASE_LO:
-        case AMDGPU::SRC_SHARED_BASE:
-        case AMDGPU::SRC_SHARED_LIMIT_LO:
-        case AMDGPU::SRC_SHARED_LIMIT:
-        case AMDGPU::SRC_PRIVATE_BASE_LO:
-        case AMDGPU::SRC_PRIVATE_BASE:
-        case AMDGPU::SRC_PRIVATE_LIMIT_LO:
-        case AMDGPU::SRC_PRIVATE_LIMIT:
-        case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
-        case AMDGPU::SGPR_NULL:
-        case AMDGPU::SGPR_NULL64:
-        case AMDGPU::MODE:
-          continue;
-
         case AMDGPU::NoRegister:
           assert(MI.isDebugInstr() &&
                  "Instruction uses invalid noreg register");
           continue;
 
-        case AMDGPU::VCC:
-        case AMDGPU::VCC_LO:
-        case AMDGPU::VCC_HI:
-        case AMDGPU::VCC_LO_LO16:
-        case AMDGPU::VCC_LO_HI16:
-        case AMDGPU::VCC_HI_LO16:
-        case AMDGPU::VCC_HI_HI16:
-          Info.UsesVCC = true;
-          continue;
-
-        case AMDGPU::FLAT_SCR:
-        case AMDGPU::FLAT_SCR_LO:
-        case AMDGPU::FLAT_SCR_HI:
-          continue;
-
         case AMDGPU::XNACK_MASK:
         case AMDGPU::XNACK_MASK_LO:
         case AMDGPU::XNACK_MASK_HI:
@@ -267,170 +229,22 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
           break;
         }
 
-        if (AMDGPU::SGPR_32RegClass.contains(Reg) ||
-            AMDGPU::SGPR_LO16RegClass.contains(Reg) ||
-            AMDGPU::SGPR_HI16RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 1;
-        } else if (AMDGPU::VGPR_32RegClass.contains(Reg) ||
-                   AMDGPU::VGPR_16RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 1;
-        } else if (AMDGPU::AGPR_32RegClass.contains(Reg) ||
-                   AMDGPU::AGPR_LO16RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 1;
-        } else if (AMDGPU::SGPR_64RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 2;
-        } else if (AMDGPU::VReg_64RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 2;
-        } else if (AMDGPU::AReg_64RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 2;
-        } else if (AMDGPU::VReg_96RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 3;
-        } else if (AMDGPU::SReg_96RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 3;
-        } else if (AMDGPU::AReg_96RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 3;
-        } else if (AMDGPU::SGPR_128RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 4;
-        } else if (AMDGPU::VReg_128RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 4;
-        } else if (AMDGPU::AReg_128RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 4;
-        } else if (AMDGPU::VReg_160RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 5;
-        } else if (AMDGPU::SReg_160RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 5;
-        } else if (AMDGPU::AReg_160RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 5;
-        } else if (AMDGPU::VReg_192RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 6;
-        } else if (AMDGPU::SReg_192RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 6;
-        } else if (AMDGPU::AReg_192RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 6;
-        } else if (AMDGPU::VReg_224RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 7;
-        } else if (AMDGPU::SReg_224RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 7;
-        } else if (AMDGPU::AReg_224RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 7;
-        } else if (AMDGPU::SReg_256RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 8;
-        } else if (AMDGPU::VReg_256RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 8;
-        } else if (AMDGPU::AReg_256RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 8;
-        } else if (AMDGPU::VReg_288RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 9;
-        } else if (AMDGPU::SReg_288RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 9;
-        } else if (AMDGPU::AReg_288RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 9;
-        } else if (AMDGPU::VReg_320RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 10;
-        } else if (AMDGPU::SReg_320RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 10;
-        } else if (AMDGPU::AReg_320RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 10;
-        } else if (AMDGPU::VReg_352RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 11;
-        } else if (AMDGPU::SReg_352RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 11;
-        } else if (AMDGPU::AReg_352RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 11;
-        } else if (AMDGPU::VReg_384RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 12;
-        } else if (AMDGPU::SReg_384RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 12;
-        } else if (AMDGPU::AReg_384RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 12;
-        } else if (AMDGPU::SReg_512RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 16;
-        } else if (AMDGPU::VReg_512RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 16;
-        } else if (AMDGPU::AReg_512RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 16;
-        } else if (AMDGPU::SReg_1024RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 32;
-        } else if (AMDGPU::VReg_1024RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 32;
-        } else if (AMDGPU::AReg_1024RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 32;
-        } else {
-          // We only expect TTMP registers or registers that do not belong to
-          // any RC.
-          assert((AMDGPU::TTMP_32RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_64RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_128RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_256RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_512RegClass.contains(Reg) ||
-                  !TRI.getPhysRegBaseClass(Reg)) &&
-                 "Unknown register class");
-        }
+        const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg);
+        assert((!RC || TRI.isVGPRClass(RC) || TRI.isSGPRClass(RC) ||
+                TRI.isAGPRClass(RC) || AMDGPU::TTMP_32RegClass.contains(Reg) ||
+                AMDGPU::TTMP_64RegClass.contains(Reg) ||
+                AMDGPU::TTMP_128RegClass.contains(Reg) ||
+                AMDGPU::TTMP_256RegClass.contains(Reg) ||
+                AMDGPU::TTMP_512RegClass.contains(Reg)) &&
+               "Unknown register class");
+
+        if (!RC || !TRI.isVGPRClass(RC))
+          continue;
+
+        unsigned Width = divideCeil(TRI.getRegSizeInBits(*RC), 32);
         unsigned HWReg = TRI.getHWRegIndex(Reg);
         int MaxUsed = HWReg + Width - 1;
-        if (IsSGPR) {
-          MaxSGPR = MaxUsed > MaxSGPR ? MaxUsed : MaxSGPR;
-        } else if (IsAGPR) {
-          MaxAGPR = MaxUsed > MaxAGPR ? MaxUsed : MaxAGPR;
-        } else {
-          MaxVGPR = MaxUsed > MaxVGPR ? MaxUsed : MaxVGPR;
-        }
+        MaxVGPR = std::max(MaxUsed, MaxVGPR);
       }
 
       if (MI.isCall()) {
@@ -492,9 +306,7 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
     }
   }
 
-  Info.NumExplicitSGPR = MaxSGPR + 1;
   Info.NumVGPR = MaxVGPR + 1;
-  Info.NumAGPR = MaxAGPR + 1;
 
   return Info;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 31a80e00edd3b..c865082a1dcea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -25,6 +25,7 @@
 #include "AMDGPUMacroFusion.h"
 #include "AMDGPUPerfHintAnalysis.h"
 #include "AMDGPUPreloadKernArgProlog.h"
+#include "AMDGPUPrepareAGPRAlloc.h"
 #include "AMDGPURemoveIncompatibleFunctions.h"
 #include "AMDGPUReserveWWMRegs.h"
 #include "AMDGPUResourceUsageAnalysis.h"
@@ -499,6 +500,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeGlobalISel(*PR);
   initializeAMDGPUAsmPrinterPass(*PR);
   initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
+  initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
   initializeGCNDPPCombineLegacyPass(*PR);
   initializeSILowerI1CopiesLegacyPass(*PR);
   initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);
@@ -1196,6 +1198,7 @@ class GCNPassConfig final : public AMDGPUPassConfig {
   bool addRegBankSelect() override;
   void addPreGlobalInstructionSelect() override;
   bool addGlobalInstructionSelect() override;
+  void addPreRegAlloc() override;
   void addFastRegAlloc() override;
   void addOptimizedRegAlloc() override;
 
@@ -1539,6 +1542,11 @@ void GCNPassConfig::addFastRegAlloc() {
   TargetPassConfig::addFastRegAlloc();
 }
 
+void GCNPassConfig::addPreRegAlloc() {
+  if (getOptLevel() != CodeGenOptLevel::None)
+    addPass(&AMDGPUPrepareAGPRAllocLegacyID);
+}
+
 void GCNPassConfig::addOptimizedRegAlloc() {
   if (EnableDCEInRA)
     insertPass(&DetectDeadLanesID, &DeadMachineInstructionElimID);
@@ -2235,6 +2243,11 @@ void AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
   Base::addOptimizedRegAlloc(addPass);
 }
 
+void AMDGPUCodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const {
+  if (getOptLevel() != CodeGenOptLevel::None)
+    addPass(AMDGPUPrepareAGPRAllocPass());
+}
+
 Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
     AddMachinePass &addPass) const {
   // TODO: Check --regalloc-npm option
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 3b2f39c14a9bc..e0f1296ddded8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -181,7 +181,9 @@ class AMDGPUCodeGenPassBuilder
   void addMachineSSAOptimization(AddMachinePass &) const;
   void addPostRegAlloc(AddMachinePass &) const;
   void addPreEmitPass(AddMachinePass &) const;
+  void addPreEmitRegAlloc(AddMachinePass &) const;
   Error addRegAssignmentOptimized(AddMachinePass &) const;
+  void addPreRegAlloc(AddMachinePass &) const;
   void addOptimizedRegAlloc(AddMachinePass &) const;
   void addPreSched2(AddMachinePass &) const;
 
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index e3519f192137c..42edec0d01493 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -74,6 +74,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPULowerKernelArguments.cpp
   AMDGPULowerKernelAttributes.cpp
   AMDGPULowerModuleLDSPass.cpp
+  AMDGPUPrepareAGPRAlloc.cpp
   AMDGPUSwLowerLDS.cpp
   AMDGPUMachineFunction.cpp
   AMDGPUMachineModuleInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 400795c29b0e4..0c76ff2ec5ea7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -33,6 +33,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -46,6 +47,7 @@
 #include <optional>
 
 using namespace llvm;
+using namespace llvm::SDPatternMatch;
 
 #define DEBUG_TYPE "si-lower"
 
@@ -14561,7 +14563,7 @@ static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
 // instead of a tree.
 SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
                                             DAGCombinerInfo &DCI) const {
-  assert(N->getOpcode() == ISD::ADD);
+  assert(N->isAnyAdd());
 
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
@@ -14594,7 +14596,7 @@ SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
     for (SDNode *User : LHS->users()) {
       // There is a use that does not feed into addition, so the multiply can't
       // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
-      if (User->getOpcode() != ISD::ADD)
+      if (!User->isAnyAdd())
         return SDValue();
 
       // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
@@ -14706,8 +14708,11 @@ SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
 
     SDValue Hi = getHiHalf64(LHS, DAG);
     SDValue ConstHi32 = DAG.getConstant(Hi_32(Val), SL, MVT::i32);
+    unsigned Opcode = N->getOpcode();
+    if (Opcode == ISD::PTRADD)
+      Opcode = ISD::ADD;
     SDValue AddHi =
-        DAG.getNode(N->getOpcode(), SL, MVT::i32, Hi, ConstHi32, N->getFlags());
+        DAG.getNode(Opcode, SL, MVT::i32, Hi, ConstHi32, N->getFlags());
 
     SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
     return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, Lo, AddHi);
@@ -15181,42 +15186,123 @@ SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
+  EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
-  if (N1.getOpcode() == ISD::ADD) {
-    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
-    //    y is not, and (add y, z) is used only once.
-    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
-    //    z is not, and (add y, z) is used only once.
-    // The goal is to move constant offsets to the outermost ptradd, to create
-    // more opportunities to fold offsets into memory instructions.
-    // Together with the generic combines in DAGCombiner.cpp, this also
-    // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
-    //
-    // This transform is here instead of in the general DAGCombiner as it can
-    // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
-    // AArch64's CPA.
-    SDValue X = N0;
-    SDValue Y = N1.getOperand(0);
-    SDValue Z = N1.getOperand(1);
-    if (N1.hasOneUse()) {
-      bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
-      bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
-      if (ZIsConstant != YIsConstant) {
-        // If both additions in the original were NUW, the new ones are as well.
-        SDNodeFlags Flags =
-            (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
-        if (YIsConstant)
-          std::swap(Y, Z);
+  // The following folds transform PTRADDs into regular arithmetic in cases
+  // where the PTRADD wouldn't be folded as an immediate offset into memory
+  // instructions anyway. They are target-specific in that other targets might
+  // prefer to not lose information about the pointer arithmetic.
+
+  // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
+  // Adapted from DAGCombiner::visitADDLikeCommutative.
+  SDValue V, K;
+  if (sd_match(N1, m_Shl(m_Neg(m_Value(V)), m_Value(K)))) {
+    SDNodeFlags ShlFlags = N1->getFlags();
+    // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
+    // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
+    // preserved.
+    SDNodeFlags NewShlFlags =
+        ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
+            ? SDNodeFlags::NoSignedWrap
+            : SDNodeFlags();
+    SDValue Inner = DAG.getNode(ISD::SHL, DL, VT, V, K, NewShlFlags);
+    DCI.AddToWorklist(Inner.getNode());
+    return DAG.getNode(ISD::SUB, DL, VT, N0, Inner);
+  }
+
+  // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
+  // performAddCombine.
+  if (N1.getOpcode() == ISD::MUL) {
+    if (Subtarget->hasMad64_32()) {
+      if (SDValue Folded = tryFoldToMad64_32(N, DCI))
+        return Folded;
+    }
+  }
 
-        SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, Flags);
+  // If the 32 low bits of the constant are all zero, there is nothing to fold
+  // into an immediate offset, so it's better to eliminate the unnecessary
+  // addition for the lower 32 bits than to preserve the PTRADD.
+  // Analogous to a fold in performAddCombine.
+  if (VT == MVT::i64) {
+    if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
+      return Folded;
+  }
+
+  if (N0.getOpcode() == ISD::PTRADD && N1.getOpcode() == ISD::Constant) {
+    // Fold (ptradd (ptradd GA, v), c) -> (ptradd (ptradd GA, c) v) with
+    // global address GA and constant c, such that c can be folded into GA.
+    SDValue GAValue = N0.getOperand(0);
+    if (const GlobalAddressSDNode *GA =
+            dyn_cast<GlobalAddressSDNode>(GAValue)) {
+      if (DCI.isBeforeLegalizeOps() && isOffsetFoldingLegal(GA)) {
+        // If both additions in the original were NUW, reassociation preserves
+        // that.
+        SDNodeFlags Flags =
+            (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+        SDValue Inner = DAG.getMemBasePlusOffset(GAValue, N1, DL, Flags);
         DCI.AddToWorklist(Inner.getNode());
-        return DAG.getMemBasePlusOffset(Inner, Z, DL, Flags);
+        return DAG.getMemBasePlusOffset(Inner, N0.getOperand(1), DL, Flags);
       }
     }
   }
 
+  if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
+    return SDValue();
+
+  // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
+  //    y is not, and (add y, z) is used only once.
+  // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
+  //    z is not, and (add y, z) is used only once.
+  // The goal is to move constant offsets to the outermost ptradd, to create
+  // more opportunities to fold offsets into memory instructions.
+  // Together with the generic combines in DAGCombiner.cpp, this also
+  // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
+  //
+  // This transform is here instead of in the general DAGCombiner as it can
+  // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
+  // AArch64's CPA.
+  SDValue X = N0;
+  SDValue Y = N1.getOperand(0);
+  SDValue Z = N1.getOperand(1);
+  bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
+  bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
+
+  // If both additions in the original were NUW, reassociation preserves that.
+  SDNodeFlags ReassocFlags =
+      (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+
+  if (ZIsConstant != YIsConstant) {
+    if (YIsConstant)
+      std::swap(Y, Z);
+    SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
+    DCI.AddToWorklist(Inner.getNode());
+    return DAG.getMemBasePlusOffset(Inner, Z, DL, ReassocFlags);
+  }
+
+  // If one of Y and Z is constant, they have been handled above. If both were
+  // constant, the addition would have been folded in SelectionDAG::getNode
+  // already. This ensures that the generic DAG combines won't undo the
+  // following reassociation.
+  assert(!YIsConstant && !ZIsConstant);
+
+  if (!X->isDivergent() && Y->isDivergent() != Z->isDivergent()) {
+    // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
+    // y are uniform and z isn't.
+    // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
+    // z are uniform and y isn't.
+    // The goal is to push uniform operands up in the computation, so that they
+    // can be handled with scalar operations. We can't use reassociateScalarOps
+    // for this since it requires two identical commutative operations to
+    // reassociate.
+    if (Y->isDivergent())
+      std::swap(Y, Z);
+    SDValue UniformInner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
+    DCI.AddToWorklist(UniformInner.getNode());
+    return DAG.getMemBasePlusOffset(UniformInner, Z, DL, ReassocFlags);
+  }
+
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 2764ed3d3f0b1..5e92921f3ea21 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1113,7 +1113,6 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   // that will not require an additional 4-bytes; this function assumes that it
   // will.
   bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
-    assert(!MO.isReg() && "isInlineConstant called on register operand!");
     if (!MO.isImm())
       return false;
     return isInlineConstant(MO.getImm(), OperandType);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 9173041a7bccd..fa2b8db6ba55a 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4052,11 +4052,11 @@ SIRegisterInfo::getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
   return 0;
 }
 
-unsigned
-SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
-                                   const TargetRegisterClass &RC) const {
+unsigned SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
+                                            const TargetRegisterClass &RC,
+                                            bool IncludeCalls) const {
   for (MCPhysReg Reg : reverse(RC.getRegisters()))
-    if (MRI.isPhysRegUsed(Reg))
+    if (MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/!IncludeCalls))
       return getHWRegIndex(Reg) + 1;
   return 0;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 06a7a17b0246b..0008e5f8cf3b4 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -486,9 +486,11 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
                                      unsigned SubReg) const;
 
   // \returns a number of registers of a given \p RC used in a function.
-  // Does not go inside function calls.
+  // Does not go inside function calls. If \p IncludeCalls is true, it will
+  // include registers that may be clobbered by calls.
   unsigned getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
-                              const TargetRegisterClass &RC) const;
+                              const TargetRegisterClass &RC,
+                              bool IncludeCalls = true) const;
 
   std::optional<uint8_t> getVRegFlagValue(StringRef Name) const override {
     return Name == "WWM_REG" ? AMDGPU::VirtRegFlag::WWM_REG
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 1bbbb610305e9..3ee90857b34b8 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -1148,6 +1148,7 @@ defm V_MOV_B64 : VOP1_Real_FULL <GFX1250Gen, 0x1d>;
 defm V_TANH_F32              : VOP1_Real_FULL<GFX1250Gen, 0x01e>;
 defm V_TANH_F16              : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x01f>;
 defm V_TANH_BF16             : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x04a>;
+defm V_PRNG_B32              : VOP1_Real_FULL<GFX1250Gen, 0x04b>;
 defm V_CVT_F32_BF16          : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x072, "v_cvt_f32_bf16", "V_CVT_F32_BF16_gfx1250">;
 defm V_CVT_PK_F16_FP8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x075>;
 defm V_CVT_PK_F16_BF8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x076>;
diff --git a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
index ac5e7f3891c72..1493bf4cba695 100644
--- a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
@@ -158,7 +158,12 @@ void LoongArchFrameLowering::processFunctionBeforeFrameFinalized(
   // estimateStackSize has been observed to under-estimate the final stack
   // size, so give ourselves wiggle-room by checking for stack size
   // representable an 11-bit signed field rather than 12-bits.
-  if (!isInt<11>(MFI.estimateStackSize(MF)))
+  // For [x]vstelm.{b/h/w/d} memory instructions with 8 imm offset, 7-bit
+  // signed field is fine.
+  unsigned EstimateStackSize = MFI.estimateStackSize(MF);
+  if (!isInt<11>(EstimateStackSize) ||
+      (MF.getSubtarget<LoongArchSubtarget>().hasExtLSX() &&
+       !isInt<7>(EstimateStackSize)))
     ScavSlotsNum = std::max(ScavSlotsNum, 1u);
 
   // For CFR spill.
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 6c8e3da80b932..23b4554349003 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -95,6 +95,11 @@ static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
     /* -21, -22, -23, -24 are reserved */
 };
 
+/// Returns true if DWARF CFI instructions ("frame moves") should be emitted.
+static bool needsDwarfCFI(const MachineFunction &MF) {
+  return MF.needsFrameMoves();
+}
+
 // For now we use x3, a.k.a gp, as pointer to shadow call stack.
 // User should not use x3 in their asm.
 static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -141,6 +146,9 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
       .addImm(-SlotSize)
       .setMIFlag(MachineInstr::FrameSetup);
 
+  if (!needsDwarfCFI(MF))
+    return;
+
   // Emit a CFI instruction that causes SlotSize to be subtracted from the value
   // of the shadow stack pointer when unwinding past this frame.
   char DwarfSCSReg = TRI->getDwarfRegNum(SCSPReg, /*IsEH*/ true);
@@ -199,8 +207,10 @@ static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
       .addReg(SCSPReg)
       .addImm(-SlotSize)
       .setMIFlag(MachineInstr::FrameDestroy);
-  // Restore the SCS pointer
-  CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
+  if (needsDwarfCFI(MF)) {
+    // Restore the SCS pointer
+    CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
+  }
 }
 
 // Insert instruction to swap mscratchsw with sp
@@ -935,6 +945,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   MBBI = std::prev(MBBI, getRVVCalleeSavedInfo(MF, CSI).size() +
                              getUnmanagedCSI(MF, CSI).size());
   CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
+  bool NeedsDwarfCFI = needsDwarfCFI(MF);
 
   // If libcalls are used to spill and restore callee-saved registers, the frame
   // has two sections; the opaque section managed by the libcalls, and the
@@ -962,10 +973,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
         alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
     RVFI->setLibCallStackSize(LibCallFrameSize);
 
-    CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
-    for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-      CFIBuilder.buildOffset(CS.getReg(),
-                             MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
+      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+        CFIBuilder.buildOffset(CS.getReg(),
+                               MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   // FIXME (note copied from Lanai): This appears to be overallocating.  Needs
@@ -996,14 +1009,17 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     // could only be the next instruction.
     ++PossiblePush;
 
-    // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
-    // could be. The PUSH will also get its own CFI metadata for its own
-    // modifications, which should come after the PUSH.
-    CFIInstBuilder PushCFIBuilder(MBB, PossiblePush, MachineInstr::FrameSetup);
-    PushCFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
-    for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
-      PushCFIBuilder.buildOffset(CS.getReg(),
-                                 MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
+      // could be. The PUSH will also get its own CFI metadata for its own
+      // modifications, which should come after the PUSH.
+      CFIInstBuilder PushCFIBuilder(MBB, PossiblePush,
+                                    MachineInstr::FrameSetup);
+      PushCFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
+      for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
+        PushCFIBuilder.buildOffset(CS.getReg(),
+                                   MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   if (RVFI->isPushable(MF) && PossiblePush != MBB.end() &&
@@ -1017,10 +1033,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     PossiblePush->getOperand(1).setImm(StackAdj);
     StackSize -= StackAdj;
 
-    CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
-    for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-      CFIBuilder.buildOffset(CS.getReg(),
-                             MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
+      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+        CFIBuilder.buildOffset(CS.getReg(),
+                               MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   // Allocate space on the stack if necessary.
@@ -1031,7 +1049,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   bool DynAllocation =
       MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
   if (StackSize != 0)
-    allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, /*EmitCFI=*/true,
+    allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, NeedsDwarfCFI,
                   NeedProbe, ProbeSize, DynAllocation,
                   MachineInstr::FrameSetup);
 
@@ -1049,8 +1067,10 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
 
   // Iterate over list of callee-saved registers and emit .cfi_offset
   // directives.
-  for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
-    CFIBuilder.buildOffset(CS.getReg(), MFI.getObjectOffset(CS.getFrameIdx()));
+  if (NeedsDwarfCFI)
+    for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+      CFIBuilder.buildOffset(CS.getReg(),
+                             MFI.getObjectOffset(CS.getFrameIdx()));
 
   // Generate new FP.
   if (hasFP(MF)) {
@@ -1069,7 +1089,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
           MachineInstr::FrameSetup, getStackAlign());
     }
 
-    CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
+    if (NeedsDwarfCFI)
+      CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
   }
 
   uint64_t SecondSPAdjustAmount = 0;
@@ -1080,15 +1101,16 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
            "SecondSPAdjustAmount should be greater than zero");
 
     allocateStack(MBB, MBBI, MF, SecondSPAdjustAmount,
-                  getStackSizeWithRVVPadding(MF), !hasFP(MF), NeedProbe,
-                  ProbeSize, DynAllocation, MachineInstr::FrameSetup);
+                  getStackSizeWithRVVPadding(MF), NeedsDwarfCFI && !hasFP(MF),
+                  NeedProbe, ProbeSize, DynAllocation,
+                  MachineInstr::FrameSetup);
   }
 
   if (RVVStackSize) {
     if (NeedProbe) {
       allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, RVVStackSize,
-                                  MachineInstr::FrameSetup, !hasFP(MF),
-                                  DynAllocation);
+                                  MachineInstr::FrameSetup,
+                                  NeedsDwarfCFI && !hasFP(MF), DynAllocation);
     } else {
       // We must keep the stack pointer aligned through any intermediate
       // updates.
@@ -1097,14 +1119,15 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
                     MachineInstr::FrameSetup, getStackAlign());
     }
 
-    if (!hasFP(MF)) {
+    if (NeedsDwarfCFI && !hasFP(MF)) {
       // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
       CFIBuilder.insertCFIInst(createDefCFAExpression(
           *RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8));
     }
 
     std::advance(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
-    emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
+    if (NeedsDwarfCFI)
+      emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
   }
 
   if (hasFP(MF)) {
@@ -1171,8 +1194,9 @@ void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
                 MachineInstr::FrameDestroy, getStackAlign());
   StackSize = 0;
 
-  CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
-      .buildDefCFAOffset(CFAOffset);
+  if (needsDwarfCFI(MF))
+    CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
+        .buildDefCFAOffset(CFAOffset);
 }
 
 void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
@@ -1212,6 +1236,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
       std::next(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
   CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
                             MachineInstr::FrameDestroy);
+  bool NeedsDwarfCFI = needsDwarfCFI(MF);
 
   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
   uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
@@ -1232,10 +1257,11 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                     StackOffset::getScalable(RVVStackSize),
                     MachineInstr::FrameDestroy, getStackAlign());
 
-    if (!hasFP(MF))
-      CFIBuilder.buildDefCFA(SPReg, RealStackSize);
-
-    emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
+    if (NeedsDwarfCFI) {
+      if (!hasFP(MF))
+        CFIBuilder.buildDefCFA(SPReg, RealStackSize);
+      emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
+    }
   }
 
   if (FirstSPAdjustAmount) {
@@ -1251,7 +1277,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                     StackOffset::getFixed(SecondSPAdjustAmount),
                     MachineInstr::FrameDestroy, getStackAlign());
 
-    if (!hasFP(MF))
+    if (NeedsDwarfCFI && !hasFP(MF))
       CFIBuilder.buildDefCFAOffset(FirstSPAdjustAmount);
   }
 
@@ -1272,7 +1298,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                   getStackAlign());
   }
 
-  if (hasFP(MF))
+  if (NeedsDwarfCFI && hasFP(MF))
     CFIBuilder.buildDefCFA(SPReg, RealStackSize);
 
   // Skip to after the restores of scalar callee-saved registers
@@ -1295,8 +1321,9 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 
   // Recover callee-saved registers.
-  for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
-    CFIBuilder.buildRestore(CS.getReg());
+  if (NeedsDwarfCFI)
+    for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+      CFIBuilder.buildRestore(CS.getReg());
 
   if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(MBBI->getOpcode())) {
     // Use available stack adjustment in pop instruction to deallocate stack
@@ -1315,15 +1342,17 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
     auto NextI = next_nodbg(MBBI, MBB.end());
     if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
       ++MBBI;
-      CFIBuilder.setInsertPoint(MBBI);
+      if (NeedsDwarfCFI) {
+        CFIBuilder.setInsertPoint(MBBI);
 
-      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-        CFIBuilder.buildRestore(CS.getReg());
+        for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+          CFIBuilder.buildRestore(CS.getReg());
 
-      // Update CFA Offset. If this is a QCI interrupt function, there will be a
-      // leftover offset which is deallocated by `QC.C.MILEAVERET`, otherwise
-      // getQCIInterruptStackSize() will be 0.
-      CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
+        // Update CFA Offset. If this is a QCI interrupt function, there will
+        // be a leftover offset which is deallocated by `QC.C.MILEAVERET`,
+        // otherwise getQCIInterruptStackSize() will be 0.
+        CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
+      }
     }
   }
 
@@ -1812,7 +1841,8 @@ MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
         // allocateStack.
         bool DynAllocation =
             MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
-        allocateStack(MBB, MI, MF, -Amount, -Amount, !hasFP(MF),
+        allocateStack(MBB, MI, MF, -Amount, -Amount,
+                      needsDwarfCFI(MF) && !hasFP(MF),
                       /*NeedProbe=*/true, ProbeSize, DynAllocation,
                       MachineInstr::NoFlags);
       } else {
diff --git a/llvm/lib/Target/X86/X86CallingConv.cpp b/llvm/lib/Target/X86/X86CallingConv.cpp
index 82e8ce4e0bd7c..5d5a705893242 100644
--- a/llvm/lib/Target/X86/X86CallingConv.cpp
+++ b/llvm/lib/Target/X86/X86CallingConv.cpp
@@ -389,7 +389,7 @@ static bool CC_X86_32_I128_FP128(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
   if (!ArgFlags.isInConsecutiveRegsLast())
     return true;
 
-  assert(PendingMembers.size() == 4 && "Should have two parts");
+  assert(PendingMembers.size() == 4 && "Should have four parts");
 
   int64_t Offset = State.AllocateStack(16, Align(16));
   PendingMembers[0].convertToMem(Offset);
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 57fbc71fa22ee..9cd35e35d4bc9 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -176,10 +176,10 @@ constexpr FeatureBitset FeaturesArrowlakeS =
     FeaturesArrowlake | FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 |
     FeatureSM4;
 constexpr FeatureBitset FeaturesPantherlake =
-    FeaturesArrowlakeS ^ FeatureWIDEKL | FeaturePREFETCHI;
+    (FeaturesArrowlakeS ^ FeatureWIDEKL) | FeaturePREFETCHI;
 constexpr FeatureBitset FeaturesClearwaterforest =
-    FeaturesSierraforest ^ FeatureWIDEKL | FeatureAVXVNNIINT16 | FeatureSHA512 |
-    FeatureSM3 | FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
+    (FeaturesSierraforest ^ FeatureWIDEKL) | FeatureAVXVNNIINT16 |
+    FeatureSHA512 | FeatureSM3 | FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
 
 // Geode Processor.
 constexpr FeatureBitset FeaturesGeode =
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index fbeb7218ba9a3..a65d0fb54c212 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1103,14 +1103,13 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
               FrameTy->getElementType(FrameData.getFieldIndex(E.first)), GEP,
               SpillAlignment, E.first->getName() + Twine(".reload"));
 
-        TinyPtrVector<DbgDeclareInst *> DIs = findDbgDeclares(Def);
         TinyPtrVector<DbgVariableRecord *> DVRs = findDVRDeclares(Def);
         // Try best to find dbg.declare. If the spill is a temp, there may not
         // be a direct dbg.declare. Walk up the load chain to find one from an
         // alias.
         if (F->getSubprogram()) {
           auto *CurDef = Def;
-          while (DIs.empty() && DVRs.empty() && isa<LoadInst>(CurDef)) {
+          while (DVRs.empty() && isa<LoadInst>(CurDef)) {
             auto *LdInst = cast<LoadInst>(CurDef);
             // Only consider ptr to ptr same type load.
             if (LdInst->getPointerOperandType() != LdInst->getType())
@@ -1118,12 +1117,11 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
             CurDef = LdInst->getPointerOperand();
             if (!isa<AllocaInst, LoadInst>(CurDef))
               break;
-            DIs = findDbgDeclares(CurDef);
             DVRs = findDVRDeclares(CurDef);
           }
         }
 
-        auto SalvageOne = [&](auto *DDI) {
+        auto SalvageOne = [&](DbgVariableRecord *DDI) {
           // This dbg.declare is preserved for all coro-split function
           // fragments. It will be unreachable in the main function, and
           // processed by coro::salvageDebugInfo() by the Cloner.
@@ -1137,7 +1135,6 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
           // will be deleted in all coro-split functions.
           coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
         };
-        for_each(DIs, SalvageOne);
         for_each(DVRs, SalvageOne);
       }
 
@@ -1225,8 +1222,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
     SmallVector<DbgVariableIntrinsic *, 4> DIs;
     SmallVector<DbgVariableRecord *> DbgVariableRecords;
     findDbgUsers(DIs, Alloca, &DbgVariableRecords);
-    for (auto *DVI : DIs)
-      DVI->replaceUsesOfWith(Alloca, G);
+    assert(DIs.empty() && "Should never see debug-intrinsics");
     for (auto *DVR : DbgVariableRecords)
       DVR->replaceVariableLocationOp(Alloca, G);
 
@@ -1920,48 +1916,6 @@ salvageDebugInfoImpl(SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
   return {{*Storage, *Expr}};
 }
 
-void coro::salvageDebugInfo(
-    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
-    DbgVariableIntrinsic &DVI, bool UseEntryValue) {
-
-  Function *F = DVI.getFunction();
-  // Follow the pointer arithmetic all the way to the incoming
-  // function argument and convert into a DIExpression.
-  bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
-  Value *OriginalStorage = DVI.getVariableLocationOp(0);
-
-  auto SalvagedInfo =
-      ::salvageDebugInfoImpl(ArgToAllocaMap, UseEntryValue, F, OriginalStorage,
-                             DVI.getExpression(), SkipOutermostLoad);
-  if (!SalvagedInfo)
-    return;
-
-  Value *Storage = &SalvagedInfo->first;
-  DIExpression *Expr = &SalvagedInfo->second;
-
-  DVI.replaceVariableLocationOp(OriginalStorage, Storage);
-  DVI.setExpression(Expr);
-  // We only hoist dbg.declare today since it doesn't make sense to hoist
-  // dbg.value since it does not have the same function wide guarantees that
-  // dbg.declare does.
-  if (isa<DbgDeclareInst>(DVI)) {
-    std::optional<BasicBlock::iterator> InsertPt;
-    if (auto *I = dyn_cast<Instruction>(Storage)) {
-      InsertPt = I->getInsertionPointAfterDef();
-      // Update DILocation only if variable was not inlined.
-      DebugLoc ILoc = I->getDebugLoc();
-      DebugLoc DVILoc = DVI.getDebugLoc();
-      if (ILoc && DVILoc &&
-          DVILoc->getScope()->getSubprogram() ==
-              ILoc->getScope()->getSubprogram())
-        DVI.setDebugLoc(I->getDebugLoc());
-    } else if (isa<Argument>(Storage))
-      InsertPt = F->getEntryBlock().begin();
-    if (InsertPt)
-      DVI.moveBefore(*(*InsertPt)->getParent(), *InsertPt);
-  }
-}
-
 void coro::salvageDebugInfo(
     SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
     DbgVariableRecord &DVR, bool UseEntryValue) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index b53c5a48eb10b..52f4ffe292dae 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -34,14 +34,11 @@ void suppressCoroAllocs(CoroIdInst *CoroId);
 void suppressCoroAllocs(LLVMContext &Context,
                         ArrayRef<CoroAllocInst *> CoroAllocs);
 
-/// Attempts to rewrite the location operand of debug intrinsics in terms of
+/// Attempts to rewrite the location operand of debug records in terms of
 /// the coroutine frame pointer, folding pointer offsets into the DIExpression
 /// of the intrinsic.
 /// If the frame pointer is an Argument, store it into an alloca to enhance the
 /// debugability.
-void salvageDebugInfo(
-    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
-    DbgVariableIntrinsic &DVI, bool IsEntryPoint);
 void salvageDebugInfo(
     SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
     DbgVariableRecord &DVR, bool UseEntryValue);
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 5a8a41f0dc432..64b33e46404f0 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -618,19 +618,15 @@ static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape,
   }
 }
 
-/// Returns all DbgVariableIntrinsic in F.
-static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,
-                 SmallVector<DbgVariableRecord *>>
-collectDbgVariableIntrinsics(Function &F) {
-  SmallVector<DbgVariableIntrinsic *, 8> Intrinsics;
+/// Returns all debug records in F.
+static SmallVector<DbgVariableRecord *>
+collectDbgVariableRecords(Function &F) {
   SmallVector<DbgVariableRecord *> DbgVariableRecords;
   for (auto &I : instructions(F)) {
     for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
       DbgVariableRecords.push_back(&DVR);
-    if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
-      Intrinsics.push_back(DVI);
   }
-  return {Intrinsics, DbgVariableRecords};
+  return DbgVariableRecords;
 }
 
 void coro::BaseCloner::replaceSwiftErrorOps() {
@@ -638,13 +634,11 @@ void coro::BaseCloner::replaceSwiftErrorOps() {
 }
 
 void coro::BaseCloner::salvageDebugInfo() {
-  auto [Worklist, DbgVariableRecords] = collectDbgVariableIntrinsics(*NewF);
+  auto DbgVariableRecords = collectDbgVariableRecords(*NewF);
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
 
   // Only 64-bit ABIs have a register we can refer to with the entry value.
   bool UseEntryValue = OrigF.getParent()->getTargetTriple().isArch64Bit();
-  for (DbgVariableIntrinsic *DVI : Worklist)
-    coro::salvageDebugInfo(ArgToAllocaMap, *DVI, UseEntryValue);
   for (DbgVariableRecord *DVR : DbgVariableRecords)
     coro::salvageDebugInfo(ArgToAllocaMap, *DVR, UseEntryValue);
 
@@ -655,7 +649,7 @@ void coro::BaseCloner::salvageDebugInfo() {
     return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
                                    &DomTree);
   };
-  auto RemoveOne = [&](auto *DVI) {
+  auto RemoveOne = [&](DbgVariableRecord *DVI) {
     if (IsUnreachableBlock(DVI->getParent()))
       DVI->eraseFromParent();
     else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
@@ -669,7 +663,6 @@ void coro::BaseCloner::salvageDebugInfo() {
         DVI->eraseFromParent();
     }
   };
-  for_each(Worklist, RemoveOne);
   for_each(DbgVariableRecords, RemoveOne);
 }
 
@@ -2022,9 +2015,7 @@ static void doSplitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
   // original function. The Cloner has already salvaged debug info in the new
   // coroutine funclets.
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
-  auto [DbgInsts, DbgVariableRecords] = collectDbgVariableIntrinsics(F);
-  for (auto *DDI : DbgInsts)
-    coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
+  auto DbgVariableRecords = collectDbgVariableRecords(F);
   for (DbgVariableRecord *DVR : DbgVariableRecords)
     coro::salvageDebugInfo(ArgToAllocaMap, *DVR, false /*UseEntryValue*/);
 
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index 8017db1cfe146..5fd5f7d9dad09 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -514,7 +514,7 @@ void collectSpillsAndAllocasFromInsts(
 void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F,
                               const SuspendCrossingInfo &Checker) {
   // We don't want the layout of coroutine frame to be affected
-  // by debug information. So we only choose to salvage DbgValueInst for
+  // by debug information. So we only choose to salvage dbg.values for
   // whose value is already in the frame.
   // We would handle the dbg.values for allocas specially
   for (auto &Iter : Spills) {
@@ -522,9 +522,7 @@ void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F,
     SmallVector<DbgValueInst *, 16> DVIs;
     SmallVector<DbgVariableRecord *, 16> DVRs;
     findDbgValues(DVIs, V, &DVRs);
-    for (DbgValueInst *DVI : DVIs)
-      if (Checker.isDefinitionAcrossSuspend(*V, DVI))
-        Spills[V].push_back(DVI);
+    assert(DVIs.empty());
     // Add the instructions which carry debug info that is in the frame.
     for (DbgVariableRecord *DVR : DVRs)
       if (Checker.isDefinitionAcrossSuspend(*V, DVR->Marker->MarkedInstr))
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index d4555e9435f1d..f5525deb0172f 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -572,7 +572,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
 
   // Work out whether a dbg.value intrinsic or an equivalent DbgVariableRecord
   // is a parameter to be preserved.
-  auto ExamineDbgValue = [](auto *DbgVal, auto &Container) {
+  auto ExamineDbgValue = [&PDVRRelated](DbgVariableRecord *DbgVal) {
     LLVM_DEBUG(dbgs() << " Deciding: ");
     LLVM_DEBUG(DbgVal->print(dbgs()));
     LLVM_DEBUG(dbgs() << "\n");
@@ -581,7 +581,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
       LLVM_DEBUG(dbgs() << "  Include (parameter): ");
       LLVM_DEBUG(DbgVal->print(dbgs()));
       LLVM_DEBUG(dbgs() << "\n");
-      Container.insert(DbgVal);
+      PDVRRelated.insert(DbgVal);
     } else {
       LLVM_DEBUG(dbgs() << "  Delete (!parameter): ");
       LLVM_DEBUG(DbgVal->print(dbgs()));
@@ -589,7 +589,8 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
     }
   };
 
-  auto ExamineDbgDeclare = [&PDIRelated](auto *DbgDecl, auto &Container) {
+  auto ExamineDbgDeclare = [&PDIRelated,
+                            &PDVRRelated](DbgVariableRecord *DbgDecl) {
     LLVM_DEBUG(dbgs() << " Deciding: ");
     LLVM_DEBUG(DbgDecl->print(dbgs()));
     LLVM_DEBUG(dbgs() << "\n");
@@ -616,7 +617,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
                 LLVM_DEBUG(dbgs() << "  Include: ");
                 LLVM_DEBUG(DbgDecl->print(dbgs()));
                 LLVM_DEBUG(dbgs() << "\n");
-                Container.insert(DbgDecl);
+                PDVRRelated.insert(DbgDecl);
               } else {
                 LLVM_DEBUG(dbgs() << "   Delete (!parameter): ");
                 LLVM_DEBUG(SI->print(dbgs()));
@@ -647,18 +648,14 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
     // they connected to parameters?
     for (DbgVariableRecord &DVR : filterDbgVars(BI->getDbgRecordRange())) {
       if (DVR.isDbgValue() || DVR.isDbgAssign()) {
-        ExamineDbgValue(&DVR, PDVRRelated);
+        ExamineDbgValue(&DVR);
       } else {
         assert(DVR.isDbgDeclare());
-        ExamineDbgDeclare(&DVR, PDVRRelated);
+        ExamineDbgDeclare(&DVR);
       }
     }
 
-    if (auto *DVI = dyn_cast<DbgValueInst>(&*BI)) {
-      ExamineDbgValue(DVI, PDIRelated);
-    } else if (auto *DDI = dyn_cast<DbgDeclareInst>(&*BI)) {
-      ExamineDbgDeclare(DDI, PDIRelated);
-    } else if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
+    if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
       LLVM_DEBUG(dbgs() << " Will Include Terminator: ");
       LLVM_DEBUG(BI->print(dbgs()));
       LLVM_DEBUG(dbgs() << "\n");
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 9d7c025ccff86..f7fbf0815df03 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -825,9 +825,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Value *EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned);
 
   bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock);
-  void tryToSinkInstructionDbgValues(
-      Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
-      BasicBlock *DestBlock, SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers);
   void tryToSinkInstructionDbgVariableRecords(
       Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
       BasicBlock *DestBlock, SmallVectorImpl<DbgVariableRecord *> &DPUsers);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 684b9a1f90161..503611a4fc32c 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1420,21 +1420,16 @@ void InstCombinerImpl::freelyInvertAllUsersOf(Value *I, Value *IgnoredUser) {
   SmallVector<DbgValueInst *, 4> DbgValues;
   SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
   llvm::findDbgValues(DbgValues, I, &DbgVariableRecords);
+  assert(DbgValues.empty());
 
-  auto InvertDbgValueUse = [&](auto *DbgVal) {
+  for (DbgVariableRecord *DbgVal : DbgVariableRecords) {
     SmallVector<uint64_t, 1> Ops = {dwarf::DW_OP_not};
     for (unsigned Idx = 0, End = DbgVal->getNumVariableLocationOps();
          Idx != End; ++Idx)
       if (DbgVal->getVariableLocationOp(Idx) == I)
         DbgVal->setExpression(
             DIExpression::appendOpsToArg(DbgVal->getExpression(), Ops, Idx));
-  };
-
-  for (DbgValueInst *DVI : DbgValues)
-    InvertDbgValueUse(DVI);
-
-  for (DbgVariableRecord *DVR : DbgVariableRecords)
-    InvertDbgValueUse(DVR);
+  }
 }
 
 /// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a
@@ -3575,6 +3570,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
   std::unique_ptr<DIBuilder> DIB;
   if (isa<AllocaInst>(MI)) {
     findDbgUsers(DVIs, &MI, &DVRs);
+    assert(DVIs.empty());
     DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
   }
 
@@ -5253,8 +5249,7 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I,
   SmallVector<DbgVariableIntrinsic *, 2> DbgUsers;
   SmallVector<DbgVariableRecord *, 2> DbgVariableRecords;
   findDbgUsers(DbgUsers, I, &DbgVariableRecords);
-  if (!DbgUsers.empty())
-    tryToSinkInstructionDbgValues(I, InsertPos, SrcBlock, DestBlock, DbgUsers);
+  assert(DbgUsers.empty());
   if (!DbgVariableRecords.empty())
     tryToSinkInstructionDbgVariableRecords(I, InsertPos, SrcBlock, DestBlock,
                                            DbgVariableRecords);
@@ -5271,71 +5266,12 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I,
   return true;
 }
 
-void InstCombinerImpl::tryToSinkInstructionDbgValues(
-    Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
-    BasicBlock *DestBlock, SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers) {
-  // For all debug values in the destination block, the sunk instruction
-  // will still be available, so they do not need to be dropped.
-  SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSalvage;
-  for (auto &DbgUser : DbgUsers)
-    if (DbgUser->getParent() != DestBlock)
-      DbgUsersToSalvage.push_back(DbgUser);
-
-  // Process the sinking DbgUsersToSalvage in reverse order, as we only want
-  // to clone the last appearing debug intrinsic for each given variable.
-  SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSink;
-  for (DbgVariableIntrinsic *DVI : DbgUsersToSalvage)
-    if (DVI->getParent() == SrcBlock)
-      DbgUsersToSink.push_back(DVI);
-  llvm::sort(DbgUsersToSink,
-             [](auto *A, auto *B) { return B->comesBefore(A); });
-
-  SmallVector<DbgVariableIntrinsic *, 2> DIIClones;
-  SmallSet<DebugVariable, 4> SunkVariables;
-  for (auto *User : DbgUsersToSink) {
-    // A dbg.declare instruction should not be cloned, since there can only be
-    // one per variable fragment. It should be left in the original place
-    // because the sunk instruction is not an alloca (otherwise we could not be
-    // here).
-    if (isa<DbgDeclareInst>(User))
-      continue;
-
-    DebugVariable DbgUserVariable =
-        DebugVariable(User->getVariable(), User->getExpression(),
-                      User->getDebugLoc()->getInlinedAt());
-
-    if (!SunkVariables.insert(DbgUserVariable).second)
-      continue;
-
-    // Leave dbg.assign intrinsics in their original positions and there should
-    // be no need to insert a clone.
-    if (isa<DbgAssignIntrinsic>(User))
-      continue;
-
-    DIIClones.emplace_back(cast<DbgVariableIntrinsic>(User->clone()));
-    if (isa<DbgDeclareInst>(User) && isa<CastInst>(I))
-      DIIClones.back()->replaceVariableLocationOp(I, I->getOperand(0));
-    LLVM_DEBUG(dbgs() << "CLONE: " << *DIIClones.back() << '\n');
-  }
-
-  // Perform salvaging without the clones, then sink the clones.
-  if (!DIIClones.empty()) {
-    salvageDebugInfoForDbgValues(*I, DbgUsersToSalvage, {});
-    // The clones are in reverse order of original appearance, reverse again to
-    // maintain the original order.
-    for (auto &DIIClone : llvm::reverse(DIIClones)) {
-      DIIClone->insertBefore(InsertPos);
-      LLVM_DEBUG(dbgs() << "SINK: " << *DIIClone << '\n');
-    }
-  }
-}
-
 void InstCombinerImpl::tryToSinkInstructionDbgVariableRecords(
     Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
     BasicBlock *DestBlock,
     SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
-  // Implementation of tryToSinkInstructionDbgValues, but for the
-  // DbgVariableRecord of variable assignments rather than dbg.values.
+  // For all debug values in the destination block, the sunk instruction
+  // will still be available, so they do not need to be dropped.
 
   // Fetch all DbgVariableRecords not already in the destination.
   SmallVector<DbgVariableRecord *, 2> DbgVariableRecordsToSalvage;
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 2786d81773ed9..df3160233c510 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1489,6 +1489,7 @@ static bool checkAndReplaceCondition(
     SmallVector<DbgVariableIntrinsic *> DbgUsers;
     SmallVector<DbgVariableRecord *> DVRUsers;
     findDbgUsers(DbgUsers, Cmp, &DVRUsers);
+    assert(DbgUsers.empty());
 
     for (auto *DVR : DVRUsers) {
       auto *DTN = DT.getNode(DVR->getParent());
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 85dd9a1bf7161..0f63ed0166cf4 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -2079,6 +2079,7 @@ struct DSEState {
       AllocFnKind AllocKind =
           Attrs.getFnAttr(Attribute::AllocKind).getAllocKind() |
           AllocFnKind::Zeroed;
+      AllocKind &= ~AllocFnKind::Uninitialized;
       Attrs =
           Attrs.addFnAttribute(Ctx, Attribute::getWithAllocKind(Ctx, AllocKind))
               .removeFnAttribute(Ctx, "alloc-variant-zeroed");
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index b5dbef13289ac..4d1f44076db7e 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1979,15 +1979,13 @@ void JumpThreadingPass::updateSSA(BasicBlock *BB, BasicBlock *NewBB,
 
     // Find debug values outside of the block
     findDbgValues(DbgValues, &I, &DbgVariableRecords);
-    llvm::erase_if(DbgValues, [&](const DbgValueInst *DbgVal) {
-      return DbgVal->getParent() == BB;
-    });
+    assert(DbgValues.empty());
     llvm::erase_if(DbgVariableRecords, [&](const DbgVariableRecord *DbgVarRec) {
       return DbgVarRec->getParent() == BB;
     });
 
     // If there are no uses outside the block, we're done with this instruction.
-    if (UsesToRename.empty() && DbgValues.empty() && DbgVariableRecords.empty())
+    if (UsesToRename.empty() && DbgVariableRecords.empty())
       continue;
     LLVM_DEBUG(dbgs() << "JT: Renaming non-local uses of: " << I << "\n");
 
@@ -2000,8 +1998,7 @@ void JumpThreadingPass::updateSSA(BasicBlock *BB, BasicBlock *NewBB,
 
     while (!UsesToRename.empty())
       SSAUpdate.RewriteUse(*UsesToRename.pop_back_val());
-    if (!DbgValues.empty() || !DbgVariableRecords.empty()) {
-      SSAUpdate.UpdateDebugValues(&I, DbgValues);
+    if (!DbgVariableRecords.empty()) {
       SSAUpdate.UpdateDebugValues(&I, DbgVariableRecords);
       DbgValues.clear();
       DbgVariableRecords.clear();
@@ -2032,32 +2029,7 @@ void JumpThreadingPass::cloneInstructions(ValueToValueMapTy &ValueMapping,
   // copy of the block 'NewBB'.  If there are PHI nodes in the source basic
   // block, evaluate them to account for entry from PredBB.
 
-  // Retargets llvm.dbg.value to any renamed variables.
-  auto RetargetDbgValueIfPossible = [&](Instruction *NewInst) -> bool {
-    auto DbgInstruction = dyn_cast<DbgValueInst>(NewInst);
-    if (!DbgInstruction)
-      return false;
-
-    SmallSet<std::pair<Value *, Value *>, 16> OperandsToRemap;
-    for (auto DbgOperand : DbgInstruction->location_ops()) {
-      auto DbgOperandInstruction = dyn_cast<Instruction>(DbgOperand);
-      if (!DbgOperandInstruction)
-        continue;
-
-      auto I = ValueMapping.find(DbgOperandInstruction);
-      if (I != ValueMapping.end()) {
-        OperandsToRemap.insert(
-            std::pair<Value *, Value *>(DbgOperand, I->second));
-      }
-    }
-
-    for (auto &[OldOp, MappedOp] : OperandsToRemap)
-      DbgInstruction->replaceVariableLocationOp(OldOp, MappedOp);
-    return true;
-  };
-
-  // Duplicate implementation of the above dbg.value code, using
-  // DbgVariableRecords instead.
+  // Retargets dbg.value to any renamed variables.
   auto RetargetDbgVariableRecordIfPossible = [&](DbgVariableRecord *DVR) {
     SmallSet<std::pair<Value *, Value *>, 16> OperandsToRemap;
     for (auto *Op : DVR->location_ops()) {
@@ -2116,9 +2088,6 @@ void JumpThreadingPass::cloneInstructions(ValueToValueMapTy &ValueMapping,
     if (const DebugLoc &DL = New->getDebugLoc())
       mapAtomInstance(DL, ValueMapping);
 
-    if (RetargetDbgValueIfPossible(New))
-      continue;
-
     // Remap operands to patch up intra-block references.
     for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)
       if (Instruction *Inst = dyn_cast<Instruction>(New->getOperand(i))) {
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index a0f9f3c4a35a5..70e9eee5339a7 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -78,6 +78,7 @@ enum class RuleTy {
   PerLoopCacheAnalysis,
   PerInstrOrderCost,
   ForVectorization,
+  Ignore
 };
 
 } // end anonymous namespace
@@ -106,14 +107,20 @@ static cl::list<RuleTy> Profitabilities(
                clEnumValN(RuleTy::PerInstrOrderCost, "instorder",
                           "Prioritize the IVs order of each instruction"),
                clEnumValN(RuleTy::ForVectorization, "vectorize",
-                          "Prioritize vectorization")));
+                          "Prioritize vectorization"),
+               clEnumValN(RuleTy::Ignore, "ignore",
+                          "Ignore profitability, force interchange (does not "
+                          "work with other options)")));
 
 #ifndef NDEBUG
-static bool noDuplicateRules(ArrayRef<RuleTy> Rules) {
+static bool noDuplicateRulesAndIgnore(ArrayRef<RuleTy> Rules) {
   SmallSet<RuleTy, 4> Set;
-  for (RuleTy Rule : Rules)
+  for (RuleTy Rule : Rules) {
     if (!Set.insert(Rule).second)
       return false;
+    if (Rule == RuleTy::Ignore)
+      return false;
+  }
   return true;
 }
 
@@ -1357,6 +1364,13 @@ std::optional<bool> LoopInterchangeProfitability::isProfitableForVectorization(
 bool LoopInterchangeProfitability::isProfitable(
     const Loop *InnerLoop, const Loop *OuterLoop, unsigned InnerLoopId,
     unsigned OuterLoopId, CharMatrix &DepMatrix, CacheCostManager &CCM) {
+
+  // Return true if interchange is forced and the cost-model ignored.
+  if (Profitabilities.size() == 1 && Profitabilities[0] == RuleTy::Ignore)
+    return true;
+  assert(noDuplicateRulesAndIgnore(Profitabilities) &&
+         "Duplicate rules and option 'ignore' are not allowed");
+
   // isProfitable() is structured to avoid endless loop interchange. If the
   // highest priority rule (isProfitablePerLoopCacheAnalysis by default) could
   // decide the profitability then, profitability check will stop and return the
@@ -1365,7 +1379,6 @@ bool LoopInterchangeProfitability::isProfitable(
   // second highest priority rule (isProfitablePerInstrOrderCost by default).
   // Likewise, if it failed to analysis the profitability then only, the last
   // rule (isProfitableForVectorization by default) will decide.
-  assert(noDuplicateRules(Profitabilities) && "Detect duplicate rules");
   std::optional<bool> shouldInterchange;
   for (RuleTy RT : Profitabilities) {
     switch (RT) {
@@ -1382,6 +1395,9 @@ bool LoopInterchangeProfitability::isProfitable(
       shouldInterchange =
           isProfitableForVectorization(InnerLoopId, OuterLoopId, DepMatrix);
       break;
+    case RuleTy::Ignore:
+      llvm_unreachable("Option 'ignore' is not supported with other options");
+      break;
     }
 
     // If this rule could determine the profitability, don't call subsequent
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index dc8fa4379752f..9e318b04c2c99 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -3790,6 +3790,11 @@ LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
             continue;
         }
 
+        // Do not consider uses inside lifetime intrinsics. These are not
+        // actually materialized.
+        if (UserInst->isLifetimeStartOrEnd())
+          continue;
+
         std::pair<size_t, Immediate> P =
             getUse(S, LSRUse::Basic, MemAccessTy());
         size_t LUIdx = P.first;
@@ -6630,13 +6635,10 @@ struct SCEVDbgValueBuilder {
 /// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
 /// and DIExpression.
 struct DVIRecoveryRec {
-  DVIRecoveryRec(DbgValueInst *DbgValue)
-      : DbgRef(DbgValue), Expr(DbgValue->getExpression()),
-        HadLocationArgList(false) {}
   DVIRecoveryRec(DbgVariableRecord *DVR)
       : DbgRef(DVR), Expr(DVR->getExpression()), HadLocationArgList(false) {}
 
-  PointerUnion<DbgValueInst *, DbgVariableRecord *> DbgRef;
+  DbgVariableRecord *DbgRef;
   DIExpression *Expr;
   bool HadLocationArgList;
   SmallVector<WeakVH, 2> LocationOps;
@@ -6695,44 +6697,38 @@ static void updateDVIWithLocations(T &DbgVal,
 }
 
 /// Write the new expression and new location ops for the dbg.value. If possible
-/// reduce the szie of the dbg.value intrinsic by omitting DIArglist. This
+/// reduce the szie of the dbg.value by omitting DIArglist. This
 /// can be omitted if:
 /// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
 /// 2. The DW_OP_LLVM_arg is the first operand in the expression.
-static void UpdateDbgValueInst(DVIRecoveryRec &DVIRec,
-                               SmallVectorImpl<Value *> &NewLocationOps,
-                               SmallVectorImpl<uint64_t> &NewExpr) {
-  auto UpdateDbgValueInstImpl = [&](auto *DbgVal) {
-    unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
-    if (NumLLVMArgs == 0) {
-      // Location assumed to be on the stack.
-      updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
-    } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
-      // There is only a single DW_OP_llvm_arg at the start of the expression,
-      // so it can be omitted along with DIArglist.
-      assert(NewExpr[1] == 0 &&
-             "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
-      llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2));
-      updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
-    } else {
-      // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
-      updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
-    }
+static void UpdateDbgValue(DVIRecoveryRec &DVIRec,
+                           SmallVectorImpl<Value *> &NewLocationOps,
+                           SmallVectorImpl<uint64_t> &NewExpr) {
+  DbgVariableRecord *DbgVal = DVIRec.DbgRef;
+  unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
+  if (NumLLVMArgs == 0) {
+    // Location assumed to be on the stack.
+    updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
+  } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
+    // There is only a single DW_OP_llvm_arg at the start of the expression,
+    // so it can be omitted along with DIArglist.
+    assert(NewExpr[1] == 0 &&
+           "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
+    llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2));
+    updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
+  } else {
+    // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
+    updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
+  }
 
-    // If the DIExpression was previously empty then add the stack terminator.
-    // Non-empty expressions have only had elements inserted into them and so
-    // the terminator should already be present e.g. stack_value or fragment.
-    DIExpression *SalvageExpr = DbgVal->getExpression();
-    if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
-      SalvageExpr =
-          DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
-      DbgVal->setExpression(SalvageExpr);
-    }
-  };
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    UpdateDbgValueInstImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
-  else
-    UpdateDbgValueInstImpl(cast<DbgVariableRecord *>(DVIRec.DbgRef));
+  // If the DIExpression was previously empty then add the stack terminator.
+  // Non-empty expressions have only had elements inserted into them and so
+  // the terminator should already be present e.g. stack_value or fragment.
+  DIExpression *SalvageExpr = DbgVal->getExpression();
+  if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
+    SalvageExpr = DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
+    DbgVal->setExpression(SalvageExpr);
+  }
 }
 
 /// Cached location ops may be erased during LSR, in which case a poison is
@@ -6746,39 +6742,34 @@ static Value *getValueOrPoison(WeakVH &VH, LLVMContext &C) {
 
 /// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
 static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
-  auto RestorePreTransformStateImpl = [&](auto *DbgVal) {
-    LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
-                      << "scev-salvage: post-LSR: " << *DbgVal << '\n');
-    assert(DVIRec.Expr && "Expected an expression");
-    DbgVal->setExpression(DVIRec.Expr);
-
-    // Even a single location-op may be inside a DIArgList and referenced with
-    // DW_OP_LLVM_arg, which is valid only with a DIArgList.
-    if (!DVIRec.HadLocationArgList) {
-      assert(DVIRec.LocationOps.size() == 1 &&
-             "Unexpected number of location ops.");
-      // LSR's unsuccessful salvage attempt may have added DIArgList, which in
-      // this case was not present before, so force the location back to a
-      // single uncontained Value.
-      Value *CachedValue =
-          getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
-      DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
-    } else {
-      SmallVector<ValueAsMetadata *, 3> MetadataLocs;
-      for (WeakVH VH : DVIRec.LocationOps) {
-        Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
-        MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
-      }
-      auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
-      DbgVal->setRawLocation(
-          llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
+  DbgVariableRecord *DbgVal = DVIRec.DbgRef;
+  LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
+                    << "scev-salvage: post-LSR: " << *DbgVal << '\n');
+  assert(DVIRec.Expr && "Expected an expression");
+  DbgVal->setExpression(DVIRec.Expr);
+
+  // Even a single location-op may be inside a DIArgList and referenced with
+  // DW_OP_LLVM_arg, which is valid only with a DIArgList.
+  if (!DVIRec.HadLocationArgList) {
+    assert(DVIRec.LocationOps.size() == 1 &&
+           "Unexpected number of location ops.");
+    // LSR's unsuccessful salvage attempt may have added DIArgList, which in
+    // this case was not present before, so force the location back to a
+    // single uncontained Value.
+    Value *CachedValue =
+        getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
+    DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
+  } else {
+    SmallVector<ValueAsMetadata *, 3> MetadataLocs;
+    for (WeakVH VH : DVIRec.LocationOps) {
+      Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
+      MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
     }
-    LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
-  };
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    RestorePreTransformStateImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
-  else
-    RestorePreTransformStateImpl(cast<DbgVariableRecord *>(DVIRec.DbgRef));
+    auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
+    DbgVal->setRawLocation(
+        llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
+  }
+  LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
 }
 
 static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
@@ -6786,9 +6777,7 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
                        const SCEV *SCEVInductionVar,
                        SCEVDbgValueBuilder IterCountExpr) {
 
-  if (isa<DbgValueInst *>(DVIRec.DbgRef)
-          ? !cast<DbgValueInst *>(DVIRec.DbgRef)->isKillLocation()
-          : !cast<DbgVariableRecord *>(DVIRec.DbgRef)->isKillLocation())
+  if (!DVIRec.DbgRef->isKillLocation())
     return false;
 
   // LSR may have caused several changes to the dbg.value in the failed salvage
@@ -6882,13 +6871,8 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
     DbgBuilder->appendToVectors(NewExpr, NewLocationOps);
   }
 
-  UpdateDbgValueInst(DVIRec, NewLocationOps, NewExpr);
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
-                      << *cast<DbgValueInst *>(DVIRec.DbgRef) << "\n");
-  else
-    LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
-                      << *cast<DbgVariableRecord *>(DVIRec.DbgRef) << "\n");
+  UpdateDbgValue(DVIRec, NewLocationOps, NewExpr);
+  LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DbgRef << "\n");
   return true;
 }
 
@@ -6934,21 +6918,23 @@ static void DbgRewriteSalvageableDVIs(
 /// cacheing and salvaging.
 static void DbgGatherSalvagableDVI(
     Loop *L, ScalarEvolution &SE,
-    SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs,
-    SmallSet<AssertingVH<DbgValueInst>, 2> &DVIHandles) {
+    SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs) {
   for (const auto &B : L->getBlocks()) {
     for (auto &I : *B) {
-      auto ProcessDbgValue = [&](auto *DbgVal) -> bool {
+      for (DbgVariableRecord &DbgVal : filterDbgVars(I.getDbgRecordRange())) {
+        if (!DbgVal.isDbgValue() && !DbgVal.isDbgAssign())
+          continue;
+
         // Ensure that if any location op is undef that the dbg.vlue is not
         // cached.
-        if (DbgVal->isKillLocation())
-          return false;
+        if (DbgVal.isKillLocation())
+          continue;
 
         // Check that the location op SCEVs are suitable for translation to
         // DIExpression.
         const auto &HasTranslatableLocationOps =
-            [&](const auto *DbgValToTranslate) -> bool {
-          for (const auto LocOp : DbgValToTranslate->location_ops()) {
+            [&](const DbgVariableRecord &DbgValToTranslate) -> bool {
+          for (const auto LocOp : DbgValToTranslate.location_ops()) {
             if (!LocOp)
               return false;
 
@@ -6963,31 +6949,21 @@ static void DbgGatherSalvagableDVI(
         };
 
         if (!HasTranslatableLocationOps(DbgVal))
-          return false;
+          continue;
 
         std::unique_ptr<DVIRecoveryRec> NewRec =
-            std::make_unique<DVIRecoveryRec>(DbgVal);
+            std::make_unique<DVIRecoveryRec>(&DbgVal);
         // Each location Op may need a SCEVDbgValueBuilder in order to recover
         // it. Pre-allocating a vector will enable quick lookups of the builder
         // later during the salvage.
-        NewRec->RecoveryExprs.resize(DbgVal->getNumVariableLocationOps());
-        for (const auto LocOp : DbgVal->location_ops()) {
+        NewRec->RecoveryExprs.resize(DbgVal.getNumVariableLocationOps());
+        for (const auto LocOp : DbgVal.location_ops()) {
           NewRec->SCEVs.push_back(SE.getSCEV(LocOp));
           NewRec->LocationOps.push_back(LocOp);
-          NewRec->HadLocationArgList = DbgVal->hasArgList();
+          NewRec->HadLocationArgList = DbgVal.hasArgList();
         }
         SalvageableDVISCEVs.push_back(std::move(NewRec));
-        return true;
-      };
-      for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
-        if (DVR.isDbgValue() || DVR.isDbgAssign())
-          ProcessDbgValue(&DVR);
       }
-      auto DVI = dyn_cast<DbgValueInst>(&I);
-      if (!DVI)
-        continue;
-      if (ProcessDbgValue(DVI))
-        DVIHandles.insert(DVI);
     }
   }
 }
@@ -7036,8 +7012,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   // Debug preservation - before we start removing anything identify which DVI
   // meet the salvageable criteria and store their DIExpression and SCEVs.
   SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> SalvageableDVIRecords;
-  SmallSet<AssertingVH<DbgValueInst>, 2> DVIHandles;
-  DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords, DVIHandles);
+  DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords);
 
   bool Changed = false;
   std::unique_ptr<MemorySSAUpdater> MSSAU;
@@ -7105,7 +7080,6 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   for (auto &Rec : SalvageableDVIRecords)
     Rec->clear();
   SalvageableDVIRecords.clear();
-  DVIHandles.clear();
   return Changed;
 }
 
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index eacaf42e4e8ba..1d1af42153325 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1222,9 +1222,7 @@ static void eraseDebugIntrinsicsWithNonLocalRefs(Function &F) {
     SmallVector<DbgVariableIntrinsic *, 4> DbgUsers;
     SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
     findDbgUsers(DbgUsers, &I, &DbgVariableRecords);
-    for (DbgVariableIntrinsic *DVI : DbgUsers)
-      if (DVI->getFunction() != &F)
-        DVI->eraseFromParent();
+    assert(DbgUsers.empty());
     for (DbgVariableRecord *DVR : DbgVariableRecords)
       if (DVR->getFunction() != &F)
         DVR->eraseFromParent();
@@ -1289,14 +1287,12 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
     SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
     SmallVector<DbgVariableRecord *, 1> DPUsers;
     findDbgUsers(DbgUsers, Input, &DPUsers);
+    assert(DbgUsers.empty());
     DIExpression *Expr = DIB.createExpression();
 
     // Iterate the debud users of the Input values. If they are in the extracted
     // function then update their location with the new value. If they are in
     // the parent function then create a similar debug record.
-    for (auto *DVI : DbgUsers)
-      UpdateOrInsertDebugRecord(DVI, Input, NewVal, Expr,
-                                isa<DbgDeclareInst>(DVI));
     for (auto *DVR : DPUsers)
       UpdateOrInsertDebugRecord(DVR, Input, NewVal, Expr, DVR->isDbgDeclare());
   }
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index a1f030a336c15..4210ce6da1eb2 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -808,9 +808,6 @@ bool checkDebugifyMetadata(Module &M,
 
     // Find missing lines.
     for (Instruction &I : instructions(F)) {
-      if (isa<DbgValueInst>(&I))
-        continue;
-
       auto DL = I.getDebugLoc();
       if (DL && DL.getLine() != 0) {
         MissingLines.reset(DL.getLine() - 1);
@@ -839,10 +836,6 @@ bool checkDebugifyMetadata(Module &M,
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
         if (DVR.isDbgValue() || DVR.isDbgAssign())
           CheckForMisSized(&DVR);
-      auto *DVI = dyn_cast<DbgValueInst>(&I);
-      if (!DVI)
-        continue;
-      CheckForMisSized(DVI);
     }
   }
 
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 6929d14bc56ea..ed3dca2f7c307 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1978,14 +1978,13 @@ static at::StorageToVarsMap collectEscapedLocals(const DataLayout &DL,
       continue;
 
     // Find all local variables associated with the backing storage.
-    auto CollectAssignsForStorage = [&](auto *DbgAssign) {
+    auto CollectAssignsForStorage = [&](DbgVariableRecord *DbgAssign) {
       // Skip variables from inlined functions - they are not local variables.
       if (DbgAssign->getDebugLoc().getInlinedAt())
         return;
       LLVM_DEBUG(errs() << " > DEF : " << *DbgAssign << "\n");
       EscapedLocals[Base].insert(at::VarRecord(DbgAssign));
     };
-    for_each(at::getAssignmentMarkers(Base), CollectAssignsForStorage);
     for_each(at::getDVRAssignmentMarkers(Base), CollectAssignsForStorage);
   }
   return EscapedLocals;
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index b14bbeac97675..d481ad9dee181 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -613,11 +613,10 @@ bool llvm::replaceDbgUsesWithUndef(Instruction *I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, I, &DPUsers);
-  for (auto *DII : DbgUsers)
-    DII->setKillLocation();
+  assert(DbgUsers.empty());
   for (auto *DVR : DPUsers)
     DVR->setKillLocation();
-  return !DbgUsers.empty() || !DPUsers.empty();
+  return !DPUsers.empty();
 }
 
 /// areAllUsesEqual - Check whether the uses of a value are all the same.
@@ -1607,12 +1606,8 @@ static bool PhiHasDebugValue(DILocalVariable *DIVar,
   SmallVector<DbgValueInst *, 1> DbgValues;
   SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
   findDbgValues(DbgValues, APN, &DbgVariableRecords);
-  for (auto *DVI : DbgValues) {
-    assert(is_contained(DVI->getValues(), APN));
-    if ((DVI->getVariable() == DIVar) && (DVI->getExpression() == DIExpr))
-      return true;
-  }
-  for (auto *DVR : DbgVariableRecords) {
+  assert(DbgValues.empty());
+  for (DbgVariableRecord *DVR : DbgVariableRecords) {
     assert(is_contained(DVR->location_ops(), APN));
     if ((DVR->getVariable() == DIVar) && (DVR->getExpression() == DIExpr))
       return true;
@@ -1971,7 +1966,6 @@ bool llvm::replaceDbgDeclare(Value *Address, Value *NewAddress,
 static void updateOneDbgValueForAlloca(const DebugLoc &Loc,
                                        DILocalVariable *DIVar,
                                        DIExpression *DIExpr, Value *NewAddress,
-                                       DbgValueInst *DVI,
                                        DbgVariableRecord *DVR,
                                        DIBuilder &Builder, int Offset) {
   assert(DIVar && "Missing variable");
@@ -1987,14 +1981,8 @@ static void updateOneDbgValueForAlloca(const DebugLoc &Loc,
   if (Offset)
     DIExpr = DIExpression::prepend(DIExpr, 0, Offset);
 
-  if (DVI) {
-    DVI->setExpression(DIExpr);
-    DVI->replaceVariableLocationOp(0u, NewAddress);
-  } else {
-    assert(DVR);
-    DVR->setExpression(DIExpr);
-    DVR->replaceVariableLocationOp(0u, NewAddress);
-  }
+  DVR->setExpression(DIExpr);
+  DVR->replaceVariableLocationOp(0u, NewAddress);
 }
 
 void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
@@ -2002,18 +1990,13 @@ void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
   SmallVector<DbgValueInst *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgValues(DbgUsers, AI, &DPUsers);
-
-  // Attempt to replace dbg.values that use this alloca.
-  for (auto *DVI : DbgUsers)
-    updateOneDbgValueForAlloca(DVI->getDebugLoc(), DVI->getVariable(),
-                               DVI->getExpression(), NewAllocaAddress, DVI,
-                               nullptr, Builder, Offset);
+  assert(DbgUsers.empty());
 
   // Replace any DbgVariableRecords that use this alloca.
   for (DbgVariableRecord *DVR : DPUsers)
     updateOneDbgValueForAlloca(DVR->getDebugLoc(), DVR->getVariable(),
-                               DVR->getExpression(), NewAllocaAddress, nullptr,
-                               DVR, Builder, Offset);
+                               DVR->getExpression(), NewAllocaAddress, DVR,
+                               Builder, Offset);
 }
 
 /// Where possible to salvage debug information for \p I do so.
@@ -2022,6 +2005,7 @@ void llvm::salvageDebugInfo(Instruction &I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, &I, &DPUsers);
+  assert(DbgUsers.empty());
   salvageDebugInfoForDbgValues(I, DbgUsers, DPUsers);
 }
 
@@ -2070,66 +2054,9 @@ void llvm::salvageDebugInfoForDbgValues(
   const unsigned MaxExpressionSize = 128;
   bool Salvaged = false;
 
-  for (auto *DII : DbgUsers) {
-    if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DII)) {
-      if (DAI->getAddress() == &I) {
-        salvageDbgAssignAddress(DAI);
-        Salvaged = true;
-      }
-      if (DAI->getValue() != &I)
-        continue;
-    }
+  // We should never see debug intrinsics nowadays.
+  assert(DbgUsers.empty());
 
-    // Do not add DW_OP_stack_value for DbgDeclare, because they are implicitly
-    // pointing out the value as a DWARF memory location description.
-    bool StackValue = isa<DbgValueInst>(DII);
-    auto DIILocation = DII->location_ops();
-    assert(
-        is_contained(DIILocation, &I) &&
-        "DbgVariableIntrinsic must use salvaged instruction as its location");
-    SmallVector<Value *, 4> AdditionalValues;
-    // `I` may appear more than once in DII's location ops, and each use of `I`
-    // must be updated in the DIExpression and potentially have additional
-    // values added; thus we call salvageDebugInfoImpl for each `I` instance in
-    // DIILocation.
-    Value *Op0 = nullptr;
-    DIExpression *SalvagedExpr = DII->getExpression();
-    auto LocItr = find(DIILocation, &I);
-    while (SalvagedExpr && LocItr != DIILocation.end()) {
-      SmallVector<uint64_t, 16> Ops;
-      unsigned LocNo = std::distance(DIILocation.begin(), LocItr);
-      uint64_t CurrentLocOps = SalvagedExpr->getNumLocationOperands();
-      Op0 = salvageDebugInfoImpl(I, CurrentLocOps, Ops, AdditionalValues);
-      if (!Op0)
-        break;
-      SalvagedExpr =
-          DIExpression::appendOpsToArg(SalvagedExpr, Ops, LocNo, StackValue);
-      LocItr = std::find(++LocItr, DIILocation.end(), &I);
-    }
-    // salvageDebugInfoImpl should fail on examining the first element of
-    // DbgUsers, or none of them.
-    if (!Op0)
-      break;
-
-    SalvagedExpr = SalvagedExpr->foldConstantMath();
-    DII->replaceVariableLocationOp(&I, Op0);
-    bool IsValidSalvageExpr = SalvagedExpr->getNumElements() <= MaxExpressionSize;
-    if (AdditionalValues.empty() && IsValidSalvageExpr) {
-      DII->setExpression(SalvagedExpr);
-    } else if (isa<DbgValueInst>(DII) && IsValidSalvageExpr &&
-               DII->getNumVariableLocationOps() + AdditionalValues.size() <=
-                   MaxDebugArgs) {
-      DII->addVariableLocationOps(AdditionalValues, SalvagedExpr);
-    } else {
-      // Do not salvage using DIArgList for dbg.declare, as it is not currently
-      // supported in those instructions. Also do not salvage if the resulting
-      // DIArgList would contain an unreasonably large number of values.
-      DII->setKillLocation();
-    }
-    LLVM_DEBUG(dbgs() << "SALVAGE: " << *DII << '\n');
-    Salvaged = true;
-  }
-  // Duplicate of above block for DbgVariableRecords.
   for (auto *DVR : DPUsers) {
     if (DVR->isDbgAssign()) {
       if (DVR->getAddress() == &I) {
@@ -2198,9 +2125,6 @@ void llvm::salvageDebugInfoForDbgValues(
   if (Salvaged)
     return;
 
-  for (auto *DII : DbgUsers)
-    DII->setKillLocation();
-
   for (auto *DVR : DPUsers)
     DVR->setKillLocation();
 }
@@ -3429,8 +3353,7 @@ void llvm::dropDebugUsers(Instruction &I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, &I, &DPUsers);
-  for (auto *DII : DbgUsers)
-    DII->eraseFromParent();
+  assert(DbgUsers.empty());
   for (auto *DVR : DPUsers)
     DVR->eraseFromParent();
 }
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 66d0573e83f65..06115e0741ade 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -161,29 +161,8 @@ static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader,
     SmallVector<DbgValueInst *, 1> DbgValues;
     SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
     llvm::findDbgValues(DbgValues, OrigHeaderVal, &DbgVariableRecords);
-    for (auto &DbgValue : DbgValues) {
-      // The original users in the OrigHeader are already using the original
-      // definitions.
-      BasicBlock *UserBB = DbgValue->getParent();
-      if (UserBB == OrigHeader)
-        continue;
-
-      // Users in the OrigPreHeader need to use the value to which the
-      // original definitions are mapped and anything else can be handled by
-      // the SSAUpdater. To avoid adding PHINodes, check if the value is
-      // available in UserBB, if not substitute poison.
-      Value *NewVal;
-      if (UserBB == OrigPreheader)
-        NewVal = OrigPreHeaderVal;
-      else if (SSA.HasValueForBlock(UserBB))
-        NewVal = SSA.GetValueInMiddleOfBlock(UserBB);
-      else
-        NewVal = PoisonValue::get(OrigHeaderVal->getType());
-      DbgValue->replaceVariableLocationOp(OrigHeaderVal, NewVal);
-    }
+    assert(DbgValues.empty());
 
-    // RemoveDIs: duplicate implementation for non-instruction debug-info
-    // storage in DbgVariableRecords.
     for (DbgVariableRecord *DVR : DbgVariableRecords) {
       // The original users in the OrigHeader are already using the original
       // definitions.
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 154f50acf3d1d..e7623aaff105d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1103,7 +1103,6 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
       (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
        RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
        RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
-    // TODO: Add float minnum/maxnum support when FMF nnan is set.
     Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
     return Builder.CreateIntrinsic(Ty, Id, {Left, Right}, nullptr,
                                    "rdx.minmax");
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index ccd7ee360e014..73b5f48796b7a 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -190,7 +190,6 @@ class AssignmentTrackingInfo {
 };
 
 struct AllocaInfo {
-  using DbgUserVec = SmallVector<DbgVariableIntrinsic *, 1>;
   using DPUserVec = SmallVector<DbgVariableRecord *, 1>;
 
   SmallVector<BasicBlock *, 32> DefiningBlocks;
@@ -201,7 +200,6 @@ struct AllocaInfo {
   bool OnlyUsedInOneBlock;
 
   /// Debug users of the alloca - does not include dbg.assign intrinsics.
-  DbgUserVec DbgUsers;
   DPUserVec DPUsers;
   /// Helper to update assignment tracking debug info.
   AssignmentTrackingInfo AssignmentTracking;
@@ -212,7 +210,6 @@ struct AllocaInfo {
     OnlyStore = nullptr;
     OnlyBlock = nullptr;
     OnlyUsedInOneBlock = true;
-    DbgUsers.clear();
     DPUsers.clear();
     AssignmentTracking.clear();
   }
@@ -246,13 +243,10 @@ struct AllocaInfo {
           OnlyUsedInOneBlock = false;
       }
     }
-    DbgUserVec AllDbgUsers;
+    SmallVector<DbgVariableIntrinsic *> AllDbgUsers;
     SmallVector<DbgVariableRecord *> AllDPUsers;
     findDbgUsers(AllDbgUsers, AI, &AllDPUsers);
-    std::copy_if(AllDbgUsers.begin(), AllDbgUsers.end(),
-                 std::back_inserter(DbgUsers), [](DbgVariableIntrinsic *DII) {
-                   return !isa<DbgAssignIntrinsic>(DII);
-                 });
+    assert(AllDbgUsers.empty());
     std::copy_if(AllDPUsers.begin(), AllDPUsers.end(),
                  std::back_inserter(DPUsers),
                  [](DbgVariableRecord *DVR) { return !DVR->isDbgAssign(); });
@@ -380,10 +374,9 @@ struct PromoteMem2Reg {
   /// to.
   DenseMap<PHINode *, unsigned> PhiToAllocaMap;
 
-  /// For each alloca, we keep track of the dbg.declare intrinsic that
+  /// For each alloca, we keep track of the dbg.declare record that
   /// describes it, if any, so that we can convert it to a dbg.value
-  /// intrinsic if the alloca gets promoted.
-  SmallVector<AllocaInfo::DbgUserVec, 8> AllocaDbgUsers;
+  /// record if the alloca gets promoted.
   SmallVector<AllocaInfo::DPUserVec, 8> AllocaDPUsers;
 
   /// For each alloca, keep an instance of a helper class that gives us an easy
@@ -741,14 +734,11 @@ promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
   AI->eraseFromParent();
 
   // The alloca's debuginfo can be removed as well.
-  auto DbgUpdateForAlloca = [&](auto &Container) {
-    for (auto *DbgItem : Container)
-      if (DbgItem->isAddressOfVariable() ||
-          DbgItem->getExpression()->startsWithDeref())
-        DbgItem->eraseFromParent();
-  };
-  DbgUpdateForAlloca(Info.DbgUsers);
-  DbgUpdateForAlloca(Info.DPUsers);
+  for (DbgVariableRecord *DbgItem : Info.DPUsers) {
+    if (DbgItem->isAddressOfVariable() ||
+        DbgItem->getExpression()->startsWithDeref())
+      DbgItem->eraseFromParent();
+  }
 
   ++NumLocalPromoted;
   return true;
@@ -757,7 +747,6 @@ promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
 void PromoteMem2Reg::run() {
   Function &F = *DT.getRoot()->getParent();
 
-  AllocaDbgUsers.resize(Allocas.size());
   AllocaATInfo.resize(Allocas.size());
   AllocaDPUsers.resize(Allocas.size());
 
@@ -816,9 +805,7 @@ void PromoteMem2Reg::run() {
     if (BBNumPreds.empty())
       BBNumPreds.resize(F.getMaxBlockNumber());
 
-    // Remember the dbg.declare intrinsic describing this alloca, if any.
-    if (!Info.DbgUsers.empty())
-      AllocaDbgUsers[AllocaNum] = Info.DbgUsers;
+    // Remember the dbg.declare record describing this alloca, if any.
     if (!Info.AssignmentTracking.empty())
       AllocaATInfo[AllocaNum] = Info.AssignmentTracking;
     if (!Info.DPUsers.empty())
@@ -894,16 +881,12 @@ void PromoteMem2Reg::run() {
   }
 
   // Remove alloca's dbg.declare intrinsics from the function.
-  auto RemoveDbgDeclares = [&](auto &Container) {
-    for (auto &DbgUsers : Container) {
-      for (auto *DbgItem : DbgUsers)
-        if (DbgItem->isAddressOfVariable() ||
-            DbgItem->getExpression()->startsWithDeref())
-          DbgItem->eraseFromParent();
-    }
-  };
-  RemoveDbgDeclares(AllocaDbgUsers);
-  RemoveDbgDeclares(AllocaDPUsers);
+  for (auto &DbgUsers : AllocaDPUsers) {
+    for (DbgVariableRecord *DbgItem : DbgUsers)
+      if (DbgItem->isAddressOfVariable() ||
+          DbgItem->getExpression()->startsWithDeref())
+        DbgItem->eraseFromParent();
+  }
 
   // Loop over all of the PHI nodes and see if there are any that we can get
   // rid of because they merge all of the same incoming values.  This can
diff --git a/llvm/lib/Transforms/Utils/SSAUpdater.cpp b/llvm/lib/Transforms/Utils/SSAUpdater.cpp
index 5db7fc956c497..561c898ec55d8 100644
--- a/llvm/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/llvm/lib/Transforms/Utils/SSAUpdater.cpp
@@ -200,11 +200,7 @@ void SSAUpdater::UpdateDebugValues(Instruction *I) {
   SmallVector<DbgValueInst *, 4> DbgValues;
   SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
   llvm::findDbgValues(DbgValues, I, &DbgVariableRecords);
-  for (auto &DbgValue : DbgValues) {
-    if (DbgValue->getParent() == I->getParent())
-      continue;
-    UpdateDebugValue(I, DbgValue);
-  }
+  assert(DbgValues.empty());
   for (auto &DVR : DbgVariableRecords) {
     if (DVR->getParent() == I->getParent())
       continue;
@@ -212,13 +208,6 @@ void SSAUpdater::UpdateDebugValues(Instruction *I) {
   }
 }
 
-void SSAUpdater::UpdateDebugValues(Instruction *I,
-                                   SmallVectorImpl<DbgValueInst *> &DbgValues) {
-  for (auto &DbgValue : DbgValues) {
-    UpdateDebugValue(I, DbgValue);
-  }
-}
-
 void SSAUpdater::UpdateDebugValues(
     Instruction *I, SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
   for (auto &DVR : DbgVariableRecords) {
@@ -226,15 +215,6 @@ void SSAUpdater::UpdateDebugValues(
   }
 }
 
-void SSAUpdater::UpdateDebugValue(Instruction *I, DbgValueInst *DbgValue) {
-  BasicBlock *UserBB = DbgValue->getParent();
-  if (HasValueForBlock(UserBB)) {
-    Value *NewVal = GetValueAtEndOfBlock(UserBB);
-    DbgValue->replaceVariableLocationOp(I, NewVal);
-  } else
-    DbgValue->setKillLocation();
-}
-
 void SSAUpdater::UpdateDebugValue(Instruction *I, DbgVariableRecord *DVR) {
   BasicBlock *UserBB = DVR->getParent();
   if (HasValueForBlock(UserBB)) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index db0902c589d49..6e420632d83e5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -947,9 +947,8 @@ class LoopVectorizationCostModel {
   /// user options, for the given register kind.
   bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind);
 
-  /// \return True if maximizing vector bandwidth is enabled by the target or
-  /// user options, for the given vector factor.
-  bool useMaxBandwidth(ElementCount VF);
+  /// \return True if register pressure should be calculated for the given VF.
+  bool shouldCalculateRegPressureForVF(ElementCount VF);
 
   /// \return The size (in bits) of the smallest and widest types in the code
   /// that needs to be vectorized. We ignore values that remain scalar such as
@@ -1736,6 +1735,9 @@ class LoopVectorizationCostModel {
   /// Whether this loop should be optimized for size based on function attribute
   /// or profile information.
   bool OptForSize;
+
+  /// The highest VF possible for this loop, without using MaxBandwidth.
+  FixedScalableVFPair MaxPermissibleVFWithoutMaxBW;
 };
 } // end namespace llvm
 
@@ -3832,10 +3834,16 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   return FixedScalableVFPair::getNone();
 }
 
-bool LoopVectorizationCostModel::useMaxBandwidth(ElementCount VF) {
-  return useMaxBandwidth(VF.isScalable()
-                             ? TargetTransformInfo::RGK_ScalableVector
-                             : TargetTransformInfo::RGK_FixedWidthVector);
+bool LoopVectorizationCostModel::shouldCalculateRegPressureForVF(
+    ElementCount VF) {
+  if (!useMaxBandwidth(VF.isScalable()
+                           ? TargetTransformInfo::RGK_ScalableVector
+                           : TargetTransformInfo::RGK_FixedWidthVector))
+    return false;
+  // Only calculate register pressure for VFs enabled by MaxBandwidth.
+  return ElementCount::isKnownGT(
+      VF, VF.isScalable() ? MaxPermissibleVFWithoutMaxBW.ScalableVF
+                          : MaxPermissibleVFWithoutMaxBW.FixedVF);
 }
 
 bool LoopVectorizationCostModel::useMaxBandwidth(
@@ -3911,6 +3919,12 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
       ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
                            : TargetTransformInfo::RGK_FixedWidthVector;
   ElementCount MaxVF = MaxVectorElementCount;
+
+  if (MaxVF.isScalable())
+    MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
+  else
+    MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
+
   if (useMaxBandwidth(RegKind)) {
     auto MaxVectorElementCountMaxBW = ElementCount::get(
         llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
@@ -4264,9 +4278,11 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
       if (VF.isScalar())
         continue;
 
-      /// Don't consider the VF if it exceeds the number of registers for the
-      /// target.
-      if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI))
+      /// If the register pressure needs to be considered for VF,
+      /// don't consider the VF as valid if it exceeds the number
+      /// of registers for the target.
+      if (CM.shouldCalculateRegPressureForVF(VF) &&
+          RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs))
         continue;
 
       InstructionCost C = CM.expectedCost(VF);
@@ -7048,7 +7064,8 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
       InstructionCost Cost = cost(*P, VF);
       VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
 
-      if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI)) {
+      if (CM.shouldCalculateRegPressureForVF(VF) &&
+          RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs)) {
         LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
                           << VF << " because it uses too many registers\n");
         continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 66657b98b094b..3499e650ae853 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -405,9 +405,12 @@ static unsigned getVFScaleFactor(VPRecipeBase *R) {
   return 1;
 }
 
-bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI) const {
-  return any_of(MaxLocalUsers, [&TTI](auto &LU) {
-    return LU.second > TTI.getNumberOfRegisters(LU.first);
+bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI,
+                                        unsigned OverrideMaxNumRegs) const {
+  return any_of(MaxLocalUsers, [&TTI, &OverrideMaxNumRegs](auto &LU) {
+    return LU.second > (OverrideMaxNumRegs > 0
+                            ? OverrideMaxNumRegs
+                            : TTI.getNumberOfRegisters(LU.first));
   });
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 7bcf9dba8c311..cd86d27cf9122 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -85,8 +85,10 @@ struct VPRegisterUsage {
   SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
 
   /// Check if any of the tracked live intervals exceeds the number of
-  /// available registers for the target.
-  bool exceedsMaxNumRegs(const TargetTransformInfo &TTI) const;
+  /// available registers for the target. If non-zero, OverrideMaxNumRegs
+  /// is used in place of the target's number of registers.
+  bool exceedsMaxNumRegs(const TargetTransformInfo &TTI,
+                         unsigned OverrideMaxNumRegs = 0) const;
 };
 
 /// Estimate the register usage for \p Plan and vectorization factors in \p VFs
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
index 4153f0be611a1..9698f1a6768fd 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
@@ -231,3 +231,27 @@ define <vscale x 8 x i64> @sload_8i8_8i64(ptr %a) {
   %aext = sext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
   ret <vscale x 8 x i64> %aext
 }
+
+; Ensure we don't try to promote a predicate load to a sign-extended load.
+define <vscale x 16 x i8> @sload_16i1_16i8(ptr %addr) {
+; CHECK-LABEL: sload_16i1_16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr p0, [x0]
+; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    ret
+  %load = load <vscale x 16 x i1>, ptr %addr
+  %zext = sext <vscale x 16 x i1> %load to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %zext
+}
+
+; Ensure we don't try to promote a predicate load to a zero-extended load.
+define <vscale x 16 x i8> @zload_16i1_16i8(ptr %addr) {
+; CHECK-LABEL: zload_16i1_16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr p0, [x0]
+; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    ret
+  %load = load <vscale x 16 x i1>, ptr %addr
+  %zext = zext <vscale x 16 x i1> %load to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %zext
+}
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
index 6742ae6c1d584..f6465de86fa4f 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
@@ -6,17 +6,17 @@
 define amdgpu_kernel void @remat_constant_voids_spill(ptr addrspace(1) %p) #1 {
 ; GFX908-LABEL: remat_constant_voids_spill:
 ; GFX908:       ; %bb.0:
-; GFX908-NEXT:    v_accvgpr_write_b32 a1, 1
-; GFX908-NEXT:    v_accvgpr_write_b32 a5, 6
-; GFX908-NEXT:    v_accvgpr_write_b32 a6, 7
-; GFX908-NEXT:    v_accvgpr_write_b32 a7, 8
-; GFX908-NEXT:    v_accvgpr_write_b32 a0, 9
-; GFX908-NEXT:    v_accvgpr_write_b32 a2, 2
-; GFX908-NEXT:    v_accvgpr_write_b32 a3, 3
-; GFX908-NEXT:    v_accvgpr_write_b32 a4, 4
+; GFX908-NEXT:    v_accvgpr_write_b32 a0, 1
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, 2
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 3
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, 4
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_accvgpr_write_b32 a1, 5
+; GFX908-NEXT:    v_accvgpr_write_b32 a0, 5
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, 6
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 7
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, 8
+; GFX908-NEXT:    v_accvgpr_write_b32 a4, 9
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ;;#ASMEND
 ; GFX908-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
new file mode 100644
index 0000000000000..69bdb1f5066f0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
@@ -0,0 +1,95 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX90A %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX908 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx906 -passes=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefix=NO-AGPR %s
+
+--- |
+  define void @func() {
+    ret void
+  }
+
+  ; Attribute is ignored for gfx90a
+  define void @no_agprs() "amdgpu-agpr-alloc"="0,0" {
+    ret void
+  }
+
+...
+---
+name: func
+tracksRegLiveness: true
+stack:
+  - { id: 0, size: 4 }
+body:             |
+  ; HAS-AGPR-LABEL: name: func
+  ; HAS-AGPR: bb.0:
+  ; HAS-AGPR-NEXT:   successors: %bb.1(0x80000000)
+  ; HAS-AGPR-NEXT:   liveins: $vgpr0
+  ; HAS-AGPR-NEXT: {{  $}}
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_2:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 6, implicit $exec
+  ; HAS-AGPR-NEXT: {{  $}}
+  ; HAS-AGPR-NEXT: bb.1:
+  ; HAS-AGPR-NEXT:   [[AV_MOV_3:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 3, implicit $exec
+  ;
+  ; NO-AGPR-LABEL: name: func
+  ; NO-AGPR: bb.0:
+  ; NO-AGPR-NEXT:   successors: %bb.1(0x80000000)
+  ; NO-AGPR-NEXT:   liveins: $vgpr0
+  ; NO-AGPR-NEXT: {{  $}}
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_2:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec
+  ; NO-AGPR-NEXT: {{  $}}
+  ; NO-AGPR-NEXT: bb.1:
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  bb.0:
+    liveins: $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+    %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+    %4:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %5:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    %6:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec
+
+  bb.1:
+    %7:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+
+...
+
+---
+name: no_agprs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+    ; GFX90A-LABEL: name: no_agprs
+    ; GFX90A: liveins: $vgpr0
+    ; GFX90A-NEXT: {{  $}}
+    ; GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX90A-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    ;
+    ; GFX908-LABEL: name: no_agprs
+    ; GFX908: liveins: $vgpr0
+    ; GFX908-NEXT: {{  $}}
+    ; GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+    ; GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+    ;
+    ; NO-AGPR-LABEL: name: no_agprs
+    ; NO-AGPR: liveins: $vgpr0
+    ; NO-AGPR-NEXT: {{  $}}
+    ; NO-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    %0:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
new file mode 100644
index 0000000000000..1a457c94778fd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
@@ -0,0 +1,74 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+
+---
+name:            bad_ra
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sgpr_64, preferred-register: '$sgpr4_sgpr5' }
+  - { id: 1, class: sgpr_128, preferred-register: '%2' }
+  - { id: 2, class: areg_128, preferred-register: '%1' }
+  - { id: 3, class: areg_128, preferred-register: '%4' }
+  - { id: 4, class: av_128, preferred-register: '%3' }
+  - { id: 5, class: areg_128, preferred-register: '%6' }
+  - { id: 6, class: vreg_128, preferred-register: '%5' }
+  - { id: 7, class: areg_128, preferred-register: '%4' }
+  - { id: 8, class: vgpr_32 }
+  - { id: 9, class: vgpr_32 }
+  - { id: 10, class: vgpr_32 }
+  - { id: 11, class: areg_128 }
+liveins:
+  - { reg: '$sgpr4_sgpr5', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  vgprForAGPRCopy: '$vgpr255'
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  bb.0:
+    liveins: $sgpr4_sgpr5
+
+    ; CHECK-LABEL: name: bad_ra
+    ; CHECK: liveins: $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1)
+    ; CHECK-NEXT: renamable $vgpr4 = V_MOV_B32_e32 1065353216, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr5 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr6 = V_MOV_B32_e32 1073741824, implicit $exec
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr1 = COPY renamable $agpr1
+    ; CHECK-NEXT: renamable $vgpr0 = COPY renamable $agpr0
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr3 = COPY renamable $agpr1
+    ; CHECK-NEXT: renamable $vgpr2 = COPY killed renamable $agpr0
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 killed $vgpr4, killed $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr5, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1)
+    %8:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+    %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %10:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec
+    %2:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
+    %3:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %2, 0, 0, 0, implicit $mode, implicit $exec
+    undef %4.sub1:av_128 = COPY %3.sub1
+    %4.sub0:av_128 = COPY %3.sub0
+    %11:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %3, 0, 0, 0, implicit $mode, implicit $exec
+    %4.sub3:av_128 = COPY %11.sub1
+    %4.sub2:av_128 = COPY %11.sub0
+    %7:areg_128 = COPY %4
+    %5:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %7, 0, 0, 0, implicit $mode, implicit $exec
+    %6:vreg_128 = COPY %5
+    GLOBAL_STORE_DWORDX4_SADDR %9, %6, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index ae90cfb631e8d..7eb7d72e6cb97 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -25,7 +25,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr17, 8, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_CSELECT_B64 -1, 0, implicit killed $scc
   ; GFX90A-NEXT:   renamable $sgpr30_sgpr31 = S_XOR_B64 killed renamable $sgpr18_sgpr19, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = DS_READ_B32_gfx9 renamable $vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3)
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr28_sgpr29, implicit-def dead $scc
@@ -56,8 +56,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr30 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $vgpr15 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr17 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr15 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr17 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_VCCZ %bb.57, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.4.bb15:
@@ -112,14 +112,14 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   successors: %bb.7(0x80000000)
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr21 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr20 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr23 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr22 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr25 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr24 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr19 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr18 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr21 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr20 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr23 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr22 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr25 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr24 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.7.Flow19:
   ; GFX90A-NEXT:   successors: %bb.62(0x40000000), %bb.8(0x40000000)
@@ -671,7 +671,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr50_sgpr51, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr30, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.54, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
@@ -759,7 +759,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr53 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $sgpr64_sgpr65, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr10 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr14_vgpr15 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr21, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from %ir.7, addrspace 3)
@@ -801,12 +801,12 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr42_vgpr43 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr40_vgpr41 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr46_vgpr47 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr52 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr16 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr53 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr14 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr52 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr16 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr53 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr13 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr12 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_BRANCH %bb.7
   ; GFX90A-NEXT: {{  $}}
@@ -814,7 +814,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   successors: %bb.3(0x80000000)
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3)
@@ -913,7 +913,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr25, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr3 = V_OR_B32_e32 killed $vgpr11, killed $vgpr19, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U32_sdwa 0, killed $vgpr53, 0, $vgpr3, 0, 0, 6, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr10 = V_OR_B32_e32 killed $vgpr52, killed $vgpr13, implicit $exec
@@ -955,7 +955,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr26, $vgpr2, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr3, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr10, killed $vgpr3, killed $vcc, 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr27 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr27 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr49 = COPY renamable $vgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr35 = COPY renamable $vgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr39 = COPY renamable $vgpr27, implicit $exec
@@ -989,7 +989,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr10 = V_OR_B32_e32 $vgpr50, killed $vgpr16, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr54 = V_OR_B32_e32 killed $vgpr10, killed $vgpr14, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr55 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr55 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 killed renamable $vgpr55, renamable $vgpr54_vgpr55, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_BRANCH %bb.69
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 50fa7ac2a19aa..4f81d3599d1de 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -10,9 +10,9 @@
 
 ; GCN-O0: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(atomic-expand,verify,gc-lowering,lower-constant-intrinsics,unreachableblockelim,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(lower-switch,lower-invoke,unreachableblockelim,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa,require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,localstackalloc))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,phi-node-elimination,two-address-instruction,regallocfast,si-fix-vgpr-copies,remove-redundant-debug-values,fixup-statepoint-caller-saved,prolog-epilog,post-ra-pseudos,si-post-ra-bundler,fentry-insert,xray-instrumentation,patchable-function,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
-; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
+; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
-; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
+; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
 define void @empty() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index af3241e95e91d..2a5c65278f7dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -329,6 +329,7 @@
 ; GCN-O1-NEXT:        Remove dead machine instructions
 ; GCN-O1-NEXT:        SI Shrink Instructions
 ; GCN-O1-NEXT:        Register Usage Information Propagation
+; GCN-O1-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O1-NEXT:        Detect Dead Lanes
 ; GCN-O1-NEXT:        Remove dead machine instructions
 ; GCN-O1-NEXT:        Init Undef Pass
@@ -640,6 +641,7 @@
 ; GCN-O1-OPTS-NEXT:        Remove dead machine instructions
 ; GCN-O1-OPTS-NEXT:        SI Shrink Instructions
 ; GCN-O1-OPTS-NEXT:        Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O1-OPTS-NEXT:        Detect Dead Lanes
 ; GCN-O1-OPTS-NEXT:        Remove dead machine instructions
 ; GCN-O1-OPTS-NEXT:        Init Undef Pass
@@ -956,6 +958,7 @@
 ; GCN-O2-NEXT:        Remove dead machine instructions
 ; GCN-O2-NEXT:        SI Shrink Instructions
 ; GCN-O2-NEXT:        Register Usage Information Propagation
+; GCN-O2-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O2-NEXT:        Detect Dead Lanes
 ; GCN-O2-NEXT:        Remove dead machine instructions
 ; GCN-O2-NEXT:        Init Undef Pass
@@ -1286,6 +1289,7 @@
 ; GCN-O3-NEXT:        Remove dead machine instructions
 ; GCN-O3-NEXT:        SI Shrink Instructions
 ; GCN-O3-NEXT:        Register Usage Information Propagation
+; GCN-O3-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O3-NEXT:        Detect Dead Lanes
 ; GCN-O3-NEXT:        Remove dead machine instructions
 ; GCN-O3-NEXT:        Init Undef Pass
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 67ae05eb6f0b8..561eaca3b77df 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -4365,8 +4365,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
 ; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
 ; NOLIT-SRCC:       ; %bb.0: ; %bb
 ; NOLIT-SRCC-NEXT:    v_mov_b32_e32 v0, 1.0
-; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a0, 1.0
+; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a2, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a3, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a4, 0
@@ -4465,8 +4465,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
 ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
 ; LIT-SRCC:       ; %bb.0: ; %bb
 ; LIT-SRCC-NEXT:    v_mov_b32_e32 v0, 1.0
-; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a0, 1.0
+; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a2, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a3, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a4, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
index 2faf375a97a86..465414c5471ee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s
 
 declare i32 @llvm.amdgcn.prng.b32(i32) #0
 
@@ -29,4 +31,4 @@ define amdgpu_kernel void @prng_b32_constant_100(ptr addrspace(1) %out) #1 {
 
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind }
\ No newline at end of file
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
index 3844d6054e130..cf244f0b1f884 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
@@ -6,16 +6,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
 ; GFX942-LABEL: matmul_kernel:
 ; GFX942:       ; %bb.0: ; %entry
 ; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942-NEXT:    v_mov_b32_e32 v0, 0
-; GFX942-NEXT:    v_accvgpr_write_b32 a2, v0
+; GFX942-NEXT:    v_accvgpr_write_b32 a2, 0
 ; GFX942-NEXT:    s_mov_b32 s2, 0
 ; GFX942-NEXT:    v_accvgpr_write_b32 a1, 0
+; GFX942-NEXT:    s_mov_b32 s3, 0
 ; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX942-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX942-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
 ; GFX942-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX942-NEXT:    s_mov_b32 s3, 0
 ; GFX942-NEXT:    s_branch .LBB0_2
 ; GFX942-NEXT:  .LBB0_1: ; %bb2
 ; GFX942-NEXT:    ; in Loop: Header=BB0_2 Depth=1
@@ -43,16 +42,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
 ; GFX908-LABEL: matmul_kernel:
 ; GFX908:       ; %bb.0: ; %entry
 ; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX908-NEXT:    v_mov_b32_e32 v0, 0
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 0
 ; GFX908-NEXT:    v_accvgpr_write_b32 a1, 0
 ; GFX908-NEXT:    s_mov_b32 s2, 0
-; GFX908-NEXT:    v_accvgpr_write_b32 a2, v0
+; GFX908-NEXT:    s_mov_b32 s3, 0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX908-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX908-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX908-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
 ; GFX908-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX908-NEXT:    s_mov_b32 s3, 0
 ; GFX908-NEXT:    s_branch .LBB0_2
 ; GFX908-NEXT:  .LBB0_1: ; %bb2
 ; GFX908-NEXT:    ; in Loop: Header=BB0_2 Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
index ee5481617cf59..01506d0af1913 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
@@ -80,16 +80,16 @@ body:             |
   ; COALESCE-NEXT:   S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
   ; COALESCE-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0
   ; COALESCE-NEXT:   [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec
   ; COALESCE-NEXT:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.1:
   ; COALESCE-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; COALESCE-NEXT: {{  $}}
-  ; COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
   ; COALESCE-NEXT:   $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; COALESCE-NEXT:   S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
@@ -103,10 +103,10 @@ body:             |
   ; COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]]
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.3:
@@ -134,16 +134,16 @@ body:             |
   ; GFX908-COALESCE-NEXT:   S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
   ; GFX908-COALESCE-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0
   ; GFX908-COALESCE-NEXT:   [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.1:
   ; GFX908-COALESCE-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
   ; GFX908-COALESCE-NEXT:   $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; GFX908-COALESCE-NEXT:   S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
@@ -157,10 +157,10 @@ body:             |
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]]
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
index 49c0aaf9fb390..a9207de317ea1 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
@@ -67,7 +67,7 @@ body:             |
   ; COALESCE-NEXT: bb.1:
   ; COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; COALESCE-NEXT: {{  $}}
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   S_BRANCH %bb.3
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.2:
@@ -78,13 +78,13 @@ body:             |
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], 0, 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+  ; COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.3:
-  ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; COALESCE-NEXT:   [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec
   ; COALESCE-NEXT:   undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec
-  ; COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+  ; COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
@@ -105,28 +105,28 @@ body:             |
   ; GFX908-COALESCE-NEXT: bb.1:
   ; GFX908-COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   S_BRANCH %bb.3
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.2:
   ; GFX908-COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[AV_MOV_1]].sub0
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub0
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]].sub0_sub1
-  ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.3:
-  ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; GFX908-COALESCE-NEXT:   [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 663fd98b46bf7..ce96766116089 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -17,9 +17,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
@@ -42,8 +42,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX908-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
-  ; PEI-GFX908-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-  ; PEI-GFX908-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+  ; PEI-GFX908-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; PEI-GFX908-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
   ; PEI-GFX908-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
   ; PEI-GFX908-NEXT:   $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
@@ -62,9 +62,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX90A-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; REGALLOC-GFX90A-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; REGALLOC-GFX90A-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; REGALLOC-GFX90A-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64_align2, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64_align2, [[V_MFMA_I32_4X4X4I8_e64_]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   S_ENDPGM 0
@@ -85,8 +85,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX90A-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
-  ; PEI-GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-  ; PEI-GFX90A-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+  ; PEI-GFX90A-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; PEI-GFX90A-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
   ; PEI-GFX90A-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
   ; PEI-GFX90A-NEXT:   $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index 1ec94162951a6..d48bfe0bb7f21 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -145,49 +145,29 @@ entry:
 
 ; Test skipping the lower-32-bit addition if it is unnecessary.
 define ptr @huge_offset_low_32_unused(ptr %p) {
-; GFX942_PTRADD-LABEL: huge_offset_low_32_unused:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    s_mov_b32 s0, 0
-; GFX942_PTRADD-NEXT:    s_mov_b32 s1, 1
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: huge_offset_low_32_unused:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+; GFX942-LABEL: huge_offset_low_32_unused:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_add_u32_e32 v1, 1, v1
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr inbounds i8, ptr %p, i64 u0x100000000
   ret ptr %gep
 }
 
 ; Reassociate address computation if it leads to more scalar operations.
 define amdgpu_kernel void @reassoc_scalar_r(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
-; GFX942_PTRADD-LABEL: reassoc_scalar_r:
-; GFX942_PTRADD:       ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_PTRADD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], v[0:1], 0, s[6:7]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: reassoc_scalar_r:
-; GFX942_LEGACY:       ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    s_add_u32 s2, s2, s6
-; GFX942_LEGACY-NEXT:    s_addc_u32 s3, s3, s7
-; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: reassoc_scalar_r:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_u32 s2, s2, s6
+; GFX942-NEXT:    s_addc_u32 s3, s3, s7
+; GFX942-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
 entry:
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
@@ -198,30 +178,18 @@ entry:
 }
 
 define amdgpu_kernel void @reassoc_scalar_l(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
-; GFX942_PTRADD-LABEL: reassoc_scalar_l:
-; GFX942_PTRADD:       ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_PTRADD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[6:7], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: reassoc_scalar_l:
-; GFX942_LEGACY:       ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    s_add_u32 s2, s2, s6
-; GFX942_LEGACY-NEXT:    s_addc_u32 s3, s3, s7
-; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: reassoc_scalar_l:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_u32 s2, s2, s6
+; GFX942-NEXT:    s_addc_u32 s3, s3, s7
+; GFX942-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
 entry:
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
@@ -233,24 +201,14 @@ entry:
 
 ; Tests the target-specific (ptradd x, shl(0 - y, k)) -> sub(x, shl(y, k)) fold
 define ptr addrspace(1) @shl_neg_offset(ptr addrspace(1) %p, i64 %noffset, i64 %shift) {
-; GFX942_PTRADD-LABEL: shl_neg_offset:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_sub_co_u32_e32 v2, vcc, 0, v2
-; GFX942_PTRADD-NEXT:    s_nop 1
-; GFX942_PTRADD-NEXT:    v_subb_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: shl_neg_offset:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX942_LEGACY-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
-; GFX942_LEGACY-NEXT:    s_nop 1
-; GFX942_LEGACY-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+; GFX942-LABEL: shl_neg_offset:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %offset = sub i64 0, %noffset
   %x = shl i64 %offset, %shift
   %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 %x
@@ -268,10 +226,9 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
 ; GFX942_PTRADD:       ; %bb.0:
 ; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942_PTRADD-NEXT:    s_getpc_b64 s[0:1]
-; GFX942_PTRADD-NEXT:    s_add_u32 s0, s0, v0 at rel32@lo+4
-; GFX942_PTRADD-NEXT:    s_addc_u32 s1, s1, v0 at rel32@hi+12
+; GFX942_PTRADD-NEXT:    s_add_u32 s0, s0, v0 at rel32@lo+14
+; GFX942_PTRADD-NEXT:    s_addc_u32 s1, s1, v0 at rel32@hi+22
 ; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 10
 ; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942_LEGACY-LABEL: complextype_global_gep:
@@ -291,30 +248,63 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
 
 ; Tests the tryFoldToMad64_32 PTRADD combine.
 define amdgpu_kernel void @fold_mad64(ptr addrspace(1) %p) {
-; GFX942_PTRADD-LABEL: fold_mad64:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    v_mul_hi_u32_u24_e32 v1, 12, v0
-; GFX942_PTRADD-NEXT:    v_mul_u32_u24_e32 v0, 12, v0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, 1.0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    global_store_dword v[0:1], v2, off
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: fold_mad64:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, 1.0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dword v[0:1], v2, off
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: fold_mad64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 1.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
+; GFX942-NEXT:    global_store_dword v[0:1], v2, off
+; GFX942-NEXT:    s_endpgm
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %p, i64 %voffset, i32 0
   store float 1.0, ptr addrspace(1) %p1
   ret void
 }
+
+; Use non-zero shift amounts in v_lshl_add_u64.
+define ptr @select_v_lshl_add_u64(ptr %base, i64 %voffset) {
+; GFX942_PTRADD-LABEL: select_v_lshl_add_u64:
+; GFX942_PTRADD:       ; %bb.0:
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], 3, v[2:3]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: select_v_lshl_add_u64:
+; GFX942_LEGACY:       ; %bb.0:
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[2:3], 3, v[0:1]
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr inbounds i64, ptr %base, i64 %voffset
+  ret ptr %gep
+}
+
+; Fold mul and add into v_mad, even if amdgpu-codegenprepare-mul24 turned the
+; mul into a mul24.
+define ptr @fold_mul24_into_mad(ptr %base, i64 %a, i64 %b) {
+; GFX942_PTRADD-LABEL: fold_mul24_into_mad:
+; GFX942_PTRADD:       ; %bb.0:
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v2, 0xfffff, v2
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v4, 0xfffff, v4
+; GFX942_PTRADD-NEXT:    v_mul_hi_u32_u24_e32 v3, v2, v4
+; GFX942_PTRADD-NEXT:    v_mul_u32_u24_e32 v2, v2, v4
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: fold_mul24_into_mad:
+; GFX942_LEGACY:       ; %bb.0:
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v2, 0xfffff, v2
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v3, 0xfffff, v4
+; GFX942_LEGACY-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v2, v3, v[0:1]
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+  %a_masked = and i64 %a, u0xfffff
+  %b_masked = and i64 %b, u0xfffff
+  %mul = mul i64 %a_masked, %b_masked
+  %gep = getelementptr inbounds i8, ptr %base, i64 %mul
+  ret ptr %gep
+}
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index bd255e88b9512..648b59f69ea79 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -9,9 +9,9 @@ define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; GCN-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; GCN-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; GCN-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GCN-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; GCN-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; GCN-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; GCN-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 2228234 /* regdef:VGPR_32 */, def undef %14.sub0
   ; GCN-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
   ; GCN-NEXT:   GLOBAL_STORE_DWORDX4 undef %24:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
index d07e2914c753a..f7653af1fa9ba 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
@@ -122,23 +122,23 @@ define i64 @callee_large_scalars(i256 %a, i256 %b) nounwind {
 define i64 @caller_large_scalars() nounwind {
 ; CHECK-LABEL: caller_large_scalars:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $zero, $sp, 24
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $zero, $sp, 40
 ; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 8
+; CHECK-NEXT:    vst $vr0, $sp, 24
 ; CHECK-NEXT:    ori $a0, $zero, 2
-; CHECK-NEXT:    st.d $a0, $sp, 0
-; CHECK-NEXT:    st.d $zero, $sp, 56
-; CHECK-NEXT:    vst $vr0, $sp, 40
+; CHECK-NEXT:    st.d $a0, $sp, 16
+; CHECK-NEXT:    st.d $zero, $sp, 72
+; CHECK-NEXT:    vst $vr0, $sp, 56
 ; CHECK-NEXT:    ori $a2, $zero, 1
-; CHECK-NEXT:    addi.d $a0, $sp, 32
-; CHECK-NEXT:    addi.d $a1, $sp, 0
-; CHECK-NEXT:    st.d $a2, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 48
+; CHECK-NEXT:    addi.d $a1, $sp, 16
+; CHECK-NEXT:    st.d $a2, $sp, 48
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(callee_large_scalars)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %1 = call i64 @callee_large_scalars(i256 1, i256 2)
   ret i64 %1
@@ -177,20 +177,20 @@ define i64 @callee_large_scalars_exhausted_regs(i64 %a, i64 %b, i64 %c, i64 %d,
 define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; CHECK-LABEL: caller_large_scalars_exhausted_regs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -96
-; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $a0, $sp, 16
+; CHECK-NEXT:    addi.d $sp, $sp, -112
+; CHECK-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    st.d $a0, $sp, 8
 ; CHECK-NEXT:    ori $a0, $zero, 9
 ; CHECK-NEXT:    st.d $a0, $sp, 0
-; CHECK-NEXT:    st.d $zero, $sp, 40
+; CHECK-NEXT:    st.d $zero, $sp, 56
 ; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 24
+; CHECK-NEXT:    vst $vr0, $sp, 40
 ; CHECK-NEXT:    ori $a0, $zero, 10
-; CHECK-NEXT:    st.d $a0, $sp, 16
-; CHECK-NEXT:    st.d $zero, $sp, 72
+; CHECK-NEXT:    st.d $a0, $sp, 32
+; CHECK-NEXT:    st.d $zero, $sp, 88
 ; CHECK-NEXT:    ori $a0, $zero, 8
-; CHECK-NEXT:    st.d $a0, $sp, 48
+; CHECK-NEXT:    st.d $a0, $sp, 64
 ; CHECK-NEXT:    ori $a0, $zero, 1
 ; CHECK-NEXT:    ori $a1, $zero, 2
 ; CHECK-NEXT:    ori $a2, $zero, 3
@@ -198,12 +198,12 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; CHECK-NEXT:    ori $a4, $zero, 5
 ; CHECK-NEXT:    ori $a5, $zero, 6
 ; CHECK-NEXT:    ori $a6, $zero, 7
-; CHECK-NEXT:    addi.d $a7, $sp, 48
-; CHECK-NEXT:    vst $vr0, $sp, 56
+; CHECK-NEXT:    addi.d $a7, $sp, 64
+; CHECK-NEXT:    vst $vr0, $sp, 72
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(callee_large_scalars_exhausted_regs)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 96
+; CHECK-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 112
 ; CHECK-NEXT:    ret
   %1 = call i64 @callee_large_scalars_exhausted_regs(
       i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i256 8, i64 9,
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-half.ll b/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
index c88b67f13d1e7..da8c3e93f6842 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
@@ -1252,8 +1252,8 @@ define i32 @caller_half_on_stack() nounwind {
 ;
 ; LA64F-LP64S-LABEL: caller_half_on_stack:
 ; LA64F-LP64S:       # %bb.0:
-; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -80
-; LA64F-LP64S-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -96
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
 ; LA64F-LP64S-NEXT:    lu12i.w $a0, -12
 ; LA64F-LP64S-NEXT:    ori $a1, $a0, 3200
 ; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
@@ -1292,8 +1292,8 @@ define i32 @caller_half_on_stack() nounwind {
 ; LA64F-LP64S-NEXT:    st.w $t0, $sp, 0
 ; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
 ; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
-; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 80
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 96
 ; LA64F-LP64S-NEXT:    ret
 ;
 ; LA64F-LP64D-LABEL: caller_half_on_stack:
@@ -1336,8 +1336,8 @@ define i32 @caller_half_on_stack() nounwind {
 ;
 ; LA64D-LP64S-LABEL: caller_half_on_stack:
 ; LA64D-LP64S:       # %bb.0:
-; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -80
-; LA64D-LP64S-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -96
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
 ; LA64D-LP64S-NEXT:    lu12i.w $a0, -12
 ; LA64D-LP64S-NEXT:    ori $a1, $a0, 3200
 ; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
@@ -1376,8 +1376,8 @@ define i32 @caller_half_on_stack() nounwind {
 ; LA64D-LP64S-NEXT:    st.w $t0, $sp, 0
 ; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
 ; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
-; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 80
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 96
 ; LA64D-LP64S-NEXT:    ret
 ;
 ; LA64D-LP64D-LABEL: caller_half_on_stack:
diff --git a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
index 52d8dd05aaa4c..1a9de3b0ef3d1 100644
--- a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
+++ b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
@@ -14,41 +14,41 @@
 define dso_local noundef signext i32 @main() nounwind {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -272
-; CHECK-NEXT:    st.d $ra, $sp, 264 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -288
+; CHECK-NEXT:    st.d $ra, $sp, 280 # 8-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
 ; CHECK-NEXT:    xvld $xr0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    xvst $xr0, $sp, 96 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 112 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_1)
 ; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI0_1)
-; CHECK-NEXT:    xvst $xr1, $sp, 64 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr1, $sp, 80 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_2)
 ; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_2)
-; CHECK-NEXT:    xvst $xr2, $sp, 32 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr2, $sp, 48 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_3)
 ; CHECK-NEXT:    xvld $xr3, $a0, %pc_lo12(.LCPI0_3)
-; CHECK-NEXT:    xvst $xr3, $sp, 0 # 32-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 136
-; CHECK-NEXT:    xvst $xr1, $sp, 168
-; CHECK-NEXT:    xvst $xr2, $sp, 200
-; CHECK-NEXT:    xvst $xr3, $sp, 232
-; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    xvst $xr3, $sp, 16 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 152
+; CHECK-NEXT:    xvst $xr1, $sp, 184
+; CHECK-NEXT:    xvst $xr2, $sp, 216
+; CHECK-NEXT:    xvst $xr3, $sp, 248
+; CHECK-NEXT:    addi.d $a0, $sp, 152
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(foo)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 96 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 136
-; CHECK-NEXT:    xvld $xr0, $sp, 64 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 168
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 200
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 232
-; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    xvld $xr0, $sp, 112 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 152
+; CHECK-NEXT:    xvld $xr0, $sp, 80 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 184
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 216
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 248
+; CHECK-NEXT:    addi.d $a0, $sp, 152
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(bar)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    move $a0, $zero
-; CHECK-NEXT:    ld.d $ra, $sp, 264 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 272
+; CHECK-NEXT:    ld.d $ra, $sp, 280 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 288
 ; CHECK-NEXT:    ret
 entry:
   %s = alloca %struct.S, align 2
diff --git a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
index ccc5c703e71ed..15ac95dfc6c55 100644
--- a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
+++ b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
@@ -28,12 +28,12 @@ define void @func() {
 ; CHECK-NEXT:    ld.w $a3, $a1, 0
 ; CHECK-NEXT:    ld.w $a2, $a1, 0
 ; CHECK-NEXT:    ld.w $a0, $a1, 0
-; CHECK-NEXT:    st.d $fp, $sp, 0
+; CHECK-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
 ; CHECK-NEXT:    lu12i.w $fp, 1
 ; CHECK-NEXT:    ori $fp, $fp, 12
 ; CHECK-NEXT:    add.d $fp, $sp, $fp
 ; CHECK-NEXT:    st.w $t8, $fp, 0
-; CHECK-NEXT:    ld.d $fp, $sp, 0
+; CHECK-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
 ; CHECK-NEXT:    st.w $t8, $a1, 0
 ; CHECK-NEXT:    st.w $t7, $a1, 0
 ; CHECK-NEXT:    st.w $t6, $a1, 0
diff --git a/llvm/test/CodeGen/LoongArch/frame.ll b/llvm/test/CodeGen/LoongArch/frame.ll
index 048703029d8c6..b29d8634854f3 100644
--- a/llvm/test/CodeGen/LoongArch/frame.ll
+++ b/llvm/test/CodeGen/LoongArch/frame.ll
@@ -1,5 +1,6 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+d,-lsx < %s | FileCheck %s --check-prefixes=CHECK,NOLSX
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lsx < %s | FileCheck %s --check-prefixes=CHECK,LSX
 
 %struct.key_t = type { i32, [16 x i8] }
 
@@ -7,20 +8,35 @@ declare void @llvm.memset.p0.i64(ptr, i8, i64, i1)
 declare void @test1(ptr)
 
 define i32 @test() nounwind {
-; CHECK-LABEL: test:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -32
-; CHECK-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; CHECK-NEXT:    st.w $zero, $sp, 16
-; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 4
-; CHECK-NEXT:    pcaddu18i $ra, %call36(test1)
-; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    move $a0, $zero
-; CHECK-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 32
-; CHECK-NEXT:    ret
+; NOLSX-LABEL: test:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -32
+; NOLSX-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; NOLSX-NEXT:    st.w $zero, $sp, 16
+; NOLSX-NEXT:    st.d $zero, $sp, 8
+; NOLSX-NEXT:    st.d $zero, $sp, 0
+; NOLSX-NEXT:    addi.d $a0, $sp, 4
+; NOLSX-NEXT:    pcaddu18i $ra, %call36(test1)
+; NOLSX-NEXT:    jirl $ra, $ra, 0
+; NOLSX-NEXT:    move $a0, $zero
+; NOLSX-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; NOLSX-NEXT:    addi.d $sp, $sp, 32
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -32
+; LSX-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LSX-NEXT:    st.w $zero, $sp, 16
+; LSX-NEXT:    vrepli.b $vr0, 0
+; LSX-NEXT:    vst $vr0, $sp, 0
+; LSX-NEXT:    addi.d $a0, $sp, 4
+; LSX-NEXT:    pcaddu18i $ra, %call36(test1)
+; LSX-NEXT:    jirl $ra, $ra, 0
+; LSX-NEXT:    move $a0, $zero
+; LSX-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LSX-NEXT:    addi.d $sp, $sp, 32
+; LSX-NEXT:    ret
   %key = alloca %struct.key_t, align 4
   call void @llvm.memset.p0.i64(ptr %key, i8 0, i64 20, i1 false)
   %1 = getelementptr inbounds %struct.key_t, ptr %key, i64 0, i32 1, i64 0
@@ -98,3 +114,62 @@ define void @test_large_frame_size_1234576() "frame-pointer"="all" {
   %1 = alloca i8, i32 1234567
   ret void
 }
+
+;; Note: will create an emergency spill slot, if (!isInt<7>(StackSize)).
+;; Should involve only one SP-adjusting addi per adjustment.
+;; LSX 112 + 16(emergency solt) = 128
+define void @test_frame_size_112() {
+; NOLSX-LABEL: test_frame_size_112:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -112
+; NOLSX-NEXT:    .cfi_def_cfa_offset 112
+; NOLSX-NEXT:    addi.d $sp, $sp, 112
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_112:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -128
+; LSX-NEXT:    .cfi_def_cfa_offset 128
+; LSX-NEXT:    addi.d $sp, $sp, 128
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 112
+  ret void
+}
+
+;; LSX 128 + 16(emergency solt) = 144
+define void @test_frame_size_128() {
+; NOLSX-LABEL: test_frame_size_128:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -128
+; NOLSX-NEXT:    .cfi_def_cfa_offset 128
+; NOLSX-NEXT:    addi.d $sp, $sp, 128
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_128:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -144
+; LSX-NEXT:    .cfi_def_cfa_offset 144
+; LSX-NEXT:    addi.d $sp, $sp, 144
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 128
+  ret void
+}
+
+;; LSX 144 + 16(emergency solt) = 160
+define void @test_frame_size_144() {
+; NOLSX-LABEL: test_frame_size_144:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -144
+; NOLSX-NEXT:    .cfi_def_cfa_offset 144
+; NOLSX-NEXT:    addi.d $sp, $sp, 144
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_144:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -160
+; LSX-NEXT:    .cfi_def_cfa_offset 160
+; LSX-NEXT:    addi.d $sp, $sp, 160
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 144
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
index 402ddb9ad941b..5a55b253c77bb 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
@@ -6,11 +6,11 @@
 define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 dereferenceable(48) %b, i64 %i) {
 ; CHECK-LABEL: box:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -96
-; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    addi.d $sp, $sp, -112
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
 ; CHECK-NEXT:    slli.d $a2, $a1, 5
 ; CHECK-NEXT:    alsl.d $a1, $a1, $a2, 4
-; CHECK-NEXT:    addi.d $a2, $sp, 0
+; CHECK-NEXT:    addi.d $a2, $sp, 16
 ; CHECK-NEXT:    add.d $a3, $a2, $a1
 ; CHECK-NEXT:    vldx $vr0, $a1, $a2
 ; CHECK-NEXT:    vld $vr1, $a3, 32
@@ -18,7 +18,7 @@ define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 der
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    vst $vr1, $a0, 32
 ; CHECK-NEXT:    vst $vr2, $a0, 16
-; CHECK-NEXT:    addi.d $sp, $sp, 96
+; CHECK-NEXT:    addi.d $sp, $sp, 112
 ; CHECK-NEXT:    ret
   %1 = alloca [2 x %Box], align 16
   %2 = getelementptr inbounds [2 x %Box], ptr %1, i64 0, i64 %i
diff --git a/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll b/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
index 789b51d9b5e5b..9528280d181a3 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
@@ -6,10 +6,10 @@ declare <8 x float> @llvm.powi.v8f32.i32(<8 x float>, i32)
 define <8 x float> @powi_v8f32(<8 x float> %va, i32 %b) nounwind {
 ; CHECK-LABEL: powi_v8f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 0 # 32-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 16 # 32-byte Folded Spill
 ; CHECK-NEXT:    addi.w $fp, $a0, 0
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 0
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
@@ -18,79 +18,79 @@ define <8 x float> @powi_v8f32(<8 x float> %va, i32 %b) nounwind {
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 1
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 2
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 2
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 3
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 3
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 4
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 4
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 5
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 5
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 6
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 6
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 7
-; CHECK-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
 entry:
   %res = call <8 x float> @llvm.powi.v8f32.i32(<8 x float> %va, i32 %b)
@@ -102,10 +102,10 @@ declare <4 x double> @llvm.powi.v4f64.i32(<4 x double>, i32)
 define <4 x double> @powi_v4f64(<4 x double> %va, i32 %b) nounwind {
 ; CHECK-LABEL: powi_v4f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 0 # 32-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 16 # 32-byte Folded Spill
 ; CHECK-NEXT:    addi.w $fp, $a0, 0
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
@@ -114,39 +114,39 @@ define <4 x double> @powi_v4f64(<4 x double> %va, i32 %b) nounwind {
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 2
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 3
-; CHECK-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
 entry:
   %res = call <4 x double> @llvm.powi.v4f64.i32(<4 x double> %va, i32 %b)
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
index 04214f5dfa9d2..2e1618748688a 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
@@ -76,21 +76,21 @@ define void @extract_4xdouble(ptr %src, ptr %dst) nounwind {
 define void @extract_32xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_32xi8_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 0
 ; CHECK-NEXT:    ld.b $a0, $a0, 0
 ; CHECK-NEXT:    st.b $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <32 x i8>, ptr %src
   %e = extractelement <32 x i8> %v, i32 %idx
@@ -101,21 +101,21 @@ define void @extract_32xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_16xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_16xi16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 1
 ; CHECK-NEXT:    ld.h $a0, $a0, 0
 ; CHECK-NEXT:    st.h $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <16 x i16>, ptr %src
   %e = extractelement <16 x i16> %v, i32 %idx
@@ -126,21 +126,21 @@ define void @extract_16xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_8xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_8xi32_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    ld.w $a0, $a0, 0
 ; CHECK-NEXT:    st.w $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x i32>, ptr %src
   %e = extractelement <8 x i32> %v, i32 %idx
@@ -151,21 +151,21 @@ define void @extract_8xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_4xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_4xi64_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    ld.d $a0, $a0, 0
 ; CHECK-NEXT:    st.d $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x i64>, ptr %src
   %e = extractelement <4 x i64> %v, i32 %idx
@@ -176,21 +176,21 @@ define void @extract_4xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_8xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_8xfloat_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    fld.s $fa0, $a0, 0
 ; CHECK-NEXT:    fst.s $fa0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x float>, ptr %src
   %e = extractelement <8 x float> %v, i32 %idx
@@ -201,21 +201,21 @@ define void @extract_8xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_4xdouble_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_4xdouble_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    fld.d $fa0, $a0, 0
 ; CHECK-NEXT:    fst.d $fa0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x double>, ptr %src
   %e = extractelement <4 x double> %v, i32 %idx
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
index 3a4f6efd2c893..b24f95e676d10 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
@@ -114,22 +114,22 @@ define void @insert_4xdouble(ptr %src, ptr %dst, double %in) nounwind {
 define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_32xi8_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 0
 ; CHECK-NEXT:    st.b $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <32 x i8>, ptr %src
   %v_new = insertelement <32 x i8> %v, i8 %in, i32 %idx
@@ -140,22 +140,22 @@ define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
 define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_16xi16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 1
 ; CHECK-NEXT:    st.h $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <16 x i16>, ptr %src
   %v_new = insertelement <16 x i16> %v, i16 %in, i32 %idx
@@ -166,22 +166,22 @@ define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
 define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_8xi32_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 2
 ; CHECK-NEXT:    st.w $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x i32>, ptr %src
   %v_new = insertelement <8 x i32> %v, i32 %in, i32 %idx
@@ -192,22 +192,22 @@ define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
 define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_4xi64_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 3
 ; CHECK-NEXT:    st.d $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x i64>, ptr %src
   %v_new = insertelement <4 x i64> %v, i64 %in, i32 %idx
@@ -218,22 +218,22 @@ define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
 define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_8xfloat_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
-; CHECK-NEXT:    xvst $xr1, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr1, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    fst.s $fa0, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x float>, ptr %src
   %v_new = insertelement <8 x float> %v, float %in, i32 %idx
@@ -244,22 +244,22 @@ define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwin
 define void @insert_4xdouble_idx(ptr %src, ptr %dst, double %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_4xdouble_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
-; CHECK-NEXT:    xvst $xr1, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr1, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    fst.d $fa0, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x double>, ptr %src
   %v_new = insertelement <4 x double> %v, double %in, i32 %idx
diff --git a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
index ffedd7f9e9438..648c19d509715 100644
--- a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
+++ b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
@@ -347,42 +347,42 @@ define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    vpackev.w $vr0, $vr0, $vr1
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vpackev.w $vr1, $vr0, $vr1
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
   %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
   ret { <2 x float>, <2 x float> } %result
@@ -439,48 +439,48 @@ define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v3f32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -96
-; LA64-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -112
+; LA64-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 2
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 72
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 88
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 68
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 84
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 64
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 80
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 56
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 72
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 52
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 68
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 48
-; LA64-NEXT:    vld $vr0, $sp, 64
-; LA64-NEXT:    vld $vr1, $sp, 48
-; LA64-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 96
+; LA64-NEXT:    fst.s $fa0, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 80
+; LA64-NEXT:    vld $vr1, $sp, 64
+; LA64-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 112
 ; LA64-NEXT:    ret
   %result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a)
   ret { <3 x float>, <3 x float> } %result
@@ -568,44 +568,44 @@ define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f64:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
-; LA64-NEXT:    vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
 ; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.d $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 1
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 0
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
-; LA64-NEXT:    vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    vinsgr2vr.d $vr1, $a0, 1
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
   %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
   ret { <2 x double>, <2 x double> } %result
@@ -801,17 +801,17 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f128:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -80
-; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s6, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s7, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -96
+; LA64-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s1, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s2, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s3, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s4, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s5, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s6, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s7, $sp, 16 # 8-byte Folded Spill
 ; LA64-NEXT:    ld.d $fp, $a1, 16
 ; LA64-NEXT:    ld.d $s0, $a1, 24
 ; LA64-NEXT:    ld.d $s1, $a1, 0
@@ -847,17 +847,17 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
 ; LA64-NEXT:    st.d $s6, $s3, 16
 ; LA64-NEXT:    st.d $s5, $s3, 8
 ; LA64-NEXT:    st.d $s4, $s3, 0
-; LA64-NEXT:    ld.d $s7, $sp, 0 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s6, $sp, 8 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 80
+; LA64-NEXT:    ld.d $s7, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s6, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s5, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s4, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s3, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s2, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s1, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 96
 ; LA64-NEXT:    ret
   %result = call { <2 x fp128>, <2 x fp128> } @llvm.sincos.v2f128(<2 x fp128> %a)
   ret { <2 x fp128>, <2 x fp128> } %result
diff --git a/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll b/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
new file mode 100644
index 0000000000000..96159e5884d3f
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lsx --verify-machineinstrs < %s | FileCheck %s
+define void @eliminate_frame_index(<16 x i8> %a) nounwind {
+; CHECK-LABEL: eliminate_frame_index:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.d $sp, $sp, -240
+; CHECK-NEXT:    st.d $ra, $sp, 232 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 224 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s0, $sp, 216 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s1, $sp, 208 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s2, $sp, 200 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s3, $sp, 192 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s4, $sp, 184 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s5, $sp, 176 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s6, $sp, 168 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s7, $sp, 160 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s8, $sp, 152 # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $zero, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $ra, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $tp, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t8, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $fp, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s8, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    st.d $a0, $sp, 0 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    vstelm.b $vr0, $a0, 0, 0
+; CHECK-NEXT:    ld.d $a0, $sp, 0 # 8-byte Folded Reload
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $zero
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $ra
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $tp
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t8
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $fp
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s8
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    ld.d $s8, $sp, 152 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s7, $sp, 160 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s6, $sp, 168 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s5, $sp, 176 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s4, $sp, 184 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s3, $sp, 192 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s2, $sp, 200 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s1, $sp, 208 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s0, $sp, 216 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $fp, $sp, 224 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 232 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 240
+; CHECK-NEXT:    ret
+  %s = alloca [16 x i8]
+  %ss = alloca [128 x i8]
+
+  %zero =  call i64 asm sideeffect "addi.d $$zero, $$zero, 1", "={r0}"()
+  %ra =  call i64 asm sideeffect "addi.d $$ra, $$zero, 1", "={r1}"()
+  %tp =  call i64 asm sideeffect "addi.d $$tp, $$zero, 1", "={r2}"()
+  %a0 =  call i64 asm sideeffect "addi.d $$a0, $$zero, 1", "={r4}"()
+  %a1 =  call i64 asm sideeffect "addi.d $$a1, $$zero, 1", "={r5}"()
+  %a2 =  call i64 asm sideeffect "addi.d $$a2, $$zero, 1", "={r6}"()
+  %a3 =  call i64 asm sideeffect "addi.d $$a3, $$zero, 1", "={r7}"()
+  %a4 =  call i64 asm sideeffect "addi.d $$a4, $$zero, 1", "={r8}"()
+  %a5 =  call i64 asm sideeffect "addi.d $$a5, $$zero, 1", "={r9}"()
+  %a6 =  call i64 asm sideeffect "addi.d $$a6, $$zero, 1", "={r10}"()
+  %a7 =  call i64 asm sideeffect "addi.d $$a7, $$zero, 1", "={r11}"()
+  %t0 =  call i64 asm sideeffect "addi.d $$t0, $$zero, 1", "={r12}"()
+  %t1 =  call i64 asm sideeffect "addi.d $$t1, $$zero, 1", "={r13}"()
+  %t2 =  call i64 asm sideeffect "addi.d $$t2, $$zero, 1", "={r14}"()
+  %t3 =  call i64 asm sideeffect "addi.d $$t3, $$zero, 1", "={r15}"()
+  %t4 =  call i64 asm sideeffect "addi.d $$t4, $$zero, 1", "={r16}"()
+  %t5 =  call i64 asm sideeffect "addi.d $$t5, $$zero, 1", "={r17}"()
+  %t6 =  call i64 asm sideeffect "addi.d $$t6, $$zero, 1", "={r18}"()
+  %t7 =  call i64 asm sideeffect "addi.d $$t7, $$zero, 1", "={r19}"()
+  %t8 =  call i64 asm sideeffect "addi.d $$t8, $$zero, 1", "={r20}"()
+  ;; r21 Reserved (Non-allocatable)
+  %s9 =  call i64 asm sideeffect "addi.d $$s9, $$zero, 1", "={r22}"()
+  %s0 =  call i64 asm sideeffect "addi.d $$s0, $$zero, 1", "={r23}"()
+  %s1 =  call i64 asm sideeffect "addi.d $$s1, $$zero, 1", "={r24}"()
+  %s2 =  call i64 asm sideeffect "addi.d $$s2, $$zero, 1", "={r25}"()
+  %s3 =  call i64 asm sideeffect "addi.d $$s3, $$zero, 1", "={r26}"()
+  %s4 =  call i64 asm sideeffect "addi.d $$s4, $$zero, 1", "={r27}"()
+  %s5 =  call i64 asm sideeffect "addi.d $$s5, $$zero, 1", "={r28}"()
+  %s6 =  call i64 asm sideeffect "addi.d $$s6, $$zero, 1", "={r29}"()
+  %s7 =  call i64 asm sideeffect "addi.d $$s7, $$zero, 1", "={r30}"()
+  %s8 =  call i64 asm sideeffect "addi.d $$s8, $$zero, 1", "={r31}"()
+
+  %e = extractelement <16 x i8> %a, i64 0
+
+  store volatile i8 %e, ptr %s
+
+  call void asm sideeffect "# reg use $0", "{r0}"(i64 %zero)
+  call void asm sideeffect "# reg use $0", "{r1}"(i64 %ra)
+  call void asm sideeffect "# reg use $0", "{r2}"(i64 %tp)
+  call void asm sideeffect "# reg use $0", "{r4}"(i64 %a0)
+  call void asm sideeffect "# reg use $0", "{r5}"(i64 %a1)
+  call void asm sideeffect "# reg use $0", "{r6}"(i64 %a2)
+  call void asm sideeffect "# reg use $0", "{r7}"(i64 %a3)
+  call void asm sideeffect "# reg use $0", "{r8}"(i64 %a4)
+  call void asm sideeffect "# reg use $0", "{r9}"(i64 %a5)
+  call void asm sideeffect "# reg use $0", "{r10}"(i64 %a6)
+  call void asm sideeffect "# reg use $0", "{r11}"(i64 %a7)
+  call void asm sideeffect "# reg use $0", "{r12}"(i64 %t0)
+  call void asm sideeffect "# reg use $0", "{r13}"(i64 %t1)
+  call void asm sideeffect "# reg use $0", "{r14}"(i64 %t2)
+  call void asm sideeffect "# reg use $0", "{r15}"(i64 %t3)
+  call void asm sideeffect "# reg use $0", "{r16}"(i64 %t4)
+  call void asm sideeffect "# reg use $0", "{r17}"(i64 %t5)
+  call void asm sideeffect "# reg use $0", "{r18}"(i64 %t6)
+  call void asm sideeffect "# reg use $0", "{r19}"(i64 %t7)
+  call void asm sideeffect "# reg use $0", "{r20}"(i64 %t8)
+  ;; r21 Reserved (Non-allocatable)
+  call void asm sideeffect "# reg use $0", "{r22}"(i64 %s9)
+  call void asm sideeffect "# reg use $0", "{r23}"(i64 %s0)
+  call void asm sideeffect "# reg use $0", "{r24}"(i64 %s1)
+  call void asm sideeffect "# reg use $0", "{r25}"(i64 %s2)
+  call void asm sideeffect "# reg use $0", "{r26}"(i64 %s3)
+  call void asm sideeffect "# reg use $0", "{r27}"(i64 %s4)
+  call void asm sideeffect "# reg use $0", "{r28}"(i64 %s5)
+  call void asm sideeffect "# reg use $0", "{r29}"(i64 %s6)
+  call void asm sideeffect "# reg use $0", "{r30}"(i64 %s7)
+  call void asm sideeffect "# reg use $0", "{r31}"(i64 %s8)
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
index 9f15604fcca6b..69995a0721f8a 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
@@ -36,15 +36,15 @@ define void @caller(i32 %n) {
 ;
 ; LA64-LABEL: caller:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s8, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -128
+; LA64-NEXT:    .cfi_def_cfa_offset 128
+; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s8, $sp, 104 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
 ; LA64-NEXT:    .cfi_offset 31, -24
-; LA64-NEXT:    addi.d $fp, $sp, 64
+; LA64-NEXT:    addi.d $fp, $sp, 128
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 5, 0
 ; LA64-NEXT:    move $s8, $sp
@@ -54,14 +54,14 @@ define void @caller(i32 %n) {
 ; LA64-NEXT:    slli.d $a0, $a0, 4
 ; LA64-NEXT:    sub.d $a0, $sp, $a0
 ; LA64-NEXT:    move $sp, $a0
-; LA64-NEXT:    addi.d $a1, $s8, 0
+; LA64-NEXT:    addi.d $a1, $s8, 64
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -64
-; LA64-NEXT:    ld.d $s8, $sp, 40 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    addi.d $sp, $fp, -128
+; LA64-NEXT:    ld.d $s8, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 128
 ; LA64-NEXT:    ret
   %1 = alloca i8, i32 %n
   %2 = alloca i32, align 64
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment.ll b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
index 0645339358b64..0188884543adb 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
@@ -28,22 +28,22 @@ define void @caller32() {
 ;
 ; LA64-LABEL: caller32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -32
-; LA64-NEXT:    .cfi_def_cfa_offset 32
-; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -64
+; LA64-NEXT:    .cfi_def_cfa_offset 64
+; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 32
+; LA64-NEXT:    addi.d $fp, $sp, 64
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 4, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 32
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -32
-; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    addi.d $sp, $fp, -64
+; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 64
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 32
   call void @callee(ptr %1)
@@ -102,22 +102,22 @@ define void @caller64() {
 ;
 ; LA64-LABEL: caller64:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -128
+; LA64-NEXT:    .cfi_def_cfa_offset 128
+; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 64
+; LA64-NEXT:    addi.d $fp, $sp, 128
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 5, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 64
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -64
-; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    addi.d $sp, $fp, -128
+; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 128
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 64
   call void @callee(ptr %1)
@@ -176,22 +176,22 @@ define void @caller128() {
 ;
 ; LA64-LABEL: caller128:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -128
-; LA64-NEXT:    .cfi_def_cfa_offset 128
-; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -256
+; LA64-NEXT:    .cfi_def_cfa_offset 256
+; LA64-NEXT:    st.d $ra, $sp, 248 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 240 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 128
+; LA64-NEXT:    addi.d $fp, $sp, 256
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 6, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 128
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -128
-; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 128
+; LA64-NEXT:    addi.d $sp, $fp, -256
+; LA64-NEXT:    ld.d $fp, $sp, 240 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 248 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 256
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 128
   call void @callee(ptr %1)
@@ -250,22 +250,22 @@ define void @caller256() {
 ;
 ; LA64-LABEL: caller256:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -256
-; LA64-NEXT:    .cfi_def_cfa_offset 256
-; LA64-NEXT:    st.d $ra, $sp, 248 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 240 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -512
+; LA64-NEXT:    .cfi_def_cfa_offset 512
+; LA64-NEXT:    st.d $ra, $sp, 504 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 496 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 256
+; LA64-NEXT:    addi.d $fp, $sp, 512
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 7, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 256
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -256
-; LA64-NEXT:    ld.d $fp, $sp, 240 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 248 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 256
+; LA64-NEXT:    addi.d $sp, $fp, -512
+; LA64-NEXT:    ld.d $fp, $sp, 496 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 504 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 512
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 256
   call void @callee(ptr %1)
diff --git a/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll b/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
index 925fdf3d60646..0d441e66a0c84 100644
--- a/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
+++ b/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
@@ -121,19 +121,19 @@ define void @t3() {
 ;
 ; LA64-LABEL: t3:
 ; LA64:       # %bb.0: # %entry
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    .cfi_def_cfa_offset 80
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(.L.str)
 ; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(.L.str)
 ; LA64-NEXT:    ld.h $a1, $a0, 20
 ; LA64-NEXT:    ld.w $a2, $a0, 16
 ; LA64-NEXT:    ld.d $a3, $a0, 8
 ; LA64-NEXT:    ld.d $a0, $a0, 0
-; LA64-NEXT:    st.h $a1, $sp, 20
-; LA64-NEXT:    st.w $a2, $sp, 16
-; LA64-NEXT:    st.d $a3, $sp, 8
-; LA64-NEXT:    st.d $a0, $sp, 0
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    st.h $a1, $sp, 36
+; LA64-NEXT:    st.w $a2, $sp, 32
+; LA64-NEXT:    st.d $a3, $sp, 24
+; LA64-NEXT:    st.d $a0, $sp, 16
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
 entry:
   %msgbuf = alloca [64 x i8], align 1
diff --git a/llvm/test/CodeGen/LoongArch/vararg.ll b/llvm/test/CodeGen/LoongArch/vararg.ll
index 939cd2015c5b1..bc4b8a77c7e15 100644
--- a/llvm/test/CodeGen/LoongArch/vararg.ll
+++ b/llvm/test/CodeGen/LoongArch/vararg.ll
@@ -47,7 +47,7 @@ define i64 @va1(ptr %fmt, ...) {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a1, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a1, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $a1, $fp, -32
 ; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
@@ -94,7 +94,7 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a1, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a1, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $a1, $fp, -32
 ; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
@@ -112,11 +112,11 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind {
 define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-LABEL: va1_va_arg_alloca:
 ; LA64-FPELIM:       # %bb.0:
-; LA64-FPELIM-NEXT:    addi.d $sp, $sp, -96
-; LA64-FPELIM-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    addi.d $fp, $sp, 32
+; LA64-FPELIM-NEXT:    addi.d $sp, $sp, -112
+; LA64-FPELIM-NEXT:    st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    addi.d $fp, $sp, 48
 ; LA64-FPELIM-NEXT:    move $s0, $a1
 ; LA64-FPELIM-NEXT:    st.d $a7, $fp, 56
 ; LA64-FPELIM-NEXT:    st.d $a6, $fp, 48
@@ -126,7 +126,7 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-NEXT:    st.d $a2, $fp, 16
 ; LA64-FPELIM-NEXT:    st.d $a1, $fp, 8
 ; LA64-FPELIM-NEXT:    addi.d $a0, $fp, 16
-; LA64-FPELIM-NEXT:    st.d $a0, $fp, -32
+; LA64-FPELIM-NEXT:    st.d $a0, $fp, -40
 ; LA64-FPELIM-NEXT:    addi.d $a0, $a1, 15
 ; LA64-FPELIM-NEXT:    bstrins.d $a0, $zero, 3, 0
 ; LA64-FPELIM-NEXT:    sub.d $a0, $sp, $a0
@@ -134,20 +134,20 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-NEXT:    pcaddu18i $ra, %call36(notdead)
 ; LA64-FPELIM-NEXT:    jirl $ra, $ra, 0
 ; LA64-FPELIM-NEXT:    move $a0, $s0
-; LA64-FPELIM-NEXT:    addi.d $sp, $fp, -32
-; LA64-FPELIM-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    addi.d $sp, $sp, 96
+; LA64-FPELIM-NEXT:    addi.d $sp, $fp, -48
+; LA64-FPELIM-NEXT:    ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    addi.d $sp, $sp, 112
 ; LA64-FPELIM-NEXT:    ret
 ;
 ; LA64-WITHFP-LABEL: va1_va_arg_alloca:
 ; LA64-WITHFP:       # %bb.0:
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -96
-; LA64-WITHFP-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 32
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -112
+; LA64-WITHFP-NEXT:    st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 48
 ; LA64-WITHFP-NEXT:    move $s0, $a1
 ; LA64-WITHFP-NEXT:    st.d $a7, $fp, 56
 ; LA64-WITHFP-NEXT:    st.d $a6, $fp, 48
@@ -157,7 +157,7 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a0, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a0, $fp, -32
+; LA64-WITHFP-NEXT:    st.d $a0, $fp, -40
 ; LA64-WITHFP-NEXT:    addi.d $a0, $a1, 15
 ; LA64-WITHFP-NEXT:    bstrins.d $a0, $zero, 3, 0
 ; LA64-WITHFP-NEXT:    sub.d $a0, $sp, $a0
@@ -165,11 +165,11 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    pcaddu18i $ra, %call36(notdead)
 ; LA64-WITHFP-NEXT:    jirl $ra, $ra, 0
 ; LA64-WITHFP-NEXT:    move $a0, $s0
-; LA64-WITHFP-NEXT:    addi.d $sp, $fp, -32
-; LA64-WITHFP-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
+; LA64-WITHFP-NEXT:    addi.d $sp, $fp, -48
+; LA64-WITHFP-NEXT:    ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 112
 ; LA64-WITHFP-NEXT:    ret
   %va = alloca ptr, align 8
   call void @llvm.va_start(ptr %va)
@@ -314,10 +314,10 @@ define void @va_aligned_stack_caller() nounwind {
 ;
 ; LA64-WITHFP-LABEL: va_aligned_stack_caller:
 ; LA64-WITHFP:       # %bb.0:
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -112
-; LA64-WITHFP-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $fp, $sp, 96 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 112
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -128
+; LA64-WITHFP-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 128
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 17
 ; LA64-WITHFP-NEXT:    st.d $a0, $sp, 48
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 16
@@ -336,23 +336,23 @@ define void @va_aligned_stack_caller() nounwind {
 ; LA64-WITHFP-NEXT:    lu32i.d $a0, 335544
 ; LA64-WITHFP-NEXT:    lu52i.d $a0, $a0, -328
 ; LA64-WITHFP-NEXT:    st.d $a0, $sp, 16
-; LA64-WITHFP-NEXT:    st.d $zero, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $zero, $fp, -40
 ; LA64-WITHFP-NEXT:    vrepli.b $vr0, 0
-; LA64-WITHFP-NEXT:    vst $vr0, $fp, -40
+; LA64-WITHFP-NEXT:    vst $vr0, $fp, -56
 ; LA64-WITHFP-NEXT:    ori $a5, $zero, 1000
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 1
 ; LA64-WITHFP-NEXT:    ori $a1, $zero, 11
-; LA64-WITHFP-NEXT:    addi.d $a2, $fp, -48
+; LA64-WITHFP-NEXT:    addi.d $a2, $fp, -64
 ; LA64-WITHFP-NEXT:    ori $a3, $zero, 12
 ; LA64-WITHFP-NEXT:    ori $a4, $zero, 13
 ; LA64-WITHFP-NEXT:    ori $a7, $zero, 1
-; LA64-WITHFP-NEXT:    st.d $a5, $fp, -48
+; LA64-WITHFP-NEXT:    st.d $a5, $fp, -64
 ; LA64-WITHFP-NEXT:    move $a6, $zero
 ; LA64-WITHFP-NEXT:    pcaddu18i $ra, %call36(va_aligned_stack_callee)
 ; LA64-WITHFP-NEXT:    jirl $ra, $ra, 0
-; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 96 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 112
+; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 128
 ; LA64-WITHFP-NEXT:    ret
   %1 = call i32 (i32, ...) @va_aligned_stack_callee(i32 1, i32 11,
     i256 1000, i32 12, i32 13, i128 18446744073709551616, i32 14,
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index cbc9f700b1f01..aba20e6b0f27f 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -2311,4 +2311,51 @@ entry:
   ret void
 }
 
+define <4 x float> @test_uitofp_v4i8(<4 x i8> %a) {
+; CHECK-LABEL: test_uitofp_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_uitofp_v4i8_param_0];
+; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 0x7773U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r3, %r2;
+; CHECK-NEXT:    prmt.b32 %r4, %r1, 0, 0x7772U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r5, %r4;
+; CHECK-NEXT:    prmt.b32 %r6, %r1, 0, 0x7771U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r7, %r6;
+; CHECK-NEXT:    prmt.b32 %r8, %r1, 0, 0x7770U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r9, %r8;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r7, %r5, %r3};
+; CHECK-NEXT:    ret;
+  %r = uitofp <4 x i8> %a to <4 x float>
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_sitofp_v4i8(<4 x i8> %a) {
+; CHECK-LABEL: test_sitofp_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sitofp_v4i8_param_0];
+; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 0xbbb3U;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r3, %rs1;
+; CHECK-NEXT:    prmt.b32 %r4, %r1, 0, 0xaaa2U;
+; CHECK-NEXT:    cvt.u16.u32 %rs2, %r4;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r5, %rs2;
+; CHECK-NEXT:    prmt.b32 %r6, %r1, 0, 0x9991U;
+; CHECK-NEXT:    cvt.u16.u32 %rs3, %r6;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r7, %rs3;
+; CHECK-NEXT:    prmt.b32 %r8, %r1, 0, 0x8880U;
+; CHECK-NEXT:    cvt.u16.u32 %rs4, %r8;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r9, %rs4;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r7, %r5, %r3};
+; CHECK-NEXT:    ret;
+  %r = sitofp <4 x i8> %a to <4 x float>
+  ret <4 x float> %r
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/NVPTX/trunc-tofp.ll b/llvm/test/CodeGen/NVPTX/trunc-tofp.ll
new file mode 100644
index 0000000000000..404c423cc026a
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/trunc-tofp.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_80 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mcpu=sm_80 | %ptxas-verify %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define float @uitofp_trunc_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: uitofp_trunc_nuw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [uitofp_trunc_nuw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [uitofp_trunc_nuw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r4, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+  %v = add i32 %x, %y
+  %t = trunc nuw i32 %v to i16
+  %f = uitofp i16 %t to float
+  ret float %f
+}
+
+define float @sitofp_trunc_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: sitofp_trunc_nsw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sitofp_trunc_nsw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [sitofp_trunc_nsw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.s32 %r4, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+  %v = add i32 %x, %y
+  %t = trunc nsw i32 %v to i16
+  %f = sitofp i16 %t to float
+  ret float %f
+}
+
+define float @uitofp_trunc_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: uitofp_trunc_nsw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [uitofp_trunc_nsw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [uitofp_trunc_nsw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
+; CHECK-NEXT:    cvt.rn.f32.u16 %r4, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+    %v = add i32 %x, %y
+    %t = trunc nsw i32 %v to i16
+    %f = uitofp i16 %t to float
+    ret float %f
+}
+
+define float @sitofp_trunc_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: sitofp_trunc_nuw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sitofp_trunc_nuw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [sitofp_trunc_nuw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r4, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+    %v = add i32 %x, %y
+    %t = trunc nuw i32 %v to i16
+    %f = sitofp i16 %t to float
+    ret float %f
+}
diff --git a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
index 061435c45ad0e..59a702ab6b17f 100644
--- a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
+++ b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
@@ -798,12 +798,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccor:
 ; RV64SFBSIFIVEU74:       # %bb.0: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
-; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s0, a3
+; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
 ; RV64SFBSIFIVEU74-NEXT:    mv s1, a2
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB15_4
 ; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    or s0, a3, a1
@@ -824,11 +824,11 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV64SFBANDESAX45-LABEL: sextw_removal_ccor:
 ; RV64SFBANDESAX45:       # %bb.0: # %bb
 ; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
-; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s0, a3
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s1, a2
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB15_2
 ; RV64SFBANDESAX45-NEXT:  # %bb.1:
 ; RV64SFBANDESAX45-NEXT:    or s0, s0, a1
@@ -848,12 +848,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; ZICOND-LABEL: sextw_removal_ccor:
 ; ZICOND:       # %bb.0: # %bb
 ; ZICOND-NEXT:    addi sp, sp, -32
-; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s0, a3
+; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    andi a0, a0, 1
 ; ZICOND-NEXT:    mv s1, a2
+; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    beqz a0, .LBB15_4
 ; ZICOND-NEXT:  # %bb.3: # %bb
 ; ZICOND-NEXT:    or s0, a3, a1
@@ -874,12 +874,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV32SFB-LABEL: sextw_removal_ccor:
 ; RV32SFB:       # %bb.0: # %bb
 ; RV32SFB-NEXT:    addi sp, sp, -16
-; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
-; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s0, a3
+; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    andi a0, a0, 1
 ; RV32SFB-NEXT:    mv s1, a2
+; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    beqz a0, .LBB15_4
 ; RV32SFB-NEXT:  # %bb.3: # %bb
 ; RV32SFB-NEXT:    or s0, a3, a1
@@ -941,11 +941,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccaddw:
 ; RV64SFBSIFIVEU74:       # %bb.0: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
-; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s1, a1
 ; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
+; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s0, a2
 ; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB16_4
 ; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
@@ -967,11 +967,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV64SFBANDESAX45-LABEL: sextw_removal_ccaddw:
 ; RV64SFBANDESAX45:       # %bb.0: # %bb
 ; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
-; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s0, a2
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s1, a1
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB16_2
 ; RV64SFBANDESAX45-NEXT:  # %bb.1:
 ; RV64SFBANDESAX45-NEXT:    addw s1, s1, a3
@@ -991,11 +991,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; ZICOND-LABEL: sextw_removal_ccaddw:
 ; ZICOND:       # %bb.0: # %bb
 ; ZICOND-NEXT:    addi sp, sp, -32
-; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s1, a1
 ; ZICOND-NEXT:    andi a0, a0, 1
+; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s0, a2
 ; ZICOND-NEXT:    beqz a0, .LBB16_4
 ; ZICOND-NEXT:  # %bb.3: # %bb
@@ -1017,11 +1017,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV32SFB-LABEL: sextw_removal_ccaddw:
 ; RV32SFB:       # %bb.0: # %bb
 ; RV32SFB-NEXT:    addi sp, sp, -16
-; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s1, a1
 ; RV32SFB-NEXT:    andi a0, a0, 1
+; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s0, a2
 ; RV32SFB-NEXT:    beqz a0, .LBB16_4
 ; RV32SFB-NEXT:  # %bb.3: # %bb
diff --git a/llvm/test/CodeGen/RISCV/zdinx-spill.ll b/llvm/test/CodeGen/RISCV/zdinx-spill.ll
index d7a700622bf8c..6f206fe571c17 100644
--- a/llvm/test/CodeGen/RISCV/zdinx-spill.ll
+++ b/llvm/test/CodeGen/RISCV/zdinx-spill.ll
@@ -9,7 +9,6 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   liveins: $x10, $x11, $x8, $x9, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $x2 = frame-setup ADDI $x2, -64
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION def_cfa_offset 64
   ; CHECK-NEXT:   frame-setup SW killed $x8, $x2, 60 :: (store (s32) into %stack.1)
   ; CHECK-NEXT:   frame-setup SW killed $x9, $x2, 56 :: (store (s32) into %stack.2)
   ; CHECK-NEXT:   frame-setup SW killed $x18, $x2, 52 :: (store (s32) into %stack.3)
@@ -22,18 +21,6 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   frame-setup SW killed $x25, $x2, 24 :: (store (s32) into %stack.10)
   ; CHECK-NEXT:   frame-setup SW killed $x26, $x2, 20 :: (store (s32) into %stack.11)
   ; CHECK-NEXT:   frame-setup SW killed $x27, $x2, 16 :: (store (s32) into %stack.12)
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x8, -4
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x9, -8
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x18, -12
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x19, -16
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x20, -20
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x21, -24
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x22, -28
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x23, -32
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x24, -36
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x25, -40
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x26, -44
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x27, -48
   ; CHECK-NEXT:   renamable $x10_x11 = nofpexcept FADD_D_IN32X killed renamable $x10_x11, renamable $x10_x11, 7, implicit $frm
   ; CHECK-NEXT:   PseudoRV32ZdinxSD killed renamable $x10_x11, $x2, 8 :: (store (s64) into %stack.0, align 4)
   ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 12 /* clobber */, implicit-def dead early-clobber $x6, 12 /* clobber */, implicit-def dead early-clobber $x7, 12 /* clobber */, implicit-def dead early-clobber $x8, 12 /* clobber */, implicit-def dead early-clobber $x9, 12 /* clobber */, implicit-def dead early-clobber $x10, 12 /* clobber */, implicit-def dead early-clobber $x11, 12 /* clobber */, implicit-def dead early-clobber $x12, 12 /* clobber */, implicit-def dead early-clobber $x13, 12 /* clobber */, implicit-def dead early-clobber $x14, 12 /* clobber */, implicit-def dead early-clobber $x15, 12 /* clobber */, implicit-def dead early-clobber $x16, 12 /* clobber */, implicit-def dead early-clobber $x17, 12 /* clobber */, implicit-def dead early-clobber $x18, 12 /* clobber */, implicit-def dead early-clobber $x19, 12 /* clobber */, implicit-def dead early-clobber $x20, 12 /* clobber */, implicit-def dead early-clobber $x21, 12 /* clobber */, implicit-def dead early-clobber $x22, 12 /* clobber */, implicit-def dead early-clobber $x23, 12 /* clobber */, implicit-def dead early-clobber $x24, 12 /* clobber */, implicit-def dead early-clobber $x25, 12 /* clobber */, implicit-def dead early-clobber $x26, 12 /* clobber */, implicit-def dead early-clobber $x27, 12 /* clobber */, implicit-def dead early-clobber $x28, 12 /* clobber */, implicit-def dead early-clobber $x29, 12 /* clobber */, implicit-def dead early-clobber $x31
@@ -50,20 +37,7 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   $x25 = frame-destroy LW $x2, 24 :: (load (s32) from %stack.10)
   ; CHECK-NEXT:   $x26 = frame-destroy LW $x2, 20 :: (load (s32) from %stack.11)
   ; CHECK-NEXT:   $x27 = frame-destroy LW $x2, 16 :: (load (s32) from %stack.12)
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x8
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x9
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x18
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x19
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x20
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x21
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x22
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x23
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x24
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x25
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x26
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x27
   ; CHECK-NEXT:   $x2 = frame-destroy ADDI $x2, 64
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION def_cfa_offset 0
   ; CHECK-NEXT:   PseudoRET implicit $x10, implicit $x11
   %a = fadd double %x, %x
   call void asm sideeffect "", "~{x6},~{x7},~{x8},~{x9},~{x10},~{x11},~{x12},~{x13},~{x14},~{x15},~{x16},~{x17},~{x18},~{x19},~{x20},~{x21},~{x22},~{x23},~{x24},~{x25},~{x26},~{x27},~{x28},~{x29},~{xr0},~{x31}"()
diff --git a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
index 1d073cdc3bdb9..a878dbe94d11d 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; Test handling of llvm.lifetime intrinsics.
 ; RUN: opt < %s -passes=asan -asan-use-after-scope -asan-use-after-return=never -asan-use-stack-safety=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT
 ; RUN: opt < %s -passes=asan -asan-use-after-scope -asan-use-after-return=never -asan-use-stack-safety=0 -asan-instrument-dynamic-allocas=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK-NO-DYNAMIC
@@ -8,109 +9,389 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
 
-; CHECK-LABEL: define void @lifetime_no_size(
 define void @lifetime_no_size(i64 %i) sanitize_address {
+; CHECK-LABEL: define void @lifetime_no_size(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @lifetime_no_size to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i64 -868083117767659023, ptr [[TMP11]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP2]])
+; CHECK-NEXT:    [[AI:%.*]] = getelementptr inbounds [2 x i32], ptr [[TMP2]], i64 0, i64 [[I]]
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[AI]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP13]], 2147450880
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, ptr [[TMP15]], align 1
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i8 [[TMP16]], 0
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[BB18:.*]], label %[[BB23:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i8
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp sge i8 [[TMP20]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB23]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    call void @__asan_report_store1(i64 [[TMP12]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    store volatile i8 0, ptr [[AI]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[TMP2]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-NEXT:    store i64 0, ptr [[TMP25]], align 1
+; CHECK-NEXT:    ret void
+;
 entry:
   %a = alloca [2 x i32], align 4
 
   ; Poison memory in prologue: 0xf3f3f300f1f1f1f1
-  ; CHECK: store i64 -868083117767659023, ptr %[[#]]
 
   call void @llvm.lifetime.start.p0(i64 -1, ptr %a)
   ; Check that lifetime with no size are ignored.
-  ; CHECK-NOT: store
-  ; CHECK: call void @llvm.lifetime.start
 
   %ai = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 %i
   store volatile i8 0, ptr %ai, align 4
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 -1, ptr %a)
   ; Check that lifetime with no size are ignored.
-  ; CHECK-NOT: store
-  ; CHECK: call void @llvm.lifetime.end
 
   ; Unpoison stack frame on exit.
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK: ret void
   ret void
 }
 
 ; Generic case of lifetime analysis.
 define void @lifetime() sanitize_address {
-  ; CHECK-LABEL: define void @lifetime()
+; CHECK-DEFAULT-LABEL: define void @lifetime(
+; CHECK-DEFAULT-SAME: ) #[[ATTR1]] {
+; CHECK-DEFAULT-NEXT:    [[TMP1:%.*]] = alloca i64, align 32
+; CHECK-DEFAULT-NEXT:    store i64 0, ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-DEFAULT-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 32
+; CHECK-DEFAULT-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 1102416563, ptr [[TMP5]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP6:%.*]] = add i64 [[TMP2]], 8
+; CHECK-DEFAULT-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP7]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP8:%.*]] = add i64 [[TMP2]], 16
+; CHECK-DEFAULT-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP9]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP10:%.*]] = lshr i64 [[TMP2]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 0
+; CHECK-DEFAULT-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 -868082052615769615, ptr [[TMP13]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 4, ptr [[TMP15]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 3, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP16]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB27:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB22]]:
+; CHECK-DEFAULT-NEXT:    [[TMP23:%.*]] = and i64 [[TMP16]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP25:%.*]] = icmp sge i8 [[TMP24]], [[TMP20]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP25]], label %[[BB26:.*]], label %[[BB27]]
+; CHECK-DEFAULT:       [[BB26]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP16]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB27]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP4]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP28:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP29]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP30:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP31:%.*]] = inttoptr i64 [[TMP30]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP31]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP32:%.*]] = alloca i8, i64 128, align 32
+; CHECK-DEFAULT-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP32]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP34:%.*]] = add i64 [[TMP33]], 32
+; CHECK-DEFAULT-NEXT:    call void @__asan_alloca_poison(i64 [[TMP34]], i64 40)
+; CHECK-DEFAULT-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP32]] to i64
+; CHECK-DEFAULT-NEXT:    store i64 [[TMP35]], ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP36:%.*]] = inttoptr i64 [[TMP34]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP37:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    call void @__asan_unpoison_stack_memory(i64 [[TMP37]], i64 40)
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[TMP36]])
+; CHECK-DEFAULT-NEXT:    [[TMP38:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP39:%.*]] = lshr i64 [[TMP38]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP40:%.*]] = add i64 [[TMP39]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP41:%.*]] = inttoptr i64 [[TMP40]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP42:%.*]] = load i8, ptr [[TMP41]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP43:%.*]] = icmp ne i8 [[TMP42]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP43]], label %[[BB44:.*]], label %[[BB49:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB44]]:
+; CHECK-DEFAULT-NEXT:    [[TMP45:%.*]] = and i64 [[TMP38]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP45]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP47:%.*]] = icmp sge i8 [[TMP46]], [[TMP42]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP47]], label %[[BB48:.*]], label %[[BB49]]
+; CHECK-DEFAULT:       [[BB48]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP38]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB49]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP36]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP50:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    call void @__asan_poison_stack_memory(i64 [[TMP50]], i64 40)
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[TMP36]])
+; CHECK-DEFAULT-NEXT:    [[TMP51:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP52:%.*]] = inttoptr i64 [[TMP51]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 4, ptr [[TMP52]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP53:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP54:%.*]] = lshr i64 [[TMP53]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP55:%.*]] = add i64 [[TMP54]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP56:%.*]] = inttoptr i64 [[TMP55]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP57:%.*]] = load i8, ptr [[TMP56]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP58:%.*]] = icmp ne i8 [[TMP57]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP58]], label %[[BB59:.*]], label %[[BB64:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB59]]:
+; CHECK-DEFAULT-NEXT:    [[TMP60:%.*]] = and i64 [[TMP53]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP62:%.*]] = icmp sge i8 [[TMP61]], [[TMP57]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP62]], label %[[BB63:.*]], label %[[BB64]]
+; CHECK-DEFAULT:       [[BB63]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP53]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB64]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP4]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP65:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP66:%.*]] = inttoptr i64 [[TMP65]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP66]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP67:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP68:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP68]], i64 [[TMP67]])
+; CHECK-DEFAULT-NEXT:    store i64 1172321806, ptr [[TMP5]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP69:%.*]] = add i64 [[TMP11]], 0
+; CHECK-DEFAULT-NEXT:    [[TMP70:%.*]] = inttoptr i64 [[TMP69]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 0, ptr [[TMP70]], align 1
+; CHECK-DEFAULT-NEXT:    ret void
+;
+; CHECK-NO-DYNAMIC-LABEL: define void @lifetime(
+; CHECK-NO-DYNAMIC-SAME: ) #[[ATTR1]] {
+; CHECK-NO-DYNAMIC-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 32
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 1102416563, ptr [[TMP4]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP6]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP7:%.*]] = add i64 [[TMP1]], 16
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP8]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 -868082052615769615, ptr [[TMP12]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP13:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 4, ptr [[TMP14]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 3, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP16:%.*]] = lshr i64 [[TMP15]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP17:%.*]] = add i64 [[TMP16]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP19]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP20]], label %[[BB21:.*]], label %[[BB26:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB21]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP22:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP22]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP24:%.*]] = icmp sge i8 [[TMP23]], [[TMP19]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP24]], label %[[BB25:.*]], label %[[BB26]]
+; CHECK-NO-DYNAMIC:       [[BB25]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP15]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB26]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[TMP3]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP27:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP28:%.*]] = inttoptr i64 [[TMP27]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP28]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP29:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP30:%.*]] = inttoptr i64 [[TMP29]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP30]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[ARR:%.*]] = alloca [10 x i32], align 16
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[ARR]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP33:%.*]] = add i64 [[TMP32]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP34:%.*]] = inttoptr i64 [[TMP33]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP36:%.*]] = icmp ne i8 [[TMP35]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP36]], label %[[BB37:.*]], label %[[BB42:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB37]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP38:%.*]] = and i64 [[TMP31]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP38]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP40:%.*]] = icmp sge i8 [[TMP39]], [[TMP35]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP40]], label %[[BB41:.*]], label %[[BB42]]
+; CHECK-NO-DYNAMIC:       [[BB41]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP31]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB42]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[ARR]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP43:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP44:%.*]] = inttoptr i64 [[TMP43]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 4, ptr [[TMP44]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP45:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP46:%.*]] = lshr i64 [[TMP45]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP47:%.*]] = add i64 [[TMP46]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP48:%.*]] = inttoptr i64 [[TMP47]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP50:%.*]] = icmp ne i8 [[TMP49]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP50]], label %[[BB51:.*]], label %[[BB56:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB51]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP52:%.*]] = and i64 [[TMP45]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP53:%.*]] = trunc i64 [[TMP52]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP54:%.*]] = icmp sge i8 [[TMP53]], [[TMP49]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP54]], label %[[BB55:.*]], label %[[BB56]]
+; CHECK-NO-DYNAMIC:       [[BB55]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP45]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB56]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[TMP3]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP57:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP58:%.*]] = inttoptr i64 [[TMP57]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP58]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    store i64 1172321806, ptr [[TMP4]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP59:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP60:%.*]] = inttoptr i64 [[TMP59]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 0, ptr [[TMP60]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    ret void
+;
 
   ; Regular variable lifetime intrinsics.
   %i = alloca i32, align 4
 
   ; Poison memory in prologue: F1F1F1F1F8F3F3F3
-  ; CHECK: store i64 -868082052615769615, ptr %{{[0-9]+}}
 
   ; Memory is unpoisoned at llvm.lifetime.start
   call void @llvm.lifetime.start.p0(i64 3, ptr %i)
-  ; CHECK: store i8 4, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 4, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ; Memory is poisoned at every call to llvm.lifetime.end
   call void @llvm.lifetime.end.p0(i64 2, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ; Lifetime intrinsics for array.
   %arr = alloca [10 x i32], align 16
 
   call void @llvm.lifetime.start.p0(i64 40, ptr %arr)
-  ; CHECK-DEFAULT: call void @__asan_unpoison_stack_memory(i64 %{{[^ ]+}}, i64 40)
-  ; CHECK-NO-DYNAMIC-NOT: call void @__asan_unpoison_stack_memory(i64 %{{[^ ]+}}, i64 40)
 
   store volatile i8 0, ptr %arr
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 40, ptr %arr)
-  ; CHECK-DEFAULT: call void @__asan_poison_stack_memory(i64 %{{[^ ]+}}, i64 40)
-  ; CHECK-NO-DYNAMIC-NOT: call void @__asan_poison_stack_memory(i64 %{{[^ ]+}}, i64 40)
 
   ; One more lifetime start/end for the same variable %i.
   call void @llvm.lifetime.start.p0(i64 2, ptr %i)
-  ; CHECK: store i8 4, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 4, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.end
 
   ; Memory is unpoisoned at function exit (only once).
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: ret void
   ret void
 }
 
 ; Check that arguments of lifetime may come from phi nodes.
 define void @phi_args(i1 %x) sanitize_address {
-  ; CHECK-LABEL: define void @phi_args(i1 %x)
+; CHECK-LABEL: define void @phi_args(
+; CHECK-SAME: i1 [[X:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.2 to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @phi_args to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i64 -868082052615769615, ptr [[TMP11]], align 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    store i8 0, ptr [[TMP13]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[TMP15]], 2147450880
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    [[TMP18:%.*]] = load i8, ptr [[TMP17]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne i8 [[TMP18]], 0
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[BB20:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp sge i8 [[TMP22]], [[TMP18]]
+; CHECK-NEXT:    br i1 [[TMP23]], label %[[BB24:.*]], label %[[BB25]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__asan_report_store1(i64 [[TMP14]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    store volatile i8 0, ptr [[TMP2]], align 1
+; CHECK-NEXT:    br i1 [[X]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB0]]:
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[I_PHI:%.*]] = phi ptr [ [[TMP2]], %[[BB25]] ], [ [[TMP2]], %[[BB0]] ]
+; CHECK-NEXT:    [[TMP26:%.*]] = add i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr
+; CHECK-NEXT:    store i8 -8, ptr [[TMP27]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[I_PHI]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-NEXT:    store i64 0, ptr [[TMP29]], align 1
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %i = alloca i64, align 4
 
   ; Poison memory in prologue: F1F1F1F1F8F3F3F3
-  ; CHECK: store i64 -868082052615769615, ptr %{{[0-9]+}}
 
   call void @llvm.lifetime.start.p0(i64 8, ptr %i)
-  ; CHECK: store i8 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   br i1 %x, label %bb0, label %bb1
 
@@ -120,49 +401,101 @@ bb0:
 bb1:
   %i.phi = phi ptr [ %i, %entry ], [ %i, %bb0 ]
   call void @llvm.lifetime.end.p0(i64 8, ptr %i.phi)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.end
 
   ret void
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: ret void
 }
 
 ; Check that arguments of lifetime may come from getelementptr nodes.
 define void @getelementptr_args(i64 %i) sanitize_address{
-  ; CHECK-LABEL: define void @getelementptr_args
+; CHECK-LABEL: define void @getelementptr_args(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 1216, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP0]], 1184
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.3 to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @getelementptr_args to i64), ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 2147450880
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    store i32 -235802127, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP14]], i64 128)
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP11]], 132
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    store i64 -940422246894996750, ptr [[TMP16]], align 1
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP11]], 140
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    store i64 -940422246894996750, ptr [[TMP18]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP11]], 150
+; CHECK-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT:    store i16 -3085, ptr [[TMP20]], align 1
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP21]], i64 128)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1024, ptr [[TMP2]])
+; CHECK-NEXT:    [[AI:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP4]], i64 0, i64 [[I]]
+; CHECK-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[AI]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[TMP23]], 2147450880
+; CHECK-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[TMP25]], align 1
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne i8 [[TMP26]], 0
+; CHECK-NEXT:    br i1 [[TMP27]], label %[[BB28:.*]], label %[[BB29:.*]]
+; CHECK:       [[BB28]]:
+; CHECK-NEXT:    call void @__asan_report_store8(i64 [[TMP22]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB29]]:
+; CHECK-NEXT:    store ptr [[TMP2]], ptr [[AI]], align 8
+; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP30]], i64 128)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1024, ptr [[TMP2]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[TMP11]], 0
+; CHECK-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP31]], i64 148)
+; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[TMP11]], 150
+; CHECK-NEXT:    [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT:    store i16 0, ptr [[TMP33]], align 1
+; CHECK-NEXT:    ret void
+;
 entry:
   %x = alloca [1024 x i8], align 16
   %a = alloca [2 x ptr], align 8
 
   ; F1F1F1F1
-  ; CHECK: store i32 -235802127, ptr %{{[0-9]+}}
-  ; CHECK: call void @__asan_set_shadow_f8(i64 %[[#]], i64 128)
   ; 0xf2f2f2f2f2f2f2f2
-  ; CHECK: store i64 -940422246894996750, ptr %[[#]]
   ; 0xf2f2f2f2f2f2f2f2
-  ; CHECK: store i64 -940422246894996750, ptr %[[#]]
 
   call void @llvm.lifetime.start.p0(i64 1024, ptr %x)
-  ; CHECK: call void @__asan_set_shadow_00(i64 %{{[0-9]+}}, i64 128)
-  ; CHECK-NEXT: call void @llvm.lifetime.start
 
   %ai = getelementptr inbounds [2 x ptr], ptr %a, i64 0, i64 %i
   store ptr %x, ptr %ai, align 8
-  ; CHECK: store ptr
 
   call void @llvm.lifetime.end.p0(i64 1024, ptr %x)
-  ; CHECK: call void @__asan_set_shadow_f8(i64 %{{[0-9]+}}, i64 128)
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ret void
-  ; CHECK: call void @__asan_set_shadow_00(i64 %{{[0-9]+}}, i64 148)
-  ; CHECK: store i16 0, ptr %[[#]], align 1
-  ; CHECK-NEXT: ret void
 }
 
 define void @zero_sized(i64 %a) #0 {
-; CHECK-LABEL: define void @zero_sized(i64 %a)
+; CHECK-LABEL: define void @zero_sized(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    [[B:%.*]] = alloca [0 x i8], align 1
+; CHECK-NEXT:    store i64 [[A]], ptr [[A_ADDR]], align 8
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 0, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 0, ptr [[B]])
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %a.addr = alloca i64, align 8
@@ -170,11 +503,13 @@ entry:
   store i64 %a, ptr %a.addr, align 8
 
   call void @llvm.lifetime.start.p0(i64 0, ptr %b) #2
-  ; CHECK: call void @llvm.lifetime.start
 
   call void @llvm.lifetime.end.p0(i64 0, ptr %b) #2
-  ; CHECK: call void @llvm.lifetime.end
 
   ret void
-  ; CHECK-NEXT: ret void
 }
+;.
+; CHECK-DEFAULT: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
+; CHECK-NO-DYNAMIC: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
index 279bb262bff04..5f310a9954ad0 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
@@ -163,6 +163,51 @@ v_tanh_bf16 v5, src_scc
 v_tanh_bf16 v127, 0x8000
 // GFX1250: v_tanh_bf16_e32 v127, 0x8000            ; encoding: [0xff,0x94,0xfe,0x7e,0x00,0x80,0x00,0x00]
 
+v_prng_b32 v5, v1
+// GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, v255
+// GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, s1
+// GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, s105
+// GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_lo
+// GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_hi
+// GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, ttmp15
+// GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, m0
+// GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_lo
+// GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_hi
+// GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, null
+// GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, -1
+// GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, 0.5
+// GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, src_scc
+// GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+v_prng_b32 v255, 0xaf123456
+// GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
 v_rcp_bf16 v5, v1
 // GFX1250: v_rcp_bf16_e32 v5, v1                   ; encoding: [0x01,0xf3,0x0a,0x7e]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
index 76272d25d92d4..aa2e028f661e1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
@@ -169,6 +169,51 @@ v_tanh_bf16 v127, 0x8000
 v_tanh_bf16 v5.h, v1.h
 // GFX1250: v_tanh_bf16_e32 v5.h, v1.h              ; encoding: [0x81,0x95,0x0a,0x7f]
 
+v_prng_b32 v5, v1
+// GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, v255
+// GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, s1
+// GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, s105
+// GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_lo
+// GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_hi
+// GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, ttmp15
+// GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, m0
+// GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_lo
+// GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_hi
+// GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, null
+// GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, -1
+// GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, 0.5
+// GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, src_scc
+// GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+v_prng_b32 v255, 0xaf123456
+// GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
 v_rcp_bf16 v5, v1
 // GFX1250: v_rcp_bf16_e32 v5, v1                   ; encoding: [0x01,0xf3,0x0a,0x7e]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
index 0a8ee84561d33..e1cd2e3043693 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
@@ -170,6 +170,58 @@ v_tanh_bf16 v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 f
 // GFX1250: v_tanh_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x94,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf2,0x0a,0x7e,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
index d4afb9d9b2d9a..c1d3238b65cbd 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
@@ -178,6 +178,58 @@ v_tanh_bf16 v5.h, v1.h quad_perm:[3,2,1,0]
 // GFX1250: v_tanh_bf16_dpp v5.h, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf2,0x0a,0x7e,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
index a7cb6bf8de69c..100e9f92ff58b 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
@@ -38,6 +38,18 @@ v_tanh_bf16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX1250: v_tanh_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x94,0xfe,0x7e,0x7f,0x00,0x00,0x00]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf2,0x0a,0x7e,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
index 6acab7edc0d49..2ae103545443c 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
@@ -46,6 +46,18 @@ v_tanh_bf16 v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_tanh_bf16_dpp v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x94,0x0a,0x7f,0x81,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf2,0x0a,0x7e,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
index 7486d849253e8..9c6a9127d82e4 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
@@ -127,6 +127,42 @@ v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp
 v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp
 // GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2 clamp ; encoding: [0x01,0x88,0xec,0xd5,0x03,0x01,0x00,0x00]
 
+v_prng_b32_e64 v5, v1
+// GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, v255
+// GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, s1
+// GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, s105
+// GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_lo
+// GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_hi
+// GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, ttmp15
+// GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, m0
+// GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_lo
+// GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_hi
+// GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, null
+// GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, -1
+// GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
 v_tanh_f32_e64 v5, v1
 // GFX1250: v_tanh_f32_e64 v5, v1                   ; encoding: [0x05,0x00,0x9e,0xd5,0x01,0x01,0x00,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
index b59b8b31e2d5f..2f57d1c331c42 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
@@ -223,6 +223,42 @@ v_tanh_f16_e64 v255, -|0x8000| clamp div:2
 v_tanh_f16 v5.l, v128.h
 // GFX1250: v_tanh_f16_e64 v5.l, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0x9f,0xd5,0x80,0x01,0x00,0x00]
 
+v_prng_b32_e64 v5, v1
+// GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, v255
+// GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, s1
+// GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, s105
+// GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_lo
+// GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_hi
+// GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, ttmp15
+// GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, m0
+// GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_lo
+// GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_hi
+// GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, null
+// GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, -1
+// GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
 v_rcp_bf16_e64 v5, v1
 // GFX1250: v_rcp_bf16_e64 v5, v1                   ; encoding: [0x05,0x00,0xf9,0xd5,0x01,0x01,0x00,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
index f7f20f46161ce..29bb842b529b7 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
@@ -170,6 +170,50 @@ v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mas
 // GFX1250: v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xca,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
index e1241b01ccae1..7df92751c38d1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
@@ -178,6 +178,50 @@ v_tanh_bf16_e64_dpp v5.h, v128.h quad_perm:[3,2,1,0]
 // GFX1250: v_tanh_bf16_e64_dpp v5.h, v128.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xca,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
index 0106175301d20..d235aeb9f3e62 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
@@ -50,6 +50,10 @@ v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX1250: v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xca,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
index 93b86f3ffb841..f25e2a5882436 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
@@ -58,6 +58,10 @@ v_tanh_bf16_e64_dpp v5.h, v128.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_tanh_bf16_e64_dpp v5.h, v128.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xca,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
index 5f37ba91e071b..aa968b2bb2bee 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
@@ -200,6 +200,51 @@
 0x81,0x95,0x0a,0x7f
 # GFX1250-REAL16: v_tanh_bf16_e32 v5.h, v1.h              ; encoding: [0x81,0x95,0x0a,0x7f]
 
+0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf
+# GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
+0xc1,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+0xf0,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+0x7f,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+0x7e,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+0x7d,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+0x7c,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+0x01,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+0x69,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+0xfd,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+0x7b,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+0x01,0x97,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+0xff,0x97,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+0x6b,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+0x6a,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
 0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_e32 v127.l, 0x8000           ; encoding: [0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_e32 v127, 0x8000             ; encoding: [0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
index 57bee2766ce44..913a2a916ff62 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
@@ -162,6 +162,45 @@
 0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff
 # GFX1250-REAL16: v_tanh_bf16_dpp v5.h, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff]
 
+0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01
+# GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13
+# GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+
 0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30
 # GFX1250-REAL16: v_rcp_bf16_dpp v127.l, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
 # GFX1250-FAKE16: v_rcp_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
index 28ec6b11b4de3..4afe44e241bf3 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
@@ -43,6 +43,15 @@
 # GFX1250-REAL16: v_tanh_bf16_dpp v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x94,0x0a,0x7f,0x81,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_add_f64_e32 v[156:157], v[129:130], v[187:188] ; encoding: [0x81,0x77,0x39,0x05]
 
+0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00
+# GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+
+0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
 0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_dpp v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
index 5004762729701..1cf3b8807d044 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
@@ -175,6 +175,42 @@
 # GFX1250-REAL16: v_tanh_bf16_e64 v5.l, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0xca,0xd5,0x80,0x01,0x00,0x00]
 # GFX1250-FAKE16: v_tanh_bf16_e64 v5, v128                ; encoding: [0x05,0x00,0xca,0xd5,0x80,0x01,0x00,0x00]
 
+0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
 0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_e64 v255.l, -|0x8000| clamp div:2 ; encoding: [0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_e64 v255, -|0x8000| clamp div:2 ; encoding: [0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
index de908b95d94f9..83a647ad7c658 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
@@ -104,6 +104,39 @@
 # GFX1250-REAL16: v_tanh_f16_e64_dpp v5.h, v128.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x9f,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 # GFX1250-FAKE16: v_tanh_f16_e64_dpp v5, v128 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x9f,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+
 0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30
 # GFX1250-REAL16: v_rsq_bf16_e64_dpp v255.l, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
 # GFX1250-FAKE16: v_rsq_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
index cfe7173c383b3..ef5ede4d1d453 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
 
 0xff,0x81,0x9e,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00
 # GFX1250: v_tanh_f32_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0x9e,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
@@ -34,6 +34,9 @@
 # GFX1250-REAL16: v_tanh_f16_e64_dpp v5.h, v128.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0x9f,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_tanh_f16_e64_dpp v5, v128 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x9f,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 
+0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
 0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00
 # GFX1250-REAL16: v_rsq_bf16_e64_dpp v255.l, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
 # GFX1250-FAKE16: v_rsq_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll b/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
index e390d4bdca632..303afc207c023 100644
--- a/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
@@ -12,6 +12,6 @@ define ptr @undeclared_customalloc(i64 %size, i64 %align) {
   ret ptr %call
 }
 
-declare ptr @customalloc2(i64, i64) allockind("alloc") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
+declare ptr @customalloc2(i64, i64) allockind("alloc,uninitialized") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
 ; CHECK-DAG: declare ptr @customalloc2_zeroed(i64, i64) #[[CA2ATTR:[0-9]+]]
 ; CHECK-DAG: attributes #[[CA2ATTR]] = { allockind("alloc,zeroed") "alloc-family"="customalloc2" }
diff --git a/llvm/test/Transforms/LoopInterchange/force-interchange.ll b/llvm/test/Transforms/LoopInterchange/force-interchange.ll
new file mode 100644
index 0000000000000..c33ecdf7d9905
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/force-interchange.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -passes=loop-interchange -pass-remarks-output=%t -disable-output -loop-interchange-profitabilities=ignore -S
+; RUN: FileCheck --input-file=%t %s
+
+; There should be no reason to interchange this, unless it is forced.
+;
+;     for (int i = 0; i<1024; i++)
+;       for (int j = 0; j<1024; j++)
+;         A[i][j] = 42;
+;
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Interchanged
+; CHECK-NEXT: Function:        f
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Loop interchanged with enclosing loop.
+; CHECK-NEXT: ...
+
+ at A = dso_local local_unnamed_addr global [1024 x [1024 x i32]] zeroinitializer, align 4
+
+define dso_local void @f() local_unnamed_addr #0 {
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %inner.header ]
+  br label %inner.body
+
+inner.header:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond20.not = icmp eq i64 %i.next, 1024
+  br i1 %exitcond20.not, label %exit, label %outer.header
+
+inner.body:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
+  %arrayidx6 = getelementptr inbounds nuw [1024 x [1024 x i32]], ptr @A, i64 0, i64 %i, i64 %j
+  store i32 42, ptr %arrayidx6, align 4
+  %j.next = add nuw nsw i64 %j, 1
+  %exitcond.not = icmp eq i64 %j.next, 1024
+  br i1 %exitcond.not, label %inner.header, label %inner.body
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
new file mode 100644
index 0000000000000..c7a0de22b200b
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=loop-reduce -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(ptr %p, i64 %idx) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4 x [4 x i32]], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[IDX]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 48
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[ALLOCA]], i64 48
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], %[[LOOP]] ], [ -8, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[LSR_IV]], 2
+; CHECK-NEXT:    [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[SCEVGEP8]], i64 32
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[SCEVGEP9]], align 4
+; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[SCEVGEP6]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[SCEVGEP7]], align 4
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[SCEVGEP5]], align 4
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[SCEVGEP2]], align 4
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], 4
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %alloca = alloca [4 x [4 x i32]], align 16
+  call void @llvm.lifetime.start.p0(i64 64, ptr %alloca)
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %gep1 = getelementptr [4 x [12 x [4 x [4 x i32]]]], ptr %p, i64 0, i64 0, i64 0, i64 %indvars.iv, i64 0
+  %0 = load i32, ptr %gep1, align 4
+  %gep2 = getelementptr [6 x [4 x [4 x i32]]], ptr %p, i64 0, i64 0, i64 0, i64 %indvars.iv
+  %1 = load i32, ptr %gep2, align 4
+  %gep3 = getelementptr [4 x [4 x i32]], ptr %alloca, i64 0, i64 3, i64 %indvars.iv
+  %2 = load i32, ptr %gep3, align 4
+  %gep4 = getelementptr [6 x [4 x [4 x i32]]], ptr %p, i64 0, i64 %idx, i64 3, i64 %indvars.iv
+  %3 = load i32, ptr %gep4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv, 1
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  call void @llvm.lifetime.end.p0(i64 64, ptr %alloca)
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll b/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
new file mode 100644
index 0000000000000..2d1543185098f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
@@ -0,0 +1,38 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-REGS-VP
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -force-target-num-vector-regs=1 -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NOREGS-VP
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @dotp(ptr %a, ptr %b) #0 {
+; CHECK-REGS-VP-NOT: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; CHECK-REGS-VP: LV: Selecting VF: vscale x 8.
+;
+; CHECK-NOREGS-VP: LV(REG): Not considering vector loop of width vscale x 8 because it uses too many registers
+; CHECK-NOREGS-VP: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; CHECK-NOREGS-VP: LV: Selecting VF: vscale x 4.
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %gep.b = getelementptr i8, ptr %b, i64 %iv
+  %load.b = load i8, ptr %gep.b, align 1
+  %ext.b = zext i8 %load.b to i32
+  %mul = mul i32 %ext.b, %ext.a
+  %sub = sub i32 0, %mul
+  %add = add i32 %accum, %sub
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
index 8e09847e9264e..cfff11758a37a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
@@ -58,7 +58,6 @@ define void @test(i32 %0, i8 %1, i64 %2, float %3) {
 ; CHECK-NEXT:    br label %[[BB54:.*]]
 ; CHECK:       [[BB54]]:
 ; CHECK-NEXT:    [[TMP54:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP17]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 0, ptr null)
 ; CHECK-NEXT:    [[TMP55:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP21]])
 ; CHECK-NEXT:    [[TMP56:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <8 x float> [[TMP56]], <8 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison>
@@ -198,7 +197,6 @@ define void @test(i32 %0, i8 %1, i64 %2, float %3) {
   %95 = or i64 %94, %91
   %96 = or i64 %95, %37
   store i64 %96, ptr null, align 1
-  call void @llvm.lifetime.start.p0(i64 0, ptr null)
   store i64 %42, ptr null, align 1
   %97 = bitcast float %3 to i32
   %98 = icmp ult i32 %97, 1325400064
diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp
index 422eb855ba2cf..2b43d27f292a0 100644
--- a/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -131,20 +131,6 @@ class CommentWriter : public AssemblyAnnotationWriter {
         printDebugLoc(DL,OS);
         OS << "]";
       }
-      if (const DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(I)) {
-        if (!Padded) {
-          OS.PadToColumn(50);
-          OS << ";";
-        }
-        OS << " [debug variable = " << DDI->getVariable()->getName() << "]";
-      }
-      else if (const DbgValueInst *DVI = dyn_cast<DbgValueInst>(I)) {
-        if (!Padded) {
-          OS.PadToColumn(50);
-          OS << ";";
-        }
-        OS << " [debug variable = " << DVI->getVariable()->getName() << "]";
-      }
     }
   }
 };
diff --git a/llvm/unittests/CodeGen/LexicalScopesTest.cpp b/llvm/unittests/CodeGen/LexicalScopesTest.cpp
index 3d707462fa615..563d496d1e600 100644
--- a/llvm/unittests/CodeGen/LexicalScopesTest.cpp
+++ b/llvm/unittests/CodeGen/LexicalScopesTest.cpp
@@ -67,7 +67,7 @@ class LexicalScopesTest : public testing::Test {
     BeanInst.Opcode = 1;
     BeanInst.Size = 1;
 
-    memset(&DbgValueInst, 0, sizeof(DbgValueInst));
+    memset(&DbgValueInst, 0, sizeof(MCInstrDesc));
     DbgValueInst.Opcode = TargetOpcode::DBG_VALUE;
     DbgValueInst.Size = 1;
     DbgValueInst.Flags = 1U << MCID::Meta;
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index 41bf863420304..baa13e1199eea 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -188,6 +188,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgRecord) {
   SmallVector<DbgValueInst *, 1> DVIs;
   SmallVector<DbgVariableRecord *, 1> DVRs;
   findDbgValues(DVIs, &I, &DVRs);
+  assert(DVIs.empty());
 
   // Delete %b. The dbg.value should now point to undef.
   I.eraseFromParent();
@@ -314,6 +315,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
   SmallVector<DbgValueInst *, 2> DVIs;
   SmallVector<DbgVariableRecord *, 2> DVRs;
   findDbgValues(DVIs, &I, &DVRs);
+  assert(DVIs.empty());
   ASSERT_EQ(DVRs.size(), 2u);
 
   // Delete %b. The DbgVariableRecord should now point to undef.
diff --git a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
index 0b00734fc4d75..1daf381ee2862 100644
--- a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
+++ b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
@@ -54,20 +54,13 @@ struct DebugInfoDrop : public FunctionPass {
 struct DebugValueDrop : public FunctionPass {
   static char ID;
   bool runOnFunction(Function &F) override {
-    SmallVector<DbgVariableIntrinsic *, 4> Dbgs;
     for (BasicBlock &BB : F) {
-      // Remove dbg var intrinsics.
       for (Instruction &I : BB) {
-        if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
-          Dbgs.push_back(DVI);
-        // If there are any non-intrinsic records (DbgRecords), drop those too.
+        // If there are any debug records, drop them.
         I.dropDbgRecords();
       }
     }
 
-    for (auto &I : Dbgs)
-      I->eraseFromParent();
-
     return true;
   }
 
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index b922216ef8893..dd2a6249c7cf9 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -633,62 +633,6 @@ TEST(Local, ChangeToUnreachable) {
   EXPECT_EQ(DLA, DLB);
 }
 
-TEST(Local, FindDbgUsers) {
-  LLVMContext Ctx;
-  std::unique_ptr<Module> M = parseIR(Ctx,
-                                      R"(
-  define dso_local void @fun(ptr %a) #0 !dbg !11 {
-  entry:
-      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
-    ret void
-  }
-
-  !llvm.dbg.cu = !{!0}
-  !llvm.module.flags = !{!2, !3, !9}
-  !llvm.ident = !{!10}
-
-  !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-  !1 = !DIFile(filename: "test.cpp", directory: "/")
-  !2 = !{i32 7, !"Dwarf Version", i32 5}
-  !3 = !{i32 2, !"Debug Info Version", i32 3}
-  !4 = !{i32 1, !"wchar_size", i32 4}
-  !9 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
-  !10 = !{!"clang version 17.0.0"}
-  !11 = distinct !DISubprogram(name: "fun", linkageName: "fun", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !14)
-  !12 = !DISubroutineType(types: !13)
-  !13 = !{null}
-  !14 = !{}
-  !15 = distinct !DIAssignID()
-  !16 = !DILocalVariable(name: "x", scope: !11, file: !1, line: 2, type: !17)
-  !17 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 64)
-  !18 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-  !19 = !DILocation(line: 0, scope: !11)
-  )");
-
-  bool BrokenDebugInfo = true;
-  verifyModule(*M, &errs(), &BrokenDebugInfo);
-  ASSERT_FALSE(BrokenDebugInfo);
-
-  // Convert to debug intrinsics as we want to test findDbgUsers and
-  // findDbgValue's debug-intrinsic-finding code here.
-  // TODO: Remove this test when debug intrinsics are removed.
-  M->convertFromNewDbgValues();
-
-  Function &Fun = *cast<Function>(M->getNamedValue("fun"));
-  Value *Arg = Fun.getArg(0);
-  SmallVector<DbgVariableIntrinsic *> Users;
-  // Arg (%a) is used twice by a single dbg.assign. Check findDbgUsers returns
-  // only 1 pointer to it rather than 2.
-  findDbgUsers(Users, Arg);
-  EXPECT_EQ(Users.size(), 1u);
-
-  SmallVector<DbgValueInst *> Vals;
-  // Arg (%a) is used twice by a single dbg.assign. Check findDbgValues returns
-  // only 1 pointer to it rather than 2.
-  findDbgValues(Vals, Arg);
-  EXPECT_EQ(Vals.size(), 1u);
-}
-
 TEST(Local, FindDbgRecords) {
   // DbgRecord copy of the FindDbgUsers test above.
   LLVMContext Ctx;
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 82ec8121548c9..5309b5d095134 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -35,7 +35,6 @@ if (current_toolchain == default_toolchain) {
       "_LIBCPP_HAS_LOCALIZATION=1",
       "_LIBCPP_HAS_UNICODE=1",
       "_LIBCPP_HAS_WIDE_CHARACTERS=1",
-      "_LIBCPP_HAS_NO_STD_MODULES=",
       "_LIBCPP_HAS_TERMINAL=1",
       "_LIBCPP_INSTRUMENTED_WITH_ASAN=",
       "_LIBCPP_ABI_DEFINES=",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index d4adeddd9b4e4..3d11ce566207a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -176,6 +176,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUPreLegalizerCombiner.cpp",
     "AMDGPUPreloadKernArgProlog.cpp",
     "AMDGPUPreloadKernelArguments.cpp",
+    "AMDGPUPrepareAGPRAlloc.cpp",
     "AMDGPUPrintfRuntimeBinding.cpp",
     "AMDGPUPromoteAlloca.cpp",
     "AMDGPUPromoteKernelArguments.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 946b63f8a54fb..7ed0d3c6824e2 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -286,6 +286,7 @@ group("test") {
     "//llvm/tools/llvm-extract",
     "//llvm/tools/llvm-gsymutil:llvm-gsymutil",
     "//llvm/tools/llvm-ifs",
+    "//llvm/tools/llvm-ir2vec",
     "//llvm/tools/llvm-isel-fuzzer",
     "//llvm/tools/llvm-jitlink",
     "//llvm/tools/llvm-jitlink/llvm-jitlink-executor",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
new file mode 100644
index 0000000000000..07a795122c76c
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
@@ -0,0 +1,9 @@
+executable("llvm-ir2vec") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+  ]
+  sources = [ "llvm-ir2vec.cpp" ]
+}
diff --git a/mlir/docs/Tutorials/transform/Ch2.md b/mlir/docs/Tutorials/transform/Ch2.md
index 0f45f5607bab9..08c4e0f9d859c 100644
--- a/mlir/docs/Tutorials/transform/Ch2.md
+++ b/mlir/docs/Tutorials/transform/Ch2.md
@@ -133,6 +133,8 @@ This will generate two files, `MyExtension.h.inc` and `MyExtension.cpp.inc`, tha
 ```c++
 // In MyExtension.cpp.
 
+#include "MyExtension.h"
+
 #define GET_OP_CLASSES
 #include "MyExtension.cpp.inc"
 
@@ -283,7 +285,7 @@ void registerMyExtension(::mlir::DialectRegistry &registry) {
 }
 ```
 
-After registering the extension, it becomes possible to use our new operation in the Transform dialect interpreter. The upstream testing pass can be used as is.
+After registering the extension, it becomes possible to use our new operation in the Transform dialect interpreter. The upstream testing pass can be used as is. It actually exists in `mlir/test/Examples/transform/Ch2/sequence.mlir`, which contains the `microkernel` implementation. 
 
 ```mlir
 module attributes {transform.with_named_sequence} {
@@ -300,7 +302,7 @@ module attributes {transform.with_named_sequence} {
 
     // The actual tiling transformation takes tile sizes as attributes. It
     // produces a handle to the loop generated during tiling.
-    %loop, %tiled = transform.structured.tile_using_forall %max
+    %tiled, %loop = transform.structured.tile_using_forall %max
                     tile_sizes [8, 32]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
@@ -311,32 +313,32 @@ module attributes {transform.with_named_sequence} {
     // a single handle to all operations and give it to
     // `fuse_into_containing_op` that would take care of the ordering in this
     // case.
-    %add_fused = transform.structured.fuse_into_containing_op %add into %loop
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
-    %matmul_fused = transform.structured.fuse_into_containing_op %arg1
-                    into %loop
+    %add_fused, %loop2 = transform.structured.fuse_into_containing_op %add into %loop
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_fused, %loop3 = transform.structured.fuse_into_containing_op %arg1
+                    into %loop2
         : (!transform.op<"linalg.matmul">, !transform.any_op)
-       -> !transform.any_op
+       -> (!transform.any_op, !transform.any_op)
 
     // Tile again to get the desired size. Note that this time this tiles the
     // "add" operation and fuses matmul into the loop, but doesn't affect the
     // "max" operation. This illustrates the precise targeting with the
     // transform dialect. Otherwise, it is difficult to differentiate "add" and
     // "max", both of which having the same kind.
-    %loop_2, %tiled_2 = transform.structured.tile_using_forall %add_fused
+    %tiled_second, %loop_second = transform.structured.tile_using_forall %add_fused
                         tile_sizes [4, 4]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
-    %matmul_fused_2 = transform.structured.fuse_into_containing_op %matmul_fused
-                      into %loop_2
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
+    %matmul_fused_2, %loop_second_2 = transform.structured.fuse_into_containing_op %matmul_fused
+                      into %loop_second
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     // Since outlining is currently only implemented for region-holding
     // operations such as loops, use tiling to size 1 to materialize the outer
     // loop that is going to be outlined.
-    %outline_target, %_ = transform.structured.tile_using_forall %tiled_2 tile_sizes [1]
+    %_0, %loop_third = transform.structured.tile_using_forall %tiled_second tile_sizes [1]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
-    transform.structured.fuse_into_containing_op %matmul_fused_2 into %outline_target
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
+    %_1, %outline_target = transform.structured.fuse_into_containing_op %matmul_fused_2 into %loop_third
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
     %func, %call = transform.loop.outline %outline_target
                    {func_name = "outlined"}
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
diff --git a/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h b/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
index ac4915901fdec..ff99d7ce96daf 100644
--- a/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
+++ b/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
@@ -21,9 +21,6 @@ class RewritePatternSet;
 /// Collect a set of patterns to convert from the vector to XeGPU ops.
 void populateVectorToXeGPUConversionPatterns(RewritePatternSet &patterns);
 
-/// Create a pass to convert ops from vector to XeGPU.
-std::unique_ptr<Pass> createConvertVectorToXeGPUPass();
-
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_VECTORTOXEGPU_VECTORTOXEGPU_H
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 12c7cc62905c9..d4400547f9568 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -1115,6 +1115,18 @@ struct AMDGPUStreamTy {
     return Plugin::success();
   }
 
+  /// Complete pending post actions until and including the event in target
+  /// slot.
+  Error completeUntil(uint32_t TargetSlot) {
+    for (uint32_t Slot = 0; Slot <= TargetSlot; ++Slot) {
+      // Take the post action of the operation if any.
+      if (auto Err = Slots[Slot].performAction())
+        return Err;
+    }
+
+    return Plugin::success();
+  }
+
   /// Make the current stream wait on a specific operation of another stream.
   /// The idea is to make the current stream waiting on two signals: 1) the last
   /// signal of the current stream, and 2) the last signal of the other stream.
@@ -1502,6 +1514,11 @@ struct AMDGPUStreamTy {
     return complete();
   }
 
+  /// Synchronize the stream until the given event. The current thread waits
+  /// until the provided event is finalized, and it performs the pending post
+  /// actions for that and prior events.
+  Error synchronizeOn(AMDGPUEventTy &Event);
+
   /// Query the stream and complete pending post actions if operations finished.
   /// Return whether all the operations completed. This operation does not block
   /// the calling thread.
@@ -1575,6 +1592,21 @@ struct AMDGPUEventTy {
     return Stream.waitEvent(*this);
   }
 
+  Error sync() {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    if (!RecordedStream)
+      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
+                           "event does not have any recorded stream");
+
+    // No need to wait on anything, the recorded stream already finished the
+    // corresponding operation.
+    if (RecordedSlot < 0)
+      return Plugin::success();
+
+    return RecordedStream->synchronizeOn(*this);
+  }
+
 protected:
   /// The stream registered in this event.
   AMDGPUStreamTy *RecordedStream;
@@ -1630,6 +1662,22 @@ Error AMDGPUStreamTy::waitEvent(const AMDGPUEventTy &Event) {
   return waitOnStreamOperation(RecordedStream, Event.RecordedSlot);
 }
 
+Error AMDGPUStreamTy::synchronizeOn(AMDGPUEventTy &Event) {
+  std::lock_guard<std::mutex> Lock(Mutex);
+
+  // Wait until the requested slot has completed
+  if (auto Err = Slots[Event.RecordedSlot].Signal->wait(
+          StreamBusyWaitMicroseconds, &Device))
+    return Err;
+
+  // If the event is the last one in the stream, just do a full finalize
+  if (Event.RecordedSlot == last())
+    return complete();
+
+  // Otherwise, only finalize until the appropriate event
+  return completeUntil(Event.RecordedSlot);
+}
+
 struct AMDGPUStreamManagerTy final
     : GenericDeviceResourceManagerTy<AMDGPUResourceRef<AMDGPUStreamTy>> {
   using ResourceRef = AMDGPUResourceRef<AMDGPUStreamTy>;
@@ -2540,8 +2588,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
   /// Synchronize the current thread with the event.
   Error syncEventImpl(void *EventPtr) override {
-    return Plugin::error(ErrorCode::UNIMPLEMENTED,
-                         "synchronize event not implemented");
+    AMDGPUEventTy *Event = reinterpret_cast<AMDGPUEventTy *>(EventPtr);
+    return Event->sync();
   }
 
   /// Print information about the device.
@@ -2562,7 +2610,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     Status2 = hsa_system_get_info(HSA_SYSTEM_INFO_VERSION_MINOR, &Minor);
     if (Status == HSA_STATUS_SUCCESS && Status2 == HSA_STATUS_SUCCESS)
       Info.add("HSA Runtime Version",
-               std::to_string(Major) + "." + std::to_string(Minor));
+               std::to_string(Major) + "." + std::to_string(Minor), "",
+               DeviceInfo::DRIVER_VERSION);
 
     Info.add("HSA OpenMP Device Number", DeviceId);
 
@@ -2572,11 +2621,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_NAME, TmpChar);
     if (Status == HSA_STATUS_SUCCESS)
-      Info.add("Device Name", TmpChar);
+      Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_VENDOR_NAME, TmpChar);
     if (Status == HSA_STATUS_SUCCESS)
-      Info.add("Vendor Name", TmpChar);
+      Info.add("Vendor Name", TmpChar, "", DeviceInfo::VENDOR);
 
     hsa_device_type_t DevType;
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_DEVICE, DevType);
@@ -2652,7 +2701,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_WORKGROUP_MAX_DIM, WorkgrpMaxDim);
     if (Status == HSA_STATUS_SUCCESS) {
-      auto &MaxSize = *Info.add("Workgroup Max Size per Dimension");
+      auto &MaxSize =
+          *Info.add("Workgroup Max Size per Dimension", std::monostate{}, "",
+                    DeviceInfo::MAX_WORK_GROUP_SIZE);
       MaxSize.add("x", WorkgrpMaxDim[0]);
       MaxSize.add("y", WorkgrpMaxDim[1]);
       MaxSize.add("z", WorkgrpMaxDim[2]);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 162b149ab483e..8c17a2ee07047 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -113,6 +113,12 @@ struct AsyncInfoWrapperTy {
   __tgt_async_info *AsyncInfoPtr;
 };
 
+enum class DeviceInfo {
+#define OFFLOAD_DEVINFO(Name, _, Value) Name = Value,
+#include "OffloadInfo.inc"
+#undef OFFLOAD_DEVINFO
+};
+
 /// Tree node for device information
 ///
 /// This information is either printed or used by liboffload to extract certain
@@ -133,6 +139,8 @@ struct InfoTreeNode {
   // * The same key can appear multiple times
   std::unique_ptr<llvm::SmallVector<InfoTreeNode, 8>> Children;
 
+  llvm::DenseMap<DeviceInfo, size_t> DeviceInfoMap;
+
   InfoTreeNode() : InfoTreeNode("", std::monostate{}, "") {}
   InfoTreeNode(std::string Key, VariantType Value, std::string Units)
       : Key(Key), Value(Value), Units(Units) {}
@@ -140,10 +148,12 @@ struct InfoTreeNode {
   /// Add a new info entry as a child of this node. The entry requires at least
   /// a key string in \p Key. The value in \p Value is optional and can be any
   /// type that is representable as a string. The units in \p Units is optional
-  /// and must be a string.
+  /// and must be a string. Providing a device info key allows liboffload to
+  /// use that value for an appropriate olGetDeviceInfo query
   template <typename T = std::monostate>
   InfoTreeNode *add(std::string Key, T Value = T(),
-                    const std::string &Units = std::string()) {
+                    const std::string &Units = std::string(),
+                    std::optional<DeviceInfo> DeviceInfoKey = std::nullopt) {
     assert(!Key.empty() && "Invalid info key");
 
     if (!Children)
@@ -157,7 +167,12 @@ struct InfoTreeNode {
     else
       ValueVariant = std::string{Value};
 
-    return &Children->emplace_back(Key, ValueVariant, Units);
+    auto Ptr = &Children->emplace_back(Key, ValueVariant, Units);
+
+    if (DeviceInfoKey)
+      DeviceInfoMap[*DeviceInfoKey] = Children->size() - 1;
+
+    return Ptr;
   }
 
   std::optional<InfoTreeNode *> get(StringRef Key) {
@@ -171,6 +186,13 @@ struct InfoTreeNode {
     return It;
   }
 
+  std::optional<InfoTreeNode *> get(DeviceInfo Info) {
+    auto Result = DeviceInfoMap.find(Info);
+    if (Result != DeviceInfoMap.end())
+      return &(*Children)[Result->second];
+    return std::nullopt;
+  }
+
   /// Print all info entries in the tree
   void print() const {
     // Fake an additional indent so that values are offset from the keys
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index b787376eb1770..728bf07c572bb 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -935,15 +935,16 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Res == CUDA_SUCCESS)
       // For consistency with other drivers, store the version as a string
       // rather than an integer
-      Info.add("CUDA Driver Version", std::to_string(TmpInt));
+      Info.add("CUDA Driver Version", std::to_string(TmpInt), "",
+               DeviceInfo::DRIVER_VERSION);
 
     Info.add("CUDA OpenMP Device Number", DeviceId);
 
     Res = cuDeviceGetName(TmpChar, 1000, Device);
     if (Res == CUDA_SUCCESS)
-      Info.add("Device Name", TmpChar);
+      Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
 
-    Info.add("Vendor Name", "NVIDIA");
+    Info.add("Vendor Name", "NVIDIA", "", DeviceInfo::VENDOR);
 
     Res = cuDeviceTotalMem(&TmpSt, Device);
     if (Res == CUDA_SUCCESS)
@@ -978,7 +979,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Res == CUDA_SUCCESS)
       Info.add("Maximum Threads per Block", TmpInt);
 
-    auto &MaxBlock = *Info.add("Maximum Block Dimensions", "");
+    auto &MaxBlock = *Info.add("Maximum Block Dimensions", std::monostate{}, "",
+                               DeviceInfo::MAX_WORK_GROUP_SIZE);
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, TmpInt);
     if (Res == CUDA_SUCCESS)
       MaxBlock.add("x", TmpInt);
diff --git a/offload/unittests/OffloadAPI/common/Fixtures.hpp b/offload/unittests/OffloadAPI/common/Fixtures.hpp
index e5d815ecda965..546921164f691 100644
--- a/offload/unittests/OffloadAPI/common/Fixtures.hpp
+++ b/offload/unittests/OffloadAPI/common/Fixtures.hpp
@@ -171,9 +171,6 @@ struct OffloadQueueTest : OffloadDeviceTest {
 struct OffloadEventTest : OffloadQueueTest {
   void SetUp() override {
     RETURN_ON_FATAL_FAILURE(OffloadQueueTest::SetUp());
-    if (getPlatformBackend() == OL_PLATFORM_BACKEND_AMDGPU)
-      GTEST_SKIP() << "AMDGPU synchronize event not implemented";
-
     // Get an event from a memcpy. We can still use it in olGetEventInfo etc
     // after it has been waited on.
     void *Alloc;
diff --git a/offload/unittests/OffloadAPI/event/olWaitEvent.cpp b/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
index 05356d4ef8d75..f80dabb4fc93f 100644
--- a/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
+++ b/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
@@ -14,9 +14,6 @@ using olWaitEventTest = OffloadQueueTest;
 OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olWaitEventTest);
 
 TEST_P(olWaitEventTest, Success) {
-  if (getPlatformBackend() == OL_PLATFORM_BACKEND_AMDGPU)
-    GTEST_SKIP() << "AMDGPU synchronize event not implemented";
-
   uint32_t Src = 42;
   void *DstPtr;
 

>From bfe68539b78d8610bd0cd315c9a3e8fe9b456a3f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 16:45:02 +0100
Subject: [PATCH 6/6] !fixup address latest comments, thanks

---
 llvm/lib/Analysis/IVDescriptors.cpp           |   6 +-
 .../Vectorize/VPlanConstruction.cpp           |  49 ++++----
 .../Transforms/Vectorize/VPlanTransforms.h    |   8 +-
 .../fmax-without-fast-math-flags.ll           | 116 ++++++++++++++++++
 4 files changed, 150 insertions(+), 29 deletions(-)

diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index a88bbd7b65d53..8be5de3bf356f 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -947,10 +947,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
       InstDesc Res = isMinMaxPattern(I, Kind, Prev);
       if (!Res.isRecurrence())
         return InstDesc(false, I);
-
       if (HasRequiredFMF())
         return Res;
-
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
       // intrinsics with extra checks ensuring the vector loop handles only
       // non-NaN inputs.
@@ -964,9 +962,9 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                "unexpected recurrence kind for minnum");
         return InstDesc(I, RecurKind::FMinNum);
       }
-
       return InstDesc(false, I);
-    } else if (isFMulAddIntrinsic(I))
+    }
+    if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
     return InstDesc(false, I);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a426d06b4befe..98c2fe2d6129d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -654,10 +654,6 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
 }
 
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
-  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-  VPReductionPHIRecipe *RedPhiR = nullptr;
-  bool HasUnsupportedPhi = false;
-
   auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
     auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
         RedPhiR->getBackedgeValue()->getDefiningRecipe());
@@ -666,7 +662,7 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
 
     auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
     if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
-        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+        !(RepR && isa<IntrinsicInst>(RepR->getUnderlyingInstr())))
       return nullptr;
 
 #ifndef NDEBUG
@@ -690,41 +686,47 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     return MinMaxR->getOperand(0);
   };
 
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
-    // TODO: Also support fixed-order recurrence phis.
-    HasUnsupportedPhi |=
-        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
-             VPReductionPHIRecipe>(&R);
+    if (isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe>(&R))
+      continue;
     auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
-    if (!Cur)
+    if (!Cur) {
+      // TODO: Also support fixed-order recurrence phis.
+      HasUnsupportedPhi = true;
       continue;
+    }
     // For now, only a single reduction is supported.
     // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
     if (RedPhiR)
       return false;
     if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
-        Cur->getRecurrenceKind() != RecurKind::FMinNum)
+        Cur->getRecurrenceKind() != RecurKind::FMinNum) {
+      HasUnsupportedPhi = true;
       continue;
+    }
     RedPhiR = Cur;
   }
 
   if (!RedPhiR)
     return true;
 
-  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
-  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
-         "unsupported reduction");
-
-  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
-  if (!MinMaxOp)
-    return false;
-
   // We won't be able to resume execution in the scalar tail, if there are
   // unsupported header phis or there is no scalar tail at all, due to
   // tail-folding.
   if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
+  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+  if (!MinMaxOp)
+    return false;
+
+  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+         "unsupported reduction");
+
   /// Check if the vector loop of \p Plan can early exit and restart
   /// execution of last vector iteration in the scalar loop. This requires all
   /// recipes up to early exit point be side-effect free as they are
@@ -774,7 +776,7 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   RdxResult->setOperand(1, NewSel);
 
   auto *ScalarPH = Plan.getScalarPreheader();
-  // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+  // Update resume phis for inductions in the scalar preheader. If AnyNaN is
   // true, the resume from the start of the last vector iteration via the
   // canonical IV, otherwise from the original value.
   for (auto &R : ScalarPH->phis()) {
@@ -782,8 +784,13 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     VPValue *VecV = ResumeR->getOperand(0);
     if (VecV == RdxResult)
       continue;
-    if (VecV != &Plan.getVectorTripCount())
+    // Bail out and abandon the current, partially modified, VPlan if we
+    // encounter resume phi that cannot be updated yet.
+    if (VecV != &Plan.getVectorTripCount()) {
+      LLVM_DEBUG(dbgs() << "Found resume phi we cannot update for VPlan with "
+                           "FMaxNum/FMinNum reduction.\n");
       return false;
+    }
     auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
     ResumeR->setOperand(0, NewSel);
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index b1692d3e6b337..04cb7a7a5c19b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -103,10 +103,10 @@ struct VPlanTransforms {
   /// not valid.
   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
 
-  /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
-  /// If they do, try to update the vector loop to exit early if any input is
-  /// NaN and resume executing in the scalar loop to handle the NaNs there.
-  /// Return false if this attempt was unsuccessful.
+  /// Check if \p Plan contains any FMaxNum or FMinNum reductions. If they do,
+  /// try to update the vector loop to exit early if any input is NaN and resume
+  /// executing in the scalar loop to handle the NaNs there. Return false if
+  /// this attempt was unsuccessful.
   static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
 
   /// Clear NSW/NUW flags from reduction instructions if necessary.
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 6204d540df40c..49c25b02ce964 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -324,6 +324,122 @@ exit:
   ret float %max.next
 }
 
+define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_induction_starts_at_10(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 10, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 10, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret float %max.next
+}
+
+define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_induction_starts_at_value(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret float %max.next
+}
+
+define float @fmaxnum_with_additional_add(ptr noalias %src, ptr noalias %src.2, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_with_additional_add(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[SRC_2:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC_2]], i64 [[IV]]
+; CHECK-NEXT:    [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4
+; CHECK-NEXT:    [[SUM_NEXT]] = add i32 [[SUM]], [[L_SRC_2]]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    store i32 [[SUM_NEXT_LCSSA]], ptr [[SRC_2]], align 4
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %sum.next, %loop ]
+  %gep.src.2 = getelementptr inbounds nuw i32, ptr %src.2, i64 %iv
+  %l.src.2 = load i32, ptr %gep.src.2, align 4
+  %sum.next = add i32 %sum, %l.src.2
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  store i32 %sum.next, ptr %src.2
+  ret float %max.next
+}
+
 define float @fmax_select_with_blend(ptr %A, ptr %B) {
 ; CHECK-LABEL: define float @fmax_select_with_blend(
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {


From flang-commits at lists.llvm.org  Fri Jul 18 09:58:13 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 09:58:13 -0700 (PDT)
Subject: [flang-commits] [flang] 2c2567d - [flang] Fixed a crash with
 undeclared variable in implicit-do loop (#149513)
Message-ID: <687a7d25.a70a0220.22c9e4.bd4d@mx.google.com>


Author: Eugene Epshteyn
Date: 2025-07-18T12:58:09-04:00
New Revision: 2c2567da95c64ee1ed9104ee2539894242922b83

URL: https://github.com/llvm/llvm-project/commit/2c2567da95c64ee1ed9104ee2539894242922b83
DIFF: https://github.com/llvm/llvm-project/commit/2c2567da95c64ee1ed9104ee2539894242922b83.diff

LOG: [flang] Fixed a crash with undeclared variable in implicit-do loop (#149513)

Fixed a crash in the following example:
```
subroutine sub()
  implicit none
  print *, (i, i = 1, 2)  ! Problem: using undefined var in implied-do loop
end subroutine sub
```
The error message was already generated, but the compiler crashed before
it could display it.

Added: 
    

Modified: 
    flang/lib/Semantics/check-do-forall.cpp
    flang/test/Semantics/resolve40.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index cc1d4bf58745a..e258df86a4b1c 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -1180,7 +1180,9 @@ void DoForallChecker::Leave(const parser::IoControlSpec &ioControlSpec) {
 void DoForallChecker::Leave(const parser::OutputImpliedDo &outputImpliedDo) {
   const auto &control{std::get<parser::IoImpliedDoControl>(outputImpliedDo.t)};
   const parser::Name &name{control.name.thing.thing};
-  context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  if (name.symbol) {
+    context_.CheckIndexVarRedefine(name.source, *name.symbol);
+  }
 }
 
 void DoForallChecker::Leave(const parser::StatVariable &statVariable) {

diff  --git a/flang/test/Semantics/resolve40.f90 b/flang/test/Semantics/resolve40.f90
index a91507aa62282..81bb5f989ec48 100644
--- a/flang/test/Semantics/resolve40.f90
+++ b/flang/test/Semantics/resolve40.f90
@@ -96,3 +96,10 @@ subroutine s12(x)
   !BECAUSE: 'x' is an INTENT(IN) dummy argument
   read(*,nml=nl)
 end
+
+subroutine s13()
+  implicit none
+  !ERROR: No explicit type declared for 'i'
+  !ERROR: No explicit type declared for 'i'
+  print *, (i, i = 1, 2)
+end


        

From flang-commits at lists.llvm.org  Fri Jul 18 09:58:19 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Fri, 18 Jul 2025 09:58:19 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Fixed a crash with undeclared
 variable in implicit-do loop (PR #149513)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149513@github.com>
Message-ID: <687a7d2b.630a0220.226a44.b464@mx.google.com>


https://github.com/eugeneepshteyn closed https://github.com/llvm/llvm-project/pull/149513

From flang-commits at lists.llvm.org  Fri Jul 18 10:11:10 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 10:11:10 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a802e.170a0220.35d9d8.be42@mx.google.com>


================
@@ -103,6 +103,12 @@ struct VPlanTransforms {
   /// not valid.
   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
 
+  /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
+  /// If they do, try to update the vector loop to exit early if any input is
+  /// NaN and resume executing in the scalar loop to handle the NaNs there.
+  /// Return false if this attempt was unsuccessful.
+  static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
----------------
ayalz wrote:

Worth shortening, being the general case?

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 10:12:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 10:12:57 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687a8099.170a0220.d296b.bd82@mx.google.com>


https://github.com/ayalz approved this pull request.

Thanks, now it LGTM!

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 12:34:07 2025
From: flang-commits at lists.llvm.org (Carlo Bramini via flang-commits)
Date: Fri, 18 Jul 2025 12:34:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][CMake] CYGWIN: Fix undefined
 references at link time. (PR #67105)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/67105@github.com>
Message-ID: <687aa1af.170a0220.129548.d58d@mx.google.com>


carlo-bramini wrote:

Than you very much for your replies.

> @carlo-bramini do you have write access?

No, I have not.

> If not, you can ask the reviewer to merge it for you, but you should change to a proper email address first: https://llvm.org/docs/DeveloperPolicy.html#email-addresses https://github.com/llvm/llvm-project/blob/b8264293a714347a77f150b109cfdde8665eeadc/.github/workflows/email-check.yaml#L35C41-L35C99

The "Keep my email addresses private" option in my setting is already OFF.

https://github.com/llvm/llvm-project/pull/67105

From flang-commits at lists.llvm.org  Fri Jul 18 12:57:23 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Fri, 18 Jul 2025 12:57:23 -0700 (PDT)
Subject: [flang-commits] [flang] early feedback commit (PR #149579)
Message-ID: <llvm.org/llvm/llvm-project/pull/149579@github.com>


https://github.com/akuhlens created https://github.com/llvm/llvm-project/pull/149579

WIP Just posting for early feedback

>From 757f85b5743c86e81d3b28bd54b6c85d885fbfeb Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <akuhlenschmi at nvidia.com>
Date: Fri, 18 Jul 2025 12:56:11 -0700
Subject: [PATCH] early feedback commit

---
 flang/lib/Semantics/check-acc-structure.cpp   | 153 ++++++++++++++++--
 flang/lib/Semantics/check-acc-structure.h     |  17 ++
 .../Semantics/OpenACC/acc-atomic-validity.f90 |   1 +
 3 files changed, 161 insertions(+), 10 deletions(-)

diff --git a/flang/lib/Semantics/check-acc-structure.cpp b/flang/lib/Semantics/check-acc-structure.cpp
index 9cbea9742a6a2..a1b877c64b968 100644
--- a/flang/lib/Semantics/check-acc-structure.cpp
+++ b/flang/lib/Semantics/check-acc-structure.cpp
@@ -7,8 +7,13 @@
 //===----------------------------------------------------------------------===//
 #include "check-acc-structure.h"
 #include "flang/Common/enum-set.h"
+#include "flang/Evaluate/tools.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Support/Fortran.h"
+#include "llvm/Support/raw_ostream.h"
+#include <optional>
 
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
   void AccStructureChecker::Enter(const parser::AccClause::X &) { \
@@ -342,21 +347,149 @@ void AccStructureChecker::Leave(const parser::OpenACCAtomicConstruct &x) {
   dirContext_.pop_back();
 }
 
-void AccStructureChecker::Enter(const parser::AccAtomicUpdate &x) {
-  const parser::AssignmentStmt &assignment{
-      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement};
-  const auto &var{std::get<parser::Variable>(assignment.t)};
-  const auto &expr{std::get<parser::Expr>(assignment.t)};
+void AccStructureChecker::CheckAtomicStmt(
+    const parser::AssignmentStmt &assign, std::string &construct) {
+  const auto &var{std::get<parser::Variable>(assign.t)};
+  const auto &expr{std::get<parser::Expr>(assign.t)};
   const auto *rhs{GetExpr(context_, expr)};
   const auto *lhs{GetExpr(context_, var)};
-  if (lhs && rhs) {
-    if (lhs->Rank() != 0)
+
+  if (lhs) {
+    if (lhs->Rank() != 0) {
       context_.Say(expr.source,
-          "LHS of atomic update statement must be scalar"_err_en_US);
-    if (rhs->Rank() != 0)
+          "LHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+  if (rhs) {
+    if (rhs->Rank() != 0) {
       context_.Say(var.GetSource(),
-          "RHS of atomic update statement must be scalar"_err_en_US);
+          "RHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+}
+
+void AccStructureChecker::CheckAtomicUpdateStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"update"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicWriteStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"write"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicCaptureStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"capture"};
+  CheckAtomicStmt(assign, construct);
+}
+
+const parser::Variable *AccStructureChecker::GetIfAtomicUpdateVar(
+    const parser::AssignmentStmt &assign) const {
+  // OpenACC 3.4, 2893-2898
+  const auto &updatedVar{std::get<parser::Variable>(assign.t)};
+  const auto &rhs{std::get<parser::Expr>(assign.t)};
+
+  // Is the rhs something a valid operations that references the updatedVar?
+  const auto expr{GetExpr(context_, rhs)};
+  if (!expr) {
+    llvm::errs() << "IsAtomicUpdateStmt: no expr\n";
+    return nullptr;
+  }
+  const auto [op, args] = evaluate::GetTopLevelOperation(*expr);
+  switch (op) {
+  // 2909-2910 operator is one of +, *, -, /, .and., .or., .eqv., or .neqv..
+  case evaluate::operation::Operator::Add:
+  case evaluate::operation::Operator::Mul:
+  case evaluate::operation::Operator::Sub:
+  case evaluate::operation::Operator::Div:
+  case evaluate::operation::Operator::And:
+  case evaluate::operation::Operator::Or:
+  case evaluate::operation::Operator::Eqv:
+  case evaluate::operation::Operator::Neqv:
+  // 2909 intrinsic-procedure name is one of max, min, iand, ior, or ieor.
+  case evaluate::operation::Operator::Max:
+  case evaluate::operation::Operator::Min:
+    // case evaluate::operation::Operator::Iand:
+    // case evaluate::operation::Operator::Ior:
+    // case evaluate::operation::Operator::Ieor:
+    break;
+  // x = (x) is not allowed.
+  case evaluate::operation::Operator::Identity:
+  default:
+    return nullptr;
+  }
+  const auto &updatedExpr{GetExpr(context_, updatedVar)};
+  if (!updatedExpr) {
+    return nullptr;
+  }
+  for (const auto &arg : args) {
+    if (*updatedExpr == arg) {
+      return &updatedVar;
+    }
   }
+  return nullptr;
+}
+
+bool AccStructureChecker::IsAtomicCaptureStmt(
+    const parser::AssignmentStmt &assign,
+    const parser::Variable &updatedVar) const {
+  const auto &var{std::get<parser::Variable>(assign.t)};
+  const auto &expr{std::get<parser::Expr>(assign.t)};
+  const auto *varExpr{GetExpr(context_, var)};
+  const auto *updatedVarExpr{GetExpr(context_, updatedVar)};
+  const auto *exprExpr{GetExpr(context_, expr)};
+  if (!varExpr || !updatedVarExpr || !exprExpr) {
+    return false;
+  }
+  return updatedVarExpr == exprExpr && !(updatedVarExpr == varExpr);
+}
+
+void AccStructureChecker::Enter(const parser::AccAtomicCapture &capture) {
+  const Fortran::parser::AssignmentStmt &stmt1 =
+      std::get<Fortran::parser::AccAtomicCapture::Stmt1>(capture.t).v.statement;
+  const Fortran::parser::AssignmentStmt &stmt2 =
+      std::get<Fortran::parser::AccAtomicCapture::Stmt2>(capture.t).v.statement;
+  if (const parser::Variable *updatedVar{GetIfAtomicUpdateVar(stmt1)};
+      updatedVar && IsAtomicCaptureStmt(stmt2, *updatedVar)) {
+    CheckAtomicUpdateStmt(stmt1);
+    CheckAtomicCaptureStmt(stmt2);
+  } else if (const parser::Variable *updatedVar{GetIfAtomicUpdateVar(stmt2)};
+      updatedVar && IsAtomicCaptureStmt(stmt1, *updatedVar)) {
+    CheckAtomicCaptureStmt(stmt1);
+    CheckAtomicUpdateStmt(stmt2);
+  } else {
+    // Note, the write-statement when unified with is just an assignment
+    // statement x is unconstrained. This is moral equivalent of `if (...
+    // updatedVar{IsAtomicWriteStmt(stmt2)}; updatedVar && ...)`
+    const auto &updatedVarV{std::get<parser::Variable>(stmt2.t)};
+    if (IsAtomicCaptureStmt(stmt1, updatedVarV)) {
+      CheckAtomicCaptureStmt(stmt1);
+      CheckAtomicWriteStmt(stmt2);
+    } else {
+      context_.Say(std::get<parser::Verbatim>(capture.t).source,
+          "The assignments in this atomic capture construct do not update a variable and capture either its initial or final value"_err_en_US);
+    }
+  }
+}
+
+void AccStructureChecker::Enter(const parser::AccAtomicUpdate &x) {
+  CheckAtomicUpdateStmt(
+      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement);
+}
+
+void AccStructureChecker::Enter(const parser::AccAtomicWrite &x) {
+  CheckAtomicWriteStmt(
+      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement);
+}
+
+void AccStructureChecker::Enter(const parser::AccAtomicRead &x) {
+  CheckAtomicCaptureStmt(
+      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement);
 }
 
 void AccStructureChecker::Enter(const parser::OpenACCCacheConstruct &x) {
diff --git a/flang/lib/Semantics/check-acc-structure.h b/flang/lib/Semantics/check-acc-structure.h
index 6a9aa01a9bb13..207311be85973 100644
--- a/flang/lib/Semantics/check-acc-structure.h
+++ b/flang/lib/Semantics/check-acc-structure.h
@@ -63,6 +63,9 @@ class AccStructureChecker
   void Enter(const parser::OpenACCCacheConstruct &);
   void Leave(const parser::OpenACCCacheConstruct &);
   void Enter(const parser::AccAtomicUpdate &);
+  void Enter(const parser::AccAtomicCapture &);
+  void Enter(const parser::AccAtomicWrite &);
+  void Enter(const parser::AccAtomicRead &);
   void Enter(const parser::OpenACCEndConstruct &);
 
   // Clauses
@@ -80,6 +83,20 @@ class AccStructureChecker
 #include "llvm/Frontend/OpenACC/ACC.inc"
 
 private:
+  bool IsAtomicUpdateOperator(
+      const parser::Expr &expr, const parser::Variable &updatedVar) const;
+  bool IsAtomicUpdateIntrinsic(
+      const parser::Expr &expr, const parser::Variable &updatedVar) const;
+  const parser::Variable *GetIfAtomicUpdateVar(
+      const parser::AssignmentStmt &assign) const;
+  bool IsAtomicCaptureStmt(const parser::AssignmentStmt &assign,
+      const parser::Variable &updatedVar) const;
+  void CheckAtomicStmt(
+      const parser::AssignmentStmt &assign, std::string &construct);
+  void CheckAtomicUpdateStmt(const parser::AssignmentStmt &assign);
+  void CheckAtomicCaptureStmt(const parser::AssignmentStmt &assign);
+  void CheckAtomicWriteStmt(const parser::AssignmentStmt &assign);
+
   bool CheckAllowedModifier(llvm::acc::Clause clause);
   bool IsComputeConstruct(llvm::acc::Directive directive) const;
   bool IsInsideComputeConstruct() const;
diff --git a/flang/test/Semantics/OpenACC/acc-atomic-validity.f90 b/flang/test/Semantics/OpenACC/acc-atomic-validity.f90
index 07fb864695737..6e3cab41749db 100644
--- a/flang/test/Semantics/OpenACC/acc-atomic-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-atomic-validity.f90
@@ -54,6 +54,7 @@ program openacc_atomic_validity
   i = c(i)
   !$acc end atomic
 
+  !TODO: Should error because c(i) references i which is the atomic update variable.
   !$acc atomic capture
   c(i) = i
   i = i + 1


From flang-commits at lists.llvm.org  Fri Jul 18 13:03:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 13:03:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][CMake] CYGWIN: Fix undefined
 references at link time. (PR #67105)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/67105@github.com>
Message-ID: <687aa8ad.170a0220.2b0b4d.a087@mx.google.com>


jeremyd2019 wrote:

you might need to amend the commit with
`git commit --amend  --reset-author` and force-push to get the authorship right then

https://github.com/llvm/llvm-project/pull/67105

From flang-commits at lists.llvm.org  Fri Jul 18 12:44:48 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 12:44:48 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687aa430.050a0220.3bf9dc.cf9e@mx.google.com>


https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/148239

>From 42d11944daf17b42cc9eae6a1b14e25740f37de4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 18 Jun 2025 14:07:04 +0100
Subject: [PATCH 1/6] [LV] Vectorize maxnum/minnum w/o fast-math flags.

Update LV to vectorize maxnum/minnum reductions without fast-math flags,
by adding an extra check in the loop if any inputs to maxnum/minnum are
NaN. If any input is NaN,
 *exit the vector loop,
 *compute the reduction result up to the vector iteration that contained NaN
  inputs and
 * resume in the scalar loop

New recurrence kinds are added for reductions using maxnum/minnum
without fast-math flags. The new recurrence kinds are not supported in
the code to generate IR to perform the reductions to prevent accidential
mis-use. Users need to add the required checks ensuring no NaN inputs,
and convert to regular FMin/FMax recurrence kinds.
---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   2 +
 llvm/lib/Analysis/IVDescriptors.cpp           |  23 ++-
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   2 +
 .../Transforms/Vectorize/LoopVectorize.cpp    |  14 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |   6 +
 llvm/lib/Transforms/Vectorize/VPlan.h         |   8 +-
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   1 +
 .../Vectorize/VPlanConstruction.cpp           | 137 ++++++++++++++++++
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |   8 +-
 .../Transforms/Vectorize/VPlanTransforms.h    |   5 +
 .../AArch64/fmax-without-fast-math-flags.ll   |  54 ++++++-
 .../AArch64/fmin-without-fast-math-flags.ll   |  54 ++++++-
 ...fmax-without-fast-math-flags-interleave.ll |  54 ++++++-
 .../fmax-without-fast-math-flags.ll           |  90 ++++++++++--
 .../fmin-without-fast-math-flags.ll           |  90 ++++++++++--
 .../LoopVectorize/minmax_reduction.ll         |   6 +-
 16 files changed, 492 insertions(+), 62 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index b985292ccee40..e28901c072e29 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,6 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
+  FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
+  FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 39f74beca082f..ef095b73d5419 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -941,10 +941,27 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                   m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
             match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
     };
-    if (isIntMinMaxRecurrenceKind(Kind) ||
-        (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+    if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
-    else if (isFMulAddIntrinsic(I))
+    if (isFPMinMaxRecurrenceKind(Kind)) {
+      if (HasRequiredFMF())
+        return isMinMaxPattern(I, Kind, Prev);
+      // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
+      // intrinsics with extra checks ensuring the inputs are not NaN.
+      auto *StartV = dyn_cast<ConstantFP>(
+          OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
+      if (StartV && !StartV->getValue().isNaN() &&
+          isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
+        if (((Kind == RecurKind::FMax &&
+              match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
+             Kind == RecurKind::FMaxNumNoFMFs))
+          return InstDesc(I, RecurKind::FMaxNumNoFMFs);
+        if (((Kind == RecurKind::FMin &&
+              match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
+             Kind == RecurKind::FMinNumNoFMFs))
+          return InstDesc(I, RecurKind::FMinNumNoFMFs);
+      }
+    } else if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
     return InstDesc(false, I);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 200d1fb854155..0cf7bc7af878c 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -938,8 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
   case RecurKind::UMin:
     return Intrinsic::vector_reduce_umin;
   case RecurKind::FMax:
+  case RecurKind::FMaxNumNoFMFs:
     return Intrinsic::vector_reduce_fmax;
   case RecurKind::FMin:
+  case RecurKind::FMinNumNoFMFs:
     return Intrinsic::vector_reduce_fmin;
   case RecurKind::FMaximum:
     return Intrinsic::vector_reduce_fmaximum;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ceeabd65cced3..b070b04cc3c35 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4347,8 +4347,15 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
   // Cross iteration phis such as reductions need special handling and are
   // currently unsupported.
-  if (any_of(OrigLoop->getHeader()->phis(),
-             [&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
+  if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
+        if (Legal->isReductionVariable(&Phi)) {
+          RecurKind RK =
+              Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
+          return RK == RecurKind::FMinNumNoFMFs ||
+                 RK == RecurKind::FMaxNumNoFMFs;
+        }
+        return Legal->isFixedOrderRecurrence(&Phi);
+      }))
     return false;
 
   // Phis with uses outside of the loop require special handling and are
@@ -8769,6 +8776,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
 
   // Adjust the recipes for any inloop reductions.
   adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+  if (!VPlanTransforms::runPass(
+          VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
+    return nullptr;
 
   // Transform recipes to abstract recipes if it is legal and beneficial and
   // clamp the range for better cost estimation.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index da6af353c709f..b3040d0317c95 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23193,6 +23193,8 @@ class HorizontalReduction {
         case RecurKind::FindFirstIVUMin:
         case RecurKind::FindLastIVSMax:
         case RecurKind::FindLastIVUMax:
+        case RecurKind::FMaxNumNoFMFs:
+        case RecurKind::FMinNumNoFMFs:
         case RecurKind::FMaximumNum:
         case RecurKind::FMinimumNum:
         case RecurKind::None:
@@ -23330,6 +23332,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
+    case RecurKind::FMaxNumNoFMFs:
+    case RecurKind::FMinNumNoFMFs:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
@@ -23432,6 +23436,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
+    case RecurKind::FMaxNumNoFMFs:
+    case RecurKind::FMinNumNoFMFs:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 204268e586b43..70c34ec3c25cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1356,16 +1356,16 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
   unsigned Opcode;
 
 public:
+  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+        Opcode(I.getOpcode()) {}
+
   VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags, const VPIRMetadata &Metadata,
                 DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
         VPIRMetadata(Metadata), Opcode(Opcode) {}
 
-  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
-      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
-        Opcode(I.getOpcode()) {}
-
   ~VPWidenRecipe() override = default;
 
   VPWidenRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index b27a7ffeed208..66657b98b094b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -84,6 +84,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
     return ResTy;
   }
   case Instruction::ICmp:
+  case Instruction::FCmp:
   case VPInstruction::ActiveLaneMask:
     assert(inferScalarType(R->getOperand(0)) ==
                inferScalarType(R->getOperand(1)) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 7fb5e82f9d32b..69efb9f89b479 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -652,3 +652,140 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
+                                      VPReductionPHIRecipe *RedPhi) {
+  auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+  if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+      !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+    return nullptr;
+
+  if (MinMaxOp->getOperand(0) == RedPhi)
+    return MinMaxOp->getOperand(1);
+  return MinMaxOp->getOperand(0);
+}
+
+/// Returns true if there VPlan is read-only and execution can be resumed at the
+/// beginning of the last vector iteration in the scalar loop
+static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
+    if (!VPBB)
+      return false;
+    for (auto &R : *VPBB) {
+      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        continue;
+      if (R.mayWriteToMemory())
+        return false;
+    }
+  }
+  return true;
+}
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPValue *AnyNaN = nullptr;
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  VPRecipeWithIRFlags *MinMaxOp = nullptr;
+  bool HasUnsupportedPhi = false;
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
+    if (RedPhiR)
+      return false;
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+      continue;
+
+    RedPhiR = Cur;
+    MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxOp)
+      return false;
+    VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
+    if (!In)
+      return false;
+
+    auto *IsNaN =
+        new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
+    IsNaN->insertBefore(MinMaxOp);
+    AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
+    AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
+  }
+
+  if (!AnyNaN)
+    return true;
+
+  if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+    return false;
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
+  auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
+  if (!RdxResult ||
+      RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
+      RdxResult->getOperand(0) != RedPhiR)
+    return false;
+
+  auto *ScalarPH = Plan.getScalarPreheader();
+  // Update the resume phis in the scalar preheader. They all must either resume
+  // from the reduction result or the canonical induction. Bail out if there are
+  // other resume phis.
+  for (auto &R : ScalarPH->phis()) {
+    auto *ResumeR = cast<VPPhi>(&R);
+    VPValue *VecV = ResumeR->getOperand(0);
+    VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
+    if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
+      return false;
+    ResumeR->setOperand(
+        1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
+    ResumeR->addOperand(BypassV);
+  }
+
+  // Create a new reduction phi recipe with either FMin/FMax, replacing
+  // FMinNumNoFMFs/FMaxNumNoFMFs.
+  RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
+                        ? RecurKind::FMin
+                        : RecurKind::FMax;
+  auto *NewRedPhiR = new VPReductionPHIRecipe(
+      cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
+      *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
+  NewRedPhiR->addOperand(RedPhiR->getOperand(1));
+  NewRedPhiR->insertBefore(RedPhiR);
+  RedPhiR->replaceAllUsesWith(NewRedPhiR);
+  RedPhiR->eraseFromParent();
+
+  // Update the loop exit condition to exit if either any of the inputs is NaN
+  // or the vector trip count is reached.
+  VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
+  VPBuilder Builder(LatchVPBB->getTerminator());
+  auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+  assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
+         "Unexpected terminator");
+  auto *IsLatchExitTaken =
+      Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
+                         LatchExitingBranch->getOperand(1));
+  auto *AnyExitTaken =
+      Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
+  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
+  LatchExitingBranch->eraseFromParent();
+
+  // Split the middle block and introduce a new block, branching to the scalar
+  // preheader to resume iteration in the scalar loop if any NaNs have been
+  // encountered.
+  MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
+  Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
+  auto *NewSel =
+      Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+  RdxResult->setOperand(1, NewSel);
+  Builder.setInsertPoint(MiddleVPBB);
+  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
+  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
+  MiddleVPBB->swapSuccessors();
+  std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
+  return true;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1664bcc3881aa..57b713d3dfcb9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -587,6 +587,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     Value *Op = State.get(getOperand(0), vputils::onlyFirstLaneUsed(this));
     return Builder.CreateFreeze(Op, Name);
   }
+  case Instruction::FCmp:
   case Instruction::ICmp: {
     bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
     Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
@@ -860,7 +861,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     Value *Res = State.get(getOperand(0));
     for (VPValue *Op : drop_begin(operands()))
       Res = Builder.CreateOr(Res, State.get(Op));
-    return Builder.CreateOrReduce(Res);
+    return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res);
   }
   case VPInstruction::FirstActiveLane: {
     if (getNumOperands() == 1) {
@@ -1033,6 +1034,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   switch (getOpcode()) {
   case Instruction::ExtractElement:
   case Instruction::Freeze:
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
   case VPInstruction::AnyOf:
@@ -1068,6 +1070,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
     return Op == getOperand(1);
   case Instruction::PHI:
     return true;
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
   case Instruction::Or:
@@ -1100,6 +1103,7 @@ bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const {
   switch (getOpcode()) {
   default:
     return false;
+  case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
     return vputils::onlyFirstPartUsed(this);
@@ -1786,7 +1790,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
     return Opcode == Instruction::ZExt;
     break;
   case OperationType::Cmp:
-    return Opcode == Instruction::ICmp;
+    return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
   case OperationType::Other:
     return true;
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 84a12470f45e4..52e33ad0d4f99 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -103,6 +103,11 @@ struct VPlanTransforms {
   /// not valid.
   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
 
+  /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
+  /// If they do, try to update the vector loop to exit early if any input is
+  /// NaN and resume executing in the scalar loop to handle the NaNs there.
+  static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
+
   /// Clear NSW/NUW flags from reduction instructions if necessary.
   static void clearReductionWrapFlags(VPlan &Plan);
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 451574a258c2b..1a9752a1ef9fd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -42,18 +42,58 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index e93ee5563b057..3deaa5f0e1f60 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -42,18 +42,58 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0x7FF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index b2e080fef2e57..8ef5b2720e4cd 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -42,18 +42,58 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 5661406b88a5a..332f3a60c5b3e 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -192,18 +192,49 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum_1(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -227,18 +258,49 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fmaxnum_2(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 148beb64a3609..0e3566b18432a 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -192,18 +192,49 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum_1(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[L]], float [[MAX]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -227,18 +258,49 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-LABEL: define float @fminnum_2(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
 ; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index 85a90f2e04c5e..1429dc26c5207 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -1001,8 +1001,9 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
+; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmin_intrinsic_nofast(
-; CHECK-NOT: <2 x float> @llvm.minnum.v2f32
+; CHECK: <2 x float> @llvm.minnum.v2f32
 define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body
@@ -1021,8 +1022,9 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
+; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmax_intrinsic_nofast(
-; CHECK-NOT: <2 x float> @llvm.maxnum.v2f32
+; CHECK: <2 x float> @llvm.maxnum.v2f32
 define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body

>From de2d6f2c89a3ead527a03c23eaaa4b7ce2ade11b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Jul 2025 09:53:59 +0100
Subject: [PATCH 2/6] !fixup address latest comments, thanks

---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   4 +-
 llvm/lib/Analysis/IVDescriptors.cpp           |  31 ++--
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   4 +-
 .../Vectorize/LoopVectorizationPlanner.h      |  12 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  10 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  12 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |   8 +-
 .../Vectorize/VPlanConstruction.cpp           | 144 +++++++++---------
 .../Transforms/Vectorize/VPlanTransforms.h    |   1 +
 .../AArch64/fmax-without-fast-math-flags.ll   |  26 ++--
 .../AArch64/fmin-without-fast-math-flags.ll   |  26 ++--
 ...fmax-without-fast-math-flags-interleave.ll |  26 ++--
 .../fmax-without-fast-math-flags.ll           |  44 +++---
 .../fmin-without-fast-math-flags.ll           |  44 +++---
 .../LoopVectorize/minmax_reduction.ll         |   2 +
 15 files changed, 212 insertions(+), 182 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index e28901c072e29..84a7672cd47f3 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,8 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
-  FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
-  FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
+  FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
+  FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index ef095b73d5419..accda5a9713fc 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -944,23 +944,24 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
     if (isIntMinMaxRecurrenceKind(Kind))
       return isMinMaxPattern(I, Kind, Prev);
     if (isFPMinMaxRecurrenceKind(Kind)) {
+      InstDesc Res = isMinMaxPattern(I, Kind, Prev);
       if (HasRequiredFMF())
-        return isMinMaxPattern(I, Kind, Prev);
+        return Res;
+
+      if (!Res.isRecurrence())
+        return InstDesc(false, I);
+
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
-      // intrinsics with extra checks ensuring the inputs are not NaN.
-      auto *StartV = dyn_cast<ConstantFP>(
-          OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
-      if (StartV && !StartV->getValue().isNaN() &&
-          isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
-        if (((Kind == RecurKind::FMax &&
-              match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
-             Kind == RecurKind::FMaxNumNoFMFs))
-          return InstDesc(I, RecurKind::FMaxNumNoFMFs);
-        if (((Kind == RecurKind::FMin &&
-              match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
-             Kind == RecurKind::FMinNumNoFMFs))
-          return InstDesc(I, RecurKind::FMinNumNoFMFs);
-      }
+      // intrinsics with extra checks ensuring the vector loop handles only
+      // non-NaN inputs.
+      if (Kind == RecurKind::FMax &&
+          match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+        return InstDesc(I, RecurKind::FMaxNum);
+      if (Kind == RecurKind::FMin &&
+          match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+        return InstDesc(I, RecurKind::FMinNum);
+
+      return InstDesc(false, I);
     } else if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 0cf7bc7af878c..7bb7a7b116384 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -938,10 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
   case RecurKind::UMin:
     return Intrinsic::vector_reduce_umin;
   case RecurKind::FMax:
-  case RecurKind::FMaxNumNoFMFs:
+  case RecurKind::FMaxNum:
     return Intrinsic::vector_reduce_fmax;
   case RecurKind::FMin:
-  case RecurKind::FMinNumNoFMFs:
+  case RecurKind::FMinNum:
     return Intrinsic::vector_reduce_fmin;
   case RecurKind::FMaximum:
     return Intrinsic::vector_reduce_fmaximum;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 11853859484e3..f57ce0c3ccb4d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -230,7 +230,6 @@ class VPBuilder {
 
   /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
   /// and \p B.
-  /// TODO: add createFCmp when needed.
   VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
                             DebugLoc DL = DebugLoc::getUnknown(),
                             const Twine &Name = "") {
@@ -240,6 +239,17 @@ class VPBuilder {
         new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name));
   }
 
+  /// Create a new FCmp VPInstruction with predicate \p Pred and operands \p A
+  /// and \p B.
+  VPInstruction *createFCmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
+                            DebugLoc DL = DebugLoc::getUnknown(),
+                            const Twine &Name = "") {
+    assert(Pred >= CmpInst::FIRST_FCMP_PREDICATE &&
+           Pred <= CmpInst::LAST_FCMP_PREDICATE && "invalid predicate");
+    return tryInsertInstruction(
+        new VPInstruction(Instruction::FCmp, {A, B}, Pred, DL, Name));
+  }
+
   VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
                               DebugLoc DL = DebugLoc::getUnknown(),
                               const Twine &Name = "") {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b070b04cc3c35..712c5f8eb4c56 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4345,14 +4345,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
 
 bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
-  // Cross iteration phis such as reductions need special handling and are
-  // currently unsupported.
+  // Cross iteration phis such as first-order recurrences and FMaxNum/FMinNum
+  // reductions need special handling and are currently unsupported.
   if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
         if (Legal->isReductionVariable(&Phi)) {
           RecurKind RK =
               Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
-          return RK == RecurKind::FMinNumNoFMFs ||
-                 RK == RecurKind::FMaxNumNoFMFs;
+          return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
         }
         return Legal->isFixedOrderRecurrence(&Phi);
       }))
@@ -8776,6 +8775,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
 
   // Adjust the recipes for any inloop reductions.
   adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+
+  // Apply mandatory transformation to handle FP maxnum/minnum reduction with
+  // NaNs if possible, bail out otherwise.
   if (!VPlanTransforms::runPass(
           VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
     return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b3040d0317c95..c1e4ff00a65e9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23193,8 +23193,8 @@ class HorizontalReduction {
         case RecurKind::FindFirstIVUMin:
         case RecurKind::FindLastIVSMax:
         case RecurKind::FindLastIVUMax:
-        case RecurKind::FMaxNumNoFMFs:
-        case RecurKind::FMinNumNoFMFs:
+        case RecurKind::FMaxNum:
+        case RecurKind::FMinNum:
         case RecurKind::FMaximumNum:
         case RecurKind::FMinimumNum:
         case RecurKind::None:
@@ -23332,8 +23332,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
-    case RecurKind::FMaxNumNoFMFs:
-    case RecurKind::FMinNumNoFMFs:
+    case RecurKind::FMaxNum:
+    case RecurKind::FMinNum:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
@@ -23436,8 +23436,8 @@ class HorizontalReduction {
     case RecurKind::FindFirstIVUMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
-    case RecurKind::FMaxNumNoFMFs:
-    case RecurKind::FMinNumNoFMFs:
+    case RecurKind::FMaxNum:
+    case RecurKind::FMinNum:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
     case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 70c34ec3c25cd..204268e586b43 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1356,16 +1356,16 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
   unsigned Opcode;
 
 public:
-  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
-      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
-        Opcode(I.getOpcode()) {}
-
   VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags, const VPIRMetadata &Metadata,
                 DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
         VPIRMetadata(Metadata), Opcode(Opcode) {}
 
+  VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+        Opcode(I.getOpcode()) {}
+
   ~VPWidenRecipe() override = default;
 
   VPWidenRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 69efb9f89b479..8d1ec5b68457e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -653,77 +653,75 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
   }
 }
 
-static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
-                                      VPReductionPHIRecipe *RedPhi) {
-  auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
-  if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
-      !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
-    return nullptr;
-
-  if (MinMaxOp->getOperand(0) == RedPhi)
-    return MinMaxOp->getOperand(1);
-  return MinMaxOp->getOperand(0);
-}
-
-/// Returns true if there VPlan is read-only and execution can be resumed at the
-/// beginning of the last vector iteration in the scalar loop
-static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
-  for (VPBlockBase *VPB : vp_depth_first_shallow(
-           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
-    auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
-    if (!VPBB)
-      return false;
-    for (auto &R : *VPBB) {
-      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
-        continue;
-      if (R.mayWriteToMemory())
-        return false;
-    }
-  }
-  return true;
-}
-
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-  VPValue *AnyNaN = nullptr;
   VPReductionPHIRecipe *RedPhiR = nullptr;
-  VPRecipeWithIRFlags *MinMaxOp = nullptr;
+  VPValue *MinMaxOp = nullptr;
   bool HasUnsupportedPhi = false;
+
+  auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
+                                  VPReductionPHIRecipe *RedPhi) -> VPValue * {
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+      return nullptr;
+
+    if (MinMaxOp->getOperand(0) == RedPhi)
+      return MinMaxOp->getOperand(1);
+    assert(MinMaxOp->getOperand(1) == RedPhi &&
+           "Reduction phi operand expected");
+    return MinMaxOp->getOperand(0);
+  };
+
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    // TODO: Also support first-order recurrence phis.
     HasUnsupportedPhi |=
         !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
              VPReductionPHIRecipe>(&R);
     auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
     if (!Cur)
       continue;
+    // For now, only a single reduction is supported.
+    // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
     if (RedPhiR)
       return false;
-    if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
-        Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNum)
       continue;
 
     RedPhiR = Cur;
-    MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
         RedPhiR->getBackedgeValue()->getDefiningRecipe());
-    if (!MinMaxOp)
+    if (!MinMaxR)
       return false;
-    VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
-    if (!In)
+    MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
+    if (!MinMaxOp)
       return false;
-
-    auto *IsNaN =
-        new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
-    IsNaN->insertBefore(MinMaxOp);
-    AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
-    AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
   }
 
-  if (!AnyNaN)
+  if (!RedPhiR)
     return true;
 
-  if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+  if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
+  /// Check if the vector loop of \p Plan can early exit and restart
+  /// execution of last vector iteration in the scalar loop. This requires all
+  /// recipes up to early exit point be side-effect free as they are
+  /// re-executed. Currently we check that the loop is free of any recipe that
+  /// may write to memory. Expected to operate on an early VPlan w/o nested
+  /// regions.
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = cast<VPBasicBlock>(VPB);
+    for (auto &R : *VPBB) {
+      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        continue;
+      if (R.mayWriteToMemory())
+        return false;
+    }
+  }
+
   auto *MiddleVPBB = Plan.getMiddleBlock();
   auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
   if (!RdxResult ||
@@ -731,24 +729,9 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
       RdxResult->getOperand(0) != RedPhiR)
     return false;
 
-  auto *ScalarPH = Plan.getScalarPreheader();
-  // Update the resume phis in the scalar preheader. They all must either resume
-  // from the reduction result or the canonical induction. Bail out if there are
-  // other resume phis.
-  for (auto &R : ScalarPH->phis()) {
-    auto *ResumeR = cast<VPPhi>(&R);
-    VPValue *VecV = ResumeR->getOperand(0);
-    VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
-    if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
-      return false;
-    ResumeR->setOperand(
-        1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
-    ResumeR->addOperand(BypassV);
-  }
-
   // Create a new reduction phi recipe with either FMin/FMax, replacing
-  // FMinNumNoFMFs/FMaxNumNoFMFs.
-  RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
+  // FMinNum/FMaxNum.
+  RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
                         ? RecurKind::FMin
                         : RecurKind::FMax;
   auto *NewRedPhiR = new VPReductionPHIRecipe(
@@ -769,23 +752,40 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   auto *IsLatchExitTaken =
       Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
                          LatchExitingBranch->getOperand(1));
+
+  VPValue *IsNaN = Builder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
+  VPValue *AnyNaN = Builder.createNaryOp(VPInstruction::AnyOf, {IsNaN});
   auto *AnyExitTaken =
       Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
   Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
   LatchExitingBranch->eraseFromParent();
 
-  // Split the middle block and introduce a new block, branching to the scalar
-  // preheader to resume iteration in the scalar loop if any NaNs have been
-  // encountered.
-  MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
+  // If we exit early due to NaNs, compute the final reduction result based on
+  // the reduction phi at the beginning of the last vector iteration.
   Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
   auto *NewSel =
       Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
   RdxResult->setOperand(1, NewSel);
-  Builder.setInsertPoint(MiddleVPBB);
-  Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
-  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
-  MiddleVPBB->swapSuccessors();
-  std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
+
+  auto *ScalarPH = Plan.getScalarPreheader();
+  // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+  // true, the resume from the start of the last vector iteration via the
+  // canonical IV, otherwise from the original value.
+  for (auto &R : ScalarPH->phis()) {
+    auto *ResumeR = cast<VPPhi>(&R);
+    VPValue *VecV = ResumeR->getOperand(0);
+    if (VecV == RdxResult)
+      continue;
+    if (VecV != &Plan.getVectorTripCount())
+      return false;
+    auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
+    ResumeR->setOperand(0, NewSel);
+  }
+
+  auto *MiddleTerm = MiddleVPBB->getTerminator();
+  Builder.setInsertPoint(MiddleTerm);
+  VPValue *MiddleCond = MiddleTerm->getOperand(0);
+  VPValue *NewCond = Builder.createAnd(MiddleCond, Builder.createNot(AnyNaN));
+  MiddleTerm->setOperand(0, NewCond);
   return true;
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 52e33ad0d4f99..b1692d3e6b337 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -106,6 +106,7 @@ struct VPlanTransforms {
   /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
   /// If they do, try to update the vector loop to exit early if any input is
   /// NaN and resume executing in the scalar loop to handle the NaNs there.
+  /// Return false if this attempt was unsuccessful.
   static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
 
   /// Clear NSW/NUW flags from reduction instructions if necessary.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 1a9752a1ef9fd..6810c90f0b829 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -57,31 +57,33 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index 3deaa5f0e1f60..a18714f94343b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -57,31 +57,33 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 8ef5b2720e4cd..5e60b32e3ae6b 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -57,31 +57,33 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 332f3a60c5b3e..5199bc2bad1da 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -204,25 +204,27 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -234,7 +236,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -270,25 +272,27 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -300,7 +304,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 0e3566b18432a..f220b900d4d0f 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -204,25 +204,27 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -234,7 +236,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
@@ -270,25 +272,27 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK:       [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -300,7 +304,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index 1429dc26c5207..e7ab02cd98a5e 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -1004,6 +1004,7 @@ for.body:                                         ; preds = %entry, %for.body
 ; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmin_intrinsic_nofast(
 ; CHECK: <2 x float> @llvm.minnum.v2f32
+; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
 define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body
@@ -1025,6 +1026,7 @@ for.body:                                         ; preds = %entry, %for.body
 ; This can be vectorized with additional runtime checks for NaNs.
 ; CHECK-LABEL: @fmax_intrinsic_nofast(
 ; CHECK: <2 x float> @llvm.maxnum.v2f32
+; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
 define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
 entry:
   br label %for.body

>From 8ebc7ff41e3f68c42a99492fce9a5f55e5f23bde Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Jul 2025 15:36:12 +0100
Subject: [PATCH 3/6] !fixup add assertion for intrinsic ID

---
 .../lib/Transforms/Vectorize/VPlanConstruction.cpp | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8d1ec5b68457e..21e3ec3e3f0fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -697,6 +697,20 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
     if (!MinMaxOp)
       return false;
+
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        Cur->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                       : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (isa<VPReplicateRecipe>(MinMaxR) &&
+            cast<IntrinsicInst>(
+                cast<VPReplicateRecipe>(MinMaxR)->getUnderlyingInstr())
+                    ->getIntrinsicID() == RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
   }
 
   if (!RedPhiR)

>From 914a0b4b32617cc63b1c902402fd670e4055267d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 10:27:24 +0100
Subject: [PATCH 4/6] !fixup address latest comments, thanks

---
 llvm/include/llvm/Analysis/IVDescriptors.h    |   5 +-
 llvm/lib/Analysis/IVDescriptors.cpp           |  18 +--
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   7 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  12 +-
 .../Vectorize/VPlanConstruction.cpp           | 108 ++++++++----------
 .../AArch64/fmax-without-fast-math-flags.ll   |   7 +-
 .../AArch64/fmin-without-fast-math-flags.ll   |   7 +-
 ...fmax-without-fast-math-flags-interleave.ll |   7 +-
 .../fmax-without-fast-math-flags.ll           |   4 +-
 .../fmin-without-fast-math-flags.ll           |   4 +-
 10 files changed, 88 insertions(+), 91 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 84a7672cd47f3..1dc73205a0ebb 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,8 +47,8 @@ enum class RecurKind {
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
   FMax,     ///< FP max implemented in terms of select(cmp()).
-  FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
-  FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
+  FMinNum,  ///< FP min with llvm.minnum semantics including NaNs.
+  FMaxNum,  ///< FP max with llvm.maxnum semantics including NaNs.
   FMinimum, ///< FP min with llvm.minimum semantics
   FMaximum, ///< FP max with llvm.maximum semantics
   FMinimumNum, ///< FP min with llvm.minimumnum semantics
@@ -252,6 +252,7 @@ class RecurrenceDescriptor {
   /// Returns true if the recurrence kind is a floating-point min/max kind.
   static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
     return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
+           Kind == RecurKind::FMinNum || Kind == RecurKind::FMaxNum ||
            Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
            Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum;
   }
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index accda5a9713fc..a88bbd7b65d53 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -945,21 +945,25 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
       return isMinMaxPattern(I, Kind, Prev);
     if (isFPMinMaxRecurrenceKind(Kind)) {
       InstDesc Res = isMinMaxPattern(I, Kind, Prev);
-      if (HasRequiredFMF())
-        return Res;
-
       if (!Res.isRecurrence())
         return InstDesc(false, I);
 
+      if (HasRequiredFMF())
+        return Res;
+
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
       // intrinsics with extra checks ensuring the vector loop handles only
       // non-NaN inputs.
-      if (Kind == RecurKind::FMax &&
-          match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+      if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMax &&
+               "unexpected recurrence kind for maxnum");
         return InstDesc(I, RecurKind::FMaxNum);
-      if (Kind == RecurKind::FMin &&
-          match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+      }
+      if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) {
+        assert(Kind == RecurKind::FMin &&
+               "unexpected recurrence kind for minnum");
         return InstDesc(I, RecurKind::FMinNum);
+      }
 
       return InstDesc(false, I);
     } else if (isFMulAddIntrinsic(I))
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 7bb7a7b116384..154f50acf3d1d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1039,8 +1039,10 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) {
   case RecurKind::SMax:
     return Intrinsic::smax;
   case RecurKind::FMin:
+  case RecurKind::FMinNum:
     return Intrinsic::minnum;
   case RecurKind::FMax:
+  case RecurKind::FMaxNum:
     return Intrinsic::maxnum;
   case RecurKind::FMinimum:
     return Intrinsic::minimum;
@@ -1098,7 +1100,8 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
                             Value *Right) {
   Type *Ty = Left->getType();
   if (Ty->isIntOrIntVectorTy() ||
-      (RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
+      (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
+       RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
        RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
     // TODO: Add float minnum/maxnum support when FMF nnan is set.
     Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
@@ -1310,6 +1313,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
   case RecurKind::UMin:
   case RecurKind::FMax:
   case RecurKind::FMin:
+  case RecurKind::FMinNum:
+  case RecurKind::FMaxNum:
   case RecurKind::FMinimum:
   case RecurKind::FMaximum:
   case RecurKind::FMinimumNum:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 712c5f8eb4c56..db0902c589d49 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4345,15 +4345,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
 
 bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
     ElementCount VF) const {
-  // Cross iteration phis such as first-order recurrences and FMaxNum/FMinNum
+  // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
   // reductions need special handling and are currently unsupported.
   if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
-        if (Legal->isReductionVariable(&Phi)) {
-          RecurKind RK =
-              Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
-          return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
-        }
-        return Legal->isFixedOrderRecurrence(&Phi);
+        if (!Legal->isReductionVariable(&Phi))
+          return Legal->isFixedOrderRecurrence(&Phi);
+        RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
+        return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
       }))
     return false;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 21e3ec3e3f0fe..a426d06b4befe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -656,25 +656,42 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   VPReductionPHIRecipe *RedPhiR = nullptr;
-  VPValue *MinMaxOp = nullptr;
   bool HasUnsupportedPhi = false;
 
-  auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
-                                  VPReductionPHIRecipe *RedPhi) -> VPValue * {
-    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
-    if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+  auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxR)
+      return nullptr;
+
+    auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+    if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
         !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
       return nullptr;
 
-    if (MinMaxOp->getOperand(0) == RedPhi)
-      return MinMaxOp->getOperand(1);
-    assert(MinMaxOp->getOperand(1) == RedPhi &&
+#ifndef NDEBUG
+    Intrinsic::ID RdxIntrinsicId =
+        RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+                                                           : Intrinsic::minnum;
+    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+                RdxIntrinsicId) ||
+           (RepR &&
+            cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+                RdxIntrinsicId) &&
+               "Intrinsic did not match recurrence kind");
+#endif
+
+    if (MinMaxR->getOperand(0) == RedPhiR)
+      return MinMaxR->getOperand(1);
+
+    assert(MinMaxR->getOperand(1) == RedPhiR &&
            "Reduction phi operand expected");
-    return MinMaxOp->getOperand(0);
+    return MinMaxR->getOperand(0);
   };
 
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
-    // TODO: Also support first-order recurrence phis.
+    // TODO: Also support fixed-order recurrence phis.
     HasUnsupportedPhi |=
         !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
              VPReductionPHIRecipe>(&R);
@@ -688,34 +705,23 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
         Cur->getRecurrenceKind() != RecurKind::FMinNum)
       continue;
-
     RedPhiR = Cur;
-    auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
-        RedPhiR->getBackedgeValue()->getDefiningRecipe());
-    if (!MinMaxR)
-      return false;
-    MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
-    if (!MinMaxOp)
-      return false;
-
-#ifndef NDEBUG
-    Intrinsic::ID RdxIntrinsicId =
-        Cur->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
-                                                       : Intrinsic::minnum;
-    assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
-            cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
-                RdxIntrinsicId) ||
-           (isa<VPReplicateRecipe>(MinMaxR) &&
-            cast<IntrinsicInst>(
-                cast<VPReplicateRecipe>(MinMaxR)->getUnderlyingInstr())
-                    ->getIntrinsicID() == RdxIntrinsicId) &&
-               "Intrinsic did not match recurrence kind");
-#endif
   }
 
   if (!RedPhiR)
     return true;
 
+  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+         "unsupported reduction");
+
+  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+  if (!MinMaxOp)
+    return false;
+
+  // We won't be able to resume execution in the scalar tail, if there are
+  // unsupported header phis or there is no scalar tail at all, due to
+  // tail-folding.
   if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
@@ -729,35 +735,12 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
            Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
     auto *VPBB = cast<VPBasicBlock>(VPB);
     for (auto &R : *VPBB) {
-      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
-        continue;
-      if (R.mayWriteToMemory())
+      if (R.mayWriteToMemory() &&
+          !match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
         return false;
     }
   }
 
-  auto *MiddleVPBB = Plan.getMiddleBlock();
-  auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
-  if (!RdxResult ||
-      RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
-      RdxResult->getOperand(0) != RedPhiR)
-    return false;
-
-  // Create a new reduction phi recipe with either FMin/FMax, replacing
-  // FMinNum/FMaxNum.
-  RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
-                        ? RecurKind::FMin
-                        : RecurKind::FMax;
-  auto *NewRedPhiR = new VPReductionPHIRecipe(
-      cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
-      *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
-  NewRedPhiR->addOperand(RedPhiR->getOperand(1));
-  NewRedPhiR->insertBefore(RedPhiR);
-  RedPhiR->replaceAllUsesWith(NewRedPhiR);
-  RedPhiR->eraseFromParent();
-
-  // Update the loop exit condition to exit if either any of the inputs is NaN
-  // or the vector trip count is reached.
   VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
   VPBuilder Builder(LatchVPBB->getTerminator());
   auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
@@ -776,9 +759,18 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
 
   // If we exit early due to NaNs, compute the final reduction result based on
   // the reduction phi at the beginning of the last vector iteration.
+  auto *RdxResult = find_singleton<VPSingleDefRecipe>(
+      RedPhiR->users(), [](VPUser *U, bool) -> VPSingleDefRecipe * {
+        auto *VPI = dyn_cast<VPInstruction>(U);
+        if (VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult)
+          return VPI;
+        return nullptr;
+      });
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
   Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
   auto *NewSel =
-      Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+      Builder.createSelect(AnyNaN, RedPhiR, RdxResult->getOperand(1));
   RdxResult->setOperand(1, NewSel);
 
   auto *ScalarPH = Plan.getScalarPreheader();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 6810c90f0b829..427a05cc1c843 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -50,8 +50,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index a18714f94343b..1a8e5940d88e7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -50,8 +50,8 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0x7FF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 5e60b32e3ae6b..a2eddad179216 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -50,8 +50,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 5199bc2bad1da..6204d540df40c 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -200,7 +200,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
@@ -268,7 +268,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index f220b900d4d0f..68bc8d0640a3f 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -200,7 +200,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
@@ -268,7 +268,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4

>From 18312dc501beba7c0c77b50c7f6e344f44174ee3 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 16:19:36 +0100
Subject: [PATCH 5/6] Merge remote-tracking branch 'origin/main' into
 lv-vectorize-fmaxnum-without-fmfs

---
 .../workflows/build-ci-container-windows.yml  |   2 -
 .github/workflows/build-ci-container.yml      |   2 -
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/Basic/BuiltinsAMDGPU.def  |   1 +
 clang/include/clang/Sema/Overload.h           |   2 -
 .../Core/PathSensitive/ExprEngine.h           |   7 +-
 clang/lib/AST/ByteCode/Descriptor.cpp         |   2 +
 clang/lib/AST/ByteCode/Descriptor.h           |   4 +
 clang/lib/AST/ByteCode/Disasm.cpp             |   1 +
 clang/lib/AST/ByteCode/Interp.cpp             |   5 +-
 clang/lib/AST/ByteCode/InterpBlock.h          |   5 -
 clang/lib/AST/ByteCode/InterpBuiltin.cpp      |   3 +-
 clang/lib/AST/ByteCode/InterpFrame.cpp        |   5 +
 clang/lib/AST/ByteCode/Pointer.h              |   5 +
 clang/lib/CodeGen/CGCall.cpp                  |  15 +-
 clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp   |   1 +
 clang/lib/Driver/ToolChain.cpp                |  40 +-
 clang/lib/Sema/SemaConcept.cpp                |   5 +
 clang/lib/Sema/SemaOverload.cpp               |  49 +-
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp  |  46 +-
 clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp |  48 --
 clang/test/AST/ByteCode/mutable.cpp           |  56 ++-
 clang/test/AST/ByteCode/unions.cpp            |  14 +
 clang/test/Analysis/div-zero-cxx20.cpp        |  61 +++
 clang/test/Analysis/div-zero.cpp              |  60 +++
 clang/test/CodeGen/64bit-swiftcall.c          |   2 +-
 clang/test/CodeGen/AArch64/byval-temp.c       |  16 +-
 .../AArch64/pure-scalable-args-empty-union.c  |   2 +-
 .../test/CodeGen/AArch64/pure-scalable-args.c |  42 +-
 .../AArch64/struct-coerce-using-ptr.cpp       |   4 +-
 ...-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c |   6 +-
 ...cle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp |   2 +-
 clang/test/CodeGen/LoongArch/bitint.c         |   6 +-
 clang/test/CodeGen/PowerPC/ppc64-vector.c     |   2 +-
 clang/test/CodeGen/RISCV/riscv-abi.cpp        |   8 +-
 .../RISCV/riscv-vector-callingconv-llvm-ir.c  |  10 +-
 .../riscv-vector-callingconv-llvm-ir.cpp      |  10 +-
 clang/test/CodeGen/RISCV/riscv32-abi.c        |  74 +--
 clang/test/CodeGen/RISCV/riscv32-vararg.c     |   2 +-
 clang/test/CodeGen/RISCV/riscv64-abi.c        |  18 +-
 clang/test/CodeGen/RISCV/riscv64-vararg.c     |   2 +-
 .../test/CodeGen/SystemZ/systemz-abi-vector.c |  52 +-
 clang/test/CodeGen/SystemZ/systemz-abi.c      |  38 +-
 .../test/CodeGen/SystemZ/systemz-inline-asm.c |   2 +-
 clang/test/CodeGen/X86/cx-complex-range.c     |   2 +-
 .../test/CodeGen/X86/x86_32-arguments-win32.c |  14 +-
 .../test/CodeGen/X86/x86_64-arguments-win32.c |   2 +-
 clang/test/CodeGen/aapcs64-align.cpp          |   4 +-
 clang/test/CodeGen/arm-aapcs-vfp.c            |   2 +-
 clang/test/CodeGen/arm-abi-vector.c           |   6 +-
 clang/test/CodeGen/arm-swiftcall.c            |   2 +-
 clang/test/CodeGen/arm64-abi-vector.c         |  14 +-
 clang/test/CodeGen/arm64-arguments.c          |  26 +-
 .../CodeGen/arm64-microsoft-arguments.cpp     |   2 +-
 clang/test/CodeGen/armv7k-abi.c               |   2 +-
 clang/test/CodeGen/atomic-arm64.c             |   2 +-
 clang/test/CodeGen/attr-noundef.cpp           |   7 +-
 clang/test/CodeGen/cx-complex-range.c         |  36 +-
 clang/test/CodeGen/ext-int-cc.c               |  36 +-
 clang/test/CodeGen/isfpclass.c                |   2 +-
 .../math-libcalls-tbaa-indirect-args.c        |  86 ++--
 clang/test/CodeGen/mingw-long-double.c        |   6 +-
 clang/test/CodeGen/ms_abi.c                   |   4 +-
 clang/test/CodeGen/pass-by-value-noalias.c    |   4 +-
 clang/test/CodeGen/ptrauth-in-c-struct.c      |   4 +-
 clang/test/CodeGen/regcall.c                  |  10 +-
 clang/test/CodeGen/regcall2.c                 |   2 +-
 clang/test/CodeGen/regcall4.c                 |  10 +-
 clang/test/CodeGen/sparcv9-abi.c              |   4 +-
 clang/test/CodeGen/vectorcall.c               |  46 +-
 clang/test/CodeGen/win-fp128.c                |   2 +-
 clang/test/CodeGen/win64-i128.c               |   4 +-
 clang/test/CodeGen/windows-swiftcall.c        |   2 +-
 .../CodeGenCXX/aarch64-mangle-sve-vectors.cpp |   4 +-
 clang/test/CodeGenCXX/arm-cc.cpp              |   2 +-
 .../CodeGenCXX/attr-target-mv-inalloca.cpp    |   8 +-
 clang/test/CodeGenCXX/copy-initialization.cpp |   2 +-
 clang/test/CodeGenCXX/debug-info.cpp          |   2 +-
 .../empty-nontrivially-copyable.cpp           |   4 +-
 clang/test/CodeGenCXX/fastcall.cpp            |   2 +-
 .../CodeGenCXX/homogeneous-aggregates.cpp     |  14 +-
 clang/test/CodeGenCXX/inalloca-lambda.cpp     |   6 +-
 .../test/CodeGenCXX/inalloca-overaligned.cpp  |   8 +-
 clang/test/CodeGenCXX/inalloca-vector.cpp     |   4 +-
 .../CodeGenCXX/inheriting-constructor.cpp     |   8 +-
 .../member-function-pointer-calls.cpp         |   4 +-
 .../CodeGenCXX/microsoft-abi-arg-order.cpp    |   4 +-
 .../CodeGenCXX/microsoft-abi-byval-thunks.cpp |  12 +-
 .../microsoft-abi-member-pointers.cpp         |   6 +-
 .../microsoft-abi-sret-and-byval.cpp          |  26 +-
 .../CodeGenCXX/microsoft-abi-unknown-arch.cpp |   2 +-
 clang/test/CodeGenCXX/ms-property.cpp         |   4 +-
 clang/test/CodeGenCXX/nrvo.cpp                |   4 +-
 .../test/CodeGenCXX/pass-by-value-noalias.cpp |  12 +-
 .../CodeGenCXX/ptrauth-qualifier-struct.cpp   |   2 +-
 clang/test/CodeGenCXX/regparm.cpp             |   2 +-
 clang/test/CodeGenCXX/trivial_abi.cpp         |   8 +-
 clang/test/CodeGenCXX/uncopyable-args.cpp     |  32 +-
 clang/test/CodeGenCXX/wasm-args-returns.cpp   |  12 +-
 .../test/CodeGenCXX/windows-x86-swiftcall.cpp |   4 +-
 .../nontrivial-c-struct-exception.m           |   2 +-
 .../test/CodeGenObjC/pass-by-value-noalias.m  |   4 +-
 clang/test/CodeGenObjC/weak-in-c-struct.m     |   6 +-
 .../test/CodeGenObjCXX/objc-struct-cxx-abi.mm |  12 +-
 clang/test/CodeGenObjCXX/property-objects.mm  |   2 +-
 .../CodeGenObjCXX/ptrauth-struct-cxx-abi.mm   |   2 +-
 .../CodeGenOpenCL/builtins-amdgcn-gfx1250.cl  |  38 ++
 .../test/Driver/print-multi-selection-flags.c |  19 +
 clang/test/Headers/stdarg.cpp                 |   4 +-
 .../test/OpenMP/for_firstprivate_codegen.cpp  |  30 +-
 .../OpenMP/parallel_firstprivate_codegen.cpp  | 216 ++++-----
 .../OpenMP/sections_firstprivate_codegen.cpp  |  14 +-
 .../OpenMP/single_firstprivate_codegen.cpp    |  14 +-
 ..._teams_distribute_firstprivate_codegen.cpp |  82 ++--
 ...bute_parallel_for_firstprivate_codegen.cpp | 224 ++++-----
 .../teams_distribute_firstprivate_codegen.cpp |  90 ++--
 ...bute_parallel_for_firstprivate_codegen.cpp | 124 ++---
 .../OpenMP/teams_firstprivate_codegen.cpp     |  80 ++--
 clang/test/Sema/dllexport.c                   |   4 +
 ...overload-resolution-deferred-templates.cpp |  28 ++
 .../test/SemaTemplate/concepts-using-decl.cpp |  21 +
 flang/lib/Lower/Bridge.cpp                    |  15 +-
 flang/lib/Lower/ConvertConstant.cpp           |   2 +-
 flang/lib/Lower/ConvertExpr.cpp               |  26 +-
 flang/lib/Lower/ConvertVariable.cpp           |  26 +-
 flang/lib/Lower/Runtime.cpp                   |   4 +-
 flang/lib/Lower/VectorSubscripts.cpp          |   2 +-
 flang/lib/Optimizer/Builder/FIRBuilder.cpp    |   2 +-
 .../Optimizer/Builder/Runtime/Intrinsics.cpp  |   6 +-
 flang/lib/Optimizer/Builder/Runtime/Stop.cpp  |   2 +-
 .../LowerHLFIROrderedAssignments.cpp          |   4 +-
 .../lib/Optimizer/Transforms/MemoryUtils.cpp  |   2 +-
 .../Transforms/PolymorphicOpConversion.cpp    |   2 +-
 .../Transforms/SimplifyFIROperations.cpp      |   2 +-
 flang/test/Lower/CUDA/cuda-set-allocator.cuf  |  32 ++
 .../Optimizer/FortranVariableTest.cpp         |   6 +-
 libc/cmake/modules/LLVMLibCTestRules.cmake    |  17 +-
 libcxx/include/__config_site.in               |   1 -
 libcxx/test/libcxx/fuzzing/random.pass.cpp    |   7 +-
 .../std/depr/depr.c.headers/math_h.pass.cpp   |   7 +-
 .../test/std/numerics/c.math/cmath.pass.cpp   |   7 +-
 lld/ELF/BPSectionOrderer.cpp                  |   6 +-
 lld/test/ELF/bp-section-orderer.s             |  21 +-
 lldb/source/Expression/IRExecutionUnit.cpp    |  16 +-
 .../TestNSDictionarySynthetic.py              |   3 +-
 .../PDB/Inputs/UdtLayoutTest.script           |   1 +
 llvm/Maintainers.md                           |  14 +
 llvm/docs/CodingStandards.rst                 |  46 +-
 .../llvm/CodeGen/MachineBlockFrequencyInfo.h  |   4 +-
 llvm/include/llvm/IR/DebugInfo.h              |   5 +-
 llvm/include/llvm/IR/DebugInfoMetadata.h      |   3 -
 .../llvm/Transforms/Utils/SSAUpdater.h        |   4 -
 llvm/lib/CodeGen/CodeGenPrepare.cpp           |  59 +--
 .../lib/CodeGen/MachineBlockFrequencyInfo.cpp |   4 +-
 llvm/lib/CodeGen/MachineDebugify.cpp          |  18 +-
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  16 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   4 +-
 llvm/lib/IR/DebugInfo.cpp                     |  25 +-
 llvm/lib/IR/DebugInfoMetadata.cpp             |   9 -
 llvm/lib/Object/IRSymtab.cpp                  |  45 +-
 .../Target/AArch64/AArch64ISelLowering.cpp    |   4 +-
 llvm/lib/Target/AMDGPU/AMDGPU.h               |   3 +
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |   1 +
 .../Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp  | 108 +++++
 .../Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h    |  23 +
 .../AMDGPU/AMDGPUResourceUsageAnalysis.cpp    | 234 +--------
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  13 +
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h  |   2 +
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     | 146 ++++--
 llvm/lib/Target/AMDGPU/SIInstrInfo.h          |   1 -
 llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp     |   8 +-
 llvm/lib/Target/AMDGPU/SIRegisterInfo.h       |   6 +-
 llvm/lib/Target/AMDGPU/VOP1Instructions.td    |   1 +
 .../LoongArch/LoongArchFrameLowering.cpp      |   7 +-
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp  | 122 +++--
 llvm/lib/Target/X86/X86CallingConv.cpp        |   2 +-
 llvm/lib/TargetParser/X86TargetParser.cpp     |   6 +-
 llvm/lib/Transforms/Coroutines/CoroFrame.cpp  |  52 +-
 llvm/lib/Transforms/Coroutines/CoroInternal.h |   5 +-
 llvm/lib/Transforms/Coroutines/CoroSplit.cpp  |  23 +-
 llvm/lib/Transforms/Coroutines/SpillUtils.cpp |   6 +-
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |  19 +-
 .../InstCombine/InstCombineInternal.h         |   3 -
 .../InstCombine/InstructionCombining.cpp      |  78 +--
 .../Scalar/ConstraintElimination.cpp          |   1 +
 .../Scalar/DeadStoreElimination.cpp           |   1 +
 llvm/lib/Transforms/Scalar/JumpThreading.cpp  |  39 +-
 .../lib/Transforms/Scalar/LoopInterchange.cpp |  24 +-
 .../Transforms/Scalar/LoopStrengthReduce.cpp  | 184 ++++---
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |   8 +-
 llvm/lib/Transforms/Utils/Debugify.cpp        |   7 -
 llvm/lib/Transforms/Utils/InlineFunction.cpp  |   3 +-
 llvm/lib/Transforms/Utils/Local.cpp           | 103 +---
 .../Transforms/Utils/LoopRotationUtils.cpp    |  23 +-
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   1 -
 .../Utils/PromoteMemoryToRegister.cpp         |  49 +-
 llvm/lib/Transforms/Utils/SSAUpdater.cpp      |  22 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  39 +-
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   9 +-
 llvm/lib/Transforms/Vectorize/VPlanAnalysis.h |   6 +-
 .../AArch64/sve-intrinsics-ldst-ext.ll        |  24 +
 llvm/test/CodeGen/AMDGPU/agpr-remat.ll        |  18 +-
 .../AMDGPU/amdgpu-prepare-agpr-alloc.mir      |  95 ++++
 .../bad-agpr-vgpr-regalloc-priority.mir       |  74 +++
 .../branch-folding-implicit-def-subreg.ll     |  46 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll  |   4 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |   4 +
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll  |   4 +-
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll  |   4 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-mov.ll     |  10 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-mov.mir    |  28 +-
 .../CodeGen/AMDGPU/no-fold-accvgpr-read.mir   |  26 +-
 ...al-regcopy-and-spill-missed-at-regalloc.ll |  20 +-
 .../AMDGPU/ptradd-sdag-optimizations.ll       | 196 ++++----
 .../CodeGen/AMDGPU/spill-vector-superclass.ll |   6 +-
 .../CodeGen/LoongArch/calling-conv-common.ll  |  48 +-
 .../CodeGen/LoongArch/calling-conv-half.ll    |  16 +-
 .../LoongArch/can-not-realign-stack.ll        |  44 +-
 .../CodeGen/LoongArch/emergency-spill-slot.ll |   4 +-
 llvm/test/CodeGen/LoongArch/frame.ll          | 107 ++++-
 .../CodeGen/LoongArch/intrinsic-memcpy.ll     |   8 +-
 llvm/test/CodeGen/LoongArch/lasx/fpowi.ll     |  88 ++--
 .../lasx/ir-instruction/extractelement.ll     | 120 ++---
 .../lasx/ir-instruction/insertelement.ll      | 132 +++---
 llvm/test/CodeGen/LoongArch/llvm.sincos.ll    | 150 +++---
 llvm/test/CodeGen/LoongArch/lsx/pr146455.ll   | 287 +++++++++++
 ...realignment-with-variable-sized-objects.ll |  24 +-
 .../CodeGen/LoongArch/stack-realignment.ll    |  80 ++--
 .../LoongArch/unaligned-memcpy-inline.ll      |  14 +-
 llvm/test/CodeGen/LoongArch/vararg.ll         |  70 +--
 llvm/test/CodeGen/NVPTX/i8x4-instructions.ll  |  47 ++
 llvm/test/CodeGen/NVPTX/trunc-tofp.ll         |  81 ++++
 .../CodeGen/RISCV/short-forward-branch-opt.ll |  32 +-
 llvm/test/CodeGen/RISCV/zdinx-spill.ll        |  26 -
 .../AddressSanitizer/lifetime.ll              | 447 +++++++++++++++---
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s |  45 ++
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s        |  45 ++
 .../MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s |  52 ++
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s  |  52 ++
 .../MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s  |  12 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s   |  12 +
 .../gfx1250_asm_vop3_from_vop1-fake16.s       |  36 ++
 .../MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s    |  36 ++
 .../gfx1250_asm_vop3_from_vop1_dpp16-fake16.s |  44 ++
 .../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s |  44 ++
 .../gfx1250_asm_vop3_from_vop1_dpp8-fake16.s  |   4 +
 .../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s  |   4 +
 .../Disassembler/AMDGPU/gfx1250_dasm_vop1.txt |  45 ++
 .../AMDGPU/gfx1250_dasm_vop1_dpp16.txt        |  39 ++
 .../AMDGPU/gfx1250_dasm_vop1_dpp8.txt         |   9 +
 .../AMDGPU/gfx1250_dasm_vop3_from_vop1.txt    |  36 ++
 .../gfx1250_dasm_vop3_from_vop1_dpp16.txt     |  33 ++
 .../gfx1250_dasm_vop3_from_vop1_dpp8.txt      |   7 +-
 .../DeadStoreElimination/zeroed-missing.ll    |   2 +-
 .../LoopInterchange/force-interchange.ll      |  43 ++
 .../LoopStrengthReduce/X86/lifetime-use.ll    |  59 +++
 .../AArch64/maxbandwidth-regpressure.ll       |  38 ++
 .../X86/split-node-reorder-node-with-ops.ll   |   2 -
 llvm/tools/llvm-dis/llvm-dis.cpp              |  14 -
 llvm/unittests/CodeGen/LexicalScopesTest.cpp  |   2 +-
 llvm/unittests/IR/DebugInfoTest.cpp           |   2 +
 .../Transforms/Utils/DebugifyTest.cpp         |   9 +-
 llvm/unittests/Transforms/Utils/LocalTest.cpp |  56 ---
 .../gn/secondary/libcxx/include/BUILD.gn      |   1 -
 .../secondary/llvm/lib/Target/AMDGPU/BUILD.gn |   1 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |   1 +
 .../secondary/llvm/tools/llvm-ir2vec/BUILD.gn |   9 +
 mlir/docs/Tutorials/transform/Ch2.md          |  30 +-
 .../Conversion/VectorToXeGPU/VectorToXeGPU.h  |   3 -
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |  63 ++-
 .../common/include/PluginInterface.h          |  28 +-
 offload/plugins-nextgen/cuda/src/rtl.cpp      |  10 +-
 .../unittests/OffloadAPI/common/Fixtures.hpp  |   3 -
 .../OffloadAPI/event/olWaitEvent.cpp          |   3 -
 275 files changed, 4637 insertions(+), 2846 deletions(-)
 create mode 100644 clang/test/Analysis/div-zero-cxx20.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
 create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
 create mode 100644 llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/trunc-tofp.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/force-interchange.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
 create mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn

diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index 59079f057d021..f76c69f29fb30 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -11,8 +11,6 @@ on:
       - .github/workflows/build-ci-container-windows.yml
       - '.github/workflows/containers/github-action-ci-windows/**'
   pull_request:
-    branches:
-      - main
     paths:
       - .github/workflows/build-ci-container-windows.yml
       - '.github/workflows/containers/github-action-ci-windows/**'
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 3159aae32ca51..7f01264af8534 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -11,8 +11,6 @@ on:
       - .github/workflows/build-ci-container.yml
       - '.github/workflows/containers/github-action-ci/**'
   pull_request:
-    branches:
-      - main
     paths:
       - .github/workflows/build-ci-container.yml
       - '.github/workflows/containers/github-action-ci/**'
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index fcd3887ec7a09..ea16029268dba 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -808,6 +808,7 @@ Bug Fixes in This Version
   nested scopes. (#GH147495)
 - Fixed a failed assertion with an operator call expression which comes from a
   macro expansion when performing analysis for nullability attributes. (#GH138371)
+- Fixed a concept equivalent checking crash due to untransformed constraint expressions. (#GH146614)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1206,6 +1207,8 @@ Static Analyzer
 ---------------
 - Fixed a crash when C++20 parenthesized initializer lists are used. This issue
   was causing a crash in clang-tidy. (#GH136041)
+- The Clang Static Analyzer now handles parenthesized initialization.
+  (#GH148875)
 
 New features
 ^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index ed51f1d5de447..a916af7e0c2df 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -673,6 +673,7 @@ TARGET_BUILTIN(__builtin_amdgcn_tanhf, "ff", "nc", "tanh-insts")
 TARGET_BUILTIN(__builtin_amdgcn_tanhh, "hh", "nc", "tanh-insts")
 TARGET_BUILTIN(__builtin_amdgcn_tanh_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_rcp_bf16, "yy", "nc", "bf16-trans-insts")
+TARGET_BUILTIN(__builtin_amdgcn_sqrt_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_rsq_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_log_bf16, "yy", "nc", "bf16-trans-insts")
 TARGET_BUILTIN(__builtin_amdgcn_exp2_bf16, "yy", "nc", "bf16-trans-insts")
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index a70335bef9dd4..d34a4146ddbd6 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -1491,8 +1491,6 @@ class Sema;
     OverloadingResult
     BestViableFunctionImpl(Sema &S, SourceLocation Loc,
                            OverloadCandidateSet::iterator &Best);
-    void PerfectViableFunction(Sema &S, SourceLocation Loc,
-                               OverloadCandidateSet::iterator &Best);
   };
 
   bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index 6370586e218ef..fbb34340a5c67 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -499,9 +499,6 @@ class ExprEngine {
   void VisitGuardedExpr(const Expr *Ex, const Expr *L, const Expr *R,
                         ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
-  void VisitInitListExpr(const InitListExpr *E, ExplodedNode *Pred,
-                         ExplodedNodeSet &Dst);
-
   /// VisitAttributedStmt - Transfer function logic for AttributedStmt.
   void VisitAttributedStmt(const AttributedStmt *A, ExplodedNode *Pred,
                            ExplodedNodeSet &Dst);
@@ -591,6 +588,10 @@ class ExprEngine {
                                 ExplodedNode *Pred,
                                 ExplodedNodeSet &Dst);
 
+  void ConstructInitList(const Expr *Source, ArrayRef<Expr *> Args,
+                         bool IsTransparent, ExplodedNode *Pred,
+                         ExplodedNodeSet &Dst);
+
   /// evalEagerlyAssumeBifurcation - Given the nodes in 'Src', eagerly assume
   /// concrete boolean values for 'Ex', storing the resulting nodes in 'Dst'.
   void evalEagerlyAssumeBifurcation(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index c89eca9bef440..5b9f44518fcc2 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -162,6 +162,8 @@ static void initField(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable,
   Desc->IsConst = IsConst || D->IsConst;
   Desc->IsFieldMutable = IsMutable || D->IsMutable;
   Desc->IsVolatile = IsVolatile || D->IsVolatile;
+  // True if this field is const AND the parent is mutable.
+  Desc->IsConstInMutable = Desc->IsConst && IsMutable;
 
   if (auto Fn = D->CtorFn)
     Fn(B, Ptr + FieldOffset, Desc->IsConst, Desc->IsFieldMutable,
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 4591eabb69bb4..0227e4c0c7e38 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -101,6 +101,10 @@ struct InlineDescriptor {
   /// Flag indicating if the field is mutable (if in a record).
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsFieldMutable : 1;
+  /// Flag indicating if this field is a const field nested in
+  /// a mutable parent field.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned IsConstInMutable : 1;
   /// Flag indicating if the field is an element of a composite array.
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsArrayElement : 1;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index f64501f4a31e8..74399d177b5a2 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -445,6 +445,7 @@ LLVM_DUMP_METHOD void InlineDescriptor::dump(llvm::raw_ostream &OS) const {
   OS << "InUnion: " << InUnion << "\n";
   OS << "IsFieldMutable: " << IsFieldMutable << "\n";
   OS << "IsArrayElement: " << IsArrayElement << "\n";
+  OS << "IsConstInMutable: " << IsConstInMutable << '\n';
   OS << "Desc: ";
   if (Desc)
     Desc->dump(OS);
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index e8b519478c026..df5e3be83d741 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -566,7 +566,10 @@ bool CheckDowncast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
 bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   assert(Ptr.isLive() && "Pointer is not live");
-  if (!Ptr.isConst() || Ptr.isMutable())
+  if (!Ptr.isConst())
+    return true;
+
+  if (Ptr.isMutable() && !Ptr.isConstInMutable())
     return true;
 
   if (!Ptr.isBlockPointer())
diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
index 7798b6f886a85..51622238e275c 100644
--- a/clang/lib/AST/ByteCode/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -14,11 +14,6 @@
 #define LLVM_CLANG_AST_INTERP_BLOCK_H
 
 #include "Descriptor.h"
-#include "clang/AST/ComparisonCategories.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclCXX.h"
-#include "clang/AST/Expr.h"
-#include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index de0b97fd93c76..462b9a11e0a5c 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 #include "../ExprConstShared.h"
 #include "Boolean.h"
-#include "Compiler.h"
 #include "EvalEmitter.h"
 #include "Interp.h"
 #include "InterpBuiltinBitCast.h"
@@ -2906,6 +2905,8 @@ static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,
         if (!copyField(F, /*Activate=*/true))
           return false;
       } else {
+        if (!CheckMutable(S, OpPC, Src.atField(F.Offset)))
+          return false;
         Pointer DestField = Dest.atField(F.Offset);
         zeroAll(DestField);
       }
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index a5a4bd25fe712..d62a4f6275b50 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -128,6 +128,11 @@ static bool shouldSkipInBacktrace(const Function *F) {
   if (FD->getDeclName().getCXXOverloadedOperator() == OO_New ||
       FD->getDeclName().getCXXOverloadedOperator() == OO_Array_New)
     return true;
+
+  if (const auto *MD = dyn_cast<CXXMethodDecl>(FD);
+      MD && MD->getParent()->isAnonymousStructOrUnion())
+    return true;
+
   return false;
 }
 
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index e6a64e6658f06..da74013cf83a6 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -576,6 +576,11 @@ class Pointer {
       return true;
     return isRoot() ? getDeclDesc()->IsConst : getInlineDesc()->IsConst;
   }
+  bool isConstInMutable() const {
+    if (!isBlockPointer())
+      return false;
+    return isRoot() ? false : getInlineDesc()->IsConstInMutable;
+  }
 
   /// Checks if an object or a subfield is volatile.
   bool isVolatile() const {
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index c8c3d6b20c496..19d8ba26d44d8 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2852,8 +2852,21 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
       if (AI.getInReg())
         Attrs.addAttribute(llvm::Attribute::InReg);
 
-      if (AI.getIndirectByVal())
+      // Depending on the ABI, this may be either a byval or a dead_on_return
+      // argument.
+      if (AI.getIndirectByVal()) {
         Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
+      } else {
+        // Add dead_on_return when the object's lifetime ends in the callee.
+        // This includes trivially-destructible objects, as well as objects
+        // whose destruction / clean-up is carried out within the callee (e.g.,
+        // Obj-C ARC-managed structs, MSVC callee-destroyed objects).
+        if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
+            ParamType->castAs<RecordType>()
+                ->getDecl()
+                ->isParamDestroyedInCallee())
+          Attrs.addAttribute(llvm::Attribute::DeadOnReturn);
+      }
 
       auto *Decl = ParamType->getAsRecordDecl();
       if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index a7d796ecccc61..ee736a2816218 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -416,6 +416,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_sqrt:
   case AMDGPU::BI__builtin_amdgcn_sqrtf:
   case AMDGPU::BI__builtin_amdgcn_sqrth:
+  case AMDGPU::BI__builtin_amdgcn_sqrt_bf16:
     return emitBuiltinWithOneOverloadedType<1>(*this, E,
                                                Intrinsic::amdgcn_sqrt);
   case AMDGPU::BI__builtin_amdgcn_rsq:
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..481f575518b93 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -258,10 +258,10 @@ static void getAArch64MultilibFlags(const Driver &D,
   processMultilibCustomFlags(Result, Args);
 }
 
-static void getARMMultilibFlags(const Driver &D,
-                                      const llvm::Triple &Triple,
-                                      const llvm::opt::ArgList &Args,
-                                      Multilib::flags_list &Result) {
+static void getARMMultilibFlags(const Driver &D, const llvm::Triple &Triple,
+                                llvm::Reloc::Model RelocationModel,
+                                const llvm::opt::ArgList &Args,
+                                Multilib::flags_list &Result) {
   std::vector<StringRef> Features;
   llvm::ARM::FPUKind FPUKind = tools::arm::getARMTargetFeatures(
       D, Triple, Args, Features, false /*ForAs*/, true /*ForMultilib*/);
@@ -304,6 +304,18 @@ static void getARMMultilibFlags(const Driver &D,
     llvm_unreachable("Invalid float ABI");
   }
 
+  if (RelocationModel == llvm::Reloc::ROPI ||
+      RelocationModel == llvm::Reloc::ROPI_RWPI)
+    Result.push_back("-fropi");
+  else
+    Result.push_back("-fno-ropi");
+
+  if (RelocationModel == llvm::Reloc::RWPI ||
+      RelocationModel == llvm::Reloc::ROPI_RWPI)
+    Result.push_back("-frwpi");
+  else
+    Result.push_back("-fno-rwpi");
+
   const Arg *BranchProtectionArg =
       Args.getLastArgNoClaim(options::OPT_mbranch_protection_EQ);
   if (BranchProtectionArg) {
@@ -344,6 +356,18 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   const llvm::Triple Triple(ComputeEffectiveClangTriple(Args));
   Result.push_back("--target=" + Triple.str());
 
+  // A difference of relocation model (absolutely addressed data, PIC, Arm
+  // ROPI/RWPI) is likely to change whether a particular multilib variant is
+  // compatible with a given link. Determine the relocation model of the
+  // current link, so as to add appropriate multilib flags.
+  llvm::Reloc::Model RelocationModel;
+  unsigned PICLevel;
+  bool IsPIE;
+  {
+    RegisterEffectiveTriple TripleRAII(*this, Triple);
+    std::tie(RelocationModel, PICLevel, IsPIE) = ParsePICArgs(*this, Args);
+  }
+
   switch (Triple.getArch()) {
   case llvm::Triple::aarch64:
   case llvm::Triple::aarch64_32:
@@ -354,7 +378,7 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   case llvm::Triple::armeb:
   case llvm::Triple::thumb:
   case llvm::Triple::thumbeb:
-    getARMMultilibFlags(D, Triple, Args, Result);
+    getARMMultilibFlags(D, Triple, RelocationModel, Args, Result);
     break;
   case llvm::Triple::riscv32:
   case llvm::Triple::riscv64:
@@ -376,6 +400,12 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   else
     Result.push_back("-fexceptions");
 
+  if (RelocationModel == llvm::Reloc::PIC_)
+    Result.push_back(IsPIE ? (PICLevel > 1 ? "-fPIE" : "-fpie")
+                           : (PICLevel > 1 ? "-fPIC" : "-fpic"));
+  else
+    Result.push_back("-fno-pic");
+
   // Sort and remove duplicates.
   std::sort(Result.begin(), Result.end());
   Result.erase(llvm::unique(Result), Result.end());
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 834417f8e15ac..5205ca0bca6fa 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -925,7 +925,12 @@ static const Expr *SubstituteConstraintExpressionWithoutSatisfaction(
       ND && ND->isFunctionOrFunctionTemplate()) {
     ScopeForParameters.emplace(S, /*CombineWithOuterScope=*/true);
     const FunctionDecl *FD = ND->getAsFunction();
+    if (FunctionTemplateDecl *Template = FD->getDescribedFunctionTemplate();
+        Template && Template->getInstantiatedFromMemberTemplate())
+      FD = Template->getInstantiatedFromMemberTemplate()->getTemplatedDecl();
     for (auto *PVD : FD->parameters()) {
+      if (ScopeForParameters->getInstantiationOfIfExists(PVD))
+        continue;
       if (!PVD->isParameterPack()) {
         ScopeForParameters->InstantiatedLocal(PVD, PVD);
         continue;
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 1b54628c5e564..5dd5b495480d9 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -11354,55 +11354,18 @@ OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
       DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
 
   if (TwoPhaseResolution) {
-
-    PerfectViableFunction(S, Loc, Best);
-    if (Best != end())
-      return ResultForBestCandidate(Best);
+    OverloadingResult Res = BestViableFunctionImpl(S, Loc, Best);
+    if (Best != end() && Best->isPerfectMatch(S.Context)) {
+      if (!(HasDeferredTemplateConstructors &&
+            isa_and_nonnull<CXXConversionDecl>(Best->Function)))
+        return Res;
+    }
   }
 
   InjectNonDeducedTemplateCandidates(S);
   return BestViableFunctionImpl(S, Loc, Best);
 }
 
-void OverloadCandidateSet::PerfectViableFunction(
-    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
-
-  Best = end();
-  for (auto It = Candidates.begin(); It != Candidates.end(); ++It) {
-
-    if (!It->isPerfectMatch(S.getASTContext()))
-      continue;
-
-    // We found a suitable conversion function
-    // but if there is a template constructor in the target class
-    // we might prefer that instead.
-    if (HasDeferredTemplateConstructors &&
-        isa_and_nonnull<CXXConversionDecl>(It->Function)) {
-      Best = end();
-      break;
-    }
-
-    if (Best == end()) {
-      Best = It;
-      continue;
-    }
-    if (Best->Function && It->Function) {
-      FunctionDecl *D =
-          S.getMoreConstrainedFunction(Best->Function, It->Function);
-      if (D == nullptr) {
-        Best = end();
-        break;
-      }
-      if (D == It->Function)
-        Best = It;
-      continue;
-    }
-    // ambiguous
-    Best = end();
-    break;
-  }
-}
-
 OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
     Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
 
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index c77ef26da568d..d87484470f8b5 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1941,7 +1941,6 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::ConceptSpecializationExprClass:
     case Stmt::CXXRewrittenBinaryOperatorClass:
     case Stmt::RequiresExprClass:
-    case Expr::CXXParenListInitExprClass:
     case Stmt::EmbedExprClass:
       // Fall through.
 
@@ -2315,11 +2314,22 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
       break;
     }
 
-    case Stmt::InitListExprClass:
+    case Stmt::InitListExprClass: {
+      const InitListExpr *E = cast<InitListExpr>(S);
       Bldr.takeNodes(Pred);
-      VisitInitListExpr(cast<InitListExpr>(S), Pred, Dst);
+      ConstructInitList(E, E->inits(), E->isTransparent(), Pred, Dst);
       Bldr.addNodes(Dst);
       break;
+    }
+
+    case Expr::CXXParenListInitExprClass: {
+      const CXXParenListInitExpr *E = cast<CXXParenListInitExpr>(S);
+      Bldr.takeNodes(Pred);
+      ConstructInitList(E, E->getInitExprs(), /*IsTransparent*/ false, Pred,
+                        Dst);
+      Bldr.addNodes(Dst);
+      break;
+    }
 
     case Stmt::MemberExprClass:
       Bldr.takeNodes(Pred);
@@ -4114,3 +4124,33 @@ void *ProgramStateTrait<ReplayWithoutInlining>::GDMIndex() {
 }
 
 void ExprEngine::anchor() { }
+
+void ExprEngine::ConstructInitList(const Expr *E, ArrayRef<Expr *> Args,
+                                   bool IsTransparent, ExplodedNode *Pred,
+                                   ExplodedNodeSet &Dst) {
+  assert((isa<InitListExpr, CXXParenListInitExpr>(E)));
+
+  const LocationContext *LC = Pred->getLocationContext();
+
+  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
+  ProgramStateRef S = Pred->getState();
+  QualType T = E->getType().getCanonicalType();
+
+  bool IsCompound = T->isArrayType() || T->isRecordType() ||
+                    T->isAnyComplexType() || T->isVectorType();
+
+  if (Args.size() > 1 || (E->isPRValue() && IsCompound && !IsTransparent)) {
+    llvm::ImmutableList<SVal> ArgList = getBasicVals().getEmptySValList();
+    for (Expr *E : llvm::reverse(Args))
+      ArgList = getBasicVals().prependSVal(S->getSVal(E, LC), ArgList);
+
+    B.generateNode(E, Pred,
+                   S->BindExpr(E, LC, svalBuilder.makeCompoundVal(T, ArgList)));
+  } else {
+    B.generateNode(E, Pred,
+                   S->BindExpr(E, LC,
+                               Args.size() == 0
+                                   ? getSValBuilder().makeZeroVal(T)
+                                   : S->getSVal(Args.front(), LC)));
+  }
+}
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index fa8e669b6bb2f..f1a25a750dd0d 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -771,54 +771,6 @@ void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
   Bldr.generateNode(B, Pred, state->BindExpr(B, Pred->getLocationContext(), X));
 }
 
-void ExprEngine::VisitInitListExpr(const InitListExpr *IE,
-                                   ExplodedNode *Pred,
-                                   ExplodedNodeSet &Dst) {
-  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
-
-  ProgramStateRef state = Pred->getState();
-  const LocationContext *LCtx = Pred->getLocationContext();
-  QualType T = getContext().getCanonicalType(IE->getType());
-  unsigned NumInitElements = IE->getNumInits();
-
-  if (!IE->isGLValue() && !IE->isTransparent() &&
-      (T->isArrayType() || T->isRecordType() || T->isVectorType() ||
-       T->isAnyComplexType())) {
-    llvm::ImmutableList<SVal> vals = getBasicVals().getEmptySValList();
-
-    // Handle base case where the initializer has no elements.
-    // e.g: static int* myArray[] = {};
-    if (NumInitElements == 0) {
-      SVal V = svalBuilder.makeCompoundVal(T, vals);
-      B.generateNode(IE, Pred, state->BindExpr(IE, LCtx, V));
-      return;
-    }
-
-    for (const Stmt *S : llvm::reverse(*IE)) {
-      SVal V = state->getSVal(cast<Expr>(S), LCtx);
-      vals = getBasicVals().prependSVal(V, vals);
-    }
-
-    B.generateNode(IE, Pred,
-                   state->BindExpr(IE, LCtx,
-                                   svalBuilder.makeCompoundVal(T, vals)));
-    return;
-  }
-
-  // Handle scalars: int{5} and int{} and GLvalues.
-  // Note, if the InitListExpr is a GLvalue, it means that there is an address
-  // representing it, so it must have a single init element.
-  assert(NumInitElements <= 1);
-
-  SVal V;
-  if (NumInitElements == 0)
-    V = getSValBuilder().makeZeroVal(T);
-  else
-    V = state->getSVal(IE->getInit(0), LCtx);
-
-  B.generateNode(IE, Pred, state->BindExpr(IE, LCtx, V));
-}
-
 void ExprEngine::VisitGuardedExpr(const Expr *Ex,
                                   const Expr *L,
                                   const Expr *R,
diff --git a/clang/test/AST/ByteCode/mutable.cpp b/clang/test/AST/ByteCode/mutable.cpp
index aebbea920578c..35c5a0389921e 100644
--- a/clang/test/AST/ByteCode/mutable.cpp
+++ b/clang/test/AST/ByteCode/mutable.cpp
@@ -1,11 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++11 -verify=expected,expected11,both,both11 %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++14 -verify=expected,expected14,both %s
-// RUN: %clang_cc1 -std=c++11 -verify=ref,ref11,both,both11 %s
-// RUN: %clang_cc1 -std=c++14 -verify=ref,ref14,both %s
-
-
-
-
+// RUN: %clang_cc1 -std=c++11 -verify=expected,expected11,both,both11 %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++14 -verify=expected,expected14,both        %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++11 -verify=ref,ref11,both,both11           %s
+// RUN: %clang_cc1 -std=c++14 -verify=ref,ref14,both                  %s
 
 namespace Simple {
   struct S {
@@ -26,3 +22,47 @@ namespace Simple {
   static_assert(s2.a2 == 12, ""); // both11-error {{not an integral constant expression}} \
                                   // both11-note {{initializer of 's2' is not a constant expression}}
 }
+#if __cplusplus >= 201402L
+namespace ConstInMutable {
+  class B {
+    public:
+
+    const int f;
+    constexpr B() : f(12) {}
+  };
+  class A {
+    public:
+    mutable B b;
+    constexpr A() = default;
+  };
+  constexpr int constInMutable() {
+    A a;
+
+    int *m = (int*)&a.b.f;
+    *m = 12; // both-note {{modification of object of const-qualified type 'const int' is not allowed in a constant expression}}
+    return 1;
+  }
+  static_assert(constInMutable() == 1, ""); // both-error {{not an integral constant expression}} \
+                                            // both-note {{in call to}}
+}
+
+namespace MutableInConst {
+  class C {
+  public:
+    mutable int c;
+    constexpr C() : c(50) {}
+  };
+  class D {
+  public:
+    C c;
+    constexpr D() {}
+  };
+  constexpr int mutableInConst() {
+    const D d{};
+    int *m = (int*)&d.c.c;
+    *m = 12;
+    return 1;
+  }
+  static_assert(mutableInConst() == 1, "");
+}
+#endif
diff --git a/clang/test/AST/ByteCode/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
index 0fa44a259a4ff..7cfd0d677a7b3 100644
--- a/clang/test/AST/ByteCode/unions.cpp
+++ b/clang/test/AST/ByteCode/unions.cpp
@@ -847,6 +847,20 @@ namespace Activation2 {
   }
   static_assert(change_member_indirectly() == 4);
 }
+
+namespace CopyCtorMutable {
+  struct E {
+    union { // expected-note {{read of mutable member 'b'}}
+      int a;
+      mutable int b; // both-note {{here}}
+    };
+  };
+  constexpr E e1 = {{1}};
+  constexpr E e2 = e1; // both-error {{constant}} \
+                       // ref-note {{read of mutable member 'b'}} \
+                       // both-note {{in call}}
+}
+
 #endif
 
 namespace AddressComparison {
diff --git a/clang/test/Analysis/div-zero-cxx20.cpp b/clang/test/Analysis/div-zero-cxx20.cpp
new file mode 100644
index 0000000000000..00ea96e796777
--- /dev/null
+++ b/clang/test/Analysis/div-zero-cxx20.cpp
@@ -0,0 +1,61 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core.DivideZero -std=c++20 -verify %s
+
+namespace GH148875 {
+struct A {
+  int x;
+  A(int v) : x(v) {}
+};
+
+struct B {
+  int x;
+  B() : x(0) {}
+};
+
+struct C {
+  int x, y;
+  C(int a, int b) : x(a), y(b) {}
+};
+
+struct D {
+  int x;
+};
+
+struct E {
+  D d;
+  E(int a) : d(a) {}
+};
+
+struct F {
+  int x;
+};
+
+int t1() {
+  A a{42};
+  return 1 / (a.x - 42); // expected-warning {{Division by zero}}
+}
+
+int t2() {
+  B b{};
+  return 1 / b.x; // expected-warning {{Division by zero}}
+}
+
+int t3() {
+  C c1{1, -1};
+  return 1 / (c1.x + c1.y); // expected-warning {{Division by zero}}
+}
+
+int t4() {
+  C c2{0, 0};
+  return 1 / (c2.x + c2.y); // expected-warning {{Division by zero}}
+}
+
+int t5() {
+  E e{32};
+  return 1 / (e.d.x - 32); // expected-warning {{Division by zero}}
+}
+
+int t6() {
+  F f(32);
+  return 1 / (f.x - 32); // expected-warning {{Division by zero}}
+}
+} // namespace GH148875
diff --git a/clang/test/Analysis/div-zero.cpp b/clang/test/Analysis/div-zero.cpp
index 063450d8883b0..51ea25e828a18 100644
--- a/clang/test/Analysis/div-zero.cpp
+++ b/clang/test/Analysis/div-zero.cpp
@@ -11,3 +11,63 @@ int fooPR10616 (int qX ) {
   return (a % (qX-1)); // expected-warning {{Division by zero}}
 
 }
+
+namespace GH148875 {
+struct A {
+  int x;
+  A(int v) : x(v) {}
+};
+
+struct B {
+  int x;
+  B() : x(0) {}
+};
+
+struct C {
+  int x, y;
+  C(int a, int b) : x(a), y(b) {}
+};
+
+struct D {
+  int x;
+};
+
+struct E {
+  D d;
+  E(int a) : d{a} {}
+};
+
+struct F {
+  int x;
+};
+
+int t1() {
+  A a{42};
+  return 1 / (a.x - 42); // expected-warning {{Division by zero}}
+}
+
+int t2() {
+  B b{};
+  return 1 / b.x; // expected-warning {{Division by zero}}
+}
+
+int t3() {
+  C c1{1, -1};
+  return 1 / (c1.x + c1.y); // expected-warning {{Division by zero}}
+}
+
+int t4() {
+  C c2{0, 0};
+  return 1 / (c2.x + c2.y); // expected-warning {{Division by zero}}
+}
+
+int t5() {
+  E e{32};
+  return 1 / (e.d.x - 32); // expected-warning {{Division by zero}}
+}
+
+int t6() {
+  F f{32};
+  return 1 / (f.x - 32); // expected-warning {{Division by zero}}
+}
+}
diff --git a/clang/test/CodeGen/64bit-swiftcall.c b/clang/test/CodeGen/64bit-swiftcall.c
index 7f8aa02d97ce1..448bca7acbca3 100644
--- a/clang/test/CodeGen/64bit-swiftcall.c
+++ b/clang/test/CodeGen/64bit-swiftcall.c
@@ -239,7 +239,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define {{.*}} void @return_struct_big_1(ptr dead_on_unwind noalias writable sret
 
 // Should not be byval.
-// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr{{( %.*)?}})
+// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr dead_on_return{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGen/AArch64/byval-temp.c b/clang/test/CodeGen/AArch64/byval-temp.c
index 0ee0312b2362d..5033b6cf5ac03 100644
--- a/clang/test/CodeGen/AArch64/byval-temp.c
+++ b/clang/test/CodeGen/AArch64/byval-temp.c
@@ -30,10 +30,10 @@ void example(void) {
 // Then, memcpy `l` to the temporary stack space.
 // CHECK-O0-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
 // Finally, call using a pointer to the temporary stack space.
-// CHECK-O0-NEXT: call void @pass_large(ptr noundef %[[byvaltemp]])
+// CHECK-O0-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
 // Now, do the same for the second call, using the second temporary alloca.
 // CHECK-O0-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
-// CHECK-O0-NEXT: call void @pass_large(ptr noundef %[[byvaltemp1]])
+// CHECK-O0-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
 // CHECK-O0-NEXT: ret void
 //
 // At O3, we should have lifetime markers to help the optimizer re-use the temporary allocas.
@@ -58,7 +58,7 @@ void example(void) {
 // Then, memcpy `l` to the temporary stack space.
 // CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
 // Finally, call using a pointer to the temporary stack space.
-// CHECK-O3-NEXT: call void @pass_large(ptr noundef %[[byvaltemp]])
+// CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
 //
 // The lifetime of the temporary used to pass a pointer to the struct ends here.
 // CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp]])
@@ -66,7 +66,7 @@ void example(void) {
 // Now, do the same for the second call, using the second temporary alloca.
 // CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %[[byvaltemp1]])
 // CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
-// CHECK-O3-NEXT: call void @pass_large(ptr noundef %[[byvaltemp1]])
+// CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
 // CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp1]])
 //
 // Mark the end of the lifetime of `l`.
@@ -88,12 +88,12 @@ void example_BitInt(void) {
 // CHECK-O0-NEXT:    [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
 // CHECK-O0-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
 // CHECK-O0-NEXT:    store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16
-// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP]])
+// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
 // CHECK-O0-NEXT:    [[TMP1:%.*]] = load i256, ptr [[L]], align 16
 // CHECK-O0-NEXT:    [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
 // CHECK-O0-NEXT:    [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
 // CHECK-O0-NEXT:    store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16
-// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP1]])
+// CHECK-O0-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
 // CHECK-O0-NEXT:    ret void
 //
 // CHECK-O3-LABEL: define dso_local void @example_BitInt(
@@ -108,13 +108,13 @@ void example_BitInt(void) {
 // CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]]) 
 // CHECK-O3-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
 // CHECK-O3-NEXT:    store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16, !tbaa [[TBAA6]]
-// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP]])
+// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]]) 
 // CHECK-O3-NEXT:    [[TMP1:%.*]] = load i256, ptr [[L]], align 16, !tbaa [[TBAA6]]
 // CHECK-O3-NEXT:    [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
 // CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]]) 
 // CHECK-O3-NEXT:    [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
 // CHECK-O3-NEXT:    store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16, !tbaa [[TBAA6]]
-// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP1]])
+// CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]]) 
 // CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[L]]) 
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c b/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
index 546910068c78a..804e14a2ea34b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
@@ -19,7 +19,7 @@ void f0(S0 *p) {
   use0(*p);
 }
 // CHECK-C:   declare void @use0(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
-// CHECK-CXX: declare void @use0(ptr noundef)
+// CHECK-CXX: declare void @use0(ptr dead_on_return noundef)
 
 #ifdef __cplusplus
 
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args.c b/clang/test/CodeGen/AArch64/pure-scalable-args.c
index fecd370d09be3..48988f7a1722b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args.c
@@ -92,7 +92,7 @@ void test_argpass_simple(PST *p) {
 // CHECK-AAPCS-NEXT: ret void
 
 // CHECK-AAPCS:  declare void @argpass_simple_callee(<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_simple_callee(ptr noundef)
+// CHECK-DARWIN: declare void @argpass_simple_callee(ptr dead_on_return noundef)
 
 // Boundary case of using the last available Z-reg, PST expanded.
 //   0.0  -> d0-d3
@@ -107,7 +107,7 @@ void test_argpass_last_z(PST *p) {
     argpass_last_z_callee(.0, .0, .0, .0, *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr dead_on_return noundef)
 
 
 // Like the above, but using a tuple type to occupy some registers.
@@ -123,7 +123,7 @@ void test_argpass_last_z_tuple(PST *p, svfloat64x4_t x) {
   argpass_last_z_tuple_callee(x, *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr dead_on_return noundef)
 
 
 // Boundary case of using the last available P-reg, PST expanded.
@@ -139,7 +139,7 @@ void test_argpass_last_p(PST *p) {
     argpass_last_p_callee(svpfalse(), svpfalse_c(), *p);
 }
 // CHECK-AAPCS:  declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr dead_on_return noundef)
 
 
 // Not enough Z-regs, push PST to memory and pass a pointer, Z-regs and
@@ -157,7 +157,7 @@ void test_argpass_no_z(PST *p, double dummy, svmfloat8_t u, int8x16_t v, mfloat8
     void argpass_no_z_callee(svmfloat8_t, int8x16_t, mfloat8x16_t, double, double, int, PST, int, double, svbool_t);
     argpass_no_z_callee(u, v, w, .0, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Like the above, using a tuple to occupy some registers.
@@ -173,7 +173,7 @@ void test_argpass_no_z_tuple_f64(PST *p, float dummy, svfloat64x4_t x) {
                                      double, svbool_t);
   argpass_no_z_tuple_f64_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Likewise, using a different tuple.
@@ -189,7 +189,7 @@ void test_argpass_no_z_tuple_mfp8(PST *p, float dummy, svmfloat8x4_t x) {
                                       double, svbool_t);
   argpass_no_z_tuple_mfp8_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Not enough Z-regs (consumed by a HFA), PST passed indirectly
@@ -204,8 +204,8 @@ void test_argpass_no_z_hfa(HFA *h, PST *p) {
     void argpass_no_z_hfa_callee(double, HFA, int, PST, int, svbool_t);
     argpass_no_z_hfa_callee(.0, *h, 1, *p, 2, svptrue_b64());
 }
-// CHECK-AAPCS:  declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
 
 // Not enough Z-regs (consumed by a HVA), PST passed indirectly
 //   0.0  -> d0
@@ -219,8 +219,8 @@ void test_argpass_no_z_hva(HVA *h, PST *p) {
     void argpass_no_z_hva_callee(double, HVA, int, PST, int, svbool_t);
     argpass_no_z_hva_callee(.0, *h, 1, *p, 2, svptrue_b64());
 }
-// CHECK-AAPCS:  declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
 
 // Not enough P-regs, PST passed indirectly, Z-regs and P-regs still available.
 //   true -> p0-p2
@@ -233,7 +233,7 @@ void test_argpass_no_p(PST *p) {
     void argpass_no_p_callee(svbool_t, svbool_t, svbool_t, int, PST, int, double, svbool_t);
     argpass_no_p_callee(svptrue_b8(), svptrue_b16(), svptrue_b32(), 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // Like above, using a tuple to occupy some registers.
@@ -250,7 +250,7 @@ void test_argpass_no_p_tuple(PST *p, svbool_t u, svboolx2_t v) {
                                  svbool_t);
   argpass_no_p_tuple_callee(v, u, 1, *p, 2, 3.0, svptrue_b64());
 }
-// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
 
 
 // HFAs go back-to-back to memory, afterwards Z-regs not available, PST passed indirectly.
@@ -263,8 +263,8 @@ void test_after_hfa(HFA *h, PST *p) {
     void after_hfa_callee(double, double, double, double, double, HFA, PST, HFA, svbool_t);
     after_hfa_callee(.0, .0, .0, .0, .0, *h, *p, *h, svpfalse());
 }
-// CHECK-AAPCS:  declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr noundef, [4 x float], <vscale x 16 x i1>)
+// CHECK-AAPCS:  declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr dead_on_return noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr dead_on_return noundef, [4 x float], <vscale x 16 x i1>)
 
 // Small PST, not enough registers, passed indirectly, unlike other small
 // aggregates.
@@ -277,7 +277,7 @@ void test_small_pst(SmallPST *p, SmallAgg *s) {
     void small_pst_callee(SmallAgg, double, double, double, double, double, double, double, double, double, SmallPST, double);
     small_pst_callee(*s, .0, .0, .0, .0, .0, .0, .0, .0, 1.0, *p, 2.0);
 }
-// CHECK-AAPCS:  declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr noundef, double noundef)
+// CHECK-AAPCS:  declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr dead_on_return noundef, double noundef)
 // CHECK-DARWIN: declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, i128, double noundef)
 
 
@@ -326,12 +326,12 @@ void test_pass_variadic(PST *p, PST *q) {
     pass_variadic_callee(*p, *q);
 }
 // CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
-// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr noundef nonnull %byval-temp)
+// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr dead_on_return noundef nonnull %byval-temp)
 
 // CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %p, i64 96, i1 false)
 // CHECK-DARWIN: call void @llvm.lifetime.start.p0(i64 96, ptr nonnull %byval-temp1)
 // CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp1, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
-// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr noundef nonnull %byval-temp, ptr noundef nonnull %byval-temp1)
+// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr dead_on_return noundef nonnull %byval-temp, ptr dead_on_return noundef nonnull %byval-temp1)
 
 
 // Test passing a small PST, still passed indirectly, despite being <= 128 bits
@@ -340,7 +340,7 @@ void test_small_pst_variadic(SmallPST *p) {
     small_pst_variadic_callee(0, *p);
 }
 // CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) %byval-temp, ptr noundef nonnull align 16 dereferenceable(16) %p, i64 16, i1 false)
-// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr noundef nonnull %byval-temp)
+// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr dead_on_return noundef nonnull %byval-temp)
 
 // CHECK-DARWIN: %0 = load i128, ptr %p, align 16
 // CHECK-DARWIN: tail call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, i128 %0)
@@ -467,7 +467,7 @@ void test_tuple_reg_count(svfloat32_t x, svfloat32x2_t y) {
                                    svfloat32_t, svfloat32_t, svfloat32_t, svfloat32x2_t);
   test_tuple_reg_count_callee(x, x, x, x, x, x, x, y);
 }
-// CHECK-AAPCS: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, ptr noundef)
+// CHECK-AAPCS: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, ptr dead_on_return noundef)
 // CHECK-DARWIN: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
 
 // Regression test for incorrect passing of SVE vector tuples
@@ -476,5 +476,5 @@ void test_tuple_reg_count_bool(svboolx4_t x, svboolx4_t y) {
   void test_tuple_reg_count_bool_callee(svboolx4_t, svboolx4_t);
   test_tuple_reg_count_bool_callee(x, y);
 }
-// CHECK-AAPCS:  declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, ptr noundef)
+// CHECK-AAPCS:  declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, ptr dead_on_return noundef)
 // CHECK-DARWIN: declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
diff --git a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
index b1232921df363..f0c9ef28201a5 100644
--- a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
+++ b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
@@ -82,7 +82,7 @@ struct Sppp {
     int *x, *y, *z;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z4Tppp4Sppp(
-// CHECK-A64-SAME: ptr noundef [[S:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr dead_on_return noundef [[S:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-A64-NEXT:    store ptr [[S]], ptr [[S_INDIRECT_ADDR]], align 8
@@ -490,7 +490,7 @@ struct Spa3 {
     int* xs[3];
 };
 // CHECK-A64-LABEL: define dso_local void @_Z4Tpa34Spa3(
-// CHECK-A64-SAME: ptr noundef [[S:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr dead_on_return noundef [[S:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-A64-NEXT:    store ptr [[S]], ptr [[S_INDIRECT_ADDR]], align 8
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
index c3d0541229fac..d244a8ba88572 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
@@ -58,7 +58,7 @@ typedef int8_t vec_int8 __attribute__((vector_size(N / 8)));
 // CHECK128-NEXT:    ret <16 x i8> [[CASTFIXEDSVE]]
 
 // CHECK-LABEL: define{{.*}} void @f2(
-// CHECK-SAME:   ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr noundef readonly captures(none) %0)
+// CHECK-SAME:   ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr dead_on_return noundef readonly captures(none) %0)
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   [[X:%.*]] = load <[[#div(VBITS,8)]] x i8>, ptr [[TMP0:%.*]], align 16, [[TBAA6:!tbaa !.*]]
 // CHECK-NEXT:   [[CASTSCALABLESVE:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v[[#div(VBITS,8)]]i8(<vscale x 16 x i8> poison, <[[#div(VBITS,8)]] x i8> [[X]], i64 0)
@@ -88,13 +88,13 @@ typedef svint8_t vec2 __attribute__((arm_sve_vector_bits(N)));
 // CHECK-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS,8)]] x i8> @llvm.vector.extract.v[[#div(VBITS,8)]]i8.nxv16i8(<vscale x 16 x i8> [[X_COERCE:%.*]], i64 0)
 // CHECK-NEXT:   call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:   store <[[#div(VBITS,8)]] x i8> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6]]
-// CHECK-NEXT:   call void @f3(ptr noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
+// CHECK-NEXT:   call void @f3(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
 // CHECK-NEXT:   call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:   ret void
 
 // CHECK128-LABEL: declare void @f3(<16 x i8> noundef)
 
 // CHECK-LABEL: declare void @f3(
-// CHECK-SAME:   ptr noundef)
+// CHECK-SAME:   ptr dead_on_return noundef)
 void g(vec2 x) { f3(x); } // OK
 #endif
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
index e82069aab2486..d42ecb663050f 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
@@ -75,7 +75,7 @@ typedef svint16_t vec2 __attribute__((arm_sve_vector_bits(N)));
 // CHECKWIDE-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS, 16)]] x i16> @llvm.vector.extract.v[[#div(VBITS, 16)]]i16.nxv8i16(<vscale x 8 x i16> [[X_COERCE:%.*]], i64 0)
 // CHECKWIDE-NEXT:   call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECKWIDE-NEXT:   store <[[#div(VBITS, 16)]] x i16> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6:!tbaa !.*]]
-// CHECKWIDE-NEXT:   call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
+// CHECKWIDE-NEXT:   call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
 // CHECKWIDE-NEXT:   call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECKWIDE-NEXT:   ret void
 void g(vec2 x) { f(x); } // OK
diff --git a/clang/test/CodeGen/LoongArch/bitint.c b/clang/test/CodeGen/LoongArch/bitint.c
index f346f569d0eb0..950f5308e32cc 100644
--- a/clang/test/CodeGen/LoongArch/bitint.c
+++ b/clang/test/CodeGen/LoongArch/bitint.c
@@ -26,12 +26,12 @@ void pass_BitInt129(_BitInt(129));
 // LA32-NEXT:    [[LOADEDV1:%.*]] = trunc i128 [[TMP1]] to i65
 // LA32-NEXT:    [[STOREDV:%.*]] = sext i65 [[LOADEDV1]] to i128
 // LA32-NEXT:    store i128 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
-// LA32-NEXT:    call void @pass_BitInt65(ptr noundef [[BYVAL_TEMP]])
+// LA32-NEXT:    call void @pass_BitInt65(ptr dead_on_return noundef [[BYVAL_TEMP]])
 // LA32-NEXT:    [[TMP2:%.*]] = load i256, ptr [[L129]], align 16
 // LA32-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
 // LA32-NEXT:    [[STOREDV4:%.*]] = sext i129 [[LOADEDV2]] to i256
 // LA32-NEXT:    store i256 [[STOREDV4]], ptr [[BYVAL_TEMP3]], align 16
-// LA32-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP3]])
+// LA32-NEXT:    call void @pass_BitInt129(ptr dead_on_return noundef [[BYVAL_TEMP3]])
 // LA32-NEXT:    ret void
 //
 // LA64-LABEL: define dso_local void @example_BitInt(
@@ -54,7 +54,7 @@ void pass_BitInt129(_BitInt(129));
 // LA64-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
 // LA64-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV2]] to i256
 // LA64-NEXT:    store i256 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
-// LA64-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP]])
+// LA64-NEXT:    call void @pass_BitInt129(ptr dead_on_return noundef [[BYVAL_TEMP]])
 // LA64-NEXT:    ret void
 //
 void example_BitInt(void) {
diff --git a/clang/test/CodeGen/PowerPC/ppc64-vector.c b/clang/test/CodeGen/PowerPC/ppc64-vector.c
index 5d3dd86a009d5..2e99781f84910 100644
--- a/clang/test/CodeGen/PowerPC/ppc64-vector.c
+++ b/clang/test/CodeGen/PowerPC/ppc64-vector.c
@@ -39,7 +39,7 @@ v8i16 test_v8i16(v8i16 x)
   return x;
 }
 
-// CHECK: define{{.*}} void @test_v16i16(ptr dead_on_unwind noalias writable sret(<16 x i16>) align 32 %agg.result, ptr noundef %0)
+// CHECK: define{{.*}} void @test_v16i16(ptr dead_on_unwind noalias writable sret(<16 x i16>) align 32 %agg.result, ptr dead_on_return noundef %0)
 v16i16 test_v16i16(v16i16 x)
 {
   return x;
diff --git a/clang/test/CodeGen/RISCV/riscv-abi.cpp b/clang/test/CodeGen/RISCV/riscv-abi.cpp
index fe1a2b6d8595c..d2e080829e72f 100644
--- a/clang/test/CodeGen/RISCV/riscv-abi.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-abi.cpp
@@ -75,7 +75,7 @@ struct child3_int64_s : parent3_float_s {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @_Z30float_int64_struct_inheritance14child3_int64_s
-// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD3_INT64_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD3_INT64_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 // LP64-LABEL: define dso_local [2 x i64] @_Z30float_int64_struct_inheritance14child3_int64_s
@@ -99,7 +99,7 @@ struct child4_double_s : parent4_double_s {
 };
 
 // ILP32-ILP32F-LABEL: define dso_local void @_Z32double_double_struct_inheritance15child4_double_s
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD4_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD4_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @_Z32double_double_struct_inheritance15child4_double_s
@@ -130,11 +130,11 @@ struct child5_virtual_s : virtual parent5_virtual_s {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @_Z38int32_float_virtual_struct_inheritance16child5_virtual_s
-// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 // LP64-LP64F-LP64D-LABEL: define dso_local void @_Z38int32_float_virtual_struct_inheritance16child5_virtual_s
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct child5_virtual_s int32_float_virtual_struct_inheritance(struct child5_virtual_s a) {
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
index 82e43fff0c3aa..bc89cb532bdcc 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
@@ -138,7 +138,7 @@ struct st_i32x4x9 {
 
 typedef int __attribute__((vector_size(256))) int32x64_t;
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_too_large(ptr noundef %0)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_too_large(ptr dead_on_return noundef %0)
 void __attribute__((riscv_vls_cc)) test_too_large(int32x64_t arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_too_large_256(<vscale x 16 x i32> noundef %arg.coerce)
 void __attribute__((riscv_vls_cc(256))) test_too_large_256(int32x64_t arg) {}
@@ -173,9 +173,9 @@ void __attribute__((riscv_vls_cc)) test_st_i32x8x2(struct st_i32x8x2 arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x8x2_256(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc)) test_st_i32x64x2(struct st_i32x64x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
@@ -188,7 +188,7 @@ void __attribute__((riscv_vls_cc)) test_st_i32x4x8(struct st_i32x4x8 arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x8_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc)) test_st_i32x4x9(struct st_i32x4x9 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
index 5f6539796c20d..128610e578c26 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
@@ -118,7 +118,7 @@ struct st_i32x4x9 {
 
 typedef int __attribute__((vector_size(256))) int32x64_t;
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z14test_too_largeDv64_i(ptr noundef %0)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z14test_too_largeDv64_i(ptr dead_on_return noundef %0)
 [[riscv::vls_cc]] void test_too_large(int32x64_t arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z18test_too_large_256Dv64_i(<vscale x 16 x i32> noundef %arg.coerce)
 [[riscv::vls_cc(256)]] void test_too_large_256(int32x64_t arg) {}
@@ -153,9 +153,9 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x8x2_25610st_i32x8x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc]] void test_st_i32x64x2(struct st_i32x64x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x310st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
@@ -168,7 +168,7 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x8_25610st_i32x4x8(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc]] void test_st_i32x4x9(struct st_i32x4x9 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x9_25610st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x9_25610st_i32x4x9(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
diff --git a/clang/test/CodeGen/RISCV/riscv32-abi.c b/clang/test/CodeGen/RISCV/riscv32-abi.c
index b53f9a9169146..a9e56d40817ae 100644
--- a/clang/test/CodeGen/RISCV/riscv32-abi.c
+++ b/clang/test/CodeGen/RISCV/riscv32-abi.c
@@ -246,7 +246,7 @@ struct large {
 };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_agg_large
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[X:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[X:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_agg_large(struct large x) {
@@ -266,7 +266,7 @@ struct large f_agg_large_ret(int32_t i, int8_t j) {
 typedef unsigned char v16i8 __attribute__((vector_size(16)));
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_vec_large_v16i8
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[TMP0:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[TMP0:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_vec_large_v16i8(v16i8 x) {
@@ -285,7 +285,7 @@ v16i8 f_vec_large_v16i8_ret(void) {
 // if they were passed in registers.
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local i32 @f_scalar_stack_1
-// ILP32-ILP32F-ILP32D-SAME: (i32 [[A_COERCE:%.*]], [2 x i32] [[B_COERCE:%.*]], i64 [[C_COERCE:%.*]], ptr noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (i32 [[A_COERCE:%.*]], [2 x i32] [[B_COERCE:%.*]], i64 [[C_COERCE:%.*]], ptr dead_on_return noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
@@ -343,7 +343,7 @@ struct large f_scalar_stack_6(float a, int64_t b, double c, long double d,
 // they would be if passed via registers.
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_agg_stack
-// ILP32-ILP32F-ILP32D-SAME: (double noundef [[A:%.*]], i64 noundef [[B:%.*]], double noundef [[C:%.*]], i64 noundef [[D:%.*]], i32 [[E_COERCE:%.*]], [2 x i32] [[F_COERCE:%.*]], i64 [[G_COERCE:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (double noundef [[A:%.*]], i64 noundef [[B:%.*]], double noundef [[C:%.*]], i64 noundef [[D:%.*]], i32 [[E_COERCE:%.*]], [2 x i32] [[F_COERCE:%.*]], i64 [[G_COERCE:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_agg_stack(double a, int64_t b, double c, int64_t d, struct tiny e,
@@ -366,7 +366,7 @@ struct double_int8_s { double d; int64_t i; };
 struct int_double_s { int a; double b; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_int_double_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_int_double_s_arg
@@ -482,7 +482,7 @@ struct zbf_double_zbf_s f_ret_zbf_double_zbf_s(void) {
 struct double_float_s { double f; float g; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_double_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_double_s_arg
@@ -504,7 +504,7 @@ struct double_double_s f_ret_double_double_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_float_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_float_s_arg
@@ -526,7 +526,7 @@ struct double_float_s f_ret_double_float_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_double_s_arg_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_double_s_arg_insufficient_fprs(float a, double b, double c, double d,
@@ -543,7 +543,7 @@ struct double_int64bf_s { double d; int64_t i : 32; };
 struct double_int8_zbf_s { double d; int8_t i; int : 0; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int8_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int8_s_arg(struct double_int8_s a) {}
@@ -557,7 +557,7 @@ struct double_int8_s f_ret_double_int8_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_uint8_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_uint8_s_arg
@@ -579,7 +579,7 @@ struct double_uint8_s f_ret_double_uint8_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int32_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int32_s_arg
@@ -601,7 +601,7 @@ struct double_int32_s f_ret_double_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int64_s_arg(struct double_int64_s a) {}
@@ -615,7 +615,7 @@ struct double_int64_s f_ret_double_int64_s(void) {
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int64bf_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int64bf_s_arg
@@ -640,7 +640,7 @@ struct double_int64bf_s f_ret_double_int64bf_s(void) {
 // floating point calling convention.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_int8_zbf_s
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_int8_zbf_s
@@ -662,14 +662,14 @@ struct double_int8_zbf_s f_ret_double_int8_zbf_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int8_s_arg_insufficient_gprs
-// ILP32-ILP32F-ILP32D-SAME: (i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], i32 noundef [[H:%.*]], ptr noundef [[I:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], i32 noundef [[H:%.*]], ptr dead_on_return noundef [[I:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_int8_s_arg_insufficient_gprs(int a, int b, int c, int d, int e,
                                           int f, int g, int h, struct double_int8_s i) {}
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_struct_double_int8_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], double noundef [[H:%.*]], ptr noundef [[I:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], double noundef [[H:%.*]], ptr dead_on_return noundef [[I:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_struct_double_int8_insufficient_fprs(float a, double b, double c, double d,
@@ -679,7 +679,7 @@ void f_struct_double_int8_insufficient_fprs(float a, double b, double c, double
 // floating-point value should be passed as if it were an fp+fp struct.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublecomplex
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublecomplex
@@ -703,7 +703,7 @@ double __complex__ f_ret_doublecomplex(void) {
 struct doublecomplex_s { double __complex__ c; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublecomplex_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublecomplex_s_arg
@@ -754,7 +754,7 @@ struct doublearr1_s f_ret_doublearr1_s(void) {
 struct doublearr2_s { double a[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_s_arg
@@ -778,7 +778,7 @@ struct doublearr2_s f_ret_doublearr2_s(void) {
 struct doublearr2_tricky1_s { struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky1_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky1_s_arg
@@ -802,7 +802,7 @@ struct doublearr2_tricky1_s f_ret_doublearr2_tricky1_s(void) {
 struct doublearr2_tricky2_s { struct {}; struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky2_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky2_s_arg
@@ -826,7 +826,7 @@ struct doublearr2_tricky2_s f_ret_doublearr2_tricky2_s(void) {
 struct doublearr2_tricky3_s { union {}; struct { double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky3_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky3_s_arg
@@ -850,7 +850,7 @@ struct doublearr2_tricky3_s f_ret_doublearr2_tricky3_s(void) {
 struct doublearr2_tricky4_s { union {}; struct { struct {}; double f[1]; } g[2]; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky4_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_doublearr2_tricky4_s_arg
@@ -877,7 +877,7 @@ struct doublearr2_tricky4_s f_ret_doublearr2_tricky4_s(void) {
 struct int_double_int_s { int a; double b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_double_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_double_int_s_arg(struct int_double_int_s a) {}
@@ -893,7 +893,7 @@ struct int_double_int_s f_ret_int_double_int_s(void) {
 struct int64_double_s { int64_t a; double b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_double_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_double_s_arg(struct int64_double_s a) {}
@@ -909,7 +909,7 @@ struct int64_double_s f_ret_int64_double_s(void) {
 struct char_char_double_s { char a; char b; double c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_char_char_double_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_char_char_double_s_arg(struct char_char_double_s a) {}
@@ -948,7 +948,7 @@ union double_u f_ret_double_u(void) {
 // double+double structs by the ABI.
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_double_int32_s_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_INT32_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_INT32_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, i32 } @f_ret_double_int32_s_double_int32_s_just_sufficient_gprs
@@ -961,7 +961,7 @@ struct double_int32_s f_ret_double_int32_s_double_int32_s_just_sufficient_gprs(
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_double_double_s_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @f_ret_double_double_s_double_int32_s_just_sufficient_gprs
@@ -974,7 +974,7 @@ struct double_double_s f_ret_double_double_s_double_int32_s_just_sufficient_gprs
 }
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_ret_doublecomplex_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local { double, double } @f_ret_doublecomplex_double_int32_s_just_sufficient_gprs
@@ -1189,7 +1189,7 @@ struct float_int32_s f_ret_float_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_float_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_float_int64_s_arg(struct float_int64_s a) {}
@@ -1465,7 +1465,7 @@ struct floatarr2_tricky4_s f_ret_floatarr2_tricky4_s(void) {
 struct int_float_int_s { int a; float b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_float_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_float_int_s_arg(struct int_float_int_s a) {}
@@ -1481,7 +1481,7 @@ struct int_float_int_s f_ret_int_float_int_s(void) {
 struct int64_float_s { int64_t a; float b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_float_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_float_s_arg(struct int64_float_s a) {}
@@ -1619,7 +1619,7 @@ struct zbf_float16_zbf_s f_ret_zbf_float16_zbf_s(void) {
 struct double_float16_s { double f; _Float16 g; };
 
 // ILP32-ILP32F-LABEL: define dso_local void @f_double_float16_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F:  entry:
 //
 // ILP32D-LABEL: define dso_local void @f_double_float16_s_arg
@@ -1641,7 +1641,7 @@ struct double_float16_s f_ret_double_float16_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_float16_s_arg_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_double_float16_s_arg_insufficient_fprs(float a, double b, double c, double d,
@@ -1725,7 +1725,7 @@ struct float16_int32_s f_ret_float16_int32_s(void) {
 }
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_float16_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_float16_int64_s_arg(struct float16_int64_s a) {}
@@ -2001,7 +2001,7 @@ struct float16arr2_tricky4_s f_ret_float16arr2_tricky4_s(void) {
 struct int_float16_int_s { int a; _Float16 b; int c; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_float16_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int_float16_int_s_arg(struct int_float16_int_s a) {}
@@ -2017,7 +2017,7 @@ struct int_float16_int_s f_ret_int_float16_int_s(void) {
 struct int64_float16_s { int64_t a; _Float16 b; };
 
 // ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_float16_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // ILP32-ILP32F-ILP32D:  entry:
 //
 void f_int64_float16_s_arg(struct int64_float16_s a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv32-vararg.c b/clang/test/CodeGen/RISCV/riscv32-vararg.c
index 2b332410f8637..ed301f9269bb8 100644
--- a/clang/test/CodeGen/RISCV/riscv32-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv32-vararg.c
@@ -64,7 +64,7 @@ int f_va_callee(int, ...);
 // CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL4]], i32 0, i32 0
 // CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[COERCE_DIVE]], align 8
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[BYVAL_TEMP]], ptr align 4 [[DOTCOMPOUNDLITERAL6]], i32 16, i1 false)
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 (i32, ...) @f_va_callee(i32 noundef 1, i32 noundef 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i32 [[TMP0]], [2 x i32] [[TMP1]], i64 [[TMP2]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 (i32, ...) @f_va_callee(i32 noundef 1, i32 noundef 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i32 [[TMP0]], [2 x i32] [[TMP1]], i64 [[TMP2]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    ret void
 //
 void f_va_caller(void) {
diff --git a/clang/test/CodeGen/RISCV/riscv64-abi.c b/clang/test/CodeGen/RISCV/riscv64-abi.c
index 021565238904e..dc01750e56970 100644
--- a/clang/test/CodeGen/RISCV/riscv64-abi.c
+++ b/clang/test/CodeGen/RISCV/riscv64-abi.c
@@ -242,7 +242,7 @@ struct large {
 };
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_agg_large
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[X:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[X:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_agg_large(struct large x) {
@@ -262,7 +262,7 @@ struct large f_agg_large_ret(int32_t i, int8_t j) {
 typedef unsigned char v32i8 __attribute__((vector_size(32)));
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_vec_large_v32i8
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[TMP0:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[TMP0:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_vec_large_v32i8(v32i8 x) {
@@ -281,7 +281,7 @@ v32i8 f_vec_large_v32i8_ret(void) {
 // if they were passed in registers.
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_1
-// LP64-LP64F-LP64D-SAME: (i64 [[A_COERCE:%.*]], [2 x i64] [[B_COERCE:%.*]], i128 [[C_COERCE:%.*]], ptr noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i64 [[A_COERCE:%.*]], [2 x i64] [[B_COERCE:%.*]], i128 [[C_COERCE:%.*]], ptr dead_on_return noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
@@ -290,7 +290,7 @@ int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_2
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], i64 noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], i64 noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_2(int32_t a, __int128_t b, int64_t c, long double d, v32i8 e,
@@ -299,7 +299,7 @@ int f_scalar_stack_2(int32_t a, __int128_t b, int64_t c, long double d, v32i8 e,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_3
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], double noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], double noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_3(int32_t a, __int128_t b, double c, long double d, v32i8 e,
@@ -312,7 +312,7 @@ int f_scalar_stack_3(int32_t a, __int128_t b, double c, long double d, v32i8 e,
 // to pass a pointer.
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_scalar_stack_4
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct large f_scalar_stack_4(uint32_t a, __int128_t b, long double c, v32i8 d,
@@ -321,7 +321,7 @@ struct large f_scalar_stack_4(uint32_t a, __int128_t b, long double c, v32i8 d,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_scalar_stack_5
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], double noundef [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], double noundef [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 struct large f_scalar_stack_5(double a, __int128_t b, long double c, v32i8 d,
@@ -330,7 +330,7 @@ struct large f_scalar_stack_5(double a, __int128_t b, long double c, v32i8 d,
 }
 
 // LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_6
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], float noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]], half noundef [[I:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], float noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]], half noundef [[I:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 int f_scalar_stack_6(int32_t a, __int128_t b, float c, long double d, v32i8 e,
@@ -1440,7 +1440,7 @@ struct doublearr2_tricky4_s f_ret_doublearr2_tricky4_s(void) {
 struct int_double_int_s { int a; double b; int c; };
 
 // LP64-LP64F-LP64D-LABEL: define dso_local void @f_int_double_int_s_arg
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
 // LP64-LP64F-LP64D:  entry:
 //
 void f_int_double_int_s_arg(struct int_double_int_s a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv64-vararg.c b/clang/test/CodeGen/RISCV/riscv64-vararg.c
index a278f74ca4a86..17802553c795a 100644
--- a/clang/test/CodeGen/RISCV/riscv64-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv64-vararg.c
@@ -74,7 +74,7 @@ int f_va_callee(int, ...);
 // CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL4]], i32 0, i32 0
 // CHECK-NEXT:    [[TMP2:%.*]] = load i128, ptr [[COERCE_DIVE]], align 16
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[BYVAL_TEMP]], ptr align 8 [[DOTCOMPOUNDLITERAL6]], i64 32, i1 false)
-// CHECK-NEXT:    [[CALL:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i64 [[TMP0]], [2 x i64] [[TMP1]], i128 [[TMP2]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    [[CALL:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i64 [[TMP0]], [2 x i64] [[TMP1]], i128 [[TMP2]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    [[CALL11:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i32 noundef signext 3, i32 noundef signext 4, fp128 noundef 0xL00000000000000004001400000000000, i32 noundef signext 6, i32 noundef signext 7, i32 noundef signext 8, i32 noundef signext 9)
 // CHECK-NEXT:    [[A13:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL12]], i32 0, i32 0
 // CHECK-NEXT:    store i128 5, ptr [[A13]], align 16
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi-vector.c b/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
index e5704709a3a33..fab6050a0d876 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
@@ -58,91 +58,91 @@ unsigned int align = __alignof__ (v16i8);
 // CHECK-VECTOR: @align ={{.*}} global i32 8
 
 v1i8 pass_v1i8(v1i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i8(ptr dead_on_unwind noalias writable sret(<1 x i8>) align 1 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i8(ptr dead_on_unwind noalias writable sret(<1 x i8>) align 1 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i8> @pass_v1i8(<1 x i8> %{{.*}})
 
 v2i8 pass_v2i8(v2i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i8(ptr dead_on_unwind noalias writable sret(<2 x i8>) align 2 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i8(ptr dead_on_unwind noalias writable sret(<2 x i8>) align 2 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i8> @pass_v2i8(<2 x i8> %{{.*}})
 
 v4i8 pass_v4i8(v4i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i8(ptr dead_on_unwind noalias writable sret(<4 x i8>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i8(ptr dead_on_unwind noalias writable sret(<4 x i8>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i8> @pass_v4i8(<4 x i8> %{{.*}})
 
 v8i8 pass_v8i8(v8i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v8i8(ptr dead_on_unwind noalias writable sret(<8 x i8>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v8i8(ptr dead_on_unwind noalias writable sret(<8 x i8>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <8 x i8> @pass_v8i8(<8 x i8> %{{.*}})
 
 v16i8 pass_v16i8(v16i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v16i8(ptr dead_on_unwind noalias writable sret(<16 x i8>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v16i8(ptr dead_on_unwind noalias writable sret(<16 x i8>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <16 x i8> @pass_v16i8(<16 x i8> %{{.*}})
 
 v32i8 pass_v32i8(v32i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 32 %{{.*}}, ptr %0)
-// CHECK-VECTOR-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 32 %{{.*}}, ptr dead_on_return %0)
+// CHECK-VECTOR-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 8 %{{.*}}, ptr dead_on_return %0)
 
 v1i16 pass_v1i16(v1i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i16(ptr dead_on_unwind noalias writable sret(<1 x i16>) align 2 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i16(ptr dead_on_unwind noalias writable sret(<1 x i16>) align 2 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i16> @pass_v1i16(<1 x i16> %{{.*}})
 
 v2i16 pass_v2i16(v2i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i16(ptr dead_on_unwind noalias writable sret(<2 x i16>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i16(ptr dead_on_unwind noalias writable sret(<2 x i16>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i16> @pass_v2i16(<2 x i16> %{{.*}})
 
 v4i16 pass_v4i16(v4i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i16(ptr dead_on_unwind noalias writable sret(<4 x i16>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i16(ptr dead_on_unwind noalias writable sret(<4 x i16>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i16> @pass_v4i16(<4 x i16> %{{.*}})
 
 v8i16 pass_v8i16(v8i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v8i16(ptr dead_on_unwind noalias writable sret(<8 x i16>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v8i16(ptr dead_on_unwind noalias writable sret(<8 x i16>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <8 x i16> @pass_v8i16(<8 x i16> %{{.*}})
 
 v1i32 pass_v1i32(v1i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i32(ptr dead_on_unwind noalias writable sret(<1 x i32>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i32(ptr dead_on_unwind noalias writable sret(<1 x i32>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i32> @pass_v1i32(<1 x i32> %{{.*}})
 
 v2i32 pass_v2i32(v2i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i32(ptr dead_on_unwind noalias writable sret(<2 x i32>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i32(ptr dead_on_unwind noalias writable sret(<2 x i32>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i32> @pass_v2i32(<2 x i32> %{{.*}})
 
 v4i32 pass_v4i32(v4i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i32(ptr dead_on_unwind noalias writable sret(<4 x i32>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i32(ptr dead_on_unwind noalias writable sret(<4 x i32>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x i32> @pass_v4i32(<4 x i32> %{{.*}})
 
 v1i64 pass_v1i64(v1i64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i64(ptr dead_on_unwind noalias writable sret(<1 x i64>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i64(ptr dead_on_unwind noalias writable sret(<1 x i64>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i64> @pass_v1i64(<1 x i64> %{{.*}})
 
 v2i64 pass_v2i64(v2i64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i64(ptr dead_on_unwind noalias writable sret(<2 x i64>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i64(ptr dead_on_unwind noalias writable sret(<2 x i64>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x i64> @pass_v2i64(<2 x i64> %{{.*}})
 
 v1i128 pass_v1i128(v1i128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i128(ptr dead_on_unwind noalias writable sret(<1 x i128>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i128(ptr dead_on_unwind noalias writable sret(<1 x i128>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x i128> @pass_v1i128(<1 x i128> %{{.*}})
 
 v1f32 pass_v1f32(v1f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f32(ptr dead_on_unwind noalias writable sret(<1 x float>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f32(ptr dead_on_unwind noalias writable sret(<1 x float>) align 4 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x float> @pass_v1f32(<1 x float> %{{.*}})
 
 v2f32 pass_v2f32(v2f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2f32(ptr dead_on_unwind noalias writable sret(<2 x float>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2f32(ptr dead_on_unwind noalias writable sret(<2 x float>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x float> @pass_v2f32(<2 x float> %{{.*}})
 
 v4f32 pass_v4f32(v4f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4f32(ptr dead_on_unwind noalias writable sret(<4 x float>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4f32(ptr dead_on_unwind noalias writable sret(<4 x float>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <4 x float> @pass_v4f32(<4 x float> %{{.*}})
 
 v1f64 pass_v1f64(v1f64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f64(ptr dead_on_unwind noalias writable sret(<1 x double>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f64(ptr dead_on_unwind noalias writable sret(<1 x double>) align 8 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x double> @pass_v1f64(<1 x double> %{{.*}})
 
 v2f64 pass_v2f64(v2f64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2f64(ptr dead_on_unwind noalias writable sret(<2 x double>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2f64(ptr dead_on_unwind noalias writable sret(<2 x double>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <2 x double> @pass_v2f64(<2 x double> %{{.*}})
 
 v1f128 pass_v1f128(v1f128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f128(ptr dead_on_unwind noalias writable sret(<1 x fp128>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f128(ptr dead_on_unwind noalias writable sret(<1 x fp128>) align 16 %{{.*}}, ptr dead_on_return %0)
 // CHECK-VECTOR-LABEL: define{{.*}} <1 x fp128> @pass_v1f128(<1 x fp128> %{{.*}})
 
 
@@ -170,13 +170,13 @@ struct agg_v8i8 pass_agg_v8i8(struct agg_v8i8 arg) { return arg; }
 
 struct agg_v16i8 { v16i8 a; };
 struct agg_v16i8 pass_agg_v16i8(struct agg_v16i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 // CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 8 %{{.*}}, <16 x i8> %{{.*}})
 
 struct agg_v32i8 { v32i8 a; };
 struct agg_v32i8 pass_agg_v32i8(struct agg_v32i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 32 %{{.*}}, ptr %{{.*}})
-// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 8 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 32 %{{.*}}, ptr dead_on_return %{{.*}})
+// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 8 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Verify that the following are *not* vector-like aggregate types
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi.c b/clang/test/CodeGen/SystemZ/systemz-abi.c
index f26084ab44eae..83137ae6d5f82 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi.c
@@ -50,7 +50,7 @@ long long pass_longlong(long long arg) { return arg; }
 // CHECK-LABEL: define{{.*}} i64 @pass_longlong(i64 %{{.*}})
 
 __int128 pass_int128(__int128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr dead_on_return %0)
 
 _Float16 pass__Float16(_Float16 arg) { return arg; }
 // CHECK-LABEL: define{{.*}} half @pass__Float16(half %{{.*}})
@@ -62,37 +62,37 @@ double pass_double(double arg) { return arg; }
 // CHECK-LABEL: define{{.*}} double @pass_double(double %{{.*}})
 
 long double pass_longdouble(long double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_longdouble(ptr dead_on_unwind noalias writable sret(fp128) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_longdouble(ptr dead_on_unwind noalias writable sret(fp128) align 8 %{{.*}}, ptr dead_on_return %0)
 
 
 // Complex types
 
 _Complex char pass_complex_char(_Complex char arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_char(ptr dead_on_unwind noalias writable sret({ i8, i8 }) align 1 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_char(ptr dead_on_unwind noalias writable sret({ i8, i8 }) align 1 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex short pass_complex_short(_Complex short arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_short(ptr dead_on_unwind noalias writable sret({ i16, i16 }) align 2 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_short(ptr dead_on_unwind noalias writable sret({ i16, i16 }) align 2 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex int pass_complex_int(_Complex int arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_int(ptr dead_on_unwind noalias writable sret({ i32, i32 }) align 4 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_int(ptr dead_on_unwind noalias writable sret({ i32, i32 }) align 4 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long pass_complex_long(_Complex long arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_long(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_long(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long long pass_complex_longlong(_Complex long long arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex _Float16 pass_complex__Float16(_Complex _Float16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex float pass_complex_float(_Complex float arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex double pass_complex_double(_Complex double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_double(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_double(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 _Complex long double pass_complex_longdouble(_Complex long double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_longdouble(ptr dead_on_unwind noalias writable sret({ fp128, fp128 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_longdouble(ptr dead_on_unwind noalias writable sret({ fp128, fp128 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
 
 
 // Aggregate types
@@ -107,7 +107,7 @@ struct agg_2byte pass_agg_2byte(struct agg_2byte arg) { return arg; }
 
 struct agg_3byte { char a[3]; };
 struct agg_3byte pass_agg_3byte(struct agg_3byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_3byte(ptr dead_on_unwind noalias writable sret(%struct.agg_3byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_3byte(ptr dead_on_unwind noalias writable sret(%struct.agg_3byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_4byte { char a[4]; };
 struct agg_4byte pass_agg_4byte(struct agg_4byte arg) { return arg; }
@@ -115,15 +115,15 @@ struct agg_4byte pass_agg_4byte(struct agg_4byte arg) { return arg; }
 
 struct agg_5byte { char a[5]; };
 struct agg_5byte pass_agg_5byte(struct agg_5byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_5byte(ptr dead_on_unwind noalias writable sret(%struct.agg_5byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_5byte(ptr dead_on_unwind noalias writable sret(%struct.agg_5byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_6byte { char a[6]; };
 struct agg_6byte pass_agg_6byte(struct agg_6byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_6byte(ptr dead_on_unwind noalias writable sret(%struct.agg_6byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_6byte(ptr dead_on_unwind noalias writable sret(%struct.agg_6byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_7byte { char a[7]; };
 struct agg_7byte pass_agg_7byte(struct agg_7byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_7byte(ptr dead_on_unwind noalias writable sret(%struct.agg_7byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_7byte(ptr dead_on_unwind noalias writable sret(%struct.agg_7byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_8byte { char a[8]; };
 struct agg_8byte pass_agg_8byte(struct agg_8byte arg) { return arg; }
@@ -131,7 +131,7 @@ struct agg_8byte pass_agg_8byte(struct agg_8byte arg) { return arg; }
 
 struct agg_16byte { char a[16]; };
 struct agg_16byte pass_agg_16byte(struct agg_16byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_16byte(ptr dead_on_unwind noalias writable sret(%struct.agg_16byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_16byte(ptr dead_on_unwind noalias writable sret(%struct.agg_16byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Float-like aggregate types
@@ -153,7 +153,7 @@ struct agg_double pass_agg_double(struct agg_double arg) { return arg; }
 
 struct agg_longdouble { long double a; };
 struct agg_longdouble pass_agg_longdouble(struct agg_longdouble arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg__Float16_a4 { _Float16 a __attribute__((aligned (4))); };
 struct agg__Float16_a4 pass_agg__Float16_a4(struct agg__Float16_a4 arg) { return arg; }
@@ -167,7 +167,7 @@ struct agg__Float16_a8 pass_agg__Float16_a8(struct agg__Float16_a8 arg) { return
 
 struct agg__Float16_a16 { _Float16 a __attribute__((aligned (16))); };
 struct agg__Float16_a16 pass_agg__Float16_a16(struct agg__Float16_a16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 
 struct agg_float_a8 { float a __attribute__((aligned (8))); };
 struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
@@ -176,7 +176,7 @@ struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
 
 struct agg_float_a16 { float a __attribute__((aligned (16))); };
 struct agg_float_a16 pass_agg_float_a16(struct agg_float_a16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_float_a16(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a16) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_float_a16(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a16) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
 
 
 // Verify that the following are *not* float-like aggregate types
diff --git a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
index 434937a66389c..d76fb4bd1fda6 100644
--- a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
+++ b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
@@ -131,7 +131,7 @@ double test_f64(double f, double g) {
 long double test_f128(long double f, long double g) {
   asm("axbr %0, %2" : "=f" (f) : "0" (f), "f" (g));
   return f;
-// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr noundef readonly captures(none) %0, ptr noundef readonly captures(none) %1)
+// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr dead_on_return noundef readonly captures(none) %0, ptr dead_on_return noundef readonly captures(none) %1)
 // CHECK: %f = load fp128, ptr %0
 // CHECK: %g = load fp128, ptr %1
 // CHECK: [[RESULT:%.*]] = tail call fp128 asm "axbr $0, $2", "=f,0,f"(fp128 %f, fp128 %g)
diff --git a/clang/test/CodeGen/X86/cx-complex-range.c b/clang/test/CodeGen/X86/cx-complex-range.c
index f87091427df71..b16b10b7b8a21 100644
--- a/clang/test/CodeGen/X86/cx-complex-range.c
+++ b/clang/test/CodeGen/X86/cx-complex-range.c
@@ -1064,7 +1064,7 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // PRMTD-NEXT:    ret <2 x half> [[TMP33]]
 //
 // X86WINPRMTD-LABEL: define dso_local i32 @f1(
-// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[RETVAL:%.*]] = alloca { half, half }, align 2
 // X86WINPRMTD-NEXT:    [[A:%.*]] = alloca { half, half }, align 2
diff --git a/clang/test/CodeGen/X86/x86_32-arguments-win32.c b/clang/test/CodeGen/X86/x86_32-arguments-win32.c
index 5b81c43f4bbb8..53d040af725b0 100644
--- a/clang/test/CodeGen/X86/x86_32-arguments-win32.c
+++ b/clang/test/CodeGen/X86/x86_32-arguments-win32.c
@@ -72,10 +72,10 @@ void receive_vec_512(__m512 x, __m512 y, __m512 z, __m512 w, __m512 q) {
 void receive_vec_1024(__m1024 x, __m1024 y, __m1024 z, __m1024 w, __m1024 q) {
   gv1024 = x + y + z + w + q;
 }
-// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg noundef %x, <8 x float> inreg noundef %y, <8 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg noundef %x, <16 x float> inreg noundef %y, <16 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_1024(ptr noundef %0, ptr noundef %1, ptr noundef %2, ptr noundef %3, ptr noundef %4)
+// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg noundef %x, <8 x float> inreg noundef %y, <8 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg noundef %x, <16 x float> inreg noundef %y, <16 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_1024(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1, ptr dead_on_return noundef %2, ptr dead_on_return noundef %3, ptr dead_on_return noundef %4)
 
 void pass_vec_128(void) {
   __m128 z = {0};
@@ -83,13 +83,13 @@ void pass_vec_128(void) {
 }
 
 // CHECK-LABEL: define dso_local void @pass_vec_128()
-// CHECK: call void @receive_vec_128(<4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, ptr noundef %{{[^,)]*}}, ptr noundef %{{[^,)]*}})
+// CHECK: call void @receive_vec_128(<4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, ptr dead_on_return noundef %{{[^,)]*}}, ptr dead_on_return noundef %{{[^,)]*}})
 
 
 void __fastcall fastcall_indirect_vec(__m128 x, __m128 y, __m128 z, __m128 w, int edx, __m128 q) {
   gv128 = x + y + z + w + q;
 }
-// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01 at fastcall_indirect_vec@84"(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr inreg noundef %0, i32 inreg noundef %edx, ptr noundef %1)
+// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01 at fastcall_indirect_vec@84"(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr dead_on_return inreg noundef %0, i32 inreg noundef %edx, ptr dead_on_return noundef %1)
 
 struct __declspec(align(1)) Align1 { unsigned long long x; };
 struct __declspec(align(4)) Align4 { unsigned long long x; };
@@ -156,4 +156,4 @@ void pass_fixed_align_variadic() {
 // correctly in Clang than it is to be bug for bug compatible, so we pass such
 // arguments indirectly.
 // CHECK-LABEL: define dso_local void @pass_fixed_align_variadic()
-// CHECK: call void (ptr, ...) @receive_fixed_align_variadic(ptr noundef %{{[^)]*}}, i32 noundef 42)
+// CHECK: call void (ptr, ...) @receive_fixed_align_variadic(ptr dead_on_return noundef %{{[^)]*}}, i32 noundef 42)
diff --git a/clang/test/CodeGen/X86/x86_64-arguments-win32.c b/clang/test/CodeGen/X86/x86_64-arguments-win32.c
index 8768e73a854aa..6010e531acb00 100644
--- a/clang/test/CodeGen/X86/x86_64-arguments-win32.c
+++ b/clang/test/CodeGen/X86/x86_64-arguments-win32.c
@@ -21,7 +21,7 @@ void f4(unsigned short a) {}
 // CHECK-LABEL: define dso_local void @f5(i64 noundef %a.coerce)
 void f5(_Complex float a) {}
 
-// CHECK-LABEL: define dso_local void @f6(ptr noundef %a)
+// CHECK-LABEL: define dso_local void @f6(ptr dead_on_return noundef %a)
 void f6(_Complex double a) {}
 
 // CHECK-LABEL: define dso_local i64 @f7()
diff --git a/clang/test/CodeGen/aapcs64-align.cpp b/clang/test/CodeGen/aapcs64-align.cpp
index 1c26d68e434f4..53fc53c2f7296 100644
--- a/clang/test/CodeGen/aapcs64-align.cpp
+++ b/clang/test/CodeGen/aapcs64-align.cpp
@@ -122,8 +122,8 @@ unsigned sizeof_RidiculouslyOverSizedBitfield = sizeof(RidiculouslyOverSizedBitf
 unsigned alignof_RidiculouslyOverSizedBitfield = alignof(RidiculouslyOverSizedBitfield);
 
 // CHECK: define{{.*}} void @g9
-// CHECK: call void @f9(i32 noundef 1, ptr noundef nonnull %agg.tmp)
-// CHECK: declare void @f9(i32 noundef, ptr noundef)
+// CHECK: call void @f9(i32 noundef 1, ptr dead_on_return noundef nonnull %agg.tmp)
+// CHECK: declare void @f9(i32 noundef, ptr dead_on_return noundef)
 void f9(int a, RidiculouslyOverSizedBitfield b);
 void g9() {
   RidiculouslyOverSizedBitfield s = {42};
diff --git a/clang/test/CodeGen/arm-aapcs-vfp.c b/clang/test/CodeGen/arm-aapcs-vfp.c
index 6581929f99f14..e60ed1e52c33a 100644
--- a/clang/test/CodeGen/arm-aapcs-vfp.c
+++ b/clang/test/CodeGen/arm-aapcs-vfp.c
@@ -65,7 +65,7 @@ struct big_struct {
   float f4;
 };
 // CHECK: define{{.*}} arm_aapcs_vfpcc void @test_big([5 x i32] %{{.*}})
-// CHECK64: define{{.*}} void @test_big(ptr noundef %{{.*}})
+// CHECK64: define{{.*}} void @test_big(ptr dead_on_return noundef %{{.*}})
 // CHECK64: call void @llvm.memcpy
 // CHECK64: call void @big_callee(ptr
 extern void big_callee(struct big_struct);
diff --git a/clang/test/CodeGen/arm-abi-vector.c b/clang/test/CodeGen/arm-abi-vector.c
index c2a8902007980..93b770878c3fa 100644
--- a/clang/test/CodeGen/arm-abi-vector.c
+++ b/clang/test/CodeGen/arm-abi-vector.c
@@ -177,11 +177,11 @@ double varargs_vec_19c(int fixed, ...) {
 
 double test_19c(__char19 *in) {
 // CHECK: test_19c
-// CHECK: call arm_aapcscc double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call arm_aapcscc double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
 // APCS-GNU: test_19c
-// APCS-GNU: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// APCS-GNU: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
 // ANDROID: test_19c
-// ANDROID: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// ANDROID: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_19c(19, *in);
 }
 
diff --git a/clang/test/CodeGen/arm-swiftcall.c b/clang/test/CodeGen/arm-swiftcall.c
index 677b878c6765d..55c22a45fd8de 100644
--- a/clang/test/CodeGen/arm-swiftcall.c
+++ b/clang/test/CodeGen/arm-swiftcall.c
@@ -261,7 +261,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define{{.*}} void @return_struct_big_1({{.*}} dead_on_unwind noalias writable sret({{.*}})
 
 // Should not be byval.
-// CHECK-LABEL: define{{.*}} void @take_struct_big_1(ptr{{( %.*)?}})
+// CHECK-LABEL: define{{.*}} void @take_struct_big_1(ptr dead_on_return{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGen/arm64-abi-vector.c b/clang/test/CodeGen/arm64-abi-vector.c
index 81e42315c883b..cf50cdd2fe86e 100644
--- a/clang/test/CodeGen/arm64-abi-vector.c
+++ b/clang/test/CodeGen/arm64-abi-vector.c
@@ -128,7 +128,7 @@ double varargs_vec_19c(int fixed, ...) {
 
 double test_19c(__char19 *in) {
 // CHECK: test_19c
-// CHECK: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_19c(19, *in);
 }
 
@@ -211,7 +211,7 @@ double varargs_vec_5i(int fixed, ...) {
 
 double test_5i(__int5 *in) {
 // CHECK: test_5i
-// CHECK: call double (i32, ...) @varargs_vec_5i(i32 noundef 5, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_5i(i32 noundef 5, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_5i(5, *in);
 }
 
@@ -231,7 +231,7 @@ double varargs_vec_3d(int fixed, ...) {
 
 double test_3d(__double3 *in) {
 // CHECK: test_3d
-// CHECK: call double (i32, ...) @varargs_vec_3d(i32 noundef 3, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_3d(i32 noundef 3, ptr dead_on_return noundef {{%.*}})
   return varargs_vec_3d(3, *in);
 }
 
@@ -291,7 +291,7 @@ double test(__char3 *c3, __char5 *c5, __char9 *c9, __char19 *c19,
             __short3 *s3, __short5 *s5, __int3 *i3, __int5 *i5,
             __double3 *d3) {
   double ret = varargs_vec(3, *c3, *c5, *c9, *c19, *s3, *s5, *i3, *i5, *d3);
-// CHECK: call double (i32, ...) @varargs_vec(i32 noundef 3, i32 {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr noundef {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr noundef {{%.*}}, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec(i32 noundef 3, i32 {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr dead_on_return noundef {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr dead_on_return noundef {{%.*}}, ptr dead_on_return noundef {{%.*}})
   return ret;
 }
 
@@ -350,7 +350,7 @@ __attribute__((noinline)) double args_vec_19c(int fixed, __char19 c19) {
 
 double fixed_19c(__char19 *in) {
 // CHECK: fixed_19c
-// CHECK: call double @args_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
   return args_vec_19c(19, *in);
 }
 
@@ -409,7 +409,7 @@ __attribute__((noinline)) double args_vec_5i(int fixed, __int5 c5) {
 
 double fixed_5i(__int5 *in) {
 // CHECK: fixed_5i
-// CHECK: call double @args_vec_5i(i32 noundef 5, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_5i(i32 noundef 5, ptr dead_on_return noundef {{%.*}})
   return args_vec_5i(5, *in);
 }
 
@@ -424,6 +424,6 @@ __attribute__((noinline)) double args_vec_3d(int fixed, __double3 c3) {
 
 double fixed_3d(__double3 *in) {
 // CHECK: fixed_3d
-// CHECK: call double @args_vec_3d(i32 noundef 3, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_3d(i32 noundef 3, ptr dead_on_return noundef {{%.*}})
   return args_vec_3d(3, *in);
 }
diff --git a/clang/test/CodeGen/arm64-arguments.c b/clang/test/CodeGen/arm64-arguments.c
index 4c4f85d923e78..2e3ab388432f6 100644
--- a/clang/test/CodeGen/arm64-arguments.c
+++ b/clang/test/CodeGen/arm64-arguments.c
@@ -163,7 +163,7 @@ void f32(struct s32 s) { }
 // A composite type larger than 16 bytes should be passed indirectly.
 struct s33 { char buf[32*32]; };
 void f33(struct s33 s) { }
-// CHECK: define{{.*}} void @f33(ptr noundef %s)
+// CHECK: define{{.*}} void @f33(ptr dead_on_return noundef %s)
 
 struct s34 { char c; };
 void f34(struct s34 s);
@@ -226,9 +226,9 @@ T_float32x2 f1_0(T_float32x2 a0) { return a0; }
 // CHECK: define{{.*}} <4 x float> @f1_1(<4 x float> noundef %{{.*}})
 T_float32x4 f1_1(T_float32x4 a0) { return a0; }
 // Vector with length bigger than 16-byte is illegal and is passed indirectly.
-// CHECK: define{{.*}} void @f1_2(ptr dead_on_unwind noalias writable sret(<8 x float>) align 16 %{{.*}}, ptr noundef %0)
+// CHECK: define{{.*}} void @f1_2(ptr dead_on_unwind noalias writable sret(<8 x float>) align 16 %{{.*}}, ptr dead_on_return noundef %0)
 T_float32x8 f1_2(T_float32x8 a0) { return a0; }
-// CHECK: define{{.*}} void @f1_3(ptr dead_on_unwind noalias writable sret(<16 x float>) align 16 %{{.*}}, ptr noundef %0)
+// CHECK: define{{.*}} void @f1_3(ptr dead_on_unwind noalias writable sret(<16 x float>) align 16 %{{.*}}, ptr dead_on_return noundef %0)
 T_float32x16 f1_3(T_float32x16 a0) { return a0; }
 
 // Testing alignment with aggregates: HFA, aggregates with size <= 16 bytes and
@@ -278,7 +278,7 @@ struct s37
 typedef struct s37 s37_with_align;
 
 int32x4_t f37(int i, s37_with_align s1, s37_with_align s2) {
-// CHECK: define{{.*}} <4 x i32> @f37(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} <4 x i32> @f37(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: load <4 x i32>, ptr %s1, align 16
 // CHECK: load <4 x i32>, ptr %s2, align 16
   int32x4_t v = vaddq_s32(*(int32x4_t *)&s1,
@@ -292,7 +292,7 @@ int32x4_t caller37() {
 // CHECK: %[[b:.*]] = alloca %struct.s37, align 16
 // CHECK: call void @llvm.memcpy
 // CHECK: call void @llvm.memcpy
-// CHECK: call <4 x i32> @f37(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call <4 x i32> @f37(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f37(3, g37, g37);
 }
 
@@ -530,7 +530,7 @@ typedef struct s42 s42_no_align;
 // passing structs in registers
 __attribute__ ((noinline))
 int f42(int i, s42_no_align s1, s42_no_align s2) {
-// CHECK: define{{.*}} i32 @f42(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f42(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 1
@@ -545,14 +545,14 @@ int caller42() {
 // CHECK: %[[b:.*]] = alloca %struct.s42, align 4
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f42(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f42(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f42(3, g42, g42_2);
 }
 // passing structs on stack
 __attribute__ ((noinline))
 int f42_stack(int i, int i2, int i3, int i4, int i5, int i6, int i7, int i8,
               int i9, s42_no_align s1, s42_no_align s2) {
-// CHECK: define{{.*}} i32 @f42_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f42_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 1
@@ -565,7 +565,7 @@ int caller42_stack() {
 // CHECK: %[[b:.*]] = alloca %struct.s42, align 4
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f42_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f42_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f42_stack(1, 2, 3, 4, 5, 6, 7, 8, 9, g42, g42_2);
 }
 
@@ -583,7 +583,7 @@ typedef struct s43 s43_with_align;
 // passing aligned structs in registers
 __attribute__ ((noinline))
 int f43(int i, s43_with_align s1, s43_with_align s2) {
-// CHECK: define{{.*}} i32 @f43(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f43(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 1
@@ -598,14 +598,14 @@ int caller43() {
 // CHECK: %[[b:.*]] = alloca %struct.s43, align 16
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f43(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f43(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f43(3, g43, g43_2);
 }
 // passing aligned structs on stack
 __attribute__ ((noinline))
 int f43_stack(int i, int i2, int i3, int i4, int i5, int i6, int i7, int i8,
               int i9, s43_with_align s1, s43_with_align s2) {
-// CHECK: define{{.*}} i32 @f43_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f43_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s2, i32 0, i32 0
 // CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 1
@@ -618,7 +618,7 @@ int caller43_stack() {
 // CHECK: %[[b:.*]] = alloca %struct.s43, align 16
 // CHECK: call void @llvm.memcpy.p0.p0.i64
 // CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f43_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f43_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
   return f43_stack(1, 2, 3, 4, 5, 6, 7, 8, 9, g43, g43_2);
 }
 
diff --git a/clang/test/CodeGen/arm64-microsoft-arguments.cpp b/clang/test/CodeGen/arm64-microsoft-arguments.cpp
index 85472645acb3b..a0a81be54325f 100644
--- a/clang/test/CodeGen/arm64-microsoft-arguments.cpp
+++ b/clang/test/CodeGen/arm64-microsoft-arguments.cpp
@@ -29,7 +29,7 @@ S2 f2() {
 
 // Pass and return for type size > 16 bytes.
 // CHECK: define {{.*}} void @{{.*}}f3{{.*}}(ptr dead_on_unwind noalias writable sret(%struct.S3) align 4 %agg.result)
-// CHECK: call void {{.*}}func3{{.*}}(ptr dead_on_unwind writable sret(%struct.S3) align 4 %agg.result, ptr noundef %agg.tmp)
+// CHECK: call void {{.*}}func3{{.*}}(ptr dead_on_unwind writable sret(%struct.S3) align 4 %agg.result, ptr dead_on_return noundef %agg.tmp)
 struct S3 {
   int a[5];
 };
diff --git a/clang/test/CodeGen/armv7k-abi.c b/clang/test/CodeGen/armv7k-abi.c
index 872e6423a4a99..6a781bc04d042 100644
--- a/clang/test/CodeGen/armv7k-abi.c
+++ b/clang/test/CodeGen/armv7k-abi.c
@@ -39,7 +39,7 @@ typedef struct {
   double z;
 } BigStruct;
 
-// CHECK: define{{.*}} void @big_struct_indirect(ptr noundef %b)
+// CHECK: define{{.*}} void @big_struct_indirect(ptr dead_on_return noundef %b)
 void big_struct_indirect(BigStruct b) {}
 
 // CHECK: define{{.*}} void @return_big_struct_indirect(ptr dead_on_unwind noalias writable sret
diff --git a/clang/test/CodeGen/atomic-arm64.c b/clang/test/CodeGen/atomic-arm64.c
index d2a30a3b6e66f..d539cad0c6a7d 100644
--- a/clang/test/CodeGen/atomic-arm64.c
+++ b/clang/test/CodeGen/atomic-arm64.c
@@ -57,7 +57,7 @@ void test3(pointer_pair_t pair) {
 }
 
 // CHECK-LABEL:define{{.*}} void @test4(
-// CHECK-SAME: ptr noundef [[QUAD:%.*]])
+// CHECK-SAME: ptr dead_on_return noundef [[QUAD:%.*]])
 // CHECK:      [[QUAD_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT: [[TEMP:%.*]] = alloca [[QUAD_T:%.*]], align 8
 // CHECK-NEXT: store ptr [[QUAD]], ptr [[QUAD_INDIRECT_ADDR]]
diff --git a/clang/test/CodeGen/attr-noundef.cpp b/clang/test/CodeGen/attr-noundef.cpp
index abdf9496bd396..619dbec7678db 100644
--- a/clang/test/CodeGen/attr-noundef.cpp
+++ b/clang/test/CodeGen/attr-noundef.cpp
@@ -27,7 +27,7 @@ struct NoCopy {
 NoCopy ret_nocopy() { return {}; }
 void pass_nocopy(NoCopy e) {}
 // CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr noundef %
+// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr dead_on_return noundef %
 
 struct Huge {
   int a[1024];
@@ -35,7 +35,8 @@ struct Huge {
 Huge ret_huge() { return {}; }
 void pass_huge(Huge h) {}
 // CHECK: [[DEF]] void @{{.*}}ret_huge{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr noundef
+// CHECK-INTEL: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr noundef
+// CHECK-AARCH: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr dead_on_return noundef
 } // namespace check_structs
 
 //************ Passing unions by value
@@ -59,7 +60,7 @@ union NoCopy {
 NoCopy ret_nocopy() { return {}; }
 void pass_nocopy(NoCopy e) {}
 // CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr noundef %
+// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr dead_on_return noundef %
 } // namespace check_unions
 
 //************ Passing `this` pointers
diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c
index cf74ab2dcca3f..fba692cac4492 100644
--- a/clang/test/CodeGen/cx-complex-range.c
+++ b/clang/test/CodeGen/cx-complex-range.c
@@ -1520,7 +1520,7 @@ void mulassignf(_Complex float *a, _Complex float b) {
 // PRMTD-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @divd(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -1744,7 +1744,7 @@ void mulassignf(_Complex float *a, _Complex float b) {
 // PRMTD_FAST-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divd(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -1938,7 +1938,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @divassignd(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2180,7 +2180,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divassignd(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2325,7 +2325,7 @@ void divassignd(_Complex double *a, _Complex double b) {
 // PRMTD-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @muld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -2457,7 +2457,7 @@ void divassignd(_Complex double *a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @muld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -2594,7 +2594,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulassignd(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2744,7 +2744,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulassignd(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2922,7 +2922,7 @@ void mulassignd(_Complex double *a, _Complex double b) {
 // PRMTD-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @divld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -3190,7 +3190,7 @@ void mulassignd(_Complex double *a, _Complex double b) {
 // PRMTD_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -3432,7 +3432,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @divassignld(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -3702,7 +3702,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @divassignld(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -3895,7 +3895,7 @@ void divassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -4059,7 +4059,7 @@ void divassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -4220,7 +4220,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD-NEXT:    ret void
 //
 // X86WINPRMTD-LABEL: define dso_local void @mulassignld(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -4386,7 +4386,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret void
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local void @mulassignld(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // X86WINPRMTD_STRICT-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -4644,7 +4644,7 @@ void mulassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD-NEXT:    ret <2 x float> [[RETVAL_SROA_0_4_VEC_INSERT]]
 //
 // X86WINPRMTD-LABEL: define dso_local i64 @f1(
-// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD-NEXT:  entry:
 // X86WINPRMTD-NEXT:    [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE]] to i32
 // X86WINPRMTD-NEXT:    [[TMP0:%.*]] = bitcast i32 [[A_SROA_0_0_EXTRACT_TRUNC]] to float
@@ -5052,7 +5052,7 @@ void mulassignld(_Complex long double *a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    ret <2 x float> [[RETVAL_SROA_0_4_VEC_INSERT]]
 //
 // X86WINPRMTD_STRICT-LABEL: define dso_local i64 @f1(
-// X86WINPRMTD_STRICT-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: i64 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
 // X86WINPRMTD_STRICT-NEXT:  entry:
 // X86WINPRMTD_STRICT-NEXT:    [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE]] to i32
 // X86WINPRMTD_STRICT-NEXT:    [[TMP0:%.*]] = bitcast i32 [[A_SROA_0_0_EXTRACT_TRUNC]] to float
diff --git a/clang/test/CodeGen/ext-int-cc.c b/clang/test/CodeGen/ext-int-cc.c
index fdca4012ee4a4..7cfd992fd48b4 100644
--- a/clang/test/CodeGen/ext-int-cc.c
+++ b/clang/test/CodeGen/ext-int-cc.c
@@ -33,9 +33,9 @@
 // Make sure 128 and 64 bit versions are passed like integers.
 void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
 // LIN64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// WIN64: define dso_local void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
-// WIN32: define dso_local void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// WIN64: define dso_local void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
+// WIN32: define dso_local void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // NVPTX64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // NVPTX: define{{.*}} void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // SPARCV9: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
@@ -50,22 +50,22 @@ void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
 // ARC: define{{.*}} void @ParamPassing(ptr byval(i128) align 4 %{{.+}}, i64 inreg %{{.+}})
 // XCORE: define{{.*}} void @ParamPassing(ptr byval(i128) align 4 %{{.+}}, i64 %{{.+}})
 // RISCV64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// RISCV32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// RISCV32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // WASM: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// SYSTEMZ: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// SYSTEMZ: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 // PPC64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // PPC32: define{{.*}} void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // AARCH64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // AARCH64DARWIN: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
 // ARM: define{{.*}} arm_aapcscc void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // LA64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// LA32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// LA32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
 
 void ParamPassing2(_BitInt(127) b, _BitInt(63) c) {}
 // LIN64: define{{.*}} void @ParamPassing2(i64 %{{.+}}, i64 %{{.+}}, i64 %{{.+}})
-// WIN64: define dso_local void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
-// WIN32: define dso_local void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// WIN64: define dso_local void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
+// WIN32: define dso_local void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 // NVPTX64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // NVPTX: define{{.*}} void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // SPARCV9: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
@@ -80,16 +80,16 @@ void ParamPassing2(_BitInt(127) b, _BitInt(63) c) {}
 // ARC: define{{.*}} void @ParamPassing2(ptr byval(i128) align 4 %{{.+}}, i63 inreg %{{.+}})
 // XCORE: define{{.*}} void @ParamPassing2(ptr byval(i128) align 4 %{{.+}}, i63 %{{.+}})
 // RISCV64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
-// RISCV32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// RISCV32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 // WASM: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
-// SYSTEMZ: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 signext %{{.+}})
+// SYSTEMZ: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 signext %{{.+}})
 // PPC64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
 // PPC32: define{{.*}} void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // AARCH64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // AARCH64DARWIN: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
 // ARM: define{{.*}} arm_aapcscc void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
 // LA64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
-// LA32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// LA32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
 
 // Make sure we follow the signext rules for promotable integer types.
 void ParamPassing3(_BitInt(15) a, _BitInt(31) b) {}
@@ -129,10 +129,10 @@ void ParamPassing3(_BitInt(15) a, _BitInt(31) b) {}
 // _BitInt widths to alert us to enable the test.
 void ParamPassing4(_BitInt(129) a) {}
 // LIN64: define{{.*}} void @ParamPassing4(ptr byval([24 x i8]) align 8 %{{.+}})
-// WIN64: define dso_local void @ParamPassing4(ptr %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// WIN32: define dso_local void @ParamPassing4(ptr %{{.+}})
-// AARCH64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// WIN64: define dso_local void @ParamPassing4(ptr dead_on_return %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
+// WIN32: define dso_local void @ParamPassing4(ptr dead_on_return %{{.+}})
+// AARCH64: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
 // NVPTX64-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // NVPTX-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // SPARCV9-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
@@ -154,8 +154,8 @@ void ParamPassing4(_BitInt(129) a) {}
 // PPC32-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // AARCH64DARWIN-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // ARM-NOT: define{{.*}} arm_aapcscc void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
-// LA64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// LA32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// LA64: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
+// LA32: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
 #endif
 
 _BitInt(63) ReturnPassing(void) { return 0; }
diff --git a/clang/test/CodeGen/isfpclass.c b/clang/test/CodeGen/isfpclass.c
index 26dd846a2bf20..ee3a22b40fefd 100644
--- a/clang/test/CodeGen/isfpclass.c
+++ b/clang/test/CodeGen/isfpclass.c
@@ -160,7 +160,7 @@ int4 check_isfpclass_nan_strict_v4f32(float4 x) {
 }
 
 // CHECK-LABEL: define dso_local void @check_isfpclass_nan_v4f64
-// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X:%.*]] = load <4 x double>, ptr [[TMP0]], align 16, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <4 x double> [[X]], zeroinitializer
diff --git a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
index eb706154300a2..0124cc5c06d43 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
+++ b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
@@ -30,37 +30,37 @@ long double powl(long double a, long double b);
 //
 // CHECK-PPC-LABEL: define dso_local ppc_fp128 @test_powl(
 // CHECK-PPC-SAME: ppc_fp128 noundef [[A:%.*]], ppc_fp128 noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @powl(ppc_fp128 noundef [[A]], ppc_fp128 noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @powl(ppc_fp128 noundef [[A]], ppc_fp128 noundef [[B]]) #[[ATTR4:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-ARM-LABEL: define dso_local double @test_powl(
 // CHECK-ARM-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-ARM:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-ARM-HF-LABEL: define dso_local double @test_powl(
 // CHECK-ARM-HF-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-THUMB-LABEL: define double @test_powl(
 // CHECK-THUMB-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-THUMB:    [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
 //
 // CHECK-AARCH-LABEL: define dso_local fp128 @test_powl(
 // CHECK-AARCH-SAME: fp128 noundef [[A:%.*]], fp128 noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @powl(fp128 noundef [[A]], fp128 noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @powl(fp128 noundef [[A]], fp128 noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func double @test_powl(
 // CHECK-SPIR-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR4:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
 //
 // CHECK-MINGW32-LABEL: define dso_local void @test_powl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]], ptr noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3:![0-9]+]]
-// CHECK-MINGW32:    [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[B]], ptr [[BYVAL_TEMP1:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    call void @powl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr noundef nonnull [[BYVAL_TEMP]], ptr noundef nonnull [[BYVAL_TEMP1]]) #[[ATTR3:[0-9]+]]
-// CHECK-MINGW32:    [[TMP2:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[TMP2]], ptr [[AGG_RESULT]], align 16, !tbaa [[TBAA3]]
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA6:![0-9]+]]
+// CHECK-MINGW32:    [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[B]], ptr [[BYVAL_TEMP1:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    call void @powl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP1]]) #[[ATTR3:[0-9]+]]
+// CHECK-MINGW32:    [[TMP2:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[TMP2]], ptr [[AGG_RESULT]], align 16, !tbaa [[TBAA6]]
 //
 long double test_powl(long double a, long double b) {
    return powl(a, b);
@@ -93,51 +93,51 @@ long double test_powl(long double a, long double b) {
 // CHECK-I686:    store x86_fp80 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 4
 //
 // CHECK-PPC-LABEL: define dso_local void @test_cargl(
-// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-PPC:    [[CLD_REAL:%.*]] = load ppc_fp128, ptr [[CLD]], align 16
 // CHECK-PPC:    [[CLD_IMAG:%.*]] = load ppc_fp128, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-PPC:    store ppc_fp128 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
 // CHECK-PPC:    store ppc_fp128 [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 16
-// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @cargl(ptr noundef nonnull byval({ ppc_fp128, ppc_fp128 }) align 16 [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call ppc_fp128 @cargl(ptr noundef nonnull byval({ ppc_fp128, ppc_fp128 }) align 16 [[BYVAL_TEMP]]) #[[ATTR4]]
 // CHECK-PPC:    store ppc_fp128 [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 16
 // CHECK-PPC:    store ppc_fp128 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 16
 //
 // CHECK-ARM-LABEL: define dso_local void @test_cargl(
-// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-ARM:    [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 // CHECK-ARM:    store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
 // CHECK-ARM:    store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
 //
 // CHECK-ARM-HF-LABEL: define dso_local { double, double } @test_cargl(
-// CHECK-ARM-HF-SAME: { double, double } noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @cargl({ double, double } noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-HF-SAME: { double, double } noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call double @cargl({ double, double } noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-THUMB-LABEL: define { double, double } @test_cargl(
-// CHECK-THUMB-SAME: [2 x double] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call double @cargl([2 x double] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-THUMB-SAME: [2 x double] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-THUMB:    [[CALL:%.*]] = tail call double @cargl([2 x double] noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-AARCH-LABEL: define dso_local { fp128, fp128 } @test_cargl(
-// CHECK-AARCH-SAME: [2 x fp128] noundef alignstack(16) [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA2]]
+// CHECK-AARCH-SAME: [2 x fp128] noundef alignstack(16) [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-AARCH:    [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA2]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func void @test_cargl(
-// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // CHECK-SPIR:    [[CLD_REAL:%.*]] = load double, ptr [[CLD]], align 8
 // CHECK-SPIR:    [[CLD_IMAG:%.*]] = load double, ptr [[CLD_IMAGP:%.*]], align 8
 // CHECK-SPIR:    store double [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 8
 // CHECK-SPIR:    store double [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 8
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @cargl(ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func double @cargl(ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]]) #[[ATTR4]]
 // CHECK-SPIR:    store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
 // CHECK-SPIR:    store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
 //
 // CHECK-MINGW32-LABEL: define dso_local void @test_cargl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-MINGW32:    [[CLD_REAL:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK-MINGW32:    [[CLD_IMAG:%.*]] = load x86_fp80, ptr [[CLD_IMAGP:%.*]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 16
-// CHECK-MINGW32:    call void @cargl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
-// CHECK-MINGW32:    [[TMP0:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA3]]
+// CHECK-MINGW32:    call void @cargl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-MINGW32:    [[TMP0:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA6]]
 // CHECK-MINGW32:    [[CLD_REAL3:%.*]] = load x86_fp80, ptr [[CLD]], align 16
 // CHECK-MINGW32:    [[CLD_IMAG5:%.*]] = load x86_fp80, ptr [[CLD_IMAGP]], align 16
 // CHECK-MINGW32:    store x86_fp80 [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 16
@@ -166,33 +166,33 @@ int ilogbl(long double a);
 //
 // CHECK-PPC-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-PPC-SAME: ppc_fp128 noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-PPC:    [[CALL:%.*]] = tail call i32 @ilogbl(ppc_fp128 noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
+// CHECK-PPC:    [[CALL:%.*]] = tail call i32 @ilogbl(ppc_fp128 noundef [[A]]) #[[ATTR4]], !tbaa [[TBAA2]]
 //
 // CHECK-ARM-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-ARM-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-ARM:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-ARM-HF-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-ARM-HF-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-ARM-HF:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-HF:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-THUMB-LABEL: define i32 @test_ilogb(
 // CHECK-THUMB-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-THUMB:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-THUMB:    [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
 //
 // CHECK-AARCH-LABEL: define dso_local i32 @test_ilogb(
 // CHECK-AARCH-SAME: fp128 noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-AARCH:    [[CALL:%.*]] = tail call i32 @ilogbl(fp128 noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA2]]
+// CHECK-AARCH:    [[CALL:%.*]] = tail call i32 @ilogbl(fp128 noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
 //
 // CHECK-SPIR-LABEL: define dso_local spir_func i32 @test_ilogb(
 // CHECK-SPIR-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
+// CHECK-SPIR:    [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR4]], !tbaa [[TBAA2]]
 //
 // CHECK-MINGW32-LABEL: define dso_local i32 @test_ilogb(
-// CHECK-MINGW32-SAME: ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32:    [[CALL:%.*]] = call i32 @ilogbl(ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-MINGW32-SAME: ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32:    [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32:    [[CALL:%.*]] = call i32 @ilogbl(ptr dead_on_return noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
 //
 int test_ilogb(long double a) {
    return ilogbl(a);
@@ -243,8 +243,8 @@ int test_ilogb(long double a) {
 // CHECK-SPIR: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
 // CHECK-SPIR: [[META5]] = !{!"Simple C/C++ TBAA"}
 //.
-// CHECK-MINGW32: [[TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
-// CHECK-MINGW32: [[META4]] = !{!"long double", [[META5:![0-9]+]], i64 0}
-// CHECK-MINGW32: [[META5]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
-// CHECK-MINGW32: [[META6]] = !{!"Simple C/C++ TBAA"}
+// CHECK-MINGW32: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+// CHECK-MINGW32: [[META7]] = !{!"long double", [[META8:![0-9]+]], i64 0}
+// CHECK-MINGW32: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+// CHECK-MINGW32: [[META9]] = !{!"Simple C/C++ TBAA"}
 //.
diff --git a/clang/test/CodeGen/mingw-long-double.c b/clang/test/CodeGen/mingw-long-double.c
index 0fc8f01509682..b98929701bc62 100644
--- a/clang/test/CodeGen/mingw-long-double.c
+++ b/clang/test/CodeGen/mingw-long-double.c
@@ -29,15 +29,15 @@ long double TestLD(long double x) {
   return x * x;
 }
 // GNU32: define dso_local x86_fp80 @TestLD(x86_fp80 noundef %x)
-// GNU64: define dso_local void @TestLD(ptr dead_on_unwind noalias writable sret(x86_fp80) align 16 %agg.result, ptr noundef %0)
+// GNU64: define dso_local void @TestLD(ptr dead_on_unwind noalias writable sret(x86_fp80) align 16 %agg.result, ptr dead_on_return noundef %0)
 // MSC64: define dso_local double @TestLD(double noundef %x)
 
 long double _Complex TestLDC(long double _Complex x) {
   return x * x;
 }
 // GNU32: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 4 %agg.result, ptr noundef byval({ x86_fp80, x86_fp80 }) align 4 %x)
-// GNU64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 16 %agg.result, ptr noundef %x)
-// MSC64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %agg.result, ptr noundef %x)
+// GNU64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 16 %agg.result, ptr dead_on_return noundef %x)
+// MSC64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %agg.result, ptr dead_on_return noundef %x)
 
 // GNU32: declare dso_local void @__mulxc3
 // GNU64: declare dso_local void @__mulxc3
diff --git a/clang/test/CodeGen/ms_abi.c b/clang/test/CodeGen/ms_abi.c
index 528e546f315d5..5d58c9816da78 100644
--- a/clang/test/CodeGen/ms_abi.c
+++ b/clang/test/CodeGen/ms_abi.c
@@ -142,7 +142,7 @@ struct i128 {
 };
 
 __attribute__((ms_abi)) struct i128 f7(struct i128 a) {
-  // WIN64: define dso_local void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr noundef %a)
-  // FREEBSD: define{{.*}} win64cc void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr noundef %a)
+  // WIN64: define dso_local void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr dead_on_return noundef %a)
+  // FREEBSD: define{{.*}} win64cc void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr dead_on_return noundef %a)
   return a;
 }
diff --git a/clang/test/CodeGen/pass-by-value-noalias.c b/clang/test/CodeGen/pass-by-value-noalias.c
index bc35d13c4df6a..e673ceb80bebe 100644
--- a/clang/test/CodeGen/pass-by-value-noalias.c
+++ b/clang/test/CodeGen/pass-by-value-noalias.c
@@ -11,6 +11,6 @@ struct Foo {
   int f;
 };
 
-// WITH_NOALIAS: define{{.*}} void @take(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @take(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
 void take(struct Foo arg) {}
diff --git a/clang/test/CodeGen/ptrauth-in-c-struct.c b/clang/test/CodeGen/ptrauth-in-c-struct.c
index 2aec31ec3baf9..c74be17b4c837 100644
--- a/clang/test/CodeGen/ptrauth-in-c-struct.c
+++ b/clang/test/CodeGen/ptrauth-in-c-struct.c
@@ -115,7 +115,7 @@ void test_move_assignment_SA(SA *p) {
   *p = getSA();
 }
 
-// CHECK: define void @test_parameter_SA(ptr noundef %{{.*}})
+// CHECK: define void @test_parameter_SA(ptr dead_on_return noundef %{{.*}})
 // CHECK-NOT: call
 // CHECK: ret void
 
@@ -128,7 +128,7 @@ void test_parameter_SA(SA a) {
 // CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
 // CHECK: call void @__copy_constructor_8_8_t0w4_pa1_50_8(ptr %[[AGG_TMP]], ptr %[[V0]])
-// CHECK: call void @calleeSA(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @calleeSA(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK-NOT: call
 // CHECK: ret void
 
diff --git a/clang/test/CodeGen/regcall.c b/clang/test/CodeGen/regcall.c
index f10da87353fa1..d4b9f00d54d41 100644
--- a/clang/test/CodeGen/regcall.c
+++ b/clang/test/CodeGen/regcall.c
@@ -28,7 +28,7 @@ struct Large { int a[5]; };
 void __regcall v4(int a, struct Large b, int c) {}
 // Win32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // Lin32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
-// Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 // Lin64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
 
 void __regcall v5(long long a, int b, int c) {}
@@ -47,7 +47,7 @@ void __regcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __regcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_regcallcc void @__regcall3__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall3__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall3__hfa2(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
@@ -61,7 +61,7 @@ void __regcall hfa3(double a, double b, double c, double d, double e, struct HFA
 // handling to ensure alignment.
 void __regcall hfa4(struct HFA5 a) {}
 // X32: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr noundef byval(%struct.HFA5) align 4 %{{.*}})
-// Win64: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr noundef %a)
+// Win64: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr dead_on_return noundef %a)
 // Lin64: define dso_local x86_regcallcc void @__regcall3__hfa4(double %a.coerce0, double %a.coerce1, double %a.coerce2, double %a.coerce3, double %a.coerce4)
 
 // Return HFAs of 4 or fewer elements in registers.
@@ -79,7 +79,7 @@ void __regcall hva1(int a, struct HVA4 b, int c) {}
 // X64: define dso_local x86_regcallcc void @__regcall3__hva1(i32 noundef %a, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 noundef %c)
 
 void __regcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {}
-// X86: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> noundef %c)
 
 void __regcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {}
@@ -95,6 +95,6 @@ void __regcall odd_size_hva(struct OddSizeHVA a) {}
 
 struct HFA6 { __m128 f[4]; };
 struct HFA6 __regcall ret_reg_reused(struct HFA6 a, struct HFA6 b, struct HFA6 c, struct HFA6 d){ struct HFA6 h; return h;}
-// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %c, ptr inreg noundef %d)
+// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %c, ptr dead_on_return inreg noundef %d)
 // Win64: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, <4 x float> %c.0, <4 x float> %c.1, <4 x float> %c.2, <4 x float> %c.3, <4 x float> %d.0, <4 x float> %d.1, <4 x float> %d.2, <4 x float> %d.3)
 // Lin64: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused([4 x <4 x float>] %a.coerce, [4 x <4 x float>] %b.coerce, [4 x <4 x float>] %c.coerce, [4 x <4 x float>] %d.coerce)
diff --git a/clang/test/CodeGen/regcall2.c b/clang/test/CodeGen/regcall2.c
index c88d4e485b104..42619369677b2 100644
--- a/clang/test/CodeGen/regcall2.c
+++ b/clang/test/CodeGen/regcall2.c
@@ -20,7 +20,7 @@ double __regcall bar(__sVector a) {
 
 // FIXME: Do we need to change for Windows?
 // Win: define dso_local x86_regcallcc void @__regcall3__foo(ptr dead_on_unwind noalias writable sret(%struct.__sVector) align 64 %agg.result, i32 noundef %a) #0
-// Win: define dso_local x86_regcallcc double @__regcall3__bar(ptr noundef %a) #0
+// Win: define dso_local x86_regcallcc double @__regcall3__bar(ptr dead_on_return noundef %a) #0
 // Win: attributes #0 = { noinline nounwind optnone "min-legal-vector-width"="0" "no-builtins" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+avx,+avx2,+avx512f,+avx512vl,+crc32,+cx8,+evex512,+f16c,+fma,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" }
 
 // Lin: define dso_local x86_regcallcc %struct.__sVector @__regcall3__foo(i32 noundef %a) #0
diff --git a/clang/test/CodeGen/regcall4.c b/clang/test/CodeGen/regcall4.c
index 5fbe77fbc7d76..d5fe5d88a0e8c 100644
--- a/clang/test/CodeGen/regcall4.c
+++ b/clang/test/CodeGen/regcall4.c
@@ -28,7 +28,7 @@ struct Large { int a[5]; };
 void __regcall v4(int a, struct Large b, int c) {}
 // Win32: define dso_local x86_regcallcc void @__regcall4__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // Lin32: define dso_local x86_regcallcc void @__regcall4__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
-// Win64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// Win64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 // Lin64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
 
 void __regcall v5(long long a, int b, int c) {}
@@ -47,7 +47,7 @@ void __regcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __regcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_regcallcc void @__regcall4__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall4__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall4__hfa2(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
@@ -61,7 +61,7 @@ void __regcall hfa3(double a, double b, double c, double d, double e, struct HFA
 // handling to ensure alignment.
 void __regcall hfa4(struct HFA5 a) {}
 // X32: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr noundef byval(%struct.HFA5) align 4 %{{.*}})
-// Win64: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr noundef %a)
+// Win64: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr dead_on_return noundef %a)
 // Lin64: define dso_local x86_regcallcc void @__regcall4__hfa4(double %a.coerce0, double %a.coerce1, double %a.coerce2, double %a.coerce3, double %a.coerce4)
 
 // Return HFAs of 4 or fewer elements in registers.
@@ -79,7 +79,7 @@ void __regcall hva1(int a, struct HVA4 b, int c) {}
 // X64: define dso_local x86_regcallcc void @__regcall4__hva1(i32 noundef %a, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 noundef %c)
 
 void __regcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {}
-// X86: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %0)
 // X64: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> noundef %c)
 
 void __regcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {}
@@ -95,6 +95,6 @@ void __regcall odd_size_hva(struct OddSizeHVA a) {}
 
 struct HFA6 { __m128 f[4]; };
 struct HFA6 __regcall ret_reg_reused(struct HFA6 a, struct HFA6 b, struct HFA6 c, struct HFA6 d){ struct HFA6 h; return h;}
-// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %c, ptr inreg noundef %d)
+// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %c, ptr dead_on_return inreg noundef %d)
 // Win64: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, <4 x float> %c.0, <4 x float> %c.1, <4 x float> %c.2, <4 x float> %c.3, <4 x float> %d.0, <4 x float> %d.1, <4 x float> %d.2, <4 x float> %d.3)
 // Lin64: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused([4 x <4 x float>] %a.coerce, [4 x <4 x float>] %b.coerce, [4 x <4 x float>] %c.coerce, [4 x <4 x float>] %d.coerce)
diff --git a/clang/test/CodeGen/sparcv9-abi.c b/clang/test/CodeGen/sparcv9-abi.c
index 616e24e7c519d..5a3d64fd37889 100644
--- a/clang/test/CodeGen/sparcv9-abi.c
+++ b/clang/test/CodeGen/sparcv9-abi.c
@@ -80,7 +80,7 @@ struct medium {
   int *c, *d;
 };
 
-// CHECK-LABEL: define{{.*}} %struct.medium @f_medium(ptr noundef %x)
+// CHECK-LABEL: define{{.*}} %struct.medium @f_medium(ptr dead_on_return noundef %x)
 struct medium f_medium(struct medium x) {
   x.a += *x.b;
   x.b = 0;
@@ -94,7 +94,7 @@ struct large {
   int x;
 };
 
-// CHECK-LABEL: define{{.*}} void @f_large(ptr dead_on_unwind noalias writable sret(%struct.large) align 8 %agg.result, ptr noundef %x)
+// CHECK-LABEL: define{{.*}} void @f_large(ptr dead_on_unwind noalias writable sret(%struct.large) align 8 %agg.result, ptr dead_on_return noundef %x)
 struct large f_large(struct large x) {
   x.a += *x.b;
   x.b = 0;
diff --git a/clang/test/CodeGen/vectorcall.c b/clang/test/CodeGen/vectorcall.c
index cab7fc0972d7b..09b3310c7c4c8 100644
--- a/clang/test/CodeGen/vectorcall.c
+++ b/clang/test/CodeGen/vectorcall.c
@@ -17,7 +17,7 @@ void __vectorcall v3(int a, struct Small b, int c) {}
 struct Large { int a[5]; };
 void __vectorcall v4(int a, struct Large b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01v4@@28"(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
 
 void __vectorcall v5(long long a, int b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01v5@@16"(i64 noundef %a, i32 inreg noundef %b, i32 inreg noundef %c)
@@ -35,21 +35,21 @@ void __vectorcall hfa1(int a, struct HFA4 b, int c) {}
 // indirectly. Additional vector arguments can consume the rest of the SSE
 // registers.
 void __vectorcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr inreg noundef %b, double inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr noundef %b, double noundef %c)
+// X86: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, double inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr dead_on_return noundef %b, double noundef %c)
 
 // Ensure that we pass builtin types directly while counting them against the
 // SSE register usage.
 void __vectorcall hfa3(double a, double b, double c, double d, double e, struct HFA2 f) {}
-// X86: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double inreg noundef %a, double inreg noundef %b, double inreg noundef %c, double inreg noundef %d, double inreg noundef %e, ptr inreg noundef %f)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double noundef %a, double noundef %b, double noundef %c, double noundef %d, double noundef %e, ptr noundef %f)
+// X86: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double inreg noundef %a, double inreg noundef %b, double inreg noundef %c, double inreg noundef %d, double inreg noundef %e, ptr dead_on_return inreg noundef %f)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double noundef %a, double noundef %b, double noundef %c, double noundef %d, double noundef %e, ptr dead_on_return noundef %f)
 
 // Aggregates with more than four elements are not HFAs and are passed byval.
 // Because they are not classified as homogeneous, they don't get special
 // handling to ensure alignment.
 void __vectorcall hfa4(struct HFA5 a) {}
 // X86: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr noundef byval(%struct.HFA5) align 4 %0)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr noundef %a)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr dead_on_return noundef %a)
 
 // Return HFAs of 4 or fewer elements in registers.
 static struct HFA2 g_hfa2;
@@ -68,26 +68,26 @@ v4f32 __vectorcall hva1(int a, struct HVA4 b, int c) {return b.w;}
 // X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva1@@80"(i32 noundef %a, %struct.HVA4 inreg %b.coerce, i32 noundef %c)
 
 v4f32 __vectorcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {return c;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c)
 
 v4f32 __vectorcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {return f.x;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> inreg noundef %a, <4 x float> inreg noundef %b, <4 x float> inreg noundef %c, <4 x float> inreg noundef %d, <4 x float> inreg noundef %e, ptr inreg noundef %f)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> noundef %a, <4 x float> noundef %b, <4 x float> noundef %c, <4 x float> noundef %d, <4 x float> noundef %e, ptr noundef %f)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> inreg noundef %a, <4 x float> inreg noundef %b, <4 x float> inreg noundef %c, <4 x float> inreg noundef %d, <4 x float> inreg noundef %e, ptr dead_on_return inreg noundef %f)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> noundef %a, <4 x float> noundef %b, <4 x float> noundef %c, <4 x float> noundef %d, <4 x float> noundef %e, ptr dead_on_return noundef %f)
 
 // vector types have higher priority then HVA structures, So vector types are allocated first
 // and HVAs are allocated if enough registers are available
 v4f32 __vectorcall hva4(struct HVA4 a, struct HVA2 b, v4f32 c) {return b.y;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c)
 
 v4f32 __vectorcall hva5(struct HVA3 a, struct HVA3 b, v4f32 c, struct HVA2 d) {return d.y;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c, %struct.HVA2 inreg %d.coerce)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c, %struct.HVA2 inreg %d.coerce)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c, %struct.HVA2 inreg %d.coerce)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c, %struct.HVA2 inreg %d.coerce)
 
 struct HVA4 __vectorcall hva6(struct HVA4 a, struct HVA4 b) { return b;}
-// X86: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b)
-// X64: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr noundef %b)
+// X86: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b)
+// X64: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b)
 
 struct HVA5 __vectorcall hva7(void) {struct HVA5 a = {}; return a;}
 // X86: define dso_local x86_vectorcallcc void @"\01hva7@@0"(ptr dead_on_unwind noalias writable sret(%struct.HVA5) align 16 %agg.result)
@@ -108,8 +108,8 @@ void __vectorcall odd_size_hva(struct OddSizeHVA a) {}
 // consider 'p7' as a register.  Instead p5 gets put into the register on the second pass.
 // x86 should pass p2, p6 and p7 in registers, then p1 in the second pass.
 struct HFA2 __vectorcall AddParticles(struct HFA2 p1, float p2, struct HFA4 p3, int p4, struct HFA2 p5, float p6, float p7, int p8){ return p1;}
-// X86: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@84"(%struct.HFA2 inreg %p1.coerce, float inreg noundef %p2, ptr inreg noundef %p3, i32 inreg noundef %p4, ptr noundef %p5, float inreg noundef %p6, float inreg noundef %p7, i32 noundef %p8)
-// X64: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@104"(%struct.HFA2 inreg %p1.coerce, float noundef %p2, ptr noundef %p3, i32 noundef %p4, %struct.HFA2 inreg %p5.coerce, float noundef %p6, float noundef %p7, i32 noundef %p8)
+// X86: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@84"(%struct.HFA2 inreg %p1.coerce, float inreg noundef %p2, ptr dead_on_return inreg noundef %p3, i32 inreg noundef %p4, ptr dead_on_return noundef %p5, float inreg noundef %p6, float inreg noundef %p7, i32 noundef %p8)
+// X64: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@104"(%struct.HFA2 inreg %p1.coerce, float noundef %p2, ptr dead_on_return noundef %p3, i32 noundef %p4, %struct.HFA2 inreg %p5.coerce, float noundef %p6, float noundef %p7, i32 noundef %p8)
 
 // Vectorcall in both architectures allows passing of an HVA as long as there is room,
 // even if it is not one of the first 6 arguments.  First pass puts p4 into a
@@ -117,8 +117,8 @@ struct HFA2 __vectorcall AddParticles(struct HFA2 p1, float p2, struct HFA4 p3,
 // in a register, does NOT put p7 in a register (since there's no room), then puts
 // p8 in a register.
 void __vectorcall HVAAnywhere(struct HFA2 p1, int p2, int p3, float p4, int p5, int p6, struct HFA4 p7, struct HFA2 p8, float p9){}
-// X86: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@88"(%struct.HFA2 inreg %p1.coerce, i32 inreg noundef %p2, i32 inreg noundef %p3, float inreg noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr noundef %p7, %struct.HFA2 inreg %p8.coerce, float inreg noundef %p9)
-// X64: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@112"(%struct.HFA2 inreg %p1.coerce, i32 noundef %p2, i32 noundef %p3, float noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr noundef %p7, %struct.HFA2 inreg %p8.coerce, float noundef %p9)
+// X86: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@88"(%struct.HFA2 inreg %p1.coerce, i32 inreg noundef %p2, i32 inreg noundef %p3, float inreg noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr dead_on_return noundef %p7, %struct.HFA2 inreg %p8.coerce, float inreg noundef %p9)
+// X64: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@112"(%struct.HFA2 inreg %p1.coerce, i32 noundef %p2, i32 noundef %p3, float noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr dead_on_return noundef %p7, %struct.HFA2 inreg %p8.coerce, float noundef %p9)
 
 #ifndef __x86_64__
 // This covers the three ways XMM values can be passed on 32-bit x86:
@@ -137,9 +137,9 @@ void __vectorcall vectorcall_indirect_vec(
 // X86-SAME: double inreg noundef %xmm3,
 // X86-SAME: double inreg noundef %xmm4,
 // X86-SAME: <4 x float> inreg noundef %xmm5,
-// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: ptr dead_on_return inreg noundef %0,
 // X86-SAME: i32 inreg noundef %edx,
-// X86-SAME: ptr noundef %1)
+// X86-SAME: ptr dead_on_return noundef %1)
 
 void __vectorcall vectorcall_indirect_fp(
     double xmm0, double xmm1, double xmm2, double xmm3, double xmm4,
@@ -153,7 +153,7 @@ void __vectorcall vectorcall_indirect_fp(
 // X86-SAME: double inreg noundef %xmm3,
 // X86-SAME: double inreg noundef %xmm4,
 // X86-SAME: <4 x float> inreg noundef %xmm5,
-// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: ptr dead_on_return inreg noundef %0,
 // X86-SAME: i32 inreg noundef %edx,
 // X86-SAME: double noundef %mem)
 #endif
diff --git a/clang/test/CodeGen/win-fp128.c b/clang/test/CodeGen/win-fp128.c
index 328a7aaa7df57..8d223741bc93e 100644
--- a/clang/test/CodeGen/win-fp128.c
+++ b/clang/test/CodeGen/win-fp128.c
@@ -6,7 +6,7 @@ __float128 fp128_ret(void) { return 0; }
 // CHECK-GNU64: define dso_local <2 x i64>  @fp128_ret()
 
 __float128 fp128_args(__float128 a, __float128 b) { return a * b; }
-// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr noundef %0, ptr noundef %1)
+// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 
 void fp128_vararg(int a, ...) {
   // CHECK-GNU64-LABEL: define dso_local void @fp128_vararg
diff --git a/clang/test/CodeGen/win64-i128.c b/clang/test/CodeGen/win64-i128.c
index e10b2be0530eb..2d83889d8f89b 100644
--- a/clang/test/CodeGen/win64-i128.c
+++ b/clang/test/CodeGen/win64-i128.c
@@ -12,8 +12,8 @@ int128_t foo(void) { return 0; }
 
 int128_t bar(int128_t a, int128_t b) { return a * b; }
 
-// GNU64: define dso_local <2 x i64> @bar(ptr noundef %0, ptr noundef %1)
-// MSC64: define dso_local <2 x i64> @bar(ptr noundef %0, ptr noundef %1)
+// GNU64: define dso_local <2 x i64> @bar(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// MSC64: define dso_local <2 x i64> @bar(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 
 void vararg(int a, ...) {
   // GNU64-LABEL: define{{.*}} void @vararg
diff --git a/clang/test/CodeGen/windows-swiftcall.c b/clang/test/CodeGen/windows-swiftcall.c
index 41569c2606622..8716f25b9ddfb 100644
--- a/clang/test/CodeGen/windows-swiftcall.c
+++ b/clang/test/CodeGen/windows-swiftcall.c
@@ -219,7 +219,7 @@ TEST(struct_big_1)
 // CHECK-LABEL: define {{.*}} void @return_struct_big_1({{.*}} dead_on_unwind noalias writable sret
 
 // Should not be byval.
-// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr noundef{{( %.*)?}})
+// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr dead_on_return noundef{{( %.*)?}})
 
 /*****************************************************************************/
 /********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
index 152be26948f28..1709c88563267 100644
--- a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
+++ b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
@@ -576,7 +576,7 @@ void f(__clang_svmfloat8x4_t, __clang_svmfloat8x4_t);
 // CHECK-NEXT:    [[COERCE74_EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 2
 // CHECK-NEXT:    [[COERCE74_EXTRACT3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 3
 // CHECK-NEXT:    store { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[BYVAL_TEMP]], align 2
-// CHECK-NEXT:    call void @_Z1f10svboolx4_tS_(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT:    call void @_Z1f10svboolx4_tS_(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // CHECK-NEXT:    store { <vscale x 16 x i8>, <vscale x 16 x i8> } zeroinitializer, ptr [[COERCE75]], align 16
 // CHECK-NEXT:    [[COERCE75_TUPLE:%.*]] = load { <vscale x 16 x i8>, <vscale x 16 x i8> }, ptr [[COERCE75]], align 16
 // CHECK-NEXT:    [[COERCE75_EXTRACT0:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[COERCE75_TUPLE]], 0
@@ -1125,7 +1125,7 @@ void f(__clang_svmfloat8x4_t, __clang_svmfloat8x4_t);
 // COMPAT_17-NEXT:    [[COERCE74_EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 2
 // COMPAT_17-NEXT:    [[COERCE74_EXTRACT3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 3
 // COMPAT_17-NEXT:    store { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[BYVAL_TEMP]], align 2
-// COMPAT_17-NEXT:    call void @_Z1f10svboolx4_t10svboolx4_t(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr noundef [[BYVAL_TEMP]])
+// COMPAT_17-NEXT:    call void @_Z1f10svboolx4_t10svboolx4_t(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr dead_on_return noundef [[BYVAL_TEMP]])
 // COMPAT_17-NEXT:    store { <vscale x 16 x i8>, <vscale x 16 x i8> } zeroinitializer, ptr [[COERCE75]], align 16
 // COMPAT_17-NEXT:    [[COERCE75_TUPLE:%.*]] = load { <vscale x 16 x i8>, <vscale x 16 x i8> }, ptr [[COERCE75]], align 16
 // COMPAT_17-NEXT:    [[COERCE75_EXTRACT0:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[COERCE75_TUPLE]], 0
diff --git a/clang/test/CodeGenCXX/arm-cc.cpp b/clang/test/CodeGenCXX/arm-cc.cpp
index 68e1b7e4e1e46..939615fcc69e0 100644
--- a/clang/test/CodeGenCXX/arm-cc.cpp
+++ b/clang/test/CodeGenCXX/arm-cc.cpp
@@ -17,4 +17,4 @@ void baz() {
 }
 
 // CHECK: declare void @_Z3fooPv(ptr dead_on_unwind writable sret(%class.SMLoc) align 4, ptr noundef)
-// CHECK: declare void @_Z3zed5SMLoc(ptr noundef)
+// CHECK: declare void @_Z3zed5SMLoc(ptr dead_on_return noundef)
diff --git a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
index c341bd2b855ff..f2d602b3b523e 100644
--- a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
+++ b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
@@ -55,18 +55,18 @@ void usage() {
 // WINDOWS-NEXT: ret i32 %[[RET]]
 
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: ret i32 %[[LOAD]]
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 1
 // WINDOWS64: ret i32 %[[ADD]]
 
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
 // WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
 // WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
 // WINDOWS64: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 2
@@ -75,7 +75,7 @@ void usage() {
 // WINDOWS64: define dso_local void @"?usage@@YAXXZ"()
 // WINDOWS64: %[[F:[0-9a-zA-Z]+]] = alloca %struct.Foo
 // WINDOWS64: %[[ARG:[0-9a-zA-Z.]+]] = alloca %struct.Foo
-// WINDOWS64: %[[CALL:[0-9a-zA-Z]+]] = call noundef i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr noundef %[[ARG]])
+// WINDOWS64: %[[CALL:[0-9a-zA-Z]+]] = call noundef i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr dead_on_return noundef %[[ARG]])
 
 // WINDOWS64: define weak_odr dso_local i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr %0)
 // WINDOWS64: %[[RET:[0-9a-zA-Z]+]] = musttail call i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr %0)
diff --git a/clang/test/CodeGenCXX/copy-initialization.cpp b/clang/test/CodeGenCXX/copy-initialization.cpp
index aa0c6395f158d..4e6194cc040c2 100644
--- a/clang/test/CodeGenCXX/copy-initialization.cpp
+++ b/clang/test/CodeGenCXX/copy-initialization.cpp
@@ -12,7 +12,7 @@ struct Bar {
 
 void f(Foo);
 
-// CHECK-LABEL: define{{.*}} void @_Z1g3Foo(ptr noundef %foo)
+// CHECK-LABEL: define{{.*}} void @_Z1g3Foo(ptr dead_on_return noundef %foo)
 void g(Foo foo) {
   // CHECK: call void @_ZN3BarC1Ev
   // CHECK: @_ZNK3BarcvRK3FooEv
diff --git a/clang/test/CodeGenCXX/debug-info.cpp b/clang/test/CodeGenCXX/debug-info.cpp
index 8594a897ef7c0..9cf26ba83ba3e 100644
--- a/clang/test/CodeGenCXX/debug-info.cpp
+++ b/clang/test/CodeGenCXX/debug-info.cpp
@@ -4,7 +4,7 @@
 // CHECK: @_ZN6pr96081xE ={{.*}} global ptr null, align 8, !dbg [[X:![0-9]+]]
 
 // CHECK: define{{.*}} void @_ZN7pr147634funcENS_3fooE
-// CHECK-SAME: ptr noundef [[param:%.*]])
+// CHECK-SAME: ptr dead_on_return noundef [[param:%.*]])
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   alloca ptr, align 8
 // CHECK-NEXT:   [[param_addr_storage:%.*]] = alloca ptr, align 8
diff --git a/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp b/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
index c7d3a017414ef..c8f5a0f7c2ea6 100644
--- a/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
+++ b/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
@@ -13,7 +13,7 @@ struct Empty {
 };
 
 bool foo(Empty e) {
-// CHECK: @_Z3foo5Empty(ptr noundef %e)
+// CHECK: @_Z3foo5Empty(ptr dead_on_return noundef %e)
 // CHECK: call {{.*}} @_ZN5Empty5checkEv(ptr {{[^,]*}} %e)
   return e.check();
 }
@@ -21,6 +21,6 @@ bool foo(Empty e) {
 void caller(Empty &e) {
 // CHECK: @_Z6callerR5Empty(ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %e)
 // CHECK: call {{.*}} @_ZN5EmptyC1ERKS_(ptr {{[^,]*}} [[NEWTMP:%.*]], ptr
-// CHECK: call {{.*}} @_Z3foo5Empty(ptr noundef [[NEWTMP]])
+// CHECK: call {{.*}} @_Z3foo5Empty(ptr dead_on_return noundef [[NEWTMP]])
   foo(e);
 }
diff --git a/clang/test/CodeGenCXX/fastcall.cpp b/clang/test/CodeGenCXX/fastcall.cpp
index 4c94c1623ee16..405917f7a14bb 100644
--- a/clang/test/CodeGenCXX/fastcall.cpp
+++ b/clang/test/CodeGenCXX/fastcall.cpp
@@ -15,6 +15,6 @@ struct S1 {
 void __attribute__((fastcall)) foo2(S1 a, int b);
 void bar2(S1 a, int b) {
   // CHECK-LABEL: define{{.*}} void @_Z4bar22S1i
-  // CHECK: call x86_fastcallcc void @_Z4foo22S1i(ptr inreg %{{.*}}, i32 inreg %
+  // CHECK: call x86_fastcallcc void @_Z4foo22S1i(ptr dead_on_return inreg %{{.*}}, i32 inreg %
   foo2(a, b);
 }
diff --git a/clang/test/CodeGenCXX/homogeneous-aggregates.cpp b/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
index 63ffc6b5bfac8..5ebeb8aad4c18 100644
--- a/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
+++ b/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
@@ -41,8 +41,8 @@ struct D5 : I1, I2, I3 {}; // homogeneous aggregate
 
 // PPC: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, [3 x i64] %x.coerce)
 // ARM32: define{{.*}} arm_aapcs_vfpcc void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, [3 x i64] %x.coerce)
-// ARM64: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr noundef %x)
-// X64: define dso_local x86_vectorcallcc void @"\01_Z7func_D12D1@@24"(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr noundef %x)
+// ARM64: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr dead_on_return noundef %x)
+// X64: define dso_local x86_vectorcallcc void @"\01_Z7func_D12D1@@24"(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr dead_on_return noundef %x)
 D1 CC func_D1(D1 x) { return x; }
 
 // PPC: define{{.*}} [3 x double] @_Z7func_D22D2([3 x double] %x.coerce)
@@ -53,7 +53,7 @@ D2 CC func_D2(D2 x) { return x; }
 
 // PPC: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, [4 x i64] %x.coerce)
 // ARM32: define{{.*}} arm_aapcs_vfpcc void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, [4 x i64] %x.coerce)
-// ARM64: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, ptr noundef %x)
+// ARM64: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, ptr dead_on_return noundef %x)
 D3 CC func_D3(D3 x) { return x; }
 
 // PPC: define{{.*}} [4 x double] @_Z7func_D42D4([4 x double] %x.coerce)
@@ -201,7 +201,7 @@ struct NonHFA {
   virtual void f1();
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at polymorphic@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at polymorphic@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace trivial_copy_assignment {
 struct HFA {
@@ -221,7 +221,7 @@ struct NonHFA {
   NonHFA &operator=(const NonHFA&);
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at non_trivial_copy_assignment@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at non_trivial_copy_assignment@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace user_provided_ctor {
 struct HFA {
@@ -251,7 +251,7 @@ struct NonHFA {
   ~NonHFA();
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at non_trivial_dtor@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at non_trivial_dtor@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace non_empty_base {
 struct non_empty_base { double d; };
@@ -272,7 +272,7 @@ struct NonHFA {
   empty e;
 };
 double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at empty_field@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at empty_field@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
 }
 namespace non_empty_field {
 struct non_empty { double d; };
diff --git a/clang/test/CodeGenCXX/inalloca-lambda.cpp b/clang/test/CodeGenCXX/inalloca-lambda.cpp
index dc78aa2773f89..0d527e1a02e5f 100644
--- a/clang/test/CodeGenCXX/inalloca-lambda.cpp
+++ b/clang/test/CodeGenCXX/inalloca-lambda.cpp
@@ -22,7 +22,7 @@ void test() {
 // CHECK: %[[V:.*]] = getelementptr inbounds nuw <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0
 // CHECK: %call = call x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %this, ptr noundef %[[V]])
+// CHECK-SAME: (ptr noundef %this, ptr dead_on_return noundef %[[V]])
 
 // CHECK: define internal noundef i32
 // CHECK-SAME: @"?__invoke@<lambda_0>@?0??test@@YAXXZ at CA?A?<auto>@@UA@@@Z"
@@ -31,12 +31,12 @@ void test() {
 // CHECK: %[[VAR:.*]] = getelementptr inbounds nuw <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0
 // CHECK: %call = call x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %unused.capture, ptr noundef %[[VAR]])
+// CHECK-SAME: (ptr noundef %unused.capture, ptr dead_on_return noundef %[[VAR]])
 // CHECK: ret i32 %call
 
 // CHECK: define internal x86_thiscallcc noundef i32
 // CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %this, ptr noundef %[[ARG:.*]])
+// CHECK-SAME: (ptr noundef %this, ptr dead_on_return noundef %[[ARG:.*]])
 // CHECK: %this.addr = alloca ptr, align 4
 // CHECK: store ptr %this, ptr %this.addr, align 4
 // CHECK: %this1 = load ptr, ptr %this.addr, align 4
diff --git a/clang/test/CodeGenCXX/inalloca-overaligned.cpp b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
index 3751751ec0bca..305b8c5dccd10 100644
--- a/clang/test/CodeGenCXX/inalloca-overaligned.cpp
+++ b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
@@ -57,7 +57,7 @@ int receive_both(Both o) {
 }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?receive_both@@Y{{.*}}"
-// CHECK-SAME: (ptr noundef %o)
+// CHECK-SAME: (ptr dead_on_return noundef %o)
 
 int pass_both() {
   gvi32 = receive_both(Both());
@@ -67,7 +67,7 @@ int pass_both() {
 // CHECK-LABEL: define dso_local noundef i32 @"?pass_both@@Y{{.*}}"
 // CHECK: [[TMP:%[^ ]*]] = alloca %struct.Both, align 8
 // CHECK: call x86_thiscallcc noundef ptr @"??0Both@@QAE at XZ"(ptr {{[^,]*}} [[TMP]])
-// CHECK: call noundef i32 @"?receive_both@@Y{{.*}}"(ptr noundef [[TMP]])
+// CHECK: call noundef i32 @"?receive_both@@Y{{.*}}"(ptr dead_on_return noundef [[TMP]])
 
 int receive_inalloca_both(NonTrivial nt, Both o) {
   return nt.x + o.x + o.y;
@@ -101,11 +101,11 @@ struct [[trivial_abi]] alignas(8) MyPtr {
 int receiveMyPtr(MyPtr o) { return *o.ptr; }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?receiveMyPtr@@Y{{.*}}"
-// CHECK-SAME: (ptr noundef %o)
+// CHECK-SAME: (ptr dead_on_return noundef %o)
 
 int passMyPtr() { return receiveMyPtr(MyPtr()); }
 
 // CHECK-LABEL: define dso_local noundef i32 @"?passMyPtr@@Y{{.*}}"
 // CHECK: [[TMP:%[^ ]*]] = alloca %struct.MyPtr, align 8
 // CHECK: call x86_thiscallcc noundef ptr @"??0MyPtr@@QAE at XZ"(ptr {{[^,]*}} [[TMP]])
-// CHECK: call noundef i32 @"?receiveMyPtr@@Y{{.*}}"(ptr noundef [[TMP]])
+// CHECK: call noundef i32 @"?receiveMyPtr@@Y{{.*}}"(ptr dead_on_return noundef [[TMP]])
diff --git a/clang/test/CodeGenCXX/inalloca-vector.cpp b/clang/test/CodeGenCXX/inalloca-vector.cpp
index d1bacb4f0dc8c..2db4c49df116a 100644
--- a/clang/test/CodeGenCXX/inalloca-vector.cpp
+++ b/clang/test/CodeGenCXX/inalloca-vector.cpp
@@ -56,7 +56,7 @@ void __fastcall fastcall_receive_vec(__m128 x, __m128 y, __m128 z, __m128 w, int
 // CHECK-SAME: (<4 x float> inreg noundef %x,
 // CHECK-SAME: <4 x float> inreg noundef %y,
 // CHECK-SAME: <4 x float> inreg noundef %z,
-// CHECK-SAME: ptr inreg noundef %0,
+// CHECK-SAME: ptr dead_on_return inreg noundef %0,
 // CHECK-SAME: i32 inreg noundef %edx,
 // CHECK-SAME: ptr inalloca(<{ ptr, %struct.NonTrivial }>) %1)
 
@@ -73,6 +73,6 @@ void __vectorcall vectorcall_receive_vec(double xmm0, double xmm1, double xmm2,
 // CHECK-SAME: <4 x float> inreg noundef %x,
 // CHECK-SAME: <4 x float> inreg noundef %y,
 // CHECK-SAME: <4 x float> inreg noundef %z,
-// CHECK-SAME: ptr inreg noundef %0,
+// CHECK-SAME: ptr dead_on_return inreg noundef %0,
 // CHECK-SAME: i32 inreg noundef %edx,
 // CHECK-SAME: ptr inalloca(<{ ptr, %struct.NonTrivial }>) %1)
diff --git a/clang/test/CodeGenCXX/inheriting-constructor.cpp b/clang/test/CodeGenCXX/inheriting-constructor.cpp
index 100ca269d7f3c..21751bea055dc 100644
--- a/clang/test/CodeGenCXX/inheriting-constructor.cpp
+++ b/clang/test/CodeGenCXX/inheriting-constructor.cpp
@@ -166,7 +166,7 @@ namespace inalloca_nonvirt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
-  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
 
@@ -202,7 +202,7 @@ namespace inalloca_nonvirt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
-  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
 }
@@ -253,7 +253,7 @@ namespace inalloca_virt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: br i1
-  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: br
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
@@ -302,7 +302,7 @@ namespace inalloca_virt {
   // WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
   // WIN64: br i1
   // WIN64: store {{.*}} @"??_8C at inalloca_virt@@7B@"
-  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+  // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
   // WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
diff --git a/clang/test/CodeGenCXX/member-function-pointer-calls.cpp b/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
index ff511c0243801..f06cda8b7684e 100644
--- a/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
+++ b/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
@@ -16,7 +16,7 @@ int f(A* a, int (A::*fp)()) {
 // CHECK-NOT: }
 // CHECK: ret i32 1
 // MINGW64-LABEL: define dso_local noundef i32 @_Z2g1v()
-// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr noundef %{{.*}})
+// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr dead_on_return noundef %{{.*}})
 int g1() {
   A a;
   return f(&a, &A::vf1);
@@ -26,7 +26,7 @@ int g1() {
 // CHECK-NOT: }
 // CHECK: ret i32 2
 // MINGW64-LABEL: define dso_local noundef i32 @_Z2g2v()
-// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr noundef %{{.*}})
+// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr dead_on_return noundef %{{.*}})
 int g2() {
   A a;
   return f(&a, &A::vf2);
diff --git a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
index b551df747c073..63a4d5525336b 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
@@ -24,7 +24,7 @@ void foo(A a, A b, A c) {
 // X86: ret void
 
 // X64-LABEL: define dso_local void @"?foo@@YAXUA@@00 at Z"
-// X64:         (ptr noundef %[[a:[^,]*]], ptr noundef %[[b:[^,]*]], ptr noundef %[[c:[^)]*]])
+// X64:         (ptr dead_on_return noundef %[[a:[^,]*]], ptr dead_on_return noundef %[[b:[^,]*]], ptr dead_on_return noundef %[[c:[^)]*]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[a]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[b]])
 // X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[c]])
@@ -64,7 +64,7 @@ void call_foo() {
 // X64: invoke noundef ptr @"??0A@@QEAA at H@Z"(ptr {{[^,]*}} %[[arg2:[^,]*]], i32 noundef 2)
 // X64: invoke noundef ptr @"??0A@@QEAA at H@Z"(ptr {{[^,]*}} %[[arg1:[^,]*]], i32 noundef 1)
 // X64: call void @"?foo@@YAXUA@@00 at Z"
-// X64:       (ptr noundef %[[arg1]], ptr noundef %[[arg2]], ptr noundef %[[arg3]])
+// X64:       (ptr dead_on_return noundef %[[arg1]], ptr dead_on_return noundef %[[arg2]], ptr dead_on_return noundef %[[arg3]])
 // X64: ret void
 //
 //   lpad2:
diff --git a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
index 5654db3ba8151..813abb03a7810 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
@@ -22,10 +22,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret void
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@byval_thunk@@W7EAAXUAgg at 2@@Z"
-// CHECK64:             (ptr noundef %this, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@byval_thunk@@UEAAXUAgg at 2@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
@@ -54,10 +54,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret void
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@stdcall_thunk@@W7EAAXUAgg at 2@@Z"
-// CHECK64:             (ptr noundef %this, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@stdcall_thunk@@UEAAXUAgg at 2@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
@@ -86,10 +86,10 @@ C::C() {} // force emission
 // CHECK32-NEXT: ret ptr %[[rv]]
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@sret_thunk@@W7EAA?AUAgg at 2@U32@@Z"
-// CHECK64:             (ptr noundef %this, ptr dead_on_unwind noalias writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr noundef %x)
+// CHECK64:             (ptr noundef %this, ptr dead_on_unwind noalias writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr dead_on_return noundef %x)
 // CHECK64:   getelementptr i8, ptr %{{.*}}, i32 -8
 // CHECK64:   call void @"?foo at C@sret_thunk@@UEAA?AUAgg at 2@U32@@Z"
-// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr noundef %x)
+// CHECK64:       (ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr dead_on_return noundef %x)
 // CHECK64-NOT: call
 // CHECK64:   ret void
 }
diff --git a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
index 806bc5b63ef02..8defb68c668b2 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
@@ -428,7 +428,7 @@ bool nullTestDataUnspecified(int Unspecified::*mp) {
 
 // Pass this large type indirectly.
 // X64-LABEL: define dso_local noundef zeroext i1 @"?nullTestDataUnspecified@@
-// X64:             (ptr noundef %0)
+// X64:             (ptr dead_on_return noundef %0)
 }
 
 bool nullTestFunctionUnspecified(void (Unspecified::*mp)()) {
@@ -590,7 +590,7 @@ bool unspecFuncMemptrEq(void (Unspecified::*l)(), void (Unspecified::*r)()) {
 // CHECK: }
 
 // X64-LABEL: define dso_local noundef zeroext i1 @"?unspecFuncMemptrEq@@
-// X64:             (ptr noundef %0, ptr noundef %1)
+// X64:             (ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 }
 
 bool unspecFuncMemptrNeq(void (Unspecified::*l)(), void (Unspecified::*r)()) {
@@ -635,7 +635,7 @@ bool unspecDataMemptrEq(int Unspecified::*l, int Unspecified::*r) {
 // CHECK: }
 
 // X64-LABEL: define dso_local noundef zeroext i1 @"?unspecDataMemptrEq@@
-// X64:             (ptr noundef %0, ptr noundef %1)
+// X64:             (ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
 }
 
 void (Multiple::*convertB2FuncToMultiple(void (B2::*mp)()))() {
diff --git a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
index 6e8ba3953b2cf..767bf168633ae 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
@@ -175,7 +175,7 @@ void multibyte_arg(Multibyte s) {}
 void packed_arg(Packed s) {}
 // LINUX-LABEL: define{{.*}} void @_Z10packed_arg6Packed(ptr noundef byval(%struct.Packed) align 4 %s)
 // WIN32: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr noundef byval(%struct.Packed) align 4 %s)
-// WIN64: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr dead_on_return noundef %s)
 
 // Test that dtors are invoked in the callee.
 void small_arg_with_dtor(SmallWithDtor s) {}
@@ -190,7 +190,7 @@ void small_arg_with_dtor(SmallWithDtor s) {}
 // WOA64: }
 
 // FIXME: MSVC incompatible!
-// WOA: define dso_local arm_aapcs_vfpcc void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(ptr noundef %s) {{.*}} {
+// WOA: define dso_local arm_aapcs_vfpcc void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(ptr dead_on_return noundef %s) {{.*}} {
 // WOA:   call arm_aapcs_vfpcc void @"??1SmallWithDtor@@QAA at XZ"(ptr {{[^,]*}} %s)
 // WOA: }
 
@@ -220,7 +220,7 @@ void ref_small_arg_with_dtor(const SmallWithDtor &s) { }
 // WIN64-LABEL: define dso_local void @"?ref_small_arg_with_dtor@@YAXAEBUSmallWithDtor@@@Z"(ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %s)
 
 void big_arg_with_dtor(BigWithDtor s) {}
-// WIN64-LABEL: define dso_local void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr noundef %s)
+// WIN64-LABEL: define dso_local void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr dead_on_return noundef %s)
 // WIN64:   call void @"??1BigWithDtor@@QEAA at XZ"
 // WIN64: }
 
@@ -231,7 +231,7 @@ void call_big_arg_with_dtor() {
 // larger than 8 bytes and is passed indirectly.
 // WIN64-LABEL: define dso_local void @"?call_big_arg_with_dtor@@YAXXZ"()
 // WIN64:   call noundef ptr @"??0BigWithDtor@@QEAA at XZ"
-// WIN64:   call void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr noundef %{{.*}})
+// WIN64:   call void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr dead_on_return noundef %{{.*}})
 // WIN64-NOT: call void @"??1BigWithDtor@@QEAA at XZ"
 // WIN64:   ret void
 
@@ -259,22 +259,22 @@ void eh_cleanup_arg_with_dtor() {
 // WIN32: }
 
 void small_arg_with_vftable(SmallWithVftable s) {}
-// LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(ptr noundef %s)
+// LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(ptr dead_on_return noundef %s)
 // WIN32: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr inalloca(<{ %struct.SmallWithVftable }>) %0)
-// WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr noundef %s)
-// WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr dead_on_return noundef %s)
+// WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr dead_on_return noundef %s)
 
 void medium_arg_with_copy_ctor(MediumWithCopyCtor s) {}
-// LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(ptr noundef %s)
+// LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(ptr dead_on_return noundef %s)
 // WIN32: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr inalloca(<{ %struct.MediumWithCopyCtor }>) %0)
-// WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
-// WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
-// WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
+// WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
+// WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
 
 void big_arg(Big s) {}
 // LINUX-LABEL: define{{.*}} void @_Z7big_arg3Big(ptr noundef byval(%struct.Big) align 4 %s)
 // WIN32: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr noundef byval(%struct.Big) align 4 %s)
-// WIN64: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr dead_on_return noundef %s)
 
 // PR27607: We would attempt to load i32 value out of the reference instead of
 // just loading the pointer from the struct during argument expansion.
@@ -346,7 +346,7 @@ class Class {
   void thiscall_method_arg(Big s) {}
   // LINUX: define {{.*}} void @_ZN5Class19thiscall_method_argE3Big(ptr {{[^,]*}} %this, ptr noundef byval(%struct.Big) align 4 %s)
   // WIN32: define {{.*}} void @"?thiscall_method_arg at Class@@QAEXUBig@@@Z"(ptr {{[^,]*}} %this, ptr noundef byval(%struct.Big) align 4 %s)
-  // WIN64: define linkonce_odr dso_local void @"?thiscall_method_arg at Class@@QEAAXUBig@@@Z"(ptr {{[^,]*}} %this, ptr noundef %s)
+  // WIN64: define linkonce_odr dso_local void @"?thiscall_method_arg at Class@@QEAAXUBig@@@Z"(ptr {{[^,]*}} %this, ptr dead_on_return noundef %s)
 };
 
 void use_class() {
diff --git a/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp b/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
index 9e37e71e257fd..b7653632cf882 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
@@ -18,4 +18,4 @@ A B::foo(A x) {
   return x;
 }
 
-// CHECK-LABEL: define{{.*}} void @"?foo at B@@QEAA?AUA@@U2@@Z"(ptr {{[^,]*}} %this, ptr dead_on_unwind noalias writable sret(%struct.A) align 4 %agg.result, ptr noundef %x)
+// CHECK-LABEL: define{{.*}} void @"?foo at B@@QEAA?AUA@@U2@@Z"(ptr {{[^,]*}} %this, ptr dead_on_unwind noalias writable sret(%struct.A) align 4 %agg.result, ptr dead_on_return noundef %x)
diff --git a/clang/test/CodeGenCXX/ms-property.cpp b/clang/test/CodeGenCXX/ms-property.cpp
index 744de224b2f9a..d9fbf46dacb25 100644
--- a/clang/test/CodeGenCXX/ms-property.cpp
+++ b/clang/test/CodeGenCXX/ms-property.cpp
@@ -105,7 +105,7 @@ int main(int argc, char **argv) {
   // CHECK: [[ARGC:%.+]] = load i32, ptr %
   // CHECK: [[P1_X_ARGC_0:%.+]] = call noundef i32 @"?GetX at S@@QEAAHHH at Z"(ptr {{[^,]*}} [[P1]], i32 noundef [[ARGC]], i32 noundef 0)
   // CHECK: [[CAST:%.+]] = trunc i32 [[P1_X_ARGC_0]] to i8
-  // CHECK: [[P2_Y_p1_X_ARGC_0_T:%.+]] = call noundef i8 @"?GetY@?$St at M@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[P2_2]], i8 noundef [[CAST]], ptr noundef %{{.+}})
+  // CHECK: [[P2_Y_p1_X_ARGC_0_T:%.+]] = call noundef i8 @"?GetY@?$St at M@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[P2_2]], i8 noundef [[CAST]], ptr dead_on_return noundef %{{.+}})
   // CHECK: [[CAST:%.+]] = sitofp i8 [[P2_Y_p1_X_ARGC_0_T]] to float
   // CHECK: [[J:%.+]] = load i32, ptr %
   // CHECK: [[CAST1:%.+]] = sitofp i32 [[J]] to float
@@ -124,6 +124,6 @@ int main(int argc, char **argv) {
 // CHECK: call noundef i32 @"?GetX@?$St at H@@QEAAHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}} i32 noundef %{{.+}})
 // CHECK: call void @"?PutY@?$St at H@@QEAAXDHN at Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, i32 noundef %{{.+}}, double noundef %{{.+}}
 // CHECK: call noundef i32 @"?GetX@?$St at H@@QEAAHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}} i32 noundef %{{.+}})
-// CHECK: call noundef i8 @"?GetY@?$St at H@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, ptr noundef %{{.+}})
+// CHECK: call noundef i8 @"?GetY@?$St at H@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, ptr dead_on_return noundef %{{.+}})
 // CHECK: call noundef i32 @"?PutX@?$St at H@@QEAAHHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}}, i32 noundef %{{.+}}, i32 noundef %{{.+}})
 #endif //HEADER
diff --git a/clang/test/CodeGenCXX/nrvo.cpp b/clang/test/CodeGenCXX/nrvo.cpp
index 1141bc35de582..5b0fc914120e3 100644
--- a/clang/test/CodeGenCXX/nrvo.cpp
+++ b/clang/test/CodeGenCXX/nrvo.cpp
@@ -2197,7 +2197,7 @@ void test16() { // http://wg21.link/p2025r2#ex-9
 // CHECK-EH-11-NEXT:    br i1 [[CMP9]], label [[IF_THEN10:%.*]], label [[IF_END11:%.*]]
 // CHECK-EH-11:       if.then10:
 // CHECK-EH-11-NEXT:    store i32 3, ptr [[CLEANUP_DEST_SLOT]], align 4
-// CHECK-EH-11-NEXT:    br label [[CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK-EH-11-NEXT:    br label [[CLEANUP]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK-EH-11:       if.end11:
 // CHECK-EH-11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-EH-11-NEXT:    [[CMP12:%.*]] = icmp eq i32 [[TMP4]], 3
@@ -2239,7 +2239,7 @@ void test16() { // http://wg21.link/p2025r2#ex-9
 // CHECK-EH-11-NEXT:      i32 2, label [[IMPOSSIBLE]]
 // CHECK-EH-11-NEXT:    ]
 // CHECK-EH-11:       cleanup.cont:
-// CHECK-EH-11-NEXT:    br label [[WHILE_BODY]], !llvm.loop [[LOOP3]]
+// CHECK-EH-11-NEXT:    br label [[WHILE_BODY]], !llvm.loop [[LOOP4]]
 // CHECK-EH-11:       while.end:
 // CHECK-EH-11-NEXT:    call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]])
 // CHECK-EH-11-NEXT:    br label [[RETURN]]
diff --git a/clang/test/CodeGenCXX/pass-by-value-noalias.cpp b/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
index 773cf6b81c3b2..947379d9b8b92 100644
--- a/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
+++ b/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
@@ -14,8 +14,8 @@ struct Foo {
 // Make sure noalias is added to indirect arguments with trivially copyable types
 // if -fpass-by-value-is-noalias is provided.
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr dead_on_return noundef %arg)
 void take(Foo arg) {}
 
 int G;
@@ -38,8 +38,8 @@ struct NonTrivial {
 // Make sure noalias is not added to indirect arguments that are not trivially
 // copyable even if -fpass-by-value-is-noalias is provided.
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr noundef %arg)
-// NO_NOALIAS:   define{{.*}} void @_Z4take10NonTrivial(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr dead_on_return noundef %arg)
+// NO_NOALIAS:   define{{.*}} void @_Z4take10NonTrivial(ptr dead_on_return noundef %arg)
 void take(NonTrivial arg) {}
 
 // Escape examples. Pointers to the objects passed to take() may escape, depending on whether a temporary copy is created or not (e.g. due to NRVO).
@@ -54,8 +54,8 @@ struct A {
 };
 A *p;
 
-// WITH_NOALIAS: define{{.*}} void @_Z4take1A(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take1A(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take1A(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take1A(ptr dead_on_return noundef %arg)
 void take(A arg) {}
 
 // WITH_NOALIAS: define{{.*}} void @_Z7CreateAPP1A(ptr dead_on_unwind noalias writable sret(%struct.A) align 1 %agg.result, ptr noundef %where)
diff --git a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
index daeea77774ec8..0310535362e3d 100644
--- a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
@@ -24,7 +24,7 @@ struct __attribute__((trivial_abi)) TrivialSA {
 // Check that TrivialSA is passed indirectly despite being annotated with
 // 'trivial_abi'.
 
-// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr noundef %{{.*}})
+// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr dead_on_return noundef %{{.*}})
 
 void testParamTrivialSA(TrivialSA a) {
 }
diff --git a/clang/test/CodeGenCXX/regparm.cpp b/clang/test/CodeGenCXX/regparm.cpp
index b9735485db8de..a31394016fb55 100644
--- a/clang/test/CodeGenCXX/regparm.cpp
+++ b/clang/test/CodeGenCXX/regparm.cpp
@@ -11,7 +11,7 @@ struct S1 {
 };
 
 void __attribute__((regparm(3))) foo2(S1 a, int b);
-// CHECK: declare void @_Z4foo22S1i(ptr inreg noundef, i32 inreg noundef)
+// CHECK: declare void @_Z4foo22S1i(ptr dead_on_return inreg noundef, i32 inreg noundef)
 void bar2(S1 a, int b) {
   foo2(a, b);
 }
diff --git a/clang/test/CodeGenCXX/trivial_abi.cpp b/clang/test/CodeGenCXX/trivial_abi.cpp
index b8cc0d1cc6528..eacbde594e517 100644
--- a/clang/test/CodeGenCXX/trivial_abi.cpp
+++ b/clang/test/CodeGenCXX/trivial_abi.cpp
@@ -140,7 +140,7 @@ void testIgnoredSmall() {
   testReturnSmall();
 }
 
-// CHECK: define{{.*}} void @_Z14testParamLarge5Large(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5LargeD1Ev(ptr {{[^,]*}} %[[A]])
 // CHECK: ret void
 // CHECK: }
@@ -163,7 +163,7 @@ Large testReturnLarge() {
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_LARGE]], align 8
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[T]])
 // CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN5LargeC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(520) %[[T]])
-// CHECK: call void @_Z14testParamLarge5Large(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: %[[CALL2:.*]] = call noundef ptr @_ZN5LargeD1Ev(ptr {{[^,]*}} %[[T]])
 // CHECK: ret void
 // CHECK: }
@@ -176,7 +176,7 @@ void testCallLarge0() {
 // CHECK: define{{.*}} void @_Z14testCallLarge1v()
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_LARGE:.*]], align 8
 // CHECK: call void @_Z15testReturnLargev(ptr dead_on_unwind writable sret(%[[STRUCT_LARGE]]) align 8 %[[AGG_TMP]])
-// CHECK: call void @_Z14testParamLarge5Large(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: ret void
 // CHECK: }
 
@@ -244,7 +244,7 @@ void testExceptionSmall() {
 // CHECK: call noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[AGG_TMP]])
 // CHECK: invoke noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[AGG_TMP1]])
 
-// CHECK: call void @_Z20calleeExceptionLarge5LargeS_(ptr noundef %[[AGG_TMP]], ptr noundef %[[AGG_TMP1]])
+// CHECK: call void @_Z20calleeExceptionLarge5LargeS_(ptr dead_on_return noundef %[[AGG_TMP]], ptr dead_on_return noundef %[[AGG_TMP1]])
 // CHECK-NEXT: ret void
 
 // CHECK: landingpad { ptr, i32 }
diff --git a/clang/test/CodeGenCXX/uncopyable-args.cpp b/clang/test/CodeGenCXX/uncopyable-args.cpp
index 31192b65cc362..2d09732f9d3c2 100644
--- a/clang/test/CodeGenCXX/uncopyable-args.cpp
+++ b/clang/test/CodeGenCXX/uncopyable-args.cpp
@@ -59,12 +59,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN9move_ctor3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN9move_ctor3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN9move_ctor3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN9move_ctor3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at move_ctor@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at move_ctor@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace all_deleted {
@@ -81,12 +81,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN11all_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN11all_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN11all_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN11all_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at all_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at all_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace implicitly_deleted {
@@ -102,14 +102,14 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN18implicitly_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr)
 
 // In MSVC 2013, the copy ctor is not deleted by a move assignment. In MSVC 2015, it is.
 // WIN64-18-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(i64
-// WIN64-19-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-19-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace one_deleted {
@@ -125,12 +125,12 @@ void bar() {
 // CHECK-LABEL: define{{.*}} void @_ZN11one_deleted3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
 // CHECK-NOT: call
-// NEWABI: call void @_ZN11one_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN11one_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN11one_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at one_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at one_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace copy_defaulted {
@@ -170,7 +170,7 @@ void bar() {
 // CHECK: call void @_ZN14move_defaulted3fooENS_1AE(ptr %{{.*}})
 // CHECK-LABEL: declare void @_ZN14move_defaulted3fooENS_1AE(ptr)
 
-// WIN64-LABEL: declare dso_local void @"?foo at move_defaulted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at move_defaulted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace trivial_defaulted {
@@ -207,12 +207,12 @@ void bar() {
 }
 // CHECK-LABEL: define{{.*}} void @_ZN14two_copy_ctors3barEv()
 // CHECK: call void @_Z{{.*}}C1Ev(
-// NEWABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr dead_on_return noundef %{{.*}})
 // OLDABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef byval
-// NEWABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr dead_on_return noundef)
 // OLDABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef byval
 
-// WIN64-LABEL: declare dso_local void @"?foo at two_copy_ctors@@YAXUB at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at two_copy_ctors@@YAXUB at 1@@Z"(ptr dead_on_return noundef)
 }
 
 namespace definition_only {
diff --git a/clang/test/CodeGenCXX/wasm-args-returns.cpp b/clang/test/CodeGenCXX/wasm-args-returns.cpp
index fbb152ac1bb3d..7b1c27bf0808d 100644
--- a/clang/test/CodeGenCXX/wasm-args-returns.cpp
+++ b/clang/test/CodeGenCXX/wasm-args-returns.cpp
@@ -46,17 +46,17 @@ struct copy_ctor {
   copy_ctor(copy_ctor const &);
 };
 test(copy_ctor);
-// CHECK: define void @_Z7forward9copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.copy_ctor) align 8 %{{.*}}, ptr nonnull %{{.*}})
+// CHECK: define void @_Z7forward9copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.copy_ctor) align 8 %{{.*}}, ptr dead_on_return nonnull %{{.*}})
 //
 // CHECK: declare ptr @_ZN9copy_ctorC1ERKS_(ptr {{[^,]*}} returned {{[^,]*}}, ptr nonnull align 8 dereferenceable(8))
 //
 // CHECK: define void @_Z14test_copy_ctorv()
 // CHECK: %[[tmp:.*]] = alloca %struct.copy_ctor, align 8
 // CHECK: call void @_Z13def_copy_ctorv(ptr dead_on_unwind nonnull writable sret(%struct.copy_ctor) align 8 %[[tmp]])
-// CHECK: call void @_Z3use9copy_ctor(ptr nonnull %[[tmp]])
+// CHECK: call void @_Z3use9copy_ctor(ptr dead_on_return nonnull %[[tmp]])
 // CHECK: ret void
 //
-// CHECK: declare void @_Z3use9copy_ctor(ptr)
+// CHECK: declare void @_Z3use9copy_ctor(ptr dead_on_return)
 // CHECK: declare void @_Z13def_copy_ctorv(ptr dead_on_unwind writable sret(%struct.copy_ctor) align 8)
 
 struct __attribute__((aligned(16))) aligned_copy_ctor {
@@ -64,17 +64,17 @@ struct __attribute__((aligned(16))) aligned_copy_ctor {
   aligned_copy_ctor(aligned_copy_ctor const &);
 };
 test(aligned_copy_ctor);
-// CHECK: define void @_Z7forward17aligned_copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.aligned_copy_ctor) align 16 %{{.*}}, ptr nonnull %{{.*}})
+// CHECK: define void @_Z7forward17aligned_copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.aligned_copy_ctor) align 16 %{{.*}}, ptr dead_on_return nonnull %{{.*}})
 //
 // CHECK: declare ptr @_ZN17aligned_copy_ctorC1ERKS_(ptr {{[^,]*}} returned {{[^,]*}}, ptr nonnull align 16 dereferenceable(16))
 //
 // CHECK: define void @_Z22test_aligned_copy_ctorv()
 // CHECK: %[[tmp:.*]] = alloca %struct.aligned_copy_ctor, align 16
 // CHECK: call void @_Z21def_aligned_copy_ctorv(ptr dead_on_unwind nonnull writable sret(%struct.aligned_copy_ctor) align 16 %[[tmp]])
-// CHECK: call void @_Z3use17aligned_copy_ctor(ptr nonnull %[[tmp]])
+// CHECK: call void @_Z3use17aligned_copy_ctor(ptr dead_on_return nonnull %[[tmp]])
 // CHECK: ret void
 //
-// CHECK: declare void @_Z3use17aligned_copy_ctor(ptr)
+// CHECK: declare void @_Z3use17aligned_copy_ctor(ptr dead_on_return)
 // CHECK: declare void @_Z21def_aligned_copy_ctorv(ptr dead_on_unwind writable sret(%struct.aligned_copy_ctor) align 16)
 
 struct empty {};
diff --git a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
index 9927d0b24799e..78c45753b5bbd 100644
--- a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
+++ b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
@@ -18,7 +18,7 @@ struct NonTrivial {
 
 SWIFTCALL int receiveNonTrivial(NonTrivial o) { return o.o; }
 
-// CHECK-LABEL: define dso_local swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr noundef %o)
+// CHECK-LABEL: define dso_local swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr dead_on_return noundef %o)
 
 int passNonTrivial() {
   return receiveNonTrivial({});
@@ -26,4 +26,4 @@ int passNonTrivial() {
 
 // CHECK-LABEL: define dso_local noundef i32 @"?passNonTrivial@@YAHXZ"()
 // CHECK-NOT: stacksave
-// CHECK: call swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr noundef %{{.*}})
+// CHECK: call swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr dead_on_return noundef %{{.*}})
diff --git a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m b/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
index d2a954ae26a04..ba8a04b52716e 100644
--- a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
+++ b/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
@@ -42,7 +42,7 @@ void testStrongException(void) {
 // CHECK: call void @genWeak(ptr dead_on_unwind writable sret(%[[STRUCT_WEAK]]) align 8 %[[AGG_TMP]])
 // CHECK: invoke void @genWeak(ptr dead_on_unwind writable sret(%[[STRUCT_WEAK]]) align 8 %[[AGG_TMP1]])
 
-// CHECK: call void @calleeWeak(ptr noundef %[[AGG_TMP]], ptr noundef %[[AGG_TMP1]])
+// CHECK: call void @calleeWeak(ptr dead_on_return noundef %[[AGG_TMP]], ptr dead_on_return noundef %[[AGG_TMP1]])
 // CHECK: ret void
 
 // CHECK: landingpad { ptr, i32 }
diff --git a/clang/test/CodeGenObjC/pass-by-value-noalias.m b/clang/test/CodeGenObjC/pass-by-value-noalias.m
index ed94d4c80b525..86a4ba36a1b8f 100644
--- a/clang/test/CodeGenObjC/pass-by-value-noalias.m
+++ b/clang/test/CodeGenObjC/pass-by-value-noalias.m
@@ -17,6 +17,6 @@ @interface Bar
   Bar *__weak f;
 };
 
-// WITH_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
-// NO_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
+// NO_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
 void take(struct Foo arg) {}
diff --git a/clang/test/CodeGenObjC/weak-in-c-struct.m b/clang/test/CodeGenObjC/weak-in-c-struct.m
index be80edd1ff11d..6809360d03da1 100644
--- a/clang/test/CodeGenObjC/weak-in-c-struct.m
+++ b/clang/test/CodeGenObjC/weak-in-c-struct.m
@@ -130,7 +130,7 @@ void test_move_assignment_Weak(Weak *p) {
   *p = getWeak();
 }
 
-// COMMON: define{{.*}} void @test_parameter_Weak(ptr noundef %[[A:.*]])
+// COMMON: define{{.*}} void @test_parameter_Weak(ptr dead_on_return noundef %[[A:.*]])
 // COMMON: call void @__destructor_{{.*}}(ptr %[[A]])
 
 void test_parameter_Weak(Weak a) {
@@ -142,7 +142,7 @@ void test_parameter_Weak(Weak a) {
 // COMMON: store ptr %[[A]], ptr %[[A_ADDR]]
 // COMMON: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]]
 // COMMON: call void @__copy_constructor_{{.*}}(ptr %[[AGG_TMP]], ptr %[[V0]])
-// COMMON: call void @calleeWeak(ptr noundef %[[AGG_TMP]])
+// COMMON: call void @calleeWeak(ptr dead_on_return noundef %[[AGG_TMP]])
 // COMMON-NEXT: ret
 
 void test_argument_Weak(Weak *a) {
@@ -164,7 +164,7 @@ Weak test_return_Weak(Weak *a) {
 // COMMON: %[[AGG_TMP:.*]] = alloca %[[STRUCT_WEAK]]
 // COMMON: br i1
 
-// COMMON: call void @objc_msgSend({{.*}}, ptr noundef %[[AGG_TMP]])
+// COMMON: call void @objc_msgSend({{.*}}, ptr dead_on_return noundef %[[AGG_TMP]])
 // COMMON: br
 
 // COMMON: call void @__destructor_{{.*}}(ptr %[[AGG_TMP]])
diff --git a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
index 4d76796d86d1f..3a043c4892981 100644
--- a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
@@ -75,7 +75,7 @@ - (void)passStrongWeak:(StrongWeak)a;
 - (void)passNonTrivial:(NonTrivial)a;
 @end
 
-// CHECK: define{{.*}} void @_Z19testParamStrongWeak10StrongWeak(ptr noundef %{{.*}})
+// CHECK: define{{.*}} void @_Z19testParamStrongWeak10StrongWeak(ptr dead_on_return noundef %{{.*}})
 // CHECK: call noundef ptr @_ZN10StrongWeakD1Ev(
 // CHECK-NEXT: ret void
 
@@ -88,7 +88,7 @@ void testParamStrongWeak(StrongWeak a) {
 // CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN10StrongWeakC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(16) %[[V0]])
-// CHECK: call void @_Z19testParamStrongWeak10StrongWeak(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z19testParamStrongWeak10StrongWeak(ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK-NOT: call
 // CHECK: ret void
 
@@ -107,13 +107,13 @@ StrongWeak testReturnStrongWeak(StrongWeak *a) {
   return *a;
 }
 
-// CHECK: define{{.*}} void @_Z27testParamContainsStrongWeak18ContainsStrongWeak(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z27testParamContainsStrongWeak18ContainsStrongWeak(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN18ContainsStrongWeakD1Ev(ptr {{[^,]*}} %[[A]])
 
 void testParamContainsStrongWeak(ContainsStrongWeak a) {
 }
 
-// CHECK: define{{.*}} void @_Z26testParamDerivedStrongWeak17DerivedStrongWeak(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z26testParamDerivedStrongWeak17DerivedStrongWeak(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN17DerivedStrongWeakD1Ev(ptr {{[^,]*}} %[[A]])
 
 void testParamDerivedStrongWeak(DerivedStrongWeak a) {
@@ -163,7 +163,7 @@ Strong testReturnStrong(Strong *a) {
   return *a;
 }
 
-// CHECK: define{{.*}} void @_Z21testParamWeakTemplate1SIU6__weakP11objc_objectE(ptr noundef %{{.*}})
+// CHECK: define{{.*}} void @_Z21testParamWeakTemplate1SIU6__weakP11objc_objectE(ptr dead_on_return noundef %{{.*}})
 // CHECK: call noundef ptr @_ZN1SIU6__weakP11objc_objectED1Ev(
 // CHECK-NEXT: ret void
 
@@ -237,7 +237,7 @@ void test0(C *c) {
 // CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_STRONGWEAK]], align 8
 // CHECK: br i1
 
-// CHECK: call void @objc_msgSend({{.*}}, ptr noundef %[[AGG_TMP]])
+// CHECK: call void @objc_msgSend({{.*}}, ptr dead_on_return noundef %[[AGG_TMP]])
 // CHECK: br
 
 // CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN10StrongWeakD1Ev(ptr noundef nonnull align 8 dereferenceable(16) %[[AGG_TMP]])
diff --git a/clang/test/CodeGenObjCXX/property-objects.mm b/clang/test/CodeGenObjCXX/property-objects.mm
index 7ae20f66177bd..8354794254933 100644
--- a/clang/test/CodeGenObjCXX/property-objects.mm
+++ b/clang/test/CodeGenObjCXX/property-objects.mm
@@ -60,7 +60,7 @@ - (struct CGRect)extent {return bounds;}
 
 // CHECK-LABEL: define{{.*}} i32 @main
 // CHECK: call void @_ZN1SC1ERKS_(ptr {{[^,]*}} [[AGGTMP:%[a-zA-Z0-9\.]+]], ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) {{%[a-zA-Z0-9\.]+}})
-// CHECK: call void @objc_msgSend(ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef [[AGGTMP]])
+// CHECK: call void @objc_msgSend(ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr dead_on_return noundef [[AGGTMP]])
 // CHECK-NEXT: ret i32 0
 int main() {
   I *i;
diff --git a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
index e5cb71bad47c0..9428940d6da48 100644
--- a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
@@ -15,7 +15,7 @@
 
 // Check that AddrDiscStrong0 is destructed in the callee.
 
-// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr noundef %[[A:.*]])
+// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr dead_on_return noundef %[[A:.*]])
 // CHECK: call noundef ptr @_ZN15AddrDiscStrong0D1Ev(ptr noundef nonnull align {{[0-9]+}} dereferenceable(16) %[[A]])
 // CHECK: ret void
 
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
index 738b7ab7f2b75..d42e51d04ab9d 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
@@ -4,6 +4,7 @@
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
+typedef unsigned int uint;
 typedef half __attribute__((ext_vector_type(2))) half2;
 
 // CHECK-LABEL: @test_setprio_inc_wg(
@@ -42,6 +43,24 @@ void test_s_wait_tensorcnt() {
   __builtin_amdgcn_s_wait_tensorcnt(0);
 }
 
+// CHECK-LABEL: @test_prng_b32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.prng.b32(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(1) [[TMP2]], align 4
+// CHECK-NEXT:    ret void
+//
+void test_prng_b32(global uint* out, uint a) {
+  *out = __builtin_amdgcn_prng_b32(a);
+}
+
 // CHECK-LABEL: @test_tanh_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -119,6 +138,25 @@ void test_rcp_bf16(global __bf16* out, __bf16 a)
   *out = __builtin_amdgcn_rcp_bf16(a);
 }
 
+// CHECK-LABEL: @test_sqrt_bf16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = call bfloat @llvm.amdgcn.sqrt.bf16(bfloat [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[TMP1]], ptr addrspace(1) [[TMP2]], align 2
+// CHECK-NEXT:    ret void
+//
+void test_sqrt_bf16(global __bf16* out, __bf16 a)
+{
+  *out = __builtin_amdgcn_sqrt_bf16(a);
+}
+
 // CHECK-LABEL: @test_rsq_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
diff --git a/clang/test/Driver/print-multi-selection-flags.c b/clang/test/Driver/print-multi-selection-flags.c
index 5f9383fbed8f4..8cf8f04bb6b48 100644
--- a/clang/test/Driver/print-multi-selection-flags.c
+++ b/clang/test/Driver/print-multi-selection-flags.c
@@ -107,3 +107,22 @@
 // CHECK-AARCH64-MULTILIB-CUSTOM-FLAG: --target=aarch64-unknown-none-eabi
 // CHECK-MULTILIB-CUSTOM-FLAG-DAG:     -fmultilib-flag=foo
 // CHECK-MULTILIB-CUSTOM-FLAG-DAG:     -fmultilib-flag=bar
+
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fropi              | FileCheck --check-prefixes=CHECK-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -frwpi              | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fropi -frwpi       | FileCheck --check-prefixes=CHECK-ROPI,CHECK-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fno-ropi -fno-rwpi | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a                     | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fpic               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIC1 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fPIC               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIC2 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fpie               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIE1 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fPIE               | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIE2 %s
+// CHECK-PIC2: -fPIC
+// CHECK-PIE2: -fPIE
+// CHECK-NO-PIC: -fno-pic
+// CHECK-NO-ROPI: -fno-ropi
+// CHECK-NO-RWPI: -fno-rwpi
+// CHECK-PIC1: -fpic
+// CHECK-PIE1: -fpie
+// CHECK-ROPI: -fropi
+// CHECK-RWPI: -frwpi
diff --git a/clang/test/Headers/stdarg.cpp b/clang/test/Headers/stdarg.cpp
index 20bf17caf15f7..bfc3af11a23b6 100644
--- a/clang/test/Headers/stdarg.cpp
+++ b/clang/test/Headers/stdarg.cpp
@@ -15,8 +15,8 @@
 
 #include <stdarg.h>
 
-// AARCH64-C: define {{.*}} @f(i32 noundef %n, ptr noundef %list)
-// AARCH64-CXX: define {{.*}} @_Z1fiSt9__va_list(i32 noundef %n, ptr noundef %list)
+// AARCH64-C: define {{.*}} @f(i32 noundef %n, ptr dead_on_return noundef %list)
+// AARCH64-CXX: define {{.*}} @_Z1fiSt9__va_list(i32 noundef %n, ptr dead_on_return noundef %list)
 // X86_64-C: define {{.*}} @f(i32 noundef %n, ptr noundef %list)
 // X86_64-CXX: define {{.*}} @_Z1fiP13__va_list_tag(i32 noundef %n, ptr noundef %list)
 // PPC64-C: define {{.*}} @f(i32 noundef signext %n, ptr noundef %list)
diff --git a/clang/test/OpenMP/for_firstprivate_codegen.cpp b/clang/test/OpenMP/for_firstprivate_codegen.cpp
index 0255e1e3d4aea..83b5939799642 100644
--- a/clang/test/OpenMP/for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/for_firstprivate_codegen.cpp
@@ -427,7 +427,7 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[TMP0]])
 // CHECK1-NEXT:    store i32 0, ptr [[RETVAL]], align 4
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
@@ -469,7 +469,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -543,12 +543,12 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store ptr [[TMP3]], ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store ptr [[TMP4]], ptr [[_TMP1]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -572,7 +572,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP6]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE6]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done6:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP8]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR7]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP7]], ptr noundef [[AGG_TMP8]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP8]]) #[[ATTR2]]
@@ -611,7 +611,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC4]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP9]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP9]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM11:%.*]] = sext i32 [[TMP19]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR5]], i64 0, i64 [[IDXPROM11]]
@@ -711,7 +711,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1044,8 +1044,8 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr @g1, align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr @g1, align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
 // CHECK4-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1086,7 +1086,7 @@ int main() {
 // CHECK4-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK4-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[G]], align 4
-// CHECK4-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK4-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store volatile i32 1, ptr [[TMP13]], align 4
 // CHECK4-NEXT:    store i32 2, ptr [[SIVAR3]], align 4
 // CHECK4-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 0
@@ -1103,7 +1103,7 @@ int main() {
 // CHECK4-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[G]], align 4
 // CHECK4-NEXT:    store volatile i32 [[TMP14]], ptr [[BLOCK_CAPTURED]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 5
-// CHECK4-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK4-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store ptr [[TMP15]], ptr [[BLOCK_CAPTURED5]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 7
 // CHECK4-NEXT:    [[TMP16:%.*]] = load i32, ptr [[SIVAR3]], align 4
@@ -1137,7 +1137,7 @@ int main() {
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
 // CHECK4-NEXT:    store i32 2, ptr [[BLOCK_CAPTURE_ADDR]], align 8
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR1]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK4-NEXT:    store i32 2, ptr [[TMP0]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
 // CHECK4-NEXT:    store i32 4, ptr [[BLOCK_CAPTURE_ADDR2]], align 4
diff --git a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
index 93e71b9a8312e..441e809dc59e5 100644
--- a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
@@ -350,9 +350,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i32 8, i1 false)
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i32 2
@@ -524,7 +524,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK1-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK1-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK1-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -535,22 +535,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK1-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK1-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK1-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK1-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK1-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK1-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK1-NEXT:    ret void
 //
@@ -578,25 +578,25 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK1-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK1-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK1-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK1-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP5]], -1
 // CHECK1-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP7]], 1
 // CHECK1-NEXT:    store i32 [[DIV]], ptr [[TMP6]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP8]], i32 0, i32 2
 // CHECK1-NEXT:    store i32 1111, ptr [[ARRAYIDX]], align 4
 // CHECK1-NEXT:    ret void
@@ -656,7 +656,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -738,9 +738,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i32 8, i1 false)
 // CHECK1-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i32 2
@@ -840,7 +840,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 0, ptr [[A]], align 4
 // CHECK1-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SST]], ptr [[THIS1]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[A_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_CASTED]], align 4
@@ -862,7 +862,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
@@ -896,7 +896,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -965,7 +965,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK3-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK3-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK3-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -976,22 +976,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK3-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK3-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK3-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK3-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK3-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK3-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK3-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK3-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK3-NEXT:    ret void
 //
@@ -1020,22 +1020,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK3-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK3-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP5]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK3-NEXT:    store ptr [[B_ADDR]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK3-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    store ptr [[TMP8]], ptr [[TMP7]], align 4
 // CHECK3-NEXT:    call void @_ZZN2SSC1ERiENKUlvE_clEv(ptr nonnull align 4 dereferenceable(16) [[REF_TMP]])
 // CHECK3-NEXT:    ret void
@@ -1053,32 +1053,32 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0:%.*]], ptr [[THIS1]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK3-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP7]], -1
 // CHECK3-NEXT:    store i32 [[DEC]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 4
+// CHECK3-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
 // CHECK3-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP10]], 1
 // CHECK3-NEXT:    store i32 [[DIV]], ptr [[TMP9]], align 4
 // CHECK3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK3-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 4
+// CHECK3-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
 // CHECK3-NEXT:    store i32 [[TMP17]], ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[B_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK3-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 4
+// CHECK3-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK3-NEXT:    store i32 [[TMP21]], ptr [[C_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP22:%.*]] = load i32, ptr [[C_CASTED]], align 4
@@ -1106,14 +1106,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK3-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK3-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK3-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK3-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
 // CHECK3-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK3-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -1252,7 +1252,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK4-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK4-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
 // CHECK4-NEXT:    store ptr [[TMP0]], ptr [[C]], align 4
 // CHECK4-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK4-NEXT:    store ptr [[A3]], ptr [[A2]], align 4
@@ -1263,22 +1263,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK4-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK4-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 4
 // CHECK4-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK4-NEXT:    store ptr [[E9]], ptr [[E]], align 4
-// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK4-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK4-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK4-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
 // CHECK4-NEXT:    ret void
 //
@@ -1307,11 +1307,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4
 // CHECK4-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 4
 // CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK4-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
 // CHECK4-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK4-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
 // CHECK4-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 4
 // CHECK4-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 0
@@ -1327,13 +1327,13 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BLOCK_CAPTURED_THIS_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 5
 // CHECK4-NEXT:    store ptr [[TMP0]], ptr [[BLOCK_CAPTURED_THIS_ADDR]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP3]], ptr [[BLOCK_CAPTURED]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 7
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK4-NEXT:    store i32 [[TMP4]], ptr [[BLOCK_CAPTURED5]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    store ptr [[TMP5]], ptr [[BLOCK_CAPTURED6]], align 4
 // CHECK4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___BLOCK_LITERAL_GENERIC:%.*]], ptr [[BLOCK]], i32 0, i32 3
 // CHECK4-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 4
@@ -1354,7 +1354,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[BLOCK_CAPTURED_THIS:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
 // CHECK4-NEXT:    [[THIS:%.*]] = load ptr, ptr [[BLOCK_CAPTURED_THIS]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 4
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK4-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
 // CHECK4-NEXT:    store i32 [[INC]], ptr [[TMP0]], align 4
@@ -1363,12 +1363,12 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP2]], -1
 // CHECK4-NEXT:    store i32 [[DEC]], ptr [[BLOCK_CAPTURE_ADDR1]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 4
+// CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK4-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP4]], 1
 // CHECK4-NEXT:    store i32 [[DIV]], ptr [[TMP3]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 4
+// CHECK4-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
 // CHECK4-NEXT:    store i32 [[TMP6]], ptr [[A_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_CASTED]], align 4
@@ -1377,7 +1377,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    store i32 [[TMP8]], ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[B_CASTED]], align 4
 // CHECK4-NEXT:    [[BLOCK_CAPTURE_ADDR5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 8
-// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 4
+// CHECK4-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
 // CHECK4-NEXT:    store i32 [[TMP11]], ptr [[C_CASTED]], align 4
 // CHECK4-NEXT:    [[TMP12:%.*]] = load i32, ptr [[C_CASTED]], align 4
@@ -1405,14 +1405,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK4-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 4
 // CHECK4-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
-// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK4-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK4-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK4-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK4-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK4-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META6]], !align [[META7]]
 // CHECK4-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK4-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK4-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -1530,9 +1530,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i64 2
@@ -1704,7 +1704,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK9-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK9-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK9-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -1715,22 +1715,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK9-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK9-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK9-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK9-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK9-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK9-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK9-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK9-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK9-NEXT:    ret void
 //
@@ -1758,25 +1758,25 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK9-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK9-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK9-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
-// CHECK9-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK9-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK9-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK9-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK9-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP5]], -1
 // CHECK9-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK9-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK9-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP7]], 1
 // CHECK9-NEXT:    store i32 [[DIV]], ptr [[TMP6]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP8]], i64 0, i64 2
 // CHECK9-NEXT:    store i32 1111, ptr [[ARRAYIDX]], align 4
 // CHECK9-NEXT:    ret void
@@ -1836,7 +1836,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1918,9 +1918,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i64 2
@@ -2020,7 +2020,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i32 0, ptr [[A]], align 4
 // CHECK9-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SST]], ptr [[THIS1]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK9-NEXT:    store i32 [[TMP1]], ptr [[A_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP2:%.*]] = load i64, ptr [[A_CASTED]], align 8
@@ -2042,7 +2042,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[A]], ptr [[A_ADDR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK9-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK9-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
@@ -2076,7 +2076,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2145,7 +2145,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK11-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK11-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK11-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK11-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK11-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -2156,22 +2156,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK11-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK11-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK11-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK11-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK11-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK11-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK11-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK11-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK11-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK11-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK11-NEXT:    ret void
 //
@@ -2200,22 +2200,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK11-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK11-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK11-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK11-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK11-NEXT:    store ptr [[TMP0]], ptr [[TMP3]], align 8
 // CHECK11-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK11-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP5]], ptr [[TMP4]], align 8
 // CHECK11-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK11-NEXT:    store ptr [[B_ADDR]], ptr [[TMP6]], align 8
 // CHECK11-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK11-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    store ptr [[TMP8]], ptr [[TMP7]], align 8
 // CHECK11-NEXT:    call void @_ZZN2SSC1ERiENKUlvE_clEv(ptr nonnull align 8 dereferenceable(32) [[REF_TMP]])
 // CHECK11-NEXT:    ret void
@@ -2233,32 +2233,32 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0:%.*]], ptr [[THIS1]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
 // CHECK11-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8
+// CHECK11-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK11-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
 // CHECK11-NEXT:    store i32 [[INC]], ptr [[TMP3]], align 4
 // CHECK11-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8
+// CHECK11-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK11-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP7]], -1
 // CHECK11-NEXT:    store i32 [[DEC]], ptr [[TMP6]], align 4
 // CHECK11-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8
+// CHECK11-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
 // CHECK11-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP10]], 1
 // CHECK11-NEXT:    store i32 [[DIV]], ptr [[TMP9]], align 4
 // CHECK11-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK11-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 8
+// CHECK11-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 // CHECK11-NEXT:    store i32 [[TMP13]], ptr [[A_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP14:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8
+// CHECK11-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
 // CHECK11-NEXT:    store i32 [[TMP17]], ptr [[B_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP18:%.*]] = load i64, ptr [[B_CASTED]], align 8
 // CHECK11-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK11-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+// CHECK11-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK11-NEXT:    store i32 [[TMP21]], ptr [[C_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP22:%.*]] = load i64, ptr [[C_CASTED]], align 8
@@ -2286,14 +2286,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK11-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK11-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK11-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK11-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK11-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK11-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK11-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK11-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK11-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK11-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -2432,7 +2432,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
 // CHECK12-NEXT:    store i8 [[BF_SET]], ptr [[B]], align 4
 // CHECK12-NEXT:    [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK12-NEXT:    store ptr [[TMP0]], ptr [[C]], align 8
 // CHECK12-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
 // CHECK12-NEXT:    store ptr [[A3]], ptr [[A2]], align 8
@@ -2443,22 +2443,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
 // CHECK12-NEXT:    store i32 [[BF_CAST]], ptr [[B4]], align 4
 // CHECK12-NEXT:    [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP1]], ptr [[C7]], align 8
 // CHECK12-NEXT:    [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
 // CHECK12-NEXT:    store ptr [[E9]], ptr [[E]], align 8
-// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK12-NEXT:    store i32 [[TMP3]], ptr [[A_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
 // CHECK12-NEXT:    [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
 // CHECK12-NEXT:    store i32 [[TMP5]], ptr [[B_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK12-NEXT:    store i32 [[TMP8]], ptr [[C_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
 // CHECK12-NEXT:    ret void
 //
@@ -2487,11 +2487,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    store i64 [[C]], ptr [[C_ADDR]], align 8
 // CHECK12-NEXT:    store ptr [[E]], ptr [[E_ADDR]], align 8
 // CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK12-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
 // CHECK12-NEXT:    store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK12-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
 // CHECK12-NEXT:    store ptr [[E3]], ptr [[_TMP4]], align 8
 // CHECK12-NEXT:    [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 0
@@ -2507,13 +2507,13 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BLOCK_CAPTURED_THIS_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 5
 // CHECK12-NEXT:    store ptr [[TMP0]], ptr [[BLOCK_CAPTURED_THIS_ADDR]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP3]], ptr [[BLOCK_CAPTURED]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 8
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK12-NEXT:    store i32 [[TMP4]], ptr [[BLOCK_CAPTURED5]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    store ptr [[TMP5]], ptr [[BLOCK_CAPTURED6]], align 8
 // CHECK12-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___BLOCK_LITERAL_GENERIC:%.*]], ptr [[BLOCK]], i32 0, i32 3
 // CHECK12-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 8
@@ -2534,7 +2534,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[BLOCK_CAPTURED_THIS:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
 // CHECK12-NEXT:    [[THIS:%.*]] = load ptr, ptr [[BLOCK_CAPTURED_THIS]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 8
+// CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK12-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
 // CHECK12-NEXT:    store i32 [[INC]], ptr [[TMP0]], align 4
@@ -2543,12 +2543,12 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP2]], -1
 // CHECK12-NEXT:    store i32 [[DEC]], ptr [[BLOCK_CAPTURE_ADDR1]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 8
+// CHECK12-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK12-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP4]], 1
 // CHECK12-NEXT:    store i32 [[DIV]], ptr [[TMP3]], align 4
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 8
+// CHECK12-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
 // CHECK12-NEXT:    store i32 [[TMP6]], ptr [[A_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP7:%.*]] = load i64, ptr [[A_CASTED]], align 8
@@ -2557,7 +2557,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    store i32 [[TMP8]], ptr [[B_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP9:%.*]] = load i64, ptr [[B_CASTED]], align 8
 // CHECK12-NEXT:    [[BLOCK_CAPTURE_ADDR5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
-// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 8
+// CHECK12-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
 // CHECK12-NEXT:    store i32 [[TMP11]], ptr [[C_CASTED]], align 4
 // CHECK12-NEXT:    [[TMP12:%.*]] = load i64, ptr [[C_CASTED]], align 8
@@ -2585,14 +2585,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK12-NEXT:    store ptr [[A_ADDR]], ptr [[TMP]], align 8
 // CHECK12-NEXT:    store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
-// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK12-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK12-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP2]], 1
 // CHECK12-NEXT:    store i32 [[INC]], ptr [[TMP1]], align 4
 // CHECK12-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK12-NEXT:    [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
 // CHECK12-NEXT:    store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK12-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK12-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK12-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK12-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
 // CHECK12-NEXT:    store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -2660,11 +2660,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    store i64 [[VLA2]], ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    store i64 [[VLA4]], ptr [[VLA_ADDR5]], align 8
 // CHECK17-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META3]], !align [[META5:![0-9]+]]
 // CHECK17-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2751,8 +2751,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META3]], !align [[META5]]
+// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
 // CHECK17-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 // CHECK17-NEXT:    [[TMP8:%.*]] = add nuw i64 [[TMP7]], 127
diff --git a/clang/test/OpenMP/sections_firstprivate_codegen.cpp b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
index 7c6d1839fb10e..32c5826e6f75d 100644
--- a/clang/test/OpenMP/sections_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
@@ -448,7 +448,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -517,10 +517,10 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_SECTIONS_LB_]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_UB_]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_ST_]], align 4
@@ -668,7 +668,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -996,7 +996,7 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK4-NEXT:    store i32 0, ptr [[DOTOMP_SECTIONS_LB_]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_UB_]], align 4
 // CHECK4-NEXT:    store i32 1, ptr [[DOTOMP_SECTIONS_ST_]], align 4
diff --git a/clang/test/OpenMP/single_firstprivate_codegen.cpp b/clang/test/OpenMP/single_firstprivate_codegen.cpp
index 27cd220adf225..31ea1ca4952fc 100644
--- a/clang/test/OpenMP/single_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/single_firstprivate_codegen.cpp
@@ -403,7 +403,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -467,10 +467,10 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK1-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_single(ptr @[[GLOB1]], i32 [[TMP5]])
@@ -585,7 +585,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -908,7 +908,7 @@ int main() {
 // CHECK4-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK4-NEXT:    store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
+// CHECK4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
 // CHECK4-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK4-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
 // CHECK4-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_single(ptr @[[GLOB1]], i32 [[TMP2]])
diff --git a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
index aa50d8fb3aabd..a171827a18646 100644
--- a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
@@ -354,9 +354,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -396,9 +396,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -550,12 +550,12 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -655,7 +655,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -711,14 +711,14 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -752,9 +752,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -776,7 +776,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -815,7 +815,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i64 0, i64 [[IDXPROM9]]
@@ -912,7 +912,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1155,9 +1155,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1197,9 +1197,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1349,12 +1349,12 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1454,7 +1454,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1510,14 +1510,14 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1551,9 +1551,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1575,7 +1575,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1613,7 +1613,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP14]]
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP15]], i32 4, i1 false)
@@ -1709,7 +1709,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1877,7 +1877,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -1943,13 +1943,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP8]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP9]], align 8
 // CHECK9-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP11]], ptr [[TMP10]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP12]], align 8
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
index 78e40e54671ac..678770520f677 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -415,9 +415,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -459,9 +459,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -616,9 +616,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -741,12 +741,12 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -846,7 +846,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -902,14 +902,14 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -944,9 +944,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -968,7 +968,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1005,7 +1005,7 @@ int main() {
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
@@ -1082,9 +1082,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1112,7 +1112,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done5:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR2]]
@@ -1151,7 +1151,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -1233,7 +1233,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1476,9 +1476,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1520,9 +1520,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1675,9 +1675,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -1796,12 +1796,12 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[TMP4]], align 4
 // CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1901,7 +1901,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1957,14 +1957,14 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1999,9 +1999,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -2023,7 +2023,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2058,7 +2058,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK3-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK3:       omp.inner.for.inc:
@@ -2135,9 +2135,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -2163,7 +2163,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2201,7 +2201,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -2282,7 +2282,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2450,7 +2450,7 @@ int main() {
 // CHECK5-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK5-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK5-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK5-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK5-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2520,7 +2520,7 @@ int main() {
 // CHECK5-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK5-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK5-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK5-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2607,13 +2607,13 @@ int main() {
 // CHECK5-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK5-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK5-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK5-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK5-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK5-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK5-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK5-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK5-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK5-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK5-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
@@ -2659,9 +2659,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -2703,9 +2703,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -2860,9 +2860,9 @@ int main() {
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK13-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -2987,14 +2987,14 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK13-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK13-NEXT:    ret void
 //
@@ -3029,9 +3029,9 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -3053,7 +3053,7 @@ int main() {
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done4:
-// CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3090,7 +3090,7 @@ int main() {
 // CHECK13-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK13-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK13-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK13-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK13-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK13-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK13:       omp.inner.for.inc:
@@ -3167,9 +3167,9 @@ int main() {
 // CHECK13-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK13-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK13-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -3197,7 +3197,7 @@ int main() {
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done5:
-// CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR3]]
@@ -3236,7 +3236,7 @@ int main() {
 // CHECK13-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK13-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK13-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK13-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK13-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK13-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK13-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -3321,7 +3321,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK13-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -3352,7 +3352,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK13-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK13-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -3379,9 +3379,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9:![0-9]+]], !align [[META10:![0-9]+]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -3423,9 +3423,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -3578,9 +3578,9 @@ int main() {
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -3701,14 +3701,14 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK15-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK15-NEXT:    ret void
 //
@@ -3743,9 +3743,9 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -3767,7 +3767,7 @@ int main() {
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
-// CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3802,7 +3802,7 @@ int main() {
 // CHECK15-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK15-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK15-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK15-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK15-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK15:       omp.inner.for.inc:
@@ -3879,9 +3879,9 @@ int main() {
 // CHECK15-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK15-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK15-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -3907,7 +3907,7 @@ int main() {
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
-// CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3945,7 +3945,7 @@ int main() {
 // CHECK15-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK15-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK15-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK15-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK15-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK15-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -4029,7 +4029,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK15-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -4060,7 +4060,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
 // CHECK15-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK15-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -4089,7 +4089,7 @@ int main() {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK17-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK17-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
 // CHECK17-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK17-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -4159,7 +4159,7 @@ int main() {
 // CHECK17-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK17-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK17-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK17-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -4246,13 +4246,13 @@ int main() {
 // CHECK17-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK17-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK17-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK17-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK17-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK17-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK17-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK17-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK17-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK17-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK17-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
diff --git a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
index 294fcba7872b3..f3c9565a17656 100644
--- a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
@@ -357,9 +357,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -399,9 +399,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -553,14 +553,14 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store i64 [[TMP2]], ptr [[TMP6]], align 8
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -660,7 +660,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -716,14 +716,14 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -757,9 +757,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -781,7 +781,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -820,7 +820,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i64 0, i64 [[IDXPROM9]]
@@ -917,7 +917,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1160,9 +1160,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1202,9 +1202,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1354,14 +1354,14 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store i32 [[TMP2]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1461,7 +1461,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1517,14 +1517,14 @@ int main() {
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1558,9 +1558,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1582,7 +1582,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1620,7 +1620,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP14]]
 // CHECK3-NEXT:    store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP15]], i32 4, i1 false)
@@ -1716,7 +1716,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1884,7 +1884,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -1950,13 +1950,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP8]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP9]], align 8
 // CHECK9-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP11]], ptr [[TMP10]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP12]], align 8
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
index d742b0a85af42..037aa12d57226 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -391,9 +391,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -435,9 +435,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -592,9 +592,9 @@ int main() {
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK1-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -717,14 +717,14 @@ int main() {
 // CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK1-NEXT:    store i64 [[TMP2]], ptr [[TMP6]], align 8
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -824,7 +824,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -880,14 +880,14 @@ int main() {
 // CHECK1-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK1-NEXT:    ret void
 //
@@ -922,9 +922,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -946,7 +946,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -983,7 +983,7 @@ int main() {
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
@@ -1060,9 +1060,9 @@ int main() {
 // CHECK1-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1090,7 +1090,7 @@ int main() {
 // CHECK1-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK1-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK1:       omp.arraycpy.done5:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
 // CHECK1-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR2]]
@@ -1129,7 +1129,7 @@ int main() {
 // CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
 // CHECK1-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
 // CHECK1-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -1211,7 +1211,7 @@ int main() {
 // CHECK1-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
 // CHECK1-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK1-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1454,9 +1454,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1498,9 +1498,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1653,9 +1653,9 @@ int main() {
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -1774,14 +1774,14 @@ int main() {
 // CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
 // CHECK3-NEXT:    store i32 [[TMP2]], ptr [[TMP6]], align 4
 // CHECK3-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1881,7 +1881,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1937,14 +1937,14 @@ int main() {
 // CHECK3-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
 // CHECK3-NEXT:    ret void
 //
@@ -1979,9 +1979,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -2003,7 +2003,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2038,7 +2038,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
 // CHECK3-NEXT:    [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
 // CHECK3-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK3:       omp.inner.for.inc:
@@ -2115,9 +2115,9 @@ int main() {
 // CHECK3-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK3-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 4
 // CHECK3-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK3-NEXT:    store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -2143,7 +2143,7 @@ int main() {
 // CHECK3-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK3-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK3:       omp.arraycpy.done4:
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
 // CHECK3-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2181,7 +2181,7 @@ int main() {
 // CHECK3-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
 // CHECK3-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK3-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -2262,7 +2262,7 @@ int main() {
 // CHECK3-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK3-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK3-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK3-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2430,7 +2430,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2500,7 +2500,7 @@ int main() {
 // CHECK9-NEXT:    [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP11]], ptr [[G_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
 // CHECK9-NEXT:    store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2587,13 +2587,13 @@ int main() {
 // CHECK9-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK9-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
 // CHECK9-NEXT:    store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store volatile i32 1, ptr [[TMP10]], align 4
 // CHECK9-NEXT:    store i32 2, ptr [[SIVAR_ADDR]], align 4
 // CHECK9-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
 // CHECK9-NEXT:    store ptr [[G_ADDR]], ptr [[TMP11]], align 8
 // CHECK9-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
 // CHECK9-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK9-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK9-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
diff --git a/clang/test/OpenMP/teams_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
index 6f21c9e31bd8d..fec8fcb78f91e 100644
--- a/clang/test/OpenMP/teams_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
@@ -458,9 +458,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11:![0-9]+]], !align [[META12:![0-9]+]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -493,9 +493,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
 // CHECK9-NEXT:    store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i64 2
@@ -808,7 +808,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -872,9 +872,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK9-NEXT:    [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -902,9 +902,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
 // CHECK9-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i64 8, i1 false)
 // CHECK9-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i64 2
@@ -1032,7 +1032,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK9-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK9-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
 // CHECK9-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK9-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1250,9 +1250,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12:![0-9]+]], !align [[META13:![0-9]+]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1285,9 +1285,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i32 8, i1 false)
 // CHECK11-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i32 2
@@ -1600,7 +1600,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK11-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK11-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load float, ptr [[F2]], align 4
 // CHECK11-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1664,9 +1664,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
 // CHECK11-NEXT:    [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1694,9 +1694,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
 // CHECK11-NEXT:    store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i32 8, i1 false)
 // CHECK11-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i32 2
@@ -1824,7 +1824,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK11-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK11-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
 // CHECK11-NEXT:    [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
 // CHECK11-NEXT:    [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
 // CHECK11-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1997,7 +1997,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
 // CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[S_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[A_ADDR]], align 8
@@ -2031,11 +2031,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    store i64 [[VLA2]], ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    store i64 [[VLA4]], ptr [[VLA_ADDR5]], align 8
 // CHECK17-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META8]], !align [[META10:![0-9]+]]
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK17-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2235,7 +2235,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP0:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
 // CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[S_ADDR]], align 8
@@ -2273,8 +2273,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK17-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK17-NEXT:    [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
 // CHECK17-NEXT:    [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK17-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META8]], !align [[META10]]
 // CHECK17-NEXT:    [[TMP6:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK17-NEXT:    store ptr [[TMP6]], ptr [[SAVED_STACK]], align 8
 // CHECK17-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2460,7 +2460,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
 // CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[S_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[A_ADDR]], align 4
@@ -2494,11 +2494,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    store i32 [[VLA2]], ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    store i32 [[VLA4]], ptr [[VLA_ADDR5]], align 4
 // CHECK19-NEXT:    store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 4
-// CHECK19-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP3:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK19-NEXT:    store ptr [[TMP5]], ptr [[SAVED_STACK]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = mul nuw i32 [[TMP2]], [[TMP3]]
@@ -2696,7 +2696,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[S_ADDR]], align 4
@@ -2734,8 +2734,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK19-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK19-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
 // CHECK19-NEXT:    [[TMP3:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
-// CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 4
+// CHECK19-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK19-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
 // CHECK19-NEXT:    [[TMP6:%.*]] = call ptr @llvm.stacksave.p0()
 // CHECK19-NEXT:    store ptr [[TMP6]], ptr [[SAVED_STACK]], align 4
 // CHECK19-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP2]], [[TMP3]]
diff --git a/clang/test/Sema/dllexport.c b/clang/test/Sema/dllexport.c
index 3f911fb095c0f..5f6ff36e290e9 100644
--- a/clang/test/Sema/dllexport.c
+++ b/clang/test/Sema/dllexport.c
@@ -2,6 +2,10 @@
 // RUN: %clang_cc1 -triple x86_64-win32   -fsyntax-only -fms-extensions -verify -std=c11 %s
 // RUN: %clang_cc1 -triple i686-mingw32   -fsyntax-only -fms-extensions -verify -std=c11 %s
 // RUN: %clang_cc1 -triple x86_64-mingw32 -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple i686-windows-itanium   -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple x86_64-windows-itanium -fsyntax-only -fms-extensions -verify -std=c11 %s
+// RUN: %clang_cc1 -triple x86_64-sie-ps5         -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple x86_64-sie-ps5         -fsyntax-only -fms-extensions -verify -std=c11 %s
 
 // Invalid usage.
 __declspec(dllexport) typedef int typedef1;
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
index 46c3670848529..135865c8450f5 100644
--- a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
+++ b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -283,3 +283,31 @@ void f() {
 }
 
 #endif
+
+namespace GH147374 {
+
+struct String {};
+template <typename T> void operator+(T, String &&) = delete;
+
+struct Bar {
+    void operator+(String) const; // expected-note {{candidate function}}
+    friend void operator+(Bar, String) {};  // expected-note {{candidate function}}
+};
+
+struct Baz {
+    void operator+(String); // expected-note {{candidate function}}
+    friend void operator+(Baz, String) {}; // expected-note {{candidate function}}
+};
+
+void test() {
+    Bar a;
+    String b;
+    a + b;
+    //expected-error at -1 {{use of overloaded operator '+' is ambiguous (with operand types 'Bar' and 'String')}}
+
+    Baz z;
+    z + b;
+    //expected-error at -1 {{use of overloaded operator '+' is ambiguous (with operand types 'Baz' and 'String')}}
+}
+
+}
diff --git a/clang/test/SemaTemplate/concepts-using-decl.cpp b/clang/test/SemaTemplate/concepts-using-decl.cpp
index fca69dea5c88f..41f7b6d2f8faa 100644
--- a/clang/test/SemaTemplate/concepts-using-decl.cpp
+++ b/clang/test/SemaTemplate/concepts-using-decl.cpp
@@ -176,3 +176,24 @@ void func() {
   f.foo<10, 10>(); // expected-error {{no matching member function for call to 'foo'}}
 }
 } // namespace heads_without_concepts.
+
+namespace GH146614 {
+
+template <typename T>
+struct base {
+    template <typename A>
+    void foo(A x)
+        requires (requires{x;})
+    {}
+};
+
+
+struct child : base<int> {
+  using base<int>::foo;
+  template <typename A>
+  void foo(A x)
+      requires (false)
+  {}
+};
+
+}
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4241d12601242..5f0783f869bf6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1466,8 +1466,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     assert(falseTarget && "missing conditional branch false block");
     mlir::Location loc = toLocation();
     mlir::Value bcc = builder->createConvert(loc, builder->getI1Type(), cond);
-    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget, std::nullopt,
-                                            falseTarget, std::nullopt);
+    builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget,
+                                            mlir::ValueRange{}, falseTarget,
+                                            mlir::ValueRange{});
   }
   void genConditionalBranch(mlir::Value cond,
                             Fortran::lower::pft::Evaluation *trueTarget,
@@ -2556,8 +2557,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       builder->setInsertionPointToEnd(loopWrapperOp.getBody());
       auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
           loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
-          /*local_vars=*/std::nullopt,
-          /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+          /*local_vars=*/mlir::ValueRange{},
+          /*local_syms=*/nullptr, /*reduce_vars=*/mlir::ValueRange{},
           /*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
           /*reduce_attrs=*/nullptr);
 
@@ -3810,9 +3811,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       mlir::Block *selectCaseBlock = insertBlock(blockList[0]);
       mlir::Block *assumedSizeBlock =
           rankStarBlock ? rankStarBlock : defaultBlock;
-      builder->create<mlir::cf::CondBranchOp>(loc, isAssumedSize,
-                                              assumedSizeBlock, std::nullopt,
-                                              selectCaseBlock, std::nullopt);
+      builder->create<mlir::cf::CondBranchOp>(
+          loc, isAssumedSize, assumedSizeBlock, mlir::ValueRange{},
+          selectCaseBlock, mlir::ValueRange{});
       startBlock(selectCaseBlock);
     }
     // Create fir.select_case for the other rank cases.
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1850b67898126..b8ab5d09e3e08 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -303,7 +303,7 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
     return builder.create<fir::StringLitOp>(
-        loc, llvm::ArrayRef<mlir::Type>{type}, std::nullopt, attrs);
+        loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
   }
 }
 
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 0a1cd67789772..281ab229d1b6a 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1003,9 +1003,9 @@ class ScalarExprLowering {
           },
           [&](const fir::MutableBoxValue &toBox) {
             if (toBox.isPointer()) {
-              Fortran::lower::associateMutableBox(converter, loc, toBox, expr,
-                                                  /*lbounds=*/std::nullopt,
-                                                  stmtCtx);
+              Fortran::lower::associateMutableBox(
+                  converter, loc, toBox, expr,
+                  /*lbounds=*/mlir::ValueRange{}, stmtCtx);
               return;
             }
             // For allocatable components, a deep copy is needed.
@@ -3604,8 +3604,9 @@ class ArrayExprLowering {
       mlir::Value castTo =
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
-      return builder.create<fir::ArrayLoadOp>(
-          loc, seqTy, castTo, shapeOp, /*slice=*/mlir::Value{}, std::nullopt);
+      return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
+                                              /*slice=*/mlir::Value{},
+                                              mlir::ValueRange{});
     };
     // Custom lowering of the element store to deal with the extra indirection
     // to the lazy allocated buffer.
@@ -4207,7 +4208,7 @@ class ArrayExprLowering {
       auto addr =
           builder->create<fir::ArrayCoorOp>(loc, eleRefTy, tmp, shape,
                                             /*slice=*/mlir::Value{}, indices,
-                                            /*typeParams=*/std::nullopt);
+                                            /*typeParams=*/mlir::ValueRange{});
       auto load = builder->create<fir::LoadOp>(loc, addr);
       return builder->createConvert(loc, i1Ty, load);
     };
@@ -4522,17 +4523,18 @@ class ArrayExprLowering {
         fir::isRecordWithAllocatableMember(eleTy))
       TODO(loc, "creating an array temp where the element type has "
                 "allocatable members");
-    mlir::Value temp = !seqTy.hasDynamicExtents()
-                           ? builder.create<fir::AllocMemOp>(loc, type)
-                           : builder.create<fir::AllocMemOp>(
-                                 loc, type, ".array.expr", std::nullopt, shape);
+    mlir::Value temp =
+        !seqTy.hasDynamicExtents()
+            ? builder.create<fir::AllocMemOp>(loc, type)
+            : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
+                                              mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
     return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
                                             /*slice=*/mlir::Value{},
-                                            std::nullopt);
+                                            mlir::ValueRange{});
   }
 
   static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -6483,7 +6485,7 @@ class ArrayExprLowering {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
         mem = builder.create<fir::AllocMemOp>(
-            loc, eleTy, /*typeparams=*/std::nullopt, initBuffSz);
+            loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
         builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
       }
     } else {
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 6cda742874ccf..cacf4e249aa28 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -813,6 +813,10 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
       if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(baseTy))
         baseTy = boxTy.getEleTy();
       baseTy = fir::unwrapRefType(baseTy);
+
+      if (mlir::isa<fir::SequenceType>(baseTy))
+        TODO(loc, "array of derived-type with device component");
+
       auto recTy =
           mlir::dyn_cast<fir::RecordType>(fir::unwrapSequenceType(baseTy));
       assert(recTy && "expected fir::RecordType");
@@ -823,7 +827,7 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
         if (Fortran::semantics::IsDeviceAllocatable(sym)) {
           unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
           mlir::Type fieldTy;
-          std::vector<mlir::Value> coordinates;
+          llvm::SmallVector<mlir::Value> coordinates;
 
           if (fieldIdx != std::numeric_limits<unsigned>::max()) {
             // Field found in the base record type.
@@ -866,8 +870,24 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
             TODO(loc, "device resident component in complex derived-type "
                       "hierarchy");
 
-          mlir::Value comp = builder.create<fir::CoordinateOp>(
-              loc, builder.getRefType(fieldTy), fir::getBase(exv), coordinates);
+          mlir::Value base = fir::getBase(exv);
+          mlir::Value comp;
+          if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(base.getType()))) {
+            mlir::Value box = builder.create<fir::LoadOp>(loc, base);
+            mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, box);
+            llvm::SmallVector<mlir::Value> lenParams;
+            assert(coordinates.size() == 1 && "expect one coordinate");
+            auto field = mlir::dyn_cast<fir::FieldIndexOp>(
+                coordinates[0].getDefiningOp());
+            comp = builder.create<hlfir::DesignateOp>(
+                loc, builder.getRefType(fieldTy), addr,
+                /*component=*/field.getFieldName(),
+                /*componentShape=*/mlir::Value{},
+                hlfir::DesignateOp::Subscripts{});
+          } else {
+            comp = builder.create<fir::CoordinateOp>(
+                loc, builder.getRefType(fieldTy), base, coordinates);
+          }
           cuf::DataAttributeAttr dataAttr =
               Fortran::lower::translateSymbolCUFDataAttribute(
                   builder.getContext(), sym);
diff --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index 2be5ef76e46b8..5f73335242336 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
   genUnreachable(builder, loc);
 }
 
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, std::nullopt);
+  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
 }
 
 void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index 389a89ddcf102..c7b3e11728cea 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -122,7 +122,7 @@ class VectorSubscriptBoxBuilder {
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
-        loc, fldTy, componentName, recTy, /*typeParams*/ std::nullopt));
+        loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
 
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index acd5a88a2582d..5b1dbc4435d6c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -620,7 +620,7 @@ fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc,
   mlir::NamedAttribute sizeAttr(sizeTag, getI64IntegerAttr(data.size()));
   llvm::SmallVector<mlir::NamedAttribute> attrs{dataAttr, sizeAttr};
   return create<fir::StringLitOp>(loc, llvm::ArrayRef<mlir::Type>{type},
-                                  std::nullopt, attrs);
+                                  mlir::ValueRange{}, attrs);
 }
 
 mlir::Value fir::FirOpBuilder::genShape(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 773d6408079cc..04703f7911176 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -59,7 +59,8 @@ mlir::Value fir::runtime::genCpuTime(fir::FirOpBuilder &builder,
                                      mlir::Location loc) {
   mlir::func::FuncOp func =
       fir::runtime::getRuntimeFunc<mkRTKey(CpuTime)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
@@ -280,7 +281,8 @@ void fir::runtime::genRename(fir::FirOpBuilder &builder, mlir::Location loc,
 mlir::Value fir::runtime::genTime(fir::FirOpBuilder &builder,
                                   mlir::Location loc) {
   auto func = fir::runtime::getRuntimeFunc<mkRTKey(time)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+      .getResult(0);
 }
 
 /// generate runtime call to transfer intrinsic with no size argument
diff --git a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
index 411181cc6dd1c..9b5e43b80b1f6 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
@@ -25,7 +25,7 @@ void fir::runtime::genExit(fir::FirOpBuilder &builder, mlir::Location loc,
 void fir::runtime::genAbort(fir::FirOpBuilder &builder, mlir::Location loc) {
   mlir::func::FuncOp abortFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(Abort)>(loc, builder);
-  builder.create<fir::CallOp>(loc, abortFunc, std::nullopt);
+  builder.create<fir::CallOp>(loc, abortFunc, mlir::ValueRange{});
 }
 
 void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 03cc92e975b19..c5cf01ed98357 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -405,7 +405,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
   mlir::Location loc = forallMaskOp.getLoc();
   mlir::Value mask = generateYieldedScalarValue(forallMaskOp.getMaskRegion(),
                                                 builder.getI1Type());
-  auto ifOp = builder.create<fir::IfOp>(loc, std::nullopt, mask, false);
+  auto ifOp = builder.create<fir::IfOp>(loc, mlir::TypeRange{}, mask, false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   constructStack.push_back(ifOp);
 }
@@ -530,7 +530,7 @@ void OrderedAssignmentRewriter::generateMaskIfOp(mlir::Value cdt) {
   mlir::Location loc = cdt.getLoc();
   cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
   cdt = builder.createConvert(loc, builder.getI1Type(), cdt);
-  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), std::nullopt, cdt,
+  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), mlir::TypeRange{}, cdt,
                                         /*withElseRegion=*/false);
   constructStack.push_back(ifOp.getOperation());
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
diff --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index 1f8edf851de9b..bc4fcd8b0112e 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -222,7 +222,7 @@ void AllocaReplaceImpl::genIndirectDeallocation(
         rewriter.create<fir::ConvertOp>(loc, intPtrTy, ptrVal);
     mlir::Value isAllocated = rewriter.create<mlir::arith::CmpIOp>(
         loc, mlir::arith::CmpIPredicate::ne, ptrToInt, c0);
-    auto ifOp = rewriter.create<fir::IfOp>(loc, std::nullopt, isAllocated,
+    auto ifOp = rewriter.create<fir::IfOp>(loc, mlir::TypeRange{}, isAllocated,
                                            /*withElseRegion=*/false);
     rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value cast = fir::factory::createConvert(
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 57eae1ff052a2..6e45aae4246d0 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -456,7 +456,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
   rewriter.setInsertionPointToEnd(thisBlock);
   if (destOps.has_value())
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, destOps.value(),
-                                            newBlock, std::nullopt);
+                                            newBlock, mlir::ValueRange{});
   else
     rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, newBlock);
   rewriter.setInsertionPointToEnd(newBlock);
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 506c8e66dbdfa..ad8464b495888 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -261,7 +261,7 @@ class DoConcurrentConversion
       innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
-          /*iterArgs=*/std::nullopt, loop.getReduceVars(),
+          /*iterArgs=*/mlir::ValueRange{}, loop.getReduceVars(),
           loop.getReduceAttrsAttr());
       ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
       rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
diff --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index ee89ea38a3fc7..e3bb181f65398 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -21,4 +21,36 @@ contains
 ! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]]#0, z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: cuf.set_allocator_idx %[[Z]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
 
+  subroutine sub2()
+    type(ty_device), pointer :: d1
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub2()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub2Ed1"} -> !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMm1Fsub2Ed1"} : (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
+  subroutine sub3()
+    type(ty_device), allocatable :: d1
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub3()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub3Ed1"} -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMm1Fsub3Ed1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
 end module
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index 98270adaa7c73..59808779aa6ef 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -48,7 +48,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
   mlir::Value addr = builder->create<fir::AllocaOp>(loc, eleType);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      /*shape=*/mlir::Value{}, /*typeParams=*/std::nullopt,
+      /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{},
       /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
@@ -102,11 +102,11 @@ TEST_F(FortranVariableTest, SimpleArray) {
       extents.size(), fir::SequenceType::getUnknownExtent());
   mlir::Type seqTy = fir::SequenceType::get(typeShape, eleType);
   mlir::Value addr = builder->create<fir::AllocaOp>(
-      loc, seqTy, /*pinned=*/false, /*typeParams=*/std::nullopt, extents);
+      loc, seqTy, /*pinned=*/false, /*typeParams=*/mlir::ValueRange{}, extents);
   mlir::Value shape = createShape(extents);
   auto name = mlir::StringAttr::get(&context, "x");
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
-      shape, /*typeParams*/ std::nullopt, /*dummy_scope=*/nullptr, name,
+      shape, /*typeParams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
       /*data_attr=*/cuf::DataAttributeAttr{});
 
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index e210992c5111a..3fb62788c1168 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -571,6 +571,8 @@ function(add_integration_test test_name)
   target_compile_options(${fq_build_target_name} PRIVATE
                          ${compile_options} ${INTEGRATION_TEST_COMPILE_OPTIONS})
 
+  set(compiler_runtime "")
+
   if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU)
     target_link_options(${fq_build_target_name} PRIVATE
       ${LIBC_COMPILE_OPTIONS_DEFAULT} ${INTEGRATION_TEST_COMPILE_OPTIONS}
@@ -599,17 +601,19 @@ function(add_integration_test test_name)
     set(link_options
       -nolibc
       -nostartfiles
-      -static
+      -nostdlib
       ${LIBC_LINK_OPTIONS_DEFAULT}
       ${LIBC_TEST_LINK_OPTIONS_DEFAULT}
     )
     target_link_options(${fq_build_target_name} PRIVATE ${link_options})
+    list(APPEND compiler_runtime ${LIBGCC_S_LOCATION})
   endif()
   target_link_libraries(
     ${fq_build_target_name}
-    ${fq_target_name}.__libc__
     libc.startup.${LIBC_TARGET_OS}.crt1
     libc.test.IntegrationTest.test
+    ${fq_target_name}.__libc__
+    ${compiler_runtime}
   )
   add_dependencies(${fq_build_target_name}
                    libc.test.IntegrationTest.test
@@ -770,6 +774,7 @@ function(add_libc_hermetic test_name)
                          ${HERMETIC_TEST_COMPILE_OPTIONS})
 
   set(link_libraries "")
+  set(compiler_runtime "")
   foreach(lib IN LISTS HERMETIC_TEST_LINK_LIBRARIES)
     if(TARGET ${lib}.hermetic)
       list(APPEND link_libraries ${lib}.hermetic)
@@ -807,12 +812,12 @@ function(add_libc_hermetic test_name)
     set(link_options
       -nolibc
       -nostartfiles
-      -static
+      -nostdlib
       ${LIBC_LINK_OPTIONS_DEFAULT}
       ${LIBC_TEST_LINK_OPTIONS_DEFAULT}
     )
     target_link_options(${fq_build_target_name} PRIVATE ${link_options})
-    list(APPEND link_libraries ${LIBGCC_S_LOCATION})
+    list(APPEND compiler_runtime ${LIBGCC_S_LOCATION})
   endif()
   target_link_libraries(
     ${fq_build_target_name}
@@ -820,7 +825,9 @@ function(add_libc_hermetic test_name)
       libc.startup.${LIBC_TARGET_OS}.crt1
       ${link_libraries}
       LibcHermeticTestSupport.hermetic
-      ${fq_target_name}.__libc__)
+      ${fq_target_name}.__libc__
+      ${compiler_runtime}
+    )
   add_dependencies(${fq_build_target_name}
                    LibcTest.hermetic
                    libc.test.UnitTest.ErrnoSetterMatcher
diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in
index fc01aaf2d8746..b68c0c8258366 100644
--- a/libcxx/include/__config_site.in
+++ b/libcxx/include/__config_site.in
@@ -30,7 +30,6 @@
 #cmakedefine01 _LIBCPP_HAS_LOCALIZATION
 #cmakedefine01 _LIBCPP_HAS_UNICODE
 #cmakedefine01 _LIBCPP_HAS_WIDE_CHARACTERS
-#cmakedefine _LIBCPP_HAS_NO_STD_MODULES
 #cmakedefine01 _LIBCPP_HAS_TIME_ZONE_DATABASE
 #cmakedefine01 _LIBCPP_INSTRUMENTED_WITH_ASAN
 
diff --git a/libcxx/test/libcxx/fuzzing/random.pass.cpp b/libcxx/test/libcxx/fuzzing/random.pass.cpp
index cb074bd60fdc8..f0256a01f29ae 100644
--- a/libcxx/test/libcxx/fuzzing/random.pass.cpp
+++ b/libcxx/test/libcxx/fuzzing/random.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // UNSUPPORTED: c++03, c++11
 
diff --git a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
index 1ba0063c1dada..f9f81d22ff80e 100644
--- a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // <math.h>
 
diff --git a/libcxx/test/std/numerics/c.math/cmath.pass.cpp b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
index 48c2918802fc3..8d261e9fcbdb2 100644
--- a/libcxx/test/std/numerics/c.math/cmath.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
 
 // <cmath>
 
diff --git a/lld/ELF/BPSectionOrderer.cpp b/lld/ELF/BPSectionOrderer.cpp
index f464b1d4518a4..06152046d13d4 100644
--- a/lld/ELF/BPSectionOrderer.cpp
+++ b/lld/ELF/BPSectionOrderer.cpp
@@ -76,10 +76,10 @@ DenseMap<const InputSectionBase *, int> elf::runBalancedPartitioning(
     if (!d)
       return;
     auto *sec = dyn_cast_or_null<InputSection>(d->section);
-    // Skip empty, discarded, ICF folded sections. Skipping ICF folded sections
-    // reduces duplicate detection work in BPSectionOrderer.
+    // Skip empty, discarded, ICF folded sections, .bss. Skipping ICF folded
+    // sections reduces duplicate detection work in BPSectionOrderer.
     if (!sec || sec->size == 0 || !sec->isLive() || sec->repl != sec ||
-        !orderer.secToSym.try_emplace(sec, d).second)
+        !sec->content().data() || !orderer.secToSym.try_emplace(sec, d).second)
       return;
     rootSymbolToSectionIdxs[CachedHashStringRef(
                                 lld::utils::getRootSymbol(sym.getName()))]
diff --git a/lld/test/ELF/bp-section-orderer.s b/lld/test/ELF/bp-section-orderer.s
index 4df2e8d43022e..438d7c2da0f76 100644
--- a/lld/test/ELF/bp-section-orderer.s
+++ b/lld/test/ELF/bp-section-orderer.s
@@ -26,28 +26,28 @@
 
 # RUN: ld.lld -o out.s a.o --irpgo-profile=a.profdata --bp-startup-sort=function
 # RUN: llvm-nm -jn out.s | tr '\n' , | FileCheck %s --check-prefix=STARTUP
-# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,{{$}}
+# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.os a.o --irpgo-profile=a.profdata --bp-startup-sort=function --symbol-ordering-file a.txt
 # RUN: llvm-nm -jn out.os | tr '\n' , | FileCheck %s --check-prefix=ORDER-STARTUP
-# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,{{$}}
+# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.cf a.o --verbose-bp-section-orderer --bp-compression-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-FUNC
 # RUN: ld.lld -o out.cf.icf a.o --verbose-bp-section-orderer --bp-compression-sort=function --icf=all --gc-sections 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-ICF-FUNC
 # RUN: llvm-nm -jn out.cf | tr '\n' , | FileCheck %s --check-prefix=CFUNC
-# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,{{$}}
+# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.cd a.o --verbose-bp-section-orderer --bp-compression-sort=data 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-DATA
 # RUN: llvm-nm -jn out.cd | tr '\n' , | FileCheck %s --check-prefix=CDATA
-# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,{{$}}
+# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
 
 # RUN: ld.lld -o out.cb a.o --verbose-bp-section-orderer --bp-compression-sort=both 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
 # RUN: llvm-nm -jn out.cb | tr '\n' , | FileCheck %s --check-prefix=CBOTH
-# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,{{$}}
+# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,g1,{{$}}
 
 # RUN: ld.lld -o out.cbs a.o --verbose-bp-section-orderer --bp-compression-sort=both --irpgo-profile=a.profdata --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
 # RUN: llvm-nm -jn out.cbs | tr '\n' , | FileCheck %s --check-prefix=CBOTH-STARTUP
-# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,{{$}}
+# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
 
 # BP-COMPRESSION-FUNC: Ordered 9 sections ([[#]] bytes) using balanced partitioning
 # BP-COMPRESSION-ICF-FUNC: Ordered 8 sections ([[#]] bytes) using balanced partitioning
@@ -108,6 +108,7 @@ d3
 d2
 
 #--- a.c
+int g1;
 const char s5[] = "engineering";
 const char s4[] = "computer program";
 const char s3[] = "hardware engineer";
@@ -377,6 +378,14 @@ d1:
 	.word	6                               // 0x6
 	.size	d1, 16
 
+	.type	g1, at object                      // @g1
+	.section	.bss.g1,"aw", at nobits
+	.globl	g1
+	.p2align	2, 0x0
+g1:
+	.word	0                               // 0x0
+	.size	g1, 4
+
 	.section	".note.GNU-stack","", at progbits
 	.addrsig
 	.addrsig_sym F
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index e445fa8833022..6f812b91a8b1d 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -700,7 +700,7 @@ void IRExecutionUnit::CollectCandidateCPlusPlusNames(
 
 class LoadAddressResolver {
 public:
-  LoadAddressResolver(Target *target, bool &symbol_was_missing_weak)
+  LoadAddressResolver(Target &target, bool &symbol_was_missing_weak)
       : m_target(target), m_symbol_was_missing_weak(symbol_was_missing_weak) {}
 
   std::optional<lldb::addr_t> Resolve(SymbolContextList &sc_list) {
@@ -722,11 +722,11 @@ class LoadAddressResolver {
 
       // First try the symbol.
       if (candidate_sc.symbol) {
-        load_address = candidate_sc.symbol->ResolveCallableAddress(*m_target);
+        load_address = candidate_sc.symbol->ResolveCallableAddress(m_target);
         if (load_address == LLDB_INVALID_ADDRESS) {
           Address addr = candidate_sc.symbol->GetAddress();
-          load_address = m_target->GetProcessSP()
-                             ? addr.GetLoadAddress(m_target)
+          load_address = m_target.GetProcessSP()
+                             ? addr.GetLoadAddress(&m_target)
                              : addr.GetFileAddress();
         }
       }
@@ -734,8 +734,8 @@ class LoadAddressResolver {
       // If that didn't work, try the function.
       if (load_address == LLDB_INVALID_ADDRESS && candidate_sc.function) {
         Address addr = candidate_sc.function->GetAddress();
-        load_address = m_target->GetProcessSP() ? addr.GetLoadAddress(m_target)
-                                                : addr.GetFileAddress();
+        load_address = m_target.GetProcessSP() ? addr.GetLoadAddress(&m_target)
+                                               : addr.GetFileAddress();
       }
 
       // We found a load address.
@@ -766,7 +766,7 @@ class LoadAddressResolver {
   }
 
 private:
-  Target *m_target;
+  Target &m_target;
   bool &m_symbol_was_missing_weak;
   lldb::addr_t m_best_internal_load_address = LLDB_INVALID_ADDRESS;
 };
@@ -790,7 +790,7 @@ IRExecutionUnit::FindInSymbols(const std::vector<ConstString> &names,
   for (size_t i = 0; i < m_preferred_modules.GetSize(); ++i)
     non_local_images.Remove(m_preferred_modules.GetModuleAtIndex(i));
 
-  LoadAddressResolver resolver(target, symbol_was_missing_weak);
+  LoadAddressResolver resolver(*target, symbol_was_missing_weak);
 
   ModuleFunctionSearchOptions function_options;
   function_options.include_symbols = true;
diff --git a/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
index e1d7e42bdd1a9..cd60227572be4 100644
--- a/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
+++ b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
@@ -2,7 +2,6 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -120,6 +119,8 @@ def cleanup():
                 '@"2 elements"',
             ],
         )
+
+        self.runCmd("settings set target.max-children-depth 6")
         self.expect(
             "frame variable mutabledict --ptr-depth 3",
             substrs=[
diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script b/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
index 91de55f4ade4a..43018eacf709b 100644
--- a/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
+++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
@@ -1,3 +1,4 @@
+settings set target.max-children-depth 10
 breakpoint set --file UdtLayoutTest.cpp --line 60
 run
 target variable
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index de5f66ce1584c..87d2a9ac3bf88 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -263,6 +263,20 @@ amaclean at nvidia.com (email), [AlexMaclean](https://github.com/AlexMaclean) (GitH
 
 #### PowerPC backend
 
+Amy Kwan (esp. release issues) \
+Amy.Kwan1 at ibm.com (email), [amy-kwan](https://github.com/amy-kwan) (GitHub) \
+Lei Huang \
+lei at ca.ibm.com (email), [lei137](https://github.com/lei137) (GitHub) \
+Sean Fertile (esp. ABI/ELF/XCOFF) \
+sfertile at ca.ibm.com (email), [mandlebug](https://github.com/mandlebug) (GitHub) \
+Zhijian Lin \
+zhijian at ca.ibm.com (email), [diggerlin](https://github.com/diggerlin) (GitHub) \
+Maryam Moghadas \
+maryammo at ca.ibm.com (email), [maryammo](https://github.com/maryammo) (GitHub) \
+Roland Froese \
+froese at ca.ibm.com (email), [RolandF77](https://github.com/RolandF77) (GitHub) \
+llvmonpower \
+powerllvm at ca.ibm.com (email), [llvmonpower](https://github.com/llvmonpower) (GitHub)
 
 #### RISCV backend
 
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index c614a6d7ace9e..732227b98ab9e 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -30,7 +30,7 @@ because the naming and other conventions are dictated by the C++ standard.
 
 There are some conventions that are not uniformly followed in the code base
 (e.g. the naming convention).  This is because they are relatively new, and a
-lot of code was written before they were put in place.  Our long term goal is
+lot of code was written before they were put in place.  Our long-term goal is
 for the entire codebase to follow the convention, but we explicitly *do not*
 want patches that do large-scale reformatting of existing code.  On the other
 hand, it is reasonable to rename the methods of a class if you're about to
@@ -50,7 +50,7 @@ code imported into the tree. Generally, our preference is for standards
 conforming, modern, and portable C++ code as the implementation language of
 choice.
 
-For automation, build-systems and utility scripts Python is preferred and
+For automation, build-systems, and utility scripts, Python is preferred and
 is widely used in the LLVM repository already.
 
 C++ Standard Versions
@@ -92,7 +92,7 @@ LLVM support libraries (for example, `ADT
 <https://github.com/llvm/llvm-project/tree/main/llvm/include/llvm/ADT>`_)
 implement specialized data structures or functionality missing in the standard
 library. Such libraries are usually implemented in the ``llvm`` namespace and
-follow the expected standard interface, when there is one.
+follow the expected standard interface when there is one.
 
 When both C++ and the LLVM support libraries provide similar functionality, and
 there isn't a specific reason to favor the C++ implementation, it is generally
@@ -325,8 +325,8 @@ implementation file.  In any case, implementation files can include additional
 comments (not necessarily in Doxygen markup) to explain implementation details
 as needed.
 
-Don't duplicate function or class name at the beginning of the comment.
-For humans it is obvious which function or class is being documented;
+Don't duplicate the function or class name at the beginning of the comment.
+For humans, it is obvious which function or class is being documented;
 automatic documentation processing tools are smart enough to bind the comment
 to the correct declaration.
 
@@ -369,7 +369,7 @@ lower-case letter, and finish the last sentence without a period, if it would
 end in one otherwise. Sentences which end with different punctuation, such as
 "did you forget ';'?", should still do so.
 
-For example this is a good error message:
+For example, this is a good error message:
 
 .. code-block:: none
 
@@ -443,7 +443,7 @@ Write your code to fit within 80 columns.
 There must be some limit to the width of the code in
 order to allow developers to have multiple files side-by-side in
 windows on a modest display.  If you are going to pick a width limit, it is
-somewhat arbitrary but you might as well pick something standard.  Going with 90
+somewhat arbitrary, but you might as well pick something standard.  Going with 90
 columns (for example) instead of 80 columns wouldn't add any significant value
 and would be detrimental to printing out code.  Also many other projects have
 standardized on 80 columns, so some people have already configured their editors
@@ -520,7 +520,7 @@ within each other and within function calls in order to build up aggregates
 The historically common formatting of braced initialization of aggregate
 variables does not mix cleanly with deep nesting, general expression contexts,
 function arguments, and lambdas. We suggest new code use a simple rule for
-formatting braced initialization lists: act as-if the braces were parentheses
+formatting braced initialization lists: act as if the braces were parentheses
 in a function call. The formatting rules exactly match those already well
 understood for formatting nested function calls. Examples:
 
@@ -607,11 +607,11 @@ Static constructors and destructors (e.g., global variables whose types have a
 constructor or destructor) should not be added to the code base, and should be
 removed wherever possible.
 
-Globals in different source files are initialized in `arbitrary order
+Globals in different source files are initialized in an `arbitrary order
 <https://yosefk.com/c++fqa/ctors.html#fqa-10.12>`_, making the code more
 difficult to reason about.
 
-Static constructors have negative impact on launch time of programs that use
+Static constructors have a negative impact on the launch time of programs that use
 LLVM as a library. We would really like for there to be zero cost for linking
 in an additional LLVM target or other library into an application, but static
 constructors undermine this goal.
@@ -698,7 +698,7 @@ If you use a braced initializer list when initializing a variable, use an equals
 Use ``auto`` Type Deduction to Make Code More Readable
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Some are advocating a policy of "almost always ``auto``" in C++11, however LLVM
+Some are advocating a policy of "almost always ``auto``" in C++11; however, LLVM
 uses a more moderate stance. Use ``auto`` if and only if it makes the code more
 readable or easier to maintain. Don't "almost always" use ``auto``, but do use
 ``auto`` with initializers like ``cast<Foo>(...)`` or other places where the
@@ -783,14 +783,14 @@ guards, and might not include their prerequisites. Name such files with the
 
 In general, a header should be implemented by one or more ``.cpp`` files.  Each
 of these ``.cpp`` files should include the header that defines their interface
-first.  This ensures that all of the dependences of the header have been
+first.  This ensures that all of the dependencies of the header have been
 properly added to the header itself, and are not implicit.  System headers
 should be included after user headers for a translation unit.
 
 Library Layering
 ^^^^^^^^^^^^^^^^
 
-A directory of header files (for example ``include/llvm/Foo``) defines a
+A directory of header files (for example, ``include/llvm/Foo``) defines a
 library (``Foo``). One library (both
 its headers and implementation) should only use things from the libraries
 listed in its dependencies.
@@ -822,7 +822,7 @@ especially in header files.
 
 But wait! Sometimes you need to have the definition of a class to use it, or to
 inherit from it.  In these cases go ahead and ``#include`` that header file.  Be
-aware however that there are many cases where you don't need to have the full
+aware, however, that there are many cases where you don't need to have the full
 definition of a class.  If you are using a pointer or reference to a class, you
 don't need the header file.  If you are simply returning a class instance from a
 prototyped function or method, you don't need it.  In fact, for most cases, you
@@ -970,7 +970,7 @@ loops.  A silly example is something like this:
 When you have very, very small loops, this sort of structure is fine. But if it
 exceeds more than 10-15 lines, it becomes difficult for people to read and
 understand at a glance. The problem with this sort of code is that it gets very
-nested very quickly. Meaning that the reader of the code has to keep a lot of
+nested very quickly. This means that the reader of the code has to keep a lot of
 context in their brain to remember what is going immediately on in the loop,
 because they don't know if/when the ``if`` conditions will have ``else``\s etc.
 It is strongly preferred to structure the loop like this:
@@ -988,7 +988,7 @@ It is strongly preferred to structure the loop like this:
     ...
   }
 
-This has all the benefits of using early exits for functions: it reduces nesting
+This has all the benefits of using early exits for functions: it reduces the nesting
 of the loop, it makes it easier to describe why the conditions are true, and it
 makes it obvious to the reader that there is no ``else`` coming up that they
 have to push context into their brain for.  If a loop is large, this can be a
@@ -1149,12 +1149,12 @@ In general, names should be in camel case (e.g. ``TextFileReader`` and
   nouns and start with an upper-case letter (e.g. ``TextFileReader``).
 
 * **Variable names** should be nouns (as they represent state).  The name should
-  be camel case, and start with an upper case letter (e.g. ``Leader`` or
+  be camel case, and start with an upper-case letter (e.g. ``Leader`` or
   ``Boats``).
 
 * **Function names** should be verb phrases (as they represent actions), and
   command-like function should be imperative.  The name should be camel case,
-  and start with a lower case letter (e.g. ``openFile()`` or ``isFoo()``).
+  and start with a lower-case letter (e.g. ``openFile()`` or ``isFoo()``).
 
 * **Enum declarations** (e.g. ``enum Foo {...}``) are types, so they should
   follow the naming conventions for types.  A common use for enums is as a
@@ -1207,7 +1207,7 @@ Assert Liberally
 ^^^^^^^^^^^^^^^^
 
 Use the "``assert``" macro to its fullest.  Check all of your preconditions and
-assumptions, you never know when a bug (not necessarily even yours) might be
+assumptions.  You never know when a bug (not necessarily even yours) might be
 caught early by an assertion, which reduces debugging time dramatically.  The
 "``<cassert>``" header file is probably already included by the header files you
 are using, so it doesn't cost anything to use it.
@@ -1302,7 +1302,7 @@ preferred to write the code like this:
   assert(NewToSet && "The value shouldn't be in the set yet");
 
 In C code where ``[[maybe_unused]]`` is not supported, use ``void`` cast to
-suppress unused variable warning as follows:
+suppress an unused variable warning as follows:
 
 .. code-block:: c
 
@@ -1546,7 +1546,7 @@ whenever possible.
 The semantics of postincrement include making a copy of the value being
 incremented, returning it, and then preincrementing the "work value".  For
 primitive types, this isn't a big deal. But for iterators, it can be a huge
-issue (for example, some iterators contains stack and set objects in them...
+issue (for example, some iterators contain stack and set objects in them...
 copying an iterator could invoke the copy ctor's of these as well).  In general,
 get in the habit of always using preincrement, and you won't have a problem.
 
@@ -1663,7 +1663,7 @@ Don't Use Braces on Simple Single-Statement Bodies of if/else/loop Statements
 
 When writing the body of an ``if``, ``else``, or for/while loop statement, we
 prefer to omit the braces to avoid unnecessary line noise. However, braces
-should be used in cases where the omission of braces harm the readability and
+should be used in cases where the omission of braces harms the readability and
 maintainability of the code.
 
 We consider that readability is harmed when omitting the brace in the presence
@@ -1763,7 +1763,7 @@ would help to avoid running into a "dangling else" situation.
         handleAttrOnDecl(D, A, i);
   }
 
-  // Use braces on the outer block because of a nested `if`; otherwise the
+  // Use braces on the outer block because of a nested `if`; otherwise, the
   // compiler would warn: `add explicit braces to avoid dangling else`
   if (auto *D = dyn_cast<FunctionDecl>(D)) {
     if (shouldProcess(D))
diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index aef91909dd17b..794075201d646 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -39,8 +39,8 @@ class MachineBlockFrequencyInfo {
 public:
   LLVM_ABI MachineBlockFrequencyInfo(); // Legacy pass manager only.
   LLVM_ABI explicit MachineBlockFrequencyInfo(
-      MachineFunction &F, MachineBranchProbabilityInfo &MBPI,
-      MachineLoopInfo &MLI);
+      const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
+      const MachineLoopInfo &MLI);
   LLVM_ABI MachineBlockFrequencyInfo(MachineBlockFrequencyInfo &&);
   LLVM_ABI ~MachineBlockFrequencyInfo();
 
diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h
index 77cee875f16e7..f8241a3cdf160 100644
--- a/llvm/include/llvm/IR/DebugInfo.h
+++ b/llvm/include/llvm/IR/DebugInfo.h
@@ -115,8 +115,7 @@ class DebugInfoFinder {
   LLVM_ABI void processVariable(DILocalVariable *DVI);
   /// Process debug info location.
   LLVM_ABI void processLocation(const Module &M, const DILocation *Loc);
-  /// Process a DbgRecord (e.g, treat a DbgVariableRecord like a
-  /// DbgVariableIntrinsic).
+  /// Process a DbgRecord.
   LLVM_ABI void processDbgRecord(const Module &M, const DbgRecord &DR);
 
   /// Process subprogram.
@@ -290,8 +289,6 @@ struct VarRecord {
   DILocalVariable *Var;
   DILocation *DL;
 
-  VarRecord(DbgVariableIntrinsic *DVI)
-      : Var(DVI->getVariable()), DL(getDebugValueLoc(DVI)) {}
   VarRecord(DbgVariableRecord *DVR)
       : Var(DVR->getVariable()), DL(getDebugValueLoc(DVR)) {}
   VarRecord(DILocalVariable *Var, DILocation *DL) : Var(Var), DL(DL) {}
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 9345f95015301..f1f0c18949c35 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -66,7 +66,6 @@ namespace dwarf {
 enum Tag : uint16_t;
 }
 
-class DbgVariableIntrinsic;
 class DbgVariableRecord;
 
 LLVM_ABI extern cl::opt<bool> EnableFSDiscriminator;
@@ -4613,7 +4612,6 @@ class DebugVariable {
   LLVM_ABI static const FragmentInfo DefaultFragment;
 
 public:
-  LLVM_ABI DebugVariable(const DbgVariableIntrinsic *DII);
   LLVM_ABI DebugVariable(const DbgVariableRecord *DVR);
 
   DebugVariable(const DILocalVariable *Var,
@@ -4681,7 +4679,6 @@ template <> struct DenseMapInfo<DebugVariable> {
 /// information).
 class DebugVariableAggregate : public DebugVariable {
 public:
-  LLVM_ABI DebugVariableAggregate(const DbgVariableIntrinsic *DVI);
   DebugVariableAggregate(const DebugVariable &V)
       : DebugVariable(V.getVariable(), std::nullopt, V.getInlinedAt()) {}
 };
diff --git a/llvm/include/llvm/Transforms/Utils/SSAUpdater.h b/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
index 4e5da81a7e885..9500b1f160ea9 100644
--- a/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
+++ b/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
@@ -29,7 +29,6 @@ template <typename T> class SSAUpdaterTraits;
 class Type;
 class Use;
 class Value;
-class DbgValueInst;
 
 /// Helper class for SSA formation on a set of values defined in
 /// multiple blocks.
@@ -122,8 +121,6 @@ class SSAUpdater {
   /// the instruction. Anything outside of its block will have its
   /// value set to the new SSA value if available, and undef if not.
   void UpdateDebugValues(Instruction *I);
-  void UpdateDebugValues(Instruction *I,
-                         SmallVectorImpl<DbgValueInst *> &DbgValues);
   void UpdateDebugValues(Instruction *I,
                          SmallVectorImpl<DbgVariableRecord *> &DbgValues);
 
@@ -136,7 +133,6 @@ class SSAUpdater {
 
 private:
   Value *GetValueAtEndOfBlockInternal(BasicBlock *BB);
-  void UpdateDebugValue(Instruction *I, DbgValueInst *DbgValue);
   void UpdateDebugValue(Instruction *I, DbgVariableRecord *DbgValue);
 };
 
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index d9d41f1d72e35..dc8184394f74d 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -445,7 +445,6 @@ class CodeGenPrepare {
   bool optimizeSwitchInst(SwitchInst *SI);
   bool optimizeExtractElementInst(Instruction *Inst);
   bool dupRetToEnableTailCallOpts(BasicBlock *BB, ModifyDT &ModifiedDT);
-  bool fixupDbgValue(Instruction *I);
   bool fixupDbgVariableRecord(DbgVariableRecord &I);
   bool fixupDbgVariableRecordsOnInst(Instruction &I);
   bool placeDbgValues(Function &F);
@@ -2762,9 +2761,6 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
     case Intrinsic::fshl:
     case Intrinsic::fshr:
       return optimizeFunnelShift(II);
-    case Intrinsic::dbg_assign:
-    case Intrinsic::dbg_value:
-      return fixupDbgValue(II);
     case Intrinsic::masked_gather:
       return optimizeGatherScatterInst(II, II->getArgOperand(0));
     case Intrinsic::masked_scatter:
@@ -3554,8 +3550,6 @@ class TypePromotionTransaction {
     /// Keep track of the original uses (pair Instruction, Index).
     SmallVector<InstructionAndIdx, 4> OriginalUses;
     /// Keep track of the debug users.
-    SmallVector<DbgValueInst *, 1> DbgValues;
-    /// And non-instruction debug-users too.
     SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
 
     /// Keep track of the new value so that we can undo it by replacing
@@ -3577,7 +3571,9 @@ class TypePromotionTransaction {
       }
       // Record the debug uses separately. They are not in the instruction's
       // use list, but they are replaced by RAUW.
+      SmallVector<DbgValueInst *> DbgValues;
       findDbgValues(DbgValues, Inst, &DbgVariableRecords);
+      assert(DbgValues.empty());
 
       // Now, we can replace the uses.
       Inst->replaceAllUsesWith(New);
@@ -3591,11 +3587,7 @@ class TypePromotionTransaction {
       // RAUW has replaced all original uses with references to the new value,
       // including the debug uses. Since we are undoing the replacements,
       // the original debug uses must also be reinstated to maintain the
-      // correctness and utility of debug value instructions.
-      for (auto *DVI : DbgValues)
-        DVI->replaceVariableLocationOp(New, Inst);
-      // Similar story with DbgVariableRecords, the non-instruction
-      // representation of dbg.values.
+      // correctness and utility of debug value records.
       for (DbgVariableRecord *DVR : DbgVariableRecords)
         DVR->replaceVariableLocationOp(New, Inst);
     }
@@ -8933,32 +8925,6 @@ bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT) {
   return MadeChange;
 }
 
-// Some CGP optimizations may move or alter what's computed in a block. Check
-// whether a dbg.value intrinsic could be pointed at a more appropriate operand.
-bool CodeGenPrepare::fixupDbgValue(Instruction *I) {
-  assert(isa<DbgValueInst>(I));
-  DbgValueInst &DVI = *cast<DbgValueInst>(I);
-
-  // Does this dbg.value refer to a sunk address calculation?
-  bool AnyChange = false;
-  SmallDenseSet<Value *> LocationOps(DVI.location_ops().begin(),
-                                     DVI.location_ops().end());
-  for (Value *Location : LocationOps) {
-    WeakTrackingVH SunkAddrVH = SunkAddrs[Location];
-    Value *SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;
-    if (SunkAddr) {
-      // Point dbg.value at locally computed address, which should give the best
-      // opportunity to be accurately lowered. This update may change the type
-      // of pointer being referred to; however this makes no difference to
-      // debugging information, and we can't generate bitcasts that may affect
-      // codegen.
-      DVI.replaceVariableLocationOp(Location, SunkAddr);
-      AnyChange = true;
-    }
-  }
-  return AnyChange;
-}
-
 bool CodeGenPrepare::fixupDbgVariableRecordsOnInst(Instruction &I) {
   bool AnyChange = false;
   for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
@@ -8993,14 +8959,6 @@ bool CodeGenPrepare::fixupDbgVariableRecord(DbgVariableRecord &DVR) {
   return AnyChange;
 }
 
-static void DbgInserterHelper(DbgValueInst *DVI, BasicBlock::iterator VI) {
-  DVI->removeFromParent();
-  if (isa<PHINode>(VI))
-    DVI->insertBefore(VI->getParent()->getFirstInsertionPt());
-  else
-    DVI->insertAfter(VI);
-}
-
 static void DbgInserterHelper(DbgVariableRecord *DVR, BasicBlock::iterator VI) {
   DVR->removeFromParent();
   BasicBlock *VIBB = VI->getParent();
@@ -9065,15 +9023,8 @@ bool CodeGenPrepare::placeDbgValues(Function &F) {
 
   for (BasicBlock &BB : F) {
     for (Instruction &Insn : llvm::make_early_inc_range(BB)) {
-      // Process dbg.value intrinsics.
-      DbgValueInst *DVI = dyn_cast<DbgValueInst>(&Insn);
-      if (DVI) {
-        DbgProcessor(DVI, DVI);
-        continue;
-      }
-
-      // If this isn't a dbg.value, process any attached DbgVariableRecord
-      // records attached to this instruction.
+      // Process any DbgVariableRecord records attached to this
+      // instruction.
       for (DbgVariableRecord &DVR : llvm::make_early_inc_range(
                filterDbgVars(Insn.getDbgRecordRange()))) {
         if (DVR.Type != DbgVariableRecord::LocationType::Value)
diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index 9daacfd399787..e7fa0824fd98a 100644
--- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -202,8 +202,8 @@ MachineBlockFrequencyInfo::MachineBlockFrequencyInfo(
     MachineBlockFrequencyInfo &&) = default;
 
 MachineBlockFrequencyInfo::MachineBlockFrequencyInfo(
-    MachineFunction &F, MachineBranchProbabilityInfo &MBPI,
-    MachineLoopInfo &MLI) {
+    const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
+    const MachineLoopInfo &MLI) {
   calculate(F, MBPI, MLI);
 }
 
diff --git a/llvm/lib/CodeGen/MachineDebugify.cpp b/llvm/lib/CodeGen/MachineDebugify.cpp
index 9b9cebc74054d..1a20fe586e951 100644
--- a/llvm/lib/CodeGen/MachineDebugify.cpp
+++ b/llvm/lib/CodeGen/MachineDebugify.cpp
@@ -63,24 +63,9 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
   // which cover a wide range of lines can help stress the debug info passes:
   // if we can't do that, fall back to using the local variable which precedes
   // all the others.
-  Function *DbgValF = M.getFunction("llvm.dbg.value");
-  DbgValueInst *EarliestDVI = nullptr;
   DbgVariableRecord *EarliestDVR = nullptr;
   DenseMap<unsigned, DILocalVariable *> Line2Var;
   DIExpression *Expr = nullptr;
-  if (DbgValF) {
-    for (const Use &U : DbgValF->uses()) {
-      auto *DVI = dyn_cast<DbgValueInst>(U.getUser());
-      if (!DVI || DVI->getFunction() != &F)
-        continue;
-      unsigned Line = DVI->getDebugLoc().getLine();
-      assert(Line != 0 && "debugify should not insert line 0 locations");
-      Line2Var[Line] = DVI->getVariable();
-      if (!EarliestDVI || Line < EarliestDVI->getDebugLoc().getLine())
-        EarliestDVI = DVI;
-      Expr = DVI->getExpression();
-    }
-  }
   for (BasicBlock &BB : F) {
     for (Instruction &I : BB) {
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
@@ -125,8 +110,7 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
       unsigned Line = MI.getDebugLoc().getLine();
       auto It = Line2Var.find(Line);
       if (It == Line2Var.end()) {
-        Line = EarliestDVI ? EarliestDVI->getDebugLoc().getLine()
-                           : EarliestDVR->getDebugLoc().getLine();
+        Line = EarliestDVR->getDebugLoc().getLine();
         It = Line2Var.find(Line);
         assert(It != Line2Var.end());
       }
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 40464e91f9efc..b7e41427f2c27 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7641,7 +7641,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
 
     if (SDValue(GN0, 0).hasOneUse() &&
         isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
-        TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+        TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
       SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
                        GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
 
@@ -15745,7 +15745,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   // fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
   if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
     if (SDValue(GN0, 0).hasOneUse() && ExtVT == GN0->getMemoryVT() &&
-        TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+        TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
       SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
                        GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
 
@@ -18727,6 +18727,12 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
   if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
     return FTrunc;
 
+  // fold (sint_to_fp (trunc nsw x)) -> (sint_to_fp x)
+  if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoSignedWrap() &&
+      TLI.isTypeDesirableForOp(ISD::SINT_TO_FP,
+                               N0.getOperand(0).getValueType()))
+    return DAG.getNode(ISD::SINT_TO_FP, DL, VT, N0.getOperand(0));
+
   return SDValue();
 }
 
@@ -18764,6 +18770,12 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
   if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
     return FTrunc;
 
+  // fold (uint_to_fp (trunc nuw x)) -> (uint_to_fp x)
+  if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoUnsignedWrap() &&
+      TLI.isTypeDesirableForOp(ISD::UINT_TO_FP,
+                               N0.getOperand(0).getValueType()))
+    return DAG.getNode(ISD::UINT_TO_FP, DL, VT, N0.getOperand(0));
+
   return SDValue();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 56c8bb441ddf8..5453828177c72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6751,7 +6751,9 @@ SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
     return SDValue();
   int64_t Offset = C2->getSExtValue();
   switch (Opcode) {
-  case ISD::ADD: break;
+  case ISD::ADD:
+  case ISD::PTRADD:
+    break;
   case ISD::SUB: Offset = -uint64_t(Offset); break;
   default: return SDValue();
   }
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 84a56058de834..8fb33c30e5cac 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -2288,39 +2288,36 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
   // Collect a map of {backing storage : dbg.declares} (currently "backing
   // storage" is limited to Allocas). We'll use this to find dbg.declares to
   // delete after running `trackAssignments`.
-  DenseMap<const AllocaInst *, SmallPtrSet<DbgDeclareInst *, 2>> DbgDeclares;
   DenseMap<const AllocaInst *, SmallPtrSet<DbgVariableRecord *, 2>> DVRDeclares;
   // Create another similar map of {storage : variables} that we'll pass to
   // trackAssignments.
   StorageToVarsMap Vars;
-  auto ProcessDeclare = [&](auto *Declare, auto &DeclareList) {
+  auto ProcessDeclare = [&](DbgVariableRecord &Declare) {
     // FIXME: trackAssignments doesn't let you specify any modifiers to the
     // variable (e.g. fragment) or location (e.g. offset), so we have to
     // leave dbg.declares with non-empty expressions in place.
-    if (Declare->getExpression()->getNumElements() != 0)
+    if (Declare.getExpression()->getNumElements() != 0)
       return;
-    if (!Declare->getAddress())
+    if (!Declare.getAddress())
       return;
     if (AllocaInst *Alloca =
-            dyn_cast<AllocaInst>(Declare->getAddress()->stripPointerCasts())) {
+            dyn_cast<AllocaInst>(Declare.getAddress()->stripPointerCasts())) {
       // FIXME: Skip VLAs for now (let these variables use dbg.declares).
       if (!Alloca->isStaticAlloca())
         return;
       // Similarly, skip scalable vectors (use dbg.declares instead).
       if (auto Sz = Alloca->getAllocationSize(*DL); Sz && Sz->isScalable())
         return;
-      DeclareList[Alloca].insert(Declare);
-      Vars[Alloca].insert(VarRecord(Declare));
+      DVRDeclares[Alloca].insert(&Declare);
+      Vars[Alloca].insert(VarRecord(&Declare));
     }
   };
   for (auto &BB : F) {
     for (auto &I : BB) {
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
         if (DVR.isDbgDeclare())
-          ProcessDeclare(&DVR, DVRDeclares);
+          ProcessDeclare(DVR);
       }
-      if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(&I))
-        ProcessDeclare(DDI, DbgDeclares);
     }
   }
 
@@ -2336,8 +2333,8 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
   trackAssignments(F.begin(), F.end(), Vars, *DL);
 
   // Delete dbg.declares for variables now tracked with assignment tracking.
-  auto DeleteSubsumedDeclare = [&](const auto &Markers, auto &Declares) {
-    (void)Markers;
+  for (auto &[Insts, Declares] : DVRDeclares) {
+    auto Markers = at::getDVRAssignmentMarkers(Insts);
     for (auto *Declare : Declares) {
       // Assert that the alloca that Declare uses is now linked to a dbg.assign
       // describing the same variable (i.e. check that this dbg.declare has
@@ -2356,10 +2353,6 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
       Changed = true;
     }
   };
-  for (auto &P : DbgDeclares)
-    DeleteSubsumedDeclare(at::getAssignmentMarkers(P.first), P.second);
-  for (auto &P : DVRDeclares)
-    DeleteSubsumedDeclare(at::getDVRAssignmentMarkers(P.first), P.second);
   return Changed;
 }
 
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 2270923bd3719..f16963dce56e1 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -49,20 +49,11 @@ uint32_t DIType::getAlignInBits() const {
 const DIExpression::FragmentInfo DebugVariable::DefaultFragment = {
     std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::min()};
 
-DebugVariable::DebugVariable(const DbgVariableIntrinsic *DII)
-    : Variable(DII->getVariable()),
-      Fragment(DII->getExpression()->getFragmentInfo()),
-      InlinedAt(DII->getDebugLoc().getInlinedAt()) {}
-
 DebugVariable::DebugVariable(const DbgVariableRecord *DVR)
     : Variable(DVR->getVariable()),
       Fragment(DVR->getExpression()->getFragmentInfo()),
       InlinedAt(DVR->getDebugLoc().getInlinedAt()) {}
 
-DebugVariableAggregate::DebugVariableAggregate(const DbgVariableIntrinsic *DVI)
-    : DebugVariable(DVI->getVariable(), std::nullopt,
-                    DVI->getDebugLoc()->getInlinedAt()) {}
-
 DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
                        unsigned Column, uint64_t AtomGroup, uint8_t AtomRank,
                        ArrayRef<Metadata *> MDs, bool ImplicitCode)
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 79eeb08cddeef..2579fa37935f0 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -54,11 +54,6 @@ static const char *PreservedSymbols[] = {
     "__stack_chk_guard",
 };
 
-static bool isPreservedGlobalVarName(StringRef Name) {
-  return StringRef(PreservedSymbols[0]) == Name ||
-         StringRef(PreservedSymbols[1]) == Name;
-}
-
 namespace {
 
 const char *getExpectedProducerName() {
@@ -86,16 +81,12 @@ struct Builder {
   // The StringTableBuilder does not create a copy of any strings added to it,
   // so this provides somewhere to store any strings that we create.
   Builder(SmallVector<char, 0> &Symtab, StringTableBuilder &StrtabBuilder,
-          BumpPtrAllocator &Alloc, const Triple &TT)
-      : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc), TT(TT),
-        Libcalls(TT) {}
+          BumpPtrAllocator &Alloc)
+      : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc) {}
 
   DenseMap<const Comdat *, int> ComdatMap;
   Mangler Mang;
-  const Triple &TT;
-
-  // FIXME: This shouldn't be here.
-  RTLIB::RuntimeLibcallsInfo Libcalls;
+  Triple TT;
 
   std::vector<storage::Comdat> Comdats;
   std::vector<storage::Module> Mods;
@@ -107,10 +98,6 @@ struct Builder {
 
   std::vector<storage::Str> DependentLibraries;
 
-  bool isPreservedLibFuncName(StringRef Name) {
-    return Libcalls.getSupportedLibcallImpl(Name) != RTLIB::Unsupported;
-  }
-
   void setStr(storage::Str &S, StringRef Value) {
     S.Offset = StrtabBuilder.add(Value);
     S.Size = Value.size();
@@ -226,6 +213,18 @@ Expected<int> Builder::getComdatIndex(const Comdat *C, const Module *M) {
   return P.first->second;
 }
 
+static DenseSet<StringRef> buildPreservedSymbolsSet(const Triple &TT) {
+  DenseSet<StringRef> PreservedSymbolSet(std::begin(PreservedSymbols),
+                                         std::end(PreservedSymbols));
+  // FIXME: Do we need to pass in ABI fields from TargetOptions?
+  RTLIB::RuntimeLibcallsInfo Libcalls(TT);
+  for (RTLIB::LibcallImpl Impl : Libcalls.getLibcallImpls()) {
+    if (Impl != RTLIB::Unsupported)
+      PreservedSymbolSet.insert(Libcalls.getLibcallImplName(Impl));
+  }
+  return PreservedSymbolSet;
+}
+
 Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
                          const SmallPtrSet<GlobalValue *, 4> &Used,
                          ModuleSymbolTable::Symbol Msym) {
@@ -279,11 +278,13 @@ Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
     return Error::success();
   }
 
-  StringRef GVName = GV->getName();
-  setStr(Sym.IRName, GVName);
+  setStr(Sym.IRName, GV->getName());
+
+  static const DenseSet<StringRef> PreservedSymbolsSet =
+      buildPreservedSymbolsSet(GV->getParent()->getTargetTriple());
+  bool IsPreservedSymbol = PreservedSymbolsSet.contains(GV->getName());
 
-  if (Used.count(GV) || isPreservedLibFuncName(GVName) ||
-      isPreservedGlobalVarName(GVName))
+  if (Used.count(GV) || IsPreservedSymbol)
     Sym.Flags |= 1 << storage::Symbol::FB_used;
   if (GV->isThreadLocal())
     Sym.Flags |= 1 << storage::Symbol::FB_tls;
@@ -350,6 +351,7 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
   setStr(Hdr.Producer, kExpectedProducerName);
   setStr(Hdr.TargetTriple, IRMods[0]->getTargetTriple().str());
   setStr(Hdr.SourceFileName, IRMods[0]->getSourceFileName());
+  TT = IRMods[0]->getTargetTriple();
 
   for (auto *M : IRMods)
     if (Error Err = addModule(M))
@@ -375,8 +377,7 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
 Error irsymtab::build(ArrayRef<Module *> Mods, SmallVector<char, 0> &Symtab,
                       StringTableBuilder &StrtabBuilder,
                       BumpPtrAllocator &Alloc) {
-  const Triple &TT = Mods[0]->getTargetTriple();
-  return Builder(Symtab, StrtabBuilder, Alloc, TT).build(Mods);
+  return Builder(Symtab, StrtabBuilder, Alloc).build(Mods);
 }
 
 // Upgrade a vector of bitcode modules created by an old version of LLVM by
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d04e6c45e2103..f026726c3f484 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -6439,7 +6439,9 @@ bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
     }
   }
 
-  return true;
+  EVT PreExtScalarVT = ExtVal->getOperand(0).getValueType().getScalarType();
+  return PreExtScalarVT == MVT::i8 || PreExtScalarVT == MVT::i16 ||
+         PreExtScalarVT == MVT::i32 || PreExtScalarVT == MVT::i64;
 }
 
 unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 23f106a9c1d4d..007b481f84960 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -153,6 +153,9 @@ struct AMDGPULowerBufferFatPointersPass
   const TargetMachine &TM;
 };
 
+void initializeAMDGPUPrepareAGPRAllocLegacyPass(PassRegistry &);
+extern char &AMDGPUPrepareAGPRAllocLegacyID;
+
 void initializeAMDGPUReserveWWMRegsLegacyPass(PassRegistry &);
 extern char &AMDGPUReserveWWMRegsLegacyID;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 250547acb1ee7..b6c6d927d0e89 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -114,6 +114,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUse
 MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
 MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
 MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
+MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass())
 MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
 MACHINE_FUNCTION_PASS("amdgpu-wait-sgpr-hazards", AMDGPUWaitSGPRHazardsPass())
 MACHINE_FUNCTION_PASS("gcn-create-vopd", GCNCreateVOPDPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
new file mode 100644
index 0000000000000..3b06e9b00ac69
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
@@ -0,0 +1,108 @@
+//===-- AMDGPUPrepareAGPRAlloc.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Make simple transformations to relax register constraints for cases which can
+// allocate to AGPRs or VGPRs. Replace materialize of inline immediates into
+// AGPR or VGPR with a pseudo with an AV_* class register constraint. This
+// allows later passes to inflate the register class if necessary. The register
+// allocator does not know to replace instructions to relax constraints.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUPrepareAGPRAlloc.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-prepare-agpr-alloc"
+
+namespace {
+
+class AMDGPUPrepareAGPRAllocImpl {
+private:
+  const SIInstrInfo &TII;
+  MachineRegisterInfo &MRI;
+
+public:
+  AMDGPUPrepareAGPRAllocImpl(const GCNSubtarget &ST, MachineRegisterInfo &MRI)
+      : TII(*ST.getInstrInfo()), MRI(MRI) {}
+  bool run(MachineFunction &MF);
+};
+
+class AMDGPUPrepareAGPRAllocLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  AMDGPUPrepareAGPRAllocLegacy() : MachineFunctionPass(ID) {
+    initializeAMDGPUPrepareAGPRAllocLegacyPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override { return "AMDGPU Prepare AGPR Alloc"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(AMDGPUPrepareAGPRAllocLegacy, DEBUG_TYPE,
+                      "AMDGPU Prepare AGPR Alloc", false, false)
+INITIALIZE_PASS_END(AMDGPUPrepareAGPRAllocLegacy, DEBUG_TYPE,
+                    "AMDGPU Prepare AGPR Alloc", false, false)
+
+char AMDGPUPrepareAGPRAllocLegacy::ID = 0;
+
+char &llvm::AMDGPUPrepareAGPRAllocLegacyID = AMDGPUPrepareAGPRAllocLegacy::ID;
+
+bool AMDGPUPrepareAGPRAllocLegacy::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  return AMDGPUPrepareAGPRAllocImpl(ST, MF.getRegInfo()).run(MF);
+}
+
+PreservedAnalyses
+AMDGPUPrepareAGPRAllocPass::run(MachineFunction &MF,
+                                MachineFunctionAnalysisManager &MFAM) {
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  AMDGPUPrepareAGPRAllocImpl(ST, MF.getRegInfo()).run(MF);
+  return PreservedAnalyses::all();
+}
+
+bool AMDGPUPrepareAGPRAllocImpl::run(MachineFunction &MF) {
+  if (MRI.isReserved(AMDGPU::AGPR0))
+    return false;
+
+  const MCInstrDesc &AVImmPseudo = TII.get(AMDGPU::AV_MOV_B32_IMM_PSEUDO);
+
+  bool Changed = false;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      if ((MI.getOpcode() == AMDGPU::V_MOV_B32_e32 &&
+           TII.isInlineConstant(MI, 1)) ||
+          (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
+           MI.getOperand(1).isImm())) {
+        MI.setDesc(AVImmPseudo);
+        Changed = true;
+      }
+    }
+  }
+
+  return Changed;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
new file mode 100644
index 0000000000000..dc598c98f241b
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
@@ -0,0 +1,23 @@
+//===- AMDGPUPrepareAGPRAlloc.h ---------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class AMDGPUPrepareAGPRAllocPass
+    : public PassInfoMixin<AMDGPUPrepareAGPRAllocPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
index 46027b8890234..8101c68986241 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
@@ -167,77 +167,39 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
 
   Info.UsesVCC =
       MRI.isPhysRegUsed(AMDGPU::VCC_LO) || MRI.isPhysRegUsed(AMDGPU::VCC_HI);
+  Info.NumExplicitSGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::SGPR_32RegClass,
+                                                /*IncludeCalls=*/false);
+  if (ST.hasMAIInsts())
+    Info.NumAGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass,
+                                          /*IncludeCalls=*/false);
 
   // If there are no calls, MachineRegisterInfo can tell us the used register
   // count easily.
   // A tail call isn't considered a call for MachineFrameInfo's purposes.
   if (!FrameInfo.hasCalls() && !FrameInfo.hasTailCall()) {
-    Info.NumVGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass);
-    Info.NumExplicitSGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::SGPR_32RegClass);
-    if (ST.hasMAIInsts())
-      Info.NumAGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass);
+    Info.NumVGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass,
+                                          /*IncludeCalls=*/false);
     return Info;
   }
 
   int32_t MaxVGPR = -1;
-  int32_t MaxAGPR = -1;
-  int32_t MaxSGPR = -1;
   Info.CalleeSegmentSize = 0;
 
   for (const MachineBasicBlock &MBB : MF) {
     for (const MachineInstr &MI : MBB) {
-      // TODO: Check regmasks? Do they occur anywhere except calls?
-      for (const MachineOperand &MO : MI.operands()) {
-        unsigned Width = 0;
-        bool IsSGPR = false;
-        bool IsAGPR = false;
+      for (unsigned I = 0; I < MI.getNumOperands(); ++I) {
+        const MachineOperand &MO = MI.getOperand(I);
 
         if (!MO.isReg())
           continue;
 
         Register Reg = MO.getReg();
         switch (Reg) {
-        case AMDGPU::EXEC:
-        case AMDGPU::EXEC_LO:
-        case AMDGPU::EXEC_HI:
-        case AMDGPU::SCC:
-        case AMDGPU::M0:
-        case AMDGPU::M0_LO16:
-        case AMDGPU::M0_HI16:
-        case AMDGPU::SRC_SHARED_BASE_LO:
-        case AMDGPU::SRC_SHARED_BASE:
-        case AMDGPU::SRC_SHARED_LIMIT_LO:
-        case AMDGPU::SRC_SHARED_LIMIT:
-        case AMDGPU::SRC_PRIVATE_BASE_LO:
-        case AMDGPU::SRC_PRIVATE_BASE:
-        case AMDGPU::SRC_PRIVATE_LIMIT_LO:
-        case AMDGPU::SRC_PRIVATE_LIMIT:
-        case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
-        case AMDGPU::SGPR_NULL:
-        case AMDGPU::SGPR_NULL64:
-        case AMDGPU::MODE:
-          continue;
-
         case AMDGPU::NoRegister:
           assert(MI.isDebugInstr() &&
                  "Instruction uses invalid noreg register");
           continue;
 
-        case AMDGPU::VCC:
-        case AMDGPU::VCC_LO:
-        case AMDGPU::VCC_HI:
-        case AMDGPU::VCC_LO_LO16:
-        case AMDGPU::VCC_LO_HI16:
-        case AMDGPU::VCC_HI_LO16:
-        case AMDGPU::VCC_HI_HI16:
-          Info.UsesVCC = true;
-          continue;
-
-        case AMDGPU::FLAT_SCR:
-        case AMDGPU::FLAT_SCR_LO:
-        case AMDGPU::FLAT_SCR_HI:
-          continue;
-
         case AMDGPU::XNACK_MASK:
         case AMDGPU::XNACK_MASK_LO:
         case AMDGPU::XNACK_MASK_HI:
@@ -267,170 +229,22 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
           break;
         }
 
-        if (AMDGPU::SGPR_32RegClass.contains(Reg) ||
-            AMDGPU::SGPR_LO16RegClass.contains(Reg) ||
-            AMDGPU::SGPR_HI16RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 1;
-        } else if (AMDGPU::VGPR_32RegClass.contains(Reg) ||
-                   AMDGPU::VGPR_16RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 1;
-        } else if (AMDGPU::AGPR_32RegClass.contains(Reg) ||
-                   AMDGPU::AGPR_LO16RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 1;
-        } else if (AMDGPU::SGPR_64RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 2;
-        } else if (AMDGPU::VReg_64RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 2;
-        } else if (AMDGPU::AReg_64RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 2;
-        } else if (AMDGPU::VReg_96RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 3;
-        } else if (AMDGPU::SReg_96RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 3;
-        } else if (AMDGPU::AReg_96RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 3;
-        } else if (AMDGPU::SGPR_128RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 4;
-        } else if (AMDGPU::VReg_128RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 4;
-        } else if (AMDGPU::AReg_128RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 4;
-        } else if (AMDGPU::VReg_160RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 5;
-        } else if (AMDGPU::SReg_160RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 5;
-        } else if (AMDGPU::AReg_160RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 5;
-        } else if (AMDGPU::VReg_192RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 6;
-        } else if (AMDGPU::SReg_192RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 6;
-        } else if (AMDGPU::AReg_192RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 6;
-        } else if (AMDGPU::VReg_224RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 7;
-        } else if (AMDGPU::SReg_224RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 7;
-        } else if (AMDGPU::AReg_224RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 7;
-        } else if (AMDGPU::SReg_256RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 8;
-        } else if (AMDGPU::VReg_256RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 8;
-        } else if (AMDGPU::AReg_256RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 8;
-        } else if (AMDGPU::VReg_288RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 9;
-        } else if (AMDGPU::SReg_288RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 9;
-        } else if (AMDGPU::AReg_288RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 9;
-        } else if (AMDGPU::VReg_320RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 10;
-        } else if (AMDGPU::SReg_320RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 10;
-        } else if (AMDGPU::AReg_320RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 10;
-        } else if (AMDGPU::VReg_352RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 11;
-        } else if (AMDGPU::SReg_352RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 11;
-        } else if (AMDGPU::AReg_352RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 11;
-        } else if (AMDGPU::VReg_384RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 12;
-        } else if (AMDGPU::SReg_384RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 12;
-        } else if (AMDGPU::AReg_384RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 12;
-        } else if (AMDGPU::SReg_512RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 16;
-        } else if (AMDGPU::VReg_512RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 16;
-        } else if (AMDGPU::AReg_512RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 16;
-        } else if (AMDGPU::SReg_1024RegClass.contains(Reg)) {
-          IsSGPR = true;
-          Width = 32;
-        } else if (AMDGPU::VReg_1024RegClass.contains(Reg)) {
-          IsSGPR = false;
-          Width = 32;
-        } else if (AMDGPU::AReg_1024RegClass.contains(Reg)) {
-          IsSGPR = false;
-          IsAGPR = true;
-          Width = 32;
-        } else {
-          // We only expect TTMP registers or registers that do not belong to
-          // any RC.
-          assert((AMDGPU::TTMP_32RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_64RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_128RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_256RegClass.contains(Reg) ||
-                  AMDGPU::TTMP_512RegClass.contains(Reg) ||
-                  !TRI.getPhysRegBaseClass(Reg)) &&
-                 "Unknown register class");
-        }
+        const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg);
+        assert((!RC || TRI.isVGPRClass(RC) || TRI.isSGPRClass(RC) ||
+                TRI.isAGPRClass(RC) || AMDGPU::TTMP_32RegClass.contains(Reg) ||
+                AMDGPU::TTMP_64RegClass.contains(Reg) ||
+                AMDGPU::TTMP_128RegClass.contains(Reg) ||
+                AMDGPU::TTMP_256RegClass.contains(Reg) ||
+                AMDGPU::TTMP_512RegClass.contains(Reg)) &&
+               "Unknown register class");
+
+        if (!RC || !TRI.isVGPRClass(RC))
+          continue;
+
+        unsigned Width = divideCeil(TRI.getRegSizeInBits(*RC), 32);
         unsigned HWReg = TRI.getHWRegIndex(Reg);
         int MaxUsed = HWReg + Width - 1;
-        if (IsSGPR) {
-          MaxSGPR = MaxUsed > MaxSGPR ? MaxUsed : MaxSGPR;
-        } else if (IsAGPR) {
-          MaxAGPR = MaxUsed > MaxAGPR ? MaxUsed : MaxAGPR;
-        } else {
-          MaxVGPR = MaxUsed > MaxVGPR ? MaxUsed : MaxVGPR;
-        }
+        MaxVGPR = std::max(MaxUsed, MaxVGPR);
       }
 
       if (MI.isCall()) {
@@ -492,9 +306,7 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
     }
   }
 
-  Info.NumExplicitSGPR = MaxSGPR + 1;
   Info.NumVGPR = MaxVGPR + 1;
-  Info.NumAGPR = MaxAGPR + 1;
 
   return Info;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 31a80e00edd3b..c865082a1dcea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -25,6 +25,7 @@
 #include "AMDGPUMacroFusion.h"
 #include "AMDGPUPerfHintAnalysis.h"
 #include "AMDGPUPreloadKernArgProlog.h"
+#include "AMDGPUPrepareAGPRAlloc.h"
 #include "AMDGPURemoveIncompatibleFunctions.h"
 #include "AMDGPUReserveWWMRegs.h"
 #include "AMDGPUResourceUsageAnalysis.h"
@@ -499,6 +500,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeGlobalISel(*PR);
   initializeAMDGPUAsmPrinterPass(*PR);
   initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
+  initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
   initializeGCNDPPCombineLegacyPass(*PR);
   initializeSILowerI1CopiesLegacyPass(*PR);
   initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);
@@ -1196,6 +1198,7 @@ class GCNPassConfig final : public AMDGPUPassConfig {
   bool addRegBankSelect() override;
   void addPreGlobalInstructionSelect() override;
   bool addGlobalInstructionSelect() override;
+  void addPreRegAlloc() override;
   void addFastRegAlloc() override;
   void addOptimizedRegAlloc() override;
 
@@ -1539,6 +1542,11 @@ void GCNPassConfig::addFastRegAlloc() {
   TargetPassConfig::addFastRegAlloc();
 }
 
+void GCNPassConfig::addPreRegAlloc() {
+  if (getOptLevel() != CodeGenOptLevel::None)
+    addPass(&AMDGPUPrepareAGPRAllocLegacyID);
+}
+
 void GCNPassConfig::addOptimizedRegAlloc() {
   if (EnableDCEInRA)
     insertPass(&DetectDeadLanesID, &DeadMachineInstructionElimID);
@@ -2235,6 +2243,11 @@ void AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
   Base::addOptimizedRegAlloc(addPass);
 }
 
+void AMDGPUCodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const {
+  if (getOptLevel() != CodeGenOptLevel::None)
+    addPass(AMDGPUPrepareAGPRAllocPass());
+}
+
 Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
     AddMachinePass &addPass) const {
   // TODO: Check --regalloc-npm option
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 3b2f39c14a9bc..e0f1296ddded8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -181,7 +181,9 @@ class AMDGPUCodeGenPassBuilder
   void addMachineSSAOptimization(AddMachinePass &) const;
   void addPostRegAlloc(AddMachinePass &) const;
   void addPreEmitPass(AddMachinePass &) const;
+  void addPreEmitRegAlloc(AddMachinePass &) const;
   Error addRegAssignmentOptimized(AddMachinePass &) const;
+  void addPreRegAlloc(AddMachinePass &) const;
   void addOptimizedRegAlloc(AddMachinePass &) const;
   void addPreSched2(AddMachinePass &) const;
 
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index e3519f192137c..42edec0d01493 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -74,6 +74,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPULowerKernelArguments.cpp
   AMDGPULowerKernelAttributes.cpp
   AMDGPULowerModuleLDSPass.cpp
+  AMDGPUPrepareAGPRAlloc.cpp
   AMDGPUSwLowerLDS.cpp
   AMDGPUMachineFunction.cpp
   AMDGPUMachineModuleInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 400795c29b0e4..0c76ff2ec5ea7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -33,6 +33,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -46,6 +47,7 @@
 #include <optional>
 
 using namespace llvm;
+using namespace llvm::SDPatternMatch;
 
 #define DEBUG_TYPE "si-lower"
 
@@ -14561,7 +14563,7 @@ static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
 // instead of a tree.
 SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
                                             DAGCombinerInfo &DCI) const {
-  assert(N->getOpcode() == ISD::ADD);
+  assert(N->isAnyAdd());
 
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
@@ -14594,7 +14596,7 @@ SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
     for (SDNode *User : LHS->users()) {
       // There is a use that does not feed into addition, so the multiply can't
       // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
-      if (User->getOpcode() != ISD::ADD)
+      if (!User->isAnyAdd())
         return SDValue();
 
       // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
@@ -14706,8 +14708,11 @@ SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
 
     SDValue Hi = getHiHalf64(LHS, DAG);
     SDValue ConstHi32 = DAG.getConstant(Hi_32(Val), SL, MVT::i32);
+    unsigned Opcode = N->getOpcode();
+    if (Opcode == ISD::PTRADD)
+      Opcode = ISD::ADD;
     SDValue AddHi =
-        DAG.getNode(N->getOpcode(), SL, MVT::i32, Hi, ConstHi32, N->getFlags());
+        DAG.getNode(Opcode, SL, MVT::i32, Hi, ConstHi32, N->getFlags());
 
     SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
     return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, Lo, AddHi);
@@ -15181,42 +15186,123 @@ SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
+  EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
-  if (N1.getOpcode() == ISD::ADD) {
-    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
-    //    y is not, and (add y, z) is used only once.
-    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
-    //    z is not, and (add y, z) is used only once.
-    // The goal is to move constant offsets to the outermost ptradd, to create
-    // more opportunities to fold offsets into memory instructions.
-    // Together with the generic combines in DAGCombiner.cpp, this also
-    // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
-    //
-    // This transform is here instead of in the general DAGCombiner as it can
-    // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
-    // AArch64's CPA.
-    SDValue X = N0;
-    SDValue Y = N1.getOperand(0);
-    SDValue Z = N1.getOperand(1);
-    if (N1.hasOneUse()) {
-      bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
-      bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
-      if (ZIsConstant != YIsConstant) {
-        // If both additions in the original were NUW, the new ones are as well.
-        SDNodeFlags Flags =
-            (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
-        if (YIsConstant)
-          std::swap(Y, Z);
+  // The following folds transform PTRADDs into regular arithmetic in cases
+  // where the PTRADD wouldn't be folded as an immediate offset into memory
+  // instructions anyway. They are target-specific in that other targets might
+  // prefer to not lose information about the pointer arithmetic.
+
+  // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
+  // Adapted from DAGCombiner::visitADDLikeCommutative.
+  SDValue V, K;
+  if (sd_match(N1, m_Shl(m_Neg(m_Value(V)), m_Value(K)))) {
+    SDNodeFlags ShlFlags = N1->getFlags();
+    // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
+    // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
+    // preserved.
+    SDNodeFlags NewShlFlags =
+        ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
+            ? SDNodeFlags::NoSignedWrap
+            : SDNodeFlags();
+    SDValue Inner = DAG.getNode(ISD::SHL, DL, VT, V, K, NewShlFlags);
+    DCI.AddToWorklist(Inner.getNode());
+    return DAG.getNode(ISD::SUB, DL, VT, N0, Inner);
+  }
+
+  // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
+  // performAddCombine.
+  if (N1.getOpcode() == ISD::MUL) {
+    if (Subtarget->hasMad64_32()) {
+      if (SDValue Folded = tryFoldToMad64_32(N, DCI))
+        return Folded;
+    }
+  }
 
-        SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, Flags);
+  // If the 32 low bits of the constant are all zero, there is nothing to fold
+  // into an immediate offset, so it's better to eliminate the unnecessary
+  // addition for the lower 32 bits than to preserve the PTRADD.
+  // Analogous to a fold in performAddCombine.
+  if (VT == MVT::i64) {
+    if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
+      return Folded;
+  }
+
+  if (N0.getOpcode() == ISD::PTRADD && N1.getOpcode() == ISD::Constant) {
+    // Fold (ptradd (ptradd GA, v), c) -> (ptradd (ptradd GA, c) v) with
+    // global address GA and constant c, such that c can be folded into GA.
+    SDValue GAValue = N0.getOperand(0);
+    if (const GlobalAddressSDNode *GA =
+            dyn_cast<GlobalAddressSDNode>(GAValue)) {
+      if (DCI.isBeforeLegalizeOps() && isOffsetFoldingLegal(GA)) {
+        // If both additions in the original were NUW, reassociation preserves
+        // that.
+        SDNodeFlags Flags =
+            (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+        SDValue Inner = DAG.getMemBasePlusOffset(GAValue, N1, DL, Flags);
         DCI.AddToWorklist(Inner.getNode());
-        return DAG.getMemBasePlusOffset(Inner, Z, DL, Flags);
+        return DAG.getMemBasePlusOffset(Inner, N0.getOperand(1), DL, Flags);
       }
     }
   }
 
+  if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
+    return SDValue();
+
+  // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
+  //    y is not, and (add y, z) is used only once.
+  // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
+  //    z is not, and (add y, z) is used only once.
+  // The goal is to move constant offsets to the outermost ptradd, to create
+  // more opportunities to fold offsets into memory instructions.
+  // Together with the generic combines in DAGCombiner.cpp, this also
+  // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
+  //
+  // This transform is here instead of in the general DAGCombiner as it can
+  // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
+  // AArch64's CPA.
+  SDValue X = N0;
+  SDValue Y = N1.getOperand(0);
+  SDValue Z = N1.getOperand(1);
+  bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
+  bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
+
+  // If both additions in the original were NUW, reassociation preserves that.
+  SDNodeFlags ReassocFlags =
+      (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+
+  if (ZIsConstant != YIsConstant) {
+    if (YIsConstant)
+      std::swap(Y, Z);
+    SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
+    DCI.AddToWorklist(Inner.getNode());
+    return DAG.getMemBasePlusOffset(Inner, Z, DL, ReassocFlags);
+  }
+
+  // If one of Y and Z is constant, they have been handled above. If both were
+  // constant, the addition would have been folded in SelectionDAG::getNode
+  // already. This ensures that the generic DAG combines won't undo the
+  // following reassociation.
+  assert(!YIsConstant && !ZIsConstant);
+
+  if (!X->isDivergent() && Y->isDivergent() != Z->isDivergent()) {
+    // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
+    // y are uniform and z isn't.
+    // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
+    // z are uniform and y isn't.
+    // The goal is to push uniform operands up in the computation, so that they
+    // can be handled with scalar operations. We can't use reassociateScalarOps
+    // for this since it requires two identical commutative operations to
+    // reassociate.
+    if (Y->isDivergent())
+      std::swap(Y, Z);
+    SDValue UniformInner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
+    DCI.AddToWorklist(UniformInner.getNode());
+    return DAG.getMemBasePlusOffset(UniformInner, Z, DL, ReassocFlags);
+  }
+
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 2764ed3d3f0b1..5e92921f3ea21 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1113,7 +1113,6 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   // that will not require an additional 4-bytes; this function assumes that it
   // will.
   bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
-    assert(!MO.isReg() && "isInlineConstant called on register operand!");
     if (!MO.isImm())
       return false;
     return isInlineConstant(MO.getImm(), OperandType);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 9173041a7bccd..fa2b8db6ba55a 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4052,11 +4052,11 @@ SIRegisterInfo::getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
   return 0;
 }
 
-unsigned
-SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
-                                   const TargetRegisterClass &RC) const {
+unsigned SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
+                                            const TargetRegisterClass &RC,
+                                            bool IncludeCalls) const {
   for (MCPhysReg Reg : reverse(RC.getRegisters()))
-    if (MRI.isPhysRegUsed(Reg))
+    if (MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/!IncludeCalls))
       return getHWRegIndex(Reg) + 1;
   return 0;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 06a7a17b0246b..0008e5f8cf3b4 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -486,9 +486,11 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
                                      unsigned SubReg) const;
 
   // \returns a number of registers of a given \p RC used in a function.
-  // Does not go inside function calls.
+  // Does not go inside function calls. If \p IncludeCalls is true, it will
+  // include registers that may be clobbered by calls.
   unsigned getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
-                              const TargetRegisterClass &RC) const;
+                              const TargetRegisterClass &RC,
+                              bool IncludeCalls = true) const;
 
   std::optional<uint8_t> getVRegFlagValue(StringRef Name) const override {
     return Name == "WWM_REG" ? AMDGPU::VirtRegFlag::WWM_REG
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 1bbbb610305e9..3ee90857b34b8 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -1148,6 +1148,7 @@ defm V_MOV_B64 : VOP1_Real_FULL <GFX1250Gen, 0x1d>;
 defm V_TANH_F32              : VOP1_Real_FULL<GFX1250Gen, 0x01e>;
 defm V_TANH_F16              : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x01f>;
 defm V_TANH_BF16             : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x04a>;
+defm V_PRNG_B32              : VOP1_Real_FULL<GFX1250Gen, 0x04b>;
 defm V_CVT_F32_BF16          : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x072, "v_cvt_f32_bf16", "V_CVT_F32_BF16_gfx1250">;
 defm V_CVT_PK_F16_FP8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x075>;
 defm V_CVT_PK_F16_BF8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x076>;
diff --git a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
index ac5e7f3891c72..1493bf4cba695 100644
--- a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
@@ -158,7 +158,12 @@ void LoongArchFrameLowering::processFunctionBeforeFrameFinalized(
   // estimateStackSize has been observed to under-estimate the final stack
   // size, so give ourselves wiggle-room by checking for stack size
   // representable an 11-bit signed field rather than 12-bits.
-  if (!isInt<11>(MFI.estimateStackSize(MF)))
+  // For [x]vstelm.{b/h/w/d} memory instructions with 8 imm offset, 7-bit
+  // signed field is fine.
+  unsigned EstimateStackSize = MFI.estimateStackSize(MF);
+  if (!isInt<11>(EstimateStackSize) ||
+      (MF.getSubtarget<LoongArchSubtarget>().hasExtLSX() &&
+       !isInt<7>(EstimateStackSize)))
     ScavSlotsNum = std::max(ScavSlotsNum, 1u);
 
   // For CFR spill.
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 6c8e3da80b932..23b4554349003 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -95,6 +95,11 @@ static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
     /* -21, -22, -23, -24 are reserved */
 };
 
+/// Returns true if DWARF CFI instructions ("frame moves") should be emitted.
+static bool needsDwarfCFI(const MachineFunction &MF) {
+  return MF.needsFrameMoves();
+}
+
 // For now we use x3, a.k.a gp, as pointer to shadow call stack.
 // User should not use x3 in their asm.
 static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -141,6 +146,9 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
       .addImm(-SlotSize)
       .setMIFlag(MachineInstr::FrameSetup);
 
+  if (!needsDwarfCFI(MF))
+    return;
+
   // Emit a CFI instruction that causes SlotSize to be subtracted from the value
   // of the shadow stack pointer when unwinding past this frame.
   char DwarfSCSReg = TRI->getDwarfRegNum(SCSPReg, /*IsEH*/ true);
@@ -199,8 +207,10 @@ static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
       .addReg(SCSPReg)
       .addImm(-SlotSize)
       .setMIFlag(MachineInstr::FrameDestroy);
-  // Restore the SCS pointer
-  CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
+  if (needsDwarfCFI(MF)) {
+    // Restore the SCS pointer
+    CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
+  }
 }
 
 // Insert instruction to swap mscratchsw with sp
@@ -935,6 +945,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   MBBI = std::prev(MBBI, getRVVCalleeSavedInfo(MF, CSI).size() +
                              getUnmanagedCSI(MF, CSI).size());
   CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
+  bool NeedsDwarfCFI = needsDwarfCFI(MF);
 
   // If libcalls are used to spill and restore callee-saved registers, the frame
   // has two sections; the opaque section managed by the libcalls, and the
@@ -962,10 +973,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
         alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
     RVFI->setLibCallStackSize(LibCallFrameSize);
 
-    CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
-    for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-      CFIBuilder.buildOffset(CS.getReg(),
-                             MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
+      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+        CFIBuilder.buildOffset(CS.getReg(),
+                               MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   // FIXME (note copied from Lanai): This appears to be overallocating.  Needs
@@ -996,14 +1009,17 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     // could only be the next instruction.
     ++PossiblePush;
 
-    // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
-    // could be. The PUSH will also get its own CFI metadata for its own
-    // modifications, which should come after the PUSH.
-    CFIInstBuilder PushCFIBuilder(MBB, PossiblePush, MachineInstr::FrameSetup);
-    PushCFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
-    for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
-      PushCFIBuilder.buildOffset(CS.getReg(),
-                                 MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
+      // could be. The PUSH will also get its own CFI metadata for its own
+      // modifications, which should come after the PUSH.
+      CFIInstBuilder PushCFIBuilder(MBB, PossiblePush,
+                                    MachineInstr::FrameSetup);
+      PushCFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
+      for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
+        PushCFIBuilder.buildOffset(CS.getReg(),
+                                   MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   if (RVFI->isPushable(MF) && PossiblePush != MBB.end() &&
@@ -1017,10 +1033,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     PossiblePush->getOperand(1).setImm(StackAdj);
     StackSize -= StackAdj;
 
-    CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
-    for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-      CFIBuilder.buildOffset(CS.getReg(),
-                             MFI.getObjectOffset(CS.getFrameIdx()));
+    if (NeedsDwarfCFI) {
+      CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
+      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+        CFIBuilder.buildOffset(CS.getReg(),
+                               MFI.getObjectOffset(CS.getFrameIdx()));
+    }
   }
 
   // Allocate space on the stack if necessary.
@@ -1031,7 +1049,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   bool DynAllocation =
       MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
   if (StackSize != 0)
-    allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, /*EmitCFI=*/true,
+    allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, NeedsDwarfCFI,
                   NeedProbe, ProbeSize, DynAllocation,
                   MachineInstr::FrameSetup);
 
@@ -1049,8 +1067,10 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
 
   // Iterate over list of callee-saved registers and emit .cfi_offset
   // directives.
-  for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
-    CFIBuilder.buildOffset(CS.getReg(), MFI.getObjectOffset(CS.getFrameIdx()));
+  if (NeedsDwarfCFI)
+    for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+      CFIBuilder.buildOffset(CS.getReg(),
+                             MFI.getObjectOffset(CS.getFrameIdx()));
 
   // Generate new FP.
   if (hasFP(MF)) {
@@ -1069,7 +1089,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
           MachineInstr::FrameSetup, getStackAlign());
     }
 
-    CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
+    if (NeedsDwarfCFI)
+      CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
   }
 
   uint64_t SecondSPAdjustAmount = 0;
@@ -1080,15 +1101,16 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
            "SecondSPAdjustAmount should be greater than zero");
 
     allocateStack(MBB, MBBI, MF, SecondSPAdjustAmount,
-                  getStackSizeWithRVVPadding(MF), !hasFP(MF), NeedProbe,
-                  ProbeSize, DynAllocation, MachineInstr::FrameSetup);
+                  getStackSizeWithRVVPadding(MF), NeedsDwarfCFI && !hasFP(MF),
+                  NeedProbe, ProbeSize, DynAllocation,
+                  MachineInstr::FrameSetup);
   }
 
   if (RVVStackSize) {
     if (NeedProbe) {
       allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, RVVStackSize,
-                                  MachineInstr::FrameSetup, !hasFP(MF),
-                                  DynAllocation);
+                                  MachineInstr::FrameSetup,
+                                  NeedsDwarfCFI && !hasFP(MF), DynAllocation);
     } else {
       // We must keep the stack pointer aligned through any intermediate
       // updates.
@@ -1097,14 +1119,15 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
                     MachineInstr::FrameSetup, getStackAlign());
     }
 
-    if (!hasFP(MF)) {
+    if (NeedsDwarfCFI && !hasFP(MF)) {
       // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
       CFIBuilder.insertCFIInst(createDefCFAExpression(
           *RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8));
     }
 
     std::advance(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
-    emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
+    if (NeedsDwarfCFI)
+      emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
   }
 
   if (hasFP(MF)) {
@@ -1171,8 +1194,9 @@ void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
                 MachineInstr::FrameDestroy, getStackAlign());
   StackSize = 0;
 
-  CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
-      .buildDefCFAOffset(CFAOffset);
+  if (needsDwarfCFI(MF))
+    CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
+        .buildDefCFAOffset(CFAOffset);
 }
 
 void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
@@ -1212,6 +1236,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
       std::next(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
   CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
                             MachineInstr::FrameDestroy);
+  bool NeedsDwarfCFI = needsDwarfCFI(MF);
 
   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
   uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
@@ -1232,10 +1257,11 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                     StackOffset::getScalable(RVVStackSize),
                     MachineInstr::FrameDestroy, getStackAlign());
 
-    if (!hasFP(MF))
-      CFIBuilder.buildDefCFA(SPReg, RealStackSize);
-
-    emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
+    if (NeedsDwarfCFI) {
+      if (!hasFP(MF))
+        CFIBuilder.buildDefCFA(SPReg, RealStackSize);
+      emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
+    }
   }
 
   if (FirstSPAdjustAmount) {
@@ -1251,7 +1277,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                     StackOffset::getFixed(SecondSPAdjustAmount),
                     MachineInstr::FrameDestroy, getStackAlign());
 
-    if (!hasFP(MF))
+    if (NeedsDwarfCFI && !hasFP(MF))
       CFIBuilder.buildDefCFAOffset(FirstSPAdjustAmount);
   }
 
@@ -1272,7 +1298,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                   getStackAlign());
   }
 
-  if (hasFP(MF))
+  if (NeedsDwarfCFI && hasFP(MF))
     CFIBuilder.buildDefCFA(SPReg, RealStackSize);
 
   // Skip to after the restores of scalar callee-saved registers
@@ -1295,8 +1321,9 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 
   // Recover callee-saved registers.
-  for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
-    CFIBuilder.buildRestore(CS.getReg());
+  if (NeedsDwarfCFI)
+    for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+      CFIBuilder.buildRestore(CS.getReg());
 
   if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(MBBI->getOpcode())) {
     // Use available stack adjustment in pop instruction to deallocate stack
@@ -1315,15 +1342,17 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
     auto NextI = next_nodbg(MBBI, MBB.end());
     if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
       ++MBBI;
-      CFIBuilder.setInsertPoint(MBBI);
+      if (NeedsDwarfCFI) {
+        CFIBuilder.setInsertPoint(MBBI);
 
-      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
-        CFIBuilder.buildRestore(CS.getReg());
+        for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+          CFIBuilder.buildRestore(CS.getReg());
 
-      // Update CFA Offset. If this is a QCI interrupt function, there will be a
-      // leftover offset which is deallocated by `QC.C.MILEAVERET`, otherwise
-      // getQCIInterruptStackSize() will be 0.
-      CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
+        // Update CFA Offset. If this is a QCI interrupt function, there will
+        // be a leftover offset which is deallocated by `QC.C.MILEAVERET`,
+        // otherwise getQCIInterruptStackSize() will be 0.
+        CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
+      }
     }
   }
 
@@ -1812,7 +1841,8 @@ MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
         // allocateStack.
         bool DynAllocation =
             MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
-        allocateStack(MBB, MI, MF, -Amount, -Amount, !hasFP(MF),
+        allocateStack(MBB, MI, MF, -Amount, -Amount,
+                      needsDwarfCFI(MF) && !hasFP(MF),
                       /*NeedProbe=*/true, ProbeSize, DynAllocation,
                       MachineInstr::NoFlags);
       } else {
diff --git a/llvm/lib/Target/X86/X86CallingConv.cpp b/llvm/lib/Target/X86/X86CallingConv.cpp
index 82e8ce4e0bd7c..5d5a705893242 100644
--- a/llvm/lib/Target/X86/X86CallingConv.cpp
+++ b/llvm/lib/Target/X86/X86CallingConv.cpp
@@ -389,7 +389,7 @@ static bool CC_X86_32_I128_FP128(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
   if (!ArgFlags.isInConsecutiveRegsLast())
     return true;
 
-  assert(PendingMembers.size() == 4 && "Should have two parts");
+  assert(PendingMembers.size() == 4 && "Should have four parts");
 
   int64_t Offset = State.AllocateStack(16, Align(16));
   PendingMembers[0].convertToMem(Offset);
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 57fbc71fa22ee..9cd35e35d4bc9 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -176,10 +176,10 @@ constexpr FeatureBitset FeaturesArrowlakeS =
     FeaturesArrowlake | FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 |
     FeatureSM4;
 constexpr FeatureBitset FeaturesPantherlake =
-    FeaturesArrowlakeS ^ FeatureWIDEKL | FeaturePREFETCHI;
+    (FeaturesArrowlakeS ^ FeatureWIDEKL) | FeaturePREFETCHI;
 constexpr FeatureBitset FeaturesClearwaterforest =
-    FeaturesSierraforest ^ FeatureWIDEKL | FeatureAVXVNNIINT16 | FeatureSHA512 |
-    FeatureSM3 | FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
+    (FeaturesSierraforest ^ FeatureWIDEKL) | FeatureAVXVNNIINT16 |
+    FeatureSHA512 | FeatureSM3 | FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
 
 // Geode Processor.
 constexpr FeatureBitset FeaturesGeode =
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index fbeb7218ba9a3..a65d0fb54c212 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1103,14 +1103,13 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
               FrameTy->getElementType(FrameData.getFieldIndex(E.first)), GEP,
               SpillAlignment, E.first->getName() + Twine(".reload"));
 
-        TinyPtrVector<DbgDeclareInst *> DIs = findDbgDeclares(Def);
         TinyPtrVector<DbgVariableRecord *> DVRs = findDVRDeclares(Def);
         // Try best to find dbg.declare. If the spill is a temp, there may not
         // be a direct dbg.declare. Walk up the load chain to find one from an
         // alias.
         if (F->getSubprogram()) {
           auto *CurDef = Def;
-          while (DIs.empty() && DVRs.empty() && isa<LoadInst>(CurDef)) {
+          while (DVRs.empty() && isa<LoadInst>(CurDef)) {
             auto *LdInst = cast<LoadInst>(CurDef);
             // Only consider ptr to ptr same type load.
             if (LdInst->getPointerOperandType() != LdInst->getType())
@@ -1118,12 +1117,11 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
             CurDef = LdInst->getPointerOperand();
             if (!isa<AllocaInst, LoadInst>(CurDef))
               break;
-            DIs = findDbgDeclares(CurDef);
             DVRs = findDVRDeclares(CurDef);
           }
         }
 
-        auto SalvageOne = [&](auto *DDI) {
+        auto SalvageOne = [&](DbgVariableRecord *DDI) {
           // This dbg.declare is preserved for all coro-split function
           // fragments. It will be unreachable in the main function, and
           // processed by coro::salvageDebugInfo() by the Cloner.
@@ -1137,7 +1135,6 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
           // will be deleted in all coro-split functions.
           coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
         };
-        for_each(DIs, SalvageOne);
         for_each(DVRs, SalvageOne);
       }
 
@@ -1225,8 +1222,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
     SmallVector<DbgVariableIntrinsic *, 4> DIs;
     SmallVector<DbgVariableRecord *> DbgVariableRecords;
     findDbgUsers(DIs, Alloca, &DbgVariableRecords);
-    for (auto *DVI : DIs)
-      DVI->replaceUsesOfWith(Alloca, G);
+    assert(DIs.empty() && "Should never see debug-intrinsics");
     for (auto *DVR : DbgVariableRecords)
       DVR->replaceVariableLocationOp(Alloca, G);
 
@@ -1920,48 +1916,6 @@ salvageDebugInfoImpl(SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
   return {{*Storage, *Expr}};
 }
 
-void coro::salvageDebugInfo(
-    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
-    DbgVariableIntrinsic &DVI, bool UseEntryValue) {
-
-  Function *F = DVI.getFunction();
-  // Follow the pointer arithmetic all the way to the incoming
-  // function argument and convert into a DIExpression.
-  bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
-  Value *OriginalStorage = DVI.getVariableLocationOp(0);
-
-  auto SalvagedInfo =
-      ::salvageDebugInfoImpl(ArgToAllocaMap, UseEntryValue, F, OriginalStorage,
-                             DVI.getExpression(), SkipOutermostLoad);
-  if (!SalvagedInfo)
-    return;
-
-  Value *Storage = &SalvagedInfo->first;
-  DIExpression *Expr = &SalvagedInfo->second;
-
-  DVI.replaceVariableLocationOp(OriginalStorage, Storage);
-  DVI.setExpression(Expr);
-  // We only hoist dbg.declare today since it doesn't make sense to hoist
-  // dbg.value since it does not have the same function wide guarantees that
-  // dbg.declare does.
-  if (isa<DbgDeclareInst>(DVI)) {
-    std::optional<BasicBlock::iterator> InsertPt;
-    if (auto *I = dyn_cast<Instruction>(Storage)) {
-      InsertPt = I->getInsertionPointAfterDef();
-      // Update DILocation only if variable was not inlined.
-      DebugLoc ILoc = I->getDebugLoc();
-      DebugLoc DVILoc = DVI.getDebugLoc();
-      if (ILoc && DVILoc &&
-          DVILoc->getScope()->getSubprogram() ==
-              ILoc->getScope()->getSubprogram())
-        DVI.setDebugLoc(I->getDebugLoc());
-    } else if (isa<Argument>(Storage))
-      InsertPt = F->getEntryBlock().begin();
-    if (InsertPt)
-      DVI.moveBefore(*(*InsertPt)->getParent(), *InsertPt);
-  }
-}
-
 void coro::salvageDebugInfo(
     SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
     DbgVariableRecord &DVR, bool UseEntryValue) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index b53c5a48eb10b..52f4ffe292dae 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -34,14 +34,11 @@ void suppressCoroAllocs(CoroIdInst *CoroId);
 void suppressCoroAllocs(LLVMContext &Context,
                         ArrayRef<CoroAllocInst *> CoroAllocs);
 
-/// Attempts to rewrite the location operand of debug intrinsics in terms of
+/// Attempts to rewrite the location operand of debug records in terms of
 /// the coroutine frame pointer, folding pointer offsets into the DIExpression
 /// of the intrinsic.
 /// If the frame pointer is an Argument, store it into an alloca to enhance the
 /// debugability.
-void salvageDebugInfo(
-    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
-    DbgVariableIntrinsic &DVI, bool IsEntryPoint);
 void salvageDebugInfo(
     SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
     DbgVariableRecord &DVR, bool UseEntryValue);
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 5a8a41f0dc432..64b33e46404f0 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -618,19 +618,15 @@ static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape,
   }
 }
 
-/// Returns all DbgVariableIntrinsic in F.
-static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,
-                 SmallVector<DbgVariableRecord *>>
-collectDbgVariableIntrinsics(Function &F) {
-  SmallVector<DbgVariableIntrinsic *, 8> Intrinsics;
+/// Returns all debug records in F.
+static SmallVector<DbgVariableRecord *>
+collectDbgVariableRecords(Function &F) {
   SmallVector<DbgVariableRecord *> DbgVariableRecords;
   for (auto &I : instructions(F)) {
     for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
       DbgVariableRecords.push_back(&DVR);
-    if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
-      Intrinsics.push_back(DVI);
   }
-  return {Intrinsics, DbgVariableRecords};
+  return DbgVariableRecords;
 }
 
 void coro::BaseCloner::replaceSwiftErrorOps() {
@@ -638,13 +634,11 @@ void coro::BaseCloner::replaceSwiftErrorOps() {
 }
 
 void coro::BaseCloner::salvageDebugInfo() {
-  auto [Worklist, DbgVariableRecords] = collectDbgVariableIntrinsics(*NewF);
+  auto DbgVariableRecords = collectDbgVariableRecords(*NewF);
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
 
   // Only 64-bit ABIs have a register we can refer to with the entry value.
   bool UseEntryValue = OrigF.getParent()->getTargetTriple().isArch64Bit();
-  for (DbgVariableIntrinsic *DVI : Worklist)
-    coro::salvageDebugInfo(ArgToAllocaMap, *DVI, UseEntryValue);
   for (DbgVariableRecord *DVR : DbgVariableRecords)
     coro::salvageDebugInfo(ArgToAllocaMap, *DVR, UseEntryValue);
 
@@ -655,7 +649,7 @@ void coro::BaseCloner::salvageDebugInfo() {
     return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
                                    &DomTree);
   };
-  auto RemoveOne = [&](auto *DVI) {
+  auto RemoveOne = [&](DbgVariableRecord *DVI) {
     if (IsUnreachableBlock(DVI->getParent()))
       DVI->eraseFromParent();
     else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
@@ -669,7 +663,6 @@ void coro::BaseCloner::salvageDebugInfo() {
         DVI->eraseFromParent();
     }
   };
-  for_each(Worklist, RemoveOne);
   for_each(DbgVariableRecords, RemoveOne);
 }
 
@@ -2022,9 +2015,7 @@ static void doSplitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
   // original function. The Cloner has already salvaged debug info in the new
   // coroutine funclets.
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
-  auto [DbgInsts, DbgVariableRecords] = collectDbgVariableIntrinsics(F);
-  for (auto *DDI : DbgInsts)
-    coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
+  auto DbgVariableRecords = collectDbgVariableRecords(F);
   for (DbgVariableRecord *DVR : DbgVariableRecords)
     coro::salvageDebugInfo(ArgToAllocaMap, *DVR, false /*UseEntryValue*/);
 
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index 8017db1cfe146..5fd5f7d9dad09 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -514,7 +514,7 @@ void collectSpillsAndAllocasFromInsts(
 void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F,
                               const SuspendCrossingInfo &Checker) {
   // We don't want the layout of coroutine frame to be affected
-  // by debug information. So we only choose to salvage DbgValueInst for
+  // by debug information. So we only choose to salvage dbg.values for
   // whose value is already in the frame.
   // We would handle the dbg.values for allocas specially
   for (auto &Iter : Spills) {
@@ -522,9 +522,7 @@ void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F,
     SmallVector<DbgValueInst *, 16> DVIs;
     SmallVector<DbgVariableRecord *, 16> DVRs;
     findDbgValues(DVIs, V, &DVRs);
-    for (DbgValueInst *DVI : DVIs)
-      if (Checker.isDefinitionAcrossSuspend(*V, DVI))
-        Spills[V].push_back(DVI);
+    assert(DVIs.empty());
     // Add the instructions which carry debug info that is in the frame.
     for (DbgVariableRecord *DVR : DVRs)
       if (Checker.isDefinitionAcrossSuspend(*V, DVR->Marker->MarkedInstr))
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index d4555e9435f1d..f5525deb0172f 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -572,7 +572,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
 
   // Work out whether a dbg.value intrinsic or an equivalent DbgVariableRecord
   // is a parameter to be preserved.
-  auto ExamineDbgValue = [](auto *DbgVal, auto &Container) {
+  auto ExamineDbgValue = [&PDVRRelated](DbgVariableRecord *DbgVal) {
     LLVM_DEBUG(dbgs() << " Deciding: ");
     LLVM_DEBUG(DbgVal->print(dbgs()));
     LLVM_DEBUG(dbgs() << "\n");
@@ -581,7 +581,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
       LLVM_DEBUG(dbgs() << "  Include (parameter): ");
       LLVM_DEBUG(DbgVal->print(dbgs()));
       LLVM_DEBUG(dbgs() << "\n");
-      Container.insert(DbgVal);
+      PDVRRelated.insert(DbgVal);
     } else {
       LLVM_DEBUG(dbgs() << "  Delete (!parameter): ");
       LLVM_DEBUG(DbgVal->print(dbgs()));
@@ -589,7 +589,8 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
     }
   };
 
-  auto ExamineDbgDeclare = [&PDIRelated](auto *DbgDecl, auto &Container) {
+  auto ExamineDbgDeclare = [&PDIRelated,
+                            &PDVRRelated](DbgVariableRecord *DbgDecl) {
     LLVM_DEBUG(dbgs() << " Deciding: ");
     LLVM_DEBUG(DbgDecl->print(dbgs()));
     LLVM_DEBUG(dbgs() << "\n");
@@ -616,7 +617,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
                 LLVM_DEBUG(dbgs() << "  Include: ");
                 LLVM_DEBUG(DbgDecl->print(dbgs()));
                 LLVM_DEBUG(dbgs() << "\n");
-                Container.insert(DbgDecl);
+                PDVRRelated.insert(DbgDecl);
               } else {
                 LLVM_DEBUG(dbgs() << "   Delete (!parameter): ");
                 LLVM_DEBUG(SI->print(dbgs()));
@@ -647,18 +648,14 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
     // they connected to parameters?
     for (DbgVariableRecord &DVR : filterDbgVars(BI->getDbgRecordRange())) {
       if (DVR.isDbgValue() || DVR.isDbgAssign()) {
-        ExamineDbgValue(&DVR, PDVRRelated);
+        ExamineDbgValue(&DVR);
       } else {
         assert(DVR.isDbgDeclare());
-        ExamineDbgDeclare(&DVR, PDVRRelated);
+        ExamineDbgDeclare(&DVR);
       }
     }
 
-    if (auto *DVI = dyn_cast<DbgValueInst>(&*BI)) {
-      ExamineDbgValue(DVI, PDIRelated);
-    } else if (auto *DDI = dyn_cast<DbgDeclareInst>(&*BI)) {
-      ExamineDbgDeclare(DDI, PDIRelated);
-    } else if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
+    if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
       LLVM_DEBUG(dbgs() << " Will Include Terminator: ");
       LLVM_DEBUG(BI->print(dbgs()));
       LLVM_DEBUG(dbgs() << "\n");
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 9d7c025ccff86..f7fbf0815df03 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -825,9 +825,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Value *EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned);
 
   bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock);
-  void tryToSinkInstructionDbgValues(
-      Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
-      BasicBlock *DestBlock, SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers);
   void tryToSinkInstructionDbgVariableRecords(
       Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
       BasicBlock *DestBlock, SmallVectorImpl<DbgVariableRecord *> &DPUsers);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 684b9a1f90161..503611a4fc32c 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1420,21 +1420,16 @@ void InstCombinerImpl::freelyInvertAllUsersOf(Value *I, Value *IgnoredUser) {
   SmallVector<DbgValueInst *, 4> DbgValues;
   SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
   llvm::findDbgValues(DbgValues, I, &DbgVariableRecords);
+  assert(DbgValues.empty());
 
-  auto InvertDbgValueUse = [&](auto *DbgVal) {
+  for (DbgVariableRecord *DbgVal : DbgVariableRecords) {
     SmallVector<uint64_t, 1> Ops = {dwarf::DW_OP_not};
     for (unsigned Idx = 0, End = DbgVal->getNumVariableLocationOps();
          Idx != End; ++Idx)
       if (DbgVal->getVariableLocationOp(Idx) == I)
         DbgVal->setExpression(
             DIExpression::appendOpsToArg(DbgVal->getExpression(), Ops, Idx));
-  };
-
-  for (DbgValueInst *DVI : DbgValues)
-    InvertDbgValueUse(DVI);
-
-  for (DbgVariableRecord *DVR : DbgVariableRecords)
-    InvertDbgValueUse(DVR);
+  }
 }
 
 /// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a
@@ -3575,6 +3570,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
   std::unique_ptr<DIBuilder> DIB;
   if (isa<AllocaInst>(MI)) {
     findDbgUsers(DVIs, &MI, &DVRs);
+    assert(DVIs.empty());
     DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
   }
 
@@ -5253,8 +5249,7 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I,
   SmallVector<DbgVariableIntrinsic *, 2> DbgUsers;
   SmallVector<DbgVariableRecord *, 2> DbgVariableRecords;
   findDbgUsers(DbgUsers, I, &DbgVariableRecords);
-  if (!DbgUsers.empty())
-    tryToSinkInstructionDbgValues(I, InsertPos, SrcBlock, DestBlock, DbgUsers);
+  assert(DbgUsers.empty());
   if (!DbgVariableRecords.empty())
     tryToSinkInstructionDbgVariableRecords(I, InsertPos, SrcBlock, DestBlock,
                                            DbgVariableRecords);
@@ -5271,71 +5266,12 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I,
   return true;
 }
 
-void InstCombinerImpl::tryToSinkInstructionDbgValues(
-    Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
-    BasicBlock *DestBlock, SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers) {
-  // For all debug values in the destination block, the sunk instruction
-  // will still be available, so they do not need to be dropped.
-  SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSalvage;
-  for (auto &DbgUser : DbgUsers)
-    if (DbgUser->getParent() != DestBlock)
-      DbgUsersToSalvage.push_back(DbgUser);
-
-  // Process the sinking DbgUsersToSalvage in reverse order, as we only want
-  // to clone the last appearing debug intrinsic for each given variable.
-  SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSink;
-  for (DbgVariableIntrinsic *DVI : DbgUsersToSalvage)
-    if (DVI->getParent() == SrcBlock)
-      DbgUsersToSink.push_back(DVI);
-  llvm::sort(DbgUsersToSink,
-             [](auto *A, auto *B) { return B->comesBefore(A); });
-
-  SmallVector<DbgVariableIntrinsic *, 2> DIIClones;
-  SmallSet<DebugVariable, 4> SunkVariables;
-  for (auto *User : DbgUsersToSink) {
-    // A dbg.declare instruction should not be cloned, since there can only be
-    // one per variable fragment. It should be left in the original place
-    // because the sunk instruction is not an alloca (otherwise we could not be
-    // here).
-    if (isa<DbgDeclareInst>(User))
-      continue;
-
-    DebugVariable DbgUserVariable =
-        DebugVariable(User->getVariable(), User->getExpression(),
-                      User->getDebugLoc()->getInlinedAt());
-
-    if (!SunkVariables.insert(DbgUserVariable).second)
-      continue;
-
-    // Leave dbg.assign intrinsics in their original positions and there should
-    // be no need to insert a clone.
-    if (isa<DbgAssignIntrinsic>(User))
-      continue;
-
-    DIIClones.emplace_back(cast<DbgVariableIntrinsic>(User->clone()));
-    if (isa<DbgDeclareInst>(User) && isa<CastInst>(I))
-      DIIClones.back()->replaceVariableLocationOp(I, I->getOperand(0));
-    LLVM_DEBUG(dbgs() << "CLONE: " << *DIIClones.back() << '\n');
-  }
-
-  // Perform salvaging without the clones, then sink the clones.
-  if (!DIIClones.empty()) {
-    salvageDebugInfoForDbgValues(*I, DbgUsersToSalvage, {});
-    // The clones are in reverse order of original appearance, reverse again to
-    // maintain the original order.
-    for (auto &DIIClone : llvm::reverse(DIIClones)) {
-      DIIClone->insertBefore(InsertPos);
-      LLVM_DEBUG(dbgs() << "SINK: " << *DIIClone << '\n');
-    }
-  }
-}
-
 void InstCombinerImpl::tryToSinkInstructionDbgVariableRecords(
     Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
     BasicBlock *DestBlock,
     SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
-  // Implementation of tryToSinkInstructionDbgValues, but for the
-  // DbgVariableRecord of variable assignments rather than dbg.values.
+  // For all debug values in the destination block, the sunk instruction
+  // will still be available, so they do not need to be dropped.
 
   // Fetch all DbgVariableRecords not already in the destination.
   SmallVector<DbgVariableRecord *, 2> DbgVariableRecordsToSalvage;
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 2786d81773ed9..df3160233c510 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1489,6 +1489,7 @@ static bool checkAndReplaceCondition(
     SmallVector<DbgVariableIntrinsic *> DbgUsers;
     SmallVector<DbgVariableRecord *> DVRUsers;
     findDbgUsers(DbgUsers, Cmp, &DVRUsers);
+    assert(DbgUsers.empty());
 
     for (auto *DVR : DVRUsers) {
       auto *DTN = DT.getNode(DVR->getParent());
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 85dd9a1bf7161..0f63ed0166cf4 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -2079,6 +2079,7 @@ struct DSEState {
       AllocFnKind AllocKind =
           Attrs.getFnAttr(Attribute::AllocKind).getAllocKind() |
           AllocFnKind::Zeroed;
+      AllocKind &= ~AllocFnKind::Uninitialized;
       Attrs =
           Attrs.addFnAttribute(Ctx, Attribute::getWithAllocKind(Ctx, AllocKind))
               .removeFnAttribute(Ctx, "alloc-variant-zeroed");
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index b5dbef13289ac..4d1f44076db7e 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1979,15 +1979,13 @@ void JumpThreadingPass::updateSSA(BasicBlock *BB, BasicBlock *NewBB,
 
     // Find debug values outside of the block
     findDbgValues(DbgValues, &I, &DbgVariableRecords);
-    llvm::erase_if(DbgValues, [&](const DbgValueInst *DbgVal) {
-      return DbgVal->getParent() == BB;
-    });
+    assert(DbgValues.empty());
     llvm::erase_if(DbgVariableRecords, [&](const DbgVariableRecord *DbgVarRec) {
       return DbgVarRec->getParent() == BB;
     });
 
     // If there are no uses outside the block, we're done with this instruction.
-    if (UsesToRename.empty() && DbgValues.empty() && DbgVariableRecords.empty())
+    if (UsesToRename.empty() && DbgVariableRecords.empty())
       continue;
     LLVM_DEBUG(dbgs() << "JT: Renaming non-local uses of: " << I << "\n");
 
@@ -2000,8 +1998,7 @@ void JumpThreadingPass::updateSSA(BasicBlock *BB, BasicBlock *NewBB,
 
     while (!UsesToRename.empty())
       SSAUpdate.RewriteUse(*UsesToRename.pop_back_val());
-    if (!DbgValues.empty() || !DbgVariableRecords.empty()) {
-      SSAUpdate.UpdateDebugValues(&I, DbgValues);
+    if (!DbgVariableRecords.empty()) {
       SSAUpdate.UpdateDebugValues(&I, DbgVariableRecords);
       DbgValues.clear();
       DbgVariableRecords.clear();
@@ -2032,32 +2029,7 @@ void JumpThreadingPass::cloneInstructions(ValueToValueMapTy &ValueMapping,
   // copy of the block 'NewBB'.  If there are PHI nodes in the source basic
   // block, evaluate them to account for entry from PredBB.
 
-  // Retargets llvm.dbg.value to any renamed variables.
-  auto RetargetDbgValueIfPossible = [&](Instruction *NewInst) -> bool {
-    auto DbgInstruction = dyn_cast<DbgValueInst>(NewInst);
-    if (!DbgInstruction)
-      return false;
-
-    SmallSet<std::pair<Value *, Value *>, 16> OperandsToRemap;
-    for (auto DbgOperand : DbgInstruction->location_ops()) {
-      auto DbgOperandInstruction = dyn_cast<Instruction>(DbgOperand);
-      if (!DbgOperandInstruction)
-        continue;
-
-      auto I = ValueMapping.find(DbgOperandInstruction);
-      if (I != ValueMapping.end()) {
-        OperandsToRemap.insert(
-            std::pair<Value *, Value *>(DbgOperand, I->second));
-      }
-    }
-
-    for (auto &[OldOp, MappedOp] : OperandsToRemap)
-      DbgInstruction->replaceVariableLocationOp(OldOp, MappedOp);
-    return true;
-  };
-
-  // Duplicate implementation of the above dbg.value code, using
-  // DbgVariableRecords instead.
+  // Retargets dbg.value to any renamed variables.
   auto RetargetDbgVariableRecordIfPossible = [&](DbgVariableRecord *DVR) {
     SmallSet<std::pair<Value *, Value *>, 16> OperandsToRemap;
     for (auto *Op : DVR->location_ops()) {
@@ -2116,9 +2088,6 @@ void JumpThreadingPass::cloneInstructions(ValueToValueMapTy &ValueMapping,
     if (const DebugLoc &DL = New->getDebugLoc())
       mapAtomInstance(DL, ValueMapping);
 
-    if (RetargetDbgValueIfPossible(New))
-      continue;
-
     // Remap operands to patch up intra-block references.
     for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)
       if (Instruction *Inst = dyn_cast<Instruction>(New->getOperand(i))) {
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index a0f9f3c4a35a5..70e9eee5339a7 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -78,6 +78,7 @@ enum class RuleTy {
   PerLoopCacheAnalysis,
   PerInstrOrderCost,
   ForVectorization,
+  Ignore
 };
 
 } // end anonymous namespace
@@ -106,14 +107,20 @@ static cl::list<RuleTy> Profitabilities(
                clEnumValN(RuleTy::PerInstrOrderCost, "instorder",
                           "Prioritize the IVs order of each instruction"),
                clEnumValN(RuleTy::ForVectorization, "vectorize",
-                          "Prioritize vectorization")));
+                          "Prioritize vectorization"),
+               clEnumValN(RuleTy::Ignore, "ignore",
+                          "Ignore profitability, force interchange (does not "
+                          "work with other options)")));
 
 #ifndef NDEBUG
-static bool noDuplicateRules(ArrayRef<RuleTy> Rules) {
+static bool noDuplicateRulesAndIgnore(ArrayRef<RuleTy> Rules) {
   SmallSet<RuleTy, 4> Set;
-  for (RuleTy Rule : Rules)
+  for (RuleTy Rule : Rules) {
     if (!Set.insert(Rule).second)
       return false;
+    if (Rule == RuleTy::Ignore)
+      return false;
+  }
   return true;
 }
 
@@ -1357,6 +1364,13 @@ std::optional<bool> LoopInterchangeProfitability::isProfitableForVectorization(
 bool LoopInterchangeProfitability::isProfitable(
     const Loop *InnerLoop, const Loop *OuterLoop, unsigned InnerLoopId,
     unsigned OuterLoopId, CharMatrix &DepMatrix, CacheCostManager &CCM) {
+
+  // Return true if interchange is forced and the cost-model ignored.
+  if (Profitabilities.size() == 1 && Profitabilities[0] == RuleTy::Ignore)
+    return true;
+  assert(noDuplicateRulesAndIgnore(Profitabilities) &&
+         "Duplicate rules and option 'ignore' are not allowed");
+
   // isProfitable() is structured to avoid endless loop interchange. If the
   // highest priority rule (isProfitablePerLoopCacheAnalysis by default) could
   // decide the profitability then, profitability check will stop and return the
@@ -1365,7 +1379,6 @@ bool LoopInterchangeProfitability::isProfitable(
   // second highest priority rule (isProfitablePerInstrOrderCost by default).
   // Likewise, if it failed to analysis the profitability then only, the last
   // rule (isProfitableForVectorization by default) will decide.
-  assert(noDuplicateRules(Profitabilities) && "Detect duplicate rules");
   std::optional<bool> shouldInterchange;
   for (RuleTy RT : Profitabilities) {
     switch (RT) {
@@ -1382,6 +1395,9 @@ bool LoopInterchangeProfitability::isProfitable(
       shouldInterchange =
           isProfitableForVectorization(InnerLoopId, OuterLoopId, DepMatrix);
       break;
+    case RuleTy::Ignore:
+      llvm_unreachable("Option 'ignore' is not supported with other options");
+      break;
     }
 
     // If this rule could determine the profitability, don't call subsequent
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index dc8fa4379752f..9e318b04c2c99 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -3790,6 +3790,11 @@ LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
             continue;
         }
 
+        // Do not consider uses inside lifetime intrinsics. These are not
+        // actually materialized.
+        if (UserInst->isLifetimeStartOrEnd())
+          continue;
+
         std::pair<size_t, Immediate> P =
             getUse(S, LSRUse::Basic, MemAccessTy());
         size_t LUIdx = P.first;
@@ -6630,13 +6635,10 @@ struct SCEVDbgValueBuilder {
 /// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
 /// and DIExpression.
 struct DVIRecoveryRec {
-  DVIRecoveryRec(DbgValueInst *DbgValue)
-      : DbgRef(DbgValue), Expr(DbgValue->getExpression()),
-        HadLocationArgList(false) {}
   DVIRecoveryRec(DbgVariableRecord *DVR)
       : DbgRef(DVR), Expr(DVR->getExpression()), HadLocationArgList(false) {}
 
-  PointerUnion<DbgValueInst *, DbgVariableRecord *> DbgRef;
+  DbgVariableRecord *DbgRef;
   DIExpression *Expr;
   bool HadLocationArgList;
   SmallVector<WeakVH, 2> LocationOps;
@@ -6695,44 +6697,38 @@ static void updateDVIWithLocations(T &DbgVal,
 }
 
 /// Write the new expression and new location ops for the dbg.value. If possible
-/// reduce the szie of the dbg.value intrinsic by omitting DIArglist. This
+/// reduce the szie of the dbg.value by omitting DIArglist. This
 /// can be omitted if:
 /// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
 /// 2. The DW_OP_LLVM_arg is the first operand in the expression.
-static void UpdateDbgValueInst(DVIRecoveryRec &DVIRec,
-                               SmallVectorImpl<Value *> &NewLocationOps,
-                               SmallVectorImpl<uint64_t> &NewExpr) {
-  auto UpdateDbgValueInstImpl = [&](auto *DbgVal) {
-    unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
-    if (NumLLVMArgs == 0) {
-      // Location assumed to be on the stack.
-      updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
-    } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
-      // There is only a single DW_OP_llvm_arg at the start of the expression,
-      // so it can be omitted along with DIArglist.
-      assert(NewExpr[1] == 0 &&
-             "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
-      llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2));
-      updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
-    } else {
-      // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
-      updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
-    }
+static void UpdateDbgValue(DVIRecoveryRec &DVIRec,
+                           SmallVectorImpl<Value *> &NewLocationOps,
+                           SmallVectorImpl<uint64_t> &NewExpr) {
+  DbgVariableRecord *DbgVal = DVIRec.DbgRef;
+  unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
+  if (NumLLVMArgs == 0) {
+    // Location assumed to be on the stack.
+    updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
+  } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
+    // There is only a single DW_OP_llvm_arg at the start of the expression,
+    // so it can be omitted along with DIArglist.
+    assert(NewExpr[1] == 0 &&
+           "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
+    llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2));
+    updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
+  } else {
+    // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
+    updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
+  }
 
-    // If the DIExpression was previously empty then add the stack terminator.
-    // Non-empty expressions have only had elements inserted into them and so
-    // the terminator should already be present e.g. stack_value or fragment.
-    DIExpression *SalvageExpr = DbgVal->getExpression();
-    if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
-      SalvageExpr =
-          DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
-      DbgVal->setExpression(SalvageExpr);
-    }
-  };
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    UpdateDbgValueInstImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
-  else
-    UpdateDbgValueInstImpl(cast<DbgVariableRecord *>(DVIRec.DbgRef));
+  // If the DIExpression was previously empty then add the stack terminator.
+  // Non-empty expressions have only had elements inserted into them and so
+  // the terminator should already be present e.g. stack_value or fragment.
+  DIExpression *SalvageExpr = DbgVal->getExpression();
+  if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
+    SalvageExpr = DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
+    DbgVal->setExpression(SalvageExpr);
+  }
 }
 
 /// Cached location ops may be erased during LSR, in which case a poison is
@@ -6746,39 +6742,34 @@ static Value *getValueOrPoison(WeakVH &VH, LLVMContext &C) {
 
 /// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
 static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
-  auto RestorePreTransformStateImpl = [&](auto *DbgVal) {
-    LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
-                      << "scev-salvage: post-LSR: " << *DbgVal << '\n');
-    assert(DVIRec.Expr && "Expected an expression");
-    DbgVal->setExpression(DVIRec.Expr);
-
-    // Even a single location-op may be inside a DIArgList and referenced with
-    // DW_OP_LLVM_arg, which is valid only with a DIArgList.
-    if (!DVIRec.HadLocationArgList) {
-      assert(DVIRec.LocationOps.size() == 1 &&
-             "Unexpected number of location ops.");
-      // LSR's unsuccessful salvage attempt may have added DIArgList, which in
-      // this case was not present before, so force the location back to a
-      // single uncontained Value.
-      Value *CachedValue =
-          getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
-      DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
-    } else {
-      SmallVector<ValueAsMetadata *, 3> MetadataLocs;
-      for (WeakVH VH : DVIRec.LocationOps) {
-        Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
-        MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
-      }
-      auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
-      DbgVal->setRawLocation(
-          llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
+  DbgVariableRecord *DbgVal = DVIRec.DbgRef;
+  LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
+                    << "scev-salvage: post-LSR: " << *DbgVal << '\n');
+  assert(DVIRec.Expr && "Expected an expression");
+  DbgVal->setExpression(DVIRec.Expr);
+
+  // Even a single location-op may be inside a DIArgList and referenced with
+  // DW_OP_LLVM_arg, which is valid only with a DIArgList.
+  if (!DVIRec.HadLocationArgList) {
+    assert(DVIRec.LocationOps.size() == 1 &&
+           "Unexpected number of location ops.");
+    // LSR's unsuccessful salvage attempt may have added DIArgList, which in
+    // this case was not present before, so force the location back to a
+    // single uncontained Value.
+    Value *CachedValue =
+        getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
+    DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
+  } else {
+    SmallVector<ValueAsMetadata *, 3> MetadataLocs;
+    for (WeakVH VH : DVIRec.LocationOps) {
+      Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
+      MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
     }
-    LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
-  };
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    RestorePreTransformStateImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
-  else
-    RestorePreTransformStateImpl(cast<DbgVariableRecord *>(DVIRec.DbgRef));
+    auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
+    DbgVal->setRawLocation(
+        llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
+  }
+  LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
 }
 
 static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
@@ -6786,9 +6777,7 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
                        const SCEV *SCEVInductionVar,
                        SCEVDbgValueBuilder IterCountExpr) {
 
-  if (isa<DbgValueInst *>(DVIRec.DbgRef)
-          ? !cast<DbgValueInst *>(DVIRec.DbgRef)->isKillLocation()
-          : !cast<DbgVariableRecord *>(DVIRec.DbgRef)->isKillLocation())
+  if (!DVIRec.DbgRef->isKillLocation())
     return false;
 
   // LSR may have caused several changes to the dbg.value in the failed salvage
@@ -6882,13 +6871,8 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
     DbgBuilder->appendToVectors(NewExpr, NewLocationOps);
   }
 
-  UpdateDbgValueInst(DVIRec, NewLocationOps, NewExpr);
-  if (isa<DbgValueInst *>(DVIRec.DbgRef))
-    LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
-                      << *cast<DbgValueInst *>(DVIRec.DbgRef) << "\n");
-  else
-    LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
-                      << *cast<DbgVariableRecord *>(DVIRec.DbgRef) << "\n");
+  UpdateDbgValue(DVIRec, NewLocationOps, NewExpr);
+  LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DbgRef << "\n");
   return true;
 }
 
@@ -6934,21 +6918,23 @@ static void DbgRewriteSalvageableDVIs(
 /// cacheing and salvaging.
 static void DbgGatherSalvagableDVI(
     Loop *L, ScalarEvolution &SE,
-    SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs,
-    SmallSet<AssertingVH<DbgValueInst>, 2> &DVIHandles) {
+    SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs) {
   for (const auto &B : L->getBlocks()) {
     for (auto &I : *B) {
-      auto ProcessDbgValue = [&](auto *DbgVal) -> bool {
+      for (DbgVariableRecord &DbgVal : filterDbgVars(I.getDbgRecordRange())) {
+        if (!DbgVal.isDbgValue() && !DbgVal.isDbgAssign())
+          continue;
+
         // Ensure that if any location op is undef that the dbg.vlue is not
         // cached.
-        if (DbgVal->isKillLocation())
-          return false;
+        if (DbgVal.isKillLocation())
+          continue;
 
         // Check that the location op SCEVs are suitable for translation to
         // DIExpression.
         const auto &HasTranslatableLocationOps =
-            [&](const auto *DbgValToTranslate) -> bool {
-          for (const auto LocOp : DbgValToTranslate->location_ops()) {
+            [&](const DbgVariableRecord &DbgValToTranslate) -> bool {
+          for (const auto LocOp : DbgValToTranslate.location_ops()) {
             if (!LocOp)
               return false;
 
@@ -6963,31 +6949,21 @@ static void DbgGatherSalvagableDVI(
         };
 
         if (!HasTranslatableLocationOps(DbgVal))
-          return false;
+          continue;
 
         std::unique_ptr<DVIRecoveryRec> NewRec =
-            std::make_unique<DVIRecoveryRec>(DbgVal);
+            std::make_unique<DVIRecoveryRec>(&DbgVal);
         // Each location Op may need a SCEVDbgValueBuilder in order to recover
         // it. Pre-allocating a vector will enable quick lookups of the builder
         // later during the salvage.
-        NewRec->RecoveryExprs.resize(DbgVal->getNumVariableLocationOps());
-        for (const auto LocOp : DbgVal->location_ops()) {
+        NewRec->RecoveryExprs.resize(DbgVal.getNumVariableLocationOps());
+        for (const auto LocOp : DbgVal.location_ops()) {
           NewRec->SCEVs.push_back(SE.getSCEV(LocOp));
           NewRec->LocationOps.push_back(LocOp);
-          NewRec->HadLocationArgList = DbgVal->hasArgList();
+          NewRec->HadLocationArgList = DbgVal.hasArgList();
         }
         SalvageableDVISCEVs.push_back(std::move(NewRec));
-        return true;
-      };
-      for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
-        if (DVR.isDbgValue() || DVR.isDbgAssign())
-          ProcessDbgValue(&DVR);
       }
-      auto DVI = dyn_cast<DbgValueInst>(&I);
-      if (!DVI)
-        continue;
-      if (ProcessDbgValue(DVI))
-        DVIHandles.insert(DVI);
     }
   }
 }
@@ -7036,8 +7012,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   // Debug preservation - before we start removing anything identify which DVI
   // meet the salvageable criteria and store their DIExpression and SCEVs.
   SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> SalvageableDVIRecords;
-  SmallSet<AssertingVH<DbgValueInst>, 2> DVIHandles;
-  DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords, DVIHandles);
+  DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords);
 
   bool Changed = false;
   std::unique_ptr<MemorySSAUpdater> MSSAU;
@@ -7105,7 +7080,6 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   for (auto &Rec : SalvageableDVIRecords)
     Rec->clear();
   SalvageableDVIRecords.clear();
-  DVIHandles.clear();
   return Changed;
 }
 
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index eacaf42e4e8ba..1d1af42153325 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1222,9 +1222,7 @@ static void eraseDebugIntrinsicsWithNonLocalRefs(Function &F) {
     SmallVector<DbgVariableIntrinsic *, 4> DbgUsers;
     SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
     findDbgUsers(DbgUsers, &I, &DbgVariableRecords);
-    for (DbgVariableIntrinsic *DVI : DbgUsers)
-      if (DVI->getFunction() != &F)
-        DVI->eraseFromParent();
+    assert(DbgUsers.empty());
     for (DbgVariableRecord *DVR : DbgVariableRecords)
       if (DVR->getFunction() != &F)
         DVR->eraseFromParent();
@@ -1289,14 +1287,12 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
     SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
     SmallVector<DbgVariableRecord *, 1> DPUsers;
     findDbgUsers(DbgUsers, Input, &DPUsers);
+    assert(DbgUsers.empty());
     DIExpression *Expr = DIB.createExpression();
 
     // Iterate the debud users of the Input values. If they are in the extracted
     // function then update their location with the new value. If they are in
     // the parent function then create a similar debug record.
-    for (auto *DVI : DbgUsers)
-      UpdateOrInsertDebugRecord(DVI, Input, NewVal, Expr,
-                                isa<DbgDeclareInst>(DVI));
     for (auto *DVR : DPUsers)
       UpdateOrInsertDebugRecord(DVR, Input, NewVal, Expr, DVR->isDbgDeclare());
   }
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index a1f030a336c15..4210ce6da1eb2 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -808,9 +808,6 @@ bool checkDebugifyMetadata(Module &M,
 
     // Find missing lines.
     for (Instruction &I : instructions(F)) {
-      if (isa<DbgValueInst>(&I))
-        continue;
-
       auto DL = I.getDebugLoc();
       if (DL && DL.getLine() != 0) {
         MissingLines.reset(DL.getLine() - 1);
@@ -839,10 +836,6 @@ bool checkDebugifyMetadata(Module &M,
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
         if (DVR.isDbgValue() || DVR.isDbgAssign())
           CheckForMisSized(&DVR);
-      auto *DVI = dyn_cast<DbgValueInst>(&I);
-      if (!DVI)
-        continue;
-      CheckForMisSized(DVI);
     }
   }
 
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 6929d14bc56ea..ed3dca2f7c307 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1978,14 +1978,13 @@ static at::StorageToVarsMap collectEscapedLocals(const DataLayout &DL,
       continue;
 
     // Find all local variables associated with the backing storage.
-    auto CollectAssignsForStorage = [&](auto *DbgAssign) {
+    auto CollectAssignsForStorage = [&](DbgVariableRecord *DbgAssign) {
       // Skip variables from inlined functions - they are not local variables.
       if (DbgAssign->getDebugLoc().getInlinedAt())
         return;
       LLVM_DEBUG(errs() << " > DEF : " << *DbgAssign << "\n");
       EscapedLocals[Base].insert(at::VarRecord(DbgAssign));
     };
-    for_each(at::getAssignmentMarkers(Base), CollectAssignsForStorage);
     for_each(at::getDVRAssignmentMarkers(Base), CollectAssignsForStorage);
   }
   return EscapedLocals;
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index b14bbeac97675..d481ad9dee181 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -613,11 +613,10 @@ bool llvm::replaceDbgUsesWithUndef(Instruction *I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, I, &DPUsers);
-  for (auto *DII : DbgUsers)
-    DII->setKillLocation();
+  assert(DbgUsers.empty());
   for (auto *DVR : DPUsers)
     DVR->setKillLocation();
-  return !DbgUsers.empty() || !DPUsers.empty();
+  return !DPUsers.empty();
 }
 
 /// areAllUsesEqual - Check whether the uses of a value are all the same.
@@ -1607,12 +1606,8 @@ static bool PhiHasDebugValue(DILocalVariable *DIVar,
   SmallVector<DbgValueInst *, 1> DbgValues;
   SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
   findDbgValues(DbgValues, APN, &DbgVariableRecords);
-  for (auto *DVI : DbgValues) {
-    assert(is_contained(DVI->getValues(), APN));
-    if ((DVI->getVariable() == DIVar) && (DVI->getExpression() == DIExpr))
-      return true;
-  }
-  for (auto *DVR : DbgVariableRecords) {
+  assert(DbgValues.empty());
+  for (DbgVariableRecord *DVR : DbgVariableRecords) {
     assert(is_contained(DVR->location_ops(), APN));
     if ((DVR->getVariable() == DIVar) && (DVR->getExpression() == DIExpr))
       return true;
@@ -1971,7 +1966,6 @@ bool llvm::replaceDbgDeclare(Value *Address, Value *NewAddress,
 static void updateOneDbgValueForAlloca(const DebugLoc &Loc,
                                        DILocalVariable *DIVar,
                                        DIExpression *DIExpr, Value *NewAddress,
-                                       DbgValueInst *DVI,
                                        DbgVariableRecord *DVR,
                                        DIBuilder &Builder, int Offset) {
   assert(DIVar && "Missing variable");
@@ -1987,14 +1981,8 @@ static void updateOneDbgValueForAlloca(const DebugLoc &Loc,
   if (Offset)
     DIExpr = DIExpression::prepend(DIExpr, 0, Offset);
 
-  if (DVI) {
-    DVI->setExpression(DIExpr);
-    DVI->replaceVariableLocationOp(0u, NewAddress);
-  } else {
-    assert(DVR);
-    DVR->setExpression(DIExpr);
-    DVR->replaceVariableLocationOp(0u, NewAddress);
-  }
+  DVR->setExpression(DIExpr);
+  DVR->replaceVariableLocationOp(0u, NewAddress);
 }
 
 void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
@@ -2002,18 +1990,13 @@ void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
   SmallVector<DbgValueInst *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgValues(DbgUsers, AI, &DPUsers);
-
-  // Attempt to replace dbg.values that use this alloca.
-  for (auto *DVI : DbgUsers)
-    updateOneDbgValueForAlloca(DVI->getDebugLoc(), DVI->getVariable(),
-                               DVI->getExpression(), NewAllocaAddress, DVI,
-                               nullptr, Builder, Offset);
+  assert(DbgUsers.empty());
 
   // Replace any DbgVariableRecords that use this alloca.
   for (DbgVariableRecord *DVR : DPUsers)
     updateOneDbgValueForAlloca(DVR->getDebugLoc(), DVR->getVariable(),
-                               DVR->getExpression(), NewAllocaAddress, nullptr,
-                               DVR, Builder, Offset);
+                               DVR->getExpression(), NewAllocaAddress, DVR,
+                               Builder, Offset);
 }
 
 /// Where possible to salvage debug information for \p I do so.
@@ -2022,6 +2005,7 @@ void llvm::salvageDebugInfo(Instruction &I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, &I, &DPUsers);
+  assert(DbgUsers.empty());
   salvageDebugInfoForDbgValues(I, DbgUsers, DPUsers);
 }
 
@@ -2070,66 +2054,9 @@ void llvm::salvageDebugInfoForDbgValues(
   const unsigned MaxExpressionSize = 128;
   bool Salvaged = false;
 
-  for (auto *DII : DbgUsers) {
-    if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DII)) {
-      if (DAI->getAddress() == &I) {
-        salvageDbgAssignAddress(DAI);
-        Salvaged = true;
-      }
-      if (DAI->getValue() != &I)
-        continue;
-    }
+  // We should never see debug intrinsics nowadays.
+  assert(DbgUsers.empty());
 
-    // Do not add DW_OP_stack_value for DbgDeclare, because they are implicitly
-    // pointing out the value as a DWARF memory location description.
-    bool StackValue = isa<DbgValueInst>(DII);
-    auto DIILocation = DII->location_ops();
-    assert(
-        is_contained(DIILocation, &I) &&
-        "DbgVariableIntrinsic must use salvaged instruction as its location");
-    SmallVector<Value *, 4> AdditionalValues;
-    // `I` may appear more than once in DII's location ops, and each use of `I`
-    // must be updated in the DIExpression and potentially have additional
-    // values added; thus we call salvageDebugInfoImpl for each `I` instance in
-    // DIILocation.
-    Value *Op0 = nullptr;
-    DIExpression *SalvagedExpr = DII->getExpression();
-    auto LocItr = find(DIILocation, &I);
-    while (SalvagedExpr && LocItr != DIILocation.end()) {
-      SmallVector<uint64_t, 16> Ops;
-      unsigned LocNo = std::distance(DIILocation.begin(), LocItr);
-      uint64_t CurrentLocOps = SalvagedExpr->getNumLocationOperands();
-      Op0 = salvageDebugInfoImpl(I, CurrentLocOps, Ops, AdditionalValues);
-      if (!Op0)
-        break;
-      SalvagedExpr =
-          DIExpression::appendOpsToArg(SalvagedExpr, Ops, LocNo, StackValue);
-      LocItr = std::find(++LocItr, DIILocation.end(), &I);
-    }
-    // salvageDebugInfoImpl should fail on examining the first element of
-    // DbgUsers, or none of them.
-    if (!Op0)
-      break;
-
-    SalvagedExpr = SalvagedExpr->foldConstantMath();
-    DII->replaceVariableLocationOp(&I, Op0);
-    bool IsValidSalvageExpr = SalvagedExpr->getNumElements() <= MaxExpressionSize;
-    if (AdditionalValues.empty() && IsValidSalvageExpr) {
-      DII->setExpression(SalvagedExpr);
-    } else if (isa<DbgValueInst>(DII) && IsValidSalvageExpr &&
-               DII->getNumVariableLocationOps() + AdditionalValues.size() <=
-                   MaxDebugArgs) {
-      DII->addVariableLocationOps(AdditionalValues, SalvagedExpr);
-    } else {
-      // Do not salvage using DIArgList for dbg.declare, as it is not currently
-      // supported in those instructions. Also do not salvage if the resulting
-      // DIArgList would contain an unreasonably large number of values.
-      DII->setKillLocation();
-    }
-    LLVM_DEBUG(dbgs() << "SALVAGE: " << *DII << '\n');
-    Salvaged = true;
-  }
-  // Duplicate of above block for DbgVariableRecords.
   for (auto *DVR : DPUsers) {
     if (DVR->isDbgAssign()) {
       if (DVR->getAddress() == &I) {
@@ -2198,9 +2125,6 @@ void llvm::salvageDebugInfoForDbgValues(
   if (Salvaged)
     return;
 
-  for (auto *DII : DbgUsers)
-    DII->setKillLocation();
-
   for (auto *DVR : DPUsers)
     DVR->setKillLocation();
 }
@@ -3429,8 +3353,7 @@ void llvm::dropDebugUsers(Instruction &I) {
   SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(DbgUsers, &I, &DPUsers);
-  for (auto *DII : DbgUsers)
-    DII->eraseFromParent();
+  assert(DbgUsers.empty());
   for (auto *DVR : DPUsers)
     DVR->eraseFromParent();
 }
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 66d0573e83f65..06115e0741ade 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -161,29 +161,8 @@ static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader,
     SmallVector<DbgValueInst *, 1> DbgValues;
     SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
     llvm::findDbgValues(DbgValues, OrigHeaderVal, &DbgVariableRecords);
-    for (auto &DbgValue : DbgValues) {
-      // The original users in the OrigHeader are already using the original
-      // definitions.
-      BasicBlock *UserBB = DbgValue->getParent();
-      if (UserBB == OrigHeader)
-        continue;
-
-      // Users in the OrigPreHeader need to use the value to which the
-      // original definitions are mapped and anything else can be handled by
-      // the SSAUpdater. To avoid adding PHINodes, check if the value is
-      // available in UserBB, if not substitute poison.
-      Value *NewVal;
-      if (UserBB == OrigPreheader)
-        NewVal = OrigPreHeaderVal;
-      else if (SSA.HasValueForBlock(UserBB))
-        NewVal = SSA.GetValueInMiddleOfBlock(UserBB);
-      else
-        NewVal = PoisonValue::get(OrigHeaderVal->getType());
-      DbgValue->replaceVariableLocationOp(OrigHeaderVal, NewVal);
-    }
+    assert(DbgValues.empty());
 
-    // RemoveDIs: duplicate implementation for non-instruction debug-info
-    // storage in DbgVariableRecords.
     for (DbgVariableRecord *DVR : DbgVariableRecords) {
       // The original users in the OrigHeader are already using the original
       // definitions.
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 154f50acf3d1d..e7623aaff105d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1103,7 +1103,6 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
       (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
        RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
        RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
-    // TODO: Add float minnum/maxnum support when FMF nnan is set.
     Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
     return Builder.CreateIntrinsic(Ty, Id, {Left, Right}, nullptr,
                                    "rdx.minmax");
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index ccd7ee360e014..73b5f48796b7a 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -190,7 +190,6 @@ class AssignmentTrackingInfo {
 };
 
 struct AllocaInfo {
-  using DbgUserVec = SmallVector<DbgVariableIntrinsic *, 1>;
   using DPUserVec = SmallVector<DbgVariableRecord *, 1>;
 
   SmallVector<BasicBlock *, 32> DefiningBlocks;
@@ -201,7 +200,6 @@ struct AllocaInfo {
   bool OnlyUsedInOneBlock;
 
   /// Debug users of the alloca - does not include dbg.assign intrinsics.
-  DbgUserVec DbgUsers;
   DPUserVec DPUsers;
   /// Helper to update assignment tracking debug info.
   AssignmentTrackingInfo AssignmentTracking;
@@ -212,7 +210,6 @@ struct AllocaInfo {
     OnlyStore = nullptr;
     OnlyBlock = nullptr;
     OnlyUsedInOneBlock = true;
-    DbgUsers.clear();
     DPUsers.clear();
     AssignmentTracking.clear();
   }
@@ -246,13 +243,10 @@ struct AllocaInfo {
           OnlyUsedInOneBlock = false;
       }
     }
-    DbgUserVec AllDbgUsers;
+    SmallVector<DbgVariableIntrinsic *> AllDbgUsers;
     SmallVector<DbgVariableRecord *> AllDPUsers;
     findDbgUsers(AllDbgUsers, AI, &AllDPUsers);
-    std::copy_if(AllDbgUsers.begin(), AllDbgUsers.end(),
-                 std::back_inserter(DbgUsers), [](DbgVariableIntrinsic *DII) {
-                   return !isa<DbgAssignIntrinsic>(DII);
-                 });
+    assert(AllDbgUsers.empty());
     std::copy_if(AllDPUsers.begin(), AllDPUsers.end(),
                  std::back_inserter(DPUsers),
                  [](DbgVariableRecord *DVR) { return !DVR->isDbgAssign(); });
@@ -380,10 +374,9 @@ struct PromoteMem2Reg {
   /// to.
   DenseMap<PHINode *, unsigned> PhiToAllocaMap;
 
-  /// For each alloca, we keep track of the dbg.declare intrinsic that
+  /// For each alloca, we keep track of the dbg.declare record that
   /// describes it, if any, so that we can convert it to a dbg.value
-  /// intrinsic if the alloca gets promoted.
-  SmallVector<AllocaInfo::DbgUserVec, 8> AllocaDbgUsers;
+  /// record if the alloca gets promoted.
   SmallVector<AllocaInfo::DPUserVec, 8> AllocaDPUsers;
 
   /// For each alloca, keep an instance of a helper class that gives us an easy
@@ -741,14 +734,11 @@ promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
   AI->eraseFromParent();
 
   // The alloca's debuginfo can be removed as well.
-  auto DbgUpdateForAlloca = [&](auto &Container) {
-    for (auto *DbgItem : Container)
-      if (DbgItem->isAddressOfVariable() ||
-          DbgItem->getExpression()->startsWithDeref())
-        DbgItem->eraseFromParent();
-  };
-  DbgUpdateForAlloca(Info.DbgUsers);
-  DbgUpdateForAlloca(Info.DPUsers);
+  for (DbgVariableRecord *DbgItem : Info.DPUsers) {
+    if (DbgItem->isAddressOfVariable() ||
+        DbgItem->getExpression()->startsWithDeref())
+      DbgItem->eraseFromParent();
+  }
 
   ++NumLocalPromoted;
   return true;
@@ -757,7 +747,6 @@ promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
 void PromoteMem2Reg::run() {
   Function &F = *DT.getRoot()->getParent();
 
-  AllocaDbgUsers.resize(Allocas.size());
   AllocaATInfo.resize(Allocas.size());
   AllocaDPUsers.resize(Allocas.size());
 
@@ -816,9 +805,7 @@ void PromoteMem2Reg::run() {
     if (BBNumPreds.empty())
       BBNumPreds.resize(F.getMaxBlockNumber());
 
-    // Remember the dbg.declare intrinsic describing this alloca, if any.
-    if (!Info.DbgUsers.empty())
-      AllocaDbgUsers[AllocaNum] = Info.DbgUsers;
+    // Remember the dbg.declare record describing this alloca, if any.
     if (!Info.AssignmentTracking.empty())
       AllocaATInfo[AllocaNum] = Info.AssignmentTracking;
     if (!Info.DPUsers.empty())
@@ -894,16 +881,12 @@ void PromoteMem2Reg::run() {
   }
 
   // Remove alloca's dbg.declare intrinsics from the function.
-  auto RemoveDbgDeclares = [&](auto &Container) {
-    for (auto &DbgUsers : Container) {
-      for (auto *DbgItem : DbgUsers)
-        if (DbgItem->isAddressOfVariable() ||
-            DbgItem->getExpression()->startsWithDeref())
-          DbgItem->eraseFromParent();
-    }
-  };
-  RemoveDbgDeclares(AllocaDbgUsers);
-  RemoveDbgDeclares(AllocaDPUsers);
+  for (auto &DbgUsers : AllocaDPUsers) {
+    for (DbgVariableRecord *DbgItem : DbgUsers)
+      if (DbgItem->isAddressOfVariable() ||
+          DbgItem->getExpression()->startsWithDeref())
+        DbgItem->eraseFromParent();
+  }
 
   // Loop over all of the PHI nodes and see if there are any that we can get
   // rid of because they merge all of the same incoming values.  This can
diff --git a/llvm/lib/Transforms/Utils/SSAUpdater.cpp b/llvm/lib/Transforms/Utils/SSAUpdater.cpp
index 5db7fc956c497..561c898ec55d8 100644
--- a/llvm/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/llvm/lib/Transforms/Utils/SSAUpdater.cpp
@@ -200,11 +200,7 @@ void SSAUpdater::UpdateDebugValues(Instruction *I) {
   SmallVector<DbgValueInst *, 4> DbgValues;
   SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
   llvm::findDbgValues(DbgValues, I, &DbgVariableRecords);
-  for (auto &DbgValue : DbgValues) {
-    if (DbgValue->getParent() == I->getParent())
-      continue;
-    UpdateDebugValue(I, DbgValue);
-  }
+  assert(DbgValues.empty());
   for (auto &DVR : DbgVariableRecords) {
     if (DVR->getParent() == I->getParent())
       continue;
@@ -212,13 +208,6 @@ void SSAUpdater::UpdateDebugValues(Instruction *I) {
   }
 }
 
-void SSAUpdater::UpdateDebugValues(Instruction *I,
-                                   SmallVectorImpl<DbgValueInst *> &DbgValues) {
-  for (auto &DbgValue : DbgValues) {
-    UpdateDebugValue(I, DbgValue);
-  }
-}
-
 void SSAUpdater::UpdateDebugValues(
     Instruction *I, SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
   for (auto &DVR : DbgVariableRecords) {
@@ -226,15 +215,6 @@ void SSAUpdater::UpdateDebugValues(
   }
 }
 
-void SSAUpdater::UpdateDebugValue(Instruction *I, DbgValueInst *DbgValue) {
-  BasicBlock *UserBB = DbgValue->getParent();
-  if (HasValueForBlock(UserBB)) {
-    Value *NewVal = GetValueAtEndOfBlock(UserBB);
-    DbgValue->replaceVariableLocationOp(I, NewVal);
-  } else
-    DbgValue->setKillLocation();
-}
-
 void SSAUpdater::UpdateDebugValue(Instruction *I, DbgVariableRecord *DVR) {
   BasicBlock *UserBB = DVR->getParent();
   if (HasValueForBlock(UserBB)) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index db0902c589d49..6e420632d83e5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -947,9 +947,8 @@ class LoopVectorizationCostModel {
   /// user options, for the given register kind.
   bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind);
 
-  /// \return True if maximizing vector bandwidth is enabled by the target or
-  /// user options, for the given vector factor.
-  bool useMaxBandwidth(ElementCount VF);
+  /// \return True if register pressure should be calculated for the given VF.
+  bool shouldCalculateRegPressureForVF(ElementCount VF);
 
   /// \return The size (in bits) of the smallest and widest types in the code
   /// that needs to be vectorized. We ignore values that remain scalar such as
@@ -1736,6 +1735,9 @@ class LoopVectorizationCostModel {
   /// Whether this loop should be optimized for size based on function attribute
   /// or profile information.
   bool OptForSize;
+
+  /// The highest VF possible for this loop, without using MaxBandwidth.
+  FixedScalableVFPair MaxPermissibleVFWithoutMaxBW;
 };
 } // end namespace llvm
 
@@ -3832,10 +3834,16 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   return FixedScalableVFPair::getNone();
 }
 
-bool LoopVectorizationCostModel::useMaxBandwidth(ElementCount VF) {
-  return useMaxBandwidth(VF.isScalable()
-                             ? TargetTransformInfo::RGK_ScalableVector
-                             : TargetTransformInfo::RGK_FixedWidthVector);
+bool LoopVectorizationCostModel::shouldCalculateRegPressureForVF(
+    ElementCount VF) {
+  if (!useMaxBandwidth(VF.isScalable()
+                           ? TargetTransformInfo::RGK_ScalableVector
+                           : TargetTransformInfo::RGK_FixedWidthVector))
+    return false;
+  // Only calculate register pressure for VFs enabled by MaxBandwidth.
+  return ElementCount::isKnownGT(
+      VF, VF.isScalable() ? MaxPermissibleVFWithoutMaxBW.ScalableVF
+                          : MaxPermissibleVFWithoutMaxBW.FixedVF);
 }
 
 bool LoopVectorizationCostModel::useMaxBandwidth(
@@ -3911,6 +3919,12 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
       ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
                            : TargetTransformInfo::RGK_FixedWidthVector;
   ElementCount MaxVF = MaxVectorElementCount;
+
+  if (MaxVF.isScalable())
+    MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
+  else
+    MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
+
   if (useMaxBandwidth(RegKind)) {
     auto MaxVectorElementCountMaxBW = ElementCount::get(
         llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
@@ -4264,9 +4278,11 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
       if (VF.isScalar())
         continue;
 
-      /// Don't consider the VF if it exceeds the number of registers for the
-      /// target.
-      if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI))
+      /// If the register pressure needs to be considered for VF,
+      /// don't consider the VF as valid if it exceeds the number
+      /// of registers for the target.
+      if (CM.shouldCalculateRegPressureForVF(VF) &&
+          RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs))
         continue;
 
       InstructionCost C = CM.expectedCost(VF);
@@ -7048,7 +7064,8 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
       InstructionCost Cost = cost(*P, VF);
       VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
 
-      if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI)) {
+      if (CM.shouldCalculateRegPressureForVF(VF) &&
+          RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs)) {
         LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
                           << VF << " because it uses too many registers\n");
         continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 66657b98b094b..3499e650ae853 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -405,9 +405,12 @@ static unsigned getVFScaleFactor(VPRecipeBase *R) {
   return 1;
 }
 
-bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI) const {
-  return any_of(MaxLocalUsers, [&TTI](auto &LU) {
-    return LU.second > TTI.getNumberOfRegisters(LU.first);
+bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI,
+                                        unsigned OverrideMaxNumRegs) const {
+  return any_of(MaxLocalUsers, [&TTI, &OverrideMaxNumRegs](auto &LU) {
+    return LU.second > (OverrideMaxNumRegs > 0
+                            ? OverrideMaxNumRegs
+                            : TTI.getNumberOfRegisters(LU.first));
   });
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 7bcf9dba8c311..cd86d27cf9122 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -85,8 +85,10 @@ struct VPRegisterUsage {
   SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
 
   /// Check if any of the tracked live intervals exceeds the number of
-  /// available registers for the target.
-  bool exceedsMaxNumRegs(const TargetTransformInfo &TTI) const;
+  /// available registers for the target. If non-zero, OverrideMaxNumRegs
+  /// is used in place of the target's number of registers.
+  bool exceedsMaxNumRegs(const TargetTransformInfo &TTI,
+                         unsigned OverrideMaxNumRegs = 0) const;
 };
 
 /// Estimate the register usage for \p Plan and vectorization factors in \p VFs
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
index 4153f0be611a1..9698f1a6768fd 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
@@ -231,3 +231,27 @@ define <vscale x 8 x i64> @sload_8i8_8i64(ptr %a) {
   %aext = sext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
   ret <vscale x 8 x i64> %aext
 }
+
+; Ensure we don't try to promote a predicate load to a sign-extended load.
+define <vscale x 16 x i8> @sload_16i1_16i8(ptr %addr) {
+; CHECK-LABEL: sload_16i1_16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr p0, [x0]
+; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    ret
+  %load = load <vscale x 16 x i1>, ptr %addr
+  %zext = sext <vscale x 16 x i1> %load to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %zext
+}
+
+; Ensure we don't try to promote a predicate load to a zero-extended load.
+define <vscale x 16 x i8> @zload_16i1_16i8(ptr %addr) {
+; CHECK-LABEL: zload_16i1_16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr p0, [x0]
+; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    ret
+  %load = load <vscale x 16 x i1>, ptr %addr
+  %zext = zext <vscale x 16 x i1> %load to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %zext
+}
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
index 6742ae6c1d584..f6465de86fa4f 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
@@ -6,17 +6,17 @@
 define amdgpu_kernel void @remat_constant_voids_spill(ptr addrspace(1) %p) #1 {
 ; GFX908-LABEL: remat_constant_voids_spill:
 ; GFX908:       ; %bb.0:
-; GFX908-NEXT:    v_accvgpr_write_b32 a1, 1
-; GFX908-NEXT:    v_accvgpr_write_b32 a5, 6
-; GFX908-NEXT:    v_accvgpr_write_b32 a6, 7
-; GFX908-NEXT:    v_accvgpr_write_b32 a7, 8
-; GFX908-NEXT:    v_accvgpr_write_b32 a0, 9
-; GFX908-NEXT:    v_accvgpr_write_b32 a2, 2
-; GFX908-NEXT:    v_accvgpr_write_b32 a3, 3
-; GFX908-NEXT:    v_accvgpr_write_b32 a4, 4
+; GFX908-NEXT:    v_accvgpr_write_b32 a0, 1
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, 2
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 3
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, 4
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_accvgpr_write_b32 a1, 5
+; GFX908-NEXT:    v_accvgpr_write_b32 a0, 5
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, 6
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 7
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, 8
+; GFX908-NEXT:    v_accvgpr_write_b32 a4, 9
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ;;#ASMEND
 ; GFX908-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
new file mode 100644
index 0000000000000..69bdb1f5066f0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
@@ -0,0 +1,95 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX90A %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX908 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx906 -passes=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefix=NO-AGPR %s
+
+--- |
+  define void @func() {
+    ret void
+  }
+
+  ; Attribute is ignored for gfx90a
+  define void @no_agprs() "amdgpu-agpr-alloc"="0,0" {
+    ret void
+  }
+
+...
+---
+name: func
+tracksRegLiveness: true
+stack:
+  - { id: 0, size: 4 }
+body:             |
+  ; HAS-AGPR-LABEL: name: func
+  ; HAS-AGPR: bb.0:
+  ; HAS-AGPR-NEXT:   successors: %bb.1(0x80000000)
+  ; HAS-AGPR-NEXT:   liveins: $vgpr0
+  ; HAS-AGPR-NEXT: {{  $}}
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_2:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 6, implicit $exec
+  ; HAS-AGPR-NEXT: {{  $}}
+  ; HAS-AGPR-NEXT: bb.1:
+  ; HAS-AGPR-NEXT:   [[AV_MOV_3:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 3, implicit $exec
+  ;
+  ; NO-AGPR-LABEL: name: func
+  ; NO-AGPR: bb.0:
+  ; NO-AGPR-NEXT:   successors: %bb.1(0x80000000)
+  ; NO-AGPR-NEXT:   liveins: $vgpr0
+  ; NO-AGPR-NEXT: {{  $}}
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_2:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec
+  ; NO-AGPR-NEXT: {{  $}}
+  ; NO-AGPR-NEXT: bb.1:
+  ; NO-AGPR-NEXT:   [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  bb.0:
+    liveins: $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+    %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+    %4:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %5:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    %6:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec
+
+  bb.1:
+    %7:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+
+...
+
+---
+name: no_agprs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+    ; GFX90A-LABEL: name: no_agprs
+    ; GFX90A: liveins: $vgpr0
+    ; GFX90A-NEXT: {{  $}}
+    ; GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX90A-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    ;
+    ; GFX908-LABEL: name: no_agprs
+    ; GFX908: liveins: $vgpr0
+    ; GFX908-NEXT: {{  $}}
+    ; GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+    ; GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+    ;
+    ; NO-AGPR-LABEL: name: no_agprs
+    ; NO-AGPR: liveins: $vgpr0
+    ; NO-AGPR-NEXT: {{  $}}
+    ; NO-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    %0:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
new file mode 100644
index 0000000000000..1a457c94778fd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
@@ -0,0 +1,74 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+
+---
+name:            bad_ra
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sgpr_64, preferred-register: '$sgpr4_sgpr5' }
+  - { id: 1, class: sgpr_128, preferred-register: '%2' }
+  - { id: 2, class: areg_128, preferred-register: '%1' }
+  - { id: 3, class: areg_128, preferred-register: '%4' }
+  - { id: 4, class: av_128, preferred-register: '%3' }
+  - { id: 5, class: areg_128, preferred-register: '%6' }
+  - { id: 6, class: vreg_128, preferred-register: '%5' }
+  - { id: 7, class: areg_128, preferred-register: '%4' }
+  - { id: 8, class: vgpr_32 }
+  - { id: 9, class: vgpr_32 }
+  - { id: 10, class: vgpr_32 }
+  - { id: 11, class: areg_128 }
+liveins:
+  - { reg: '$sgpr4_sgpr5', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  vgprForAGPRCopy: '$vgpr255'
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  bb.0:
+    liveins: $sgpr4_sgpr5
+
+    ; CHECK-LABEL: name: bad_ra
+    ; CHECK: liveins: $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1)
+    ; CHECK-NEXT: renamable $vgpr4 = V_MOV_B32_e32 1065353216, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr5 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr6 = V_MOV_B32_e32 1073741824, implicit $exec
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr1 = COPY renamable $agpr1
+    ; CHECK-NEXT: renamable $vgpr0 = COPY renamable $agpr0
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr3 = COPY renamable $agpr1
+    ; CHECK-NEXT: renamable $vgpr2 = COPY killed renamable $agpr0
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 killed $vgpr4, killed $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr5, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+    renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1)
+    %8:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+    %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %10:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec
+    %2:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
+    %3:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %2, 0, 0, 0, implicit $mode, implicit $exec
+    undef %4.sub1:av_128 = COPY %3.sub1
+    %4.sub0:av_128 = COPY %3.sub0
+    %11:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %3, 0, 0, 0, implicit $mode, implicit $exec
+    %4.sub3:av_128 = COPY %11.sub1
+    %4.sub2:av_128 = COPY %11.sub0
+    %7:areg_128 = COPY %4
+    %5:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %7, 0, 0, 0, implicit $mode, implicit $exec
+    %6:vreg_128 = COPY %5
+    GLOBAL_STORE_DWORDX4_SADDR %9, %6, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index ae90cfb631e8d..7eb7d72e6cb97 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -25,7 +25,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr17, 8, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_CSELECT_B64 -1, 0, implicit killed $scc
   ; GFX90A-NEXT:   renamable $sgpr30_sgpr31 = S_XOR_B64 killed renamable $sgpr18_sgpr19, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = DS_READ_B32_gfx9 renamable $vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3)
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr28_sgpr29, implicit-def dead $scc
@@ -56,8 +56,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr30 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $vgpr15 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr17 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr15 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr17 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_VCCZ %bb.57, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.4.bb15:
@@ -112,14 +112,14 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   successors: %bb.7(0x80000000)
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr21 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr20 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr23 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr22 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr25 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr24 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr19 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr18 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr21 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr20 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr23 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr22 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr25 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr24 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.7.Flow19:
   ; GFX90A-NEXT:   successors: %bb.62(0x40000000), %bb.8(0x40000000)
@@ -671,7 +671,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr50_sgpr51, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr30, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.54, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
@@ -759,7 +759,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr53 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $sgpr64_sgpr65, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr10 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr14_vgpr15 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr21, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from %ir.7, addrspace 3)
@@ -801,12 +801,12 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr42_vgpr43 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr40_vgpr41 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr46_vgpr47 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr52 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr16 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr53 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr14 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr52 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr16 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr53 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr13 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr12 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_BRANCH %bb.7
   ; GFX90A-NEXT: {{  $}}
@@ -814,7 +814,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   successors: %bb.3(0x80000000)
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3)
@@ -913,7 +913,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr25, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr3 = V_OR_B32_e32 killed $vgpr11, killed $vgpr19, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U32_sdwa 0, killed $vgpr53, 0, $vgpr3, 0, 0, 6, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr10 = V_OR_B32_e32 killed $vgpr52, killed $vgpr13, implicit $exec
@@ -955,7 +955,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr26, $vgpr2, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr3, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr10, killed $vgpr3, killed $vcc, 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr27 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr27 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr49 = COPY renamable $vgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr35 = COPY renamable $vgpr27, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr39 = COPY renamable $vgpr27, implicit $exec
@@ -989,7 +989,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr10 = V_OR_B32_e32 $vgpr50, killed $vgpr16, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr54 = V_OR_B32_e32 killed $vgpr10, killed $vgpr14, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr55 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr55 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 killed renamable $vgpr55, renamable $vgpr54_vgpr55, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_BRANCH %bb.69
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 50fa7ac2a19aa..4f81d3599d1de 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -10,9 +10,9 @@
 
 ; GCN-O0: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(atomic-expand,verify,gc-lowering,lower-constant-intrinsics,unreachableblockelim,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(lower-switch,lower-invoke,unreachableblockelim,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa,require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,localstackalloc))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,phi-node-elimination,two-address-instruction,regallocfast,si-fix-vgpr-copies,remove-redundant-debug-values,fixup-statepoint-caller-saved,prolog-epilog,post-ra-pseudos,si-post-ra-bundler,fentry-insert,xray-instrumentation,patchable-function,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
-; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
+; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
-; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
+; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
 
 define void @empty() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index af3241e95e91d..2a5c65278f7dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -329,6 +329,7 @@
 ; GCN-O1-NEXT:        Remove dead machine instructions
 ; GCN-O1-NEXT:        SI Shrink Instructions
 ; GCN-O1-NEXT:        Register Usage Information Propagation
+; GCN-O1-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O1-NEXT:        Detect Dead Lanes
 ; GCN-O1-NEXT:        Remove dead machine instructions
 ; GCN-O1-NEXT:        Init Undef Pass
@@ -640,6 +641,7 @@
 ; GCN-O1-OPTS-NEXT:        Remove dead machine instructions
 ; GCN-O1-OPTS-NEXT:        SI Shrink Instructions
 ; GCN-O1-OPTS-NEXT:        Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O1-OPTS-NEXT:        Detect Dead Lanes
 ; GCN-O1-OPTS-NEXT:        Remove dead machine instructions
 ; GCN-O1-OPTS-NEXT:        Init Undef Pass
@@ -956,6 +958,7 @@
 ; GCN-O2-NEXT:        Remove dead machine instructions
 ; GCN-O2-NEXT:        SI Shrink Instructions
 ; GCN-O2-NEXT:        Register Usage Information Propagation
+; GCN-O2-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O2-NEXT:        Detect Dead Lanes
 ; GCN-O2-NEXT:        Remove dead machine instructions
 ; GCN-O2-NEXT:        Init Undef Pass
@@ -1286,6 +1289,7 @@
 ; GCN-O3-NEXT:        Remove dead machine instructions
 ; GCN-O3-NEXT:        SI Shrink Instructions
 ; GCN-O3-NEXT:        Register Usage Information Propagation
+; GCN-O3-NEXT:        AMDGPU Prepare AGPR Alloc
 ; GCN-O3-NEXT:        Detect Dead Lanes
 ; GCN-O3-NEXT:        Remove dead machine instructions
 ; GCN-O3-NEXT:        Init Undef Pass
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 67ae05eb6f0b8..561eaca3b77df 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -4365,8 +4365,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
 ; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
 ; NOLIT-SRCC:       ; %bb.0: ; %bb
 ; NOLIT-SRCC-NEXT:    v_mov_b32_e32 v0, 1.0
-; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a0, 1.0
+; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a2, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a3, 0
 ; NOLIT-SRCC-NEXT:    v_accvgpr_write_b32 a4, 0
@@ -4465,8 +4465,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
 ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
 ; LIT-SRCC:       ; %bb.0: ; %bb
 ; LIT-SRCC-NEXT:    v_mov_b32_e32 v0, 1.0
-; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a0, 1.0
+; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a1, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a2, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a3, 0
 ; LIT-SRCC-NEXT:    v_accvgpr_write_b32 a4, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
index 2faf375a97a86..465414c5471ee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s
 
 declare i32 @llvm.amdgcn.prng.b32(i32) #0
 
@@ -29,4 +31,4 @@ define amdgpu_kernel void @prng_b32_constant_100(ptr addrspace(1) %out) #1 {
 
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind }
\ No newline at end of file
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
index 3844d6054e130..cf244f0b1f884 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
@@ -6,16 +6,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
 ; GFX942-LABEL: matmul_kernel:
 ; GFX942:       ; %bb.0: ; %entry
 ; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942-NEXT:    v_mov_b32_e32 v0, 0
-; GFX942-NEXT:    v_accvgpr_write_b32 a2, v0
+; GFX942-NEXT:    v_accvgpr_write_b32 a2, 0
 ; GFX942-NEXT:    s_mov_b32 s2, 0
 ; GFX942-NEXT:    v_accvgpr_write_b32 a1, 0
+; GFX942-NEXT:    s_mov_b32 s3, 0
 ; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX942-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX942-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
 ; GFX942-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX942-NEXT:    s_mov_b32 s3, 0
 ; GFX942-NEXT:    s_branch .LBB0_2
 ; GFX942-NEXT:  .LBB0_1: ; %bb2
 ; GFX942-NEXT:    ; in Loop: Header=BB0_2 Depth=1
@@ -43,16 +42,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
 ; GFX908-LABEL: matmul_kernel:
 ; GFX908:       ; %bb.0: ; %entry
 ; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX908-NEXT:    v_mov_b32_e32 v0, 0
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 0
 ; GFX908-NEXT:    v_accvgpr_write_b32 a1, 0
 ; GFX908-NEXT:    s_mov_b32 s2, 0
-; GFX908-NEXT:    v_accvgpr_write_b32 a2, v0
+; GFX908-NEXT:    s_mov_b32 s3, 0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX908-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX908-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX908-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
 ; GFX908-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX908-NEXT:    s_mov_b32 s3, 0
 ; GFX908-NEXT:    s_branch .LBB0_2
 ; GFX908-NEXT:  .LBB0_1: ; %bb2
 ; GFX908-NEXT:    ; in Loop: Header=BB0_2 Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
index ee5481617cf59..01506d0af1913 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
@@ -80,16 +80,16 @@ body:             |
   ; COALESCE-NEXT:   S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
   ; COALESCE-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0
   ; COALESCE-NEXT:   [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec
   ; COALESCE-NEXT:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.1:
   ; COALESCE-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; COALESCE-NEXT: {{  $}}
-  ; COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
   ; COALESCE-NEXT:   $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; COALESCE-NEXT:   S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
@@ -103,10 +103,10 @@ body:             |
   ; COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]]
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.3:
@@ -134,16 +134,16 @@ body:             |
   ; GFX908-COALESCE-NEXT:   S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
   ; GFX908-COALESCE-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0
   ; GFX908-COALESCE-NEXT:   [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.1:
   ; GFX908-COALESCE-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
   ; GFX908-COALESCE-NEXT:   $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; GFX908-COALESCE-NEXT:   S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
@@ -157,10 +157,10 @@ body:             |
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]]
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
index 49c0aaf9fb390..a9207de317ea1 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
@@ -67,7 +67,7 @@ body:             |
   ; COALESCE-NEXT: bb.1:
   ; COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; COALESCE-NEXT: {{  $}}
-  ; COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   S_BRANCH %bb.3
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.2:
@@ -78,13 +78,13 @@ body:             |
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], 0, 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
-  ; COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+  ; COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
   ; COALESCE-NEXT: {{  $}}
   ; COALESCE-NEXT: bb.3:
-  ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; COALESCE-NEXT:   [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec
   ; COALESCE-NEXT:   undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec
-  ; COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+  ; COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
@@ -105,28 +105,28 @@ body:             |
   ; GFX908-COALESCE-NEXT: bb.1:
   ; GFX908-COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   S_BRANCH %bb.3
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.2:
   ; GFX908-COALESCE-NEXT:   successors: %bb.3(0x80000000)
   ; GFX908-COALESCE-NEXT: {{  $}}
-  ; GFX908-COALESCE-NEXT:   undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
+  ; GFX908-COALESCE-NEXT:   undef [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[AV_MOV_1]].sub0
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub0
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]].sub0_sub1
-  ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT: {{  $}}
   ; GFX908-COALESCE-NEXT: bb.3:
-  ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+  ; GFX908-COALESCE-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
   ; GFX908-COALESCE-NEXT:   [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec
   ; GFX908-COALESCE-NEXT:   undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec
-  ; GFX908-COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX908-COALESCE-NEXT:   [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; GFX908-COALESCE-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 663fd98b46bf7..ce96766116089 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -17,9 +17,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
@@ -42,8 +42,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX908-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
-  ; PEI-GFX908-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-  ; PEI-GFX908-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+  ; PEI-GFX908-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; PEI-GFX908-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
   ; PEI-GFX908-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
   ; PEI-GFX908-NEXT:   $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
@@ -62,9 +62,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX90A-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; REGALLOC-GFX90A-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; REGALLOC-GFX90A-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; REGALLOC-GFX90A-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; REGALLOC-GFX90A-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64_align2, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64_align2, [[V_MFMA_I32_4X4X4I8_e64_]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   S_ENDPGM 0
@@ -85,8 +85,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX90A-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
-  ; PEI-GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-  ; PEI-GFX90A-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+  ; PEI-GFX90A-NEXT:   renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; PEI-GFX90A-NEXT:   renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
   ; PEI-GFX90A-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
   ; PEI-GFX90A-NEXT:   $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index 1ec94162951a6..d48bfe0bb7f21 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -145,49 +145,29 @@ entry:
 
 ; Test skipping the lower-32-bit addition if it is unnecessary.
 define ptr @huge_offset_low_32_unused(ptr %p) {
-; GFX942_PTRADD-LABEL: huge_offset_low_32_unused:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    s_mov_b32 s0, 0
-; GFX942_PTRADD-NEXT:    s_mov_b32 s1, 1
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: huge_offset_low_32_unused:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+; GFX942-LABEL: huge_offset_low_32_unused:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_add_u32_e32 v1, 1, v1
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr inbounds i8, ptr %p, i64 u0x100000000
   ret ptr %gep
 }
 
 ; Reassociate address computation if it leads to more scalar operations.
 define amdgpu_kernel void @reassoc_scalar_r(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
-; GFX942_PTRADD-LABEL: reassoc_scalar_r:
-; GFX942_PTRADD:       ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_PTRADD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], v[0:1], 0, s[6:7]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: reassoc_scalar_r:
-; GFX942_LEGACY:       ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    s_add_u32 s2, s2, s6
-; GFX942_LEGACY-NEXT:    s_addc_u32 s3, s3, s7
-; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: reassoc_scalar_r:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_u32 s2, s2, s6
+; GFX942-NEXT:    s_addc_u32 s3, s3, s7
+; GFX942-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
 entry:
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
@@ -198,30 +178,18 @@ entry:
 }
 
 define amdgpu_kernel void @reassoc_scalar_l(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
-; GFX942_PTRADD-LABEL: reassoc_scalar_l:
-; GFX942_PTRADD:       ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_PTRADD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[6:7], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: reassoc_scalar_l:
-; GFX942_LEGACY:       ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, 0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    s_add_u32 s2, s2, s6
-; GFX942_LEGACY-NEXT:    s_addc_u32 s3, s3, s7
-; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: reassoc_scalar_l:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_u32 s2, s2, s6
+; GFX942-NEXT:    s_addc_u32 s3, s3, s7
+; GFX942-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
 entry:
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
@@ -233,24 +201,14 @@ entry:
 
 ; Tests the target-specific (ptradd x, shl(0 - y, k)) -> sub(x, shl(y, k)) fold
 define ptr addrspace(1) @shl_neg_offset(ptr addrspace(1) %p, i64 %noffset, i64 %shift) {
-; GFX942_PTRADD-LABEL: shl_neg_offset:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_sub_co_u32_e32 v2, vcc, 0, v2
-; GFX942_PTRADD-NEXT:    s_nop 1
-; GFX942_PTRADD-NEXT:    v_subb_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: shl_neg_offset:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX942_LEGACY-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
-; GFX942_LEGACY-NEXT:    s_nop 1
-; GFX942_LEGACY-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+; GFX942-LABEL: shl_neg_offset:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX942-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %offset = sub i64 0, %noffset
   %x = shl i64 %offset, %shift
   %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 %x
@@ -268,10 +226,9 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
 ; GFX942_PTRADD:       ; %bb.0:
 ; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942_PTRADD-NEXT:    s_getpc_b64 s[0:1]
-; GFX942_PTRADD-NEXT:    s_add_u32 s0, s0, v0 at rel32@lo+4
-; GFX942_PTRADD-NEXT:    s_addc_u32 s1, s1, v0 at rel32@hi+12
+; GFX942_PTRADD-NEXT:    s_add_u32 s0, s0, v0 at rel32@lo+14
+; GFX942_PTRADD-NEXT:    s_addc_u32 s1, s1, v0 at rel32@hi+22
 ; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 10
 ; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942_LEGACY-LABEL: complextype_global_gep:
@@ -291,30 +248,63 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
 
 ; Tests the tryFoldToMad64_32 PTRADD combine.
 define amdgpu_kernel void @fold_mad64(ptr addrspace(1) %p) {
-; GFX942_PTRADD-LABEL: fold_mad64:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT:    v_mul_hi_u32_u24_e32 v1, 12, v0
-; GFX942_PTRADD-NEXT:    v_mul_u32_u24_e32 v0, 12, v0
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, 1.0
-; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX942_PTRADD-NEXT:    global_store_dword v[0:1], v2, off
-; GFX942_PTRADD-NEXT:    s_endpgm
-;
-; GFX942_LEGACY-LABEL: fold_mad64:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, 1.0
-; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
-; GFX942_LEGACY-NEXT:    global_store_dword v[0:1], v2, off
-; GFX942_LEGACY-NEXT:    s_endpgm
+; GFX942-LABEL: fold_mad64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 1.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
+; GFX942-NEXT:    global_store_dword v[0:1], v2, off
+; GFX942-NEXT:    s_endpgm
   %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
   %voffset = zext i32 %voffset32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %p, i64 %voffset, i32 0
   store float 1.0, ptr addrspace(1) %p1
   ret void
 }
+
+; Use non-zero shift amounts in v_lshl_add_u64.
+define ptr @select_v_lshl_add_u64(ptr %base, i64 %voffset) {
+; GFX942_PTRADD-LABEL: select_v_lshl_add_u64:
+; GFX942_PTRADD:       ; %bb.0:
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], 3, v[2:3]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: select_v_lshl_add_u64:
+; GFX942_LEGACY:       ; %bb.0:
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[2:3], 3, v[0:1]
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr inbounds i64, ptr %base, i64 %voffset
+  ret ptr %gep
+}
+
+; Fold mul and add into v_mad, even if amdgpu-codegenprepare-mul24 turned the
+; mul into a mul24.
+define ptr @fold_mul24_into_mad(ptr %base, i64 %a, i64 %b) {
+; GFX942_PTRADD-LABEL: fold_mul24_into_mad:
+; GFX942_PTRADD:       ; %bb.0:
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v2, 0xfffff, v2
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v4, 0xfffff, v4
+; GFX942_PTRADD-NEXT:    v_mul_hi_u32_u24_e32 v3, v2, v4
+; GFX942_PTRADD-NEXT:    v_mul_u32_u24_e32 v2, v2, v4
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: fold_mul24_into_mad:
+; GFX942_LEGACY:       ; %bb.0:
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v2, 0xfffff, v2
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v3, 0xfffff, v4
+; GFX942_LEGACY-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v2, v3, v[0:1]
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+  %a_masked = and i64 %a, u0xfffff
+  %b_masked = and i64 %b, u0xfffff
+  %mul = mul i64 %a_masked, %b_masked
+  %gep = getelementptr inbounds i8, ptr %base, i64 %mul
+  ret ptr %gep
+}
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index bd255e88b9512..648b59f69ea79 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -9,9 +9,9 @@ define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; GCN-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-  ; GCN-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
-  ; GCN-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GCN-NEXT:   [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+  ; GCN-NEXT:   [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+  ; GCN-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; GCN-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 2228234 /* regdef:VGPR_32 */, def undef %14.sub0
   ; GCN-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
   ; GCN-NEXT:   GLOBAL_STORE_DWORDX4 undef %24:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
index d07e2914c753a..f7653af1fa9ba 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
@@ -122,23 +122,23 @@ define i64 @callee_large_scalars(i256 %a, i256 %b) nounwind {
 define i64 @caller_large_scalars() nounwind {
 ; CHECK-LABEL: caller_large_scalars:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $zero, $sp, 24
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $zero, $sp, 40
 ; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 8
+; CHECK-NEXT:    vst $vr0, $sp, 24
 ; CHECK-NEXT:    ori $a0, $zero, 2
-; CHECK-NEXT:    st.d $a0, $sp, 0
-; CHECK-NEXT:    st.d $zero, $sp, 56
-; CHECK-NEXT:    vst $vr0, $sp, 40
+; CHECK-NEXT:    st.d $a0, $sp, 16
+; CHECK-NEXT:    st.d $zero, $sp, 72
+; CHECK-NEXT:    vst $vr0, $sp, 56
 ; CHECK-NEXT:    ori $a2, $zero, 1
-; CHECK-NEXT:    addi.d $a0, $sp, 32
-; CHECK-NEXT:    addi.d $a1, $sp, 0
-; CHECK-NEXT:    st.d $a2, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 48
+; CHECK-NEXT:    addi.d $a1, $sp, 16
+; CHECK-NEXT:    st.d $a2, $sp, 48
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(callee_large_scalars)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %1 = call i64 @callee_large_scalars(i256 1, i256 2)
   ret i64 %1
@@ -177,20 +177,20 @@ define i64 @callee_large_scalars_exhausted_regs(i64 %a, i64 %b, i64 %c, i64 %d,
 define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; CHECK-LABEL: caller_large_scalars_exhausted_regs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -96
-; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $a0, $sp, 16
+; CHECK-NEXT:    addi.d $sp, $sp, -112
+; CHECK-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    st.d $a0, $sp, 8
 ; CHECK-NEXT:    ori $a0, $zero, 9
 ; CHECK-NEXT:    st.d $a0, $sp, 0
-; CHECK-NEXT:    st.d $zero, $sp, 40
+; CHECK-NEXT:    st.d $zero, $sp, 56
 ; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 24
+; CHECK-NEXT:    vst $vr0, $sp, 40
 ; CHECK-NEXT:    ori $a0, $zero, 10
-; CHECK-NEXT:    st.d $a0, $sp, 16
-; CHECK-NEXT:    st.d $zero, $sp, 72
+; CHECK-NEXT:    st.d $a0, $sp, 32
+; CHECK-NEXT:    st.d $zero, $sp, 88
 ; CHECK-NEXT:    ori $a0, $zero, 8
-; CHECK-NEXT:    st.d $a0, $sp, 48
+; CHECK-NEXT:    st.d $a0, $sp, 64
 ; CHECK-NEXT:    ori $a0, $zero, 1
 ; CHECK-NEXT:    ori $a1, $zero, 2
 ; CHECK-NEXT:    ori $a2, $zero, 3
@@ -198,12 +198,12 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; CHECK-NEXT:    ori $a4, $zero, 5
 ; CHECK-NEXT:    ori $a5, $zero, 6
 ; CHECK-NEXT:    ori $a6, $zero, 7
-; CHECK-NEXT:    addi.d $a7, $sp, 48
-; CHECK-NEXT:    vst $vr0, $sp, 56
+; CHECK-NEXT:    addi.d $a7, $sp, 64
+; CHECK-NEXT:    vst $vr0, $sp, 72
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(callee_large_scalars_exhausted_regs)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 96
+; CHECK-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 112
 ; CHECK-NEXT:    ret
   %1 = call i64 @callee_large_scalars_exhausted_regs(
       i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i256 8, i64 9,
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-half.ll b/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
index c88b67f13d1e7..da8c3e93f6842 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
@@ -1252,8 +1252,8 @@ define i32 @caller_half_on_stack() nounwind {
 ;
 ; LA64F-LP64S-LABEL: caller_half_on_stack:
 ; LA64F-LP64S:       # %bb.0:
-; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -80
-; LA64F-LP64S-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -96
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
 ; LA64F-LP64S-NEXT:    lu12i.w $a0, -12
 ; LA64F-LP64S-NEXT:    ori $a1, $a0, 3200
 ; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
@@ -1292,8 +1292,8 @@ define i32 @caller_half_on_stack() nounwind {
 ; LA64F-LP64S-NEXT:    st.w $t0, $sp, 0
 ; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
 ; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
-; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 80
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 96
 ; LA64F-LP64S-NEXT:    ret
 ;
 ; LA64F-LP64D-LABEL: caller_half_on_stack:
@@ -1336,8 +1336,8 @@ define i32 @caller_half_on_stack() nounwind {
 ;
 ; LA64D-LP64S-LABEL: caller_half_on_stack:
 ; LA64D-LP64S:       # %bb.0:
-; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -80
-; LA64D-LP64S-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -96
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
 ; LA64D-LP64S-NEXT:    lu12i.w $a0, -12
 ; LA64D-LP64S-NEXT:    ori $a1, $a0, 3200
 ; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
@@ -1376,8 +1376,8 @@ define i32 @caller_half_on_stack() nounwind {
 ; LA64D-LP64S-NEXT:    st.w $t0, $sp, 0
 ; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
 ; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
-; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 80
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 96
 ; LA64D-LP64S-NEXT:    ret
 ;
 ; LA64D-LP64D-LABEL: caller_half_on_stack:
diff --git a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
index 52d8dd05aaa4c..1a9de3b0ef3d1 100644
--- a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
+++ b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
@@ -14,41 +14,41 @@
 define dso_local noundef signext i32 @main() nounwind {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -272
-; CHECK-NEXT:    st.d $ra, $sp, 264 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -288
+; CHECK-NEXT:    st.d $ra, $sp, 280 # 8-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
 ; CHECK-NEXT:    xvld $xr0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    xvst $xr0, $sp, 96 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 112 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_1)
 ; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI0_1)
-; CHECK-NEXT:    xvst $xr1, $sp, 64 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr1, $sp, 80 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_2)
 ; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_2)
-; CHECK-NEXT:    xvst $xr2, $sp, 32 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr2, $sp, 48 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_3)
 ; CHECK-NEXT:    xvld $xr3, $a0, %pc_lo12(.LCPI0_3)
-; CHECK-NEXT:    xvst $xr3, $sp, 0 # 32-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 136
-; CHECK-NEXT:    xvst $xr1, $sp, 168
-; CHECK-NEXT:    xvst $xr2, $sp, 200
-; CHECK-NEXT:    xvst $xr3, $sp, 232
-; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    xvst $xr3, $sp, 16 # 32-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 152
+; CHECK-NEXT:    xvst $xr1, $sp, 184
+; CHECK-NEXT:    xvst $xr2, $sp, 216
+; CHECK-NEXT:    xvst $xr3, $sp, 248
+; CHECK-NEXT:    addi.d $a0, $sp, 152
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(foo)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 96 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 136
-; CHECK-NEXT:    xvld $xr0, $sp, 64 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 168
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 200
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
-; CHECK-NEXT:    xvst $xr0, $sp, 232
-; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    xvld $xr0, $sp, 112 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 152
+; CHECK-NEXT:    xvld $xr0, $sp, 80 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 184
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 216
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 248
+; CHECK-NEXT:    addi.d $a0, $sp, 152
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(bar)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    move $a0, $zero
-; CHECK-NEXT:    ld.d $ra, $sp, 264 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 272
+; CHECK-NEXT:    ld.d $ra, $sp, 280 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 288
 ; CHECK-NEXT:    ret
 entry:
   %s = alloca %struct.S, align 2
diff --git a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
index ccc5c703e71ed..15ac95dfc6c55 100644
--- a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
+++ b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
@@ -28,12 +28,12 @@ define void @func() {
 ; CHECK-NEXT:    ld.w $a3, $a1, 0
 ; CHECK-NEXT:    ld.w $a2, $a1, 0
 ; CHECK-NEXT:    ld.w $a0, $a1, 0
-; CHECK-NEXT:    st.d $fp, $sp, 0
+; CHECK-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
 ; CHECK-NEXT:    lu12i.w $fp, 1
 ; CHECK-NEXT:    ori $fp, $fp, 12
 ; CHECK-NEXT:    add.d $fp, $sp, $fp
 ; CHECK-NEXT:    st.w $t8, $fp, 0
-; CHECK-NEXT:    ld.d $fp, $sp, 0
+; CHECK-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
 ; CHECK-NEXT:    st.w $t8, $a1, 0
 ; CHECK-NEXT:    st.w $t7, $a1, 0
 ; CHECK-NEXT:    st.w $t6, $a1, 0
diff --git a/llvm/test/CodeGen/LoongArch/frame.ll b/llvm/test/CodeGen/LoongArch/frame.ll
index 048703029d8c6..b29d8634854f3 100644
--- a/llvm/test/CodeGen/LoongArch/frame.ll
+++ b/llvm/test/CodeGen/LoongArch/frame.ll
@@ -1,5 +1,6 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+d,-lsx < %s | FileCheck %s --check-prefixes=CHECK,NOLSX
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lsx < %s | FileCheck %s --check-prefixes=CHECK,LSX
 
 %struct.key_t = type { i32, [16 x i8] }
 
@@ -7,20 +8,35 @@ declare void @llvm.memset.p0.i64(ptr, i8, i64, i1)
 declare void @test1(ptr)
 
 define i32 @test() nounwind {
-; CHECK-LABEL: test:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -32
-; CHECK-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; CHECK-NEXT:    st.w $zero, $sp, 16
-; CHECK-NEXT:    vrepli.b $vr0, 0
-; CHECK-NEXT:    vst $vr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 4
-; CHECK-NEXT:    pcaddu18i $ra, %call36(test1)
-; CHECK-NEXT:    jirl $ra, $ra, 0
-; CHECK-NEXT:    move $a0, $zero
-; CHECK-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 32
-; CHECK-NEXT:    ret
+; NOLSX-LABEL: test:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -32
+; NOLSX-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; NOLSX-NEXT:    st.w $zero, $sp, 16
+; NOLSX-NEXT:    st.d $zero, $sp, 8
+; NOLSX-NEXT:    st.d $zero, $sp, 0
+; NOLSX-NEXT:    addi.d $a0, $sp, 4
+; NOLSX-NEXT:    pcaddu18i $ra, %call36(test1)
+; NOLSX-NEXT:    jirl $ra, $ra, 0
+; NOLSX-NEXT:    move $a0, $zero
+; NOLSX-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; NOLSX-NEXT:    addi.d $sp, $sp, 32
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -32
+; LSX-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LSX-NEXT:    st.w $zero, $sp, 16
+; LSX-NEXT:    vrepli.b $vr0, 0
+; LSX-NEXT:    vst $vr0, $sp, 0
+; LSX-NEXT:    addi.d $a0, $sp, 4
+; LSX-NEXT:    pcaddu18i $ra, %call36(test1)
+; LSX-NEXT:    jirl $ra, $ra, 0
+; LSX-NEXT:    move $a0, $zero
+; LSX-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LSX-NEXT:    addi.d $sp, $sp, 32
+; LSX-NEXT:    ret
   %key = alloca %struct.key_t, align 4
   call void @llvm.memset.p0.i64(ptr %key, i8 0, i64 20, i1 false)
   %1 = getelementptr inbounds %struct.key_t, ptr %key, i64 0, i32 1, i64 0
@@ -98,3 +114,62 @@ define void @test_large_frame_size_1234576() "frame-pointer"="all" {
   %1 = alloca i8, i32 1234567
   ret void
 }
+
+;; Note: will create an emergency spill slot, if (!isInt<7>(StackSize)).
+;; Should involve only one SP-adjusting addi per adjustment.
+;; LSX 112 + 16(emergency solt) = 128
+define void @test_frame_size_112() {
+; NOLSX-LABEL: test_frame_size_112:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -112
+; NOLSX-NEXT:    .cfi_def_cfa_offset 112
+; NOLSX-NEXT:    addi.d $sp, $sp, 112
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_112:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -128
+; LSX-NEXT:    .cfi_def_cfa_offset 128
+; LSX-NEXT:    addi.d $sp, $sp, 128
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 112
+  ret void
+}
+
+;; LSX 128 + 16(emergency solt) = 144
+define void @test_frame_size_128() {
+; NOLSX-LABEL: test_frame_size_128:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -128
+; NOLSX-NEXT:    .cfi_def_cfa_offset 128
+; NOLSX-NEXT:    addi.d $sp, $sp, 128
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_128:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -144
+; LSX-NEXT:    .cfi_def_cfa_offset 144
+; LSX-NEXT:    addi.d $sp, $sp, 144
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 128
+  ret void
+}
+
+;; LSX 144 + 16(emergency solt) = 160
+define void @test_frame_size_144() {
+; NOLSX-LABEL: test_frame_size_144:
+; NOLSX:       # %bb.0:
+; NOLSX-NEXT:    addi.d $sp, $sp, -144
+; NOLSX-NEXT:    .cfi_def_cfa_offset 144
+; NOLSX-NEXT:    addi.d $sp, $sp, 144
+; NOLSX-NEXT:    ret
+;
+; LSX-LABEL: test_frame_size_144:
+; LSX:       # %bb.0:
+; LSX-NEXT:    addi.d $sp, $sp, -160
+; LSX-NEXT:    .cfi_def_cfa_offset 160
+; LSX-NEXT:    addi.d $sp, $sp, 160
+; LSX-NEXT:    ret
+  %1 = alloca i8, i32 144
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
index 402ddb9ad941b..5a55b253c77bb 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
@@ -6,11 +6,11 @@
 define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 dereferenceable(48) %b, i64 %i) {
 ; CHECK-LABEL: box:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -96
-; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    addi.d $sp, $sp, -112
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
 ; CHECK-NEXT:    slli.d $a2, $a1, 5
 ; CHECK-NEXT:    alsl.d $a1, $a1, $a2, 4
-; CHECK-NEXT:    addi.d $a2, $sp, 0
+; CHECK-NEXT:    addi.d $a2, $sp, 16
 ; CHECK-NEXT:    add.d $a3, $a2, $a1
 ; CHECK-NEXT:    vldx $vr0, $a1, $a2
 ; CHECK-NEXT:    vld $vr1, $a3, 32
@@ -18,7 +18,7 @@ define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 der
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    vst $vr1, $a0, 32
 ; CHECK-NEXT:    vst $vr2, $a0, 16
-; CHECK-NEXT:    addi.d $sp, $sp, 96
+; CHECK-NEXT:    addi.d $sp, $sp, 112
 ; CHECK-NEXT:    ret
   %1 = alloca [2 x %Box], align 16
   %2 = getelementptr inbounds [2 x %Box], ptr %1, i64 0, i64 %i
diff --git a/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll b/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
index 789b51d9b5e5b..9528280d181a3 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
@@ -6,10 +6,10 @@ declare <8 x float> @llvm.powi.v8f32.i32(<8 x float>, i32)
 define <8 x float> @powi_v8f32(<8 x float> %va, i32 %b) nounwind {
 ; CHECK-LABEL: powi_v8f32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 0 # 32-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 16 # 32-byte Folded Spill
 ; CHECK-NEXT:    addi.w $fp, $a0, 0
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 0
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
@@ -18,79 +18,79 @@ define <8 x float> @powi_v8f32(<8 x float> %va, i32 %b) nounwind {
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 1
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 2
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 2
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 3
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 3
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 4
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 4
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 5
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 5
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 6
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 6
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
 ; CHECK-NEXT:    movgr2fr.w $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powisf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.s $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 7
-; CHECK-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
 entry:
   %res = call <8 x float> @llvm.powi.v8f32.i32(<8 x float> %va, i32 %b)
@@ -102,10 +102,10 @@ declare <4 x double> @llvm.powi.v4f64.i32(<4 x double>, i32)
 define <4 x double> @powi_v4f64(<4 x double> %va, i32 %b) nounwind {
 ; CHECK-LABEL: powi_v4f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi.d $sp, $sp, -80
-; CHECK-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; CHECK-NEXT:    xvst $xr0, $sp, 0 # 32-byte Folded Spill
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    xvst $xr0, $sp, 16 # 32-byte Folded Spill
 ; CHECK-NEXT:    addi.w $fp, $a0, 0
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
@@ -114,39 +114,39 @@ define <4 x double> @powi_v4f64(<4 x double> %va, i32 %b) nounwind {
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 2
-; CHECK-NEXT:    xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT:    xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT:    xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT:    xvld $xr0, $sp, 16 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
 ; CHECK-NEXT:    movgr2fr.d $fa0, $a0
 ; CHECK-NEXT:    move $a0, $fp
 ; CHECK-NEXT:    pcaddu18i $ra, %call36(__powidf2)
 ; CHECK-NEXT:    jirl $ra, $ra, 0
 ; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT:    xvld $xr0, $sp, 48 # 32-byte Folded Reload
 ; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 3
-; CHECK-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 80
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
 entry:
   %res = call <4 x double> @llvm.powi.v4f64.i32(<4 x double> %va, i32 %b)
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
index 04214f5dfa9d2..2e1618748688a 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
@@ -76,21 +76,21 @@ define void @extract_4xdouble(ptr %src, ptr %dst) nounwind {
 define void @extract_32xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_32xi8_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 0
 ; CHECK-NEXT:    ld.b $a0, $a0, 0
 ; CHECK-NEXT:    st.b $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <32 x i8>, ptr %src
   %e = extractelement <32 x i8> %v, i32 %idx
@@ -101,21 +101,21 @@ define void @extract_32xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_16xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_16xi16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 1
 ; CHECK-NEXT:    ld.h $a0, $a0, 0
 ; CHECK-NEXT:    st.h $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <16 x i16>, ptr %src
   %e = extractelement <16 x i16> %v, i32 %idx
@@ -126,21 +126,21 @@ define void @extract_16xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_8xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_8xi32_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    ld.w $a0, $a0, 0
 ; CHECK-NEXT:    st.w $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x i32>, ptr %src
   %e = extractelement <8 x i32> %v, i32 %idx
@@ -151,21 +151,21 @@ define void @extract_8xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_4xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_4xi64_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    ld.d $a0, $a0, 0
 ; CHECK-NEXT:    st.d $a0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x i64>, ptr %src
   %e = extractelement <4 x i64> %v, i32 %idx
@@ -176,21 +176,21 @@ define void @extract_4xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_8xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_8xfloat_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    fld.s $fa0, $a0, 0
 ; CHECK-NEXT:    fst.s $fa0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x float>, ptr %src
   %e = extractelement <8 x float> %v, i32 %idx
@@ -201,21 +201,21 @@ define void @extract_8xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 define void @extract_4xdouble_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
 ; CHECK-LABEL: extract_4xdouble_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    fld.d $fa0, $a0, 0
 ; CHECK-NEXT:    fst.d $fa0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x double>, ptr %src
   %e = extractelement <4 x double> %v, i32 %idx
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
index 3a4f6efd2c893..b24f95e676d10 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
@@ -114,22 +114,22 @@ define void @insert_4xdouble(ptr %src, ptr %dst, double %in) nounwind {
 define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_32xi8_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 0
 ; CHECK-NEXT:    st.b $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <32 x i8>, ptr %src
   %v_new = insertelement <32 x i8> %v, i8 %in, i32 %idx
@@ -140,22 +140,22 @@ define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
 define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_16xi16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 1
 ; CHECK-NEXT:    st.h $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <16 x i16>, ptr %src
   %v_new = insertelement <16 x i16> %v, i16 %in, i32 %idx
@@ -166,22 +166,22 @@ define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
 define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_8xi32_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 2
 ; CHECK-NEXT:    st.w $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x i32>, ptr %src
   %v_new = insertelement <8 x i32> %v, i32 %in, i32 %idx
@@ -192,22 +192,22 @@ define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
 define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_4xi64_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a3, 4, 3
 ; CHECK-NEXT:    st.d $a2, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x i64>, ptr %src
   %v_new = insertelement <4 x i64> %v, i64 %in, i32 %idx
@@ -218,22 +218,22 @@ define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
 define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_8xfloat_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
-; CHECK-NEXT:    xvst $xr1, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr1, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 2
 ; CHECK-NEXT:    fst.s $fa0, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <8 x float>, ptr %src
   %v_new = insertelement <8 x float> %v, float %in, i32 %idx
@@ -244,22 +244,22 @@ define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwin
 define void @insert_4xdouble_idx(ptr %src, ptr %dst, double %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_4xdouble_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $sp, -96
+; CHECK-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 96
 ; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
-; CHECK-NEXT:    xvst $xr1, $sp, 0
-; CHECK-NEXT:    addi.d $a0, $sp, 0
+; CHECK-NEXT:    xvst $xr1, $sp, 32
+; CHECK-NEXT:    addi.d $a0, $sp, 32
 ; CHECK-NEXT:    bstrins.d $a0, $a2, 4, 3
 ; CHECK-NEXT:    fst.d $fa0, $a0, 0
-; CHECK-NEXT:    xvld $xr0, $sp, 0
+; CHECK-NEXT:    xvld $xr0, $sp, 32
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    addi.d $sp, $fp, -96
+; CHECK-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 96
 ; CHECK-NEXT:    ret
   %v = load volatile <4 x double>, ptr %src
   %v_new = insertelement <4 x double> %v, double %in, i32 %idx
diff --git a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
index ffedd7f9e9438..648c19d509715 100644
--- a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
+++ b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
@@ -347,42 +347,42 @@ define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    vpackev.w $vr0, $vr0, $vr1
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vpackev.w $vr1, $vr0, $vr1
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
   %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
   ret { <2 x float>, <2 x float> } %result
@@ -439,48 +439,48 @@ define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v3f32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -96
-; LA64-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -112
+; LA64-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 2
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 72
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 88
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 68
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 84
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.w $vr0, $vr0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 64
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 80
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 56
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 72
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 52
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    fst.s $fa0, $sp, 68
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fst.s $fa0, $sp, 48
-; LA64-NEXT:    vld $vr0, $sp, 64
-; LA64-NEXT:    vld $vr1, $sp, 48
-; LA64-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 96
+; LA64-NEXT:    fst.s $fa0, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 80
+; LA64-NEXT:    vld $vr1, $sp, 64
+; LA64-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 112
 ; LA64-NEXT:    ret
   %result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a)
   ret { <3 x float>, <3 x float> } %result
@@ -568,44 +568,44 @@ define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f64:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
-; LA64-NEXT:    vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
 ; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 0
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    vreplvei.d $vr0, $vr0, 1
-; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 1
-; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
 ; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 0
-; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
 ; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    movfr2gr.d $a0, $fa0
-; LA64-NEXT:    vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
 ; LA64-NEXT:    vinsgr2vr.d $vr1, $a0, 1
-; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 48 # 16-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
   %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
   ret { <2 x double>, <2 x double> } %result
@@ -801,17 +801,17 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
 ;
 ; LA64-LABEL: test_sincos_v2f128:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -80
-; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s6, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s7, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -96
+; LA64-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 80 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s1, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s2, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s3, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s4, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s5, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s6, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s7, $sp, 16 # 8-byte Folded Spill
 ; LA64-NEXT:    ld.d $fp, $a1, 16
 ; LA64-NEXT:    ld.d $s0, $a1, 24
 ; LA64-NEXT:    ld.d $s1, $a1, 0
@@ -847,17 +847,17 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
 ; LA64-NEXT:    st.d $s6, $s3, 16
 ; LA64-NEXT:    st.d $s5, $s3, 8
 ; LA64-NEXT:    st.d $s4, $s3, 0
-; LA64-NEXT:    ld.d $s7, $sp, 0 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s6, $sp, 8 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 80
+; LA64-NEXT:    ld.d $s7, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s6, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s5, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s4, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s3, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s2, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s1, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 96
 ; LA64-NEXT:    ret
   %result = call { <2 x fp128>, <2 x fp128> } @llvm.sincos.v2f128(<2 x fp128> %a)
   ret { <2 x fp128>, <2 x fp128> } %result
diff --git a/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll b/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
new file mode 100644
index 0000000000000..96159e5884d3f
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lsx --verify-machineinstrs < %s | FileCheck %s
+define void @eliminate_frame_index(<16 x i8> %a) nounwind {
+; CHECK-LABEL: eliminate_frame_index:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.d $sp, $sp, -240
+; CHECK-NEXT:    st.d $ra, $sp, 232 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 224 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s0, $sp, 216 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s1, $sp, 208 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s2, $sp, 200 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s3, $sp, 192 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s4, $sp, 184 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s5, $sp, 176 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s6, $sp, 168 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s7, $sp, 160 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $s8, $sp, 152 # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $zero, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $ra, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $tp, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $a7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $t8, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $fp, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s0, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s1, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s2, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s3, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s4, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s5, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s6, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s7, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    addi.d $s8, $zero, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    st.d $a0, $sp, 0 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $a0, $sp, 136
+; CHECK-NEXT:    vstelm.b $vr0, $a0, 0, 0
+; CHECK-NEXT:    ld.d $a0, $sp, 0 # 8-byte Folded Reload
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $zero
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $ra
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $tp
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $a7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $t8
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $fp
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s3
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s4
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s5
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s6
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s7
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # reg use $s8
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    ld.d $s8, $sp, 152 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s7, $sp, 160 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s6, $sp, 168 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s5, $sp, 176 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s4, $sp, 184 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s3, $sp, 192 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s2, $sp, 200 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s1, $sp, 208 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $s0, $sp, 216 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $fp, $sp, 224 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 232 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 240
+; CHECK-NEXT:    ret
+  %s = alloca [16 x i8]
+  %ss = alloca [128 x i8]
+
+  %zero =  call i64 asm sideeffect "addi.d $$zero, $$zero, 1", "={r0}"()
+  %ra =  call i64 asm sideeffect "addi.d $$ra, $$zero, 1", "={r1}"()
+  %tp =  call i64 asm sideeffect "addi.d $$tp, $$zero, 1", "={r2}"()
+  %a0 =  call i64 asm sideeffect "addi.d $$a0, $$zero, 1", "={r4}"()
+  %a1 =  call i64 asm sideeffect "addi.d $$a1, $$zero, 1", "={r5}"()
+  %a2 =  call i64 asm sideeffect "addi.d $$a2, $$zero, 1", "={r6}"()
+  %a3 =  call i64 asm sideeffect "addi.d $$a3, $$zero, 1", "={r7}"()
+  %a4 =  call i64 asm sideeffect "addi.d $$a4, $$zero, 1", "={r8}"()
+  %a5 =  call i64 asm sideeffect "addi.d $$a5, $$zero, 1", "={r9}"()
+  %a6 =  call i64 asm sideeffect "addi.d $$a6, $$zero, 1", "={r10}"()
+  %a7 =  call i64 asm sideeffect "addi.d $$a7, $$zero, 1", "={r11}"()
+  %t0 =  call i64 asm sideeffect "addi.d $$t0, $$zero, 1", "={r12}"()
+  %t1 =  call i64 asm sideeffect "addi.d $$t1, $$zero, 1", "={r13}"()
+  %t2 =  call i64 asm sideeffect "addi.d $$t2, $$zero, 1", "={r14}"()
+  %t3 =  call i64 asm sideeffect "addi.d $$t3, $$zero, 1", "={r15}"()
+  %t4 =  call i64 asm sideeffect "addi.d $$t4, $$zero, 1", "={r16}"()
+  %t5 =  call i64 asm sideeffect "addi.d $$t5, $$zero, 1", "={r17}"()
+  %t6 =  call i64 asm sideeffect "addi.d $$t6, $$zero, 1", "={r18}"()
+  %t7 =  call i64 asm sideeffect "addi.d $$t7, $$zero, 1", "={r19}"()
+  %t8 =  call i64 asm sideeffect "addi.d $$t8, $$zero, 1", "={r20}"()
+  ;; r21 Reserved (Non-allocatable)
+  %s9 =  call i64 asm sideeffect "addi.d $$s9, $$zero, 1", "={r22}"()
+  %s0 =  call i64 asm sideeffect "addi.d $$s0, $$zero, 1", "={r23}"()
+  %s1 =  call i64 asm sideeffect "addi.d $$s1, $$zero, 1", "={r24}"()
+  %s2 =  call i64 asm sideeffect "addi.d $$s2, $$zero, 1", "={r25}"()
+  %s3 =  call i64 asm sideeffect "addi.d $$s3, $$zero, 1", "={r26}"()
+  %s4 =  call i64 asm sideeffect "addi.d $$s4, $$zero, 1", "={r27}"()
+  %s5 =  call i64 asm sideeffect "addi.d $$s5, $$zero, 1", "={r28}"()
+  %s6 =  call i64 asm sideeffect "addi.d $$s6, $$zero, 1", "={r29}"()
+  %s7 =  call i64 asm sideeffect "addi.d $$s7, $$zero, 1", "={r30}"()
+  %s8 =  call i64 asm sideeffect "addi.d $$s8, $$zero, 1", "={r31}"()
+
+  %e = extractelement <16 x i8> %a, i64 0
+
+  store volatile i8 %e, ptr %s
+
+  call void asm sideeffect "# reg use $0", "{r0}"(i64 %zero)
+  call void asm sideeffect "# reg use $0", "{r1}"(i64 %ra)
+  call void asm sideeffect "# reg use $0", "{r2}"(i64 %tp)
+  call void asm sideeffect "# reg use $0", "{r4}"(i64 %a0)
+  call void asm sideeffect "# reg use $0", "{r5}"(i64 %a1)
+  call void asm sideeffect "# reg use $0", "{r6}"(i64 %a2)
+  call void asm sideeffect "# reg use $0", "{r7}"(i64 %a3)
+  call void asm sideeffect "# reg use $0", "{r8}"(i64 %a4)
+  call void asm sideeffect "# reg use $0", "{r9}"(i64 %a5)
+  call void asm sideeffect "# reg use $0", "{r10}"(i64 %a6)
+  call void asm sideeffect "# reg use $0", "{r11}"(i64 %a7)
+  call void asm sideeffect "# reg use $0", "{r12}"(i64 %t0)
+  call void asm sideeffect "# reg use $0", "{r13}"(i64 %t1)
+  call void asm sideeffect "# reg use $0", "{r14}"(i64 %t2)
+  call void asm sideeffect "# reg use $0", "{r15}"(i64 %t3)
+  call void asm sideeffect "# reg use $0", "{r16}"(i64 %t4)
+  call void asm sideeffect "# reg use $0", "{r17}"(i64 %t5)
+  call void asm sideeffect "# reg use $0", "{r18}"(i64 %t6)
+  call void asm sideeffect "# reg use $0", "{r19}"(i64 %t7)
+  call void asm sideeffect "# reg use $0", "{r20}"(i64 %t8)
+  ;; r21 Reserved (Non-allocatable)
+  call void asm sideeffect "# reg use $0", "{r22}"(i64 %s9)
+  call void asm sideeffect "# reg use $0", "{r23}"(i64 %s0)
+  call void asm sideeffect "# reg use $0", "{r24}"(i64 %s1)
+  call void asm sideeffect "# reg use $0", "{r25}"(i64 %s2)
+  call void asm sideeffect "# reg use $0", "{r26}"(i64 %s3)
+  call void asm sideeffect "# reg use $0", "{r27}"(i64 %s4)
+  call void asm sideeffect "# reg use $0", "{r28}"(i64 %s5)
+  call void asm sideeffect "# reg use $0", "{r29}"(i64 %s6)
+  call void asm sideeffect "# reg use $0", "{r30}"(i64 %s7)
+  call void asm sideeffect "# reg use $0", "{r31}"(i64 %s8)
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
index 9f15604fcca6b..69995a0721f8a 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
@@ -36,15 +36,15 @@ define void @caller(i32 %n) {
 ;
 ; LA64-LABEL: caller:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $s8, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -128
+; LA64-NEXT:    .cfi_def_cfa_offset 128
+; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s8, $sp, 104 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
 ; LA64-NEXT:    .cfi_offset 31, -24
-; LA64-NEXT:    addi.d $fp, $sp, 64
+; LA64-NEXT:    addi.d $fp, $sp, 128
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 5, 0
 ; LA64-NEXT:    move $s8, $sp
@@ -54,14 +54,14 @@ define void @caller(i32 %n) {
 ; LA64-NEXT:    slli.d $a0, $a0, 4
 ; LA64-NEXT:    sub.d $a0, $sp, $a0
 ; LA64-NEXT:    move $sp, $a0
-; LA64-NEXT:    addi.d $a1, $s8, 0
+; LA64-NEXT:    addi.d $a1, $s8, 64
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -64
-; LA64-NEXT:    ld.d $s8, $sp, 40 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    addi.d $sp, $fp, -128
+; LA64-NEXT:    ld.d $s8, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 128
 ; LA64-NEXT:    ret
   %1 = alloca i8, i32 %n
   %2 = alloca i32, align 64
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment.ll b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
index 0645339358b64..0188884543adb 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
@@ -28,22 +28,22 @@ define void @caller32() {
 ;
 ; LA64-LABEL: caller32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -32
-; LA64-NEXT:    .cfi_def_cfa_offset 32
-; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -64
+; LA64-NEXT:    .cfi_def_cfa_offset 64
+; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 32
+; LA64-NEXT:    addi.d $fp, $sp, 64
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 4, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 32
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -32
-; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    addi.d $sp, $fp, -64
+; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 64
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 32
   call void @callee(ptr %1)
@@ -102,22 +102,22 @@ define void @caller64() {
 ;
 ; LA64-LABEL: caller64:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
-; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -128
+; LA64-NEXT:    .cfi_def_cfa_offset 128
+; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 64
+; LA64-NEXT:    addi.d $fp, $sp, 128
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 5, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 64
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -64
-; LA64-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    addi.d $sp, $fp, -128
+; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 128
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 64
   call void @callee(ptr %1)
@@ -176,22 +176,22 @@ define void @caller128() {
 ;
 ; LA64-LABEL: caller128:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -128
-; LA64-NEXT:    .cfi_def_cfa_offset 128
-; LA64-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -256
+; LA64-NEXT:    .cfi_def_cfa_offset 256
+; LA64-NEXT:    st.d $ra, $sp, 248 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 240 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 128
+; LA64-NEXT:    addi.d $fp, $sp, 256
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 6, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 128
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -128
-; LA64-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 128
+; LA64-NEXT:    addi.d $sp, $fp, -256
+; LA64-NEXT:    ld.d $fp, $sp, 240 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 248 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 256
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 128
   call void @callee(ptr %1)
@@ -250,22 +250,22 @@ define void @caller256() {
 ;
 ; LA64-LABEL: caller256:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    addi.d $sp, $sp, -256
-; LA64-NEXT:    .cfi_def_cfa_offset 256
-; LA64-NEXT:    st.d $ra, $sp, 248 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 240 # 8-byte Folded Spill
+; LA64-NEXT:    addi.d $sp, $sp, -512
+; LA64-NEXT:    .cfi_def_cfa_offset 512
+; LA64-NEXT:    st.d $ra, $sp, 504 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 496 # 8-byte Folded Spill
 ; LA64-NEXT:    .cfi_offset 1, -8
 ; LA64-NEXT:    .cfi_offset 22, -16
-; LA64-NEXT:    addi.d $fp, $sp, 256
+; LA64-NEXT:    addi.d $fp, $sp, 512
 ; LA64-NEXT:    .cfi_def_cfa 22, 0
 ; LA64-NEXT:    bstrins.d $sp, $zero, 7, 0
-; LA64-NEXT:    addi.d $a0, $sp, 0
+; LA64-NEXT:    addi.d $a0, $sp, 256
 ; LA64-NEXT:    pcaddu18i $ra, %call36(callee)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    addi.d $sp, $fp, -256
-; LA64-NEXT:    ld.d $fp, $sp, 240 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $ra, $sp, 248 # 8-byte Folded Reload
-; LA64-NEXT:    addi.d $sp, $sp, 256
+; LA64-NEXT:    addi.d $sp, $fp, -512
+; LA64-NEXT:    ld.d $fp, $sp, 496 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 504 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 512
 ; LA64-NEXT:    ret
   %1 = alloca i8, align 256
   call void @callee(ptr %1)
diff --git a/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll b/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
index 925fdf3d60646..0d441e66a0c84 100644
--- a/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
+++ b/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
@@ -121,19 +121,19 @@ define void @t3() {
 ;
 ; LA64-LABEL: t3:
 ; LA64:       # %bb.0: # %entry
-; LA64-NEXT:    addi.d $sp, $sp, -64
-; LA64-NEXT:    .cfi_def_cfa_offset 64
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    .cfi_def_cfa_offset 80
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(.L.str)
 ; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(.L.str)
 ; LA64-NEXT:    ld.h $a1, $a0, 20
 ; LA64-NEXT:    ld.w $a2, $a0, 16
 ; LA64-NEXT:    ld.d $a3, $a0, 8
 ; LA64-NEXT:    ld.d $a0, $a0, 0
-; LA64-NEXT:    st.h $a1, $sp, 20
-; LA64-NEXT:    st.w $a2, $sp, 16
-; LA64-NEXT:    st.d $a3, $sp, 8
-; LA64-NEXT:    st.d $a0, $sp, 0
-; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    st.h $a1, $sp, 36
+; LA64-NEXT:    st.w $a2, $sp, 32
+; LA64-NEXT:    st.d $a3, $sp, 24
+; LA64-NEXT:    st.d $a0, $sp, 16
+; LA64-NEXT:    addi.d $sp, $sp, 80
 ; LA64-NEXT:    ret
 entry:
   %msgbuf = alloca [64 x i8], align 1
diff --git a/llvm/test/CodeGen/LoongArch/vararg.ll b/llvm/test/CodeGen/LoongArch/vararg.ll
index 939cd2015c5b1..bc4b8a77c7e15 100644
--- a/llvm/test/CodeGen/LoongArch/vararg.ll
+++ b/llvm/test/CodeGen/LoongArch/vararg.ll
@@ -47,7 +47,7 @@ define i64 @va1(ptr %fmt, ...) {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a1, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a1, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $a1, $fp, -32
 ; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
@@ -94,7 +94,7 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a1, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a1, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $a1, $fp, -32
 ; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
 ; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
@@ -112,11 +112,11 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind {
 define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-LABEL: va1_va_arg_alloca:
 ; LA64-FPELIM:       # %bb.0:
-; LA64-FPELIM-NEXT:    addi.d $sp, $sp, -96
-; LA64-FPELIM-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT:    addi.d $fp, $sp, 32
+; LA64-FPELIM-NEXT:    addi.d $sp, $sp, -112
+; LA64-FPELIM-NEXT:    st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT:    addi.d $fp, $sp, 48
 ; LA64-FPELIM-NEXT:    move $s0, $a1
 ; LA64-FPELIM-NEXT:    st.d $a7, $fp, 56
 ; LA64-FPELIM-NEXT:    st.d $a6, $fp, 48
@@ -126,7 +126,7 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-NEXT:    st.d $a2, $fp, 16
 ; LA64-FPELIM-NEXT:    st.d $a1, $fp, 8
 ; LA64-FPELIM-NEXT:    addi.d $a0, $fp, 16
-; LA64-FPELIM-NEXT:    st.d $a0, $fp, -32
+; LA64-FPELIM-NEXT:    st.d $a0, $fp, -40
 ; LA64-FPELIM-NEXT:    addi.d $a0, $a1, 15
 ; LA64-FPELIM-NEXT:    bstrins.d $a0, $zero, 3, 0
 ; LA64-FPELIM-NEXT:    sub.d $a0, $sp, $a0
@@ -134,20 +134,20 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-FPELIM-NEXT:    pcaddu18i $ra, %call36(notdead)
 ; LA64-FPELIM-NEXT:    jirl $ra, $ra, 0
 ; LA64-FPELIM-NEXT:    move $a0, $s0
-; LA64-FPELIM-NEXT:    addi.d $sp, $fp, -32
-; LA64-FPELIM-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT:    addi.d $sp, $sp, 96
+; LA64-FPELIM-NEXT:    addi.d $sp, $fp, -48
+; LA64-FPELIM-NEXT:    ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT:    addi.d $sp, $sp, 112
 ; LA64-FPELIM-NEXT:    ret
 ;
 ; LA64-WITHFP-LABEL: va1_va_arg_alloca:
 ; LA64-WITHFP:       # %bb.0:
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -96
-; LA64-WITHFP-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 32
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -112
+; LA64-WITHFP-NEXT:    st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 48
 ; LA64-WITHFP-NEXT:    move $s0, $a1
 ; LA64-WITHFP-NEXT:    st.d $a7, $fp, 56
 ; LA64-WITHFP-NEXT:    st.d $a6, $fp, 48
@@ -157,7 +157,7 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    st.d $a2, $fp, 16
 ; LA64-WITHFP-NEXT:    st.d $a1, $fp, 8
 ; LA64-WITHFP-NEXT:    addi.d $a0, $fp, 16
-; LA64-WITHFP-NEXT:    st.d $a0, $fp, -32
+; LA64-WITHFP-NEXT:    st.d $a0, $fp, -40
 ; LA64-WITHFP-NEXT:    addi.d $a0, $a1, 15
 ; LA64-WITHFP-NEXT:    bstrins.d $a0, $zero, 3, 0
 ; LA64-WITHFP-NEXT:    sub.d $a0, $sp, $a0
@@ -165,11 +165,11 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LA64-WITHFP-NEXT:    pcaddu18i $ra, %call36(notdead)
 ; LA64-WITHFP-NEXT:    jirl $ra, $ra, 0
 ; LA64-WITHFP-NEXT:    move $a0, $s0
-; LA64-WITHFP-NEXT:    addi.d $sp, $fp, -32
-; LA64-WITHFP-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 96
+; LA64-WITHFP-NEXT:    addi.d $sp, $fp, -48
+; LA64-WITHFP-NEXT:    ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 112
 ; LA64-WITHFP-NEXT:    ret
   %va = alloca ptr, align 8
   call void @llvm.va_start(ptr %va)
@@ -314,10 +314,10 @@ define void @va_aligned_stack_caller() nounwind {
 ;
 ; LA64-WITHFP-LABEL: va_aligned_stack_caller:
 ; LA64-WITHFP:       # %bb.0:
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -112
-; LA64-WITHFP-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    st.d $fp, $sp, 96 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 112
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, -128
+; LA64-WITHFP-NEXT:    st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT:    addi.d $fp, $sp, 128
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 17
 ; LA64-WITHFP-NEXT:    st.d $a0, $sp, 48
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 16
@@ -336,23 +336,23 @@ define void @va_aligned_stack_caller() nounwind {
 ; LA64-WITHFP-NEXT:    lu32i.d $a0, 335544
 ; LA64-WITHFP-NEXT:    lu52i.d $a0, $a0, -328
 ; LA64-WITHFP-NEXT:    st.d $a0, $sp, 16
-; LA64-WITHFP-NEXT:    st.d $zero, $fp, -24
+; LA64-WITHFP-NEXT:    st.d $zero, $fp, -40
 ; LA64-WITHFP-NEXT:    vrepli.b $vr0, 0
-; LA64-WITHFP-NEXT:    vst $vr0, $fp, -40
+; LA64-WITHFP-NEXT:    vst $vr0, $fp, -56
 ; LA64-WITHFP-NEXT:    ori $a5, $zero, 1000
 ; LA64-WITHFP-NEXT:    ori $a0, $zero, 1
 ; LA64-WITHFP-NEXT:    ori $a1, $zero, 11
-; LA64-WITHFP-NEXT:    addi.d $a2, $fp, -48
+; LA64-WITHFP-NEXT:    addi.d $a2, $fp, -64
 ; LA64-WITHFP-NEXT:    ori $a3, $zero, 12
 ; LA64-WITHFP-NEXT:    ori $a4, $zero, 13
 ; LA64-WITHFP-NEXT:    ori $a7, $zero, 1
-; LA64-WITHFP-NEXT:    st.d $a5, $fp, -48
+; LA64-WITHFP-NEXT:    st.d $a5, $fp, -64
 ; LA64-WITHFP-NEXT:    move $a6, $zero
 ; LA64-WITHFP-NEXT:    pcaddu18i $ra, %call36(va_aligned_stack_callee)
 ; LA64-WITHFP-NEXT:    jirl $ra, $ra, 0
-; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 96 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 112
+; LA64-WITHFP-NEXT:    ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT:    addi.d $sp, $sp, 128
 ; LA64-WITHFP-NEXT:    ret
   %1 = call i32 (i32, ...) @va_aligned_stack_callee(i32 1, i32 11,
     i256 1000, i32 12, i32 13, i128 18446744073709551616, i32 14,
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index cbc9f700b1f01..aba20e6b0f27f 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -2311,4 +2311,51 @@ entry:
   ret void
 }
 
+define <4 x float> @test_uitofp_v4i8(<4 x i8> %a) {
+; CHECK-LABEL: test_uitofp_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_uitofp_v4i8_param_0];
+; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 0x7773U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r3, %r2;
+; CHECK-NEXT:    prmt.b32 %r4, %r1, 0, 0x7772U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r5, %r4;
+; CHECK-NEXT:    prmt.b32 %r6, %r1, 0, 0x7771U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r7, %r6;
+; CHECK-NEXT:    prmt.b32 %r8, %r1, 0, 0x7770U;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r9, %r8;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r7, %r5, %r3};
+; CHECK-NEXT:    ret;
+  %r = uitofp <4 x i8> %a to <4 x float>
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_sitofp_v4i8(<4 x i8> %a) {
+; CHECK-LABEL: test_sitofp_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sitofp_v4i8_param_0];
+; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 0xbbb3U;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r3, %rs1;
+; CHECK-NEXT:    prmt.b32 %r4, %r1, 0, 0xaaa2U;
+; CHECK-NEXT:    cvt.u16.u32 %rs2, %r4;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r5, %rs2;
+; CHECK-NEXT:    prmt.b32 %r6, %r1, 0, 0x9991U;
+; CHECK-NEXT:    cvt.u16.u32 %rs3, %r6;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r7, %rs3;
+; CHECK-NEXT:    prmt.b32 %r8, %r1, 0, 0x8880U;
+; CHECK-NEXT:    cvt.u16.u32 %rs4, %r8;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r9, %rs4;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r7, %r5, %r3};
+; CHECK-NEXT:    ret;
+  %r = sitofp <4 x i8> %a to <4 x float>
+  ret <4 x float> %r
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/NVPTX/trunc-tofp.ll b/llvm/test/CodeGen/NVPTX/trunc-tofp.ll
new file mode 100644
index 0000000000000..404c423cc026a
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/trunc-tofp.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_80 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mcpu=sm_80 | %ptxas-verify %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define float @uitofp_trunc_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: uitofp_trunc_nuw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [uitofp_trunc_nuw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [uitofp_trunc_nuw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.u32 %r4, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+  %v = add i32 %x, %y
+  %t = trunc nuw i32 %v to i16
+  %f = uitofp i16 %t to float
+  ret float %f
+}
+
+define float @sitofp_trunc_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: sitofp_trunc_nsw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sitofp_trunc_nsw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [sitofp_trunc_nsw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f32.s32 %r4, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+  %v = add i32 %x, %y
+  %t = trunc nsw i32 %v to i16
+  %f = sitofp i16 %t to float
+  ret float %f
+}
+
+define float @uitofp_trunc_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: uitofp_trunc_nsw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [uitofp_trunc_nsw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [uitofp_trunc_nsw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
+; CHECK-NEXT:    cvt.rn.f32.u16 %r4, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+    %v = add i32 %x, %y
+    %t = trunc nsw i32 %v to i16
+    %f = uitofp i16 %t to float
+    ret float %f
+}
+
+define float @sitofp_trunc_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: sitofp_trunc_nuw(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sitofp_trunc_nuw_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [sitofp_trunc_nuw_param_1];
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
+; CHECK-NEXT:    cvt.rn.f32.s16 %r4, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+    %v = add i32 %x, %y
+    %t = trunc nuw i32 %v to i16
+    %f = sitofp i16 %t to float
+    ret float %f
+}
diff --git a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
index 061435c45ad0e..59a702ab6b17f 100644
--- a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
+++ b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
@@ -798,12 +798,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccor:
 ; RV64SFBSIFIVEU74:       # %bb.0: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
-; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s0, a3
+; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
 ; RV64SFBSIFIVEU74-NEXT:    mv s1, a2
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB15_4
 ; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    or s0, a3, a1
@@ -824,11 +824,11 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV64SFBANDESAX45-LABEL: sextw_removal_ccor:
 ; RV64SFBANDESAX45:       # %bb.0: # %bb
 ; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
-; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s0, a3
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s1, a2
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB15_2
 ; RV64SFBANDESAX45-NEXT:  # %bb.1:
 ; RV64SFBANDESAX45-NEXT:    or s0, s0, a1
@@ -848,12 +848,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; ZICOND-LABEL: sextw_removal_ccor:
 ; ZICOND:       # %bb.0: # %bb
 ; ZICOND-NEXT:    addi sp, sp, -32
-; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s0, a3
+; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    andi a0, a0, 1
 ; ZICOND-NEXT:    mv s1, a2
+; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    beqz a0, .LBB15_4
 ; ZICOND-NEXT:  # %bb.3: # %bb
 ; ZICOND-NEXT:    or s0, a3, a1
@@ -874,12 +874,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; RV32SFB-LABEL: sextw_removal_ccor:
 ; RV32SFB:       # %bb.0: # %bb
 ; RV32SFB-NEXT:    addi sp, sp, -16
-; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
-; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s0, a3
+; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    andi a0, a0, 1
 ; RV32SFB-NEXT:    mv s1, a2
+; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    beqz a0, .LBB15_4
 ; RV32SFB-NEXT:  # %bb.3: # %bb
 ; RV32SFB-NEXT:    or s0, a3, a1
@@ -941,11 +941,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccaddw:
 ; RV64SFBSIFIVEU74:       # %bb.0: # %bb
 ; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
-; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s1, a1
 ; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
+; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBSIFIVEU74-NEXT:    mv s0, a2
 ; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB16_4
 ; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
@@ -967,11 +967,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV64SFBANDESAX45-LABEL: sextw_removal_ccaddw:
 ; RV64SFBANDESAX45:       # %bb.0: # %bb
 ; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
-; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s0, a2
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    mv s1, a1
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB16_2
 ; RV64SFBANDESAX45-NEXT:  # %bb.1:
 ; RV64SFBANDESAX45-NEXT:    addw s1, s1, a3
@@ -991,11 +991,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; ZICOND-LABEL: sextw_removal_ccaddw:
 ; ZICOND:       # %bb.0: # %bb
 ; ZICOND-NEXT:    addi sp, sp, -32
-; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s1, a1
 ; ZICOND-NEXT:    andi a0, a0, 1
+; ZICOND-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; ZICOND-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; ZICOND-NEXT:    mv s0, a2
 ; ZICOND-NEXT:    beqz a0, .LBB16_4
 ; ZICOND-NEXT:  # %bb.3: # %bb
@@ -1017,11 +1017,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; RV32SFB-LABEL: sextw_removal_ccaddw:
 ; RV32SFB:       # %bb.0: # %bb
 ; RV32SFB-NEXT:    addi sp, sp, -16
-; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s1, a1
 ; RV32SFB-NEXT:    andi a0, a0, 1
+; RV32SFB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32SFB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32SFB-NEXT:    mv s0, a2
 ; RV32SFB-NEXT:    beqz a0, .LBB16_4
 ; RV32SFB-NEXT:  # %bb.3: # %bb
diff --git a/llvm/test/CodeGen/RISCV/zdinx-spill.ll b/llvm/test/CodeGen/RISCV/zdinx-spill.ll
index d7a700622bf8c..6f206fe571c17 100644
--- a/llvm/test/CodeGen/RISCV/zdinx-spill.ll
+++ b/llvm/test/CodeGen/RISCV/zdinx-spill.ll
@@ -9,7 +9,6 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   liveins: $x10, $x11, $x8, $x9, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $x2 = frame-setup ADDI $x2, -64
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION def_cfa_offset 64
   ; CHECK-NEXT:   frame-setup SW killed $x8, $x2, 60 :: (store (s32) into %stack.1)
   ; CHECK-NEXT:   frame-setup SW killed $x9, $x2, 56 :: (store (s32) into %stack.2)
   ; CHECK-NEXT:   frame-setup SW killed $x18, $x2, 52 :: (store (s32) into %stack.3)
@@ -22,18 +21,6 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   frame-setup SW killed $x25, $x2, 24 :: (store (s32) into %stack.10)
   ; CHECK-NEXT:   frame-setup SW killed $x26, $x2, 20 :: (store (s32) into %stack.11)
   ; CHECK-NEXT:   frame-setup SW killed $x27, $x2, 16 :: (store (s32) into %stack.12)
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x8, -4
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x9, -8
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x18, -12
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x19, -16
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x20, -20
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x21, -24
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x22, -28
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x23, -32
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x24, -36
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x25, -40
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x26, -44
-  ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x27, -48
   ; CHECK-NEXT:   renamable $x10_x11 = nofpexcept FADD_D_IN32X killed renamable $x10_x11, renamable $x10_x11, 7, implicit $frm
   ; CHECK-NEXT:   PseudoRV32ZdinxSD killed renamable $x10_x11, $x2, 8 :: (store (s64) into %stack.0, align 4)
   ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 12 /* clobber */, implicit-def dead early-clobber $x6, 12 /* clobber */, implicit-def dead early-clobber $x7, 12 /* clobber */, implicit-def dead early-clobber $x8, 12 /* clobber */, implicit-def dead early-clobber $x9, 12 /* clobber */, implicit-def dead early-clobber $x10, 12 /* clobber */, implicit-def dead early-clobber $x11, 12 /* clobber */, implicit-def dead early-clobber $x12, 12 /* clobber */, implicit-def dead early-clobber $x13, 12 /* clobber */, implicit-def dead early-clobber $x14, 12 /* clobber */, implicit-def dead early-clobber $x15, 12 /* clobber */, implicit-def dead early-clobber $x16, 12 /* clobber */, implicit-def dead early-clobber $x17, 12 /* clobber */, implicit-def dead early-clobber $x18, 12 /* clobber */, implicit-def dead early-clobber $x19, 12 /* clobber */, implicit-def dead early-clobber $x20, 12 /* clobber */, implicit-def dead early-clobber $x21, 12 /* clobber */, implicit-def dead early-clobber $x22, 12 /* clobber */, implicit-def dead early-clobber $x23, 12 /* clobber */, implicit-def dead early-clobber $x24, 12 /* clobber */, implicit-def dead early-clobber $x25, 12 /* clobber */, implicit-def dead early-clobber $x26, 12 /* clobber */, implicit-def dead early-clobber $x27, 12 /* clobber */, implicit-def dead early-clobber $x28, 12 /* clobber */, implicit-def dead early-clobber $x29, 12 /* clobber */, implicit-def dead early-clobber $x31
@@ -50,20 +37,7 @@ define double @foo(double %x) nounwind {
   ; CHECK-NEXT:   $x25 = frame-destroy LW $x2, 24 :: (load (s32) from %stack.10)
   ; CHECK-NEXT:   $x26 = frame-destroy LW $x2, 20 :: (load (s32) from %stack.11)
   ; CHECK-NEXT:   $x27 = frame-destroy LW $x2, 16 :: (load (s32) from %stack.12)
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x8
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x9
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x18
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x19
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x20
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x21
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x22
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x23
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x24
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x25
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x26
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION restore $x27
   ; CHECK-NEXT:   $x2 = frame-destroy ADDI $x2, 64
-  ; CHECK-NEXT:   frame-destroy CFI_INSTRUCTION def_cfa_offset 0
   ; CHECK-NEXT:   PseudoRET implicit $x10, implicit $x11
   %a = fadd double %x, %x
   call void asm sideeffect "", "~{x6},~{x7},~{x8},~{x9},~{x10},~{x11},~{x12},~{x13},~{x14},~{x15},~{x16},~{x17},~{x18},~{x19},~{x20},~{x21},~{x22},~{x23},~{x24},~{x25},~{x26},~{x27},~{x28},~{x29},~{xr0},~{x31}"()
diff --git a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
index 1d073cdc3bdb9..a878dbe94d11d 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; Test handling of llvm.lifetime intrinsics.
 ; RUN: opt < %s -passes=asan -asan-use-after-scope -asan-use-after-return=never -asan-use-stack-safety=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT
 ; RUN: opt < %s -passes=asan -asan-use-after-scope -asan-use-after-return=never -asan-use-stack-safety=0 -asan-instrument-dynamic-allocas=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK-NO-DYNAMIC
@@ -8,109 +9,389 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
 
-; CHECK-LABEL: define void @lifetime_no_size(
 define void @lifetime_no_size(i64 %i) sanitize_address {
+; CHECK-LABEL: define void @lifetime_no_size(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @lifetime_no_size to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i64 -868083117767659023, ptr [[TMP11]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP2]])
+; CHECK-NEXT:    [[AI:%.*]] = getelementptr inbounds [2 x i32], ptr [[TMP2]], i64 0, i64 [[I]]
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[AI]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP13]], 2147450880
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, ptr [[TMP15]], align 1
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i8 [[TMP16]], 0
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[BB18:.*]], label %[[BB23:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i8
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp sge i8 [[TMP20]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB23]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    call void @__asan_report_store1(i64 [[TMP12]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    store volatile i8 0, ptr [[AI]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[TMP2]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-NEXT:    store i64 0, ptr [[TMP25]], align 1
+; CHECK-NEXT:    ret void
+;
 entry:
   %a = alloca [2 x i32], align 4
 
   ; Poison memory in prologue: 0xf3f3f300f1f1f1f1
-  ; CHECK: store i64 -868083117767659023, ptr %[[#]]
 
   call void @llvm.lifetime.start.p0(i64 -1, ptr %a)
   ; Check that lifetime with no size are ignored.
-  ; CHECK-NOT: store
-  ; CHECK: call void @llvm.lifetime.start
 
   %ai = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 %i
   store volatile i8 0, ptr %ai, align 4
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 -1, ptr %a)
   ; Check that lifetime with no size are ignored.
-  ; CHECK-NOT: store
-  ; CHECK: call void @llvm.lifetime.end
 
   ; Unpoison stack frame on exit.
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK: ret void
   ret void
 }
 
 ; Generic case of lifetime analysis.
 define void @lifetime() sanitize_address {
-  ; CHECK-LABEL: define void @lifetime()
+; CHECK-DEFAULT-LABEL: define void @lifetime(
+; CHECK-DEFAULT-SAME: ) #[[ATTR1]] {
+; CHECK-DEFAULT-NEXT:    [[TMP1:%.*]] = alloca i64, align 32
+; CHECK-DEFAULT-NEXT:    store i64 0, ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-DEFAULT-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 32
+; CHECK-DEFAULT-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 1102416563, ptr [[TMP5]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP6:%.*]] = add i64 [[TMP2]], 8
+; CHECK-DEFAULT-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP7]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP8:%.*]] = add i64 [[TMP2]], 16
+; CHECK-DEFAULT-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP9]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP10:%.*]] = lshr i64 [[TMP2]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 0
+; CHECK-DEFAULT-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 -868082052615769615, ptr [[TMP13]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 4, ptr [[TMP15]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 3, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP16]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB27:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB22]]:
+; CHECK-DEFAULT-NEXT:    [[TMP23:%.*]] = and i64 [[TMP16]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP25:%.*]] = icmp sge i8 [[TMP24]], [[TMP20]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP25]], label %[[BB26:.*]], label %[[BB27]]
+; CHECK-DEFAULT:       [[BB26]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP16]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB27]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP4]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP28:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP29]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP30:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP31:%.*]] = inttoptr i64 [[TMP30]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP31]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP32:%.*]] = alloca i8, i64 128, align 32
+; CHECK-DEFAULT-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP32]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP34:%.*]] = add i64 [[TMP33]], 32
+; CHECK-DEFAULT-NEXT:    call void @__asan_alloca_poison(i64 [[TMP34]], i64 40)
+; CHECK-DEFAULT-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP32]] to i64
+; CHECK-DEFAULT-NEXT:    store i64 [[TMP35]], ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP36:%.*]] = inttoptr i64 [[TMP34]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP37:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    call void @__asan_unpoison_stack_memory(i64 [[TMP37]], i64 40)
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[TMP36]])
+; CHECK-DEFAULT-NEXT:    [[TMP38:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP39:%.*]] = lshr i64 [[TMP38]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP40:%.*]] = add i64 [[TMP39]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP41:%.*]] = inttoptr i64 [[TMP40]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP42:%.*]] = load i8, ptr [[TMP41]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP43:%.*]] = icmp ne i8 [[TMP42]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP43]], label %[[BB44:.*]], label %[[BB49:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB44]]:
+; CHECK-DEFAULT-NEXT:    [[TMP45:%.*]] = and i64 [[TMP38]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP45]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP47:%.*]] = icmp sge i8 [[TMP46]], [[TMP42]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP47]], label %[[BB48:.*]], label %[[BB49]]
+; CHECK-DEFAULT:       [[BB48]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP38]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB49]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP36]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP50:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT:    call void @__asan_poison_stack_memory(i64 [[TMP50]], i64 40)
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[TMP36]])
+; CHECK-DEFAULT-NEXT:    [[TMP51:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP52:%.*]] = inttoptr i64 [[TMP51]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 4, ptr [[TMP52]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP53:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP54:%.*]] = lshr i64 [[TMP53]], 3
+; CHECK-DEFAULT-NEXT:    [[TMP55:%.*]] = add i64 [[TMP54]], 2147450880
+; CHECK-DEFAULT-NEXT:    [[TMP56:%.*]] = inttoptr i64 [[TMP55]] to ptr
+; CHECK-DEFAULT-NEXT:    [[TMP57:%.*]] = load i8, ptr [[TMP56]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP58:%.*]] = icmp ne i8 [[TMP57]], 0
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP58]], label %[[BB59:.*]], label %[[BB64:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT:       [[BB59]]:
+; CHECK-DEFAULT-NEXT:    [[TMP60:%.*]] = and i64 [[TMP53]], 7
+; CHECK-DEFAULT-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i8
+; CHECK-DEFAULT-NEXT:    [[TMP62:%.*]] = icmp sge i8 [[TMP61]], [[TMP57]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP62]], label %[[BB63:.*]], label %[[BB64]]
+; CHECK-DEFAULT:       [[BB63]]:
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP53]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    unreachable
+; CHECK-DEFAULT:       [[BB64]]:
+; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP4]], align 1
+; CHECK-DEFAULT-NEXT:    [[TMP65:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT:    [[TMP66:%.*]] = inttoptr i64 [[TMP65]] to ptr
+; CHECK-DEFAULT-NEXT:    store i8 -8, ptr [[TMP66]], align 1
+; CHECK-DEFAULT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT:    [[TMP67:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; CHECK-DEFAULT-NEXT:    [[TMP68:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP68]], i64 [[TMP67]])
+; CHECK-DEFAULT-NEXT:    store i64 1172321806, ptr [[TMP5]], align 8
+; CHECK-DEFAULT-NEXT:    [[TMP69:%.*]] = add i64 [[TMP11]], 0
+; CHECK-DEFAULT-NEXT:    [[TMP70:%.*]] = inttoptr i64 [[TMP69]] to ptr
+; CHECK-DEFAULT-NEXT:    store i64 0, ptr [[TMP70]], align 1
+; CHECK-DEFAULT-NEXT:    ret void
+;
+; CHECK-NO-DYNAMIC-LABEL: define void @lifetime(
+; CHECK-NO-DYNAMIC-SAME: ) #[[ATTR1]] {
+; CHECK-NO-DYNAMIC-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 32
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 1102416563, ptr [[TMP4]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP6]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP7:%.*]] = add i64 [[TMP1]], 16
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP8]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 -868082052615769615, ptr [[TMP12]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP13:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 4, ptr [[TMP14]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 3, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP16:%.*]] = lshr i64 [[TMP15]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP17:%.*]] = add i64 [[TMP16]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP19]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP20]], label %[[BB21:.*]], label %[[BB26:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB21]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP22:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP22]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP24:%.*]] = icmp sge i8 [[TMP23]], [[TMP19]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP24]], label %[[BB25:.*]], label %[[BB26]]
+; CHECK-NO-DYNAMIC:       [[BB25]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP15]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB26]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[TMP3]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP27:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP28:%.*]] = inttoptr i64 [[TMP27]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP28]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP29:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP30:%.*]] = inttoptr i64 [[TMP29]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP30]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[ARR:%.*]] = alloca [10 x i32], align 16
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[ARR]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP33:%.*]] = add i64 [[TMP32]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP34:%.*]] = inttoptr i64 [[TMP33]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP36:%.*]] = icmp ne i8 [[TMP35]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP36]], label %[[BB37:.*]], label %[[BB42:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB37]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP38:%.*]] = and i64 [[TMP31]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP38]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP40:%.*]] = icmp sge i8 [[TMP39]], [[TMP35]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP40]], label %[[BB41:.*]], label %[[BB42]]
+; CHECK-NO-DYNAMIC:       [[BB41]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP31]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB42]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[ARR]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP43:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP44:%.*]] = inttoptr i64 [[TMP43]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 4, ptr [[TMP44]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP45:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP46:%.*]] = lshr i64 [[TMP45]], 3
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP47:%.*]] = add i64 [[TMP46]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP48:%.*]] = inttoptr i64 [[TMP47]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP50:%.*]] = icmp ne i8 [[TMP49]], 0
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP50]], label %[[BB51:.*]], label %[[BB56:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC:       [[BB51]]:
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP52:%.*]] = and i64 [[TMP45]], 7
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP53:%.*]] = trunc i64 [[TMP52]] to i8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP54:%.*]] = icmp sge i8 [[TMP53]], [[TMP49]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP54]], label %[[BB55:.*]], label %[[BB56]]
+; CHECK-NO-DYNAMIC:       [[BB55]]:
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP45]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    unreachable
+; CHECK-NO-DYNAMIC:       [[BB56]]:
+; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[TMP3]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP57:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP58:%.*]] = inttoptr i64 [[TMP57]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP58]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT:    store i64 1172321806, ptr [[TMP4]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP59:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NO-DYNAMIC-NEXT:    [[TMP60:%.*]] = inttoptr i64 [[TMP59]] to ptr
+; CHECK-NO-DYNAMIC-NEXT:    store i64 0, ptr [[TMP60]], align 1
+; CHECK-NO-DYNAMIC-NEXT:    ret void
+;
 
   ; Regular variable lifetime intrinsics.
   %i = alloca i32, align 4
 
   ; Poison memory in prologue: F1F1F1F1F8F3F3F3
-  ; CHECK: store i64 -868082052615769615, ptr %{{[0-9]+}}
 
   ; Memory is unpoisoned at llvm.lifetime.start
   call void @llvm.lifetime.start.p0(i64 3, ptr %i)
-  ; CHECK: store i8 4, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 4, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ; Memory is poisoned at every call to llvm.lifetime.end
   call void @llvm.lifetime.end.p0(i64 2, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ; Lifetime intrinsics for array.
   %arr = alloca [10 x i32], align 16
 
   call void @llvm.lifetime.start.p0(i64 40, ptr %arr)
-  ; CHECK-DEFAULT: call void @__asan_unpoison_stack_memory(i64 %{{[^ ]+}}, i64 40)
-  ; CHECK-NO-DYNAMIC-NOT: call void @__asan_unpoison_stack_memory(i64 %{{[^ ]+}}, i64 40)
 
   store volatile i8 0, ptr %arr
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 40, ptr %arr)
-  ; CHECK-DEFAULT: call void @__asan_poison_stack_memory(i64 %{{[^ ]+}}, i64 40)
-  ; CHECK-NO-DYNAMIC-NOT: call void @__asan_poison_stack_memory(i64 %{{[^ ]+}}, i64 40)
 
   ; One more lifetime start/end for the same variable %i.
   call void @llvm.lifetime.start.p0(i64 2, ptr %i)
-  ; CHECK: store i8 4, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   call void @llvm.lifetime.end.p0(i64 4, ptr %i)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.end
 
   ; Memory is unpoisoned at function exit (only once).
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: ret void
   ret void
 }
 
 ; Check that arguments of lifetime may come from phi nodes.
 define void @phi_args(i1 %x) sanitize_address {
-  ; CHECK-LABEL: define void @phi_args(i1 %x)
+; CHECK-LABEL: define void @phi_args(
+; CHECK-SAME: i1 [[X:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.2 to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @phi_args to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i64 -868082052615769615, ptr [[TMP11]], align 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    store i8 0, ptr [[TMP13]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[TMP15]], 2147450880
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    [[TMP18:%.*]] = load i8, ptr [[TMP17]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne i8 [[TMP18]], 0
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[BB20:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp sge i8 [[TMP22]], [[TMP18]]
+; CHECK-NEXT:    br i1 [[TMP23]], label %[[BB24:.*]], label %[[BB25]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__asan_report_store1(i64 [[TMP14]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    store volatile i8 0, ptr [[TMP2]], align 1
+; CHECK-NEXT:    br i1 [[X]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB0]]:
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[I_PHI:%.*]] = phi ptr [ [[TMP2]], %[[BB25]] ], [ [[TMP2]], %[[BB0]] ]
+; CHECK-NEXT:    [[TMP26:%.*]] = add i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr
+; CHECK-NEXT:    store i8 -8, ptr [[TMP27]], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[I_PHI]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-NEXT:    store i64 0, ptr [[TMP29]], align 1
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %i = alloca i64, align 4
 
   ; Poison memory in prologue: F1F1F1F1F8F3F3F3
-  ; CHECK: store i64 -868082052615769615, ptr %{{[0-9]+}}
 
   call void @llvm.lifetime.start.p0(i64 8, ptr %i)
-  ; CHECK: store i8 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.start
 
   store volatile i8 0, ptr %i
-  ; CHECK: store volatile
 
   br i1 %x, label %bb0, label %bb1
 
@@ -120,49 +401,101 @@ bb0:
 bb1:
   %i.phi = phi ptr [ %i, %entry ], [ %i, %bb0 ]
   call void @llvm.lifetime.end.p0(i64 8, ptr %i.phi)
-  ; CHECK: store i8 -8, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: llvm.lifetime.end
 
   ret void
-  ; CHECK: store i64 0, ptr %{{[0-9]+}}
-  ; CHECK-NEXT: ret void
 }
 
 ; Check that arguments of lifetime may come from getelementptr nodes.
 define void @getelementptr_args(i64 %i) sanitize_address{
-  ; CHECK-LABEL: define void @getelementptr_args
+; CHECK-LABEL: define void @getelementptr_args(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 1216, align 32
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP0]], 1184
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT:    store i64 1102416563, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.3 to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    store i64 ptrtoint (ptr @getelementptr_args to i64), ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 2147450880
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    store i32 -235802127, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP14]], i64 128)
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP11]], 132
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    store i64 -940422246894996750, ptr [[TMP16]], align 1
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP11]], 140
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    store i64 -940422246894996750, ptr [[TMP18]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP11]], 150
+; CHECK-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT:    store i16 -3085, ptr [[TMP20]], align 1
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP21]], i64 128)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1024, ptr [[TMP2]])
+; CHECK-NEXT:    [[AI:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP4]], i64 0, i64 [[I]]
+; CHECK-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[AI]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[TMP23]], 2147450880
+; CHECK-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[TMP25]], align 1
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne i8 [[TMP26]], 0
+; CHECK-NEXT:    br i1 [[TMP27]], label %[[BB28:.*]], label %[[BB29:.*]]
+; CHECK:       [[BB28]]:
+; CHECK-NEXT:    call void @__asan_report_store8(i64 [[TMP22]]) #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB29]]:
+; CHECK-NEXT:    store ptr [[TMP2]], ptr [[AI]], align 8
+; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP30]], i64 128)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1024, ptr [[TMP2]])
+; CHECK-NEXT:    store i64 1172321806, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[TMP11]], 0
+; CHECK-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP31]], i64 148)
+; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[TMP11]], 150
+; CHECK-NEXT:    [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT:    store i16 0, ptr [[TMP33]], align 1
+; CHECK-NEXT:    ret void
+;
 entry:
   %x = alloca [1024 x i8], align 16
   %a = alloca [2 x ptr], align 8
 
   ; F1F1F1F1
-  ; CHECK: store i32 -235802127, ptr %{{[0-9]+}}
-  ; CHECK: call void @__asan_set_shadow_f8(i64 %[[#]], i64 128)
   ; 0xf2f2f2f2f2f2f2f2
-  ; CHECK: store i64 -940422246894996750, ptr %[[#]]
   ; 0xf2f2f2f2f2f2f2f2
-  ; CHECK: store i64 -940422246894996750, ptr %[[#]]
 
   call void @llvm.lifetime.start.p0(i64 1024, ptr %x)
-  ; CHECK: call void @__asan_set_shadow_00(i64 %{{[0-9]+}}, i64 128)
-  ; CHECK-NEXT: call void @llvm.lifetime.start
 
   %ai = getelementptr inbounds [2 x ptr], ptr %a, i64 0, i64 %i
   store ptr %x, ptr %ai, align 8
-  ; CHECK: store ptr
 
   call void @llvm.lifetime.end.p0(i64 1024, ptr %x)
-  ; CHECK: call void @__asan_set_shadow_f8(i64 %{{[0-9]+}}, i64 128)
-  ; CHECK-NEXT: call void @llvm.lifetime.end
 
   ret void
-  ; CHECK: call void @__asan_set_shadow_00(i64 %{{[0-9]+}}, i64 148)
-  ; CHECK: store i16 0, ptr %[[#]], align 1
-  ; CHECK-NEXT: ret void
 }
 
 define void @zero_sized(i64 %a) #0 {
-; CHECK-LABEL: define void @zero_sized(i64 %a)
+; CHECK-LABEL: define void @zero_sized(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    [[B:%.*]] = alloca [0 x i8], align 1
+; CHECK-NEXT:    store i64 [[A]], ptr [[A_ADDR]], align 8
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 0, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 0, ptr [[B]])
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %a.addr = alloca i64, align 8
@@ -170,11 +503,13 @@ entry:
   store i64 %a, ptr %a.addr, align 8
 
   call void @llvm.lifetime.start.p0(i64 0, ptr %b) #2
-  ; CHECK: call void @llvm.lifetime.start
 
   call void @llvm.lifetime.end.p0(i64 0, ptr %b) #2
-  ; CHECK: call void @llvm.lifetime.end
 
   ret void
-  ; CHECK-NEXT: ret void
 }
+;.
+; CHECK-DEFAULT: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
+; CHECK-NO-DYNAMIC: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
index 279bb262bff04..5f310a9954ad0 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
@@ -163,6 +163,51 @@ v_tanh_bf16 v5, src_scc
 v_tanh_bf16 v127, 0x8000
 // GFX1250: v_tanh_bf16_e32 v127, 0x8000            ; encoding: [0xff,0x94,0xfe,0x7e,0x00,0x80,0x00,0x00]
 
+v_prng_b32 v5, v1
+// GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, v255
+// GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, s1
+// GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, s105
+// GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_lo
+// GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_hi
+// GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, ttmp15
+// GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, m0
+// GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_lo
+// GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_hi
+// GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, null
+// GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, -1
+// GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, 0.5
+// GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, src_scc
+// GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+v_prng_b32 v255, 0xaf123456
+// GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
 v_rcp_bf16 v5, v1
 // GFX1250: v_rcp_bf16_e32 v5, v1                   ; encoding: [0x01,0xf3,0x0a,0x7e]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
index 76272d25d92d4..aa2e028f661e1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
@@ -169,6 +169,51 @@ v_tanh_bf16 v127, 0x8000
 v_tanh_bf16 v5.h, v1.h
 // GFX1250: v_tanh_bf16_e32 v5.h, v1.h              ; encoding: [0x81,0x95,0x0a,0x7f]
 
+v_prng_b32 v5, v1
+// GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, v255
+// GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, s1
+// GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, s105
+// GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_lo
+// GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_hi
+// GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, ttmp15
+// GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, m0
+// GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_lo
+// GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_hi
+// GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, null
+// GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, -1
+// GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, 0.5
+// GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, src_scc
+// GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+v_prng_b32 v255, 0xaf123456
+// GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
 v_rcp_bf16 v5, v1
 // GFX1250: v_rcp_bf16_e32 v5, v1                   ; encoding: [0x01,0xf3,0x0a,0x7e]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
index 0a8ee84561d33..e1cd2e3043693 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
@@ -170,6 +170,58 @@ v_tanh_bf16 v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 f
 // GFX1250: v_tanh_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x94,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf2,0x0a,0x7e,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
index d4afb9d9b2d9a..c1d3238b65cbd 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
@@ -178,6 +178,58 @@ v_tanh_bf16 v5.h, v1.h quad_perm:[3,2,1,0]
 // GFX1250: v_tanh_bf16_dpp v5.h, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf2,0x0a,0x7e,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
index a7cb6bf8de69c..100e9f92ff58b 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
@@ -38,6 +38,18 @@ v_tanh_bf16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX1250: v_tanh_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x94,0xfe,0x7e,0x7f,0x00,0x00,0x00]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf2,0x0a,0x7e,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
index 6acab7edc0d49..2ae103545443c 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
@@ -46,6 +46,18 @@ v_tanh_bf16 v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_tanh_bf16_dpp v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x94,0x0a,0x7f,0x81,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf2,0x0a,0x7e,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
index 7486d849253e8..9c6a9127d82e4 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
@@ -127,6 +127,42 @@ v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp
 v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp
 // GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2 clamp ; encoding: [0x01,0x88,0xec,0xd5,0x03,0x01,0x00,0x00]
 
+v_prng_b32_e64 v5, v1
+// GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, v255
+// GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, s1
+// GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, s105
+// GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_lo
+// GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_hi
+// GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, ttmp15
+// GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, m0
+// GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_lo
+// GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_hi
+// GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, null
+// GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, -1
+// GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
 v_tanh_f32_e64 v5, v1
 // GFX1250: v_tanh_f32_e64 v5, v1                   ; encoding: [0x05,0x00,0x9e,0xd5,0x01,0x01,0x00,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
index b59b8b31e2d5f..2f57d1c331c42 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
@@ -223,6 +223,42 @@ v_tanh_f16_e64 v255, -|0x8000| clamp div:2
 v_tanh_f16 v5.l, v128.h
 // GFX1250: v_tanh_f16_e64 v5.l, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0x9f,0xd5,0x80,0x01,0x00,0x00]
 
+v_prng_b32_e64 v5, v1
+// GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, v255
+// GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, s1
+// GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, s105
+// GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_lo
+// GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_hi
+// GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, ttmp15
+// GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, m0
+// GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_lo
+// GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_hi
+// GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, null
+// GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, -1
+// GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
 v_rcp_bf16_e64 v5, v1
 // GFX1250: v_rcp_bf16_e64 v5, v1                   ; encoding: [0x05,0x00,0xf9,0xd5,0x01,0x01,0x00,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
index f7f20f46161ce..29bb842b529b7 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
@@ -170,6 +170,50 @@ v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mas
 // GFX1250: v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xca,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
index e1241b01ccae1..7df92751c38d1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
@@ -178,6 +178,50 @@ v_tanh_bf16_e64_dpp v5.h, v128.h quad_perm:[3,2,1,0]
 // GFX1250: v_tanh_bf16_e64_dpp v5.h, v128.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xca,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
index 0106175301d20..d235aeb9f3e62 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
@@ -50,6 +50,10 @@ v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX1250: v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xca,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
index 93b86f3ffb841..f25e2a5882436 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
@@ -58,6 +58,10 @@ v_tanh_bf16_e64_dpp v5.h, v128.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_tanh_bf16_e64_dpp v5.h, v128.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xca,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
index 5f37ba91e071b..aa968b2bb2bee 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
@@ -200,6 +200,51 @@
 0x81,0x95,0x0a,0x7f
 # GFX1250-REAL16: v_tanh_bf16_e32 v5.h, v1.h              ; encoding: [0x81,0x95,0x0a,0x7f]
 
+0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf
+# GFX1250: v_prng_b32_e32 v255, 0xaf123456         ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
+0xc1,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, -1                   ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+0xf0,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, 0.5                  ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+0x7f,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, exec_hi              ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+0x7e,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, exec_lo              ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+0x7d,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, m0                   ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+0x7c,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, null                 ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+0x01,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, s1                   ; encoding: [0x01,0x96,0x0a,0x7e]
+
+0x69,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, s105                 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+0xfd,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, src_scc              ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+0x7b,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, ttmp15               ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+0x01,0x97,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, v1                   ; encoding: [0x01,0x97,0x0a,0x7e]
+
+0xff,0x97,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, v255                 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+0x6b,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, vcc_hi               ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+0x6a,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, vcc_lo               ; encoding: [0x6a,0x96,0x0a,0x7e]
+
 0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_e32 v127.l, 0x8000           ; encoding: [0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_e32 v127, 0x8000             ; encoding: [0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
index 57bee2766ce44..913a2a916ff62 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
@@ -162,6 +162,45 @@
 0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff
 # GFX1250-REAL16: v_tanh_bf16_dpp v5.h, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff]
 
+0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01
+# GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13
+# GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+
 0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30
 # GFX1250-REAL16: v_rcp_bf16_dpp v127.l, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
 # GFX1250-FAKE16: v_rcp_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
index 28ec6b11b4de3..4afe44e241bf3 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
@@ -43,6 +43,15 @@
 # GFX1250-REAL16: v_tanh_bf16_dpp v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x94,0x0a,0x7f,0x81,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_add_f64_e32 v[156:157], v[129:130], v[187:188] ; encoding: [0x81,0x77,0x39,0x05]
 
+0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00
+# GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+
+0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
 0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_dpp v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
index 5004762729701..1cf3b8807d044 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
@@ -175,6 +175,42 @@
 # GFX1250-REAL16: v_tanh_bf16_e64 v5.l, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0xca,0xd5,0x80,0x01,0x00,0x00]
 # GFX1250-FAKE16: v_tanh_bf16_e64 v5, v128                ; encoding: [0x05,0x00,0xca,0xd5,0x80,0x01,0x00,0x00]
 
+0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, -1                   ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, exec_hi              ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, exec_lo              ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, m0                   ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, null                 ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, s1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, s105                 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, ttmp15               ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, v1                   ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, v255                 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, vcc_hi               ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, vcc_lo               ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
 0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00
 # GFX1250-REAL16: v_rcp_bf16_e64 v255.l, -|0x8000| clamp div:2 ; encoding: [0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00]
 # GFX1250-FAKE16: v_rcp_bf16_e64 v255, -|0x8000| clamp div:2 ; encoding: [0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
index de908b95d94f9..83a647ad7c658 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
@@ -104,6 +104,39 @@
 # GFX1250-REAL16: v_tanh_f16_e64_dpp v5.h, v128.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x9f,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 # GFX1250-FAKE16: v_tanh_f16_e64_dpp v5, v128 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x9f,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+
 0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30
 # GFX1250-REAL16: v_rsq_bf16_e64_dpp v255.l, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
 # GFX1250-FAKE16: v_rsq_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
index cfe7173c383b3..ef5ede4d1d453 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
 
 0xff,0x81,0x9e,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00
 # GFX1250: v_tanh_f32_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0x9e,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
@@ -34,6 +34,9 @@
 # GFX1250-REAL16: v_tanh_f16_e64_dpp v5.h, v128.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0x9f,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_tanh_f16_e64_dpp v5, v128 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x9f,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 
+0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
 0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00
 # GFX1250-REAL16: v_rsq_bf16_e64_dpp v255.l, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
 # GFX1250-FAKE16: v_rsq_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll b/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
index e390d4bdca632..303afc207c023 100644
--- a/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
@@ -12,6 +12,6 @@ define ptr @undeclared_customalloc(i64 %size, i64 %align) {
   ret ptr %call
 }
 
-declare ptr @customalloc2(i64, i64) allockind("alloc") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
+declare ptr @customalloc2(i64, i64) allockind("alloc,uninitialized") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
 ; CHECK-DAG: declare ptr @customalloc2_zeroed(i64, i64) #[[CA2ATTR:[0-9]+]]
 ; CHECK-DAG: attributes #[[CA2ATTR]] = { allockind("alloc,zeroed") "alloc-family"="customalloc2" }
diff --git a/llvm/test/Transforms/LoopInterchange/force-interchange.ll b/llvm/test/Transforms/LoopInterchange/force-interchange.ll
new file mode 100644
index 0000000000000..c33ecdf7d9905
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/force-interchange.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -passes=loop-interchange -pass-remarks-output=%t -disable-output -loop-interchange-profitabilities=ignore -S
+; RUN: FileCheck --input-file=%t %s
+
+; There should be no reason to interchange this, unless it is forced.
+;
+;     for (int i = 0; i<1024; i++)
+;       for (int j = 0; j<1024; j++)
+;         A[i][j] = 42;
+;
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Interchanged
+; CHECK-NEXT: Function:        f
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Loop interchanged with enclosing loop.
+; CHECK-NEXT: ...
+
+ at A = dso_local local_unnamed_addr global [1024 x [1024 x i32]] zeroinitializer, align 4
+
+define dso_local void @f() local_unnamed_addr #0 {
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %inner.header ]
+  br label %inner.body
+
+inner.header:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond20.not = icmp eq i64 %i.next, 1024
+  br i1 %exitcond20.not, label %exit, label %outer.header
+
+inner.body:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
+  %arrayidx6 = getelementptr inbounds nuw [1024 x [1024 x i32]], ptr @A, i64 0, i64 %i, i64 %j
+  store i32 42, ptr %arrayidx6, align 4
+  %j.next = add nuw nsw i64 %j, 1
+  %exitcond.not = icmp eq i64 %j.next, 1024
+  br i1 %exitcond.not, label %inner.header, label %inner.body
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
new file mode 100644
index 0000000000000..c7a0de22b200b
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=loop-reduce -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(ptr %p, i64 %idx) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4 x [4 x i32]], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[IDX]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 48
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[ALLOCA]], i64 48
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], %[[LOOP]] ], [ -8, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[LSR_IV]], 2
+; CHECK-NEXT:    [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[SCEVGEP8]], i64 32
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[SCEVGEP9]], align 4
+; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[SCEVGEP6]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[SCEVGEP7]], align 4
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[SCEVGEP5]], align 4
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[LSR_IV]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[SCEVGEP2]], align 4
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], 4
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %alloca = alloca [4 x [4 x i32]], align 16
+  call void @llvm.lifetime.start.p0(i64 64, ptr %alloca)
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %gep1 = getelementptr [4 x [12 x [4 x [4 x i32]]]], ptr %p, i64 0, i64 0, i64 0, i64 %indvars.iv, i64 0
+  %0 = load i32, ptr %gep1, align 4
+  %gep2 = getelementptr [6 x [4 x [4 x i32]]], ptr %p, i64 0, i64 0, i64 0, i64 %indvars.iv
+  %1 = load i32, ptr %gep2, align 4
+  %gep3 = getelementptr [4 x [4 x i32]], ptr %alloca, i64 0, i64 3, i64 %indvars.iv
+  %2 = load i32, ptr %gep3, align 4
+  %gep4 = getelementptr [6 x [4 x [4 x i32]]], ptr %p, i64 0, i64 %idx, i64 3, i64 %indvars.iv
+  %3 = load i32, ptr %gep4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv, 1
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  call void @llvm.lifetime.end.p0(i64 64, ptr %alloca)
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll b/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
new file mode 100644
index 0000000000000..2d1543185098f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
@@ -0,0 +1,38 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-REGS-VP
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -force-target-num-vector-regs=1 -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NOREGS-VP
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @dotp(ptr %a, ptr %b) #0 {
+; CHECK-REGS-VP-NOT: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; CHECK-REGS-VP: LV: Selecting VF: vscale x 8.
+;
+; CHECK-NOREGS-VP: LV(REG): Not considering vector loop of width vscale x 8 because it uses too many registers
+; CHECK-NOREGS-VP: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; CHECK-NOREGS-VP: LV: Selecting VF: vscale x 4.
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %gep.b = getelementptr i8, ptr %b, i64 %iv
+  %load.b = load i8, ptr %gep.b, align 1
+  %ext.b = zext i8 %load.b to i32
+  %mul = mul i32 %ext.b, %ext.a
+  %sub = sub i32 0, %mul
+  %add = add i32 %accum, %sub
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
index 8e09847e9264e..cfff11758a37a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
@@ -58,7 +58,6 @@ define void @test(i32 %0, i8 %1, i64 %2, float %3) {
 ; CHECK-NEXT:    br label %[[BB54:.*]]
 ; CHECK:       [[BB54]]:
 ; CHECK-NEXT:    [[TMP54:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP17]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 0, ptr null)
 ; CHECK-NEXT:    [[TMP55:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP21]])
 ; CHECK-NEXT:    [[TMP56:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <8 x float> [[TMP56]], <8 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison>
@@ -198,7 +197,6 @@ define void @test(i32 %0, i8 %1, i64 %2, float %3) {
   %95 = or i64 %94, %91
   %96 = or i64 %95, %37
   store i64 %96, ptr null, align 1
-  call void @llvm.lifetime.start.p0(i64 0, ptr null)
   store i64 %42, ptr null, align 1
   %97 = bitcast float %3 to i32
   %98 = icmp ult i32 %97, 1325400064
diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp
index 422eb855ba2cf..2b43d27f292a0 100644
--- a/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -131,20 +131,6 @@ class CommentWriter : public AssemblyAnnotationWriter {
         printDebugLoc(DL,OS);
         OS << "]";
       }
-      if (const DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(I)) {
-        if (!Padded) {
-          OS.PadToColumn(50);
-          OS << ";";
-        }
-        OS << " [debug variable = " << DDI->getVariable()->getName() << "]";
-      }
-      else if (const DbgValueInst *DVI = dyn_cast<DbgValueInst>(I)) {
-        if (!Padded) {
-          OS.PadToColumn(50);
-          OS << ";";
-        }
-        OS << " [debug variable = " << DVI->getVariable()->getName() << "]";
-      }
     }
   }
 };
diff --git a/llvm/unittests/CodeGen/LexicalScopesTest.cpp b/llvm/unittests/CodeGen/LexicalScopesTest.cpp
index 3d707462fa615..563d496d1e600 100644
--- a/llvm/unittests/CodeGen/LexicalScopesTest.cpp
+++ b/llvm/unittests/CodeGen/LexicalScopesTest.cpp
@@ -67,7 +67,7 @@ class LexicalScopesTest : public testing::Test {
     BeanInst.Opcode = 1;
     BeanInst.Size = 1;
 
-    memset(&DbgValueInst, 0, sizeof(DbgValueInst));
+    memset(&DbgValueInst, 0, sizeof(MCInstrDesc));
     DbgValueInst.Opcode = TargetOpcode::DBG_VALUE;
     DbgValueInst.Size = 1;
     DbgValueInst.Flags = 1U << MCID::Meta;
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index 41bf863420304..baa13e1199eea 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -188,6 +188,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgRecord) {
   SmallVector<DbgValueInst *, 1> DVIs;
   SmallVector<DbgVariableRecord *, 1> DVRs;
   findDbgValues(DVIs, &I, &DVRs);
+  assert(DVIs.empty());
 
   // Delete %b. The dbg.value should now point to undef.
   I.eraseFromParent();
@@ -314,6 +315,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
   SmallVector<DbgValueInst *, 2> DVIs;
   SmallVector<DbgVariableRecord *, 2> DVRs;
   findDbgValues(DVIs, &I, &DVRs);
+  assert(DVIs.empty());
   ASSERT_EQ(DVRs.size(), 2u);
 
   // Delete %b. The DbgVariableRecord should now point to undef.
diff --git a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
index 0b00734fc4d75..1daf381ee2862 100644
--- a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
+++ b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
@@ -54,20 +54,13 @@ struct DebugInfoDrop : public FunctionPass {
 struct DebugValueDrop : public FunctionPass {
   static char ID;
   bool runOnFunction(Function &F) override {
-    SmallVector<DbgVariableIntrinsic *, 4> Dbgs;
     for (BasicBlock &BB : F) {
-      // Remove dbg var intrinsics.
       for (Instruction &I : BB) {
-        if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
-          Dbgs.push_back(DVI);
-        // If there are any non-intrinsic records (DbgRecords), drop those too.
+        // If there are any debug records, drop them.
         I.dropDbgRecords();
       }
     }
 
-    for (auto &I : Dbgs)
-      I->eraseFromParent();
-
     return true;
   }
 
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index b922216ef8893..dd2a6249c7cf9 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -633,62 +633,6 @@ TEST(Local, ChangeToUnreachable) {
   EXPECT_EQ(DLA, DLB);
 }
 
-TEST(Local, FindDbgUsers) {
-  LLVMContext Ctx;
-  std::unique_ptr<Module> M = parseIR(Ctx,
-                                      R"(
-  define dso_local void @fun(ptr %a) #0 !dbg !11 {
-  entry:
-      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
-    ret void
-  }
-
-  !llvm.dbg.cu = !{!0}
-  !llvm.module.flags = !{!2, !3, !9}
-  !llvm.ident = !{!10}
-
-  !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-  !1 = !DIFile(filename: "test.cpp", directory: "/")
-  !2 = !{i32 7, !"Dwarf Version", i32 5}
-  !3 = !{i32 2, !"Debug Info Version", i32 3}
-  !4 = !{i32 1, !"wchar_size", i32 4}
-  !9 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
-  !10 = !{!"clang version 17.0.0"}
-  !11 = distinct !DISubprogram(name: "fun", linkageName: "fun", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !14)
-  !12 = !DISubroutineType(types: !13)
-  !13 = !{null}
-  !14 = !{}
-  !15 = distinct !DIAssignID()
-  !16 = !DILocalVariable(name: "x", scope: !11, file: !1, line: 2, type: !17)
-  !17 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 64)
-  !18 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-  !19 = !DILocation(line: 0, scope: !11)
-  )");
-
-  bool BrokenDebugInfo = true;
-  verifyModule(*M, &errs(), &BrokenDebugInfo);
-  ASSERT_FALSE(BrokenDebugInfo);
-
-  // Convert to debug intrinsics as we want to test findDbgUsers and
-  // findDbgValue's debug-intrinsic-finding code here.
-  // TODO: Remove this test when debug intrinsics are removed.
-  M->convertFromNewDbgValues();
-
-  Function &Fun = *cast<Function>(M->getNamedValue("fun"));
-  Value *Arg = Fun.getArg(0);
-  SmallVector<DbgVariableIntrinsic *> Users;
-  // Arg (%a) is used twice by a single dbg.assign. Check findDbgUsers returns
-  // only 1 pointer to it rather than 2.
-  findDbgUsers(Users, Arg);
-  EXPECT_EQ(Users.size(), 1u);
-
-  SmallVector<DbgValueInst *> Vals;
-  // Arg (%a) is used twice by a single dbg.assign. Check findDbgValues returns
-  // only 1 pointer to it rather than 2.
-  findDbgValues(Vals, Arg);
-  EXPECT_EQ(Vals.size(), 1u);
-}
-
 TEST(Local, FindDbgRecords) {
   // DbgRecord copy of the FindDbgUsers test above.
   LLVMContext Ctx;
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 82ec8121548c9..5309b5d095134 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -35,7 +35,6 @@ if (current_toolchain == default_toolchain) {
       "_LIBCPP_HAS_LOCALIZATION=1",
       "_LIBCPP_HAS_UNICODE=1",
       "_LIBCPP_HAS_WIDE_CHARACTERS=1",
-      "_LIBCPP_HAS_NO_STD_MODULES=",
       "_LIBCPP_HAS_TERMINAL=1",
       "_LIBCPP_INSTRUMENTED_WITH_ASAN=",
       "_LIBCPP_ABI_DEFINES=",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index d4adeddd9b4e4..3d11ce566207a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -176,6 +176,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUPreLegalizerCombiner.cpp",
     "AMDGPUPreloadKernArgProlog.cpp",
     "AMDGPUPreloadKernelArguments.cpp",
+    "AMDGPUPrepareAGPRAlloc.cpp",
     "AMDGPUPrintfRuntimeBinding.cpp",
     "AMDGPUPromoteAlloca.cpp",
     "AMDGPUPromoteKernelArguments.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 946b63f8a54fb..7ed0d3c6824e2 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -286,6 +286,7 @@ group("test") {
     "//llvm/tools/llvm-extract",
     "//llvm/tools/llvm-gsymutil:llvm-gsymutil",
     "//llvm/tools/llvm-ifs",
+    "//llvm/tools/llvm-ir2vec",
     "//llvm/tools/llvm-isel-fuzzer",
     "//llvm/tools/llvm-jitlink",
     "//llvm/tools/llvm-jitlink/llvm-jitlink-executor",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
new file mode 100644
index 0000000000000..07a795122c76c
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
@@ -0,0 +1,9 @@
+executable("llvm-ir2vec") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+  ]
+  sources = [ "llvm-ir2vec.cpp" ]
+}
diff --git a/mlir/docs/Tutorials/transform/Ch2.md b/mlir/docs/Tutorials/transform/Ch2.md
index 0f45f5607bab9..08c4e0f9d859c 100644
--- a/mlir/docs/Tutorials/transform/Ch2.md
+++ b/mlir/docs/Tutorials/transform/Ch2.md
@@ -133,6 +133,8 @@ This will generate two files, `MyExtension.h.inc` and `MyExtension.cpp.inc`, tha
 ```c++
 // In MyExtension.cpp.
 
+#include "MyExtension.h"
+
 #define GET_OP_CLASSES
 #include "MyExtension.cpp.inc"
 
@@ -283,7 +285,7 @@ void registerMyExtension(::mlir::DialectRegistry &registry) {
 }
 ```
 
-After registering the extension, it becomes possible to use our new operation in the Transform dialect interpreter. The upstream testing pass can be used as is.
+After registering the extension, it becomes possible to use our new operation in the Transform dialect interpreter. The upstream testing pass can be used as is. It actually exists in `mlir/test/Examples/transform/Ch2/sequence.mlir`, which contains the `microkernel` implementation. 
 
 ```mlir
 module attributes {transform.with_named_sequence} {
@@ -300,7 +302,7 @@ module attributes {transform.with_named_sequence} {
 
     // The actual tiling transformation takes tile sizes as attributes. It
     // produces a handle to the loop generated during tiling.
-    %loop, %tiled = transform.structured.tile_using_forall %max
+    %tiled, %loop = transform.structured.tile_using_forall %max
                     tile_sizes [8, 32]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
@@ -311,32 +313,32 @@ module attributes {transform.with_named_sequence} {
     // a single handle to all operations and give it to
     // `fuse_into_containing_op` that would take care of the ordering in this
     // case.
-    %add_fused = transform.structured.fuse_into_containing_op %add into %loop
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
-    %matmul_fused = transform.structured.fuse_into_containing_op %arg1
-                    into %loop
+    %add_fused, %loop2 = transform.structured.fuse_into_containing_op %add into %loop
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_fused, %loop3 = transform.structured.fuse_into_containing_op %arg1
+                    into %loop2
         : (!transform.op<"linalg.matmul">, !transform.any_op)
-       -> !transform.any_op
+       -> (!transform.any_op, !transform.any_op)
 
     // Tile again to get the desired size. Note that this time this tiles the
     // "add" operation and fuses matmul into the loop, but doesn't affect the
     // "max" operation. This illustrates the precise targeting with the
     // transform dialect. Otherwise, it is difficult to differentiate "add" and
     // "max", both of which having the same kind.
-    %loop_2, %tiled_2 = transform.structured.tile_using_forall %add_fused
+    %tiled_second, %loop_second = transform.structured.tile_using_forall %add_fused
                         tile_sizes [4, 4]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
-    %matmul_fused_2 = transform.structured.fuse_into_containing_op %matmul_fused
-                      into %loop_2
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
+    %matmul_fused_2, %loop_second_2 = transform.structured.fuse_into_containing_op %matmul_fused
+                      into %loop_second
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     // Since outlining is currently only implemented for region-holding
     // operations such as loops, use tiling to size 1 to materialize the outer
     // loop that is going to be outlined.
-    %outline_target, %_ = transform.structured.tile_using_forall %tiled_2 tile_sizes [1]
+    %_0, %loop_third = transform.structured.tile_using_forall %tiled_second tile_sizes [1]
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
-    transform.structured.fuse_into_containing_op %matmul_fused_2 into %outline_target
-        : (!transform.any_op, !transform.any_op) -> !transform.any_op
+    %_1, %outline_target = transform.structured.fuse_into_containing_op %matmul_fused_2 into %loop_third
+        : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
     %func, %call = transform.loop.outline %outline_target
                    {func_name = "outlined"}
         : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
diff --git a/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h b/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
index ac4915901fdec..ff99d7ce96daf 100644
--- a/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
+++ b/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
@@ -21,9 +21,6 @@ class RewritePatternSet;
 /// Collect a set of patterns to convert from the vector to XeGPU ops.
 void populateVectorToXeGPUConversionPatterns(RewritePatternSet &patterns);
 
-/// Create a pass to convert ops from vector to XeGPU.
-std::unique_ptr<Pass> createConvertVectorToXeGPUPass();
-
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_VECTORTOXEGPU_VECTORTOXEGPU_H
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 12c7cc62905c9..d4400547f9568 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -1115,6 +1115,18 @@ struct AMDGPUStreamTy {
     return Plugin::success();
   }
 
+  /// Complete pending post actions until and including the event in target
+  /// slot.
+  Error completeUntil(uint32_t TargetSlot) {
+    for (uint32_t Slot = 0; Slot <= TargetSlot; ++Slot) {
+      // Take the post action of the operation if any.
+      if (auto Err = Slots[Slot].performAction())
+        return Err;
+    }
+
+    return Plugin::success();
+  }
+
   /// Make the current stream wait on a specific operation of another stream.
   /// The idea is to make the current stream waiting on two signals: 1) the last
   /// signal of the current stream, and 2) the last signal of the other stream.
@@ -1502,6 +1514,11 @@ struct AMDGPUStreamTy {
     return complete();
   }
 
+  /// Synchronize the stream until the given event. The current thread waits
+  /// until the provided event is finalized, and it performs the pending post
+  /// actions for that and prior events.
+  Error synchronizeOn(AMDGPUEventTy &Event);
+
   /// Query the stream and complete pending post actions if operations finished.
   /// Return whether all the operations completed. This operation does not block
   /// the calling thread.
@@ -1575,6 +1592,21 @@ struct AMDGPUEventTy {
     return Stream.waitEvent(*this);
   }
 
+  Error sync() {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    if (!RecordedStream)
+      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
+                           "event does not have any recorded stream");
+
+    // No need to wait on anything, the recorded stream already finished the
+    // corresponding operation.
+    if (RecordedSlot < 0)
+      return Plugin::success();
+
+    return RecordedStream->synchronizeOn(*this);
+  }
+
 protected:
   /// The stream registered in this event.
   AMDGPUStreamTy *RecordedStream;
@@ -1630,6 +1662,22 @@ Error AMDGPUStreamTy::waitEvent(const AMDGPUEventTy &Event) {
   return waitOnStreamOperation(RecordedStream, Event.RecordedSlot);
 }
 
+Error AMDGPUStreamTy::synchronizeOn(AMDGPUEventTy &Event) {
+  std::lock_guard<std::mutex> Lock(Mutex);
+
+  // Wait until the requested slot has completed
+  if (auto Err = Slots[Event.RecordedSlot].Signal->wait(
+          StreamBusyWaitMicroseconds, &Device))
+    return Err;
+
+  // If the event is the last one in the stream, just do a full finalize
+  if (Event.RecordedSlot == last())
+    return complete();
+
+  // Otherwise, only finalize until the appropriate event
+  return completeUntil(Event.RecordedSlot);
+}
+
 struct AMDGPUStreamManagerTy final
     : GenericDeviceResourceManagerTy<AMDGPUResourceRef<AMDGPUStreamTy>> {
   using ResourceRef = AMDGPUResourceRef<AMDGPUStreamTy>;
@@ -2540,8 +2588,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
   /// Synchronize the current thread with the event.
   Error syncEventImpl(void *EventPtr) override {
-    return Plugin::error(ErrorCode::UNIMPLEMENTED,
-                         "synchronize event not implemented");
+    AMDGPUEventTy *Event = reinterpret_cast<AMDGPUEventTy *>(EventPtr);
+    return Event->sync();
   }
 
   /// Print information about the device.
@@ -2562,7 +2610,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     Status2 = hsa_system_get_info(HSA_SYSTEM_INFO_VERSION_MINOR, &Minor);
     if (Status == HSA_STATUS_SUCCESS && Status2 == HSA_STATUS_SUCCESS)
       Info.add("HSA Runtime Version",
-               std::to_string(Major) + "." + std::to_string(Minor));
+               std::to_string(Major) + "." + std::to_string(Minor), "",
+               DeviceInfo::DRIVER_VERSION);
 
     Info.add("HSA OpenMP Device Number", DeviceId);
 
@@ -2572,11 +2621,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_NAME, TmpChar);
     if (Status == HSA_STATUS_SUCCESS)
-      Info.add("Device Name", TmpChar);
+      Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_VENDOR_NAME, TmpChar);
     if (Status == HSA_STATUS_SUCCESS)
-      Info.add("Vendor Name", TmpChar);
+      Info.add("Vendor Name", TmpChar, "", DeviceInfo::VENDOR);
 
     hsa_device_type_t DevType;
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_DEVICE, DevType);
@@ -2652,7 +2701,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_WORKGROUP_MAX_DIM, WorkgrpMaxDim);
     if (Status == HSA_STATUS_SUCCESS) {
-      auto &MaxSize = *Info.add("Workgroup Max Size per Dimension");
+      auto &MaxSize =
+          *Info.add("Workgroup Max Size per Dimension", std::monostate{}, "",
+                    DeviceInfo::MAX_WORK_GROUP_SIZE);
       MaxSize.add("x", WorkgrpMaxDim[0]);
       MaxSize.add("y", WorkgrpMaxDim[1]);
       MaxSize.add("z", WorkgrpMaxDim[2]);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 162b149ab483e..8c17a2ee07047 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -113,6 +113,12 @@ struct AsyncInfoWrapperTy {
   __tgt_async_info *AsyncInfoPtr;
 };
 
+enum class DeviceInfo {
+#define OFFLOAD_DEVINFO(Name, _, Value) Name = Value,
+#include "OffloadInfo.inc"
+#undef OFFLOAD_DEVINFO
+};
+
 /// Tree node for device information
 ///
 /// This information is either printed or used by liboffload to extract certain
@@ -133,6 +139,8 @@ struct InfoTreeNode {
   // * The same key can appear multiple times
   std::unique_ptr<llvm::SmallVector<InfoTreeNode, 8>> Children;
 
+  llvm::DenseMap<DeviceInfo, size_t> DeviceInfoMap;
+
   InfoTreeNode() : InfoTreeNode("", std::monostate{}, "") {}
   InfoTreeNode(std::string Key, VariantType Value, std::string Units)
       : Key(Key), Value(Value), Units(Units) {}
@@ -140,10 +148,12 @@ struct InfoTreeNode {
   /// Add a new info entry as a child of this node. The entry requires at least
   /// a key string in \p Key. The value in \p Value is optional and can be any
   /// type that is representable as a string. The units in \p Units is optional
-  /// and must be a string.
+  /// and must be a string. Providing a device info key allows liboffload to
+  /// use that value for an appropriate olGetDeviceInfo query
   template <typename T = std::monostate>
   InfoTreeNode *add(std::string Key, T Value = T(),
-                    const std::string &Units = std::string()) {
+                    const std::string &Units = std::string(),
+                    std::optional<DeviceInfo> DeviceInfoKey = std::nullopt) {
     assert(!Key.empty() && "Invalid info key");
 
     if (!Children)
@@ -157,7 +167,12 @@ struct InfoTreeNode {
     else
       ValueVariant = std::string{Value};
 
-    return &Children->emplace_back(Key, ValueVariant, Units);
+    auto Ptr = &Children->emplace_back(Key, ValueVariant, Units);
+
+    if (DeviceInfoKey)
+      DeviceInfoMap[*DeviceInfoKey] = Children->size() - 1;
+
+    return Ptr;
   }
 
   std::optional<InfoTreeNode *> get(StringRef Key) {
@@ -171,6 +186,13 @@ struct InfoTreeNode {
     return It;
   }
 
+  std::optional<InfoTreeNode *> get(DeviceInfo Info) {
+    auto Result = DeviceInfoMap.find(Info);
+    if (Result != DeviceInfoMap.end())
+      return &(*Children)[Result->second];
+    return std::nullopt;
+  }
+
   /// Print all info entries in the tree
   void print() const {
     // Fake an additional indent so that values are offset from the keys
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index b787376eb1770..728bf07c572bb 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -935,15 +935,16 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Res == CUDA_SUCCESS)
       // For consistency with other drivers, store the version as a string
       // rather than an integer
-      Info.add("CUDA Driver Version", std::to_string(TmpInt));
+      Info.add("CUDA Driver Version", std::to_string(TmpInt), "",
+               DeviceInfo::DRIVER_VERSION);
 
     Info.add("CUDA OpenMP Device Number", DeviceId);
 
     Res = cuDeviceGetName(TmpChar, 1000, Device);
     if (Res == CUDA_SUCCESS)
-      Info.add("Device Name", TmpChar);
+      Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
 
-    Info.add("Vendor Name", "NVIDIA");
+    Info.add("Vendor Name", "NVIDIA", "", DeviceInfo::VENDOR);
 
     Res = cuDeviceTotalMem(&TmpSt, Device);
     if (Res == CUDA_SUCCESS)
@@ -978,7 +979,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Res == CUDA_SUCCESS)
       Info.add("Maximum Threads per Block", TmpInt);
 
-    auto &MaxBlock = *Info.add("Maximum Block Dimensions", "");
+    auto &MaxBlock = *Info.add("Maximum Block Dimensions", std::monostate{}, "",
+                               DeviceInfo::MAX_WORK_GROUP_SIZE);
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, TmpInt);
     if (Res == CUDA_SUCCESS)
       MaxBlock.add("x", TmpInt);
diff --git a/offload/unittests/OffloadAPI/common/Fixtures.hpp b/offload/unittests/OffloadAPI/common/Fixtures.hpp
index e5d815ecda965..546921164f691 100644
--- a/offload/unittests/OffloadAPI/common/Fixtures.hpp
+++ b/offload/unittests/OffloadAPI/common/Fixtures.hpp
@@ -171,9 +171,6 @@ struct OffloadQueueTest : OffloadDeviceTest {
 struct OffloadEventTest : OffloadQueueTest {
   void SetUp() override {
     RETURN_ON_FATAL_FAILURE(OffloadQueueTest::SetUp());
-    if (getPlatformBackend() == OL_PLATFORM_BACKEND_AMDGPU)
-      GTEST_SKIP() << "AMDGPU synchronize event not implemented";
-
     // Get an event from a memcpy. We can still use it in olGetEventInfo etc
     // after it has been waited on.
     void *Alloc;
diff --git a/offload/unittests/OffloadAPI/event/olWaitEvent.cpp b/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
index 05356d4ef8d75..f80dabb4fc93f 100644
--- a/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
+++ b/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
@@ -14,9 +14,6 @@ using olWaitEventTest = OffloadQueueTest;
 OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olWaitEventTest);
 
 TEST_P(olWaitEventTest, Success) {
-  if (getPlatformBackend() == OL_PLATFORM_BACKEND_AMDGPU)
-    GTEST_SKIP() << "AMDGPU synchronize event not implemented";
-
   uint32_t Src = 42;
   void *DstPtr;
 

>From bfe68539b78d8610bd0cd315c9a3e8fe9b456a3f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 16:45:02 +0100
Subject: [PATCH 6/6] !fixup address latest comments, thanks

---
 llvm/lib/Analysis/IVDescriptors.cpp           |   6 +-
 .../Vectorize/VPlanConstruction.cpp           |  49 ++++----
 .../Transforms/Vectorize/VPlanTransforms.h    |   8 +-
 .../fmax-without-fast-math-flags.ll           | 116 ++++++++++++++++++
 4 files changed, 150 insertions(+), 29 deletions(-)

diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index a88bbd7b65d53..8be5de3bf356f 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -947,10 +947,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
       InstDesc Res = isMinMaxPattern(I, Kind, Prev);
       if (!Res.isRecurrence())
         return InstDesc(false, I);
-
       if (HasRequiredFMF())
         return Res;
-
       // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
       // intrinsics with extra checks ensuring the vector loop handles only
       // non-NaN inputs.
@@ -964,9 +962,9 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                "unexpected recurrence kind for minnum");
         return InstDesc(I, RecurKind::FMinNum);
       }
-
       return InstDesc(false, I);
-    } else if (isFMulAddIntrinsic(I))
+    }
+    if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,
                       I->hasAllowReassoc() ? nullptr : I);
     return InstDesc(false, I);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a426d06b4befe..98c2fe2d6129d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -654,10 +654,6 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
 }
 
 bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
-  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-  VPReductionPHIRecipe *RedPhiR = nullptr;
-  bool HasUnsupportedPhi = false;
-
   auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
     auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
         RedPhiR->getBackedgeValue()->getDefiningRecipe());
@@ -666,7 +662,7 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
 
     auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
     if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
-        !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+        !(RepR && isa<IntrinsicInst>(RepR->getUnderlyingInstr())))
       return nullptr;
 
 #ifndef NDEBUG
@@ -690,41 +686,47 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     return MinMaxR->getOperand(0);
   };
 
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  bool HasUnsupportedPhi = false;
   for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
-    // TODO: Also support fixed-order recurrence phis.
-    HasUnsupportedPhi |=
-        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
-             VPReductionPHIRecipe>(&R);
+    if (isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe>(&R))
+      continue;
     auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
-    if (!Cur)
+    if (!Cur) {
+      // TODO: Also support fixed-order recurrence phis.
+      HasUnsupportedPhi = true;
       continue;
+    }
     // For now, only a single reduction is supported.
     // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
     if (RedPhiR)
       return false;
     if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
-        Cur->getRecurrenceKind() != RecurKind::FMinNum)
+        Cur->getRecurrenceKind() != RecurKind::FMinNum) {
+      HasUnsupportedPhi = true;
       continue;
+    }
     RedPhiR = Cur;
   }
 
   if (!RedPhiR)
     return true;
 
-  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
-  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
-         "unsupported reduction");
-
-  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
-  if (!MinMaxOp)
-    return false;
-
   // We won't be able to resume execution in the scalar tail, if there are
   // unsupported header phis or there is no scalar tail at all, due to
   // tail-folding.
   if (HasUnsupportedPhi || !Plan.hasScalarTail())
     return false;
 
+  VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+  if (!MinMaxOp)
+    return false;
+
+  RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+  assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+         "unsupported reduction");
+
   /// Check if the vector loop of \p Plan can early exit and restart
   /// execution of last vector iteration in the scalar loop. This requires all
   /// recipes up to early exit point be side-effect free as they are
@@ -774,7 +776,7 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
   RdxResult->setOperand(1, NewSel);
 
   auto *ScalarPH = Plan.getScalarPreheader();
-  // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+  // Update resume phis for inductions in the scalar preheader. If AnyNaN is
   // true, the resume from the start of the last vector iteration via the
   // canonical IV, otherwise from the original value.
   for (auto &R : ScalarPH->phis()) {
@@ -782,8 +784,13 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
     VPValue *VecV = ResumeR->getOperand(0);
     if (VecV == RdxResult)
       continue;
-    if (VecV != &Plan.getVectorTripCount())
+    // Bail out and abandon the current, partially modified, VPlan if we
+    // encounter resume phi that cannot be updated yet.
+    if (VecV != &Plan.getVectorTripCount()) {
+      LLVM_DEBUG(dbgs() << "Found resume phi we cannot update for VPlan with "
+                           "FMaxNum/FMinNum reduction.\n");
       return false;
+    }
     auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
     ResumeR->setOperand(0, NewSel);
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index b1692d3e6b337..04cb7a7a5c19b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -103,10 +103,10 @@ struct VPlanTransforms {
   /// not valid.
   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
 
-  /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
-  /// If they do, try to update the vector loop to exit early if any input is
-  /// NaN and resume executing in the scalar loop to handle the NaNs there.
-  /// Return false if this attempt was unsuccessful.
+  /// Check if \p Plan contains any FMaxNum or FMinNum reductions. If they do,
+  /// try to update the vector loop to exit early if any input is NaN and resume
+  /// executing in the scalar loop to handle the NaNs there. Return false if
+  /// this attempt was unsuccessful.
   static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
 
   /// Clear NSW/NUW flags from reduction instructions if necessary.
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 6204d540df40c..49c25b02ce964 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -324,6 +324,122 @@ exit:
   ret float %max.next
 }
 
+define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_induction_starts_at_10(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 10, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 10, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret float %max.next
+}
+
+define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_induction_starts_at_value(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret float %max.next
+}
+
+define float @fmaxnum_with_additional_add(ptr noalias %src, ptr noalias %src.2, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_with_additional_add(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[SRC_2:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC_2]], i64 [[IV]]
+; CHECK-NEXT:    [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4
+; CHECK-NEXT:    [[SUM_NEXT]] = add i32 [[SUM]], [[L_SRC_2]]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    store i32 [[SUM_NEXT_LCSSA]], ptr [[SRC_2]], align 4
+; CHECK-NEXT:    ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %sum.next, %loop ]
+  %gep.src.2 = getelementptr inbounds nuw i32, ptr %src.2, i64 %iv
+  %l.src.2 = load i32, ptr %gep.src.2, align 4
+  %sum.next = add i32 %sum, %l.src.2
+  %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+  %l = load float, ptr %gep.src, align 4
+  %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  store i32 %sum.next, ptr %src.2
+  ret float %max.next
+}
+
 define float @fmax_select_with_blend(ptr %A, ptr %B) {
 ; CHECK-LABEL: define float @fmax_select_with_blend(
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {


From flang-commits at lists.llvm.org  Fri Jul 18 12:52:24 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Fri, 18 Jul 2025 12:52:24 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687aa5f8.050a0220.179512.d28d@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 1/9] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 2/9] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 3/9] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 4/9] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 5/9] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 6/9] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 

>From 498d60cf39c237c654c0cfddbe444d63bc87bc6c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 09:57:18 +0200
Subject: [PATCH 7/9] CI test

---
 flang-rt/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 5ffa1bcc34a41..d987115c8e9e8 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,40 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
 enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
@@ -159,6 +193,7 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)

>From 1bb0144eb21c2b1c8e1e66029d5573d8f5619c4c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 14:24:44 +0200
Subject: [PATCH 8/9] Enforce REAL(16) support

---
 flang-rt/CMakeLists.txt      | 45 ++++-------------------
 llvm/runtimes/CMakeLists.txt |  8 ++++
 runtimes/CMakeLists.txt      | 71 ++++++++++++++++++------------------
 3 files changed, 51 insertions(+), 73 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d987115c8e9e8..bfbd0af0c31dc 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,41 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
@@ -79,6 +44,11 @@ include(CMakePushCheckState)
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
+# Fortran compiler not optional for building Flang-RT
+enable_language(Fortran)
+
+flang_module_fortran_enable()
+
 
 #################
 # Build Options #
@@ -193,7 +163,9 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+# Look for support of REAL(16), if not already defined via command line.
+# NOTE: Does not work with Flang and CMake < 3.24
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
@@ -206,7 +178,6 @@ check_fortran_source_compiles([[
 )
 cmake_pop_check_state()
 
-flang_module_fortran_enable()
 
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 2f2f6db16255e..2bf2b0ee8ed50 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -528,6 +528,14 @@ if(build_runtimes)
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
+  if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    # Ensure REAL(16) support in runtimes to be consistent with compiler
+    if (FLANG_RUNTIME_F128_MATH_LIB OR HAVE_LDBL_MANT_DIG_113)
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=TRUE")
+   else ()
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=FALSE")
+   endif ()
+  endif ()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 987d173a283be..51214a46f558e 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -94,6 +94,41 @@ include(CheckCXXCompilerFlag)
 include(ExtendPath)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang, if used.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
+
 # Determine whether we are in the runtimes/runtimes-bins directory of a
 # bootstrapping build.
 set(LLVM_TREE_AVAILABLE OFF)
@@ -334,42 +369,6 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
-      _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-
-    enable_language(Fortran)
-  endif ()
-endif ()
-
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 8ec6d03939c3a7e2f79bb3ea5911b7607028dd7c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 21:52:02 +0200
Subject: [PATCH 9/9] Also add dependency barrier for WIN32

---
 flang-rt/lib/runtime/CMakeLists.txt | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 1b8114c102205..20f5d84bb2b69 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -281,14 +281,18 @@ else()
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
 
-    add_flangrt_library(${name}.intrinsics OBJECT
+    add_flangrt_library(${name}.intrinsics.obj OBJECT
       ${intrinsics_sources}
     )
+    add_custom_target(${name}.intrinsics
+      COMMENT "Intrinsic module dependency barrier"
+    )
+    add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
 
     add_flangrt_library(${name} ${libtype}
-        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics.obj>
         ${ARGN}
-        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -311,8 +315,9 @@ else()
     endif ()
 
     get_target_property(compile_target ${name}.compile ALIASED_TARGET)
-    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
+    add_dependencies(${compile_target} ${name}.intrinsics)
     add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")


From flang-commits at lists.llvm.org  Fri Jul 18 13:15:33 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Fri, 18 Jul 2025 13:15:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][openacc] Add semantic check for do
 contruct containing cache directive (PR #148077)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148077@github.com>
Message-ID: <687aab65.050a0220.2c3f5b.d4b1@mx.google.com>


https://github.com/akuhlens closed https://github.com/llvm/llvm-project/pull/148077

From flang-commits at lists.llvm.org  Fri Jul 18 13:38:42 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Fri, 18 Jul 2025 13:38:42 -0700 (PDT)
Subject: [flang-commits] [flang] [NFC][flang] Added engineering option for
 triaging local-alloc-tbaa. (PR #149587)
Message-ID: <llvm.org/llvm/llvm-project/pull/149587@github.com>


https://github.com/vzakhari created https://github.com/llvm/llvm-project/pull/149587

I triaged a benchmark that showed inaccurate results, when local-alloc-tbaa
was enabled. It turned out to be not a real TBAA issue, but rather
TBAA affecting optimizations that affect FMA generation, which introduced
an expected accuracy variation. I would like to keep this threshold
control for future uses.


>From 7c79e23426968538f9932a71fb2d5b19af76e697 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin at nvidia.com>
Date: Fri, 18 Jul 2025 10:19:19 -0700
Subject: [PATCH] [NFC][flang] Added engineering option for triaging
 local-alloc-tbaa.

I triaged a benchmark that showed inaccurate results, when local-alloc-tbaa
was enabled. It turned out to be not a real TBAA issue, but rather
TBAA affecting optimizations that affect FMA generation, which introduced
an expected accuracy variation. I would like to keep this threshold
control for future uses.
---
 .../lib/Optimizer/Transforms/AddAliasTags.cpp | 41 ++++++++++++++++---
 .../Transforms/tbaa-local-alloc-threshold.fir | 23 +++++++++++
 2 files changed, 59 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Transforms/tbaa-local-alloc-threshold.fir

diff --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index b27c1b26dedb3..85403ad257657 100644
--- a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -48,12 +48,21 @@ static llvm::cl::opt<bool>
                       llvm::cl::Hidden,
                       llvm::cl::desc("Add TBAA tags to local allocations."));
 
+// Engineering option to triage TBAA tags attachment for accesses
+// of allocatable entities.
+static llvm::cl::opt<unsigned> localAllocsThreshold(
+    "local-alloc-tbaa-threshold", llvm::cl::init(0), llvm::cl::ReallyHidden,
+    llvm::cl::desc("If present, stops generating TBAA tags for accesses of "
+                   "local allocations after N accesses in a module"));
+
 namespace {
 
 /// Shared state per-module
 class PassState {
 public:
-  PassState(mlir::DominanceInfo &domInfo) : domInfo(domInfo) {}
+  PassState(mlir::DominanceInfo &domInfo,
+            std::optional<unsigned> localAllocsThreshold)
+      : domInfo(domInfo), localAllocsThreshold(localAllocsThreshold) {}
   /// memoised call to fir::AliasAnalysis::getSource
   inline const fir::AliasAnalysis::Source &getSource(mlir::Value value) {
     if (!analysisCache.contains(value))
@@ -84,6 +93,11 @@ class PassState {
   // (e.g. !fir.ref<!fir.type<Derived{f:!fir.box<!fir.heap<f32>>}>>).
   bool typeReferencesDescriptor(mlir::Type type);
 
+  // Returns true if we can attach a TBAA tag to an access of an allocatable
+  // entities. It checks if localAllocsThreshold allows the next tag
+  // attachment.
+  bool attachLocalAllocTag();
+
 private:
   mlir::DominanceInfo &domInfo;
   fir::AliasAnalysis analysis;
@@ -103,6 +117,8 @@ class PassState {
   // Local pass cache for derived types that contain descriptor
   // member(s), to avoid the cost of isRecordWithDescriptorMember().
   llvm::DenseSet<mlir::Type> typesContainingDescriptors;
+
+  std::optional<unsigned> localAllocsThreshold;
 };
 
 // Process fir.dummy_scope operations in the given func:
@@ -169,6 +185,19 @@ bool PassState::typeReferencesDescriptor(mlir::Type type) {
   return false;
 }
 
+bool PassState::attachLocalAllocTag() {
+  if (!localAllocsThreshold)
+    return true;
+  if (*localAllocsThreshold == 0) {
+    LLVM_DEBUG(llvm::dbgs().indent(2)
+               << "WARN: not assigning TBAA tag for an allocated entity access "
+                  "due to the threshold\n");
+    return false;
+  }
+  --*localAllocsThreshold;
+  return true;
+}
+
 class AddAliasTagsPass : public fir::impl::AddAliasTagsBase<AddAliasTagsPass> {
 public:
   void runOnOperation() override;
@@ -335,16 +364,16 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
       LLVM_DEBUG(llvm::dbgs().indent(2)
                  << "WARN: unknown defining op for SourceKind::Allocate " << *op
                  << "\n");
-    } else if (source.isPointer()) {
+    } else if (source.isPointer() && state.attachLocalAllocTag()) {
       LLVM_DEBUG(llvm::dbgs().indent(2)
                  << "Found reference to allocation at " << *op << "\n");
       tag = state.getFuncTreeWithScope(func, scopeOp).targetDataTree.getTag();
-    } else if (name) {
+    } else if (name && state.attachLocalAllocTag()) {
       LLVM_DEBUG(llvm::dbgs().indent(2) << "Found reference to allocation "
                                         << name << " at " << *op << "\n");
       tag = state.getFuncTreeWithScope(func, scopeOp)
                 .allocatedDataTree.getTag(*name);
-    } else {
+    } else if (state.attachLocalAllocTag()) {
       LLVM_DEBUG(llvm::dbgs().indent(2)
                  << "WARN: couldn't find a name for allocation " << *op
                  << "\n");
@@ -372,7 +401,9 @@ void AddAliasTagsPass::runOnOperation() {
   // thinks the pass operates on), then the real work of the pass is done in
   // runOnAliasInterface
   auto &domInfo = getAnalysis<mlir::DominanceInfo>();
-  PassState state(domInfo);
+  PassState state(domInfo, localAllocsThreshold.getPosition()
+                               ? std::optional<unsigned>(localAllocsThreshold)
+                               : std::nullopt);
 
   mlir::ModuleOp mod = getOperation();
   mod.walk(
diff --git a/flang/test/Transforms/tbaa-local-alloc-threshold.fir b/flang/test/Transforms/tbaa-local-alloc-threshold.fir
new file mode 100644
index 0000000000000..27c19a6e23095
--- /dev/null
+++ b/flang/test/Transforms/tbaa-local-alloc-threshold.fir
@@ -0,0 +1,23 @@
+// Check that -local-alloc-tbaa-threshold option limits
+// the attachment of TBAA tags to accesses of locally allocated entities.
+// RUN: fir-opt --fir-add-alias-tags -local-alloc-tbaa-threshold=2 %s | FileCheck %s --check-prefixes=ALL,COUNT2
+// RUN: fir-opt --fir-add-alias-tags -local-alloc-tbaa-threshold=1 %s | FileCheck %s --check-prefixes=ALL,COUNT1
+// RUN: fir-opt --fir-add-alias-tags -local-alloc-tbaa-threshold=0 %s | FileCheck %s --check-prefixes=ALL,COUNT0
+
+// ALL-LABEL:   func.func @_QPtest() {
+// COUNT2: fir.load{{.*}}{tbaa =
+// COUNT2: fir.store{{.*}}{tbaa =
+// COUNT1: fir.load{{.*}}{tbaa =
+// COUNT1-NOT: fir.store{{.*}}{tbaa =
+// COUNT0-NOT: fir.load{{.*}}{tbaa =
+// COUNT0-NOT: fir.store{{.*}}{tbaa =
+func.func @_QPtest() {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %2 = fir.declare %1 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  %3 = fir.alloca f32 {bindc_name = "y", uniq_name = "_QFtestEy"}
+  %4 = fir.declare %3 {uniq_name = "_QFtestEy"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  %5 = fir.load %4 : !fir.ref<f32>
+  fir.store %5 to %2 : !fir.ref<f32>
+  return
+}


From flang-commits at lists.llvm.org  Fri Jul 18 13:39:18 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 13:39:18 -0700 (PDT)
Subject: [flang-commits] [flang] [NFC][flang] Added engineering option for
 triaging local-alloc-tbaa. (PR #149587)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149587@github.com>
Message-ID: <687ab0f6.170a0220.343223.d5a0@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Slava Zakharin (vzakhari)

<details>
<summary>Changes</summary>

I triaged a benchmark that showed inaccurate results, when local-alloc-tbaa
was enabled. It turned out to be not a real TBAA issue, but rather
TBAA affecting optimizations that affect FMA generation, which introduced
an expected accuracy variation. I would like to keep this threshold
control for future uses.


---
Full diff: https://github.com/llvm/llvm-project/pull/149587.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/AddAliasTags.cpp (+36-5) 
- (added) flang/test/Transforms/tbaa-local-alloc-threshold.fir (+23) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index b27c1b26dedb3..85403ad257657 100644
--- a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -48,12 +48,21 @@ static llvm::cl::opt<bool>
                       llvm::cl::Hidden,
                       llvm::cl::desc("Add TBAA tags to local allocations."));
 
+// Engineering option to triage TBAA tags attachment for accesses
+// of allocatable entities.
+static llvm::cl::opt<unsigned> localAllocsThreshold(
+    "local-alloc-tbaa-threshold", llvm::cl::init(0), llvm::cl::ReallyHidden,
+    llvm::cl::desc("If present, stops generating TBAA tags for accesses of "
+                   "local allocations after N accesses in a module"));
+
 namespace {
 
 /// Shared state per-module
 class PassState {
 public:
-  PassState(mlir::DominanceInfo &domInfo) : domInfo(domInfo) {}
+  PassState(mlir::DominanceInfo &domInfo,
+            std::optional<unsigned> localAllocsThreshold)
+      : domInfo(domInfo), localAllocsThreshold(localAllocsThreshold) {}
   /// memoised call to fir::AliasAnalysis::getSource
   inline const fir::AliasAnalysis::Source &getSource(mlir::Value value) {
     if (!analysisCache.contains(value))
@@ -84,6 +93,11 @@ class PassState {
   // (e.g. !fir.ref<!fir.type<Derived{f:!fir.box<!fir.heap<f32>>}>>).
   bool typeReferencesDescriptor(mlir::Type type);
 
+  // Returns true if we can attach a TBAA tag to an access of an allocatable
+  // entities. It checks if localAllocsThreshold allows the next tag
+  // attachment.
+  bool attachLocalAllocTag();
+
 private:
   mlir::DominanceInfo &domInfo;
   fir::AliasAnalysis analysis;
@@ -103,6 +117,8 @@ class PassState {
   // Local pass cache for derived types that contain descriptor
   // member(s), to avoid the cost of isRecordWithDescriptorMember().
   llvm::DenseSet<mlir::Type> typesContainingDescriptors;
+
+  std::optional<unsigned> localAllocsThreshold;
 };
 
 // Process fir.dummy_scope operations in the given func:
@@ -169,6 +185,19 @@ bool PassState::typeReferencesDescriptor(mlir::Type type) {
   return false;
 }
 
+bool PassState::attachLocalAllocTag() {
+  if (!localAllocsThreshold)
+    return true;
+  if (*localAllocsThreshold == 0) {
+    LLVM_DEBUG(llvm::dbgs().indent(2)
+               << "WARN: not assigning TBAA tag for an allocated entity access "
+                  "due to the threshold\n");
+    return false;
+  }
+  --*localAllocsThreshold;
+  return true;
+}
+
 class AddAliasTagsPass : public fir::impl::AddAliasTagsBase<AddAliasTagsPass> {
 public:
   void runOnOperation() override;
@@ -335,16 +364,16 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
       LLVM_DEBUG(llvm::dbgs().indent(2)
                  << "WARN: unknown defining op for SourceKind::Allocate " << *op
                  << "\n");
-    } else if (source.isPointer()) {
+    } else if (source.isPointer() && state.attachLocalAllocTag()) {
       LLVM_DEBUG(llvm::dbgs().indent(2)
                  << "Found reference to allocation at " << *op << "\n");
       tag = state.getFuncTreeWithScope(func, scopeOp).targetDataTree.getTag();
-    } else if (name) {
+    } else if (name && state.attachLocalAllocTag()) {
       LLVM_DEBUG(llvm::dbgs().indent(2) << "Found reference to allocation "
                                         << name << " at " << *op << "\n");
       tag = state.getFuncTreeWithScope(func, scopeOp)
                 .allocatedDataTree.getTag(*name);
-    } else {
+    } else if (state.attachLocalAllocTag()) {
       LLVM_DEBUG(llvm::dbgs().indent(2)
                  << "WARN: couldn't find a name for allocation " << *op
                  << "\n");
@@ -372,7 +401,9 @@ void AddAliasTagsPass::runOnOperation() {
   // thinks the pass operates on), then the real work of the pass is done in
   // runOnAliasInterface
   auto &domInfo = getAnalysis<mlir::DominanceInfo>();
-  PassState state(domInfo);
+  PassState state(domInfo, localAllocsThreshold.getPosition()
+                               ? std::optional<unsigned>(localAllocsThreshold)
+                               : std::nullopt);
 
   mlir::ModuleOp mod = getOperation();
   mod.walk(
diff --git a/flang/test/Transforms/tbaa-local-alloc-threshold.fir b/flang/test/Transforms/tbaa-local-alloc-threshold.fir
new file mode 100644
index 0000000000000..27c19a6e23095
--- /dev/null
+++ b/flang/test/Transforms/tbaa-local-alloc-threshold.fir
@@ -0,0 +1,23 @@
+// Check that -local-alloc-tbaa-threshold option limits
+// the attachment of TBAA tags to accesses of locally allocated entities.
+// RUN: fir-opt --fir-add-alias-tags -local-alloc-tbaa-threshold=2 %s | FileCheck %s --check-prefixes=ALL,COUNT2
+// RUN: fir-opt --fir-add-alias-tags -local-alloc-tbaa-threshold=1 %s | FileCheck %s --check-prefixes=ALL,COUNT1
+// RUN: fir-opt --fir-add-alias-tags -local-alloc-tbaa-threshold=0 %s | FileCheck %s --check-prefixes=ALL,COUNT0
+
+// ALL-LABEL:   func.func @_QPtest() {
+// COUNT2: fir.load{{.*}}{tbaa =
+// COUNT2: fir.store{{.*}}{tbaa =
+// COUNT1: fir.load{{.*}}{tbaa =
+// COUNT1-NOT: fir.store{{.*}}{tbaa =
+// COUNT0-NOT: fir.load{{.*}}{tbaa =
+// COUNT0-NOT: fir.store{{.*}}{tbaa =
+func.func @_QPtest() {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %2 = fir.declare %1 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  %3 = fir.alloca f32 {bindc_name = "y", uniq_name = "_QFtestEy"}
+  %4 = fir.declare %3 {uniq_name = "_QFtestEy"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  %5 = fir.load %4 : !fir.ref<f32>
+  fir.store %5 to %2 : !fir.ref<f32>
+  return
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/149587

From flang-commits at lists.llvm.org  Fri Jul 18 13:45:12 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 13:45:12 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][runtime] Preserve type when
 remapping monomorphic pointers (PR #149427)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149427@github.com>
Message-ID: <687ab258.170a0220.3d0691.5ef5@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/149427

From flang-commits at lists.llvm.org  Fri Jul 18 13:45:29 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 13:45:29 -0700 (PDT)
Subject: [flang-commits] [flang] b6ea04a - [flang][NFC] Fix build-time
	warning (#149549)
Message-ID: <687ab269.050a0220.34944b.d71f@mx.google.com>


Author: Peter Klausler
Date: 2025-07-18T13:45:25-07:00
New Revision: b6ea04a37b2a41e24bb999e5a9b6a7bd2b576085

URL: https://github.com/llvm/llvm-project/commit/b6ea04a37b2a41e24bb999e5a9b6a7bd2b576085
DIFF: https://github.com/llvm/llvm-project/commit/b6ea04a37b2a41e24bb999e5a9b6a7bd2b576085.diff

LOG: [flang][NFC] Fix build-time warning (#149549)

Don't increment the LHS variable of an assignment that also uses that
variable on the RHS.

Added: 
    

Modified: 
    flang/lib/Semantics/resolve-labels.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index 27e259fab3873..9454ef9fe928a 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -492,10 +492,9 @@ class ParseTreeAnalyzer {
     // Uppercase the name of the main program, so that its symbol name
     // would be unique from similarly named non-main-program symbols.
     auto upperCaseCharBlock = [](const parser::CharBlock &cb) {
-      char *ch{const_cast<char *>(cb.begin())};
-      char *endCh{ch + cb.size()};
-      while (ch != endCh) {
-        *ch++ = parser::ToUpperCaseLetter(*ch);
+      auto ch{const_cast<char *>(cb.begin())};
+      for (char *endCh{ch + cb.size()}; ch != endCh; ++ch) {
+        *ch = parser::ToUpperCaseLetter(*ch);
       }
     };
     const parser::CharBlock *progName{nullptr};


        

From flang-commits at lists.llvm.org  Fri Jul 18 13:45:32 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 13:45:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][NFC] Fix build-time warning (PR
	#149549)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149549@github.com>
Message-ID: <687ab26c.170a0220.4ced.d973@mx.google.com>


https://github.com/klausler closed https://github.com/llvm/llvm-project/pull/149549

From flang-commits at lists.llvm.org  Fri Jul 18 13:50:13 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 13:50:13 -0700 (PDT)
Subject: [flang-commits] [flang] abdd453 - [flang][openacc] fix bugs with
 default(none) checking (#149220)
Message-ID: <687ab385.630a0220.1f06c.cf15@mx.google.com>


Author: Andre Kuhlenschmidt
Date: 2025-07-18T13:50:09-07:00
New Revision: abdd4536ce0fc75c7a4ddcc1da5913ec5e028091

URL: https://github.com/llvm/llvm-project/commit/abdd4536ce0fc75c7a4ddcc1da5913ec5e028091
DIFF: https://github.com/llvm/llvm-project/commit/abdd4536ce0fc75c7a4ddcc1da5913ec5e028091.diff

LOG: [flang][openacc] fix bugs with default(none) checking (#149220)

A report of the following code not generating an error led to fixing two bugs in directive checking.

- We should treat CombinedConstructs as OpenACC Constructs
- We should treat DoConstruct index variables as private. 

```fortran
subroutine sub(nn)
  integer :: nn, ii
  !$acc serial loop default(none)
  do ii = 1, nn
  end do
  !$acc end serial loop
end subroutine
```
Here `nn` should be flagged as needing a data clause while `ii` should
still get one implicitly.

Added: 
    

Modified: 
    flang/lib/Semantics/resolve-directives.cpp
    flang/test/Lower/OpenACC/acc-loop.f90
    flang/test/Semantics/OpenACC/acc-kernels-loop.f90

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 151f4ccae634e..521c7432d9fbb 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -138,6 +138,9 @@ class AccAttributeVisitor : DirectiveAttributeVisitor<llvm::acc::Directive> {
   void Post(const parser::OpenACCBlockConstruct &) { PopContext(); }
   bool Pre(const parser::OpenACCCombinedConstruct &);
   void Post(const parser::OpenACCCombinedConstruct &) { PopContext(); }
+  void Post(const parser::AccBeginCombinedDirective &) {
+    GetContext().withinConstruct = true;
+  }
 
   bool Pre(const parser::OpenACCDeclarativeConstruct &);
   void Post(const parser::OpenACCDeclarativeConstruct &) { PopContext(); }
@@ -160,6 +163,18 @@ class AccAttributeVisitor : DirectiveAttributeVisitor<llvm::acc::Directive> {
     GetContext().withinConstruct = true;
   }
 
+  // TODO: We should probably also privatize ConcurrentBounds.
+  template <typename A>
+  bool Pre(const parser::LoopBounds<parser::ScalarName, A> &x) {
+    if (!dirContext_.empty() && GetContext().withinConstruct) {
+      if (auto *symbol{ResolveAcc(
+              x.name.thing, Symbol::Flag::AccPrivate, currScope())}) {
+        AddToContextObjectWithDSA(*symbol, Symbol::Flag::AccPrivate);
+      }
+    }
+    return true;
+  }
+
   bool Pre(const parser::OpenACCStandaloneConstruct &);
   void Post(const parser::OpenACCStandaloneConstruct &) { PopContext(); }
   void Post(const parser::AccStandaloneDirective &) {

diff  --git a/flang/test/Lower/OpenACC/acc-loop.f90 b/flang/test/Lower/OpenACC/acc-loop.f90
index c6df28ec5e000..f9f5e8c2165d5 100644
--- a/flang/test/Lower/OpenACC/acc-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-loop.f90
@@ -372,12 +372,15 @@ subroutine sub1(i, j, k)
 end subroutine
 
 ! CHECK: func.func @_QPsub1
-! CHECK: acc.parallel
-! CHECK: %[[DC_K:.*]] = fir.alloca i32 {bindc_name = "k"}
-! CHECK: %[[DC_J:.*]] = fir.alloca i32 {bindc_name = "j"}
-! CHECK: %[[DC_I:.*]] = fir.alloca i32 {bindc_name = "i"}
-! CHECK: %[[P_I:.*]] = acc.private varPtr(%[[DC_I]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
-! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "j"}
-! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "k"}
+! CHECK-SAME: %[[ARG_I:.*]]: !fir.ref<i32> {fir.bindc_name = "i"}
+! CHECK-SAME: %[[ARG_J:.*]]: !fir.ref<i32> {fir.bindc_name = "j"}
+! CHECK-SAME: %[[ARG_K:.*]]: !fir.ref<i32> {fir.bindc_name = "k"}
+! CHECK: %[[DC_I:.*]]:2 = hlfir.declare %[[ARG_I]] dummy_scope %0
+! CHECK: %[[DC_J:.*]]:2 = hlfir.declare %[[ARG_J]] dummy_scope %0 
+! CHECK: %[[DC_K:.*]]:2 = hlfir.declare %[[ARG_K]] dummy_scope %0 
+! CHECK: acc.parallel combined(loop)
+! CHECK: %[[P_I:.*]] = acc.private varPtr(%[[DC_I]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "j"}
+! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "k"}
 ! CHECK: acc.loop combined(parallel) private(@privatization_ref_i32 -> %[[P_I]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_J]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_K]] : !fir.ref<i32>) control(%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%c10{{.*}}, %c100{{.*}}, %c200{{.*}} : i32, i32, i32)  step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32)
 ! CHECK: } attributes {inclusiveUpperbound = array<i1: true, true, true>, independent = [#acc.device_type<none>]}

diff  --git a/flang/test/Semantics/OpenACC/acc-kernels-loop.f90 b/flang/test/Semantics/OpenACC/acc-kernels-loop.f90
index 29985a02eb6ef..cfe27e4f8fca1 100644
--- a/flang/test/Semantics/OpenACC/acc-kernels-loop.f90
+++ b/flang/test/Semantics/OpenACC/acc-kernels-loop.f90
@@ -243,8 +243,15 @@ program openacc_kernels_loop_validity
     a(i) = 3.14
   end do
 
+  !$acc kernels loop default(none) private(N, a)
+  do i = 1, N
+    a(i) = 3.14
+  end do
+
   !$acc kernels loop default(none)
+  !ERROR: The DEFAULT(NONE) clause requires that 'n' must be listed in a data-mapping clause
   do i = 1, N
+    !ERROR: The DEFAULT(NONE) clause requires that 'a' must be listed in a data-mapping clause
     a(i) = 3.14
   end do
 


        

From flang-commits at lists.llvm.org  Fri Jul 18 13:50:16 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Fri, 18 Jul 2025 13:50:16 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][openacc] fix bugs with
	default(none) checking (PR #149220)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149220@github.com>
Message-ID: <687ab388.630a0220.248baf.da54@mx.google.com>


https://github.com/akuhlens closed https://github.com/llvm/llvm-project/pull/149220

From flang-commits at lists.llvm.org  Fri Jul 18 13:58:29 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Fri, 18 Jul 2025 13:58:29 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687ab575.170a0220.3ad293.d5b4@mx.google.com>


https://github.com/fhahn closed https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Fri Jul 18 14:08:37 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 14:08:37 -0700 (PDT)
Subject: [flang-commits] [flang] early feedback commit (PR #149579)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149579@github.com>
Message-ID: <687ab7d5.050a0220.373199.cb52@mx.google.com>


================
@@ -342,21 +347,149 @@ void AccStructureChecker::Leave(const parser::OpenACCAtomicConstruct &x) {
   dirContext_.pop_back();
 }
 
-void AccStructureChecker::Enter(const parser::AccAtomicUpdate &x) {
-  const parser::AssignmentStmt &assignment{
-      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement};
-  const auto &var{std::get<parser::Variable>(assignment.t)};
-  const auto &expr{std::get<parser::Expr>(assignment.t)};
+void AccStructureChecker::CheckAtomicStmt(
+    const parser::AssignmentStmt &assign, std::string &construct) {
+  const auto &var{std::get<parser::Variable>(assign.t)};
+  const auto &expr{std::get<parser::Expr>(assign.t)};
   const auto *rhs{GetExpr(context_, expr)};
   const auto *lhs{GetExpr(context_, var)};
-  if (lhs && rhs) {
-    if (lhs->Rank() != 0)
+
+  if (lhs) {
+    if (lhs->Rank() != 0) {
       context_.Say(expr.source,
-          "LHS of atomic update statement must be scalar"_err_en_US);
-    if (rhs->Rank() != 0)
+          "LHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+  if (rhs) {
+    if (rhs->Rank() != 0) {
       context_.Say(var.GetSource(),
-          "RHS of atomic update statement must be scalar"_err_en_US);
+          "RHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
----------------
klausler wrote:

`rhs`

Do they have to be the same type?

https://github.com/llvm/llvm-project/pull/149579

From flang-commits at lists.llvm.org  Fri Jul 18 14:08:37 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 14:08:37 -0700 (PDT)
Subject: [flang-commits] [flang] early feedback commit (PR #149579)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149579@github.com>
Message-ID: <687ab7d5.050a0220.3bdc25.e1ad@mx.google.com>


================
@@ -342,21 +347,149 @@ void AccStructureChecker::Leave(const parser::OpenACCAtomicConstruct &x) {
   dirContext_.pop_back();
 }
 
-void AccStructureChecker::Enter(const parser::AccAtomicUpdate &x) {
-  const parser::AssignmentStmt &assignment{
-      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement};
-  const auto &var{std::get<parser::Variable>(assignment.t)};
-  const auto &expr{std::get<parser::Expr>(assignment.t)};
+void AccStructureChecker::CheckAtomicStmt(
+    const parser::AssignmentStmt &assign, std::string &construct) {
+  const auto &var{std::get<parser::Variable>(assign.t)};
+  const auto &expr{std::get<parser::Expr>(assign.t)};
   const auto *rhs{GetExpr(context_, expr)};
   const auto *lhs{GetExpr(context_, var)};
-  if (lhs && rhs) {
-    if (lhs->Rank() != 0)
+
+  if (lhs) {
+    if (lhs->Rank() != 0) {
       context_.Say(expr.source,
-          "LHS of atomic update statement must be scalar"_err_en_US);
-    if (rhs->Rank() != 0)
+          "LHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+  if (rhs) {
+    if (rhs->Rank() != 0) {
       context_.Say(var.GetSource(),
-          "RHS of atomic update statement must be scalar"_err_en_US);
+          "RHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+}
+
+void AccStructureChecker::CheckAtomicUpdateStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"update"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicWriteStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"write"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicCaptureStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"capture"};
+  CheckAtomicStmt(assign, construct);
+}
+
+const parser::Variable *AccStructureChecker::GetIfAtomicUpdateVar(
+    const parser::AssignmentStmt &assign) const {
+  // OpenACC 3.4, 2893-2898
+  const auto &updatedVar{std::get<parser::Variable>(assign.t)};
+  const auto &rhs{std::get<parser::Expr>(assign.t)};
+
+  // Is the rhs something a valid operations that references the updatedVar?
+  const auto expr{GetExpr(context_, rhs)};
+  if (!expr) {
+    llvm::errs() << "IsAtomicUpdateStmt: no expr\n";
+    return nullptr;
+  }
+  const auto [op, args] = evaluate::GetTopLevelOperation(*expr);
----------------
klausler wrote:

should there be braces here?

https://github.com/llvm/llvm-project/pull/149579

From flang-commits at lists.llvm.org  Fri Jul 18 14:08:37 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 14:08:37 -0700 (PDT)
Subject: [flang-commits] [flang] early feedback commit (PR #149579)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149579@github.com>
Message-ID: <687ab7d5.170a0220.90079.8771@mx.google.com>


================
@@ -342,21 +347,149 @@ void AccStructureChecker::Leave(const parser::OpenACCAtomicConstruct &x) {
   dirContext_.pop_back();
 }
 
-void AccStructureChecker::Enter(const parser::AccAtomicUpdate &x) {
-  const parser::AssignmentStmt &assignment{
-      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement};
-  const auto &var{std::get<parser::Variable>(assignment.t)};
-  const auto &expr{std::get<parser::Expr>(assignment.t)};
+void AccStructureChecker::CheckAtomicStmt(
+    const parser::AssignmentStmt &assign, std::string &construct) {
+  const auto &var{std::get<parser::Variable>(assign.t)};
+  const auto &expr{std::get<parser::Expr>(assign.t)};
   const auto *rhs{GetExpr(context_, expr)};
   const auto *lhs{GetExpr(context_, var)};
-  if (lhs && rhs) {
-    if (lhs->Rank() != 0)
+
+  if (lhs) {
+    if (lhs->Rank() != 0) {
       context_.Say(expr.source,
-          "LHS of atomic update statement must be scalar"_err_en_US);
-    if (rhs->Rank() != 0)
+          "LHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+  if (rhs) {
+    if (rhs->Rank() != 0) {
       context_.Say(var.GetSource(),
-          "RHS of atomic update statement must be scalar"_err_en_US);
+          "RHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+}
+
+void AccStructureChecker::CheckAtomicUpdateStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"update"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicWriteStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"write"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicCaptureStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"capture"};
+  CheckAtomicStmt(assign, construct);
+}
+
+const parser::Variable *AccStructureChecker::GetIfAtomicUpdateVar(
+    const parser::AssignmentStmt &assign) const {
+  // OpenACC 3.4, 2893-2898
+  const auto &updatedVar{std::get<parser::Variable>(assign.t)};
+  const auto &rhs{std::get<parser::Expr>(assign.t)};
+
+  // Is the rhs something a valid operations that references the updatedVar?
+  const auto expr{GetExpr(context_, rhs)};
+  if (!expr) {
+    llvm::errs() << "IsAtomicUpdateStmt: no expr\n";
+    return nullptr;
+  }
+  const auto [op, args] = evaluate::GetTopLevelOperation(*expr);
+  switch (op) {
+  // 2909-2910 operator is one of +, *, -, /, .and., .or., .eqv., or .neqv..
----------------
klausler wrote:

what about `integervar = integervar * 2.5`?  the top-level operator will be a conversion here.  maybe check that the rhs type is the same as the lhs type.

https://github.com/llvm/llvm-project/pull/149579

From flang-commits at lists.llvm.org  Fri Jul 18 14:08:38 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 14:08:38 -0700 (PDT)
Subject: [flang-commits] [flang] early feedback commit (PR #149579)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149579@github.com>
Message-ID: <687ab7d6.170a0220.25bd29.da8e@mx.google.com>


================
@@ -342,21 +347,149 @@ void AccStructureChecker::Leave(const parser::OpenACCAtomicConstruct &x) {
   dirContext_.pop_back();
 }
 
-void AccStructureChecker::Enter(const parser::AccAtomicUpdate &x) {
-  const parser::AssignmentStmt &assignment{
-      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement};
-  const auto &var{std::get<parser::Variable>(assignment.t)};
-  const auto &expr{std::get<parser::Expr>(assignment.t)};
+void AccStructureChecker::CheckAtomicStmt(
+    const parser::AssignmentStmt &assign, std::string &construct) {
+  const auto &var{std::get<parser::Variable>(assign.t)};
+  const auto &expr{std::get<parser::Expr>(assign.t)};
   const auto *rhs{GetExpr(context_, expr)};
   const auto *lhs{GetExpr(context_, var)};
-  if (lhs && rhs) {
-    if (lhs->Rank() != 0)
+
+  if (lhs) {
+    if (lhs->Rank() != 0) {
       context_.Say(expr.source,
-          "LHS of atomic update statement must be scalar"_err_en_US);
-    if (rhs->Rank() != 0)
+          "LHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+  if (rhs) {
+    if (rhs->Rank() != 0) {
       context_.Say(var.GetSource(),
-          "RHS of atomic update statement must be scalar"_err_en_US);
+          "RHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+}
+
+void AccStructureChecker::CheckAtomicUpdateStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"update"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicWriteStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"write"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicCaptureStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"capture"};
+  CheckAtomicStmt(assign, construct);
+}
+
+const parser::Variable *AccStructureChecker::GetIfAtomicUpdateVar(
+    const parser::AssignmentStmt &assign) const {
+  // OpenACC 3.4, 2893-2898
+  const auto &updatedVar{std::get<parser::Variable>(assign.t)};
+  const auto &rhs{std::get<parser::Expr>(assign.t)};
+
+  // Is the rhs something a valid operations that references the updatedVar?
+  const auto expr{GetExpr(context_, rhs)};
----------------
klausler wrote:

`const auto *` please, otherwise it makes me wonder whether you omitted a `&`.

https://github.com/llvm/llvm-project/pull/149579

From flang-commits at lists.llvm.org  Fri Jul 18 14:08:38 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Fri, 18 Jul 2025 14:08:38 -0700 (PDT)
Subject: [flang-commits] [flang] early feedback commit (PR #149579)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149579@github.com>
Message-ID: <687ab7d6.050a0220.7dffd.cfce@mx.google.com>


================
@@ -342,21 +347,149 @@ void AccStructureChecker::Leave(const parser::OpenACCAtomicConstruct &x) {
   dirContext_.pop_back();
 }
 
-void AccStructureChecker::Enter(const parser::AccAtomicUpdate &x) {
-  const parser::AssignmentStmt &assignment{
-      std::get<parser::Statement<parser::AssignmentStmt>>(x.t).statement};
-  const auto &var{std::get<parser::Variable>(assignment.t)};
-  const auto &expr{std::get<parser::Expr>(assignment.t)};
+void AccStructureChecker::CheckAtomicStmt(
+    const parser::AssignmentStmt &assign, std::string &construct) {
+  const auto &var{std::get<parser::Variable>(assign.t)};
+  const auto &expr{std::get<parser::Expr>(assign.t)};
   const auto *rhs{GetExpr(context_, expr)};
   const auto *lhs{GetExpr(context_, var)};
-  if (lhs && rhs) {
-    if (lhs->Rank() != 0)
+
+  if (lhs) {
+    if (lhs->Rank() != 0) {
       context_.Say(expr.source,
-          "LHS of atomic update statement must be scalar"_err_en_US);
-    if (rhs->Rank() != 0)
+          "LHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+  if (rhs) {
+    if (rhs->Rank() != 0) {
       context_.Say(var.GetSource(),
-          "RHS of atomic update statement must be scalar"_err_en_US);
+          "RHS of atomic %s statement must be scalar"_err_en_US, construct);
+    }
+    // TODO: Check if lhs is intrinsic type
+  }
+}
+
+void AccStructureChecker::CheckAtomicUpdateStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"update"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicWriteStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"write"};
+  CheckAtomicStmt(assign, construct);
+}
+
+void AccStructureChecker::CheckAtomicCaptureStmt(
+    const parser::AssignmentStmt &assign) {
+  static std::string construct{"capture"};
+  CheckAtomicStmt(assign, construct);
+}
+
+const parser::Variable *AccStructureChecker::GetIfAtomicUpdateVar(
+    const parser::AssignmentStmt &assign) const {
+  // OpenACC 3.4, 2893-2898
+  const auto &updatedVar{std::get<parser::Variable>(assign.t)};
+  const auto &rhs{std::get<parser::Expr>(assign.t)};
+
+  // Is the rhs something a valid operations that references the updatedVar?
+  const auto expr{GetExpr(context_, rhs)};
+  if (!expr) {
+    llvm::errs() << "IsAtomicUpdateStmt: no expr\n";
+    return nullptr;
+  }
+  const auto [op, args] = evaluate::GetTopLevelOperation(*expr);
+  switch (op) {
+  // 2909-2910 operator is one of +, *, -, /, .and., .or., .eqv., or .neqv..
+  case evaluate::operation::Operator::Add:
+  case evaluate::operation::Operator::Mul:
+  case evaluate::operation::Operator::Sub:
+  case evaluate::operation::Operator::Div:
+  case evaluate::operation::Operator::And:
+  case evaluate::operation::Operator::Or:
+  case evaluate::operation::Operator::Eqv:
+  case evaluate::operation::Operator::Neqv:
+  // 2909 intrinsic-procedure name is one of max, min, iand, ior, or ieor.
+  case evaluate::operation::Operator::Max:
+  case evaluate::operation::Operator::Min:
+    // case evaluate::operation::Operator::Iand:
+    // case evaluate::operation::Operator::Ior:
+    // case evaluate::operation::Operator::Ieor:
+    break;
+  // x = (x) is not allowed.
+  case evaluate::operation::Operator::Identity:
+  default:
+    return nullptr;
+  }
+  const auto &updatedExpr{GetExpr(context_, updatedVar)};
+  if (!updatedExpr) {
+    return nullptr;
+  }
+  for (const auto &arg : args) {
+    if (*updatedExpr == arg) {
+      return &updatedVar;
+    }
   }
+  return nullptr;
+}
+
+bool AccStructureChecker::IsAtomicCaptureStmt(
+    const parser::AssignmentStmt &assign,
+    const parser::Variable &updatedVar) const {
+  const auto &var{std::get<parser::Variable>(assign.t)};
+  const auto &expr{std::get<parser::Expr>(assign.t)};
+  const auto *varExpr{GetExpr(context_, var)};
+  const auto *updatedVarExpr{GetExpr(context_, updatedVar)};
+  const auto *exprExpr{GetExpr(context_, expr)};
+  if (!varExpr || !updatedVarExpr || !exprExpr) {
+    return false;
+  }
+  return updatedVarExpr == exprExpr && !(updatedVarExpr == varExpr);
----------------
klausler wrote:

you probably want to compare pointees rather than pointers

https://github.com/llvm/llvm-project/pull/149579

From flang-commits at lists.llvm.org  Fri Jul 18 14:16:40 2025
From: flang-commits at lists.llvm.org (Alexey Bataev via flang-commits)
Date: Fri, 18 Jul 2025 14:16:40 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp]
 [Clang][OpenMP][LoopTransformations] Add support for "#pragma omp fuse"
 loop transformation directive and "looprange" clause (PR #139293)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/139293@github.com>
Message-ID: <687ab9b8.630a0220.2b5166.d465@mx.google.com>


alexey-bataev wrote:

Need a rebase

https://github.com/llvm/llvm-project/pull/139293

From flang-commits at lists.llvm.org  Fri Jul 18 14:43:06 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 14:43:06 -0700 (PDT)
Subject: [flang-commits] [flang] c99c213 - [mlir][Flang][NFC] Replace use of
 `vector.insertelement/extractelement` (#143272)
Message-ID: <687abfea.630a0220.18c80.d493@mx.google.com>


Author: Diego Caballero
Date: 2025-07-18T14:43:03-07:00
New Revision: c99c213e725adee22c386d2059118a49f0c57054

URL: https://github.com/llvm/llvm-project/commit/c99c213e725adee22c386d2059118a49f0c57054
DIFF: https://github.com/llvm/llvm-project/commit/c99c213e725adee22c386d2059118a49f0c57054.diff

LOG: [mlir][Flang][NFC] Replace use of `vector.insertelement/extractelement` (#143272)

This PR is part of the last step to remove `vector.extractelement` and
`vector.insertelement` ops (RFC:
https://discourse.llvm.org/t/rfc-psa-remove-vector-extractelement-and-vector-insertelement-ops-in-favor-of-vector-extract-and-vector-insert-ops).
It replaces `vector.insertelement` and `vector.extractelement` with
`vector.insert` and `vector.extract` in Flang. It looks like no lit
tests are impacted?

Added: 
    

Modified: 
    flang/include/flang/Optimizer/Support/InitFIR.h
    flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
    flang/lib/Optimizer/CodeGen/CMakeLists.txt
    flang/lib/Optimizer/CodeGen/CodeGen.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Support/InitFIR.h b/flang/include/flang/Optimizer/Support/InitFIR.h
index 8bb4791859bb7..aacba233a2b32 100644
--- a/flang/include/flang/Optimizer/Support/InitFIR.h
+++ b/flang/include/flang/Optimizer/Support/InitFIR.h
@@ -23,6 +23,7 @@
 #include "mlir/Dialect/Affine/Passes.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/Extensions/InlinerExtension.h"
+#include "mlir/Dialect/Index/IR/IndexDialect.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 #include "mlir/Dialect/OpenACC/Transforms/Passes.h"
 #include "mlir/Dialect/SCF/Transforms/Passes.h"
@@ -41,7 +42,8 @@ namespace fir::support {
       mlir::cf::ControlFlowDialect, mlir::func::FuncDialect,                   \
       mlir::vector::VectorDialect, mlir::math::MathDialect,                    \
       mlir::complex::ComplexDialect, mlir::DLTIDialect, cuf::CUFDialect,       \
-      mlir::NVVM::NVVMDialect, mlir::gpu::GPUDialect
+      mlir::NVVM::NVVMDialect, mlir::gpu::GPUDialect,                          \
+      mlir::index::IndexDialect
 
 #define FLANG_CODEGEN_DIALECT_LIST FIRCodeGenDialect, mlir::LLVM::LLVMDialect
 

diff  --git a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
index 0094ce892d61b..db12c84496b10 100644
--- a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
@@ -17,6 +17,7 @@
 #include "flang/Evaluate/common.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/MutableBox.h"
+#include "mlir/Dialect/Index/IR/IndexOps.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 
 namespace fir {
@@ -1685,7 +1686,9 @@ PPCIntrinsicLibrary::genVecExtract(mlir::Type resultType,
   if (!isNativeVecElemOrderOnLE())
     uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp);
 
-  return builder.create<mlir::vector::ExtractElementOp>(loc, varg0, uremOp);
+  mlir::Value index = builder.createOrFold<mlir::index::CastUOp>(
+      loc, builder.getIndexType(), uremOp);
+  return builder.create<mlir::vector::ExtractOp>(loc, varg0, index);
 }
 
 // VEC_INSERT
@@ -1706,8 +1709,10 @@ PPCIntrinsicLibrary::genVecInsert(mlir::Type resultType,
   if (!isNativeVecElemOrderOnLE())
     uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp);
 
-  auto res{builder.create<mlir::vector::InsertElementOp>(loc, argBases[0],
-                                                         varg1, uremOp)};
+  mlir::Value index = builder.createOrFold<mlir::index::CastUOp>(
+      loc, builder.getIndexType(), uremOp);
+  mlir::Value res =
+      builder.create<mlir::vector::InsertOp>(loc, argBases[0], varg1, index);
   return builder.create<fir::ConvertOp>(loc, vecTyInfo.toFirVectorType(), res);
 }
 

diff  --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 16c7944a885a1..d5ea3c7a8e282 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -37,6 +37,7 @@ add_flang_library(FIRCodeGen
   MLIRComplexToROCDLLibraryCalls
   MLIRComplexToStandard
   MLIRGPUDialect
+  MLIRIndexToLLVM
   MLIRMathToFuncs
   MLIRMathToLLVM
   MLIRMathToLibm

diff  --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index d879382555c39..609ba27bc212b 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -37,6 +37,7 @@
 #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
+#include "mlir/Conversion/IndexToLLVM/IndexToLLVM.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/MathToFuncs/MathToFuncs.h"
 #include "mlir/Conversion/MathToLLVM/MathToLLVM.h"
@@ -4224,6 +4225,7 @@ class FIRToLLVMLowering
     if (!isAMDGCN)
       mlir::populateMathToLibmConversionPatterns(pattern);
     mlir::populateComplexToLLVMConversionPatterns(typeConverter, pattern);
+    mlir::index::populateIndexToLLVMConversionPatterns(typeConverter, pattern);
     mlir::populateVectorToLLVMConversionPatterns(typeConverter, pattern);
 
     // Flang specific overloads for OpenMP operations, to allow for special


        

From flang-commits at lists.llvm.org  Fri Jul 18 14:43:10 2025
From: flang-commits at lists.llvm.org (Diego Caballero via flang-commits)
Date: Fri, 18 Jul 2025 14:43:10 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][Flang][NFC] Replace use of
 `vector.insertelement/extractelement` (PR #143272)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143272@github.com>
Message-ID: <687abfee.a70a0220.3dc274.d55f@mx.google.com>


https://github.com/dcaballe closed https://github.com/llvm/llvm-project/pull/143272

From flang-commits at lists.llvm.org  Fri Jul 18 22:06:31 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Fri, 18 Jul 2025 22:06:31 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Minor revision of the getting
	started guide. (PR #149632)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149632@github.com>
Message-ID: <687b27d7.050a0220.135890.c0a6@mx.google.com>


https://github.com/mjklemm approved this pull request.


https://github.com/llvm/llvm-project/pull/149632

From flang-commits at lists.llvm.org  Fri Jul 18 22:07:00 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Fri, 18 Jul 2025 22:07:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Minor revision of the getting
	started guide. (PR #149632)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149632@github.com>
Message-ID: <687b27f4.050a0220.3ba056.e7b5@mx.google.com>


mjklemm wrote:

LGTM, thanks for catching this

https://github.com/llvm/llvm-project/pull/149632

From flang-commits at lists.llvm.org  Fri Jul 18 22:41:11 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 22:41:11 -0700 (PDT)
Subject: [flang-commits] [flang] c875bb8 - [flang] Minor revision of the
 getting started guide. (#149632)
Message-ID: <687b2ff7.050a0220.951e.e6ea@mx.google.com>


Author: Connector Switch
Date: 2025-07-19T13:41:07+08:00
New Revision: c875bb8eef6c60e7cd5814fdbab149abb86efa30

URL: https://github.com/llvm/llvm-project/commit/c875bb8eef6c60e7cd5814fdbab149abb86efa30
DIFF: https://github.com/llvm/llvm-project/commit/c875bb8eef6c60e7cd5814fdbab149abb86efa30.diff

LOG: [flang] Minor revision of the getting started guide. (#149632)

- C++17 is now the default, no need to specify explicitly.

https://github.com/llvm/llvm-project/blob/b3c9ed151f18fbbfe027cf93dd7957f36bcbaccf/flang/CMakeLists.txt#L13

- The current recommended way to set up OpenMP is as a runtime, not as a
project.

https://github.com/llvm/llvm-project/blob/b3c9ed151f18fbbfe027cf93dd7957f36bcbaccf/llvm/CMakeLists.txt#L209-L214

Added: 
    

Modified: 
    flang/docs/GettingStarted.md

Removed: 
    


################################################################################
diff  --git a/flang/docs/GettingStarted.md b/flang/docs/GettingStarted.md
index 0b3b551ffbfba..2ea8093b607cf 100644
--- a/flang/docs/GettingStarted.md
+++ b/flang/docs/GettingStarted.md
@@ -74,15 +74,14 @@ cmake \
   -G Ninja \
   -DCMAKE_BUILD_TYPE=Release \
   -DCMAKE_INSTALL_PREFIX=$INSTALLDIR \
-  -DCMAKE_CXX_STANDARD=17 \
   -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
   -DCMAKE_CXX_LINK_FLAGS="-Wl,-rpath,$LD_LIBRARY_PATH" \
   -DFLANG_ENABLE_WERROR=ON \
   -DLLVM_ENABLE_ASSERTIONS=ON \
   -DLLVM_TARGETS_TO_BUILD=host \
   -DLLVM_LIT_ARGS=-v \
-  -DLLVM_ENABLE_PROJECTS="clang;mlir;flang;openmp" \
-  -DLLVM_ENABLE_RUNTIMES="compiler-rt;flang-rt" \
+  -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" \
+  -DLLVM_ENABLE_RUNTIMES="compiler-rt;flang-rt;openmp" \
   ../llvm-project/llvm
 
 ninja
@@ -141,7 +140,6 @@ cd build
 cmake \
   -G Ninja \
   -DCMAKE_BUILD_TYPE=Release \
-  -DCMAKE_CXX_STANDARD=17 \
   -DCMAKE_CXX_LINK_FLAGS="-Wl,-rpath,$LD_LIBRARY_PATH" \
   -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
   -DFLANG_ENABLE_WERROR=ON \


        

From flang-commits at lists.llvm.org  Fri Jul 18 22:41:13 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Fri, 18 Jul 2025 22:41:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Minor revision of the getting
	started guide. (PR #149632)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149632@github.com>
Message-ID: <687b2ff9.170a0220.25f985.edb8@mx.google.com>


https://github.com/c8ef closed https://github.com/llvm/llvm-project/pull/149632

From flang-commits at lists.llvm.org  Sat Jul 19 05:32:04 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Sat, 19 Jul 2025 05:32:04 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687b9044.170a0220.2f342b.fe74@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Sat Jul 19 05:32:34 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Sat, 19 Jul 2025 05:32:34 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687b9062.170a0220.38494f.fbc2@mx.google.com>


https://github.com/Meinersbur ready_for_review https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Sat Jul 19 05:33:59 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Sat, 19 Jul 2025 05:33:59 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][Flang][NFC] Replace use of
 `vector.insertelement/extractelement` (PR #143272)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143272@github.com>
Message-ID: <687b90b7.050a0220.2f9a7b.f639@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `flang` at step 6 "test-build-unified-tree-check-flang".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/34311

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (test-build-unified-tree-check-flang) failure: test (failure)
******************** TEST 'Flang :: Lower/PowerPC/ppc-vec-splat-elem-order.f90' FAILED ********************
Exit Code: 1

Command Output (stderr):
--
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -flang-experimental-hlfir -emit-llvm /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90 -fno-ppc-native-vector-element-order -triple ppc64le-unknown-linux -o - | /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck --check-prefixes="LLVMIR" /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90 # RUN: at line 1
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck --check-prefixes=LLVMIR /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -flang-experimental-hlfir -emit-llvm /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90 -fno-ppc-native-vector-element-order -triple ppc64le-unknown-linux -o -
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90:22:11: error: LLVMIR: expected string not found in input
! LLVMIR: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 15
          ^
<stdin>:18:39: note: scanning from here
 %3 = load <16 x i8>, ptr %0, align 16
                                      ^
<stdin>:18:39: note: with "x" equal to "3"
 %3 = load <16 x i8>, ptr %0, align 16
                                      ^
<stdin>:19:2: note: possible intended match here
 %4 = extractelement <16 x i8> %3, i64 15
 ^

Input file: <stdin>
Check file: /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90

-dump-input=help explains the following input dump.

Input was:
<<<<<<
            .
            .
            .
           13:  ret void 
           14: } 
           15:  
           16: define void @vec_splat_testu8i16_(ptr %0) #0 { 
           17:  %2 = alloca <16 x i8>, i64 1, align 16 
           18:  %3 = load <16 x i8>, ptr %0, align 16 
check:22'0                                           X error: no match found
check:22'1                                             with "x" equal to "3"
           19:  %4 = extractelement <16 x i8> %3, i64 15 
check:22'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
check:22'2      ?                                         possible intended match
           20:  %5 = insertelement <16 x i8> poison, i8 %4, i32 0 
check:22'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           21:  %6 = shufflevector <16 x i8> %5, <16 x i8> poison, <16 x i32> zeroinitializer 
check:22'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           22:  store <16 x i8> %6, ptr %2, align 16 
check:22'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           23:  ret void 
check:22'0     ~~~~~~~~~~
...

```

</details>

https://github.com/llvm/llvm-project/pull/143272

From flang-commits at lists.llvm.org  Fri Jul 18 14:43:18 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Fri, 18 Jul 2025 14:43:18 -0700 (PDT)
Subject: [flang-commits] [flang] 2e67dcf - [flang] update ppc lit tests
 after using vector.insert and vector.extract (NFC) (#148775)
Message-ID: <687abff6.170a0220.43545.d8a3@mx.google.com>


Author: Kelvin Li
Date: 2025-07-18T14:43:15-07:00
New Revision: 2e67dcfdcd023df2f06e0823eeea23990ce41534

URL: https://github.com/llvm/llvm-project/commit/2e67dcfdcd023df2f06e0823eeea23990ce41534
DIFF: https://github.com/llvm/llvm-project/commit/2e67dcfdcd023df2f06e0823eeea23990ce41534.diff

LOG: [flang] update ppc lit tests after using vector.insert and vector.extract (NFC) (#148775)

See https://github.com/llvm/llvm-project/pull/143272

Added: 
    

Modified: 
    flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90
    flang/test/Lower/PowerPC/ppc-vec-extract.f90
    flang/test/Lower/PowerPC/ppc-vec-insert-elem-order.f90
    flang/test/Lower/PowerPC/ppc-vec-insert.f90
    flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90
    flang/test/Lower/PowerPC/ppc-vec-splat.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90 b/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90
index 73669c25b339e..d7d14581b4b7f 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -flang-experimental-hlfir -emit-llvm %s -fno-ppc-native-vector-element-order -triple ppc64le-unknown-linux -o - | FileCheck --check-prefixes="LLVMIR" %s
+! RUN: %flang_fc1 -emit-llvm %s -fno-ppc-native-vector-element-order -triple ppc64le-unknown-linux -o - | FileCheck --check-prefixes="LLVMIR" %s
 ! REQUIRES: target=powerpc{{.*}}
 
 !CHECK-LABEL: vec_extract_testr4i8
@@ -27,6 +27,7 @@ subroutine vec_extract_testi8i1(arg1, arg2, r)
 ! LLVMIR: %[[arg2:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[urem:.*]] = urem i8 %[[arg2]], 2
 ! LLVMIR: %[[sub:.*]] = sub i8 1, %[[urem]]
-! LLVMIR: %[[r:.*]] = extractelement <2 x i64> %[[arg1]], i8 %[[sub]]
+! LLVMIR: %[[idx:.*]] = zext i8 %[[sub]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <2 x i64> %[[arg1]], i64 %[[idx]]
 ! LLVMIR: store i64 %[[r]], ptr %{{[0-9]}}, align 8
 end subroutine vec_extract_testi8i1

diff  --git a/flang/test/Lower/PowerPC/ppc-vec-extract.f90 b/flang/test/Lower/PowerPC/ppc-vec-extract.f90
index 0f279347b6b75..32c0dcfd66013 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-extract.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-extract.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -flang-experimental-hlfir -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-LE" %s
-! RUN: %flang_fc1 -flang-experimental-hlfir -triple powerpc64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-BE" %s
+! RUN: %flang_fc1 -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-LE" %s
+! RUN: %flang_fc1 -triple powerpc64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-BE" %s
 ! REQUIRES: target=powerpc{{.*}}
 
 !-------------
@@ -19,8 +19,9 @@ subroutine vec_extract_testf32(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[u:.*]] = urem i8 %[[i1]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i8 3, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <4 x float> %[[x]], i8 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <4 x float> %[[x]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <4 x float> %[[x]], i64 %[[idx]]
 ! LLVMIR: store float %[[r]], ptr %{{[0-9]}}, align 4
 
   r = vec_extract(x, i2)
@@ -29,8 +30,9 @@ subroutine vec_extract_testf32(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[u:.*]] = urem i16 %[[i2]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i16 3, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <4 x float> %[[x]], i16 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <4 x float> %[[x]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <4 x float> %[[x]], i64 %[[idx]]
 ! LLVMIR: store float %[[r]], ptr %{{[0-9]}}, align 4
 
   r = vec_extract(x, i4)
@@ -39,18 +41,19 @@ subroutine vec_extract_testf32(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[u:.*]] = urem i32 %[[i4]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i32 3, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <4 x float> %[[x]], i32 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <4 x float> %[[x]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <4 x float> %[[x]], i64 %[[idx]]
 ! LLVMIR: store float %[[r]], ptr %{{[0-9]}}, align 4
 
   r = vec_extract(x, i8)
 
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[u:.*]] = urem i64 %[[i8]], 4
-! LLVMIR-BE: %[[s:.*]] = sub i64 3, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <4 x float> %[[x]], i64 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <4 x float> %[[x]], i64 %[[s]]
+! LLVMIR-BE: %[[u:.*]] = urem i64 %[[i8]], 4
+! LLVMIR-BE: %[[idx:.*]] = sub i64 3, %[[u]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 4
+! LLVMIR: %[[r:.*]] = extractelement <4 x float> %[[x]], i64 %[[idx]]
 ! LLVMIR: store float %[[r]], ptr %{{[0-9]}}, align 4
 end subroutine vec_extract_testf32
 
@@ -68,8 +71,9 @@ subroutine vec_extract_testf64(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[u:.*]] = urem i8 %[[i1]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i8 1, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <2 x double> %[[x]], i8 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <2 x double> %[[x]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <2 x double> %[[x]], i64 %[[idx]]
 ! LLVMIR: store double %[[r]], ptr %{{[0-9]}}, align 8
 
   r = vec_extract(x, i2)
@@ -78,8 +82,9 @@ subroutine vec_extract_testf64(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[u:.*]] = urem i16 %[[i2]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i16 1, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <2 x double> %[[x]], i16 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <2 x double> %[[x]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <2 x double> %[[x]], i64 %[[idx]]
 ! LLVMIR: store double %[[r]], ptr %{{[0-9]}}, align 8
 
 
@@ -89,18 +94,19 @@ subroutine vec_extract_testf64(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[u:.*]] = urem i32 %[[i4]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i32 1, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <2 x double> %[[x]], i32 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <2 x double> %[[x]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <2 x double> %[[x]], i64 %[[idx]]
 ! LLVMIR: store double %[[r]], ptr %{{[0-9]}}, align 8
 
   r = vec_extract(x, i8)
 
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[u:.*]] = urem i64 %[[i8]], 2
-! LLVMIR-BE: %[[s:.*]] = sub i64 1, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <2 x double> %[[x]], i64 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <2 x double> %[[x]], i64 %[[s]]
+! LLVMIR-BE: %[[u:.*]] = urem i64 %[[i8]], 2
+! LLVMIR-BE: %[[idx:.*]] = sub i64 1, %[[u]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 2
+! LLVMIR: %[[r:.*]] = extractelement <2 x double> %[[x]], i64 %[[idx]]
 ! LLVMIR: store double %[[r]], ptr %{{[0-9]}}, align 8
 end subroutine vec_extract_testf64
 
@@ -118,8 +124,9 @@ subroutine vec_extract_testi8(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[u:.*]] = urem i8 %[[i1]], 16
 ! LLVMIR-BE: %[[s:.*]] = sub i8 15, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <16 x i8> %[[x]], i8 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <16 x i8> %[[x]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <16 x i8> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i8 %[[r]], ptr %{{[0-9]}}, align 1
 
   r = vec_extract(x, i2)
@@ -128,8 +135,9 @@ subroutine vec_extract_testi8(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[u:.*]] = urem i16 %[[i2]], 16
 ! LLVMIR-BE: %[[s:.*]] = sub i16 15, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <16 x i8> %[[x]], i16 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <16 x i8> %[[x]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <16 x i8> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i8 %[[r]], ptr %{{[0-9]}}, align 1
 
   r = vec_extract(x, i4)
@@ -138,18 +146,19 @@ subroutine vec_extract_testi8(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[u:.*]] = urem i32 %[[i4]], 16
 ! LLVMIR-BE: %[[s:.*]] = sub i32 15, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <16 x i8> %[[x]], i32 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <16 x i8> %[[x]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <16 x i8> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i8 %[[r]], ptr %{{[0-9]}}, align 1
 
   r = vec_extract(x, i8)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[u:.*]] = urem i64 %[[i8]], 16
-! LLVMIR-BE: %[[s:.*]] = sub i64 15, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <16 x i8> %[[x]], i64 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <16 x i8> %[[x]], i64 %[[s]]
+! LLVMIR-BE: %[[u:.*]] = urem i64 %[[i8]], 16
+! LLVMIR-BE: %[[idx:.*]] = sub i64 15, %[[u]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 16
+! LLVMIR: %[[r:.*]] = extractelement <16 x i8> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i8 %[[r]], ptr %{{[0-9]}}, align 1
 end subroutine vec_extract_testi8
 
@@ -167,8 +176,9 @@ subroutine vec_extract_testi16(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[u:.*]] = urem i8 %[[i1]], 8
 ! LLVMIR-BE: %[[s:.*]] = sub i8 7, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <8 x i16> %[[x]], i8 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <8 x i16> %[[x]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <8 x i16> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i16 %[[r]], ptr %{{[0-9]}}, align 2
 
   r = vec_extract(x, i2)
@@ -177,8 +187,9 @@ subroutine vec_extract_testi16(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[u:.*]] = urem i16 %[[i2]], 8
 ! LLVMIR-BE: %[[s:.*]] = sub i16 7, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <8 x i16> %[[x]], i16 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <8 x i16> %[[x]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <8 x i16> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i16 %[[r]], ptr %{{[0-9]}}, align 2
 
   r = vec_extract(x, i4)
@@ -187,18 +198,19 @@ subroutine vec_extract_testi16(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[u:.*]] = urem i32 %[[i4]], 8
 ! LLVMIR-BE: %[[s:.*]] = sub i32 7, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <8 x i16> %[[x]], i32 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <8 x i16> %[[x]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <8 x i16> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i16 %[[r]], ptr %{{[0-9]}}, align 2
 
   r = vec_extract(x, i8)
 
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[u:.*]] = urem i64 %[[i8]], 8
-! LLVMIR-BE: %[[s:.*]] = sub i64 7, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <8 x i16> %[[x]], i64 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <8 x i16> %[[x]], i64 %[[s]]
+! LLVMIR-BE: %[[u:.*]] = urem i64 %[[i8]], 8
+! LLVMIR-BE: %[[idx:.*]] = sub i64 7, %[[u]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 8
+! LLVMIR: %[[r:.*]] = extractelement <8 x i16> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i16 %[[r]], ptr %{{[0-9]}}, align 2
 end subroutine vec_extract_testi16
 
@@ -216,8 +228,9 @@ subroutine vec_extract_testi32(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[u:.*]] = urem i8 %[[i1]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i8 3, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <4 x i32> %[[x]], i8 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <4 x i32> %[[x]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <4 x i32> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i32 %[[r]], ptr %{{[0-9]}}, align 4
 
   r = vec_extract(x, i2)
@@ -226,8 +239,9 @@ subroutine vec_extract_testi32(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[u:.*]] = urem i16 %[[i2]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i16 3, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <4 x i32> %[[x]], i16 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <4 x i32> %[[x]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <4 x i32> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i32 %[[r]], ptr %{{[0-9]}}, align 4
 
   r = vec_extract(x, i4)
@@ -236,18 +250,19 @@ subroutine vec_extract_testi32(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[u:.*]] = urem i32 %[[i4]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i32 3, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <4 x i32> %[[x]], i32 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <4 x i32> %[[x]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <4 x i32> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i32 %[[r]], ptr %{{[0-9]}}, align 4
 
   r = vec_extract(x, i8)
 
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[u:.*]] = urem i64 %[[i8]], 4
-! LLVMIR-BE: %[[s:.*]] = sub i64 3, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <4 x i32> %[[x]], i64 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <4 x i32> %[[x]], i64 %[[s]]
+! LLVMIR-BE: %[[u:.*]] = urem i64 %[[i8]], 4
+! LLVMIR-BE: %[[idx:.*]] = sub i64 3, %[[u]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 4
+! LLVMIR: %[[r:.*]] = extractelement <4 x i32> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i32 %[[r]], ptr %{{[0-9]}}, align 4
 end subroutine vec_extract_testi32
 
@@ -265,8 +280,9 @@ subroutine vec_extract_testi64(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[u:.*]] = urem i8 %[[i1]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i8 1, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <2 x i64> %[[x]], i8 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <2 x i64> %[[x]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <2 x i64> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i64 %[[r]], ptr %{{[0-9]}}, align 8
 
   r = vec_extract(x, i2)
@@ -275,8 +291,9 @@ subroutine vec_extract_testi64(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[u:.*]] = urem i16 %[[i2]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i16 1, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <2 x i64> %[[x]], i16 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <2 x i64> %[[x]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <2 x i64> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i64 %[[r]], ptr %{{[0-9]}}, align 8
 
   r = vec_extract(x, i4)
@@ -285,17 +302,18 @@ subroutine vec_extract_testi64(x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[u:.*]] = urem i32 %[[i4]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i32 1, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <2 x i64> %[[x]], i32 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <2 x i64> %[[x]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[u]] to i64
+! LLVMIR: %[[r:.*]] = extractelement <2 x i64> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i64 %[[r]], ptr %{{[0-9]}}, align 8
 
   r = vec_extract(x, i8)
 
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[u:.*]] = urem i64 %[[i8]], 2
-! LLVMIR-BE: %[[s:.*]] = sub i64 1, %[[u]]
-! LLVMIR-LE: %[[r:.*]] = extractelement <2 x i64> %[[x]], i64 %[[u]]
-! LLVMIR-BE: %[[r:.*]] = extractelement <2 x i64> %[[x]], i64 %[[s]]
+! LLVMIR-BE: %[[u:.*]] = urem i64 %[[i8]], 2
+! LLVMIR-BE: %[[idx:.*]] = sub i64 1, %[[u]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 2
+! LLVMIR: %[[r:.*]] = extractelement <2 x i64> %[[x]], i64 %[[idx]]
 ! LLVMIR: store i64 %[[r]], ptr %{{[0-9]}}, align 8
 end subroutine vec_extract_testi64

diff  --git a/flang/test/Lower/PowerPC/ppc-vec-insert-elem-order.f90 b/flang/test/Lower/PowerPC/ppc-vec-insert-elem-order.f90
index f64df46f170ab..b30065d74e46b 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-insert-elem-order.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-insert-elem-order.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -flang-experimental-hlfir -emit-llvm %s -fno-ppc-native-vector-element-order -triple ppc64le-unknown-linux -o - | FileCheck --check-prefixes="LLVMIR" %s
+! RUN: %flang_fc1 -emit-llvm %s -fno-ppc-native-vector-element-order -triple ppc64le-unknown-linux -o - | FileCheck --check-prefixes="LLVMIR" %s
 ! REQUIRES: target=powerpc{{.*}}
 
 !CHECK-LABEL: vec_insert_testf32i64
@@ -31,6 +31,7 @@ subroutine vec_insert_testi64i8(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[urem:.*]] = urem i8 %[[i1]], 2
 ! LLVMIR: %[[sub:.*]] = sub i8 1, %[[urem]]
-! LLVMIR: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i8 %[[sub]]
+! LLVMIR: %[[idx:.*]] = zext i8 %[[sub]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x i64> %[[r]], ptr %{{[0-9]}}, align 16
 end subroutine vec_insert_testi64i8

diff  --git a/flang/test/Lower/PowerPC/ppc-vec-insert.f90 b/flang/test/Lower/PowerPC/ppc-vec-insert.f90
index dd57fcc67be08..26bc7fc114cec 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-insert.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-insert.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -flang-experimental-hlfir -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-LE" %s
-! RUN: %flang_fc1 -flang-experimental-hlfir -triple powerpc64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-BE" %s
+! RUN: %flang_fc1 -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-LE" %s
+! RUN: %flang_fc1 -triple powerpc64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-BE" %s
 ! REQUIRES: target=powerpc{{.*}}
 
 ! vec_insert
@@ -20,8 +20,9 @@ subroutine vec_insert_testf32(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[urem:.*]] = urem i8 %[[i1]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i8 3, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i8 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i64 %[[idx]]
 ! LLVMIR: store <4 x float> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i2)
@@ -31,8 +32,9 @@ subroutine vec_insert_testf32(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[urem:.*]] = urem i16 %[[i2]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i16 3, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i16 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i64 %[[idx]]
 ! LLVMIR: store <4 x float> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i4)
@@ -42,8 +44,9 @@ subroutine vec_insert_testf32(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[urem:.*]] = urem i32 %[[i4]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i32 3, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i32 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i64 %[[idx]]
 ! LLVMIR: store <4 x float> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i8)
@@ -51,10 +54,10 @@ subroutine vec_insert_testf32(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[v:.*]] = load float, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[urem:.*]] = urem i64 %[[i8]], 4
-! LLVMIR-BE: %[[s:.*]] = sub i64 3, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i64 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i64 %[[s]]
+! LLVMIR-BE: %[[urem:.*]] = urem i64 %[[i8]], 4
+! LLVMIR-BE: %[[idx:.*]] = sub i64 3, %[[urem]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 4
+! LLVMIR: %[[r:.*]] = insertelement <4 x float> %[[x]], float %[[v]], i64 %[[idx]]
 ! LLVMIR: store <4 x float> %[[r]], ptr %{{[0-9]}}, align 16
 end subroutine vec_insert_testf32
 
@@ -74,8 +77,9 @@ subroutine vec_insert_testf64(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[urem:.*]] = urem i8 %[[i1]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i8 1, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i8 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x double> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i2)
@@ -85,8 +89,9 @@ subroutine vec_insert_testf64(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[urem:.*]] = urem i16 %[[i2]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i16 1, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i16 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x double> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i4)
@@ -96,8 +101,9 @@ subroutine vec_insert_testf64(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[urem:.*]] = urem i32 %[[i4]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i32 1, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i32 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x double> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i8)
@@ -105,10 +111,10 @@ subroutine vec_insert_testf64(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[v:.*]] = load double, ptr %{{[0-9]}}, align 8
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[urem:.*]] = urem i64 %[[i8]], 2
-! LLVMIR-BE: %[[s:.*]] = sub i64 1, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i64 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i64 %[[s]]
+! LLVMIR-BE: %[[urem:.*]] = urem i64 %[[i8]], 2
+! LLVMIR-BE: %[[idx:.*]] = sub i64 1, %[[urem]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 2
+! LLVMIR: %[[r:.*]] = insertelement <2 x double> %[[x]], double %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x double> %[[r]], ptr %{{[0-9]}}, align 16
 end subroutine vec_insert_testf64
 
@@ -128,8 +134,9 @@ subroutine vec_insert_testi8(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[urem:.*]] = urem i8 %[[i1]], 16
 ! LLVMIR-BE: %[[s:.*]] = sub i8 15, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i8 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <16 x i8> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i2)
@@ -139,8 +146,9 @@ subroutine vec_insert_testi8(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[urem:.*]] = urem i16 %[[i2]], 16
 ! LLVMIR-BE: %[[s:.*]] = sub i16 15, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i16 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <16 x i8> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i4)
@@ -150,8 +158,9 @@ subroutine vec_insert_testi8(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[urem:.*]] = urem i32 %[[i4]], 16
 ! LLVMIR-BE: %[[s:.*]] = sub i32 15, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i32 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <16 x i8> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i8)
@@ -159,10 +168,10 @@ subroutine vec_insert_testi8(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[v:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[urem:.*]] = urem i64 %[[i8]], 16
-! LLVMIR-BE: %[[s:.*]] = sub i64 15, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i64 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i64 %[[s]]
+! LLVMIR-BE: %[[urem:.*]] = urem i64 %[[i8]], 16
+! LLVMIR-BE: %[[idx:.*]] = sub i64 15, %[[urem]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 16
+! LLVMIR: %[[r:.*]] = insertelement <16 x i8> %[[x]], i8 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <16 x i8> %[[r]], ptr %{{[0-9]}}, align 16
 end subroutine vec_insert_testi8
 
@@ -182,8 +191,9 @@ subroutine vec_insert_testi16(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[urem:.*]] = urem i8 %[[i1]], 8
 ! LLVMIR-BE: %[[s:.*]] = sub i8 7, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i8 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <8 x i16> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i2)
@@ -193,8 +203,9 @@ subroutine vec_insert_testi16(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[urem:.*]] = urem i16 %[[i2]], 8
 ! LLVMIR-BE: %[[s:.*]] = sub i16 7, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i16 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <8 x i16> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i4)
@@ -204,8 +215,9 @@ subroutine vec_insert_testi16(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[urem:.*]] = urem i32 %[[i4]], 8
 ! LLVMIR-BE: %[[s:.*]] = sub i32 7, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i32 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <8 x i16> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i8)
@@ -213,10 +225,10 @@ subroutine vec_insert_testi16(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[v:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[urem:.*]] = urem i64 %[[i8]], 8
-! LLVMIR-BE: %[[s:.*]] = sub i64 7, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i64 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i64 %[[s]]
+! LLVMIR-BE: %[[urem:.*]] = urem i64 %[[i8]], 8
+! LLVMIR-BE: %[[idx:.*]] = sub i64 7, %[[urem]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 8
+! LLVMIR: %[[r:.*]] = insertelement <8 x i16> %[[x]], i16 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <8 x i16> %[[r]], ptr %{{[0-9]}}, align 16
 end subroutine vec_insert_testi16
 
@@ -236,8 +248,9 @@ subroutine vec_insert_testi32(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[urem:.*]] = urem i8 %[[i1]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i8 3, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i8 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <4 x i32> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i2)
@@ -247,8 +260,9 @@ subroutine vec_insert_testi32(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[urem:.*]] = urem i16 %[[i2]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i16 3, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i16 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <4 x i32> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i4)
@@ -258,8 +272,9 @@ subroutine vec_insert_testi32(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[urem:.*]] = urem i32 %[[i4]], 4
 ! LLVMIR-BE: %[[s:.*]] = sub i32 3, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i32 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <4 x i32> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i8)
@@ -267,10 +282,10 @@ subroutine vec_insert_testi32(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[v:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[urem:.*]] = urem i64 %[[i8]], 4
-! LLVMIR-BE: %[[s:.*]] = sub i64 3, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i64 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i64 %[[s]]
+! LLVMIR-BE: %[[urem:.*]] = urem i64 %[[i8]], 4
+! LLVMIR-BE: %[[idx:.*]] = sub i64 3, %[[urem]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 4
+! LLVMIR: %[[r:.*]] = insertelement <4 x i32> %[[x]], i32 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <4 x i32> %[[r]], ptr %{{[0-9]}}, align 16
 end subroutine vec_insert_testi32
 
@@ -290,8 +305,9 @@ subroutine vec_insert_testi64(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i1:.*]] = load i8, ptr %{{[0-9]}}, align 1
 ! LLVMIR: %[[urem:.*]] = urem i8 %[[i1]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i8 1, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i8 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i8 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i8 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i8 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x i64> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i2)
@@ -301,8 +317,9 @@ subroutine vec_insert_testi64(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i2:.*]] = load i16, ptr %{{[0-9]}}, align 2
 ! LLVMIR: %[[urem:.*]] = urem i16 %[[i2]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i16 1, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i16 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i16 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i16 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i16 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x i64> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i4)
@@ -312,8 +329,9 @@ subroutine vec_insert_testi64(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[i4:.*]] = load i32, ptr %{{[0-9]}}, align 4
 ! LLVMIR: %[[urem:.*]] = urem i32 %[[i4]], 2
 ! LLVMIR-BE: %[[s:.*]] = sub i32 1, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i32 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i32 %[[s]]
+! LLVMIR-BE: %[[idx:.*]] = zext i32 %[[s]] to i64
+! LLVMIR-LE: %[[idx:.*]] = zext i32 %[[urem]] to i64
+! LLVMIR: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x i64> %[[r]], ptr %{{[0-9]}}, align 16
 
   r = vec_insert(v, x, i8)
@@ -321,9 +339,9 @@ subroutine vec_insert_testi64(v, x, i1, i2, i4, i8)
 ! LLVMIR: %[[v:.*]] = load i64, ptr %{{[0-9]}}, align 8
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
 ! LLVMIR: %[[i8:.*]] = load i64, ptr %{{[0-9]}}, align 8
-! LLVMIR: %[[urem:.*]] = urem i64 %[[i8]], 2
-! LLVMIR-BE: %[[s:.*]] = sub i64 1, %[[urem]]
-! LLVMIR-LE: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i64 %[[urem]]
-! LLVMIR-BE: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i64 %[[s]]
+! LLVMIR-BE: %[[urem:.*]] = urem i64 %[[i8]], 2
+! LLVMIR-BE: %[[idx:.*]] = sub i64 1, %[[urem]]
+! LLVMIR-LE: %[[idx:.*]] = urem i64 %[[i8]], 2
+! LLVMIR: %[[r:.*]] = insertelement <2 x i64> %[[x]], i64 %[[v]], i64 %[[idx]]
 ! LLVMIR: store <2 x i64> %[[r]], ptr %{{[0-9]}}, align 16
 end subroutine vec_insert_testi64

diff  --git a/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90 b/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90
index 50604e1f720f3..ca8c0c3f6f1d8 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-splat-elem-order.f90
@@ -19,7 +19,7 @@ subroutine vec_splat_testu8i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
-! LLVMIR: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 15
+! LLVMIR: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
 ! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16

diff  --git a/flang/test/Lower/PowerPC/ppc-vec-splat.f90 b/flang/test/Lower/PowerPC/ppc-vec-splat.f90
index f3c1f19d5877d..55614c75d1ad8 100644
--- a/flang/test/Lower/PowerPC/ppc-vec-splat.f90
+++ b/flang/test/Lower/PowerPC/ppc-vec-splat.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -flang-experimental-hlfir -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-LE" %s
-! RUN: %flang_fc1 -flang-experimental-hlfir -triple powerpc64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-BE" %s
+! RUN: %flang_fc1 -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-LE" %s
+! RUN: %flang_fc1 -triple powerpc64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefixes="LLVMIR","LLVMIR-BE" %s
 ! REQUIRES: target=powerpc{{.*}}
 
 !----------------
@@ -12,8 +12,8 @@ subroutine vec_splat_testi8i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i8 15
+! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
 ! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
@@ -25,8 +25,8 @@ subroutine vec_splat_testi8i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 15
+! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
 ! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
@@ -38,8 +38,8 @@ subroutine vec_splat_testi8i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i32 15
+! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
 ! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
@@ -64,8 +64,8 @@ subroutine vec_splat_testi16i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i8 7
+! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 7
 ! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
@@ -77,8 +77,8 @@ subroutine vec_splat_testi16i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i16 7
+! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 7
 ! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
@@ -90,8 +90,8 @@ subroutine vec_splat_testi16i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i32 7
+! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 7
 ! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
@@ -116,8 +116,8 @@ subroutine vec_splat_testi32i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i8 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
@@ -129,8 +129,8 @@ subroutine vec_splat_testi32i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i16 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
@@ -142,8 +142,8 @@ subroutine vec_splat_testi32i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i32 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
@@ -168,8 +168,8 @@ subroutine vec_splat_testi64i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i8 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
@@ -181,8 +181,8 @@ subroutine vec_splat_testi64i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i16 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
@@ -194,8 +194,8 @@ subroutine vec_splat_testi64i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i32 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
@@ -220,8 +220,8 @@ subroutine vec_splat_testf32i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i8 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
@@ -233,8 +233,8 @@ subroutine vec_splat_testf32i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i16 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
@@ -246,8 +246,8 @@ subroutine vec_splat_testf32i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <4 x float>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i32 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x float> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x float> poison, float %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x float> %[[ins]], <4 x float> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x float> %[[y]], ptr %{{[0-9]}}, align 16
@@ -272,8 +272,8 @@ subroutine vec_splat_testf64i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i8 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x double> poison, double %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x double> %[[y]], ptr %{{[0-9]}}, align 16
@@ -285,8 +285,8 @@ subroutine vec_splat_testf64i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i16 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x double> poison, double %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x double> %[[y]], ptr %{{[0-9]}}, align 16
@@ -298,8 +298,8 @@ subroutine vec_splat_testf64i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <2 x double>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i32 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x double> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x double> poison, double %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x double> %[[ins]], <2 x double> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x double> %[[y]], ptr %{{[0-9]}}, align 16
@@ -324,8 +324,8 @@ subroutine vec_splat_testu8i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i8 15
+! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
 ! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
@@ -337,8 +337,8 @@ subroutine vec_splat_testu8i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i16 15
+! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
 ! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
@@ -350,8 +350,8 @@ subroutine vec_splat_testu8i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <16 x i8>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i32 15
+! LLVMIR-LE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <16 x i8> %[[x]], i64 15
 ! LLVMIR: %[[ins:.*]] = insertelement <16 x i8> poison, i8 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <16 x i8> %[[ins]], <16 x i8> poison, <16 x i32> zeroinitializer
 ! LLVMIR: store <16 x i8> %[[y]], ptr %{{[0-9]}}, align 16
@@ -376,8 +376,8 @@ subroutine vec_splat_testu16i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i8 7
+! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 7
 ! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
@@ -389,8 +389,8 @@ subroutine vec_splat_testu16i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i16 7
+! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 7
 ! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
@@ -402,8 +402,8 @@ subroutine vec_splat_testu16i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <8 x i16>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i32 7
+! LLVMIR-LE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <8 x i16> %[[x]], i64 7
 ! LLVMIR: %[[ins:.*]] = insertelement <8 x i16> poison, i16 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <8 x i16> %[[ins]], <8 x i16> poison, <8 x i32> zeroinitializer
 ! LLVMIR: store <8 x i16> %[[y]], ptr %{{[0-9]}}, align 16
@@ -428,8 +428,8 @@ subroutine vec_splat_testu32i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i8 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
@@ -441,8 +441,8 @@ subroutine vec_splat_testu32i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i16 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
@@ -454,8 +454,8 @@ subroutine vec_splat_testu32i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <4 x i32>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i32 3
+! LLVMIR-LE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <4 x i32> %[[x]], i64 3
 ! LLVMIR: %[[ins:.*]] = insertelement <4 x i32> poison, i32 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <4 x i32> %[[ins]], <4 x i32> poison, <4 x i32> zeroinitializer
 ! LLVMIR: store <4 x i32> %[[y]], ptr %{{[0-9]}}, align 16
@@ -480,8 +480,8 @@ subroutine vec_splat_testu64i8(x)
   y = vec_splat(x, 0_1)
 
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i8 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i8 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
@@ -493,8 +493,8 @@ subroutine vec_splat_testu64i16(x)
   y = vec_splat(x, 0_2)
 
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i16 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i16 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16
@@ -506,8 +506,8 @@ subroutine vec_splat_testu64i32(x)
   y = vec_splat(x, 0_4)
 
 ! LLVMIR: %[[x:.*]] = load <2 x i64>, ptr %{{[0-9]}}, align 16
-! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i32 0
-! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i32 1
+! LLVMIR-LE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 0
+! LLVMIR-BE: %[[ele:.*]] = extractelement <2 x i64> %[[x]], i64 1
 ! LLVMIR: %[[ins:.*]] = insertelement <2 x i64> poison, i64 %[[ele]], i32 0
 ! LLVMIR: %[[y:.*]] = shufflevector <2 x i64> %[[ins]], <2 x i64> poison, <2 x i32> zeroinitializer
 ! LLVMIR: store <2 x i64> %[[y]], ptr %{{[0-9]}}, align 16


        

From flang-commits at lists.llvm.org  Sat Jul 19 04:17:07 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Sat, 19 Jul 2025 04:17:07 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687b7eb3.a70a0220.31f104.ed48@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 01/11] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 02/11] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 03/11] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 04/11] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 05/11] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 06/11] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 

>From 498d60cf39c237c654c0cfddbe444d63bc87bc6c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 09:57:18 +0200
Subject: [PATCH 07/11] CI test

---
 flang-rt/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 5ffa1bcc34a41..d987115c8e9e8 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,40 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
 enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
@@ -159,6 +193,7 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)

>From 1bb0144eb21c2b1c8e1e66029d5573d8f5619c4c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 14:24:44 +0200
Subject: [PATCH 08/11] Enforce REAL(16) support

---
 flang-rt/CMakeLists.txt      | 45 ++++-------------------
 llvm/runtimes/CMakeLists.txt |  8 ++++
 runtimes/CMakeLists.txt      | 71 ++++++++++++++++++------------------
 3 files changed, 51 insertions(+), 73 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d987115c8e9e8..bfbd0af0c31dc 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,41 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
@@ -79,6 +44,11 @@ include(CMakePushCheckState)
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
+# Fortran compiler not optional for building Flang-RT
+enable_language(Fortran)
+
+flang_module_fortran_enable()
+
 
 #################
 # Build Options #
@@ -193,7 +163,9 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+# Look for support of REAL(16), if not already defined via command line.
+# NOTE: Does not work with Flang and CMake < 3.24
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
@@ -206,7 +178,6 @@ check_fortran_source_compiles([[
 )
 cmake_pop_check_state()
 
-flang_module_fortran_enable()
 
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 2f2f6db16255e..2bf2b0ee8ed50 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -528,6 +528,14 @@ if(build_runtimes)
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
+  if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    # Ensure REAL(16) support in runtimes to be consistent with compiler
+    if (FLANG_RUNTIME_F128_MATH_LIB OR HAVE_LDBL_MANT_DIG_113)
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=TRUE")
+   else ()
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=FALSE")
+   endif ()
+  endif ()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 987d173a283be..51214a46f558e 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -94,6 +94,41 @@ include(CheckCXXCompilerFlag)
 include(ExtendPath)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang, if used.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
+
 # Determine whether we are in the runtimes/runtimes-bins directory of a
 # bootstrapping build.
 set(LLVM_TREE_AVAILABLE OFF)
@@ -334,42 +369,6 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
-      _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-
-    enable_language(Fortran)
-  endif ()
-endif ()
-
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 8ec6d03939c3a7e2f79bb3ea5911b7607028dd7c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 21:52:02 +0200
Subject: [PATCH 09/11] Also add dependency barrier for WIN32

---
 flang-rt/lib/runtime/CMakeLists.txt | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 1b8114c102205..20f5d84bb2b69 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -281,14 +281,18 @@ else()
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
 
-    add_flangrt_library(${name}.intrinsics OBJECT
+    add_flangrt_library(${name}.intrinsics.obj OBJECT
       ${intrinsics_sources}
     )
+    add_custom_target(${name}.intrinsics
+      COMMENT "Intrinsic module dependency barrier"
+    )
+    add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
 
     add_flangrt_library(${name} ${libtype}
-        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics.obj>
         ${ARGN}
-        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -311,8 +315,9 @@ else()
     endif ()
 
     get_target_property(compile_target ${name}.compile ALIASED_TARGET)
-    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
+    add_dependencies(${compile_target} ${name}.intrinsics)
     add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")

>From 580a0c56535a9cd1c65fd5a5e6309b73c36ae8b6 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:05:34 +0200
Subject: [PATCH 10/11] Dependency barrier test

---
 flang-rt/lib/runtime/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 20f5d84bb2b69..bdd4318832473 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,6 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
+      COMMAND echo "Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
@@ -295,6 +296,7 @@ else()
         LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
 
     if (msvc_lib)
       set_target_properties(${name}
@@ -311,14 +313,13 @@ else()
       set(is_public "")
     else ()
       set(is_public PUBLIC)
+      add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
       set(_has_public_intrinsics "YES" PARENT_SCOPE)
     endif ()
 
-    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
     flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
     add_dependencies(${compile_target} ${name}.intrinsics)
-    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")

>From 979691a5bba4888be8c7c82d1bed4e8cdc71fff9 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:06:40 +0200
Subject: [PATCH 11/11] Dependency barrier info

---
 flang-rt/lib/runtime/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index bdd4318832473..24b84b11f9513 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,7 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
-      COMMAND echo "Dependency barrier"
+      COMMAND echo "${name} Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)


From flang-commits at lists.llvm.org  Sat Jul 19 19:13:05 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Sat, 19 Jul 2025 19:13:05 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (PR #149652)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149652@github.com>
Message-ID: <687c50b1.170a0220.342198.34f4@mx.google.com>


https://github.com/koachan updated https://github.com/llvm/llvm-project/pull/149652

>From 84da1b197113869b684f87bf580b37122d6a15e3 Mon Sep 17 00:00:00 2001
From: Koakuma <koachan at protonmail.com>
Date: Sat, 19 Jul 2025 05:40:56 +0700
Subject: [PATCH 1/2] [SPARC][Driver] Move feature mode selection to
 Arch/Sparc.cpp

This is so that it's performed globally and not just for clang.
---
 clang/lib/Driver/ToolChains/Arch/Sparc.cpp | 11 ++++++++++-
 clang/lib/Driver/ToolChains/Arch/Sparc.h   |  3 ++-
 clang/lib/Driver/ToolChains/Clang.cpp      | 10 ----------
 clang/lib/Driver/ToolChains/CommonArgs.cpp |  2 +-
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
index 9595ee8383c85..504f110eac87c 100644
--- a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
@@ -130,7 +130,8 @@ std::string sparc::getSparcTargetCPU(const Driver &D, const ArgList &Args,
   return "";
 }
 
-void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args,
+void sparc::getSparcTargetFeatures(const Driver &D, const llvm::Triple &Triple,
+                                   const ArgList &Args,
                                    std::vector<StringRef> &Features) {
   sparc::FloatABI FloatABI = sparc::getSparcFloatABI(D, Args);
   if (FloatABI == sparc::FloatABI::Soft)
@@ -150,11 +151,19 @@ void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args,
       Features.push_back("-popc");
   }
 
+  // Those OSes default to enabling VIS on 64-bit SPARC.
+  // See also the corresponding code for external assemblers in
+  // sparc::getSparcAsmModeForCPU().
+  bool IsSparcV9ATarget =
+      (Triple.getArch() == llvm::Triple::sparcv9) &&
+      (Triple.isOSLinux() || Triple.isOSFreeBSD() || Triple.isOSOpenBSD());
   if (Arg *A = Args.getLastArg(options::OPT_mvis, options::OPT_mno_vis)) {
     if (A->getOption().matches(options::OPT_mvis))
       Features.push_back("+vis");
     else
       Features.push_back("-vis");
+  } else if (IsSparcV9ATarget) {
+    Features.push_back("+vis");
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_mvis2, options::OPT_mno_vis2)) {
diff --git a/clang/lib/Driver/ToolChains/Arch/Sparc.h b/clang/lib/Driver/ToolChains/Arch/Sparc.h
index 2b178d9df1ee3..fa25b4992cc8b 100644
--- a/clang/lib/Driver/ToolChains/Arch/Sparc.h
+++ b/clang/lib/Driver/ToolChains/Arch/Sparc.h
@@ -31,7 +31,8 @@ FloatABI getSparcFloatABI(const Driver &D, const llvm::opt::ArgList &Args);
 std::string getSparcTargetCPU(const Driver &D, const llvm::opt::ArgList &Args,
                               const llvm::Triple &Triple);
 
-void getSparcTargetFeatures(const Driver &D, const llvm::opt::ArgList &Args,
+void getSparcTargetFeatures(const Driver &D, const llvm::Triple &Triple,
+                            const llvm::opt::ArgList &Args,
                             std::vector<llvm::StringRef> &Features);
 const char *getSparcAsmModeForCPU(llvm::StringRef Name,
                                   const llvm::Triple &Triple);
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 456bfe885f354..bf5f427fbfbb4 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2731,16 +2731,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
     CmdArgs.push_back(MipsTargetFeature);
   }
 
-  // Those OSes default to enabling VIS on 64-bit SPARC.
-  // See also the corresponding code for external assemblers in
-  // sparc::getSparcAsmModeForCPU().
-  bool IsSparcV9ATarget =
-      (C.getDefaultToolChain().getArch() == llvm::Triple::sparcv9) &&
-      (Triple.isOSLinux() || Triple.isOSFreeBSD() || Triple.isOSOpenBSD());
-  if (IsSparcV9ATarget && SparcTargetFeatures.empty()) {
-    CmdArgs.push_back("-target-feature");
-    CmdArgs.push_back("+vis");
-  }
   for (const char *Feature : SparcTargetFeatures) {
     CmdArgs.push_back("-target-feature");
     CmdArgs.push_back(Feature);
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 651a39c03bb54..826e2ea7eb06d 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -856,7 +856,7 @@ void tools::getTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   case llvm::Triple::sparc:
   case llvm::Triple::sparcel:
   case llvm::Triple::sparcv9:
-    sparc::getSparcTargetFeatures(D, Args, Features);
+    sparc::getSparcTargetFeatures(D, Triple, Args, Features);
     break;
   case llvm::Triple::r600:
   case llvm::Triple::amdgcn:

>From 31ce96bde166a118016dc6f1d8068d0d4fdd9afd Mon Sep 17 00:00:00 2001
From: Koakuma <koachan at protonmail.com>
Date: Sun, 20 Jul 2025 09:06:41 +0700
Subject: [PATCH 2/2] Add flang tests

---
 clang/lib/Driver/ToolChains/Flang.cpp     | 1 +
 flang/test/Driver/target-cpu-features.f90 | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..7ab41e9b85a04 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -447,6 +447,7 @@ void Flang::addTargetOptions(const ArgList &Args,
   // Add the target features.
   switch (TC.getArch()) {
   default:
+    getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false);
     break;
   case llvm::Triple::aarch64:
     getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false);
diff --git a/flang/test/Driver/target-cpu-features.f90 b/flang/test/Driver/target-cpu-features.f90
index 5a3fd0d838002..e7da964184c85 100644
--- a/flang/test/Driver/target-cpu-features.f90
+++ b/flang/test/Driver/target-cpu-features.f90
@@ -44,6 +44,10 @@
 ! RUN: %flang --target=loongarch64-linux-gnu -c %s -### 2>&1 \
 ! RUN: | FileCheck %s -check-prefix=CHECK-LOONGARCH64
 
+! RUN: %flang --target=sparc64-linux-gnu -c -### %s 2>&1  | FileCheck %s -check-prefix=CHECK-SPARC-VIS
+! RUN: %flang --target=sparc64-freebsd -c -### %s 2>&1  | FileCheck %s -check-prefix=CHECK-SPARC-VIS
+! RUN: %flang --target=sparc64-openbsd -c -### %s 2>&1  | FileCheck %s -check-prefix=CHECK-SPARC-VIS
+
 ! CHECK-A57: "-fc1" "-triple" "aarch64-unknown-linux-gnu"
 ! CHECK-A57-SAME: "-target-cpu" "cortex-a57"
 ! CHECK-A57-SAME: "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2
@@ -92,3 +96,6 @@
 
 ! CHECK-LOONGARCH64: "-fc1" "-triple" "loongarch64-unknown-linux-gnu"
 ! CHECK-LOONGARCH64-SAME: "-target-cpu" "loongarch64" "-target-feature" "+lsx" "-target-feature" "+64bit" "-target-feature" "+f" "-target-feature" "+d" "-target-feature" "+ual"
+
+! CHECK-SPARC-VIS: "-fc1" "-triple" "sparc64-{{[^"]+}}"
+! CHECK-SPARC-VIS-SAME: "-target-feature" "+vis"


From flang-commits at lists.llvm.org  Sat Jul 19 21:41:19 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Sat, 19 Jul 2025 21:41:19 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (PR #149652)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149652@github.com>
Message-ID: <687c736f.630a0220.248baf.12c2@mx.google.com>


https://github.com/koachan edited https://github.com/llvm/llvm-project/pull/149652

From flang-commits at lists.llvm.org  Sun Jul 20 02:52:59 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Sun, 20 Jul 2025 02:52:59 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][Flang][NFC] Replace use of
 `vector.insertelement/extractelement` (PR #143272)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/143272@github.com>
Message-ID: <687cbc7b.050a0220.37a1d8.17db@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64-flang-aix` running on `ppc64-flang-aix-test` while building `flang` at step 6 "test-build-unified-tree-check-flang".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/201/builds/5467

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (test-build-unified-tree-check-flang) failure: test (failure)
******************** TEST 'Flang :: Lower/PowerPC/ppc-vec-extract-elem-order.f90' FAILED ********************
Exit Code: 1

Command Output (stderr):
--
/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/bin/flang -fc1 -flang-experimental-hlfir -emit-llvm /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90 -fno-ppc-native-vector-element-order -triple ppc64le-unknown-linux -o - | /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/bin/FileCheck --check-prefixes="LLVMIR" /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90 # RUN: at line 1
+ /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/bin/flang -fc1 -flang-experimental-hlfir -emit-llvm /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90 -fno-ppc-native-vector-element-order -triple ppc64le-unknown-linux -o -
+ /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/build/bin/FileCheck --check-prefixes=LLVMIR /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90
/home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90:30:11: error: LLVMIR: expected string not found in input
! LLVMIR: %[[r:.*]] = extractelement <2 x i64> %[[arg1]], i8 %[[sub]]
          ^
<stdin>:20:19: note: scanning from here
 %7 = sub i8 1, %6
                  ^
<stdin>:20:19: note: with "arg1" equal to "4"
 %7 = sub i8 1, %6
                  ^
<stdin>:20:19: note: with "sub" equal to "7"
 %7 = sub i8 1, %6
                  ^
<stdin>:22:2: note: possible intended match here
 %9 = extractelement <2 x i64> %4, i64 %8
 ^

Input file: <stdin>
Check file: /home/llvm/llvm-external-buildbots/workers/ppc64-flang-aix-test/ppc64-flang-aix-build/llvm-project/flang/test/Lower/PowerPC/ppc-vec-extract-elem-order.f90

-dump-input=help explains the following input dump.

Input was:
<<<<<<
            .
            .
            .
           15:  
           16: define void @vec_extract_testi8i1_(ptr %0, ptr %1, ptr %2) #0 { 
           17:  %4 = load <2 x i64>, ptr %0, align 16 
           18:  %5 = load i8, ptr %1, align 1 
           19:  %6 = urem i8 %5, 2 
           20:  %7 = sub i8 1, %6 
check:30'0                       X error: no match found
check:30'1                         with "arg1" equal to "4"
check:30'2                         with "sub" equal to "7"
           21:  %8 = zext i8 %7 to i64 
check:30'0     ~~~~~~~~~~~~~~~~~~~~~~~~
           22:  %9 = extractelement <2 x i64> %4, i64 %8 
check:30'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
check:30'3      ?                                         possible intended match
           23:  store i64 %9, ptr %2, align 8 
check:30'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
...

```

</details>

https://github.com/llvm/llvm-project/pull/143272

From flang-commits at lists.llvm.org  Sun Jul 20 03:02:01 2025
From: flang-commits at lists.llvm.org (Rainer Orth via flang-commits)
Date: Sun, 20 Jul 2025 03:02:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (PR #149652)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149652@github.com>
Message-ID: <687cbe99.630a0220.14445.0d07@mx.google.com>


rorth wrote:

I've now tested the (original version of) the patch on both `sparc64-unknown-linux-gnu` and `sparcv9-sun-solaris2.11`.  The Linux/sparc64 build failure is gone, no regressions on either target.

Btw., prompted by this I've discovered that GCC's `-mcpu=v9` default on Solaris/SPARC is way outdated: the Oracle Studio compilers default to `-xarch=sparcvis2` since version 12.6 (from 2017 no less), which is `-mcpu=ultrasparc3` in `gcc`/`clang` lingo.  The 32 and 64-bit `libc.so.1` match this.  I'll update the `gcc` default after testing, and provide a similar patch for `clang` afterwards.  Thanks for pointing this out!

https://github.com/llvm/llvm-project/pull/149652

From flang-commits at lists.llvm.org  Sun Jul 20 12:50:23 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Sun, 20 Jul 2025 12:50:23 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687d487f.170a0220.35d9d8.2fbf@mx.google.com>


fhahn wrote:

/cherry-pick 004c67ea257039e4e98abc26dd4ac6e8f3d7a171

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Sun Jul 20 12:56:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Sun, 20 Jul 2025 12:56:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687d49d1.170a0220.3150c3.34b0@mx.google.com>


llvmbot wrote:

<!--IGNORE-->
>/cherry-pick 004c67ea257039e4e98abc26dd4ac6e8f3d7a171

Error: Command failed due to missing milestone.

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Sun Jul 20 12:59:50 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Sun, 20 Jul 2025 12:59:50 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687d4ab6.170a0220.2ab000.317d@mx.google.com>


https://github.com/fhahn milestoned https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Sun Jul 20 12:59:58 2025
From: flang-commits at lists.llvm.org (Florian Hahn via flang-commits)
Date: Sun, 20 Jul 2025 12:59:58 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687d4abe.a70a0220.3dc274.2d7f@mx.google.com>


fhahn wrote:

/cherry-pick https://github.com/llvm/llvm-project/commit/004c67ea257039e4e98abc26dd4ac6e8f3d7a171

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Sun Jul 20 13:05:56 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Sun, 20 Jul 2025 13:05:56 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm]
 [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148239@github.com>
Message-ID: <687d4c24.170a0220.1777a6.32c4@mx.google.com>


llvmbot wrote:

/pull-request llvm/llvm-project#149736

https://github.com/llvm/llvm-project/pull/148239

From flang-commits at lists.llvm.org  Sun Jul 20 14:16:26 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Sun, 20 Jul 2025 14:16:26 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Control alignment of constant
	folded reals (PR #149381)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149381@github.com>
Message-ID: <687d5caa.170a0220.a8279.31fc@mx.google.com>


https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/149381

>From a155a355cc1cef2169c37d069ad72b4c180f0633 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 14:56:11 -0400
Subject: [PATCH 1/3] [flang] Control alignment of constant folded reals

When REAL types are constant folded, the underneath implementation uses
arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the
resulting code for flang compiler ended up using SSE2 aligned load
instructions for REAL(16) constant folding on x86_64, and these
instructions require that the values are loaded from the aligned
addresses.
---
 flang/include/flang/Evaluate/integer.h | 1 +
 flang/include/flang/Evaluate/real.h    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index fccc2ad774a8f..5953fc81cb111 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,6 +74,7 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
+  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 03294881850a1..fa74792a0377f 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,7 +490,7 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  Word word_{}; // an Integer<>
+  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format

>From 43102f9b97cc055124b7e0e730cecf71fecdc29a Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 15:43:08 -0400
Subject: [PATCH 2/3] Added comment

---
 flang/include/flang/Evaluate/real.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index fa74792a0377f..af17adc8ad8a3 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,6 +490,9 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
+  // Require alignment, in case code generation code on x86_64 decides that
+  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 

>From 26432b55e826ea49030252b43714baf26afa36ac Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 17 Jul 2025 19:15:09 -0400
Subject: [PATCH 3/3] Comment update

---
 flang/include/flang/Evaluate/real.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index af17adc8ad8a3..76d25d9fe2670 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,8 +490,8 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  // Require alignment, in case code generation code on x86_64 decides that
-  // our Real object is suitable for SSE2 instructions and then gets surprised
+  // Require alignment, in case code generation on x86_64 decides that our
+  // Real object is suitable for SSE2 instructions and then gets surprised
   // by unaligned address.
   alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };


From flang-commits at lists.llvm.org  Sun Jul 20 18:28:00 2025
From: flang-commits at lists.llvm.org (Fangrui Song via flang-commits)
Date: Sun, 20 Jul 2025 18:28:00 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (PR #149652)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149652@github.com>
Message-ID: <687d97a0.050a0220.30231d.367d@mx.google.com>


https://github.com/MaskRay approved this pull request.


https://github.com/llvm/llvm-project/pull/149652

From flang-commits at lists.llvm.org  Sun Jul 20 18:33:09 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Sun, 20 Jul 2025 18:33:09 -0700 (PDT)
Subject: [flang-commits] [flang] 38fc453 - [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (#149652)
Message-ID: <687d98d5.170a0220.2f7960.40ef@mx.google.com>


Author: Koakuma
Date: 2025-07-20T21:33:06-04:00
New Revision: 38fc453afdb6a4511b7c8e189f12a92559ecc396

URL: https://github.com/llvm/llvm-project/commit/38fc453afdb6a4511b7c8e189f12a92559ecc396
DIFF: https://github.com/llvm/llvm-project/commit/38fc453afdb6a4511b7c8e189f12a92559ecc396.diff

LOG: [SPARC][Driver] Move feature mode selection to Arch/Sparc.cpp (#149652)

This is so that it's performed also for flang and not just for clang.

This should fix https://github.com/llvm/llvm-project/issues/138494.

Added: 
    

Modified: 
    clang/lib/Driver/ToolChains/Arch/Sparc.cpp
    clang/lib/Driver/ToolChains/Arch/Sparc.h
    clang/lib/Driver/ToolChains/Clang.cpp
    clang/lib/Driver/ToolChains/CommonArgs.cpp
    clang/lib/Driver/ToolChains/Flang.cpp
    flang/test/Driver/target-cpu-features.f90

Removed: 
    


################################################################################
diff  --git a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
index 9595ee8383c85..504f110eac87c 100644
--- a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
@@ -130,7 +130,8 @@ std::string sparc::getSparcTargetCPU(const Driver &D, const ArgList &Args,
   return "";
 }
 
-void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args,
+void sparc::getSparcTargetFeatures(const Driver &D, const llvm::Triple &Triple,
+                                   const ArgList &Args,
                                    std::vector<StringRef> &Features) {
   sparc::FloatABI FloatABI = sparc::getSparcFloatABI(D, Args);
   if (FloatABI == sparc::FloatABI::Soft)
@@ -150,11 +151,19 @@ void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args,
       Features.push_back("-popc");
   }
 
+  // Those OSes default to enabling VIS on 64-bit SPARC.
+  // See also the corresponding code for external assemblers in
+  // sparc::getSparcAsmModeForCPU().
+  bool IsSparcV9ATarget =
+      (Triple.getArch() == llvm::Triple::sparcv9) &&
+      (Triple.isOSLinux() || Triple.isOSFreeBSD() || Triple.isOSOpenBSD());
   if (Arg *A = Args.getLastArg(options::OPT_mvis, options::OPT_mno_vis)) {
     if (A->getOption().matches(options::OPT_mvis))
       Features.push_back("+vis");
     else
       Features.push_back("-vis");
+  } else if (IsSparcV9ATarget) {
+    Features.push_back("+vis");
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_mvis2, options::OPT_mno_vis2)) {

diff  --git a/clang/lib/Driver/ToolChains/Arch/Sparc.h b/clang/lib/Driver/ToolChains/Arch/Sparc.h
index 2b178d9df1ee3..fa25b4992cc8b 100644
--- a/clang/lib/Driver/ToolChains/Arch/Sparc.h
+++ b/clang/lib/Driver/ToolChains/Arch/Sparc.h
@@ -31,7 +31,8 @@ FloatABI getSparcFloatABI(const Driver &D, const llvm::opt::ArgList &Args);
 std::string getSparcTargetCPU(const Driver &D, const llvm::opt::ArgList &Args,
                               const llvm::Triple &Triple);
 
-void getSparcTargetFeatures(const Driver &D, const llvm::opt::ArgList &Args,
+void getSparcTargetFeatures(const Driver &D, const llvm::Triple &Triple,
+                            const llvm::opt::ArgList &Args,
                             std::vector<llvm::StringRef> &Features);
 const char *getSparcAsmModeForCPU(llvm::StringRef Name,
                                   const llvm::Triple &Triple);

diff  --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 8880c9375143f..241ca6ed5d604 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2731,16 +2731,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
     CmdArgs.push_back(MipsTargetFeature);
   }
 
-  // Those OSes default to enabling VIS on 64-bit SPARC.
-  // See also the corresponding code for external assemblers in
-  // sparc::getSparcAsmModeForCPU().
-  bool IsSparcV9ATarget =
-      (C.getDefaultToolChain().getArch() == llvm::Triple::sparcv9) &&
-      (Triple.isOSLinux() || Triple.isOSFreeBSD() || Triple.isOSOpenBSD());
-  if (IsSparcV9ATarget && SparcTargetFeatures.empty()) {
-    CmdArgs.push_back("-target-feature");
-    CmdArgs.push_back("+vis");
-  }
   for (const char *Feature : SparcTargetFeatures) {
     CmdArgs.push_back("-target-feature");
     CmdArgs.push_back(Feature);

diff  --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 651a39c03bb54..826e2ea7eb06d 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -856,7 +856,7 @@ void tools::getTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   case llvm::Triple::sparc:
   case llvm::Triple::sparcel:
   case llvm::Triple::sparcv9:
-    sparc::getSparcTargetFeatures(D, Args, Features);
+    sparc::getSparcTargetFeatures(D, Triple, Args, Features);
     break;
   case llvm::Triple::r600:
   case llvm::Triple::amdgcn:

diff  --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..7ab41e9b85a04 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -447,6 +447,7 @@ void Flang::addTargetOptions(const ArgList &Args,
   // Add the target features.
   switch (TC.getArch()) {
   default:
+    getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false);
     break;
   case llvm::Triple::aarch64:
     getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false);

diff  --git a/flang/test/Driver/target-cpu-features.f90 b/flang/test/Driver/target-cpu-features.f90
index 5a3fd0d838002..e7da964184c85 100644
--- a/flang/test/Driver/target-cpu-features.f90
+++ b/flang/test/Driver/target-cpu-features.f90
@@ -44,6 +44,10 @@
 ! RUN: %flang --target=loongarch64-linux-gnu -c %s -### 2>&1 \
 ! RUN: | FileCheck %s -check-prefix=CHECK-LOONGARCH64
 
+! RUN: %flang --target=sparc64-linux-gnu -c -### %s 2>&1  | FileCheck %s -check-prefix=CHECK-SPARC-VIS
+! RUN: %flang --target=sparc64-freebsd -c -### %s 2>&1  | FileCheck %s -check-prefix=CHECK-SPARC-VIS
+! RUN: %flang --target=sparc64-openbsd -c -### %s 2>&1  | FileCheck %s -check-prefix=CHECK-SPARC-VIS
+
 ! CHECK-A57: "-fc1" "-triple" "aarch64-unknown-linux-gnu"
 ! CHECK-A57-SAME: "-target-cpu" "cortex-a57"
 ! CHECK-A57-SAME: "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2
@@ -92,3 +96,6 @@
 
 ! CHECK-LOONGARCH64: "-fc1" "-triple" "loongarch64-unknown-linux-gnu"
 ! CHECK-LOONGARCH64-SAME: "-target-cpu" "loongarch64" "-target-feature" "+lsx" "-target-feature" "+64bit" "-target-feature" "+f" "-target-feature" "+d" "-target-feature" "+ual"
+
+! CHECK-SPARC-VIS: "-fc1" "-triple" "sparc64-{{[^"]+}}"
+! CHECK-SPARC-VIS-SAME: "-target-feature" "+vis"


        

From flang-commits at lists.llvm.org  Sun Jul 20 18:33:13 2025
From: flang-commits at lists.llvm.org (Brad Smith via flang-commits)
Date: Sun, 20 Jul 2025 18:33:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (PR #149652)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149652@github.com>
Message-ID: <687d98d9.170a0220.15968b.3d74@mx.google.com>


https://github.com/brad0 closed https://github.com/llvm/llvm-project/pull/149652

From flang-commits at lists.llvm.org  Mon Jul 21 00:00:58 2025
From: flang-commits at lists.llvm.org (Rainer Orth via flang-commits)
Date: Mon, 21 Jul 2025 00:00:58 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (PR #149652)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149652@github.com>
Message-ID: <687de5aa.050a0220.2db721.ef14@mx.google.com>


https://github.com/rorth milestoned https://github.com/llvm/llvm-project/pull/149652

From flang-commits at lists.llvm.org  Mon Jul 21 00:01:15 2025
From: flang-commits at lists.llvm.org (Rainer Orth via flang-commits)
Date: Mon, 21 Jul 2025 00:01:15 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (PR #149652)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149652@github.com>
Message-ID: <687de5bb.170a0220.250438.20c9@mx.google.com>


rorth wrote:

/cherry-pick 38fc453afdb6a4511b7c8e189f12a92559ecc396

https://github.com/llvm/llvm-project/pull/149652

From flang-commits at lists.llvm.org  Mon Jul 21 00:01:22 2025
From: flang-commits at lists.llvm.org (Rainer Orth via flang-commits)
Date: Mon, 21 Jul 2025 00:01:22 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [SPARC][Driver] Move feature mode
 selection to Arch/Sparc.cpp (PR #149652)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149652@github.com>
Message-ID: <687de5c2.050a0220.fc8f.40a3@mx.google.com>


rorth wrote:

This needs to go to the `release/21.x` branch, too, to unbreak the Linux/sparc64 `flang` build.

https://github.com/llvm/llvm-project/pull/149652

From flang-commits at lists.llvm.org  Mon Jul 21 02:45:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 02:45:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][Driver] Enable
	FLANG_DEFAULT_LINKER (PR #149786)
Message-ID: <llvm.org/llvm/llvm-project/pull/149786@github.com>


https://github.com/parabola94 created https://github.com/llvm/llvm-project/pull/149786

The default linker could be changed by a CMake variable CLANG_DEFAULT_LINKER before. However, it also changes the default linker invoked by clang. This patch distincts the default linkers of clang and flang.

Note that FLANG_DEFAULT_LINKER will be the same as CLANG_DEFAULT_LINKER unless it is defined explicitly. That means this patch does not affect the current behavior.

Fixes #73153

>From c6d3516b04946c9529083b293ce640f14c1879fe Mon Sep 17 00:00:00 2001
From: parabola94 <heavybaby5000 at toki.waseda.jp>
Date: Mon, 21 Jul 2025 18:00:05 +0900
Subject: [PATCH 1/2] [clang][Driver] Add a new member for CLANG_DEFAULT_LINKER
 to clang::driver::Driver (NFC)

The default linker can be changed by a CMake variable CLANG_DEFAULT_LINKER,
but it is shared in all toolchains. This patch intends to resolve this.
---
 clang/include/clang/Driver/Driver.h     | 9 +++++++++
 clang/lib/Driver/Driver.cpp             | 4 ++--
 clang/lib/Driver/ToolChain.cpp          | 2 +-
 clang/lib/Driver/ToolChains/MSVC.cpp    | 4 ++--
 clang/lib/Driver/ToolChains/MinGW.cpp   | 2 +-
 clang/lib/Driver/ToolChains/Solaris.cpp | 6 +++---
 clang/lib/Driver/ToolChains/UEFI.cpp    | 4 ++--
 7 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..107b3d95dde42 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -337,6 +337,10 @@ class Driver {
   /// "clang" as it's first argument.
   const char *PrependArg;
 
+  /// The default value of -fuse-ld= option. An empty string means the default
+  /// system linker.
+  std::string PreferredLinker;
+
   /// Whether to check that input files exist when constructing compilation
   /// jobs.
   LLVM_PREFERRED_TYPE(bool)
@@ -450,6 +454,11 @@ class Driver {
     return ClangExecutable.c_str();
   }
 
+  StringRef getPreferredLinker() const { return PreferredLinker; }
+  void setPreferredLinker(std::string Value) {
+    PreferredLinker = std::move(Value);
+  }
+
   bool isSaveTempsEnabled() const { return SaveTemps != SaveTempsNone; }
   bool isSaveTempsObj() const { return SaveTemps == SaveTempsObj; }
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..46c8ffbc493d2 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -266,8 +266,8 @@ Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
       CCLogDiagnostics(false), CCGenDiagnostics(false),
       CCPrintProcessStats(false), CCPrintInternalStats(false),
       TargetTriple(TargetTriple), Saver(Alloc), PrependArg(nullptr),
-      CheckInputsExist(true), ProbePrecompiled(true),
-      SuppressMissingInputWarning(false) {
+      PreferredLinker(CLANG_DEFAULT_LINKER), CheckInputsExist(true),
+      ProbePrecompiled(true), SuppressMissingInputWarning(false) {
   // Provide a sane fallback if no VFS is specified.
   if (!this->VFS)
     this->VFS = llvm::vfs::getRealFileSystem();
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 481f575518b93..38616a3f97183 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1100,7 +1100,7 @@ std::string ToolChain::GetLinkerPath(bool *LinkerIsLLD) const {
   // Get -fuse-ld= first to prevent -Wunused-command-line-argument. -fuse-ld= is
   // considered as the linker flavor, e.g. "bfd", "gold", or "lld".
   const Arg* A = Args.getLastArg(options::OPT_fuse_ld_EQ);
-  StringRef UseLinker = A ? A->getValue() : CLANG_DEFAULT_LINKER;
+  StringRef UseLinker = A ? A->getValue() : getDriver().getPreferredLinker();
 
   // --ld-path= takes precedence over -fuse-ld= and specifies the executable
   // name. -B, COMPILER_PATH and PATH and consulted if the value does not
diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index 7d31eea603087..bb469ff095cd4 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -279,8 +279,8 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     AddRunTimeLibs(TC, TC.getDriver(), CmdArgs, Args);
   }
 
-  StringRef Linker =
-      Args.getLastArgValue(options::OPT_fuse_ld_EQ, CLANG_DEFAULT_LINKER);
+  StringRef Linker = Args.getLastArgValue(options::OPT_fuse_ld_EQ,
+                                          TC.getDriver().getPreferredLinker());
   if (Linker.empty())
     Linker = "link";
   // We need to translate 'lld' into 'lld-link'.
diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp
index b2e36ae6f97c3..6abd0c04ecc0e 100644
--- a/clang/lib/Driver/ToolChains/MinGW.cpp
+++ b/clang/lib/Driver/ToolChains/MinGW.cpp
@@ -548,7 +548,7 @@ toolchains::MinGW::MinGW(const Driver &D, const llvm::Triple &Triple,
     getFilePaths().push_back(Base + "lib");
 
   NativeLLVMSupport =
-      Args.getLastArgValue(options::OPT_fuse_ld_EQ, CLANG_DEFAULT_LINKER)
+      Args.getLastArgValue(options::OPT_fuse_ld_EQ, D.getPreferredLinker())
           .equals_insensitive("lld");
 }
 
diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp
index a3574e1b701e8..02aa59817449d 100644
--- a/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -39,7 +39,7 @@ void solaris::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
 bool solaris::isLinkerGnuLd(const ToolChain &TC, const ArgList &Args) {
   // Only used if targetting Solaris.
   const Arg *A = Args.getLastArg(options::OPT_fuse_ld_EQ);
-  StringRef UseLinker = A ? A->getValue() : CLANG_DEFAULT_LINKER;
+  StringRef UseLinker = A ? A->getValue() : TC.getDriver().getPreferredLinker();
   return UseLinker == "bfd" || UseLinker == "gld";
 }
 
@@ -52,7 +52,7 @@ static bool getPIE(const ArgList &Args, const ToolChain &TC) {
                       TC.isPIEDefault(Args));
 }
 
-// FIXME: Need to handle CLANG_DEFAULT_LINKER here?
+// FIXME: Need to handle PreferredLinker here?
 std::string solaris::Linker::getLinkerPath(const ArgList &Args) const {
   const ToolChain &ToolChain = getToolChain();
   if (const Arg *A = Args.getLastArg(options::OPT_fuse_ld_EQ)) {
@@ -345,7 +345,7 @@ SanitizerMask Solaris::getSupportedSanitizers() const {
 
 const char *Solaris::getDefaultLinker() const {
   // FIXME: Only handle Solaris ld and GNU ld here.
-  return llvm::StringSwitch<const char *>(CLANG_DEFAULT_LINKER)
+  return llvm::StringSwitch<const char *>(getDriver().getPreferredLinker())
       .Cases("bfd", "gld", "/usr/gnu/bin/ld")
       .Default("/usr/bin/ld");
 }
diff --git a/clang/lib/Driver/ToolChains/UEFI.cpp b/clang/lib/Driver/ToolChains/UEFI.cpp
index ac6668e6bdd5f..2b41173543477 100644
--- a/clang/lib/Driver/ToolChains/UEFI.cpp
+++ b/clang/lib/Driver/ToolChains/UEFI.cpp
@@ -83,8 +83,8 @@ void tools::uefi::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // This should ideally be handled by ToolChain::GetLinkerPath but we need
   // to special case some linker paths. In the case of lld, we need to
   // translate 'lld' into 'lld-link'.
-  StringRef Linker =
-      Args.getLastArgValue(options::OPT_fuse_ld_EQ, CLANG_DEFAULT_LINKER);
+  StringRef Linker = Args.getLastArgValue(options::OPT_fuse_ld_EQ,
+                                          TC.getDriver().getPreferredLinker());
   if (Linker.empty() || Linker == "lld")
     Linker = "lld-link";
 

>From 7a97f531134060f7b0480cdb2d980373834d7ea2 Mon Sep 17 00:00:00 2001
From: parabola94 <heavybaby5000 at toki.waseda.jp>
Date: Mon, 21 Jul 2025 18:29:59 +0900
Subject: [PATCH 2/2] [flang][Driver] Enable FLANG_DEFAULT_LINKER

The default linker could be changed by a CMake variable CLANG_DEFAULT_LINKER before.
However, it also changes the default linker invoked by clang.
This patch distincts the default linkers of clang and flang.

Note that FLANG_DEFAULT_LINKER will be the same as CLANG_DEFAULT_LINKER
unless it is defined explicitly.
That means this patch does not affect the current behavior.

Fixes #73153
---
 flang/CMakeLists.txt                      |  2 +-
 flang/include/flang/Config/config.h.cmake | 16 +++++++++-------
 flang/tools/flang-driver/driver.cpp       |  2 ++
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..6cbe6d4ed6a3e 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -317,7 +317,7 @@ if (NOT ENABLE_LINKER_BUILD_ID)
   set(ENABLE_LINKER_BUILD_ID OFF CACHE BOOL "pass --build-id to ld")
 endif()
 
-set(FLANG_DEFAULT_LINKER "" CACHE STRING
+set(FLANG_DEFAULT_LINKER ${CLANG_DEFAULT_LINKER} CACHE STRING
   "Default linker to use (linker name or absolute path, empty for platform default)")
 
 set(FLANG_DEFAULT_RTLIB "" CACHE STRING
diff --git a/flang/include/flang/Config/config.h.cmake b/flang/include/flang/Config/config.h.cmake
index fd34d3f403631..f7ff495880fa1 100644
--- a/flang/include/flang/Config/config.h.cmake
+++ b/flang/include/flang/Config/config.h.cmake
@@ -1,10 +1,10 @@
-#===-- include/flang/Config/config.h.cmake ---------------------------------===#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===------------------------------------------------------------------------===#
+//===-- include/flang/Config/config.h.cmake ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===------------------------------------------------------------------------===//
 
 /* This generated file is for internal use. Do not include it from headers. */
 
@@ -16,6 +16,8 @@
 
 #define FLANG_VERSION            "${FLANG_VERSION}"
 
+#define FLANG_DEFAULT_LINKER     "${FLANG_DEFAULT_LINKER}"
+
 #endif
 
 
diff --git a/flang/tools/flang-driver/driver.cpp b/flang/tools/flang-driver/driver.cpp
index 3a2dffc66428f..b1e02383cd33d 100644
--- a/flang/tools/flang-driver/driver.cpp
+++ b/flang/tools/flang-driver/driver.cpp
@@ -16,6 +16,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Driver/Driver.h"
+#include "flang/Config/config.h"
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Basic/Diagnostic.h"
@@ -138,6 +139,7 @@ int main(int argc, const char **argv) {
                                   llvm::sys::getDefaultTargetTriple(), diags,
                                   "flang LLVM compiler");
   theDriver.setTargetAndMode(targetandMode);
+  theDriver.setPreferredLinker(FLANG_DEFAULT_LINKER);
 #ifdef FLANG_RUNTIME_F128_MATH_LIB
   theDriver.setFlangF128MathLibrary(FLANG_RUNTIME_F128_MATH_LIB);
 #endif


From flang-commits at lists.llvm.org  Mon Jul 21 02:59:49 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 02:59:49 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][Driver] Enable
	FLANG_DEFAULT_LINKER (PR #149786)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149786@github.com>
Message-ID: <687e0f95.630a0220.10e853.3edb@mx.google.com>


https://github.com/parabola94 edited https://github.com/llvm/llvm-project/pull/149786

From flang-commits at lists.llvm.org  Mon Jul 21 04:27:34 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 04:27:34 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang] Implement !$omp unroll using
 omp.unroll_heuristic (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <687e2426.630a0220.2d452b.4280@mx.google.com>


Meinersbur wrote:

ping

https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Mon Jul 21 04:28:13 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 04:28:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][Driver] Enable
	FLANG_DEFAULT_LINKER (PR #149786)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149786@github.com>
Message-ID: <687e244d.170a0220.c8d5c.55e7@mx.google.com>


https://github.com/parabola94 edited https://github.com/llvm/llvm-project/pull/149786

From flang-commits at lists.llvm.org  Mon Jul 21 04:28:25 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 04:28:25 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][Driver] Enable
	FLANG_DEFAULT_LINKER (PR #149786)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149786@github.com>
Message-ID: <687e2459.a70a0220.3519fd.57d8@mx.google.com>


https://github.com/parabola94 edited https://github.com/llvm/llvm-project/pull/149786

From flang-commits at lists.llvm.org  Mon Jul 21 04:34:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 04:34:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][Driver] Enable
	FLANG_DEFAULT_LINKER (PR #149786)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149786@github.com>
Message-ID: <687e25a9.170a0220.e2d29.5bf3@mx.google.com>


https://github.com/parabola94 edited https://github.com/llvm/llvm-project/pull/149786

From flang-commits at lists.llvm.org  Mon Jul 21 05:45:10 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 21 Jul 2025 05:45:10 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e3656.170a0220.1acf46.6c25@mx.google.com>


================
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
----------------
tblah wrote:

What is this needed for?

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 05:45:10 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 21 Jul 2025 05:45:10 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e3656.170a0220.291ac4.4a87@mx.google.com>


================
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
----------------
tblah wrote:

Is there documentation for this extension somewhere? Maybe it should be in flang/docs/Extensions.md?

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 05:57:43 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 05:57:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e3947.050a0220.299ca0.6cb7@mx.google.com>


================
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
----------------
kparzysz wrote:

Ah..  The decision as to whether to rewrite or not is based on the OpenMP version.  We need SemanticsContext to get it, although in this PR it's unused.  The next PR in the stack uses it though.  Let me know if you'd like to have it removed from this one.

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 05:57:44 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 05:57:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e3948.170a0220.13105e.76f5@mx.google.com>


================
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
----------------
kparzysz wrote:

I'm not sure if there is any.  Maybe in clang somewhere.  IIRC it's to make the runtime keep a mapped value forever, but it don't remember where I saw it (likely in the source code, maybe as a comment...)

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 06:02:39 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 06:02:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e3a6f.170a0220.1483cf.8511@mx.google.com>


kparzysz wrote:

Stack:
- https://github.com/llvm/llvm-project/pull/149134 (this PR)
- https://github.com/llvm/llvm-project/pull/149137

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 06:25:20 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 21 Jul 2025 06:25:20 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][Driver] Enable
	FLANG_DEFAULT_LINKER (PR #149786)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149786@github.com>
Message-ID: <687e3fc0.a70a0220.31effe.a161@mx.google.com>


https://github.com/tarunprabhu approved this pull request.

Thanks. LGTM, but please wait for someone on the clang side to take a look as well.

https://github.com/llvm/llvm-project/pull/149786

From flang-commits at lists.llvm.org  Mon Jul 21 06:26:20 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 21 Jul 2025 06:26:20 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][Driver] Enable
	FLANG_DEFAULT_LINKER (PR #149786)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149786@github.com>
Message-ID: <687e3ffc.170a0220.364154.ac9a@mx.google.com>


https://github.com/tarunprabhu edited https://github.com/llvm/llvm-project/pull/149786

From flang-commits at lists.llvm.org  Mon Jul 21 06:27:46 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 06:27:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e4052.170a0220.2415e7.9dbe@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/149134

>From 0929234481b4d6d015381fda490c05bbeff1e22f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 08:04:17 -0500
Subject: [PATCH] [flang][OpenMP] Parse OpenMP 6.0 map modifiers

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers.
The old "map-type-modifier" is retained, but is no longer created in
parsing. It will remain to take advantage of the preexisting modifier
validation for older versions: when the OpenMP version is < 6.0,
the modifiers will be rewritten back as map-type-modifiers (or map-
type in case of "delete").

In this patch the modifiers will always be rewritten in the older
format to isolate these changes to parsing as much as possible.
---
 flang/include/flang/Parser/dump-parse-tree.h  |  14 ++
 flang/include/flang/Parser/parse-tree.h       | 138 +++++++++++++++---
 .../flang/Semantics/openmp-modifiers.h        |   7 +
 flang/lib/Parser/openmp-parsers.cpp           |  53 +++++--
 flang/lib/Parser/unparse.cpp                  |   7 +
 flang/lib/Semantics/canonicalize-omp.cpp      |  48 +++++-
 flang/lib/Semantics/canonicalize-omp.h        |   9 +-
 flang/lib/Semantics/openmp-modifiers.cpp      | 115 +++++++++++++++
 flang/lib/Semantics/resolve-directives.cpp    |   2 +
 flang/lib/Semantics/semantics.cpp             |   3 +-
 .../test/Parser/OpenMP/map-modifiers-v60.f90  | 113 ++++++++++++++
 11 files changed, 466 insertions(+), 43 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/map-modifiers-v60.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 32b6ca45609b6..23e35d106c077 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -529,6 +529,8 @@ class ParseTreeDumper {
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlwaysModifier)
+  NODE_ENUM(OmpAlwaysModifier, Value)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
@@ -546,6 +548,8 @@ class ParseTreeDumper {
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
   NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpCloseModifier)
+  NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -561,6 +565,8 @@ class ParseTreeDumper {
   NODE(parser, OmpDefaultmapClause)
   NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE(parser, OmpDeleteModifier)
+  NODE_ENUM(OmpDeleteModifier, Value)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
   NODE(parser, OmpTaskDependenceType)
@@ -628,6 +634,8 @@ class ParseTreeDumper {
   NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
   NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpPresentModifier)
+  NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
@@ -637,6 +645,10 @@ class ParseTreeDumper {
   NODE(parser, OmpInReductionClause)
   NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpRefModifier)
+  NODE_ENUM(OmpRefModifier, Value)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpInitializerProc)
@@ -673,6 +685,8 @@ class ParseTreeDumper {
   NODE(parser, OmpSectionsDirective)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpxHoldModifier)
+  NODE_ENUM(OmpxHoldModifier, Value)
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cc1d032f94d4a..92a449d2f9d49 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3756,6 +3756,19 @@ struct OmpAllocatorComplexModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// always-modifier ->
+//    ALWAYS                                        // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpAlwaysModifier {
+  ENUM_CLASS(Value, Always)
+  WRAPPER_CLASS_BOILERPLATE(OmpAlwaysModifier, Value);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3767,17 +3780,29 @@ struct OmpChunkModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpChunkModifier, Value);
 };
 
-// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
-// iterator-specifier ->
-//    [iterator-type] iterator-identifier
-//        = range-specification |                   // since 5.0
-//    [iterator-type ::] iterator-identifier
-//        = range-specification                     // since 5.2
-struct OmpIteratorSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
-  CharBlock source;
-  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+// close-modifier ->
+//    CLOSE                                         // since 5.0
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpCloseModifier {
+  ENUM_CLASS(Value, Close)
+  WRAPPER_CLASS_BOILERPLATE(OmpCloseModifier, Value);
+};
+
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// delete-modifier ->
+//    DELETE                                        // since 6.0
+//
+// Until 5.2, it was a part of map-type.
+struct OmpDeleteModifier {
+  ENUM_CLASS(Value, Delete)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeleteModifier, Value);
 };
 
 // Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289]
@@ -3867,6 +3892,19 @@ struct OmpInteropType {
   WRAPPER_CLASS_BOILERPLATE(OmpInteropType, Value);
 };
 
+// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+//
+// iterator-specifier ->
+//    [iterator-type] iterator-identifier
+//        = range-specification |                   // since 5.0
+//    [iterator-type ::] iterator-identifier
+//        = range-specification                     // since 5.2
+struct OmpIteratorSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
+  CharBlock source;
+  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+};
+
 // Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
 //
 // iterator-modifier ->
@@ -3901,21 +3939,28 @@ struct OmpMapper {
   WRAPPER_CLASS_BOILERPLATE(OmpMapper, Name);
 };
 
-// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
 // map-type ->
-//    ALLOC | DELETE | FROM | RELEASE | TO | TOFROM // since 4.5
+//    ALLOC | DELETE | RELEASE |                    // since 4.5, until 5.2
+//    FROM | TO | TOFROM |                          // since 4.5
+//    STORAGE                                       // since 6.0
+//
+// Since 6.0 DELETE is a separate delete-modifier.
 struct OmpMapType {
-  ENUM_CLASS(Value, Alloc, Delete, From, Release, To, Tofrom);
+  ENUM_CLASS(Value, Alloc, Delete, From, Release, Storage, To, Tofrom);
   WRAPPER_CLASS_BOILERPLATE(OmpMapType, Value);
 };
 
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-type-modifier ->
-//    ALWAYS |                                      // since 4.5
-//    CLOSE |                                       // since 5.0
-//    PRESENT                                       // since 5.1
+//    ALWAYS |                                      // since 4.5, until 5.2
+//    CLOSE |                                       // since 5.0, until 5.2
+//    PRESENT                                       // since 5.1, until 5.2
+// Since 6.0 the map-type-modifier has been split into individual modifiers.
+//
 struct OmpMapTypeModifier {
   ENUM_CLASS(Value, Always, Close, Present, Ompx_Hold)
   WRAPPER_CLASS_BOILERPLATE(OmpMapTypeModifier, Value);
@@ -3954,6 +3999,19 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// present-modifier ->
+//    PRESENT                                       // since 5.1
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpPresentModifier {
+  ENUM_CLASS(Value, Present)
+  WRAPPER_CLASS_BOILERPLATE(OmpPresentModifier, Value);
+};
+
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3963,6 +4021,26 @@ struct OmpReductionModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpReductionModifier, Value);
 };
 
+// Ref: [6.0:279-288]
+//
+// ref-modifier ->
+//    REF_PTEE | REF_PTR | REF_PTR_PTEE             // since 6.0
+//
+struct OmpRefModifier {
+  ENUM_CLASS(Value, Ref_Ptee, Ref_Ptr, Ref_Ptr_Ptee)
+  WRAPPER_CLASS_BOILERPLATE(OmpRefModifier, Value);
+};
+
+// Ref: [6.0:279-288]
+//
+// self-modifier ->
+//    SELF                                          // since 6.0
+//
+struct OmpSelfModifier {
+  ENUM_CLASS(Value, Self)
+  WRAPPER_CLASS_BOILERPLATE(OmpSelfModifier, Value);
+};
+
 // Ref: [5.2:117-120]
 //
 // step-complex-modifier ->
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
+//
+// ompx-hold-modifier ->
+//    OMPX_HOLD                                     // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpxHoldModifier {
+  ENUM_CLASS(Value, Ompx_Hold)
+  WRAPPER_CLASS_BOILERPLATE(OmpxHoldModifier, Value);
+};
+
 // context-selector
 using OmpContextSelector = traits::OmpContextSelectorSpecification;
 } // namespace modifier
@@ -4376,13 +4466,25 @@ struct OmpLinearClause {
 // map-clause ->
 //    MAP([modifier...:] locator-list)              // since 4.5
 // modifier ->
-//    map-type-modifier |                           // since 4.5
+//    map-type-modifier [replaced] |                // since 4.5, until 5.2
+//    always-modifier |                             // since 6.0
+//    close-modifier |                              // since 6.0
+//    delete-modifier |                             // since 6.0
+//    present-modifier |                            // since 6.0
+//    ref-modifier |                                // since 6.0
+//    self-modifier |                               // since 6.0
 //    mapper |                                      // since 5.0
 //    iterator |                                    // since 5.1
 //    map-type                                      // since 4.5
+//    ompx-hold-modifier |                          // since 6.0
+//
+// Since 6.0 the map-type-modifier has been split into individual modifiers,
+// and delete-modifier has been split from map-type.
 struct OmpMapClause {
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  MODIFIER_BOILERPLATE(OmpMapTypeModifier, OmpMapper, OmpIterator, OmpMapType);
+  MODIFIER_BOILERPLATE(OmpAlwaysModifier, OmpCloseModifier, OmpDeleteModifier,
+      OmpMapTypeModifier, OmpPresentModifier, OmpRefModifier, OmpSelfModifier,
+      OmpMapper, OmpIterator, OmpMapType, OmpxHoldModifier);
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 4fbd80f989e72..a9fe911ef8807 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -71,8 +71,11 @@ DECLARE_DESCRIPTOR(parser::OmpAlignment);
 DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
+DECLARE_DESCRIPTOR(parser::OmpAlwaysModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpCloseModifier);
 DECLARE_DESCRIPTOR(parser::OmpContextSelector);
+DECLARE_DESCRIPTOR(parser::OmpDeleteModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
@@ -88,12 +91,16 @@ DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
 DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
+DECLARE_DESCRIPTOR(parser::OmpPresentModifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
+DECLARE_DESCRIPTOR(parser::OmpRefModifier);
+DECLARE_DESCRIPTOR(parser::OmpSelfModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpVariableCategory);
+DECLARE_DESCRIPTOR(parser::OmpxHoldModifier);
 
 #undef DECLARE_DESCRIPTOR
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..d349d8ceb0bb5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -274,6 +274,10 @@ TYPE_PARSER( //
     construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}) ||
     construct<OmpTypeSpecifier>(Parser<TypeSpec>{}))
 
+// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
+TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
+    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
+
 TYPE_PARSER(construct<OmpReductionSpecifier>( //
     Parser<OmpReductionIdentifier>{},
     ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
@@ -442,9 +446,18 @@ TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
 
 TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
 
+TYPE_PARSER(construct<OmpAlwaysModifier>( //
+    "ALWAYS" >> pure(OmpAlwaysModifier::Value::Always)))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpCloseModifier>( //
+    "CLOSE" >> pure(OmpCloseModifier::Value::Close)))
+
+TYPE_PARSER(construct<OmpDeleteModifier>( //
+    "DELETE" >> pure(OmpDeleteModifier::Value::Delete)))
+
 TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
@@ -502,26 +515,16 @@ TYPE_PARSER(construct<OmpLinearModifier>( //
 TYPE_PARSER(construct<OmpMapper>( //
     "MAPPER"_tok >> parenthesized(Parser<ObjectName>{})))
 
-// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+// map-type -> ALLOC | DELETE | FROM | RELEASE | STORAGE | TO | TOFROM
 TYPE_PARSER(construct<OmpMapType>( //
     "ALLOC" >> pure(OmpMapType::Value::Alloc) ||
-    "DELETE" >> pure(OmpMapType::Value::Delete) ||
+    // Parse "DELETE" as OmpDeleteModifier
     "FROM" >> pure(OmpMapType::Value::From) ||
     "RELEASE" >> pure(OmpMapType::Value::Release) ||
+    "STORAGE" >> pure(OmpMapType::Value::Storage) ||
     "TO"_id >> pure(OmpMapType::Value::To) ||
     "TOFROM" >> pure(OmpMapType::Value::Tofrom)))
 
-// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
-TYPE_PARSER(construct<OmpMapTypeModifier>(
-    "ALWAYS" >> pure(OmpMapTypeModifier::Value::Always) ||
-    "CLOSE" >> pure(OmpMapTypeModifier::Value::Close) ||
-    "OMPX_HOLD" >> pure(OmpMapTypeModifier::Value::Ompx_Hold) ||
-    "PRESENT" >> pure(OmpMapTypeModifier::Value::Present)))
-
-// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
-TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
-    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
-
 TYPE_PARSER(construct<OmpOrderModifier>(
     "REPRODUCIBLE" >> pure(OmpOrderModifier::Value::Reproducible) ||
     "UNCONSTRAINED" >> pure(OmpOrderModifier::Value::Unconstrained)))
@@ -534,11 +537,22 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
 TYPE_PARSER(construct<OmpPrescriptiveness>(
     "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
 
+TYPE_PARSER(construct<OmpPresentModifier>( //
+    "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
     "DEFAULT" >> pure(OmpReductionModifier::Value::Default)))
 
+TYPE_PARSER(construct<OmpRefModifier>( //
+    "REF_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptee) ||
+    "REF_PTR"_id >> pure(OmpRefModifier::Value::Ref_Ptr) ||
+    "REF_PTR_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptr_Ptee)))
+
+TYPE_PARSER(construct<OmpSelfModifier>( //
+    "SELF" >> pure(OmpSelfModifier::Value::Self)))
+
 TYPE_PARSER(construct<OmpStepComplexModifier>( //
     "STEP" >> parenthesized(scalarIntExpr)))
 
@@ -559,6 +573,9 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "POINTER" >> pure(OmpVariableCategory::Value::Pointer) ||
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
+TYPE_PARSER(construct<OmpxHoldModifier>( //
+    "OMPX_HOLD" >> pure(OmpxHoldModifier::Value::Ompx_Hold)))
+
 // This could be auto-generated.
 TYPE_PARSER(
     sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
@@ -611,10 +628,16 @@ TYPE_PARSER(sourced(
     construct<OmpLinearClause::Modifier>(Parser<OmpStepSimpleModifier>{})))
 
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
-    sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
+    sourced(construct<OmpMapClause::Modifier>(Parser<OmpAlwaysModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpCloseModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpDeleteModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpPresentModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpRefModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpSelfModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpIterator>{}) ||
-        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{})))))
+        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpxHoldModifier>{})))))
 
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index fbe89c668fc13..8ed16905b5099 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -3007,8 +3007,15 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
+  WALK_NESTED_ENUM(OmpAlwaysModifier, Value)
+  WALK_NESTED_ENUM(OmpCloseModifier, Value)
+  WALK_NESTED_ENUM(OmpDeleteModifier, Value)
+  WALK_NESTED_ENUM(OmpPresentModifier, Value)
+  WALK_NESTED_ENUM(OmpRefModifier, Value)
+  WALK_NESTED_ENUM(OmpSelfModifier, Value)
   WALK_NESTED_ENUM(OmpTraitSelectorName, Value)
   WALK_NESTED_ENUM(OmpTraitSetSelectorName, Value)
+  WALK_NESTED_ENUM(OmpxHoldModifier, Value)
 
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index cf05d8463277f..46aaab19ded0a 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -88,6 +89,8 @@ class CanonicalizationOfOmp {
     CanonicalizeUtilityConstructs(spec);
   }
 
+  void Post(parser::OmpMapClause &map) { CanonicalizeMapModifiers(map); }
+
 private:
   template <typename T> T *GetConstructIf(parser::ExecutionPartConstruct &x) {
     if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index c45d6bbbf9062..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,11 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
-}
+class SemanticsContext;
+
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+} // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index c84e832ee52a1..336ce4beb24ba 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -140,6 +140,22 @@ OmpGetDescriptor<parser::OmpAllocatorSimpleModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlwaysModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"always-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   static const OmpModifierDescriptor desc{
@@ -156,6 +172,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpCloseModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"close-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   static const OmpModifierDescriptor desc{
@@ -173,6 +205,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDeleteModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"delete-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique, OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   static const OmpModifierDescriptor desc{
@@ -347,6 +396,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapType>() {
       /*props=*/
       {
           {45, {OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
       },
       /*clauses=*/
       {
@@ -367,6 +417,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapTypeModifier>() {
       /*clauses=*/
       {
           {45, {Clause::OMPC_map}},
+          {60, {}},
       },
   };
   return desc;
@@ -420,6 +471,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPrescriptiveness>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPresentModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"present-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpReductionIdentifier>() {
@@ -456,6 +523,38 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpReductionModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpRefModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ref-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpSelfModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"self-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpStepComplexModifier>() {
@@ -522,4 +621,20 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpVariableCategory>() {
   };
   return desc;
 }
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpxHoldModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ompx-hold-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 151f4ccae634e..0b860314b4a1f 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -734,6 +734,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
+      default:
+        break;
       }
     }
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..d0a5b200d0d8e 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -643,8 +643,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}
diff --git a/flang/test/Parser/OpenMP/map-modifiers-v60.f90 b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..bc80886780d46
--- /dev/null
+++ b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,113 @@
+!RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x
+  !$omp target map(always, close, delete, present, ompx_hold: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(ALWAYS, CLOSE, DELETE, PRESENT, OMPX_HOLD: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpAlwaysModifier -> Value = Always
+!PARSE-TREE: | | Modifier -> OmpCloseModifier -> Value = Close
+!PARSE-TREE: | | Modifier -> OmpDeleteModifier -> Value = Delete
+!PARSE-TREE: | | Modifier -> OmpPresentModifier -> Value = Present
+!PARSE-TREE: | | Modifier -> OmpxHoldModifier -> Value = Ompx_Hold
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f01(x)
+  integer :: x
+  !$omp target map(self, storage: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: !$OMP TARGET  MAP(SELF, STORAGE: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpSelfModifier -> Value = Self
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = Storage
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f03(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f04(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f04 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'


From flang-commits at lists.llvm.org  Mon Jul 21 06:30:46 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 21 Jul 2025 06:30:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `COSPI` (PR #149343)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149343@github.com>
Message-ID: <687e4106.050a0220.112965.a83c@mx.google.com>


tblah wrote:

> I have a minor question: Does flang have a utility similar to `update_test_checks.py`? Editing the test file manually is a bit tedious.

MLIR's test checks tool mostly works for FIR output, but it doesn't do a good job of editing fortran files in place as it is expecting to find MLIR https://github.com/llvm/llvm-project/blob/main/mlir/utils/generate-test-checks.py

https://github.com/llvm/llvm-project/pull/149343

From flang-commits at lists.llvm.org  Mon Jul 21 06:50:52 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 06:50:52 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e45bc.050a0220.203351.aefb@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/149134

>From 0929234481b4d6d015381fda490c05bbeff1e22f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 08:04:17 -0500
Subject: [PATCH 1/2] [flang][OpenMP] Parse OpenMP 6.0 map modifiers

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers.
The old "map-type-modifier" is retained, but is no longer created in
parsing. It will remain to take advantage of the preexisting modifier
validation for older versions: when the OpenMP version is < 6.0,
the modifiers will be rewritten back as map-type-modifiers (or map-
type in case of "delete").

In this patch the modifiers will always be rewritten in the older
format to isolate these changes to parsing as much as possible.
---
 flang/include/flang/Parser/dump-parse-tree.h  |  14 ++
 flang/include/flang/Parser/parse-tree.h       | 138 +++++++++++++++---
 .../flang/Semantics/openmp-modifiers.h        |   7 +
 flang/lib/Parser/openmp-parsers.cpp           |  53 +++++--
 flang/lib/Parser/unparse.cpp                  |   7 +
 flang/lib/Semantics/canonicalize-omp.cpp      |  48 +++++-
 flang/lib/Semantics/canonicalize-omp.h        |   9 +-
 flang/lib/Semantics/openmp-modifiers.cpp      | 115 +++++++++++++++
 flang/lib/Semantics/resolve-directives.cpp    |   2 +
 flang/lib/Semantics/semantics.cpp             |   3 +-
 .../test/Parser/OpenMP/map-modifiers-v60.f90  | 113 ++++++++++++++
 11 files changed, 466 insertions(+), 43 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/map-modifiers-v60.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 32b6ca45609b6..23e35d106c077 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -529,6 +529,8 @@ class ParseTreeDumper {
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlwaysModifier)
+  NODE_ENUM(OmpAlwaysModifier, Value)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
@@ -546,6 +548,8 @@ class ParseTreeDumper {
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
   NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpCloseModifier)
+  NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -561,6 +565,8 @@ class ParseTreeDumper {
   NODE(parser, OmpDefaultmapClause)
   NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE(parser, OmpDeleteModifier)
+  NODE_ENUM(OmpDeleteModifier, Value)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
   NODE(parser, OmpTaskDependenceType)
@@ -628,6 +634,8 @@ class ParseTreeDumper {
   NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
   NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpPresentModifier)
+  NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
@@ -637,6 +645,10 @@ class ParseTreeDumper {
   NODE(parser, OmpInReductionClause)
   NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpRefModifier)
+  NODE_ENUM(OmpRefModifier, Value)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpInitializerProc)
@@ -673,6 +685,8 @@ class ParseTreeDumper {
   NODE(parser, OmpSectionsDirective)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpxHoldModifier)
+  NODE_ENUM(OmpxHoldModifier, Value)
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cc1d032f94d4a..92a449d2f9d49 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3756,6 +3756,19 @@ struct OmpAllocatorComplexModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// always-modifier ->
+//    ALWAYS                                        // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpAlwaysModifier {
+  ENUM_CLASS(Value, Always)
+  WRAPPER_CLASS_BOILERPLATE(OmpAlwaysModifier, Value);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3767,17 +3780,29 @@ struct OmpChunkModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpChunkModifier, Value);
 };
 
-// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
-// iterator-specifier ->
-//    [iterator-type] iterator-identifier
-//        = range-specification |                   // since 5.0
-//    [iterator-type ::] iterator-identifier
-//        = range-specification                     // since 5.2
-struct OmpIteratorSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
-  CharBlock source;
-  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+// close-modifier ->
+//    CLOSE                                         // since 5.0
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpCloseModifier {
+  ENUM_CLASS(Value, Close)
+  WRAPPER_CLASS_BOILERPLATE(OmpCloseModifier, Value);
+};
+
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// delete-modifier ->
+//    DELETE                                        // since 6.0
+//
+// Until 5.2, it was a part of map-type.
+struct OmpDeleteModifier {
+  ENUM_CLASS(Value, Delete)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeleteModifier, Value);
 };
 
 // Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289]
@@ -3867,6 +3892,19 @@ struct OmpInteropType {
   WRAPPER_CLASS_BOILERPLATE(OmpInteropType, Value);
 };
 
+// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+//
+// iterator-specifier ->
+//    [iterator-type] iterator-identifier
+//        = range-specification |                   // since 5.0
+//    [iterator-type ::] iterator-identifier
+//        = range-specification                     // since 5.2
+struct OmpIteratorSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
+  CharBlock source;
+  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+};
+
 // Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
 //
 // iterator-modifier ->
@@ -3901,21 +3939,28 @@ struct OmpMapper {
   WRAPPER_CLASS_BOILERPLATE(OmpMapper, Name);
 };
 
-// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
 // map-type ->
-//    ALLOC | DELETE | FROM | RELEASE | TO | TOFROM // since 4.5
+//    ALLOC | DELETE | RELEASE |                    // since 4.5, until 5.2
+//    FROM | TO | TOFROM |                          // since 4.5
+//    STORAGE                                       // since 6.0
+//
+// Since 6.0 DELETE is a separate delete-modifier.
 struct OmpMapType {
-  ENUM_CLASS(Value, Alloc, Delete, From, Release, To, Tofrom);
+  ENUM_CLASS(Value, Alloc, Delete, From, Release, Storage, To, Tofrom);
   WRAPPER_CLASS_BOILERPLATE(OmpMapType, Value);
 };
 
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-type-modifier ->
-//    ALWAYS |                                      // since 4.5
-//    CLOSE |                                       // since 5.0
-//    PRESENT                                       // since 5.1
+//    ALWAYS |                                      // since 4.5, until 5.2
+//    CLOSE |                                       // since 5.0, until 5.2
+//    PRESENT                                       // since 5.1, until 5.2
+// Since 6.0 the map-type-modifier has been split into individual modifiers.
+//
 struct OmpMapTypeModifier {
   ENUM_CLASS(Value, Always, Close, Present, Ompx_Hold)
   WRAPPER_CLASS_BOILERPLATE(OmpMapTypeModifier, Value);
@@ -3954,6 +3999,19 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// present-modifier ->
+//    PRESENT                                       // since 5.1
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpPresentModifier {
+  ENUM_CLASS(Value, Present)
+  WRAPPER_CLASS_BOILERPLATE(OmpPresentModifier, Value);
+};
+
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3963,6 +4021,26 @@ struct OmpReductionModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpReductionModifier, Value);
 };
 
+// Ref: [6.0:279-288]
+//
+// ref-modifier ->
+//    REF_PTEE | REF_PTR | REF_PTR_PTEE             // since 6.0
+//
+struct OmpRefModifier {
+  ENUM_CLASS(Value, Ref_Ptee, Ref_Ptr, Ref_Ptr_Ptee)
+  WRAPPER_CLASS_BOILERPLATE(OmpRefModifier, Value);
+};
+
+// Ref: [6.0:279-288]
+//
+// self-modifier ->
+//    SELF                                          // since 6.0
+//
+struct OmpSelfModifier {
+  ENUM_CLASS(Value, Self)
+  WRAPPER_CLASS_BOILERPLATE(OmpSelfModifier, Value);
+};
+
 // Ref: [5.2:117-120]
 //
 // step-complex-modifier ->
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
+//
+// ompx-hold-modifier ->
+//    OMPX_HOLD                                     // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpxHoldModifier {
+  ENUM_CLASS(Value, Ompx_Hold)
+  WRAPPER_CLASS_BOILERPLATE(OmpxHoldModifier, Value);
+};
+
 // context-selector
 using OmpContextSelector = traits::OmpContextSelectorSpecification;
 } // namespace modifier
@@ -4376,13 +4466,25 @@ struct OmpLinearClause {
 // map-clause ->
 //    MAP([modifier...:] locator-list)              // since 4.5
 // modifier ->
-//    map-type-modifier |                           // since 4.5
+//    map-type-modifier [replaced] |                // since 4.5, until 5.2
+//    always-modifier |                             // since 6.0
+//    close-modifier |                              // since 6.0
+//    delete-modifier |                             // since 6.0
+//    present-modifier |                            // since 6.0
+//    ref-modifier |                                // since 6.0
+//    self-modifier |                               // since 6.0
 //    mapper |                                      // since 5.0
 //    iterator |                                    // since 5.1
 //    map-type                                      // since 4.5
+//    ompx-hold-modifier |                          // since 6.0
+//
+// Since 6.0 the map-type-modifier has been split into individual modifiers,
+// and delete-modifier has been split from map-type.
 struct OmpMapClause {
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  MODIFIER_BOILERPLATE(OmpMapTypeModifier, OmpMapper, OmpIterator, OmpMapType);
+  MODIFIER_BOILERPLATE(OmpAlwaysModifier, OmpCloseModifier, OmpDeleteModifier,
+      OmpMapTypeModifier, OmpPresentModifier, OmpRefModifier, OmpSelfModifier,
+      OmpMapper, OmpIterator, OmpMapType, OmpxHoldModifier);
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 4fbd80f989e72..a9fe911ef8807 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -71,8 +71,11 @@ DECLARE_DESCRIPTOR(parser::OmpAlignment);
 DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
+DECLARE_DESCRIPTOR(parser::OmpAlwaysModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpCloseModifier);
 DECLARE_DESCRIPTOR(parser::OmpContextSelector);
+DECLARE_DESCRIPTOR(parser::OmpDeleteModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
@@ -88,12 +91,16 @@ DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
 DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
+DECLARE_DESCRIPTOR(parser::OmpPresentModifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
+DECLARE_DESCRIPTOR(parser::OmpRefModifier);
+DECLARE_DESCRIPTOR(parser::OmpSelfModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpVariableCategory);
+DECLARE_DESCRIPTOR(parser::OmpxHoldModifier);
 
 #undef DECLARE_DESCRIPTOR
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..d349d8ceb0bb5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -274,6 +274,10 @@ TYPE_PARSER( //
     construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}) ||
     construct<OmpTypeSpecifier>(Parser<TypeSpec>{}))
 
+// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
+TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
+    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
+
 TYPE_PARSER(construct<OmpReductionSpecifier>( //
     Parser<OmpReductionIdentifier>{},
     ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
@@ -442,9 +446,18 @@ TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
 
 TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
 
+TYPE_PARSER(construct<OmpAlwaysModifier>( //
+    "ALWAYS" >> pure(OmpAlwaysModifier::Value::Always)))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpCloseModifier>( //
+    "CLOSE" >> pure(OmpCloseModifier::Value::Close)))
+
+TYPE_PARSER(construct<OmpDeleteModifier>( //
+    "DELETE" >> pure(OmpDeleteModifier::Value::Delete)))
+
 TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
@@ -502,26 +515,16 @@ TYPE_PARSER(construct<OmpLinearModifier>( //
 TYPE_PARSER(construct<OmpMapper>( //
     "MAPPER"_tok >> parenthesized(Parser<ObjectName>{})))
 
-// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+// map-type -> ALLOC | DELETE | FROM | RELEASE | STORAGE | TO | TOFROM
 TYPE_PARSER(construct<OmpMapType>( //
     "ALLOC" >> pure(OmpMapType::Value::Alloc) ||
-    "DELETE" >> pure(OmpMapType::Value::Delete) ||
+    // Parse "DELETE" as OmpDeleteModifier
     "FROM" >> pure(OmpMapType::Value::From) ||
     "RELEASE" >> pure(OmpMapType::Value::Release) ||
+    "STORAGE" >> pure(OmpMapType::Value::Storage) ||
     "TO"_id >> pure(OmpMapType::Value::To) ||
     "TOFROM" >> pure(OmpMapType::Value::Tofrom)))
 
-// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
-TYPE_PARSER(construct<OmpMapTypeModifier>(
-    "ALWAYS" >> pure(OmpMapTypeModifier::Value::Always) ||
-    "CLOSE" >> pure(OmpMapTypeModifier::Value::Close) ||
-    "OMPX_HOLD" >> pure(OmpMapTypeModifier::Value::Ompx_Hold) ||
-    "PRESENT" >> pure(OmpMapTypeModifier::Value::Present)))
-
-// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
-TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
-    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
-
 TYPE_PARSER(construct<OmpOrderModifier>(
     "REPRODUCIBLE" >> pure(OmpOrderModifier::Value::Reproducible) ||
     "UNCONSTRAINED" >> pure(OmpOrderModifier::Value::Unconstrained)))
@@ -534,11 +537,22 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
 TYPE_PARSER(construct<OmpPrescriptiveness>(
     "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
 
+TYPE_PARSER(construct<OmpPresentModifier>( //
+    "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
     "DEFAULT" >> pure(OmpReductionModifier::Value::Default)))
 
+TYPE_PARSER(construct<OmpRefModifier>( //
+    "REF_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptee) ||
+    "REF_PTR"_id >> pure(OmpRefModifier::Value::Ref_Ptr) ||
+    "REF_PTR_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptr_Ptee)))
+
+TYPE_PARSER(construct<OmpSelfModifier>( //
+    "SELF" >> pure(OmpSelfModifier::Value::Self)))
+
 TYPE_PARSER(construct<OmpStepComplexModifier>( //
     "STEP" >> parenthesized(scalarIntExpr)))
 
@@ -559,6 +573,9 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "POINTER" >> pure(OmpVariableCategory::Value::Pointer) ||
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
+TYPE_PARSER(construct<OmpxHoldModifier>( //
+    "OMPX_HOLD" >> pure(OmpxHoldModifier::Value::Ompx_Hold)))
+
 // This could be auto-generated.
 TYPE_PARSER(
     sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
@@ -611,10 +628,16 @@ TYPE_PARSER(sourced(
     construct<OmpLinearClause::Modifier>(Parser<OmpStepSimpleModifier>{})))
 
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
-    sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
+    sourced(construct<OmpMapClause::Modifier>(Parser<OmpAlwaysModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpCloseModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpDeleteModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpPresentModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpRefModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpSelfModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpIterator>{}) ||
-        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{})))))
+        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpxHoldModifier>{})))))
 
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index fbe89c668fc13..8ed16905b5099 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -3007,8 +3007,15 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
+  WALK_NESTED_ENUM(OmpAlwaysModifier, Value)
+  WALK_NESTED_ENUM(OmpCloseModifier, Value)
+  WALK_NESTED_ENUM(OmpDeleteModifier, Value)
+  WALK_NESTED_ENUM(OmpPresentModifier, Value)
+  WALK_NESTED_ENUM(OmpRefModifier, Value)
+  WALK_NESTED_ENUM(OmpSelfModifier, Value)
   WALK_NESTED_ENUM(OmpTraitSelectorName, Value)
   WALK_NESTED_ENUM(OmpTraitSetSelectorName, Value)
+  WALK_NESTED_ENUM(OmpxHoldModifier, Value)
 
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index cf05d8463277f..46aaab19ded0a 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -88,6 +89,8 @@ class CanonicalizationOfOmp {
     CanonicalizeUtilityConstructs(spec);
   }
 
+  void Post(parser::OmpMapClause &map) { CanonicalizeMapModifiers(map); }
+
 private:
   template <typename T> T *GetConstructIf(parser::ExecutionPartConstruct &x) {
     if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index c45d6bbbf9062..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,11 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
-}
+class SemanticsContext;
+
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+} // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index c84e832ee52a1..336ce4beb24ba 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -140,6 +140,22 @@ OmpGetDescriptor<parser::OmpAllocatorSimpleModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlwaysModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"always-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   static const OmpModifierDescriptor desc{
@@ -156,6 +172,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpCloseModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"close-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   static const OmpModifierDescriptor desc{
@@ -173,6 +205,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDeleteModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"delete-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique, OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   static const OmpModifierDescriptor desc{
@@ -347,6 +396,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapType>() {
       /*props=*/
       {
           {45, {OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
       },
       /*clauses=*/
       {
@@ -367,6 +417,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapTypeModifier>() {
       /*clauses=*/
       {
           {45, {Clause::OMPC_map}},
+          {60, {}},
       },
   };
   return desc;
@@ -420,6 +471,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPrescriptiveness>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPresentModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"present-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpReductionIdentifier>() {
@@ -456,6 +523,38 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpReductionModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpRefModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ref-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpSelfModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"self-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpStepComplexModifier>() {
@@ -522,4 +621,20 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpVariableCategory>() {
   };
   return desc;
 }
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpxHoldModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ompx-hold-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 151f4ccae634e..0b860314b4a1f 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -734,6 +734,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
+      default:
+        break;
       }
     }
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..d0a5b200d0d8e 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -643,8 +643,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}
diff --git a/flang/test/Parser/OpenMP/map-modifiers-v60.f90 b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..bc80886780d46
--- /dev/null
+++ b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,113 @@
+!RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x
+  !$omp target map(always, close, delete, present, ompx_hold: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(ALWAYS, CLOSE, DELETE, PRESENT, OMPX_HOLD: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpAlwaysModifier -> Value = Always
+!PARSE-TREE: | | Modifier -> OmpCloseModifier -> Value = Close
+!PARSE-TREE: | | Modifier -> OmpDeleteModifier -> Value = Delete
+!PARSE-TREE: | | Modifier -> OmpPresentModifier -> Value = Present
+!PARSE-TREE: | | Modifier -> OmpxHoldModifier -> Value = Ompx_Hold
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f01(x)
+  integer :: x
+  !$omp target map(self, storage: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: !$OMP TARGET  MAP(SELF, STORAGE: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpSelfModifier -> Value = Self
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = Storage
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f03(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f04(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f04 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'

>From faff0d32746640c34988a65552e277198748fcf2 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 08:49:18 -0500
Subject: [PATCH 2/2] Don't pass SemanticsContext to CanonicalizeOmp yet

---
 flang/lib/Semantics/canonicalize-omp.cpp | 10 ++++------
 flang/lib/Semantics/canonicalize-omp.h   |  5 +++--
 flang/lib/Semantics/semantics.cpp        |  3 ++-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 46aaab19ded0a..bba4d72a702e9 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,8 +27,7 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(SemanticsContext &context)
-      : context_{context}, messages_{context.messages()} {}
+  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -433,13 +432,12 @@ class CanonicalizationOfOmp {
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
-  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
-  CanonicalizationOfOmp omp{context};
+bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
+  CanonicalizationOfOmp omp{messages};
   Walk(program, omp);
-  return !context.messages().AnyFatalError();
+  return !messages.AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index 3251218da35ed..23350c522a775 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,12 +11,13 @@
 
 namespace Fortran::parser {
 struct Program;
-}
+class Messages;
+} // namespace Fortran::parser
 
 namespace Fortran::semantics {
 class SemanticsContext;
 
-bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
 } // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 6db11aaf56c2a..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -642,7 +642,8 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_.messages(), program_) &&
+      CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}


From flang-commits at lists.llvm.org  Mon Jul 21 06:51:17 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 06:51:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e45d5.a70a0220.2cb65b.bc39@mx.google.com>


================
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
----------------
kparzysz wrote:

Removed now

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 07:27:36 2025
From: flang-commits at lists.llvm.org (Kiran Chandramohan via flang-commits)
Date: Mon, 21 Jul 2025 07:27:36 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang] Add
	-fintrinsic-modules-path= alias (PR #149417)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149417@github.com>
Message-ID: <687e4e58.630a0220.18ddcf.b2cb@mx.google.com>


https://github.com/kiranchandramohan approved this pull request.

LG.

https://github.com/llvm/llvm-project/pull/149417

From flang-commits at lists.llvm.org  Mon Jul 21 07:30:07 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 07:30:07 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang] Add
	-fintrinsic-modules-path= alias (PR #149417)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149417@github.com>
Message-ID: <687e4eef.170a0220.137ab5.c978@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/149417

From flang-commits at lists.llvm.org  Mon Jul 21 07:33:25 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Mon, 21 Jul 2025 07:33:25 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang] Add
	-fintrinsic-modules-path= alias (PR #149417)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149417@github.com>
Message-ID: <687e4fb5.630a0220.3d7378.b854@mx.google.com>


https://github.com/tarunprabhu approved this pull request.

LGTM. Thanks.

https://github.com/llvm/llvm-project/pull/149417

From flang-commits at lists.llvm.org  Mon Jul 21 07:42:50 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 21 Jul 2025 07:42:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687e51ea.050a0220.203351.c949@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
tblah wrote:

Why not create the real constant with the argument type in the first place? This might matter for real(16).

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Mon Jul 21 07:50:13 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 21 Jul 2025 07:50:13 -0700 (PDT)
Subject: [flang-commits] [flang] [NFC][flang] Added engineering option for
 triaging local-alloc-tbaa. (PR #149587)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149587@github.com>
Message-ID: <687e53a5.050a0220.43c38.d8ef@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM. Thanks for upstreaming

https://github.com/llvm/llvm-project/pull/149587

From flang-commits at lists.llvm.org  Mon Jul 21 07:58:32 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Mon, 21 Jul 2025 07:58:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] intrinsic and external are mutually
	exclusive (PR #149830)
Message-ID: <llvm.org/llvm/llvm-project/pull/149830@github.com>


https://github.com/c8ef created https://github.com/llvm/llvm-project/pull/149830

None

>From 7ce2a13b4d9246f9733e55ab849d0ccd4fd67470 Mon Sep 17 00:00:00 2001
From: c8ef <c8ef at outlook.com>
Date: Mon, 21 Jul 2025 14:57:47 +0000
Subject: [PATCH] [flang] intrinsic and external are mutually exclusive

---
 flang/lib/Semantics/resolve-names.cpp              | 14 +-------------
 flang/test/Semantics/null-init.f90                 |  1 -
 .../test/Semantics/resolve-intrinsic-external.f90  |  9 +++++++++
 3 files changed, 10 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Semantics/resolve-intrinsic-external.f90

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index b3268605e7c0c..cb59b3f881272 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -2351,6 +2351,7 @@ bool AttrsVisitor::IsConflictingAttr(Attr attrName) {
       HaveAttrConflict(attrName, Attr::PASS, Attr::NOPASS) || // C781
       HaveAttrConflict(attrName, Attr::PURE, Attr::IMPURE) ||
       HaveAttrConflict(attrName, Attr::PUBLIC, Attr::PRIVATE) ||
+      HaveAttrConflict(attrName, Attr::EXTERNAL, Attr::INTRINSIC) || // C840
       HaveAttrConflict(attrName, Attr::RECURSIVE, Attr::NON_RECURSIVE);
 }
 bool AttrsVisitor::CheckAndSet(Attr attrName) {
@@ -3311,11 +3312,6 @@ bool ScopeHandler::CheckPossibleBadForwardRef(const Symbol &symbol) {
 void ScopeHandler::MakeExternal(Symbol &symbol) {
   if (!symbol.attrs().test(Attr::EXTERNAL)) {
     SetImplicitAttr(symbol, Attr::EXTERNAL);
-    if (symbol.attrs().test(Attr::INTRINSIC)) { // C840
-      Say(symbol.name(),
-          "Symbol '%s' cannot have both EXTERNAL and INTRINSIC attributes"_err_en_US,
-          symbol.name());
-    }
   }
 }
 
@@ -5606,10 +5602,6 @@ bool DeclarationVisitor::Pre(const parser::ExternalStmt &x) {
         SayWithDecl(
             name, *symbol, "EXTERNAL attribute not allowed on '%s'"_err_en_US);
       }
-    } else if (symbol->attrs().test(Attr::INTRINSIC)) { // C840
-      Say(symbol->name(),
-          "Symbol '%s' cannot have both INTRINSIC and EXTERNAL attributes"_err_en_US,
-          symbol->name());
     }
   }
   return false;
@@ -5637,10 +5629,6 @@ void DeclarationVisitor::DeclareIntrinsic(const parser::Name &name) {
   } else if (!ConvertToProcEntity(symbol, name.source)) {
     SayWithDecl(
         name, symbol, "INTRINSIC attribute not allowed on '%s'"_err_en_US);
-  } else if (symbol.attrs().test(Attr::EXTERNAL)) { // C840
-    Say(symbol.name(),
-        "Symbol '%s' cannot have both EXTERNAL and INTRINSIC attributes"_err_en_US,
-        symbol.name());
   } else {
     if (symbol.GetType()) {
       // These warnings are worded so that they should make sense in either
diff --git a/flang/test/Semantics/null-init.f90 b/flang/test/Semantics/null-init.f90
index d01ad75a75a11..3f81bd907eabc 100644
--- a/flang/test/Semantics/null-init.f90
+++ b/flang/test/Semantics/null-init.f90
@@ -30,7 +30,6 @@ module m5
 end module
 
 module m6
-  !ERROR: Symbol 'null' cannot have both INTRINSIC and EXTERNAL attributes
   integer, pointer :: p => null()
   external :: null
 end module
diff --git a/flang/test/Semantics/resolve-intrinsic-external.f90 b/flang/test/Semantics/resolve-intrinsic-external.f90
new file mode 100644
index 0000000000000..269ef37207c1f
--- /dev/null
+++ b/flang/test/Semantics/resolve-intrinsic-external.f90
@@ -0,0 +1,9 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+
+real function eval(x, n)
+  real :: x
+  integer :: n
+  !ERROR: Attributes 'EXTERNAL' and 'INTRINSIC' conflict with each other
+  real, external, intrinsic :: exp
+  eval = 1.0
+end function eval


From flang-commits at lists.llvm.org  Mon Jul 21 07:58:49 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Mon, 21 Jul 2025 07:58:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] intrinsic and external are mutually
	exclusive (PR #149830)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149830@github.com>
Message-ID: <687e55a9.170a0220.2c92d3.d272@mx.google.com>


c8ef wrote:

* **#149830** <a href="https://app.graphite.dev/github/pr/llvm/llvm-project/149830?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/llvm/llvm-project/149830?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->

https://github.com/llvm/llvm-project/pull/149830

From flang-commits at lists.llvm.org  Mon Jul 21 08:02:12 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Mon, 21 Jul 2025 08:02:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] intrinsic and external are mutually
	exclusive (PR #149830)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149830@github.com>
Message-ID: <687e5674.170a0220.19cf55.e2ff@mx.google.com>


https://github.com/c8ef closed https://github.com/llvm/llvm-project/pull/149830

From flang-commits at lists.llvm.org  Mon Jul 21 08:05:40 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 21 Jul 2025 08:05:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e5744.a70a0220.2cb65b.dbf0@mx.google.com>


================
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
----------------
tblah wrote:

If even you're not sure what it does, is this ever going to be implemented? Maybe we shouldn't support it in flang.

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 08:10:56 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Mon, 21 Jul 2025 08:10:56 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e5880.050a0220.53604.8a02@mx.google.com>


================
@@ -404,3 +408,73 @@ ModuleFlagAttr::verify(function_ref<InFlightDiagnostic()> emitError,
                         "supported for unknown key '"
                      << key << "'";
 }
+
+FailureOr<Attribute> TargetFeaturesAttr::query(DataLayoutEntryKey key) {
+  if (auto stringKey = dyn_cast<StringAttr>(key))
+    if (contains(stringKey))
+      return UnitAttr::get(getContext());
+  return failure();
+}
+
+//===----------------------------------------------------------------------===//
+// LLVM_TargetAttr
+//===----------------------------------------------------------------------===//
+
+FailureOr<llvm::TargetMachine *> TargetAttr::getTargetMachine() {
+  if (targetMachine.has_value()) {
+    llvm::TargetMachine *tm = targetMachine.value();
+    if (tm != nullptr)
+      return {tm};
+    return failure();
+  }
+  llvm::InitializeAllTargets();
+  llvm::InitializeAllTargetMCs();
+
+  std::string error;
+  const llvm::Target *target =
+      llvm::TargetRegistry::lookupTarget(getTriple(), error);
+  if (!error.empty()) {
+    LLVM_DEBUG({
+      llvm::dbgs() << "Failed to retrieve the target with: `" << error << "`\n";
+    });
+    targetMachine = {nullptr};
+    return failure();
+  }
+
+  targetMachine = {target->createTargetMachine(
----------------
rolfmorel wrote:

@ftynse, would be able to help with how best to deal with this, i.e. what the best way is to have a unique_ptr on an attribute (where the unique_ptr is essentially a cache for the result of a method call on the attribute)?

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 08:14:00 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Mon, 21 Jul 2025 08:14:00 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e5938.a70a0220.f6d72.d2a2@mx.google.com>


================
@@ -14,8 +14,10 @@
 #ifndef MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 #define MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
 
-#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Target/TargetMachine.h"
----------------
rolfmorel wrote:

Thanks @fabianmcg - this makes sense! Will incorporate this in my revision of the PR this week.

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 08:18:36 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 08:18:36 -0700 (PDT)
Subject: [flang-commits] [flang] ed2bfd1 - [Flang] Add
	-fintrinsic-modules-path= alias (#149417)
Message-ID: <687e5a4c.630a0220.265440.bf3f@mx.google.com>


Author: Michael Kruse
Date: 2025-07-21T17:18:33+02:00
New Revision: ed2bfd132509da679320a1d691af4a91192297d0

URL: https://github.com/llvm/llvm-project/commit/ed2bfd132509da679320a1d691af4a91192297d0
DIFF: https://github.com/llvm/llvm-project/commit/ed2bfd132509da679320a1d691af4a91192297d0.diff

LOG: [Flang] Add -fintrinsic-modules-path= alias (#149417)

Add the syntax `-fintrinsic-modules-path=<dir>` as an alias to the
existing option `-fintrinsic-modules-path <dir>`. gfortran also supports
both alternatives.

This is particularly useful with CMake which de-duplicates command-line
options. For instance,
`-fintrinsic-modules-path /path/A -fintrinsic-modules-path /path/B`
is de-duplicated to
`-fintrinsic-modules-path /path/A /path/B`
since it conisiders the second `-fintrinsic-modules-path`
"redundant". This can be avoided using the syntax
`-fintrinsic-modules-path=/path/A -fintrinsic-modules-path=/path/B`.

Added: 
    

Modified: 
    clang/include/clang/Driver/Options.td
    flang/test/Driver/intrinsic-module-path.f90

Removed: 
    


################################################################################
diff  --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8087979b3051a..e30c152cbce2e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -7130,6 +7130,8 @@ def fintrinsic_modules_path : Separate<["-"], "fintrinsic-modules-path">,  Group
   HelpText<"Specify where to find the compiled intrinsic modules">,
   DocBrief<[{This option specifies the location of pre-compiled intrinsic modules,
   if they are not in the default location expected by the compiler.}]>;
+def fintrinsic_modules_path_EQ : Joined<["-"], "fintrinsic-modules-path=">,
+  Group<f_Group>, Alias<fintrinsic_modules_path>;
 
 defm backslash : OptInFC1FFlag<"backslash", "Specify that backslash in string introduces an escape character">;
 defm xor_operator : OptInFC1FFlag<"xor-operator", "Enable .XOR. as a synonym of .NEQV.">;

diff  --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..615d8f9a1730a 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -8,6 +8,7 @@
 !-----------------------------------------
 ! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
 ! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path=%S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found


        

From flang-commits at lists.llvm.org  Mon Jul 21 08:18:39 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 08:18:39 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [Flang] Add
	-fintrinsic-modules-path= alias (PR #149417)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149417@github.com>
Message-ID: <687e5a4f.170a0220.1dadae.d8e7@mx.google.com>


https://github.com/Meinersbur closed https://github.com/llvm/llvm-project/pull/149417

From flang-commits at lists.llvm.org  Mon Jul 21 08:22:10 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Mon, 21 Jul 2025 08:22:10 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e5b22.630a0220.96932.c26f@mx.google.com>


rolfmorel wrote:

@fabianmcg thanks for pointing that out! That should resolve bullet 4 up above I believe.

Any other feedback on this PR would be much appreciated. In particular any help with addressing the remaining bullets:

> 1. As the `LLVMTargetFeaturesAttr` attribute already exists we have the following:
>    
>    1. There's a circular dependency between `LLVMInterfaces` and `LLVMAttrs` in that the new `TargetAttrInterface` requires `LLVMTargetFeaturesAttr` (for the `features` of a target) and there are attrs in `LLVMAttrs` that require interfaces from `LLVMInterfaces`. I break the dependency by moving `LLVMTargetFeaturesAttr` to its own file and having a dedicated CMake-rule to generate `LLVMOpsAttrDefs.h.inc` by way of `LLVMAttrAndEnumDefs.td`.
>       
>       1. Is there a better way of dealing with this circular dependency?
>    2. Currently, `#nvvm.target` and `#rocdl.target` take `features` as a `StringAttr`. Seems to me they should take `LLVMTargetFeaturesAttr` which would make them eligible for/compatible with implementing `TargetAttrInterface`. I have not spent time on transitioning them yet.
> 2. For some reason, the parser generated for `` let assemblyFormat = [{`<` `triple` `=` $triple (`,` `chip` `=` $chip^)? (`,` qualified($features)^)? `>`}]; `` does not actually make the `chip` clause optional. That is, it greedily assumes that as it saw `,` from the `chip` clause it is now committed to it. For example, `#llvm.target<triple = "x86_64-unknown-linux", #llvm.target_features<["+mmx", "+sse"]>>` yields an `expected 'chip'` parser error.
>    
>    1. What's the best solution here? Just give in and write a custom parser?
> 3. How do I properly hang a `unique_ptr<llvm::TargetMachine>` of off `TargetAttr`? The current bare pointer works but probably is not safe, e.g. in case the context gets deleted. See discussion with @rengolin above.
> 
> 4. ~Had to move the `DataLayoutImporter` from `lib/Target/LLVMIR` to `Dialect/LLVMIR/IR`, in support of `#llvm.data_layout` being a wrapper around the corresponding `#dlti.dl_spec`, which causes a linking dependency on the DLTI-dialect.~
> 
> 5. What is the strategy around verification? See discussion with @rengolin above.
> 6. Test cases should be made aware of which backends the compiled `llvm-project` has available and be skipped if not applicable. CI is happy for now, which is actually wrong.



https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 08:22:11 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Mon, 21 Jul 2025 08:22:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687e5b23.050a0220.2da145.d284@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
c8ef wrote:

```
mlir::Value factor = builder.createRealConstant(loc, args[0].getType(), pi);
```

Using the argument type in the creation process seems to result in the following error:

```
error: FloatAttr type doesn't match the type implied by its value
flang: /home/c8ef/llvm-dev/mlir/include/mlir/IR/StorageUniquerSupport.h:180: static ConcreteT mlir::detail::StorageUserBase<mlir::FloatAttr, mlir::Attribute, mlir::detail::FloatAttrStorage, mlir::detail::AttributeUniquer, mlir::TypedAttr::Trait>::get(MLIRContext *, Args &&...) [ConcreteT = mlir::FloatAttr, BaseT = mlir::Attribute, StorageT = mlir::detail::FloatAttrStorage, UniquerT = mlir::detail::AttributeUniquer, Traits = <mlir::TypedAttr::Trait>, Args = <mlir::Type &, const llvm::APFloat &>]: Assertion `succeeded( ConcreteT::verifyInvariants(getDefaultDiagnosticEmitFn(ctx), args...))' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
Stack dump:
0.	Program arguments: /home/c8ef/llvm-dev/build/bin/flang -fc1 -emit-fir /home/c8ef/llvm-dev/flang/test/Lower/Intrinsics/sinpi.f90 -o -
 #0 0x0000ffffa6de9e50 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) /home/c8ef/llvm-dev/llvm/lib/Support/Unix/Signals.inc:834:11
 #1 0x0000ffffa6dea368 PrintStackTraceSignalHandler(void*) /home/c8ef/llvm-dev/llvm/lib/Support/Unix/Signals.inc:918:1
 #2 0x0000ffffa6de83f4 llvm::sys::RunSignalHandlers() /home/c8ef/llvm-dev/llvm/lib/Support/Signals.cpp:104:5
 #3 0x0000ffffa6deab18 SignalHandler(int, siginfo_t*, void*) /home/c8ef/llvm-dev/llvm/lib/Support/Unix/Signals.inc:426:38
 #4 0x0000ffffaa031808 (linux-vdso.so.1+0x808)
 #5 0x0000ffffa64d7dc0 __pthread_kill_implementation ./nptl/pthread_kill.c:44:76
 #6 0x0000ffffa6486980 raise ./signal/../sysdeps/posix/raise.c:27:6
 #7 0x0000ffffa6471ac4 abort ./stdlib/abort.c:81:3
 #8 0x0000ffffa647f9bc __assert_fail_base ./assert/assert.c:53:15
 #9 0x0000ffff9fc572cc mlir::FloatAttr mlir::detail::StorageUserBase<mlir::FloatAttr, mlir::Attribute, mlir::detail::FloatAttrStorage, mlir::detail::AttributeUniquer, mlir::TypedAttr::Trait>::get<mlir::Type&, llvm::APFloat const&>(mlir::MLIRContext*, mlir::Type&, llvm::APFloat const&) /home/c8ef/llvm-dev/mlir/include/mlir/IR/StorageUniquerSupport.h:179:5
#10 0x0000ffff9fc4c8c8 mlir::FloatAttr::get(mlir::Type, llvm::APFloat const&) /home/c8ef/llvm-dev/build/tools/mlir/include/mlir/IR/BuiltinAttributes.cpp.inc:293:10
#11 0x0000ffff9fc3b3bc mlir::Builder::getFloatAttr(mlir::Type, llvm::APFloat const&) /home/c8ef/llvm-dev/mlir/lib/IR/Builders.cpp:254:10
#12 0x0000ffff944e0b84 fir::FirOpBuilder::createRealConstant(mlir::Location, mlir::Type, llvm::APFloat const&) /home/c8ef/llvm-dev/flang/lib/Optimizer/Builder/FIRBuilder.cpp:187:17
#13 0x0000ffff94567eac fir::IntrinsicLibrary::genSinpi(mlir::Type, llvm::ArrayRef<mlir::Value>) /home/c8ef/llvm-dev/flang/lib/Optimizer/Builder/IntrinsicCall.cpp:8072:32
```

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Mon Jul 21 08:22:41 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 08:22:41 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e5b41.630a0220.397d09.be9c@mx.google.com>


================
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
----------------
kparzysz wrote:

It's already implemented.  It's a pretty old modifier.

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 08:23:32 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 08:23:32 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e5b74.170a0220.1358a9.d277@mx.google.com>


https://github.com/kparzysz edited https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 08:19:40 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Mon, 21 Jul 2025 08:19:40 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [flang][acc] Lower do and do
 concurrent loops specially in acc regions (PR #149614)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149614@github.com>
Message-ID: <687e5a8c.170a0220.26e0d3.ea0e@mx.google.com>


https://github.com/razvanlupusoru updated https://github.com/llvm/llvm-project/pull/149614

>From 78affa2a398ce503de265dba50c6988406d21697 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Fri, 18 Jul 2025 16:05:43 -0700
Subject: [PATCH 1/2] [flang][acc] Lower do and do concurrent loops specially
 in acc regions

When OpenACC is enabled and Fortran loops are annotated with `acc loop`,
they are lowered to `acc.loop` operation. And rest of the contained
loops use the normal FIR lowering path.

Hovever, the OpenACC specification has special provisions related
to contained loops and their induction variable. In order to adhere to
this, we convert all valid contained loops to `acc.loop` in order to
store this information appropriately.

The provisions in the spec that motivated this change (line numbers
are from OpenACC 3.4):
- 1353 Loop variables in Fortran do statements within a compute
construct are predetermined to be private to the thread that executes
the loop.
- 3783 When do concurrent appears without a loop construct in a kernels
construct it is treated as if it is annotated with loop auto. If it
appears in a parallel construct or an accelerator routine then it is
treated as if it is annotated with loop independent.

By valid loops - we convert do loops and do concurrent loops which have
induction variable. Loops which are unstructured are not handled.
---
 flang/include/flang/Lower/OpenACC.h           |  22 +-
 flang/lib/Lower/Bridge.cpp                    |  33 +-
 flang/lib/Lower/OpenACC.cpp                   | 385 ++++++++++++------
 .../Lower/OpenACC/do-loops-to-acc-loops.f90   | 332 +++++++++++++++
 .../mlir/Dialect/OpenACC/OpenACCOps.td        |  65 +++
 5 files changed, 716 insertions(+), 121 deletions(-)
 create mode 100644 flang/test/Lower/OpenACC/do-loops-to-acc-loops.f90

diff --git a/flang/include/flang/Lower/OpenACC.h b/flang/include/flang/Lower/OpenACC.h
index af3451023e3df..8b13ce94f7bc4 100644
--- a/flang/include/flang/Lower/OpenACC.h
+++ b/flang/include/flang/Lower/OpenACC.h
@@ -43,6 +43,7 @@ struct ProcedureDesignator;
 
 namespace parser {
 struct AccClauseList;
+struct DoConstruct;
 struct OpenACCConstruct;
 struct OpenACCDeclarativeConstruct;
 struct OpenACCRoutineConstruct;
@@ -58,6 +59,7 @@ namespace lower {
 
 class AbstractConverter;
 class StatementContext;
+class SymMap;
 
 namespace pft {
 struct Evaluation;
@@ -114,14 +116,32 @@ void attachDeclarePostDeallocAction(AbstractConverter &, fir::FirOpBuilder &,
 void genOpenACCTerminator(fir::FirOpBuilder &, mlir::Operation *,
                           mlir::Location);
 
-int64_t getLoopCountForCollapseAndTile(const Fortran::parser::AccClauseList &);
+/// Used to obtain the number of contained loops to look for
+/// since this is dependent on number of tile operands and collapse
+/// clause.
+uint64_t getLoopCountForCollapseAndTile(const Fortran::parser::AccClauseList &);
 
+/// Checks whether the current insertion point is inside OpenACC loop.
 bool isInOpenACCLoop(fir::FirOpBuilder &);
 
+/// Checks whether the current insertion point is inside OpenACC compute construct.
+bool isInsideOpenACCComputeConstruct(fir::FirOpBuilder &);
+
 void setInsertionPointAfterOpenACCLoopIfInside(fir::FirOpBuilder &);
 
 void genEarlyReturnInOpenACCLoop(fir::FirOpBuilder &, mlir::Location);
 
+/// Generates an OpenACC loop from a do construct in order to
+/// properly capture the loop bounds, parallelism determination mode,
+/// and to privatize the loop variables.
+/// When the conversion is rejected, nullptr is returned.
+mlir::Operation *genOpenACCLoopFromDoConstruct(
+    AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semanticsContext,
+    Fortran::lower::SymMap &localSymbols,
+    const Fortran::parser::DoConstruct &doConstruct,
+    pft::Evaluation &eval);
+
 } // namespace lower
 } // namespace Fortran
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 5f0783f869bf6..4073b8623e333 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2164,10 +2164,35 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   ///  - structured and unstructured concurrent loops
   void genFIR(const Fortran::parser::DoConstruct &doConstruct) {
     setCurrentPositionAt(doConstruct);
-    // Collect loop nest information.
-    // Generate begin loop code directly for infinite and while loops.
     Fortran::lower::pft::Evaluation &eval = getEval();
     bool unstructuredContext = eval.lowerAsUnstructured();
+
+    // Loops with induction variables inside OpenACC compute constructs
+    // need special handling to ensure that the IVs are privatized.
+    if (Fortran::lower::isInsideOpenACCComputeConstruct(*builder)) {
+      mlir::Operation* loopOp = Fortran::lower::genOpenACCLoopFromDoConstruct(
+                         *this, bridge.getSemanticsContext(), localSymbols,
+                         doConstruct, eval);
+      bool success = loopOp != nullptr;
+      if (success) {
+        // Sanity check that the builder insertion point is inside the newly
+        // generated loop.
+        assert(
+            loopOp->getRegion(0).isAncestor(
+                builder->getInsertionPoint()->getBlock()->getParent()) &&
+            "builder insertion point is not inside the newly generated loop");
+
+        // Loop body code.
+        auto iter = eval.getNestedEvaluations().begin();
+        for (auto end = --eval.getNestedEvaluations().end(); iter != end; ++iter)
+          genFIR(*iter, unstructuredContext);
+        return;
+      }
+      // Fall back to normal loop handling.
+    }
+
+    // Collect loop nest information.
+    // Generate begin loop code directly for infinite and while loops.
     Fortran::lower::pft::Evaluation &doStmtEval =
         eval.getFirstNestedEvaluation();
     auto *doStmt = doStmtEval.getIf<Fortran::parser::NonLabelDoStmt>();
@@ -3121,7 +3146,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     Fortran::lower::pft::Evaluation *curEval = &getEval();
 
     if (accLoop || accCombined) {
-      int64_t loopCount;
+      uint64_t loopCount;
       if (accLoop) {
         const Fortran::parser::AccBeginLoopDirective &beginLoopDir =
             std::get<Fortran::parser::AccBeginLoopDirective>(accLoop->t);
@@ -3139,7 +3164,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
       if (curEval->lowerAsStructured()) {
         curEval = &curEval->getFirstNestedEvaluation();
-        for (int64_t i = 1; i < loopCount; i++)
+        for (uint64_t i = 1; i < loopCount; i++)
           curEval = &*std::next(curEval->getNestedEvaluations().begin());
       }
     }
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 51eb33dec186b..950b02501751a 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -35,6 +35,7 @@
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/Support/LLVM.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Frontend/OpenACC/ACC.h.inc"
 #include "llvm/Support/CommandLine.h"
@@ -2138,6 +2139,168 @@ static void determineDefaultLoopParMode(
   }
 }
 
+// Extract loop bounds, steps, induction variables, and privatization info
+// for both DO CONCURRENT and regular do loops
+static void processDoLoopBounds(
+    Fortran::lower::AbstractConverter &converter,
+    mlir::Location currentLocation, Fortran::lower::StatementContext &stmtCtx,
+    fir::FirOpBuilder &builder,
+    const Fortran::parser::DoConstruct &outerDoConstruct,
+    Fortran::lower::pft::Evaluation &eval,
+    llvm::SmallVector<mlir::Value> &lowerbounds,
+    llvm::SmallVector<mlir::Value> &upperbounds,
+    llvm::SmallVector<mlir::Value> &steps,
+    llvm::SmallVector<mlir::Value> &privateOperands,
+    llvm::SmallVector<mlir::Value> &ivPrivate,
+    llvm::SmallVector<mlir::Attribute> &privatizationRecipes,
+    llvm::SmallVector<mlir::Type> &ivTypes,
+    llvm::SmallVector<mlir::Location> &ivLocs,
+    llvm::SmallVector<bool> &inclusiveBounds,
+    llvm::SmallVector<mlir::Location> &locs, uint64_t loopsToProcess) {
+  assert(loopsToProcess > 0 && "expect at least one loop");
+  locs.push_back(currentLocation); // Location of the directive
+  Fortran::lower::pft::Evaluation *crtEval = &eval.getFirstNestedEvaluation();
+  bool isDoConcurrent = outerDoConstruct.IsDoConcurrent();
+
+  if (isDoConcurrent) {
+    locs.push_back(converter.genLocation(
+        Fortran::parser::FindSourceLocation(outerDoConstruct)));
+    const Fortran::parser::LoopControl *loopControl =
+        &*outerDoConstruct.GetLoopControl();
+    const auto &concurrent =
+        std::get<Fortran::parser::LoopControl::Concurrent>(loopControl->u);
+    if (!std::get<std::list<Fortran::parser::LocalitySpec>>(concurrent.t)
+             .empty())
+      TODO(currentLocation, "DO CONCURRENT with locality spec inside ACC");
+
+    const auto &concurrentHeader =
+        std::get<Fortran::parser::ConcurrentHeader>(concurrent.t);
+    const auto &controls =
+        std::get<std::list<Fortran::parser::ConcurrentControl>>(
+            concurrentHeader.t);
+    for (const auto &control : controls) {
+      lowerbounds.push_back(fir::getBase(converter.genExprValue(
+          *Fortran::semantics::GetExpr(std::get<1>(control.t)), stmtCtx)));
+      upperbounds.push_back(fir::getBase(converter.genExprValue(
+          *Fortran::semantics::GetExpr(std::get<2>(control.t)), stmtCtx)));
+      if (const auto &expr =
+              std::get<std::optional<Fortran::parser::ScalarIntExpr>>(
+                  control.t))
+        steps.push_back(fir::getBase(converter.genExprValue(
+            *Fortran::semantics::GetExpr(*expr), stmtCtx)));
+      else // If `step` is not present, assume it is `1`.
+        steps.push_back(builder.createIntegerConstant(
+            currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1));
+
+      const auto &name = std::get<Fortran::parser::Name>(control.t);
+      privatizeIv(converter, *name.symbol, currentLocation, ivTypes, ivLocs,
+                  privateOperands, ivPrivate, privatizationRecipes,
+                  isDoConcurrent);
+
+      inclusiveBounds.push_back(true);
+    }
+  } else {
+    for (uint64_t i = 0; i < loopsToProcess; ++i) {
+      const Fortran::parser::LoopControl *loopControl;
+      if (i == 0) {
+        loopControl = &*outerDoConstruct.GetLoopControl();
+        locs.push_back(converter.genLocation(
+            Fortran::parser::FindSourceLocation(outerDoConstruct)));
+      } else {
+        auto *doCons = crtEval->getIf<Fortran::parser::DoConstruct>();
+        assert(doCons && "expect do construct");
+        loopControl = &*doCons->GetLoopControl();
+        locs.push_back(converter.genLocation(
+            Fortran::parser::FindSourceLocation(*doCons)));
+      }
+
+      const Fortran::parser::LoopControl::Bounds *bounds =
+          std::get_if<Fortran::parser::LoopControl::Bounds>(&loopControl->u);
+      assert(bounds && "Expected bounds on the loop construct");
+      lowerbounds.push_back(fir::getBase(converter.genExprValue(
+          *Fortran::semantics::GetExpr(bounds->lower), stmtCtx)));
+      upperbounds.push_back(fir::getBase(converter.genExprValue(
+          *Fortran::semantics::GetExpr(bounds->upper), stmtCtx)));
+      if (bounds->step)
+        steps.push_back(fir::getBase(converter.genExprValue(
+            *Fortran::semantics::GetExpr(bounds->step), stmtCtx)));
+      else // If `step` is not present, assume it is `1`.
+        steps.push_back(builder.createIntegerConstant(
+            currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1));
+
+      Fortran::semantics::Symbol &ivSym =
+          bounds->name.thing.symbol->GetUltimate();
+      privatizeIv(converter, ivSym, currentLocation, ivTypes, ivLocs,
+                  privateOperands, ivPrivate, privatizationRecipes);
+
+      inclusiveBounds.push_back(true);
+
+      if (i < loopsToProcess - 1)
+        crtEval = &*std::next(crtEval->getNestedEvaluations().begin());
+    }
+  }
+}
+
+static mlir::acc::LoopOp
+buildACCLoopOp(Fortran::lower::AbstractConverter &converter,
+               mlir::Location currentLocation,
+               Fortran::semantics::SemanticsContext &semanticsContext,
+               Fortran::lower::StatementContext &stmtCtx,
+               const Fortran::parser::DoConstruct &outerDoConstruct,
+               Fortran::lower::pft::Evaluation &eval,
+               llvm::SmallVector<mlir::Value> &privateOperands,
+               llvm::SmallVector<mlir::Attribute> &privatizationRecipes,
+               llvm::SmallVector<mlir::Value> &gangOperands,
+               llvm::SmallVector<mlir::Value> &workerNumOperands,
+               llvm::SmallVector<mlir::Value> &vectorOperands,
+               llvm::SmallVector<mlir::Value> &tileOperands,
+               llvm::SmallVector<mlir::Value> &cacheOperands,
+               llvm::SmallVector<mlir::Value> &reductionOperands,
+               llvm::SmallVector<mlir::Type> &retTy, mlir::Value yieldValue,
+               uint64_t loopsToProcess) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+
+  llvm::SmallVector<mlir::Value> ivPrivate;
+  llvm::SmallVector<mlir::Type> ivTypes;
+  llvm::SmallVector<mlir::Location> ivLocs;
+  llvm::SmallVector<bool> inclusiveBounds;
+  llvm::SmallVector<mlir::Location> locs;
+  llvm::SmallVector<mlir::Value> lowerbounds, upperbounds, steps;
+
+  // Look at the do/do concurrent loops to extract bounds information.
+  processDoLoopBounds(converter, currentLocation, stmtCtx, builder,
+                      outerDoConstruct, eval, lowerbounds, upperbounds, steps,
+                      privateOperands, ivPrivate, privatizationRecipes, ivTypes,
+                      ivLocs, inclusiveBounds, locs, loopsToProcess);
+
+  // Prepare the operand segment size attribute and the operands value range.
+  llvm::SmallVector<mlir::Value> operands;
+  llvm::SmallVector<int32_t> operandSegments;
+  addOperands(operands, operandSegments, lowerbounds);
+  addOperands(operands, operandSegments, upperbounds);
+  addOperands(operands, operandSegments, steps);
+  addOperands(operands, operandSegments, gangOperands);
+  addOperands(operands, operandSegments, workerNumOperands);
+  addOperands(operands, operandSegments, vectorOperands);
+  addOperands(operands, operandSegments, tileOperands);
+  addOperands(operands, operandSegments, cacheOperands);
+  addOperands(operands, operandSegments, privateOperands);
+  addOperands(operands, operandSegments, reductionOperands);
+
+  auto loopOp = createRegionOp<mlir::acc::LoopOp, mlir::acc::YieldOp>(
+      builder, builder.getFusedLoc(locs), currentLocation, eval, operands,
+      operandSegments, /*outerCombined=*/false, retTy, yieldValue, ivTypes,
+      ivLocs);
+
+  for (auto [arg, value] : llvm::zip(
+           loopOp.getLoopRegions().front()->front().getArguments(), ivPrivate))
+    builder.create<fir::StoreOp>(currentLocation, arg, value);
+
+  loopOp.setInclusiveUpperbound(inclusiveBounds);
+
+  return loopOp;
+}
+
 static mlir::acc::LoopOp createLoopOp(
     Fortran::lower::AbstractConverter &converter,
     mlir::Location currentLocation,
@@ -2150,9 +2313,9 @@ static mlir::acc::LoopOp createLoopOp(
         std::nullopt,
     bool needEarlyReturnHandling = false) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  llvm::SmallVector<mlir::Value> tileOperands, privateOperands, ivPrivate,
+  llvm::SmallVector<mlir::Value> tileOperands, privateOperands,
       reductionOperands, cacheOperands, vectorOperands, workerNumOperands,
-      gangOperands, lowerbounds, upperbounds, steps;
+      gangOperands;
   llvm::SmallVector<mlir::Attribute> privatizationRecipes, reductionRecipes;
   llvm::SmallVector<int32_t> tileOperandsSegments, gangOperandsSegments;
   llvm::SmallVector<int64_t> collapseValues;
@@ -2321,107 +2484,6 @@ static mlir::acc::LoopOp createLoopOp(
     }
   }
 
-  llvm::SmallVector<mlir::Type> ivTypes;
-  llvm::SmallVector<mlir::Location> ivLocs;
-  llvm::SmallVector<bool> inclusiveBounds;
-  llvm::SmallVector<mlir::Location> locs;
-  locs.push_back(currentLocation); // Location of the directive
-  Fortran::lower::pft::Evaluation *crtEval = &eval.getFirstNestedEvaluation();
-  bool isDoConcurrent = outerDoConstruct.IsDoConcurrent();
-  if (isDoConcurrent) {
-    locs.push_back(converter.genLocation(
-        Fortran::parser::FindSourceLocation(outerDoConstruct)));
-    const Fortran::parser::LoopControl *loopControl =
-        &*outerDoConstruct.GetLoopControl();
-    const auto &concurrent =
-        std::get<Fortran::parser::LoopControl::Concurrent>(loopControl->u);
-    if (!std::get<std::list<Fortran::parser::LocalitySpec>>(concurrent.t)
-             .empty())
-      TODO(currentLocation, "DO CONCURRENT with locality spec");
-
-    const auto &concurrentHeader =
-        std::get<Fortran::parser::ConcurrentHeader>(concurrent.t);
-    const auto &controls =
-        std::get<std::list<Fortran::parser::ConcurrentControl>>(
-            concurrentHeader.t);
-    for (const auto &control : controls) {
-      lowerbounds.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(std::get<1>(control.t)), stmtCtx)));
-      upperbounds.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(std::get<2>(control.t)), stmtCtx)));
-      if (const auto &expr =
-              std::get<std::optional<Fortran::parser::ScalarIntExpr>>(
-                  control.t))
-        steps.push_back(fir::getBase(converter.genExprValue(
-            *Fortran::semantics::GetExpr(*expr), stmtCtx)));
-      else // If `step` is not present, assume it is `1`.
-        steps.push_back(builder.createIntegerConstant(
-            currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1));
-
-      const auto &name = std::get<Fortran::parser::Name>(control.t);
-      privatizeIv(converter, *name.symbol, currentLocation, ivTypes, ivLocs,
-                  privateOperands, ivPrivate, privatizationRecipes,
-                  isDoConcurrent);
-
-      inclusiveBounds.push_back(true);
-    }
-  } else {
-    int64_t loopCount =
-        Fortran::lower::getLoopCountForCollapseAndTile(accClauseList);
-    for (unsigned i = 0; i < loopCount; ++i) {
-      const Fortran::parser::LoopControl *loopControl;
-      if (i == 0) {
-        loopControl = &*outerDoConstruct.GetLoopControl();
-        locs.push_back(converter.genLocation(
-            Fortran::parser::FindSourceLocation(outerDoConstruct)));
-      } else {
-        auto *doCons = crtEval->getIf<Fortran::parser::DoConstruct>();
-        assert(doCons && "expect do construct");
-        loopControl = &*doCons->GetLoopControl();
-        locs.push_back(converter.genLocation(
-            Fortran::parser::FindSourceLocation(*doCons)));
-      }
-
-      const Fortran::parser::LoopControl::Bounds *bounds =
-          std::get_if<Fortran::parser::LoopControl::Bounds>(&loopControl->u);
-      assert(bounds && "Expected bounds on the loop construct");
-      lowerbounds.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(bounds->lower), stmtCtx)));
-      upperbounds.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(bounds->upper), stmtCtx)));
-      if (bounds->step)
-        steps.push_back(fir::getBase(converter.genExprValue(
-            *Fortran::semantics::GetExpr(bounds->step), stmtCtx)));
-      else // If `step` is not present, assume it is `1`.
-        steps.push_back(builder.createIntegerConstant(
-            currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1));
-
-      Fortran::semantics::Symbol &ivSym =
-          bounds->name.thing.symbol->GetUltimate();
-      privatizeIv(converter, ivSym, currentLocation, ivTypes, ivLocs,
-                  privateOperands, ivPrivate, privatizationRecipes);
-
-      inclusiveBounds.push_back(true);
-
-      if (i < loopCount - 1)
-        crtEval = &*std::next(crtEval->getNestedEvaluations().begin());
-    }
-  }
-
-  // Prepare the operand segment size attribute and the operands value range.
-  llvm::SmallVector<mlir::Value> operands;
-  llvm::SmallVector<int32_t> operandSegments;
-  addOperands(operands, operandSegments, lowerbounds);
-  addOperands(operands, operandSegments, upperbounds);
-  addOperands(operands, operandSegments, steps);
-  addOperands(operands, operandSegments, gangOperands);
-  addOperands(operands, operandSegments, workerNumOperands);
-  addOperands(operands, operandSegments, vectorOperands);
-  addOperands(operands, operandSegments, tileOperands);
-  addOperands(operands, operandSegments, cacheOperands);
-  addOperands(operands, operandSegments, privateOperands);
-  addOperands(operands, operandSegments, reductionOperands);
-
   llvm::SmallVector<mlir::Type> retTy;
   mlir::Value yieldValue;
   if (needEarlyReturnHandling) {
@@ -2430,16 +2492,13 @@ static mlir::acc::LoopOp createLoopOp(
     retTy.push_back(i1Ty);
   }
 
-  auto loopOp = createRegionOp<mlir::acc::LoopOp, mlir::acc::YieldOp>(
-      builder, builder.getFusedLoc(locs), currentLocation, eval, operands,
-      operandSegments, /*outerCombined=*/false, retTy, yieldValue, ivTypes,
-      ivLocs);
-
-  for (auto [arg, value] : llvm::zip(
-           loopOp.getLoopRegions().front()->front().getArguments(), ivPrivate))
-    builder.create<fir::StoreOp>(currentLocation, arg, value);
-
-  loopOp.setInclusiveUpperbound(inclusiveBounds);
+  uint64_t loopsToProcess =
+      Fortran::lower::getLoopCountForCollapseAndTile(accClauseList);
+  auto loopOp = buildACCLoopOp(
+      converter, currentLocation, semanticsContext, stmtCtx, outerDoConstruct,
+      eval, privateOperands, privatizationRecipes, gangOperands,
+      workerNumOperands, vectorOperands, tileOperands, cacheOperands,
+      reductionOperands, retTy, yieldValue, loopsToProcess);
 
   if (!gangDeviceTypes.empty())
     loopOp.setGangAttr(builder.getArrayAttr(gangDeviceTypes));
@@ -4891,6 +4950,12 @@ bool Fortran::lower::isInOpenACCLoop(fir::FirOpBuilder &builder) {
   return false;
 }
 
+bool Fortran::lower::isInsideOpenACCComputeConstruct(
+    fir::FirOpBuilder &builder) {
+  return mlir::isa_and_nonnull<ACC_COMPUTE_CONSTRUCT_OPS>(
+      mlir::acc::getEnclosingComputeOp(builder.getRegion()));
+}
+
 void Fortran::lower::setInsertionPointAfterOpenACCLoopIfInside(
     fir::FirOpBuilder &builder) {
   if (auto loopOp =
@@ -4905,10 +4970,10 @@ void Fortran::lower::genEarlyReturnInOpenACCLoop(fir::FirOpBuilder &builder,
   builder.create<mlir::acc::YieldOp>(loc, yieldValue);
 }
 
-int64_t Fortran::lower::getLoopCountForCollapseAndTile(
+uint64_t Fortran::lower::getLoopCountForCollapseAndTile(
     const Fortran::parser::AccClauseList &clauseList) {
-  int64_t collapseLoopCount = 1;
-  int64_t tileLoopCount = 1;
+  uint64_t collapseLoopCount = 1;
+  uint64_t tileLoopCount = 1;
   for (const Fortran::parser::AccClause &clause : clauseList.v) {
     if (const auto *collapseClause =
             std::get_if<Fortran::parser::AccClause::Collapse>(&clause.u)) {
@@ -4927,3 +4992,91 @@ int64_t Fortran::lower::getLoopCountForCollapseAndTile(
     return tileLoopCount;
   return collapseLoopCount;
 }
+
+/// Create an ACC loop operation for a DO construct when inside ACC compute
+/// constructs This serves as a bridge between regular DO construct handling and
+/// ACC loop creation
+mlir::Operation *Fortran::lower::genOpenACCLoopFromDoConstruct(
+    AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semanticsContext,
+    Fortran::lower::SymMap &localSymbols,
+    const Fortran::parser::DoConstruct &doConstruct, pft::Evaluation &eval) {
+  // Only convert loops which have induction variables that need privatized.
+  if (!doConstruct.IsDoNormal() && !doConstruct.IsDoConcurrent())
+    return nullptr;
+
+  // If the evaluation is not structured, then we cannot convert the loop
+  // because acc loop does not have an unstructured form.
+  // TODO: There may be other strategies that can be employed such
+  // as generating acc.private for the loop variables without attaching
+  // them to acc.loop.
+  if (eval.lowerAsUnstructured())
+    return nullptr;
+
+  // Open up a new scope for the loop variables.
+  localSymbols.pushScope();
+  auto scopeGuard = llvm::make_scope_exit([&]() { localSymbols.popScope(); });
+
+  // Prepare empty operand vectors since there are no associated `acc loop`
+  // clauses with the Fortran do loops being handled here.
+  llvm::SmallVector<mlir::Value> privateOperands, gangOperands,
+      workerNumOperands, vectorOperands, tileOperands, cacheOperands,
+      reductionOperands;
+  llvm::SmallVector<mlir::Attribute> privatizationRecipes;
+  llvm::SmallVector<mlir::Type> retTy;
+  mlir::Value yieldValue;
+  uint64_t loopsToProcess = 1; // Single loop construct
+
+  // Use same mechanism that handles `acc loop` contained do loops to handle
+  // the implicit loop case.
+  Fortran::lower::StatementContext stmtCtx;
+  auto loopOp = buildACCLoopOp(
+      converter, converter.getCurrentLocation(), semanticsContext, stmtCtx,
+      doConstruct, eval, privateOperands, privatizationRecipes, gangOperands,
+      workerNumOperands, vectorOperands, tileOperands, cacheOperands,
+      reductionOperands, retTy, yieldValue, loopsToProcess);
+
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  if (!privatizationRecipes.empty())
+    loopOp.setPrivatizationRecipesAttr(mlir::ArrayAttr::get(
+        converter.getFirOpBuilder().getContext(), privatizationRecipes));
+
+  // Normal do loops which are not annotated with `acc loop` should be
+  // left for analysis by marking with `auto`. This is the case even in the case
+  // of `acc parallel` region because the normal rules of applying `independent`
+  // is only for loops marked with `acc loop`.
+  // For do concurrent loops, the spec says in section 2.17.2:
+  // "When do concurrent appears without a loop construct in a kernels construct
+  // it is treated as if it is annotated with loop auto. If it appears in a
+  // parallel construct or an accelerator routine then it is treated as if it is
+  // annotated with loop independent."
+  // So this means that in all cases we mark with `auto` unless it is a
+  // `do concurrent` in an `acc parallel` construct or it must be `seq` because
+  // it is in an `acc serial` construct.
+  mlir::Operation *accRegionOp =
+      mlir::acc::getEnclosingComputeOp(converter.getFirOpBuilder().getRegion());
+  mlir::acc::LoopParMode parMode =
+      mlir::isa_and_present<mlir::acc::ParallelOp>(accRegionOp) &&
+              doConstruct.IsDoConcurrent()
+          ? mlir::acc::LoopParMode::loop_independent
+      : mlir::isa_and_present<mlir::acc::SerialOp>(accRegionOp)
+          ? mlir::acc::LoopParMode::loop_seq
+          : mlir::acc::LoopParMode::loop_auto;
+
+  // Set the parallel mode based on the computed parMode
+  auto deviceNoneAttr = mlir::acc::DeviceTypeAttr::get(
+      builder.getContext(), mlir::acc::DeviceType::None);
+  auto arrOfDeviceNone =
+      mlir::ArrayAttr::get(builder.getContext(), deviceNoneAttr);
+  if (parMode == mlir::acc::LoopParMode::loop_independent) {
+    loopOp.setIndependentAttr(arrOfDeviceNone);
+  } else if (parMode == mlir::acc::LoopParMode::loop_seq) {
+    loopOp.setSeqAttr(arrOfDeviceNone);
+  } else if (parMode == mlir::acc::LoopParMode::loop_auto) {
+    loopOp.setAuto_Attr(arrOfDeviceNone);
+  } else {
+    llvm_unreachable("Unexpected loop par mode");
+  }
+
+  return loopOp;
+}
diff --git a/flang/test/Lower/OpenACC/do-loops-to-acc-loops.f90 b/flang/test/Lower/OpenACC/do-loops-to-acc-loops.f90
new file mode 100644
index 0000000000000..d34cd11795b0f
--- /dev/null
+++ b/flang/test/Lower/OpenACC/do-loops-to-acc-loops.f90
@@ -0,0 +1,332 @@
+! This test checks lowering of Fortran do loops and do concurrent loops to OpenACC loop constructs.
+! Tests the new functionality that converts Fortran iteration constructs to acc.loop with proper IV handling.
+
+! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s
+
+! CHECK-LABEL: func.func @_QPbasic_do_loop
+subroutine basic_do_loop()
+  integer :: i
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b
+
+  ! Basic do loop that should be converted to acc.loop
+  !$acc kernels
+  do i = 1, n
+    a(i) = b(i) + 1.0
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.yield
+! CHECK: attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPbasic_do_concurrent
+subroutine basic_do_concurrent()
+  integer :: i
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b
+
+  ! Basic do concurrent loop
+  !$acc kernels
+  do concurrent (i = 1:n)
+    a(i) = b(i) + 1.0
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.yield
+! CHECK: attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPbasic_do_loop_parallel
+subroutine basic_do_loop_parallel()
+  integer :: i
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b
+
+  ! Basic do loop with acc parallel that should be converted to acc.loop
+  !$acc parallel
+  do i = 1, n
+    a(i) = b(i) + 1.0
+  end do
+  !$acc end parallel
+
+! CHECK: acc.parallel {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.yield
+! CHECK: attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPbasic_do_loop_serial
+subroutine basic_do_loop_serial()
+  integer :: i
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b
+
+  ! Basic do loop with acc serial that should be converted to acc.loop
+  !$acc serial
+  do i = 1, n
+    a(i) = b(i) + 1.0
+  end do
+  !$acc end serial
+
+! CHECK: acc.serial {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.yield
+! CHECK: attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPbasic_do_concurrent_parallel
+subroutine basic_do_concurrent_parallel()
+  integer :: i
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b
+
+  ! Basic do concurrent loop with acc parallel
+  !$acc parallel
+  do concurrent (i = 1:n)
+    a(i) = b(i) + 1.0
+  end do
+  !$acc end parallel
+
+! CHECK: acc.parallel {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.yield
+! CHECK: attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPbasic_do_concurrent_serial
+subroutine basic_do_concurrent_serial()
+  integer :: i
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b
+
+  ! Basic do concurrent loop with acc serial
+  !$acc serial
+  do concurrent (i = 1:n)
+    a(i) = b(i) + 1.0
+  end do
+  !$acc end serial
+
+! CHECK: acc.serial {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.yield
+! CHECK: attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPmulti_dimension_do_concurrent
+subroutine multi_dimension_do_concurrent()
+  integer :: i, j, k
+  integer, parameter :: n = 10, m = 20, l = 5
+  real, dimension(n,m,l) :: a, b
+
+  ! Multi-dimensional do concurrent with multiple iteration variables
+  !$acc kernels
+  do concurrent (i = 1:n, j = 1:m, k = 1:l)
+    a(i,j,k) = b(i,j,k) * 2.0
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%{{.*}}, %{{.*}}, %{{.*}} : i32, i32, i32) step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32)
+! CHECK: acc.yield
+! CHECK: attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true, true>}
+end subroutine
+
+
+! CHECK-LABEL: func.func @_QPnested_do_loops
+subroutine nested_do_loops()
+  integer :: i, j
+  integer, parameter :: n = 10, m = 20
+  real, dimension(n,m) :: a, b
+
+  ! Nested do loops
+  !$acc kernels
+  do i = 1, n
+    do j = 1, m
+      a(i,j) = b(i,j) + i + j
+    end do
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.yield
+! CHECK: attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
+! CHECK: acc.yield
+! CHECK: attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPvariable_bounds_and_step
+subroutine variable_bounds_and_step(n, start_val, step_val)
+  integer, intent(in) :: n, start_val, step_val
+  integer :: i
+  real, dimension(n) :: a, b
+
+  ! Do loop with variable bounds and step
+  !$acc kernels
+  do i = start_val, n, step_val
+    a(i) = b(i) * 2.0
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.yield
+! CHECK: attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPdifferent_iv_types
+subroutine different_iv_types()
+  integer(kind=8) :: i8
+  integer(kind=4) :: i4
+  integer(kind=2) :: i2
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b, c, d
+
+  ! Test different iteration variable types
+  !$acc kernels
+  do i8 = 1_8, int(n,8)
+    a(i8) = b(i8) + 1.0
+  end do
+  !$acc end kernels
+
+  !$acc kernels
+  do i4 = 1, n
+    b(i4) = c(i4) + 1.0
+  end do
+  !$acc end kernels
+
+  !$acc kernels
+  do i2 = 1_2, int(n,2)
+    c(i2) = d(i2) + 1.0
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i64) = (%{{.*}} : i64) to (%{{.*}} : i64) step (%{{.*}} : i64)
+! CHECK: acc.kernels {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32)
+! CHECK: acc.kernels {
+! CHECK: acc.loop {{.*}} control(%{{.*}} : i16) = (%{{.*}} : i16) to (%{{.*}} : i16) step (%{{.*}} : i16)
+
+end subroutine
+
+! -----------------------------------------------------------------------------------------
+! Tests for loops that should NOT be converted to acc.loop due to unstructured control flow
+
+! CHECK-LABEL: func.func @_QPinfinite_loop_no_iv
+subroutine infinite_loop_no_iv()
+  integer :: i
+  logical :: condition
+
+  ! Infinite loop with no induction variable - should NOT convert to acc.loop
+  !$acc kernels
+  do
+    i = i + 1
+    if (i > 100) exit
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK-NOT: acc.loop
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPdo_loop_with_goto
+subroutine do_loop_with_goto()
+  integer :: i
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b
+
+  ! Do loop with goto - unstructured control flow is not converted.
+  !$acc kernels
+  do i = 1, n
+    a(i) = b(i) + 1.0
+    if (i == 5) goto 100
+    100 continue
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK-NOT: acc.loop
+
+end subroutine
+
+
+! CHECK-LABEL: func.func @_QPdo_loop_with_cycle_goto
+subroutine do_loop_with_cycle_goto()
+  integer :: i
+  integer, parameter :: n = 10
+  real, dimension(n) :: a, b
+
+  ! Do loop with cycle and goto - unstructured control flow is not converted.
+  !$acc kernels
+  do i = 1, n
+    if (i == 3) cycle
+    a(i) = b(i) + 1.0
+    if (i == 7) goto 200
+    a(i) = a(i) * 2.0
+  end do
+200 continue
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK-NOT: acc.loop
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPnested_goto_loop
+subroutine nested_goto_loop()
+  integer :: i, j
+  integer, parameter :: n = 10, m = 5
+  real, dimension(n,m) :: a, b
+
+  ! Nested loop with goto from inner to outer - should NOT convert to acc.loop
+  !$acc kernels
+  do i = 1, n
+    do j = 1, m
+      a(i,j) = b(i,j) + 1.0
+      if (i * j > 20) goto 300  ! Exit both loops
+    end do
+  end do
+300 continue
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK-NOT: acc.loop
+
+end subroutine
+
+! CHECK-LABEL: func.func @_QPwhile_like_loop
+subroutine while_like_loop()
+  integer :: i
+  logical :: condition
+
+  i = 1
+  condition = .true.
+
+  ! While-like infinite loop - should NOT convert to acc.loop
+  !$acc kernels
+  do while (condition)
+    i = i + 1
+    if (i > 100) condition = .false.
+  end do
+  !$acc end kernels
+
+! CHECK: acc.kernels {
+! CHECK-NOT: acc.loop
+
+end subroutine
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 96b9adcc53b3c..19b81267c32dd 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -134,6 +134,24 @@ def OpenACC_VariableTypeCategory : I32BitEnumAttr<
   let printBitEnumPrimaryGroups = 1;
 }
 
+// These are parallelism determination modes for `acc loop`.
+// In the enum names, we use the "loop_" prefix because "auto" is
+// a language keyword - and thus for consistency all other cases
+// do the same.
+def OpenACC_LoopSeq : I32EnumAttrCase<"loop_seq", 0>;
+def OpenACC_LoopAuto : I32EnumAttrCase<"loop_auto", 1>;
+def OpenACC_LoopIndependent : I32EnumAttrCase<"loop_independent", 2>;
+
+def OpenACC_LoopParMode : I32EnumAttr<
+    "LoopParMode",
+    "Encodes the options for loop parallelism determination mode",
+    [
+      OpenACC_LoopAuto, OpenACC_LoopIndependent,
+      OpenACC_LoopSeq]> {
+  let cppNamespace = "::mlir::acc";
+  let genSpecializedAttr = 0;
+}
+
 // Type used in operation below.
 def IntOrIndex : AnyTypeOf<[AnyInteger, Index]>;
 
@@ -2404,6 +2422,53 @@ def OpenACC_LoopOp : OpenACC_Op<"loop",
   }];
 
   let hasVerifier = 1;
+
+  let builders = [
+    OpBuilder<(ins "::mlir::ValueRange":$lowerbounds,
+                   "::mlir::ValueRange":$upperbounds,
+                   "::mlir::ValueRange":$steps,
+                   "LoopParMode":$parMode), [{
+        auto deviceNoneAttr = mlir::acc::DeviceTypeAttr::get(
+          $_builder.getContext(), mlir::acc::DeviceType::None);
+        auto arrOfDeviceNone = mlir::ArrayAttr::get(
+          $_builder.getContext(), deviceNoneAttr);
+        build($_builder, $_state,
+          /*results=*/{},
+          /*lowerbound=*/lowerbounds,
+          /*upperbound=*/upperbounds,
+          /*step=*/steps,
+          /*inclusiveUpperbound=*/nullptr,
+          /*collapse=*/nullptr,
+          /*collapseDeviceType=*/nullptr,
+          /*gangOperands=*/{},
+          /*gangOperandsArgType=*/nullptr,
+          /*gangOperandsSegments=*/nullptr,
+          /*gangOperandsDeviceType=*/nullptr,
+          /*workerNumOperands=*/{},
+          /*workerNumOperandsDeviceType=*/nullptr,
+          /*vectorOperands=*/{},
+          /*vectorOperandsDeviceType=*/nullptr,
+          /*seq=*/parMode == LoopParMode::loop_seq ?
+            arrOfDeviceNone : nullptr,
+          /*independent=*/parMode == LoopParMode::loop_independent ?
+            arrOfDeviceNone : nullptr,
+          /*auto_=*/parMode == LoopParMode::loop_auto ?
+            arrOfDeviceNone : nullptr,
+          /*gang=*/nullptr,
+          /*worker=*/nullptr,
+          /*vector=*/nullptr,
+          /*tileOperands=*/{},
+          /*tileOperandsSegments=*/nullptr,
+          /*tileOperandsDeviceType=*/nullptr,
+          /*cacheOperands=*/{},
+          /*privateOperands=*/{},
+          /*privatizationRecipes=*/nullptr,
+          /*reductionOperands=*/{},
+          /*reductionRecipes=*/nullptr,
+          /*combined=*/nullptr);
+      }]
+    >
+  ];
 }
 
 // Yield operation for the acc.loop and acc.parallel operations.

>From c1114f839e319cf685d9fbb095cc40d76049ea43 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Mon, 21 Jul 2025 08:19:28 -0700
Subject: [PATCH 2/2] Fix formatting

---
 flang/include/flang/Lower/OpenACC.h | 6 +++---
 flang/lib/Lower/Bridge.cpp          | 8 ++++----
 flang/lib/Lower/OpenACC.cpp         | 2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/flang/include/flang/Lower/OpenACC.h b/flang/include/flang/Lower/OpenACC.h
index 8b13ce94f7bc4..e974f3d6eef11 100644
--- a/flang/include/flang/Lower/OpenACC.h
+++ b/flang/include/flang/Lower/OpenACC.h
@@ -124,7 +124,8 @@ uint64_t getLoopCountForCollapseAndTile(const Fortran::parser::AccClauseList &);
 /// Checks whether the current insertion point is inside OpenACC loop.
 bool isInOpenACCLoop(fir::FirOpBuilder &);
 
-/// Checks whether the current insertion point is inside OpenACC compute construct.
+/// Checks whether the current insertion point is inside OpenACC compute
+/// construct.
 bool isInsideOpenACCComputeConstruct(fir::FirOpBuilder &);
 
 void setInsertionPointAfterOpenACCLoopIfInside(fir::FirOpBuilder &);
@@ -139,8 +140,7 @@ mlir::Operation *genOpenACCLoopFromDoConstruct(
     AbstractConverter &converter,
     Fortran::semantics::SemanticsContext &semanticsContext,
     Fortran::lower::SymMap &localSymbols,
-    const Fortran::parser::DoConstruct &doConstruct,
-    pft::Evaluation &eval);
+    const Fortran::parser::DoConstruct &doConstruct, pft::Evaluation &eval);
 
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4073b8623e333..5eb1bafbb7ea2 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2170,9 +2170,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // Loops with induction variables inside OpenACC compute constructs
     // need special handling to ensure that the IVs are privatized.
     if (Fortran::lower::isInsideOpenACCComputeConstruct(*builder)) {
-      mlir::Operation* loopOp = Fortran::lower::genOpenACCLoopFromDoConstruct(
-                         *this, bridge.getSemanticsContext(), localSymbols,
-                         doConstruct, eval);
+      mlir::Operation *loopOp = Fortran::lower::genOpenACCLoopFromDoConstruct(
+          *this, bridge.getSemanticsContext(), localSymbols, doConstruct, eval);
       bool success = loopOp != nullptr;
       if (success) {
         // Sanity check that the builder insertion point is inside the newly
@@ -2184,7 +2183,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
         // Loop body code.
         auto iter = eval.getNestedEvaluations().begin();
-        for (auto end = --eval.getNestedEvaluations().end(); iter != end; ++iter)
+        for (auto end = --eval.getNestedEvaluations().end(); iter != end;
+             ++iter)
           genFIR(*iter, unstructuredContext);
         return;
       }
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 950b02501751a..5f58ce02c11b1 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -35,8 +35,8 @@
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/Support/LLVM.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/Frontend/OpenACC/ACC.h.inc"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"


From flang-commits at lists.llvm.org  Mon Jul 21 08:24:44 2025
From: flang-commits at lists.llvm.org (Rolf Morel via flang-commits)
Date: Mon, 21 Jul 2025 08:24:44 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e5bbc.050a0220.370a26.bc40@mx.google.com>


https://github.com/rolfmorel edited https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 08:26:55 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 08:26:55 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e5c3f.050a0220.1f53af.e216@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/149134

>From 0929234481b4d6d015381fda490c05bbeff1e22f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 08:04:17 -0500
Subject: [PATCH 1/3] [flang][OpenMP] Parse OpenMP 6.0 map modifiers

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers.
The old "map-type-modifier" is retained, but is no longer created in
parsing. It will remain to take advantage of the preexisting modifier
validation for older versions: when the OpenMP version is < 6.0,
the modifiers will be rewritten back as map-type-modifiers (or map-
type in case of "delete").

In this patch the modifiers will always be rewritten in the older
format to isolate these changes to parsing as much as possible.
---
 flang/include/flang/Parser/dump-parse-tree.h  |  14 ++
 flang/include/flang/Parser/parse-tree.h       | 138 +++++++++++++++---
 .../flang/Semantics/openmp-modifiers.h        |   7 +
 flang/lib/Parser/openmp-parsers.cpp           |  53 +++++--
 flang/lib/Parser/unparse.cpp                  |   7 +
 flang/lib/Semantics/canonicalize-omp.cpp      |  48 +++++-
 flang/lib/Semantics/canonicalize-omp.h        |   9 +-
 flang/lib/Semantics/openmp-modifiers.cpp      | 115 +++++++++++++++
 flang/lib/Semantics/resolve-directives.cpp    |   2 +
 flang/lib/Semantics/semantics.cpp             |   3 +-
 .../test/Parser/OpenMP/map-modifiers-v60.f90  | 113 ++++++++++++++
 11 files changed, 466 insertions(+), 43 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/map-modifiers-v60.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 32b6ca45609b6..23e35d106c077 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -529,6 +529,8 @@ class ParseTreeDumper {
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlwaysModifier)
+  NODE_ENUM(OmpAlwaysModifier, Value)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
@@ -546,6 +548,8 @@ class ParseTreeDumper {
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
   NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpCloseModifier)
+  NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -561,6 +565,8 @@ class ParseTreeDumper {
   NODE(parser, OmpDefaultmapClause)
   NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE(parser, OmpDeleteModifier)
+  NODE_ENUM(OmpDeleteModifier, Value)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
   NODE(parser, OmpTaskDependenceType)
@@ -628,6 +634,8 @@ class ParseTreeDumper {
   NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
   NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpPresentModifier)
+  NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
@@ -637,6 +645,10 @@ class ParseTreeDumper {
   NODE(parser, OmpInReductionClause)
   NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpRefModifier)
+  NODE_ENUM(OmpRefModifier, Value)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpInitializerProc)
@@ -673,6 +685,8 @@ class ParseTreeDumper {
   NODE(parser, OmpSectionsDirective)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpxHoldModifier)
+  NODE_ENUM(OmpxHoldModifier, Value)
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cc1d032f94d4a..92a449d2f9d49 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3756,6 +3756,19 @@ struct OmpAllocatorComplexModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// always-modifier ->
+//    ALWAYS                                        // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpAlwaysModifier {
+  ENUM_CLASS(Value, Always)
+  WRAPPER_CLASS_BOILERPLATE(OmpAlwaysModifier, Value);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3767,17 +3780,29 @@ struct OmpChunkModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpChunkModifier, Value);
 };
 
-// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
-// iterator-specifier ->
-//    [iterator-type] iterator-identifier
-//        = range-specification |                   // since 5.0
-//    [iterator-type ::] iterator-identifier
-//        = range-specification                     // since 5.2
-struct OmpIteratorSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
-  CharBlock source;
-  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+// close-modifier ->
+//    CLOSE                                         // since 5.0
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpCloseModifier {
+  ENUM_CLASS(Value, Close)
+  WRAPPER_CLASS_BOILERPLATE(OmpCloseModifier, Value);
+};
+
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// delete-modifier ->
+//    DELETE                                        // since 6.0
+//
+// Until 5.2, it was a part of map-type.
+struct OmpDeleteModifier {
+  ENUM_CLASS(Value, Delete)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeleteModifier, Value);
 };
 
 // Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289]
@@ -3867,6 +3892,19 @@ struct OmpInteropType {
   WRAPPER_CLASS_BOILERPLATE(OmpInteropType, Value);
 };
 
+// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+//
+// iterator-specifier ->
+//    [iterator-type] iterator-identifier
+//        = range-specification |                   // since 5.0
+//    [iterator-type ::] iterator-identifier
+//        = range-specification                     // since 5.2
+struct OmpIteratorSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
+  CharBlock source;
+  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+};
+
 // Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
 //
 // iterator-modifier ->
@@ -3901,21 +3939,28 @@ struct OmpMapper {
   WRAPPER_CLASS_BOILERPLATE(OmpMapper, Name);
 };
 
-// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
 // map-type ->
-//    ALLOC | DELETE | FROM | RELEASE | TO | TOFROM // since 4.5
+//    ALLOC | DELETE | RELEASE |                    // since 4.5, until 5.2
+//    FROM | TO | TOFROM |                          // since 4.5
+//    STORAGE                                       // since 6.0
+//
+// Since 6.0 DELETE is a separate delete-modifier.
 struct OmpMapType {
-  ENUM_CLASS(Value, Alloc, Delete, From, Release, To, Tofrom);
+  ENUM_CLASS(Value, Alloc, Delete, From, Release, Storage, To, Tofrom);
   WRAPPER_CLASS_BOILERPLATE(OmpMapType, Value);
 };
 
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-type-modifier ->
-//    ALWAYS |                                      // since 4.5
-//    CLOSE |                                       // since 5.0
-//    PRESENT                                       // since 5.1
+//    ALWAYS |                                      // since 4.5, until 5.2
+//    CLOSE |                                       // since 5.0, until 5.2
+//    PRESENT                                       // since 5.1, until 5.2
+// Since 6.0 the map-type-modifier has been split into individual modifiers.
+//
 struct OmpMapTypeModifier {
   ENUM_CLASS(Value, Always, Close, Present, Ompx_Hold)
   WRAPPER_CLASS_BOILERPLATE(OmpMapTypeModifier, Value);
@@ -3954,6 +3999,19 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// present-modifier ->
+//    PRESENT                                       // since 5.1
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpPresentModifier {
+  ENUM_CLASS(Value, Present)
+  WRAPPER_CLASS_BOILERPLATE(OmpPresentModifier, Value);
+};
+
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3963,6 +4021,26 @@ struct OmpReductionModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpReductionModifier, Value);
 };
 
+// Ref: [6.0:279-288]
+//
+// ref-modifier ->
+//    REF_PTEE | REF_PTR | REF_PTR_PTEE             // since 6.0
+//
+struct OmpRefModifier {
+  ENUM_CLASS(Value, Ref_Ptee, Ref_Ptr, Ref_Ptr_Ptee)
+  WRAPPER_CLASS_BOILERPLATE(OmpRefModifier, Value);
+};
+
+// Ref: [6.0:279-288]
+//
+// self-modifier ->
+//    SELF                                          // since 6.0
+//
+struct OmpSelfModifier {
+  ENUM_CLASS(Value, Self)
+  WRAPPER_CLASS_BOILERPLATE(OmpSelfModifier, Value);
+};
+
 // Ref: [5.2:117-120]
 //
 // step-complex-modifier ->
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
+//
+// ompx-hold-modifier ->
+//    OMPX_HOLD                                     // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpxHoldModifier {
+  ENUM_CLASS(Value, Ompx_Hold)
+  WRAPPER_CLASS_BOILERPLATE(OmpxHoldModifier, Value);
+};
+
 // context-selector
 using OmpContextSelector = traits::OmpContextSelectorSpecification;
 } // namespace modifier
@@ -4376,13 +4466,25 @@ struct OmpLinearClause {
 // map-clause ->
 //    MAP([modifier...:] locator-list)              // since 4.5
 // modifier ->
-//    map-type-modifier |                           // since 4.5
+//    map-type-modifier [replaced] |                // since 4.5, until 5.2
+//    always-modifier |                             // since 6.0
+//    close-modifier |                              // since 6.0
+//    delete-modifier |                             // since 6.0
+//    present-modifier |                            // since 6.0
+//    ref-modifier |                                // since 6.0
+//    self-modifier |                               // since 6.0
 //    mapper |                                      // since 5.0
 //    iterator |                                    // since 5.1
 //    map-type                                      // since 4.5
+//    ompx-hold-modifier |                          // since 6.0
+//
+// Since 6.0 the map-type-modifier has been split into individual modifiers,
+// and delete-modifier has been split from map-type.
 struct OmpMapClause {
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  MODIFIER_BOILERPLATE(OmpMapTypeModifier, OmpMapper, OmpIterator, OmpMapType);
+  MODIFIER_BOILERPLATE(OmpAlwaysModifier, OmpCloseModifier, OmpDeleteModifier,
+      OmpMapTypeModifier, OmpPresentModifier, OmpRefModifier, OmpSelfModifier,
+      OmpMapper, OmpIterator, OmpMapType, OmpxHoldModifier);
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 4fbd80f989e72..a9fe911ef8807 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -71,8 +71,11 @@ DECLARE_DESCRIPTOR(parser::OmpAlignment);
 DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
+DECLARE_DESCRIPTOR(parser::OmpAlwaysModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpCloseModifier);
 DECLARE_DESCRIPTOR(parser::OmpContextSelector);
+DECLARE_DESCRIPTOR(parser::OmpDeleteModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
@@ -88,12 +91,16 @@ DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
 DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
+DECLARE_DESCRIPTOR(parser::OmpPresentModifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
+DECLARE_DESCRIPTOR(parser::OmpRefModifier);
+DECLARE_DESCRIPTOR(parser::OmpSelfModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpVariableCategory);
+DECLARE_DESCRIPTOR(parser::OmpxHoldModifier);
 
 #undef DECLARE_DESCRIPTOR
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..d349d8ceb0bb5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -274,6 +274,10 @@ TYPE_PARSER( //
     construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}) ||
     construct<OmpTypeSpecifier>(Parser<TypeSpec>{}))
 
+// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
+TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
+    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
+
 TYPE_PARSER(construct<OmpReductionSpecifier>( //
     Parser<OmpReductionIdentifier>{},
     ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
@@ -442,9 +446,18 @@ TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
 
 TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
 
+TYPE_PARSER(construct<OmpAlwaysModifier>( //
+    "ALWAYS" >> pure(OmpAlwaysModifier::Value::Always)))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpCloseModifier>( //
+    "CLOSE" >> pure(OmpCloseModifier::Value::Close)))
+
+TYPE_PARSER(construct<OmpDeleteModifier>( //
+    "DELETE" >> pure(OmpDeleteModifier::Value::Delete)))
+
 TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
@@ -502,26 +515,16 @@ TYPE_PARSER(construct<OmpLinearModifier>( //
 TYPE_PARSER(construct<OmpMapper>( //
     "MAPPER"_tok >> parenthesized(Parser<ObjectName>{})))
 
-// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+// map-type -> ALLOC | DELETE | FROM | RELEASE | STORAGE | TO | TOFROM
 TYPE_PARSER(construct<OmpMapType>( //
     "ALLOC" >> pure(OmpMapType::Value::Alloc) ||
-    "DELETE" >> pure(OmpMapType::Value::Delete) ||
+    // Parse "DELETE" as OmpDeleteModifier
     "FROM" >> pure(OmpMapType::Value::From) ||
     "RELEASE" >> pure(OmpMapType::Value::Release) ||
+    "STORAGE" >> pure(OmpMapType::Value::Storage) ||
     "TO"_id >> pure(OmpMapType::Value::To) ||
     "TOFROM" >> pure(OmpMapType::Value::Tofrom)))
 
-// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
-TYPE_PARSER(construct<OmpMapTypeModifier>(
-    "ALWAYS" >> pure(OmpMapTypeModifier::Value::Always) ||
-    "CLOSE" >> pure(OmpMapTypeModifier::Value::Close) ||
-    "OMPX_HOLD" >> pure(OmpMapTypeModifier::Value::Ompx_Hold) ||
-    "PRESENT" >> pure(OmpMapTypeModifier::Value::Present)))
-
-// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
-TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
-    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
-
 TYPE_PARSER(construct<OmpOrderModifier>(
     "REPRODUCIBLE" >> pure(OmpOrderModifier::Value::Reproducible) ||
     "UNCONSTRAINED" >> pure(OmpOrderModifier::Value::Unconstrained)))
@@ -534,11 +537,22 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
 TYPE_PARSER(construct<OmpPrescriptiveness>(
     "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
 
+TYPE_PARSER(construct<OmpPresentModifier>( //
+    "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
     "DEFAULT" >> pure(OmpReductionModifier::Value::Default)))
 
+TYPE_PARSER(construct<OmpRefModifier>( //
+    "REF_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptee) ||
+    "REF_PTR"_id >> pure(OmpRefModifier::Value::Ref_Ptr) ||
+    "REF_PTR_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptr_Ptee)))
+
+TYPE_PARSER(construct<OmpSelfModifier>( //
+    "SELF" >> pure(OmpSelfModifier::Value::Self)))
+
 TYPE_PARSER(construct<OmpStepComplexModifier>( //
     "STEP" >> parenthesized(scalarIntExpr)))
 
@@ -559,6 +573,9 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "POINTER" >> pure(OmpVariableCategory::Value::Pointer) ||
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
+TYPE_PARSER(construct<OmpxHoldModifier>( //
+    "OMPX_HOLD" >> pure(OmpxHoldModifier::Value::Ompx_Hold)))
+
 // This could be auto-generated.
 TYPE_PARSER(
     sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
@@ -611,10 +628,16 @@ TYPE_PARSER(sourced(
     construct<OmpLinearClause::Modifier>(Parser<OmpStepSimpleModifier>{})))
 
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
-    sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
+    sourced(construct<OmpMapClause::Modifier>(Parser<OmpAlwaysModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpCloseModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpDeleteModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpPresentModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpRefModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpSelfModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpIterator>{}) ||
-        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{})))))
+        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpxHoldModifier>{})))))
 
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index fbe89c668fc13..8ed16905b5099 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -3007,8 +3007,15 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
+  WALK_NESTED_ENUM(OmpAlwaysModifier, Value)
+  WALK_NESTED_ENUM(OmpCloseModifier, Value)
+  WALK_NESTED_ENUM(OmpDeleteModifier, Value)
+  WALK_NESTED_ENUM(OmpPresentModifier, Value)
+  WALK_NESTED_ENUM(OmpRefModifier, Value)
+  WALK_NESTED_ENUM(OmpSelfModifier, Value)
   WALK_NESTED_ENUM(OmpTraitSelectorName, Value)
   WALK_NESTED_ENUM(OmpTraitSetSelectorName, Value)
+  WALK_NESTED_ENUM(OmpxHoldModifier, Value)
 
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index cf05d8463277f..46aaab19ded0a 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -88,6 +89,8 @@ class CanonicalizationOfOmp {
     CanonicalizeUtilityConstructs(spec);
   }
 
+  void Post(parser::OmpMapClause &map) { CanonicalizeMapModifiers(map); }
+
 private:
   template <typename T> T *GetConstructIf(parser::ExecutionPartConstruct &x) {
     if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index c45d6bbbf9062..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,11 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
-}
+class SemanticsContext;
+
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+} // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index c84e832ee52a1..336ce4beb24ba 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -140,6 +140,22 @@ OmpGetDescriptor<parser::OmpAllocatorSimpleModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlwaysModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"always-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   static const OmpModifierDescriptor desc{
@@ -156,6 +172,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpCloseModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"close-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   static const OmpModifierDescriptor desc{
@@ -173,6 +205,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDeleteModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"delete-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique, OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   static const OmpModifierDescriptor desc{
@@ -347,6 +396,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapType>() {
       /*props=*/
       {
           {45, {OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
       },
       /*clauses=*/
       {
@@ -367,6 +417,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapTypeModifier>() {
       /*clauses=*/
       {
           {45, {Clause::OMPC_map}},
+          {60, {}},
       },
   };
   return desc;
@@ -420,6 +471,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPrescriptiveness>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPresentModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"present-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpReductionIdentifier>() {
@@ -456,6 +523,38 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpReductionModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpRefModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ref-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpSelfModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"self-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpStepComplexModifier>() {
@@ -522,4 +621,20 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpVariableCategory>() {
   };
   return desc;
 }
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpxHoldModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ompx-hold-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 151f4ccae634e..0b860314b4a1f 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -734,6 +734,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
+      default:
+        break;
       }
     }
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..d0a5b200d0d8e 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -643,8 +643,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}
diff --git a/flang/test/Parser/OpenMP/map-modifiers-v60.f90 b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..bc80886780d46
--- /dev/null
+++ b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,113 @@
+!RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x
+  !$omp target map(always, close, delete, present, ompx_hold: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(ALWAYS, CLOSE, DELETE, PRESENT, OMPX_HOLD: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpAlwaysModifier -> Value = Always
+!PARSE-TREE: | | Modifier -> OmpCloseModifier -> Value = Close
+!PARSE-TREE: | | Modifier -> OmpDeleteModifier -> Value = Delete
+!PARSE-TREE: | | Modifier -> OmpPresentModifier -> Value = Present
+!PARSE-TREE: | | Modifier -> OmpxHoldModifier -> Value = Ompx_Hold
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f01(x)
+  integer :: x
+  !$omp target map(self, storage: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: !$OMP TARGET  MAP(SELF, STORAGE: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpSelfModifier -> Value = Self
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = Storage
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f03(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f04(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f04 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'

>From faff0d32746640c34988a65552e277198748fcf2 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 08:49:18 -0500
Subject: [PATCH 2/3] Don't pass SemanticsContext to CanonicalizeOmp yet

---
 flang/lib/Semantics/canonicalize-omp.cpp | 10 ++++------
 flang/lib/Semantics/canonicalize-omp.h   |  5 +++--
 flang/lib/Semantics/semantics.cpp        |  3 ++-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 46aaab19ded0a..bba4d72a702e9 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,8 +27,7 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(SemanticsContext &context)
-      : context_{context}, messages_{context.messages()} {}
+  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -433,13 +432,12 @@ class CanonicalizationOfOmp {
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
-  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
-  CanonicalizationOfOmp omp{context};
+bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
+  CanonicalizationOfOmp omp{messages};
   Walk(program, omp);
-  return !context.messages().AnyFatalError();
+  return !messages.AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index 3251218da35ed..23350c522a775 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,12 +11,13 @@
 
 namespace Fortran::parser {
 struct Program;
-}
+class Messages;
+} // namespace Fortran::parser
 
 namespace Fortran::semantics {
 class SemanticsContext;
 
-bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
 } // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 6db11aaf56c2a..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -642,7 +642,8 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_.messages(), program_) &&
+      CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}

>From 64ec502991cddd823f7c1a1834255776d1cbab00 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 10:26:43 -0500
Subject: [PATCH 3/3] Add link to the description of ompx_hold

---
 flang/include/flang/Parser/parse-tree.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 92a449d2f9d49..0b3dec1010312 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4079,7 +4079,8 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
-// Extension
+// Extension:
+// https://openmp.llvm.org//openacc/OpenMPExtensions.html#ompx-hold
 //
 // ompx-hold-modifier ->
 //    OMPX_HOLD                                     // since 4.5


From flang-commits at lists.llvm.org  Mon Jul 21 08:27:21 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 08:27:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e5c59.170a0220.2f82f7.ee7f@mx.google.com>


================
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
----------------
kparzysz wrote:

Found a link to the documentation and added it in a comment.

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 08:32:27 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 08:32:27 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang] Implement !$omp unroll using
 omp.unroll_heuristic (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <687e5d8b.170a0220.188b9e.b4d4@mx.google.com>


https://github.com/kparzysz approved this pull request.


https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Mon Jul 21 08:51:44 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Mon, 21 Jul 2025 08:51:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687e6210.050a0220.2a832d.93fe@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
c8ef wrote:

```
LogicalResult FloatAttr::verify(function_ref<InFlightDiagnostic()> emitError,
                                Type type, APFloat value) {
  // Verify that the type is correct.
  if (!llvm::isa<FloatType>(type))
    return emitError() << "expected floating point type";

  // Verify that the type semantics match that of the value.
  if (&llvm::cast<FloatType>(type).getFloatSemantics() !=
      &value.getSemantics()) {
    return emitError()
           << "FloatAttr type doesn't match the type implied by its value";
  }
  return success();
}
```

It appears these two floating-point types have different semantics. To ensure pi uses the same semantics as the argument type, we'd still need to convert it from APFloat. Either way, a conversion seems necessary.

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Mon Jul 21 08:54:37 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 21 Jul 2025 08:54:37 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e62bd.170a0220.35d9d8.e999@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM, thanks!

https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 08:55:40 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 08:55:40 -0700 (PDT)
Subject: [flang-commits] [flang] 2aa1e54 - [flang][OpenMP] Parse OpenMP 6.0
	map modifiers (#149134)
Message-ID: <687e62fc.050a0220.112965.ea24@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-21T10:55:37-05:00
New Revision: 2aa1e54fa1ff7f7c347e7108fe8650e94014c941

URL: https://github.com/llvm/llvm-project/commit/2aa1e54fa1ff7f7c347e7108fe8650e94014c941
DIFF: https://github.com/llvm/llvm-project/commit/2aa1e54fa1ff7f7c347e7108fe8650e94014c941.diff

LOG: [flang][OpenMP] Parse OpenMP 6.0 map modifiers (#149134)

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers. The
old "map-type-modifier" is retained, but is no longer created in
parsing. It will remain to take advantage of the preexisting modifier
validation for older versions: when the OpenMP version is < 6.0, the
modifiers will be rewritten back as map-type-modifiers (or map- type in
case of "delete").

In this patch the modifiers will always be rewritten in the older format
to isolate these changes to parsing as much as possible.

Added: 
    flang/test/Parser/OpenMP/map-modifiers-v60.f90

Modified: 
    flang/include/flang/Parser/dump-parse-tree.h
    flang/include/flang/Parser/parse-tree.h
    flang/include/flang/Semantics/openmp-modifiers.h
    flang/lib/Parser/openmp-parsers.cpp
    flang/lib/Parser/unparse.cpp
    flang/lib/Semantics/canonicalize-omp.cpp
    flang/lib/Semantics/canonicalize-omp.h
    flang/lib/Semantics/openmp-modifiers.cpp
    flang/lib/Semantics/resolve-directives.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 32b6ca45609b6..23e35d106c077 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -529,6 +529,8 @@ class ParseTreeDumper {
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlwaysModifier)
+  NODE_ENUM(OmpAlwaysModifier, Value)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
@@ -546,6 +548,8 @@ class ParseTreeDumper {
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
   NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpCloseModifier)
+  NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -561,6 +565,8 @@ class ParseTreeDumper {
   NODE(parser, OmpDefaultmapClause)
   NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE(parser, OmpDeleteModifier)
+  NODE_ENUM(OmpDeleteModifier, Value)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
   NODE(parser, OmpTaskDependenceType)
@@ -628,6 +634,8 @@ class ParseTreeDumper {
   NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
   NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpPresentModifier)
+  NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
@@ -637,6 +645,10 @@ class ParseTreeDumper {
   NODE(parser, OmpInReductionClause)
   NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpRefModifier)
+  NODE_ENUM(OmpRefModifier, Value)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpInitializerProc)
@@ -673,6 +685,8 @@ class ParseTreeDumper {
   NODE(parser, OmpSectionsDirective)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpxHoldModifier)
+  NODE_ENUM(OmpxHoldModifier, Value)
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)

diff  --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cc1d032f94d4a..0b3dec1010312 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3756,6 +3756,19 @@ struct OmpAllocatorComplexModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// always-modifier ->
+//    ALWAYS                                        // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpAlwaysModifier {
+  ENUM_CLASS(Value, Always)
+  WRAPPER_CLASS_BOILERPLATE(OmpAlwaysModifier, Value);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3767,17 +3780,29 @@ struct OmpChunkModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpChunkModifier, Value);
 };
 
-// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
-// iterator-specifier ->
-//    [iterator-type] iterator-identifier
-//        = range-specification |                   // since 5.0
-//    [iterator-type ::] iterator-identifier
-//        = range-specification                     // since 5.2
-struct OmpIteratorSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
-  CharBlock source;
-  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+// close-modifier ->
+//    CLOSE                                         // since 5.0
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpCloseModifier {
+  ENUM_CLASS(Value, Close)
+  WRAPPER_CLASS_BOILERPLATE(OmpCloseModifier, Value);
+};
+
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// delete-modifier ->
+//    DELETE                                        // since 6.0
+//
+// Until 5.2, it was a part of map-type.
+struct OmpDeleteModifier {
+  ENUM_CLASS(Value, Delete)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeleteModifier, Value);
 };
 
 // Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289]
@@ -3867,6 +3892,19 @@ struct OmpInteropType {
   WRAPPER_CLASS_BOILERPLATE(OmpInteropType, Value);
 };
 
+// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+//
+// iterator-specifier ->
+//    [iterator-type] iterator-identifier
+//        = range-specification |                   // since 5.0
+//    [iterator-type ::] iterator-identifier
+//        = range-specification                     // since 5.2
+struct OmpIteratorSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
+  CharBlock source;
+  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+};
+
 // Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
 //
 // iterator-modifier ->
@@ -3901,21 +3939,28 @@ struct OmpMapper {
   WRAPPER_CLASS_BOILERPLATE(OmpMapper, Name);
 };
 
-// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
 // map-type ->
-//    ALLOC | DELETE | FROM | RELEASE | TO | TOFROM // since 4.5
+//    ALLOC | DELETE | RELEASE |                    // since 4.5, until 5.2
+//    FROM | TO | TOFROM |                          // since 4.5
+//    STORAGE                                       // since 6.0
+//
+// Since 6.0 DELETE is a separate delete-modifier.
 struct OmpMapType {
-  ENUM_CLASS(Value, Alloc, Delete, From, Release, To, Tofrom);
+  ENUM_CLASS(Value, Alloc, Delete, From, Release, Storage, To, Tofrom);
   WRAPPER_CLASS_BOILERPLATE(OmpMapType, Value);
 };
 
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-type-modifier ->
-//    ALWAYS |                                      // since 4.5
-//    CLOSE |                                       // since 5.0
-//    PRESENT                                       // since 5.1
+//    ALWAYS |                                      // since 4.5, until 5.2
+//    CLOSE |                                       // since 5.0, until 5.2
+//    PRESENT                                       // since 5.1, until 5.2
+// Since 6.0 the map-type-modifier has been split into individual modifiers.
+//
 struct OmpMapTypeModifier {
   ENUM_CLASS(Value, Always, Close, Present, Ompx_Hold)
   WRAPPER_CLASS_BOILERPLATE(OmpMapTypeModifier, Value);
@@ -3954,6 +3999,19 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// present-modifier ->
+//    PRESENT                                       // since 5.1
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpPresentModifier {
+  ENUM_CLASS(Value, Present)
+  WRAPPER_CLASS_BOILERPLATE(OmpPresentModifier, Value);
+};
+
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3963,6 +4021,26 @@ struct OmpReductionModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpReductionModifier, Value);
 };
 
+// Ref: [6.0:279-288]
+//
+// ref-modifier ->
+//    REF_PTEE | REF_PTR | REF_PTR_PTEE             // since 6.0
+//
+struct OmpRefModifier {
+  ENUM_CLASS(Value, Ref_Ptee, Ref_Ptr, Ref_Ptr_Ptee)
+  WRAPPER_CLASS_BOILERPLATE(OmpRefModifier, Value);
+};
+
+// Ref: [6.0:279-288]
+//
+// self-modifier ->
+//    SELF                                          // since 6.0
+//
+struct OmpSelfModifier {
+  ENUM_CLASS(Value, Self)
+  WRAPPER_CLASS_BOILERPLATE(OmpSelfModifier, Value);
+};
+
 // Ref: [5.2:117-120]
 //
 // step-complex-modifier ->
@@ -4001,6 +4079,19 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension:
+// https://openmp.llvm.org//openacc/OpenMPExtensions.html#ompx-hold
+//
+// ompx-hold-modifier ->
+//    OMPX_HOLD                                     // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpxHoldModifier {
+  ENUM_CLASS(Value, Ompx_Hold)
+  WRAPPER_CLASS_BOILERPLATE(OmpxHoldModifier, Value);
+};
+
 // context-selector
 using OmpContextSelector = traits::OmpContextSelectorSpecification;
 } // namespace modifier
@@ -4376,13 +4467,25 @@ struct OmpLinearClause {
 // map-clause ->
 //    MAP([modifier...:] locator-list)              // since 4.5
 // modifier ->
-//    map-type-modifier |                           // since 4.5
+//    map-type-modifier [replaced] |                // since 4.5, until 5.2
+//    always-modifier |                             // since 6.0
+//    close-modifier |                              // since 6.0
+//    delete-modifier |                             // since 6.0
+//    present-modifier |                            // since 6.0
+//    ref-modifier |                                // since 6.0
+//    self-modifier |                               // since 6.0
 //    mapper |                                      // since 5.0
 //    iterator |                                    // since 5.1
 //    map-type                                      // since 4.5
+//    ompx-hold-modifier |                          // since 6.0
+//
+// Since 6.0 the map-type-modifier has been split into individual modifiers,
+// and delete-modifier has been split from map-type.
 struct OmpMapClause {
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  MODIFIER_BOILERPLATE(OmpMapTypeModifier, OmpMapper, OmpIterator, OmpMapType);
+  MODIFIER_BOILERPLATE(OmpAlwaysModifier, OmpCloseModifier, OmpDeleteModifier,
+      OmpMapTypeModifier, OmpPresentModifier, OmpRefModifier, OmpSelfModifier,
+      OmpMapper, OmpIterator, OmpMapType, OmpxHoldModifier);
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 

diff  --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 4fbd80f989e72..a9fe911ef8807 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -71,8 +71,11 @@ DECLARE_DESCRIPTOR(parser::OmpAlignment);
 DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
+DECLARE_DESCRIPTOR(parser::OmpAlwaysModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpCloseModifier);
 DECLARE_DESCRIPTOR(parser::OmpContextSelector);
+DECLARE_DESCRIPTOR(parser::OmpDeleteModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
@@ -88,12 +91,16 @@ DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
 DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
+DECLARE_DESCRIPTOR(parser::OmpPresentModifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
+DECLARE_DESCRIPTOR(parser::OmpRefModifier);
+DECLARE_DESCRIPTOR(parser::OmpSelfModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpVariableCategory);
+DECLARE_DESCRIPTOR(parser::OmpxHoldModifier);
 
 #undef DECLARE_DESCRIPTOR
 

diff  --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..d349d8ceb0bb5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -274,6 +274,10 @@ TYPE_PARSER( //
     construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}) ||
     construct<OmpTypeSpecifier>(Parser<TypeSpec>{}))
 
+// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
+TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
+    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
+
 TYPE_PARSER(construct<OmpReductionSpecifier>( //
     Parser<OmpReductionIdentifier>{},
     ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
@@ -442,9 +446,18 @@ TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
 
 TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
 
+TYPE_PARSER(construct<OmpAlwaysModifier>( //
+    "ALWAYS" >> pure(OmpAlwaysModifier::Value::Always)))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpCloseModifier>( //
+    "CLOSE" >> pure(OmpCloseModifier::Value::Close)))
+
+TYPE_PARSER(construct<OmpDeleteModifier>( //
+    "DELETE" >> pure(OmpDeleteModifier::Value::Delete)))
+
 TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
@@ -502,26 +515,16 @@ TYPE_PARSER(construct<OmpLinearModifier>( //
 TYPE_PARSER(construct<OmpMapper>( //
     "MAPPER"_tok >> parenthesized(Parser<ObjectName>{})))
 
-// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+// map-type -> ALLOC | DELETE | FROM | RELEASE | STORAGE | TO | TOFROM
 TYPE_PARSER(construct<OmpMapType>( //
     "ALLOC" >> pure(OmpMapType::Value::Alloc) ||
-    "DELETE" >> pure(OmpMapType::Value::Delete) ||
+    // Parse "DELETE" as OmpDeleteModifier
     "FROM" >> pure(OmpMapType::Value::From) ||
     "RELEASE" >> pure(OmpMapType::Value::Release) ||
+    "STORAGE" >> pure(OmpMapType::Value::Storage) ||
     "TO"_id >> pure(OmpMapType::Value::To) ||
     "TOFROM" >> pure(OmpMapType::Value::Tofrom)))
 
-// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
-TYPE_PARSER(construct<OmpMapTypeModifier>(
-    "ALWAYS" >> pure(OmpMapTypeModifier::Value::Always) ||
-    "CLOSE" >> pure(OmpMapTypeModifier::Value::Close) ||
-    "OMPX_HOLD" >> pure(OmpMapTypeModifier::Value::Ompx_Hold) ||
-    "PRESENT" >> pure(OmpMapTypeModifier::Value::Present)))
-
-// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
-TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
-    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
-
 TYPE_PARSER(construct<OmpOrderModifier>(
     "REPRODUCIBLE" >> pure(OmpOrderModifier::Value::Reproducible) ||
     "UNCONSTRAINED" >> pure(OmpOrderModifier::Value::Unconstrained)))
@@ -534,11 +537,22 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
 TYPE_PARSER(construct<OmpPrescriptiveness>(
     "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
 
+TYPE_PARSER(construct<OmpPresentModifier>( //
+    "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
     "DEFAULT" >> pure(OmpReductionModifier::Value::Default)))
 
+TYPE_PARSER(construct<OmpRefModifier>( //
+    "REF_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptee) ||
+    "REF_PTR"_id >> pure(OmpRefModifier::Value::Ref_Ptr) ||
+    "REF_PTR_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptr_Ptee)))
+
+TYPE_PARSER(construct<OmpSelfModifier>( //
+    "SELF" >> pure(OmpSelfModifier::Value::Self)))
+
 TYPE_PARSER(construct<OmpStepComplexModifier>( //
     "STEP" >> parenthesized(scalarIntExpr)))
 
@@ -559,6 +573,9 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "POINTER" >> pure(OmpVariableCategory::Value::Pointer) ||
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
+TYPE_PARSER(construct<OmpxHoldModifier>( //
+    "OMPX_HOLD" >> pure(OmpxHoldModifier::Value::Ompx_Hold)))
+
 // This could be auto-generated.
 TYPE_PARSER(
     sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
@@ -611,10 +628,16 @@ TYPE_PARSER(sourced(
     construct<OmpLinearClause::Modifier>(Parser<OmpStepSimpleModifier>{})))
 
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
-    sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
+    sourced(construct<OmpMapClause::Modifier>(Parser<OmpAlwaysModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpCloseModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpDeleteModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpPresentModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpRefModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpSelfModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpIterator>{}) ||
-        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{})))))
+        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpxHoldModifier>{})))))
 
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))

diff  --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index fbe89c668fc13..8ed16905b5099 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -3007,8 +3007,15 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
+  WALK_NESTED_ENUM(OmpAlwaysModifier, Value)
+  WALK_NESTED_ENUM(OmpCloseModifier, Value)
+  WALK_NESTED_ENUM(OmpDeleteModifier, Value)
+  WALK_NESTED_ENUM(OmpPresentModifier, Value)
+  WALK_NESTED_ENUM(OmpRefModifier, Value)
+  WALK_NESTED_ENUM(OmpSelfModifier, Value)
   WALK_NESTED_ENUM(OmpTraitSelectorName, Value)
   WALK_NESTED_ENUM(OmpTraitSetSelectorName, Value)
+  WALK_NESTED_ENUM(OmpxHoldModifier, Value)
 
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {

diff  --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index cf05d8463277f..bba4d72a702e9 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -88,6 +88,8 @@ class CanonicalizationOfOmp {
     CanonicalizeUtilityConstructs(spec);
   }
 
+  void Post(parser::OmpMapClause &map) { CanonicalizeMapModifiers(map); }
+
 private:
   template <typename T> T *GetConstructIf(parser::ExecutionPartConstruct &x) {
     if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
@@ -390,6 +392,42 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.

diff  --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index c45d6bbbf9062..23350c522a775 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -15,7 +15,9 @@ class Messages;
 } // namespace Fortran::parser
 
 namespace Fortran::semantics {
+class SemanticsContext;
+
 bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
-}
+} // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_

diff  --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index c84e832ee52a1..336ce4beb24ba 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -140,6 +140,22 @@ OmpGetDescriptor<parser::OmpAllocatorSimpleModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlwaysModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"always-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   static const OmpModifierDescriptor desc{
@@ -156,6 +172,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpCloseModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"close-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   static const OmpModifierDescriptor desc{
@@ -173,6 +205,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDeleteModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"delete-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique, OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   static const OmpModifierDescriptor desc{
@@ -347,6 +396,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapType>() {
       /*props=*/
       {
           {45, {OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
       },
       /*clauses=*/
       {
@@ -367,6 +417,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapTypeModifier>() {
       /*clauses=*/
       {
           {45, {Clause::OMPC_map}},
+          {60, {}},
       },
   };
   return desc;
@@ -420,6 +471,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPrescriptiveness>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPresentModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"present-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpReductionIdentifier>() {
@@ -456,6 +523,38 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpReductionModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpRefModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ref-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpSelfModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"self-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpStepComplexModifier>() {
@@ -522,4 +621,20 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpVariableCategory>() {
   };
   return desc;
 }
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpxHoldModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ompx-hold-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
 } // namespace Fortran::semantics

diff  --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 521c7432d9fbb..332291a2aefe8 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -749,6 +749,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
+      default:
+        break;
       }
     }
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};

diff  --git a/flang/test/Parser/OpenMP/map-modifiers-v60.f90 b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..bc80886780d46
--- /dev/null
+++ b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,113 @@
+!RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x
+  !$omp target map(always, close, delete, present, ompx_hold: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(ALWAYS, CLOSE, DELETE, PRESENT, OMPX_HOLD: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpAlwaysModifier -> Value = Always
+!PARSE-TREE: | | Modifier -> OmpCloseModifier -> Value = Close
+!PARSE-TREE: | | Modifier -> OmpDeleteModifier -> Value = Delete
+!PARSE-TREE: | | Modifier -> OmpPresentModifier -> Value = Present
+!PARSE-TREE: | | Modifier -> OmpxHoldModifier -> Value = Ompx_Hold
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f01(x)
+  integer :: x
+  !$omp target map(self, storage: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: !$OMP TARGET  MAP(SELF, STORAGE: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpSelfModifier -> Value = Self
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = Storage
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f03(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f04(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f04 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'


        

From flang-commits at lists.llvm.org  Mon Jul 21 08:55:44 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 08:55:44 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Parse OpenMP 6.0 map
	modifiers (PR #149134)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149134@github.com>
Message-ID: <687e6300.170a0220.4b401.df78@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/149134

From flang-commits at lists.llvm.org  Mon Jul 21 08:55:49 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 08:55:49 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <687e6305.630a0220.265440.cb75@mx.google.com>


https://github.com/kparzysz edited https://github.com/llvm/llvm-project/pull/149137

From flang-commits at lists.llvm.org  Mon Jul 21 09:11:08 2025
From: flang-commits at lists.llvm.org (Fabian Mora via flang-commits)
Date: Mon, 21 Jul 2025 09:11:08 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e669c.170a0220.1483cf.f370@mx.google.com>


fabianmcg wrote:

> 1. There's a circular dependency between `LLVMInterfaces` and `LLVMAttrs` in that the new `TargetAttrInterface` requires `LLVMTargetFeaturesAttr` (for the `features` of a target) and there are attrs in `LLVMAttrs` that require interfaces from `LLVMInterfaces`. I break the dependency by moving `LLVMTargetFeaturesAttr` to its own file and having a dedicated CMake-rule to generate `LLVMOpsAttrDefs.h.inc` by way of `LLVMAttrAndEnumDefs.td`

I'd argue is preferable to create `TargetAttrInterfece.td` to break the dependency. That way other dialects wanting to implement the interface don't have to take `LLVMInterfaces` as a dep.


> 2\. For some reason, the parser generated for `` let assemblyFormat = [{`<` `triple` `=` $triple (`,` `chip` `=` $chip^)? (`,` qualified($features)^)? `>`}]; `` does not actually make the `chip` clause optional. That is, it greedily assumes that as it saw `,` from the `chip` clause it is now committed to it. For example, `#llvm.target<triple = "x86_64-unknown-linux", #llvm.target_features<["+mmx", "+sse"]>>` yields an `expected 'chip'` parser error.

I'd recommend using the struct directive, that way the elements can appear in any order, and should handle the optional elements correctly.

https://mlir.llvm.org/docs/DefiningDialects/AttributesAndTypes/#struct-directive

If I recall correctly, the above issue is caused because the parser generated MLIR uses only one lookahead token, so the parser doesn't know whether it's parsing chip or features because they both start with a `,`.

Also, I'm not sure whether making `chip` optional is a good idea.



> 3\. How do I properly hang a `unique_ptr<llvm::TargetMachine>` of off `TargetAttr`? The current bare pointer works but probably is not safe, e.g. in case the context gets deleted. See discussion with @rengolin above.

I imagine you want this for caching the TM. But why do you want this? When is it going to be reused? I think we should use to import the DL for now.

> 6\. Test cases should be made aware of which backends the compiled `llvm-project` has available and be skipped if not applicable. CI is happy for now, which is actually wrong.

See this:
https://github.com/llvm/llvm-project/blob/main/mlir/test/Dialect/GPU/module-to-binary-nvvm.mlir#L1


https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 08:57:08 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 08:57:08 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <687e6354.050a0220.1430fe.c9e6@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/149137

>From 0929234481b4d6d015381fda490c05bbeff1e22f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 08:04:17 -0500
Subject: [PATCH 01/10] [flang][OpenMP] Parse OpenMP 6.0 map modifiers

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers.
The old "map-type-modifier" is retained, but is no longer created in
parsing. It will remain to take advantage of the preexisting modifier
validation for older versions: when the OpenMP version is < 6.0,
the modifiers will be rewritten back as map-type-modifiers (or map-
type in case of "delete").

In this patch the modifiers will always be rewritten in the older
format to isolate these changes to parsing as much as possible.
---
 flang/include/flang/Parser/dump-parse-tree.h  |  14 ++
 flang/include/flang/Parser/parse-tree.h       | 138 +++++++++++++++---
 .../flang/Semantics/openmp-modifiers.h        |   7 +
 flang/lib/Parser/openmp-parsers.cpp           |  53 +++++--
 flang/lib/Parser/unparse.cpp                  |   7 +
 flang/lib/Semantics/canonicalize-omp.cpp      |  48 +++++-
 flang/lib/Semantics/canonicalize-omp.h        |   9 +-
 flang/lib/Semantics/openmp-modifiers.cpp      | 115 +++++++++++++++
 flang/lib/Semantics/resolve-directives.cpp    |   2 +
 flang/lib/Semantics/semantics.cpp             |   3 +-
 .../test/Parser/OpenMP/map-modifiers-v60.f90  | 113 ++++++++++++++
 11 files changed, 466 insertions(+), 43 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/map-modifiers-v60.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 32b6ca45609b6..23e35d106c077 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -529,6 +529,8 @@ class ParseTreeDumper {
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlwaysModifier)
+  NODE_ENUM(OmpAlwaysModifier, Value)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
@@ -546,6 +548,8 @@ class ParseTreeDumper {
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
   NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpCloseModifier)
+  NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -561,6 +565,8 @@ class ParseTreeDumper {
   NODE(parser, OmpDefaultmapClause)
   NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE(parser, OmpDeleteModifier)
+  NODE_ENUM(OmpDeleteModifier, Value)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
   NODE(parser, OmpTaskDependenceType)
@@ -628,6 +634,8 @@ class ParseTreeDumper {
   NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
   NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpPresentModifier)
+  NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
@@ -637,6 +645,10 @@ class ParseTreeDumper {
   NODE(parser, OmpInReductionClause)
   NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpRefModifier)
+  NODE_ENUM(OmpRefModifier, Value)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpInitializerProc)
@@ -673,6 +685,8 @@ class ParseTreeDumper {
   NODE(parser, OmpSectionsDirective)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpxHoldModifier)
+  NODE_ENUM(OmpxHoldModifier, Value)
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cc1d032f94d4a..92a449d2f9d49 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3756,6 +3756,19 @@ struct OmpAllocatorComplexModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// always-modifier ->
+//    ALWAYS                                        // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpAlwaysModifier {
+  ENUM_CLASS(Value, Always)
+  WRAPPER_CLASS_BOILERPLATE(OmpAlwaysModifier, Value);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3767,17 +3780,29 @@ struct OmpChunkModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpChunkModifier, Value);
 };
 
-// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
-// iterator-specifier ->
-//    [iterator-type] iterator-identifier
-//        = range-specification |                   // since 5.0
-//    [iterator-type ::] iterator-identifier
-//        = range-specification                     // since 5.2
-struct OmpIteratorSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
-  CharBlock source;
-  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+// close-modifier ->
+//    CLOSE                                         // since 5.0
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpCloseModifier {
+  ENUM_CLASS(Value, Close)
+  WRAPPER_CLASS_BOILERPLATE(OmpCloseModifier, Value);
+};
+
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// delete-modifier ->
+//    DELETE                                        // since 6.0
+//
+// Until 5.2, it was a part of map-type.
+struct OmpDeleteModifier {
+  ENUM_CLASS(Value, Delete)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeleteModifier, Value);
 };
 
 // Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289]
@@ -3867,6 +3892,19 @@ struct OmpInteropType {
   WRAPPER_CLASS_BOILERPLATE(OmpInteropType, Value);
 };
 
+// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+//
+// iterator-specifier ->
+//    [iterator-type] iterator-identifier
+//        = range-specification |                   // since 5.0
+//    [iterator-type ::] iterator-identifier
+//        = range-specification                     // since 5.2
+struct OmpIteratorSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
+  CharBlock source;
+  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+};
+
 // Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
 //
 // iterator-modifier ->
@@ -3901,21 +3939,28 @@ struct OmpMapper {
   WRAPPER_CLASS_BOILERPLATE(OmpMapper, Name);
 };
 
-// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
 // map-type ->
-//    ALLOC | DELETE | FROM | RELEASE | TO | TOFROM // since 4.5
+//    ALLOC | DELETE | RELEASE |                    // since 4.5, until 5.2
+//    FROM | TO | TOFROM |                          // since 4.5
+//    STORAGE                                       // since 6.0
+//
+// Since 6.0 DELETE is a separate delete-modifier.
 struct OmpMapType {
-  ENUM_CLASS(Value, Alloc, Delete, From, Release, To, Tofrom);
+  ENUM_CLASS(Value, Alloc, Delete, From, Release, Storage, To, Tofrom);
   WRAPPER_CLASS_BOILERPLATE(OmpMapType, Value);
 };
 
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-type-modifier ->
-//    ALWAYS |                                      // since 4.5
-//    CLOSE |                                       // since 5.0
-//    PRESENT                                       // since 5.1
+//    ALWAYS |                                      // since 4.5, until 5.2
+//    CLOSE |                                       // since 5.0, until 5.2
+//    PRESENT                                       // since 5.1, until 5.2
+// Since 6.0 the map-type-modifier has been split into individual modifiers.
+//
 struct OmpMapTypeModifier {
   ENUM_CLASS(Value, Always, Close, Present, Ompx_Hold)
   WRAPPER_CLASS_BOILERPLATE(OmpMapTypeModifier, Value);
@@ -3954,6 +3999,19 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// present-modifier ->
+//    PRESENT                                       // since 5.1
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpPresentModifier {
+  ENUM_CLASS(Value, Present)
+  WRAPPER_CLASS_BOILERPLATE(OmpPresentModifier, Value);
+};
+
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3963,6 +4021,26 @@ struct OmpReductionModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpReductionModifier, Value);
 };
 
+// Ref: [6.0:279-288]
+//
+// ref-modifier ->
+//    REF_PTEE | REF_PTR | REF_PTR_PTEE             // since 6.0
+//
+struct OmpRefModifier {
+  ENUM_CLASS(Value, Ref_Ptee, Ref_Ptr, Ref_Ptr_Ptee)
+  WRAPPER_CLASS_BOILERPLATE(OmpRefModifier, Value);
+};
+
+// Ref: [6.0:279-288]
+//
+// self-modifier ->
+//    SELF                                          // since 6.0
+//
+struct OmpSelfModifier {
+  ENUM_CLASS(Value, Self)
+  WRAPPER_CLASS_BOILERPLATE(OmpSelfModifier, Value);
+};
+
 // Ref: [5.2:117-120]
 //
 // step-complex-modifier ->
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
+//
+// ompx-hold-modifier ->
+//    OMPX_HOLD                                     // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpxHoldModifier {
+  ENUM_CLASS(Value, Ompx_Hold)
+  WRAPPER_CLASS_BOILERPLATE(OmpxHoldModifier, Value);
+};
+
 // context-selector
 using OmpContextSelector = traits::OmpContextSelectorSpecification;
 } // namespace modifier
@@ -4376,13 +4466,25 @@ struct OmpLinearClause {
 // map-clause ->
 //    MAP([modifier...:] locator-list)              // since 4.5
 // modifier ->
-//    map-type-modifier |                           // since 4.5
+//    map-type-modifier [replaced] |                // since 4.5, until 5.2
+//    always-modifier |                             // since 6.0
+//    close-modifier |                              // since 6.0
+//    delete-modifier |                             // since 6.0
+//    present-modifier |                            // since 6.0
+//    ref-modifier |                                // since 6.0
+//    self-modifier |                               // since 6.0
 //    mapper |                                      // since 5.0
 //    iterator |                                    // since 5.1
 //    map-type                                      // since 4.5
+//    ompx-hold-modifier |                          // since 6.0
+//
+// Since 6.0 the map-type-modifier has been split into individual modifiers,
+// and delete-modifier has been split from map-type.
 struct OmpMapClause {
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  MODIFIER_BOILERPLATE(OmpMapTypeModifier, OmpMapper, OmpIterator, OmpMapType);
+  MODIFIER_BOILERPLATE(OmpAlwaysModifier, OmpCloseModifier, OmpDeleteModifier,
+      OmpMapTypeModifier, OmpPresentModifier, OmpRefModifier, OmpSelfModifier,
+      OmpMapper, OmpIterator, OmpMapType, OmpxHoldModifier);
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 4fbd80f989e72..a9fe911ef8807 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -71,8 +71,11 @@ DECLARE_DESCRIPTOR(parser::OmpAlignment);
 DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
+DECLARE_DESCRIPTOR(parser::OmpAlwaysModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpCloseModifier);
 DECLARE_DESCRIPTOR(parser::OmpContextSelector);
+DECLARE_DESCRIPTOR(parser::OmpDeleteModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
@@ -88,12 +91,16 @@ DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
 DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
+DECLARE_DESCRIPTOR(parser::OmpPresentModifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
+DECLARE_DESCRIPTOR(parser::OmpRefModifier);
+DECLARE_DESCRIPTOR(parser::OmpSelfModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpVariableCategory);
+DECLARE_DESCRIPTOR(parser::OmpxHoldModifier);
 
 #undef DECLARE_DESCRIPTOR
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..d349d8ceb0bb5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -274,6 +274,10 @@ TYPE_PARSER( //
     construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}) ||
     construct<OmpTypeSpecifier>(Parser<TypeSpec>{}))
 
+// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
+TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
+    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
+
 TYPE_PARSER(construct<OmpReductionSpecifier>( //
     Parser<OmpReductionIdentifier>{},
     ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
@@ -442,9 +446,18 @@ TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
 
 TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
 
+TYPE_PARSER(construct<OmpAlwaysModifier>( //
+    "ALWAYS" >> pure(OmpAlwaysModifier::Value::Always)))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpCloseModifier>( //
+    "CLOSE" >> pure(OmpCloseModifier::Value::Close)))
+
+TYPE_PARSER(construct<OmpDeleteModifier>( //
+    "DELETE" >> pure(OmpDeleteModifier::Value::Delete)))
+
 TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
@@ -502,26 +515,16 @@ TYPE_PARSER(construct<OmpLinearModifier>( //
 TYPE_PARSER(construct<OmpMapper>( //
     "MAPPER"_tok >> parenthesized(Parser<ObjectName>{})))
 
-// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+// map-type -> ALLOC | DELETE | FROM | RELEASE | STORAGE | TO | TOFROM
 TYPE_PARSER(construct<OmpMapType>( //
     "ALLOC" >> pure(OmpMapType::Value::Alloc) ||
-    "DELETE" >> pure(OmpMapType::Value::Delete) ||
+    // Parse "DELETE" as OmpDeleteModifier
     "FROM" >> pure(OmpMapType::Value::From) ||
     "RELEASE" >> pure(OmpMapType::Value::Release) ||
+    "STORAGE" >> pure(OmpMapType::Value::Storage) ||
     "TO"_id >> pure(OmpMapType::Value::To) ||
     "TOFROM" >> pure(OmpMapType::Value::Tofrom)))
 
-// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
-TYPE_PARSER(construct<OmpMapTypeModifier>(
-    "ALWAYS" >> pure(OmpMapTypeModifier::Value::Always) ||
-    "CLOSE" >> pure(OmpMapTypeModifier::Value::Close) ||
-    "OMPX_HOLD" >> pure(OmpMapTypeModifier::Value::Ompx_Hold) ||
-    "PRESENT" >> pure(OmpMapTypeModifier::Value::Present)))
-
-// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
-TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
-    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
-
 TYPE_PARSER(construct<OmpOrderModifier>(
     "REPRODUCIBLE" >> pure(OmpOrderModifier::Value::Reproducible) ||
     "UNCONSTRAINED" >> pure(OmpOrderModifier::Value::Unconstrained)))
@@ -534,11 +537,22 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
 TYPE_PARSER(construct<OmpPrescriptiveness>(
     "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
 
+TYPE_PARSER(construct<OmpPresentModifier>( //
+    "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
     "DEFAULT" >> pure(OmpReductionModifier::Value::Default)))
 
+TYPE_PARSER(construct<OmpRefModifier>( //
+    "REF_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptee) ||
+    "REF_PTR"_id >> pure(OmpRefModifier::Value::Ref_Ptr) ||
+    "REF_PTR_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptr_Ptee)))
+
+TYPE_PARSER(construct<OmpSelfModifier>( //
+    "SELF" >> pure(OmpSelfModifier::Value::Self)))
+
 TYPE_PARSER(construct<OmpStepComplexModifier>( //
     "STEP" >> parenthesized(scalarIntExpr)))
 
@@ -559,6 +573,9 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "POINTER" >> pure(OmpVariableCategory::Value::Pointer) ||
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
+TYPE_PARSER(construct<OmpxHoldModifier>( //
+    "OMPX_HOLD" >> pure(OmpxHoldModifier::Value::Ompx_Hold)))
+
 // This could be auto-generated.
 TYPE_PARSER(
     sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
@@ -611,10 +628,16 @@ TYPE_PARSER(sourced(
     construct<OmpLinearClause::Modifier>(Parser<OmpStepSimpleModifier>{})))
 
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
-    sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
+    sourced(construct<OmpMapClause::Modifier>(Parser<OmpAlwaysModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpCloseModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpDeleteModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpPresentModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpRefModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpSelfModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpIterator>{}) ||
-        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{})))))
+        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpxHoldModifier>{})))))
 
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index fbe89c668fc13..8ed16905b5099 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -3007,8 +3007,15 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
+  WALK_NESTED_ENUM(OmpAlwaysModifier, Value)
+  WALK_NESTED_ENUM(OmpCloseModifier, Value)
+  WALK_NESTED_ENUM(OmpDeleteModifier, Value)
+  WALK_NESTED_ENUM(OmpPresentModifier, Value)
+  WALK_NESTED_ENUM(OmpRefModifier, Value)
+  WALK_NESTED_ENUM(OmpSelfModifier, Value)
   WALK_NESTED_ENUM(OmpTraitSelectorName, Value)
   WALK_NESTED_ENUM(OmpTraitSetSelectorName, Value)
+  WALK_NESTED_ENUM(OmpxHoldModifier, Value)
 
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index cf05d8463277f..46aaab19ded0a 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -88,6 +89,8 @@ class CanonicalizationOfOmp {
     CanonicalizeUtilityConstructs(spec);
   }
 
+  void Post(parser::OmpMapClause &map) { CanonicalizeMapModifiers(map); }
+
 private:
   template <typename T> T *GetConstructIf(parser::ExecutionPartConstruct &x) {
     if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index c45d6bbbf9062..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,11 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
-}
+class SemanticsContext;
+
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+} // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index c84e832ee52a1..336ce4beb24ba 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -140,6 +140,22 @@ OmpGetDescriptor<parser::OmpAllocatorSimpleModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlwaysModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"always-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   static const OmpModifierDescriptor desc{
@@ -156,6 +172,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpCloseModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"close-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   static const OmpModifierDescriptor desc{
@@ -173,6 +205,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDeleteModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"delete-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique, OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   static const OmpModifierDescriptor desc{
@@ -347,6 +396,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapType>() {
       /*props=*/
       {
           {45, {OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
       },
       /*clauses=*/
       {
@@ -367,6 +417,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapTypeModifier>() {
       /*clauses=*/
       {
           {45, {Clause::OMPC_map}},
+          {60, {}},
       },
   };
   return desc;
@@ -420,6 +471,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPrescriptiveness>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPresentModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"present-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpReductionIdentifier>() {
@@ -456,6 +523,38 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpReductionModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpRefModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ref-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpSelfModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"self-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpStepComplexModifier>() {
@@ -522,4 +621,20 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpVariableCategory>() {
   };
   return desc;
 }
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpxHoldModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ompx-hold-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 151f4ccae634e..0b860314b4a1f 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -734,6 +734,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
+      default:
+        break;
       }
     }
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..d0a5b200d0d8e 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -643,8 +643,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}
diff --git a/flang/test/Parser/OpenMP/map-modifiers-v60.f90 b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..bc80886780d46
--- /dev/null
+++ b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,113 @@
+!RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x
+  !$omp target map(always, close, delete, present, ompx_hold: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(ALWAYS, CLOSE, DELETE, PRESENT, OMPX_HOLD: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpAlwaysModifier -> Value = Always
+!PARSE-TREE: | | Modifier -> OmpCloseModifier -> Value = Close
+!PARSE-TREE: | | Modifier -> OmpDeleteModifier -> Value = Delete
+!PARSE-TREE: | | Modifier -> OmpPresentModifier -> Value = Present
+!PARSE-TREE: | | Modifier -> OmpxHoldModifier -> Value = Ompx_Hold
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f01(x)
+  integer :: x
+  !$omp target map(self, storage: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: !$OMP TARGET  MAP(SELF, STORAGE: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpSelfModifier -> Value = Self
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = Storage
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f03(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f04(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f04 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'

>From acc1a7d3da0d2bd5a60096a9f5dc27338342b952 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 08:04:17 -0500
Subject: [PATCH 02/10] [flang][OpenMP] Sema checks, lowering with new format
 of MAP modifiers

OpenMP 6.0 has changed the modifiers on the MAP clause. Previous patch
has introduced parsing support for them. This patch introduces processing
of the new forms in semantic checks and in lowering. This only applies
to existing modifiers, which were updated in the 6.0 spec. Any of the
newly introduced modifiers (SELF and REF) are ignored.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  16 +--
 flang/lib/Lower/OpenMP/Clauses.cpp            | 113 ++++++++++++------
 flang/lib/Semantics/canonicalize-omp.cpp      |   5 +
 flang/lib/Semantics/check-omp-structure.cpp   | 107 +++++++++++------
 flang/lib/Semantics/check-omp-structure.h     |   4 +-
 flang/lib/Semantics/openmp-utils.cpp          |  25 ++++
 flang/lib/Semantics/openmp-utils.h            |   3 +
 flang/lib/Semantics/resolve-directives.cpp    |  50 ++++++--
 .../Semantics/OpenMP/combined-constructs.f90  |   8 +-
 .../Semantics/OpenMP/device-constructs.f90    |   6 +-
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |   9 +-
 .../Frontend/OpenMP/ConstructDecompositionT.h |   5 +-
 llvm/include/llvm/Frontend/OpenMP/OMP.h       |   2 +
 llvm/lib/Frontend/OpenMP/OMP.cpp              |  14 +++
 14 files changed, 260 insertions(+), 107 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 74087d42a8e6e..ec71014c36093 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1315,7 +1315,8 @@ bool ClauseProcessor::processMap(
                      const parser::CharBlock &source) {
     using Map = omp::clause::Map;
     mlir::Location clauseLocation = converter.genLocation(source);
-    const auto &[mapType, typeMods, mappers, iterator, objects] = clause.t;
+    const auto &[mapType, typeMods, refMod, mappers, iterator, objects] =
+        clause.t;
     llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
         llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
     std::string mapperIdName = "__implicit_mapper";
@@ -1342,16 +1343,13 @@ bool ClauseProcessor::processMap(
       mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
                      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
       break;
-    case Map::MapType::Alloc:
-    case Map::MapType::Release:
+    case Map::MapType::Storage:
       // alloc and release is the default map_type for the Target Data
       // Ops, i.e. if no bits for map_type is supplied then alloc/release
-      // is implicitly assumed based on the target directive. Default
-      // value for Target Data and Enter Data is alloc and for Exit Data
-      // it is release.
+      // (aka storage in 6.0+) is implicitly assumed based on the target
+      // directive. Default value for Target Data and Enter Data is alloc
+      // and for Exit Data it is release.
       break;
-    case Map::MapType::Delete:
-      mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
     }
 
     if (typeMods) {
@@ -1362,6 +1360,8 @@ bool ClauseProcessor::processMap(
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
       if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Close))
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
+      if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Delete))
+        mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
       if (llvm::is_contained(*typeMods, Map::MapTypeModifier::OmpxHold))
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
     }
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 22a07219d3a50..18e49719b6013 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1001,64 +1001,105 @@ Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpMapType::Value, Map::MapType,
+      convert1, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
       // clang-format off
-      MS(Alloc,    Alloc)
-      MS(Delete,   Delete)
-      MS(From,     From)
-      MS(Release,  Release)
-      MS(To,       To)
-      MS(Tofrom,   Tofrom)
+      MS(Always,    Always)
+      MS(Close,     Close)
+      MS(Ompx_Hold, OmpxHold)
+      MS(Present,   Present)
       // clang-format on
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert2, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
+      convert2, parser::OmpRefModifier::Value, Map::RefModifier,
       // clang-format off
-      MS(Always,    Always)
-      MS(Close,     Close)
-      MS(Ompx_Hold, OmpxHold)
-      MS(Present,   Present)
+      MS(Ref_Ptee,     RefPtee)
+      MS(Ref_Ptr,      RefPtr)
+      MS(Ref_Ptr_Ptee, RefPtrPtee)
       // clang-format on
   );
 
+  // Treat always, close, present, self, delete modifiers as map-type-
+  // modifiers.
   auto &mods = semantics::OmpGetModifiers(inp.v);
-  auto *t1 = semantics::OmpGetUniqueModifier<parser::OmpMapper>(mods);
-  auto *t2 = semantics::OmpGetUniqueModifier<parser::OmpIterator>(mods);
-  auto *t3 = semantics::OmpGetUniqueModifier<parser::OmpMapType>(mods);
-  auto &t4 = std::get<parser::OmpObjectList>(inp.v.t);
 
-  auto mappers = [&]() -> std::optional<List<Mapper>> {
-    if (t1)
-      return List<Mapper>{Mapper{makeObject(t1->v, semaCtx)}};
-    return std::nullopt;
-  }();
-
-  auto iterator = [&]() -> std::optional<Iterator> {
-    if (t2)
-      return makeIterator(*t2, semaCtx);
-    return std::nullopt;
-  }();
+  auto *t1 = semantics::OmpGetUniqueModifier<parser::OmpMapType>(mods);
+  auto &t2 = std::get<parser::OmpObjectList>(inp.v.t);
 
   auto type = [&]() -> std::optional<Map::MapType> {
-    if (t3)
-      return convert1(t3->v);
-    return std::nullopt;
+    std::optional<Map::MapType> type;
+    if (t1) {
+      switch (t1->v) {
+      case parser::OmpMapType::Value::Alloc:
+      case parser::OmpMapType::Value::Delete:
+      case parser::OmpMapType::Value::Release:
+      case parser::OmpMapType::Value::Storage:
+        type = Map::MapType::Storage;
+        break;
+      case parser::OmpMapType::Value::From:
+        type = Map::MapType::From;
+        break;
+      case parser::OmpMapType::Value::To:
+        type = Map::MapType::To;
+        break;
+      case parser::OmpMapType::Value::Tofrom:
+        type = Map::MapType::Tofrom;
+        break;
+      default:
+        break;
+      }
+    }
+    return type;
   }();
 
-  Map::MapTypeModifiers typeMods;
+  llvm::DenseSet<Map::MapTypeModifier> modSet;
+  if (t1 && t1->v == parser::OmpMapType::Value::Delete)
+    modSet.insert(Map::MapTypeModifier::Delete);
+
   for (auto *typeMod :
        semantics::OmpGetRepeatableModifier<parser::OmpMapTypeModifier>(mods)) {
-    typeMods.push_back(convert2(typeMod->v));
+    modSet.insert(convert1(typeMod->v));
   }
+  if (semantics::OmpGetUniqueModifier<parser::OmpAlwaysModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Always);
+  if (semantics::OmpGetUniqueModifier<parser::OmpCloseModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Close);
+  if (semantics::OmpGetUniqueModifier<parser::OmpDeleteModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Delete);
+  if (semantics::OmpGetUniqueModifier<parser::OmpPresentModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Present);
+  if (semantics::OmpGetUniqueModifier<parser::OmpSelfModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Self);
+  if (semantics::OmpGetUniqueModifier<parser::OmpxHoldModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::OmpxHold);
+
   std::optional<Map::MapTypeModifiers> maybeTypeMods{};
-  if (!typeMods.empty())
-    maybeTypeMods = std::move(typeMods);
+  if (!modSet.empty())
+    maybeTypeMods = Map::MapTypeModifiers(modSet.begin(), modSet.end());
+
+  auto refMod = [&]() -> std::optional<Map::RefModifier> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpRefModifier>(mods))
+      return convert2(t->v);
+    return std::nullopt;
+  }();
+
+  auto mappers = [&]() -> std::optional<List<Mapper>> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpMapper>(mods))
+      return List<Mapper>{Mapper{makeObject(t->v, semaCtx)}};
+    return std::nullopt;
+  }();
+
+  auto iterator = [&]() -> std::optional<Iterator> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpIterator>(mods))
+      return makeIterator(*t, semaCtx);
+    return std::nullopt;
+  }();
 
   return Map{{/*MapType=*/std::move(type),
               /*MapTypeModifiers=*/std::move(maybeTypeMods),
-              /*Mapper=*/std::move(mappers), /*Iterator=*/std::move(iterator),
-              /*LocatorList=*/makeObjects(t4, semaCtx)}};
+              /*RefModifier=*/std::move(refMod), /*Mapper=*/std::move(mappers),
+              /*Iterator=*/std::move(iterator),
+              /*LocatorList=*/makeObjects(t2, semaCtx)}};
 }
 
 Match make(const parser::OmpClause::Match &inp,
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 46aaab19ded0a..77e2fd6ca5097 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -402,6 +402,11 @@ class CanonicalizationOfOmp {
   // if the specified OpenMP version is less than 6.0, rewrite the affected
   // modifiers back into the pre-6.0 forms.
   void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    unsigned version{context_.langOptions().OpenMPVersion};
+    if (version >= 60) {
+      return;
+    }
+
     // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
     // OmpDeleteModifier -> OmpMapType
     using Modifier = parser::OmpMapClause::Modifier;
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2425265e196c6..94dee3d7afe21 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -37,6 +37,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
@@ -3399,23 +3400,22 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Detach &x) {
   }
 }
 
-void OmpStructureChecker::CheckAllowedMapTypes(
-    const parser::OmpMapType::Value &type,
-    const std::list<parser::OmpMapType::Value> &allowedMapTypeList) {
-  if (!llvm::is_contained(allowedMapTypeList, type)) {
-    std::string commaSeparatedMapTypes;
-    llvm::interleave(
-        allowedMapTypeList.begin(), allowedMapTypeList.end(),
-        [&](const parser::OmpMapType::Value &mapType) {
-          commaSeparatedMapTypes.append(parser::ToUpperCaseLetters(
-              parser::OmpMapType::EnumToString(mapType)));
-        },
-        [&] { commaSeparatedMapTypes.append(", "); });
-    context_.Say(GetContext().clauseSource,
-        "Only the %s map types are permitted "
-        "for MAP clauses on the %s directive"_err_en_US,
-        commaSeparatedMapTypes, ContextDirectiveAsFortran());
+void OmpStructureChecker::CheckAllowedMapTypes(parser::OmpMapType::Value type,
+    llvm::ArrayRef<parser::OmpMapType::Value> allowed) {
+  if (llvm::is_contained(allowed, type)) {
+    return;
   }
+
+  llvm::SmallVector<std::string> names;
+  llvm::transform(
+      allowed, std::back_inserter(names), [](parser::OmpMapType::Value val) {
+        return parser::ToUpperCaseLetters(
+            parser::OmpMapType::EnumToString(val));
+      });
+  llvm::sort(names);
+  context_.Say(GetContext().clauseSource,
+      "Only the %s map types are permitted for MAP clauses on the %s directive"_err_en_US,
+      llvm::join(names, ", "), ContextDirectiveAsFortran());
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
@@ -3436,27 +3436,62 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
     CheckIteratorModifier(*iter);
   }
   if (auto *type{OmpGetUniqueModifier<parser::OmpMapType>(modifiers)}) {
+    using Directive = llvm::omp::Directive;
     using Value = parser::OmpMapType::Value;
-    switch (GetContext().directive) {
-    case llvm::omp::Directive::OMPD_target:
-    case llvm::omp::Directive::OMPD_target_teams:
-    case llvm::omp::Directive::OMPD_target_teams_distribute:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
-    case llvm::omp::Directive::OMPD_target_data:
-      CheckAllowedMapTypes(
-          type->v, {Value::To, Value::From, Value::Tofrom, Value::Alloc});
-      break;
-    case llvm::omp::Directive::OMPD_target_enter_data:
-      CheckAllowedMapTypes(type->v, {Value::To, Value::Alloc});
-      break;
-    case llvm::omp::Directive::OMPD_target_exit_data:
-      CheckAllowedMapTypes(
-          type->v, {Value::From, Value::Release, Value::Delete});
-      break;
-    default:
-      break;
+
+    static auto isValidForVersion{
+        [](parser::OmpMapType::Value t, unsigned version) {
+          switch (t) {
+          case parser::OmpMapType::Value::Alloc:
+          case parser::OmpMapType::Value::Delete:
+          case parser::OmpMapType::Value::Release:
+            return version < 60;
+          case parser::OmpMapType::Value::Storage:
+            return version >= 60;
+          default:
+            return true;
+          }
+        }};
+
+    llvm::SmallVector<parser::OmpMapType::Value> mapEnteringTypes{[&]() {
+      llvm::SmallVector<parser::OmpMapType::Value> result;
+      for (size_t i{0}; i != parser::OmpMapType::Value_enumSize; ++i) {
+        auto t{static_cast<parser::OmpMapType::Value>(i)};
+        if (isValidForVersion(t, version) && IsMapEnteringType(t)) {
+          result.push_back(t);
+        }
+      }
+      return result;
+    }()};
+    llvm::SmallVector<parser::OmpMapType::Value> mapExitingTypes{[&]() {
+      llvm::SmallVector<parser::OmpMapType::Value> result;
+      for (size_t i{0}; i != parser::OmpMapType::Value_enumSize; ++i) {
+        auto t{static_cast<parser::OmpMapType::Value>(i)};
+        if (isValidForVersion(t, version) && IsMapExitingType(t)) {
+          result.push_back(t);
+        }
+      }
+      return result;
+    }()};
+
+    llvm::omp::Directive dir{GetContext().directive};
+    llvm::ArrayRef<llvm::omp::Directive> leafs{
+        llvm::omp::getLeafConstructsOrSelf(dir)};
+
+    if (llvm::is_contained(leafs, Directive::OMPD_target) ||
+        llvm::is_contained(leafs, Directive::OMPD_target_data)) {
+      if (version >= 60) {
+        // Map types listed in the decay table. [6.0:276]
+        CheckAllowedMapTypes(
+            type->v, {Value::Storage, Value::From, Value::To, Value::Tofrom});
+      } else {
+        CheckAllowedMapTypes(
+            type->v, {Value::Alloc, Value::From, Value::To, Value::Tofrom});
+      }
+    } else if (llvm::is_contained(leafs, Directive::OMPD_target_enter_data)) {
+      CheckAllowedMapTypes(type->v, mapEnteringTypes);
+    } else if (llvm::is_contained(leafs, Directive::OMPD_target_exit_data)) {
+      CheckAllowedMapTypes(type->v, mapExitingTypes);
     }
   }
 
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 6a877a5d0a7c0..f4a291dc255c8 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -179,8 +179,8 @@ class OmpStructureChecker
   void HasInvalidDistributeNesting(const parser::OpenMPLoopConstruct &x);
   void HasInvalidLoopBinding(const parser::OpenMPLoopConstruct &x);
   // specific clause related
-  void CheckAllowedMapTypes(const parser::OmpMapType::Value &,
-      const std::list<parser::OmpMapType::Value> &);
+  void CheckAllowedMapTypes(
+      parser::OmpMapType::Value, llvm::ArrayRef<parser::OmpMapType::Value>);
 
   const std::list<parser::OmpTraitProperty> &GetTraitPropertyList(
       const parser::OmpTraitSelector &);
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index f43d2cc75620e..da14507aa9fe6 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -143,6 +143,31 @@ bool IsVarOrFunctionRef(const MaybeExpr &expr) {
   }
 }
 
+bool IsMapEnteringType(parser::OmpMapType::Value type) {
+  switch (type) {
+  case parser::OmpMapType::Value::Alloc:
+  case parser::OmpMapType::Value::Storage:
+  case parser::OmpMapType::Value::To:
+  case parser::OmpMapType::Value::Tofrom:
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool IsMapExitingType(parser::OmpMapType::Value type) {
+  switch (type) {
+  case parser::OmpMapType::Value::Delete:
+  case parser::OmpMapType::Value::From:
+  case parser::OmpMapType::Value::Release:
+  case parser::OmpMapType::Value::Storage:
+  case parser::OmpMapType::Value::Tofrom:
+    return true;
+  default:
+    return false;
+  }
+}
+
 std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
   const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
   // ForwardOwningPointer           typedExpr
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index a96c008fb26e7..001fbeb45ceec 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -59,6 +59,9 @@ bool IsExtendedListItem(const Symbol &sym);
 bool IsVariableListItem(const Symbol &sym);
 bool IsVarOrFunctionRef(const MaybeExpr &expr);
 
+bool IsMapEnteringType(parser::OmpMapType::Value type);
+bool IsMapExitingType(parser::OmpMapType::Value type);
+
 std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr);
 std::optional<evaluate::DynamicType> GetDynamicType(
     const parser::Expr &parserExpr);
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 0b860314b4a1f..cbd14bc259d38 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -712,7 +712,15 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   void Post(const parser::EorLabel &eorLabel) { CheckSourceLabel(eorLabel.v); }
 
   void Post(const parser::OmpMapClause &x) {
-    Symbol::Flag ompFlag = Symbol::Flag::OmpMapToFrom;
+    unsigned version{context_.langOptions().OpenMPVersion};
+    llvm::omp::Directive id{GetContext().directive};
+    std::optional<Symbol::Flag> ompFlag;
+    // Expand "storage" into either "alloc" or "release", depending on the
+    // type of the construct.
+    Symbol::Flag ompFlagForStorage = llvm::omp::isMapEnteringConstruct(id)
+        ? Symbol::Flag::OmpMapAlloc
+        : Symbol::Flag::OmpMapRelease;
+
     auto &mods{OmpGetModifiers(x)};
     if (auto *mapType{OmpGetUniqueModifier<parser::OmpMapType>(mods)}) {
       switch (mapType->v) {
@@ -734,10 +742,29 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
-      default:
+      case parser::OmpMapType::Value::Storage:
+        ompFlag = ompFlagForStorage;
         break;
       }
     }
+    if (!ompFlag) {
+      if (version >= 60) {
+        // [6.0:275:12-15]
+        // When a map-type is not specified for a clause on which it may be
+        // specified, the map-type defaults to storage if the delete-modifier
+        // is present on the clause or if the list item for which the map-type
+        // is not specified is an assumed-size array.
+        if (OmpGetUniqueModifier<parser::OmpDeleteModifier>(mods)) {
+          ompFlag = ompFlagForStorage;
+        }
+        // Otherwise, if delete-modifier is absent, leave ompFlag unset.
+      } else {
+        // [5.2:151:10]
+        // If a map-type is not specified, the map-type defaults to tofrom.
+        ompFlag = Symbol::Flag::OmpMapToFrom;
+      }
+    }
+
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
     for (const auto &ompObj : ompObjList.v) {
       common::visit(
@@ -746,15 +773,14 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
                 if (const auto *name{
                         semantics::getDesignatorNameIfDataRef(designator)}) {
                   if (name->symbol) {
-                    name->symbol->set(ompFlag);
-                    AddToContextObjectWithDSA(*name->symbol, ompFlag);
-                  }
-                  if (name->symbol &&
-                      semantics::IsAssumedSizeArray(*name->symbol)) {
-                    context_.Say(designator.source,
-                        "Assumed-size whole arrays may not appear on the %s "
-                        "clause"_err_en_US,
-                        "MAP");
+                    name->symbol->set(ompFlag.value_or(ompFlagForStorage));
+                    AddToContextObjectWithDSA(*name->symbol, *ompFlag);
+                    if (semantics::IsAssumedSizeArray(*name->symbol)) {
+                      context_.Say(designator.source,
+                          "Assumed-size whole arrays may not appear on the %s "
+                          "clause"_err_en_US,
+                          "MAP");
+                    }
                   }
                 }
               },
@@ -762,7 +788,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
           },
           ompObj.u);
 
-      ResolveOmpObject(ompObj, ompFlag);
+      ResolveOmpObject(ompObj, ompFlag.value_or(ompFlagForStorage));
     }
   }
 
diff --git a/flang/test/Semantics/OpenMP/combined-constructs.f90 b/flang/test/Semantics/OpenMP/combined-constructs.f90
index 4f2a4a4f501b9..2298d33ef33eb 100644
--- a/flang/test/Semantics/OpenMP/combined-constructs.f90
+++ b/flang/test/Semantics/OpenMP/combined-constructs.f90
@@ -207,7 +207,7 @@ program main
   enddo
   !$omp end target teams
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS directive
   !$omp target teams map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -307,7 +307,7 @@ program main
   enddo
   !$omp end target teams distribute
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -400,7 +400,7 @@ program main
   enddo
   !$omp end target teams distribute parallel do
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -500,7 +500,7 @@ program main
   enddo
   !$omp end target teams distribute parallel do simd
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd map(delete:a)
   do i = 1, N
      a(i) = 3.14
diff --git a/flang/test/Semantics/OpenMP/device-constructs.f90 b/flang/test/Semantics/OpenMP/device-constructs.f90
index 6f545b9021966..431e0f88e3237 100644
--- a/flang/test/Semantics/OpenMP/device-constructs.f90
+++ b/flang/test/Semantics/OpenMP/device-constructs.f90
@@ -123,7 +123,7 @@ program main
   enddo
   !$omp end target
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET directive
   !$omp target map(delete:a)
   do i = 1, N
      a = 3.14
@@ -160,7 +160,7 @@ program main
   !ERROR: At most one IF clause can appear on the TARGET ENTER DATA directive
   !$omp target enter data map(to:a) if(.true.) if(.false.)
 
-  !ERROR: Only the TO, ALLOC map types are permitted for MAP clauses on the TARGET ENTER DATA directive
+  !ERROR: Only the ALLOC, TO, TOFROM map types are permitted for MAP clauses on the TARGET ENTER DATA directive
   !$omp target enter data map(from:a)
 
   !$omp target exit data map(delete:a)
@@ -168,7 +168,7 @@ program main
   !ERROR: At most one DEVICE clause can appear on the TARGET EXIT DATA directive
   !$omp target exit data map(from:a) device(0) device(1)
 
-  !ERROR: Only the FROM, RELEASE, DELETE map types are permitted for MAP clauses on the TARGET EXIT DATA directive
+  !ERROR: Only the DELETE, FROM, RELEASE, TOFROM map types are permitted for MAP clauses on the TARGET EXIT DATA directive
   !$omp target exit data map(to:a)
 
   !$omp target update if(.true.) device(1) to(a) from(b) depend(inout:c) nowait
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index de888ff86fe91..7919f7a8b0c34 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -779,16 +779,17 @@ struct LinkT {
 template <typename T, typename I, typename E> //
 struct MapT {
   using LocatorList = ObjectListT<I, E>;
-  ENUM(MapType, To, From, Tofrom, Alloc, Release, Delete);
-  ENUM(MapTypeModifier, Always, Close, Present, OmpxHold);
+  ENUM(MapType, To, From, Tofrom, Storage);
+  ENUM(MapTypeModifier, Always, Close, Delete, Present, Self, OmpxHold);
+  ENUM(RefModifier, RefPtee, RefPtr, RefPtrPtee);
   // See note at the definition of the MapperT type.
   using Mappers = ListT<type::MapperT<I, E>>; // Not a spec name
   using Iterator = type::IteratorT<T, I, E>;
   using MapTypeModifiers = ListT<MapTypeModifier>; // Not a spec name
 
   using TupleTrait = std::true_type;
-  std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(Mappers), OPT(Iterator),
-             LocatorList>
+  std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(RefModifier),
+             OPT(Mappers), OPT(Iterator), LocatorList>
       t;
 };
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
index 611bfe3f8aced..047baa3a79f5d 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -708,6 +708,7 @@ bool ConstructDecompositionT<C, H>::applyClause(
                      tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
                          {/*MapType=*/MapType::Tofrom,
                           /*MapTypeModifier=*/std::nullopt,
+                          /*RefModifier=*/std::nullopt,
                           /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
                           /*LocatorList=*/std::move(tofrom)}});
       dirTarget->clauses.push_back(map);
@@ -969,8 +970,8 @@ bool ConstructDecompositionT<C, H>::applyClause(
           llvm::omp::Clause::OMPC_map,
           tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
               {/*MapType=*/MapType::Tofrom, /*MapTypeModifier=*/std::nullopt,
-               /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
-               /*LocatorList=*/std::move(tofrom)}});
+               /*RefModifier=*/std::nullopt, /*Mapper=*/std::nullopt,
+               /*Iterator=*/std::nullopt, /*LocatorList=*/std::move(tofrom)}});
 
       dirTarget->clauses.push_back(map);
       applied = true;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index d44c33301bde7..a8b4503147034 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -64,6 +64,8 @@ static constexpr inline bool isPrivatizingClause(Clause C) {
   }
 }
 
+LLVM_ABI bool isMapEnteringConstruct(Directive D);
+
 static constexpr unsigned FallbackVersion = 52;
 LLVM_ABI ArrayRef<unsigned> getOpenMPVersions();
 
diff --git a/llvm/lib/Frontend/OpenMP/OMP.cpp b/llvm/lib/Frontend/OpenMP/OMP.cpp
index 555e2a61e411e..c23fb400666d6 100644
--- a/llvm/lib/Frontend/OpenMP/OMP.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMP.cpp
@@ -189,6 +189,20 @@ bool isCombinedConstruct(Directive D) {
   return !getLeafConstructs(D).empty() && !isCompositeConstruct(D);
 }
 
+bool isMapEnteringConstruct(Directive D) {
+  for (Directive L : getLeafConstructsOrSelf(D)) {
+    switch (L) {
+    case OMPD_target:
+    case OMPD_target_data:
+    case OMPD_target_enter_data:
+      return true;
+    default:
+      break;
+    }
+  }
+  return false;
+}
+
 ArrayRef<unsigned> getOpenMPVersions() {
   static unsigned Versions[]{31, 40, 45, 50, 51, 52, 60};
   return Versions;

>From 0733ec13ae69e2ec3d462987c279ef083890e95c Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 11:41:55 -0500
Subject: [PATCH 03/10] Simplify converting MapType, NFC

---
 flang/lib/Lower/OpenMP/Clauses.cpp | 47 +++++++++++++-----------------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 18e49719b6013..ece4a8eaac8cd 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1001,7 +1001,20 @@ Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
+      convert1, parser::OmpMapType::Value, Map::MapType,
+      // clang-format off
+      MS(Alloc,   Storage)
+      MS(Delete,  Storage)
+      MS(Release, Storage)
+      MS(Storage, Storage)
+      MS(From,    From)
+      MS(To,      To)
+      MS(Tofrom,  Tofrom)
+      // clang-format on
+  );
+
+  CLAUSET_ENUM_CONVERT( //
+      convert2, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
       // clang-format off
       MS(Always,    Always)
       MS(Close,     Close)
@@ -1011,7 +1024,7 @@ Map make(const parser::OmpClause::Map &inp,
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert2, parser::OmpRefModifier::Value, Map::RefModifier,
+      convert3, parser::OmpRefModifier::Value, Map::RefModifier,
       // clang-format off
       MS(Ref_Ptee,     RefPtee)
       MS(Ref_Ptr,      RefPtr)
@@ -1027,29 +1040,9 @@ Map make(const parser::OmpClause::Map &inp,
   auto &t2 = std::get<parser::OmpObjectList>(inp.v.t);
 
   auto type = [&]() -> std::optional<Map::MapType> {
-    std::optional<Map::MapType> type;
-    if (t1) {
-      switch (t1->v) {
-      case parser::OmpMapType::Value::Alloc:
-      case parser::OmpMapType::Value::Delete:
-      case parser::OmpMapType::Value::Release:
-      case parser::OmpMapType::Value::Storage:
-        type = Map::MapType::Storage;
-        break;
-      case parser::OmpMapType::Value::From:
-        type = Map::MapType::From;
-        break;
-      case parser::OmpMapType::Value::To:
-        type = Map::MapType::To;
-        break;
-      case parser::OmpMapType::Value::Tofrom:
-        type = Map::MapType::Tofrom;
-        break;
-      default:
-        break;
-      }
-    }
-    return type;
+    if (t1)
+      return convert1(t1->v);
+    return std::nullopt;
   }();
 
   llvm::DenseSet<Map::MapTypeModifier> modSet;
@@ -1058,7 +1051,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   for (auto *typeMod :
        semantics::OmpGetRepeatableModifier<parser::OmpMapTypeModifier>(mods)) {
-    modSet.insert(convert1(typeMod->v));
+    modSet.insert(convert2(typeMod->v));
   }
   if (semantics::OmpGetUniqueModifier<parser::OmpAlwaysModifier>(mods))
     modSet.insert(Map::MapTypeModifier::Always);
@@ -1079,7 +1072,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   auto refMod = [&]() -> std::optional<Map::RefModifier> {
     if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpRefModifier>(mods))
-      return convert2(t->v);
+      return convert3(t->v);
     return std::nullopt;
   }();
 

>From a7476b1ca983ecf0244b71ab873d3775343e5c46 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 12:43:14 -0500
Subject: [PATCH 04/10] Fix unit test

---
 .../Frontend/OpenMPDecompositionTest.cpp         | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
index 6189d0954891b..95c26b10c9a0c 100644
--- a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
@@ -431,8 +431,8 @@ TEST_F(OpenMPDecompositionTest, Firstprivate3) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (12), (27)
-  ASSERT_EQ(Dir1, "teams shared(x)");          // (6), (17)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (12), (27)
+  ASSERT_EQ(Dir1, "teams shared(x)");            // (6), (17)
   ASSERT_EQ(Dir2, "distribute firstprivate(x) lastprivate(, (x))"); // (5), (21)
 }
 
@@ -574,9 +574,9 @@ TEST_F(OpenMPDecompositionTest, Lastprivate3) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (21), (27)
-  ASSERT_EQ(Dir1, "parallel shared(x)");       // (22)
-  ASSERT_EQ(Dir2, "do lastprivate(, (x))");    // (21)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (21), (27)
+  ASSERT_EQ(Dir1, "parallel shared(x)");         // (22)
+  ASSERT_EQ(Dir2, "do lastprivate(, (x))");      // (21)
 }
 
 // SHARED
@@ -984,9 +984,9 @@ TEST_F(OpenMPDecompositionTest, Reduction7) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (36), (10)
-  ASSERT_EQ(Dir1, "parallel shared(x)");       // (36), (1), (4)
-  ASSERT_EQ(Dir2, "do reduction(, (3), (x))"); // (36)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (36), (10)
+  ASSERT_EQ(Dir1, "parallel shared(x)");         // (36), (1), (4)
+  ASSERT_EQ(Dir2, "do reduction(, (3), (x))");   // (36)
 }
 
 // IF

>From faff0d32746640c34988a65552e277198748fcf2 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 08:49:18 -0500
Subject: [PATCH 05/10] Don't pass SemanticsContext to CanonicalizeOmp yet

---
 flang/lib/Semantics/canonicalize-omp.cpp | 10 ++++------
 flang/lib/Semantics/canonicalize-omp.h   |  5 +++--
 flang/lib/Semantics/semantics.cpp        |  3 ++-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 46aaab19ded0a..bba4d72a702e9 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,8 +27,7 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(SemanticsContext &context)
-      : context_{context}, messages_{context.messages()} {}
+  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -433,13 +432,12 @@ class CanonicalizationOfOmp {
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
-  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
-  CanonicalizationOfOmp omp{context};
+bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
+  CanonicalizationOfOmp omp{messages};
   Walk(program, omp);
-  return !context.messages().AnyFatalError();
+  return !messages.AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index 3251218da35ed..23350c522a775 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,12 +11,13 @@
 
 namespace Fortran::parser {
 struct Program;
-}
+class Messages;
+} // namespace Fortran::parser
 
 namespace Fortran::semantics {
 class SemanticsContext;
 
-bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
 } // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 6db11aaf56c2a..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -642,7 +642,8 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_.messages(), program_) &&
+      CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}

>From 724cc8306faae084c4a6ff8295073bd65bc0f609 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 08:54:34 -0500
Subject: [PATCH 06/10] Pass SemanticsContext to CanonicalizeOmp

---
 flang/lib/Semantics/canonicalize-omp.cpp | 10 ++++++----
 flang/lib/Semantics/canonicalize-omp.h   |  5 ++---
 flang/lib/Semantics/semantics.cpp        |  3 +--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 9b74a153b233e..77e2fd6ca5097 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -437,12 +438,13 @@ class CanonicalizationOfOmp {
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index 23350c522a775..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,13 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
 class SemanticsContext;
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
 } // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index b15ed057b52f2..6db11aaf56c2a 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -642,8 +642,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}

>From 3886bbcd0477b5932b1c9dc1303bbe3a940bd179 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 09:20:25 -0500
Subject: [PATCH 07/10] Invent OmpMapStorage for map flags

We're using "storage" instead of "alloc"/"release" in lowering, this will
make things more consistent.
---
 flang/include/flang/Semantics/symbol.h     |  5 ++---
 flang/lib/Semantics/resolve-directives.cpp | 26 +++++++---------------
 flang/lib/Semantics/symbol.cpp             |  3 +--
 3 files changed, 11 insertions(+), 23 deletions(-)

diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index b319e2c7e5e74..5bde9f39ca0b0 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -812,9 +812,8 @@ class Symbol {
       // OpenMP data-sharing attribute
       OmpShared, OmpPrivate, OmpLinear, OmpFirstPrivate, OmpLastPrivate,
       // OpenMP data-mapping attribute
-      OmpMapTo, OmpMapFrom, OmpMapToFrom, OmpMapAlloc, OmpMapRelease,
-      OmpMapDelete, OmpUseDevicePtr, OmpUseDeviceAddr, OmpIsDevicePtr,
-      OmpHasDeviceAddr,
+      OmpMapTo, OmpMapFrom, OmpMapToFrom, OmpMapStorage, OmpMapDelete,
+      OmpUseDevicePtr, OmpUseDeviceAddr, OmpIsDevicePtr, OmpHasDeviceAddr,
       // OpenMP data-copying attribute
       OmpCopyIn, OmpCopyPrivate,
       // OpenMP miscellaneous flags
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index e02ff97dc6917..4c3e509b5a36d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -728,13 +728,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
 
   void Post(const parser::OmpMapClause &x) {
     unsigned version{context_.langOptions().OpenMPVersion};
-    llvm::omp::Directive id{GetContext().directive};
     std::optional<Symbol::Flag> ompFlag;
-    // Expand "storage" into either "alloc" or "release", depending on the
-    // type of the construct.
-    Symbol::Flag ompFlagForStorage = llvm::omp::isMapEnteringConstruct(id)
-        ? Symbol::Flag::OmpMapAlloc
-        : Symbol::Flag::OmpMapRelease;
 
     auto &mods{OmpGetModifiers(x)};
     if (auto *mapType{OmpGetUniqueModifier<parser::OmpMapType>(mods)}) {
@@ -749,17 +743,13 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
         ompFlag = Symbol::Flag::OmpMapToFrom;
         break;
       case parser::OmpMapType::Value::Alloc:
-        ompFlag = Symbol::Flag::OmpMapAlloc;
-        break;
       case parser::OmpMapType::Value::Release:
-        ompFlag = Symbol::Flag::OmpMapRelease;
+      case parser::OmpMapType::Value::Storage:
+        ompFlag = Symbol::Flag::OmpMapStorage;
         break;
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
-      case parser::OmpMapType::Value::Storage:
-        ompFlag = ompFlagForStorage;
-        break;
       }
     }
     if (!ompFlag) {
@@ -770,7 +760,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
         // is present on the clause or if the list item for which the map-type
         // is not specified is an assumed-size array.
         if (OmpGetUniqueModifier<parser::OmpDeleteModifier>(mods)) {
-          ompFlag = ompFlagForStorage;
+          ompFlag = Symbol::Flag::OmpMapStorage;
         }
         // Otherwise, if delete-modifier is absent, leave ompFlag unset.
       } else {
@@ -788,7 +778,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
                 if (const auto *name{
                         semantics::getDesignatorNameIfDataRef(designator)}) {
                   if (name->symbol) {
-                    name->symbol->set(ompFlag.value_or(ompFlagForStorage));
+                    name->symbol->set(
+                        ompFlag.value_or(Symbol::Flag::OmpMapStorage));
                     AddToContextObjectWithDSA(*name->symbol, *ompFlag);
                     if (semantics::IsAssumedSizeArray(*name->symbol)) {
                       context_.Say(designator.source,
@@ -803,7 +794,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
           },
           ompObj.u);
 
-      ResolveOmpObject(ompObj, ompFlag.value_or(ompFlagForStorage));
+      ResolveOmpObject(ompObj, ompFlag.value_or(Symbol::Flag::OmpMapStorage));
     }
   }
 
@@ -2802,9 +2793,8 @@ void OmpAttributeVisitor::ResolveOmpObject(
                   }
                   Symbol::Flag dataMappingAttributeFlags[] = {
                       Symbol::Flag::OmpMapTo, Symbol::Flag::OmpMapFrom,
-                      Symbol::Flag::OmpMapToFrom, Symbol::Flag::OmpMapAlloc,
-                      Symbol::Flag::OmpMapRelease, Symbol::Flag::OmpMapDelete,
-                      Symbol::Flag::OmpIsDevicePtr,
+                      Symbol::Flag::OmpMapToFrom, Symbol::Flag::OmpMapStorage,
+                      Symbol::Flag::OmpMapDelete, Symbol::Flag::OmpIsDevicePtr,
                       Symbol::Flag::OmpHasDeviceAddr};
 
                   Symbol::Flag dataSharingAttributeFlags[] = {
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index 0380207927ad3..2259cfcf23ece 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -861,8 +861,7 @@ std::string Symbol::OmpFlagToClauseName(Symbol::Flag ompFlag) {
   case Symbol::Flag::OmpMapTo:
   case Symbol::Flag::OmpMapFrom:
   case Symbol::Flag::OmpMapToFrom:
-  case Symbol::Flag::OmpMapAlloc:
-  case Symbol::Flag::OmpMapRelease:
+  case Symbol::Flag::OmpMapStorage:
   case Symbol::Flag::OmpMapDelete:
     clauseName = "MAP";
     break;

>From 20bc43526e52676b4571fe85c3fad35cad70a115 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 09:31:44 -0500
Subject: [PATCH 08/10] Add testcase for using v60 modifiers in v52

---
 .../Semantics/OpenMP/map-modifiers-v60.f90    | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 flang/test/Semantics/OpenMP/map-modifiers-v60.f90

diff --git a/flang/test/Semantics/OpenMP/map-modifiers-v60.f90 b/flang/test/Semantics/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..b3f2e5171d6d6
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,35 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52 -Werror
+
+subroutine f00(x)
+  integer :: x
+!WARNING: 'self-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(self: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f01(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptr: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f02(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptee: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f03(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptr_ptee: x)
+  x = x + 1
+  !$omp end target
+end
+
+

>From 7d232c536a47aa705107db336259e4470335e350 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 09:35:48 -0500
Subject: [PATCH 09/10] Clearer names for enum-converting functions for map
 modifiers

---
 flang/lib/Lower/OpenMP/Clauses.cpp | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index ece4a8eaac8cd..686fba0154f44 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1001,7 +1001,7 @@ Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpMapType::Value, Map::MapType,
+      convertMapType, parser::OmpMapType::Value, Map::MapType,
       // clang-format off
       MS(Alloc,   Storage)
       MS(Delete,  Storage)
@@ -1014,7 +1014,8 @@ Map make(const parser::OmpClause::Map &inp,
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert2, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
+      convertMapTypeMod, parser::OmpMapTypeModifier::Value,
+      Map::MapTypeModifier,
       // clang-format off
       MS(Always,    Always)
       MS(Close,     Close)
@@ -1024,7 +1025,7 @@ Map make(const parser::OmpClause::Map &inp,
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert3, parser::OmpRefModifier::Value, Map::RefModifier,
+      convertRefMod, parser::OmpRefModifier::Value, Map::RefModifier,
       // clang-format off
       MS(Ref_Ptee,     RefPtee)
       MS(Ref_Ptr,      RefPtr)
@@ -1041,7 +1042,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   auto type = [&]() -> std::optional<Map::MapType> {
     if (t1)
-      return convert1(t1->v);
+      return convertMapType(t1->v);
     return std::nullopt;
   }();
 
@@ -1051,7 +1052,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   for (auto *typeMod :
        semantics::OmpGetRepeatableModifier<parser::OmpMapTypeModifier>(mods)) {
-    modSet.insert(convert2(typeMod->v));
+    modSet.insert(convertMapTypeMod(typeMod->v));
   }
   if (semantics::OmpGetUniqueModifier<parser::OmpAlwaysModifier>(mods))
     modSet.insert(Map::MapTypeModifier::Always);
@@ -1072,7 +1073,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   auto refMod = [&]() -> std::optional<Map::RefModifier> {
     if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpRefModifier>(mods))
-      return convert3(t->v);
+      return convertRefMod(t->v);
     return std::nullopt;
   }();
 

>From dd339d61a4e10e88edf2c569af7be3fd9684beea Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 09:57:08 -0500
Subject: [PATCH 10/10] Remove now unused isMapEnteringConstruct

---
 llvm/include/llvm/Frontend/OpenMP/OMP.h |  2 --
 llvm/lib/Frontend/OpenMP/OMP.cpp        | 14 --------------
 2 files changed, 16 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index 48217384afca6..9d0a55432e1ae 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -68,8 +68,6 @@ static constexpr inline bool isPrivatizingClause(Clause C) {
   }
 }
 
-LLVM_ABI bool isMapEnteringConstruct(Directive D);
-
 static constexpr unsigned FallbackVersion = 52;
 LLVM_ABI ArrayRef<unsigned> getOpenMPVersions();
 
diff --git a/llvm/lib/Frontend/OpenMP/OMP.cpp b/llvm/lib/Frontend/OpenMP/OMP.cpp
index c23fb400666d6..555e2a61e411e 100644
--- a/llvm/lib/Frontend/OpenMP/OMP.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMP.cpp
@@ -189,20 +189,6 @@ bool isCombinedConstruct(Directive D) {
   return !getLeafConstructs(D).empty() && !isCompositeConstruct(D);
 }
 
-bool isMapEnteringConstruct(Directive D) {
-  for (Directive L : getLeafConstructsOrSelf(D)) {
-    switch (L) {
-    case OMPD_target:
-    case OMPD_target_data:
-    case OMPD_target_enter_data:
-      return true;
-    default:
-      break;
-    }
-  }
-  return false;
-}
-
 ArrayRef<unsigned> getOpenMPVersions() {
   static unsigned Versions[]{31, 40, 45, 50, 51, 52, 60};
   return Versions;


From flang-commits at lists.llvm.org  Mon Jul 21 09:08:59 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 09:08:59 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <687e661b.170a0220.4964b.e505@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/149137

>From 0929234481b4d6d015381fda490c05bbeff1e22f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 08:04:17 -0500
Subject: [PATCH 01/11] [flang][OpenMP] Parse OpenMP 6.0 map modifiers

OpenMP 6.0 has changed the modifiers on the MAP clause:
- map-type-modifier has been split into individual modifiers,
- map-type "delete" has become a modifier,
- new modifiers have been added.

This patch adds parsing support for all of the OpenMP 6.0 modifiers.
The old "map-type-modifier" is retained, but is no longer created in
parsing. It will remain to take advantage of the preexisting modifier
validation for older versions: when the OpenMP version is < 6.0,
the modifiers will be rewritten back as map-type-modifiers (or map-
type in case of "delete").

In this patch the modifiers will always be rewritten in the older
format to isolate these changes to parsing as much as possible.
---
 flang/include/flang/Parser/dump-parse-tree.h  |  14 ++
 flang/include/flang/Parser/parse-tree.h       | 138 +++++++++++++++---
 .../flang/Semantics/openmp-modifiers.h        |   7 +
 flang/lib/Parser/openmp-parsers.cpp           |  53 +++++--
 flang/lib/Parser/unparse.cpp                  |   7 +
 flang/lib/Semantics/canonicalize-omp.cpp      |  48 +++++-
 flang/lib/Semantics/canonicalize-omp.h        |   9 +-
 flang/lib/Semantics/openmp-modifiers.cpp      | 115 +++++++++++++++
 flang/lib/Semantics/resolve-directives.cpp    |   2 +
 flang/lib/Semantics/semantics.cpp             |   3 +-
 .../test/Parser/OpenMP/map-modifiers-v60.f90  | 113 ++++++++++++++
 11 files changed, 466 insertions(+), 43 deletions(-)
 create mode 100644 flang/test/Parser/OpenMP/map-modifiers-v60.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 32b6ca45609b6..23e35d106c077 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -529,6 +529,8 @@ class ParseTreeDumper {
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlwaysModifier)
+  NODE_ENUM(OmpAlwaysModifier, Value)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
@@ -546,6 +548,8 @@ class ParseTreeDumper {
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
   NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpCloseModifier)
+  NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -561,6 +565,8 @@ class ParseTreeDumper {
   NODE(parser, OmpDefaultmapClause)
   NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE(parser, OmpDeleteModifier)
+  NODE_ENUM(OmpDeleteModifier, Value)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
   NODE(parser, OmpTaskDependenceType)
@@ -628,6 +634,8 @@ class ParseTreeDumper {
   NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
   NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpPresentModifier)
+  NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
@@ -637,6 +645,10 @@ class ParseTreeDumper {
   NODE(parser, OmpInReductionClause)
   NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpRefModifier)
+  NODE_ENUM(OmpRefModifier, Value)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
   NODE(parser, OmpTaskReductionClause)
   NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpInitializerProc)
@@ -673,6 +685,8 @@ class ParseTreeDumper {
   NODE(parser, OmpSectionsDirective)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpxHoldModifier)
+  NODE_ENUM(OmpxHoldModifier, Value)
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cc1d032f94d4a..92a449d2f9d49 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3756,6 +3756,19 @@ struct OmpAllocatorComplexModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// always-modifier ->
+//    ALWAYS                                        // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpAlwaysModifier {
+  ENUM_CLASS(Value, Always)
+  WRAPPER_CLASS_BOILERPLATE(OmpAlwaysModifier, Value);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3767,17 +3780,29 @@ struct OmpChunkModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpChunkModifier, Value);
 };
 
-// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
-// iterator-specifier ->
-//    [iterator-type] iterator-identifier
-//        = range-specification |                   // since 5.0
-//    [iterator-type ::] iterator-identifier
-//        = range-specification                     // since 5.2
-struct OmpIteratorSpecifier {
-  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
-  CharBlock source;
-  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+// close-modifier ->
+//    CLOSE                                         // since 5.0
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpCloseModifier {
+  ENUM_CLASS(Value, Close)
+  WRAPPER_CLASS_BOILERPLATE(OmpCloseModifier, Value);
+};
+
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// delete-modifier ->
+//    DELETE                                        // since 6.0
+//
+// Until 5.2, it was a part of map-type.
+struct OmpDeleteModifier {
+  ENUM_CLASS(Value, Delete)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeleteModifier, Value);
 };
 
 // Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289]
@@ -3867,6 +3892,19 @@ struct OmpInteropType {
   WRAPPER_CLASS_BOILERPLATE(OmpInteropType, Value);
 };
 
+// Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
+//
+// iterator-specifier ->
+//    [iterator-type] iterator-identifier
+//        = range-specification |                   // since 5.0
+//    [iterator-type ::] iterator-identifier
+//        = range-specification                     // since 5.2
+struct OmpIteratorSpecifier {
+  TUPLE_CLASS_BOILERPLATE(OmpIteratorSpecifier);
+  CharBlock source;
+  std::tuple<TypeDeclarationStmt, SubscriptTriplet> t;
+};
+
 // Ref: [5.0:47-49], [5.1:49-51], [5.2:67-69]
 //
 // iterator-modifier ->
@@ -3901,21 +3939,28 @@ struct OmpMapper {
   WRAPPER_CLASS_BOILERPLATE(OmpMapper, Name);
 };
 
-// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
 //
 // map-type ->
-//    ALLOC | DELETE | FROM | RELEASE | TO | TOFROM // since 4.5
+//    ALLOC | DELETE | RELEASE |                    // since 4.5, until 5.2
+//    FROM | TO | TOFROM |                          // since 4.5
+//    STORAGE                                       // since 6.0
+//
+// Since 6.0 DELETE is a separate delete-modifier.
 struct OmpMapType {
-  ENUM_CLASS(Value, Alloc, Delete, From, Release, To, Tofrom);
+  ENUM_CLASS(Value, Alloc, Delete, From, Release, Storage, To, Tofrom);
   WRAPPER_CLASS_BOILERPLATE(OmpMapType, Value);
 };
 
 // Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158]
 //
 // map-type-modifier ->
-//    ALWAYS |                                      // since 4.5
-//    CLOSE |                                       // since 5.0
-//    PRESENT                                       // since 5.1
+//    ALWAYS |                                      // since 4.5, until 5.2
+//    CLOSE |                                       // since 5.0, until 5.2
+//    PRESENT                                       // since 5.1, until 5.2
+// Since 6.0 the map-type-modifier has been split into individual modifiers.
+//
 struct OmpMapTypeModifier {
   ENUM_CLASS(Value, Always, Close, Present, Ompx_Hold)
   WRAPPER_CLASS_BOILERPLATE(OmpMapTypeModifier, Value);
@@ -3954,6 +3999,19 @@ struct OmpPrescriptiveness {
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
+// Ref: [4.5:216-219], [5.0:315-324], [5.1:347-355], [5.2:150-158],
+// [6.0:279-288]
+//
+// present-modifier ->
+//    PRESENT                                       // since 5.1
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpPresentModifier {
+  ENUM_CLASS(Value, Present)
+  WRAPPER_CLASS_BOILERPLATE(OmpPresentModifier, Value);
+};
+
 // Ref: [5.0:300-302], [5.1:332-334], [5.2:134-137]
 //
 // reduction-modifier ->
@@ -3963,6 +4021,26 @@ struct OmpReductionModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpReductionModifier, Value);
 };
 
+// Ref: [6.0:279-288]
+//
+// ref-modifier ->
+//    REF_PTEE | REF_PTR | REF_PTR_PTEE             // since 6.0
+//
+struct OmpRefModifier {
+  ENUM_CLASS(Value, Ref_Ptee, Ref_Ptr, Ref_Ptr_Ptee)
+  WRAPPER_CLASS_BOILERPLATE(OmpRefModifier, Value);
+};
+
+// Ref: [6.0:279-288]
+//
+// self-modifier ->
+//    SELF                                          // since 6.0
+//
+struct OmpSelfModifier {
+  ENUM_CLASS(Value, Self)
+  WRAPPER_CLASS_BOILERPLATE(OmpSelfModifier, Value);
+};
+
 // Ref: [5.2:117-120]
 //
 // step-complex-modifier ->
@@ -4001,6 +4079,18 @@ struct OmpVariableCategory {
   WRAPPER_CLASS_BOILERPLATE(OmpVariableCategory, Value);
 };
 
+// Extension
+//
+// ompx-hold-modifier ->
+//    OMPX_HOLD                                     // since 4.5
+//
+// Until 5.2, it was a part of map-type-modifier. Since 6.0 the
+// map-type-modifier has been split into individual modifiers.
+struct OmpxHoldModifier {
+  ENUM_CLASS(Value, Ompx_Hold)
+  WRAPPER_CLASS_BOILERPLATE(OmpxHoldModifier, Value);
+};
+
 // context-selector
 using OmpContextSelector = traits::OmpContextSelectorSpecification;
 } // namespace modifier
@@ -4376,13 +4466,25 @@ struct OmpLinearClause {
 // map-clause ->
 //    MAP([modifier...:] locator-list)              // since 4.5
 // modifier ->
-//    map-type-modifier |                           // since 4.5
+//    map-type-modifier [replaced] |                // since 4.5, until 5.2
+//    always-modifier |                             // since 6.0
+//    close-modifier |                              // since 6.0
+//    delete-modifier |                             // since 6.0
+//    present-modifier |                            // since 6.0
+//    ref-modifier |                                // since 6.0
+//    self-modifier |                               // since 6.0
 //    mapper |                                      // since 5.0
 //    iterator |                                    // since 5.1
 //    map-type                                      // since 4.5
+//    ompx-hold-modifier |                          // since 6.0
+//
+// Since 6.0 the map-type-modifier has been split into individual modifiers,
+// and delete-modifier has been split from map-type.
 struct OmpMapClause {
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  MODIFIER_BOILERPLATE(OmpMapTypeModifier, OmpMapper, OmpIterator, OmpMapType);
+  MODIFIER_BOILERPLATE(OmpAlwaysModifier, OmpCloseModifier, OmpDeleteModifier,
+      OmpMapTypeModifier, OmpPresentModifier, OmpRefModifier, OmpSelfModifier,
+      OmpMapper, OmpIterator, OmpMapType, OmpxHoldModifier);
   std::tuple<MODIFIERS(), OmpObjectList, /*CommaSeparated=*/bool> t;
 };
 
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 4fbd80f989e72..a9fe911ef8807 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -71,8 +71,11 @@ DECLARE_DESCRIPTOR(parser::OmpAlignment);
 DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
+DECLARE_DESCRIPTOR(parser::OmpAlwaysModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
+DECLARE_DESCRIPTOR(parser::OmpCloseModifier);
 DECLARE_DESCRIPTOR(parser::OmpContextSelector);
+DECLARE_DESCRIPTOR(parser::OmpDeleteModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
 DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
@@ -88,12 +91,16 @@ DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
 DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
+DECLARE_DESCRIPTOR(parser::OmpPresentModifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
+DECLARE_DESCRIPTOR(parser::OmpRefModifier);
+DECLARE_DESCRIPTOR(parser::OmpSelfModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepComplexModifier);
 DECLARE_DESCRIPTOR(parser::OmpStepSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
 DECLARE_DESCRIPTOR(parser::OmpVariableCategory);
+DECLARE_DESCRIPTOR(parser::OmpxHoldModifier);
 
 #undef DECLARE_DESCRIPTOR
 
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 76c9499410017..d349d8ceb0bb5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -274,6 +274,10 @@ TYPE_PARSER( //
     construct<OmpTypeSpecifier>(Parser<DeclarationTypeSpec>{}) ||
     construct<OmpTypeSpecifier>(Parser<TypeSpec>{}))
 
+// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
+TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
+    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
+
 TYPE_PARSER(construct<OmpReductionSpecifier>( //
     Parser<OmpReductionIdentifier>{},
     ":"_tok >> nonemptyList(Parser<OmpTypeSpecifier>{}),
@@ -442,9 +446,18 @@ TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
 
 TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
 
+TYPE_PARSER(construct<OmpAlwaysModifier>( //
+    "ALWAYS" >> pure(OmpAlwaysModifier::Value::Always)))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpCloseModifier>( //
+    "CLOSE" >> pure(OmpCloseModifier::Value::Close)))
+
+TYPE_PARSER(construct<OmpDeleteModifier>( //
+    "DELETE" >> pure(OmpDeleteModifier::Value::Delete)))
+
 TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
@@ -502,26 +515,16 @@ TYPE_PARSER(construct<OmpLinearModifier>( //
 TYPE_PARSER(construct<OmpMapper>( //
     "MAPPER"_tok >> parenthesized(Parser<ObjectName>{})))
 
-// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+// map-type -> ALLOC | DELETE | FROM | RELEASE | STORAGE | TO | TOFROM
 TYPE_PARSER(construct<OmpMapType>( //
     "ALLOC" >> pure(OmpMapType::Value::Alloc) ||
-    "DELETE" >> pure(OmpMapType::Value::Delete) ||
+    // Parse "DELETE" as OmpDeleteModifier
     "FROM" >> pure(OmpMapType::Value::From) ||
     "RELEASE" >> pure(OmpMapType::Value::Release) ||
+    "STORAGE" >> pure(OmpMapType::Value::Storage) ||
     "TO"_id >> pure(OmpMapType::Value::To) ||
     "TOFROM" >> pure(OmpMapType::Value::Tofrom)))
 
-// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
-TYPE_PARSER(construct<OmpMapTypeModifier>(
-    "ALWAYS" >> pure(OmpMapTypeModifier::Value::Always) ||
-    "CLOSE" >> pure(OmpMapTypeModifier::Value::Close) ||
-    "OMPX_HOLD" >> pure(OmpMapTypeModifier::Value::Ompx_Hold) ||
-    "PRESENT" >> pure(OmpMapTypeModifier::Value::Present)))
-
-// 2.15.3.6 REDUCTION (reduction-identifier: variable-name-list)
-TYPE_PARSER(construct<OmpReductionIdentifier>(Parser<DefinedOperator>{}) ||
-    construct<OmpReductionIdentifier>(Parser<ProcedureDesignator>{}))
-
 TYPE_PARSER(construct<OmpOrderModifier>(
     "REPRODUCIBLE" >> pure(OmpOrderModifier::Value::Reproducible) ||
     "UNCONSTRAINED" >> pure(OmpOrderModifier::Value::Unconstrained)))
@@ -534,11 +537,22 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
 TYPE_PARSER(construct<OmpPrescriptiveness>(
     "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
 
+TYPE_PARSER(construct<OmpPresentModifier>( //
+    "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
     "DEFAULT" >> pure(OmpReductionModifier::Value::Default)))
 
+TYPE_PARSER(construct<OmpRefModifier>( //
+    "REF_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptee) ||
+    "REF_PTR"_id >> pure(OmpRefModifier::Value::Ref_Ptr) ||
+    "REF_PTR_PTEE" >> pure(OmpRefModifier::Value::Ref_Ptr_Ptee)))
+
+TYPE_PARSER(construct<OmpSelfModifier>( //
+    "SELF" >> pure(OmpSelfModifier::Value::Self)))
+
 TYPE_PARSER(construct<OmpStepComplexModifier>( //
     "STEP" >> parenthesized(scalarIntExpr)))
 
@@ -559,6 +573,9 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "POINTER" >> pure(OmpVariableCategory::Value::Pointer) ||
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
+TYPE_PARSER(construct<OmpxHoldModifier>( //
+    "OMPX_HOLD" >> pure(OmpxHoldModifier::Value::Ompx_Hold)))
+
 // This could be auto-generated.
 TYPE_PARSER(
     sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
@@ -611,10 +628,16 @@ TYPE_PARSER(sourced(
     construct<OmpLinearClause::Modifier>(Parser<OmpStepSimpleModifier>{})))
 
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
-    sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
+    sourced(construct<OmpMapClause::Modifier>(Parser<OmpAlwaysModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpCloseModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpDeleteModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpPresentModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpRefModifier>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpSelfModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpIterator>{}) ||
-        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{})))))
+        construct<OmpMapClause::Modifier>(Parser<OmpMapType>{}) ||
+        construct<OmpMapClause::Modifier>(Parser<OmpxHoldModifier>{})))))
 
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index fbe89c668fc13..8ed16905b5099 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -3007,8 +3007,15 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
+  WALK_NESTED_ENUM(OmpAlwaysModifier, Value)
+  WALK_NESTED_ENUM(OmpCloseModifier, Value)
+  WALK_NESTED_ENUM(OmpDeleteModifier, Value)
+  WALK_NESTED_ENUM(OmpPresentModifier, Value)
+  WALK_NESTED_ENUM(OmpRefModifier, Value)
+  WALK_NESTED_ENUM(OmpSelfModifier, Value)
   WALK_NESTED_ENUM(OmpTraitSelectorName, Value)
   WALK_NESTED_ENUM(OmpTraitSetSelectorName, Value)
+  WALK_NESTED_ENUM(OmpxHoldModifier, Value)
 
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index cf05d8463277f..46aaab19ded0a 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -88,6 +89,8 @@ class CanonicalizationOfOmp {
     CanonicalizeUtilityConstructs(spec);
   }
 
+  void Post(parser::OmpMapClause &map) { CanonicalizeMapModifiers(map); }
+
 private:
   template <typename T> T *GetConstructIf(parser::ExecutionPartConstruct &x) {
     if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
@@ -390,16 +393,53 @@ class CanonicalizationOfOmp {
     omps.erase(rlast.base(), omps.end());
   }
 
+  // Map clause modifiers are parsed as per OpenMP 6.0 spec. That spec has
+  // changed properties of some of the modifiers, for example it has expanded
+  // map-type-modifier into 3 individual modifiers (one for each of the
+  // possible values of the original modifier), and the "map-type" modifier
+  // is no longer ultimate.
+  // To utilize the modifier validation framework for semantic checks,
+  // if the specified OpenMP version is less than 6.0, rewrite the affected
+  // modifiers back into the pre-6.0 forms.
+  void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
+    // OmpDeleteModifier -> OmpMapType
+    using Modifier = parser::OmpMapClause::Modifier;
+    using Modifiers = std::optional<std::list<Modifier>>;
+    auto &modifiers{std::get<Modifiers>(map.t)};
+    if (!modifiers) {
+      return;
+    }
+
+    using MapTypeModifier = parser::OmpMapTypeModifier;
+    using MapType = parser::OmpMapType;
+
+    for (auto &mod : *modifiers) {
+      if (std::holds_alternative<parser::OmpAlwaysModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Always);
+      } else if (std::holds_alternative<parser::OmpCloseModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Close);
+      } else if (std::holds_alternative<parser::OmpPresentModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Present);
+      } else if (std::holds_alternative<parser::OmpxHoldModifier>(mod.u)) {
+        mod.u = MapTypeModifier(MapTypeModifier::Value::Ompx_Hold);
+      } else if (std::holds_alternative<parser::OmpDeleteModifier>(mod.u)) {
+        mod.u = MapType(MapType::Value::Delete);
+      }
+    }
+  }
+
   // Mapping from the specification parts to the blocks that follow in the
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index c45d6bbbf9062..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,11 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
-}
+class SemanticsContext;
+
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+} // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index c84e832ee52a1..336ce4beb24ba 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -140,6 +140,22 @@ OmpGetDescriptor<parser::OmpAllocatorSimpleModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlwaysModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"always-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   static const OmpModifierDescriptor desc{
@@ -156,6 +172,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpCloseModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"close-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   static const OmpModifierDescriptor desc{
@@ -173,6 +205,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpContextSelector>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDeleteModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"delete-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique, OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   static const OmpModifierDescriptor desc{
@@ -347,6 +396,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapType>() {
       /*props=*/
       {
           {45, {OmpProperty::Ultimate}},
+          {60, {OmpProperty::Unique}},
       },
       /*clauses=*/
       {
@@ -367,6 +417,7 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpMapTypeModifier>() {
       /*clauses=*/
       {
           {45, {Clause::OMPC_map}},
+          {60, {}},
       },
   };
   return desc;
@@ -420,6 +471,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPrescriptiveness>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPresentModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"present-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpReductionIdentifier>() {
@@ -456,6 +523,38 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpReductionModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpRefModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ref-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpSelfModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"self-modifier",
+      /*props=*/
+      {
+          {60, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {60, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpStepComplexModifier>() {
@@ -522,4 +621,20 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpVariableCategory>() {
   };
   return desc;
 }
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpxHoldModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"ompx-hold-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_map}},
+      },
+  };
+  return desc;
+}
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 151f4ccae634e..0b860314b4a1f 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -734,6 +734,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
+      default:
+        break;
       }
     }
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index ab78605d01f4c..d0a5b200d0d8e 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -643,8 +643,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}
diff --git a/flang/test/Parser/OpenMP/map-modifiers-v60.f90 b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..bc80886780d46
--- /dev/null
+++ b/flang/test/Parser/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,113 @@
+!RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree-no-sema -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x
+  !$omp target map(always, close, delete, present, ompx_hold: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(ALWAYS, CLOSE, DELETE, PRESENT, OMPX_HOLD: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpAlwaysModifier -> Value = Always
+!PARSE-TREE: | | Modifier -> OmpCloseModifier -> Value = Close
+!PARSE-TREE: | | Modifier -> OmpDeleteModifier -> Value = Delete
+!PARSE-TREE: | | Modifier -> OmpPresentModifier -> Value = Present
+!PARSE-TREE: | | Modifier -> OmpxHoldModifier -> Value = Ompx_Hold
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f01(x)
+  integer :: x
+  !$omp target map(self, storage: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: !$OMP TARGET  MAP(SELF, STORAGE: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpSelfModifier -> Value = Self
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = Storage
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f02(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f03(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'
+
+subroutine f04(x)
+  integer, pointer :: x
+  !$omp target map(ref_ptr_ptee, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f04 (x)
+!UNPARSE:  INTEGER, POINTER :: x
+!UNPARSE: !$OMP TARGET  MAP(REF_PTR_PTEE, TO: x)
+!UNPARSE:   x = x+1
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Modifier -> OmpRefModifier -> Value = Ref_Ptr_Ptee
+!PARSE-TREE: | | Modifier -> OmpMapType -> Value = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | bool = 'true'

>From acc1a7d3da0d2bd5a60096a9f5dc27338342b952 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 08:04:17 -0500
Subject: [PATCH 02/11] [flang][OpenMP] Sema checks, lowering with new format
 of MAP modifiers

OpenMP 6.0 has changed the modifiers on the MAP clause. Previous patch
has introduced parsing support for them. This patch introduces processing
of the new forms in semantic checks and in lowering. This only applies
to existing modifiers, which were updated in the 6.0 spec. Any of the
newly introduced modifiers (SELF and REF) are ignored.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  16 +--
 flang/lib/Lower/OpenMP/Clauses.cpp            | 113 ++++++++++++------
 flang/lib/Semantics/canonicalize-omp.cpp      |   5 +
 flang/lib/Semantics/check-omp-structure.cpp   | 107 +++++++++++------
 flang/lib/Semantics/check-omp-structure.h     |   4 +-
 flang/lib/Semantics/openmp-utils.cpp          |  25 ++++
 flang/lib/Semantics/openmp-utils.h            |   3 +
 flang/lib/Semantics/resolve-directives.cpp    |  50 ++++++--
 .../Semantics/OpenMP/combined-constructs.f90  |   8 +-
 .../Semantics/OpenMP/device-constructs.f90    |   6 +-
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |   9 +-
 .../Frontend/OpenMP/ConstructDecompositionT.h |   5 +-
 llvm/include/llvm/Frontend/OpenMP/OMP.h       |   2 +
 llvm/lib/Frontend/OpenMP/OMP.cpp              |  14 +++
 14 files changed, 260 insertions(+), 107 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 74087d42a8e6e..ec71014c36093 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1315,7 +1315,8 @@ bool ClauseProcessor::processMap(
                      const parser::CharBlock &source) {
     using Map = omp::clause::Map;
     mlir::Location clauseLocation = converter.genLocation(source);
-    const auto &[mapType, typeMods, mappers, iterator, objects] = clause.t;
+    const auto &[mapType, typeMods, refMod, mappers, iterator, objects] =
+        clause.t;
     llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
         llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
     std::string mapperIdName = "__implicit_mapper";
@@ -1342,16 +1343,13 @@ bool ClauseProcessor::processMap(
       mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
                      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
       break;
-    case Map::MapType::Alloc:
-    case Map::MapType::Release:
+    case Map::MapType::Storage:
       // alloc and release is the default map_type for the Target Data
       // Ops, i.e. if no bits for map_type is supplied then alloc/release
-      // is implicitly assumed based on the target directive. Default
-      // value for Target Data and Enter Data is alloc and for Exit Data
-      // it is release.
+      // (aka storage in 6.0+) is implicitly assumed based on the target
+      // directive. Default value for Target Data and Enter Data is alloc
+      // and for Exit Data it is release.
       break;
-    case Map::MapType::Delete:
-      mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
     }
 
     if (typeMods) {
@@ -1362,6 +1360,8 @@ bool ClauseProcessor::processMap(
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
       if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Close))
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
+      if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Delete))
+        mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
       if (llvm::is_contained(*typeMods, Map::MapTypeModifier::OmpxHold))
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
     }
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 22a07219d3a50..18e49719b6013 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1001,64 +1001,105 @@ Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpMapType::Value, Map::MapType,
+      convert1, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
       // clang-format off
-      MS(Alloc,    Alloc)
-      MS(Delete,   Delete)
-      MS(From,     From)
-      MS(Release,  Release)
-      MS(To,       To)
-      MS(Tofrom,   Tofrom)
+      MS(Always,    Always)
+      MS(Close,     Close)
+      MS(Ompx_Hold, OmpxHold)
+      MS(Present,   Present)
       // clang-format on
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert2, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
+      convert2, parser::OmpRefModifier::Value, Map::RefModifier,
       // clang-format off
-      MS(Always,    Always)
-      MS(Close,     Close)
-      MS(Ompx_Hold, OmpxHold)
-      MS(Present,   Present)
+      MS(Ref_Ptee,     RefPtee)
+      MS(Ref_Ptr,      RefPtr)
+      MS(Ref_Ptr_Ptee, RefPtrPtee)
       // clang-format on
   );
 
+  // Treat always, close, present, self, delete modifiers as map-type-
+  // modifiers.
   auto &mods = semantics::OmpGetModifiers(inp.v);
-  auto *t1 = semantics::OmpGetUniqueModifier<parser::OmpMapper>(mods);
-  auto *t2 = semantics::OmpGetUniqueModifier<parser::OmpIterator>(mods);
-  auto *t3 = semantics::OmpGetUniqueModifier<parser::OmpMapType>(mods);
-  auto &t4 = std::get<parser::OmpObjectList>(inp.v.t);
 
-  auto mappers = [&]() -> std::optional<List<Mapper>> {
-    if (t1)
-      return List<Mapper>{Mapper{makeObject(t1->v, semaCtx)}};
-    return std::nullopt;
-  }();
-
-  auto iterator = [&]() -> std::optional<Iterator> {
-    if (t2)
-      return makeIterator(*t2, semaCtx);
-    return std::nullopt;
-  }();
+  auto *t1 = semantics::OmpGetUniqueModifier<parser::OmpMapType>(mods);
+  auto &t2 = std::get<parser::OmpObjectList>(inp.v.t);
 
   auto type = [&]() -> std::optional<Map::MapType> {
-    if (t3)
-      return convert1(t3->v);
-    return std::nullopt;
+    std::optional<Map::MapType> type;
+    if (t1) {
+      switch (t1->v) {
+      case parser::OmpMapType::Value::Alloc:
+      case parser::OmpMapType::Value::Delete:
+      case parser::OmpMapType::Value::Release:
+      case parser::OmpMapType::Value::Storage:
+        type = Map::MapType::Storage;
+        break;
+      case parser::OmpMapType::Value::From:
+        type = Map::MapType::From;
+        break;
+      case parser::OmpMapType::Value::To:
+        type = Map::MapType::To;
+        break;
+      case parser::OmpMapType::Value::Tofrom:
+        type = Map::MapType::Tofrom;
+        break;
+      default:
+        break;
+      }
+    }
+    return type;
   }();
 
-  Map::MapTypeModifiers typeMods;
+  llvm::DenseSet<Map::MapTypeModifier> modSet;
+  if (t1 && t1->v == parser::OmpMapType::Value::Delete)
+    modSet.insert(Map::MapTypeModifier::Delete);
+
   for (auto *typeMod :
        semantics::OmpGetRepeatableModifier<parser::OmpMapTypeModifier>(mods)) {
-    typeMods.push_back(convert2(typeMod->v));
+    modSet.insert(convert1(typeMod->v));
   }
+  if (semantics::OmpGetUniqueModifier<parser::OmpAlwaysModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Always);
+  if (semantics::OmpGetUniqueModifier<parser::OmpCloseModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Close);
+  if (semantics::OmpGetUniqueModifier<parser::OmpDeleteModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Delete);
+  if (semantics::OmpGetUniqueModifier<parser::OmpPresentModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Present);
+  if (semantics::OmpGetUniqueModifier<parser::OmpSelfModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Self);
+  if (semantics::OmpGetUniqueModifier<parser::OmpxHoldModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::OmpxHold);
+
   std::optional<Map::MapTypeModifiers> maybeTypeMods{};
-  if (!typeMods.empty())
-    maybeTypeMods = std::move(typeMods);
+  if (!modSet.empty())
+    maybeTypeMods = Map::MapTypeModifiers(modSet.begin(), modSet.end());
+
+  auto refMod = [&]() -> std::optional<Map::RefModifier> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpRefModifier>(mods))
+      return convert2(t->v);
+    return std::nullopt;
+  }();
+
+  auto mappers = [&]() -> std::optional<List<Mapper>> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpMapper>(mods))
+      return List<Mapper>{Mapper{makeObject(t->v, semaCtx)}};
+    return std::nullopt;
+  }();
+
+  auto iterator = [&]() -> std::optional<Iterator> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpIterator>(mods))
+      return makeIterator(*t, semaCtx);
+    return std::nullopt;
+  }();
 
   return Map{{/*MapType=*/std::move(type),
               /*MapTypeModifiers=*/std::move(maybeTypeMods),
-              /*Mapper=*/std::move(mappers), /*Iterator=*/std::move(iterator),
-              /*LocatorList=*/makeObjects(t4, semaCtx)}};
+              /*RefModifier=*/std::move(refMod), /*Mapper=*/std::move(mappers),
+              /*Iterator=*/std::move(iterator),
+              /*LocatorList=*/makeObjects(t2, semaCtx)}};
 }
 
 Match make(const parser::OmpClause::Match &inp,
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 46aaab19ded0a..77e2fd6ca5097 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -402,6 +402,11 @@ class CanonicalizationOfOmp {
   // if the specified OpenMP version is less than 6.0, rewrite the affected
   // modifiers back into the pre-6.0 forms.
   void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    unsigned version{context_.langOptions().OpenMPVersion};
+    if (version >= 60) {
+      return;
+    }
+
     // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
     // OmpDeleteModifier -> OmpMapType
     using Modifier = parser::OmpMapClause::Modifier;
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2425265e196c6..94dee3d7afe21 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -37,6 +37,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
@@ -3399,23 +3400,22 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Detach &x) {
   }
 }
 
-void OmpStructureChecker::CheckAllowedMapTypes(
-    const parser::OmpMapType::Value &type,
-    const std::list<parser::OmpMapType::Value> &allowedMapTypeList) {
-  if (!llvm::is_contained(allowedMapTypeList, type)) {
-    std::string commaSeparatedMapTypes;
-    llvm::interleave(
-        allowedMapTypeList.begin(), allowedMapTypeList.end(),
-        [&](const parser::OmpMapType::Value &mapType) {
-          commaSeparatedMapTypes.append(parser::ToUpperCaseLetters(
-              parser::OmpMapType::EnumToString(mapType)));
-        },
-        [&] { commaSeparatedMapTypes.append(", "); });
-    context_.Say(GetContext().clauseSource,
-        "Only the %s map types are permitted "
-        "for MAP clauses on the %s directive"_err_en_US,
-        commaSeparatedMapTypes, ContextDirectiveAsFortran());
+void OmpStructureChecker::CheckAllowedMapTypes(parser::OmpMapType::Value type,
+    llvm::ArrayRef<parser::OmpMapType::Value> allowed) {
+  if (llvm::is_contained(allowed, type)) {
+    return;
   }
+
+  llvm::SmallVector<std::string> names;
+  llvm::transform(
+      allowed, std::back_inserter(names), [](parser::OmpMapType::Value val) {
+        return parser::ToUpperCaseLetters(
+            parser::OmpMapType::EnumToString(val));
+      });
+  llvm::sort(names);
+  context_.Say(GetContext().clauseSource,
+      "Only the %s map types are permitted for MAP clauses on the %s directive"_err_en_US,
+      llvm::join(names, ", "), ContextDirectiveAsFortran());
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
@@ -3436,27 +3436,62 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
     CheckIteratorModifier(*iter);
   }
   if (auto *type{OmpGetUniqueModifier<parser::OmpMapType>(modifiers)}) {
+    using Directive = llvm::omp::Directive;
     using Value = parser::OmpMapType::Value;
-    switch (GetContext().directive) {
-    case llvm::omp::Directive::OMPD_target:
-    case llvm::omp::Directive::OMPD_target_teams:
-    case llvm::omp::Directive::OMPD_target_teams_distribute:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
-    case llvm::omp::Directive::OMPD_target_data:
-      CheckAllowedMapTypes(
-          type->v, {Value::To, Value::From, Value::Tofrom, Value::Alloc});
-      break;
-    case llvm::omp::Directive::OMPD_target_enter_data:
-      CheckAllowedMapTypes(type->v, {Value::To, Value::Alloc});
-      break;
-    case llvm::omp::Directive::OMPD_target_exit_data:
-      CheckAllowedMapTypes(
-          type->v, {Value::From, Value::Release, Value::Delete});
-      break;
-    default:
-      break;
+
+    static auto isValidForVersion{
+        [](parser::OmpMapType::Value t, unsigned version) {
+          switch (t) {
+          case parser::OmpMapType::Value::Alloc:
+          case parser::OmpMapType::Value::Delete:
+          case parser::OmpMapType::Value::Release:
+            return version < 60;
+          case parser::OmpMapType::Value::Storage:
+            return version >= 60;
+          default:
+            return true;
+          }
+        }};
+
+    llvm::SmallVector<parser::OmpMapType::Value> mapEnteringTypes{[&]() {
+      llvm::SmallVector<parser::OmpMapType::Value> result;
+      for (size_t i{0}; i != parser::OmpMapType::Value_enumSize; ++i) {
+        auto t{static_cast<parser::OmpMapType::Value>(i)};
+        if (isValidForVersion(t, version) && IsMapEnteringType(t)) {
+          result.push_back(t);
+        }
+      }
+      return result;
+    }()};
+    llvm::SmallVector<parser::OmpMapType::Value> mapExitingTypes{[&]() {
+      llvm::SmallVector<parser::OmpMapType::Value> result;
+      for (size_t i{0}; i != parser::OmpMapType::Value_enumSize; ++i) {
+        auto t{static_cast<parser::OmpMapType::Value>(i)};
+        if (isValidForVersion(t, version) && IsMapExitingType(t)) {
+          result.push_back(t);
+        }
+      }
+      return result;
+    }()};
+
+    llvm::omp::Directive dir{GetContext().directive};
+    llvm::ArrayRef<llvm::omp::Directive> leafs{
+        llvm::omp::getLeafConstructsOrSelf(dir)};
+
+    if (llvm::is_contained(leafs, Directive::OMPD_target) ||
+        llvm::is_contained(leafs, Directive::OMPD_target_data)) {
+      if (version >= 60) {
+        // Map types listed in the decay table. [6.0:276]
+        CheckAllowedMapTypes(
+            type->v, {Value::Storage, Value::From, Value::To, Value::Tofrom});
+      } else {
+        CheckAllowedMapTypes(
+            type->v, {Value::Alloc, Value::From, Value::To, Value::Tofrom});
+      }
+    } else if (llvm::is_contained(leafs, Directive::OMPD_target_enter_data)) {
+      CheckAllowedMapTypes(type->v, mapEnteringTypes);
+    } else if (llvm::is_contained(leafs, Directive::OMPD_target_exit_data)) {
+      CheckAllowedMapTypes(type->v, mapExitingTypes);
     }
   }
 
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 6a877a5d0a7c0..f4a291dc255c8 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -179,8 +179,8 @@ class OmpStructureChecker
   void HasInvalidDistributeNesting(const parser::OpenMPLoopConstruct &x);
   void HasInvalidLoopBinding(const parser::OpenMPLoopConstruct &x);
   // specific clause related
-  void CheckAllowedMapTypes(const parser::OmpMapType::Value &,
-      const std::list<parser::OmpMapType::Value> &);
+  void CheckAllowedMapTypes(
+      parser::OmpMapType::Value, llvm::ArrayRef<parser::OmpMapType::Value>);
 
   const std::list<parser::OmpTraitProperty> &GetTraitPropertyList(
       const parser::OmpTraitSelector &);
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index f43d2cc75620e..da14507aa9fe6 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -143,6 +143,31 @@ bool IsVarOrFunctionRef(const MaybeExpr &expr) {
   }
 }
 
+bool IsMapEnteringType(parser::OmpMapType::Value type) {
+  switch (type) {
+  case parser::OmpMapType::Value::Alloc:
+  case parser::OmpMapType::Value::Storage:
+  case parser::OmpMapType::Value::To:
+  case parser::OmpMapType::Value::Tofrom:
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool IsMapExitingType(parser::OmpMapType::Value type) {
+  switch (type) {
+  case parser::OmpMapType::Value::Delete:
+  case parser::OmpMapType::Value::From:
+  case parser::OmpMapType::Value::Release:
+  case parser::OmpMapType::Value::Storage:
+  case parser::OmpMapType::Value::Tofrom:
+    return true;
+  default:
+    return false;
+  }
+}
+
 std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
   const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
   // ForwardOwningPointer           typedExpr
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index a96c008fb26e7..001fbeb45ceec 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -59,6 +59,9 @@ bool IsExtendedListItem(const Symbol &sym);
 bool IsVariableListItem(const Symbol &sym);
 bool IsVarOrFunctionRef(const MaybeExpr &expr);
 
+bool IsMapEnteringType(parser::OmpMapType::Value type);
+bool IsMapExitingType(parser::OmpMapType::Value type);
+
 std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr);
 std::optional<evaluate::DynamicType> GetDynamicType(
     const parser::Expr &parserExpr);
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 0b860314b4a1f..cbd14bc259d38 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -712,7 +712,15 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   void Post(const parser::EorLabel &eorLabel) { CheckSourceLabel(eorLabel.v); }
 
   void Post(const parser::OmpMapClause &x) {
-    Symbol::Flag ompFlag = Symbol::Flag::OmpMapToFrom;
+    unsigned version{context_.langOptions().OpenMPVersion};
+    llvm::omp::Directive id{GetContext().directive};
+    std::optional<Symbol::Flag> ompFlag;
+    // Expand "storage" into either "alloc" or "release", depending on the
+    // type of the construct.
+    Symbol::Flag ompFlagForStorage = llvm::omp::isMapEnteringConstruct(id)
+        ? Symbol::Flag::OmpMapAlloc
+        : Symbol::Flag::OmpMapRelease;
+
     auto &mods{OmpGetModifiers(x)};
     if (auto *mapType{OmpGetUniqueModifier<parser::OmpMapType>(mods)}) {
       switch (mapType->v) {
@@ -734,10 +742,29 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
-      default:
+      case parser::OmpMapType::Value::Storage:
+        ompFlag = ompFlagForStorage;
         break;
       }
     }
+    if (!ompFlag) {
+      if (version >= 60) {
+        // [6.0:275:12-15]
+        // When a map-type is not specified for a clause on which it may be
+        // specified, the map-type defaults to storage if the delete-modifier
+        // is present on the clause or if the list item for which the map-type
+        // is not specified is an assumed-size array.
+        if (OmpGetUniqueModifier<parser::OmpDeleteModifier>(mods)) {
+          ompFlag = ompFlagForStorage;
+        }
+        // Otherwise, if delete-modifier is absent, leave ompFlag unset.
+      } else {
+        // [5.2:151:10]
+        // If a map-type is not specified, the map-type defaults to tofrom.
+        ompFlag = Symbol::Flag::OmpMapToFrom;
+      }
+    }
+
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
     for (const auto &ompObj : ompObjList.v) {
       common::visit(
@@ -746,15 +773,14 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
                 if (const auto *name{
                         semantics::getDesignatorNameIfDataRef(designator)}) {
                   if (name->symbol) {
-                    name->symbol->set(ompFlag);
-                    AddToContextObjectWithDSA(*name->symbol, ompFlag);
-                  }
-                  if (name->symbol &&
-                      semantics::IsAssumedSizeArray(*name->symbol)) {
-                    context_.Say(designator.source,
-                        "Assumed-size whole arrays may not appear on the %s "
-                        "clause"_err_en_US,
-                        "MAP");
+                    name->symbol->set(ompFlag.value_or(ompFlagForStorage));
+                    AddToContextObjectWithDSA(*name->symbol, *ompFlag);
+                    if (semantics::IsAssumedSizeArray(*name->symbol)) {
+                      context_.Say(designator.source,
+                          "Assumed-size whole arrays may not appear on the %s "
+                          "clause"_err_en_US,
+                          "MAP");
+                    }
                   }
                 }
               },
@@ -762,7 +788,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
           },
           ompObj.u);
 
-      ResolveOmpObject(ompObj, ompFlag);
+      ResolveOmpObject(ompObj, ompFlag.value_or(ompFlagForStorage));
     }
   }
 
diff --git a/flang/test/Semantics/OpenMP/combined-constructs.f90 b/flang/test/Semantics/OpenMP/combined-constructs.f90
index 4f2a4a4f501b9..2298d33ef33eb 100644
--- a/flang/test/Semantics/OpenMP/combined-constructs.f90
+++ b/flang/test/Semantics/OpenMP/combined-constructs.f90
@@ -207,7 +207,7 @@ program main
   enddo
   !$omp end target teams
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS directive
   !$omp target teams map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -307,7 +307,7 @@ program main
   enddo
   !$omp end target teams distribute
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -400,7 +400,7 @@ program main
   enddo
   !$omp end target teams distribute parallel do
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -500,7 +500,7 @@ program main
   enddo
   !$omp end target teams distribute parallel do simd
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd map(delete:a)
   do i = 1, N
      a(i) = 3.14
diff --git a/flang/test/Semantics/OpenMP/device-constructs.f90 b/flang/test/Semantics/OpenMP/device-constructs.f90
index 6f545b9021966..431e0f88e3237 100644
--- a/flang/test/Semantics/OpenMP/device-constructs.f90
+++ b/flang/test/Semantics/OpenMP/device-constructs.f90
@@ -123,7 +123,7 @@ program main
   enddo
   !$omp end target
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET directive
   !$omp target map(delete:a)
   do i = 1, N
      a = 3.14
@@ -160,7 +160,7 @@ program main
   !ERROR: At most one IF clause can appear on the TARGET ENTER DATA directive
   !$omp target enter data map(to:a) if(.true.) if(.false.)
 
-  !ERROR: Only the TO, ALLOC map types are permitted for MAP clauses on the TARGET ENTER DATA directive
+  !ERROR: Only the ALLOC, TO, TOFROM map types are permitted for MAP clauses on the TARGET ENTER DATA directive
   !$omp target enter data map(from:a)
 
   !$omp target exit data map(delete:a)
@@ -168,7 +168,7 @@ program main
   !ERROR: At most one DEVICE clause can appear on the TARGET EXIT DATA directive
   !$omp target exit data map(from:a) device(0) device(1)
 
-  !ERROR: Only the FROM, RELEASE, DELETE map types are permitted for MAP clauses on the TARGET EXIT DATA directive
+  !ERROR: Only the DELETE, FROM, RELEASE, TOFROM map types are permitted for MAP clauses on the TARGET EXIT DATA directive
   !$omp target exit data map(to:a)
 
   !$omp target update if(.true.) device(1) to(a) from(b) depend(inout:c) nowait
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index de888ff86fe91..7919f7a8b0c34 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -779,16 +779,17 @@ struct LinkT {
 template <typename T, typename I, typename E> //
 struct MapT {
   using LocatorList = ObjectListT<I, E>;
-  ENUM(MapType, To, From, Tofrom, Alloc, Release, Delete);
-  ENUM(MapTypeModifier, Always, Close, Present, OmpxHold);
+  ENUM(MapType, To, From, Tofrom, Storage);
+  ENUM(MapTypeModifier, Always, Close, Delete, Present, Self, OmpxHold);
+  ENUM(RefModifier, RefPtee, RefPtr, RefPtrPtee);
   // See note at the definition of the MapperT type.
   using Mappers = ListT<type::MapperT<I, E>>; // Not a spec name
   using Iterator = type::IteratorT<T, I, E>;
   using MapTypeModifiers = ListT<MapTypeModifier>; // Not a spec name
 
   using TupleTrait = std::true_type;
-  std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(Mappers), OPT(Iterator),
-             LocatorList>
+  std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(RefModifier),
+             OPT(Mappers), OPT(Iterator), LocatorList>
       t;
 };
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
index 611bfe3f8aced..047baa3a79f5d 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -708,6 +708,7 @@ bool ConstructDecompositionT<C, H>::applyClause(
                      tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
                          {/*MapType=*/MapType::Tofrom,
                           /*MapTypeModifier=*/std::nullopt,
+                          /*RefModifier=*/std::nullopt,
                           /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
                           /*LocatorList=*/std::move(tofrom)}});
       dirTarget->clauses.push_back(map);
@@ -969,8 +970,8 @@ bool ConstructDecompositionT<C, H>::applyClause(
           llvm::omp::Clause::OMPC_map,
           tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
               {/*MapType=*/MapType::Tofrom, /*MapTypeModifier=*/std::nullopt,
-               /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
-               /*LocatorList=*/std::move(tofrom)}});
+               /*RefModifier=*/std::nullopt, /*Mapper=*/std::nullopt,
+               /*Iterator=*/std::nullopt, /*LocatorList=*/std::move(tofrom)}});
 
       dirTarget->clauses.push_back(map);
       applied = true;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index d44c33301bde7..a8b4503147034 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -64,6 +64,8 @@ static constexpr inline bool isPrivatizingClause(Clause C) {
   }
 }
 
+LLVM_ABI bool isMapEnteringConstruct(Directive D);
+
 static constexpr unsigned FallbackVersion = 52;
 LLVM_ABI ArrayRef<unsigned> getOpenMPVersions();
 
diff --git a/llvm/lib/Frontend/OpenMP/OMP.cpp b/llvm/lib/Frontend/OpenMP/OMP.cpp
index 555e2a61e411e..c23fb400666d6 100644
--- a/llvm/lib/Frontend/OpenMP/OMP.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMP.cpp
@@ -189,6 +189,20 @@ bool isCombinedConstruct(Directive D) {
   return !getLeafConstructs(D).empty() && !isCompositeConstruct(D);
 }
 
+bool isMapEnteringConstruct(Directive D) {
+  for (Directive L : getLeafConstructsOrSelf(D)) {
+    switch (L) {
+    case OMPD_target:
+    case OMPD_target_data:
+    case OMPD_target_enter_data:
+      return true;
+    default:
+      break;
+    }
+  }
+  return false;
+}
+
 ArrayRef<unsigned> getOpenMPVersions() {
   static unsigned Versions[]{31, 40, 45, 50, 51, 52, 60};
   return Versions;

>From 0733ec13ae69e2ec3d462987c279ef083890e95c Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 11:41:55 -0500
Subject: [PATCH 03/11] Simplify converting MapType, NFC

---
 flang/lib/Lower/OpenMP/Clauses.cpp | 47 +++++++++++++-----------------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 18e49719b6013..ece4a8eaac8cd 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1001,7 +1001,20 @@ Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
+      convert1, parser::OmpMapType::Value, Map::MapType,
+      // clang-format off
+      MS(Alloc,   Storage)
+      MS(Delete,  Storage)
+      MS(Release, Storage)
+      MS(Storage, Storage)
+      MS(From,    From)
+      MS(To,      To)
+      MS(Tofrom,  Tofrom)
+      // clang-format on
+  );
+
+  CLAUSET_ENUM_CONVERT( //
+      convert2, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
       // clang-format off
       MS(Always,    Always)
       MS(Close,     Close)
@@ -1011,7 +1024,7 @@ Map make(const parser::OmpClause::Map &inp,
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert2, parser::OmpRefModifier::Value, Map::RefModifier,
+      convert3, parser::OmpRefModifier::Value, Map::RefModifier,
       // clang-format off
       MS(Ref_Ptee,     RefPtee)
       MS(Ref_Ptr,      RefPtr)
@@ -1027,29 +1040,9 @@ Map make(const parser::OmpClause::Map &inp,
   auto &t2 = std::get<parser::OmpObjectList>(inp.v.t);
 
   auto type = [&]() -> std::optional<Map::MapType> {
-    std::optional<Map::MapType> type;
-    if (t1) {
-      switch (t1->v) {
-      case parser::OmpMapType::Value::Alloc:
-      case parser::OmpMapType::Value::Delete:
-      case parser::OmpMapType::Value::Release:
-      case parser::OmpMapType::Value::Storage:
-        type = Map::MapType::Storage;
-        break;
-      case parser::OmpMapType::Value::From:
-        type = Map::MapType::From;
-        break;
-      case parser::OmpMapType::Value::To:
-        type = Map::MapType::To;
-        break;
-      case parser::OmpMapType::Value::Tofrom:
-        type = Map::MapType::Tofrom;
-        break;
-      default:
-        break;
-      }
-    }
-    return type;
+    if (t1)
+      return convert1(t1->v);
+    return std::nullopt;
   }();
 
   llvm::DenseSet<Map::MapTypeModifier> modSet;
@@ -1058,7 +1051,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   for (auto *typeMod :
        semantics::OmpGetRepeatableModifier<parser::OmpMapTypeModifier>(mods)) {
-    modSet.insert(convert1(typeMod->v));
+    modSet.insert(convert2(typeMod->v));
   }
   if (semantics::OmpGetUniqueModifier<parser::OmpAlwaysModifier>(mods))
     modSet.insert(Map::MapTypeModifier::Always);
@@ -1079,7 +1072,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   auto refMod = [&]() -> std::optional<Map::RefModifier> {
     if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpRefModifier>(mods))
-      return convert2(t->v);
+      return convert3(t->v);
     return std::nullopt;
   }();
 

>From a7476b1ca983ecf0244b71ab873d3775343e5c46 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 16 Jul 2025 12:43:14 -0500
Subject: [PATCH 04/11] Fix unit test

---
 .../Frontend/OpenMPDecompositionTest.cpp         | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
index 6189d0954891b..95c26b10c9a0c 100644
--- a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
@@ -431,8 +431,8 @@ TEST_F(OpenMPDecompositionTest, Firstprivate3) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (12), (27)
-  ASSERT_EQ(Dir1, "teams shared(x)");          // (6), (17)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (12), (27)
+  ASSERT_EQ(Dir1, "teams shared(x)");            // (6), (17)
   ASSERT_EQ(Dir2, "distribute firstprivate(x) lastprivate(, (x))"); // (5), (21)
 }
 
@@ -574,9 +574,9 @@ TEST_F(OpenMPDecompositionTest, Lastprivate3) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (21), (27)
-  ASSERT_EQ(Dir1, "parallel shared(x)");       // (22)
-  ASSERT_EQ(Dir2, "do lastprivate(, (x))");    // (21)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (21), (27)
+  ASSERT_EQ(Dir1, "parallel shared(x)");         // (22)
+  ASSERT_EQ(Dir2, "do lastprivate(, (x))");      // (21)
 }
 
 // SHARED
@@ -984,9 +984,9 @@ TEST_F(OpenMPDecompositionTest, Reduction7) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (36), (10)
-  ASSERT_EQ(Dir1, "parallel shared(x)");       // (36), (1), (4)
-  ASSERT_EQ(Dir2, "do reduction(, (3), (x))"); // (36)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (36), (10)
+  ASSERT_EQ(Dir1, "parallel shared(x)");         // (36), (1), (4)
+  ASSERT_EQ(Dir2, "do reduction(, (3), (x))");   // (36)
 }
 
 // IF

>From faff0d32746640c34988a65552e277198748fcf2 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 08:49:18 -0500
Subject: [PATCH 05/11] Don't pass SemanticsContext to CanonicalizeOmp yet

---
 flang/lib/Semantics/canonicalize-omp.cpp | 10 ++++------
 flang/lib/Semantics/canonicalize-omp.h   |  5 +++--
 flang/lib/Semantics/semantics.cpp        |  3 ++-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 46aaab19ded0a..bba4d72a702e9 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,8 +27,7 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(SemanticsContext &context)
-      : context_{context}, messages_{context.messages()} {}
+  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -433,13 +432,12 @@ class CanonicalizationOfOmp {
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
-  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
-  CanonicalizationOfOmp omp{context};
+bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
+  CanonicalizationOfOmp omp{messages};
   Walk(program, omp);
-  return !context.messages().AnyFatalError();
+  return !messages.AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index 3251218da35ed..23350c522a775 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,12 +11,13 @@
 
 namespace Fortran::parser {
 struct Program;
-}
+class Messages;
+} // namespace Fortran::parser
 
 namespace Fortran::semantics {
 class SemanticsContext;
 
-bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
+bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
 } // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 6db11aaf56c2a..b15ed057b52f2 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -642,7 +642,8 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_.messages(), program_) &&
+      CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}

>From 724cc8306faae084c4a6ff8295073bd65bc0f609 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 08:54:34 -0500
Subject: [PATCH 06/11] Pass SemanticsContext to CanonicalizeOmp

---
 flang/lib/Semantics/canonicalize-omp.cpp | 10 ++++++----
 flang/lib/Semantics/canonicalize-omp.h   |  5 ++---
 flang/lib/Semantics/semantics.cpp        |  3 +--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 9b74a153b233e..77e2fd6ca5097 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -437,12 +438,13 @@ class CanonicalizationOfOmp {
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index 23350c522a775..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,13 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
 class SemanticsContext;
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
 } // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index b15ed057b52f2..6db11aaf56c2a 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -642,8 +642,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}

>From 3886bbcd0477b5932b1c9dc1303bbe3a940bd179 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 09:20:25 -0500
Subject: [PATCH 07/11] Invent OmpMapStorage for map flags

We're using "storage" instead of "alloc"/"release" in lowering, this will
make things more consistent.
---
 flang/include/flang/Semantics/symbol.h     |  5 ++---
 flang/lib/Semantics/resolve-directives.cpp | 26 +++++++---------------
 flang/lib/Semantics/symbol.cpp             |  3 +--
 3 files changed, 11 insertions(+), 23 deletions(-)

diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index b319e2c7e5e74..5bde9f39ca0b0 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -812,9 +812,8 @@ class Symbol {
       // OpenMP data-sharing attribute
       OmpShared, OmpPrivate, OmpLinear, OmpFirstPrivate, OmpLastPrivate,
       // OpenMP data-mapping attribute
-      OmpMapTo, OmpMapFrom, OmpMapToFrom, OmpMapAlloc, OmpMapRelease,
-      OmpMapDelete, OmpUseDevicePtr, OmpUseDeviceAddr, OmpIsDevicePtr,
-      OmpHasDeviceAddr,
+      OmpMapTo, OmpMapFrom, OmpMapToFrom, OmpMapStorage, OmpMapDelete,
+      OmpUseDevicePtr, OmpUseDeviceAddr, OmpIsDevicePtr, OmpHasDeviceAddr,
       // OpenMP data-copying attribute
       OmpCopyIn, OmpCopyPrivate,
       // OpenMP miscellaneous flags
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index e02ff97dc6917..4c3e509b5a36d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -728,13 +728,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
 
   void Post(const parser::OmpMapClause &x) {
     unsigned version{context_.langOptions().OpenMPVersion};
-    llvm::omp::Directive id{GetContext().directive};
     std::optional<Symbol::Flag> ompFlag;
-    // Expand "storage" into either "alloc" or "release", depending on the
-    // type of the construct.
-    Symbol::Flag ompFlagForStorage = llvm::omp::isMapEnteringConstruct(id)
-        ? Symbol::Flag::OmpMapAlloc
-        : Symbol::Flag::OmpMapRelease;
 
     auto &mods{OmpGetModifiers(x)};
     if (auto *mapType{OmpGetUniqueModifier<parser::OmpMapType>(mods)}) {
@@ -749,17 +743,13 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
         ompFlag = Symbol::Flag::OmpMapToFrom;
         break;
       case parser::OmpMapType::Value::Alloc:
-        ompFlag = Symbol::Flag::OmpMapAlloc;
-        break;
       case parser::OmpMapType::Value::Release:
-        ompFlag = Symbol::Flag::OmpMapRelease;
+      case parser::OmpMapType::Value::Storage:
+        ompFlag = Symbol::Flag::OmpMapStorage;
         break;
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
-      case parser::OmpMapType::Value::Storage:
-        ompFlag = ompFlagForStorage;
-        break;
       }
     }
     if (!ompFlag) {
@@ -770,7 +760,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
         // is present on the clause or if the list item for which the map-type
         // is not specified is an assumed-size array.
         if (OmpGetUniqueModifier<parser::OmpDeleteModifier>(mods)) {
-          ompFlag = ompFlagForStorage;
+          ompFlag = Symbol::Flag::OmpMapStorage;
         }
         // Otherwise, if delete-modifier is absent, leave ompFlag unset.
       } else {
@@ -788,7 +778,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
                 if (const auto *name{
                         semantics::getDesignatorNameIfDataRef(designator)}) {
                   if (name->symbol) {
-                    name->symbol->set(ompFlag.value_or(ompFlagForStorage));
+                    name->symbol->set(
+                        ompFlag.value_or(Symbol::Flag::OmpMapStorage));
                     AddToContextObjectWithDSA(*name->symbol, *ompFlag);
                     if (semantics::IsAssumedSizeArray(*name->symbol)) {
                       context_.Say(designator.source,
@@ -803,7 +794,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
           },
           ompObj.u);
 
-      ResolveOmpObject(ompObj, ompFlag.value_or(ompFlagForStorage));
+      ResolveOmpObject(ompObj, ompFlag.value_or(Symbol::Flag::OmpMapStorage));
     }
   }
 
@@ -2802,9 +2793,8 @@ void OmpAttributeVisitor::ResolveOmpObject(
                   }
                   Symbol::Flag dataMappingAttributeFlags[] = {
                       Symbol::Flag::OmpMapTo, Symbol::Flag::OmpMapFrom,
-                      Symbol::Flag::OmpMapToFrom, Symbol::Flag::OmpMapAlloc,
-                      Symbol::Flag::OmpMapRelease, Symbol::Flag::OmpMapDelete,
-                      Symbol::Flag::OmpIsDevicePtr,
+                      Symbol::Flag::OmpMapToFrom, Symbol::Flag::OmpMapStorage,
+                      Symbol::Flag::OmpMapDelete, Symbol::Flag::OmpIsDevicePtr,
                       Symbol::Flag::OmpHasDeviceAddr};
 
                   Symbol::Flag dataSharingAttributeFlags[] = {
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index 0380207927ad3..2259cfcf23ece 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -861,8 +861,7 @@ std::string Symbol::OmpFlagToClauseName(Symbol::Flag ompFlag) {
   case Symbol::Flag::OmpMapTo:
   case Symbol::Flag::OmpMapFrom:
   case Symbol::Flag::OmpMapToFrom:
-  case Symbol::Flag::OmpMapAlloc:
-  case Symbol::Flag::OmpMapRelease:
+  case Symbol::Flag::OmpMapStorage:
   case Symbol::Flag::OmpMapDelete:
     clauseName = "MAP";
     break;

>From 20bc43526e52676b4571fe85c3fad35cad70a115 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 09:31:44 -0500
Subject: [PATCH 08/11] Add testcase for using v60 modifiers in v52

---
 .../Semantics/OpenMP/map-modifiers-v60.f90    | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 flang/test/Semantics/OpenMP/map-modifiers-v60.f90

diff --git a/flang/test/Semantics/OpenMP/map-modifiers-v60.f90 b/flang/test/Semantics/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..b3f2e5171d6d6
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,35 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52 -Werror
+
+subroutine f00(x)
+  integer :: x
+!WARNING: 'self-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(self: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f01(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptr: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f02(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptee: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f03(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptr_ptee: x)
+  x = x + 1
+  !$omp end target
+end
+
+

>From 7d232c536a47aa705107db336259e4470335e350 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 09:35:48 -0500
Subject: [PATCH 09/11] Clearer names for enum-converting functions for map
 modifiers

---
 flang/lib/Lower/OpenMP/Clauses.cpp | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index ece4a8eaac8cd..686fba0154f44 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1001,7 +1001,7 @@ Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpMapType::Value, Map::MapType,
+      convertMapType, parser::OmpMapType::Value, Map::MapType,
       // clang-format off
       MS(Alloc,   Storage)
       MS(Delete,  Storage)
@@ -1014,7 +1014,8 @@ Map make(const parser::OmpClause::Map &inp,
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert2, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
+      convertMapTypeMod, parser::OmpMapTypeModifier::Value,
+      Map::MapTypeModifier,
       // clang-format off
       MS(Always,    Always)
       MS(Close,     Close)
@@ -1024,7 +1025,7 @@ Map make(const parser::OmpClause::Map &inp,
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert3, parser::OmpRefModifier::Value, Map::RefModifier,
+      convertRefMod, parser::OmpRefModifier::Value, Map::RefModifier,
       // clang-format off
       MS(Ref_Ptee,     RefPtee)
       MS(Ref_Ptr,      RefPtr)
@@ -1041,7 +1042,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   auto type = [&]() -> std::optional<Map::MapType> {
     if (t1)
-      return convert1(t1->v);
+      return convertMapType(t1->v);
     return std::nullopt;
   }();
 
@@ -1051,7 +1052,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   for (auto *typeMod :
        semantics::OmpGetRepeatableModifier<parser::OmpMapTypeModifier>(mods)) {
-    modSet.insert(convert2(typeMod->v));
+    modSet.insert(convertMapTypeMod(typeMod->v));
   }
   if (semantics::OmpGetUniqueModifier<parser::OmpAlwaysModifier>(mods))
     modSet.insert(Map::MapTypeModifier::Always);
@@ -1072,7 +1073,7 @@ Map make(const parser::OmpClause::Map &inp,
 
   auto refMod = [&]() -> std::optional<Map::RefModifier> {
     if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpRefModifier>(mods))
-      return convert3(t->v);
+      return convertRefMod(t->v);
     return std::nullopt;
   }();
 

>From dd339d61a4e10e88edf2c569af7be3fd9684beea Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 09:57:08 -0500
Subject: [PATCH 10/11] Remove now unused isMapEnteringConstruct

---
 llvm/include/llvm/Frontend/OpenMP/OMP.h |  2 --
 llvm/lib/Frontend/OpenMP/OMP.cpp        | 14 --------------
 2 files changed, 16 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
index 48217384afca6..9d0a55432e1ae 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -68,8 +68,6 @@ static constexpr inline bool isPrivatizingClause(Clause C) {
   }
 }
 
-LLVM_ABI bool isMapEnteringConstruct(Directive D);
-
 static constexpr unsigned FallbackVersion = 52;
 LLVM_ABI ArrayRef<unsigned> getOpenMPVersions();
 
diff --git a/llvm/lib/Frontend/OpenMP/OMP.cpp b/llvm/lib/Frontend/OpenMP/OMP.cpp
index c23fb400666d6..555e2a61e411e 100644
--- a/llvm/lib/Frontend/OpenMP/OMP.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMP.cpp
@@ -189,20 +189,6 @@ bool isCombinedConstruct(Directive D) {
   return !getLeafConstructs(D).empty() && !isCompositeConstruct(D);
 }
 
-bool isMapEnteringConstruct(Directive D) {
-  for (Directive L : getLeafConstructsOrSelf(D)) {
-    switch (L) {
-    case OMPD_target:
-    case OMPD_target_data:
-    case OMPD_target_enter_data:
-      return true;
-    default:
-      break;
-    }
-  }
-  return false;
-}
-
 ArrayRef<unsigned> getOpenMPVersions() {
   static unsigned Versions[]{31, 40, 45, 50, 51, 52, 60};
   return Versions;

>From 20c75913d2db8bba3f6245e56741367c0a76a601 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 11:08:38 -0500
Subject: [PATCH 11/11] Check translation of map-type-modifiers

---
 flang/test/Lower/OpenMP/map-modifiers.f90 | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/flang/test/Lower/OpenMP/map-modifiers.f90 b/flang/test/Lower/OpenMP/map-modifiers.f90
index 64d7869cbb836..be93c14627f9a 100644
--- a/flang/test/Lower/OpenMP/map-modifiers.f90
+++ b/flang/test/Lower/OpenMP/map-modifiers.f90
@@ -1,4 +1,6 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=60 %s -o - | FileCheck %s
 
 subroutine map_present_target_data
     integer :: x
@@ -15,6 +17,14 @@ subroutine map_present_update
 !$omp target update to(present: x)
 end subroutine
 
+subroutine map_always
+    integer :: x
+!CHECK: %[[MAP:.*]] = omp.map.info {{.*}} map_clauses(always, tofrom) {{.*}} {name = "x"}
+!CHECK: omp.target_data map_entries(%[[MAP]] : {{.*}}) {
+!$omp target data map(always, tofrom: x)
+!$omp end target data
+end subroutine
+
 subroutine map_close
     integer :: x
 !CHECK: %[[MAP:.*]] = omp.map.info {{.*}} map_clauses(close, tofrom) {{.*}} {name = "x"}


From flang-commits at lists.llvm.org  Mon Jul 21 09:31:27 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Mon, 21 Jul 2025 09:31:27 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <687e6b5f.170a0220.3bc76a.e4e3@mx.google.com>


kparzysz wrote:

> Ahh okay. Maybe we only need one test then, but it would be good to make sure this makes it through the clause processor okay.

I added more RUN lines to flang/test/Lower/OpenMP/map-modifiers.f90 with different versions, plus a test for "always" modifier.

https://github.com/llvm/llvm-project/pull/149137

From flang-commits at lists.llvm.org  Mon Jul 21 10:05:42 2025
From: flang-commits at lists.llvm.org (Renato Golin via flang-commits)
Date: Mon, 21 Jul 2025 10:05:42 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e7366.170a0220.22de5.a5f8@mx.google.com>


rengolin wrote:

> Also, I'm not sure whether making `chip` optional is a good idea.

Older Arm targets didn't need a chip because the micro-architecture was encoded in the target triple. There can be the case where passing a chip name is not just confusing but wrong (if the target doesn't recognize the chip but can build a target with the correct set of features). For x86 and GPUs the chip names carry a lot of information, that is not encoded in the triple, so become necessary.

Arguably we don't usually care about armv7 or lower, but I'll let @banach-space comment on that. 

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 10:08:00 2025
From: flang-commits at lists.llvm.org (Renato Golin via flang-commits)
Date: Mon, 21 Jul 2025 10:08:00 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e73f0.170a0220.16e4de.f990@mx.google.com>


rengolin wrote:

> I imagine you want this for caching the TM. But why do you want this? When is it going to be reused? I think we should use it only to import the DL for now.

The DL generator is one of the users, not the only one. The cost model will be making a lot of enquires about the target over the life of the compiler passes, so this cache is necessary to avoid constructing it for every invocation. A storage class should do the trick, but may need some massaging at this level.



https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 10:08:50 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Mon, 21 Jul 2025 10:08:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687e7422.170a0220.26acd6.ef99@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
tblah wrote:

Ahh okay what I think is happening here is that APFloat is forcing its size to f64 because `llvm::numbers::pi` is defined as a double. I'm concerned that this could produce inaccurate results for real(16) (aka floating point types with more precision than `f64`: f128 on aarch64 systems, I think f80 on x86).

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Mon Jul 21 10:10:17 2025
From: flang-commits at lists.llvm.org (Fabian Mora via flang-commits)
Date: Mon, 21 Jul 2025 10:10:17 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e7479.630a0220.1a60cb.d6fe@mx.google.com>


fabianmcg wrote:

> Older Arm targets didn't need a chip because the micro-architecture was encoded in the target triple. There can be the case where passing a chip name is not just confusing but wrong (if the target doesn't recognize the chip but can build a target with the correct set of features). For x86 and GPUs the chip names carry a lot of information, that is not encoded in the triple, so become necessary.

In that particular case, I'd prefer an empty chip or a string indicating this is a special case. Because as you say, there are a lot of targets where the chip does carry a lot info, therefore I think we should handle the common case of required chip.

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 10:15:16 2025
From: flang-commits at lists.llvm.org (Fabian Mora via flang-commits)
Date: Mon, 21 Jul 2025 10:15:16 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e75a4.170a0220.3d3ec6.edee@mx.google.com>


fabianmcg wrote:

>  The cost model will be making a lot of enquires about the target over the life of the compiler passes, so this cache is necessary to avoid constructing it for every invocation.

Do you already have the design of how that looks? I'm not saying it's not necessary, I'm arguing that until such use is ready to land we shouldn't put it in, as we would be adding unused features, and it's easy to add later when the use is ready.

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 10:15:49 2025
From: flang-commits at lists.llvm.org (Renato Golin via flang-commits)
Date: Mon, 21 Jul 2025 10:15:49 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e75c5.a70a0220.2cb65b.f19f@mx.google.com>


rengolin wrote:

> In that particular case, I'd prefer an empty chip or a string indicating this is a special case. Because as you say, there are a lot of targets where the chip does carry a lot info, therefore I think we should handle the common case of required chip.

Empty string would be fine, yes. If Arm is fine with making it compulsory (and not empty), we should. Validation would be so much easier. 

If not, then we may need some custom validation per target to make sure that for the ones where the info is in the chip, it must not be empty.

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 10:16:40 2025
From: flang-commits at lists.llvm.org (Renato Golin via flang-commits)
Date: Mon, 21 Jul 2025 10:16:40 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [MLIR][LLVMIR][DLTI] Add
 #llvm.target, #llvm.data_layout and TargetAttrInterface (PR #145899)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145899@github.com>
Message-ID: <687e75f8.170a0220.60494.f182@mx.google.com>


rengolin wrote:

> Do you already have the design of how that looks? I'm not saying it's not necessary, I'm arguing that until such use is ready to land we shouldn't put it in, as we would be adding unused features, and it's easy to add later when the use is ready.

I'd be ok with that, too.

https://github.com/llvm/llvm-project/pull/145899

From flang-commits at lists.llvm.org  Mon Jul 21 10:29:30 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Mon, 21 Jul 2025 10:29:30 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687e78fa.630a0220.2ec9ca.edb0@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
c8ef wrote:

Yeah so I think the ultimate solution for trig-pi functions is to use the glibc version, when we have glibc>=2.41(it does provide f128 version). And if not maybe we can use this as a fallback or something like libquadmath. Or maybe we need a interface return `pi` based on fltSemantic. Not quite sure how to do it right since there are a lot of them.

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Mon Jul 21 11:53:52 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Mon, 21 Jul 2025 11:53:52 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Implement LOWER argument for
	C_F_POINTER (PR #149870)
Message-ID: <llvm.org/llvm/llvm-project/pull/149870@github.com>


https://github.com/mjklemm created https://github.com/llvm/llvm-project/pull/149870

This PR resolves issue #147819 and adds support for the F2023 extension of the `LOWER=` argument for `C_F_POINTER`.

>From aaa6fbb0429fd09b9fb250136bac832966bbcb08 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Mon, 21 Jul 2025 15:59:17 +0200
Subject: [PATCH 1/3] [Flang] Implement LOWER argument for C_F_POINTER

This PR adds support for the optional LOWER argument for C_F_POINTER
(Fortran 2023, 18.2.3.3)
---
 flang/docs/F202X.md                           |  1 -
 flang/lib/Evaluate/intrinsics.cpp             | 44 +++++++++++++++++--
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 26 +++++++++--
 3 files changed, 62 insertions(+), 9 deletions(-)

diff --git a/flang/docs/F202X.md b/flang/docs/F202X.md
index 67ea7fd97449a..7504561a64aa6 100644
--- a/flang/docs/F202X.md
+++ b/flang/docs/F202X.md
@@ -268,7 +268,6 @@ Addressing some issues and omissions in intrinsic modules:
  * LOGICAL8/16/32/64 and REAL16
  * IEEE module facilities upgraded to match latest IEEE FP standard
  * C_F_STRPOINTER, F_C_STRING for NUL-terminated strings
- * C_F_POINTER(LOWER=)
 
 #### Intrinsic Procedure Extensions
 
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9957010684d48..c48dea4b54b3a 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -3073,10 +3073,11 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
     ActualArguments &arguments, FoldingContext &context) const {
   characteristics::Procedure::Attrs attrs;
   attrs.set(characteristics::Procedure::Attr::Subroutine);
-  static const char *const keywords[]{"cptr", "fptr", "shape", nullptr};
+  static const char *const keywords[]{
+      "cptr", "fptr", "shape", "lower", nullptr};
   characteristics::DummyArguments dummies;
-  if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 1)) {
-    CHECK(arguments.size() == 3);
+  if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 2)) {
+    CHECK(arguments.size() == 4);
     if (const auto *expr{arguments[0].value().UnwrapExpr()}) {
       // General semantic checks will catch an actual argument that's not
       // scalar.
@@ -3169,11 +3170,30 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
           }
         }
       }
+      if (arguments[3] && fptrRank == 0) {
+        context.messages().Say(arguments[3]->sourceLocation(),
+            "LOWER= argument to C_F_POINTER() may not appear when FPTR= is scalar"_err_en_US);
+      } else if (arguments[3]) {
+        if (const auto *argExpr{arguments[3].value().UnwrapExpr()}) {
+          if (argExpr->Rank() > 1) {
+            context.messages().Say(arguments[3]->sourceLocation(),
+                "LOWER= argument to C_F_POINTER() must be a rank-one array."_err_en_US);
+          } else if (argExpr->Rank() == 1) {
+            if (auto constShape{GetConstantShape(context, *argExpr)}) {
+              if (constShape->At(ConstantSubscripts{1}).ToInt64() != fptrRank) {
+                context.messages().Say(arguments[3]->sourceLocation(),
+                    "LOWER= argument to C_F_POINTER() must have size equal to the rank of FPTR="_err_en_US);
+              }
+            }
+          }
+        }
+      }
     }
   }
   if (dummies.size() == 2) {
+    // Handle SHAPE
     DynamicType shapeType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
-    if (arguments[2]) {
+    if (arguments.size() >=3 && arguments[2]) {
       if (auto type{arguments[2]->GetType()}) {
         if (type->category() == TypeCategory::Integer) {
           shapeType = *type;
@@ -3185,6 +3205,22 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
     shape.intent = common::Intent::In;
     shape.attrs.set(characteristics::DummyDataObject::Attr::Optional);
     dummies.emplace_back("shape"s, std::move(shape));
+
+    // Handle LOWER
+    DynamicType lowerType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
+    if (arguments.size() >= 4 && arguments[3]) {
+      if (auto type{arguments[3]->GetType()}) {
+        if (type->category() == TypeCategory::Integer) {
+          lowerType = *type;
+        }
+      }
+    }
+    characteristics::DummyDataObject lower{
+        characteristics::TypeAndShape{lowerType, 1}};
+    lower.intent = common::Intent::In;
+    lower.attrs.set(characteristics::DummyDataObject::Attr::Optional);
+    dummies.emplace_back("lower"s, std::move(lower));
+
     return SpecificCall{
         SpecificIntrinsic{"__builtin_c_f_pointer"s,
             characteristics::Procedure{std::move(dummies), attrs}},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d77a656158a37..a99c4cb112c21 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -369,7 +369,8 @@ static constexpr IntrinsicHandler handlers[]{
      &I::genCFPointer,
      {{{"cptr", asValue},
        {"fptr", asInquired},
-       {"shape", asAddr, handleDynamicOptional}}},
+       {"shape", asAddr, handleDynamicOptional},
+       {"lower", asAddr, handleDynamicOptional}}},
      /*isElemental=*/false},
     {"c_f_procpointer",
      &I::genCFProcPointer,
@@ -3403,7 +3404,7 @@ IntrinsicLibrary::genCDevLoc(mlir::Type resultType,
 
 // C_F_POINTER
 void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
-  assert(args.size() == 3);
+  assert(args.size() == 4);
   // Handle CPTR argument
   // Get the value of the C address or the result of a reference to C_LOC.
   mlir::Value cPtr = fir::getBase(args[0]);
@@ -3418,9 +3419,12 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
     mlir::Value addr =
         builder.createConvert(loc, fPtr->getMemTy(), cPtrAddrVal);
     mlir::SmallVector<mlir::Value> extents;
+    mlir::SmallVector<mlir::Value> lbounds;
     if (box.hasRank()) {
       assert(isStaticallyPresent(args[2]) &&
              "FPTR argument must be an array if SHAPE argument exists");
+
+      // Handle and unpack SHAPE argument
       mlir::Value shape = fir::getBase(args[2]);
       int arrayRank = box.rank();
       mlir::Type shapeElementType =
@@ -3433,17 +3437,31 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
         mlir::Value load = builder.create<fir::LoadOp>(loc, var);
         extents.push_back(builder.createConvert(loc, idxType, load));
       }
+
+      // Handle and unpack LOWER argument if present
+      if (isStaticallyPresent(args[3])) {
+        mlir::Value lower = fir::getBase(args[3]);
+        mlir::Type lowerElementType =
+            fir::unwrapSequenceType(fir::unwrapPassByRefType(lower.getType()));
+        for (int i = 0; i < arrayRank; ++i) {
+          mlir::Value index = builder.createIntegerConstant(loc, idxType, i);
+          mlir::Value var = builder.create<fir::CoordinateOp>(
+              loc, builder.getRefType(lowerElementType), lower, index);
+          mlir::Value load = builder.create<fir::LoadOp>(loc, var);
+          lbounds.push_back(builder.createConvert(loc, idxType, load));
+        }
+      }
     }
     if (box.isCharacter()) {
       mlir::Value len = box.nonDeferredLenParams()[0];
       if (box.hasRank())
-        return fir::CharArrayBoxValue{addr, len, extents};
+        return fir::CharArrayBoxValue{addr, len, extents, lbounds};
       return fir::CharBoxValue{addr, len};
     }
     if (box.isDerivedWithLenParameters())
       TODO(loc, "get length parameters of derived type");
     if (box.hasRank())
-      return fir::ArrayBoxValue{addr, extents};
+      return fir::ArrayBoxValue{addr, extents, lbounds};
     return addr;
   };
 

>From 3e5f80c749997a49583c6fd40cd0d46fb3313b67 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Mon, 21 Jul 2025 19:01:17 +0200
Subject: [PATCH 2/3] Add LOWER= to semantics test

---
 flang/test/Semantics/c_f_pointer.f90 | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/flang/test/Semantics/c_f_pointer.f90 b/flang/test/Semantics/c_f_pointer.f90
index e2d00536cdacb..8a22175ffe19e 100644
--- a/flang/test/Semantics/c_f_pointer.f90
+++ b/flang/test/Semantics/c_f_pointer.f90
@@ -54,4 +54,12 @@ program test
   call c_f_pointer(scalarC, c2ptr)
   !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type or kind CHARACTER(KIND=4,LEN=1_8) [-Winteroperability]
   call c_f_pointer(scalarC, unicodePtr)
+
+  !ERROR: SHAPE= argument to C_F_POINTER() may not appear when FPTR= is scalar
+  !ERROR: LOWER= argument to C_F_POINTER() may not appear when FPTR= is scalar
+  call c_f_pointer(scalarC, scalarIntF, [1_8], [0_8])
+  !ERROR: LOWER= argument to C_F_POINTER() must be a rank-one array.
+  call c_f_pointer(scalarC, arrayIntF, shape=[1_8], lower=rankTwoArray)
+  !ERROR: SHAPE= argument to C_F_POINTER() must appear when FPTR= is an array
+  call c_f_pointer(scalarC, arrayIntF, lower=[0])
 end program

>From 117fe8e9e113e625d36f3fb6deca588e4ed23105 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Mon, 21 Jul 2025 20:30:49 +0200
Subject: [PATCH 3/3] Add LOWER= to lowering test

---
 flang/test/Lower/Intrinsics/c_f_pointer.f90 | 75 +++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/flang/test/Lower/Intrinsics/c_f_pointer.f90 b/flang/test/Lower/Intrinsics/c_f_pointer.f90
index 67817e39d5c2b..f41d5e471f540 100644
--- a/flang/test/Lower/Intrinsics/c_f_pointer.f90
+++ b/flang/test/Lower/Intrinsics/c_f_pointer.f90
@@ -140,3 +140,78 @@ subroutine dynamic_shape_size_2(cptr, fptr, shape, n)
 ! CHECK:         %[[VAL_16:.*]] = fir.shape %[[VAL_11]], %[[VAL_15]] : (index, index) -> !fir.shape<2>
   call c_f_pointer(cptr, fptr, shape)
 end subroutine
+
+! CHECK-LABEL: func.func @_QPdynamic_shape_lower(
+subroutine dynamic_shape_lower(cptr, fpr, shape, lower)
+  use iso_c_binding
+  type(c_ptr)  :: cptr
+  real, pointer :: fptr(:, :)
+  integer :: n
+  integer :: shape(:)
+  integer :: lower(:)
+! CHECK: %[[C_0:.*]] = arith.constant 0 : index
+! CHECK: %[[VAL_2:.*]] = fir.shape %[[C_0]], %[[C_0]] : (index, index) -> !fir.shape<2>
+! CHECK: %[[VAL_3:.*]] = fir.embox %[[VAL_1:.*]](%[[VAL_2]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK: fir.store %[[VAL_3]] to %[[VAL_0:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFdynamic_shape_lowerEn"}
+! CHECK: %[[VAL_5:.*]] = fir.coordinate_of %[[ARG_0:.*]], __address : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> !fir.ref<i64>
+! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<i64>
+! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (i64) -> !fir.ptr<!fir.array<?x?xf32>>
+! CHECK: %[[C_0:.*]]_0 = arith.constant 0 : index
+! CHECK: %[[VAL_8:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_0]]_0 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref<i32>
+! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+! CHECK: %[[C_1:.*]] = arith.constant 1 : index
+! CHECK: %[[VAL_11:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_1]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_11]] : !fir.ref<i32>
+! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i32) -> index
+! CHECK: %[[C_0:.*]]_1 = arith.constant 0 : index
+! CHECK: %[[VAL_14:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_0]]_1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
+! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> index
+! CHECK: %[[C_1:.*]]_2 = arith.constant 1 : index
+! CHECK: %[[VAL_17:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_1]]_2 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> index
+! CHECK: %[[VAL_20:.*]] = fir.shape_shift %[[VAL_16]], %[[VAL_10]], %[[VAL_19]], %[[VAL_13]] : (index, index, index, index) -> !fir.shapeshift<2>
+! CHECK: %[[VAL_21:.*]] = fir.embox %[[VAL_7]](%[[VAL_20]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK: fir.store %[[VAL_21:.*]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  call c_f_pointer(cptr, fptr, shape, lower)
+end subroutine dynamic_shape_lower
+
+! CHECK-LABEL: func.func @_QPdynamic_shape_lower_2(
+subroutine dynamic_shape_lower_2(cptr, fpr, shape, lower, n)
+  use iso_c_binding
+  type(c_ptr)  :: cptr
+  real, pointer :: fptr(:, :)
+  integer :: n
+  integer :: shape(n)
+  integer :: lower(n)
+!CHECK: %[[C_0:.*]] = arith.constant 0 : index
+!CHECK: %[[VAL_2:.*]] = fir.shape %[[C_0]], %[[C_0]] : (index, index) -> !fir.shape<2>
+!CHECK: %[[VAL_3:.*]] = fir.embox %[[ARG1:.*]](%[[VAL_2]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+!CHECK: fir.store %[[VAL_3]] to %[[VAL_0:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+!CHECK: %[[VAL_4:.*]] = fir.coordinate_of %[[ARG_0:.*]], __address : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> !fir.ref<i64>
+!CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<i64>
+!CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i64) -> !fir.ptr<!fir.array<?x?xf32>>
+!CHECK: %[[C_0:.*]]_0 = arith.constant 0 : index
+!CHECK: %[[VAL_7:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_0]]_0 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+!CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i32) -> index
+!CHECK: %[[C_1:.*]] = arith.constant 1 : index
+!CHECK: %[[VAL_10:.*]] = fir.coordinate_of %[[ARG_2]], %[[C_1]] : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+!CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+!CHECK: %[[C_0:.*]]_1 = arith.constant 0 : index
+!CHECK: %[[VAL_13:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_0]]_1 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<i32>
+!CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i32) -> index
+!CHECK: %[[C_1:.*]]_2 = arith.constant 1 : index
+!CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[ARG_3]], %[[C_1]]_2 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
+!CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i32) -> index
+!CHECK: %[[VAL_19:.*]] = fir.shape_shift %[[VAL_15]], %[[VAL_9]], %[[VAL_18]], %[[VAL_12]] : (index, index, index, index) -> !fir.shapeshift<2>
+!CHECK: %[[VAL_20:.*]] = fir.embox %[[VAL_6]](%[[VAL_19]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>
+!CHECK: fir.store %[[VAL_20]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  call c_f_pointer(cptr, fptr, shape, lower)
+end subroutine dynamic_shape_lower_2


From flang-commits at lists.llvm.org  Mon Jul 21 11:54:09 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Mon, 21 Jul 2025 11:54:09 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Implement LOWER= argument for
 C_F_POINTER (Fortran 2023) (PR #149870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149870@github.com>
Message-ID: <687e8cd1.170a0220.32fb4.ff77@mx.google.com>


https://github.com/mjklemm edited https://github.com/llvm/llvm-project/pull/149870

From flang-commits at lists.llvm.org  Mon Jul 21 11:54:24 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 11:54:24 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Implement LOWER= argument for
 C_F_POINTER (Fortran 2023) (PR #149870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149870@github.com>
Message-ID: <687e8ce0.170a0220.18523f.00f4@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Michael Klemm (mjklemm)

<details>
<summary>Changes</summary>

This PR resolves issue #<!-- -->147819 and adds support for the F2023 extension of the `LOWER=` argument for `C_F_POINTER`.

---
Full diff: https://github.com/llvm/llvm-project/pull/149870.diff


5 Files Affected:

- (modified) flang/docs/F202X.md (-1) 
- (modified) flang/lib/Evaluate/intrinsics.cpp (+40-4) 
- (modified) flang/lib/Optimizer/Builder/IntrinsicCall.cpp (+22-4) 
- (modified) flang/test/Lower/Intrinsics/c_f_pointer.f90 (+75) 
- (modified) flang/test/Semantics/c_f_pointer.f90 (+8) 


``````````diff
diff --git a/flang/docs/F202X.md b/flang/docs/F202X.md
index 67ea7fd97449a..7504561a64aa6 100644
--- a/flang/docs/F202X.md
+++ b/flang/docs/F202X.md
@@ -268,7 +268,6 @@ Addressing some issues and omissions in intrinsic modules:
  * LOGICAL8/16/32/64 and REAL16
  * IEEE module facilities upgraded to match latest IEEE FP standard
  * C_F_STRPOINTER, F_C_STRING for NUL-terminated strings
- * C_F_POINTER(LOWER=)
 
 #### Intrinsic Procedure Extensions
 
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9957010684d48..c48dea4b54b3a 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -3073,10 +3073,11 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
     ActualArguments &arguments, FoldingContext &context) const {
   characteristics::Procedure::Attrs attrs;
   attrs.set(characteristics::Procedure::Attr::Subroutine);
-  static const char *const keywords[]{"cptr", "fptr", "shape", nullptr};
+  static const char *const keywords[]{
+      "cptr", "fptr", "shape", "lower", nullptr};
   characteristics::DummyArguments dummies;
-  if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 1)) {
-    CHECK(arguments.size() == 3);
+  if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 2)) {
+    CHECK(arguments.size() == 4);
     if (const auto *expr{arguments[0].value().UnwrapExpr()}) {
       // General semantic checks will catch an actual argument that's not
       // scalar.
@@ -3169,11 +3170,30 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
           }
         }
       }
+      if (arguments[3] && fptrRank == 0) {
+        context.messages().Say(arguments[3]->sourceLocation(),
+            "LOWER= argument to C_F_POINTER() may not appear when FPTR= is scalar"_err_en_US);
+      } else if (arguments[3]) {
+        if (const auto *argExpr{arguments[3].value().UnwrapExpr()}) {
+          if (argExpr->Rank() > 1) {
+            context.messages().Say(arguments[3]->sourceLocation(),
+                "LOWER= argument to C_F_POINTER() must be a rank-one array."_err_en_US);
+          } else if (argExpr->Rank() == 1) {
+            if (auto constShape{GetConstantShape(context, *argExpr)}) {
+              if (constShape->At(ConstantSubscripts{1}).ToInt64() != fptrRank) {
+                context.messages().Say(arguments[3]->sourceLocation(),
+                    "LOWER= argument to C_F_POINTER() must have size equal to the rank of FPTR="_err_en_US);
+              }
+            }
+          }
+        }
+      }
     }
   }
   if (dummies.size() == 2) {
+    // Handle SHAPE
     DynamicType shapeType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
-    if (arguments[2]) {
+    if (arguments.size() >=3 && arguments[2]) {
       if (auto type{arguments[2]->GetType()}) {
         if (type->category() == TypeCategory::Integer) {
           shapeType = *type;
@@ -3185,6 +3205,22 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
     shape.intent = common::Intent::In;
     shape.attrs.set(characteristics::DummyDataObject::Attr::Optional);
     dummies.emplace_back("shape"s, std::move(shape));
+
+    // Handle LOWER
+    DynamicType lowerType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
+    if (arguments.size() >= 4 && arguments[3]) {
+      if (auto type{arguments[3]->GetType()}) {
+        if (type->category() == TypeCategory::Integer) {
+          lowerType = *type;
+        }
+      }
+    }
+    characteristics::DummyDataObject lower{
+        characteristics::TypeAndShape{lowerType, 1}};
+    lower.intent = common::Intent::In;
+    lower.attrs.set(characteristics::DummyDataObject::Attr::Optional);
+    dummies.emplace_back("lower"s, std::move(lower));
+
     return SpecificCall{
         SpecificIntrinsic{"__builtin_c_f_pointer"s,
             characteristics::Procedure{std::move(dummies), attrs}},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d77a656158a37..a99c4cb112c21 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -369,7 +369,8 @@ static constexpr IntrinsicHandler handlers[]{
      &I::genCFPointer,
      {{{"cptr", asValue},
        {"fptr", asInquired},
-       {"shape", asAddr, handleDynamicOptional}}},
+       {"shape", asAddr, handleDynamicOptional},
+       {"lower", asAddr, handleDynamicOptional}}},
      /*isElemental=*/false},
     {"c_f_procpointer",
      &I::genCFProcPointer,
@@ -3403,7 +3404,7 @@ IntrinsicLibrary::genCDevLoc(mlir::Type resultType,
 
 // C_F_POINTER
 void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
-  assert(args.size() == 3);
+  assert(args.size() == 4);
   // Handle CPTR argument
   // Get the value of the C address or the result of a reference to C_LOC.
   mlir::Value cPtr = fir::getBase(args[0]);
@@ -3418,9 +3419,12 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
     mlir::Value addr =
         builder.createConvert(loc, fPtr->getMemTy(), cPtrAddrVal);
     mlir::SmallVector<mlir::Value> extents;
+    mlir::SmallVector<mlir::Value> lbounds;
     if (box.hasRank()) {
       assert(isStaticallyPresent(args[2]) &&
              "FPTR argument must be an array if SHAPE argument exists");
+
+      // Handle and unpack SHAPE argument
       mlir::Value shape = fir::getBase(args[2]);
       int arrayRank = box.rank();
       mlir::Type shapeElementType =
@@ -3433,17 +3437,31 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
         mlir::Value load = builder.create<fir::LoadOp>(loc, var);
         extents.push_back(builder.createConvert(loc, idxType, load));
       }
+
+      // Handle and unpack LOWER argument if present
+      if (isStaticallyPresent(args[3])) {
+        mlir::Value lower = fir::getBase(args[3]);
+        mlir::Type lowerElementType =
+            fir::unwrapSequenceType(fir::unwrapPassByRefType(lower.getType()));
+        for (int i = 0; i < arrayRank; ++i) {
+          mlir::Value index = builder.createIntegerConstant(loc, idxType, i);
+          mlir::Value var = builder.create<fir::CoordinateOp>(
+              loc, builder.getRefType(lowerElementType), lower, index);
+          mlir::Value load = builder.create<fir::LoadOp>(loc, var);
+          lbounds.push_back(builder.createConvert(loc, idxType, load));
+        }
+      }
     }
     if (box.isCharacter()) {
       mlir::Value len = box.nonDeferredLenParams()[0];
       if (box.hasRank())
-        return fir::CharArrayBoxValue{addr, len, extents};
+        return fir::CharArrayBoxValue{addr, len, extents, lbounds};
       return fir::CharBoxValue{addr, len};
     }
     if (box.isDerivedWithLenParameters())
       TODO(loc, "get length parameters of derived type");
     if (box.hasRank())
-      return fir::ArrayBoxValue{addr, extents};
+      return fir::ArrayBoxValue{addr, extents, lbounds};
     return addr;
   };
 
diff --git a/flang/test/Lower/Intrinsics/c_f_pointer.f90 b/flang/test/Lower/Intrinsics/c_f_pointer.f90
index 67817e39d5c2b..f41d5e471f540 100644
--- a/flang/test/Lower/Intrinsics/c_f_pointer.f90
+++ b/flang/test/Lower/Intrinsics/c_f_pointer.f90
@@ -140,3 +140,78 @@ subroutine dynamic_shape_size_2(cptr, fptr, shape, n)
 ! CHECK:         %[[VAL_16:.*]] = fir.shape %[[VAL_11]], %[[VAL_15]] : (index, index) -> !fir.shape<2>
   call c_f_pointer(cptr, fptr, shape)
 end subroutine
+
+! CHECK-LABEL: func.func @_QPdynamic_shape_lower(
+subroutine dynamic_shape_lower(cptr, fpr, shape, lower)
+  use iso_c_binding
+  type(c_ptr)  :: cptr
+  real, pointer :: fptr(:, :)
+  integer :: n
+  integer :: shape(:)
+  integer :: lower(:)
+! CHECK: %[[C_0:.*]] = arith.constant 0 : index
+! CHECK: %[[VAL_2:.*]] = fir.shape %[[C_0]], %[[C_0]] : (index, index) -> !fir.shape<2>
+! CHECK: %[[VAL_3:.*]] = fir.embox %[[VAL_1:.*]](%[[VAL_2]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK: fir.store %[[VAL_3]] to %[[VAL_0:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFdynamic_shape_lowerEn"}
+! CHECK: %[[VAL_5:.*]] = fir.coordinate_of %[[ARG_0:.*]], __address : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> !fir.ref<i64>
+! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<i64>
+! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (i64) -> !fir.ptr<!fir.array<?x?xf32>>
+! CHECK: %[[C_0:.*]]_0 = arith.constant 0 : index
+! CHECK: %[[VAL_8:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_0]]_0 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref<i32>
+! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+! CHECK: %[[C_1:.*]] = arith.constant 1 : index
+! CHECK: %[[VAL_11:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_1]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_11]] : !fir.ref<i32>
+! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i32) -> index
+! CHECK: %[[C_0:.*]]_1 = arith.constant 0 : index
+! CHECK: %[[VAL_14:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_0]]_1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
+! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> index
+! CHECK: %[[C_1:.*]]_2 = arith.constant 1 : index
+! CHECK: %[[VAL_17:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_1]]_2 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> index
+! CHECK: %[[VAL_20:.*]] = fir.shape_shift %[[VAL_16]], %[[VAL_10]], %[[VAL_19]], %[[VAL_13]] : (index, index, index, index) -> !fir.shapeshift<2>
+! CHECK: %[[VAL_21:.*]] = fir.embox %[[VAL_7]](%[[VAL_20]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK: fir.store %[[VAL_21:.*]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  call c_f_pointer(cptr, fptr, shape, lower)
+end subroutine dynamic_shape_lower
+
+! CHECK-LABEL: func.func @_QPdynamic_shape_lower_2(
+subroutine dynamic_shape_lower_2(cptr, fpr, shape, lower, n)
+  use iso_c_binding
+  type(c_ptr)  :: cptr
+  real, pointer :: fptr(:, :)
+  integer :: n
+  integer :: shape(n)
+  integer :: lower(n)
+!CHECK: %[[C_0:.*]] = arith.constant 0 : index
+!CHECK: %[[VAL_2:.*]] = fir.shape %[[C_0]], %[[C_0]] : (index, index) -> !fir.shape<2>
+!CHECK: %[[VAL_3:.*]] = fir.embox %[[ARG1:.*]](%[[VAL_2]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+!CHECK: fir.store %[[VAL_3]] to %[[VAL_0:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+!CHECK: %[[VAL_4:.*]] = fir.coordinate_of %[[ARG_0:.*]], __address : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> !fir.ref<i64>
+!CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<i64>
+!CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i64) -> !fir.ptr<!fir.array<?x?xf32>>
+!CHECK: %[[C_0:.*]]_0 = arith.constant 0 : index
+!CHECK: %[[VAL_7:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_0]]_0 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+!CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i32) -> index
+!CHECK: %[[C_1:.*]] = arith.constant 1 : index
+!CHECK: %[[VAL_10:.*]] = fir.coordinate_of %[[ARG_2]], %[[C_1]] : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+!CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+!CHECK: %[[C_0:.*]]_1 = arith.constant 0 : index
+!CHECK: %[[VAL_13:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_0]]_1 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<i32>
+!CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i32) -> index
+!CHECK: %[[C_1:.*]]_2 = arith.constant 1 : index
+!CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[ARG_3]], %[[C_1]]_2 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
+!CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i32) -> index
+!CHECK: %[[VAL_19:.*]] = fir.shape_shift %[[VAL_15]], %[[VAL_9]], %[[VAL_18]], %[[VAL_12]] : (index, index, index, index) -> !fir.shapeshift<2>
+!CHECK: %[[VAL_20:.*]] = fir.embox %[[VAL_6]](%[[VAL_19]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>
+!CHECK: fir.store %[[VAL_20]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  call c_f_pointer(cptr, fptr, shape, lower)
+end subroutine dynamic_shape_lower_2
diff --git a/flang/test/Semantics/c_f_pointer.f90 b/flang/test/Semantics/c_f_pointer.f90
index e2d00536cdacb..8a22175ffe19e 100644
--- a/flang/test/Semantics/c_f_pointer.f90
+++ b/flang/test/Semantics/c_f_pointer.f90
@@ -54,4 +54,12 @@ program test
   call c_f_pointer(scalarC, c2ptr)
   !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type or kind CHARACTER(KIND=4,LEN=1_8) [-Winteroperability]
   call c_f_pointer(scalarC, unicodePtr)
+
+  !ERROR: SHAPE= argument to C_F_POINTER() may not appear when FPTR= is scalar
+  !ERROR: LOWER= argument to C_F_POINTER() may not appear when FPTR= is scalar
+  call c_f_pointer(scalarC, scalarIntF, [1_8], [0_8])
+  !ERROR: LOWER= argument to C_F_POINTER() must be a rank-one array.
+  call c_f_pointer(scalarC, arrayIntF, shape=[1_8], lower=rankTwoArray)
+  !ERROR: SHAPE= argument to C_F_POINTER() must appear when FPTR= is an array
+  call c_f_pointer(scalarC, arrayIntF, lower=[0])
 end program

``````````

</details>


https://github.com/llvm/llvm-project/pull/149870

From flang-commits at lists.llvm.org  Mon Jul 21 11:54:25 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 11:54:25 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Implement LOWER= argument for
 C_F_POINTER (Fortran 2023) (PR #149870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149870@github.com>
Message-ID: <687e8ce1.050a0220.f8142.ed5f@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Michael Klemm (mjklemm)

<details>
<summary>Changes</summary>

This PR resolves issue #<!-- -->147819 and adds support for the F2023 extension of the `LOWER=` argument for `C_F_POINTER`.

---
Full diff: https://github.com/llvm/llvm-project/pull/149870.diff


5 Files Affected:

- (modified) flang/docs/F202X.md (-1) 
- (modified) flang/lib/Evaluate/intrinsics.cpp (+40-4) 
- (modified) flang/lib/Optimizer/Builder/IntrinsicCall.cpp (+22-4) 
- (modified) flang/test/Lower/Intrinsics/c_f_pointer.f90 (+75) 
- (modified) flang/test/Semantics/c_f_pointer.f90 (+8) 


``````````diff
diff --git a/flang/docs/F202X.md b/flang/docs/F202X.md
index 67ea7fd97449a..7504561a64aa6 100644
--- a/flang/docs/F202X.md
+++ b/flang/docs/F202X.md
@@ -268,7 +268,6 @@ Addressing some issues and omissions in intrinsic modules:
  * LOGICAL8/16/32/64 and REAL16
  * IEEE module facilities upgraded to match latest IEEE FP standard
  * C_F_STRPOINTER, F_C_STRING for NUL-terminated strings
- * C_F_POINTER(LOWER=)
 
 #### Intrinsic Procedure Extensions
 
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9957010684d48..c48dea4b54b3a 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -3073,10 +3073,11 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
     ActualArguments &arguments, FoldingContext &context) const {
   characteristics::Procedure::Attrs attrs;
   attrs.set(characteristics::Procedure::Attr::Subroutine);
-  static const char *const keywords[]{"cptr", "fptr", "shape", nullptr};
+  static const char *const keywords[]{
+      "cptr", "fptr", "shape", "lower", nullptr};
   characteristics::DummyArguments dummies;
-  if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 1)) {
-    CHECK(arguments.size() == 3);
+  if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 2)) {
+    CHECK(arguments.size() == 4);
     if (const auto *expr{arguments[0].value().UnwrapExpr()}) {
       // General semantic checks will catch an actual argument that's not
       // scalar.
@@ -3169,11 +3170,30 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
           }
         }
       }
+      if (arguments[3] && fptrRank == 0) {
+        context.messages().Say(arguments[3]->sourceLocation(),
+            "LOWER= argument to C_F_POINTER() may not appear when FPTR= is scalar"_err_en_US);
+      } else if (arguments[3]) {
+        if (const auto *argExpr{arguments[3].value().UnwrapExpr()}) {
+          if (argExpr->Rank() > 1) {
+            context.messages().Say(arguments[3]->sourceLocation(),
+                "LOWER= argument to C_F_POINTER() must be a rank-one array."_err_en_US);
+          } else if (argExpr->Rank() == 1) {
+            if (auto constShape{GetConstantShape(context, *argExpr)}) {
+              if (constShape->At(ConstantSubscripts{1}).ToInt64() != fptrRank) {
+                context.messages().Say(arguments[3]->sourceLocation(),
+                    "LOWER= argument to C_F_POINTER() must have size equal to the rank of FPTR="_err_en_US);
+              }
+            }
+          }
+        }
+      }
     }
   }
   if (dummies.size() == 2) {
+    // Handle SHAPE
     DynamicType shapeType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
-    if (arguments[2]) {
+    if (arguments.size() >=3 && arguments[2]) {
       if (auto type{arguments[2]->GetType()}) {
         if (type->category() == TypeCategory::Integer) {
           shapeType = *type;
@@ -3185,6 +3205,22 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
     shape.intent = common::Intent::In;
     shape.attrs.set(characteristics::DummyDataObject::Attr::Optional);
     dummies.emplace_back("shape"s, std::move(shape));
+
+    // Handle LOWER
+    DynamicType lowerType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
+    if (arguments.size() >= 4 && arguments[3]) {
+      if (auto type{arguments[3]->GetType()}) {
+        if (type->category() == TypeCategory::Integer) {
+          lowerType = *type;
+        }
+      }
+    }
+    characteristics::DummyDataObject lower{
+        characteristics::TypeAndShape{lowerType, 1}};
+    lower.intent = common::Intent::In;
+    lower.attrs.set(characteristics::DummyDataObject::Attr::Optional);
+    dummies.emplace_back("lower"s, std::move(lower));
+
     return SpecificCall{
         SpecificIntrinsic{"__builtin_c_f_pointer"s,
             characteristics::Procedure{std::move(dummies), attrs}},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d77a656158a37..a99c4cb112c21 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -369,7 +369,8 @@ static constexpr IntrinsicHandler handlers[]{
      &I::genCFPointer,
      {{{"cptr", asValue},
        {"fptr", asInquired},
-       {"shape", asAddr, handleDynamicOptional}}},
+       {"shape", asAddr, handleDynamicOptional},
+       {"lower", asAddr, handleDynamicOptional}}},
      /*isElemental=*/false},
     {"c_f_procpointer",
      &I::genCFProcPointer,
@@ -3403,7 +3404,7 @@ IntrinsicLibrary::genCDevLoc(mlir::Type resultType,
 
 // C_F_POINTER
 void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
-  assert(args.size() == 3);
+  assert(args.size() == 4);
   // Handle CPTR argument
   // Get the value of the C address or the result of a reference to C_LOC.
   mlir::Value cPtr = fir::getBase(args[0]);
@@ -3418,9 +3419,12 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
     mlir::Value addr =
         builder.createConvert(loc, fPtr->getMemTy(), cPtrAddrVal);
     mlir::SmallVector<mlir::Value> extents;
+    mlir::SmallVector<mlir::Value> lbounds;
     if (box.hasRank()) {
       assert(isStaticallyPresent(args[2]) &&
              "FPTR argument must be an array if SHAPE argument exists");
+
+      // Handle and unpack SHAPE argument
       mlir::Value shape = fir::getBase(args[2]);
       int arrayRank = box.rank();
       mlir::Type shapeElementType =
@@ -3433,17 +3437,31 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
         mlir::Value load = builder.create<fir::LoadOp>(loc, var);
         extents.push_back(builder.createConvert(loc, idxType, load));
       }
+
+      // Handle and unpack LOWER argument if present
+      if (isStaticallyPresent(args[3])) {
+        mlir::Value lower = fir::getBase(args[3]);
+        mlir::Type lowerElementType =
+            fir::unwrapSequenceType(fir::unwrapPassByRefType(lower.getType()));
+        for (int i = 0; i < arrayRank; ++i) {
+          mlir::Value index = builder.createIntegerConstant(loc, idxType, i);
+          mlir::Value var = builder.create<fir::CoordinateOp>(
+              loc, builder.getRefType(lowerElementType), lower, index);
+          mlir::Value load = builder.create<fir::LoadOp>(loc, var);
+          lbounds.push_back(builder.createConvert(loc, idxType, load));
+        }
+      }
     }
     if (box.isCharacter()) {
       mlir::Value len = box.nonDeferredLenParams()[0];
       if (box.hasRank())
-        return fir::CharArrayBoxValue{addr, len, extents};
+        return fir::CharArrayBoxValue{addr, len, extents, lbounds};
       return fir::CharBoxValue{addr, len};
     }
     if (box.isDerivedWithLenParameters())
       TODO(loc, "get length parameters of derived type");
     if (box.hasRank())
-      return fir::ArrayBoxValue{addr, extents};
+      return fir::ArrayBoxValue{addr, extents, lbounds};
     return addr;
   };
 
diff --git a/flang/test/Lower/Intrinsics/c_f_pointer.f90 b/flang/test/Lower/Intrinsics/c_f_pointer.f90
index 67817e39d5c2b..f41d5e471f540 100644
--- a/flang/test/Lower/Intrinsics/c_f_pointer.f90
+++ b/flang/test/Lower/Intrinsics/c_f_pointer.f90
@@ -140,3 +140,78 @@ subroutine dynamic_shape_size_2(cptr, fptr, shape, n)
 ! CHECK:         %[[VAL_16:.*]] = fir.shape %[[VAL_11]], %[[VAL_15]] : (index, index) -> !fir.shape<2>
   call c_f_pointer(cptr, fptr, shape)
 end subroutine
+
+! CHECK-LABEL: func.func @_QPdynamic_shape_lower(
+subroutine dynamic_shape_lower(cptr, fpr, shape, lower)
+  use iso_c_binding
+  type(c_ptr)  :: cptr
+  real, pointer :: fptr(:, :)
+  integer :: n
+  integer :: shape(:)
+  integer :: lower(:)
+! CHECK: %[[C_0:.*]] = arith.constant 0 : index
+! CHECK: %[[VAL_2:.*]] = fir.shape %[[C_0]], %[[C_0]] : (index, index) -> !fir.shape<2>
+! CHECK: %[[VAL_3:.*]] = fir.embox %[[VAL_1:.*]](%[[VAL_2]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK: fir.store %[[VAL_3]] to %[[VAL_0:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFdynamic_shape_lowerEn"}
+! CHECK: %[[VAL_5:.*]] = fir.coordinate_of %[[ARG_0:.*]], __address : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> !fir.ref<i64>
+! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<i64>
+! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (i64) -> !fir.ptr<!fir.array<?x?xf32>>
+! CHECK: %[[C_0:.*]]_0 = arith.constant 0 : index
+! CHECK: %[[VAL_8:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_0]]_0 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref<i32>
+! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+! CHECK: %[[C_1:.*]] = arith.constant 1 : index
+! CHECK: %[[VAL_11:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_1]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_11]] : !fir.ref<i32>
+! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i32) -> index
+! CHECK: %[[C_0:.*]]_1 = arith.constant 0 : index
+! CHECK: %[[VAL_14:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_0]]_1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
+! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> index
+! CHECK: %[[C_1:.*]]_2 = arith.constant 1 : index
+! CHECK: %[[VAL_17:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_1]]_2 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> index
+! CHECK: %[[VAL_20:.*]] = fir.shape_shift %[[VAL_16]], %[[VAL_10]], %[[VAL_19]], %[[VAL_13]] : (index, index, index, index) -> !fir.shapeshift<2>
+! CHECK: %[[VAL_21:.*]] = fir.embox %[[VAL_7]](%[[VAL_20]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK: fir.store %[[VAL_21:.*]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  call c_f_pointer(cptr, fptr, shape, lower)
+end subroutine dynamic_shape_lower
+
+! CHECK-LABEL: func.func @_QPdynamic_shape_lower_2(
+subroutine dynamic_shape_lower_2(cptr, fpr, shape, lower, n)
+  use iso_c_binding
+  type(c_ptr)  :: cptr
+  real, pointer :: fptr(:, :)
+  integer :: n
+  integer :: shape(n)
+  integer :: lower(n)
+!CHECK: %[[C_0:.*]] = arith.constant 0 : index
+!CHECK: %[[VAL_2:.*]] = fir.shape %[[C_0]], %[[C_0]] : (index, index) -> !fir.shape<2>
+!CHECK: %[[VAL_3:.*]] = fir.embox %[[ARG1:.*]](%[[VAL_2]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+!CHECK: fir.store %[[VAL_3]] to %[[VAL_0:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+!CHECK: %[[VAL_4:.*]] = fir.coordinate_of %[[ARG_0:.*]], __address : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> !fir.ref<i64>
+!CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<i64>
+!CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i64) -> !fir.ptr<!fir.array<?x?xf32>>
+!CHECK: %[[C_0:.*]]_0 = arith.constant 0 : index
+!CHECK: %[[VAL_7:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_0]]_0 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+!CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i32) -> index
+!CHECK: %[[C_1:.*]] = arith.constant 1 : index
+!CHECK: %[[VAL_10:.*]] = fir.coordinate_of %[[ARG_2]], %[[C_1]] : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+!CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+!CHECK: %[[C_0:.*]]_1 = arith.constant 0 : index
+!CHECK: %[[VAL_13:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_0]]_1 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<i32>
+!CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i32) -> index
+!CHECK: %[[C_1:.*]]_2 = arith.constant 1 : index
+!CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[ARG_3]], %[[C_1]]_2 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
+!CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i32) -> index
+!CHECK: %[[VAL_19:.*]] = fir.shape_shift %[[VAL_15]], %[[VAL_9]], %[[VAL_18]], %[[VAL_12]] : (index, index, index, index) -> !fir.shapeshift<2>
+!CHECK: %[[VAL_20:.*]] = fir.embox %[[VAL_6]](%[[VAL_19]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>
+!CHECK: fir.store %[[VAL_20]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  call c_f_pointer(cptr, fptr, shape, lower)
+end subroutine dynamic_shape_lower_2
diff --git a/flang/test/Semantics/c_f_pointer.f90 b/flang/test/Semantics/c_f_pointer.f90
index e2d00536cdacb..8a22175ffe19e 100644
--- a/flang/test/Semantics/c_f_pointer.f90
+++ b/flang/test/Semantics/c_f_pointer.f90
@@ -54,4 +54,12 @@ program test
   call c_f_pointer(scalarC, c2ptr)
   !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type or kind CHARACTER(KIND=4,LEN=1_8) [-Winteroperability]
   call c_f_pointer(scalarC, unicodePtr)
+
+  !ERROR: SHAPE= argument to C_F_POINTER() may not appear when FPTR= is scalar
+  !ERROR: LOWER= argument to C_F_POINTER() may not appear when FPTR= is scalar
+  call c_f_pointer(scalarC, scalarIntF, [1_8], [0_8])
+  !ERROR: LOWER= argument to C_F_POINTER() must be a rank-one array.
+  call c_f_pointer(scalarC, arrayIntF, shape=[1_8], lower=rankTwoArray)
+  !ERROR: SHAPE= argument to C_F_POINTER() must appear when FPTR= is an array
+  call c_f_pointer(scalarC, arrayIntF, lower=[0])
 end program

``````````

</details>


https://github.com/llvm/llvm-project/pull/149870

From flang-commits at lists.llvm.org  Mon Jul 21 12:48:15 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 12:48:15 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687e997f.050a0220.845a8.fb17@mx.google.com>


================
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
----------------
Meinersbur wrote:

With this PR, the default intrinsic path is now determined by the driver, not by the frontend. To emulate driver behavior, the test needs to add the default intrinsics path manually.

Alternatively, this test could invoke the driver instead. It's in the `test/Driver` directory after all.

https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 12:58:44 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Mon, 21 Jul 2025 12:58:44 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687e9bf4.050a0220.3c93d6.048d@mx.google.com>


================
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
----------------
vzakhari wrote:

Oh, I see. Thanks!

https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 12:56:48 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 12:56:48 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687e9b80.170a0220.3552ac.000b@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 01/12] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 02/12] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 03/12] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 04/12] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 05/12] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 06/12] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 

>From 498d60cf39c237c654c0cfddbe444d63bc87bc6c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 09:57:18 +0200
Subject: [PATCH 07/12] CI test

---
 flang-rt/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 5ffa1bcc34a41..d987115c8e9e8 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,40 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
 enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
@@ -159,6 +193,7 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)

>From 1bb0144eb21c2b1c8e1e66029d5573d8f5619c4c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 14:24:44 +0200
Subject: [PATCH 08/12] Enforce REAL(16) support

---
 flang-rt/CMakeLists.txt      | 45 ++++-------------------
 llvm/runtimes/CMakeLists.txt |  8 ++++
 runtimes/CMakeLists.txt      | 71 ++++++++++++++++++------------------
 3 files changed, 51 insertions(+), 73 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d987115c8e9e8..bfbd0af0c31dc 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,41 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
@@ -79,6 +44,11 @@ include(CMakePushCheckState)
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
+# Fortran compiler not optional for building Flang-RT
+enable_language(Fortran)
+
+flang_module_fortran_enable()
+
 
 #################
 # Build Options #
@@ -193,7 +163,9 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+# Look for support of REAL(16), if not already defined via command line.
+# NOTE: Does not work with Flang and CMake < 3.24
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
@@ -206,7 +178,6 @@ check_fortran_source_compiles([[
 )
 cmake_pop_check_state()
 
-flang_module_fortran_enable()
 
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 2f2f6db16255e..2bf2b0ee8ed50 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -528,6 +528,14 @@ if(build_runtimes)
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
+  if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    # Ensure REAL(16) support in runtimes to be consistent with compiler
+    if (FLANG_RUNTIME_F128_MATH_LIB OR HAVE_LDBL_MANT_DIG_113)
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=TRUE")
+   else ()
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=FALSE")
+   endif ()
+  endif ()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 987d173a283be..51214a46f558e 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -94,6 +94,41 @@ include(CheckCXXCompilerFlag)
 include(ExtendPath)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang, if used.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
+
 # Determine whether we are in the runtimes/runtimes-bins directory of a
 # bootstrapping build.
 set(LLVM_TREE_AVAILABLE OFF)
@@ -334,42 +369,6 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
-      _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-
-    enable_language(Fortran)
-  endif ()
-endif ()
-
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 8ec6d03939c3a7e2f79bb3ea5911b7607028dd7c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 21:52:02 +0200
Subject: [PATCH 09/12] Also add dependency barrier for WIN32

---
 flang-rt/lib/runtime/CMakeLists.txt | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 1b8114c102205..20f5d84bb2b69 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -281,14 +281,18 @@ else()
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
 
-    add_flangrt_library(${name}.intrinsics OBJECT
+    add_flangrt_library(${name}.intrinsics.obj OBJECT
       ${intrinsics_sources}
     )
+    add_custom_target(${name}.intrinsics
+      COMMENT "Intrinsic module dependency barrier"
+    )
+    add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
 
     add_flangrt_library(${name} ${libtype}
-        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics.obj>
         ${ARGN}
-        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -311,8 +315,9 @@ else()
     endif ()
 
     get_target_property(compile_target ${name}.compile ALIASED_TARGET)
-    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
+    add_dependencies(${compile_target} ${name}.intrinsics)
     add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")

>From 580a0c56535a9cd1c65fd5a5e6309b73c36ae8b6 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:05:34 +0200
Subject: [PATCH 10/12] Dependency barrier test

---
 flang-rt/lib/runtime/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 20f5d84bb2b69..bdd4318832473 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,6 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
+      COMMAND echo "Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
@@ -295,6 +296,7 @@ else()
         LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
 
     if (msvc_lib)
       set_target_properties(${name}
@@ -311,14 +313,13 @@ else()
       set(is_public "")
     else ()
       set(is_public PUBLIC)
+      add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
       set(_has_public_intrinsics "YES" PARENT_SCOPE)
     endif ()
 
-    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
     flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
     add_dependencies(${compile_target} ${name}.intrinsics)
-    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")

>From 979691a5bba4888be8c7c82d1bed4e8cdc71fff9 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:06:40 +0200
Subject: [PATCH 11/12] Dependency barrier info

---
 flang-rt/lib/runtime/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index bdd4318832473..24b84b11f9513 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,7 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
-      COMMAND echo "Dependency barrier"
+      COMMAND echo "${name} Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)

>From b4adeab58e615b9059c0a6e5bbcb376d1fe21bb2 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Mon, 21 Jul 2025 21:56:26 +0200
Subject: [PATCH 12/12] Avoid unrelated changes

---
 clang/include/clang/Driver/Driver.h     | 1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 9343fed36b6ac..14e1e644e51aa 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,6 +403,7 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
+
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
   /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 688b5aacc4bcd..609ba27bc212b 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1376,10 +1376,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(
-        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
-                 "' was not generated and skipExternalRttiDefinition and "
-                 "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(loc,
+                        "runtime derived type info descriptor was not "
+                        "generated and skipExternalRttiDefinition and "
+                        "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition


From flang-commits at lists.llvm.org  Mon Jul 21 13:51:55 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 13:51:55 -0700 (PDT)
Subject: [flang-commits] [flang] 45a6c02 - [flang] Control alignment of
 constant folded reals (#149381)
Message-ID: <687ea86b.170a0220.f8aad.0231@mx.google.com>


Author: Eugene Epshteyn
Date: 2025-07-21T16:51:52-04:00
New Revision: 45a6c02c2123f1d4764a8ad981193b15851df744

URL: https://github.com/llvm/llvm-project/commit/45a6c02c2123f1d4764a8ad981193b15851df744
DIFF: https://github.com/llvm/llvm-project/commit/45a6c02c2123f1d4764a8ad981193b15851df744.diff

LOG: [flang] Control alignment of constant folded reals (#149381)

When REAL types are constant folded, the underneath implementation uses
arrays of integers. Ensure that these arrays are properly aligned.

This matters when building flang with clang. In some cases, the
resulting code for flang compiler ended up using SSE2 aligned load
instructions for REAL(16) constant folding on x86_64, and these
instructions require that the values are loaded from the aligned
addresses.

Added: 
    

Modified: 
    flang/include/flang/Evaluate/integer.h
    flang/include/flang/Evaluate/real.h

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index fccc2ad774a8f..5953fc81cb111 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -74,6 +74,7 @@ class Integer {
   static_assert(std::is_unsigned_v<BigPart>);
   static_assert(CHAR_BIT * sizeof(BigPart) >= 2 * partBits);
   static constexpr bool littleEndian{IS_LITTLE_ENDIAN};
+  static constexpr int alignment{ALIGNMENT};
 
 private:
   static constexpr int maxPartBits{CHAR_BIT * sizeof(Part)};

diff  --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 03294881850a1..76d25d9fe2670 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -490,7 +490,10 @@ template <typename WORD, int PREC> class Real {
       bool isNegative, int exponent, const Fraction &, Rounding, RoundingBits,
       bool multiply = false);
 
-  Word word_{}; // an Integer<>
+  // Require alignment, in case code generation on x86_64 decides that our
+  // Real object is suitable for SSE2 instructions and then gets surprised
+  // by unaligned address.
+  alignas(Word::alignment / 8) Word word_{}; // an Integer<>
 };
 
 extern template class Real<Integer<16>, 11>; // IEEE half format


        

From flang-commits at lists.llvm.org  Mon Jul 21 13:51:59 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Mon, 21 Jul 2025 13:51:59 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Control alignment of constant
	folded reals (PR #149381)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149381@github.com>
Message-ID: <687ea86f.170a0220.9674b.fdbd@mx.google.com>


https://github.com/eugeneepshteyn closed https://github.com/llvm/llvm-project/pull/149381

From flang-commits at lists.llvm.org  Mon Jul 21 14:55:20 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 21 Jul 2025 14:55:20 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][NFC] update `flang/Lower` create
 APIs (8/n) (#149687) (PR #149912)
Message-ID: <llvm.org/llvm/llvm-project/pull/149912@github.com>


https://github.com/makslevental created https://github.com/llvm/llvm-project/pull/149912

See https://github.com/llvm/llvm-project/pull/147168 for more info.

>From 64f5ec9d1951434a4f52f8ecd158019dcc3297fb Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 21 Jul 2025 17:54:53 -0400
Subject: [PATCH] [mlir][NFC] update `flang/Lower` create APIs (8/n) (#149687)

See https://github.com/llvm/llvm-project/pull/147168 for more info.
---
 flang/lib/Lower/Allocatable.cpp               |  52 +-
 flang/lib/Lower/Bridge.cpp                    |  34 +-
 flang/lib/Lower/ConvertArrayConstructor.cpp   |  22 +-
 flang/lib/Lower/ConvertCall.cpp               |  98 ++--
 flang/lib/Lower/ConvertConstant.cpp           |  56 +-
 flang/lib/Lower/ConvertExpr.cpp               | 504 +++++++++---------
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |  70 +--
 .../lib/Lower/ConvertProcedureDesignator.cpp  |  12 +-
 flang/lib/Lower/ConvertVariable.cpp           | 132 ++---
 flang/lib/Lower/CustomIntrinsicCall.cpp       |  18 +-
 flang/lib/Lower/HlfirIntrinsics.cpp           |  12 +-
 flang/lib/Lower/HostAssociations.cpp          |  38 +-
 flang/lib/Lower/IO.cpp                        | 192 +++----
 flang/lib/Lower/OpenACC.cpp                   | 246 ++++-----
 flang/lib/Lower/OpenMP/Atomic.cpp             |  16 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  28 +-
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp |  18 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  46 +-
 flang/lib/Lower/OpenMP/Utils.cpp              |  12 +-
 flang/lib/Lower/Runtime.cpp                   |  14 +-
 .../Lower/Support/PrivateReductionUtils.cpp   |  62 +--
 .../lib/Lower/Support/ReductionProcessor.cpp  |  64 +--
 flang/lib/Lower/Support/Utils.cpp             |  12 +-
 flang/lib/Lower/VectorSubscripts.cpp          |  26 +-
 24 files changed, 892 insertions(+), 892 deletions(-)

diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index 5536bfe8d63ca..f6119cf8a37d1 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -78,7 +78,7 @@ struct ErrorManager {
         statExpr && errMsgExpr
             ? builder.createBox(loc,
                                 converter.genExprAddr(loc, errMsgExpr, stmtCtx))
-            : builder.create<fir::AbsentOp>(
+            : fir::AbsentOp::create(builder,
                   loc,
                   fir::BoxType::get(mlir::NoneType::get(builder.getContext())));
     sourceFile = fir::factory::locationToFilename(builder, loc);
@@ -92,9 +92,9 @@ struct ErrorManager {
     if (statValue) {
       mlir::Value zero =
           builder.createIntegerConstant(loc, statValue.getType(), 0);
-      auto cmp = builder.create<mlir::arith::CmpIOp>(
+      auto cmp = mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::eq, statValue, zero);
-      auto ifOp = builder.create<fir::IfOp>(loc, cmp,
+      auto ifOp = fir::IfOp::create(builder, loc, cmp,
                                             /*withElseRegion=*/false);
       builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     }
@@ -106,7 +106,7 @@ struct ErrorManager {
       assert(stat && "missing stat value");
       mlir::Value castStat = builder.createConvert(
           loc, fir::dyn_cast_ptrEleTy(statAddr.getType()), stat);
-      builder.create<fir::StoreOp>(loc, castStat, statAddr);
+      fir::StoreOp::create(builder, loc, castStat, statAddr);
       statValue = stat;
     }
   }
@@ -141,7 +141,7 @@ static void genRuntimeSetBounds(fir::FirOpBuilder &builder, mlir::Location loc,
   const auto args = fir::runtime::createArguments(
       builder, loc, callee.getFunctionType(), box.getAddr(), dimIndex,
       lowerBound, upperBound);
-  builder.create<fir::CallOp>(loc, callee, args);
+  fir::CallOp::create(builder, loc, callee, args);
 }
 
 /// Generate runtime call to set the lengths of a character allocatable or
@@ -171,7 +171,7 @@ static void genRuntimeInitCharacter(fir::FirOpBuilder &builder,
   args.push_back(builder.createIntegerConstant(loc, inputTypes[4], corank));
   const auto convertedArgs = fir::runtime::createArguments(
       builder, loc, callee.getFunctionType(), args);
-  builder.create<fir::CallOp>(loc, callee, convertedArgs);
+  fir::CallOp::create(builder, loc, callee, convertedArgs);
 }
 
 /// Generate a sequence of runtime calls to allocate memory.
@@ -194,7 +194,7 @@ static mlir::Value genRuntimeAllocate(fir::FirOpBuilder &builder,
   args.push_back(errorManager.sourceLine);
   const auto convertedArgs = fir::runtime::createArguments(
       builder, loc, callee.getFunctionType(), args);
-  return builder.create<fir::CallOp>(loc, callee, convertedArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, callee, convertedArgs).getResult(0);
 }
 
 /// Generate a sequence of runtime calls to allocate memory and assign with the
@@ -214,7 +214,7 @@ static mlir::Value genRuntimeAllocateSource(fir::FirOpBuilder &builder,
       builder, loc, callee.getFunctionType(), box.getAddr(),
       fir::getBase(source), errorManager.hasStat, errorManager.errMsgAddr,
       errorManager.sourceFile, errorManager.sourceLine);
-  return builder.create<fir::CallOp>(loc, callee, args).getResult(0);
+  return fir::CallOp::create(builder, loc, callee, args).getResult(0);
 }
 
 /// Generate runtime call to apply mold to the descriptor.
@@ -233,7 +233,7 @@ static void genRuntimeAllocateApplyMold(fir::FirOpBuilder &builder,
       fir::factory::getMutableIRBox(builder, loc, box), fir::getBase(mold),
       builder.createIntegerConstant(
           loc, callee.getFunctionType().getInputs()[2], rank));
-  builder.create<fir::CallOp>(loc, callee, args);
+  fir::CallOp::create(builder, loc, callee, args);
 }
 
 /// Generate a runtime call to deallocate memory.
@@ -270,7 +270,7 @@ static mlir::Value genRuntimeDeallocate(fir::FirOpBuilder &builder,
         errorManager.hasStat, errorManager.errMsgAddr, errorManager.sourceFile,
         errorManager.sourceLine);
   }
-  return builder.create<fir::CallOp>(loc, callee, operands).getResult(0);
+  return fir::CallOp::create(builder, loc, callee, operands).getResult(0);
 }
 
 //===----------------------------------------------------------------------===//
@@ -433,9 +433,9 @@ class AllocateStmtHelper {
           loc, Fortran::semantics::GetExpr(std::get<1>(shapeSpec.t)), stmtCtx));
       ub = builder.createConvert(loc, idxTy, ub);
       if (lb) {
-        mlir::Value diff = builder.create<mlir::arith::SubIOp>(loc, ub, lb);
+        mlir::Value diff = mlir::arith::SubIOp::create(builder, loc, ub, lb);
         extents.emplace_back(
-            builder.create<mlir::arith::AddIOp>(loc, diff, one));
+            mlir::arith::AddIOp::create(builder, loc, diff, one));
       } else {
         extents.emplace_back(ub);
       }
@@ -461,7 +461,7 @@ class AllocateStmtHelper {
     mlir::Value falseValue = builder.createBool(loc, false);
     mlir::Value falseConv = builder.createConvert(
         loc, fir::unwrapRefType(pinned.getType()), falseValue);
-    builder.create<fir::StoreOp>(loc, falseConv, pinned);
+    fir::StoreOp::create(builder, loc, falseConv, pinned);
   }
 
   void genSimpleAllocation(const Allocation &alloc,
@@ -557,7 +557,7 @@ class AllocateStmtHelper {
       mlir::Value nullPointer = fir::factory::createUnallocatedBox(
           builder, loc, box.getBoxTy(), box.nonDeferredLenParams(),
           /*typeSourceBox=*/{}, allocatorIdx);
-      builder.create<fir::StoreOp>(loc, nullPointer, box.getAddr());
+      fir::StoreOp::create(builder, loc, nullPointer, box.getAddr());
     } else {
       assert(box.isAllocatable() && "must be an allocatable");
       // For allocatables, sync the MutableBoxValue and descriptor before the
@@ -597,13 +597,13 @@ class AllocateStmtHelper {
       assert(sourceBox && "source expression should be lowered to one box");
       for (int i = 0; i < sourceExpr->Rank(); ++i) {
         auto dimVal = builder.createIntegerConstant(loc, idxTy, i);
-        auto dimInfo = builder.create<fir::BoxDimsOp>(
+        auto dimInfo = fir::BoxDimsOp::create(builder,
             loc, idxTy, idxTy, idxTy, sourceBox->getAddr(), dimVal);
         mlir::Value lb =
             fir::factory::readLowerBound(builder, loc, sourceExv, i, one);
         mlir::Value extent = dimInfo.getResult(1);
-        mlir::Value ub = builder.create<mlir::arith::SubIOp>(
-            loc, builder.create<mlir::arith::AddIOp>(loc, extent, lb), one);
+        mlir::Value ub = mlir::arith::SubIOp::create(builder,
+            loc, mlir::arith::AddIOp::create(builder, loc, extent, lb), one);
         mlir::Value dimIndex = builder.createIntegerConstant(loc, i32Ty, i);
         genRuntimeSetBounds(builder, loc, box, dimIndex, lb, ub);
       }
@@ -668,7 +668,7 @@ class AllocateStmtHelper {
     const auto args = fir::runtime::createArguments(
         builder, loc, callee.getFunctionType(), box.getAddr(), typeDescAddr,
         rankValue, corankValue);
-    builder.create<fir::CallOp>(loc, callee, args);
+    fir::CallOp::create(builder, loc, callee, args);
   }
 
   /// Generate call to PointerNullifyIntrinsic or AllocatableInitIntrinsic to
@@ -697,7 +697,7 @@ class AllocateStmtHelper {
     const auto args = fir::runtime::createArguments(
         builder, loc, callee.getFunctionType(), box.getAddr(), categoryValue,
         kindValue, rankValue, corankValue);
-    builder.create<fir::CallOp>(loc, callee, args);
+    fir::CallOp::create(builder, loc, callee, args);
   }
 
   /// Generate call to the AllocatableInitDerived to set up the type descriptor
@@ -909,7 +909,7 @@ void Fortran::lower::genDeallocateIfAllocated(
       .genThen([&]() {
         if (mlir::Type eleType = box.getEleTy();
             mlir::isa<fir::RecordType>(eleType) && box.isPolymorphic()) {
-          mlir::Value declaredTypeDesc = builder.create<fir::TypeDescOp>(
+          mlir::Value declaredTypeDesc = fir::TypeDescOp::create(builder,
               loc, mlir::TypeAttr::get(eleType));
           genDeallocateBox(converter, box, loc, sym, declaredTypeDesc);
         } else {
@@ -1151,7 +1151,7 @@ mlir::Value Fortran::lower::getAssumedCharAllocatableOrPointerLen(
   // here).
   auto readLength = [&]() {
     fir::BoxValue boxLoad =
-        builder.create<fir::LoadOp>(loc, fir::getBase(box)).getResult();
+        fir::LoadOp::create(builder, loc, fir::getBase(box)).getResult();
     return fir::factory::readCharLen(builder, loc, boxLoad);
   };
   if (Fortran::semantics::IsOptional(sym)) {
@@ -1160,15 +1160,15 @@ mlir::Value Fortran::lower::getAssumedCharAllocatableOrPointerLen(
     // they are absents. According to 15.5.2.12 3 (9), it is illegal to
     // inquire the length of absent optional, even if non deferred, so
     // it's fine to use undefOp in this case.
-    auto isPresent = builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
+    auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
                                                       fir::getBase(box));
     mlir::Value len =
         builder.genIfOp(loc, {idxTy}, isPresent, true)
             .genThen(
-                [&]() { builder.create<fir::ResultOp>(loc, readLength()); })
+                [&]() { fir::ResultOp::create(builder, loc, readLength()); })
             .genElse([&]() {
-              auto undef = builder.create<fir::UndefOp>(loc, idxTy);
-              builder.create<fir::ResultOp>(loc, undef.getResult());
+              auto undef = fir::UndefOp::create(builder, loc, idxTy);
+              fir::ResultOp::create(builder, loc, undef.getResult());
             })
             .getResults()[0];
     return len;
@@ -1183,5 +1183,5 @@ mlir::Value Fortran::lower::getTypeDescAddr(
   mlir::Type typeDesc =
       Fortran::lower::translateDerivedTypeToFIRType(converter, typeSpec);
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  return builder.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(typeDesc));
+  return fir::TypeDescOp::create(builder, loc, mlir::TypeAttr::get(typeDesc));
 }
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 7ce397a11861b..557fdef272357 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -333,11 +333,11 @@ class TypeInfoConverter {
         if (details.numPrivatesNotOverridden() > 0)
           tbpName += "."s + std::to_string(details.numPrivatesNotOverridden());
         std::string bindingName = converter.mangleName(details.symbol());
-        builder.create<fir::DTEntryOp>(
+        fir::DTEntryOp::create(builder,
             info.loc, mlir::StringAttr::get(builder.getContext(), tbpName),
             mlir::SymbolRefAttr::get(builder.getContext(), bindingName));
       }
-      builder.create<fir::FirEndOp>(info.loc);
+      fir::FirEndOp::create(builder, info.loc);
     }
     // Gather info about components that is not reflected in fir.type and may be
     // needed later: component initial values and array component non default
@@ -360,11 +360,11 @@ class TypeInfoConverter {
           componentInfo = builder.createBlock(&dt.getComponentInfo());
         auto compName = mlir::StringAttr::get(builder.getContext(),
                                               toStringRef(component.name()));
-        builder.create<fir::DTComponentOp>(info.loc, compName, lbs, init_val);
+        fir::DTComponentOp::create(builder, info.loc, compName, lbs, init_val);
       }
     }
     if (componentInfo)
-      builder.create<fir::FirEndOp>(info.loc);
+      fir::FirEndOp::create(builder, info.loc);
     builder.restoreInsertionPoint(insertPointIfCreated);
   }
 
@@ -4829,17 +4829,17 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           base = convertOp.getValue();
         // Special case if the rhs is a constant.
         if (matchPattern(base.getDefiningOp(), mlir::m_Constant())) {
-          builder.create<cuf::DataTransferOp>(loc, base, lhsVal, shape,
+          cuf::DataTransferOp::create(builder, loc, base, lhsVal, shape,
                                               transferKindAttr);
         } else {
           auto associate = hlfir::genAssociateExpr(
               loc, builder, rhs, rhs.getType(), ".cuf_host_tmp");
-          builder.create<cuf::DataTransferOp>(loc, associate.getBase(), lhsVal,
+          cuf::DataTransferOp::create(builder, loc, associate.getBase(), lhsVal,
                                               shape, transferKindAttr);
-          builder.create<hlfir::EndAssociateOp>(loc, associate);
+          hlfir::EndAssociateOp::create(builder, loc, associate);
         }
       } else {
-        builder.create<cuf::DataTransferOp>(loc, rhsVal, lhsVal, shape,
+        cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
                                             transferKindAttr);
       }
       return;
@@ -4849,7 +4849,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     if (!lhsIsDevice && rhsIsDevice) {
       auto transferKindAttr = cuf::DataTransferKindAttr::get(
           builder.getContext(), cuf::DataTransferKind::DeviceHost);
-      builder.create<cuf::DataTransferOp>(loc, rhsVal, lhsVal, shape,
+      cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
                                           transferKindAttr);
       return;
     }
@@ -4859,7 +4859,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       assert(rhs.isVariable() && "CUDA Fortran assignment rhs is not legal");
       auto transferKindAttr = cuf::DataTransferKindAttr::get(
           builder.getContext(), cuf::DataTransferKind::DeviceDevice);
-      builder.create<cuf::DataTransferOp>(loc, rhsVal, lhsVal, shape,
+      cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
                                           transferKindAttr);
       return;
     }
@@ -4906,7 +4906,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           addSymbol(sym,
                     hlfir::translateToExtendedValue(loc, builder, temp).first,
                     /*forced=*/true);
-          builder.create<cuf::DataTransferOp>(
+          cuf::DataTransferOp::create(builder,
               loc, addr, temp, /*shape=*/mlir::Value{}, transferKindAttr);
           ++nbDeviceResidentObject;
         }
@@ -4996,13 +4996,13 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       if (isCUDATransfer && !hasCUDAImplicitTransfer)
         genCUDADataTransfer(builder, loc, assign, lhs, rhs);
       else
-        builder.create<hlfir::AssignOp>(loc, rhs, lhs,
+        hlfir::AssignOp::create(builder, loc, rhs, lhs,
                                         isWholeAllocatableAssignment,
                                         keepLhsLengthInAllocatableAssignment);
       if (hasCUDAImplicitTransfer && !isInDeviceContext) {
         localSymbols.popScope();
         for (mlir::Value temp : implicitTemps)
-          builder.create<fir::FreeMemOp>(loc, temp);
+          fir::FreeMemOp::create(builder, loc, temp);
       }
       return;
     }
@@ -5010,13 +5010,13 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // left-hand side requires using an hlfir.region_assign in HLFIR. The
     // right-hand side and left-hand side must be evaluated inside the
     // hlfir.region_assign regions.
-    auto regionAssignOp = builder.create<hlfir::RegionAssignOp>(loc);
+    auto regionAssignOp = hlfir::RegionAssignOp::create(builder, loc);
 
     // Lower RHS in its own region.
     builder.createBlock(&regionAssignOp.getRhsRegion());
     Fortran::lower::StatementContext rhsContext;
     hlfir::Entity rhs = evaluateRhs(rhsContext);
-    auto rhsYieldOp = builder.create<hlfir::YieldOp>(loc, rhs);
+    auto rhsYieldOp = hlfir::YieldOp::create(builder, loc, rhs);
     Fortran::lower::genCleanUpInRegionIfAny(
         loc, builder, rhsYieldOp.getCleanup(), rhsContext);
     // Lower LHS in its own region.
@@ -5025,7 +5025,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::Value lhsYield = nullptr;
     if (!lhsHasVectorSubscripts) {
       hlfir::Entity lhs = evaluateLhs(lhsContext);
-      auto lhsYieldOp = builder.create<hlfir::YieldOp>(loc, lhs);
+      auto lhsYieldOp = hlfir::YieldOp::create(builder, loc, lhs);
       Fortran::lower::genCleanUpInRegionIfAny(
           loc, builder, lhsYieldOp.getCleanup(), lhsContext);
       lhsYield = lhs;
@@ -5054,7 +5054,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       builder.createBlock(&regionAssignOp.getUserDefinedAssignment(),
                           mlir::Region::iterator{}, {rhsType, lhsType},
                           {loc, loc});
-      auto end = builder.create<fir::FirEndOp>(loc);
+      auto end = fir::FirEndOp::create(builder, loc);
       builder.setInsertionPoint(end);
       hlfir::Entity lhsBlockArg{regionAssignOp.getUserAssignmentLhs()};
       hlfir::Entity rhsBlockArg{regionAssignOp.getUserAssignmentRhs()};
diff --git a/flang/lib/Lower/ConvertArrayConstructor.cpp b/flang/lib/Lower/ConvertArrayConstructor.cpp
index 7e2142693eac5..a352133b70fa4 100644
--- a/flang/lib/Lower/ConvertArrayConstructor.cpp
+++ b/flang/lib/Lower/ConvertArrayConstructor.cpp
@@ -137,7 +137,7 @@ class InlinedTempStrategyImpl : public StrategyBase,
                              mlir::Value stride) {
     if constexpr (!hasLoops)
       fir::emitFatalError(loc, "array constructor lowering is inconsistent");
-    auto loop = builder.create<fir::DoLoopOp>(loc, lower, upper, stride,
+    auto loop = fir::DoLoopOp::create(builder, loc, lower, upper, stride,
                                               /*unordered=*/false,
                                               /*finalCount=*/false);
     builder.setInsertionPointToStart(loop.getBody());
@@ -213,15 +213,15 @@ class AsElementalStrategy : public StrategyBase {
     assert(!elementalOp && "expected only one implied-do");
     mlir::Value one =
         builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-    elementalOp = builder.create<hlfir::ElementalOp>(
+    elementalOp = hlfir::ElementalOp::create(builder,
         loc, exprType, shape,
         /*mold=*/nullptr, lengthParams, /*isUnordered=*/true);
     builder.setInsertionPointToStart(elementalOp.getBody());
     // implied-do-index = lower+((i-1)*stride)
-    mlir::Value diff = builder.create<mlir::arith::SubIOp>(
+    mlir::Value diff = mlir::arith::SubIOp::create(builder,
         loc, elementalOp.getIndices()[0], one);
-    mlir::Value mul = builder.create<mlir::arith::MulIOp>(loc, diff, stride);
-    mlir::Value add = builder.create<mlir::arith::AddIOp>(loc, lower, mul);
+    mlir::Value mul = mlir::arith::MulIOp::create(builder, loc, diff, stride);
+    mlir::Value add = mlir::arith::AddIOp::create(builder, loc, lower, mul);
     return add;
   }
 
@@ -260,7 +260,7 @@ class AsElementalStrategy : public StrategyBase {
     if (destroyOp)
       destroyOp->erase();
 
-    builder.create<hlfir::YieldElementOp>(loc, elementResult);
+    hlfir::YieldElementOp::create(builder, loc, elementResult);
   }
 
   // Override the default, because the context scope must be popped in
@@ -315,7 +315,7 @@ class RuntimeTempStrategy : public StrategyBase {
       mlir::Value tempStorage = builder.createHeapTemporary(
           loc, declaredType, tempName, extents, lengths);
       mlir::Value shape = builder.genShape(loc, extents);
-      declare = builder.create<hlfir::DeclareOp>(
+      declare = hlfir::DeclareOp::create(builder,
           loc, tempStorage, tempName, shape, lengths,
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       initialBoxValue =
@@ -347,7 +347,7 @@ class RuntimeTempStrategy : public StrategyBase {
                                           /*slice=*/mlir::Value{}, emboxLengths,
                                           /*tdesc=*/{});
     }
-    builder.create<fir::StoreOp>(loc, initialBoxValue, allocatableTemp);
+    fir::StoreOp::create(builder, loc, initialBoxValue, allocatableTemp);
     arrayConstructorVector = fir::runtime::genInitArrayConstructorVector(
         loc, builder, allocatableTemp,
         builder.createBool(loc, missingLengthParameters));
@@ -369,7 +369,7 @@ class RuntimeTempStrategy : public StrategyBase {
           loc, builder, value, arrayConstructorElementType);
       mlir::Value addr = fir::getBase(addrExv);
       if (mlir::isa<fir::BaseBoxType>(addr.getType()))
-        addr = builder.create<fir::BoxAddrOp>(loc, addr);
+        addr = fir::BoxAddrOp::create(builder, loc, addr);
       fir::runtime::genPushArrayConstructorSimpleScalar(
           loc, builder, arrayConstructorVector, addr);
       if (cleanUp)
@@ -389,7 +389,7 @@ class RuntimeTempStrategy : public StrategyBase {
   mlir::Value startImpliedDo(mlir::Location loc, fir::FirOpBuilder &builder,
                              mlir::Value lower, mlir::Value upper,
                              mlir::Value stride) {
-    auto loop = builder.create<fir::DoLoopOp>(loc, lower, upper, stride,
+    auto loop = fir::DoLoopOp::create(builder, loc, lower, upper, stride,
                                               /*unordered=*/false,
                                               /*finalCount=*/false);
     builder.setInsertionPointToStart(loop.getBody());
@@ -409,7 +409,7 @@ class RuntimeTempStrategy : public StrategyBase {
     else
       temp = hlfir::derefPointersAndAllocatables(
           loc, builder, hlfir::Entity{allocatableTemp});
-    auto hlfirExpr = builder.create<hlfir::AsExprOp>(loc, temp, mustFree);
+    auto hlfirExpr = hlfir::AsExprOp::create(builder, loc, temp, mustFree);
     return hlfir::Entity{hlfirExpr};
   }
 
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 071513303da25..4c7ec7a10bcbd 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -85,7 +85,7 @@ static mlir::Value genRecordCPtrValueArg(fir::FirOpBuilder &builder,
                                          mlir::Location loc, mlir::Value rec,
                                          mlir::Type ty) {
   mlir::Value cAddr = fir::factory::genCPtrOrCFunptrAddr(builder, loc, rec, ty);
-  mlir::Value cVal = builder.create<fir::LoadOp>(loc, cAddr);
+  mlir::Value cVal = fir::LoadOp::create(builder, loc, cAddr);
   return builder.createConvert(loc, cAddr.getType(), cVal);
 }
 
@@ -159,7 +159,7 @@ static mlir::Value readDim3Value(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Type refI32Ty = fir::ReferenceType::get(i32Ty);
   llvm::SmallVector<mlir::Value> lenParams;
 
-  mlir::Value designate = builder.create<hlfir::DesignateOp>(
+  mlir::Value designate = hlfir::DesignateOp::create(builder,
       loc, refI32Ty, dim3Addr, /*component=*/comp,
       /*componentShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
       /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
@@ -229,7 +229,7 @@ static mlir::Value remapActualToDummyDescriptor(
   if (fir::isPolymorphicType(dummyBoxType))
     mold = explicitArgument;
   mlir::Value remapped =
-      builder.create<fir::EmboxOp>(loc, dummyBoxType, baseAddr, shape,
+      fir::EmboxOp::create(builder, loc, dummyBoxType, baseAddr, shape,
                                    /*slice=*/mlir::Value{}, lengths, mold);
   if (mapSymbols)
     symMap.popScope();
@@ -273,12 +273,12 @@ static void remapActualToDummyDescriptors(
                   mlir::Value newBox = remapActualToDummyDescriptor(
                       loc, converter, symMap, argLambdaCapture, caller,
                       isBindcCall);
-                  builder.create<fir::ResultOp>(loc, newBox);
+                  fir::ResultOp::create(builder, loc, newBox);
                 })
                 .genElse([&]() {
                   mlir::Value absent =
-                      builder.create<fir::AbsentOp>(loc, dummyType);
-                  builder.create<fir::ResultOp>(loc, absent);
+                      fir::AbsentOp::create(builder, loc, dummyType);
+                  fir::ResultOp::create(builder, loc, absent);
                 })
                 .getResults()[0];
         caller.placeInput(arg, remapped);
@@ -381,7 +381,7 @@ Fortran::lower::genCallOpAndResult(
 
     if (isExprCall) {
       mlir::Type exprType = hlfir::getExprType(type);
-      evaluateInMemory = builder.create<hlfir::EvaluateInMemoryOp>(
+      evaluateInMemory = hlfir::EvaluateInMemoryOp::create(builder,
           loc, exprType, arrayResultShape, resultLengths);
       builder.setInsertionPointToStart(&evaluateInMemory.getBody().front());
       return toExtendedValue(loc, evaluateInMemory.getMemory(), extents,
@@ -454,7 +454,7 @@ Fortran::lower::genCallOpAndResult(
     if (!addHostAssociations &&
         mustCastFuncOpToCopeWithImplicitInterfaceMismatch(
             loc, converter, callSiteType, funcOpType))
-      funcPointer = builder.create<fir::AddrOfOp>(loc, funcOpType, symbolAttr);
+      funcPointer = fir::AddrOfOp::create(builder, loc, funcOpType, symbolAttr);
     else
       funcSymbolAttr = symbolAttr;
 
@@ -482,7 +482,7 @@ Fortran::lower::genCallOpAndResult(
   if (funcPointer) {
     operands.push_back(
         mlir::isa<fir::BoxProcType>(funcPointer.getType())
-            ? builder.create<fir::BoxAddrOp>(loc, funcType, funcPointer)
+            ? fir::BoxAddrOp::create(builder, loc, funcType, funcPointer)
             : builder.createConvert(loc, funcType, funcPointer));
   }
 
@@ -499,10 +499,10 @@ Fortran::lower::genCallOpAndResult(
       auto funcTy = mlir::FunctionType::get(context, {}, {});
       auto boxProcTy = builder.getBoxProcType(funcTy);
       if (mlir::Value host = argumentHostAssocs(converter, fst)) {
-        cast = builder.create<fir::EmboxProcOp>(
+        cast = fir::EmboxProcOp::create(builder,
             loc, boxProcTy, llvm::ArrayRef<mlir::Value>{fst, host});
       } else {
-        cast = builder.create<fir::EmboxProcOp>(loc, boxProcTy, fst);
+        cast = fir::EmboxProcOp::create(builder, loc, boxProcTy, fst);
       }
     } else {
       mlir::Type fromTy = fir::unwrapRefType(fst.getType());
@@ -613,7 +613,7 @@ Fortran::lower::genCallOpAndResult(
       stream = fir::getBase(converter.genExprAddr(
           caller.getCallDescription().chevrons()[3], stmtCtx));
 
-    builder.create<cuf::KernelLaunchOp>(
+    cuf::KernelLaunchOp::create(builder,
         loc, funcType.getResults(), funcSymbolAttr, grid_x, grid_y, grid_z,
         block_x, block_y, block_z, bytes, stream, operands,
         /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr);
@@ -640,7 +640,7 @@ Fortran::lower::genCallOpAndResult(
       // passed object because interface mismatch issues may have inserted a
       // cast to the operand with a different declared type, which would break
       // later type bound call resolution in the FIR to FIR pass.
-      dispatch = builder.create<fir::DispatchOp>(
+      dispatch = fir::DispatchOp::create(builder,
           loc, funcType.getResults(), builder.getStringAttr(procName),
           caller.getInputs()[*passArg], operands,
           builder.getI32IntegerAttr(*passArg), /*arg_attrs=*/nullptr,
@@ -656,8 +656,8 @@ Fortran::lower::genCallOpAndResult(
       mlir::Value passObject = fir::getBase(dataRefValue);
 
       if (fir::isa_ref_type(passObject.getType()))
-        passObject = builder.create<fir::LoadOp>(loc, passObject);
-      dispatch = builder.create<fir::DispatchOp>(
+        passObject = fir::LoadOp::create(builder, loc, passObject);
+      dispatch = fir::DispatchOp::create(builder,
           loc, funcType.getResults(), builder.getStringAttr(procName),
           passObject, operands, nullptr, /*arg_attrs=*/nullptr,
           /*res_attrs=*/nullptr, procAttrs);
@@ -667,7 +667,7 @@ Fortran::lower::genCallOpAndResult(
       callResult = dispatch.getResult(0);
   } else {
     // Standard procedure call with fir.call.
-    auto call = builder.create<fir::CallOp>(
+    auto call = fir::CallOp::create(builder,
         loc, funcType.getResults(), funcSymbolAttr, operands,
         /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, procAttrs);
 
@@ -691,7 +691,7 @@ Fortran::lower::genCallOpAndResult(
 
   if (caller.mustSaveResult()) {
     assert(allocatedResult.has_value());
-    builder.create<fir::SaveResultOp>(loc, callResult,
+    fir::SaveResultOp::create(builder, loc, callResult,
                                       fir::getBase(*allocatedResult),
                                       arrayResultShape, resultLengths);
   }
@@ -864,9 +864,9 @@ static hlfir::EntityWithAttributes genStmtFunctionRef(
   // The result must not be a variable.
   result = hlfir::loadTrivialScalar(loc, builder, result);
   if (result.isVariable())
-    result = hlfir::Entity{builder.create<hlfir::AsExprOp>(loc, result)};
+    result = hlfir::Entity{hlfir::AsExprOp::create(builder, loc, result)};
   for (auto associate : exprAssociations)
-    builder.create<hlfir::EndAssociateOp>(loc, associate);
+    hlfir::EndAssociateOp::create(builder, loc, associate);
   return hlfir::EntityWithAttributes{result};
 }
 
@@ -951,8 +951,8 @@ extendedValueToHlfirEntity(mlir::Location loc, fir::FirOpBuilder &builder,
     // rid of the memory indirection in a = char(b), so there is
     // little incentive to increase the compiler complexity.
     hlfir::Entity storage{builder.createTemporary(loc, charTy)};
-    builder.create<fir::StoreOp>(loc, firBase, storage);
-    auto asExpr = builder.create<hlfir::AsExprOp>(
+    fir::StoreOp::create(builder, loc, firBase, storage);
+    auto asExpr = hlfir::AsExprOp::create(builder,
         loc, storage, /*mustFree=*/builder.createBool(loc, false));
     return hlfir::EntityWithAttributes{asExpr.getResult()};
   }
@@ -965,7 +965,7 @@ namespace {
 struct CallCleanUp {
   struct CopyIn {
     void genCleanUp(mlir::Location loc, fir::FirOpBuilder &builder) {
-      builder.create<hlfir::CopyOutOp>(loc, tempBox, wasCopied, copyBackVar);
+      hlfir::CopyOutOp::create(builder, loc, tempBox, wasCopied, copyBackVar);
     }
     // address of the descriptor holding the temp if a temp was created.
     mlir::Value tempBox;
@@ -976,7 +976,7 @@ struct CallCleanUp {
   };
   struct ExprAssociate {
     void genCleanUp(mlir::Location loc, fir::FirOpBuilder &builder) {
-      builder.create<hlfir::EndAssociateOp>(loc, tempVar, mustFree);
+      hlfir::EndAssociateOp::create(builder, loc, tempVar, mustFree);
     }
     mlir::Value tempVar;
     mlir::Value mustFree;
@@ -1074,7 +1074,7 @@ struct ConditionallyPreparedDummy {
   /// Generate the "fir.result %preparedDummy" in the then branch of the
   /// wrapping fir.if.
   void genThenResult(mlir::Location loc, fir::FirOpBuilder &builder) const {
-    builder.create<fir::ResultOp>(loc, thenResultValues);
+    fir::ResultOp::create(builder, loc, thenResultValues);
   }
 
   /// Generate the "fir.result %absent" in the else branch of the
@@ -1089,7 +1089,7 @@ struct ConditionallyPreparedDummy {
       else
         elseResultValues.push_back(builder.genAbsentOp(loc, type));
     }
-    builder.create<fir::ResultOp>(loc, elseResultValues);
+    fir::ResultOp::create(builder, loc, elseResultValues);
   }
 
   /// Once the fir.if has been created, get the resulting %conditionallyPrepared
@@ -1134,7 +1134,7 @@ static hlfir::Entity fixProcedureDummyMismatch(mlir::Location loc,
   if (mlir::isa<fir::BoxProcType>(actual.getType()) &&
       fir::isCharacterProcedureTuple(dummyType)) {
     mlir::Value length =
-        builder.create<fir::UndefOp>(loc, builder.getCharacterLengthType());
+        fir::UndefOp::create(builder, loc, builder.getCharacterLengthType());
     mlir::Value tuple = fir::factory::createCharacterProcedureTuple(
         builder, loc, dummyType, actual, length);
     return hlfir::Entity{tuple};
@@ -1317,7 +1317,7 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     mlir::Type tempBoxType = baseBoxTy.getBoxTypeWithNewAttr(
         fir::BaseBoxType::Attribute::Allocatable);
     mlir::Value tempBox = builder.createTemporary(loc, tempBoxType);
-    auto copyIn = builder.create<hlfir::CopyInOp>(
+    auto copyIn = hlfir::CopyInOp::create(builder,
         loc, var, tempBox, /*var_is_present=*/mlir::Value{});
     // Register the copy-out after the call.
     preparedDummy.pushCopyInCleanUp(copyIn.getTempBox(), copyIn.getWasCopied(),
@@ -1330,14 +1330,14 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     fir::BaseBoxType boxType = fir::BoxType::get(
         hlfir::getFortranElementOrSequenceType(dummyTypeWithActualRank));
     if (actualIsAssumedRank)
-      return hlfir::Entity{builder.create<fir::ReboxAssumedRankOp>(
+      return hlfir::Entity{fir::ReboxAssumedRankOp::create(builder,
           loc, boxType, var, fir::LowerBoundModifierAttribute::SetToOnes)};
     // Use actual shape when creating descriptor with dummy type, the dummy
     // shape may be unknown in case of sequence association.
     mlir::Type actualTy =
         hlfir::getFortranElementOrSequenceType(actual.getType());
     boxType = boxType.getBoxTypeWithNewShape(actualTy);
-    return hlfir::Entity{builder.create<fir::ReboxOp>(loc, boxType, var,
+    return hlfir::Entity{fir::ReboxOp::create(builder, loc, boxType, var,
                                                       /*shape=*/mlir::Value{},
                                                       /*slice=*/mlir::Value{})};
   };
@@ -1361,7 +1361,7 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
         // generated writes in copy-out.
         isParameterObjectOrSubObject(entity)) {
       // Make a copy in a temporary.
-      auto copy = builder.create<hlfir::AsExprOp>(loc, entity);
+      auto copy = hlfir::AsExprOp::create(builder, loc, entity);
       mlir::Type storageType = entity.getType();
       mlir::NamedAttribute byRefAttr = fir::getAdaptToByRefAttr(builder);
       hlfir::AssociateOp associate = hlfir::genAssociateExpr(
@@ -1441,13 +1441,13 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
         auto lbModifier = needsZeroLowerBounds
                               ? fir::LowerBoundModifierAttribute::SetToZeroes
                               : fir::LowerBoundModifierAttribute::SetToOnes;
-        entity = hlfir::Entity{builder.create<fir::ReboxAssumedRankOp>(
+        entity = hlfir::Entity{fir::ReboxAssumedRankOp::create(builder,
             loc, dummyTypeWithActualRank, entity, lbModifier)};
       } else {
         mlir::Value shift{};
         if (needsZeroLowerBounds)
           shift = getZeroLowerBounds(loc, builder, entity);
-        entity = hlfir::Entity{builder.create<fir::ReboxOp>(
+        entity = hlfir::Entity{fir::ReboxOp::create(builder,
             loc, dummyTypeWithActualRank, entity, /*shape=*/shift,
             /*slice=*/mlir::Value{})};
       }
@@ -1501,7 +1501,7 @@ static PreparedDummyArgument prepareUserCallActualArgument(
   // for this unusual if/then/else generation is that the number
   // and types of the if results will depend on how the argument
   // is prepared, and forecasting that here would be brittle.
-  auto badIfOp = builder.create<fir::IfOp>(loc, dummyType, isPresent,
+  auto badIfOp = fir::IfOp::create(builder, loc, dummyType, isPresent,
                                            /*withElseRegion=*/false);
   mlir::Block *preparationBlock = &badIfOp.getThenRegion().front();
   builder.setInsertionPointToStart(preparationBlock);
@@ -1520,7 +1520,7 @@ static PreparedDummyArgument prepareUserCallActualArgument(
   // badIfOp cannot be modified and used here).
   llvm::SmallVector<mlir::Type> ifOpResultTypes;
   ConditionallyPreparedDummy conditionalDummy(unconditionalDummy);
-  auto ifOp = builder.create<fir::IfOp>(loc, conditionalDummy.getIfResulTypes(),
+  auto ifOp = fir::IfOp::create(builder, loc, conditionalDummy.getIfResulTypes(),
                                         isPresent,
                                         /*withElseRegion=*/true);
   // Move "preparationBlock" into the "then" of the new
@@ -1559,7 +1559,7 @@ static PreparedDummyArgument prepareProcedurePointerActualArgument(
     auto tempBoxProc{builder.createTemporary(loc, boxTy)};
     hlfir::Entity nullBoxProc(
         fir::factory::createNullBoxProc(builder, loc, boxTy));
-    builder.create<fir::StoreOp>(loc, nullBoxProc, tempBoxProc);
+    fir::StoreOp::create(builder, loc, nullBoxProc, tempBoxProc);
     return PreparedDummyArgument{tempBoxProc, /*cleanups=*/{}};
   }
   hlfir::Entity actual = preparedActual.getActual(loc, builder);
@@ -1568,7 +1568,7 @@ static PreparedDummyArgument prepareProcedurePointerActualArgument(
   assert(actual.isProcedure());
   // Procedure actual to procedure pointer dummy.
   auto tempBoxProc{builder.createTemporary(loc, actual.getType())};
-  builder.create<fir::StoreOp>(loc, actual, tempBoxProc);
+  fir::StoreOp::create(builder, loc, actual, tempBoxProc);
   return PreparedDummyArgument{tempBoxProc, /*cleanups=*/{}};
 }
 
@@ -1607,7 +1607,7 @@ void prepareUserCallArguments(
                                                    "adapt.cptrbyval");
           value = hlfir::Entity{genRecordCPtrValueArg(
               builder, loc, associate.getFirBase(), eleTy)};
-          builder.create<hlfir::EndAssociateOp>(loc, associate);
+          hlfir::EndAssociateOp::create(builder, loc, associate);
         } else {
           value =
               hlfir::Entity{genRecordCPtrValueArg(builder, loc, value, eleTy)};
@@ -1626,7 +1626,7 @@ void prepareUserCallArguments(
             loadedValue = builder.createConvert(
                 loc, fir::ReferenceType::get(argTy), loadedValue);
         if (fir::isa_ref_type(loadedValue.getType()))
-          loadedValue = builder.create<fir::LoadOp>(loc, loadedValue);
+          loadedValue = fir::LoadOp::create(builder, loc, loadedValue);
         caller.placeInput(arg, loadedValue);
         if (cleanup)
           (*cleanup)();
@@ -1811,7 +1811,7 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals,
       // In such case, the expression should not be freed after its use since
       // the result is stack allocated or deallocation (for allocatable results)
       // was already inserted in genCallOpAndResult.
-      auto asExpr = builder.create<hlfir::AsExprOp>(
+      auto asExpr = hlfir::AsExprOp::create(builder,
           loc, resultEntity, /*mustFree=*/builder.createBool(loc, false));
       return hlfir::EntityWithAttributes{asExpr.getResult()};
     }
@@ -1860,12 +1860,12 @@ static ExvAndCleanup genOptionalValue(fir::FirOpBuilder &builder,
                        "must be a numerical or logical scalar");
                 mlir::Value val =
                     hlfir::loadTrivialScalar(loc, builder, entity);
-                builder.create<fir::ResultOp>(loc, val);
+                fir::ResultOp::create(builder, loc, val);
               })
               .genElse([&]() {
                 mlir::Value zero =
                     fir::factory::createZeroValue(builder, loc, eleType);
-                builder.create<fir::ResultOp>(loc, zero);
+                fir::ResultOp::create(builder, loc, zero);
               })
               .getResults()[0],
           std::nullopt};
@@ -1912,8 +1912,8 @@ static ExvAndCleanup genOptionalBox(fir::FirOpBuilder &builder,
   // ensures it won't be.
   mlir::Value box = builder.createBox(loc, newExv);
   mlir::Type boxType = box.getType();
-  auto absent = builder.create<fir::AbsentOp>(loc, boxType);
-  auto boxOrAbsent = builder.create<mlir::arith::SelectOp>(
+  auto absent = fir::AbsentOp::create(builder, loc, boxType);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder,
       loc, boxType, isPresent, box, absent);
   return {fir::BoxValue(boxOrAbsent), cleanup};
 }
@@ -2142,9 +2142,9 @@ genIntrinsicRefCore(Fortran::lower::PreparedActualArguments &loweredActuals,
     // ownership of this address cannot be taken here since it may not be a
     // temp.
     if (intrinsicName == "merge")
-      asExpr = builder.create<hlfir::AsExprOp>(loc, resultEntity);
+      asExpr = hlfir::AsExprOp::create(builder, loc, resultEntity);
     else
-      asExpr = builder.create<hlfir::AsExprOp>(
+      asExpr = hlfir::AsExprOp::create(builder,
           loc, resultEntity, builder.createBool(loc, mustBeFreed));
     resultEntity = hlfir::EntityWithAttributes{asExpr.getResult()};
   }
@@ -2525,7 +2525,7 @@ genIsPresentIfArgMaybeAbsent(mlir::Location loc, hlfir::Entity actual,
   // May fall into the category above if the allocatable is not optional.
 
   // Passing an optional to an optional.
-  return builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), actual)
+  return fir::IsPresentOp::create(builder, loc, builder.getI1Type(), actual)
       .getResult();
 }
 
@@ -2814,7 +2814,7 @@ genProcedureRef(CallContext &callContext) {
           // need to cover the case of passing an ALLOCATABLE/POINTER to an
           // OPTIONAL.
           isPresent =
-              builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), actual)
+              fir::IsPresentOp::create(builder, loc, builder.getI1Type(), actual)
                   .getResult();
         }
         loweredActuals.push_back(Fortran::lower::PreparedActualArgument{
@@ -2931,7 +2931,7 @@ std::optional<hlfir::EntityWithAttributes> Fortran::lower::convertCallToHLFIR(
     // this can be enforced whenscheduling forall/where expression evaluations.
     Fortran::lower::StatementContext localStmtCtx;
     mlir::Type bogusType = builder.getIndexType();
-    auto exactlyOnce = builder.create<hlfir::ExactlyOnceOp>(loc, bogusType);
+    auto exactlyOnce = hlfir::ExactlyOnceOp::create(builder, loc, bogusType);
     mlir::Block *block = builder.createBlock(&exactlyOnce.getBody());
     builder.setInsertionPointToStart(block);
     CallContext callContext(procRef, resultType, loc, converter, symMap,
@@ -2939,7 +2939,7 @@ std::optional<hlfir::EntityWithAttributes> Fortran::lower::convertCallToHLFIR(
     std::optional<hlfir::EntityWithAttributes> res =
         genProcedureRef(callContext);
     assert(res.has_value() && "must be a function");
-    auto yield = builder.create<hlfir::YieldOp>(loc, *res);
+    auto yield = hlfir::YieldOp::create(builder, loc, *res);
     Fortran::lower::genCleanUpInRegionIfAny(loc, builder, yield.getCleanup(),
                                             localStmtCtx);
     builder.setInsertionPointAfter(exactlyOnce);
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index b8ab5d09e3e08..cd4be37076e48 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -237,7 +237,7 @@ static mlir::Value genScalarLit(
                                     ? value.UnsignedDecimal()
                                     : value.SignedDecimal(),
                                 10);
-      return builder.create<mlir::arith::ConstantOp>(
+      return mlir::arith::ConstantOp::create(builder,
           loc, ty, mlir::IntegerAttr::get(ty, bigInt));
     }
     return builder.createIntegerConstant(loc, ty, value.ToInt64());
@@ -302,7 +302,7 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     auto sizeTag = mlir::StringAttr::get(context, fir::StringLitOp::size());
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
-    return builder.create<fir::StringLitOp>(
+    return fir::StringLitOp::create(builder,
         loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
   }
 }
@@ -340,10 +340,10 @@ genScalarLit(fir::FirOpBuilder &builder, mlir::Location loc,
         [&](fir::FirOpBuilder &builder) {
           fir::StringLitOp str =
               createStringLitOp<KIND>(builder, loc, value, len);
-          builder.create<fir::HasValueOp>(loc, str);
+          fir::HasValueOp::create(builder, loc, str);
         },
         builder.createLinkOnceLinkage());
-  return builder.create<fir::AddrOfOp>(loc, global.resultType(),
+  return fir::AddrOfOp::create(builder, loc, global.resultType(),
                                        global.getSymbol());
 }
 
@@ -364,7 +364,7 @@ static mlir::Value genStructureComponentInit(
   auto fieldTy = fir::FieldType::get(recTy.getContext());
   assert(componentTy && "failed to retrieve component");
   // FIXME: type parameters must come from the derived-type-spec
-  auto field = builder.create<fir::FieldIndexOp>(
+  auto field = fir::FieldIndexOp::create(builder,
       loc, fieldTy, name, recTy,
       /*typeParams=*/mlir::ValueRange{} /*TODO*/);
 
@@ -378,7 +378,7 @@ static mlir::Value genStructureComponentInit(
           fir::factory::createUnallocatedBox(builder, loc, componentTy, {})};
       componentValue = builder.createConvert(loc, componentTy, componentValue);
 
-      return builder.create<fir::InsertValueOp>(
+      return fir::InsertValueOp::create(builder,
           loc, recTy, res, componentValue,
           builder.getArrayAttr(field.getAttributes()));
     }
@@ -400,7 +400,7 @@ static mlir::Value genStructureComponentInit(
     } else
       initialTarget = Fortran::lower::genInitialDataTarget(converter, loc,
                                                            componentTy, expr);
-    res = builder.create<fir::InsertValueOp>(
+    res = fir::InsertValueOp::create(builder,
         loc, recTy, res, initialTarget,
         builder.getArrayAttr(field.getAttributes()));
     return res;
@@ -426,7 +426,7 @@ static mlir::Value genStructureComponentInit(
     mlir::Value addr = fir::getBase(
         Fortran::lower::genExtAddrInInitializer(converter, loc, expr));
     if (mlir::isa<fir::BoxProcType>(addr.getType()))
-      addr = builder.create<fir::BoxAddrOp>(loc, addr);
+      addr = fir::BoxAddrOp::create(builder, loc, addr);
     assert((fir::isa_ref_type(addr.getType()) ||
             mlir::isa<mlir::FunctionType>(addr.getType())) &&
            "expect reference type for address field");
@@ -435,15 +435,15 @@ static mlir::Value genStructureComponentInit(
     auto cPtrRecTy = mlir::cast<fir::RecordType>(componentTy);
     llvm::StringRef addrFieldName = Fortran::lower::builtin::cptrFieldName;
     mlir::Type addrFieldTy = cPtrRecTy.getType(addrFieldName);
-    auto addrField = builder.create<fir::FieldIndexOp>(
+    auto addrField = fir::FieldIndexOp::create(builder,
         loc, fieldTy, addrFieldName, componentTy,
         /*typeParams=*/mlir::ValueRange{});
     mlir::Value castAddr = builder.createConvert(loc, addrFieldTy, addr);
-    auto undef = builder.create<fir::UndefOp>(loc, componentTy);
-    addr = builder.create<fir::InsertValueOp>(
+    auto undef = fir::UndefOp::create(builder, loc, componentTy);
+    addr = fir::InsertValueOp::create(builder,
         loc, componentTy, undef, castAddr,
         builder.getArrayAttr(addrField.getAttributes()));
-    res = builder.create<fir::InsertValueOp>(
+    res = fir::InsertValueOp::create(builder,
         loc, recTy, res, addr, builder.getArrayAttr(field.getAttributes()));
     return res;
   }
@@ -451,7 +451,7 @@ static mlir::Value genStructureComponentInit(
   mlir::Value val = fir::getBase(genConstantValue(converter, loc, expr));
   assert(!fir::isa_ref_type(val.getType()) && "expecting a constant value");
   mlir::Value castVal = builder.createConvert(loc, componentTy, val);
-  res = builder.create<fir::InsertValueOp>(
+  res = fir::InsertValueOp::create(builder,
       loc, recTy, res, castVal, builder.getArrayAttr(field.getAttributes()));
   return res;
 }
@@ -465,7 +465,7 @@ static mlir::Value genInlinedStructureCtorLitImpl(
   auto recTy = mlir::cast<fir::RecordType>(type);
 
   if (!converter.getLoweringOptions().getLowerToHighLevelFIR()) {
-    mlir::Value res = builder.create<fir::UndefOp>(loc, recTy);
+    mlir::Value res = fir::UndefOp::create(builder, loc, recTy);
     for (const auto &[sym, expr] : ctor.values()) {
       // Parent components need more work because they do not appear in the
       // fir.rec type.
@@ -495,12 +495,12 @@ static mlir::Value genInlinedStructureCtorLitImpl(
         break;
     }
     for (mlir::Type parentType : llvm::reverse(parentTypes)) {
-      auto undef = builder.create<fir::UndefOp>(loc, parentType);
+      auto undef = fir::UndefOp::create(builder, loc, parentType);
       fir::RecordType parentRecTy = mlir::cast<fir::RecordType>(parentType);
-      auto field = builder.create<fir::FieldIndexOp>(
+      auto field = fir::FieldIndexOp::create(builder,
           loc, fieldTy, parentRecTy.getTypeList()[0].first, parentType,
           /*typeParams=*/mlir::ValueRange{} /*TODO*/);
-      res = builder.create<fir::InsertValueOp>(
+      res = fir::InsertValueOp::create(builder,
           loc, parentRecTy, undef, res,
           builder.getArrayAttr(field.getAttributes()));
     }
@@ -514,7 +514,7 @@ static mlir::Value genInlinedStructureCtorLitImpl(
     if (!res) {
       mlir::Type parentType = converter.genType(*componentParentType);
       curentType = componentParentType;
-      res = builder.create<fir::UndefOp>(loc, parentType);
+      res = fir::UndefOp::create(builder, loc, parentType);
     } else if (*componentParentType != *curentType) {
       mlir::Type parentType = converter.genType(*componentParentType);
       insertParentValueIntoExtension(parentType);
@@ -524,7 +524,7 @@ static mlir::Value genInlinedStructureCtorLitImpl(
   }
 
   if (!res) // structure constructor for empty type.
-    return builder.create<fir::UndefOp>(loc, recTy);
+    return fir::UndefOp::create(builder, loc, recTy);
 
   // The last component may belong to a parent type.
   if (res.getType() != recTy)
@@ -550,11 +550,11 @@ static mlir::Value genScalarLit(
         [&](fir::FirOpBuilder &builder) {
           mlir::Value result =
               genInlinedStructureCtorLitImpl(converter, loc, value, eleTy);
-          builder.create<fir::HasValueOp>(loc, result);
+          fir::HasValueOp::create(builder, loc, result);
         },
         builder.createInternalLinkage());
   }
-  return builder.create<fir::AddrOfOp>(loc, global.resultType(),
+  return fir::AddrOfOp::create(builder, loc, global.resultType(),
                                        global.getSymbol());
 }
 
@@ -576,7 +576,7 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
           builder.getIntegerAttr(idxTy, subscripts[i] - con.lbounds()[i]));
     return idx;
   };
-  mlir::Value array = builder.create<fir::UndefOp>(loc, arrayTy);
+  mlir::Value array = fir::UndefOp::create(builder, loc, arrayTy);
   if (Fortran::evaluate::GetSize(con.shape()) == 0)
     return array;
   if constexpr (T::category == Fortran::common::TypeCategory::Character) {
@@ -584,7 +584,7 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       mlir::Value elementVal =
           genScalarLit<T::kind>(builder, loc, con.At(subscripts), con.LEN(),
                                 /*outlineInReadOnlyMemory=*/false);
-      array = builder.create<fir::InsertValueOp>(
+      array = fir::InsertValueOp::create(builder,
           loc, arrayTy, array, elementVal, builder.getArrayAttr(createIdx()));
     } while (con.IncrementSubscripts(subscripts));
   } else if constexpr (T::category == Fortran::common::TypeCategory::Derived) {
@@ -594,7 +594,7 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       mlir::Value elementVal =
           genScalarLit(converter, loc, con.At(subscripts), eleTy,
                        /*outlineInReadOnlyMemory=*/false);
-      array = builder.create<fir::InsertValueOp>(
+      array = fir::InsertValueOp::create(builder,
           loc, arrayTy, array, elementVal, builder.getArrayAttr(createIdx()));
     } while (con.IncrementSubscripts(subscripts));
   } else {
@@ -611,7 +611,7 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       bool nextIsSame = con.IncrementSubscripts(nextSubscripts) &&
                         con.At(subscripts) == con.At(nextSubscripts);
       if (!rangeSize && !nextIsSame) { // single (non-range) value
-        array = builder.create<fir::InsertValueOp>(
+        array = fir::InsertValueOp::create(builder,
             loc, arrayTy, array, getElementVal(),
             builder.getArrayAttr(createIdx()));
       } else if (!rangeSize) { // start a range
@@ -629,7 +629,7 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
           rangeBounds.push_back(
               mlir::cast<mlir::IntegerAttr>(idx[i]).getValue().getSExtValue());
         }
-        array = builder.create<fir::InsertOnRangeOp>(
+        array = fir::InsertOnRangeOp::create(builder,
             loc, arrayTy, array, getElementVal(),
             builder.getIndexVectorAttr(rangeBounds));
         rangeSize = 0;
@@ -679,11 +679,11 @@ genOutlineArrayLit(Fortran::lower::AbstractConverter &converter,
           [&](fir::FirOpBuilder &builder) {
             mlir::Value result =
                 genInlinedArrayLit(converter, loc, arrayTy, constant);
-            builder.create<fir::HasValueOp>(loc, result);
+            fir::HasValueOp::create(builder, loc, result);
           },
           builder.createInternalLinkage());
   }
-  return builder.create<fir::AddrOfOp>(loc, global.resultType(),
+  return fir::AddrOfOp::create(builder, loc, global.resultType(),
                                        global.getSymbol());
 }
 
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 281ab229d1b6a..a2fbec65187db 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -267,7 +267,7 @@ static mlir::Value genActualIsPresentTest(fir::FirOpBuilder &builder,
   // Optional case (not that optional allocatable/pointer cannot be absent
   // when passed to CMPLX as per 15.5.2.12 point 3 (7) and (8)). It is
   // therefore possible to catch them in the `then` case above.
-  return builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
+  return fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
                                           fir::getBase(actual));
 }
 
@@ -345,7 +345,7 @@ arrayLoadExtValue(fir::FirOpBuilder &builder, mlir::Location loc,
     auto origins = fir::factory::getNonDefaultLowerBounds(builder, loc, exv);
     if (shapeVal) {
       // shapeVal is a ShiftOp and load.memref() is a boxed value.
-      newBase = builder.create<fir::ReboxOp>(loc, oldBox.getType(), oldBox,
+      newBase = fir::ReboxOp::create(builder, loc, oldBox.getType(), oldBox,
                                              shapeVal, /*slice=*/mlir::Value{});
       origins = fir::factory::getOrigins(shapeVal);
     }
@@ -378,7 +378,7 @@ placeScalarValueInMemory(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value temp = builder.createTemporary(
       loc, storageType,
       llvm::ArrayRef<mlir::NamedAttribute>{fir::getAdaptToByRefAttr(builder)});
-  builder.create<fir::StoreOp>(loc, val, temp);
+  fir::StoreOp::create(builder, loc, val, temp);
   return fir::substBase(exv, temp);
 }
 
@@ -434,14 +434,14 @@ static fir::ExtendedValue genLoad(fir::FirOpBuilder &builder,
         if (mlir::isa<fir::RecordType>(
                 fir::unwrapRefType(fir::getBase(p).getType())))
           return p;
-        mlir::Value load = builder.create<fir::LoadOp>(loc, fir::getBase(p));
+        mlir::Value load = fir::LoadOp::create(builder, loc, fir::getBase(p));
         return fir::PolymorphicValue(load, p.getSourceBox());
       },
       [&](const fir::UnboxedValue &v) -> fir::ExtendedValue {
         if (mlir::isa<fir::RecordType>(
                 fir::unwrapRefType(fir::getBase(v).getType())))
           return v;
-        return builder.create<fir::LoadOp>(loc, fir::getBase(v));
+        return fir::LoadOp::create(builder, loc, fir::getBase(v));
       },
       [&](const fir::MutableBoxValue &box) -> fir::ExtendedValue {
         return genLoad(builder, loc,
@@ -473,11 +473,11 @@ static fir::ExtendedValue genOptionalValue(fir::FirOpBuilder &builder,
                /*withElseRegion=*/true)
       .genThen([&]() {
         mlir::Value val = fir::getBase(genLoad(builder, loc, exv));
-        builder.create<fir::ResultOp>(loc, val);
+        fir::ResultOp::create(builder, loc, val);
       })
       .genElse([&]() {
         mlir::Value zero = fir::factory::createZeroValue(builder, loc, eleType);
-        builder.create<fir::ResultOp>(loc, zero);
+        fir::ResultOp::create(builder, loc, zero);
       })
       .getResults()[0];
 }
@@ -521,8 +521,8 @@ static fir::ExtendedValue genOptionalBox(fir::FirOpBuilder &builder,
   // ensures it won't be.
   mlir::Value box = builder.createBox(loc, newExv);
   mlir::Type boxType = box.getType();
-  auto absent = builder.create<fir::AbsentOp>(loc, boxType);
-  auto boxOrAbsent = builder.create<mlir::arith::SelectOp>(
+  auto absent = fir::AbsentOp::create(builder, loc, boxType);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder,
       loc, boxType, isPresent, box, absent);
   return fir::BoxValue(boxOrAbsent);
 }
@@ -569,13 +569,13 @@ createBoxProcCharTuple(Fortran::lower::AbstractConverter &converter,
   if (fir::isa_ref_type(fromTy))
     funcAddr = builder.createConvert(loc, toTy, funcAddr);
   else if (mlir::isa<fir::BoxProcType>(fromTy))
-    funcAddr = builder.create<fir::BoxAddrOp>(loc, toTy, funcAddr);
+    funcAddr = fir::BoxAddrOp::create(builder, loc, toTy, funcAddr);
 
   auto boxProc = [&]() -> mlir::Value {
     if (auto host = Fortran::lower::argumentHostAssocs(converter, funcAddr))
-      return builder.create<fir::EmboxProcOp>(
+      return fir::EmboxProcOp::create(builder,
           loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
-    return builder.create<fir::EmboxProcOp>(loc, boxTy, funcAddr);
+    return fir::EmboxProcOp::create(builder, loc, boxTy, funcAddr);
   }();
   return fir::factory::createCharacterProcedureTuple(builder, loc, argTy,
                                                      boxProc, charLen);
@@ -598,7 +598,7 @@ absentBoxToUnallocatedBox(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value emptyBox =
       fir::factory::createUnallocatedBox(builder, loc, boxType, {});
   auto safeToReadBox =
-      builder.create<mlir::arith::SelectOp>(loc, isPresent, box, emptyBox);
+      mlir::arith::SelectOp::create(builder, loc, isPresent, box, emptyBox);
   return fir::substBase(exv, safeToReadBox);
 }
 
@@ -822,9 +822,9 @@ class ScalarExprLowering {
               Fortran::common::TypeCategory::Integer, *unsignedKind);
           mlir::Value lhsSL = builder.createConvert(loc, signlessType, *lhs);
           mlir::Value rhsSL = builder.createConvert(loc, signlessType, *rhs);
-          return builder.create<OpTy>(loc, pred, lhsSL, rhsSL);
+          return OpTy::create(builder, loc, pred, lhsSL, rhsSL);
         }
-        return builder.create<OpTy>(loc, pred, *lhs, *rhs);
+        return OpTy::create(builder, loc, pred, *lhs, *rhs);
       }
     }
     fir::emitFatalError(getLoc(), "array compare should be handled in genarr");
@@ -841,7 +841,7 @@ class ScalarExprLowering {
                              const ExtValue &left, const ExtValue &right) {
     if (const fir::UnboxedValue *lhs = left.getUnboxed())
       if (const fir::UnboxedValue *rhs = right.getUnboxed())
-        return builder.create<OpTy>(getLoc(), pred, *lhs, *rhs);
+        return OpTy::create(builder, getLoc(), pred, *lhs, *rhs);
     fir::emitFatalError(getLoc(), "array compare should be handled in genarr");
   }
   template <typename OpTy, typename A>
@@ -904,7 +904,7 @@ class ScalarExprLowering {
 
           mlir::Value cnvrt = Fortran::lower::addCrayPointerInst(
               loc, builder, ptrVal, ptrTy, pteVal.getType());
-          addr = builder.create<fir::LoadOp>(loc, cnvrt);
+          addr = fir::LoadOp::create(builder, loc, cnvrt);
         }
         return genLoad(addr);
       }
@@ -970,11 +970,11 @@ class ScalarExprLowering {
 
       std::string name = converter.getRecordTypeFieldName(sym);
       // FIXME: type parameters must come from the derived-type-spec
-      mlir::Value field = builder.create<fir::FieldIndexOp>(
+      mlir::Value field = fir::FieldIndexOp::create(builder,
           loc, fieldTy, name, ty,
           /*typeParams=*/mlir::ValueRange{} /*TODO*/);
       mlir::Type coorTy = builder.getRefType(recTy.getType(name));
-      auto coor = builder.create<fir::CoordinateOp>(loc, coorTy,
+      auto coor = fir::CoordinateOp::create(builder, loc, coorTy,
                                                     fir::getBase(res), field);
       ExtValue to = fir::factory::componentToExtendedValue(builder, loc, coor);
       to.match(
@@ -1077,7 +1077,7 @@ class ScalarExprLowering {
     mlir::Value input = genunbox(op.left());
     // Like LLVM, integer negation is the binary op "0 - value"
     mlir::Value zero = genIntegerConstant<KIND>(builder.getContext(), 0);
-    return builder.create<mlir::arith::SubIOp>(getLoc(), zero, input);
+    return mlir::arith::SubIOp::create(builder, getLoc(), zero, input);
   }
   template <int KIND>
   ExtValue genval(const Fortran::evaluate::Negate<Fortran::evaluate::Type<
@@ -1088,18 +1088,18 @@ class ScalarExprLowering {
     mlir::Value input = genunbox(op.left());
     mlir::Value signless = builder.createConvert(loc, signlessType, input);
     mlir::Value zero = genIntegerConstant<KIND>(builder.getContext(), 0);
-    mlir::Value neg = builder.create<mlir::arith::SubIOp>(loc, zero, signless);
+    mlir::Value neg = mlir::arith::SubIOp::create(builder, loc, zero, signless);
     return builder.createConvert(loc, input.getType(), neg);
   }
   template <int KIND>
   ExtValue genval(const Fortran::evaluate::Negate<Fortran::evaluate::Type<
                       Fortran::common::TypeCategory::Real, KIND>> &op) {
-    return builder.create<mlir::arith::NegFOp>(getLoc(), genunbox(op.left()));
+    return mlir::arith::NegFOp::create(builder, getLoc(), genunbox(op.left()));
   }
   template <int KIND>
   ExtValue genval(const Fortran::evaluate::Negate<Fortran::evaluate::Type<
                       Fortran::common::TypeCategory::Complex, KIND>> &op) {
-    return builder.create<fir::NegcOp>(getLoc(), genunbox(op.left()));
+    return fir::NegcOp::create(builder, getLoc(), genunbox(op.left()));
   }
 
   template <typename OpTy>
@@ -1312,7 +1312,7 @@ class ScalarExprLowering {
     ExtValue input = genval(op.left());
     mlir::Value base = fir::getBase(input);
     mlir::Value newBase =
-        builder.create<fir::NoReassocOp>(getLoc(), base.getType(), base);
+        fir::NoReassocOp::create(builder, getLoc(), base.getType(), base);
     return fir::substBase(input, newBase);
   }
 
@@ -1322,7 +1322,7 @@ class ScalarExprLowering {
     mlir::Value one = genBoolConstant(true);
     mlir::Value val =
         builder.createConvert(getLoc(), builder.getI1Type(), logical);
-    return builder.create<mlir::arith::XOrIOp>(getLoc(), val, one);
+    return mlir::arith::XOrIOp::create(builder, getLoc(), val, one);
   }
 
   template <int KIND>
@@ -1384,7 +1384,7 @@ class ScalarExprLowering {
     mlir::Value offset = builder.createIntegerConstant(
         loc, idxTy,
         x.part() == Fortran::evaluate::ComplexPart::Part::RE ? 0 : 1);
-    mlir::Value result = builder.create<fir::CoordinateOp>(
+    mlir::Value result = fir::CoordinateOp::create(builder,
         loc, builder.getRefType(eleTy), base, mlir::ValueRange{offset});
     return {result};
   }
@@ -1500,7 +1500,7 @@ class ScalarExprLowering {
       auto recTy = mlir::cast<fir::RecordType>(ty);
       const Fortran::semantics::Symbol &sym = getLastSym(*field);
       std::string name = converter.getRecordTypeFieldName(sym);
-      coorArgs.push_back(builder.create<fir::FieldIndexOp>(
+      coorArgs.push_back(fir::FieldIndexOp::create(builder,
           loc, fldTy, name, recTy, fir::getTypeParams(obj)));
       ty = recTy.getType(name);
     }
@@ -1510,7 +1510,7 @@ class ScalarExprLowering {
     ty = builder.getRefType(ty);
     return fir::factory::componentToExtendedValue(
         builder, loc,
-        builder.create<fir::CoordinateOp>(loc, ty, fir::getBase(obj),
+        fir::CoordinateOp::create(builder, loc, ty, fir::getBase(obj),
                                           coorArgs));
   }
 
@@ -1587,7 +1587,7 @@ class ScalarExprLowering {
       mlir::Value val = fir::getBase(subVal);
       mlir::Type ty = val.getType();
       mlir::Value lb = getLBound(array, subsc.index(), ty);
-      args.push_back(builder.create<mlir::arith::SubIOp>(loc, ty, val, lb));
+      args.push_back(mlir::arith::SubIOp::create(builder, loc, ty, val, lb));
     }
     mlir::Value base = fir::getBase(array);
 
@@ -1602,7 +1602,7 @@ class ScalarExprLowering {
 
       mlir::Value cnvrt = Fortran::lower::addCrayPointerInst(
           loc, builder, ptrVal, ptrTy, base.getType());
-      base = builder.create<fir::LoadOp>(loc, cnvrt);
+      base = fir::LoadOp::create(builder, loc, cnvrt);
     }
 
     mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(base.getType());
@@ -1611,7 +1611,7 @@ class ScalarExprLowering {
     auto seqTy = mlir::cast<fir::SequenceType>(eleTy);
     assert(args.size() == seqTy.getDimension());
     mlir::Type ty = builder.getRefType(seqTy.getEleTy());
-    auto addr = builder.create<fir::CoordinateOp>(loc, ty, base, args);
+    auto addr = fir::CoordinateOp::create(builder, loc, ty, base, args);
     return fir::factory::arrayElementToExtendedValue(builder, loc, array, addr);
   }
 
@@ -1648,12 +1648,12 @@ class ScalarExprLowering {
         mlir::Value val =
             builder.createConvert(loc, idxTy, fir::getBase(subVal));
         mlir::Value lb = builder.createConvert(loc, idxTy, getLB(arr, dim));
-        mlir::Value diff = builder.create<mlir::arith::SubIOp>(loc, val, lb);
+        mlir::Value diff = mlir::arith::SubIOp::create(builder, loc, val, lb);
         mlir::Value prod =
-            builder.create<mlir::arith::MulIOp>(loc, delta, diff);
-        total = builder.create<mlir::arith::AddIOp>(loc, prod, total);
+            mlir::arith::MulIOp::create(builder, loc, delta, diff);
+        total = mlir::arith::AddIOp::create(builder, loc, prod, total);
         if (ext)
-          delta = builder.create<mlir::arith::MulIOp>(loc, delta, ext);
+          delta = mlir::arith::MulIOp::create(builder, loc, delta, ext);
         ++dim;
       }
       mlir::Type origRefTy = refTy;
@@ -1672,7 +1672,7 @@ class ScalarExprLowering {
           base = builder.createConvert(loc, seqRefTy, base);
         }
       }
-      auto coor = builder.create<fir::CoordinateOp>(
+      auto coor = fir::CoordinateOp::create(builder,
           loc, refTy, base, llvm::ArrayRef<mlir::Value>{total});
       // Convert to expected, original type after address arithmetic.
       return builder.createConvert(loc, origRefTy, coor);
@@ -1725,7 +1725,7 @@ class ScalarExprLowering {
           builder.createConvert(loc, idxTy, fir::getBase(subVal)));
     }
     mlir::Value shape = builder.createShape(loc, exv);
-    mlir::Value elementAddr = builder.create<fir::ArrayCoorOp>(
+    mlir::Value elementAddr = fir::ArrayCoorOp::create(builder,
         loc, refTy, addr, shape, /*slice=*/mlir::Value{}, arrayCoorArgs,
         fir::getTypeParams(exv));
     return fir::factory::arrayElementToExtendedValue(builder, loc, exv,
@@ -1826,7 +1826,7 @@ class ScalarExprLowering {
     if (mlir::isa<mlir::FunctionType>(exvTy)) {
       auto boxProcTy =
           builder.getBoxProcType(mlir::cast<mlir::FunctionType>(exvTy));
-      return builder.create<fir::EmboxProcOp>(loc, boxProcTy,
+      return fir::EmboxProcOp::create(builder, loc, boxProcTy,
                                               fir::getBase(exv));
     }
     mlir::Value box = builder.createBox(loc, exv, exv.isPolymorphic());
@@ -2073,7 +2073,7 @@ class ScalarExprLowering {
       TODO(loc, "creating temporary for derived type with length parameters");
     }
 
-    mlir::Value temp = builder.create<fir::AllocMemOp>(
+    mlir::Value temp = fir::AllocMemOp::create(builder,
         loc, type, tempName, allocMemTypeParams, extents);
     if (mlir::isa<fir::CharacterType>(fir::unwrapSequenceType(type)))
       return fir::CharArrayBoxValue{temp, charLen, extents};
@@ -2124,9 +2124,9 @@ class ScalarExprLowering {
           mlir::Type type = v.getType();
           mlir::Value value = v;
           if (fir::isa_ref_type(type))
-            value = builder.create<fir::LoadOp>(loc, value);
+            value = fir::LoadOp::create(builder, loc, value);
           mlir::Value temp = builder.createTemporary(loc, value.getType());
-          builder.create<fir::StoreOp>(loc, value, temp);
+          fir::StoreOp::create(builder, loc, value, temp);
           return temp;
         },
         [&](const fir::BoxValue &x) -> ExtValue {
@@ -2141,9 +2141,9 @@ class ScalarExprLowering {
           // created always has the declared type.
           mlir::Value var =
               fir::getBase(fir::factory::readBoxValue(builder, loc, x));
-          auto value = builder.create<fir::LoadOp>(loc, var);
+          auto value = fir::LoadOp::create(builder, loc, var);
           mlir::Value temp = builder.createTemporary(loc, value.getType());
-          builder.create<fir::StoreOp>(loc, value, temp);
+          fir::StoreOp::create(builder, loc, value, temp);
           return temp;
         },
         [&](const fir::PolymorphicValue &p) -> ExtValue {
@@ -2242,7 +2242,7 @@ class ScalarExprLowering {
       // at this point.
       mlir::Value destBox = fir::getBase(builder.createBox(loc, temp));
       mlir::Value boxRef = builder.createTemporary(loc, destBox.getType());
-      builder.create<fir::StoreOp>(loc, destBox, boxRef);
+      fir::StoreOp::create(builder, loc, destBox, boxRef);
       fir::runtime::genAssignTemporary(builder, loc, boxRef,
                                        fir::getBase(actualArg));
       return temp;
@@ -2250,20 +2250,20 @@ class ScalarExprLowering {
 
     auto noCopy = [&]() {
       mlir::Value box = fir::getBase(actualArg);
-      mlir::Value boxAddr = builder.create<fir::BoxAddrOp>(loc, addrType, box);
-      builder.create<fir::ResultOp>(loc, boxAddr);
+      mlir::Value boxAddr = fir::BoxAddrOp::create(builder, loc, addrType, box);
+      fir::ResultOp::create(builder, loc, boxAddr);
     };
 
     auto combinedCondition = [&]() {
       if (isActualArgBox) {
         mlir::Value zero =
             builder.createIntegerConstant(loc, builder.getI1Type(), 0);
-        mlir::Value notContiguous = builder.create<mlir::arith::CmpIOp>(
+        mlir::Value notContiguous = mlir::arith::CmpIOp::create(builder,
             loc, mlir::arith::CmpIPredicate::eq, isContiguousResult, zero);
         if (!restrictCopyAtRuntime) {
           restrictCopyAtRuntime = notContiguous;
         } else {
-          mlir::Value cond = builder.create<mlir::arith::AndIOp>(
+          mlir::Value cond = mlir::arith::AndIOp::create(builder,
               loc, *restrictCopyAtRuntime, notContiguous);
           restrictCopyAtRuntime = cond;
         }
@@ -2280,7 +2280,7 @@ class ScalarExprLowering {
                 .genThen([&]() { noCopy(); })
                 .genElse([&] {
                   ExtValue temp = doCopyIn();
-                  builder.create<fir::ResultOp>(loc, fir::getBase(temp));
+                  fir::ResultOp::create(builder, loc, fir::getBase(temp));
                 })
                 .getResults()[0];
         fir::ExtendedValue temp =
@@ -2312,19 +2312,19 @@ class ScalarExprLowering {
                         .genThen([&]() { noCopy(); })
                         .genElse([&]() {
                           ExtValue temp = doCopyIn();
-                          builder.create<fir::ResultOp>(loc,
+                          fir::ResultOp::create(builder, loc,
                                                         fir::getBase(temp));
                         })
                         .getResults()[0];
-                builder.create<fir::ResultOp>(loc, addr1);
+                fir::ResultOp::create(builder, loc, addr1);
               } else {
                 ExtValue temp = doCopyIn();
-                builder.create<fir::ResultOp>(loc, fir::getBase(temp));
+                fir::ResultOp::create(builder, loc, fir::getBase(temp));
               }
             })
             .genElse([&]() {
               mlir::Value nullPtr = builder.createNullConstant(loc, addrType);
-              builder.create<fir::ResultOp>(loc, nullPtr);
+              fir::ResultOp::create(builder, loc, nullPtr);
             })
             .getResults()[0];
     // Associate the temp address with actualArg lengths and extents if a
@@ -2357,7 +2357,7 @@ class ScalarExprLowering {
                                                                  tempBox);
         }
         // Deallocate the top-level entity of the temporary.
-        builder.create<fir::FreeMemOp>(loc, fir::getBase(copyOutPair.temp));
+        fir::FreeMemOp::create(builder, loc, fir::getBase(copyOutPair.temp));
         return;
       }
       // Generate CopyOutAssign() call to copy data from the temporary
@@ -2376,11 +2376,11 @@ class ScalarExprLowering {
       mlir::Type allocBoxTy =
           mlir::cast<fir::BaseBoxType>(srcBox.getType())
               .getBoxTypeWithNewAttr(fir::BaseBoxType::Attribute::Allocatable);
-      srcBox = builder.create<fir::ReboxOp>(loc, allocBoxTy, srcBox,
+      srcBox = fir::ReboxOp::create(builder, loc, allocBoxTy, srcBox,
                                             /*shift=*/mlir::Value{},
                                             /*slice=*/mlir::Value{});
       mlir::Value srcBoxRef = builder.createTemporary(loc, srcBox.getType());
-      builder.create<fir::StoreOp>(loc, srcBox, srcBoxRef);
+      fir::StoreOp::create(builder, loc, srcBox, srcBoxRef);
       // Create descriptor pointer to variable descriptor if copy out is needed,
       // and nullptr otherwise.
       mlir::Value destBoxRef;
@@ -2388,9 +2388,9 @@ class ScalarExprLowering {
         mlir::Value destBox =
             fir::getBase(builder.createBox(loc, copyOutPair.var));
         destBoxRef = builder.createTemporary(loc, destBox.getType());
-        builder.create<fir::StoreOp>(loc, destBox, destBoxRef);
+        fir::StoreOp::create(builder, loc, destBox, destBoxRef);
       } else {
-        destBoxRef = builder.create<fir::ZeroOp>(loc, srcBoxRef.getType());
+        destBoxRef = fir::ZeroOp::create(builder, loc, srcBoxRef.getType());
       }
       fir::runtime::genCopyOutAssign(builder, loc, destBoxRef, srcBoxRef);
     };
@@ -2436,7 +2436,7 @@ class ScalarExprLowering {
     // fir.box is absent.
     ExtValue actualArg = gen(expr);
     mlir::Value actualArgBase = fir::getBase(actualArg);
-    mlir::Value isPresent = builder.create<fir::IsPresentOp>(
+    mlir::Value isPresent = fir::IsPresentOp::create(builder,
         loc, builder.getI1Type(), actualArgBase);
     if (!mlir::isa<fir::BoxType>(actualArgBase.getType()))
       return {actualArg, isPresent};
@@ -2457,7 +2457,7 @@ class ScalarExprLowering {
     if (const fir::CharBoxValue *charBox = actualArg.getCharBox()) {
       mlir::Value len = charBox->getLen();
       mlir::Value zero = builder.createIntegerConstant(loc, len.getType(), 0);
-      len = builder.create<mlir::arith::SelectOp>(loc, isPresent, len, zero);
+      len = mlir::arith::SelectOp::create(builder, loc, isPresent, len, zero);
       mlir::Value temp =
           builder.createTemporary(loc, type, /*name=*/{},
                                   /*shape=*/{}, mlir::ValueRange{len},
@@ -2538,12 +2538,12 @@ class ScalarExprLowering {
                 .genThen([&]() {
                   fir::factory::genScalarAssignment(builder, loc, temp,
                                                     actualArg);
-                  builder.create<fir::ResultOp>(loc, fir::getBase(temp));
+                  fir::ResultOp::create(builder, loc, fir::getBase(temp));
                 })
                 .genElse([&]() {
                   mlir::Value absent =
-                      builder.create<fir::AbsentOp>(loc, tempAddrTy);
-                  builder.create<fir::ResultOp>(loc, absent);
+                      fir::AbsentOp::create(builder, loc, tempAddrTy);
+                  fir::ResultOp::create(builder, loc, absent);
                 })
                 .getResults()[0];
         return {fir::substBase(temp, selectAddr), isPresent};
@@ -2647,7 +2647,7 @@ class ScalarExprLowering {
           mlir::Value boxStorage = builder.createTemporary(loc, boxTy);
           mlir::Value nullBox = fir::factory::createUnallocatedBox(
               builder, loc, boxTy, /*nonDeferredParams=*/{});
-          builder.create<fir::StoreOp>(loc, nullBox, boxStorage);
+          fir::StoreOp::create(builder, loc, nullBox, boxStorage);
           caller.placeInput(arg, boxStorage);
           continue;
         }
@@ -2706,9 +2706,9 @@ class ScalarExprLowering {
                       mlir::cast<fir::BoxCharType>(funcTy.getResult(0));
                   mlir::Value ref = builder.createConvertWithVolatileCast(
                       loc, builder.getRefType(boxTy.getEleTy()), x.getAddr());
-                  auto len = builder.create<fir::UndefOp>(
+                  auto len = fir::UndefOp::create(builder,
                       loc, builder.getCharacterLengthType());
-                  return builder.create<fir::EmboxCharOp>(loc, boxTy, ref, len);
+                  return fir::EmboxCharOp::create(builder, loc, boxTy, ref, len);
                 }
                 return helper.createEmbox(x);
               },
@@ -2758,9 +2758,9 @@ class ScalarExprLowering {
           mlir::Value box = builder.createBox(loc, argAddr);
           if (isPresentValue) {
             mlir::Value convertedBox = builder.createConvert(loc, argTy, box);
-            auto absent = builder.create<fir::AbsentOp>(loc, argTy);
+            auto absent = fir::AbsentOp::create(builder, loc, argTy);
             caller.placeInput(arg,
-                              builder.create<mlir::arith::SelectOp>(
+                              mlir::arith::SelectOp::create(builder,
                                   loc, *isPresentValue, convertedBox, absent));
           } else {
             caller.placeInput(arg, builder.createBox(loc, argAddr));
@@ -2782,7 +2782,7 @@ class ScalarExprLowering {
           mlir::Value isAllocated =
               fir::factory::genIsAllocatedOrAssociatedTest(builder, loc,
                                                            mutableBox);
-          auto absent = builder.create<fir::AbsentOp>(loc, argTy);
+          auto absent = fir::AbsentOp::create(builder, loc, argTy);
           /// For now, assume it is not OK to pass the allocatable/pointer
           /// descriptor to a non pointer/allocatable dummy. That is a strict
           /// interpretation of 18.3.6 point 4 that stipulates the descriptor
@@ -2801,13 +2801,13 @@ class ScalarExprLowering {
                 box);
           } else if (mlir::isa<fir::BoxType>(box.getType()) &&
                      fir::isPolymorphicType(argTy)) {
-            box = builder.create<fir::ReboxOp>(loc, argTy, box, mlir::Value{},
+            box = fir::ReboxOp::create(builder, loc, argTy, box, mlir::Value{},
                                                /*slice=*/mlir::Value{});
           }
 
           // Need the box types to be exactly similar for the selectOp.
           mlir::Value convertedBox = builder.createConvert(loc, argTy, box);
-          caller.placeInput(arg, builder.create<mlir::arith::SelectOp>(
+          caller.placeInput(arg, mlir::arith::SelectOp::create(builder,
                                      loc, isAllocated, convertedBox, absent));
         } else {
           auto dynamicType = expr->GetType();
@@ -2830,12 +2830,12 @@ class ScalarExprLowering {
                     .genThen([&]() {
                       auto boxed = builder.createBox(
                           loc, genBoxArg(*expr), fir::isPolymorphicType(argTy));
-                      builder.create<fir::ResultOp>(loc, boxed);
+                      fir::ResultOp::create(builder, loc, boxed);
                     })
                     .genElse([&]() {
                       auto absent =
-                          builder.create<fir::AbsentOp>(loc, argTy).getResult();
-                      builder.create<fir::ResultOp>(loc, absent);
+                          fir::AbsentOp::create(builder, loc, argTy).getResult();
+                      fir::ResultOp::create(builder, loc, absent);
                     })
                     .getResults()[0];
           } else {
@@ -2867,17 +2867,17 @@ class ScalarExprLowering {
                                         loc, actualTy, box, mlir::Value{},
                                         /*slice=*/mlir::Value{})
                                     .getResult();
-                            builder.create<fir::ResultOp>(loc, rebox);
+                            fir::ResultOp::create(builder, loc, rebox);
                           })
                           .genElse([&]() {
                             auto absent =
-                                builder.create<fir::AbsentOp>(loc, actualTy)
+                                fir::AbsentOp::create(builder, loc, actualTy)
                                     .getResult();
-                            builder.create<fir::ResultOp>(loc, absent);
+                            fir::ResultOp::create(builder, loc, absent);
                           })
                           .getResults()[0];
               } else {
-                box = builder.create<fir::ReboxOp>(loc, actualTy, box,
+                box = fir::ReboxOp::create(builder, loc, actualTy, box,
                                                    mlir::Value{},
                                                    /*slice=*/mlir::Value{});
               }
@@ -3132,12 +3132,12 @@ static void genScalarUserDefinedAssignmentCall(fir::FirOpBuilder &builder,
     if (argBaseType != fir::unwrapRefType(from.getType())) {
       // With logicals, it is possible that from is i1 here.
       if (fir::isa_ref_type(from.getType()))
-        from = builder.create<fir::LoadOp>(loc, from);
+        from = fir::LoadOp::create(builder, loc, from);
       from = builder.createConvert(loc, argBaseType, from);
     }
     if (!fir::isa_ref_type(from.getType())) {
       mlir::Value temp = builder.createTemporary(loc, argBaseType);
-      builder.create<fir::StoreOp>(loc, from, temp);
+      fir::StoreOp::create(builder, loc, from, temp);
       from = temp;
     }
     return builder.createConvert(loc, argType, from);
@@ -3147,7 +3147,7 @@ static void genScalarUserDefinedAssignmentCall(fir::FirOpBuilder &builder,
   mlir::Type rhsType = func.getFunctionType().getInput(1);
   mlir::Value lhsArg = prepareUserDefinedArg(builder, loc, lhs, lhsType);
   mlir::Value rhsArg = prepareUserDefinedArg(builder, loc, rhs, rhsType);
-  builder.create<fir::CallOp>(loc, func, mlir::ValueRange{lhsArg, rhsArg});
+  fir::CallOp::create(builder, loc, func, mlir::ValueRange{lhsArg, rhsArg});
 }
 
 /// Convert the result of a fir.array_modify to an ExtendedValue given the
@@ -3187,16 +3187,16 @@ createDerivedArrayAmend(mlir::Location loc, fir::ArrayLoadOp destLoad,
     fir::factory::genRecordAssignment(builder, loc, destAcc, elementExv);
   } else {
     auto boxTy = fir::BoxType::get(eleTy);
-    auto toBox = builder.create<fir::EmboxOp>(loc, boxTy, destAcc.getResult(),
+    auto toBox = fir::EmboxOp::create(builder, loc, boxTy, destAcc.getResult(),
                                               mlir::Value{}, mlir::Value{},
                                               destLoad.getTypeparams());
-    auto fromBox = builder.create<fir::EmboxOp>(
+    auto fromBox = fir::EmboxOp::create(builder,
         loc, boxTy, fir::getBase(elementExv), mlir::Value{}, mlir::Value{},
         destLoad.getTypeparams());
     fir::factory::genRecordAssignment(builder, loc, fir::BoxValue(toBox),
                                       fir::BoxValue(fromBox));
   }
-  return builder.create<fir::ArrayAmendOp>(loc, innerArg.getType(), innerArg,
+  return fir::ArrayAmendOp::create(builder, loc, innerArg.getType(), innerArg,
                                            destAcc);
 }
 
@@ -3219,7 +3219,7 @@ createCharArrayAmend(mlir::Location loc, fir::FirOpBuilder &builder,
   helper.createAssign(fir::ExtendedValue{dstChar}, srcExv);
   // Mark this array element as amended.
   mlir::Type ty = innerArg.getType();
-  auto amend = builder.create<fir::ArrayAmendOp>(loc, ty, innerArg, dstOp);
+  auto amend = fir::ArrayAmendOp::create(builder, loc, ty, innerArg, dstOp);
   return amend;
 }
 
@@ -3235,7 +3235,7 @@ convertToArrayBoxValue(mlir::Location loc, fir::FirOpBuilder &builder,
   mlir::Type ty = fir::unwrapRefType(val.getType());
   mlir::IndexType idxTy = builder.getIndexType();
   auto seqTy = mlir::cast<fir::SequenceType>(ty);
-  auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+  auto undef = fir::UndefOp::create(builder, loc, idxTy);
   llvm::SmallVector<mlir::Value> extents(seqTy.getDimension(), undef);
   if (fir::isa_char(seqTy.getEleTy()))
     return fir::CharArrayBoxValue(val, len ? len : undef, extents);
@@ -3315,9 +3315,9 @@ class ArrayExprLowering {
     ExtValue exv = lowerArrayExpression(rhs);
     if (explicitSpaceIsActive()) {
       explicitSpace->finalizeContext();
-      builder.create<fir::ResultOp>(loc, fir::getBase(exv));
+      fir::ResultOp::create(builder, loc, fir::getBase(exv));
     } else {
-      builder.create<fir::ArrayMergeStoreOp>(
+      fir::ArrayMergeStoreOp::create(builder,
           loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     }
@@ -3432,7 +3432,7 @@ class ArrayExprLowering {
     assert(destination && "destination must have been set");
     ExtValue exv = lowerArrayExpression(rhsCC, destination.getType());
     if (!explicitSpaceIsActive())
-      builder.create<fir::ArrayMergeStoreOp>(
+      fir::ArrayMergeStoreOp::create(builder,
           loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     // destShape may originally be null, if rhs did not define a shape.
@@ -3502,7 +3502,7 @@ class ArrayExprLowering {
                                          lengthParams, assignToStorage);
     if (explicitSpaceIsActive()) {
       explicitSpace->finalizeContext();
-      builder.create<fir::ResultOp>(loc, fir::getBase(realloc.newValue));
+      fir::ResultOp::create(builder, loc, fir::getBase(realloc.newValue));
     }
     fir::factory::finalizeRealloc(builder, loc, mutableBox, lbounds,
                                   takeLboundsIfRealloc, realloc);
@@ -3547,7 +3547,7 @@ class ArrayExprLowering {
     mlir::Value tempRes = dest.getMemref();
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     mlir::Location loc = converter.getCurrentLocation();
-    builder.create<fir::ArrayMergeStoreOp>(loc, dest, fir::getBase(loopRes),
+    fir::ArrayMergeStoreOp::create(builder, loc, dest, fir::getBase(loopRes),
                                            tempRes, dest.getSlice(),
                                            dest.getTypeparams());
 
@@ -3595,16 +3595,16 @@ class ArrayExprLowering {
     // as there isn't any necessity for it.
     ccLoadDest = [=](llvm::ArrayRef<mlir::Value> shape) -> fir::ArrayLoadOp {
       mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-      auto var = builder.create<fir::CoordinateOp>(
+      auto var = fir::CoordinateOp::create(builder,
           loc, builder.getRefType(hdrTy.getType(1)), header, one);
-      auto load = builder.create<fir::LoadOp>(loc, var);
+      auto load = fir::LoadOp::create(builder, loc, var);
       mlir::Type eleTy =
           fir::unwrapSequenceType(fir::unwrapRefType(load.getType()));
       auto seqTy = fir::SequenceType::get(eleTy, shape.size());
       mlir::Value castTo =
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
-      return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
+      return fir::ArrayLoadOp::create(builder, loc, seqTy, castTo, shapeOp,
                                               /*slice=*/mlir::Value{},
                                               mlir::ValueRange{});
     };
@@ -3612,9 +3612,9 @@ class ArrayExprLowering {
     // to the lazy allocated buffer.
     ccStoreToDest = [=](IterSpace iters) {
       mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-      auto var = builder.create<fir::CoordinateOp>(
+      auto var = fir::CoordinateOp::create(builder,
           loc, builder.getRefType(hdrTy.getType(1)), header, one);
-      auto load = builder.create<fir::LoadOp>(loc, var);
+      auto load = fir::LoadOp::create(builder, loc, var);
       mlir::Type eleTy =
           fir::unwrapSequenceType(fir::unwrapRefType(load.getType()));
       auto seqTy = fir::SequenceType::get(eleTy, iters.iterVec().size());
@@ -3623,12 +3623,12 @@ class ArrayExprLowering {
       mlir::Value shape = builder.genShape(loc, genIterationShape());
       llvm::SmallVector<mlir::Value> indices = fir::factory::originateIndices(
           loc, builder, castTo.getType(), shape, iters.iterVec());
-      auto eleAddr = builder.create<fir::ArrayCoorOp>(
+      auto eleAddr = fir::ArrayCoorOp::create(builder,
           loc, builder.getRefType(eleTy), castTo, shape,
           /*slice=*/mlir::Value{}, indices, destination.getTypeparams());
       mlir::Value eleVal =
           builder.createConvert(loc, eleTy, iters.getElement());
-      builder.create<fir::StoreOp>(loc, eleVal, eleAddr);
+      fir::StoreOp::create(builder, loc, eleVal, eleAddr);
       return iters.innerArgument();
     };
 
@@ -3684,9 +3684,9 @@ class ArrayExprLowering {
     auto exv = lowerArrayExpression(rhs);
     if (explicitSpaceIsActive()) {
       explicitSpace->finalizeContext();
-      builder.create<fir::ResultOp>(loc, fir::getBase(exv));
+      fir::ResultOp::create(builder, loc, fir::getBase(exv));
     } else {
-      builder.create<fir::ArrayMergeStoreOp>(
+      fir::ArrayMergeStoreOp::create(builder,
           loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     }
@@ -3766,7 +3766,7 @@ class ArrayExprLowering {
       std::size_t offset = explicitSpace->argPosition(oldInnerArg);
       explicitSpace->setInnerArg(offset, fir::getBase(lexv));
       finalizeElementCtx();
-      builder.create<fir::ResultOp>(loc, fir::getBase(lexv));
+      fir::ResultOp::create(builder, loc, fir::getBase(lexv));
     };
     if (mlir::Operation *defOp = fir::getBase(lexv).getDefiningOp()) {
       llvm::TypeSwitch<mlir::Operation *>(defOp)
@@ -3836,7 +3836,7 @@ class ArrayExprLowering {
     // 5). Thread the array value updated forward.
     if (!isIllFormedLHS) {
       finalizeElementCtx();
-      builder.create<fir::ResultOp>(getLoc(), fir::getBase(lexv));
+      fir::ResultOp::create(builder, getLoc(), fir::getBase(lexv));
     }
     return lexv;
   }
@@ -3979,7 +3979,7 @@ class ArrayExprLowering {
     if (auto origEleTy = fir::dyn_cast_ptrEleTy(origVal.getType()))
       if (mlir::isa<fir::BaseBoxType>(origEleTy)) {
         // If origVal is a box variable, load it so it is in the value domain.
-        origVal = builder.create<fir::LoadOp>(loc, origVal);
+        origVal = fir::LoadOp::create(builder, loc, origVal);
       }
     if (mlir::isa<fir::BoxType>(origVal.getType()) &&
         !mlir::isa<fir::BoxType>(eleTy)) {
@@ -3998,7 +3998,7 @@ class ArrayExprLowering {
         TODO(loc, "TARGET of pointer assignment with runtime size/shape");
       auto memrefTy = fir::boxMemRefType(mlir::cast<fir::BoxType>(eleTy));
       auto castTo = builder.createConvert(loc, memrefTy, origVal);
-      origVal = builder.create<fir::EmboxOp>(loc, eleTy, castTo);
+      origVal = fir::EmboxOp::create(builder, loc, eleTy, castTo);
     }
     mlir::Value val = builder.convertWithSemantics(loc, eleTy, origVal);
     if (isBoundsSpec()) {
@@ -4007,8 +4007,8 @@ class ArrayExprLowering {
       if (lbs.size() > 0) {
         // Rebox the value with user-specified shift.
         auto shiftTy = fir::ShiftType::get(eleTy.getContext(), lbs.size());
-        mlir::Value shiftOp = builder.create<fir::ShiftOp>(loc, shiftTy, lbs);
-        val = builder.create<fir::ReboxOp>(loc, eleTy, val, shiftOp,
+        mlir::Value shiftOp = fir::ShiftOp::create(builder, loc, shiftTy, lbs);
+        val = fir::ReboxOp::create(builder, loc, eleTy, val, shiftOp,
                                            mlir::Value{});
       }
     } else if (isBoundsRemap()) {
@@ -4020,8 +4020,8 @@ class ArrayExprLowering {
         auto shapeShiftArgs = flatZip(lbs, *ubounds);
         auto shapeTy = fir::ShapeShiftType::get(eleTy.getContext(), lbs.size());
         mlir::Value shapeShift =
-            builder.create<fir::ShapeShiftOp>(loc, shapeTy, shapeShiftArgs);
-        val = builder.create<fir::ReboxOp>(loc, eleTy, val, shapeShift,
+            fir::ShapeShiftOp::create(builder, loc, shapeTy, shapeShiftArgs);
+        val = fir::ReboxOp::create(builder, loc, eleTy, val, shapeShift,
                                            mlir::Value{});
       }
     }
@@ -4045,7 +4045,7 @@ class ArrayExprLowering {
         // memory into the destination array.
         mlir::Type resRefTy = builder.getRefType(eleTy);
         // Get a reference to the array element to be amended.
-        auto arrayOp = builder.create<fir::ArrayAccessOp>(
+        auto arrayOp = fir::ArrayAccessOp::create(builder,
             loc, resRefTy, innerArg, iterSpace.iterVec(),
             fir::factory::getTypeParams(loc, builder, destination));
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
@@ -4067,7 +4067,7 @@ class ArrayExprLowering {
       }
       // By value semantics. The element is being assigned by value.
       auto ele = convertElementForUpdate(loc, eleTy, fir::getBase(exv));
-      auto update = builder.create<fir::ArrayUpdateOp>(
+      auto update = fir::ArrayUpdateOp::create(builder,
           loc, arrTy, innerArg, ele, iterSpace.iterVec(),
           destination.getTypeparams());
       return abstractArrayExtValue(update);
@@ -4094,7 +4094,7 @@ class ArrayExprLowering {
                       : defaultStoreToDestination(/*substring=*/nullptr);
     mlir::Value updVal = fir::getBase(lambda(iterSpace));
     finalizeElementCtx();
-    builder.create<fir::ResultOp>(loc, updVal);
+    fir::ResultOp::create(builder, loc, updVal);
     builder.restoreInsertionPoint(insPt);
     return abstractArrayExtValue(iterSpace.outerResult());
   }
@@ -4247,31 +4247,31 @@ class ArrayExprLowering {
       // Compute the dynamic position into the header.
       llvm::SmallVector<mlir::Value> offsets;
       for (auto doLoop : loopStack[i]) {
-        auto m = builder.create<mlir::arith::SubIOp>(
+        auto m = mlir::arith::SubIOp::create(builder,
             loc, doLoop.getInductionVar(), doLoop.getLowerBound());
-        auto n = builder.create<mlir::arith::DivSIOp>(loc, m, doLoop.getStep());
+        auto n = mlir::arith::DivSIOp::create(builder, loc, m, doLoop.getStep());
         mlir::Value one = builder.createIntegerConstant(loc, n.getType(), 1);
-        offsets.push_back(builder.create<mlir::arith::AddIOp>(loc, n, one));
+        offsets.push_back(mlir::arith::AddIOp::create(builder, loc, n, one));
       }
       mlir::IntegerType i32Ty = builder.getIntegerType(32);
       mlir::Value uno = builder.createIntegerConstant(loc, i32Ty, 1);
       mlir::Type coorTy = builder.getRefType(raggedTy.getType(1));
-      auto hdOff = builder.create<fir::CoordinateOp>(loc, coorTy, header, uno);
+      auto hdOff = fir::CoordinateOp::create(builder, loc, coorTy, header, uno);
       auto toTy = fir::SequenceType::get(raggedTy, offsets.size());
       mlir::Type toRefTy = builder.getRefType(toTy);
-      auto ldHdr = builder.create<fir::LoadOp>(loc, hdOff);
+      auto ldHdr = fir::LoadOp::create(builder, loc, hdOff);
       mlir::Value hdArr = builder.createConvert(loc, toRefTy, ldHdr);
       auto shapeOp = builder.genShape(loc, extents);
-      header = builder.create<fir::ArrayCoorOp>(
+      header = fir::ArrayCoorOp::create(builder,
           loc, builder.getRefType(raggedTy), hdArr, shapeOp,
           /*slice=*/mlir::Value{}, offsets,
           /*typeparams=*/mlir::ValueRange{});
-      auto hdrVar = builder.create<fir::CoordinateOp>(loc, coorTy, header, uno);
-      auto inVar = builder.create<fir::LoadOp>(loc, hdrVar);
+      auto hdrVar = fir::CoordinateOp::create(builder, loc, coorTy, header, uno);
+      auto inVar = fir::LoadOp::create(builder, loc, hdrVar);
       mlir::Value two = builder.createIntegerConstant(loc, i32Ty, 2);
       mlir::Type coorTy2 = builder.getRefType(raggedTy.getType(2));
-      auto hdrSh = builder.create<fir::CoordinateOp>(loc, coorTy2, header, two);
-      auto shapePtr = builder.create<fir::LoadOp>(loc, hdrSh);
+      auto hdrSh = fir::CoordinateOp::create(builder, loc, coorTy2, header, two);
+      auto shapePtr = fir::LoadOp::create(builder, loc, hdrSh);
       // Replace the binding.
       implicitSpace->rebind(expr, genMaskAccess(inVar, shapePtr));
       if (i < depth - 1)
@@ -4301,7 +4301,7 @@ class ArrayExprLowering {
             Fortran::lower::createLazyArrayTempValue(converter, *e, header,
                                                      symMap, stmtCtx);
             // Close the explicit loops.
-            builder.create<fir::ResultOp>(loc, explicitSpace->getInnerArgs());
+            fir::ResultOp::create(builder, loc, explicitSpace->getInnerArgs());
             builder.setInsertionPointAfter(explicitSpace->getOuterLoop());
             // Open a new copy of the explicit loop nest.
             explicitSpace->genLoopNest();
@@ -4327,9 +4327,9 @@ class ArrayExprLowering {
               fir::factory::getRaggedArrayHeaderType(builder);
           mlir::IntegerType i32Ty = builder.getIntegerType(32);
           mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-          auto coor1 = builder.create<fir::CoordinateOp>(
+          auto coor1 = fir::CoordinateOp::create(builder,
               loc, builder.getRefType(raggedTy.getType(1)), header, one);
-          auto db = builder.create<fir::LoadOp>(loc, coor1);
+          auto db = fir::LoadOp::create(builder, loc, coor1);
           mlir::Type eleTy =
               fir::unwrapSequenceType(fir::unwrapRefType(db.getType()));
           mlir::Type buffTy =
@@ -4338,17 +4338,17 @@ class ArrayExprLowering {
           mlir::Value buff = builder.createConvert(loc, buffTy, db);
 
           mlir::Value two = builder.createIntegerConstant(loc, i32Ty, 2);
-          auto coor2 = builder.create<fir::CoordinateOp>(
+          auto coor2 = fir::CoordinateOp::create(builder,
               loc, builder.getRefType(raggedTy.getType(2)), header, two);
-          auto shBuff = builder.create<fir::LoadOp>(loc, coor2);
+          auto shBuff = fir::LoadOp::create(builder, loc, coor2);
           mlir::IntegerType i64Ty = builder.getIntegerType(64);
           mlir::IndexType idxTy = builder.getIndexType();
           llvm::SmallVector<mlir::Value> extents;
           for (std::remove_const_t<decltype(rank)> i = 0; i < rank; ++i) {
             mlir::Value off = builder.createIntegerConstant(loc, i32Ty, i);
-            auto coor = builder.create<fir::CoordinateOp>(
+            auto coor = fir::CoordinateOp::create(builder,
                 loc, builder.getRefType(i64Ty), shBuff, off);
-            auto ldExt = builder.create<fir::LoadOp>(loc, coor);
+            auto ldExt = fir::LoadOp::create(builder, loc, coor);
             extents.push_back(builder.createConvert(loc, idxTy, ldExt));
           }
           if (destShape.empty())
@@ -4376,7 +4376,7 @@ class ArrayExprLowering {
     // run from 0 to `extent - 1` inclusive.
     for (auto extent : shape)
       loopUppers.push_back(
-          builder.create<mlir::arith::SubIOp>(loc, extent, one));
+          mlir::arith::SubIOp::create(builder, loc, extent, one));
 
     // Iteration space is created with outermost columns, innermost rows
     llvm::SmallVector<fir::DoLoopOp> loops;
@@ -4391,14 +4391,14 @@ class ArrayExprLowering {
       }
       fir::DoLoopOp loop;
       if (innerArg) {
-        loop = builder.create<fir::DoLoopOp>(
+        loop = fir::DoLoopOp::create(builder,
             loc, zero, i.value(), one, isUnordered(),
             /*finalCount=*/false, mlir::ValueRange{innerArg});
         innerArg = loop.getRegionIterArgs().front();
         if (explicitSpaceIsActive())
           explicitSpace->setInnerArg(0, innerArg);
       } else {
-        loop = builder.create<fir::DoLoopOp>(loc, zero, i.value(), one,
+        loop = fir::DoLoopOp::create(builder, loc, zero, i.value(), one,
                                              isUnordered(),
                                              /*finalCount=*/false);
       }
@@ -4410,7 +4410,7 @@ class ArrayExprLowering {
       for (std::remove_const_t<decltype(loopDepth)> i = 0; i + 1 < loopDepth;
            ++i) {
         builder.setInsertionPointToEnd(loops[i].getBody());
-        builder.create<fir::ResultOp>(loc, loops[i + 1].getResult(0));
+        fir::ResultOp::create(builder, loc, loops[i + 1].getResult(0));
       }
 
     // Move insertion point to the start of the innermost loop in the nest.
@@ -4468,21 +4468,21 @@ class ArrayExprLowering {
            implicitSpace->getMasks()) {
         const std::size_t size = maskExprs.size() - 1;
         auto genFalseBlock = [&](const auto *e, auto &&cond) {
-          auto ifOp = builder.create<fir::IfOp>(
+          auto ifOp = fir::IfOp::create(builder,
               loc, mlir::TypeRange{innerArg.getType()}, fir::getBase(cond),
               /*withElseRegion=*/true);
-          builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
+          fir::ResultOp::create(builder, loc, ifOp.getResult(0));
           builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-          builder.create<fir::ResultOp>(loc, innerArg);
+          fir::ResultOp::create(builder, loc, innerArg);
           builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
         };
         auto genTrueBlock = [&](const auto *e, auto &&cond) {
-          auto ifOp = builder.create<fir::IfOp>(
+          auto ifOp = fir::IfOp::create(builder,
               loc, mlir::TypeRange{innerArg.getType()}, fir::getBase(cond),
               /*withElseRegion=*/true);
-          builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
+          fir::ResultOp::create(builder, loc, ifOp.getResult(0));
           builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-          builder.create<fir::ResultOp>(loc, innerArg);
+          fir::ResultOp::create(builder, loc, innerArg);
           builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         };
         for (std::remove_const_t<decltype(size)> i = 0; i < size; ++i)
@@ -4525,14 +4525,14 @@ class ArrayExprLowering {
                 "allocatable members");
     mlir::Value temp =
         !seqTy.hasDynamicExtents()
-            ? builder.create<fir::AllocMemOp>(loc, type)
-            : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
+            ? fir::AllocMemOp::create(builder, loc, type)
+            : fir::AllocMemOp::create(builder, loc, type, ".array.expr",
                                               mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
-    return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
+    return fir::ArrayLoadOp::create(builder, loc, seqTy, temp, shapeOp,
                                             /*slice=*/mlir::Value{},
                                             mlir::ValueRange{});
   }
@@ -4543,7 +4543,7 @@ class ArrayExprLowering {
     llvm::SmallVector<mlir::Value> idxShape;
     for (auto s : shape)
       idxShape.push_back(builder.createConvert(loc, idxTy, s));
-    return builder.create<fir::ShapeOp>(loc, idxShape);
+    return fir::ShapeOp::create(builder, loc, idxShape);
   }
 
   fir::ShapeOp genShapeOp(llvm::ArrayRef<mlir::Value> shape) {
@@ -4791,7 +4791,7 @@ class ArrayExprLowering {
       mlir::Type argTy = callSiteType.getInput(arg.firArgument);
       if (!actual) {
         // Optional dummy argument for which there is no actual argument.
-        auto absent = builder.create<fir::AbsentOp>(loc, argTy);
+        auto absent = fir::AbsentOp::create(builder, loc, argTy);
         operands.emplace_back([=](IterSpace) { return absent; });
         continue;
       }
@@ -4824,7 +4824,7 @@ class ArrayExprLowering {
               builder.createTemporary(loc, val.getType(),
                                       llvm::ArrayRef<mlir::NamedAttribute>{
                                           fir::getAdaptToByRefAttr(builder)});
-          builder.create<fir::StoreOp>(loc, val, temp);
+          fir::StoreOp::create(builder, loc, val, temp);
           operands.emplace_back(
               [=](IterSpace iters) -> ExtValue { return temp; });
         }
@@ -4883,12 +4883,12 @@ class ArrayExprLowering {
                 fir::dyn_cast_ptrOrBoxEleTy(fir::getBase(exv).getType());
             mlir::Type innerTy = fir::unwrapSequenceType(baseTy);
             operands.emplace_back([=](IterSpace iters) -> ExtValue {
-              mlir::Value coord = builder.create<fir::CoordinateOp>(
+              mlir::Value coord = fir::CoordinateOp::create(builder,
                   loc, fir::ReferenceType::get(innerTy), fir::getBase(exv),
                   iters.iterVec());
               mlir::Value empty;
               mlir::ValueRange emptyRange;
-              return builder.create<fir::EmboxOp>(
+              return fir::EmboxOp::create(builder,
                   loc, fir::ClassType::get(innerTy), coord, empty, empty,
                   emptyRange, sourceBox);
             });
@@ -4903,7 +4903,7 @@ class ArrayExprLowering {
               operands.emplace_back([=](IterSpace iters) -> ExtValue {
                 mlir::Value empty;
                 mlir::ValueRange emptyRange;
-                return builder.create<fir::EmboxOp>(
+                return fir::EmboxOp::create(builder,
                     loc, fir::ClassType::get(baseTy), fir::getBase(exv), empty,
                     empty, emptyRange);
               });
@@ -5098,7 +5098,7 @@ class ArrayExprLowering {
         [&](const fir::CharBoxValue &cb) -> ExtValue {
           mlir::Value len = cb.getLen();
           auto mem =
-              builder.create<fir::AllocaOp>(loc, toType, mlir::ValueRange{len});
+              fir::AllocaOp::create(builder, loc, toType, mlir::ValueRange{len});
           fir::CharBoxValue result(mem, len);
           fir::factory::CharacterExprHelper{builder, loc}.createAssign(
               ExtValue{result}, exv);
@@ -5153,7 +5153,7 @@ class ArrayExprLowering {
       auto val = f(iters);
       mlir::Value base = fir::getBase(val);
       auto newBase =
-          builder.create<fir::NoReassocOp>(loc, base.getType(), base);
+          fir::NoReassocOp::create(builder, loc, base.getType(), base);
       return fir::substBase(val, newBase);
     };
   }
@@ -5170,10 +5170,10 @@ class ArrayExprLowering {
       if constexpr (CAT == Fortran::common::TypeCategory::Unsigned) {
         mlir::Value signless = builder.createConvert(loc, ty, val);
         mlir::Value neg =
-            builder.create<mlir::arith::SubIOp>(loc, zero, signless);
+            mlir::arith::SubIOp::create(builder, loc, zero, signless);
         return builder.createConvert(loc, val.getType(), neg);
       }
-      return builder.create<mlir::arith::SubIOp>(loc, zero, val);
+      return mlir::arith::SubIOp::create(builder, loc, zero, val);
     };
   }
   template <int KIND>
@@ -5192,7 +5192,7 @@ class ArrayExprLowering {
     mlir::Location loc = getLoc();
     auto f = genarr(x.left());
     return [=](IterSpace iters) -> ExtValue {
-      return builder.create<mlir::arith::NegFOp>(loc, fir::getBase(f(iters)));
+      return mlir::arith::NegFOp::create(builder, loc, fir::getBase(f(iters)));
     };
   }
   template <int KIND>
@@ -5201,7 +5201,7 @@ class ArrayExprLowering {
     mlir::Location loc = getLoc();
     auto f = genarr(x.left());
     return [=](IterSpace iters) -> ExtValue {
-      return builder.create<fir::NegcOp>(loc, fir::getBase(f(iters)));
+      return fir::NegcOp::create(builder, loc, fir::getBase(f(iters)));
     };
   }
 
@@ -5426,8 +5426,8 @@ class ArrayExprLowering {
     mlir::Location loc = getLoc();
     mlir::Value lb = getLBound(x, dim, one);
     mlir::Value extent = fir::factory::readExtent(builder, loc, x, dim);
-    auto add = builder.create<mlir::arith::AddIOp>(loc, lb, extent);
-    return builder.create<mlir::arith::SubIOp>(loc, add, one);
+    auto add = mlir::arith::AddIOp::create(builder, loc, lb, extent);
+    return mlir::arith::SubIOp::create(builder, loc, add, one);
   }
 
   /// Return the extent of the boxed array `x` in dimesion `dim`.
@@ -5469,10 +5469,10 @@ class ArrayExprLowering {
     if (destShape[0] != savedDestShape[dim]) {
       // Not the same, so choose the smaller value.
       mlir::Location loc = getLoc();
-      auto cmp = builder.create<mlir::arith::CmpIOp>(
+      auto cmp = mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::sgt, destShape[0],
           savedDestShape[dim]);
-      auto sel = builder.create<mlir::arith::SelectOp>(
+      auto sel = mlir::arith::SelectOp::create(builder,
           loc, cmp, savedDestShape[dim], destShape[0]);
       savedDestShape[dim] = sel;
       destShape = savedDestShape;
@@ -5549,12 +5549,12 @@ class ArrayExprLowering {
                     // FIXME: must use the lower bound of this component.
                     auto arrLowerBound =
                         atBase ? getLBound(arrayExv, subsIndex, one) : one;
-                    auto initial = builder.create<mlir::arith::SubIOp>(
+                    auto initial = mlir::arith::SubIOp::create(builder,
                         loc, lowerBound, arrLowerBound);
-                    auto prod = builder.create<mlir::arith::MulIOp>(
+                    auto prod = mlir::arith::MulIOp::create(builder,
                         loc, impliedIter, stride);
                     auto result =
-                        builder.create<mlir::arith::AddIOp>(loc, initial, prod);
+                        mlir::arith::AddIOp::create(builder, loc, initial, prod);
                     newIters.setIndexValue(subsIndex, result);
                     return newIters;
                   };
@@ -5588,7 +5588,7 @@ class ArrayExprLowering {
                     // using the base array's lower bound value.
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
-                    auto origin = builder.create<mlir::arith::SubIOp>(
+                    auto origin = mlir::arith::SubIOp::create(builder,
                         loc, idxTy, val, lb);
                     newIters.setIndexValue(subsIndex, origin);
                     return newIters;
@@ -5596,7 +5596,7 @@ class ArrayExprLowering {
                   if (useTripsForSlice) {
                     LLVM_ATTRIBUTE_UNUSED auto vectorSubscriptShape =
                         getShape(arrayOperands.back());
-                    auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+                    auto undef = fir::UndefOp::create(builder, loc, idxTy);
                     trips.push_back(undef);
                     trips.push_back(undef);
                     trips.push_back(undef);
@@ -5614,7 +5614,7 @@ class ArrayExprLowering {
                     // the array's declared rank.
                     mlir::Value v = fir::getBase(asScalarArray(e));
                     trips.push_back(v);
-                    auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+                    auto undef = fir::UndefOp::create(builder, loc, idxTy);
                     trips.push_back(undef);
                     trips.push_back(undef);
                     auto currentPC = pc;
@@ -5624,7 +5624,7 @@ class ArrayExprLowering {
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
                     mlir::Value ivAdj =
-                        builder.create<mlir::arith::SubIOp>(loc, idxTy, iv, lb);
+                        mlir::arith::SubIOp::create(builder, loc, idxTy, iv, lb);
                     // Add lbound adjusted value of `e` to the iteration vector
                     // (except when creating a box because the iteration vector
                     // is empty).
@@ -5641,7 +5641,7 @@ class ArrayExprLowering {
                         builder.createConvert(loc, idxTy, newValue);
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
-                    result = builder.create<mlir::arith::SubIOp>(loc, idxTy,
+                    result = mlir::arith::SubIOp::create(builder, loc, idxTy,
                                                                  result, lb);
                     pc = [=](IterSpace iters) {
                       IterationSpace newIters = currentPC(iters);
@@ -5671,7 +5671,7 @@ class ArrayExprLowering {
     auto seqType = mlir::cast<fir::SequenceType>(ty);
     for (auto extent : seqType.getShape()) {
       auto v = extent == fir::SequenceType::getUnknownExtent()
-                   ? builder.create<fir::UndefOp>(loc, idxTy).getResult()
+                   ? fir::UndefOp::create(builder, loc, idxTy).getResult()
                    : builder.createIntegerConstant(loc, idxTy, extent);
       result.push_back(v);
     }
@@ -5764,18 +5764,18 @@ class ArrayExprLowering {
         mlir::Value one =
             builder.createIntegerConstant(loc, substringBounds[0].getType(), 1);
         substringBounds[0] =
-            builder.create<mlir::arith::SubIOp>(loc, substringBounds[0], one);
+            mlir::arith::SubIOp::create(builder, loc, substringBounds[0], one);
         // Convert the upper bound to a length.
         mlir::Value cast = builder.createConvert(loc, iTy, substringBounds[1]);
         mlir::Value zero = builder.createIntegerConstant(loc, iTy, 0);
         auto size =
-            builder.create<mlir::arith::SubIOp>(loc, cast, substringBounds[0]);
-        auto cmp = builder.create<mlir::arith::CmpIOp>(
+            mlir::arith::SubIOp::create(builder, loc, cast, substringBounds[0]);
+        auto cmp = mlir::arith::CmpIOp::create(builder,
             loc, mlir::arith::CmpIPredicate::sgt, size, zero);
         // size = MAX(upper - (lower - 1), 0)
         substringBounds[1] =
-            builder.create<mlir::arith::SelectOp>(loc, cmp, size, zero);
-        slice = builder.create<fir::SliceOp>(
+            mlir::arith::SelectOp::create(builder, loc, cmp, size, zero);
+        slice = fir::SliceOp::create(builder,
             loc, padSlice(components.trips, shape), components.suffixComponents,
             substringBounds);
       } else {
@@ -5827,7 +5827,7 @@ class ArrayExprLowering {
       }
       mlir::Value embox =
           mlir::isa<fir::BaseBoxType>(memref.getType())
-              ? builder.create<fir::ReboxOp>(loc, boxTy, memref, shape, slice)
+              ? fir::ReboxOp::create(builder, loc, boxTy, memref, shape, slice)
                     .getResult()
               : builder
                     .create<fir::EmboxOp>(loc, boxTy, memref, shape, slice,
@@ -5849,7 +5849,7 @@ class ArrayExprLowering {
         // ArrayCoorOp does not expect zero based indices.
         llvm::SmallVector<mlir::Value> indices = fir::factory::originateIndices(
             loc, builder, memref.getType(), shape, iters.iterVec());
-        mlir::Value coor = builder.create<fir::ArrayCoorOp>(
+        mlir::Value coor = fir::ArrayCoorOp::create(builder,
             loc, refEleTy, memref, shape, slice, indices,
             fir::getTypeParams(extMemref));
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
@@ -5869,15 +5869,15 @@ class ArrayExprLowering {
             builder, loc, extMemref, coor, slice);
       };
     }
-    auto arrLoad = builder.create<fir::ArrayLoadOp>(
+    auto arrLoad = fir::ArrayLoadOp::create(builder,
         loc, arrTy, memref, shape, slice, fir::getTypeParams(extMemref));
 
     if (CrayPtr) {
       mlir::Type ptrTy = CrayPtr.getType();
       mlir::Value cnvrt = Fortran::lower::addCrayPointerInst(
           loc, builder, CrayPtr, ptrTy, memref.getType());
-      auto addr = builder.create<fir::LoadOp>(loc, cnvrt);
-      arrLoad = builder.create<fir::ArrayLoadOp>(loc, arrTy, addr, shape, slice,
+      auto addr = fir::LoadOp::create(builder, loc, cnvrt);
+      arrLoad = fir::ArrayLoadOp::create(builder, loc, arrTy, addr, shape, slice,
                                                  fir::getTypeParams(extMemref));
     }
 
@@ -5906,7 +5906,7 @@ class ArrayExprLowering {
         mlir::Type eleTy = fir::applyPathToType(resTy, iters.iterVec());
         mlir::Type refEleTy =
             fir::isa_ref_type(eleTy) ? eleTy : builder.getRefType(eleTy);
-        auto arrModify = builder.create<fir::ArrayModifyOp>(
+        auto arrModify = fir::ArrayModifyOp::create(builder,
             loc, mlir::TypeRange{refEleTy, resTy}, innerArg, iters.iterVec(),
             destination.getTypeparams());
         return abstractArrayExtValue(arrModify.getResult(1));
@@ -5932,17 +5932,17 @@ class ArrayExprLowering {
         mlir::Type eleTy = fir::applyPathToType(arrTy, iters.iterVec());
         if (isAdjustedArrayElementType(eleTy)) {
           mlir::Type eleRefTy = builder.getRefType(eleTy);
-          base = builder.create<fir::ArrayAccessOp>(
+          base = fir::ArrayAccessOp::create(builder,
               loc, eleRefTy, arrLd, iters.iterVec(), arrLdTypeParams);
         } else {
-          base = builder.create<fir::ArrayFetchOp>(
+          base = fir::ArrayFetchOp::create(builder,
               loc, eleTy, arrLd, iters.iterVec(), arrLdTypeParams);
         }
         mlir::Value temp =
             builder.createTemporary(loc, base.getType(),
                                     llvm::ArrayRef<mlir::NamedAttribute>{
                                         fir::getAdaptToByRefAttr(builder)});
-        builder.create<fir::StoreOp>(loc, base, temp);
+        fir::StoreOp::create(builder, loc, base, temp);
         return fir::factory::arraySectionElementToExtendedValue(
             builder, loc, extMemref, temp, slice);
       };
@@ -5953,7 +5953,7 @@ class ArrayExprLowering {
       mlir::Type eleTy = fir::applyPathToType(arrTy, iters.iterVec());
       if (isAdjustedArrayElementType(eleTy)) {
         mlir::Type eleRefTy = builder.getRefType(eleTy);
-        mlir::Value arrayOp = builder.create<fir::ArrayAccessOp>(
+        mlir::Value arrayOp = fir::ArrayAccessOp::create(builder,
             loc, eleRefTy, arrLd, iters.iterVec(), arrLdTypeParams);
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
           llvm::SmallVector<mlir::Value> substringBounds;
@@ -5969,7 +5969,7 @@ class ArrayExprLowering {
         return fir::factory::arraySectionElementToExtendedValue(
             builder, loc, extMemref, arrayOp, slice);
       }
-      auto arrFetch = builder.create<fir::ArrayFetchOp>(
+      auto arrFetch = fir::ArrayFetchOp::create(builder,
           loc, eleTy, arrLd, iters.iterVec(), arrLdTypeParams);
       return fir::factory::arraySectionElementToExtendedValue(
           builder, loc, extMemref, arrFetch, slice);
@@ -6009,7 +6009,7 @@ class ArrayExprLowering {
     mlir::Value memref = fir::getBase(exv);
     mlir::Value shape = builder.createShape(loc, exv);
     mlir::Value noSlice;
-    auto arrLoad = builder.create<fir::ArrayLoadOp>(
+    auto arrLoad = fir::ArrayLoadOp::create(builder,
         loc, arrType, memref, shape, noSlice, fir::getTypeParams(exv));
     mlir::Operation::operand_range arrLdTypeParams = arrLoad.getTypeparams();
     mlir::Value arrLd = arrLoad.getResult();
@@ -6025,7 +6025,7 @@ class ArrayExprLowering {
 
     // By value semantics.
     auto cc = [=](IterSpace iters) -> ExtValue {
-      auto arrFetch = builder.create<fir::ArrayFetchOp>(
+      auto arrFetch = fir::ArrayFetchOp::create(builder,
           loc, eleType, arrLd, iters.iterVec(), arrLdTypeParams);
       return fir::factory::arraySectionElementToExtendedValue(
           builder, loc, exv, arrFetch, noSlice);
@@ -6064,12 +6064,12 @@ class ArrayExprLowering {
               .genIfOp(loc, {eleType}, isPresent,
                        /*withElseRegion=*/true)
               .genThen([&]() {
-                builder.create<fir::ResultOp>(loc, fir::getBase(cc(iters)));
+                fir::ResultOp::create(builder, loc, fir::getBase(cc(iters)));
               })
               .genElse([&]() {
                 mlir::Value zero =
                     fir::factory::createZeroValue(builder, loc, eleType);
-                builder.create<fir::ResultOp>(loc, zero);
+                fir::ResultOp::create(builder, loc, zero);
               })
               .getResults()[0];
       return elementValue;
@@ -6178,7 +6178,7 @@ class ArrayExprLowering {
     mlir::Type eleRefTy = builder.getRefType(eleTy);
     mlir::Type resRefTy = builder.getRefType(resTy);
     mlir::Value nullPtr = builder.createNullConstant(loc, resRefTy);
-    auto offset = builder.create<fir::CoordinateOp>(
+    auto offset = fir::CoordinateOp::create(builder,
         loc, eleRefTy, nullPtr, mlir::ValueRange{multiplier});
     return builder.createConvert(loc, idxTy, offset);
   }
@@ -6193,7 +6193,7 @@ class ArrayExprLowering {
   /// Create a call to the LLVM memcpy intrinsic.
   void createCallMemcpy(llvm::ArrayRef<mlir::Value> args, bool isVolatile) {
     mlir::Location loc = getLoc();
-    builder.create<mlir::LLVM::MemcpyOp>(loc, args[0], args[1], args[2],
+    mlir::LLVM::MemcpyOp::create(builder, loc, args[0], args[1], args[2],
                                          isVolatile);
   }
 
@@ -6204,32 +6204,32 @@ class ArrayExprLowering {
                          mlir::Value eleSz) {
     mlir::Location loc = getLoc();
     mlir::func::FuncOp reallocFunc = fir::factory::getRealloc(builder);
-    auto cond = builder.create<mlir::arith::CmpIOp>(
+    auto cond = mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::sle, bufferSize, needed);
-    auto ifOp = builder.create<fir::IfOp>(loc, mem.getType(), cond,
+    auto ifOp = fir::IfOp::create(builder, loc, mem.getType(), cond,
                                           /*withElseRegion=*/true);
     auto insPt = builder.saveInsertionPoint();
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     // Not enough space, resize the buffer.
     mlir::IndexType idxTy = builder.getIndexType();
     mlir::Value two = builder.createIntegerConstant(loc, idxTy, 2);
-    auto newSz = builder.create<mlir::arith::MulIOp>(loc, needed, two);
-    builder.create<fir::StoreOp>(loc, newSz, buffSize);
-    mlir::Value byteSz = builder.create<mlir::arith::MulIOp>(loc, newSz, eleSz);
+    auto newSz = mlir::arith::MulIOp::create(builder, loc, needed, two);
+    fir::StoreOp::create(builder, loc, newSz, buffSize);
+    mlir::Value byteSz = mlir::arith::MulIOp::create(builder, loc, newSz, eleSz);
     mlir::SymbolRefAttr funcSymAttr =
         builder.getSymbolRefAttr(reallocFunc.getName());
     mlir::FunctionType funcTy = reallocFunc.getFunctionType();
-    auto newMem = builder.create<fir::CallOp>(
+    auto newMem = fir::CallOp::create(builder,
         loc, funcSymAttr, funcTy.getResults(),
         llvm::ArrayRef<mlir::Value>{
             builder.createConvert(loc, funcTy.getInputs()[0], mem),
             builder.createConvert(loc, funcTy.getInputs()[1], byteSz)});
     mlir::Value castNewMem =
         builder.createConvert(loc, mem.getType(), newMem.getResult(0));
-    builder.create<fir::ResultOp>(loc, castNewMem);
+    fir::ResultOp::create(builder, loc, castNewMem);
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
     // Otherwise, just forward the buffer.
-    builder.create<fir::ResultOp>(loc, mem);
+    fir::ResultOp::create(builder, loc, mem);
     builder.restoreInsertionPoint(insPt);
     return ifOp.getResult(0);
   }
@@ -6241,8 +6241,8 @@ class ArrayExprLowering {
                                        mlir::Value eleSz, mlir::Type eleTy,
                                        mlir::Type eleRefTy, mlir::Type resTy) {
     mlir::Location loc = getLoc();
-    auto off = builder.create<fir::LoadOp>(loc, buffPos);
-    auto limit = builder.create<fir::LoadOp>(loc, buffSize);
+    auto off = fir::LoadOp::create(builder, loc, buffPos);
+    auto limit = fir::LoadOp::create(builder, loc, buffSize);
     mlir::IndexType idxTy = builder.getIndexType();
     mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
 
@@ -6260,7 +6260,7 @@ class ArrayExprLowering {
         mlir::Value length = fir::getLen(exv);
         if (!length)
           fir::emitFatalError(loc, "result is not boxed character");
-        eleSz = builder.create<mlir::arith::MulIOp>(loc, bytes, length);
+        eleSz = mlir::arith::MulIOp::create(builder, loc, bytes, length);
       } else {
         TODO(loc, "PDT size");
         // Will call the PDT's size function with the type parameters.
@@ -6279,12 +6279,12 @@ class ArrayExprLowering {
           refTy = builder.getRefType(chTy);
           mlir::Type toTy = builder.getRefType(builder.getVarLenSeqTy(chTy));
           buff = builder.createConvert(loc, toTy, buff);
-          off = builder.create<mlir::arith::MulIOp>(loc, off, eleSz);
+          off = mlir::arith::MulIOp::create(builder, loc, off, eleSz);
         } else {
           TODO(loc, "PDT offset");
         }
       }
-      auto coor = builder.create<fir::CoordinateOp>(loc, refTy, buff,
+      auto coor = fir::CoordinateOp::create(builder, loc, refTy, buff,
                                                     mlir::ValueRange{off});
       return builder.createConvert(loc, eleRefTy, coor);
     };
@@ -6294,15 +6294,15 @@ class ArrayExprLowering {
       // Compute the array size.
       mlir::Value arrSz = one;
       for (auto ext : v.getExtents())
-        arrSz = builder.create<mlir::arith::MulIOp>(loc, arrSz, ext);
+        arrSz = mlir::arith::MulIOp::create(builder, loc, arrSz, ext);
 
       // Grow the buffer as needed.
-      auto endOff = builder.create<mlir::arith::AddIOp>(loc, off, arrSz);
+      auto endOff = mlir::arith::AddIOp::create(builder, loc, off, arrSz);
       mem = growBuffer(mem, endOff, limit, buffSize, eleSz);
 
       // Copy the elements to the buffer.
       mlir::Value byteSz =
-          builder.create<mlir::arith::MulIOp>(loc, arrSz, eleSz);
+          mlir::arith::MulIOp::create(builder, loc, arrSz, eleSz);
       auto buff = builder.createConvert(loc, fir::HeapType::get(resTy), mem);
       mlir::Value buffi = computeCoordinate(buff, off);
       llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
@@ -6311,13 +6311,13 @@ class ArrayExprLowering {
       createCallMemcpy(args, isVolatile);
 
       // Save the incremented buffer position.
-      builder.create<fir::StoreOp>(loc, endOff, buffPos);
+      fir::StoreOp::create(builder, loc, endOff, buffPos);
     };
 
     // Copy a trivial scalar value into the buffer.
     auto doTrivialScalar = [&](const ExtValue &v, mlir::Value len = {}) {
       // Increment the buffer position.
-      auto plusOne = builder.create<mlir::arith::AddIOp>(loc, off, one);
+      auto plusOne = mlir::arith::AddIOp::create(builder, loc, off, one);
 
       // Grow the buffer as needed.
       mem = growBuffer(mem, plusOne, limit, buffSize, eleSz);
@@ -6325,7 +6325,7 @@ class ArrayExprLowering {
       // Store the element in the buffer.
       mlir::Value buff =
           builder.createConvert(loc, fir::HeapType::get(resTy), mem);
-      auto buffi = builder.create<fir::CoordinateOp>(loc, eleRefTy, buff,
+      auto buffi = fir::CoordinateOp::create(builder, loc, eleRefTy, buff,
                                                      mlir::ValueRange{off});
       fir::factory::genScalarAssignment(
           builder, loc,
@@ -6335,7 +6335,7 @@ class ArrayExprLowering {
             return buffi;
           }(),
           v);
-      builder.create<fir::StoreOp>(loc, plusOne, buffPos);
+      fir::StoreOp::create(builder, loc, plusOne, buffPos);
     };
 
     // Copy the value.
@@ -6347,7 +6347,7 @@ class ArrayExprLowering {
             doTrivialScalar(exv, eleSz);
           } else {
             // Increment the buffer position.
-            auto plusOne = builder.create<mlir::arith::AddIOp>(loc, off, one);
+            auto plusOne = mlir::arith::AddIOp::create(builder, loc, off, one);
 
             // Grow the buffer as needed.
             mem = growBuffer(mem, plusOne, limit, buffSize, eleSz);
@@ -6362,7 +6362,7 @@ class ArrayExprLowering {
                 fir::isa_volatile_type(v.getAddr().getType());
             createCallMemcpy(args, isVolatile);
 
-            builder.create<fir::StoreOp>(loc, plusOne, buffPos);
+            fir::StoreOp::create(builder, loc, plusOne, buffPos);
           }
         },
         [&](const fir::ArrayBoxValue &v) { doAbstractArray(v); },
@@ -6403,7 +6403,7 @@ class ArrayExprLowering {
     auto seqTy = mlir::cast<fir::SequenceType>(resTy);
     mlir::Type eleTy = fir::unwrapSequenceType(seqTy);
     auto loop =
-        builder.create<fir::DoLoopOp>(loc, lo, up, step, /*unordered=*/false,
+        fir::DoLoopOp::create(builder, loc, lo, up, step, /*unordered=*/false,
                                       /*finalCount=*/false, mem);
     // create a new binding for x.name(), to ac-do-variable, to the iteration
     // value.
@@ -6434,22 +6434,22 @@ class ArrayExprLowering {
         mlir::Value castLen =
             builder.createConvert(loc, builder.getI64Type(), fir::getLen(exv));
         assert(charLen.has_value());
-        builder.create<fir::StoreOp>(loc, castLen, *charLen);
+        fir::StoreOp::create(builder, loc, castLen, *charLen);
       }
     }
     stmtCtx.finalizeAndPop();
 
-    builder.create<fir::ResultOp>(loc, mem);
+    fir::ResultOp::create(builder, loc, mem);
     builder.restoreInsertionPoint(insPt);
     mem = loop.getResult(0);
     symMap.popImpliedDoBinding();
     llvm::SmallVector<mlir::Value> extents = {
-        builder.create<fir::LoadOp>(loc, buffPos).getResult()};
+        fir::LoadOp::create(builder, loc, buffPos).getResult()};
 
     // Convert to extended value.
     if (fir::isa_char(seqTy.getEleTy())) {
       assert(charLen.has_value());
-      auto len = builder.create<fir::LoadOp>(loc, *charLen);
+      auto len = fir::LoadOp::create(builder, loc, *charLen);
       return {fir::CharArrayBoxValue{mem, len, extents}, /*needCopy=*/false};
     }
     return {fir::ArrayBoxValue{mem, extents}, /*needCopy=*/false};
@@ -6472,7 +6472,7 @@ class ArrayExprLowering {
     mlir::Value buffSize = builder.createTemporary(loc, idxTy, ".buff.size");
     mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
     mlir::Value buffPos = builder.createTemporary(loc, idxTy, ".buff.pos");
-    builder.create<fir::StoreOp>(loc, zero, buffPos);
+    fir::StoreOp::create(builder, loc, zero, buffPos);
     // Allocate space for the array to be constructed.
     mlir::Value mem;
     if (fir::hasDynamicSize(resTy)) {
@@ -6480,22 +6480,22 @@ class ArrayExprLowering {
         // The size of each element may depend on a general expression. Defer
         // creating the buffer until after the expression is evaluated.
         mem = builder.createNullConstant(loc, builder.getRefType(eleTy));
-        builder.create<fir::StoreOp>(loc, zero, buffSize);
+        fir::StoreOp::create(builder, loc, zero, buffSize);
       } else {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
-        mem = builder.create<fir::AllocMemOp>(
+        mem = fir::AllocMemOp::create(builder,
             loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
-        builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
+        fir::StoreOp::create(builder, loc, initBuffSz, buffSize);
       }
     } else {
-      mem = builder.create<fir::AllocMemOp>(loc, resTy);
+      mem = fir::AllocMemOp::create(builder, loc, resTy);
       int64_t buffSz = 1;
       for (auto extent : seqTy.getShape())
         buffSz *= extent;
       mlir::Value initBuffSz =
           builder.createIntegerConstant(loc, idxTy, buffSz);
-      builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
+      fir::StoreOp::create(builder, loc, initBuffSz, buffSize);
     }
     // Compute size of element
     mlir::Type eleRefTy = builder.getRefType(eleTy);
@@ -6517,12 +6517,12 @@ class ArrayExprLowering {
         charLen = builder.createTemporary(loc, builder.getI64Type());
         mlir::Value castLen =
             builder.createConvert(loc, builder.getI64Type(), fir::getLen(exv));
-        builder.create<fir::StoreOp>(loc, castLen, *charLen);
+        fir::StoreOp::create(builder, loc, castLen, *charLen);
       }
     }
     mem = builder.createConvert(loc, fir::HeapType::get(resTy), mem);
     llvm::SmallVector<mlir::Value> extents = {
-        builder.create<fir::LoadOp>(loc, buffPos)};
+        fir::LoadOp::create(builder, loc, buffPos)};
 
     // Cleanup the temporary.
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
@@ -6532,7 +6532,7 @@ class ArrayExprLowering {
     // Return the continuation.
     if (fir::isa_char(seqTy.getEleTy())) {
       if (charLen) {
-        auto len = builder.create<fir::LoadOp>(loc, *charLen);
+        auto len = fir::LoadOp::create(builder, loc, *charLen);
         return genarr(fir::CharArrayBoxValue{mem, len, extents});
       }
       return genarr(fir::CharArrayBoxValue{mem, zero, extents});
@@ -6569,7 +6569,7 @@ class ArrayExprLowering {
     return [=](IterSpace iters) -> ExtValue {
       mlir::Value logical = fir::getBase(lambda(iters));
       mlir::Value val = builder.createConvert(loc, i1Ty, logical);
-      return builder.create<mlir::arith::XOrIOp>(loc, val, truth);
+      return mlir::arith::XOrIOp::create(builder, loc, val, truth);
     };
   }
   template <typename OP, typename A>
@@ -6583,7 +6583,7 @@ class ArrayExprLowering {
       mlir::Value right = fir::getBase(rf(iters));
       mlir::Value lhs = builder.createConvert(loc, i1Ty, left);
       mlir::Value rhs = builder.createConvert(loc, i1Ty, right);
-      return builder.create<OP>(loc, lhs, rhs);
+      return OP::create(builder, loc, lhs, rhs);
     };
   }
   template <typename OP, typename A>
@@ -6597,7 +6597,7 @@ class ArrayExprLowering {
       mlir::Value right = fir::getBase(rf(iters));
       mlir::Value lhs = builder.createConvert(loc, i1Ty, left);
       mlir::Value rhs = builder.createConvert(loc, i1Ty, right);
-      return builder.create<OP>(loc, pred, lhs, rhs);
+      return OP::create(builder, loc, pred, lhs, rhs);
     };
   }
   template <int KIND>
@@ -6637,9 +6637,9 @@ class ArrayExprLowering {
             Fortran::common::TypeCategory::Integer, *unsignedKind);
         mlir::Value lhsSL = builder.createConvert(loc, signlessType, lhs);
         mlir::Value rhsSL = builder.createConvert(loc, signlessType, rhs);
-        return builder.create<OP>(loc, pred, lhsSL, rhsSL);
+        return OP::create(builder, loc, pred, lhsSL, rhsSL);
       }
-      return builder.create<OP>(loc, pred, lhs, rhs);
+      return OP::create(builder, loc, pred, lhs, rhs);
     };
   }
   template <typename A>
@@ -6807,7 +6807,7 @@ class ArrayExprLowering {
                               mlir::Value val = builder.createConvert(
                                   loc, idxTy, subscriptVal);
                               mlir::Value ivAdj =
-                                  builder.create<mlir::arith::SubIOp>(
+                                  mlir::arith::SubIOp::create(builder,
                                       loc, idxTy, val, lb);
                               componentsToAdd.push_back(
                                   builder.createConvert(loc, idxTy, ivAdj));
@@ -6830,7 +6830,7 @@ class ArrayExprLowering {
                     converter.getRecordTypeFieldName(getLastSym(*x));
                 if (auto recTy = mlir::dyn_cast<fir::RecordType>(ty)) {
                   ty = recTy.getType(name);
-                  auto fld = builder.create<fir::FieldIndexOp>(
+                  auto fld = fir::FieldIndexOp::create(builder,
                       loc, fieldTy, name, recTy, fir::getTypeParams(arrayExv));
                   addComponentList(ty, {fld});
                   if (index != revPath.size() - 1 || !isPointerAssignment()) {
@@ -6852,7 +6852,7 @@ class ArrayExprLowering {
                   ty = fir::unwrapRefType(boxTy.getEleTy());
                   auto recTy = mlir::cast<fir::RecordType>(ty);
                   ty = recTy.getType(name);
-                  auto fld = builder.create<fir::FieldIndexOp>(
+                  auto fld = fir::FieldIndexOp::create(builder,
                       loc, fieldTy, name, recTy, fir::getTypeParams(arrayExv));
                   extendComponent(components, ty, {fld});
                 } else {
@@ -6896,7 +6896,7 @@ class ArrayExprLowering {
         mlir::Value innerArg = esp->findArgumentOfLoad(load);
         if (isAdjustedArrayElementType(eleTy)) {
           mlir::Type eleRefTy = builder.getRefType(eleTy);
-          auto arrayOp = builder.create<fir::ArrayAccessOp>(
+          auto arrayOp = fir::ArrayAccessOp::create(builder,
               loc, eleRefTy, innerArg, iters.iterVec(),
               fir::factory::getTypeParams(loc, builder, load));
           if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
@@ -6924,7 +6924,7 @@ class ArrayExprLowering {
           if (!eleBoxTy || !mlir::isa<fir::BoxType>(eleBoxTy))
             TODO(loc, "assignment in a FORALL involving a designator with a "
                       "POINTER or ALLOCATABLE component part-ref");
-          auto arrayOp = builder.create<fir::ArrayAccessOp>(
+          auto arrayOp = fir::ArrayAccessOp::create(builder,
               loc, builder.getRefType(eleBoxTy), innerArg, iters.iterVec(),
               fir::factory::getTypeParams(loc, builder, load));
           mlir::Value addr = components.getExtendCoorRef()(arrayOp);
@@ -6935,18 +6935,18 @@ class ArrayExprLowering {
           if (!isPointerAssignment()) {
             if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(eleTy)) {
               eleTy = fir::boxMemRefType(boxTy);
-              addr = builder.create<fir::BoxAddrOp>(loc, eleTy, addr);
+              addr = fir::BoxAddrOp::create(builder, loc, eleTy, addr);
               eleTy = fir::unwrapRefType(eleTy);
             }
           }
           auto ele = convertElementForUpdate(loc, eleTy, iters.getElement());
-          builder.create<fir::StoreOp>(loc, ele, addr);
-          auto amend = builder.create<fir::ArrayAmendOp>(
+          fir::StoreOp::create(builder, loc, ele, addr);
+          auto amend = fir::ArrayAmendOp::create(builder,
               loc, innerArg.getType(), innerArg, arrayOp);
           return arrayLoadExtValue(builder, loc, load, iters.iterVec(), amend);
         }
         auto ele = convertElementForUpdate(loc, eleTy, iters.getElement());
-        auto update = builder.create<fir::ArrayUpdateOp>(
+        auto update = fir::ArrayUpdateOp::create(builder,
             loc, innerArg.getType(), innerArg, ele, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         return arrayLoadExtValue(builder, loc, load, iters.iterVec(), update);
@@ -6961,7 +6961,7 @@ class ArrayExprLowering {
         mlir::Value innerArg = explicitSpace->findArgumentOfLoad(load);
         mlir::Type refEleTy =
             fir::isa_ref_type(eleTy) ? eleTy : builder.getRefType(eleTy);
-        auto arrModify = builder.create<fir::ArrayModifyOp>(
+        auto arrModify = fir::ArrayModifyOp::create(builder,
             loc, mlir::TypeRange{refEleTy, innerArg.getType()}, innerArg,
             iters.iterVec(), load.getTypeparams());
         return arrayLoadExtValue(builder, loc, load, iters.iterVec(),
@@ -6974,7 +6974,7 @@ class ArrayExprLowering {
           isAdjustedArrayElementType(eleTy)) {
         mlir::Type resTy = builder.getRefType(eleTy);
         // Use array element reference semantics.
-        auto access = builder.create<fir::ArrayAccessOp>(
+        auto access = fir::ArrayAccessOp::create(builder,
             loc, resTy, load, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         mlir::Value newBase = access;
@@ -6997,7 +6997,7 @@ class ArrayExprLowering {
         if (!eleBoxTy || !mlir::isa<fir::BoxType>(eleBoxTy))
           TODO(loc, "assignment in a FORALL involving a designator with a "
                     "POINTER or ALLOCATABLE component part-ref");
-        auto access = builder.create<fir::ArrayAccessOp>(
+        auto access = fir::ArrayAccessOp::create(builder,
             loc, builder.getRefType(eleBoxTy), load, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         mlir::Value addr = components.getExtendCoorRef()(access);
@@ -7010,7 +7010,7 @@ class ArrayExprLowering {
           // Rhs is a regular expression that will need to be boxed before
           // assigning to the boxed variable.
           auto typeParams = fir::factory::getTypeParams(loc, builder, load);
-          auto access = builder.create<fir::ArrayAccessOp>(
+          auto access = fir::ArrayAccessOp::create(builder,
               loc, builder.getRefType(eleTy), load, iters.iterVec(),
               typeParams);
           auto addr = components.getExtendCoorRef()(access);
@@ -7025,13 +7025,13 @@ class ArrayExprLowering {
             TODO(loc, "need to adjust typeparameter(s) to reflect the final "
                       "component");
           mlir::Value embox =
-              builder.create<fir::EmboxOp>(loc, boxTy, ptrAddr,
+              fir::EmboxOp::create(builder, loc, boxTy, ptrAddr,
                                            /*shape=*/mlir::Value{},
                                            /*slice=*/mlir::Value{}, typeParams);
           return arrayLoadExtValue(builder, loc, load, iters.iterVec(), embox);
         }
       }
-      auto fetch = builder.create<fir::ArrayFetchOp>(
+      auto fetch = fir::ArrayFetchOp::create(builder,
           loc, eleTy, load, iters.iterVec(), load.getTypeparams());
       return arrayLoadExtValue(builder, loc, load, iters.iterVec(), fetch);
     };
@@ -7541,13 +7541,13 @@ fir::ExtendedValue Fortran::lower::updateBoxForParentComponent(
 
   if (op) {
     if (auto embox = mlir::dyn_cast<fir::EmboxOp>(op)) {
-      auto newBox = builder.create<fir::EmboxOp>(
+      auto newBox = fir::EmboxOp::create(builder,
           loc, fir::BoxType::get(actualTy), embox.getMemref(), embox.getShape(),
           embox.getSlice(), embox.getTypeparams());
       return fir::substBase(box, newBox);
     }
     if (auto rebox = mlir::dyn_cast<fir::ReboxOp>(op)) {
-      auto newBox = builder.create<fir::ReboxOp>(
+      auto newBox = fir::ReboxOp::create(builder,
           loc, fir::BoxType::get(actualTy), rebox.getBox(), rebox.getShape(),
           rebox.getSlice());
       return fir::substBase(box, newBox);
@@ -7556,7 +7556,7 @@ fir::ExtendedValue Fortran::lower::updateBoxForParentComponent(
 
   mlir::Value empty;
   mlir::ValueRange emptyRange;
-  return builder.create<fir::ReboxOp>(loc, fir::BoxType::get(actualTy), boxBase,
+  return fir::ReboxOp::create(builder, loc, fir::BoxType::get(actualTy), boxBase,
                                       /*shape=*/empty,
                                       /*slice=*/empty);
 }
@@ -7649,7 +7649,7 @@ fir::ArrayLoadOp genArrayLoad(mlir::Location loc,
   mlir::Value addr = fir::getBase(exv);
   mlir::Value shapeOp = builder.createShape(loc, exv);
   mlir::Type arrTy = fir::dyn_cast_ptrOrBoxEleTy(addr.getType());
-  return builder.create<fir::ArrayLoadOp>(loc, arrTy, addr, shapeOp,
+  return fir::ArrayLoadOp::create(builder, loc, arrTy, addr, shapeOp,
                                           /*slice=*/mlir::Value{},
                                           fir::getTypeParams(exv));
 }
@@ -7698,7 +7698,7 @@ void Fortran::lower::createArrayMergeStores(
   for (auto i : llvm::enumerate(esp.getOuterLoop().getResults()))
     if (std::optional<fir::ArrayLoadOp> ldOpt = esp.getLhsLoad(i.index())) {
       fir::ArrayLoadOp load = *ldOpt;
-      builder.create<fir::ArrayMergeStoreOp>(loc, load, i.value(),
+      fir::ArrayMergeStoreOp::create(builder, loc, load, i.value(),
                                              load.getMemref(), load.getSlice(),
                                              load.getTypeparams());
     }
@@ -7722,12 +7722,12 @@ mlir::Value Fortran::lower::addCrayPointerInst(mlir::Location loc,
   mlir::Value empty;
   mlir::ValueRange emptyRange;
   auto boxTy = fir::BoxType::get(ptrTy);
-  auto box = builder.create<fir::EmboxOp>(loc, boxTy, ptrVal, empty, empty,
+  auto box = fir::EmboxOp::create(builder, loc, boxTy, ptrVal, empty, empty,
                                           emptyRange);
   mlir::Value addrof =
       (mlir::isa<fir::ReferenceType>(ptrTy))
-          ? builder.create<fir::BoxAddrOp>(loc, ptrTy, box)
-          : builder.create<fir::BoxAddrOp>(loc, builder.getRefType(ptrTy), box);
+          ? fir::BoxAddrOp::create(builder, loc, ptrTy, box)
+          : fir::BoxAddrOp::create(builder, loc, builder.getRefType(ptrTy), box);
 
   auto refPtrTy =
       builder.getRefType(fir::PointerType::get(fir::dyn_cast_ptrEleTy(pteTy)));
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 7de433d6a201a..a709c7a362e2c 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -167,7 +167,7 @@ class HlfirDesignatorBuilder {
       extents.push_back(builder.createIntegerConstant(loc, idxTy, extent));
     }
     if (!mayHaveNonDefaultLowerBounds(componentSym))
-      return builder.create<fir::ShapeOp>(loc, extents);
+      return fir::ShapeOp::create(builder, loc, extents);
 
     llvm::SmallVector<mlir::Value> lbounds;
     if (const auto *objDetails =
@@ -312,7 +312,7 @@ class HlfirDesignatorBuilder {
     // hlfir.elemental_addr.
     if (auto elementalAddrOp = getVectorSubscriptElementAddrOp())
       builder.setInsertionPointToEnd(&elementalAddrOp->getBody().front());
-    auto designate = builder.create<hlfir::DesignateOp>(
+    auto designate = hlfir::DesignateOp::create(builder,
         getLoc(), designatorType, partInfo.base.value().getBase(),
         partInfo.componentName, partInfo.componentShape, partInfo.subscripts,
         partInfo.substring, partInfo.complexPart, partInfo.resultShape,
@@ -344,7 +344,7 @@ class HlfirDesignatorBuilder {
         mlir::Type refPtrType = builder.getRefType(
             fir::PointerType::get(fir::dyn_cast_ptrEleTy(ptrAddr.getType())));
         mlir::Value cast = builder.createConvert(loc, refPtrType, ptrAddr);
-        mlir::Value ptrVal = builder.create<fir::LoadOp>(loc, cast);
+        mlir::Value ptrVal = fir::LoadOp::create(builder, loc, cast);
 
         // Update the base_addr to the value of the Cray pointer.
         // This is a hacky way to do the update, and it may harm
@@ -442,9 +442,9 @@ class HlfirDesignatorBuilder {
     } else {
       // Compute "len = max(ub-lb+1,0)" (Fortran 2018 9.4.1).
       mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-      auto boundsDiff = builder.create<mlir::arith::SubIOp>(
+      auto boundsDiff = mlir::arith::SubIOp::create(builder,
           loc, partInfo.substring[1], partInfo.substring[0]);
-      auto rawLen = builder.create<mlir::arith::AddIOp>(loc, boundsDiff, one);
+      auto rawLen = mlir::arith::AddIOp::create(builder, loc, boundsDiff, one);
       partInfo.typeParams[0] =
           fir::factory::genMaxWithZero(builder, loc, rawLen);
     }
@@ -803,10 +803,10 @@ class HlfirDesignatorBuilder {
     mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
     oneBasedIndex = builder.createConvert(loc, idxTy, oneBasedIndex);
     mlir::Value zeroBased =
-        builder.create<mlir::arith::SubIOp>(loc, oneBasedIndex, one);
+        mlir::arith::SubIOp::create(builder, loc, oneBasedIndex, one);
     mlir::Value offset =
-        builder.create<mlir::arith::MulIOp>(loc, zeroBased, step);
-    return builder.create<mlir::arith::AddIOp>(loc, lb, offset);
+        mlir::arith::MulIOp::create(builder, loc, zeroBased, step);
+    return mlir::arith::AddIOp::create(builder, loc, lb, offset);
   }
 
   /// Create an hlfir.element_addr operation to deal with vector subscripted
@@ -836,7 +836,7 @@ class HlfirDesignatorBuilder {
     assert(partInfo.base.has_value() &&
            "vector subscripted part must have a base");
     mlir::Value mold = *partInfo.base;
-    auto elementalAddrOp = builder.create<hlfir::ElementalAddrOp>(
+    auto elementalAddrOp = hlfir::ElementalAddrOp::create(builder,
         loc, shape, mold, mlir::ValueRange{},
         /*isUnordered=*/true);
     setVectorSubscriptElementAddrOp(elementalAddrOp);
@@ -881,7 +881,7 @@ class HlfirDesignatorBuilder {
     builder.setInsertionPointToEnd(&elementalAddrOp.getBody().front());
     if (!elementAddr.isPolymorphic())
       elementalAddrOp.getMoldMutable().clear();
-    builder.create<hlfir::YieldOp>(loc, elementAddr);
+    hlfir::YieldOp::create(builder, loc, elementAddr);
     builder.setInsertionPointAfter(elementalAddrOp);
   }
 
@@ -1001,7 +1001,7 @@ HlfirDesignatorBuilder::convertVectorSubscriptedExprToElementalAddr(
     elementalAddrOp.getMoldMutable().clear();
   // Create the hlfir.yield terminator inside the hlfir.elemental_body.
   builder.setInsertionPointToEnd(&elementalAddrOp.getBody().front());
-  builder.create<hlfir::YieldOp>(loc, elementAddrEntity);
+  hlfir::YieldOp::create(builder, loc, elementAddrEntity);
   builder.setInsertionPointAfter(elementalAddrOp);
   // Reset the HlfirDesignatorBuilder state, in case it is used on a new
   // designator.
@@ -1034,7 +1034,7 @@ struct BinaryOp {};
                                                 rhs)};                         \
       } else {                                                                 \
         return hlfir::EntityWithAttributes{                                    \
-            builder.create<GenBinFirOp>(loc, lhs, rhs)};                       \
+            GenBinFirOp::create(builder, loc, lhs, rhs)};                       \
       }                                                                        \
     }                                                                          \
   };
@@ -1075,7 +1075,7 @@ struct BinaryOp<Fortran::evaluate::Divide<
           fir::genDivC(builder, loc, ty, lhs, rhs)};
     } else {
       return hlfir::EntityWithAttributes{
-          builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
+          mlir::complex::DivOp::create(builder, loc, lhs, rhs)};
     }
   }
 };
@@ -1219,7 +1219,7 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = builder.create<mlir::arith::CmpIOp>(
+    auto cmp = mlir::arith::CmpIOp::create(builder,
         loc, translateSignedRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
@@ -1241,7 +1241,7 @@ struct BinaryOp<Fortran::evaluate::Relational<
         mlir::IntegerType::SignednessSemantics::Signless);
     mlir::Value lhsSL = builder.createConvert(loc, signlessType, lhs);
     mlir::Value rhsSL = builder.createConvert(loc, signlessType, rhs);
-    auto cmp = builder.create<mlir::arith::CmpIOp>(
+    auto cmp = mlir::arith::CmpIOp::create(builder,
         loc, translateUnsignedRelational(op.opr), lhsSL, rhsSL);
     return hlfir::EntityWithAttributes{cmp};
   }
@@ -1256,7 +1256,7 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = builder.create<mlir::arith::CmpFOp>(
+    auto cmp = mlir::arith::CmpFOp::create(builder,
         loc, translateFloatRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
@@ -1271,7 +1271,7 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = builder.create<fir::CmpcOp>(
+    auto cmp = fir::CmpcOp::create(builder,
         loc, translateFloatRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
@@ -1313,15 +1313,15 @@ struct BinaryOp<Fortran::evaluate::LogicalOperation<KIND>> {
     switch (op.logicalOperator) {
     case Fortran::evaluate::LogicalOperator::And:
       return hlfir::EntityWithAttributes{
-          builder.create<mlir::arith::AndIOp>(loc, i1Lhs, i1Rhs)};
+          mlir::arith::AndIOp::create(builder, loc, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Or:
       return hlfir::EntityWithAttributes{
-          builder.create<mlir::arith::OrIOp>(loc, i1Lhs, i1Rhs)};
+          mlir::arith::OrIOp::create(builder, loc, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Eqv:
-      return hlfir::EntityWithAttributes{builder.create<mlir::arith::CmpIOp>(
+      return hlfir::EntityWithAttributes{mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::eq, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Neqv:
-      return hlfir::EntityWithAttributes{builder.create<mlir::arith::CmpIOp>(
+      return hlfir::EntityWithAttributes{mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::ne, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Not:
       // lib/evaluate expression for .NOT. is Fortran::evaluate::Not<KIND>.
@@ -1354,7 +1354,7 @@ struct BinaryOp<Fortran::evaluate::SetLength<KIND>> {
     // Fortran 2018 7.4.4.2 point 5.
     mlir::Value safeLength = fir::factory::genMaxWithZero(builder, loc, length);
     return hlfir::EntityWithAttributes{
-        builder.create<hlfir::SetLengthOp>(loc, string, safeLength)};
+        hlfir::SetLengthOp::create(builder, loc, string, safeLength)};
   }
   static void
   genResultTypeParams(mlir::Location, fir::FirOpBuilder &, hlfir::Entity,
@@ -1372,7 +1372,7 @@ struct BinaryOp<Fortran::evaluate::Concat<KIND>> {
                                   hlfir::Entity lhs, hlfir::Entity rhs) {
     assert(len && "genResultTypeParams must have been called");
     auto concat =
-        builder.create<hlfir::ConcatOp>(loc, mlir::ValueRange{lhs, rhs}, len);
+        hlfir::ConcatOp::create(builder, loc, mlir::ValueRange{lhs, rhs}, len);
     return hlfir::EntityWithAttributes{concat.getResult()};
   }
   void
@@ -1386,7 +1386,7 @@ struct BinaryOp<Fortran::evaluate::Concat<KIND>> {
     mlir::Type idxType = builder.getIndexType();
     mlir::Value lhsLen = builder.createConvert(loc, idxType, lengths[0]);
     mlir::Value rhsLen = builder.createConvert(loc, idxType, lengths[1]);
-    len = builder.create<mlir::arith::AddIOp>(loc, lhsLen, rhsLen);
+    len = mlir::arith::AddIOp::create(builder, loc, lhsLen, rhsLen);
     resultTypeParams.push_back(len);
   }
 
@@ -1410,7 +1410,7 @@ struct UnaryOp<Fortran::evaluate::Not<KIND>> {
     mlir::Value one = builder.createBool(loc, true);
     mlir::Value val = builder.createConvert(loc, builder.getI1Type(), lhs);
     return hlfir::EntityWithAttributes{
-        builder.create<mlir::arith::XOrIOp>(loc, val, one)};
+        mlir::arith::XOrIOp::create(builder, loc, val, one)};
   }
 };
 
@@ -1428,7 +1428,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
         /*params=*/{});
     mlir::Value zero = builder.createIntegerConstant(loc, type, 0);
     return hlfir::EntityWithAttributes{
-        builder.create<mlir::arith::SubIOp>(loc, zero, lhs)};
+        mlir::arith::SubIOp::create(builder, loc, zero, lhs)};
   }
 };
 
@@ -1448,7 +1448,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
     mlir::Value zero = builder.createIntegerConstant(loc, signlessType, 0);
     mlir::Value signless = builder.createConvert(loc, signlessType, lhs);
     mlir::Value negated =
-        builder.create<mlir::arith::SubIOp>(loc, zero, signless);
+        mlir::arith::SubIOp::create(builder, loc, zero, signless);
     return hlfir::EntityWithAttributes(
         builder.createConvert(loc, lhs.getType(), negated));
   }
@@ -1463,7 +1463,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
                                          fir::FirOpBuilder &builder, const Op &,
                                          hlfir::Entity lhs) {
     return hlfir::EntityWithAttributes{
-        builder.create<mlir::arith::NegFOp>(loc, lhs)};
+        mlir::arith::NegFOp::create(builder, loc, lhs)};
   }
 };
 
@@ -1475,7 +1475,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
   static hlfir::EntityWithAttributes gen(mlir::Location loc,
                                          fir::FirOpBuilder &builder, const Op &,
                                          hlfir::Entity lhs) {
-    return hlfir::EntityWithAttributes{builder.create<fir::NegcOp>(loc, lhs)};
+    return hlfir::EntityWithAttributes{fir::NegcOp::create(builder, loc, lhs)};
   }
 };
 
@@ -1499,9 +1499,9 @@ struct UnaryOp<Fortran::evaluate::Parentheses<T>> {
                                          const Op &op, hlfir::Entity lhs) {
     if (lhs.isVariable())
       return hlfir::EntityWithAttributes{
-          builder.create<hlfir::AsExprOp>(loc, lhs)};
+          hlfir::AsExprOp::create(builder, loc, lhs)};
     return hlfir::EntityWithAttributes{
-        builder.create<hlfir::NoReassocOp>(loc, lhs.getType(), lhs)};
+        hlfir::NoReassocOp::create(builder, loc, lhs.getType(), lhs)};
   }
 
   static void
@@ -1822,7 +1822,7 @@ class HlfirBuilder {
     // Allocate scalar temporary that will be initialized
     // with the values specified by the constructor.
     mlir::Value storagePtr = builder.createTemporary(loc, recTy);
-    auto varOp = hlfir::EntityWithAttributes{builder.create<hlfir::DeclareOp>(
+    auto varOp = hlfir::EntityWithAttributes{hlfir::DeclareOp::create(builder,
         loc, storagePtr, "ctor.temp", /*shape=*/nullptr,
         /*typeparams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{})};
@@ -1855,7 +1855,7 @@ class HlfirBuilder {
         auto parentCompType = baseRecTy.getType(parentName);
         assert(parentCompType && "failed to retrieve parent component type");
         mlir::Type designatorType = builder.getRefType(parentCompType);
-        mlir::Value newParent = builder.create<hlfir::DesignateOp>(
+        mlir::Value newParent = hlfir::DesignateOp::create(builder,
             loc, designatorType, currentParent, parentName,
             /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
             /*substring=*/mlir::ValueRange{},
@@ -1912,7 +1912,7 @@ class HlfirBuilder {
                                                     extraAttributeFlags);
 
       // Get the component designator.
-      auto lhs = builder.create<hlfir::DesignateOp>(
+      auto lhs = hlfir::DesignateOp::create(builder,
           loc, designatorType, baseOp, name, compShape,
           hlfir::DesignateOp::Subscripts{},
           /*substring=*/mlir::ValueRange{},
@@ -1997,7 +1997,7 @@ class HlfirBuilder {
         auto rhsCastAndCleanup =
             hlfir::genTypeAndKindConvert(loc, builder, rhs, lhs.getType(),
                                          /*preserveLowerBounds=*/allowRealloc);
-        builder.create<hlfir::AssignOp>(loc, rhsCastAndCleanup.first, lhs,
+        hlfir::AssignOp::create(builder, loc, rhsCastAndCleanup.first, lhs,
                                         allowRealloc,
                                         allowRealloc ? keepLhsLength : false,
                                         /*temporary_lhs=*/true);
diff --git a/flang/lib/Lower/ConvertProcedureDesignator.cpp b/flang/lib/Lower/ConvertProcedureDesignator.cpp
index b528544ec245c..e46717070032a 100644
--- a/flang/lib/Lower/ConvertProcedureDesignator.cpp
+++ b/flang/lib/Lower/ConvertProcedureDesignator.cpp
@@ -49,7 +49,7 @@ fir::ExtendedValue Fortran::lower::convertProcedureDesignator(
         fir::getUnrestrictedIntrinsicSymbolRefAttr(builder, loc, genericName,
                                                    signature);
     mlir::Value funcPtr =
-        builder.create<fir::AddrOfOp>(loc, signature, symbolRefAttr);
+        fir::AddrOfOp::create(builder, loc, signature, symbolRefAttr);
     return funcPtr;
   }
   const Fortran::semantics::Symbol *symbol = proc.GetSymbol();
@@ -69,7 +69,7 @@ fir::ExtendedValue Fortran::lower::convertProcedureDesignator(
         Fortran::lower::getOrDeclareFunction(proc, converter);
     mlir::SymbolRefAttr nameAttr = builder.getSymbolRefAttr(func.getSymName());
     funcPtr =
-        builder.create<fir::AddrOfOp>(loc, func.getFunctionType(), nameAttr);
+        fir::AddrOfOp::create(builder, loc, func.getFunctionType(), nameAttr);
   }
   if (Fortran::lower::mustPassLengthWithDummyProcedure(proc, converter)) {
     // The result length, if available here, must be propagated along the
@@ -114,7 +114,7 @@ static hlfir::EntityWithAttributes designateProcedurePointerComponent(
   /// Passed argument may be a descriptor. This is a scalar reference, so the
   /// base address can be directly addressed.
   if (mlir::isa<fir::BaseBoxType>(base.getType()))
-    base = builder.create<fir::BoxAddrOp>(loc, base);
+    base = fir::BoxAddrOp::create(builder, loc, base);
   std::string fieldName = converter.getRecordTypeFieldName(procComponentSym);
   auto recordType =
       mlir::cast<fir::RecordType>(hlfir::getFortranElementType(base.getType()));
@@ -124,7 +124,7 @@ static hlfir::EntityWithAttributes designateProcedurePointerComponent(
   if (!fieldType)
     TODO(loc, "passing type bound procedure (extension)");
   mlir::Type designatorType = fir::ReferenceType::get(fieldType);
-  mlir::Value compRef = builder.create<hlfir::DesignateOp>(
+  mlir::Value compRef = hlfir::DesignateOp::create(builder,
       loc, designatorType, base, fieldName,
       /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
       /*substring=*/mlir::ValueRange{},
@@ -174,10 +174,10 @@ hlfir::EntityWithAttributes Fortran::lower::convertProcedureDesignatorToHLFIR(
     mlir::Type boxTy =
         Fortran::lower::getUntypedBoxProcType(&converter.getMLIRContext());
     if (auto host = Fortran::lower::argumentHostAssocs(converter, funcAddr))
-      funcAddr = builder.create<fir::EmboxProcOp>(
+      funcAddr = fir::EmboxProcOp::create(builder,
           loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
     else
-      funcAddr = builder.create<fir::EmboxProcOp>(loc, boxTy, funcAddr);
+      funcAddr = fir::EmboxProcOp::create(builder, loc, boxTy, funcAddr);
   }
 
   mlir::Value res = procExv.match(
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index cacf4e249aa28..11e0feb889873 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -312,7 +312,7 @@ mlir::Value Fortran::lower::genInitialDataTarget(
   // initial value of the descriptor).
   // Create a fir.rebox to set the attribute correctly, and use targetShift
   // to preserve the target lower bounds if any.
-  return builder.create<fir::ReboxOp>(loc, boxType, targetBox, targetShift,
+  return fir::ReboxOp::create(builder, loc, boxType, targetBox, targetShift,
                                       /*slice=*/mlir::Value{});
 }
 
@@ -362,7 +362,7 @@ static mlir::Value genComponentDefaultInit(
     } else {
       // Component has no initial value. Set its bits to zero by extension
       // to match what is expected because other compilers are doing it.
-      componentValue = builder.create<fir::ZeroOp>(loc, componentTy);
+      componentValue = fir::ZeroOp::create(builder, loc, componentTy);
     }
   } else if (const auto *proc{
                  component
@@ -377,16 +377,16 @@ static mlir::Value genComponentDefaultInit(
         componentValue =
             fir::factory::createNullBoxProc(builder, loc, componentTy);
     } else
-      componentValue = builder.create<fir::ZeroOp>(loc, componentTy);
+      componentValue = fir::ZeroOp::create(builder, loc, componentTy);
   }
   assert(componentValue && "must have been computed");
   componentValue = builder.createConvert(loc, componentTy, componentValue);
   auto fieldTy = fir::FieldType::get(recTy.getContext());
   // FIXME: type parameters must come from the derived-type-spec
-  auto field = builder.create<fir::FieldIndexOp>(
+  auto field = fir::FieldIndexOp::create(builder,
       loc, fieldTy, name, recTy,
       /*typeParams=*/mlir::ValueRange{} /*TODO*/);
-  return builder.create<fir::InsertValueOp>(
+  return fir::InsertValueOp::create(builder,
       loc, recTy, insertInto, componentValue,
       builder.getArrayAttr(field.getAttributes()));
 }
@@ -405,7 +405,7 @@ static mlir::Value genDefaultInitializerValue(
   // Build a scalar default value of the symbol type, looping through the
   // components to build each component initial value.
   auto recTy = mlir::cast<fir::RecordType>(scalarType);
-  mlir::Value initialValue = builder.create<fir::UndefOp>(loc, scalarType);
+  mlir::Value initialValue = fir::UndefOp::create(builder, loc, scalarType);
   const Fortran::semantics::DeclTypeSpec *declTy = sym.GetType();
   assert(declTy && "var with default initialization must have a type");
 
@@ -444,7 +444,7 @@ static mlir::Value genDefaultInitializerValue(
   if (sequenceType) {
     // For arrays, duplicate the scalar value to all elements with an
     // fir.insert_range covering the whole array.
-    auto arrayInitialValue = builder.create<fir::UndefOp>(loc, sequenceType);
+    auto arrayInitialValue = fir::UndefOp::create(builder, loc, sequenceType);
     llvm::SmallVector<int64_t> rangeBounds;
     for (int64_t extent : sequenceType.getShape()) {
       if (extent == fir::SequenceType::getUnknownExtent())
@@ -453,7 +453,7 @@ static mlir::Value genDefaultInitializerValue(
       rangeBounds.push_back(0);
       rangeBounds.push_back(extent - 1);
     }
-    return builder.create<fir::InsertOnRangeOp>(
+    return fir::InsertOnRangeOp::create(builder,
         loc, sequenceType, arrayInitialValue, initialValue,
         builder.getIndexVectorAttr(rangeBounds));
   }
@@ -546,7 +546,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
       createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &b) {
         mlir::Value box =
             Fortran::lower::genInitialDataTarget(converter, loc, symTy, expr);
-        b.create<fir::HasValueOp>(loc, box);
+        fir::HasValueOp::create(b, loc, box);
       });
     } else {
       // Create unallocated/disassociated descriptor if no explicit init
@@ -555,7 +555,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
             b, loc, symTy,
             /*nonDeferredParams=*/{},
             /*typeSourceBox=*/{}, getAllocatorIdxFromDataAttr(dataAttr));
-        b.create<fir::HasValueOp>(loc, box);
+        fir::HasValueOp::create(b, loc, box);
       });
     }
   } else if (const auto *details =
@@ -569,7 +569,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
                 converter, loc, details->init().value(), stmtCtx);
             mlir::Value castTo =
                 builder.createConvert(loc, symTy, fir::getBase(initVal));
-            builder.create<fir::HasValueOp>(loc, castTo);
+            fir::HasValueOp::create(builder, loc, castTo);
           });
     } else if (Fortran::lower::hasDefaultInitialization(sym)) {
       createGlobalInitialization(
@@ -579,7 +579,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
             mlir::Value initVal =
                 genDefaultInitializerValue(converter, loc, sym, symTy, stmtCtx);
             mlir::Value castTo = builder.createConvert(loc, symTy, initVal);
-            builder.create<fir::HasValueOp>(loc, castTo);
+            fir::HasValueOp::create(builder, loc, castTo);
           });
     }
   } else if (Fortran::semantics::IsProcedurePointer(sym)) {
@@ -594,19 +594,19 @@ fir::GlobalOp Fortran::lower::defineGlobal(
               auto box{Fortran::lower::convertProcedureDesignatorInitialTarget(
                   converter, loc, *sym)};
               auto castTo{builder.createConvert(loc, symTy, box)};
-              b.create<fir::HasValueOp>(loc, castTo);
+              fir::HasValueOp::create(b, loc, castTo);
             });
       else { // Has NULL() target.
         createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &b) {
           auto box{fir::factory::createNullBoxProc(b, loc, symTy)};
-          b.create<fir::HasValueOp>(loc, box);
+          fir::HasValueOp::create(b, loc, box);
         });
       }
     } else {
       // No initialization.
       createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &b) {
         auto box{fir::factory::createNullBoxProc(b, loc, symTy)};
-        b.create<fir::HasValueOp>(loc, box);
+        fir::HasValueOp::create(b, loc, box);
       });
     }
   } else if (sym.has<Fortran::semantics::CommonBlockDetails>()) {
@@ -632,10 +632,10 @@ fir::GlobalOp Fortran::lower::defineGlobal(
         builder, global, [&](fir::FirOpBuilder &builder) {
           mlir::Value initValue;
           if (converter.getLoweringOptions().getInitGlobalZero())
-            initValue = builder.create<fir::ZeroOp>(loc, symTy);
+            initValue = fir::ZeroOp::create(builder, loc, symTy);
           else
-            initValue = builder.create<fir::UndefOp>(loc, symTy);
-          builder.create<fir::HasValueOp>(loc, initValue);
+            initValue = fir::UndefOp::create(builder, loc, symTy);
+          fir::HasValueOp::create(builder, loc, initValue);
         });
   }
   // Set public visibility to prevent global definition to be optimized out
@@ -690,7 +690,7 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
                                                         sym);
     global = defineGlobal(converter, var, globalName, linkage, dataAttr);
   }
-  auto addrOf = builder.create<fir::AddrOfOp>(loc, global.resultType(),
+  auto addrOf = fir::AddrOfOp::create(builder, loc, global.resultType(),
                                               global.getSymbol());
   // The type of the global cannot be trusted to be the same as the one
   // of the variable as some existing programs map common blocks to
@@ -752,7 +752,7 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   // would be a waste of space, and incorrect if the pointee is a non dummy
   // assumed-size (possible with cray pointee).
   if (ultimateSymbol.test(Fortran::semantics::Symbol::Flag::CrayPointee))
-    return builder.create<fir::ZeroOp>(loc, fir::ReferenceType::get(ty));
+    return fir::ZeroOp::create(builder, loc, fir::ReferenceType::get(ty));
 
   if (needCUDAAlloc(ultimateSymbol)) {
     cuf::DataAttributeAttr dataAttr =
@@ -767,11 +767,11 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
     for (mlir::Value sh : elidedShape)
       indices.push_back(builder.createConvert(loc, idxTy, sh));
     if (dataAttr.getValue() == cuf::DataAttribute::Shared)
-      return builder.create<cuf::SharedMemoryOp>(loc, ty, nm, symNm, lenParams,
+      return cuf::SharedMemoryOp::create(builder, loc, ty, nm, symNm, lenParams,
                                                  indices);
 
     if (!cuf::isCUDADeviceContext(builder.getRegion()))
-      return builder.create<cuf::AllocOp>(loc, ty, nm, symNm, dataAttr,
+      return cuf::AllocOp::create(builder, loc, ty, nm, symNm, dataAttr,
                                           lenParams, indices);
   }
 
@@ -782,7 +782,7 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   // Local procedure pointer.
   auto res{builder.allocateLocal(loc, ty, nm, symNm, shape, lenParams, isTarg)};
   auto box{fir::factory::createNullBoxProc(builder, loc, ty)};
-  builder.create<fir::StoreOp>(loc, box, res);
+  fir::StoreOp::create(builder, loc, box, res);
   return res;
 }
 
@@ -833,7 +833,7 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
             // Field found in the base record type.
             auto fieldName = recTy.getTypeList()[fieldIdx].first;
             fieldTy = recTy.getTypeList()[fieldIdx].second;
-            mlir::Value fieldIndex = builder.create<fir::FieldIndexOp>(
+            mlir::Value fieldIndex = fir::FieldIndexOp::create(builder,
                 loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
                 recTy,
                 /*typeParams=*/mlir::ValueRange{});
@@ -847,7 +847,7 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
                 fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
                 if (fieldIdx != std::numeric_limits<unsigned>::max()) {
                   mlir::Value parentFieldIndex =
-                      builder.create<fir::FieldIndexOp>(
+                      fir::FieldIndexOp::create(builder,
                           loc, fir::FieldType::get(childRecTy.getContext()),
                           component.first, recTy,
                           /*typeParams=*/mlir::ValueRange{});
@@ -855,7 +855,7 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
                   auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
                   fieldTy = childRecTy.getTypeList()[fieldIdx].second;
                   mlir::Value childFieldIndex =
-                      builder.create<fir::FieldIndexOp>(
+                      fir::FieldIndexOp::create(builder,
                           loc, fir::FieldType::get(fieldTy.getContext()),
                           fieldName, childRecTy,
                           /*typeParams=*/mlir::ValueRange{});
@@ -873,25 +873,25 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
           mlir::Value base = fir::getBase(exv);
           mlir::Value comp;
           if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(base.getType()))) {
-            mlir::Value box = builder.create<fir::LoadOp>(loc, base);
-            mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, box);
+            mlir::Value box = fir::LoadOp::create(builder, loc, base);
+            mlir::Value addr = fir::BoxAddrOp::create(builder, loc, box);
             llvm::SmallVector<mlir::Value> lenParams;
             assert(coordinates.size() == 1 && "expect one coordinate");
             auto field = mlir::dyn_cast<fir::FieldIndexOp>(
                 coordinates[0].getDefiningOp());
-            comp = builder.create<hlfir::DesignateOp>(
+            comp = hlfir::DesignateOp::create(builder,
                 loc, builder.getRefType(fieldTy), addr,
                 /*component=*/field.getFieldName(),
                 /*componentShape=*/mlir::Value{},
                 hlfir::DesignateOp::Subscripts{});
           } else {
-            comp = builder.create<fir::CoordinateOp>(
+            comp = fir::CoordinateOp::create(builder,
                 loc, builder.getRefType(fieldTy), base, coordinates);
           }
           cuf::DataAttributeAttr dataAttr =
               Fortran::lower::translateSymbolCUFDataAttribute(
                   builder.getContext(), sym);
-          builder.create<cuf::SetAllocatorIndexOp>(loc, comp, dataAttr);
+          cuf::SetAllocatorIndexOp::create(builder, loc, comp, dataAttr);
         }
       }
     }
@@ -933,7 +933,7 @@ void Fortran::lower::defaultInitializeAtRuntime(
     // 15.5.2.12 point 3, absent optional dummies are not initialized.
     // Creating descriptor/passing null descriptor to the runtime would
     // create runtime crashes.
-    auto isPresent = builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
+    auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
                                                       fir::getBase(exv));
     builder.genIfThen(loc, isPresent)
         .genThen([&]() {
@@ -976,7 +976,7 @@ void Fortran::lower::defaultInitializeAtRuntime(
                   converter, loc, details->init().value(), stmtCtx);
               mlir::Value castTo =
                   builder.createConvert(loc, symTy, fir::getBase(initVal));
-              builder.create<fir::HasValueOp>(loc, castTo);
+              fir::HasValueOp::create(builder, loc, castTo);
             });
       } else if (!global) {
         global = builder.createGlobal(loc, symTy, globalName, linkage,
@@ -991,12 +991,12 @@ void Fortran::lower::defaultInitializeAtRuntime(
               mlir::Value initVal = genDefaultInitializerValue(
                   converter, loc, sym, symTy, stmtCtx);
               mlir::Value castTo = builder.createConvert(loc, symTy, initVal);
-              builder.create<fir::HasValueOp>(loc, castTo);
+              fir::HasValueOp::create(builder, loc, castTo);
             });
       }
-      auto addrOf = builder.create<fir::AddrOfOp>(loc, global.resultType(),
+      auto addrOf = fir::AddrOfOp::create(builder, loc, global.resultType(),
                                                   global.getSymbol());
-      builder.create<fir::CopyOp>(loc, addrOf, fir::getBase(exv),
+      fir::CopyOp::create(builder, loc, addrOf, fir::getBase(exv),
                                   /*noOverlap=*/true);
     } else {
       mlir::Value box = builder.createBox(loc, exv);
@@ -1097,7 +1097,7 @@ static void finalizeAtRuntime(Fortran::lower::AbstractConverter &converter,
   fir::ExtendedValue exv = converter.getSymbolExtendedValue(sym, &symMap);
   if (Fortran::semantics::IsOptional(sym)) {
     // Only finalize if present.
-    auto isPresent = builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
+    auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
                                                       fir::getBase(exv));
     builder.genIfThen(loc, isPresent)
         .genThen([&]() {
@@ -1144,7 +1144,7 @@ static void deallocateIntentOut(Fortran::lower::AbstractConverter &converter,
       fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
       if (Fortran::semantics::IsOptional(sym)) {
-        auto isPresent = builder.create<fir::IsPresentOp>(
+        auto isPresent = fir::IsPresentOp::create(builder,
             loc, builder.getI1Type(), fir::getBase(extVal));
         builder.genIfThen(loc, isPresent)
             .genThen([&]() {
@@ -1358,7 +1358,7 @@ static fir::GlobalOp defineGlobalAggregateStore(
               Fortran::lower::StatementContext stmtCtx;
               mlir::Value initVal = fir::getBase(genInitializerExprValue(
                   converter, loc, objectDetails->init().value(), stmtCtx));
-              builder.create<fir::HasValueOp>(loc, initVal);
+              fir::HasValueOp::create(builder, loc, initVal);
             });
         return global;
       }
@@ -1367,8 +1367,8 @@ static fir::GlobalOp defineGlobalAggregateStore(
   // of the linkage.
   createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &builder) {
     Fortran::lower::StatementContext stmtCtx;
-    mlir::Value initVal = builder.create<fir::ZeroOp>(loc, aggTy);
-    builder.create<fir::HasValueOp>(loc, initVal);
+    mlir::Value initVal = fir::ZeroOp::create(builder, loc, aggTy);
+    fir::HasValueOp::create(builder, loc, initVal);
   });
   return global;
 }
@@ -1418,7 +1418,7 @@ instantiateAggregateStore(Fortran::lower::AbstractConverter &converter,
       global =
           defineGlobalAggregateStore(converter, aggregate, aggName, linkage);
     }
-    auto addr = builder.create<fir::AddrOfOp>(loc, global.resultType(),
+    auto addr = fir::AddrOfOp::create(builder, loc, global.resultType(),
                                               global.getSymbol());
     auto size = std::get<1>(var.getInterval());
     fir::SequenceType::Shape shape(1, size);
@@ -1466,7 +1466,7 @@ static void instantiateAlias(Fortran::lower::AbstractConverter &converter,
   std::size_t off = sym.GetUltimate().offset() - var.getAliasOffset();
   mlir::Value storeAddr = getAggregateStore(storeMap, var);
   mlir::Value offset = builder.createIntegerConstant(loc, idxTy, off);
-  mlir::Value bytePtr = builder.create<fir::CoordinateOp>(
+  mlir::Value bytePtr = fir::CoordinateOp::create(builder,
       loc, i8Ptr, storeAddr, mlir::ValueRange{offset});
   mlir::Value typedPtr = castAliasToPointer(builder, loc, symType, bytePtr);
   Fortran::lower::StatementContext stmtCtx;
@@ -1656,7 +1656,7 @@ static void finalizeCommonBlockDefinition(
   mlir::TupleType commonTy = mlir::cast<mlir::TupleType>(global.getType());
   auto initFunc = [&](fir::FirOpBuilder &builder) {
     mlir::IndexType idxTy = builder.getIndexType();
-    mlir::Value cb = builder.create<fir::ZeroOp>(loc, commonTy);
+    mlir::Value cb = fir::ZeroOp::create(builder, loc, commonTy);
     unsigned tupIdx = 0;
     std::size_t offset = 0;
     LLVM_DEBUG(llvm::dbgs() << "block {\n");
@@ -1680,7 +1680,7 @@ static void finalizeCommonBlockDefinition(
           mlir::IntegerAttr offVal = builder.getIntegerAttr(idxTy, tupIdx);
           mlir::Value castVal = builder.createConvert(
               loc, commonTy.getType(tupIdx), fir::getBase(initVal));
-          cb = builder.create<fir::InsertValueOp>(loc, commonTy, cb, castVal,
+          cb = fir::InsertValueOp::create(builder, loc, commonTy, cb, castVal,
                                                   builder.getArrayAttr(offVal));
           ++tupIdx;
           offset = mem->offset() + mem->size();
@@ -1688,7 +1688,7 @@ static void finalizeCommonBlockDefinition(
       }
     }
     LLVM_DEBUG(llvm::dbgs() << "}\n");
-    builder.create<fir::HasValueOp>(loc, cb);
+    fir::HasValueOp::create(builder, loc, cb);
   };
   createGlobalInitialization(builder, global, initFunc);
 }
@@ -1722,7 +1722,7 @@ mlir::Value Fortran::lower::genCommonBlockMember(
 
   mlir::Value offs =
       builder.createIntegerConstant(loc, builder.getIndexType(), byteOffset);
-  mlir::Value varAddr = builder.create<fir::CoordinateOp>(
+  mlir::Value varAddr = fir::CoordinateOp::create(builder,
       loc, i8Ptr, base, mlir::ValueRange{offs});
   mlir::Type symType = converter.genType(sym);
 
@@ -1748,7 +1748,7 @@ static void instantiateCommon(Fortran::lower::AbstractConverter &converter,
   if (!commonAddr) {
     // introduce a local AddrOf and add it to the map
     fir::GlobalOp global = getCommonBlockGlobal(converter, common);
-    commonAddr = builder.create<fir::AddrOfOp>(loc, global.resultType(),
+    commonAddr = fir::AddrOfOp::create(builder, loc, global.resultType(),
                                                global.getSymbol());
 
     symMap.addSymbol(common, commonAddr);
@@ -2031,7 +2031,7 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
           Fortran::semantics::GetCUDADataAttr(&sym.GetUltimate()));
 
       // Declare a local pointer variable.
-      auto newBase = builder.create<hlfir::DeclareOp>(
+      auto newBase = hlfir::DeclareOp::create(builder,
           loc, boxAlloc, name, /*shape=*/nullptr, lenParams,
           /*dummy_scope=*/nullptr, attributes);
       mlir::Value nullAddr = builder.createNullConstant(
@@ -2047,9 +2047,9 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
       // Inherit the shape (and maybe length parameters) from the pointee
       // declaration.
       mlir::Value initVal =
-          builder.create<fir::EmboxOp>(loc, ptrBoxType, nullAddr, shapeOrShift,
+          fir::EmboxOp::create(builder, loc, ptrBoxType, nullAddr, shapeOrShift,
                                        /*slice=*/nullptr, lenParams);
-      builder.create<fir::StoreOp>(loc, initVal, newBase.getBase());
+      fir::StoreOp::create(builder, loc, initVal, newBase.getBase());
 
       // Any reference to the pointee is going to be using the pointer
       // box from now on. The base_addr of the descriptor must be updated
@@ -2063,7 +2063,7 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
     mlir::Value dummyScope;
     if (converter.isRegisteredDummySymbol(sym))
       dummyScope = converter.dummyArgsScopeValue();
-    auto newBase = builder.create<hlfir::DeclareOp>(
+    auto newBase = hlfir::DeclareOp::create(builder,
         loc, base, name, shapeOrShift, lenParams, dummyScope, attributes,
         dataAttr);
     symMap.addVariableDefinition(sym, newBase, force);
@@ -2214,7 +2214,7 @@ void Fortran::lower::mapSymbolAttributes(
       // Additional discussion below.
       mlir::Type dummyProcType =
           Fortran::lower::getDummyProcedureType(sym, converter);
-      mlir::Value undefOp = builder.create<fir::UndefOp>(loc, dummyProcType);
+      mlir::Value undefOp = fir::UndefOp::create(builder, loc, dummyProcType);
 
       Fortran::lower::genDeclareSymbol(converter, symMap, sym, undefOp);
     }
@@ -2304,7 +2304,7 @@ void Fortran::lower::mapSymbolAttributes(
           mlir::Type lenType = builder.getCharacterLengthType();
           mlir::Value addr, len;
           if (Fortran::semantics::IsOptional(sym)) {
-            auto isPresent = builder.create<fir::IsPresentOp>(
+            auto isPresent = fir::IsPresentOp::create(builder,
                 loc, builder.getI1Type(), dummyArg);
             auto addrAndLen =
                 builder
@@ -2312,24 +2312,24 @@ void Fortran::lower::mapSymbolAttributes(
                              /*withElseRegion=*/true)
                     .genThen([&]() {
                       mlir::Value readAddr =
-                          builder.create<fir::BoxAddrOp>(loc, refTy, dummyArg);
+                          fir::BoxAddrOp::create(builder, loc, refTy, dummyArg);
                       mlir::Value readLength =
                           charHelp.readLengthFromBox(dummyArg);
-                      builder.create<fir::ResultOp>(
+                      fir::ResultOp::create(builder,
                           loc, mlir::ValueRange{readAddr, readLength});
                     })
                     .genElse([&] {
                       mlir::Value readAddr = builder.genAbsentOp(loc, refTy);
                       mlir::Value readLength =
                           fir::factory::createZeroValue(builder, loc, lenType);
-                      builder.create<fir::ResultOp>(
+                      fir::ResultOp::create(builder,
                           loc, mlir::ValueRange{readAddr, readLength});
                     })
                     .getResults();
             addr = addrAndLen[0];
             len = addrAndLen[1];
           } else {
-            addr = builder.create<fir::BoxAddrOp>(loc, refTy, dummyArg);
+            addr = fir::BoxAddrOp::create(builder, loc, refTy, dummyArg);
             len = charHelp.readLengthFromBox(dummyArg);
           }
           if (!explicitParams.empty())
@@ -2428,7 +2428,7 @@ void Fortran::lower::mapSymbolAttributes(
         mlir::Value dim =
             builder.createIntegerConstant(loc, idxTy, iter.index());
         auto dimInfo =
-            builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, dim);
+            fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, dim);
         shapes.emplace_back(dimInfo.getResult(1));
       } else if (spec->ubound().isStar()) {
         shapes.emplace_back(getAssumedSizeExtent(loc, builder));
@@ -2452,7 +2452,7 @@ void Fortran::lower::mapSymbolAttributes(
         mlir::Value dim =
             builder.createIntegerConstant(loc, idxTy, iter.index());
         dimInfo =
-            builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, dim);
+            fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, dim);
         extents.emplace_back(dimInfo.getResult(1));
         if (auto low = spec->lbound().GetExplicit()) {
           auto expr = Fortran::lower::SomeExpr{*low};
@@ -2501,7 +2501,7 @@ void Fortran::lower::mapSymbolAttributes(
     if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(arg.getType())) {
       // Contiguous assumed shape that can be tracked without a fir.box.
       mlir::Type refTy = builder.getRefType(boxTy.getEleTy());
-      addr = builder.create<fir::BoxAddrOp>(loc, refTy, arg);
+      addr = fir::BoxAddrOp::create(builder, loc, refTy, arg);
     }
 
   // Compute/Extract character length.
@@ -2512,8 +2512,8 @@ void Fortran::lower::mapSymbolAttributes(
         std::tie(addr, len) = charHelp.createUnboxChar(arg);
       } else if (mlir::isa<fir::CharacterType>(arg.getType())) {
         // fir.char<1> passed by value (BIND(C) with VALUE attribute).
-        addr = builder.create<fir::AllocaOp>(loc, arg.getType());
-        builder.create<fir::StoreOp>(loc, arg, addr);
+        addr = fir::AllocaOp::create(builder, loc, arg.getType());
+        fir::StoreOp::create(builder, loc, arg, addr);
       } else if (!addr) {
         addr = arg;
       }
@@ -2583,7 +2583,7 @@ void Fortran::lower::mapSymbolAttributes(
         // Dummy argument passed in register. Place the value in memory at that
         // point since lowering expect symbols to be mapped to memory addresses.
         mlir::Type symType = converter.genType(sym);
-        addr = builder.create<fir::AllocaOp>(loc, symType);
+        addr = fir::AllocaOp::create(builder, loc, symType);
         if (isCptrByVal) {
           // Place the void* address into the CPTR address component.
           mlir::Value addrComponent =
@@ -2803,7 +2803,7 @@ Fortran::lower::genPackArray(Fortran::lower::AbstractConverter &converter,
   mlir::Type elementType = boxType.unwrapInnerType();
   llvm::SmallVector<mlir::Value> elidedLenParams =
       fir::factory::elideLengthsAlreadyInType(elementType, lenParams);
-  auto packOp = builder.create<fir::PackArrayOp>(
+  auto packOp = fir::PackArrayOp::create(builder,
       loc, fir::getBase(exv), stackAlloc, isInnermostMode, noCopy,
       /*max_size=*/mlir::IntegerAttr{},
       /*max_element_size=*/mlir::IntegerAttr{},
@@ -2842,6 +2842,6 @@ void Fortran::lower::genUnpackArray(
   // Avoid copy-out for 'intent(in)' variables.
   bool noCopy = Fortran::semantics::IsIntentIn(sym);
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  builder.create<fir::UnpackArrayOp>(loc, temp, original, stackAlloc, noCopy,
+  fir::UnpackArrayOp::create(builder, loc, temp, original, stackAlloc, noCopy,
                                      getSafeRepackAttrs(converter));
 }
diff --git a/flang/lib/Lower/CustomIntrinsicCall.cpp b/flang/lib/Lower/CustomIntrinsicCall.cpp
index 30c6ce7f53b3f..5040989b1b6c3 100644
--- a/flang/lib/Lower/CustomIntrinsicCall.cpp
+++ b/flang/lib/Lower/CustomIntrinsicCall.cpp
@@ -101,7 +101,7 @@ Fortran::lower::genIntrinsicCall(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value addr = fir::getBase(result);
     if (auto *box = result.getBoxOf<fir::BoxValue>())
       addr =
-          builder.create<fir::BoxAddrOp>(loc, box->getMemTy(), box->getAddr());
+          fir::BoxAddrOp::create(builder, loc, box->getMemTy(), box->getAddr());
     fir::FirOpBuilder *bldr = &builder;
     stmtCtx.attachCleanup([=]() { bldr->create<fir::FreeMemOp>(loc, addr); });
   }
@@ -171,9 +171,9 @@ lowerMinOrMax(fir::FirOpBuilder &builder, mlir::Location loc,
                 args.emplace_back(getOperand(opIndex, loadOperand));
                 fir::ExtendedValue newExtremum = genIntrinsicCall(
                     builder, loc, name, resultType, args, stmtCtx);
-                builder.create<fir::ResultOp>(loc, fir::getBase(newExtremum));
+                fir::ResultOp::create(builder, loc, fir::getBase(newExtremum));
               })
-              .genElse([&]() { builder.create<fir::ResultOp>(loc, extremum); })
+              .genElse([&]() { fir::ResultOp::create(builder, loc, extremum); })
               .getResults()[0];
     } else {
       // Argument is know to be present at compile time.
@@ -235,13 +235,13 @@ lowerIshftc(fir::FirOpBuilder &builder, mlir::Location loc,
             fir::ExtendedValue sizeExv = getOperand(2, loadOperand);
             mlir::Value size =
                 builder.createConvert(loc, resultType, fir::getBase(sizeExv));
-            builder.create<fir::ResultOp>(loc, size);
+            fir::ResultOp::create(builder, loc, size);
           })
           .genElse([&]() {
             mlir::Value bitSize = builder.createIntegerConstant(
                 loc, resultType,
                 mlir::cast<mlir::IntegerType>(resultType).getWidth());
-            builder.create<fir::ResultOp>(loc, bitSize);
+            fir::ResultOp::create(builder, loc, bitSize);
           })
           .getResults()[0]);
   return genIntrinsicCall(builder, loc, name, resultType, args, stmtCtx);
@@ -280,7 +280,7 @@ lowerAssociated(fir::FirOpBuilder &builder, mlir::Location loc,
   // while the optionality of the target pointer/allocatable is what must be
   // checked here.
   mlir::Value isPresent =
-      builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), targetBase);
+      fir::IsPresentOp::create(builder, loc, builder.getI1Type(), targetBase);
   mlir::Type targetType = fir::unwrapRefType(targetBase.getType());
   mlir::Type targetValueType = fir::unwrapPassByRefType(targetType);
   mlir::Type boxType = mlir::isa<fir::BaseBoxType>(targetType)
@@ -293,11 +293,11 @@ lowerAssociated(fir::FirOpBuilder &builder, mlir::Location loc,
           .genThen([&]() {
             mlir::Value box = builder.createBox(loc, targetExv);
             mlir::Value cast = builder.createConvert(loc, boxType, box);
-            builder.create<fir::ResultOp>(loc, cast);
+            fir::ResultOp::create(builder, loc, cast);
           })
           .genElse([&]() {
-            mlir::Value absentBox = builder.create<fir::AbsentOp>(loc, boxType);
-            builder.create<fir::ResultOp>(loc, absentBox);
+            mlir::Value absentBox = fir::AbsentOp::create(builder, loc, boxType);
+            fir::ResultOp::create(builder, loc, absentBox);
           })
           .getResults()[0];
   args.emplace_back(std::move(targetBox));
diff --git a/flang/lib/Lower/HlfirIntrinsics.cpp b/flang/lib/Lower/HlfirIntrinsics.cpp
index 8b96b209ddb00..116742b7cded0 100644
--- a/flang/lib/Lower/HlfirIntrinsics.cpp
+++ b/flang/lib/Lower/HlfirIntrinsics.cpp
@@ -63,7 +63,7 @@ class HlfirTransformationalIntrinsic {
 
   template <typename OP, typename... BUILD_ARGS>
   inline OP createOp(BUILD_ARGS... args) {
-    return builder.create<OP>(loc, args...);
+    return OP::create(builder, loc, args...);
   }
 
   mlir::Value loadBoxAddress(
@@ -195,7 +195,7 @@ mlir::Value HlfirTransformationalIntrinsic::loadBoxAddress(
       // this is a box address type but is not dynamically optional. Just load
       // the box, assuming it is well formed (!fir.ref<!fir.box<...>> ->
       // !fir.box<...>)
-      return builder.create<fir::LoadOp>(loc, actual.getBase());
+      return fir::LoadOp::create(builder, loc, actual.getBase());
     }
     return actual;
   }
@@ -209,8 +209,8 @@ mlir::Value HlfirTransformationalIntrinsic::loadBoxAddress(
   // ensures it won't be.
   mlir::Value box = builder.createBox(loc, exv);
   mlir::Type boxType = box.getType();
-  auto absent = builder.create<fir::AbsentOp>(loc, boxType);
-  auto boxOrAbsent = builder.create<mlir::arith::SelectOp>(
+  auto absent = fir::AbsentOp::create(builder, loc, boxType);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder,
       loc, boxType, isPresent, box, absent);
 
   return boxOrAbsent;
@@ -232,11 +232,11 @@ static mlir::Value loadOptionalValue(
         assert(actual.isScalar() && fir::isa_trivial(eleType) &&
                "must be a numerical or logical scalar");
         hlfir::Entity val = hlfir::loadTrivialScalar(loc, builder, actual);
-        builder.create<fir::ResultOp>(loc, val);
+        fir::ResultOp::create(builder, loc, val);
       })
       .genElse([&]() {
         mlir::Value zero = fir::factory::createZeroValue(builder, loc, eleType);
-        builder.create<fir::ResultOp>(loc, zero);
+        fir::ResultOp::create(builder, loc, zero);
       })
       .getResults()[0];
 }
diff --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp
index 95ea74b791b47..aeaa6e27b99cd 100644
--- a/flang/lib/Lower/HostAssociations.cpp
+++ b/flang/lib/Lower/HostAssociations.cpp
@@ -165,7 +165,7 @@ class CapturedSimpleScalars : public CapturedSymbols<CapturedSimpleScalars> {
     assert(typeInTuple && "addrInTuple must be an address");
     mlir::Value castBox = builder.createConvertWithVolatileCast(
         args.loc, typeInTuple, fir::getBase(args.hostValue));
-    builder.create<fir::StoreOp>(args.loc, castBox, args.addrInTuple);
+    fir::StoreOp::create(builder, args.loc, castBox, args.addrInTuple);
   }
 
   static void getFromTuple(const GetFromTuple &args,
@@ -196,7 +196,7 @@ class CapturedProcedure : public CapturedSymbols<CapturedProcedure> {
     assert(typeInTuple && "addrInTuple must be an address");
     mlir::Value castBox = builder.createConvertWithVolatileCast(
         args.loc, typeInTuple, fir::getBase(args.hostValue));
-    builder.create<fir::StoreOp>(args.loc, castBox, args.addrInTuple);
+    fir::StoreOp::create(builder, args.loc, castBox, args.addrInTuple);
   }
 
   static void getFromTuple(const GetFromTuple &args,
@@ -231,7 +231,7 @@ class CapturedCharacterScalars
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     mlir::Value boxchar = fir::factory::CharacterExprHelper(builder, args.loc)
                               .createEmbox(*charBox);
-    builder.create<fir::StoreOp>(args.loc, boxchar, args.addrInTuple);
+    fir::StoreOp::create(builder, args.loc, boxchar, args.addrInTuple);
   }
 
   static void getFromTuple(const GetFromTuple &args,
@@ -269,20 +269,20 @@ class CapturedPolymorphicScalar
         args.loc, typeInTuple, fir::getBase(args.hostValue));
     if (Fortran::semantics::IsOptional(sym)) {
       auto isPresent =
-          builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), castBox);
+          fir::IsPresentOp::create(builder, loc, builder.getI1Type(), castBox);
       builder.genIfThenElse(loc, isPresent)
           .genThen([&]() {
-            builder.create<fir::StoreOp>(loc, castBox, args.addrInTuple);
+            fir::StoreOp::create(builder, loc, castBox, args.addrInTuple);
           })
           .genElse([&]() {
             mlir::Value null = fir::factory::createUnallocatedBox(
                 builder, loc, typeInTuple,
                 /*nonDeferredParams=*/mlir::ValueRange{});
-            builder.create<fir::StoreOp>(loc, null, args.addrInTuple);
+            fir::StoreOp::create(builder, loc, null, args.addrInTuple);
           })
           .end();
     } else {
-      builder.create<fir::StoreOp>(loc, castBox, args.addrInTuple);
+      fir::StoreOp::create(builder, loc, castBox, args.addrInTuple);
     }
   }
   static void getFromTuple(const GetFromTuple &args,
@@ -297,11 +297,11 @@ class CapturedPolymorphicScalar
       auto eleTy = boxTy.getEleTy();
       if (!fir::isa_ref_type(eleTy))
         eleTy = builder.getRefType(eleTy);
-      auto addr = builder.create<fir::BoxAddrOp>(loc, eleTy, box);
+      auto addr = fir::BoxAddrOp::create(builder, loc, eleTy, box);
       mlir::Value isPresent = builder.genIsNotNullAddr(loc, addr);
-      auto absentBox = builder.create<fir::AbsentOp>(loc, boxTy);
+      auto absentBox = fir::AbsentOp::create(builder, loc, boxTy);
       box =
-          builder.create<mlir::arith::SelectOp>(loc, isPresent, box, absentBox);
+          mlir::arith::SelectOp::create(builder, loc, isPresent, box, absentBox);
     }
     bindCapturedSymbol(sym, box, converter, args.symMap);
   }
@@ -331,7 +331,7 @@ class CapturedAllocatableAndPointer
     assert(typeInTuple && "addrInTuple must be an address");
     mlir::Value castBox = builder.createConvertWithVolatileCast(
         args.loc, typeInTuple, fir::getBase(args.hostValue));
-    builder.create<fir::StoreOp>(args.loc, castBox, args.addrInTuple);
+    fir::StoreOp::create(builder, args.loc, castBox, args.addrInTuple);
   }
   static void getFromTuple(const GetFromTuple &args,
                            Fortran::lower::AbstractConverter &converter,
@@ -404,7 +404,7 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
       // done on present optional. For absent optionals, simply create a
       // disassociated pointer (it is illegal to inquire about lower bounds or
       // lengths of optional according to 15.5.2.12 3 (9) and 10.1.11 2 (7)b).
-      auto isPresent = builder.create<fir::IsPresentOp>(
+      auto isPresent = fir::IsPresentOp::create(builder,
           loc, builder.getI1Type(), fir::getBase(args.hostValue));
       builder.genIfThenElse(loc, isPresent)
           .genThen([&]() {
@@ -441,7 +441,7 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
         const unsigned rank = sym.Rank();
         for (unsigned dim = 0; dim < rank; ++dim) {
           mlir::Value dimVal = builder.createIntegerConstant(loc, idxTy, dim);
-          auto dims = builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy,
+          auto dims = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
                                                      box, dimVal);
           lbounds.emplace_back(dims.getResult(0));
         }
@@ -464,10 +464,10 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
         auto eleTy = boxTy.getEleTy();
         if (!fir::isa_ref_type(eleTy))
           eleTy = builder.getRefType(eleTy);
-        auto addr = builder.create<fir::BoxAddrOp>(loc, eleTy, box);
+        auto addr = fir::BoxAddrOp::create(builder, loc, eleTy, box);
         mlir::Value isPresent = builder.genIsNotNullAddr(loc, addr);
-        auto absentBox = builder.create<fir::AbsentOp>(loc, boxTy);
-        box = builder.create<mlir::arith::SelectOp>(loc, isPresent, box,
+        auto absentBox = fir::AbsentOp::create(builder, loc, boxTy);
+        box = mlir::arith::SelectOp::create(builder, loc, isPresent, box,
                                                     absentBox);
       }
       fir::BoxValue boxValue(box, lbounds, /*explicitParams=*/{});
@@ -540,7 +540,7 @@ static mlir::Value genTupleCoor(fir::FirOpBuilder &builder, mlir::Location loc,
   auto ty = mlir::isa<fir::ReferenceType>(varTy)
                 ? mlir::Type(fir::LLVMPointerType::get(varTy))
                 : mlir::Type(builder.getRefType(varTy));
-  return builder.create<fir::CoordinateOp>(loc, ty, tupleArg, offset);
+  return fir::CoordinateOp::create(builder, loc, ty, tupleArg, offset);
 }
 
 void Fortran::lower::HostAssociations::addSymbolsToBind(
@@ -572,7 +572,7 @@ void Fortran::lower::HostAssociations::hostProcedureBindings(
   mlir::TupleType tupTy = unwrapTupleTy(getArgumentType(converter));
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   mlir::Location loc = converter.getCurrentLocation();
-  auto hostTuple = builder.create<fir::AllocaOp>(loc, tupTy);
+  auto hostTuple = fir::AllocaOp::create(builder, loc, tupTy);
   mlir::IntegerType offTy = builder.getIntegerType(32);
 
   // Walk the list of tupleSymbols and update the pointers in the tuple.
@@ -639,7 +639,7 @@ void Fortran::lower::HostAssociations::internalProcedureBindings(
     mlir::Value off = builder.createIntegerConstant(loc, offTy, s.index());
     mlir::Type varTy = tupTy.getType(s.index());
     mlir::Value eleOff = genTupleCoor(builder, loc, varTy, tupleArg, off);
-    mlir::Value valueInTuple = builder.create<fir::LoadOp>(loc, eleOff);
+    mlir::Value valueInTuple = fir::LoadOp::create(builder, loc, eleOff);
     GetFromTuple getFromTuple{symMap, valueInTuple, loc};
     walkCaptureCategories(getFromTuple, converter, *s.value());
   }
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 53bf61922392d..32ddd0ea9594f 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -153,7 +153,7 @@ static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
   if (csi.ioMsg) {
     mlir::func::FuncOp getIoMsg =
         fir::runtime::getIORuntimeFunc<mkIOKey(GetIoMsg)>(loc, builder);
-    builder.create<fir::CallOp>(
+    fir::CallOp::create(builder,
         loc, getIoMsg,
         mlir::ValueRange{
             cookie,
@@ -164,12 +164,12 @@ static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
   }
   mlir::func::FuncOp endIoStatement =
       fir::runtime::getIORuntimeFunc<mkIOKey(EndIoStatement)>(loc, builder);
-  auto call = builder.create<fir::CallOp>(loc, endIoStatement,
+  auto call = fir::CallOp::create(builder, loc, endIoStatement,
                                           mlir::ValueRange{cookie});
   mlir::Value iostat = call.getResult(0);
   if (csi.bigUnitIfOp) {
     stmtCtx.finalizeAndPop();
-    builder.create<fir::ResultOp>(loc, iostat);
+    fir::ResultOp::create(builder, loc, iostat);
     builder.setInsertionPointAfter(csi.bigUnitIfOp);
     iostat = csi.bigUnitIfOp.getResult(0);
   }
@@ -178,7 +178,7 @@ static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
         fir::getBase(converter.genExprAddr(loc, csi.ioStatExpr, stmtCtx));
     mlir::Value ioStatResult =
         builder.createConvert(loc, converter.genType(*csi.ioStatExpr), iostat);
-    builder.create<fir::StoreOp>(loc, ioStatResult, ioStatVar);
+    fir::StoreOp::create(builder, loc, ioStatResult, ioStatVar);
   }
   return csi.hasTransferConditionSpec() ? iostat : mlir::Value{};
 }
@@ -203,7 +203,7 @@ static void makeNextConditionalOn(fir::FirOpBuilder &builder,
   mlir::IntegerType boolTy = builder.getI1Type();
   if (inLoop)
     resTy = boolTy;
-  auto ifOp = builder.create<fir::IfOp>(loc, resTy, ok,
+  auto ifOp = fir::IfOp::create(builder, loc, resTy, ok,
                                         /*withElseRegion=*/inLoop);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
 }
@@ -261,7 +261,7 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   if (auto table = builder.getNamedGlobal(tableMangleName))
     return builder.createConvert(
         loc, refTy,
-        builder.create<fir::AddrOfOp>(loc, table.resultType(),
+        fir::AddrOfOp::create(builder, loc, table.resultType(),
                                       table.getSymbol()));
 
   mlir::StringAttr linkOnce = builder.createLinkOnceLinkage();
@@ -282,10 +282,10 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   bool tableIsLocal =
       !definedIoProcMap.empty() && hasLocalDefinedIoProc(definedIoProcMap);
   mlir::Value listAddr =
-      tableIsLocal ? builder.create<fir::AllocaOp>(loc, listTy) : mlir::Value{};
+      tableIsLocal ? fir::AllocaOp::create(builder, loc, listTy) : mlir::Value{};
   std::string listMangleName = tableMangleName + ".list";
   auto listFunc = [&](fir::FirOpBuilder &builder) {
-    mlir::Value list = builder.create<fir::UndefOp>(loc, listTy);
+    mlir::Value list = fir::UndefOp::create(builder, loc, listTy);
     mlir::IntegerAttr intAttr[4];
     for (int i = 0; i < 4; ++i)
       intAttr[i] = builder.getIntegerAttr(idxTy, i);
@@ -294,7 +294,7 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
     int n0 = 0, n1;
     auto insert = [&](mlir::Value val) {
       idx[1] = intAttr[n1++];
-      list = builder.create<fir::InsertValueOp>(loc, listTy, list, val,
+      list = fir::InsertValueOp::create(builder, loc, listTy, list, val,
                                                 builder.getArrayAttr(idx));
     };
     for (auto &iface : definedIoProcMap) {
@@ -305,7 +305,7 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       std::string dtName = converter.mangleName(dtSym);
       insert(builder.createConvert(
           loc, refTy,
-          builder.create<fir::AddrOfOp>(
+          fir::AddrOfOp::create(builder,
               loc, fir::ReferenceType::get(converter.genType(dtSym)),
               builder.getSymbolRefAttr(dtName))));
       // defined IO procedure [void (*subroutine)()], may be null
@@ -316,7 +316,7 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
           TODO(loc, "defined IO procedure pointers");
         } else if (Fortran::semantics::IsDummy(*procSym)) {
           Fortran::lower::StatementContext stmtCtx;
-          insert(builder.create<fir::BoxAddrOp>(
+          insert(fir::BoxAddrOp::create(builder,
               loc, refTy,
               fir::getBase(converter.genExprAddr(
                   loc,
@@ -330,7 +330,7 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
               builder.getSymbolRefAttr(procDef.getSymName());
           insert(builder.createConvert(
               loc, refTy,
-              builder.create<fir::AddrOfOp>(loc, procDef.getFunctionType(),
+              fir::AddrOfOp::create(builder, loc, procDef.getFunctionType(),
                                             nameAttr)));
         }
       } else {
@@ -346,9 +346,9 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       insert(builder.createIntegerConstant(loc, byteTy, iface.second.flags));
     }
     if (tableIsLocal)
-      builder.create<fir::StoreOp>(loc, list, listAddr);
+      fir::StoreOp::create(builder, loc, list, listAddr);
     else
-      builder.create<fir::HasValueOp>(loc, list);
+      fir::HasValueOp::create(builder, loc, list);
   };
   if (!definedIoProcMap.empty()) {
     if (tableIsLocal)
@@ -360,12 +360,12 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
 
   // Define the NonTbpDefinedIoTable.
   mlir::Value tableAddr = tableIsLocal
-                              ? builder.create<fir::AllocaOp>(loc, tableTy)
+                              ? fir::AllocaOp::create(builder, loc, tableTy)
                               : mlir::Value{};
   auto tableFunc = [&](fir::FirOpBuilder &builder) {
-    mlir::Value table = builder.create<fir::UndefOp>(loc, tableTy);
+    mlir::Value table = fir::UndefOp::create(builder, loc, tableTy);
     // list item count [std::size_t items]
-    table = builder.create<fir::InsertValueOp>(
+    table = fir::InsertValueOp::create(builder,
         loc, tableTy, table,
         builder.createIntegerConstant(loc, sizeTy, definedIoProcMap.size()),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 0)));
@@ -373,20 +373,20 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
     if (definedIoProcMap.empty())
       listAddr = builder.createNullConstant(loc, builder.getRefType(listTy));
     else if (fir::GlobalOp list = builder.getNamedGlobal(listMangleName))
-      listAddr = builder.create<fir::AddrOfOp>(loc, list.resultType(),
+      listAddr = fir::AddrOfOp::create(builder, loc, list.resultType(),
                                                list.getSymbol());
     assert(listAddr && "missing namelist object list");
-    table = builder.create<fir::InsertValueOp>(
+    table = fir::InsertValueOp::create(builder,
         loc, tableTy, table, listAddr,
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 1)));
     // [bool ignoreNonTbpEntries] conservatively set to true
-    table = builder.create<fir::InsertValueOp>(
+    table = fir::InsertValueOp::create(builder,
         loc, tableTy, table, builder.createIntegerConstant(loc, boolTy, true),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 2)));
     if (tableIsLocal)
-      builder.create<fir::StoreOp>(loc, table, tableAddr);
+      fir::StoreOp::create(builder, loc, table, tableAddr);
     else
-      builder.create<fir::HasValueOp>(loc, table);
+      fir::HasValueOp::create(builder, loc, table);
   };
   if (tableIsLocal) {
     tableFunc(builder);
@@ -394,7 +394,7 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
     fir::GlobalOp table = builder.createGlobal(
         loc, tableTy, tableMangleName,
         /*isConst=*/true, /*isTarget=*/false, tableFunc, linkOnce);
-    tableAddr = builder.create<fir::AddrOfOp>(
+    tableAddr = fir::AddrOfOp::create(builder,
         loc, fir::ReferenceType::get(tableTy), table.getSymbol());
   }
   assert(tableAddr && "missing NonTbpDefinedIo table result");
@@ -420,7 +420,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
   mlir::Location loc = converter.getCurrentLocation();
   std::string groupMangleName = converter.mangleName(symbol);
   if (auto group = builder.getNamedGlobal(groupMangleName))
-    return builder.create<fir::AddrOfOp>(loc, group.resultType(),
+    return fir::AddrOfOp::create(builder, loc, group.resultType(),
                                          group.getSymbol());
 
   const auto &details =
@@ -468,7 +468,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       auto descFunc = [&](fir::FirOpBuilder &b) {
         auto box = Fortran::lower::genInitialDataTarget(
             converter, loc, boxTy, *expr, /*couldBeInEquivalence=*/true);
-        b.create<fir::HasValueOp>(loc, box);
+        fir::HasValueOp::create(b, loc, box);
       };
       builder.createGlobalConstant(loc, boxTy, mangleName, descFunc, linkOnce);
     }
@@ -476,10 +476,10 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
 
   // Define the list of Items.
   mlir::Value listAddr =
-      groupIsLocal ? builder.create<fir::AllocaOp>(loc, listTy) : mlir::Value{};
+      groupIsLocal ? fir::AllocaOp::create(builder, loc, listTy) : mlir::Value{};
   std::string listMangleName = groupMangleName + ".list";
   auto listFunc = [&](fir::FirOpBuilder &builder) {
-    mlir::Value list = builder.create<fir::UndefOp>(loc, listTy);
+    mlir::Value list = fir::UndefOp::create(builder, loc, listTy);
     mlir::IntegerAttr zero = builder.getIntegerAttr(idxTy, 0);
     mlir::IntegerAttr one = builder.getIntegerAttr(idxTy, 1);
     llvm::SmallVector<mlir::Attribute, 2> idx = {mlir::Attribute{},
@@ -490,13 +490,13 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       idx[1] = zero;
       mlir::Value nameAddr =
           builder.createConvert(loc, charRefTy, fir::getBase(stringAddress(s)));
-      list = builder.create<fir::InsertValueOp>(loc, listTy, list, nameAddr,
+      list = fir::InsertValueOp::create(builder, loc, listTy, list, nameAddr,
                                                 builder.getArrayAttr(idx));
       idx[1] = one;
       mlir::Value descAddr;
       if (auto desc = builder.getNamedGlobal(
               Fortran::lower::mangle::globalNamelistDescriptorName(s))) {
-        descAddr = builder.create<fir::AddrOfOp>(loc, desc.resultType(),
+        descAddr = fir::AddrOfOp::create(builder, loc, desc.resultType(),
                                                  desc.getSymbol());
       } else if (Fortran::semantics::FindCommonBlockContaining(s) &&
                  IsAllocatableOrPointer(s)) {
@@ -505,7 +505,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
             Fortran::semantics::FindCommonBlockContaining(s);
         std::string commonBlockName = converter.mangleName(*commonBlockSym);
         fir::GlobalOp commonGlobal = builder.getNamedGlobal(commonBlockName);
-        mlir::Value commonBlockAddr = builder.create<fir::AddrOfOp>(
+        mlir::Value commonBlockAddr = fir::AddrOfOp::create(builder,
             loc, commonGlobal.resultType(), commonGlobal.getSymbol());
         mlir::IntegerType i8Ty = builder.getIntegerType(8);
         mlir::Type i8Ptr = builder.getRefType(i8Ty);
@@ -514,7 +514,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
         std::size_t byteOffset = s.GetUltimate().offset();
         mlir::Value offs = builder.createIntegerConstant(
             loc, builder.getIndexType(), byteOffset);
-        mlir::Value varAddr = builder.create<fir::CoordinateOp>(
+        mlir::Value varAddr = fir::CoordinateOp::create(builder,
             loc, i8Ptr, base, mlir::ValueRange{offs});
         descAddr =
             builder.createConvert(loc, builder.getRefType(symType), varAddr);
@@ -531,13 +531,13 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
                                           /*lbounds=*/{});
       }
       descAddr = builder.createConvert(loc, descRefTy, descAddr);
-      list = builder.create<fir::InsertValueOp>(loc, listTy, list, descAddr,
+      list = fir::InsertValueOp::create(builder, loc, listTy, list, descAddr,
                                                 builder.getArrayAttr(idx));
     }
     if (groupIsLocal)
-      builder.create<fir::StoreOp>(loc, list, listAddr);
+      fir::StoreOp::create(builder, loc, list, listAddr);
     else
-      builder.create<fir::HasValueOp>(loc, list);
+      fir::HasValueOp::create(builder, loc, list);
   };
   if (groupIsLocal)
     listFunc(builder);
@@ -547,39 +547,39 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
 
   // Define the group.
   mlir::Value groupAddr = groupIsLocal
-                              ? builder.create<fir::AllocaOp>(loc, groupTy)
+                              ? fir::AllocaOp::create(builder, loc, groupTy)
                               : mlir::Value{};
   auto groupFunc = [&](fir::FirOpBuilder &builder) {
-    mlir::Value group = builder.create<fir::UndefOp>(loc, groupTy);
+    mlir::Value group = fir::UndefOp::create(builder, loc, groupTy);
     // group name [const char *groupName]
-    group = builder.create<fir::InsertValueOp>(
+    group = fir::InsertValueOp::create(builder,
         loc, groupTy, group,
         builder.createConvert(loc, charRefTy,
                               fir::getBase(stringAddress(symbol))),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 0)));
     // list item count [std::size_t items]
-    group = builder.create<fir::InsertValueOp>(
+    group = fir::InsertValueOp::create(builder,
         loc, groupTy, group,
         builder.createIntegerConstant(loc, sizeTy, details.objects().size()),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 1)));
     // item list [const Item *item]
     if (fir::GlobalOp list = builder.getNamedGlobal(listMangleName))
-      listAddr = builder.create<fir::AddrOfOp>(loc, list.resultType(),
+      listAddr = fir::AddrOfOp::create(builder, loc, list.resultType(),
                                                list.getSymbol());
     assert(listAddr && "missing namelist object list");
-    group = builder.create<fir::InsertValueOp>(
+    group = fir::InsertValueOp::create(builder,
         loc, groupTy, group, listAddr,
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 2)));
     // non-type-bound defined IO procedures
     // [const NonTbpDefinedIoTable *nonTbpDefinedIo]
-    group = builder.create<fir::InsertValueOp>(
+    group = fir::InsertValueOp::create(builder,
         loc, groupTy, group,
         getNonTbpDefinedIoTableAddr(converter, definedIoProcMap),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 3)));
     if (groupIsLocal)
-      builder.create<fir::StoreOp>(loc, group, groupAddr);
+      fir::StoreOp::create(builder, loc, group, groupAddr);
     else
-      builder.create<fir::HasValueOp>(loc, group);
+      fir::HasValueOp::create(builder, loc, group);
   };
   if (groupIsLocal) {
     groupFunc(builder);
@@ -587,7 +587,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
     fir::GlobalOp group = builder.createGlobal(
         loc, groupTy, groupMangleName,
         /*isConst=*/true, /*isTarget=*/false, groupFunc, linkOnce);
-    groupAddr = builder.create<fir::AddrOfOp>(loc, group.resultType(),
+    groupAddr = fir::AddrOfOp::create(builder, loc, group.resultType(),
                                               group.getSymbol());
   }
   assert(groupAddr && "missing namelist group result");
@@ -608,7 +608,7 @@ static void genNamelistIO(Fortran::lower::AbstractConverter &converter,
       getNamelistGroup(converter, symbol.GetUltimate(), stmtCtx);
   groupAddr = builder.createConvert(loc, argType, groupAddr);
   llvm::SmallVector<mlir::Value> args = {cookie, groupAddr};
-  ok = builder.create<fir::CallOp>(loc, funcOp, args).getResult(0);
+  ok = fir::CallOp::create(builder, loc, funcOp, args).getResult(0);
 }
 
 /// Is \p type a derived type or an array of derived type?
@@ -751,7 +751,7 @@ static void genOutputItemList(
         outputFuncArgs.push_back(itemValue);
       }
     }
-    ok = builder.create<fir::CallOp>(loc, outputFunc, outputFuncArgs)
+    ok = fir::CallOp::create(builder, loc, outputFunc, outputFuncArgs)
              .getResult(0);
   }
 }
@@ -812,12 +812,12 @@ static void boolRefToLogical(mlir::Location loc, fir::FirOpBuilder &builder,
                              mlir::Value addr) {
   auto boolType = builder.getRefType(builder.getI1Type());
   auto boolAddr = builder.createConvert(loc, boolType, addr);
-  auto boolValue = builder.create<fir::LoadOp>(loc, boolAddr);
+  auto boolValue = fir::LoadOp::create(builder, loc, boolAddr);
   auto logicalType = fir::unwrapPassByRefType(addr.getType());
   // The convert avoid making any assumptions about how LOGICALs are actually
   // represented (it might end-up being either a signed or zero extension).
   auto logicalValue = builder.createConvert(loc, logicalType, boolValue);
-  builder.create<fir::StoreOp>(loc, logicalValue, addr);
+  fir::StoreOp::create(builder, loc, logicalValue, addr);
 }
 
 static mlir::Value
@@ -849,12 +849,12 @@ createIoRuntimeCallForItem(Fortran::lower::AbstractConverter &converter,
       inputFuncArgs.push_back(builder.createConvert(
           loc, inputFunc.getFunctionType().getInput(2), len));
     } else if (mlir::isa<mlir::IntegerType>(itemTy)) {
-      inputFuncArgs.push_back(builder.create<mlir::arith::ConstantOp>(
+      inputFuncArgs.push_back(mlir::arith::ConstantOp::create(builder,
           loc, builder.getI32IntegerAttr(
                    mlir::cast<mlir::IntegerType>(itemTy).getWidth() / 8)));
     }
   }
-  auto call = builder.create<fir::CallOp>(loc, inputFunc, inputFuncArgs);
+  auto call = fir::CallOp::create(builder, loc, inputFunc, inputFuncArgs);
   auto itemAddr = fir::getBase(item);
   auto itemTy = fir::unwrapRefType(itemAddr.getType());
   if (mlir::isa<fir::LogicalType>(itemTy))
@@ -951,7 +951,7 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   mlir::Value stepValue =
       control.step.has_value()
           ? genControlValue(*control.step)
-          : builder.create<mlir::arith::ConstantIndexOp>(loc, 1);
+          : mlir::arith::ConstantIndexOp::create(builder, loc, 1);
   auto genItemList = [&](const D &ioImpliedDo) {
     if constexpr (std::is_same_v<D, Fortran::parser::InputImpliedDo>)
       genInputItemList(converter, cookie, itemList, isFormatted, checkResult,
@@ -962,35 +962,35 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   };
   if (!checkResult) {
     // No IO call result checks - the loop is a fir.do_loop op.
-    auto doLoopOp = builder.create<fir::DoLoopOp>(
+    auto doLoopOp = fir::DoLoopOp::create(builder,
         loc, lowerValue, upperValue, stepValue, /*unordered=*/false,
         /*finalCountValue=*/true);
     builder.setInsertionPointToStart(doLoopOp.getBody());
     mlir::Value lcv = builder.createConvert(
         loc, fir::unwrapRefType(loopVar.getType()), doLoopOp.getInductionVar());
-    builder.create<fir::StoreOp>(loc, lcv, loopVar);
+    fir::StoreOp::create(builder, loc, lcv, loopVar);
     genItemList(ioImpliedDo);
     builder.setInsertionPointToEnd(doLoopOp.getBody());
-    mlir::Value result = builder.create<mlir::arith::AddIOp>(
+    mlir::Value result = mlir::arith::AddIOp::create(builder,
         loc, doLoopOp.getInductionVar(), doLoopOp.getStep(), iofAttr);
-    builder.create<fir::ResultOp>(loc, result);
+    fir::ResultOp::create(builder, loc, result);
     builder.setInsertionPointAfter(doLoopOp);
     // The loop control variable may be used after the loop.
     lcv = builder.createConvert(loc, fir::unwrapRefType(loopVar.getType()),
                                 doLoopOp.getResult(0));
-    builder.create<fir::StoreOp>(loc, lcv, loopVar);
+    fir::StoreOp::create(builder, loc, lcv, loopVar);
     return;
   }
   // Check IO call results - the loop is a fir.iterate_while op.
   if (!ok)
     ok = builder.createBool(loc, true);
-  auto iterWhileOp = builder.create<fir::IterWhileOp>(
+  auto iterWhileOp = fir::IterWhileOp::create(builder,
       loc, lowerValue, upperValue, stepValue, ok, /*finalCountValue*/ true);
   builder.setInsertionPointToStart(iterWhileOp.getBody());
   mlir::Value lcv =
       builder.createConvert(loc, fir::unwrapRefType(loopVar.getType()),
                             iterWhileOp.getInductionVar());
-  builder.create<fir::StoreOp>(loc, lcv, loopVar);
+  fir::StoreOp::create(builder, loc, lcv, loopVar);
   ok = iterWhileOp.getIterateVar();
   mlir::Value falseValue =
       builder.createIntegerConstant(loc, builder.getI1Type(), 0);
@@ -1003,28 +1003,28 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
     builder.setInsertionPointAfter(lastOp);
     // The primary ifOp result is the result of an IO call or loop.
     if (mlir::isa<fir::CallOp, fir::IfOp>(*lastOp))
-      builder.create<fir::ResultOp>(loc, lastOp->getResult(0));
+      fir::ResultOp::create(builder, loc, lastOp->getResult(0));
     else
-      builder.create<fir::ResultOp>(loc, ok); // loop result
+      fir::ResultOp::create(builder, loc, ok); // loop result
     // The else branch propagates an early exit false result.
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-    builder.create<fir::ResultOp>(loc, falseValue);
+    fir::ResultOp::create(builder, loc, falseValue);
   }
   builder.setInsertionPointToEnd(iterWhileOp.getBody());
   mlir::OpResult iterateResult = builder.getBlock()->back().getResult(0);
   mlir::Value inductionResult0 = iterWhileOp.getInductionVar();
-  auto inductionResult1 = builder.create<mlir::arith::AddIOp>(
+  auto inductionResult1 = mlir::arith::AddIOp::create(builder,
       loc, inductionResult0, iterWhileOp.getStep(), iofAttr);
-  auto inductionResult = builder.create<mlir::arith::SelectOp>(
+  auto inductionResult = mlir::arith::SelectOp::create(builder,
       loc, iterateResult, inductionResult1, inductionResult0);
   llvm::SmallVector<mlir::Value> results = {inductionResult, iterateResult};
-  builder.create<fir::ResultOp>(loc, results);
+  fir::ResultOp::create(builder, loc, results);
   ok = iterWhileOp.getResult(1);
   builder.setInsertionPointAfter(iterWhileOp);
   // The loop control variable may be used after the loop.
   lcv = builder.createConvert(loc, fir::unwrapRefType(loopVar.getType()),
                               iterWhileOp.getResult(0));
-  builder.create<fir::StoreOp>(loc, lcv, loopVar);
+  fir::StoreOp::create(builder, loc, lcv, loopVar);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1046,14 +1046,14 @@ static mlir::Value locToLineNo(Fortran::lower::AbstractConverter &converter,
 
 static mlir::Value getDefaultScratch(fir::FirOpBuilder &builder,
                                      mlir::Location loc, mlir::Type toType) {
-  mlir::Value null = builder.create<mlir::arith::ConstantOp>(
+  mlir::Value null = mlir::arith::ConstantOp::create(builder,
       loc, builder.getI64IntegerAttr(0));
   return builder.createConvert(loc, toType, null);
 }
 
 static mlir::Value getDefaultScratchLen(fir::FirOpBuilder &builder,
                                         mlir::Location loc, mlir::Type toType) {
-  return builder.create<mlir::arith::ConstantOp>(
+  return mlir::arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(toType, 0));
 }
 
@@ -1105,7 +1105,7 @@ lowerStringLit(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
   mlir::Value kind;
   if (ty2) {
     auto kindVal = expr->GetType().value().kind();
-    kind = builder.create<mlir::arith::ConstantOp>(
+    kind = mlir::arith::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(ty2, kindVal));
   }
   return {buff, len, kind};
@@ -1146,7 +1146,7 @@ mlir::Value genIntIOOption(Fortran::lower::AbstractConverter &converter,
       loc, Fortran::semantics::GetExpr(spec.v), localStatementCtx));
   mlir::Value val = builder.createConvert(loc, ioFuncTy.getInput(1), expr);
   llvm::SmallVector<mlir::Value> ioArgs = {cookie, val};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 /// Generic to build a string argument to the runtime. This passes a CHARACTER
@@ -1164,7 +1164,7 @@ mlir::Value genCharIOOption(Fortran::lower::AbstractConverter &converter,
                      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> ioArgs = {cookie, std::get<0>(tup),
                                            std::get<1>(tup)};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 template <typename A>
@@ -1197,7 +1197,7 @@ mlir::Value genIOOption<Fortran::parser::FileNameExpr>(
                      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> ioArgs{cookie, std::get<0>(tup),
                                         std::get<1>(tup)};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 template <>
@@ -1262,7 +1262,7 @@ mlir::Value genIOOption<Fortran::parser::ConnectSpec::CharExpr>(
                      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> ioArgs = {cookie, std::get<0>(tup),
                                            std::get<1>(tup)};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 template <>
@@ -1316,7 +1316,7 @@ mlir::Value genIOOption<Fortran::parser::IoControlSpec::CharExpr>(
                      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> ioArgs = {cookie, std::get<0>(tup),
                                            std::get<1>(tup)};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 template <>
@@ -1352,7 +1352,7 @@ static void genIOGetVar(Fortran::lower::AbstractConverter &converter,
   mlir::func::FuncOp ioFunc =
       fir::runtime::getIORuntimeFunc<IoRuntimeKey>(loc, builder);
   mlir::Value value =
-      builder.create<fir::CallOp>(loc, ioFunc, mlir::ValueRange{cookie})
+      fir::CallOp::create(builder, loc, ioFunc, mlir::ValueRange{cookie})
           .getResult(0);
   Fortran::lower::StatementContext localStatementCtx;
   fir::ExtendedValue var = converter.genExprAddr(
@@ -1480,7 +1480,7 @@ genConditionHandlerCall(Fortran::lower::AbstractConverter &converter,
       fir::runtime::getIORuntimeFunc<mkIOKey(EnableHandlers)>(loc, builder);
   mlir::Type boolType = enableHandlers.getFunctionType().getInput(1);
   auto boolValue = [&](bool specifierIsPresent) {
-    return builder.create<mlir::arith::ConstantOp>(
+    return mlir::arith::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(boolType, specifierIsPresent));
   };
   llvm::SmallVector<mlir::Value> ioArgs = {cookie,
@@ -1489,7 +1489,7 @@ genConditionHandlerCall(Fortran::lower::AbstractConverter &converter,
                                            boolValue(csi.hasEnd),
                                            boolValue(csi.hasEor),
                                            boolValue(csi.ioMsg.has_value())};
-  builder.create<fir::CallOp>(loc, enableHandlers, ioArgs);
+  fir::CallOp::create(builder, loc, enableHandlers, ioArgs);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1663,7 +1663,7 @@ lowerReferenceAsStringSelect(Fortran::lower::AbstractConverter &converter,
     // Pass the format string reference and the string length out of the select
     // statement.
     llvm::SmallVector<mlir::Value> args = {stringRef, stringLen};
-    builder.create<mlir::cf::BranchOp>(loc, endBlock, args);
+    mlir::cf::BranchOp::create(builder, loc, endBlock, args);
 
     // Add block to the list of cases and make a new one.
     blockList.push_back(block);
@@ -1678,13 +1678,13 @@ lowerReferenceAsStringSelect(Fortran::lower::AbstractConverter &converter,
       builder, loc,
       "Assigned format variable '" + symbol->name().ToString() +
           "' has not been assigned a valid format label");
-  builder.create<fir::UnreachableOp>(loc);
+  fir::UnreachableOp::create(builder, loc);
   blockList.push_back(unitBlock);
 
   // Lower the selectOp.
   builder.setInsertionPointToEnd(startBlock);
   auto label = fir::getBase(converter.genExprValue(loc, &expr, stmtCtx));
-  builder.create<fir::SelectOp>(loc, label, indexList, blockList);
+  fir::SelectOp::create(builder, loc, label, indexList, blockList);
 
   builder.setInsertionPointToEnd(endBlock);
   endBlock->addArgument(strTy, loc);
@@ -1814,17 +1814,17 @@ static mlir::Value genIOUnitNumber(Fortran::lower::AbstractConverter &converter,
     mlir::Value line = locToLineNo(converter, loc, funcTy.getInput(5));
     args.push_back(file);
     args.push_back(line);
-    auto checkCall = builder.create<fir::CallOp>(loc, check, args);
+    auto checkCall = fir::CallOp::create(builder, loc, check, args);
     if (csi.hasErrorConditionSpec()) {
       mlir::Value iostat = checkCall.getResult(0);
       mlir::Type iostatTy = iostat.getType();
       mlir::Value zero = fir::factory::createZeroValue(builder, loc, iostatTy);
-      mlir::Value unitIsOK = builder.create<mlir::arith::CmpIOp>(
+      mlir::Value unitIsOK = mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::eq, iostat, zero);
-      auto ifOp = builder.create<fir::IfOp>(loc, iostatTy, unitIsOK,
+      auto ifOp = fir::IfOp::create(builder, loc, iostatTy, unitIsOK,
                                             /*withElseRegion=*/true);
       builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, iostat);
+      fir::ResultOp::create(builder, loc, iostat);
       builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
       stmtCtx.pushScope();
       csi.bigUnitIfOp = ifOp;
@@ -1846,7 +1846,7 @@ static mlir::Value genIOUnit(Fortran::lower::AbstractConverter &converter,
                 &iounit->u))
       return genIOUnitNumber(converter, loc, Fortran::semantics::GetExpr(*e),
                              ty, csi, stmtCtx);
-  return builder.create<mlir::arith::ConstantOp>(
+  return mlir::arith::ConstantOp::create(builder,
       loc, builder.getIntegerAttr(ty, defaultUnitNumber));
 }
 
@@ -1879,7 +1879,7 @@ static mlir::Value genBasicIOStmt(Fortran::lower::AbstractConverter &converter,
   mlir::Value un = builder.createConvert(loc, beginFuncTy.getInput(0), unit);
   mlir::Value file = locToFilename(converter, loc, beginFuncTy.getInput(1));
   mlir::Value line = locToLineNo(converter, loc, beginFuncTy.getInput(2));
-  auto call = builder.create<fir::CallOp>(loc, beginFunc,
+  auto call = fir::CallOp::create(builder, loc, beginFunc,
                                           mlir::ValueRange{un, file, line});
   mlir::Value cookie = call.getResult(0);
   genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
@@ -1934,7 +1934,7 @@ genNewunitSpec(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
       auto kind = builder.createIntegerConstant(loc, ioFuncTy.getInput(2),
                                                 var->GetType().value().kind());
       llvm::SmallVector<mlir::Value> ioArgs = {cookie, addr, kind};
-      return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+      return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
     }
   llvm_unreachable("missing Newunit spec");
 }
@@ -1969,7 +1969,7 @@ Fortran::lower::genOpenStatement(Fortran::lower::AbstractConverter &converter,
     beginArgs.push_back(locToLineNo(converter, loc, beginFuncTy.getInput(1)));
   }
   auto cookie =
-      builder.create<fir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+      fir::CallOp::create(builder, loc, beginFunc, beginArgs).getResult(0);
   genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
   mlir::Value ok;
   auto insertPt = builder.saveInsertionPoint();
@@ -2013,7 +2013,7 @@ Fortran::lower::genWaitStatement(Fortran::lower::AbstractConverter &converter,
     args.push_back(locToFilename(converter, loc, beginFuncTy.getInput(1)));
     args.push_back(locToLineNo(converter, loc, beginFuncTy.getInput(2)));
   }
-  auto cookie = builder.create<fir::CallOp>(loc, beginFunc, args).getResult(0);
+  auto cookie = fir::CallOp::create(builder, loc, beginFunc, args).getResult(0);
   genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
   return genEndIO(converter, converter.getCurrentLocation(), cookie, csi,
                   stmtCtx);
@@ -2149,7 +2149,7 @@ void genBeginDataTransferCallArgs(
     }
   } else { // PRINT - maybe explicit format; default unit
     maybeGetFormatArgs();
-    ioArgs.push_back(builder.create<mlir::arith::ConstantOp>(
+    ioArgs.push_back(mlir::arith::ConstantOp::create(builder,
         loc, builder.getIntegerAttr(ioFuncTy.getInput(ioArgs.size()),
                                     defaultUnitNumber)));
   }
@@ -2198,7 +2198,7 @@ genDataTransferStmt(Fortran::lower::AbstractConverter &converter,
       ioArgs, converter, loc, stmt, ioFunc.getFunctionType(), isFormatted,
       isList || isNml, isInternal, descRef, csi, stmtCtx);
   mlir::Value cookie =
-      builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+      fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 
   auto insertPt = builder.saveInsertionPoint();
   mlir::Value ok;
@@ -2332,7 +2332,7 @@ mlir::Value genInquireSpec<Fortran::parser::InquireSpec::CharVar>(
                                                        .c_str())),
       builder.createConvert(loc, specFuncTy.getInput(2), fir::getBase(str)),
       builder.createConvert(loc, specFuncTy.getInput(3), fir::getLen(str))};
-  return builder.create<fir::CallOp>(loc, specFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, specFunc, args).getResult(0);
 }
 /// Specialization for INTEGER.
 template <>
@@ -2369,7 +2369,7 @@ mlir::Value genInquireSpec<Fortran::parser::InquireSpec::IntVar>(
                                                        .c_str())),
       builder.createConvert(loc, specFuncTy.getInput(2), addr),
       builder.createConvert(loc, specFuncTy.getInput(3), kind)};
-  return builder.create<fir::CallOp>(loc, specFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, specFunc, args).getResult(0);
 }
 /// Specialization for LOGICAL and (PENDING + ID).
 template <>
@@ -2406,7 +2406,7 @@ mlir::Value genInquireSpec<Fortran::parser::InquireSpec::LogVar>(
             Fortran::parser::InquireSpec::LogVar::EnumToString(logVarKind)}
                                                      .c_str())));
   args.push_back(builder.createConvert(loc, specFuncTy.getInput(2), addr));
-  auto call = builder.create<fir::CallOp>(loc, specFunc, args);
+  auto call = fir::CallOp::create(builder, loc, specFunc, args);
   boolRefToLogical(loc, builder, addr);
   return call.getResult(0);
 }
@@ -2502,7 +2502,7 @@ mlir::Value Fortran::lower::genInquireStatement(
     beginArgs = {locToFilename(converter, loc, beginFuncTy.getInput(0)),
                  locToLineNo(converter, loc, beginFuncTy.getInput(1))};
     auto cookie =
-        builder.create<fir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+        fir::CallOp::create(builder, loc, beginFunc, beginArgs).getResult(0);
     mlir::Value ok;
     genOutputItemList(
         converter, cookie,
@@ -2523,14 +2523,14 @@ mlir::Value Fortran::lower::genInquireStatement(
             .getResult(0);
     mlir::Value length1 =
         builder.createConvert(loc, converter.genType(*ioLengthVar), length);
-    builder.create<fir::StoreOp>(loc, length1, ioLengthVarAddr);
+    fir::StoreOp::create(builder, loc, length1, ioLengthVarAddr);
     return genEndIO(converter, loc, cookie, csi, stmtCtx);
   }
 
   // Common handling for inquire by unit or file.
   assert(list && "inquire-spec list must be present");
   auto cookie =
-      builder.create<fir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+      fir::CallOp::create(builder, loc, beginFunc, beginArgs).getResult(0);
   genConditionHandlerCall(converter, loc, cookie, *list, csi);
   // Handle remaining arguments in specifier list.
   mlir::Value ok;
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 51eb33dec186b..13b2d5155bddb 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -132,21 +132,21 @@ createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
                        /*withElseRegion=*/true)
               .genThen([&]() {
                 if (fir::isBoxAddress(baseAddr.getType()))
-                  baseAddr = builder.create<fir::LoadOp>(loc, baseAddr);
+                  baseAddr = fir::LoadOp::create(builder, loc, baseAddr);
                 mlir::Value boxAddr =
-                    builder.create<fir::BoxAddrOp>(loc, baseAddr);
-                builder.create<fir::ResultOp>(loc, mlir::ValueRange{boxAddr});
+                    fir::BoxAddrOp::create(builder, loc, baseAddr);
+                fir::ResultOp::create(builder, loc, mlir::ValueRange{boxAddr});
               })
               .genElse([&] {
                 mlir::Value absent =
-                    builder.create<fir::AbsentOp>(loc, ifRetTy);
-                builder.create<fir::ResultOp>(loc, mlir::ValueRange{absent});
+                    fir::AbsentOp::create(builder, loc, ifRetTy);
+                fir::ResultOp::create(builder, loc, mlir::ValueRange{absent});
               })
               .getResults()[0];
     } else {
       if (fir::isBoxAddress(baseAddr.getType()))
-        baseAddr = builder.create<fir::LoadOp>(loc, baseAddr);
-      baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
+        baseAddr = fir::LoadOp::create(builder, loc, baseAddr);
+      baseAddr = fir::BoxAddrOp::create(builder, loc, baseAddr);
     }
     retTy = baseAddr.getType();
   }
@@ -159,7 +159,7 @@ createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
   addOperands(operands, operandSegments, bounds);
   addOperands(operands, operandSegments, async);
 
-  Op op = builder.create<Op>(loc, retTy, operands);
+  Op op = Op::create(builder, loc, retTy, operands);
   op.setNameAttr(builder.getStringAttr(name.str()));
   op.setStructured(structured);
   op.setImplicit(implicit);
@@ -198,12 +198,12 @@ createDeclareFunc(mlir::OpBuilder &modBuilder, fir::FirOpBuilder &builder,
                   llvm::SmallVector<mlir::Type> argsTy = {},
                   llvm::SmallVector<mlir::Location> locs = {}) {
   auto funcTy = mlir::FunctionType::get(modBuilder.getContext(), argsTy, {});
-  auto funcOp = modBuilder.create<mlir::func::FuncOp>(loc, funcName, funcTy);
+  auto funcOp = mlir::func::FuncOp::create(modBuilder, loc, funcName, funcTy);
   funcOp.setVisibility(mlir::SymbolTable::Visibility::Private);
   builder.createBlock(&funcOp.getRegion(), funcOp.getRegion().end(), argsTy,
                       locs);
   builder.setInsertionPointToEnd(&funcOp.getRegion().back());
-  builder.create<mlir::func::ReturnOp>(loc);
+  mlir::func::ReturnOp::create(builder, loc);
   builder.setInsertionPointToStart(&funcOp.getRegion().back());
   return funcOp;
 }
@@ -214,7 +214,7 @@ createSimpleOp(fir::FirOpBuilder &builder, mlir::Location loc,
                const llvm::SmallVectorImpl<mlir::Value> &operands,
                const llvm::SmallVectorImpl<int32_t> &operandSegments) {
   llvm::ArrayRef<mlir::Type> argTy;
-  Op op = builder.create<Op>(loc, argTy, operands);
+  Op op = Op::create(builder, loc, argTy, operands);
   op->setAttr(Op::getOperandSegmentSizeAttr(),
               builder.getDenseI32ArrayAttr(operandSegments));
   return op;
@@ -257,14 +257,14 @@ static void createDeclareAllocFuncWithArg(mlir::OpBuilder &modBuilder,
 
   if (unwrapFirBox) {
     mlir::Value desc =
-        builder.create<fir::LoadOp>(loc, registerFuncOp.getArgument(0));
-    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, desc);
+        fir::LoadOp::create(builder, loc, registerFuncOp.getArgument(0));
+    fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, desc);
     addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
     EntryOp entryOp = createDataEntryOp<EntryOp>(
         builder, loc, boxAddrOp.getResult(), asFortran, bounds,
         /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-    builder.create<mlir::acc::DeclareEnterOp>(
+    mlir::acc::DeclareEnterOp::create(builder,
         loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
         mlir::ValueRange(entryOp.getAccVar()));
   }
@@ -291,8 +291,8 @@ static void createDeclareDeallocFuncWithArg(
   mlir::Value var = preDeallocOp.getArgument(0);
   if (unwrapFirBox) {
     mlir::Value loadOp =
-        builder.create<fir::LoadOp>(loc, preDeallocOp.getArgument(0));
-    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+        fir::LoadOp::create(builder, loc, preDeallocOp.getArgument(0));
+    fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, loadOp);
     addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
     var = boxAddrOp.getResult();
   }
@@ -303,12 +303,12 @@ static void createDeclareDeallocFuncWithArg(
           builder, loc, var, asFortran, bounds,
           /*structured=*/false, /*implicit=*/false, clause, var.getType(),
           /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-  builder.create<mlir::acc::DeclareExitOp>(
+  mlir::acc::DeclareExitOp::create(builder,
       loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccVar()));
 
   if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                 std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
+    ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
                            entryOp.getVar(), entryOp.getVarType(),
                            entryOp.getBounds(), entryOp.getAsyncOperands(),
                            entryOp.getAsyncOperandsDeviceTypeAttr(),
@@ -316,7 +316,7 @@ static void createDeclareDeallocFuncWithArg(
                            /*structured=*/false, /*implicit=*/false,
                            builder.getStringAttr(*entryOp.getName()));
   else
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
+    ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
                            entryOp.getBounds(), entryOp.getAsyncOperands(),
                            entryOp.getAsyncOperandsDeviceTypeAttr(),
                            entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
@@ -333,7 +333,7 @@ static void createDeclareDeallocFuncWithArg(
 
   var = postDeallocOp.getArgument(0);
   if (unwrapFirBox) {
-    var = builder.create<fir::LoadOp>(loc, postDeallocOp.getArgument(0));
+    var = fir::LoadOp::create(builder, loc, postDeallocOp.getArgument(0));
     asFortran << accFirDescriptorPostfix.str();
   }
 
@@ -385,7 +385,7 @@ genAtomicCaptureStatement(Fortran::lower::AbstractConverter &converter,
   // Generate `atomic.read` operation for atomic assigment statements
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
-  firOpBuilder.create<mlir::acc::AtomicReadOp>(
+  mlir::acc::AtomicReadOp::create(firOpBuilder,
       loc, fromAddress, toAddress, mlir::TypeAttr::get(elementType));
 }
 
@@ -406,7 +406,7 @@ genAtomicWriteStatement(Fortran::lower::AbstractConverter &converter,
   rhsExpr = firOpBuilder.createConvert(loc, varType, rhsExpr);
   firOpBuilder.restoreInsertionPoint(insertionPoint);
 
-  firOpBuilder.create<mlir::acc::AtomicWriteOp>(loc, lhsAddr, rhsExpr);
+  mlir::acc::AtomicWriteOp::create(firOpBuilder, loc, lhsAddr, rhsExpr);
 }
 
 /// Used to generate atomic.update operation which is created in existing
@@ -522,7 +522,7 @@ static inline void genAtomicUpdateStatement(
 
   mlir::Operation *atomicUpdateOp = nullptr;
   atomicUpdateOp =
-      firOpBuilder.create<mlir::acc::AtomicUpdateOp>(currentLocation, lhsAddr);
+      mlir::acc::AtomicUpdateOp::create(firOpBuilder, currentLocation, lhsAddr);
 
   llvm::SmallVector<mlir::Type> varTys = {varType};
   llvm::SmallVector<mlir::Location> locs = {currentLocation};
@@ -540,7 +540,7 @@ static inline void genAtomicUpdateStatement(
         *Fortran::semantics::GetExpr(assignmentStmtExpr), atomicStmtCtx));
     mlir::Value convertResult =
         firOpBuilder.createConvert(currentLocation, varType, rhsExpr);
-    firOpBuilder.create<mlir::acc::YieldOp>(currentLocation, convertResult);
+    mlir::acc::YieldOp::create(firOpBuilder, currentLocation, convertResult);
     converter.resetExprOverrides();
   }
   firOpBuilder.setInsertionPointAfter(atomicUpdateOp);
@@ -647,7 +647,7 @@ void genAtomicCapture(Fortran::lower::AbstractConverter &converter,
       fir::getBase(converter.genExprValue(assign2.lhs, stmtCtx)).getType();
 
   mlir::Operation *atomicCaptureOp = nullptr;
-  atomicCaptureOp = firOpBuilder.create<mlir::acc::AtomicCaptureOp>(loc);
+  atomicCaptureOp = mlir::acc::AtomicCaptureOp::create(firOpBuilder, loc);
 
   firOpBuilder.createBlock(&(atomicCaptureOp->getRegion(0)));
   mlir::Block &block = atomicCaptureOp->getRegion(0).back();
@@ -688,7 +688,7 @@ void genAtomicCapture(Fortran::lower::AbstractConverter &converter,
                               loc);
   }
   firOpBuilder.setInsertionPointToEnd(&block);
-  firOpBuilder.create<mlir::acc::TerminatorOp>(loc);
+  mlir::acc::TerminatorOp::create(firOpBuilder, loc);
   // The clean-ups associated with the statements inside the capture
   // construct must be generated after the AtomicCaptureOp.
   firOpBuilder.setInsertionPointAfter(atomicCaptureOp);
@@ -839,14 +839,14 @@ genDataExitOperations(fir::FirOpBuilder &builder,
     mlir::Location opLoc = exitLoc ? *exitLoc : entryOp.getLoc();
     if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                   std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-      builder.create<ExitOp>(
+      ExitOp::create(builder,
           opLoc, entryOp.getAccVar(), entryOp.getVar(), entryOp.getVarType(),
           entryOp.getBounds(), entryOp.getAsyncOperands(),
           entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
           entryOp.getDataClause(), structured, entryOp.getImplicit(),
           builder.getStringAttr(*entryOp.getName()));
     else
-      builder.create<ExitOp>(
+      ExitOp::create(builder,
           opLoc, entryOp.getAccVar(), entryOp.getBounds(),
           entryOp.getAsyncOperands(), entryOp.getAsyncOperandsDeviceTypeAttr(),
           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(), structured,
@@ -859,9 +859,9 @@ fir::ShapeOp genShapeOp(mlir::OpBuilder &builder, fir::SequenceType seqTy,
   llvm::SmallVector<mlir::Value> extents;
   mlir::Type idxTy = builder.getIndexType();
   for (auto extent : seqTy.getShape())
-    extents.push_back(builder.create<mlir::arith::ConstantOp>(
+    extents.push_back(mlir::arith::ConstantOp::create(builder,
         loc, idxTy, builder.getIntegerAttr(idxTy, extent)));
-  return builder.create<fir::ShapeOp>(loc, extents);
+  return fir::ShapeOp::create(builder, loc, extents);
 }
 
 /// Get the initial value for reduction operator.
@@ -936,7 +936,7 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
     return builder.createBool(loc, value);
   }
   if (ty.isIntOrIndex())
-    return builder.create<mlir::arith::ConstantOp>(
+    return mlir::arith::ConstantOp::create(builder,
         loc, ty,
         builder.getIntegerAttr(ty, getReductionInitValue<llvm::APInt>(op, ty)));
   if (op == mlir::acc::ReductionOperator::AccMin ||
@@ -945,12 +945,12 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
       llvm::report_fatal_error(
           "min/max reduction not supported for complex type");
     if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::ConstantOp>(
+      return mlir::arith::ConstantOp::create(builder,
           loc, ty,
           builder.getFloatAttr(ty,
                                getReductionInitValue<llvm::APFloat>(op, ty)));
   } else if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty)) {
-    return builder.create<mlir::arith::ConstantOp>(
+    return mlir::arith::ConstantOp::create(builder,
         loc, ty,
         builder.getFloatAttr(ty, getReductionInitValue<int64_t>(op, ty)));
   } else if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
@@ -985,10 +985,10 @@ static RecipeOp genRecipeOp(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   RecipeOp recipe;
   if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>) {
-    recipe = modBuilder.create<mlir::acc::ReductionRecipeOp>(loc, recipeName,
+    recipe = mlir::acc::ReductionRecipeOp::create(modBuilder, loc, recipeName,
                                                              ty, op);
   } else {
-    recipe = modBuilder.create<RecipeOp>(loc, recipeName, ty);
+    recipe = RecipeOp::create(modBuilder, loc, recipeName, ty);
   }
 
   llvm::SmallVector<mlir::Type> argsTy{ty};
@@ -1032,7 +1032,7 @@ static RecipeOp genRecipeOp(
       initName,
       initBlock->getArguments().take_back(initBlock->getArguments().size() - 1),
       initValue);
-  builder.create<mlir::acc::YieldOp>(loc, retVal ? retVal
+  mlir::acc::YieldOp::create(builder, loc, retVal ? retVal
                                                  : initBlock->getArgument(0));
   return recipe;
 }
@@ -1132,15 +1132,15 @@ static mlir::Value genShapeFromBoundsOrArgs(
   mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
   for (unsigned i = 0; i < args.size(); i += 3) {
     mlir::Value s1 =
-        builder.create<mlir::arith::SubIOp>(loc, args[i + 1], args[0]);
-    mlir::Value s2 = builder.create<mlir::arith::AddIOp>(loc, s1, one);
-    mlir::Value s3 = builder.create<mlir::arith::DivSIOp>(loc, s2, args[i + 2]);
-    mlir::Value cmp = builder.create<mlir::arith::CmpIOp>(
+        mlir::arith::SubIOp::create(builder, loc, args[i + 1], args[0]);
+    mlir::Value s2 = mlir::arith::AddIOp::create(builder, loc, s1, one);
+    mlir::Value s3 = mlir::arith::DivSIOp::create(builder, loc, s2, args[i + 2]);
+    mlir::Value cmp = mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::sgt, s3, zero);
-    mlir::Value ext = builder.create<mlir::arith::SelectOp>(loc, cmp, s3, zero);
+    mlir::Value ext = mlir::arith::SelectOp::create(builder, loc, cmp, s3, zero);
     extents.push_back(ext);
   }
-  return builder.create<fir::ShapeOp>(loc, extents);
+  return fir::ShapeOp::create(builder, loc, extents);
 }
 
 static hlfir::DesignateOp::Subscripts
@@ -1157,7 +1157,7 @@ static hlfir::Entity genDesignateWithTriplets(
     hlfir::DesignateOp::Subscripts &triplets, mlir::Value shape) {
   llvm::SmallVector<mlir::Value> lenParams;
   hlfir::genLengthParameters(loc, builder, entity, lenParams);
-  auto designate = builder.create<hlfir::DesignateOp>(
+  auto designate = hlfir::DesignateOp::create(builder,
       loc, entity.getBase().getType(), entity, /*component=*/"",
       /*componentShape=*/mlir::Value{}, triplets,
       /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt, shape,
@@ -1198,20 +1198,20 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
   builder.setInsertionPointToEnd(&recipe.getCopyRegion().back());
   ty = fir::unwrapRefType(ty);
   if (fir::isa_trivial(ty)) {
-    mlir::Value initValue = builder.create<fir::LoadOp>(
+    mlir::Value initValue = fir::LoadOp::create(builder,
         loc, recipe.getCopyRegion().front().getArgument(0));
-    builder.create<fir::StoreOp>(loc, initValue,
+    fir::StoreOp::create(builder, loc, initValue,
                                  recipe.getCopyRegion().front().getArgument(1));
   } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
     fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
     auto shape = genShapeFromBoundsOrArgs(
         loc, firBuilder, seqTy, bounds, recipe.getCopyRegion().getArguments());
 
-    auto leftDeclOp = builder.create<hlfir::DeclareOp>(
+    auto leftDeclOp = hlfir::DeclareOp::create(builder,
         loc, recipe.getCopyRegion().getArgument(0), llvm::StringRef{}, shape,
         llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{});
-    auto rightDeclOp = builder.create<hlfir::DeclareOp>(
+    auto rightDeclOp = hlfir::DeclareOp::create(builder,
         loc, recipe.getCopyRegion().getArgument(1), llvm::StringRef{}, shape,
         llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{});
@@ -1225,7 +1225,7 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
     auto right =
         genDesignateWithTriplets(firBuilder, loc, rightEntity, triplets, shape);
 
-    firBuilder.create<hlfir::AssignOp>(loc, left, right);
+    hlfir::AssignOp::create(firBuilder, loc, left, right);
 
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
     fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
@@ -1246,10 +1246,10 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
     auto rightEntity = hlfir::Entity{recipe.getCopyRegion().getArgument(1)};
     auto right =
         genDesignateWithTriplets(firBuilder, loc, rightEntity, triplets, shape);
-    firBuilder.create<hlfir::AssignOp>(loc, left, right);
+    hlfir::AssignOp::create(firBuilder, loc, left, right);
   }
 
-  builder.create<mlir::acc::TerminatorOp>(loc);
+  mlir::acc::TerminatorOp::create(builder, loc);
   builder.restoreInsertionPoint(ip);
   return recipe;
 }
@@ -1414,10 +1414,10 @@ static mlir::Value genLogicalCombiner(fir::FirOpBuilder &builder,
                                       mlir::Location loc, mlir::Value value1,
                                       mlir::Value value2) {
   mlir::Type i1 = builder.getI1Type();
-  mlir::Value v1 = builder.create<fir::ConvertOp>(loc, i1, value1);
-  mlir::Value v2 = builder.create<fir::ConvertOp>(loc, i1, value2);
-  mlir::Value combined = builder.create<Op>(loc, v1, v2);
-  return builder.create<fir::ConvertOp>(loc, value1.getType(), combined);
+  mlir::Value v1 = fir::ConvertOp::create(builder, loc, i1, value1);
+  mlir::Value v2 = fir::ConvertOp::create(builder, loc, i1, value2);
+  mlir::Value combined = Op::create(builder, loc, v1, v2);
+  return fir::ConvertOp::create(builder, loc, value1.getType(), combined);
 }
 
 static mlir::Value genComparisonCombiner(fir::FirOpBuilder &builder,
@@ -1426,10 +1426,10 @@ static mlir::Value genComparisonCombiner(fir::FirOpBuilder &builder,
                                          mlir::Value value1,
                                          mlir::Value value2) {
   mlir::Type i1 = builder.getI1Type();
-  mlir::Value v1 = builder.create<fir::ConvertOp>(loc, i1, value1);
-  mlir::Value v2 = builder.create<fir::ConvertOp>(loc, i1, value2);
-  mlir::Value add = builder.create<mlir::arith::CmpIOp>(loc, pred, v1, v2);
-  return builder.create<fir::ConvertOp>(loc, value1.getType(), add);
+  mlir::Value v1 = fir::ConvertOp::create(builder, loc, i1, value1);
+  mlir::Value v2 = fir::ConvertOp::create(builder, loc, i1, value2);
+  mlir::Value add = mlir::arith::CmpIOp::create(builder, loc, pred, v1, v2);
+  return fir::ConvertOp::create(builder, loc, value1.getType(), add);
 }
 
 static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
@@ -1441,21 +1441,21 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
   value2 = builder.loadIfRef(loc, value2);
   if (op == mlir::acc::ReductionOperator::AccAdd) {
     if (ty.isIntOrIndex())
-      return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+      return mlir::arith::AddIOp::create(builder, loc, value1, value2);
     if (mlir::isa<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+      return mlir::arith::AddFOp::create(builder, loc, value1, value2);
     if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty))
-      return builder.create<fir::AddcOp>(loc, value1, value2);
+      return fir::AddcOp::create(builder, loc, value1, value2);
     TODO(loc, "reduction add type");
   }
 
   if (op == mlir::acc::ReductionOperator::AccMul) {
     if (ty.isIntOrIndex())
-      return builder.create<mlir::arith::MulIOp>(loc, value1, value2);
+      return mlir::arith::MulIOp::create(builder, loc, value1, value2);
     if (mlir::isa<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::MulFOp>(loc, value1, value2);
+      return mlir::arith::MulFOp::create(builder, loc, value1, value2);
     if (mlir::isa<mlir::ComplexType>(ty))
-      return builder.create<fir::MulcOp>(loc, value1, value2);
+      return fir::MulcOp::create(builder, loc, value1, value2);
     TODO(loc, "reduction mul type");
   }
 
@@ -1466,13 +1466,13 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
     return fir::genMax(builder, loc, {value1, value2});
 
   if (op == mlir::acc::ReductionOperator::AccIand)
-    return builder.create<mlir::arith::AndIOp>(loc, value1, value2);
+    return mlir::arith::AndIOp::create(builder, loc, value1, value2);
 
   if (op == mlir::acc::ReductionOperator::AccIor)
-    return builder.create<mlir::arith::OrIOp>(loc, value1, value2);
+    return mlir::arith::OrIOp::create(builder, loc, value1, value2);
 
   if (op == mlir::acc::ReductionOperator::AccXor)
-    return builder.create<mlir::arith::XOrIOp>(loc, value1, value2);
+    return mlir::arith::XOrIOp::create(builder, loc, value1, value2);
 
   if (op == mlir::acc::ReductionOperator::AccLand)
     return genLogicalCombiner<mlir::arith::AndIOp>(builder, loc, value1,
@@ -1520,10 +1520,10 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       auto shape =
           genShapeFromBoundsOrArgs(loc, builder, seqTy, bounds,
                                    recipe.getCombinerRegion().getArguments());
-      auto v1DeclareOp = builder.create<hlfir::DeclareOp>(
+      auto v1DeclareOp = hlfir::DeclareOp::create(builder,
           loc, value1, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
-      auto v2DeclareOp = builder.create<hlfir::DeclareOp>(
+      auto v2DeclareOp = hlfir::DeclareOp::create(builder,
           loc, value2, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       hlfir::DesignateOp::Subscripts triplets = getTripletsFromArgs(recipe);
@@ -1531,7 +1531,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       llvm::SmallVector<mlir::Value> lenParamsLeft;
       auto leftEntity = hlfir::Entity{v1DeclareOp.getBase()};
       hlfir::genLengthParameters(loc, builder, leftEntity, lenParamsLeft);
-      auto leftDesignate = builder.create<hlfir::DesignateOp>(
+      auto leftDesignate = hlfir::DesignateOp::create(builder,
           loc, v1DeclareOp.getBase().getType(), v1DeclareOp.getBase(),
           /*component=*/"",
           /*componentShape=*/mlir::Value{}, triplets,
@@ -1542,7 +1542,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       llvm::SmallVector<mlir::Value> lenParamsRight;
       auto rightEntity = hlfir::Entity{v2DeclareOp.getBase()};
       hlfir::genLengthParameters(loc, builder, rightEntity, lenParamsLeft);
-      auto rightDesignate = builder.create<hlfir::DesignateOp>(
+      auto rightDesignate = hlfir::DesignateOp::create(builder,
           loc, v2DeclareOp.getBase().getType(), v2DeclareOp.getBase(),
           /*component=*/"",
           /*componentShape=*/mlir::Value{}, triplets,
@@ -1564,20 +1564,20 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       mlir::Value elemental = hlfir::genElementalOp(
           loc, builder, seqTy.getEleTy(), shape, typeParams, genKernel,
           /*isUnordered=*/true);
-      builder.create<hlfir::AssignOp>(loc, elemental, v1DeclareOp.getBase());
+      hlfir::AssignOp::create(builder, loc, elemental, v1DeclareOp.getBase());
       return;
     }
     if (bounds.empty()) {
       llvm::SmallVector<mlir::Value> extents;
       mlir::Type idxTy = builder.getIndexType();
       for (auto extent : seqTy.getShape()) {
-        mlir::Value lb = builder.create<mlir::arith::ConstantOp>(
+        mlir::Value lb = mlir::arith::ConstantOp::create(builder,
             loc, idxTy, builder.getIntegerAttr(idxTy, 0));
-        mlir::Value ub = builder.create<mlir::arith::ConstantOp>(
+        mlir::Value ub = mlir::arith::ConstantOp::create(builder,
             loc, idxTy, builder.getIntegerAttr(idxTy, extent - 1));
-        mlir::Value step = builder.create<mlir::arith::ConstantOp>(
+        mlir::Value step = mlir::arith::ConstantOp::create(builder,
             loc, idxTy, builder.getIntegerAttr(idxTy, 1));
-        auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
+        auto loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
                                                   /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
@@ -1594,7 +1594,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
         llvm::SmallVector<mlir::Value> values =
             genConstantBounds(builder, loc, dataBound);
         auto loop =
-            builder.create<fir::DoLoopOp>(loc, values[0], values[1], values[2],
+            fir::DoLoopOp::create(builder, loc, values[0], values[1], values[2],
                                           /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
@@ -1611,31 +1611,31 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
         mlir::Value lb = recipe.getCombinerRegion().getArgument(i);
         mlir::Value ub = recipe.getCombinerRegion().getArgument(i + 1);
         mlir::Value step = recipe.getCombinerRegion().getArgument(i + 2);
-        auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
+        auto loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
                                                   /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
         ivs.push_back(loop.getInductionVar());
       }
     }
-    auto addr1 = builder.create<fir::CoordinateOp>(loc, refTy, value1, ivs);
-    auto addr2 = builder.create<fir::CoordinateOp>(loc, refTy, value2, ivs);
-    auto load1 = builder.create<fir::LoadOp>(loc, addr1);
-    auto load2 = builder.create<fir::LoadOp>(loc, addr2);
+    auto addr1 = fir::CoordinateOp::create(builder, loc, refTy, value1, ivs);
+    auto addr2 = fir::CoordinateOp::create(builder, loc, refTy, value2, ivs);
+    auto load1 = fir::LoadOp::create(builder, loc, addr1);
+    auto load2 = fir::LoadOp::create(builder, loc, addr2);
     mlir::Value res =
         genScalarCombiner(builder, loc, op, seqTy.getEleTy(), load1, load2);
-    builder.create<fir::StoreOp>(loc, res, addr1);
+    fir::StoreOp::create(builder, loc, res, addr1);
     builder.setInsertionPointAfter(loops[0]);
   } else if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {
     mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
     if (fir::isa_trivial(innerTy)) {
       mlir::Value boxAddr1 = value1, boxAddr2 = value2;
       if (fir::isBoxAddress(boxAddr1.getType()))
-        boxAddr1 = builder.create<fir::LoadOp>(loc, boxAddr1);
+        boxAddr1 = fir::LoadOp::create(builder, loc, boxAddr1);
       if (fir::isBoxAddress(boxAddr2.getType()))
-        boxAddr2 = builder.create<fir::LoadOp>(loc, boxAddr2);
-      boxAddr1 = builder.create<fir::BoxAddrOp>(loc, boxAddr1);
-      boxAddr2 = builder.create<fir::BoxAddrOp>(loc, boxAddr2);
+        boxAddr2 = fir::LoadOp::create(builder, loc, boxAddr2);
+      boxAddr1 = fir::BoxAddrOp::create(builder, loc, boxAddr1);
+      boxAddr2 = fir::BoxAddrOp::create(builder, loc, boxAddr2);
       auto leftEntity = hlfir::Entity{boxAddr1};
       auto rightEntity = hlfir::Entity{boxAddr2};
 
@@ -1643,7 +1643,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       auto rightVal = hlfir::loadTrivialScalar(loc, builder, rightEntity);
       mlir::Value res =
           genScalarCombiner(builder, loc, op, innerTy, leftVal, rightVal);
-      builder.create<hlfir::AssignOp>(loc, res, boxAddr1);
+      hlfir::AssignOp::create(builder, loc, res, boxAddr1);
     } else {
       mlir::Type innerTy = fir::extractSequenceType(boxTy);
       fir::SequenceType seqTy =
@@ -1659,13 +1659,13 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       auto leftEntity = hlfir::Entity{value1};
       if (fir::isBoxAddress(value1.getType()))
         leftEntity =
-            hlfir::Entity{builder.create<fir::LoadOp>(loc, value1).getResult()};
+            hlfir::Entity{fir::LoadOp::create(builder, loc, value1).getResult()};
       auto left =
           genDesignateWithTriplets(builder, loc, leftEntity, triplets, shape);
       auto rightEntity = hlfir::Entity{value2};
       if (fir::isBoxAddress(value2.getType()))
         rightEntity =
-            hlfir::Entity{builder.create<fir::LoadOp>(loc, value2).getResult()};
+            hlfir::Entity{fir::LoadOp::create(builder, loc, value2).getResult()};
       auto right =
           genDesignateWithTriplets(builder, loc, rightEntity, triplets, shape);
 
@@ -1683,11 +1683,11 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       mlir::Value elemental = hlfir::genElementalOp(
           loc, builder, seqTy.getEleTy(), shape, typeParams, genKernel,
           /*isUnordered=*/true);
-      builder.create<hlfir::AssignOp>(loc, elemental, value1);
+      hlfir::AssignOp::create(builder, loc, elemental, value1);
     }
   } else {
     mlir::Value res = genScalarCombiner(builder, loc, op, ty, value1, value2);
-    builder.create<fir::StoreOp>(loc, res, value1);
+    fir::StoreOp::create(builder, loc, res, value1);
   }
 }
 
@@ -1729,7 +1729,7 @@ mlir::acc::ReductionRecipeOp Fortran::lower::createOrGetReductionRecipe(
   mlir::Value v1 = recipe.getCombinerRegion().front().getArgument(0);
   mlir::Value v2 = recipe.getCombinerRegion().front().getArgument(1);
   genCombiner(builder, loc, op, ty, v1, v2, recipe, bounds, allConstantBound);
-  builder.create<mlir::acc::YieldOp>(loc, v1);
+  mlir::acc::YieldOp::create(builder, loc, v1);
   builder.restoreInsertionPoint(ip);
   return recipe;
 }
@@ -1821,7 +1821,7 @@ createRegionOp(fir::FirOpBuilder &builder, mlir::Location loc,
                llvm::SmallVector<mlir::Type> retTy = {},
                mlir::Value yieldValue = {}, mlir::TypeRange argsTy = {},
                llvm::SmallVector<mlir::Location> locs = {}) {
-  Op op = builder.create<Op>(loc, retTy, operands);
+  Op op = Op::create(builder, loc, retTy, operands);
   builder.createBlock(&op.getRegion(), op.getRegion().end(), argsTy, locs);
   mlir::Block &block = op.getRegion().back();
   builder.setInsertionPointToStart(&block);
@@ -1841,13 +1841,13 @@ createRegionOp(fir::FirOpBuilder &builder, mlir::Location loc,
 
   if (yieldValue) {
     if constexpr (std::is_same_v<Terminator, mlir::acc::YieldOp>) {
-      Terminator yieldOp = builder.create<Terminator>(returnLoc, yieldValue);
+      Terminator yieldOp = Terminator::create(builder, returnLoc, yieldValue);
       yieldValue.getDefiningOp()->moveBefore(yieldOp);
     } else {
-      builder.create<Terminator>(returnLoc);
+      Terminator::create(builder, returnLoc);
     }
   } else {
-    builder.create<Terminator>(returnLoc);
+    Terminator::create(builder, returnLoc);
   }
   builder.setInsertionPointToStart(&block);
   return op;
@@ -2437,7 +2437,7 @@ static mlir::acc::LoopOp createLoopOp(
 
   for (auto [arg, value] : llvm::zip(
            loopOp.getLoopRegions().front()->front().getArguments(), ivPrivate))
-    builder.create<fir::StoreOp>(currentLocation, arg, value);
+    fir::StoreOp::create(builder, currentLocation, arg, value);
 
   loopOp.setInclusiveUpperbound(inclusiveBounds);
 
@@ -3750,7 +3750,7 @@ genACCUpdateOp(Fortran::lower::AbstractConverter &converter,
 
   dataClauseOperands.append(updateHostOperands);
 
-  builder.create<mlir::acc::UpdateOp>(
+  mlir::acc::UpdateOp::create(builder,
       currentLocation, ifCond, asyncOperands,
       getArrayAttr(builder, asyncOperandsDeviceTypes),
       getArrayAttr(builder, asyncOnlyDeviceTypes), waitOperands,
@@ -3873,12 +3873,12 @@ static void createDeclareGlobalOp(mlir::OpBuilder &modBuilder,
                                   const std::string &declareGlobalName,
                                   bool implicit, std::stringstream &asFortran) {
   GlobalOp declareGlobalOp =
-      modBuilder.create<GlobalOp>(loc, declareGlobalName);
+      GlobalOp::create(modBuilder, loc, declareGlobalName);
   builder.createBlock(&declareGlobalOp.getRegion(),
                       declareGlobalOp.getRegion().end(), {}, {});
   builder.setInsertionPointToEnd(&declareGlobalOp.getRegion().back());
 
-  fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(builder,
       loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
   addDeclareAttr(builder, addrOp, clause);
 
@@ -3888,21 +3888,21 @@ static void createDeclareGlobalOp(mlir::OpBuilder &modBuilder,
       /*structured=*/false, implicit, clause, addrOp.getResTy().getType(),
       /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
   if constexpr (std::is_same_v<DeclareOp, mlir::acc::DeclareEnterOp>)
-    builder.create<DeclareOp>(
+    DeclareOp::create(builder,
         loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
         mlir::ValueRange(entryOp.getAccVar()));
   else
-    builder.create<DeclareOp>(loc, mlir::Value{},
+    DeclareOp::create(builder, loc, mlir::Value{},
                               mlir::ValueRange(entryOp.getAccVar()));
   if constexpr (std::is_same_v<GlobalOp, mlir::acc::GlobalDestructorOp>) {
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
+    ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
                            entryOp.getBounds(), entryOp.getAsyncOperands(),
                            entryOp.getAsyncOperandsDeviceTypeAttr(),
                            entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
                            /*structured=*/false, /*implicit=*/false,
                            builder.getStringAttr(*entryOp.getName()));
   }
-  builder.create<mlir::acc::TerminatorOp>(loc);
+  mlir::acc::TerminatorOp::create(builder, loc);
   modBuilder.setInsertionPointAfter(declareGlobalOp);
 }
 
@@ -3917,7 +3917,7 @@ static void createDeclareAllocFunc(mlir::OpBuilder &modBuilder,
   auto registerFuncOp =
       createDeclareFunc(modBuilder, builder, loc, registerFuncName.str());
 
-  fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(builder,
       loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
 
   std::stringstream asFortran;
@@ -3941,14 +3941,14 @@ static void createDeclareAllocFunc(mlir::OpBuilder &modBuilder,
   createSimpleOp<mlir::acc::UpdateOp>(builder, loc, operands, operandSegments);
 
   if (unwrapFirBox) {
-    auto loadOp = builder.create<fir::LoadOp>(loc, addrOp.getResult());
-    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+    auto loadOp = fir::LoadOp::create(builder, loc, addrOp.getResult());
+    fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, loadOp);
     addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
     EntryOp entryOp = createDataEntryOp<EntryOp>(
         builder, loc, boxAddrOp.getResult(), asFortran, bounds,
         /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-    builder.create<mlir::acc::DeclareEnterOp>(
+    mlir::acc::DeclareEnterOp::create(builder,
         loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
         mlir::ValueRange(entryOp.getAccVar()));
   }
@@ -3980,10 +3980,10 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
     auto preDeallocOp =
         createDeclareFunc(modBuilder, builder, loc, preDeallocFuncName.str());
 
-    fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
+    fir::AddrOfOp addrOp = fir::AddrOfOp::create(builder,
         loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
-    auto loadOp = builder.create<fir::LoadOp>(loc, addrOp.getResult());
-    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+    auto loadOp = fir::LoadOp::create(builder, loc, addrOp.getResult());
+    fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, loadOp);
     mlir::Value var = boxAddrOp.getResult();
     addDeclareAttr(builder, var.getDefiningOp(), clause);
 
@@ -3994,12 +3994,12 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
             /*structured=*/false, /*implicit=*/false, clause, var.getType(),
             /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
 
-    builder.create<mlir::acc::DeclareExitOp>(
+    mlir::acc::DeclareExitOp::create(builder,
         loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccVar()));
 
     if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                   std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-      builder.create<ExitOp>(
+      ExitOp::create(builder,
           entryOp.getLoc(), entryOp.getAccVar(), entryOp.getVar(),
           entryOp.getBounds(), entryOp.getAsyncOperands(),
           entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
@@ -4007,7 +4007,7 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
           /*structured=*/false, /*implicit=*/false,
           builder.getStringAttr(*entryOp.getName()));
     else
-      builder.create<ExitOp>(
+      ExitOp::create(builder,
           entryOp.getLoc(), entryOp.getAccVar(), entryOp.getBounds(),
           entryOp.getAsyncOperands(), entryOp.getAsyncOperandsDeviceTypeAttr(),
           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
@@ -4024,7 +4024,7 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
   auto postDeallocOp =
       createDeclareFunc(modBuilder, builder, loc, postDeallocFuncName.str());
 
-  fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(builder,
       loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
   if (unwrapFirBox)
     asFortran << accFirDescriptorPostfix.str();
@@ -4262,12 +4262,12 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
   auto ops = funcOp.getOps<mlir::acc::DeclareEnterOp>();
   mlir::Value declareToken;
   if (ops.empty()) {
-    declareToken = builder.create<mlir::acc::DeclareEnterOp>(
+    declareToken = mlir::acc::DeclareEnterOp::create(builder,
         loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
         dataClauseOperands);
   } else {
     auto declareOp = *ops.begin();
-    auto newDeclareOp = builder.create<mlir::acc::DeclareEnterOp>(
+    auto newDeclareOp = mlir::acc::DeclareEnterOp::create(builder,
         loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
         declareOp.getDataClauseOperands());
     newDeclareOp.getDataClauseOperandsMutable().append(dataClauseOperands);
@@ -4290,7 +4290,7 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
     mlir::func::FuncOp funcOp = builder.getFunction();
     auto ops = funcOp.getOps<mlir::acc::DeclareExitOp>();
     if (ops.empty()) {
-      builder.create<mlir::acc::DeclareExitOp>(loc, declareToken, operands);
+      mlir::acc::DeclareExitOp::create(builder, loc, declareToken, operands);
     } else {
       auto declareOp = *ops.begin();
       declareOp.getDataClauseOperandsMutable().append(operands);
@@ -4535,7 +4535,7 @@ void createOpenACCRoutineConstruct(
   std::string routineOpStr = routineOpName.str();
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  modBuilder.create<mlir::acc::RoutineOp>(
+  mlir::acc::RoutineOp::create(modBuilder,
       loc, routineOpStr,
       mlir::SymbolRefAttr::get(builder.getContext(), funcName),
       getArrayAttrOrNull(builder, bindIdNames),
@@ -4880,9 +4880,9 @@ void Fortran::lower::genOpenACCTerminator(fir::FirOpBuilder &builder,
                                           mlir::Operation *op,
                                           mlir::Location loc) {
   if (mlir::isa<mlir::acc::ParallelOp, mlir::acc::LoopOp>(op))
-    builder.create<mlir::acc::YieldOp>(loc);
+    mlir::acc::YieldOp::create(builder, loc);
   else
-    builder.create<mlir::acc::TerminatorOp>(loc);
+    mlir::acc::TerminatorOp::create(builder, loc);
 }
 
 bool Fortran::lower::isInOpenACCLoop(fir::FirOpBuilder &builder) {
@@ -4902,7 +4902,7 @@ void Fortran::lower::genEarlyReturnInOpenACCLoop(fir::FirOpBuilder &builder,
                                                  mlir::Location loc) {
   mlir::Value yieldValue =
       builder.createIntegerConstant(loc, builder.getI1Type(), 1);
-  builder.create<mlir::acc::YieldOp>(loc, yieldValue);
+  mlir::acc::YieldOp::create(builder, loc, yieldValue);
 }
 
 int64_t Fortran::lower::getLoopCountForCollapseAndTile(
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 6ea331c370640..6794144de20d3 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -528,7 +528,7 @@ genAtomicRead(lower::AbstractConverter &converter,
   }();
 
   builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
+  mlir::Operation *op = mlir::omp::AtomicReadOp::create(builder,
       loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
       makeMemOrderAttr(converter, memOrder));
 
@@ -537,7 +537,7 @@ genAtomicRead(lower::AbstractConverter &converter,
     // The READ operation could be a part of UPDATE CAPTURE, so make sure
     // we don't emit extra code into the body of the atomic op.
     builder.restoreInsertionPoint(postAt);
-    mlir::Value load = builder.create<fir::LoadOp>(loc, toAddr);
+    mlir::Value load = fir::LoadOp::create(builder, loc, toAddr);
     overrides.try_emplace(&atom, load);
 
     converter.overrideExprValues(&overrides);
@@ -545,7 +545,7 @@ genAtomicRead(lower::AbstractConverter &converter,
         fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
     converter.resetExprOverrides();
 
-    builder.create<fir::StoreOp>(loc, value, storeAddr);
+    fir::StoreOp::create(builder, loc, value, storeAddr);
   }
   return op;
 }
@@ -581,7 +581,7 @@ genAtomicWrite(lower::AbstractConverter &converter,
   mlir::Value converted = builder.createConvert(loc, atomType, value);
 
   builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
+  mlir::Operation *op = mlir::omp::AtomicWriteOp::create(builder,
       loc, atomAddr, converted, hint, makeMemOrderAttr(converter, memOrder));
   return op;
 }
@@ -635,7 +635,7 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   }
 
   builder.restoreInsertionPoint(atomicAt);
-  auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
+  auto updateOp = mlir::omp::AtomicUpdateOp::create(builder,
       loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
@@ -647,7 +647,7 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Value updated =
       fir::getBase(converter.genExprValue(rhs, stmtCtx, &loc));
   mlir::Value converted = builder.createConvert(loc, atomType, updated);
-  builder.create<mlir::omp::YieldOp>(loc, converted);
+  mlir::omp::YieldOp::create(builder, loc, converted);
   converter.resetExprOverrides();
 
   builder.restoreInsertionPoint(postAt); // For naCtx cleanups
@@ -731,7 +731,7 @@ void Fortran::lower::omp::lowerAtomic(
              "Expexcing two actions");
       (void)action0;
       (void)action1;
-      captureOp = builder.create<mlir::omp::AtomicCaptureOp>(
+      captureOp = mlir::omp::AtomicCaptureOp::create(builder,
           loc, hint, makeMemOrderAttr(converter, memOrder));
       // Set the non-atomic insertion point to before the atomic.capture.
       preAt = getInsertionPointBefore(captureOp);
@@ -740,7 +740,7 @@ void Fortran::lower::omp::lowerAtomic(
       builder.setInsertionPointToEnd(block);
       // Set the atomic insertion point to before the terminator inside
       // atomic.capture.
-      mlir::Operation *term = builder.create<mlir::omp::TerminatorOp>(loc);
+      mlir::Operation *term = mlir::omp::TerminatorOp::create(builder, loc);
       atomicAt = getInsertionPointBefore(term);
       postAt = getInsertionPointAfter(captureOp);
       hint = nullptr;
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 74087d42a8e6e..d118bfffa32a3 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -723,7 +723,7 @@ bool ClauseProcessor::processCopyin() const {
   // barrier is inserted following all of them.
   firOpBuilder.restoreInsertionPoint(insPt);
   if (hasCopyin)
-    firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
+    mlir::omp::BarrierOp::create(firOpBuilder, converter.getCurrentLocation());
   return hasCopyin;
 }
 
@@ -803,7 +803,7 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
   llvm::SmallVector<mlir::Type> argsTy = {varType, varType};
   auto funcType = mlir::FunctionType::get(builder.getContext(), argsTy, {});
   mlir::func::FuncOp funcOp =
-      modBuilder.create<mlir::func::FuncOp>(loc, copyFuncName, funcType);
+      mlir::func::FuncOp::create(modBuilder, loc, copyFuncName, funcType);
   funcOp.setVisibility(mlir::SymbolTable::Visibility::Private);
   fir::factory::setInternalLinkage(funcOp);
   builder.createBlock(&funcOp.getRegion(), funcOp.getRegion().end(), argsTy,
@@ -819,21 +819,21 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
     for (auto extent : typeInfo.getShape())
       extents.push_back(
           builder.createIntegerConstant(loc, builder.getIndexType(), extent));
-    shape = builder.create<fir::ShapeOp>(loc, extents);
+    shape = fir::ShapeOp::create(builder, loc, extents);
   }
   mlir::Value dst = funcOp.getArgument(0);
   mlir::Value src = funcOp.getArgument(1);
   llvm::SmallVector<mlir::Value> typeparams;
   if (typeInfo.isBoxChar()) {
     // fir.boxchar will be passed here as fir.ref<fir.boxchar>
-    auto loadDst = builder.create<fir::LoadOp>(loc, dst);
-    auto loadSrc = builder.create<fir::LoadOp>(loc, src);
+    auto loadDst = fir::LoadOp::create(builder, loc, dst);
+    auto loadSrc = fir::LoadOp::create(builder, loc, src);
     // get the actual fir.ref<fir.char> type
     mlir::Type refType =
         fir::ReferenceType::get(mlir::cast<fir::BoxCharType>(eleTy).getEleTy());
-    auto unboxedDst = builder.create<fir::UnboxCharOp>(
+    auto unboxedDst = fir::UnboxCharOp::create(builder,
         loc, refType, builder.getIndexType(), loadDst);
-    auto unboxedSrc = builder.create<fir::UnboxCharOp>(
+    auto unboxedSrc = fir::UnboxCharOp::create(builder,
         loc, refType, builder.getIndexType(), loadSrc);
     // Add length to type parameters
     typeparams.push_back(unboxedDst.getResult(1));
@@ -844,14 +844,14 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
         loc, builder.getCharacterLengthType(), *typeInfo.getCharLength());
     typeparams.push_back(charLen);
   }
-  auto declDst = builder.create<hlfir::DeclareOp>(
+  auto declDst = hlfir::DeclareOp::create(builder,
       loc, dst, copyFuncName + "_dst", shape, typeparams,
       /*dummy_scope=*/nullptr, attrs);
-  auto declSrc = builder.create<hlfir::DeclareOp>(
+  auto declSrc = hlfir::DeclareOp::create(builder,
       loc, src, copyFuncName + "_src", shape, typeparams,
       /*dummy_scope=*/nullptr, attrs);
   converter.copyVar(loc, declDst.getBase(), declSrc.getBase(), varAttrs);
-  builder.create<mlir::func::ReturnOp>(loc);
+  mlir::func::ReturnOp::create(builder, loc);
   return funcOp;
 }
 
@@ -882,8 +882,8 @@ bool ClauseProcessor::processCopyprivate(
     if (mlir::isa<fir::BaseBoxType>(symType) ||
         mlir::isa<fir::BoxCharType>(symType)) {
       fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-      auto alloca = builder.create<fir::AllocaOp>(currentLocation, symType);
-      builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
+      auto alloca = fir::AllocaOp::create(builder, currentLocation, symType);
+      fir::StoreOp::create(builder, currentLocation, symVal, alloca);
       cpVar = alloca;
     }
 
@@ -1002,7 +1002,7 @@ bool ClauseProcessor::processDepend(lower::SymMap &symMap,
       // allocations so this is not a reliable way to identify the dependency.
       if (auto ref = mlir::dyn_cast<fir::ReferenceType>(dependVar.getType()))
         if (fir::isa_box_type(ref.getElementType()))
-          dependVar = builder.create<fir::LoadOp>(
+          dependVar = fir::LoadOp::create(builder,
               converter.getCurrentLocation(), dependVar);
 
       // The openmp dialect doesn't know what to do with boxes (and it would
@@ -1012,7 +1012,7 @@ bool ClauseProcessor::processDepend(lower::SymMap &symMap,
       // Getting the address of the box data is okay because all the runtime
       // ultimately cares about is the base address of the array.
       if (fir::isa_box_type(dependVar.getType()))
-        dependVar = builder.create<fir::BoxAddrOp>(
+        dependVar = fir::BoxAddrOp::create(builder,
             converter.getCurrentLocation(), dependVar);
 
       result.dependVars.push_back(dependVar);
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 675a58e4f35a1..def5297ba8036 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -291,7 +291,7 @@ void DataSharingProcessor::insertBarrier(
       clauseOps->privateNeedsBarrier =
           mlir::UnitAttr::get(&converter.getMLIRContext());
   } else {
-    firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
+    mlir::omp::BarrierOp::create(firOpBuilder, converter.getCurrentLocation());
   }
 }
 
@@ -351,32 +351,32 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
              loopOp.getIVs(), result.loopUpperBounds, result.loopSteps)) {
       // v = iv + step
       // cmp = step < 0 ? v < ub : v > ub
-      mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
+      mlir::Value v = mlir::arith::AddIOp::create(firOpBuilder, loc, iv, step);
       vs.push_back(v);
       mlir::Value zero =
           firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
-      mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
+      mlir::Value negativeStep = mlir::arith::CmpIOp::create(firOpBuilder,
           loc, mlir::arith::CmpIPredicate::slt, step, zero);
-      mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
+      mlir::Value vLT = mlir::arith::CmpIOp::create(firOpBuilder,
           loc, mlir::arith::CmpIPredicate::slt, v, ub);
-      mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
+      mlir::Value vGT = mlir::arith::CmpIOp::create(firOpBuilder,
           loc, mlir::arith::CmpIPredicate::sgt, v, ub);
-      mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
+      mlir::Value icmpOp = mlir::arith::SelectOp::create(firOpBuilder,
           loc, negativeStep, vLT, vGT);
 
       if (cmpOp)
-        cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
+        cmpOp = mlir::arith::AndIOp::create(firOpBuilder, loc, cmpOp, icmpOp);
       else
         cmpOp = icmpOp;
     }
 
-    auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
+    auto ifOp = fir::IfOp::create(firOpBuilder, loc, cmpOp, /*else*/ false);
     firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
       hlfir::Entity loopIVEntity{loopIV};
       loopIVEntity =
           hlfir::derefPointersAndAllocatables(loc, firOpBuilder, loopIVEntity);
-      firOpBuilder.create<hlfir::AssignOp>(loc, v, loopIVEntity);
+      hlfir::AssignOp::create(firOpBuilder, loc, v, loopIVEntity);
     }
     lastPrivIP = firOpBuilder.saveInsertionPoint();
   } else if (mlir::isa<mlir::omp::SectionsOp>(op)) {
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index fcb20fdf187ff..8d906550a6065 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -642,7 +642,7 @@ static void threadPrivatizeVars(lower::AbstractConverter &converter,
       op = declOp.getMemref().getDefiningOp();
     if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
       symValue = mlir::dyn_cast<mlir::omp::ThreadprivateOp>(op).getSymAddr();
-    return firOpBuilder.create<mlir::omp::ThreadprivateOp>(
+    return mlir::omp::ThreadprivateOp::create(firOpBuilder,
         currentLocation, symValue.getType(), symValue);
   };
 
@@ -710,7 +710,7 @@ createAndSetPrivatizedLoopVar(lower::AbstractConverter &converter,
   lhs = hlfir::derefPointersAndAllocatables(loc, firOpBuilder, lhs);
 
   mlir::Operation *storeOp =
-      firOpBuilder.create<hlfir::AssignOp>(loc, cvtVal, lhs);
+      hlfir::AssignOp::create(firOpBuilder, loc, cvtVal, lhs);
   return storeOp;
 }
 
@@ -1156,7 +1156,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
   fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder();
 
   auto insertMarker = [](fir::FirOpBuilder &builder) {
-    mlir::Value undef = builder.create<fir::UndefOp>(builder.getUnknownLoc(),
+    mlir::Value undef = fir::UndefOp::create(builder, builder.getUnknownLoc(),
                                                      builder.getIndexType());
     return undef.getDefiningOp();
   };
@@ -1271,7 +1271,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
     mlir::Block *exit = firOpBuilder.createBlock(&region);
     for (mlir::Block *b : exits) {
       firOpBuilder.setInsertionPointToEnd(b);
-      firOpBuilder.create<mlir::cf::BranchOp>(info.loc, exit);
+      mlir::cf::BranchOp::create(firOpBuilder, info.loc, exit);
     }
     return exit;
   };
@@ -1332,7 +1332,7 @@ static void genBodyOfTargetDataOp(
   // Remembering the position for further insertion is important since
   // there are hlfir.declares inserted above while setting block arguments
   // and new code from the body should be inserted after that.
-  mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
+  mlir::Value undefMarker = fir::UndefOp::create(firOpBuilder,
       dataOp.getLoc(), firOpBuilder.getIndexType());
 
   // Create blocks for unstructured regions. This has to be done since
@@ -1342,7 +1342,7 @@ static void genBodyOfTargetDataOp(
         firOpBuilder, eval.getNestedEvaluations());
   }
 
-  firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
+  mlir::omp::TerminatorOp::create(firOpBuilder, currentLocation);
 
   // Set the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
@@ -1496,7 +1496,7 @@ static void genBodyOfTargetOp(
             insertIndex, copyVal.getType(), copyVal.getLoc());
 
         firOpBuilder.setInsertionPointToStart(entryBlock);
-        auto loadOp = firOpBuilder.create<fir::LoadOp>(clonedValArg.getLoc(),
+        auto loadOp = fir::LoadOp::create(firOpBuilder, clonedValArg.getLoc(),
                                                        clonedValArg);
         val.replaceUsesWithIf(loadOp->getResult(0),
                               [entryBlock](mlir::OpOperand &use) {
@@ -1513,7 +1513,7 @@ static void genBodyOfTargetOp(
   // marker will be deleted since there are not uses.
   // In the HLFIR flow there are hlfir.declares inserted above while
   // setting block arguments.
-  mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
+  mlir::Value undefMarker = fir::UndefOp::create(firOpBuilder,
       targetOp.getLoc(), firOpBuilder.getIndexType());
 
   // Create blocks for unstructured regions. This has to be done since
@@ -1524,7 +1524,7 @@ static void genBodyOfTargetOp(
         firOpBuilder, eval.getNestedEvaluations());
   }
 
-  firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
+  mlir::omp::TerminatorOp::create(firOpBuilder, currentLocation);
 
   // Create the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
@@ -1570,7 +1570,7 @@ static OpTy genWrapperOp(lower::AbstractConverter &converter,
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
   // Create wrapper.
-  auto op = firOpBuilder.create<OpTy>(loc, clauseOps);
+  auto op = OpTy::create(firOpBuilder, loc, clauseOps);
 
   // Create entry block with arguments.
   genEntryBlock(firOpBuilder, args, op.getRegion());
@@ -1983,7 +1983,7 @@ genCriticalOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
                                 clauseOps, nameStr);
 
       mlir::OpBuilder modBuilder(mod.getBodyRegion());
-      global = modBuilder.create<mlir::omp::CriticalDeclareOp>(loc, clauseOps);
+      global = mlir::omp::CriticalDeclareOp::create(modBuilder, loc, clauseOps);
     }
     nameAttr = mlir::FlatSymbolRefAttr::get(firOpBuilder.getContext(),
                                             global.getSymName());
@@ -2201,7 +2201,7 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   }
 
   // SECTIONS construct.
-  auto sectionsOp = builder.create<mlir::omp::SectionsOp>(loc, clauseOps);
+  auto sectionsOp = mlir::omp::SectionsOp::create(builder, loc, clauseOps);
 
   // Create entry block with reduction variables as arguments.
   EntryBlockArgs args;
@@ -2277,7 +2277,7 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   // races on post-update of lastprivate variables when `nowait`
   // clause is present.
   if (clauseOps.nowait && !lastprivates.empty())
-    builder.create<mlir::omp::BarrierOp>(loc);
+    mlir::omp::BarrierOp::create(builder, loc);
 
   return sectionsOp;
 }
@@ -2429,7 +2429,7 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   };
   lower::pft::visitAllSymbols(eval, captureImplicitMap);
 
-  auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
+  auto targetOp = mlir::omp::TargetOp::create(firOpBuilder, loc, clauseOps);
 
   llvm::SmallVector<mlir::Value> hasDeviceAddrBaseValues, mapBaseValues;
   extractMappedBaseValues(clauseOps.hasDeviceAddrVars, hasDeviceAddrBaseValues);
@@ -2509,7 +2509,7 @@ static OpTy genTargetEnterExitUpdateDataOp(
   genTargetEnterExitUpdateDataClauses(converter, semaCtx, symTable, stmtCtx,
                                       item->clauses, loc, directive, clauseOps);
 
-  return firOpBuilder.create<OpTy>(loc, clauseOps);
+  return OpTy::create(firOpBuilder, loc, clauseOps);
 }
 
 static mlir::omp::TaskOp
@@ -3342,7 +3342,7 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
   auto mlirType = converter.genType(varType.declTypeSpec->derivedTypeSpec());
-  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
+  auto declMapperOp = mlir::omp::DeclareMapperOp::create(firOpBuilder,
       loc, mapperNameStr, mlirType);
   auto &region = declMapperOp.getRegion();
   firOpBuilder.createBlock(&region);
@@ -3356,7 +3356,7 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   List<Clause> clauses = makeClauses(*clauseList, semaCtx);
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processMap(loc, stmtCtx, clauseOps);
-  firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
+  mlir::omp::DeclareMapperInfoOp::create(firOpBuilder, loc, clauseOps.mapVars);
 }
 
 static void
@@ -3758,8 +3758,8 @@ mlir::Operation *Fortran::lower::genOpenMPTerminator(fir::FirOpBuilder &builder,
                                                      mlir::Location loc) {
   if (mlir::isa<mlir::omp::AtomicUpdateOp, mlir::omp::DeclareReductionOp,
                 mlir::omp::LoopNestOp>(op))
-    return builder.create<mlir::omp::YieldOp>(loc);
-  return builder.create<mlir::omp::TerminatorOp>(loc);
+    return mlir::omp::YieldOp::create(builder, loc);
+  return mlir::omp::TerminatorOp::create(builder, loc);
 }
 
 void Fortran::lower::genOpenMPConstruct(lower::AbstractConverter &converter,
@@ -3820,7 +3820,7 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
     }
     // Generate ThreadprivateOp and rebind the common block.
     mlir::Value commonThreadprivateValue =
-        firOpBuilder.create<mlir::omp::ThreadprivateOp>(
+        mlir::omp::ThreadprivateOp::create(firOpBuilder,
             currentLocation, commonValue.getType(), commonValue);
     converter.bindSymbol(*common, commonThreadprivateValue);
     // Generate the threadprivate value for the common block member.
@@ -3841,9 +3841,9 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
       global = globalInitialization(converter, firOpBuilder, sym, var,
                                     currentLocation);
 
-    mlir::Value symValue = firOpBuilder.create<fir::AddrOfOp>(
+    mlir::Value symValue = fir::AddrOfOp::create(firOpBuilder,
         currentLocation, global.resultType(), global.getSymbol());
-    symThreadprivateValue = firOpBuilder.create<mlir::omp::ThreadprivateOp>(
+    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(firOpBuilder,
         currentLocation, symValue.getType(), symValue);
   } else {
     mlir::Value symValue = converter.getSymbolAddress(sym);
@@ -3859,7 +3859,7 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
     if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
       return;
 
-    symThreadprivateValue = firOpBuilder.create<mlir::omp::ThreadprivateOp>(
+    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(firOpBuilder,
         currentLocation, symValue.getType(), symValue);
   }
 
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index b194150c0f7f0..83752fa62c891 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -115,7 +115,7 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
                 mlir::omp::VariableCaptureKind mapCaptureType, mlir::Type retTy,
                 bool partialMap, mlir::FlatSymbolRefAttr mapperId) {
   if (auto boxTy = llvm::dyn_cast<fir::BaseBoxType>(baseAddr.getType())) {
-    baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
+    baseAddr = fir::BoxAddrOp::create(builder, loc, baseAddr);
     retTy = baseAddr.getType();
   }
 
@@ -129,7 +129,7 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
     if (seqType.hasDynamicExtents())
       varType = mlir::TypeAttr::get(seqType.getEleTy());
 
-  mlir::omp::MapInfoOp op = builder.create<mlir::omp::MapInfoOp>(
+  mlir::omp::MapInfoOp op = mlir::omp::MapInfoOp::create(builder,
       loc, retTy, baseAddr, varType,
       builder.getIntegerAttr(builder.getIntegerType(64, false), mapType),
       builder.getAttr<mlir::omp::VariableCaptureKindAttr>(mapCaptureType),
@@ -195,7 +195,7 @@ static void generateArrayIndices(lower::AbstractConverter &converter,
         clauseLocation, firOpBuilder.getIndexType(), 1);
     subscript = firOpBuilder.createConvert(
         clauseLocation, firOpBuilder.getIndexType(), subscript);
-    indices.push_back(firOpBuilder.create<mlir::arith::SubIOp>(clauseLocation,
+    indices.push_back(mlir::arith::SubIOp::create(firOpBuilder, clauseLocation,
                                                                subscript, one));
   }
 }
@@ -329,7 +329,7 @@ mlir::Value createParentSymAndGenIntermediateMaps(
                              subscriptIndices, objectList[i]);
         assert(!subscriptIndices.empty() &&
                "missing expected indices for map clause");
-        curValue = firOpBuilder.create<fir::CoordinateOp>(
+        curValue = fir::CoordinateOp::create(firOpBuilder,
             clauseLocation, firOpBuilder.getRefType(arrType.getEleTy()),
             curValue, subscriptIndices);
       }
@@ -345,7 +345,7 @@ mlir::Value createParentSymAndGenIntermediateMaps(
       fir::IntOrValue idxConst = mlir::IntegerAttr::get(
           firOpBuilder.getI32Type(), indices[currentIndicesIdx]);
       mlir::Type memberTy = recordType.getType(indices[currentIndicesIdx]);
-      curValue = firOpBuilder.create<fir::CoordinateOp>(
+      curValue = fir::CoordinateOp::create(firOpBuilder,
           clauseLocation, firOpBuilder.getRefType(memberTy), curValue,
           llvm::SmallVector<fir::IntOrValue, 1>{idxConst});
 
@@ -417,7 +417,7 @@ mlir::Value createParentSymAndGenIntermediateMaps(
 
       // Load the currently accessed member, so we can continue to access
       // further segments.
-      curValue = firOpBuilder.create<fir::LoadOp>(clauseLocation, curValue);
+      curValue = fir::LoadOp::create(firOpBuilder, clauseLocation, curValue);
       currentIndicesIdx++;
     }
   }
diff --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index ae8bf0e1630aa..fc59a2414d539 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -43,7 +43,7 @@ static void genUnreachable(fir::FirOpBuilder &builder, mlir::Location loc) {
            mlir::acc::OpenACCDialect::getDialectNamespace())
     Fortran::lower::genOpenACCTerminator(builder, parentOp, loc);
   else
-    builder.create<fir::UnreachableOp>(loc);
+    fir::UnreachableOp::create(builder, loc);
   mlir::Block *newBlock = curBlock->splitBlock(builder.getInsertionPoint());
   builder.setInsertionPointToStart(newBlock);
 }
@@ -118,7 +118,7 @@ void Fortran::lower::genStopStatement(
         loc, calleeType.getInput(operands.size()), 0));
   }
 
-  builder.create<fir::CallOp>(loc, callee, operands);
+  fir::CallOp::create(builder, loc, callee, operands);
   auto blockIsUnterminated = [&builder]() {
     mlir::Block *currentBlock = builder.getBlock();
     return currentBlock->empty() ||
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
+  fir::CallOp::create(builder, loc, callee, mlir::ValueRange{});
   genUnreachable(builder, loc);
 }
 
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
+  fir::CallOp::create(builder, loc, callee, mlir::ValueRange{});
 }
 
 void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
@@ -210,7 +210,7 @@ void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(PointerAssociate)>(loc, builder);
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, func.getFunctionType(), pointer, target);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void Fortran::lower::genPointerAssociateRemapping(
@@ -229,7 +229,7 @@ void Fortran::lower::genPointerAssociateRemapping(
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, func.getFunctionType(), pointer, target, bounds, sourceFile,
       sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void Fortran::lower::genPointerAssociateLowerBounds(fir::FirOpBuilder &builder,
@@ -242,5 +242,5 @@ void Fortran::lower::genPointerAssociateLowerBounds(fir::FirOpBuilder &builder,
           loc, builder);
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, func.getFunctionType(), pointer, target, lbounds);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index c3a5b6101ce00..4c446c324699c 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -75,9 +75,9 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
                                         /*mutableProperties=*/{}};
         Fortran::lower::genDeallocateIfAllocated(converter, mutableBox, loc);
         if (isDoConcurrent)
-          builder.create<fir::YieldOp>(loc);
+          fir::YieldOp::create(builder, loc);
         else
-          builder.create<mlir::omp::YieldOp>(loc);
+          mlir::omp::YieldOp::create(builder, loc);
         return;
       }
     }
@@ -97,18 +97,18 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
         hlfir::genVariableRawAddress(loc, builder, hlfir::Entity{arg});
     mlir::Value isAllocated = builder.genIsNotNullAddr(loc, addr);
     fir::IfOp ifOp =
-        builder.create<fir::IfOp>(loc, isAllocated, /*withElseRegion=*/false);
+        fir::IfOp::create(builder, loc, isAllocated, /*withElseRegion=*/false);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
     mlir::Value cast = builder.createConvert(
         loc, fir::HeapType::get(fir::dyn_cast_ptrEleTy(addr.getType())), addr);
-    builder.create<fir::FreeMemOp>(loc, cast);
+    fir::FreeMemOp::create(builder, loc, cast);
 
     builder.setInsertionPointAfter(ifOp);
     if (isDoConcurrent)
-      builder.create<fir::YieldOp>(loc);
+      fir::YieldOp::create(builder, loc);
     else
-      builder.create<mlir::omp::YieldOp>(loc);
+      mlir::omp::YieldOp::create(builder, loc);
     return;
   }
 
@@ -122,11 +122,11 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
     auto heapTy = fir::HeapType::get(refTy.getEleTy());
     addr = builder.createConvert(loc, heapTy, addr);
 
-    builder.create<fir::FreeMemOp>(loc, addr);
+    fir::FreeMemOp::create(builder, loc, addr);
     if (isDoConcurrent)
-      builder.create<fir::YieldOp>(loc);
+      fir::YieldOp::create(builder, loc);
     else
-      builder.create<mlir::omp::YieldOp>(loc);
+      mlir::omp::YieldOp::create(builder, loc);
 
     return;
   }
@@ -172,7 +172,7 @@ fir::ShapeShiftOp Fortran::lower::getShapeShift(
       // OpenACC does
       mlir::Value dim = builder.createIntegerConstant(loc, idxTy, i);
       auto dimInfo =
-          builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, dim);
+          fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, dim);
       lbAndExtents.push_back(useDefaultLowerBounds ? one()
                                                    : dimInfo.getLowerBound());
       lbAndExtents.push_back(dimInfo.getExtent());
@@ -181,7 +181,7 @@ fir::ShapeShiftOp Fortran::lower::getShapeShift(
 
   auto shapeShiftTy = fir::ShapeShiftType::get(builder.getContext(), rank);
   auto shapeShift =
-      builder.create<fir::ShapeShiftOp>(loc, shapeShiftTy, lbAndExtents);
+      fir::ShapeShiftOp::create(builder, loc, shapeShiftTy, lbAndExtents);
   return shapeShift;
 }
 
@@ -270,7 +270,7 @@ static mlir::Value generateZeroShapeForRank(fir::FirOpBuilder &builder,
   mlir::SmallVector<mlir::Value> dims;
   dims.resize(rank, zero);
   mlir::Type shapeTy = fir::ShapeType::get(builder.getContext(), rank);
-  return builder.create<fir::ShapeOp>(loc, shapeTy, dims);
+  return fir::ShapeOp::create(builder, loc, shapeTy, dims);
 }
 
 namespace {
@@ -341,9 +341,9 @@ class PopulateInitAndCleanupRegionsHelper {
 
   void createYield(mlir::Value ret) {
     if (isDoConcurrent)
-      builder.create<fir::YieldOp>(loc, ret);
+      fir::YieldOp::create(builder, loc, ret);
     else
-      builder.create<mlir::omp::YieldOp>(loc, ret);
+      mlir::omp::YieldOp::create(builder, loc, ret);
   }
 
   void initTrivialType() {
@@ -392,9 +392,9 @@ void PopulateInitAndCleanupRegionsHelper::initBoxedPrivatePointer(
   // Just incase, do initialize the box with a null value
   mlir::Value null = builder.createNullConstant(loc, boxTy.getEleTy());
   mlir::Value nullBox;
-  nullBox = builder.create<fir::EmboxOp>(loc, boxTy, null, shape,
+  nullBox = fir::EmboxOp::create(builder, loc, boxTy, null, shape,
                                          /*slice=*/mlir::Value{}, lenParams);
-  builder.create<fir::StoreOp>(loc, nullBox, allocatedPrivVarArg);
+  fir::StoreOp::create(builder, loc, nullBox, allocatedPrivVarArg);
   createYield(allocatedPrivVarArg);
 }
 /// Check if an allocatable box is unallocated. If so, initialize the boxAlloca
@@ -410,9 +410,9 @@ void PopulateInitAndCleanupRegionsHelper::initBoxedPrivatePointer(
 /// }
 /// omp.yield %box_alloca
 fir::IfOp PopulateInitAndCleanupRegionsHelper::handleNullAllocatable() {
-  mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, getLoadedMoldArg());
+  mlir::Value addr = fir::BoxAddrOp::create(builder, loc, getLoadedMoldArg());
   mlir::Value isNotAllocated = builder.genIsNullAddr(loc, addr);
-  fir::IfOp ifOp = builder.create<fir::IfOp>(loc, isNotAllocated,
+  fir::IfOp ifOp = fir::IfOp::create(builder, loc, isNotAllocated,
                                              /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   // Just embox the null address and return.
@@ -421,9 +421,9 @@ fir::IfOp PopulateInitAndCleanupRegionsHelper::handleNullAllocatable() {
   mlir::Value shape = generateZeroShapeForRank(builder, loc, moldArg);
 
   mlir::Value nullBox =
-      builder.create<fir::EmboxOp>(loc, valType, addr, shape,
+      fir::EmboxOp::create(builder, loc, valType, addr, shape,
                                    /*slice=*/mlir::Value{}, lenParams);
-  builder.create<fir::StoreOp>(loc, nullBox, allocatedPrivVarArg);
+  fir::StoreOp::create(builder, loc, nullBox, allocatedPrivVarArg);
   return ifOp;
 }
 
@@ -442,14 +442,14 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedScalar(
                                                      /*shape=*/{}, lenParams);
   if (scalarInitValue)
     builder.createStoreWithConvert(loc, scalarInitValue, valAlloc);
-  mlir::Value box = builder.create<fir::EmboxOp>(
+  mlir::Value box = fir::EmboxOp::create(builder,
       loc, valType, valAlloc, /*shape=*/mlir::Value{},
       /*slice=*/mlir::Value{}, lenParams);
   initializeIfDerivedTypeBox(
       builder, loc, box, getLoadedMoldArg(), needsInitialization,
       /*isFirstPrivate=*/kind == DeclOperationKind::FirstPrivateOrLocalInit);
   fir::StoreOp lastOp =
-      builder.create<fir::StoreOp>(loc, box, allocatedPrivVarArg);
+      fir::StoreOp::create(builder, loc, box, allocatedPrivVarArg);
 
   createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
                       isDoConcurrent);
@@ -483,14 +483,14 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
     fir::ShapeShiftOp shape =
         getShapeShift(builder, loc, source, cannotHaveNonDefaultLowerBounds);
     mlir::Type arrayType = source.getElementOrSequenceType();
-    mlir::Value allocatedArray = builder.create<fir::AllocMemOp>(
+    mlir::Value allocatedArray = fir::AllocMemOp::create(builder,
         loc, arrayType, /*typeparams=*/mlir::ValueRange{}, shape.getExtents());
-    mlir::Value firClass = builder.create<fir::EmboxOp>(loc, source.getType(),
+    mlir::Value firClass = fir::EmboxOp::create(builder, loc, source.getType(),
                                                         allocatedArray, shape);
     initializeIfDerivedTypeBox(
         builder, loc, firClass, source, needsInitialization,
         /*isFirstprivate=*/kind == DeclOperationKind::FirstPrivateOrLocalInit);
-    builder.create<fir::StoreOp>(loc, firClass, allocatedPrivVarArg);
+    fir::StoreOp::create(builder, loc, firClass, allocatedPrivVarArg);
     if (ifUnallocated)
       builder.setInsertionPointAfter(ifUnallocated);
     createYield(allocatedPrivVarArg);
@@ -543,22 +543,22 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
   if (mlir::isa<fir::BaseBoxType>(temp.getType()))
     // the box created by the declare form createTempFromMold is missing
     // lower bounds info
-    box = builder.create<fir::ReboxOp>(loc, boxType, temp, shapeShift,
+    box = fir::ReboxOp::create(builder, loc, boxType, temp, shapeShift,
                                        /*shift=*/mlir::Value{});
   else
-    box = builder.create<fir::EmboxOp>(
+    box = fir::EmboxOp::create(builder,
         loc, boxType, temp, shapeShift,
         /*slice=*/mlir::Value{},
         /*typeParams=*/llvm::ArrayRef<mlir::Value>{});
 
   if (scalarInitValue)
-    builder.create<hlfir::AssignOp>(loc, scalarInitValue, box);
+    hlfir::AssignOp::create(builder, loc, scalarInitValue, box);
 
   initializeIfDerivedTypeBox(
       builder, loc, box, getLoadedMoldArg(), needsInitialization,
       /*isFirstPrivate=*/kind == DeclOperationKind::FirstPrivateOrLocalInit);
 
-  builder.create<fir::StoreOp>(loc, box, allocatedPrivVarArg);
+  fir::StoreOp::create(builder, loc, box, allocatedPrivVarArg);
   if (ifUnallocated)
     builder.setInsertionPointAfter(ifUnallocated);
   createYield(allocatedPrivVarArg);
@@ -596,8 +596,8 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupUnboxedDerivedType(
   builder.setInsertionPointToStart(initBlock);
   mlir::Type boxedTy = fir::BoxType::get(valType);
   mlir::Value newBox =
-      builder.create<fir::EmboxOp>(loc, boxedTy, allocatedPrivVarArg);
-  mlir::Value moldBox = builder.create<fir::EmboxOp>(loc, boxedTy, moldArg);
+      fir::EmboxOp::create(builder, loc, boxedTy, allocatedPrivVarArg);
+  mlir::Value moldBox = fir::EmboxOp::create(builder, loc, boxedTy, moldArg);
   initializeIfDerivedTypeBox(builder, loc, newBox, moldBox, needsInitialization,
                              /*isFirstPrivate=*/kind ==
                                  DeclOperationKind::FirstPrivateOrLocalInit);
diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index c0be1e229f825..ee73ead799109 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -260,19 +260,19 @@ ReductionProcessor::getReductionInitValue(mlir::Location loc, mlir::Type type,
                                                                initIm);
     }
     if (mlir::isa<mlir::FloatType>(type))
-      return builder.create<mlir::arith::ConstantOp>(
+      return mlir::arith::ConstantOp::create(builder,
           loc, type,
           builder.getFloatAttr(type, (double)getOperationIdentity(redId, loc)));
 
     if (mlir::isa<fir::LogicalType>(type)) {
-      mlir::Value intConst = builder.create<mlir::arith::ConstantOp>(
+      mlir::Value intConst = mlir::arith::ConstantOp::create(builder,
           loc, builder.getI1Type(),
           builder.getIntegerAttr(builder.getI1Type(),
                                  getOperationIdentity(redId, loc)));
       return builder.createConvert(loc, type, intConst);
     }
 
-    return builder.create<mlir::arith::ConstantOp>(
+    return mlir::arith::ConstantOp::create(builder,
         loc, type,
         builder.getIntegerAttr(type, getOperationIdentity(redId, loc)));
   case ReductionIdentifier::ID:
@@ -301,15 +301,15 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     break;
   case ReductionIdentifier::IOR:
     assert((type.isIntOrIndex()) && "only integer is expected");
-    reductionOp = builder.create<mlir::arith::OrIOp>(loc, op1, op2);
+    reductionOp = mlir::arith::OrIOp::create(builder, loc, op1, op2);
     break;
   case ReductionIdentifier::IEOR:
     assert((type.isIntOrIndex()) && "only integer is expected");
-    reductionOp = builder.create<mlir::arith::XOrIOp>(loc, op1, op2);
+    reductionOp = mlir::arith::XOrIOp::create(builder, loc, op1, op2);
     break;
   case ReductionIdentifier::IAND:
     assert((type.isIntOrIndex()) && "only integer is expected");
-    reductionOp = builder.create<mlir::arith::AndIOp>(loc, op1, op2);
+    reductionOp = mlir::arith::AndIOp::create(builder, loc, op1, op2);
     break;
   case ReductionIdentifier::ADD:
     reductionOp =
@@ -325,7 +325,7 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value andiOp = builder.create<mlir::arith::AndIOp>(loc, op1I1, op2I1);
+    mlir::Value andiOp = mlir::arith::AndIOp::create(builder, loc, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, andiOp);
     break;
@@ -334,7 +334,7 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value oriOp = builder.create<mlir::arith::OrIOp>(loc, op1I1, op2I1);
+    mlir::Value oriOp = mlir::arith::OrIOp::create(builder, loc, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, oriOp);
     break;
@@ -343,7 +343,7 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value cmpiOp = builder.create<mlir::arith::CmpIOp>(
+    mlir::Value cmpiOp = mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::eq, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, cmpiOp);
@@ -353,7 +353,7 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value cmpiOp = builder.create<mlir::arith::CmpIOp>(
+    mlir::Value cmpiOp = mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::ne, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, cmpiOp);
@@ -370,9 +370,9 @@ template <typename ParentDeclOpType>
 static void genYield(fir::FirOpBuilder &builder, mlir::Location loc,
                      mlir::Value yieldedValue) {
   if constexpr (std::is_same_v<ParentDeclOpType, mlir::omp::DeclareReductionOp>)
-    builder.create<mlir::omp::YieldOp>(loc, yieldedValue);
+    mlir::omp::YieldOp::create(builder, loc, yieldedValue);
   else
-    builder.create<fir::YieldOp>(loc, yieldedValue);
+    fir::YieldOp::create(builder, loc, yieldedValue);
 }
 
 /// Create reduction combiner region for reduction variables which are boxed
@@ -393,24 +393,24 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
 
   // load fir.ref<fir.box<...>>
   mlir::Value lhsAddr = lhs;
-  lhs = builder.create<fir::LoadOp>(loc, lhs);
-  rhs = builder.create<fir::LoadOp>(loc, rhs);
+  lhs = fir::LoadOp::create(builder, loc, lhs);
+  rhs = fir::LoadOp::create(builder, loc, rhs);
 
   if ((heapTy || ptrTy) && !seqTy) {
     // get box contents (heap pointers)
-    lhs = builder.create<fir::BoxAddrOp>(loc, lhs);
-    rhs = builder.create<fir::BoxAddrOp>(loc, rhs);
+    lhs = fir::BoxAddrOp::create(builder, loc, lhs);
+    rhs = fir::BoxAddrOp::create(builder, loc, rhs);
     mlir::Value lhsValAddr = lhs;
 
     // load heap pointers
-    lhs = builder.create<fir::LoadOp>(loc, lhs);
-    rhs = builder.create<fir::LoadOp>(loc, rhs);
+    lhs = fir::LoadOp::create(builder, loc, lhs);
+    rhs = fir::LoadOp::create(builder, loc, rhs);
 
     mlir::Type eleTy = heapTy ? heapTy.getEleTy() : ptrTy.getEleTy();
 
     mlir::Value result = ReductionProcessor::createScalarCombiner(
         builder, loc, redId, eleTy, lhs, rhs);
-    builder.create<fir::StoreOp>(loc, result, lhsValAddr);
+    fir::StoreOp::create(builder, loc, result, lhsValAddr);
     genYield<DeclRedOpType>(builder, loc, lhsAddr);
     return;
   }
@@ -437,17 +437,17 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
   builder.setInsertionPointToStart(nest.body);
   const bool seqIsVolatile = fir::isa_volatile_type(seqTy.getEleTy());
   mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy(), seqIsVolatile);
-  auto lhsEleAddr = builder.create<fir::ArrayCoorOp>(
+  auto lhsEleAddr = fir::ArrayCoorOp::create(builder,
       loc, refTy, lhs, shapeShift, /*slice=*/mlir::Value{},
       nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});
-  auto rhsEleAddr = builder.create<fir::ArrayCoorOp>(
+  auto rhsEleAddr = fir::ArrayCoorOp::create(builder,
       loc, refTy, rhs, shapeShift, /*slice=*/mlir::Value{},
       nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});
-  auto lhsEle = builder.create<fir::LoadOp>(loc, lhsEleAddr);
-  auto rhsEle = builder.create<fir::LoadOp>(loc, rhsEleAddr);
+  auto lhsEle = fir::LoadOp::create(builder, loc, lhsEleAddr);
+  auto rhsEle = fir::LoadOp::create(builder, loc, rhsEleAddr);
   mlir::Value scalarReduction = ReductionProcessor::createScalarCombiner(
       builder, loc, redId, refTy, lhsEle, rhsEle);
-  builder.create<fir::StoreOp>(loc, scalarReduction, lhsEleAddr);
+  fir::StoreOp::create(builder, loc, scalarReduction, lhsEleAddr);
 
   builder.setInsertionPointAfter(nest.outerOp);
   genYield<DeclRedOpType>(builder, loc, lhsAddr);
@@ -468,7 +468,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value result = ReductionProcessor::createScalarCombiner(
         builder, loc, redId, ty, lhsLoaded, rhsLoaded);
     if (isByRef) {
-      builder.create<fir::StoreOp>(loc, result, lhs);
+      fir::StoreOp::create(builder, loc, result, lhs);
       genYield<DeclRedOpType>(builder, loc, lhs);
     } else {
       genYield<DeclRedOpType>(builder, loc, result);
@@ -539,7 +539,7 @@ static void createReductionAllocAndInitRegions(
     if (isByRef) {
       // alloc region
       builder.setInsertionPointToEnd(allocBlock);
-      mlir::Value alloca = builder.create<fir::AllocaOp>(loc, ty);
+      mlir::Value alloca = fir::AllocaOp::create(builder, loc, ty);
       yield(alloca);
       return;
     }
@@ -551,7 +551,7 @@ static void createReductionAllocAndInitRegions(
 
   // alloc region
   builder.setInsertionPointToEnd(allocBlock);
-  mlir::Value boxAlloca = builder.create<fir::AllocaOp>(loc, ty);
+  mlir::Value boxAlloca = fir::AllocaOp::create(builder, loc, ty);
   yield(boxAlloca);
 }
 
@@ -575,7 +575,7 @@ OpType ReductionProcessor::createDeclareReduction(
   if (!isByRef)
     type = valTy;
 
-  decl = modBuilder.create<OpType>(loc, reductionOpName, type);
+  decl = OpType::create(modBuilder, loc, reductionOpName, type);
   createReductionAllocAndInitRegions(converter, loc, decl, redId, type,
                                      isByRef);
 
@@ -672,8 +672,8 @@ void ReductionProcessor::processReductionArguments(
       // Always pass the box by reference so that the OpenMP dialect
       // verifiers don't need to know anything about fir.box
       auto alloca =
-          builder.create<fir::AllocaOp>(currentLocation, box.getType());
-      builder.create<fir::StoreOp>(currentLocation, box, alloca);
+          fir::AllocaOp::create(builder, currentLocation, box.getType());
+      fir::StoreOp::create(builder, currentLocation, box, alloca);
 
       symVal = alloca;
     } else if (mlir::isa<fir::BaseBoxType>(symVal.getType())) {
@@ -683,9 +683,9 @@ void ReductionProcessor::processReductionArguments(
       auto oldIP = builder.saveInsertionPoint();
       builder.setInsertionPointToStart(builder.getAllocaBlock());
       auto alloca =
-          builder.create<fir::AllocaOp>(currentLocation, symVal.getType());
+          fir::AllocaOp::create(builder, currentLocation, symVal.getType());
       builder.restoreInsertionPoint(oldIP);
-      builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
+      fir::StoreOp::create(builder, currentLocation, symVal, alloca);
       symVal = alloca;
     }
 
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index b9d2574a76ad0..887732294060c 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -702,9 +702,9 @@ void privatizeSymbol(
     // Boxes should be passed by reference into nested regions:
     auto oldIP = firOpBuilder.saveInsertionPoint();
     firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
-    auto alloca = firOpBuilder.create<fir::AllocaOp>(symLoc, privVal.getType());
+    auto alloca = fir::AllocaOp::create(firOpBuilder, symLoc, privVal.getType());
     firOpBuilder.restoreInsertionPoint(oldIP);
-    firOpBuilder.create<fir::StoreOp>(symLoc, privVal, alloca);
+    fir::StoreOp::create(firOpBuilder, symLoc, privVal, alloca);
     privVal = alloca;
   }
 
@@ -726,12 +726,12 @@ void privatizeSymbol(
     OpType result;
 
     if constexpr (std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) {
-      result = firOpBuilder.create<OpType>(
+      result = OpType::create(firOpBuilder,
           symLoc, uniquePrivatizerName, allocType,
           emitCopyRegion ? mlir::omp::DataSharingClauseType::FirstPrivate
                          : mlir::omp::DataSharingClauseType::Private);
     } else {
-      result = firOpBuilder.create<OpType>(
+      result = OpType::create(firOpBuilder,
           symLoc, uniquePrivatizerName, allocType,
           emitCopyRegion ? fir::LocalitySpecifierType::LocalInit
                          : fir::LocalitySpecifierType::Local);
@@ -815,11 +815,11 @@ void privatizeSymbol(
       copyFirstPrivateSymbol(converter, symToPrivatize, &ip);
 
       if constexpr (std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) {
-        firOpBuilder.create<mlir::omp::YieldOp>(
+        mlir::omp::YieldOp::create(firOpBuilder,
             hsb.getAddr().getLoc(),
             symTable.shallowLookupSymbol(*symToPrivatize).getAddr());
       } else {
-        firOpBuilder.create<fir::YieldOp>(
+        fir::YieldOp::create(firOpBuilder,
             hsb.getAddr().getLoc(),
             symTable.shallowLookupSymbol(*symToPrivatize).getAddr());
       }
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index c7b3e11728cea..a859d9a2872a0 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -121,7 +121,7 @@ class VectorSubscriptBoxBuilder {
     if (recTy.getNumLenParams() != 0)
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-    componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
+    componentPath.emplace_back(fir::FieldIndexOp::create(builder,
         loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
@@ -270,15 +270,15 @@ mlir::Value Fortran::lower::VectorSubscriptBox::loopOverElementsBase(
     LoopType loop;
     if constexpr (std::is_same_v<LoopType, fir::IterWhileOp>) {
       loop =
-          builder.create<fir::IterWhileOp>(loc, lb, ub, step, initialCondition);
+          fir::IterWhileOp::create(builder, loc, lb, ub, step, initialCondition);
       initialCondition = loop.getIterateVar();
       if (!outerLoop)
         outerLoop = loop;
       else
-        builder.create<fir::ResultOp>(loc, loop.getResult(0));
+        fir::ResultOp::create(builder, loc, loop.getResult(0));
     } else {
       loop =
-          builder.create<fir::DoLoopOp>(loc, lb, ub, step, /*unordered=*/false);
+          fir::DoLoopOp::create(builder, loc, lb, ub, step, /*unordered=*/false);
       if (!outerLoop)
         outerLoop = loop;
     }
@@ -293,7 +293,7 @@ mlir::Value Fortran::lower::VectorSubscriptBox::loopOverElementsBase(
 
   if constexpr (std::is_same_v<LoopType, fir::IterWhileOp>) {
     auto res = elementalGenerator(elem);
-    builder.create<fir::ResultOp>(loc, res);
+    fir::ResultOp::create(builder, loc, res);
     builder.setInsertionPointAfter(outerLoop);
     return outerLoop.getResult(0);
   } else {
@@ -326,7 +326,7 @@ Fortran::lower::VectorSubscriptBox::createSlice(fir::FirOpBuilder &builder,
   mlir::Type idxTy = builder.getIndexType();
   llvm::SmallVector<mlir::Value> triples;
   mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-  auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+  auto undef = fir::UndefOp::create(builder, loc, idxTy);
   for (const LoweredSubscript &subscript : loweredSubscripts)
     Fortran::common::visit(Fortran::common::visitors{
                                [&](const LoweredTriplet &triplet) {
@@ -346,7 +346,7 @@ Fortran::lower::VectorSubscriptBox::createSlice(fir::FirOpBuilder &builder,
                                },
                            },
                            subscript);
-  return builder.create<fir::SliceOp>(loc, triples, componentPath);
+  return fir::SliceOp::create(builder, loc, triples, componentPath);
 }
 
 llvm::SmallVector<std::tuple<mlir::Value, mlir::Value, mlir::Value>>
@@ -369,13 +369,13 @@ Fortran::lower::VectorSubscriptBox::genLoopBounds(fir::FirOpBuilder &builder,
           builder, loc, loweredBase, dimension, one);
       baseLb = builder.createConvert(loc, idxTy, baseLb);
       lb = baseLb;
-      ub = builder.create<mlir::arith::SubIOp>(loc, idxTy, extent, one);
-      ub = builder.create<mlir::arith::AddIOp>(loc, idxTy, ub, baseLb);
+      ub = mlir::arith::SubIOp::create(builder, loc, idxTy, extent, one);
+      ub = mlir::arith::AddIOp::create(builder, loc, idxTy, ub, baseLb);
       step = one;
     } else {
       const auto &vector = std::get<LoweredVectorSubscript>(subscript);
       lb = zero;
-      ub = builder.create<mlir::arith::SubIOp>(loc, idxTy, vector.size, one);
+      ub = mlir::arith::SubIOp::create(builder, loc, idxTy, vector.size, one);
       step = one;
     }
     bounds.emplace_back(lb, ub, step);
@@ -402,10 +402,10 @@ fir::ExtendedValue Fortran::lower::VectorSubscriptBox::getElementAt(
               mlir::Type vecEleTy = fir::unwrapSequenceType(
                   fir::unwrapPassByRefType(vecBase.getType()));
               mlir::Type refTy = builder.getRefType(vecEleTy);
-              auto vecEltRef = builder.create<fir::CoordinateOp>(
+              auto vecEltRef = fir::CoordinateOp::create(builder,
                   loc, refTy, vecBase, vecIndex);
               auto vecElt =
-                  builder.create<fir::LoadOp>(loc, vecEleTy, vecEltRef);
+                  fir::LoadOp::create(builder, loc, vecEleTy, vecEltRef);
               indexes.emplace_back(builder.createConvert(loc, idxTy, vecElt));
             },
             [&](const mlir::Value &i) {
@@ -414,7 +414,7 @@ fir::ExtendedValue Fortran::lower::VectorSubscriptBox::getElementAt(
         },
         subscript);
   mlir::Type refTy = builder.getRefType(getElementType());
-  auto elementAddr = builder.create<fir::ArrayCoorOp>(
+  auto elementAddr = fir::ArrayCoorOp::create(builder,
       loc, refTy, fir::getBase(loweredBase), shape, slice, indexes,
       fir::getTypeParams(loweredBase));
   fir::ExtendedValue element = fir::factory::arraySectionElementToExtendedValue(


From flang-commits at lists.llvm.org  Mon Jul 21 14:57:37 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 14:57:37 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][NFC] update `flang/Lower` create
 APIs (8/n) (#149687) (PR #149912)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149912@github.com>
Message-ID: <687eb7d1.a70a0220.f81cd.ff2d@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions cpp -- flang/lib/Lower/Allocatable.cpp flang/lib/Lower/Bridge.cpp flang/lib/Lower/ConvertArrayConstructor.cpp flang/lib/Lower/ConvertCall.cpp flang/lib/Lower/ConvertConstant.cpp flang/lib/Lower/ConvertExpr.cpp flang/lib/Lower/ConvertExprToHLFIR.cpp flang/lib/Lower/ConvertProcedureDesignator.cpp flang/lib/Lower/ConvertVariable.cpp flang/lib/Lower/CustomIntrinsicCall.cpp flang/lib/Lower/HlfirIntrinsics.cpp flang/lib/Lower/HostAssociations.cpp flang/lib/Lower/IO.cpp flang/lib/Lower/OpenACC.cpp flang/lib/Lower/OpenMP/Atomic.cpp flang/lib/Lower/OpenMP/ClauseProcessor.cpp flang/lib/Lower/OpenMP/DataSharingProcessor.cpp flang/lib/Lower/OpenMP/OpenMP.cpp flang/lib/Lower/OpenMP/Utils.cpp flang/lib/Lower/Runtime.cpp flang/lib/Lower/Support/PrivateReductionUtils.cpp flang/lib/Lower/Support/ReductionProcessor.cpp flang/lib/Lower/Support/Utils.cpp flang/lib/Lower/VectorSubscripts.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index f6119cf8a..15cd9770b 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -78,8 +78,8 @@ struct ErrorManager {
         statExpr && errMsgExpr
             ? builder.createBox(loc,
                                 converter.genExprAddr(loc, errMsgExpr, stmtCtx))
-            : fir::AbsentOp::create(builder,
-                  loc,
+            : fir::AbsentOp::create(
+                  builder, loc,
                   fir::BoxType::get(mlir::NoneType::get(builder.getContext())));
     sourceFile = fir::factory::locationToFilename(builder, loc);
     sourceLine = fir::factory::locationToLineNo(builder, loc,
@@ -92,10 +92,10 @@ struct ErrorManager {
     if (statValue) {
       mlir::Value zero =
           builder.createIntegerConstant(loc, statValue.getType(), 0);
-      auto cmp = mlir::arith::CmpIOp::create(builder,
-          loc, mlir::arith::CmpIPredicate::eq, statValue, zero);
+      auto cmp = mlir::arith::CmpIOp::create(
+          builder, loc, mlir::arith::CmpIPredicate::eq, statValue, zero);
       auto ifOp = fir::IfOp::create(builder, loc, cmp,
-                                            /*withElseRegion=*/false);
+                                    /*withElseRegion=*/false);
       builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     }
   }
@@ -597,13 +597,14 @@ private:
       assert(sourceBox && "source expression should be lowered to one box");
       for (int i = 0; i < sourceExpr->Rank(); ++i) {
         auto dimVal = builder.createIntegerConstant(loc, idxTy, i);
-        auto dimInfo = fir::BoxDimsOp::create(builder,
-            loc, idxTy, idxTy, idxTy, sourceBox->getAddr(), dimVal);
+        auto dimInfo = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
+                                              sourceBox->getAddr(), dimVal);
         mlir::Value lb =
             fir::factory::readLowerBound(builder, loc, sourceExv, i, one);
         mlir::Value extent = dimInfo.getResult(1);
-        mlir::Value ub = mlir::arith::SubIOp::create(builder,
-            loc, mlir::arith::AddIOp::create(builder, loc, extent, lb), one);
+        mlir::Value ub = mlir::arith::SubIOp::create(
+            builder, loc, mlir::arith::AddIOp::create(builder, loc, extent, lb),
+            one);
         mlir::Value dimIndex = builder.createIntegerConstant(loc, i32Ty, i);
         genRuntimeSetBounds(builder, loc, box, dimIndex, lb, ub);
       }
@@ -909,8 +910,8 @@ void Fortran::lower::genDeallocateIfAllocated(
       .genThen([&]() {
         if (mlir::Type eleType = box.getEleTy();
             mlir::isa<fir::RecordType>(eleType) && box.isPolymorphic()) {
-          mlir::Value declaredTypeDesc = fir::TypeDescOp::create(builder,
-              loc, mlir::TypeAttr::get(eleType));
+          mlir::Value declaredTypeDesc = fir::TypeDescOp::create(
+              builder, loc, mlir::TypeAttr::get(eleType));
           genDeallocateBox(converter, box, loc, sym, declaredTypeDesc);
         } else {
           genDeallocateBox(converter, box, loc, sym);
@@ -1161,7 +1162,7 @@ mlir::Value Fortran::lower::getAssumedCharAllocatableOrPointerLen(
     // inquire the length of absent optional, even if non deferred, so
     // it's fine to use undefOp in this case.
     auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
-                                                      fir::getBase(box));
+                                              fir::getBase(box));
     mlir::Value len =
         builder.genIfOp(loc, {idxTy}, isPresent, true)
             .genThen(
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 557fdef27..b94833d85 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -333,8 +333,9 @@ private:
         if (details.numPrivatesNotOverridden() > 0)
           tbpName += "."s + std::to_string(details.numPrivatesNotOverridden());
         std::string bindingName = converter.mangleName(details.symbol());
-        fir::DTEntryOp::create(builder,
-            info.loc, mlir::StringAttr::get(builder.getContext(), tbpName),
+        fir::DTEntryOp::create(
+            builder, info.loc,
+            mlir::StringAttr::get(builder.getContext(), tbpName),
             mlir::SymbolRefAttr::get(builder.getContext(), bindingName));
       }
       fir::FirEndOp::create(builder, info.loc);
@@ -4830,17 +4831,17 @@ private:
         // Special case if the rhs is a constant.
         if (matchPattern(base.getDefiningOp(), mlir::m_Constant())) {
           cuf::DataTransferOp::create(builder, loc, base, lhsVal, shape,
-                                              transferKindAttr);
+                                      transferKindAttr);
         } else {
           auto associate = hlfir::genAssociateExpr(
               loc, builder, rhs, rhs.getType(), ".cuf_host_tmp");
           cuf::DataTransferOp::create(builder, loc, associate.getBase(), lhsVal,
-                                              shape, transferKindAttr);
+                                      shape, transferKindAttr);
           hlfir::EndAssociateOp::create(builder, loc, associate);
         }
       } else {
         cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
-                                            transferKindAttr);
+                                    transferKindAttr);
       }
       return;
     }
@@ -4850,7 +4851,7 @@ private:
       auto transferKindAttr = cuf::DataTransferKindAttr::get(
           builder.getContext(), cuf::DataTransferKind::DeviceHost);
       cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
-                                          transferKindAttr);
+                                  transferKindAttr);
       return;
     }
 
@@ -4860,7 +4861,7 @@ private:
       auto transferKindAttr = cuf::DataTransferKindAttr::get(
           builder.getContext(), cuf::DataTransferKind::DeviceDevice);
       cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
-                                          transferKindAttr);
+                                  transferKindAttr);
       return;
     }
     llvm_unreachable("Unhandled CUDA data transfer");
@@ -4906,8 +4907,9 @@ private:
           addSymbol(sym,
                     hlfir::translateToExtendedValue(loc, builder, temp).first,
                     /*forced=*/true);
-          cuf::DataTransferOp::create(builder,
-              loc, addr, temp, /*shape=*/mlir::Value{}, transferKindAttr);
+          cuf::DataTransferOp::create(builder, loc, addr, temp,
+                                      /*shape=*/mlir::Value{},
+                                      transferKindAttr);
           ++nbDeviceResidentObject;
         }
       }
@@ -4997,8 +4999,8 @@ private:
         genCUDADataTransfer(builder, loc, assign, lhs, rhs);
       else
         hlfir::AssignOp::create(builder, loc, rhs, lhs,
-                                        isWholeAllocatableAssignment,
-                                        keepLhsLengthInAllocatableAssignment);
+                                isWholeAllocatableAssignment,
+                                keepLhsLengthInAllocatableAssignment);
       if (hasCUDAImplicitTransfer && !isInDeviceContext) {
         localSymbols.popScope();
         for (mlir::Value temp : implicitTemps)
diff --git a/flang/lib/Lower/ConvertArrayConstructor.cpp b/flang/lib/Lower/ConvertArrayConstructor.cpp
index a352133b7..55c4b4555 100644
--- a/flang/lib/Lower/ConvertArrayConstructor.cpp
+++ b/flang/lib/Lower/ConvertArrayConstructor.cpp
@@ -138,8 +138,8 @@ public:
     if constexpr (!hasLoops)
       fir::emitFatalError(loc, "array constructor lowering is inconsistent");
     auto loop = fir::DoLoopOp::create(builder, loc, lower, upper, stride,
-                                              /*unordered=*/false,
-                                              /*finalCount=*/false);
+                                      /*unordered=*/false,
+                                      /*finalCount=*/false);
     builder.setInsertionPointToStart(loop.getBody());
     return loop.getInductionVar();
   }
@@ -213,13 +213,13 @@ public:
     assert(!elementalOp && "expected only one implied-do");
     mlir::Value one =
         builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-    elementalOp = hlfir::ElementalOp::create(builder,
-        loc, exprType, shape,
-        /*mold=*/nullptr, lengthParams, /*isUnordered=*/true);
+    elementalOp = hlfir::ElementalOp::create(builder, loc, exprType, shape,
+                                             /*mold=*/nullptr, lengthParams,
+                                             /*isUnordered=*/true);
     builder.setInsertionPointToStart(elementalOp.getBody());
     // implied-do-index = lower+((i-1)*stride)
-    mlir::Value diff = mlir::arith::SubIOp::create(builder,
-        loc, elementalOp.getIndices()[0], one);
+    mlir::Value diff = mlir::arith::SubIOp::create(
+        builder, loc, elementalOp.getIndices()[0], one);
     mlir::Value mul = mlir::arith::MulIOp::create(builder, loc, diff, stride);
     mlir::Value add = mlir::arith::AddIOp::create(builder, loc, lower, mul);
     return add;
@@ -315,8 +315,8 @@ public:
       mlir::Value tempStorage = builder.createHeapTemporary(
           loc, declaredType, tempName, extents, lengths);
       mlir::Value shape = builder.genShape(loc, extents);
-      declare = hlfir::DeclareOp::create(builder,
-          loc, tempStorage, tempName, shape, lengths,
+      declare = hlfir::DeclareOp::create(
+          builder, loc, tempStorage, tempName, shape, lengths,
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       initialBoxValue =
           builder.createBox(loc, boxType, declare->getOriginalBase(), shape,
@@ -390,8 +390,8 @@ public:
                              mlir::Value lower, mlir::Value upper,
                              mlir::Value stride) {
     auto loop = fir::DoLoopOp::create(builder, loc, lower, upper, stride,
-                                              /*unordered=*/false,
-                                              /*finalCount=*/false);
+                                      /*unordered=*/false,
+                                      /*finalCount=*/false);
     builder.setInsertionPointToStart(loop.getBody());
     return loop.getInductionVar();
   }
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 4c7ec7a10..8c3648bcb 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -159,8 +159,8 @@ static mlir::Value readDim3Value(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Type refI32Ty = fir::ReferenceType::get(i32Ty);
   llvm::SmallVector<mlir::Value> lenParams;
 
-  mlir::Value designate = hlfir::DesignateOp::create(builder,
-      loc, refI32Ty, dim3Addr, /*component=*/comp,
+  mlir::Value designate = hlfir::DesignateOp::create(
+      builder, loc, refI32Ty, dim3Addr, /*component=*/comp,
       /*componentShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
       /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
       mlir::Value{}, lenParams);
@@ -230,7 +230,7 @@ static mlir::Value remapActualToDummyDescriptor(
     mold = explicitArgument;
   mlir::Value remapped =
       fir::EmboxOp::create(builder, loc, dummyBoxType, baseAddr, shape,
-                                   /*slice=*/mlir::Value{}, lengths, mold);
+                           /*slice=*/mlir::Value{}, lengths, mold);
   if (mapSymbols)
     symMap.popScope();
   return remapped;
@@ -381,8 +381,8 @@ Fortran::lower::genCallOpAndResult(
 
     if (isExprCall) {
       mlir::Type exprType = hlfir::getExprType(type);
-      evaluateInMemory = hlfir::EvaluateInMemoryOp::create(builder,
-          loc, exprType, arrayResultShape, resultLengths);
+      evaluateInMemory = hlfir::EvaluateInMemoryOp::create(
+          builder, loc, exprType, arrayResultShape, resultLengths);
       builder.setInsertionPointToStart(&evaluateInMemory.getBody().front());
       return toExtendedValue(loc, evaluateInMemory.getMemory(), extents,
                              lengths);
@@ -499,8 +499,8 @@ Fortran::lower::genCallOpAndResult(
       auto funcTy = mlir::FunctionType::get(context, {}, {});
       auto boxProcTy = builder.getBoxProcType(funcTy);
       if (mlir::Value host = argumentHostAssocs(converter, fst)) {
-        cast = fir::EmboxProcOp::create(builder,
-            loc, boxProcTy, llvm::ArrayRef<mlir::Value>{fst, host});
+        cast = fir::EmboxProcOp::create(builder, loc, boxProcTy,
+                                        llvm::ArrayRef<mlir::Value>{fst, host});
       } else {
         cast = fir::EmboxProcOp::create(builder, loc, boxProcTy, fst);
       }
@@ -613,10 +613,10 @@ Fortran::lower::genCallOpAndResult(
       stream = fir::getBase(converter.genExprAddr(
           caller.getCallDescription().chevrons()[3], stmtCtx));
 
-    cuf::KernelLaunchOp::create(builder,
-        loc, funcType.getResults(), funcSymbolAttr, grid_x, grid_y, grid_z,
-        block_x, block_y, block_z, bytes, stream, operands,
-        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr);
+    cuf::KernelLaunchOp::create(builder, loc, funcType.getResults(),
+                                funcSymbolAttr, grid_x, grid_y, grid_z, block_x,
+                                block_y, block_z, bytes, stream, operands,
+                                /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr);
     callNumResults = 0;
   } else if (caller.requireDispatchCall()) {
     // Procedure call requiring a dynamic dispatch. Call is created with
@@ -640,8 +640,8 @@ Fortran::lower::genCallOpAndResult(
       // passed object because interface mismatch issues may have inserted a
       // cast to the operand with a different declared type, which would break
       // later type bound call resolution in the FIR to FIR pass.
-      dispatch = fir::DispatchOp::create(builder,
-          loc, funcType.getResults(), builder.getStringAttr(procName),
+      dispatch = fir::DispatchOp::create(
+          builder, loc, funcType.getResults(), builder.getStringAttr(procName),
           caller.getInputs()[*passArg], operands,
           builder.getI32IntegerAttr(*passArg), /*arg_attrs=*/nullptr,
           /*res_attrs=*/nullptr, procAttrs);
@@ -657,8 +657,8 @@ Fortran::lower::genCallOpAndResult(
 
       if (fir::isa_ref_type(passObject.getType()))
         passObject = fir::LoadOp::create(builder, loc, passObject);
-      dispatch = fir::DispatchOp::create(builder,
-          loc, funcType.getResults(), builder.getStringAttr(procName),
+      dispatch = fir::DispatchOp::create(
+          builder, loc, funcType.getResults(), builder.getStringAttr(procName),
           passObject, operands, nullptr, /*arg_attrs=*/nullptr,
           /*res_attrs=*/nullptr, procAttrs);
     }
@@ -667,8 +667,8 @@ Fortran::lower::genCallOpAndResult(
       callResult = dispatch.getResult(0);
   } else {
     // Standard procedure call with fir.call.
-    auto call = fir::CallOp::create(builder,
-        loc, funcType.getResults(), funcSymbolAttr, operands,
+    auto call = fir::CallOp::create(
+        builder, loc, funcType.getResults(), funcSymbolAttr, operands,
         /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, procAttrs);
 
     callNumResults = call.getNumResults();
@@ -692,8 +692,8 @@ Fortran::lower::genCallOpAndResult(
   if (caller.mustSaveResult()) {
     assert(allocatedResult.has_value());
     fir::SaveResultOp::create(builder, loc, callResult,
-                                      fir::getBase(*allocatedResult),
-                                      arrayResultShape, resultLengths);
+                              fir::getBase(*allocatedResult), arrayResultShape,
+                              resultLengths);
   }
 
   if (evaluateInMemory) {
@@ -952,8 +952,8 @@ extendedValueToHlfirEntity(mlir::Location loc, fir::FirOpBuilder &builder,
     // little incentive to increase the compiler complexity.
     hlfir::Entity storage{builder.createTemporary(loc, charTy)};
     fir::StoreOp::create(builder, loc, firBase, storage);
-    auto asExpr = hlfir::AsExprOp::create(builder,
-        loc, storage, /*mustFree=*/builder.createBool(loc, false));
+    auto asExpr = hlfir::AsExprOp::create(
+        builder, loc, storage, /*mustFree=*/builder.createBool(loc, false));
     return hlfir::EntityWithAttributes{asExpr.getResult()};
   }
   return hlfir::genDeclare(loc, builder, exv, name,
@@ -1317,8 +1317,8 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     mlir::Type tempBoxType = baseBoxTy.getBoxTypeWithNewAttr(
         fir::BaseBoxType::Attribute::Allocatable);
     mlir::Value tempBox = builder.createTemporary(loc, tempBoxType);
-    auto copyIn = hlfir::CopyInOp::create(builder,
-        loc, var, tempBox, /*var_is_present=*/mlir::Value{});
+    auto copyIn = hlfir::CopyInOp::create(builder, loc, var, tempBox,
+                                          /*var_is_present=*/mlir::Value{});
     // Register the copy-out after the call.
     preparedDummy.pushCopyInCleanUp(copyIn.getTempBox(), copyIn.getWasCopied(),
                                     doCopyOut ? copyIn.getVar()
@@ -1330,16 +1330,17 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     fir::BaseBoxType boxType = fir::BoxType::get(
         hlfir::getFortranElementOrSequenceType(dummyTypeWithActualRank));
     if (actualIsAssumedRank)
-      return hlfir::Entity{fir::ReboxAssumedRankOp::create(builder,
-          loc, boxType, var, fir::LowerBoundModifierAttribute::SetToOnes)};
+      return hlfir::Entity{fir::ReboxAssumedRankOp::create(
+          builder, loc, boxType, var,
+          fir::LowerBoundModifierAttribute::SetToOnes)};
     // Use actual shape when creating descriptor with dummy type, the dummy
     // shape may be unknown in case of sequence association.
     mlir::Type actualTy =
         hlfir::getFortranElementOrSequenceType(actual.getType());
     boxType = boxType.getBoxTypeWithNewShape(actualTy);
     return hlfir::Entity{fir::ReboxOp::create(builder, loc, boxType, var,
-                                                      /*shape=*/mlir::Value{},
-                                                      /*slice=*/mlir::Value{})};
+                                              /*shape=*/mlir::Value{},
+                                              /*slice=*/mlir::Value{})};
   };
 
   // Step 2: prepare the storage for the dummy arguments, ensuring that it
@@ -1441,14 +1442,14 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
         auto lbModifier = needsZeroLowerBounds
                               ? fir::LowerBoundModifierAttribute::SetToZeroes
                               : fir::LowerBoundModifierAttribute::SetToOnes;
-        entity = hlfir::Entity{fir::ReboxAssumedRankOp::create(builder,
-            loc, dummyTypeWithActualRank, entity, lbModifier)};
+        entity = hlfir::Entity{fir::ReboxAssumedRankOp::create(
+            builder, loc, dummyTypeWithActualRank, entity, lbModifier)};
       } else {
         mlir::Value shift{};
         if (needsZeroLowerBounds)
           shift = getZeroLowerBounds(loc, builder, entity);
-        entity = hlfir::Entity{fir::ReboxOp::create(builder,
-            loc, dummyTypeWithActualRank, entity, /*shape=*/shift,
+        entity = hlfir::Entity{fir::ReboxOp::create(
+            builder, loc, dummyTypeWithActualRank, entity, /*shape=*/shift,
             /*slice=*/mlir::Value{})};
       }
     }
@@ -1502,7 +1503,7 @@ static PreparedDummyArgument prepareUserCallActualArgument(
   // and types of the if results will depend on how the argument
   // is prepared, and forecasting that here would be brittle.
   auto badIfOp = fir::IfOp::create(builder, loc, dummyType, isPresent,
-                                           /*withElseRegion=*/false);
+                                   /*withElseRegion=*/false);
   mlir::Block *preparationBlock = &badIfOp.getThenRegion().front();
   builder.setInsertionPointToStart(preparationBlock);
   PreparedDummyArgument unconditionalDummy =
@@ -1520,9 +1521,9 @@ static PreparedDummyArgument prepareUserCallActualArgument(
   // badIfOp cannot be modified and used here).
   llvm::SmallVector<mlir::Type> ifOpResultTypes;
   ConditionallyPreparedDummy conditionalDummy(unconditionalDummy);
-  auto ifOp = fir::IfOp::create(builder, loc, conditionalDummy.getIfResulTypes(),
-                                        isPresent,
-                                        /*withElseRegion=*/true);
+  auto ifOp = fir::IfOp::create(builder, loc,
+                                conditionalDummy.getIfResulTypes(), isPresent,
+                                /*withElseRegion=*/true);
   // Move "preparationBlock" into the "then" of the new
   // fir.if operation and create fir.result propagating
   // unconditionalDummy.
@@ -1811,8 +1812,9 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals,
       // In such case, the expression should not be freed after its use since
       // the result is stack allocated or deallocation (for allocatable results)
       // was already inserted in genCallOpAndResult.
-      auto asExpr = hlfir::AsExprOp::create(builder,
-          loc, resultEntity, /*mustFree=*/builder.createBool(loc, false));
+      auto asExpr =
+          hlfir::AsExprOp::create(builder, loc, resultEntity,
+                                  /*mustFree=*/builder.createBool(loc, false));
       return hlfir::EntityWithAttributes{asExpr.getResult()};
     }
     return hlfir::EntityWithAttributes{resultEntity};
@@ -1913,8 +1915,8 @@ static ExvAndCleanup genOptionalBox(fir::FirOpBuilder &builder,
   mlir::Value box = builder.createBox(loc, newExv);
   mlir::Type boxType = box.getType();
   auto absent = fir::AbsentOp::create(builder, loc, boxType);
-  auto boxOrAbsent = mlir::arith::SelectOp::create(builder,
-      loc, boxType, isPresent, box, absent);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder, loc, boxType,
+                                                   isPresent, box, absent);
   return {fir::BoxValue(boxOrAbsent), cleanup};
 }
 
@@ -2144,8 +2146,8 @@ genIntrinsicRefCore(Fortran::lower::PreparedActualArguments &loweredActuals,
     if (intrinsicName == "merge")
       asExpr = hlfir::AsExprOp::create(builder, loc, resultEntity);
     else
-      asExpr = hlfir::AsExprOp::create(builder,
-          loc, resultEntity, builder.createBool(loc, mustBeFreed));
+      asExpr = hlfir::AsExprOp::create(builder, loc, resultEntity,
+                                       builder.createBool(loc, mustBeFreed));
     resultEntity = hlfir::EntityWithAttributes{asExpr.getResult()};
   }
   return resultEntity;
@@ -2813,9 +2815,9 @@ genProcedureRef(CallContext &callContext) {
           // TYPE(*) cannot be ALLOCATABLE/POINTER (C709) so there is no
           // need to cover the case of passing an ALLOCATABLE/POINTER to an
           // OPTIONAL.
-          isPresent =
-              fir::IsPresentOp::create(builder, loc, builder.getI1Type(), actual)
-                  .getResult();
+          isPresent = fir::IsPresentOp::create(builder, loc,
+                                               builder.getI1Type(), actual)
+                          .getResult();
         }
         loweredActuals.push_back(Fortran::lower::PreparedActualArgument{
             hlfir::Entity{*var}, isPresent});
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index cd4be3707..768a237c9 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -237,8 +237,8 @@ static mlir::Value genScalarLit(
                                     ? value.UnsignedDecimal()
                                     : value.SignedDecimal(),
                                 10);
-      return mlir::arith::ConstantOp::create(builder,
-          loc, ty, mlir::IntegerAttr::get(ty, bigInt));
+      return mlir::arith::ConstantOp::create(
+          builder, loc, ty, mlir::IntegerAttr::get(ty, bigInt));
     }
     return builder.createIntegerConstant(loc, ty, value.ToInt64());
   } else if constexpr (TC == Fortran::common::TypeCategory::Logical) {
@@ -302,8 +302,9 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     auto sizeTag = mlir::StringAttr::get(context, fir::StringLitOp::size());
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
-    return fir::StringLitOp::create(builder,
-        loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
+    return fir::StringLitOp::create(builder, loc,
+                                    llvm::ArrayRef<mlir::Type>{type},
+                                    mlir::ValueRange{}, attrs);
   }
 }
 
@@ -344,7 +345,7 @@ genScalarLit(fir::FirOpBuilder &builder, mlir::Location loc,
         },
         builder.createLinkOnceLinkage());
   return fir::AddrOfOp::create(builder, loc, global.resultType(),
-                                       global.getSymbol());
+                               global.getSymbol());
 }
 
 // Helper to generate StructureConstructor component values.
@@ -364,9 +365,9 @@ static mlir::Value genStructureComponentInit(
   auto fieldTy = fir::FieldType::get(recTy.getContext());
   assert(componentTy && "failed to retrieve component");
   // FIXME: type parameters must come from the derived-type-spec
-  auto field = fir::FieldIndexOp::create(builder,
-      loc, fieldTy, name, recTy,
-      /*typeParams=*/mlir::ValueRange{} /*TODO*/);
+  auto field =
+      fir::FieldIndexOp::create(builder, loc, fieldTy, name, recTy,
+                                /*typeParams=*/mlir::ValueRange{} /*TODO*/);
 
   if (Fortran::semantics::IsAllocatable(sym)) {
     if (!Fortran::evaluate::IsNullPointerOrAllocatable(&expr)) {
@@ -378,8 +379,8 @@ static mlir::Value genStructureComponentInit(
           fir::factory::createUnallocatedBox(builder, loc, componentTy, {})};
       componentValue = builder.createConvert(loc, componentTy, componentValue);
 
-      return fir::InsertValueOp::create(builder,
-          loc, recTy, res, componentValue,
+      return fir::InsertValueOp::create(
+          builder, loc, recTy, res, componentValue,
           builder.getArrayAttr(field.getAttributes()));
     }
   }
@@ -400,9 +401,9 @@ static mlir::Value genStructureComponentInit(
     } else
       initialTarget = Fortran::lower::genInitialDataTarget(converter, loc,
                                                            componentTy, expr);
-    res = fir::InsertValueOp::create(builder,
-        loc, recTy, res, initialTarget,
-        builder.getArrayAttr(field.getAttributes()));
+    res =
+        fir::InsertValueOp::create(builder, loc, recTy, res, initialTarget,
+                                   builder.getArrayAttr(field.getAttributes()));
     return res;
   }
 
@@ -435,24 +436,25 @@ static mlir::Value genStructureComponentInit(
     auto cPtrRecTy = mlir::cast<fir::RecordType>(componentTy);
     llvm::StringRef addrFieldName = Fortran::lower::builtin::cptrFieldName;
     mlir::Type addrFieldTy = cPtrRecTy.getType(addrFieldName);
-    auto addrField = fir::FieldIndexOp::create(builder,
-        loc, fieldTy, addrFieldName, componentTy,
+    auto addrField = fir::FieldIndexOp::create(
+        builder, loc, fieldTy, addrFieldName, componentTy,
         /*typeParams=*/mlir::ValueRange{});
     mlir::Value castAddr = builder.createConvert(loc, addrFieldTy, addr);
     auto undef = fir::UndefOp::create(builder, loc, componentTy);
-    addr = fir::InsertValueOp::create(builder,
-        loc, componentTy, undef, castAddr,
+    addr = fir::InsertValueOp::create(
+        builder, loc, componentTy, undef, castAddr,
         builder.getArrayAttr(addrField.getAttributes()));
-    res = fir::InsertValueOp::create(builder,
-        loc, recTy, res, addr, builder.getArrayAttr(field.getAttributes()));
+    res =
+        fir::InsertValueOp::create(builder, loc, recTy, res, addr,
+                                   builder.getArrayAttr(field.getAttributes()));
     return res;
   }
 
   mlir::Value val = fir::getBase(genConstantValue(converter, loc, expr));
   assert(!fir::isa_ref_type(val.getType()) && "expecting a constant value");
   mlir::Value castVal = builder.createConvert(loc, componentTy, val);
-  res = fir::InsertValueOp::create(builder,
-      loc, recTy, res, castVal, builder.getArrayAttr(field.getAttributes()));
+  res = fir::InsertValueOp::create(builder, loc, recTy, res, castVal,
+                                   builder.getArrayAttr(field.getAttributes()));
   return res;
 }
 
@@ -497,11 +499,11 @@ static mlir::Value genInlinedStructureCtorLitImpl(
     for (mlir::Type parentType : llvm::reverse(parentTypes)) {
       auto undef = fir::UndefOp::create(builder, loc, parentType);
       fir::RecordType parentRecTy = mlir::cast<fir::RecordType>(parentType);
-      auto field = fir::FieldIndexOp::create(builder,
-          loc, fieldTy, parentRecTy.getTypeList()[0].first, parentType,
+      auto field = fir::FieldIndexOp::create(
+          builder, loc, fieldTy, parentRecTy.getTypeList()[0].first, parentType,
           /*typeParams=*/mlir::ValueRange{} /*TODO*/);
-      res = fir::InsertValueOp::create(builder,
-          loc, parentRecTy, undef, res,
+      res = fir::InsertValueOp::create(
+          builder, loc, parentRecTy, undef, res,
           builder.getArrayAttr(field.getAttributes()));
     }
   };
@@ -555,7 +557,7 @@ static mlir::Value genScalarLit(
         builder.createInternalLinkage());
   }
   return fir::AddrOfOp::create(builder, loc, global.resultType(),
-                                       global.getSymbol());
+                               global.getSymbol());
 }
 
 /// Create an evaluate::Constant<T> array to a fir.array<> value
@@ -584,8 +586,9 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       mlir::Value elementVal =
           genScalarLit<T::kind>(builder, loc, con.At(subscripts), con.LEN(),
                                 /*outlineInReadOnlyMemory=*/false);
-      array = fir::InsertValueOp::create(builder,
-          loc, arrayTy, array, elementVal, builder.getArrayAttr(createIdx()));
+      array =
+          fir::InsertValueOp::create(builder, loc, arrayTy, array, elementVal,
+                                     builder.getArrayAttr(createIdx()));
     } while (con.IncrementSubscripts(subscripts));
   } else if constexpr (T::category == Fortran::common::TypeCategory::Derived) {
     do {
@@ -594,8 +597,9 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       mlir::Value elementVal =
           genScalarLit(converter, loc, con.At(subscripts), eleTy,
                        /*outlineInReadOnlyMemory=*/false);
-      array = fir::InsertValueOp::create(builder,
-          loc, arrayTy, array, elementVal, builder.getArrayAttr(createIdx()));
+      array =
+          fir::InsertValueOp::create(builder, loc, arrayTy, array, elementVal,
+                                     builder.getArrayAttr(createIdx()));
     } while (con.IncrementSubscripts(subscripts));
   } else {
     llvm::SmallVector<mlir::Attribute> rangeStartIdx;
@@ -611,9 +615,9 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       bool nextIsSame = con.IncrementSubscripts(nextSubscripts) &&
                         con.At(subscripts) == con.At(nextSubscripts);
       if (!rangeSize && !nextIsSame) { // single (non-range) value
-        array = fir::InsertValueOp::create(builder,
-            loc, arrayTy, array, getElementVal(),
-            builder.getArrayAttr(createIdx()));
+        array = fir::InsertValueOp::create(builder, loc, arrayTy, array,
+                                           getElementVal(),
+                                           builder.getArrayAttr(createIdx()));
       } else if (!rangeSize) { // start a range
         rangeStartIdx = createIdx();
         rangeSize = 1;
@@ -629,8 +633,8 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
           rangeBounds.push_back(
               mlir::cast<mlir::IntegerAttr>(idx[i]).getValue().getSExtValue());
         }
-        array = fir::InsertOnRangeOp::create(builder,
-            loc, arrayTy, array, getElementVal(),
+        array = fir::InsertOnRangeOp::create(
+            builder, loc, arrayTy, array, getElementVal(),
             builder.getIndexVectorAttr(rangeBounds));
         rangeSize = 0;
       }
@@ -684,7 +688,7 @@ genOutlineArrayLit(Fortran::lower::AbstractConverter &converter,
           builder.createInternalLinkage());
   }
   return fir::AddrOfOp::create(builder, loc, global.resultType(),
-                                       global.getSymbol());
+                               global.getSymbol());
 }
 
 /// Convert an evaluate::Constant<T> array into an fir::ExtendedValue.
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index a2fbec651..3578f941e 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -268,7 +268,7 @@ static mlir::Value genActualIsPresentTest(fir::FirOpBuilder &builder,
   // when passed to CMPLX as per 15.5.2.12 point 3 (7) and (8)). It is
   // therefore possible to catch them in the `then` case above.
   return fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
-                                          fir::getBase(actual));
+                                  fir::getBase(actual));
 }
 
 /// Convert the array_load, `load`, to an extended value. If `path` is not
@@ -346,7 +346,7 @@ arrayLoadExtValue(fir::FirOpBuilder &builder, mlir::Location loc,
     if (shapeVal) {
       // shapeVal is a ShiftOp and load.memref() is a boxed value.
       newBase = fir::ReboxOp::create(builder, loc, oldBox.getType(), oldBox,
-                                             shapeVal, /*slice=*/mlir::Value{});
+                                     shapeVal, /*slice=*/mlir::Value{});
       origins = fir::factory::getOrigins(shapeVal);
     }
     return fir::substBase(arrayToExtendedValue(extents, origins), newBase);
@@ -522,8 +522,8 @@ static fir::ExtendedValue genOptionalBox(fir::FirOpBuilder &builder,
   mlir::Value box = builder.createBox(loc, newExv);
   mlir::Type boxType = box.getType();
   auto absent = fir::AbsentOp::create(builder, loc, boxType);
-  auto boxOrAbsent = mlir::arith::SelectOp::create(builder,
-      loc, boxType, isPresent, box, absent);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder, loc, boxType,
+                                                   isPresent, box, absent);
   return fir::BoxValue(boxOrAbsent);
 }
 
@@ -573,8 +573,8 @@ createBoxProcCharTuple(Fortran::lower::AbstractConverter &converter,
 
   auto boxProc = [&]() -> mlir::Value {
     if (auto host = Fortran::lower::argumentHostAssocs(converter, funcAddr))
-      return fir::EmboxProcOp::create(builder,
-          loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
+      return fir::EmboxProcOp::create(
+          builder, loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
     return fir::EmboxProcOp::create(builder, loc, boxTy, funcAddr);
   }();
   return fir::factory::createCharacterProcedureTuple(builder, loc, argTy,
@@ -970,12 +970,12 @@ public:
 
       std::string name = converter.getRecordTypeFieldName(sym);
       // FIXME: type parameters must come from the derived-type-spec
-      mlir::Value field = fir::FieldIndexOp::create(builder,
-          loc, fieldTy, name, ty,
-          /*typeParams=*/mlir::ValueRange{} /*TODO*/);
+      mlir::Value field =
+          fir::FieldIndexOp::create(builder, loc, fieldTy, name, ty,
+                                    /*typeParams=*/mlir::ValueRange{} /*TODO*/);
       mlir::Type coorTy = builder.getRefType(recTy.getType(name));
       auto coor = fir::CoordinateOp::create(builder, loc, coorTy,
-                                                    fir::getBase(res), field);
+                                            fir::getBase(res), field);
       ExtValue to = fir::factory::componentToExtendedValue(builder, loc, coor);
       to.match(
           [&](const fir::UnboxedValue &toPtr) {
@@ -1384,8 +1384,9 @@ public:
     mlir::Value offset = builder.createIntegerConstant(
         loc, idxTy,
         x.part() == Fortran::evaluate::ComplexPart::Part::RE ? 0 : 1);
-    mlir::Value result = fir::CoordinateOp::create(builder,
-        loc, builder.getRefType(eleTy), base, mlir::ValueRange{offset});
+    mlir::Value result =
+        fir::CoordinateOp::create(builder, loc, builder.getRefType(eleTy), base,
+                                  mlir::ValueRange{offset});
     return {result};
   }
   ExtValue genval(const Fortran::evaluate::ComplexPart &x) {
@@ -1500,8 +1501,8 @@ public:
       auto recTy = mlir::cast<fir::RecordType>(ty);
       const Fortran::semantics::Symbol &sym = getLastSym(*field);
       std::string name = converter.getRecordTypeFieldName(sym);
-      coorArgs.push_back(fir::FieldIndexOp::create(builder,
-          loc, fldTy, name, recTy, fir::getTypeParams(obj)));
+      coorArgs.push_back(fir::FieldIndexOp::create(
+          builder, loc, fldTy, name, recTy, fir::getTypeParams(obj)));
       ty = recTy.getType(name);
     }
     // If parent component is referred then it has no coordinate argument.
@@ -1511,7 +1512,7 @@ public:
     return fir::factory::componentToExtendedValue(
         builder, loc,
         fir::CoordinateOp::create(builder, loc, ty, fir::getBase(obj),
-                                          coorArgs));
+                                  coorArgs));
   }
 
   ExtValue gen(const Fortran::evaluate::Component &cmpt) {
@@ -1672,8 +1673,8 @@ public:
           base = builder.createConvert(loc, seqRefTy, base);
         }
       }
-      auto coor = fir::CoordinateOp::create(builder,
-          loc, refTy, base, llvm::ArrayRef<mlir::Value>{total});
+      auto coor = fir::CoordinateOp::create(builder, loc, refTy, base,
+                                            llvm::ArrayRef<mlir::Value>{total});
       // Convert to expected, original type after address arithmetic.
       return builder.createConvert(loc, origRefTy, coor);
     };
@@ -1725,9 +1726,9 @@ public:
           builder.createConvert(loc, idxTy, fir::getBase(subVal)));
     }
     mlir::Value shape = builder.createShape(loc, exv);
-    mlir::Value elementAddr = fir::ArrayCoorOp::create(builder,
-        loc, refTy, addr, shape, /*slice=*/mlir::Value{}, arrayCoorArgs,
-        fir::getTypeParams(exv));
+    mlir::Value elementAddr = fir::ArrayCoorOp::create(
+        builder, loc, refTy, addr, shape, /*slice=*/mlir::Value{},
+        arrayCoorArgs, fir::getTypeParams(exv));
     return fir::factory::arrayElementToExtendedValue(builder, loc, exv,
                                                      elementAddr);
   }
@@ -1827,7 +1828,7 @@ public:
       auto boxProcTy =
           builder.getBoxProcType(mlir::cast<mlir::FunctionType>(exvTy));
       return fir::EmboxProcOp::create(builder, loc, boxProcTy,
-                                              fir::getBase(exv));
+                                      fir::getBase(exv));
     }
     mlir::Value box = builder.createBox(loc, exv, exv.isPolymorphic());
     if (Fortran::lower::isParentComponent(expr)) {
@@ -2073,8 +2074,8 @@ public:
       TODO(loc, "creating temporary for derived type with length parameters");
     }
 
-    mlir::Value temp = fir::AllocMemOp::create(builder,
-        loc, type, tempName, allocMemTypeParams, extents);
+    mlir::Value temp = fir::AllocMemOp::create(builder, loc, type, tempName,
+                                               allocMemTypeParams, extents);
     if (mlir::isa<fir::CharacterType>(fir::unwrapSequenceType(type)))
       return fir::CharArrayBoxValue{temp, charLen, extents};
     return fir::ArrayBoxValue{temp, extents};
@@ -2258,13 +2259,14 @@ public:
       if (isActualArgBox) {
         mlir::Value zero =
             builder.createIntegerConstant(loc, builder.getI1Type(), 0);
-        mlir::Value notContiguous = mlir::arith::CmpIOp::create(builder,
-            loc, mlir::arith::CmpIPredicate::eq, isContiguousResult, zero);
+        mlir::Value notContiguous = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::eq, isContiguousResult,
+            zero);
         if (!restrictCopyAtRuntime) {
           restrictCopyAtRuntime = notContiguous;
         } else {
-          mlir::Value cond = mlir::arith::AndIOp::create(builder,
-              loc, *restrictCopyAtRuntime, notContiguous);
+          mlir::Value cond = mlir::arith::AndIOp::create(
+              builder, loc, *restrictCopyAtRuntime, notContiguous);
           restrictCopyAtRuntime = cond;
         }
       }
@@ -2313,7 +2315,7 @@ public:
                         .genElse([&]() {
                           ExtValue temp = doCopyIn();
                           fir::ResultOp::create(builder, loc,
-                                                        fir::getBase(temp));
+                                                fir::getBase(temp));
                         })
                         .getResults()[0];
                 fir::ResultOp::create(builder, loc, addr1);
@@ -2377,8 +2379,8 @@ public:
           mlir::cast<fir::BaseBoxType>(srcBox.getType())
               .getBoxTypeWithNewAttr(fir::BaseBoxType::Attribute::Allocatable);
       srcBox = fir::ReboxOp::create(builder, loc, allocBoxTy, srcBox,
-                                            /*shift=*/mlir::Value{},
-                                            /*slice=*/mlir::Value{});
+                                    /*shift=*/mlir::Value{},
+                                    /*slice=*/mlir::Value{});
       mlir::Value srcBoxRef = builder.createTemporary(loc, srcBox.getType());
       fir::StoreOp::create(builder, loc, srcBox, srcBoxRef);
       // Create descriptor pointer to variable descriptor if copy out is needed,
@@ -2436,8 +2438,8 @@ public:
     // fir.box is absent.
     ExtValue actualArg = gen(expr);
     mlir::Value actualArgBase = fir::getBase(actualArg);
-    mlir::Value isPresent = fir::IsPresentOp::create(builder,
-        loc, builder.getI1Type(), actualArgBase);
+    mlir::Value isPresent = fir::IsPresentOp::create(
+        builder, loc, builder.getI1Type(), actualArgBase);
     if (!mlir::isa<fir::BoxType>(actualArgBase.getType()))
       return {actualArg, isPresent};
     ExtValue safeToReadBox =
@@ -2706,9 +2708,10 @@ public:
                       mlir::cast<fir::BoxCharType>(funcTy.getResult(0));
                   mlir::Value ref = builder.createConvertWithVolatileCast(
                       loc, builder.getRefType(boxTy.getEleTy()), x.getAddr());
-                  auto len = fir::UndefOp::create(builder,
-                      loc, builder.getCharacterLengthType());
-                  return fir::EmboxCharOp::create(builder, loc, boxTy, ref, len);
+                  auto len = fir::UndefOp::create(
+                      builder, loc, builder.getCharacterLengthType());
+                  return fir::EmboxCharOp::create(builder, loc, boxTy, ref,
+                                                  len);
                 }
                 return helper.createEmbox(x);
               },
@@ -2759,9 +2762,9 @@ public:
           if (isPresentValue) {
             mlir::Value convertedBox = builder.createConvert(loc, argTy, box);
             auto absent = fir::AbsentOp::create(builder, loc, argTy);
-            caller.placeInput(arg,
-                              mlir::arith::SelectOp::create(builder,
-                                  loc, *isPresentValue, convertedBox, absent));
+            caller.placeInput(
+                arg, mlir::arith::SelectOp::create(
+                         builder, loc, *isPresentValue, convertedBox, absent));
           } else {
             caller.placeInput(arg, builder.createBox(loc, argAddr));
           }
@@ -2802,13 +2805,14 @@ public:
           } else if (mlir::isa<fir::BoxType>(box.getType()) &&
                      fir::isPolymorphicType(argTy)) {
             box = fir::ReboxOp::create(builder, loc, argTy, box, mlir::Value{},
-                                               /*slice=*/mlir::Value{});
+                                       /*slice=*/mlir::Value{});
           }
 
           // Need the box types to be exactly similar for the selectOp.
           mlir::Value convertedBox = builder.createConvert(loc, argTy, box);
-          caller.placeInput(arg, mlir::arith::SelectOp::create(builder,
-                                     loc, isAllocated, convertedBox, absent));
+          caller.placeInput(
+              arg, mlir::arith::SelectOp::create(builder, loc, isAllocated,
+                                                 convertedBox, absent));
         } else {
           auto dynamicType = expr->GetType();
           mlir::Value box;
@@ -2833,8 +2837,8 @@ public:
                       fir::ResultOp::create(builder, loc, boxed);
                     })
                     .genElse([&]() {
-                      auto absent =
-                          fir::AbsentOp::create(builder, loc, argTy).getResult();
+                      auto absent = fir::AbsentOp::create(builder, loc, argTy)
+                                        .getResult();
                       fir::ResultOp::create(builder, loc, absent);
                     })
                     .getResults()[0];
@@ -2878,8 +2882,8 @@ public:
                           .getResults()[0];
               } else {
                 box = fir::ReboxOp::create(builder, loc, actualTy, box,
-                                                   mlir::Value{},
-                                                   /*slice=*/mlir::Value{});
+                                           mlir::Value{},
+                                           /*slice=*/mlir::Value{});
               }
             } else if (Fortran::lower::isParentComponent(*expr)) {
               fir::ExtendedValue newExv =
@@ -3188,16 +3192,16 @@ createDerivedArrayAmend(mlir::Location loc, fir::ArrayLoadOp destLoad,
   } else {
     auto boxTy = fir::BoxType::get(eleTy);
     auto toBox = fir::EmboxOp::create(builder, loc, boxTy, destAcc.getResult(),
-                                              mlir::Value{}, mlir::Value{},
-                                              destLoad.getTypeparams());
-    auto fromBox = fir::EmboxOp::create(builder,
-        loc, boxTy, fir::getBase(elementExv), mlir::Value{}, mlir::Value{},
-        destLoad.getTypeparams());
+                                      mlir::Value{}, mlir::Value{},
+                                      destLoad.getTypeparams());
+    auto fromBox = fir::EmboxOp::create(
+        builder, loc, boxTy, fir::getBase(elementExv), mlir::Value{},
+        mlir::Value{}, destLoad.getTypeparams());
     fir::factory::genRecordAssignment(builder, loc, fir::BoxValue(toBox),
                                       fir::BoxValue(fromBox));
   }
   return fir::ArrayAmendOp::create(builder, loc, innerArg.getType(), innerArg,
-                                           destAcc);
+                                   destAcc);
 }
 
 inline static fir::ArrayAmendOp
@@ -3317,8 +3321,8 @@ public:
       explicitSpace->finalizeContext();
       fir::ResultOp::create(builder, loc, fir::getBase(exv));
     } else {
-      fir::ArrayMergeStoreOp::create(builder,
-          loc, destination, fir::getBase(exv), destination.getMemref(),
+      fir::ArrayMergeStoreOp::create(
+          builder, loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     }
   }
@@ -3432,8 +3436,8 @@ public:
     assert(destination && "destination must have been set");
     ExtValue exv = lowerArrayExpression(rhsCC, destination.getType());
     if (!explicitSpaceIsActive())
-      fir::ArrayMergeStoreOp::create(builder,
-          loc, destination, fir::getBase(exv), destination.getMemref(),
+      fir::ArrayMergeStoreOp::create(
+          builder, loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     // destShape may originally be null, if rhs did not define a shape.
     // In this case the destShape is computed from lhs, and we may have
@@ -3548,8 +3552,8 @@ public:
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     mlir::Location loc = converter.getCurrentLocation();
     fir::ArrayMergeStoreOp::create(builder, loc, dest, fir::getBase(loopRes),
-                                           tempRes, dest.getSlice(),
-                                           dest.getTypeparams());
+                                   tempRes, dest.getSlice(),
+                                   dest.getTypeparams());
 
     auto arrTy = mlir::cast<fir::SequenceType>(
         fir::dyn_cast_ptrEleTy(tempRes.getType()));
@@ -3595,8 +3599,8 @@ public:
     // as there isn't any necessity for it.
     ccLoadDest = [=](llvm::ArrayRef<mlir::Value> shape) -> fir::ArrayLoadOp {
       mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-      auto var = fir::CoordinateOp::create(builder,
-          loc, builder.getRefType(hdrTy.getType(1)), header, one);
+      auto var = fir::CoordinateOp::create(
+          builder, loc, builder.getRefType(hdrTy.getType(1)), header, one);
       auto load = fir::LoadOp::create(builder, loc, var);
       mlir::Type eleTy =
           fir::unwrapSequenceType(fir::unwrapRefType(load.getType()));
@@ -3605,15 +3609,15 @@ public:
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
       return fir::ArrayLoadOp::create(builder, loc, seqTy, castTo, shapeOp,
-                                              /*slice=*/mlir::Value{},
-                                              mlir::ValueRange{});
+                                      /*slice=*/mlir::Value{},
+                                      mlir::ValueRange{});
     };
     // Custom lowering of the element store to deal with the extra indirection
     // to the lazy allocated buffer.
     ccStoreToDest = [=](IterSpace iters) {
       mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-      auto var = fir::CoordinateOp::create(builder,
-          loc, builder.getRefType(hdrTy.getType(1)), header, one);
+      auto var = fir::CoordinateOp::create(
+          builder, loc, builder.getRefType(hdrTy.getType(1)), header, one);
       auto load = fir::LoadOp::create(builder, loc, var);
       mlir::Type eleTy =
           fir::unwrapSequenceType(fir::unwrapRefType(load.getType()));
@@ -3623,8 +3627,8 @@ public:
       mlir::Value shape = builder.genShape(loc, genIterationShape());
       llvm::SmallVector<mlir::Value> indices = fir::factory::originateIndices(
           loc, builder, castTo.getType(), shape, iters.iterVec());
-      auto eleAddr = fir::ArrayCoorOp::create(builder,
-          loc, builder.getRefType(eleTy), castTo, shape,
+      auto eleAddr = fir::ArrayCoorOp::create(
+          builder, loc, builder.getRefType(eleTy), castTo, shape,
           /*slice=*/mlir::Value{}, indices, destination.getTypeparams());
       mlir::Value eleVal =
           builder.createConvert(loc, eleTy, iters.getElement());
@@ -3686,8 +3690,8 @@ public:
       explicitSpace->finalizeContext();
       fir::ResultOp::create(builder, loc, fir::getBase(exv));
     } else {
-      fir::ArrayMergeStoreOp::create(builder,
-          loc, destination, fir::getBase(exv), destination.getMemref(),
+      fir::ArrayMergeStoreOp::create(
+          builder, loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     }
   }
@@ -4009,7 +4013,7 @@ private:
         auto shiftTy = fir::ShiftType::get(eleTy.getContext(), lbs.size());
         mlir::Value shiftOp = fir::ShiftOp::create(builder, loc, shiftTy, lbs);
         val = fir::ReboxOp::create(builder, loc, eleTy, val, shiftOp,
-                                           mlir::Value{});
+                                   mlir::Value{});
       }
     } else if (isBoundsRemap()) {
       assert(lbounds.has_value());
@@ -4022,7 +4026,7 @@ private:
         mlir::Value shapeShift =
             fir::ShapeShiftOp::create(builder, loc, shapeTy, shapeShiftArgs);
         val = fir::ReboxOp::create(builder, loc, eleTy, val, shapeShift,
-                                           mlir::Value{});
+                                   mlir::Value{});
       }
     }
     return val;
@@ -4045,8 +4049,8 @@ private:
         // memory into the destination array.
         mlir::Type resRefTy = builder.getRefType(eleTy);
         // Get a reference to the array element to be amended.
-        auto arrayOp = fir::ArrayAccessOp::create(builder,
-            loc, resRefTy, innerArg, iterSpace.iterVec(),
+        auto arrayOp = fir::ArrayAccessOp::create(
+            builder, loc, resRefTy, innerArg, iterSpace.iterVec(),
             fir::factory::getTypeParams(loc, builder, destination));
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
           llvm::SmallVector<mlir::Value> substringBounds;
@@ -4067,9 +4071,9 @@ private:
       }
       // By value semantics. The element is being assigned by value.
       auto ele = convertElementForUpdate(loc, eleTy, fir::getBase(exv));
-      auto update = fir::ArrayUpdateOp::create(builder,
-          loc, arrTy, innerArg, ele, iterSpace.iterVec(),
-          destination.getTypeparams());
+      auto update = fir::ArrayUpdateOp::create(builder, loc, arrTy, innerArg,
+                                               ele, iterSpace.iterVec(),
+                                               destination.getTypeparams());
       return abstractArrayExtValue(update);
     };
   }
@@ -4247,9 +4251,10 @@ private:
       // Compute the dynamic position into the header.
       llvm::SmallVector<mlir::Value> offsets;
       for (auto doLoop : loopStack[i]) {
-        auto m = mlir::arith::SubIOp::create(builder,
-            loc, doLoop.getInductionVar(), doLoop.getLowerBound());
-        auto n = mlir::arith::DivSIOp::create(builder, loc, m, doLoop.getStep());
+        auto m = mlir::arith::SubIOp::create(
+            builder, loc, doLoop.getInductionVar(), doLoop.getLowerBound());
+        auto n =
+            mlir::arith::DivSIOp::create(builder, loc, m, doLoop.getStep());
         mlir::Value one = builder.createIntegerConstant(loc, n.getType(), 1);
         offsets.push_back(mlir::arith::AddIOp::create(builder, loc, n, one));
       }
@@ -4262,15 +4267,17 @@ private:
       auto ldHdr = fir::LoadOp::create(builder, loc, hdOff);
       mlir::Value hdArr = builder.createConvert(loc, toRefTy, ldHdr);
       auto shapeOp = builder.genShape(loc, extents);
-      header = fir::ArrayCoorOp::create(builder,
-          loc, builder.getRefType(raggedTy), hdArr, shapeOp,
+      header = fir::ArrayCoorOp::create(
+          builder, loc, builder.getRefType(raggedTy), hdArr, shapeOp,
           /*slice=*/mlir::Value{}, offsets,
           /*typeparams=*/mlir::ValueRange{});
-      auto hdrVar = fir::CoordinateOp::create(builder, loc, coorTy, header, uno);
+      auto hdrVar =
+          fir::CoordinateOp::create(builder, loc, coorTy, header, uno);
       auto inVar = fir::LoadOp::create(builder, loc, hdrVar);
       mlir::Value two = builder.createIntegerConstant(loc, i32Ty, 2);
       mlir::Type coorTy2 = builder.getRefType(raggedTy.getType(2));
-      auto hdrSh = fir::CoordinateOp::create(builder, loc, coorTy2, header, two);
+      auto hdrSh =
+          fir::CoordinateOp::create(builder, loc, coorTy2, header, two);
       auto shapePtr = fir::LoadOp::create(builder, loc, hdrSh);
       // Replace the binding.
       implicitSpace->rebind(expr, genMaskAccess(inVar, shapePtr));
@@ -4327,8 +4334,9 @@ private:
               fir::factory::getRaggedArrayHeaderType(builder);
           mlir::IntegerType i32Ty = builder.getIntegerType(32);
           mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-          auto coor1 = fir::CoordinateOp::create(builder,
-              loc, builder.getRefType(raggedTy.getType(1)), header, one);
+          auto coor1 = fir::CoordinateOp::create(
+              builder, loc, builder.getRefType(raggedTy.getType(1)), header,
+              one);
           auto db = fir::LoadOp::create(builder, loc, coor1);
           mlir::Type eleTy =
               fir::unwrapSequenceType(fir::unwrapRefType(db.getType()));
@@ -4338,16 +4346,17 @@ private:
           mlir::Value buff = builder.createConvert(loc, buffTy, db);
 
           mlir::Value two = builder.createIntegerConstant(loc, i32Ty, 2);
-          auto coor2 = fir::CoordinateOp::create(builder,
-              loc, builder.getRefType(raggedTy.getType(2)), header, two);
+          auto coor2 = fir::CoordinateOp::create(
+              builder, loc, builder.getRefType(raggedTy.getType(2)), header,
+              two);
           auto shBuff = fir::LoadOp::create(builder, loc, coor2);
           mlir::IntegerType i64Ty = builder.getIntegerType(64);
           mlir::IndexType idxTy = builder.getIndexType();
           llvm::SmallVector<mlir::Value> extents;
           for (std::remove_const_t<decltype(rank)> i = 0; i < rank; ++i) {
             mlir::Value off = builder.createIntegerConstant(loc, i32Ty, i);
-            auto coor = fir::CoordinateOp::create(builder,
-                loc, builder.getRefType(i64Ty), shBuff, off);
+            auto coor = fir::CoordinateOp::create(
+                builder, loc, builder.getRefType(i64Ty), shBuff, off);
             auto ldExt = fir::LoadOp::create(builder, loc, coor);
             extents.push_back(builder.createConvert(loc, idxTy, ldExt));
           }
@@ -4391,16 +4400,16 @@ private:
       }
       fir::DoLoopOp loop;
       if (innerArg) {
-        loop = fir::DoLoopOp::create(builder,
-            loc, zero, i.value(), one, isUnordered(),
+        loop = fir::DoLoopOp::create(
+            builder, loc, zero, i.value(), one, isUnordered(),
             /*finalCount=*/false, mlir::ValueRange{innerArg});
         innerArg = loop.getRegionIterArgs().front();
         if (explicitSpaceIsActive())
           explicitSpace->setInnerArg(0, innerArg);
       } else {
         loop = fir::DoLoopOp::create(builder, loc, zero, i.value(), one,
-                                             isUnordered(),
-                                             /*finalCount=*/false);
+                                     isUnordered(),
+                                     /*finalCount=*/false);
       }
       ivars.push_back(loop.getInductionVar());
       loops.push_back(loop);
@@ -4468,18 +4477,20 @@ private:
            implicitSpace->getMasks()) {
         const std::size_t size = maskExprs.size() - 1;
         auto genFalseBlock = [&](const auto *e, auto &&cond) {
-          auto ifOp = fir::IfOp::create(builder,
-              loc, mlir::TypeRange{innerArg.getType()}, fir::getBase(cond),
-              /*withElseRegion=*/true);
+          auto ifOp = fir::IfOp::create(builder, loc,
+                                        mlir::TypeRange{innerArg.getType()},
+                                        fir::getBase(cond),
+                                        /*withElseRegion=*/true);
           fir::ResultOp::create(builder, loc, ifOp.getResult(0));
           builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
           fir::ResultOp::create(builder, loc, innerArg);
           builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
         };
         auto genTrueBlock = [&](const auto *e, auto &&cond) {
-          auto ifOp = fir::IfOp::create(builder,
-              loc, mlir::TypeRange{innerArg.getType()}, fir::getBase(cond),
-              /*withElseRegion=*/true);
+          auto ifOp = fir::IfOp::create(builder, loc,
+                                        mlir::TypeRange{innerArg.getType()},
+                                        fir::getBase(cond),
+                                        /*withElseRegion=*/true);
           fir::ResultOp::create(builder, loc, ifOp.getResult(0));
           builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
           fir::ResultOp::create(builder, loc, innerArg);
@@ -4527,14 +4538,14 @@ private:
         !seqTy.hasDynamicExtents()
             ? fir::AllocMemOp::create(builder, loc, type)
             : fir::AllocMemOp::create(builder, loc, type, ".array.expr",
-                                              mlir::ValueRange{}, shape);
+                                      mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
     return fir::ArrayLoadOp::create(builder, loc, seqTy, temp, shapeOp,
-                                            /*slice=*/mlir::Value{},
-                                            mlir::ValueRange{});
+                                    /*slice=*/mlir::Value{},
+                                    mlir::ValueRange{});
   }
 
   static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -4883,14 +4894,14 @@ private:
                 fir::dyn_cast_ptrOrBoxEleTy(fir::getBase(exv).getType());
             mlir::Type innerTy = fir::unwrapSequenceType(baseTy);
             operands.emplace_back([=](IterSpace iters) -> ExtValue {
-              mlir::Value coord = fir::CoordinateOp::create(builder,
-                  loc, fir::ReferenceType::get(innerTy), fir::getBase(exv),
-                  iters.iterVec());
+              mlir::Value coord = fir::CoordinateOp::create(
+                  builder, loc, fir::ReferenceType::get(innerTy),
+                  fir::getBase(exv), iters.iterVec());
               mlir::Value empty;
               mlir::ValueRange emptyRange;
-              return fir::EmboxOp::create(builder,
-                  loc, fir::ClassType::get(innerTy), coord, empty, empty,
-                  emptyRange, sourceBox);
+              return fir::EmboxOp::create(builder, loc,
+                                          fir::ClassType::get(innerTy), coord,
+                                          empty, empty, emptyRange, sourceBox);
             });
           } else {
             ExtValue exv = asScalarRef(*expr);
@@ -4903,9 +4914,9 @@ private:
               operands.emplace_back([=](IterSpace iters) -> ExtValue {
                 mlir::Value empty;
                 mlir::ValueRange emptyRange;
-                return fir::EmboxOp::create(builder,
-                    loc, fir::ClassType::get(baseTy), fir::getBase(exv), empty,
-                    empty, emptyRange);
+                return fir::EmboxOp::create(
+                    builder, loc, fir::ClassType::get(baseTy),
+                    fir::getBase(exv), empty, empty, emptyRange);
               });
             }
           }
@@ -5097,8 +5108,8 @@ private:
     return exv.match(
         [&](const fir::CharBoxValue &cb) -> ExtValue {
           mlir::Value len = cb.getLen();
-          auto mem =
-              fir::AllocaOp::create(builder, loc, toType, mlir::ValueRange{len});
+          auto mem = fir::AllocaOp::create(builder, loc, toType,
+                                           mlir::ValueRange{len});
           fir::CharBoxValue result(mem, len);
           fir::factory::CharacterExprHelper{builder, loc}.createAssign(
               ExtValue{result}, exv);
@@ -5469,11 +5480,11 @@ private:
     if (destShape[0] != savedDestShape[dim]) {
       // Not the same, so choose the smaller value.
       mlir::Location loc = getLoc();
-      auto cmp = mlir::arith::CmpIOp::create(builder,
-          loc, mlir::arith::CmpIPredicate::sgt, destShape[0],
-          savedDestShape[dim]);
-      auto sel = mlir::arith::SelectOp::create(builder,
-          loc, cmp, savedDestShape[dim], destShape[0]);
+      auto cmp = mlir::arith::CmpIOp::create(builder, loc,
+                                             mlir::arith::CmpIPredicate::sgt,
+                                             destShape[0], savedDestShape[dim]);
+      auto sel = mlir::arith::SelectOp::create(
+          builder, loc, cmp, savedDestShape[dim], destShape[0]);
       savedDestShape[dim] = sel;
       destShape = savedDestShape;
     }
@@ -5549,12 +5560,12 @@ private:
                     // FIXME: must use the lower bound of this component.
                     auto arrLowerBound =
                         atBase ? getLBound(arrayExv, subsIndex, one) : one;
-                    auto initial = mlir::arith::SubIOp::create(builder,
-                        loc, lowerBound, arrLowerBound);
-                    auto prod = mlir::arith::MulIOp::create(builder,
-                        loc, impliedIter, stride);
-                    auto result =
-                        mlir::arith::AddIOp::create(builder, loc, initial, prod);
+                    auto initial = mlir::arith::SubIOp::create(
+                        builder, loc, lowerBound, arrLowerBound);
+                    auto prod = mlir::arith::MulIOp::create(
+                        builder, loc, impliedIter, stride);
+                    auto result = mlir::arith::AddIOp::create(builder, loc,
+                                                              initial, prod);
                     newIters.setIndexValue(subsIndex, result);
                     return newIters;
                   };
@@ -5588,8 +5599,8 @@ private:
                     // using the base array's lower bound value.
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
-                    auto origin = mlir::arith::SubIOp::create(builder,
-                        loc, idxTy, val, lb);
+                    auto origin = mlir::arith::SubIOp::create(builder, loc,
+                                                              idxTy, val, lb);
                     newIters.setIndexValue(subsIndex, origin);
                     return newIters;
                   };
@@ -5623,8 +5634,8 @@ private:
                     // Normalize `e` by subtracting the declared lbound.
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
-                    mlir::Value ivAdj =
-                        mlir::arith::SubIOp::create(builder, loc, idxTy, iv, lb);
+                    mlir::Value ivAdj = mlir::arith::SubIOp::create(
+                        builder, loc, idxTy, iv, lb);
                     // Add lbound adjusted value of `e` to the iteration vector
                     // (except when creating a box because the iteration vector
                     // is empty).
@@ -5642,7 +5653,7 @@ private:
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
                     result = mlir::arith::SubIOp::create(builder, loc, idxTy,
-                                                                 result, lb);
+                                                         result, lb);
                     pc = [=](IterSpace iters) {
                       IterationSpace newIters = currentPC(iters);
                       newIters.insertIndexValue(subsIndex, result);
@@ -5770,14 +5781,14 @@ private:
         mlir::Value zero = builder.createIntegerConstant(loc, iTy, 0);
         auto size =
             mlir::arith::SubIOp::create(builder, loc, cast, substringBounds[0]);
-        auto cmp = mlir::arith::CmpIOp::create(builder,
-            loc, mlir::arith::CmpIPredicate::sgt, size, zero);
+        auto cmp = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::sgt, size, zero);
         // size = MAX(upper - (lower - 1), 0)
         substringBounds[1] =
             mlir::arith::SelectOp::create(builder, loc, cmp, size, zero);
-        slice = fir::SliceOp::create(builder,
-            loc, padSlice(components.trips, shape), components.suffixComponents,
-            substringBounds);
+        slice = fir::SliceOp::create(
+            builder, loc, padSlice(components.trips, shape),
+            components.suffixComponents, substringBounds);
       } else {
         slice = builder.createSlice(loc, extMemref, components.trips,
                                     components.suffixComponents);
@@ -5849,8 +5860,8 @@ private:
         // ArrayCoorOp does not expect zero based indices.
         llvm::SmallVector<mlir::Value> indices = fir::factory::originateIndices(
             loc, builder, memref.getType(), shape, iters.iterVec());
-        mlir::Value coor = fir::ArrayCoorOp::create(builder,
-            loc, refEleTy, memref, shape, slice, indices,
+        mlir::Value coor = fir::ArrayCoorOp::create(
+            builder, loc, refEleTy, memref, shape, slice, indices,
             fir::getTypeParams(extMemref));
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
           llvm::SmallVector<mlir::Value> substringBounds;
@@ -5869,16 +5880,17 @@ private:
             builder, loc, extMemref, coor, slice);
       };
     }
-    auto arrLoad = fir::ArrayLoadOp::create(builder,
-        loc, arrTy, memref, shape, slice, fir::getTypeParams(extMemref));
+    auto arrLoad =
+        fir::ArrayLoadOp::create(builder, loc, arrTy, memref, shape, slice,
+                                 fir::getTypeParams(extMemref));
 
     if (CrayPtr) {
       mlir::Type ptrTy = CrayPtr.getType();
       mlir::Value cnvrt = Fortran::lower::addCrayPointerInst(
           loc, builder, CrayPtr, ptrTy, memref.getType());
       auto addr = fir::LoadOp::create(builder, loc, cnvrt);
-      arrLoad = fir::ArrayLoadOp::create(builder, loc, arrTy, addr, shape, slice,
-                                                 fir::getTypeParams(extMemref));
+      arrLoad = fir::ArrayLoadOp::create(builder, loc, arrTy, addr, shape,
+                                         slice, fir::getTypeParams(extMemref));
     }
 
     mlir::Value arrLd = arrLoad.getResult();
@@ -5906,9 +5918,9 @@ private:
         mlir::Type eleTy = fir::applyPathToType(resTy, iters.iterVec());
         mlir::Type refEleTy =
             fir::isa_ref_type(eleTy) ? eleTy : builder.getRefType(eleTy);
-        auto arrModify = fir::ArrayModifyOp::create(builder,
-            loc, mlir::TypeRange{refEleTy, resTy}, innerArg, iters.iterVec(),
-            destination.getTypeparams());
+        auto arrModify = fir::ArrayModifyOp::create(
+            builder, loc, mlir::TypeRange{refEleTy, resTy}, innerArg,
+            iters.iterVec(), destination.getTypeparams());
         return abstractArrayExtValue(arrModify.getResult(1));
       };
     }
@@ -5932,11 +5944,11 @@ private:
         mlir::Type eleTy = fir::applyPathToType(arrTy, iters.iterVec());
         if (isAdjustedArrayElementType(eleTy)) {
           mlir::Type eleRefTy = builder.getRefType(eleTy);
-          base = fir::ArrayAccessOp::create(builder,
-              loc, eleRefTy, arrLd, iters.iterVec(), arrLdTypeParams);
+          base = fir::ArrayAccessOp::create(builder, loc, eleRefTy, arrLd,
+                                            iters.iterVec(), arrLdTypeParams);
         } else {
-          base = fir::ArrayFetchOp::create(builder,
-              loc, eleTy, arrLd, iters.iterVec(), arrLdTypeParams);
+          base = fir::ArrayFetchOp::create(builder, loc, eleTy, arrLd,
+                                           iters.iterVec(), arrLdTypeParams);
         }
         mlir::Value temp =
             builder.createTemporary(loc, base.getType(),
@@ -5953,8 +5965,8 @@ private:
       mlir::Type eleTy = fir::applyPathToType(arrTy, iters.iterVec());
       if (isAdjustedArrayElementType(eleTy)) {
         mlir::Type eleRefTy = builder.getRefType(eleTy);
-        mlir::Value arrayOp = fir::ArrayAccessOp::create(builder,
-            loc, eleRefTy, arrLd, iters.iterVec(), arrLdTypeParams);
+        mlir::Value arrayOp = fir::ArrayAccessOp::create(
+            builder, loc, eleRefTy, arrLd, iters.iterVec(), arrLdTypeParams);
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
           llvm::SmallVector<mlir::Value> substringBounds;
           populateBounds(substringBounds, components.substring);
@@ -5969,8 +5981,8 @@ private:
         return fir::factory::arraySectionElementToExtendedValue(
             builder, loc, extMemref, arrayOp, slice);
       }
-      auto arrFetch = fir::ArrayFetchOp::create(builder,
-          loc, eleTy, arrLd, iters.iterVec(), arrLdTypeParams);
+      auto arrFetch = fir::ArrayFetchOp::create(
+          builder, loc, eleTy, arrLd, iters.iterVec(), arrLdTypeParams);
       return fir::factory::arraySectionElementToExtendedValue(
           builder, loc, extMemref, arrFetch, slice);
     };
@@ -6009,8 +6021,8 @@ private:
     mlir::Value memref = fir::getBase(exv);
     mlir::Value shape = builder.createShape(loc, exv);
     mlir::Value noSlice;
-    auto arrLoad = fir::ArrayLoadOp::create(builder,
-        loc, arrType, memref, shape, noSlice, fir::getTypeParams(exv));
+    auto arrLoad = fir::ArrayLoadOp::create(
+        builder, loc, arrType, memref, shape, noSlice, fir::getTypeParams(exv));
     mlir::Operation::operand_range arrLdTypeParams = arrLoad.getTypeparams();
     mlir::Value arrLd = arrLoad.getResult();
     // Mark the load to tell later passes it is unsafe to use this array_load
@@ -6025,8 +6037,8 @@ private:
 
     // By value semantics.
     auto cc = [=](IterSpace iters) -> ExtValue {
-      auto arrFetch = fir::ArrayFetchOp::create(builder,
-          loc, eleType, arrLd, iters.iterVec(), arrLdTypeParams);
+      auto arrFetch = fir::ArrayFetchOp::create(
+          builder, loc, eleType, arrLd, iters.iterVec(), arrLdTypeParams);
       return fir::factory::arraySectionElementToExtendedValue(
           builder, loc, exv, arrFetch, noSlice);
     };
@@ -6178,8 +6190,8 @@ private:
     mlir::Type eleRefTy = builder.getRefType(eleTy);
     mlir::Type resRefTy = builder.getRefType(resTy);
     mlir::Value nullPtr = builder.createNullConstant(loc, resRefTy);
-    auto offset = fir::CoordinateOp::create(builder,
-        loc, eleRefTy, nullPtr, mlir::ValueRange{multiplier});
+    auto offset = fir::CoordinateOp::create(builder, loc, eleRefTy, nullPtr,
+                                            mlir::ValueRange{multiplier});
     return builder.createConvert(loc, idxTy, offset);
   }
 
@@ -6194,7 +6206,7 @@ private:
   void createCallMemcpy(llvm::ArrayRef<mlir::Value> args, bool isVolatile) {
     mlir::Location loc = getLoc();
     mlir::LLVM::MemcpyOp::create(builder, loc, args[0], args[1], args[2],
-                                         isVolatile);
+                                 isVolatile);
   }
 
   // Construct code to check for a buffer overrun and realloc the buffer when
@@ -6204,10 +6216,10 @@ private:
                          mlir::Value eleSz) {
     mlir::Location loc = getLoc();
     mlir::func::FuncOp reallocFunc = fir::factory::getRealloc(builder);
-    auto cond = mlir::arith::CmpIOp::create(builder,
-        loc, mlir::arith::CmpIPredicate::sle, bufferSize, needed);
+    auto cond = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::sle, bufferSize, needed);
     auto ifOp = fir::IfOp::create(builder, loc, mem.getType(), cond,
-                                          /*withElseRegion=*/true);
+                                  /*withElseRegion=*/true);
     auto insPt = builder.saveInsertionPoint();
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     // Not enough space, resize the buffer.
@@ -6215,12 +6227,13 @@ private:
     mlir::Value two = builder.createIntegerConstant(loc, idxTy, 2);
     auto newSz = mlir::arith::MulIOp::create(builder, loc, needed, two);
     fir::StoreOp::create(builder, loc, newSz, buffSize);
-    mlir::Value byteSz = mlir::arith::MulIOp::create(builder, loc, newSz, eleSz);
+    mlir::Value byteSz =
+        mlir::arith::MulIOp::create(builder, loc, newSz, eleSz);
     mlir::SymbolRefAttr funcSymAttr =
         builder.getSymbolRefAttr(reallocFunc.getName());
     mlir::FunctionType funcTy = reallocFunc.getFunctionType();
-    auto newMem = fir::CallOp::create(builder,
-        loc, funcSymAttr, funcTy.getResults(),
+    auto newMem = fir::CallOp::create(
+        builder, loc, funcSymAttr, funcTy.getResults(),
         llvm::ArrayRef<mlir::Value>{
             builder.createConvert(loc, funcTy.getInputs()[0], mem),
             builder.createConvert(loc, funcTy.getInputs()[1], byteSz)});
@@ -6285,7 +6298,7 @@ private:
         }
       }
       auto coor = fir::CoordinateOp::create(builder, loc, refTy, buff,
-                                                    mlir::ValueRange{off});
+                                            mlir::ValueRange{off});
       return builder.createConvert(loc, eleRefTy, coor);
     };
 
@@ -6326,7 +6339,7 @@ private:
       mlir::Value buff =
           builder.createConvert(loc, fir::HeapType::get(resTy), mem);
       auto buffi = fir::CoordinateOp::create(builder, loc, eleRefTy, buff,
-                                                     mlir::ValueRange{off});
+                                             mlir::ValueRange{off});
       fir::factory::genScalarAssignment(
           builder, loc,
           [&]() -> ExtValue {
@@ -6404,7 +6417,7 @@ private:
     mlir::Type eleTy = fir::unwrapSequenceType(seqTy);
     auto loop =
         fir::DoLoopOp::create(builder, loc, lo, up, step, /*unordered=*/false,
-                                      /*finalCount=*/false, mem);
+                              /*finalCount=*/false, mem);
     // create a new binding for x.name(), to ac-do-variable, to the iteration
     // value.
     symMap.pushImpliedDoBinding(toStringRef(x.name()), loop.getInductionVar());
@@ -6484,8 +6497,8 @@ private:
       } else {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
-        mem = fir::AllocMemOp::create(builder,
-            loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
+        mem = fir::AllocMemOp::create(
+            builder, loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
         fir::StoreOp::create(builder, loc, initBuffSz, buffSize);
       }
     } else {
@@ -6806,9 +6819,8 @@ private:
                                          : one;
                               mlir::Value val = builder.createConvert(
                                   loc, idxTy, subscriptVal);
-                              mlir::Value ivAdj =
-                                  mlir::arith::SubIOp::create(builder,
-                                      loc, idxTy, val, lb);
+                              mlir::Value ivAdj = mlir::arith::SubIOp::create(
+                                  builder, loc, idxTy, val, lb);
                               componentsToAdd.push_back(
                                   builder.createConvert(loc, idxTy, ivAdj));
                             },
@@ -6830,8 +6842,9 @@ private:
                     converter.getRecordTypeFieldName(getLastSym(*x));
                 if (auto recTy = mlir::dyn_cast<fir::RecordType>(ty)) {
                   ty = recTy.getType(name);
-                  auto fld = fir::FieldIndexOp::create(builder,
-                      loc, fieldTy, name, recTy, fir::getTypeParams(arrayExv));
+                  auto fld = fir::FieldIndexOp::create(
+                      builder, loc, fieldTy, name, recTy,
+                      fir::getTypeParams(arrayExv));
                   addComponentList(ty, {fld});
                   if (index != revPath.size() - 1 || !isPointerAssignment()) {
                     // Need an intermediate  dereference if the boxed value
@@ -6852,8 +6865,9 @@ private:
                   ty = fir::unwrapRefType(boxTy.getEleTy());
                   auto recTy = mlir::cast<fir::RecordType>(ty);
                   ty = recTy.getType(name);
-                  auto fld = fir::FieldIndexOp::create(builder,
-                      loc, fieldTy, name, recTy, fir::getTypeParams(arrayExv));
+                  auto fld = fir::FieldIndexOp::create(
+                      builder, loc, fieldTy, name, recTy,
+                      fir::getTypeParams(arrayExv));
                   extendComponent(components, ty, {fld});
                 } else {
                   TODO(loc, "other component type");
@@ -6896,8 +6910,8 @@ private:
         mlir::Value innerArg = esp->findArgumentOfLoad(load);
         if (isAdjustedArrayElementType(eleTy)) {
           mlir::Type eleRefTy = builder.getRefType(eleTy);
-          auto arrayOp = fir::ArrayAccessOp::create(builder,
-              loc, eleRefTy, innerArg, iters.iterVec(),
+          auto arrayOp = fir::ArrayAccessOp::create(
+              builder, loc, eleRefTy, innerArg, iters.iterVec(),
               fir::factory::getTypeParams(loc, builder, load));
           if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
             mlir::Value dstLen = fir::factory::genLenOfCharacter(
@@ -6924,9 +6938,9 @@ private:
           if (!eleBoxTy || !mlir::isa<fir::BoxType>(eleBoxTy))
             TODO(loc, "assignment in a FORALL involving a designator with a "
                       "POINTER or ALLOCATABLE component part-ref");
-          auto arrayOp = fir::ArrayAccessOp::create(builder,
-              loc, builder.getRefType(eleBoxTy), innerArg, iters.iterVec(),
-              fir::factory::getTypeParams(loc, builder, load));
+          auto arrayOp = fir::ArrayAccessOp::create(
+              builder, loc, builder.getRefType(eleBoxTy), innerArg,
+              iters.iterVec(), fir::factory::getTypeParams(loc, builder, load));
           mlir::Value addr = components.getExtendCoorRef()(arrayOp);
           components.resetExtendCoorRef();
           // When the lhs is a boxed value and the context is not a pointer
@@ -6941,13 +6955,13 @@ private:
           }
           auto ele = convertElementForUpdate(loc, eleTy, iters.getElement());
           fir::StoreOp::create(builder, loc, ele, addr);
-          auto amend = fir::ArrayAmendOp::create(builder,
-              loc, innerArg.getType(), innerArg, arrayOp);
+          auto amend = fir::ArrayAmendOp::create(
+              builder, loc, innerArg.getType(), innerArg, arrayOp);
           return arrayLoadExtValue(builder, loc, load, iters.iterVec(), amend);
         }
         auto ele = convertElementForUpdate(loc, eleTy, iters.getElement());
-        auto update = fir::ArrayUpdateOp::create(builder,
-            loc, innerArg.getType(), innerArg, ele, iters.iterVec(),
+        auto update = fir::ArrayUpdateOp::create(
+            builder, loc, innerArg.getType(), innerArg, ele, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         return arrayLoadExtValue(builder, loc, load, iters.iterVec(), update);
       };
@@ -6961,9 +6975,9 @@ private:
         mlir::Value innerArg = explicitSpace->findArgumentOfLoad(load);
         mlir::Type refEleTy =
             fir::isa_ref_type(eleTy) ? eleTy : builder.getRefType(eleTy);
-        auto arrModify = fir::ArrayModifyOp::create(builder,
-            loc, mlir::TypeRange{refEleTy, innerArg.getType()}, innerArg,
-            iters.iterVec(), load.getTypeparams());
+        auto arrModify = fir::ArrayModifyOp::create(
+            builder, loc, mlir::TypeRange{refEleTy, innerArg.getType()},
+            innerArg, iters.iterVec(), load.getTypeparams());
         return arrayLoadExtValue(builder, loc, load, iters.iterVec(),
                                  arrModify.getResult(1));
       };
@@ -6974,8 +6988,8 @@ private:
           isAdjustedArrayElementType(eleTy)) {
         mlir::Type resTy = builder.getRefType(eleTy);
         // Use array element reference semantics.
-        auto access = fir::ArrayAccessOp::create(builder,
-            loc, resTy, load, iters.iterVec(),
+        auto access = fir::ArrayAccessOp::create(
+            builder, loc, resTy, load, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         mlir::Value newBase = access;
         if (fir::isa_char(eleTy)) {
@@ -6997,8 +7011,8 @@ private:
         if (!eleBoxTy || !mlir::isa<fir::BoxType>(eleBoxTy))
           TODO(loc, "assignment in a FORALL involving a designator with a "
                     "POINTER or ALLOCATABLE component part-ref");
-        auto access = fir::ArrayAccessOp::create(builder,
-            loc, builder.getRefType(eleBoxTy), load, iters.iterVec(),
+        auto access = fir::ArrayAccessOp::create(
+            builder, loc, builder.getRefType(eleBoxTy), load, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         mlir::Value addr = components.getExtendCoorRef()(access);
         components.resetExtendCoorRef();
@@ -7010,8 +7024,8 @@ private:
           // Rhs is a regular expression that will need to be boxed before
           // assigning to the boxed variable.
           auto typeParams = fir::factory::getTypeParams(loc, builder, load);
-          auto access = fir::ArrayAccessOp::create(builder,
-              loc, builder.getRefType(eleTy), load, iters.iterVec(),
+          auto access = fir::ArrayAccessOp::create(
+              builder, loc, builder.getRefType(eleTy), load, iters.iterVec(),
               typeParams);
           auto addr = components.getExtendCoorRef()(access);
           components.resetExtendCoorRef();
@@ -7026,13 +7040,13 @@ private:
                       "component");
           mlir::Value embox =
               fir::EmboxOp::create(builder, loc, boxTy, ptrAddr,
-                                           /*shape=*/mlir::Value{},
-                                           /*slice=*/mlir::Value{}, typeParams);
+                                   /*shape=*/mlir::Value{},
+                                   /*slice=*/mlir::Value{}, typeParams);
           return arrayLoadExtValue(builder, loc, load, iters.iterVec(), embox);
         }
       }
-      auto fetch = fir::ArrayFetchOp::create(builder,
-          loc, eleTy, load, iters.iterVec(), load.getTypeparams());
+      auto fetch = fir::ArrayFetchOp::create(
+          builder, loc, eleTy, load, iters.iterVec(), load.getTypeparams());
       return arrayLoadExtValue(builder, loc, load, iters.iterVec(), fetch);
     };
     return [=](IterSpace iters) mutable { return lambda(pc(iters)); };
@@ -7541,24 +7555,25 @@ fir::ExtendedValue Fortran::lower::updateBoxForParentComponent(
 
   if (op) {
     if (auto embox = mlir::dyn_cast<fir::EmboxOp>(op)) {
-      auto newBox = fir::EmboxOp::create(builder,
-          loc, fir::BoxType::get(actualTy), embox.getMemref(), embox.getShape(),
-          embox.getSlice(), embox.getTypeparams());
+      auto newBox = fir::EmboxOp::create(
+          builder, loc, fir::BoxType::get(actualTy), embox.getMemref(),
+          embox.getShape(), embox.getSlice(), embox.getTypeparams());
       return fir::substBase(box, newBox);
     }
     if (auto rebox = mlir::dyn_cast<fir::ReboxOp>(op)) {
-      auto newBox = fir::ReboxOp::create(builder,
-          loc, fir::BoxType::get(actualTy), rebox.getBox(), rebox.getShape(),
-          rebox.getSlice());
+      auto newBox = fir::ReboxOp::create(
+          builder, loc, fir::BoxType::get(actualTy), rebox.getBox(),
+          rebox.getShape(), rebox.getSlice());
       return fir::substBase(box, newBox);
     }
   }
 
   mlir::Value empty;
   mlir::ValueRange emptyRange;
-  return fir::ReboxOp::create(builder, loc, fir::BoxType::get(actualTy), boxBase,
-                                      /*shape=*/empty,
-                                      /*slice=*/empty);
+  return fir::ReboxOp::create(builder, loc, fir::BoxType::get(actualTy),
+                              boxBase,
+                              /*shape=*/empty,
+                              /*slice=*/empty);
 }
 
 fir::ExtendedValue Fortran::lower::createBoxValue(
@@ -7650,8 +7665,8 @@ fir::ArrayLoadOp genArrayLoad(mlir::Location loc,
   mlir::Value shapeOp = builder.createShape(loc, exv);
   mlir::Type arrTy = fir::dyn_cast_ptrOrBoxEleTy(addr.getType());
   return fir::ArrayLoadOp::create(builder, loc, arrTy, addr, shapeOp,
-                                          /*slice=*/mlir::Value{},
-                                          fir::getTypeParams(exv));
+                                  /*slice=*/mlir::Value{},
+                                  fir::getTypeParams(exv));
 }
 template <>
 fir::ArrayLoadOp
@@ -7699,8 +7714,8 @@ void Fortran::lower::createArrayMergeStores(
     if (std::optional<fir::ArrayLoadOp> ldOpt = esp.getLhsLoad(i.index())) {
       fir::ArrayLoadOp load = *ldOpt;
       fir::ArrayMergeStoreOp::create(builder, loc, load, i.value(),
-                                             load.getMemref(), load.getSlice(),
-                                             load.getTypeparams());
+                                     load.getMemref(), load.getSlice(),
+                                     load.getTypeparams());
     }
   if (esp.loopCleanup) {
     (*esp.loopCleanup)(builder);
@@ -7723,11 +7738,11 @@ mlir::Value Fortran::lower::addCrayPointerInst(mlir::Location loc,
   mlir::ValueRange emptyRange;
   auto boxTy = fir::BoxType::get(ptrTy);
   auto box = fir::EmboxOp::create(builder, loc, boxTy, ptrVal, empty, empty,
-                                          emptyRange);
-  mlir::Value addrof =
-      (mlir::isa<fir::ReferenceType>(ptrTy))
-          ? fir::BoxAddrOp::create(builder, loc, ptrTy, box)
-          : fir::BoxAddrOp::create(builder, loc, builder.getRefType(ptrTy), box);
+                                  emptyRange);
+  mlir::Value addrof = (mlir::isa<fir::ReferenceType>(ptrTy))
+                           ? fir::BoxAddrOp::create(builder, loc, ptrTy, box)
+                           : fir::BoxAddrOp::create(
+                                 builder, loc, builder.getRefType(ptrTy), box);
 
   auto refPtrTy =
       builder.getRefType(fir::PointerType::get(fir::dyn_cast_ptrEleTy(pteTy)));
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index a709c7a36..46be11124 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -312,8 +312,8 @@ private:
     // hlfir.elemental_addr.
     if (auto elementalAddrOp = getVectorSubscriptElementAddrOp())
       builder.setInsertionPointToEnd(&elementalAddrOp->getBody().front());
-    auto designate = hlfir::DesignateOp::create(builder,
-        getLoc(), designatorType, partInfo.base.value().getBase(),
+    auto designate = hlfir::DesignateOp::create(
+        builder, getLoc(), designatorType, partInfo.base.value().getBase(),
         partInfo.componentName, partInfo.componentShape, partInfo.subscripts,
         partInfo.substring, partInfo.complexPart, partInfo.resultShape,
         partInfo.typeParams, attributes);
@@ -442,8 +442,8 @@ private:
     } else {
       // Compute "len = max(ub-lb+1,0)" (Fortran 2018 9.4.1).
       mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-      auto boundsDiff = mlir::arith::SubIOp::create(builder,
-          loc, partInfo.substring[1], partInfo.substring[0]);
+      auto boundsDiff = mlir::arith::SubIOp::create(
+          builder, loc, partInfo.substring[1], partInfo.substring[0]);
       auto rawLen = mlir::arith::AddIOp::create(builder, loc, boundsDiff, one);
       partInfo.typeParams[0] =
           fir::factory::genMaxWithZero(builder, loc, rawLen);
@@ -836,8 +836,8 @@ private:
     assert(partInfo.base.has_value() &&
            "vector subscripted part must have a base");
     mlir::Value mold = *partInfo.base;
-    auto elementalAddrOp = hlfir::ElementalAddrOp::create(builder,
-        loc, shape, mold, mlir::ValueRange{},
+    auto elementalAddrOp = hlfir::ElementalAddrOp::create(
+        builder, loc, shape, mold, mlir::ValueRange{},
         /*isUnordered=*/true);
     setVectorSubscriptElementAddrOp(elementalAddrOp);
     builder.setInsertionPointToEnd(&elementalAddrOp.getBody().front());
@@ -1034,7 +1034,7 @@ struct BinaryOp {};
                                                 rhs)};                         \
       } else {                                                                 \
         return hlfir::EntityWithAttributes{                                    \
-            GenBinFirOp::create(builder, loc, lhs, rhs)};                       \
+            GenBinFirOp::create(builder, loc, lhs, rhs)};                      \
       }                                                                        \
     }                                                                          \
   };
@@ -1219,8 +1219,8 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = mlir::arith::CmpIOp::create(builder,
-        loc, translateSignedRelational(op.opr), lhs, rhs);
+    auto cmp = mlir::arith::CmpIOp::create(
+        builder, loc, translateSignedRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
 };
@@ -1241,8 +1241,8 @@ struct BinaryOp<Fortran::evaluate::Relational<
         mlir::IntegerType::SignednessSemantics::Signless);
     mlir::Value lhsSL = builder.createConvert(loc, signlessType, lhs);
     mlir::Value rhsSL = builder.createConvert(loc, signlessType, rhs);
-    auto cmp = mlir::arith::CmpIOp::create(builder,
-        loc, translateUnsignedRelational(op.opr), lhsSL, rhsSL);
+    auto cmp = mlir::arith::CmpIOp::create(
+        builder, loc, translateUnsignedRelational(op.opr), lhsSL, rhsSL);
     return hlfir::EntityWithAttributes{cmp};
   }
 };
@@ -1256,8 +1256,8 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = mlir::arith::CmpFOp::create(builder,
-        loc, translateFloatRelational(op.opr), lhs, rhs);
+    auto cmp = mlir::arith::CmpFOp::create(
+        builder, loc, translateFloatRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
 };
@@ -1271,8 +1271,8 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = fir::CmpcOp::create(builder,
-        loc, translateFloatRelational(op.opr), lhs, rhs);
+    auto cmp = fir::CmpcOp::create(builder, loc,
+                                   translateFloatRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
 };
@@ -1318,11 +1318,11 @@ struct BinaryOp<Fortran::evaluate::LogicalOperation<KIND>> {
       return hlfir::EntityWithAttributes{
           mlir::arith::OrIOp::create(builder, loc, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Eqv:
-      return hlfir::EntityWithAttributes{mlir::arith::CmpIOp::create(builder,
-          loc, mlir::arith::CmpIPredicate::eq, i1Lhs, i1Rhs)};
+      return hlfir::EntityWithAttributes{mlir::arith::CmpIOp::create(
+          builder, loc, mlir::arith::CmpIPredicate::eq, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Neqv:
-      return hlfir::EntityWithAttributes{mlir::arith::CmpIOp::create(builder,
-          loc, mlir::arith::CmpIPredicate::ne, i1Lhs, i1Rhs)};
+      return hlfir::EntityWithAttributes{mlir::arith::CmpIOp::create(
+          builder, loc, mlir::arith::CmpIPredicate::ne, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Not:
       // lib/evaluate expression for .NOT. is Fortran::evaluate::Not<KIND>.
       llvm_unreachable(".NOT. is not a binary operator");
@@ -1822,8 +1822,8 @@ private:
     // Allocate scalar temporary that will be initialized
     // with the values specified by the constructor.
     mlir::Value storagePtr = builder.createTemporary(loc, recTy);
-    auto varOp = hlfir::EntityWithAttributes{hlfir::DeclareOp::create(builder,
-        loc, storagePtr, "ctor.temp", /*shape=*/nullptr,
+    auto varOp = hlfir::EntityWithAttributes{hlfir::DeclareOp::create(
+        builder, loc, storagePtr, "ctor.temp", /*shape=*/nullptr,
         /*typeparams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{})};
 
@@ -1855,8 +1855,8 @@ private:
         auto parentCompType = baseRecTy.getType(parentName);
         assert(parentCompType && "failed to retrieve parent component type");
         mlir::Type designatorType = builder.getRefType(parentCompType);
-        mlir::Value newParent = hlfir::DesignateOp::create(builder,
-            loc, designatorType, currentParent, parentName,
+        mlir::Value newParent = hlfir::DesignateOp::create(
+            builder, loc, designatorType, currentParent, parentName,
             /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
             /*substring=*/mlir::ValueRange{},
             /*complexPart=*/std::nullopt,
@@ -1912,8 +1912,8 @@ private:
                                                     extraAttributeFlags);
 
       // Get the component designator.
-      auto lhs = hlfir::DesignateOp::create(builder,
-          loc, designatorType, baseOp, name, compShape,
+      auto lhs = hlfir::DesignateOp::create(
+          builder, loc, designatorType, baseOp, name, compShape,
           hlfir::DesignateOp::Subscripts{},
           /*substring=*/mlir::ValueRange{},
           /*complexPart=*/std::nullopt,
@@ -1998,9 +1998,9 @@ private:
             hlfir::genTypeAndKindConvert(loc, builder, rhs, lhs.getType(),
                                          /*preserveLowerBounds=*/allowRealloc);
         hlfir::AssignOp::create(builder, loc, rhsCastAndCleanup.first, lhs,
-                                        allowRealloc,
-                                        allowRealloc ? keepLhsLength : false,
-                                        /*temporary_lhs=*/true);
+                                allowRealloc,
+                                allowRealloc ? keepLhsLength : false,
+                                /*temporary_lhs=*/true);
         if (rhsCastAndCleanup.second)
           (*rhsCastAndCleanup.second)();
       };
diff --git a/flang/lib/Lower/ConvertProcedureDesignator.cpp b/flang/lib/Lower/ConvertProcedureDesignator.cpp
index e46717070..d4c535d71 100644
--- a/flang/lib/Lower/ConvertProcedureDesignator.cpp
+++ b/flang/lib/Lower/ConvertProcedureDesignator.cpp
@@ -124,8 +124,8 @@ static hlfir::EntityWithAttributes designateProcedurePointerComponent(
   if (!fieldType)
     TODO(loc, "passing type bound procedure (extension)");
   mlir::Type designatorType = fir::ReferenceType::get(fieldType);
-  mlir::Value compRef = hlfir::DesignateOp::create(builder,
-      loc, designatorType, base, fieldName,
+  mlir::Value compRef = hlfir::DesignateOp::create(
+      builder, loc, designatorType, base, fieldName,
       /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
       /*substring=*/mlir::ValueRange{},
       /*complexPart=*/std::nullopt,
@@ -174,8 +174,8 @@ hlfir::EntityWithAttributes Fortran::lower::convertProcedureDesignatorToHLFIR(
     mlir::Type boxTy =
         Fortran::lower::getUntypedBoxProcType(&converter.getMLIRContext());
     if (auto host = Fortran::lower::argumentHostAssocs(converter, funcAddr))
-      funcAddr = fir::EmboxProcOp::create(builder,
-          loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
+      funcAddr = fir::EmboxProcOp::create(
+          builder, loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
     else
       funcAddr = fir::EmboxProcOp::create(builder, loc, boxTy, funcAddr);
   }
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 11e0feb88..647bd0d07 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -313,7 +313,7 @@ mlir::Value Fortran::lower::genInitialDataTarget(
   // Create a fir.rebox to set the attribute correctly, and use targetShift
   // to preserve the target lower bounds if any.
   return fir::ReboxOp::create(builder, loc, boxType, targetBox, targetShift,
-                                      /*slice=*/mlir::Value{});
+                              /*slice=*/mlir::Value{});
 }
 
 /// Generate default initial value for a derived type object \p sym with mlir
@@ -383,11 +383,11 @@ static mlir::Value genComponentDefaultInit(
   componentValue = builder.createConvert(loc, componentTy, componentValue);
   auto fieldTy = fir::FieldType::get(recTy.getContext());
   // FIXME: type parameters must come from the derived-type-spec
-  auto field = fir::FieldIndexOp::create(builder,
-      loc, fieldTy, name, recTy,
-      /*typeParams=*/mlir::ValueRange{} /*TODO*/);
-  return fir::InsertValueOp::create(builder,
-      loc, recTy, insertInto, componentValue,
+  auto field =
+      fir::FieldIndexOp::create(builder, loc, fieldTy, name, recTy,
+                                /*typeParams=*/mlir::ValueRange{} /*TODO*/);
+  return fir::InsertValueOp::create(
+      builder, loc, recTy, insertInto, componentValue,
       builder.getArrayAttr(field.getAttributes()));
 }
 
@@ -453,8 +453,8 @@ static mlir::Value genDefaultInitializerValue(
       rangeBounds.push_back(0);
       rangeBounds.push_back(extent - 1);
     }
-    return fir::InsertOnRangeOp::create(builder,
-        loc, sequenceType, arrayInitialValue, initialValue,
+    return fir::InsertOnRangeOp::create(
+        builder, loc, sequenceType, arrayInitialValue, initialValue,
         builder.getIndexVectorAttr(rangeBounds));
   }
   return initialValue;
@@ -691,7 +691,7 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
     global = defineGlobal(converter, var, globalName, linkage, dataAttr);
   }
   auto addrOf = fir::AddrOfOp::create(builder, loc, global.resultType(),
-                                              global.getSymbol());
+                                      global.getSymbol());
   // The type of the global cannot be trusted to be the same as the one
   // of the variable as some existing programs map common blocks to
   // BIND(C) module variables (e.g. mpi_argv_null in MPI and MPI_F08).
@@ -768,11 +768,11 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
       indices.push_back(builder.createConvert(loc, idxTy, sh));
     if (dataAttr.getValue() == cuf::DataAttribute::Shared)
       return cuf::SharedMemoryOp::create(builder, loc, ty, nm, symNm, lenParams,
-                                                 indices);
+                                         indices);
 
     if (!cuf::isCUDADeviceContext(builder.getRegion()))
       return cuf::AllocOp::create(builder, loc, ty, nm, symNm, dataAttr,
-                                          lenParams, indices);
+                                  lenParams, indices);
   }
 
   // Let the builder do all the heavy lifting.
@@ -833,9 +833,9 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
             // Field found in the base record type.
             auto fieldName = recTy.getTypeList()[fieldIdx].first;
             fieldTy = recTy.getTypeList()[fieldIdx].second;
-            mlir::Value fieldIndex = fir::FieldIndexOp::create(builder,
-                loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
-                recTy,
+            mlir::Value fieldIndex = fir::FieldIndexOp::create(
+                builder, loc, fir::FieldType::get(fieldTy.getContext()),
+                fieldName, recTy,
                 /*typeParams=*/mlir::ValueRange{});
             coordinates.push_back(fieldIndex);
           } else {
@@ -846,19 +846,18 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
                       mlir::dyn_cast<fir::RecordType>(component.second)) {
                 fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
                 if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                  mlir::Value parentFieldIndex =
-                      fir::FieldIndexOp::create(builder,
-                          loc, fir::FieldType::get(childRecTy.getContext()),
-                          component.first, recTy,
-                          /*typeParams=*/mlir::ValueRange{});
+                  mlir::Value parentFieldIndex = fir::FieldIndexOp::create(
+                      builder, loc,
+                      fir::FieldType::get(childRecTy.getContext()),
+                      component.first, recTy,
+                      /*typeParams=*/mlir::ValueRange{});
                   coordinates.push_back(parentFieldIndex);
                   auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
                   fieldTy = childRecTy.getTypeList()[fieldIdx].second;
-                  mlir::Value childFieldIndex =
-                      fir::FieldIndexOp::create(builder,
-                          loc, fir::FieldType::get(fieldTy.getContext()),
-                          fieldName, childRecTy,
-                          /*typeParams=*/mlir::ValueRange{});
+                  mlir::Value childFieldIndex = fir::FieldIndexOp::create(
+                      builder, loc, fir::FieldType::get(fieldTy.getContext()),
+                      fieldName, childRecTy,
+                      /*typeParams=*/mlir::ValueRange{});
                   coordinates.push_back(childFieldIndex);
                   break;
                 }
@@ -879,14 +878,14 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
             assert(coordinates.size() == 1 && "expect one coordinate");
             auto field = mlir::dyn_cast<fir::FieldIndexOp>(
                 coordinates[0].getDefiningOp());
-            comp = hlfir::DesignateOp::create(builder,
-                loc, builder.getRefType(fieldTy), addr,
+            comp = hlfir::DesignateOp::create(
+                builder, loc, builder.getRefType(fieldTy), addr,
                 /*component=*/field.getFieldName(),
                 /*componentShape=*/mlir::Value{},
                 hlfir::DesignateOp::Subscripts{});
           } else {
-            comp = fir::CoordinateOp::create(builder,
-                loc, builder.getRefType(fieldTy), base, coordinates);
+            comp = fir::CoordinateOp::create(
+                builder, loc, builder.getRefType(fieldTy), base, coordinates);
           }
           cuf::DataAttributeAttr dataAttr =
               Fortran::lower::translateSymbolCUFDataAttribute(
@@ -934,7 +933,7 @@ void Fortran::lower::defaultInitializeAtRuntime(
     // Creating descriptor/passing null descriptor to the runtime would
     // create runtime crashes.
     auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
-                                                      fir::getBase(exv));
+                                              fir::getBase(exv));
     builder.genIfThen(loc, isPresent)
         .genThen([&]() {
           auto box = builder.createBox(loc, exv);
@@ -995,9 +994,9 @@ void Fortran::lower::defaultInitializeAtRuntime(
             });
       }
       auto addrOf = fir::AddrOfOp::create(builder, loc, global.resultType(),
-                                                  global.getSymbol());
+                                          global.getSymbol());
       fir::CopyOp::create(builder, loc, addrOf, fir::getBase(exv),
-                                  /*noOverlap=*/true);
+                          /*noOverlap=*/true);
     } else {
       mlir::Value box = builder.createBox(loc, exv);
       fir::runtime::genDerivedTypeInitialize(builder, loc, box);
@@ -1098,7 +1097,7 @@ static void finalizeAtRuntime(Fortran::lower::AbstractConverter &converter,
   if (Fortran::semantics::IsOptional(sym)) {
     // Only finalize if present.
     auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
-                                                      fir::getBase(exv));
+                                              fir::getBase(exv));
     builder.genIfThen(loc, isPresent)
         .genThen([&]() {
           auto box = builder.createBox(loc, exv);
@@ -1144,8 +1143,8 @@ static void deallocateIntentOut(Fortran::lower::AbstractConverter &converter,
       fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
       if (Fortran::semantics::IsOptional(sym)) {
-        auto isPresent = fir::IsPresentOp::create(builder,
-            loc, builder.getI1Type(), fir::getBase(extVal));
+        auto isPresent = fir::IsPresentOp::create(
+            builder, loc, builder.getI1Type(), fir::getBase(extVal));
         builder.genIfThen(loc, isPresent)
             .genThen([&]() {
               Fortran::lower::genDeallocateIfAllocated(converter, *mutBox, loc);
@@ -1419,7 +1418,7 @@ instantiateAggregateStore(Fortran::lower::AbstractConverter &converter,
           defineGlobalAggregateStore(converter, aggregate, aggName, linkage);
     }
     auto addr = fir::AddrOfOp::create(builder, loc, global.resultType(),
-                                              global.getSymbol());
+                                      global.getSymbol());
     auto size = std::get<1>(var.getInterval());
     fir::SequenceType::Shape shape(1, size);
     auto seqTy = fir::SequenceType::get(shape, i8Ty);
@@ -1466,8 +1465,8 @@ static void instantiateAlias(Fortran::lower::AbstractConverter &converter,
   std::size_t off = sym.GetUltimate().offset() - var.getAliasOffset();
   mlir::Value storeAddr = getAggregateStore(storeMap, var);
   mlir::Value offset = builder.createIntegerConstant(loc, idxTy, off);
-  mlir::Value bytePtr = fir::CoordinateOp::create(builder,
-      loc, i8Ptr, storeAddr, mlir::ValueRange{offset});
+  mlir::Value bytePtr = fir::CoordinateOp::create(
+      builder, loc, i8Ptr, storeAddr, mlir::ValueRange{offset});
   mlir::Value typedPtr = castAliasToPointer(builder, loc, symType, bytePtr);
   Fortran::lower::StatementContext stmtCtx;
   mapSymbolAttributes(converter, var, symMap, stmtCtx, typedPtr);
@@ -1681,7 +1680,7 @@ static void finalizeCommonBlockDefinition(
           mlir::Value castVal = builder.createConvert(
               loc, commonTy.getType(tupIdx), fir::getBase(initVal));
           cb = fir::InsertValueOp::create(builder, loc, commonTy, cb, castVal,
-                                                  builder.getArrayAttr(offVal));
+                                          builder.getArrayAttr(offVal));
           ++tupIdx;
           offset = mem->offset() + mem->size();
         }
@@ -1722,8 +1721,8 @@ mlir::Value Fortran::lower::genCommonBlockMember(
 
   mlir::Value offs =
       builder.createIntegerConstant(loc, builder.getIndexType(), byteOffset);
-  mlir::Value varAddr = fir::CoordinateOp::create(builder,
-      loc, i8Ptr, base, mlir::ValueRange{offs});
+  mlir::Value varAddr = fir::CoordinateOp::create(builder, loc, i8Ptr, base,
+                                                  mlir::ValueRange{offs});
   mlir::Type symType = converter.genType(sym);
 
   return Fortran::semantics::FindEquivalenceSet(sym) != nullptr
@@ -1749,7 +1748,7 @@ static void instantiateCommon(Fortran::lower::AbstractConverter &converter,
     // introduce a local AddrOf and add it to the map
     fir::GlobalOp global = getCommonBlockGlobal(converter, common);
     commonAddr = fir::AddrOfOp::create(builder, loc, global.resultType(),
-                                               global.getSymbol());
+                                       global.getSymbol());
 
     symMap.addSymbol(common, commonAddr);
   }
@@ -2031,8 +2030,8 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
           Fortran::semantics::GetCUDADataAttr(&sym.GetUltimate()));
 
       // Declare a local pointer variable.
-      auto newBase = hlfir::DeclareOp::create(builder,
-          loc, boxAlloc, name, /*shape=*/nullptr, lenParams,
+      auto newBase = hlfir::DeclareOp::create(
+          builder, loc, boxAlloc, name, /*shape=*/nullptr, lenParams,
           /*dummy_scope=*/nullptr, attributes);
       mlir::Value nullAddr = builder.createNullConstant(
           loc, llvm::cast<fir::BaseBoxType>(ptrBoxType).getEleTy());
@@ -2048,7 +2047,7 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
       // declaration.
       mlir::Value initVal =
           fir::EmboxOp::create(builder, loc, ptrBoxType, nullAddr, shapeOrShift,
-                                       /*slice=*/nullptr, lenParams);
+                               /*slice=*/nullptr, lenParams);
       fir::StoreOp::create(builder, loc, initVal, newBase.getBase());
 
       // Any reference to the pointee is going to be using the pointer
@@ -2063,9 +2062,9 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
     mlir::Value dummyScope;
     if (converter.isRegisteredDummySymbol(sym))
       dummyScope = converter.dummyArgsScopeValue();
-    auto newBase = hlfir::DeclareOp::create(builder,
-        loc, base, name, shapeOrShift, lenParams, dummyScope, attributes,
-        dataAttr);
+    auto newBase =
+        hlfir::DeclareOp::create(builder, loc, base, name, shapeOrShift,
+                                 lenParams, dummyScope, attributes, dataAttr);
     symMap.addVariableDefinition(sym, newBase, force);
     return;
   }
@@ -2304,8 +2303,8 @@ void Fortran::lower::mapSymbolAttributes(
           mlir::Type lenType = builder.getCharacterLengthType();
           mlir::Value addr, len;
           if (Fortran::semantics::IsOptional(sym)) {
-            auto isPresent = fir::IsPresentOp::create(builder,
-                loc, builder.getI1Type(), dummyArg);
+            auto isPresent = fir::IsPresentOp::create(
+                builder, loc, builder.getI1Type(), dummyArg);
             auto addrAndLen =
                 builder
                     .genIfOp(loc, {refTy, lenType}, isPresent,
@@ -2315,15 +2314,15 @@ void Fortran::lower::mapSymbolAttributes(
                           fir::BoxAddrOp::create(builder, loc, refTy, dummyArg);
                       mlir::Value readLength =
                           charHelp.readLengthFromBox(dummyArg);
-                      fir::ResultOp::create(builder,
-                          loc, mlir::ValueRange{readAddr, readLength});
+                      fir::ResultOp::create(
+                          builder, loc, mlir::ValueRange{readAddr, readLength});
                     })
                     .genElse([&] {
                       mlir::Value readAddr = builder.genAbsentOp(loc, refTy);
                       mlir::Value readLength =
                           fir::factory::createZeroValue(builder, loc, lenType);
-                      fir::ResultOp::create(builder,
-                          loc, mlir::ValueRange{readAddr, readLength});
+                      fir::ResultOp::create(
+                          builder, loc, mlir::ValueRange{readAddr, readLength});
                     })
                     .getResults();
             addr = addrAndLen[0];
@@ -2803,8 +2802,8 @@ Fortran::lower::genPackArray(Fortran::lower::AbstractConverter &converter,
   mlir::Type elementType = boxType.unwrapInnerType();
   llvm::SmallVector<mlir::Value> elidedLenParams =
       fir::factory::elideLengthsAlreadyInType(elementType, lenParams);
-  auto packOp = fir::PackArrayOp::create(builder,
-      loc, fir::getBase(exv), stackAlloc, isInnermostMode, noCopy,
+  auto packOp = fir::PackArrayOp::create(
+      builder, loc, fir::getBase(exv), stackAlloc, isInnermostMode, noCopy,
       /*max_size=*/mlir::IntegerAttr{},
       /*max_element_size=*/mlir::IntegerAttr{},
       /*min_stride=*/mlir::IntegerAttr{}, fir::PackArrayHeuristics::None,
@@ -2843,5 +2842,5 @@ void Fortran::lower::genUnpackArray(
   bool noCopy = Fortran::semantics::IsIntentIn(sym);
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   fir::UnpackArrayOp::create(builder, loc, temp, original, stackAlloc, noCopy,
-                                     getSafeRepackAttrs(converter));
+                             getSafeRepackAttrs(converter));
 }
diff --git a/flang/lib/Lower/CustomIntrinsicCall.cpp b/flang/lib/Lower/CustomIntrinsicCall.cpp
index 5040989b1..2c5233bdd 100644
--- a/flang/lib/Lower/CustomIntrinsicCall.cpp
+++ b/flang/lib/Lower/CustomIntrinsicCall.cpp
@@ -296,7 +296,8 @@ lowerAssociated(fir::FirOpBuilder &builder, mlir::Location loc,
             fir::ResultOp::create(builder, loc, cast);
           })
           .genElse([&]() {
-            mlir::Value absentBox = fir::AbsentOp::create(builder, loc, boxType);
+            mlir::Value absentBox =
+                fir::AbsentOp::create(builder, loc, boxType);
             fir::ResultOp::create(builder, loc, absentBox);
           })
           .getResults()[0];
diff --git a/flang/lib/Lower/HlfirIntrinsics.cpp b/flang/lib/Lower/HlfirIntrinsics.cpp
index 116742b7c..6e1d06a25 100644
--- a/flang/lib/Lower/HlfirIntrinsics.cpp
+++ b/flang/lib/Lower/HlfirIntrinsics.cpp
@@ -210,8 +210,8 @@ mlir::Value HlfirTransformationalIntrinsic::loadBoxAddress(
   mlir::Value box = builder.createBox(loc, exv);
   mlir::Type boxType = box.getType();
   auto absent = fir::AbsentOp::create(builder, loc, boxType);
-  auto boxOrAbsent = mlir::arith::SelectOp::create(builder,
-      loc, boxType, isPresent, box, absent);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder, loc, boxType,
+                                                   isPresent, box, absent);
 
   return boxOrAbsent;
 }
diff --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp
index aeaa6e27b..2a330ccc4 100644
--- a/flang/lib/Lower/HostAssociations.cpp
+++ b/flang/lib/Lower/HostAssociations.cpp
@@ -300,8 +300,8 @@ public:
       auto addr = fir::BoxAddrOp::create(builder, loc, eleTy, box);
       mlir::Value isPresent = builder.genIsNotNullAddr(loc, addr);
       auto absentBox = fir::AbsentOp::create(builder, loc, boxTy);
-      box =
-          mlir::arith::SelectOp::create(builder, loc, isPresent, box, absentBox);
+      box = mlir::arith::SelectOp::create(builder, loc, isPresent, box,
+                                          absentBox);
     }
     bindCapturedSymbol(sym, box, converter, args.symMap);
   }
@@ -404,8 +404,8 @@ public:
       // done on present optional. For absent optionals, simply create a
       // disassociated pointer (it is illegal to inquire about lower bounds or
       // lengths of optional according to 15.5.2.12 3 (9) and 10.1.11 2 (7)b).
-      auto isPresent = fir::IsPresentOp::create(builder,
-          loc, builder.getI1Type(), fir::getBase(args.hostValue));
+      auto isPresent = fir::IsPresentOp::create(
+          builder, loc, builder.getI1Type(), fir::getBase(args.hostValue));
       builder.genIfThenElse(loc, isPresent)
           .genThen([&]() {
             fir::factory::associateMutableBox(builder, loc, boxInTuple,
@@ -442,7 +442,7 @@ public:
         for (unsigned dim = 0; dim < rank; ++dim) {
           mlir::Value dimVal = builder.createIntegerConstant(loc, idxTy, dim);
           auto dims = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
-                                                     box, dimVal);
+                                             box, dimVal);
           lbounds.emplace_back(dims.getResult(0));
         }
       }
@@ -468,7 +468,7 @@ public:
         mlir::Value isPresent = builder.genIsNotNullAddr(loc, addr);
         auto absentBox = fir::AbsentOp::create(builder, loc, boxTy);
         box = mlir::arith::SelectOp::create(builder, loc, isPresent, box,
-                                                    absentBox);
+                                            absentBox);
       }
       fir::BoxValue boxValue(box, lbounds, /*explicitParams=*/{});
       bindCapturedSymbol(sym, boxValue, converter, args.symMap);
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 32ddd0ea9..c95c3404a 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -153,8 +153,8 @@ static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
   if (csi.ioMsg) {
     mlir::func::FuncOp getIoMsg =
         fir::runtime::getIORuntimeFunc<mkIOKey(GetIoMsg)>(loc, builder);
-    fir::CallOp::create(builder,
-        loc, getIoMsg,
+    fir::CallOp::create(
+        builder, loc, getIoMsg,
         mlir::ValueRange{
             cookie,
             builder.createConvert(loc, getIoMsg.getFunctionType().getInput(1),
@@ -165,7 +165,7 @@ static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
   mlir::func::FuncOp endIoStatement =
       fir::runtime::getIORuntimeFunc<mkIOKey(EndIoStatement)>(loc, builder);
   auto call = fir::CallOp::create(builder, loc, endIoStatement,
-                                          mlir::ValueRange{cookie});
+                                  mlir::ValueRange{cookie});
   mlir::Value iostat = call.getResult(0);
   if (csi.bigUnitIfOp) {
     stmtCtx.finalizeAndPop();
@@ -204,7 +204,7 @@ static void makeNextConditionalOn(fir::FirOpBuilder &builder,
   if (inLoop)
     resTy = boolTy;
   auto ifOp = fir::IfOp::create(builder, loc, resTy, ok,
-                                        /*withElseRegion=*/inLoop);
+                                /*withElseRegion=*/inLoop);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
 }
 
@@ -259,10 +259,10 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
           ? fir::NameUniquer::doGenerated("default" + suffix)
           : converter.mangleName(suffix);
   if (auto table = builder.getNamedGlobal(tableMangleName))
-    return builder.createConvert(
-        loc, refTy,
-        fir::AddrOfOp::create(builder, loc, table.resultType(),
-                                      table.getSymbol()));
+    return builder.createConvert(loc, refTy,
+                                 fir::AddrOfOp::create(builder, loc,
+                                                       table.resultType(),
+                                                       table.getSymbol()));
 
   mlir::StringAttr linkOnce = builder.createLinkOnceLinkage();
   mlir::Type idxTy = builder.getIndexType();
@@ -281,8 +281,9 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   // Define the list of NonTbpDefinedIo procedures.
   bool tableIsLocal =
       !definedIoProcMap.empty() && hasLocalDefinedIoProc(definedIoProcMap);
-  mlir::Value listAddr =
-      tableIsLocal ? fir::AllocaOp::create(builder, loc, listTy) : mlir::Value{};
+  mlir::Value listAddr = tableIsLocal
+                             ? fir::AllocaOp::create(builder, loc, listTy)
+                             : mlir::Value{};
   std::string listMangleName = tableMangleName + ".list";
   auto listFunc = [&](fir::FirOpBuilder &builder) {
     mlir::Value list = fir::UndefOp::create(builder, loc, listTy);
@@ -295,7 +296,7 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
     auto insert = [&](mlir::Value val) {
       idx[1] = intAttr[n1++];
       list = fir::InsertValueOp::create(builder, loc, listTy, list, val,
-                                                builder.getArrayAttr(idx));
+                                        builder.getArrayAttr(idx));
     };
     for (auto &iface : definedIoProcMap) {
       idx[0] = builder.getIntegerAttr(idxTy, n0++);
@@ -305,8 +306,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       std::string dtName = converter.mangleName(dtSym);
       insert(builder.createConvert(
           loc, refTy,
-          fir::AddrOfOp::create(builder,
-              loc, fir::ReferenceType::get(converter.genType(dtSym)),
+          fir::AddrOfOp::create(
+              builder, loc, fir::ReferenceType::get(converter.genType(dtSym)),
               builder.getSymbolRefAttr(dtName))));
       // defined IO procedure [void (*subroutine)()], may be null
       const Fortran::semantics::Symbol *procSym = iface.second.subroutine;
@@ -316,8 +317,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
           TODO(loc, "defined IO procedure pointers");
         } else if (Fortran::semantics::IsDummy(*procSym)) {
           Fortran::lower::StatementContext stmtCtx;
-          insert(fir::BoxAddrOp::create(builder,
-              loc, refTy,
+          insert(fir::BoxAddrOp::create(
+              builder, loc, refTy,
               fir::getBase(converter.genExprAddr(
                   loc,
                   Fortran::lower::SomeExpr{
@@ -331,7 +332,7 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
           insert(builder.createConvert(
               loc, refTy,
               fir::AddrOfOp::create(builder, loc, procDef.getFunctionType(),
-                                            nameAttr)));
+                                    nameAttr)));
         }
       } else {
         insert(builder.createNullConstant(loc, refTy));
@@ -365,8 +366,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   auto tableFunc = [&](fir::FirOpBuilder &builder) {
     mlir::Value table = fir::UndefOp::create(builder, loc, tableTy);
     // list item count [std::size_t items]
-    table = fir::InsertValueOp::create(builder,
-        loc, tableTy, table,
+    table = fir::InsertValueOp::create(
+        builder, loc, tableTy, table,
         builder.createIntegerConstant(loc, sizeTy, definedIoProcMap.size()),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 0)));
     // item list [const NonTbpDefinedIo *item]
@@ -374,14 +375,15 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       listAddr = builder.createNullConstant(loc, builder.getRefType(listTy));
     else if (fir::GlobalOp list = builder.getNamedGlobal(listMangleName))
       listAddr = fir::AddrOfOp::create(builder, loc, list.resultType(),
-                                               list.getSymbol());
+                                       list.getSymbol());
     assert(listAddr && "missing namelist object list");
-    table = fir::InsertValueOp::create(builder,
-        loc, tableTy, table, listAddr,
+    table = fir::InsertValueOp::create(
+        builder, loc, tableTy, table, listAddr,
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 1)));
     // [bool ignoreNonTbpEntries] conservatively set to true
-    table = fir::InsertValueOp::create(builder,
-        loc, tableTy, table, builder.createIntegerConstant(loc, boolTy, true),
+    table = fir::InsertValueOp::create(
+        builder, loc, tableTy, table,
+        builder.createIntegerConstant(loc, boolTy, true),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 2)));
     if (tableIsLocal)
       fir::StoreOp::create(builder, loc, table, tableAddr);
@@ -394,8 +396,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
     fir::GlobalOp table = builder.createGlobal(
         loc, tableTy, tableMangleName,
         /*isConst=*/true, /*isTarget=*/false, tableFunc, linkOnce);
-    tableAddr = fir::AddrOfOp::create(builder,
-        loc, fir::ReferenceType::get(tableTy), table.getSymbol());
+    tableAddr = fir::AddrOfOp::create(
+        builder, loc, fir::ReferenceType::get(tableTy), table.getSymbol());
   }
   assert(tableAddr && "missing NonTbpDefinedIo table result");
   return builder.createConvert(loc, refTy, tableAddr);
@@ -421,7 +423,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
   std::string groupMangleName = converter.mangleName(symbol);
   if (auto group = builder.getNamedGlobal(groupMangleName))
     return fir::AddrOfOp::create(builder, loc, group.resultType(),
-                                         group.getSymbol());
+                                 group.getSymbol());
 
   const auto &details =
       symbol.GetUltimate().get<Fortran::semantics::NamelistDetails>();
@@ -475,8 +477,9 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
   }
 
   // Define the list of Items.
-  mlir::Value listAddr =
-      groupIsLocal ? fir::AllocaOp::create(builder, loc, listTy) : mlir::Value{};
+  mlir::Value listAddr = groupIsLocal
+                             ? fir::AllocaOp::create(builder, loc, listTy)
+                             : mlir::Value{};
   std::string listMangleName = groupMangleName + ".list";
   auto listFunc = [&](fir::FirOpBuilder &builder) {
     mlir::Value list = fir::UndefOp::create(builder, loc, listTy);
@@ -491,13 +494,13 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       mlir::Value nameAddr =
           builder.createConvert(loc, charRefTy, fir::getBase(stringAddress(s)));
       list = fir::InsertValueOp::create(builder, loc, listTy, list, nameAddr,
-                                                builder.getArrayAttr(idx));
+                                        builder.getArrayAttr(idx));
       idx[1] = one;
       mlir::Value descAddr;
       if (auto desc = builder.getNamedGlobal(
               Fortran::lower::mangle::globalNamelistDescriptorName(s))) {
         descAddr = fir::AddrOfOp::create(builder, loc, desc.resultType(),
-                                                 desc.getSymbol());
+                                         desc.getSymbol());
       } else if (Fortran::semantics::FindCommonBlockContaining(s) &&
                  IsAllocatableOrPointer(s)) {
         mlir::Type symType = converter.genType(s);
@@ -505,8 +508,8 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
             Fortran::semantics::FindCommonBlockContaining(s);
         std::string commonBlockName = converter.mangleName(*commonBlockSym);
         fir::GlobalOp commonGlobal = builder.getNamedGlobal(commonBlockName);
-        mlir::Value commonBlockAddr = fir::AddrOfOp::create(builder,
-            loc, commonGlobal.resultType(), commonGlobal.getSymbol());
+        mlir::Value commonBlockAddr = fir::AddrOfOp::create(
+            builder, loc, commonGlobal.resultType(), commonGlobal.getSymbol());
         mlir::IntegerType i8Ty = builder.getIntegerType(8);
         mlir::Type i8Ptr = builder.getRefType(i8Ty);
         mlir::Type seqTy = builder.getRefType(builder.getVarLenSeqTy(i8Ty));
@@ -514,8 +517,8 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
         std::size_t byteOffset = s.GetUltimate().offset();
         mlir::Value offs = builder.createIntegerConstant(
             loc, builder.getIndexType(), byteOffset);
-        mlir::Value varAddr = fir::CoordinateOp::create(builder,
-            loc, i8Ptr, base, mlir::ValueRange{offs});
+        mlir::Value varAddr = fir::CoordinateOp::create(
+            builder, loc, i8Ptr, base, mlir::ValueRange{offs});
         descAddr =
             builder.createConvert(loc, builder.getRefType(symType), varAddr);
       } else {
@@ -532,7 +535,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       }
       descAddr = builder.createConvert(loc, descRefTy, descAddr);
       list = fir::InsertValueOp::create(builder, loc, listTy, list, descAddr,
-                                                builder.getArrayAttr(idx));
+                                        builder.getArrayAttr(idx));
     }
     if (groupIsLocal)
       fir::StoreOp::create(builder, loc, list, listAddr);
@@ -552,28 +555,28 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
   auto groupFunc = [&](fir::FirOpBuilder &builder) {
     mlir::Value group = fir::UndefOp::create(builder, loc, groupTy);
     // group name [const char *groupName]
-    group = fir::InsertValueOp::create(builder,
-        loc, groupTy, group,
+    group = fir::InsertValueOp::create(
+        builder, loc, groupTy, group,
         builder.createConvert(loc, charRefTy,
                               fir::getBase(stringAddress(symbol))),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 0)));
     // list item count [std::size_t items]
-    group = fir::InsertValueOp::create(builder,
-        loc, groupTy, group,
+    group = fir::InsertValueOp::create(
+        builder, loc, groupTy, group,
         builder.createIntegerConstant(loc, sizeTy, details.objects().size()),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 1)));
     // item list [const Item *item]
     if (fir::GlobalOp list = builder.getNamedGlobal(listMangleName))
       listAddr = fir::AddrOfOp::create(builder, loc, list.resultType(),
-                                               list.getSymbol());
+                                       list.getSymbol());
     assert(listAddr && "missing namelist object list");
-    group = fir::InsertValueOp::create(builder,
-        loc, groupTy, group, listAddr,
+    group = fir::InsertValueOp::create(
+        builder, loc, groupTy, group, listAddr,
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 2)));
     // non-type-bound defined IO procedures
     // [const NonTbpDefinedIoTable *nonTbpDefinedIo]
-    group = fir::InsertValueOp::create(builder,
-        loc, groupTy, group,
+    group = fir::InsertValueOp::create(
+        builder, loc, groupTy, group,
         getNonTbpDefinedIoTableAddr(converter, definedIoProcMap),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 3)));
     if (groupIsLocal)
@@ -588,7 +591,7 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
         loc, groupTy, groupMangleName,
         /*isConst=*/true, /*isTarget=*/false, groupFunc, linkOnce);
     groupAddr = fir::AddrOfOp::create(builder, loc, group.resultType(),
-                                              group.getSymbol());
+                                      group.getSymbol());
   }
   assert(groupAddr && "missing namelist group result");
   return groupAddr;
@@ -849,9 +852,10 @@ createIoRuntimeCallForItem(Fortran::lower::AbstractConverter &converter,
       inputFuncArgs.push_back(builder.createConvert(
           loc, inputFunc.getFunctionType().getInput(2), len));
     } else if (mlir::isa<mlir::IntegerType>(itemTy)) {
-      inputFuncArgs.push_back(mlir::arith::ConstantOp::create(builder,
-          loc, builder.getI32IntegerAttr(
-                   mlir::cast<mlir::IntegerType>(itemTy).getWidth() / 8)));
+      inputFuncArgs.push_back(mlir::arith::ConstantOp::create(
+          builder, loc,
+          builder.getI32IntegerAttr(
+              mlir::cast<mlir::IntegerType>(itemTy).getWidth() / 8)));
     }
   }
   auto call = fir::CallOp::create(builder, loc, inputFunc, inputFuncArgs);
@@ -962,17 +966,17 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   };
   if (!checkResult) {
     // No IO call result checks - the loop is a fir.do_loop op.
-    auto doLoopOp = fir::DoLoopOp::create(builder,
-        loc, lowerValue, upperValue, stepValue, /*unordered=*/false,
-        /*finalCountValue=*/true);
+    auto doLoopOp = fir::DoLoopOp::create(builder, loc, lowerValue, upperValue,
+                                          stepValue, /*unordered=*/false,
+                                          /*finalCountValue=*/true);
     builder.setInsertionPointToStart(doLoopOp.getBody());
     mlir::Value lcv = builder.createConvert(
         loc, fir::unwrapRefType(loopVar.getType()), doLoopOp.getInductionVar());
     fir::StoreOp::create(builder, loc, lcv, loopVar);
     genItemList(ioImpliedDo);
     builder.setInsertionPointToEnd(doLoopOp.getBody());
-    mlir::Value result = mlir::arith::AddIOp::create(builder,
-        loc, doLoopOp.getInductionVar(), doLoopOp.getStep(), iofAttr);
+    mlir::Value result = mlir::arith::AddIOp::create(
+        builder, loc, doLoopOp.getInductionVar(), doLoopOp.getStep(), iofAttr);
     fir::ResultOp::create(builder, loc, result);
     builder.setInsertionPointAfter(doLoopOp);
     // The loop control variable may be used after the loop.
@@ -984,8 +988,9 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   // Check IO call results - the loop is a fir.iterate_while op.
   if (!ok)
     ok = builder.createBool(loc, true);
-  auto iterWhileOp = fir::IterWhileOp::create(builder,
-      loc, lowerValue, upperValue, stepValue, ok, /*finalCountValue*/ true);
+  auto iterWhileOp =
+      fir::IterWhileOp::create(builder, loc, lowerValue, upperValue, stepValue,
+                               ok, /*finalCountValue*/ true);
   builder.setInsertionPointToStart(iterWhileOp.getBody());
   mlir::Value lcv =
       builder.createConvert(loc, fir::unwrapRefType(loopVar.getType()),
@@ -1013,10 +1018,10 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   builder.setInsertionPointToEnd(iterWhileOp.getBody());
   mlir::OpResult iterateResult = builder.getBlock()->back().getResult(0);
   mlir::Value inductionResult0 = iterWhileOp.getInductionVar();
-  auto inductionResult1 = mlir::arith::AddIOp::create(builder,
-      loc, inductionResult0, iterWhileOp.getStep(), iofAttr);
-  auto inductionResult = mlir::arith::SelectOp::create(builder,
-      loc, iterateResult, inductionResult1, inductionResult0);
+  auto inductionResult1 = mlir::arith::AddIOp::create(
+      builder, loc, inductionResult0, iterWhileOp.getStep(), iofAttr);
+  auto inductionResult = mlir::arith::SelectOp::create(
+      builder, loc, iterateResult, inductionResult1, inductionResult0);
   llvm::SmallVector<mlir::Value> results = {inductionResult, iterateResult};
   fir::ResultOp::create(builder, loc, results);
   ok = iterWhileOp.getResult(1);
@@ -1046,15 +1051,15 @@ static mlir::Value locToLineNo(Fortran::lower::AbstractConverter &converter,
 
 static mlir::Value getDefaultScratch(fir::FirOpBuilder &builder,
                                      mlir::Location loc, mlir::Type toType) {
-  mlir::Value null = mlir::arith::ConstantOp::create(builder,
-      loc, builder.getI64IntegerAttr(0));
+  mlir::Value null = mlir::arith::ConstantOp::create(
+      builder, loc, builder.getI64IntegerAttr(0));
   return builder.createConvert(loc, toType, null);
 }
 
 static mlir::Value getDefaultScratchLen(fir::FirOpBuilder &builder,
                                         mlir::Location loc, mlir::Type toType) {
-  return mlir::arith::ConstantOp::create(builder,
-      loc, builder.getIntegerAttr(toType, 0));
+  return mlir::arith::ConstantOp::create(builder, loc,
+                                         builder.getIntegerAttr(toType, 0));
 }
 
 /// Generate a reference to a buffer and the length of buffer given
@@ -1105,8 +1110,8 @@ lowerStringLit(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
   mlir::Value kind;
   if (ty2) {
     auto kindVal = expr->GetType().value().kind();
-    kind = mlir::arith::ConstantOp::create(builder,
-        loc, builder.getIntegerAttr(ty2, kindVal));
+    kind = mlir::arith::ConstantOp::create(
+        builder, loc, builder.getIntegerAttr(ty2, kindVal));
   }
   return {buff, len, kind};
 }
@@ -1480,8 +1485,8 @@ genConditionHandlerCall(Fortran::lower::AbstractConverter &converter,
       fir::runtime::getIORuntimeFunc<mkIOKey(EnableHandlers)>(loc, builder);
   mlir::Type boolType = enableHandlers.getFunctionType().getInput(1);
   auto boolValue = [&](bool specifierIsPresent) {
-    return mlir::arith::ConstantOp::create(builder,
-        loc, builder.getIntegerAttr(boolType, specifierIsPresent));
+    return mlir::arith::ConstantOp::create(
+        builder, loc, builder.getIntegerAttr(boolType, specifierIsPresent));
   };
   llvm::SmallVector<mlir::Value> ioArgs = {cookie,
                                            boolValue(csi.ioStatExpr != nullptr),
@@ -1819,10 +1824,10 @@ static mlir::Value genIOUnitNumber(Fortran::lower::AbstractConverter &converter,
       mlir::Value iostat = checkCall.getResult(0);
       mlir::Type iostatTy = iostat.getType();
       mlir::Value zero = fir::factory::createZeroValue(builder, loc, iostatTy);
-      mlir::Value unitIsOK = mlir::arith::CmpIOp::create(builder,
-          loc, mlir::arith::CmpIPredicate::eq, iostat, zero);
+      mlir::Value unitIsOK = mlir::arith::CmpIOp::create(
+          builder, loc, mlir::arith::CmpIPredicate::eq, iostat, zero);
       auto ifOp = fir::IfOp::create(builder, loc, iostatTy, unitIsOK,
-                                            /*withElseRegion=*/true);
+                                    /*withElseRegion=*/true);
       builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
       fir::ResultOp::create(builder, loc, iostat);
       builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
@@ -1846,8 +1851,8 @@ static mlir::Value genIOUnit(Fortran::lower::AbstractConverter &converter,
                 &iounit->u))
       return genIOUnitNumber(converter, loc, Fortran::semantics::GetExpr(*e),
                              ty, csi, stmtCtx);
-  return mlir::arith::ConstantOp::create(builder,
-      loc, builder.getIntegerAttr(ty, defaultUnitNumber));
+  return mlir::arith::ConstantOp::create(
+      builder, loc, builder.getIntegerAttr(ty, defaultUnitNumber));
 }
 
 template <typename A>
@@ -1880,7 +1885,7 @@ static mlir::Value genBasicIOStmt(Fortran::lower::AbstractConverter &converter,
   mlir::Value file = locToFilename(converter, loc, beginFuncTy.getInput(1));
   mlir::Value line = locToLineNo(converter, loc, beginFuncTy.getInput(2));
   auto call = fir::CallOp::create(builder, loc, beginFunc,
-                                          mlir::ValueRange{un, file, line});
+                                  mlir::ValueRange{un, file, line});
   mlir::Value cookie = call.getResult(0);
   genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
   mlir::Value ok;
@@ -2149,9 +2154,10 @@ void genBeginDataTransferCallArgs(
     }
   } else { // PRINT - maybe explicit format; default unit
     maybeGetFormatArgs();
-    ioArgs.push_back(mlir::arith::ConstantOp::create(builder,
-        loc, builder.getIntegerAttr(ioFuncTy.getInput(ioArgs.size()),
-                                    defaultUnitNumber)));
+    ioArgs.push_back(mlir::arith::ConstantOp::create(
+        builder, loc,
+        builder.getIntegerAttr(ioFuncTy.getInput(ioArgs.size()),
+                               defaultUnitNumber)));
   }
   // File name and line number are always the last two arguments.
   ioArgs.push_back(
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 13b2d5155..471f36859 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -264,8 +264,8 @@ static void createDeclareAllocFuncWithArg(mlir::OpBuilder &modBuilder,
         builder, loc, boxAddrOp.getResult(), asFortran, bounds,
         /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-    mlir::acc::DeclareEnterOp::create(builder,
-        loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
+    mlir::acc::DeclareEnterOp::create(
+        builder, loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
         mlir::ValueRange(entryOp.getAccVar()));
   }
 
@@ -303,25 +303,25 @@ static void createDeclareDeallocFuncWithArg(
           builder, loc, var, asFortran, bounds,
           /*structured=*/false, /*implicit=*/false, clause, var.getType(),
           /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-  mlir::acc::DeclareExitOp::create(builder,
-      loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccVar()));
+  mlir::acc::DeclareExitOp::create(builder, loc, mlir::Value{},
+                                   mlir::ValueRange(entryOp.getAccVar()));
 
   if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                 std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
     ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
-                           entryOp.getVar(), entryOp.getVarType(),
-                           entryOp.getBounds(), entryOp.getAsyncOperands(),
-                           entryOp.getAsyncOperandsDeviceTypeAttr(),
-                           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-                           /*structured=*/false, /*implicit=*/false,
-                           builder.getStringAttr(*entryOp.getName()));
+                   entryOp.getVar(), entryOp.getVarType(), entryOp.getBounds(),
+                   entryOp.getAsyncOperands(),
+                   entryOp.getAsyncOperandsDeviceTypeAttr(),
+                   entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                   /*structured=*/false, /*implicit=*/false,
+                   builder.getStringAttr(*entryOp.getName()));
   else
     ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
-                           entryOp.getBounds(), entryOp.getAsyncOperands(),
-                           entryOp.getAsyncOperandsDeviceTypeAttr(),
-                           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-                           /*structured=*/false, /*implicit=*/false,
-                           builder.getStringAttr(*entryOp.getName()));
+                   entryOp.getBounds(), entryOp.getAsyncOperands(),
+                   entryOp.getAsyncOperandsDeviceTypeAttr(),
+                   entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                   /*structured=*/false, /*implicit=*/false,
+                   builder.getStringAttr(*entryOp.getName()));
 
   // Generate the post dealloc function.
   modBuilder.setInsertionPointAfter(preDeallocOp);
@@ -385,8 +385,8 @@ genAtomicCaptureStatement(Fortran::lower::AbstractConverter &converter,
   // Generate `atomic.read` operation for atomic assigment statements
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
-  mlir::acc::AtomicReadOp::create(firOpBuilder,
-      loc, fromAddress, toAddress, mlir::TypeAttr::get(elementType));
+  mlir::acc::AtomicReadOp::create(firOpBuilder, loc, fromAddress, toAddress,
+                                  mlir::TypeAttr::get(elementType));
 }
 
 /// Used to generate atomic.write operation which is created in existing
@@ -839,15 +839,15 @@ genDataExitOperations(fir::FirOpBuilder &builder,
     mlir::Location opLoc = exitLoc ? *exitLoc : entryOp.getLoc();
     if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                   std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-      ExitOp::create(builder,
-          opLoc, entryOp.getAccVar(), entryOp.getVar(), entryOp.getVarType(),
-          entryOp.getBounds(), entryOp.getAsyncOperands(),
+      ExitOp::create(
+          builder, opLoc, entryOp.getAccVar(), entryOp.getVar(),
+          entryOp.getVarType(), entryOp.getBounds(), entryOp.getAsyncOperands(),
           entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
           entryOp.getDataClause(), structured, entryOp.getImplicit(),
           builder.getStringAttr(*entryOp.getName()));
     else
-      ExitOp::create(builder,
-          opLoc, entryOp.getAccVar(), entryOp.getBounds(),
+      ExitOp::create(
+          builder, opLoc, entryOp.getAccVar(), entryOp.getBounds(),
           entryOp.getAsyncOperands(), entryOp.getAsyncOperandsDeviceTypeAttr(),
           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(), structured,
           entryOp.getImplicit(), builder.getStringAttr(*entryOp.getName()));
@@ -859,8 +859,8 @@ fir::ShapeOp genShapeOp(mlir::OpBuilder &builder, fir::SequenceType seqTy,
   llvm::SmallVector<mlir::Value> extents;
   mlir::Type idxTy = builder.getIndexType();
   for (auto extent : seqTy.getShape())
-    extents.push_back(mlir::arith::ConstantOp::create(builder,
-        loc, idxTy, builder.getIntegerAttr(idxTy, extent)));
+    extents.push_back(mlir::arith::ConstantOp::create(
+        builder, loc, idxTy, builder.getIntegerAttr(idxTy, extent)));
   return fir::ShapeOp::create(builder, loc, extents);
 }
 
@@ -936,8 +936,8 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
     return builder.createBool(loc, value);
   }
   if (ty.isIntOrIndex())
-    return mlir::arith::ConstantOp::create(builder,
-        loc, ty,
+    return mlir::arith::ConstantOp::create(
+        builder, loc, ty,
         builder.getIntegerAttr(ty, getReductionInitValue<llvm::APInt>(op, ty)));
   if (op == mlir::acc::ReductionOperator::AccMin ||
       op == mlir::acc::ReductionOperator::AccMax) {
@@ -945,13 +945,13 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
       llvm::report_fatal_error(
           "min/max reduction not supported for complex type");
     if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty))
-      return mlir::arith::ConstantOp::create(builder,
-          loc, ty,
+      return mlir::arith::ConstantOp::create(
+          builder, loc, ty,
           builder.getFloatAttr(ty,
                                getReductionInitValue<llvm::APFloat>(op, ty)));
   } else if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty)) {
-    return mlir::arith::ConstantOp::create(builder,
-        loc, ty,
+    return mlir::arith::ConstantOp::create(
+        builder, loc, ty,
         builder.getFloatAttr(ty, getReductionInitValue<int64_t>(op, ty)));
   } else if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
     mlir::Type floatTy = cmplxTy.getElementType();
@@ -986,7 +986,7 @@ static RecipeOp genRecipeOp(
   RecipeOp recipe;
   if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>) {
     recipe = mlir::acc::ReductionRecipeOp::create(modBuilder, loc, recipeName,
-                                                             ty, op);
+                                                  ty, op);
   } else {
     recipe = RecipeOp::create(modBuilder, loc, recipeName, ty);
   }
@@ -1032,8 +1032,8 @@ static RecipeOp genRecipeOp(
       initName,
       initBlock->getArguments().take_back(initBlock->getArguments().size() - 1),
       initValue);
-  mlir::acc::YieldOp::create(builder, loc, retVal ? retVal
-                                                 : initBlock->getArgument(0));
+  mlir::acc::YieldOp::create(builder, loc,
+                             retVal ? retVal : initBlock->getArgument(0));
   return recipe;
 }
 
@@ -1134,10 +1134,12 @@ static mlir::Value genShapeFromBoundsOrArgs(
     mlir::Value s1 =
         mlir::arith::SubIOp::create(builder, loc, args[i + 1], args[0]);
     mlir::Value s2 = mlir::arith::AddIOp::create(builder, loc, s1, one);
-    mlir::Value s3 = mlir::arith::DivSIOp::create(builder, loc, s2, args[i + 2]);
-    mlir::Value cmp = mlir::arith::CmpIOp::create(builder,
-        loc, mlir::arith::CmpIPredicate::sgt, s3, zero);
-    mlir::Value ext = mlir::arith::SelectOp::create(builder, loc, cmp, s3, zero);
+    mlir::Value s3 =
+        mlir::arith::DivSIOp::create(builder, loc, s2, args[i + 2]);
+    mlir::Value cmp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::sgt, s3, zero);
+    mlir::Value ext =
+        mlir::arith::SelectOp::create(builder, loc, cmp, s3, zero);
     extents.push_back(ext);
   }
   return fir::ShapeOp::create(builder, loc, extents);
@@ -1157,8 +1159,8 @@ static hlfir::Entity genDesignateWithTriplets(
     hlfir::DesignateOp::Subscripts &triplets, mlir::Value shape) {
   llvm::SmallVector<mlir::Value> lenParams;
   hlfir::genLengthParameters(loc, builder, entity, lenParams);
-  auto designate = hlfir::DesignateOp::create(builder,
-      loc, entity.getBase().getType(), entity, /*component=*/"",
+  auto designate = hlfir::DesignateOp::create(
+      builder, loc, entity.getBase().getType(), entity, /*component=*/"",
       /*componentShape=*/mlir::Value{}, triplets,
       /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt, shape,
       lenParams);
@@ -1198,22 +1200,22 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
   builder.setInsertionPointToEnd(&recipe.getCopyRegion().back());
   ty = fir::unwrapRefType(ty);
   if (fir::isa_trivial(ty)) {
-    mlir::Value initValue = fir::LoadOp::create(builder,
-        loc, recipe.getCopyRegion().front().getArgument(0));
+    mlir::Value initValue = fir::LoadOp::create(
+        builder, loc, recipe.getCopyRegion().front().getArgument(0));
     fir::StoreOp::create(builder, loc, initValue,
-                                 recipe.getCopyRegion().front().getArgument(1));
+                         recipe.getCopyRegion().front().getArgument(1));
   } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
     fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
     auto shape = genShapeFromBoundsOrArgs(
         loc, firBuilder, seqTy, bounds, recipe.getCopyRegion().getArguments());
 
-    auto leftDeclOp = hlfir::DeclareOp::create(builder,
-        loc, recipe.getCopyRegion().getArgument(0), llvm::StringRef{}, shape,
-        llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+    auto leftDeclOp = hlfir::DeclareOp::create(
+        builder, loc, recipe.getCopyRegion().getArgument(0), llvm::StringRef{},
+        shape, llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{});
-    auto rightDeclOp = hlfir::DeclareOp::create(builder,
-        loc, recipe.getCopyRegion().getArgument(1), llvm::StringRef{}, shape,
-        llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+    auto rightDeclOp = hlfir::DeclareOp::create(
+        builder, loc, recipe.getCopyRegion().getArgument(1), llvm::StringRef{},
+        shape, llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{});
 
     hlfir::DesignateOp::Subscripts triplets =
@@ -1520,19 +1522,21 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       auto shape =
           genShapeFromBoundsOrArgs(loc, builder, seqTy, bounds,
                                    recipe.getCombinerRegion().getArguments());
-      auto v1DeclareOp = hlfir::DeclareOp::create(builder,
-          loc, value1, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
+      auto v1DeclareOp = hlfir::DeclareOp::create(
+          builder, loc, value1, llvm::StringRef{}, shape,
+          llvm::ArrayRef<mlir::Value>{},
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
-      auto v2DeclareOp = hlfir::DeclareOp::create(builder,
-          loc, value2, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
+      auto v2DeclareOp = hlfir::DeclareOp::create(
+          builder, loc, value2, llvm::StringRef{}, shape,
+          llvm::ArrayRef<mlir::Value>{},
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       hlfir::DesignateOp::Subscripts triplets = getTripletsFromArgs(recipe);
 
       llvm::SmallVector<mlir::Value> lenParamsLeft;
       auto leftEntity = hlfir::Entity{v1DeclareOp.getBase()};
       hlfir::genLengthParameters(loc, builder, leftEntity, lenParamsLeft);
-      auto leftDesignate = hlfir::DesignateOp::create(builder,
-          loc, v1DeclareOp.getBase().getType(), v1DeclareOp.getBase(),
+      auto leftDesignate = hlfir::DesignateOp::create(
+          builder, loc, v1DeclareOp.getBase().getType(), v1DeclareOp.getBase(),
           /*component=*/"",
           /*componentShape=*/mlir::Value{}, triplets,
           /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
@@ -1542,8 +1546,8 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       llvm::SmallVector<mlir::Value> lenParamsRight;
       auto rightEntity = hlfir::Entity{v2DeclareOp.getBase()};
       hlfir::genLengthParameters(loc, builder, rightEntity, lenParamsLeft);
-      auto rightDesignate = hlfir::DesignateOp::create(builder,
-          loc, v2DeclareOp.getBase().getType(), v2DeclareOp.getBase(),
+      auto rightDesignate = hlfir::DesignateOp::create(
+          builder, loc, v2DeclareOp.getBase().getType(), v2DeclareOp.getBase(),
           /*component=*/"",
           /*componentShape=*/mlir::Value{}, triplets,
           /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
@@ -1571,14 +1575,14 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       llvm::SmallVector<mlir::Value> extents;
       mlir::Type idxTy = builder.getIndexType();
       for (auto extent : seqTy.getShape()) {
-        mlir::Value lb = mlir::arith::ConstantOp::create(builder,
-            loc, idxTy, builder.getIntegerAttr(idxTy, 0));
-        mlir::Value ub = mlir::arith::ConstantOp::create(builder,
-            loc, idxTy, builder.getIntegerAttr(idxTy, extent - 1));
-        mlir::Value step = mlir::arith::ConstantOp::create(builder,
-            loc, idxTy, builder.getIntegerAttr(idxTy, 1));
+        mlir::Value lb = mlir::arith::ConstantOp::create(
+            builder, loc, idxTy, builder.getIntegerAttr(idxTy, 0));
+        mlir::Value ub = mlir::arith::ConstantOp::create(
+            builder, loc, idxTy, builder.getIntegerAttr(idxTy, extent - 1));
+        mlir::Value step = mlir::arith::ConstantOp::create(
+            builder, loc, idxTy, builder.getIntegerAttr(idxTy, 1));
         auto loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
-                                                  /*unordered=*/false);
+                                          /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
         ivs.push_back(loop.getInductionVar());
@@ -1595,7 +1599,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
             genConstantBounds(builder, loc, dataBound);
         auto loop =
             fir::DoLoopOp::create(builder, loc, values[0], values[1], values[2],
-                                          /*unordered=*/false);
+                                  /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
         ivs.push_back(loop.getInductionVar());
@@ -1612,7 +1616,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
         mlir::Value ub = recipe.getCombinerRegion().getArgument(i + 1);
         mlir::Value step = recipe.getCombinerRegion().getArgument(i + 2);
         auto loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
-                                                  /*unordered=*/false);
+                                          /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
         ivs.push_back(loop.getInductionVar());
@@ -1658,14 +1662,14 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
           getSubscriptsFromArgs(recipe.getCombinerRegion().getArguments());
       auto leftEntity = hlfir::Entity{value1};
       if (fir::isBoxAddress(value1.getType()))
-        leftEntity =
-            hlfir::Entity{fir::LoadOp::create(builder, loc, value1).getResult()};
+        leftEntity = hlfir::Entity{
+            fir::LoadOp::create(builder, loc, value1).getResult()};
       auto left =
           genDesignateWithTriplets(builder, loc, leftEntity, triplets, shape);
       auto rightEntity = hlfir::Entity{value2};
       if (fir::isBoxAddress(value2.getType()))
-        rightEntity =
-            hlfir::Entity{fir::LoadOp::create(builder, loc, value2).getResult()};
+        rightEntity = hlfir::Entity{
+            fir::LoadOp::create(builder, loc, value2).getResult()};
       auto right =
           genDesignateWithTriplets(builder, loc, rightEntity, triplets, shape);
 
@@ -3750,8 +3754,8 @@ genACCUpdateOp(Fortran::lower::AbstractConverter &converter,
 
   dataClauseOperands.append(updateHostOperands);
 
-  mlir::acc::UpdateOp::create(builder,
-      currentLocation, ifCond, asyncOperands,
+  mlir::acc::UpdateOp::create(
+      builder, currentLocation, ifCond, asyncOperands,
       getArrayAttr(builder, asyncOperandsDeviceTypes),
       getArrayAttr(builder, asyncOnlyDeviceTypes), waitOperands,
       getDenseI32ArrayAttr(builder, waitOperandsSegments),
@@ -3878,8 +3882,9 @@ static void createDeclareGlobalOp(mlir::OpBuilder &modBuilder,
                       declareGlobalOp.getRegion().end(), {}, {});
   builder.setInsertionPointToEnd(&declareGlobalOp.getRegion().back());
 
-  fir::AddrOfOp addrOp = fir::AddrOfOp::create(builder,
-      loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(
+      builder, loc, fir::ReferenceType::get(globalOp.getType()),
+      globalOp.getSymbol());
   addDeclareAttr(builder, addrOp, clause);
 
   llvm::SmallVector<mlir::Value> bounds;
@@ -3888,19 +3893,19 @@ static void createDeclareGlobalOp(mlir::OpBuilder &modBuilder,
       /*structured=*/false, implicit, clause, addrOp.getResTy().getType(),
       /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
   if constexpr (std::is_same_v<DeclareOp, mlir::acc::DeclareEnterOp>)
-    DeclareOp::create(builder,
-        loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
-        mlir::ValueRange(entryOp.getAccVar()));
+    DeclareOp::create(builder, loc,
+                      mlir::acc::DeclareTokenType::get(entryOp.getContext()),
+                      mlir::ValueRange(entryOp.getAccVar()));
   else
     DeclareOp::create(builder, loc, mlir::Value{},
-                              mlir::ValueRange(entryOp.getAccVar()));
+                      mlir::ValueRange(entryOp.getAccVar()));
   if constexpr (std::is_same_v<GlobalOp, mlir::acc::GlobalDestructorOp>) {
     ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
-                           entryOp.getBounds(), entryOp.getAsyncOperands(),
-                           entryOp.getAsyncOperandsDeviceTypeAttr(),
-                           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-                           /*structured=*/false, /*implicit=*/false,
-                           builder.getStringAttr(*entryOp.getName()));
+                   entryOp.getBounds(), entryOp.getAsyncOperands(),
+                   entryOp.getAsyncOperandsDeviceTypeAttr(),
+                   entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                   /*structured=*/false, /*implicit=*/false,
+                   builder.getStringAttr(*entryOp.getName()));
   }
   mlir::acc::TerminatorOp::create(builder, loc);
   modBuilder.setInsertionPointAfter(declareGlobalOp);
@@ -3917,8 +3922,9 @@ static void createDeclareAllocFunc(mlir::OpBuilder &modBuilder,
   auto registerFuncOp =
       createDeclareFunc(modBuilder, builder, loc, registerFuncName.str());
 
-  fir::AddrOfOp addrOp = fir::AddrOfOp::create(builder,
-      loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(
+      builder, loc, fir::ReferenceType::get(globalOp.getType()),
+      globalOp.getSymbol());
 
   std::stringstream asFortran;
   asFortran << Fortran::lower::mangle::demangleName(globalOp.getSymName());
@@ -3948,8 +3954,8 @@ static void createDeclareAllocFunc(mlir::OpBuilder &modBuilder,
         builder, loc, boxAddrOp.getResult(), asFortran, bounds,
         /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-    mlir::acc::DeclareEnterOp::create(builder,
-        loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
+    mlir::acc::DeclareEnterOp::create(
+        builder, loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
         mlir::ValueRange(entryOp.getAccVar()));
   }
 
@@ -3980,8 +3986,9 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
     auto preDeallocOp =
         createDeclareFunc(modBuilder, builder, loc, preDeallocFuncName.str());
 
-    fir::AddrOfOp addrOp = fir::AddrOfOp::create(builder,
-        loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
+    fir::AddrOfOp addrOp = fir::AddrOfOp::create(
+        builder, loc, fir::ReferenceType::get(globalOp.getType()),
+        globalOp.getSymbol());
     auto loadOp = fir::LoadOp::create(builder, loc, addrOp.getResult());
     fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, loadOp);
     mlir::Value var = boxAddrOp.getResult();
@@ -3994,25 +4001,25 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
             /*structured=*/false, /*implicit=*/false, clause, var.getType(),
             /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
 
-    mlir::acc::DeclareExitOp::create(builder,
-        loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccVar()));
+    mlir::acc::DeclareExitOp::create(builder, loc, mlir::Value{},
+                                     mlir::ValueRange(entryOp.getAccVar()));
 
     if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                   std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-      ExitOp::create(builder,
-          entryOp.getLoc(), entryOp.getAccVar(), entryOp.getVar(),
-          entryOp.getBounds(), entryOp.getAsyncOperands(),
-          entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
-          entryOp.getDataClause(),
-          /*structured=*/false, /*implicit=*/false,
-          builder.getStringAttr(*entryOp.getName()));
+      ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
+                     entryOp.getVar(), entryOp.getBounds(),
+                     entryOp.getAsyncOperands(),
+                     entryOp.getAsyncOperandsDeviceTypeAttr(),
+                     entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                     /*structured=*/false, /*implicit=*/false,
+                     builder.getStringAttr(*entryOp.getName()));
     else
-      ExitOp::create(builder,
-          entryOp.getLoc(), entryOp.getAccVar(), entryOp.getBounds(),
-          entryOp.getAsyncOperands(), entryOp.getAsyncOperandsDeviceTypeAttr(),
-          entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-          /*structured=*/false, /*implicit=*/false,
-          builder.getStringAttr(*entryOp.getName()));
+      ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
+                     entryOp.getBounds(), entryOp.getAsyncOperands(),
+                     entryOp.getAsyncOperandsDeviceTypeAttr(),
+                     entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                     /*structured=*/false, /*implicit=*/false,
+                     builder.getStringAttr(*entryOp.getName()));
 
     // Generate the post dealloc function.
     modBuilder.setInsertionPointAfter(preDeallocOp);
@@ -4024,8 +4031,9 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
   auto postDeallocOp =
       createDeclareFunc(modBuilder, builder, loc, postDeallocFuncName.str());
 
-  fir::AddrOfOp addrOp = fir::AddrOfOp::create(builder,
-      loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(
+      builder, loc, fir::ReferenceType::get(globalOp.getType()),
+      globalOp.getSymbol());
   if (unwrapFirBox)
     asFortran << accFirDescriptorPostfix.str();
   llvm::SmallVector<mlir::Value> bounds;
@@ -4262,13 +4270,13 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
   auto ops = funcOp.getOps<mlir::acc::DeclareEnterOp>();
   mlir::Value declareToken;
   if (ops.empty()) {
-    declareToken = mlir::acc::DeclareEnterOp::create(builder,
-        loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
+    declareToken = mlir::acc::DeclareEnterOp::create(
+        builder, loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
         dataClauseOperands);
   } else {
     auto declareOp = *ops.begin();
-    auto newDeclareOp = mlir::acc::DeclareEnterOp::create(builder,
-        loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
+    auto newDeclareOp = mlir::acc::DeclareEnterOp::create(
+        builder, loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
         declareOp.getDataClauseOperands());
     newDeclareOp.getDataClauseOperandsMutable().append(dataClauseOperands);
     declareToken = newDeclareOp.getToken();
@@ -4535,8 +4543,8 @@ void createOpenACCRoutineConstruct(
   std::string routineOpStr = routineOpName.str();
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  mlir::acc::RoutineOp::create(modBuilder,
-      loc, routineOpStr,
+  mlir::acc::RoutineOp::create(
+      modBuilder, loc, routineOpStr,
       mlir::SymbolRefAttr::get(builder.getContext(), funcName),
       getArrayAttrOrNull(builder, bindIdNames),
       getArrayAttrOrNull(builder, bindStrNames),
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 6794144de..9a233d2d8 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -528,8 +528,8 @@ genAtomicRead(lower::AbstractConverter &converter,
   }();
 
   builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = mlir::omp::AtomicReadOp::create(builder,
-      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
+  mlir::Operation *op = mlir::omp::AtomicReadOp::create(
+      builder, loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
       makeMemOrderAttr(converter, memOrder));
 
   if (atomType != storeType) {
@@ -581,8 +581,9 @@ genAtomicWrite(lower::AbstractConverter &converter,
   mlir::Value converted = builder.createConvert(loc, atomType, value);
 
   builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = mlir::omp::AtomicWriteOp::create(builder,
-      loc, atomAddr, converted, hint, makeMemOrderAttr(converter, memOrder));
+  mlir::Operation *op =
+      mlir::omp::AtomicWriteOp::create(builder, loc, atomAddr, converted, hint,
+                                       makeMemOrderAttr(converter, memOrder));
   return op;
 }
 
@@ -635,8 +636,8 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   }
 
   builder.restoreInsertionPoint(atomicAt);
-  auto updateOp = mlir::omp::AtomicUpdateOp::create(builder,
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+  auto updateOp = mlir::omp::AtomicUpdateOp::create(
+      builder, loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
@@ -731,8 +732,8 @@ void Fortran::lower::omp::lowerAtomic(
              "Expexcing two actions");
       (void)action0;
       (void)action1;
-      captureOp = mlir::omp::AtomicCaptureOp::create(builder,
-          loc, hint, makeMemOrderAttr(converter, memOrder));
+      captureOp = mlir::omp::AtomicCaptureOp::create(
+          builder, loc, hint, makeMemOrderAttr(converter, memOrder));
       // Set the non-atomic insertion point to before the atomic.capture.
       preAt = getInsertionPointBefore(captureOp);
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index d118bfffa..498eea471 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -831,10 +831,10 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
     // get the actual fir.ref<fir.char> type
     mlir::Type refType =
         fir::ReferenceType::get(mlir::cast<fir::BoxCharType>(eleTy).getEleTy());
-    auto unboxedDst = fir::UnboxCharOp::create(builder,
-        loc, refType, builder.getIndexType(), loadDst);
-    auto unboxedSrc = fir::UnboxCharOp::create(builder,
-        loc, refType, builder.getIndexType(), loadSrc);
+    auto unboxedDst = fir::UnboxCharOp::create(builder, loc, refType,
+                                               builder.getIndexType(), loadDst);
+    auto unboxedSrc = fir::UnboxCharOp::create(builder, loc, refType,
+                                               builder.getIndexType(), loadSrc);
     // Add length to type parameters
     typeparams.push_back(unboxedDst.getResult(1));
     dst = unboxedDst.getResult(0);
@@ -844,11 +844,11 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
         loc, builder.getCharacterLengthType(), *typeInfo.getCharLength());
     typeparams.push_back(charLen);
   }
-  auto declDst = hlfir::DeclareOp::create(builder,
-      loc, dst, copyFuncName + "_dst", shape, typeparams,
+  auto declDst = hlfir::DeclareOp::create(
+      builder, loc, dst, copyFuncName + "_dst", shape, typeparams,
       /*dummy_scope=*/nullptr, attrs);
-  auto declSrc = hlfir::DeclareOp::create(builder,
-      loc, src, copyFuncName + "_src", shape, typeparams,
+  auto declSrc = hlfir::DeclareOp::create(
+      builder, loc, src, copyFuncName + "_src", shape, typeparams,
       /*dummy_scope=*/nullptr, attrs);
   converter.copyVar(loc, declDst.getBase(), declSrc.getBase(), varAttrs);
   mlir::func::ReturnOp::create(builder, loc);
@@ -1002,8 +1002,8 @@ bool ClauseProcessor::processDepend(lower::SymMap &symMap,
       // allocations so this is not a reliable way to identify the dependency.
       if (auto ref = mlir::dyn_cast<fir::ReferenceType>(dependVar.getType()))
         if (fir::isa_box_type(ref.getElementType()))
-          dependVar = fir::LoadOp::create(builder,
-              converter.getCurrentLocation(), dependVar);
+          dependVar = fir::LoadOp::create(
+              builder, converter.getCurrentLocation(), dependVar);
 
       // The openmp dialect doesn't know what to do with boxes (and it would
       // break layering to teach it about them). The dependency variable can be
@@ -1012,8 +1012,8 @@ bool ClauseProcessor::processDepend(lower::SymMap &symMap,
       // Getting the address of the box data is okay because all the runtime
       // ultimately cares about is the base address of the array.
       if (fir::isa_box_type(dependVar.getType()))
-        dependVar = fir::BoxAddrOp::create(builder,
-            converter.getCurrentLocation(), dependVar);
+        dependVar = fir::BoxAddrOp::create(
+            builder, converter.getCurrentLocation(), dependVar);
 
       result.dependVars.push_back(dependVar);
     }
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index def5297ba..11e488371 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -355,14 +355,14 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
       vs.push_back(v);
       mlir::Value zero =
           firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
-      mlir::Value negativeStep = mlir::arith::CmpIOp::create(firOpBuilder,
-          loc, mlir::arith::CmpIPredicate::slt, step, zero);
-      mlir::Value vLT = mlir::arith::CmpIOp::create(firOpBuilder,
-          loc, mlir::arith::CmpIPredicate::slt, v, ub);
-      mlir::Value vGT = mlir::arith::CmpIOp::create(firOpBuilder,
-          loc, mlir::arith::CmpIPredicate::sgt, v, ub);
-      mlir::Value icmpOp = mlir::arith::SelectOp::create(firOpBuilder,
-          loc, negativeStep, vLT, vGT);
+      mlir::Value negativeStep = mlir::arith::CmpIOp::create(
+          firOpBuilder, loc, mlir::arith::CmpIPredicate::slt, step, zero);
+      mlir::Value vLT = mlir::arith::CmpIOp::create(
+          firOpBuilder, loc, mlir::arith::CmpIPredicate::slt, v, ub);
+      mlir::Value vGT = mlir::arith::CmpIOp::create(
+          firOpBuilder, loc, mlir::arith::CmpIPredicate::sgt, v, ub);
+      mlir::Value icmpOp = mlir::arith::SelectOp::create(
+          firOpBuilder, loc, negativeStep, vLT, vGT);
 
       if (cmpOp)
         cmpOp = mlir::arith::AndIOp::create(firOpBuilder, loc, cmpOp, icmpOp);
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 8d906550a..80905aa29 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -642,8 +642,8 @@ static void threadPrivatizeVars(lower::AbstractConverter &converter,
       op = declOp.getMemref().getDefiningOp();
     if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
       symValue = mlir::dyn_cast<mlir::omp::ThreadprivateOp>(op).getSymAddr();
-    return mlir::omp::ThreadprivateOp::create(firOpBuilder,
-        currentLocation, symValue.getType(), symValue);
+    return mlir::omp::ThreadprivateOp::create(firOpBuilder, currentLocation,
+                                              symValue.getType(), symValue);
   };
 
   llvm::SetVector<const semantics::Symbol *> threadprivateSyms;
@@ -1157,7 +1157,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
 
   auto insertMarker = [](fir::FirOpBuilder &builder) {
     mlir::Value undef = fir::UndefOp::create(builder, builder.getUnknownLoc(),
-                                                     builder.getIndexType());
+                                             builder.getIndexType());
     return undef.getDefiningOp();
   };
 
@@ -1332,8 +1332,8 @@ static void genBodyOfTargetDataOp(
   // Remembering the position for further insertion is important since
   // there are hlfir.declares inserted above while setting block arguments
   // and new code from the body should be inserted after that.
-  mlir::Value undefMarker = fir::UndefOp::create(firOpBuilder,
-      dataOp.getLoc(), firOpBuilder.getIndexType());
+  mlir::Value undefMarker = fir::UndefOp::create(firOpBuilder, dataOp.getLoc(),
+                                                 firOpBuilder.getIndexType());
 
   // Create blocks for unstructured regions. This has to be done since
   // blocks are initially allocated with the function as the parent region.
@@ -1497,7 +1497,7 @@ static void genBodyOfTargetOp(
 
         firOpBuilder.setInsertionPointToStart(entryBlock);
         auto loadOp = fir::LoadOp::create(firOpBuilder, clonedValArg.getLoc(),
-                                                       clonedValArg);
+                                          clonedValArg);
         val.replaceUsesWithIf(loadOp->getResult(0),
                               [entryBlock](mlir::OpOperand &use) {
                                 return use.getOwner()->getBlock() == entryBlock;
@@ -1513,8 +1513,8 @@ static void genBodyOfTargetOp(
   // marker will be deleted since there are not uses.
   // In the HLFIR flow there are hlfir.declares inserted above while
   // setting block arguments.
-  mlir::Value undefMarker = fir::UndefOp::create(firOpBuilder,
-      targetOp.getLoc(), firOpBuilder.getIndexType());
+  mlir::Value undefMarker = fir::UndefOp::create(
+      firOpBuilder, targetOp.getLoc(), firOpBuilder.getIndexType());
 
   // Create blocks for unstructured regions. This has to be done since
   // blocks are initially allocated with the function as the parent region.
@@ -3342,8 +3342,8 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
   auto mlirType = converter.genType(varType.declTypeSpec->derivedTypeSpec());
-  auto declMapperOp = mlir::omp::DeclareMapperOp::create(firOpBuilder,
-      loc, mapperNameStr, mlirType);
+  auto declMapperOp = mlir::omp::DeclareMapperOp::create(
+      firOpBuilder, loc, mapperNameStr, mlirType);
   auto &region = declMapperOp.getRegion();
   firOpBuilder.createBlock(&region);
   auto varVal = region.addArgument(firOpBuilder.getRefType(mlirType), loc);
@@ -3819,9 +3819,8 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
       return;
     }
     // Generate ThreadprivateOp and rebind the common block.
-    mlir::Value commonThreadprivateValue =
-        mlir::omp::ThreadprivateOp::create(firOpBuilder,
-            currentLocation, commonValue.getType(), commonValue);
+    mlir::Value commonThreadprivateValue = mlir::omp::ThreadprivateOp::create(
+        firOpBuilder, currentLocation, commonValue.getType(), commonValue);
     converter.bindSymbol(*common, commonThreadprivateValue);
     // Generate the threadprivate value for the common block member.
     symThreadprivateValue = genCommonBlockMember(converter, currentLocation,
@@ -3841,10 +3840,10 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
       global = globalInitialization(converter, firOpBuilder, sym, var,
                                     currentLocation);
 
-    mlir::Value symValue = fir::AddrOfOp::create(firOpBuilder,
-        currentLocation, global.resultType(), global.getSymbol());
-    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(firOpBuilder,
-        currentLocation, symValue.getType(), symValue);
+    mlir::Value symValue = fir::AddrOfOp::create(
+        firOpBuilder, currentLocation, global.resultType(), global.getSymbol());
+    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(
+        firOpBuilder, currentLocation, symValue.getType(), symValue);
   } else {
     mlir::Value symValue = converter.getSymbolAddress(sym);
 
@@ -3859,8 +3858,8 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
     if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
       return;
 
-    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(firOpBuilder,
-        currentLocation, symValue.getType(), symValue);
+    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(
+        firOpBuilder, currentLocation, symValue.getType(), symValue);
   }
 
   fir::ExtendedValue sexv = converter.getSymbolExtendedValue(sym);
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 83752fa62..b1716d6af 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -129,8 +129,8 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
     if (seqType.hasDynamicExtents())
       varType = mlir::TypeAttr::get(seqType.getEleTy());
 
-  mlir::omp::MapInfoOp op = mlir::omp::MapInfoOp::create(builder,
-      loc, retTy, baseAddr, varType,
+  mlir::omp::MapInfoOp op = mlir::omp::MapInfoOp::create(
+      builder, loc, retTy, baseAddr, varType,
       builder.getIntegerAttr(builder.getIntegerType(64, false), mapType),
       builder.getAttr<mlir::omp::VariableCaptureKindAttr>(mapCaptureType),
       varPtrPtr, members, membersIndex, bounds, mapperId,
@@ -196,7 +196,7 @@ static void generateArrayIndices(lower::AbstractConverter &converter,
     subscript = firOpBuilder.createConvert(
         clauseLocation, firOpBuilder.getIndexType(), subscript);
     indices.push_back(mlir::arith::SubIOp::create(firOpBuilder, clauseLocation,
-                                                               subscript, one));
+                                                  subscript, one));
   }
 }
 
@@ -329,9 +329,10 @@ mlir::Value createParentSymAndGenIntermediateMaps(
                              subscriptIndices, objectList[i]);
         assert(!subscriptIndices.empty() &&
                "missing expected indices for map clause");
-        curValue = fir::CoordinateOp::create(firOpBuilder,
-            clauseLocation, firOpBuilder.getRefType(arrType.getEleTy()),
-            curValue, subscriptIndices);
+        curValue = fir::CoordinateOp::create(
+            firOpBuilder, clauseLocation,
+            firOpBuilder.getRefType(arrType.getEleTy()), curValue,
+            subscriptIndices);
       }
     }
 
@@ -345,9 +346,9 @@ mlir::Value createParentSymAndGenIntermediateMaps(
       fir::IntOrValue idxConst = mlir::IntegerAttr::get(
           firOpBuilder.getI32Type(), indices[currentIndicesIdx]);
       mlir::Type memberTy = recordType.getType(indices[currentIndicesIdx]);
-      curValue = fir::CoordinateOp::create(firOpBuilder,
-          clauseLocation, firOpBuilder.getRefType(memberTy), curValue,
-          llvm::SmallVector<fir::IntOrValue, 1>{idxConst});
+      curValue = fir::CoordinateOp::create(
+          firOpBuilder, clauseLocation, firOpBuilder.getRefType(memberTy),
+          curValue, llvm::SmallVector<fir::IntOrValue, 1>{idxConst});
 
       // If we're a final member, the map will be generated by the processMap
       // call that invoked this function.
diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index 4c446c324..fff060b79 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -393,7 +393,7 @@ void PopulateInitAndCleanupRegionsHelper::initBoxedPrivatePointer(
   mlir::Value null = builder.createNullConstant(loc, boxTy.getEleTy());
   mlir::Value nullBox;
   nullBox = fir::EmboxOp::create(builder, loc, boxTy, null, shape,
-                                         /*slice=*/mlir::Value{}, lenParams);
+                                 /*slice=*/mlir::Value{}, lenParams);
   fir::StoreOp::create(builder, loc, nullBox, allocatedPrivVarArg);
   createYield(allocatedPrivVarArg);
 }
@@ -413,7 +413,7 @@ fir::IfOp PopulateInitAndCleanupRegionsHelper::handleNullAllocatable() {
   mlir::Value addr = fir::BoxAddrOp::create(builder, loc, getLoadedMoldArg());
   mlir::Value isNotAllocated = builder.genIsNullAddr(loc, addr);
   fir::IfOp ifOp = fir::IfOp::create(builder, loc, isNotAllocated,
-                                             /*withElseRegion=*/true);
+                                     /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   // Just embox the null address and return.
   // We have to give the embox a shape so that the LLVM box structure has the
@@ -422,7 +422,7 @@ fir::IfOp PopulateInitAndCleanupRegionsHelper::handleNullAllocatable() {
 
   mlir::Value nullBox =
       fir::EmboxOp::create(builder, loc, valType, addr, shape,
-                                   /*slice=*/mlir::Value{}, lenParams);
+                           /*slice=*/mlir::Value{}, lenParams);
   fir::StoreOp::create(builder, loc, nullBox, allocatedPrivVarArg);
   return ifOp;
 }
@@ -442,9 +442,9 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedScalar(
                                                      /*shape=*/{}, lenParams);
   if (scalarInitValue)
     builder.createStoreWithConvert(loc, scalarInitValue, valAlloc);
-  mlir::Value box = fir::EmboxOp::create(builder,
-      loc, valType, valAlloc, /*shape=*/mlir::Value{},
-      /*slice=*/mlir::Value{}, lenParams);
+  mlir::Value box = fir::EmboxOp::create(builder, loc, valType, valAlloc,
+                                         /*shape=*/mlir::Value{},
+                                         /*slice=*/mlir::Value{}, lenParams);
   initializeIfDerivedTypeBox(
       builder, loc, box, getLoadedMoldArg(), needsInitialization,
       /*isFirstPrivate=*/kind == DeclOperationKind::FirstPrivateOrLocalInit);
@@ -483,10 +483,11 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
     fir::ShapeShiftOp shape =
         getShapeShift(builder, loc, source, cannotHaveNonDefaultLowerBounds);
     mlir::Type arrayType = source.getElementOrSequenceType();
-    mlir::Value allocatedArray = fir::AllocMemOp::create(builder,
-        loc, arrayType, /*typeparams=*/mlir::ValueRange{}, shape.getExtents());
+    mlir::Value allocatedArray = fir::AllocMemOp::create(
+        builder, loc, arrayType, /*typeparams=*/mlir::ValueRange{},
+        shape.getExtents());
     mlir::Value firClass = fir::EmboxOp::create(builder, loc, source.getType(),
-                                                        allocatedArray, shape);
+                                                allocatedArray, shape);
     initializeIfDerivedTypeBox(
         builder, loc, firClass, source, needsInitialization,
         /*isFirstprivate=*/kind == DeclOperationKind::FirstPrivateOrLocalInit);
@@ -544,12 +545,11 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
     // the box created by the declare form createTempFromMold is missing
     // lower bounds info
     box = fir::ReboxOp::create(builder, loc, boxType, temp, shapeShift,
-                                       /*shift=*/mlir::Value{});
+                               /*shift=*/mlir::Value{});
   else
-    box = fir::EmboxOp::create(builder,
-        loc, boxType, temp, shapeShift,
-        /*slice=*/mlir::Value{},
-        /*typeParams=*/llvm::ArrayRef<mlir::Value>{});
+    box = fir::EmboxOp::create(builder, loc, boxType, temp, shapeShift,
+                               /*slice=*/mlir::Value{},
+                               /*typeParams=*/llvm::ArrayRef<mlir::Value>{});
 
   if (scalarInitValue)
     hlfir::AssignOp::create(builder, loc, scalarInitValue, box);
diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index ee73ead79..80c32d066 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -260,20 +260,20 @@ ReductionProcessor::getReductionInitValue(mlir::Location loc, mlir::Type type,
                                                                initIm);
     }
     if (mlir::isa<mlir::FloatType>(type))
-      return mlir::arith::ConstantOp::create(builder,
-          loc, type,
+      return mlir::arith::ConstantOp::create(
+          builder, loc, type,
           builder.getFloatAttr(type, (double)getOperationIdentity(redId, loc)));
 
     if (mlir::isa<fir::LogicalType>(type)) {
-      mlir::Value intConst = mlir::arith::ConstantOp::create(builder,
-          loc, builder.getI1Type(),
+      mlir::Value intConst = mlir::arith::ConstantOp::create(
+          builder, loc, builder.getI1Type(),
           builder.getIntegerAttr(builder.getI1Type(),
                                  getOperationIdentity(redId, loc)));
       return builder.createConvert(loc, type, intConst);
     }
 
-    return mlir::arith::ConstantOp::create(builder,
-        loc, type,
+    return mlir::arith::ConstantOp::create(
+        builder, loc, type,
         builder.getIntegerAttr(type, getOperationIdentity(redId, loc)));
   case ReductionIdentifier::ID:
   case ReductionIdentifier::USER_DEF_OP:
@@ -325,7 +325,8 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value andiOp = mlir::arith::AndIOp::create(builder, loc, op1I1, op2I1);
+    mlir::Value andiOp =
+        mlir::arith::AndIOp::create(builder, loc, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, andiOp);
     break;
@@ -343,8 +344,8 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value cmpiOp = mlir::arith::CmpIOp::create(builder,
-        loc, mlir::arith::CmpIPredicate::eq, op1I1, op2I1);
+    mlir::Value cmpiOp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::eq, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, cmpiOp);
     break;
@@ -353,8 +354,8 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value cmpiOp = mlir::arith::CmpIOp::create(builder,
-        loc, mlir::arith::CmpIPredicate::ne, op1I1, op2I1);
+    mlir::Value cmpiOp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::ne, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, cmpiOp);
     break;
@@ -437,11 +438,11 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
   builder.setInsertionPointToStart(nest.body);
   const bool seqIsVolatile = fir::isa_volatile_type(seqTy.getEleTy());
   mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy(), seqIsVolatile);
-  auto lhsEleAddr = fir::ArrayCoorOp::create(builder,
-      loc, refTy, lhs, shapeShift, /*slice=*/mlir::Value{},
+  auto lhsEleAddr = fir::ArrayCoorOp::create(
+      builder, loc, refTy, lhs, shapeShift, /*slice=*/mlir::Value{},
       nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});
-  auto rhsEleAddr = fir::ArrayCoorOp::create(builder,
-      loc, refTy, rhs, shapeShift, /*slice=*/mlir::Value{},
+  auto rhsEleAddr = fir::ArrayCoorOp::create(
+      builder, loc, refTy, rhs, shapeShift, /*slice=*/mlir::Value{},
       nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});
   auto lhsEle = fir::LoadOp::create(builder, loc, lhsEleAddr);
   auto rhsEle = fir::LoadOp::create(builder, loc, rhsEleAddr);
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index 887732294..881401e11 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -702,7 +702,8 @@ void privatizeSymbol(
     // Boxes should be passed by reference into nested regions:
     auto oldIP = firOpBuilder.saveInsertionPoint();
     firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
-    auto alloca = fir::AllocaOp::create(firOpBuilder, symLoc, privVal.getType());
+    auto alloca =
+        fir::AllocaOp::create(firOpBuilder, symLoc, privVal.getType());
     firOpBuilder.restoreInsertionPoint(oldIP);
     fir::StoreOp::create(firOpBuilder, symLoc, privVal, alloca);
     privVal = alloca;
@@ -726,15 +727,15 @@ void privatizeSymbol(
     OpType result;
 
     if constexpr (std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) {
-      result = OpType::create(firOpBuilder,
-          symLoc, uniquePrivatizerName, allocType,
+      result = OpType::create(
+          firOpBuilder, symLoc, uniquePrivatizerName, allocType,
           emitCopyRegion ? mlir::omp::DataSharingClauseType::FirstPrivate
                          : mlir::omp::DataSharingClauseType::Private);
     } else {
-      result = OpType::create(firOpBuilder,
-          symLoc, uniquePrivatizerName, allocType,
-          emitCopyRegion ? fir::LocalitySpecifierType::LocalInit
-                         : fir::LocalitySpecifierType::Local);
+      result =
+          OpType::create(firOpBuilder, symLoc, uniquePrivatizerName, allocType,
+                         emitCopyRegion ? fir::LocalitySpecifierType::LocalInit
+                                        : fir::LocalitySpecifierType::Local);
     }
 
     fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
@@ -815,12 +816,12 @@ void privatizeSymbol(
       copyFirstPrivateSymbol(converter, symToPrivatize, &ip);
 
       if constexpr (std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) {
-        mlir::omp::YieldOp::create(firOpBuilder,
-            hsb.getAddr().getLoc(),
+        mlir::omp::YieldOp::create(
+            firOpBuilder, hsb.getAddr().getLoc(),
             symTable.shallowLookupSymbol(*symToPrivatize).getAddr());
       } else {
-        fir::YieldOp::create(firOpBuilder,
-            hsb.getAddr().getLoc(),
+        fir::YieldOp::create(
+            firOpBuilder, hsb.getAddr().getLoc(),
             symTable.shallowLookupSymbol(*symToPrivatize).getAddr());
       }
     }
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index a859d9a28..4d1d6fb28 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -121,8 +121,9 @@ private:
     if (recTy.getNumLenParams() != 0)
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-    componentPath.emplace_back(fir::FieldIndexOp::create(builder,
-        loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
+    componentPath.emplace_back(
+        fir::FieldIndexOp::create(builder, loc, fldTy, componentName, recTy,
+                                  /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
 
@@ -269,16 +270,16 @@ mlir::Value Fortran::lower::VectorSubscriptBox::loopOverElementsBase(
   for (auto [lb, ub, step] : genLoopBounds(builder, loc)) {
     LoopType loop;
     if constexpr (std::is_same_v<LoopType, fir::IterWhileOp>) {
-      loop =
-          fir::IterWhileOp::create(builder, loc, lb, ub, step, initialCondition);
+      loop = fir::IterWhileOp::create(builder, loc, lb, ub, step,
+                                      initialCondition);
       initialCondition = loop.getIterateVar();
       if (!outerLoop)
         outerLoop = loop;
       else
         fir::ResultOp::create(builder, loc, loop.getResult(0));
     } else {
-      loop =
-          fir::DoLoopOp::create(builder, loc, lb, ub, step, /*unordered=*/false);
+      loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
+                                   /*unordered=*/false);
       if (!outerLoop)
         outerLoop = loop;
     }
@@ -402,8 +403,8 @@ fir::ExtendedValue Fortran::lower::VectorSubscriptBox::getElementAt(
               mlir::Type vecEleTy = fir::unwrapSequenceType(
                   fir::unwrapPassByRefType(vecBase.getType()));
               mlir::Type refTy = builder.getRefType(vecEleTy);
-              auto vecEltRef = fir::CoordinateOp::create(builder,
-                  loc, refTy, vecBase, vecIndex);
+              auto vecEltRef = fir::CoordinateOp::create(builder, loc, refTy,
+                                                         vecBase, vecIndex);
               auto vecElt =
                   fir::LoadOp::create(builder, loc, vecEleTy, vecEltRef);
               indexes.emplace_back(builder.createConvert(loc, idxTy, vecElt));
@@ -414,8 +415,8 @@ fir::ExtendedValue Fortran::lower::VectorSubscriptBox::getElementAt(
         },
         subscript);
   mlir::Type refTy = builder.getRefType(getElementType());
-  auto elementAddr = fir::ArrayCoorOp::create(builder,
-      loc, refTy, fir::getBase(loweredBase), shape, slice, indexes,
+  auto elementAddr = fir::ArrayCoorOp::create(
+      builder, loc, refTy, fir::getBase(loweredBase), shape, slice, indexes,
       fir::getTypeParams(loweredBase));
   fir::ExtendedValue element = fir::factory::arraySectionElementToExtendedValue(
       builder, loc, loweredBase, elementAddr, slice);

``````````

</details>


https://github.com/llvm/llvm-project/pull/149912

From flang-commits at lists.llvm.org  Mon Jul 21 14:58:57 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 21 Jul 2025 14:58:57 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][NFC] update `flang` create APIs
	(13/n) (#149687) (PR #149913)
Message-ID: <llvm.org/llvm/llvm-project/pull/149913@github.com>


https://github.com/makslevental created https://github.com/llvm/llvm-project/pull/149913

See https://github.com/llvm/llvm-project/pull/147168 for more info.

>From c2805351265edb5dea16d7b770701e33a59ea0e4 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 21 Jul 2025 17:58:28 -0400
Subject: [PATCH] [mlir][NFC] update `flang` create APIs (13/n) (#149687)

See https://github.com/llvm/llvm-project/pull/147168 for more info.
---
 flang/include/flang/Lower/DirectivesCommon.h  | 36 ++++++-------
 .../flang/Lower/Support/ReductionProcessor.h  | 10 ++--
 .../include/flang/Optimizer/Builder/Complex.h |  4 +-
 .../Optimizer/Builder/DirectivesCommon.h      | 44 ++++++++--------
 .../flang/Optimizer/CodeGen/FIROpPatterns.h   |  2 +-
 .../unittests/Frontend/CodeGenActionTest.cpp  |  2 +-
 .../Optimizer/Builder/CharacterTest.cpp       |  4 +-
 .../Optimizer/Builder/ComplexTest.cpp         |  4 +-
 .../Optimizer/Builder/FIRBuilderTest.cpp      | 42 ++++++++--------
 .../Optimizer/Builder/HLFIRToolsTest.cpp      | 14 +++---
 .../Builder/Runtime/CharacterTest.cpp         | 46 ++++++++---------
 .../Optimizer/Builder/Runtime/NumericTest.cpp | 12 ++---
 .../Builder/Runtime/ReductionTest.cpp         | 50 +++++++++----------
 .../Builder/Runtime/RuntimeCallTestBase.h     |  4 +-
 .../Builder/Runtime/TransformationalTest.cpp  | 42 ++++++++--------
 15 files changed, 158 insertions(+), 158 deletions(-)

diff --git a/flang/include/flang/Lower/DirectivesCommon.h b/flang/include/flang/Lower/DirectivesCommon.h
index 93ab2e350d035..c8d5780bcd9d7 100644
--- a/flang/include/flang/Lower/DirectivesCommon.h
+++ b/flang/include/flang/Lower/DirectivesCommon.h
@@ -193,26 +193,26 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
                     mlir::Value box =
                         !fir::isBoxAddress(info.addr.getType())
                             ? info.addr
-                            : builder.create<fir::LoadOp>(loc, info.addr);
+                            : fir::LoadOp::create(builder, loc, info.addr);
                     mlir::Value d =
                         builder.createIntegerConstant(loc, idxTy, dimension);
-                    auto dimInfo = builder.create<fir::BoxDimsOp>(
+                    auto dimInfo = fir::BoxDimsOp::create(builder,
                         loc, idxTy, idxTy, idxTy, box, d);
-                    builder.create<fir::ResultOp>(loc, dimInfo.getByteStride());
+                    fir::ResultOp::create(builder, loc, dimInfo.getByteStride());
                   })
                   .genElse([&] {
                     mlir::Value zero =
                         builder.createIntegerConstant(loc, idxTy, 0);
-                    builder.create<fir::ResultOp>(loc, zero);
+                    fir::ResultOp::create(builder, loc, zero);
                   })
                   .getResults()[0];
         } else {
           mlir::Value box = !fir::isBoxAddress(info.addr.getType())
                                 ? info.addr
-                                : builder.create<fir::LoadOp>(loc, info.addr);
+                                : fir::LoadOp::create(builder, loc, info.addr);
           mlir::Value d = builder.createIntegerConstant(loc, idxTy, dimension);
           auto dimInfo =
-              builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, d);
+              fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, d);
           stride = dimInfo.getByteStride();
         }
         strideInBytes = true;
@@ -242,14 +242,14 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
             lbound = builder.createIntegerConstant(loc, idxTy, *lval - 1);
           } else {
             mlir::Value lb = builder.createIntegerConstant(loc, idxTy, *lval);
-            lbound = builder.create<mlir::arith::SubIOp>(loc, lb, baseLb);
+            lbound = mlir::arith::SubIOp::create(builder, loc, lb, baseLb);
           }
           asFortran << *lval;
         } else {
           mlir::Value lb =
               fir::getBase(converter.genExprValue(loc, *lower, stmtCtx));
           lb = builder.createConvert(loc, baseLb.getType(), lb);
-          lbound = builder.create<mlir::arith::SubIOp>(loc, lb, baseLb);
+          lbound = mlir::arith::SubIOp::create(builder, loc, lb, baseLb);
           asFortran << detail::peelOuterConvert(*lower).AsFortran();
         }
       } else {
@@ -276,14 +276,14 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
               ubound = builder.createIntegerConstant(loc, idxTy, *uval - 1);
             } else {
               mlir::Value ub = builder.createIntegerConstant(loc, idxTy, *uval);
-              ubound = builder.create<mlir::arith::SubIOp>(loc, ub, baseLb);
+              ubound = mlir::arith::SubIOp::create(builder, loc, ub, baseLb);
             }
             asFortran << *uval;
           } else {
             mlir::Value ub =
                 fir::getBase(converter.genExprValue(loc, *upper, stmtCtx));
             ub = builder.createConvert(loc, baseLb.getType(), ub);
-            ubound = builder.create<mlir::arith::SubIOp>(loc, ub, baseLb);
+            ubound = mlir::arith::SubIOp::create(builder, loc, ub, baseLb);
             asFortran << detail::peelOuterConvert(*upper).AsFortran();
           }
         }
@@ -310,12 +310,12 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
                   .genThen([&]() {
                     mlir::Value ext = fir::factory::readExtent(
                         builder, loc, dataExv, dimension);
-                    builder.create<fir::ResultOp>(loc, ext);
+                    fir::ResultOp::create(builder, loc, ext);
                   })
                   .genElse([&] {
                     mlir::Value zero =
                         builder.createIntegerConstant(loc, idxTy, 0);
-                    builder.create<fir::ResultOp>(loc, zero);
+                    fir::ResultOp::create(builder, loc, zero);
                   })
                   .getResults()[0];
         } else {
@@ -326,8 +326,8 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
           extent = zero;
           if (ubound && lbound) {
             mlir::Value diff =
-                builder.create<mlir::arith::SubIOp>(loc, ubound, lbound);
-            extent = builder.create<mlir::arith::AddIOp>(loc, diff, one);
+                mlir::arith::SubIOp::create(builder, loc, ubound, lbound);
+            extent = mlir::arith::AddIOp::create(builder, loc, diff, one);
           }
           if (!ubound)
             ubound = lbound;
@@ -335,7 +335,7 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
 
         if (!ubound) {
           // ub = extent - 1
-          ubound = builder.create<mlir::arith::SubIOp>(loc, extent, one);
+          ubound = mlir::arith::SubIOp::create(builder, loc, extent, one);
         }
       }
 
@@ -347,7 +347,7 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
             loc, cumulativeExtent, extent);
       }
 
-      mlir::Value bound = builder.create<BoundsOp>(
+      mlir::Value bound = BoundsOp::create(builder,
           loc, boundTy, lbound, ubound, extent, stride, strideInBytes, baseLb);
       bounds.push_back(bound);
       ++dimension;
@@ -461,7 +461,7 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
     asFortran << designator.AsFortran();
 
     if (semantics::IsOptional(compRef->GetLastSymbol())) {
-      info.isPresent = builder.create<fir::IsPresentOp>(
+      info.isPresent = fir::IsPresentOp::create(builder,
           operandLocation, builder.getI1Type(), info.rawInput);
     }
 
@@ -472,7 +472,7 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
             fir::isPointerType(loadOp.getType())) {
           info.boxType = info.addr.getType();
           info.addr =
-              builder.create<fir::BoxAddrOp>(operandLocation, info.addr);
+              fir::BoxAddrOp::create(builder, operandLocation, info.addr);
         }
         info.rawInput = info.addr;
       }
diff --git a/flang/include/flang/Lower/Support/ReductionProcessor.h b/flang/include/flang/Lower/Support/ReductionProcessor.h
index 72d8a0096f511..e97ebeca4ebc7 100644
--- a/flang/include/flang/Lower/Support/ReductionProcessor.h
+++ b/flang/include/flang/Lower/Support/ReductionProcessor.h
@@ -142,8 +142,8 @@ ReductionProcessor::getReductionOperation(fir::FirOpBuilder &builder,
   assert(type.isIntOrIndexOrFloat() &&
          "only integer, float and complex types are currently supported");
   if (type.isIntOrIndex())
-    return builder.create<IntegerOp>(loc, op1, op2);
-  return builder.create<FloatOp>(loc, op1, op2);
+    return IntegerOp::create(builder, loc, op1, op2);
+  return FloatOp::create(builder, loc, op1, op2);
 }
 
 template <typename FloatOp, typename IntegerOp, typename ComplexOp>
@@ -154,10 +154,10 @@ ReductionProcessor::getReductionOperation(fir::FirOpBuilder &builder,
   assert((type.isIntOrIndexOrFloat() || fir::isa_complex(type)) &&
          "only integer, float and complex types are currently supported");
   if (type.isIntOrIndex())
-    return builder.create<IntegerOp>(loc, op1, op2);
+    return IntegerOp::create(builder, loc, op1, op2);
   if (fir::isa_real(type))
-    return builder.create<FloatOp>(loc, op1, op2);
-  return builder.create<ComplexOp>(loc, op1, op2);
+    return FloatOp::create(builder, loc, op1, op2);
+  return ComplexOp::create(builder, loc, op1, op2);
 }
 
 } // namespace omp
diff --git a/flang/include/flang/Optimizer/Builder/Complex.h b/flang/include/flang/Optimizer/Builder/Complex.h
index cd0a876a4cef0..063cd4b026f5e 100644
--- a/flang/include/flang/Optimizer/Builder/Complex.h
+++ b/flang/include/flang/Optimizer/Builder/Complex.h
@@ -58,7 +58,7 @@ class Complex {
 protected:
   template <Part partId>
   mlir::Value extract(mlir::Value cplx) {
-    return builder.create<fir::ExtractValueOp>(
+    return fir::ExtractValueOp::create(builder,
         loc, getComplexPartType(cplx), cplx,
         builder.getArrayAttr({builder.getIntegerAttr(
             builder.getIndexType(), static_cast<int>(partId))}));
@@ -66,7 +66,7 @@ class Complex {
 
   template <Part partId>
   mlir::Value insert(mlir::Value cplx, mlir::Value part) {
-    return builder.create<fir::InsertValueOp>(
+    return fir::InsertValueOp::create(builder,
         loc, cplx.getType(), cplx, part,
         builder.getArrayAttr({builder.getIntegerAttr(
             builder.getIndexType(), static_cast<int>(partId))}));
diff --git a/flang/include/flang/Optimizer/Builder/DirectivesCommon.h b/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
index 78c1bb359c804..8c9b22a1342f2 100644
--- a/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
+++ b/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
@@ -71,7 +71,7 @@ inline AddrAndBoundsInfo getDataOperandBaseAddr(fir::FirOpBuilder &builder,
   mlir::Value isPresent;
   if (isOptional)
     isPresent =
-        builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), rawInput);
+        fir::IsPresentOp::create(builder, loc, builder.getI1Type(), rawInput);
 
   if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(
           fir::unwrapRefType(symAddr.getType()))) {
@@ -82,7 +82,7 @@ inline AddrAndBoundsInfo getDataOperandBaseAddr(fir::FirOpBuilder &builder,
     // if branches.
     if (unwrapFirBox && mlir::isa<fir::ReferenceType>(symAddr.getType()) &&
         !isOptional) {
-      mlir::Value addr = builder.create<fir::LoadOp>(loc, symAddr);
+      mlir::Value addr = fir::LoadOp::create(builder, loc, symAddr);
       return AddrAndBoundsInfo(addr, rawInput, isPresent, boxTy);
     }
 
@@ -94,7 +94,7 @@ inline AddrAndBoundsInfo getDataOperandBaseAddr(fir::FirOpBuilder &builder,
   if (auto boxCharType = mlir::dyn_cast<fir::BoxCharType>(
           fir::unwrapRefType((symAddr.getType())))) {
     if (!isOptional && mlir::isa<fir::ReferenceType>(symAddr.getType())) {
-      mlir::Value boxChar = builder.create<fir::LoadOp>(loc, symAddr);
+      mlir::Value boxChar = fir::LoadOp::create(builder, loc, symAddr);
       return AddrAndBoundsInfo(boxChar, rawInput, isPresent);
     }
   }
@@ -117,10 +117,10 @@ gatherBoundsOrBoundValues(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value baseLb =
         fir::factory::readLowerBound(builder, loc, dataExv, dim, one);
     auto dimInfo =
-        builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, d);
+        fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, d);
     mlir::Value lb = builder.createIntegerConstant(loc, idxTy, 0);
     mlir::Value ub =
-        builder.create<mlir::arith::SubIOp>(loc, dimInfo.getExtent(), one);
+        mlir::arith::SubIOp::create(builder, loc, dimInfo.getExtent(), one);
     if (dim == 0) // First stride is the element size.
       byteStride = dimInfo.getByteStride();
     if (collectValuesOnly) {
@@ -130,12 +130,12 @@ gatherBoundsOrBoundValues(fir::FirOpBuilder &builder, mlir::Location loc,
       values.push_back(byteStride);
       values.push_back(baseLb);
     } else {
-      mlir::Value bound = builder.create<BoundsOp>(
+      mlir::Value bound = BoundsOp::create(builder,
           loc, boundTy, lb, ub, dimInfo.getExtent(), byteStride, true, baseLb);
       values.push_back(bound);
     }
     // Compute the stride for the next dimension.
-    byteStride = builder.create<mlir::arith::MulIOp>(loc, byteStride,
+    byteStride = mlir::arith::MulIOp::create(builder, loc, byteStride,
                                                      dimInfo.getExtent());
   }
   return values;
@@ -162,20 +162,20 @@ genBoundsOpFromBoxChar(fir::FirOpBuilder &builder, mlir::Location loc,
               .genThen([&]() {
                 mlir::Value boxChar =
                     fir::isa_ref_type(info.addr.getType())
-                        ? builder.create<fir::LoadOp>(loc, info.addr)
+                        ? fir::LoadOp::create(builder, loc, info.addr)
                         : info.addr;
                 fir::BoxCharType boxCharType =
                     mlir::cast<fir::BoxCharType>(boxChar.getType());
                 mlir::Type refType = builder.getRefType(boxCharType.getEleTy());
-                auto unboxed = builder.create<fir::UnboxCharOp>(
+                auto unboxed = fir::UnboxCharOp::create(builder,
                     loc, refType, lenType, boxChar);
                 mlir::SmallVector<mlir::Value> results = {unboxed.getResult(1),
                                                           one};
-                builder.create<fir::ResultOp>(loc, results);
+                fir::ResultOp::create(builder, loc, results);
               })
               .genElse([&]() {
                 mlir::SmallVector<mlir::Value> results = {zero, zero};
-                builder.create<fir::ResultOp>(loc, results);
+                fir::ResultOp::create(builder, loc, results);
               })
               .getResults();
       return {ifRes[0], ifRes[1]};
@@ -183,19 +183,19 @@ genBoundsOpFromBoxChar(fir::FirOpBuilder &builder, mlir::Location loc,
     // We have already established that info.addr.getType() is a boxchar
     // or a boxchar address. If an address, load the boxchar.
     mlir::Value boxChar = fir::isa_ref_type(info.addr.getType())
-                              ? builder.create<fir::LoadOp>(loc, info.addr)
+                              ? fir::LoadOp::create(builder, loc, info.addr)
                               : info.addr;
     fir::BoxCharType boxCharType =
         mlir::cast<fir::BoxCharType>(boxChar.getType());
     mlir::Type refType = builder.getRefType(boxCharType.getEleTy());
     auto unboxed =
-        builder.create<fir::UnboxCharOp>(loc, refType, lenType, boxChar);
+        fir::UnboxCharOp::create(builder, loc, refType, lenType, boxChar);
     return {unboxed.getResult(1), one};
   }();
 
-  mlir::Value ub = builder.create<mlir::arith::SubIOp>(loc, extent, one);
+  mlir::Value ub = mlir::arith::SubIOp::create(builder, loc, extent, one);
   mlir::Type boundTy = builder.getType<BoundsType>();
-  return builder.create<BoundsOp>(loc, boundTy,
+  return BoundsOp::create(builder, loc, boundTy,
                                   /*lower_bound=*/zero,
                                   /*upper_bound=*/ub,
                                   /*extent=*/extent,
@@ -230,12 +230,12 @@ genBoundsOpsFromBox(fir::FirOpBuilder &builder, mlir::Location loc,
               mlir::Value box =
                   !fir::isBoxAddress(info.addr.getType())
                       ? info.addr
-                      : builder.create<fir::LoadOp>(loc, info.addr);
+                      : fir::LoadOp::create(builder, loc, info.addr);
               llvm::SmallVector<mlir::Value> boundValues =
                   gatherBoundsOrBoundValues<BoundsOp, BoundsType>(
                       builder, loc, dataExv, box,
                       /*collectValuesOnly=*/true);
-              builder.create<fir::ResultOp>(loc, boundValues);
+              fir::ResultOp::create(builder, loc, boundValues);
             })
             .genElse([&] {
               // Box is not present. Populate bound values with default values.
@@ -249,13 +249,13 @@ genBoundsOpsFromBox(fir::FirOpBuilder &builder, mlir::Location loc,
                 boundValues.push_back(zero); // byteStride
                 boundValues.push_back(zero); // baseLb
               }
-              builder.create<fir::ResultOp>(loc, boundValues);
+              fir::ResultOp::create(builder, loc, boundValues);
             })
             .getResults();
     // Create the bound operations outside the if-then-else with the if op
     // results.
     for (unsigned i = 0; i < ifRes.size(); i += nbValuesPerBound) {
-      mlir::Value bound = builder.create<BoundsOp>(
+      mlir::Value bound = BoundsOp::create(builder,
           loc, boundTy, ifRes[i], ifRes[i + 1], ifRes[i + 2], ifRes[i + 3],
           true, ifRes[i + 4]);
       bounds.push_back(bound);
@@ -263,7 +263,7 @@ genBoundsOpsFromBox(fir::FirOpBuilder &builder, mlir::Location loc,
   } else {
     mlir::Value box = !fir::isBoxAddress(info.addr.getType())
                           ? info.addr
-                          : builder.create<fir::LoadOp>(loc, info.addr);
+                          : fir::LoadOp::create(builder, loc, info.addr);
     bounds = gatherBoundsOrBoundValues<BoundsOp, BoundsType>(builder, loc,
                                                              dataExv, box);
   }
@@ -299,7 +299,7 @@ genBaseBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
       ub = lb;
     } else {
       // ub = extent - 1
-      ub = builder.create<mlir::arith::SubIOp>(loc, extent, one);
+      ub = mlir::arith::SubIOp::create(builder, loc, extent, one);
     }
     mlir::Value stride = one;
     if (strideIncludeLowerExtent) {
@@ -308,7 +308,7 @@ genBaseBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
           loc, cumulativeExtent, extent);
     }
 
-    mlir::Value bound = builder.create<BoundsOp>(loc, boundTy, lb, ub, extent,
+    mlir::Value bound = BoundsOp::create(builder, loc, boundTy, lb, ub, extent,
                                                  stride, false, baseLb);
     bounds.push_back(bound);
   }
diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
index cbdcf6b501b8e..b7fa8fc3848f2 100644
--- a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
+++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
@@ -144,7 +144,7 @@ class ConvertFIRToLLVMPattern : public mlir::ConvertToLLVMPattern {
     llvm::SmallVector<mlir::LLVM::GEPArg> cv = {args...};
     auto llvmPtrTy =
         mlir::LLVM::LLVMPointerType::get(ty.getContext(), /*addressSpace=*/0);
-    return rewriter.create<mlir::LLVM::GEPOp>(loc, llvmPtrTy, ty, base, cv);
+    return mlir::LLVM::GEPOp::create(rewriter, loc, llvmPtrTy, ty, base, cv);
   }
 
   // Find the Block in which the alloca should be inserted.
diff --git a/flang/unittests/Frontend/CodeGenActionTest.cpp b/flang/unittests/Frontend/CodeGenActionTest.cpp
index 6020abc463eda..419a76f218f28 100644
--- a/flang/unittests/Frontend/CodeGenActionTest.cpp
+++ b/flang/unittests/Frontend/CodeGenActionTest.cpp
@@ -77,7 +77,7 @@ class LLVMConversionFailureCodeGenAction : public CodeGenAction {
     mlir::OpBuilder builder(mlirCtx.get());
     builder.setInsertionPointToStart(&mlirModule->getRegion().front());
     // Create a fake op to trip conversion to LLVM.
-    builder.create<test::dummy::FakeOp>(loc);
+    test::dummy::FakeOp::create(builder, loc);
 
     llvmCtx = std::make_unique<llvm::LLVMContext>();
   }
diff --git a/flang/unittests/Optimizer/Builder/CharacterTest.cpp b/flang/unittests/Optimizer/Builder/CharacterTest.cpp
index d8d2da40ba9a6..824bcbb11c7cf 100644
--- a/flang/unittests/Optimizer/Builder/CharacterTest.cpp
+++ b/flang/unittests/Optimizer/Builder/CharacterTest.cpp
@@ -26,9 +26,9 @@ struct CharacterTest : public testing::Test {
 
     // Set up a Module with a dummy function operation inside.
     // Set the insertion point in the function entry block.
-    moduleOp = builder.create<mlir::ModuleOp>(loc);
+    moduleOp = mlir::ModuleOp::create(builder, loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
-    mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
+    mlir::func::FuncOp func = mlir::func::FuncOp::create(builder,
         loc, "func1", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
diff --git a/flang/unittests/Optimizer/Builder/ComplexTest.cpp b/flang/unittests/Optimizer/Builder/ComplexTest.cpp
index d5f00c9b61108..1794e69e915a1 100644
--- a/flang/unittests/Optimizer/Builder/ComplexTest.cpp
+++ b/flang/unittests/Optimizer/Builder/ComplexTest.cpp
@@ -22,9 +22,9 @@ struct ComplexTest : public testing::Test {
 
     // Set up a Module with a dummy function operation inside.
     // Set the insertion point in the function entry block.
-    moduleOp = builder.create<mlir::ModuleOp>(loc);
+    moduleOp = mlir::ModuleOp::create(builder, loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
-    mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
+    mlir::func::FuncOp func = mlir::func::FuncOp::create(builder,
         loc, "func1", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
diff --git a/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp b/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
index e4c21f6b65a36..1c54c6ed33962 100644
--- a/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
+++ b/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
@@ -26,9 +26,9 @@ struct FIRBuilderTest : public testing::Test {
 
     // Set up a Module with a dummy function operation inside.
     // Set the insertion point in the function entry block.
-    moduleOp = builder.create<mlir::ModuleOp>(loc);
+    moduleOp = mlir::ModuleOp::create(builder, loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
-    mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
+    mlir::func::FuncOp func = mlir::func::FuncOp::create(builder,
         loc, "func1", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
@@ -47,7 +47,7 @@ static arith::CmpIOp createCondition(fir::FirOpBuilder &builder) {
   auto loc = builder.getUnknownLoc();
   auto zero1 = builder.createIntegerConstant(loc, builder.getIndexType(), 0);
   auto zero2 = builder.createIntegerConstant(loc, builder.getIndexType(), 0);
-  return builder.create<arith::CmpIOp>(
+  return arith::CmpIOp::create(builder,
       loc, arith::CmpIPredicate::eq, zero1, zero2);
 }
 
@@ -413,7 +413,7 @@ TEST_F(FIRBuilderTest, getExtents) {
   llvm::SmallVector<mlir::Value> extents = {c10, c100};
   fir::SequenceType::Shape shape(2, fir::SequenceType::getUnknownExtent());
   auto arrayTy = fir::SequenceType::get(shape, builder.getI64Type());
-  mlir::Value array = builder.create<fir::UndefOp>(loc, arrayTy);
+  mlir::Value array = fir::UndefOp::create(builder, loc, arrayTy);
   fir::ArrayBoxValue aab(array, extents, {});
   fir::ExtendedValue ex(aab);
   auto readExtents = fir::factory::getExtents(loc, builder, ex);
@@ -468,12 +468,12 @@ TEST_F(FIRBuilderTest, getBaseTypeOf) {
     auto boxTyArray = fir::BoxType::get(arrayType);
     auto boxTyScalar = fir::BoxType::get(elementType);
 
-    auto ptrValArray = builder.create<fir::UndefOp>(loc, ptrTyArray);
-    auto ptrValScalar = builder.create<fir::UndefOp>(loc, ptrTyScalar);
-    auto boxRefValArray = builder.create<fir::UndefOp>(loc, boxRefTyArray);
-    auto boxRefValScalar = builder.create<fir::UndefOp>(loc, boxRefTyScalar);
-    auto boxValArray = builder.create<fir::UndefOp>(loc, boxTyArray);
-    auto boxValScalar = builder.create<fir::UndefOp>(loc, boxTyScalar);
+    auto ptrValArray = fir::UndefOp::create(builder, loc, ptrTyArray);
+    auto ptrValScalar = fir::UndefOp::create(builder, loc, ptrTyScalar);
+    auto boxRefValArray = fir::UndefOp::create(builder, loc, boxRefTyArray);
+    auto boxRefValScalar = fir::UndefOp::create(builder, loc, boxRefTyScalar);
+    auto boxValArray = fir::UndefOp::create(builder, loc, boxTyArray);
+    auto boxValScalar = fir::UndefOp::create(builder, loc, boxTyScalar);
 
     llvm::SmallVector<fir::ExtendedValue, 4> scalars;
     scalars.emplace_back(fir::UnboxedValue(ptrValScalar));
@@ -482,7 +482,7 @@ TEST_F(FIRBuilderTest, getBaseTypeOf) {
         fir::MutableBoxValue(boxRefValScalar, mlir::ValueRange(), {}));
 
     llvm::SmallVector<fir::ExtendedValue, 4> arrays;
-    auto extent = builder.create<fir::UndefOp>(loc, builder.getIndexType());
+    auto extent = fir::UndefOp::create(builder, loc, builder.getIndexType());
     llvm::SmallVector<mlir::Value> extents(
         mlir::dyn_cast<fir::SequenceType>(arrayType).getDimension(),
         extent.getResult());
@@ -537,10 +537,10 @@ TEST_F(FIRBuilderTest, genArithFastMath) {
   auto loc = builder.getUnknownLoc();
 
   auto realTy = mlir::Float32Type::get(ctx);
-  auto arg = builder.create<fir::UndefOp>(loc, realTy);
+  auto arg = fir::UndefOp::create(builder, loc, realTy);
 
   // Test that FastMathFlags is 'none' by default.
-  mlir::Operation *op1 = builder.create<mlir::arith::AddFOp>(loc, arg, arg);
+  mlir::Operation *op1 = mlir::arith::AddFOp::create(builder, loc, arg, arg);
   auto op1_fmi =
       mlir::dyn_cast_or_null<mlir::arith::ArithFastMathInterface>(op1);
   EXPECT_TRUE(op1_fmi);
@@ -558,7 +558,7 @@ TEST_F(FIRBuilderTest, genArithFastMath) {
   builder_copy.setFastMathFlags(FMF2);
 
   // Modifying FastMathFlags for the copy must not affect the original builder.
-  mlir::Operation *op2 = builder.create<mlir::arith::AddFOp>(loc, arg, arg);
+  mlir::Operation *op2 = mlir::arith::AddFOp::create(builder, loc, arg, arg);
   auto op2_fmi =
       mlir::dyn_cast_or_null<mlir::arith::ArithFastMathInterface>(op2);
   EXPECT_TRUE(op2_fmi);
@@ -567,7 +567,7 @@ TEST_F(FIRBuilderTest, genArithFastMath) {
 
   // Modifying FastMathFlags for the original builder must not affect the copy.
   mlir::Operation *op3 =
-      builder_copy.create<mlir::arith::AddFOp>(loc, arg, arg);
+      mlir::arith::AddFOp::create(builder_copy, loc, arg, arg);
   auto op3_fmi =
       mlir::dyn_cast_or_null<mlir::arith::ArithFastMathInterface>(op3);
   EXPECT_TRUE(op3_fmi);
@@ -578,7 +578,7 @@ TEST_F(FIRBuilderTest, genArithFastMath) {
   fir::FirOpBuilder builder_copy2(builder);
 
   mlir::Operation *op4 =
-      builder_copy2.create<mlir::arith::AddFOp>(loc, arg, arg);
+      mlir::arith::AddFOp::create(builder_copy2, loc, arg, arg);
   auto op4_fmi =
       mlir::dyn_cast_or_null<mlir::arith::ArithFastMathInterface>(op4);
   EXPECT_TRUE(op4_fmi);
@@ -592,10 +592,10 @@ TEST_F(FIRBuilderTest, genArithIntegerOverflow) {
   auto loc = builder.getUnknownLoc();
 
   auto intTy = IntegerType::get(ctx, 32);
-  auto arg = builder.create<fir::UndefOp>(loc, intTy);
+  auto arg = fir::UndefOp::create(builder, loc, intTy);
 
   // Test that IntegerOverflowFlags is 'none' by default.
-  mlir::Operation *op1 = builder.create<mlir::arith::AddIOp>(loc, arg, arg);
+  mlir::Operation *op1 = mlir::arith::AddIOp::create(builder, loc, arg, arg);
   auto op1_iofi =
       mlir::dyn_cast_or_null<mlir::arith::ArithIntegerOverflowFlagsInterface>(
           op1);
@@ -613,7 +613,7 @@ TEST_F(FIRBuilderTest, genArithIntegerOverflow) {
 
   // Modifying IntegerOverflowFlags for the copy must not affect the original
   // builder.
-  mlir::Operation *op2 = builder.create<mlir::arith::AddIOp>(loc, arg, arg);
+  mlir::Operation *op2 = mlir::arith::AddIOp::create(builder, loc, arg, arg);
   auto op2_iofi =
       mlir::dyn_cast_or_null<mlir::arith::ArithIntegerOverflowFlagsInterface>(
           op2);
@@ -624,7 +624,7 @@ TEST_F(FIRBuilderTest, genArithIntegerOverflow) {
   // Modifying IntegerOverflowFlags for the original builder must not affect the
   // copy.
   mlir::Operation *op3 =
-      builder_copy.create<mlir::arith::AddIOp>(loc, arg, arg);
+      mlir::arith::AddIOp::create(builder_copy, loc, arg, arg);
   auto op3_iofi =
       mlir::dyn_cast_or_null<mlir::arith::ArithIntegerOverflowFlagsInterface>(
           op3);
@@ -636,7 +636,7 @@ TEST_F(FIRBuilderTest, genArithIntegerOverflow) {
   fir::FirOpBuilder builder_copy2(builder);
 
   mlir::Operation *op4 =
-      builder_copy2.create<mlir::arith::AddIOp>(loc, arg, arg);
+      mlir::arith::AddIOp::create(builder_copy2, loc, arg, arg);
   auto op4_iofi =
       mlir::dyn_cast_or_null<mlir::arith::ArithIntegerOverflowFlagsInterface>(
           op4);
diff --git a/flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp b/flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp
index a0785198b078d..783b2a38266a9 100644
--- a/flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp
+++ b/flang/unittests/Optimizer/Builder/HLFIRToolsTest.cpp
@@ -25,9 +25,9 @@ struct HLFIRToolsTest : public testing::Test {
 
     // Set up a Module with a dummy function operation inside.
     // Set the insertion point in the function entry block.
-    moduleOp = builder.create<mlir::ModuleOp>(loc);
+    moduleOp = mlir::ModuleOp::create(builder, loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
-    mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
+    mlir::func::FuncOp func = mlir::func::FuncOp::create(builder,
         loc, "func1", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
@@ -61,7 +61,7 @@ TEST_F(HLFIRToolsTest, testScalarRoundTrip) {
   mlir::Location loc = getLoc();
   mlir::Type f32Type = mlir::Float32Type::get(&context);
   mlir::Type scalarf32Type = builder.getRefType(f32Type);
-  mlir::Value scalarf32Addr = builder.create<fir::UndefOp>(loc, scalarf32Type);
+  mlir::Value scalarf32Addr = fir::UndefOp::create(builder, loc, scalarf32Type);
   fir::ExtendedValue scalarf32{scalarf32Addr};
   hlfir::EntityWithAttributes scalarf32Entity(createDeclare(scalarf32));
   auto [scalarf32Result, cleanup] =
@@ -85,7 +85,7 @@ TEST_F(HLFIRToolsTest, testArrayRoundTrip) {
   mlir::Type f32Type = mlir::Float32Type::get(&context);
   mlir::Type seqf32Type = builder.getVarLenSeqTy(f32Type, 2);
   mlir::Type arrayf32Type = builder.getRefType(seqf32Type);
-  mlir::Value arrayf32Addr = builder.create<fir::UndefOp>(loc, arrayf32Type);
+  mlir::Value arrayf32Addr = fir::UndefOp::create(builder, loc, arrayf32Type);
   fir::ArrayBoxValue arrayf32{arrayf32Addr, extents, lbounds};
   hlfir::EntityWithAttributes arrayf32Entity(createDeclare(arrayf32));
   auto [arrayf32Result, cleanup] =
@@ -113,7 +113,7 @@ TEST_F(HLFIRToolsTest, testScalarCharRoundTrip) {
   mlir::Type charType = fir::CharacterType::getUnknownLen(&context, 1);
   mlir::Type scalarCharType = builder.getRefType(charType);
   mlir::Value scalarCharAddr =
-      builder.create<fir::UndefOp>(loc, scalarCharType);
+      fir::UndefOp::create(builder, loc, scalarCharType);
   fir::CharBoxValue scalarChar{scalarCharAddr, len};
   hlfir::EntityWithAttributes scalarCharEntity(createDeclare(scalarChar));
   auto [scalarCharResult, cleanup] =
@@ -138,7 +138,7 @@ TEST_F(HLFIRToolsTest, testArrayCharRoundTrip) {
   mlir::Type charType = fir::CharacterType::getUnknownLen(&context, 1);
   mlir::Type seqCharType = builder.getVarLenSeqTy(charType, 2);
   mlir::Type arrayCharType = builder.getRefType(seqCharType);
-  mlir::Value arrayCharAddr = builder.create<fir::UndefOp>(loc, arrayCharType);
+  mlir::Value arrayCharAddr = fir::UndefOp::create(builder, loc, arrayCharType);
   fir::CharArrayBoxValue arrayChar{arrayCharAddr, len, extents, lbounds};
   hlfir::EntityWithAttributes arrayCharEntity(createDeclare(arrayChar));
   auto [arrayCharResult, cleanup] =
@@ -170,7 +170,7 @@ TEST_F(HLFIRToolsTest, testArrayCharBoxRoundTrip) {
   mlir::Type seqCharType = builder.getVarLenSeqTy(charType, 2);
   mlir::Type arrayCharBoxType = fir::BoxType::get(seqCharType);
   mlir::Value arrayCharAddr =
-      builder.create<fir::UndefOp>(loc, arrayCharBoxType);
+      fir::UndefOp::create(builder, loc, arrayCharBoxType);
   llvm::SmallVector<mlir::Value> explicitTypeParams{len};
   fir::BoxValue arrayChar{arrayCharAddr, lbounds, explicitTypeParams};
   hlfir::EntityWithAttributes arrayCharEntity(createDeclare(arrayChar));
diff --git a/flang/unittests/Optimizer/Builder/Runtime/CharacterTest.cpp b/flang/unittests/Optimizer/Builder/Runtime/CharacterTest.cpp
index 315f8c80dc33f..cfd2d489ac578 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/CharacterTest.cpp
+++ b/flang/unittests/Optimizer/Builder/Runtime/CharacterTest.cpp
@@ -33,10 +33,10 @@ void checkCharCompare1(
     fir::FirOpBuilder &builder, mlir::Type type, llvm::StringRef fctName) {
   auto loc = builder.getUnknownLoc();
   mlir::Type i32Ty = IntegerType::get(builder.getContext(), 32);
-  mlir::Value lhsBuff = builder.create<fir::UndefOp>(loc, type);
-  mlir::Value lhsLen = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value rhsBuff = builder.create<fir::UndefOp>(loc, type);
-  mlir::Value rhsLen = builder.create<fir::UndefOp>(loc, i32Ty);
+  mlir::Value lhsBuff = fir::UndefOp::create(builder, loc, type);
+  mlir::Value lhsLen = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value rhsBuff = fir::UndefOp::create(builder, loc, type);
+  mlir::Value rhsLen = fir::UndefOp::create(builder, loc, i32Ty);
   mlir::Value res = fir::runtime::genCharCompare(builder, loc,
       mlir::arith::CmpIPredicate::eq, lhsBuff, lhsLen, rhsBuff, rhsLen);
   checkCallOpFromResultBox(lhsBuff, fctName, 4, /*addLocArgs=*/false);
@@ -72,10 +72,10 @@ void checkCharCompare2(
   fir::factory::CharacterExprHelper charHelper(builder, loc);
   mlir::Type i32Ty = IntegerType::get(builder.getContext(), 32);
   mlir::Type boxCharTy = fir::BoxCharType::get(builder.getContext(), kind);
-  mlir::Value lhsBuff = builder.create<fir::UndefOp>(loc, boxCharTy);
-  mlir::Value lhsLen = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value rhsBuff = builder.create<fir::UndefOp>(loc, boxCharTy);
-  mlir::Value rhsLen = builder.create<fir::UndefOp>(loc, i32Ty);
+  mlir::Value lhsBuff = fir::UndefOp::create(builder, loc, boxCharTy);
+  mlir::Value lhsLen = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value rhsBuff = fir::UndefOp::create(builder, loc, boxCharTy);
+  mlir::Value rhsLen = fir::UndefOp::create(builder, loc, i32Ty);
   fir::ExtendedValue lhs = charHelper.toExtendedValue(lhsBuff, lhsLen);
   fir::ExtendedValue rhs = charHelper.toExtendedValue(rhsBuff, rhsLen);
   mlir::Value res = fir::runtime::genCharCompare(
@@ -97,11 +97,11 @@ void checkGenIndex(
     fir::FirOpBuilder &builder, llvm::StringRef fctName, unsigned kind) {
   auto loc = builder.getUnknownLoc();
   mlir::Type i32Ty = IntegerType::get(builder.getContext(), 32);
-  mlir::Value stringBase = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value stringLen = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value substringBase = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value substringLen = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value back = builder.create<fir::UndefOp>(loc, i32Ty);
+  mlir::Value stringBase = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value stringLen = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value substringBase = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value substringLen = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value back = fir::UndefOp::create(builder, loc, i32Ty);
   mlir::Value res = fir::runtime::genIndex(builder, loc, kind, stringBase,
       stringLen, substringBase, substringLen, back);
   checkCallOp(res.getDefiningOp(), fctName, 5, /*addLocArgs=*/false);
@@ -160,11 +160,11 @@ void checkGenScan(
   mlir::Type charTy = fir::CharacterType::get(builder.getContext(), kind, 10);
   mlir::Type boxTy = fir::BoxType::get(charTy);
   mlir::Type i32Ty = IntegerType::get(builder.getContext(), 32);
-  mlir::Value stringBase = builder.create<fir::UndefOp>(loc, boxTy);
-  mlir::Value stringLen = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value setBase = builder.create<fir::UndefOp>(loc, boxTy);
-  mlir::Value setLen = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value back = builder.create<fir::UndefOp>(loc, i32Ty);
+  mlir::Value stringBase = fir::UndefOp::create(builder, loc, boxTy);
+  mlir::Value stringLen = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value setBase = fir::UndefOp::create(builder, loc, boxTy);
+  mlir::Value setLen = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value back = fir::UndefOp::create(builder, loc, i32Ty);
   mlir::Value res = fir::runtime::genScan(
       builder, loc, kind, stringBase, stringLen, setBase, setLen, back);
   checkCallOp(res.getDefiningOp(), fctName, 5, /*addLocArgs=*/false);
@@ -194,11 +194,11 @@ void checkGenVerify(
   mlir::Type charTy = fir::CharacterType::get(builder.getContext(), kind, 10);
   mlir::Type boxTy = fir::BoxType::get(charTy);
   mlir::Type i32Ty = IntegerType::get(builder.getContext(), 32);
-  mlir::Value stringBase = builder.create<fir::UndefOp>(loc, boxTy);
-  mlir::Value stringLen = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value setBase = builder.create<fir::UndefOp>(loc, boxTy);
-  mlir::Value setLen = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value back = builder.create<fir::UndefOp>(loc, i32Ty);
+  mlir::Value stringBase = fir::UndefOp::create(builder, loc, boxTy);
+  mlir::Value stringLen = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value setBase = fir::UndefOp::create(builder, loc, boxTy);
+  mlir::Value setLen = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value back = fir::UndefOp::create(builder, loc, i32Ty);
   mlir::Value res = fir::runtime::genVerify(
       builder, loc, kind, stringBase, stringLen, setBase, setLen, back);
   checkCallOp(res.getDefiningOp(), fctName, 5, /*addLocArgs=*/false);
diff --git a/flang/unittests/Optimizer/Builder/Runtime/NumericTest.cpp b/flang/unittests/Optimizer/Builder/Runtime/NumericTest.cpp
index 47342da07f060..0e867e98a5eef 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/NumericTest.cpp
+++ b/flang/unittests/Optimizer/Builder/Runtime/NumericTest.cpp
@@ -15,7 +15,7 @@ using namespace mlir;
 void testGenExponent(fir::FirOpBuilder &builder, mlir::Type resultType,
     mlir::Type xType, llvm::StringRef fctName) {
   auto loc = builder.getUnknownLoc();
-  mlir::Value x = builder.create<fir::UndefOp>(loc, xType);
+  mlir::Value x = fir::UndefOp::create(builder, loc, xType);
   mlir::Value exp = fir::runtime::genExponent(builder, loc, resultType, x);
   checkCallOp(exp.getDefiningOp(), fctName, 1, /*addLocArg=*/false);
 }
@@ -35,7 +35,7 @@ void testGenX(fir::FirOpBuilder &builder, mlir::Type xType,
     mlir::Value (*genFct)(fir::FirOpBuilder &, Location, mlir::Value),
     llvm::StringRef fctName) {
   auto loc = builder.getUnknownLoc();
-  mlir::Value x = builder.create<fir::UndefOp>(loc, xType);
+  mlir::Value x = fir::UndefOp::create(builder, loc, xType);
   mlir::Value val = genFct(builder, loc, x);
   checkCallOp(val.getDefiningOp(), fctName, 1, /*addLocArg=*/false);
 }
@@ -52,8 +52,8 @@ TEST_F(RuntimeCallTest, genFractionTest) {
 void testGenNearest(fir::FirOpBuilder &builder, mlir::Type xType,
     mlir::Type sType, llvm::StringRef fctName) {
   auto loc = builder.getUnknownLoc();
-  mlir::Value x = builder.create<fir::UndefOp>(loc, xType);
-  mlir::Value s = builder.create<fir::UndefOp>(loc, sType);
+  mlir::Value x = fir::UndefOp::create(builder, loc, xType);
+  mlir::Value s = fir::UndefOp::create(builder, loc, sType);
   mlir::Value nearest = fir::runtime::genNearest(builder, loc, x, s);
   checkCallOp(nearest.getDefiningOp(), fctName, 2, /*addLocArg=*/false);
 }
@@ -81,8 +81,8 @@ void testGenXI(fir::FirOpBuilder &builder, mlir::Type xType, mlir::Type iType,
         fir::FirOpBuilder &, Location, mlir::Value, mlir::Value),
     llvm::StringRef fctName) {
   auto loc = builder.getUnknownLoc();
-  mlir::Value x = builder.create<fir::UndefOp>(loc, xType);
-  mlir::Value i = builder.create<fir::UndefOp>(loc, iType);
+  mlir::Value x = fir::UndefOp::create(builder, loc, xType);
+  mlir::Value i = fir::UndefOp::create(builder, loc, iType);
   mlir::Value val = genFct(builder, loc, x, i);
   checkCallOp(val.getDefiningOp(), fctName, 2, /*addLocArg=*/false);
 }
diff --git a/flang/unittests/Optimizer/Builder/Runtime/ReductionTest.cpp b/flang/unittests/Optimizer/Builder/Runtime/ReductionTest.cpp
index a4006c6b59c12..91e2d0b41cd47 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/ReductionTest.cpp
+++ b/flang/unittests/Optimizer/Builder/Runtime/ReductionTest.cpp
@@ -68,8 +68,8 @@ void testGenMaxVal(
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), eleTy);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value undef = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value mask = builder.create<fir::UndefOp>(loc, seqTy);
+  mlir::Value undef = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value mask = fir::UndefOp::create(builder, loc, seqTy);
   mlir::Value max = fir::runtime::genMaxval(builder, loc, undef, mask);
   checkCallOp(max.getDefiningOp(), fctName, 3);
 }
@@ -93,8 +93,8 @@ void testGenMinVal(
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), eleTy);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value undef = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value mask = builder.create<fir::UndefOp>(loc, seqTy);
+  mlir::Value undef = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value mask = fir::UndefOp::create(builder, loc, seqTy);
   mlir::Value min = fir::runtime::genMinval(builder, loc, undef, mask);
   checkCallOp(min.getDefiningOp(), fctName, 3);
 }
@@ -135,9 +135,9 @@ void testGenSum(
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), eleTy);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value undef = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value mask = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
+  mlir::Value undef = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value mask = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
   mlir::Value sum = fir::runtime::genSum(builder, loc, undef, mask, result);
   if (fir::isa_complex(eleTy))
     checkCallOpFromResultBox(result, fctName, 4);
@@ -167,9 +167,9 @@ void testGenProduct(
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), eleTy);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value undef = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value mask = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
+  mlir::Value undef = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value mask = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
   mlir::Value prod =
       fir::runtime::genProduct(builder, loc, undef, mask, result);
   if (fir::isa_complex(eleTy))
@@ -200,10 +200,10 @@ void testGenDotProduct(
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), eleTy);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value a = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value b = builder.create<fir::UndefOp>(loc, refSeqTy);
+  mlir::Value a = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value b = fir::UndefOp::create(builder, loc, refSeqTy);
   mlir::Value result =
-      builder.create<fir::UndefOp>(loc, fir::ReferenceType::get(eleTy));
+      fir::UndefOp::create(builder, loc, fir::ReferenceType::get(eleTy));
   mlir::Value prod = fir::runtime::genDotProduct(builder, loc, a, b, result);
   if (fir::isa_complex(eleTy))
     checkCallOpFromResultBox(result, fctName, 3);
@@ -236,9 +236,9 @@ void checkGenMxxloc(fir::FirOpBuilder &builder, mlir::Type eleTy,
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), eleTy);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value a = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value mask = builder.create<fir::UndefOp>(loc, seqTy);
+  mlir::Value a = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value mask = fir::UndefOp::create(builder, loc, seqTy);
   mlir::Value kind = builder.createIntegerConstant(loc, i32Ty, 1);
   mlir::Value back = builder.createIntegerConstant(loc, i32Ty, 1);
   genFct(builder, loc, result, a, mask, kind, back);
@@ -308,9 +308,9 @@ void checkGenMxxlocDim(fir::FirOpBuilder &builder,
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), i32Ty);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value a = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value mask = builder.create<fir::UndefOp>(loc, seqTy);
+  mlir::Value a = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value mask = fir::UndefOp::create(builder, loc, seqTy);
   mlir::Value kind = builder.createIntegerConstant(loc, i32Ty, 1);
   mlir::Value dim = builder.createIntegerConstant(loc, i32Ty, 1);
   mlir::Value back = builder.createIntegerConstant(loc, i32Ty, 1);
@@ -337,9 +337,9 @@ void checkGenMxxvalChar(fir::FirOpBuilder &builder,
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), i32Ty);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value a = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value mask = builder.create<fir::UndefOp>(loc, seqTy);
+  mlir::Value a = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value mask = fir::UndefOp::create(builder, loc, seqTy);
   genFct(builder, loc, result, a, mask);
   checkCallOpFromResultBox(result, fctName, nbArgs);
 }
@@ -363,9 +363,9 @@ void checkGen4argsDim(fir::FirOpBuilder &builder,
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), i32Ty);
   mlir::Type refSeqTy = fir::ReferenceType::get(seqTy);
-  mlir::Value a = builder.create<fir::UndefOp>(loc, refSeqTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value mask = builder.create<fir::UndefOp>(loc, seqTy);
+  mlir::Value a = fir::UndefOp::create(builder, loc, refSeqTy);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value mask = fir::UndefOp::create(builder, loc, seqTy);
   mlir::Value dim = builder.createIntegerConstant(loc, i32Ty, 1);
   genFct(builder, loc, result, a, dim, mask);
   checkCallOpFromResultBox(result, fctName, nbArgs);
diff --git a/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h b/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
index 4ecec92f42dc2..37f97df0352f7 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
+++ b/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
@@ -24,9 +24,9 @@ struct RuntimeCallTest : public testing::Test {
 
     // Set up a Module with a dummy function operation inside.
     // Set the insertion point in the function entry block.
-    moduleOp = builder.create<mlir::ModuleOp>(loc);
+    moduleOp = mlir::ModuleOp::create(builder, loc);
     builder.setInsertionPointToStart(moduleOp->getBody());
-    mlir::func::FuncOp func = builder.create<mlir::func::FuncOp>(
+    mlir::func::FuncOp func = mlir::func::FuncOp::create(builder,
         loc, "runtime_unit_tests_func", builder.getFunctionType({}, {}));
     auto *entryBlock = func.addEntryBlock();
     builder.setInsertionPointToStart(entryBlock);
diff --git a/flang/unittests/Optimizer/Builder/Runtime/TransformationalTest.cpp b/flang/unittests/Optimizer/Builder/Runtime/TransformationalTest.cpp
index 28266bb90400b..b8e89556f1fc8 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/TransformationalTest.cpp
+++ b/flang/unittests/Optimizer/Builder/Runtime/TransformationalTest.cpp
@@ -16,12 +16,12 @@ void testGenBesselJn(
   mlir::Type i32Ty = builder.getIntegerType(32);
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), realTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value n1 = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value n2 = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value x = builder.create<fir::UndefOp>(loc, realTy);
-  mlir::Value bn1 = builder.create<fir::UndefOp>(loc, realTy);
-  mlir::Value bn2 = builder.create<fir::UndefOp>(loc, realTy);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value n1 = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value n2 = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value x = fir::UndefOp::create(builder, loc, realTy);
+  mlir::Value bn1 = fir::UndefOp::create(builder, loc, realTy);
+  mlir::Value bn2 = fir::UndefOp::create(builder, loc, realTy);
   fir::runtime::genBesselJn(builder, loc, result, n1, n2, x, bn1, bn2);
   checkCallOpFromResultBox(result, fctName, 6);
 }
@@ -39,9 +39,9 @@ void testGenBesselJnX0(
   mlir::Type i32Ty = builder.getIntegerType(32);
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), realTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value n1 = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value n2 = builder.create<fir::UndefOp>(loc, i32Ty);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value n1 = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value n2 = fir::UndefOp::create(builder, loc, i32Ty);
   fir::runtime::genBesselJnX0(builder, loc, realTy, result, n1, n2);
   checkCallOpFromResultBox(result, fctName, 3);
 }
@@ -59,12 +59,12 @@ void testGenBesselYn(
   mlir::Type i32Ty = builder.getIntegerType(32);
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), realTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value n1 = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value n2 = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value x = builder.create<fir::UndefOp>(loc, realTy);
-  mlir::Value bn1 = builder.create<fir::UndefOp>(loc, realTy);
-  mlir::Value bn2 = builder.create<fir::UndefOp>(loc, realTy);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value n1 = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value n2 = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value x = fir::UndefOp::create(builder, loc, realTy);
+  mlir::Value bn1 = fir::UndefOp::create(builder, loc, realTy);
+  mlir::Value bn2 = fir::UndefOp::create(builder, loc, realTy);
   fir::runtime::genBesselYn(builder, loc, result, n1, n2, x, bn1, bn2);
   checkCallOpFromResultBox(result, fctName, 6);
 }
@@ -82,9 +82,9 @@ void testGenBesselYnX0(
   mlir::Type i32Ty = builder.getIntegerType(32);
   mlir::Type seqTy =
       fir::SequenceType::get(fir::SequenceType::Shape(1, 10), realTy);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, seqTy);
-  mlir::Value n1 = builder.create<fir::UndefOp>(loc, i32Ty);
-  mlir::Value n2 = builder.create<fir::UndefOp>(loc, i32Ty);
+  mlir::Value result = fir::UndefOp::create(builder, loc, seqTy);
+  mlir::Value n1 = fir::UndefOp::create(builder, loc, i32Ty);
+  mlir::Value n2 = fir::UndefOp::create(builder, loc, i32Ty);
   fir::runtime::genBesselYnX0(builder, loc, realTy, result, n1, n2);
   checkCallOpFromResultBox(result, fctName, 3);
 }
@@ -155,9 +155,9 @@ void testGenMatmul(fir::FirOpBuilder &builder, mlir::Type eleTy1,
       fir::SequenceType::get(fir::SequenceType::Shape(2, 10), eleTy2);
   mlir::Type boxTy1 = fir::BoxType::get(seqTy1);
   mlir::Type boxTy2 = fir::BoxType::get(seqTy2);
-  mlir::Value result = builder.create<fir::UndefOp>(loc, resultTy);
-  mlir::Value matrixA = builder.create<fir::UndefOp>(loc, boxTy1);
-  mlir::Value matrixB = builder.create<fir::UndefOp>(loc, boxTy2);
+  mlir::Value result = fir::UndefOp::create(builder, loc, resultTy);
+  mlir::Value matrixA = fir::UndefOp::create(builder, loc, boxTy1);
+  mlir::Value matrixB = fir::UndefOp::create(builder, loc, boxTy2);
   fir::runtime::genMatmul(builder, loc, result, matrixA, matrixB);
   checkCallOpFromResultBox(result, funcName, 3);
 }


From flang-commits at lists.llvm.org  Mon Jul 21 14:59:03 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 21 Jul 2025 14:59:03 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][NFC] update `flang/lib` create APIs
 (12/n) (#149687) (PR #149914)
Message-ID: <llvm.org/llvm/llvm-project/pull/149914@github.com>


https://github.com/makslevental created https://github.com/llvm/llvm-project/pull/149914

See https://github.com/llvm/llvm-project/pull/147168 for more info.

>From 1d837af11ba902e5b3b6f4700cf6d35084a02e96 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 21 Jul 2025 17:57:58 -0400
Subject: [PATCH] [mlir][NFC] update `flang/lib` create APIs (12/n) (#149687)

See https://github.com/llvm/llvm-project/pull/147168 for more info.
---
 flang/lib/Frontend/FrontendActions.cpp        |   2 +-
 .../lib/Optimizer/CodeGen/BoxedProcedure.cpp  |   8 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 332 +++++++++---------
 flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp |  32 +-
 .../Optimizer/CodeGen/LowerRepackArrays.cpp   |  32 +-
 flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp  |  12 +-
 flang/lib/Optimizer/Dialect/FIRDialect.cpp    |   2 +-
 flang/lib/Optimizer/Dialect/FIROps.cpp        |  16 +-
 flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp |   4 +-
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp     |   6 +-
 .../HLFIR/Transforms/BufferizeHLFIR.cpp       |  38 +-
 .../HLFIR/Transforms/ConvertToFIR.cpp         |  70 ++--
 .../HLFIR/Transforms/InlineHLFIRAssign.cpp    |   2 +-
 .../HLFIR/Transforms/InlineHLFIRCopyIn.cpp    |  18 +-
 .../HLFIR/Transforms/LowerHLFIRIntrinsics.cpp |   2 +-
 .../LowerHLFIROrderedAssignments.cpp          |  16 +-
 .../Transforms/OptimizedBufferization.cpp     |  12 +-
 .../Transforms/SimplifyHLFIRIntrinsics.cpp    | 166 ++++-----
 .../Support/FIROpenACCTypeInterfaces.cpp      |  42 +--
 .../OpenMP/DoConcurrentConversion.cpp         |  16 +-
 .../Optimizer/OpenMP/FunctionFiltering.cpp    |   2 +-
 .../OpenMP/GenericLoopConversion.cpp          |  10 +-
 flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp |  26 +-
 .../Optimizer/OpenMP/MapInfoFinalization.cpp  |  24 +-
 .../OpenMP/MapsForPrivatizedSymbols.cpp       |   6 +-
 25 files changed, 448 insertions(+), 448 deletions(-)

diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index b5f4f9421f633..99cc17fcfacc5 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -164,7 +164,7 @@ static void addDependentLibs(mlir::ModuleOp mlirModule, CompilerInstance &ci) {
   // Add linker options specified by --dependent-lib
   auto builder = mlir::OpBuilder(mlirModule.getRegion());
   for (const std::string &lib : libs) {
-    builder.create<mlir::LLVM::LinkerOptionsOp>(
+    mlir::LLVM::LinkerOptionsOp::create(builder,
         mlirModule.getLoc(), builder.getStrArrayAttr({"/DEFAULTLIB:" + lib}));
   }
 }
diff --git a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
index 69bdb48146a54..9750f0750a4f7 100644
--- a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
+++ b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
@@ -180,7 +180,7 @@ class BoxprocTypeRewriter : public mlir::TypeConverter {
                                           mlir::ValueRange inputs,
                                           mlir::Location loc) {
     assert(inputs.size() == 1);
-    return builder.create<ConvertOp>(loc, unwrapRefType(type.getEleTy()),
+    return ConvertOp::create(builder, loc, unwrapRefType(type.getEleTy()),
                                      inputs[0]);
   }
 
@@ -282,16 +282,16 @@ class BoxedProcedurePass
             // 32 bytes.
             fir::SequenceType::Extent thunkSize = triple.getTrampolineSize();
             mlir::Type buffTy = SequenceType::get({thunkSize}, i8Ty);
-            auto buffer = builder.create<AllocaOp>(loc, buffTy);
+            auto buffer = AllocaOp::create(builder, loc, buffTy);
             mlir::Value closure =
                 builder.createConvert(loc, i8Ptr, embox.getHost());
             mlir::Value tramp = builder.createConvert(loc, i8Ptr, buffer);
             mlir::Value func =
                 builder.createConvert(loc, i8Ptr, embox.getFunc());
-            builder.create<fir::CallOp>(
+            fir::CallOp::create(builder,
                 loc, factory::getLlvmInitTrampoline(builder),
                 llvm::ArrayRef<mlir::Value>{tramp, func, closure});
-            auto adjustCall = builder.create<fir::CallOp>(
+            auto adjustCall = fir::CallOp::create(builder,
                 loc, factory::getLlvmAdjustTrampoline(builder),
                 llvm::ArrayRef<mlir::Value>{tramp});
             rewriter.replaceOpWithNewOp<ConvertOp>(embox, toTy,
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 609ba27bc212b..1d80c5b44798f 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -92,7 +92,7 @@ genConstantIndex(mlir::Location loc, mlir::Type ity,
                  mlir::ConversionPatternRewriter &rewriter,
                  std::int64_t offset) {
   auto cattr = rewriter.getI64IntegerAttr(offset);
-  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+  return mlir::LLVM::ConstantOp::create(rewriter, loc, ity, cattr);
 }
 
 static mlir::Block *createBlock(mlir::ConversionPatternRewriter &rewriter,
@@ -148,27 +148,27 @@ mlir::Value replaceWithAddrOfOrASCast(mlir::ConversionPatternRewriter &rewriter,
                                       mlir::Operation *replaceOp = nullptr) {
   if (mlir::isa<mlir::LLVM::LLVMPointerType>(type)) {
     if (globalAS != programAS) {
-      auto llvmAddrOp = rewriter.create<mlir::LLVM::AddressOfOp>(
+      auto llvmAddrOp = mlir::LLVM::AddressOfOp::create(rewriter,
           loc, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
       if (replaceOp)
         return rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(
             replaceOp, ::getLlvmPtrType(rewriter.getContext(), programAS),
             llvmAddrOp);
-      return rewriter.create<mlir::LLVM::AddrSpaceCastOp>(
+      return mlir::LLVM::AddrSpaceCastOp::create(rewriter,
           loc, getLlvmPtrType(rewriter.getContext(), programAS), llvmAddrOp);
     }
 
     if (replaceOp)
       return rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
           replaceOp, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
-    return rewriter.create<mlir::LLVM::AddressOfOp>(
+    return mlir::LLVM::AddressOfOp::create(rewriter,
         loc, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
   }
 
   if (replaceOp)
     return rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(replaceOp, type,
                                                                 symName);
-  return rewriter.create<mlir::LLVM::AddressOfOp>(loc, type, symName);
+  return mlir::LLVM::AddressOfOp::create(rewriter, loc, type, symName);
 }
 
 /// Lower `fir.address_of` operation to `llvm.address_of` operation.
@@ -250,7 +250,7 @@ struct DeclareOpConversion : public fir::FIROpConversion<fir::cg::XDeclareOp> {
       if (auto varAttr =
               mlir::dyn_cast_or_null<mlir::LLVM::DILocalVariableAttr>(
                   fusedLoc.getMetadata())) {
-        rewriter.create<mlir::LLVM::DbgDeclareOp>(memRef.getLoc(), memRef,
+        mlir::LLVM::DbgDeclareOp::create(rewriter, memRef.getLoc(), memRef,
                                                   varAttr, nullptr);
       }
     }
@@ -294,7 +294,7 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
           emitError(loc, "did not find allocation function");
         mlir::NamedAttribute attr = rewriter.getNamedAttr(
             "callee", mlir::SymbolRefAttr::get(memSizeFn));
-        auto call = rewriter.create<mlir::LLVM::CallOp>(
+        auto call = mlir::LLVM::CallOp::create(rewriter,
             loc, ity, lenParams,
             addLLVMOpBundleAttrs(rewriter, {attr}, lenParams.size()));
         size = call.getResult();
@@ -339,7 +339,7 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
     // pointers! Only propagate pinned and bindc_name to help debugging, but
     // this should have no functional purpose (and passing the operand segment
     // attribute like before is certainly bad).
-    auto llvmAlloc = rewriter.create<mlir::LLVM::AllocaOp>(
+    auto llvmAlloc = mlir::LLVM::AllocaOp::create(rewriter,
         loc, ::getLlvmPtrType(alloc.getContext(), allocaAs), llvmObjectType,
         size);
     if (alloc.getPinned())
@@ -401,7 +401,7 @@ struct BoxCharLenOpConversion : public fir::FIROpConversion<fir::BoxCharLenOp> {
     mlir::Type returnValTy = boxCharLen.getResult().getType();
 
     constexpr int boxcharLenIdx = 1;
-    auto len = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, boxChar,
+    auto len = mlir::LLVM::ExtractValueOp::create(rewriter, loc, boxChar,
                                                            boxcharLenIdx);
     mlir::Value lenAfterCast = integerCast(loc, rewriter, returnValTy, len);
     rewriter.replaceOp(boxCharLen, lenAfterCast);
@@ -597,9 +597,9 @@ struct StringLitOpConversion : public fir::FIROpConversion<fir::StringLitOp> {
     unsigned bits = lowerTy().characterBitsize(charTy);
     mlir::Type intTy = rewriter.getIntegerType(bits);
     mlir::Location loc = constop.getLoc();
-    mlir::Value cst = rewriter.create<mlir::LLVM::UndefOp>(loc, ty);
+    mlir::Value cst = mlir::LLVM::UndefOp::create(rewriter, loc, ty);
     if (auto arr = mlir::dyn_cast<mlir::DenseElementsAttr>(attr)) {
-      cst = rewriter.create<mlir::LLVM::ConstantOp>(loc, ty, arr);
+      cst = mlir::LLVM::ConstantOp::create(rewriter, loc, ty, arr);
     } else if (auto arr = mlir::dyn_cast<mlir::ArrayAttr>(attr)) {
       for (auto a : llvm::enumerate(arr.getValue())) {
         // convert each character to a precise bitsize
@@ -608,8 +608,8 @@ struct StringLitOpConversion : public fir::FIROpConversion<fir::StringLitOp> {
             mlir::cast<mlir::IntegerAttr>(a.value()).getValue().zextOrTrunc(
                 bits));
         auto elemCst =
-            rewriter.create<mlir::LLVM::ConstantOp>(loc, intTy, elemAttr);
-        cst = rewriter.create<mlir::LLVM::InsertValueOp>(loc, cst, elemCst,
+            mlir::LLVM::ConstantOp::create(rewriter, loc, intTy, elemAttr);
+        cst = mlir::LLVM::InsertValueOp::create(rewriter, loc, cst, elemCst,
                                                          a.index());
       }
     } else {
@@ -706,14 +706,14 @@ struct CmpcOpConversion : public fir::FIROpConversion<fir::CmpcOp> {
         mlir::arith::convertArithFastMathFlagsToLLVM(cmp.getFastmath());
     mlir::LLVM::FCmpPredicate pred =
         static_cast<mlir::LLVM::FCmpPredicate>(cmp.getPredicate());
-    auto rcp = rewriter.create<mlir::LLVM::FCmpOp>(
+    auto rcp = mlir::LLVM::FCmpOp::create(rewriter,
         loc, resTy, pred,
-        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, operands[0], 0),
-        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, operands[1], 0), fmf);
-    auto icp = rewriter.create<mlir::LLVM::FCmpOp>(
+        mlir::LLVM::ExtractValueOp::create(rewriter, loc, operands[0], 0),
+        mlir::LLVM::ExtractValueOp::create(rewriter, loc, operands[1], 0), fmf);
+    auto icp = mlir::LLVM::FCmpOp::create(rewriter,
         loc, resTy, pred,
-        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, operands[0], 1),
-        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, operands[1], 1), fmf);
+        mlir::LLVM::ExtractValueOp::create(rewriter, loc, operands[0], 1),
+        mlir::LLVM::ExtractValueOp::create(rewriter, loc, operands[1], 1), fmf);
     llvm::SmallVector<mlir::Value, 2> cp = {rcp, icp};
     switch (cmp.getPredicate()) {
     case mlir::arith::CmpFPredicate::OEQ: // .EQ.
@@ -778,16 +778,16 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
              "incompatible record types");
 
       auto toStTy = mlir::cast<mlir::LLVM::LLVMStructType>(toTy);
-      mlir::Value val = rewriter.create<mlir::LLVM::UndefOp>(loc, toStTy);
+      mlir::Value val = mlir::LLVM::UndefOp::create(rewriter, loc, toStTy);
       auto indexTypeMap = toStTy.getSubelementIndexMap();
       assert(indexTypeMap.has_value() && "invalid record type");
 
       for (auto [attr, type] : indexTypeMap.value()) {
         int64_t index = mlir::cast<mlir::IntegerAttr>(attr).getInt();
         auto extVal =
-            rewriter.create<mlir::LLVM::ExtractValueOp>(loc, op0, index);
+            mlir::LLVM::ExtractValueOp::create(rewriter, loc, op0, index);
         val =
-            rewriter.create<mlir::LLVM::InsertValueOp>(loc, val, extVal, index);
+            mlir::LLVM::InsertValueOp::create(rewriter, loc, val, extVal, index);
       }
 
       rewriter.replaceOp(convert, val);
@@ -831,7 +831,7 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
 
       // Compare the input with zero.
       mlir::Value zero = genConstantIndex(loc, fromTy, rewriter, 0);
-      auto isTrue = rewriter.create<mlir::LLVM::ICmpOp>(
+      auto isTrue = mlir::LLVM::ICmpOp::create(rewriter,
           loc, mlir::LLVM::ICmpPredicate::ne, op0, zero);
 
       // Zero extend the i1 isTrue result to the required type (unless it is i1
@@ -859,23 +859,23 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
         return {};
       }
       if (fromBits > toBits)
-        return rewriter.create<mlir::LLVM::FPTruncOp>(loc, toTy, val);
-      return rewriter.create<mlir::LLVM::FPExtOp>(loc, toTy, val);
+        return mlir::LLVM::FPTruncOp::create(rewriter, loc, toTy, val);
+      return mlir::LLVM::FPExtOp::create(rewriter, loc, toTy, val);
     };
     // Complex to complex conversion.
     if (fir::isa_complex(fromFirTy) && fir::isa_complex(toFirTy)) {
       // Special case: handle the conversion of a complex such that both the
       // real and imaginary parts are converted together.
       auto ty = convertType(getComplexEleTy(convert.getValue().getType()));
-      auto rp = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, op0, 0);
-      auto ip = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, op0, 1);
+      auto rp = mlir::LLVM::ExtractValueOp::create(rewriter, loc, op0, 0);
+      auto ip = mlir::LLVM::ExtractValueOp::create(rewriter, loc, op0, 1);
       auto nt = convertType(getComplexEleTy(convert.getRes().getType()));
       auto fromBits = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
       auto toBits = mlir::LLVM::getPrimitiveTypeSizeInBits(nt);
       auto rc = convertFpToFp(rp, fromBits, toBits, nt);
       auto ic = convertFpToFp(ip, fromBits, toBits, nt);
-      auto un = rewriter.create<mlir::LLVM::UndefOp>(loc, toTy);
-      auto i1 = rewriter.create<mlir::LLVM::InsertValueOp>(loc, un, rc, 0);
+      auto un = mlir::LLVM::UndefOp::create(rewriter, loc, toTy);
+      auto i1 = mlir::LLVM::InsertValueOp::create(rewriter, loc, un, rc, 0);
       rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(convert, i1, ic,
                                                              1);
       return mlir::success();
@@ -1023,7 +1023,7 @@ struct EmboxCharOpConversion : public fir::FIROpConversion<fir::EmboxCharOp> {
 
     mlir::Location loc = emboxChar.getLoc();
     mlir::Type llvmStructTy = convertType(emboxChar.getType());
-    auto llvmStruct = rewriter.create<mlir::LLVM::UndefOp>(loc, llvmStructTy);
+    auto llvmStruct = mlir::LLVM::UndefOp::create(rewriter, loc, llvmStructTy);
 
     mlir::Type lenTy =
         mlir::cast<mlir::LLVM::LLVMStructType>(llvmStructTy).getBody()[1];
@@ -1033,9 +1033,9 @@ struct EmboxCharOpConversion : public fir::FIROpConversion<fir::EmboxCharOp> {
         mlir::cast<mlir::LLVM::LLVMStructType>(llvmStructTy).getBody()[0];
     if (addrTy != charBuffer.getType())
       charBuffer =
-          rewriter.create<mlir::LLVM::BitcastOp>(loc, addrTy, charBuffer);
+          mlir::LLVM::BitcastOp::create(rewriter, loc, addrTy, charBuffer);
 
-    auto insertBufferOp = rewriter.create<mlir::LLVM::InsertValueOp>(
+    auto insertBufferOp = mlir::LLVM::InsertValueOp::create(rewriter,
         loc, llvmStruct, charBuffer, 0);
     rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
         emboxChar, insertBufferOp, lenAfterCast, 1);
@@ -1059,7 +1059,7 @@ getMallocInModule(ModuleOp mod, fir::AllocMemOp op,
     return mlir::SymbolRefAttr::get(userMalloc);
 
   mlir::OpBuilder moduleBuilder(mod.getBodyRegion());
-  auto mallocDecl = moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+  auto mallocDecl = mlir::LLVM::LLVMFuncOp::create(moduleBuilder,
       op.getLoc(), mallocName,
       mlir::LLVM::LLVMFunctionType::get(getLlvmPtrType(op.getContext()),
                                         indexType,
@@ -1120,9 +1120,9 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
       TODO(loc, "fir.allocmem codegen of derived type with length parameters");
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
     if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
-      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+      size = mlir::LLVM::MulOp::create(rewriter, loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands())
-      size = rewriter.create<mlir::LLVM::MulOp>(
+      size = mlir::LLVM::MulOp::create(rewriter,
           loc, ity, size, integerCast(loc, rewriter, ity, opnd));
 
     // As the return value of malloc(0) is implementation defined, allocate one
@@ -1130,9 +1130,9 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     // what the runtime has.
     mlir::Value zero = genConstantIndex(loc, ity, rewriter, 0);
     mlir::Value one = genConstantIndex(loc, ity, rewriter, 1);
-    mlir::Value cmp = rewriter.create<mlir::LLVM::ICmpOp>(
+    mlir::Value cmp = mlir::LLVM::ICmpOp::create(rewriter,
         loc, mlir::LLVM::ICmpPredicate::sgt, size, zero);
-    size = rewriter.create<mlir::LLVM::SelectOp>(loc, cmp, size, one);
+    size = mlir::LLVM::SelectOp::create(rewriter, loc, cmp, size, one);
 
     auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
     auto mallocTy =
@@ -1173,7 +1173,7 @@ getFreeInModule(ModuleOp mod, fir::FreeMemOp op,
   // Create llvm declaration for free.
   mlir::OpBuilder moduleBuilder(mod.getBodyRegion());
   auto voidType = mlir::LLVM::LLVMVoidType::get(op.getContext());
-  auto freeDecl = moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
+  auto freeDecl = mlir::LLVM::LLVMFuncOp::create(moduleBuilder,
       rewriter.getUnknownLoc(), freeName,
       mlir::LLVM::LLVMFunctionType::get(voidType,
                                         getLlvmPtrType(op.getContext()),
@@ -1209,7 +1209,7 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
                   mlir::ConversionPatternRewriter &rewriter) const override {
     mlir::Location loc = freemem.getLoc();
     freemem->setAttr("callee", getFree(freemem, rewriter));
-    rewriter.create<mlir::LLVM::CallOp>(
+    mlir::LLVM::CallOp::create(rewriter,
         loc, mlir::TypeRange{}, mlir::ValueRange{adaptor.getHeapref()},
         addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 1));
     rewriter.eraseOp(freemem);
@@ -1265,38 +1265,38 @@ static mlir::Value genSourceFile(mlir::Location loc, mlir::ModuleOp mod,
     std::string globalName = fir::factory::uniqueCGIdent("cl", fn);
 
     if (auto g = mod.lookupSymbol<fir::GlobalOp>(globalName)) {
-      return rewriter.create<mlir::LLVM::AddressOfOp>(loc, ptrTy, g.getName());
+      return mlir::LLVM::AddressOfOp::create(rewriter, loc, ptrTy, g.getName());
     } else if (auto g = mod.lookupSymbol<mlir::LLVM::GlobalOp>(globalName)) {
-      return rewriter.create<mlir::LLVM::AddressOfOp>(loc, ptrTy, g.getName());
+      return mlir::LLVM::AddressOfOp::create(rewriter, loc, ptrTy, g.getName());
     }
 
     auto crtInsPt = rewriter.saveInsertionPoint();
     rewriter.setInsertionPoint(mod.getBody(), mod.getBody()->end());
     auto arrayTy = mlir::LLVM::LLVMArrayType::get(
         mlir::IntegerType::get(rewriter.getContext(), 8), fn.size());
-    mlir::LLVM::GlobalOp globalOp = rewriter.create<mlir::LLVM::GlobalOp>(
+    mlir::LLVM::GlobalOp globalOp = mlir::LLVM::GlobalOp::create(rewriter,
         loc, arrayTy, /*constant=*/true, mlir::LLVM::Linkage::Linkonce,
         globalName, mlir::Attribute());
 
     mlir::Region &region = globalOp.getInitializerRegion();
     mlir::Block *block = rewriter.createBlock(&region);
     rewriter.setInsertionPoint(block, block->begin());
-    mlir::Value constValue = rewriter.create<mlir::LLVM::ConstantOp>(
+    mlir::Value constValue = mlir::LLVM::ConstantOp::create(rewriter,
         loc, arrayTy, rewriter.getStringAttr(fn));
-    rewriter.create<mlir::LLVM::ReturnOp>(loc, constValue);
+    mlir::LLVM::ReturnOp::create(rewriter, loc, constValue);
     rewriter.restoreInsertionPoint(crtInsPt);
-    return rewriter.create<mlir::LLVM::AddressOfOp>(loc, ptrTy,
+    return mlir::LLVM::AddressOfOp::create(rewriter, loc, ptrTy,
                                                     globalOp.getName());
   }
-  return rewriter.create<mlir::LLVM::ZeroOp>(loc, ptrTy);
+  return mlir::LLVM::ZeroOp::create(rewriter, loc, ptrTy);
 }
 
 static mlir::Value genSourceLine(mlir::Location loc,
                                  mlir::ConversionPatternRewriter &rewriter) {
   if (auto flc = mlir::dyn_cast<mlir::FileLineColLoc>(loc))
-    return rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI32Type(),
+    return mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
                                                    flc.getLine());
-  return rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI32Type(), 0);
+  return mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), 0);
 }
 
 static mlir::Value
@@ -1373,7 +1373,7 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
   if (options.ignoreMissingTypeDescriptors ||
       fir::NameUniquer::belongsToModule(
           name, Fortran::semantics::typeInfoBuiltinModule))
-    return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
+    return mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
     fir::emitFatalError(loc,
@@ -1386,11 +1386,11 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
   // option. Generate the object declaration now.
   auto insertPt = rewriter.saveInsertionPoint();
   rewriter.setInsertionPoint(mod.getBody(), mod.getBody()->end());
-  mlir::LLVM::GlobalOp global = rewriter.create<mlir::LLVM::GlobalOp>(
+  mlir::LLVM::GlobalOp global = mlir::LLVM::GlobalOp::create(rewriter,
       loc, llvmPtrTy, /*constant=*/true, mlir::LLVM::Linkage::External, name,
       mlir::Attribute());
   rewriter.restoreInsertionPoint(insertPt);
-  return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
+  return mlir::LLVM::AddressOfOp::create(rewriter, loc, llvmPtrTy,
                                                   global.getSymName());
 }
 
@@ -1422,7 +1422,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
     assert(!lenParams.empty());
     auto len64 = fir::FIROpConversion<OP>::integerCast(loc, rewriter, i64Ty,
                                                        lenParams.back());
-    return rewriter.create<mlir::LLVM::MulOp>(loc, i64Ty, size, len64);
+    return mlir::LLVM::MulOp::create(rewriter, loc, i64Ty, size, len64);
   }
 
   // Get the element size and CFI type code of the boxed value.
@@ -1437,7 +1437,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
       return getSizeAndTypeCode(loc, rewriter, seqTy.getEleTy(), lenParams);
     if (mlir::isa<mlir::NoneType>(
             boxEleTy)) // unlimited polymorphic or assumed type
-      return {rewriter.create<mlir::LLVM::ConstantOp>(loc, i64Ty, 0),
+      return {mlir::LLVM::ConstantOp::create(rewriter, loc, i64Ty, 0),
               this->genConstantOffset(loc, rewriter, CFI_type_other)};
     mlir::Value typeCodeVal = this->genConstantOffset(
         loc, rewriter,
@@ -1473,7 +1473,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
     if (!bitcast)
       value = this->integerCast(loc, rewriter, fldTy, value);
     // bitcast are no-ops with LLVM opaque pointers.
-    return rewriter.create<mlir::LLVM::InsertValueOp>(loc, dest, value,
+    return mlir::LLVM::InsertValueOp::create(rewriter, loc, dest, value,
                                                       fldIndexes);
   }
 
@@ -1518,7 +1518,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
     bool isUnlimitedPolymorphic = fir::isUnlimitedPolymorphicType(boxTy);
     bool useInputType = fir::isPolymorphicType(boxTy) || isUnlimitedPolymorphic;
     mlir::Value descriptor =
-        rewriter.create<mlir::LLVM::UndefOp>(loc, llvmBoxTy);
+        mlir::LLVM::UndefOp::create(rewriter, loc, llvmBoxTy);
     descriptor =
         insertField(rewriter, loc, descriptor, {kElemLenPosInBox}, eleSize);
     descriptor = insertField(rewriter, loc, descriptor, {kVersionPosInBox},
@@ -1539,16 +1539,16 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
         auto maskAttr = mlir::IntegerAttr::get(
             rewriter.getIntegerType(8, /*isSigned=*/false),
             llvm::APInt(8, (uint64_t)_CFI_ADDENDUM_FLAG, /*isSigned=*/false));
-        mlir::LLVM::ConstantOp mask = rewriter.create<mlir::LLVM::ConstantOp>(
+        mlir::LLVM::ConstantOp mask = mlir::LLVM::ConstantOp::create(rewriter,
             loc, rewriter.getI8Type(), maskAttr);
-        extraField = rewriter.create<mlir::LLVM::OrOp>(loc, extraField, mask);
+        extraField = mlir::LLVM::OrOp::create(rewriter, loc, extraField, mask);
       } else {
         auto maskAttr = mlir::IntegerAttr::get(
             rewriter.getIntegerType(8, /*isSigned=*/false),
             llvm::APInt(8, (uint64_t)~_CFI_ADDENDUM_FLAG, /*isSigned=*/true));
-        mlir::LLVM::ConstantOp mask = rewriter.create<mlir::LLVM::ConstantOp>(
+        mlir::LLVM::ConstantOp mask = mlir::LLVM::ConstantOp::create(rewriter,
             loc, rewriter.getI8Type(), maskAttr);
-        extraField = rewriter.create<mlir::LLVM::AndOp>(loc, extraField, mask);
+        extraField = mlir::LLVM::AndOp::create(rewriter, loc, extraField, mask);
       }
       // Extra field value is provided so just use it.
       descriptor =
@@ -1575,7 +1575,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
           } else {
             // Unlimited polymorphic type descriptor with no record type. Set
             // type descriptor address to a clean state.
-            typeDesc = rewriter.create<mlir::LLVM::ZeroOp>(
+            typeDesc = mlir::LLVM::ZeroOp::create(rewriter,
                 loc, ::getLlvmPtrType(mod.getContext()));
           }
         } else {
@@ -1756,12 +1756,12 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
         mlir::Value cast =
             this->integerCast(loc, rewriter, outterOffsetTy, *substringOffset);
 
-        gepArgs[0] = rewriter.create<mlir::LLVM::AddOp>(
+        gepArgs[0] = mlir::LLVM::AddOp::create(rewriter,
             loc, outterOffsetTy, llvm::cast<mlir::Value>(gepArgs[0]), cast);
       }
     }
     mlir::Type llvmPtrTy = ::getLlvmPtrType(resultTy.getContext());
-    return rewriter.create<mlir::LLVM::GEPOp>(
+    return mlir::LLVM::GEPOp::create(rewriter,
         loc, llvmPtrTy, llvmBaseObjectType, base, gepArgs);
   }
 
@@ -1809,7 +1809,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
       storage = this->genAllocaAndAddrCastWithType(loc, llvmBoxTy, defaultAlign,
                                                    rewriter);
     }
-    auto storeOp = rewriter.create<mlir::LLVM::StoreOp>(loc, boxValue, storage);
+    auto storeOp = mlir::LLVM::StoreOp::create(rewriter, loc, boxValue, storage);
     this->attachTBAATag(storeOp, boxTy, boxTy, nullptr);
     return storage;
   }
@@ -1823,14 +1823,14 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
     ub = this->integerCast(loc, rewriter, type, ub);
     step = this->integerCast(loc, rewriter, type, step);
     zero = this->integerCast(loc, rewriter, type, zero);
-    mlir::Value extent = rewriter.create<mlir::LLVM::SubOp>(loc, type, ub, lb);
-    extent = rewriter.create<mlir::LLVM::AddOp>(loc, type, extent, step);
-    extent = rewriter.create<mlir::LLVM::SDivOp>(loc, type, extent, step);
+    mlir::Value extent = mlir::LLVM::SubOp::create(rewriter, loc, type, ub, lb);
+    extent = mlir::LLVM::AddOp::create(rewriter, loc, type, extent, step);
+    extent = mlir::LLVM::SDivOp::create(rewriter, loc, type, extent, step);
     // If the resulting extent is negative (`ub-lb` and `step` have different
     // signs), zero must be returned instead.
-    auto cmp = rewriter.create<mlir::LLVM::ICmpOp>(
+    auto cmp = mlir::LLVM::ICmpOp::create(rewriter,
         loc, mlir::LLVM::ICmpPredicate::sgt, extent, zero);
-    return rewriter.create<mlir::LLVM::SelectOp>(loc, cmp, extent, zero);
+    return mlir::LLVM::SelectOp::create(rewriter, loc, cmp, extent, zero);
   }
 };
 
@@ -2005,14 +2005,14 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
         mlir::Value adj = one;
         if (hasShift)
           adj = integerCast(loc, rewriter, i64Ty, operands[shiftOffset]);
-        auto ao = rewriter.create<mlir::LLVM::SubOp>(loc, i64Ty, off, adj);
+        auto ao = mlir::LLVM::SubOp::create(rewriter, loc, i64Ty, off, adj);
         if (constRows > 0) {
           cstInteriorIndices.push_back(ao);
         } else {
           auto dimOff =
-              rewriter.create<mlir::LLVM::MulOp>(loc, i64Ty, ao, prevPtrOff);
+              mlir::LLVM::MulOp::create(rewriter, loc, i64Ty, ao, prevPtrOff);
           ptrOffset =
-              rewriter.create<mlir::LLVM::AddOp>(loc, i64Ty, dimOff, ptrOffset);
+              mlir::LLVM::AddOp::create(rewriter, loc, i64Ty, dimOff, ptrOffset);
         }
         if (mlir::isa_and_nonnull<fir::UndefOp>(
                 xbox.getSlice()[3 * di + 1].getDefiningOp())) {
@@ -2042,9 +2042,9 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
         if (hasShift && !(hasSlice || hasSubcomp || hasSubstr) &&
             (isaPointerOrAllocatable || !normalizedLowerBound(xbox))) {
           lb = integerCast(loc, rewriter, i64Ty, operands[shiftOffset]);
-          auto extentIsEmpty = rewriter.create<mlir::LLVM::ICmpOp>(
+          auto extentIsEmpty = mlir::LLVM::ICmpOp::create(rewriter,
               loc, mlir::LLVM::ICmpPredicate::eq, extent, zero);
-          lb = rewriter.create<mlir::LLVM::SelectOp>(loc, extentIsEmpty, one,
+          lb = mlir::LLVM::SelectOp::create(rewriter, loc, extentIsEmpty, one,
                                                      lb);
         }
         dest = insertLowerBound(rewriter, loc, dest, descIdx, lb);
@@ -2057,17 +2057,17 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
           mlir::Value sliceStep =
               integerCast(loc, rewriter, i64Ty, operands[sliceOffset + 2]);
           step =
-              rewriter.create<mlir::LLVM::MulOp>(loc, i64Ty, step, sliceStep);
+              mlir::LLVM::MulOp::create(rewriter, loc, i64Ty, step, sliceStep);
         }
         dest = insertStride(rewriter, loc, dest, descIdx, step);
         ++descIdx;
       }
 
       // compute the stride and offset for the next natural dimension
-      prevDimByteStride = rewriter.create<mlir::LLVM::MulOp>(
+      prevDimByteStride = mlir::LLVM::MulOp::create(rewriter,
           loc, i64Ty, prevDimByteStride, outerExtent);
       if (constRows == 0)
-        prevPtrOff = rewriter.create<mlir::LLVM::MulOp>(loc, i64Ty, prevPtrOff,
+        prevPtrOff = mlir::LLVM::MulOp::create(rewriter, loc, i64Ty, prevPtrOff,
                                                         outerExtent);
       else
         --constRows;
@@ -2153,7 +2153,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
                  "character target in global op must have constant length");
           mlir::Value width =
               genConstantIndex(loc, idxTy, rewriter, charTy.getFKind());
-          len = rewriter.create<mlir::LLVM::SDivOp>(loc, idxTy, len, width);
+          len = mlir::LLVM::SDivOp::create(rewriter, loc, idxTy, len, width);
         }
         lenParams.emplace_back(len);
       }
@@ -2214,9 +2214,9 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       mlir::Value lb = one;
       if (!lbounds.empty()) {
         lb = integerCast(loc, rewriter, lowerTy().indexType(), lbounds[dim]);
-        auto extentIsEmpty = rewriter.create<mlir::LLVM::ICmpOp>(
+        auto extentIsEmpty = mlir::LLVM::ICmpOp::create(rewriter,
             loc, mlir::LLVM::ICmpPredicate::eq, extent, zero);
-        lb = rewriter.create<mlir::LLVM::SelectOp>(loc, extentIsEmpty, one, lb);
+        lb = mlir::LLVM::SelectOp::create(rewriter, loc, extentIsEmpty, one, lb);
       };
       dest = insertLowerBound(rewriter, loc, dest, dim, lb);
       dest = insertExtent(rewriter, loc, dest, dim, extent);
@@ -2284,9 +2284,9 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
               ? integerCast(loc, rewriter, idxTy, operands[shiftOps])
               : one;
       mlir::Value diff =
-          rewriter.create<mlir::LLVM::SubOp>(loc, idxTy, sliceLb, sliceOrigin);
+          mlir::LLVM::SubOp::create(rewriter, loc, idxTy, sliceLb, sliceOrigin);
       mlir::Value offset =
-          rewriter.create<mlir::LLVM::MulOp>(loc, idxTy, diff, inputStride);
+          mlir::LLVM::MulOp::create(rewriter, loc, idxTy, diff, inputStride);
       // Strides from the fir.box are in bytes.
       base = genGEP(loc, byteTy, rewriter, base, offset);
       // Apply upper bound and step if this is a triplet. Otherwise, the
@@ -2304,7 +2304,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
         slicedExtents.emplace_back(extent);
         // stride = step*input_stride
         mlir::Value stride =
-            rewriter.create<mlir::LLVM::MulOp>(loc, idxTy, step, inputStride);
+            mlir::LLVM::MulOp::create(rewriter, loc, idxTy, step, inputStride);
         slicedStrides.emplace_back(stride);
       }
     }
@@ -2348,7 +2348,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       newExtents.emplace_back(extent);
       newStrides.emplace_back(stride);
       // nextStride = extent * stride;
-      stride = rewriter.create<mlir::LLVM::MulOp>(loc, idxTy, extent, stride);
+      stride = mlir::LLVM::MulOp::create(rewriter, loc, idxTy, extent, stride);
     }
     return finalizeRebox(rebox, adaptor, destBoxTy, dest, base, reboxShifts,
                          newExtents, newStrides, rewriter);
@@ -2536,7 +2536,7 @@ struct InsertOnRangeOpConversion
     mlir::Value insertVal = adaptor.getVal();
 
     while (subscripts != uBounds) {
-      lastOp = rewriter.create<mlir::LLVM::InsertValueOp>(
+      lastOp = mlir::LLVM::InsertValueOp::create(rewriter,
           loc, lastOp, insertVal, subscripts);
 
       incrementSubscripts(dims, subscripts);
@@ -2606,15 +2606,15 @@ struct XArrayCoorOpConversion
         if (normalSlice)
           step = integerCast(loc, rewriter, idxTy, operands[sliceOffset + 2]);
       }
-      auto idx = rewriter.create<mlir::LLVM::SubOp>(loc, idxTy, index, lb, nsw);
+      auto idx = mlir::LLVM::SubOp::create(rewriter, loc, idxTy, index, lb, nsw);
       mlir::Value diff =
-          rewriter.create<mlir::LLVM::MulOp>(loc, idxTy, idx, step, nsw);
+          mlir::LLVM::MulOp::create(rewriter, loc, idxTy, idx, step, nsw);
       if (normalSlice) {
         mlir::Value sliceLb =
             integerCast(loc, rewriter, idxTy, operands[sliceOffset]);
         auto adj =
-            rewriter.create<mlir::LLVM::SubOp>(loc, idxTy, sliceLb, lb, nsw);
-        diff = rewriter.create<mlir::LLVM::AddOp>(loc, idxTy, diff, adj, nsw);
+            mlir::LLVM::SubOp::create(rewriter, loc, idxTy, sliceLb, lb, nsw);
+        diff = mlir::LLVM::AddOp::create(rewriter, loc, idxTy, diff, adj, nsw);
       }
       // Update the offset given the stride and the zero based index `diff`
       // that was just computed.
@@ -2623,19 +2623,19 @@ struct XArrayCoorOpConversion
         mlir::Value stride =
             getStrideFromBox(loc, baseBoxTyPair, operands[0], i, rewriter);
         auto sc =
-            rewriter.create<mlir::LLVM::MulOp>(loc, idxTy, diff, stride, nsw);
+            mlir::LLVM::MulOp::create(rewriter, loc, idxTy, diff, stride, nsw);
         offset =
-            rewriter.create<mlir::LLVM::AddOp>(loc, idxTy, sc, offset, nsw);
+            mlir::LLVM::AddOp::create(rewriter, loc, idxTy, sc, offset, nsw);
       } else {
         // Use stride computed at last iteration.
         auto sc =
-            rewriter.create<mlir::LLVM::MulOp>(loc, idxTy, diff, prevExt, nsw);
+            mlir::LLVM::MulOp::create(rewriter, loc, idxTy, diff, prevExt, nsw);
         offset =
-            rewriter.create<mlir::LLVM::AddOp>(loc, idxTy, sc, offset, nsw);
+            mlir::LLVM::AddOp::create(rewriter, loc, idxTy, sc, offset, nsw);
         // Compute next stride assuming contiguity of the base array
         // (in element number).
         auto nextExt = integerCast(loc, rewriter, idxTy, operands[shapeOffset]);
-        prevExt = rewriter.create<mlir::LLVM::MulOp>(loc, idxTy, prevExt,
+        prevExt = mlir::LLVM::MulOp::create(rewriter, loc, idxTy, prevExt,
                                                      nextExt, nsw);
       }
     }
@@ -2648,7 +2648,7 @@ struct XArrayCoorOpConversion
       mlir::Value base =
           getBaseAddrFromBox(loc, baseBoxTyPair, operands[0], rewriter);
       llvm::SmallVector<mlir::LLVM::GEPArg> args{offset};
-      auto addr = rewriter.create<mlir::LLVM::GEPOp>(loc, llvmPtrTy, byteTy,
+      auto addr = mlir::LLVM::GEPOp::create(rewriter, loc, llvmPtrTy, byteTy,
                                                      base, args);
       if (coor.getSubcomponent().empty()) {
         rewriter.replaceOp(coor, addr);
@@ -2696,7 +2696,7 @@ struct XArrayCoorOpConversion
           assert(coor.getLenParams().size() == 1);
           auto length = integerCast(loc, rewriter, idxTy,
                                     operands[coor.getLenParamsOperandIndex()]);
-          offset = rewriter.create<mlir::LLVM::MulOp>(loc, idxTy, offset,
+          offset = mlir::LLVM::MulOp::create(rewriter, loc, idxTy, offset,
                                                       length, nsw);
         } else {
           TODO(loc, "compute size of derived type with type parameters");
@@ -2912,12 +2912,12 @@ struct CoordinateOpConversion
         for (unsigned dim = 0; dim < arrayDim && it != end; ++dim, ++it) {
           mlir::Value stride =
               getStrideFromBox(loc, boxTyPair, operands[0], dim, rewriter);
-          auto sc = rewriter.create<mlir::LLVM::MulOp>(
+          auto sc = mlir::LLVM::MulOp::create(rewriter,
               loc, idxTy, operands[nextIndexValue + dim], stride, nsw);
-          off = rewriter.create<mlir::LLVM::AddOp>(loc, idxTy, sc, off, nsw);
+          off = mlir::LLVM::AddOp::create(rewriter, loc, idxTy, sc, off, nsw);
         }
         nextIndexValue += arrayDim;
-        resultAddr = rewriter.create<mlir::LLVM::GEPOp>(
+        resultAddr = mlir::LLVM::GEPOp::create(rewriter,
             loc, llvmPtrTy, byteTy, resultAddr,
             llvm::ArrayRef<mlir::LLVM::GEPArg>{off});
         cpnTy = arrTy.getEleTy();
@@ -2930,7 +2930,7 @@ struct CoordinateOpConversion
         ++it;
         cpnTy = recTy.getType(fieldIndex);
         auto llvmRecTy = lowerTy().convertType(recTy);
-        resultAddr = rewriter.create<mlir::LLVM::GEPOp>(
+        resultAddr = mlir::LLVM::GEPOp::create(rewriter,
             loc, llvmPtrTy, llvmRecTy, resultAddr,
             llvm::ArrayRef<mlir::LLVM::GEPArg>{0, fieldIndex});
       } else {
@@ -3184,7 +3184,7 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
     auto isConst = global.getConstant().has_value();
     mlir::SymbolRefAttr comdat;
     llvm::ArrayRef<mlir::NamedAttribute> attrs;
-    auto g = rewriter.create<mlir::LLVM::GlobalOp>(
+    auto g = mlir::LLVM::GlobalOp::create(rewriter,
         loc, tyAttr, isConst, linkage, global.getSymName(), initAttr, 0,
         getGlobalAddressSpace(rewriter), false, false, comdat, attrs, dbgExprs);
 
@@ -3276,14 +3276,14 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
         module.lookupSymbol<mlir::LLVM::ComdatOp>(comdatName);
     if (!comdatOp) {
       comdatOp =
-          rewriter.create<mlir::LLVM::ComdatOp>(module.getLoc(), comdatName);
+          mlir::LLVM::ComdatOp::create(rewriter, module.getLoc(), comdatName);
     }
     if (auto select = comdatOp.lookupSymbol<mlir::LLVM::ComdatSelectorOp>(
             global.getSymName()))
       return;
     mlir::OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPointToEnd(&comdatOp.getBody().back());
-    auto selectorOp = rewriter.create<mlir::LLVM::ComdatSelectorOp>(
+    auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(rewriter,
         comdatOp.getLoc(), global.getSymName(),
         mlir::LLVM::comdat::Comdat::Any);
     global.setComdatAttr(mlir::SymbolRefAttr::get(
@@ -3331,7 +3331,7 @@ struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
       TypePair boxTypePair{boxTy, llvmLoadTy};
       mlir::Value boxSize =
           computeBoxSize(loc, boxTypePair, inputBoxStorage, rewriter);
-      auto memcpy = rewriter.create<mlir::LLVM::MemcpyOp>(
+      auto memcpy = mlir::LLVM::MemcpyOp::create(rewriter,
           loc, newBoxStorage, inputBoxStorage, boxSize, isVolatile);
 
       if (std::optional<mlir::ArrayAttr> optionalTag = load.getTbaa())
@@ -3340,7 +3340,7 @@ struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
         attachTBAATag(memcpy, boxTy, boxTy, nullptr);
       rewriter.replaceOp(load, newBoxStorage);
     } else {
-      mlir::LLVM::LoadOp loadOp = rewriter.create<mlir::LLVM::LoadOp>(
+      mlir::LLVM::LoadOp loadOp = mlir::LLVM::LoadOp::create(rewriter,
           load.getLoc(), llvmLoadTy, adaptor.getOperands(), load->getAttrs());
       loadOp.setVolatile_(isVolatile);
       if (std::optional<mlir::ArrayAttr> optionalTag = load.getTbaa())
@@ -3396,10 +3396,10 @@ static void genCondBrOp(mlir::Location loc, mlir::Value cmp, mlir::Block *dest,
                         mlir::ConversionPatternRewriter &rewriter,
                         mlir::Block *newBlock) {
   if (destOps)
-    rewriter.create<mlir::LLVM::CondBrOp>(loc, cmp, dest, *destOps, newBlock,
+    mlir::LLVM::CondBrOp::create(rewriter, loc, cmp, dest, *destOps, newBlock,
                                           mlir::ValueRange());
   else
-    rewriter.create<mlir::LLVM::CondBrOp>(loc, cmp, dest, newBlock);
+    mlir::LLVM::CondBrOp::create(rewriter, loc, cmp, dest, newBlock);
 }
 
 template <typename A, typename B>
@@ -3466,35 +3466,35 @@ struct SelectCaseOpConversion : public fir::FIROpConversion<fir::SelectCaseOp> {
       mlir::Attribute attr = cases[t];
       assert(mlir::isa<mlir::UnitAttr>(attr) || cmpOps.has_value());
       if (mlir::isa<fir::PointIntervalAttr>(attr)) {
-        auto cmp = rewriter.create<mlir::LLVM::ICmpOp>(
+        auto cmp = mlir::LLVM::ICmpOp::create(rewriter,
             loc, mlir::LLVM::ICmpPredicate::eq, selector, cmpOps->front());
         genCaseLadderStep(loc, cmp, dest, destOps, rewriter);
         continue;
       }
       if (mlir::isa<fir::LowerBoundAttr>(attr)) {
-        auto cmp = rewriter.create<mlir::LLVM::ICmpOp>(
+        auto cmp = mlir::LLVM::ICmpOp::create(rewriter,
             loc, mlir::LLVM::ICmpPredicate::sle, cmpOps->front(), selector);
         genCaseLadderStep(loc, cmp, dest, destOps, rewriter);
         continue;
       }
       if (mlir::isa<fir::UpperBoundAttr>(attr)) {
-        auto cmp = rewriter.create<mlir::LLVM::ICmpOp>(
+        auto cmp = mlir::LLVM::ICmpOp::create(rewriter,
             loc, mlir::LLVM::ICmpPredicate::sle, selector, cmpOps->front());
         genCaseLadderStep(loc, cmp, dest, destOps, rewriter);
         continue;
       }
       if (mlir::isa<fir::ClosedIntervalAttr>(attr)) {
         mlir::Value caseArg0 = *cmpOps->begin();
-        auto cmp0 = rewriter.create<mlir::LLVM::ICmpOp>(
+        auto cmp0 = mlir::LLVM::ICmpOp::create(rewriter,
             loc, mlir::LLVM::ICmpPredicate::sle, caseArg0, selector);
         auto *thisBlock = rewriter.getInsertionBlock();
         auto *newBlock1 = createBlock(rewriter, dest);
         auto *newBlock2 = createBlock(rewriter, dest);
         rewriter.setInsertionPointToEnd(thisBlock);
-        rewriter.create<mlir::LLVM::CondBrOp>(loc, cmp0, newBlock1, newBlock2);
+        mlir::LLVM::CondBrOp::create(rewriter, loc, cmp0, newBlock1, newBlock2);
         rewriter.setInsertionPointToEnd(newBlock1);
         mlir::Value caseArg1 = *(cmpOps->begin() + 1);
-        auto cmp1 = rewriter.create<mlir::LLVM::ICmpOp>(
+        auto cmp1 = mlir::LLVM::ICmpOp::create(rewriter,
             loc, mlir::LLVM::ICmpPredicate::sle, selector, caseArg1);
         genCondBrOp(loc, cmp1, dest, destOps, rewriter, newBlock2);
         rewriter.setInsertionPointToEnd(newBlock2);
@@ -3581,7 +3581,7 @@ selectMatchAndRewrite(const fir::LLVMTypeConverter &lowering, OP select,
 
   // LLVM::SwitchOp takes a i32 type for the selector.
   if (select.getSelector().getType() != rewriter.getI32Type())
-    selector = rewriter.create<mlir::LLVM::TruncOp>(loc, rewriter.getI32Type(),
+    selector = mlir::LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(),
                                                     selector);
 
   rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(
@@ -3654,11 +3654,11 @@ struct StoreOpConversion : public fir::FIROpConversion<fir::StoreOp> {
       TypePair boxTypePair{boxTy, llvmBoxTy};
       mlir::Value boxSize =
           computeBoxSize(loc, boxTypePair, llvmValue, rewriter);
-      newOp = rewriter.create<mlir::LLVM::MemcpyOp>(loc, llvmMemref, llvmValue,
+      newOp = mlir::LLVM::MemcpyOp::create(rewriter, loc, llvmMemref, llvmValue,
                                                     boxSize, isVolatile);
     } else {
       mlir::LLVM::StoreOp storeOp =
-          rewriter.create<mlir::LLVM::StoreOp>(loc, llvmValue, llvmMemref);
+          mlir::LLVM::StoreOp::create(rewriter, loc, llvmValue, llvmMemref);
 
       if (isVolatile)
         storeOp.setVolatile_(true);
@@ -3697,10 +3697,10 @@ struct CopyOpConversion : public fir::FIROpConversion<fir::CopyOp> {
 
     mlir::LLVM::AliasAnalysisOpInterface newOp;
     if (copy.getNoOverlap())
-      newOp = rewriter.create<mlir::LLVM::MemcpyOp>(
+      newOp = mlir::LLVM::MemcpyOp::create(rewriter,
           loc, llvmDestination, llvmSource, copySize, isVolatile);
     else
-      newOp = rewriter.create<mlir::LLVM::MemmoveOp>(
+      newOp = mlir::LLVM::MemmoveOp::create(rewriter,
           loc, llvmDestination, llvmSource, copySize, isVolatile);
 
     // TODO: propagate TBAA once FirAliasTagOpInterface added to CopyOp.
@@ -3725,9 +3725,9 @@ struct UnboxCharOpConversion : public fir::FIROpConversion<fir::UnboxCharOp> {
 
     mlir::Location loc = unboxchar.getLoc();
     mlir::Value ptrToBuffer =
-        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, tuple, 0);
+        mlir::LLVM::ExtractValueOp::create(rewriter, loc, tuple, 0);
 
-    auto len = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, tuple, 1);
+    auto len = mlir::LLVM::ExtractValueOp::create(rewriter, loc, tuple, 1);
     mlir::Value lenAfterCast = integerCast(loc, rewriter, lenTy, len);
 
     rewriter.replaceOp(unboxchar,
@@ -3817,11 +3817,11 @@ struct IsPresentOpConversion : public fir::FIROpConversion<fir::IsPresentOp> {
           mlir::cast<mlir::LLVM::LLVMStructType>(ptr.getType());
       assert(!structTy.isOpaque() && !structTy.getBody().empty());
 
-      ptr = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, ptr, 0);
+      ptr = mlir::LLVM::ExtractValueOp::create(rewriter, loc, ptr, 0);
     }
     mlir::LLVM::ConstantOp c0 =
         genConstantIndex(isPresent.getLoc(), idxTy, rewriter, 0);
-    auto addr = rewriter.create<mlir::LLVM::PtrToIntOp>(loc, idxTy, ptr);
+    auto addr = mlir::LLVM::PtrToIntOp::create(rewriter, loc, idxTy, ptr);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         isPresent, mlir::LLVM::ICmpPredicate::ne, addr, c0);
 
@@ -3866,15 +3866,15 @@ complexSum(OPTY sumop, mlir::ValueRange opnds,
   auto loc = sumop.getLoc();
   mlir::Type eleTy = lowering.convertType(getComplexEleTy(sumop.getType()));
   mlir::Type ty = lowering.convertType(sumop.getType());
-  auto x0 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, a, 0);
-  auto y0 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, a, 1);
-  auto x1 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, b, 0);
-  auto y1 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, b, 1);
-  auto rx = rewriter.create<LLVMOP>(loc, eleTy, x0, x1, fmf);
-  auto ry = rewriter.create<LLVMOP>(loc, eleTy, y0, y1, fmf);
-  auto r0 = rewriter.create<mlir::LLVM::UndefOp>(loc, ty);
-  auto r1 = rewriter.create<mlir::LLVM::InsertValueOp>(loc, r0, rx, 0);
-  return rewriter.create<mlir::LLVM::InsertValueOp>(loc, r1, ry, 1);
+  auto x0 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, a, 0);
+  auto y0 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, a, 1);
+  auto x1 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, b, 0);
+  auto y1 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, b, 1);
+  auto rx = LLVMOP::create(rewriter, loc, eleTy, x0, x1, fmf);
+  auto ry = LLVMOP::create(rewriter, loc, eleTy, y0, y1, fmf);
+  auto r0 = mlir::LLVM::UndefOp::create(rewriter, loc, ty);
+  auto r1 = mlir::LLVM::InsertValueOp::create(rewriter, loc, r0, rx, 0);
+  return mlir::LLVM::InsertValueOp::create(rewriter, loc, r1, ry, 1);
 }
 } // namespace
 
@@ -3925,19 +3925,19 @@ struct MulcOpConversion : public fir::FIROpConversion<fir::MulcOp> {
     auto loc = mulc.getLoc();
     mlir::Type eleTy = convertType(getComplexEleTy(mulc.getType()));
     mlir::Type ty = convertType(mulc.getType());
-    auto x0 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, a, 0);
-    auto y0 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, a, 1);
-    auto x1 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, b, 0);
-    auto y1 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, b, 1);
-    auto xx = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, x0, x1, fmf);
-    auto yx = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, y0, x1, fmf);
-    auto xy = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, x0, y1, fmf);
-    auto ri = rewriter.create<mlir::LLVM::FAddOp>(loc, eleTy, xy, yx, fmf);
-    auto yy = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, y0, y1, fmf);
-    auto rr = rewriter.create<mlir::LLVM::FSubOp>(loc, eleTy, xx, yy, fmf);
-    auto ra = rewriter.create<mlir::LLVM::UndefOp>(loc, ty);
-    auto r1 = rewriter.create<mlir::LLVM::InsertValueOp>(loc, ra, rr, 0);
-    auto r0 = rewriter.create<mlir::LLVM::InsertValueOp>(loc, r1, ri, 1);
+    auto x0 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, a, 0);
+    auto y0 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, a, 1);
+    auto x1 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, b, 0);
+    auto y1 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, b, 1);
+    auto xx = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, x0, x1, fmf);
+    auto yx = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, y0, x1, fmf);
+    auto xy = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, x0, y1, fmf);
+    auto ri = mlir::LLVM::FAddOp::create(rewriter, loc, eleTy, xy, yx, fmf);
+    auto yy = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, y0, y1, fmf);
+    auto rr = mlir::LLVM::FSubOp::create(rewriter, loc, eleTy, xx, yy, fmf);
+    auto ra = mlir::LLVM::UndefOp::create(rewriter, loc, ty);
+    auto r1 = mlir::LLVM::InsertValueOp::create(rewriter, loc, ra, rr, 0);
+    auto r0 = mlir::LLVM::InsertValueOp::create(rewriter, loc, r1, ri, 1);
     rewriter.replaceOp(mulc, r0.getResult());
     return mlir::success();
   }
@@ -3960,24 +3960,24 @@ struct DivcOpConversion : public fir::FIROpConversion<fir::DivcOp> {
     auto loc = divc.getLoc();
     mlir::Type eleTy = convertType(getComplexEleTy(divc.getType()));
     mlir::Type ty = convertType(divc.getType());
-    auto x0 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, a, 0);
-    auto y0 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, a, 1);
-    auto x1 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, b, 0);
-    auto y1 = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, b, 1);
-    auto xx = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, x0, x1, fmf);
-    auto x1x1 = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, x1, x1, fmf);
-    auto yx = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, y0, x1, fmf);
-    auto xy = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, x0, y1, fmf);
-    auto yy = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, y0, y1, fmf);
-    auto y1y1 = rewriter.create<mlir::LLVM::FMulOp>(loc, eleTy, y1, y1, fmf);
-    auto d = rewriter.create<mlir::LLVM::FAddOp>(loc, eleTy, x1x1, y1y1, fmf);
-    auto rrn = rewriter.create<mlir::LLVM::FAddOp>(loc, eleTy, xx, yy, fmf);
-    auto rin = rewriter.create<mlir::LLVM::FSubOp>(loc, eleTy, yx, xy, fmf);
-    auto rr = rewriter.create<mlir::LLVM::FDivOp>(loc, eleTy, rrn, d, fmf);
-    auto ri = rewriter.create<mlir::LLVM::FDivOp>(loc, eleTy, rin, d, fmf);
-    auto ra = rewriter.create<mlir::LLVM::UndefOp>(loc, ty);
-    auto r1 = rewriter.create<mlir::LLVM::InsertValueOp>(loc, ra, rr, 0);
-    auto r0 = rewriter.create<mlir::LLVM::InsertValueOp>(loc, r1, ri, 1);
+    auto x0 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, a, 0);
+    auto y0 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, a, 1);
+    auto x1 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, b, 0);
+    auto y1 = mlir::LLVM::ExtractValueOp::create(rewriter, loc, b, 1);
+    auto xx = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, x0, x1, fmf);
+    auto x1x1 = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, x1, x1, fmf);
+    auto yx = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, y0, x1, fmf);
+    auto xy = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, x0, y1, fmf);
+    auto yy = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, y0, y1, fmf);
+    auto y1y1 = mlir::LLVM::FMulOp::create(rewriter, loc, eleTy, y1, y1, fmf);
+    auto d = mlir::LLVM::FAddOp::create(rewriter, loc, eleTy, x1x1, y1y1, fmf);
+    auto rrn = mlir::LLVM::FAddOp::create(rewriter, loc, eleTy, xx, yy, fmf);
+    auto rin = mlir::LLVM::FSubOp::create(rewriter, loc, eleTy, yx, xy, fmf);
+    auto rr = mlir::LLVM::FDivOp::create(rewriter, loc, eleTy, rrn, d, fmf);
+    auto ri = mlir::LLVM::FDivOp::create(rewriter, loc, eleTy, rin, d, fmf);
+    auto ra = mlir::LLVM::UndefOp::create(rewriter, loc, ty);
+    auto r1 = mlir::LLVM::InsertValueOp::create(rewriter, loc, ra, rr, 0);
+    auto r0 = mlir::LLVM::InsertValueOp::create(rewriter, loc, r1, ri, 1);
     rewriter.replaceOp(divc, r0.getResult());
     return mlir::success();
   }
@@ -3995,11 +3995,11 @@ struct NegcOpConversion : public fir::FIROpConversion<fir::NegcOp> {
     auto eleTy = convertType(getComplexEleTy(neg.getType()));
     auto loc = neg.getLoc();
     mlir::Value o0 = adaptor.getOperands()[0];
-    auto rp = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, o0, 0);
-    auto ip = rewriter.create<mlir::LLVM::ExtractValueOp>(loc, o0, 1);
-    auto nrp = rewriter.create<mlir::LLVM::FNegOp>(loc, eleTy, rp);
-    auto nip = rewriter.create<mlir::LLVM::FNegOp>(loc, eleTy, ip);
-    auto r = rewriter.create<mlir::LLVM::InsertValueOp>(loc, o0, nrp, 0);
+    auto rp = mlir::LLVM::ExtractValueOp::create(rewriter, loc, o0, 0);
+    auto ip = mlir::LLVM::ExtractValueOp::create(rewriter, loc, o0, 1);
+    auto nrp = mlir::LLVM::FNegOp::create(rewriter, loc, eleTy, rp);
+    auto nip = mlir::LLVM::FNegOp::create(rewriter, loc, eleTy, ip);
+    auto r = mlir::LLVM::InsertValueOp::create(rewriter, loc, o0, nrp, 0);
     rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(neg, r, nip, 1);
     return mlir::success();
   }
diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
index 75a54fd0ee14f..ea234e20c6856 100644
--- a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
+++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
@@ -48,7 +48,7 @@ mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genI32Constant(
     int value) const {
   mlir::Type i32Ty = rewriter.getI32Type();
   mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value);
-  return rewriter.create<mlir::LLVM::ConstantOp>(loc, i32Ty, attr);
+  return mlir::LLVM::ConstantOp::create(rewriter, loc, i32Ty, attr);
 }
 
 mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genConstantOffset(
@@ -56,7 +56,7 @@ mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genConstantOffset(
     int offset) const {
   mlir::Type ity = lowerTy().offsetType();
   mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset);
-  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+  return mlir::LLVM::ConstantOp::create(rewriter, loc, ity, cattr);
 }
 
 /// Perform an extension or truncation as needed on an integer value. Lowering
@@ -80,9 +80,9 @@ mlir::Value ConvertFIRToLLVMPattern::integerCast(
       return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
   } else {
     if (toSize < fromSize)
-      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+      return mlir::LLVM::TruncOp::create(rewriter, loc, ty, val);
     if (toSize > fromSize)
-      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+      return mlir::LLVM::SExtOp::create(rewriter, loc, ty, val);
   }
   return val;
 }
@@ -100,16 +100,16 @@ mlir::Value ConvertFIRToLLVMPattern::getValueFromBox(
     mlir::ConversionPatternRewriter &rewriter, int boxValue) const {
   if (mlir::isa<mlir::LLVM::LLVMPointerType>(box.getType())) {
     auto pty = getLlvmPtrType(resultTy.getContext());
-    auto p = rewriter.create<mlir::LLVM::GEPOp>(
+    auto p = mlir::LLVM::GEPOp::create(rewriter,
         loc, pty, boxTy.llvm, box,
         llvm::ArrayRef<mlir::LLVM::GEPArg>{0, boxValue});
     auto fldTy = getBoxEleTy(boxTy.llvm, {boxValue});
-    auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, fldTy, p);
+    auto loadOp = mlir::LLVM::LoadOp::create(rewriter, loc, fldTy, p);
     auto castOp = integerCast(loc, rewriter, resultTy, loadOp);
     attachTBAATag(loadOp, boxTy.fir, nullptr, p);
     return castOp;
   }
-  return rewriter.create<mlir::LLVM::ExtractValueOp>(loc, box, boxValue);
+  return mlir::LLVM::ExtractValueOp::create(rewriter, loc, box, boxValue);
 }
 
 /// Method to construct code sequence to get the triple for dimension `dim`
@@ -147,7 +147,7 @@ mlir::Value ConvertFIRToLLVMPattern::loadDimFieldFromBox(
          "in memory");
   mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0,
                                static_cast<int>(kDimsPosInBox), dim, off);
-  auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, ty, p);
+  auto loadOp = mlir::LLVM::LoadOp::create(rewriter, loc, ty, p);
   attachTBAATag(loadOp, boxTy.fir, nullptr, p);
   return loadOp;
 }
@@ -158,11 +158,11 @@ mlir::Value ConvertFIRToLLVMPattern::getDimFieldFromBox(
   if (mlir::isa<mlir::LLVM::LLVMPointerType>(box.getType())) {
     mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0,
                                  static_cast<int>(kDimsPosInBox), dim, off);
-    auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, ty, p);
+    auto loadOp = mlir::LLVM::LoadOp::create(rewriter, loc, ty, p);
     attachTBAATag(loadOp, boxTy.fir, nullptr, p);
     return loadOp;
   }
-  return rewriter.create<mlir::LLVM::ExtractValueOp>(
+  return mlir::LLVM::ExtractValueOp::create(rewriter,
       loc, box, llvm::ArrayRef<std::int64_t>{kDimsPosInBox, dim, off});
 }
 
@@ -251,9 +251,9 @@ mlir::Value ConvertFIRToLLVMPattern::genBoxAttributeCheck(
       getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox);
   mlir::LLVM::ConstantOp attrMask = genConstantOffset(loc, rewriter, maskValue);
   auto maskRes =
-      rewriter.create<mlir::LLVM::AndOp>(loc, attrTy, attribute, attrMask);
+      mlir::LLVM::AndOp::create(rewriter, loc, attrTy, attribute, attrMask);
   mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0);
-  return rewriter.create<mlir::LLVM::ICmpOp>(loc, mlir::LLVM::ICmpPredicate::ne,
+  return mlir::LLVM::ICmpOp::create(rewriter, loc, mlir::LLVM::ICmpPredicate::ne,
                                              maskRes, c0);
 }
 
@@ -281,9 +281,9 @@ mlir::Value ConvertFIRToLLVMPattern::computeBoxSize(
               firBoxType.getBoxTypeWithNewShape(1)))) &&
          "descriptor layout requires adding padding for dim field");
   mlir::Value sizePerDim = genConstantOffset(loc, rewriter, sizePerDimCst);
-  mlir::Value dimsSize = rewriter.create<mlir::LLVM::MulOp>(
+  mlir::Value dimsSize = mlir::LLVM::MulOp::create(rewriter,
       loc, sizePerDim.getType(), sizePerDim, rank);
-  mlir::Value size = rewriter.create<mlir::LLVM::AddOp>(
+  mlir::Value size = mlir::LLVM::AddOp::create(rewriter,
       loc, scalarBoxSize.getType(), scalarBoxSize, dimsSize);
   return size;
 }
@@ -324,7 +324,7 @@ mlir::Value ConvertFIRToLLVMPattern::genAllocaAndAddrCastWithType(
   unsigned allocaAs = getAllocaAddressSpace(rewriter);
   unsigned programAs = getProgramAddressSpace(rewriter);
 
-  mlir::Value al = rewriter.create<mlir::LLVM::AllocaOp>(
+  mlir::Value al = mlir::LLVM::AllocaOp::create(rewriter,
       loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs), llvmObjectTy,
       size, alignment);
 
@@ -334,7 +334,7 @@ mlir::Value ConvertFIRToLLVMPattern::genAllocaAndAddrCastWithType(
   // the numeric value 5 (private), and the program address space is 0
   // (generic).
   if (allocaAs != programAs) {
-    al = rewriter.create<mlir::LLVM::AddrSpaceCastOp>(
+    al = mlir::LLVM::AddrSpaceCastOp::create(rewriter,
         loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al);
   }
 
diff --git a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
index d2cf85bedd54c..926c3e57d271f 100644
--- a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
+++ b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
@@ -152,20 +152,20 @@ PackArrayConversion::matchAndRewrite(fir::PackArrayOp op,
 
   // For now we have to always check if the box is present.
   auto isPresent =
-      builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), box);
+      fir::IsPresentOp::create(builder, loc, builder.getI1Type(), box);
 
-  fir::IfOp ifOp = builder.create<fir::IfOp>(loc, boxType, isPresent,
+  fir::IfOp ifOp = fir::IfOp::create(builder, loc, boxType, isPresent,
                                              /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   // The box is present.
   auto newBox = genRepackedBox(builder, loc, op);
   if (mlir::failed(newBox))
     return newBox;
-  builder.create<fir::ResultOp>(loc, *newBox);
+  fir::ResultOp::create(builder, loc, *newBox);
 
   // The box is not present. Return original box.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<fir::ResultOp>(loc, box);
+  fir::ResultOp::create(builder, loc, box);
 
   rewriter.replaceOp(op, ifOp.getResult(0));
   return mlir::success();
@@ -199,7 +199,7 @@ mlir::Value PackArrayConversion::allocateTempBuffer(
     // We need to reset the CFI_attribute_allocatable before
     // returning the temporary box to avoid any mishandling
     // of the temporary box in Fortran runtime.
-    base = builder.create<fir::BoxAddrOp>(loc, fir::boxMemRefType(tempBoxType),
+    base = fir::BoxAddrOp::create(builder, loc, fir::boxMemRefType(tempBoxType),
                                           base);
     ptrType = base.getType();
   }
@@ -263,22 +263,22 @@ PackArrayConversion::genRepackedBox(fir::FirOpBuilder &builder,
 
   // Create a temporay iff the original is not contigous and is not empty.
   auto isNotContiguous = builder.genNot(
-      loc, builder.create<fir::IsContiguousBoxOp>(loc, box, op.getInnermost()));
+      loc, fir::IsContiguousBoxOp::create(builder, loc, box, op.getInnermost()));
   auto dataAddr =
-      builder.create<fir::BoxAddrOp>(loc, fir::boxMemRefType(boxType), box);
+      fir::BoxAddrOp::create(builder, loc, fir::boxMemRefType(boxType), box);
   auto isNotEmpty =
-      builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), dataAddr);
+      fir::IsPresentOp::create(builder, loc, builder.getI1Type(), dataAddr);
   auto doPack =
-      builder.create<mlir::arith::AndIOp>(loc, isNotContiguous, isNotEmpty);
+      mlir::arith::AndIOp::create(builder, loc, isNotContiguous, isNotEmpty);
 
   fir::IfOp ifOp =
-      builder.create<fir::IfOp>(loc, boxType, doPack, /*withElseRegion=*/true);
+      fir::IfOp::create(builder, loc, boxType, doPack, /*withElseRegion=*/true);
   // Assume that the repacking is unlikely.
   ifOp.setUnlikelyIfWeights();
 
   // Return original box.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-  builder.create<fir::ResultOp>(loc, box);
+  fir::ResultOp::create(builder, loc, box);
 
   // Create a new box.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
@@ -308,7 +308,7 @@ PackArrayConversion::genRepackedBox(fir::FirOpBuilder &builder,
   if (!op.getNoCopy())
     fir::runtime::genShallowCopy(builder, loc, tempBox, box,
                                  /*resultIsAllocated=*/true);
-  builder.create<fir::ResultOp>(loc, tempBox);
+  fir::ResultOp::create(builder, loc, tempBox);
 
   return ifOp.getResult(0);
 }
@@ -330,15 +330,15 @@ UnpackArrayConversion::matchAndRewrite(fir::UnpackArrayOp op,
 
   // For now we have to always check if the box is present.
   auto isPresent =
-      builder.create<fir::IsPresentOp>(loc, predicateType, originalBox);
+      fir::IsPresentOp::create(builder, loc, predicateType, originalBox);
 
   builder.genIfThen(loc, isPresent).genThen([&]() {
     mlir::Type addrType =
         fir::HeapType::get(fir::extractSequenceType(tempBox.getType()));
     mlir::Value tempAddr =
-        builder.create<fir::BoxAddrOp>(loc, addrType, tempBox);
+        fir::BoxAddrOp::create(builder, loc, addrType, tempBox);
     mlir::Value originalAddr =
-        builder.create<fir::BoxAddrOp>(loc, addrType, originalBox);
+        fir::BoxAddrOp::create(builder, loc, addrType, originalBox);
 
     auto isNotSame = builder.genPtrCompare(loc, mlir::arith::CmpIPredicate::ne,
                                            tempAddr, originalAddr);
@@ -356,7 +356,7 @@ UnpackArrayConversion::matchAndRewrite(fir::UnpackArrayOp op,
           // to the runtime that uses heap memory, even when the stack
           // attribute is set on fir.pack_array.
           if (!op.getStack() || !canAllocateTempOnStack(originalBox))
-            builder.create<fir::FreeMemOp>(loc, tempAddr);
+            fir::FreeMemOp::create(builder, loc, tempAddr);
         })
         .getIfOp()
         .setUnlikelyIfWeights();
diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index b60ac11c7795a..a26f21e5be792 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -103,10 +103,10 @@ class EmboxConversion : public mlir::OpRewritePattern<fir::EmboxOp> {
     auto idxTy = rewriter.getIndexType();
     for (auto ext : seqTy.getShape()) {
       auto iAttr = rewriter.getIndexAttr(ext);
-      auto extVal = rewriter.create<mlir::arith::ConstantOp>(loc, idxTy, iAttr);
+      auto extVal = mlir::arith::ConstantOp::create(rewriter, loc, idxTy, iAttr);
       shapeOpers.push_back(extVal);
     }
-    auto xbox = rewriter.create<fir::cg::XEmboxOp>(
+    auto xbox = fir::cg::XEmboxOp::create(rewriter,
         loc, embox.getType(), embox.getMemref(), shapeOpers, mlir::ValueRange{},
         mlir::ValueRange{}, mlir::ValueRange{}, mlir::ValueRange{},
         embox.getTypeparams(), embox.getSourceBox(),
@@ -143,7 +143,7 @@ class EmboxConversion : public mlir::OpRewritePattern<fir::EmboxOp> {
         substrOpers.assign(sliceOp.getSubstr().begin(),
                            sliceOp.getSubstr().end());
       }
-    auto xbox = rewriter.create<fir::cg::XEmboxOp>(
+    auto xbox = fir::cg::XEmboxOp::create(rewriter,
         loc, embox.getType(), embox.getMemref(), shapeOpers, shiftOpers,
         sliceOpers, subcompOpers, substrOpers, embox.getTypeparams(),
         embox.getSourceBox(), embox.getAllocatorIdxAttr());
@@ -201,7 +201,7 @@ class ReboxConversion : public mlir::OpRewritePattern<fir::ReboxOp> {
                            sliceOp.getSubstr().end());
       }
 
-    auto xRebox = rewriter.create<fir::cg::XReboxOp>(
+    auto xRebox = fir::cg::XReboxOp::create(rewriter,
         loc, rebox.getType(), rebox.getBox(), shapeOpers, shiftOpers,
         sliceOpers, subcompOpers, substrOpers);
     LLVM_DEBUG(llvm::dbgs()
@@ -259,7 +259,7 @@ class ArrayCoorConversion : public mlir::OpRewritePattern<fir::ArrayCoorOp> {
                "Don't allow substring operations on array_coor. This "
                "restriction may be lifted in the future.");
       }
-    auto xArrCoor = rewriter.create<fir::cg::XArrayCoorOp>(
+    auto xArrCoor = fir::cg::XArrayCoorOp::create(rewriter,
         loc, arrCoor.getType(), arrCoor.getMemref(), shapeOpers, shiftOpers,
         sliceOpers, subcompOpers, arrCoor.getIndices(),
         arrCoor.getTypeparams());
@@ -301,7 +301,7 @@ class DeclareOpConversion : public mlir::OpRewritePattern<fir::DeclareOp> {
         return mlir::failure();
     }
     // FIXME: Add FortranAttrs and CudaAttrs
-    auto xDeclOp = rewriter.create<fir::cg::XDeclareOp>(
+    auto xDeclOp = fir::cg::XDeclareOp::create(rewriter,
         loc, declareOp.getType(), declareOp.getMemref(), shapeOpers, shiftOpers,
         declareOp.getTypeparams(), declareOp.getDummyScope(),
         declareOp.getUniqName());
diff --git a/flang/lib/Optimizer/Dialect/FIRDialect.cpp b/flang/lib/Optimizer/Dialect/FIRDialect.cpp
index 4b1dadaac6728..12f1baf39be45 100644
--- a/flang/lib/Optimizer/Dialect/FIRDialect.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRDialect.cpp
@@ -56,7 +56,7 @@ struct FIRInlinerInterface : public mlir::DialectInlinerInterface {
                                              mlir::Value input,
                                              mlir::Type resultType,
                                              mlir::Location loc) const final {
-    return builder.create<fir::ConvertOp>(loc, resultType, input);
+    return fir::ConvertOp::create(builder, loc, resultType, input);
   }
 };
 } // namespace
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index cf20d84cbbcdb..e8ef2f73b05ab 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -782,7 +782,7 @@ struct SimplifyArrayCoorOp : public mlir::OpRewritePattern<fir::ArrayCoorOp> {
       return nullptr;
     mlir::OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPoint(shapeShiftOp);
-    return rewriter.create<fir::ShapeOp>(shapeShiftOp.getLoc(),
+    return fir::ShapeOp::create(rewriter, shapeShiftOp.getLoc(),
                                          shapeShiftOp.getExtents());
   }
 
@@ -797,19 +797,19 @@ struct SimplifyArrayCoorOp : public mlir::OpRewritePattern<fir::ArrayCoorOp> {
       rewriter.setInsertionPoint(op);
       mlir::Location loc = op->getLoc();
       mlir::Type idxTy = rewriter.getIndexType();
-      mlir::Value one = rewriter.create<mlir::arith::ConstantOp>(
+      mlir::Value one = mlir::arith::ConstantOp::create(rewriter,
           loc, idxTy, rewriter.getIndexAttr(1));
       rewriter.restoreInsertionPoint(savedIP);
       auto nsw = mlir::arith::IntegerOverflowFlags::nsw;
 
       IndicesVectorTy shiftedIndices;
       for (auto [lb, idx] : llvm::zip(lbs, indices)) {
-        mlir::Value extLb = rewriter.create<fir::ConvertOp>(loc, idxTy, lb);
-        mlir::Value extIdx = rewriter.create<fir::ConvertOp>(loc, idxTy, idx);
+        mlir::Value extLb = fir::ConvertOp::create(rewriter, loc, idxTy, lb);
+        mlir::Value extIdx = fir::ConvertOp::create(rewriter, loc, idxTy, idx);
         mlir::Value add =
-            rewriter.create<mlir::arith::AddIOp>(loc, extIdx, extLb, nsw);
+            mlir::arith::AddIOp::create(rewriter, loc, extIdx, extLb, nsw);
         mlir::Value sub =
-            rewriter.create<mlir::arith::SubIOp>(loc, add, one, nsw);
+            mlir::arith::SubIOp::create(rewriter, loc, add, one, nsw);
         shiftedIndices.push_back(sub);
       }
 
@@ -4711,7 +4711,7 @@ mlir::func::FuncOp fir::createFuncOp(mlir::Location loc, mlir::ModuleOp module,
     return f;
   mlir::OpBuilder modBuilder(module.getBodyRegion());
   modBuilder.setInsertionPointToEnd(module.getBody());
-  auto result = modBuilder.create<mlir::func::FuncOp>(loc, name, type, attrs);
+  auto result = mlir::func::FuncOp::create(modBuilder, loc, name, type, attrs);
   result.setVisibility(mlir::SymbolTable::Visibility::Private);
   return result;
 }
@@ -4731,7 +4731,7 @@ fir::GlobalOp fir::createGlobalOp(mlir::Location loc, mlir::ModuleOp module,
   if (auto g = module.lookupSymbol<fir::GlobalOp>(name))
     return g;
   mlir::OpBuilder modBuilder(module.getBodyRegion());
-  auto result = modBuilder.create<fir::GlobalOp>(loc, name, type, attrs);
+  auto result = fir::GlobalOp::create(modBuilder, loc, name, type, attrs);
   result.setVisibility(mlir::SymbolTable::Visibility::Private);
   return result;
 }
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp
index cb77aef74acd5..818e6ef1c27a2 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp
@@ -201,13 +201,13 @@ mlir::Value hlfir::genExprShape(mlir::OpBuilder &builder,
   for (std::int64_t extent : expr.getShape()) {
     if (extent == hlfir::ExprType::getUnknownExtent())
       return {};
-    extents.emplace_back(builder.create<mlir::arith::ConstantOp>(
+    extents.emplace_back(mlir::arith::ConstantOp::create(builder,
         loc, indexTy, builder.getIntegerAttr(indexTy, extent)));
   }
 
   fir::ShapeType shapeTy =
       fir::ShapeType::get(builder.getContext(), expr.getRank());
-  fir::ShapeOp shape = builder.create<fir::ShapeOp>(loc, shapeTy, extents);
+  fir::ShapeOp shape = fir::ShapeOp::create(builder, loc, shapeTy, extents);
   return shape.getResult();
 }
 
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index df6ce12215d26..974a1120a5b1d 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -257,7 +257,7 @@ updateDeclaredInputTypeWithVolatility(mlir::Type inputType, mlir::Value memref,
   llvm::TypeSwitch<mlir::Type>(inputType)
       .Case<fir::ReferenceType, fir::BoxType, fir::ClassType>(updateType);
   memref =
-      builder.create<fir::VolatileCastOp>(memref.getLoc(), inputType, memref);
+      fir::VolatileCastOp::create(builder, memref.getLoc(), inputType, memref);
   return std::make_pair(inputType, memref);
 }
 
@@ -1293,7 +1293,7 @@ hlfir::MatmulOp::canonicalize(MatmulOp matmulOp,
     if (isOtherwiseUnused(transposeOp)) {
       mlir::Location loc = matmulOp.getLoc();
       mlir::Type resultTy = matmulOp.getResult().getType();
-      auto matmulTransposeOp = rewriter.create<hlfir::MatmulTransposeOp>(
+      auto matmulTransposeOp = hlfir::MatmulTransposeOp::create(rewriter,
           loc, resultTy, transposeOp.getArray(), matmulOp.getRhs(),
           matmulOp.getFastmathAttr());
 
@@ -2271,7 +2271,7 @@ hlfir::GetLengthOp::canonicalize(GetLengthOp getLength,
     return mlir::failure();
 
   mlir::Type indexTy = rewriter.getIndexType();
-  auto cstLen = rewriter.create<mlir::arith::ConstantOp>(
+  auto cstLen = mlir::arith::ConstantOp::create(rewriter,
       loc, indexTy, mlir::IntegerAttr::get(indexTy, charTy.getLen()));
   rewriter.replaceOp(getLength, cstLen);
   return mlir::success();
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 00ca6731c035b..2ea0a967d7425 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -53,12 +53,12 @@ static mlir::Value packageBufferizedExpr(mlir::Location loc,
   auto tupleType = mlir::TupleType::get(
       builder.getContext(),
       mlir::TypeRange{storage.getType(), mustFree.getType()});
-  auto undef = builder.create<fir::UndefOp>(loc, tupleType);
-  auto insert = builder.create<fir::InsertValueOp>(
+  auto undef = fir::UndefOp::create(builder, loc, tupleType);
+  auto insert = fir::InsertValueOp::create(builder,
       loc, tupleType, undef, mustFree,
       builder.getArrayAttr(
           {builder.getIntegerAttr(builder.getIndexType(), 1)}));
-  return builder.create<fir::InsertValueOp>(
+  return fir::InsertValueOp::create(builder,
       loc, tupleType, insert, storage,
       builder.getArrayAttr(
           {builder.getIntegerAttr(builder.getIndexType(), 0)}));
@@ -117,7 +117,7 @@ createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder,
          llvm::ArrayRef<mlir::Value> typeParams,
          fir::FortranVariableFlagsAttr attrs) -> mlir::Value {
     auto declareOp =
-        builder.create<hlfir::DeclareOp>(loc, memref, name, shape, typeParams,
+        hlfir::DeclareOp::create(builder, loc, memref, name, shape, typeParams,
                                          /*dummy_scope=*/nullptr, attrs);
     return declareOp.getBase();
   };
@@ -137,7 +137,7 @@ static mlir::Value copyInTempAndPackage(mlir::Location loc,
                                         hlfir::Entity source) {
   auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source);
   assert(!temp.isAllocatable() && "expect temp to already be allocated");
-  builder.create<hlfir::AssignOp>(loc, source, temp, /*realloc=*/false,
+  hlfir::AssignOp::create(builder, loc, source, temp, /*realloc=*/false,
                                   /*keep_lhs_length_if_realloc=*/false,
                                   /*temporary_lhs=*/true);
   return packageBufferizedExpr(loc, builder, temp, cleanup);
@@ -210,11 +210,11 @@ struct ApplyOpConversion : public mlir::OpConversionPattern<hlfir::ApplyOp> {
     mlir::Location loc = apply->getLoc();
     hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());
     mlir::Type resultType = hlfir::getVariableElementType(bufferizedExpr);
-    mlir::Value result = rewriter.create<hlfir::DesignateOp>(
+    mlir::Value result = hlfir::DesignateOp::create(rewriter,
         loc, resultType, bufferizedExpr, adaptor.getIndices(),
         adaptor.getTypeparams());
     if (fir::isa_trivial(apply.getType())) {
-      result = rewriter.create<fir::LoadOp>(loc, result);
+      result = fir::LoadOp::create(rewriter, loc, result);
     } else {
       fir::FirOpBuilder builder(rewriter, apply.getOperation());
       result =
@@ -297,12 +297,12 @@ struct SetLengthOpConversion
     llvm::SmallVector<mlir::Value, 1> lenParams{adaptor.getLength()};
     auto alloca = builder.createTemporary(loc, charType, tmpName,
                                           /*shape=*/{}, lenParams);
-    auto declareOp = builder.create<hlfir::DeclareOp>(
+    auto declareOp = hlfir::DeclareOp::create(builder,
         loc, alloca, tmpName, /*shape=*/mlir::Value{}, lenParams,
         /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
     hlfir::Entity temp{declareOp.getBase()};
     // Assign string value to the created temp.
-    builder.create<hlfir::AssignOp>(loc, string, temp,
+    hlfir::AssignOp::create(builder, loc, string, temp,
                                     /*realloc=*/false,
                                     /*keep_lhs_length_if_realloc=*/false,
                                     /*temporary_lhs=*/true);
@@ -445,7 +445,7 @@ struct AssociateOpConversion
              !mlir::isa<fir::BaseBoxType>(assocType)) ||
             ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
               !mlir::isa<fir::BoxCharType>(assocType)))) {
-          sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
+          sourceVar = fir::BoxAddrOp::create(builder, loc, assocType, sourceVar);
         } else {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
         }
@@ -511,7 +511,7 @@ struct AssociateOpConversion
         name = *associate.getUniqName();
       auto temp =
           builder.createTemporary(loc, bufferizedExpr.getType(), name, attrs);
-      builder.create<fir::StoreOp>(loc, bufferizedExpr, temp);
+      fir::StoreOp::create(builder, loc, bufferizedExpr, temp);
       mlir::Value mustFree = builder.createBool(loc, false);
       replaceWith(temp, temp, mustFree);
       return mlir::success();
@@ -544,10 +544,10 @@ static void genBufferDestruction(mlir::Location loc, fir::FirOpBuilder &builder,
       if (mustFinalize && !mlir::isa<fir::BaseBoxType>(var.getType()))
         fir::emitFatalError(loc, "non-finalizable variable");
 
-      addr = builder.create<fir::BoxAddrOp>(loc, heapType, var);
+      addr = fir::BoxAddrOp::create(builder, loc, heapType, var);
     } else {
       if (!mlir::isa<fir::HeapType>(var.getType()))
-        addr = builder.create<fir::ConvertOp>(loc, heapType, var);
+        addr = fir::ConvertOp::create(builder, loc, heapType, var);
 
       if (mustFinalize || deallocComponents) {
         // Embox the raw pointer using proper shape and type params
@@ -577,7 +577,7 @@ static void genBufferDestruction(mlir::Location loc, fir::FirOpBuilder &builder,
       fir::runtime::genDerivedTypeDestroyWithoutFinalization(builder, loc, var);
 
     if (doFree)
-      builder.create<fir::FreeMemOp>(loc, addr);
+      fir::FreeMemOp::create(builder, loc, addr);
   };
   bool deallocComponents = hlfir::mayHaveAllocatableComponent(var.getType());
 
@@ -657,7 +657,7 @@ struct NoReassocOpConversion
     fir::FirOpBuilder builder(rewriter, noreassoc.getOperation());
     mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getVal());
     mlir::Value result =
-        builder.create<hlfir::NoReassocOp>(loc, bufferizedExpr);
+        hlfir::NoReassocOp::create(builder, loc, bufferizedExpr);
 
     if (!fir::isa_trivial(bufferizedExpr.getType())) {
       // NoReassocOp should not be needed on the mustFree path.
@@ -775,10 +775,10 @@ struct ElementalOpConversion
     if (asExpr && asExpr.isMove() && mlir::isa<fir::RecordType>(elemType) &&
         hlfir::mayHaveAllocatableComponent(elemType) &&
         wasCreatedInCurrentBlock(elementValue, builder)) {
-      auto load = builder.create<fir::LoadOp>(loc, asExpr.getVar());
-      builder.create<fir::StoreOp>(loc, load, tempElement);
+      auto load = fir::LoadOp::create(builder, loc, asExpr.getVar());
+      fir::StoreOp::create(builder, loc, load, tempElement);
     } else {
-      builder.create<hlfir::AssignOp>(loc, elementValue, tempElement,
+      hlfir::AssignOp::create(builder, loc, elementValue, tempElement,
                                       /*realloc=*/false,
                                       /*keep_lhs_length_if_realloc=*/false,
                                       /*temporary_lhs=*/true);
@@ -792,7 +792,7 @@ struct ElementalOpConversion
       // loop, this will ensure the buffer properly deallocated.
       if (mlir::isa<hlfir::ExprType>(elementValue.getType()) &&
           wasCreatedInCurrentBlock(elementValue, builder))
-        builder.create<hlfir::DestroyOp>(loc, elementValue);
+        hlfir::DestroyOp::create(builder, loc, elementValue);
     }
     builder.restoreInsertionPoint(insPt);
 
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index 33f687db08f9a..74ad958d9ed8a 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -82,8 +82,8 @@ class AssignOpConversion : public mlir::OpRewritePattern<hlfir::AssignOp> {
           rhsType = fir::LogicalType::get(builder.getContext(), 4);
           rhsVal = builder.createConvert(loc, rhsType, rhsVal);
         }
-        mlir::Value temp = builder.create<fir::AllocaOp>(loc, rhsType);
-        builder.create<fir::StoreOp>(loc, rhsVal, temp);
+        mlir::Value temp = fir::AllocaOp::create(builder, loc, rhsType);
+        fir::StoreOp::create(builder, loc, rhsVal, temp);
         rhsExv = temp;
       }
       return fir::getBase(builder.createBox(loc, rhsExv));
@@ -136,7 +136,7 @@ class AssignOpConversion : public mlir::OpRewritePattern<hlfir::AssignOp> {
       // reallocate and modify "toMutableBox" even if it is taking it by
       // reference.
       auto toMutableBox = builder.createTemporary(loc, to.getType());
-      builder.create<fir::StoreOp>(loc, to, toMutableBox);
+      fir::StoreOp::create(builder, loc, to, toMutableBox);
       if (assignOp.isTemporaryLHS())
         fir::runtime::genAssignTemporary(builder, loc, toMutableBox, from);
       else
@@ -182,7 +182,7 @@ class CopyInOpConversion : public mlir::OpRewritePattern<hlfir::CopyInOp> {
             .genIfOp(loc, {resultAddrType}, isContiguous,
                      /*withElseRegion=*/true)
             .genThen(
-                [&]() { builder.create<fir::ResultOp>(loc, inputVariable); })
+                [&]() { fir::ResultOp::create(builder, loc, inputVariable); })
             .genElse([&] {
               // Create temporary on the heap. Note that the runtime is used and
               // that is desired: since the data copy happens under a runtime
@@ -191,17 +191,17 @@ class CopyInOpConversion : public mlir::OpRewritePattern<hlfir::CopyInOp> {
               // compilation time on these loops.
               mlir::Value temp = copyInOp.getTempBox();
               fir::runtime::genCopyInAssign(builder, loc, temp, inputVariable);
-              mlir::Value copy = builder.create<fir::LoadOp>(loc, temp);
+              mlir::Value copy = fir::LoadOp::create(builder, loc, temp);
               // Get rid of allocatable flag in the fir.box.
               if (mlir::cast<fir::BaseBoxType>(resultAddrType).isAssumedRank())
-                copy = builder.create<fir::ReboxAssumedRankOp>(
+                copy = fir::ReboxAssumedRankOp::create(builder,
                     loc, resultAddrType, copy,
                     fir::LowerBoundModifierAttribute::Preserve);
               else
-                copy = builder.create<fir::ReboxOp>(loc, resultAddrType, copy,
+                copy = fir::ReboxOp::create(builder, loc, resultAddrType, copy,
                                                     /*shape=*/mlir::Value{},
                                                     /*slice=*/mlir::Value{});
-              builder.create<fir::ResultOp>(loc, copy);
+              fir::ResultOp::create(builder, loc, copy);
             })
             .getResults()[0];
     return {addr, builder.genNot(loc, isContiguous)};
@@ -218,13 +218,13 @@ class CopyInOpConversion : public mlir::OpRewritePattern<hlfir::CopyInOp> {
                      /*withElseRegion=*/true)
             .genThen([&]() {
               CopyInResult res = genNonOptionalCopyIn(loc, builder, copyInOp);
-              builder.create<fir::ResultOp>(
+              fir::ResultOp::create(builder,
                   loc, mlir::ValueRange{res.addr, res.wasCopied});
             })
             .genElse([&] {
               mlir::Value absent =
-                  builder.create<fir::AbsentOp>(loc, resultAddrType);
-              builder.create<fir::ResultOp>(
+                  fir::AbsentOp::create(builder, loc, resultAddrType);
+              fir::ResultOp::create(builder,
                   loc, mlir::ValueRange{absent, isPresent});
             })
             .getResults();
@@ -269,12 +269,12 @@ class CopyOutOpConversion : public mlir::OpRewritePattern<hlfir::CopyOutOp> {
             // CopyOutAssign() guarantees that there will be no finalization for
             // the LHS even if it is of a derived type with finalization.
             varMutableBox = builder.createTemporary(loc, var.getType());
-            builder.create<fir::StoreOp>(loc, var, varMutableBox);
+            fir::StoreOp::create(builder, loc, var, varMutableBox);
           } else {
             // Even when there is no need to copy back the data (e.g., the dummy
             // argument was intent(in), CopyOutAssign is called to
             // destroy/deallocate the temporary.
-            varMutableBox = builder.create<fir::ZeroOp>(loc, temp.getType());
+            varMutableBox = fir::ZeroOp::create(builder, loc, temp.getType());
           }
           fir::runtime::genCopyOutAssign(builder, loc, varMutableBox,
                                          copyOutOp.getTemp());
@@ -302,7 +302,7 @@ class DeclareOpConversion : public mlir::OpRewritePattern<hlfir::DeclareOp> {
           fir::FortranVariableFlagsAttr::get(rewriter.getContext(), *attrs);
     if (auto attr = declareOp.getDataAttr())
       dataAttr = cuf::DataAttributeAttr::get(rewriter.getContext(), *attr);
-    auto firDeclareOp = rewriter.create<fir::DeclareOp>(
+    auto firDeclareOp = fir::DeclareOp::create(rewriter,
         loc, memref.getType(), memref, declareOp.getShape(),
         declareOp.getTypeparams(), declareOp.getDummyScope(),
         declareOp.getUniqName(), fortranAttrs, dataAttr);
@@ -328,13 +328,13 @@ class DeclareOpConversion : public mlir::OpRewritePattern<hlfir::DeclareOp> {
                 mlir::dyn_cast<fir::BaseBoxType>(firBase.getType())) {
           // Rebox so that lower bounds and attributes are correct.
           if (baseBoxType.isAssumedRank())
-            return builder.create<fir::ReboxAssumedRankOp>(
+            return fir::ReboxAssumedRankOp::create(builder,
                 loc, hlfirBaseType, firBase,
                 fir::LowerBoundModifierAttribute::SetToOnes);
           if (!fir::extractSequenceType(baseBoxType.getEleTy()) &&
               baseBoxType == hlfirBaseType)
             return firBase;
-          return builder.create<fir::ReboxOp>(loc, hlfirBaseType, firBase,
+          return fir::ReboxOp::create(builder, loc, hlfirBaseType, firBase,
                                               declareOp.getShape(),
                                               /*slice=*/mlir::Value{});
         } else {
@@ -344,7 +344,7 @@ class DeclareOpConversion : public mlir::OpRewritePattern<hlfir::DeclareOp> {
           if (!maybeCharType || maybeCharType.hasDynamicLen())
             typeParams.append(declareOp.getTypeparams().begin(),
                               declareOp.getTypeparams().end());
-          return builder.create<fir::EmboxOp>(
+          return fir::EmboxOp::create(builder,
               loc, hlfirBaseType, firBase, declareOp.getShape(),
               /*slice=*/mlir::Value{}, typeParams);
         }
@@ -368,24 +368,24 @@ class DeclareOpConversion : public mlir::OpRewritePattern<hlfir::DeclareOp> {
         // the entity is absent so that later fir.is_present on the hlfir base
         // are valid.
         mlir::Value isPresent =
-            builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), firBase);
+            fir::IsPresentOp::create(builder, loc, builder.getI1Type(), firBase);
         hlfirBase = builder
                         .genIfOp(loc, {hlfirBaseType}, isPresent,
                                  /*withElseRegion=*/true)
                         .genThen([&] {
-                          builder.create<fir::ResultOp>(loc, genHlfirBox());
+                          fir::ResultOp::create(builder, loc, genHlfirBox());
                         })
                         .genElse([&]() {
                           mlir::Value absent =
-                              builder.create<fir::AbsentOp>(loc, hlfirBaseType);
-                          builder.create<fir::ResultOp>(loc, absent);
+                              fir::AbsentOp::create(builder, loc, hlfirBaseType);
+                          fir::ResultOp::create(builder, loc, absent);
                         })
                         .getResults()[0];
       }
     } else if (mlir::isa<fir::BoxCharType>(hlfirBaseType)) {
       assert(declareOp.getTypeparams().size() == 1 &&
              "must contain character length");
-      hlfirBase = rewriter.create<fir::EmboxCharOp>(
+      hlfirBase = fir::EmboxCharOp::create(rewriter,
           loc, hlfirBaseType, firBase, declareOp.getTypeparams()[0]);
     } else {
       if (hlfirBaseType != firBase.getType()) {
@@ -426,7 +426,7 @@ class DesignateOpConversion
     const bool isVolatile = fir::isa_volatile_type(originalDesignateType);
     mlir::Type arrayCoorType = fir::ReferenceType::get(baseEleTy, isVolatile);
 
-    base = builder.create<fir::ArrayCoorOp>(
+    base = fir::ArrayCoorOp::create(builder,
         loc, arrayCoorType, base, shape,
         /*slice=*/mlir::Value{}, firstElementIndices, firBaseTypeParameters);
     return base;
@@ -461,7 +461,7 @@ class DesignateOpConversion
       mlir::Type baseRecordType = baseEntity.getFortranElementType();
       if (fir::isRecordWithTypeParameters(baseRecordType))
         TODO(loc, "hlfir.designate with a parametrized derived type base");
-      fieldIndex = builder.create<fir::FieldIndexOp>(
+      fieldIndex = fir::FieldIndexOp::create(builder,
           loc, fir::FieldType::get(builder.getContext()),
           designate.getComponent().value(), baseRecordType,
           /*typeParams=*/mlir::ValueRange{});
@@ -475,7 +475,7 @@ class DesignateOpConversion
                 designate.getComponent().value());
         mlir::Type coorTy = fir::ReferenceType::get(componentType, isVolatile);
 
-        base = builder.create<fir::CoordinateOp>(loc, coorTy, base, fieldIndex);
+        base = fir::CoordinateOp::create(builder, loc, coorTy, base, fieldIndex);
         if (mlir::isa<fir::BaseBoxType>(componentType)) {
           auto variableInterface = mlir::cast<fir::FortranVariableOpInterface>(
               designate.getOperation());
@@ -532,12 +532,12 @@ class DesignateOpConversion
             mlir::Value iIdx = builder.createConvert(loc, idxTy, i);
             mlir::Value lbIdx = builder.createConvert(loc, idxTy, lb);
             sliceFields.emplace_back(
-                builder.create<mlir::arith::SubIOp>(loc, iIdx, lbIdx));
+                mlir::arith::SubIOp::create(builder, loc, iIdx, lbIdx));
           }
         }
       } else if (!isScalarDesignator) {
         // Otherwise, this is an array section with triplets.
-        auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+        auto undef = fir::UndefOp::create(builder, loc, idxTy);
         unsigned i = 0;
         for (auto isTriplet : designate.getIsTriplet()) {
           triples.push_back(subscripts[i++]);
@@ -558,7 +558,7 @@ class DesignateOpConversion
         mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
         substring[0] = builder.createConvert(loc, idxTy, substring[0]);
         substring[0] =
-            builder.create<mlir::arith::SubIOp>(loc, substring[0], one);
+            mlir::arith::SubIOp::create(builder, loc, substring[0], one);
         substring.push_back(designate.getTypeparams()[0]);
       }
       if (designate.getComplexPart()) {
@@ -570,7 +570,7 @@ class DesignateOpConversion
       mlir::Value slice;
       if (!triples.empty())
         slice =
-            builder.create<fir::SliceOp>(loc, triples, sliceFields, substring);
+            fir::SliceOp::create(builder, loc, triples, sliceFields, substring);
       else
         assert(sliceFields.empty() && substring.empty());
 
@@ -580,10 +580,10 @@ class DesignateOpConversion
       mlir::Value resultBox;
       if (mlir::isa<fir::BaseBoxType>(base.getType())) {
         resultBox =
-            builder.create<fir::ReboxOp>(loc, resultType, base, shape, slice);
+            fir::ReboxOp::create(builder, loc, resultType, base, shape, slice);
       } else {
         resultBox =
-            builder.create<fir::EmboxOp>(loc, resultType, base, shape, slice,
+            fir::EmboxOp::create(builder, loc, resultType, base, shape, slice,
                                          firBaseTypeParameters, sourceBox);
       }
       rewriter.replaceOp(designate, resultBox);
@@ -623,14 +623,14 @@ class DesignateOpConversion
                                                  *designate.getComplexPart());
       auto coorTy = fir::ReferenceType::get(resultEleTy, isVolatile);
 
-      base = builder.create<fir::CoordinateOp>(loc, coorTy, base, index);
+      base = fir::CoordinateOp::create(builder, loc, coorTy, base, index);
     }
 
     // Cast/embox the computed scalar address if needed.
     if (mlir::isa<fir::BoxCharType>(designateResultType)) {
       assert(designate.getTypeparams().size() == 1 &&
              "must have character length");
-      auto emboxChar = builder.create<fir::EmboxCharOp>(
+      auto emboxChar = fir::EmboxCharOp::create(builder,
           loc, designateResultType, base, designate.getTypeparams()[0]);
 
       rewriter.replaceOp(designate, emboxChar.getResult());
@@ -679,7 +679,7 @@ class ParentComponentOpConversion
       // and the output will be monomorphic, the base address can be extracted
       // from the fir.class.
       if (mlir::isa<fir::BaseBoxType>(baseAddr.getType()))
-        baseAddr = rewriter.create<fir::BoxAddrOp>(loc, baseAddr);
+        baseAddr = fir::BoxAddrOp::create(rewriter, loc, baseAddr);
       rewriter.replaceOpWithNewOp<fir::ConvertOp>(parentComponent, resultType,
                                                   baseAddr);
       return mlir::success();
@@ -697,7 +697,7 @@ class ParentComponentOpConversion
           fir::BoxType::get(base.getElementOrSequenceType());
       assert(!base.hasLengthParameters() &&
              "base must be a box if it has any type parameters");
-      baseAddr = rewriter.create<fir::EmboxOp>(
+      baseAddr = fir::EmboxOp::create(rewriter,
           loc, baseBoxType, baseAddr, parentComponent.getShape(),
           /*slice=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{});
     }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
index 6c4a07be52a48..86d39749df93d 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
@@ -126,7 +126,7 @@ class InlineHLFIRAssignConversion
     rhsArrayElement = hlfir::loadTrivialScalar(loc, builder, rhsArrayElement);
     auto lhsArrayElement =
         hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-    builder.create<hlfir::AssignOp>(loc, rhsArrayElement, lhsArrayElement);
+    hlfir::AssignOp::create(builder, loc, rhsArrayElement, lhsArrayElement);
     rewriter.eraseOp(assign);
     return mlir::success();
   }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp
index 7e8acc515ee26..8c4ce934b809a 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp
@@ -82,7 +82,7 @@ InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
       hlfir::getFortranElementOrSequenceType(inputVariable.getType());
   fir::BoxType resultBoxType = fir::BoxType::get(sequenceType);
   mlir::Value isContiguous =
-      builder.create<fir::IsContiguousBoxOp>(loc, inputVariable);
+      fir::IsContiguousBoxOp::create(builder, loc, inputVariable);
   mlir::Operation::result_range results =
       builder
           .genIfOp(loc, {resultBoxType, builder.getI1Type()}, isContiguous,
@@ -90,11 +90,11 @@ InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
           .genThen([&]() {
             mlir::Value result = inputVariable;
             if (fir::isPointerType(inputVariable.getType())) {
-              result = builder.create<fir::ReboxOp>(
+              result = fir::ReboxOp::create(builder,
                   loc, resultBoxType, inputVariable, mlir::Value{},
                   mlir::Value{});
             }
-            builder.create<fir::ResultOp>(
+            fir::ResultOp::create(builder,
                 loc, mlir::ValueRange{result, builder.createBool(loc, false)});
           })
           .genElse([&] {
@@ -106,7 +106,7 @@ InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
             mlir::Value alloc = builder.createHeapTemporary(
                 loc, sequenceType, tmpName, extents, lenParams);
 
-            auto declareOp = builder.create<hlfir::DeclareOp>(
+            auto declareOp = hlfir::DeclareOp::create(builder,
                 loc, alloc, tmpName, shape, lenParams,
                 /*dummy_scope=*/nullptr);
             hlfir::Entity temp{declareOp.getBase()};
@@ -120,7 +120,7 @@ InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
             elem = hlfir::loadTrivialScalar(loc, builder, elem);
             hlfir::Entity tempElem = hlfir::getElementAt(
                 loc, builder, temp, loopNest.oneBasedIndices);
-            builder.create<hlfir::AssignOp>(loc, elem, tempElem);
+            hlfir::AssignOp::create(builder, loc, elem, tempElem);
             builder.setInsertionPointAfter(loopNest.outerOp);
 
             mlir::Value result;
@@ -132,11 +132,11 @@ InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
               fir::ReferenceType refTy =
                   fir::ReferenceType::get(temp.getElementOrSequenceType());
               mlir::Value refVal = builder.createConvert(loc, refTy, temp);
-              result = builder.create<fir::EmboxOp>(loc, resultBoxType, refVal,
+              result = fir::EmboxOp::create(builder, loc, resultBoxType, refVal,
                                                     shape);
             }
 
-            builder.create<fir::ResultOp>(
+            fir::ResultOp::create(builder,
                 loc, mlir::ValueRange{result, builder.createBool(loc, true)});
           })
           .getResults();
@@ -145,8 +145,8 @@ InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
   mlir::OpResult needsCleanup = results[1];
 
   // Prepare the corresponding copyOut to free the temporary if it is required
-  auto alloca = builder.create<fir::AllocaOp>(loc, resultBox.getType());
-  auto store = builder.create<fir::StoreOp>(loc, resultBox, alloca);
+  auto alloca = fir::AllocaOp::create(builder, loc, resultBox.getType());
+  auto store = fir::StoreOp::create(builder, loc, resultBox, alloca);
   rewriter.startOpModification(copyOut);
   copyOut->setOperand(0, store.getMemref());
   copyOut->setOperand(1, needsCleanup);
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
index 31e5bc1193e22..1c7c8624031c3 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
@@ -169,7 +169,7 @@ class HlfirIntrinsicConversion : public mlir::OpRewritePattern<OP> {
     }
 
     if (resultEntity->isVariable()) {
-      hlfir::AsExprOp asExpr = builder.create<hlfir::AsExprOp>(
+      hlfir::AsExprOp asExpr = hlfir::AsExprOp::create(builder,
           loc, *resultEntity, builder.createBool(loc, mustBeFreed));
       resultEntity = hlfir::EntityWithAttributes{asExpr.getResult()};
     }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index c5cf01ed98357..3e2b0be34a83f 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -377,7 +377,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallOp forallOp) {
   } else {
     step = generateYieldedScalarValue(forallOp.getStepRegion(), idxTy);
   }
-  auto doLoop = builder.create<fir::DoLoopOp>(loc, lb, ub, step);
+  auto doLoop = fir::DoLoopOp::create(builder, loc, lb, ub, step);
   builder.setInsertionPointToStart(doLoop.getBody());
   mlir::Value oldIndex = forallOp.getForallIndexValue();
   mlir::Value newIndex =
@@ -405,7 +405,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
   mlir::Location loc = forallMaskOp.getLoc();
   mlir::Value mask = generateYieldedScalarValue(forallMaskOp.getMaskRegion(),
                                                 builder.getI1Type());
-  auto ifOp = builder.create<fir::IfOp>(loc, mlir::TypeRange{}, mask, false);
+  auto ifOp = fir::IfOp::create(builder, loc, mlir::TypeRange{}, mask, false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   constructStack.push_back(ifOp);
 }
@@ -431,11 +431,11 @@ convertToMoldType(mlir::Location loc, fir::FirOpBuilder &builder,
   if (input.isVariable() && mold.isValue()) {
     if (fir::isa_trivial(mold.getType())) {
       // fir.ref<T> to T.
-      mlir::Value load = builder.create<fir::LoadOp>(loc, input);
+      mlir::Value load = fir::LoadOp::create(builder, loc, input);
       return hlfir::Entity{builder.createConvert(loc, mold.getType(), load)};
     }
     // fir.ref<T> to hlfir.expr<T>.
-    mlir::Value asExpr = builder.create<hlfir::AsExprOp>(loc, input);
+    mlir::Value asExpr = hlfir::AsExprOp::create(builder, loc, input);
     if (asExpr.getType() != mold.getType())
       TODO(loc, "hlfir.expr conversion");
     cleanups.emplace_back([=]() { b->create<hlfir::DestroyOp>(loc, asExpr); });
@@ -517,7 +517,7 @@ void OrderedAssignmentRewriter::pre(hlfir::RegionAssignOp regionAssignOp) {
   } else {
     // TODO: preserve allocatable assignment aspects for forall once
     // they are conveyed in hlfir.region_assign.
-    builder.create<hlfir::AssignOp>(loc, rhsEntity, lhsEntity);
+    hlfir::AssignOp::create(builder, loc, rhsEntity, lhsEntity);
   }
   generateCleanupIfAny(loweredLhs.elementalCleanup);
   if (loweredLhs.vectorSubscriptLoopNest)
@@ -530,7 +530,7 @@ void OrderedAssignmentRewriter::generateMaskIfOp(mlir::Value cdt) {
   mlir::Location loc = cdt.getLoc();
   cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
   cdt = builder.createConvert(loc, builder.getI1Type(), cdt);
-  auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), mlir::TypeRange{}, cdt,
+  auto ifOp = fir::IfOp::create(builder, cdt.getLoc(), mlir::TypeRange{}, cdt,
                                         /*withElseRegion=*/false);
   constructStack.push_back(ifOp.getOperation());
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
@@ -604,7 +604,7 @@ void OrderedAssignmentRewriter::enterElsewhere(hlfir::ElseWhereOp elseWhereOp) {
   if (ifOp.getElseRegion().empty()) {
     mlir::Location loc = elseWhereOp.getLoc();
     builder.createBlock(&ifOp.getElseRegion());
-    auto end = builder.create<fir::ResultOp>(loc);
+    auto end = fir::ResultOp::create(builder, loc);
     builder.setInsertionPoint(end);
   } else {
     builder.setInsertionPoint(&ifOp.getElseRegion().back().back());
@@ -1150,7 +1150,7 @@ computeLoopNestIterationNumber(mlir::Location loc, fir::FirOpBuilder &builder,
     if (!loopExtent)
       loopExtent = extent;
     else
-      loopExtent = builder.create<mlir::arith::MulIOp>(loc, loopExtent, extent);
+      loopExtent = mlir::arith::MulIOp::create(builder, loc, loopExtent, extent);
   }
   assert(loopExtent && "loopNest must not be empty");
   return loopExtent;
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index abcbf145410ab..14dd36b2cb741 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -727,7 +727,7 @@ llvm::LogicalResult ElementalAssignBufferization::matchAndRewrite(
   // Assign the element value to the array element for this iteration.
   auto arrayElement =
       hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-  builder.create<hlfir::AssignOp>(
+  hlfir::AssignOp::create(builder,
       loc, elementValue, arrayElement, /*realloc=*/false,
       /*keep_lhs_length_if_realloc=*/false, match->assign.getTemporaryLhs());
 
@@ -793,7 +793,7 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
     // optimized.
     mlir::Value n = extents[0];
     for (size_t i = 1; i < extents.size(); ++i)
-      n = builder.create<mlir::arith::MulIOp>(loc, n, extents[i]);
+      n = mlir::arith::MulIOp::create(builder, loc, n, extents[i]);
     llvm::SmallVector<mlir::Value> flatExtents = {n};
 
     mlir::Type flatArrayType;
@@ -801,7 +801,7 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
     if (mlir::isa<fir::BoxType>(lhs.getType())) {
       shape = builder.genShape(loc, flatExtents);
       flatArrayType = fir::BoxType::get(fir::SequenceType::get(eleTy, 1));
-      flatArray = builder.create<fir::ReboxOp>(loc, flatArrayType, flatArray,
+      flatArray = fir::ReboxOp::create(builder, loc, flatArrayType, flatArray,
                                                shape, /*slice=*/mlir::Value{});
     } else {
       // Array references must have fixed shape, when used in assignments.
@@ -822,9 +822,9 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
     builder.setInsertionPointToStart(loopNest.body);
 
     mlir::Value arrayElement =
-        builder.create<hlfir::DesignateOp>(loc, fir::ReferenceType::get(eleTy),
+        hlfir::DesignateOp::create(builder, loc, fir::ReferenceType::get(eleTy),
                                            flatArray, loopNest.oneBasedIndices);
-    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+    hlfir::AssignOp::create(builder, loc, rhs, arrayElement);
   } else {
     hlfir::LoopNest loopNest =
         hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
@@ -832,7 +832,7 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
     builder.setInsertionPointToStart(loopNest.body);
     auto arrayElement =
         hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+    hlfir::AssignOp::create(builder, loc, rhs, arrayElement);
   }
 
   rewriter.eraseOp(assign);
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index 79582390d1294..0b461d6ea73ef 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -60,22 +60,22 @@ class ProductFactory {
     v2 = castToProductType(v2, resultType);
     mlir::Value result;
     if (mlir::isa<mlir::FloatType>(resultType)) {
-      result = builder.create<mlir::arith::AddFOp>(
-          loc, acc, builder.create<mlir::arith::MulFOp>(loc, v1, v2));
+      result = mlir::arith::AddFOp::create(builder,
+          loc, acc, mlir::arith::MulFOp::create(builder, loc, v1, v2));
     } else if (mlir::isa<mlir::ComplexType>(resultType)) {
       if constexpr (CONJ)
         result = fir::IntrinsicLibrary{builder, loc}.genConjg(resultType, v1);
       else
         result = v1;
 
-      result = builder.create<fir::AddcOp>(
-          loc, acc, builder.create<fir::MulcOp>(loc, result, v2));
+      result = fir::AddcOp::create(builder,
+          loc, acc, fir::MulcOp::create(builder, loc, result, v2));
     } else if (mlir::isa<mlir::IntegerType>(resultType)) {
-      result = builder.create<mlir::arith::AddIOp>(
-          loc, acc, builder.create<mlir::arith::MulIOp>(loc, v1, v2));
+      result = mlir::arith::AddIOp::create(builder,
+          loc, acc, mlir::arith::MulIOp::create(builder, loc, v1, v2));
     } else if (mlir::isa<fir::LogicalType>(resultType)) {
-      result = builder.create<mlir::arith::OrIOp>(
-          loc, acc, builder.create<mlir::arith::AndIOp>(loc, v1, v2));
+      result = mlir::arith::OrIOp::create(builder,
+          loc, acc, mlir::arith::AndIOp::create(builder, loc, v1, v2));
     } else {
       llvm_unreachable("unsupported type");
     }
@@ -168,7 +168,7 @@ class TransposeAsElementalConversion
 
     // transpose indices
     assert(inExtents.size() == 2 && "checked in TransposeOp::validate");
-    return builder.create<fir::ShapeOp>(
+    return fir::ShapeOp::create(builder,
         loc, mlir::ValueRange{inExtents[1], inExtents[0]});
   }
 };
@@ -385,19 +385,19 @@ genMinMaxComparison(mlir::Location loc, fir::FirOpBuilder &builder,
     // is not NaN. A OGL/OLT condition will usually work for this unless all
     // the values are Nan or Inf. This follows the same logic as
     // NumericCompare for Minloc/Maxloc in extrema.cpp.
-    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
+    mlir::Value cmp = mlir::arith::CmpFOp::create(builder,
         loc,
         IS_MAX ? mlir::arith::CmpFPredicate::OGT
                : mlir::arith::CmpFPredicate::OLT,
         elem, reduction);
-    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+    mlir::Value cmpNan = mlir::arith::CmpFOp::create(builder,
         loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+    mlir::Value cmpNan2 = mlir::arith::CmpFOp::create(builder,
         loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+    cmpNan = mlir::arith::AndIOp::create(builder, loc, cmpNan, cmpNan2);
+    return mlir::arith::OrIOp::create(builder, loc, cmp, cmpNan);
   } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
-    return builder.create<mlir::arith::CmpIOp>(
+    return mlir::arith::CmpIOp::create(builder,
         loc,
         IS_MAX ? mlir::arith::CmpIPredicate::sgt
                : mlir::arith::CmpIPredicate::slt,
@@ -415,9 +415,9 @@ genIsNotEmptyArrayExtents(mlir::Location loc, fir::FirOpBuilder &builder,
   for (auto extent : extents) {
     mlir::Value zero =
         fir::factory::createZeroValue(builder, loc, extent.getType());
-    mlir::Value cmp = builder.create<mlir::arith::CmpIOp>(
+    mlir::Value cmp = mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::ne, extent, zero);
-    isNotEmpty = builder.create<mlir::arith::AndIOp>(loc, isNotEmpty, cmp);
+    isNotEmpty = mlir::arith::AndIOp::create(builder, loc, isNotEmpty, cmp);
   }
   return isNotEmpty;
 }
@@ -581,7 +581,7 @@ MinMaxlocAsElementalConverter<T>::genReductionInitValues(
     llvm::SmallVector<mlir::Type> ifTypes(getNumCoors(),
                                           getResultElementType());
     ifTypes.push_back(getSourceElementType());
-    ifOp = builder.create<fir::IfOp>(loc, ifTypes, isNotEmpty,
+    ifOp = fir::IfOp::create(builder, loc, ifTypes, isNotEmpty,
                                      /*withElseRegion=*/true);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value one =
@@ -590,7 +590,7 @@ MinMaxlocAsElementalConverter<T>::genReductionInitValues(
     mlir::Value minMaxFirst =
         hlfir::loadElementAt(loc, builder, hlfir::Entity{getSource()}, indices);
     results.push_back(minMaxFirst);
-    builder.create<fir::ResultOp>(loc, results);
+    fir::ResultOp::create(builder, loc, results);
 
     // In the 'else' block use default init values.
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -607,7 +607,7 @@ MinMaxlocAsElementalConverter<T>::genReductionInitValues(
   result.push_back(minMaxInit);
 
   if (ifOp) {
-    builder.create<fir::ResultOp>(loc, result);
+    fir::ResultOp::create(builder, loc, result);
     builder.setInsertionPointAfter(ifOp);
     result = ifOp.getResults();
   } else if (useIsFirst()) {
@@ -635,7 +635,7 @@ MinMaxlocAsElementalConverter<T>::reduceOneElement(
     // If isFirst is true, then do the reduction update regardless
     // of the FP comparison.
     cmp =
-        builder.create<mlir::arith::OrIOp>(loc, cmp, getIsFirst(currentValue));
+        mlir::arith::OrIOp::create(builder, loc, cmp, getIsFirst(currentValue));
   }
 
   llvm::SmallVector<mlir::Value> newIndices;
@@ -654,11 +654,11 @@ MinMaxlocAsElementalConverter<T>::reduceOneElement(
     mlir::Value newCoor = builder.createConvert(
         loc, currentCoor.getType(), oneBasedIndices[coorIdx + dim - 1]);
     mlir::Value update =
-        builder.create<mlir::arith::SelectOp>(loc, cmp, newCoor, currentCoor);
+        mlir::arith::SelectOp::create(builder, loc, cmp, newCoor, currentCoor);
     newIndices.push_back(update);
   }
 
-  mlir::Value newMinMax = builder.create<mlir::arith::SelectOp>(
+  mlir::Value newMinMax = mlir::arith::SelectOp::create(builder,
       loc, cmp, elementValue, getCurrentMinMax(currentValue));
   newIndices.push_back(newMinMax);
 
@@ -711,9 +711,9 @@ hlfir::Entity MinMaxlocAsElementalConverter<T>::genFinalResult(
     mlir::Value idx = builder.createIntegerConstant(loc, indexType, i + 1);
     mlir::Value resultElement =
         hlfir::getElementAt(loc, builder, hlfir::Entity{tempArray}, {idx});
-    builder.create<hlfir::AssignOp>(loc, coor, resultElement);
+    hlfir::AssignOp::create(builder, loc, coor, resultElement);
   }
-  mlir::Value tempExpr = builder.create<hlfir::AsExprOp>(
+  mlir::Value tempExpr = hlfir::AsExprOp::create(builder,
       loc, tempArray, builder.createBool(loc, false));
   return hlfir::Entity{tempExpr};
 }
@@ -792,9 +792,9 @@ class MinMaxvalAsElementalConverter
     mlir::Value cmp =
         genMinMaxComparison<isMax>(loc, builder, elementValue, currentMinMax);
     if (useIsFirst())
-      cmp = builder.create<mlir::arith::OrIOp>(loc, cmp,
+      cmp = mlir::arith::OrIOp::create(builder, loc, cmp,
                                                getIsFirst(currentValue));
-    mlir::Value newMinMax = builder.create<mlir::arith::SelectOp>(
+    mlir::Value newMinMax = mlir::arith::SelectOp::create(builder,
         loc, cmp, elementValue, currentMinMax);
     result.push_back(newMinMax);
     if (useIsFirst())
@@ -868,12 +868,12 @@ MinMaxvalAsElementalConverter<T>::genReductionInitValues(
         this->getSourceRank(), oneBasedIndices);
 
     ifOp =
-        builder.create<fir::IfOp>(loc, this->getResultElementType(), isNotEmpty,
+        fir::IfOp::create(builder, loc, this->getResultElementType(), isNotEmpty,
                                   /*withElseRegion=*/true);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value minMaxFirst = hlfir::loadElementAt(
         loc, builder, hlfir::Entity{this->getSource()}, indices);
-    builder.create<fir::ResultOp>(loc, minMaxFirst);
+    fir::ResultOp::create(builder, loc, minMaxFirst);
 
     // In the 'else' block use default init values.
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -884,7 +884,7 @@ MinMaxvalAsElementalConverter<T>::genReductionInitValues(
   result.push_back(init);
 
   if (ifOp) {
-    builder.create<fir::ResultOp>(loc, result);
+    fir::ResultOp::create(builder, loc, result);
     builder.setInsertionPointAfter(ifOp);
     result = ifOp.getResults();
   } else if (useIsFirst()) {
@@ -992,9 +992,9 @@ class AllAnyAsElementalConverter
     mlir::Value mask =
         builder.createConvert(loc, builder.getI1Type(), elementValue);
     if constexpr (isAll)
-      return {builder.create<mlir::arith::AndIOp>(loc, mask, currentValue[0])};
+      return {mlir::arith::AndIOp::create(builder, loc, mask, currentValue[0])};
     else
-      return {builder.create<mlir::arith::OrIOp>(loc, mask, currentValue[0])};
+      return {mlir::arith::OrIOp::create(builder, loc, mask, currentValue[0])};
   }
 
   virtual hlfir::Entity genFinalResult(
@@ -1034,8 +1034,8 @@ class CountAsElementalConverter
     mlir::Value one =
         builder.createIntegerConstant(loc, getResultElementType(), 1);
     mlir::Value add1 =
-        builder.create<mlir::arith::AddIOp>(loc, currentValue[0], one);
-    return {builder.create<mlir::arith::SelectOp>(loc, cond, add1,
+        mlir::arith::AddIOp::create(builder, loc, currentValue[0], one);
+    return {mlir::arith::SelectOp::create(builder, loc, cond, add1,
                                                   currentValue[0])};
   }
 };
@@ -1068,7 +1068,7 @@ mlir::LogicalResult ReductionAsElementalConverter::convert() {
       // MASK represented by a box might be dynamically optional,
       // so we have to check for its presence before accessing it.
       isPresentPred =
-          builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
+          fir::IsPresentOp::create(builder, loc, builder.getI1Type(), mask);
     }
 
     if (hlfir::Entity{mask}.isScalar())
@@ -1120,19 +1120,19 @@ mlir::LogicalResult ReductionAsElementalConverter::convert() {
           maskValue = genMaskValue(mask, isPresentPred, indices);
         }
         mlir::Value isUnmasked =
-            builder.create<fir::ConvertOp>(loc, builder.getI1Type(), maskValue);
-        ifOp = builder.create<fir::IfOp>(loc, reductionTypes, isUnmasked,
+            fir::ConvertOp::create(builder, loc, builder.getI1Type(), maskValue);
+        ifOp = fir::IfOp::create(builder, loc, reductionTypes, isUnmasked,
                                          /*withElseRegion=*/true);
         // In the 'else' block return the current reduction value.
         builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-        builder.create<fir::ResultOp>(loc, reductionValues);
+        fir::ResultOp::create(builder, loc, reductionValues);
 
         // In the 'then' block do the actual addition.
         builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
       }
       reductionValues = reduceOneElement(reductionValues, array, indices);
       if (ifOp) {
-        builder.create<fir::ResultOp>(loc, reductionValues);
+        fir::ResultOp::create(builder, loc, reductionValues);
         builder.setInsertionPointAfter(ifOp);
         reductionValues = ifOp.getResults();
       }
@@ -1177,7 +1177,7 @@ ReductionAsElementalConverter::genResultShapeForPartialReduction(
 
   mlir::Value dimExtent = inExtents[dimVal - 1];
   inExtents.erase(inExtents.begin() + dimVal - 1);
-  return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
+  return {fir::ShapeOp::create(builder, loc, inExtents), dimExtent};
 }
 
 mlir::Value SumAsElementalConverter::genScalarAdd(mlir::Value value1,
@@ -1185,11 +1185,11 @@ mlir::Value SumAsElementalConverter::genScalarAdd(mlir::Value value1,
   mlir::Type ty = value1.getType();
   assert(ty == value2.getType() && "reduction values' types do not match");
   if (mlir::isa<mlir::FloatType>(ty))
-    return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+    return mlir::arith::AddFOp::create(builder, loc, value1, value2);
   else if (mlir::isa<mlir::ComplexType>(ty))
-    return builder.create<fir::AddcOp>(loc, value1, value2);
+    return fir::AddcOp::create(builder, loc, value1, value2);
   else if (mlir::isa<mlir::IntegerType>(ty))
-    return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+    return mlir::arith::AddIOp::create(builder, loc, value1, value2);
 
   llvm_unreachable("unsupported SUM reduction type");
 }
@@ -1201,14 +1201,14 @@ mlir::Value ReductionAsElementalConverter::genMaskValue(
   mlir::Type maskType =
       hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
   if (isPresentPred) {
-    ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
+    ifOp = fir::IfOp::create(builder, loc, maskType, isPresentPred,
                                      /*withElseRegion=*/true);
 
     // Use 'true', if the mask is not present.
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
     mlir::Value trueValue = builder.createBool(loc, true);
     trueValue = builder.createConvert(loc, maskType, trueValue);
-    builder.create<fir::ResultOp>(loc, trueValue);
+    fir::ResultOp::create(builder, loc, trueValue);
 
     // Load the mask value, if the mask is present.
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
@@ -1219,7 +1219,7 @@ mlir::Value ReductionAsElementalConverter::genMaskValue(
     if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
       // MASK may be a boxed scalar.
       mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
-      mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
+      mask = fir::LoadOp::create(builder, loc, hlfir::Entity{addr});
     } else {
       mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
     }
@@ -1233,7 +1233,7 @@ mlir::Value ReductionAsElementalConverter::genMaskValue(
   if (!isPresentPred)
     return mask;
 
-  builder.create<fir::ResultOp>(loc, mask);
+  fir::ResultOp::create(builder, loc, mask);
   return ifOp.getResult(0);
 }
 
@@ -1335,9 +1335,9 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
     // which extent value we use.
     mlir::Value zero = builder.createIntegerConstant(loc, calcType, 0);
     mlir::Value one = builder.createIntegerConstant(loc, calcType, 1);
-    mlir::Value isZero = builder.create<mlir::arith::CmpIOp>(
+    mlir::Value isZero = mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::eq, extent, zero);
-    extent = builder.create<mlir::arith::SelectOp>(loc, isZero, one, extent);
+    extent = mlir::arith::SelectOp::create(builder, loc, isZero, one, extent);
     shiftVal = fir::IntrinsicLibrary{builder, loc}.genModulo(
         calcType, {shiftVal, extent});
     return builder.createConvert(loc, calcType, shiftVal);
@@ -1408,17 +1408,17 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
       // Such index computation allows for further loop vectorization
       // in LLVM.
       mlir::Value wrapBound =
-          builder.create<mlir::arith::SubIOp>(loc, shiftDimExtent, shiftVal);
+          mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);
       mlir::Value adjustedShiftVal =
-          builder.create<mlir::arith::SubIOp>(loc, shiftVal, shiftDimExtent);
+          mlir::arith::SubIOp::create(builder, loc, shiftVal, shiftDimExtent);
       mlir::Value index =
           builder.createConvert(loc, calcType, inputIndices[dimVal - 1]);
-      mlir::Value wrapCheck = builder.create<mlir::arith::CmpIOp>(
+      mlir::Value wrapCheck = mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::sle, index, wrapBound);
-      mlir::Value actualShift = builder.create<mlir::arith::SelectOp>(
+      mlir::Value actualShift = mlir::arith::SelectOp::create(builder,
           loc, wrapCheck, shiftVal, adjustedShiftVal);
       mlir::Value newIndex =
-          builder.create<mlir::arith::AddIOp>(loc, index, actualShift);
+          mlir::arith::AddIOp::create(builder, loc, index, actualShift);
       newIndex = builder.createConvert(loc, builder.getIndexType(), newIndex);
       indices[dimVal - 1] = newIndex;
       hlfir::Entity element = hlfir::getElementAt(loc, builder, array, indices);
@@ -1485,7 +1485,7 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
     }
 
     hlfir::EvaluateInMemoryOp evalOp =
-        builder.create<hlfir::EvaluateInMemoryOp>(
+        hlfir::EvaluateInMemoryOp::create(builder,
             loc, mlir::cast<hlfir::ExprType>(cshift.getType()), arrayShape);
     builder.setInsertionPointToStart(&evalOp.getBody().front());
 
@@ -1550,13 +1550,13 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
         srcIndices[dimVal - 1] = srcIndex;
         hlfir::Entity srcElementValue =
             hlfir::loadElementAt(loc, builder, srcArray, srcIndices);
-        mlir::Value dstIndex = builder.create<mlir::arith::AddIOp>(
+        mlir::Value dstIndex = mlir::arith::AddIOp::create(builder,
             loc, srcIndex,
-            builder.create<mlir::arith::SubIOp>(loc, shiftDimExtent, shiftVal));
+            mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal));
         dstIndices[dimVal - 1] = dstIndex;
         hlfir::Entity dstElement = hlfir::getElementAt(
             loc, builder, hlfir::Entity{resultArray}, dstIndices);
-        builder.create<hlfir::AssignOp>(loc, srcElementValue, dstElement);
+        hlfir::AssignOp::create(builder, loc, srcElementValue, dstElement);
         return {};
       };
 
@@ -1576,20 +1576,20 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
         assert(index.size() == 1 && "expected single loop");
         mlir::Value dstIndex = builder.createConvert(loc, calcType, index[0]);
         mlir::Value srcIndex =
-            builder.create<mlir::arith::AddIOp>(loc, dstIndex, shiftVal);
+            mlir::arith::AddIOp::create(builder, loc, dstIndex, shiftVal);
         srcIndices[dimVal - 1] = srcIndex;
         hlfir::Entity srcElementValue =
             hlfir::loadElementAt(loc, builder, srcArray, srcIndices);
         dstIndices[dimVal - 1] = dstIndex;
         hlfir::Entity dstElement = hlfir::getElementAt(
             loc, builder, hlfir::Entity{resultArray}, dstIndices);
-        builder.create<hlfir::AssignOp>(loc, srcElementValue, dstElement);
+        hlfir::AssignOp::create(builder, loc, srcElementValue, dstElement);
         return {};
       };
 
       // Generate the second loop.
       mlir::Value bound =
-          builder.create<mlir::arith::SubIOp>(loc, shiftDimExtent, shiftVal);
+          mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);
       hlfir::genLoopNestWithReductions(loc, builder, {bound},
                                        /*reductionInits=*/{}, genAssign2,
                                        /*isUnordered=*/true);
@@ -1625,10 +1625,10 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
       if (dimVal == 1 && mlir::isa<fir::BaseBoxType>(array.getType())) {
         mlir::Type indexType = builder.getIndexType();
         elemSize =
-            builder.create<fir::BoxEleSizeOp>(loc, indexType, array.getBase());
+            fir::BoxEleSizeOp::create(builder, loc, indexType, array.getBase());
         mlir::Value dimIdx =
             builder.createIntegerConstant(loc, indexType, dimVal - 1);
-        auto boxDim = builder.create<fir::BoxDimsOp>(
+        auto boxDim = fir::BoxDimsOp::create(builder,
             loc, indexType, indexType, indexType, array.getBase(), dimIdx);
         stride = boxDim.getByteStride();
       }
@@ -1639,7 +1639,7 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
         return {};
       }
 
-      mlir::Value isContiguous = builder.create<mlir::arith::CmpIOp>(
+      mlir::Value isContiguous = mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::eq, elemSize, stride);
       builder.genIfOp(loc, {}, isContiguous, /*withElseRegion=*/true)
           .genThen([&]() {
@@ -1711,7 +1711,7 @@ class MatmulConversion : public mlir::OpRewritePattern<Op> {
     // from Fortran runtime. The implementation needs to operate
     // with the result array as an in-memory object.
     hlfir::EvaluateInMemoryOp evalOp =
-        builder.create<hlfir::EvaluateInMemoryOp>(
+        hlfir::EvaluateInMemoryOp::create(builder,
             loc, mlir::cast<hlfir::ExprType>(matmul.getType()), resultShape);
     builder.setInsertionPointToStart(&evalOp.getBody().front());
 
@@ -1813,7 +1813,7 @@ class MatmulConversion : public mlir::OpRewritePattern<Op> {
     llvm::SmallVector<mlir::Value> innerProductExtent =
         fir::factory::deduceOptimalExtents({innerProduct1Extent},
                                            {innerProduct2Extent});
-    return {builder.create<fir::ShapeOp>(loc, newExtents),
+    return {fir::ShapeOp::create(builder, loc, newExtents),
             innerProductExtent[0]};
   }
 
@@ -1853,7 +1853,7 @@ class MatmulConversion : public mlir::OpRewritePattern<Op> {
         -> llvm::SmallVector<mlir::Value, 0> {
       hlfir::Entity resultElement =
           hlfir::getElementAt(loc, builder, result, oneBasedIndices);
-      builder.create<hlfir::AssignOp>(loc, initValue, resultElement);
+      hlfir::AssignOp::create(builder, loc, initValue, resultElement);
       return {};
     };
 
@@ -1887,7 +1887,7 @@ class MatmulConversion : public mlir::OpRewritePattern<Op> {
         mlir::Value productValue =
             ProductFactory{loc, builder}.genAccumulateProduct(
                 resultElementValue, lhsElementValue, rhsElementValue);
-        builder.create<hlfir::AssignOp>(loc, productValue, resultElement);
+        hlfir::AssignOp::create(builder, loc, productValue, resultElement);
         return {};
       };
 
@@ -1924,7 +1924,7 @@ class MatmulConversion : public mlir::OpRewritePattern<Op> {
         mlir::Value productValue =
             ProductFactory{loc, builder}.genAccumulateProduct(
                 resultElementValue, lhsElementValue, rhsElementValue);
-        builder.create<hlfir::AssignOp>(loc, productValue, resultElement);
+        hlfir::AssignOp::create(builder, loc, productValue, resultElement);
         return {};
       };
       hlfir::genLoopNestWithReductions(
@@ -1956,7 +1956,7 @@ class MatmulConversion : public mlir::OpRewritePattern<Op> {
         mlir::Value productValue =
             ProductFactory{loc, builder}.genAccumulateProduct(
                 resultElementValue, lhsElementValue, rhsElementValue);
-        builder.create<hlfir::AssignOp>(loc, productValue, resultElement);
+        hlfir::AssignOp::create(builder, loc, productValue, resultElement);
         return {};
       };
       hlfir::genLoopNestWithReductions(
@@ -2172,7 +2172,7 @@ class ReshapeAsElementalConversion
       resultExtents.push_back(hlfir::loadElementAt(
           loc, builder, shape,
           builder.createIntegerConstant(loc, indexType, idx + 1)));
-    auto resultShape = builder.create<fir::ShapeOp>(loc, resultExtents);
+    auto resultShape = fir::ShapeOp::create(builder, loc, resultExtents);
 
     auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
                          mlir::ValueRange inputIndices) -> hlfir::Entity {
@@ -2181,9 +2181,9 @@ class ReshapeAsElementalConversion
       fir::IfOp ifOp;
       if (pad) {
         // PAD is present. Check if this element comes from the PAD array.
-        mlir::Value isInsideArray = builder.create<mlir::arith::CmpIOp>(
+        mlir::Value isInsideArray = mlir::arith::CmpIOp::create(builder,
             loc, mlir::arith::CmpIPredicate::ult, linearIndex, arraySize);
-        ifOp = builder.create<fir::IfOp>(loc, elementType, isInsideArray,
+        ifOp = fir::IfOp::create(builder, loc, elementType, isInsideArray,
                                          /*withElseRegion=*/true);
 
         // In the 'else' block, return an element from the PAD.
@@ -2196,13 +2196,13 @@ class ReshapeAsElementalConversion
         // Subtract the ARRAY size from the zero-based linear index
         // to get the zero-based linear index into PAD.
         mlir::Value padLinearIndex =
-            builder.create<mlir::arith::SubIOp>(loc, linearIndex, arraySize);
+            mlir::arith::SubIOp::create(builder, loc, linearIndex, arraySize);
         llvm::SmallVector<mlir::Value, Fortran::common::maxRank> padIndices =
             delinearizeIndex(loc, builder, padExtents, padLinearIndex,
                              /*wrapAround=*/true);
         mlir::Value padElement =
             hlfir::loadElementAt(loc, builder, hlfir::Entity{pad}, padIndices);
-        builder.create<fir::ResultOp>(loc, padElement);
+        fir::ResultOp::create(builder, loc, padElement);
 
         // In the 'then' block, return an element from the ARRAY.
         builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
@@ -2215,7 +2215,7 @@ class ReshapeAsElementalConversion
           hlfir::loadElementAt(loc, builder, array, arrayIndices);
 
       if (ifOp) {
-        builder.create<fir::ResultOp>(loc, arrayElement);
+        fir::ResultOp::create(builder, loc, arrayElement);
         builder.setInsertionPointAfter(ifOp);
         arrayElement = ifOp.getResult(0);
       }
@@ -2252,11 +2252,11 @@ class ReshapeAsElementalConversion
     mlir::Value linearIndex = zero;
     std::size_t idx = 0;
     for (auto index : llvm::reverse(indices)) {
-      mlir::Value tmp = builder.create<mlir::arith::SubIOp>(
+      mlir::Value tmp = mlir::arith::SubIOp::create(builder,
           loc, builder.createConvert(loc, indexType, index), one);
-      tmp = builder.create<mlir::arith::AddIOp>(loc, linearIndex, tmp);
+      tmp = mlir::arith::AddIOp::create(builder, loc, linearIndex, tmp);
       if (idx + 1 < rank)
-        tmp = builder.create<mlir::arith::MulIOp>(
+        tmp = mlir::arith::MulIOp::create(builder,
             loc, tmp,
             builder.createConvert(loc, indexType, extents[rank - idx - 2]));
 
@@ -2297,12 +2297,12 @@ class ReshapeAsElementalConversion
       mlir::Value currentIndex = linearIndex;
       if (dim != extents.size() - 1 || wrapAround)
         currentIndex =
-            builder.create<mlir::arith::RemUIOp>(loc, linearIndex, extent);
+            mlir::arith::RemUIOp::create(builder, loc, linearIndex, extent);
       // The result of the last division is unused, so it will be DCEd.
       linearIndex =
-          builder.create<mlir::arith::DivUIOp>(loc, linearIndex, extent);
+          mlir::arith::DivUIOp::create(builder, loc, linearIndex, extent);
       indices.push_back(
-          builder.create<mlir::arith::AddIOp>(loc, currentIndex, one));
+          mlir::arith::AddIOp::create(builder, loc, currentIndex, one));
     }
     return indices;
   }
@@ -2314,7 +2314,7 @@ class ReshapeAsElementalConversion
     mlir::Type indexType = builder.getIndexType();
     mlir::Value size = builder.createIntegerConstant(loc, indexType, 1);
     for (auto extent : extents)
-      size = builder.create<mlir::arith::MulIOp>(
+      size = mlir::arith::MulIOp::create(builder,
           loc, size, builder.createConvert(loc, indexType, extent));
     return size;
   }
diff --git a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
index 8b40c643405a9..6f3ddb4be3113 100644
--- a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
@@ -244,14 +244,14 @@ generateSeqTyAccBounds(fir::SequenceType seqType, mlir::Value var,
         mlir::Value cummulativeExtent = one;
         for (auto extent : shapeOp.getExtents()) {
           mlir::Value upperbound =
-              builder.create<mlir::arith::SubIOp>(loc, extent, one);
+              mlir::arith::SubIOp::create(builder, loc, extent, one);
           mlir::Value stride = one;
           if (strideIncludeLowerExtent) {
             stride = cummulativeExtent;
-            cummulativeExtent = builder.create<mlir::arith::MulIOp>(
+            cummulativeExtent = mlir::arith::MulIOp::create(builder,
                 loc, cummulativeExtent, extent);
           }
-          auto accBound = builder.create<mlir::acc::DataBoundsOp>(
+          auto accBound = mlir::acc::DataBoundsOp::create(builder,
               loc, mlir::acc::DataBoundsType::get(builder.getContext()),
               /*lowerbound=*/zero, /*upperbound=*/upperbound,
               /*extent=*/extent, /*stride=*/stride, /*strideInBytes=*/false,
@@ -269,16 +269,16 @@ generateSeqTyAccBounds(fir::SequenceType seqType, mlir::Value var,
           } else {
             mlir::Value extent = val;
             mlir::Value upperbound =
-                builder.create<mlir::arith::SubIOp>(loc, extent, one);
-            upperbound = builder.create<mlir::arith::AddIOp>(loc, lowerbound,
+                mlir::arith::SubIOp::create(builder, loc, extent, one);
+            upperbound = mlir::arith::AddIOp::create(builder, loc, lowerbound,
                                                              upperbound);
             mlir::Value stride = one;
             if (strideIncludeLowerExtent) {
               stride = cummulativeExtent;
-              cummulativeExtent = builder.create<mlir::arith::MulIOp>(
+              cummulativeExtent = mlir::arith::MulIOp::create(builder,
                   loc, cummulativeExtent, extent);
             }
-            auto accBound = builder.create<mlir::acc::DataBoundsOp>(
+            auto accBound = mlir::acc::DataBoundsOp::create(builder,
                 loc, mlir::acc::DataBoundsType::get(builder.getContext()),
                 /*lowerbound=*/zero, /*upperbound=*/upperbound,
                 /*extent=*/extent, /*stride=*/stride, /*strideInBytes=*/false,
@@ -531,9 +531,9 @@ static fir::ShapeOp genShapeOp(mlir::OpBuilder &builder,
   llvm::SmallVector<mlir::Value> extents;
   mlir::Type idxTy = builder.getIndexType();
   for (auto extent : seqTy.getShape())
-    extents.push_back(builder.create<mlir::arith::ConstantOp>(
+    extents.push_back(mlir::arith::ConstantOp::create(builder,
         loc, idxTy, builder.getIntegerAttr(idxTy, extent)));
-  return builder.create<fir::ShapeOp>(loc, extents);
+  return fir::ShapeOp::create(builder, loc, extents);
 }
 
 template <typename Ty>
@@ -549,8 +549,8 @@ mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
   fir::FirOpBuilder firBuilder(builder, mod);
 
   auto getDeclareOpForType = [&](mlir::Type ty) -> hlfir::DeclareOp {
-    auto alloca = firBuilder.create<fir::AllocaOp>(loc, ty);
-    return firBuilder.create<hlfir::DeclareOp>(
+    auto alloca = fir::AllocaOp::create(firBuilder, loc, ty);
+    return hlfir::DeclareOp::create(firBuilder,
         loc, alloca, varName, /*shape=*/nullptr, llvm::ArrayRef<mlir::Value>{},
         /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
   };
@@ -559,7 +559,7 @@ mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
     auto declareOp = getDeclareOpForType(unwrappedTy);
     if (initVal) {
       auto convert = firBuilder.createConvert(loc, unwrappedTy, initVal);
-      firBuilder.create<fir::StoreOp>(loc, convert, declareOp.getBase());
+      fir::StoreOp::create(firBuilder, loc, convert, declareOp.getBase());
     }
     retVal = declareOp.getBase();
   } else if (auto seqTy =
@@ -567,13 +567,13 @@ mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
     if (fir::isa_trivial(seqTy.getEleTy())) {
       mlir::Value shape;
       if (seqTy.hasDynamicExtents()) {
-        shape = firBuilder.create<fir::ShapeOp>(loc, llvm::to_vector(extents));
+        shape = fir::ShapeOp::create(firBuilder, loc, llvm::to_vector(extents));
       } else {
         shape = genShapeOp(firBuilder, seqTy, loc);
       }
-      auto alloca = firBuilder.create<fir::AllocaOp>(
+      auto alloca = fir::AllocaOp::create(firBuilder,
           loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
-      auto declareOp = firBuilder.create<hlfir::DeclareOp>(
+      auto declareOp = hlfir::DeclareOp::create(firBuilder,
           loc, alloca, varName, shape, llvm::ArrayRef<mlir::Value>{},
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
 
@@ -584,21 +584,21 @@ mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
         llvm::SmallVector<mlir::Value> ivs;
 
         if (seqTy.hasDynamicExtents()) {
-          firBuilder.create<hlfir::AssignOp>(loc, initVal, declareOp.getBase());
+          hlfir::AssignOp::create(firBuilder, loc, initVal, declareOp.getBase());
         } else {
           for (auto ext : seqTy.getShape()) {
             auto lb = firBuilder.createIntegerConstant(loc, idxTy, 0);
             auto ub = firBuilder.createIntegerConstant(loc, idxTy, ext - 1);
             auto step = firBuilder.createIntegerConstant(loc, idxTy, 1);
-            auto loop = firBuilder.create<fir::DoLoopOp>(loc, lb, ub, step,
+            auto loop = fir::DoLoopOp::create(firBuilder, loc, lb, ub, step,
                                                          /*unordered=*/false);
             firBuilder.setInsertionPointToStart(loop.getBody());
             loops.push_back(loop);
             ivs.push_back(loop.getInductionVar());
           }
-          auto coord = firBuilder.create<fir::CoordinateOp>(
+          auto coord = fir::CoordinateOp::create(firBuilder,
               loc, refTy, declareOp.getBase(), ivs);
-          firBuilder.create<fir::StoreOp>(loc, initVal, coord);
+          fir::StoreOp::create(firBuilder, loc, initVal, coord);
           firBuilder.setInsertionPointAfter(loops[0]);
         }
       }
@@ -626,7 +626,7 @@ mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
           storeDst = firBuilder.createConvert(
               loc, firBuilder.getRefType(temp.getType()), retVal);
         }
-        builder.create<fir::StoreOp>(loc, temp, storeDst);
+        fir::StoreOp::create(builder, loc, temp, storeDst);
       } else {
         retVal = temp;
       }
@@ -634,7 +634,7 @@ mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
       TODO(loc, "Unsupported boxed type for OpenACC private-like recipe");
     }
     if (initVal) {
-      builder.create<hlfir::AssignOp>(loc, initVal, retVal);
+      hlfir::AssignOp::create(builder, loc, initVal, retVal);
     }
   }
   return retVal;
diff --git a/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp b/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
index 31076f6eb328f..6d3a32f9d37e0 100644
--- a/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
+++ b/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
@@ -246,9 +246,9 @@ class DoConcurrentConversion
   genParallelOp(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
                 looputils::InductionVariableInfos &ivInfos,
                 mlir::IRMapping &mapper) const {
-    auto parallelOp = rewriter.create<mlir::omp::ParallelOp>(loc);
+    auto parallelOp = mlir::omp::ParallelOp::create(rewriter, loc);
     rewriter.createBlock(&parallelOp.getRegion());
-    rewriter.setInsertionPoint(rewriter.create<mlir::omp::TerminatorOp>(loc));
+    rewriter.setInsertionPoint(mlir::omp::TerminatorOp::create(rewriter, loc));
 
     genLoopNestIndVarAllocs(rewriter, ivInfos, mapper);
     return parallelOp;
@@ -319,7 +319,7 @@ class DoConcurrentConversion
         auto firYield =
             mlir::cast<fir::YieldOp>(ompRegion.back().getTerminator());
         rewriter.setInsertionPoint(firYield);
-        rewriter.create<mlir::omp::YieldOp>(firYield.getLoc(),
+        mlir::omp::YieldOp::create(rewriter, firYield.getLoc(),
                                             firYield.getOperands());
         rewriter.eraseOp(firYield);
       }
@@ -342,7 +342,7 @@ class DoConcurrentConversion
         mlir::OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointAfter(localizer);
 
-        auto privatizer = rewriter.create<mlir::omp::PrivateClauseOp>(
+        auto privatizer = mlir::omp::PrivateClauseOp::create(rewriter,
             localizer.getLoc(), sym.getLeafReference().str() + ".omp",
             localizer.getTypeAttr().getValue(),
             mlir::omp::DataSharingClauseType::Private);
@@ -369,7 +369,7 @@ class DoConcurrentConversion
         mlir::OpBuilder::InsertionGuard guard(rewriter);
         rewriter.setInsertionPointAfter(firReducer);
 
-        auto ompReducer = rewriter.create<mlir::omp::DeclareReductionOp>(
+        auto ompReducer = mlir::omp::DeclareReductionOp::create(rewriter,
             firReducer.getLoc(), sym.getLeafReference().str() + ".omp",
             firReducer.getTypeAttr().getValue());
 
@@ -392,7 +392,7 @@ class DoConcurrentConversion
     }
 
     auto wsloopOp =
-        rewriter.create<mlir::omp::WsloopOp>(loop.getLoc(), wsloopClauseOps);
+        mlir::omp::WsloopOp::create(rewriter, loop.getLoc(), wsloopClauseOps);
     wsloopOp.setComposite(isComposite);
 
     Fortran::common::openmp::EntryBlockArgs wsloopArgs;
@@ -402,7 +402,7 @@ class DoConcurrentConversion
                                            wsloopOp.getRegion());
 
     auto loopNestOp =
-        rewriter.create<mlir::omp::LoopNestOp>(loop.getLoc(), clauseOps);
+        mlir::omp::LoopNestOp::create(rewriter, loop.getLoc(), clauseOps);
 
     // Clone the loop's body inside the loop nest construct using the
     // mapped values.
@@ -410,7 +410,7 @@ class DoConcurrentConversion
                                loopNestOp.getRegion().begin(), mapper);
 
     rewriter.setInsertionPointToEnd(&loopNestOp.getRegion().back());
-    rewriter.create<mlir::omp::YieldOp>(loop->getLoc());
+    mlir::omp::YieldOp::create(rewriter, loop->getLoc());
 
     // `local` region arguments are transferred/cloned from the `do concurrent`
     // loop to the loopnest op when the region is cloned above. Instead, these
diff --git a/flang/lib/Optimizer/OpenMP/FunctionFiltering.cpp b/flang/lib/Optimizer/OpenMP/FunctionFiltering.cpp
index 9554808824ac3..ae5c0ecc5b7f6 100644
--- a/flang/lib/Optimizer/OpenMP/FunctionFiltering.cpp
+++ b/flang/lib/Optimizer/OpenMP/FunctionFiltering.cpp
@@ -83,7 +83,7 @@ class FunctionFilteringPass
             for (Value res : callOp->getResults()) {
               opBuilder.setInsertionPoint(callOp);
               undefResults.emplace_back(
-                  opBuilder.create<fir::UndefOp>(res.getLoc(), res.getType()));
+                  fir::UndefOp::create(opBuilder, res.getLoc(), res.getType()));
             }
             callOp->replaceAllUsesWith(undefResults);
           }
diff --git a/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp b/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp
index 3009746954984..50dc19c6659c8 100644
--- a/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp
+++ b/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp
@@ -280,7 +280,7 @@ class GenericLoopConversionPattern
       args.reduction.vars = clauseOps.reductionVars;
     }
 
-    auto wrapperOp = rewriter.create<OpTy>(loopOp.getLoc(), clauseOps);
+    auto wrapperOp = OpTy::create(rewriter, loopOp.getLoc(), clauseOps);
     mlir::Block *opBlock = genEntryBlock(rewriter, args, wrapperOp.getRegion());
 
     mlir::IRMapping mapper;
@@ -307,15 +307,15 @@ class GenericLoopConversionPattern
     Fortran::common::openmp::EntryBlockArgs parallelArgs;
     parallelArgs.priv.vars = parallelClauseOps.privateVars;
 
-    auto parallelOp = rewriter.create<mlir::omp::ParallelOp>(loopOp.getLoc(),
+    auto parallelOp = mlir::omp::ParallelOp::create(rewriter, loopOp.getLoc(),
                                                              parallelClauseOps);
     genEntryBlock(rewriter, parallelArgs, parallelOp.getRegion());
     parallelOp.setComposite(true);
     rewriter.setInsertionPoint(
-        rewriter.create<mlir::omp::TerminatorOp>(loopOp.getLoc()));
+        mlir::omp::TerminatorOp::create(rewriter, loopOp.getLoc()));
 
     mlir::omp::DistributeOperands distributeClauseOps;
-    auto distributeOp = rewriter.create<mlir::omp::DistributeOp>(
+    auto distributeOp = mlir::omp::DistributeOp::create(rewriter,
         loopOp.getLoc(), distributeClauseOps);
     distributeOp.setComposite(true);
     rewriter.createBlock(&distributeOp.getRegion());
@@ -326,7 +326,7 @@ class GenericLoopConversionPattern
     wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
 
     auto wsloopOp =
-        rewriter.create<mlir::omp::WsloopOp>(loopOp.getLoc(), wsloopClauseOps);
+        mlir::omp::WsloopOp::create(rewriter, loopOp.getLoc(), wsloopClauseOps);
     wsloopOp.setComposite(true);
     genEntryBlock(rewriter, wsloopArgs, wsloopOp.getRegion());
 
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp
index 27a57f73d4e97..f6af684f06edb 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp
@@ -160,17 +160,17 @@ static mlir::func::FuncOp createCopyFunc(mlir::Location loc, mlir::Type varType,
   llvm::SmallVector<mlir::Type> argsTy = {varType, varType};
   auto funcType = mlir::FunctionType::get(builder.getContext(), argsTy, {});
   mlir::func::FuncOp funcOp =
-      modBuilder.create<mlir::func::FuncOp>(loc, copyFuncName, funcType);
+      mlir::func::FuncOp::create(modBuilder, loc, copyFuncName, funcType);
   funcOp.setVisibility(mlir::SymbolTable::Visibility::Private);
   fir::factory::setInternalLinkage(funcOp);
   builder.createBlock(&funcOp.getRegion(), funcOp.getRegion().end(), argsTy,
                       {loc, loc});
   builder.setInsertionPointToStart(&funcOp.getRegion().back());
 
-  Value loaded = builder.create<fir::LoadOp>(loc, funcOp.getArgument(1));
-  builder.create<fir::StoreOp>(loc, loaded, funcOp.getArgument(0));
+  Value loaded = fir::LoadOp::create(builder, loc, funcOp.getArgument(1));
+  fir::StoreOp::create(builder, loc, loaded, funcOp.getArgument(0));
 
-  builder.create<mlir::func::ReturnOp>(loc);
+  mlir::func::ReturnOp::create(builder, loc);
   return funcOp;
 }
 
@@ -234,9 +234,9 @@ static void parallelizeRegion(Region &sourceRegion, Region &targetRegion,
     if (auto reloaded = rootMapping.lookupOrNull(v))
       return nullptr;
     Type ty = v.getType();
-    Value alloc = allocaBuilder.create<fir::AllocaOp>(loc, ty);
-    singleBuilder.create<fir::StoreOp>(loc, singleMapping.lookup(v), alloc);
-    Value reloaded = parallelBuilder.create<fir::LoadOp>(loc, ty, alloc);
+    Value alloc = fir::AllocaOp::create(allocaBuilder, loc, ty);
+    fir::StoreOp::create(singleBuilder, loc, singleMapping.lookup(v), alloc);
+    Value reloaded = fir::LoadOp::create(parallelBuilder, loc, ty, alloc);
     rootMapping.map(v, reloaded);
     return alloc;
   };
@@ -293,7 +293,7 @@ static void parallelizeRegion(Region &sourceRegion, Region &targetRegion,
         allParallelized = false;
       }
     }
-    singleBuilder.create<omp::TerminatorOp>(loc);
+    omp::TerminatorOp::create(singleBuilder, loc);
     return {allParallelized, copyPrivate};
   };
 
@@ -370,7 +370,7 @@ static void parallelizeRegion(Region &sourceRegion, Region &targetRegion,
                 SymbolRefAttr::get(funcOp));
           }
           omp::SingleOp singleOp =
-              rootBuilder.create<omp::SingleOp>(loc, singleOperands);
+              omp::SingleOp::create(rootBuilder, loc, singleOperands);
           singleOp.getRegion().push_back(singleBlock);
           targetRegion.front().getOperations().splice(
               singleOp->getIterator(), allocaBlock->getOperations());
@@ -386,7 +386,7 @@ static void parallelizeRegion(Region &sourceRegion, Region &targetRegion,
           if (isLast)
             wsloopOperands.nowait = rootBuilder.getUnitAttr();
           auto wsloop =
-              rootBuilder.create<mlir::omp::WsloopOp>(loc, wsloopOperands);
+              mlir::omp::WsloopOp::create(rootBuilder, loc, wsloopOperands);
           auto clonedWslw = cast<omp::WorkshareLoopWrapperOp>(
               rootBuilder.clone(*wslw, rootMapping));
           wsloop.getRegion().takeBody(clonedWslw.getRegion());
@@ -465,9 +465,9 @@ LogicalResult lowerWorkshare(mlir::omp::WorkshareOp wsOp, DominanceInfo &di) {
     // it because our `parallelizeRegion` function works on regions and not
     // blocks.
     omp::WorkshareOp newOp =
-        rootBuilder.create<omp::WorkshareOp>(loc, omp::WorkshareOperands());
+        omp::WorkshareOp::create(rootBuilder, loc, omp::WorkshareOperands());
     if (!wsOp.getNowait())
-      rootBuilder.create<omp::BarrierOp>(loc);
+      omp::BarrierOp::create(rootBuilder, loc);
 
     parallelizeRegion(wsOp.getRegion(), newOp.getRegion(), rootMapping, loc,
                       di);
@@ -505,7 +505,7 @@ LogicalResult lowerWorkshare(mlir::omp::WorkshareOp wsOp, DominanceInfo &di) {
 
     omp::SingleOperands operands;
     operands.nowait = wsOp.getNowaitAttr();
-    omp::SingleOp newOp = rootBuilder.create<omp::SingleOp>(loc, operands);
+    omp::SingleOp newOp = omp::SingleOp::create(rootBuilder, loc, operands);
 
     newOp.getRegion().getBlocks().splice(newOp.getRegion().getBlocks().begin(),
                                          wsOp.getRegion().getBlocks());
diff --git a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
index f052cf8d22d1e..0db273ab88ed9 100644
--- a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
@@ -158,17 +158,17 @@ class MapInfoFinalizationPass
     mlir::Type allocaType = descriptor.getType();
     if (fir::isBoxAddress(allocaType))
       allocaType = fir::unwrapRefType(allocaType);
-    auto alloca = builder.create<fir::AllocaOp>(loc, allocaType);
+    auto alloca = fir::AllocaOp::create(builder, loc, allocaType);
     builder.restoreInsertionPoint(insPt);
     // We should only emit a store if the passed in data is present, it is
     // possible a user passes in no argument to an optional parameter, in which
     // case we cannot store or we'll segfault on the emitted memcpy.
     auto isPresent =
-        builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), descriptor);
+        fir::IsPresentOp::create(builder, loc, builder.getI1Type(), descriptor);
     builder.genIfOp(loc, {}, isPresent, false)
         .genThen([&]() {
           descriptor = builder.loadIfRef(loc, descriptor);
-          builder.create<fir::StoreOp>(loc, descriptor, alloca);
+          fir::StoreOp::create(builder, loc, descriptor, alloca);
         })
         .end();
     return slot = alloca;
@@ -183,7 +183,7 @@ class MapInfoFinalizationPass
                                       int64_t mapType,
                                       fir::FirOpBuilder &builder) {
     mlir::Location loc = descriptor.getLoc();
-    mlir::Value baseAddrAddr = builder.create<fir::BoxOffsetOp>(
+    mlir::Value baseAddrAddr = fir::BoxOffsetOp::create(builder,
         loc, descriptor, fir::BoxFieldAttr::base_addr);
 
     mlir::Type underlyingVarType =
@@ -195,7 +195,7 @@ class MapInfoFinalizationPass
         underlyingVarType = seqType.getEleTy();
 
     // Member of the descriptor pointing at the allocated data
-    return builder.create<mlir::omp::MapInfoOp>(
+    return mlir::omp::MapInfoOp::create(builder,
         loc, baseAddrAddr.getType(), descriptor,
         mlir::TypeAttr::get(underlyingVarType),
         builder.getIntegerAttr(builder.getIntegerType(64, false), mapType),
@@ -293,11 +293,11 @@ class MapInfoFinalizationPass
     mlir::Value boxChar = op.getVarPtr();
 
     if (mlir::isa<fir::ReferenceType>(op.getVarPtr().getType()))
-      boxChar = builder.create<fir::LoadOp>(loc, op.getVarPtr());
+      boxChar = fir::LoadOp::create(builder, loc, op.getVarPtr());
 
     fir::BoxCharType boxCharType =
         mlir::dyn_cast<fir::BoxCharType>(boxChar.getType());
-    mlir::Value boxAddr = builder.create<fir::BoxOffsetOp>(
+    mlir::Value boxAddr = fir::BoxOffsetOp::create(builder,
         loc, op.getVarPtr(), fir::BoxFieldAttr::base_addr);
 
     uint64_t mapTypeToImplicit = static_cast<
@@ -310,7 +310,7 @@ class MapInfoFinalizationPass
     newMembersAttr = builder.create2DI64ArrayAttr(memberIdx);
 
     mlir::Value varPtr = op.getVarPtr();
-    mlir::omp::MapInfoOp memberMapInfoOp = builder.create<mlir::omp::MapInfoOp>(
+    mlir::omp::MapInfoOp memberMapInfoOp = mlir::omp::MapInfoOp::create(builder,
         op.getLoc(), varPtr.getType(), varPtr,
         mlir::TypeAttr::get(boxCharType.getEleTy()),
         builder.getIntegerAttr(builder.getIntegerType(64, /*isSigned=*/false),
@@ -324,7 +324,7 @@ class MapInfoFinalizationPass
         /*mapperId=*/mlir::FlatSymbolRefAttr(), /*name=*/op.getNameAttr(),
         builder.getBoolAttr(false));
 
-    mlir::omp::MapInfoOp newMapInfoOp = builder.create<mlir::omp::MapInfoOp>(
+    mlir::omp::MapInfoOp newMapInfoOp = mlir::omp::MapInfoOp::create(builder,
         op.getLoc(), op.getResult().getType(), varPtr,
         mlir::TypeAttr::get(
             llvm::cast<mlir::omp::PointerLikeType>(varPtr.getType())
@@ -426,7 +426,7 @@ class MapInfoFinalizationPass
     }
 
     mlir::omp::MapInfoOp newDescParentMapOp =
-        builder.create<mlir::omp::MapInfoOp>(
+        mlir::omp::MapInfoOp::create(builder,
             op->getLoc(), op.getResult().getType(), descriptor,
             mlir::TypeAttr::get(fir::unwrapRefType(descriptor.getType())),
             builder.getIntegerAttr(builder.getIntegerType(64, false),
@@ -739,7 +739,7 @@ class MapInfoFinalizationPass
           builder.setInsertionPoint(op);
           fir::IntOrValue idxConst =
               mlir::IntegerAttr::get(builder.getI32Type(), fieldIdx);
-          auto fieldCoord = builder.create<fir::CoordinateOp>(
+          auto fieldCoord = fir::CoordinateOp::create(builder,
               op.getLoc(), builder.getRefType(memTy), op.getVarPtr(),
               llvm::SmallVector<fir::IntOrValue, 1>{idxConst});
           fir::factory::AddrAndBoundsInfo info =
@@ -755,7 +755,7 @@ class MapInfoFinalizationPass
                   /*dataExvIsAssumedSize=*/false, op.getLoc());
 
           mlir::omp::MapInfoOp fieldMapOp =
-              builder.create<mlir::omp::MapInfoOp>(
+              mlir::omp::MapInfoOp::create(builder,
                   op.getLoc(), fieldCoord.getResult().getType(),
                   fieldCoord.getResult(),
                   mlir::TypeAttr::get(
diff --git a/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp b/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
index 19566af11db04..e007a80512513 100644
--- a/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
@@ -80,9 +80,9 @@ class MapsForPrivatizedSymbolsPass
       mlir::Block *allocaBlock = builder.getAllocaBlock();
       assert(allocaBlock && "No allocablock  found for a funcOp");
       builder.setInsertionPointToStart(allocaBlock);
-      auto alloca = builder.create<fir::AllocaOp>(loc, varPtr.getType());
+      auto alloca = fir::AllocaOp::create(builder, loc, varPtr.getType());
       builder.restoreInsertionPoint(savedInsPoint);
-      builder.create<fir::StoreOp>(loc, varPtr, alloca);
+      fir::StoreOp::create(builder, loc, varPtr, alloca);
       varPtr = alloca;
     }
     assert(mlir::isa<omp::PointerLikeType>(varPtr.getType()) &&
@@ -94,7 +94,7 @@ class MapsForPrivatizedSymbolsPass
     if (needsBoundsOps(varPtr))
       genBoundsOps(builder, varPtr, boundsOps);
 
-    return builder.create<omp::MapInfoOp>(
+    return omp::MapInfoOp::create(builder,
         loc, varPtr.getType(), varPtr,
         TypeAttr::get(llvm::cast<omp::PointerLikeType>(varPtr.getType())
                           .getElementType()),


From flang-commits at lists.llvm.org  Mon Jul 21 14:59:18 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 21 Jul 2025 14:59:18 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][NFC] update
 `flang/Optimizer/Transforms` create APIs (11/n) (PR #149915)
Message-ID: <llvm.org/llvm/llvm-project/pull/149915@github.com>


https://github.com/makslevental created https://github.com/llvm/llvm-project/pull/149915

See https://github.com/llvm/llvm-project/pull/147168 for more info.

>From c46be1a7b6242bf07ca83e8cf5ef759717542b69 Mon Sep 17 00:00:00 2001
From: max <maksim.levental at gmail.com>
Date: Mon, 21 Jul 2025 17:57:16 -0400
Subject: [PATCH] [mlir][NFC] update `flang/Optimizer/Transforms` create APIs
 (11/n) (#149687)

See https://github.com/llvm/llvm-project/pull/147168 for more info.
---
 .../Optimizer/Transforms/AbstractResult.cpp   |  16 +-
 .../Optimizer/Transforms/AffineDemotion.cpp   |   4 +-
 .../Optimizer/Transforms/AffinePromotion.cpp  |  20 +-
 .../Optimizer/Transforms/ArrayValueCopy.cpp   |  44 ++---
 .../Transforms/AssumedRankOpConversion.cpp    |   4 +-
 .../Transforms/CUFAddConstructor.cpp          |  18 +-
 .../CUFComputeSharedMemoryOffsetsAndSize.cpp  |   6 +-
 .../Transforms/CUFGPUToLLVMConversion.cpp     |  44 ++---
 .../Optimizer/Transforms/CUFOpConversion.cpp  |  86 ++++-----
 .../Transforms/CharacterConversion.cpp        |  24 +--
 .../ConstantArgumentGlobalisation.cpp         |   6 +-
 .../Transforms/ControlFlowConverter.cpp       |  64 +++---
 .../Transforms/DebugTypeGenerator.cpp         |   4 +-
 flang/lib/Optimizer/Transforms/FIRToSCF.cpp   |  18 +-
 .../Transforms/GenRuntimeCallsForTest.cpp     |   4 +-
 .../Optimizer/Transforms/LoopVersioning.cpp   |  28 +--
 .../Optimizer/Transforms/MemoryAllocation.cpp |   4 +-
 .../lib/Optimizer/Transforms/MemoryUtils.cpp  |  18 +-
 .../Transforms/PolymorphicOpConversion.cpp    |  54 +++---
 .../Transforms/SimplifyFIROperations.cpp      |  14 +-
 .../Transforms/SimplifyIntrinsics.cpp         | 182 +++++++++---------
 .../lib/Optimizer/Transforms/StackArrays.cpp  |   4 +-
 22 files changed, 333 insertions(+), 333 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/AbstractResult.cpp b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
index 59e2eeb76c715..4899822bbb9f4 100644
--- a/flang/lib/Optimizer/Transforms/AbstractResult.cpp
+++ b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
@@ -137,7 +137,7 @@ class CallConversion : public mlir::OpRewritePattern<Op> {
     auto buffer = saveResult.getMemref();
     mlir::Value arg = buffer;
     if (mustEmboxResult(result.getType(), shouldBoxResult))
-      arg = rewriter.create<fir::EmboxOp>(
+      arg = fir::EmboxOp::create(rewriter,
           loc, argType, buffer, saveResult.getShape(), /*slice*/ mlir::Value{},
           saveResult.getTypeparams());
 
@@ -155,7 +155,7 @@ class CallConversion : public mlir::OpRewritePattern<Op> {
         if (!isResultBuiltinCPtr)
           newOperands.emplace_back(arg);
         newOperands.append(op.getOperands().begin(), op.getOperands().end());
-        newOp = rewriter.create<fir::CallOp>(loc, *op.getCallee(),
+        newOp = fir::CallOp::create(rewriter, loc, *op.getCallee(),
                                              newResultTypes, newOperands);
       } else {
         // Indirect calls.
@@ -169,12 +169,12 @@ class CallConversion : public mlir::OpRewritePattern<Op> {
 
         llvm::SmallVector<mlir::Value> newOperands;
         newOperands.push_back(
-            rewriter.create<fir::ConvertOp>(loc, newFuncTy, op.getOperand(0)));
+            fir::ConvertOp::create(rewriter, loc, newFuncTy, op.getOperand(0)));
         if (!isResultBuiltinCPtr)
           newOperands.push_back(arg);
         newOperands.append(op.getOperands().begin() + 1,
                            op.getOperands().end());
-        newOp = rewriter.create<fir::CallOp>(loc, mlir::SymbolRefAttr{},
+        newOp = fir::CallOp::create(rewriter, loc, mlir::SymbolRefAttr{},
                                              newResultTypes, newOperands);
       }
     }
@@ -191,7 +191,7 @@ class CallConversion : public mlir::OpRewritePattern<Op> {
         passArgPos =
             rewriter.getI32IntegerAttr(*op.getPassArgPos() + passArgShift);
       // TODO: propagate argument and result attributes (need to be shifted).
-      newOp = rewriter.create<fir::DispatchOp>(
+      newOp = fir::DispatchOp::create(rewriter,
           loc, newResultTypes, rewriter.getStringAttr(op.getMethod()),
           op.getOperands()[0], newOperands, passArgPos,
           /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
@@ -280,7 +280,7 @@ processReturnLikeOp(OpTy ret, mlir::Value newArg,
     // register pass, this is possible for fir.box results, or fir.record
     // with no length parameters. Simply store the result in the result
     // storage. at the return point.
-    rewriter.create<fir::StoreOp>(loc, resultValue, newArg);
+    fir::StoreOp::create(rewriter, loc, resultValue, newArg);
     rewriter.replaceOpWithNewOp<OpTy>(ret);
   }
   // Delete result old local storage if unused.
@@ -337,7 +337,7 @@ class AddrOfOpConversion : public mlir::OpRewritePattern<fir::AddrOfOp> {
       newFuncTy = getCPtrFunctionType(oldFuncTy);
     else
       newFuncTy = getNewFunctionType(oldFuncTy, shouldBoxResult);
-    auto newAddrOf = rewriter.create<fir::AddrOfOp>(addrOf.getLoc(), newFuncTy,
+    auto newAddrOf = fir::AddrOfOp::create(rewriter, addrOf.getLoc(), newFuncTy,
                                                     addrOf.getSymbol());
     // Rather than converting all op a function pointer might transit through
     // (e.g calls, stores, loads, converts...), cast new type to the abstract
@@ -397,7 +397,7 @@ class AbstractResultOpt
         if (mustEmboxResult(resultType, shouldBoxResult)) {
           auto bufferType = fir::ReferenceType::get(resultType);
           rewriter.setInsertionPointToStart(&func.front());
-          newArg = rewriter.create<fir::BoxAddrOp>(loc, bufferType, newArg);
+          newArg = fir::BoxAddrOp::create(rewriter, loc, bufferType, newArg);
         }
         patterns.insert<ReturnOpConversion>(context, newArg);
         target.addDynamicallyLegalOp<mlir::func::ReturnOp>(
diff --git a/flang/lib/Optimizer/Transforms/AffineDemotion.cpp b/flang/lib/Optimizer/Transforms/AffineDemotion.cpp
index d45f855c9078e..d08c9d83b9e68 100644
--- a/flang/lib/Optimizer/Transforms/AffineDemotion.cpp
+++ b/flang/lib/Optimizer/Transforms/AffineDemotion.cpp
@@ -60,7 +60,7 @@ class AffineLoadConversion
     if (!maybeExpandedMap)
       return failure();
 
-    auto coorOp = rewriter.create<fir::CoordinateOp>(
+    auto coorOp = fir::CoordinateOp::create(rewriter,
         op.getLoc(), fir::ReferenceType::get(op.getResult().getType()),
         adaptor.getMemref(), *maybeExpandedMap);
 
@@ -83,7 +83,7 @@ class AffineStoreConversion
     if (!maybeExpandedMap)
       return failure();
 
-    auto coorOp = rewriter.create<fir::CoordinateOp>(
+    auto coorOp = fir::CoordinateOp::create(rewriter,
         op.getLoc(), fir::ReferenceType::get(op.getValueToStore().getType()),
         adaptor.getMemref(), *maybeExpandedMap);
     rewriter.replaceOpWithNewOp<fir::StoreOp>(op, adaptor.getValue(),
diff --git a/flang/lib/Optimizer/Transforms/AffinePromotion.cpp b/flang/lib/Optimizer/Transforms/AffinePromotion.cpp
index ef82e400bea14..d8d971f9aa3d0 100644
--- a/flang/lib/Optimizer/Transforms/AffinePromotion.cpp
+++ b/flang/lib/Optimizer/Transforms/AffinePromotion.cpp
@@ -366,7 +366,7 @@ static mlir::Type coordinateArrayElement(fir::ArrayCoorOp op) {
 static void populateIndexArgs(fir::ArrayCoorOp acoOp, fir::ShapeOp shape,
                               SmallVectorImpl<mlir::Value> &indexArgs,
                               mlir::PatternRewriter &rewriter) {
-  auto one = rewriter.create<mlir::arith::ConstantOp>(
+  auto one = mlir::arith::ConstantOp::create(rewriter,
       acoOp.getLoc(), rewriter.getIndexType(), rewriter.getIndexAttr(1));
   auto extents = shape.getExtents();
   for (auto i = extents.begin(); i < extents.end(); i++) {
@@ -379,7 +379,7 @@ static void populateIndexArgs(fir::ArrayCoorOp acoOp, fir::ShapeOp shape,
 static void populateIndexArgs(fir::ArrayCoorOp acoOp, fir::ShapeShiftOp shape,
                               SmallVectorImpl<mlir::Value> &indexArgs,
                               mlir::PatternRewriter &rewriter) {
-  auto one = rewriter.create<mlir::arith::ConstantOp>(
+  auto one = mlir::arith::ConstantOp::create(rewriter,
       acoOp.getLoc(), rewriter.getIndexType(), rewriter.getIndexAttr(1));
   auto extents = shape.getPairs();
   for (auto i = extents.begin(); i < extents.end();) {
@@ -422,12 +422,12 @@ createAffineOps(mlir::Value arrayRef, mlir::PatternRewriter &rewriter) {
 
   populateIndexArgs(acoOp, indexArgs, rewriter);
 
-  auto affineApply = rewriter.create<affine::AffineApplyOp>(
+  auto affineApply = affine::AffineApplyOp::create(rewriter,
       acoOp.getLoc(), affineMap, indexArgs);
   auto arrayElementType = coordinateArrayElement(acoOp);
   auto newType =
       mlir::MemRefType::get({mlir::ShapedType::kDynamic}, arrayElementType);
-  auto arrayConvert = rewriter.create<fir::ConvertOp>(acoOp.getLoc(), newType,
+  auto arrayConvert = fir::ConvertOp::create(rewriter, acoOp.getLoc(), newType,
                                                       acoOp.getMemref());
   return std::make_pair(affineApply, arrayConvert);
 }
@@ -495,7 +495,7 @@ class AffineLoopConversion : public mlir::OpRewritePattern<fir::DoLoopOp> {
                                 affineFor.getRegionIterArgs());
     if (!results.empty()) {
       rewriter.setInsertionPointToEnd(affineFor.getBody());
-      rewriter.create<affine::AffineYieldOp>(resultOp->getLoc(), results);
+      affine::AffineYieldOp::create(rewriter, resultOp->getLoc(), results);
     }
     rewriter.finalizeOpModification(affineFor.getOperation());
 
@@ -525,7 +525,7 @@ class AffineLoopConversion : public mlir::OpRewritePattern<fir::DoLoopOp> {
   std::pair<affine::AffineForOp, mlir::Value>
   positiveConstantStep(fir::DoLoopOp op, int64_t step,
                        mlir::PatternRewriter &rewriter) const {
-    auto affineFor = rewriter.create<affine::AffineForOp>(
+    auto affineFor = affine::AffineForOp::create(rewriter,
         op.getLoc(), ValueRange(op.getLowerBound()),
         mlir::AffineMap::get(0, 1,
                              mlir::getAffineSymbolExpr(0, op.getContext())),
@@ -543,7 +543,7 @@ class AffineLoopConversion : public mlir::OpRewritePattern<fir::DoLoopOp> {
     auto step = mlir::getAffineSymbolExpr(2, op.getContext());
     mlir::AffineMap upperBoundMap = mlir::AffineMap::get(
         0, 3, (upperBound - lowerBound + step).floorDiv(step));
-    auto genericUpperBound = rewriter.create<affine::AffineApplyOp>(
+    auto genericUpperBound = affine::AffineApplyOp::create(rewriter,
         op.getLoc(), upperBoundMap,
         ValueRange({op.getLowerBound(), op.getUpperBound(), op.getStep()}));
     auto actualIndexMap = mlir::AffineMap::get(
@@ -551,7 +551,7 @@ class AffineLoopConversion : public mlir::OpRewritePattern<fir::DoLoopOp> {
         (lowerBound + mlir::getAffineDimExpr(0, op.getContext())) *
             mlir::getAffineSymbolExpr(1, op.getContext()));
 
-    auto affineFor = rewriter.create<affine::AffineForOp>(
+    auto affineFor = affine::AffineForOp::create(rewriter,
         op.getLoc(), ValueRange(),
         AffineMap::getConstantMap(0, op.getContext()),
         genericUpperBound.getResult(),
@@ -559,7 +559,7 @@ class AffineLoopConversion : public mlir::OpRewritePattern<fir::DoLoopOp> {
                              1 + mlir::getAffineSymbolExpr(0, op.getContext())),
         1, op.getIterOperands());
     rewriter.setInsertionPointToStart(affineFor.getBody());
-    auto actualIndex = rewriter.create<affine::AffineApplyOp>(
+    auto actualIndex = affine::AffineApplyOp::create(rewriter,
         op.getLoc(), actualIndexMap,
         ValueRange(
             {affineFor.getInductionVar(), op.getLowerBound(), op.getStep()}));
@@ -588,7 +588,7 @@ class AffineIfConversion : public mlir::OpRewritePattern<fir::IfOp> {
               << "AffineIfConversion: couldn't calculate affine condition\n";);
       return failure();
     }
-    auto affineIf = rewriter.create<affine::AffineIfOp>(
+    auto affineIf = affine::AffineIfOp::create(rewriter,
         op.getLoc(), affineCondition.getIntegerSet(),
         affineCondition.getAffineArgs(), !op.getElseRegion().empty());
     rewriter.startOpModification(affineIf);
diff --git a/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp b/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp
index 8544d17f62248..7f24624a47788 100644
--- a/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp
+++ b/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp
@@ -856,7 +856,7 @@ static bool getAdjustedExtents(mlir::Location loc,
   auto idxTy = rewriter.getIndexType();
   if (isAssumedSize(result)) {
     // Use slice information to compute the extent of the column.
-    auto one = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 1);
+    auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
     mlir::Value size = one;
     if (mlir::Value sliceArg = arrLoad.getSlice()) {
       if (auto sliceOp =
@@ -896,14 +896,14 @@ static mlir::Value getOrReadExtentsAndShapeOp(
         mlir::cast<SequenceType>(dyn_cast_ptrOrBoxEleTy(boxTy)).getDimension();
     auto idxTy = rewriter.getIndexType();
     for (decltype(rank) dim = 0; dim < rank; ++dim) {
-      auto dimVal = rewriter.create<mlir::arith::ConstantIndexOp>(loc, dim);
-      auto dimInfo = rewriter.create<BoxDimsOp>(loc, idxTy, idxTy, idxTy,
+      auto dimVal = mlir::arith::ConstantIndexOp::create(rewriter, loc, dim);
+      auto dimInfo = BoxDimsOp::create(rewriter, loc, idxTy, idxTy, idxTy,
                                                 arrLoad.getMemref(), dimVal);
       result.emplace_back(dimInfo.getResult(1));
     }
     if (!arrLoad.getShape()) {
       auto shapeType = ShapeType::get(rewriter.getContext(), rank);
-      return rewriter.create<ShapeOp>(loc, shapeType, result);
+      return ShapeOp::create(rewriter, loc, shapeType, result);
     }
     auto shiftOp = arrLoad.getShape().getDefiningOp<ShiftOp>();
     auto shapeShiftType = ShapeShiftType::get(rewriter.getContext(), rank);
@@ -912,7 +912,7 @@ static mlir::Value getOrReadExtentsAndShapeOp(
       shapeShiftOperands.push_back(lb);
       shapeShiftOperands.push_back(extent);
     }
-    return rewriter.create<ShapeShiftOp>(loc, shapeShiftType,
+    return ShapeShiftOp::create(rewriter, loc, shapeShiftType,
                                          shapeShiftOperands);
   }
   copyUsingSlice =
@@ -952,12 +952,12 @@ static mlir::Value genCoorOp(mlir::PatternRewriter &rewriter,
   auto module = load->getParentOfType<mlir::ModuleOp>();
   FirOpBuilder builder(rewriter, module);
   auto typeparams = getTypeParamsIfRawData(loc, builder, load, alloc.getType());
-  mlir::Value result = rewriter.create<ArrayCoorOp>(
+  mlir::Value result = ArrayCoorOp::create(rewriter,
       loc, eleTy, alloc, shape, slice,
       llvm::ArrayRef<mlir::Value>{originated}.take_front(dimension),
       typeparams);
   if (dimension < originated.size())
-    result = rewriter.create<fir::CoordinateOp>(
+    result = fir::CoordinateOp::create(rewriter,
         loc, resTy, result,
         llvm::ArrayRef<mlir::Value>{originated}.drop_front(dimension));
   return result;
@@ -971,12 +971,12 @@ static mlir::Value getCharacterLen(mlir::Location loc, FirOpBuilder &builder,
       // The loaded array is an emboxed value. Get the CHARACTER length from
       // the box value.
       auto eleSzInBytes =
-          builder.create<BoxEleSizeOp>(loc, charLenTy, load.getMemref());
+          BoxEleSizeOp::create(builder, loc, charLenTy, load.getMemref());
       auto kindSize =
           builder.getKindMap().getCharacterBitsize(charTy.getFKind());
       auto kindByteSize =
           builder.createIntegerConstant(loc, charLenTy, kindSize / 8);
-      return builder.create<mlir::arith::DivSIOp>(loc, eleSzInBytes,
+      return mlir::arith::DivSIOp::create(builder, loc, eleSzInBytes,
                                                   kindByteSize);
     }
     // The loaded array is a (set of) unboxed values. If the CHARACTER's
@@ -1003,11 +1003,11 @@ void genArrayCopy(mlir::Location loc, mlir::PatternRewriter &rewriter,
   auto idxTy = rewriter.getIndexType();
   // Build loop nest from column to row.
   for (auto sh : llvm::reverse(extents)) {
-    auto ubi = rewriter.create<ConvertOp>(loc, idxTy, sh);
-    auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
-    auto one = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 1);
-    auto ub = rewriter.create<mlir::arith::SubIOp>(loc, idxTy, ubi, one);
-    auto loop = rewriter.create<DoLoopOp>(loc, zero, ub, one);
+    auto ubi = ConvertOp::create(rewriter, loc, idxTy, sh);
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto ub = mlir::arith::SubIOp::create(rewriter, loc, idxTy, ubi, one);
+    auto loop = DoLoopOp::create(rewriter, loc, zero, ub, one);
     rewriter.setInsertionPointToStart(loop.getBody());
     indices.push_back(loop.getInductionVar());
   }
@@ -1015,12 +1015,12 @@ void genArrayCopy(mlir::Location loc, mlir::PatternRewriter &rewriter,
   std::reverse(indices.begin(), indices.end());
   auto module = arrLoad->getParentOfType<mlir::ModuleOp>();
   FirOpBuilder builder(rewriter, module);
-  auto fromAddr = rewriter.create<ArrayCoorOp>(
+  auto fromAddr = ArrayCoorOp::create(rewriter,
       loc, getEleTy(src.getType()), src, shapeOp,
       CopyIn && copyUsingSlice ? sliceOp : mlir::Value{},
       factory::originateIndices(loc, rewriter, src.getType(), shapeOp, indices),
       getTypeParamsIfRawData(loc, builder, arrLoad, src.getType()));
-  auto toAddr = rewriter.create<ArrayCoorOp>(
+  auto toAddr = ArrayCoorOp::create(rewriter,
       loc, getEleTy(dst.getType()), dst, shapeOp,
       !CopyIn && copyUsingSlice ? sliceOp : mlir::Value{},
       factory::originateIndices(loc, rewriter, dst.getType(), shapeOp, indices),
@@ -1093,14 +1093,14 @@ allocateArrayTemp(mlir::Location loc, mlir::PatternRewriter &rewriter,
       findNonconstantExtents(baseType, extents);
   llvm::SmallVector<mlir::Value> typeParams =
       genArrayLoadTypeParameters(loc, rewriter, load);
-  mlir::Value allocmem = rewriter.create<AllocMemOp>(
+  mlir::Value allocmem = AllocMemOp::create(rewriter,
       loc, dyn_cast_ptrOrBoxEleTy(baseType), typeParams, nonconstantExtents);
   mlir::Type eleType =
       fir::unwrapSequenceType(fir::unwrapPassByRefType(baseType));
   if (fir::isRecordWithAllocatableMember(eleType)) {
     // The allocatable component descriptors need to be set to a clean
     // deallocated status before anything is done with them.
-    mlir::Value box = rewriter.create<fir::EmboxOp>(
+    mlir::Value box = fir::EmboxOp::create(rewriter,
         loc, fir::BoxType::get(allocmem.getType()), allocmem, shape,
         /*slice=*/mlir::Value{}, typeParams);
     auto module = load->getParentOfType<mlir::ModuleOp>();
@@ -1111,12 +1111,12 @@ allocateArrayTemp(mlir::Location loc, mlir::PatternRewriter &rewriter,
     auto cleanup = [=](mlir::PatternRewriter &r) {
       FirOpBuilder builder(r, module);
       runtime::genDerivedTypeDestroy(builder, loc, box);
-      r.create<FreeMemOp>(loc, allocmem);
+      FreeMemOp::create(r, loc, allocmem);
     };
     return {allocmem, cleanup};
   }
   auto cleanup = [=](mlir::PatternRewriter &r) {
-    r.create<FreeMemOp>(loc, allocmem);
+    FreeMemOp::create(r, loc, allocmem);
   };
   return {allocmem, cleanup};
 }
@@ -1257,7 +1257,7 @@ class ArrayUpdateConversion : public ArrayUpdateConversionBase<ArrayUpdateOp> {
       if (auto inEleTy = dyn_cast_ptrEleTy(input.getType())) {
         emitFatalError(loc, "array_update on references not supported");
       } else {
-        rewriter.create<fir::StoreOp>(loc, input, coor);
+        fir::StoreOp::create(rewriter, loc, input, coor);
       }
     };
     auto lhsEltRefType = toRefType(update.getMerge().getType());
@@ -1368,7 +1368,7 @@ class ArrayAmendConversion : public mlir::OpRewritePattern<ArrayAmendOp> {
     auto *op = amend.getOperation();
     rewriter.setInsertionPoint(op);
     auto loc = amend.getLoc();
-    auto undef = rewriter.create<UndefOp>(loc, amend.getType());
+    auto undef = UndefOp::create(rewriter, loc, amend.getType());
     rewriter.replaceOp(amend, undef.getResult());
     return mlir::success();
   }
diff --git a/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp b/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
index 6af1cb988a4c1..72c5c8cf1f65c 100644
--- a/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
@@ -88,7 +88,7 @@ class ReboxAssumedRankConv
         (fir::isPolymorphicType(oldBoxType) ||
          (newEleType != oldBoxType.unwrapInnerType())) &&
         !fir::isPolymorphicType(newBoxType)) {
-      newDtype = builder.create<fir::TypeDescOp>(
+      newDtype = fir::TypeDescOp::create(builder,
           loc, mlir::TypeAttr::get(newDerivedType));
     } else {
       newDtype = builder.createNullConstant(loc);
@@ -103,7 +103,7 @@ class ReboxAssumedRankConv
                                              rebox.getBox(), newDtype,
                                              newAttribute, lowerBoundModifier);
 
-    mlir::Value descValue = builder.create<fir::LoadOp>(loc, tempDesc);
+    mlir::Value descValue = fir::LoadOp::create(builder, loc, tempDesc);
     mlir::Value castDesc = builder.createConvert(loc, newBoxType, descValue);
     rewriter.replaceOp(rebox, castDesc);
     return mlir::success();
diff --git a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
index 2dd6950b34897..a74586653c260 100644
--- a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
@@ -68,7 +68,7 @@ struct CUFAddConstructor
 
     // Symbol reference to CUFRegisterAllocator.
     builder.setInsertionPointToEnd(mod.getBody());
-    auto registerFuncOp = builder.create<mlir::LLVM::LLVMFuncOp>(
+    auto registerFuncOp = mlir::LLVM::LLVMFuncOp::create(builder,
         loc, RTNAME_STRING(CUFRegisterAllocator), funcTy);
     registerFuncOp.setVisibility(mlir::SymbolTable::Visibility::Private);
     auto cufRegisterAllocatorRef = mlir::SymbolRefAttr::get(
@@ -76,16 +76,16 @@ struct CUFAddConstructor
     builder.setInsertionPointToEnd(mod.getBody());
 
     // Create the constructor function that call CUFRegisterAllocator.
-    auto func = builder.create<mlir::LLVM::LLVMFuncOp>(loc, cudaFortranCtorName,
+    auto func = mlir::LLVM::LLVMFuncOp::create(builder, loc, cudaFortranCtorName,
                                                        funcTy);
     func.setLinkage(mlir::LLVM::Linkage::Internal);
     builder.setInsertionPointToStart(func.addEntryBlock(builder));
-    builder.create<mlir::LLVM::CallOp>(loc, funcTy, cufRegisterAllocatorRef);
+    mlir::LLVM::CallOp::create(builder, loc, funcTy, cufRegisterAllocatorRef);
 
     auto gpuMod = symTab.lookup<mlir::gpu::GPUModuleOp>(cudaDeviceModuleName);
     if (gpuMod) {
       auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(ctx);
-      auto registeredMod = builder.create<cuf::RegisterModuleOp>(
+      auto registeredMod = cuf::RegisterModuleOp::create(builder,
           loc, llvmPtrTy, mlir::SymbolRefAttr::get(ctx, gpuMod.getName()));
 
       fir::LLVMTypeConverter typeConverter(mod, /*applyTBAA=*/false,
@@ -96,7 +96,7 @@ struct CUFAddConstructor
           auto kernelName = mlir::SymbolRefAttr::get(
               builder.getStringAttr(cudaDeviceModuleName),
               {mlir::SymbolRefAttr::get(builder.getContext(), func.getName())});
-          builder.create<cuf::RegisterKernelOp>(loc, kernelName, registeredMod);
+          cuf::RegisterKernelOp::create(builder, loc, kernelName, registeredMod);
         }
       }
 
@@ -140,19 +140,19 @@ struct CUFAddConstructor
           auto sizeVal = builder.createIntegerConstant(loc, idxTy, *size);
 
           // Global variable address
-          mlir::Value addr = builder.create<fir::AddrOfOp>(
+          mlir::Value addr = fir::AddrOfOp::create(builder,
               loc, globalOp.resultType(), globalOp.getSymbol());
 
           llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
               builder, loc, fTy, registeredMod, addr, gblName, sizeVal)};
-          builder.create<fir::CallOp>(loc, func, args);
+          fir::CallOp::create(builder, loc, func, args);
         } break;
         default:
           break;
         }
       }
     }
-    builder.create<mlir::LLVM::ReturnOp>(loc, mlir::ValueRange{});
+    mlir::LLVM::ReturnOp::create(builder, loc, mlir::ValueRange{});
 
     // Create the llvm.global_ctor with the function.
     // TODO: We might want to have a utility that retrieve it if already
@@ -165,7 +165,7 @@ struct CUFAddConstructor
     llvm::SmallVector<mlir::Attribute> data;
     priorities.push_back(0);
     data.push_back(mlir::LLVM::ZeroAttr::get(mod.getContext()));
-    builder.create<mlir::LLVM::GlobalCtorsOp>(
+    mlir::LLVM::GlobalCtorsOp::create(builder,
         mod.getLoc(), builder.getArrayAttr(funcs),
         builder.getI32ArrayAttr(priorities), builder.getArrayAttr(data));
   }
diff --git a/flang/lib/Optimizer/Transforms/CUFComputeSharedMemoryOffsetsAndSize.cpp b/flang/lib/Optimizer/Transforms/CUFComputeSharedMemoryOffsetsAndSize.cpp
index f6381ef8a8a21..beba8b822e467 100644
--- a/flang/lib/Optimizer/Transforms/CUFComputeSharedMemoryOffsetsAndSize.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFComputeSharedMemoryOffsetsAndSize.cpp
@@ -93,10 +93,10 @@ struct CUFComputeSharedMemoryOffsetsAndSize
           mlir::Value dynSize =
               builder.createIntegerConstant(loc, idxTy, tySize);
           for (auto extent : sharedOp.getShape())
-            dynSize = builder.create<mlir::arith::MulIOp>(loc, dynSize, extent);
+            dynSize = mlir::arith::MulIOp::create(builder, loc, dynSize, extent);
           if (crtDynOffset)
             crtDynOffset =
-                builder.create<mlir::arith::AddIOp>(loc, crtDynOffset, dynSize);
+                mlir::arith::AddIOp::create(builder, loc, crtDynOffset, dynSize);
           else
             crtDynOffset = dynSize;
 
@@ -142,7 +142,7 @@ struct CUFComputeSharedMemoryOffsetsAndSize
           fir::GlobalOp::getDataAttrAttrName(globalOpName),
           cuf::DataAttributeAttr::get(gpuMod.getContext(),
                                       cuf::DataAttribute::Shared)));
-      auto sharedMem = builder.create<fir::GlobalOp>(
+      auto sharedMem = fir::GlobalOp::create(builder,
           funcOp.getLoc(), sharedMemGlobalName, false, false, sharedMemType,
           init, linkage, attrs);
       sharedMem.setAlignment(alignment);
diff --git a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
index fe69ffa8350af..374d9b3381b01 100644
--- a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
@@ -46,27 +46,27 @@ static mlir::Value createKernelArgArray(mlir::Location loc,
   auto structTy = mlir::LLVM::LLVMStructType::getLiteral(ctx, structTypes);
   auto ptrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
   mlir::Type i32Ty = rewriter.getI32Type();
-  auto zero = rewriter.create<mlir::LLVM::ConstantOp>(
+  auto zero = mlir::LLVM::ConstantOp::create(rewriter,
       loc, i32Ty, rewriter.getIntegerAttr(i32Ty, 0));
-  auto one = rewriter.create<mlir::LLVM::ConstantOp>(
+  auto one = mlir::LLVM::ConstantOp::create(rewriter,
       loc, i32Ty, rewriter.getIntegerAttr(i32Ty, 1));
   mlir::Value argStruct =
-      rewriter.create<mlir::LLVM::AllocaOp>(loc, ptrTy, structTy, one);
-  auto size = rewriter.create<mlir::LLVM::ConstantOp>(
+      mlir::LLVM::AllocaOp::create(rewriter, loc, ptrTy, structTy, one);
+  auto size = mlir::LLVM::ConstantOp::create(rewriter,
       loc, i32Ty, rewriter.getIntegerAttr(i32Ty, structTypes.size()));
   mlir::Value argArray =
-      rewriter.create<mlir::LLVM::AllocaOp>(loc, ptrTy, ptrTy, size);
+      mlir::LLVM::AllocaOp::create(rewriter, loc, ptrTy, ptrTy, size);
 
   for (auto [i, arg] : llvm::enumerate(operands)) {
-    auto indice = rewriter.create<mlir::LLVM::ConstantOp>(
+    auto indice = mlir::LLVM::ConstantOp::create(rewriter,
         loc, i32Ty, rewriter.getIntegerAttr(i32Ty, i));
-    mlir::Value structMember = rewriter.create<LLVM::GEPOp>(
+    mlir::Value structMember = LLVM::GEPOp::create(rewriter,
         loc, ptrTy, structTy, argStruct,
         mlir::ArrayRef<mlir::Value>({zero, indice}));
-    rewriter.create<LLVM::StoreOp>(loc, arg, structMember);
-    mlir::Value arrayMember = rewriter.create<LLVM::GEPOp>(
+    LLVM::StoreOp::create(rewriter, loc, arg, structMember);
+    mlir::Value arrayMember = LLVM::GEPOp::create(rewriter,
         loc, ptrTy, ptrTy, argArray, mlir::ArrayRef<mlir::Value>({indice}));
-    rewriter.create<LLVM::StoreOp>(loc, structMember, arrayMember);
+    LLVM::StoreOp::create(rewriter, loc, structMember, arrayMember);
   }
   return argArray;
 }
@@ -94,7 +94,7 @@ struct GPULaunchKernelConversion
     mlir::Value dynamicMemorySize = op.getDynamicSharedMemorySize();
     mlir::Type i32Ty = rewriter.getI32Type();
     if (!dynamicMemorySize)
-      dynamicMemorySize = rewriter.create<mlir::LLVM::ConstantOp>(
+      dynamicMemorySize = mlir::LLVM::ConstantOp::create(rewriter,
           loc, i32Ty, rewriter.getIntegerAttr(i32Ty, 0));
 
     mlir::Value kernelArgs =
@@ -108,17 +108,17 @@ struct GPULaunchKernelConversion
       if (!funcOp)
         return mlir::failure();
       kernelPtr =
-          rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, funcOp.getName());
+          LLVM::AddressOfOp::create(rewriter, loc, ptrTy, funcOp.getName());
     } else {
       kernelPtr =
-          rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, kernel.getName());
+          LLVM::AddressOfOp::create(rewriter, loc, ptrTy, kernel.getName());
     }
 
     auto llvmIntPtrType = mlir::IntegerType::get(
         ctx, this->getTypeConverter()->getPointerBitwidth(0));
     auto voidTy = mlir::LLVM::LLVMVoidType::get(ctx);
 
-    mlir::Value nullPtr = rewriter.create<LLVM::ZeroOp>(loc, ptrTy);
+    mlir::Value nullPtr = LLVM::ZeroOp::create(rewriter, loc, ptrTy);
 
     if (op.hasClusterSize()) {
       auto funcOp = mod.lookupSymbol<mlir::LLVM::LLVMFuncOp>(
@@ -134,7 +134,7 @@ struct GPULaunchKernelConversion
       if (!funcOp) {
         mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
         rewriter.setInsertionPointToStart(mod.getBody());
-        auto launchKernelFuncOp = rewriter.create<mlir::LLVM::LLVMFuncOp>(
+        auto launchKernelFuncOp = mlir::LLVM::LLVMFuncOp::create(rewriter,
             loc, RTNAME_STRING(CUFLaunchClusterKernel), funcTy);
         launchKernelFuncOp.setVisibility(
             mlir::SymbolTable::Visibility::Private);
@@ -148,7 +148,7 @@ struct GPULaunchKernelConversion
         stream = adaptor.getAsyncDependencies().front();
       }
 
-      rewriter.create<mlir::LLVM::CallOp>(
+      mlir::LLVM::CallOp::create(rewriter,
           loc, funcTy, cufLaunchClusterKernel,
           mlir::ValueRange{kernelPtr, adaptor.getClusterSizeX(),
                            adaptor.getClusterSizeY(), adaptor.getClusterSizeZ(),
@@ -178,7 +178,7 @@ struct GPULaunchKernelConversion
         mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
         rewriter.setInsertionPointToStart(mod.getBody());
         auto launchKernelFuncOp =
-            rewriter.create<mlir::LLVM::LLVMFuncOp>(loc, fctName, funcTy);
+            mlir::LLVM::LLVMFuncOp::create(rewriter, loc, fctName, funcTy);
         launchKernelFuncOp.setVisibility(
             mlir::SymbolTable::Visibility::Private);
       }
@@ -191,7 +191,7 @@ struct GPULaunchKernelConversion
         stream = adaptor.getAsyncDependencies().front();
       }
 
-      rewriter.create<mlir::LLVM::CallOp>(
+      mlir::LLVM::CallOp::create(rewriter,
           loc, funcTy, cufLaunchKernel,
           mlir::ValueRange{kernelPtr, adaptor.getGridSizeX(),
                            adaptor.getGridSizeY(), adaptor.getGridSizeZ(),
@@ -222,10 +222,10 @@ static mlir::Value createAddressOfOp(mlir::ConversionPatternRewriter &rewriter,
   auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(
       rewriter.getContext(), mlir::NVVM::NVVMMemorySpace::kSharedMemorySpace);
   if (auto g = gpuMod.lookupSymbol<fir::GlobalOp>(sharedGlobalName))
-    return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
+    return mlir::LLVM::AddressOfOp::create(rewriter, loc, llvmPtrTy,
                                                     g.getSymName());
   if (auto g = gpuMod.lookupSymbol<mlir::LLVM::GlobalOp>(sharedGlobalName))
-    return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
+    return mlir::LLVM::AddressOfOp::create(rewriter, loc, llvmPtrTy,
                                                     g.getSymName());
   return {};
 }
@@ -255,12 +255,12 @@ struct CUFSharedMemoryOpConversion
     if (!sharedGlobalAddr)
       mlir::emitError(loc, "Could not find the shared global operation\n");
 
-    auto castPtr = rewriter.create<mlir::LLVM::AddrSpaceCastOp>(
+    auto castPtr = mlir::LLVM::AddrSpaceCastOp::create(rewriter,
         loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),
         sharedGlobalAddr);
     mlir::Type baseType = castPtr->getResultTypes().front();
     llvm::SmallVector<mlir::LLVM::GEPArg> gepArgs = {op.getOffset()};
-    mlir::Value shmemPtr = rewriter.create<mlir::LLVM::GEPOp>(
+    mlir::Value shmemPtr = mlir::LLVM::GEPOp::create(rewriter,
         loc, baseType, rewriter.getI8Type(), castPtr, gepArgs);
     rewriter.replaceOp(op, {shmemPtr});
     return mlir::success();
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 750569c126642..5e5ee2905e83a 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -89,7 +89,7 @@ static mlir::Value createConvertOp(mlir::PatternRewriter &rewriter,
                                    mlir::Location loc, mlir::Type toTy,
                                    mlir::Value val) {
   if (val.getType() != toTy)
-    return rewriter.create<fir::ConvertOp>(loc, toTy, val);
+    return fir::ConvertOp::create(rewriter, loc, toTy, val);
   return val;
 }
 
@@ -118,7 +118,7 @@ static mlir::LogicalResult convertOpToCall(OpTy op,
     errmsg = op.getErrmsg();
   } else {
     mlir::Type boxNoneTy = fir::BoxType::get(builder.getNoneType());
-    errmsg = builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+    errmsg = fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   }
   llvm::SmallVector<mlir::Value> args;
   if constexpr (std::is_same_v<OpTy, cuf::AllocateOp>) {
@@ -148,7 +148,7 @@ static mlir::LogicalResult convertOpToCall(OpTy op,
         fir::runtime::createArguments(builder, loc, fTy, op.getBox(), hasStat,
                                       errmsg, sourceFile, sourceLine);
   }
-  auto callOp = builder.create<fir::CallOp>(loc, func, args);
+  auto callOp = fir::CallOp::create(builder, loc, func, args);
   rewriter.replaceOp(op, callOp);
   return mlir::success();
 }
@@ -301,7 +301,7 @@ struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
     if (inDeviceContext(op.getOperation())) {
       // In device context just replace the cuf.alloc operation with a fir.alloc
       // the cuf.free will be removed.
-      auto allocaOp = rewriter.create<fir::AllocaOp>(
+      auto allocaOp = fir::AllocaOp::create(rewriter,
           loc, op.getInType(), op.getUniqName() ? *op.getUniqName() : "",
           op.getBindcName() ? *op.getBindcName() : "", op.getTypeparams(),
           op.getShape());
@@ -338,14 +338,14 @@ struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
           assert(!op.getShape().empty() && "expect shape with dynamic arrays");
           nbElem = builder.loadIfRef(loc, op.getShape()[0]);
           for (unsigned i = 1; i < op.getShape().size(); ++i) {
-            nbElem = rewriter.create<mlir::arith::MulIOp>(
+            nbElem = mlir::arith::MulIOp::create(rewriter,
                 loc, nbElem, builder.loadIfRef(loc, op.getShape()[i]));
           }
         } else {
           nbElem = builder.createIntegerConstant(loc, builder.getIndexType(),
                                                  seqTy.getConstantArraySize());
         }
-        bytes = rewriter.create<mlir::arith::MulIOp>(loc, nbElem, width);
+        bytes = mlir::arith::MulIOp::create(rewriter, loc, nbElem, width);
       } else if (fir::isa_derived(op.getInType())) {
         mlir::Type structTy = typeConverter->convertType(op.getInType());
         std::size_t structSize = dl->getTypeSizeInBits(structTy) / 8;
@@ -363,7 +363,7 @@ struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
           loc, builder.getI32Type(), getMemType(op.getDataAttr()));
       llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
           builder, loc, fTy, bytes, memTy, sourceFile, sourceLine)};
-      auto callOp = builder.create<fir::CallOp>(loc, func, args);
+      auto callOp = fir::CallOp::create(builder, loc, func, args);
       callOp->setAttr(cuf::getDataAttrName(), op.getDataAttrAttr());
       auto convOp = builder.createConvert(loc, op.getResult().getType(),
                                           callOp.getResult(0));
@@ -386,7 +386,7 @@ struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
 
     llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
         builder, loc, fTy, sizeInBytes, sourceFile, sourceLine)};
-    auto callOp = builder.create<fir::CallOp>(loc, func, args);
+    auto callOp = fir::CallOp::create(builder, loc, func, args);
     callOp->setAttr(cuf::getDataAttrName(), op.getDataAttrAttr());
     auto convOp = builder.createConvert(loc, op.getResult().getType(),
                                         callOp.getResult(0));
@@ -414,7 +414,7 @@ struct CUFDeviceAddressOpConversion
             op.getHostSymbol().getRootReference().getValue())) {
       auto mod = op->getParentOfType<mlir::ModuleOp>();
       mlir::Location loc = op.getLoc();
-      auto hostAddr = rewriter.create<fir::AddrOfOp>(
+      auto hostAddr = fir::AddrOfOp::create(rewriter,
           loc, fir::ReferenceType::get(global.getType()), op.getHostSymbol());
       fir::FirOpBuilder builder(rewriter, mod);
       mlir::func::FuncOp callee =
@@ -428,7 +428,7 @@ struct CUFDeviceAddressOpConversion
           fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
       llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
           builder, loc, fTy, conv, sourceFile, sourceLine)};
-      auto call = rewriter.create<fir::CallOp>(loc, callee, args);
+      auto call = fir::CallOp::create(rewriter, loc, callee, args);
       mlir::Value addr = createConvertOp(rewriter, loc, hostAddr.getType(),
                                          call->getResult(0));
       rewriter.replaceOp(op, addr.getDefiningOp());
@@ -456,7 +456,7 @@ struct DeclareOpConversion : public mlir::OpRewritePattern<fir::DeclareOp> {
               addrOfOp.getSymbol().getRootReference().getValue())) {
         if (cuf::isRegisteredDeviceGlobal(global)) {
           rewriter.setInsertionPointAfter(addrOfOp);
-          mlir::Value devAddr = rewriter.create<cuf::DeviceAddressOp>(
+          mlir::Value devAddr = cuf::DeviceAddressOp::create(rewriter,
               op.getLoc(), addrOfOp.getType(), addrOfOp.getSymbol());
           rewriter.startOpModification(op);
           op.getMemrefMutable().assign(devAddr);
@@ -502,7 +502,7 @@ struct CUFFreeOpConversion : public mlir::OpRewritePattern<cuf::FreeOp> {
           loc, builder.getI32Type(), getMemType(op.getDataAttr()));
       llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
           builder, loc, fTy, op.getDevptr(), memTy, sourceFile, sourceLine)};
-      builder.create<fir::CallOp>(loc, func, args);
+      fir::CallOp::create(builder, loc, func, args);
       rewriter.eraseOp(op);
       return mlir::success();
     }
@@ -515,7 +515,7 @@ struct CUFFreeOpConversion : public mlir::OpRewritePattern<cuf::FreeOp> {
         fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
     llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
         builder, loc, fTy, op.getDevptr(), sourceFile, sourceLine)};
-    auto callOp = builder.create<fir::CallOp>(loc, func, args);
+    auto callOp = fir::CallOp::create(builder, loc, func, args);
     callOp->setAttr(cuf::getDataAttrName(), op.getDataAttrAttr());
     rewriter.eraseOp(op);
     return mlir::success();
@@ -558,18 +558,18 @@ static mlir::Value emboxSrc(mlir::PatternRewriter &rewriter,
       srcTy = fir::LogicalType::get(rewriter.getContext(), 4);
       src = createConvertOp(rewriter, loc, srcTy, src);
       addr = builder.createTemporary(loc, srcTy);
-      builder.create<fir::StoreOp>(loc, src, addr);
+      fir::StoreOp::create(builder, loc, src, addr);
     } else {
       if (dstEleTy && fir::isa_trivial(dstEleTy) && srcTy != dstEleTy) {
         // Use dstEleTy and convert to avoid assign mismatch.
         addr = builder.createTemporary(loc, dstEleTy);
-        auto conv = builder.create<fir::ConvertOp>(loc, dstEleTy, src);
-        builder.create<fir::StoreOp>(loc, conv, addr);
+        auto conv = fir::ConvertOp::create(builder, loc, dstEleTy, src);
+        fir::StoreOp::create(builder, loc, conv, addr);
         srcTy = dstEleTy;
       } else {
         // Put constant in memory if it is not.
         addr = builder.createTemporary(loc, srcTy);
-        builder.create<fir::StoreOp>(loc, src, addr);
+        fir::StoreOp::create(builder, loc, src, addr);
       }
     }
   } else {
@@ -582,7 +582,7 @@ static mlir::Value emboxSrc(mlir::PatternRewriter &rewriter,
                         /*slice=*/nullptr, lenParams,
                         /*tdesc=*/nullptr);
   mlir::Value src = builder.createTemporary(loc, box.getType());
-  builder.create<fir::StoreOp>(loc, box, src);
+  fir::StoreOp::create(builder, loc, box, src);
   return src;
 }
 
@@ -601,7 +601,7 @@ static mlir::Value emboxDst(mlir::PatternRewriter &rewriter,
                         /*slice=*/nullptr, lenParams,
                         /*tdesc=*/nullptr);
   mlir::Value dst = builder.createTemporary(loc, dstBox.getType());
-  builder.create<fir::StoreOp>(loc, dstBox, dst);
+  fir::StoreOp::create(builder, loc, dstBox, dst);
   return dst;
 }
 
@@ -660,7 +660,7 @@ struct CUFDataTransferOpConversion
             fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
         llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
             builder, loc, fTy, dst, src, modeValue, sourceFile, sourceLine)};
-        builder.create<fir::CallOp>(loc, func, args);
+        fir::CallOp::create(builder, loc, func, args);
         rewriter.eraseOp(op);
         return mlir::success();
       }
@@ -679,12 +679,12 @@ struct CUFDataTransferOpConversion
               extents.push_back(i.value());
         }
 
-        nbElement = rewriter.create<fir::ConvertOp>(loc, i64Ty, extents[0]);
+        nbElement = fir::ConvertOp::create(rewriter, loc, i64Ty, extents[0]);
         for (unsigned i = 1; i < extents.size(); ++i) {
           auto operand =
-              rewriter.create<fir::ConvertOp>(loc, i64Ty, extents[i]);
+              fir::ConvertOp::create(rewriter, loc, i64Ty, extents[i]);
           nbElement =
-              rewriter.create<mlir::arith::MulIOp>(loc, nbElement, operand);
+              mlir::arith::MulIOp::create(rewriter, loc, nbElement, operand);
         }
       } else {
         if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(dstTy))
@@ -699,11 +699,11 @@ struct CUFDataTransferOpConversion
       } else {
         width = computeWidth(loc, dstTy, kindMap);
       }
-      mlir::Value widthValue = rewriter.create<mlir::arith::ConstantOp>(
+      mlir::Value widthValue = mlir::arith::ConstantOp::create(rewriter,
           loc, i64Ty, rewriter.getIntegerAttr(i64Ty, width));
       mlir::Value bytes =
           nbElement
-              ? rewriter.create<mlir::arith::MulIOp>(loc, nbElement, widthValue)
+              ? mlir::arith::MulIOp::create(rewriter, loc, nbElement, widthValue)
               : widthValue;
 
       mlir::func::FuncOp func =
@@ -719,13 +719,13 @@ struct CUFDataTransferOpConversion
       // Materialize the src if constant.
       if (matchPattern(src.getDefiningOp(), mlir::m_Constant())) {
         mlir::Value temp = builder.createTemporary(loc, srcTy);
-        builder.create<fir::StoreOp>(loc, src, temp);
+        fir::StoreOp::create(builder, loc, src, temp);
         src = temp;
       }
       llvm::SmallVector<mlir::Value> args{
           fir::runtime::createArguments(builder, loc, fTy, dst, src, bytes,
                                         modeValue, sourceFile, sourceLine)};
-      builder.create<fir::CallOp>(loc, func, args);
+      fir::CallOp::create(builder, loc, func, args);
       rewriter.eraseOp(op);
       return mlir::success();
     }
@@ -734,7 +734,7 @@ struct CUFDataTransferOpConversion
       if (mlir::isa<fir::EmboxOp, fir::ReboxOp>(val.getDefiningOp())) {
         // Materialize the box to memory to be able to call the runtime.
         mlir::Value box = builder.createTemporary(loc, val.getType());
-        builder.create<fir::StoreOp>(loc, val, box);
+        fir::StoreOp::create(builder, loc, val, box);
         return box;
       }
       return val;
@@ -768,7 +768,7 @@ struct CUFDataTransferOpConversion
           fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
       llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
           builder, loc, fTy, dst, src, modeValue, sourceFile, sourceLine)};
-      builder.create<fir::CallOp>(loc, func, args);
+      fir::CallOp::create(builder, loc, func, args);
       rewriter.eraseOp(op);
     } else {
       // Transfer from a descriptor.
@@ -784,7 +784,7 @@ struct CUFDataTransferOpConversion
           fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
       llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
           builder, loc, fTy, dst, src, modeValue, sourceFile, sourceLine)};
-      builder.create<fir::CallOp>(loc, func, args);
+      fir::CallOp::create(builder, loc, func, args);
       rewriter.eraseOp(op);
     }
     return mlir::success();
@@ -810,20 +810,20 @@ struct CUFLaunchOpConversion
                   mlir::PatternRewriter &rewriter) const override {
     mlir::Location loc = op.getLoc();
     auto idxTy = mlir::IndexType::get(op.getContext());
-    mlir::Value zero = rewriter.create<mlir::arith::ConstantOp>(
+    mlir::Value zero = mlir::arith::ConstantOp::create(rewriter,
         loc, rewriter.getIntegerType(32), rewriter.getI32IntegerAttr(0));
     auto gridSizeX =
-        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getGridX());
+        mlir::arith::IndexCastOp::create(rewriter, loc, idxTy, op.getGridX());
     auto gridSizeY =
-        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getGridY());
+        mlir::arith::IndexCastOp::create(rewriter, loc, idxTy, op.getGridY());
     auto gridSizeZ =
-        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getGridZ());
+        mlir::arith::IndexCastOp::create(rewriter, loc, idxTy, op.getGridZ());
     auto blockSizeX =
-        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getBlockX());
+        mlir::arith::IndexCastOp::create(rewriter, loc, idxTy, op.getBlockX());
     auto blockSizeY =
-        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getBlockY());
+        mlir::arith::IndexCastOp::create(rewriter, loc, idxTy, op.getBlockY());
     auto blockSizeZ =
-        rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getBlockZ());
+        mlir::arith::IndexCastOp::create(rewriter, loc, idxTy, op.getBlockZ());
     auto kernelName = mlir::SymbolRefAttr::get(
         rewriter.getStringAttr(cudaDeviceModuleName),
         {mlir::SymbolRefAttr::get(
@@ -835,11 +835,11 @@ struct CUFLaunchOpConversion
             op.getCallee().getLeafReference())) {
       if (auto clusterDimsAttr = funcOp->getAttrOfType<cuf::ClusterDimsAttr>(
               cuf::getClusterDimsAttrName())) {
-        clusterDimX = rewriter.create<mlir::arith::ConstantIndexOp>(
+        clusterDimX = mlir::arith::ConstantIndexOp::create(rewriter,
             loc, clusterDimsAttr.getX().getInt());
-        clusterDimY = rewriter.create<mlir::arith::ConstantIndexOp>(
+        clusterDimY = mlir::arith::ConstantIndexOp::create(rewriter,
             loc, clusterDimsAttr.getY().getInt());
-        clusterDimZ = rewriter.create<mlir::arith::ConstantIndexOp>(
+        clusterDimZ = mlir::arith::ConstantIndexOp::create(rewriter,
             loc, clusterDimsAttr.getZ().getInt());
       }
       procAttr =
@@ -870,7 +870,7 @@ struct CUFLaunchOpConversion
       args.push_back(arg);
     }
     mlir::Value dynamicShmemSize = op.getBytes() ? op.getBytes() : zero;
-    auto gpuLaunchOp = rewriter.create<mlir::gpu::LaunchFuncOp>(
+    auto gpuLaunchOp = mlir::gpu::LaunchFuncOp::create(rewriter,
         loc, kernelName, mlir::gpu::KernelDim3{gridSizeX, gridSizeY, gridSizeZ},
         mlir::gpu::KernelDim3{blockSizeX, blockSizeY, blockSizeZ},
         dynamicShmemSize, args);
@@ -883,7 +883,7 @@ struct CUFLaunchOpConversion
       mlir::OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPoint(gpuLaunchOp);
       mlir::Value stream =
-          rewriter.create<cuf::StreamCastOp>(loc, op.getStream());
+          cuf::StreamCastOp::create(rewriter, loc, op.getStream());
       gpuLaunchOp.getAsyncDependenciesMutable().append(stream);
     }
     if (procAttr)
@@ -916,7 +916,7 @@ struct CUFSyncDescriptorOpConversion
     if (!globalOp)
       return mlir::failure();
 
-    auto hostAddr = builder.create<fir::AddrOfOp>(
+    auto hostAddr = fir::AddrOfOp::create(builder,
         loc, fir::ReferenceType::get(globalOp.getType()), op.getGlobalName());
     fir::runtime::cuda::genSyncGlobalDescriptor(builder, loc, hostAddr);
     op.erase();
diff --git a/flang/lib/Optimizer/Transforms/CharacterConversion.cpp b/flang/lib/Optimizer/Transforms/CharacterConversion.cpp
index aee7e8ca5cb66..2c2db1d4d6ded 100644
--- a/flang/lib/Optimizer/Transforms/CharacterConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CharacterConversion.cpp
@@ -48,12 +48,12 @@ class CharacterConvertConversion
                << "running character conversion on " << conv << '\n');
 
     // Establish a loop that executes count iterations.
-    auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
-    auto one = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 1);
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto idxTy = rewriter.getIndexType();
-    auto castCnt = rewriter.create<fir::ConvertOp>(loc, idxTy, conv.getCount());
-    auto countm1 = rewriter.create<mlir::arith::SubIOp>(loc, castCnt, one);
-    auto loop = rewriter.create<fir::DoLoopOp>(loc, zero, countm1, one);
+    auto castCnt = fir::ConvertOp::create(rewriter, loc, idxTy, conv.getCount());
+    auto countm1 = mlir::arith::SubIOp::create(rewriter, loc, castCnt, one);
+    auto loop = fir::DoLoopOp::create(rewriter, loc, zero, countm1, one);
     auto insPt = rewriter.saveInsertionPoint();
     rewriter.setInsertionPointToStart(loop.getBody());
 
@@ -75,21 +75,21 @@ class CharacterConvertConversion
     auto toTy = rewriter.getIntegerType(toBits);
     auto toPtrTy = pointerType(toBits);
     auto fromPtr =
-        rewriter.create<fir::ConvertOp>(loc, fromPtrTy, conv.getFrom());
-    auto toPtr = rewriter.create<fir::ConvertOp>(loc, toPtrTy, conv.getTo());
+        fir::ConvertOp::create(rewriter, loc, fromPtrTy, conv.getFrom());
+    auto toPtr = fir::ConvertOp::create(rewriter, loc, toPtrTy, conv.getTo());
     auto getEleTy = [&](unsigned bits) {
       return fir::ReferenceType::get(rewriter.getIntegerType(bits));
     };
-    auto fromi = rewriter.create<fir::CoordinateOp>(
+    auto fromi = fir::CoordinateOp::create(rewriter,
         loc, getEleTy(fromBits), fromPtr,
         mlir::ValueRange{loop.getInductionVar()});
-    auto toi = rewriter.create<fir::CoordinateOp>(
+    auto toi = fir::CoordinateOp::create(rewriter,
         loc, getEleTy(toBits), toPtr, mlir::ValueRange{loop.getInductionVar()});
-    auto load = rewriter.create<fir::LoadOp>(loc, fromi);
+    auto load = fir::LoadOp::create(rewriter, loc, fromi);
     mlir::Value icast =
         (fromBits >= toBits)
-            ? rewriter.create<fir::ConvertOp>(loc, toTy, load).getResult()
-            : rewriter.create<mlir::arith::ExtUIOp>(loc, toTy, load)
+            ? fir::ConvertOp::create(rewriter, loc, toTy, load).getResult()
+            : mlir::arith::ExtUIOp::create(rewriter, loc, toTy, load)
                   .getResult();
     rewriter.replaceOpWithNewOp<fir::StoreOp>(conv, icast, toi);
     rewriter.restoreInsertionPoint(insPt);
diff --git a/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp b/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
index 239a7cdaa4cf2..09cec03ee0916 100644
--- a/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
+++ b/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
@@ -111,10 +111,10 @@ class CallOpRewriter : public mlir::OpRewritePattern<fir::CallOp> {
             builder.insert(cln);
             mlir::Value val =
                 builder.createConvert(loc, varTy, cln->getResult(0));
-            builder.create<fir::HasValueOp>(loc, val);
+            fir::HasValueOp::create(builder, loc, val);
           },
           builder.createInternalLinkage());
-      mlir::Value addr = builder.create<fir::AddrOfOp>(loc, global.resultType(),
+      mlir::Value addr = fir::AddrOfOp::create(builder, loc, global.resultType(),
                                                        global.getSymbol());
       newOperands.push_back(addr);
       needUpdate = true;
@@ -125,7 +125,7 @@ class CallOpRewriter : public mlir::OpRewritePattern<fir::CallOp> {
       llvm::SmallVector<mlir::Type> newResultTypes;
       newResultTypes.append(callOp.getResultTypes().begin(),
                             callOp.getResultTypes().end());
-      fir::CallOp newOp = builder.create<fir::CallOp>(
+      fir::CallOp newOp = fir::CallOp::create(builder,
           loc,
           callOp.getCallee().has_value() ? callOp.getCallee().value()
                                          : mlir::SymbolRefAttr{},
diff --git a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
index 3d35803e6a2d3..0c070ed6d9c3f 100644
--- a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
+++ b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
@@ -83,17 +83,17 @@ class CfgLoopConv : public mlir::OpRewritePattern<fir::DoLoopOp> {
 
     // Initalization block
     rewriter.setInsertionPointToEnd(initBlock);
-    auto diff = rewriter.create<mlir::arith::SubIOp>(loc, high, low);
-    auto distance = rewriter.create<mlir::arith::AddIOp>(loc, diff, step);
+    auto diff = mlir::arith::SubIOp::create(rewriter, loc, high, low);
+    auto distance = mlir::arith::AddIOp::create(rewriter, loc, diff, step);
     mlir::Value iters =
-        rewriter.create<mlir::arith::DivSIOp>(loc, distance, step);
+        mlir::arith::DivSIOp::create(rewriter, loc, distance, step);
 
     if (forceLoopToExecuteOnce) {
-      auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
-      auto cond = rewriter.create<mlir::arith::CmpIOp>(
+      auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
+      auto cond = mlir::arith::CmpIOp::create(rewriter,
           loc, arith::CmpIPredicate::sle, iters, zero);
-      auto one = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 1);
-      iters = rewriter.create<mlir::arith::SelectOp>(loc, cond, one, iters);
+      auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
+      iters = mlir::arith::SelectOp::create(rewriter, loc, cond, one, iters);
     }
 
     llvm::SmallVector<mlir::Value> loopOperands;
@@ -102,20 +102,20 @@ class CfgLoopConv : public mlir::OpRewritePattern<fir::DoLoopOp> {
     loopOperands.append(operands.begin(), operands.end());
     loopOperands.push_back(iters);
 
-    rewriter.create<mlir::cf::BranchOp>(loc, conditionalBlock, loopOperands);
+    mlir::cf::BranchOp::create(rewriter, loc, conditionalBlock, loopOperands);
 
     // Last loop block
     auto *terminator = lastBlock->getTerminator();
     rewriter.setInsertionPointToEnd(lastBlock);
     auto iv = conditionalBlock->getArgument(0);
     mlir::Value steppedIndex =
-        rewriter.create<mlir::arith::AddIOp>(loc, iv, step, iofAttr);
+        mlir::arith::AddIOp::create(rewriter, loc, iv, step, iofAttr);
     assert(steppedIndex && "must be a Value");
     auto lastArg = conditionalBlock->getNumArguments() - 1;
     auto itersLeft = conditionalBlock->getArgument(lastArg);
-    auto one = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 1);
+    auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
     mlir::Value itersMinusOne =
-        rewriter.create<mlir::arith::SubIOp>(loc, itersLeft, one);
+        mlir::arith::SubIOp::create(rewriter, loc, itersLeft, one);
 
     llvm::SmallVector<mlir::Value> loopCarried;
     loopCarried.push_back(steppedIndex);
@@ -124,7 +124,7 @@ class CfgLoopConv : public mlir::OpRewritePattern<fir::DoLoopOp> {
     loopCarried.append(begin, terminator->operand_end());
     loopCarried.push_back(itersMinusOne);
     auto backEdge =
-        rewriter.create<mlir::cf::BranchOp>(loc, conditionalBlock, loopCarried);
+        mlir::cf::BranchOp::create(rewriter, loc, conditionalBlock, loopCarried);
     rewriter.eraseOp(terminator);
 
     // Copy loop annotations from the do loop to the loop back edge.
@@ -133,11 +133,11 @@ class CfgLoopConv : public mlir::OpRewritePattern<fir::DoLoopOp> {
 
     // Conditional block
     rewriter.setInsertionPointToEnd(conditionalBlock);
-    auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
-    auto comparison = rewriter.create<mlir::arith::CmpIOp>(
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto comparison = mlir::arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sgt, itersLeft, zero);
 
-    rewriter.create<mlir::cf::CondBranchOp>(
+    mlir::cf::CondBranchOp::create(rewriter,
         loc, comparison, firstBlock, llvm::ArrayRef<mlir::Value>(), endBlock,
         llvm::ArrayRef<mlir::Value>());
 
@@ -180,7 +180,7 @@ class CfgIfConv : public mlir::OpRewritePattern<fir::IfOp> {
       continueBlock = rewriter.createBlock(
           remainingOpsBlock, ifOp.getResultTypes(),
           llvm::SmallVector<mlir::Location>(ifOp.getNumResults(), loc));
-      rewriter.create<mlir::cf::BranchOp>(loc, remainingOpsBlock);
+      mlir::cf::BranchOp::create(rewriter, loc, remainingOpsBlock);
     }
 
     // Move blocks from the "then" region to the region containing 'fir.if',
@@ -190,7 +190,7 @@ class CfgIfConv : public mlir::OpRewritePattern<fir::IfOp> {
     auto *ifOpTerminator = ifOpRegion.back().getTerminator();
     auto ifOpTerminatorOperands = ifOpTerminator->getOperands();
     rewriter.setInsertionPointToEnd(&ifOpRegion.back());
-    rewriter.create<mlir::cf::BranchOp>(loc, continueBlock,
+    mlir::cf::BranchOp::create(rewriter, loc, continueBlock,
                                         ifOpTerminatorOperands);
     rewriter.eraseOp(ifOpTerminator);
     rewriter.inlineRegionBefore(ifOpRegion, continueBlock);
@@ -205,14 +205,14 @@ class CfgIfConv : public mlir::OpRewritePattern<fir::IfOp> {
       auto *otherwiseTerm = otherwiseRegion.back().getTerminator();
       auto otherwiseTermOperands = otherwiseTerm->getOperands();
       rewriter.setInsertionPointToEnd(&otherwiseRegion.back());
-      rewriter.create<mlir::cf::BranchOp>(loc, continueBlock,
+      mlir::cf::BranchOp::create(rewriter, loc, continueBlock,
                                           otherwiseTermOperands);
       rewriter.eraseOp(otherwiseTerm);
       rewriter.inlineRegionBefore(otherwiseRegion, continueBlock);
     }
 
     rewriter.setInsertionPointToEnd(condBlock);
-    auto branchOp = rewriter.create<mlir::cf::CondBranchOp>(
+    auto branchOp = mlir::cf::CondBranchOp::create(rewriter,
         loc, ifOp.getCondition(), ifOpBlock, llvm::ArrayRef<mlir::Value>(),
         otherwiseBlock, llvm::ArrayRef<mlir::Value>());
     llvm::ArrayRef<int32_t> weights = ifOp.getWeights();
@@ -269,7 +269,7 @@ class CfgIterWhileConv : public mlir::OpRewritePattern<fir::IterWhileOp> {
     rewriter.setInsertionPointToEnd(lastBodyBlock);
     auto step = whileOp.getStep();
     mlir::Value stepped =
-        rewriter.create<mlir::arith::AddIOp>(loc, iv, step, iofAttr);
+        mlir::arith::AddIOp::create(rewriter, loc, iv, step, iofAttr);
     assert(stepped && "must be a Value");
 
     llvm::SmallVector<mlir::Value> loopCarried;
@@ -278,7 +278,7 @@ class CfgIterWhileConv : public mlir::OpRewritePattern<fir::IterWhileOp> {
                      ? std::next(terminator->operand_begin())
                      : terminator->operand_begin();
     loopCarried.append(begin, terminator->operand_end());
-    rewriter.create<mlir::cf::BranchOp>(loc, conditionBlock, loopCarried);
+    mlir::cf::BranchOp::create(rewriter, loc, conditionBlock, loopCarried);
     rewriter.eraseOp(terminator);
 
     // Compute loop bounds before branching to the condition.
@@ -293,29 +293,29 @@ class CfgIterWhileConv : public mlir::OpRewritePattern<fir::IterWhileOp> {
     destOperands.push_back(lowerBound);
     auto iterOperands = whileOp.getIterOperands();
     destOperands.append(iterOperands.begin(), iterOperands.end());
-    rewriter.create<mlir::cf::BranchOp>(loc, conditionBlock, destOperands);
+    mlir::cf::BranchOp::create(rewriter, loc, conditionBlock, destOperands);
 
     // With the body block done, we can fill in the condition block.
     rewriter.setInsertionPointToEnd(conditionBlock);
     // The comparison depends on the sign of the step value. We fully expect
     // this expression to be folded by the optimizer or LLVM. This expression
     // is written this way so that `step == 0` always returns `false`.
-    auto zero = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
-    auto compl0 = rewriter.create<mlir::arith::CmpIOp>(
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto compl0 = mlir::arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, zero, step);
-    auto compl1 = rewriter.create<mlir::arith::CmpIOp>(
+    auto compl1 = mlir::arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sle, iv, upperBound);
-    auto compl2 = rewriter.create<mlir::arith::CmpIOp>(
+    auto compl2 = mlir::arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::slt, step, zero);
-    auto compl3 = rewriter.create<mlir::arith::CmpIOp>(
+    auto compl3 = mlir::arith::CmpIOp::create(rewriter,
         loc, arith::CmpIPredicate::sle, upperBound, iv);
-    auto cmp0 = rewriter.create<mlir::arith::AndIOp>(loc, compl0, compl1);
-    auto cmp1 = rewriter.create<mlir::arith::AndIOp>(loc, compl2, compl3);
-    auto cmp2 = rewriter.create<mlir::arith::OrIOp>(loc, cmp0, cmp1);
+    auto cmp0 = mlir::arith::AndIOp::create(rewriter, loc, compl0, compl1);
+    auto cmp1 = mlir::arith::AndIOp::create(rewriter, loc, compl2, compl3);
+    auto cmp2 = mlir::arith::OrIOp::create(rewriter, loc, cmp0, cmp1);
     // Remember to AND in the early-exit bool.
     auto comparison =
-        rewriter.create<mlir::arith::AndIOp>(loc, iterateVar, cmp2);
-    rewriter.create<mlir::cf::CondBranchOp>(
+        mlir::arith::AndIOp::create(rewriter, loc, iterateVar, cmp2);
+    mlir::cf::CondBranchOp::create(rewriter,
         loc, comparison, firstBodyBlock, llvm::ArrayRef<mlir::Value>(),
         endBlock, llvm::ArrayRef<mlir::Value>());
     // The result of the loop operation is the values of the condition block
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index abad500d3f657..8d16c8cc74044 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -103,13 +103,13 @@ mlir::LLVM::DILocalVariableAttr DebugTypeGenerator::generateArtificialVariable(
   mlir::Type type = val.getType();
   if (!mlir::isa<mlir::IntegerType>(type) || !type.isSignlessInteger()) {
     type = builder.getIntegerType(64);
-    val = builder.create<fir::ConvertOp>(declOp.getLoc(), type, val);
+    val = fir::ConvertOp::create(builder, declOp.getLoc(), type, val);
   }
   mlir::LLVM::DITypeAttr Ty = convertType(type, fileAttr, scope, declOp);
   auto lvAttr = mlir::LLVM::DILocalVariableAttr::get(
       context, scope, name, fileAttr, /*line=*/0, /*argNo=*/0,
       /*alignInBits=*/0, Ty, mlir::LLVM::DIFlags::Artificial);
-  builder.create<mlir::LLVM::DbgValueOp>(declOp.getLoc(), val, lvAttr, nullptr);
+  mlir::LLVM::DbgValueOp::create(builder, declOp.getLoc(), val, lvAttr, nullptr);
   return lvAttr;
 }
 
diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index f06ad2db90d55..d7d1865bc56ba 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -49,13 +49,13 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     // must be a positive value.
     // For easier conversion, we calculate the trip count and use a canonical
     // induction variable.
-    auto diff = rewriter.create<arith::SubIOp>(loc, high, low);
-    auto distance = rewriter.create<arith::AddIOp>(loc, diff, step);
-    auto tripCount = rewriter.create<arith::DivSIOp>(loc, distance, step);
-    auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    auto diff = arith::SubIOp::create(rewriter, loc, high, low);
+    auto distance = arith::AddIOp::create(rewriter, loc, diff, step);
+    auto tripCount = arith::DivSIOp::create(rewriter, loc, distance, step);
+    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto scfForOp =
-        rewriter.create<scf::ForOp>(loc, zero, tripCount, one, iterArgs);
+        scf::ForOp::create(rewriter, loc, zero, tripCount, one, iterArgs);
 
     auto &loopOps = doLoopOp.getBody()->getOperations();
     auto resultOp = cast<fir::ResultOp>(doLoopOp.getBody()->getTerminator());
@@ -68,12 +68,12 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
 
     rewriter.setInsertionPointToStart(loweredBody);
     Value iv =
-        rewriter.create<arith::MulIOp>(loc, scfForOp.getInductionVar(), step);
-    iv = rewriter.create<arith::AddIOp>(loc, low, iv);
+        arith::MulIOp::create(rewriter, loc, scfForOp.getInductionVar(), step);
+    iv = arith::AddIOp::create(rewriter, loc, low, iv);
 
     if (!results.empty()) {
       rewriter.setInsertionPointToEnd(loweredBody);
-      rewriter.create<scf::YieldOp>(resultOp->getLoc(), results);
+      scf::YieldOp::create(rewriter, resultOp->getLoc(), results);
     }
     doLoopOp.getInductionVar().replaceAllUsesWith(iv);
     rewriter.replaceAllUsesWith(doLoopOp.getRegionIterArgs(),
diff --git a/flang/lib/Optimizer/Transforms/GenRuntimeCallsForTest.cpp b/flang/lib/Optimizer/Transforms/GenRuntimeCallsForTest.cpp
index 7ea3b9c670c69..699be12178881 100644
--- a/flang/lib/Optimizer/Transforms/GenRuntimeCallsForTest.cpp
+++ b/flang/lib/Optimizer/Transforms/GenRuntimeCallsForTest.cpp
@@ -91,8 +91,8 @@ void GenRuntimeCallsForTestPass::runOnOperation() {
     // Generate the wrapper function body that consists of a call and return.
     builder.setInsertionPointToStart(callerFunc.addEntryBlock());
     mlir::Block::BlockArgListType args = callerFunc.front().getArguments();
-    auto callOp = builder.create<fir::CallOp>(loc, funcOp, args);
-    builder.create<mlir::func::ReturnOp>(loc, callOp.getResults());
+    auto callOp = fir::CallOp::create(builder, loc, funcOp, args);
+    mlir::func::ReturnOp::create(builder, loc, callOp.getResults());
 
     newFuncs.push_back(callerFunc.getOperation());
     builder.restoreInsertionPoint(insertPt);
diff --git a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
index 056bdf63d914f..dbbc740a6f027 100644
--- a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
@@ -285,7 +285,7 @@ static mlir::Value getIndex(fir::FirOpBuilder &builder, mlir::Operation *op,
   // index_0 = index - lb;
   if (lb.getType() != index.getType())
     lb = builder.createConvert(coop.getLoc(), index.getType(), lb);
-  return builder.create<mlir::arith::SubIOp>(coop.getLoc(), index, lb);
+  return mlir::arith::SubIOp::create(builder, coop.getLoc(), index, lb);
 }
 
 void LoopVersioningPass::runOnOperation() {
@@ -483,25 +483,25 @@ void LoopVersioningPass::runOnOperation() {
       unsigned ndims = arg.rank;
       for (unsigned i = 0; i < ndims; i++) {
         mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i);
-        arg.dims[i] = builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy,
+        arg.dims[i] = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
                                                      arg.arg, dimIdx);
       }
       // We only care about lowest order dimension, here.
       mlir::Value elemSize =
           builder.createIntegerConstant(loc, idxTy, arg.size);
-      mlir::Value cmp = builder.create<mlir::arith::CmpIOp>(
+      mlir::Value cmp = mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::eq, arg.dims[0].getResult(2),
           elemSize);
       if (!allCompares) {
         allCompares = cmp;
       } else {
         allCompares =
-            builder.create<mlir::arith::AndIOp>(loc, cmp, allCompares);
+            mlir::arith::AndIOp::create(builder, loc, cmp, allCompares);
       }
     }
 
     auto ifOp =
-        builder.create<fir::IfOp>(loc, op.op->getResultTypes(), allCompares,
+        fir::IfOp::create(builder, loc, op.op->getResultTypes(), allCompares,
                                   /*withElse=*/true);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
@@ -515,8 +515,8 @@ void LoopVersioningPass::runOnOperation() {
       mlir::Type arrTy = fir::SequenceType::get(newShape, elementType);
       mlir::Type boxArrTy = fir::BoxType::get(arrTy);
       mlir::Type refArrTy = builder.getRefType(arrTy);
-      auto carg = builder.create<fir::ConvertOp>(loc, boxArrTy, arg.arg);
-      auto caddr = builder.create<fir::BoxAddrOp>(loc, refArrTy, carg);
+      auto carg = fir::ConvertOp::create(builder, loc, boxArrTy, arg.arg);
+      auto caddr = fir::BoxAddrOp::create(builder, loc, refArrTy, carg);
       auto insPt = builder.saveInsertionPoint();
       // Use caddr instead of arg.
       clonedLoop->walk([&](mlir::Operation *coop) {
@@ -540,8 +540,8 @@ void LoopVersioningPass::runOnOperation() {
             mlir::Value scale =
                 builder.createConvert(loc, idxTy, arg.dims[i].getResult(2));
             curIndex =
-                builder.create<mlir::arith::MulIOp>(loc, scale, curIndex);
-            totalIndex = (totalIndex) ? builder.create<mlir::arith::AddIOp>(
+                mlir::arith::MulIOp::create(builder, loc, scale, curIndex);
+            totalIndex = (totalIndex) ? mlir::arith::AddIOp::create(builder,
                                             loc, curIndex, totalIndex)
                                       : curIndex;
           }
@@ -554,15 +554,15 @@ void LoopVersioningPass::runOnOperation() {
             unsigned bits = llvm::Log2_32(arg.size);
             mlir::Value elemShift =
                 builder.createIntegerConstant(loc, idxTy, bits);
-            totalIndex = builder.create<mlir::arith::AddIOp>(
+            totalIndex = mlir::arith::AddIOp::create(builder,
                 loc,
-                builder.create<mlir::arith::ShRSIOp>(loc, totalIndex,
+                mlir::arith::ShRSIOp::create(builder, loc, totalIndex,
                                                      elemShift),
                 finalIndex);
           } else {
             totalIndex = finalIndex;
           }
-          auto newOp = builder.create<fir::CoordinateOp>(
+          auto newOp = fir::CoordinateOp::create(builder,
               loc, builder.getRefType(elementType), caddr,
               mlir::ValueRange{totalIndex});
           LLVM_DEBUG(newOp->dump());
@@ -582,7 +582,7 @@ void LoopVersioningPass::runOnOperation() {
     mlir::ResultRange results = clonedLoop->getResults();
     bool hasResults = (results.size() > 0);
     if (hasResults)
-      builder.create<fir::ResultOp>(loc, results);
+      fir::ResultOp::create(builder, loc, results);
 
     // Add the original loop in the else-side of the if operation.
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -591,7 +591,7 @@ void LoopVersioningPass::runOnOperation() {
     builder.insert(op.op);
     // Rely on "cloned loop has results, so original loop also has results".
     if (hasResults) {
-      builder.create<fir::ResultOp>(loc, op.op->getResults());
+      fir::ResultOp::create(builder, loc, op.op->getResults());
     } else {
       // Use an assert to check this.
       assert(op.op->getResults().size() == 0 &&
diff --git a/flang/lib/Optimizer/Transforms/MemoryAllocation.cpp b/flang/lib/Optimizer/Transforms/MemoryAllocation.cpp
index 3f308a8f4b560..1a0bb6c98e1d8 100644
--- a/flang/lib/Optimizer/Transforms/MemoryAllocation.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryAllocation.cpp
@@ -68,7 +68,7 @@ static mlir::Value genAllocmem(mlir::OpBuilder &builder, fir::AllocaOp alloca,
   };
   llvm::StringRef uniqName = unpackName(alloca.getUniqName());
   llvm::StringRef bindcName = unpackName(alloca.getBindcName());
-  auto heap = builder.create<fir::AllocMemOp>(alloca.getLoc(), varTy, uniqName,
+  auto heap = fir::AllocMemOp::create(builder, alloca.getLoc(), varTy, uniqName,
                                               bindcName, alloca.getTypeparams(),
                                               alloca.getShape());
   LLVM_DEBUG(llvm::dbgs() << "memory allocation opt: replaced " << alloca
@@ -78,7 +78,7 @@ static mlir::Value genAllocmem(mlir::OpBuilder &builder, fir::AllocaOp alloca,
 
 static void genFreemem(mlir::Location loc, mlir::OpBuilder &builder,
                        mlir::Value allocmem) {
-  [[maybe_unused]] auto free = builder.create<fir::FreeMemOp>(loc, allocmem);
+  [[maybe_unused]] auto free = fir::FreeMemOp::create(builder, loc, allocmem);
   LLVM_DEBUG(llvm::dbgs() << "memory allocation opt: add free " << free
                           << " for " << allocmem << '\n');
 }
diff --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index bc4fcd8b0112e..d295758abd852 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -200,29 +200,29 @@ void AllocaReplaceImpl::genIndirectDeallocation(
   // and access it indirectly in the entry points that do not dominate.
   rewriter.setInsertionPointToStart(&owningRegion.front());
   mlir::Type heapType = fir::HeapType::get(alloca.getInType());
-  mlir::Value ptrVar = rewriter.create<fir::AllocaOp>(loc, heapType);
-  mlir::Value nullPtr = rewriter.create<fir::ZeroOp>(loc, heapType);
-  rewriter.create<fir::StoreOp>(loc, nullPtr, ptrVar);
+  mlir::Value ptrVar = fir::AllocaOp::create(rewriter, loc, heapType);
+  mlir::Value nullPtr = fir::ZeroOp::create(rewriter, loc, heapType);
+  fir::StoreOp::create(rewriter, loc, nullPtr, ptrVar);
   // TODO: introducing a pointer compare op in FIR would help
   // generating less IR here.
   mlir::Type intPtrTy = fir::getIntPtrType(rewriter);
-  mlir::Value c0 = rewriter.create<mlir::arith::ConstantOp>(
+  mlir::Value c0 = mlir::arith::ConstantOp::create(rewriter,
       loc, intPtrTy, rewriter.getIntegerAttr(intPtrTy, 0));
 
   // Store new storage address right after its creation.
   rewriter.restoreInsertionPoint(replacementInsertPoint);
   mlir::Value castReplacement =
       fir::factory::createConvert(rewriter, loc, heapType, replacement);
-  rewriter.create<fir::StoreOp>(loc, castReplacement, ptrVar);
+  fir::StoreOp::create(rewriter, loc, castReplacement, ptrVar);
 
   // Generate conditional deallocation at every deallocation point.
   auto genConditionalDealloc = [&](mlir::Location loc) {
-    mlir::Value ptrVal = rewriter.create<fir::LoadOp>(loc, ptrVar);
+    mlir::Value ptrVal = fir::LoadOp::create(rewriter, loc, ptrVar);
     mlir::Value ptrToInt =
-        rewriter.create<fir::ConvertOp>(loc, intPtrTy, ptrVal);
-    mlir::Value isAllocated = rewriter.create<mlir::arith::CmpIOp>(
+        fir::ConvertOp::create(rewriter, loc, intPtrTy, ptrVal);
+    mlir::Value isAllocated = mlir::arith::CmpIOp::create(rewriter,
         loc, mlir::arith::CmpIPredicate::ne, ptrToInt, c0);
-    auto ifOp = rewriter.create<fir::IfOp>(loc, mlir::TypeRange{}, isAllocated,
+    auto ifOp = fir::IfOp::create(rewriter, loc, mlir::TypeRange{}, isAllocated,
                                            /*withElseRegion=*/false);
     rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
     mlir::Value cast = fir::factory::createConvert(
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 6e45aae4246d0..29b7d9dd77bd2 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -183,49 +183,49 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
     mlir::Type tdescType =
         fir::TypeDescType::get(mlir::NoneType::get(rewriter.getContext()));
     mlir::Value boxDesc =
-        rewriter.create<fir::BoxTypeDescOp>(loc, tdescType, passedObject);
-    boxDesc = rewriter.create<fir::ConvertOp>(
+        fir::BoxTypeDescOp::create(rewriter, loc, tdescType, passedObject);
+    boxDesc = fir::ConvertOp::create(rewriter,
         loc, fir::ReferenceType::get(typeDescTy), boxDesc);
 
     // Load the bindings descriptor.
     auto bindingsCompName = Fortran::semantics::bindingDescCompName;
     fir::RecordType typeDescRecTy = mlir::cast<fir::RecordType>(typeDescTy);
-    mlir::Value field = rewriter.create<fir::FieldIndexOp>(
+    mlir::Value field = fir::FieldIndexOp::create(rewriter,
         loc, fieldTy, bindingsCompName, typeDescRecTy, mlir::ValueRange{});
     mlir::Type coorTy =
         fir::ReferenceType::get(typeDescRecTy.getType(bindingsCompName));
     mlir::Value bindingBoxAddr =
-        rewriter.create<fir::CoordinateOp>(loc, coorTy, boxDesc, field);
-    mlir::Value bindingBox = rewriter.create<fir::LoadOp>(loc, bindingBoxAddr);
+        fir::CoordinateOp::create(rewriter, loc, coorTy, boxDesc, field);
+    mlir::Value bindingBox = fir::LoadOp::create(rewriter, loc, bindingBoxAddr);
 
     // Load the correct binding.
-    mlir::Value bindings = rewriter.create<fir::BoxAddrOp>(loc, bindingBox);
+    mlir::Value bindings = fir::BoxAddrOp::create(rewriter, loc, bindingBox);
     fir::RecordType bindingTy = fir::unwrapIfDerived(
         mlir::cast<fir::BaseBoxType>(bindingBox.getType()));
     mlir::Type bindingAddrTy = fir::ReferenceType::get(bindingTy);
-    mlir::Value bindingIdxVal = rewriter.create<mlir::arith::ConstantOp>(
+    mlir::Value bindingIdxVal = mlir::arith::ConstantOp::create(rewriter,
         loc, rewriter.getIndexType(), rewriter.getIndexAttr(bindingIdx));
-    mlir::Value bindingAddr = rewriter.create<fir::CoordinateOp>(
+    mlir::Value bindingAddr = fir::CoordinateOp::create(rewriter,
         loc, bindingAddrTy, bindings, bindingIdxVal);
 
     // Get the function pointer.
     auto procCompName = Fortran::semantics::procCompName;
-    mlir::Value procField = rewriter.create<fir::FieldIndexOp>(
+    mlir::Value procField = fir::FieldIndexOp::create(rewriter,
         loc, fieldTy, procCompName, bindingTy, mlir::ValueRange{});
     fir::RecordType procTy =
         mlir::cast<fir::RecordType>(bindingTy.getType(procCompName));
     mlir::Type procRefTy = fir::ReferenceType::get(procTy);
-    mlir::Value procRef = rewriter.create<fir::CoordinateOp>(
+    mlir::Value procRef = fir::CoordinateOp::create(rewriter,
         loc, procRefTy, bindingAddr, procField);
 
     auto addressFieldName = Fortran::lower::builtin::cptrFieldName;
-    mlir::Value addressField = rewriter.create<fir::FieldIndexOp>(
+    mlir::Value addressField = fir::FieldIndexOp::create(rewriter,
         loc, fieldTy, addressFieldName, procTy, mlir::ValueRange{});
     mlir::Type addressTy = procTy.getType(addressFieldName);
     mlir::Type addressRefTy = fir::ReferenceType::get(addressTy);
-    mlir::Value addressRef = rewriter.create<fir::CoordinateOp>(
+    mlir::Value addressRef = fir::CoordinateOp::create(rewriter,
         loc, addressRefTy, procRef, addressField);
-    mlir::Value address = rewriter.create<fir::LoadOp>(loc, addressRef);
+    mlir::Value address = fir::LoadOp::create(rewriter, loc, addressRef);
 
     // Get the function type.
     llvm::SmallVector<mlir::Type> argTypes;
@@ -237,7 +237,7 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
 
     mlir::Type funTy =
         mlir::FunctionType::get(rewriter.getContext(), argTypes, resTypes);
-    mlir::Value funcPtr = rewriter.create<fir::ConvertOp>(loc, funTy, address);
+    mlir::Value funcPtr = fir::ConvertOp::create(rewriter, loc, funTy, address);
 
     // Make the call.
     llvm::SmallVector<mlir::Value> args{funcPtr};
@@ -398,11 +398,11 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
       if (code == 0)
         return mlir::emitError(loc)
                << "type code unavailable for " << a.getType();
-      mlir::Value typeCode = rewriter.create<mlir::arith::ConstantOp>(
+      mlir::Value typeCode = mlir::arith::ConstantOp::create(rewriter,
           loc, rewriter.getI8IntegerAttr(code));
-      mlir::Value selectorTypeCode = rewriter.create<fir::BoxTypeCodeOp>(
+      mlir::Value selectorTypeCode = fir::BoxTypeCodeOp::create(rewriter,
           loc, rewriter.getI8Type(), selector);
-      cmp = rewriter.create<mlir::arith::CmpIOp>(
+      cmp = mlir::arith::CmpIOp::create(rewriter,
           loc, mlir::arith::CmpIPredicate::eq, selectorTypeCode, typeCode);
     } else {
       // Flang inline the kind parameter in the type descriptor so we can
@@ -419,15 +419,15 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
     // Retrieve the type descriptor from the type guard statement record type.
     assert(mlir::isa<fir::RecordType>(a.getType()) && "expect fir.record type");
     mlir::Value typeDescAddr =
-        rewriter.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(a.getType()));
+        fir::TypeDescOp::create(rewriter, loc, mlir::TypeAttr::get(a.getType()));
     mlir::Type refNoneType = ReferenceType::get(rewriter.getNoneType());
     mlir::Value typeDesc =
-        rewriter.create<ConvertOp>(loc, refNoneType, typeDescAddr);
+        ConvertOp::create(rewriter, loc, refNoneType, typeDescAddr);
 
     // Prepare the selector descriptor for the runtime call.
     mlir::Type descNoneTy = fir::BoxType::get(rewriter.getNoneType());
     mlir::Value descSelector =
-        rewriter.create<ConvertOp>(loc, descNoneTy, selector);
+        ConvertOp::create(rewriter, loc, descNoneTy, selector);
 
     // Generate runtime call.
     llvm::StringRef fctName = RTNAME_STRING(ClassIs);
@@ -455,10 +455,10 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
       rewriter.createBlock(dest->getParent(), mlir::Region::iterator(dest));
   rewriter.setInsertionPointToEnd(thisBlock);
   if (destOps.has_value())
-    rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, destOps.value(),
+    mlir::cf::CondBranchOp::create(rewriter, loc, cmp, dest, destOps.value(),
                                             newBlock, mlir::ValueRange{});
   else
-    rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, newBlock);
+    mlir::cf::CondBranchOp::create(rewriter, loc, cmp, dest, newBlock);
   rewriter.setInsertionPointToEnd(newBlock);
   return mlir::success();
 }
@@ -470,15 +470,15 @@ SelectTypeConv::genTypeDescCompare(mlir::Location loc, mlir::Value selector,
                                    mlir::PatternRewriter &rewriter) const {
   assert(mlir::isa<fir::RecordType>(ty) && "expect fir.record type");
   mlir::Value typeDescAddr =
-      rewriter.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(ty));
-  mlir::Value selectorTdescAddr = rewriter.create<fir::BoxTypeDescOp>(
+      fir::TypeDescOp::create(rewriter, loc, mlir::TypeAttr::get(ty));
+  mlir::Value selectorTdescAddr = fir::BoxTypeDescOp::create(rewriter,
       loc, typeDescAddr.getType(), selector);
   auto intPtrTy = rewriter.getIndexType();
   auto typeDescInt =
-      rewriter.create<fir::ConvertOp>(loc, intPtrTy, typeDescAddr);
+      fir::ConvertOp::create(rewriter, loc, intPtrTy, typeDescAddr);
   auto selectorTdescInt =
-      rewriter.create<fir::ConvertOp>(loc, intPtrTy, selectorTdescAddr);
-  return rewriter.create<mlir::arith::CmpIOp>(
+      fir::ConvertOp::create(rewriter, loc, intPtrTy, selectorTdescAddr);
+  return mlir::arith::CmpIOp::create(rewriter,
       loc, mlir::arith::CmpIPredicate::eq, typeDescInt, selectorTdescInt);
 }
 
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index ad8464b495888..d3441f9b22ac0 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -88,18 +88,18 @@ mlir::LogicalResult IsContiguousBoxCoversion::matchAndRewrite(
     // The scalar cases are supposed to be optimized by the canonicalization.
     if (rank == 1 || (op.getInnermost() && rank > 0)) {
       mlir::Type idxTy = builder.getIndexType();
-      auto eleSize = builder.create<fir::BoxEleSizeOp>(loc, idxTy, box);
+      auto eleSize = fir::BoxEleSizeOp::create(builder, loc, idxTy, box);
       mlir::Value zero = fir::factory::createZeroValue(builder, loc, idxTy);
       auto dimInfo =
-          builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, zero);
+          fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, zero);
       mlir::Value stride = dimInfo.getByteStride();
-      mlir::Value pred1 = builder.create<mlir::arith::CmpIOp>(
+      mlir::Value pred1 = mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::eq, eleSize, stride);
       mlir::Value extent = dimInfo.getExtent();
-      mlir::Value pred2 = builder.create<mlir::arith::CmpIOp>(
+      mlir::Value pred2 = mlir::arith::CmpIOp::create(builder,
           loc, mlir::arith::CmpIPredicate::eq, extent, zero);
       mlir::Value result =
-          builder.create<mlir::arith::OrIOp>(loc, pred1, pred2);
+          mlir::arith::OrIOp::create(builder, loc, pred1, pred2);
       result = builder.createConvert(loc, op.getType(), result);
       rewriter.replaceOp(op, result);
       return mlir::success();
@@ -192,7 +192,7 @@ class DoConcurrentConversion
         // TODO Should this be a heap allocation instead? For now, we allocate
         // on the stack for each loop iteration.
         mlir::Value localAlloc =
-            rewriter.create<fir::AllocaOp>(loop.getLoc(), localizer.getType());
+            fir::AllocaOp::create(rewriter, loop.getLoc(), localizer.getType());
 
         auto cloneLocalizerRegion = [&](mlir::Region &region,
                                         mlir::ValueRange regionArgs,
@@ -258,7 +258,7 @@ class DoConcurrentConversion
     for (auto [lb, ub, st, iv] :
          llvm::zip_equal(loop.getLowerBound(), loop.getUpperBound(),
                          loop.getStep(), *loop.getLoopInductionVars())) {
-      innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
+      innermostUnorderdLoop = fir::DoLoopOp::create(rewriter,
           doConcurentOp.getLoc(), lb, ub, st,
           /*unordred=*/true, /*finalCountValue=*/false,
           /*iterArgs=*/mlir::ValueRange{}, loop.getReduceVars(),
diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
index 4d25a02bf18ba..43aedb564ed81 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
@@ -284,7 +284,7 @@ genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp,
                                      fir::SequenceType::getUnknownExtent());
   mlir::Type arrTy = fir::SequenceType::get(flatShape, elementType);
   mlir::Type boxArrTy = fir::BoxType::get(arrTy);
-  mlir::Value array = builder.create<fir::ConvertOp>(loc, boxArrTy, arg);
+  mlir::Value array = fir::ConvertOp::create(builder, loc, boxArrTy, arg);
   mlir::Type resultType = funcOp.getResultTypes()[0];
   mlir::Value init = initVal(builder, loc, resultType);
 
@@ -300,10 +300,10 @@ genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp,
   for (unsigned i = 0; i < rank; ++i) {
     mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i);
     auto dims =
-        builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, array, dimIdx);
+        fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, array, dimIdx);
     mlir::Value len = dims.getResult(1);
     // We use C indexing here, so len-1 as loopcount
-    mlir::Value loopCount = builder.create<mlir::arith::SubIOp>(loc, len, one);
+    mlir::Value loopCount = mlir::arith::SubIOp::create(builder, loc, len, one);
     bounds.push_back(loopCount);
   }
   // Create a loop nest consisting of OP operations.
@@ -316,7 +316,7 @@ genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp,
   for (unsigned i = rank; 0 < i; --i) {
     mlir::Value step = one;
     mlir::Value loopCount = bounds[i - 1];
-    auto loop = builder.create<OP>(loc, zeroIdx, loopCount, step,
+    auto loop = OP::create(builder, loc, zeroIdx, loopCount, step,
                                    unorderedOrInitialLoopCond,
                                    /*finalCountValue=*/false, init);
     init = loop.getRegionIterArgs()[resultIndex];
@@ -332,8 +332,8 @@ genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp,
   // We are in the innermost loop: generate the reduction body.
   mlir::Type eleRefTy = builder.getRefType(elementType);
   mlir::Value addr =
-      builder.create<fir::CoordinateOp>(loc, eleRefTy, array, indices);
-  mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
+      fir::CoordinateOp::create(builder, loc, eleRefTy, array, indices);
+  mlir::Value elem = fir::LoadOp::create(builder, loc, addr);
   mlir::Value reductionVal = genBody(builder, loc, elementType, elem, init);
   // Generate vector with condition to continue while loop at [0] and result
   // from current loop at [1] for IterWhileOp loops, just result at [0] for
@@ -344,7 +344,7 @@ genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp,
   // to return the updated value of the reduction to the enclosing
   // loops.
   for (unsigned i = 0; i < rank; ++i) {
-    auto result = builder.create<fir::ResultOp>(loc, results);
+    auto result = fir::ResultOp::create(builder, loc, results);
     // Proceed to the outer loop.
     auto loop = mlir::cast<OP>(result->getParentOp());
     results = loop.getResults();
@@ -354,7 +354,7 @@ genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp,
   }
   // End of loop nest. The insertion point is after the outermost loop.
   // Return the reduction value from the function.
-  builder.create<mlir::func::ReturnOp>(loc, results[resultIndex]);
+  mlir::func::ReturnOp::create(builder, loc, results[resultIndex]);
 }
 
 static llvm::SmallVector<mlir::Value> nopLoopCond(fir::FirOpBuilder &builder,
@@ -394,9 +394,9 @@ static void genRuntimeSumBody(fir::FirOpBuilder &builder,
                       mlir::Type elementType, mlir::Value elem1,
                       mlir::Value elem2) -> mlir::Value {
     if (mlir::isa<mlir::FloatType>(elementType))
-      return builder.create<mlir::arith::AddFOp>(loc, elem1, elem2);
+      return mlir::arith::AddFOp::create(builder, loc, elem1, elem2);
     if (mlir::isa<mlir::IntegerType>(elementType))
-      return builder.create<mlir::arith::AddIOp>(loc, elem1, elem2);
+      return mlir::arith::AddIOp::create(builder, loc, elem1, elem2);
 
     llvm_unreachable("unsupported type");
     return {};
@@ -436,12 +436,12 @@ static void genRuntimeMaxvalBody(fir::FirOpBuilder &builder,
       // This libm function may not work properly for F128 arguments
       // on targets where long double is not F128. It is an LLVM issue,
       // but we just use normal select here to resolve all the cases.
-      auto compare = builder.create<mlir::arith::CmpFOp>(
+      auto compare = mlir::arith::CmpFOp::create(builder,
           loc, mlir::arith::CmpFPredicate::OGT, elem1, elem2);
-      return builder.create<mlir::arith::SelectOp>(loc, compare, elem1, elem2);
+      return mlir::arith::SelectOp::create(builder, loc, compare, elem1, elem2);
     }
     if (mlir::isa<mlir::IntegerType>(elementType))
-      return builder.create<mlir::arith::MaxSIOp>(loc, elem1, elem2);
+      return mlir::arith::MaxSIOp::create(builder, loc, elem1, elem2);
 
     llvm_unreachable("unsupported type");
     return {};
@@ -472,11 +472,11 @@ static void genRuntimeCountBody(fir::FirOpBuilder &builder,
     auto zero64 = builder.createIntegerConstant(loc, builder.getI64Type(), 0);
     auto one64 = builder.createIntegerConstant(loc, builder.getI64Type(), 1);
 
-    auto compare = builder.create<mlir::arith::CmpIOp>(
+    auto compare = mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::eq, elem1, zero32);
     auto select =
-        builder.create<mlir::arith::SelectOp>(loc, compare, zero64, one64);
-    return builder.create<mlir::arith::AddIOp>(loc, select, elem2);
+        mlir::arith::SelectOp::create(builder, loc, compare, zero64, one64);
+    return mlir::arith::AddIOp::create(builder, loc, select, elem2);
   };
 
   // Count always gets I32 for elementType as it converts logical input to
@@ -501,14 +501,14 @@ static void genRuntimeAnyBody(fir::FirOpBuilder &builder,
                       mlir::Type elementType, mlir::Value elem1,
                       mlir::Value elem2) -> mlir::Value {
     auto zero = builder.createIntegerConstant(loc, elementType, 0);
-    return builder.create<mlir::arith::CmpIOp>(
+    return mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::ne, elem1, zero);
   };
 
   auto continueCond = [](fir::FirOpBuilder builder, mlir::Location loc,
                          mlir::Value reductionVal) {
     auto one1 = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
-    auto eor = builder.create<mlir::arith::XOrIOp>(loc, reductionVal, one1);
+    auto eor = mlir::arith::XOrIOp::create(builder, loc, reductionVal, one1);
     llvm::SmallVector<mlir::Value> results = {eor, reductionVal};
     return results;
   };
@@ -534,7 +534,7 @@ static void genRuntimeAllBody(fir::FirOpBuilder &builder,
                       mlir::Type elementType, mlir::Value elem1,
                       mlir::Value elem2) -> mlir::Value {
     auto zero = builder.createIntegerConstant(loc, elementType, 0);
-    return builder.create<mlir::arith::CmpIOp>(
+    return mlir::arith::CmpIOp::create(builder,
         loc, mlir::arith::CmpIPredicate::ne, elem1, zero);
   };
 
@@ -577,13 +577,13 @@ void fir::genMinMaxlocReductionLoop(
                                      fir::SequenceType::getUnknownExtent());
   mlir::Type arrTy = fir::SequenceType::get(flatShape, elementType);
   mlir::Type boxArrTy = fir::BoxType::get(arrTy);
-  array = builder.create<fir::ConvertOp>(loc, boxArrTy, array);
+  array = fir::ConvertOp::create(builder, loc, boxArrTy, array);
 
   mlir::Type resultElemType = hlfir::getFortranElementType(resultArr.getType());
   mlir::Value flagSet = builder.createIntegerConstant(loc, resultElemType, 1);
   mlir::Value zero = builder.createIntegerConstant(loc, resultElemType, 0);
   mlir::Value flagRef = builder.createTemporary(loc, resultElemType);
-  builder.create<fir::StoreOp>(loc, zero, flagRef);
+  fir::StoreOp::create(builder, loc, zero, flagRef);
 
   mlir::Value init = initVal(builder, loc, elementType);
   llvm::SmallVector<mlir::Value, Fortran::common::maxRank> bounds;
@@ -598,10 +598,10 @@ void fir::genMinMaxlocReductionLoop(
   for (unsigned i = 0; i < rank; ++i) {
     mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i);
     auto dims =
-        builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, array, dimIdx);
+        fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, array, dimIdx);
     mlir::Value len = dims.getResult(1);
     // We use C indexing here, so len-1 as loopcount
-    mlir::Value loopCount = builder.create<mlir::arith::SubIOp>(loc, len, one);
+    mlir::Value loopCount = mlir::arith::SubIOp::create(builder, loc, len, one);
     bounds.push_back(loopCount);
   }
   // Create a loop nest consisting of OP operations.
@@ -615,7 +615,7 @@ void fir::genMinMaxlocReductionLoop(
     mlir::Value step = one;
     mlir::Value loopCount = bounds[i - 1];
     auto loop =
-        builder.create<fir::DoLoopOp>(loc, zeroIdx, loopCount, step, false,
+        fir::DoLoopOp::create(builder, loc, zeroIdx, loopCount, step, false,
                                       /*finalCountValue=*/false, init);
     init = loop.getRegionIterArgs()[0];
     indices.push_back(loop.getInductionVar());
@@ -634,7 +634,7 @@ void fir::genMinMaxlocReductionLoop(
   // to return the updated value of the reduction to the enclosing
   // loops.
   for (unsigned i = 0; i < rank; ++i) {
-    auto result = builder.create<fir::ResultOp>(loc, reductionVal);
+    auto result = fir::ResultOp::create(builder, loc, reductionVal);
     // Proceed to the outer loop.
     auto loop = mlir::cast<fir::DoLoopOp>(result->getParentOp());
     reductionVal = loop.getResult(0);
@@ -646,7 +646,7 @@ void fir::genMinMaxlocReductionLoop(
   if (maskMayBeLogicalScalar) {
     if (fir::IfOp ifOp =
             mlir::dyn_cast<fir::IfOp>(builder.getBlock()->getParentOp())) {
-      builder.create<fir::ResultOp>(loc, reductionVal);
+      fir::ResultOp::create(builder, loc, reductionVal);
       builder.setInsertionPointAfter(ifOp);
       // Redefine flagSet to escape scope of ifOp
       flagSet = builder.createIntegerConstant(loc, resultElemType, 1);
@@ -689,9 +689,9 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
   mlir::Value returnValue = builder.createIntegerConstant(loc, resultElemTy, 0);
   mlir::Value resultArrSize = builder.createIntegerConstant(loc, idxTy, rank);
 
-  mlir::Value resultArrInit = builder.create<fir::AllocMemOp>(loc, resultTy);
-  mlir::Value resultArrShape = builder.create<fir::ShapeOp>(loc, resultArrSize);
-  mlir::Value resultArr = builder.create<fir::EmboxOp>(
+  mlir::Value resultArrInit = fir::AllocMemOp::create(builder, loc, resultTy);
+  mlir::Value resultArrShape = fir::ShapeOp::create(builder, loc, resultArrSize);
+  mlir::Value resultArr = fir::EmboxOp::create(builder,
       loc, resultBoxTy, resultArrInit, resultArrShape);
 
   mlir::Type resultRefTy = builder.getRefType(resultElemTy);
@@ -701,14 +701,14 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
                                        fir::SequenceType::getUnknownExtent());
     mlir::Type maskTy = fir::SequenceType::get(flatShape, maskElemType);
     mlir::Type boxMaskTy = fir::BoxType::get(maskTy);
-    mask = builder.create<fir::ConvertOp>(loc, boxMaskTy, mask);
+    mask = fir::ConvertOp::create(builder, loc, boxMaskTy, mask);
   }
 
   for (unsigned int i = 0; i < rank; ++i) {
     mlir::Value index = builder.createIntegerConstant(loc, idxTy, i);
     mlir::Value resultElemAddr =
-        builder.create<fir::CoordinateOp>(loc, resultRefTy, resultArr, index);
-    builder.create<fir::StoreOp>(loc, returnValue, resultElemAddr);
+        fir::CoordinateOp::create(builder, loc, resultRefTy, resultArr, index);
+    fir::StoreOp::create(builder, loc, returnValue, resultElemAddr);
   }
 
   auto genBodyOp =
@@ -720,17 +720,17 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
     if (maskRank > 0) {
       mlir::Type logicalRef = builder.getRefType(maskElemType);
       mlir::Value maskAddr =
-          builder.create<fir::CoordinateOp>(loc, logicalRef, mask, indices);
-      mlir::Value maskElem = builder.create<fir::LoadOp>(loc, maskAddr);
+          fir::CoordinateOp::create(builder, loc, logicalRef, mask, indices);
+      mlir::Value maskElem = fir::LoadOp::create(builder, loc, maskAddr);
 
       // fir::IfOp requires argument to be I1 - won't accept logical or any
       // other Integer.
       mlir::Type ifCompatType = builder.getI1Type();
       mlir::Value ifCompatElem =
-          builder.create<fir::ConvertOp>(loc, ifCompatType, maskElem);
+          fir::ConvertOp::create(builder, loc, ifCompatType, maskElem);
 
       llvm::SmallVector<mlir::Type> resultsTy = {elementType, elementType};
-      fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, ifCompatElem,
+      fir::IfOp ifOp = fir::IfOp::create(builder, loc, elementType, ifCompatElem,
                                                  /*withElseRegion=*/true);
       builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     }
@@ -738,11 +738,11 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
     // Set flag that mask was true at some point
     mlir::Value flagSet = builder.createIntegerConstant(
         loc, mlir::cast<fir::ReferenceType>(flagRef.getType()).getEleTy(), 1);
-    mlir::Value isFirst = builder.create<fir::LoadOp>(loc, flagRef);
+    mlir::Value isFirst = fir::LoadOp::create(builder, loc, flagRef);
     mlir::Type eleRefTy = builder.getRefType(elementType);
     mlir::Value addr =
-        builder.create<fir::CoordinateOp>(loc, eleRefTy, array, indices);
-    mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
+        fir::CoordinateOp::create(builder, loc, eleRefTy, array, indices);
+    mlir::Value elem = fir::LoadOp::create(builder, loc, addr);
 
     mlir::Value cmp;
     if (mlir::isa<mlir::FloatType>(elementType)) {
@@ -750,20 +750,20 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
       // is not NaN. A OGL/OLT condition will usually work for this unless all
       // the values are Nan or Inf. This follows the same logic as
       // NumericCompare for Minloc/Maxlox in extrema.cpp.
-      cmp = builder.create<mlir::arith::CmpFOp>(
+      cmp = mlir::arith::CmpFOp::create(builder,
           loc,
           isMax ? mlir::arith::CmpFPredicate::OGT
                 : mlir::arith::CmpFPredicate::OLT,
           elem, reduction);
 
-      mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+      mlir::Value cmpNan = mlir::arith::CmpFOp::create(builder,
           loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-      mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+      mlir::Value cmpNan2 = mlir::arith::CmpFOp::create(builder,
           loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-      cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-      cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+      cmpNan = mlir::arith::AndIOp::create(builder, loc, cmpNan, cmpNan2);
+      cmp = mlir::arith::OrIOp::create(builder, loc, cmp, cmpNan);
     } else if (mlir::isa<mlir::IntegerType>(elementType)) {
-      cmp = builder.create<mlir::arith::CmpIOp>(
+      cmp = mlir::arith::CmpIOp::create(builder,
           loc,
           isMax ? mlir::arith::CmpIPredicate::sgt
                 : mlir::arith::CmpIPredicate::slt,
@@ -773,15 +773,15 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
     }
 
     // The condition used for the loop is isFirst || <the condition above>.
-    isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
-    isFirst = builder.create<mlir::arith::XOrIOp>(
+    isFirst = fir::ConvertOp::create(builder, loc, cmp.getType(), isFirst);
+    isFirst = mlir::arith::XOrIOp::create(builder,
         loc, isFirst, builder.createIntegerConstant(loc, cmp.getType(), 1));
-    cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, isFirst);
-    fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, cmp,
+    cmp = mlir::arith::OrIOp::create(builder, loc, cmp, isFirst);
+    fir::IfOp ifOp = fir::IfOp::create(builder, loc, elementType, cmp,
                                                /*withElseRegion*/ true);
 
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-    builder.create<fir::StoreOp>(loc, flagSet, flagRef);
+    fir::StoreOp::create(builder, loc, flagSet, flagRef);
     mlir::Type resultElemTy = hlfir::getFortranElementType(resultArr.getType());
     mlir::Type returnRefTy = builder.getRefType(resultElemTy);
     mlir::IndexType idxTy = builder.getIndexType();
@@ -791,16 +791,16 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
     for (unsigned int i = 0; i < rank; ++i) {
       mlir::Value index = builder.createIntegerConstant(loc, idxTy, i);
       mlir::Value resultElemAddr =
-          builder.create<fir::CoordinateOp>(loc, returnRefTy, resultArr, index);
+          fir::CoordinateOp::create(builder, loc, returnRefTy, resultArr, index);
       mlir::Value convert =
-          builder.create<fir::ConvertOp>(loc, resultElemTy, indices[i]);
+          fir::ConvertOp::create(builder, loc, resultElemTy, indices[i]);
       mlir::Value fortranIndex =
-          builder.create<mlir::arith::AddIOp>(loc, convert, one);
-      builder.create<fir::StoreOp>(loc, fortranIndex, resultElemAddr);
+          mlir::arith::AddIOp::create(builder, loc, convert, one);
+      fir::StoreOp::create(builder, loc, fortranIndex, resultElemAddr);
     }
-    builder.create<fir::ResultOp>(loc, elem);
+    fir::ResultOp::create(builder, loc, elem);
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-    builder.create<fir::ResultOp>(loc, reduction);
+    fir::ResultOp::create(builder, loc, reduction);
     builder.setInsertionPointAfter(ifOp);
     mlir::Value reductionVal = ifOp.getResult(0);
 
@@ -808,9 +808,9 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
     if (maskRank > 0) {
       fir::IfOp ifOp =
           mlir::dyn_cast<fir::IfOp>(builder.getBlock()->getParentOp());
-      builder.create<fir::ResultOp>(loc, reductionVal);
+      fir::ResultOp::create(builder, loc, reductionVal);
       builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, reduction);
+      fir::ResultOp::create(builder, loc, reduction);
       reductionVal = ifOp.getResult(0);
       builder.setInsertionPointAfter(ifOp);
     }
@@ -825,11 +825,11 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
     mlir::Type logical = maskElemType;
     mlir::Type logicalRefTy = builder.getRefType(logical);
     mlir::Value condAddr =
-        builder.create<fir::BoxAddrOp>(loc, logicalRefTy, mask);
-    mlir::Value cond = builder.create<fir::LoadOp>(loc, condAddr);
-    mlir::Value condI1 = builder.create<fir::ConvertOp>(loc, i1Type, cond);
+        fir::BoxAddrOp::create(builder, loc, logicalRefTy, mask);
+    mlir::Value cond = fir::LoadOp::create(builder, loc, condAddr);
+    mlir::Value condI1 = fir::ConvertOp::create(builder, loc, i1Type, cond);
 
-    fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, condI1,
+    fir::IfOp ifOp = fir::IfOp::create(builder, loc, elementType, condI1,
                                                /*withElseRegion=*/true);
 
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -839,7 +839,7 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
     } else {
       basicValue = builder.createRealConstant(loc, elementType, 0);
     }
-    builder.create<fir::ResultOp>(loc, basicValue);
+    fir::ResultOp::create(builder, loc, basicValue);
 
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   }
@@ -847,7 +847,7 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
                       const mlir::Type &resultElemType, mlir::Value resultArr,
                       mlir::Value index) {
     mlir::Type resultRefTy = builder.getRefType(resultElemType);
-    return builder.create<fir::CoordinateOp>(loc, resultRefTy, resultArr,
+    return fir::CoordinateOp::create(builder, loc, resultRefTy, resultArr,
                                              index);
   };
 
@@ -859,25 +859,25 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
   if (isDim) {
     mlir::Type resultBoxTy =
         fir::BoxType::get(fir::HeapType::get(resultElemTy));
-    mlir::Value outputArr = builder.create<fir::ConvertOp>(
+    mlir::Value outputArr = fir::ConvertOp::create(builder,
         loc, builder.getRefType(resultBoxTy), funcOp.front().getArgument(0));
-    mlir::Value resultArrScalar = builder.create<fir::ConvertOp>(
+    mlir::Value resultArrScalar = fir::ConvertOp::create(builder,
         loc, fir::HeapType::get(resultElemTy), resultArrInit);
     mlir::Value resultBox =
-        builder.create<fir::EmboxOp>(loc, resultBoxTy, resultArrScalar);
-    builder.create<fir::StoreOp>(loc, resultBox, outputArr);
+        fir::EmboxOp::create(builder, loc, resultBoxTy, resultArrScalar);
+    fir::StoreOp::create(builder, loc, resultBox, outputArr);
   } else {
     fir::SequenceType::Shape resultShape(1, rank);
     mlir::Type outputArrTy = fir::SequenceType::get(resultShape, resultElemTy);
     mlir::Type outputHeapTy = fir::HeapType::get(outputArrTy);
     mlir::Type outputBoxTy = fir::BoxType::get(outputHeapTy);
     mlir::Type outputRefTy = builder.getRefType(outputBoxTy);
-    mlir::Value outputArr = builder.create<fir::ConvertOp>(
+    mlir::Value outputArr = fir::ConvertOp::create(builder,
         loc, outputRefTy, funcOp.front().getArgument(0));
-    builder.create<fir::StoreOp>(loc, resultArr, outputArr);
+    fir::StoreOp::create(builder, loc, resultArr, outputArr);
   }
 
-  builder.create<mlir::func::ReturnOp>(loc);
+  mlir::func::ReturnOp::create(builder, loc);
 }
 
 /// Generate function type for the simplified version of RTNAME(DotProduct)
@@ -929,10 +929,10 @@ static void genRuntimeDotBody(fir::FirOpBuilder &builder,
   fir::SequenceType::Shape flatShape = {fir::SequenceType::getUnknownExtent()};
   mlir::Type arrTy1 = fir::SequenceType::get(flatShape, arg1ElementTy);
   mlir::Type boxArrTy1 = fir::BoxType::get(arrTy1);
-  mlir::Value array1 = builder.create<fir::ConvertOp>(loc, boxArrTy1, arg1);
+  mlir::Value array1 = fir::ConvertOp::create(builder, loc, boxArrTy1, arg1);
   mlir::Type arrTy2 = fir::SequenceType::get(flatShape, arg2ElementTy);
   mlir::Type boxArrTy2 = fir::BoxType::get(arrTy2);
-  mlir::Value array2 = builder.create<fir::ConvertOp>(loc, boxArrTy2, arg2);
+  mlir::Value array2 = fir::ConvertOp::create(builder, loc, boxArrTy2, arg2);
   // This version takes the loop trip count from the first argument.
   // If the first argument's box has unknown (at compilation time)
   // extent, then it may be better to take the extent from the second
@@ -942,14 +942,14 @@ static void genRuntimeDotBody(fir::FirOpBuilder &builder,
   // is more profitable to call.
   // Note that we can assume that both arguments have the same extent.
   auto dims =
-      builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, array1, zeroIdx);
+      fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, array1, zeroIdx);
   mlir::Value len = dims.getResult(1);
   mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
   mlir::Value step = one;
 
   // We use C indexing here, so len-1 as loopcount
-  mlir::Value loopCount = builder.create<mlir::arith::SubIOp>(loc, len, one);
-  auto loop = builder.create<fir::DoLoopOp>(loc, zeroIdx, loopCount, step,
+  mlir::Value loopCount = mlir::arith::SubIOp::create(builder, loc, len, one);
+  auto loop = fir::DoLoopOp::create(builder, loc, zeroIdx, loopCount, step,
                                             /*unordered=*/false,
                                             /*finalCountValue=*/false, zero);
   mlir::Value sumVal = loop.getRegionIterArgs()[0];
@@ -961,33 +961,33 @@ static void genRuntimeDotBody(fir::FirOpBuilder &builder,
   mlir::Type eleRef1Ty = builder.getRefType(arg1ElementTy);
   mlir::Value index = loop.getInductionVar();
   mlir::Value addr1 =
-      builder.create<fir::CoordinateOp>(loc, eleRef1Ty, array1, index);
-  mlir::Value elem1 = builder.create<fir::LoadOp>(loc, addr1);
+      fir::CoordinateOp::create(builder, loc, eleRef1Ty, array1, index);
+  mlir::Value elem1 = fir::LoadOp::create(builder, loc, addr1);
   // Convert to the result type.
-  elem1 = builder.create<fir::ConvertOp>(loc, resultElementType, elem1);
+  elem1 = fir::ConvertOp::create(builder, loc, resultElementType, elem1);
 
   mlir::Type eleRef2Ty = builder.getRefType(arg2ElementTy);
   mlir::Value addr2 =
-      builder.create<fir::CoordinateOp>(loc, eleRef2Ty, array2, index);
-  mlir::Value elem2 = builder.create<fir::LoadOp>(loc, addr2);
+      fir::CoordinateOp::create(builder, loc, eleRef2Ty, array2, index);
+  mlir::Value elem2 = fir::LoadOp::create(builder, loc, addr2);
   // Convert to the result type.
-  elem2 = builder.create<fir::ConvertOp>(loc, resultElementType, elem2);
+  elem2 = fir::ConvertOp::create(builder, loc, resultElementType, elem2);
 
   if (mlir::isa<mlir::FloatType>(resultElementType))
-    sumVal = builder.create<mlir::arith::AddFOp>(
-        loc, builder.create<mlir::arith::MulFOp>(loc, elem1, elem2), sumVal);
+    sumVal = mlir::arith::AddFOp::create(builder,
+        loc, mlir::arith::MulFOp::create(builder, loc, elem1, elem2), sumVal);
   else if (mlir::isa<mlir::IntegerType>(resultElementType))
-    sumVal = builder.create<mlir::arith::AddIOp>(
-        loc, builder.create<mlir::arith::MulIOp>(loc, elem1, elem2), sumVal);
+    sumVal = mlir::arith::AddIOp::create(builder,
+        loc, mlir::arith::MulIOp::create(builder, loc, elem1, elem2), sumVal);
   else
     llvm_unreachable("unsupported type");
 
-  builder.create<fir::ResultOp>(loc, sumVal);
+  fir::ResultOp::create(builder, loc, sumVal);
   // End of loop.
   builder.restoreInsertionPoint(loopEndPt);
 
   mlir::Value resultVal = loop.getResult(0);
-  builder.create<mlir::func::ReturnOp>(loc, resultVal);
+  mlir::func::ReturnOp::create(builder, loc, resultVal);
 }
 
 mlir::func::FuncOp SimplifyIntrinsicsPass::getOrCreateFunction(
@@ -1229,7 +1229,7 @@ void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction(
 
   mlir::func::FuncOp newFunc =
       getOrCreateFunction(builder, funcName, typeGenerator, bodyGenerator);
-  builder.create<fir::CallOp>(loc, newFunc,
+  fir::CallOp::create(builder, loc, newFunc,
                               mlir::ValueRange{args[0], args[1], mask});
   call->dropAllReferences();
   call->erase();
@@ -1259,7 +1259,7 @@ void SimplifyIntrinsicsPass::simplifyReductionBody(
   mlir::func::FuncOp newFunc =
       getOrCreateFunction(builder, funcName, typeGenerator, bodyGenerator);
   auto newCall =
-      builder.create<fir::CallOp>(loc, newFunc, mlir::ValueRange{args[0]});
+      fir::CallOp::create(builder, loc, newFunc, mlir::ValueRange{args[0]});
   call->replaceAllUsesWith(newCall.getResults());
   call->dropAllReferences();
   call->erase();
@@ -1344,7 +1344,7 @@ void SimplifyIntrinsicsPass::runOnOperation() {
 
           mlir::func::FuncOp newFunc = getOrCreateFunction(
               builder, typedFuncName, typeGenerator, bodyGenerator);
-          auto newCall = builder.create<fir::CallOp>(loc, newFunc,
+          auto newCall = fir::CallOp::create(builder, loc, newFunc,
                                                      mlir::ValueRange{v1, v2});
           call->replaceAllUsesWith(newCall.getResults());
           call->dropAllReferences();
diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index bc8a9497fbb70..e34f56367a7ad 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -569,7 +569,7 @@ static mlir::Value convertAllocationType(mlir::PatternRewriter &rewriter,
   auto insertionPoint = rewriter.saveInsertionPoint();
   rewriter.setInsertionPointAfter(stack.getDefiningOp());
   mlir::Value conv =
-      rewriter.create<fir::ConvertOp>(loc, firHeapTy, stack).getResult();
+      fir::ConvertOp::create(rewriter, loc, firHeapTy, stack).getResult();
   rewriter.restoreInsertionPoint(insertionPoint);
   return conv;
 }
@@ -758,7 +758,7 @@ AllocMemConversion::insertAlloca(fir::AllocMemOp &oldAlloc,
 
   llvm::StringRef uniqName = unpackName(oldAlloc.getUniqName());
   llvm::StringRef bindcName = unpackName(oldAlloc.getBindcName());
-  auto alloca = rewriter.create<fir::AllocaOp>(loc, varTy, uniqName, bindcName,
+  auto alloca = fir::AllocaOp::create(rewriter, loc, varTy, uniqName, bindcName,
                                                oldAlloc.getTypeparams(),
                                                oldAlloc.getShape());
   if (emitLifetimeMarkers)


From flang-commits at lists.llvm.org  Mon Jul 21 16:53:50 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:53:50 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed30e.170a0220.282d23.0a01@mx.google.com>


================
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
----------------
Meinersbur wrote:

Some additional notes:
%flang_bare` actually is the driver (i.e. not `flang -fc1`, have to change the comments). The problem is that the intrinsic modules may not be in the resource directory, but in the directory defined by `FLANG_INTRINSIC_MODULES_DIR`, as otherwise it would not be possible run check-flang tests in a Flang-standalone build (that does not build flang-rt, hence no intrinsic modules).
`FLANG_INTRINSIC_MODULES_DIR` is implemented by replacing `flang` with `flang -fintrinsic-modules-path ${FLANG_INTRINSIC_MODULES_DIR}`, i.e. always comes first in the command line. This means it will always have priority over `-fintrinsic-modules-path %S/Inputs/`. This is incompatible with the second RUN line that depends on the files in  `%S/Inputs/` being picked up.
Hence, this test adds `-fintrinsic-modules-path ${FLANG_INTRINSIC_MODULES_DIR}` (if necessary) after  `-fintrinsic-modules-path %S/Inputs/`, since with a `%flang` substitution it is not possible to put it behind all other flags. `%flang_bare` avoids that `-fintrinsic-modules-path ${FLANG_INTRINSIC_MODULES_DIR}` is (also) added before `%S/Inputs/`.

One could also redesign this test, i.e. that it expects the default intrinsic modules to be matched, not the ones in `%S/Inputs/`.

https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:54:14 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:54:14 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed326.170a0220.1affc.144e@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:54:35 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 21 Jul 2025 16:54:35 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][NFC] update `flang/Lower` create
	APIs (8/n) (PR #149912)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149912@github.com>
Message-ID: <687ed33b.630a0220.1e11ca.eee0@mx.google.com>


https://github.com/makslevental closed https://github.com/llvm/llvm-project/pull/149912

From flang-commits at lists.llvm.org  Mon Jul 21 16:54:59 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:54:59 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed353.170a0220.37699d.06a4@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:55:32 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:55:32 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed374.050a0220.23e52f.0733@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:56:19 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:56:19 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed3a3.050a0220.1634f6.fa9d@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:54:33 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 16:54:33 -0700 (PDT)
Subject: [flang-commits] [flang] a3a007a - [mlir][NFC] update `flang/Lower`
 create APIs (8/n) (#149912)
Message-ID: <687ed339.170a0220.122c7f.01c5@mx.google.com>


Author: Maksim Levental
Date: 2025-07-21T19:54:29-04:00
New Revision: a3a007ad5fa20abc90ead4e1030b481bf109b4cf

URL: https://github.com/llvm/llvm-project/commit/a3a007ad5fa20abc90ead4e1030b481bf109b4cf
DIFF: https://github.com/llvm/llvm-project/commit/a3a007ad5fa20abc90ead4e1030b481bf109b4cf.diff

LOG: [mlir][NFC] update `flang/Lower` create APIs (8/n) (#149912)

See https://github.com/llvm/llvm-project/pull/147168 for more info.

Added: 
    

Modified: 
    flang/lib/Lower/Allocatable.cpp
    flang/lib/Lower/Bridge.cpp
    flang/lib/Lower/ConvertArrayConstructor.cpp
    flang/lib/Lower/ConvertCall.cpp
    flang/lib/Lower/ConvertConstant.cpp
    flang/lib/Lower/ConvertExpr.cpp
    flang/lib/Lower/ConvertExprToHLFIR.cpp
    flang/lib/Lower/ConvertProcedureDesignator.cpp
    flang/lib/Lower/ConvertVariable.cpp
    flang/lib/Lower/CustomIntrinsicCall.cpp
    flang/lib/Lower/HlfirIntrinsics.cpp
    flang/lib/Lower/HostAssociations.cpp
    flang/lib/Lower/IO.cpp
    flang/lib/Lower/OpenACC.cpp
    flang/lib/Lower/OpenMP/Atomic.cpp
    flang/lib/Lower/OpenMP/ClauseProcessor.cpp
    flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
    flang/lib/Lower/OpenMP/OpenMP.cpp
    flang/lib/Lower/OpenMP/Utils.cpp
    flang/lib/Lower/Runtime.cpp
    flang/lib/Lower/Support/PrivateReductionUtils.cpp
    flang/lib/Lower/Support/ReductionProcessor.cpp
    flang/lib/Lower/Support/Utils.cpp
    flang/lib/Lower/VectorSubscripts.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index 5536bfe8d63ca..15cd9770b35ba 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -78,8 +78,8 @@ struct ErrorManager {
         statExpr && errMsgExpr
             ? builder.createBox(loc,
                                 converter.genExprAddr(loc, errMsgExpr, stmtCtx))
-            : builder.create<fir::AbsentOp>(
-                  loc,
+            : fir::AbsentOp::create(
+                  builder, loc,
                   fir::BoxType::get(mlir::NoneType::get(builder.getContext())));
     sourceFile = fir::factory::locationToFilename(builder, loc);
     sourceLine = fir::factory::locationToLineNo(builder, loc,
@@ -92,10 +92,10 @@ struct ErrorManager {
     if (statValue) {
       mlir::Value zero =
           builder.createIntegerConstant(loc, statValue.getType(), 0);
-      auto cmp = builder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::eq, statValue, zero);
-      auto ifOp = builder.create<fir::IfOp>(loc, cmp,
-                                            /*withElseRegion=*/false);
+      auto cmp = mlir::arith::CmpIOp::create(
+          builder, loc, mlir::arith::CmpIPredicate::eq, statValue, zero);
+      auto ifOp = fir::IfOp::create(builder, loc, cmp,
+                                    /*withElseRegion=*/false);
       builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     }
   }
@@ -106,7 +106,7 @@ struct ErrorManager {
       assert(stat && "missing stat value");
       mlir::Value castStat = builder.createConvert(
           loc, fir::dyn_cast_ptrEleTy(statAddr.getType()), stat);
-      builder.create<fir::StoreOp>(loc, castStat, statAddr);
+      fir::StoreOp::create(builder, loc, castStat, statAddr);
       statValue = stat;
     }
   }
@@ -141,7 +141,7 @@ static void genRuntimeSetBounds(fir::FirOpBuilder &builder, mlir::Location loc,
   const auto args = fir::runtime::createArguments(
       builder, loc, callee.getFunctionType(), box.getAddr(), dimIndex,
       lowerBound, upperBound);
-  builder.create<fir::CallOp>(loc, callee, args);
+  fir::CallOp::create(builder, loc, callee, args);
 }
 
 /// Generate runtime call to set the lengths of a character allocatable or
@@ -171,7 +171,7 @@ static void genRuntimeInitCharacter(fir::FirOpBuilder &builder,
   args.push_back(builder.createIntegerConstant(loc, inputTypes[4], corank));
   const auto convertedArgs = fir::runtime::createArguments(
       builder, loc, callee.getFunctionType(), args);
-  builder.create<fir::CallOp>(loc, callee, convertedArgs);
+  fir::CallOp::create(builder, loc, callee, convertedArgs);
 }
 
 /// Generate a sequence of runtime calls to allocate memory.
@@ -194,7 +194,7 @@ static mlir::Value genRuntimeAllocate(fir::FirOpBuilder &builder,
   args.push_back(errorManager.sourceLine);
   const auto convertedArgs = fir::runtime::createArguments(
       builder, loc, callee.getFunctionType(), args);
-  return builder.create<fir::CallOp>(loc, callee, convertedArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, callee, convertedArgs).getResult(0);
 }
 
 /// Generate a sequence of runtime calls to allocate memory and assign with the
@@ -214,7 +214,7 @@ static mlir::Value genRuntimeAllocateSource(fir::FirOpBuilder &builder,
       builder, loc, callee.getFunctionType(), box.getAddr(),
       fir::getBase(source), errorManager.hasStat, errorManager.errMsgAddr,
       errorManager.sourceFile, errorManager.sourceLine);
-  return builder.create<fir::CallOp>(loc, callee, args).getResult(0);
+  return fir::CallOp::create(builder, loc, callee, args).getResult(0);
 }
 
 /// Generate runtime call to apply mold to the descriptor.
@@ -233,7 +233,7 @@ static void genRuntimeAllocateApplyMold(fir::FirOpBuilder &builder,
       fir::factory::getMutableIRBox(builder, loc, box), fir::getBase(mold),
       builder.createIntegerConstant(
           loc, callee.getFunctionType().getInputs()[2], rank));
-  builder.create<fir::CallOp>(loc, callee, args);
+  fir::CallOp::create(builder, loc, callee, args);
 }
 
 /// Generate a runtime call to deallocate memory.
@@ -270,7 +270,7 @@ static mlir::Value genRuntimeDeallocate(fir::FirOpBuilder &builder,
         errorManager.hasStat, errorManager.errMsgAddr, errorManager.sourceFile,
         errorManager.sourceLine);
   }
-  return builder.create<fir::CallOp>(loc, callee, operands).getResult(0);
+  return fir::CallOp::create(builder, loc, callee, operands).getResult(0);
 }
 
 //===----------------------------------------------------------------------===//
@@ -433,9 +433,9 @@ class AllocateStmtHelper {
           loc, Fortran::semantics::GetExpr(std::get<1>(shapeSpec.t)), stmtCtx));
       ub = builder.createConvert(loc, idxTy, ub);
       if (lb) {
-        mlir::Value 
diff  = builder.create<mlir::arith::SubIOp>(loc, ub, lb);
+        mlir::Value 
diff  = mlir::arith::SubIOp::create(builder, loc, ub, lb);
         extents.emplace_back(
-            builder.create<mlir::arith::AddIOp>(loc, 
diff , one));
+            mlir::arith::AddIOp::create(builder, loc, 
diff , one));
       } else {
         extents.emplace_back(ub);
       }
@@ -461,7 +461,7 @@ class AllocateStmtHelper {
     mlir::Value falseValue = builder.createBool(loc, false);
     mlir::Value falseConv = builder.createConvert(
         loc, fir::unwrapRefType(pinned.getType()), falseValue);
-    builder.create<fir::StoreOp>(loc, falseConv, pinned);
+    fir::StoreOp::create(builder, loc, falseConv, pinned);
   }
 
   void genSimpleAllocation(const Allocation &alloc,
@@ -557,7 +557,7 @@ class AllocateStmtHelper {
       mlir::Value nullPointer = fir::factory::createUnallocatedBox(
           builder, loc, box.getBoxTy(), box.nonDeferredLenParams(),
           /*typeSourceBox=*/{}, allocatorIdx);
-      builder.create<fir::StoreOp>(loc, nullPointer, box.getAddr());
+      fir::StoreOp::create(builder, loc, nullPointer, box.getAddr());
     } else {
       assert(box.isAllocatable() && "must be an allocatable");
       // For allocatables, sync the MutableBoxValue and descriptor before the
@@ -597,13 +597,14 @@ class AllocateStmtHelper {
       assert(sourceBox && "source expression should be lowered to one box");
       for (int i = 0; i < sourceExpr->Rank(); ++i) {
         auto dimVal = builder.createIntegerConstant(loc, idxTy, i);
-        auto dimInfo = builder.create<fir::BoxDimsOp>(
-            loc, idxTy, idxTy, idxTy, sourceBox->getAddr(), dimVal);
+        auto dimInfo = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
+                                              sourceBox->getAddr(), dimVal);
         mlir::Value lb =
             fir::factory::readLowerBound(builder, loc, sourceExv, i, one);
         mlir::Value extent = dimInfo.getResult(1);
-        mlir::Value ub = builder.create<mlir::arith::SubIOp>(
-            loc, builder.create<mlir::arith::AddIOp>(loc, extent, lb), one);
+        mlir::Value ub = mlir::arith::SubIOp::create(
+            builder, loc, mlir::arith::AddIOp::create(builder, loc, extent, lb),
+            one);
         mlir::Value dimIndex = builder.createIntegerConstant(loc, i32Ty, i);
         genRuntimeSetBounds(builder, loc, box, dimIndex, lb, ub);
       }
@@ -668,7 +669,7 @@ class AllocateStmtHelper {
     const auto args = fir::runtime::createArguments(
         builder, loc, callee.getFunctionType(), box.getAddr(), typeDescAddr,
         rankValue, corankValue);
-    builder.create<fir::CallOp>(loc, callee, args);
+    fir::CallOp::create(builder, loc, callee, args);
   }
 
   /// Generate call to PointerNullifyIntrinsic or AllocatableInitIntrinsic to
@@ -697,7 +698,7 @@ class AllocateStmtHelper {
     const auto args = fir::runtime::createArguments(
         builder, loc, callee.getFunctionType(), box.getAddr(), categoryValue,
         kindValue, rankValue, corankValue);
-    builder.create<fir::CallOp>(loc, callee, args);
+    fir::CallOp::create(builder, loc, callee, args);
   }
 
   /// Generate call to the AllocatableInitDerived to set up the type descriptor
@@ -909,8 +910,8 @@ void Fortran::lower::genDeallocateIfAllocated(
       .genThen([&]() {
         if (mlir::Type eleType = box.getEleTy();
             mlir::isa<fir::RecordType>(eleType) && box.isPolymorphic()) {
-          mlir::Value declaredTypeDesc = builder.create<fir::TypeDescOp>(
-              loc, mlir::TypeAttr::get(eleType));
+          mlir::Value declaredTypeDesc = fir::TypeDescOp::create(
+              builder, loc, mlir::TypeAttr::get(eleType));
           genDeallocateBox(converter, box, loc, sym, declaredTypeDesc);
         } else {
           genDeallocateBox(converter, box, loc, sym);
@@ -1151,7 +1152,7 @@ mlir::Value Fortran::lower::getAssumedCharAllocatableOrPointerLen(
   // here).
   auto readLength = [&]() {
     fir::BoxValue boxLoad =
-        builder.create<fir::LoadOp>(loc, fir::getBase(box)).getResult();
+        fir::LoadOp::create(builder, loc, fir::getBase(box)).getResult();
     return fir::factory::readCharLen(builder, loc, boxLoad);
   };
   if (Fortran::semantics::IsOptional(sym)) {
@@ -1160,15 +1161,15 @@ mlir::Value Fortran::lower::getAssumedCharAllocatableOrPointerLen(
     // they are absents. According to 15.5.2.12 3 (9), it is illegal to
     // inquire the length of absent optional, even if non deferred, so
     // it's fine to use undefOp in this case.
-    auto isPresent = builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
-                                                      fir::getBase(box));
+    auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
+                                              fir::getBase(box));
     mlir::Value len =
         builder.genIfOp(loc, {idxTy}, isPresent, true)
             .genThen(
-                [&]() { builder.create<fir::ResultOp>(loc, readLength()); })
+                [&]() { fir::ResultOp::create(builder, loc, readLength()); })
             .genElse([&]() {
-              auto undef = builder.create<fir::UndefOp>(loc, idxTy);
-              builder.create<fir::ResultOp>(loc, undef.getResult());
+              auto undef = fir::UndefOp::create(builder, loc, idxTy);
+              fir::ResultOp::create(builder, loc, undef.getResult());
             })
             .getResults()[0];
     return len;
@@ -1183,5 +1184,5 @@ mlir::Value Fortran::lower::getTypeDescAddr(
   mlir::Type typeDesc =
       Fortran::lower::translateDerivedTypeToFIRType(converter, typeSpec);
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  return builder.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(typeDesc));
+  return fir::TypeDescOp::create(builder, loc, mlir::TypeAttr::get(typeDesc));
 }

diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 7ce397a11861b..b94833d852b2e 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -333,11 +333,12 @@ class TypeInfoConverter {
         if (details.numPrivatesNotOverridden() > 0)
           tbpName += "."s + std::to_string(details.numPrivatesNotOverridden());
         std::string bindingName = converter.mangleName(details.symbol());
-        builder.create<fir::DTEntryOp>(
-            info.loc, mlir::StringAttr::get(builder.getContext(), tbpName),
+        fir::DTEntryOp::create(
+            builder, info.loc,
+            mlir::StringAttr::get(builder.getContext(), tbpName),
             mlir::SymbolRefAttr::get(builder.getContext(), bindingName));
       }
-      builder.create<fir::FirEndOp>(info.loc);
+      fir::FirEndOp::create(builder, info.loc);
     }
     // Gather info about components that is not reflected in fir.type and may be
     // needed later: component initial values and array component non default
@@ -360,11 +361,11 @@ class TypeInfoConverter {
           componentInfo = builder.createBlock(&dt.getComponentInfo());
         auto compName = mlir::StringAttr::get(builder.getContext(),
                                               toStringRef(component.name()));
-        builder.create<fir::DTComponentOp>(info.loc, compName, lbs, init_val);
+        fir::DTComponentOp::create(builder, info.loc, compName, lbs, init_val);
       }
     }
     if (componentInfo)
-      builder.create<fir::FirEndOp>(info.loc);
+      fir::FirEndOp::create(builder, info.loc);
     builder.restoreInsertionPoint(insertPointIfCreated);
   }
 
@@ -4829,18 +4830,18 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           base = convertOp.getValue();
         // Special case if the rhs is a constant.
         if (matchPattern(base.getDefiningOp(), mlir::m_Constant())) {
-          builder.create<cuf::DataTransferOp>(loc, base, lhsVal, shape,
-                                              transferKindAttr);
+          cuf::DataTransferOp::create(builder, loc, base, lhsVal, shape,
+                                      transferKindAttr);
         } else {
           auto associate = hlfir::genAssociateExpr(
               loc, builder, rhs, rhs.getType(), ".cuf_host_tmp");
-          builder.create<cuf::DataTransferOp>(loc, associate.getBase(), lhsVal,
-                                              shape, transferKindAttr);
-          builder.create<hlfir::EndAssociateOp>(loc, associate);
+          cuf::DataTransferOp::create(builder, loc, associate.getBase(), lhsVal,
+                                      shape, transferKindAttr);
+          hlfir::EndAssociateOp::create(builder, loc, associate);
         }
       } else {
-        builder.create<cuf::DataTransferOp>(loc, rhsVal, lhsVal, shape,
-                                            transferKindAttr);
+        cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
+                                    transferKindAttr);
       }
       return;
     }
@@ -4849,8 +4850,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     if (!lhsIsDevice && rhsIsDevice) {
       auto transferKindAttr = cuf::DataTransferKindAttr::get(
           builder.getContext(), cuf::DataTransferKind::DeviceHost);
-      builder.create<cuf::DataTransferOp>(loc, rhsVal, lhsVal, shape,
-                                          transferKindAttr);
+      cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
+                                  transferKindAttr);
       return;
     }
 
@@ -4859,8 +4860,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       assert(rhs.isVariable() && "CUDA Fortran assignment rhs is not legal");
       auto transferKindAttr = cuf::DataTransferKindAttr::get(
           builder.getContext(), cuf::DataTransferKind::DeviceDevice);
-      builder.create<cuf::DataTransferOp>(loc, rhsVal, lhsVal, shape,
-                                          transferKindAttr);
+      cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
+                                  transferKindAttr);
       return;
     }
     llvm_unreachable("Unhandled CUDA data transfer");
@@ -4906,8 +4907,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           addSymbol(sym,
                     hlfir::translateToExtendedValue(loc, builder, temp).first,
                     /*forced=*/true);
-          builder.create<cuf::DataTransferOp>(
-              loc, addr, temp, /*shape=*/mlir::Value{}, transferKindAttr);
+          cuf::DataTransferOp::create(builder, loc, addr, temp,
+                                      /*shape=*/mlir::Value{},
+                                      transferKindAttr);
           ++nbDeviceResidentObject;
         }
       }
@@ -4996,13 +4998,13 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       if (isCUDATransfer && !hasCUDAImplicitTransfer)
         genCUDADataTransfer(builder, loc, assign, lhs, rhs);
       else
-        builder.create<hlfir::AssignOp>(loc, rhs, lhs,
-                                        isWholeAllocatableAssignment,
-                                        keepLhsLengthInAllocatableAssignment);
+        hlfir::AssignOp::create(builder, loc, rhs, lhs,
+                                isWholeAllocatableAssignment,
+                                keepLhsLengthInAllocatableAssignment);
       if (hasCUDAImplicitTransfer && !isInDeviceContext) {
         localSymbols.popScope();
         for (mlir::Value temp : implicitTemps)
-          builder.create<fir::FreeMemOp>(loc, temp);
+          fir::FreeMemOp::create(builder, loc, temp);
       }
       return;
     }
@@ -5010,13 +5012,13 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // left-hand side requires using an hlfir.region_assign in HLFIR. The
     // right-hand side and left-hand side must be evaluated inside the
     // hlfir.region_assign regions.
-    auto regionAssignOp = builder.create<hlfir::RegionAssignOp>(loc);
+    auto regionAssignOp = hlfir::RegionAssignOp::create(builder, loc);
 
     // Lower RHS in its own region.
     builder.createBlock(&regionAssignOp.getRhsRegion());
     Fortran::lower::StatementContext rhsContext;
     hlfir::Entity rhs = evaluateRhs(rhsContext);
-    auto rhsYieldOp = builder.create<hlfir::YieldOp>(loc, rhs);
+    auto rhsYieldOp = hlfir::YieldOp::create(builder, loc, rhs);
     Fortran::lower::genCleanUpInRegionIfAny(
         loc, builder, rhsYieldOp.getCleanup(), rhsContext);
     // Lower LHS in its own region.
@@ -5025,7 +5027,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::Value lhsYield = nullptr;
     if (!lhsHasVectorSubscripts) {
       hlfir::Entity lhs = evaluateLhs(lhsContext);
-      auto lhsYieldOp = builder.create<hlfir::YieldOp>(loc, lhs);
+      auto lhsYieldOp = hlfir::YieldOp::create(builder, loc, lhs);
       Fortran::lower::genCleanUpInRegionIfAny(
           loc, builder, lhsYieldOp.getCleanup(), lhsContext);
       lhsYield = lhs;
@@ -5054,7 +5056,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       builder.createBlock(&regionAssignOp.getUserDefinedAssignment(),
                           mlir::Region::iterator{}, {rhsType, lhsType},
                           {loc, loc});
-      auto end = builder.create<fir::FirEndOp>(loc);
+      auto end = fir::FirEndOp::create(builder, loc);
       builder.setInsertionPoint(end);
       hlfir::Entity lhsBlockArg{regionAssignOp.getUserAssignmentLhs()};
       hlfir::Entity rhsBlockArg{regionAssignOp.getUserAssignmentRhs()};

diff  --git a/flang/lib/Lower/ConvertArrayConstructor.cpp b/flang/lib/Lower/ConvertArrayConstructor.cpp
index 7e2142693eac5..55c4b45554f78 100644
--- a/flang/lib/Lower/ConvertArrayConstructor.cpp
+++ b/flang/lib/Lower/ConvertArrayConstructor.cpp
@@ -137,9 +137,9 @@ class InlinedTempStrategyImpl : public StrategyBase,
                              mlir::Value stride) {
     if constexpr (!hasLoops)
       fir::emitFatalError(loc, "array constructor lowering is inconsistent");
-    auto loop = builder.create<fir::DoLoopOp>(loc, lower, upper, stride,
-                                              /*unordered=*/false,
-                                              /*finalCount=*/false);
+    auto loop = fir::DoLoopOp::create(builder, loc, lower, upper, stride,
+                                      /*unordered=*/false,
+                                      /*finalCount=*/false);
     builder.setInsertionPointToStart(loop.getBody());
     return loop.getInductionVar();
   }
@@ -213,15 +213,15 @@ class AsElementalStrategy : public StrategyBase {
     assert(!elementalOp && "expected only one implied-do");
     mlir::Value one =
         builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-    elementalOp = builder.create<hlfir::ElementalOp>(
-        loc, exprType, shape,
-        /*mold=*/nullptr, lengthParams, /*isUnordered=*/true);
+    elementalOp = hlfir::ElementalOp::create(builder, loc, exprType, shape,
+                                             /*mold=*/nullptr, lengthParams,
+                                             /*isUnordered=*/true);
     builder.setInsertionPointToStart(elementalOp.getBody());
     // implied-do-index = lower+((i-1)*stride)
-    mlir::Value 
diff  = builder.create<mlir::arith::SubIOp>(
-        loc, elementalOp.getIndices()[0], one);
-    mlir::Value mul = builder.create<mlir::arith::MulIOp>(loc, 
diff , stride);
-    mlir::Value add = builder.create<mlir::arith::AddIOp>(loc, lower, mul);
+    mlir::Value 
diff  = mlir::arith::SubIOp::create(
+        builder, loc, elementalOp.getIndices()[0], one);
+    mlir::Value mul = mlir::arith::MulIOp::create(builder, loc, 
diff , stride);
+    mlir::Value add = mlir::arith::AddIOp::create(builder, loc, lower, mul);
     return add;
   }
 
@@ -260,7 +260,7 @@ class AsElementalStrategy : public StrategyBase {
     if (destroyOp)
       destroyOp->erase();
 
-    builder.create<hlfir::YieldElementOp>(loc, elementResult);
+    hlfir::YieldElementOp::create(builder, loc, elementResult);
   }
 
   // Override the default, because the context scope must be popped in
@@ -315,8 +315,8 @@ class RuntimeTempStrategy : public StrategyBase {
       mlir::Value tempStorage = builder.createHeapTemporary(
           loc, declaredType, tempName, extents, lengths);
       mlir::Value shape = builder.genShape(loc, extents);
-      declare = builder.create<hlfir::DeclareOp>(
-          loc, tempStorage, tempName, shape, lengths,
+      declare = hlfir::DeclareOp::create(
+          builder, loc, tempStorage, tempName, shape, lengths,
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       initialBoxValue =
           builder.createBox(loc, boxType, declare->getOriginalBase(), shape,
@@ -347,7 +347,7 @@ class RuntimeTempStrategy : public StrategyBase {
                                           /*slice=*/mlir::Value{}, emboxLengths,
                                           /*tdesc=*/{});
     }
-    builder.create<fir::StoreOp>(loc, initialBoxValue, allocatableTemp);
+    fir::StoreOp::create(builder, loc, initialBoxValue, allocatableTemp);
     arrayConstructorVector = fir::runtime::genInitArrayConstructorVector(
         loc, builder, allocatableTemp,
         builder.createBool(loc, missingLengthParameters));
@@ -369,7 +369,7 @@ class RuntimeTempStrategy : public StrategyBase {
           loc, builder, value, arrayConstructorElementType);
       mlir::Value addr = fir::getBase(addrExv);
       if (mlir::isa<fir::BaseBoxType>(addr.getType()))
-        addr = builder.create<fir::BoxAddrOp>(loc, addr);
+        addr = fir::BoxAddrOp::create(builder, loc, addr);
       fir::runtime::genPushArrayConstructorSimpleScalar(
           loc, builder, arrayConstructorVector, addr);
       if (cleanUp)
@@ -389,9 +389,9 @@ class RuntimeTempStrategy : public StrategyBase {
   mlir::Value startImpliedDo(mlir::Location loc, fir::FirOpBuilder &builder,
                              mlir::Value lower, mlir::Value upper,
                              mlir::Value stride) {
-    auto loop = builder.create<fir::DoLoopOp>(loc, lower, upper, stride,
-                                              /*unordered=*/false,
-                                              /*finalCount=*/false);
+    auto loop = fir::DoLoopOp::create(builder, loc, lower, upper, stride,
+                                      /*unordered=*/false,
+                                      /*finalCount=*/false);
     builder.setInsertionPointToStart(loop.getBody());
     return loop.getInductionVar();
   }
@@ -409,7 +409,7 @@ class RuntimeTempStrategy : public StrategyBase {
     else
       temp = hlfir::derefPointersAndAllocatables(
           loc, builder, hlfir::Entity{allocatableTemp});
-    auto hlfirExpr = builder.create<hlfir::AsExprOp>(loc, temp, mustFree);
+    auto hlfirExpr = hlfir::AsExprOp::create(builder, loc, temp, mustFree);
     return hlfir::Entity{hlfirExpr};
   }
 

diff  --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 071513303da25..8c3648bcb0f35 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -85,7 +85,7 @@ static mlir::Value genRecordCPtrValueArg(fir::FirOpBuilder &builder,
                                          mlir::Location loc, mlir::Value rec,
                                          mlir::Type ty) {
   mlir::Value cAddr = fir::factory::genCPtrOrCFunptrAddr(builder, loc, rec, ty);
-  mlir::Value cVal = builder.create<fir::LoadOp>(loc, cAddr);
+  mlir::Value cVal = fir::LoadOp::create(builder, loc, cAddr);
   return builder.createConvert(loc, cAddr.getType(), cVal);
 }
 
@@ -159,8 +159,8 @@ static mlir::Value readDim3Value(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Type refI32Ty = fir::ReferenceType::get(i32Ty);
   llvm::SmallVector<mlir::Value> lenParams;
 
-  mlir::Value designate = builder.create<hlfir::DesignateOp>(
-      loc, refI32Ty, dim3Addr, /*component=*/comp,
+  mlir::Value designate = hlfir::DesignateOp::create(
+      builder, loc, refI32Ty, dim3Addr, /*component=*/comp,
       /*componentShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
       /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
       mlir::Value{}, lenParams);
@@ -229,8 +229,8 @@ static mlir::Value remapActualToDummyDescriptor(
   if (fir::isPolymorphicType(dummyBoxType))
     mold = explicitArgument;
   mlir::Value remapped =
-      builder.create<fir::EmboxOp>(loc, dummyBoxType, baseAddr, shape,
-                                   /*slice=*/mlir::Value{}, lengths, mold);
+      fir::EmboxOp::create(builder, loc, dummyBoxType, baseAddr, shape,
+                           /*slice=*/mlir::Value{}, lengths, mold);
   if (mapSymbols)
     symMap.popScope();
   return remapped;
@@ -273,12 +273,12 @@ static void remapActualToDummyDescriptors(
                   mlir::Value newBox = remapActualToDummyDescriptor(
                       loc, converter, symMap, argLambdaCapture, caller,
                       isBindcCall);
-                  builder.create<fir::ResultOp>(loc, newBox);
+                  fir::ResultOp::create(builder, loc, newBox);
                 })
                 .genElse([&]() {
                   mlir::Value absent =
-                      builder.create<fir::AbsentOp>(loc, dummyType);
-                  builder.create<fir::ResultOp>(loc, absent);
+                      fir::AbsentOp::create(builder, loc, dummyType);
+                  fir::ResultOp::create(builder, loc, absent);
                 })
                 .getResults()[0];
         caller.placeInput(arg, remapped);
@@ -381,8 +381,8 @@ Fortran::lower::genCallOpAndResult(
 
     if (isExprCall) {
       mlir::Type exprType = hlfir::getExprType(type);
-      evaluateInMemory = builder.create<hlfir::EvaluateInMemoryOp>(
-          loc, exprType, arrayResultShape, resultLengths);
+      evaluateInMemory = hlfir::EvaluateInMemoryOp::create(
+          builder, loc, exprType, arrayResultShape, resultLengths);
       builder.setInsertionPointToStart(&evaluateInMemory.getBody().front());
       return toExtendedValue(loc, evaluateInMemory.getMemory(), extents,
                              lengths);
@@ -454,7 +454,7 @@ Fortran::lower::genCallOpAndResult(
     if (!addHostAssociations &&
         mustCastFuncOpToCopeWithImplicitInterfaceMismatch(
             loc, converter, callSiteType, funcOpType))
-      funcPointer = builder.create<fir::AddrOfOp>(loc, funcOpType, symbolAttr);
+      funcPointer = fir::AddrOfOp::create(builder, loc, funcOpType, symbolAttr);
     else
       funcSymbolAttr = symbolAttr;
 
@@ -482,7 +482,7 @@ Fortran::lower::genCallOpAndResult(
   if (funcPointer) {
     operands.push_back(
         mlir::isa<fir::BoxProcType>(funcPointer.getType())
-            ? builder.create<fir::BoxAddrOp>(loc, funcType, funcPointer)
+            ? fir::BoxAddrOp::create(builder, loc, funcType, funcPointer)
             : builder.createConvert(loc, funcType, funcPointer));
   }
 
@@ -499,10 +499,10 @@ Fortran::lower::genCallOpAndResult(
       auto funcTy = mlir::FunctionType::get(context, {}, {});
       auto boxProcTy = builder.getBoxProcType(funcTy);
       if (mlir::Value host = argumentHostAssocs(converter, fst)) {
-        cast = builder.create<fir::EmboxProcOp>(
-            loc, boxProcTy, llvm::ArrayRef<mlir::Value>{fst, host});
+        cast = fir::EmboxProcOp::create(builder, loc, boxProcTy,
+                                        llvm::ArrayRef<mlir::Value>{fst, host});
       } else {
-        cast = builder.create<fir::EmboxProcOp>(loc, boxProcTy, fst);
+        cast = fir::EmboxProcOp::create(builder, loc, boxProcTy, fst);
       }
     } else {
       mlir::Type fromTy = fir::unwrapRefType(fst.getType());
@@ -613,10 +613,10 @@ Fortran::lower::genCallOpAndResult(
       stream = fir::getBase(converter.genExprAddr(
           caller.getCallDescription().chevrons()[3], stmtCtx));
 
-    builder.create<cuf::KernelLaunchOp>(
-        loc, funcType.getResults(), funcSymbolAttr, grid_x, grid_y, grid_z,
-        block_x, block_y, block_z, bytes, stream, operands,
-        /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr);
+    cuf::KernelLaunchOp::create(builder, loc, funcType.getResults(),
+                                funcSymbolAttr, grid_x, grid_y, grid_z, block_x,
+                                block_y, block_z, bytes, stream, operands,
+                                /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr);
     callNumResults = 0;
   } else if (caller.requireDispatchCall()) {
     // Procedure call requiring a dynamic dispatch. Call is created with
@@ -640,8 +640,8 @@ Fortran::lower::genCallOpAndResult(
       // passed object because interface mismatch issues may have inserted a
       // cast to the operand with a 
diff erent declared type, which would break
       // later type bound call resolution in the FIR to FIR pass.
-      dispatch = builder.create<fir::DispatchOp>(
-          loc, funcType.getResults(), builder.getStringAttr(procName),
+      dispatch = fir::DispatchOp::create(
+          builder, loc, funcType.getResults(), builder.getStringAttr(procName),
           caller.getInputs()[*passArg], operands,
           builder.getI32IntegerAttr(*passArg), /*arg_attrs=*/nullptr,
           /*res_attrs=*/nullptr, procAttrs);
@@ -656,9 +656,9 @@ Fortran::lower::genCallOpAndResult(
       mlir::Value passObject = fir::getBase(dataRefValue);
 
       if (fir::isa_ref_type(passObject.getType()))
-        passObject = builder.create<fir::LoadOp>(loc, passObject);
-      dispatch = builder.create<fir::DispatchOp>(
-          loc, funcType.getResults(), builder.getStringAttr(procName),
+        passObject = fir::LoadOp::create(builder, loc, passObject);
+      dispatch = fir::DispatchOp::create(
+          builder, loc, funcType.getResults(), builder.getStringAttr(procName),
           passObject, operands, nullptr, /*arg_attrs=*/nullptr,
           /*res_attrs=*/nullptr, procAttrs);
     }
@@ -667,8 +667,8 @@ Fortran::lower::genCallOpAndResult(
       callResult = dispatch.getResult(0);
   } else {
     // Standard procedure call with fir.call.
-    auto call = builder.create<fir::CallOp>(
-        loc, funcType.getResults(), funcSymbolAttr, operands,
+    auto call = fir::CallOp::create(
+        builder, loc, funcType.getResults(), funcSymbolAttr, operands,
         /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, procAttrs);
 
     callNumResults = call.getNumResults();
@@ -691,9 +691,9 @@ Fortran::lower::genCallOpAndResult(
 
   if (caller.mustSaveResult()) {
     assert(allocatedResult.has_value());
-    builder.create<fir::SaveResultOp>(loc, callResult,
-                                      fir::getBase(*allocatedResult),
-                                      arrayResultShape, resultLengths);
+    fir::SaveResultOp::create(builder, loc, callResult,
+                              fir::getBase(*allocatedResult), arrayResultShape,
+                              resultLengths);
   }
 
   if (evaluateInMemory) {
@@ -864,9 +864,9 @@ static hlfir::EntityWithAttributes genStmtFunctionRef(
   // The result must not be a variable.
   result = hlfir::loadTrivialScalar(loc, builder, result);
   if (result.isVariable())
-    result = hlfir::Entity{builder.create<hlfir::AsExprOp>(loc, result)};
+    result = hlfir::Entity{hlfir::AsExprOp::create(builder, loc, result)};
   for (auto associate : exprAssociations)
-    builder.create<hlfir::EndAssociateOp>(loc, associate);
+    hlfir::EndAssociateOp::create(builder, loc, associate);
   return hlfir::EntityWithAttributes{result};
 }
 
@@ -951,9 +951,9 @@ extendedValueToHlfirEntity(mlir::Location loc, fir::FirOpBuilder &builder,
     // rid of the memory indirection in a = char(b), so there is
     // little incentive to increase the compiler complexity.
     hlfir::Entity storage{builder.createTemporary(loc, charTy)};
-    builder.create<fir::StoreOp>(loc, firBase, storage);
-    auto asExpr = builder.create<hlfir::AsExprOp>(
-        loc, storage, /*mustFree=*/builder.createBool(loc, false));
+    fir::StoreOp::create(builder, loc, firBase, storage);
+    auto asExpr = hlfir::AsExprOp::create(
+        builder, loc, storage, /*mustFree=*/builder.createBool(loc, false));
     return hlfir::EntityWithAttributes{asExpr.getResult()};
   }
   return hlfir::genDeclare(loc, builder, exv, name,
@@ -965,7 +965,7 @@ namespace {
 struct CallCleanUp {
   struct CopyIn {
     void genCleanUp(mlir::Location loc, fir::FirOpBuilder &builder) {
-      builder.create<hlfir::CopyOutOp>(loc, tempBox, wasCopied, copyBackVar);
+      hlfir::CopyOutOp::create(builder, loc, tempBox, wasCopied, copyBackVar);
     }
     // address of the descriptor holding the temp if a temp was created.
     mlir::Value tempBox;
@@ -976,7 +976,7 @@ struct CallCleanUp {
   };
   struct ExprAssociate {
     void genCleanUp(mlir::Location loc, fir::FirOpBuilder &builder) {
-      builder.create<hlfir::EndAssociateOp>(loc, tempVar, mustFree);
+      hlfir::EndAssociateOp::create(builder, loc, tempVar, mustFree);
     }
     mlir::Value tempVar;
     mlir::Value mustFree;
@@ -1074,7 +1074,7 @@ struct ConditionallyPreparedDummy {
   /// Generate the "fir.result %preparedDummy" in the then branch of the
   /// wrapping fir.if.
   void genThenResult(mlir::Location loc, fir::FirOpBuilder &builder) const {
-    builder.create<fir::ResultOp>(loc, thenResultValues);
+    fir::ResultOp::create(builder, loc, thenResultValues);
   }
 
   /// Generate the "fir.result %absent" in the else branch of the
@@ -1089,7 +1089,7 @@ struct ConditionallyPreparedDummy {
       else
         elseResultValues.push_back(builder.genAbsentOp(loc, type));
     }
-    builder.create<fir::ResultOp>(loc, elseResultValues);
+    fir::ResultOp::create(builder, loc, elseResultValues);
   }
 
   /// Once the fir.if has been created, get the resulting %conditionallyPrepared
@@ -1134,7 +1134,7 @@ static hlfir::Entity fixProcedureDummyMismatch(mlir::Location loc,
   if (mlir::isa<fir::BoxProcType>(actual.getType()) &&
       fir::isCharacterProcedureTuple(dummyType)) {
     mlir::Value length =
-        builder.create<fir::UndefOp>(loc, builder.getCharacterLengthType());
+        fir::UndefOp::create(builder, loc, builder.getCharacterLengthType());
     mlir::Value tuple = fir::factory::createCharacterProcedureTuple(
         builder, loc, dummyType, actual, length);
     return hlfir::Entity{tuple};
@@ -1317,8 +1317,8 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     mlir::Type tempBoxType = baseBoxTy.getBoxTypeWithNewAttr(
         fir::BaseBoxType::Attribute::Allocatable);
     mlir::Value tempBox = builder.createTemporary(loc, tempBoxType);
-    auto copyIn = builder.create<hlfir::CopyInOp>(
-        loc, var, tempBox, /*var_is_present=*/mlir::Value{});
+    auto copyIn = hlfir::CopyInOp::create(builder, loc, var, tempBox,
+                                          /*var_is_present=*/mlir::Value{});
     // Register the copy-out after the call.
     preparedDummy.pushCopyInCleanUp(copyIn.getTempBox(), copyIn.getWasCopied(),
                                     doCopyOut ? copyIn.getVar()
@@ -1330,16 +1330,17 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     fir::BaseBoxType boxType = fir::BoxType::get(
         hlfir::getFortranElementOrSequenceType(dummyTypeWithActualRank));
     if (actualIsAssumedRank)
-      return hlfir::Entity{builder.create<fir::ReboxAssumedRankOp>(
-          loc, boxType, var, fir::LowerBoundModifierAttribute::SetToOnes)};
+      return hlfir::Entity{fir::ReboxAssumedRankOp::create(
+          builder, loc, boxType, var,
+          fir::LowerBoundModifierAttribute::SetToOnes)};
     // Use actual shape when creating descriptor with dummy type, the dummy
     // shape may be unknown in case of sequence association.
     mlir::Type actualTy =
         hlfir::getFortranElementOrSequenceType(actual.getType());
     boxType = boxType.getBoxTypeWithNewShape(actualTy);
-    return hlfir::Entity{builder.create<fir::ReboxOp>(loc, boxType, var,
-                                                      /*shape=*/mlir::Value{},
-                                                      /*slice=*/mlir::Value{})};
+    return hlfir::Entity{fir::ReboxOp::create(builder, loc, boxType, var,
+                                              /*shape=*/mlir::Value{},
+                                              /*slice=*/mlir::Value{})};
   };
 
   // Step 2: prepare the storage for the dummy arguments, ensuring that it
@@ -1361,7 +1362,7 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
         // generated writes in copy-out.
         isParameterObjectOrSubObject(entity)) {
       // Make a copy in a temporary.
-      auto copy = builder.create<hlfir::AsExprOp>(loc, entity);
+      auto copy = hlfir::AsExprOp::create(builder, loc, entity);
       mlir::Type storageType = entity.getType();
       mlir::NamedAttribute byRefAttr = fir::getAdaptToByRefAttr(builder);
       hlfir::AssociateOp associate = hlfir::genAssociateExpr(
@@ -1441,14 +1442,14 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
         auto lbModifier = needsZeroLowerBounds
                               ? fir::LowerBoundModifierAttribute::SetToZeroes
                               : fir::LowerBoundModifierAttribute::SetToOnes;
-        entity = hlfir::Entity{builder.create<fir::ReboxAssumedRankOp>(
-            loc, dummyTypeWithActualRank, entity, lbModifier)};
+        entity = hlfir::Entity{fir::ReboxAssumedRankOp::create(
+            builder, loc, dummyTypeWithActualRank, entity, lbModifier)};
       } else {
         mlir::Value shift{};
         if (needsZeroLowerBounds)
           shift = getZeroLowerBounds(loc, builder, entity);
-        entity = hlfir::Entity{builder.create<fir::ReboxOp>(
-            loc, dummyTypeWithActualRank, entity, /*shape=*/shift,
+        entity = hlfir::Entity{fir::ReboxOp::create(
+            builder, loc, dummyTypeWithActualRank, entity, /*shape=*/shift,
             /*slice=*/mlir::Value{})};
       }
     }
@@ -1501,8 +1502,8 @@ static PreparedDummyArgument prepareUserCallActualArgument(
   // for this unusual if/then/else generation is that the number
   // and types of the if results will depend on how the argument
   // is prepared, and forecasting that here would be brittle.
-  auto badIfOp = builder.create<fir::IfOp>(loc, dummyType, isPresent,
-                                           /*withElseRegion=*/false);
+  auto badIfOp = fir::IfOp::create(builder, loc, dummyType, isPresent,
+                                   /*withElseRegion=*/false);
   mlir::Block *preparationBlock = &badIfOp.getThenRegion().front();
   builder.setInsertionPointToStart(preparationBlock);
   PreparedDummyArgument unconditionalDummy =
@@ -1520,9 +1521,9 @@ static PreparedDummyArgument prepareUserCallActualArgument(
   // badIfOp cannot be modified and used here).
   llvm::SmallVector<mlir::Type> ifOpResultTypes;
   ConditionallyPreparedDummy conditionalDummy(unconditionalDummy);
-  auto ifOp = builder.create<fir::IfOp>(loc, conditionalDummy.getIfResulTypes(),
-                                        isPresent,
-                                        /*withElseRegion=*/true);
+  auto ifOp = fir::IfOp::create(builder, loc,
+                                conditionalDummy.getIfResulTypes(), isPresent,
+                                /*withElseRegion=*/true);
   // Move "preparationBlock" into the "then" of the new
   // fir.if operation and create fir.result propagating
   // unconditionalDummy.
@@ -1559,7 +1560,7 @@ static PreparedDummyArgument prepareProcedurePointerActualArgument(
     auto tempBoxProc{builder.createTemporary(loc, boxTy)};
     hlfir::Entity nullBoxProc(
         fir::factory::createNullBoxProc(builder, loc, boxTy));
-    builder.create<fir::StoreOp>(loc, nullBoxProc, tempBoxProc);
+    fir::StoreOp::create(builder, loc, nullBoxProc, tempBoxProc);
     return PreparedDummyArgument{tempBoxProc, /*cleanups=*/{}};
   }
   hlfir::Entity actual = preparedActual.getActual(loc, builder);
@@ -1568,7 +1569,7 @@ static PreparedDummyArgument prepareProcedurePointerActualArgument(
   assert(actual.isProcedure());
   // Procedure actual to procedure pointer dummy.
   auto tempBoxProc{builder.createTemporary(loc, actual.getType())};
-  builder.create<fir::StoreOp>(loc, actual, tempBoxProc);
+  fir::StoreOp::create(builder, loc, actual, tempBoxProc);
   return PreparedDummyArgument{tempBoxProc, /*cleanups=*/{}};
 }
 
@@ -1607,7 +1608,7 @@ void prepareUserCallArguments(
                                                    "adapt.cptrbyval");
           value = hlfir::Entity{genRecordCPtrValueArg(
               builder, loc, associate.getFirBase(), eleTy)};
-          builder.create<hlfir::EndAssociateOp>(loc, associate);
+          hlfir::EndAssociateOp::create(builder, loc, associate);
         } else {
           value =
               hlfir::Entity{genRecordCPtrValueArg(builder, loc, value, eleTy)};
@@ -1626,7 +1627,7 @@ void prepareUserCallArguments(
             loadedValue = builder.createConvert(
                 loc, fir::ReferenceType::get(argTy), loadedValue);
         if (fir::isa_ref_type(loadedValue.getType()))
-          loadedValue = builder.create<fir::LoadOp>(loc, loadedValue);
+          loadedValue = fir::LoadOp::create(builder, loc, loadedValue);
         caller.placeInput(arg, loadedValue);
         if (cleanup)
           (*cleanup)();
@@ -1811,8 +1812,9 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals,
       // In such case, the expression should not be freed after its use since
       // the result is stack allocated or deallocation (for allocatable results)
       // was already inserted in genCallOpAndResult.
-      auto asExpr = builder.create<hlfir::AsExprOp>(
-          loc, resultEntity, /*mustFree=*/builder.createBool(loc, false));
+      auto asExpr =
+          hlfir::AsExprOp::create(builder, loc, resultEntity,
+                                  /*mustFree=*/builder.createBool(loc, false));
       return hlfir::EntityWithAttributes{asExpr.getResult()};
     }
     return hlfir::EntityWithAttributes{resultEntity};
@@ -1860,12 +1862,12 @@ static ExvAndCleanup genOptionalValue(fir::FirOpBuilder &builder,
                        "must be a numerical or logical scalar");
                 mlir::Value val =
                     hlfir::loadTrivialScalar(loc, builder, entity);
-                builder.create<fir::ResultOp>(loc, val);
+                fir::ResultOp::create(builder, loc, val);
               })
               .genElse([&]() {
                 mlir::Value zero =
                     fir::factory::createZeroValue(builder, loc, eleType);
-                builder.create<fir::ResultOp>(loc, zero);
+                fir::ResultOp::create(builder, loc, zero);
               })
               .getResults()[0],
           std::nullopt};
@@ -1912,9 +1914,9 @@ static ExvAndCleanup genOptionalBox(fir::FirOpBuilder &builder,
   // ensures it won't be.
   mlir::Value box = builder.createBox(loc, newExv);
   mlir::Type boxType = box.getType();
-  auto absent = builder.create<fir::AbsentOp>(loc, boxType);
-  auto boxOrAbsent = builder.create<mlir::arith::SelectOp>(
-      loc, boxType, isPresent, box, absent);
+  auto absent = fir::AbsentOp::create(builder, loc, boxType);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder, loc, boxType,
+                                                   isPresent, box, absent);
   return {fir::BoxValue(boxOrAbsent), cleanup};
 }
 
@@ -2142,10 +2144,10 @@ genIntrinsicRefCore(Fortran::lower::PreparedActualArguments &loweredActuals,
     // ownership of this address cannot be taken here since it may not be a
     // temp.
     if (intrinsicName == "merge")
-      asExpr = builder.create<hlfir::AsExprOp>(loc, resultEntity);
+      asExpr = hlfir::AsExprOp::create(builder, loc, resultEntity);
     else
-      asExpr = builder.create<hlfir::AsExprOp>(
-          loc, resultEntity, builder.createBool(loc, mustBeFreed));
+      asExpr = hlfir::AsExprOp::create(builder, loc, resultEntity,
+                                       builder.createBool(loc, mustBeFreed));
     resultEntity = hlfir::EntityWithAttributes{asExpr.getResult()};
   }
   return resultEntity;
@@ -2525,7 +2527,7 @@ genIsPresentIfArgMaybeAbsent(mlir::Location loc, hlfir::Entity actual,
   // May fall into the category above if the allocatable is not optional.
 
   // Passing an optional to an optional.
-  return builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), actual)
+  return fir::IsPresentOp::create(builder, loc, builder.getI1Type(), actual)
       .getResult();
 }
 
@@ -2813,9 +2815,9 @@ genProcedureRef(CallContext &callContext) {
           // TYPE(*) cannot be ALLOCATABLE/POINTER (C709) so there is no
           // need to cover the case of passing an ALLOCATABLE/POINTER to an
           // OPTIONAL.
-          isPresent =
-              builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), actual)
-                  .getResult();
+          isPresent = fir::IsPresentOp::create(builder, loc,
+                                               builder.getI1Type(), actual)
+                          .getResult();
         }
         loweredActuals.push_back(Fortran::lower::PreparedActualArgument{
             hlfir::Entity{*var}, isPresent});
@@ -2931,7 +2933,7 @@ std::optional<hlfir::EntityWithAttributes> Fortran::lower::convertCallToHLFIR(
     // this can be enforced whenscheduling forall/where expression evaluations.
     Fortran::lower::StatementContext localStmtCtx;
     mlir::Type bogusType = builder.getIndexType();
-    auto exactlyOnce = builder.create<hlfir::ExactlyOnceOp>(loc, bogusType);
+    auto exactlyOnce = hlfir::ExactlyOnceOp::create(builder, loc, bogusType);
     mlir::Block *block = builder.createBlock(&exactlyOnce.getBody());
     builder.setInsertionPointToStart(block);
     CallContext callContext(procRef, resultType, loc, converter, symMap,
@@ -2939,7 +2941,7 @@ std::optional<hlfir::EntityWithAttributes> Fortran::lower::convertCallToHLFIR(
     std::optional<hlfir::EntityWithAttributes> res =
         genProcedureRef(callContext);
     assert(res.has_value() && "must be a function");
-    auto yield = builder.create<hlfir::YieldOp>(loc, *res);
+    auto yield = hlfir::YieldOp::create(builder, loc, *res);
     Fortran::lower::genCleanUpInRegionIfAny(loc, builder, yield.getCleanup(),
                                             localStmtCtx);
     builder.setInsertionPointAfter(exactlyOnce);

diff  --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index b8ab5d09e3e08..768a237c92396 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -237,8 +237,8 @@ static mlir::Value genScalarLit(
                                     ? value.UnsignedDecimal()
                                     : value.SignedDecimal(),
                                 10);
-      return builder.create<mlir::arith::ConstantOp>(
-          loc, ty, mlir::IntegerAttr::get(ty, bigInt));
+      return mlir::arith::ConstantOp::create(
+          builder, loc, ty, mlir::IntegerAttr::get(ty, bigInt));
     }
     return builder.createIntegerConstant(loc, ty, value.ToInt64());
   } else if constexpr (TC == Fortran::common::TypeCategory::Logical) {
@@ -302,8 +302,9 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
     auto sizeTag = mlir::StringAttr::get(context, fir::StringLitOp::size());
     mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
     llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
-    return builder.create<fir::StringLitOp>(
-        loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
+    return fir::StringLitOp::create(builder, loc,
+                                    llvm::ArrayRef<mlir::Type>{type},
+                                    mlir::ValueRange{}, attrs);
   }
 }
 
@@ -340,11 +341,11 @@ genScalarLit(fir::FirOpBuilder &builder, mlir::Location loc,
         [&](fir::FirOpBuilder &builder) {
           fir::StringLitOp str =
               createStringLitOp<KIND>(builder, loc, value, len);
-          builder.create<fir::HasValueOp>(loc, str);
+          fir::HasValueOp::create(builder, loc, str);
         },
         builder.createLinkOnceLinkage());
-  return builder.create<fir::AddrOfOp>(loc, global.resultType(),
-                                       global.getSymbol());
+  return fir::AddrOfOp::create(builder, loc, global.resultType(),
+                               global.getSymbol());
 }
 
 // Helper to generate StructureConstructor component values.
@@ -364,9 +365,9 @@ static mlir::Value genStructureComponentInit(
   auto fieldTy = fir::FieldType::get(recTy.getContext());
   assert(componentTy && "failed to retrieve component");
   // FIXME: type parameters must come from the derived-type-spec
-  auto field = builder.create<fir::FieldIndexOp>(
-      loc, fieldTy, name, recTy,
-      /*typeParams=*/mlir::ValueRange{} /*TODO*/);
+  auto field =
+      fir::FieldIndexOp::create(builder, loc, fieldTy, name, recTy,
+                                /*typeParams=*/mlir::ValueRange{} /*TODO*/);
 
   if (Fortran::semantics::IsAllocatable(sym)) {
     if (!Fortran::evaluate::IsNullPointerOrAllocatable(&expr)) {
@@ -378,8 +379,8 @@ static mlir::Value genStructureComponentInit(
           fir::factory::createUnallocatedBox(builder, loc, componentTy, {})};
       componentValue = builder.createConvert(loc, componentTy, componentValue);
 
-      return builder.create<fir::InsertValueOp>(
-          loc, recTy, res, componentValue,
+      return fir::InsertValueOp::create(
+          builder, loc, recTy, res, componentValue,
           builder.getArrayAttr(field.getAttributes()));
     }
   }
@@ -400,9 +401,9 @@ static mlir::Value genStructureComponentInit(
     } else
       initialTarget = Fortran::lower::genInitialDataTarget(converter, loc,
                                                            componentTy, expr);
-    res = builder.create<fir::InsertValueOp>(
-        loc, recTy, res, initialTarget,
-        builder.getArrayAttr(field.getAttributes()));
+    res =
+        fir::InsertValueOp::create(builder, loc, recTy, res, initialTarget,
+                                   builder.getArrayAttr(field.getAttributes()));
     return res;
   }
 
@@ -426,7 +427,7 @@ static mlir::Value genStructureComponentInit(
     mlir::Value addr = fir::getBase(
         Fortran::lower::genExtAddrInInitializer(converter, loc, expr));
     if (mlir::isa<fir::BoxProcType>(addr.getType()))
-      addr = builder.create<fir::BoxAddrOp>(loc, addr);
+      addr = fir::BoxAddrOp::create(builder, loc, addr);
     assert((fir::isa_ref_type(addr.getType()) ||
             mlir::isa<mlir::FunctionType>(addr.getType())) &&
            "expect reference type for address field");
@@ -435,24 +436,25 @@ static mlir::Value genStructureComponentInit(
     auto cPtrRecTy = mlir::cast<fir::RecordType>(componentTy);
     llvm::StringRef addrFieldName = Fortran::lower::builtin::cptrFieldName;
     mlir::Type addrFieldTy = cPtrRecTy.getType(addrFieldName);
-    auto addrField = builder.create<fir::FieldIndexOp>(
-        loc, fieldTy, addrFieldName, componentTy,
+    auto addrField = fir::FieldIndexOp::create(
+        builder, loc, fieldTy, addrFieldName, componentTy,
         /*typeParams=*/mlir::ValueRange{});
     mlir::Value castAddr = builder.createConvert(loc, addrFieldTy, addr);
-    auto undef = builder.create<fir::UndefOp>(loc, componentTy);
-    addr = builder.create<fir::InsertValueOp>(
-        loc, componentTy, undef, castAddr,
+    auto undef = fir::UndefOp::create(builder, loc, componentTy);
+    addr = fir::InsertValueOp::create(
+        builder, loc, componentTy, undef, castAddr,
         builder.getArrayAttr(addrField.getAttributes()));
-    res = builder.create<fir::InsertValueOp>(
-        loc, recTy, res, addr, builder.getArrayAttr(field.getAttributes()));
+    res =
+        fir::InsertValueOp::create(builder, loc, recTy, res, addr,
+                                   builder.getArrayAttr(field.getAttributes()));
     return res;
   }
 
   mlir::Value val = fir::getBase(genConstantValue(converter, loc, expr));
   assert(!fir::isa_ref_type(val.getType()) && "expecting a constant value");
   mlir::Value castVal = builder.createConvert(loc, componentTy, val);
-  res = builder.create<fir::InsertValueOp>(
-      loc, recTy, res, castVal, builder.getArrayAttr(field.getAttributes()));
+  res = fir::InsertValueOp::create(builder, loc, recTy, res, castVal,
+                                   builder.getArrayAttr(field.getAttributes()));
   return res;
 }
 
@@ -465,7 +467,7 @@ static mlir::Value genInlinedStructureCtorLitImpl(
   auto recTy = mlir::cast<fir::RecordType>(type);
 
   if (!converter.getLoweringOptions().getLowerToHighLevelFIR()) {
-    mlir::Value res = builder.create<fir::UndefOp>(loc, recTy);
+    mlir::Value res = fir::UndefOp::create(builder, loc, recTy);
     for (const auto &[sym, expr] : ctor.values()) {
       // Parent components need more work because they do not appear in the
       // fir.rec type.
@@ -495,13 +497,13 @@ static mlir::Value genInlinedStructureCtorLitImpl(
         break;
     }
     for (mlir::Type parentType : llvm::reverse(parentTypes)) {
-      auto undef = builder.create<fir::UndefOp>(loc, parentType);
+      auto undef = fir::UndefOp::create(builder, loc, parentType);
       fir::RecordType parentRecTy = mlir::cast<fir::RecordType>(parentType);
-      auto field = builder.create<fir::FieldIndexOp>(
-          loc, fieldTy, parentRecTy.getTypeList()[0].first, parentType,
+      auto field = fir::FieldIndexOp::create(
+          builder, loc, fieldTy, parentRecTy.getTypeList()[0].first, parentType,
           /*typeParams=*/mlir::ValueRange{} /*TODO*/);
-      res = builder.create<fir::InsertValueOp>(
-          loc, parentRecTy, undef, res,
+      res = fir::InsertValueOp::create(
+          builder, loc, parentRecTy, undef, res,
           builder.getArrayAttr(field.getAttributes()));
     }
   };
@@ -514,7 +516,7 @@ static mlir::Value genInlinedStructureCtorLitImpl(
     if (!res) {
       mlir::Type parentType = converter.genType(*componentParentType);
       curentType = componentParentType;
-      res = builder.create<fir::UndefOp>(loc, parentType);
+      res = fir::UndefOp::create(builder, loc, parentType);
     } else if (*componentParentType != *curentType) {
       mlir::Type parentType = converter.genType(*componentParentType);
       insertParentValueIntoExtension(parentType);
@@ -524,7 +526,7 @@ static mlir::Value genInlinedStructureCtorLitImpl(
   }
 
   if (!res) // structure constructor for empty type.
-    return builder.create<fir::UndefOp>(loc, recTy);
+    return fir::UndefOp::create(builder, loc, recTy);
 
   // The last component may belong to a parent type.
   if (res.getType() != recTy)
@@ -550,12 +552,12 @@ static mlir::Value genScalarLit(
         [&](fir::FirOpBuilder &builder) {
           mlir::Value result =
               genInlinedStructureCtorLitImpl(converter, loc, value, eleTy);
-          builder.create<fir::HasValueOp>(loc, result);
+          fir::HasValueOp::create(builder, loc, result);
         },
         builder.createInternalLinkage());
   }
-  return builder.create<fir::AddrOfOp>(loc, global.resultType(),
-                                       global.getSymbol());
+  return fir::AddrOfOp::create(builder, loc, global.resultType(),
+                               global.getSymbol());
 }
 
 /// Create an evaluate::Constant<T> array to a fir.array<> value
@@ -576,7 +578,7 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
           builder.getIntegerAttr(idxTy, subscripts[i] - con.lbounds()[i]));
     return idx;
   };
-  mlir::Value array = builder.create<fir::UndefOp>(loc, arrayTy);
+  mlir::Value array = fir::UndefOp::create(builder, loc, arrayTy);
   if (Fortran::evaluate::GetSize(con.shape()) == 0)
     return array;
   if constexpr (T::category == Fortran::common::TypeCategory::Character) {
@@ -584,8 +586,9 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       mlir::Value elementVal =
           genScalarLit<T::kind>(builder, loc, con.At(subscripts), con.LEN(),
                                 /*outlineInReadOnlyMemory=*/false);
-      array = builder.create<fir::InsertValueOp>(
-          loc, arrayTy, array, elementVal, builder.getArrayAttr(createIdx()));
+      array =
+          fir::InsertValueOp::create(builder, loc, arrayTy, array, elementVal,
+                                     builder.getArrayAttr(createIdx()));
     } while (con.IncrementSubscripts(subscripts));
   } else if constexpr (T::category == Fortran::common::TypeCategory::Derived) {
     do {
@@ -594,8 +597,9 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       mlir::Value elementVal =
           genScalarLit(converter, loc, con.At(subscripts), eleTy,
                        /*outlineInReadOnlyMemory=*/false);
-      array = builder.create<fir::InsertValueOp>(
-          loc, arrayTy, array, elementVal, builder.getArrayAttr(createIdx()));
+      array =
+          fir::InsertValueOp::create(builder, loc, arrayTy, array, elementVal,
+                                     builder.getArrayAttr(createIdx()));
     } while (con.IncrementSubscripts(subscripts));
   } else {
     llvm::SmallVector<mlir::Attribute> rangeStartIdx;
@@ -611,9 +615,9 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
       bool nextIsSame = con.IncrementSubscripts(nextSubscripts) &&
                         con.At(subscripts) == con.At(nextSubscripts);
       if (!rangeSize && !nextIsSame) { // single (non-range) value
-        array = builder.create<fir::InsertValueOp>(
-            loc, arrayTy, array, getElementVal(),
-            builder.getArrayAttr(createIdx()));
+        array = fir::InsertValueOp::create(builder, loc, arrayTy, array,
+                                           getElementVal(),
+                                           builder.getArrayAttr(createIdx()));
       } else if (!rangeSize) { // start a range
         rangeStartIdx = createIdx();
         rangeSize = 1;
@@ -629,8 +633,8 @@ genInlinedArrayLit(Fortran::lower::AbstractConverter &converter,
           rangeBounds.push_back(
               mlir::cast<mlir::IntegerAttr>(idx[i]).getValue().getSExtValue());
         }
-        array = builder.create<fir::InsertOnRangeOp>(
-            loc, arrayTy, array, getElementVal(),
+        array = fir::InsertOnRangeOp::create(
+            builder, loc, arrayTy, array, getElementVal(),
             builder.getIndexVectorAttr(rangeBounds));
         rangeSize = 0;
       }
@@ -679,12 +683,12 @@ genOutlineArrayLit(Fortran::lower::AbstractConverter &converter,
           [&](fir::FirOpBuilder &builder) {
             mlir::Value result =
                 genInlinedArrayLit(converter, loc, arrayTy, constant);
-            builder.create<fir::HasValueOp>(loc, result);
+            fir::HasValueOp::create(builder, loc, result);
           },
           builder.createInternalLinkage());
   }
-  return builder.create<fir::AddrOfOp>(loc, global.resultType(),
-                                       global.getSymbol());
+  return fir::AddrOfOp::create(builder, loc, global.resultType(),
+                               global.getSymbol());
 }
 
 /// Convert an evaluate::Constant<T> array into an fir::ExtendedValue.

diff  --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 281ab229d1b6a..3578f941ec1b4 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -267,8 +267,8 @@ static mlir::Value genActualIsPresentTest(fir::FirOpBuilder &builder,
   // Optional case (not that optional allocatable/pointer cannot be absent
   // when passed to CMPLX as per 15.5.2.12 point 3 (7) and (8)). It is
   // therefore possible to catch them in the `then` case above.
-  return builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
-                                          fir::getBase(actual));
+  return fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
+                                  fir::getBase(actual));
 }
 
 /// Convert the array_load, `load`, to an extended value. If `path` is not
@@ -345,8 +345,8 @@ arrayLoadExtValue(fir::FirOpBuilder &builder, mlir::Location loc,
     auto origins = fir::factory::getNonDefaultLowerBounds(builder, loc, exv);
     if (shapeVal) {
       // shapeVal is a ShiftOp and load.memref() is a boxed value.
-      newBase = builder.create<fir::ReboxOp>(loc, oldBox.getType(), oldBox,
-                                             shapeVal, /*slice=*/mlir::Value{});
+      newBase = fir::ReboxOp::create(builder, loc, oldBox.getType(), oldBox,
+                                     shapeVal, /*slice=*/mlir::Value{});
       origins = fir::factory::getOrigins(shapeVal);
     }
     return fir::substBase(arrayToExtendedValue(extents, origins), newBase);
@@ -378,7 +378,7 @@ placeScalarValueInMemory(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value temp = builder.createTemporary(
       loc, storageType,
       llvm::ArrayRef<mlir::NamedAttribute>{fir::getAdaptToByRefAttr(builder)});
-  builder.create<fir::StoreOp>(loc, val, temp);
+  fir::StoreOp::create(builder, loc, val, temp);
   return fir::substBase(exv, temp);
 }
 
@@ -434,14 +434,14 @@ static fir::ExtendedValue genLoad(fir::FirOpBuilder &builder,
         if (mlir::isa<fir::RecordType>(
                 fir::unwrapRefType(fir::getBase(p).getType())))
           return p;
-        mlir::Value load = builder.create<fir::LoadOp>(loc, fir::getBase(p));
+        mlir::Value load = fir::LoadOp::create(builder, loc, fir::getBase(p));
         return fir::PolymorphicValue(load, p.getSourceBox());
       },
       [&](const fir::UnboxedValue &v) -> fir::ExtendedValue {
         if (mlir::isa<fir::RecordType>(
                 fir::unwrapRefType(fir::getBase(v).getType())))
           return v;
-        return builder.create<fir::LoadOp>(loc, fir::getBase(v));
+        return fir::LoadOp::create(builder, loc, fir::getBase(v));
       },
       [&](const fir::MutableBoxValue &box) -> fir::ExtendedValue {
         return genLoad(builder, loc,
@@ -473,11 +473,11 @@ static fir::ExtendedValue genOptionalValue(fir::FirOpBuilder &builder,
                /*withElseRegion=*/true)
       .genThen([&]() {
         mlir::Value val = fir::getBase(genLoad(builder, loc, exv));
-        builder.create<fir::ResultOp>(loc, val);
+        fir::ResultOp::create(builder, loc, val);
       })
       .genElse([&]() {
         mlir::Value zero = fir::factory::createZeroValue(builder, loc, eleType);
-        builder.create<fir::ResultOp>(loc, zero);
+        fir::ResultOp::create(builder, loc, zero);
       })
       .getResults()[0];
 }
@@ -521,9 +521,9 @@ static fir::ExtendedValue genOptionalBox(fir::FirOpBuilder &builder,
   // ensures it won't be.
   mlir::Value box = builder.createBox(loc, newExv);
   mlir::Type boxType = box.getType();
-  auto absent = builder.create<fir::AbsentOp>(loc, boxType);
-  auto boxOrAbsent = builder.create<mlir::arith::SelectOp>(
-      loc, boxType, isPresent, box, absent);
+  auto absent = fir::AbsentOp::create(builder, loc, boxType);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder, loc, boxType,
+                                                   isPresent, box, absent);
   return fir::BoxValue(boxOrAbsent);
 }
 
@@ -569,13 +569,13 @@ createBoxProcCharTuple(Fortran::lower::AbstractConverter &converter,
   if (fir::isa_ref_type(fromTy))
     funcAddr = builder.createConvert(loc, toTy, funcAddr);
   else if (mlir::isa<fir::BoxProcType>(fromTy))
-    funcAddr = builder.create<fir::BoxAddrOp>(loc, toTy, funcAddr);
+    funcAddr = fir::BoxAddrOp::create(builder, loc, toTy, funcAddr);
 
   auto boxProc = [&]() -> mlir::Value {
     if (auto host = Fortran::lower::argumentHostAssocs(converter, funcAddr))
-      return builder.create<fir::EmboxProcOp>(
-          loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
-    return builder.create<fir::EmboxProcOp>(loc, boxTy, funcAddr);
+      return fir::EmboxProcOp::create(
+          builder, loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
+    return fir::EmboxProcOp::create(builder, loc, boxTy, funcAddr);
   }();
   return fir::factory::createCharacterProcedureTuple(builder, loc, argTy,
                                                      boxProc, charLen);
@@ -598,7 +598,7 @@ absentBoxToUnallocatedBox(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value emptyBox =
       fir::factory::createUnallocatedBox(builder, loc, boxType, {});
   auto safeToReadBox =
-      builder.create<mlir::arith::SelectOp>(loc, isPresent, box, emptyBox);
+      mlir::arith::SelectOp::create(builder, loc, isPresent, box, emptyBox);
   return fir::substBase(exv, safeToReadBox);
 }
 
@@ -822,9 +822,9 @@ class ScalarExprLowering {
               Fortran::common::TypeCategory::Integer, *unsignedKind);
           mlir::Value lhsSL = builder.createConvert(loc, signlessType, *lhs);
           mlir::Value rhsSL = builder.createConvert(loc, signlessType, *rhs);
-          return builder.create<OpTy>(loc, pred, lhsSL, rhsSL);
+          return OpTy::create(builder, loc, pred, lhsSL, rhsSL);
         }
-        return builder.create<OpTy>(loc, pred, *lhs, *rhs);
+        return OpTy::create(builder, loc, pred, *lhs, *rhs);
       }
     }
     fir::emitFatalError(getLoc(), "array compare should be handled in genarr");
@@ -841,7 +841,7 @@ class ScalarExprLowering {
                              const ExtValue &left, const ExtValue &right) {
     if (const fir::UnboxedValue *lhs = left.getUnboxed())
       if (const fir::UnboxedValue *rhs = right.getUnboxed())
-        return builder.create<OpTy>(getLoc(), pred, *lhs, *rhs);
+        return OpTy::create(builder, getLoc(), pred, *lhs, *rhs);
     fir::emitFatalError(getLoc(), "array compare should be handled in genarr");
   }
   template <typename OpTy, typename A>
@@ -904,7 +904,7 @@ class ScalarExprLowering {
 
           mlir::Value cnvrt = Fortran::lower::addCrayPointerInst(
               loc, builder, ptrVal, ptrTy, pteVal.getType());
-          addr = builder.create<fir::LoadOp>(loc, cnvrt);
+          addr = fir::LoadOp::create(builder, loc, cnvrt);
         }
         return genLoad(addr);
       }
@@ -970,12 +970,12 @@ class ScalarExprLowering {
 
       std::string name = converter.getRecordTypeFieldName(sym);
       // FIXME: type parameters must come from the derived-type-spec
-      mlir::Value field = builder.create<fir::FieldIndexOp>(
-          loc, fieldTy, name, ty,
-          /*typeParams=*/mlir::ValueRange{} /*TODO*/);
+      mlir::Value field =
+          fir::FieldIndexOp::create(builder, loc, fieldTy, name, ty,
+                                    /*typeParams=*/mlir::ValueRange{} /*TODO*/);
       mlir::Type coorTy = builder.getRefType(recTy.getType(name));
-      auto coor = builder.create<fir::CoordinateOp>(loc, coorTy,
-                                                    fir::getBase(res), field);
+      auto coor = fir::CoordinateOp::create(builder, loc, coorTy,
+                                            fir::getBase(res), field);
       ExtValue to = fir::factory::componentToExtendedValue(builder, loc, coor);
       to.match(
           [&](const fir::UnboxedValue &toPtr) {
@@ -1077,7 +1077,7 @@ class ScalarExprLowering {
     mlir::Value input = genunbox(op.left());
     // Like LLVM, integer negation is the binary op "0 - value"
     mlir::Value zero = genIntegerConstant<KIND>(builder.getContext(), 0);
-    return builder.create<mlir::arith::SubIOp>(getLoc(), zero, input);
+    return mlir::arith::SubIOp::create(builder, getLoc(), zero, input);
   }
   template <int KIND>
   ExtValue genval(const Fortran::evaluate::Negate<Fortran::evaluate::Type<
@@ -1088,18 +1088,18 @@ class ScalarExprLowering {
     mlir::Value input = genunbox(op.left());
     mlir::Value signless = builder.createConvert(loc, signlessType, input);
     mlir::Value zero = genIntegerConstant<KIND>(builder.getContext(), 0);
-    mlir::Value neg = builder.create<mlir::arith::SubIOp>(loc, zero, signless);
+    mlir::Value neg = mlir::arith::SubIOp::create(builder, loc, zero, signless);
     return builder.createConvert(loc, input.getType(), neg);
   }
   template <int KIND>
   ExtValue genval(const Fortran::evaluate::Negate<Fortran::evaluate::Type<
                       Fortran::common::TypeCategory::Real, KIND>> &op) {
-    return builder.create<mlir::arith::NegFOp>(getLoc(), genunbox(op.left()));
+    return mlir::arith::NegFOp::create(builder, getLoc(), genunbox(op.left()));
   }
   template <int KIND>
   ExtValue genval(const Fortran::evaluate::Negate<Fortran::evaluate::Type<
                       Fortran::common::TypeCategory::Complex, KIND>> &op) {
-    return builder.create<fir::NegcOp>(getLoc(), genunbox(op.left()));
+    return fir::NegcOp::create(builder, getLoc(), genunbox(op.left()));
   }
 
   template <typename OpTy>
@@ -1312,7 +1312,7 @@ class ScalarExprLowering {
     ExtValue input = genval(op.left());
     mlir::Value base = fir::getBase(input);
     mlir::Value newBase =
-        builder.create<fir::NoReassocOp>(getLoc(), base.getType(), base);
+        fir::NoReassocOp::create(builder, getLoc(), base.getType(), base);
     return fir::substBase(input, newBase);
   }
 
@@ -1322,7 +1322,7 @@ class ScalarExprLowering {
     mlir::Value one = genBoolConstant(true);
     mlir::Value val =
         builder.createConvert(getLoc(), builder.getI1Type(), logical);
-    return builder.create<mlir::arith::XOrIOp>(getLoc(), val, one);
+    return mlir::arith::XOrIOp::create(builder, getLoc(), val, one);
   }
 
   template <int KIND>
@@ -1384,8 +1384,9 @@ class ScalarExprLowering {
     mlir::Value offset = builder.createIntegerConstant(
         loc, idxTy,
         x.part() == Fortran::evaluate::ComplexPart::Part::RE ? 0 : 1);
-    mlir::Value result = builder.create<fir::CoordinateOp>(
-        loc, builder.getRefType(eleTy), base, mlir::ValueRange{offset});
+    mlir::Value result =
+        fir::CoordinateOp::create(builder, loc, builder.getRefType(eleTy), base,
+                                  mlir::ValueRange{offset});
     return {result};
   }
   ExtValue genval(const Fortran::evaluate::ComplexPart &x) {
@@ -1500,8 +1501,8 @@ class ScalarExprLowering {
       auto recTy = mlir::cast<fir::RecordType>(ty);
       const Fortran::semantics::Symbol &sym = getLastSym(*field);
       std::string name = converter.getRecordTypeFieldName(sym);
-      coorArgs.push_back(builder.create<fir::FieldIndexOp>(
-          loc, fldTy, name, recTy, fir::getTypeParams(obj)));
+      coorArgs.push_back(fir::FieldIndexOp::create(
+          builder, loc, fldTy, name, recTy, fir::getTypeParams(obj)));
       ty = recTy.getType(name);
     }
     // If parent component is referred then it has no coordinate argument.
@@ -1510,8 +1511,8 @@ class ScalarExprLowering {
     ty = builder.getRefType(ty);
     return fir::factory::componentToExtendedValue(
         builder, loc,
-        builder.create<fir::CoordinateOp>(loc, ty, fir::getBase(obj),
-                                          coorArgs));
+        fir::CoordinateOp::create(builder, loc, ty, fir::getBase(obj),
+                                  coorArgs));
   }
 
   ExtValue gen(const Fortran::evaluate::Component &cmpt) {
@@ -1587,7 +1588,7 @@ class ScalarExprLowering {
       mlir::Value val = fir::getBase(subVal);
       mlir::Type ty = val.getType();
       mlir::Value lb = getLBound(array, subsc.index(), ty);
-      args.push_back(builder.create<mlir::arith::SubIOp>(loc, ty, val, lb));
+      args.push_back(mlir::arith::SubIOp::create(builder, loc, ty, val, lb));
     }
     mlir::Value base = fir::getBase(array);
 
@@ -1602,7 +1603,7 @@ class ScalarExprLowering {
 
       mlir::Value cnvrt = Fortran::lower::addCrayPointerInst(
           loc, builder, ptrVal, ptrTy, base.getType());
-      base = builder.create<fir::LoadOp>(loc, cnvrt);
+      base = fir::LoadOp::create(builder, loc, cnvrt);
     }
 
     mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(base.getType());
@@ -1611,7 +1612,7 @@ class ScalarExprLowering {
     auto seqTy = mlir::cast<fir::SequenceType>(eleTy);
     assert(args.size() == seqTy.getDimension());
     mlir::Type ty = builder.getRefType(seqTy.getEleTy());
-    auto addr = builder.create<fir::CoordinateOp>(loc, ty, base, args);
+    auto addr = fir::CoordinateOp::create(builder, loc, ty, base, args);
     return fir::factory::arrayElementToExtendedValue(builder, loc, array, addr);
   }
 
@@ -1648,12 +1649,12 @@ class ScalarExprLowering {
         mlir::Value val =
             builder.createConvert(loc, idxTy, fir::getBase(subVal));
         mlir::Value lb = builder.createConvert(loc, idxTy, getLB(arr, dim));
-        mlir::Value 
diff  = builder.create<mlir::arith::SubIOp>(loc, val, lb);
+        mlir::Value 
diff  = mlir::arith::SubIOp::create(builder, loc, val, lb);
         mlir::Value prod =
-            builder.create<mlir::arith::MulIOp>(loc, delta, 
diff );
-        total = builder.create<mlir::arith::AddIOp>(loc, prod, total);
+            mlir::arith::MulIOp::create(builder, loc, delta, 
diff );
+        total = mlir::arith::AddIOp::create(builder, loc, prod, total);
         if (ext)
-          delta = builder.create<mlir::arith::MulIOp>(loc, delta, ext);
+          delta = mlir::arith::MulIOp::create(builder, loc, delta, ext);
         ++dim;
       }
       mlir::Type origRefTy = refTy;
@@ -1672,8 +1673,8 @@ class ScalarExprLowering {
           base = builder.createConvert(loc, seqRefTy, base);
         }
       }
-      auto coor = builder.create<fir::CoordinateOp>(
-          loc, refTy, base, llvm::ArrayRef<mlir::Value>{total});
+      auto coor = fir::CoordinateOp::create(builder, loc, refTy, base,
+                                            llvm::ArrayRef<mlir::Value>{total});
       // Convert to expected, original type after address arithmetic.
       return builder.createConvert(loc, origRefTy, coor);
     };
@@ -1725,9 +1726,9 @@ class ScalarExprLowering {
           builder.createConvert(loc, idxTy, fir::getBase(subVal)));
     }
     mlir::Value shape = builder.createShape(loc, exv);
-    mlir::Value elementAddr = builder.create<fir::ArrayCoorOp>(
-        loc, refTy, addr, shape, /*slice=*/mlir::Value{}, arrayCoorArgs,
-        fir::getTypeParams(exv));
+    mlir::Value elementAddr = fir::ArrayCoorOp::create(
+        builder, loc, refTy, addr, shape, /*slice=*/mlir::Value{},
+        arrayCoorArgs, fir::getTypeParams(exv));
     return fir::factory::arrayElementToExtendedValue(builder, loc, exv,
                                                      elementAddr);
   }
@@ -1826,8 +1827,8 @@ class ScalarExprLowering {
     if (mlir::isa<mlir::FunctionType>(exvTy)) {
       auto boxProcTy =
           builder.getBoxProcType(mlir::cast<mlir::FunctionType>(exvTy));
-      return builder.create<fir::EmboxProcOp>(loc, boxProcTy,
-                                              fir::getBase(exv));
+      return fir::EmboxProcOp::create(builder, loc, boxProcTy,
+                                      fir::getBase(exv));
     }
     mlir::Value box = builder.createBox(loc, exv, exv.isPolymorphic());
     if (Fortran::lower::isParentComponent(expr)) {
@@ -2073,8 +2074,8 @@ class ScalarExprLowering {
       TODO(loc, "creating temporary for derived type with length parameters");
     }
 
-    mlir::Value temp = builder.create<fir::AllocMemOp>(
-        loc, type, tempName, allocMemTypeParams, extents);
+    mlir::Value temp = fir::AllocMemOp::create(builder, loc, type, tempName,
+                                               allocMemTypeParams, extents);
     if (mlir::isa<fir::CharacterType>(fir::unwrapSequenceType(type)))
       return fir::CharArrayBoxValue{temp, charLen, extents};
     return fir::ArrayBoxValue{temp, extents};
@@ -2124,9 +2125,9 @@ class ScalarExprLowering {
           mlir::Type type = v.getType();
           mlir::Value value = v;
           if (fir::isa_ref_type(type))
-            value = builder.create<fir::LoadOp>(loc, value);
+            value = fir::LoadOp::create(builder, loc, value);
           mlir::Value temp = builder.createTemporary(loc, value.getType());
-          builder.create<fir::StoreOp>(loc, value, temp);
+          fir::StoreOp::create(builder, loc, value, temp);
           return temp;
         },
         [&](const fir::BoxValue &x) -> ExtValue {
@@ -2141,9 +2142,9 @@ class ScalarExprLowering {
           // created always has the declared type.
           mlir::Value var =
               fir::getBase(fir::factory::readBoxValue(builder, loc, x));
-          auto value = builder.create<fir::LoadOp>(loc, var);
+          auto value = fir::LoadOp::create(builder, loc, var);
           mlir::Value temp = builder.createTemporary(loc, value.getType());
-          builder.create<fir::StoreOp>(loc, value, temp);
+          fir::StoreOp::create(builder, loc, value, temp);
           return temp;
         },
         [&](const fir::PolymorphicValue &p) -> ExtValue {
@@ -2242,7 +2243,7 @@ class ScalarExprLowering {
       // at this point.
       mlir::Value destBox = fir::getBase(builder.createBox(loc, temp));
       mlir::Value boxRef = builder.createTemporary(loc, destBox.getType());
-      builder.create<fir::StoreOp>(loc, destBox, boxRef);
+      fir::StoreOp::create(builder, loc, destBox, boxRef);
       fir::runtime::genAssignTemporary(builder, loc, boxRef,
                                        fir::getBase(actualArg));
       return temp;
@@ -2250,21 +2251,22 @@ class ScalarExprLowering {
 
     auto noCopy = [&]() {
       mlir::Value box = fir::getBase(actualArg);
-      mlir::Value boxAddr = builder.create<fir::BoxAddrOp>(loc, addrType, box);
-      builder.create<fir::ResultOp>(loc, boxAddr);
+      mlir::Value boxAddr = fir::BoxAddrOp::create(builder, loc, addrType, box);
+      fir::ResultOp::create(builder, loc, boxAddr);
     };
 
     auto combinedCondition = [&]() {
       if (isActualArgBox) {
         mlir::Value zero =
             builder.createIntegerConstant(loc, builder.getI1Type(), 0);
-        mlir::Value notContiguous = builder.create<mlir::arith::CmpIOp>(
-            loc, mlir::arith::CmpIPredicate::eq, isContiguousResult, zero);
+        mlir::Value notContiguous = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::eq, isContiguousResult,
+            zero);
         if (!restrictCopyAtRuntime) {
           restrictCopyAtRuntime = notContiguous;
         } else {
-          mlir::Value cond = builder.create<mlir::arith::AndIOp>(
-              loc, *restrictCopyAtRuntime, notContiguous);
+          mlir::Value cond = mlir::arith::AndIOp::create(
+              builder, loc, *restrictCopyAtRuntime, notContiguous);
           restrictCopyAtRuntime = cond;
         }
       }
@@ -2280,7 +2282,7 @@ class ScalarExprLowering {
                 .genThen([&]() { noCopy(); })
                 .genElse([&] {
                   ExtValue temp = doCopyIn();
-                  builder.create<fir::ResultOp>(loc, fir::getBase(temp));
+                  fir::ResultOp::create(builder, loc, fir::getBase(temp));
                 })
                 .getResults()[0];
         fir::ExtendedValue temp =
@@ -2312,19 +2314,19 @@ class ScalarExprLowering {
                         .genThen([&]() { noCopy(); })
                         .genElse([&]() {
                           ExtValue temp = doCopyIn();
-                          builder.create<fir::ResultOp>(loc,
-                                                        fir::getBase(temp));
+                          fir::ResultOp::create(builder, loc,
+                                                fir::getBase(temp));
                         })
                         .getResults()[0];
-                builder.create<fir::ResultOp>(loc, addr1);
+                fir::ResultOp::create(builder, loc, addr1);
               } else {
                 ExtValue temp = doCopyIn();
-                builder.create<fir::ResultOp>(loc, fir::getBase(temp));
+                fir::ResultOp::create(builder, loc, fir::getBase(temp));
               }
             })
             .genElse([&]() {
               mlir::Value nullPtr = builder.createNullConstant(loc, addrType);
-              builder.create<fir::ResultOp>(loc, nullPtr);
+              fir::ResultOp::create(builder, loc, nullPtr);
             })
             .getResults()[0];
     // Associate the temp address with actualArg lengths and extents if a
@@ -2357,7 +2359,7 @@ class ScalarExprLowering {
                                                                  tempBox);
         }
         // Deallocate the top-level entity of the temporary.
-        builder.create<fir::FreeMemOp>(loc, fir::getBase(copyOutPair.temp));
+        fir::FreeMemOp::create(builder, loc, fir::getBase(copyOutPair.temp));
         return;
       }
       // Generate CopyOutAssign() call to copy data from the temporary
@@ -2376,11 +2378,11 @@ class ScalarExprLowering {
       mlir::Type allocBoxTy =
           mlir::cast<fir::BaseBoxType>(srcBox.getType())
               .getBoxTypeWithNewAttr(fir::BaseBoxType::Attribute::Allocatable);
-      srcBox = builder.create<fir::ReboxOp>(loc, allocBoxTy, srcBox,
-                                            /*shift=*/mlir::Value{},
-                                            /*slice=*/mlir::Value{});
+      srcBox = fir::ReboxOp::create(builder, loc, allocBoxTy, srcBox,
+                                    /*shift=*/mlir::Value{},
+                                    /*slice=*/mlir::Value{});
       mlir::Value srcBoxRef = builder.createTemporary(loc, srcBox.getType());
-      builder.create<fir::StoreOp>(loc, srcBox, srcBoxRef);
+      fir::StoreOp::create(builder, loc, srcBox, srcBoxRef);
       // Create descriptor pointer to variable descriptor if copy out is needed,
       // and nullptr otherwise.
       mlir::Value destBoxRef;
@@ -2388,9 +2390,9 @@ class ScalarExprLowering {
         mlir::Value destBox =
             fir::getBase(builder.createBox(loc, copyOutPair.var));
         destBoxRef = builder.createTemporary(loc, destBox.getType());
-        builder.create<fir::StoreOp>(loc, destBox, destBoxRef);
+        fir::StoreOp::create(builder, loc, destBox, destBoxRef);
       } else {
-        destBoxRef = builder.create<fir::ZeroOp>(loc, srcBoxRef.getType());
+        destBoxRef = fir::ZeroOp::create(builder, loc, srcBoxRef.getType());
       }
       fir::runtime::genCopyOutAssign(builder, loc, destBoxRef, srcBoxRef);
     };
@@ -2436,8 +2438,8 @@ class ScalarExprLowering {
     // fir.box is absent.
     ExtValue actualArg = gen(expr);
     mlir::Value actualArgBase = fir::getBase(actualArg);
-    mlir::Value isPresent = builder.create<fir::IsPresentOp>(
-        loc, builder.getI1Type(), actualArgBase);
+    mlir::Value isPresent = fir::IsPresentOp::create(
+        builder, loc, builder.getI1Type(), actualArgBase);
     if (!mlir::isa<fir::BoxType>(actualArgBase.getType()))
       return {actualArg, isPresent};
     ExtValue safeToReadBox =
@@ -2457,7 +2459,7 @@ class ScalarExprLowering {
     if (const fir::CharBoxValue *charBox = actualArg.getCharBox()) {
       mlir::Value len = charBox->getLen();
       mlir::Value zero = builder.createIntegerConstant(loc, len.getType(), 0);
-      len = builder.create<mlir::arith::SelectOp>(loc, isPresent, len, zero);
+      len = mlir::arith::SelectOp::create(builder, loc, isPresent, len, zero);
       mlir::Value temp =
           builder.createTemporary(loc, type, /*name=*/{},
                                   /*shape=*/{}, mlir::ValueRange{len},
@@ -2538,12 +2540,12 @@ class ScalarExprLowering {
                 .genThen([&]() {
                   fir::factory::genScalarAssignment(builder, loc, temp,
                                                     actualArg);
-                  builder.create<fir::ResultOp>(loc, fir::getBase(temp));
+                  fir::ResultOp::create(builder, loc, fir::getBase(temp));
                 })
                 .genElse([&]() {
                   mlir::Value absent =
-                      builder.create<fir::AbsentOp>(loc, tempAddrTy);
-                  builder.create<fir::ResultOp>(loc, absent);
+                      fir::AbsentOp::create(builder, loc, tempAddrTy);
+                  fir::ResultOp::create(builder, loc, absent);
                 })
                 .getResults()[0];
         return {fir::substBase(temp, selectAddr), isPresent};
@@ -2647,7 +2649,7 @@ class ScalarExprLowering {
           mlir::Value boxStorage = builder.createTemporary(loc, boxTy);
           mlir::Value nullBox = fir::factory::createUnallocatedBox(
               builder, loc, boxTy, /*nonDeferredParams=*/{});
-          builder.create<fir::StoreOp>(loc, nullBox, boxStorage);
+          fir::StoreOp::create(builder, loc, nullBox, boxStorage);
           caller.placeInput(arg, boxStorage);
           continue;
         }
@@ -2706,9 +2708,10 @@ class ScalarExprLowering {
                       mlir::cast<fir::BoxCharType>(funcTy.getResult(0));
                   mlir::Value ref = builder.createConvertWithVolatileCast(
                       loc, builder.getRefType(boxTy.getEleTy()), x.getAddr());
-                  auto len = builder.create<fir::UndefOp>(
-                      loc, builder.getCharacterLengthType());
-                  return builder.create<fir::EmboxCharOp>(loc, boxTy, ref, len);
+                  auto len = fir::UndefOp::create(
+                      builder, loc, builder.getCharacterLengthType());
+                  return fir::EmboxCharOp::create(builder, loc, boxTy, ref,
+                                                  len);
                 }
                 return helper.createEmbox(x);
               },
@@ -2758,10 +2761,10 @@ class ScalarExprLowering {
           mlir::Value box = builder.createBox(loc, argAddr);
           if (isPresentValue) {
             mlir::Value convertedBox = builder.createConvert(loc, argTy, box);
-            auto absent = builder.create<fir::AbsentOp>(loc, argTy);
-            caller.placeInput(arg,
-                              builder.create<mlir::arith::SelectOp>(
-                                  loc, *isPresentValue, convertedBox, absent));
+            auto absent = fir::AbsentOp::create(builder, loc, argTy);
+            caller.placeInput(
+                arg, mlir::arith::SelectOp::create(
+                         builder, loc, *isPresentValue, convertedBox, absent));
           } else {
             caller.placeInput(arg, builder.createBox(loc, argAddr));
           }
@@ -2782,7 +2785,7 @@ class ScalarExprLowering {
           mlir::Value isAllocated =
               fir::factory::genIsAllocatedOrAssociatedTest(builder, loc,
                                                            mutableBox);
-          auto absent = builder.create<fir::AbsentOp>(loc, argTy);
+          auto absent = fir::AbsentOp::create(builder, loc, argTy);
           /// For now, assume it is not OK to pass the allocatable/pointer
           /// descriptor to a non pointer/allocatable dummy. That is a strict
           /// interpretation of 18.3.6 point 4 that stipulates the descriptor
@@ -2801,14 +2804,15 @@ class ScalarExprLowering {
                 box);
           } else if (mlir::isa<fir::BoxType>(box.getType()) &&
                      fir::isPolymorphicType(argTy)) {
-            box = builder.create<fir::ReboxOp>(loc, argTy, box, mlir::Value{},
-                                               /*slice=*/mlir::Value{});
+            box = fir::ReboxOp::create(builder, loc, argTy, box, mlir::Value{},
+                                       /*slice=*/mlir::Value{});
           }
 
           // Need the box types to be exactly similar for the selectOp.
           mlir::Value convertedBox = builder.createConvert(loc, argTy, box);
-          caller.placeInput(arg, builder.create<mlir::arith::SelectOp>(
-                                     loc, isAllocated, convertedBox, absent));
+          caller.placeInput(
+              arg, mlir::arith::SelectOp::create(builder, loc, isAllocated,
+                                                 convertedBox, absent));
         } else {
           auto dynamicType = expr->GetType();
           mlir::Value box;
@@ -2830,12 +2834,12 @@ class ScalarExprLowering {
                     .genThen([&]() {
                       auto boxed = builder.createBox(
                           loc, genBoxArg(*expr), fir::isPolymorphicType(argTy));
-                      builder.create<fir::ResultOp>(loc, boxed);
+                      fir::ResultOp::create(builder, loc, boxed);
                     })
                     .genElse([&]() {
-                      auto absent =
-                          builder.create<fir::AbsentOp>(loc, argTy).getResult();
-                      builder.create<fir::ResultOp>(loc, absent);
+                      auto absent = fir::AbsentOp::create(builder, loc, argTy)
+                                        .getResult();
+                      fir::ResultOp::create(builder, loc, absent);
                     })
                     .getResults()[0];
           } else {
@@ -2867,19 +2871,19 @@ class ScalarExprLowering {
                                         loc, actualTy, box, mlir::Value{},
                                         /*slice=*/mlir::Value{})
                                     .getResult();
-                            builder.create<fir::ResultOp>(loc, rebox);
+                            fir::ResultOp::create(builder, loc, rebox);
                           })
                           .genElse([&]() {
                             auto absent =
-                                builder.create<fir::AbsentOp>(loc, actualTy)
+                                fir::AbsentOp::create(builder, loc, actualTy)
                                     .getResult();
-                            builder.create<fir::ResultOp>(loc, absent);
+                            fir::ResultOp::create(builder, loc, absent);
                           })
                           .getResults()[0];
               } else {
-                box = builder.create<fir::ReboxOp>(loc, actualTy, box,
-                                                   mlir::Value{},
-                                                   /*slice=*/mlir::Value{});
+                box = fir::ReboxOp::create(builder, loc, actualTy, box,
+                                           mlir::Value{},
+                                           /*slice=*/mlir::Value{});
               }
             } else if (Fortran::lower::isParentComponent(*expr)) {
               fir::ExtendedValue newExv =
@@ -3132,12 +3136,12 @@ static void genScalarUserDefinedAssignmentCall(fir::FirOpBuilder &builder,
     if (argBaseType != fir::unwrapRefType(from.getType())) {
       // With logicals, it is possible that from is i1 here.
       if (fir::isa_ref_type(from.getType()))
-        from = builder.create<fir::LoadOp>(loc, from);
+        from = fir::LoadOp::create(builder, loc, from);
       from = builder.createConvert(loc, argBaseType, from);
     }
     if (!fir::isa_ref_type(from.getType())) {
       mlir::Value temp = builder.createTemporary(loc, argBaseType);
-      builder.create<fir::StoreOp>(loc, from, temp);
+      fir::StoreOp::create(builder, loc, from, temp);
       from = temp;
     }
     return builder.createConvert(loc, argType, from);
@@ -3147,7 +3151,7 @@ static void genScalarUserDefinedAssignmentCall(fir::FirOpBuilder &builder,
   mlir::Type rhsType = func.getFunctionType().getInput(1);
   mlir::Value lhsArg = prepareUserDefinedArg(builder, loc, lhs, lhsType);
   mlir::Value rhsArg = prepareUserDefinedArg(builder, loc, rhs, rhsType);
-  builder.create<fir::CallOp>(loc, func, mlir::ValueRange{lhsArg, rhsArg});
+  fir::CallOp::create(builder, loc, func, mlir::ValueRange{lhsArg, rhsArg});
 }
 
 /// Convert the result of a fir.array_modify to an ExtendedValue given the
@@ -3187,17 +3191,17 @@ createDerivedArrayAmend(mlir::Location loc, fir::ArrayLoadOp destLoad,
     fir::factory::genRecordAssignment(builder, loc, destAcc, elementExv);
   } else {
     auto boxTy = fir::BoxType::get(eleTy);
-    auto toBox = builder.create<fir::EmboxOp>(loc, boxTy, destAcc.getResult(),
-                                              mlir::Value{}, mlir::Value{},
-                                              destLoad.getTypeparams());
-    auto fromBox = builder.create<fir::EmboxOp>(
-        loc, boxTy, fir::getBase(elementExv), mlir::Value{}, mlir::Value{},
-        destLoad.getTypeparams());
+    auto toBox = fir::EmboxOp::create(builder, loc, boxTy, destAcc.getResult(),
+                                      mlir::Value{}, mlir::Value{},
+                                      destLoad.getTypeparams());
+    auto fromBox = fir::EmboxOp::create(
+        builder, loc, boxTy, fir::getBase(elementExv), mlir::Value{},
+        mlir::Value{}, destLoad.getTypeparams());
     fir::factory::genRecordAssignment(builder, loc, fir::BoxValue(toBox),
                                       fir::BoxValue(fromBox));
   }
-  return builder.create<fir::ArrayAmendOp>(loc, innerArg.getType(), innerArg,
-                                           destAcc);
+  return fir::ArrayAmendOp::create(builder, loc, innerArg.getType(), innerArg,
+                                   destAcc);
 }
 
 inline static fir::ArrayAmendOp
@@ -3219,7 +3223,7 @@ createCharArrayAmend(mlir::Location loc, fir::FirOpBuilder &builder,
   helper.createAssign(fir::ExtendedValue{dstChar}, srcExv);
   // Mark this array element as amended.
   mlir::Type ty = innerArg.getType();
-  auto amend = builder.create<fir::ArrayAmendOp>(loc, ty, innerArg, dstOp);
+  auto amend = fir::ArrayAmendOp::create(builder, loc, ty, innerArg, dstOp);
   return amend;
 }
 
@@ -3235,7 +3239,7 @@ convertToArrayBoxValue(mlir::Location loc, fir::FirOpBuilder &builder,
   mlir::Type ty = fir::unwrapRefType(val.getType());
   mlir::IndexType idxTy = builder.getIndexType();
   auto seqTy = mlir::cast<fir::SequenceType>(ty);
-  auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+  auto undef = fir::UndefOp::create(builder, loc, idxTy);
   llvm::SmallVector<mlir::Value> extents(seqTy.getDimension(), undef);
   if (fir::isa_char(seqTy.getEleTy()))
     return fir::CharArrayBoxValue(val, len ? len : undef, extents);
@@ -3315,10 +3319,10 @@ class ArrayExprLowering {
     ExtValue exv = lowerArrayExpression(rhs);
     if (explicitSpaceIsActive()) {
       explicitSpace->finalizeContext();
-      builder.create<fir::ResultOp>(loc, fir::getBase(exv));
+      fir::ResultOp::create(builder, loc, fir::getBase(exv));
     } else {
-      builder.create<fir::ArrayMergeStoreOp>(
-          loc, destination, fir::getBase(exv), destination.getMemref(),
+      fir::ArrayMergeStoreOp::create(
+          builder, loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     }
   }
@@ -3432,8 +3436,8 @@ class ArrayExprLowering {
     assert(destination && "destination must have been set");
     ExtValue exv = lowerArrayExpression(rhsCC, destination.getType());
     if (!explicitSpaceIsActive())
-      builder.create<fir::ArrayMergeStoreOp>(
-          loc, destination, fir::getBase(exv), destination.getMemref(),
+      fir::ArrayMergeStoreOp::create(
+          builder, loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     // destShape may originally be null, if rhs did not define a shape.
     // In this case the destShape is computed from lhs, and we may have
@@ -3502,7 +3506,7 @@ class ArrayExprLowering {
                                          lengthParams, assignToStorage);
     if (explicitSpaceIsActive()) {
       explicitSpace->finalizeContext();
-      builder.create<fir::ResultOp>(loc, fir::getBase(realloc.newValue));
+      fir::ResultOp::create(builder, loc, fir::getBase(realloc.newValue));
     }
     fir::factory::finalizeRealloc(builder, loc, mutableBox, lbounds,
                                   takeLboundsIfRealloc, realloc);
@@ -3547,9 +3551,9 @@ class ArrayExprLowering {
     mlir::Value tempRes = dest.getMemref();
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     mlir::Location loc = converter.getCurrentLocation();
-    builder.create<fir::ArrayMergeStoreOp>(loc, dest, fir::getBase(loopRes),
-                                           tempRes, dest.getSlice(),
-                                           dest.getTypeparams());
+    fir::ArrayMergeStoreOp::create(builder, loc, dest, fir::getBase(loopRes),
+                                   tempRes, dest.getSlice(),
+                                   dest.getTypeparams());
 
     auto arrTy = mlir::cast<fir::SequenceType>(
         fir::dyn_cast_ptrEleTy(tempRes.getType()));
@@ -3595,26 +3599,26 @@ class ArrayExprLowering {
     // as there isn't any necessity for it.
     ccLoadDest = [=](llvm::ArrayRef<mlir::Value> shape) -> fir::ArrayLoadOp {
       mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-      auto var = builder.create<fir::CoordinateOp>(
-          loc, builder.getRefType(hdrTy.getType(1)), header, one);
-      auto load = builder.create<fir::LoadOp>(loc, var);
+      auto var = fir::CoordinateOp::create(
+          builder, loc, builder.getRefType(hdrTy.getType(1)), header, one);
+      auto load = fir::LoadOp::create(builder, loc, var);
       mlir::Type eleTy =
           fir::unwrapSequenceType(fir::unwrapRefType(load.getType()));
       auto seqTy = fir::SequenceType::get(eleTy, shape.size());
       mlir::Value castTo =
           builder.createConvert(loc, fir::HeapType::get(seqTy), load);
       mlir::Value shapeOp = builder.genShape(loc, shape);
-      return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
-                                              /*slice=*/mlir::Value{},
-                                              mlir::ValueRange{});
+      return fir::ArrayLoadOp::create(builder, loc, seqTy, castTo, shapeOp,
+                                      /*slice=*/mlir::Value{},
+                                      mlir::ValueRange{});
     };
     // Custom lowering of the element store to deal with the extra indirection
     // to the lazy allocated buffer.
     ccStoreToDest = [=](IterSpace iters) {
       mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-      auto var = builder.create<fir::CoordinateOp>(
-          loc, builder.getRefType(hdrTy.getType(1)), header, one);
-      auto load = builder.create<fir::LoadOp>(loc, var);
+      auto var = fir::CoordinateOp::create(
+          builder, loc, builder.getRefType(hdrTy.getType(1)), header, one);
+      auto load = fir::LoadOp::create(builder, loc, var);
       mlir::Type eleTy =
           fir::unwrapSequenceType(fir::unwrapRefType(load.getType()));
       auto seqTy = fir::SequenceType::get(eleTy, iters.iterVec().size());
@@ -3623,12 +3627,12 @@ class ArrayExprLowering {
       mlir::Value shape = builder.genShape(loc, genIterationShape());
       llvm::SmallVector<mlir::Value> indices = fir::factory::originateIndices(
           loc, builder, castTo.getType(), shape, iters.iterVec());
-      auto eleAddr = builder.create<fir::ArrayCoorOp>(
-          loc, builder.getRefType(eleTy), castTo, shape,
+      auto eleAddr = fir::ArrayCoorOp::create(
+          builder, loc, builder.getRefType(eleTy), castTo, shape,
           /*slice=*/mlir::Value{}, indices, destination.getTypeparams());
       mlir::Value eleVal =
           builder.createConvert(loc, eleTy, iters.getElement());
-      builder.create<fir::StoreOp>(loc, eleVal, eleAddr);
+      fir::StoreOp::create(builder, loc, eleVal, eleAddr);
       return iters.innerArgument();
     };
 
@@ -3684,10 +3688,10 @@ class ArrayExprLowering {
     auto exv = lowerArrayExpression(rhs);
     if (explicitSpaceIsActive()) {
       explicitSpace->finalizeContext();
-      builder.create<fir::ResultOp>(loc, fir::getBase(exv));
+      fir::ResultOp::create(builder, loc, fir::getBase(exv));
     } else {
-      builder.create<fir::ArrayMergeStoreOp>(
-          loc, destination, fir::getBase(exv), destination.getMemref(),
+      fir::ArrayMergeStoreOp::create(
+          builder, loc, destination, fir::getBase(exv), destination.getMemref(),
           destination.getSlice(), destination.getTypeparams());
     }
   }
@@ -3766,7 +3770,7 @@ class ArrayExprLowering {
       std::size_t offset = explicitSpace->argPosition(oldInnerArg);
       explicitSpace->setInnerArg(offset, fir::getBase(lexv));
       finalizeElementCtx();
-      builder.create<fir::ResultOp>(loc, fir::getBase(lexv));
+      fir::ResultOp::create(builder, loc, fir::getBase(lexv));
     };
     if (mlir::Operation *defOp = fir::getBase(lexv).getDefiningOp()) {
       llvm::TypeSwitch<mlir::Operation *>(defOp)
@@ -3836,7 +3840,7 @@ class ArrayExprLowering {
     // 5). Thread the array value updated forward.
     if (!isIllFormedLHS) {
       finalizeElementCtx();
-      builder.create<fir::ResultOp>(getLoc(), fir::getBase(lexv));
+      fir::ResultOp::create(builder, getLoc(), fir::getBase(lexv));
     }
     return lexv;
   }
@@ -3979,7 +3983,7 @@ class ArrayExprLowering {
     if (auto origEleTy = fir::dyn_cast_ptrEleTy(origVal.getType()))
       if (mlir::isa<fir::BaseBoxType>(origEleTy)) {
         // If origVal is a box variable, load it so it is in the value domain.
-        origVal = builder.create<fir::LoadOp>(loc, origVal);
+        origVal = fir::LoadOp::create(builder, loc, origVal);
       }
     if (mlir::isa<fir::BoxType>(origVal.getType()) &&
         !mlir::isa<fir::BoxType>(eleTy)) {
@@ -3998,7 +4002,7 @@ class ArrayExprLowering {
         TODO(loc, "TARGET of pointer assignment with runtime size/shape");
       auto memrefTy = fir::boxMemRefType(mlir::cast<fir::BoxType>(eleTy));
       auto castTo = builder.createConvert(loc, memrefTy, origVal);
-      origVal = builder.create<fir::EmboxOp>(loc, eleTy, castTo);
+      origVal = fir::EmboxOp::create(builder, loc, eleTy, castTo);
     }
     mlir::Value val = builder.convertWithSemantics(loc, eleTy, origVal);
     if (isBoundsSpec()) {
@@ -4007,9 +4011,9 @@ class ArrayExprLowering {
       if (lbs.size() > 0) {
         // Rebox the value with user-specified shift.
         auto shiftTy = fir::ShiftType::get(eleTy.getContext(), lbs.size());
-        mlir::Value shiftOp = builder.create<fir::ShiftOp>(loc, shiftTy, lbs);
-        val = builder.create<fir::ReboxOp>(loc, eleTy, val, shiftOp,
-                                           mlir::Value{});
+        mlir::Value shiftOp = fir::ShiftOp::create(builder, loc, shiftTy, lbs);
+        val = fir::ReboxOp::create(builder, loc, eleTy, val, shiftOp,
+                                   mlir::Value{});
       }
     } else if (isBoundsRemap()) {
       assert(lbounds.has_value());
@@ -4020,9 +4024,9 @@ class ArrayExprLowering {
         auto shapeShiftArgs = flatZip(lbs, *ubounds);
         auto shapeTy = fir::ShapeShiftType::get(eleTy.getContext(), lbs.size());
         mlir::Value shapeShift =
-            builder.create<fir::ShapeShiftOp>(loc, shapeTy, shapeShiftArgs);
-        val = builder.create<fir::ReboxOp>(loc, eleTy, val, shapeShift,
-                                           mlir::Value{});
+            fir::ShapeShiftOp::create(builder, loc, shapeTy, shapeShiftArgs);
+        val = fir::ReboxOp::create(builder, loc, eleTy, val, shapeShift,
+                                   mlir::Value{});
       }
     }
     return val;
@@ -4045,8 +4049,8 @@ class ArrayExprLowering {
         // memory into the destination array.
         mlir::Type resRefTy = builder.getRefType(eleTy);
         // Get a reference to the array element to be amended.
-        auto arrayOp = builder.create<fir::ArrayAccessOp>(
-            loc, resRefTy, innerArg, iterSpace.iterVec(),
+        auto arrayOp = fir::ArrayAccessOp::create(
+            builder, loc, resRefTy, innerArg, iterSpace.iterVec(),
             fir::factory::getTypeParams(loc, builder, destination));
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
           llvm::SmallVector<mlir::Value> substringBounds;
@@ -4067,9 +4071,9 @@ class ArrayExprLowering {
       }
       // By value semantics. The element is being assigned by value.
       auto ele = convertElementForUpdate(loc, eleTy, fir::getBase(exv));
-      auto update = builder.create<fir::ArrayUpdateOp>(
-          loc, arrTy, innerArg, ele, iterSpace.iterVec(),
-          destination.getTypeparams());
+      auto update = fir::ArrayUpdateOp::create(builder, loc, arrTy, innerArg,
+                                               ele, iterSpace.iterVec(),
+                                               destination.getTypeparams());
       return abstractArrayExtValue(update);
     };
   }
@@ -4094,7 +4098,7 @@ class ArrayExprLowering {
                       : defaultStoreToDestination(/*substring=*/nullptr);
     mlir::Value updVal = fir::getBase(lambda(iterSpace));
     finalizeElementCtx();
-    builder.create<fir::ResultOp>(loc, updVal);
+    fir::ResultOp::create(builder, loc, updVal);
     builder.restoreInsertionPoint(insPt);
     return abstractArrayExtValue(iterSpace.outerResult());
   }
@@ -4247,31 +4251,34 @@ class ArrayExprLowering {
       // Compute the dynamic position into the header.
       llvm::SmallVector<mlir::Value> offsets;
       for (auto doLoop : loopStack[i]) {
-        auto m = builder.create<mlir::arith::SubIOp>(
-            loc, doLoop.getInductionVar(), doLoop.getLowerBound());
-        auto n = builder.create<mlir::arith::DivSIOp>(loc, m, doLoop.getStep());
+        auto m = mlir::arith::SubIOp::create(
+            builder, loc, doLoop.getInductionVar(), doLoop.getLowerBound());
+        auto n =
+            mlir::arith::DivSIOp::create(builder, loc, m, doLoop.getStep());
         mlir::Value one = builder.createIntegerConstant(loc, n.getType(), 1);
-        offsets.push_back(builder.create<mlir::arith::AddIOp>(loc, n, one));
+        offsets.push_back(mlir::arith::AddIOp::create(builder, loc, n, one));
       }
       mlir::IntegerType i32Ty = builder.getIntegerType(32);
       mlir::Value uno = builder.createIntegerConstant(loc, i32Ty, 1);
       mlir::Type coorTy = builder.getRefType(raggedTy.getType(1));
-      auto hdOff = builder.create<fir::CoordinateOp>(loc, coorTy, header, uno);
+      auto hdOff = fir::CoordinateOp::create(builder, loc, coorTy, header, uno);
       auto toTy = fir::SequenceType::get(raggedTy, offsets.size());
       mlir::Type toRefTy = builder.getRefType(toTy);
-      auto ldHdr = builder.create<fir::LoadOp>(loc, hdOff);
+      auto ldHdr = fir::LoadOp::create(builder, loc, hdOff);
       mlir::Value hdArr = builder.createConvert(loc, toRefTy, ldHdr);
       auto shapeOp = builder.genShape(loc, extents);
-      header = builder.create<fir::ArrayCoorOp>(
-          loc, builder.getRefType(raggedTy), hdArr, shapeOp,
+      header = fir::ArrayCoorOp::create(
+          builder, loc, builder.getRefType(raggedTy), hdArr, shapeOp,
           /*slice=*/mlir::Value{}, offsets,
           /*typeparams=*/mlir::ValueRange{});
-      auto hdrVar = builder.create<fir::CoordinateOp>(loc, coorTy, header, uno);
-      auto inVar = builder.create<fir::LoadOp>(loc, hdrVar);
+      auto hdrVar =
+          fir::CoordinateOp::create(builder, loc, coorTy, header, uno);
+      auto inVar = fir::LoadOp::create(builder, loc, hdrVar);
       mlir::Value two = builder.createIntegerConstant(loc, i32Ty, 2);
       mlir::Type coorTy2 = builder.getRefType(raggedTy.getType(2));
-      auto hdrSh = builder.create<fir::CoordinateOp>(loc, coorTy2, header, two);
-      auto shapePtr = builder.create<fir::LoadOp>(loc, hdrSh);
+      auto hdrSh =
+          fir::CoordinateOp::create(builder, loc, coorTy2, header, two);
+      auto shapePtr = fir::LoadOp::create(builder, loc, hdrSh);
       // Replace the binding.
       implicitSpace->rebind(expr, genMaskAccess(inVar, shapePtr));
       if (i < depth - 1)
@@ -4301,7 +4308,7 @@ class ArrayExprLowering {
             Fortran::lower::createLazyArrayTempValue(converter, *e, header,
                                                      symMap, stmtCtx);
             // Close the explicit loops.
-            builder.create<fir::ResultOp>(loc, explicitSpace->getInnerArgs());
+            fir::ResultOp::create(builder, loc, explicitSpace->getInnerArgs());
             builder.setInsertionPointAfter(explicitSpace->getOuterLoop());
             // Open a new copy of the explicit loop nest.
             explicitSpace->genLoopNest();
@@ -4327,9 +4334,10 @@ class ArrayExprLowering {
               fir::factory::getRaggedArrayHeaderType(builder);
           mlir::IntegerType i32Ty = builder.getIntegerType(32);
           mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-          auto coor1 = builder.create<fir::CoordinateOp>(
-              loc, builder.getRefType(raggedTy.getType(1)), header, one);
-          auto db = builder.create<fir::LoadOp>(loc, coor1);
+          auto coor1 = fir::CoordinateOp::create(
+              builder, loc, builder.getRefType(raggedTy.getType(1)), header,
+              one);
+          auto db = fir::LoadOp::create(builder, loc, coor1);
           mlir::Type eleTy =
               fir::unwrapSequenceType(fir::unwrapRefType(db.getType()));
           mlir::Type buffTy =
@@ -4338,17 +4346,18 @@ class ArrayExprLowering {
           mlir::Value buff = builder.createConvert(loc, buffTy, db);
 
           mlir::Value two = builder.createIntegerConstant(loc, i32Ty, 2);
-          auto coor2 = builder.create<fir::CoordinateOp>(
-              loc, builder.getRefType(raggedTy.getType(2)), header, two);
-          auto shBuff = builder.create<fir::LoadOp>(loc, coor2);
+          auto coor2 = fir::CoordinateOp::create(
+              builder, loc, builder.getRefType(raggedTy.getType(2)), header,
+              two);
+          auto shBuff = fir::LoadOp::create(builder, loc, coor2);
           mlir::IntegerType i64Ty = builder.getIntegerType(64);
           mlir::IndexType idxTy = builder.getIndexType();
           llvm::SmallVector<mlir::Value> extents;
           for (std::remove_const_t<decltype(rank)> i = 0; i < rank; ++i) {
             mlir::Value off = builder.createIntegerConstant(loc, i32Ty, i);
-            auto coor = builder.create<fir::CoordinateOp>(
-                loc, builder.getRefType(i64Ty), shBuff, off);
-            auto ldExt = builder.create<fir::LoadOp>(loc, coor);
+            auto coor = fir::CoordinateOp::create(
+                builder, loc, builder.getRefType(i64Ty), shBuff, off);
+            auto ldExt = fir::LoadOp::create(builder, loc, coor);
             extents.push_back(builder.createConvert(loc, idxTy, ldExt));
           }
           if (destShape.empty())
@@ -4376,7 +4385,7 @@ class ArrayExprLowering {
     // run from 0 to `extent - 1` inclusive.
     for (auto extent : shape)
       loopUppers.push_back(
-          builder.create<mlir::arith::SubIOp>(loc, extent, one));
+          mlir::arith::SubIOp::create(builder, loc, extent, one));
 
     // Iteration space is created with outermost columns, innermost rows
     llvm::SmallVector<fir::DoLoopOp> loops;
@@ -4391,16 +4400,16 @@ class ArrayExprLowering {
       }
       fir::DoLoopOp loop;
       if (innerArg) {
-        loop = builder.create<fir::DoLoopOp>(
-            loc, zero, i.value(), one, isUnordered(),
+        loop = fir::DoLoopOp::create(
+            builder, loc, zero, i.value(), one, isUnordered(),
             /*finalCount=*/false, mlir::ValueRange{innerArg});
         innerArg = loop.getRegionIterArgs().front();
         if (explicitSpaceIsActive())
           explicitSpace->setInnerArg(0, innerArg);
       } else {
-        loop = builder.create<fir::DoLoopOp>(loc, zero, i.value(), one,
-                                             isUnordered(),
-                                             /*finalCount=*/false);
+        loop = fir::DoLoopOp::create(builder, loc, zero, i.value(), one,
+                                     isUnordered(),
+                                     /*finalCount=*/false);
       }
       ivars.push_back(loop.getInductionVar());
       loops.push_back(loop);
@@ -4410,7 +4419,7 @@ class ArrayExprLowering {
       for (std::remove_const_t<decltype(loopDepth)> i = 0; i + 1 < loopDepth;
            ++i) {
         builder.setInsertionPointToEnd(loops[i].getBody());
-        builder.create<fir::ResultOp>(loc, loops[i + 1].getResult(0));
+        fir::ResultOp::create(builder, loc, loops[i + 1].getResult(0));
       }
 
     // Move insertion point to the start of the innermost loop in the nest.
@@ -4468,21 +4477,23 @@ class ArrayExprLowering {
            implicitSpace->getMasks()) {
         const std::size_t size = maskExprs.size() - 1;
         auto genFalseBlock = [&](const auto *e, auto &&cond) {
-          auto ifOp = builder.create<fir::IfOp>(
-              loc, mlir::TypeRange{innerArg.getType()}, fir::getBase(cond),
-              /*withElseRegion=*/true);
-          builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
+          auto ifOp = fir::IfOp::create(builder, loc,
+                                        mlir::TypeRange{innerArg.getType()},
+                                        fir::getBase(cond),
+                                        /*withElseRegion=*/true);
+          fir::ResultOp::create(builder, loc, ifOp.getResult(0));
           builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-          builder.create<fir::ResultOp>(loc, innerArg);
+          fir::ResultOp::create(builder, loc, innerArg);
           builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
         };
         auto genTrueBlock = [&](const auto *e, auto &&cond) {
-          auto ifOp = builder.create<fir::IfOp>(
-              loc, mlir::TypeRange{innerArg.getType()}, fir::getBase(cond),
-              /*withElseRegion=*/true);
-          builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
+          auto ifOp = fir::IfOp::create(builder, loc,
+                                        mlir::TypeRange{innerArg.getType()},
+                                        fir::getBase(cond),
+                                        /*withElseRegion=*/true);
+          fir::ResultOp::create(builder, loc, ifOp.getResult(0));
           builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-          builder.create<fir::ResultOp>(loc, innerArg);
+          fir::ResultOp::create(builder, loc, innerArg);
           builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         };
         for (std::remove_const_t<decltype(size)> i = 0; i < size; ++i)
@@ -4525,16 +4536,16 @@ class ArrayExprLowering {
                 "allocatable members");
     mlir::Value temp =
         !seqTy.hasDynamicExtents()
-            ? builder.create<fir::AllocMemOp>(loc, type)
-            : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
-                                              mlir::ValueRange{}, shape);
+            ? fir::AllocMemOp::create(builder, loc, type)
+            : fir::AllocMemOp::create(builder, loc, type, ".array.expr",
+                                      mlir::ValueRange{}, shape);
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
     stmtCtx.attachCleanup(
         [bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
     mlir::Value shapeOp = genShapeOp(shape);
-    return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
-                                            /*slice=*/mlir::Value{},
-                                            mlir::ValueRange{});
+    return fir::ArrayLoadOp::create(builder, loc, seqTy, temp, shapeOp,
+                                    /*slice=*/mlir::Value{},
+                                    mlir::ValueRange{});
   }
 
   static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -4543,7 +4554,7 @@ class ArrayExprLowering {
     llvm::SmallVector<mlir::Value> idxShape;
     for (auto s : shape)
       idxShape.push_back(builder.createConvert(loc, idxTy, s));
-    return builder.create<fir::ShapeOp>(loc, idxShape);
+    return fir::ShapeOp::create(builder, loc, idxShape);
   }
 
   fir::ShapeOp genShapeOp(llvm::ArrayRef<mlir::Value> shape) {
@@ -4791,7 +4802,7 @@ class ArrayExprLowering {
       mlir::Type argTy = callSiteType.getInput(arg.firArgument);
       if (!actual) {
         // Optional dummy argument for which there is no actual argument.
-        auto absent = builder.create<fir::AbsentOp>(loc, argTy);
+        auto absent = fir::AbsentOp::create(builder, loc, argTy);
         operands.emplace_back([=](IterSpace) { return absent; });
         continue;
       }
@@ -4824,7 +4835,7 @@ class ArrayExprLowering {
               builder.createTemporary(loc, val.getType(),
                                       llvm::ArrayRef<mlir::NamedAttribute>{
                                           fir::getAdaptToByRefAttr(builder)});
-          builder.create<fir::StoreOp>(loc, val, temp);
+          fir::StoreOp::create(builder, loc, val, temp);
           operands.emplace_back(
               [=](IterSpace iters) -> ExtValue { return temp; });
         }
@@ -4883,14 +4894,14 @@ class ArrayExprLowering {
                 fir::dyn_cast_ptrOrBoxEleTy(fir::getBase(exv).getType());
             mlir::Type innerTy = fir::unwrapSequenceType(baseTy);
             operands.emplace_back([=](IterSpace iters) -> ExtValue {
-              mlir::Value coord = builder.create<fir::CoordinateOp>(
-                  loc, fir::ReferenceType::get(innerTy), fir::getBase(exv),
-                  iters.iterVec());
+              mlir::Value coord = fir::CoordinateOp::create(
+                  builder, loc, fir::ReferenceType::get(innerTy),
+                  fir::getBase(exv), iters.iterVec());
               mlir::Value empty;
               mlir::ValueRange emptyRange;
-              return builder.create<fir::EmboxOp>(
-                  loc, fir::ClassType::get(innerTy), coord, empty, empty,
-                  emptyRange, sourceBox);
+              return fir::EmboxOp::create(builder, loc,
+                                          fir::ClassType::get(innerTy), coord,
+                                          empty, empty, emptyRange, sourceBox);
             });
           } else {
             ExtValue exv = asScalarRef(*expr);
@@ -4903,9 +4914,9 @@ class ArrayExprLowering {
               operands.emplace_back([=](IterSpace iters) -> ExtValue {
                 mlir::Value empty;
                 mlir::ValueRange emptyRange;
-                return builder.create<fir::EmboxOp>(
-                    loc, fir::ClassType::get(baseTy), fir::getBase(exv), empty,
-                    empty, emptyRange);
+                return fir::EmboxOp::create(
+                    builder, loc, fir::ClassType::get(baseTy),
+                    fir::getBase(exv), empty, empty, emptyRange);
               });
             }
           }
@@ -5097,8 +5108,8 @@ class ArrayExprLowering {
     return exv.match(
         [&](const fir::CharBoxValue &cb) -> ExtValue {
           mlir::Value len = cb.getLen();
-          auto mem =
-              builder.create<fir::AllocaOp>(loc, toType, mlir::ValueRange{len});
+          auto mem = fir::AllocaOp::create(builder, loc, toType,
+                                           mlir::ValueRange{len});
           fir::CharBoxValue result(mem, len);
           fir::factory::CharacterExprHelper{builder, loc}.createAssign(
               ExtValue{result}, exv);
@@ -5153,7 +5164,7 @@ class ArrayExprLowering {
       auto val = f(iters);
       mlir::Value base = fir::getBase(val);
       auto newBase =
-          builder.create<fir::NoReassocOp>(loc, base.getType(), base);
+          fir::NoReassocOp::create(builder, loc, base.getType(), base);
       return fir::substBase(val, newBase);
     };
   }
@@ -5170,10 +5181,10 @@ class ArrayExprLowering {
       if constexpr (CAT == Fortran::common::TypeCategory::Unsigned) {
         mlir::Value signless = builder.createConvert(loc, ty, val);
         mlir::Value neg =
-            builder.create<mlir::arith::SubIOp>(loc, zero, signless);
+            mlir::arith::SubIOp::create(builder, loc, zero, signless);
         return builder.createConvert(loc, val.getType(), neg);
       }
-      return builder.create<mlir::arith::SubIOp>(loc, zero, val);
+      return mlir::arith::SubIOp::create(builder, loc, zero, val);
     };
   }
   template <int KIND>
@@ -5192,7 +5203,7 @@ class ArrayExprLowering {
     mlir::Location loc = getLoc();
     auto f = genarr(x.left());
     return [=](IterSpace iters) -> ExtValue {
-      return builder.create<mlir::arith::NegFOp>(loc, fir::getBase(f(iters)));
+      return mlir::arith::NegFOp::create(builder, loc, fir::getBase(f(iters)));
     };
   }
   template <int KIND>
@@ -5201,7 +5212,7 @@ class ArrayExprLowering {
     mlir::Location loc = getLoc();
     auto f = genarr(x.left());
     return [=](IterSpace iters) -> ExtValue {
-      return builder.create<fir::NegcOp>(loc, fir::getBase(f(iters)));
+      return fir::NegcOp::create(builder, loc, fir::getBase(f(iters)));
     };
   }
 
@@ -5426,8 +5437,8 @@ class ArrayExprLowering {
     mlir::Location loc = getLoc();
     mlir::Value lb = getLBound(x, dim, one);
     mlir::Value extent = fir::factory::readExtent(builder, loc, x, dim);
-    auto add = builder.create<mlir::arith::AddIOp>(loc, lb, extent);
-    return builder.create<mlir::arith::SubIOp>(loc, add, one);
+    auto add = mlir::arith::AddIOp::create(builder, loc, lb, extent);
+    return mlir::arith::SubIOp::create(builder, loc, add, one);
   }
 
   /// Return the extent of the boxed array `x` in dimesion `dim`.
@@ -5469,11 +5480,11 @@ class ArrayExprLowering {
     if (destShape[0] != savedDestShape[dim]) {
       // Not the same, so choose the smaller value.
       mlir::Location loc = getLoc();
-      auto cmp = builder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::sgt, destShape[0],
-          savedDestShape[dim]);
-      auto sel = builder.create<mlir::arith::SelectOp>(
-          loc, cmp, savedDestShape[dim], destShape[0]);
+      auto cmp = mlir::arith::CmpIOp::create(builder, loc,
+                                             mlir::arith::CmpIPredicate::sgt,
+                                             destShape[0], savedDestShape[dim]);
+      auto sel = mlir::arith::SelectOp::create(
+          builder, loc, cmp, savedDestShape[dim], destShape[0]);
       savedDestShape[dim] = sel;
       destShape = savedDestShape;
     }
@@ -5549,12 +5560,12 @@ class ArrayExprLowering {
                     // FIXME: must use the lower bound of this component.
                     auto arrLowerBound =
                         atBase ? getLBound(arrayExv, subsIndex, one) : one;
-                    auto initial = builder.create<mlir::arith::SubIOp>(
-                        loc, lowerBound, arrLowerBound);
-                    auto prod = builder.create<mlir::arith::MulIOp>(
-                        loc, impliedIter, stride);
-                    auto result =
-                        builder.create<mlir::arith::AddIOp>(loc, initial, prod);
+                    auto initial = mlir::arith::SubIOp::create(
+                        builder, loc, lowerBound, arrLowerBound);
+                    auto prod = mlir::arith::MulIOp::create(
+                        builder, loc, impliedIter, stride);
+                    auto result = mlir::arith::AddIOp::create(builder, loc,
+                                                              initial, prod);
                     newIters.setIndexValue(subsIndex, result);
                     return newIters;
                   };
@@ -5588,15 +5599,15 @@ class ArrayExprLowering {
                     // using the base array's lower bound value.
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
-                    auto origin = builder.create<mlir::arith::SubIOp>(
-                        loc, idxTy, val, lb);
+                    auto origin = mlir::arith::SubIOp::create(builder, loc,
+                                                              idxTy, val, lb);
                     newIters.setIndexValue(subsIndex, origin);
                     return newIters;
                   };
                   if (useTripsForSlice) {
                     LLVM_ATTRIBUTE_UNUSED auto vectorSubscriptShape =
                         getShape(arrayOperands.back());
-                    auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+                    auto undef = fir::UndefOp::create(builder, loc, idxTy);
                     trips.push_back(undef);
                     trips.push_back(undef);
                     trips.push_back(undef);
@@ -5614,7 +5625,7 @@ class ArrayExprLowering {
                     // the array's declared rank.
                     mlir::Value v = fir::getBase(asScalarArray(e));
                     trips.push_back(v);
-                    auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+                    auto undef = fir::UndefOp::create(builder, loc, idxTy);
                     trips.push_back(undef);
                     trips.push_back(undef);
                     auto currentPC = pc;
@@ -5623,8 +5634,8 @@ class ArrayExprLowering {
                     // Normalize `e` by subtracting the declared lbound.
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
-                    mlir::Value ivAdj =
-                        builder.create<mlir::arith::SubIOp>(loc, idxTy, iv, lb);
+                    mlir::Value ivAdj = mlir::arith::SubIOp::create(
+                        builder, loc, idxTy, iv, lb);
                     // Add lbound adjusted value of `e` to the iteration vector
                     // (except when creating a box because the iteration vector
                     // is empty).
@@ -5641,8 +5652,8 @@ class ArrayExprLowering {
                         builder.createConvert(loc, idxTy, newValue);
                     mlir::Value lb = fir::factory::readLowerBound(
                         builder, loc, arrayExv, subsIndex, one);
-                    result = builder.create<mlir::arith::SubIOp>(loc, idxTy,
-                                                                 result, lb);
+                    result = mlir::arith::SubIOp::create(builder, loc, idxTy,
+                                                         result, lb);
                     pc = [=](IterSpace iters) {
                       IterationSpace newIters = currentPC(iters);
                       newIters.insertIndexValue(subsIndex, result);
@@ -5671,7 +5682,7 @@ class ArrayExprLowering {
     auto seqType = mlir::cast<fir::SequenceType>(ty);
     for (auto extent : seqType.getShape()) {
       auto v = extent == fir::SequenceType::getUnknownExtent()
-                   ? builder.create<fir::UndefOp>(loc, idxTy).getResult()
+                   ? fir::UndefOp::create(builder, loc, idxTy).getResult()
                    : builder.createIntegerConstant(loc, idxTy, extent);
       result.push_back(v);
     }
@@ -5764,20 +5775,20 @@ class ArrayExprLowering {
         mlir::Value one =
             builder.createIntegerConstant(loc, substringBounds[0].getType(), 1);
         substringBounds[0] =
-            builder.create<mlir::arith::SubIOp>(loc, substringBounds[0], one);
+            mlir::arith::SubIOp::create(builder, loc, substringBounds[0], one);
         // Convert the upper bound to a length.
         mlir::Value cast = builder.createConvert(loc, iTy, substringBounds[1]);
         mlir::Value zero = builder.createIntegerConstant(loc, iTy, 0);
         auto size =
-            builder.create<mlir::arith::SubIOp>(loc, cast, substringBounds[0]);
-        auto cmp = builder.create<mlir::arith::CmpIOp>(
-            loc, mlir::arith::CmpIPredicate::sgt, size, zero);
+            mlir::arith::SubIOp::create(builder, loc, cast, substringBounds[0]);
+        auto cmp = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::sgt, size, zero);
         // size = MAX(upper - (lower - 1), 0)
         substringBounds[1] =
-            builder.create<mlir::arith::SelectOp>(loc, cmp, size, zero);
-        slice = builder.create<fir::SliceOp>(
-            loc, padSlice(components.trips, shape), components.suffixComponents,
-            substringBounds);
+            mlir::arith::SelectOp::create(builder, loc, cmp, size, zero);
+        slice = fir::SliceOp::create(
+            builder, loc, padSlice(components.trips, shape),
+            components.suffixComponents, substringBounds);
       } else {
         slice = builder.createSlice(loc, extMemref, components.trips,
                                     components.suffixComponents);
@@ -5827,7 +5838,7 @@ class ArrayExprLowering {
       }
       mlir::Value embox =
           mlir::isa<fir::BaseBoxType>(memref.getType())
-              ? builder.create<fir::ReboxOp>(loc, boxTy, memref, shape, slice)
+              ? fir::ReboxOp::create(builder, loc, boxTy, memref, shape, slice)
                     .getResult()
               : builder
                     .create<fir::EmboxOp>(loc, boxTy, memref, shape, slice,
@@ -5849,8 +5860,8 @@ class ArrayExprLowering {
         // ArrayCoorOp does not expect zero based indices.
         llvm::SmallVector<mlir::Value> indices = fir::factory::originateIndices(
             loc, builder, memref.getType(), shape, iters.iterVec());
-        mlir::Value coor = builder.create<fir::ArrayCoorOp>(
-            loc, refEleTy, memref, shape, slice, indices,
+        mlir::Value coor = fir::ArrayCoorOp::create(
+            builder, loc, refEleTy, memref, shape, slice, indices,
             fir::getTypeParams(extMemref));
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
           llvm::SmallVector<mlir::Value> substringBounds;
@@ -5869,16 +5880,17 @@ class ArrayExprLowering {
             builder, loc, extMemref, coor, slice);
       };
     }
-    auto arrLoad = builder.create<fir::ArrayLoadOp>(
-        loc, arrTy, memref, shape, slice, fir::getTypeParams(extMemref));
+    auto arrLoad =
+        fir::ArrayLoadOp::create(builder, loc, arrTy, memref, shape, slice,
+                                 fir::getTypeParams(extMemref));
 
     if (CrayPtr) {
       mlir::Type ptrTy = CrayPtr.getType();
       mlir::Value cnvrt = Fortran::lower::addCrayPointerInst(
           loc, builder, CrayPtr, ptrTy, memref.getType());
-      auto addr = builder.create<fir::LoadOp>(loc, cnvrt);
-      arrLoad = builder.create<fir::ArrayLoadOp>(loc, arrTy, addr, shape, slice,
-                                                 fir::getTypeParams(extMemref));
+      auto addr = fir::LoadOp::create(builder, loc, cnvrt);
+      arrLoad = fir::ArrayLoadOp::create(builder, loc, arrTy, addr, shape,
+                                         slice, fir::getTypeParams(extMemref));
     }
 
     mlir::Value arrLd = arrLoad.getResult();
@@ -5906,9 +5918,9 @@ class ArrayExprLowering {
         mlir::Type eleTy = fir::applyPathToType(resTy, iters.iterVec());
         mlir::Type refEleTy =
             fir::isa_ref_type(eleTy) ? eleTy : builder.getRefType(eleTy);
-        auto arrModify = builder.create<fir::ArrayModifyOp>(
-            loc, mlir::TypeRange{refEleTy, resTy}, innerArg, iters.iterVec(),
-            destination.getTypeparams());
+        auto arrModify = fir::ArrayModifyOp::create(
+            builder, loc, mlir::TypeRange{refEleTy, resTy}, innerArg,
+            iters.iterVec(), destination.getTypeparams());
         return abstractArrayExtValue(arrModify.getResult(1));
       };
     }
@@ -5932,17 +5944,17 @@ class ArrayExprLowering {
         mlir::Type eleTy = fir::applyPathToType(arrTy, iters.iterVec());
         if (isAdjustedArrayElementType(eleTy)) {
           mlir::Type eleRefTy = builder.getRefType(eleTy);
-          base = builder.create<fir::ArrayAccessOp>(
-              loc, eleRefTy, arrLd, iters.iterVec(), arrLdTypeParams);
+          base = fir::ArrayAccessOp::create(builder, loc, eleRefTy, arrLd,
+                                            iters.iterVec(), arrLdTypeParams);
         } else {
-          base = builder.create<fir::ArrayFetchOp>(
-              loc, eleTy, arrLd, iters.iterVec(), arrLdTypeParams);
+          base = fir::ArrayFetchOp::create(builder, loc, eleTy, arrLd,
+                                           iters.iterVec(), arrLdTypeParams);
         }
         mlir::Value temp =
             builder.createTemporary(loc, base.getType(),
                                     llvm::ArrayRef<mlir::NamedAttribute>{
                                         fir::getAdaptToByRefAttr(builder)});
-        builder.create<fir::StoreOp>(loc, base, temp);
+        fir::StoreOp::create(builder, loc, base, temp);
         return fir::factory::arraySectionElementToExtendedValue(
             builder, loc, extMemref, temp, slice);
       };
@@ -5953,8 +5965,8 @@ class ArrayExprLowering {
       mlir::Type eleTy = fir::applyPathToType(arrTy, iters.iterVec());
       if (isAdjustedArrayElementType(eleTy)) {
         mlir::Type eleRefTy = builder.getRefType(eleTy);
-        mlir::Value arrayOp = builder.create<fir::ArrayAccessOp>(
-            loc, eleRefTy, arrLd, iters.iterVec(), arrLdTypeParams);
+        mlir::Value arrayOp = fir::ArrayAccessOp::create(
+            builder, loc, eleRefTy, arrLd, iters.iterVec(), arrLdTypeParams);
         if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
           llvm::SmallVector<mlir::Value> substringBounds;
           populateBounds(substringBounds, components.substring);
@@ -5969,8 +5981,8 @@ class ArrayExprLowering {
         return fir::factory::arraySectionElementToExtendedValue(
             builder, loc, extMemref, arrayOp, slice);
       }
-      auto arrFetch = builder.create<fir::ArrayFetchOp>(
-          loc, eleTy, arrLd, iters.iterVec(), arrLdTypeParams);
+      auto arrFetch = fir::ArrayFetchOp::create(
+          builder, loc, eleTy, arrLd, iters.iterVec(), arrLdTypeParams);
       return fir::factory::arraySectionElementToExtendedValue(
           builder, loc, extMemref, arrFetch, slice);
     };
@@ -6009,8 +6021,8 @@ class ArrayExprLowering {
     mlir::Value memref = fir::getBase(exv);
     mlir::Value shape = builder.createShape(loc, exv);
     mlir::Value noSlice;
-    auto arrLoad = builder.create<fir::ArrayLoadOp>(
-        loc, arrType, memref, shape, noSlice, fir::getTypeParams(exv));
+    auto arrLoad = fir::ArrayLoadOp::create(
+        builder, loc, arrType, memref, shape, noSlice, fir::getTypeParams(exv));
     mlir::Operation::operand_range arrLdTypeParams = arrLoad.getTypeparams();
     mlir::Value arrLd = arrLoad.getResult();
     // Mark the load to tell later passes it is unsafe to use this array_load
@@ -6025,8 +6037,8 @@ class ArrayExprLowering {
 
     // By value semantics.
     auto cc = [=](IterSpace iters) -> ExtValue {
-      auto arrFetch = builder.create<fir::ArrayFetchOp>(
-          loc, eleType, arrLd, iters.iterVec(), arrLdTypeParams);
+      auto arrFetch = fir::ArrayFetchOp::create(
+          builder, loc, eleType, arrLd, iters.iterVec(), arrLdTypeParams);
       return fir::factory::arraySectionElementToExtendedValue(
           builder, loc, exv, arrFetch, noSlice);
     };
@@ -6064,12 +6076,12 @@ class ArrayExprLowering {
               .genIfOp(loc, {eleType}, isPresent,
                        /*withElseRegion=*/true)
               .genThen([&]() {
-                builder.create<fir::ResultOp>(loc, fir::getBase(cc(iters)));
+                fir::ResultOp::create(builder, loc, fir::getBase(cc(iters)));
               })
               .genElse([&]() {
                 mlir::Value zero =
                     fir::factory::createZeroValue(builder, loc, eleType);
-                builder.create<fir::ResultOp>(loc, zero);
+                fir::ResultOp::create(builder, loc, zero);
               })
               .getResults()[0];
       return elementValue;
@@ -6178,8 +6190,8 @@ class ArrayExprLowering {
     mlir::Type eleRefTy = builder.getRefType(eleTy);
     mlir::Type resRefTy = builder.getRefType(resTy);
     mlir::Value nullPtr = builder.createNullConstant(loc, resRefTy);
-    auto offset = builder.create<fir::CoordinateOp>(
-        loc, eleRefTy, nullPtr, mlir::ValueRange{multiplier});
+    auto offset = fir::CoordinateOp::create(builder, loc, eleRefTy, nullPtr,
+                                            mlir::ValueRange{multiplier});
     return builder.createConvert(loc, idxTy, offset);
   }
 
@@ -6193,8 +6205,8 @@ class ArrayExprLowering {
   /// Create a call to the LLVM memcpy intrinsic.
   void createCallMemcpy(llvm::ArrayRef<mlir::Value> args, bool isVolatile) {
     mlir::Location loc = getLoc();
-    builder.create<mlir::LLVM::MemcpyOp>(loc, args[0], args[1], args[2],
-                                         isVolatile);
+    mlir::LLVM::MemcpyOp::create(builder, loc, args[0], args[1], args[2],
+                                 isVolatile);
   }
 
   // Construct code to check for a buffer overrun and realloc the buffer when
@@ -6204,32 +6216,33 @@ class ArrayExprLowering {
                          mlir::Value eleSz) {
     mlir::Location loc = getLoc();
     mlir::func::FuncOp reallocFunc = fir::factory::getRealloc(builder);
-    auto cond = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::sle, bufferSize, needed);
-    auto ifOp = builder.create<fir::IfOp>(loc, mem.getType(), cond,
-                                          /*withElseRegion=*/true);
+    auto cond = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::sle, bufferSize, needed);
+    auto ifOp = fir::IfOp::create(builder, loc, mem.getType(), cond,
+                                  /*withElseRegion=*/true);
     auto insPt = builder.saveInsertionPoint();
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     // Not enough space, resize the buffer.
     mlir::IndexType idxTy = builder.getIndexType();
     mlir::Value two = builder.createIntegerConstant(loc, idxTy, 2);
-    auto newSz = builder.create<mlir::arith::MulIOp>(loc, needed, two);
-    builder.create<fir::StoreOp>(loc, newSz, buffSize);
-    mlir::Value byteSz = builder.create<mlir::arith::MulIOp>(loc, newSz, eleSz);
+    auto newSz = mlir::arith::MulIOp::create(builder, loc, needed, two);
+    fir::StoreOp::create(builder, loc, newSz, buffSize);
+    mlir::Value byteSz =
+        mlir::arith::MulIOp::create(builder, loc, newSz, eleSz);
     mlir::SymbolRefAttr funcSymAttr =
         builder.getSymbolRefAttr(reallocFunc.getName());
     mlir::FunctionType funcTy = reallocFunc.getFunctionType();
-    auto newMem = builder.create<fir::CallOp>(
-        loc, funcSymAttr, funcTy.getResults(),
+    auto newMem = fir::CallOp::create(
+        builder, loc, funcSymAttr, funcTy.getResults(),
         llvm::ArrayRef<mlir::Value>{
             builder.createConvert(loc, funcTy.getInputs()[0], mem),
             builder.createConvert(loc, funcTy.getInputs()[1], byteSz)});
     mlir::Value castNewMem =
         builder.createConvert(loc, mem.getType(), newMem.getResult(0));
-    builder.create<fir::ResultOp>(loc, castNewMem);
+    fir::ResultOp::create(builder, loc, castNewMem);
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
     // Otherwise, just forward the buffer.
-    builder.create<fir::ResultOp>(loc, mem);
+    fir::ResultOp::create(builder, loc, mem);
     builder.restoreInsertionPoint(insPt);
     return ifOp.getResult(0);
   }
@@ -6241,8 +6254,8 @@ class ArrayExprLowering {
                                        mlir::Value eleSz, mlir::Type eleTy,
                                        mlir::Type eleRefTy, mlir::Type resTy) {
     mlir::Location loc = getLoc();
-    auto off = builder.create<fir::LoadOp>(loc, buffPos);
-    auto limit = builder.create<fir::LoadOp>(loc, buffSize);
+    auto off = fir::LoadOp::create(builder, loc, buffPos);
+    auto limit = fir::LoadOp::create(builder, loc, buffSize);
     mlir::IndexType idxTy = builder.getIndexType();
     mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
 
@@ -6260,7 +6273,7 @@ class ArrayExprLowering {
         mlir::Value length = fir::getLen(exv);
         if (!length)
           fir::emitFatalError(loc, "result is not boxed character");
-        eleSz = builder.create<mlir::arith::MulIOp>(loc, bytes, length);
+        eleSz = mlir::arith::MulIOp::create(builder, loc, bytes, length);
       } else {
         TODO(loc, "PDT size");
         // Will call the PDT's size function with the type parameters.
@@ -6279,13 +6292,13 @@ class ArrayExprLowering {
           refTy = builder.getRefType(chTy);
           mlir::Type toTy = builder.getRefType(builder.getVarLenSeqTy(chTy));
           buff = builder.createConvert(loc, toTy, buff);
-          off = builder.create<mlir::arith::MulIOp>(loc, off, eleSz);
+          off = mlir::arith::MulIOp::create(builder, loc, off, eleSz);
         } else {
           TODO(loc, "PDT offset");
         }
       }
-      auto coor = builder.create<fir::CoordinateOp>(loc, refTy, buff,
-                                                    mlir::ValueRange{off});
+      auto coor = fir::CoordinateOp::create(builder, loc, refTy, buff,
+                                            mlir::ValueRange{off});
       return builder.createConvert(loc, eleRefTy, coor);
     };
 
@@ -6294,15 +6307,15 @@ class ArrayExprLowering {
       // Compute the array size.
       mlir::Value arrSz = one;
       for (auto ext : v.getExtents())
-        arrSz = builder.create<mlir::arith::MulIOp>(loc, arrSz, ext);
+        arrSz = mlir::arith::MulIOp::create(builder, loc, arrSz, ext);
 
       // Grow the buffer as needed.
-      auto endOff = builder.create<mlir::arith::AddIOp>(loc, off, arrSz);
+      auto endOff = mlir::arith::AddIOp::create(builder, loc, off, arrSz);
       mem = growBuffer(mem, endOff, limit, buffSize, eleSz);
 
       // Copy the elements to the buffer.
       mlir::Value byteSz =
-          builder.create<mlir::arith::MulIOp>(loc, arrSz, eleSz);
+          mlir::arith::MulIOp::create(builder, loc, arrSz, eleSz);
       auto buff = builder.createConvert(loc, fir::HeapType::get(resTy), mem);
       mlir::Value buffi = computeCoordinate(buff, off);
       llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
@@ -6311,13 +6324,13 @@ class ArrayExprLowering {
       createCallMemcpy(args, isVolatile);
 
       // Save the incremented buffer position.
-      builder.create<fir::StoreOp>(loc, endOff, buffPos);
+      fir::StoreOp::create(builder, loc, endOff, buffPos);
     };
 
     // Copy a trivial scalar value into the buffer.
     auto doTrivialScalar = [&](const ExtValue &v, mlir::Value len = {}) {
       // Increment the buffer position.
-      auto plusOne = builder.create<mlir::arith::AddIOp>(loc, off, one);
+      auto plusOne = mlir::arith::AddIOp::create(builder, loc, off, one);
 
       // Grow the buffer as needed.
       mem = growBuffer(mem, plusOne, limit, buffSize, eleSz);
@@ -6325,8 +6338,8 @@ class ArrayExprLowering {
       // Store the element in the buffer.
       mlir::Value buff =
           builder.createConvert(loc, fir::HeapType::get(resTy), mem);
-      auto buffi = builder.create<fir::CoordinateOp>(loc, eleRefTy, buff,
-                                                     mlir::ValueRange{off});
+      auto buffi = fir::CoordinateOp::create(builder, loc, eleRefTy, buff,
+                                             mlir::ValueRange{off});
       fir::factory::genScalarAssignment(
           builder, loc,
           [&]() -> ExtValue {
@@ -6335,7 +6348,7 @@ class ArrayExprLowering {
             return buffi;
           }(),
           v);
-      builder.create<fir::StoreOp>(loc, plusOne, buffPos);
+      fir::StoreOp::create(builder, loc, plusOne, buffPos);
     };
 
     // Copy the value.
@@ -6347,7 +6360,7 @@ class ArrayExprLowering {
             doTrivialScalar(exv, eleSz);
           } else {
             // Increment the buffer position.
-            auto plusOne = builder.create<mlir::arith::AddIOp>(loc, off, one);
+            auto plusOne = mlir::arith::AddIOp::create(builder, loc, off, one);
 
             // Grow the buffer as needed.
             mem = growBuffer(mem, plusOne, limit, buffSize, eleSz);
@@ -6362,7 +6375,7 @@ class ArrayExprLowering {
                 fir::isa_volatile_type(v.getAddr().getType());
             createCallMemcpy(args, isVolatile);
 
-            builder.create<fir::StoreOp>(loc, plusOne, buffPos);
+            fir::StoreOp::create(builder, loc, plusOne, buffPos);
           }
         },
         [&](const fir::ArrayBoxValue &v) { doAbstractArray(v); },
@@ -6403,8 +6416,8 @@ class ArrayExprLowering {
     auto seqTy = mlir::cast<fir::SequenceType>(resTy);
     mlir::Type eleTy = fir::unwrapSequenceType(seqTy);
     auto loop =
-        builder.create<fir::DoLoopOp>(loc, lo, up, step, /*unordered=*/false,
-                                      /*finalCount=*/false, mem);
+        fir::DoLoopOp::create(builder, loc, lo, up, step, /*unordered=*/false,
+                              /*finalCount=*/false, mem);
     // create a new binding for x.name(), to ac-do-variable, to the iteration
     // value.
     symMap.pushImpliedDoBinding(toStringRef(x.name()), loop.getInductionVar());
@@ -6434,22 +6447,22 @@ class ArrayExprLowering {
         mlir::Value castLen =
             builder.createConvert(loc, builder.getI64Type(), fir::getLen(exv));
         assert(charLen.has_value());
-        builder.create<fir::StoreOp>(loc, castLen, *charLen);
+        fir::StoreOp::create(builder, loc, castLen, *charLen);
       }
     }
     stmtCtx.finalizeAndPop();
 
-    builder.create<fir::ResultOp>(loc, mem);
+    fir::ResultOp::create(builder, loc, mem);
     builder.restoreInsertionPoint(insPt);
     mem = loop.getResult(0);
     symMap.popImpliedDoBinding();
     llvm::SmallVector<mlir::Value> extents = {
-        builder.create<fir::LoadOp>(loc, buffPos).getResult()};
+        fir::LoadOp::create(builder, loc, buffPos).getResult()};
 
     // Convert to extended value.
     if (fir::isa_char(seqTy.getEleTy())) {
       assert(charLen.has_value());
-      auto len = builder.create<fir::LoadOp>(loc, *charLen);
+      auto len = fir::LoadOp::create(builder, loc, *charLen);
       return {fir::CharArrayBoxValue{mem, len, extents}, /*needCopy=*/false};
     }
     return {fir::ArrayBoxValue{mem, extents}, /*needCopy=*/false};
@@ -6472,7 +6485,7 @@ class ArrayExprLowering {
     mlir::Value buffSize = builder.createTemporary(loc, idxTy, ".buff.size");
     mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
     mlir::Value buffPos = builder.createTemporary(loc, idxTy, ".buff.pos");
-    builder.create<fir::StoreOp>(loc, zero, buffPos);
+    fir::StoreOp::create(builder, loc, zero, buffPos);
     // Allocate space for the array to be constructed.
     mlir::Value mem;
     if (fir::hasDynamicSize(resTy)) {
@@ -6480,22 +6493,22 @@ class ArrayExprLowering {
         // The size of each element may depend on a general expression. Defer
         // creating the buffer until after the expression is evaluated.
         mem = builder.createNullConstant(loc, builder.getRefType(eleTy));
-        builder.create<fir::StoreOp>(loc, zero, buffSize);
+        fir::StoreOp::create(builder, loc, zero, buffSize);
       } else {
         mlir::Value initBuffSz =
             builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
-        mem = builder.create<fir::AllocMemOp>(
-            loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
-        builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
+        mem = fir::AllocMemOp::create(
+            builder, loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
+        fir::StoreOp::create(builder, loc, initBuffSz, buffSize);
       }
     } else {
-      mem = builder.create<fir::AllocMemOp>(loc, resTy);
+      mem = fir::AllocMemOp::create(builder, loc, resTy);
       int64_t buffSz = 1;
       for (auto extent : seqTy.getShape())
         buffSz *= extent;
       mlir::Value initBuffSz =
           builder.createIntegerConstant(loc, idxTy, buffSz);
-      builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
+      fir::StoreOp::create(builder, loc, initBuffSz, buffSize);
     }
     // Compute size of element
     mlir::Type eleRefTy = builder.getRefType(eleTy);
@@ -6517,12 +6530,12 @@ class ArrayExprLowering {
         charLen = builder.createTemporary(loc, builder.getI64Type());
         mlir::Value castLen =
             builder.createConvert(loc, builder.getI64Type(), fir::getLen(exv));
-        builder.create<fir::StoreOp>(loc, castLen, *charLen);
+        fir::StoreOp::create(builder, loc, castLen, *charLen);
       }
     }
     mem = builder.createConvert(loc, fir::HeapType::get(resTy), mem);
     llvm::SmallVector<mlir::Value> extents = {
-        builder.create<fir::LoadOp>(loc, buffPos)};
+        fir::LoadOp::create(builder, loc, buffPos)};
 
     // Cleanup the temporary.
     fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
@@ -6532,7 +6545,7 @@ class ArrayExprLowering {
     // Return the continuation.
     if (fir::isa_char(seqTy.getEleTy())) {
       if (charLen) {
-        auto len = builder.create<fir::LoadOp>(loc, *charLen);
+        auto len = fir::LoadOp::create(builder, loc, *charLen);
         return genarr(fir::CharArrayBoxValue{mem, len, extents});
       }
       return genarr(fir::CharArrayBoxValue{mem, zero, extents});
@@ -6569,7 +6582,7 @@ class ArrayExprLowering {
     return [=](IterSpace iters) -> ExtValue {
       mlir::Value logical = fir::getBase(lambda(iters));
       mlir::Value val = builder.createConvert(loc, i1Ty, logical);
-      return builder.create<mlir::arith::XOrIOp>(loc, val, truth);
+      return mlir::arith::XOrIOp::create(builder, loc, val, truth);
     };
   }
   template <typename OP, typename A>
@@ -6583,7 +6596,7 @@ class ArrayExprLowering {
       mlir::Value right = fir::getBase(rf(iters));
       mlir::Value lhs = builder.createConvert(loc, i1Ty, left);
       mlir::Value rhs = builder.createConvert(loc, i1Ty, right);
-      return builder.create<OP>(loc, lhs, rhs);
+      return OP::create(builder, loc, lhs, rhs);
     };
   }
   template <typename OP, typename A>
@@ -6597,7 +6610,7 @@ class ArrayExprLowering {
       mlir::Value right = fir::getBase(rf(iters));
       mlir::Value lhs = builder.createConvert(loc, i1Ty, left);
       mlir::Value rhs = builder.createConvert(loc, i1Ty, right);
-      return builder.create<OP>(loc, pred, lhs, rhs);
+      return OP::create(builder, loc, pred, lhs, rhs);
     };
   }
   template <int KIND>
@@ -6637,9 +6650,9 @@ class ArrayExprLowering {
             Fortran::common::TypeCategory::Integer, *unsignedKind);
         mlir::Value lhsSL = builder.createConvert(loc, signlessType, lhs);
         mlir::Value rhsSL = builder.createConvert(loc, signlessType, rhs);
-        return builder.create<OP>(loc, pred, lhsSL, rhsSL);
+        return OP::create(builder, loc, pred, lhsSL, rhsSL);
       }
-      return builder.create<OP>(loc, pred, lhs, rhs);
+      return OP::create(builder, loc, pred, lhs, rhs);
     };
   }
   template <typename A>
@@ -6806,9 +6819,8 @@ class ArrayExprLowering {
                                          : one;
                               mlir::Value val = builder.createConvert(
                                   loc, idxTy, subscriptVal);
-                              mlir::Value ivAdj =
-                                  builder.create<mlir::arith::SubIOp>(
-                                      loc, idxTy, val, lb);
+                              mlir::Value ivAdj = mlir::arith::SubIOp::create(
+                                  builder, loc, idxTy, val, lb);
                               componentsToAdd.push_back(
                                   builder.createConvert(loc, idxTy, ivAdj));
                             },
@@ -6830,8 +6842,9 @@ class ArrayExprLowering {
                     converter.getRecordTypeFieldName(getLastSym(*x));
                 if (auto recTy = mlir::dyn_cast<fir::RecordType>(ty)) {
                   ty = recTy.getType(name);
-                  auto fld = builder.create<fir::FieldIndexOp>(
-                      loc, fieldTy, name, recTy, fir::getTypeParams(arrayExv));
+                  auto fld = fir::FieldIndexOp::create(
+                      builder, loc, fieldTy, name, recTy,
+                      fir::getTypeParams(arrayExv));
                   addComponentList(ty, {fld});
                   if (index != revPath.size() - 1 || !isPointerAssignment()) {
                     // Need an intermediate  dereference if the boxed value
@@ -6852,8 +6865,9 @@ class ArrayExprLowering {
                   ty = fir::unwrapRefType(boxTy.getEleTy());
                   auto recTy = mlir::cast<fir::RecordType>(ty);
                   ty = recTy.getType(name);
-                  auto fld = builder.create<fir::FieldIndexOp>(
-                      loc, fieldTy, name, recTy, fir::getTypeParams(arrayExv));
+                  auto fld = fir::FieldIndexOp::create(
+                      builder, loc, fieldTy, name, recTy,
+                      fir::getTypeParams(arrayExv));
                   extendComponent(components, ty, {fld});
                 } else {
                   TODO(loc, "other component type");
@@ -6896,8 +6910,8 @@ class ArrayExprLowering {
         mlir::Value innerArg = esp->findArgumentOfLoad(load);
         if (isAdjustedArrayElementType(eleTy)) {
           mlir::Type eleRefTy = builder.getRefType(eleTy);
-          auto arrayOp = builder.create<fir::ArrayAccessOp>(
-              loc, eleRefTy, innerArg, iters.iterVec(),
+          auto arrayOp = fir::ArrayAccessOp::create(
+              builder, loc, eleRefTy, innerArg, iters.iterVec(),
               fir::factory::getTypeParams(loc, builder, load));
           if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
             mlir::Value dstLen = fir::factory::genLenOfCharacter(
@@ -6924,9 +6938,9 @@ class ArrayExprLowering {
           if (!eleBoxTy || !mlir::isa<fir::BoxType>(eleBoxTy))
             TODO(loc, "assignment in a FORALL involving a designator with a "
                       "POINTER or ALLOCATABLE component part-ref");
-          auto arrayOp = builder.create<fir::ArrayAccessOp>(
-              loc, builder.getRefType(eleBoxTy), innerArg, iters.iterVec(),
-              fir::factory::getTypeParams(loc, builder, load));
+          auto arrayOp = fir::ArrayAccessOp::create(
+              builder, loc, builder.getRefType(eleBoxTy), innerArg,
+              iters.iterVec(), fir::factory::getTypeParams(loc, builder, load));
           mlir::Value addr = components.getExtendCoorRef()(arrayOp);
           components.resetExtendCoorRef();
           // When the lhs is a boxed value and the context is not a pointer
@@ -6935,19 +6949,19 @@ class ArrayExprLowering {
           if (!isPointerAssignment()) {
             if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(eleTy)) {
               eleTy = fir::boxMemRefType(boxTy);
-              addr = builder.create<fir::BoxAddrOp>(loc, eleTy, addr);
+              addr = fir::BoxAddrOp::create(builder, loc, eleTy, addr);
               eleTy = fir::unwrapRefType(eleTy);
             }
           }
           auto ele = convertElementForUpdate(loc, eleTy, iters.getElement());
-          builder.create<fir::StoreOp>(loc, ele, addr);
-          auto amend = builder.create<fir::ArrayAmendOp>(
-              loc, innerArg.getType(), innerArg, arrayOp);
+          fir::StoreOp::create(builder, loc, ele, addr);
+          auto amend = fir::ArrayAmendOp::create(
+              builder, loc, innerArg.getType(), innerArg, arrayOp);
           return arrayLoadExtValue(builder, loc, load, iters.iterVec(), amend);
         }
         auto ele = convertElementForUpdate(loc, eleTy, iters.getElement());
-        auto update = builder.create<fir::ArrayUpdateOp>(
-            loc, innerArg.getType(), innerArg, ele, iters.iterVec(),
+        auto update = fir::ArrayUpdateOp::create(
+            builder, loc, innerArg.getType(), innerArg, ele, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         return arrayLoadExtValue(builder, loc, load, iters.iterVec(), update);
       };
@@ -6961,9 +6975,9 @@ class ArrayExprLowering {
         mlir::Value innerArg = explicitSpace->findArgumentOfLoad(load);
         mlir::Type refEleTy =
             fir::isa_ref_type(eleTy) ? eleTy : builder.getRefType(eleTy);
-        auto arrModify = builder.create<fir::ArrayModifyOp>(
-            loc, mlir::TypeRange{refEleTy, innerArg.getType()}, innerArg,
-            iters.iterVec(), load.getTypeparams());
+        auto arrModify = fir::ArrayModifyOp::create(
+            builder, loc, mlir::TypeRange{refEleTy, innerArg.getType()},
+            innerArg, iters.iterVec(), load.getTypeparams());
         return arrayLoadExtValue(builder, loc, load, iters.iterVec(),
                                  arrModify.getResult(1));
       };
@@ -6974,8 +6988,8 @@ class ArrayExprLowering {
           isAdjustedArrayElementType(eleTy)) {
         mlir::Type resTy = builder.getRefType(eleTy);
         // Use array element reference semantics.
-        auto access = builder.create<fir::ArrayAccessOp>(
-            loc, resTy, load, iters.iterVec(),
+        auto access = fir::ArrayAccessOp::create(
+            builder, loc, resTy, load, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         mlir::Value newBase = access;
         if (fir::isa_char(eleTy)) {
@@ -6997,8 +7011,8 @@ class ArrayExprLowering {
         if (!eleBoxTy || !mlir::isa<fir::BoxType>(eleBoxTy))
           TODO(loc, "assignment in a FORALL involving a designator with a "
                     "POINTER or ALLOCATABLE component part-ref");
-        auto access = builder.create<fir::ArrayAccessOp>(
-            loc, builder.getRefType(eleBoxTy), load, iters.iterVec(),
+        auto access = fir::ArrayAccessOp::create(
+            builder, loc, builder.getRefType(eleBoxTy), load, iters.iterVec(),
             fir::factory::getTypeParams(loc, builder, load));
         mlir::Value addr = components.getExtendCoorRef()(access);
         components.resetExtendCoorRef();
@@ -7010,8 +7024,8 @@ class ArrayExprLowering {
           // Rhs is a regular expression that will need to be boxed before
           // assigning to the boxed variable.
           auto typeParams = fir::factory::getTypeParams(loc, builder, load);
-          auto access = builder.create<fir::ArrayAccessOp>(
-              loc, builder.getRefType(eleTy), load, iters.iterVec(),
+          auto access = fir::ArrayAccessOp::create(
+              builder, loc, builder.getRefType(eleTy), load, iters.iterVec(),
               typeParams);
           auto addr = components.getExtendCoorRef()(access);
           components.resetExtendCoorRef();
@@ -7025,14 +7039,14 @@ class ArrayExprLowering {
             TODO(loc, "need to adjust typeparameter(s) to reflect the final "
                       "component");
           mlir::Value embox =
-              builder.create<fir::EmboxOp>(loc, boxTy, ptrAddr,
-                                           /*shape=*/mlir::Value{},
-                                           /*slice=*/mlir::Value{}, typeParams);
+              fir::EmboxOp::create(builder, loc, boxTy, ptrAddr,
+                                   /*shape=*/mlir::Value{},
+                                   /*slice=*/mlir::Value{}, typeParams);
           return arrayLoadExtValue(builder, loc, load, iters.iterVec(), embox);
         }
       }
-      auto fetch = builder.create<fir::ArrayFetchOp>(
-          loc, eleTy, load, iters.iterVec(), load.getTypeparams());
+      auto fetch = fir::ArrayFetchOp::create(
+          builder, loc, eleTy, load, iters.iterVec(), load.getTypeparams());
       return arrayLoadExtValue(builder, loc, load, iters.iterVec(), fetch);
     };
     return [=](IterSpace iters) mutable { return lambda(pc(iters)); };
@@ -7541,24 +7555,25 @@ fir::ExtendedValue Fortran::lower::updateBoxForParentComponent(
 
   if (op) {
     if (auto embox = mlir::dyn_cast<fir::EmboxOp>(op)) {
-      auto newBox = builder.create<fir::EmboxOp>(
-          loc, fir::BoxType::get(actualTy), embox.getMemref(), embox.getShape(),
-          embox.getSlice(), embox.getTypeparams());
+      auto newBox = fir::EmboxOp::create(
+          builder, loc, fir::BoxType::get(actualTy), embox.getMemref(),
+          embox.getShape(), embox.getSlice(), embox.getTypeparams());
       return fir::substBase(box, newBox);
     }
     if (auto rebox = mlir::dyn_cast<fir::ReboxOp>(op)) {
-      auto newBox = builder.create<fir::ReboxOp>(
-          loc, fir::BoxType::get(actualTy), rebox.getBox(), rebox.getShape(),
-          rebox.getSlice());
+      auto newBox = fir::ReboxOp::create(
+          builder, loc, fir::BoxType::get(actualTy), rebox.getBox(),
+          rebox.getShape(), rebox.getSlice());
       return fir::substBase(box, newBox);
     }
   }
 
   mlir::Value empty;
   mlir::ValueRange emptyRange;
-  return builder.create<fir::ReboxOp>(loc, fir::BoxType::get(actualTy), boxBase,
-                                      /*shape=*/empty,
-                                      /*slice=*/empty);
+  return fir::ReboxOp::create(builder, loc, fir::BoxType::get(actualTy),
+                              boxBase,
+                              /*shape=*/empty,
+                              /*slice=*/empty);
 }
 
 fir::ExtendedValue Fortran::lower::createBoxValue(
@@ -7649,9 +7664,9 @@ fir::ArrayLoadOp genArrayLoad(mlir::Location loc,
   mlir::Value addr = fir::getBase(exv);
   mlir::Value shapeOp = builder.createShape(loc, exv);
   mlir::Type arrTy = fir::dyn_cast_ptrOrBoxEleTy(addr.getType());
-  return builder.create<fir::ArrayLoadOp>(loc, arrTy, addr, shapeOp,
-                                          /*slice=*/mlir::Value{},
-                                          fir::getTypeParams(exv));
+  return fir::ArrayLoadOp::create(builder, loc, arrTy, addr, shapeOp,
+                                  /*slice=*/mlir::Value{},
+                                  fir::getTypeParams(exv));
 }
 template <>
 fir::ArrayLoadOp
@@ -7698,9 +7713,9 @@ void Fortran::lower::createArrayMergeStores(
   for (auto i : llvm::enumerate(esp.getOuterLoop().getResults()))
     if (std::optional<fir::ArrayLoadOp> ldOpt = esp.getLhsLoad(i.index())) {
       fir::ArrayLoadOp load = *ldOpt;
-      builder.create<fir::ArrayMergeStoreOp>(loc, load, i.value(),
-                                             load.getMemref(), load.getSlice(),
-                                             load.getTypeparams());
+      fir::ArrayMergeStoreOp::create(builder, loc, load, i.value(),
+                                     load.getMemref(), load.getSlice(),
+                                     load.getTypeparams());
     }
   if (esp.loopCleanup) {
     (*esp.loopCleanup)(builder);
@@ -7722,12 +7737,12 @@ mlir::Value Fortran::lower::addCrayPointerInst(mlir::Location loc,
   mlir::Value empty;
   mlir::ValueRange emptyRange;
   auto boxTy = fir::BoxType::get(ptrTy);
-  auto box = builder.create<fir::EmboxOp>(loc, boxTy, ptrVal, empty, empty,
-                                          emptyRange);
-  mlir::Value addrof =
-      (mlir::isa<fir::ReferenceType>(ptrTy))
-          ? builder.create<fir::BoxAddrOp>(loc, ptrTy, box)
-          : builder.create<fir::BoxAddrOp>(loc, builder.getRefType(ptrTy), box);
+  auto box = fir::EmboxOp::create(builder, loc, boxTy, ptrVal, empty, empty,
+                                  emptyRange);
+  mlir::Value addrof = (mlir::isa<fir::ReferenceType>(ptrTy))
+                           ? fir::BoxAddrOp::create(builder, loc, ptrTy, box)
+                           : fir::BoxAddrOp::create(
+                                 builder, loc, builder.getRefType(ptrTy), box);
 
   auto refPtrTy =
       builder.getRefType(fir::PointerType::get(fir::dyn_cast_ptrEleTy(pteTy)));

diff  --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 7de433d6a201a..46be111242bf7 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -167,7 +167,7 @@ class HlfirDesignatorBuilder {
       extents.push_back(builder.createIntegerConstant(loc, idxTy, extent));
     }
     if (!mayHaveNonDefaultLowerBounds(componentSym))
-      return builder.create<fir::ShapeOp>(loc, extents);
+      return fir::ShapeOp::create(builder, loc, extents);
 
     llvm::SmallVector<mlir::Value> lbounds;
     if (const auto *objDetails =
@@ -312,8 +312,8 @@ class HlfirDesignatorBuilder {
     // hlfir.elemental_addr.
     if (auto elementalAddrOp = getVectorSubscriptElementAddrOp())
       builder.setInsertionPointToEnd(&elementalAddrOp->getBody().front());
-    auto designate = builder.create<hlfir::DesignateOp>(
-        getLoc(), designatorType, partInfo.base.value().getBase(),
+    auto designate = hlfir::DesignateOp::create(
+        builder, getLoc(), designatorType, partInfo.base.value().getBase(),
         partInfo.componentName, partInfo.componentShape, partInfo.subscripts,
         partInfo.substring, partInfo.complexPart, partInfo.resultShape,
         partInfo.typeParams, attributes);
@@ -344,7 +344,7 @@ class HlfirDesignatorBuilder {
         mlir::Type refPtrType = builder.getRefType(
             fir::PointerType::get(fir::dyn_cast_ptrEleTy(ptrAddr.getType())));
         mlir::Value cast = builder.createConvert(loc, refPtrType, ptrAddr);
-        mlir::Value ptrVal = builder.create<fir::LoadOp>(loc, cast);
+        mlir::Value ptrVal = fir::LoadOp::create(builder, loc, cast);
 
         // Update the base_addr to the value of the Cray pointer.
         // This is a hacky way to do the update, and it may harm
@@ -442,9 +442,9 @@ class HlfirDesignatorBuilder {
     } else {
       // Compute "len = max(ub-lb+1,0)" (Fortran 2018 9.4.1).
       mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-      auto boundsDiff = builder.create<mlir::arith::SubIOp>(
-          loc, partInfo.substring[1], partInfo.substring[0]);
-      auto rawLen = builder.create<mlir::arith::AddIOp>(loc, boundsDiff, one);
+      auto boundsDiff = mlir::arith::SubIOp::create(
+          builder, loc, partInfo.substring[1], partInfo.substring[0]);
+      auto rawLen = mlir::arith::AddIOp::create(builder, loc, boundsDiff, one);
       partInfo.typeParams[0] =
           fir::factory::genMaxWithZero(builder, loc, rawLen);
     }
@@ -803,10 +803,10 @@ class HlfirDesignatorBuilder {
     mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
     oneBasedIndex = builder.createConvert(loc, idxTy, oneBasedIndex);
     mlir::Value zeroBased =
-        builder.create<mlir::arith::SubIOp>(loc, oneBasedIndex, one);
+        mlir::arith::SubIOp::create(builder, loc, oneBasedIndex, one);
     mlir::Value offset =
-        builder.create<mlir::arith::MulIOp>(loc, zeroBased, step);
-    return builder.create<mlir::arith::AddIOp>(loc, lb, offset);
+        mlir::arith::MulIOp::create(builder, loc, zeroBased, step);
+    return mlir::arith::AddIOp::create(builder, loc, lb, offset);
   }
 
   /// Create an hlfir.element_addr operation to deal with vector subscripted
@@ -836,8 +836,8 @@ class HlfirDesignatorBuilder {
     assert(partInfo.base.has_value() &&
            "vector subscripted part must have a base");
     mlir::Value mold = *partInfo.base;
-    auto elementalAddrOp = builder.create<hlfir::ElementalAddrOp>(
-        loc, shape, mold, mlir::ValueRange{},
+    auto elementalAddrOp = hlfir::ElementalAddrOp::create(
+        builder, loc, shape, mold, mlir::ValueRange{},
         /*isUnordered=*/true);
     setVectorSubscriptElementAddrOp(elementalAddrOp);
     builder.setInsertionPointToEnd(&elementalAddrOp.getBody().front());
@@ -881,7 +881,7 @@ class HlfirDesignatorBuilder {
     builder.setInsertionPointToEnd(&elementalAddrOp.getBody().front());
     if (!elementAddr.isPolymorphic())
       elementalAddrOp.getMoldMutable().clear();
-    builder.create<hlfir::YieldOp>(loc, elementAddr);
+    hlfir::YieldOp::create(builder, loc, elementAddr);
     builder.setInsertionPointAfter(elementalAddrOp);
   }
 
@@ -1001,7 +1001,7 @@ HlfirDesignatorBuilder::convertVectorSubscriptedExprToElementalAddr(
     elementalAddrOp.getMoldMutable().clear();
   // Create the hlfir.yield terminator inside the hlfir.elemental_body.
   builder.setInsertionPointToEnd(&elementalAddrOp.getBody().front());
-  builder.create<hlfir::YieldOp>(loc, elementAddrEntity);
+  hlfir::YieldOp::create(builder, loc, elementAddrEntity);
   builder.setInsertionPointAfter(elementalAddrOp);
   // Reset the HlfirDesignatorBuilder state, in case it is used on a new
   // designator.
@@ -1034,7 +1034,7 @@ struct BinaryOp {};
                                                 rhs)};                         \
       } else {                                                                 \
         return hlfir::EntityWithAttributes{                                    \
-            builder.create<GenBinFirOp>(loc, lhs, rhs)};                       \
+            GenBinFirOp::create(builder, loc, lhs, rhs)};                      \
       }                                                                        \
     }                                                                          \
   };
@@ -1075,7 +1075,7 @@ struct BinaryOp<Fortran::evaluate::Divide<
           fir::genDivC(builder, loc, ty, lhs, rhs)};
     } else {
       return hlfir::EntityWithAttributes{
-          builder.create<mlir::complex::DivOp>(loc, lhs, rhs)};
+          mlir::complex::DivOp::create(builder, loc, lhs, rhs)};
     }
   }
 };
@@ -1219,8 +1219,8 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = builder.create<mlir::arith::CmpIOp>(
-        loc, translateSignedRelational(op.opr), lhs, rhs);
+    auto cmp = mlir::arith::CmpIOp::create(
+        builder, loc, translateSignedRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
 };
@@ -1241,8 +1241,8 @@ struct BinaryOp<Fortran::evaluate::Relational<
         mlir::IntegerType::SignednessSemantics::Signless);
     mlir::Value lhsSL = builder.createConvert(loc, signlessType, lhs);
     mlir::Value rhsSL = builder.createConvert(loc, signlessType, rhs);
-    auto cmp = builder.create<mlir::arith::CmpIOp>(
-        loc, translateUnsignedRelational(op.opr), lhsSL, rhsSL);
+    auto cmp = mlir::arith::CmpIOp::create(
+        builder, loc, translateUnsignedRelational(op.opr), lhsSL, rhsSL);
     return hlfir::EntityWithAttributes{cmp};
   }
 };
@@ -1256,8 +1256,8 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = builder.create<mlir::arith::CmpFOp>(
-        loc, translateFloatRelational(op.opr), lhs, rhs);
+    auto cmp = mlir::arith::CmpFOp::create(
+        builder, loc, translateFloatRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
 };
@@ -1271,8 +1271,8 @@ struct BinaryOp<Fortran::evaluate::Relational<
                                          fir::FirOpBuilder &builder,
                                          const Op &op, hlfir::Entity lhs,
                                          hlfir::Entity rhs) {
-    auto cmp = builder.create<fir::CmpcOp>(
-        loc, translateFloatRelational(op.opr), lhs, rhs);
+    auto cmp = fir::CmpcOp::create(builder, loc,
+                                   translateFloatRelational(op.opr), lhs, rhs);
     return hlfir::EntityWithAttributes{cmp};
   }
 };
@@ -1313,16 +1313,16 @@ struct BinaryOp<Fortran::evaluate::LogicalOperation<KIND>> {
     switch (op.logicalOperator) {
     case Fortran::evaluate::LogicalOperator::And:
       return hlfir::EntityWithAttributes{
-          builder.create<mlir::arith::AndIOp>(loc, i1Lhs, i1Rhs)};
+          mlir::arith::AndIOp::create(builder, loc, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Or:
       return hlfir::EntityWithAttributes{
-          builder.create<mlir::arith::OrIOp>(loc, i1Lhs, i1Rhs)};
+          mlir::arith::OrIOp::create(builder, loc, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Eqv:
-      return hlfir::EntityWithAttributes{builder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::eq, i1Lhs, i1Rhs)};
+      return hlfir::EntityWithAttributes{mlir::arith::CmpIOp::create(
+          builder, loc, mlir::arith::CmpIPredicate::eq, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Neqv:
-      return hlfir::EntityWithAttributes{builder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::ne, i1Lhs, i1Rhs)};
+      return hlfir::EntityWithAttributes{mlir::arith::CmpIOp::create(
+          builder, loc, mlir::arith::CmpIPredicate::ne, i1Lhs, i1Rhs)};
     case Fortran::evaluate::LogicalOperator::Not:
       // lib/evaluate expression for .NOT. is Fortran::evaluate::Not<KIND>.
       llvm_unreachable(".NOT. is not a binary operator");
@@ -1354,7 +1354,7 @@ struct BinaryOp<Fortran::evaluate::SetLength<KIND>> {
     // Fortran 2018 7.4.4.2 point 5.
     mlir::Value safeLength = fir::factory::genMaxWithZero(builder, loc, length);
     return hlfir::EntityWithAttributes{
-        builder.create<hlfir::SetLengthOp>(loc, string, safeLength)};
+        hlfir::SetLengthOp::create(builder, loc, string, safeLength)};
   }
   static void
   genResultTypeParams(mlir::Location, fir::FirOpBuilder &, hlfir::Entity,
@@ -1372,7 +1372,7 @@ struct BinaryOp<Fortran::evaluate::Concat<KIND>> {
                                   hlfir::Entity lhs, hlfir::Entity rhs) {
     assert(len && "genResultTypeParams must have been called");
     auto concat =
-        builder.create<hlfir::ConcatOp>(loc, mlir::ValueRange{lhs, rhs}, len);
+        hlfir::ConcatOp::create(builder, loc, mlir::ValueRange{lhs, rhs}, len);
     return hlfir::EntityWithAttributes{concat.getResult()};
   }
   void
@@ -1386,7 +1386,7 @@ struct BinaryOp<Fortran::evaluate::Concat<KIND>> {
     mlir::Type idxType = builder.getIndexType();
     mlir::Value lhsLen = builder.createConvert(loc, idxType, lengths[0]);
     mlir::Value rhsLen = builder.createConvert(loc, idxType, lengths[1]);
-    len = builder.create<mlir::arith::AddIOp>(loc, lhsLen, rhsLen);
+    len = mlir::arith::AddIOp::create(builder, loc, lhsLen, rhsLen);
     resultTypeParams.push_back(len);
   }
 
@@ -1410,7 +1410,7 @@ struct UnaryOp<Fortran::evaluate::Not<KIND>> {
     mlir::Value one = builder.createBool(loc, true);
     mlir::Value val = builder.createConvert(loc, builder.getI1Type(), lhs);
     return hlfir::EntityWithAttributes{
-        builder.create<mlir::arith::XOrIOp>(loc, val, one)};
+        mlir::arith::XOrIOp::create(builder, loc, val, one)};
   }
 };
 
@@ -1428,7 +1428,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
         /*params=*/{});
     mlir::Value zero = builder.createIntegerConstant(loc, type, 0);
     return hlfir::EntityWithAttributes{
-        builder.create<mlir::arith::SubIOp>(loc, zero, lhs)};
+        mlir::arith::SubIOp::create(builder, loc, zero, lhs)};
   }
 };
 
@@ -1448,7 +1448,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
     mlir::Value zero = builder.createIntegerConstant(loc, signlessType, 0);
     mlir::Value signless = builder.createConvert(loc, signlessType, lhs);
     mlir::Value negated =
-        builder.create<mlir::arith::SubIOp>(loc, zero, signless);
+        mlir::arith::SubIOp::create(builder, loc, zero, signless);
     return hlfir::EntityWithAttributes(
         builder.createConvert(loc, lhs.getType(), negated));
   }
@@ -1463,7 +1463,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
                                          fir::FirOpBuilder &builder, const Op &,
                                          hlfir::Entity lhs) {
     return hlfir::EntityWithAttributes{
-        builder.create<mlir::arith::NegFOp>(loc, lhs)};
+        mlir::arith::NegFOp::create(builder, loc, lhs)};
   }
 };
 
@@ -1475,7 +1475,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
   static hlfir::EntityWithAttributes gen(mlir::Location loc,
                                          fir::FirOpBuilder &builder, const Op &,
                                          hlfir::Entity lhs) {
-    return hlfir::EntityWithAttributes{builder.create<fir::NegcOp>(loc, lhs)};
+    return hlfir::EntityWithAttributes{fir::NegcOp::create(builder, loc, lhs)};
   }
 };
 
@@ -1499,9 +1499,9 @@ struct UnaryOp<Fortran::evaluate::Parentheses<T>> {
                                          const Op &op, hlfir::Entity lhs) {
     if (lhs.isVariable())
       return hlfir::EntityWithAttributes{
-          builder.create<hlfir::AsExprOp>(loc, lhs)};
+          hlfir::AsExprOp::create(builder, loc, lhs)};
     return hlfir::EntityWithAttributes{
-        builder.create<hlfir::NoReassocOp>(loc, lhs.getType(), lhs)};
+        hlfir::NoReassocOp::create(builder, loc, lhs.getType(), lhs)};
   }
 
   static void
@@ -1822,8 +1822,8 @@ class HlfirBuilder {
     // Allocate scalar temporary that will be initialized
     // with the values specified by the constructor.
     mlir::Value storagePtr = builder.createTemporary(loc, recTy);
-    auto varOp = hlfir::EntityWithAttributes{builder.create<hlfir::DeclareOp>(
-        loc, storagePtr, "ctor.temp", /*shape=*/nullptr,
+    auto varOp = hlfir::EntityWithAttributes{hlfir::DeclareOp::create(
+        builder, loc, storagePtr, "ctor.temp", /*shape=*/nullptr,
         /*typeparams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{})};
 
@@ -1855,8 +1855,8 @@ class HlfirBuilder {
         auto parentCompType = baseRecTy.getType(parentName);
         assert(parentCompType && "failed to retrieve parent component type");
         mlir::Type designatorType = builder.getRefType(parentCompType);
-        mlir::Value newParent = builder.create<hlfir::DesignateOp>(
-            loc, designatorType, currentParent, parentName,
+        mlir::Value newParent = hlfir::DesignateOp::create(
+            builder, loc, designatorType, currentParent, parentName,
             /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
             /*substring=*/mlir::ValueRange{},
             /*complexPart=*/std::nullopt,
@@ -1912,8 +1912,8 @@ class HlfirBuilder {
                                                     extraAttributeFlags);
 
       // Get the component designator.
-      auto lhs = builder.create<hlfir::DesignateOp>(
-          loc, designatorType, baseOp, name, compShape,
+      auto lhs = hlfir::DesignateOp::create(
+          builder, loc, designatorType, baseOp, name, compShape,
           hlfir::DesignateOp::Subscripts{},
           /*substring=*/mlir::ValueRange{},
           /*complexPart=*/std::nullopt,
@@ -1997,10 +1997,10 @@ class HlfirBuilder {
         auto rhsCastAndCleanup =
             hlfir::genTypeAndKindConvert(loc, builder, rhs, lhs.getType(),
                                          /*preserveLowerBounds=*/allowRealloc);
-        builder.create<hlfir::AssignOp>(loc, rhsCastAndCleanup.first, lhs,
-                                        allowRealloc,
-                                        allowRealloc ? keepLhsLength : false,
-                                        /*temporary_lhs=*/true);
+        hlfir::AssignOp::create(builder, loc, rhsCastAndCleanup.first, lhs,
+                                allowRealloc,
+                                allowRealloc ? keepLhsLength : false,
+                                /*temporary_lhs=*/true);
         if (rhsCastAndCleanup.second)
           (*rhsCastAndCleanup.second)();
       };

diff  --git a/flang/lib/Lower/ConvertProcedureDesignator.cpp b/flang/lib/Lower/ConvertProcedureDesignator.cpp
index b528544ec245c..d4c535d71cb5f 100644
--- a/flang/lib/Lower/ConvertProcedureDesignator.cpp
+++ b/flang/lib/Lower/ConvertProcedureDesignator.cpp
@@ -49,7 +49,7 @@ fir::ExtendedValue Fortran::lower::convertProcedureDesignator(
         fir::getUnrestrictedIntrinsicSymbolRefAttr(builder, loc, genericName,
                                                    signature);
     mlir::Value funcPtr =
-        builder.create<fir::AddrOfOp>(loc, signature, symbolRefAttr);
+        fir::AddrOfOp::create(builder, loc, signature, symbolRefAttr);
     return funcPtr;
   }
   const Fortran::semantics::Symbol *symbol = proc.GetSymbol();
@@ -69,7 +69,7 @@ fir::ExtendedValue Fortran::lower::convertProcedureDesignator(
         Fortran::lower::getOrDeclareFunction(proc, converter);
     mlir::SymbolRefAttr nameAttr = builder.getSymbolRefAttr(func.getSymName());
     funcPtr =
-        builder.create<fir::AddrOfOp>(loc, func.getFunctionType(), nameAttr);
+        fir::AddrOfOp::create(builder, loc, func.getFunctionType(), nameAttr);
   }
   if (Fortran::lower::mustPassLengthWithDummyProcedure(proc, converter)) {
     // The result length, if available here, must be propagated along the
@@ -114,7 +114,7 @@ static hlfir::EntityWithAttributes designateProcedurePointerComponent(
   /// Passed argument may be a descriptor. This is a scalar reference, so the
   /// base address can be directly addressed.
   if (mlir::isa<fir::BaseBoxType>(base.getType()))
-    base = builder.create<fir::BoxAddrOp>(loc, base);
+    base = fir::BoxAddrOp::create(builder, loc, base);
   std::string fieldName = converter.getRecordTypeFieldName(procComponentSym);
   auto recordType =
       mlir::cast<fir::RecordType>(hlfir::getFortranElementType(base.getType()));
@@ -124,8 +124,8 @@ static hlfir::EntityWithAttributes designateProcedurePointerComponent(
   if (!fieldType)
     TODO(loc, "passing type bound procedure (extension)");
   mlir::Type designatorType = fir::ReferenceType::get(fieldType);
-  mlir::Value compRef = builder.create<hlfir::DesignateOp>(
-      loc, designatorType, base, fieldName,
+  mlir::Value compRef = hlfir::DesignateOp::create(
+      builder, loc, designatorType, base, fieldName,
       /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
       /*substring=*/mlir::ValueRange{},
       /*complexPart=*/std::nullopt,
@@ -174,10 +174,10 @@ hlfir::EntityWithAttributes Fortran::lower::convertProcedureDesignatorToHLFIR(
     mlir::Type boxTy =
         Fortran::lower::getUntypedBoxProcType(&converter.getMLIRContext());
     if (auto host = Fortran::lower::argumentHostAssocs(converter, funcAddr))
-      funcAddr = builder.create<fir::EmboxProcOp>(
-          loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
+      funcAddr = fir::EmboxProcOp::create(
+          builder, loc, boxTy, llvm::ArrayRef<mlir::Value>{funcAddr, host});
     else
-      funcAddr = builder.create<fir::EmboxProcOp>(loc, boxTy, funcAddr);
+      funcAddr = fir::EmboxProcOp::create(builder, loc, boxTy, funcAddr);
   }
 
   mlir::Value res = procExv.match(

diff  --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index cacf4e249aa28..647bd0d079985 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -312,8 +312,8 @@ mlir::Value Fortran::lower::genInitialDataTarget(
   // initial value of the descriptor).
   // Create a fir.rebox to set the attribute correctly, and use targetShift
   // to preserve the target lower bounds if any.
-  return builder.create<fir::ReboxOp>(loc, boxType, targetBox, targetShift,
-                                      /*slice=*/mlir::Value{});
+  return fir::ReboxOp::create(builder, loc, boxType, targetBox, targetShift,
+                              /*slice=*/mlir::Value{});
 }
 
 /// Generate default initial value for a derived type object \p sym with mlir
@@ -362,7 +362,7 @@ static mlir::Value genComponentDefaultInit(
     } else {
       // Component has no initial value. Set its bits to zero by extension
       // to match what is expected because other compilers are doing it.
-      componentValue = builder.create<fir::ZeroOp>(loc, componentTy);
+      componentValue = fir::ZeroOp::create(builder, loc, componentTy);
     }
   } else if (const auto *proc{
                  component
@@ -377,17 +377,17 @@ static mlir::Value genComponentDefaultInit(
         componentValue =
             fir::factory::createNullBoxProc(builder, loc, componentTy);
     } else
-      componentValue = builder.create<fir::ZeroOp>(loc, componentTy);
+      componentValue = fir::ZeroOp::create(builder, loc, componentTy);
   }
   assert(componentValue && "must have been computed");
   componentValue = builder.createConvert(loc, componentTy, componentValue);
   auto fieldTy = fir::FieldType::get(recTy.getContext());
   // FIXME: type parameters must come from the derived-type-spec
-  auto field = builder.create<fir::FieldIndexOp>(
-      loc, fieldTy, name, recTy,
-      /*typeParams=*/mlir::ValueRange{} /*TODO*/);
-  return builder.create<fir::InsertValueOp>(
-      loc, recTy, insertInto, componentValue,
+  auto field =
+      fir::FieldIndexOp::create(builder, loc, fieldTy, name, recTy,
+                                /*typeParams=*/mlir::ValueRange{} /*TODO*/);
+  return fir::InsertValueOp::create(
+      builder, loc, recTy, insertInto, componentValue,
       builder.getArrayAttr(field.getAttributes()));
 }
 
@@ -405,7 +405,7 @@ static mlir::Value genDefaultInitializerValue(
   // Build a scalar default value of the symbol type, looping through the
   // components to build each component initial value.
   auto recTy = mlir::cast<fir::RecordType>(scalarType);
-  mlir::Value initialValue = builder.create<fir::UndefOp>(loc, scalarType);
+  mlir::Value initialValue = fir::UndefOp::create(builder, loc, scalarType);
   const Fortran::semantics::DeclTypeSpec *declTy = sym.GetType();
   assert(declTy && "var with default initialization must have a type");
 
@@ -444,7 +444,7 @@ static mlir::Value genDefaultInitializerValue(
   if (sequenceType) {
     // For arrays, duplicate the scalar value to all elements with an
     // fir.insert_range covering the whole array.
-    auto arrayInitialValue = builder.create<fir::UndefOp>(loc, sequenceType);
+    auto arrayInitialValue = fir::UndefOp::create(builder, loc, sequenceType);
     llvm::SmallVector<int64_t> rangeBounds;
     for (int64_t extent : sequenceType.getShape()) {
       if (extent == fir::SequenceType::getUnknownExtent())
@@ -453,8 +453,8 @@ static mlir::Value genDefaultInitializerValue(
       rangeBounds.push_back(0);
       rangeBounds.push_back(extent - 1);
     }
-    return builder.create<fir::InsertOnRangeOp>(
-        loc, sequenceType, arrayInitialValue, initialValue,
+    return fir::InsertOnRangeOp::create(
+        builder, loc, sequenceType, arrayInitialValue, initialValue,
         builder.getIndexVectorAttr(rangeBounds));
   }
   return initialValue;
@@ -546,7 +546,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
       createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &b) {
         mlir::Value box =
             Fortran::lower::genInitialDataTarget(converter, loc, symTy, expr);
-        b.create<fir::HasValueOp>(loc, box);
+        fir::HasValueOp::create(b, loc, box);
       });
     } else {
       // Create unallocated/disassociated descriptor if no explicit init
@@ -555,7 +555,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
             b, loc, symTy,
             /*nonDeferredParams=*/{},
             /*typeSourceBox=*/{}, getAllocatorIdxFromDataAttr(dataAttr));
-        b.create<fir::HasValueOp>(loc, box);
+        fir::HasValueOp::create(b, loc, box);
       });
     }
   } else if (const auto *details =
@@ -569,7 +569,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
                 converter, loc, details->init().value(), stmtCtx);
             mlir::Value castTo =
                 builder.createConvert(loc, symTy, fir::getBase(initVal));
-            builder.create<fir::HasValueOp>(loc, castTo);
+            fir::HasValueOp::create(builder, loc, castTo);
           });
     } else if (Fortran::lower::hasDefaultInitialization(sym)) {
       createGlobalInitialization(
@@ -579,7 +579,7 @@ fir::GlobalOp Fortran::lower::defineGlobal(
             mlir::Value initVal =
                 genDefaultInitializerValue(converter, loc, sym, symTy, stmtCtx);
             mlir::Value castTo = builder.createConvert(loc, symTy, initVal);
-            builder.create<fir::HasValueOp>(loc, castTo);
+            fir::HasValueOp::create(builder, loc, castTo);
           });
     }
   } else if (Fortran::semantics::IsProcedurePointer(sym)) {
@@ -594,19 +594,19 @@ fir::GlobalOp Fortran::lower::defineGlobal(
               auto box{Fortran::lower::convertProcedureDesignatorInitialTarget(
                   converter, loc, *sym)};
               auto castTo{builder.createConvert(loc, symTy, box)};
-              b.create<fir::HasValueOp>(loc, castTo);
+              fir::HasValueOp::create(b, loc, castTo);
             });
       else { // Has NULL() target.
         createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &b) {
           auto box{fir::factory::createNullBoxProc(b, loc, symTy)};
-          b.create<fir::HasValueOp>(loc, box);
+          fir::HasValueOp::create(b, loc, box);
         });
       }
     } else {
       // No initialization.
       createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &b) {
         auto box{fir::factory::createNullBoxProc(b, loc, symTy)};
-        b.create<fir::HasValueOp>(loc, box);
+        fir::HasValueOp::create(b, loc, box);
       });
     }
   } else if (sym.has<Fortran::semantics::CommonBlockDetails>()) {
@@ -632,10 +632,10 @@ fir::GlobalOp Fortran::lower::defineGlobal(
         builder, global, [&](fir::FirOpBuilder &builder) {
           mlir::Value initValue;
           if (converter.getLoweringOptions().getInitGlobalZero())
-            initValue = builder.create<fir::ZeroOp>(loc, symTy);
+            initValue = fir::ZeroOp::create(builder, loc, symTy);
           else
-            initValue = builder.create<fir::UndefOp>(loc, symTy);
-          builder.create<fir::HasValueOp>(loc, initValue);
+            initValue = fir::UndefOp::create(builder, loc, symTy);
+          fir::HasValueOp::create(builder, loc, initValue);
         });
   }
   // Set public visibility to prevent global definition to be optimized out
@@ -690,8 +690,8 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
                                                         sym);
     global = defineGlobal(converter, var, globalName, linkage, dataAttr);
   }
-  auto addrOf = builder.create<fir::AddrOfOp>(loc, global.resultType(),
-                                              global.getSymbol());
+  auto addrOf = fir::AddrOfOp::create(builder, loc, global.resultType(),
+                                      global.getSymbol());
   // The type of the global cannot be trusted to be the same as the one
   // of the variable as some existing programs map common blocks to
   // BIND(C) module variables (e.g. mpi_argv_null in MPI and MPI_F08).
@@ -752,7 +752,7 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   // would be a waste of space, and incorrect if the pointee is a non dummy
   // assumed-size (possible with cray pointee).
   if (ultimateSymbol.test(Fortran::semantics::Symbol::Flag::CrayPointee))
-    return builder.create<fir::ZeroOp>(loc, fir::ReferenceType::get(ty));
+    return fir::ZeroOp::create(builder, loc, fir::ReferenceType::get(ty));
 
   if (needCUDAAlloc(ultimateSymbol)) {
     cuf::DataAttributeAttr dataAttr =
@@ -767,12 +767,12 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
     for (mlir::Value sh : elidedShape)
       indices.push_back(builder.createConvert(loc, idxTy, sh));
     if (dataAttr.getValue() == cuf::DataAttribute::Shared)
-      return builder.create<cuf::SharedMemoryOp>(loc, ty, nm, symNm, lenParams,
-                                                 indices);
+      return cuf::SharedMemoryOp::create(builder, loc, ty, nm, symNm, lenParams,
+                                         indices);
 
     if (!cuf::isCUDADeviceContext(builder.getRegion()))
-      return builder.create<cuf::AllocOp>(loc, ty, nm, symNm, dataAttr,
-                                          lenParams, indices);
+      return cuf::AllocOp::create(builder, loc, ty, nm, symNm, dataAttr,
+                                  lenParams, indices);
   }
 
   // Let the builder do all the heavy lifting.
@@ -782,7 +782,7 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   // Local procedure pointer.
   auto res{builder.allocateLocal(loc, ty, nm, symNm, shape, lenParams, isTarg)};
   auto box{fir::factory::createNullBoxProc(builder, loc, ty)};
-  builder.create<fir::StoreOp>(loc, box, res);
+  fir::StoreOp::create(builder, loc, box, res);
   return res;
 }
 
@@ -833,9 +833,9 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
             // Field found in the base record type.
             auto fieldName = recTy.getTypeList()[fieldIdx].first;
             fieldTy = recTy.getTypeList()[fieldIdx].second;
-            mlir::Value fieldIndex = builder.create<fir::FieldIndexOp>(
-                loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
-                recTy,
+            mlir::Value fieldIndex = fir::FieldIndexOp::create(
+                builder, loc, fir::FieldType::get(fieldTy.getContext()),
+                fieldName, recTy,
                 /*typeParams=*/mlir::ValueRange{});
             coordinates.push_back(fieldIndex);
           } else {
@@ -846,19 +846,18 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
                       mlir::dyn_cast<fir::RecordType>(component.second)) {
                 fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
                 if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                  mlir::Value parentFieldIndex =
-                      builder.create<fir::FieldIndexOp>(
-                          loc, fir::FieldType::get(childRecTy.getContext()),
-                          component.first, recTy,
-                          /*typeParams=*/mlir::ValueRange{});
+                  mlir::Value parentFieldIndex = fir::FieldIndexOp::create(
+                      builder, loc,
+                      fir::FieldType::get(childRecTy.getContext()),
+                      component.first, recTy,
+                      /*typeParams=*/mlir::ValueRange{});
                   coordinates.push_back(parentFieldIndex);
                   auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
                   fieldTy = childRecTy.getTypeList()[fieldIdx].second;
-                  mlir::Value childFieldIndex =
-                      builder.create<fir::FieldIndexOp>(
-                          loc, fir::FieldType::get(fieldTy.getContext()),
-                          fieldName, childRecTy,
-                          /*typeParams=*/mlir::ValueRange{});
+                  mlir::Value childFieldIndex = fir::FieldIndexOp::create(
+                      builder, loc, fir::FieldType::get(fieldTy.getContext()),
+                      fieldName, childRecTy,
+                      /*typeParams=*/mlir::ValueRange{});
                   coordinates.push_back(childFieldIndex);
                   break;
                 }
@@ -873,25 +872,25 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
           mlir::Value base = fir::getBase(exv);
           mlir::Value comp;
           if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(base.getType()))) {
-            mlir::Value box = builder.create<fir::LoadOp>(loc, base);
-            mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, box);
+            mlir::Value box = fir::LoadOp::create(builder, loc, base);
+            mlir::Value addr = fir::BoxAddrOp::create(builder, loc, box);
             llvm::SmallVector<mlir::Value> lenParams;
             assert(coordinates.size() == 1 && "expect one coordinate");
             auto field = mlir::dyn_cast<fir::FieldIndexOp>(
                 coordinates[0].getDefiningOp());
-            comp = builder.create<hlfir::DesignateOp>(
-                loc, builder.getRefType(fieldTy), addr,
+            comp = hlfir::DesignateOp::create(
+                builder, loc, builder.getRefType(fieldTy), addr,
                 /*component=*/field.getFieldName(),
                 /*componentShape=*/mlir::Value{},
                 hlfir::DesignateOp::Subscripts{});
           } else {
-            comp = builder.create<fir::CoordinateOp>(
-                loc, builder.getRefType(fieldTy), base, coordinates);
+            comp = fir::CoordinateOp::create(
+                builder, loc, builder.getRefType(fieldTy), base, coordinates);
           }
           cuf::DataAttributeAttr dataAttr =
               Fortran::lower::translateSymbolCUFDataAttribute(
                   builder.getContext(), sym);
-          builder.create<cuf::SetAllocatorIndexOp>(loc, comp, dataAttr);
+          cuf::SetAllocatorIndexOp::create(builder, loc, comp, dataAttr);
         }
       }
     }
@@ -933,8 +932,8 @@ void Fortran::lower::defaultInitializeAtRuntime(
     // 15.5.2.12 point 3, absent optional dummies are not initialized.
     // Creating descriptor/passing null descriptor to the runtime would
     // create runtime crashes.
-    auto isPresent = builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
-                                                      fir::getBase(exv));
+    auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
+                                              fir::getBase(exv));
     builder.genIfThen(loc, isPresent)
         .genThen([&]() {
           auto box = builder.createBox(loc, exv);
@@ -976,7 +975,7 @@ void Fortran::lower::defaultInitializeAtRuntime(
                   converter, loc, details->init().value(), stmtCtx);
               mlir::Value castTo =
                   builder.createConvert(loc, symTy, fir::getBase(initVal));
-              builder.create<fir::HasValueOp>(loc, castTo);
+              fir::HasValueOp::create(builder, loc, castTo);
             });
       } else if (!global) {
         global = builder.createGlobal(loc, symTy, globalName, linkage,
@@ -991,13 +990,13 @@ void Fortran::lower::defaultInitializeAtRuntime(
               mlir::Value initVal = genDefaultInitializerValue(
                   converter, loc, sym, symTy, stmtCtx);
               mlir::Value castTo = builder.createConvert(loc, symTy, initVal);
-              builder.create<fir::HasValueOp>(loc, castTo);
+              fir::HasValueOp::create(builder, loc, castTo);
             });
       }
-      auto addrOf = builder.create<fir::AddrOfOp>(loc, global.resultType(),
-                                                  global.getSymbol());
-      builder.create<fir::CopyOp>(loc, addrOf, fir::getBase(exv),
-                                  /*noOverlap=*/true);
+      auto addrOf = fir::AddrOfOp::create(builder, loc, global.resultType(),
+                                          global.getSymbol());
+      fir::CopyOp::create(builder, loc, addrOf, fir::getBase(exv),
+                          /*noOverlap=*/true);
     } else {
       mlir::Value box = builder.createBox(loc, exv);
       fir::runtime::genDerivedTypeInitialize(builder, loc, box);
@@ -1097,8 +1096,8 @@ static void finalizeAtRuntime(Fortran::lower::AbstractConverter &converter,
   fir::ExtendedValue exv = converter.getSymbolExtendedValue(sym, &symMap);
   if (Fortran::semantics::IsOptional(sym)) {
     // Only finalize if present.
-    auto isPresent = builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
-                                                      fir::getBase(exv));
+    auto isPresent = fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
+                                              fir::getBase(exv));
     builder.genIfThen(loc, isPresent)
         .genThen([&]() {
           auto box = builder.createBox(loc, exv);
@@ -1144,8 +1143,8 @@ static void deallocateIntentOut(Fortran::lower::AbstractConverter &converter,
       fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
       if (Fortran::semantics::IsOptional(sym)) {
-        auto isPresent = builder.create<fir::IsPresentOp>(
-            loc, builder.getI1Type(), fir::getBase(extVal));
+        auto isPresent = fir::IsPresentOp::create(
+            builder, loc, builder.getI1Type(), fir::getBase(extVal));
         builder.genIfThen(loc, isPresent)
             .genThen([&]() {
               Fortran::lower::genDeallocateIfAllocated(converter, *mutBox, loc);
@@ -1358,7 +1357,7 @@ static fir::GlobalOp defineGlobalAggregateStore(
               Fortran::lower::StatementContext stmtCtx;
               mlir::Value initVal = fir::getBase(genInitializerExprValue(
                   converter, loc, objectDetails->init().value(), stmtCtx));
-              builder.create<fir::HasValueOp>(loc, initVal);
+              fir::HasValueOp::create(builder, loc, initVal);
             });
         return global;
       }
@@ -1367,8 +1366,8 @@ static fir::GlobalOp defineGlobalAggregateStore(
   // of the linkage.
   createGlobalInitialization(builder, global, [&](fir::FirOpBuilder &builder) {
     Fortran::lower::StatementContext stmtCtx;
-    mlir::Value initVal = builder.create<fir::ZeroOp>(loc, aggTy);
-    builder.create<fir::HasValueOp>(loc, initVal);
+    mlir::Value initVal = fir::ZeroOp::create(builder, loc, aggTy);
+    fir::HasValueOp::create(builder, loc, initVal);
   });
   return global;
 }
@@ -1418,8 +1417,8 @@ instantiateAggregateStore(Fortran::lower::AbstractConverter &converter,
       global =
           defineGlobalAggregateStore(converter, aggregate, aggName, linkage);
     }
-    auto addr = builder.create<fir::AddrOfOp>(loc, global.resultType(),
-                                              global.getSymbol());
+    auto addr = fir::AddrOfOp::create(builder, loc, global.resultType(),
+                                      global.getSymbol());
     auto size = std::get<1>(var.getInterval());
     fir::SequenceType::Shape shape(1, size);
     auto seqTy = fir::SequenceType::get(shape, i8Ty);
@@ -1466,8 +1465,8 @@ static void instantiateAlias(Fortran::lower::AbstractConverter &converter,
   std::size_t off = sym.GetUltimate().offset() - var.getAliasOffset();
   mlir::Value storeAddr = getAggregateStore(storeMap, var);
   mlir::Value offset = builder.createIntegerConstant(loc, idxTy, off);
-  mlir::Value bytePtr = builder.create<fir::CoordinateOp>(
-      loc, i8Ptr, storeAddr, mlir::ValueRange{offset});
+  mlir::Value bytePtr = fir::CoordinateOp::create(
+      builder, loc, i8Ptr, storeAddr, mlir::ValueRange{offset});
   mlir::Value typedPtr = castAliasToPointer(builder, loc, symType, bytePtr);
   Fortran::lower::StatementContext stmtCtx;
   mapSymbolAttributes(converter, var, symMap, stmtCtx, typedPtr);
@@ -1656,7 +1655,7 @@ static void finalizeCommonBlockDefinition(
   mlir::TupleType commonTy = mlir::cast<mlir::TupleType>(global.getType());
   auto initFunc = [&](fir::FirOpBuilder &builder) {
     mlir::IndexType idxTy = builder.getIndexType();
-    mlir::Value cb = builder.create<fir::ZeroOp>(loc, commonTy);
+    mlir::Value cb = fir::ZeroOp::create(builder, loc, commonTy);
     unsigned tupIdx = 0;
     std::size_t offset = 0;
     LLVM_DEBUG(llvm::dbgs() << "block {\n");
@@ -1680,15 +1679,15 @@ static void finalizeCommonBlockDefinition(
           mlir::IntegerAttr offVal = builder.getIntegerAttr(idxTy, tupIdx);
           mlir::Value castVal = builder.createConvert(
               loc, commonTy.getType(tupIdx), fir::getBase(initVal));
-          cb = builder.create<fir::InsertValueOp>(loc, commonTy, cb, castVal,
-                                                  builder.getArrayAttr(offVal));
+          cb = fir::InsertValueOp::create(builder, loc, commonTy, cb, castVal,
+                                          builder.getArrayAttr(offVal));
           ++tupIdx;
           offset = mem->offset() + mem->size();
         }
       }
     }
     LLVM_DEBUG(llvm::dbgs() << "}\n");
-    builder.create<fir::HasValueOp>(loc, cb);
+    fir::HasValueOp::create(builder, loc, cb);
   };
   createGlobalInitialization(builder, global, initFunc);
 }
@@ -1722,8 +1721,8 @@ mlir::Value Fortran::lower::genCommonBlockMember(
 
   mlir::Value offs =
       builder.createIntegerConstant(loc, builder.getIndexType(), byteOffset);
-  mlir::Value varAddr = builder.create<fir::CoordinateOp>(
-      loc, i8Ptr, base, mlir::ValueRange{offs});
+  mlir::Value varAddr = fir::CoordinateOp::create(builder, loc, i8Ptr, base,
+                                                  mlir::ValueRange{offs});
   mlir::Type symType = converter.genType(sym);
 
   return Fortran::semantics::FindEquivalenceSet(sym) != nullptr
@@ -1748,8 +1747,8 @@ static void instantiateCommon(Fortran::lower::AbstractConverter &converter,
   if (!commonAddr) {
     // introduce a local AddrOf and add it to the map
     fir::GlobalOp global = getCommonBlockGlobal(converter, common);
-    commonAddr = builder.create<fir::AddrOfOp>(loc, global.resultType(),
-                                               global.getSymbol());
+    commonAddr = fir::AddrOfOp::create(builder, loc, global.resultType(),
+                                       global.getSymbol());
 
     symMap.addSymbol(common, commonAddr);
   }
@@ -2031,8 +2030,8 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
           Fortran::semantics::GetCUDADataAttr(&sym.GetUltimate()));
 
       // Declare a local pointer variable.
-      auto newBase = builder.create<hlfir::DeclareOp>(
-          loc, boxAlloc, name, /*shape=*/nullptr, lenParams,
+      auto newBase = hlfir::DeclareOp::create(
+          builder, loc, boxAlloc, name, /*shape=*/nullptr, lenParams,
           /*dummy_scope=*/nullptr, attributes);
       mlir::Value nullAddr = builder.createNullConstant(
           loc, llvm::cast<fir::BaseBoxType>(ptrBoxType).getEleTy());
@@ -2047,9 +2046,9 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
       // Inherit the shape (and maybe length parameters) from the pointee
       // declaration.
       mlir::Value initVal =
-          builder.create<fir::EmboxOp>(loc, ptrBoxType, nullAddr, shapeOrShift,
-                                       /*slice=*/nullptr, lenParams);
-      builder.create<fir::StoreOp>(loc, initVal, newBase.getBase());
+          fir::EmboxOp::create(builder, loc, ptrBoxType, nullAddr, shapeOrShift,
+                               /*slice=*/nullptr, lenParams);
+      fir::StoreOp::create(builder, loc, initVal, newBase.getBase());
 
       // Any reference to the pointee is going to be using the pointer
       // box from now on. The base_addr of the descriptor must be updated
@@ -2063,9 +2062,9 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
     mlir::Value dummyScope;
     if (converter.isRegisteredDummySymbol(sym))
       dummyScope = converter.dummyArgsScopeValue();
-    auto newBase = builder.create<hlfir::DeclareOp>(
-        loc, base, name, shapeOrShift, lenParams, dummyScope, attributes,
-        dataAttr);
+    auto newBase =
+        hlfir::DeclareOp::create(builder, loc, base, name, shapeOrShift,
+                                 lenParams, dummyScope, attributes, dataAttr);
     symMap.addVariableDefinition(sym, newBase, force);
     return;
   }
@@ -2214,7 +2213,7 @@ void Fortran::lower::mapSymbolAttributes(
       // Additional discussion below.
       mlir::Type dummyProcType =
           Fortran::lower::getDummyProcedureType(sym, converter);
-      mlir::Value undefOp = builder.create<fir::UndefOp>(loc, dummyProcType);
+      mlir::Value undefOp = fir::UndefOp::create(builder, loc, dummyProcType);
 
       Fortran::lower::genDeclareSymbol(converter, symMap, sym, undefOp);
     }
@@ -2304,32 +2303,32 @@ void Fortran::lower::mapSymbolAttributes(
           mlir::Type lenType = builder.getCharacterLengthType();
           mlir::Value addr, len;
           if (Fortran::semantics::IsOptional(sym)) {
-            auto isPresent = builder.create<fir::IsPresentOp>(
-                loc, builder.getI1Type(), dummyArg);
+            auto isPresent = fir::IsPresentOp::create(
+                builder, loc, builder.getI1Type(), dummyArg);
             auto addrAndLen =
                 builder
                     .genIfOp(loc, {refTy, lenType}, isPresent,
                              /*withElseRegion=*/true)
                     .genThen([&]() {
                       mlir::Value readAddr =
-                          builder.create<fir::BoxAddrOp>(loc, refTy, dummyArg);
+                          fir::BoxAddrOp::create(builder, loc, refTy, dummyArg);
                       mlir::Value readLength =
                           charHelp.readLengthFromBox(dummyArg);
-                      builder.create<fir::ResultOp>(
-                          loc, mlir::ValueRange{readAddr, readLength});
+                      fir::ResultOp::create(
+                          builder, loc, mlir::ValueRange{readAddr, readLength});
                     })
                     .genElse([&] {
                       mlir::Value readAddr = builder.genAbsentOp(loc, refTy);
                       mlir::Value readLength =
                           fir::factory::createZeroValue(builder, loc, lenType);
-                      builder.create<fir::ResultOp>(
-                          loc, mlir::ValueRange{readAddr, readLength});
+                      fir::ResultOp::create(
+                          builder, loc, mlir::ValueRange{readAddr, readLength});
                     })
                     .getResults();
             addr = addrAndLen[0];
             len = addrAndLen[1];
           } else {
-            addr = builder.create<fir::BoxAddrOp>(loc, refTy, dummyArg);
+            addr = fir::BoxAddrOp::create(builder, loc, refTy, dummyArg);
             len = charHelp.readLengthFromBox(dummyArg);
           }
           if (!explicitParams.empty())
@@ -2428,7 +2427,7 @@ void Fortran::lower::mapSymbolAttributes(
         mlir::Value dim =
             builder.createIntegerConstant(loc, idxTy, iter.index());
         auto dimInfo =
-            builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, dim);
+            fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, dim);
         shapes.emplace_back(dimInfo.getResult(1));
       } else if (spec->ubound().isStar()) {
         shapes.emplace_back(getAssumedSizeExtent(loc, builder));
@@ -2452,7 +2451,7 @@ void Fortran::lower::mapSymbolAttributes(
         mlir::Value dim =
             builder.createIntegerConstant(loc, idxTy, iter.index());
         dimInfo =
-            builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, dim);
+            fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, dim);
         extents.emplace_back(dimInfo.getResult(1));
         if (auto low = spec->lbound().GetExplicit()) {
           auto expr = Fortran::lower::SomeExpr{*low};
@@ -2501,7 +2500,7 @@ void Fortran::lower::mapSymbolAttributes(
     if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(arg.getType())) {
       // Contiguous assumed shape that can be tracked without a fir.box.
       mlir::Type refTy = builder.getRefType(boxTy.getEleTy());
-      addr = builder.create<fir::BoxAddrOp>(loc, refTy, arg);
+      addr = fir::BoxAddrOp::create(builder, loc, refTy, arg);
     }
 
   // Compute/Extract character length.
@@ -2512,8 +2511,8 @@ void Fortran::lower::mapSymbolAttributes(
         std::tie(addr, len) = charHelp.createUnboxChar(arg);
       } else if (mlir::isa<fir::CharacterType>(arg.getType())) {
         // fir.char<1> passed by value (BIND(C) with VALUE attribute).
-        addr = builder.create<fir::AllocaOp>(loc, arg.getType());
-        builder.create<fir::StoreOp>(loc, arg, addr);
+        addr = fir::AllocaOp::create(builder, loc, arg.getType());
+        fir::StoreOp::create(builder, loc, arg, addr);
       } else if (!addr) {
         addr = arg;
       }
@@ -2583,7 +2582,7 @@ void Fortran::lower::mapSymbolAttributes(
         // Dummy argument passed in register. Place the value in memory at that
         // point since lowering expect symbols to be mapped to memory addresses.
         mlir::Type symType = converter.genType(sym);
-        addr = builder.create<fir::AllocaOp>(loc, symType);
+        addr = fir::AllocaOp::create(builder, loc, symType);
         if (isCptrByVal) {
           // Place the void* address into the CPTR address component.
           mlir::Value addrComponent =
@@ -2803,8 +2802,8 @@ Fortran::lower::genPackArray(Fortran::lower::AbstractConverter &converter,
   mlir::Type elementType = boxType.unwrapInnerType();
   llvm::SmallVector<mlir::Value> elidedLenParams =
       fir::factory::elideLengthsAlreadyInType(elementType, lenParams);
-  auto packOp = builder.create<fir::PackArrayOp>(
-      loc, fir::getBase(exv), stackAlloc, isInnermostMode, noCopy,
+  auto packOp = fir::PackArrayOp::create(
+      builder, loc, fir::getBase(exv), stackAlloc, isInnermostMode, noCopy,
       /*max_size=*/mlir::IntegerAttr{},
       /*max_element_size=*/mlir::IntegerAttr{},
       /*min_stride=*/mlir::IntegerAttr{}, fir::PackArrayHeuristics::None,
@@ -2842,6 +2841,6 @@ void Fortran::lower::genUnpackArray(
   // Avoid copy-out for 'intent(in)' variables.
   bool noCopy = Fortran::semantics::IsIntentIn(sym);
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  builder.create<fir::UnpackArrayOp>(loc, temp, original, stackAlloc, noCopy,
-                                     getSafeRepackAttrs(converter));
+  fir::UnpackArrayOp::create(builder, loc, temp, original, stackAlloc, noCopy,
+                             getSafeRepackAttrs(converter));
 }

diff  --git a/flang/lib/Lower/CustomIntrinsicCall.cpp b/flang/lib/Lower/CustomIntrinsicCall.cpp
index 30c6ce7f53b3f..2c5233bdd15ee 100644
--- a/flang/lib/Lower/CustomIntrinsicCall.cpp
+++ b/flang/lib/Lower/CustomIntrinsicCall.cpp
@@ -101,7 +101,7 @@ Fortran::lower::genIntrinsicCall(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value addr = fir::getBase(result);
     if (auto *box = result.getBoxOf<fir::BoxValue>())
       addr =
-          builder.create<fir::BoxAddrOp>(loc, box->getMemTy(), box->getAddr());
+          fir::BoxAddrOp::create(builder, loc, box->getMemTy(), box->getAddr());
     fir::FirOpBuilder *bldr = &builder;
     stmtCtx.attachCleanup([=]() { bldr->create<fir::FreeMemOp>(loc, addr); });
   }
@@ -171,9 +171,9 @@ lowerMinOrMax(fir::FirOpBuilder &builder, mlir::Location loc,
                 args.emplace_back(getOperand(opIndex, loadOperand));
                 fir::ExtendedValue newExtremum = genIntrinsicCall(
                     builder, loc, name, resultType, args, stmtCtx);
-                builder.create<fir::ResultOp>(loc, fir::getBase(newExtremum));
+                fir::ResultOp::create(builder, loc, fir::getBase(newExtremum));
               })
-              .genElse([&]() { builder.create<fir::ResultOp>(loc, extremum); })
+              .genElse([&]() { fir::ResultOp::create(builder, loc, extremum); })
               .getResults()[0];
     } else {
       // Argument is know to be present at compile time.
@@ -235,13 +235,13 @@ lowerIshftc(fir::FirOpBuilder &builder, mlir::Location loc,
             fir::ExtendedValue sizeExv = getOperand(2, loadOperand);
             mlir::Value size =
                 builder.createConvert(loc, resultType, fir::getBase(sizeExv));
-            builder.create<fir::ResultOp>(loc, size);
+            fir::ResultOp::create(builder, loc, size);
           })
           .genElse([&]() {
             mlir::Value bitSize = builder.createIntegerConstant(
                 loc, resultType,
                 mlir::cast<mlir::IntegerType>(resultType).getWidth());
-            builder.create<fir::ResultOp>(loc, bitSize);
+            fir::ResultOp::create(builder, loc, bitSize);
           })
           .getResults()[0]);
   return genIntrinsicCall(builder, loc, name, resultType, args, stmtCtx);
@@ -280,7 +280,7 @@ lowerAssociated(fir::FirOpBuilder &builder, mlir::Location loc,
   // while the optionality of the target pointer/allocatable is what must be
   // checked here.
   mlir::Value isPresent =
-      builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), targetBase);
+      fir::IsPresentOp::create(builder, loc, builder.getI1Type(), targetBase);
   mlir::Type targetType = fir::unwrapRefType(targetBase.getType());
   mlir::Type targetValueType = fir::unwrapPassByRefType(targetType);
   mlir::Type boxType = mlir::isa<fir::BaseBoxType>(targetType)
@@ -293,11 +293,12 @@ lowerAssociated(fir::FirOpBuilder &builder, mlir::Location loc,
           .genThen([&]() {
             mlir::Value box = builder.createBox(loc, targetExv);
             mlir::Value cast = builder.createConvert(loc, boxType, box);
-            builder.create<fir::ResultOp>(loc, cast);
+            fir::ResultOp::create(builder, loc, cast);
           })
           .genElse([&]() {
-            mlir::Value absentBox = builder.create<fir::AbsentOp>(loc, boxType);
-            builder.create<fir::ResultOp>(loc, absentBox);
+            mlir::Value absentBox =
+                fir::AbsentOp::create(builder, loc, boxType);
+            fir::ResultOp::create(builder, loc, absentBox);
           })
           .getResults()[0];
   args.emplace_back(std::move(targetBox));

diff  --git a/flang/lib/Lower/HlfirIntrinsics.cpp b/flang/lib/Lower/HlfirIntrinsics.cpp
index 8b96b209ddb00..6e1d06a25924b 100644
--- a/flang/lib/Lower/HlfirIntrinsics.cpp
+++ b/flang/lib/Lower/HlfirIntrinsics.cpp
@@ -63,7 +63,7 @@ class HlfirTransformationalIntrinsic {
 
   template <typename OP, typename... BUILD_ARGS>
   inline OP createOp(BUILD_ARGS... args) {
-    return builder.create<OP>(loc, args...);
+    return OP::create(builder, loc, args...);
   }
 
   mlir::Value loadBoxAddress(
@@ -195,7 +195,7 @@ mlir::Value HlfirTransformationalIntrinsic::loadBoxAddress(
       // this is a box address type but is not dynamically optional. Just load
       // the box, assuming it is well formed (!fir.ref<!fir.box<...>> ->
       // !fir.box<...>)
-      return builder.create<fir::LoadOp>(loc, actual.getBase());
+      return fir::LoadOp::create(builder, loc, actual.getBase());
     }
     return actual;
   }
@@ -209,9 +209,9 @@ mlir::Value HlfirTransformationalIntrinsic::loadBoxAddress(
   // ensures it won't be.
   mlir::Value box = builder.createBox(loc, exv);
   mlir::Type boxType = box.getType();
-  auto absent = builder.create<fir::AbsentOp>(loc, boxType);
-  auto boxOrAbsent = builder.create<mlir::arith::SelectOp>(
-      loc, boxType, isPresent, box, absent);
+  auto absent = fir::AbsentOp::create(builder, loc, boxType);
+  auto boxOrAbsent = mlir::arith::SelectOp::create(builder, loc, boxType,
+                                                   isPresent, box, absent);
 
   return boxOrAbsent;
 }
@@ -232,11 +232,11 @@ static mlir::Value loadOptionalValue(
         assert(actual.isScalar() && fir::isa_trivial(eleType) &&
                "must be a numerical or logical scalar");
         hlfir::Entity val = hlfir::loadTrivialScalar(loc, builder, actual);
-        builder.create<fir::ResultOp>(loc, val);
+        fir::ResultOp::create(builder, loc, val);
       })
       .genElse([&]() {
         mlir::Value zero = fir::factory::createZeroValue(builder, loc, eleType);
-        builder.create<fir::ResultOp>(loc, zero);
+        fir::ResultOp::create(builder, loc, zero);
       })
       .getResults()[0];
 }

diff  --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp
index 95ea74b791b47..2a330ccc4eebb 100644
--- a/flang/lib/Lower/HostAssociations.cpp
+++ b/flang/lib/Lower/HostAssociations.cpp
@@ -165,7 +165,7 @@ class CapturedSimpleScalars : public CapturedSymbols<CapturedSimpleScalars> {
     assert(typeInTuple && "addrInTuple must be an address");
     mlir::Value castBox = builder.createConvertWithVolatileCast(
         args.loc, typeInTuple, fir::getBase(args.hostValue));
-    builder.create<fir::StoreOp>(args.loc, castBox, args.addrInTuple);
+    fir::StoreOp::create(builder, args.loc, castBox, args.addrInTuple);
   }
 
   static void getFromTuple(const GetFromTuple &args,
@@ -196,7 +196,7 @@ class CapturedProcedure : public CapturedSymbols<CapturedProcedure> {
     assert(typeInTuple && "addrInTuple must be an address");
     mlir::Value castBox = builder.createConvertWithVolatileCast(
         args.loc, typeInTuple, fir::getBase(args.hostValue));
-    builder.create<fir::StoreOp>(args.loc, castBox, args.addrInTuple);
+    fir::StoreOp::create(builder, args.loc, castBox, args.addrInTuple);
   }
 
   static void getFromTuple(const GetFromTuple &args,
@@ -231,7 +231,7 @@ class CapturedCharacterScalars
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     mlir::Value boxchar = fir::factory::CharacterExprHelper(builder, args.loc)
                               .createEmbox(*charBox);
-    builder.create<fir::StoreOp>(args.loc, boxchar, args.addrInTuple);
+    fir::StoreOp::create(builder, args.loc, boxchar, args.addrInTuple);
   }
 
   static void getFromTuple(const GetFromTuple &args,
@@ -269,20 +269,20 @@ class CapturedPolymorphicScalar
         args.loc, typeInTuple, fir::getBase(args.hostValue));
     if (Fortran::semantics::IsOptional(sym)) {
       auto isPresent =
-          builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), castBox);
+          fir::IsPresentOp::create(builder, loc, builder.getI1Type(), castBox);
       builder.genIfThenElse(loc, isPresent)
           .genThen([&]() {
-            builder.create<fir::StoreOp>(loc, castBox, args.addrInTuple);
+            fir::StoreOp::create(builder, loc, castBox, args.addrInTuple);
           })
           .genElse([&]() {
             mlir::Value null = fir::factory::createUnallocatedBox(
                 builder, loc, typeInTuple,
                 /*nonDeferredParams=*/mlir::ValueRange{});
-            builder.create<fir::StoreOp>(loc, null, args.addrInTuple);
+            fir::StoreOp::create(builder, loc, null, args.addrInTuple);
           })
           .end();
     } else {
-      builder.create<fir::StoreOp>(loc, castBox, args.addrInTuple);
+      fir::StoreOp::create(builder, loc, castBox, args.addrInTuple);
     }
   }
   static void getFromTuple(const GetFromTuple &args,
@@ -297,11 +297,11 @@ class CapturedPolymorphicScalar
       auto eleTy = boxTy.getEleTy();
       if (!fir::isa_ref_type(eleTy))
         eleTy = builder.getRefType(eleTy);
-      auto addr = builder.create<fir::BoxAddrOp>(loc, eleTy, box);
+      auto addr = fir::BoxAddrOp::create(builder, loc, eleTy, box);
       mlir::Value isPresent = builder.genIsNotNullAddr(loc, addr);
-      auto absentBox = builder.create<fir::AbsentOp>(loc, boxTy);
-      box =
-          builder.create<mlir::arith::SelectOp>(loc, isPresent, box, absentBox);
+      auto absentBox = fir::AbsentOp::create(builder, loc, boxTy);
+      box = mlir::arith::SelectOp::create(builder, loc, isPresent, box,
+                                          absentBox);
     }
     bindCapturedSymbol(sym, box, converter, args.symMap);
   }
@@ -331,7 +331,7 @@ class CapturedAllocatableAndPointer
     assert(typeInTuple && "addrInTuple must be an address");
     mlir::Value castBox = builder.createConvertWithVolatileCast(
         args.loc, typeInTuple, fir::getBase(args.hostValue));
-    builder.create<fir::StoreOp>(args.loc, castBox, args.addrInTuple);
+    fir::StoreOp::create(builder, args.loc, castBox, args.addrInTuple);
   }
   static void getFromTuple(const GetFromTuple &args,
                            Fortran::lower::AbstractConverter &converter,
@@ -404,8 +404,8 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
       // done on present optional. For absent optionals, simply create a
       // disassociated pointer (it is illegal to inquire about lower bounds or
       // lengths of optional according to 15.5.2.12 3 (9) and 10.1.11 2 (7)b).
-      auto isPresent = builder.create<fir::IsPresentOp>(
-          loc, builder.getI1Type(), fir::getBase(args.hostValue));
+      auto isPresent = fir::IsPresentOp::create(
+          builder, loc, builder.getI1Type(), fir::getBase(args.hostValue));
       builder.genIfThenElse(loc, isPresent)
           .genThen([&]() {
             fir::factory::associateMutableBox(builder, loc, boxInTuple,
@@ -441,8 +441,8 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
         const unsigned rank = sym.Rank();
         for (unsigned dim = 0; dim < rank; ++dim) {
           mlir::Value dimVal = builder.createIntegerConstant(loc, idxTy, dim);
-          auto dims = builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy,
-                                                     box, dimVal);
+          auto dims = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
+                                             box, dimVal);
           lbounds.emplace_back(dims.getResult(0));
         }
       }
@@ -464,11 +464,11 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
         auto eleTy = boxTy.getEleTy();
         if (!fir::isa_ref_type(eleTy))
           eleTy = builder.getRefType(eleTy);
-        auto addr = builder.create<fir::BoxAddrOp>(loc, eleTy, box);
+        auto addr = fir::BoxAddrOp::create(builder, loc, eleTy, box);
         mlir::Value isPresent = builder.genIsNotNullAddr(loc, addr);
-        auto absentBox = builder.create<fir::AbsentOp>(loc, boxTy);
-        box = builder.create<mlir::arith::SelectOp>(loc, isPresent, box,
-                                                    absentBox);
+        auto absentBox = fir::AbsentOp::create(builder, loc, boxTy);
+        box = mlir::arith::SelectOp::create(builder, loc, isPresent, box,
+                                            absentBox);
       }
       fir::BoxValue boxValue(box, lbounds, /*explicitParams=*/{});
       bindCapturedSymbol(sym, boxValue, converter, args.symMap);
@@ -540,7 +540,7 @@ static mlir::Value genTupleCoor(fir::FirOpBuilder &builder, mlir::Location loc,
   auto ty = mlir::isa<fir::ReferenceType>(varTy)
                 ? mlir::Type(fir::LLVMPointerType::get(varTy))
                 : mlir::Type(builder.getRefType(varTy));
-  return builder.create<fir::CoordinateOp>(loc, ty, tupleArg, offset);
+  return fir::CoordinateOp::create(builder, loc, ty, tupleArg, offset);
 }
 
 void Fortran::lower::HostAssociations::addSymbolsToBind(
@@ -572,7 +572,7 @@ void Fortran::lower::HostAssociations::hostProcedureBindings(
   mlir::TupleType tupTy = unwrapTupleTy(getArgumentType(converter));
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   mlir::Location loc = converter.getCurrentLocation();
-  auto hostTuple = builder.create<fir::AllocaOp>(loc, tupTy);
+  auto hostTuple = fir::AllocaOp::create(builder, loc, tupTy);
   mlir::IntegerType offTy = builder.getIntegerType(32);
 
   // Walk the list of tupleSymbols and update the pointers in the tuple.
@@ -639,7 +639,7 @@ void Fortran::lower::HostAssociations::internalProcedureBindings(
     mlir::Value off = builder.createIntegerConstant(loc, offTy, s.index());
     mlir::Type varTy = tupTy.getType(s.index());
     mlir::Value eleOff = genTupleCoor(builder, loc, varTy, tupleArg, off);
-    mlir::Value valueInTuple = builder.create<fir::LoadOp>(loc, eleOff);
+    mlir::Value valueInTuple = fir::LoadOp::create(builder, loc, eleOff);
     GetFromTuple getFromTuple{symMap, valueInTuple, loc};
     walkCaptureCategories(getFromTuple, converter, *s.value());
   }

diff  --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 53bf61922392d..c95c3404a8e26 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -153,8 +153,8 @@ static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
   if (csi.ioMsg) {
     mlir::func::FuncOp getIoMsg =
         fir::runtime::getIORuntimeFunc<mkIOKey(GetIoMsg)>(loc, builder);
-    builder.create<fir::CallOp>(
-        loc, getIoMsg,
+    fir::CallOp::create(
+        builder, loc, getIoMsg,
         mlir::ValueRange{
             cookie,
             builder.createConvert(loc, getIoMsg.getFunctionType().getInput(1),
@@ -164,12 +164,12 @@ static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
   }
   mlir::func::FuncOp endIoStatement =
       fir::runtime::getIORuntimeFunc<mkIOKey(EndIoStatement)>(loc, builder);
-  auto call = builder.create<fir::CallOp>(loc, endIoStatement,
-                                          mlir::ValueRange{cookie});
+  auto call = fir::CallOp::create(builder, loc, endIoStatement,
+                                  mlir::ValueRange{cookie});
   mlir::Value iostat = call.getResult(0);
   if (csi.bigUnitIfOp) {
     stmtCtx.finalizeAndPop();
-    builder.create<fir::ResultOp>(loc, iostat);
+    fir::ResultOp::create(builder, loc, iostat);
     builder.setInsertionPointAfter(csi.bigUnitIfOp);
     iostat = csi.bigUnitIfOp.getResult(0);
   }
@@ -178,7 +178,7 @@ static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
         fir::getBase(converter.genExprAddr(loc, csi.ioStatExpr, stmtCtx));
     mlir::Value ioStatResult =
         builder.createConvert(loc, converter.genType(*csi.ioStatExpr), iostat);
-    builder.create<fir::StoreOp>(loc, ioStatResult, ioStatVar);
+    fir::StoreOp::create(builder, loc, ioStatResult, ioStatVar);
   }
   return csi.hasTransferConditionSpec() ? iostat : mlir::Value{};
 }
@@ -203,8 +203,8 @@ static void makeNextConditionalOn(fir::FirOpBuilder &builder,
   mlir::IntegerType boolTy = builder.getI1Type();
   if (inLoop)
     resTy = boolTy;
-  auto ifOp = builder.create<fir::IfOp>(loc, resTy, ok,
-                                        /*withElseRegion=*/inLoop);
+  auto ifOp = fir::IfOp::create(builder, loc, resTy, ok,
+                                /*withElseRegion=*/inLoop);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
 }
 
@@ -259,10 +259,10 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
           ? fir::NameUniquer::doGenerated("default" + suffix)
           : converter.mangleName(suffix);
   if (auto table = builder.getNamedGlobal(tableMangleName))
-    return builder.createConvert(
-        loc, refTy,
-        builder.create<fir::AddrOfOp>(loc, table.resultType(),
-                                      table.getSymbol()));
+    return builder.createConvert(loc, refTy,
+                                 fir::AddrOfOp::create(builder, loc,
+                                                       table.resultType(),
+                                                       table.getSymbol()));
 
   mlir::StringAttr linkOnce = builder.createLinkOnceLinkage();
   mlir::Type idxTy = builder.getIndexType();
@@ -281,11 +281,12 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
   // Define the list of NonTbpDefinedIo procedures.
   bool tableIsLocal =
       !definedIoProcMap.empty() && hasLocalDefinedIoProc(definedIoProcMap);
-  mlir::Value listAddr =
-      tableIsLocal ? builder.create<fir::AllocaOp>(loc, listTy) : mlir::Value{};
+  mlir::Value listAddr = tableIsLocal
+                             ? fir::AllocaOp::create(builder, loc, listTy)
+                             : mlir::Value{};
   std::string listMangleName = tableMangleName + ".list";
   auto listFunc = [&](fir::FirOpBuilder &builder) {
-    mlir::Value list = builder.create<fir::UndefOp>(loc, listTy);
+    mlir::Value list = fir::UndefOp::create(builder, loc, listTy);
     mlir::IntegerAttr intAttr[4];
     for (int i = 0; i < 4; ++i)
       intAttr[i] = builder.getIntegerAttr(idxTy, i);
@@ -294,8 +295,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
     int n0 = 0, n1;
     auto insert = [&](mlir::Value val) {
       idx[1] = intAttr[n1++];
-      list = builder.create<fir::InsertValueOp>(loc, listTy, list, val,
-                                                builder.getArrayAttr(idx));
+      list = fir::InsertValueOp::create(builder, loc, listTy, list, val,
+                                        builder.getArrayAttr(idx));
     };
     for (auto &iface : definedIoProcMap) {
       idx[0] = builder.getIntegerAttr(idxTy, n0++);
@@ -305,8 +306,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       std::string dtName = converter.mangleName(dtSym);
       insert(builder.createConvert(
           loc, refTy,
-          builder.create<fir::AddrOfOp>(
-              loc, fir::ReferenceType::get(converter.genType(dtSym)),
+          fir::AddrOfOp::create(
+              builder, loc, fir::ReferenceType::get(converter.genType(dtSym)),
               builder.getSymbolRefAttr(dtName))));
       // defined IO procedure [void (*subroutine)()], may be null
       const Fortran::semantics::Symbol *procSym = iface.second.subroutine;
@@ -316,8 +317,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
           TODO(loc, "defined IO procedure pointers");
         } else if (Fortran::semantics::IsDummy(*procSym)) {
           Fortran::lower::StatementContext stmtCtx;
-          insert(builder.create<fir::BoxAddrOp>(
-              loc, refTy,
+          insert(fir::BoxAddrOp::create(
+              builder, loc, refTy,
               fir::getBase(converter.genExprAddr(
                   loc,
                   Fortran::lower::SomeExpr{
@@ -330,8 +331,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
               builder.getSymbolRefAttr(procDef.getSymName());
           insert(builder.createConvert(
               loc, refTy,
-              builder.create<fir::AddrOfOp>(loc, procDef.getFunctionType(),
-                                            nameAttr)));
+              fir::AddrOfOp::create(builder, loc, procDef.getFunctionType(),
+                                    nameAttr)));
         }
       } else {
         insert(builder.createNullConstant(loc, refTy));
@@ -346,9 +347,9 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
       insert(builder.createIntegerConstant(loc, byteTy, iface.second.flags));
     }
     if (tableIsLocal)
-      builder.create<fir::StoreOp>(loc, list, listAddr);
+      fir::StoreOp::create(builder, loc, list, listAddr);
     else
-      builder.create<fir::HasValueOp>(loc, list);
+      fir::HasValueOp::create(builder, loc, list);
   };
   if (!definedIoProcMap.empty()) {
     if (tableIsLocal)
@@ -360,33 +361,34 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
 
   // Define the NonTbpDefinedIoTable.
   mlir::Value tableAddr = tableIsLocal
-                              ? builder.create<fir::AllocaOp>(loc, tableTy)
+                              ? fir::AllocaOp::create(builder, loc, tableTy)
                               : mlir::Value{};
   auto tableFunc = [&](fir::FirOpBuilder &builder) {
-    mlir::Value table = builder.create<fir::UndefOp>(loc, tableTy);
+    mlir::Value table = fir::UndefOp::create(builder, loc, tableTy);
     // list item count [std::size_t items]
-    table = builder.create<fir::InsertValueOp>(
-        loc, tableTy, table,
+    table = fir::InsertValueOp::create(
+        builder, loc, tableTy, table,
         builder.createIntegerConstant(loc, sizeTy, definedIoProcMap.size()),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 0)));
     // item list [const NonTbpDefinedIo *item]
     if (definedIoProcMap.empty())
       listAddr = builder.createNullConstant(loc, builder.getRefType(listTy));
     else if (fir::GlobalOp list = builder.getNamedGlobal(listMangleName))
-      listAddr = builder.create<fir::AddrOfOp>(loc, list.resultType(),
-                                               list.getSymbol());
+      listAddr = fir::AddrOfOp::create(builder, loc, list.resultType(),
+                                       list.getSymbol());
     assert(listAddr && "missing namelist object list");
-    table = builder.create<fir::InsertValueOp>(
-        loc, tableTy, table, listAddr,
+    table = fir::InsertValueOp::create(
+        builder, loc, tableTy, table, listAddr,
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 1)));
     // [bool ignoreNonTbpEntries] conservatively set to true
-    table = builder.create<fir::InsertValueOp>(
-        loc, tableTy, table, builder.createIntegerConstant(loc, boolTy, true),
+    table = fir::InsertValueOp::create(
+        builder, loc, tableTy, table,
+        builder.createIntegerConstant(loc, boolTy, true),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 2)));
     if (tableIsLocal)
-      builder.create<fir::StoreOp>(loc, table, tableAddr);
+      fir::StoreOp::create(builder, loc, table, tableAddr);
     else
-      builder.create<fir::HasValueOp>(loc, table);
+      fir::HasValueOp::create(builder, loc, table);
   };
   if (tableIsLocal) {
     tableFunc(builder);
@@ -394,8 +396,8 @@ getNonTbpDefinedIoTableAddr(Fortran::lower::AbstractConverter &converter,
     fir::GlobalOp table = builder.createGlobal(
         loc, tableTy, tableMangleName,
         /*isConst=*/true, /*isTarget=*/false, tableFunc, linkOnce);
-    tableAddr = builder.create<fir::AddrOfOp>(
-        loc, fir::ReferenceType::get(tableTy), table.getSymbol());
+    tableAddr = fir::AddrOfOp::create(
+        builder, loc, fir::ReferenceType::get(tableTy), table.getSymbol());
   }
   assert(tableAddr && "missing NonTbpDefinedIo table result");
   return builder.createConvert(loc, refTy, tableAddr);
@@ -420,8 +422,8 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
   mlir::Location loc = converter.getCurrentLocation();
   std::string groupMangleName = converter.mangleName(symbol);
   if (auto group = builder.getNamedGlobal(groupMangleName))
-    return builder.create<fir::AddrOfOp>(loc, group.resultType(),
-                                         group.getSymbol());
+    return fir::AddrOfOp::create(builder, loc, group.resultType(),
+                                 group.getSymbol());
 
   const auto &details =
       symbol.GetUltimate().get<Fortran::semantics::NamelistDetails>();
@@ -468,18 +470,19 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       auto descFunc = [&](fir::FirOpBuilder &b) {
         auto box = Fortran::lower::genInitialDataTarget(
             converter, loc, boxTy, *expr, /*couldBeInEquivalence=*/true);
-        b.create<fir::HasValueOp>(loc, box);
+        fir::HasValueOp::create(b, loc, box);
       };
       builder.createGlobalConstant(loc, boxTy, mangleName, descFunc, linkOnce);
     }
   }
 
   // Define the list of Items.
-  mlir::Value listAddr =
-      groupIsLocal ? builder.create<fir::AllocaOp>(loc, listTy) : mlir::Value{};
+  mlir::Value listAddr = groupIsLocal
+                             ? fir::AllocaOp::create(builder, loc, listTy)
+                             : mlir::Value{};
   std::string listMangleName = groupMangleName + ".list";
   auto listFunc = [&](fir::FirOpBuilder &builder) {
-    mlir::Value list = builder.create<fir::UndefOp>(loc, listTy);
+    mlir::Value list = fir::UndefOp::create(builder, loc, listTy);
     mlir::IntegerAttr zero = builder.getIntegerAttr(idxTy, 0);
     mlir::IntegerAttr one = builder.getIntegerAttr(idxTy, 1);
     llvm::SmallVector<mlir::Attribute, 2> idx = {mlir::Attribute{},
@@ -490,14 +493,14 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       idx[1] = zero;
       mlir::Value nameAddr =
           builder.createConvert(loc, charRefTy, fir::getBase(stringAddress(s)));
-      list = builder.create<fir::InsertValueOp>(loc, listTy, list, nameAddr,
-                                                builder.getArrayAttr(idx));
+      list = fir::InsertValueOp::create(builder, loc, listTy, list, nameAddr,
+                                        builder.getArrayAttr(idx));
       idx[1] = one;
       mlir::Value descAddr;
       if (auto desc = builder.getNamedGlobal(
               Fortran::lower::mangle::globalNamelistDescriptorName(s))) {
-        descAddr = builder.create<fir::AddrOfOp>(loc, desc.resultType(),
-                                                 desc.getSymbol());
+        descAddr = fir::AddrOfOp::create(builder, loc, desc.resultType(),
+                                         desc.getSymbol());
       } else if (Fortran::semantics::FindCommonBlockContaining(s) &&
                  IsAllocatableOrPointer(s)) {
         mlir::Type symType = converter.genType(s);
@@ -505,8 +508,8 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
             Fortran::semantics::FindCommonBlockContaining(s);
         std::string commonBlockName = converter.mangleName(*commonBlockSym);
         fir::GlobalOp commonGlobal = builder.getNamedGlobal(commonBlockName);
-        mlir::Value commonBlockAddr = builder.create<fir::AddrOfOp>(
-            loc, commonGlobal.resultType(), commonGlobal.getSymbol());
+        mlir::Value commonBlockAddr = fir::AddrOfOp::create(
+            builder, loc, commonGlobal.resultType(), commonGlobal.getSymbol());
         mlir::IntegerType i8Ty = builder.getIntegerType(8);
         mlir::Type i8Ptr = builder.getRefType(i8Ty);
         mlir::Type seqTy = builder.getRefType(builder.getVarLenSeqTy(i8Ty));
@@ -514,8 +517,8 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
         std::size_t byteOffset = s.GetUltimate().offset();
         mlir::Value offs = builder.createIntegerConstant(
             loc, builder.getIndexType(), byteOffset);
-        mlir::Value varAddr = builder.create<fir::CoordinateOp>(
-            loc, i8Ptr, base, mlir::ValueRange{offs});
+        mlir::Value varAddr = fir::CoordinateOp::create(
+            builder, loc, i8Ptr, base, mlir::ValueRange{offs});
         descAddr =
             builder.createConvert(loc, builder.getRefType(symType), varAddr);
       } else {
@@ -531,13 +534,13 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
                                           /*lbounds=*/{});
       }
       descAddr = builder.createConvert(loc, descRefTy, descAddr);
-      list = builder.create<fir::InsertValueOp>(loc, listTy, list, descAddr,
-                                                builder.getArrayAttr(idx));
+      list = fir::InsertValueOp::create(builder, loc, listTy, list, descAddr,
+                                        builder.getArrayAttr(idx));
     }
     if (groupIsLocal)
-      builder.create<fir::StoreOp>(loc, list, listAddr);
+      fir::StoreOp::create(builder, loc, list, listAddr);
     else
-      builder.create<fir::HasValueOp>(loc, list);
+      fir::HasValueOp::create(builder, loc, list);
   };
   if (groupIsLocal)
     listFunc(builder);
@@ -547,39 +550,39 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
 
   // Define the group.
   mlir::Value groupAddr = groupIsLocal
-                              ? builder.create<fir::AllocaOp>(loc, groupTy)
+                              ? fir::AllocaOp::create(builder, loc, groupTy)
                               : mlir::Value{};
   auto groupFunc = [&](fir::FirOpBuilder &builder) {
-    mlir::Value group = builder.create<fir::UndefOp>(loc, groupTy);
+    mlir::Value group = fir::UndefOp::create(builder, loc, groupTy);
     // group name [const char *groupName]
-    group = builder.create<fir::InsertValueOp>(
-        loc, groupTy, group,
+    group = fir::InsertValueOp::create(
+        builder, loc, groupTy, group,
         builder.createConvert(loc, charRefTy,
                               fir::getBase(stringAddress(symbol))),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 0)));
     // list item count [std::size_t items]
-    group = builder.create<fir::InsertValueOp>(
-        loc, groupTy, group,
+    group = fir::InsertValueOp::create(
+        builder, loc, groupTy, group,
         builder.createIntegerConstant(loc, sizeTy, details.objects().size()),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 1)));
     // item list [const Item *item]
     if (fir::GlobalOp list = builder.getNamedGlobal(listMangleName))
-      listAddr = builder.create<fir::AddrOfOp>(loc, list.resultType(),
-                                               list.getSymbol());
+      listAddr = fir::AddrOfOp::create(builder, loc, list.resultType(),
+                                       list.getSymbol());
     assert(listAddr && "missing namelist object list");
-    group = builder.create<fir::InsertValueOp>(
-        loc, groupTy, group, listAddr,
+    group = fir::InsertValueOp::create(
+        builder, loc, groupTy, group, listAddr,
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 2)));
     // non-type-bound defined IO procedures
     // [const NonTbpDefinedIoTable *nonTbpDefinedIo]
-    group = builder.create<fir::InsertValueOp>(
-        loc, groupTy, group,
+    group = fir::InsertValueOp::create(
+        builder, loc, groupTy, group,
         getNonTbpDefinedIoTableAddr(converter, definedIoProcMap),
         builder.getArrayAttr(builder.getIntegerAttr(idxTy, 3)));
     if (groupIsLocal)
-      builder.create<fir::StoreOp>(loc, group, groupAddr);
+      fir::StoreOp::create(builder, loc, group, groupAddr);
     else
-      builder.create<fir::HasValueOp>(loc, group);
+      fir::HasValueOp::create(builder, loc, group);
   };
   if (groupIsLocal) {
     groupFunc(builder);
@@ -587,8 +590,8 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
     fir::GlobalOp group = builder.createGlobal(
         loc, groupTy, groupMangleName,
         /*isConst=*/true, /*isTarget=*/false, groupFunc, linkOnce);
-    groupAddr = builder.create<fir::AddrOfOp>(loc, group.resultType(),
-                                              group.getSymbol());
+    groupAddr = fir::AddrOfOp::create(builder, loc, group.resultType(),
+                                      group.getSymbol());
   }
   assert(groupAddr && "missing namelist group result");
   return groupAddr;
@@ -608,7 +611,7 @@ static void genNamelistIO(Fortran::lower::AbstractConverter &converter,
       getNamelistGroup(converter, symbol.GetUltimate(), stmtCtx);
   groupAddr = builder.createConvert(loc, argType, groupAddr);
   llvm::SmallVector<mlir::Value> args = {cookie, groupAddr};
-  ok = builder.create<fir::CallOp>(loc, funcOp, args).getResult(0);
+  ok = fir::CallOp::create(builder, loc, funcOp, args).getResult(0);
 }
 
 /// Is \p type a derived type or an array of derived type?
@@ -751,7 +754,7 @@ static void genOutputItemList(
         outputFuncArgs.push_back(itemValue);
       }
     }
-    ok = builder.create<fir::CallOp>(loc, outputFunc, outputFuncArgs)
+    ok = fir::CallOp::create(builder, loc, outputFunc, outputFuncArgs)
              .getResult(0);
   }
 }
@@ -812,12 +815,12 @@ static void boolRefToLogical(mlir::Location loc, fir::FirOpBuilder &builder,
                              mlir::Value addr) {
   auto boolType = builder.getRefType(builder.getI1Type());
   auto boolAddr = builder.createConvert(loc, boolType, addr);
-  auto boolValue = builder.create<fir::LoadOp>(loc, boolAddr);
+  auto boolValue = fir::LoadOp::create(builder, loc, boolAddr);
   auto logicalType = fir::unwrapPassByRefType(addr.getType());
   // The convert avoid making any assumptions about how LOGICALs are actually
   // represented (it might end-up being either a signed or zero extension).
   auto logicalValue = builder.createConvert(loc, logicalType, boolValue);
-  builder.create<fir::StoreOp>(loc, logicalValue, addr);
+  fir::StoreOp::create(builder, loc, logicalValue, addr);
 }
 
 static mlir::Value
@@ -849,12 +852,13 @@ createIoRuntimeCallForItem(Fortran::lower::AbstractConverter &converter,
       inputFuncArgs.push_back(builder.createConvert(
           loc, inputFunc.getFunctionType().getInput(2), len));
     } else if (mlir::isa<mlir::IntegerType>(itemTy)) {
-      inputFuncArgs.push_back(builder.create<mlir::arith::ConstantOp>(
-          loc, builder.getI32IntegerAttr(
-                   mlir::cast<mlir::IntegerType>(itemTy).getWidth() / 8)));
+      inputFuncArgs.push_back(mlir::arith::ConstantOp::create(
+          builder, loc,
+          builder.getI32IntegerAttr(
+              mlir::cast<mlir::IntegerType>(itemTy).getWidth() / 8)));
     }
   }
-  auto call = builder.create<fir::CallOp>(loc, inputFunc, inputFuncArgs);
+  auto call = fir::CallOp::create(builder, loc, inputFunc, inputFuncArgs);
   auto itemAddr = fir::getBase(item);
   auto itemTy = fir::unwrapRefType(itemAddr.getType());
   if (mlir::isa<fir::LogicalType>(itemTy))
@@ -951,7 +955,7 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   mlir::Value stepValue =
       control.step.has_value()
           ? genControlValue(*control.step)
-          : builder.create<mlir::arith::ConstantIndexOp>(loc, 1);
+          : mlir::arith::ConstantIndexOp::create(builder, loc, 1);
   auto genItemList = [&](const D &ioImpliedDo) {
     if constexpr (std::is_same_v<D, Fortran::parser::InputImpliedDo>)
       genInputItemList(converter, cookie, itemList, isFormatted, checkResult,
@@ -962,35 +966,36 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   };
   if (!checkResult) {
     // No IO call result checks - the loop is a fir.do_loop op.
-    auto doLoopOp = builder.create<fir::DoLoopOp>(
-        loc, lowerValue, upperValue, stepValue, /*unordered=*/false,
-        /*finalCountValue=*/true);
+    auto doLoopOp = fir::DoLoopOp::create(builder, loc, lowerValue, upperValue,
+                                          stepValue, /*unordered=*/false,
+                                          /*finalCountValue=*/true);
     builder.setInsertionPointToStart(doLoopOp.getBody());
     mlir::Value lcv = builder.createConvert(
         loc, fir::unwrapRefType(loopVar.getType()), doLoopOp.getInductionVar());
-    builder.create<fir::StoreOp>(loc, lcv, loopVar);
+    fir::StoreOp::create(builder, loc, lcv, loopVar);
     genItemList(ioImpliedDo);
     builder.setInsertionPointToEnd(doLoopOp.getBody());
-    mlir::Value result = builder.create<mlir::arith::AddIOp>(
-        loc, doLoopOp.getInductionVar(), doLoopOp.getStep(), iofAttr);
-    builder.create<fir::ResultOp>(loc, result);
+    mlir::Value result = mlir::arith::AddIOp::create(
+        builder, loc, doLoopOp.getInductionVar(), doLoopOp.getStep(), iofAttr);
+    fir::ResultOp::create(builder, loc, result);
     builder.setInsertionPointAfter(doLoopOp);
     // The loop control variable may be used after the loop.
     lcv = builder.createConvert(loc, fir::unwrapRefType(loopVar.getType()),
                                 doLoopOp.getResult(0));
-    builder.create<fir::StoreOp>(loc, lcv, loopVar);
+    fir::StoreOp::create(builder, loc, lcv, loopVar);
     return;
   }
   // Check IO call results - the loop is a fir.iterate_while op.
   if (!ok)
     ok = builder.createBool(loc, true);
-  auto iterWhileOp = builder.create<fir::IterWhileOp>(
-      loc, lowerValue, upperValue, stepValue, ok, /*finalCountValue*/ true);
+  auto iterWhileOp =
+      fir::IterWhileOp::create(builder, loc, lowerValue, upperValue, stepValue,
+                               ok, /*finalCountValue*/ true);
   builder.setInsertionPointToStart(iterWhileOp.getBody());
   mlir::Value lcv =
       builder.createConvert(loc, fir::unwrapRefType(loopVar.getType()),
                             iterWhileOp.getInductionVar());
-  builder.create<fir::StoreOp>(loc, lcv, loopVar);
+  fir::StoreOp::create(builder, loc, lcv, loopVar);
   ok = iterWhileOp.getIterateVar();
   mlir::Value falseValue =
       builder.createIntegerConstant(loc, builder.getI1Type(), 0);
@@ -1003,28 +1008,28 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
     builder.setInsertionPointAfter(lastOp);
     // The primary ifOp result is the result of an IO call or loop.
     if (mlir::isa<fir::CallOp, fir::IfOp>(*lastOp))
-      builder.create<fir::ResultOp>(loc, lastOp->getResult(0));
+      fir::ResultOp::create(builder, loc, lastOp->getResult(0));
     else
-      builder.create<fir::ResultOp>(loc, ok); // loop result
+      fir::ResultOp::create(builder, loc, ok); // loop result
     // The else branch propagates an early exit false result.
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-    builder.create<fir::ResultOp>(loc, falseValue);
+    fir::ResultOp::create(builder, loc, falseValue);
   }
   builder.setInsertionPointToEnd(iterWhileOp.getBody());
   mlir::OpResult iterateResult = builder.getBlock()->back().getResult(0);
   mlir::Value inductionResult0 = iterWhileOp.getInductionVar();
-  auto inductionResult1 = builder.create<mlir::arith::AddIOp>(
-      loc, inductionResult0, iterWhileOp.getStep(), iofAttr);
-  auto inductionResult = builder.create<mlir::arith::SelectOp>(
-      loc, iterateResult, inductionResult1, inductionResult0);
+  auto inductionResult1 = mlir::arith::AddIOp::create(
+      builder, loc, inductionResult0, iterWhileOp.getStep(), iofAttr);
+  auto inductionResult = mlir::arith::SelectOp::create(
+      builder, loc, iterateResult, inductionResult1, inductionResult0);
   llvm::SmallVector<mlir::Value> results = {inductionResult, iterateResult};
-  builder.create<fir::ResultOp>(loc, results);
+  fir::ResultOp::create(builder, loc, results);
   ok = iterWhileOp.getResult(1);
   builder.setInsertionPointAfter(iterWhileOp);
   // The loop control variable may be used after the loop.
   lcv = builder.createConvert(loc, fir::unwrapRefType(loopVar.getType()),
                               iterWhileOp.getResult(0));
-  builder.create<fir::StoreOp>(loc, lcv, loopVar);
+  fir::StoreOp::create(builder, loc, lcv, loopVar);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1046,15 +1051,15 @@ static mlir::Value locToLineNo(Fortran::lower::AbstractConverter &converter,
 
 static mlir::Value getDefaultScratch(fir::FirOpBuilder &builder,
                                      mlir::Location loc, mlir::Type toType) {
-  mlir::Value null = builder.create<mlir::arith::ConstantOp>(
-      loc, builder.getI64IntegerAttr(0));
+  mlir::Value null = mlir::arith::ConstantOp::create(
+      builder, loc, builder.getI64IntegerAttr(0));
   return builder.createConvert(loc, toType, null);
 }
 
 static mlir::Value getDefaultScratchLen(fir::FirOpBuilder &builder,
                                         mlir::Location loc, mlir::Type toType) {
-  return builder.create<mlir::arith::ConstantOp>(
-      loc, builder.getIntegerAttr(toType, 0));
+  return mlir::arith::ConstantOp::create(builder, loc,
+                                         builder.getIntegerAttr(toType, 0));
 }
 
 /// Generate a reference to a buffer and the length of buffer given
@@ -1105,8 +1110,8 @@ lowerStringLit(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
   mlir::Value kind;
   if (ty2) {
     auto kindVal = expr->GetType().value().kind();
-    kind = builder.create<mlir::arith::ConstantOp>(
-        loc, builder.getIntegerAttr(ty2, kindVal));
+    kind = mlir::arith::ConstantOp::create(
+        builder, loc, builder.getIntegerAttr(ty2, kindVal));
   }
   return {buff, len, kind};
 }
@@ -1146,7 +1151,7 @@ mlir::Value genIntIOOption(Fortran::lower::AbstractConverter &converter,
       loc, Fortran::semantics::GetExpr(spec.v), localStatementCtx));
   mlir::Value val = builder.createConvert(loc, ioFuncTy.getInput(1), expr);
   llvm::SmallVector<mlir::Value> ioArgs = {cookie, val};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 /// Generic to build a string argument to the runtime. This passes a CHARACTER
@@ -1164,7 +1169,7 @@ mlir::Value genCharIOOption(Fortran::lower::AbstractConverter &converter,
                      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> ioArgs = {cookie, std::get<0>(tup),
                                            std::get<1>(tup)};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 template <typename A>
@@ -1197,7 +1202,7 @@ mlir::Value genIOOption<Fortran::parser::FileNameExpr>(
                      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> ioArgs{cookie, std::get<0>(tup),
                                         std::get<1>(tup)};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 template <>
@@ -1262,7 +1267,7 @@ mlir::Value genIOOption<Fortran::parser::ConnectSpec::CharExpr>(
                      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> ioArgs = {cookie, std::get<0>(tup),
                                            std::get<1>(tup)};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 template <>
@@ -1316,7 +1321,7 @@ mlir::Value genIOOption<Fortran::parser::IoControlSpec::CharExpr>(
                      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> ioArgs = {cookie, std::get<0>(tup),
                                            std::get<1>(tup)};
-  return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+  return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 }
 
 template <>
@@ -1352,7 +1357,7 @@ static void genIOGetVar(Fortran::lower::AbstractConverter &converter,
   mlir::func::FuncOp ioFunc =
       fir::runtime::getIORuntimeFunc<IoRuntimeKey>(loc, builder);
   mlir::Value value =
-      builder.create<fir::CallOp>(loc, ioFunc, mlir::ValueRange{cookie})
+      fir::CallOp::create(builder, loc, ioFunc, mlir::ValueRange{cookie})
           .getResult(0);
   Fortran::lower::StatementContext localStatementCtx;
   fir::ExtendedValue var = converter.genExprAddr(
@@ -1480,8 +1485,8 @@ genConditionHandlerCall(Fortran::lower::AbstractConverter &converter,
       fir::runtime::getIORuntimeFunc<mkIOKey(EnableHandlers)>(loc, builder);
   mlir::Type boolType = enableHandlers.getFunctionType().getInput(1);
   auto boolValue = [&](bool specifierIsPresent) {
-    return builder.create<mlir::arith::ConstantOp>(
-        loc, builder.getIntegerAttr(boolType, specifierIsPresent));
+    return mlir::arith::ConstantOp::create(
+        builder, loc, builder.getIntegerAttr(boolType, specifierIsPresent));
   };
   llvm::SmallVector<mlir::Value> ioArgs = {cookie,
                                            boolValue(csi.ioStatExpr != nullptr),
@@ -1489,7 +1494,7 @@ genConditionHandlerCall(Fortran::lower::AbstractConverter &converter,
                                            boolValue(csi.hasEnd),
                                            boolValue(csi.hasEor),
                                            boolValue(csi.ioMsg.has_value())};
-  builder.create<fir::CallOp>(loc, enableHandlers, ioArgs);
+  fir::CallOp::create(builder, loc, enableHandlers, ioArgs);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1663,7 +1668,7 @@ lowerReferenceAsStringSelect(Fortran::lower::AbstractConverter &converter,
     // Pass the format string reference and the string length out of the select
     // statement.
     llvm::SmallVector<mlir::Value> args = {stringRef, stringLen};
-    builder.create<mlir::cf::BranchOp>(loc, endBlock, args);
+    mlir::cf::BranchOp::create(builder, loc, endBlock, args);
 
     // Add block to the list of cases and make a new one.
     blockList.push_back(block);
@@ -1678,13 +1683,13 @@ lowerReferenceAsStringSelect(Fortran::lower::AbstractConverter &converter,
       builder, loc,
       "Assigned format variable '" + symbol->name().ToString() +
           "' has not been assigned a valid format label");
-  builder.create<fir::UnreachableOp>(loc);
+  fir::UnreachableOp::create(builder, loc);
   blockList.push_back(unitBlock);
 
   // Lower the selectOp.
   builder.setInsertionPointToEnd(startBlock);
   auto label = fir::getBase(converter.genExprValue(loc, &expr, stmtCtx));
-  builder.create<fir::SelectOp>(loc, label, indexList, blockList);
+  fir::SelectOp::create(builder, loc, label, indexList, blockList);
 
   builder.setInsertionPointToEnd(endBlock);
   endBlock->addArgument(strTy, loc);
@@ -1814,17 +1819,17 @@ static mlir::Value genIOUnitNumber(Fortran::lower::AbstractConverter &converter,
     mlir::Value line = locToLineNo(converter, loc, funcTy.getInput(5));
     args.push_back(file);
     args.push_back(line);
-    auto checkCall = builder.create<fir::CallOp>(loc, check, args);
+    auto checkCall = fir::CallOp::create(builder, loc, check, args);
     if (csi.hasErrorConditionSpec()) {
       mlir::Value iostat = checkCall.getResult(0);
       mlir::Type iostatTy = iostat.getType();
       mlir::Value zero = fir::factory::createZeroValue(builder, loc, iostatTy);
-      mlir::Value unitIsOK = builder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::eq, iostat, zero);
-      auto ifOp = builder.create<fir::IfOp>(loc, iostatTy, unitIsOK,
-                                            /*withElseRegion=*/true);
+      mlir::Value unitIsOK = mlir::arith::CmpIOp::create(
+          builder, loc, mlir::arith::CmpIPredicate::eq, iostat, zero);
+      auto ifOp = fir::IfOp::create(builder, loc, iostatTy, unitIsOK,
+                                    /*withElseRegion=*/true);
       builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, iostat);
+      fir::ResultOp::create(builder, loc, iostat);
       builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
       stmtCtx.pushScope();
       csi.bigUnitIfOp = ifOp;
@@ -1846,8 +1851,8 @@ static mlir::Value genIOUnit(Fortran::lower::AbstractConverter &converter,
                 &iounit->u))
       return genIOUnitNumber(converter, loc, Fortran::semantics::GetExpr(*e),
                              ty, csi, stmtCtx);
-  return builder.create<mlir::arith::ConstantOp>(
-      loc, builder.getIntegerAttr(ty, defaultUnitNumber));
+  return mlir::arith::ConstantOp::create(
+      builder, loc, builder.getIntegerAttr(ty, defaultUnitNumber));
 }
 
 template <typename A>
@@ -1879,8 +1884,8 @@ static mlir::Value genBasicIOStmt(Fortran::lower::AbstractConverter &converter,
   mlir::Value un = builder.createConvert(loc, beginFuncTy.getInput(0), unit);
   mlir::Value file = locToFilename(converter, loc, beginFuncTy.getInput(1));
   mlir::Value line = locToLineNo(converter, loc, beginFuncTy.getInput(2));
-  auto call = builder.create<fir::CallOp>(loc, beginFunc,
-                                          mlir::ValueRange{un, file, line});
+  auto call = fir::CallOp::create(builder, loc, beginFunc,
+                                  mlir::ValueRange{un, file, line});
   mlir::Value cookie = call.getResult(0);
   genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
   mlir::Value ok;
@@ -1934,7 +1939,7 @@ genNewunitSpec(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
       auto kind = builder.createIntegerConstant(loc, ioFuncTy.getInput(2),
                                                 var->GetType().value().kind());
       llvm::SmallVector<mlir::Value> ioArgs = {cookie, addr, kind};
-      return builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+      return fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
     }
   llvm_unreachable("missing Newunit spec");
 }
@@ -1969,7 +1974,7 @@ Fortran::lower::genOpenStatement(Fortran::lower::AbstractConverter &converter,
     beginArgs.push_back(locToLineNo(converter, loc, beginFuncTy.getInput(1)));
   }
   auto cookie =
-      builder.create<fir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+      fir::CallOp::create(builder, loc, beginFunc, beginArgs).getResult(0);
   genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
   mlir::Value ok;
   auto insertPt = builder.saveInsertionPoint();
@@ -2013,7 +2018,7 @@ Fortran::lower::genWaitStatement(Fortran::lower::AbstractConverter &converter,
     args.push_back(locToFilename(converter, loc, beginFuncTy.getInput(1)));
     args.push_back(locToLineNo(converter, loc, beginFuncTy.getInput(2)));
   }
-  auto cookie = builder.create<fir::CallOp>(loc, beginFunc, args).getResult(0);
+  auto cookie = fir::CallOp::create(builder, loc, beginFunc, args).getResult(0);
   genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
   return genEndIO(converter, converter.getCurrentLocation(), cookie, csi,
                   stmtCtx);
@@ -2149,9 +2154,10 @@ void genBeginDataTransferCallArgs(
     }
   } else { // PRINT - maybe explicit format; default unit
     maybeGetFormatArgs();
-    ioArgs.push_back(builder.create<mlir::arith::ConstantOp>(
-        loc, builder.getIntegerAttr(ioFuncTy.getInput(ioArgs.size()),
-                                    defaultUnitNumber)));
+    ioArgs.push_back(mlir::arith::ConstantOp::create(
+        builder, loc,
+        builder.getIntegerAttr(ioFuncTy.getInput(ioArgs.size()),
+                               defaultUnitNumber)));
   }
   // File name and line number are always the last two arguments.
   ioArgs.push_back(
@@ -2198,7 +2204,7 @@ genDataTransferStmt(Fortran::lower::AbstractConverter &converter,
       ioArgs, converter, loc, stmt, ioFunc.getFunctionType(), isFormatted,
       isList || isNml, isInternal, descRef, csi, stmtCtx);
   mlir::Value cookie =
-      builder.create<fir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+      fir::CallOp::create(builder, loc, ioFunc, ioArgs).getResult(0);
 
   auto insertPt = builder.saveInsertionPoint();
   mlir::Value ok;
@@ -2332,7 +2338,7 @@ mlir::Value genInquireSpec<Fortran::parser::InquireSpec::CharVar>(
                                                        .c_str())),
       builder.createConvert(loc, specFuncTy.getInput(2), fir::getBase(str)),
       builder.createConvert(loc, specFuncTy.getInput(3), fir::getLen(str))};
-  return builder.create<fir::CallOp>(loc, specFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, specFunc, args).getResult(0);
 }
 /// Specialization for INTEGER.
 template <>
@@ -2369,7 +2375,7 @@ mlir::Value genInquireSpec<Fortran::parser::InquireSpec::IntVar>(
                                                        .c_str())),
       builder.createConvert(loc, specFuncTy.getInput(2), addr),
       builder.createConvert(loc, specFuncTy.getInput(3), kind)};
-  return builder.create<fir::CallOp>(loc, specFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, specFunc, args).getResult(0);
 }
 /// Specialization for LOGICAL and (PENDING + ID).
 template <>
@@ -2406,7 +2412,7 @@ mlir::Value genInquireSpec<Fortran::parser::InquireSpec::LogVar>(
             Fortran::parser::InquireSpec::LogVar::EnumToString(logVarKind)}
                                                      .c_str())));
   args.push_back(builder.createConvert(loc, specFuncTy.getInput(2), addr));
-  auto call = builder.create<fir::CallOp>(loc, specFunc, args);
+  auto call = fir::CallOp::create(builder, loc, specFunc, args);
   boolRefToLogical(loc, builder, addr);
   return call.getResult(0);
 }
@@ -2502,7 +2508,7 @@ mlir::Value Fortran::lower::genInquireStatement(
     beginArgs = {locToFilename(converter, loc, beginFuncTy.getInput(0)),
                  locToLineNo(converter, loc, beginFuncTy.getInput(1))};
     auto cookie =
-        builder.create<fir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+        fir::CallOp::create(builder, loc, beginFunc, beginArgs).getResult(0);
     mlir::Value ok;
     genOutputItemList(
         converter, cookie,
@@ -2523,14 +2529,14 @@ mlir::Value Fortran::lower::genInquireStatement(
             .getResult(0);
     mlir::Value length1 =
         builder.createConvert(loc, converter.genType(*ioLengthVar), length);
-    builder.create<fir::StoreOp>(loc, length1, ioLengthVarAddr);
+    fir::StoreOp::create(builder, loc, length1, ioLengthVarAddr);
     return genEndIO(converter, loc, cookie, csi, stmtCtx);
   }
 
   // Common handling for inquire by unit or file.
   assert(list && "inquire-spec list must be present");
   auto cookie =
-      builder.create<fir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+      fir::CallOp::create(builder, loc, beginFunc, beginArgs).getResult(0);
   genConditionHandlerCall(converter, loc, cookie, *list, csi);
   // Handle remaining arguments in specifier list.
   mlir::Value ok;

diff  --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 51eb33dec186b..471f3685974cd 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -132,21 +132,21 @@ createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
                        /*withElseRegion=*/true)
               .genThen([&]() {
                 if (fir::isBoxAddress(baseAddr.getType()))
-                  baseAddr = builder.create<fir::LoadOp>(loc, baseAddr);
+                  baseAddr = fir::LoadOp::create(builder, loc, baseAddr);
                 mlir::Value boxAddr =
-                    builder.create<fir::BoxAddrOp>(loc, baseAddr);
-                builder.create<fir::ResultOp>(loc, mlir::ValueRange{boxAddr});
+                    fir::BoxAddrOp::create(builder, loc, baseAddr);
+                fir::ResultOp::create(builder, loc, mlir::ValueRange{boxAddr});
               })
               .genElse([&] {
                 mlir::Value absent =
-                    builder.create<fir::AbsentOp>(loc, ifRetTy);
-                builder.create<fir::ResultOp>(loc, mlir::ValueRange{absent});
+                    fir::AbsentOp::create(builder, loc, ifRetTy);
+                fir::ResultOp::create(builder, loc, mlir::ValueRange{absent});
               })
               .getResults()[0];
     } else {
       if (fir::isBoxAddress(baseAddr.getType()))
-        baseAddr = builder.create<fir::LoadOp>(loc, baseAddr);
-      baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
+        baseAddr = fir::LoadOp::create(builder, loc, baseAddr);
+      baseAddr = fir::BoxAddrOp::create(builder, loc, baseAddr);
     }
     retTy = baseAddr.getType();
   }
@@ -159,7 +159,7 @@ createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
   addOperands(operands, operandSegments, bounds);
   addOperands(operands, operandSegments, async);
 
-  Op op = builder.create<Op>(loc, retTy, operands);
+  Op op = Op::create(builder, loc, retTy, operands);
   op.setNameAttr(builder.getStringAttr(name.str()));
   op.setStructured(structured);
   op.setImplicit(implicit);
@@ -198,12 +198,12 @@ createDeclareFunc(mlir::OpBuilder &modBuilder, fir::FirOpBuilder &builder,
                   llvm::SmallVector<mlir::Type> argsTy = {},
                   llvm::SmallVector<mlir::Location> locs = {}) {
   auto funcTy = mlir::FunctionType::get(modBuilder.getContext(), argsTy, {});
-  auto funcOp = modBuilder.create<mlir::func::FuncOp>(loc, funcName, funcTy);
+  auto funcOp = mlir::func::FuncOp::create(modBuilder, loc, funcName, funcTy);
   funcOp.setVisibility(mlir::SymbolTable::Visibility::Private);
   builder.createBlock(&funcOp.getRegion(), funcOp.getRegion().end(), argsTy,
                       locs);
   builder.setInsertionPointToEnd(&funcOp.getRegion().back());
-  builder.create<mlir::func::ReturnOp>(loc);
+  mlir::func::ReturnOp::create(builder, loc);
   builder.setInsertionPointToStart(&funcOp.getRegion().back());
   return funcOp;
 }
@@ -214,7 +214,7 @@ createSimpleOp(fir::FirOpBuilder &builder, mlir::Location loc,
                const llvm::SmallVectorImpl<mlir::Value> &operands,
                const llvm::SmallVectorImpl<int32_t> &operandSegments) {
   llvm::ArrayRef<mlir::Type> argTy;
-  Op op = builder.create<Op>(loc, argTy, operands);
+  Op op = Op::create(builder, loc, argTy, operands);
   op->setAttr(Op::getOperandSegmentSizeAttr(),
               builder.getDenseI32ArrayAttr(operandSegments));
   return op;
@@ -257,15 +257,15 @@ static void createDeclareAllocFuncWithArg(mlir::OpBuilder &modBuilder,
 
   if (unwrapFirBox) {
     mlir::Value desc =
-        builder.create<fir::LoadOp>(loc, registerFuncOp.getArgument(0));
-    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, desc);
+        fir::LoadOp::create(builder, loc, registerFuncOp.getArgument(0));
+    fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, desc);
     addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
     EntryOp entryOp = createDataEntryOp<EntryOp>(
         builder, loc, boxAddrOp.getResult(), asFortran, bounds,
         /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-    builder.create<mlir::acc::DeclareEnterOp>(
-        loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
+    mlir::acc::DeclareEnterOp::create(
+        builder, loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
         mlir::ValueRange(entryOp.getAccVar()));
   }
 
@@ -291,8 +291,8 @@ static void createDeclareDeallocFuncWithArg(
   mlir::Value var = preDeallocOp.getArgument(0);
   if (unwrapFirBox) {
     mlir::Value loadOp =
-        builder.create<fir::LoadOp>(loc, preDeallocOp.getArgument(0));
-    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+        fir::LoadOp::create(builder, loc, preDeallocOp.getArgument(0));
+    fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, loadOp);
     addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
     var = boxAddrOp.getResult();
   }
@@ -303,25 +303,25 @@ static void createDeclareDeallocFuncWithArg(
           builder, loc, var, asFortran, bounds,
           /*structured=*/false, /*implicit=*/false, clause, var.getType(),
           /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-  builder.create<mlir::acc::DeclareExitOp>(
-      loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccVar()));
+  mlir::acc::DeclareExitOp::create(builder, loc, mlir::Value{},
+                                   mlir::ValueRange(entryOp.getAccVar()));
 
   if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                 std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
-                           entryOp.getVar(), entryOp.getVarType(),
-                           entryOp.getBounds(), entryOp.getAsyncOperands(),
-                           entryOp.getAsyncOperandsDeviceTypeAttr(),
-                           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-                           /*structured=*/false, /*implicit=*/false,
-                           builder.getStringAttr(*entryOp.getName()));
+    ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
+                   entryOp.getVar(), entryOp.getVarType(), entryOp.getBounds(),
+                   entryOp.getAsyncOperands(),
+                   entryOp.getAsyncOperandsDeviceTypeAttr(),
+                   entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                   /*structured=*/false, /*implicit=*/false,
+                   builder.getStringAttr(*entryOp.getName()));
   else
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
-                           entryOp.getBounds(), entryOp.getAsyncOperands(),
-                           entryOp.getAsyncOperandsDeviceTypeAttr(),
-                           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-                           /*structured=*/false, /*implicit=*/false,
-                           builder.getStringAttr(*entryOp.getName()));
+    ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
+                   entryOp.getBounds(), entryOp.getAsyncOperands(),
+                   entryOp.getAsyncOperandsDeviceTypeAttr(),
+                   entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                   /*structured=*/false, /*implicit=*/false,
+                   builder.getStringAttr(*entryOp.getName()));
 
   // Generate the post dealloc function.
   modBuilder.setInsertionPointAfter(preDeallocOp);
@@ -333,7 +333,7 @@ static void createDeclareDeallocFuncWithArg(
 
   var = postDeallocOp.getArgument(0);
   if (unwrapFirBox) {
-    var = builder.create<fir::LoadOp>(loc, postDeallocOp.getArgument(0));
+    var = fir::LoadOp::create(builder, loc, postDeallocOp.getArgument(0));
     asFortran << accFirDescriptorPostfix.str();
   }
 
@@ -385,8 +385,8 @@ genAtomicCaptureStatement(Fortran::lower::AbstractConverter &converter,
   // Generate `atomic.read` operation for atomic assigment statements
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
-  firOpBuilder.create<mlir::acc::AtomicReadOp>(
-      loc, fromAddress, toAddress, mlir::TypeAttr::get(elementType));
+  mlir::acc::AtomicReadOp::create(firOpBuilder, loc, fromAddress, toAddress,
+                                  mlir::TypeAttr::get(elementType));
 }
 
 /// Used to generate atomic.write operation which is created in existing
@@ -406,7 +406,7 @@ genAtomicWriteStatement(Fortran::lower::AbstractConverter &converter,
   rhsExpr = firOpBuilder.createConvert(loc, varType, rhsExpr);
   firOpBuilder.restoreInsertionPoint(insertionPoint);
 
-  firOpBuilder.create<mlir::acc::AtomicWriteOp>(loc, lhsAddr, rhsExpr);
+  mlir::acc::AtomicWriteOp::create(firOpBuilder, loc, lhsAddr, rhsExpr);
 }
 
 /// Used to generate atomic.update operation which is created in existing
@@ -522,7 +522,7 @@ static inline void genAtomicUpdateStatement(
 
   mlir::Operation *atomicUpdateOp = nullptr;
   atomicUpdateOp =
-      firOpBuilder.create<mlir::acc::AtomicUpdateOp>(currentLocation, lhsAddr);
+      mlir::acc::AtomicUpdateOp::create(firOpBuilder, currentLocation, lhsAddr);
 
   llvm::SmallVector<mlir::Type> varTys = {varType};
   llvm::SmallVector<mlir::Location> locs = {currentLocation};
@@ -540,7 +540,7 @@ static inline void genAtomicUpdateStatement(
         *Fortran::semantics::GetExpr(assignmentStmtExpr), atomicStmtCtx));
     mlir::Value convertResult =
         firOpBuilder.createConvert(currentLocation, varType, rhsExpr);
-    firOpBuilder.create<mlir::acc::YieldOp>(currentLocation, convertResult);
+    mlir::acc::YieldOp::create(firOpBuilder, currentLocation, convertResult);
     converter.resetExprOverrides();
   }
   firOpBuilder.setInsertionPointAfter(atomicUpdateOp);
@@ -647,7 +647,7 @@ void genAtomicCapture(Fortran::lower::AbstractConverter &converter,
       fir::getBase(converter.genExprValue(assign2.lhs, stmtCtx)).getType();
 
   mlir::Operation *atomicCaptureOp = nullptr;
-  atomicCaptureOp = firOpBuilder.create<mlir::acc::AtomicCaptureOp>(loc);
+  atomicCaptureOp = mlir::acc::AtomicCaptureOp::create(firOpBuilder, loc);
 
   firOpBuilder.createBlock(&(atomicCaptureOp->getRegion(0)));
   mlir::Block &block = atomicCaptureOp->getRegion(0).back();
@@ -688,7 +688,7 @@ void genAtomicCapture(Fortran::lower::AbstractConverter &converter,
                               loc);
   }
   firOpBuilder.setInsertionPointToEnd(&block);
-  firOpBuilder.create<mlir::acc::TerminatorOp>(loc);
+  mlir::acc::TerminatorOp::create(firOpBuilder, loc);
   // The clean-ups associated with the statements inside the capture
   // construct must be generated after the AtomicCaptureOp.
   firOpBuilder.setInsertionPointAfter(atomicCaptureOp);
@@ -839,15 +839,15 @@ genDataExitOperations(fir::FirOpBuilder &builder,
     mlir::Location opLoc = exitLoc ? *exitLoc : entryOp.getLoc();
     if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                   std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-      builder.create<ExitOp>(
-          opLoc, entryOp.getAccVar(), entryOp.getVar(), entryOp.getVarType(),
-          entryOp.getBounds(), entryOp.getAsyncOperands(),
+      ExitOp::create(
+          builder, opLoc, entryOp.getAccVar(), entryOp.getVar(),
+          entryOp.getVarType(), entryOp.getBounds(), entryOp.getAsyncOperands(),
           entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
           entryOp.getDataClause(), structured, entryOp.getImplicit(),
           builder.getStringAttr(*entryOp.getName()));
     else
-      builder.create<ExitOp>(
-          opLoc, entryOp.getAccVar(), entryOp.getBounds(),
+      ExitOp::create(
+          builder, opLoc, entryOp.getAccVar(), entryOp.getBounds(),
           entryOp.getAsyncOperands(), entryOp.getAsyncOperandsDeviceTypeAttr(),
           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(), structured,
           entryOp.getImplicit(), builder.getStringAttr(*entryOp.getName()));
@@ -859,9 +859,9 @@ fir::ShapeOp genShapeOp(mlir::OpBuilder &builder, fir::SequenceType seqTy,
   llvm::SmallVector<mlir::Value> extents;
   mlir::Type idxTy = builder.getIndexType();
   for (auto extent : seqTy.getShape())
-    extents.push_back(builder.create<mlir::arith::ConstantOp>(
-        loc, idxTy, builder.getIntegerAttr(idxTy, extent)));
-  return builder.create<fir::ShapeOp>(loc, extents);
+    extents.push_back(mlir::arith::ConstantOp::create(
+        builder, loc, idxTy, builder.getIntegerAttr(idxTy, extent)));
+  return fir::ShapeOp::create(builder, loc, extents);
 }
 
 /// Get the initial value for reduction operator.
@@ -936,8 +936,8 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
     return builder.createBool(loc, value);
   }
   if (ty.isIntOrIndex())
-    return builder.create<mlir::arith::ConstantOp>(
-        loc, ty,
+    return mlir::arith::ConstantOp::create(
+        builder, loc, ty,
         builder.getIntegerAttr(ty, getReductionInitValue<llvm::APInt>(op, ty)));
   if (op == mlir::acc::ReductionOperator::AccMin ||
       op == mlir::acc::ReductionOperator::AccMax) {
@@ -945,13 +945,13 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
       llvm::report_fatal_error(
           "min/max reduction not supported for complex type");
     if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::ConstantOp>(
-          loc, ty,
+      return mlir::arith::ConstantOp::create(
+          builder, loc, ty,
           builder.getFloatAttr(ty,
                                getReductionInitValue<llvm::APFloat>(op, ty)));
   } else if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty)) {
-    return builder.create<mlir::arith::ConstantOp>(
-        loc, ty,
+    return mlir::arith::ConstantOp::create(
+        builder, loc, ty,
         builder.getFloatAttr(ty, getReductionInitValue<int64_t>(op, ty)));
   } else if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
     mlir::Type floatTy = cmplxTy.getElementType();
@@ -985,10 +985,10 @@ static RecipeOp genRecipeOp(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   RecipeOp recipe;
   if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>) {
-    recipe = modBuilder.create<mlir::acc::ReductionRecipeOp>(loc, recipeName,
-                                                             ty, op);
+    recipe = mlir::acc::ReductionRecipeOp::create(modBuilder, loc, recipeName,
+                                                  ty, op);
   } else {
-    recipe = modBuilder.create<RecipeOp>(loc, recipeName, ty);
+    recipe = RecipeOp::create(modBuilder, loc, recipeName, ty);
   }
 
   llvm::SmallVector<mlir::Type> argsTy{ty};
@@ -1032,8 +1032,8 @@ static RecipeOp genRecipeOp(
       initName,
       initBlock->getArguments().take_back(initBlock->getArguments().size() - 1),
       initValue);
-  builder.create<mlir::acc::YieldOp>(loc, retVal ? retVal
-                                                 : initBlock->getArgument(0));
+  mlir::acc::YieldOp::create(builder, loc,
+                             retVal ? retVal : initBlock->getArgument(0));
   return recipe;
 }
 
@@ -1132,15 +1132,17 @@ static mlir::Value genShapeFromBoundsOrArgs(
   mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
   for (unsigned i = 0; i < args.size(); i += 3) {
     mlir::Value s1 =
-        builder.create<mlir::arith::SubIOp>(loc, args[i + 1], args[0]);
-    mlir::Value s2 = builder.create<mlir::arith::AddIOp>(loc, s1, one);
-    mlir::Value s3 = builder.create<mlir::arith::DivSIOp>(loc, s2, args[i + 2]);
-    mlir::Value cmp = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::sgt, s3, zero);
-    mlir::Value ext = builder.create<mlir::arith::SelectOp>(loc, cmp, s3, zero);
+        mlir::arith::SubIOp::create(builder, loc, args[i + 1], args[0]);
+    mlir::Value s2 = mlir::arith::AddIOp::create(builder, loc, s1, one);
+    mlir::Value s3 =
+        mlir::arith::DivSIOp::create(builder, loc, s2, args[i + 2]);
+    mlir::Value cmp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::sgt, s3, zero);
+    mlir::Value ext =
+        mlir::arith::SelectOp::create(builder, loc, cmp, s3, zero);
     extents.push_back(ext);
   }
-  return builder.create<fir::ShapeOp>(loc, extents);
+  return fir::ShapeOp::create(builder, loc, extents);
 }
 
 static hlfir::DesignateOp::Subscripts
@@ -1157,8 +1159,8 @@ static hlfir::Entity genDesignateWithTriplets(
     hlfir::DesignateOp::Subscripts &triplets, mlir::Value shape) {
   llvm::SmallVector<mlir::Value> lenParams;
   hlfir::genLengthParameters(loc, builder, entity, lenParams);
-  auto designate = builder.create<hlfir::DesignateOp>(
-      loc, entity.getBase().getType(), entity, /*component=*/"",
+  auto designate = hlfir::DesignateOp::create(
+      builder, loc, entity.getBase().getType(), entity, /*component=*/"",
       /*componentShape=*/mlir::Value{}, triplets,
       /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt, shape,
       lenParams);
@@ -1198,22 +1200,22 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
   builder.setInsertionPointToEnd(&recipe.getCopyRegion().back());
   ty = fir::unwrapRefType(ty);
   if (fir::isa_trivial(ty)) {
-    mlir::Value initValue = builder.create<fir::LoadOp>(
-        loc, recipe.getCopyRegion().front().getArgument(0));
-    builder.create<fir::StoreOp>(loc, initValue,
-                                 recipe.getCopyRegion().front().getArgument(1));
+    mlir::Value initValue = fir::LoadOp::create(
+        builder, loc, recipe.getCopyRegion().front().getArgument(0));
+    fir::StoreOp::create(builder, loc, initValue,
+                         recipe.getCopyRegion().front().getArgument(1));
   } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
     fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
     auto shape = genShapeFromBoundsOrArgs(
         loc, firBuilder, seqTy, bounds, recipe.getCopyRegion().getArguments());
 
-    auto leftDeclOp = builder.create<hlfir::DeclareOp>(
-        loc, recipe.getCopyRegion().getArgument(0), llvm::StringRef{}, shape,
-        llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+    auto leftDeclOp = hlfir::DeclareOp::create(
+        builder, loc, recipe.getCopyRegion().getArgument(0), llvm::StringRef{},
+        shape, llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{});
-    auto rightDeclOp = builder.create<hlfir::DeclareOp>(
-        loc, recipe.getCopyRegion().getArgument(1), llvm::StringRef{}, shape,
-        llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+    auto rightDeclOp = hlfir::DeclareOp::create(
+        builder, loc, recipe.getCopyRegion().getArgument(1), llvm::StringRef{},
+        shape, llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{});
 
     hlfir::DesignateOp::Subscripts triplets =
@@ -1225,7 +1227,7 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
     auto right =
         genDesignateWithTriplets(firBuilder, loc, rightEntity, triplets, shape);
 
-    firBuilder.create<hlfir::AssignOp>(loc, left, right);
+    hlfir::AssignOp::create(firBuilder, loc, left, right);
 
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
     fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
@@ -1246,10 +1248,10 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
     auto rightEntity = hlfir::Entity{recipe.getCopyRegion().getArgument(1)};
     auto right =
         genDesignateWithTriplets(firBuilder, loc, rightEntity, triplets, shape);
-    firBuilder.create<hlfir::AssignOp>(loc, left, right);
+    hlfir::AssignOp::create(firBuilder, loc, left, right);
   }
 
-  builder.create<mlir::acc::TerminatorOp>(loc);
+  mlir::acc::TerminatorOp::create(builder, loc);
   builder.restoreInsertionPoint(ip);
   return recipe;
 }
@@ -1414,10 +1416,10 @@ static mlir::Value genLogicalCombiner(fir::FirOpBuilder &builder,
                                       mlir::Location loc, mlir::Value value1,
                                       mlir::Value value2) {
   mlir::Type i1 = builder.getI1Type();
-  mlir::Value v1 = builder.create<fir::ConvertOp>(loc, i1, value1);
-  mlir::Value v2 = builder.create<fir::ConvertOp>(loc, i1, value2);
-  mlir::Value combined = builder.create<Op>(loc, v1, v2);
-  return builder.create<fir::ConvertOp>(loc, value1.getType(), combined);
+  mlir::Value v1 = fir::ConvertOp::create(builder, loc, i1, value1);
+  mlir::Value v2 = fir::ConvertOp::create(builder, loc, i1, value2);
+  mlir::Value combined = Op::create(builder, loc, v1, v2);
+  return fir::ConvertOp::create(builder, loc, value1.getType(), combined);
 }
 
 static mlir::Value genComparisonCombiner(fir::FirOpBuilder &builder,
@@ -1426,10 +1428,10 @@ static mlir::Value genComparisonCombiner(fir::FirOpBuilder &builder,
                                          mlir::Value value1,
                                          mlir::Value value2) {
   mlir::Type i1 = builder.getI1Type();
-  mlir::Value v1 = builder.create<fir::ConvertOp>(loc, i1, value1);
-  mlir::Value v2 = builder.create<fir::ConvertOp>(loc, i1, value2);
-  mlir::Value add = builder.create<mlir::arith::CmpIOp>(loc, pred, v1, v2);
-  return builder.create<fir::ConvertOp>(loc, value1.getType(), add);
+  mlir::Value v1 = fir::ConvertOp::create(builder, loc, i1, value1);
+  mlir::Value v2 = fir::ConvertOp::create(builder, loc, i1, value2);
+  mlir::Value add = mlir::arith::CmpIOp::create(builder, loc, pred, v1, v2);
+  return fir::ConvertOp::create(builder, loc, value1.getType(), add);
 }
 
 static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
@@ -1441,21 +1443,21 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
   value2 = builder.loadIfRef(loc, value2);
   if (op == mlir::acc::ReductionOperator::AccAdd) {
     if (ty.isIntOrIndex())
-      return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+      return mlir::arith::AddIOp::create(builder, loc, value1, value2);
     if (mlir::isa<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+      return mlir::arith::AddFOp::create(builder, loc, value1, value2);
     if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty))
-      return builder.create<fir::AddcOp>(loc, value1, value2);
+      return fir::AddcOp::create(builder, loc, value1, value2);
     TODO(loc, "reduction add type");
   }
 
   if (op == mlir::acc::ReductionOperator::AccMul) {
     if (ty.isIntOrIndex())
-      return builder.create<mlir::arith::MulIOp>(loc, value1, value2);
+      return mlir::arith::MulIOp::create(builder, loc, value1, value2);
     if (mlir::isa<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::MulFOp>(loc, value1, value2);
+      return mlir::arith::MulFOp::create(builder, loc, value1, value2);
     if (mlir::isa<mlir::ComplexType>(ty))
-      return builder.create<fir::MulcOp>(loc, value1, value2);
+      return fir::MulcOp::create(builder, loc, value1, value2);
     TODO(loc, "reduction mul type");
   }
 
@@ -1466,13 +1468,13 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
     return fir::genMax(builder, loc, {value1, value2});
 
   if (op == mlir::acc::ReductionOperator::AccIand)
-    return builder.create<mlir::arith::AndIOp>(loc, value1, value2);
+    return mlir::arith::AndIOp::create(builder, loc, value1, value2);
 
   if (op == mlir::acc::ReductionOperator::AccIor)
-    return builder.create<mlir::arith::OrIOp>(loc, value1, value2);
+    return mlir::arith::OrIOp::create(builder, loc, value1, value2);
 
   if (op == mlir::acc::ReductionOperator::AccXor)
-    return builder.create<mlir::arith::XOrIOp>(loc, value1, value2);
+    return mlir::arith::XOrIOp::create(builder, loc, value1, value2);
 
   if (op == mlir::acc::ReductionOperator::AccLand)
     return genLogicalCombiner<mlir::arith::AndIOp>(builder, loc, value1,
@@ -1520,19 +1522,21 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       auto shape =
           genShapeFromBoundsOrArgs(loc, builder, seqTy, bounds,
                                    recipe.getCombinerRegion().getArguments());
-      auto v1DeclareOp = builder.create<hlfir::DeclareOp>(
-          loc, value1, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
+      auto v1DeclareOp = hlfir::DeclareOp::create(
+          builder, loc, value1, llvm::StringRef{}, shape,
+          llvm::ArrayRef<mlir::Value>{},
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
-      auto v2DeclareOp = builder.create<hlfir::DeclareOp>(
-          loc, value2, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
+      auto v2DeclareOp = hlfir::DeclareOp::create(
+          builder, loc, value2, llvm::StringRef{}, shape,
+          llvm::ArrayRef<mlir::Value>{},
           /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       hlfir::DesignateOp::Subscripts triplets = getTripletsFromArgs(recipe);
 
       llvm::SmallVector<mlir::Value> lenParamsLeft;
       auto leftEntity = hlfir::Entity{v1DeclareOp.getBase()};
       hlfir::genLengthParameters(loc, builder, leftEntity, lenParamsLeft);
-      auto leftDesignate = builder.create<hlfir::DesignateOp>(
-          loc, v1DeclareOp.getBase().getType(), v1DeclareOp.getBase(),
+      auto leftDesignate = hlfir::DesignateOp::create(
+          builder, loc, v1DeclareOp.getBase().getType(), v1DeclareOp.getBase(),
           /*component=*/"",
           /*componentShape=*/mlir::Value{}, triplets,
           /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
@@ -1542,8 +1546,8 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       llvm::SmallVector<mlir::Value> lenParamsRight;
       auto rightEntity = hlfir::Entity{v2DeclareOp.getBase()};
       hlfir::genLengthParameters(loc, builder, rightEntity, lenParamsLeft);
-      auto rightDesignate = builder.create<hlfir::DesignateOp>(
-          loc, v2DeclareOp.getBase().getType(), v2DeclareOp.getBase(),
+      auto rightDesignate = hlfir::DesignateOp::create(
+          builder, loc, v2DeclareOp.getBase().getType(), v2DeclareOp.getBase(),
           /*component=*/"",
           /*componentShape=*/mlir::Value{}, triplets,
           /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
@@ -1564,21 +1568,21 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       mlir::Value elemental = hlfir::genElementalOp(
           loc, builder, seqTy.getEleTy(), shape, typeParams, genKernel,
           /*isUnordered=*/true);
-      builder.create<hlfir::AssignOp>(loc, elemental, v1DeclareOp.getBase());
+      hlfir::AssignOp::create(builder, loc, elemental, v1DeclareOp.getBase());
       return;
     }
     if (bounds.empty()) {
       llvm::SmallVector<mlir::Value> extents;
       mlir::Type idxTy = builder.getIndexType();
       for (auto extent : seqTy.getShape()) {
-        mlir::Value lb = builder.create<mlir::arith::ConstantOp>(
-            loc, idxTy, builder.getIntegerAttr(idxTy, 0));
-        mlir::Value ub = builder.create<mlir::arith::ConstantOp>(
-            loc, idxTy, builder.getIntegerAttr(idxTy, extent - 1));
-        mlir::Value step = builder.create<mlir::arith::ConstantOp>(
-            loc, idxTy, builder.getIntegerAttr(idxTy, 1));
-        auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
-                                                  /*unordered=*/false);
+        mlir::Value lb = mlir::arith::ConstantOp::create(
+            builder, loc, idxTy, builder.getIntegerAttr(idxTy, 0));
+        mlir::Value ub = mlir::arith::ConstantOp::create(
+            builder, loc, idxTy, builder.getIntegerAttr(idxTy, extent - 1));
+        mlir::Value step = mlir::arith::ConstantOp::create(
+            builder, loc, idxTy, builder.getIntegerAttr(idxTy, 1));
+        auto loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
+                                          /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
         ivs.push_back(loop.getInductionVar());
@@ -1594,8 +1598,8 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
         llvm::SmallVector<mlir::Value> values =
             genConstantBounds(builder, loc, dataBound);
         auto loop =
-            builder.create<fir::DoLoopOp>(loc, values[0], values[1], values[2],
-                                          /*unordered=*/false);
+            fir::DoLoopOp::create(builder, loc, values[0], values[1], values[2],
+                                  /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
         ivs.push_back(loop.getInductionVar());
@@ -1611,31 +1615,31 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
         mlir::Value lb = recipe.getCombinerRegion().getArgument(i);
         mlir::Value ub = recipe.getCombinerRegion().getArgument(i + 1);
         mlir::Value step = recipe.getCombinerRegion().getArgument(i + 2);
-        auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
-                                                  /*unordered=*/false);
+        auto loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
+                                          /*unordered=*/false);
         builder.setInsertionPointToStart(loop.getBody());
         loops.push_back(loop);
         ivs.push_back(loop.getInductionVar());
       }
     }
-    auto addr1 = builder.create<fir::CoordinateOp>(loc, refTy, value1, ivs);
-    auto addr2 = builder.create<fir::CoordinateOp>(loc, refTy, value2, ivs);
-    auto load1 = builder.create<fir::LoadOp>(loc, addr1);
-    auto load2 = builder.create<fir::LoadOp>(loc, addr2);
+    auto addr1 = fir::CoordinateOp::create(builder, loc, refTy, value1, ivs);
+    auto addr2 = fir::CoordinateOp::create(builder, loc, refTy, value2, ivs);
+    auto load1 = fir::LoadOp::create(builder, loc, addr1);
+    auto load2 = fir::LoadOp::create(builder, loc, addr2);
     mlir::Value res =
         genScalarCombiner(builder, loc, op, seqTy.getEleTy(), load1, load2);
-    builder.create<fir::StoreOp>(loc, res, addr1);
+    fir::StoreOp::create(builder, loc, res, addr1);
     builder.setInsertionPointAfter(loops[0]);
   } else if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {
     mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
     if (fir::isa_trivial(innerTy)) {
       mlir::Value boxAddr1 = value1, boxAddr2 = value2;
       if (fir::isBoxAddress(boxAddr1.getType()))
-        boxAddr1 = builder.create<fir::LoadOp>(loc, boxAddr1);
+        boxAddr1 = fir::LoadOp::create(builder, loc, boxAddr1);
       if (fir::isBoxAddress(boxAddr2.getType()))
-        boxAddr2 = builder.create<fir::LoadOp>(loc, boxAddr2);
-      boxAddr1 = builder.create<fir::BoxAddrOp>(loc, boxAddr1);
-      boxAddr2 = builder.create<fir::BoxAddrOp>(loc, boxAddr2);
+        boxAddr2 = fir::LoadOp::create(builder, loc, boxAddr2);
+      boxAddr1 = fir::BoxAddrOp::create(builder, loc, boxAddr1);
+      boxAddr2 = fir::BoxAddrOp::create(builder, loc, boxAddr2);
       auto leftEntity = hlfir::Entity{boxAddr1};
       auto rightEntity = hlfir::Entity{boxAddr2};
 
@@ -1643,7 +1647,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       auto rightVal = hlfir::loadTrivialScalar(loc, builder, rightEntity);
       mlir::Value res =
           genScalarCombiner(builder, loc, op, innerTy, leftVal, rightVal);
-      builder.create<hlfir::AssignOp>(loc, res, boxAddr1);
+      hlfir::AssignOp::create(builder, loc, res, boxAddr1);
     } else {
       mlir::Type innerTy = fir::extractSequenceType(boxTy);
       fir::SequenceType seqTy =
@@ -1658,14 +1662,14 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
           getSubscriptsFromArgs(recipe.getCombinerRegion().getArguments());
       auto leftEntity = hlfir::Entity{value1};
       if (fir::isBoxAddress(value1.getType()))
-        leftEntity =
-            hlfir::Entity{builder.create<fir::LoadOp>(loc, value1).getResult()};
+        leftEntity = hlfir::Entity{
+            fir::LoadOp::create(builder, loc, value1).getResult()};
       auto left =
           genDesignateWithTriplets(builder, loc, leftEntity, triplets, shape);
       auto rightEntity = hlfir::Entity{value2};
       if (fir::isBoxAddress(value2.getType()))
-        rightEntity =
-            hlfir::Entity{builder.create<fir::LoadOp>(loc, value2).getResult()};
+        rightEntity = hlfir::Entity{
+            fir::LoadOp::create(builder, loc, value2).getResult()};
       auto right =
           genDesignateWithTriplets(builder, loc, rightEntity, triplets, shape);
 
@@ -1683,11 +1687,11 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
       mlir::Value elemental = hlfir::genElementalOp(
           loc, builder, seqTy.getEleTy(), shape, typeParams, genKernel,
           /*isUnordered=*/true);
-      builder.create<hlfir::AssignOp>(loc, elemental, value1);
+      hlfir::AssignOp::create(builder, loc, elemental, value1);
     }
   } else {
     mlir::Value res = genScalarCombiner(builder, loc, op, ty, value1, value2);
-    builder.create<fir::StoreOp>(loc, res, value1);
+    fir::StoreOp::create(builder, loc, res, value1);
   }
 }
 
@@ -1729,7 +1733,7 @@ mlir::acc::ReductionRecipeOp Fortran::lower::createOrGetReductionRecipe(
   mlir::Value v1 = recipe.getCombinerRegion().front().getArgument(0);
   mlir::Value v2 = recipe.getCombinerRegion().front().getArgument(1);
   genCombiner(builder, loc, op, ty, v1, v2, recipe, bounds, allConstantBound);
-  builder.create<mlir::acc::YieldOp>(loc, v1);
+  mlir::acc::YieldOp::create(builder, loc, v1);
   builder.restoreInsertionPoint(ip);
   return recipe;
 }
@@ -1821,7 +1825,7 @@ createRegionOp(fir::FirOpBuilder &builder, mlir::Location loc,
                llvm::SmallVector<mlir::Type> retTy = {},
                mlir::Value yieldValue = {}, mlir::TypeRange argsTy = {},
                llvm::SmallVector<mlir::Location> locs = {}) {
-  Op op = builder.create<Op>(loc, retTy, operands);
+  Op op = Op::create(builder, loc, retTy, operands);
   builder.createBlock(&op.getRegion(), op.getRegion().end(), argsTy, locs);
   mlir::Block &block = op.getRegion().back();
   builder.setInsertionPointToStart(&block);
@@ -1841,13 +1845,13 @@ createRegionOp(fir::FirOpBuilder &builder, mlir::Location loc,
 
   if (yieldValue) {
     if constexpr (std::is_same_v<Terminator, mlir::acc::YieldOp>) {
-      Terminator yieldOp = builder.create<Terminator>(returnLoc, yieldValue);
+      Terminator yieldOp = Terminator::create(builder, returnLoc, yieldValue);
       yieldValue.getDefiningOp()->moveBefore(yieldOp);
     } else {
-      builder.create<Terminator>(returnLoc);
+      Terminator::create(builder, returnLoc);
     }
   } else {
-    builder.create<Terminator>(returnLoc);
+    Terminator::create(builder, returnLoc);
   }
   builder.setInsertionPointToStart(&block);
   return op;
@@ -2437,7 +2441,7 @@ static mlir::acc::LoopOp createLoopOp(
 
   for (auto [arg, value] : llvm::zip(
            loopOp.getLoopRegions().front()->front().getArguments(), ivPrivate))
-    builder.create<fir::StoreOp>(currentLocation, arg, value);
+    fir::StoreOp::create(builder, currentLocation, arg, value);
 
   loopOp.setInclusiveUpperbound(inclusiveBounds);
 
@@ -3750,8 +3754,8 @@ genACCUpdateOp(Fortran::lower::AbstractConverter &converter,
 
   dataClauseOperands.append(updateHostOperands);
 
-  builder.create<mlir::acc::UpdateOp>(
-      currentLocation, ifCond, asyncOperands,
+  mlir::acc::UpdateOp::create(
+      builder, currentLocation, ifCond, asyncOperands,
       getArrayAttr(builder, asyncOperandsDeviceTypes),
       getArrayAttr(builder, asyncOnlyDeviceTypes), waitOperands,
       getDenseI32ArrayAttr(builder, waitOperandsSegments),
@@ -3873,13 +3877,14 @@ static void createDeclareGlobalOp(mlir::OpBuilder &modBuilder,
                                   const std::string &declareGlobalName,
                                   bool implicit, std::stringstream &asFortran) {
   GlobalOp declareGlobalOp =
-      modBuilder.create<GlobalOp>(loc, declareGlobalName);
+      GlobalOp::create(modBuilder, loc, declareGlobalName);
   builder.createBlock(&declareGlobalOp.getRegion(),
                       declareGlobalOp.getRegion().end(), {}, {});
   builder.setInsertionPointToEnd(&declareGlobalOp.getRegion().back());
 
-  fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
-      loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(
+      builder, loc, fir::ReferenceType::get(globalOp.getType()),
+      globalOp.getSymbol());
   addDeclareAttr(builder, addrOp, clause);
 
   llvm::SmallVector<mlir::Value> bounds;
@@ -3888,21 +3893,21 @@ static void createDeclareGlobalOp(mlir::OpBuilder &modBuilder,
       /*structured=*/false, implicit, clause, addrOp.getResTy().getType(),
       /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
   if constexpr (std::is_same_v<DeclareOp, mlir::acc::DeclareEnterOp>)
-    builder.create<DeclareOp>(
-        loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
-        mlir::ValueRange(entryOp.getAccVar()));
+    DeclareOp::create(builder, loc,
+                      mlir::acc::DeclareTokenType::get(entryOp.getContext()),
+                      mlir::ValueRange(entryOp.getAccVar()));
   else
-    builder.create<DeclareOp>(loc, mlir::Value{},
-                              mlir::ValueRange(entryOp.getAccVar()));
+    DeclareOp::create(builder, loc, mlir::Value{},
+                      mlir::ValueRange(entryOp.getAccVar()));
   if constexpr (std::is_same_v<GlobalOp, mlir::acc::GlobalDestructorOp>) {
-    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccVar(),
-                           entryOp.getBounds(), entryOp.getAsyncOperands(),
-                           entryOp.getAsyncOperandsDeviceTypeAttr(),
-                           entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-                           /*structured=*/false, /*implicit=*/false,
-                           builder.getStringAttr(*entryOp.getName()));
+    ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
+                   entryOp.getBounds(), entryOp.getAsyncOperands(),
+                   entryOp.getAsyncOperandsDeviceTypeAttr(),
+                   entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                   /*structured=*/false, /*implicit=*/false,
+                   builder.getStringAttr(*entryOp.getName()));
   }
-  builder.create<mlir::acc::TerminatorOp>(loc);
+  mlir::acc::TerminatorOp::create(builder, loc);
   modBuilder.setInsertionPointAfter(declareGlobalOp);
 }
 
@@ -3917,8 +3922,9 @@ static void createDeclareAllocFunc(mlir::OpBuilder &modBuilder,
   auto registerFuncOp =
       createDeclareFunc(modBuilder, builder, loc, registerFuncName.str());
 
-  fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
-      loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(
+      builder, loc, fir::ReferenceType::get(globalOp.getType()),
+      globalOp.getSymbol());
 
   std::stringstream asFortran;
   asFortran << Fortran::lower::mangle::demangleName(globalOp.getSymName());
@@ -3941,15 +3947,15 @@ static void createDeclareAllocFunc(mlir::OpBuilder &modBuilder,
   createSimpleOp<mlir::acc::UpdateOp>(builder, loc, operands, operandSegments);
 
   if (unwrapFirBox) {
-    auto loadOp = builder.create<fir::LoadOp>(loc, addrOp.getResult());
-    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+    auto loadOp = fir::LoadOp::create(builder, loc, addrOp.getResult());
+    fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, loadOp);
     addDeclareAttr(builder, boxAddrOp.getOperation(), clause);
     EntryOp entryOp = createDataEntryOp<EntryOp>(
         builder, loc, boxAddrOp.getResult(), asFortran, bounds,
         /*structured=*/false, /*implicit=*/false, clause, boxAddrOp.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
-    builder.create<mlir::acc::DeclareEnterOp>(
-        loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
+    mlir::acc::DeclareEnterOp::create(
+        builder, loc, mlir::acc::DeclareTokenType::get(entryOp.getContext()),
         mlir::ValueRange(entryOp.getAccVar()));
   }
 
@@ -3980,10 +3986,11 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
     auto preDeallocOp =
         createDeclareFunc(modBuilder, builder, loc, preDeallocFuncName.str());
 
-    fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
-        loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
-    auto loadOp = builder.create<fir::LoadOp>(loc, addrOp.getResult());
-    fir::BoxAddrOp boxAddrOp = builder.create<fir::BoxAddrOp>(loc, loadOp);
+    fir::AddrOfOp addrOp = fir::AddrOfOp::create(
+        builder, loc, fir::ReferenceType::get(globalOp.getType()),
+        globalOp.getSymbol());
+    auto loadOp = fir::LoadOp::create(builder, loc, addrOp.getResult());
+    fir::BoxAddrOp boxAddrOp = fir::BoxAddrOp::create(builder, loc, loadOp);
     mlir::Value var = boxAddrOp.getResult();
     addDeclareAttr(builder, var.getDefiningOp(), clause);
 
@@ -3994,25 +4001,25 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
             /*structured=*/false, /*implicit=*/false, clause, var.getType(),
             /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
 
-    builder.create<mlir::acc::DeclareExitOp>(
-        loc, mlir::Value{}, mlir::ValueRange(entryOp.getAccVar()));
+    mlir::acc::DeclareExitOp::create(builder, loc, mlir::Value{},
+                                     mlir::ValueRange(entryOp.getAccVar()));
 
     if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                   std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
-      builder.create<ExitOp>(
-          entryOp.getLoc(), entryOp.getAccVar(), entryOp.getVar(),
-          entryOp.getBounds(), entryOp.getAsyncOperands(),
-          entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
-          entryOp.getDataClause(),
-          /*structured=*/false, /*implicit=*/false,
-          builder.getStringAttr(*entryOp.getName()));
+      ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
+                     entryOp.getVar(), entryOp.getBounds(),
+                     entryOp.getAsyncOperands(),
+                     entryOp.getAsyncOperandsDeviceTypeAttr(),
+                     entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                     /*structured=*/false, /*implicit=*/false,
+                     builder.getStringAttr(*entryOp.getName()));
     else
-      builder.create<ExitOp>(
-          entryOp.getLoc(), entryOp.getAccVar(), entryOp.getBounds(),
-          entryOp.getAsyncOperands(), entryOp.getAsyncOperandsDeviceTypeAttr(),
-          entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
-          /*structured=*/false, /*implicit=*/false,
-          builder.getStringAttr(*entryOp.getName()));
+      ExitOp::create(builder, entryOp.getLoc(), entryOp.getAccVar(),
+                     entryOp.getBounds(), entryOp.getAsyncOperands(),
+                     entryOp.getAsyncOperandsDeviceTypeAttr(),
+                     entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
+                     /*structured=*/false, /*implicit=*/false,
+                     builder.getStringAttr(*entryOp.getName()));
 
     // Generate the post dealloc function.
     modBuilder.setInsertionPointAfter(preDeallocOp);
@@ -4024,8 +4031,9 @@ static void createDeclareDeallocFunc(mlir::OpBuilder &modBuilder,
   auto postDeallocOp =
       createDeclareFunc(modBuilder, builder, loc, postDeallocFuncName.str());
 
-  fir::AddrOfOp addrOp = builder.create<fir::AddrOfOp>(
-      loc, fir::ReferenceType::get(globalOp.getType()), globalOp.getSymbol());
+  fir::AddrOfOp addrOp = fir::AddrOfOp::create(
+      builder, loc, fir::ReferenceType::get(globalOp.getType()),
+      globalOp.getSymbol());
   if (unwrapFirBox)
     asFortran << accFirDescriptorPostfix.str();
   llvm::SmallVector<mlir::Value> bounds;
@@ -4262,13 +4270,13 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
   auto ops = funcOp.getOps<mlir::acc::DeclareEnterOp>();
   mlir::Value declareToken;
   if (ops.empty()) {
-    declareToken = builder.create<mlir::acc::DeclareEnterOp>(
-        loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
+    declareToken = mlir::acc::DeclareEnterOp::create(
+        builder, loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
         dataClauseOperands);
   } else {
     auto declareOp = *ops.begin();
-    auto newDeclareOp = builder.create<mlir::acc::DeclareEnterOp>(
-        loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
+    auto newDeclareOp = mlir::acc::DeclareEnterOp::create(
+        builder, loc, mlir::acc::DeclareTokenType::get(builder.getContext()),
         declareOp.getDataClauseOperands());
     newDeclareOp.getDataClauseOperandsMutable().append(dataClauseOperands);
     declareToken = newDeclareOp.getToken();
@@ -4290,7 +4298,7 @@ genDeclareInFunction(Fortran::lower::AbstractConverter &converter,
     mlir::func::FuncOp funcOp = builder.getFunction();
     auto ops = funcOp.getOps<mlir::acc::DeclareExitOp>();
     if (ops.empty()) {
-      builder.create<mlir::acc::DeclareExitOp>(loc, declareToken, operands);
+      mlir::acc::DeclareExitOp::create(builder, loc, declareToken, operands);
     } else {
       auto declareOp = *ops.begin();
       declareOp.getDataClauseOperandsMutable().append(operands);
@@ -4535,8 +4543,8 @@ void createOpenACCRoutineConstruct(
   std::string routineOpStr = routineOpName.str();
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  modBuilder.create<mlir::acc::RoutineOp>(
-      loc, routineOpStr,
+  mlir::acc::RoutineOp::create(
+      modBuilder, loc, routineOpStr,
       mlir::SymbolRefAttr::get(builder.getContext(), funcName),
       getArrayAttrOrNull(builder, bindIdNames),
       getArrayAttrOrNull(builder, bindStrNames),
@@ -4880,9 +4888,9 @@ void Fortran::lower::genOpenACCTerminator(fir::FirOpBuilder &builder,
                                           mlir::Operation *op,
                                           mlir::Location loc) {
   if (mlir::isa<mlir::acc::ParallelOp, mlir::acc::LoopOp>(op))
-    builder.create<mlir::acc::YieldOp>(loc);
+    mlir::acc::YieldOp::create(builder, loc);
   else
-    builder.create<mlir::acc::TerminatorOp>(loc);
+    mlir::acc::TerminatorOp::create(builder, loc);
 }
 
 bool Fortran::lower::isInOpenACCLoop(fir::FirOpBuilder &builder) {
@@ -4902,7 +4910,7 @@ void Fortran::lower::genEarlyReturnInOpenACCLoop(fir::FirOpBuilder &builder,
                                                  mlir::Location loc) {
   mlir::Value yieldValue =
       builder.createIntegerConstant(loc, builder.getI1Type(), 1);
-  builder.create<mlir::acc::YieldOp>(loc, yieldValue);
+  mlir::acc::YieldOp::create(builder, loc, yieldValue);
 }
 
 int64_t Fortran::lower::getLoopCountForCollapseAndTile(

diff  --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index 6ea331c370640..9a233d2d8cb08 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -528,8 +528,8 @@ genAtomicRead(lower::AbstractConverter &converter,
   }();
 
   builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
-      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
+  mlir::Operation *op = mlir::omp::AtomicReadOp::create(
+      builder, loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
       makeMemOrderAttr(converter, memOrder));
 
   if (atomType != storeType) {
@@ -537,7 +537,7 @@ genAtomicRead(lower::AbstractConverter &converter,
     // The READ operation could be a part of UPDATE CAPTURE, so make sure
     // we don't emit extra code into the body of the atomic op.
     builder.restoreInsertionPoint(postAt);
-    mlir::Value load = builder.create<fir::LoadOp>(loc, toAddr);
+    mlir::Value load = fir::LoadOp::create(builder, loc, toAddr);
     overrides.try_emplace(&atom, load);
 
     converter.overrideExprValues(&overrides);
@@ -545,7 +545,7 @@ genAtomicRead(lower::AbstractConverter &converter,
         fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
     converter.resetExprOverrides();
 
-    builder.create<fir::StoreOp>(loc, value, storeAddr);
+    fir::StoreOp::create(builder, loc, value, storeAddr);
   }
   return op;
 }
@@ -581,8 +581,9 @@ genAtomicWrite(lower::AbstractConverter &converter,
   mlir::Value converted = builder.createConvert(loc, atomType, value);
 
   builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
-      loc, atomAddr, converted, hint, makeMemOrderAttr(converter, memOrder));
+  mlir::Operation *op =
+      mlir::omp::AtomicWriteOp::create(builder, loc, atomAddr, converted, hint,
+                                       makeMemOrderAttr(converter, memOrder));
   return op;
 }
 
@@ -635,8 +636,8 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   }
 
   builder.restoreInsertionPoint(atomicAt);
-  auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+  auto updateOp = mlir::omp::AtomicUpdateOp::create(
+      builder, loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
@@ -647,7 +648,7 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Value updated =
       fir::getBase(converter.genExprValue(rhs, stmtCtx, &loc));
   mlir::Value converted = builder.createConvert(loc, atomType, updated);
-  builder.create<mlir::omp::YieldOp>(loc, converted);
+  mlir::omp::YieldOp::create(builder, loc, converted);
   converter.resetExprOverrides();
 
   builder.restoreInsertionPoint(postAt); // For naCtx cleanups
@@ -731,8 +732,8 @@ void Fortran::lower::omp::lowerAtomic(
              "Expexcing two actions");
       (void)action0;
       (void)action1;
-      captureOp = builder.create<mlir::omp::AtomicCaptureOp>(
-          loc, hint, makeMemOrderAttr(converter, memOrder));
+      captureOp = mlir::omp::AtomicCaptureOp::create(
+          builder, loc, hint, makeMemOrderAttr(converter, memOrder));
       // Set the non-atomic insertion point to before the atomic.capture.
       preAt = getInsertionPointBefore(captureOp);
 
@@ -740,7 +741,7 @@ void Fortran::lower::omp::lowerAtomic(
       builder.setInsertionPointToEnd(block);
       // Set the atomic insertion point to before the terminator inside
       // atomic.capture.
-      mlir::Operation *term = builder.create<mlir::omp::TerminatorOp>(loc);
+      mlir::Operation *term = mlir::omp::TerminatorOp::create(builder, loc);
       atomicAt = getInsertionPointBefore(term);
       postAt = getInsertionPointAfter(captureOp);
       hint = nullptr;

diff  --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 74087d42a8e6e..498eea471f3db 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -723,7 +723,7 @@ bool ClauseProcessor::processCopyin() const {
   // barrier is inserted following all of them.
   firOpBuilder.restoreInsertionPoint(insPt);
   if (hasCopyin)
-    firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
+    mlir::omp::BarrierOp::create(firOpBuilder, converter.getCurrentLocation());
   return hasCopyin;
 }
 
@@ -803,7 +803,7 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
   llvm::SmallVector<mlir::Type> argsTy = {varType, varType};
   auto funcType = mlir::FunctionType::get(builder.getContext(), argsTy, {});
   mlir::func::FuncOp funcOp =
-      modBuilder.create<mlir::func::FuncOp>(loc, copyFuncName, funcType);
+      mlir::func::FuncOp::create(modBuilder, loc, copyFuncName, funcType);
   funcOp.setVisibility(mlir::SymbolTable::Visibility::Private);
   fir::factory::setInternalLinkage(funcOp);
   builder.createBlock(&funcOp.getRegion(), funcOp.getRegion().end(), argsTy,
@@ -819,22 +819,22 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
     for (auto extent : typeInfo.getShape())
       extents.push_back(
           builder.createIntegerConstant(loc, builder.getIndexType(), extent));
-    shape = builder.create<fir::ShapeOp>(loc, extents);
+    shape = fir::ShapeOp::create(builder, loc, extents);
   }
   mlir::Value dst = funcOp.getArgument(0);
   mlir::Value src = funcOp.getArgument(1);
   llvm::SmallVector<mlir::Value> typeparams;
   if (typeInfo.isBoxChar()) {
     // fir.boxchar will be passed here as fir.ref<fir.boxchar>
-    auto loadDst = builder.create<fir::LoadOp>(loc, dst);
-    auto loadSrc = builder.create<fir::LoadOp>(loc, src);
+    auto loadDst = fir::LoadOp::create(builder, loc, dst);
+    auto loadSrc = fir::LoadOp::create(builder, loc, src);
     // get the actual fir.ref<fir.char> type
     mlir::Type refType =
         fir::ReferenceType::get(mlir::cast<fir::BoxCharType>(eleTy).getEleTy());
-    auto unboxedDst = builder.create<fir::UnboxCharOp>(
-        loc, refType, builder.getIndexType(), loadDst);
-    auto unboxedSrc = builder.create<fir::UnboxCharOp>(
-        loc, refType, builder.getIndexType(), loadSrc);
+    auto unboxedDst = fir::UnboxCharOp::create(builder, loc, refType,
+                                               builder.getIndexType(), loadDst);
+    auto unboxedSrc = fir::UnboxCharOp::create(builder, loc, refType,
+                                               builder.getIndexType(), loadSrc);
     // Add length to type parameters
     typeparams.push_back(unboxedDst.getResult(1));
     dst = unboxedDst.getResult(0);
@@ -844,14 +844,14 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
         loc, builder.getCharacterLengthType(), *typeInfo.getCharLength());
     typeparams.push_back(charLen);
   }
-  auto declDst = builder.create<hlfir::DeclareOp>(
-      loc, dst, copyFuncName + "_dst", shape, typeparams,
+  auto declDst = hlfir::DeclareOp::create(
+      builder, loc, dst, copyFuncName + "_dst", shape, typeparams,
       /*dummy_scope=*/nullptr, attrs);
-  auto declSrc = builder.create<hlfir::DeclareOp>(
-      loc, src, copyFuncName + "_src", shape, typeparams,
+  auto declSrc = hlfir::DeclareOp::create(
+      builder, loc, src, copyFuncName + "_src", shape, typeparams,
       /*dummy_scope=*/nullptr, attrs);
   converter.copyVar(loc, declDst.getBase(), declSrc.getBase(), varAttrs);
-  builder.create<mlir::func::ReturnOp>(loc);
+  mlir::func::ReturnOp::create(builder, loc);
   return funcOp;
 }
 
@@ -882,8 +882,8 @@ bool ClauseProcessor::processCopyprivate(
     if (mlir::isa<fir::BaseBoxType>(symType) ||
         mlir::isa<fir::BoxCharType>(symType)) {
       fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-      auto alloca = builder.create<fir::AllocaOp>(currentLocation, symType);
-      builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
+      auto alloca = fir::AllocaOp::create(builder, currentLocation, symType);
+      fir::StoreOp::create(builder, currentLocation, symVal, alloca);
       cpVar = alloca;
     }
 
@@ -1002,8 +1002,8 @@ bool ClauseProcessor::processDepend(lower::SymMap &symMap,
       // allocations so this is not a reliable way to identify the dependency.
       if (auto ref = mlir::dyn_cast<fir::ReferenceType>(dependVar.getType()))
         if (fir::isa_box_type(ref.getElementType()))
-          dependVar = builder.create<fir::LoadOp>(
-              converter.getCurrentLocation(), dependVar);
+          dependVar = fir::LoadOp::create(
+              builder, converter.getCurrentLocation(), dependVar);
 
       // The openmp dialect doesn't know what to do with boxes (and it would
       // break layering to teach it about them). The dependency variable can be
@@ -1012,8 +1012,8 @@ bool ClauseProcessor::processDepend(lower::SymMap &symMap,
       // Getting the address of the box data is okay because all the runtime
       // ultimately cares about is the base address of the array.
       if (fir::isa_box_type(dependVar.getType()))
-        dependVar = builder.create<fir::BoxAddrOp>(
-            converter.getCurrentLocation(), dependVar);
+        dependVar = fir::BoxAddrOp::create(
+            builder, converter.getCurrentLocation(), dependVar);
 
       result.dependVars.push_back(dependVar);
     }

diff  --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 675a58e4f35a1..11e488371b886 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -291,7 +291,7 @@ void DataSharingProcessor::insertBarrier(
       clauseOps->privateNeedsBarrier =
           mlir::UnitAttr::get(&converter.getMLIRContext());
   } else {
-    firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
+    mlir::omp::BarrierOp::create(firOpBuilder, converter.getCurrentLocation());
   }
 }
 
@@ -351,32 +351,32 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
              loopOp.getIVs(), result.loopUpperBounds, result.loopSteps)) {
       // v = iv + step
       // cmp = step < 0 ? v < ub : v > ub
-      mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
+      mlir::Value v = mlir::arith::AddIOp::create(firOpBuilder, loc, iv, step);
       vs.push_back(v);
       mlir::Value zero =
           firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
-      mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::slt, step, zero);
-      mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::slt, v, ub);
-      mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
-          loc, mlir::arith::CmpIPredicate::sgt, v, ub);
-      mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
-          loc, negativeStep, vLT, vGT);
+      mlir::Value negativeStep = mlir::arith::CmpIOp::create(
+          firOpBuilder, loc, mlir::arith::CmpIPredicate::slt, step, zero);
+      mlir::Value vLT = mlir::arith::CmpIOp::create(
+          firOpBuilder, loc, mlir::arith::CmpIPredicate::slt, v, ub);
+      mlir::Value vGT = mlir::arith::CmpIOp::create(
+          firOpBuilder, loc, mlir::arith::CmpIPredicate::sgt, v, ub);
+      mlir::Value icmpOp = mlir::arith::SelectOp::create(
+          firOpBuilder, loc, negativeStep, vLT, vGT);
 
       if (cmpOp)
-        cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
+        cmpOp = mlir::arith::AndIOp::create(firOpBuilder, loc, cmpOp, icmpOp);
       else
         cmpOp = icmpOp;
     }
 
-    auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
+    auto ifOp = fir::IfOp::create(firOpBuilder, loc, cmpOp, /*else*/ false);
     firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
       hlfir::Entity loopIVEntity{loopIV};
       loopIVEntity =
           hlfir::derefPointersAndAllocatables(loc, firOpBuilder, loopIVEntity);
-      firOpBuilder.create<hlfir::AssignOp>(loc, v, loopIVEntity);
+      hlfir::AssignOp::create(firOpBuilder, loc, v, loopIVEntity);
     }
     lastPrivIP = firOpBuilder.saveInsertionPoint();
   } else if (mlir::isa<mlir::omp::SectionsOp>(op)) {

diff  --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index fcb20fdf187ff..80905aa2979a8 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -642,8 +642,8 @@ static void threadPrivatizeVars(lower::AbstractConverter &converter,
       op = declOp.getMemref().getDefiningOp();
     if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
       symValue = mlir::dyn_cast<mlir::omp::ThreadprivateOp>(op).getSymAddr();
-    return firOpBuilder.create<mlir::omp::ThreadprivateOp>(
-        currentLocation, symValue.getType(), symValue);
+    return mlir::omp::ThreadprivateOp::create(firOpBuilder, currentLocation,
+                                              symValue.getType(), symValue);
   };
 
   llvm::SetVector<const semantics::Symbol *> threadprivateSyms;
@@ -710,7 +710,7 @@ createAndSetPrivatizedLoopVar(lower::AbstractConverter &converter,
   lhs = hlfir::derefPointersAndAllocatables(loc, firOpBuilder, lhs);
 
   mlir::Operation *storeOp =
-      firOpBuilder.create<hlfir::AssignOp>(loc, cvtVal, lhs);
+      hlfir::AssignOp::create(firOpBuilder, loc, cvtVal, lhs);
   return storeOp;
 }
 
@@ -1156,8 +1156,8 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
   fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder();
 
   auto insertMarker = [](fir::FirOpBuilder &builder) {
-    mlir::Value undef = builder.create<fir::UndefOp>(builder.getUnknownLoc(),
-                                                     builder.getIndexType());
+    mlir::Value undef = fir::UndefOp::create(builder, builder.getUnknownLoc(),
+                                             builder.getIndexType());
     return undef.getDefiningOp();
   };
 
@@ -1271,7 +1271,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
     mlir::Block *exit = firOpBuilder.createBlock(&region);
     for (mlir::Block *b : exits) {
       firOpBuilder.setInsertionPointToEnd(b);
-      firOpBuilder.create<mlir::cf::BranchOp>(info.loc, exit);
+      mlir::cf::BranchOp::create(firOpBuilder, info.loc, exit);
     }
     return exit;
   };
@@ -1332,8 +1332,8 @@ static void genBodyOfTargetDataOp(
   // Remembering the position for further insertion is important since
   // there are hlfir.declares inserted above while setting block arguments
   // and new code from the body should be inserted after that.
-  mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
-      dataOp.getLoc(), firOpBuilder.getIndexType());
+  mlir::Value undefMarker = fir::UndefOp::create(firOpBuilder, dataOp.getLoc(),
+                                                 firOpBuilder.getIndexType());
 
   // Create blocks for unstructured regions. This has to be done since
   // blocks are initially allocated with the function as the parent region.
@@ -1342,7 +1342,7 @@ static void genBodyOfTargetDataOp(
         firOpBuilder, eval.getNestedEvaluations());
   }
 
-  firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
+  mlir::omp::TerminatorOp::create(firOpBuilder, currentLocation);
 
   // Set the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
@@ -1496,8 +1496,8 @@ static void genBodyOfTargetOp(
             insertIndex, copyVal.getType(), copyVal.getLoc());
 
         firOpBuilder.setInsertionPointToStart(entryBlock);
-        auto loadOp = firOpBuilder.create<fir::LoadOp>(clonedValArg.getLoc(),
-                                                       clonedValArg);
+        auto loadOp = fir::LoadOp::create(firOpBuilder, clonedValArg.getLoc(),
+                                          clonedValArg);
         val.replaceUsesWithIf(loadOp->getResult(0),
                               [entryBlock](mlir::OpOperand &use) {
                                 return use.getOwner()->getBlock() == entryBlock;
@@ -1513,8 +1513,8 @@ static void genBodyOfTargetOp(
   // marker will be deleted since there are not uses.
   // In the HLFIR flow there are hlfir.declares inserted above while
   // setting block arguments.
-  mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
-      targetOp.getLoc(), firOpBuilder.getIndexType());
+  mlir::Value undefMarker = fir::UndefOp::create(
+      firOpBuilder, targetOp.getLoc(), firOpBuilder.getIndexType());
 
   // Create blocks for unstructured regions. This has to be done since
   // blocks are initially allocated with the function as the parent region.
@@ -1524,7 +1524,7 @@ static void genBodyOfTargetOp(
         firOpBuilder, eval.getNestedEvaluations());
   }
 
-  firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
+  mlir::omp::TerminatorOp::create(firOpBuilder, currentLocation);
 
   // Create the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
@@ -1570,7 +1570,7 @@ static OpTy genWrapperOp(lower::AbstractConverter &converter,
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
   // Create wrapper.
-  auto op = firOpBuilder.create<OpTy>(loc, clauseOps);
+  auto op = OpTy::create(firOpBuilder, loc, clauseOps);
 
   // Create entry block with arguments.
   genEntryBlock(firOpBuilder, args, op.getRegion());
@@ -1983,7 +1983,7 @@ genCriticalOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
                                 clauseOps, nameStr);
 
       mlir::OpBuilder modBuilder(mod.getBodyRegion());
-      global = modBuilder.create<mlir::omp::CriticalDeclareOp>(loc, clauseOps);
+      global = mlir::omp::CriticalDeclareOp::create(modBuilder, loc, clauseOps);
     }
     nameAttr = mlir::FlatSymbolRefAttr::get(firOpBuilder.getContext(),
                                             global.getSymName());
@@ -2201,7 +2201,7 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   }
 
   // SECTIONS construct.
-  auto sectionsOp = builder.create<mlir::omp::SectionsOp>(loc, clauseOps);
+  auto sectionsOp = mlir::omp::SectionsOp::create(builder, loc, clauseOps);
 
   // Create entry block with reduction variables as arguments.
   EntryBlockArgs args;
@@ -2277,7 +2277,7 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   // races on post-update of lastprivate variables when `nowait`
   // clause is present.
   if (clauseOps.nowait && !lastprivates.empty())
-    builder.create<mlir::omp::BarrierOp>(loc);
+    mlir::omp::BarrierOp::create(builder, loc);
 
   return sectionsOp;
 }
@@ -2429,7 +2429,7 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   };
   lower::pft::visitAllSymbols(eval, captureImplicitMap);
 
-  auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
+  auto targetOp = mlir::omp::TargetOp::create(firOpBuilder, loc, clauseOps);
 
   llvm::SmallVector<mlir::Value> hasDeviceAddrBaseValues, mapBaseValues;
   extractMappedBaseValues(clauseOps.hasDeviceAddrVars, hasDeviceAddrBaseValues);
@@ -2509,7 +2509,7 @@ static OpTy genTargetEnterExitUpdateDataOp(
   genTargetEnterExitUpdateDataClauses(converter, semaCtx, symTable, stmtCtx,
                                       item->clauses, loc, directive, clauseOps);
 
-  return firOpBuilder.create<OpTy>(loc, clauseOps);
+  return OpTy::create(firOpBuilder, loc, clauseOps);
 }
 
 static mlir::omp::TaskOp
@@ -3342,8 +3342,8 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
   auto mlirType = converter.genType(varType.declTypeSpec->derivedTypeSpec());
-  auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
-      loc, mapperNameStr, mlirType);
+  auto declMapperOp = mlir::omp::DeclareMapperOp::create(
+      firOpBuilder, loc, mapperNameStr, mlirType);
   auto &region = declMapperOp.getRegion();
   firOpBuilder.createBlock(&region);
   auto varVal = region.addArgument(firOpBuilder.getRefType(mlirType), loc);
@@ -3356,7 +3356,7 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   List<Clause> clauses = makeClauses(*clauseList, semaCtx);
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processMap(loc, stmtCtx, clauseOps);
-  firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
+  mlir::omp::DeclareMapperInfoOp::create(firOpBuilder, loc, clauseOps.mapVars);
 }
 
 static void
@@ -3758,8 +3758,8 @@ mlir::Operation *Fortran::lower::genOpenMPTerminator(fir::FirOpBuilder &builder,
                                                      mlir::Location loc) {
   if (mlir::isa<mlir::omp::AtomicUpdateOp, mlir::omp::DeclareReductionOp,
                 mlir::omp::LoopNestOp>(op))
-    return builder.create<mlir::omp::YieldOp>(loc);
-  return builder.create<mlir::omp::TerminatorOp>(loc);
+    return mlir::omp::YieldOp::create(builder, loc);
+  return mlir::omp::TerminatorOp::create(builder, loc);
 }
 
 void Fortran::lower::genOpenMPConstruct(lower::AbstractConverter &converter,
@@ -3819,9 +3819,8 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
       return;
     }
     // Generate ThreadprivateOp and rebind the common block.
-    mlir::Value commonThreadprivateValue =
-        firOpBuilder.create<mlir::omp::ThreadprivateOp>(
-            currentLocation, commonValue.getType(), commonValue);
+    mlir::Value commonThreadprivateValue = mlir::omp::ThreadprivateOp::create(
+        firOpBuilder, currentLocation, commonValue.getType(), commonValue);
     converter.bindSymbol(*common, commonThreadprivateValue);
     // Generate the threadprivate value for the common block member.
     symThreadprivateValue = genCommonBlockMember(converter, currentLocation,
@@ -3841,10 +3840,10 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
       global = globalInitialization(converter, firOpBuilder, sym, var,
                                     currentLocation);
 
-    mlir::Value symValue = firOpBuilder.create<fir::AddrOfOp>(
-        currentLocation, global.resultType(), global.getSymbol());
-    symThreadprivateValue = firOpBuilder.create<mlir::omp::ThreadprivateOp>(
-        currentLocation, symValue.getType(), symValue);
+    mlir::Value symValue = fir::AddrOfOp::create(
+        firOpBuilder, currentLocation, global.resultType(), global.getSymbol());
+    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(
+        firOpBuilder, currentLocation, symValue.getType(), symValue);
   } else {
     mlir::Value symValue = converter.getSymbolAddress(sym);
 
@@ -3859,8 +3858,8 @@ void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
     if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
       return;
 
-    symThreadprivateValue = firOpBuilder.create<mlir::omp::ThreadprivateOp>(
-        currentLocation, symValue.getType(), symValue);
+    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(
+        firOpBuilder, currentLocation, symValue.getType(), symValue);
   }
 
   fir::ExtendedValue sexv = converter.getSymbolExtendedValue(sym);

diff  --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index b194150c0f7f0..b1716d6afb200 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -115,7 +115,7 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
                 mlir::omp::VariableCaptureKind mapCaptureType, mlir::Type retTy,
                 bool partialMap, mlir::FlatSymbolRefAttr mapperId) {
   if (auto boxTy = llvm::dyn_cast<fir::BaseBoxType>(baseAddr.getType())) {
-    baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
+    baseAddr = fir::BoxAddrOp::create(builder, loc, baseAddr);
     retTy = baseAddr.getType();
   }
 
@@ -129,8 +129,8 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
     if (seqType.hasDynamicExtents())
       varType = mlir::TypeAttr::get(seqType.getEleTy());
 
-  mlir::omp::MapInfoOp op = builder.create<mlir::omp::MapInfoOp>(
-      loc, retTy, baseAddr, varType,
+  mlir::omp::MapInfoOp op = mlir::omp::MapInfoOp::create(
+      builder, loc, retTy, baseAddr, varType,
       builder.getIntegerAttr(builder.getIntegerType(64, false), mapType),
       builder.getAttr<mlir::omp::VariableCaptureKindAttr>(mapCaptureType),
       varPtrPtr, members, membersIndex, bounds, mapperId,
@@ -195,8 +195,8 @@ static void generateArrayIndices(lower::AbstractConverter &converter,
         clauseLocation, firOpBuilder.getIndexType(), 1);
     subscript = firOpBuilder.createConvert(
         clauseLocation, firOpBuilder.getIndexType(), subscript);
-    indices.push_back(firOpBuilder.create<mlir::arith::SubIOp>(clauseLocation,
-                                                               subscript, one));
+    indices.push_back(mlir::arith::SubIOp::create(firOpBuilder, clauseLocation,
+                                                  subscript, one));
   }
 }
 
@@ -329,9 +329,10 @@ mlir::Value createParentSymAndGenIntermediateMaps(
                              subscriptIndices, objectList[i]);
         assert(!subscriptIndices.empty() &&
                "missing expected indices for map clause");
-        curValue = firOpBuilder.create<fir::CoordinateOp>(
-            clauseLocation, firOpBuilder.getRefType(arrType.getEleTy()),
-            curValue, subscriptIndices);
+        curValue = fir::CoordinateOp::create(
+            firOpBuilder, clauseLocation,
+            firOpBuilder.getRefType(arrType.getEleTy()), curValue,
+            subscriptIndices);
       }
     }
 
@@ -345,9 +346,9 @@ mlir::Value createParentSymAndGenIntermediateMaps(
       fir::IntOrValue idxConst = mlir::IntegerAttr::get(
           firOpBuilder.getI32Type(), indices[currentIndicesIdx]);
       mlir::Type memberTy = recordType.getType(indices[currentIndicesIdx]);
-      curValue = firOpBuilder.create<fir::CoordinateOp>(
-          clauseLocation, firOpBuilder.getRefType(memberTy), curValue,
-          llvm::SmallVector<fir::IntOrValue, 1>{idxConst});
+      curValue = fir::CoordinateOp::create(
+          firOpBuilder, clauseLocation, firOpBuilder.getRefType(memberTy),
+          curValue, llvm::SmallVector<fir::IntOrValue, 1>{idxConst});
 
       // If we're a final member, the map will be generated by the processMap
       // call that invoked this function.
@@ -417,7 +418,7 @@ mlir::Value createParentSymAndGenIntermediateMaps(
 
       // Load the currently accessed member, so we can continue to access
       // further segments.
-      curValue = firOpBuilder.create<fir::LoadOp>(clauseLocation, curValue);
+      curValue = fir::LoadOp::create(firOpBuilder, clauseLocation, curValue);
       currentIndicesIdx++;
     }
   }

diff  --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index ae8bf0e1630aa..fc59a2414d539 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -43,7 +43,7 @@ static void genUnreachable(fir::FirOpBuilder &builder, mlir::Location loc) {
            mlir::acc::OpenACCDialect::getDialectNamespace())
     Fortran::lower::genOpenACCTerminator(builder, parentOp, loc);
   else
-    builder.create<fir::UnreachableOp>(loc);
+    fir::UnreachableOp::create(builder, loc);
   mlir::Block *newBlock = curBlock->splitBlock(builder.getInsertionPoint());
   builder.setInsertionPointToStart(newBlock);
 }
@@ -118,7 +118,7 @@ void Fortran::lower::genStopStatement(
         loc, calleeType.getInput(operands.size()), 0));
   }
 
-  builder.create<fir::CallOp>(loc, callee, operands);
+  fir::CallOp::create(builder, loc, callee, operands);
   auto blockIsUnterminated = [&builder]() {
     mlir::Block *currentBlock = builder.getBlock();
     return currentBlock->empty() ||
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
+  fir::CallOp::create(builder, loc, callee, mlir::ValueRange{});
   genUnreachable(builder, loc);
 }
 
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
   mlir::Location loc = converter.getCurrentLocation();
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
-  builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
+  fir::CallOp::create(builder, loc, callee, mlir::ValueRange{});
 }
 
 void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
@@ -210,7 +210,7 @@ void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(PointerAssociate)>(loc, builder);
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, func.getFunctionType(), pointer, target);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void Fortran::lower::genPointerAssociateRemapping(
@@ -229,7 +229,7 @@ void Fortran::lower::genPointerAssociateRemapping(
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, func.getFunctionType(), pointer, target, bounds, sourceFile,
       sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void Fortran::lower::genPointerAssociateLowerBounds(fir::FirOpBuilder &builder,
@@ -242,5 +242,5 @@ void Fortran::lower::genPointerAssociateLowerBounds(fir::FirOpBuilder &builder,
           loc, builder);
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, func.getFunctionType(), pointer, target, lbounds);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index c3a5b6101ce00..fff060b79c9fe 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -75,9 +75,9 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
                                         /*mutableProperties=*/{}};
         Fortran::lower::genDeallocateIfAllocated(converter, mutableBox, loc);
         if (isDoConcurrent)
-          builder.create<fir::YieldOp>(loc);
+          fir::YieldOp::create(builder, loc);
         else
-          builder.create<mlir::omp::YieldOp>(loc);
+          mlir::omp::YieldOp::create(builder, loc);
         return;
       }
     }
@@ -97,18 +97,18 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
         hlfir::genVariableRawAddress(loc, builder, hlfir::Entity{arg});
     mlir::Value isAllocated = builder.genIsNotNullAddr(loc, addr);
     fir::IfOp ifOp =
-        builder.create<fir::IfOp>(loc, isAllocated, /*withElseRegion=*/false);
+        fir::IfOp::create(builder, loc, isAllocated, /*withElseRegion=*/false);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
     mlir::Value cast = builder.createConvert(
         loc, fir::HeapType::get(fir::dyn_cast_ptrEleTy(addr.getType())), addr);
-    builder.create<fir::FreeMemOp>(loc, cast);
+    fir::FreeMemOp::create(builder, loc, cast);
 
     builder.setInsertionPointAfter(ifOp);
     if (isDoConcurrent)
-      builder.create<fir::YieldOp>(loc);
+      fir::YieldOp::create(builder, loc);
     else
-      builder.create<mlir::omp::YieldOp>(loc);
+      mlir::omp::YieldOp::create(builder, loc);
     return;
   }
 
@@ -122,11 +122,11 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
     auto heapTy = fir::HeapType::get(refTy.getEleTy());
     addr = builder.createConvert(loc, heapTy, addr);
 
-    builder.create<fir::FreeMemOp>(loc, addr);
+    fir::FreeMemOp::create(builder, loc, addr);
     if (isDoConcurrent)
-      builder.create<fir::YieldOp>(loc);
+      fir::YieldOp::create(builder, loc);
     else
-      builder.create<mlir::omp::YieldOp>(loc);
+      mlir::omp::YieldOp::create(builder, loc);
 
     return;
   }
@@ -172,7 +172,7 @@ fir::ShapeShiftOp Fortran::lower::getShapeShift(
       // OpenACC does
       mlir::Value dim = builder.createIntegerConstant(loc, idxTy, i);
       auto dimInfo =
-          builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, dim);
+          fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, dim);
       lbAndExtents.push_back(useDefaultLowerBounds ? one()
                                                    : dimInfo.getLowerBound());
       lbAndExtents.push_back(dimInfo.getExtent());
@@ -181,7 +181,7 @@ fir::ShapeShiftOp Fortran::lower::getShapeShift(
 
   auto shapeShiftTy = fir::ShapeShiftType::get(builder.getContext(), rank);
   auto shapeShift =
-      builder.create<fir::ShapeShiftOp>(loc, shapeShiftTy, lbAndExtents);
+      fir::ShapeShiftOp::create(builder, loc, shapeShiftTy, lbAndExtents);
   return shapeShift;
 }
 
@@ -270,7 +270,7 @@ static mlir::Value generateZeroShapeForRank(fir::FirOpBuilder &builder,
   mlir::SmallVector<mlir::Value> dims;
   dims.resize(rank, zero);
   mlir::Type shapeTy = fir::ShapeType::get(builder.getContext(), rank);
-  return builder.create<fir::ShapeOp>(loc, shapeTy, dims);
+  return fir::ShapeOp::create(builder, loc, shapeTy, dims);
 }
 
 namespace {
@@ -341,9 +341,9 @@ class PopulateInitAndCleanupRegionsHelper {
 
   void createYield(mlir::Value ret) {
     if (isDoConcurrent)
-      builder.create<fir::YieldOp>(loc, ret);
+      fir::YieldOp::create(builder, loc, ret);
     else
-      builder.create<mlir::omp::YieldOp>(loc, ret);
+      mlir::omp::YieldOp::create(builder, loc, ret);
   }
 
   void initTrivialType() {
@@ -392,9 +392,9 @@ void PopulateInitAndCleanupRegionsHelper::initBoxedPrivatePointer(
   // Just incase, do initialize the box with a null value
   mlir::Value null = builder.createNullConstant(loc, boxTy.getEleTy());
   mlir::Value nullBox;
-  nullBox = builder.create<fir::EmboxOp>(loc, boxTy, null, shape,
-                                         /*slice=*/mlir::Value{}, lenParams);
-  builder.create<fir::StoreOp>(loc, nullBox, allocatedPrivVarArg);
+  nullBox = fir::EmboxOp::create(builder, loc, boxTy, null, shape,
+                                 /*slice=*/mlir::Value{}, lenParams);
+  fir::StoreOp::create(builder, loc, nullBox, allocatedPrivVarArg);
   createYield(allocatedPrivVarArg);
 }
 /// Check if an allocatable box is unallocated. If so, initialize the boxAlloca
@@ -410,10 +410,10 @@ void PopulateInitAndCleanupRegionsHelper::initBoxedPrivatePointer(
 /// }
 /// omp.yield %box_alloca
 fir::IfOp PopulateInitAndCleanupRegionsHelper::handleNullAllocatable() {
-  mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, getLoadedMoldArg());
+  mlir::Value addr = fir::BoxAddrOp::create(builder, loc, getLoadedMoldArg());
   mlir::Value isNotAllocated = builder.genIsNullAddr(loc, addr);
-  fir::IfOp ifOp = builder.create<fir::IfOp>(loc, isNotAllocated,
-                                             /*withElseRegion=*/true);
+  fir::IfOp ifOp = fir::IfOp::create(builder, loc, isNotAllocated,
+                                     /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   // Just embox the null address and return.
   // We have to give the embox a shape so that the LLVM box structure has the
@@ -421,9 +421,9 @@ fir::IfOp PopulateInitAndCleanupRegionsHelper::handleNullAllocatable() {
   mlir::Value shape = generateZeroShapeForRank(builder, loc, moldArg);
 
   mlir::Value nullBox =
-      builder.create<fir::EmboxOp>(loc, valType, addr, shape,
-                                   /*slice=*/mlir::Value{}, lenParams);
-  builder.create<fir::StoreOp>(loc, nullBox, allocatedPrivVarArg);
+      fir::EmboxOp::create(builder, loc, valType, addr, shape,
+                           /*slice=*/mlir::Value{}, lenParams);
+  fir::StoreOp::create(builder, loc, nullBox, allocatedPrivVarArg);
   return ifOp;
 }
 
@@ -442,14 +442,14 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedScalar(
                                                      /*shape=*/{}, lenParams);
   if (scalarInitValue)
     builder.createStoreWithConvert(loc, scalarInitValue, valAlloc);
-  mlir::Value box = builder.create<fir::EmboxOp>(
-      loc, valType, valAlloc, /*shape=*/mlir::Value{},
-      /*slice=*/mlir::Value{}, lenParams);
+  mlir::Value box = fir::EmboxOp::create(builder, loc, valType, valAlloc,
+                                         /*shape=*/mlir::Value{},
+                                         /*slice=*/mlir::Value{}, lenParams);
   initializeIfDerivedTypeBox(
       builder, loc, box, getLoadedMoldArg(), needsInitialization,
       /*isFirstPrivate=*/kind == DeclOperationKind::FirstPrivateOrLocalInit);
   fir::StoreOp lastOp =
-      builder.create<fir::StoreOp>(loc, box, allocatedPrivVarArg);
+      fir::StoreOp::create(builder, loc, box, allocatedPrivVarArg);
 
   createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
                       isDoConcurrent);
@@ -483,14 +483,15 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
     fir::ShapeShiftOp shape =
         getShapeShift(builder, loc, source, cannotHaveNonDefaultLowerBounds);
     mlir::Type arrayType = source.getElementOrSequenceType();
-    mlir::Value allocatedArray = builder.create<fir::AllocMemOp>(
-        loc, arrayType, /*typeparams=*/mlir::ValueRange{}, shape.getExtents());
-    mlir::Value firClass = builder.create<fir::EmboxOp>(loc, source.getType(),
-                                                        allocatedArray, shape);
+    mlir::Value allocatedArray = fir::AllocMemOp::create(
+        builder, loc, arrayType, /*typeparams=*/mlir::ValueRange{},
+        shape.getExtents());
+    mlir::Value firClass = fir::EmboxOp::create(builder, loc, source.getType(),
+                                                allocatedArray, shape);
     initializeIfDerivedTypeBox(
         builder, loc, firClass, source, needsInitialization,
         /*isFirstprivate=*/kind == DeclOperationKind::FirstPrivateOrLocalInit);
-    builder.create<fir::StoreOp>(loc, firClass, allocatedPrivVarArg);
+    fir::StoreOp::create(builder, loc, firClass, allocatedPrivVarArg);
     if (ifUnallocated)
       builder.setInsertionPointAfter(ifUnallocated);
     createYield(allocatedPrivVarArg);
@@ -543,22 +544,21 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
   if (mlir::isa<fir::BaseBoxType>(temp.getType()))
     // the box created by the declare form createTempFromMold is missing
     // lower bounds info
-    box = builder.create<fir::ReboxOp>(loc, boxType, temp, shapeShift,
-                                       /*shift=*/mlir::Value{});
+    box = fir::ReboxOp::create(builder, loc, boxType, temp, shapeShift,
+                               /*shift=*/mlir::Value{});
   else
-    box = builder.create<fir::EmboxOp>(
-        loc, boxType, temp, shapeShift,
-        /*slice=*/mlir::Value{},
-        /*typeParams=*/llvm::ArrayRef<mlir::Value>{});
+    box = fir::EmboxOp::create(builder, loc, boxType, temp, shapeShift,
+                               /*slice=*/mlir::Value{},
+                               /*typeParams=*/llvm::ArrayRef<mlir::Value>{});
 
   if (scalarInitValue)
-    builder.create<hlfir::AssignOp>(loc, scalarInitValue, box);
+    hlfir::AssignOp::create(builder, loc, scalarInitValue, box);
 
   initializeIfDerivedTypeBox(
       builder, loc, box, getLoadedMoldArg(), needsInitialization,
       /*isFirstPrivate=*/kind == DeclOperationKind::FirstPrivateOrLocalInit);
 
-  builder.create<fir::StoreOp>(loc, box, allocatedPrivVarArg);
+  fir::StoreOp::create(builder, loc, box, allocatedPrivVarArg);
   if (ifUnallocated)
     builder.setInsertionPointAfter(ifUnallocated);
   createYield(allocatedPrivVarArg);
@@ -596,8 +596,8 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupUnboxedDerivedType(
   builder.setInsertionPointToStart(initBlock);
   mlir::Type boxedTy = fir::BoxType::get(valType);
   mlir::Value newBox =
-      builder.create<fir::EmboxOp>(loc, boxedTy, allocatedPrivVarArg);
-  mlir::Value moldBox = builder.create<fir::EmboxOp>(loc, boxedTy, moldArg);
+      fir::EmboxOp::create(builder, loc, boxedTy, allocatedPrivVarArg);
+  mlir::Value moldBox = fir::EmboxOp::create(builder, loc, boxedTy, moldArg);
   initializeIfDerivedTypeBox(builder, loc, newBox, moldBox, needsInitialization,
                              /*isFirstPrivate=*/kind ==
                                  DeclOperationKind::FirstPrivateOrLocalInit);

diff  --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index c0be1e229f825..80c32d066a38d 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -260,20 +260,20 @@ ReductionProcessor::getReductionInitValue(mlir::Location loc, mlir::Type type,
                                                                initIm);
     }
     if (mlir::isa<mlir::FloatType>(type))
-      return builder.create<mlir::arith::ConstantOp>(
-          loc, type,
+      return mlir::arith::ConstantOp::create(
+          builder, loc, type,
           builder.getFloatAttr(type, (double)getOperationIdentity(redId, loc)));
 
     if (mlir::isa<fir::LogicalType>(type)) {
-      mlir::Value intConst = builder.create<mlir::arith::ConstantOp>(
-          loc, builder.getI1Type(),
+      mlir::Value intConst = mlir::arith::ConstantOp::create(
+          builder, loc, builder.getI1Type(),
           builder.getIntegerAttr(builder.getI1Type(),
                                  getOperationIdentity(redId, loc)));
       return builder.createConvert(loc, type, intConst);
     }
 
-    return builder.create<mlir::arith::ConstantOp>(
-        loc, type,
+    return mlir::arith::ConstantOp::create(
+        builder, loc, type,
         builder.getIntegerAttr(type, getOperationIdentity(redId, loc)));
   case ReductionIdentifier::ID:
   case ReductionIdentifier::USER_DEF_OP:
@@ -301,15 +301,15 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     break;
   case ReductionIdentifier::IOR:
     assert((type.isIntOrIndex()) && "only integer is expected");
-    reductionOp = builder.create<mlir::arith::OrIOp>(loc, op1, op2);
+    reductionOp = mlir::arith::OrIOp::create(builder, loc, op1, op2);
     break;
   case ReductionIdentifier::IEOR:
     assert((type.isIntOrIndex()) && "only integer is expected");
-    reductionOp = builder.create<mlir::arith::XOrIOp>(loc, op1, op2);
+    reductionOp = mlir::arith::XOrIOp::create(builder, loc, op1, op2);
     break;
   case ReductionIdentifier::IAND:
     assert((type.isIntOrIndex()) && "only integer is expected");
-    reductionOp = builder.create<mlir::arith::AndIOp>(loc, op1, op2);
+    reductionOp = mlir::arith::AndIOp::create(builder, loc, op1, op2);
     break;
   case ReductionIdentifier::ADD:
     reductionOp =
@@ -325,7 +325,8 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value andiOp = builder.create<mlir::arith::AndIOp>(loc, op1I1, op2I1);
+    mlir::Value andiOp =
+        mlir::arith::AndIOp::create(builder, loc, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, andiOp);
     break;
@@ -334,7 +335,7 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value oriOp = builder.create<mlir::arith::OrIOp>(loc, op1I1, op2I1);
+    mlir::Value oriOp = mlir::arith::OrIOp::create(builder, loc, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, oriOp);
     break;
@@ -343,8 +344,8 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value cmpiOp = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::eq, op1I1, op2I1);
+    mlir::Value cmpiOp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::eq, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, cmpiOp);
     break;
@@ -353,8 +354,8 @@ mlir::Value ReductionProcessor::createScalarCombiner(
     mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);
     mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);
 
-    mlir::Value cmpiOp = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::ne, op1I1, op2I1);
+    mlir::Value cmpiOp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::ne, op1I1, op2I1);
 
     reductionOp = builder.createConvert(loc, type, cmpiOp);
     break;
@@ -370,9 +371,9 @@ template <typename ParentDeclOpType>
 static void genYield(fir::FirOpBuilder &builder, mlir::Location loc,
                      mlir::Value yieldedValue) {
   if constexpr (std::is_same_v<ParentDeclOpType, mlir::omp::DeclareReductionOp>)
-    builder.create<mlir::omp::YieldOp>(loc, yieldedValue);
+    mlir::omp::YieldOp::create(builder, loc, yieldedValue);
   else
-    builder.create<fir::YieldOp>(loc, yieldedValue);
+    fir::YieldOp::create(builder, loc, yieldedValue);
 }
 
 /// Create reduction combiner region for reduction variables which are boxed
@@ -393,24 +394,24 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
 
   // load fir.ref<fir.box<...>>
   mlir::Value lhsAddr = lhs;
-  lhs = builder.create<fir::LoadOp>(loc, lhs);
-  rhs = builder.create<fir::LoadOp>(loc, rhs);
+  lhs = fir::LoadOp::create(builder, loc, lhs);
+  rhs = fir::LoadOp::create(builder, loc, rhs);
 
   if ((heapTy || ptrTy) && !seqTy) {
     // get box contents (heap pointers)
-    lhs = builder.create<fir::BoxAddrOp>(loc, lhs);
-    rhs = builder.create<fir::BoxAddrOp>(loc, rhs);
+    lhs = fir::BoxAddrOp::create(builder, loc, lhs);
+    rhs = fir::BoxAddrOp::create(builder, loc, rhs);
     mlir::Value lhsValAddr = lhs;
 
     // load heap pointers
-    lhs = builder.create<fir::LoadOp>(loc, lhs);
-    rhs = builder.create<fir::LoadOp>(loc, rhs);
+    lhs = fir::LoadOp::create(builder, loc, lhs);
+    rhs = fir::LoadOp::create(builder, loc, rhs);
 
     mlir::Type eleTy = heapTy ? heapTy.getEleTy() : ptrTy.getEleTy();
 
     mlir::Value result = ReductionProcessor::createScalarCombiner(
         builder, loc, redId, eleTy, lhs, rhs);
-    builder.create<fir::StoreOp>(loc, result, lhsValAddr);
+    fir::StoreOp::create(builder, loc, result, lhsValAddr);
     genYield<DeclRedOpType>(builder, loc, lhsAddr);
     return;
   }
@@ -437,17 +438,17 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
   builder.setInsertionPointToStart(nest.body);
   const bool seqIsVolatile = fir::isa_volatile_type(seqTy.getEleTy());
   mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy(), seqIsVolatile);
-  auto lhsEleAddr = builder.create<fir::ArrayCoorOp>(
-      loc, refTy, lhs, shapeShift, /*slice=*/mlir::Value{},
+  auto lhsEleAddr = fir::ArrayCoorOp::create(
+      builder, loc, refTy, lhs, shapeShift, /*slice=*/mlir::Value{},
       nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});
-  auto rhsEleAddr = builder.create<fir::ArrayCoorOp>(
-      loc, refTy, rhs, shapeShift, /*slice=*/mlir::Value{},
+  auto rhsEleAddr = fir::ArrayCoorOp::create(
+      builder, loc, refTy, rhs, shapeShift, /*slice=*/mlir::Value{},
       nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});
-  auto lhsEle = builder.create<fir::LoadOp>(loc, lhsEleAddr);
-  auto rhsEle = builder.create<fir::LoadOp>(loc, rhsEleAddr);
+  auto lhsEle = fir::LoadOp::create(builder, loc, lhsEleAddr);
+  auto rhsEle = fir::LoadOp::create(builder, loc, rhsEleAddr);
   mlir::Value scalarReduction = ReductionProcessor::createScalarCombiner(
       builder, loc, redId, refTy, lhsEle, rhsEle);
-  builder.create<fir::StoreOp>(loc, scalarReduction, lhsEleAddr);
+  fir::StoreOp::create(builder, loc, scalarReduction, lhsEleAddr);
 
   builder.setInsertionPointAfter(nest.outerOp);
   genYield<DeclRedOpType>(builder, loc, lhsAddr);
@@ -468,7 +469,7 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value result = ReductionProcessor::createScalarCombiner(
         builder, loc, redId, ty, lhsLoaded, rhsLoaded);
     if (isByRef) {
-      builder.create<fir::StoreOp>(loc, result, lhs);
+      fir::StoreOp::create(builder, loc, result, lhs);
       genYield<DeclRedOpType>(builder, loc, lhs);
     } else {
       genYield<DeclRedOpType>(builder, loc, result);
@@ -539,7 +540,7 @@ static void createReductionAllocAndInitRegions(
     if (isByRef) {
       // alloc region
       builder.setInsertionPointToEnd(allocBlock);
-      mlir::Value alloca = builder.create<fir::AllocaOp>(loc, ty);
+      mlir::Value alloca = fir::AllocaOp::create(builder, loc, ty);
       yield(alloca);
       return;
     }
@@ -551,7 +552,7 @@ static void createReductionAllocAndInitRegions(
 
   // alloc region
   builder.setInsertionPointToEnd(allocBlock);
-  mlir::Value boxAlloca = builder.create<fir::AllocaOp>(loc, ty);
+  mlir::Value boxAlloca = fir::AllocaOp::create(builder, loc, ty);
   yield(boxAlloca);
 }
 
@@ -575,7 +576,7 @@ OpType ReductionProcessor::createDeclareReduction(
   if (!isByRef)
     type = valTy;
 
-  decl = modBuilder.create<OpType>(loc, reductionOpName, type);
+  decl = OpType::create(modBuilder, loc, reductionOpName, type);
   createReductionAllocAndInitRegions(converter, loc, decl, redId, type,
                                      isByRef);
 
@@ -672,8 +673,8 @@ void ReductionProcessor::processReductionArguments(
       // Always pass the box by reference so that the OpenMP dialect
       // verifiers don't need to know anything about fir.box
       auto alloca =
-          builder.create<fir::AllocaOp>(currentLocation, box.getType());
-      builder.create<fir::StoreOp>(currentLocation, box, alloca);
+          fir::AllocaOp::create(builder, currentLocation, box.getType());
+      fir::StoreOp::create(builder, currentLocation, box, alloca);
 
       symVal = alloca;
     } else if (mlir::isa<fir::BaseBoxType>(symVal.getType())) {
@@ -683,9 +684,9 @@ void ReductionProcessor::processReductionArguments(
       auto oldIP = builder.saveInsertionPoint();
       builder.setInsertionPointToStart(builder.getAllocaBlock());
       auto alloca =
-          builder.create<fir::AllocaOp>(currentLocation, symVal.getType());
+          fir::AllocaOp::create(builder, currentLocation, symVal.getType());
       builder.restoreInsertionPoint(oldIP);
-      builder.create<fir::StoreOp>(currentLocation, symVal, alloca);
+      fir::StoreOp::create(builder, currentLocation, symVal, alloca);
       symVal = alloca;
     }
 

diff  --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index b9d2574a76ad0..881401e11fee4 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -702,9 +702,10 @@ void privatizeSymbol(
     // Boxes should be passed by reference into nested regions:
     auto oldIP = firOpBuilder.saveInsertionPoint();
     firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
-    auto alloca = firOpBuilder.create<fir::AllocaOp>(symLoc, privVal.getType());
+    auto alloca =
+        fir::AllocaOp::create(firOpBuilder, symLoc, privVal.getType());
     firOpBuilder.restoreInsertionPoint(oldIP);
-    firOpBuilder.create<fir::StoreOp>(symLoc, privVal, alloca);
+    fir::StoreOp::create(firOpBuilder, symLoc, privVal, alloca);
     privVal = alloca;
   }
 
@@ -726,15 +727,15 @@ void privatizeSymbol(
     OpType result;
 
     if constexpr (std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) {
-      result = firOpBuilder.create<OpType>(
-          symLoc, uniquePrivatizerName, allocType,
+      result = OpType::create(
+          firOpBuilder, symLoc, uniquePrivatizerName, allocType,
           emitCopyRegion ? mlir::omp::DataSharingClauseType::FirstPrivate
                          : mlir::omp::DataSharingClauseType::Private);
     } else {
-      result = firOpBuilder.create<OpType>(
-          symLoc, uniquePrivatizerName, allocType,
-          emitCopyRegion ? fir::LocalitySpecifierType::LocalInit
-                         : fir::LocalitySpecifierType::Local);
+      result =
+          OpType::create(firOpBuilder, symLoc, uniquePrivatizerName, allocType,
+                         emitCopyRegion ? fir::LocalitySpecifierType::LocalInit
+                                        : fir::LocalitySpecifierType::Local);
     }
 
     fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
@@ -815,12 +816,12 @@ void privatizeSymbol(
       copyFirstPrivateSymbol(converter, symToPrivatize, &ip);
 
       if constexpr (std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) {
-        firOpBuilder.create<mlir::omp::YieldOp>(
-            hsb.getAddr().getLoc(),
+        mlir::omp::YieldOp::create(
+            firOpBuilder, hsb.getAddr().getLoc(),
             symTable.shallowLookupSymbol(*symToPrivatize).getAddr());
       } else {
-        firOpBuilder.create<fir::YieldOp>(
-            hsb.getAddr().getLoc(),
+        fir::YieldOp::create(
+            firOpBuilder, hsb.getAddr().getLoc(),
             symTable.shallowLookupSymbol(*symToPrivatize).getAddr());
       }
     }

diff  --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index c7b3e11728cea..4d1d6fb28b365 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -121,8 +121,9 @@ class VectorSubscriptBoxBuilder {
     if (recTy.getNumLenParams() != 0)
       TODO(loc, "threading length parameters in field index op");
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-    componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
-        loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
+    componentPath.emplace_back(
+        fir::FieldIndexOp::create(builder, loc, fldTy, componentName, recTy,
+                                  /*typeParams=*/mlir::ValueRange{}));
     return fir::unwrapSequenceType(recTy.getType(componentName));
   }
 
@@ -269,16 +270,16 @@ mlir::Value Fortran::lower::VectorSubscriptBox::loopOverElementsBase(
   for (auto [lb, ub, step] : genLoopBounds(builder, loc)) {
     LoopType loop;
     if constexpr (std::is_same_v<LoopType, fir::IterWhileOp>) {
-      loop =
-          builder.create<fir::IterWhileOp>(loc, lb, ub, step, initialCondition);
+      loop = fir::IterWhileOp::create(builder, loc, lb, ub, step,
+                                      initialCondition);
       initialCondition = loop.getIterateVar();
       if (!outerLoop)
         outerLoop = loop;
       else
-        builder.create<fir::ResultOp>(loc, loop.getResult(0));
+        fir::ResultOp::create(builder, loc, loop.getResult(0));
     } else {
-      loop =
-          builder.create<fir::DoLoopOp>(loc, lb, ub, step, /*unordered=*/false);
+      loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
+                                   /*unordered=*/false);
       if (!outerLoop)
         outerLoop = loop;
     }
@@ -293,7 +294,7 @@ mlir::Value Fortran::lower::VectorSubscriptBox::loopOverElementsBase(
 
   if constexpr (std::is_same_v<LoopType, fir::IterWhileOp>) {
     auto res = elementalGenerator(elem);
-    builder.create<fir::ResultOp>(loc, res);
+    fir::ResultOp::create(builder, loc, res);
     builder.setInsertionPointAfter(outerLoop);
     return outerLoop.getResult(0);
   } else {
@@ -326,7 +327,7 @@ Fortran::lower::VectorSubscriptBox::createSlice(fir::FirOpBuilder &builder,
   mlir::Type idxTy = builder.getIndexType();
   llvm::SmallVector<mlir::Value> triples;
   mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-  auto undef = builder.create<fir::UndefOp>(loc, idxTy);
+  auto undef = fir::UndefOp::create(builder, loc, idxTy);
   for (const LoweredSubscript &subscript : loweredSubscripts)
     Fortran::common::visit(Fortran::common::visitors{
                                [&](const LoweredTriplet &triplet) {
@@ -346,7 +347,7 @@ Fortran::lower::VectorSubscriptBox::createSlice(fir::FirOpBuilder &builder,
                                },
                            },
                            subscript);
-  return builder.create<fir::SliceOp>(loc, triples, componentPath);
+  return fir::SliceOp::create(builder, loc, triples, componentPath);
 }
 
 llvm::SmallVector<std::tuple<mlir::Value, mlir::Value, mlir::Value>>
@@ -369,13 +370,13 @@ Fortran::lower::VectorSubscriptBox::genLoopBounds(fir::FirOpBuilder &builder,
           builder, loc, loweredBase, dimension, one);
       baseLb = builder.createConvert(loc, idxTy, baseLb);
       lb = baseLb;
-      ub = builder.create<mlir::arith::SubIOp>(loc, idxTy, extent, one);
-      ub = builder.create<mlir::arith::AddIOp>(loc, idxTy, ub, baseLb);
+      ub = mlir::arith::SubIOp::create(builder, loc, idxTy, extent, one);
+      ub = mlir::arith::AddIOp::create(builder, loc, idxTy, ub, baseLb);
       step = one;
     } else {
       const auto &vector = std::get<LoweredVectorSubscript>(subscript);
       lb = zero;
-      ub = builder.create<mlir::arith::SubIOp>(loc, idxTy, vector.size, one);
+      ub = mlir::arith::SubIOp::create(builder, loc, idxTy, vector.size, one);
       step = one;
     }
     bounds.emplace_back(lb, ub, step);
@@ -402,10 +403,10 @@ fir::ExtendedValue Fortran::lower::VectorSubscriptBox::getElementAt(
               mlir::Type vecEleTy = fir::unwrapSequenceType(
                   fir::unwrapPassByRefType(vecBase.getType()));
               mlir::Type refTy = builder.getRefType(vecEleTy);
-              auto vecEltRef = builder.create<fir::CoordinateOp>(
-                  loc, refTy, vecBase, vecIndex);
+              auto vecEltRef = fir::CoordinateOp::create(builder, loc, refTy,
+                                                         vecBase, vecIndex);
               auto vecElt =
-                  builder.create<fir::LoadOp>(loc, vecEleTy, vecEltRef);
+                  fir::LoadOp::create(builder, loc, vecEleTy, vecEltRef);
               indexes.emplace_back(builder.createConvert(loc, idxTy, vecElt));
             },
             [&](const mlir::Value &i) {
@@ -414,8 +415,8 @@ fir::ExtendedValue Fortran::lower::VectorSubscriptBox::getElementAt(
         },
         subscript);
   mlir::Type refTy = builder.getRefType(getElementType());
-  auto elementAddr = builder.create<fir::ArrayCoorOp>(
-      loc, refTy, fir::getBase(loweredBase), shape, slice, indexes,
+  auto elementAddr = fir::ArrayCoorOp::create(
+      builder, loc, refTy, fir::getBase(loweredBase), shape, slice, indexes,
       fir::getTypeParams(loweredBase));
   fir::ExtendedValue element = fir::factory::arraySectionElementToExtendedValue(
       builder, loc, loweredBase, elementAddr, slice);


        

From flang-commits at lists.llvm.org  Mon Jul 21 16:56:47 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 21 Jul 2025 16:56:47 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][NFC] update
 `flang/Optimizer/Builder/Runtime` create APIs (10/n) (PR #149916)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149916@github.com>
Message-ID: <687ed3bf.050a0220.114a96.0f5e@mx.google.com>


https://github.com/makslevental closed https://github.com/llvm/llvm-project/pull/149916

From flang-commits at lists.llvm.org  Mon Jul 21 16:57:05 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:57:05 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed3d1.170a0220.1833f5.0646@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:57:06 2025
From: flang-commits at lists.llvm.org (Maksim Levental via flang-commits)
Date: Mon, 21 Jul 2025 16:57:06 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir][NFC] update
 `flang/Optimizer/Builder` create APIs (9/n) (PR #149917)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149917@github.com>
Message-ID: <687ed3d2.050a0220.32f5ca.f5bf@mx.google.com>


https://github.com/makslevental closed https://github.com/llvm/llvm-project/pull/149917

From flang-commits at lists.llvm.org  Mon Jul 21 16:57:59 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:57:59 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed407.630a0220.31eae0.f556@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:58:39 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:58:39 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed42f.170a0220.3082ea.01ab@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:59:13 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Mon, 21 Jul 2025 16:59:13 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687ed451.170a0220.40ba3.0082@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Mon Jul 21 16:56:44 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 16:56:44 -0700 (PDT)
Subject: [flang-commits] [flang] 5547c6c - [mlir][NFC] update
 `flang/Optimizer/Builder/Runtime` create APIs (10/n) (#149916)
Message-ID: <687ed3bc.a70a0220.22c9e4.0d78@mx.google.com>


Author: Maksim Levental
Date: 2025-07-21T19:56:41-04:00
New Revision: 5547c6cd03ddddd405a09e51624e1f19955a85b1

URL: https://github.com/llvm/llvm-project/commit/5547c6cd03ddddd405a09e51624e1f19955a85b1
DIFF: https://github.com/llvm/llvm-project/commit/5547c6cd03ddddd405a09e51624e1f19955a85b1.diff

LOG: [mlir][NFC] update `flang/Optimizer/Builder/Runtime` create APIs (10/n) (#149916)

See https://github.com/llvm/llvm-project/pull/147168 for more info.

Added: 
    

Modified: 
    flang/lib/Optimizer/Builder/Runtime/Allocatable.cpp
    flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp
    flang/lib/Optimizer/Builder/Runtime/Assign.cpp
    flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
    flang/lib/Optimizer/Builder/Runtime/Character.cpp
    flang/lib/Optimizer/Builder/Runtime/Command.cpp
    flang/lib/Optimizer/Builder/Runtime/Derived.cpp
    flang/lib/Optimizer/Builder/Runtime/EnvironmentDefaults.cpp
    flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp
    flang/lib/Optimizer/Builder/Runtime/Execute.cpp
    flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp
    flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
    flang/lib/Optimizer/Builder/Runtime/Main.cpp
    flang/lib/Optimizer/Builder/Runtime/Numeric.cpp
    flang/lib/Optimizer/Builder/Runtime/Pointer.cpp
    flang/lib/Optimizer/Builder/Runtime/Ragged.cpp
    flang/lib/Optimizer/Builder/Runtime/Reduction.cpp
    flang/lib/Optimizer/Builder/Runtime/Stop.cpp
    flang/lib/Optimizer/Builder/Runtime/Support.cpp
    flang/lib/Optimizer/Builder/Runtime/TemporaryStack.cpp
    flang/lib/Optimizer/Builder/Runtime/Transformational.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Builder/Runtime/Allocatable.cpp b/flang/lib/Optimizer/Builder/Runtime/Allocatable.cpp
index cd5f1f6d098c3..cc9f8280a172c 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Allocatable.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Allocatable.cpp
@@ -30,7 +30,7 @@ mlir::Value fir::runtime::genMoveAlloc(fir::FirOpBuilder &builder,
         mlir::dyn_cast<fir::ClassType>(fir::dyn_cast_ptrEleTy(from.getType()));
     mlir::Type derivedType = fir::unwrapInnerType(clTy.getEleTy());
     declaredTypeDesc =
-        builder.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(derivedType));
+        fir::TypeDescOp::create(builder, loc, mlir::TypeAttr::get(derivedType));
   } else {
     declaredTypeDesc = builder.createNullConstant(loc);
   }
@@ -38,7 +38,7 @@ mlir::Value fir::runtime::genMoveAlloc(fir::FirOpBuilder &builder,
       builder, loc, fTy, to, from, declaredTypeDesc, hasStat, errMsg,
       sourceFile, sourceLine)};
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 void fir::runtime::genAllocatableApplyMold(fir::FirOpBuilder &builder,
@@ -52,7 +52,7 @@ void fir::runtime::genAllocatableApplyMold(fir::FirOpBuilder &builder,
       builder.createIntegerConstant(loc, fTy.getInput(2), rank);
   llvm::SmallVector<mlir::Value> args{
       fir::runtime::createArguments(builder, loc, fTy, desc, mold, rankVal)};
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genAllocatableSetBounds(fir::FirOpBuilder &builder,
@@ -66,7 +66,7 @@ void fir::runtime::genAllocatableSetBounds(fir::FirOpBuilder &builder,
   mlir::FunctionType fTy{func.getFunctionType()};
   llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
       builder, loc, fTy, desc, dimIndex, lowerBound, upperBound)};
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genAllocatableAllocate(fir::FirOpBuilder &builder,
@@ -84,10 +84,10 @@ void fir::runtime::genAllocatableAllocate(fir::FirOpBuilder &builder,
     hasStat = builder.createBool(loc, false);
   if (!errMsg) {
     mlir::Type boxNoneTy = fir::BoxType::get(builder.getNoneType());
-    errMsg = builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+    errMsg = fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   }
   llvm::SmallVector<mlir::Value> args{
       fir::runtime::createArguments(builder, loc, fTy, desc, asyncObject,
                                     hasStat, errMsg, sourceFile, sourceLine)};
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp b/flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp
index 0d56cd2edc99b..8c9825efaaa4e 100644
--- a/flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp
@@ -50,7 +50,7 @@ mlir::Value fir::runtime::genInitArrayConstructorVector(
   auto args = fir::runtime::createArguments(builder, loc, funcType, cookie,
                                             toBox, useValueLengthParameters,
                                             sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
   return cookie;
 }
 
@@ -63,7 +63,7 @@ void fir::runtime::genPushArrayConstructorValue(
   mlir::FunctionType funcType = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcType,
                                             arrayConstructorVector, fromBox);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genPushArrayConstructorSimpleScalar(
@@ -75,5 +75,5 @@ void fir::runtime::genPushArrayConstructorSimpleScalar(
   mlir::FunctionType funcType = func.getFunctionType();
   auto args = fir::runtime::createArguments(
       builder, loc, funcType, arrayConstructorVector, fromAddress);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Assign.cpp b/flang/lib/Optimizer/Builder/Runtime/Assign.cpp
index 62f03f7d48665..336dbdc89c04a 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Assign.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Assign.cpp
@@ -22,7 +22,7 @@ void fir::runtime::genAssign(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, destBox,
                                             sourceBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genAssignPolymorphic(fir::FirOpBuilder &builder,
@@ -36,7 +36,7 @@ void fir::runtime::genAssignPolymorphic(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, destBox,
                                             sourceBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genAssignExplicitLengthCharacter(fir::FirOpBuilder &builder,
@@ -52,7 +52,7 @@ void fir::runtime::genAssignExplicitLengthCharacter(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, destBox,
                                             sourceBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genAssignTemporary(fir::FirOpBuilder &builder,
@@ -66,7 +66,7 @@ void fir::runtime::genAssignTemporary(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, destBox,
                                             sourceBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genCopyInAssign(fir::FirOpBuilder &builder,
@@ -79,7 +79,7 @@ void fir::runtime::genCopyInAssign(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, destBox,
                                             sourceBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genCopyOutAssign(fir::FirOpBuilder &builder,
@@ -93,5 +93,5 @@ void fir::runtime::genCopyOutAssign(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, destBox,
                                             sourceBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp b/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
index 62a0652cc2e5d..a6ee98685f3c9 100644
--- a/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
@@ -30,7 +30,7 @@ void fir::runtime::cuda::genSyncGlobalDescriptor(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
   llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
       builder, loc, fTy, hostPtr, sourceFile, sourceLine)};
-  builder.create<fir::CallOp>(loc, callee, args);
+  fir::CallOp::create(builder, loc, callee, args);
 }
 
 void fir::runtime::cuda::genDescriptorCheckSection(fir::FirOpBuilder &builder,
@@ -45,7 +45,7 @@ void fir::runtime::cuda::genDescriptorCheckSection(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
   llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
       builder, loc, fTy, desc, sourceFile, sourceLine)};
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::cuda::genSetAllocatorIndex(fir::FirOpBuilder &builder,
@@ -60,5 +60,5 @@ void fir::runtime::cuda::genSetAllocatorIndex(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
       builder, loc, fTy, desc, index, sourceFile, sourceLine)};
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Character.cpp b/flang/lib/Optimizer/Builder/Runtime/Character.cpp
index b16819915d5ab..57fb0cccf6863 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Character.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Character.cpp
@@ -34,7 +34,7 @@ static void genCharacterSearch(FN func, fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             string1Box, string2Box, backBox,
                                             kind, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Helper function to recover the KIND from the FIR type.
@@ -72,7 +72,7 @@ static void genAdjust(fir::FirOpBuilder &builder, mlir::Location loc,
   auto sourceFile = fir::factory::locationToFilename(builder, loc);
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             stringBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, adjustFunc, args);
+  fir::CallOp::create(builder, loc, adjustFunc, args);
 }
 
 void fir::runtime::genAdjustL(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -114,9 +114,9 @@ fir::runtime::genCharCompare(fir::FirOpBuilder &builder, mlir::Location loc,
   auto fTy = beginFunc.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, lhsBuff, rhsBuff,
                                             lhsLen, rhsLen);
-  auto tri = builder.create<fir::CallOp>(loc, beginFunc, args).getResult(0);
+  auto tri = fir::CallOp::create(builder, loc, beginFunc, args).getResult(0);
   auto zero = builder.createIntegerConstant(loc, tri.getType(), 0);
-  return builder.create<mlir::arith::CmpIOp>(loc, cmp, tri, zero);
+  return mlir::arith::CmpIOp::create(builder, loc, cmp, tri, zero);
 }
 
 mlir::Value fir::runtime::genCharCompare(fir::FirOpBuilder &builder,
@@ -130,8 +130,8 @@ mlir::Value fir::runtime::genCharCompare(fir::FirOpBuilder &builder,
     if (fir::isa_ref_type(base.getType()))
       return base;
     auto mem =
-        builder.create<fir::AllocaOp>(loc, base.getType(), /*pinned=*/false);
-    builder.create<fir::StoreOp>(loc, base, mem);
+        fir::AllocaOp::create(builder, loc, base.getType(), /*pinned=*/false);
+    fir::StoreOp::create(builder, loc, base, mem);
     return mem;
   };
   auto lhsBuffer = allocateIfNotInMemory(fir::getBase(lhs));
@@ -165,7 +165,7 @@ mlir::Value fir::runtime::genIndex(fir::FirOpBuilder &builder,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, stringBase, stringLen,
                                     substringBase, substringLen, back);
-  return builder.create<fir::CallOp>(loc, indexFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, indexFunc, args).getResult(0);
 }
 
 void fir::runtime::genIndexDescriptor(fir::FirOpBuilder &builder,
@@ -189,7 +189,7 @@ void fir::runtime::genRepeat(fir::FirOpBuilder &builder, mlir::Location loc,
 
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, stringBox, ncopies, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, repeatFunc, args);
+  fir::CallOp::create(builder, loc, repeatFunc, args);
 }
 
 void fir::runtime::genTrim(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -202,7 +202,7 @@ void fir::runtime::genTrim(fir::FirOpBuilder &builder, mlir::Location loc,
 
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             stringBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, trimFunc, args);
+  fir::CallOp::create(builder, loc, trimFunc, args);
 }
 
 void fir::runtime::genScanDescriptor(fir::FirOpBuilder &builder,
@@ -237,7 +237,7 @@ mlir::Value fir::runtime::genScan(fir::FirOpBuilder &builder,
   auto fTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, stringBase,
                                             stringLen, setBase, setLen, back);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 void fir::runtime::genVerifyDescriptor(fir::FirOpBuilder &builder,
@@ -274,5 +274,5 @@ mlir::Value fir::runtime::genVerify(fir::FirOpBuilder &builder,
   auto fTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, stringBase,
                                             stringLen, setBase, setLen, back);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Command.cpp b/flang/lib/Optimizer/Builder/Runtime/Command.cpp
index 35aa529a9a727..e65e2b6df7557 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Command.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Command.cpp
@@ -30,7 +30,7 @@ mlir::Value fir::runtime::genCommandArgumentCount(fir::FirOpBuilder &builder,
                                                   mlir::Location loc) {
   auto argumentCountFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(ArgumentCount)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, argumentCountFunc).getResult(0);
+  return fir::CallOp::create(builder, loc, argumentCountFunc).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetCommand(fir::FirOpBuilder &builder,
@@ -46,7 +46,7 @@ mlir::Value fir::runtime::genGetCommand(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args =
       fir::runtime::createArguments(builder, loc, runtimeFuncTy, command,
                                     length, errmsg, sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, runtimeFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, runtimeFunc, args).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetPID(fir::FirOpBuilder &builder,
@@ -54,7 +54,7 @@ mlir::Value fir::runtime::genGetPID(fir::FirOpBuilder &builder,
   auto runtimeFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(GetPID)>(loc, builder);
 
-  return builder.create<fir::CallOp>(loc, runtimeFunc).getResult(0);
+  return fir::CallOp::create(builder, loc, runtimeFunc).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetCommandArgument(
@@ -69,7 +69,7 @@ mlir::Value fir::runtime::genGetCommandArgument(
   llvm::SmallVector<mlir::Value> args =
       fir::runtime::createArguments(builder, loc, runtimeFuncTy, number, value,
                                     length, errmsg, sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, runtimeFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, runtimeFunc, args).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetEnvVariable(fir::FirOpBuilder &builder,
@@ -87,7 +87,7 @@ mlir::Value fir::runtime::genGetEnvVariable(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, runtimeFuncTy, name, value, length, trimName, errmsg,
       sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, runtimeFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, runtimeFunc, args).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetCwd(fir::FirOpBuilder &builder,
@@ -100,7 +100,7 @@ mlir::Value fir::runtime::genGetCwd(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, runtimeFuncTy, cwd, sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 mlir::Value fir::runtime::genHostnm(fir::FirOpBuilder &builder,
@@ -113,7 +113,7 @@ mlir::Value fir::runtime::genHostnm(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(2));
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, runtimeFuncTy, res, sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 void fir::runtime::genPerror(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -123,7 +123,7 @@ void fir::runtime::genPerror(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::FunctionType runtimeFuncTy = runtimeFunc.getFunctionType();
   llvm::SmallVector<mlir::Value> args =
       fir::runtime::createArguments(builder, loc, runtimeFuncTy, string);
-  builder.create<fir::CallOp>(loc, runtimeFunc, args);
+  fir::CallOp::create(builder, loc, runtimeFunc, args);
 }
 
 mlir::Value fir::runtime::genPutEnv(fir::FirOpBuilder &builder,
@@ -137,7 +137,7 @@ mlir::Value fir::runtime::genPutEnv(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(1));
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, runtimeFuncTy, str, strLength, sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 mlir::Value fir::runtime::genUnlink(fir::FirOpBuilder &builder,
@@ -151,5 +151,5 @@ mlir::Value fir::runtime::genUnlink(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(1));
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, runtimeFuncTy, path, pathLength, sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Derived.cpp b/flang/lib/Optimizer/Builder/Runtime/Derived.cpp
index 25b41518a90e5..1b0457bb2d0a1 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Derived.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Derived.cpp
@@ -26,7 +26,7 @@ void fir::runtime::genDerivedTypeInitialize(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
   auto args = fir::runtime::createArguments(builder, loc, fTy, box, sourceFile,
                                             sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genDerivedTypeInitializeClone(fir::FirOpBuilder &builder,
@@ -41,7 +41,7 @@ void fir::runtime::genDerivedTypeInitializeClone(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, newBox, box,
                                             sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genDerivedTypeDestroy(fir::FirOpBuilder &builder,
@@ -49,7 +49,7 @@ void fir::runtime::genDerivedTypeDestroy(fir::FirOpBuilder &builder,
   auto func = fir::runtime::getRuntimeFunc<mkRTKey(Destroy)>(loc, builder);
   auto fTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, box);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genDerivedTypeFinalize(fir::FirOpBuilder &builder,
@@ -61,7 +61,7 @@ void fir::runtime::genDerivedTypeFinalize(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
   auto args = fir::runtime::createArguments(builder, loc, fTy, box, sourceFile,
                                             sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genDerivedTypeDestroyWithoutFinalization(
@@ -70,7 +70,7 @@ void fir::runtime::genDerivedTypeDestroyWithoutFinalization(
       loc, builder);
   auto fTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, box);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genNullifyDerivedType(fir::FirOpBuilder &builder,
@@ -78,7 +78,7 @@ void fir::runtime::genNullifyDerivedType(fir::FirOpBuilder &builder,
                                          fir::RecordType derivedType,
                                          unsigned rank) {
   mlir::Value typeDesc =
-      builder.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(derivedType));
+      fir::TypeDescOp::create(builder, loc, mlir::TypeAttr::get(derivedType));
   mlir::func::FuncOp callee =
       fir::runtime::getRuntimeFunc<mkRTKey(PointerNullifyDerived)>(loc,
                                                                    builder);
@@ -90,7 +90,7 @@ void fir::runtime::genNullifyDerivedType(fir::FirOpBuilder &builder,
   mlir::Value c0 = builder.createIntegerConstant(loc, inputTypes[3], 0);
   args.push_back(rankCst);
   args.push_back(c0);
-  builder.create<fir::CallOp>(loc, callee, args);
+  fir::CallOp::create(builder, loc, callee, args);
 }
 
 mlir::Value fir::runtime::genSameTypeAs(fir::FirOpBuilder &builder,
@@ -100,7 +100,7 @@ mlir::Value fir::runtime::genSameTypeAs(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(SameTypeAs)>(loc, builder);
   auto fTy = sameTypeAsFunc.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, a, b);
-  return builder.create<fir::CallOp>(loc, sameTypeAsFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, sameTypeAsFunc, args).getResult(0);
 }
 
 mlir::Value fir::runtime::genExtendsTypeOf(fir::FirOpBuilder &builder,
@@ -110,5 +110,6 @@ mlir::Value fir::runtime::genExtendsTypeOf(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(ExtendsTypeOf)>(loc, builder);
   auto fTy = extendsTypeOfFunc.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, a, mold);
-  return builder.create<fir::CallOp>(loc, extendsTypeOfFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, extendsTypeOfFunc, args)
+      .getResult(0);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/EnvironmentDefaults.cpp b/flang/lib/Optimizer/Builder/Runtime/EnvironmentDefaults.cpp
index bf5fd6af0eafa..fa3d00e8b844f 100755
--- a/flang/lib/Optimizer/Builder/Runtime/EnvironmentDefaults.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/EnvironmentDefaults.cpp
@@ -44,7 +44,7 @@ mlir::Value fir::runtime::genEnvironmentDefaults(
   mlir::IntegerAttr one = builder.getIntegerAttr(idxTy, 1);
   std::string itemListName = envDefaultListPtrName + ".items";
   auto listBuilder = [&](fir::FirOpBuilder &builder) {
-    mlir::Value list = builder.create<fir::UndefOp>(loc, itemListTy);
+    mlir::Value list = fir::UndefOp::create(builder, loc, itemListTy);
     llvm::SmallVector<mlir::Attribute, 2> idx = {mlir::Attribute{},
                                                  mlir::Attribute{}};
     auto insertStringField = [&](const std::string &s,
@@ -52,8 +52,8 @@ mlir::Value fir::runtime::genEnvironmentDefaults(
       mlir::Value stringAddress = fir::getBase(
           fir::factory::createStringLiteral(builder, loc, s + '\0'));
       mlir::Value addr = builder.createConvert(loc, charRefTy, stringAddress);
-      return builder.create<fir::InsertValueOp>(loc, itemListTy, list, addr,
-                                                builder.getArrayAttr(idx));
+      return fir::InsertValueOp::create(builder, loc, itemListTy, list, addr,
+                                        builder.getArrayAttr(idx));
     };
 
     size_t n = 0;
@@ -65,7 +65,7 @@ mlir::Value fir::runtime::genEnvironmentDefaults(
       list = insertStringField(def.defaultValue, idx);
       ++n;
     }
-    builder.create<fir::HasValueOp>(loc, list);
+    fir::HasValueOp::create(builder, loc, list);
   };
   builder.createGlobalConstant(loc, itemListTy, itemListName, listBuilder,
                                linkOnce);
@@ -73,27 +73,27 @@ mlir::Value fir::runtime::genEnvironmentDefaults(
   // Define the EnviornmentDefaultList object.
   auto envDefaultListBuilder = [&](fir::FirOpBuilder &builder) {
     mlir::Value envDefaultList =
-        builder.create<fir::UndefOp>(loc, envDefaultListTy);
+        fir::UndefOp::create(builder, loc, envDefaultListTy);
     mlir::Value numItems =
         builder.createIntegerConstant(loc, intTy, envDefaults.size());
-    envDefaultList = builder.create<fir::InsertValueOp>(
-        loc, envDefaultListTy, envDefaultList, numItems,
-        builder.getArrayAttr(zero));
+    envDefaultList = fir::InsertValueOp::create(builder, loc, envDefaultListTy,
+                                                envDefaultList, numItems,
+                                                builder.getArrayAttr(zero));
     fir::GlobalOp itemList = builder.getNamedGlobal(itemListName);
     assert(itemList && "missing environment default list");
-    mlir::Value listAddr = builder.create<fir::AddrOfOp>(
-        loc, itemList.resultType(), itemList.getSymbol());
-    envDefaultList = builder.create<fir::InsertValueOp>(
-        loc, envDefaultListTy, envDefaultList, listAddr,
-        builder.getArrayAttr(one));
-    builder.create<fir::HasValueOp>(loc, envDefaultList);
+    mlir::Value listAddr = fir::AddrOfOp::create(
+        builder, loc, itemList.resultType(), itemList.getSymbol());
+    envDefaultList = fir::InsertValueOp::create(builder, loc, envDefaultListTy,
+                                                envDefaultList, listAddr,
+                                                builder.getArrayAttr(one));
+    fir::HasValueOp::create(builder, loc, envDefaultList);
   };
   fir::GlobalOp envDefaultList = builder.createGlobalConstant(
       loc, envDefaultListTy, envDefaultListPtrName + ".list",
       envDefaultListBuilder, linkOnce);
 
   // Define the pointer to the list used by the runtime.
-  mlir::Value addr = builder.create<fir::AddrOfOp>(
-      loc, envDefaultList.resultType(), envDefaultList.getSymbol());
+  mlir::Value addr = fir::AddrOfOp::create(
+      builder, loc, envDefaultList.resultType(), envDefaultList.getSymbol());
   return addr;
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp b/flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp
index 0f66315696ac7..0256644ed06e7 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp
@@ -18,21 +18,21 @@ mlir::Value fir::runtime::genMapExcept(fir::FirOpBuilder &builder,
                                        mlir::Value excepts) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(MapException)>(loc, builder)};
-  return builder.create<fir::CallOp>(loc, func, excepts).getResult(0);
+  return fir::CallOp::create(builder, loc, func, excepts).getResult(0);
 }
 
 void fir::runtime::genFeclearexcept(fir::FirOpBuilder &builder,
                                     mlir::Location loc, mlir::Value excepts) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(feclearexcept)>(loc, builder)};
-  builder.create<fir::CallOp>(loc, func, excepts);
+  fir::CallOp::create(builder, loc, func, excepts);
 }
 
 void fir::runtime::genFeraiseexcept(fir::FirOpBuilder &builder,
                                     mlir::Location loc, mlir::Value excepts) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(feraiseexcept)>(loc, builder)};
-  builder.create<fir::CallOp>(loc, func, excepts);
+  fir::CallOp::create(builder, loc, func, excepts);
 }
 
 mlir::Value fir::runtime::genFetestexcept(fir::FirOpBuilder &builder,
@@ -40,28 +40,28 @@ mlir::Value fir::runtime::genFetestexcept(fir::FirOpBuilder &builder,
                                           mlir::Value excepts) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(fetestexcept)>(loc, builder)};
-  return builder.create<fir::CallOp>(loc, func, excepts).getResult(0);
+  return fir::CallOp::create(builder, loc, func, excepts).getResult(0);
 }
 
 void fir::runtime::genFedisableexcept(fir::FirOpBuilder &builder,
                                       mlir::Location loc, mlir::Value excepts) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(fedisableexcept)>(loc, builder)};
-  builder.create<fir::CallOp>(loc, func, excepts);
+  fir::CallOp::create(builder, loc, func, excepts);
 }
 
 void fir::runtime::genFeenableexcept(fir::FirOpBuilder &builder,
                                      mlir::Location loc, mlir::Value excepts) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(feenableexcept)>(loc, builder)};
-  builder.create<fir::CallOp>(loc, func, excepts);
+  fir::CallOp::create(builder, loc, func, excepts);
 }
 
 mlir::Value fir::runtime::genFegetexcept(fir::FirOpBuilder &builder,
                                          mlir::Location loc) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(fegetexcept)>(loc, builder)};
-  return builder.create<fir::CallOp>(loc, func).getResult(0);
+  return fir::CallOp::create(builder, loc, func).getResult(0);
 }
 
 mlir::Value fir::runtime::genSupportHalting(fir::FirOpBuilder &builder,
@@ -69,33 +69,33 @@ mlir::Value fir::runtime::genSupportHalting(fir::FirOpBuilder &builder,
                                             mlir::Value excepts) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(SupportHalting)>(loc, builder)};
-  return builder.create<fir::CallOp>(loc, func, excepts).getResult(0);
+  return fir::CallOp::create(builder, loc, func, excepts).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetUnderflowMode(fir::FirOpBuilder &builder,
                                               mlir::Location loc) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(GetUnderflowMode)>(loc, builder)};
-  return builder.create<fir::CallOp>(loc, func).getResult(0);
+  return fir::CallOp::create(builder, loc, func).getResult(0);
 }
 
 void fir::runtime::genSetUnderflowMode(fir::FirOpBuilder &builder,
                                        mlir::Location loc, mlir::Value flag) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(SetUnderflowMode)>(loc, builder)};
-  builder.create<fir::CallOp>(loc, func, flag);
+  fir::CallOp::create(builder, loc, func, flag);
 }
 
 mlir::Value fir::runtime::genGetModesTypeSize(fir::FirOpBuilder &builder,
                                               mlir::Location loc) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(GetModesTypeSize)>(loc, builder)};
-  return builder.create<fir::CallOp>(loc, func).getResult(0);
+  return fir::CallOp::create(builder, loc, func).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetStatusTypeSize(fir::FirOpBuilder &builder,
                                                mlir::Location loc) {
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(GetStatusTypeSize)>(loc, builder)};
-  return builder.create<fir::CallOp>(loc, func).getResult(0);
+  return fir::CallOp::create(builder, loc, func).getResult(0);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Execute.cpp b/flang/lib/Optimizer/Builder/Runtime/Execute.cpp
index 71ee3996ac0da..2f85fb45ea0d2 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Execute.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Execute.cpp
@@ -40,5 +40,5 @@ void fir::runtime::genExecuteCommandLine(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, runtimeFuncTy, command, wait, exitstat, cmdstat, cmdmsg,
       sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, runtimeFunc, args);
+  fir::CallOp::create(builder, loc, runtimeFunc, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp b/flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp
index 718c3533564e8..5e3f022310973 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp
@@ -26,7 +26,7 @@ mlir::Value fir::runtime::genLboundDim(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, array, dim,
                                             sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, lboundFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, lboundFunc, args).getResult(0);
 }
 
 void fir::runtime::genLbound(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -40,7 +40,7 @@ void fir::runtime::genLbound(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultAddr, array, kind, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `Ubound` runtime routine.  Calls to UBOUND with a DIM
@@ -57,7 +57,7 @@ void fir::runtime::genUbound(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, array,
                                             kind, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, uboundFunc, args);
+  fir::CallOp::create(builder, loc, uboundFunc, args);
 }
 
 /// Generate call to `Size` runtime routine. This routine is a version when
@@ -73,7 +73,7 @@ mlir::Value fir::runtime::genSizeDim(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, array, dim,
                                             sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, sizeFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, sizeFunc, args).getResult(0);
 }
 
 /// Generate call to `Size` runtime routine. This routine is a version when
@@ -88,7 +88,7 @@ mlir::Value fir::runtime::genSize(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
   auto args = fir::runtime::createArguments(builder, loc, fTy, array,
                                             sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, sizeFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, sizeFunc, args).getResult(0);
 }
 
 /// Generate call to `IsContiguous` runtime routine.
@@ -99,7 +99,7 @@ mlir::Value fir::runtime::genIsContiguous(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(IsContiguous)>(loc, builder);
   auto fTy = isContiguousFunc.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, array);
-  return builder.create<fir::CallOp>(loc, isContiguousFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, isContiguousFunc, args).getResult(0);
 }
 
 /// Generate call to `IsContiguousUpTo` runtime routine.
@@ -111,7 +111,7 @@ mlir::Value fir::runtime::genIsContiguousUpTo(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(IsContiguousUpTo)>(loc, builder);
   auto fTy = isContiguousFunc.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, array, dim);
-  return builder.create<fir::CallOp>(loc, isContiguousFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, isContiguousFunc, args).getResult(0);
 }
 
 void fir::runtime::genShape(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -125,5 +125,5 @@ void fir::runtime::genShape(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultAddr, array, kind, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 04703f7911176..4b4954a3e738c 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -52,14 +52,14 @@ mlir::Value fir::runtime::genAssociated(fir::FirOpBuilder &builder,
                                                                      builder);
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, func.getFunctionType(), pointer, target);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 mlir::Value fir::runtime::genCpuTime(fir::FirOpBuilder &builder,
                                      mlir::Location loc) {
   mlir::func::FuncOp func =
       fir::runtime::getRuntimeFunc<mkRTKey(CpuTime)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+  return fir::CallOp::create(builder, loc, func, mlir::ValueRange{})
       .getResult(0);
 }
 
@@ -103,7 +103,7 @@ void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, funcTy, dateBuffer, dateLen, timeBuffer, timeLen,
       zoneBuffer, zoneLen, sourceFile, sourceLine, values);
-  builder.create<fir::CallOp>(loc, callee, args);
+  fir::CallOp::create(builder, loc, callee, args);
 }
 
 void fir::runtime::genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -117,7 +117,7 @@ void fir::runtime::genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
 
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, runtimeFuncTy, values, time, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, runtimeFunc, args);
+  fir::CallOp::create(builder, loc, runtimeFunc, args);
 }
 
 void fir::runtime::genFree(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -125,8 +125,8 @@ void fir::runtime::genFree(fir::FirOpBuilder &builder, mlir::Location loc,
   auto runtimeFunc = fir::runtime::getRuntimeFunc<mkRTKey(Free)>(loc, builder);
   mlir::Type intPtrTy = builder.getIntPtrType();
 
-  builder.create<fir::CallOp>(loc, runtimeFunc,
-                              builder.createConvert(loc, intPtrTy, ptr));
+  fir::CallOp::create(builder, loc, runtimeFunc,
+                      builder.createConvert(loc, intPtrTy, ptr));
 }
 
 mlir::Value fir::runtime::genFseek(fir::FirOpBuilder &builder,
@@ -140,7 +140,7 @@ mlir::Value fir::runtime::genFseek(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args =
       fir::runtime::createArguments(builder, loc, runtimeFuncTy, unit, offset,
                                     whence, sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, runtimeFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, runtimeFunc, args).getResult(0);
   ;
 }
 
@@ -150,7 +150,7 @@ mlir::Value fir::runtime::genFtell(fir::FirOpBuilder &builder,
   mlir::FunctionType runtimeFuncTy = runtimeFunc.getFunctionType();
   llvm::SmallVector<mlir::Value> args =
       fir::runtime::createArguments(builder, loc, runtimeFuncTy, unit);
-  return builder.create<fir::CallOp>(loc, runtimeFunc, args).getResult(0);
+  return fir::CallOp::create(builder, loc, runtimeFunc, args).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetGID(fir::FirOpBuilder &builder,
@@ -158,7 +158,7 @@ mlir::Value fir::runtime::genGetGID(fir::FirOpBuilder &builder,
   auto runtimeFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(GetGID)>(loc, builder);
 
-  return builder.create<fir::CallOp>(loc, runtimeFunc).getResult(0);
+  return fir::CallOp::create(builder, loc, runtimeFunc).getResult(0);
 }
 
 mlir::Value fir::runtime::genGetUID(fir::FirOpBuilder &builder,
@@ -166,7 +166,7 @@ mlir::Value fir::runtime::genGetUID(fir::FirOpBuilder &builder,
   auto runtimeFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(GetUID)>(loc, builder);
 
-  return builder.create<fir::CallOp>(loc, runtimeFunc).getResult(0);
+  return fir::CallOp::create(builder, loc, runtimeFunc).getResult(0);
 }
 
 mlir::Value fir::runtime::genMalloc(fir::FirOpBuilder &builder,
@@ -187,7 +187,7 @@ void fir::runtime::genRandomInit(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::runtime::getRuntimeFunc<mkRTKey(RandomInit)>(loc, builder);
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, func.getFunctionType(), repeatable, imageDistinct);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genRandomNumber(fir::FirOpBuilder &builder,
@@ -207,7 +207,7 @@ void fir::runtime::genRandomNumber(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, funcTy.getInput(2));
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, funcTy, harvest, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genRandomSeed(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -224,7 +224,7 @@ void fir::runtime::genRandomSeed(fir::FirOpBuilder &builder, mlir::Location loc,
   if (staticArgCount == 0) {
     func = fir::runtime::getRuntimeFunc<mkRTKey(RandomSeedDefaultPut)>(loc,
                                                                        builder);
-    builder.create<fir::CallOp>(loc, func);
+    fir::CallOp::create(builder, loc, func);
     return;
   }
   mlir::FunctionType funcTy;
@@ -239,7 +239,7 @@ void fir::runtime::genRandomSeed(fir::FirOpBuilder &builder, mlir::Location loc,
         fir::factory::locationToLineNo(builder, loc, funcTy.getInput(4));
     args = fir::runtime::createArguments(builder, loc, funcTy, size, put, get,
                                          sourceFile, sourceLine);
-    builder.create<fir::CallOp>(loc, func, args);
+    fir::CallOp::create(builder, loc, func, args);
     return;
   }
   if (sizeIsPresent) {
@@ -256,7 +256,7 @@ void fir::runtime::genRandomSeed(fir::FirOpBuilder &builder, mlir::Location loc,
   sourceLine = fir::factory::locationToLineNo(builder, loc, funcTy.getInput(2));
   args = fir::runtime::createArguments(builder, loc, funcTy, argBox, sourceFile,
                                        sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// generate rename runtime call
@@ -274,14 +274,14 @@ void fir::runtime::genRename(fir::FirOpBuilder &builder, mlir::Location loc,
   llvm::SmallVector<mlir::Value> args =
       fir::runtime::createArguments(builder, loc, runtimeFuncTy, path1, path2,
                                     status, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, runtimeFunc, args);
+  fir::CallOp::create(builder, loc, runtimeFunc, args);
 }
 
 /// generate runtime call to time intrinsic
 mlir::Value fir::runtime::genTime(fir::FirOpBuilder &builder,
                                   mlir::Location loc) {
   auto func = fir::runtime::getRuntimeFunc<mkRTKey(time)>(loc, builder);
-  return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+  return fir::CallOp::create(builder, loc, func, mlir::ValueRange{})
       .getResult(0);
 }
 
@@ -298,7 +298,7 @@ void fir::runtime::genTransfer(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, sourceBox, moldBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// generate runtime call to transfer intrinsic with size argument
@@ -315,7 +315,7 @@ void fir::runtime::genTransferSize(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, sourceBox,
                                     moldBox, sourceFile, sourceLine, size);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// generate system_clock runtime call/s
@@ -332,11 +332,12 @@ void fir::runtime::genSystemClock(fir::FirOpBuilder &builder,
         mlir::dyn_cast<fir::HeapType>(type)) {
       // Check for a disassociated pointer or an unallocated allocatable.
       assert(!isOptionalArg && "invalid optional argument");
-      ifOp = builder.create<fir::IfOp>(loc, builder.genIsNotNullAddr(loc, arg),
-                                       /*withElseRegion=*/false);
+      ifOp = fir::IfOp::create(builder, loc, builder.genIsNotNullAddr(loc, arg),
+                               /*withElseRegion=*/false);
     } else if (isOptionalArg) {
-      ifOp = builder.create<fir::IfOp>(
-          loc, builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), arg),
+      ifOp = fir::IfOp::create(
+          builder, loc,
+          fir::IsPresentOp::create(builder, loc, builder.getI1Type(), arg),
           /*withElseRegion=*/false);
     }
     if (ifOp)
@@ -348,11 +349,11 @@ void fir::runtime::genSystemClock(fir::FirOpBuilder &builder,
       integerKind = intType.getWidth() / 8;
     mlir::Value kind = builder.createIntegerConstant(loc, kindTy, integerKind);
     mlir::Value res =
-        builder.create<fir::CallOp>(loc, func, mlir::ValueRange{kind})
+        fir::CallOp::create(builder, loc, func, mlir::ValueRange{kind})
             .getResult(0);
     mlir::Value castRes =
         builder.createConvert(loc, fir::dyn_cast_ptrEleTy(type), res);
-    builder.create<fir::StoreOp>(loc, castRes, arg);
+    fir::StoreOp::create(builder, loc, castRes, arg);
     if (ifOp)
       builder.setInsertionPointAfter(ifOp);
   };
@@ -373,24 +374,24 @@ void fir::runtime::genSignal(fir::FirOpBuilder &builder, mlir::Location loc,
                              mlir::Value status) {
   assert(mlir::isa<mlir::IntegerType>(number.getType()));
   mlir::Type int64 = builder.getIntegerType(64);
-  number = builder.create<fir::ConvertOp>(loc, int64, number);
+  number = fir::ConvertOp::create(builder, loc, int64, number);
 
   mlir::Type handlerUnwrappedTy = fir::unwrapRefType(handler.getType());
   if (mlir::isa_and_nonnull<mlir::IntegerType>(handlerUnwrappedTy)) {
     // pass the integer as a function pointer like one would to signal(2)
-    handler = builder.create<fir::LoadOp>(loc, handler);
+    handler = fir::LoadOp::create(builder, loc, handler);
     mlir::Type fnPtrTy = fir::LLVMPointerType::get(
         mlir::FunctionType::get(handler.getContext(), {}, {}));
-    handler = builder.create<fir::ConvertOp>(loc, fnPtrTy, handler);
+    handler = fir::ConvertOp::create(builder, loc, fnPtrTy, handler);
   } else {
     assert(mlir::isa<fir::BoxProcType>(handler.getType()));
-    handler = builder.create<fir::BoxAddrOp>(loc, handler);
+    handler = fir::BoxAddrOp::create(builder, loc, handler);
   }
 
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(Signal)>(loc, builder)};
   mlir::Value stat =
-      builder.create<fir::CallOp>(loc, func, mlir::ValueRange{number, handler})
+      fir::CallOp::create(builder, loc, func, mlir::ValueRange{number, handler})
           ->getResult(0);
 
   // return status code via status argument (if present)
@@ -398,12 +399,12 @@ void fir::runtime::genSignal(fir::FirOpBuilder &builder, mlir::Location loc,
     assert(mlir::isa<mlir::IntegerType>(fir::unwrapRefType(status.getType())));
     // status might be dynamically optional, so test if it is present
     mlir::Value isPresent =
-        builder.create<IsPresentOp>(loc, builder.getI1Type(), status);
+        IsPresentOp::create(builder, loc, builder.getI1Type(), status);
     builder.genIfOp(loc, /*results=*/{}, isPresent, /*withElseRegion=*/false)
         .genThen([&]() {
-          stat = builder.create<fir::ConvertOp>(
-              loc, fir::unwrapRefType(status.getType()), stat);
-          builder.create<fir::StoreOp>(loc, stat, status);
+          stat = fir::ConvertOp::create(
+              builder, loc, fir::unwrapRefType(status.getType()), stat);
+          fir::StoreOp::create(builder, loc, stat, status);
         })
         .end();
   }
@@ -412,10 +413,10 @@ void fir::runtime::genSignal(fir::FirOpBuilder &builder, mlir::Location loc,
 void fir::runtime::genSleep(fir::FirOpBuilder &builder, mlir::Location loc,
                             mlir::Value seconds) {
   mlir::Type int64 = builder.getIntegerType(64);
-  seconds = builder.create<fir::ConvertOp>(loc, int64, seconds);
+  seconds = fir::ConvertOp::create(builder, loc, int64, seconds);
   mlir::func::FuncOp func{
       fir::runtime::getRuntimeFunc<mkRTKey(Sleep)>(loc, builder)};
-  builder.create<fir::CallOp>(loc, func, seconds);
+  fir::CallOp::create(builder, loc, func, seconds);
 }
 
 /// generate chdir runtime call
@@ -425,5 +426,5 @@ mlir::Value fir::runtime::genChdir(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(Chdir)>(loc, builder)};
   llvm::SmallVector<mlir::Value> args =
       fir::runtime::createArguments(builder, loc, func.getFunctionType(), name);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Main.cpp b/flang/lib/Optimizer/Builder/Runtime/Main.cpp
index 973744837d378..d35f687167b05 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Main.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Main.cpp
@@ -62,17 +62,17 @@ void fir::runtime::genMain(
   llvm::SmallVector<mlir::Value, 4> args(block->getArguments());
   args.push_back(env);
 
-  builder.create<fir::CallOp>(loc, startFn, args);
+  fir::CallOp::create(builder, loc, startFn, args);
 
   if (initCuda) {
     auto initFn = builder.createFunction(
         loc, RTNAME_STRING(CUFInit), mlir::FunctionType::get(context, {}, {}));
-    builder.create<fir::CallOp>(loc, initFn);
+    fir::CallOp::create(builder, loc, initFn);
   }
 
-  builder.create<fir::CallOp>(loc, qqMainFn);
-  builder.create<fir::CallOp>(loc, stopFn);
+  fir::CallOp::create(builder, loc, qqMainFn);
+  fir::CallOp::create(builder, loc, stopFn);
 
   mlir::Value ret = builder.createIntegerConstant(loc, argcTy, 0);
-  builder.create<mlir::func::ReturnOp>(loc, ret);
+  mlir::func::ReturnOp::create(builder, loc, ret);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Numeric.cpp b/flang/lib/Optimizer/Builder/Runtime/Numeric.cpp
index 4ff7c86bb0a24..62d5e506cedf6 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Numeric.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Numeric.cpp
@@ -317,7 +317,7 @@ mlir::Value fir::runtime::genExponent(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args = {
       builder.createConvert(loc, funcTy.getInput(0), x)};
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Fraction intrinsic runtime routine.
@@ -340,7 +340,7 @@ mlir::Value fir::runtime::genFraction(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args = {
       builder.createConvert(loc, funcTy.getInput(0), x)};
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Mod intrinsic runtime routine.
@@ -370,7 +370,7 @@ mlir::Value fir::runtime::genMod(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args = fir::runtime::createArguments(builder, loc, funcTy, a, p,
                                             sourceFile, sourceLine);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Modulo intrinsic runtime routine.
@@ -403,7 +403,7 @@ mlir::Value fir::runtime::genModulo(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(builder, loc, funcTy, a, p,
                                             sourceFile, sourceLine);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Nearest intrinsic or a "Next" intrinsic module procedure.
@@ -427,7 +427,7 @@ mlir::Value fir::runtime::genNearest(fir::FirOpBuilder &builder,
   auto funcTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcTy, x, valueUp);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to RRSpacing intrinsic runtime routine.
@@ -451,7 +451,7 @@ mlir::Value fir::runtime::genRRSpacing(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args = {
       builder.createConvert(loc, funcTy.getInput(0), x)};
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to ErfcScaled intrinsic runtime routine.
@@ -475,7 +475,7 @@ mlir::Value fir::runtime::genErfcScaled(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args = {
       builder.createConvert(loc, funcTy.getInput(0), x)};
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Scale intrinsic runtime routine.
@@ -499,7 +499,7 @@ mlir::Value fir::runtime::genScale(fir::FirOpBuilder &builder,
   auto funcTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcTy, x, i);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Selected_char_kind intrinsic runtime routine.
@@ -519,7 +519,7 @@ mlir::Value fir::runtime::genSelectedCharKind(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(builder, loc, fTy, sourceFile,
                                             sourceLine, name, length);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Selected_int_kind intrinsic runtime routine.
@@ -540,7 +540,7 @@ mlir::Value fir::runtime::genSelectedIntKind(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(builder, loc, fTy, sourceFile,
                                             sourceLine, x, xKind);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Selected_logical_kind intrinsic runtime routine.
@@ -561,7 +561,7 @@ mlir::Value fir::runtime::genSelectedLogicalKind(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(builder, loc, fTy, sourceFile,
                                             sourceLine, x, xKind);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Selected_real_kind intrinsic runtime routine.
@@ -593,7 +593,7 @@ mlir::Value fir::runtime::genSelectedRealKind(fir::FirOpBuilder &builder,
                                             sourceLine, precision, pKind, range,
                                             rKind, radix, dKind);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Set_exponent intrinsic runtime routine.
@@ -617,7 +617,7 @@ mlir::Value fir::runtime::genSetExponent(fir::FirOpBuilder &builder,
   auto funcTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcTy, x, i);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to Spacing intrinsic runtime routine.
@@ -649,6 +649,6 @@ mlir::Value fir::runtime::genSpacing(fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> args = {
       builder.createConvert(loc, funcTy.getInput(0), x)};
 
-  mlir::Value res = builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  mlir::Value res = fir::CallOp::create(builder, loc, func, args).getResult(0);
   return builder.createConvert(loc, fltTy, res);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Pointer.cpp b/flang/lib/Optimizer/Builder/Runtime/Pointer.cpp
index 160c6515a7a9d..c03ff5885d91e 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Pointer.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Pointer.cpp
@@ -23,5 +23,5 @@ void fir::runtime::genPointerAssociateScalar(fir::FirOpBuilder &builder,
   mlir::FunctionType fTy{func.getFunctionType()};
   llvm::SmallVector<mlir::Value> args{
       fir::runtime::createArguments(builder, loc, fTy, desc, target)};
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Ragged.cpp b/flang/lib/Optimizer/Builder/Runtime/Ragged.cpp
index e5d0fb0fb27a9..e5cf96359ea38 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Ragged.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Ragged.cpp
@@ -34,25 +34,25 @@ void fir::runtime::genRaggedArrayAllocate(mlir::Location loc,
   auto eleTy = fir::unwrapSequenceType(fir::unwrapRefType(header.getType()));
   auto ptrTy =
       builder.getRefType(mlir::cast<mlir::TupleType>(eleTy).getType(1));
-  auto ptr = builder.create<fir::CoordinateOp>(loc, ptrTy, header, one);
-  auto heap = builder.create<fir::LoadOp>(loc, ptr);
+  auto ptr = fir::CoordinateOp::create(builder, loc, ptrTy, header, one);
+  auto heap = fir::LoadOp::create(builder, loc, ptr);
   auto cmp = builder.genIsNullAddr(loc, heap);
   builder.genIfThen(loc, cmp)
       .genThen([&]() {
         auto asHeadersVal = builder.createIntegerConstant(loc, i1Ty, asHeaders);
         auto rankVal = builder.createIntegerConstant(loc, i64Ty, rank);
-        auto buff = builder.create<fir::AllocMemOp>(loc, extentTy);
+        auto buff = fir::AllocMemOp::create(builder, loc, extentTy);
         // Convert all the extents to i64 and pack them in a buffer on the heap.
         for (auto i : llvm::enumerate(extents)) {
           auto offset = builder.createIntegerConstant(loc, i32Ty, i.index());
           auto addr =
-              builder.create<fir::CoordinateOp>(loc, refTy, buff, offset);
+              fir::CoordinateOp::create(builder, loc, refTy, buff, offset);
           auto castVal = builder.createConvert(loc, i64Ty, i.value());
-          builder.create<fir::StoreOp>(loc, castVal, addr);
+          fir::StoreOp::create(builder, loc, castVal, addr);
         }
         auto args = fir::runtime::createArguments(
             builder, loc, fTy, header, asHeadersVal, rankVal, eleSize, buff);
-        builder.create<fir::CallOp>(loc, func, args);
+        fir::CallOp::create(builder, loc, func, args);
       })
       .end();
 }
@@ -64,5 +64,5 @@ void fir::runtime::genRaggedArrayDeallocate(mlir::Location loc,
       loc, builder);
   auto fTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, header);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Reduction.cpp b/flang/lib/Optimizer/Builder/Runtime/Reduction.cpp
index f778b963c59ca..157d4358329ce 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Reduction.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Reduction.cpp
@@ -1155,7 +1155,7 @@ mlir::Value genSpecial2Args(FN func, fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
   auto args = fir::runtime::createArguments(builder, loc, fTy, maskBox,
                                             sourceFile, sourceLine, dim);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate calls to reduction intrinsics such as All and Any.
@@ -1171,7 +1171,7 @@ static void genReduction2Args(FN func, fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, maskBox, dim, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate calls to reduction intrinsics such as Maxval and Minval.
@@ -1189,7 +1189,7 @@ static void genReduction3Args(FN func, fir::FirOpBuilder &builder,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox, dim,
                                     sourceFile, sourceLine, maskBox);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate calls to reduction intrinsics such as Maxloc and Minloc.
@@ -1206,7 +1206,7 @@ static void genReduction4Args(FN func, fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             arrayBox, kind, sourceFile,
                                             sourceLine, maskBox, back);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate calls to reduction intrinsics such as Maxloc and Minloc.
@@ -1223,7 +1223,7 @@ genReduction5Args(FN func, fir::FirOpBuilder &builder, mlir::Location loc,
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             arrayBox, kind, dim, sourceFile,
                                             sourceLine, maskBox, back);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `AllDim` runtime routine.
@@ -1296,7 +1296,7 @@ void fir::runtime::genCountDim(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(5));
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, maskBox, dim, kind, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `Findloc` intrinsic runtime routine. This is the version
@@ -1313,7 +1313,7 @@ void fir::runtime::genFindloc(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             arrayBox, valBox, kind, sourceFile,
                                             sourceLine, maskBox, back);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `FindlocDim` intrinsic runtime routine. This is the version
@@ -1331,7 +1331,7 @@ void fir::runtime::genFindlocDim(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, arrayBox, valBox, kind, dim, sourceFile,
       sourceLine, maskBox, back);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `Maxloc` intrinsic runtime routine. This is the version
@@ -1392,7 +1392,7 @@ mlir::Value fir::runtime::genMaxval(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, arrayBox, sourceFile, sourceLine, dim, maskBox);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to `MaxvalDim` intrinsic runtime routine. This is the version
@@ -1417,7 +1417,7 @@ void fir::runtime::genMaxvalChar(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, arrayBox, sourceFile, sourceLine, maskBox);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `Minloc` intrinsic runtime routine. This is the version
@@ -1476,7 +1476,7 @@ void fir::runtime::genMinvalChar(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, arrayBox, sourceFile, sourceLine, maskBox);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `Minval` intrinsic runtime routine. This is the version
@@ -1504,7 +1504,7 @@ mlir::Value fir::runtime::genMinval(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, arrayBox, sourceFile, sourceLine, dim, maskBox);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to `Norm2Dim` intrinsic runtime routine. This is the version
@@ -1527,7 +1527,7 @@ void fir::runtime::genNorm2Dim(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, arrayBox, dim, sourceFile, sourceLine);
 
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `Norm2` intrinsic runtime routine. This is the version
@@ -1558,7 +1558,7 @@ mlir::Value fir::runtime::genNorm2(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(builder, loc, fTy, arrayBox,
                                             sourceFile, sourceLine, dim);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to `Parity` intrinsic runtime routine. This routine is
@@ -1604,7 +1604,7 @@ mlir::Value fir::runtime::genProduct(fir::FirOpBuilder &builder,
     auto args =
         fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox,
                                       sourceFile, sourceLine, dim, maskBox);
-    builder.create<fir::CallOp>(loc, func, args);
+    fir::CallOp::create(builder, loc, func, args);
     return resultBox;
   }
 
@@ -1613,7 +1613,7 @@ mlir::Value fir::runtime::genProduct(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, arrayBox, sourceFile, sourceLine, dim, maskBox);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 /// Generate call to `DotProduct` intrinsic runtime routine.
@@ -1645,7 +1645,7 @@ mlir::Value fir::runtime::genDotProduct(fir::FirOpBuilder &builder,
     auto args =
         fir::runtime::createArguments(builder, loc, fTy, resultBox, vectorABox,
                                       vectorBBox, sourceFile, sourceLine);
-    builder.create<fir::CallOp>(loc, func, args);
+    fir::CallOp::create(builder, loc, func, args);
     return resultBox;
   }
 
@@ -1653,7 +1653,7 @@ mlir::Value fir::runtime::genDotProduct(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, vectorABox,
                                             vectorBBox, sourceFile, sourceLine);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 /// Generate call to `SumDim` intrinsic runtime routine. This is the version
 /// that handles any rank array with the dim argument specified.
@@ -1688,7 +1688,7 @@ mlir::Value fir::runtime::genSum(fir::FirOpBuilder &builder, mlir::Location loc,
     auto args =
         fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox,
                                       sourceFile, sourceLine, dim, maskBox);
-    builder.create<fir::CallOp>(loc, func, args);
+    fir::CallOp::create(builder, loc, func, args);
     return resultBox;
   }
 
@@ -1697,7 +1697,7 @@ mlir::Value fir::runtime::genSum(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, arrayBox, sourceFile, sourceLine, dim, maskBox);
 
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 // The IAll, IAny and IParity intrinsics have essentially the same
@@ -1733,7 +1733,7 @@ mlir::Value fir::runtime::genSum(fir::FirOpBuilder &builder, mlir::Location loc,
     auto args = fir::runtime::createArguments(                                 \
         builder, loc, fTy, arrayBox, sourceFile, sourceLine, dim, maskBox);    \
                                                                                \
-    return builder.create<fir::CallOp>(loc, func, args).getResult(0);          \
+    return fir::CallOp::create(builder, loc, func, args).getResult(0);         \
   }
 
 /// Generate call to `IAllDim` intrinsic runtime routine. This is the version
@@ -1819,11 +1819,12 @@ void fir::runtime::genReduce(fir::FirOpBuilder &builder, mlir::Location loc,
   auto sourceFile = fir::factory::locationToFilename(builder, loc);
   auto sourceLine =
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
-  auto opAddr = builder.create<fir::BoxAddrOp>(loc, fTy.getInput(2), operation);
+  auto opAddr =
+      fir::BoxAddrOp::create(builder, loc, fTy.getInput(2), operation);
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, arrayBox, opAddr, sourceFile, sourceLine,
       dim, maskBox, identity, ordered);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `Reduce` intrinsic runtime routine. This is the version
@@ -1864,11 +1865,12 @@ mlir::Value fir::runtime::genReduce(fir::FirOpBuilder &builder,
   auto sourceFile = fir::factory::locationToFilename(builder, loc);
   auto sourceLine =
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
-  auto opAddr = builder.create<fir::BoxAddrOp>(loc, fTy.getInput(1), operation);
+  auto opAddr =
+      fir::BoxAddrOp::create(builder, loc, fTy.getInput(1), operation);
   auto args = fir::runtime::createArguments(builder, loc, fTy, arrayBox, opAddr,
                                             sourceFile, sourceLine, dim,
                                             maskBox, identity, ordered);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 void fir::runtime::genReduceDim(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -1912,9 +1914,10 @@ void fir::runtime::genReduceDim(fir::FirOpBuilder &builder, mlir::Location loc,
 
   auto sourceLine =
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
-  auto opAddr = builder.create<fir::BoxAddrOp>(loc, fTy.getInput(2), operation);
+  auto opAddr =
+      fir::BoxAddrOp::create(builder, loc, fTy.getInput(2), operation);
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, arrayBox, opAddr, sourceFile, sourceLine,
       dim, maskBox, identity, ordered);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
index 9b5e43b80b1f6..5629371947641 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
@@ -19,13 +19,13 @@ void fir::runtime::genExit(fir::FirOpBuilder &builder, mlir::Location loc,
   auto exitFunc = fir::runtime::getRuntimeFunc<mkRTKey(Exit)>(loc, builder);
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, exitFunc.getFunctionType(), status);
-  builder.create<fir::CallOp>(loc, exitFunc, args);
+  fir::CallOp::create(builder, loc, exitFunc, args);
 }
 
 void fir::runtime::genAbort(fir::FirOpBuilder &builder, mlir::Location loc) {
   mlir::func::FuncOp abortFunc =
       fir::runtime::getRuntimeFunc<mkRTKey(Abort)>(loc, builder);
-  builder.create<fir::CallOp>(loc, abortFunc, mlir::ValueRange{});
+  fir::CallOp::create(builder, loc, abortFunc, mlir::ValueRange{});
 }
 
 void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,
@@ -41,5 +41,5 @@ void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,
   mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
   llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
       builder, loc, funcTy, msgVal, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, crashFunc, args);
+  fir::CallOp::create(builder, loc, crashFunc, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Support.cpp b/flang/lib/Optimizer/Builder/Runtime/Support.cpp
index b5e9ddb87c7c4..d0d48ad718da4 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Support.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Support.cpp
@@ -42,7 +42,7 @@ void fir::runtime::genCopyAndUpdateDescriptor(fir::FirOpBuilder &builder,
     func.setArgAttr(0, noCapture, unitAttr);
     func.setArgAttr(1, noCapture, unitAttr);
   }
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 mlir::Value fir::runtime::genIsAssumedSize(fir::FirOpBuilder &builder,
@@ -52,5 +52,5 @@ mlir::Value fir::runtime::genIsAssumedSize(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(IsAssumedSize)>(loc, builder);
   auto fTy = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, fTy, box);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/TemporaryStack.cpp b/flang/lib/Optimizer/Builder/Runtime/TemporaryStack.cpp
index 732152c823a06..effd712079753 100644
--- a/flang/lib/Optimizer/Builder/Runtime/TemporaryStack.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/TemporaryStack.cpp
@@ -23,7 +23,7 @@ mlir::Value fir::runtime::genCreateValueStack(mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, funcType.getInput(1));
   auto args = fir::runtime::createArguments(builder, loc, funcType, sourceFile,
                                             sourceLine);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 void fir::runtime::genPushValue(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -33,7 +33,7 @@ void fir::runtime::genPushValue(mlir::Location loc, fir::FirOpBuilder &builder,
   mlir::FunctionType funcType = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcType, opaquePtr,
                                             boxValue);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genValueAt(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -44,7 +44,7 @@ void fir::runtime::genValueAt(mlir::Location loc, fir::FirOpBuilder &builder,
   mlir::FunctionType funcType = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcType, opaquePtr,
                                             i, retValueBox);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genDestroyValueStack(mlir::Location loc,
@@ -54,7 +54,7 @@ void fir::runtime::genDestroyValueStack(mlir::Location loc,
       fir::runtime::getRuntimeFunc<mkRTKey(DestroyValueStack)>(loc, builder);
   mlir::FunctionType funcType = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcType, opaquePtr);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 mlir::Value fir::runtime::genCreateDescriptorStack(mlir::Location loc,
@@ -68,7 +68,7 @@ mlir::Value fir::runtime::genCreateDescriptorStack(mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, funcType.getInput(1));
   auto args = fir::runtime::createArguments(builder, loc, funcType, sourceFile,
                                             sourceLine);
-  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+  return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
 void fir::runtime::genPushDescriptor(mlir::Location loc,
@@ -80,7 +80,7 @@ void fir::runtime::genPushDescriptor(mlir::Location loc,
   mlir::FunctionType funcType = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcType, opaquePtr,
                                             boxDescriptor);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genDescriptorAt(mlir::Location loc,
@@ -92,7 +92,7 @@ void fir::runtime::genDescriptorAt(mlir::Location loc,
   mlir::FunctionType funcType = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcType, opaquePtr,
                                             i, retDescriptorBox);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 void fir::runtime::genDestroyDescriptorStack(mlir::Location loc,
@@ -103,5 +103,5 @@ void fir::runtime::genDestroyDescriptorStack(mlir::Location loc,
                                                                     builder);
   mlir::FunctionType funcType = func.getFunctionType();
   auto args = fir::runtime::createArguments(builder, loc, funcType, opaquePtr);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }

diff  --git a/flang/lib/Optimizer/Builder/Runtime/Transformational.cpp b/flang/lib/Optimizer/Builder/Runtime/Transformational.cpp
index 47744b0facb53..6251def5cfb17 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Transformational.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Transformational.cpp
@@ -170,7 +170,7 @@ void fir::runtime::genBesselJn(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, n2, x,
                                     bn2, bn2_1, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `BesselJn` intrinsic. This is used when `x == 0.0`.
@@ -196,7 +196,7 @@ void fir::runtime::genBesselJnX0(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, n1,
                                             n2, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `BesselYn` intrinsic.
@@ -225,7 +225,7 @@ void fir::runtime::genBesselYn(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, n2, x,
                                     bn1, bn1_1, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to `BesselYn` intrinsic. This is used when `x == 0.0`.
@@ -251,7 +251,7 @@ void fir::runtime::genBesselYnX0(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, n1,
                                             n2, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to Cshift intrinsic
@@ -266,7 +266,7 @@ void fir::runtime::genCshift(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox,
                                     shiftBox, dimBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, cshiftFunc, args);
+  fir::CallOp::create(builder, loc, cshiftFunc, args);
 }
 
 /// Generate call to the vector version of the Cshift intrinsic
@@ -282,7 +282,7 @@ void fir::runtime::genCshiftVector(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
   auto args = fir::runtime::createArguments(
       builder, loc, fTy, resultBox, arrayBox, shiftBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, cshiftFunc, args);
+  fir::CallOp::create(builder, loc, cshiftFunc, args);
 }
 
 /// Generate call to Eoshift intrinsic
@@ -299,7 +299,7 @@ void fir::runtime::genEoshift(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             arrayBox, shiftBox, boundBox,
                                             dimBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, eoshiftFunc, args);
+  fir::CallOp::create(builder, loc, eoshiftFunc, args);
 }
 
 /// Generate call to the vector version of the Eoshift intrinsic
@@ -318,7 +318,7 @@ void fir::runtime::genEoshiftVector(fir::FirOpBuilder &builder,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox,
                                     shiftBox, boundBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, eoshiftFunc, args);
+  fir::CallOp::create(builder, loc, eoshiftFunc, args);
 }
 
 /// Define ForcedMatmul<ACAT><AKIND><BCAT><BKIND> models.
@@ -388,7 +388,7 @@ void fir::runtime::genMatmul(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, matrixABox,
                                     matrixBBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Define ForcedMatmulTranspose<ACAT><AKIND><BCAT><BKIND> models.
@@ -440,7 +440,7 @@ void fir::runtime::genMatmulTranspose(fir::FirOpBuilder &builder,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, matrixABox,
                                     matrixBBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to Pack intrinsic runtime routine.
@@ -455,7 +455,7 @@ void fir::runtime::genPack(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox,
                                     maskBox, vectorBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, packFunc, args);
+  fir::CallOp::create(builder, loc, packFunc, args);
 }
 
 /// Generate call to Reshape intrinsic runtime routine.
@@ -471,7 +471,7 @@ void fir::runtime::genReshape(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             sourceBox, shapeBox, padBox,
                                             orderBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to ShallowCopy[Direct] runtime routine.
@@ -491,7 +491,7 @@ void fir::runtime::genShallowCopy(fir::FirOpBuilder &builder,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             arrayBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, packFunc, args);
+  fir::CallOp::create(builder, loc, packFunc, args);
 }
 
 /// Generate call to Spread intrinsic runtime routine.
@@ -506,7 +506,7 @@ void fir::runtime::genSpread(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, sourceBox,
                                     dim, ncopies, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to Transpose intrinsic runtime routine.
@@ -519,7 +519,7 @@ void fir::runtime::genTranspose(fir::FirOpBuilder &builder, mlir::Location loc,
       fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
   auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox,
                                             sourceBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, func, args);
+  fir::CallOp::create(builder, loc, func, args);
 }
 
 /// Generate call to Unpack intrinsic runtime routine.
@@ -534,5 +534,5 @@ void fir::runtime::genUnpack(fir::FirOpBuilder &builder, mlir::Location loc,
   auto args =
       fir::runtime::createArguments(builder, loc, fTy, resultBox, vectorBox,
                                     maskBox, fieldBox, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, unpackFunc, args);
+  fir::CallOp::create(builder, loc, unpackFunc, args);
 }


        

From flang-commits at lists.llvm.org  Mon Jul 21 16:57:04 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 16:57:04 -0700 (PDT)
Subject: [flang-commits] [flang] 9844ba6 - [mlir][NFC] update
 `flang/Optimizer/Builder` create APIs (9/n) (#149917)
Message-ID: <687ed3d0.050a0220.20336f.1606@mx.google.com>


Author: Maksim Levental
Date: 2025-07-21T19:57:00-04:00
New Revision: 9844ba6d9740206129b52633c555f767eaa45581

URL: https://github.com/llvm/llvm-project/commit/9844ba6d9740206129b52633c555f767eaa45581
DIFF: https://github.com/llvm/llvm-project/commit/9844ba6d9740206129b52633c555f767eaa45581.diff

LOG: [mlir][NFC] update `flang/Optimizer/Builder` create APIs (9/n) (#149917)

See https://github.com/llvm/llvm-project/pull/147168 for more info.

Added: 
    

Modified: 
    flang/lib/Optimizer/Builder/CUFCommon.cpp
    flang/lib/Optimizer/Builder/Character.cpp
    flang/lib/Optimizer/Builder/Complex.cpp
    flang/lib/Optimizer/Builder/DoLoopHelper.cpp
    flang/lib/Optimizer/Builder/FIRBuilder.cpp
    flang/lib/Optimizer/Builder/HLFIRTools.cpp
    flang/lib/Optimizer/Builder/IntrinsicCall.cpp
    flang/lib/Optimizer/Builder/MutableBox.cpp
    flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
    flang/lib/Optimizer/Builder/TemporaryStorage.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Builder/CUFCommon.cpp b/flang/lib/Optimizer/Builder/CUFCommon.cpp
index dcbf4991907bf..cf7588f275d22 100644
--- a/flang/lib/Optimizer/Builder/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Builder/CUFCommon.cpp
@@ -25,8 +25,8 @@ mlir::gpu::GPUModuleOp cuf::getOrCreateGPUModule(mlir::ModuleOp mod,
                mlir::UnitAttr::get(ctx));
 
   mlir::OpBuilder builder(ctx);
-  auto gpuMod = builder.create<mlir::gpu::GPUModuleOp>(mod.getLoc(),
-                                                       cudaDeviceModuleName);
+  auto gpuMod = mlir::gpu::GPUModuleOp::create(builder, mod.getLoc(),
+                                               cudaDeviceModuleName);
   mlir::Block::iterator insertPt(mod.getBodyRegion().front().end());
   symTab.insert(gpuMod, insertPt);
   return gpuMod;
@@ -84,8 +84,8 @@ void cuf::genPointerSync(const mlir::Value box, fir::FirOpBuilder &builder) {
       if (auto globalOp =
               mod.lookupSymbol<fir::GlobalOp>(addrOfOp.getSymbol())) {
         if (cuf::isRegisteredDeviceGlobal(globalOp)) {
-          builder.create<cuf::SyncDescriptorOp>(box.getLoc(),
-                                                addrOfOp.getSymbol());
+          cuf::SyncDescriptorOp::create(builder, box.getLoc(),
+                                        addrOfOp.getSymbol());
         }
       }
     }

diff  --git a/flang/lib/Optimizer/Builder/Character.cpp b/flang/lib/Optimizer/Builder/Character.cpp
index 61428ac490a46..a096099a04fe8 100644
--- a/flang/lib/Optimizer/Builder/Character.cpp
+++ b/flang/lib/Optimizer/Builder/Character.cpp
@@ -112,8 +112,8 @@ fir::factory::CharacterExprHelper::materializeValue(mlir::Value str) {
   }
   auto len = builder.createIntegerConstant(
       loc, builder.getCharacterLengthType(), charTy.getLen());
-  auto temp = builder.create<fir::AllocaOp>(loc, charTy);
-  builder.create<fir::StoreOp>(loc, str, temp);
+  auto temp = fir::AllocaOp::create(builder, loc, charTy);
+  fir::StoreOp::create(builder, loc, str, temp);
   LLVM_DEBUG(llvm::dbgs() << "materialized as local: " << str << " -> (" << temp
                           << ", " << len << ")\n");
   return {temp, len};
@@ -163,7 +163,7 @@ fir::factory::CharacterExprHelper::toExtendedValue(mlir::Value character,
     }
     if (!boxCharLen) {
       auto unboxed =
-          builder.create<fir::UnboxCharOp>(loc, refType, lenType, character);
+          fir::UnboxCharOp::create(builder, loc, refType, lenType, character);
       base = builder.createConvert(loc, refType, unboxed.getResult(0));
       boxCharLen = unboxed.getResult(1);
     }
@@ -208,7 +208,7 @@ fir::factory::CharacterExprHelper::createEmbox(const fir::CharBoxValue &box) {
   // not in memory.
   if (!fir::isa_ref_type(buff.getType())) {
     auto temp = builder.createTemporary(loc, buff.getType());
-    builder.create<fir::StoreOp>(loc, buff, temp);
+    fir::StoreOp::create(builder, loc, buff, temp);
     buff = temp;
   }
   // fir.emboxchar only accepts scalar, cast array buffer to a scalar buffer.
@@ -218,7 +218,7 @@ fir::factory::CharacterExprHelper::createEmbox(const fir::CharBoxValue &box) {
   // be used in boxchar.
   auto len = builder.createConvert(loc, builder.getCharacterLengthType(),
                                    box.getLen());
-  return builder.create<fir::EmboxCharOp>(loc, boxCharType, buff, len);
+  return fir::EmboxCharOp::create(builder, loc, boxCharType, buff, len);
 }
 
 fir::CharBoxValue fir::factory::CharacterExprHelper::toScalarCharacter(
@@ -231,8 +231,8 @@ fir::CharBoxValue fir::factory::CharacterExprHelper::toScalarCharacter(
   auto lenType = builder.getCharacterLengthType();
   auto len = builder.createConvert(loc, lenType, box.getLen());
   for (auto extent : box.getExtents())
-    len = builder.create<mlir::arith::MulIOp>(
-        loc, len, builder.createConvert(loc, lenType, extent));
+    len = mlir::arith::MulIOp::create(
+        builder, loc, len, builder.createConvert(loc, lenType, extent));
 
   // TODO: typeLen can be improved in compiled constant cases
   // TODO: allow bare fir.array<> (no ref) conversion here ?
@@ -277,7 +277,7 @@ fir::factory::CharacterExprHelper::createElementAddr(mlir::Value buffer,
 
   auto coor = builder.createConvert(loc, coorTy, buffer);
   auto i = builder.createConvert(loc, builder.getIndexType(), index);
-  return builder.create<fir::CoordinateOp>(loc, singleRefTy, coor, i);
+  return fir::CoordinateOp::create(builder, loc, singleRefTy, coor, i);
 }
 
 /// Load a character out of `buff` from offset `index`.
@@ -287,7 +287,7 @@ fir::factory::CharacterExprHelper::createLoadCharAt(mlir::Value buff,
                                                     mlir::Value index) {
   LLVM_DEBUG(llvm::dbgs() << "load a char: " << buff << " type: "
                           << buff.getType() << " at: " << index << '\n');
-  return builder.create<fir::LoadOp>(loc, createElementAddr(buff, index));
+  return fir::LoadOp::create(builder, loc, createElementAddr(buff, index));
 }
 
 /// Store the singleton character `c` to `str` at offset `index`.
@@ -299,7 +299,7 @@ void fir::factory::CharacterExprHelper::createStoreCharAt(mlir::Value str,
                           << " type: " << str.getType() << " at: " << index
                           << '\n');
   auto addr = createElementAddr(str, index);
-  builder.create<fir::StoreOp>(loc, c, addr);
+  fir::StoreOp::create(builder, loc, c, addr);
 }
 
 // FIXME: this temp is useless... either fir.coordinate_of needs to
@@ -311,8 +311,8 @@ mlir::Value fir::factory::CharacterExprHelper::getCharBoxBuffer(
     const fir::CharBoxValue &box) {
   auto buff = box.getBuffer();
   if (fir::isa_char(buff.getType())) {
-    auto newBuff = builder.create<fir::AllocaOp>(loc, buff.getType());
-    builder.create<fir::StoreOp>(loc, buff, newBuff);
+    auto newBuff = fir::AllocaOp::create(builder, loc, buff.getType());
+    fir::StoreOp::create(builder, loc, buff, newBuff);
     return newBuff;
   }
   return buff;
@@ -339,19 +339,19 @@ void fir::factory::CharacterExprHelper::createCopy(
     auto kindBytes = builder.createIntegerConstant(loc, i64Ty, bytes);
     auto castCount = builder.createConvert(loc, i64Ty, count);
     auto totalBytes =
-        builder.create<mlir::arith::MulIOp>(loc, kindBytes, castCount);
+        mlir::arith::MulIOp::create(builder, loc, kindBytes, castCount);
     auto llvmPointerType =
         mlir::LLVM::LLVMPointerType::get(builder.getContext());
     auto toPtr = builder.createConvert(loc, llvmPointerType, toBuff);
     auto fromPtr = builder.createConvert(loc, llvmPointerType, fromBuff);
-    builder.create<mlir::LLVM::MemmoveOp>(loc, toPtr, fromPtr, totalBytes,
-                                          isVolatile);
+    mlir::LLVM::MemmoveOp::create(builder, loc, toPtr, fromPtr, totalBytes,
+                                  isVolatile);
     return;
   }
 
   // Convert a CHARACTER of one KIND into a CHARACTER of another KIND.
-  builder.create<fir::CharConvertOp>(loc, src.getBuffer(), count,
-                                     dest.getBuffer());
+  fir::CharConvertOp::create(builder, loc, src.getBuffer(), count,
+                             dest.getBuffer());
 }
 
 void fir::factory::CharacterExprHelper::createPadding(
@@ -397,7 +397,7 @@ fir::CharBoxValue fir::factory::CharacterExprHelper::createTempFrom(
   } else {
     auto ref = builder.createConvert(loc, builder.getRefType(sourceTy),
                                      temp.getBuffer());
-    builder.create<fir::StoreOp>(loc, charBox->getBuffer(), ref);
+    fir::StoreOp::create(builder, loc, charBox->getBuffer(), ref);
   }
   return temp;
 }
@@ -412,23 +412,23 @@ void fir::factory::CharacterExprHelper::createLengthOneAssign(
     auto fromCharLen1RefTy = builder.getRefType(getSingletonCharType(
         builder.getContext(),
         getCharacterKind(fir::unwrapRefType(val.getType()))));
-    val = builder.create<fir::LoadOp>(
-        loc, builder.createConvert(loc, fromCharLen1RefTy, val));
+    val = fir::LoadOp::create(
+        builder, loc, builder.createConvert(loc, fromCharLen1RefTy, val));
   }
   auto toCharLen1Ty =
       getSingletonCharType(builder.getContext(), getCharacterKind(toTy));
   val = builder.createConvert(loc, toCharLen1Ty, val);
-  builder.create<fir::StoreOp>(
-      loc, val,
+  fir::StoreOp::create(
+      builder, loc, val,
       builder.createConvert(loc, builder.getRefType(toCharLen1Ty), addr));
 }
 
 /// Returns the minimum of integer mlir::Value \p a and \b.
 mlir::Value genMin(fir::FirOpBuilder &builder, mlir::Location loc,
                    mlir::Value a, mlir::Value b) {
-  auto cmp = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::slt, a, b);
-  return builder.create<mlir::arith::SelectOp>(loc, cmp, a, b);
+  auto cmp = mlir::arith::CmpIOp::create(builder, loc,
+                                         mlir::arith::CmpIPredicate::slt, a, b);
+  return mlir::arith::SelectOp::create(builder, loc, cmp, a, b);
 }
 
 void fir::factory::CharacterExprHelper::createAssign(
@@ -479,7 +479,7 @@ void fir::factory::CharacterExprHelper::createAssign(
   if (!compileTimeSameLength) {
     auto one = builder.createIntegerConstant(loc, lhs.getLen().getType(), 1);
     auto maxPadding =
-        builder.create<mlir::arith::SubIOp>(loc, lhs.getLen(), one);
+        mlir::arith::SubIOp::create(builder, loc, lhs.getLen(), one);
     createPadding(lhs, copyCount, maxPadding);
   }
 }
@@ -490,18 +490,19 @@ fir::CharBoxValue fir::factory::CharacterExprHelper::createConcatenate(
                                       lhs.getLen());
   auto rhsLen = builder.createConvert(loc, builder.getCharacterLengthType(),
                                       rhs.getLen());
-  mlir::Value len = builder.create<mlir::arith::AddIOp>(loc, lhsLen, rhsLen);
+  mlir::Value len = mlir::arith::AddIOp::create(builder, loc, lhsLen, rhsLen);
   auto temp = createCharacterTemp(getCharacterType(rhs), len);
   createCopy(temp, lhs, lhsLen);
   auto one = builder.createIntegerConstant(loc, len.getType(), 1);
-  auto upperBound = builder.create<mlir::arith::SubIOp>(loc, len, one);
+  auto upperBound = mlir::arith::SubIOp::create(builder, loc, len, one);
   auto lhsLenIdx = builder.createConvert(loc, builder.getIndexType(), lhsLen);
   auto fromBuff = getCharBoxBuffer(rhs);
   auto toBuff = getCharBoxBuffer(temp);
   fir::factory::DoLoopHelper{builder, loc}.createLoop(
       lhsLenIdx, upperBound, one,
       [&](fir::FirOpBuilder &bldr, mlir::Value index) {
-        auto rhsIndex = bldr.create<mlir::arith::SubIOp>(loc, index, lhsLenIdx);
+        auto rhsIndex =
+            mlir::arith::SubIOp::create(bldr, loc, index, lhsLenIdx);
         auto charVal = createLoadCharAt(fromBuff, rhsIndex);
         createStoreCharAt(toBuff, index, charVal);
       });
@@ -514,7 +515,7 @@ mlir::Value fir::factory::CharacterExprHelper::genSubstringBase(
   if (!one)
     one = builder.createIntegerConstant(loc, lowerBound.getType(), 1);
   auto offset =
-      builder.create<mlir::arith::SubIOp>(loc, lowerBound, one).getResult();
+      mlir::arith::SubIOp::create(builder, loc, lowerBound, one).getResult();
   auto addr = createElementAddr(stringRawAddr, offset);
   return builder.createConvert(loc, substringAddrType, addr);
 }
@@ -545,19 +546,19 @@ fir::CharBoxValue fir::factory::CharacterExprHelper::createSubstring(
   mlir::Value substringLen;
   if (nbounds < 2) {
     substringLen =
-        builder.create<mlir::arith::SubIOp>(loc, box.getLen(), castBounds[0]);
+        mlir::arith::SubIOp::create(builder, loc, box.getLen(), castBounds[0]);
   } else {
     substringLen =
-        builder.create<mlir::arith::SubIOp>(loc, castBounds[1], castBounds[0]);
+        mlir::arith::SubIOp::create(builder, loc, castBounds[1], castBounds[0]);
   }
-  substringLen = builder.create<mlir::arith::AddIOp>(loc, substringLen, one);
+  substringLen = mlir::arith::AddIOp::create(builder, loc, substringLen, one);
 
   // Set length to zero if bounds were reversed (Fortran 2018 9.4.1)
   auto zero = builder.createIntegerConstant(loc, substringLen.getType(), 0);
-  auto cdt = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::slt, substringLen, zero);
+  auto cdt = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::slt, substringLen, zero);
   substringLen =
-      builder.create<mlir::arith::SelectOp>(loc, cdt, zero, substringLen);
+      mlir::arith::SelectOp::create(builder, loc, cdt, zero, substringLen);
 
   return {substringRef, substringLen};
 }
@@ -573,11 +574,11 @@ fir::factory::CharacterExprHelper::createLenTrim(const fir::CharBoxValue &str) {
   auto zero = builder.createIntegerConstant(loc, indexType, 0);
   auto trueVal = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
   auto blank = createBlankConstantCode(getCharacterType(str));
-  mlir::Value lastChar = builder.create<mlir::arith::SubIOp>(loc, len, one);
+  mlir::Value lastChar = mlir::arith::SubIOp::create(builder, loc, len, one);
 
   auto iterWhile =
-      builder.create<fir::IterWhileOp>(loc, lastChar, zero, minusOne, trueVal,
-                                       /*returnFinalCount=*/false, lastChar);
+      fir::IterWhileOp::create(builder, loc, lastChar, zero, minusOne, trueVal,
+                               /*returnFinalCount=*/false, lastChar);
   auto insPt = builder.saveInsertionPoint();
   builder.setInsertionPointToStart(iterWhile.getBody());
   auto index = iterWhile.getInductionVar();
@@ -586,17 +587,17 @@ fir::factory::CharacterExprHelper::createLenTrim(const fir::CharBoxValue &str) {
   auto elemAddr = createElementAddr(fromBuff, index);
   auto codeAddr =
       builder.createConvert(loc, builder.getRefType(blank.getType()), elemAddr);
-  auto c = builder.create<fir::LoadOp>(loc, codeAddr);
-  auto isBlank = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq, blank, c);
+  auto c = fir::LoadOp::create(builder, loc, codeAddr);
+  auto isBlank = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq, blank, c);
   llvm::SmallVector<mlir::Value> results = {isBlank, index};
-  builder.create<fir::ResultOp>(loc, results);
+  fir::ResultOp::create(builder, loc, results);
   builder.restoreInsertionPoint(insPt);
   // Compute length after iteration (zero if all blanks)
   mlir::Value newLen =
-      builder.create<mlir::arith::AddIOp>(loc, iterWhile.getResult(1), one);
-  auto result = builder.create<mlir::arith::SelectOp>(
-      loc, iterWhile.getResult(0), zero, newLen);
+      mlir::arith::AddIOp::create(builder, loc, iterWhile.getResult(1), one);
+  auto result = mlir::arith::SelectOp::create(
+      builder, loc, iterWhile.getResult(0), zero, newLen);
   return builder.createConvert(loc, builder.getCharacterLengthType(), result);
 }
 
@@ -606,7 +607,7 @@ fir::factory::CharacterExprHelper::createCharacterTemp(mlir::Type type,
   assert(len >= 0 && "expected positive length");
   auto kind = recoverCharacterType(type).getFKind();
   auto charType = fir::CharacterType::get(builder.getContext(), kind, len);
-  auto addr = builder.create<fir::AllocaOp>(loc, charType);
+  auto addr = fir::AllocaOp::create(builder, loc, charType);
   auto mlirLen =
       builder.createIntegerConstant(loc, builder.getCharacterLengthType(), len);
   return {addr, mlirLen};
@@ -690,10 +691,10 @@ fir::factory::CharacterExprHelper::createSingletonFromCode(mlir::Value code,
   auto bits = builder.getKindMap().getCharacterBitsize(kind);
   auto intType = builder.getIntegerType(bits);
   auto cast = builder.createConvert(loc, intType, code);
-  auto undef = builder.create<fir::UndefOp>(loc, charType);
+  auto undef = fir::UndefOp::create(builder, loc, charType);
   auto zero = builder.getIntegerAttr(builder.getIndexType(), 0);
-  return builder.create<fir::InsertValueOp>(loc, charType, undef, cast,
-                                            builder.getArrayAttr(zero));
+  return fir::InsertValueOp::create(builder, loc, charType, undef, cast,
+                                    builder.getArrayAttr(zero));
 }
 
 mlir::Value fir::factory::CharacterExprHelper::extractCodeFromSingleton(
@@ -703,8 +704,8 @@ mlir::Value fir::factory::CharacterExprHelper::extractCodeFromSingleton(
   auto bits = builder.getKindMap().getCharacterBitsize(type.getFKind());
   auto intType = builder.getIntegerType(bits);
   auto zero = builder.getIntegerAttr(builder.getIndexType(), 0);
-  return builder.create<fir::ExtractValueOp>(loc, intType, singleton,
-                                             builder.getArrayAttr(zero));
+  return fir::ExtractValueOp::create(builder, loc, intType, singleton,
+                                     builder.getArrayAttr(zero));
 }
 
 mlir::Value
@@ -716,12 +717,12 @@ fir::factory::CharacterExprHelper::readLengthFromBox(mlir::Value box) {
 mlir::Value fir::factory::CharacterExprHelper::readLengthFromBox(
     mlir::Value box, fir::CharacterType charTy) {
   auto lenTy = builder.getCharacterLengthType();
-  auto size = builder.create<fir::BoxEleSizeOp>(loc, lenTy, box);
+  auto size = fir::BoxEleSizeOp::create(builder, loc, lenTy, box);
   auto bits = builder.getKindMap().getCharacterBitsize(charTy.getFKind());
   auto width = bits / 8;
   if (width > 1) {
     auto widthVal = builder.createIntegerConstant(loc, lenTy, width);
-    return builder.create<mlir::arith::DivSIOp>(loc, size, widthVal);
+    return mlir::arith::DivSIOp::create(builder, loc, size, widthVal);
   }
   return size;
 }
@@ -748,18 +749,18 @@ fir::factory::extractCharacterProcedureTuple(fir::FirOpBuilder &builder,
                                              mlir::Value tuple,
                                              bool openBoxProc) {
   mlir::TupleType tupleType = mlir::cast<mlir::TupleType>(tuple.getType());
-  mlir::Value addr = builder.create<fir::ExtractValueOp>(
-      loc, tupleType.getType(0), tuple,
+  mlir::Value addr = fir::ExtractValueOp::create(
+      builder, loc, tupleType.getType(0), tuple,
       builder.getArrayAttr(
           {builder.getIntegerAttr(builder.getIndexType(), 0)}));
   mlir::Value proc = [&]() -> mlir::Value {
     if (openBoxProc)
       if (auto addrTy = mlir::dyn_cast<fir::BoxProcType>(addr.getType()))
-        return builder.create<fir::BoxAddrOp>(loc, addrTy.getEleTy(), addr);
+        return fir::BoxAddrOp::create(builder, loc, addrTy.getEleTy(), addr);
     return addr;
   }();
-  mlir::Value len = builder.create<fir::ExtractValueOp>(
-      loc, tupleType.getType(1), tuple,
+  mlir::Value len = fir::ExtractValueOp::create(
+      builder, loc, tupleType.getType(1), tuple,
       builder.getArrayAttr(
           {builder.getIntegerAttr(builder.getIndexType(), 1)}));
   return {proc, len};
@@ -773,14 +774,14 @@ mlir::Value fir::factory::createCharacterProcedureTuple(
   if (len)
     len = builder.createConvert(loc, tupleType.getType(1), len);
   else
-    len = builder.create<fir::UndefOp>(loc, tupleType.getType(1));
-  mlir::Value tuple = builder.create<fir::UndefOp>(loc, tupleType);
-  tuple = builder.create<fir::InsertValueOp>(
-      loc, tupleType, tuple, addr,
+    len = fir::UndefOp::create(builder, loc, tupleType.getType(1));
+  mlir::Value tuple = fir::UndefOp::create(builder, loc, tupleType);
+  tuple = fir::InsertValueOp::create(
+      builder, loc, tupleType, tuple, addr,
       builder.getArrayAttr(
           {builder.getIntegerAttr(builder.getIndexType(), 0)}));
-  tuple = builder.create<fir::InsertValueOp>(
-      loc, tupleType, tuple, len,
+  tuple = fir::InsertValueOp::create(
+      builder, loc, tupleType, tuple, len,
       builder.getArrayAttr(
           {builder.getIntegerAttr(builder.getIndexType(), 1)}));
   return tuple;
@@ -827,10 +828,10 @@ fir::CharBoxValue fir::factory::CharacterExprHelper::createCharExtremum(
     auto currLen = builder.createConvert(loc, builder.getCharacterLengthType(),
                                          currChar.getLen());
     // biggest len result
-    mlir::Value lhsBigger = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::uge, biggestLen, currLen);
-    biggestLen = builder.create<mlir::arith::SelectOp>(loc, lhsBigger,
-                                                       biggestLen, currLen);
+    mlir::Value lhsBigger = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::uge, biggestLen, currLen);
+    biggestLen = mlir::arith::SelectOp::create(builder, loc, lhsBigger,
+                                               biggestLen, currLen);
 
     auto cmp = predIsMin ? mlir::arith::CmpIPredicate::slt
                          : mlir::arith::CmpIPredicate::sgt;
@@ -843,10 +844,10 @@ fir::CharBoxValue fir::factory::CharacterExprHelper::createCharExtremum(
     resultBuf = builder.createConvert(loc, type, resultBuf);
     currBuf = builder.createConvert(loc, type, currBuf);
 
-    resultBuf = builder.create<mlir::arith::SelectOp>(loc, resultCmp, currBuf,
-                                                      resultBuf);
-    resultLen = builder.create<mlir::arith::SelectOp>(loc, resultCmp, currLen,
-                                                      resultLen);
+    resultBuf = mlir::arith::SelectOp::create(builder, loc, resultCmp, currBuf,
+                                              resultBuf);
+    resultLen = mlir::arith::SelectOp::create(builder, loc, resultCmp, currLen,
+                                              resultLen);
   }
 
   // now that we know the lexicographically biggest/smallest char and which char
@@ -876,7 +877,7 @@ fir::factory::convertCharacterKind(fir::FirOpBuilder &builder,
     // As a value, it ought to have a constant LEN value.
     assert(charTy.hasConstantLen() && "must have constant length");
     mlir::Value tmp = builder.createTemporary(loc, charTy);
-    builder.create<fir::StoreOp>(loc, boxCharAddr, tmp);
+    fir::StoreOp::create(builder, loc, boxCharAddr, tmp);
     boxCharAddr = tmp;
   }
   auto fromBits = kindMap.getCharacterBitsize(
@@ -886,13 +887,13 @@ fir::factory::convertCharacterKind(fir::FirOpBuilder &builder,
     // Scale by relative ratio to give a buffer of the same length.
     auto ratio = builder.createIntegerConstant(loc, bufferSize.getType(),
                                                fromBits / toBits);
-    bufferSize = builder.create<mlir::arith::MulIOp>(loc, bufferSize, ratio);
+    bufferSize = mlir::arith::MulIOp::create(builder, loc, bufferSize, ratio);
   }
   mlir::Type toType =
       fir::CharacterType::getUnknownLen(builder.getContext(), toKind);
   auto dest = builder.createTemporary(loc, toType, /*name=*/{}, /*shape=*/{},
                                       mlir::ValueRange{bufferSize});
-  builder.create<fir::CharConvertOp>(loc, boxCharAddr, srcBoxChar.getLen(),
-                                     dest);
+  fir::CharConvertOp::create(builder, loc, boxCharAddr, srcBoxChar.getLen(),
+                             dest);
   return fir::CharBoxValue{dest, srcBoxChar.getLen()};
 }

diff  --git a/flang/lib/Optimizer/Builder/Complex.cpp b/flang/lib/Optimizer/Builder/Complex.cpp
index 69f97dd654ce0..61de9880774ac 100644
--- a/flang/lib/Optimizer/Builder/Complex.cpp
+++ b/flang/lib/Optimizer/Builder/Complex.cpp
@@ -24,7 +24,7 @@ mlir::Type fir::factory::Complex::getComplexPartType(mlir::Value cplx) const {
 mlir::Value fir::factory::Complex::createComplex(mlir::Type cplxTy,
                                                  mlir::Value real,
                                                  mlir::Value imag) {
-  mlir::Value und = builder.create<fir::UndefOp>(loc, cplxTy);
+  mlir::Value und = fir::UndefOp::create(builder, loc, cplxTy);
   return insert<Part::Imag>(insert<Part::Real>(und, real), imag);
 }
 

diff  --git a/flang/lib/Optimizer/Builder/DoLoopHelper.cpp b/flang/lib/Optimizer/Builder/DoLoopHelper.cpp
index 4b12e281b5153..0ec91d5883b92 100644
--- a/flang/lib/Optimizer/Builder/DoLoopHelper.cpp
+++ b/flang/lib/Optimizer/Builder/DoLoopHelper.cpp
@@ -20,7 +20,7 @@ fir::factory::DoLoopHelper::createLoop(mlir::Value lb, mlir::Value ub,
   auto ubi = builder.convertToIndexType(loc, ub);
   assert(step && "step must be an actual Value");
   auto inc = builder.convertToIndexType(loc, step);
-  auto loop = builder.create<fir::DoLoopOp>(loc, lbi, ubi, inc);
+  auto loop = fir::DoLoopOp::create(builder, loc, lbi, ubi, inc);
   auto insertPt = builder.saveInsertionPoint();
   builder.setInsertionPointToStart(loop.getBody());
   auto index = loop.getInductionVar();
@@ -43,6 +43,6 @@ fir::factory::DoLoopHelper::createLoop(mlir::Value count,
   auto indexType = builder.getIndexType();
   auto zero = builder.createIntegerConstant(loc, indexType, 0);
   auto one = builder.createIntegerConstant(loc, count.getType(), 1);
-  auto up = builder.create<mlir::arith::SubIOp>(loc, count, one);
+  auto up = mlir::arith::SubIOp::create(builder, loc, count, one);
   return createLoop(zero, up, one, bodyGenerator);
 }

diff  --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 5b1dbc4435d6c..eaad54eb9eec2 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -406,10 +406,10 @@ mlir::Value fir::FirOpBuilder::genTempDeclareOp(
     llvm::ArrayRef<mlir::Value> typeParams,
     fir::FortranVariableFlagsAttr fortranAttrs) {
   auto nameAttr = mlir::StringAttr::get(builder.getContext(), name);
-  return builder.create<fir::DeclareOp>(loc, memref.getType(), memref, shape,
-                                        typeParams,
-                                        /*dummy_scope=*/nullptr, nameAttr,
-                                        fortranAttrs, cuf::DataAttributeAttr{});
+  return fir::DeclareOp::create(builder, loc, memref.getType(), memref, shape,
+                                typeParams,
+                                /*dummy_scope=*/nullptr, nameAttr, fortranAttrs,
+                                cuf::DataAttributeAttr{});
 }
 
 mlir::Value fir::FirOpBuilder::genStackSave(mlir::Location loc) {
@@ -585,7 +585,7 @@ mlir::Value fir::factory::createConvert(mlir::OpBuilder &builder,
             mlir::cast<fir::RecordType>(val.getType()).getTypeList() ==
                 mlir::cast<fir::RecordType>(toTy).getTypeList()) &&
            "incompatible record types");
-    return builder.create<fir::ConvertOp>(loc, toTy, val);
+    return fir::ConvertOp::create(builder, loc, toTy, val);
   }
   return val;
 }
@@ -824,7 +824,7 @@ genNullPointerComparison(fir::FirOpBuilder &builder, mlir::Location loc,
   auto intPtrTy = builder.getIntPtrType();
   auto ptrToInt = builder.createConvert(loc, intPtrTy, addr);
   auto c0 = builder.createIntegerConstant(loc, intPtrTy, 0);
-  return builder.create<mlir::arith::CmpIOp>(loc, condition, ptrToInt, c0);
+  return mlir::arith::CmpIOp::create(builder, loc, condition, ptrToInt, c0);
 }
 
 mlir::Value fir::FirOpBuilder::genIsNotNullAddr(mlir::Location loc,
@@ -1028,8 +1028,8 @@ fir::factory::readExtents(fir::FirOpBuilder &builder, mlir::Location loc,
   auto idxTy = builder.getIndexType();
   for (decltype(rank) dim = 0; dim < rank; ++dim) {
     auto dimVal = builder.createIntegerConstant(loc, idxTy, dim);
-    auto dimInfo = builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy,
-                                                  box.getAddr(), dimVal);
+    auto dimInfo = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
+                                          box.getAddr(), dimVal);
     result.emplace_back(dimInfo.getResult(1));
   }
   return result;
@@ -1061,7 +1061,7 @@ fir::ExtendedValue fir::factory::readBoxValue(fir::FirOpBuilder &builder,
   assert(!box.hasAssumedRank() &&
          "cannot read unlimited polymorphic or assumed rank fir.box");
   auto addr =
-      builder.create<fir::BoxAddrOp>(loc, box.getMemTy(), box.getAddr());
+      fir::BoxAddrOp::create(builder, loc, box.getMemTy(), box.getAddr());
   if (box.isCharacter()) {
     auto len = fir::factory::readCharLen(builder, loc, box);
     if (box.rank() == 0)
@@ -1139,13 +1139,13 @@ static llvm::SmallVector<mlir::Value> getFromBox(mlir::Location loc,
     } else if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
       if (charTy.hasDynamicLen()) {
         auto idxTy = builder.getIndexType();
-        auto eleSz = builder.create<fir::BoxEleSizeOp>(loc, idxTy, boxVal);
+        auto eleSz = fir::BoxEleSizeOp::create(builder, loc, idxTy, boxVal);
         auto kindBytes =
             builder.getKindMap().getCharacterBitsize(charTy.getFKind()) / 8;
         mlir::Value charSz =
             builder.createIntegerConstant(loc, idxTy, kindBytes);
         mlir::Value len =
-            builder.create<mlir::arith::DivSIOp>(loc, eleSz, charSz);
+            mlir::arith::DivSIOp::create(builder, loc, eleSz, charSz);
         return {len};
       }
     }
@@ -1237,11 +1237,11 @@ fir::ExtendedValue fir::factory::createStringLiteral(fir::FirOpBuilder &builder,
         loc, type, globalName,
         [&](fir::FirOpBuilder &builder) {
           auto stringLitOp = builder.createStringLitOp(loc, str);
-          builder.create<fir::HasValueOp>(loc, stringLitOp);
+          fir::HasValueOp::create(builder, loc, stringLitOp);
         },
         builder.createLinkOnceLinkage());
-  auto addr = builder.create<fir::AddrOfOp>(loc, global.resultType(),
-                                            global.getSymbol());
+  auto addr = fir::AddrOfOp::create(builder, loc, global.resultType(),
+                                    global.getSymbol());
   auto len = builder.createIntegerConstant(
       loc, builder.getCharacterLengthType(), str.size());
   return fir::CharBoxValue{addr, len};
@@ -1255,7 +1255,7 @@ fir::factory::createExtents(fir::FirOpBuilder &builder, mlir::Location loc,
   for (auto ext : seqTy.getShape())
     extents.emplace_back(
         ext == fir::SequenceType::getUnknownExtent()
-            ? builder.create<fir::UndefOp>(loc, idxTy).getResult()
+            ? fir::UndefOp::create(builder, loc, idxTy).getResult()
             : builder.createIntegerConstant(loc, idxTy, ext));
   return extents;
 }
@@ -1396,11 +1396,11 @@ void fir::factory::genScalarAssignment(fir::FirOpBuilder &builder,
     assert(!fir::hasDynamicSize(type));
     auto rhsVal = fir::getBase(rhs);
     if (fir::isa_ref_type(rhsVal.getType()))
-      rhsVal = builder.create<fir::LoadOp>(loc, rhsVal);
+      rhsVal = fir::LoadOp::create(builder, loc, rhsVal);
     mlir::Value lhsAddr = fir::getBase(lhs);
     rhsVal = builder.createConvert(loc, fir::unwrapRefType(lhsAddr.getType()),
                                    rhsVal);
-    builder.create<fir::StoreOp>(loc, rhsVal, lhsAddr);
+    fir::StoreOp::create(builder, loc, rhsVal, lhsAddr);
   }
 }
 
@@ -1421,16 +1421,18 @@ static void genComponentByComponentAssignment(fir::FirOpBuilder &builder,
     auto &[lFieldName, lFieldTy] = lhsPair;
     auto &[rFieldName, rFieldTy] = rhsPair;
     assert(!fir::hasDynamicSize(lFieldTy) && !fir::hasDynamicSize(rFieldTy));
-    mlir::Value rField = builder.create<fir::FieldIndexOp>(
-        loc, fieldIndexType, rFieldName, rhsType, fir::getTypeParams(rhs));
+    mlir::Value rField =
+        fir::FieldIndexOp::create(builder, loc, fieldIndexType, rFieldName,
+                                  rhsType, fir::getTypeParams(rhs));
     auto rFieldRefType = builder.getRefType(rFieldTy);
-    mlir::Value fromCoor = builder.create<fir::CoordinateOp>(
-        loc, rFieldRefType, fir::getBase(rhs), rField);
-    mlir::Value field = builder.create<fir::FieldIndexOp>(
-        loc, fieldIndexType, lFieldName, lhsType, fir::getTypeParams(lhs));
+    mlir::Value fromCoor = fir::CoordinateOp::create(
+        builder, loc, rFieldRefType, fir::getBase(rhs), rField);
+    mlir::Value field =
+        fir::FieldIndexOp::create(builder, loc, fieldIndexType, lFieldName,
+                                  lhsType, fir::getTypeParams(lhs));
     auto fieldRefType = builder.getRefType(lFieldTy);
-    mlir::Value toCoor = builder.create<fir::CoordinateOp>(
-        loc, fieldRefType, fir::getBase(lhs), field);
+    mlir::Value toCoor = fir::CoordinateOp::create(builder, loc, fieldRefType,
+                                                   fir::getBase(lhs), field);
     std::optional<fir::DoLoopOp> outerLoop;
     if (auto sequenceType = mlir::dyn_cast<fir::SequenceType>(lFieldTy)) {
       // Create loops to assign array components elements by elements.
@@ -1444,7 +1446,7 @@ static void genComponentByComponentAssignment(fir::FirOpBuilder &builder,
       for (auto extent : llvm::reverse(sequenceType.getShape())) {
         // TODO: add zero size test !
         mlir::Value ub = builder.createIntegerConstant(loc, idxTy, extent - 1);
-        auto loop = builder.create<fir::DoLoopOp>(loc, zero, ub, one);
+        auto loop = fir::DoLoopOp::create(builder, loc, zero, ub, one);
         if (!outerLoop)
           outerLoop = loop;
         indices.push_back(loop.getInductionVar());
@@ -1453,19 +1455,19 @@ static void genComponentByComponentAssignment(fir::FirOpBuilder &builder,
       // Set indices in column-major order.
       std::reverse(indices.begin(), indices.end());
       auto elementRefType = builder.getRefType(sequenceType.getEleTy());
-      toCoor = builder.create<fir::CoordinateOp>(loc, elementRefType, toCoor,
-                                                 indices);
-      fromCoor = builder.create<fir::CoordinateOp>(loc, elementRefType,
-                                                   fromCoor, indices);
+      toCoor = fir::CoordinateOp::create(builder, loc, elementRefType, toCoor,
+                                         indices);
+      fromCoor = fir::CoordinateOp::create(builder, loc, elementRefType,
+                                           fromCoor, indices);
     }
     if (auto fieldEleTy = fir::unwrapSequenceType(lFieldTy);
         mlir::isa<fir::BaseBoxType>(fieldEleTy)) {
       assert(mlir::isa<fir::PointerType>(
                  mlir::cast<fir::BaseBoxType>(fieldEleTy).getEleTy()) &&
              "allocatable members require deep copy");
-      auto fromPointerValue = builder.create<fir::LoadOp>(loc, fromCoor);
+      auto fromPointerValue = fir::LoadOp::create(builder, loc, fromCoor);
       auto castTo = builder.createConvert(loc, fieldEleTy, fromPointerValue);
-      builder.create<fir::StoreOp>(loc, castTo, toCoor);
+      fir::StoreOp::create(builder, loc, castTo, toCoor);
     } else {
       auto from =
           fir::factory::componentToExtendedValue(builder, loc, fromCoor);
@@ -1543,7 +1545,7 @@ void fir::factory::genRecordAssignment(fir::FirOpBuilder &builder,
     // runtime interface, but assume the fir.box is unchanged.
     // TODO: does this holds true with polymorphic entities ?
     auto toMutableBox = builder.createTemporary(loc, to.getType());
-    builder.create<fir::StoreOp>(loc, to, toMutableBox);
+    fir::StoreOp::create(builder, loc, to, toMutableBox);
     if (isTemporaryLHS)
       fir::runtime::genAssignTemporary(builder, loc, toMutableBox, from);
     else
@@ -1588,12 +1590,12 @@ mlir::Value fir::factory::genLenOfCharacter(
   auto idxTy = builder.getIndexType();
   auto zero = builder.createIntegerConstant(loc, idxTy, 0);
   auto saturatedDiff = [&](mlir::Value lower, mlir::Value upper) {
-    auto 
diff  = builder.create<mlir::arith::SubIOp>(loc, upper, lower);
+    auto 
diff  = mlir::arith::SubIOp::create(builder, loc, upper, lower);
     auto one = builder.createIntegerConstant(loc, idxTy, 1);
-    auto size = builder.create<mlir::arith::AddIOp>(loc, 
diff , one);
-    auto cmp = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::sgt, size, zero);
-    return builder.create<mlir::arith::SelectOp>(loc, cmp, size, zero);
+    auto size = mlir::arith::AddIOp::create(builder, loc, 
diff , one);
+    auto cmp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::sgt, size, zero);
+    return mlir::arith::SelectOp::create(builder, loc, cmp, size, zero);
   };
   if (substring.size() == 2) {
     auto upper = builder.createConvert(loc, idxTy, substring.back());
@@ -1615,7 +1617,7 @@ mlir::Value fir::factory::genLenOfCharacter(
   }
   if (fir::isa_box_type(memref.getType())) {
     if (mlir::isa<fir::BoxCharType>(memref.getType()))
-      return builder.create<fir::BoxCharLenOp>(loc, idxTy, memref);
+      return fir::BoxCharLenOp::create(builder, loc, idxTy, memref);
     if (mlir::isa<fir::BoxType>(memref.getType()))
       return CharacterExprHelper(builder, loc).readLengthFromBox(memref);
     fir::emitFatalError(loc, "memref has wrong type");
@@ -1684,10 +1686,10 @@ mlir::Value fir::factory::genMaxWithZero(fir::FirOpBuilder &builder,
     if (auto cst = mlir::dyn_cast<mlir::arith::ConstantOp>(definingOp))
       if (auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(cst.getValue()))
         return intAttr.getInt() > 0 ? value : zero;
-  mlir::Value valueIsGreater = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::sgt, value, zero);
-  return builder.create<mlir::arith::SelectOp>(loc, valueIsGreater, value,
-                                               zero);
+  mlir::Value valueIsGreater = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::sgt, value, zero);
+  return mlir::arith::SelectOp::create(builder, loc, valueIsGreater, value,
+                                       zero);
 }
 
 mlir::Value fir::factory::genMaxWithZero(fir::FirOpBuilder &builder,
@@ -1703,8 +1705,8 @@ mlir::Value fir::factory::computeExtent(fir::FirOpBuilder &builder,
                                         mlir::Value one) {
   mlir::Type type = lb.getType();
   // Let the folder deal with the common `ub - <const> + 1` case.
-  auto 
diff  = builder.create<mlir::arith::SubIOp>(loc, type, ub, lb);
-  auto rawExtent = builder.create<mlir::arith::AddIOp>(loc, type, 
diff , one);
+  auto 
diff  = mlir::arith::SubIOp::create(builder, loc, type, ub, lb);
+  auto rawExtent = mlir::arith::AddIOp::create(builder, loc, type, 
diff , one);
   return fir::factory::genMaxWithZero(builder, loc, rawExtent, zero);
 }
 mlir::Value fir::factory::computeExtent(fir::FirOpBuilder &builder,
@@ -1724,8 +1726,8 @@ genCPtrOrCFunptrFieldIndex(fir::FirOpBuilder &builder, mlir::Location loc,
   auto addrFieldName = recTy.getTypeList()[0].first;
   mlir::Type addrFieldTy = recTy.getTypeList()[0].second;
   auto fieldIndexType = fir::FieldType::get(cptrTy.getContext());
-  mlir::Value addrFieldIndex = builder.create<fir::FieldIndexOp>(
-      loc, fieldIndexType, addrFieldName, recTy,
+  mlir::Value addrFieldIndex = fir::FieldIndexOp::create(
+      builder, loc, fieldIndexType, addrFieldName, recTy,
       /*typeParams=*/mlir::ValueRange{});
   return {addrFieldIndex, addrFieldTy};
 }
@@ -1736,8 +1738,8 @@ mlir::Value fir::factory::genCPtrOrCFunptrAddr(fir::FirOpBuilder &builder,
                                                mlir::Type ty) {
   auto [addrFieldIndex, addrFieldTy] =
       genCPtrOrCFunptrFieldIndex(builder, loc, ty);
-  return builder.create<fir::CoordinateOp>(loc, builder.getRefType(addrFieldTy),
-                                           cPtr, addrFieldIndex);
+  return fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(addrFieldTy), cPtr, addrFieldIndex);
 }
 
 mlir::Value fir::factory::genCDevPtrAddr(fir::FirOpBuilder &builder,
@@ -1748,15 +1750,15 @@ mlir::Value fir::factory::genCDevPtrAddr(fir::FirOpBuilder &builder,
   auto cptrFieldName = recTy.getTypeList()[0].first;
   mlir::Type cptrFieldTy = recTy.getTypeList()[0].second;
   auto fieldIndexType = fir::FieldType::get(ty.getContext());
-  mlir::Value cptrFieldIndex = builder.create<fir::FieldIndexOp>(
-      loc, fieldIndexType, cptrFieldName, recTy,
+  mlir::Value cptrFieldIndex = fir::FieldIndexOp::create(
+      builder, loc, fieldIndexType, cptrFieldName, recTy,
       /*typeParams=*/mlir::ValueRange{});
-  auto cptrCoord = builder.create<fir::CoordinateOp>(
-      loc, builder.getRefType(cptrFieldTy), cDevPtr, cptrFieldIndex);
+  auto cptrCoord = fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(cptrFieldTy), cDevPtr, cptrFieldIndex);
   auto [addrFieldIndex, addrFieldTy] =
       genCPtrOrCFunptrFieldIndex(builder, loc, cptrFieldTy);
-  return builder.create<fir::CoordinateOp>(loc, builder.getRefType(addrFieldTy),
-                                           cptrCoord, addrFieldIndex);
+  return fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(addrFieldTy), cptrCoord, addrFieldIndex);
 }
 
 mlir::Value fir::factory::genCPtrOrCFunptrValue(fir::FirOpBuilder &builder,
@@ -1769,13 +1771,13 @@ mlir::Value fir::factory::genCPtrOrCFunptrValue(fir::FirOpBuilder &builder,
         genCPtrOrCFunptrFieldIndex(builder, loc, cPtrTy);
     mlir::Value cPtrCoor;
     if (fir::isa_ref_type(cPtr.getType())) {
-      cPtrCoor = builder.create<fir::CoordinateOp>(
-          loc, builder.getRefType(addrFieldTy), cPtr, addrFieldIndex);
+      cPtrCoor = fir::CoordinateOp::create(
+          builder, loc, builder.getRefType(addrFieldTy), cPtr, addrFieldIndex);
     } else {
       auto arrayAttr = builder.getArrayAttr(
           {builder.getIntegerAttr(builder.getIndexType(), 0)});
-      cPtrCoor = builder.create<fir::ExtractValueOp>(loc, addrFieldTy, cPtr,
-                                                     arrayAttr);
+      cPtrCoor = fir::ExtractValueOp::create(builder, loc, addrFieldTy, cPtr,
+                                             arrayAttr);
     }
     return genCPtrOrCFunptrValue(builder, loc, cPtrCoor);
   }
@@ -1783,13 +1785,14 @@ mlir::Value fir::factory::genCPtrOrCFunptrValue(fir::FirOpBuilder &builder,
   if (fir::isa_ref_type(cPtr.getType())) {
     mlir::Value cPtrAddr =
         fir::factory::genCPtrOrCFunptrAddr(builder, loc, cPtr, cPtrTy);
-    return builder.create<fir::LoadOp>(loc, cPtrAddr);
+    return fir::LoadOp::create(builder, loc, cPtrAddr);
   }
   auto [addrFieldIndex, addrFieldTy] =
       genCPtrOrCFunptrFieldIndex(builder, loc, cPtrTy);
   auto arrayAttr =
       builder.getArrayAttr({builder.getIntegerAttr(builder.getIndexType(), 0)});
-  return builder.create<fir::ExtractValueOp>(loc, addrFieldTy, cPtr, arrayAttr);
+  return fir::ExtractValueOp::create(builder, loc, addrFieldTy, cPtr,
+                                     arrayAttr);
 }
 
 fir::BoxValue fir::factory::createBoxValue(fir::FirOpBuilder &builder,
@@ -1837,8 +1840,8 @@ mlir::Value fir::factory::createNullBoxProc(fir::FirOpBuilder &builder,
   if (!boxTy)
     fir::emitFatalError(loc, "Procedure pointer must be of BoxProcType");
   auto boxEleTy{fir::unwrapRefType(boxTy.getEleTy())};
-  mlir::Value initVal{builder.create<fir::ZeroOp>(loc, boxEleTy)};
-  return builder.create<fir::EmboxProcOp>(loc, boxTy, initVal);
+  mlir::Value initVal{fir::ZeroOp::create(builder, loc, boxEleTy)};
+  return fir::EmboxProcOp::create(builder, loc, boxTy, initVal);
 }
 
 void fir::factory::setInternalLinkage(mlir::func::FuncOp func) {
@@ -1897,15 +1900,15 @@ llvm::SmallVector<mlir::Value> fir::factory::updateRuntimeExtentsForEmptyArrays(
     mlir::Type type = extent.getType();
     mlir::Value zero = createZeroValue(builder, loc, type);
     zeroes.push_back(zero);
-    mlir::Value isZero = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::eq, extent, zero);
-    isEmpty = builder.create<mlir::arith::OrIOp>(loc, isEmpty, isZero);
+    mlir::Value isZero = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::eq, extent, zero);
+    isEmpty = mlir::arith::OrIOp::create(builder, loc, isEmpty, isZero);
   }
 
   llvm::SmallVector<mlir::Value> newExtents;
   for (auto [zero, extent] : llvm::zip_equal(zeroes, extents)) {
     newExtents.push_back(
-        builder.create<mlir::arith::SelectOp>(loc, isEmpty, zero, extent));
+        mlir::arith::SelectOp::create(builder, loc, isEmpty, zero, extent));
   }
   return newExtents;
 }
@@ -1926,7 +1929,7 @@ void fir::factory::genDimInfoFromBox(
   for (unsigned i = 0; i < rank; ++i) {
     mlir::Value dim = builder.createIntegerConstant(loc, idxTy, i);
     auto dimInfo =
-        builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, dim);
+        fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, box, dim);
     if (lbounds)
       lbounds->push_back(dimInfo.getLowerBound());
     if (extents)
@@ -1943,12 +1946,12 @@ mlir::Value fir::factory::genLifetimeStart(mlir::OpBuilder &builder,
   mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(
       alloc.getContext(), getAllocaAddressSpace(dl));
   mlir::Value cast =
-      builder.create<fir::ConvertOp>(loc, ptrTy, alloc.getResult());
-  builder.create<mlir::LLVM::LifetimeStartOp>(loc, size, cast);
+      fir::ConvertOp::create(builder, loc, ptrTy, alloc.getResult());
+  mlir::LLVM::LifetimeStartOp::create(builder, loc, size, cast);
   return cast;
 }
 
 void fir::factory::genLifetimeEnd(mlir::OpBuilder &builder, mlir::Location loc,
                                   mlir::Value cast, int64_t size) {
-  builder.create<mlir::LLVM::LifetimeEndOp>(loc, size, cast);
+  mlir::LLVM::LifetimeEndOp::create(builder, loc, size, cast);
 }

diff  --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index fb6f0dbf719fb..c3948f2caf67b 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -50,7 +50,7 @@ hlfir::getExplicitExtentsFromShape(mlir::Value shape,
       int64_t extent = exprShape[i];
       mlir::Value extentVal;
       if (extent == expr.getUnknownExtent()) {
-        auto op = builder.create<hlfir::GetExtentOp>(shape.getLoc(), shape, i);
+        auto op = hlfir::GetExtentOp::create(builder, shape.getLoc(), shape, i);
         extentVal = op.getResult();
       } else {
         extentVal =
@@ -150,7 +150,7 @@ static mlir::Value genCharacterVariableLength(mlir::Location loc,
     return builder.createIntegerConstant(loc, builder.getIndexType(),
                                          charType.getLen());
   if (var.isMutableBox())
-    var = hlfir::Entity{builder.create<fir::LoadOp>(loc, var)};
+    var = hlfir::Entity{fir::LoadOp::create(builder, loc, var)};
   mlir::Value len = fir::factory::CharacterExprHelper{builder, loc}.getLength(
       var.getFirBase());
   assert(len && "failed to retrieve length");
@@ -164,8 +164,8 @@ static fir::CharBoxValue genUnboxChar(mlir::Location loc,
     return {emboxChar.getMemref(), emboxChar.getLen()};
   mlir::Type refType = fir::ReferenceType::get(
       mlir::cast<fir::BoxCharType>(boxChar.getType()).getEleTy());
-  auto unboxed = builder.create<fir::UnboxCharOp>(
-      loc, refType, builder.getIndexType(), boxChar);
+  auto unboxed = fir::UnboxCharOp::create(builder, loc, refType,
+                                          builder.getIndexType(), boxChar);
   mlir::Value addr = unboxed.getResult(0);
   mlir::Value len = unboxed.getResult(1);
   if (auto varIface = boxChar.getDefiningOp<fir::FortranVariableOpInterface>())
@@ -278,8 +278,9 @@ hlfir::genDeclare(mlir::Location loc, fir::FirOpBuilder &builder,
                          box.nonDeferredLenParams().end());
       },
       [](const auto &) {});
-  auto declareOp = builder.create<hlfir::DeclareOp>(
-      loc, base, name, shapeOrShift, lenParams, dummyScope, flags, dataAttr);
+  auto declareOp =
+      hlfir::DeclareOp::create(builder, loc, base, name, shapeOrShift,
+                               lenParams, dummyScope, flags, dataAttr);
   return mlir::cast<fir::FortranVariableOpInterface>(declareOp.getOperation());
 }
 
@@ -312,12 +313,12 @@ hlfir::genAssociateExpr(mlir::Location loc, fir::FirOpBuilder &builder,
   genLengthParameters(loc, builder, value, lenParams);
   if (attr) {
     assert(name.empty() && "It attribute is provided, no-name is expected");
-    return builder.create<hlfir::AssociateOp>(loc, source, shape, lenParams,
-                                              fir::FortranVariableFlagsAttr{},
-                                              llvm::ArrayRef{*attr});
+    return hlfir::AssociateOp::create(builder, loc, source, shape, lenParams,
+                                      fir::FortranVariableFlagsAttr{},
+                                      llvm::ArrayRef{*attr});
   }
-  return builder.create<hlfir::AssociateOp>(loc, source, name, shape, lenParams,
-                                            fir::FortranVariableFlagsAttr{});
+  return hlfir::AssociateOp::create(builder, loc, source, name, shape,
+                                    lenParams, fir::FortranVariableFlagsAttr{});
 }
 
 mlir::Value hlfir::genVariableRawAddress(mlir::Location loc,
@@ -326,12 +327,12 @@ mlir::Value hlfir::genVariableRawAddress(mlir::Location loc,
   assert(var.isVariable() && "only address of variables can be taken");
   mlir::Value baseAddr = var.getFirBase();
   if (var.isMutableBox())
-    baseAddr = builder.create<fir::LoadOp>(loc, baseAddr);
+    baseAddr = fir::LoadOp::create(builder, loc, baseAddr);
   // Get raw address.
   if (mlir::isa<fir::BoxCharType>(var.getType()))
     baseAddr = genUnboxChar(loc, builder, var.getBase()).getAddr();
   if (mlir::isa<fir::BaseBoxType>(baseAddr.getType()))
-    baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
+    baseAddr = fir::BoxAddrOp::create(builder, loc, baseAddr);
   return baseAddr;
 }
 
@@ -350,8 +351,8 @@ mlir::Value hlfir::genVariableBoxChar(mlir::Location loc,
       fir::BoxCharType::get(builder.getContext(), charType.getFKind());
   auto scalarAddr =
       builder.createConvert(loc, fir::ReferenceType::get(charType), addr);
-  return builder.create<fir::EmboxCharOp>(loc, boxCharType, scalarAddr,
-                                          lengths[0]);
+  return fir::EmboxCharOp::create(builder, loc, boxCharType, scalarAddr,
+                                  lengths[0]);
 }
 
 static hlfir::Entity changeBoxAttributes(mlir::Location loc,
@@ -365,8 +366,8 @@ static hlfir::Entity changeBoxAttributes(mlir::Location loc,
       getNonDefaultLowerBounds(loc, builder, var);
   if (!lbounds.empty())
     shift = builder.genShift(loc, lbounds);
-  auto rebox = builder.create<fir::ReboxOp>(loc, forceBoxType, var, shift,
-                                            /*slice=*/nullptr);
+  auto rebox = fir::ReboxOp::create(builder, loc, forceBoxType, var, shift,
+                                    /*slice=*/nullptr);
   return hlfir::Entity{rebox};
 }
 
@@ -404,9 +405,8 @@ hlfir::Entity hlfir::genVariableBox(mlir::Location loc,
         fir::ReferenceType::get(fir::unwrapRefType(forceBoxType.getEleTy()));
     addr = builder.createConvert(loc, baseType, addr);
   }
-  auto embox =
-      builder.create<fir::EmboxOp>(loc, boxType, addr, shape,
-                                   /*slice=*/mlir::Value{}, typeParams);
+  auto embox = fir::EmboxOp::create(builder, loc, boxType, addr, shape,
+                                    /*slice=*/mlir::Value{}, typeParams);
   return hlfir::Entity{embox.getResult()};
 }
 
@@ -416,7 +416,7 @@ hlfir::Entity hlfir::loadTrivialScalar(mlir::Location loc,
   entity = derefPointersAndAllocatables(loc, builder, entity);
   if (entity.isVariable() && entity.isScalar() &&
       fir::isa_trivial(entity.getFortranElementType())) {
-    return Entity{builder.create<fir::LoadOp>(loc, entity)};
+    return Entity{fir::LoadOp::create(builder, loc, entity)};
   }
   return entity;
 }
@@ -429,8 +429,8 @@ hlfir::Entity hlfir::getElementAt(mlir::Location loc,
   llvm::SmallVector<mlir::Value> lenParams;
   genLengthParameters(loc, builder, entity, lenParams);
   if (mlir::isa<hlfir::ExprType>(entity.getType()))
-    return hlfir::Entity{builder.create<hlfir::ApplyOp>(
-        loc, entity, oneBasedIndices, lenParams)};
+    return hlfir::Entity{hlfir::ApplyOp::create(builder, loc, entity,
+                                                oneBasedIndices, lenParams)};
   // Build hlfir.designate. The lower bounds may need to be added to
   // the oneBasedIndices since hlfir.designate expect indices
   // based on the array operand lower bounds.
@@ -445,16 +445,16 @@ hlfir::Entity hlfir::getElementAt(mlir::Location loc,
     for (auto [oneBased, lb] : llvm::zip(oneBasedIndices, lbounds)) {
       auto lbIdx = builder.createConvert(loc, idxTy, lb);
       auto oneBasedIdx = builder.createConvert(loc, idxTy, oneBased);
-      auto shift = builder.create<mlir::arith::SubIOp>(loc, lbIdx, one);
+      auto shift = mlir::arith::SubIOp::create(builder, loc, lbIdx, one);
       mlir::Value index =
-          builder.create<mlir::arith::AddIOp>(loc, oneBasedIdx, shift);
+          mlir::arith::AddIOp::create(builder, loc, oneBasedIdx, shift);
       indices.push_back(index);
     }
-    designate = builder.create<hlfir::DesignateOp>(loc, resultType, entity,
-                                                   indices, lenParams);
+    designate = hlfir::DesignateOp::create(builder, loc, resultType, entity,
+                                           indices, lenParams);
   } else {
-    designate = builder.create<hlfir::DesignateOp>(loc, resultType, entity,
-                                                   oneBasedIndices, lenParams);
+    designate = hlfir::DesignateOp::create(builder, loc, resultType, entity,
+                                           oneBasedIndices, lenParams);
   }
   return mlir::cast<fir::FortranVariableOpInterface>(designate.getOperation());
 }
@@ -467,8 +467,8 @@ static mlir::Value genUBound(mlir::Location loc, fir::FirOpBuilder &builder,
       return extent;
   extent = builder.createConvert(loc, one.getType(), extent);
   lb = builder.createConvert(loc, one.getType(), lb);
-  auto add = builder.create<mlir::arith::AddIOp>(loc, lb, extent);
-  return builder.create<mlir::arith::SubIOp>(loc, add, one);
+  auto add = mlir::arith::AddIOp::create(builder, loc, lb, extent);
+  return mlir::arith::SubIOp::create(builder, loc, add, one);
 }
 
 llvm::SmallVector<std::pair<mlir::Value, mlir::Value>>
@@ -557,8 +557,8 @@ static mlir::Value computeVariableExtent(mlir::Location loc,
   assert(mlir::isa<fir::BaseBoxType>(variable.getType()) &&
          "array variable with dynamic extent must be boxed");
   mlir::Value dimVal = builder.createIntegerConstant(loc, idxTy, dim);
-  auto dimInfo = builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy,
-                                                variable, dimVal);
+  auto dimInfo = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
+                                        variable, dimVal);
   return dimInfo.getExtent();
 }
 llvm::SmallVector<mlir::Value> getVariableExtents(mlir::Location loc,
@@ -608,14 +608,14 @@ mlir::Value hlfir::genShape(mlir::Location loc, fir::FirOpBuilder &builder,
       return shape;
     if (mlir::isa<fir::ShapeShiftType>(shape.getType()))
       if (auto s = shape.getDefiningOp<fir::ShapeShiftOp>())
-        return builder.create<fir::ShapeOp>(loc, s.getExtents());
+        return fir::ShapeOp::create(builder, loc, s.getExtents());
   }
   if (mlir::isa<hlfir::ExprType>(entity.getType()))
-    return builder.create<hlfir::ShapeOfOp>(loc, entity.getBase());
+    return hlfir::ShapeOfOp::create(builder, loc, entity.getBase());
   // There is no shape lying around for this entity. Retrieve the extents and
   // build a new fir.shape.
-  return builder.create<fir::ShapeOp>(loc,
-                                      getVariableExtents(loc, builder, entity));
+  return fir::ShapeOp::create(builder, loc,
+                              getVariableExtents(loc, builder, entity));
 }
 
 llvm::SmallVector<mlir::Value>
@@ -668,7 +668,7 @@ mlir::Value hlfir::genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
   mlir::Type idxTy = builder.getIndexType();
   mlir::Value dimVal = builder.createIntegerConstant(loc, idxTy, dim);
   auto dimInfo =
-      builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, entity, dimVal);
+      fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, entity, dimVal);
   return dimInfo.getLowerBound();
 }
 
@@ -708,7 +708,7 @@ void hlfir::genLengthParameters(mlir::Location loc, fir::FirOpBuilder &builder,
       return;
     }
     if (entity.isCharacter()) {
-      result.push_back(builder.create<hlfir::GetLengthOp>(loc, expr));
+      result.push_back(hlfir::GetLengthOp::create(builder, loc, expr));
       return;
     }
     TODO(loc, "inquire PDTs length parameters of hlfir.expr");
@@ -735,7 +735,7 @@ mlir::Value hlfir::genRank(mlir::Location loc, fir::FirOpBuilder &builder,
     return builder.createIntegerConstant(loc, resultType, entity.getRank());
   assert(entity.isBoxAddressOrValue() &&
          "assumed-ranks are box addresses or values");
-  return builder.create<fir::BoxRankOp>(loc, resultType, entity);
+  return fir::BoxRankOp::create(builder, loc, resultType, entity);
 }
 
 // Return a "shape" that can be used in fir.embox/fir.rebox with \p exv base.
@@ -796,20 +796,20 @@ hlfir::Entity hlfir::derefPointersAndAllocatables(mlir::Location loc,
                                                   fir::FirOpBuilder &builder,
                                                   Entity entity) {
   if (entity.isMutableBox()) {
-    hlfir::Entity boxLoad{builder.create<fir::LoadOp>(loc, entity)};
+    hlfir::Entity boxLoad{fir::LoadOp::create(builder, loc, entity)};
     if (entity.isScalar()) {
       if (!entity.isPolymorphic() && !entity.hasLengthParameters())
-        return hlfir::Entity{builder.create<fir::BoxAddrOp>(loc, boxLoad)};
+        return hlfir::Entity{fir::BoxAddrOp::create(builder, loc, boxLoad)};
       mlir::Type elementType = boxLoad.getFortranElementType();
       if (auto charType = mlir::dyn_cast<fir::CharacterType>(elementType)) {
-        mlir::Value base = builder.create<fir::BoxAddrOp>(loc, boxLoad);
+        mlir::Value base = fir::BoxAddrOp::create(builder, loc, boxLoad);
         if (charType.hasConstantLen())
           return hlfir::Entity{base};
         mlir::Value len = genCharacterVariableLength(loc, builder, entity);
         auto boxCharType =
             fir::BoxCharType::get(builder.getContext(), charType.getFKind());
         return hlfir::Entity{
-            builder.create<fir::EmboxCharOp>(loc, boxCharType, base, len)
+            fir::EmboxCharOp::create(builder, loc, boxCharType, base, len)
                 .getResult()};
       }
     }
@@ -819,7 +819,7 @@ hlfir::Entity hlfir::derefPointersAndAllocatables(mlir::Location loc,
     // information. Keep them boxed.
     return boxLoad;
   } else if (entity.isProcedurePointer()) {
-    return hlfir::Entity{builder.create<fir::LoadOp>(loc, entity)};
+    return hlfir::Entity{fir::LoadOp::create(builder, loc, entity)};
   }
   return entity;
 }
@@ -870,8 +870,8 @@ hlfir::ElementalOp hlfir::genElementalOp(
     mlir::Value polymorphicMold, mlir::Type exprType) {
   if (!exprType)
     exprType = getArrayExprType(elementType, shape, !!polymorphicMold);
-  auto elementalOp = builder.create<hlfir::ElementalOp>(
-      loc, exprType, shape, polymorphicMold, typeParams, isUnordered);
+  auto elementalOp = hlfir::ElementalOp::create(
+      builder, loc, exprType, shape, polymorphicMold, typeParams, isUnordered);
   auto insertPt = builder.saveInsertionPoint();
   builder.setInsertionPointToStart(elementalOp.getBody());
   mlir::Value elementResult = genKernel(loc, builder, elementalOp.getIndices());
@@ -881,7 +881,7 @@ hlfir::ElementalOp hlfir::genElementalOp(
   // here.
   if (fir::isa_trivial(elementResult.getType()))
     elementResult = builder.createConvert(loc, elementType, elementResult);
-  builder.create<hlfir::YieldElementOp>(loc, elementResult);
+  hlfir::YieldElementOp::create(builder, loc, elementResult);
   builder.restoreInsertionPoint(insertPt);
   return elementalOp;
 }
@@ -948,10 +948,10 @@ hlfir::LoopNest hlfir::genLoopNest(mlir::Location loc,
   mlir::OpBuilder::InsertionGuard guard(builder);
   loopNest.oneBasedIndices.assign(extents.size(), mlir::Value{});
   // Build loop nest from column to row.
-  auto one = builder.create<mlir::arith::ConstantIndexOp>(loc, 1);
+  auto one = mlir::arith::ConstantIndexOp::create(builder, loc, 1);
   mlir::Type indexType = builder.getIndexType();
   if (emitWorkshareLoop) {
-    auto wslw = builder.create<mlir::omp::WorkshareLoopWrapperOp>(loc);
+    auto wslw = mlir::omp::WorkshareLoopWrapperOp::create(builder, loc);
     loopNest.outerOp = wslw;
     builder.createBlock(&wslw.getRegion());
     mlir::omp::LoopNestOperands lnops;
@@ -961,12 +961,12 @@ hlfir::LoopNest hlfir::genLoopNest(mlir::Location loc,
       lnops.loopUpperBounds.push_back(extent);
       lnops.loopSteps.push_back(one);
     }
-    auto lnOp = builder.create<mlir::omp::LoopNestOp>(loc, lnops);
+    auto lnOp = mlir::omp::LoopNestOp::create(builder, loc, lnops);
     mlir::Block *block = builder.createBlock(&lnOp.getRegion());
     for (auto extent : llvm::reverse(extents))
       block->addArgument(extent.getType(), extent.getLoc());
     loopNest.body = block;
-    builder.create<mlir::omp::YieldOp>(loc);
+    mlir::omp::YieldOp::create(builder, loc);
     for (unsigned dim = 0; dim < extents.size(); dim++)
       loopNest.oneBasedIndices[extents.size() - dim - 1] =
           lnOp.getRegion().front().getArgument(dim);
@@ -975,7 +975,7 @@ hlfir::LoopNest hlfir::genLoopNest(mlir::Location loc,
     for (auto extent : llvm::reverse(extents)) {
       auto ub = builder.createConvert(loc, indexType, extent);
       auto doLoop =
-          builder.create<fir::DoLoopOp>(loc, one, ub, one, isUnordered);
+          fir::DoLoopOp::create(builder, loc, one, ub, one, isUnordered);
       if (!couldVectorize) {
         mlir::LLVM::LoopVectorizeAttr va{mlir::LLVM::LoopVectorizeAttr::get(
             builder.getContext(),
@@ -1002,7 +1002,7 @@ llvm::SmallVector<mlir::Value> hlfir::genLoopNestWithReductions(
     bool isUnordered) {
   assert(!extents.empty() && "must have at least one extent");
   // Build loop nest from column to row.
-  auto one = builder.create<mlir::arith::ConstantIndexOp>(loc, 1);
+  auto one = mlir::arith::ConstantIndexOp::create(builder, loc, 1);
   mlir::Type indexType = builder.getIndexType();
   unsigned dim = extents.size() - 1;
   fir::DoLoopOp outerLoop = nullptr;
@@ -1018,16 +1018,15 @@ llvm::SmallVector<mlir::Value> hlfir::genLoopNestWithReductions(
     // of its parent loop.
     fir::DoLoopOp doLoop;
     if (!parentLoop) {
-      doLoop = builder.create<fir::DoLoopOp>(loc, one, ub, one, isUnordered,
-                                             /*finalCountValue=*/false,
-                                             reductionInits);
+      doLoop = fir::DoLoopOp::create(builder, loc, one, ub, one, isUnordered,
+                                     /*finalCountValue=*/false, reductionInits);
     } else {
-      doLoop = builder.create<fir::DoLoopOp>(loc, one, ub, one, isUnordered,
-                                             /*finalCountValue=*/false,
-                                             parentLoop.getRegionIterArgs());
+      doLoop = fir::DoLoopOp::create(builder, loc, one, ub, one, isUnordered,
+                                     /*finalCountValue=*/false,
+                                     parentLoop.getRegionIterArgs());
       if (!reductionInits.empty()) {
         // Return the results of the child loop from its parent loop.
-        builder.create<fir::ResultOp>(loc, doLoop.getResults());
+        fir::ResultOp::create(builder, loc, doLoop.getResults());
       }
     }
 
@@ -1044,7 +1043,7 @@ llvm::SmallVector<mlir::Value> hlfir::genLoopNestWithReductions(
       genBody(loc, builder, oneBasedIndices, parentLoop.getRegionIterArgs());
   builder.setInsertionPointToEnd(parentLoop.getBody());
   if (!reductionValues.empty())
-    builder.create<fir::ResultOp>(loc, reductionValues);
+    fir::ResultOp::create(builder, loc, reductionValues);
   builder.setInsertionPointAfter(outerLoop);
   return outerLoop->getResults();
 }
@@ -1057,18 +1056,18 @@ conditionallyEvaluate(mlir::Location loc, fir::FirOpBuilder &builder,
 
   // Evaluate in some region that will be moved into the actual ifOp (the actual
   // ifOp can only be created when the result types are known).
-  auto badIfOp = builder.create<fir::IfOp>(loc, condition.getType(), condition,
-                                           /*withElseRegion=*/false);
+  auto badIfOp = fir::IfOp::create(builder, loc, condition.getType(), condition,
+                                   /*withElseRegion=*/false);
   mlir::Block *preparationBlock = &badIfOp.getThenRegion().front();
   builder.setInsertionPointToStart(preparationBlock);
   fir::ExtendedValue result = genIfTrue();
   fir::ResultOp resultOp = result.match(
       [&](const fir::CharBoxValue &box) -> fir::ResultOp {
-        return builder.create<fir::ResultOp>(
-            loc, mlir::ValueRange{box.getAddr(), box.getLen()});
+        return fir::ResultOp::create(
+            builder, loc, mlir::ValueRange{box.getAddr(), box.getLen()});
       },
       [&](const mlir::Value &addr) -> fir::ResultOp {
-        return builder.create<fir::ResultOp>(loc, addr);
+        return fir::ResultOp::create(builder, loc, addr);
       },
       [&](const auto &) -> fir::ResultOp {
         TODO(loc, "unboxing non scalar optional fir.box");
@@ -1077,8 +1076,8 @@ conditionallyEvaluate(mlir::Location loc, fir::FirOpBuilder &builder,
 
   // Create actual fir.if operation.
   auto ifOp =
-      builder.create<fir::IfOp>(loc, resultOp->getOperandTypes(), condition,
-                                /*withElseRegion=*/true);
+      fir::IfOp::create(builder, loc, resultOp->getOperandTypes(), condition,
+                        /*withElseRegion=*/true);
   // Move evaluation into Then block,
   preparationBlock->moveBefore(&ifOp.getThenRegion().back());
   ifOp.getThenRegion().back().erase();
@@ -1087,11 +1086,11 @@ conditionallyEvaluate(mlir::Location loc, fir::FirOpBuilder &builder,
   llvm::SmallVector<mlir::Value> absentValues;
   for (mlir::Type resTy : ifOp->getResultTypes()) {
     if (fir::isa_ref_type(resTy) || fir::isa_box_type(resTy))
-      absentValues.emplace_back(builder.create<fir::AbsentOp>(loc, resTy));
+      absentValues.emplace_back(fir::AbsentOp::create(builder, loc, resTy));
     else
-      absentValues.emplace_back(builder.create<fir::ZeroOp>(loc, resTy));
+      absentValues.emplace_back(fir::ZeroOp::create(builder, loc, resTy));
   }
-  builder.create<fir::ResultOp>(loc, absentValues);
+  fir::ResultOp::create(builder, loc, absentValues);
   badIfOp->erase();
 
   // Build fir::ExtendedValue from the result values.
@@ -1139,8 +1138,8 @@ static fir::ExtendedValue translateVariableToExtendedValue(
     }
     if (variable.mayBeOptional()) {
       if (!keepScalarOptionalBoxed && variable.isScalar()) {
-        mlir::Value isPresent = builder.create<fir::IsPresentOp>(
-            loc, builder.getI1Type(), variable);
+        mlir::Value isPresent = fir::IsPresentOp::create(
+            builder, loc, builder.getI1Type(), variable);
         return conditionallyEvaluate(
             loc, builder, isPresent, [&]() -> fir::ExtendedValue {
               mlir::Value base = genVariableRawAddress(loc, builder, variable);
@@ -1249,7 +1248,7 @@ static fir::ExtendedValue placeTrivialInMemory(mlir::Location loc,
   if (targetType != val.getType())
     builder.createStoreWithConvert(loc, val, temp);
   else
-    builder.create<fir::StoreOp>(loc, val, temp);
+    fir::StoreOp::create(builder, loc, val, temp);
   return temp;
 }
 
@@ -1369,8 +1368,8 @@ hlfir::createTempFromMold(mlir::Location loc, fir::FirOpBuilder &builder,
          llvm::ArrayRef<mlir::Value> typeParams,
          fir::FortranVariableFlagsAttr attrs) -> mlir::Value {
     auto declareOp =
-        builder.create<hlfir::DeclareOp>(loc, memref, name, shape, typeParams,
-                                         /*dummy_scope=*/nullptr, attrs);
+        hlfir::DeclareOp::create(builder, loc, memref, name, shape, typeParams,
+                                 /*dummy_scope=*/nullptr, attrs);
     return declareOp.getBase();
   };
 
@@ -1406,8 +1405,8 @@ hlfir::Entity hlfir::createStackTempFromMold(mlir::Location loc,
                                     /*shape=*/{}, lenParams);
   }
   auto declareOp =
-      builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, shape, lenParams,
-                                       /*dummy_scope=*/nullptr, declAttrs);
+      hlfir::DeclareOp::create(builder, loc, alloc, tmpName, shape, lenParams,
+                               /*dummy_scope=*/nullptr, declAttrs);
   return hlfir::Entity{declareOp.getBase()};
 }
 
@@ -1422,8 +1421,8 @@ hlfir::convertCharacterKind(mlir::Location loc, fir::FirOpBuilder &builder,
   if (src.second.has_value())
     src.second.value()();
 
-  return hlfir::EntityWithAttributes{builder.create<hlfir::DeclareOp>(
-      loc, res.getAddr(), ".temp.kindconvert", /*shape=*/nullptr,
+  return hlfir::EntityWithAttributes{hlfir::DeclareOp::create(
+      builder, loc, res.getAddr(), ".temp.kindconvert", /*shape=*/nullptr,
       /*typeparams=*/mlir::ValueRange{res.getLen()},
       /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{})};
 }
@@ -1494,10 +1493,10 @@ hlfir::genTypeAndKindConvert(mlir::Location loc, fir::FirOpBuilder &builder,
     }
     auto shapeShiftType = fir::ShapeShiftType::get(builder.getContext(), rank);
     mlir::Value shapeShift =
-        builder.create<fir::ShapeShiftOp>(loc, shapeShiftType, lbAndExtents);
-    auto declareOp = builder.create<hlfir::DeclareOp>(
-        loc, associate.getFirBase(), *associate.getUniqName(), shapeShift,
-        associate.getTypeparams(), /*dummy_scope=*/nullptr,
+        fir::ShapeShiftOp::create(builder, loc, shapeShiftType, lbAndExtents);
+    auto declareOp = hlfir::DeclareOp::create(
+        builder, loc, associate.getFirBase(), *associate.getUniqName(),
+        shapeShift, associate.getTypeparams(), /*dummy_scope=*/nullptr,
         /*flags=*/fir::FortranVariableFlagsAttr{});
     hlfir::Entity castWithLbounds =
         mlir::cast<fir::FortranVariableOpInterface>(declareOp.getOperation());
@@ -1536,8 +1535,8 @@ std::pair<hlfir::Entity, bool> hlfir::computeEvaluateOpInNewTemp(
                                                    extents, typeParams);
   mlir::Value innerMemory = evalInMem.getMemory();
   temp = builder.createConvert(loc, innerMemory.getType(), temp);
-  auto declareOp = builder.create<hlfir::DeclareOp>(
-      loc, temp, tmpName, shape, typeParams,
+  auto declareOp = hlfir::DeclareOp::create(
+      builder, loc, temp, tmpName, shape, typeParams,
       /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
   computeEvaluateOpIn(loc, builder, evalInMem, declareOp.getOriginalBase());
   return {hlfir::Entity{declareOp.getBase()}, /*heapAllocated=*/heapAllocated};
@@ -1601,7 +1600,7 @@ hlfir::Entity hlfir::gen1DSection(mlir::Location loc,
     }
   }
   mlir::Value sectionShape =
-      builder.create<fir::ShapeOp>(loc, extents[dim - 1]);
+      fir::ShapeOp::create(builder, loc, extents[dim - 1]);
 
   // The result type is one of:
   //   !fir.box/class<!fir.array<NxT>>
@@ -1617,9 +1616,9 @@ hlfir::Entity hlfir::gen1DSection(mlir::Location loc,
       fir::SequenceType::get({dimExtent}, seqType.getEleTy());
   sectionType = fir::wrapInClassOrBoxType(sectionType, array.isPolymorphic());
 
-  auto designate = builder.create<hlfir::DesignateOp>(
-      loc, sectionType, array, /*component=*/"", /*componentShape=*/nullptr,
-      subscripts,
+  auto designate = hlfir::DesignateOp::create(
+      builder, loc, sectionType, array, /*component=*/"",
+      /*componentShape=*/nullptr, subscripts,
       /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
       sectionShape, typeParams);
   return hlfir::Entity{designate.getResult()};

diff  --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d77a656158a37..323d2fe1ac16f 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -1110,7 +1110,7 @@ mlir::Value genLibCall(fir::FirOpBuilder &builder, mlir::Location loc,
     // was just created from user functions with the same name.
     funcOp->setAttr(fir::FIROpsDialect::getFirRuntimeAttrName(),
                     builder.getUnitAttr());
-    auto libCall = builder.create<fir::CallOp>(loc, funcOp, args);
+    auto libCall = fir::CallOp::create(builder, loc, funcOp, args);
     // TODO: ensure 'strictfp' setting on the call for "precise/strict"
     //       FP mode. Set appropriate Fast-Math Flags otherwise.
     // TODO: we should also mark as many libm function as possible
@@ -1124,7 +1124,7 @@ mlir::Value genLibCall(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Type soughtFuncType = funcOp.getFunctionType();
 
   if (soughtFuncType == libFuncType) {
-    libCall = builder.create<fir::CallOp>(loc, funcOp, args);
+    libCall = fir::CallOp::create(builder, loc, funcOp, args);
   } else {
     // A function with the same name might have been declared
     // before (e.g. with an explicit interface and a binding label).
@@ -1138,13 +1138,13 @@ mlir::Value genLibCall(fir::FirOpBuilder &builder, mlir::Location loc,
                  llvm::Twine("' may lead to undefined behavior.")));
     mlir::SymbolRefAttr funcSymbolAttr = builder.getSymbolRefAttr(libFuncName);
     mlir::Value funcPointer =
-        builder.create<fir::AddrOfOp>(loc, soughtFuncType, funcSymbolAttr);
+        fir::AddrOfOp::create(builder, loc, soughtFuncType, funcSymbolAttr);
     funcPointer = builder.createConvert(loc, libFuncType, funcPointer);
 
     llvm::SmallVector<mlir::Value, 3> operands{funcPointer};
     operands.append(args.begin(), args.end());
-    libCall = builder.create<fir::CallOp>(loc, mlir::SymbolRefAttr{},
-                                          libFuncType.getResults(), operands);
+    libCall = fir::CallOp::create(builder, loc, mlir::SymbolRefAttr{},
+                                  libFuncType.getResults(), operands);
   }
 
   LLVM_DEBUG(libCall.dump(); llvm::dbgs() << "\n");
@@ -1220,7 +1220,7 @@ mlir::Value genMathOp(fir::FirOpBuilder &builder, mlir::Location loc,
     LLVM_DEBUG(llvm::dbgs() << "Generating '" << mathLibFuncName
                             << "' operation with type ";
                mathLibFuncType.dump(); llvm::dbgs() << "\n");
-    result = builder.create<T>(loc, args);
+    result = T::create(builder, loc, args);
   }
   LLVM_DEBUG(result.dump(); llvm::dbgs() << "\n");
   return result;
@@ -1258,12 +1258,12 @@ mlir::Value genComplexMathOp(fir::FirOpBuilder &builder, mlir::Location loc,
   // the argument types for an operation
   if constexpr (T::template hasTrait<
                     mlir::OpTrait::SameOperandsAndResultType>()) {
-    result = builder.create<T>(loc, args);
+    result = T::create(builder, loc, args);
     result = builder.createConvert(loc, mathLibFuncType.getResult(0), result);
   } else {
     auto complexTy = mlir::cast<mlir::ComplexType>(mathLibFuncType.getInput(0));
     auto realTy = complexTy.getElementType();
-    result = builder.create<T>(loc, realTy, args);
+    result = T::create(builder, loc, realTy, args);
     result = builder.createConvert(loc, mathLibFuncType.getResult(0), result);
   }
 
@@ -2461,7 +2461,7 @@ IntrinsicLibrary::outlineInWrapper(GeneratorType generator,
     nameOS << '.' << fmfString;
   }
   mlir::func::FuncOp wrapper = getWrapper(generator, funcName, funcType);
-  return builder.create<fir::CallOp>(loc, wrapper, args).getResult(0);
+  return fir::CallOp::create(builder, loc, wrapper, args).getResult(0);
 }
 
 template <typename GeneratorType>
@@ -2477,7 +2477,7 @@ fir::ExtendedValue IntrinsicLibrary::outlineInExtendedWrapper(
     mlirArgs.emplace_back(toValue(extendedVal, builder, loc));
   mlir::FunctionType funcType = getFunctionType(resultType, mlirArgs, builder);
   mlir::func::FuncOp wrapper = getWrapper(generator, name, funcType);
-  auto call = builder.create<fir::CallOp>(loc, wrapper, mlirArgs);
+  auto call = fir::CallOp::create(builder, loc, wrapper, mlirArgs);
   if (resultType)
     return toExtendedValue(call.getResult(0), builder, loc);
   // Subroutine calls
@@ -2595,9 +2595,9 @@ IntrinsicLibrary::readAndAddCleanUp(fir::MutableBoxValue resultMutableBox,
         return box;
       },
       [&](const mlir::Value &tempAddr) -> fir::ExtendedValue {
-        auto load = builder.create<fir::LoadOp>(loc, resultType, tempAddr);
+        auto load = fir::LoadOp::create(builder, loc, resultType, tempAddr);
         // Temp can be freed right away since it was loaded.
-        builder.create<fir::FreeMemOp>(loc, tempAddr);
+        fir::FreeMemOp::create(builder, loc, tempAddr);
         return load;
       },
       [&](const fir::CharBoxValue &box) -> fir::ExtendedValue {
@@ -2650,9 +2650,9 @@ mlir::Value IntrinsicLibrary::genAbs(mlir::Type resultType,
     // So, implement abs here without branching.
     mlir::Value shift =
         builder.createIntegerConstant(loc, intType, intType.getWidth() - 1);
-    auto mask = builder.create<mlir::arith::ShRSIOp>(loc, arg, shift);
-    auto xored = builder.create<mlir::arith::XOrIOp>(loc, arg, mask);
-    return builder.create<mlir::arith::SubIOp>(loc, xored, mask);
+    auto mask = mlir::arith::ShRSIOp::create(builder, loc, arg, shift);
+    auto xored = mlir::arith::XOrIOp::create(builder, loc, arg, mask);
+    return mlir::arith::SubIOp::create(builder, loc, xored, mask);
   }
   llvm_unreachable("unexpected type in ABS argument");
 }
@@ -2671,7 +2671,7 @@ mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
   mlir::Value dfactor = builder.createRealConstant(
       loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  return builder.create<mlir::arith::MulFOp>(loc, result, factor);
+  return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
 // ADJUSTL & ADJUSTR
@@ -2824,7 +2824,7 @@ mlir::Value IntrinsicLibrary::genAsind(mlir::Type resultType,
   mlir::Value dfactor = builder.createRealConstant(
       loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  return builder.create<mlir::arith::MulFOp>(loc, result, factor);
+  return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
 // ATAND, ATAN2D
@@ -2838,8 +2838,8 @@ mlir::Value IntrinsicLibrary::genAtand(mlir::Type resultType,
 
   // atand = atan * 180/pi
   if (args.size() == 2) {
-    atan = builder.create<mlir::math::Atan2Op>(loc, fir::getBase(args[0]),
-                                               fir::getBase(args[1]));
+    atan = mlir::math::Atan2Op::create(builder, loc, fir::getBase(args[0]),
+                                       fir::getBase(args[1]));
   } else {
     mlir::FunctionType ftype =
         mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
@@ -2849,7 +2849,7 @@ mlir::Value IntrinsicLibrary::genAtand(mlir::Type resultType,
   mlir::Value dfactor = builder.createRealConstant(
       loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
-  return builder.create<mlir::arith::MulFOp>(loc, atan, factor);
+  return mlir::arith::MulFOp::create(builder, loc, atan, factor);
 }
 
 // ATANPI, ATAN2PI
@@ -2863,8 +2863,8 @@ mlir::Value IntrinsicLibrary::genAtanpi(mlir::Type resultType,
 
   // atanpi = atan / pi
   if (args.size() == 2) {
-    atan = builder.create<mlir::math::Atan2Op>(loc, fir::getBase(args[0]),
-                                               fir::getBase(args[1]));
+    atan = mlir::math::Atan2Op::create(builder, loc, fir::getBase(args[0]),
+                                       fir::getBase(args[1]));
   } else {
     mlir::FunctionType ftype =
         mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
@@ -2874,7 +2874,7 @@ mlir::Value IntrinsicLibrary::genAtanpi(mlir::Type resultType,
   mlir::Value dfactor =
       builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
   mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
-  return builder.create<mlir::arith::MulFOp>(loc, atan, factor);
+  return mlir::arith::MulFOp::create(builder, loc, atan, factor);
 }
 
 static mlir::Value genAtomBinOp(fir::FirOpBuilder &builder, mlir::Location &loc,
@@ -2882,8 +2882,8 @@ static mlir::Value genAtomBinOp(fir::FirOpBuilder &builder, mlir::Location &loc,
                                 mlir::Value arg1) {
   auto llvmPointerType = mlir::LLVM::LLVMPointerType::get(builder.getContext());
   arg0 = builder.createConvert(loc, llvmPointerType, arg0);
-  return builder.create<mlir::LLVM::AtomicRMWOp>(
-      loc, binOp, arg0, arg1, mlir::LLVM::AtomicOrdering::seq_cst);
+  return mlir::LLVM::AtomicRMWOp::create(builder, loc, binOp, arg0, arg1,
+                                         mlir::LLVM::AtomicOrdering::seq_cst);
 }
 
 mlir::Value IntrinsicLibrary::genAtomicAdd(mlir::Type resultType,
@@ -2941,11 +2941,11 @@ IntrinsicLibrary::genAtomicCas(mlir::Type resultType,
 
   auto bitCastFloat = [&](mlir::Value arg) -> mlir::Value {
     if (mlir::isa<mlir::Float32Type>(arg.getType()))
-      return builder.create<mlir::LLVM::BitcastOp>(loc, builder.getI32Type(),
-                                                   arg);
+      return mlir::LLVM::BitcastOp::create(builder, loc, builder.getI32Type(),
+                                           arg);
     if (mlir::isa<mlir::Float64Type>(arg.getType()))
-      return builder.create<mlir::LLVM::BitcastOp>(loc, builder.getI64Type(),
-                                                   arg);
+      return mlir::LLVM::BitcastOp::create(builder, loc, builder.getI64Type(),
+                                           arg);
     return arg;
   };
 
@@ -2958,11 +2958,11 @@ IntrinsicLibrary::genAtomicCas(mlir::Type resultType,
   }
 
   auto address =
-      builder.create<mlir::UnrealizedConversionCastOp>(loc, llvmPtrTy, arg0)
+      mlir::UnrealizedConversionCastOp::create(builder, loc, llvmPtrTy, arg0)
           .getResult(0);
-  auto cmpxchg = builder.create<mlir::LLVM::AtomicCmpXchgOp>(
-      loc, address, arg1, arg2, successOrdering, failureOrdering);
-  return builder.create<mlir::LLVM::ExtractValueOp>(loc, cmpxchg, 1);
+  auto cmpxchg = mlir::LLVM::AtomicCmpXchgOp::create(
+      builder, loc, address, arg1, arg2, successOrdering, failureOrdering);
+  return mlir::LLVM::ExtractValueOp::create(builder, loc, cmpxchg, 1);
 }
 
 mlir::Value IntrinsicLibrary::genAtomicDec(mlir::Type resultType,
@@ -3038,31 +3038,31 @@ IntrinsicLibrary::genAssociated(mlir::Type resultType,
                 mlir::isa<fir::BoxProcType>(ptrTy))) {
     mlir::Value pointerBoxProc =
         fir::isBoxProcAddressType(ptrTy)
-            ? builder.create<fir::LoadOp>(loc, fir::getBase(args[0]))
+            ? fir::LoadOp::create(builder, loc, fir::getBase(args[0]))
             : fir::getBase(args[0]);
     mlir::Value pointerTarget =
-        builder.create<fir::BoxAddrOp>(loc, pointerBoxProc);
+        fir::BoxAddrOp::create(builder, loc, pointerBoxProc);
     if (isStaticallyAbsent(args[1]))
       return builder.genIsNotNullAddr(loc, pointerTarget);
     mlir::Value target = fir::getBase(args[1]);
     if (fir::isBoxProcAddressType(target.getType()))
-      target = builder.create<fir::LoadOp>(loc, target);
+      target = fir::LoadOp::create(builder, loc, target);
     if (mlir::isa<fir::BoxProcType>(target.getType()))
-      target = builder.create<fir::BoxAddrOp>(loc, target);
+      target = fir::BoxAddrOp::create(builder, loc, target);
     mlir::Type intPtrTy = builder.getIntPtrType();
     mlir::Value pointerInt =
         builder.createConvert(loc, intPtrTy, pointerTarget);
     mlir::Value targetInt = builder.createConvert(loc, intPtrTy, target);
-    mlir::Value sameTarget = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::eq, pointerInt, targetInt);
+    mlir::Value sameTarget = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::eq, pointerInt, targetInt);
     mlir::Value zero = builder.createIntegerConstant(loc, intPtrTy, 0);
-    mlir::Value notNull = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::ne, zero, pointerInt);
+    mlir::Value notNull = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::ne, zero, pointerInt);
     // The not notNull test covers the following two cases:
     // - TARGET is a procedure that is OPTIONAL and absent at runtime.
     // - TARGET is a procedure pointer that is NULL.
     // In both cases, ASSOCIATED should be false if POINTER is NULL.
-    return builder.create<mlir::arith::AndIOp>(loc, sameTarget, notNull);
+    return mlir::arith::AndIOp::create(builder, loc, sameTarget, notNull);
   }
   auto *pointer =
       args[0].match([&](const fir::MutableBoxValue &x) { return &x; },
@@ -3075,7 +3075,7 @@ IntrinsicLibrary::genAssociated(mlir::Type resultType,
   mlir::Value targetBox = builder.createBox(loc, target);
   mlir::Value pointerBoxRef =
       fir::factory::getMutableIRBox(builder, loc, *pointer);
-  auto pointerBox = builder.create<fir::LoadOp>(loc, pointerBoxRef);
+  auto pointerBox = fir::LoadOp::create(builder, loc, pointerBoxRef);
   return fir::runtime::genAssociated(builder, loc, pointerBox, targetBox);
 }
 
@@ -3106,12 +3106,12 @@ IntrinsicLibrary::genBesselJn(mlir::Type resultType,
     mlir::Value resultBox =
         fir::factory::getMutableIRBox(builder, loc, resultMutableBox);
 
-    mlir::Value cmpXEq0 = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UEQ, x, zero);
-    mlir::Value cmpN1LtN2 = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::slt, n1, n2);
-    mlir::Value cmpN1EqN2 = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::eq, n1, n2);
+    mlir::Value cmpXEq0 = mlir::arith::CmpFOp::create(
+        builder, loc, mlir::arith::CmpFPredicate::UEQ, x, zero);
+    mlir::Value cmpN1LtN2 = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::slt, n1, n2);
+    mlir::Value cmpN1EqN2 = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::eq, n1, n2);
 
     auto genXEq0 = [&]() {
       fir::runtime::genBesselJnX0(builder, loc, floatTy, resultBox, n1, n2);
@@ -3123,7 +3123,7 @@ IntrinsicLibrary::genBesselJn(mlir::Type resultType,
       // https://dlmf.nist.gov/10.6.E1). When n1 < n2, this requires
       // the values of BESSEL_JN(n2) and BESSEL_JN(n2 - 1) since they
       // are the anchors of the recursion.
-      mlir::Value n2_1 = builder.create<mlir::arith::SubIOp>(loc, n2, one);
+      mlir::Value n2_1 = mlir::arith::SubIOp::create(builder, loc, n2, one);
       mlir::Value bn2 = genRuntimeCall("bessel_jn", resultType, {n2, x});
       mlir::Value bn2_1 = genRuntimeCall("bessel_jn", resultType, {n2_1, x});
       fir::runtime::genBesselJn(builder, loc, resultBox, n1, n2, x, bn2, bn2_1);
@@ -3191,12 +3191,12 @@ IntrinsicLibrary::genBesselYn(mlir::Type resultType,
     mlir::Value resultBox =
         fir::factory::getMutableIRBox(builder, loc, resultMutableBox);
 
-    mlir::Value cmpXEq0 = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UEQ, x, zero);
-    mlir::Value cmpN1LtN2 = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::slt, n1, n2);
-    mlir::Value cmpN1EqN2 = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::eq, n1, n2);
+    mlir::Value cmpXEq0 = mlir::arith::CmpFOp::create(
+        builder, loc, mlir::arith::CmpFPredicate::UEQ, x, zero);
+    mlir::Value cmpN1LtN2 = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::slt, n1, n2);
+    mlir::Value cmpN1EqN2 = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::eq, n1, n2);
 
     auto genXEq0 = [&]() {
       fir::runtime::genBesselYnX0(builder, loc, floatTy, resultBox, n1, n2);
@@ -3208,7 +3208,7 @@ IntrinsicLibrary::genBesselYn(mlir::Type resultType,
       // https://dlmf.nist.gov/10.6.E1). When n1 < n2, this requires
       // the values of BESSEL_YN(n1) and BESSEL_YN(n1 + 1) since they
       // are the anchors of the recursion.
-      mlir::Value n1_1 = builder.create<mlir::arith::AddIOp>(loc, n1, one);
+      mlir::Value n1_1 = mlir::arith::AddIOp::create(builder, loc, n1, one);
       mlir::Value bn1 = genRuntimeCall("bessel_yn", resultType, {n1, x});
       mlir::Value bn1_1 = genRuntimeCall("bessel_yn", resultType, {n1_1, x});
       fir::runtime::genBesselYn(builder, loc, resultBox, n1, n2, x, bn1, bn1_1);
@@ -3280,12 +3280,12 @@ IntrinsicLibrary::genBitwiseCompare(mlir::Type resultType,
   if (arg0Ty.isUnsignedInteger())
     arg0 = builder.createConvert(loc, signlessType, arg0);
   else if (bits0 < widest)
-    arg0 = builder.create<mlir::arith::ExtUIOp>(loc, signlessType, arg0);
+    arg0 = mlir::arith::ExtUIOp::create(builder, loc, signlessType, arg0);
   if (arg1Ty.isUnsignedInteger())
     arg1 = builder.createConvert(loc, signlessType, arg1);
   else if (bits1 < widest)
-    arg1 = builder.create<mlir::arith::ExtUIOp>(loc, signlessType, arg1);
-  return builder.create<mlir::arith::CmpIOp>(loc, pred, arg0, arg1);
+    arg1 = mlir::arith::ExtUIOp::create(builder, loc, signlessType, arg1);
+  return mlir::arith::CmpIOp::create(builder, loc, pred, arg0, arg1);
 }
 
 // BTEST
@@ -3304,9 +3304,9 @@ mlir::Value IntrinsicLibrary::genBtest(mlir::Type resultType,
     word = builder.createConvert(loc, signlessType, word);
   mlir::Value shiftCount = builder.createConvert(loc, signlessType, args[1]);
   mlir::Value shifted =
-      builder.create<mlir::arith::ShRUIOp>(loc, word, shiftCount);
+      mlir::arith::ShRUIOp::create(builder, loc, word, shiftCount);
   mlir::Value one = builder.createIntegerConstant(loc, signlessType, 1);
-  mlir::Value bit = builder.create<mlir::arith::AndIOp>(loc, shifted, one);
+  mlir::Value bit = mlir::arith::AndIOp::create(builder, loc, shifted, one);
   return builder.createConvert(loc, resultType, bit);
 }
 
@@ -3317,11 +3317,11 @@ static mlir::Value getAddrFromBox(fir::FirOpBuilder &builder,
   mlir::Value addr{nullptr};
   if (isFunc) {
     auto funcTy = mlir::cast<fir::BoxProcType>(argValue.getType()).getEleTy();
-    addr = builder.create<fir::BoxAddrOp>(loc, funcTy, argValue);
+    addr = fir::BoxAddrOp::create(builder, loc, funcTy, argValue);
   } else {
     const auto *box = arg.getBoxOf<fir::BoxValue>();
-    addr = builder.create<fir::BoxAddrOp>(loc, box->getMemTy(),
-                                          fir::getBase(*box));
+    addr = fir::BoxAddrOp::create(builder, loc, box->getMemTy(),
+                                  fir::getBase(*box));
   }
   return addr;
 }
@@ -3331,7 +3331,7 @@ genCLocOrCFunLoc(fir::FirOpBuilder &builder, mlir::Location loc,
                  mlir::Type resultType, llvm::ArrayRef<fir::ExtendedValue> args,
                  bool isFunc = false, bool isDevLoc = false) {
   assert(args.size() == 1);
-  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Value res = fir::AllocaOp::create(builder, loc, resultType);
   mlir::Value resAddr;
   if (isDevLoc)
     resAddr = fir::factory::genCDevPtrAddr(builder, loc, res, resultType);
@@ -3342,7 +3342,7 @@ genCLocOrCFunLoc(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value argAddr = getAddrFromBox(builder, loc, args[0], isFunc);
   mlir::Value argAddrVal = builder.createConvert(
       loc, fir::unwrapRefType(resAddr.getType()), argAddr);
-  builder.create<fir::StoreOp>(loc, argAddrVal, resAddr);
+  fir::StoreOp::create(builder, loc, argAddrVal, resAddr);
   return res;
 }
 
@@ -3355,8 +3355,8 @@ genCAssociated(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value cPtrVal1 =
       fir::factory::genCPtrOrCFunptrValue(builder, loc, cPtr1);
   mlir::Value zero = builder.createIntegerConstant(loc, cPtrVal1.getType(), 0);
-  mlir::Value res = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::ne, cPtrVal1, zero);
+  mlir::Value res = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::ne, cPtrVal1, zero);
 
   if (isStaticallyPresent(args[1])) {
     mlir::Type i1Ty = builder.getI1Type();
@@ -3365,15 +3365,16 @@ genCAssociated(fir::FirOpBuilder &builder, mlir::Location loc,
     res =
         builder
             .genIfOp(loc, {i1Ty}, isDynamicallyAbsent, /*withElseRegion=*/true)
-            .genThen([&]() { builder.create<fir::ResultOp>(loc, res); })
+            .genThen([&]() { fir::ResultOp::create(builder, loc, res); })
             .genElse([&]() {
               mlir::Value cPtrVal2 =
                   fir::factory::genCPtrOrCFunptrValue(builder, loc, cPtr2);
-              mlir::Value cmpVal = builder.create<mlir::arith::CmpIOp>(
-                  loc, mlir::arith::CmpIPredicate::eq, cPtrVal1, cPtrVal2);
+              mlir::Value cmpVal = mlir::arith::CmpIOp::create(
+                  builder, loc, mlir::arith::CmpIPredicate::eq, cPtrVal1,
+                  cPtrVal2);
               mlir::Value newRes =
-                  builder.create<mlir::arith::AndIOp>(loc, res, cmpVal);
-              builder.create<fir::ResultOp>(loc, newRes);
+                  mlir::arith::AndIOp::create(builder, loc, res, cmpVal);
+              fir::ResultOp::create(builder, loc, newRes);
             })
             .getResults()[0];
   }
@@ -3428,9 +3429,9 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
       mlir::Type idxType = builder.getIndexType();
       for (int i = 0; i < arrayRank; ++i) {
         mlir::Value index = builder.createIntegerConstant(loc, idxType, i);
-        mlir::Value var = builder.create<fir::CoordinateOp>(
-            loc, builder.getRefType(shapeElementType), shape, index);
-        mlir::Value load = builder.create<fir::LoadOp>(loc, var);
+        mlir::Value var = fir::CoordinateOp::create(
+            builder, loc, builder.getRefType(shapeElementType), shape, index);
+        mlir::Value load = fir::LoadOp::create(builder, loc, var);
         extents.push_back(builder.createConvert(loc, idxType, load));
       }
     }
@@ -3474,8 +3475,8 @@ void IntrinsicLibrary::genCFProcPointer(
   mlir::Value cptrCast =
       builder.createConvert(loc, boxProcType.getEleTy(), cptr);
   mlir::Value cptrBox =
-      builder.create<fir::EmboxProcOp>(loc, boxProcType, cptrCast);
-  builder.create<fir::StoreOp>(loc, cptrBox, fptr);
+      fir::EmboxProcOp::create(builder, loc, boxProcType, cptrCast);
+  fir::StoreOp::create(builder, loc, cptrBox, fptr);
 }
 
 // C_FUNLOC
@@ -3505,7 +3506,7 @@ IntrinsicLibrary::genCPtrCompare(mlir::Type resultType,
   mlir::Value cPtrVal2 =
       fir::factory::genCPtrOrCFunptrValue(builder, loc, cPtr2);
   mlir::Value cmp =
-      builder.create<mlir::arith::CmpIOp>(loc, pred, cPtrVal1, cPtrVal2);
+      mlir::arith::CmpIOp::create(builder, loc, pred, cPtrVal1, cPtrVal2);
   return builder.createConvert(loc, resultType, cmp);
 }
 
@@ -3604,7 +3605,7 @@ mlir::Value IntrinsicLibrary::genConjg(mlir::Type resultType,
   mlir::Value cplx = args[0];
   auto imag = fir::factory::Complex{builder, loc}.extractComplexPart(
       cplx, /*isImagPart=*/true);
-  auto negImag = builder.create<mlir::arith::NegFOp>(loc, imag);
+  auto negImag = mlir::arith::NegFOp::create(builder, loc, imag);
   return fir::factory::Complex{builder, loc}.insertComplexPart(
       cplx, negImag, /*isImagPart=*/true);
 }
@@ -3620,7 +3621,7 @@ mlir::Value IntrinsicLibrary::genCosd(mlir::Type resultType,
   mlir::Value dfactor = builder.createRealConstant(
       loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  mlir::Value arg = mlir::arith::MulFOp::create(builder, loc, args[0], factor);
   return getRuntimeCallGenerator("cos", ftype)(builder, loc, {arg});
 }
 
@@ -3635,7 +3636,7 @@ mlir::Value IntrinsicLibrary::genCospi(mlir::Type resultType,
   mlir::Value dfactor =
       builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  mlir::Value arg = mlir::arith::MulFOp::create(builder, loc, args[0], factor);
   return getRuntimeCallGenerator("cos", ftype)(builder, loc, {arg});
 }
 
@@ -3696,7 +3697,7 @@ void IntrinsicLibrary::genCpuTime(llvm::ArrayRef<fir::ExtendedValue> args) {
   mlir::Value res1 = fir::runtime::genCpuTime(builder, loc);
   mlir::Value res2 =
       builder.createConvert(loc, fir::dyn_cast_ptrEleTy(arg->getType()), res1);
-  builder.create<fir::StoreOp>(loc, res2, *arg);
+  fir::StoreOp::create(builder, loc, res2, *arg);
 }
 
 // CSHIFT
@@ -3723,7 +3724,7 @@ IntrinsicLibrary::genCshift(mlir::Type resultType,
     // Handle required SHIFT argument as a scalar
     const mlir::Value *shiftAddr = args[1].getUnboxed();
     assert(shiftAddr && "nonscalar CSHIFT argument");
-    auto shift = builder.create<fir::LoadOp>(loc, *shiftAddr);
+    auto shift = fir::LoadOp::create(builder, loc, *shiftAddr);
 
     fir::runtime::genCshiftVector(builder, loc, resultIrBox, array, shift);
   } else {
@@ -3749,9 +3750,9 @@ IntrinsicLibrary::genCUDALDXXFunc(mlir::Type resultType,
   assert(args.size() == 1);
   mlir::Type resTy = fir::SequenceType::get(extent, resultType);
   mlir::Value arg = fir::getBase(args[0]);
-  mlir::Value res = builder.create<fir::AllocaOp>(loc, resTy);
+  mlir::Value res = fir::AllocaOp::create(builder, loc, resTy);
   if (mlir::isa<fir::BaseBoxType>(arg.getType()))
-    arg = builder.create<fir::BoxAddrOp>(loc, arg);
+    arg = fir::BoxAddrOp::create(builder, loc, arg);
   mlir::Type refResTy = fir::ReferenceType::get(resTy);
   mlir::FunctionType ftype =
       mlir::FunctionType::get(arg.getContext(), {refResTy, refResTy}, {});
@@ -3759,7 +3760,7 @@ IntrinsicLibrary::genCUDALDXXFunc(mlir::Type resultType,
   llvm::SmallVector<mlir::Value> funcArgs;
   funcArgs.push_back(res);
   funcArgs.push_back(arg);
-  builder.create<fir::CallOp>(loc, funcOp, funcArgs);
+  fir::CallOp::create(builder, loc, funcOp, funcArgs);
   mlir::Value ext =
       builder.createIntegerConstant(loc, builder.getIndexType(), extent);
   return fir::ArrayBoxValue(res, {ext});
@@ -3775,8 +3776,8 @@ void IntrinsicLibrary::genDateAndTime(llvm::ArrayRef<fir::ExtendedValue> args) {
 
   mlir::Value values = fir::getBase(args[3]);
   if (!values)
-    values = builder.create<fir::AbsentOp>(
-        loc, fir::BoxType::get(builder.getNoneType()));
+    values = fir::AbsentOp::create(builder, loc,
+                                   fir::BoxType::get(builder.getNoneType()));
 
   fir::runtime::genDateAndTime(builder, loc, charArgs[0], charArgs[1],
                                charArgs[2], values);
@@ -3788,17 +3789,17 @@ mlir::Value IntrinsicLibrary::genDim(mlir::Type resultType,
   assert(args.size() == 2);
   if (mlir::isa<mlir::IntegerType>(resultType)) {
     mlir::Value zero = builder.createIntegerConstant(loc, resultType, 0);
-    auto 
diff  = builder.create<mlir::arith::SubIOp>(loc, args[0], args[1]);
-    auto cmp = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::sgt, 
diff , zero);
-    return builder.create<mlir::arith::SelectOp>(loc, cmp, 
diff , zero);
+    auto 
diff  = mlir::arith::SubIOp::create(builder, loc, args[0], args[1]);
+    auto cmp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::sgt, 
diff , zero);
+    return mlir::arith::SelectOp::create(builder, loc, cmp, 
diff , zero);
   }
   assert(fir::isa_real(resultType) && "Only expects real and integer in DIM");
   mlir::Value zero = builder.createRealZeroConstant(loc, resultType);
-  auto 
diff  = builder.create<mlir::arith::SubFOp>(loc, args[0], args[1]);
-  auto cmp = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::OGT, 
diff , zero);
-  return builder.create<mlir::arith::SelectOp>(loc, cmp, 
diff , zero);
+  auto 
diff  = mlir::arith::SubFOp::create(builder, loc, args[0], args[1]);
+  auto cmp = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::OGT, 
diff , zero);
+  return mlir::arith::SelectOp::create(builder, loc, cmp, 
diff , zero);
 }
 
 // DOT_PRODUCT
@@ -3816,12 +3817,13 @@ IntrinsicLibrary::genDotProduct(mlir::Type resultType,
   if (fir::isa_complex(eleTy)) {
     mlir::Value result = builder.createTemporary(loc, eleTy);
     fir::runtime::genDotProduct(builder, loc, vectorA, vectorB, result);
-    return builder.create<fir::LoadOp>(loc, result);
+    return fir::LoadOp::create(builder, loc, result);
   }
 
   // This operation is only used to pass the result type
   // information to the DotProduct generator.
-  auto resultBox = builder.create<fir::AbsentOp>(loc, fir::BoxType::get(eleTy));
+  auto resultBox =
+      fir::AbsentOp::create(builder, loc, fir::BoxType::get(eleTy));
   return fir::runtime::genDotProduct(builder, loc, vectorA, vectorB, resultBox);
 }
 
@@ -3833,7 +3835,7 @@ mlir::Value IntrinsicLibrary::genDprod(mlir::Type resultType,
          "Result must be double precision in DPROD");
   mlir::Value a = builder.createConvert(loc, resultType, args[0]);
   mlir::Value b = builder.createConvert(loc, resultType, args[1]);
-  return builder.create<mlir::arith::MulFOp>(loc, a, b);
+  return mlir::arith::MulFOp::create(builder, loc, a, b);
 }
 
 // DSHIFTL
@@ -3856,14 +3858,14 @@ mlir::Value IntrinsicLibrary::genDshiftl(mlir::Type resultType,
 
   // Per the standard, the value of DSHIFTL(I, J, SHIFT) is equal to
   // IOR (SHIFTL(I, SHIFT), SHIFTR(J, BIT_SIZE(J) - SHIFT))
-  mlir::Value 
diff  = builder.create<mlir::arith::SubIOp>(loc, bitSize, shift);
+  mlir::Value 
diff  = mlir::arith::SubIOp::create(builder, loc, bitSize, shift);
 
   mlir::Value lArgs[2]{i, shift};
   mlir::Value lft = genShift<mlir::arith::ShLIOp>(signlessType, lArgs);
 
   mlir::Value rArgs[2]{j, 
diff };
   mlir::Value rgt = genShift<mlir::arith::ShRUIOp>(signlessType, rArgs);
-  mlir::Value result = builder.create<mlir::arith::OrIOp>(loc, lft, rgt);
+  mlir::Value result = mlir::arith::OrIOp::create(builder, loc, lft, rgt);
   if (resultType.isUnsignedInteger())
     return builder.createConvert(loc, resultType, result);
   return result;
@@ -3889,14 +3891,14 @@ mlir::Value IntrinsicLibrary::genDshiftr(mlir::Type resultType,
 
   // Per the standard, the value of DSHIFTR(I, J, SHIFT) is equal to
   // IOR (SHIFTL(I, BIT_SIZE(I) - SHIFT), SHIFTR(J, SHIFT))
-  mlir::Value 
diff  = builder.create<mlir::arith::SubIOp>(loc, bitSize, shift);
+  mlir::Value 
diff  = mlir::arith::SubIOp::create(builder, loc, bitSize, shift);
 
   mlir::Value lArgs[2]{i, 
diff };
   mlir::Value lft = genShift<mlir::arith::ShLIOp>(signlessType, lArgs);
 
   mlir::Value rArgs[2]{j, shift};
   mlir::Value rgt = genShift<mlir::arith::ShRUIOp>(signlessType, rArgs);
-  mlir::Value result = builder.create<mlir::arith::OrIOp>(loc, lft, rgt);
+  mlir::Value result = mlir::arith::OrIOp::create(builder, loc, lft, rgt);
   if (resultType.isUnsignedInteger())
     return builder.createConvert(loc, resultType, result);
   return result;
@@ -3924,8 +3926,8 @@ IntrinsicLibrary::genEoshift(mlir::Type resultType,
   // Handle optional BOUNDARY argument
   mlir::Value boundary =
       isStaticallyAbsent(args[2])
-          ? builder.create<fir::AbsentOp>(
-                loc, fir::BoxType::get(builder.getNoneType()))
+          ? fir::AbsentOp::create(builder, loc,
+                                  fir::BoxType::get(builder.getNoneType()))
           : builder.createBox(loc, args[2]);
 
   if (arrayRank == 1) {
@@ -3933,7 +3935,7 @@ IntrinsicLibrary::genEoshift(mlir::Type resultType,
     // Handle required SHIFT argument as a scalar
     const mlir::Value *shiftAddr = args[1].getUnboxed();
     assert(shiftAddr && "nonscalar EOSHIFT SHIFT argument");
-    auto shift = builder.create<fir::LoadOp>(loc, *shiftAddr);
+    auto shift = fir::LoadOp::create(builder, loc, *shiftAddr);
     fir::runtime::genEoshiftVector(builder, loc, resultIrBox, array, shift,
                                    boundary);
   } else {
@@ -3981,14 +3983,15 @@ void IntrinsicLibrary::genExecuteCommandLine(
                    .genIfOp(loc, {i1Ty}, waitIsPresentAtRuntime,
                             /*withElseRegion=*/true)
                    .genThen([&]() {
-                     auto waitLoad = builder.create<fir::LoadOp>(loc, waitAddr);
+                     auto waitLoad =
+                         fir::LoadOp::create(builder, loc, waitAddr);
                      mlir::Value cast =
                          builder.createConvert(loc, i1Ty, waitLoad);
-                     builder.create<fir::ResultOp>(loc, cast);
+                     fir::ResultOp::create(builder, loc, cast);
                    })
                    .genElse([&]() {
                      mlir::Value trueVal = builder.createBool(loc, true);
-                     builder.create<fir::ResultOp>(loc, trueVal);
+                     fir::ResultOp::create(builder, loc, trueVal);
                    })
                    .getResults()[0];
   }
@@ -3996,15 +3999,15 @@ void IntrinsicLibrary::genExecuteCommandLine(
   mlir::Value exitstatBox =
       isStaticallyPresent(exitstat)
           ? fir::getBase(exitstat)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value cmdstatBox =
       isStaticallyPresent(cmdstat)
           ? fir::getBase(cmdstat)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value cmdmsgBox =
       isStaticallyPresent(cmdmsg)
           ? fir::getBase(cmdmsg)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   fir::runtime::genExecuteCommandLine(builder, loc, command, waitBool,
                                       exitstatBox, cmdstatBox, cmdmsgBox);
 }
@@ -4025,7 +4028,7 @@ IntrinsicLibrary::genEtime(std::optional<mlir::Type> resultType,
     auto timeAddr = builder.createTemporary(loc, *resultType);
     auto timeBox = builder.createBox(loc, timeAddr);
     fir::runtime::genEtime(builder, loc, values, timeBox);
-    return builder.create<fir::LoadOp>(loc, timeAddr);
+    return fir::LoadOp::create(builder, loc, timeAddr);
   } else {
     // subroutine form
     mlir::Value time = fir::getBase(args[1]);
@@ -4098,8 +4101,8 @@ IntrinsicLibrary::genFindloc(mlir::Type resultType,
 
   // Handle optional mask argument
   auto mask = isStaticallyAbsent(args[3])
-                  ? builder.create<fir::AbsentOp>(
-                        loc, fir::BoxType::get(builder.getI1Type()))
+                  ? fir::AbsentOp::create(
+                        builder, loc, fir::BoxType::get(builder.getI1Type()))
                   : builder.createBox(loc, args[3]);
 
   // Handle optional kind argument
@@ -4288,15 +4291,15 @@ void IntrinsicLibrary::genGetCommand(llvm::ArrayRef<fir::ExtendedValue> args) {
   mlir::Value commandBox =
       isStaticallyPresent(command)
           ? fir::getBase(command)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value lenBox =
       isStaticallyPresent(length)
           ? fir::getBase(length)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value errBox =
       isStaticallyPresent(errmsg)
           ? fir::getBase(errmsg)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value stat =
       fir::runtime::genGetCommand(builder, loc, commandBox, lenBox, errBox);
   if (isStaticallyPresent(status)) {
@@ -4355,15 +4358,15 @@ void IntrinsicLibrary::genGetCommandArgument(
   mlir::Value valBox =
       isStaticallyPresent(value)
           ? fir::getBase(value)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value lenBox =
       isStaticallyPresent(length)
           ? fir::getBase(length)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value errBox =
       isStaticallyPresent(errmsg)
           ? fir::getBase(errmsg)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value stat = fir::runtime::genGetCommandArgument(
       builder, loc, number, valBox, lenBox, errBox);
   if (isStaticallyPresent(status)) {
@@ -4408,13 +4411,14 @@ void IntrinsicLibrary::genGetEnvironmentVariable(
                .genIfOp(loc, {i1Ty}, trimNameIsPresentAtRuntime,
                         /*withElseRegion=*/true)
                .genThen([&]() {
-                 auto trimLoad = builder.create<fir::LoadOp>(loc, trimNameAddr);
+                 auto trimLoad =
+                     fir::LoadOp::create(builder, loc, trimNameAddr);
                  mlir::Value cast = builder.createConvert(loc, i1Ty, trimLoad);
-                 builder.create<fir::ResultOp>(loc, cast);
+                 fir::ResultOp::create(builder, loc, cast);
                })
                .genElse([&]() {
                  mlir::Value trueVal = builder.createBool(loc, true);
-                 builder.create<fir::ResultOp>(loc, trueVal);
+                 fir::ResultOp::create(builder, loc, trueVal);
                })
                .getResults()[0];
   }
@@ -4423,15 +4427,15 @@ void IntrinsicLibrary::genGetEnvironmentVariable(
   mlir::Value valBox =
       isStaticallyPresent(value)
           ? fir::getBase(value)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value lenBox =
       isStaticallyPresent(length)
           ? fir::getBase(length)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value errBox =
       isStaticallyPresent(errmsg)
           ? fir::getBase(errmsg)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   mlir::Value stat = fir::runtime::genGetEnvVariable(builder, loc, name, valBox,
                                                      lenBox, trim, errBox);
   if (isStaticallyPresent(status)) {
@@ -4516,8 +4520,8 @@ IntrinsicLibrary::genReduction(FN func, FD funcDim, llvm::StringRef errMsg,
 
   // Handle optional mask argument
   auto mask = isStaticallyAbsent(args[2])
-                  ? builder.create<fir::AbsentOp>(
-                        loc, fir::BoxType::get(builder.getI1Type()))
+                  ? fir::AbsentOp::create(
+                        builder, loc, fir::BoxType::get(builder.getI1Type()))
                   : builder.createBox(loc, args[2]);
 
   bool absentDim = isStaticallyAbsent(args[1]);
@@ -4531,10 +4535,10 @@ IntrinsicLibrary::genReduction(FN func, FD funcDim, llvm::StringRef errMsg,
     if (fir::isa_complex(eleTy)) {
       mlir::Value result = builder.createTemporary(loc, eleTy);
       func(builder, loc, array, mask, result);
-      return builder.create<fir::LoadOp>(loc, result);
+      return fir::LoadOp::create(builder, loc, result);
     }
-    auto resultBox = builder.create<fir::AbsentOp>(
-        loc, fir::BoxType::get(builder.getI1Type()));
+    auto resultBox = fir::AbsentOp::create(
+        builder, loc, fir::BoxType::get(builder.getI1Type()));
     return func(builder, loc, array, mask, resultBox);
   }
   // Handle Product/Sum cases that have an array result.
@@ -4581,8 +4585,8 @@ mlir::Value IntrinsicLibrary::genIbclr(mlir::Type resultType,
   mlir::Value one = builder.createIntegerConstant(loc, signlessType, 1);
   mlir::Value ones = builder.createAllOnesInteger(loc, signlessType);
   mlir::Value pos = builder.createConvert(loc, signlessType, args[1]);
-  mlir::Value bit = builder.create<mlir::arith::ShLIOp>(loc, one, pos);
-  mlir::Value mask = builder.create<mlir::arith::XOrIOp>(loc, ones, bit);
+  mlir::Value bit = mlir::arith::ShLIOp::create(builder, loc, one, pos);
+  mlir::Value mask = mlir::arith::XOrIOp::create(builder, loc, ones, bit);
   return builder.createUnsigned<mlir::arith::AndIOp>(loc, resultType, args[0],
                                                      mask);
 }
@@ -4610,18 +4614,18 @@ mlir::Value IntrinsicLibrary::genIbits(mlir::Type resultType,
   mlir::Value bitSize = builder.createIntegerConstant(
       loc, signlessType, mlir::cast<mlir::IntegerType>(resultType).getWidth());
   mlir::Value shiftCount =
-      builder.create<mlir::arith::SubIOp>(loc, bitSize, len);
+      mlir::arith::SubIOp::create(builder, loc, bitSize, len);
   mlir::Value zero = builder.createIntegerConstant(loc, signlessType, 0);
   mlir::Value ones = builder.createAllOnesInteger(loc, signlessType);
   mlir::Value mask =
-      builder.create<mlir::arith::ShRUIOp>(loc, ones, shiftCount);
+      mlir::arith::ShRUIOp::create(builder, loc, ones, shiftCount);
   mlir::Value res1 = builder.createUnsigned<mlir::arith::ShRSIOp>(
       loc, signlessType, word, pos);
-  mlir::Value res2 = builder.create<mlir::arith::AndIOp>(loc, res1, mask);
-  mlir::Value lenIsZero = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq, len, zero);
+  mlir::Value res2 = mlir::arith::AndIOp::create(builder, loc, res1, mask);
+  mlir::Value lenIsZero = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq, len, zero);
   mlir::Value result =
-      builder.create<mlir::arith::SelectOp>(loc, lenIsZero, zero, res2);
+      mlir::arith::SelectOp::create(builder, loc, lenIsZero, zero, res2);
   if (resultType.isUnsignedInteger())
     return builder.createConvert(loc, resultType, result);
   return result;
@@ -4640,7 +4644,7 @@ mlir::Value IntrinsicLibrary::genIbset(mlir::Type resultType,
       mlir::IntegerType::SignednessSemantics::Signless);
   mlir::Value one = builder.createIntegerConstant(loc, signlessType, 1);
   mlir::Value pos = builder.createConvert(loc, signlessType, args[1]);
-  mlir::Value mask = builder.create<mlir::arith::ShLIOp>(loc, one, pos);
+  mlir::Value mask = mlir::arith::ShLIOp::create(builder, loc, one, pos);
   return builder.createUnsigned<mlir::arith::OrIOp>(loc, resultType, args[0],
                                                     mask);
 }
@@ -4674,13 +4678,13 @@ IntrinsicLibrary::genIchar(mlir::Type resultType,
         fir::CharacterType::get(builder.getContext(), eleType.getFKind(), 1);
     mlir::Type toTy = builder.getRefType(charType);
     mlir::Value cast = builder.createConvert(loc, toTy, buffer);
-    charVal = builder.create<fir::LoadOp>(loc, cast);
+    charVal = fir::LoadOp::create(builder, loc, cast);
   }
   LLVM_DEBUG(llvm::dbgs() << "ichar(" << charVal << ")\n");
   auto code = helper.extractCodeFromSingleton(charVal);
   if (code.getType() == resultType)
     return code;
-  return builder.create<mlir::arith::ExtUIOp>(loc, resultType, code);
+  return mlir::arith::ExtUIOp::create(builder, loc, resultType, code);
 }
 
 // llvm floating point class intrinsic test values
@@ -4710,7 +4714,7 @@ mlir::Value IntrinsicLibrary::genIsFPClass(mlir::Type resultType,
   assert(args.size() == 1);
   mlir::Type i1Ty = builder.getI1Type();
   mlir::Value isfpclass =
-      builder.create<mlir::LLVM::IsFPClass>(loc, i1Ty, args[0], fpclass);
+      mlir::LLVM::IsFPClass::create(builder, loc, i1Ty, args[0], fpclass);
   return builder.createConvert(loc, resultType, isfpclass);
 }
 
@@ -4725,7 +4729,7 @@ mlir::Value IntrinsicLibrary::genQNan(mlir::Type resultType) {
 void IntrinsicLibrary::genRaiseExcept(int excepts, mlir::Value cond) {
   fir::IfOp ifOp;
   if (cond) {
-    ifOp = builder.create<fir::IfOp>(loc, cond, /*withElseRegion=*/false);
+    ifOp = fir::IfOp::create(builder, loc, cond, /*withElseRegion=*/false);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   }
   mlir::Type i32Ty = builder.getIntegerType(32);
@@ -4746,11 +4750,11 @@ getFieldRef(fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value rec,
       mlir::dyn_cast<fir::RecordType>(fir::unwrapPassByRefType(rec.getType()));
   assert(index < recType.getTypeList().size() && "not enough components");
   auto [fieldName, fieldTy] = recType.getTypeList()[index];
-  mlir::Value field = builder.create<fir::FieldIndexOp>(
-      loc, fir::FieldType::get(recType.getContext()), fieldName, recType,
-      fir::getTypeParams(rec));
-  return {builder.create<fir::CoordinateOp>(loc, builder.getRefType(fieldTy),
-                                            rec, field),
+  mlir::Value field = fir::FieldIndexOp::create(
+      builder, loc, fir::FieldType::get(recType.getContext()), fieldName,
+      recType, fir::getTypeParams(rec));
+  return {fir::CoordinateOp::create(builder, loc, builder.getRefType(fieldTy),
+                                    rec, field),
           fieldTy};
 }
 
@@ -4763,9 +4767,9 @@ IntrinsicLibrary::genIeeeTypeCompare(mlir::Type resultType,
   assert(args.size() == 2);
   auto [leftRef, fieldTy] = getFieldRef(builder, loc, args[0]);
   auto [rightRef, ignore] = getFieldRef(builder, loc, args[1]);
-  mlir::Value left = builder.create<fir::LoadOp>(loc, fieldTy, leftRef);
-  mlir::Value right = builder.create<fir::LoadOp>(loc, fieldTy, rightRef);
-  return builder.create<mlir::arith::CmpIOp>(loc, pred, left, right);
+  mlir::Value left = fir::LoadOp::create(builder, loc, fieldTy, leftRef);
+  mlir::Value right = fir::LoadOp::create(builder, loc, fieldTy, rightRef);
+  return mlir::arith::CmpIOp::create(builder, loc, pred, left, right);
 }
 
 // IEEE_CLASS
@@ -4796,7 +4800,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType,
   const unsigned intWidth = realType.getWidth();
   mlir::Type intType = builder.getIntegerType(intWidth);
   mlir::Value intVal =
-      builder.create<mlir::arith::BitcastOp>(loc, intType, realVal);
+      mlir::arith::BitcastOp::create(builder, loc, intType, realVal);
   llvm::StringRef tableName = RTNAME_STRING(IeeeClassTable);
   uint64_t highSignificandSize = (realType.getWidth() == 80) + 1;
 
@@ -4806,8 +4810,8 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType,
     return builder.createIntegerConstant(loc, intType, k);
   };
   auto createIntegerConstantAPI = [&](const llvm::APInt &apInt) {
-    return builder.create<mlir::arith::ConstantOp>(
-        loc, intType, builder.getIntegerAttr(intType, apInt));
+    return mlir::arith::ConstantOp::create(
+        builder, loc, intType, builder.getIntegerAttr(intType, apInt));
   };
   auto getMasksAndShifts = [&](uint64_t totalSize, uint64_t exponentSize,
                                uint64_t significandSize,
@@ -4854,50 +4858,52 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType,
 
   // [s] sign bit
   int pos = 3 + highSignificandSize;
-  mlir::Value index = builder.create<mlir::arith::AndIOp>(
-      loc, builder.create<mlir::arith::ShRUIOp>(loc, intVal, signShift),
+  mlir::Value index = mlir::arith::AndIOp::create(
+      builder, loc,
+      mlir::arith::ShRUIOp::create(builder, loc, intVal, signShift),
       createIntegerConstant(1ULL << pos));
 
   // [e] exponent != 0
   mlir::Value exponent =
-      builder.create<mlir::arith::AndIOp>(loc, intVal, exponentMask);
+      mlir::arith::AndIOp::create(builder, loc, intVal, exponentMask);
   mlir::Value zero = createIntegerConstant(0);
-  index = builder.create<mlir::arith::OrIOp>(
-      loc, index,
-      builder.create<mlir::arith::SelectOp>(
-          loc,
-          builder.create<mlir::arith::CmpIOp>(
-              loc, mlir::arith::CmpIPredicate::ne, exponent, zero),
+  index = mlir::arith::OrIOp::create(
+      builder, loc, index,
+      mlir::arith::SelectOp::create(
+          builder, loc,
+          mlir::arith::CmpIOp::create(
+              builder, loc, mlir::arith::CmpIPredicate::ne, exponent, zero),
           createIntegerConstant(1ULL << --pos), zero));
 
   // [m] exponent == 1..1 (max exponent)
-  index = builder.create<mlir::arith::OrIOp>(
-      loc, index,
-      builder.create<mlir::arith::SelectOp>(
-          loc,
-          builder.create<mlir::arith::CmpIOp>(
-              loc, mlir::arith::CmpIPredicate::eq, exponent, exponentMask),
+  index = mlir::arith::OrIOp::create(
+      builder, loc, index,
+      mlir::arith::SelectOp::create(
+          builder, loc,
+          mlir::arith::CmpIOp::create(builder, loc,
+                                      mlir::arith::CmpIPredicate::eq, exponent,
+                                      exponentMask),
           createIntegerConstant(1ULL << --pos), zero));
 
   // [l] low-order significand != 0
-  index = builder.create<mlir::arith::OrIOp>(
-      loc, index,
-      builder.create<mlir::arith::SelectOp>(
-          loc,
-          builder.create<mlir::arith::CmpIOp>(
-              loc, mlir::arith::CmpIPredicate::ne,
-              builder.create<mlir::arith::AndIOp>(loc, intVal,
-                                                  lowSignificandMask),
+  index = mlir::arith::OrIOp::create(
+      builder, loc, index,
+      mlir::arith::SelectOp::create(
+          builder, loc,
+          mlir::arith::CmpIOp::create(
+              builder, loc, mlir::arith::CmpIPredicate::ne,
+              mlir::arith::AndIOp::create(builder, loc, intVal,
+                                          lowSignificandMask),
               zero),
           createIntegerConstant(1ULL << --pos), zero));
 
   // [h] high-order significand (1 or 2 bits)
-  index = builder.create<mlir::arith::OrIOp>(
-      loc, index,
-      builder.create<mlir::arith::AndIOp>(
-          loc,
-          builder.create<mlir::arith::ShRUIOp>(loc, intVal,
-                                               highSignificandShift),
+  index = mlir::arith::OrIOp::create(
+      builder, loc, index,
+      mlir::arith::AndIOp::create(
+          builder, loc,
+          mlir::arith::ShRUIOp::create(builder, loc, intVal,
+                                       highSignificandShift),
           createIntegerConstant((1 << highSignificandSize) - 1)));
 
   int tableSize = 1 << (4 + highSignificandSize);
@@ -5025,10 +5031,10 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType,
             mlir::RankedTensorType::get(tableSize, int8Ty), values));
   }
 
-  return builder.create<fir::CoordinateOp>(
-      loc, builder.getRefType(resultType),
-      builder.create<fir::AddrOfOp>(loc, builder.getRefType(tableTy),
-                                    builder.getSymbolRefAttr(tableName)),
+  return fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(resultType),
+      fir::AddrOfOp::create(builder, loc, builder.getRefType(tableTy),
+                            builder.getSymbolRefAttr(tableName)),
       index);
 }
 
@@ -5055,34 +5061,36 @@ IntrinsicLibrary::genIeeeCopySign(mlir::Type resultType,
 
   // Args have the same type.
   if (xRealType == yRealType)
-    return builder.create<mlir::math::CopySignOp>(loc, xRealVal, yRealVal);
+    return mlir::math::CopySignOp::create(builder, loc, xRealVal, yRealVal);
 
   // Args have 
diff erent types.
   mlir::Type xIntType = builder.getIntegerType(xRealType.getWidth());
   mlir::Type yIntType = builder.getIntegerType(yRealType.getWidth());
   mlir::Value xIntVal =
-      builder.create<mlir::arith::BitcastOp>(loc, xIntType, xRealVal);
+      mlir::arith::BitcastOp::create(builder, loc, xIntType, xRealVal);
   mlir::Value yIntVal =
-      builder.create<mlir::arith::BitcastOp>(loc, yIntType, yRealVal);
+      mlir::arith::BitcastOp::create(builder, loc, yIntType, yRealVal);
   mlir::Value xZero = builder.createIntegerConstant(loc, xIntType, 0);
   mlir::Value yZero = builder.createIntegerConstant(loc, yIntType, 0);
   mlir::Value xOne = builder.createIntegerConstant(loc, xIntType, 1);
-  mlir::Value ySign = builder.create<mlir::arith::ShRUIOp>(
-      loc, yIntVal,
+  mlir::Value ySign = mlir::arith::ShRUIOp::create(
+      builder, loc, yIntVal,
       builder.createIntegerConstant(loc, yIntType, yRealType.getWidth() - 1));
-  mlir::Value xAbs = builder.create<mlir::arith::ShRUIOp>(
-      loc, builder.create<mlir::arith::ShLIOp>(loc, xIntVal, xOne), xOne);
-  mlir::Value xSign = builder.create<mlir::arith::SelectOp>(
-      loc,
-      builder.create<mlir::arith::CmpIOp>(loc, mlir::arith::CmpIPredicate::eq,
-                                          ySign, yZero),
+  mlir::Value xAbs = mlir::arith::ShRUIOp::create(
+      builder, loc, mlir::arith::ShLIOp::create(builder, loc, xIntVal, xOne),
+      xOne);
+  mlir::Value xSign = mlir::arith::SelectOp::create(
+      builder, loc,
+      mlir::arith::CmpIOp::create(builder, loc, mlir::arith::CmpIPredicate::eq,
+                                  ySign, yZero),
       xZero,
-      builder.create<mlir::arith::ShLIOp>(
-          loc, xOne,
+      mlir::arith::ShLIOp::create(
+          builder, loc, xOne,
           builder.createIntegerConstant(loc, xIntType,
                                         xRealType.getWidth() - 1)));
-  return builder.create<mlir::arith::BitcastOp>(
-      loc, xRealType, builder.create<mlir::arith::OrIOp>(loc, xAbs, xSign));
+  return mlir::arith::BitcastOp::create(
+      builder, loc, xRealType,
+      mlir::arith::OrIOp::create(builder, loc, xAbs, xSign));
 }
 
 // IEEE_GET_FLAG
@@ -5096,16 +5104,16 @@ void IntrinsicLibrary::genIeeeGetFlag(llvm::ArrayRef<fir::ExtendedValue> args) {
   mlir::Type i32Ty = builder.getIntegerType(32);
   mlir::Value zero = builder.createIntegerConstant(loc, i32Ty, 0);
   auto [fieldRef, ignore] = getFieldRef(builder, loc, flag);
-  mlir::Value field = builder.create<fir::LoadOp>(loc, fieldRef);
+  mlir::Value field = fir::LoadOp::create(builder, loc, fieldRef);
   mlir::Value excepts = fir::runtime::genFetestexcept(
       builder, loc,
       fir::runtime::genMapExcept(
-          builder, loc, builder.create<fir::ConvertOp>(loc, i32Ty, field)));
-  mlir::Value logicalResult = builder.create<fir::ConvertOp>(
-      loc, resultTy,
-      builder.create<mlir::arith::CmpIOp>(loc, mlir::arith::CmpIPredicate::ne,
-                                          excepts, zero));
-  builder.create<fir::StoreOp>(loc, logicalResult, flagValue);
+          builder, loc, fir::ConvertOp::create(builder, loc, i32Ty, field)));
+  mlir::Value logicalResult = fir::ConvertOp::create(
+      builder, loc, resultTy,
+      mlir::arith::CmpIOp::create(builder, loc, mlir::arith::CmpIPredicate::ne,
+                                  excepts, zero));
+  fir::StoreOp::create(builder, loc, logicalResult, flagValue);
 }
 
 // IEEE_GET_HALTING_MODE
@@ -5120,17 +5128,17 @@ void IntrinsicLibrary::genIeeeGetHaltingMode(
   mlir::Type i32Ty = builder.getIntegerType(32);
   mlir::Value zero = builder.createIntegerConstant(loc, i32Ty, 0);
   auto [fieldRef, ignore] = getFieldRef(builder, loc, flag);
-  mlir::Value field = builder.create<fir::LoadOp>(loc, fieldRef);
+  mlir::Value field = fir::LoadOp::create(builder, loc, fieldRef);
   mlir::Value haltSet = fir::runtime::genFegetexcept(builder, loc);
-  mlir::Value intResult = builder.create<mlir::arith::AndIOp>(
-      loc, haltSet,
+  mlir::Value intResult = mlir::arith::AndIOp::create(
+      builder, loc, haltSet,
       fir::runtime::genMapExcept(
-          builder, loc, builder.create<fir::ConvertOp>(loc, i32Ty, field)));
-  mlir::Value logicalResult = builder.create<fir::ConvertOp>(
-      loc, resultTy,
-      builder.create<mlir::arith::CmpIOp>(loc, mlir::arith::CmpIPredicate::ne,
-                                          intResult, zero));
-  builder.create<fir::StoreOp>(loc, logicalResult, halting);
+          builder, loc, fir::ConvertOp::create(builder, loc, i32Ty, field)));
+  mlir::Value logicalResult = fir::ConvertOp::create(
+      builder, loc, resultTy,
+      mlir::arith::CmpIOp::create(builder, loc, mlir::arith::CmpIPredicate::ne,
+                                  intResult, zero));
+  fir::StoreOp::create(builder, loc, logicalResult, halting);
 }
 
 // IEEE_GET_MODES, IEEE_SET_MODES
@@ -5155,34 +5163,34 @@ void IntrinsicLibrary::genIeeeGetOrSetModesOrStatus(
     // allotment. Allocate data space from the heap.
     auto [fieldRef, fieldTy] =
         getFieldRef(builder, loc, fir::getBase(args[0]), 1);
-    addr = builder.create<fir::BoxAddrOp>(
-        loc, builder.create<fir::LoadOp>(loc, fieldRef));
+    addr = fir::BoxAddrOp::create(builder, loc,
+                                  fir::LoadOp::create(builder, loc, fieldRef));
     mlir::Type heapTy = addr.getType();
-    mlir::Value allocated = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::ne,
+    mlir::Value allocated = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::ne,
         builder.createConvert(loc, i64Ty, addr),
         builder.createIntegerConstant(loc, i64Ty, 0));
-    auto ifOp = builder.create<fir::IfOp>(loc, heapTy, allocated,
-                                          /*withElseRegion=*/true);
+    auto ifOp = fir::IfOp::create(builder, loc, heapTy, allocated,
+                                  /*withElseRegion=*/true);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-    builder.create<fir::ResultOp>(loc, addr);
+    fir::ResultOp::create(builder, loc, addr);
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
     mlir::Value byteSize =
         isModes ? fir::runtime::genGetModesTypeSize(builder, loc)
                 : fir::runtime::genGetStatusTypeSize(builder, loc);
     byteSize = builder.createConvert(loc, builder.getIndexType(), byteSize);
-    addr = builder.create<fir::AllocMemOp>(loc, extractSequenceType(heapTy),
-                                           /*typeparams=*/mlir::ValueRange(),
-                                           byteSize);
-    mlir::Value shape = builder.create<fir::ShapeOp>(loc, byteSize);
-    builder.create<fir::StoreOp>(
-        loc, builder.create<fir::EmboxOp>(loc, fieldTy, addr, shape), fieldRef);
-    builder.create<fir::ResultOp>(loc, addr);
+    addr = fir::AllocMemOp::create(builder, loc, extractSequenceType(heapTy),
+                                   /*typeparams=*/mlir::ValueRange(), byteSize);
+    mlir::Value shape = fir::ShapeOp::create(builder, loc, byteSize);
+    fir::StoreOp::create(
+        builder, loc, fir::EmboxOp::create(builder, loc, fieldTy, addr, shape),
+        fieldRef);
+    fir::ResultOp::create(builder, loc, addr);
     builder.setInsertionPointAfter(ifOp);
-    addr = builder.create<fir::ConvertOp>(loc, ptrTy, ifOp.getResult(0));
+    addr = fir::ConvertOp::create(builder, loc, ptrTy, ifOp.getResult(0));
   } else {
     // Place floating point environment data in __data storage.
-    addr = builder.create<fir::ConvertOp>(loc, ptrTy, getBase(args[0]));
+    addr = fir::ConvertOp::create(builder, loc, ptrTy, getBase(args[0]));
   }
   llvm::StringRef func = isModes ? (isGet ? "fegetmode" : "fesetmode")
                                  : (isGet ? "fegetenv" : "fesetenv");
@@ -5193,11 +5201,11 @@ void IntrinsicLibrary::genIeeeGetOrSetModesOrStatus(
 // Check that an explicit ieee_[get|set]_rounding_mode call radix value is 2.
 static void checkRadix(fir::FirOpBuilder &builder, mlir::Location loc,
                        mlir::Value radix, std::string procName) {
-  mlir::Value notTwo = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::ne, radix,
+  mlir::Value notTwo = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::ne, radix,
       builder.createIntegerConstant(loc, radix.getType(), 2));
-  auto ifOp = builder.create<fir::IfOp>(loc, notTwo,
-                                        /*withElseRegion=*/false);
+  auto ifOp = fir::IfOp::create(builder, loc, notTwo,
+                                /*withElseRegion=*/false);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   fir::runtime::genReportFatalUserError(builder, loc,
                                         procName + " radix argument must be 2");
@@ -5215,9 +5223,9 @@ void IntrinsicLibrary::genIeeeGetRoundingMode(
     checkRadix(builder, loc, fir::getBase(args[1]), "ieee_get_rounding_mode");
   auto [fieldRef, fieldTy] = getFieldRef(builder, loc, fir::getBase(args[0]));
   mlir::func::FuncOp getRound = fir::factory::getLlvmGetRounding(builder);
-  mlir::Value mode = builder.create<fir::CallOp>(loc, getRound).getResult(0);
+  mlir::Value mode = fir::CallOp::create(builder, loc, getRound).getResult(0);
   mode = builder.createConvert(loc, fieldTy, mode);
-  builder.create<fir::StoreOp>(loc, mode, fieldRef);
+  fir::StoreOp::create(builder, loc, mode, fieldRef);
 }
 
 // IEEE_GET_UNDERFLOW_MODE
@@ -5242,44 +5250,45 @@ mlir::Value IntrinsicLibrary::genIeeeInt(mlir::Type resultType,
   mlir::FloatType realType = mlir::cast<mlir::FloatType>(args[0].getType());
   mlir::Value realResult = genIeeeRint(realType, {args[0], args[1]});
   int intWidth = mlir::cast<mlir::IntegerType>(resultType).getWidth();
-  mlir::Value intLBound = builder.create<mlir::arith::ConstantOp>(
-      loc, resultType,
+  mlir::Value intLBound = mlir::arith::ConstantOp::create(
+      builder, loc, resultType,
       builder.getIntegerAttr(resultType,
                              llvm::APInt::getBitsSet(intWidth,
                                                      /*lo=*/intWidth - 1,
                                                      /*hi=*/intWidth)));
-  mlir::Value intUBound = builder.create<mlir::arith::ConstantOp>(
-      loc, resultType,
+  mlir::Value intUBound = mlir::arith::ConstantOp::create(
+      builder, loc, resultType,
       builder.getIntegerAttr(resultType,
                              llvm::APInt::getBitsSet(intWidth, /*lo=*/0,
                                                      /*hi=*/intWidth - 1)));
   mlir::Value realLBound =
-      builder.create<fir::ConvertOp>(loc, realType, intLBound);
-  mlir::Value realUBound = builder.create<mlir::arith::NegFOp>(loc, realLBound);
-  mlir::Value aGreaterThanLBound = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::OGE, realResult, realLBound);
-  mlir::Value aLessThanUBound = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::OLT, realResult, realUBound);
-  mlir::Value resultIsValid = builder.create<mlir::arith::AndIOp>(
-      loc, aGreaterThanLBound, aLessThanUBound);
+      fir::ConvertOp::create(builder, loc, realType, intLBound);
+  mlir::Value realUBound =
+      mlir::arith::NegFOp::create(builder, loc, realLBound);
+  mlir::Value aGreaterThanLBound = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::OGE, realResult, realLBound);
+  mlir::Value aLessThanUBound = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::OLT, realResult, realUBound);
+  mlir::Value resultIsValid = mlir::arith::AndIOp::create(
+      builder, loc, aGreaterThanLBound, aLessThanUBound);
 
   // Result is valid. It may be exact or inexact.
   mlir::Value result;
-  fir::IfOp ifOp = builder.create<fir::IfOp>(loc, resultType, resultIsValid,
-                                             /*withElseRegion=*/true);
+  fir::IfOp ifOp = fir::IfOp::create(builder, loc, resultType, resultIsValid,
+                                     /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  mlir::Value inexact = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::ONE, args[0], realResult);
+  mlir::Value inexact = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::ONE, args[0], realResult);
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INEXACT, inexact);
-  result = builder.create<fir::ConvertOp>(loc, resultType, realResult);
-  builder.create<fir::ResultOp>(loc, result);
+  result = fir::ConvertOp::create(builder, loc, resultType, realResult);
+  fir::ResultOp::create(builder, loc, result);
 
   // Result is invalid.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INVALID);
-  result = builder.create<mlir::arith::SelectOp>(loc, aGreaterThanLBound,
-                                                 intUBound, intLBound);
-  builder.create<fir::ResultOp>(loc, result);
+  result = mlir::arith::SelectOp::create(builder, loc, aGreaterThanLBound,
+                                         intUBound, intLBound);
+  fir::ResultOp::create(builder, loc, result);
   builder.setInsertionPointAfter(ifOp);
   return ifOp.getResult(0);
 }
@@ -5334,7 +5343,7 @@ mlir::Value IntrinsicLibrary::genIeeeLogb(mlir::Type resultType,
   int bitWidth = realType.getWidth();
   mlir::Type intType = builder.getIntegerType(realType.getWidth());
   mlir::Value intVal =
-      builder.create<mlir::arith::BitcastOp>(loc, intType, realVal);
+      mlir::arith::BitcastOp::create(builder, loc, intType, realVal);
   mlir::Type i1Ty = builder.getI1Type();
 
   int exponentBias, significandSize, nonSignificandSize;
@@ -5381,72 +5390,72 @@ mlir::Value IntrinsicLibrary::genIeeeLogb(mlir::Type resultType,
     llvm_unreachable("unknown real type");
   }
 
-  mlir::Value isZero = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::OEQ, realVal,
+  mlir::Value isZero = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::OEQ, realVal,
       builder.createRealZeroConstant(loc, resultType));
-  auto outerIfOp = builder.create<fir::IfOp>(loc, resultType, isZero,
-                                             /*withElseRegion=*/true);
+  auto outerIfOp = fir::IfOp::create(builder, loc, resultType, isZero,
+                                     /*withElseRegion=*/true);
   // X is zero -- result is -infinity
   builder.setInsertionPointToStart(&outerIfOp.getThenRegion().front());
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_DIVIDE_BY_ZERO);
   mlir::Value ones = builder.createAllOnesInteger(loc, intType);
-  mlir::Value result = builder.create<mlir::arith::ShLIOp>(
-      loc, ones,
+  mlir::Value result = mlir::arith::ShLIOp::create(
+      builder, loc, ones,
       builder.createIntegerConstant(loc, intType,
                                     // kind=10 high-order bit is explicit
                                     significandSize - (bitWidth == 80)));
-  result = builder.create<mlir::arith::BitcastOp>(loc, resultType, result);
-  builder.create<fir::ResultOp>(loc, result);
+  result = mlir::arith::BitcastOp::create(builder, loc, resultType, result);
+  fir::ResultOp::create(builder, loc, result);
 
   builder.setInsertionPointToStart(&outerIfOp.getElseRegion().front());
   mlir::Value one = builder.createIntegerConstant(loc, intType, 1);
   mlir::Value shiftLeftOne =
-      builder.create<mlir::arith::ShLIOp>(loc, intVal, one);
+      mlir::arith::ShLIOp::create(builder, loc, intVal, one);
   mlir::Value isFinite = genIsFPClass(i1Ty, args, finiteTest);
-  auto innerIfOp = builder.create<fir::IfOp>(loc, resultType, isFinite,
-                                             /*withElseRegion=*/true);
+  auto innerIfOp = fir::IfOp::create(builder, loc, resultType, isFinite,
+                                     /*withElseRegion=*/true);
   // X is non-zero finite -- result is unbiased exponent of X
   builder.setInsertionPointToStart(&innerIfOp.getThenRegion().front());
   mlir::Value isNormal = genIsFPClass(i1Ty, args, normalTest);
-  auto normalIfOp = builder.create<fir::IfOp>(loc, resultType, isNormal,
-                                              /*withElseRegion=*/true);
+  auto normalIfOp = fir::IfOp::create(builder, loc, resultType, isNormal,
+                                      /*withElseRegion=*/true);
   // X is normal
   builder.setInsertionPointToStart(&normalIfOp.getThenRegion().front());
-  mlir::Value biasedExponent = builder.create<mlir::arith::ShRUIOp>(
-      loc, shiftLeftOne,
+  mlir::Value biasedExponent = mlir::arith::ShRUIOp::create(
+      builder, loc, shiftLeftOne,
       builder.createIntegerConstant(loc, intType, significandSize + 1));
-  result = builder.create<mlir::arith::SubIOp>(
-      loc, biasedExponent,
+  result = mlir::arith::SubIOp::create(
+      builder, loc, biasedExponent,
       builder.createIntegerConstant(loc, intType, exponentBias));
-  result = builder.create<fir::ConvertOp>(loc, resultType, result);
-  builder.create<fir::ResultOp>(loc, result);
+  result = fir::ConvertOp::create(builder, loc, resultType, result);
+  fir::ResultOp::create(builder, loc, result);
 
   // X is denormal -- result is (-exponentBias - ctlz(significand))
   builder.setInsertionPointToStart(&normalIfOp.getElseRegion().front());
-  mlir::Value significand = builder.create<mlir::arith::ShLIOp>(
-      loc, intVal,
+  mlir::Value significand = mlir::arith::ShLIOp::create(
+      builder, loc, intVal,
       builder.createIntegerConstant(loc, intType, nonSignificandSize));
   mlir::Value ctlz =
-      builder.create<mlir::math::CountLeadingZerosOp>(loc, significand);
+      mlir::math::CountLeadingZerosOp::create(builder, loc, significand);
   mlir::Type i32Ty = builder.getI32Type();
-  result = builder.create<mlir::arith::SubIOp>(
-      loc, builder.createIntegerConstant(loc, i32Ty, -exponentBias),
-      builder.create<fir::ConvertOp>(loc, i32Ty, ctlz));
-  result = builder.create<fir::ConvertOp>(loc, resultType, result);
-  builder.create<fir::ResultOp>(loc, result);
+  result = mlir::arith::SubIOp::create(
+      builder, loc, builder.createIntegerConstant(loc, i32Ty, -exponentBias),
+      fir::ConvertOp::create(builder, loc, i32Ty, ctlz));
+  result = fir::ConvertOp::create(builder, loc, resultType, result);
+  fir::ResultOp::create(builder, loc, result);
 
   builder.setInsertionPointToEnd(&innerIfOp.getThenRegion().front());
-  builder.create<fir::ResultOp>(loc, normalIfOp.getResult(0));
+  fir::ResultOp::create(builder, loc, normalIfOp.getResult(0));
 
   // X is infinity or NaN -- result is +infinity or NaN
   builder.setInsertionPointToStart(&innerIfOp.getElseRegion().front());
-  result = builder.create<mlir::arith::ShRUIOp>(loc, shiftLeftOne, one);
-  result = builder.create<mlir::arith::BitcastOp>(loc, resultType, result);
-  builder.create<fir::ResultOp>(loc, result);
+  result = mlir::arith::ShRUIOp::create(builder, loc, shiftLeftOne, one);
+  result = mlir::arith::BitcastOp::create(builder, loc, resultType, result);
+  fir::ResultOp::create(builder, loc, result);
 
   // Unwind the if nest.
   builder.setInsertionPointToEnd(&outerIfOp.getElseRegion().front());
-  builder.create<fir::ResultOp>(loc, innerIfOp.getResult(0));
+  fir::ResultOp::create(builder, loc, innerIfOp.getResult(0));
   builder.setInsertionPointAfter(outerIfOp);
   return outerIfOp.getResult(0);
 }
@@ -5480,8 +5489,8 @@ mlir::Value IntrinsicLibrary::genIeeeMaxMin(mlir::Type resultType,
   mlir::Value x1, y1; // X or ABS(X), Y or ABS(Y)
   if constexpr (isMag) {
     mlir::Value zero = builder.createRealZeroConstant(loc, resultType);
-    x1 = builder.create<mlir::math::CopySignOp>(loc, x, zero);
-    y1 = builder.create<mlir::math::CopySignOp>(loc, y, zero);
+    x1 = mlir::math::CopySignOp::create(builder, loc, x, zero);
+    y1 = mlir::math::CopySignOp::create(builder, loc, y, zero);
   } else {
     x1 = x;
     y1 = y;
@@ -5492,56 +5501,56 @@ mlir::Value IntrinsicLibrary::genIeeeMaxMin(mlir::Type resultType,
 
   // X1 < Y1 -- MAX result is Y; MIN result is X.
   pred = mlir::arith::CmpFPredicate::OLT;
-  cmp = builder.create<mlir::arith::CmpFOp>(loc, pred, x1, y1);
-  auto ifOp1 = builder.create<fir::IfOp>(loc, resultType, cmp, true);
+  cmp = mlir::arith::CmpFOp::create(builder, loc, pred, x1, y1);
+  auto ifOp1 = fir::IfOp::create(builder, loc, resultType, cmp, true);
   builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
   result = isMax ? y : x;
-  builder.create<fir::ResultOp>(loc, result);
+  fir::ResultOp::create(builder, loc, result);
 
   // X1 > Y1 -- MAX result is X; MIN result is Y.
   builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
   pred = mlir::arith::CmpFPredicate::OGT;
-  cmp = builder.create<mlir::arith::CmpFOp>(loc, pred, x1, y1);
-  auto ifOp2 = builder.create<fir::IfOp>(loc, resultType, cmp, true);
+  cmp = mlir::arith::CmpFOp::create(builder, loc, pred, x1, y1);
+  auto ifOp2 = fir::IfOp::create(builder, loc, resultType, cmp, true);
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
   result = isMax ? x : y;
-  builder.create<fir::ResultOp>(loc, result);
+  fir::ResultOp::create(builder, loc, result);
 
   // X1 == Y1 -- MAX favors a positive result; MIN favors a negative result.
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
   pred = mlir::arith::CmpFPredicate::OEQ;
-  cmp = builder.create<mlir::arith::CmpFOp>(loc, pred, x1, y1);
-  auto ifOp3 = builder.create<fir::IfOp>(loc, resultType, cmp, true);
+  cmp = mlir::arith::CmpFOp::create(builder, loc, pred, x1, y1);
+  auto ifOp3 = fir::IfOp::create(builder, loc, resultType, cmp, true);
   builder.setInsertionPointToStart(&ifOp3.getThenRegion().front());
   resultIsX = isMax ? genIsFPClass(i1Ty, x, positiveTest)
                     : genIsFPClass(i1Ty, x, negativeTest);
-  result = builder.create<mlir::arith::SelectOp>(loc, resultIsX, x, y);
-  builder.create<fir::ResultOp>(loc, result);
+  result = mlir::arith::SelectOp::create(builder, loc, resultIsX, x, y);
+  fir::ResultOp::create(builder, loc, result);
 
   // X or Y or both are NaNs -- result may be X, Y, or a qNaN
   builder.setInsertionPointToStart(&ifOp3.getElseRegion().front());
   if constexpr (isNum) {
     pred = mlir::arith::CmpFPredicate::ORD; // check for a non-NaN
-    resultIsX = builder.create<mlir::arith::CmpFOp>(loc, pred, x, x);
-    resultIsY = builder.create<mlir::arith::CmpFOp>(loc, pred, y, y);
+    resultIsX = mlir::arith::CmpFOp::create(builder, loc, pred, x, x);
+    resultIsY = mlir::arith::CmpFOp::create(builder, loc, pred, y, y);
   } else {
     resultIsX = resultIsY = builder.createBool(loc, false);
   }
-  result = builder.create<mlir::arith::SelectOp>(
-      loc, resultIsX, x,
-      builder.create<mlir::arith::SelectOp>(loc, resultIsY, y,
-                                            genQNan(resultType)));
-  mlir::Value hasSNaNOp = builder.create<mlir::arith::OrIOp>(
-      loc, genIsFPClass(builder.getI1Type(), args[0], snanTest),
+  result = mlir::arith::SelectOp::create(
+      builder, loc, resultIsX, x,
+      mlir::arith::SelectOp::create(builder, loc, resultIsY, y,
+                                    genQNan(resultType)));
+  mlir::Value hasSNaNOp = mlir::arith::OrIOp::create(
+      builder, loc, genIsFPClass(builder.getI1Type(), args[0], snanTest),
       genIsFPClass(builder.getI1Type(), args[1], snanTest));
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INVALID, hasSNaNOp);
-  builder.create<fir::ResultOp>(loc, result);
+  fir::ResultOp::create(builder, loc, result);
 
   // Unwind the if nest.
   builder.setInsertionPointAfter(ifOp3);
-  builder.create<fir::ResultOp>(loc, ifOp3.getResult(0));
+  fir::ResultOp::create(builder, loc, ifOp3.getResult(0));
   builder.setInsertionPointAfter(ifOp2);
-  builder.create<fir::ResultOp>(loc, ifOp2.getResult(0));
+  fir::ResultOp::create(builder, loc, ifOp2.getResult(0));
   builder.setInsertionPointAfter(ifOp1);
   return ifOp1.getResult(0);
 }
@@ -5554,13 +5563,13 @@ IntrinsicLibrary::genIeeeQuietCompare(mlir::Type resultType,
                                       llvm::ArrayRef<mlir::Value> args) {
   // Compare X and Y with special case treatment of NaN operands.
   assert(args.size() == 2);
-  mlir::Value hasSNaNOp = builder.create<mlir::arith::OrIOp>(
-      loc, genIsFPClass(builder.getI1Type(), args[0], snanTest),
+  mlir::Value hasSNaNOp = mlir::arith::OrIOp::create(
+      builder, loc, genIsFPClass(builder.getI1Type(), args[0], snanTest),
       genIsFPClass(builder.getI1Type(), args[1], snanTest));
   mlir::Value res =
-      builder.create<mlir::arith::CmpFOp>(loc, pred, args[0], args[1]);
+      mlir::arith::CmpFOp::create(builder, loc, pred, args[0], args[1]);
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INVALID, hasSNaNOp);
-  return builder.create<fir::ConvertOp>(loc, resultType, res);
+  return fir::ConvertOp::create(builder, loc, resultType, res);
 }
 
 // IEEE_REAL
@@ -5612,14 +5621,14 @@ mlir::Value IntrinsicLibrary::genIeeeReal(mlir::Type resultType,
   // If the argument is an sNaN, raise an invalid exception and return a qNaN.
   // Otherwise return the argument.
   auto processSnan = [&](mlir::Value x) {
-    fir::IfOp ifOp = builder.create<fir::IfOp>(loc, resultType,
-                                               genIsFPClass(i1Ty, x, snanTest),
-                                               /*withElseRegion=*/true);
+    fir::IfOp ifOp = fir::IfOp::create(builder, loc, resultType,
+                                       genIsFPClass(i1Ty, x, snanTest),
+                                       /*withElseRegion=*/true);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INVALID);
-    builder.create<fir::ResultOp>(loc, genQNan(resultType));
+    fir::ResultOp::create(builder, loc, genQNan(resultType));
     builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-    builder.create<fir::ResultOp>(loc, x);
+    fir::ResultOp::create(builder, loc, x);
     builder.setInsertionPointAfter(ifOp);
     return ifOp.getResult(0);
   };
@@ -5635,7 +5644,7 @@ mlir::Value IntrinsicLibrary::genIeeeReal(mlir::Type resultType,
     a = builder.createConvert(loc, f32Ty, a);
     aType = f32Ty;
   }
-  r = builder.create<fir::ConvertOp>(loc, resultType, a);
+  r = fir::ConvertOp::create(builder, loc, resultType, a);
 
   mlir::IntegerType aIntType = mlir::dyn_cast<mlir::IntegerType>(aType);
   mlir::FloatType aFloatType = mlir::dyn_cast<mlir::FloatType>(aType);
@@ -5647,142 +5656,144 @@ mlir::Value IntrinsicLibrary::genIeeeReal(mlir::Type resultType,
     return aIntType ? r : processSnan(r);
 
   // A possibly inexact conversion result may need to be rounded up or down.
-  mlir::Value b = builder.create<fir::ConvertOp>(loc, aType, r);
+  mlir::Value b = fir::ConvertOp::create(builder, loc, aType, r);
   mlir::Value aEqB;
   if (aIntType)
-    aEqB = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::eq, a, b);
+    aEqB = mlir::arith::CmpIOp::create(builder, loc,
+                                       mlir::arith::CmpIPredicate::eq, a, b);
   else
-    aEqB = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UEQ, a, b);
+    aEqB = mlir::arith::CmpFOp::create(builder, loc,
+                                       mlir::arith::CmpFPredicate::UEQ, a, b);
 
   // [a == b] a is a NaN or r is exact (a may be -0, +0, -inf, +inf) -- return r
-  fir::IfOp ifOp1 = builder.create<fir::IfOp>(loc, resultType, aEqB,
-                                              /*withElseRegion=*/true);
+  fir::IfOp ifOp1 = fir::IfOp::create(builder, loc, resultType, aEqB,
+                                      /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&ifOp1.getThenRegion().front());
-  builder.create<fir::ResultOp>(loc, aIntType ? r : processSnan(r));
+  fir::ResultOp::create(builder, loc, aIntType ? r : processSnan(r));
 
   // Code common to (a < b) and (a > b) branches.
   builder.setInsertionPointToStart(&ifOp1.getElseRegion().front());
   mlir::func::FuncOp getRound = fir::factory::getLlvmGetRounding(builder);
-  mlir::Value mode = builder.create<fir::CallOp>(loc, getRound).getResult(0);
+  mlir::Value mode = fir::CallOp::create(builder, loc, getRound).getResult(0);
   mlir::Value aIsNegative, aIsPositive;
   if (aIntType) {
     mlir::Value zero = builder.createIntegerConstant(loc, aIntType, 0);
-    aIsNegative = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::slt, a, zero);
-    aIsPositive = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::sgt, a, zero);
+    aIsNegative = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::slt, a, zero);
+    aIsPositive = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::sgt, a, zero);
   } else {
     mlir::Value zero = builder.createRealZeroConstant(loc, aFloatType);
-    aIsNegative = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OLT, a, zero);
-    aIsPositive = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OGT, a, zero);
+    aIsNegative = mlir::arith::CmpFOp::create(
+        builder, loc, mlir::arith::CmpFPredicate::OLT, a, zero);
+    aIsPositive = mlir::arith::CmpFOp::create(
+        builder, loc, mlir::arith::CmpFPredicate::OGT, a, zero);
   }
   mlir::Type resultIntType = builder.getIntegerType(resultFloatType.getWidth());
   mlir::Value resultCast =
-      builder.create<mlir::arith::BitcastOp>(loc, resultIntType, r);
+      mlir::arith::BitcastOp::create(builder, loc, resultIntType, r);
   mlir::Value one = builder.createIntegerConstant(loc, resultIntType, 1);
-  mlir::Value rIsOdd = builder.create<fir::ConvertOp>(
-      loc, i1Ty, builder.create<mlir::arith::AndIOp>(loc, resultCast, one));
+  mlir::Value rIsOdd = fir::ConvertOp::create(
+      builder, loc, i1Ty,
+      mlir::arith::AndIOp::create(builder, loc, resultCast, one));
   // Check for a rounding mode match.
   auto match = [&](int m) {
-    return builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::eq, mode,
+    return mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::eq, mode,
         builder.createIntegerConstant(loc, mode.getType(), m));
   };
-  mlir::Value roundToNearestBit = builder.create<mlir::arith::OrIOp>(
-      loc,
+  mlir::Value roundToNearestBit = mlir::arith::OrIOp::create(
+      builder, loc,
       // IEEE_OTHER is an alias for IEEE_NEAREST.
       match(_FORTRAN_RUNTIME_IEEE_NEAREST), match(_FORTRAN_RUNTIME_IEEE_OTHER));
   mlir::Value roundToNearest =
-      builder.create<mlir::arith::AndIOp>(loc, roundToNearestBit, rIsOdd);
+      mlir::arith::AndIOp::create(builder, loc, roundToNearestBit, rIsOdd);
   mlir::Value roundToZeroBit = match(_FORTRAN_RUNTIME_IEEE_TO_ZERO);
   mlir::Value roundAwayBit = match(_FORTRAN_RUNTIME_IEEE_AWAY);
   mlir::Value roundToZero, roundAway, mustAdjust;
   fir::IfOp adjustIfOp;
   mlir::Value aLtB;
   if (aIntType)
-    aLtB = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::slt, a, b);
+    aLtB = mlir::arith::CmpIOp::create(builder, loc,
+                                       mlir::arith::CmpIPredicate::slt, a, b);
   else
-    aLtB = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OLT, a, b);
+    aLtB = mlir::arith::CmpFOp::create(builder, loc,
+                                       mlir::arith::CmpFPredicate::OLT, a, b);
   mlir::Value upResult =
-      builder.create<mlir::arith::AddIOp>(loc, resultCast, one);
+      mlir::arith::AddIOp::create(builder, loc, resultCast, one);
   mlir::Value downResult =
-      builder.create<mlir::arith::SubIOp>(loc, resultCast, one);
+      mlir::arith::SubIOp::create(builder, loc, resultCast, one);
 
   // (a < b): r is inexact -- return r or ieee_next_down(r)
-  fir::IfOp ifOp2 = builder.create<fir::IfOp>(loc, resultType, aLtB,
-                                              /*withElseRegion=*/true);
+  fir::IfOp ifOp2 = fir::IfOp::create(builder, loc, resultType, aLtB,
+                                      /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&ifOp2.getThenRegion().front());
   roundToZero =
-      builder.create<mlir::arith::AndIOp>(loc, roundToZeroBit, aIsPositive);
+      mlir::arith::AndIOp::create(builder, loc, roundToZeroBit, aIsPositive);
   roundAway =
-      builder.create<mlir::arith::AndIOp>(loc, roundAwayBit, aIsNegative);
+      mlir::arith::AndIOp::create(builder, loc, roundAwayBit, aIsNegative);
   mlir::Value roundDown = match(_FORTRAN_RUNTIME_IEEE_DOWN);
   mustAdjust =
-      builder.create<mlir::arith::OrIOp>(loc, roundToNearest, roundToZero);
-  mustAdjust = builder.create<mlir::arith::OrIOp>(loc, mustAdjust, roundAway);
-  mustAdjust = builder.create<mlir::arith::OrIOp>(loc, mustAdjust, roundDown);
-  adjustIfOp = builder.create<fir::IfOp>(loc, resultType, mustAdjust,
-                                         /*withElseRegion=*/true);
+      mlir::arith::OrIOp::create(builder, loc, roundToNearest, roundToZero);
+  mustAdjust = mlir::arith::OrIOp::create(builder, loc, mustAdjust, roundAway);
+  mustAdjust = mlir::arith::OrIOp::create(builder, loc, mustAdjust, roundDown);
+  adjustIfOp = fir::IfOp::create(builder, loc, resultType, mustAdjust,
+                                 /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&adjustIfOp.getThenRegion().front());
   if (resultType.isF80())
     r1 = fir::runtime::genNearest(builder, loc, r,
                                   builder.createBool(loc, false));
   else
-    r1 = builder.create<mlir::arith::BitcastOp>(
-        loc, resultType,
-        builder.create<mlir::arith::SelectOp>(loc, aIsNegative, upResult,
-                                              downResult));
-  builder.create<fir::ResultOp>(loc, r1);
+    r1 = mlir::arith::BitcastOp::create(
+        builder, loc, resultType,
+        mlir::arith::SelectOp::create(builder, loc, aIsNegative, upResult,
+                                      downResult));
+  fir::ResultOp::create(builder, loc, r1);
   builder.setInsertionPointToStart(&adjustIfOp.getElseRegion().front());
-  builder.create<fir::ResultOp>(loc, r);
+  fir::ResultOp::create(builder, loc, r);
   builder.setInsertionPointAfter(adjustIfOp);
-  builder.create<fir::ResultOp>(loc, adjustIfOp.getResult(0));
+  fir::ResultOp::create(builder, loc, adjustIfOp.getResult(0));
 
   // (a > b): r is inexact -- return r or ieee_next_up(r)
   builder.setInsertionPointToStart(&ifOp2.getElseRegion().front());
   roundToZero =
-      builder.create<mlir::arith::AndIOp>(loc, roundToZeroBit, aIsNegative);
+      mlir::arith::AndIOp::create(builder, loc, roundToZeroBit, aIsNegative);
   roundAway =
-      builder.create<mlir::arith::AndIOp>(loc, roundAwayBit, aIsPositive);
+      mlir::arith::AndIOp::create(builder, loc, roundAwayBit, aIsPositive);
   mlir::Value roundUp = match(_FORTRAN_RUNTIME_IEEE_UP);
   mustAdjust =
-      builder.create<mlir::arith::OrIOp>(loc, roundToNearest, roundToZero);
-  mustAdjust = builder.create<mlir::arith::OrIOp>(loc, mustAdjust, roundAway);
-  mustAdjust = builder.create<mlir::arith::OrIOp>(loc, mustAdjust, roundUp);
-  adjustIfOp = builder.create<fir::IfOp>(loc, resultType, mustAdjust,
-                                         /*withElseRegion=*/true);
+      mlir::arith::OrIOp::create(builder, loc, roundToNearest, roundToZero);
+  mustAdjust = mlir::arith::OrIOp::create(builder, loc, mustAdjust, roundAway);
+  mustAdjust = mlir::arith::OrIOp::create(builder, loc, mustAdjust, roundUp);
+  adjustIfOp = fir::IfOp::create(builder, loc, resultType, mustAdjust,
+                                 /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&adjustIfOp.getThenRegion().front());
   if (resultType.isF80())
     r1 = fir::runtime::genNearest(builder, loc, r,
                                   builder.createBool(loc, true));
   else
-    r1 = builder.create<mlir::arith::BitcastOp>(
-        loc, resultType,
-        builder.create<mlir::arith::SelectOp>(loc, aIsPositive, upResult,
-                                              downResult));
-  builder.create<fir::ResultOp>(loc, r1);
+    r1 = mlir::arith::BitcastOp::create(
+        builder, loc, resultType,
+        mlir::arith::SelectOp::create(builder, loc, aIsPositive, upResult,
+                                      downResult));
+  fir::ResultOp::create(builder, loc, r1);
   builder.setInsertionPointToStart(&adjustIfOp.getElseRegion().front());
-  builder.create<fir::ResultOp>(loc, r);
+  fir::ResultOp::create(builder, loc, r);
   builder.setInsertionPointAfter(adjustIfOp);
-  builder.create<fir::ResultOp>(loc, adjustIfOp.getResult(0));
+  fir::ResultOp::create(builder, loc, adjustIfOp.getResult(0));
 
   // Generate exceptions for (a < b) and (a > b) branches.
   builder.setInsertionPointAfter(ifOp2);
   r = ifOp2.getResult(0);
-  fir::IfOp exceptIfOp1 = builder.create<fir::IfOp>(
-      loc, genIsFPClass(i1Ty, r, infiniteTest), /*withElseRegion=*/true);
+  fir::IfOp exceptIfOp1 =
+      fir::IfOp::create(builder, loc, genIsFPClass(i1Ty, r, infiniteTest),
+                        /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&exceptIfOp1.getThenRegion().front());
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_OVERFLOW |
                  _FORTRAN_RUNTIME_IEEE_INEXACT);
   builder.setInsertionPointToStart(&exceptIfOp1.getElseRegion().front());
-  fir::IfOp exceptIfOp2 = builder.create<fir::IfOp>(
-      loc, genIsFPClass(i1Ty, r, subnormalTest | zeroTest),
+  fir::IfOp exceptIfOp2 = fir::IfOp::create(
+      builder, loc, genIsFPClass(i1Ty, r, subnormalTest | zeroTest),
       /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&exceptIfOp2.getThenRegion().front());
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_UNDERFLOW |
@@ -5790,7 +5801,7 @@ mlir::Value IntrinsicLibrary::genIeeeReal(mlir::Type resultType,
   builder.setInsertionPointToStart(&exceptIfOp2.getElseRegion().front());
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INEXACT);
   builder.setInsertionPointAfter(exceptIfOp1);
-  builder.create<fir::ResultOp>(loc, ifOp2.getResult(0));
+  fir::ResultOp::create(builder, loc, ifOp2.getResult(0));
   builder.setInsertionPointAfter(ifOp1);
   return ifOp1.getResult(0);
 }
@@ -5806,19 +5817,19 @@ mlir::Value IntrinsicLibrary::genIeeeRem(mlir::Type resultType,
   mlir::Value y = args[1];
   if (mlir::dyn_cast<mlir::FloatType>(resultType).getWidth() < 32) {
     mlir::Type f32Ty = mlir::Float32Type::get(builder.getContext());
-    x = builder.create<fir::ConvertOp>(loc, f32Ty, x);
-    y = builder.create<fir::ConvertOp>(loc, f32Ty, y);
+    x = fir::ConvertOp::create(builder, loc, f32Ty, x);
+    y = fir::ConvertOp::create(builder, loc, f32Ty, y);
   } else {
-    x = builder.create<fir::ConvertOp>(loc, resultType, x);
-    y = builder.create<fir::ConvertOp>(loc, resultType, y);
+    x = fir::ConvertOp::create(builder, loc, resultType, x);
+    y = fir::ConvertOp::create(builder, loc, resultType, y);
   }
   // remainder calls do not signal IEEE_UNDERFLOW.
-  mlir::Value underflow = builder.create<mlir::arith::AndIOp>(
-      loc, genIsFPClass(builder.getI1Type(), x, subnormalTest),
+  mlir::Value underflow = mlir::arith::AndIOp::create(
+      builder, loc, genIsFPClass(builder.getI1Type(), x, subnormalTest),
       genIsFPClass(builder.getI1Type(), y, infiniteTest));
   mlir::Value result = genRuntimeCall("remainder", x.getType(), {x, y});
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_UNDERFLOW, underflow);
-  return builder.create<fir::ConvertOp>(loc, resultType, result);
+  return fir::ConvertOp::create(builder, loc, resultType, result);
 }
 
 // IEEE_RINT
@@ -5834,19 +5845,19 @@ mlir::Value IntrinsicLibrary::genIeeeRint(mlir::Type resultType,
   mlir::func::FuncOp setRound = fir::factory::getLlvmSetRounding(builder);
   mlir::Value mode;
   if (isStaticallyPresent(args[1])) {
-    mode = builder.create<fir::CallOp>(loc, getRound).getResult(0);
+    mode = fir::CallOp::create(builder, loc, getRound).getResult(0);
     genIeeeSetRoundingMode({args[1]});
   }
   if (mlir::cast<mlir::FloatType>(resultType).getWidth() == 16)
-    a = builder.create<fir::ConvertOp>(
-        loc, mlir::Float32Type::get(builder.getContext()), a);
-  mlir::Value result = builder.create<fir::ConvertOp>(
-      loc, resultType, genRuntimeCall("nearbyint", a.getType(), a));
+    a = fir::ConvertOp::create(builder, loc,
+                               mlir::Float32Type::get(builder.getContext()), a);
+  mlir::Value result = fir::ConvertOp::create(
+      builder, loc, resultType, genRuntimeCall("nearbyint", a.getType(), a));
   if (isStaticallyPresent(args[1])) {
-    builder.create<fir::CallOp>(loc, setRound, mode);
+    fir::CallOp::create(builder, loc, setRound, mode);
   } else {
-    mlir::Value inexact = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::ONE, args[0], result);
+    mlir::Value inexact = mlir::arith::CmpFOp::create(
+        builder, loc, mlir::arith::CmpFPredicate::ONE, args[0], result);
     genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INEXACT, inexact);
   }
   return result;
@@ -5862,18 +5873,19 @@ void IntrinsicLibrary::genIeeeSetFlagOrHaltingMode(
   mlir::Type i1Ty = builder.getI1Type();
   mlir::Type i32Ty = builder.getIntegerType(32);
   auto [fieldRef, ignore] = getFieldRef(builder, loc, getBase(args[0]));
-  mlir::Value field = builder.create<fir::LoadOp>(loc, fieldRef);
+  mlir::Value field = fir::LoadOp::create(builder, loc, fieldRef);
   mlir::Value except = fir::runtime::genMapExcept(
-      builder, loc, builder.create<fir::ConvertOp>(loc, i32Ty, field));
-  auto ifOp = builder.create<fir::IfOp>(
-      loc, builder.create<fir::ConvertOp>(loc, i1Ty, getBase(args[1])),
+      builder, loc, fir::ConvertOp::create(builder, loc, i32Ty, field));
+  auto ifOp = fir::IfOp::create(
+      builder, loc,
+      fir::ConvertOp::create(builder, loc, i1Ty, getBase(args[1])),
       /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   (isFlag ? fir::runtime::genFeraiseexcept : fir::runtime::genFeenableexcept)(
-      builder, loc, builder.create<fir::ConvertOp>(loc, i32Ty, except));
+      builder, loc, fir::ConvertOp::create(builder, loc, i32Ty, except));
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
   (isFlag ? fir::runtime::genFeclearexcept : fir::runtime::genFedisableexcept)(
-      builder, loc, builder.create<fir::ConvertOp>(loc, i32Ty, except));
+      builder, loc, fir::ConvertOp::create(builder, loc, i32Ty, except));
   builder.setInsertionPointAfter(ifOp);
 }
 
@@ -5890,7 +5902,7 @@ void IntrinsicLibrary::genIeeeSetRoundingMode(
     checkRadix(builder, loc, fir::getBase(args[1]), "ieee_set_rounding_mode");
   auto [fieldRef, fieldTy] = getFieldRef(builder, loc, fir::getBase(args[0]));
   mlir::func::FuncOp setRound = fir::factory::getLlvmSetRounding(builder);
-  mlir::Value mode = builder.create<fir::LoadOp>(loc, fieldRef);
+  mlir::Value mode = fir::LoadOp::create(builder, loc, fieldRef);
   static_assert(
       _FORTRAN_RUNTIME_IEEE_TO_ZERO >= 0 &&
       _FORTRAN_RUNTIME_IEEE_TO_ZERO <= 3 &&
@@ -5898,28 +5910,28 @@ void IntrinsicLibrary::genIeeeSetRoundingMode(
       _FORTRAN_RUNTIME_IEEE_NEAREST <= 3 && _FORTRAN_RUNTIME_IEEE_UP >= 0 &&
       _FORTRAN_RUNTIME_IEEE_UP <= 3 && _FORTRAN_RUNTIME_IEEE_DOWN >= 0 &&
       _FORTRAN_RUNTIME_IEEE_DOWN <= 3 && "unexpected rounding mode mapping");
-  mlir::Value mask = builder.create<mlir::arith::ShLIOp>(
-      loc, builder.createAllOnesInteger(loc, fieldTy),
+  mlir::Value mask = mlir::arith::ShLIOp::create(
+      builder, loc, builder.createAllOnesInteger(loc, fieldTy),
       builder.createIntegerConstant(loc, fieldTy, 2));
-  mlir::Value modeIsSupported = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq,
-      builder.create<mlir::arith::AndIOp>(loc, mode, mask),
+  mlir::Value modeIsSupported = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq,
+      mlir::arith::AndIOp::create(builder, loc, mode, mask),
       builder.createIntegerConstant(loc, fieldTy, 0));
   mlir::Value nearest = builder.createIntegerConstant(
       loc, fieldTy, _FORTRAN_RUNTIME_IEEE_NEAREST);
-  mode = builder.create<mlir::arith::SelectOp>(loc, modeIsSupported, mode,
-                                               nearest);
-  mode = builder.create<fir::ConvertOp>(
-      loc, setRound.getFunctionType().getInput(0), mode);
-  builder.create<fir::CallOp>(loc, setRound, mode);
+  mode = mlir::arith::SelectOp::create(builder, loc, modeIsSupported, mode,
+                                       nearest);
+  mode = fir::ConvertOp::create(builder, loc,
+                                setRound.getFunctionType().getInput(0), mode);
+  fir::CallOp::create(builder, loc, setRound, mode);
 }
 
 // IEEE_SET_UNDERFLOW_MODE
 void IntrinsicLibrary::genIeeeSetUnderflowMode(
     llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 1);
-  mlir::Value gradual = builder.create<fir::ConvertOp>(loc, builder.getI1Type(),
-                                                       getBase(args[0]));
+  mlir::Value gradual = fir::ConvertOp::create(
+      builder, loc, builder.getI1Type(), getBase(args[0]));
   fir::runtime::genSetUnderflowMode(builder, loc, {gradual});
 }
 
@@ -5933,9 +5945,9 @@ IntrinsicLibrary::genIeeeSignalingCompare(mlir::Type resultType,
   assert(args.size() == 2);
   mlir::Value hasNaNOp = genIeeeUnordered(mlir::Type{}, args);
   mlir::Value res =
-      builder.create<mlir::arith::CmpFOp>(loc, pred, args[0], args[1]);
+      mlir::arith::CmpFOp::create(builder, loc, pred, args[0], args[1]);
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INVALID, hasNaNOp);
-  return builder.create<fir::ConvertOp>(loc, resultType, res);
+  return fir::ConvertOp::create(builder, loc, resultType, res);
 }
 
 // IEEE_SIGNBIT
@@ -5954,9 +5966,9 @@ mlir::Value IntrinsicLibrary::genIeeeSignbit(mlir::Type resultType,
   }
   mlir::Type intType = builder.getIntegerType(bitWidth);
   mlir::Value intVal =
-      builder.create<mlir::arith::BitcastOp>(loc, intType, realVal);
+      mlir::arith::BitcastOp::create(builder, loc, intType, realVal);
   mlir::Value shift = builder.createIntegerConstant(loc, intType, bitWidth - 1);
-  mlir::Value sign = builder.create<mlir::arith::ShRUIOp>(loc, intVal, shift);
+  mlir::Value sign = mlir::arith::ShRUIOp::create(builder, loc, intVal, shift);
   return builder.createConvert(loc, resultType, sign);
 }
 
@@ -5969,21 +5981,21 @@ IntrinsicLibrary::genIeeeSupportFlag(mlir::Type resultType,
   mlir::Type i1Ty = builder.getI1Type();
   mlir::Type i32Ty = builder.getIntegerType(32);
   auto [fieldRef, fieldTy] = getFieldRef(builder, loc, getBase(args[0]));
-  mlir::Value flag = builder.create<fir::LoadOp>(loc, fieldRef);
+  mlir::Value flag = fir::LoadOp::create(builder, loc, fieldRef);
   mlir::Value standardFlagMask = builder.createIntegerConstant(
       loc, fieldTy,
       _FORTRAN_RUNTIME_IEEE_INVALID | _FORTRAN_RUNTIME_IEEE_DIVIDE_BY_ZERO |
           _FORTRAN_RUNTIME_IEEE_OVERFLOW | _FORTRAN_RUNTIME_IEEE_UNDERFLOW |
           _FORTRAN_RUNTIME_IEEE_INEXACT);
-  mlir::Value isStandardFlag = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::ne,
-      builder.create<mlir::arith::AndIOp>(loc, flag, standardFlagMask),
+  mlir::Value isStandardFlag = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::ne,
+      mlir::arith::AndIOp::create(builder, loc, flag, standardFlagMask),
       builder.createIntegerConstant(loc, fieldTy, 0));
-  fir::IfOp ifOp = builder.create<fir::IfOp>(loc, i1Ty, isStandardFlag,
-                                             /*withElseRegion=*/true);
+  fir::IfOp ifOp = fir::IfOp::create(builder, loc, i1Ty, isStandardFlag,
+                                     /*withElseRegion=*/true);
   // Standard flags are supported.
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  builder.create<fir::ResultOp>(loc, builder.createBool(loc, true));
+  fir::ResultOp::create(builder, loc, builder.createBool(loc, true));
 
   // TargetCharacteristics information for the nonstandard ieee_denorm flag
   // is not available here. So use a runtime check restricted to possibly
@@ -6007,17 +6019,17 @@ IntrinsicLibrary::genIeeeSupportFlag(mlir::Type resultType,
     }
   }
   if (mayBeSupported) {
-    mlir::Value isDenorm = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::eq, flag,
+    mlir::Value isDenorm = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::eq, flag,
         builder.createIntegerConstant(loc, fieldTy,
                                       _FORTRAN_RUNTIME_IEEE_DENORM));
-    mlir::Value result = builder.create<mlir::arith::AndIOp>(
-        loc, isDenorm,
+    mlir::Value result = mlir::arith::AndIOp::create(
+        builder, loc, isDenorm,
         fir::runtime::genSupportHalting(
-            builder, loc, builder.create<fir::ConvertOp>(loc, i32Ty, flag)));
-    builder.create<fir::ResultOp>(loc, result);
+            builder, loc, fir::ConvertOp::create(builder, loc, i32Ty, flag)));
+    fir::ResultOp::create(builder, loc, result);
   } else {
-    builder.create<fir::ResultOp>(loc, builder.createBool(loc, false));
+    fir::ResultOp::create(builder, loc, builder.createBool(loc, false));
   }
   builder.setInsertionPointAfter(ifOp);
   return builder.createConvert(loc, resultType, ifOp.getResult(0));
@@ -6032,11 +6044,11 @@ fir::ExtendedValue IntrinsicLibrary::genIeeeSupportHalting(
   assert(args.size() == 1);
   mlir::Type i32Ty = builder.getIntegerType(32);
   auto [fieldRef, ignore] = getFieldRef(builder, loc, getBase(args[0]));
-  mlir::Value field = builder.create<fir::LoadOp>(loc, fieldRef);
+  mlir::Value field = fir::LoadOp::create(builder, loc, fieldRef);
   return builder.createConvert(
       loc, resultType,
       fir::runtime::genSupportHalting(
-          builder, loc, builder.create<fir::ConvertOp>(loc, i32Ty, field)));
+          builder, loc, fir::ConvertOp::create(builder, loc, i32Ty, field)));
 }
 
 // IEEE_SUPPORT_ROUNDING
@@ -6053,16 +6065,16 @@ fir::ExtendedValue IntrinsicLibrary::genIeeeSupportRounding(
   //  4 - to nearest, ties away from zero [not supported]
   assert(args.size() == 1 || args.size() == 2);
   auto [fieldRef, fieldTy] = getFieldRef(builder, loc, getBase(args[0]));
-  mlir::Value mode = builder.create<fir::LoadOp>(loc, fieldRef);
-  mlir::Value lbOk = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::sge, mode,
+  mlir::Value mode = fir::LoadOp::create(builder, loc, fieldRef);
+  mlir::Value lbOk = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::sge, mode,
       builder.createIntegerConstant(loc, fieldTy,
                                     _FORTRAN_RUNTIME_IEEE_TO_ZERO));
-  mlir::Value ubOk = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::sle, mode,
+  mlir::Value ubOk = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::sle, mode,
       builder.createIntegerConstant(loc, fieldTy, _FORTRAN_RUNTIME_IEEE_DOWN));
   return builder.createConvert(
-      loc, resultType, builder.create<mlir::arith::AndIOp>(loc, lbOk, ubOk));
+      loc, resultType, mlir::arith::AndIOp::create(builder, loc, lbOk, ubOk));
 }
 
 // IEEE_SUPPORT_STANDARD
@@ -6086,15 +6098,15 @@ IntrinsicLibrary::genIeeeUnordered(mlir::Type resultType,
   // If there is no result type return an i1 result.
   assert(args.size() == 2);
   if (args[0].getType() == args[1].getType()) {
-    mlir::Value res = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UNO, args[0], args[1]);
+    mlir::Value res = mlir::arith::CmpFOp::create(
+        builder, loc, mlir::arith::CmpFPredicate::UNO, args[0], args[1]);
     return resultType ? builder.createConvert(loc, resultType, res) : res;
   }
   assert(resultType && "expecting a (mixed arg type) unordered result type");
   mlir::Type i1Ty = builder.getI1Type();
   mlir::Value xIsNan = genIsFPClass(i1Ty, args[0], nanTest);
   mlir::Value yIsNan = genIsFPClass(i1Ty, args[1], nanTest);
-  mlir::Value res = builder.create<mlir::arith::OrIOp>(loc, xIsNan, yIsNan);
+  mlir::Value res = mlir::arith::OrIOp::create(builder, loc, xIsNan, yIsNan);
   return builder.createConvert(loc, resultType, res);
 }
 
@@ -6219,22 +6231,22 @@ mlir::Value IntrinsicLibrary::genIeeeValue(mlir::Type resultType,
   mlir::Value which;
   if (args.size() == 2) { // user call
     auto [index, ignore] = getFieldRef(builder, loc, args[1]);
-    which = builder.create<fir::LoadOp>(loc, index);
+    which = fir::LoadOp::create(builder, loc, index);
   } else { // compiler generated call
     which = args[0];
   }
-  mlir::Value bits = builder.create<fir::LoadOp>(
-      loc,
-      builder.create<fir::CoordinateOp>(
-          loc, builder.getRefType(valueTy),
-          builder.create<fir::AddrOfOp>(loc, builder.getRefType(tableTy),
-                                        builder.getSymbolRefAttr(tableName)),
+  mlir::Value bits = fir::LoadOp::create(
+      builder, loc,
+      fir::CoordinateOp::create(
+          builder, loc, builder.getRefType(valueTy),
+          fir::AddrOfOp::create(builder, loc, builder.getRefType(tableTy),
+                                builder.getSymbolRefAttr(tableName)),
           which));
   if (bitWidth > 64)
-    bits = builder.create<mlir::arith::ShLIOp>(
-        loc, builder.createConvert(loc, intType, bits),
+    bits = mlir::arith::ShLIOp::create(
+        builder, loc, builder.createConvert(loc, intType, bits),
         builder.createIntegerConstant(loc, intType, bitWidth - 64));
-  return builder.create<mlir::arith::BitcastOp>(loc, realType, bits);
+  return mlir::arith::BitcastOp::create(builder, loc, realType, bits);
 }
 
 // IEOR
@@ -6276,13 +6288,14 @@ IntrinsicLibrary::genIndex(mlir::Type resultType,
         builder.getContext(), builder.getKindMap().defaultLogicalKind());
     mlir::Value temp = builder.createTemporary(loc, logTy);
     mlir::Value castb = builder.createConvert(loc, logTy, b);
-    builder.create<fir::StoreOp>(loc, castb, temp);
+    fir::StoreOp::create(builder, loc, castb, temp);
     return builder.createBox(loc, temp);
   };
-  mlir::Value backOpt = isStaticallyAbsent(args, 2)
-                            ? builder.create<fir::AbsentOp>(
-                                  loc, fir::BoxType::get(builder.getI1Type()))
-                            : makeRefThenEmbox(fir::getBase(args[2]));
+  mlir::Value backOpt =
+      isStaticallyAbsent(args, 2)
+          ? fir::AbsentOp::create(builder, loc,
+                                  fir::BoxType::get(builder.getI1Type()))
+          : makeRefThenEmbox(fir::getBase(args[2]));
   mlir::Value kindVal = isStaticallyAbsent(args, 3)
                             ? builder.createIntegerConstant(
                                   loc, builder.getIndexType(),
@@ -6331,8 +6344,8 @@ mlir::Value
 IntrinsicLibrary::genIsIostatValue(mlir::Type resultType,
                                    llvm::ArrayRef<mlir::Value> args) {
   assert(args.size() == 1);
-  return builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq, args[0],
+  return mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq, args[0],
       builder.createIntegerConstant(loc, args[0].getType(), value));
 }
 
@@ -6359,16 +6372,16 @@ mlir::Value IntrinsicLibrary::genIshft(mlir::Type resultType,
   mlir::Value word = args[0];
   if (word.getType().isUnsignedInteger())
     word = builder.createConvert(loc, signlessType, word);
-  auto left = builder.create<mlir::arith::ShLIOp>(loc, word, absShift);
-  auto right = builder.create<mlir::arith::ShRUIOp>(loc, word, absShift);
-  auto shiftIsLarge = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::sge, absShift, bitSize);
-  auto shiftIsNegative = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::slt, shift, zero);
+  auto left = mlir::arith::ShLIOp::create(builder, loc, word, absShift);
+  auto right = mlir::arith::ShRUIOp::create(builder, loc, word, absShift);
+  auto shiftIsLarge = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::sge, absShift, bitSize);
+  auto shiftIsNegative = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::slt, shift, zero);
   auto sel =
-      builder.create<mlir::arith::SelectOp>(loc, shiftIsNegative, right, left);
+      mlir::arith::SelectOp::create(builder, loc, shiftIsNegative, right, left);
   mlir::Value result =
-      builder.create<mlir::arith::SelectOp>(loc, shiftIsLarge, zero, sel);
+      mlir::arith::SelectOp::create(builder, loc, shiftIsLarge, zero, sel);
   if (resultType.isUnsignedInteger())
     return builder.createConvert(loc, resultType, result);
   return result;
@@ -6409,42 +6422,42 @@ mlir::Value IntrinsicLibrary::genIshftc(mlir::Type resultType,
   mlir::Value zero = builder.createIntegerConstant(loc, signlessType, 0);
   mlir::Value ones = builder.createAllOnesInteger(loc, signlessType);
   mlir::Value absShift = genAbs(signlessType, {shift});
-  auto elseSize = builder.create<mlir::arith::SubIOp>(loc, size, absShift);
-  auto shiftIsZero = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq, shift, zero);
-  auto shiftEqualsSize = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq, absShift, size);
+  auto elseSize = mlir::arith::SubIOp::create(builder, loc, size, absShift);
+  auto shiftIsZero = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq, shift, zero);
+  auto shiftEqualsSize = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq, absShift, size);
   auto shiftIsNop =
-      builder.create<mlir::arith::OrIOp>(loc, shiftIsZero, shiftEqualsSize);
-  auto shiftIsPositive = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::sgt, shift, zero);
-  auto leftSize = builder.create<mlir::arith::SelectOp>(loc, shiftIsPositive,
-                                                        absShift, elseSize);
-  auto rightSize = builder.create<mlir::arith::SelectOp>(loc, shiftIsPositive,
-                                                         elseSize, absShift);
-  auto hasUnchanged = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::ne, size, bitSize);
-  auto unchangedTmp1 = builder.create<mlir::arith::ShRUIOp>(loc, word, size);
+      mlir::arith::OrIOp::create(builder, loc, shiftIsZero, shiftEqualsSize);
+  auto shiftIsPositive = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::sgt, shift, zero);
+  auto leftSize = mlir::arith::SelectOp::create(builder, loc, shiftIsPositive,
+                                                absShift, elseSize);
+  auto rightSize = mlir::arith::SelectOp::create(builder, loc, shiftIsPositive,
+                                                 elseSize, absShift);
+  auto hasUnchanged = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::ne, size, bitSize);
+  auto unchangedTmp1 = mlir::arith::ShRUIOp::create(builder, loc, word, size);
   auto unchangedTmp2 =
-      builder.create<mlir::arith::ShLIOp>(loc, unchangedTmp1, size);
-  auto unchanged = builder.create<mlir::arith::SelectOp>(loc, hasUnchanged,
-                                                         unchangedTmp2, zero);
+      mlir::arith::ShLIOp::create(builder, loc, unchangedTmp1, size);
+  auto unchanged = mlir::arith::SelectOp::create(builder, loc, hasUnchanged,
+                                                 unchangedTmp2, zero);
   auto leftMaskShift =
-      builder.create<mlir::arith::SubIOp>(loc, bitSize, leftSize);
+      mlir::arith::SubIOp::create(builder, loc, bitSize, leftSize);
   auto leftMask =
-      builder.create<mlir::arith::ShRUIOp>(loc, ones, leftMaskShift);
-  auto leftTmp = builder.create<mlir::arith::ShRUIOp>(loc, word, rightSize);
-  auto left = builder.create<mlir::arith::AndIOp>(loc, leftTmp, leftMask);
+      mlir::arith::ShRUIOp::create(builder, loc, ones, leftMaskShift);
+  auto leftTmp = mlir::arith::ShRUIOp::create(builder, loc, word, rightSize);
+  auto left = mlir::arith::AndIOp::create(builder, loc, leftTmp, leftMask);
   auto rightMaskShift =
-      builder.create<mlir::arith::SubIOp>(loc, bitSize, rightSize);
+      mlir::arith::SubIOp::create(builder, loc, bitSize, rightSize);
   auto rightMask =
-      builder.create<mlir::arith::ShRUIOp>(loc, ones, rightMaskShift);
-  auto rightTmp = builder.create<mlir::arith::AndIOp>(loc, word, rightMask);
-  auto right = builder.create<mlir::arith::ShLIOp>(loc, rightTmp, leftSize);
-  auto resTmp = builder.create<mlir::arith::OrIOp>(loc, unchanged, left);
-  auto res = builder.create<mlir::arith::OrIOp>(loc, resTmp, right);
+      mlir::arith::ShRUIOp::create(builder, loc, ones, rightMaskShift);
+  auto rightTmp = mlir::arith::AndIOp::create(builder, loc, word, rightMask);
+  auto right = mlir::arith::ShLIOp::create(builder, loc, rightTmp, leftSize);
+  auto resTmp = mlir::arith::OrIOp::create(builder, loc, unchanged, left);
+  auto res = mlir::arith::OrIOp::create(builder, loc, resTmp, right);
   mlir::Value result =
-      builder.create<mlir::arith::SelectOp>(loc, shiftIsNop, word, res);
+      mlir::arith::SelectOp::create(builder, loc, shiftIsNop, word, res);
   if (resultType.isUnsignedInteger())
     return builder.createConvert(loc, resultType, result);
   return result;
@@ -6456,7 +6469,7 @@ mlir::Value IntrinsicLibrary::genLeadz(mlir::Type resultType,
   assert(args.size() == 1);
 
   mlir::Value result =
-      builder.create<mlir::math::CountLeadingZerosOp>(loc, args);
+      mlir::math::CountLeadingZerosOp::create(builder, loc, args);
 
   return builder.createConvert(loc, resultType, result);
 }
@@ -6524,18 +6537,18 @@ IntrinsicLibrary::genLoc(mlir::Type resultType,
   // created when preparing the argument cases, but the box can be safely be
   // used for all those cases and the address will be null if absent.
   mlir::Value isPresent =
-      builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), box);
+      fir::IsPresentOp::create(builder, loc, builder.getI1Type(), box);
   return builder
       .genIfOp(loc, {resultType}, isPresent,
                /*withElseRegion=*/true)
       .genThen([&]() {
         mlir::Value argAddr = getAddrFromBox(builder, loc, args[0], isFunc);
         mlir::Value cast = builder.createConvert(loc, resultType, argAddr);
-        builder.create<fir::ResultOp>(loc, cast);
+        fir::ResultOp::create(builder, loc, cast);
       })
       .genElse([&]() {
         mlir::Value zero = builder.createIntegerConstant(loc, resultType, 0);
-        builder.create<fir::ResultOp>(loc, zero);
+        fir::ResultOp::create(builder, loc, zero);
       })
       .getResults()[0];
 }
@@ -6568,12 +6581,12 @@ mlir::Value IntrinsicLibrary::genMask(mlir::Type resultType,
   // non-deterministic result. Other compilers don't produce a consistent result
   // in this case either, so we choose the most efficient implementation.
   mlir::Value shift =
-      builder.create<mlir::arith::SubIOp>(loc, bitSize, bitsToSet);
-  mlir::Value shifted = builder.create<Shift>(loc, ones, shift);
-  mlir::Value isZero = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq, bitsToSet, zero);
+      mlir::arith::SubIOp::create(builder, loc, bitSize, bitsToSet);
+  mlir::Value shifted = Shift::create(builder, loc, ones, shift);
+  mlir::Value isZero = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq, bitsToSet, zero);
   mlir::Value result =
-      builder.create<mlir::arith::SelectOp>(loc, isZero, zero, shifted);
+      mlir::arith::SelectOp::create(builder, loc, isZero, zero, shifted);
   if (resultType.isUnsignedInteger())
     return builder.createConvert(loc, resultType, result);
   return result;
@@ -6591,8 +6604,8 @@ IntrinsicLibrary::genMatchAllSync(mlir::Type resultType,
 
   mlir::Value arg1 = args[1];
   if (arg1.getType().isF32() || arg1.getType().isF64())
-    arg1 = builder.create<fir::ConvertOp>(
-        loc, is32 ? builder.getI32Type() : builder.getI64Type(), arg1);
+    arg1 = fir::ConvertOp::create(
+        builder, loc, is32 ? builder.getI32Type() : builder.getI64Type(), arg1);
 
   mlir::Type retTy =
       mlir::LLVM::LLVMStructType::getLiteral(context, {resultType, i1Ty});
@@ -6601,10 +6614,10 @@ IntrinsicLibrary::genMatchAllSync(mlir::Type resultType,
           .create<mlir::NVVM::MatchSyncOp>(loc, retTy, args[0], arg1,
                                            mlir::NVVM::MatchSyncKind::all)
           .getResult();
-  auto value = builder.create<mlir::LLVM::ExtractValueOp>(loc, match, 0);
-  auto pred = builder.create<mlir::LLVM::ExtractValueOp>(loc, match, 1);
-  auto conv = builder.create<mlir::LLVM::ZExtOp>(loc, resultType, pred);
-  builder.create<fir::StoreOp>(loc, conv, args[2]);
+  auto value = mlir::LLVM::ExtractValueOp::create(builder, loc, match, 0);
+  auto pred = mlir::LLVM::ExtractValueOp::create(builder, loc, match, 1);
+  auto conv = mlir::LLVM::ZExtOp::create(builder, loc, resultType, pred);
+  fir::StoreOp::create(builder, loc, conv, args[2]);
   return value;
 }
 
@@ -6614,14 +6627,14 @@ mlir::Value IntrinsicLibrary::genVoteSync(mlir::Type resultType,
                                           llvm::ArrayRef<mlir::Value> args) {
   assert(args.size() == 2);
   mlir::Value arg1 =
-      builder.create<fir::ConvertOp>(loc, builder.getI1Type(), args[1]);
+      fir::ConvertOp::create(builder, loc, builder.getI1Type(), args[1]);
   mlir::Type resTy = kind == mlir::NVVM::VoteSyncKind::ballot
                          ? builder.getI32Type()
                          : builder.getI1Type();
   auto voteRes =
-      builder.create<mlir::NVVM::VoteSyncOp>(loc, resTy, args[0], arg1, kind)
+      mlir::NVVM::VoteSyncOp::create(builder, loc, resTy, args[0], arg1, kind)
           .getResult();
-  return builder.create<fir::ConvertOp>(loc, resultType, voteRes);
+  return fir::ConvertOp::create(builder, loc, resultType, voteRes);
 }
 
 // MATCH_ANY_SYNC
@@ -6633,8 +6646,8 @@ IntrinsicLibrary::genMatchAnySync(mlir::Type resultType,
 
   mlir::Value arg1 = args[1];
   if (arg1.getType().isF32() || arg1.getType().isF64())
-    arg1 = builder.create<fir::ConvertOp>(
-        loc, is32 ? builder.getI32Type() : builder.getI64Type(), arg1);
+    arg1 = fir::ConvertOp::create(
+        builder, loc, is32 ? builder.getI32Type() : builder.getI64Type(), arg1);
 
   return builder
       .create<mlir::NVVM::MatchSyncOp>(loc, resultType, args[0], arg1,
@@ -6718,10 +6731,10 @@ IntrinsicLibrary::genMerge(mlir::Type,
                                  mlir::Value other) -> mlir::Value {
     mlir::Type otherType = other.getType();
     if (mlir::isa<fir::BaseBoxType>(otherType))
-      return builder.create<fir::ReboxOp>(loc, otherType, polymorphic,
-                                          /*shape*/ mlir::Value{},
-                                          /*slice=*/mlir::Value{});
-    return builder.create<fir::BoxAddrOp>(loc, otherType, polymorphic);
+      return fir::ReboxOp::create(builder, loc, otherType, polymorphic,
+                                  /*shape*/ mlir::Value{},
+                                  /*slice=*/mlir::Value{});
+    return fir::BoxAddrOp::create(builder, loc, otherType, polymorphic);
   };
   if (fir::isPolymorphicType(tsource.getType()) &&
       !fir::isPolymorphicType(fsource.getType())) {
@@ -6737,8 +6750,8 @@ IntrinsicLibrary::genMerge(mlir::Type,
     // fulfill mlir::SelectOp constraint that the MLIR types must be the same.
     fsourceCast = builder.createConvert(loc, tsource.getType(), fsource);
   }
-  auto rslt = builder.create<mlir::arith::SelectOp>(loc, mask, tsourceCast,
-                                                    fsourceCast);
+  auto rslt = mlir::arith::SelectOp::create(builder, loc, mask, tsourceCast,
+                                            fsourceCast);
   if (isCharRslt) {
     // Need a CharBoxValue for character results
     const fir::CharBoxValue *charBox = args[0].getCharBox();
@@ -6779,7 +6792,7 @@ mlir::Value IntrinsicLibrary::genMod(mlir::Type resultType,
                                                         args[0], args[1]);
   }
   if (mlir::isa<mlir::IntegerType>(resultType))
-    return builder.create<mlir::arith::RemSIOp>(loc, args[0], args[1]);
+    return mlir::arith::RemSIOp::create(builder, loc, args[0], args[1]);
 
   // Use runtime.
   return builder.createConvert(
@@ -6809,19 +6822,19 @@ mlir::Value IntrinsicLibrary::genModulo(mlir::Type resultType,
   }
   if (mlir::isa<mlir::IntegerType>(resultType)) {
     auto remainder =
-        builder.create<mlir::arith::RemSIOp>(loc, args[0], args[1]);
-    auto argXor = builder.create<mlir::arith::XOrIOp>(loc, args[0], args[1]);
+        mlir::arith::RemSIOp::create(builder, loc, args[0], args[1]);
+    auto argXor = mlir::arith::XOrIOp::create(builder, loc, args[0], args[1]);
     mlir::Value zero = builder.createIntegerConstant(loc, argXor.getType(), 0);
-    auto argSignDifferent = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::slt, argXor, zero);
-    auto remainderIsNotZero = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::ne, remainder, zero);
-    auto mustAddP = builder.create<mlir::arith::AndIOp>(loc, remainderIsNotZero,
-                                                        argSignDifferent);
+    auto argSignDifferent = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::slt, argXor, zero);
+    auto remainderIsNotZero = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::ne, remainder, zero);
+    auto mustAddP = mlir::arith::AndIOp::create(
+        builder, loc, remainderIsNotZero, argSignDifferent);
     auto remPlusP =
-        builder.create<mlir::arith::AddIOp>(loc, remainder, args[1]);
-    return builder.create<mlir::arith::SelectOp>(loc, mustAddP, remPlusP,
-                                                 remainder);
+        mlir::arith::AddIOp::create(builder, loc, remainder, args[1]);
+    return mlir::arith::SelectOp::create(builder, loc, mustAddP, remPlusP,
+                                         remainder);
   }
 
   auto fastMathFlags = builder.getFastMathFlags();
@@ -6834,21 +6847,21 @@ mlir::Value IntrinsicLibrary::genModulo(mlir::Type resultType,
         loc, resultType,
         fir::runtime::genModulo(builder, loc, args[0], args[1]));
 
-  auto remainder = builder.create<mlir::arith::RemFOp>(loc, args[0], args[1]);
+  auto remainder = mlir::arith::RemFOp::create(builder, loc, args[0], args[1]);
   mlir::Value zero = builder.createRealZeroConstant(loc, remainder.getType());
-  auto remainderIsNotZero = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::UNE, remainder, zero);
-  auto aLessThanZero = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::OLT, args[0], zero);
-  auto pLessThanZero = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::OLT, args[1], zero);
+  auto remainderIsNotZero = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::UNE, remainder, zero);
+  auto aLessThanZero = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::OLT, args[0], zero);
+  auto pLessThanZero = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::OLT, args[1], zero);
   auto argSignDifferent =
-      builder.create<mlir::arith::XOrIOp>(loc, aLessThanZero, pLessThanZero);
-  auto mustAddP = builder.create<mlir::arith::AndIOp>(loc, remainderIsNotZero,
-                                                      argSignDifferent);
-  auto remPlusP = builder.create<mlir::arith::AddFOp>(loc, remainder, args[1]);
-  return builder.create<mlir::arith::SelectOp>(loc, mustAddP, remPlusP,
-                                               remainder);
+      mlir::arith::XOrIOp::create(builder, loc, aLessThanZero, pLessThanZero);
+  auto mustAddP = mlir::arith::AndIOp::create(builder, loc, remainderIsNotZero,
+                                              argSignDifferent);
+  auto remPlusP = mlir::arith::AddFOp::create(builder, loc, remainder, args[1]);
+  return mlir::arith::SelectOp::create(builder, loc, mustAddP, remPlusP,
+                                       remainder);
 }
 
 void IntrinsicLibrary::genMoveAlloc(llvm::ArrayRef<fir::ExtendedValue> args) {
@@ -6863,7 +6876,7 @@ void IntrinsicLibrary::genMoveAlloc(llvm::ArrayRef<fir::ExtendedValue> args) {
   mlir::Value errBox =
       isStaticallyPresent(errMsg)
           ? fir::getBase(errMsg)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
 
   const fir::MutableBoxValue *fromBox = from.getBoxOf<fir::MutableBoxValue>();
   const fir::MutableBoxValue *toBox = to.getBoxOf<fir::MutableBoxValue>();
@@ -6920,33 +6933,34 @@ void IntrinsicLibrary::genMvbits(llvm::ArrayRef<fir::ExtendedValue> args) {
   mlir::Type toType{fir::dyn_cast_ptrEleTy(toAddr.getType())};
   assert(toType.getIntOrFloatBitWidth() == fromType.getIntOrFloatBitWidth() &&
          "mismatched mvbits types");
-  auto to = builder.create<fir::LoadOp>(loc, signlessType, toAddr);
+  auto to = fir::LoadOp::create(builder, loc, signlessType, toAddr);
   mlir::Value topos = builder.createConvert(loc, signlessType, unbox(args[4]));
   mlir::Value zero = builder.createIntegerConstant(loc, signlessType, 0);
   mlir::Value ones = builder.createAllOnesInteger(loc, signlessType);
   mlir::Value bitSize = builder.createIntegerConstant(
       loc, signlessType,
       mlir::cast<mlir::IntegerType>(signlessType).getWidth());
-  auto shiftCount = builder.create<mlir::arith::SubIOp>(loc, bitSize, len);
-  auto mask = builder.create<mlir::arith::ShRUIOp>(loc, ones, shiftCount);
-  auto unchangedTmp1 = builder.create<mlir::arith::ShLIOp>(loc, mask, topos);
+  auto shiftCount = mlir::arith::SubIOp::create(builder, loc, bitSize, len);
+  auto mask = mlir::arith::ShRUIOp::create(builder, loc, ones, shiftCount);
+  auto unchangedTmp1 = mlir::arith::ShLIOp::create(builder, loc, mask, topos);
   auto unchangedTmp2 =
-      builder.create<mlir::arith::XOrIOp>(loc, unchangedTmp1, ones);
-  auto unchanged = builder.create<mlir::arith::AndIOp>(loc, unchangedTmp2, to);
+      mlir::arith::XOrIOp::create(builder, loc, unchangedTmp1, ones);
+  auto unchanged = mlir::arith::AndIOp::create(builder, loc, unchangedTmp2, to);
   if (fromType.isUnsignedInteger())
     from = builder.createConvert(loc, signlessType, from);
-  auto frombitsTmp1 = builder.create<mlir::arith::ShRUIOp>(loc, from, frompos);
+  auto frombitsTmp1 = mlir::arith::ShRUIOp::create(builder, loc, from, frompos);
   auto frombitsTmp2 =
-      builder.create<mlir::arith::AndIOp>(loc, frombitsTmp1, mask);
-  auto frombits = builder.create<mlir::arith::ShLIOp>(loc, frombitsTmp2, topos);
-  auto resTmp = builder.create<mlir::arith::OrIOp>(loc, unchanged, frombits);
-  auto lenIsZero = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq, len, zero);
+      mlir::arith::AndIOp::create(builder, loc, frombitsTmp1, mask);
+  auto frombits =
+      mlir::arith::ShLIOp::create(builder, loc, frombitsTmp2, topos);
+  auto resTmp = mlir::arith::OrIOp::create(builder, loc, unchanged, frombits);
+  auto lenIsZero = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq, len, zero);
   mlir::Value res =
-      builder.create<mlir::arith::SelectOp>(loc, lenIsZero, to, resTmp);
+      mlir::arith::SelectOp::create(builder, loc, lenIsZero, to, resTmp);
   if (toType.isUnsignedInteger())
     res = builder.createConvert(loc, toType, res);
-  builder.create<fir::StoreOp>(loc, res, toAddr);
+  fir::StoreOp::create(builder, loc, res, toAddr);
 }
 
 // NEAREST, IEEE_NEXT_AFTER, IEEE_NEXT_DOWN, IEEE_NEXT_UP
@@ -6988,7 +7002,7 @@ mlir::Value IntrinsicLibrary::genNearest(mlir::Type resultType,
     // If isNan(Y), set X to a qNaN that will propagate to the resultIsX result.
     mlir::Value qNan = genQNan(xType);
     mlir::Value isFPClass = genIsFPClass(i1Ty, args[1], nanTest);
-    x = builder.create<mlir::arith::SelectOp>(loc, isFPClass, qNan, x);
+    x = mlir::arith::SelectOp::create(builder, loc, isFPClass, qNan, x);
   }
   mlir::Value resultIsX = genIsFPClass(i1Ty, x, nanTest);
   mlir::Type intType = builder.getIntegerType(xBitWidth);
@@ -6999,15 +7013,15 @@ mlir::Value IntrinsicLibrary::genNearest(mlir::Type resultType,
   if constexpr (proc == NearestProc::Nearest) {
     // Arg S must not be zero.
     fir::IfOp ifOp =
-        builder.create<fir::IfOp>(loc, genIsFPClass(i1Ty, args[1], zeroTest),
-                                  /*withElseRegion=*/false);
+        fir::IfOp::create(builder, loc, genIsFPClass(i1Ty, args[1], zeroTest),
+                          /*withElseRegion=*/false);
     builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     fir::runtime::genReportFatalUserError(
         builder, loc, "intrinsic nearest S argument is zero");
     builder.setInsertionPointAfter(ifOp);
     mlir::Value sSign = IntrinsicLibrary::genIeeeSignbit(intType, {args[1]});
-    valueUp = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::ne, sSign, one);
+    valueUp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::ne, sSign, one);
   } else if constexpr (proc == NearestProc::NextAfter) {
     // Convert X and Y to a common type to allow comparison. Direct conversions
     // between kinds 2, 3, 10, and 16 are not all supported. These conversions
@@ -7028,58 +7042,58 @@ mlir::Value IntrinsicLibrary::genNearest(mlir::Type resultType,
       if (xBitWidth > 32 && xBitWidth > yBitWidth)
         y = builder.createConvert(loc, xType, y);
     }
-    resultIsX = builder.create<mlir::arith::OrIOp>(
-        loc, resultIsX,
-        builder.create<mlir::arith::CmpFOp>(
-            loc, mlir::arith::CmpFPredicate::OEQ, x1, y));
-    valueUp = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OLT, x1, y);
+    resultIsX = mlir::arith::OrIOp::create(
+        builder, loc, resultIsX,
+        mlir::arith::CmpFOp::create(builder, loc,
+                                    mlir::arith::CmpFPredicate::OEQ, x1, y));
+    valueUp = mlir::arith::CmpFOp::create(
+        builder, loc, mlir::arith::CmpFPredicate::OLT, x1, y);
   } else if constexpr (proc == NearestProc::NextDown) {
     valueUp = builder.createBool(loc, false);
   } else if constexpr (proc == NearestProc::NextUp) {
     valueUp = builder.createBool(loc, true);
   }
-  mlir::Value magnitudeUp = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::ne, valueUp,
+  mlir::Value magnitudeUp = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::ne, valueUp,
       IntrinsicLibrary::genIeeeSignbit(i1Ty, {args[0]}));
-  resultIsX = builder.create<mlir::arith::OrIOp>(
-      loc, resultIsX,
-      builder.create<mlir::arith::AndIOp>(
-          loc, genIsFPClass(i1Ty, x, infiniteTest), magnitudeUp));
+  resultIsX = mlir::arith::OrIOp::create(
+      builder, loc, resultIsX,
+      mlir::arith::AndIOp::create(
+          builder, loc, genIsFPClass(i1Ty, x, infiniteTest), magnitudeUp));
 
   // Result is X. (For ieee_next_after with isNan(Y), X has been set to a NaN.)
-  fir::IfOp outerIfOp = builder.create<fir::IfOp>(loc, resultType, resultIsX,
-                                                  /*withElseRegion=*/true);
+  fir::IfOp outerIfOp = fir::IfOp::create(builder, loc, resultType, resultIsX,
+                                          /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&outerIfOp.getThenRegion().front());
   if constexpr (proc == NearestProc::NextDown || proc == NearestProc::NextUp)
     genRaiseExcept(_FORTRAN_RUNTIME_IEEE_INVALID,
                    genIsFPClass(i1Ty, x, snanTest));
-  builder.create<fir::ResultOp>(loc, x);
+  fir::ResultOp::create(builder, loc, x);
 
   // Result is minPositiveSubnormal or minNegativeSubnormal. (X is zero.)
   builder.setInsertionPointToStart(&outerIfOp.getElseRegion().front());
-  mlir::Value resultIsMinSubnormal = builder.create<mlir::arith::CmpFOp>(
-      loc, mlir::arith::CmpFPredicate::OEQ, x,
+  mlir::Value resultIsMinSubnormal = mlir::arith::CmpFOp::create(
+      builder, loc, mlir::arith::CmpFPredicate::OEQ, x,
       builder.createRealZeroConstant(loc, xType));
   fir::IfOp innerIfOp =
-      builder.create<fir::IfOp>(loc, resultType, resultIsMinSubnormal,
-                                /*withElseRegion=*/true);
+      fir::IfOp::create(builder, loc, resultType, resultIsMinSubnormal,
+                        /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&innerIfOp.getThenRegion().front());
   mlir::Value minPositiveSubnormal =
-      builder.create<mlir::arith::BitcastOp>(loc, resultType, one);
-  mlir::Value minNegativeSubnormal = builder.create<mlir::arith::BitcastOp>(
-      loc, resultType,
-      builder.create<mlir::arith::ConstantOp>(
-          loc, intType,
+      mlir::arith::BitcastOp::create(builder, loc, resultType, one);
+  mlir::Value minNegativeSubnormal = mlir::arith::BitcastOp::create(
+      builder, loc, resultType,
+      mlir::arith::ConstantOp::create(
+          builder, loc, intType,
           builder.getIntegerAttr(
               intType, llvm::APInt::getBitsSetWithWrap(
                            xBitWidth, /*lo=*/xBitWidth - 1, /*hi=*/1))));
-  mlir::Value result = builder.create<mlir::arith::SelectOp>(
-      loc, valueUp, minPositiveSubnormal, minNegativeSubnormal);
+  mlir::Value result = mlir::arith::SelectOp::create(
+      builder, loc, valueUp, minPositiveSubnormal, minNegativeSubnormal);
   if constexpr (proc == NearestProc::Nearest || proc == NearestProc::NextAfter)
     genRaiseExcept(_FORTRAN_RUNTIME_IEEE_UNDERFLOW |
                    _FORTRAN_RUNTIME_IEEE_INEXACT);
-  builder.create<fir::ResultOp>(loc, result);
+  fir::ResultOp::create(builder, loc, result);
 
   // Result is (X + minPositiveSubnormal) or (X - minPositiveSubnormal).
   builder.setInsertionPointToStart(&innerIfOp.getElseRegion().front());
@@ -7105,15 +7119,15 @@ mlir::Value IntrinsicLibrary::genNearest(mlir::Type resultType,
       genRuntimeCall("feraiseexcept", i32Ty, excepts);
       genRuntimeCall("feenableexcept", i32Ty, mask);
     }
-    builder.create<fir::ResultOp>(loc, result);
+    fir::ResultOp::create(builder, loc, result);
   } else {
     // Kind 2, 3, 4, 8, 16. Increment or decrement X cast to integer.
-    mlir::Value intX = builder.create<mlir::arith::BitcastOp>(loc, intType, x);
-    mlir::Value add = builder.create<mlir::arith::AddIOp>(loc, intX, one);
-    mlir::Value sub = builder.create<mlir::arith::SubIOp>(loc, intX, one);
-    result = builder.create<mlir::arith::BitcastOp>(
-        loc, resultType,
-        builder.create<mlir::arith::SelectOp>(loc, magnitudeUp, add, sub));
+    mlir::Value intX = mlir::arith::BitcastOp::create(builder, loc, intType, x);
+    mlir::Value add = mlir::arith::AddIOp::create(builder, loc, intX, one);
+    mlir::Value sub = mlir::arith::SubIOp::create(builder, loc, intX, one);
+    result = mlir::arith::BitcastOp::create(
+        builder, loc, resultType,
+        mlir::arith::SelectOp::create(builder, loc, magnitudeUp, add, sub));
     if constexpr (proc == NearestProc::Nearest ||
                   proc == NearestProc::NextAfter) {
       genRaiseExcept(_FORTRAN_RUNTIME_IEEE_OVERFLOW |
@@ -7123,11 +7137,11 @@ mlir::Value IntrinsicLibrary::genNearest(mlir::Type resultType,
                          _FORTRAN_RUNTIME_IEEE_INEXACT,
                      genIsFPClass(i1Ty, result, subnormalTest));
     }
-    builder.create<fir::ResultOp>(loc, result);
+    fir::ResultOp::create(builder, loc, result);
   }
 
   builder.setInsertionPointAfter(innerIfOp);
-  builder.create<fir::ResultOp>(loc, innerIfOp.getResult(0));
+  fir::ResultOp::create(builder, loc, innerIfOp.getResult(0));
   builder.setInsertionPointAfter(outerIfOp);
   return outerIfOp.getResult(0);
 }
@@ -7209,7 +7223,7 @@ IntrinsicLibrary::genNull(mlir::Type, llvm::ArrayRef<fir::ExtendedValue> args) {
   mlir::Value boxStorage = builder.createTemporary(loc, boxType);
   mlir::Value box = fir::factory::createUnallocatedBox(
       builder, loc, boxType, mold->nonDeferredLenParams());
-  builder.create<fir::StoreOp>(loc, box, boxStorage);
+  fir::StoreOp::create(builder, loc, box, boxStorage);
   return fir::MutableBoxValue(boxStorage, mold->nonDeferredLenParams(), {});
 }
 
@@ -7218,7 +7232,7 @@ template <typename OpTy>
 mlir::Value IntrinsicLibrary::genNVVMTime(mlir::Type resultType,
                                           llvm::ArrayRef<mlir::Value> args) {
   assert(args.size() == 0 && "expect no arguments");
-  return builder.create<OpTy>(loc, resultType).getResult();
+  return OpTy::create(builder, loc, resultType).getResult();
 }
 
 // PACK
@@ -7235,10 +7249,11 @@ IntrinsicLibrary::genPack(mlir::Type resultType,
   mlir::Value mask = builder.createBox(loc, args[1]);
 
   // Handle optional vector argument
-  mlir::Value vector = isStaticallyAbsent(args, 2)
-                           ? builder.create<fir::AbsentOp>(
-                                 loc, fir::BoxType::get(builder.getI1Type()))
-                           : builder.createBox(loc, args[2]);
+  mlir::Value vector =
+      isStaticallyAbsent(args, 2)
+          ? fir::AbsentOp::create(builder, loc,
+                                  fir::BoxType::get(builder.getI1Type()))
+          : builder.createBox(loc, args[2]);
 
   // Create mutable fir.box to be passed to the runtime for the result.
   mlir::Type resultArrayType = builder.getVarLenSeqTy(resultType, 1);
@@ -7298,7 +7313,7 @@ void IntrinsicLibrary::genPerror(llvm::ArrayRef<fir::ExtendedValue> args) {
   fir::ExtendedValue str = args[0];
   const auto *box = str.getBoxOf<fir::BoxValue>();
   mlir::Value addr =
-      builder.create<fir::BoxAddrOp>(loc, box->getMemTy(), fir::getBase(*box));
+      fir::BoxAddrOp::create(builder, loc, box->getMemTy(), fir::getBase(*box));
   fir::runtime::genPerror(builder, loc, addr);
 }
 
@@ -7307,7 +7322,7 @@ mlir::Value IntrinsicLibrary::genPopcnt(mlir::Type resultType,
                                         llvm::ArrayRef<mlir::Value> args) {
   assert(args.size() == 1);
 
-  mlir::Value count = builder.create<mlir::math::CtPopOp>(loc, args);
+  mlir::Value count = mlir::math::CtPopOp::create(builder, loc, args);
 
   return builder.createConvert(loc, resultType, count);
 }
@@ -7320,7 +7335,7 @@ mlir::Value IntrinsicLibrary::genPoppar(mlir::Type resultType,
   mlir::Value count = genPopcnt(resultType, args);
   mlir::Value one = builder.createIntegerConstant(loc, resultType, 1);
 
-  return builder.create<mlir::arith::AndIOp>(loc, count, one);
+  return mlir::arith::AndIOp::create(builder, loc, count, one);
 }
 
 // PRESENT
@@ -7328,8 +7343,8 @@ fir::ExtendedValue
 IntrinsicLibrary::genPresent(mlir::Type,
                              llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 1);
-  return builder.create<fir::IsPresentOp>(loc, builder.getI1Type(),
-                                          fir::getBase(args[0]));
+  return fir::IsPresentOp::create(builder, loc, builder.getI1Type(),
+                                  fir::getBase(args[0]));
 }
 
 // PRODUCT
@@ -7394,7 +7409,7 @@ void IntrinsicLibrary::genRandomSeed(llvm::ArrayRef<fir::ExtendedValue> args) {
   auto getDesc = [&](int i) {
     return isStaticallyPresent(args[i])
                ? fir::getBase(args[i])
-               : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+               : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
   };
   mlir::Value size = getDesc(0);
   mlir::Value put = getDesc(1);
@@ -7438,13 +7453,13 @@ IntrinsicLibrary::genReduce(mlir::Type resultType,
   bool absentDim = isStaticallyAbsent(args[2]);
 
   auto mask = isStaticallyAbsent(args[3])
-                  ? builder.create<fir::AbsentOp>(
-                        loc, fir::BoxType::get(builder.getI1Type()))
+                  ? fir::AbsentOp::create(
+                        builder, loc, fir::BoxType::get(builder.getI1Type()))
                   : builder.createBox(loc, args[3]);
 
   mlir::Value identity =
       isStaticallyAbsent(args[4])
-          ? builder.create<fir::AbsentOp>(loc, fir::ReferenceType::get(eleTy))
+          ? fir::AbsentOp::create(builder, loc, fir::ReferenceType::get(eleTy))
           : fir::getBase(args[4]);
 
   mlir::Value ordered = isStaticallyAbsent(args[5])
@@ -7460,7 +7475,7 @@ IntrinsicLibrary::genReduce(mlir::Type resultType,
                               ordered, result, argByRef);
       if (fir::isa_derived(eleTy))
         return result;
-      return builder.create<fir::LoadOp>(loc, result);
+      return fir::LoadOp::create(builder, loc, result);
     }
     if (fir::isa_char(eleTy)) {
       auto charTy = mlir::dyn_cast_or_null<fir::CharacterType>(resultType);
@@ -7510,7 +7525,7 @@ IntrinsicLibrary::genRename(std::optional<mlir::Type> resultType,
     auto statusAddr = builder.createTemporary(loc, *resultType);
     auto statusBox = builder.createBox(loc, statusAddr);
     fir::runtime::genRename(builder, loc, path1, path2, statusBox);
-    return builder.create<fir::LoadOp>(loc, statusAddr);
+    return fir::LoadOp::create(builder, loc, statusAddr);
   } else {
     // code-gen for the procedure form of RENAME
     mlir::Type boxNoneTy = fir::BoxType::get(builder.getNoneType());
@@ -7518,7 +7533,7 @@ IntrinsicLibrary::genRename(std::optional<mlir::Type> resultType,
     mlir::Value statusBox =
         isStaticallyPresent(status)
             ? fir::getBase(status)
-            : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+            : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
     fir::runtime::genRename(builder, loc, path1, path2, statusBox);
     return {};
   }
@@ -7563,16 +7578,18 @@ IntrinsicLibrary::genReshape(mlir::Type resultType,
     TODO(loc, "intrinsic: reshape requires computing rank of result");
 
   // Handle optional pad argument
-  mlir::Value pad = isStaticallyAbsent(args[2])
-                        ? builder.create<fir::AbsentOp>(
-                              loc, fir::BoxType::get(builder.getI1Type()))
-                        : builder.createBox(loc, args[2]);
+  mlir::Value pad =
+      isStaticallyAbsent(args[2])
+          ? fir::AbsentOp::create(builder, loc,
+                                  fir::BoxType::get(builder.getI1Type()))
+          : builder.createBox(loc, args[2]);
 
   // Handle optional order argument
-  mlir::Value order = isStaticallyAbsent(args[3])
-                          ? builder.create<fir::AbsentOp>(
-                                loc, fir::BoxType::get(builder.getI1Type()))
-                          : builder.createBox(loc, args[3]);
+  mlir::Value order =
+      isStaticallyAbsent(args[3])
+          ? fir::AbsentOp::create(builder, loc,
+                                  fir::BoxType::get(builder.getI1Type()))
+          : builder.createBox(loc, args[3]);
 
   // Create mutable fir.box to be passed to the runtime for the result.
   mlir::Type type = builder.getVarLenSeqTy(resultType, resultRank);
@@ -7643,26 +7660,27 @@ mlir::Value IntrinsicLibrary::genScale(mlir::Type resultType,
   // If X is finite and result is infinite, signal IEEE_OVERFLOW
   // If X is finite and scale(result, -I) != X, signal IEEE_UNDERFLOW
   fir::IfOp outerIfOp =
-      builder.create<fir::IfOp>(loc, genIsFPClass(i1Ty, args[0], finiteTest),
-                                /*withElseRegion=*/false);
+      fir::IfOp::create(builder, loc, genIsFPClass(i1Ty, args[0], finiteTest),
+                        /*withElseRegion=*/false);
   builder.setInsertionPointToStart(&outerIfOp.getThenRegion().front());
   fir::IfOp innerIfOp =
-      builder.create<fir::IfOp>(loc, genIsFPClass(i1Ty, result, infiniteTest),
-                                /*withElseRegion=*/true);
+      fir::IfOp::create(builder, loc, genIsFPClass(i1Ty, result, infiniteTest),
+                        /*withElseRegion=*/true);
   builder.setInsertionPointToStart(&innerIfOp.getThenRegion().front());
   genRaiseExcept(_FORTRAN_RUNTIME_IEEE_OVERFLOW |
                  _FORTRAN_RUNTIME_IEEE_INEXACT);
   builder.setInsertionPointToStart(&innerIfOp.getElseRegion().front());
-  mlir::Value minusI = builder.create<mlir::arith::MulIOp>(
-      loc, args[1], builder.createAllOnesInteger(loc, args[1].getType()));
+  mlir::Value minusI = mlir::arith::MulIOp::create(
+      builder, loc, args[1],
+      builder.createAllOnesInteger(loc, args[1].getType()));
   mlir::Value reverseResult = builder.createConvert(
       loc, resultType,
       fir::runtime::genScale(
           builder, loc, builder.createConvert(loc, f32Ty, result), minusI));
   genRaiseExcept(
       _FORTRAN_RUNTIME_IEEE_UNDERFLOW | _FORTRAN_RUNTIME_IEEE_INEXACT,
-      builder.create<mlir::arith::CmpFOp>(loc, mlir::arith::CmpFPredicate::ONE,
-                                          args[0], reverseResult));
+      mlir::arith::CmpFOp::create(builder, loc, mlir::arith::CmpFPredicate::ONE,
+                                  args[0], reverseResult));
   builder.setInsertionPointAfter(outerIfOp);
   return result;
 }
@@ -7714,13 +7732,14 @@ IntrinsicLibrary::genScan(mlir::Type resultType,
         builder.getContext(), builder.getKindMap().defaultLogicalKind());
     mlir::Value temp = builder.createTemporary(loc, logTy);
     mlir::Value castb = builder.createConvert(loc, logTy, b);
-    builder.create<fir::StoreOp>(loc, castb, temp);
+    fir::StoreOp::create(builder, loc, castb, temp);
     return builder.createBox(loc, temp);
   };
-  mlir::Value back = fir::isUnboxedValue(args[2])
-                         ? makeRefThenEmbox(*args[2].getUnboxed())
-                         : builder.create<fir::AbsentOp>(
-                               loc, fir::BoxType::get(builder.getI1Type()));
+  mlir::Value back =
+      fir::isUnboxedValue(args[2])
+          ? makeRefThenEmbox(*args[2].getUnboxed())
+          : fir::AbsentOp::create(builder, loc,
+                                  fir::BoxType::get(builder.getI1Type()));
 
   // Handle required string argument
   mlir::Value string = builder.createBox(loc, args[0]);
@@ -7761,7 +7780,7 @@ IntrinsicLibrary::genSecond(std::optional<mlir::Type> resultType,
   genCpuTime(subroutineArgs);
 
   if (resultType)
-    return builder.create<fir::LoadOp>(loc, fir::getBase(result));
+    return fir::LoadOp::create(builder, loc, fir::getBase(result));
   return {};
 }
 
@@ -7808,22 +7827,22 @@ IntrinsicLibrary::genSelectedRealKind(mlir::Type resultType,
   // Handle optional precision(P) argument
   mlir::Value precision =
       isStaticallyAbsent(args[0])
-          ? builder.create<fir::AbsentOp>(
-                loc, fir::ReferenceType::get(builder.getI1Type()))
+          ? fir::AbsentOp::create(builder, loc,
+                                  fir::ReferenceType::get(builder.getI1Type()))
           : fir::getBase(args[0]);
 
   // Handle optional range(R) argument
   mlir::Value range =
       isStaticallyAbsent(args[1])
-          ? builder.create<fir::AbsentOp>(
-                loc, fir::ReferenceType::get(builder.getI1Type()))
+          ? fir::AbsentOp::create(builder, loc,
+                                  fir::ReferenceType::get(builder.getI1Type()))
           : fir::getBase(args[1]);
 
   // Handle optional radix(RADIX) argument
   mlir::Value radix =
       isStaticallyAbsent(args[2])
-          ? builder.create<fir::AbsentOp>(
-                loc, fir::ReferenceType::get(builder.getI1Type()))
+          ? fir::AbsentOp::create(builder, loc,
+                                  fir::ReferenceType::get(builder.getI1Type()))
           : fir::getBase(args[2]);
 
   return builder.createConvert(
@@ -7861,9 +7880,9 @@ createBoxForRuntimeBoundInquiry(mlir::Location loc, fir::FirOpBuilder &builder,
         // shape information.
         mlir::Value localShape = builder.createShape(loc, array);
         mlir::Value oldBox = boxValue.getAddr();
-        return builder.create<fir::ReboxOp>(loc, oldBox.getType(), oldBox,
-                                            localShape,
-                                            /*slice=*/mlir::Value{});
+        return fir::ReboxOp::create(builder, loc, oldBox.getType(), oldBox,
+                                    localShape,
+                                    /*slice=*/mlir::Value{});
       },
       [&](const auto &) -> mlir::Value {
         // This is a pointer/allocatable, or an entity not yet tracked with a
@@ -7909,7 +7928,7 @@ genBoundInquiry(fir::FirOpBuilder &builder, mlir::Location loc,
     mlir::Value resultBase =
         builder.createConvert(loc, baseType, resultStorage);
     mlir::Value rankValue =
-        builder.create<fir::BoxRankOp>(loc, builder.getIndexType(), arrayBox);
+        fir::BoxRankOp::create(builder, loc, builder.getIndexType(), arrayBox);
     return fir::ArrayBoxValue{resultBase, {rankValue}};
   }
   // Result extent is a compile time constant in the other cases.
@@ -7939,9 +7958,9 @@ IntrinsicLibrary::genShape(mlir::Type resultType,
     mlir::Value extent = fir::factory::readExtent(builder, loc, array, dim);
     extent = builder.createConvert(loc, extentType, extent);
     auto index = builder.createIntegerConstant(loc, indexType, dim);
-    auto shapeAddr = builder.create<fir::CoordinateOp>(loc, shapeAddrType,
-                                                       shapeArray, index);
-    builder.create<fir::StoreOp>(loc, extent, shapeAddr);
+    auto shapeAddr = fir::CoordinateOp::create(builder, loc, shapeAddrType,
+                                               shapeArray, index);
+    fir::StoreOp::create(builder, loc, extent, shapeAddr);
   }
   mlir::Value shapeArrayExtent =
       builder.createIntegerConstant(loc, indexType, rank);
@@ -7967,18 +7986,18 @@ mlir::Value IntrinsicLibrary::genShift(mlir::Type resultType,
   mlir::Value zero = builder.createIntegerConstant(loc, signlessType, 0);
   mlir::Value shift = builder.createConvert(loc, signlessType, args[1]);
 
-  mlir::Value tooSmall = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::slt, shift, zero);
-  mlir::Value tooLarge = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::sge, shift, bitSize);
+  mlir::Value tooSmall = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::slt, shift, zero);
+  mlir::Value tooLarge = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::sge, shift, bitSize);
   mlir::Value outOfBounds =
-      builder.create<mlir::arith::OrIOp>(loc, tooSmall, tooLarge);
+      mlir::arith::OrIOp::create(builder, loc, tooSmall, tooLarge);
   mlir::Value word = args[0];
   if (word.getType().isUnsignedInteger())
     word = builder.createConvert(loc, signlessType, word);
-  mlir::Value shifted = builder.create<Shift>(loc, word, shift);
+  mlir::Value shifted = Shift::create(builder, loc, word, shift);
   mlir::Value result =
-      builder.create<mlir::arith::SelectOp>(loc, outOfBounds, zero, shifted);
+      mlir::arith::SelectOp::create(builder, loc, outOfBounds, zero, shifted);
   if (resultType.isUnsignedInteger())
     return builder.createConvert(loc, resultType, result);
   return result;
@@ -7993,8 +8012,8 @@ mlir::Value IntrinsicLibrary::genShiftA(mlir::Type resultType,
                              mlir::IntegerType::SignednessSemantics::Signless);
   mlir::Value bitSize = builder.createIntegerConstant(loc, signlessType, bits);
   mlir::Value shift = builder.createConvert(loc, signlessType, args[1]);
-  mlir::Value shiftGeBitSize = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::uge, shift, bitSize);
+  mlir::Value shiftGeBitSize = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::uge, shift, bitSize);
 
   // Lowering of mlir::arith::ShRSIOp is using `ashr`. `ashr` is undefined when
   // the shift amount is equal to the element size.
@@ -8006,13 +8025,13 @@ mlir::Value IntrinsicLibrary::genShiftA(mlir::Type resultType,
   mlir::Value word = args[0];
   if (word.getType().isUnsignedInteger())
     word = builder.createConvert(loc, signlessType, word);
-  mlir::Value valueIsNeg = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::slt, word, zero);
+  mlir::Value valueIsNeg = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::slt, word, zero);
   mlir::Value specialRes =
-      builder.create<mlir::arith::SelectOp>(loc, valueIsNeg, minusOne, zero);
-  mlir::Value shifted = builder.create<mlir::arith::ShRSIOp>(loc, word, shift);
-  mlir::Value result = builder.create<mlir::arith::SelectOp>(
-      loc, shiftGeBitSize, specialRes, shifted);
+      mlir::arith::SelectOp::create(builder, loc, valueIsNeg, minusOne, zero);
+  mlir::Value shifted = mlir::arith::ShRSIOp::create(builder, loc, word, shift);
+  mlir::Value result = mlir::arith::SelectOp::create(
+      builder, loc, shiftGeBitSize, specialRes, shifted);
   if (resultType.isUnsignedInteger())
     return builder.createConvert(loc, resultType, result);
   return result;
@@ -8037,10 +8056,10 @@ mlir::Value IntrinsicLibrary::genSign(mlir::Type resultType,
   if (mlir::isa<mlir::IntegerType>(resultType)) {
     mlir::Value abs = genAbs(resultType, {args[0]});
     mlir::Value zero = builder.createIntegerConstant(loc, resultType, 0);
-    auto neg = builder.create<mlir::arith::SubIOp>(loc, zero, abs);
-    auto cmp = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::slt, args[1], zero);
-    return builder.create<mlir::arith::SelectOp>(loc, cmp, neg, abs);
+    auto neg = mlir::arith::SubIOp::create(builder, loc, zero, abs);
+    auto cmp = mlir::arith::CmpIOp::create(
+        builder, loc, mlir::arith::CmpIPredicate::slt, args[1], zero);
+    return mlir::arith::SelectOp::create(builder, loc, cmp, neg, abs);
   }
   return genRuntimeCall("sign", resultType, args);
 }
@@ -8056,7 +8075,7 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   mlir::Value dfactor = builder.createRealConstant(
       loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  mlir::Value arg = mlir::arith::MulFOp::create(builder, loc, args[0], factor);
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
@@ -8100,14 +8119,14 @@ IntrinsicLibrary::genSize(mlir::Type resultType,
       .genThen([&]() {
         mlir::Value size = builder.createConvert(
             loc, resultType, fir::runtime::genSize(builder, loc, array));
-        builder.create<fir::ResultOp>(loc, size);
+        fir::ResultOp::create(builder, loc, size);
       })
       .genElse([&]() {
-        mlir::Value dimValue = builder.create<fir::LoadOp>(loc, dim);
+        mlir::Value dimValue = fir::LoadOp::create(builder, loc, dim);
         mlir::Value size = builder.createConvert(
             loc, resultType,
             fir::runtime::genSizeDim(builder, loc, array, dimValue));
-        builder.create<fir::ResultOp>(loc, size);
+        fir::ResultOp::create(builder, loc, size);
       })
       .getResults()[0];
 }
@@ -8118,12 +8137,13 @@ IntrinsicLibrary::genSizeOf(mlir::Type resultType,
                             llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 1);
   mlir::Value box = fir::getBase(args[0]);
-  mlir::Value eleSize = builder.create<fir::BoxEleSizeOp>(loc, resultType, box);
+  mlir::Value eleSize =
+      fir::BoxEleSizeOp::create(builder, loc, resultType, box);
   if (!fir::isArray(args[0]))
     return eleSize;
   mlir::Value arraySize = builder.createConvert(
       loc, resultType, fir::runtime::genSize(builder, loc, box));
-  return builder.create<mlir::arith::MulIOp>(loc, eleSize, arraySize);
+  return mlir::arith::MulIOp::create(builder, loc, eleSize, arraySize);
 }
 
 // TAND
@@ -8137,7 +8157,7 @@ mlir::Value IntrinsicLibrary::genTand(mlir::Type resultType,
   mlir::Value dfactor = builder.createRealConstant(
       loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  mlir::Value arg = mlir::arith::MulFOp::create(builder, loc, args[0], factor);
   return getRuntimeCallGenerator("tan", ftype)(builder, loc, {arg});
 }
 
@@ -8147,81 +8167,83 @@ mlir::Value IntrinsicLibrary::genThisGrid(mlir::Type resultType,
   assert(args.size() == 0);
   auto recTy = mlir::cast<fir::RecordType>(resultType);
   assert(recTy && "RecordType expepected");
-  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Value res = fir::AllocaOp::create(builder, loc, resultType);
   mlir::Type i32Ty = builder.getI32Type();
 
-  mlir::Value threadIdX = builder.create<mlir::NVVM::ThreadIdXOp>(loc, i32Ty);
-  mlir::Value threadIdY = builder.create<mlir::NVVM::ThreadIdYOp>(loc, i32Ty);
-  mlir::Value threadIdZ = builder.create<mlir::NVVM::ThreadIdZOp>(loc, i32Ty);
+  mlir::Value threadIdX = mlir::NVVM::ThreadIdXOp::create(builder, loc, i32Ty);
+  mlir::Value threadIdY = mlir::NVVM::ThreadIdYOp::create(builder, loc, i32Ty);
+  mlir::Value threadIdZ = mlir::NVVM::ThreadIdZOp::create(builder, loc, i32Ty);
 
-  mlir::Value blockIdX = builder.create<mlir::NVVM::BlockIdXOp>(loc, i32Ty);
-  mlir::Value blockIdY = builder.create<mlir::NVVM::BlockIdYOp>(loc, i32Ty);
-  mlir::Value blockIdZ = builder.create<mlir::NVVM::BlockIdZOp>(loc, i32Ty);
+  mlir::Value blockIdX = mlir::NVVM::BlockIdXOp::create(builder, loc, i32Ty);
+  mlir::Value blockIdY = mlir::NVVM::BlockIdYOp::create(builder, loc, i32Ty);
+  mlir::Value blockIdZ = mlir::NVVM::BlockIdZOp::create(builder, loc, i32Ty);
 
-  mlir::Value blockDimX = builder.create<mlir::NVVM::BlockDimXOp>(loc, i32Ty);
-  mlir::Value blockDimY = builder.create<mlir::NVVM::BlockDimYOp>(loc, i32Ty);
-  mlir::Value blockDimZ = builder.create<mlir::NVVM::BlockDimZOp>(loc, i32Ty);
-  mlir::Value gridDimX = builder.create<mlir::NVVM::GridDimXOp>(loc, i32Ty);
-  mlir::Value gridDimY = builder.create<mlir::NVVM::GridDimYOp>(loc, i32Ty);
-  mlir::Value gridDimZ = builder.create<mlir::NVVM::GridDimZOp>(loc, i32Ty);
+  mlir::Value blockDimX = mlir::NVVM::BlockDimXOp::create(builder, loc, i32Ty);
+  mlir::Value blockDimY = mlir::NVVM::BlockDimYOp::create(builder, loc, i32Ty);
+  mlir::Value blockDimZ = mlir::NVVM::BlockDimZOp::create(builder, loc, i32Ty);
+  mlir::Value gridDimX = mlir::NVVM::GridDimXOp::create(builder, loc, i32Ty);
+  mlir::Value gridDimY = mlir::NVVM::GridDimYOp::create(builder, loc, i32Ty);
+  mlir::Value gridDimZ = mlir::NVVM::GridDimZOp::create(builder, loc, i32Ty);
 
   // this_grid.size = ((blockDim.z * gridDim.z) * (blockDim.y * gridDim.y)) *
   // (blockDim.x * gridDim.x);
   mlir::Value resZ =
-      builder.create<mlir::arith::MulIOp>(loc, blockDimZ, gridDimZ);
+      mlir::arith::MulIOp::create(builder, loc, blockDimZ, gridDimZ);
   mlir::Value resY =
-      builder.create<mlir::arith::MulIOp>(loc, blockDimY, gridDimY);
+      mlir::arith::MulIOp::create(builder, loc, blockDimY, gridDimY);
   mlir::Value resX =
-      builder.create<mlir::arith::MulIOp>(loc, blockDimX, gridDimX);
-  mlir::Value resZY = builder.create<mlir::arith::MulIOp>(loc, resZ, resY);
-  mlir::Value size = builder.create<mlir::arith::MulIOp>(loc, resZY, resX);
+      mlir::arith::MulIOp::create(builder, loc, blockDimX, gridDimX);
+  mlir::Value resZY = mlir::arith::MulIOp::create(builder, loc, resZ, resY);
+  mlir::Value size = mlir::arith::MulIOp::create(builder, loc, resZY, resX);
 
   // tmp = ((blockIdx.z * gridDim.y * gridDim.x) + (blockIdx.y * gridDim.x)) +
   //   blockIdx.x;
   // this_group.rank = tmp * ((blockDim.x * blockDim.y) * blockDim.z) +
   //   ((threadIdx.z * blockDim.y) * blockDim.x) +
   //   (threadIdx.y * blockDim.x) + threadIdx.x + 1;
-  mlir::Value r1 = builder.create<mlir::arith::MulIOp>(loc, blockIdZ, gridDimY);
-  mlir::Value r2 = builder.create<mlir::arith::MulIOp>(loc, r1, gridDimX);
-  mlir::Value r3 = builder.create<mlir::arith::MulIOp>(loc, blockIdY, gridDimX);
-  mlir::Value r2r3 = builder.create<mlir::arith::AddIOp>(loc, r2, r3);
-  mlir::Value tmp = builder.create<mlir::arith::AddIOp>(loc, r2r3, blockIdX);
+  mlir::Value r1 =
+      mlir::arith::MulIOp::create(builder, loc, blockIdZ, gridDimY);
+  mlir::Value r2 = mlir::arith::MulIOp::create(builder, loc, r1, gridDimX);
+  mlir::Value r3 =
+      mlir::arith::MulIOp::create(builder, loc, blockIdY, gridDimX);
+  mlir::Value r2r3 = mlir::arith::AddIOp::create(builder, loc, r2, r3);
+  mlir::Value tmp = mlir::arith::AddIOp::create(builder, loc, r2r3, blockIdX);
 
   mlir::Value bXbY =
-      builder.create<mlir::arith::MulIOp>(loc, blockDimX, blockDimY);
+      mlir::arith::MulIOp::create(builder, loc, blockDimX, blockDimY);
   mlir::Value bXbYbZ =
-      builder.create<mlir::arith::MulIOp>(loc, bXbY, blockDimZ);
+      mlir::arith::MulIOp::create(builder, loc, bXbY, blockDimZ);
   mlir::Value tZbY =
-      builder.create<mlir::arith::MulIOp>(loc, threadIdZ, blockDimY);
+      mlir::arith::MulIOp::create(builder, loc, threadIdZ, blockDimY);
   mlir::Value tZbYbX =
-      builder.create<mlir::arith::MulIOp>(loc, tZbY, blockDimX);
+      mlir::arith::MulIOp::create(builder, loc, tZbY, blockDimX);
   mlir::Value tYbX =
-      builder.create<mlir::arith::MulIOp>(loc, threadIdY, blockDimX);
-  mlir::Value rank = builder.create<mlir::arith::MulIOp>(loc, tmp, bXbYbZ);
-  rank = builder.create<mlir::arith::AddIOp>(loc, rank, tZbYbX);
-  rank = builder.create<mlir::arith::AddIOp>(loc, rank, tYbX);
-  rank = builder.create<mlir::arith::AddIOp>(loc, rank, threadIdX);
+      mlir::arith::MulIOp::create(builder, loc, threadIdY, blockDimX);
+  mlir::Value rank = mlir::arith::MulIOp::create(builder, loc, tmp, bXbYbZ);
+  rank = mlir::arith::AddIOp::create(builder, loc, rank, tZbYbX);
+  rank = mlir::arith::AddIOp::create(builder, loc, rank, tYbX);
+  rank = mlir::arith::AddIOp::create(builder, loc, rank, threadIdX);
   mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-  rank = builder.create<mlir::arith::AddIOp>(loc, rank, one);
+  rank = mlir::arith::AddIOp::create(builder, loc, rank, one);
 
   auto sizeFieldName = recTy.getTypeList()[1].first;
   mlir::Type sizeFieldTy = recTy.getTypeList()[1].second;
   mlir::Type fieldIndexType = fir::FieldType::get(resultType.getContext());
-  mlir::Value sizeFieldIndex = builder.create<fir::FieldIndexOp>(
-      loc, fieldIndexType, sizeFieldName, recTy,
+  mlir::Value sizeFieldIndex = fir::FieldIndexOp::create(
+      builder, loc, fieldIndexType, sizeFieldName, recTy,
       /*typeParams=*/mlir::ValueRange{});
-  mlir::Value sizeCoord = builder.create<fir::CoordinateOp>(
-      loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
-  builder.create<fir::StoreOp>(loc, size, sizeCoord);
+  mlir::Value sizeCoord = fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
+  fir::StoreOp::create(builder, loc, size, sizeCoord);
 
   auto rankFieldName = recTy.getTypeList()[2].first;
   mlir::Type rankFieldTy = recTy.getTypeList()[2].second;
-  mlir::Value rankFieldIndex = builder.create<fir::FieldIndexOp>(
-      loc, fieldIndexType, rankFieldName, recTy,
+  mlir::Value rankFieldIndex = fir::FieldIndexOp::create(
+      builder, loc, fieldIndexType, rankFieldName, recTy,
       /*typeParams=*/mlir::ValueRange{});
-  mlir::Value rankCoord = builder.create<fir::CoordinateOp>(
-      loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
-  builder.create<fir::StoreOp>(loc, rank, rankCoord);
+  mlir::Value rankCoord = fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
+  fir::StoreOp::create(builder, loc, rank, rankCoord);
   return res;
 }
 
@@ -8232,50 +8254,50 @@ IntrinsicLibrary::genThisThreadBlock(mlir::Type resultType,
   assert(args.size() == 0);
   auto recTy = mlir::cast<fir::RecordType>(resultType);
   assert(recTy && "RecordType expepected");
-  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Value res = fir::AllocaOp::create(builder, loc, resultType);
   mlir::Type i32Ty = builder.getI32Type();
 
   // this_thread_block%size = blockDim.z * blockDim.y * blockDim.x;
-  mlir::Value blockDimX = builder.create<mlir::NVVM::BlockDimXOp>(loc, i32Ty);
-  mlir::Value blockDimY = builder.create<mlir::NVVM::BlockDimYOp>(loc, i32Ty);
-  mlir::Value blockDimZ = builder.create<mlir::NVVM::BlockDimZOp>(loc, i32Ty);
+  mlir::Value blockDimX = mlir::NVVM::BlockDimXOp::create(builder, loc, i32Ty);
+  mlir::Value blockDimY = mlir::NVVM::BlockDimYOp::create(builder, loc, i32Ty);
+  mlir::Value blockDimZ = mlir::NVVM::BlockDimZOp::create(builder, loc, i32Ty);
   mlir::Value size =
-      builder.create<mlir::arith::MulIOp>(loc, blockDimZ, blockDimY);
-  size = builder.create<mlir::arith::MulIOp>(loc, size, blockDimX);
+      mlir::arith::MulIOp::create(builder, loc, blockDimZ, blockDimY);
+  size = mlir::arith::MulIOp::create(builder, loc, size, blockDimX);
 
   // this_thread_block%rank = ((threadIdx.z * blockDim.y) * blockDim.x) +
   //   (threadIdx.y * blockDim.x) + threadIdx.x + 1;
-  mlir::Value threadIdX = builder.create<mlir::NVVM::ThreadIdXOp>(loc, i32Ty);
-  mlir::Value threadIdY = builder.create<mlir::NVVM::ThreadIdYOp>(loc, i32Ty);
-  mlir::Value threadIdZ = builder.create<mlir::NVVM::ThreadIdZOp>(loc, i32Ty);
+  mlir::Value threadIdX = mlir::NVVM::ThreadIdXOp::create(builder, loc, i32Ty);
+  mlir::Value threadIdY = mlir::NVVM::ThreadIdYOp::create(builder, loc, i32Ty);
+  mlir::Value threadIdZ = mlir::NVVM::ThreadIdZOp::create(builder, loc, i32Ty);
   mlir::Value r1 =
-      builder.create<mlir::arith::MulIOp>(loc, threadIdZ, blockDimY);
-  mlir::Value r2 = builder.create<mlir::arith::MulIOp>(loc, r1, blockDimX);
+      mlir::arith::MulIOp::create(builder, loc, threadIdZ, blockDimY);
+  mlir::Value r2 = mlir::arith::MulIOp::create(builder, loc, r1, blockDimX);
   mlir::Value r3 =
-      builder.create<mlir::arith::MulIOp>(loc, threadIdY, blockDimX);
-  mlir::Value r2r3 = builder.create<mlir::arith::AddIOp>(loc, r2, r3);
-  mlir::Value rank = builder.create<mlir::arith::AddIOp>(loc, r2r3, threadIdX);
+      mlir::arith::MulIOp::create(builder, loc, threadIdY, blockDimX);
+  mlir::Value r2r3 = mlir::arith::AddIOp::create(builder, loc, r2, r3);
+  mlir::Value rank = mlir::arith::AddIOp::create(builder, loc, r2r3, threadIdX);
   mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
-  rank = builder.create<mlir::arith::AddIOp>(loc, rank, one);
+  rank = mlir::arith::AddIOp::create(builder, loc, rank, one);
 
   auto sizeFieldName = recTy.getTypeList()[1].first;
   mlir::Type sizeFieldTy = recTy.getTypeList()[1].second;
   mlir::Type fieldIndexType = fir::FieldType::get(resultType.getContext());
-  mlir::Value sizeFieldIndex = builder.create<fir::FieldIndexOp>(
-      loc, fieldIndexType, sizeFieldName, recTy,
+  mlir::Value sizeFieldIndex = fir::FieldIndexOp::create(
+      builder, loc, fieldIndexType, sizeFieldName, recTy,
       /*typeParams=*/mlir::ValueRange{});
-  mlir::Value sizeCoord = builder.create<fir::CoordinateOp>(
-      loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
-  builder.create<fir::StoreOp>(loc, size, sizeCoord);
+  mlir::Value sizeCoord = fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
+  fir::StoreOp::create(builder, loc, size, sizeCoord);
 
   auto rankFieldName = recTy.getTypeList()[2].first;
   mlir::Type rankFieldTy = recTy.getTypeList()[2].second;
-  mlir::Value rankFieldIndex = builder.create<fir::FieldIndexOp>(
-      loc, fieldIndexType, rankFieldName, recTy,
+  mlir::Value rankFieldIndex = fir::FieldIndexOp::create(
+      builder, loc, fieldIndexType, rankFieldName, recTy,
       /*typeParams=*/mlir::ValueRange{});
-  mlir::Value rankCoord = builder.create<fir::CoordinateOp>(
-      loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
-  builder.create<fir::StoreOp>(loc, rank, rankCoord);
+  mlir::Value rankCoord = fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
+  fir::StoreOp::create(builder, loc, rank, rankCoord);
   return res;
 }
 
@@ -8285,7 +8307,7 @@ mlir::Value IntrinsicLibrary::genThisWarp(mlir::Type resultType,
   assert(args.size() == 0);
   auto recTy = mlir::cast<fir::RecordType>(resultType);
   assert(recTy && "RecordType expepected");
-  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Value res = fir::AllocaOp::create(builder, loc, resultType);
   mlir::Type i32Ty = builder.getI32Type();
 
   // coalesced_group%size = 32
@@ -8293,28 +8315,28 @@ mlir::Value IntrinsicLibrary::genThisWarp(mlir::Type resultType,
   auto sizeFieldName = recTy.getTypeList()[1].first;
   mlir::Type sizeFieldTy = recTy.getTypeList()[1].second;
   mlir::Type fieldIndexType = fir::FieldType::get(resultType.getContext());
-  mlir::Value sizeFieldIndex = builder.create<fir::FieldIndexOp>(
-      loc, fieldIndexType, sizeFieldName, recTy,
+  mlir::Value sizeFieldIndex = fir::FieldIndexOp::create(
+      builder, loc, fieldIndexType, sizeFieldName, recTy,
       /*typeParams=*/mlir::ValueRange{});
-  mlir::Value sizeCoord = builder.create<fir::CoordinateOp>(
-      loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
-  builder.create<fir::StoreOp>(loc, size, sizeCoord);
+  mlir::Value sizeCoord = fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
+  fir::StoreOp::create(builder, loc, size, sizeCoord);
 
   // coalesced_group%rank = threadIdx.x & 31 + 1
-  mlir::Value threadIdX = builder.create<mlir::NVVM::ThreadIdXOp>(loc, i32Ty);
+  mlir::Value threadIdX = mlir::NVVM::ThreadIdXOp::create(builder, loc, i32Ty);
   mlir::Value mask = builder.createIntegerConstant(loc, i32Ty, 31);
   mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
   mlir::Value masked =
-      builder.create<mlir::arith::AndIOp>(loc, threadIdX, mask);
-  mlir::Value rank = builder.create<mlir::arith::AddIOp>(loc, masked, one);
+      mlir::arith::AndIOp::create(builder, loc, threadIdX, mask);
+  mlir::Value rank = mlir::arith::AddIOp::create(builder, loc, masked, one);
   auto rankFieldName = recTy.getTypeList()[2].first;
   mlir::Type rankFieldTy = recTy.getTypeList()[2].second;
-  mlir::Value rankFieldIndex = builder.create<fir::FieldIndexOp>(
-      loc, fieldIndexType, rankFieldName, recTy,
+  mlir::Value rankFieldIndex = fir::FieldIndexOp::create(
+      builder, loc, fieldIndexType, rankFieldName, recTy,
       /*typeParams=*/mlir::ValueRange{});
-  mlir::Value rankCoord = builder.create<fir::CoordinateOp>(
-      loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
-  builder.create<fir::StoreOp>(loc, rank, rankCoord);
+  mlir::Value rankCoord = fir::CoordinateOp::create(
+      builder, loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
+  fir::StoreOp::create(builder, loc, rank, rankCoord);
   return res;
 }
 
@@ -8324,7 +8346,7 @@ mlir::Value IntrinsicLibrary::genTrailz(mlir::Type resultType,
   assert(args.size() == 1);
 
   mlir::Value result =
-      builder.create<mlir::math::CountTrailingZerosOp>(loc, args);
+      mlir::math::CountTrailingZerosOp::create(builder, loc, args);
 
   return builder.createConvert(loc, resultType, result);
 }
@@ -8352,10 +8374,10 @@ static mlir::Value computeLBOUND(fir::FirOpBuilder &builder, mlir::Location loc,
   zero = builder.createConvert(loc, extent.getType(), zero);
   // Note: for assumed size, the extent is -1, and the lower bound should
   // be returned. It is important to test extent == 0 and not extent > 0.
-  auto dimIsEmpty = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::eq, extent, zero);
+  auto dimIsEmpty = mlir::arith::CmpIOp::create(
+      builder, loc, mlir::arith::CmpIPredicate::eq, extent, zero);
   one = builder.createConvert(loc, lb.getType(), one);
-  return builder.create<mlir::arith::SelectOp>(loc, dimIsEmpty, one, lb);
+  return mlir::arith::SelectOp::create(builder, loc, dimIsEmpty, one, lb);
 }
 
 // LBOUND
@@ -8391,8 +8413,8 @@ IntrinsicLibrary::genLbound(mlir::Type resultType,
       lb = builder.createConvert(loc, lbType, lb);
       auto index = builder.createIntegerConstant(loc, indexType, dim);
       auto lbAddr =
-          builder.create<fir::CoordinateOp>(loc, lbAddrType, lbArray, index);
-      builder.create<fir::StoreOp>(loc, lb, lbAddr);
+          fir::CoordinateOp::create(builder, loc, lbAddrType, lbArray, index);
+      fir::StoreOp::create(builder, loc, lb, lbAddr);
     }
     mlir::Value lbArrayExtent =
         builder.createIntegerConstant(loc, indexType, rank);
@@ -8431,8 +8453,8 @@ IntrinsicLibrary::genUbound(mlir::Type resultType,
     mlir::Value lbound = fir::getBase(genLbound(resultType, args));
 
     mlir::Value one = builder.createIntegerConstant(loc, resultType, 1);
-    mlir::Value ubound = builder.create<mlir::arith::SubIOp>(loc, lbound, one);
-    return builder.create<mlir::arith::AddIOp>(loc, ubound, extent);
+    mlir::Value ubound = mlir::arith::SubIOp::create(builder, loc, lbound, one);
+    return mlir::arith::AddIOp::create(builder, loc, ubound, extent);
   }
   // Handle calls to UBOUND without the DIM argument, which return an array
   int kindPos = args.size() == 2 ? 1 : 2;
@@ -8526,9 +8548,9 @@ IntrinsicLibrary::genStorageSize(mlir::Type resultType,
 
   box = builder.createBox(loc, args[0],
                           /*isPolymorphic=*/args[0].isPolymorphic());
-  mlir::Value eleSize = builder.create<fir::BoxEleSizeOp>(loc, kindTy, box);
+  mlir::Value eleSize = fir::BoxEleSizeOp::create(builder, loc, kindTy, box);
   mlir::Value c8 = builder.createIntegerConstant(loc, kindTy, 8);
-  return builder.create<mlir::arith::MulIOp>(loc, eleSize, c8);
+  return mlir::arith::MulIOp::create(builder, loc, eleSize, c8);
 }
 
 // SUM
@@ -8541,7 +8563,7 @@ IntrinsicLibrary::genSum(mlir::Type resultType,
 
 // SYNCTHREADS
 void IntrinsicLibrary::genSyncThreads(llvm::ArrayRef<fir::ExtendedValue> args) {
-  builder.create<mlir::NVVM::Barrier0Op>(loc);
+  mlir::NVVM::Barrier0Op::create(builder, loc);
 }
 
 // SYNCTHREADS_AND
@@ -8553,7 +8575,7 @@ IntrinsicLibrary::genSyncThreadsAnd(mlir::Type resultType,
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
   auto funcOp = builder.createFunction(loc, funcName, ftype);
-  return builder.create<fir::CallOp>(loc, funcOp, args).getResult(0);
+  return fir::CallOp::create(builder, loc, funcOp, args).getResult(0);
 }
 
 // SYNCTHREADS_COUNT
@@ -8565,7 +8587,7 @@ IntrinsicLibrary::genSyncThreadsCount(mlir::Type resultType,
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
   auto funcOp = builder.createFunction(loc, funcName, ftype);
-  return builder.create<fir::CallOp>(loc, funcOp, args).getResult(0);
+  return fir::CallOp::create(builder, loc, funcOp, args).getResult(0);
 }
 
 // SYNCTHREADS_OR
@@ -8577,7 +8599,7 @@ IntrinsicLibrary::genSyncThreadsOr(mlir::Type resultType,
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
   auto funcOp = builder.createFunction(loc, funcName, ftype);
-  return builder.create<fir::CallOp>(loc, funcOp, args).getResult(0);
+  return fir::CallOp::create(builder, loc, funcOp, args).getResult(0);
 }
 
 // SYNCWARP
@@ -8589,7 +8611,7 @@ void IntrinsicLibrary::genSyncWarp(llvm::ArrayRef<fir::ExtendedValue> args) {
       mlir::FunctionType::get(builder.getContext(), {mask.getType()}, {});
   auto funcOp = builder.createFunction(loc, funcName, funcType);
   llvm::SmallVector<mlir::Value> argsList{mask};
-  builder.create<fir::CallOp>(loc, funcOp, argsList);
+  fir::CallOp::create(builder, loc, funcOp, argsList);
 }
 
 // SYSTEM
@@ -8615,25 +8637,26 @@ IntrinsicLibrary::genSystem(std::optional<mlir::Type> resultType,
   mlir::Value exitstatBox =
       isStaticallyPresent(exitstat)
           ? fir::getBase(exitstat)
-          : builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+          : fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
 
   // Create a dummmy cmdstat to prevent EXECUTE_COMMAND_LINE terminate itself
   // when cmdstat is assigned with a non-zero value but not present
   mlir::Value tempValue =
       builder.createIntegerConstant(loc, builder.getI16Type(), 0);
   mlir::Value temp = builder.createTemporary(loc, builder.getI16Type());
-  builder.create<fir::StoreOp>(loc, tempValue, temp);
+  fir::StoreOp::create(builder, loc, tempValue, temp);
   mlir::Value cmdstatBox = builder.createBox(loc, temp);
 
   mlir::Value cmdmsgBox =
-      builder.create<fir::AbsentOp>(loc, boxNoneTy).getResult();
+      fir::AbsentOp::create(builder, loc, boxNoneTy).getResult();
 
   fir::runtime::genExecuteCommandLine(builder, loc, command, waitBool,
                                       exitstatBox, cmdstatBox, cmdmsgBox);
 
   if (resultType) {
-    mlir::Value exitstatAddr = builder.create<fir::BoxAddrOp>(loc, exitstatBox);
-    return builder.create<fir::LoadOp>(loc, fir::getBase(exitstatAddr));
+    mlir::Value exitstatAddr =
+        fir::BoxAddrOp::create(builder, loc, exitstatBox);
+    return fir::LoadOp::create(builder, loc, fir::getBase(exitstatAddr));
   }
   return {};
 }
@@ -8729,7 +8752,7 @@ void IntrinsicLibrary::genThreadFence(llvm::ArrayRef<fir::ExtendedValue> args) {
       mlir::FunctionType::get(builder.getContext(), {}, {});
   auto funcOp = builder.createFunction(loc, funcName, funcType);
   llvm::SmallVector<mlir::Value> noArgs;
-  builder.create<fir::CallOp>(loc, funcOp, noArgs);
+  fir::CallOp::create(builder, loc, funcOp, noArgs);
 }
 
 // THREADFENCE_BLOCK
@@ -8740,7 +8763,7 @@ void IntrinsicLibrary::genThreadFenceBlock(
       mlir::FunctionType::get(builder.getContext(), {}, {});
   auto funcOp = builder.createFunction(loc, funcName, funcType);
   llvm::SmallVector<mlir::Value> noArgs;
-  builder.create<fir::CallOp>(loc, funcOp, noArgs);
+  fir::CallOp::create(builder, loc, funcOp, noArgs);
 }
 
 // THREADFENCE_SYSTEM
@@ -8751,7 +8774,7 @@ void IntrinsicLibrary::genThreadFenceSystem(
       mlir::FunctionType::get(builder.getContext(), {}, {});
   auto funcOp = builder.createFunction(loc, funcName, funcType);
   llvm::SmallVector<mlir::Value> noArgs;
-  builder.create<fir::CallOp>(loc, funcOp, noArgs);
+  fir::CallOp::create(builder, loc, funcOp, noArgs);
 }
 
 // TIME
@@ -8803,29 +8826,30 @@ static mlir::Value createExtremumCompare(mlir::Location loc,
       // Return the number if one of the inputs is NaN and the other is
       // a number.
       auto leftIsResult =
-          builder.create<mlir::arith::CmpFOp>(loc, orderedCmp, left, right);
-      auto rightIsNan = builder.create<mlir::arith::CmpFOp>(
-          loc, mlir::arith::CmpFPredicate::UNE, right, right);
+          mlir::arith::CmpFOp::create(builder, loc, orderedCmp, left, right);
+      auto rightIsNan = mlir::arith::CmpFOp::create(
+          builder, loc, mlir::arith::CmpFPredicate::UNE, right, right);
       result =
-          builder.create<mlir::arith::OrIOp>(loc, leftIsResult, rightIsNan);
+          mlir::arith::OrIOp::create(builder, loc, leftIsResult, rightIsNan);
     } else if constexpr (behavior == ExtremumBehavior::IeeeMinMaximum) {
       // Always return NaNs if one the input is NaNs
       auto leftIsResult =
-          builder.create<mlir::arith::CmpFOp>(loc, orderedCmp, left, right);
-      auto leftIsNan = builder.create<mlir::arith::CmpFOp>(
-          loc, mlir::arith::CmpFPredicate::UNE, left, left);
-      result = builder.create<mlir::arith::OrIOp>(loc, leftIsResult, leftIsNan);
+          mlir::arith::CmpFOp::create(builder, loc, orderedCmp, left, right);
+      auto leftIsNan = mlir::arith::CmpFOp::create(
+          builder, loc, mlir::arith::CmpFPredicate::UNE, left, left);
+      result =
+          mlir::arith::OrIOp::create(builder, loc, leftIsResult, leftIsNan);
     } else if constexpr (behavior == ExtremumBehavior::MinMaxss) {
       // If the left is a NaN, return the right whatever it is.
       result =
-          builder.create<mlir::arith::CmpFOp>(loc, orderedCmp, left, right);
+          mlir::arith::CmpFOp::create(builder, loc, orderedCmp, left, right);
     } else if constexpr (behavior == ExtremumBehavior::PgfortranLlvm) {
       // If one of the operand is a NaN, return left whatever it is.
       static constexpr auto unorderedCmp =
           extremum == Extremum::Max ? mlir::arith::CmpFPredicate::UGT
                                     : mlir::arith::CmpFPredicate::ULT;
       result =
-          builder.create<mlir::arith::CmpFOp>(loc, unorderedCmp, left, right);
+          mlir::arith::CmpFOp::create(builder, loc, unorderedCmp, left, right);
     } else {
       // TODO: ieeeMinNum/ieeeMaxNum
       static_assert(behavior == ExtremumBehavior::IeeeMinMaxNum,
@@ -8839,8 +8863,8 @@ static mlir::Value createExtremumCompare(mlir::Location loc,
       left = builder.createConvert(loc, signlessType, left);
       right = builder.createConvert(loc, signlessType, right);
     }
-    result =
-        builder.create<mlir::arith::CmpIOp>(loc, integerPredicate, left, right);
+    result = mlir::arith::CmpIOp::create(builder, loc, integerPredicate, left,
+                                         right);
   } else if (fir::isa_char(type) || fir::isa_char(fir::unwrapRefType(type))) {
     // TODO: ! character min and max is tricky because the result
     // length is the length of the longest argument!
@@ -8961,13 +8985,14 @@ IntrinsicLibrary::genVerify(mlir::Type resultType,
         builder.getContext(), builder.getKindMap().defaultLogicalKind());
     mlir::Value temp = builder.createTemporary(loc, logTy);
     mlir::Value castb = builder.createConvert(loc, logTy, b);
-    builder.create<fir::StoreOp>(loc, castb, temp);
+    fir::StoreOp::create(builder, loc, castb, temp);
     return builder.createBox(loc, temp);
   };
-  mlir::Value back = fir::isUnboxedValue(args[2])
-                         ? makeRefThenEmbox(*args[2].getUnboxed())
-                         : builder.create<fir::AbsentOp>(
-                               loc, fir::BoxType::get(builder.getI1Type()));
+  mlir::Value back =
+      fir::isUnboxedValue(args[2])
+          ? makeRefThenEmbox(*args[2].getUnboxed())
+          : fir::AbsentOp::create(builder, loc,
+                                  fir::BoxType::get(builder.getI1Type()));
 
   // Handle required string argument
   mlir::Value string = builder.createBox(loc, args[0]);
@@ -9007,8 +9032,8 @@ IntrinsicLibrary::genExtremumloc(FN func, FD funcDim, llvm::StringRef errMsg,
 
   // Handle optional mask argument
   auto mask = isStaticallyAbsent(args[2])
-                  ? builder.create<fir::AbsentOp>(
-                        loc, fir::BoxType::get(builder.getI1Type()))
+                  ? fir::AbsentOp::create(
+                        builder, loc, fir::BoxType::get(builder.getI1Type()))
                   : builder.createBox(loc, args[2]);
 
   // Handle optional kind argument
@@ -9090,8 +9115,8 @@ IntrinsicLibrary::genExtremumVal(FN func, FD funcDim, FC funcChar,
 
   // Handle optional mask argument
   auto mask = isStaticallyAbsent(args[2])
-                  ? builder.create<fir::AbsentOp>(
-                        loc, fir::BoxType::get(builder.getI1Type()))
+                  ? fir::AbsentOp::create(
+                        builder, loc, fir::BoxType::get(builder.getI1Type()))
                   : builder.createBox(loc, args[2]);
 
   bool absentDim = isStaticallyAbsent(args[1]);
@@ -9155,7 +9180,7 @@ mlir::Value IntrinsicLibrary::genExtremum(mlir::Type,
   for (auto arg : args.drop_front()) {
     mlir::Value mask =
         createExtremumCompare<extremum, behavior>(loc, builder, result, arg);
-    result = builder.create<mlir::arith::SelectOp>(loc, mask, result, arg);
+    result = mlir::arith::SelectOp::create(builder, loc, mask, result, arg);
   }
   return result;
 }

diff  --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp
index 93abedc43936d..50c945df5b465 100644
--- a/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -35,7 +35,7 @@ createNewFirBox(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value shape;
   if (!extents.empty()) {
     if (lbounds.empty()) {
-      shape = builder.create<fir::ShapeOp>(loc, extents);
+      shape = fir::ShapeOp::create(builder, loc, extents);
     } else {
       llvm::SmallVector<mlir::Value> shapeShiftBounds;
       for (auto [lb, extent] : llvm::zip(lbounds, extents)) {
@@ -44,8 +44,8 @@ createNewFirBox(fir::FirOpBuilder &builder, mlir::Location loc,
       }
       auto shapeShiftType =
           fir::ShapeShiftType::get(builder.getContext(), extents.size());
-      shape = builder.create<fir::ShapeShiftOp>(loc, shapeShiftType,
-                                                shapeShiftBounds);
+      shape = fir::ShapeShiftOp::create(builder, loc, shapeShiftType,
+                                        shapeShiftBounds);
     }
   } // Otherwise, this a scalar. Leave the shape empty.
 
@@ -78,8 +78,8 @@ createNewFirBox(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value emptySlice;
   auto boxType = fir::updateTypeWithVolatility(
       box.getBoxTy(), fir::isa_volatile_type(cleanedAddr.getType()));
-  return builder.create<fir::EmboxOp>(loc, boxType, cleanedAddr, shape,
-                                      emptySlice, cleanedLengths, tdesc);
+  return fir::EmboxOp::create(builder, loc, boxType, cleanedAddr, shape,
+                              emptySlice, cleanedLengths, tdesc);
 }
 
 //===----------------------------------------------------------------------===//
@@ -106,7 +106,7 @@ class MutablePropertyReader {
                         bool forceIRBoxRead = false)
       : builder{builder}, loc{loc}, box{box} {
     if (forceIRBoxRead || !box.isDescribedByVariables())
-      irBox = builder.create<fir::LoadOp>(loc, box.getAddr());
+      irBox = fir::LoadOp::create(builder, loc, box.getAddr());
   }
   /// Get base address of allocated/associated entity.
   mlir::Value readBaseAddress() {
@@ -114,10 +114,10 @@ class MutablePropertyReader {
       auto memrefTy = box.getBoxTy().getEleTy();
       if (!fir::isa_ref_type(memrefTy))
         memrefTy = builder.getRefType(memrefTy);
-      return builder.create<fir::BoxAddrOp>(loc, memrefTy, irBox);
+      return fir::BoxAddrOp::create(builder, loc, memrefTy, irBox);
     }
     auto addrVar = box.getMutableProperties().addr;
-    return builder.create<fir::LoadOp>(loc, addrVar);
+    return fir::LoadOp::create(builder, loc, addrVar);
   }
   /// Return {lbound, extent} values read from the MutableBoxValue given
   /// the dimension.
@@ -125,13 +125,14 @@ class MutablePropertyReader {
     auto idxTy = builder.getIndexType();
     if (irBox) {
       auto dimVal = builder.createIntegerConstant(loc, idxTy, dim);
-      auto dimInfo = builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy,
-                                                    irBox, dimVal);
+      auto dimInfo = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy,
+                                            irBox, dimVal);
       return {dimInfo.getResult(0), dimInfo.getResult(1)};
     }
     const auto &mutableProperties = box.getMutableProperties();
-    auto lb = builder.create<fir::LoadOp>(loc, mutableProperties.lbounds[dim]);
-    auto ext = builder.create<fir::LoadOp>(loc, mutableProperties.extents[dim]);
+    auto lb = fir::LoadOp::create(builder, loc, mutableProperties.lbounds[dim]);
+    auto ext =
+        fir::LoadOp::create(builder, loc, mutableProperties.extents[dim]);
     return {lb, ext};
   }
 
@@ -146,7 +147,7 @@ class MutablePropertyReader {
     const auto &deferred = box.getMutableProperties().deferredParams;
     if (deferred.empty())
       fir::emitFatalError(loc, "allocatable entity has no length property");
-    return builder.create<fir::LoadOp>(loc, deferred[0]);
+    return fir::LoadOp::create(builder, loc, deferred[0]);
   }
 
   /// Read and return all extents. If \p lbounds vector is provided, lbounds are
@@ -223,7 +224,7 @@ class MutablePropertyWriter {
   /// all that can be described in the new fir.box (e.g. non contiguous entity).
   void updateWithIrBox(mlir::Value newBox) {
     assert(!box.isDescribedByVariables());
-    builder.create<fir::StoreOp>(loc, newBox, box.getAddr());
+    fir::StoreOp::create(builder, loc, newBox, box.getAddr());
   }
   /// Set unallocated/disassociated status for the entity described by
   /// MutableBoxValue. Deallocation is not performed by this helper.
@@ -231,8 +232,8 @@ class MutablePropertyWriter {
     if (box.isDescribedByVariables()) {
       auto addrVar = box.getMutableProperties().addr;
       auto nullTy = fir::dyn_cast_ptrEleTy(addrVar.getType());
-      builder.create<fir::StoreOp>(loc, builder.createNullConstant(loc, nullTy),
-                                   addrVar);
+      fir::StoreOp::create(builder, loc,
+                           builder.createNullConstant(loc, nullTy), addrVar);
     } else {
       // Note that the dynamic type of polymorphic entities must be reset to the
       // declaration type of the mutable box. See Fortran 2018 7.8.2 NOTE 1.
@@ -246,7 +247,7 @@ class MutablePropertyWriter {
       auto deallocatedBox = fir::factory::createUnallocatedBox(
           builder, loc, box.getBoxTy(), box.nonDeferredLenParams(),
           typeSourceBox, allocator);
-      builder.create<fir::StoreOp>(loc, deallocatedBox, box.getAddr());
+      fir::StoreOp::create(builder, loc, deallocatedBox, box.getAddr());
     }
   }
 
@@ -286,7 +287,7 @@ class MutablePropertyWriter {
     const bool valueTypeIsVolatile =
         fir::isa_volatile_type(fir::unwrapRefType(box.getAddr().getType()));
     irBox = builder.createVolatileCast(loc, valueTypeIsVolatile, irBox);
-    builder.create<fir::StoreOp>(loc, irBox, box.getAddr());
+    fir::StoreOp::create(builder, loc, irBox, box.getAddr());
   }
 
   /// Update the set of property variables of the MutableBoxValue.
@@ -295,8 +296,8 @@ class MutablePropertyWriter {
                                mlir::ValueRange lengths) {
     auto castAndStore = [&](mlir::Value val, mlir::Value addr) {
       auto type = fir::dyn_cast_ptrEleTy(addr.getType());
-      builder.create<fir::StoreOp>(loc, builder.createConvert(loc, type, val),
-                                   addr);
+      fir::StoreOp::create(builder, loc, builder.createConvert(loc, type, val),
+                           addr);
     };
     const auto &mutableProperties = box.getMutableProperties();
     castAndStore(addr, mutableProperties.addr);
@@ -379,8 +380,8 @@ mlir::Value fir::factory::createUnallocatedBox(
     }
   }
   mlir::Value emptySlice;
-  auto embox = builder.create<fir::EmboxOp>(
-      loc, baseBoxType, nullAddr, shape, emptySlice, lenParams, typeSourceBox);
+  auto embox = fir::EmboxOp::create(builder, loc, baseBoxType, nullAddr, shape,
+                                    emptySlice, lenParams, typeSourceBox);
   if (allocator != 0)
     embox.setAllocatorIdx(allocator);
   if (isAssumedRank)
@@ -459,7 +460,7 @@ fir::factory::genMutableBoxRead(fir::FirOpBuilder &builder, mlir::Location loc,
   }
   mlir::Value sourceBox;
   if (box.isPolymorphic())
-    sourceBox = builder.create<fir::LoadOp>(loc, box.getAddr());
+    sourceBox = fir::LoadOp::create(builder, loc, box.getAddr());
   if (rank)
     return fir::ArrayBoxValue{addr, extents, lbounds, sourceBox};
   if (box.isPolymorphic())
@@ -490,7 +491,7 @@ static void genFreemem(fir::FirOpBuilder &builder, mlir::Location loc,
   // so make sure the heap type is restored before deallocation.
   auto cast = builder.createConvert(
       loc, fir::HeapType::get(fir::dyn_cast_ptrEleTy(addr.getType())), addr);
-  builder.create<fir::FreeMemOp>(loc, cast);
+  fir::FreeMemOp::create(builder, loc, cast);
 }
 
 void fir::factory::genFreememIfAllocated(fir::FirOpBuilder &builder,
@@ -498,8 +499,8 @@ void fir::factory::genFreememIfAllocated(fir::FirOpBuilder &builder,
                                          const fir::MutableBoxValue &box) {
   auto addr = MutablePropertyReader(builder, loc, box).readBaseAddress();
   auto isAllocated = builder.genIsNotNullAddr(loc, addr);
-  auto ifOp = builder.create<fir::IfOp>(loc, isAllocated,
-                                        /*withElseRegion=*/false);
+  auto ifOp = fir::IfOp::create(builder, loc, isAllocated,
+                                /*withElseRegion=*/false);
   auto insPt = builder.saveInsertionPoint();
   builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
   ::genFreemem(builder, loc, addr);
@@ -553,15 +554,15 @@ void fir::factory::associateMutableBox(fir::FirOpBuilder &builder,
         if (box.hasAssumedRank()) {
           assert(arr.hasAssumedRank() &&
                  "expect both arr and box to be assumed-rank");
-          mlir::Value reboxed = builder.create<fir::ReboxAssumedRankOp>(
-              loc, box.getBoxTy(), arr.getAddr(),
+          mlir::Value reboxed = fir::ReboxAssumedRankOp::create(
+              builder, loc, box.getBoxTy(), arr.getAddr(),
               fir::LowerBoundModifierAttribute::Preserve);
           writer.updateWithIrBox(reboxed);
         } else if (box.isDescribedByVariables()) {
           // LHS is a contiguous pointer described by local variables. Open RHS
           // fir.box to update the LHS.
-          auto rawAddr = builder.create<fir::BoxAddrOp>(loc, arr.getMemTy(),
-                                                        arr.getAddr());
+          auto rawAddr = fir::BoxAddrOp::create(builder, loc, arr.getMemTy(),
+                                                arr.getAddr());
           auto extents = fir::factory::getExtents(loc, builder, source);
           llvm::SmallVector<mlir::Value> lenParams;
           if (arr.isCharacter()) {
@@ -576,11 +577,11 @@ void fir::factory::associateMutableBox(fir::FirOpBuilder &builder,
           if (!newLbounds.empty()) {
             auto shiftType =
                 fir::ShiftType::get(builder.getContext(), newLbounds.size());
-            shift = builder.create<fir::ShiftOp>(loc, shiftType, newLbounds);
+            shift = fir::ShiftOp::create(builder, loc, shiftType, newLbounds);
           }
           auto reboxed =
-              builder.create<fir::ReboxOp>(loc, box.getBoxTy(), arr.getAddr(),
-                                           shift, /*slice=*/mlir::Value());
+              fir::ReboxOp::create(builder, loc, box.getBoxTy(), arr.getAddr(),
+                                   shift, /*slice=*/mlir::Value());
           writer.updateWithIrBox(reboxed);
         }
       },
@@ -608,9 +609,9 @@ void fir::factory::associateMutableBoxWithRemap(
     for (auto [lb, ub] : llvm::zip(lbounds, ubounds)) {
       auto lbi = builder.createConvert(loc, idxTy, lb);
       auto ubi = builder.createConvert(loc, idxTy, ub);
-      auto 
diff  = builder.create<mlir::arith::SubIOp>(loc, idxTy, ubi, lbi);
+      auto 
diff  = mlir::arith::SubIOp::create(builder, loc, idxTy, ubi, lbi);
       extents.emplace_back(
-          builder.create<mlir::arith::AddIOp>(loc, idxTy, 
diff , one));
+          mlir::arith::AddIOp::create(builder, loc, idxTy, 
diff , one));
     }
   } else {
     // lbounds are default. Upper bounds and extents are the same.
@@ -657,8 +658,8 @@ void fir::factory::associateMutableBoxWithRemap(
         if (box.isDescribedByVariables()) {
           // LHS is a contiguous pointer described by local variables. Open RHS
           // fir.box to update the LHS.
-          auto rawAddr = builder.create<fir::BoxAddrOp>(loc, arr.getMemTy(),
-                                                        arr.getAddr());
+          auto rawAddr = fir::BoxAddrOp::create(builder, loc, arr.getMemTy(),
+                                                arr.getAddr());
           llvm::SmallVector<mlir::Value> lenParams;
           if (arr.isCharacter()) {
             lenParams.emplace_back(
@@ -678,10 +679,10 @@ void fir::factory::associateMutableBoxWithRemap(
             shapeArgs.push_back(ext);
           }
           auto shape =
-              builder.create<fir::ShapeShiftOp>(loc, shapeType, shapeArgs);
+              fir::ShapeShiftOp::create(builder, loc, shapeType, shapeArgs);
           auto reboxed =
-              builder.create<fir::ReboxOp>(loc, box.getBoxTy(), arr.getAddr(),
-                                           shape, /*slice=*/mlir::Value());
+              fir::ReboxOp::create(builder, loc, box.getBoxTy(), arr.getAddr(),
+                                   shape, /*slice=*/mlir::Value());
           writer.updateWithIrBox(reboxed);
         }
       },
@@ -748,8 +749,8 @@ static mlir::Value allocateAndInitNewStorage(fir::FirOpBuilder &builder,
                                              mlir::ValueRange lenParams,
                                              llvm::StringRef allocName) {
   auto lengths = getNewLengths(builder, loc, box, lenParams);
-  auto newStorage = builder.create<fir::AllocMemOp>(
-      loc, box.getBaseTy(), allocName, lengths, extents);
+  auto newStorage = fir::AllocMemOp::create(builder, loc, box.getBaseTy(),
+                                            allocName, lengths, extents);
   if (mlir::isa<fir::RecordType>(box.getEleTy())) {
     // TODO: skip runtime initialization if this is not required. Currently,
     // there is no way to know here if a derived type needs it or not. But the
@@ -771,8 +772,8 @@ void fir::factory::genInlinedAllocation(
   llvm::SmallVector<mlir::Value> safeExtents;
   for (mlir::Value extent : extents)
     safeExtents.push_back(fir::factory::genMaxWithZero(builder, loc, extent));
-  auto heap = builder.create<fir::AllocMemOp>(loc, box.getBaseTy(), allocName,
-                                              lengths, safeExtents);
+  auto heap = fir::AllocMemOp::create(builder, loc, box.getBaseTy(), allocName,
+                                      lengths, safeExtents);
   MutablePropertyWriter{builder, loc, box}.updateMutableBox(
       heap, lbounds, safeExtents, lengths);
   if (mlir::isa<fir::RecordType>(box.getEleTy())) {
@@ -841,10 +842,11 @@ fir::factory::MutableBoxReallocation fir::factory::genReallocIfNeeded(
                                        mlir::Value required) {
               auto castPrevious =
                   builder.createConvert(loc, required.getType(), previous);
-              auto cmp = builder.create<mlir::arith::CmpIOp>(
-                  loc, mlir::arith::CmpIPredicate::ne, castPrevious, required);
-              mustReallocate = builder.create<mlir::arith::SelectOp>(
-                  loc, cmp, cmp, mustReallocate);
+              auto cmp = mlir::arith::CmpIOp::create(
+                  builder, loc, mlir::arith::CmpIPredicate::ne, castPrevious,
+                  required);
+              mustReallocate = mlir::arith::SelectOp::create(
+                  builder, loc, cmp, cmp, mustReallocate);
             };
             llvm::SmallVector<mlir::Value> previousExtents = reader.readShape();
             if (!shape.empty())
@@ -879,17 +881,17 @@ fir::factory::MutableBoxReallocation fir::factory::genReallocIfNeeded(
                                   ".auto.alloc");
                               if (storageHandler)
                                 storageHandler(getExtValForStorage(heap));
-                              builder.create<fir::ResultOp>(loc, heap);
+                              fir::ResultOp::create(builder, loc, heap);
                             })
                             .genElse([&]() {
                               if (storageHandler)
                                 storageHandler(getExtValForStorage(addr));
-                              builder.create<fir::ResultOp>(loc, addr);
+                              fir::ResultOp::create(builder, loc, addr);
                             });
             ifOp.end();
             auto newAddr = ifOp.getResults()[0];
-            builder.create<fir::ResultOp>(
-                loc, mlir::ValueRange{mustReallocate, newAddr});
+            fir::ResultOp::create(builder, loc,
+                                  mlir::ValueRange{mustReallocate, newAddr});
           })
           .genElse([&]() {
             auto trueValue = builder.createBool(loc, true);
@@ -900,15 +902,15 @@ fir::factory::MutableBoxReallocation fir::factory::genReallocIfNeeded(
                   builder, loc,
                   "array left hand side must be allocated when the right hand "
                   "side is a scalar");
-              builder.create<fir::ResultOp>(loc,
-                                            mlir::ValueRange{trueValue, addr});
+              fir::ResultOp::create(builder, loc,
+                                    mlir::ValueRange{trueValue, addr});
             } else {
               auto heap = allocateAndInitNewStorage(
                   builder, loc, box, shape, lengthParams, ".auto.alloc");
               if (storageHandler)
                 storageHandler(getExtValForStorage(heap));
-              builder.create<fir::ResultOp>(loc,
-                                            mlir::ValueRange{trueValue, heap});
+              fir::ResultOp::create(builder, loc,
+                                    mlir::ValueRange{trueValue, heap});
             }
           });
   ifOp.end();
@@ -976,7 +978,7 @@ mlir::Value fir::factory::genNullBoxStorage(fir::FirOpBuilder &builder,
   mlir::Value boxStorage = builder.createTemporary(loc, boxTy);
   mlir::Value nullBox = fir::factory::createUnallocatedBox(
       builder, loc, boxTy, /*nonDeferredParams=*/{});
-  builder.create<fir::StoreOp>(loc, nullBox, boxStorage);
+  fir::StoreOp::create(builder, loc, nullBox, boxStorage);
   return boxStorage;
 }
 
@@ -988,11 +990,11 @@ mlir::Value fir::factory::getAndEstablishBoxStorage(
   mlir::Value nullAddr =
       builder.createNullConstant(loc, boxTy.getBaseAddressType());
   mlir::Value box =
-      builder.create<fir::EmboxOp>(loc, boxTy, nullAddr, shape,
-                                   /*emptySlice=*/mlir::Value{},
-                                   fir::factory::elideLengthsAlreadyInType(
-                                       boxTy.unwrapInnerType(), typeParams),
-                                   polymorphicMold);
-  builder.create<fir::StoreOp>(loc, box, boxStorage);
+      fir::EmboxOp::create(builder, loc, boxTy, nullAddr, shape,
+                           /*emptySlice=*/mlir::Value{},
+                           fir::factory::elideLengthsAlreadyInType(
+                               boxTy.unwrapInnerType(), typeParams),
+                           polymorphicMold);
+  fir::StoreOp::create(builder, loc, box, boxStorage);
   return boxStorage;
 }

diff  --git a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
index db12c84496b10..03952da95b11e 100644
--- a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
@@ -1091,7 +1091,7 @@ void PPCIntrinsicLibrary::genMtfsf(llvm::ArrayRef<fir::ExtendedValue> args) {
         builder.getContext(), builder);
     funcOp = builder.createFunction(loc, "llvm.ppc.mtfsf", libFuncType);
   }
-  builder.create<fir::CallOp>(loc, funcOp, scalarArgs);
+  fir::CallOp::create(builder, loc, funcOp, scalarArgs);
 }
 
 // VEC_ABS
@@ -1118,7 +1118,7 @@ PPCIntrinsicLibrary::genVecAbs(mlir::Type resultType,
     }
 
     funcOp = builder.createFunction(loc, fname, ftype);
-    auto callOp{builder.create<fir::CallOp>(loc, funcOp, argBases[0])};
+    auto callOp{fir::CallOp::create(builder, loc, funcOp, argBases[0])};
     return callOp.getResult(0);
   } else if (auto eleTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy)) {
     // vec_abs(arg1) = max(0 - arg1, arg1)
@@ -1128,8 +1128,8 @@ PPCIntrinsicLibrary::genVecAbs(mlir::Type resultType,
     // construct vector(0,..)
     auto zeroVal{builder.createIntegerConstant(loc, eleTy, 0)};
     auto vZero{
-        builder.create<mlir::vector::BroadcastOp>(loc, newVecTy, zeroVal)};
-    auto zeroSubVarg1{builder.create<mlir::arith::SubIOp>(loc, vZero, varg1)};
+        mlir::vector::BroadcastOp::create(builder, loc, newVecTy, zeroVal)};
+    auto zeroSubVarg1{mlir::arith::SubIOp::create(builder, loc, vZero, varg1)};
 
     mlir::func::FuncOp funcOp{nullptr};
     switch (eleTy.getWidth()) {
@@ -1159,7 +1159,7 @@ PPCIntrinsicLibrary::genVecAbs(mlir::Type resultType,
     funcOp = builder.createFunction(loc, fname, ftype);
 
     mlir::Value args[] = {zeroSubVarg1, varg1};
-    auto callOp{builder.create<fir::CallOp>(loc, funcOp, args)};
+    auto callOp{fir::CallOp::create(builder, loc, funcOp, args)};
     return builder.createConvert(loc, argBases[0].getType(),
                                  callOp.getResult(0));
   }
@@ -1189,21 +1189,21 @@ fir::ExtendedValue PPCIntrinsicLibrary::genVecAddAndMulSubXor(
   switch (vop) {
   case VecOp::Add:
     if (isInteger)
-      r = builder.create<mlir::arith::AddIOp>(loc, vargs[0], vargs[1]);
+      r = mlir::arith::AddIOp::create(builder, loc, vargs[0], vargs[1]);
     else if (isFloat)
-      r = builder.create<mlir::arith::AddFOp>(loc, vargs[0], vargs[1]);
+      r = mlir::arith::AddFOp::create(builder, loc, vargs[0], vargs[1]);
     break;
   case VecOp::Mul:
     if (isInteger)
-      r = builder.create<mlir::arith::MulIOp>(loc, vargs[0], vargs[1]);
+      r = mlir::arith::MulIOp::create(builder, loc, vargs[0], vargs[1]);
     else if (isFloat)
-      r = builder.create<mlir::arith::MulFOp>(loc, vargs[0], vargs[1]);
+      r = mlir::arith::MulFOp::create(builder, loc, vargs[0], vargs[1]);
     break;
   case VecOp::Sub:
     if (isInteger)
-      r = builder.create<mlir::arith::SubIOp>(loc, vargs[0], vargs[1]);
+      r = mlir::arith::SubIOp::create(builder, loc, vargs[0], vargs[1]);
     else if (isFloat)
-      r = builder.create<mlir::arith::SubFOp>(loc, vargs[0], vargs[1]);
+      r = mlir::arith::SubFOp::create(builder, loc, vargs[0], vargs[1]);
     break;
   case VecOp::And:
   case VecOp::Xor: {
@@ -1217,16 +1217,16 @@ fir::ExtendedValue PPCIntrinsicLibrary::genVecAddAndMulSubXor(
       auto wd{mlir::dyn_cast<mlir::FloatType>(vecTyInfo.eleTy).getWidth()};
       auto ftype{builder.getIntegerType(wd)};
       auto bcVecTy{mlir::VectorType::get(vecTyInfo.len, ftype)};
-      arg1 = builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[0]);
-      arg2 = builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[1]);
+      arg1 = mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[0]);
+      arg2 = mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[1]);
     }
     if (vop == VecOp::And)
-      r = builder.create<mlir::arith::AndIOp>(loc, arg1, arg2);
+      r = mlir::arith::AndIOp::create(builder, loc, arg1, arg2);
     else if (vop == VecOp::Xor)
-      r = builder.create<mlir::arith::XOrIOp>(loc, arg1, arg2);
+      r = mlir::arith::XOrIOp::create(builder, loc, arg1, arg2);
 
     if (isFloat)
-      r = builder.create<mlir::vector::BitCastOp>(loc, vargs[0].getType(), r);
+      r = mlir::vector::BitCastOp::create(builder, loc, vargs[0].getType(), r);
 
     break;
   }
@@ -1342,7 +1342,7 @@ PPCIntrinsicLibrary::genVecAnyCompare(mlir::Type resultType,
   assert((!fname.empty() && ftype) && "invalid type");
 
   mlir::func::FuncOp funcOp{builder.createFunction(loc, fname, ftype)};
-  auto callOp{builder.create<fir::CallOp>(loc, funcOp, cmpArgs)};
+  auto callOp{fir::CallOp::create(builder, loc, funcOp, cmpArgs)};
   return callOp.getResult(0);
 }
 
@@ -1473,7 +1473,7 @@ PPCIntrinsicLibrary::genVecCmp(mlir::Type resultType,
       // arg1 < arg2 --> vcmpgt(arg2, arg1)
       mlir::Value vargs[]{argBases[argOrder[vop][0]],
                           argBases[argOrder[vop][1]]};
-      auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)};
+      auto callOp{fir::CallOp::create(builder, loc, funcOp, vargs)};
       res = callOp.getResult(0);
       break;
     }
@@ -1487,14 +1487,15 @@ PPCIntrinsicLibrary::genVecCmp(mlir::Type resultType,
       // Construct a constant vector(-1)
       auto negOneVal{builder.createIntegerConstant(
           loc, getConvertedElementType(context, eTy), -1)};
-      auto vNegOne{builder.create<mlir::vector::BroadcastOp>(
-          loc, vecTyInfo.toMlirVectorType(context), negOneVal)};
+      auto vNegOne{mlir::vector::BroadcastOp::create(
+          builder, loc, vecTyInfo.toMlirVectorType(context), negOneVal)};
 
-      auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)};
+      auto callOp{fir::CallOp::create(builder, loc, funcOp, vargs)};
       mlir::Value callRes{callOp.getResult(0)};
       auto vargs2{
           convertVecArgs(builder, loc, vecTyInfo, mlir::ValueRange{callRes})};
-      auto xorRes{builder.create<mlir::arith::XOrIOp>(loc, vargs2[0], vNegOne)};
+      auto xorRes{
+          mlir::arith::XOrIOp::create(builder, loc, vargs2[0], vNegOne)};
 
       res = builder.createConvert(loc, returnType, xorRes);
       break;
@@ -1519,7 +1520,7 @@ PPCIntrinsicLibrary::genVecCmp(mlir::Type resultType,
     default:
       llvm_unreachable("Invalid vector operation for generator");
     }
-    auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)};
+    auto callOp{fir::CallOp::create(builder, loc, funcOp, vargs)};
     res = callOp.getResult(0);
   } else
     llvm_unreachable("invalid vector type");
@@ -1535,13 +1536,13 @@ static inline mlir::Value swapVectorWordPairs(fir::FirOpBuilder &builder,
   auto vtype{mlir::VectorType::get(16, mlir::IntegerType::get(context, 8))};
 
   if (ty != vtype)
-    arg = builder.create<mlir::LLVM::BitcastOp>(loc, vtype, arg).getResult();
+    arg = mlir::LLVM::BitcastOp::create(builder, loc, vtype, arg).getResult();
 
   llvm::SmallVector<int64_t, 16> mask{4,  5,  6,  7,  0, 1, 2,  3,
                                       12, 13, 14, 15, 8, 9, 10, 11};
-  arg = builder.create<mlir::vector::ShuffleOp>(loc, arg, arg, mask);
+  arg = mlir::vector::ShuffleOp::create(builder, loc, arg, arg, mask);
   if (ty != vtype)
-    arg = builder.create<mlir::LLVM::BitcastOp>(loc, ty, arg);
+    arg = mlir::LLVM::BitcastOp::create(builder, loc, ty, arg);
   return arg;
 }
 
@@ -1576,7 +1577,7 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType,
                                                : "llvm.ppc.altivec.vcfsx"};
       auto funcOp{builder.createFunction(loc, fname, ftype)};
       mlir::Value newArgs[] = {argBases[0], convArg};
-      auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)};
+      auto callOp{fir::CallOp::create(builder, loc, funcOp, newArgs)};
 
       return callOp.getResult(0);
     } else if (width == 64) {
@@ -1585,8 +1586,8 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType,
 
       // vec_vtf(arg1, arg2) = fmul(1.0 / (1 << arg2), llvm.sitofp(arg1))
       auto convOp{(isUnsigned)
-                      ? builder.create<mlir::LLVM::UIToFPOp>(loc, ty, vArg1)
-                      : builder.create<mlir::LLVM::SIToFPOp>(loc, ty, vArg1)};
+                      ? mlir::LLVM::UIToFPOp::create(builder, loc, ty, vArg1)
+                      : mlir::LLVM::SIToFPOp::create(builder, loc, ty, vArg1)};
 
       // construct vector<1./(1<<arg1), 1.0/(1<<arg1)>
       auto constInt{mlir::dyn_cast_or_null<mlir::IntegerAttr>(
@@ -1595,11 +1596,11 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType,
       assert(constInt && "expected integer constant argument");
       double f{1.0 / (1 << constInt.getInt())};
       llvm::SmallVector<double> vals{f, f};
-      auto constOp{builder.create<mlir::arith::ConstantOp>(
-          loc, ty, builder.getF64VectorAttr(vals))};
+      auto constOp{mlir::arith::ConstantOp::create(
+          builder, loc, ty, builder.getF64VectorAttr(vals))};
 
-      auto mulOp{builder.create<mlir::LLVM::FMulOp>(
-          loc, ty, convOp->getResult(0), constOp)};
+      auto mulOp{mlir::LLVM::FMulOp::create(builder, loc, ty,
+                                            convOp->getResult(0), constOp)};
 
       return builder.createConvert(loc, fir::VectorType::get(2, fTy), mulOp);
     }
@@ -1613,7 +1614,7 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType,
     auto firTy{resTyInfo.toFirVectorType()};
 
     // vec_convert(v, mold) = bitcast v to "type of mold"
-    auto conv{builder.create<mlir::LLVM::BitcastOp>(loc, moldTy, vArg1)};
+    auto conv{mlir::LLVM::BitcastOp::create(builder, loc, moldTy, vArg1)};
 
     return builder.createConvert(loc, firTy, conv);
   }
@@ -1629,7 +1630,7 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType,
       auto ftype{
           genFuncType<Ty::RealVector<8>, Ty::RealVector<4>>(context, builder)};
       auto funcOp{builder.createFunction(loc, fname, ftype)};
-      auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)};
+      auto callOp{fir::CallOp::create(builder, loc, funcOp, newArgs)};
 
       return callOp.getResult(0);
     } else if (vecTyInfo.isFloat64()) {
@@ -1638,7 +1639,7 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType,
           genFuncType<Ty::RealVector<4>, Ty::RealVector<8>>(context, builder)};
       auto funcOp{builder.createFunction(loc, fname, ftype)};
       newArgs[0] =
-          builder.create<fir::CallOp>(loc, funcOp, newArgs).getResult(0);
+          fir::CallOp::create(builder, loc, funcOp, newArgs).getResult(0);
       auto fvf32Ty{newArgs[0].getType()};
       auto f32type{mlir::Float32Type::get(context)};
       auto mvf32Ty{mlir::VectorType::get(4, f32type)};
@@ -1662,7 +1663,7 @@ static mlir::Value convertVectorElementOrder(fir::FirOpBuilder &builder,
                                              mlir::Value idx) {
   mlir::Value numSub1{
       builder.createIntegerConstant(loc, idx.getType(), vecInfo.len - 1)};
-  return builder.create<mlir::LLVM::SubOp>(loc, idx.getType(), numSub1, idx);
+  return mlir::LLVM::SubOp::create(builder, loc, idx.getType(), numSub1, idx);
 }
 
 // VEC_EXTRACT
@@ -1681,14 +1682,14 @@ PPCIntrinsicLibrary::genVecExtract(mlir::Type resultType,
   // position
   auto numEle{builder.createIntegerConstant(loc, argTypes[1], vecTyInfo.len)};
   mlir::Value uremOp{
-      builder.create<mlir::LLVM::URemOp>(loc, argBases[1], numEle)};
+      mlir::LLVM::URemOp::create(builder, loc, argBases[1], numEle)};
 
   if (!isNativeVecElemOrderOnLE())
     uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp);
 
   mlir::Value index = builder.createOrFold<mlir::index::CastUOp>(
       loc, builder.getIndexType(), uremOp);
-  return builder.create<mlir::vector::ExtractOp>(loc, varg0, index);
+  return mlir::vector::ExtractOp::create(builder, loc, varg0, index);
 }
 
 // VEC_INSERT
@@ -1704,7 +1705,7 @@ PPCIntrinsicLibrary::genVecInsert(mlir::Type resultType,
 
   auto numEle{builder.createIntegerConstant(loc, argTypes[2], vecTyInfo.len)};
   mlir::Value uremOp{
-      builder.create<mlir::LLVM::URemOp>(loc, argBases[2], numEle)};
+      mlir::LLVM::URemOp::create(builder, loc, argBases[2], numEle)};
 
   if (!isNativeVecElemOrderOnLE())
     uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp);
@@ -1712,8 +1713,8 @@ PPCIntrinsicLibrary::genVecInsert(mlir::Type resultType,
   mlir::Value index = builder.createOrFold<mlir::index::CastUOp>(
       loc, builder.getIndexType(), uremOp);
   mlir::Value res =
-      builder.create<mlir::vector::InsertOp>(loc, argBases[0], varg1, index);
-  return builder.create<fir::ConvertOp>(loc, vecTyInfo.toFirVectorType(), res);
+      mlir::vector::InsertOp::create(builder, loc, argBases[0], varg1, index);
+  return fir::ConvertOp::create(builder, loc, vecTyInfo.toFirVectorType(), res);
 }
 
 // VEC_MERGEH, VEC_MERGEL
@@ -1799,8 +1800,8 @@ PPCIntrinsicLibrary::genVecMerge(mlir::Type resultType,
   llvm::SmallVector<int64_t, 16> &mergeMask =
       (isBEVecElemOrderOnLE()) ? rMask : mMask;
 
-  auto callOp{builder.create<mlir::vector::ShuffleOp>(loc, vargs[0], vargs[1],
-                                                      mergeMask)};
+  auto callOp{mlir::vector::ShuffleOp::create(builder, loc, vargs[0], vargs[1],
+                                              mergeMask)};
   return builder.createConvert(loc, resultType, callOp);
 }
 
@@ -1812,9 +1813,9 @@ static mlir::Value addOffsetToAddress(fir::FirOpBuilder &builder,
   auto arrRefTy{builder.getRefType(fir::SequenceType::get(
       {typeExtent}, mlir::IntegerType::get(builder.getContext(), 8)))};
   // Convert arg to !fir.ref<!ref.array<?xi8>>
-  auto resAddr{builder.create<fir::ConvertOp>(loc, arrRefTy, baseAddr)};
+  auto resAddr{fir::ConvertOp::create(builder, loc, arrRefTy, baseAddr)};
 
-  return builder.create<fir::CoordinateOp>(loc, arrRefTy, resAddr, offset);
+  return fir::CoordinateOp::create(builder, loc, arrRefTy, resAddr, offset);
 }
 
 static mlir::Value reverseVectorElements(fir::FirOpBuilder &builder,
@@ -1826,8 +1827,8 @@ static mlir::Value reverseVectorElements(fir::FirOpBuilder &builder,
   for (int64_t i = 0; i < len; ++i) {
     mask.push_back(len - 1 - i);
   }
-  auto undefVec{builder.create<fir::UndefOp>(loc, v.getType())};
-  return builder.create<mlir::vector::ShuffleOp>(loc, v, undefVec, mask);
+  auto undefVec{fir::UndefOp::create(builder, loc, v.getType())};
+  return mlir::vector::ShuffleOp::create(builder, loc, v, undefVec, mask);
 }
 
 static mlir::NamedAttribute getAlignmentAttr(fir::FirOpBuilder &builder,
@@ -1876,8 +1877,8 @@ fir::ExtendedValue PPCIntrinsicLibrary::genVecLdNoCallGrp(
 
   const auto triple{fir::getTargetTriple(builder.getModule())};
   // Need to get align 1.
-  auto result{builder.create<fir::LoadOp>(loc, mlirTy, addr,
-                                          getAlignmentAttr(builder, 1))};
+  auto result{fir::LoadOp::create(builder, loc, mlirTy, addr,
+                                  getAlignmentAttr(builder, 1))};
   if ((vop == VecOp::Xl && isBEVecElemOrderOnLE()) ||
       (vop == VecOp::Xlbe && triple.isLittleEndian()))
     return builder.createConvert(
@@ -1970,13 +1971,13 @@ PPCIntrinsicLibrary::genVecLdCallGrp(mlir::Type resultType,
       mlir::FunctionType::get(context, {addr.getType()}, {intrinResTy})};
   auto funcOp{builder.createFunction(loc, fname, funcType)};
   auto result{
-      builder.create<fir::CallOp>(loc, funcOp, parsedArgs).getResult(0)};
+      fir::CallOp::create(builder, loc, funcOp, parsedArgs).getResult(0)};
 
   if (vop == VecOp::Lxvp)
     return result;
 
   if (intrinResTy != mlirTy)
-    result = builder.create<mlir::vector::BitCastOp>(loc, mlirTy, result);
+    result = mlir::vector::BitCastOp::create(builder, loc, mlirTy, result);
 
   if (vop != VecOp::Xld2 && vop != VecOp::Xlw4 && isBEVecElemOrderOnLE())
     return builder.createConvert(
@@ -2003,13 +2004,13 @@ PPCIntrinsicLibrary::genVecLvsGrp(mlir::Type resultType,
   // Convert arg0 to i64 type if needed
   auto i64ty{mlir::IntegerType::get(context, 64)};
   if (arg0.getType() != i64ty)
-    arg0 = builder.create<fir::ConvertOp>(loc, i64ty, arg0);
+    arg0 = fir::ConvertOp::create(builder, loc, i64ty, arg0);
 
   // offset is modulo 16, so shift left 56 bits and then right 56 bits to clear
   //   upper 56 bit while preserving sign
   auto shiftVal{builder.createIntegerConstant(loc, i64ty, 56)};
-  auto offset{builder.create<mlir::arith::ShLIOp>(loc, arg0, shiftVal)};
-  auto offset2{builder.create<mlir::arith::ShRSIOp>(loc, offset, shiftVal)};
+  auto offset{mlir::arith::ShLIOp::create(builder, loc, arg0, shiftVal)};
+  auto offset2{mlir::arith::ShRSIOp::create(builder, loc, offset, shiftVal)};
 
   // Add the offsetArg to %addr of arg1
   auto addr{addOffsetToAddress(builder, loc, arg1, offset2)};
@@ -2029,7 +2030,7 @@ PPCIntrinsicLibrary::genVecLvsGrp(mlir::Type resultType,
   auto funcType{mlir::FunctionType::get(context, {addr.getType()}, {mlirTy})};
   auto funcOp{builder.createFunction(loc, fname, funcType)};
   auto result{
-      builder.create<fir::CallOp>(loc, funcOp, parsedArgs).getResult(0)};
+      fir::CallOp::create(builder, loc, funcOp, parsedArgs).getResult(0)};
 
   if (isNativeVecElemOrderOnLE())
     return builder.createConvert(
@@ -2066,19 +2067,19 @@ PPCIntrinsicLibrary::genVecNmaddMsub(mlir::Type resultType,
                                      std::get<1>(fmaMap[width]))};
   if (vop == VecOp::Nmadd) {
     // vec_nmadd(arg1, arg2, arg3) = -fma(arg1, arg2, arg3)
-    auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)};
+    auto callOp{fir::CallOp::create(builder, loc, funcOp, newArgs)};
 
     // We need to convert fir.vector to MLIR vector to use fneg and then back
     // to fir.vector to store.
     auto vCall{builder.createConvert(loc, vTypeInfo.toMlirVectorType(context),
                                      callOp.getResult(0))};
-    auto neg{builder.create<mlir::arith::NegFOp>(loc, vCall)};
+    auto neg{mlir::arith::NegFOp::create(builder, loc, vCall)};
     return builder.createConvert(loc, vTypeInfo.toFirVectorType(), neg);
   } else if (vop == VecOp::Msub) {
     // vec_msub(arg1, arg2, arg3) = fma(arg1, arg2, -arg3)
-    newArgs[2] = builder.create<mlir::arith::NegFOp>(loc, newArgs[2]);
+    newArgs[2] = mlir::arith::NegFOp::create(builder, loc, newArgs[2]);
 
-    auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)};
+    auto callOp{fir::CallOp::create(builder, loc, funcOp, newArgs)};
     return callOp.getResult(0);
   }
   llvm_unreachable("Invalid vector operation for generator");
@@ -2109,10 +2110,10 @@ PPCIntrinsicLibrary::genVecPerm(mlir::Type resultType,
     auto mMask{builder.createConvert(loc, mlirMaskTy, argBases[2])};
 
     if (mlirTy != vi32Ty) {
-      mArg0 =
-          builder.create<mlir::LLVM::BitcastOp>(loc, vi32Ty, mArg0).getResult();
-      mArg1 =
-          builder.create<mlir::LLVM::BitcastOp>(loc, vi32Ty, mArg1).getResult();
+      mArg0 = mlir::LLVM::BitcastOp::create(builder, loc, vi32Ty, mArg0)
+                  .getResult();
+      mArg1 = mlir::LLVM::BitcastOp::create(builder, loc, vi32Ty, mArg1)
+                  .getResult();
     }
 
     auto funcOp{builder.createFunction(
@@ -2127,23 +2128,23 @@ PPCIntrinsicLibrary::genVecPerm(mlir::Type resultType,
       auto v8Ty{mlir::VectorType::get(16, i8Ty)};
       auto negOne{builder.createMinusOneInteger(loc, i8Ty)};
       auto vNegOne{
-          builder.create<mlir::vector::BroadcastOp>(loc, v8Ty, negOne)};
+          mlir::vector::BroadcastOp::create(builder, loc, v8Ty, negOne)};
 
-      mMask = builder.create<mlir::arith::XOrIOp>(loc, mMask, vNegOne);
+      mMask = mlir::arith::XOrIOp::create(builder, loc, mMask, vNegOne);
       newArgs = {mArg1, mArg0, mMask};
     } else {
       newArgs = {mArg0, mArg1, mMask};
     }
 
-    auto res{builder.create<fir::CallOp>(loc, funcOp, newArgs).getResult(0)};
+    auto res{fir::CallOp::create(builder, loc, funcOp, newArgs).getResult(0)};
 
     if (res.getType() != argTypes[0]) {
       // fir.call llvm.ppc.altivec.vperm returns !fir.vector<i4:32>
       // convert the result back to the original type
       res = builder.createConvert(loc, vi32Ty, res);
       if (mlirTy != vi32Ty)
-        res =
-            builder.create<mlir::LLVM::BitcastOp>(loc, mlirTy, res).getResult();
+        res = mlir::LLVM::BitcastOp::create(builder, loc, mlirTy, res)
+                  .getResult();
     }
     return builder.createConvert(loc, resultType, res);
   }
@@ -2156,10 +2157,10 @@ PPCIntrinsicLibrary::genVecPerm(mlir::Type resultType,
     auto constInt{constIntOp.getInt()};
     // arg1, arg2, and result type share same VecTypeInfo
     if (vecTyInfo.isFloat()) {
-      mArg0 =
-          builder.create<mlir::LLVM::BitcastOp>(loc, vf64Ty, mArg0).getResult();
-      mArg1 =
-          builder.create<mlir::LLVM::BitcastOp>(loc, vf64Ty, mArg1).getResult();
+      mArg0 = mlir::LLVM::BitcastOp::create(builder, loc, vf64Ty, mArg0)
+                  .getResult();
+      mArg1 = mlir::LLVM::BitcastOp::create(builder, loc, vf64Ty, mArg1)
+                  .getResult();
     }
 
     llvm::SmallVector<int64_t, 2> nMask; // native vector element order mask
@@ -2188,9 +2189,9 @@ PPCIntrinsicLibrary::genVecPerm(mlir::Type resultType,
 
     llvm::SmallVector<int64_t, 2> mask =
         (isBEVecElemOrderOnLE()) ? rMask : nMask;
-    auto res{builder.create<mlir::vector::ShuffleOp>(loc, mArg0, mArg1, mask)};
+    auto res{mlir::vector::ShuffleOp::create(builder, loc, mArg0, mArg1, mask)};
     if (res.getType() != mlirTy) {
-      auto cast{builder.create<mlir::LLVM::BitcastOp>(loc, mlirTy, res)};
+      auto cast{mlir::LLVM::BitcastOp::create(builder, loc, mlirTy, res)};
       return builder.createConvert(loc, resultType, cast);
     }
     return builder.createConvert(loc, resultType, res);
@@ -2217,22 +2218,23 @@ PPCIntrinsicLibrary::genVecSel(mlir::Type resultType,
 
   // construct a constant <16 x i8> vector with value -1 for bitcast
   auto bcVecTy{mlir::VectorType::get(16, i8Ty)};
-  auto vNegOne{builder.create<mlir::vector::BroadcastOp>(loc, bcVecTy, negOne)};
+  auto vNegOne{
+      mlir::vector::BroadcastOp::create(builder, loc, bcVecTy, negOne)};
 
   // bitcast arguments to bcVecTy
-  auto arg1{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[0])};
-  auto arg2{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[1])};
-  auto arg3{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[2])};
+  auto arg1{mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[0])};
+  auto arg2{mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[1])};
+  auto arg3{mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[2])};
 
   // vec_sel(arg1, arg2, arg3) =
   //   (arg2 and arg3) or (arg1 and (arg3 xor vector(-1,...)))
-  auto comp{builder.create<mlir::arith::XOrIOp>(loc, arg3, vNegOne)};
-  auto a1AndComp{builder.create<mlir::arith::AndIOp>(loc, arg1, comp)};
-  auto a1OrA2{builder.create<mlir::arith::AndIOp>(loc, arg2, arg3)};
-  auto res{builder.create<mlir::arith::OrIOp>(loc, a1AndComp, a1OrA2)};
+  auto comp{mlir::arith::XOrIOp::create(builder, loc, arg3, vNegOne)};
+  auto a1AndComp{mlir::arith::AndIOp::create(builder, loc, arg1, comp)};
+  auto a1OrA2{mlir::arith::AndIOp::create(builder, loc, arg2, arg3)};
+  auto res{mlir::arith::OrIOp::create(builder, loc, a1AndComp, a1OrA2)};
 
   auto bcRes{
-      builder.create<mlir::vector::BitCastOp>(loc, vargs[0].getType(), res)};
+      mlir::vector::BitCastOp::create(builder, loc, vargs[0].getType(), res)};
 
   return builder.createConvert(loc, vecTyInfos[0].toFirVectorType(), bcRes);
 }
@@ -2269,14 +2271,14 @@ PPCIntrinsicLibrary::genVecShift(mlir::Type resultType,
     auto vecVal{builder.createIntegerConstant(
         loc, getConvertedElementType(context, vecTyInfoArgs[0].eleTy), width)};
     auto mask{
-        builder.create<mlir::vector::BroadcastOp>(loc, mlirTyArgs[1], vecVal)};
-    auto shft{builder.create<mlir::arith::RemUIOp>(loc, mlirVecArgs[1], mask)};
+        mlir::vector::BroadcastOp::create(builder, loc, mlirTyArgs[1], vecVal)};
+    auto shft{mlir::arith::RemUIOp::create(builder, loc, mlirVecArgs[1], mask)};
 
     mlir::Value res{nullptr};
     if (vop == VecOp::Sr)
-      res = builder.create<mlir::arith::ShRUIOp>(loc, mlirVecArgs[0], shft);
+      res = mlir::arith::ShRUIOp::create(builder, loc, mlirVecArgs[0], shft);
     else if (vop == VecOp::Sl)
-      res = builder.create<mlir::arith::ShLIOp>(loc, mlirVecArgs[0], shft);
+      res = mlir::arith::ShLIOp::create(builder, loc, mlirVecArgs[0], shft);
 
     shftRes = builder.createConvert(loc, argTypes[0], res);
   } else if (vop == VecOp::Sll || vop == VecOp::Slo || vop == VecOp::Srl ||
@@ -2286,11 +2288,11 @@ PPCIntrinsicLibrary::genVecShift(mlir::Type resultType,
     // Bitcast to vector<4xi32>
     auto bcVecTy{mlir::VectorType::get(4, builder.getIntegerType(32))};
     if (mlirTyArgs[0] != bcVecTy)
-      mlirVecArgs[0] =
-          builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, mlirVecArgs[0]);
+      mlirVecArgs[0] = mlir::vector::BitCastOp::create(builder, loc, bcVecTy,
+                                                       mlirVecArgs[0]);
     if (mlirTyArgs[1] != bcVecTy)
-      mlirVecArgs[1] =
-          builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, mlirVecArgs[1]);
+      mlirVecArgs[1] = mlir::vector::BitCastOp::create(builder, loc, bcVecTy,
+                                                       mlirVecArgs[1]);
 
     llvm::StringRef funcName;
     switch (vop) {
@@ -2312,13 +2314,13 @@ PPCIntrinsicLibrary::genVecShift(mlir::Type resultType,
     auto funcTy{genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,
                             Ty::IntegerVector<4>>(context, builder)};
     mlir::func::FuncOp funcOp{builder.createFunction(loc, funcName, funcTy)};
-    auto callOp{builder.create<fir::CallOp>(loc, funcOp, mlirVecArgs)};
+    auto callOp{fir::CallOp::create(builder, loc, funcOp, mlirVecArgs)};
 
     // If the result vector type is 
diff erent from the original type, need
     // to convert to mlir vector, bitcast and then convert back to fir vector.
     if (callOp.getResult(0).getType() != argTypes[0]) {
       auto res = builder.createConvert(loc, bcVecTy, callOp.getResult(0));
-      res = builder.create<mlir::vector::BitCastOp>(loc, mlirTyArgs[0], res);
+      res = mlir::vector::BitCastOp::create(builder, loc, mlirTyArgs[0], res);
       shftRes = builder.createConvert(loc, argTypes[0], res);
     } else {
       shftRes = callOp.getResult(0);
@@ -2334,10 +2336,10 @@ PPCIntrinsicLibrary::genVecShift(mlir::Type resultType,
     auto vi8Ty{mlir::VectorType::get(16, builder.getIntegerType(8))};
     if (mlirTyArgs[0] != vi8Ty) {
       mlirVecArgs[0] =
-          builder.create<mlir::LLVM::BitcastOp>(loc, vi8Ty, mlirVecArgs[0])
+          mlir::LLVM::BitcastOp::create(builder, loc, vi8Ty, mlirVecArgs[0])
               .getResult();
       mlirVecArgs[1] =
-          builder.create<mlir::LLVM::BitcastOp>(loc, vi8Ty, mlirVecArgs[1])
+          mlir::LLVM::BitcastOp::create(builder, loc, vi8Ty, mlirVecArgs[1])
               .getResult();
     }
 
@@ -2352,19 +2354,19 @@ PPCIntrinsicLibrary::genVecShift(mlir::Type resultType,
     if (triple.isLittleEndian()) {
       for (int i = 16; i < 32; ++i)
         mask.push_back(i - shiftVal);
-      shftRes = builder.create<mlir::vector::ShuffleOp>(loc, mlirVecArgs[1],
-                                                        mlirVecArgs[0], mask);
+      shftRes = mlir::vector::ShuffleOp::create(builder, loc, mlirVecArgs[1],
+                                                mlirVecArgs[0], mask);
     } else {
       for (int i = 0; i < 16; ++i)
         mask.push_back(i + shiftVal);
-      shftRes = builder.create<mlir::vector::ShuffleOp>(loc, mlirVecArgs[0],
-                                                        mlirVecArgs[1], mask);
+      shftRes = mlir::vector::ShuffleOp::create(builder, loc, mlirVecArgs[0],
+                                                mlirVecArgs[1], mask);
     }
 
     // Bitcast to the original type
     if (shftRes.getType() != mlirTyArgs[0])
       shftRes =
-          builder.create<mlir::LLVM::BitcastOp>(loc, mlirTyArgs[0], shftRes);
+          mlir::LLVM::BitcastOp::create(builder, loc, mlirTyArgs[0], shftRes);
 
     return builder.createConvert(loc, resultType, shftRes);
   } else
@@ -2389,8 +2391,9 @@ PPCIntrinsicLibrary::genVecSplat(mlir::Type resultType,
     auto vecTyInfo{getVecTypeFromFir(argBases[0])};
 
     auto extractOp{genVecExtract(resultType, args)};
-    splatOp = builder.create<mlir::vector::SplatOp>(
-        loc, *(extractOp.getUnboxed()), vecTyInfo.toMlirVectorType(context));
+    splatOp =
+        mlir::vector::SplatOp::create(builder, loc, *(extractOp.getUnboxed()),
+                                      vecTyInfo.toMlirVectorType(context));
     retTy = vecTyInfo.toFirVectorType();
     break;
   }
@@ -2398,8 +2401,8 @@ PPCIntrinsicLibrary::genVecSplat(mlir::Type resultType,
     assert(args.size() == 1);
     auto vecTyInfo{getVecTypeFromEle(argBases[0])};
 
-    splatOp = builder.create<mlir::vector::SplatOp>(
-        loc, argBases[0], vecTyInfo.toMlirVectorType(context));
+    splatOp = mlir::vector::SplatOp::create(
+        builder, loc, argBases[0], vecTyInfo.toMlirVectorType(context));
     retTy = vecTyInfo.toFirVectorType();
     break;
   }
@@ -2409,8 +2412,8 @@ PPCIntrinsicLibrary::genVecSplat(mlir::Type resultType,
     auto intOp{builder.createConvert(loc, eleTy, argBases[0])};
 
     // the intrinsic always returns vector(integer(4))
-    splatOp = builder.create<mlir::vector::SplatOp>(
-        loc, intOp, mlir::VectorType::get(4, eleTy));
+    splatOp = mlir::vector::SplatOp::create(builder, loc, intOp,
+                                            mlir::VectorType::get(4, eleTy));
     retTy = fir::VectorType::get(4, eleTy);
     break;
   }
@@ -2438,14 +2441,14 @@ PPCIntrinsicLibrary::genVecXlds(mlir::Type resultType,
   auto i64Ty{mlir::IntegerType::get(builder.getContext(), 64)};
   auto i64VecTy{mlir::VectorType::get(2, i64Ty)};
   auto i64RefTy{builder.getRefType(i64Ty)};
-  auto addrConv{builder.create<fir::ConvertOp>(loc, i64RefTy, addr)};
+  auto addrConv{fir::ConvertOp::create(builder, loc, i64RefTy, addr)};
 
-  auto addrVal{builder.create<fir::LoadOp>(loc, addrConv)};
-  auto splatRes{builder.create<mlir::vector::SplatOp>(loc, addrVal, i64VecTy)};
+  auto addrVal{fir::LoadOp::create(builder, loc, addrConv)};
+  auto splatRes{mlir::vector::SplatOp::create(builder, loc, addrVal, i64VecTy)};
 
   mlir::Value result{nullptr};
   if (mlirTy != splatRes.getType()) {
-    result = builder.create<mlir::vector::BitCastOp>(loc, mlirTy, splatRes);
+    result = mlir::vector::BitCastOp::create(builder, loc, mlirTy, splatRes);
   } else
     result = splatRes;
 
@@ -2795,7 +2798,7 @@ void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef<fir::ExtendedValue> args) {
     if (i == 0 && HandlerOp == MMAHandlerOp::FirstArgIsResult) {
       // First argument is passed in as an address. We need to load
       // the content to match the LLVM interface.
-      v = builder.create<fir::LoadOp>(loc, v);
+      v = fir::LoadOp::create(builder, loc, v);
     }
     auto vType{v.getType()};
     mlir::Type targetType{intrFuncType.getInput(j)};
@@ -2806,7 +2809,7 @@ void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef<fir::ExtendedValue> args) {
         auto len{mlir::dyn_cast<fir::VectorType>(vType).getLen()};
         mlir::VectorType mlirType = mlir::VectorType::get(len, eleTy);
         auto v0{builder.createConvert(loc, mlirType, v)};
-        auto v1{builder.create<mlir::vector::BitCastOp>(loc, targetType, v0)};
+        auto v1{mlir::vector::BitCastOp::create(builder, loc, targetType, v0)};
         intrArgs.push_back(v1);
       } else if (mlir::isa<mlir::IntegerType>(targetType) &&
                  mlir::isa<mlir::IntegerType>(vType)) {
@@ -2822,7 +2825,7 @@ void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef<fir::ExtendedValue> args) {
       intrArgs.push_back(v);
     }
   }
-  auto callSt{builder.create<fir::CallOp>(loc, funcOp, intrArgs)};
+  auto callSt{fir::CallOp::create(builder, loc, funcOp, intrArgs)};
   if (HandlerOp == MMAHandlerOp::SubToFunc ||
       HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE ||
       HandlerOp == MMAHandlerOp::FirstArgIsResult) {
@@ -2831,10 +2834,11 @@ void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef<fir::ExtendedValue> args) {
     mlir::Value destPtr{fir::getBase(args[0])};
     mlir::Type callResultPtrType{builder.getRefType(callResult.getType())};
     if (destPtr.getType() != callResultPtrType) {
-      destPtr = builder.create<fir::ConvertOp>(loc, callResultPtrType, destPtr);
+      destPtr =
+          fir::ConvertOp::create(builder, loc, callResultPtrType, destPtr);
     }
     // Copy the result.
-    builder.create<fir::StoreOp>(loc, callResult, destPtr);
+    fir::StoreOp::create(builder, loc, callResult, destPtr);
   }
 }
 
@@ -2901,7 +2905,7 @@ void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef<fir::ExtendedValue> args) {
   if (vop == VecOp::Stxvp) {
     biArgs.push_back(argBases[0]);
     biArgs.push_back(addr);
-    builder.create<fir::CallOp>(loc, funcOp, biArgs);
+    fir::CallOp::create(builder, loc, funcOp, biArgs);
     return;
   }
 
@@ -2911,7 +2915,7 @@ void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef<fir::ExtendedValue> args) {
 
   mlir::Value newArg1{nullptr};
   if (stTy != arg1TyInfo.toMlirVectorType(context))
-    newArg1 = builder.create<mlir::vector::BitCastOp>(loc, stTy, cnv);
+    newArg1 = mlir::vector::BitCastOp::create(builder, loc, stTy, cnv);
   else
     newArg1 = cnv;
 
@@ -2922,7 +2926,7 @@ void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef<fir::ExtendedValue> args) {
   biArgs.push_back(newArg1);
   biArgs.push_back(addr);
 
-  builder.create<fir::CallOp>(loc, funcOp, biArgs);
+  fir::CallOp::create(builder, loc, funcOp, biArgs);
 }
 
 // VEC_XST, VEC_XST_BE, VEC_STXV, VEC_XSTD2, VEC_XSTW4
@@ -2971,7 +2975,7 @@ void PPCIntrinsicLibrary::genVecXStore(
 
     mlir::Type srcTy{nullptr};
     if (numElem != arg1TyInfo.len) {
-      cnv = builder.create<mlir::vector::BitCastOp>(loc, mlirVecTy, cnv);
+      cnv = mlir::vector::BitCastOp::create(builder, loc, mlirVecTy, cnv);
       srcTy = firVecTy;
     } else {
       srcTy = arg1TyInfo.toFirVectorType();
@@ -2994,9 +2998,9 @@ void PPCIntrinsicLibrary::genVecXStore(
   default:
     assert(false && "Invalid vector operation for generator");
   }
-  builder.create<fir::StoreOp>(loc, mlir::TypeRange{},
-                               mlir::ValueRange{src, trg},
-                               getAlignmentAttr(builder, 1));
+  fir::StoreOp::create(builder, loc, mlir::TypeRange{},
+                       mlir::ValueRange{src, trg},
+                       getAlignmentAttr(builder, 1));
 }
 
 } // namespace fir

diff  --git a/flang/lib/Optimizer/Builder/TemporaryStorage.cpp b/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
index 9d2e9837a3df8..4c648df18b328 100644
--- a/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
+++ b/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
@@ -28,7 +28,7 @@ fir::factory::Counter::Counter(mlir::Location loc, fir::FirOpBuilder &builder,
   one = builder.createIntegerConstant(loc, type, 1);
   if (canCountThroughLoops) {
     index = builder.createTemporary(loc, type);
-    builder.create<fir::StoreOp>(loc, initialValue, index);
+    fir::StoreOp::create(builder, loc, initialValue, index);
   } else {
     index = initialValue;
   }
@@ -38,21 +38,21 @@ mlir::Value
 fir::factory::Counter::getAndIncrementIndex(mlir::Location loc,
                                             fir::FirOpBuilder &builder) {
   if (canCountThroughLoops) {
-    mlir::Value indexValue = builder.create<fir::LoadOp>(loc, index);
+    mlir::Value indexValue = fir::LoadOp::create(builder, loc, index);
     mlir::Value newValue =
-        builder.create<mlir::arith::AddIOp>(loc, indexValue, one);
-    builder.create<fir::StoreOp>(loc, newValue, index);
+        mlir::arith::AddIOp::create(builder, loc, indexValue, one);
+    fir::StoreOp::create(builder, loc, newValue, index);
     return indexValue;
   }
   mlir::Value indexValue = index;
-  index = builder.create<mlir::arith::AddIOp>(loc, indexValue, one);
+  index = mlir::arith::AddIOp::create(builder, loc, indexValue, one);
   return indexValue;
 }
 
 void fir::factory::Counter::reset(mlir::Location loc,
                                   fir::FirOpBuilder &builder) {
   if (canCountThroughLoops)
-    builder.create<fir::StoreOp>(loc, initialValue, index);
+    fir::StoreOp::create(builder, loc, initialValue, index);
   else
     index = initialValue;
 }
@@ -103,7 +103,7 @@ void fir::factory::HomogeneousScalarStack::pushValue(mlir::Location loc,
   // below should not get hit but is added as a remainder/safety.
   if (!entity.hasIntrinsicType())
     TODO(loc, "creating inlined temporary stack for derived types");
-  builder.create<hlfir::AssignOp>(loc, value, tempElement);
+  hlfir::AssignOp::create(builder, loc, value, tempElement);
 }
 
 void fir::factory::HomogeneousScalarStack::resetFetchPosition(
@@ -125,14 +125,14 @@ void fir::factory::HomogeneousScalarStack::destroy(mlir::Location loc,
   if (allocateOnHeap) {
     auto declare = temp.getDefiningOp<hlfir::DeclareOp>();
     assert(declare && "temp must have been declared");
-    builder.create<fir::FreeMemOp>(loc, declare.getMemref());
+    fir::FreeMemOp::create(builder, loc, declare.getMemref());
   }
 }
 
 hlfir::Entity fir::factory::HomogeneousScalarStack::moveStackAsArrayExpr(
     mlir::Location loc, fir::FirOpBuilder &builder) {
   mlir::Value mustFree = builder.createBool(loc, allocateOnHeap);
-  auto hlfirExpr = builder.create<hlfir::AsExprOp>(loc, temp, mustFree);
+  auto hlfirExpr = hlfir::AsExprOp::create(builder, loc, temp, mustFree);
   return hlfir::Entity{hlfirExpr};
 }
 
@@ -147,14 +147,14 @@ fir::factory::SimpleCopy::SimpleCopy(mlir::Location loc,
   // Use hlfir.as_expr and hlfir.associate to create a copy and leave
   // bufferization deals with how best to make the copy.
   if (source.isVariable())
-    source = hlfir::Entity{builder.create<hlfir::AsExprOp>(loc, source)};
+    source = hlfir::Entity{hlfir::AsExprOp::create(builder, loc, source)};
   copy = hlfir::genAssociateExpr(loc, builder, source,
                                  source.getFortranElementType(), tempName);
 }
 
 void fir::factory::SimpleCopy::destroy(mlir::Location loc,
                                        fir::FirOpBuilder &builder) {
-  builder.create<hlfir::EndAssociateOp>(loc, copy);
+  hlfir::EndAssociateOp::create(builder, loc, copy);
 }
 
 //===----------------------------------------------------------------------===//
@@ -279,7 +279,7 @@ mlir::Value fir::factory::AnyVariableStack::fetch(mlir::Location loc,
   mlir::Value indexValue = counter.getAndIncrementIndex(loc, builder);
   fir::runtime::genDescriptorAt(loc, builder, opaquePtr, indexValue,
                                 retValueBox);
-  hlfir::Entity retBox{builder.create<fir::LoadOp>(loc, retValueBox)};
+  hlfir::Entity retBox{fir::LoadOp::create(builder, loc, retValueBox)};
   // The runtime always tracks variable as address, but the form of the variable
   // that was saved may be 
diff erent (raw address, fir.boxchar), ensure
   // the returned variable has the same form of the one that was saved.
@@ -326,7 +326,7 @@ void fir::factory::AnyVectorSubscriptStack::pushShape(
         hlfir::getFortranElementOrSequenceType(*boxType));
     mlir::Value null = builder.createNullConstant(loc, refType);
     mlir::Value descriptor =
-        builder.create<fir::EmboxOp>(loc, *boxType, null, shape);
+        fir::EmboxOp::create(builder, loc, *boxType, null, shape);
     shapeTemp->pushValue(loc, builder, descriptor);
     return;
   }
@@ -372,7 +372,7 @@ void fir::factory::AnyAddressStack::pushValue(mlir::Location loc,
   mlir::Value cast = variable;
   if (auto boxProcType = llvm::dyn_cast<fir::BoxProcType>(variable.getType())) {
     cast =
-        builder.create<fir::BoxAddrOp>(loc, boxProcType.getEleTy(), variable);
+        fir::BoxAddrOp::create(builder, loc, boxProcType.getEleTy(), variable);
   }
   cast = builder.createConvert(loc, builder.getIntPtrType(), cast);
   static_cast<AnyValueStack *>(this)->pushValue(loc, builder, cast);
@@ -383,7 +383,7 @@ mlir::Value fir::factory::AnyAddressStack::fetch(mlir::Location loc,
   mlir::Value addr = static_cast<AnyValueStack *>(this)->fetch(loc, builder);
   if (auto boxProcType = llvm::dyn_cast<fir::BoxProcType>(addressType)) {
     mlir::Value cast = builder.createConvert(loc, boxProcType.getEleTy(), addr);
-    return builder.create<fir::EmboxProcOp>(loc, boxProcType, cast);
+    return fir::EmboxProcOp::create(builder, loc, boxProcType, cast);
   }
   return builder.createConvert(loc, addressType, addr);
 }


        

From flang-commits at lists.llvm.org  Mon Jul 21 21:14:45 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Mon, 21 Jul 2025 21:14:45 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
Message-ID: <llvm.org/llvm/llvm-project/pull/149959@github.com>


https://github.com/terapines-osc-mlir created https://github.com/llvm/llvm-project/pull/149959

This commmit is a supplement for https://github.com/llvm/llvm-project/pull/140374.
RFC:https://discourse.llvm.org/t/rfc-add-fir-affine-optimization-fir-pass-pipeline/86190/6

>From b145b2658498a48448d95a30ff74d9d254f4c7a1 Mon Sep 17 00:00:00 2001
From: Terapines MLIR <osc-mlir at terapines.com>
Date: Tue, 22 Jul 2025 11:09:20 +0800
Subject: [PATCH] [flang][fir]: New optimizer transform of `fir.if` to
 `scf.if`.

---
 flang/lib/Optimizer/Transforms/FIRToSCF.cpp | 50 +++++++++++++++++-
 flang/test/Fir/FirToSCF/if.fir              | 57 +++++++++++++++++++++
 2 files changed, 105 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Fir/FirToSCF/if.fir

diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index d7d1865bc56ba..ad9d8ec136b22 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,59 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                  Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = dyn_cast<fir::ResultOp>(srcTerminator);
+
+  /// Do a simple copy when fir.result is not used.
+  if (resultOp == nullptr) {
+    dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations());
+    return;
+  }
+
+  /// Drop the fir.result and insert a scf.yield in the end of the block
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    rewriter.create<scf::YieldOp>(resultOp.getLoc(), resultOp.getOperands());
+  }
+
+  rewriter.eraseOp(srcTerminator);
+}
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    mlir::Location loc = ifOp.getLoc();
+    bool hasElse = !ifOp.getElseRegion().empty();
+    auto scfIfOp = rewriter.create<scf::IfOp>(loc, ifOp->getResultTypes(),
+                                              ifOp.getCondition(), hasElse);
+
+    copyBlocksAndTransformResult(rewriter, ifOp.getThenRegion().front(),
+                                 scfIfOp.getThenRegion().front());
+
+    if (hasElse) {
+      copyBlocksAndTransformResult(rewriter, ifOp.getElseRegion().front(),
+                                   scfIfOp.getElseRegion().front());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/flang/test/Fir/FirToSCF/if.fir b/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 0000000000000..fbb2f179411cf
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,57 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_only(
+// CHECK-SAME:      %[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:           scf.if %[[ARG0]] {
+// CHECK:             %[[VAL_1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_else()   {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           scf.if %[[VAL_1]] {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           } else {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2]] -> (f32, f32) {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           } else {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}
\ No newline at end of file


From flang-commits at lists.llvm.org  Mon Jul 21 21:15:03 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 21:15:03 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f1047.170a0220.1a0fb5.0696@mx.google.com>


github-actions[bot] wrote:

<!--LLVM NEW CONTRIBUTOR COMMENT-->

Thank you for submitting a Pull Request (PR) to the LLVM Project!

This PR will be automatically labeled and the relevant teams will be notified.

If you wish to, you can add reviewers by using the "Reviewers" section on this page.

If this is not working for you, it is probably because you do not have write permissions for the repository. In which case you can instead tag reviewers by name in a comment by using `@` followed by their GitHub username.

If you have received no comments on your PR for a week, you can request a review by "ping"ing the PR by adding a comment “Ping”. The common courtesy "ping" rate is once a week. Please remember that you are asking for valuable time from other developers.

If you have further questions, they may be answered by the [LLVM GitHub User Guide](https://llvm.org/docs/GitHub.html).

You can also ask questions in a comment on this PR, on the [LLVM Discord](https://discord.com/invite/xS7Z362) or on the [forums](https://discourse.llvm.org/).

https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Mon Jul 21 21:15:34 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Mon, 21 Jul 2025 21:15:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f1066.050a0220.245136.02ac@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Terapines MLIR (terapines-osc-mlir)

<details>
<summary>Changes</summary>

This commmit is a supplement for https://github.com/llvm/llvm-project/pull/140374.
RFC:https://discourse.llvm.org/t/rfc-add-fir-affine-optimization-fir-pass-pipeline/86190/6

---
Full diff: https://github.com/llvm/llvm-project/pull/149959.diff


2 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/FIRToSCF.cpp (+48-2) 
- (added) flang/test/Fir/FirToSCF/if.fir (+57) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index d7d1865bc56ba..ad9d8ec136b22 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,59 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                  Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = dyn_cast<fir::ResultOp>(srcTerminator);
+
+  /// Do a simple copy when fir.result is not used.
+  if (resultOp == nullptr) {
+    dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations());
+    return;
+  }
+
+  /// Drop the fir.result and insert a scf.yield in the end of the block
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    rewriter.create<scf::YieldOp>(resultOp.getLoc(), resultOp.getOperands());
+  }
+
+  rewriter.eraseOp(srcTerminator);
+}
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    mlir::Location loc = ifOp.getLoc();
+    bool hasElse = !ifOp.getElseRegion().empty();
+    auto scfIfOp = rewriter.create<scf::IfOp>(loc, ifOp->getResultTypes(),
+                                              ifOp.getCondition(), hasElse);
+
+    copyBlocksAndTransformResult(rewriter, ifOp.getThenRegion().front(),
+                                 scfIfOp.getThenRegion().front());
+
+    if (hasElse) {
+      copyBlocksAndTransformResult(rewriter, ifOp.getElseRegion().front(),
+                                   scfIfOp.getElseRegion().front());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/flang/test/Fir/FirToSCF/if.fir b/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 0000000000000..fbb2f179411cf
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,57 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_only(
+// CHECK-SAME:      %[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:           scf.if %[[ARG0]] {
+// CHECK:             %[[VAL_1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_else()   {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           scf.if %[[VAL_1]] {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           } else {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2]] -> (f32, f32) {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           } else {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}
\ No newline at end of file

``````````

</details>


https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Mon Jul 21 22:31:10 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Mon, 21 Jul 2025 22:31:10 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f221e.170a0220.31d4f2.0f50@mx.google.com>


https://github.com/terapines-osc-mlir updated https://github.com/llvm/llvm-project/pull/149959

>From 358ac2c0322db026bfef41457804c00b9cfabc78 Mon Sep 17 00:00:00 2001
From: Terapines MLIR <osc-mlir at terapines.com>
Date: Tue, 22 Jul 2025 11:09:20 +0800
Subject: [PATCH] [flang][fir]: New optimizer transform of `fir.if` to
 `scf.if`.

---
 flang/lib/Optimizer/Transforms/FIRToSCF.cpp | 43 +++++++++++++++-
 flang/test/Fir/FirToSCF/if.fir              | 57 +++++++++++++++++++++
 2 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Fir/FirToSCF/if.fir

diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index d7d1865bc56ba..f3198329028e9 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                  Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = cast<fir::ResultOp>(srcTerminator);
+
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    rewriter.create<scf::YieldOp>(resultOp.getLoc(), resultOp.getOperands());
+  }
+
+  rewriter.eraseOp(srcTerminator);
+}
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    mlir::Location loc = ifOp.getLoc();
+    bool hasElse = !ifOp.getElseRegion().empty();
+    auto scfIfOp = rewriter.create<scf::IfOp>(loc, ifOp->getResultTypes(),
+                                              ifOp.getCondition(), hasElse);
+
+    copyBlocksAndTransformResult(rewriter, ifOp.getThenRegion().front(),
+                                 scfIfOp.getThenRegion().front());
+
+    if (hasElse) {
+      copyBlocksAndTransformResult(rewriter, ifOp.getElseRegion().front(),
+                                   scfIfOp.getElseRegion().front());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/flang/test/Fir/FirToSCF/if.fir b/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 0000000000000..fbb2f179411cf
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,57 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_only(
+// CHECK-SAME:      %[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:           scf.if %[[ARG0]] {
+// CHECK:             %[[VAL_1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_else()   {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           scf.if %[[VAL_1]] {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           } else {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2]] -> (f32, f32) {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           } else {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}
\ No newline at end of file


From flang-commits at lists.llvm.org  Mon Jul 21 22:43:39 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Mon, 21 Jul 2025 22:43:39 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f250b.a70a0220.e27ff.145a@mx.google.com>


https://github.com/terapines-osc-mlir updated https://github.com/llvm/llvm-project/pull/149959

>From 9ef83e54213275e742727fb025a345d003694edc Mon Sep 17 00:00:00 2001
From: Terapines MLIR <osc-mlir at terapines.com>
Date: Tue, 22 Jul 2025 11:09:20 +0800
Subject: [PATCH] [flang][fir]: New optimizer transform of `fir.if` to
 `scf.if`.

---
 flang/lib/Optimizer/Transforms/FIRToSCF.cpp | 43 +++++++++++++++-
 flang/test/Fir/FirToSCF/if.fir              | 57 +++++++++++++++++++++
 2 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Fir/FirToSCF/if.fir

diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index d7d1865bc56ba..f3198329028e9 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                  Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = cast<fir::ResultOp>(srcTerminator);
+
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    rewriter.create<scf::YieldOp>(resultOp.getLoc(), resultOp.getOperands());
+  }
+
+  rewriter.eraseOp(srcTerminator);
+}
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    mlir::Location loc = ifOp.getLoc();
+    bool hasElse = !ifOp.getElseRegion().empty();
+    auto scfIfOp = rewriter.create<scf::IfOp>(loc, ifOp->getResultTypes(),
+                                              ifOp.getCondition(), hasElse);
+
+    copyBlocksAndTransformResult(rewriter, ifOp.getThenRegion().front(),
+                                 scfIfOp.getThenRegion().front());
+
+    if (hasElse) {
+      copyBlocksAndTransformResult(rewriter, ifOp.getElseRegion().front(),
+                                   scfIfOp.getElseRegion().front());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/flang/test/Fir/FirToSCF/if.fir b/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 0000000000000..03be264c4cdf5
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,57 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_only(
+// CHECK-SAME:      %[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:           scf.if %[[ARG0]] {
+// CHECK:             %[[VAL_1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_else()   {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           scf.if %[[VAL_1]] {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           } else {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2]] -> (f32, f32) {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           } else {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}


From flang-commits at lists.llvm.org  Mon Jul 21 22:44:04 2025
From: flang-commits at lists.llvm.org (Ming Yan via flang-commits)
Date: Mon, 21 Jul 2025 22:44:04 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f2524.630a0220.98d75.008c@mx.google.com>


https://github.com/NexMing approved this pull request.


https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Tue Jul 22 00:01:08 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Tue, 22 Jul 2025 00:01:08 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f3734.170a0220.2c927d.17b1@mx.google.com>


https://github.com/terapines-osc-mlir updated https://github.com/llvm/llvm-project/pull/149959

>From 77baecd5ecbd0c42a2d0e7785a78fd2409c90a62 Mon Sep 17 00:00:00 2001
From: Terapines MLIR <osc-mlir at terapines.com>
Date: Tue, 22 Jul 2025 11:09:20 +0800
Subject: [PATCH] [flang][fir]: New optimizer transform of `fir.if` to
 `scf.if`.

---
 flang/lib/Optimizer/Transforms/FIRToSCF.cpp | 43 +++++++++++++++-
 flang/test/Fir/FirToSCF/if.fir              | 57 +++++++++++++++++++++
 2 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Fir/FirToSCF/if.fir

diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index d7d1865bc56ba..f3198329028e9 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                  Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = cast<fir::ResultOp>(srcTerminator);
+
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    rewriter.create<scf::YieldOp>(resultOp.getLoc(), resultOp.getOperands());
+  }
+
+  rewriter.eraseOp(srcTerminator);
+}
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    mlir::Location loc = ifOp.getLoc();
+    bool hasElse = !ifOp.getElseRegion().empty();
+    auto scfIfOp = rewriter.create<scf::IfOp>(loc, ifOp->getResultTypes(),
+                                              ifOp.getCondition(), hasElse);
+
+    copyBlocksAndTransformResult(rewriter, ifOp.getThenRegion().front(),
+                                 scfIfOp.getThenRegion().front());
+
+    if (hasElse) {
+      copyBlocksAndTransformResult(rewriter, ifOp.getElseRegion().front(),
+                                   scfIfOp.getElseRegion().front());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/flang/test/Fir/FirToSCF/if.fir b/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 0000000000000..03be264c4cdf5
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,57 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_only(
+// CHECK-SAME:      %[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:           scf.if %[[ARG0]] {
+// CHECK:             %[[VAL_1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_else()   {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           scf.if %[[VAL_1]] {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           } else {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2]] -> (f32, f32) {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           } else {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}


From flang-commits at lists.llvm.org  Tue Jul 22 01:19:20 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 01:19:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add conflicting checks for EXTERNAL
	attribute (PR #149973)
Message-ID: <llvm.org/llvm/llvm-project/pull/149973@github.com>


https://github.com/foxtran created https://github.com/llvm/llvm-project/pull/149973

Fixes #149771 

>From 2dd5e5c20774aa824a828c163dc61bf65b963226 Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Tue, 22 Jul 2025 10:11:03 +0200
Subject: [PATCH 1/2] Add checks of EXTERNAL attribute

---
 flang/lib/Semantics/check-declarations.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index a2f2906af10b8..a99a65b0dc0b4 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -66,6 +66,7 @@ class CheckHelper {
   void CheckVolatile(const Symbol &, const DerivedTypeSpec *);
   void CheckContiguous(const Symbol &);
   void CheckPointer(const Symbol &);
+  void CheckExternal(const Symbol &);
   void CheckPassArg(
       const Symbol &proc, const Symbol *interface, const WithPassArg &);
   void CheckProcBinding(const Symbol &, const ProcBindingDetails &);
@@ -983,6 +984,7 @@ void CheckHelper::CheckObjectEntity(
     }
   }
   if (symbol.attrs().test(Attr::EXTERNAL)) {
+    CheckExternal(symbol);
     SayWithDeclaration(symbol,
         "'%s' is a data object and may not be EXTERNAL"_err_en_US,
         symbol.name());
@@ -2460,6 +2462,11 @@ void CheckHelper::CheckPointer(const Symbol &symbol) { // C852
   }
 }
 
+void CheckHelper::CheckExternal(const Symbol &symbol) {
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::INTRINSIC); // F'2023 C842
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::PARAMETER);
+}
+
 // C760 constraints on the passed-object dummy argument
 // C757 constraints on procedure pointer components
 void CheckHelper::CheckPassArg(

>From 50bf419ed31a9a467bc1353a8a531e5dca03a889 Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Tue, 22 Jul 2025 10:14:55 +0200
Subject: [PATCH 2/2] Add tests for EXTERNAL attribute

---
 flang/test/Semantics/declarations09.f90 | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 flang/test/Semantics/declarations09.f90

diff --git a/flang/test/Semantics/declarations09.f90 b/flang/test/Semantics/declarations09.f90
new file mode 100644
index 0000000000000..4bd08a1749721
--- /dev/null
+++ b/flang/test/Semantics/declarations09.f90
@@ -0,0 +1,11 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+
+subroutine bug149771
+
+  !ERROR: 'x1' may not have both the EXTERNAL and INTRINSIC attributes
+  integer, external, intrinsic :: x1
+
+  !ERROR: 'x2' may not have both the EXTERNAL and PARAMETER attributes
+  integer, external, parameter :: x2
+
+end subroutine bug149771


From flang-commits at lists.llvm.org  Tue Jul 22 01:19:52 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 01:19:52 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add conflicting checks for EXTERNAL
	attribute (PR #149973)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149973@github.com>
Message-ID: <687f49a8.a70a0220.31effe.251c@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: foxtran (foxtran)

<details>
<summary>Changes</summary>

Fixes #<!-- -->149771 

---
Full diff: https://github.com/llvm/llvm-project/pull/149973.diff


2 Files Affected:

- (modified) flang/lib/Semantics/check-declarations.cpp (+7) 
- (added) flang/test/Semantics/declarations09.f90 (+11) 


``````````diff
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index a2f2906af10b8..a99a65b0dc0b4 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -66,6 +66,7 @@ class CheckHelper {
   void CheckVolatile(const Symbol &, const DerivedTypeSpec *);
   void CheckContiguous(const Symbol &);
   void CheckPointer(const Symbol &);
+  void CheckExternal(const Symbol &);
   void CheckPassArg(
       const Symbol &proc, const Symbol *interface, const WithPassArg &);
   void CheckProcBinding(const Symbol &, const ProcBindingDetails &);
@@ -983,6 +984,7 @@ void CheckHelper::CheckObjectEntity(
     }
   }
   if (symbol.attrs().test(Attr::EXTERNAL)) {
+    CheckExternal(symbol);
     SayWithDeclaration(symbol,
         "'%s' is a data object and may not be EXTERNAL"_err_en_US,
         symbol.name());
@@ -2460,6 +2462,11 @@ void CheckHelper::CheckPointer(const Symbol &symbol) { // C852
   }
 }
 
+void CheckHelper::CheckExternal(const Symbol &symbol) {
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::INTRINSIC); // F'2023 C842
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::PARAMETER);
+}
+
 // C760 constraints on the passed-object dummy argument
 // C757 constraints on procedure pointer components
 void CheckHelper::CheckPassArg(
diff --git a/flang/test/Semantics/declarations09.f90 b/flang/test/Semantics/declarations09.f90
new file mode 100644
index 0000000000000..4bd08a1749721
--- /dev/null
+++ b/flang/test/Semantics/declarations09.f90
@@ -0,0 +1,11 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+
+subroutine bug149771
+
+  !ERROR: 'x1' may not have both the EXTERNAL and INTRINSIC attributes
+  integer, external, intrinsic :: x1
+
+  !ERROR: 'x2' may not have both the EXTERNAL and PARAMETER attributes
+  integer, external, parameter :: x2
+
+end subroutine bug149771

``````````

</details>


https://github.com/llvm/llvm-project/pull/149973

From flang-commits at lists.llvm.org  Tue Jul 22 01:47:34 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Tue, 22 Jul 2025 01:47:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f5026.170a0220.2180ee.2f80@mx.google.com>


https://github.com/terapines-osc-mlir updated https://github.com/llvm/llvm-project/pull/149959

>From de4de9c815273820376be8df2880a5a0d1bbd841 Mon Sep 17 00:00:00 2001
From: Terapines MLIR <osc-mlir at terapines.com>
Date: Tue, 22 Jul 2025 11:09:20 +0800
Subject: [PATCH] [flang][fir]: New optimizer transform of `fir.if` to
 `scf.if`.

---
 flang/lib/Optimizer/Transforms/FIRToSCF.cpp | 43 +++++++++++++++-
 flang/test/Fir/FirToSCF/if.fir              | 57 +++++++++++++++++++++
 2 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Fir/FirToSCF/if.fir

diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index d7d1865bc56ba..f3198329028e9 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                  Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = cast<fir::ResultOp>(srcTerminator);
+
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    rewriter.create<scf::YieldOp>(resultOp.getLoc(), resultOp.getOperands());
+  }
+
+  rewriter.eraseOp(srcTerminator);
+}
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    mlir::Location loc = ifOp.getLoc();
+    bool hasElse = !ifOp.getElseRegion().empty();
+    auto scfIfOp = rewriter.create<scf::IfOp>(loc, ifOp->getResultTypes(),
+                                              ifOp.getCondition(), hasElse);
+
+    copyBlocksAndTransformResult(rewriter, ifOp.getThenRegion().front(),
+                                 scfIfOp.getThenRegion().front());
+
+    if (hasElse) {
+      copyBlocksAndTransformResult(rewriter, ifOp.getElseRegion().front(),
+                                   scfIfOp.getElseRegion().front());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/flang/test/Fir/FirToSCF/if.fir b/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 0000000000000..03be264c4cdf5
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,57 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_only(
+// CHECK-SAME:      %[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:           scf.if %[[ARG0]] {
+// CHECK:             %[[VAL_1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_else()   {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           scf.if %[[VAL_1]] {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           } else {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2]] -> (f32, f32) {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           } else {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}


From flang-commits at lists.llvm.org  Tue Jul 22 02:17:12 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 22 Jul 2025 02:17:12 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <687f5718.170a0220.1b7f28.0f5a@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks for the updates. LGTM

https://github.com/llvm/llvm-project/pull/149137

From flang-commits at lists.llvm.org  Tue Jul 22 02:30:15 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 22 Jul 2025 02:30:15 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687f5a27.170a0220.aa548.1f9b@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
tblah wrote:

There is support for libquadmath and glibc>=2.4.1 in flang-rt. It defines functions such as `RTDEF(SinF128)`.

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 02:30:40 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 22 Jul 2025 02:30:40 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687f5a40.170a0220.37a56a.20be@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 02:36:55 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Tue, 22 Jul 2025 02:36:55 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687f5bb7.630a0220.31eae0.1213@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
c8ef wrote:

I mean, by directly using the C23 functions added in glibc.

- https://www.phoronix.com/news/Glibc-2.41-More-C23

Maybe we could use this approach for now and refactor it in a later PR? Other functions, like sind, appear to suffer from the same issue.

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 02:58:01 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Tue, 22 Jul 2025 02:58:01 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang] Implement !$omp unroll using
 omp.unroll_heuristic (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <687f60a9.050a0220.37a1d8.128d@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `flang-aarch64-out-of-tree` running on `linaro-flang-aarch64-out-of-tree` while building `flang` at step 11 "test-build-flang-unified-tree-check-flang".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/53/builds/17907

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 11 (test-build-flang-unified-tree-check-flang) failure: test (failure)
******************** TEST 'Flang :: Lower/OpenMP/unroll-heuristic02.f90' FAILED ********************
Exit Code: 1

Command Output (stderr):
--
/home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/build_flang/bin/flang -fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - /home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90 2>&1 | /home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/build_llvm/bin/FileCheck /home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90 # RUN: at line 1
+ /home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/build_flang/bin/flang -fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - /home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90
+ /home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/build_llvm/bin/FileCheck /home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90
/home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90:23:14: error: CHECK-NEXT: expected string not found in input
!CHECK-NEXT: %18 = arith.cmpi slt, %17, %c0_i32 : i32
             ^
<stdin>:25:34: note: scanning from here
 %c1_i32 = arith.constant 1 : i32
                                 ^
<stdin>:26:2: note: possible intended match here
 %20 = arith.cmpi slt, %19, %c0_i32 : i32
 ^

Input file: <stdin>
Check file: /home/tcwg-buildbot/worker/flang-aarch64-out-of-tree/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90

-dump-input=help explains the following input dump.

Input was:
<<<<<<
           .
           .
           .
          20:  %16:2 = hlfir.declare %15 {uniq_name = "_QFomp_unroll_heuristic_nested02Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>) 
          21:  %17 = fir.load %9#0 : !fir.ref<i32> 
          22:  %18 = fir.load %10#0 : !fir.ref<i32> 
          23:  %19 = fir.load %8#0 : !fir.ref<i32> 
          24:  %c0_i32 = arith.constant 0 : i32 
          25:  %c1_i32 = arith.constant 1 : i32 
next:23'0                                      X error: no match found
          26:  %20 = arith.cmpi slt, %19, %c0_i32 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
next:23'1      ?                                         possible intended match
          27:  %21 = arith.subi %c0_i32, %19 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          28:  %22 = arith.select %20, %21, %19 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          29:  %23 = arith.select %20, %18, %17 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          30:  %24 = arith.select %20, %17, %18 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          31:  %25 = arith.subi %24, %23 overflow<nuw> : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           .
           .
...

```

</details>

https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Tue Jul 22 03:31:11 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Tue, 22 Jul 2025 03:31:11 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang] Implement !$omp unroll using
 omp.unroll_heuristic (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <687f686f.170a0220.db915.eb95@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `premerge-monolithic-linux` running on `premerge-linux-1` while building `flang` at step 7 "test-build-unified-tree-check-all".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/153/builds/38911

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 7 (test-build-unified-tree-check-all) failure: test (failure)
******************** TEST 'Flang :: Lower/OpenMP/unroll-heuristic02.f90' FAILED ********************
Exit Code: 1

Command Output (stderr):
--
/build/buildbot/premerge-monolithic-linux/build/bin/flang -fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - /build/buildbot/premerge-monolithic-linux/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90 2>&1 | /build/buildbot/premerge-monolithic-linux/build/bin/FileCheck /build/buildbot/premerge-monolithic-linux/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90 # RUN: at line 1
+ /build/buildbot/premerge-monolithic-linux/build/bin/flang -fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - /build/buildbot/premerge-monolithic-linux/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90
+ /build/buildbot/premerge-monolithic-linux/build/bin/FileCheck /build/buildbot/premerge-monolithic-linux/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90
/build/buildbot/premerge-monolithic-linux/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90:23:14: error: CHECK-NEXT: expected string not found in input
!CHECK-NEXT: %18 = arith.cmpi slt, %17, %c0_i32 : i32
             ^
<stdin>:25:34: note: scanning from here
 %c1_i32 = arith.constant 1 : i32
                                 ^
<stdin>:26:2: note: possible intended match here
 %20 = arith.cmpi slt, %19, %c0_i32 : i32
 ^

Input file: <stdin>
Check file: /build/buildbot/premerge-monolithic-linux/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90

-dump-input=help explains the following input dump.

Input was:
<<<<<<
           .
           .
           .
          20:  %16:2 = hlfir.declare %15 {uniq_name = "_QFomp_unroll_heuristic_nested02Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>) 
          21:  %17 = fir.load %9#0 : !fir.ref<i32> 
          22:  %18 = fir.load %10#0 : !fir.ref<i32> 
          23:  %19 = fir.load %8#0 : !fir.ref<i32> 
          24:  %c0_i32 = arith.constant 0 : i32 
          25:  %c1_i32 = arith.constant 1 : i32 
next:23'0                                      X error: no match found
          26:  %20 = arith.cmpi slt, %19, %c0_i32 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
next:23'1      ?                                         possible intended match
          27:  %21 = arith.subi %c0_i32, %19 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          28:  %22 = arith.select %20, %21, %19 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          29:  %23 = arith.select %20, %18, %17 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          30:  %24 = arith.select %20, %17, %18 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          31:  %25 = arith.subi %24, %23 overflow<nuw> : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           .
           .
...

```

</details>

https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Tue Jul 22 03:34:17 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 03:34:17 -0700 (PDT)
Subject: [flang-commits] [flang] 87d73ff - [Flang] Regenerate unroll test
Message-ID: <687f6929.170a0220.36e92.2b7d@mx.google.com>


Author: Michael Kruse
Date: 2025-07-22T12:33:44+02:00
New Revision: 87d73ff7fdc8fa7acdc2137654f4c4d8b051c423

URL: https://github.com/llvm/llvm-project/commit/87d73ff7fdc8fa7acdc2137654f4c4d8b051c423
DIFF: https://github.com/llvm/llvm-project/commit/87d73ff7fdc8fa7acdc2137654f4c4d8b051c423.diff

LOG: [Flang] Regenerate unroll test

Added: 
    

Modified: 
    flang/test/Lower/OpenMP/unroll-heuristic02.f90

Removed: 
    


################################################################################
diff  --git a/flang/test/Lower/OpenMP/unroll-heuristic02.f90 b/flang/test/Lower/OpenMP/unroll-heuristic02.f90
index 669f185f910c4..14f694d6cdb78 100644
--- a/flang/test/Lower/OpenMP/unroll-heuristic02.f90
+++ b/flang/test/Lower/OpenMP/unroll-heuristic02.f90
@@ -17,54 +17,81 @@ subroutine omp_unroll_heuristic_nested02(outer_lb, outer_ub, outer_inc, inner_lb
 end subroutine omp_unroll_heuristic_nested02
 
 
-!CHECK-LABEL: func.func @_QPomp_unroll_heuristic_nested02(%arg0: !fir.ref<i32> {fir.bindc_name = "outer_lb"}, %arg1: !fir.ref<i32> {fir.bindc_name = "outer_ub"}, %arg2: !fir.ref<i32> {fir.bindc_name = "outer_inc"}, %arg3: !fir.ref<i32> {fir.bindc_name = "inner_lb"}, %arg4: !fir.ref<i32> {fir.bindc_name = "inner_ub"}, %arg5: !fir.ref<i32> {fir.bindc_name = "inner_inc"}) {
-!CHECK:      %c0_i32 = arith.constant 0 : i32
-!CHECK-NEXT: %c1_i32 = arith.constant 1 : i32
-!CHECK-NEXT: %18 = arith.cmpi slt, %17, %c0_i32 : i32
-!CHECK-NEXT: %19 = arith.subi %c0_i32, %17 : i32
-!CHECK-NEXT: %20 = arith.select %18, %19, %17 : i32
-!CHECK-NEXT: %21 = arith.select %18, %16, %15 : i32
-!CHECK-NEXT: %22 = arith.select %18, %15, %16 : i32
-!CHECK-NEXT: %23 = arith.subi %22, %21 overflow<nuw> : i32
-!CHECK-NEXT: %24 = arith.divui %23, %20 : i32
-!CHECK-NEXT: %25 = arith.addi %24, %c1_i32 overflow<nuw> : i32
-!CHECK-NEXT: %26 = arith.cmpi slt, %22, %21 : i32
-!CHECK-NEXT: %27 = arith.select %26, %c0_i32, %25 : i32
-!CHECK-NEXT: %canonloop_s0 = omp.new_cli
-!CHECK-NEXT: omp.canonical_loop(%canonloop_s0) %iv : i32 in range(%27) {
-!CHECK-NEXT:   %28 = arith.muli %iv, %17 : i32
-!CHECK-NEXT:   %29 = arith.addi %15, %28 : i32
-!CHECK-NEXT:   hlfir.assign %29 to %14#0 : i32, !fir.ref<i32>
-!CHECK-NEXT:   %30 = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFomp_unroll_heuristic_nested02Ej"}
-!CHECK-NEXT:   %31:2 = hlfir.declare %30 {uniq_name = "_QFomp_unroll_heuristic_nested02Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK-NEXT:   %32 = fir.load %4#0 : !fir.ref<i32>
-!CHECK-NEXT:   %33 = fir.load %5#0 : !fir.ref<i32>
-!CHECK-NEXT:   %34 = fir.load %3#0 : !fir.ref<i32>
-!CHECK-NEXT:   %c0_i32_0 = arith.constant 0 : i32
-!CHECK-NEXT:   %c1_i32_1 = arith.constant 1 : i32
-!CHECK-NEXT:   %35 = arith.cmpi slt, %34, %c0_i32_0 : i32
-!CHECK-NEXT:   %36 = arith.subi %c0_i32_0, %34 : i32
-!CHECK-NEXT:   %37 = arith.select %35, %36, %34 : i32
-!CHECK-NEXT:   %38 = arith.select %35, %33, %32 : i32
-!CHECK-NEXT:   %39 = arith.select %35, %32, %33 : i32
-!CHECK-NEXT:   %40 = arith.subi %39, %38 overflow<nuw> : i32
-!CHECK-NEXT:   %41 = arith.divui %40, %37 : i32
-!CHECK-NEXT:   %42 = arith.addi %41, %c1_i32_1 overflow<nuw> : i32
-!CHECK-NEXT:   %43 = arith.cmpi slt, %39, %38 : i32
-!CHECK-NEXT:   %44 = arith.select %43, %c0_i32_0, %42 : i32
-!CHECK-NEXT:   %canonloop_s0_s0 = omp.new_cli
-!CHECK-NEXT:   omp.canonical_loop(%canonloop_s0_s0) %iv_2 : i32 in range(%44) {
-!CHECK-NEXT:     %45 = arith.muli %iv_2, %34 : i32
-!CHECK-NEXT:     %46 = arith.addi %32, %45 : i32
-!CHECK-NEXT:     hlfir.assign %46 to %31#0 : i32, !fir.ref<i32>
-!CHECK-NEXT:     %47 = fir.load %14#0 : !fir.ref<i32>
-!CHECK-NEXT:     %48 = fir.load %31#0 : !fir.ref<i32>
-!CHECK-NEXT:     %49 = arith.addi %47, %48 : i32
-!CHECK-NEXT:     hlfir.assign %49 to %12#0 : i32, !fir.ref<i32>
-!CHECK-NEXT:     omp.terminator
-!CHECK-NEXT:   }
-!CHECK-NEXT:   omp.unroll_heuristic(%canonloop_s0_s0)
-!CHECK-NEXT:   omp.terminator
-!CHECK-NEXT: }
-!CHECK-NEXT: omp.unroll_heuristic(%canonloop_s0)
-!CHECK-NEXT: return
+!CHECK-LABEL:   func.func @_QPomp_unroll_heuristic_nested02(
+!CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "outer_lb"},
+!CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "outer_ub"},
+!CHECK-SAME:      %[[ARG2:.*]]: !fir.ref<i32> {fir.bindc_name = "outer_inc"},
+!CHECK-SAME:      %[[ARG3:.*]]: !fir.ref<i32> {fir.bindc_name = "inner_lb"},
+!CHECK-SAME:      %[[ARG4:.*]]: !fir.ref<i32> {fir.bindc_name = "inner_ub"},
+!CHECK-SAME:      %[[ARG5:.*]]: !fir.ref<i32> {fir.bindc_name = "inner_inc"}) {
+!CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+!CHECK:           %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFomp_unroll_heuristic_nested02Ei"}
+!CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFomp_unroll_heuristic_nested02Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[ARG5]] dummy_scope %[[VAL_0]] {uniq_name = "_QFomp_unroll_heuristic_nested02Einner_inc"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %[[VAL_0]] {uniq_name = "_QFomp_unroll_heuristic_nested02Einner_lb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG4]] dummy_scope %[[VAL_0]] {uniq_name = "_QFomp_unroll_heuristic_nested02Einner_ub"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFomp_unroll_heuristic_nested02Ej"}
+!CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFomp_unroll_heuristic_nested02Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[VAL_0]] {uniq_name = "_QFomp_unroll_heuristic_nested02Eouter_inc"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {uniq_name = "_QFomp_unroll_heuristic_nested02Eouter_lb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %[[VAL_0]] {uniq_name = "_QFomp_unroll_heuristic_nested02Eouter_ub"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "res", uniq_name = "_QFomp_unroll_heuristic_nested02Eres"}
+!CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFomp_unroll_heuristic_nested02Eres"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_13:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFomp_unroll_heuristic_nested02Ei"}
+!CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFomp_unroll_heuristic_nested02Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_15:.*]] = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFomp_unroll_heuristic_nested02Ej"}
+!CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFomp_unroll_heuristic_nested02Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[VAL_17:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<i32>
+!CHECK:           %[[VAL_18:.*]] = fir.load %[[VAL_10]]#0 : !fir.ref<i32>
+!CHECK:           %[[VAL_19:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+!CHECK:           %[[VAL_20:.*]] = arith.constant 0 : i32
+!CHECK:           %[[VAL_21:.*]] = arith.constant 1 : i32
+!CHECK:           %[[VAL_22:.*]] = arith.cmpi slt, %[[VAL_19]], %[[VAL_20]] : i32
+!CHECK:           %[[VAL_23:.*]] = arith.subi %[[VAL_20]], %[[VAL_19]] : i32
+!CHECK:           %[[VAL_24:.*]] = arith.select %[[VAL_22]], %[[VAL_23]], %[[VAL_19]] : i32
+!CHECK:           %[[VAL_25:.*]] = arith.select %[[VAL_22]], %[[VAL_18]], %[[VAL_17]] : i32
+!CHECK:           %[[VAL_26:.*]] = arith.select %[[VAL_22]], %[[VAL_17]], %[[VAL_18]] : i32
+!CHECK:           %[[VAL_27:.*]] = arith.subi %[[VAL_26]], %[[VAL_25]] overflow<nuw> : i32
+!CHECK:           %[[VAL_28:.*]] = arith.divui %[[VAL_27]], %[[VAL_24]] : i32
+!CHECK:           %[[VAL_29:.*]] = arith.addi %[[VAL_28]], %[[VAL_21]] overflow<nuw> : i32
+!CHECK:           %[[VAL_30:.*]] = arith.cmpi slt, %[[VAL_26]], %[[VAL_25]] : i32
+!CHECK:           %[[VAL_31:.*]] = arith.select %[[VAL_30]], %[[VAL_20]], %[[VAL_29]] : i32
+!CHECK:           %[[VAL_32:.*]] = omp.new_cli
+!CHECK:           omp.canonical_loop(%[[VAL_32]]) %[[VAL_33:.*]] : i32 in range(%[[VAL_31]]) {
+!CHECK:             %[[VAL_34:.*]] = arith.muli %[[VAL_33]], %[[VAL_19]] : i32
+!CHECK:             %[[VAL_35:.*]] = arith.addi %[[VAL_17]], %[[VAL_34]] : i32
+!CHECK:             hlfir.assign %[[VAL_35]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
+!CHECK:             %[[VAL_36:.*]] = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFomp_unroll_heuristic_nested02Ej"}
+!CHECK:             %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_36]] {uniq_name = "_QFomp_unroll_heuristic_nested02Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:             %[[VAL_38:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
+!CHECK:             %[[VAL_39:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+!CHECK:             %[[VAL_40:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
+!CHECK:             %[[VAL_41:.*]] = arith.constant 0 : i32
+!CHECK:             %[[VAL_42:.*]] = arith.constant 1 : i32
+!CHECK:             %[[VAL_43:.*]] = arith.cmpi slt, %[[VAL_40]], %[[VAL_41]] : i32
+!CHECK:             %[[VAL_44:.*]] = arith.subi %[[VAL_41]], %[[VAL_40]] : i32
+!CHECK:             %[[VAL_45:.*]] = arith.select %[[VAL_43]], %[[VAL_44]], %[[VAL_40]] : i32
+!CHECK:             %[[VAL_46:.*]] = arith.select %[[VAL_43]], %[[VAL_39]], %[[VAL_38]] : i32
+!CHECK:             %[[VAL_47:.*]] = arith.select %[[VAL_43]], %[[VAL_38]], %[[VAL_39]] : i32
+!CHECK:             %[[VAL_48:.*]] = arith.subi %[[VAL_47]], %[[VAL_46]] overflow<nuw> : i32
+!CHECK:             %[[VAL_49:.*]] = arith.divui %[[VAL_48]], %[[VAL_45]] : i32
+!CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_49]], %[[VAL_42]] overflow<nuw> : i32
+!CHECK:             %[[VAL_51:.*]] = arith.cmpi slt, %[[VAL_47]], %[[VAL_46]] : i32
+!CHECK:             %[[VAL_52:.*]] = arith.select %[[VAL_51]], %[[VAL_41]], %[[VAL_50]] : i32
+!CHECK:             %[[VAL_53:.*]] = omp.new_cli
+!CHECK:             omp.canonical_loop(%[[VAL_53]]) %[[VAL_54:.*]] : i32 in range(%[[VAL_52]]) {
+!CHECK:               %[[VAL_55:.*]] = arith.muli %[[VAL_54]], %[[VAL_40]] : i32
+!CHECK:               %[[VAL_56:.*]] = arith.addi %[[VAL_38]], %[[VAL_55]] : i32
+!CHECK:               hlfir.assign %[[VAL_56]] to %[[VAL_37]]#0 : i32, !fir.ref<i32>
+!CHECK:               %[[VAL_57:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
+!CHECK:               %[[VAL_58:.*]] = fir.load %[[VAL_37]]#0 : !fir.ref<i32>
+!CHECK:               %[[VAL_59:.*]] = arith.addi %[[VAL_57]], %[[VAL_58]] : i32
+!CHECK:               hlfir.assign %[[VAL_59]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
+!CHECK:               omp.terminator
+!CHECK:             }
+!CHECK:             omp.unroll_heuristic(%[[VAL_53]])
+!CHECK:             omp.terminator
+!CHECK:           }
+!CHECK:           omp.unroll_heuristic(%[[VAL_32]])
+!CHECK:           return
+!CHECK:         }


        

From flang-commits at lists.llvm.org  Tue Jul 22 03:48:47 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 22 Jul 2025 03:48:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f6c8f.050a0220.2d0474.2884@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM thanks!

https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Tue Jul 22 04:06:52 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 22 Jul 2025 04:06:52 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f70cc.170a0220.1a507a.230a@mx.google.com>


================
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                  Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = cast<fir::ResultOp>(srcTerminator);
+
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    rewriter.create<scf::YieldOp>(resultOp.getLoc(), resultOp.getOperands());
----------------
tblah wrote:

Note that this old syntax is being replaced with `YieldOp::create(rewriter, ....)`. For example, see https://github.com/llvm/llvm-project/pull/149917

https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Tue Jul 22 04:57:10 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 04:57:10 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add conflicting checks for EXTERNAL
	attribute (PR #149973)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149973@github.com>
Message-ID: <687f7c96.170a0220.2ead1c.23e2@mx.google.com>


https://github.com/foxtran updated https://github.com/llvm/llvm-project/pull/149973

>From 2dd5e5c20774aa824a828c163dc61bf65b963226 Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Tue, 22 Jul 2025 10:11:03 +0200
Subject: [PATCH 1/3] Add checks of EXTERNAL attribute

---
 flang/lib/Semantics/check-declarations.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index a2f2906af10b8..a99a65b0dc0b4 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -66,6 +66,7 @@ class CheckHelper {
   void CheckVolatile(const Symbol &, const DerivedTypeSpec *);
   void CheckContiguous(const Symbol &);
   void CheckPointer(const Symbol &);
+  void CheckExternal(const Symbol &);
   void CheckPassArg(
       const Symbol &proc, const Symbol *interface, const WithPassArg &);
   void CheckProcBinding(const Symbol &, const ProcBindingDetails &);
@@ -983,6 +984,7 @@ void CheckHelper::CheckObjectEntity(
     }
   }
   if (symbol.attrs().test(Attr::EXTERNAL)) {
+    CheckExternal(symbol);
     SayWithDeclaration(symbol,
         "'%s' is a data object and may not be EXTERNAL"_err_en_US,
         symbol.name());
@@ -2460,6 +2462,11 @@ void CheckHelper::CheckPointer(const Symbol &symbol) { // C852
   }
 }
 
+void CheckHelper::CheckExternal(const Symbol &symbol) {
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::INTRINSIC); // F'2023 C842
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::PARAMETER);
+}
+
 // C760 constraints on the passed-object dummy argument
 // C757 constraints on procedure pointer components
 void CheckHelper::CheckPassArg(

>From 50bf419ed31a9a467bc1353a8a531e5dca03a889 Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Tue, 22 Jul 2025 10:14:55 +0200
Subject: [PATCH 2/3] Add tests for EXTERNAL attribute

---
 flang/test/Semantics/declarations09.f90 | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 flang/test/Semantics/declarations09.f90

diff --git a/flang/test/Semantics/declarations09.f90 b/flang/test/Semantics/declarations09.f90
new file mode 100644
index 0000000000000..4bd08a1749721
--- /dev/null
+++ b/flang/test/Semantics/declarations09.f90
@@ -0,0 +1,11 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+
+subroutine bug149771
+
+  !ERROR: 'x1' may not have both the EXTERNAL and INTRINSIC attributes
+  integer, external, intrinsic :: x1
+
+  !ERROR: 'x2' may not have both the EXTERNAL and PARAMETER attributes
+  integer, external, parameter :: x2
+
+end subroutine bug149771

>From abf1dbf8004084b8974dc8ef35d3892e2a8611da Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Tue, 22 Jul 2025 13:56:47 +0200
Subject: [PATCH 3/3] Rename CheckExternal -> CheckExternalAttrConflicts

---
 flang/lib/Semantics/check-declarations.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index a99a65b0dc0b4..113543825558e 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -66,7 +66,7 @@ class CheckHelper {
   void CheckVolatile(const Symbol &, const DerivedTypeSpec *);
   void CheckContiguous(const Symbol &);
   void CheckPointer(const Symbol &);
-  void CheckExternal(const Symbol &);
+  void CheckExternalAttrConflicts(const Symbol &);
   void CheckPassArg(
       const Symbol &proc, const Symbol *interface, const WithPassArg &);
   void CheckProcBinding(const Symbol &, const ProcBindingDetails &);
@@ -984,7 +984,7 @@ void CheckHelper::CheckObjectEntity(
     }
   }
   if (symbol.attrs().test(Attr::EXTERNAL)) {
-    CheckExternal(symbol);
+    CheckExternalAttrConflicts(symbol);
     SayWithDeclaration(symbol,
         "'%s' is a data object and may not be EXTERNAL"_err_en_US,
         symbol.name());
@@ -2462,7 +2462,7 @@ void CheckHelper::CheckPointer(const Symbol &symbol) { // C852
   }
 }
 
-void CheckHelper::CheckExternal(const Symbol &symbol) {
+void CheckHelper::CheckExternalAttrConflicts(const Symbol &symbol) {
   CheckConflicting(symbol, Attr::EXTERNAL, Attr::INTRINSIC); // F'2023 C842
   CheckConflicting(symbol, Attr::EXTERNAL, Attr::PARAMETER);
 }


From flang-commits at lists.llvm.org  Tue Jul 22 05:13:25 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 05:13:25 -0700 (PDT)
Subject: [flang-commits] [flang] --Fix for issue #127426 (PR #150006)
Message-ID: <llvm.org/llvm/llvm-project/pull/150006@github.com>


https://github.com/aadeshps-mcw created https://github.com/llvm/llvm-project/pull/150006

None

>From 31627756179835a88454afa0d3c376ff16c513a1 Mon Sep 17 00:00:00 2001
From: Aadesh PremKumar <aadesh.premkumar at multicorewareinc.com>
Date: Tue, 22 Jul 2025 17:42:12 +0530
Subject: [PATCH] --Fix for issue #127426

---
 flang/lib/Parser/prescan.cpp |  1 +
 flang/test/Parser/inc.h      |  4 ++++
 flang/test/Parser/test.F     | 14 ++++++++++++++
 3 files changed, 19 insertions(+)
 create mode 100644 flang/test/Parser/inc.h
 create mode 100644 flang/test/Parser/test.F

diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 3a9a475c365ee..ceb3d91c216cc 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -435,6 +435,7 @@ void Prescanner::LabelField(TokenSequence &token) {
   int outCol{1};
   const char *start{at_};
   std::optional<int> badColumn;
+  SkipCComments();
   for (; *at_ != '\n' && column_ <= 6; ++at_) {
     if (*at_ == '\t') {
       ++at_;
diff --git a/flang/test/Parser/inc.h b/flang/test/Parser/inc.h
new file mode 100644
index 0000000000000..a754bfbbbc641
--- /dev/null
+++ b/flang/test/Parser/inc.h
@@ -0,0 +1,4 @@
+/* Old style C comments
+*comments
+*/
+#define VAL 1
diff --git a/flang/test/Parser/test.F b/flang/test/Parser/test.F
new file mode 100644
index 0000000000000..94436d70faa05
--- /dev/null
+++ b/flang/test/Parser/test.F
@@ -0,0 +1,14 @@
+! RUN: %flang_fc1 -fdebug-unparse-no-sema %s 2>&1 | FileCheck %s
+ 
+!CHECK: SUBROUTINE foo
+!CHECK: INTEGER i
+!CHECK: i = 1
+!CHECK: PRINT *, i
+!CHECK: END SUBROUTINE foo
+ 
+      subroutine foo ()
+#include "inc.h"
+       integer :: i
+        i = VAL
+        print *,i
+      end subroutine foo


From flang-commits at lists.llvm.org  Tue Jul 22 05:13:44 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 05:13:44 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #127426 (PR #150006)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150006@github.com>
Message-ID: <687f8078.050a0220.19a061.17f8@mx.google.com>


https://github.com/aadeshps-mcw edited https://github.com/llvm/llvm-project/pull/150006

From flang-commits at lists.llvm.org  Tue Jul 22 05:13:57 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 05:13:57 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #127426 (PR #150006)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150006@github.com>
Message-ID: <687f8085.050a0220.34944b.2b4f@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: Aadesh Premkumar (aadeshps-mcw)

<details>
<summary>Changes</summary>



---
Full diff: https://github.com/llvm/llvm-project/pull/150006.diff


3 Files Affected:

- (modified) flang/lib/Parser/prescan.cpp (+1) 
- (added) flang/test/Parser/inc.h (+4) 
- (added) flang/test/Parser/test.F (+14) 


``````````diff
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 3a9a475c365ee..ceb3d91c216cc 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -435,6 +435,7 @@ void Prescanner::LabelField(TokenSequence &token) {
   int outCol{1};
   const char *start{at_};
   std::optional<int> badColumn;
+  SkipCComments();
   for (; *at_ != '\n' && column_ <= 6; ++at_) {
     if (*at_ == '\t') {
       ++at_;
diff --git a/flang/test/Parser/inc.h b/flang/test/Parser/inc.h
new file mode 100644
index 0000000000000..a754bfbbbc641
--- /dev/null
+++ b/flang/test/Parser/inc.h
@@ -0,0 +1,4 @@
+/* Old style C comments
+*comments
+*/
+#define VAL 1
diff --git a/flang/test/Parser/test.F b/flang/test/Parser/test.F
new file mode 100644
index 0000000000000..94436d70faa05
--- /dev/null
+++ b/flang/test/Parser/test.F
@@ -0,0 +1,14 @@
+! RUN: %flang_fc1 -fdebug-unparse-no-sema %s 2>&1 | FileCheck %s
+ 
+!CHECK: SUBROUTINE foo
+!CHECK: INTEGER i
+!CHECK: i = 1
+!CHECK: PRINT *, i
+!CHECK: END SUBROUTINE foo
+ 
+      subroutine foo ()
+#include "inc.h"
+       integer :: i
+        i = VAL
+        print *,i
+      end subroutine foo

``````````

</details>


https://github.com/llvm/llvm-project/pull/150006

From flang-commits at lists.llvm.org  Tue Jul 22 05:17:10 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 05:17:10 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #132646 (PR #150007)
Message-ID: <llvm.org/llvm/llvm-project/pull/150007@github.com>


https://github.com/aadeshps-mcw created https://github.com/llvm/llvm-project/pull/150007

None

>From fb2b31b21b007dc5dca796aa7522f475ca9cd4b6 Mon Sep 17 00:00:00 2001
From: Aadesh PremKumar <aadesh.premkumar at multicorewareinc.com>
Date: Tue, 22 Jul 2025 17:45:43 +0530
Subject: [PATCH] --Fix for issue #132646

---
 flang/lib/Evaluate/intrinsics.cpp | 15 +++++++++------
 flang/test/Evaluate/bug132646.f90 | 24 ++++++++++++++++++++++++
 flang/test/Evaluate/folding04.f90 |  4 +---
 3 files changed, 34 insertions(+), 9 deletions(-)
 create mode 100644 flang/test/Evaluate/bug132646.f90

diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index e802915945e26..466bbed93951f 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1336,9 +1336,9 @@ static const SpecificIntrinsicInterface specificIntrinsicFunction[]{
     {{"log", {{"x", DefaultReal}}, DefaultReal}},
     {{"log10", {{"x", DefaultReal}}, DefaultReal}},
     {{"max0",
-         {{"a1", DefaultInt}, {"a2", DefaultInt},
-             {"a3", DefaultInt, Rank::elemental, Optionality::repeats}},
-         DefaultInt},
+         {{"a1", OperandInt}, {"a2", OperandInt},
+             {"a3", OperandInt, Rank::elemental, Optionality::repeats}},
+         OperandInt},
         "max", true, true},
     {{"max1",
          {{"a1", DefaultReal}, {"a2", DefaultReal},
@@ -1346,9 +1346,9 @@ static const SpecificIntrinsicInterface specificIntrinsicFunction[]{
          DefaultInt},
         "max", true, true},
     {{"min0",
-         {{"a1", DefaultInt}, {"a2", DefaultInt},
-             {"a3", DefaultInt, Rank::elemental, Optionality::repeats}},
-         DefaultInt},
+         {{"a1", OperandInt}, {"a2", OperandInt},
+             {"a3", OperandInt, Rank::elemental, Optionality::repeats}},
+         OperandInt},
         "min", true, true},
     {{"min1",
          {{"a1", DefaultReal}, {"a2", DefaultReal},
@@ -3454,6 +3454,9 @@ static DynamicType GetReturnType(const SpecificIntrinsicInterface &interface,
   case KindCode::defaultRealKind:
     category = TypeCategory::Real;
     break;
+  case KindCode::operand:
+    category = TypeCategory::Integer;
+    break;
   default:
     CRASH_NO_CASE;
   }
diff --git a/flang/test/Evaluate/bug132646.f90 b/flang/test/Evaluate/bug132646.f90
new file mode 100644
index 0000000000000..3ffa95589e3c8
--- /dev/null
+++ b/flang/test/Evaluate/bug132646.f90
@@ -0,0 +1,24 @@
+! RUN: %flang %s -o %t && %t | FileCheck %s
+
+program main
+    print *,kind(max0(1_1,2_1))
+    print *,kind(max0(1_2,2_2))
+    print *,kind(max0(1_4,2_4))
+    print *,kind(max0(1_8,2_8))
+
+    print *,kind(min0(1_1,2_1))
+    print *,kind(min0(1_2,2_2))
+    print *,kind(min0(1_4,2_4))
+    print *,kind(min0(1_8,2_8))
+end program main
+
+! CHECK: 1
+! CHECK-NEXT: 2
+! CHECK-NEXT: 4
+! CHECK-NEXT: 8
+
+! CHECK-NEXT: 1
+! CHECK-NEXT: 2
+! CHECK-NEXT: 4
+! CHECK-NEXT: 8
+
diff --git a/flang/test/Evaluate/folding04.f90 b/flang/test/Evaluate/folding04.f90
index 027db20f608b2..c5d5b5ac6c99a 100644
--- a/flang/test/Evaluate/folding04.f90
+++ b/flang/test/Evaluate/folding04.f90
@@ -86,9 +86,7 @@ module specific_extremums
   ! specified for f18 (converting the result).
   integer(8), parameter :: max_i32_8 = 2_8**31-1
   integer, parameter :: expected_min0 = int(min(max_i32_8, 2_8*max_i32_8), 4)
-  !WARN: portability: Argument types do not match specific intrinsic 'min0' requirements; using 'min' generic instead and converting the result to INTEGER(4) if needed [-Wuse-generic-intrinsic-when-specific-doesnt-match]
-  integer, parameter :: result_min0 =  min0(max_i32_8, 2_8*max_i32_8)
-  ! result_min0 would be -2  if arguments were converted to default integer.
+  integer(8), parameter :: result_min0 =  min0(max_i32_8, 2_8*max_i32_8)
   logical, parameter :: test_min0 = expected_min0 .EQ. result_min0
 
   real, parameter :: expected_amax0 = real(max(max_i32_8, 2_8*max_i32_8), 4)


From flang-commits at lists.llvm.org  Tue Jul 22 05:17:18 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 05:17:18 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #127426 (PR #150006)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150006@github.com>
Message-ID: <687f814e.170a0220.21fa1.328c@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions h,cpp -- flang/test/Parser/inc.h flang/lib/Parser/prescan.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/test/Parser/inc.h b/flang/test/Parser/inc.h
index a754bfbbb..15eb75b25 100644
--- a/flang/test/Parser/inc.h
+++ b/flang/test/Parser/inc.h
@@ -1,4 +1,4 @@
 /* Old style C comments
-*comments
-*/
+ *comments
+ */
 #define VAL 1

``````````

</details>


https://github.com/llvm/llvm-project/pull/150006

From flang-commits at lists.llvm.org  Tue Jul 22 05:17:19 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 05:17:19 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #132646 (PR #150007)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150007@github.com>
Message-ID: <687f814f.170a0220.2c92d3.260a@mx.google.com>


https://github.com/aadeshps-mcw converted_to_draft https://github.com/llvm/llvm-project/pull/150007

From flang-commits at lists.llvm.org  Tue Jul 22 05:17:43 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 05:17:43 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #132646 (PR #150007)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150007@github.com>
Message-ID: <687f8167.050a0220.af705.2e23@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Aadesh Premkumar (aadeshps-mcw)

<details>
<summary>Changes</summary>



---
Full diff: https://github.com/llvm/llvm-project/pull/150007.diff


3 Files Affected:

- (modified) flang/lib/Evaluate/intrinsics.cpp (+9-6) 
- (added) flang/test/Evaluate/bug132646.f90 (+24) 
- (modified) flang/test/Evaluate/folding04.f90 (+1-3) 


``````````diff
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index e802915945e26..466bbed93951f 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1336,9 +1336,9 @@ static const SpecificIntrinsicInterface specificIntrinsicFunction[]{
     {{"log", {{"x", DefaultReal}}, DefaultReal}},
     {{"log10", {{"x", DefaultReal}}, DefaultReal}},
     {{"max0",
-         {{"a1", DefaultInt}, {"a2", DefaultInt},
-             {"a3", DefaultInt, Rank::elemental, Optionality::repeats}},
-         DefaultInt},
+         {{"a1", OperandInt}, {"a2", OperandInt},
+             {"a3", OperandInt, Rank::elemental, Optionality::repeats}},
+         OperandInt},
         "max", true, true},
     {{"max1",
          {{"a1", DefaultReal}, {"a2", DefaultReal},
@@ -1346,9 +1346,9 @@ static const SpecificIntrinsicInterface specificIntrinsicFunction[]{
          DefaultInt},
         "max", true, true},
     {{"min0",
-         {{"a1", DefaultInt}, {"a2", DefaultInt},
-             {"a3", DefaultInt, Rank::elemental, Optionality::repeats}},
-         DefaultInt},
+         {{"a1", OperandInt}, {"a2", OperandInt},
+             {"a3", OperandInt, Rank::elemental, Optionality::repeats}},
+         OperandInt},
         "min", true, true},
     {{"min1",
          {{"a1", DefaultReal}, {"a2", DefaultReal},
@@ -3454,6 +3454,9 @@ static DynamicType GetReturnType(const SpecificIntrinsicInterface &interface,
   case KindCode::defaultRealKind:
     category = TypeCategory::Real;
     break;
+  case KindCode::operand:
+    category = TypeCategory::Integer;
+    break;
   default:
     CRASH_NO_CASE;
   }
diff --git a/flang/test/Evaluate/bug132646.f90 b/flang/test/Evaluate/bug132646.f90
new file mode 100644
index 0000000000000..3ffa95589e3c8
--- /dev/null
+++ b/flang/test/Evaluate/bug132646.f90
@@ -0,0 +1,24 @@
+! RUN: %flang %s -o %t && %t | FileCheck %s
+
+program main
+    print *,kind(max0(1_1,2_1))
+    print *,kind(max0(1_2,2_2))
+    print *,kind(max0(1_4,2_4))
+    print *,kind(max0(1_8,2_8))
+
+    print *,kind(min0(1_1,2_1))
+    print *,kind(min0(1_2,2_2))
+    print *,kind(min0(1_4,2_4))
+    print *,kind(min0(1_8,2_8))
+end program main
+
+! CHECK: 1
+! CHECK-NEXT: 2
+! CHECK-NEXT: 4
+! CHECK-NEXT: 8
+
+! CHECK-NEXT: 1
+! CHECK-NEXT: 2
+! CHECK-NEXT: 4
+! CHECK-NEXT: 8
+
diff --git a/flang/test/Evaluate/folding04.f90 b/flang/test/Evaluate/folding04.f90
index 027db20f608b2..c5d5b5ac6c99a 100644
--- a/flang/test/Evaluate/folding04.f90
+++ b/flang/test/Evaluate/folding04.f90
@@ -86,9 +86,7 @@ module specific_extremums
   ! specified for f18 (converting the result).
   integer(8), parameter :: max_i32_8 = 2_8**31-1
   integer, parameter :: expected_min0 = int(min(max_i32_8, 2_8*max_i32_8), 4)
-  !WARN: portability: Argument types do not match specific intrinsic 'min0' requirements; using 'min' generic instead and converting the result to INTEGER(4) if needed [-Wuse-generic-intrinsic-when-specific-doesnt-match]
-  integer, parameter :: result_min0 =  min0(max_i32_8, 2_8*max_i32_8)
-  ! result_min0 would be -2  if arguments were converted to default integer.
+  integer(8), parameter :: result_min0 =  min0(max_i32_8, 2_8*max_i32_8)
   logical, parameter :: test_min0 = expected_min0 .EQ. result_min0
 
   real, parameter :: expected_amax0 = real(max(max_i32_8, 2_8*max_i32_8), 4)

``````````

</details>


https://github.com/llvm/llvm-project/pull/150007

From flang-commits at lists.llvm.org  Tue Jul 22 05:17:54 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 05:17:54 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #127426 (PR #150006)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150006@github.com>
Message-ID: <687f8172.050a0220.bec76.22cd@mx.google.com>


https://github.com/aadeshps-mcw closed https://github.com/llvm/llvm-project/pull/150006

From flang-commits at lists.llvm.org  Tue Jul 22 05:18:21 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 05:18:21 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #132646 (PR #150007)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150007@github.com>
Message-ID: <687f818d.170a0220.598f.2759@mx.google.com>


https://github.com/aadeshps-mcw closed https://github.com/llvm/llvm-project/pull/150007

From flang-commits at lists.llvm.org  Tue Jul 22 05:21:07 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Tue, 22 Jul 2025 05:21:07 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [Flang] Implement !$omp unroll using
 omp.unroll_heuristic (PR #144785)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/144785@github.com>
Message-ID: <687f8233.170a0220.3e88d.268d@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `flang` at step 6 "test-build-unified-tree-check-flang".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/34541

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (test-build-unified-tree-check-flang) failure: test (failure)
******************** TEST 'Flang :: Lower/OpenMP/unroll-heuristic02.f90' FAILED ********************
Exit Code: 1

Command Output (stderr):
--
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90 2>&1 | /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90 # RUN: at line 1
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/flang -fc1 -emit-hlfir -fopenmp -fopenmp-version=51 -o - /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90
+ /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/bin/FileCheck /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90:23:14: error: CHECK-NEXT: expected string not found in input
!CHECK-NEXT: %18 = arith.cmpi slt, %17, %c0_i32 : i32
             ^
<stdin>:25:34: note: scanning from here
 %c1_i32 = arith.constant 1 : i32
                                 ^
<stdin>:26:2: note: possible intended match here
 %20 = arith.cmpi slt, %19, %c0_i32 : i32
 ^

Input file: <stdin>
Check file: /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/test/Lower/OpenMP/unroll-heuristic02.f90

-dump-input=help explains the following input dump.

Input was:
<<<<<<
           .
           .
           .
          20:  %16:2 = hlfir.declare %15 {uniq_name = "_QFomp_unroll_heuristic_nested02Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>) 
          21:  %17 = fir.load %9#0 : !fir.ref<i32> 
          22:  %18 = fir.load %10#0 : !fir.ref<i32> 
          23:  %19 = fir.load %8#0 : !fir.ref<i32> 
          24:  %c0_i32 = arith.constant 0 : i32 
          25:  %c1_i32 = arith.constant 1 : i32 
next:23'0                                      X error: no match found
          26:  %20 = arith.cmpi slt, %19, %c0_i32 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
next:23'1      ?                                         possible intended match
          27:  %21 = arith.subi %c0_i32, %19 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          28:  %22 = arith.select %20, %21, %19 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          29:  %23 = arith.select %20, %18, %17 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          30:  %24 = arith.select %20, %17, %18 : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
          31:  %25 = arith.subi %24, %23 overflow<nuw> : i32 
next:23'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           .
           .
...

```

</details>

https://github.com/llvm/llvm-project/pull/144785

From flang-commits at lists.llvm.org  Tue Jul 22 05:22:31 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 05:22:31 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #127426 (PR #150008)
Message-ID: <llvm.org/llvm/llvm-project/pull/150008@github.com>


https://github.com/aadeshps-mcw created https://github.com/llvm/llvm-project/pull/150008

None

>From adf5b62e464eab1eb2834138bd0ddf4276738963 Mon Sep 17 00:00:00 2001
From: Aadesh PremKumar <aadesh.premkumar at multicorewareinc.com>
Date: Tue, 22 Jul 2025 17:51:01 +0530
Subject: [PATCH] --Fix for issue #127426

---
 flang/lib/Parser/prescan.cpp |  7 ++++---
 flang/test/Parser/inc.h      |  4 ++++
 flang/test/Parser/test.F     | 14 ++++++++++++++
 3 files changed, 22 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Parser/inc.h
 create mode 100644 flang/test/Parser/test.F

diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 3a9a475c365ee..06c9456b203a9 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -175,7 +175,7 @@ void Prescanner::Statement() {
       EmitChar(tokens, '!');
       ++at_, ++column_;
       for (const char *sp{directiveSentinel_}; *sp != '\0';
-           ++sp, ++at_, ++column_) {
+          ++sp, ++at_, ++column_) {
         EmitChar(tokens, *sp);
       }
       if (inFixedForm_) {
@@ -435,6 +435,7 @@ void Prescanner::LabelField(TokenSequence &token) {
   int outCol{1};
   const char *start{at_};
   std::optional<int> badColumn;
+  SkipCComments();
   for (; *at_ != '\n' && column_ <= 6; ++at_) {
     if (*at_ == '\t') {
       ++at_;
@@ -1218,7 +1219,7 @@ std::optional<std::size_t> Prescanner::IsIncludeLine(const char *start) const {
   }
   if (IsDecimalDigit(*p)) { // accept & ignore a numeric kind prefix
     for (p = SkipWhiteSpace(p + 1); IsDecimalDigit(*p);
-         p = SkipWhiteSpace(p + 1)) {
+        p = SkipWhiteSpace(p + 1)) {
     }
     if (*p != '_') {
       return std::nullopt;
@@ -1266,7 +1267,7 @@ void Prescanner::FortranInclude(const char *firstQuote) {
   llvm::raw_string_ostream error{buf};
   Provenance provenance{GetProvenance(nextLine_)};
   std::optional<std::string> prependPath;
-  if (const SourceFile * currentFile{allSources_.GetSourceFile(provenance)}) {
+  if (const SourceFile *currentFile{allSources_.GetSourceFile(provenance)}) {
     prependPath = DirectoryName(currentFile->path());
   }
   const SourceFile *included{
diff --git a/flang/test/Parser/inc.h b/flang/test/Parser/inc.h
new file mode 100644
index 0000000000000..1e31cb035ccf1
--- /dev/null
+++ b/flang/test/Parser/inc.h
@@ -0,0 +1,4 @@
+/* Old-style C comments
+ * Comments
+ */
+#define VAL 1
diff --git a/flang/test/Parser/test.F b/flang/test/Parser/test.F
new file mode 100644
index 0000000000000..c482bc0023b66
--- /dev/null
+++ b/flang/test/Parser/test.F
@@ -0,0 +1,14 @@
+! RUN: %flang_fc1 -fdebug-unparse-no-sema %s 2>&1 | FileCheck %s
+
+! CHECK: SUBROUTINE foo
+! CHECK: INTEGER i
+! CHECK: i = 1
+! CHECK: PRINT *, i
+! CHECK: END SUBROUTINE foo
+
+subroutine foo()
+#include "inc.h"
+  integer :: i
+  i = VAL
+  print *, i
+end subroutine foo


From flang-commits at lists.llvm.org  Tue Jul 22 05:22:34 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 05:22:34 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #127426 (PR #150008)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150008@github.com>
Message-ID: <687f828a.170a0220.16ddb8.f2c3@mx.google.com>


https://github.com/aadeshps-mcw converted_to_draft https://github.com/llvm/llvm-project/pull/150008

From flang-commits at lists.llvm.org  Tue Jul 22 05:23:09 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 05:23:09 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #127426 (PR #150008)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150008@github.com>
Message-ID: <687f82ad.170a0220.2b4b36.2a49@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-parser

Author: Aadesh Premkumar (aadeshps-mcw)

<details>
<summary>Changes</summary>



---
Full diff: https://github.com/llvm/llvm-project/pull/150008.diff


3 Files Affected:

- (modified) flang/lib/Parser/prescan.cpp (+4-3) 
- (added) flang/test/Parser/inc.h (+4) 
- (added) flang/test/Parser/test.F (+14) 


``````````diff
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 3a9a475c365ee..06c9456b203a9 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -175,7 +175,7 @@ void Prescanner::Statement() {
       EmitChar(tokens, '!');
       ++at_, ++column_;
       for (const char *sp{directiveSentinel_}; *sp != '\0';
-           ++sp, ++at_, ++column_) {
+          ++sp, ++at_, ++column_) {
         EmitChar(tokens, *sp);
       }
       if (inFixedForm_) {
@@ -435,6 +435,7 @@ void Prescanner::LabelField(TokenSequence &token) {
   int outCol{1};
   const char *start{at_};
   std::optional<int> badColumn;
+  SkipCComments();
   for (; *at_ != '\n' && column_ <= 6; ++at_) {
     if (*at_ == '\t') {
       ++at_;
@@ -1218,7 +1219,7 @@ std::optional<std::size_t> Prescanner::IsIncludeLine(const char *start) const {
   }
   if (IsDecimalDigit(*p)) { // accept & ignore a numeric kind prefix
     for (p = SkipWhiteSpace(p + 1); IsDecimalDigit(*p);
-         p = SkipWhiteSpace(p + 1)) {
+        p = SkipWhiteSpace(p + 1)) {
     }
     if (*p != '_') {
       return std::nullopt;
@@ -1266,7 +1267,7 @@ void Prescanner::FortranInclude(const char *firstQuote) {
   llvm::raw_string_ostream error{buf};
   Provenance provenance{GetProvenance(nextLine_)};
   std::optional<std::string> prependPath;
-  if (const SourceFile * currentFile{allSources_.GetSourceFile(provenance)}) {
+  if (const SourceFile *currentFile{allSources_.GetSourceFile(provenance)}) {
     prependPath = DirectoryName(currentFile->path());
   }
   const SourceFile *included{
diff --git a/flang/test/Parser/inc.h b/flang/test/Parser/inc.h
new file mode 100644
index 0000000000000..1e31cb035ccf1
--- /dev/null
+++ b/flang/test/Parser/inc.h
@@ -0,0 +1,4 @@
+/* Old-style C comments
+ * Comments
+ */
+#define VAL 1
diff --git a/flang/test/Parser/test.F b/flang/test/Parser/test.F
new file mode 100644
index 0000000000000..c482bc0023b66
--- /dev/null
+++ b/flang/test/Parser/test.F
@@ -0,0 +1,14 @@
+! RUN: %flang_fc1 -fdebug-unparse-no-sema %s 2>&1 | FileCheck %s
+
+! CHECK: SUBROUTINE foo
+! CHECK: INTEGER i
+! CHECK: i = 1
+! CHECK: PRINT *, i
+! CHECK: END SUBROUTINE foo
+
+subroutine foo()
+#include "inc.h"
+  integer :: i
+  i = VAL
+  print *, i
+end subroutine foo

``````````

</details>


https://github.com/llvm/llvm-project/pull/150008

From flang-commits at lists.llvm.org  Tue Jul 22 05:37:50 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 05:37:50 -0700 (PDT)
Subject: [flang-commits] [flang] 2914a48 - [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (#149137)
Message-ID: <687f861e.050a0220.127c85.2d4d@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-22T07:37:47-05:00
New Revision: 2914a488c7f49c4817bbfb86f74da04fd338b4eb

URL: https://github.com/llvm/llvm-project/commit/2914a488c7f49c4817bbfb86f74da04fd338b4eb
DIFF: https://github.com/llvm/llvm-project/commit/2914a488c7f49c4817bbfb86f74da04fd338b4eb.diff

LOG: [flang][OpenMP] Sema checks, lowering with new format of MAP modifiers (#149137)

OpenMP 6.0 has changed the modifiers on the MAP clause. Previous patch
has introduced parsing support for them. This patch introduces
processing of the new forms in semantic checks and in lowering. This
only applies to existing modifiers, which were updated in the 6.0 spec.
Any of the newly introduced modifiers (SELF and REF) are ignored.

Added: 
    flang/test/Semantics/OpenMP/map-modifiers-v60.f90

Modified: 
    flang/include/flang/Semantics/symbol.h
    flang/lib/Lower/OpenMP/ClauseProcessor.cpp
    flang/lib/Lower/OpenMP/Clauses.cpp
    flang/lib/Semantics/canonicalize-omp.cpp
    flang/lib/Semantics/canonicalize-omp.h
    flang/lib/Semantics/check-omp-structure.cpp
    flang/lib/Semantics/check-omp-structure.h
    flang/lib/Semantics/openmp-utils.cpp
    flang/lib/Semantics/openmp-utils.h
    flang/lib/Semantics/resolve-directives.cpp
    flang/lib/Semantics/semantics.cpp
    flang/lib/Semantics/symbol.cpp
    flang/test/Lower/OpenMP/map-modifiers.f90
    flang/test/Semantics/OpenMP/combined-constructs.f90
    flang/test/Semantics/OpenMP/device-constructs.f90
    llvm/include/llvm/Frontend/OpenMP/ClauseT.h
    llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
    llvm/unittests/Frontend/OpenMPDecompositionTest.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index b319e2c7e5e74..5bde9f39ca0b0 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -812,9 +812,8 @@ class Symbol {
       // OpenMP data-sharing attribute
       OmpShared, OmpPrivate, OmpLinear, OmpFirstPrivate, OmpLastPrivate,
       // OpenMP data-mapping attribute
-      OmpMapTo, OmpMapFrom, OmpMapToFrom, OmpMapAlloc, OmpMapRelease,
-      OmpMapDelete, OmpUseDevicePtr, OmpUseDeviceAddr, OmpIsDevicePtr,
-      OmpHasDeviceAddr,
+      OmpMapTo, OmpMapFrom, OmpMapToFrom, OmpMapStorage, OmpMapDelete,
+      OmpUseDevicePtr, OmpUseDeviceAddr, OmpIsDevicePtr, OmpHasDeviceAddr,
       // OpenMP data-copying attribute
       OmpCopyIn, OmpCopyPrivate,
       // OpenMP miscellaneous flags

diff  --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 498eea471f3db..8b3ad57c53810 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1315,7 +1315,8 @@ bool ClauseProcessor::processMap(
                      const parser::CharBlock &source) {
     using Map = omp::clause::Map;
     mlir::Location clauseLocation = converter.genLocation(source);
-    const auto &[mapType, typeMods, mappers, iterator, objects] = clause.t;
+    const auto &[mapType, typeMods, refMod, mappers, iterator, objects] =
+        clause.t;
     llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
         llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
     std::string mapperIdName = "__implicit_mapper";
@@ -1342,16 +1343,13 @@ bool ClauseProcessor::processMap(
       mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
                      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
       break;
-    case Map::MapType::Alloc:
-    case Map::MapType::Release:
+    case Map::MapType::Storage:
       // alloc and release is the default map_type for the Target Data
       // Ops, i.e. if no bits for map_type is supplied then alloc/release
-      // is implicitly assumed based on the target directive. Default
-      // value for Target Data and Enter Data is alloc and for Exit Data
-      // it is release.
+      // (aka storage in 6.0+) is implicitly assumed based on the target
+      // directive. Default value for Target Data and Enter Data is alloc
+      // and for Exit Data it is release.
       break;
-    case Map::MapType::Delete:
-      mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
     }
 
     if (typeMods) {
@@ -1362,6 +1360,8 @@ bool ClauseProcessor::processMap(
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
       if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Close))
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
+      if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Delete))
+        mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
       if (llvm::is_contained(*typeMods, Map::MapTypeModifier::OmpxHold))
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
     }

diff  --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 22a07219d3a50..686fba0154f44 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1001,19 +1001,21 @@ Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpMapType::Value, Map::MapType,
+      convertMapType, parser::OmpMapType::Value, Map::MapType,
       // clang-format off
-      MS(Alloc,    Alloc)
-      MS(Delete,   Delete)
-      MS(From,     From)
-      MS(Release,  Release)
-      MS(To,       To)
-      MS(Tofrom,   Tofrom)
+      MS(Alloc,   Storage)
+      MS(Delete,  Storage)
+      MS(Release, Storage)
+      MS(Storage, Storage)
+      MS(From,    From)
+      MS(To,      To)
+      MS(Tofrom,  Tofrom)
       // clang-format on
   );
 
   CLAUSET_ENUM_CONVERT( //
-      convert2, parser::OmpMapTypeModifier::Value, Map::MapTypeModifier,
+      convertMapTypeMod, parser::OmpMapTypeModifier::Value,
+      Map::MapTypeModifier,
       // clang-format off
       MS(Always,    Always)
       MS(Close,     Close)
@@ -1022,43 +1024,76 @@ Map make(const parser::OmpClause::Map &inp,
       // clang-format on
   );
 
+  CLAUSET_ENUM_CONVERT( //
+      convertRefMod, parser::OmpRefModifier::Value, Map::RefModifier,
+      // clang-format off
+      MS(Ref_Ptee,     RefPtee)
+      MS(Ref_Ptr,      RefPtr)
+      MS(Ref_Ptr_Ptee, RefPtrPtee)
+      // clang-format on
+  );
+
+  // Treat always, close, present, self, delete modifiers as map-type-
+  // modifiers.
   auto &mods = semantics::OmpGetModifiers(inp.v);
-  auto *t1 = semantics::OmpGetUniqueModifier<parser::OmpMapper>(mods);
-  auto *t2 = semantics::OmpGetUniqueModifier<parser::OmpIterator>(mods);
-  auto *t3 = semantics::OmpGetUniqueModifier<parser::OmpMapType>(mods);
-  auto &t4 = std::get<parser::OmpObjectList>(inp.v.t);
 
-  auto mappers = [&]() -> std::optional<List<Mapper>> {
+  auto *t1 = semantics::OmpGetUniqueModifier<parser::OmpMapType>(mods);
+  auto &t2 = std::get<parser::OmpObjectList>(inp.v.t);
+
+  auto type = [&]() -> std::optional<Map::MapType> {
     if (t1)
-      return List<Mapper>{Mapper{makeObject(t1->v, semaCtx)}};
+      return convertMapType(t1->v);
     return std::nullopt;
   }();
 
-  auto iterator = [&]() -> std::optional<Iterator> {
-    if (t2)
-      return makeIterator(*t2, semaCtx);
+  llvm::DenseSet<Map::MapTypeModifier> modSet;
+  if (t1 && t1->v == parser::OmpMapType::Value::Delete)
+    modSet.insert(Map::MapTypeModifier::Delete);
+
+  for (auto *typeMod :
+       semantics::OmpGetRepeatableModifier<parser::OmpMapTypeModifier>(mods)) {
+    modSet.insert(convertMapTypeMod(typeMod->v));
+  }
+  if (semantics::OmpGetUniqueModifier<parser::OmpAlwaysModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Always);
+  if (semantics::OmpGetUniqueModifier<parser::OmpCloseModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Close);
+  if (semantics::OmpGetUniqueModifier<parser::OmpDeleteModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Delete);
+  if (semantics::OmpGetUniqueModifier<parser::OmpPresentModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Present);
+  if (semantics::OmpGetUniqueModifier<parser::OmpSelfModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::Self);
+  if (semantics::OmpGetUniqueModifier<parser::OmpxHoldModifier>(mods))
+    modSet.insert(Map::MapTypeModifier::OmpxHold);
+
+  std::optional<Map::MapTypeModifiers> maybeTypeMods{};
+  if (!modSet.empty())
+    maybeTypeMods = Map::MapTypeModifiers(modSet.begin(), modSet.end());
+
+  auto refMod = [&]() -> std::optional<Map::RefModifier> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpRefModifier>(mods))
+      return convertRefMod(t->v);
     return std::nullopt;
   }();
 
-  auto type = [&]() -> std::optional<Map::MapType> {
-    if (t3)
-      return convert1(t3->v);
+  auto mappers = [&]() -> std::optional<List<Mapper>> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpMapper>(mods))
+      return List<Mapper>{Mapper{makeObject(t->v, semaCtx)}};
     return std::nullopt;
   }();
 
-  Map::MapTypeModifiers typeMods;
-  for (auto *typeMod :
-       semantics::OmpGetRepeatableModifier<parser::OmpMapTypeModifier>(mods)) {
-    typeMods.push_back(convert2(typeMod->v));
-  }
-  std::optional<Map::MapTypeModifiers> maybeTypeMods{};
-  if (!typeMods.empty())
-    maybeTypeMods = std::move(typeMods);
+  auto iterator = [&]() -> std::optional<Iterator> {
+    if (auto *t = semantics::OmpGetUniqueModifier<parser::OmpIterator>(mods))
+      return makeIterator(*t, semaCtx);
+    return std::nullopt;
+  }();
 
   return Map{{/*MapType=*/std::move(type),
               /*MapTypeModifiers=*/std::move(maybeTypeMods),
-              /*Mapper=*/std::move(mappers), /*Iterator=*/std::move(iterator),
-              /*LocatorList=*/makeObjects(t4, semaCtx)}};
+              /*RefModifier=*/std::move(refMod), /*Mapper=*/std::move(mappers),
+              /*Iterator=*/std::move(iterator),
+              /*LocatorList=*/makeObjects(t2, semaCtx)}};
 }
 
 Match make(const parser::OmpClause::Match &inp,

diff  --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index bba4d72a702e9..77e2fd6ca5097 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
 
   void Post(parser::Block &block) {
     for (auto it{block.begin()}; it != block.end(); ++it) {
@@ -401,6 +402,11 @@ class CanonicalizationOfOmp {
   // if the specified OpenMP version is less than 6.0, rewrite the affected
   // modifiers back into the pre-6.0 forms.
   void CanonicalizeMapModifiers(parser::OmpMapClause &map) {
+    unsigned version{context_.langOptions().OpenMPVersion};
+    if (version >= 60) {
+      return;
+    }
+
     // Omp{Always, Close, Present, xHold}Modifier -> OmpMapTypeModifier
     // OmpDeleteModifier -> OmpMapType
     using Modifier = parser::OmpMapClause::Modifier;
@@ -432,12 +438,13 @@ class CanonicalizationOfOmp {
   // same construct. This is for converting utility constructs to executable
   // constructs.
   std::map<parser::SpecificationPart *, parser::Block *> blockForSpec_;
+  SemanticsContext &context_;
   parser::Messages &messages_;
 };
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program) {
-  CanonicalizationOfOmp omp{messages};
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program) {
+  CanonicalizationOfOmp omp{context};
   Walk(program, omp);
-  return !messages.AnyFatalError();
+  return !context.messages().AnyFatalError();
 }
 } // namespace Fortran::semantics

diff  --git a/flang/lib/Semantics/canonicalize-omp.h b/flang/lib/Semantics/canonicalize-omp.h
index 23350c522a775..3251218da35ed 100644
--- a/flang/lib/Semantics/canonicalize-omp.h
+++ b/flang/lib/Semantics/canonicalize-omp.h
@@ -11,13 +11,12 @@
 
 namespace Fortran::parser {
 struct Program;
-class Messages;
-} // namespace Fortran::parser
+}
 
 namespace Fortran::semantics {
 class SemanticsContext;
 
-bool CanonicalizeOmp(parser::Messages &messages, parser::Program &program);
+bool CanonicalizeOmp(SemanticsContext &context, parser::Program &program);
 } // namespace Fortran::semantics
 
 #endif // FORTRAN_SEMANTICS_CANONICALIZE_OMP_H_

diff  --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index e4a94efcc6b55..8264e1d5e8fd9 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -37,6 +37,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
@@ -3398,23 +3399,22 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Detach &x) {
   }
 }
 
-void OmpStructureChecker::CheckAllowedMapTypes(
-    const parser::OmpMapType::Value &type,
-    const std::list<parser::OmpMapType::Value> &allowedMapTypeList) {
-  if (!llvm::is_contained(allowedMapTypeList, type)) {
-    std::string commaSeparatedMapTypes;
-    llvm::interleave(
-        allowedMapTypeList.begin(), allowedMapTypeList.end(),
-        [&](const parser::OmpMapType::Value &mapType) {
-          commaSeparatedMapTypes.append(parser::ToUpperCaseLetters(
-              parser::OmpMapType::EnumToString(mapType)));
-        },
-        [&] { commaSeparatedMapTypes.append(", "); });
-    context_.Say(GetContext().clauseSource,
-        "Only the %s map types are permitted "
-        "for MAP clauses on the %s directive"_err_en_US,
-        commaSeparatedMapTypes, ContextDirectiveAsFortran());
+void OmpStructureChecker::CheckAllowedMapTypes(parser::OmpMapType::Value type,
+    llvm::ArrayRef<parser::OmpMapType::Value> allowed) {
+  if (llvm::is_contained(allowed, type)) {
+    return;
   }
+
+  llvm::SmallVector<std::string> names;
+  llvm::transform(
+      allowed, std::back_inserter(names), [](parser::OmpMapType::Value val) {
+        return parser::ToUpperCaseLetters(
+            parser::OmpMapType::EnumToString(val));
+      });
+  llvm::sort(names);
+  context_.Say(GetContext().clauseSource,
+      "Only the %s map types are permitted for MAP clauses on the %s directive"_err_en_US,
+      llvm::join(names, ", "), ContextDirectiveAsFortran());
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
@@ -3435,27 +3435,62 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
     CheckIteratorModifier(*iter);
   }
   if (auto *type{OmpGetUniqueModifier<parser::OmpMapType>(modifiers)}) {
+    using Directive = llvm::omp::Directive;
     using Value = parser::OmpMapType::Value;
-    switch (GetContext().directive) {
-    case llvm::omp::Directive::OMPD_target:
-    case llvm::omp::Directive::OMPD_target_teams:
-    case llvm::omp::Directive::OMPD_target_teams_distribute:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
-    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
-    case llvm::omp::Directive::OMPD_target_data:
-      CheckAllowedMapTypes(
-          type->v, {Value::To, Value::From, Value::Tofrom, Value::Alloc});
-      break;
-    case llvm::omp::Directive::OMPD_target_enter_data:
-      CheckAllowedMapTypes(type->v, {Value::To, Value::Alloc});
-      break;
-    case llvm::omp::Directive::OMPD_target_exit_data:
-      CheckAllowedMapTypes(
-          type->v, {Value::From, Value::Release, Value::Delete});
-      break;
-    default:
-      break;
+
+    static auto isValidForVersion{
+        [](parser::OmpMapType::Value t, unsigned version) {
+          switch (t) {
+          case parser::OmpMapType::Value::Alloc:
+          case parser::OmpMapType::Value::Delete:
+          case parser::OmpMapType::Value::Release:
+            return version < 60;
+          case parser::OmpMapType::Value::Storage:
+            return version >= 60;
+          default:
+            return true;
+          }
+        }};
+
+    llvm::SmallVector<parser::OmpMapType::Value> mapEnteringTypes{[&]() {
+      llvm::SmallVector<parser::OmpMapType::Value> result;
+      for (size_t i{0}; i != parser::OmpMapType::Value_enumSize; ++i) {
+        auto t{static_cast<parser::OmpMapType::Value>(i)};
+        if (isValidForVersion(t, version) && IsMapEnteringType(t)) {
+          result.push_back(t);
+        }
+      }
+      return result;
+    }()};
+    llvm::SmallVector<parser::OmpMapType::Value> mapExitingTypes{[&]() {
+      llvm::SmallVector<parser::OmpMapType::Value> result;
+      for (size_t i{0}; i != parser::OmpMapType::Value_enumSize; ++i) {
+        auto t{static_cast<parser::OmpMapType::Value>(i)};
+        if (isValidForVersion(t, version) && IsMapExitingType(t)) {
+          result.push_back(t);
+        }
+      }
+      return result;
+    }()};
+
+    llvm::omp::Directive dir{GetContext().directive};
+    llvm::ArrayRef<llvm::omp::Directive> leafs{
+        llvm::omp::getLeafConstructsOrSelf(dir)};
+
+    if (llvm::is_contained(leafs, Directive::OMPD_target) ||
+        llvm::is_contained(leafs, Directive::OMPD_target_data)) {
+      if (version >= 60) {
+        // Map types listed in the decay table. [6.0:276]
+        CheckAllowedMapTypes(
+            type->v, {Value::Storage, Value::From, Value::To, Value::Tofrom});
+      } else {
+        CheckAllowedMapTypes(
+            type->v, {Value::Alloc, Value::From, Value::To, Value::Tofrom});
+      }
+    } else if (llvm::is_contained(leafs, Directive::OMPD_target_enter_data)) {
+      CheckAllowedMapTypes(type->v, mapEnteringTypes);
+    } else if (llvm::is_contained(leafs, Directive::OMPD_target_exit_data)) {
+      CheckAllowedMapTypes(type->v, mapExitingTypes);
     }
   }
 

diff  --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 6a877a5d0a7c0..f4a291dc255c8 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -179,8 +179,8 @@ class OmpStructureChecker
   void HasInvalidDistributeNesting(const parser::OpenMPLoopConstruct &x);
   void HasInvalidLoopBinding(const parser::OpenMPLoopConstruct &x);
   // specific clause related
-  void CheckAllowedMapTypes(const parser::OmpMapType::Value &,
-      const std::list<parser::OmpMapType::Value> &);
+  void CheckAllowedMapTypes(
+      parser::OmpMapType::Value, llvm::ArrayRef<parser::OmpMapType::Value>);
 
   const std::list<parser::OmpTraitProperty> &GetTraitPropertyList(
       const parser::OmpTraitSelector &);

diff  --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index f43d2cc75620e..da14507aa9fe6 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -143,6 +143,31 @@ bool IsVarOrFunctionRef(const MaybeExpr &expr) {
   }
 }
 
+bool IsMapEnteringType(parser::OmpMapType::Value type) {
+  switch (type) {
+  case parser::OmpMapType::Value::Alloc:
+  case parser::OmpMapType::Value::Storage:
+  case parser::OmpMapType::Value::To:
+  case parser::OmpMapType::Value::Tofrom:
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool IsMapExitingType(parser::OmpMapType::Value type) {
+  switch (type) {
+  case parser::OmpMapType::Value::Delete:
+  case parser::OmpMapType::Value::From:
+  case parser::OmpMapType::Value::Release:
+  case parser::OmpMapType::Value::Storage:
+  case parser::OmpMapType::Value::Tofrom:
+    return true;
+  default:
+    return false;
+  }
+}
+
 std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
   const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
   // ForwardOwningPointer           typedExpr

diff  --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
index a96c008fb26e7..001fbeb45ceec 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -59,6 +59,9 @@ bool IsExtendedListItem(const Symbol &sym);
 bool IsVariableListItem(const Symbol &sym);
 bool IsVarOrFunctionRef(const MaybeExpr &expr);
 
+bool IsMapEnteringType(parser::OmpMapType::Value type);
+bool IsMapExitingType(parser::OmpMapType::Value type);
+
 std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr);
 std::optional<evaluate::DynamicType> GetDynamicType(
     const parser::Expr &parserExpr);

diff  --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 332291a2aefe8..addd7d322a4fd 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -727,7 +727,9 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   void Post(const parser::EorLabel &eorLabel) { CheckSourceLabel(eorLabel.v); }
 
   void Post(const parser::OmpMapClause &x) {
-    Symbol::Flag ompFlag = Symbol::Flag::OmpMapToFrom;
+    unsigned version{context_.langOptions().OpenMPVersion};
+    std::optional<Symbol::Flag> ompFlag;
+
     auto &mods{OmpGetModifiers(x)};
     if (auto *mapType{OmpGetUniqueModifier<parser::OmpMapType>(mods)}) {
       switch (mapType->v) {
@@ -741,10 +743,9 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
         ompFlag = Symbol::Flag::OmpMapToFrom;
         break;
       case parser::OmpMapType::Value::Alloc:
-        ompFlag = Symbol::Flag::OmpMapAlloc;
-        break;
       case parser::OmpMapType::Value::Release:
-        ompFlag = Symbol::Flag::OmpMapRelease;
+      case parser::OmpMapType::Value::Storage:
+        ompFlag = Symbol::Flag::OmpMapStorage;
         break;
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
@@ -753,6 +754,24 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
         break;
       }
     }
+    if (!ompFlag) {
+      if (version >= 60) {
+        // [6.0:275:12-15]
+        // When a map-type is not specified for a clause on which it may be
+        // specified, the map-type defaults to storage if the delete-modifier
+        // is present on the clause or if the list item for which the map-type
+        // is not specified is an assumed-size array.
+        if (OmpGetUniqueModifier<parser::OmpDeleteModifier>(mods)) {
+          ompFlag = Symbol::Flag::OmpMapStorage;
+        }
+        // Otherwise, if delete-modifier is absent, leave ompFlag unset.
+      } else {
+        // [5.2:151:10]
+        // If a map-type is not specified, the map-type defaults to tofrom.
+        ompFlag = Symbol::Flag::OmpMapToFrom;
+      }
+    }
+
     const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
     for (const auto &ompObj : ompObjList.v) {
       common::visit(
@@ -761,15 +780,15 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
                 if (const auto *name{
                         semantics::getDesignatorNameIfDataRef(designator)}) {
                   if (name->symbol) {
-                    name->symbol->set(ompFlag);
-                    AddToContextObjectWithDSA(*name->symbol, ompFlag);
-                  }
-                  if (name->symbol &&
-                      semantics::IsAssumedSizeArray(*name->symbol)) {
-                    context_.Say(designator.source,
-                        "Assumed-size whole arrays may not appear on the %s "
-                        "clause"_err_en_US,
-                        "MAP");
+                    name->symbol->set(
+                        ompFlag.value_or(Symbol::Flag::OmpMapStorage));
+                    AddToContextObjectWithDSA(*name->symbol, *ompFlag);
+                    if (semantics::IsAssumedSizeArray(*name->symbol)) {
+                      context_.Say(designator.source,
+                          "Assumed-size whole arrays may not appear on the %s "
+                          "clause"_err_en_US,
+                          "MAP");
+                    }
                   }
                 }
               },
@@ -777,7 +796,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
           },
           ompObj.u);
 
-      ResolveOmpObject(ompObj, ompFlag);
+      ResolveOmpObject(ompObj, ompFlag.value_or(Symbol::Flag::OmpMapStorage));
     }
   }
 
@@ -2776,9 +2795,8 @@ void OmpAttributeVisitor::ResolveOmpObject(
                   }
                   Symbol::Flag dataMappingAttributeFlags[] = {
                       Symbol::Flag::OmpMapTo, Symbol::Flag::OmpMapFrom,
-                      Symbol::Flag::OmpMapToFrom, Symbol::Flag::OmpMapAlloc,
-                      Symbol::Flag::OmpMapRelease, Symbol::Flag::OmpMapDelete,
-                      Symbol::Flag::OmpIsDevicePtr,
+                      Symbol::Flag::OmpMapToFrom, Symbol::Flag::OmpMapStorage,
+                      Symbol::Flag::OmpMapDelete, Symbol::Flag::OmpIsDevicePtr,
                       Symbol::Flag::OmpHasDeviceAddr};
 
                   Symbol::Flag dataSharingAttributeFlags[] = {

diff  --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index b15ed057b52f2..6db11aaf56c2a 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -642,8 +642,7 @@ bool Semantics::Perform() {
   return ValidateLabels(context_, program_) &&
       parser::CanonicalizeDo(program_) && // force line break
       CanonicalizeAcc(context_.messages(), program_) &&
-      CanonicalizeOmp(context_.messages(), program_) &&
-      CanonicalizeCUDA(program_) &&
+      CanonicalizeOmp(context_, program_) && CanonicalizeCUDA(program_) &&
       PerformStatementSemantics(context_, program_) &&
       CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}

diff  --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index 0380207927ad3..2259cfcf23ece 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -861,8 +861,7 @@ std::string Symbol::OmpFlagToClauseName(Symbol::Flag ompFlag) {
   case Symbol::Flag::OmpMapTo:
   case Symbol::Flag::OmpMapFrom:
   case Symbol::Flag::OmpMapToFrom:
-  case Symbol::Flag::OmpMapAlloc:
-  case Symbol::Flag::OmpMapRelease:
+  case Symbol::Flag::OmpMapStorage:
   case Symbol::Flag::OmpMapDelete:
     clauseName = "MAP";
     break;

diff  --git a/flang/test/Lower/OpenMP/map-modifiers.f90 b/flang/test/Lower/OpenMP/map-modifiers.f90
index 64d7869cbb836..be93c14627f9a 100644
--- a/flang/test/Lower/OpenMP/map-modifiers.f90
+++ b/flang/test/Lower/OpenMP/map-modifiers.f90
@@ -1,4 +1,6 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=60 %s -o - | FileCheck %s
 
 subroutine map_present_target_data
     integer :: x
@@ -15,6 +17,14 @@ subroutine map_present_update
 !$omp target update to(present: x)
 end subroutine
 
+subroutine map_always
+    integer :: x
+!CHECK: %[[MAP:.*]] = omp.map.info {{.*}} map_clauses(always, tofrom) {{.*}} {name = "x"}
+!CHECK: omp.target_data map_entries(%[[MAP]] : {{.*}}) {
+!$omp target data map(always, tofrom: x)
+!$omp end target data
+end subroutine
+
 subroutine map_close
     integer :: x
 !CHECK: %[[MAP:.*]] = omp.map.info {{.*}} map_clauses(close, tofrom) {{.*}} {name = "x"}

diff  --git a/flang/test/Semantics/OpenMP/combined-constructs.f90 b/flang/test/Semantics/OpenMP/combined-constructs.f90
index 4f2a4a4f501b9..2298d33ef33eb 100644
--- a/flang/test/Semantics/OpenMP/combined-constructs.f90
+++ b/flang/test/Semantics/OpenMP/combined-constructs.f90
@@ -207,7 +207,7 @@ program main
   enddo
   !$omp end target teams
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS directive
   !$omp target teams map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -307,7 +307,7 @@ program main
   enddo
   !$omp end target teams distribute
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -400,7 +400,7 @@ program main
   enddo
   !$omp end target teams distribute parallel do
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do map(delete:a)
   do i = 1, N
      a(i) = 3.14
@@ -500,7 +500,7 @@ program main
   enddo
   !$omp end target teams distribute parallel do simd
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd map(delete:a)
   do i = 1, N
      a(i) = 3.14

diff  --git a/flang/test/Semantics/OpenMP/device-constructs.f90 b/flang/test/Semantics/OpenMP/device-constructs.f90
index 6f545b9021966..431e0f88e3237 100644
--- a/flang/test/Semantics/OpenMP/device-constructs.f90
+++ b/flang/test/Semantics/OpenMP/device-constructs.f90
@@ -123,7 +123,7 @@ program main
   enddo
   !$omp end target
 
-  !ERROR: Only the TO, FROM, TOFROM, ALLOC map types are permitted for MAP clauses on the TARGET directive
+  !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET directive
   !$omp target map(delete:a)
   do i = 1, N
      a = 3.14
@@ -160,7 +160,7 @@ program main
   !ERROR: At most one IF clause can appear on the TARGET ENTER DATA directive
   !$omp target enter data map(to:a) if(.true.) if(.false.)
 
-  !ERROR: Only the TO, ALLOC map types are permitted for MAP clauses on the TARGET ENTER DATA directive
+  !ERROR: Only the ALLOC, TO, TOFROM map types are permitted for MAP clauses on the TARGET ENTER DATA directive
   !$omp target enter data map(from:a)
 
   !$omp target exit data map(delete:a)
@@ -168,7 +168,7 @@ program main
   !ERROR: At most one DEVICE clause can appear on the TARGET EXIT DATA directive
   !$omp target exit data map(from:a) device(0) device(1)
 
-  !ERROR: Only the FROM, RELEASE, DELETE map types are permitted for MAP clauses on the TARGET EXIT DATA directive
+  !ERROR: Only the DELETE, FROM, RELEASE, TOFROM map types are permitted for MAP clauses on the TARGET EXIT DATA directive
   !$omp target exit data map(to:a)
 
   !$omp target update if(.true.) device(1) to(a) from(b) depend(inout:c) nowait

diff  --git a/flang/test/Semantics/OpenMP/map-modifiers-v60.f90 b/flang/test/Semantics/OpenMP/map-modifiers-v60.f90
new file mode 100644
index 0000000000000..b3f2e5171d6d6
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/map-modifiers-v60.f90
@@ -0,0 +1,35 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52 -Werror
+
+subroutine f00(x)
+  integer :: x
+!WARNING: 'self-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(self: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f01(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptr: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f02(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptee: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f03(x)
+  integer, pointer :: x
+!WARNING: 'ref-modifier' modifier is not supported in OpenMP v5.2, try -fopenmp-version=60
+  !$omp target map(ref_ptr_ptee: x)
+  x = x + 1
+  !$omp end target
+end
+
+

diff  --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index de888ff86fe91..7919f7a8b0c34 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -779,16 +779,17 @@ struct LinkT {
 template <typename T, typename I, typename E> //
 struct MapT {
   using LocatorList = ObjectListT<I, E>;
-  ENUM(MapType, To, From, Tofrom, Alloc, Release, Delete);
-  ENUM(MapTypeModifier, Always, Close, Present, OmpxHold);
+  ENUM(MapType, To, From, Tofrom, Storage);
+  ENUM(MapTypeModifier, Always, Close, Delete, Present, Self, OmpxHold);
+  ENUM(RefModifier, RefPtee, RefPtr, RefPtrPtee);
   // See note at the definition of the MapperT type.
   using Mappers = ListT<type::MapperT<I, E>>; // Not a spec name
   using Iterator = type::IteratorT<T, I, E>;
   using MapTypeModifiers = ListT<MapTypeModifier>; // Not a spec name
 
   using TupleTrait = std::true_type;
-  std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(Mappers), OPT(Iterator),
-             LocatorList>
+  std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(RefModifier),
+             OPT(Mappers), OPT(Iterator), LocatorList>
       t;
 };
 

diff  --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
index 611bfe3f8aced..047baa3a79f5d 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -708,6 +708,7 @@ bool ConstructDecompositionT<C, H>::applyClause(
                      tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
                          {/*MapType=*/MapType::Tofrom,
                           /*MapTypeModifier=*/std::nullopt,
+                          /*RefModifier=*/std::nullopt,
                           /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
                           /*LocatorList=*/std::move(tofrom)}});
       dirTarget->clauses.push_back(map);
@@ -969,8 +970,8 @@ bool ConstructDecompositionT<C, H>::applyClause(
           llvm::omp::Clause::OMPC_map,
           tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
               {/*MapType=*/MapType::Tofrom, /*MapTypeModifier=*/std::nullopt,
-               /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
-               /*LocatorList=*/std::move(tofrom)}});
+               /*RefModifier=*/std::nullopt, /*Mapper=*/std::nullopt,
+               /*Iterator=*/std::nullopt, /*LocatorList=*/std::move(tofrom)}});
 
       dirTarget->clauses.push_back(map);
       applied = true;

diff  --git a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
index 6189d0954891b..95c26b10c9a0c 100644
--- a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
@@ -431,8 +431,8 @@ TEST_F(OpenMPDecompositionTest, Firstprivate3) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (12), (27)
-  ASSERT_EQ(Dir1, "teams shared(x)");          // (6), (17)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (12), (27)
+  ASSERT_EQ(Dir1, "teams shared(x)");            // (6), (17)
   ASSERT_EQ(Dir2, "distribute firstprivate(x) lastprivate(, (x))"); // (5), (21)
 }
 
@@ -574,9 +574,9 @@ TEST_F(OpenMPDecompositionTest, Lastprivate3) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (21), (27)
-  ASSERT_EQ(Dir1, "parallel shared(x)");       // (22)
-  ASSERT_EQ(Dir2, "do lastprivate(, (x))");    // (21)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (21), (27)
+  ASSERT_EQ(Dir1, "parallel shared(x)");         // (22)
+  ASSERT_EQ(Dir2, "do lastprivate(, (x))");      // (21)
 }
 
 // SHARED
@@ -984,9 +984,9 @@ TEST_F(OpenMPDecompositionTest, Reduction7) {
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
   std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (36), (10)
-  ASSERT_EQ(Dir1, "parallel shared(x)");       // (36), (1), (4)
-  ASSERT_EQ(Dir2, "do reduction(, (3), (x))"); // (36)
+  ASSERT_EQ(Dir0, "target map(2, , , , , (x))"); // (36), (10)
+  ASSERT_EQ(Dir1, "parallel shared(x)");         // (36), (1), (4)
+  ASSERT_EQ(Dir2, "do reduction(, (3), (x))");   // (36)
 }
 
 // IF


        

From flang-commits at lists.llvm.org  Tue Jul 22 05:37:59 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 22 Jul 2025 05:37:59 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <687f8627.050a0220.3ba056.2c16@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/149137

From flang-commits at lists.llvm.org  Tue Jul 22 05:47:31 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 05:47:31 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][cuda] Introduce
 cuf.set_allocator_idx operation (PR #148717)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148717@github.com>
Message-ID: <687f8863.170a0220.99ec4.2a31@mx.google.com>


================
@@ -72,3 +72,13 @@ TEST(AllocatableCUFTest, DescriptorAllocationTest) {
   EXPECT_TRUE(desc != nullptr);
   RTNAME(CUFFreeDescriptor)(desc);
 }
+
+TEST(AllocatableCUFTest, CUFSetAllocatorIndex) {
+  using Fortran::common::TypeCategory;
+  RTNAME(CUFRegisterAllocator)();
+  // REAL(4), DEVICE, ALLOCATABLE :: a(:)
+  auto a{createAllocatable(TypeCategory::Real, 4)};
+  EXPECT_EQ((int)kDefaultAllocator, a->GetAllocIdx());
+  RTNAME(CUFSetAllocatorIndex)(*a, kDeviceAllocatorPos, __FILE__, __LINE__);
----------------
Meinersbur wrote:

I get a compilation error here (even after 9e9fdd433adfd4c1da46e475f4bbe8fdbf7c04f9):
```
../../../../_src/flang-rt/unittests/Runtime/CUDA/AllocatorCUF.cpp:82:3: error: no matching function for call to '_FortranACUFSetAllocatorIndex'
   82 |   RTNAME(CUFSetAllocatorIndex)(*a, kDeviceAllocatorPos, __FILE__, __LINE__);
      |   ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
../../../../_src/flang/include/flang/Runtime/entry-names.h:27:22: note: expanded from macro 'RTNAME'
   27 | #define RTNAME(name) NAME_WITH_PREFIX_AND_REVISION(_Fortran, A, name)
      |                      ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../../../../_src/flang/include/flang/Runtime/entry-names.h:26:3: note: expanded from macro 'NAME_WITH_PREFIX_AND_REVISION'
   26 |   prefix##revision##name
      |   ^~~~~~~~~~~~~~~~~~~~~~
<scratch space>:42:1: note: expanded from here
   42 | _FortranACUFSetAllocatorIndex
      | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../../../../_src/flang/include/flang/Runtime/CUDA/descriptor.h:45:6: note: candidate function not viable: no known conversion from 'Fortran::runtime::Descriptor' to 'Descriptor *' for 1st argument; take the address of the argument with &
   45 | void RTDECL(CUFSetAllocatorIndex)(Descriptor *, int index,
      |      ^                            ~~~~~~~~~~~~
../../../../_src/flang/include/flang/Runtime/entry-names.h:31:35: note: expanded from macro 'RTDECL'
   31 | #define RTDECL(name) RT_API_ATTRS RTNAME(name)
      |                                   ^
../../../../_src/flang/include/flang/Runtime/entry-names.h:27:22: note: expanded from macro 'RTNAME'
   27 | #define RTNAME(name) NAME_WITH_PREFIX_AND_REVISION(_Fortran, A, name)
      |                      ^
../../../../_src/flang/include/flang/Runtime/entry-names.h:26:3: note: expanded from macro 'NAME_WITH_PREFIX_AND_REVISION'
   26 |   prefix##revision##name
      |   ^
<scratch space>:16:1: note: expanded from here
   16 | _FortranACUFSetAllocatorIndex
      | ^
1 error generated.
```

```suggestion
RTNAME(CUFSetAllocatorIndex)(a, kDeviceAllocatorPos, __FILE__, __LINE__);
```

https://github.com/llvm/llvm-project/pull/148717

From flang-commits at lists.llvm.org  Tue Jul 22 05:47:45 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 05:47:45 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][cuda] Introduce
 cuf.set_allocator_idx operation (PR #148717)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/148717@github.com>
Message-ID: <687f8871.630a0220.2b5166.18d1@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/148717

From flang-commits at lists.llvm.org  Tue Jul 22 05:59:12 2025
From: flang-commits at lists.llvm.org (Scott Manley via flang-commits)
Date: Tue, 22 Jul 2025 05:59:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f8b20.a70a0220.2819f3.3581@mx.google.com>


================
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
----------------
rscottmanley wrote:

I think this should be a lambda instead

https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Tue Jul 22 06:00:22 2025
From: flang-commits at lists.llvm.org (Scott Manley via flang-commits)
Date: Tue, 22 Jul 2025 06:00:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <687f8b66.050a0220.245136.2263@mx.google.com>


rscottmanley wrote:

The title of this MR confused me for a minute, can it just be "add conversion of `fir.if` to `scf.if`"? or "convert `fir.if` in FIRToSCF"

https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Tue Jul 22 06:17:54 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 06:17:54 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [OpenMP] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <687f8f82.170a0220.52a67.082b@mx.google.com>


https://github.com/skc7 edited https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Tue Jul 22 06:19:07 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 06:19:07 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [OpenMP] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <687f8fcb.050a0220.17815.269a@mx.google.com>


https://github.com/skc7 edited https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Tue Jul 22 06:19:42 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 06:19:42 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [OpenMP] Introduce
 omp.target_allocmem and omp.target_freemem omp dialect ops. (PR #145464)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145464@github.com>
Message-ID: <687f8fee.170a0220.364154.4c28@mx.google.com>


https://github.com/skc7 edited https://github.com/llvm/llvm-project/pull/145464

From flang-commits at lists.llvm.org  Tue Jul 22 06:24:39 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 06:24:39 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang-rt] Add Assign_omp RT call.
	(PR #145465)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/145465@github.com>
Message-ID: <687f9117.170a0220.2d38c5.365a@mx.google.com>


https://github.com/skc7 edited https://github.com/llvm/llvm-project/pull/145465

From flang-commits at lists.llvm.org  Tue Jul 22 07:05:34 2025
From: flang-commits at lists.llvm.org (Aadesh Premkumar via flang-commits)
Date: Tue, 22 Jul 2025 07:05:34 -0700 (PDT)
Subject: [flang-commits] [flang] [FLANG] Fix for issue #127426 (PR #150008)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150008@github.com>
Message-ID: <687f9aae.170a0220.2ccc88.7fa4@mx.google.com>


https://github.com/aadeshps-mcw ready_for_review https://github.com/llvm/llvm-project/pull/150008

From flang-commits at lists.llvm.org  Tue Jul 22 07:11:38 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 22 Jul 2025 07:11:38 -0700 (PDT)
Subject: [flang-commits] [flang] 3855c49 - [flang][OpenMP] Fix build break
	with -Werror
Message-ID: <687f9c1a.630a0220.138c34.7660@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-22T09:11:18-05:00
New Revision: 3855c49f28d5ee4b32bf1478d0a50ca191bcf2b6

URL: https://github.com/llvm/llvm-project/commit/3855c49f28d5ee4b32bf1478d0a50ca191bcf2b6
DIFF: https://github.com/llvm/llvm-project/commit/3855c49f28d5ee4b32bf1478d0a50ca191bcf2b6.diff

LOG: [flang][OpenMP] Fix build break with -Werror

llvm-project/flang/lib/Semantics/resolve-directives.cpp:753:7: error: default label in switch which covers all enumeration values [-Werror,-Wcovered-switch-default]
  753 |       default:
      |       ^
1 error generated.

https://lab.llvm.org/buildbot/#/builders/53/builds/17917

Added: 
    

Modified: 
    flang/lib/Semantics/resolve-directives.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index addd7d322a4fd..4c3e509b5a36d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -750,8 +750,6 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       case parser::OmpMapType::Value::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
-      default:
-        break;
       }
     }
     if (!ompFlag) {


        

From flang-commits at lists.llvm.org  Tue Jul 22 07:39:09 2025
From: flang-commits at lists.llvm.org (Carlos Seo via flang-commits)
Date: Tue, 22 Jul 2025 07:39:09 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ASSIGN statement (PR #149941)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149941@github.com>
Message-ID: <687fa28d.170a0220.19c7be.ae69@mx.google.com>


================
@@ -5506,10 +5506,34 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   void genFIR(const Fortran::parser::AssignStmt &stmt) {
     const Fortran::semantics::Symbol &symbol =
         *std::get<Fortran::parser::Name>(stmt.t).symbol;
+
----------------
ceseo wrote:

I couldn't make it work. I think it's because hlfir.assign expects compatible types and it fails in the case where we have a boxed pointer (which is the situation uncovered by the original issue).

https://github.com/llvm/llvm-project/pull/149941

From flang-commits at lists.llvm.org  Tue Jul 22 07:44:39 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Tue, 22 Jul 2025 07:44:39 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687fa3d7.170a0220.a991.9e95@mx.google.com>


================
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
----------------
klausler wrote:

I don't understand how flang tests that depend on fortran_type_info.mod are going to work.

https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Tue Jul 22 07:56:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 07:56:01 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687fa681.170a0220.32fb4.abdb@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions h,cpp -- clang/include/clang/Driver/Driver.h clang/include/clang/Driver/ToolChain.h clang/lib/Driver/Driver.cpp clang/lib/Driver/ToolChain.cpp clang/lib/Driver/ToolChains/Flang.cpp flang/include/flang/Frontend/CompilerInvocation.h flang/lib/Frontend/CompilerInvocation.cpp flang/tools/bbc/bbc.cpp
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 451521eb7..69365dd27 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -402,7 +402,6 @@ private:
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
   /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the

``````````

</details>


https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Tue Jul 22 07:52:57 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 07:52:57 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687fa5c9.170a0220.1113e.99a7@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 01/14] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 02/14] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 03/14] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 04/14] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 05/14] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 06/14] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 

>From 498d60cf39c237c654c0cfddbe444d63bc87bc6c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 09:57:18 +0200
Subject: [PATCH 07/14] CI test

---
 flang-rt/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 5ffa1bcc34a41..d987115c8e9e8 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,40 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
 enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
@@ -159,6 +193,7 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)

>From 1bb0144eb21c2b1c8e1e66029d5573d8f5619c4c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 14:24:44 +0200
Subject: [PATCH 08/14] Enforce REAL(16) support

---
 flang-rt/CMakeLists.txt      | 45 ++++-------------------
 llvm/runtimes/CMakeLists.txt |  8 ++++
 runtimes/CMakeLists.txt      | 71 ++++++++++++++++++------------------
 3 files changed, 51 insertions(+), 73 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d987115c8e9e8..bfbd0af0c31dc 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,41 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
@@ -79,6 +44,11 @@ include(CMakePushCheckState)
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
+# Fortran compiler not optional for building Flang-RT
+enable_language(Fortran)
+
+flang_module_fortran_enable()
+
 
 #################
 # Build Options #
@@ -193,7 +163,9 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+# Look for support of REAL(16), if not already defined via command line.
+# NOTE: Does not work with Flang and CMake < 3.24
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
@@ -206,7 +178,6 @@ check_fortran_source_compiles([[
 )
 cmake_pop_check_state()
 
-flang_module_fortran_enable()
 
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 2f2f6db16255e..2bf2b0ee8ed50 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -528,6 +528,14 @@ if(build_runtimes)
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
+  if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    # Ensure REAL(16) support in runtimes to be consistent with compiler
+    if (FLANG_RUNTIME_F128_MATH_LIB OR HAVE_LDBL_MANT_DIG_113)
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=TRUE")
+   else ()
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=FALSE")
+   endif ()
+  endif ()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 987d173a283be..51214a46f558e 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -94,6 +94,41 @@ include(CheckCXXCompilerFlag)
 include(ExtendPath)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang, if used.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
+
 # Determine whether we are in the runtimes/runtimes-bins directory of a
 # bootstrapping build.
 set(LLVM_TREE_AVAILABLE OFF)
@@ -334,42 +369,6 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
-      _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-
-    enable_language(Fortran)
-  endif ()
-endif ()
-
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 8ec6d03939c3a7e2f79bb3ea5911b7607028dd7c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 21:52:02 +0200
Subject: [PATCH 09/14] Also add dependency barrier for WIN32

---
 flang-rt/lib/runtime/CMakeLists.txt | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 1b8114c102205..20f5d84bb2b69 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -281,14 +281,18 @@ else()
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
 
-    add_flangrt_library(${name}.intrinsics OBJECT
+    add_flangrt_library(${name}.intrinsics.obj OBJECT
       ${intrinsics_sources}
     )
+    add_custom_target(${name}.intrinsics
+      COMMENT "Intrinsic module dependency barrier"
+    )
+    add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
 
     add_flangrt_library(${name} ${libtype}
-        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics.obj>
         ${ARGN}
-        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -311,8 +315,9 @@ else()
     endif ()
 
     get_target_property(compile_target ${name}.compile ALIASED_TARGET)
-    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
+    add_dependencies(${compile_target} ${name}.intrinsics)
     add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")

>From 580a0c56535a9cd1c65fd5a5e6309b73c36ae8b6 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:05:34 +0200
Subject: [PATCH 10/14] Dependency barrier test

---
 flang-rt/lib/runtime/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 20f5d84bb2b69..bdd4318832473 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,6 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
+      COMMAND echo "Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
@@ -295,6 +296,7 @@ else()
         LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
 
     if (msvc_lib)
       set_target_properties(${name}
@@ -311,14 +313,13 @@ else()
       set(is_public "")
     else ()
       set(is_public PUBLIC)
+      add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
       set(_has_public_intrinsics "YES" PARENT_SCOPE)
     endif ()
 
-    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
     flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
     add_dependencies(${compile_target} ${name}.intrinsics)
-    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")

>From 979691a5bba4888be8c7c82d1bed4e8cdc71fff9 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:06:40 +0200
Subject: [PATCH 11/14] Dependency barrier info

---
 flang-rt/lib/runtime/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index bdd4318832473..24b84b11f9513 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,7 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
-      COMMAND echo "Dependency barrier"
+      COMMAND echo "${name} Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)

>From b4adeab58e615b9059c0a6e5bbcb376d1fe21bb2 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Mon, 21 Jul 2025 21:56:26 +0200
Subject: [PATCH 12/14] Avoid unrelated changes

---
 clang/include/clang/Driver/Driver.h     | 1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 9343fed36b6ac..14e1e644e51aa 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,6 +403,7 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
+
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
   /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 688b5aacc4bcd..609ba27bc212b 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1376,10 +1376,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(
-        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
-                 "' was not generated and skipExternalRttiDefinition and "
-                 "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(loc,
+                        "runtime derived type info descriptor was not "
+                        "generated and skipExternalRttiDefinition and "
+                        "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition

>From c40f43c98ff7ddbcb52c1ac35210320926369b2c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 14:16:19 +0200
Subject: [PATCH 13/14] Use -fintrinsic-modules-path=

---
 flang/test/lit.cfg.py   | 20 +++-----------------
 runtimes/CMakeLists.txt |  2 +-
 2 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index b05eba8da0b0c..bbf9a5a9f277f 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -147,26 +147,18 @@ def get_resource_module_intrinsic_dir():
 
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
+    intrinsics_search_args += [f"-fintrinsic-modules-path={flang_intrinsics_dir}"]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
     extra_intrinsics_search_args += [
-        "-fintrinsic-modules-path",
-        config.flang_intrinsic_modules_dir,
+        f"-fintrinsic-modules-path={config.flang_intrinsic_modules_dir}",
     ]
     lit_config.note(
         f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
     )
 
-config.substitutions.append(
-    (
-        "%intrinsic_module_flags",
-        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
-    )
-)
-
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
@@ -193,13 +185,7 @@ def get_resource_module_intrinsic_dir():
         "%bbc_bare",
         command=FindTool("bbc"),
         unresolved="fatal",
-    ),
-    ToolSubst(
-        "%flang_bare",
-        command=FindTool("flang"),
-        extra_args=isysroot_flag,
-        unresolved="fatal",
-    ),
+    )
 ]
 
 # Flang has several unimplemented features. TODO messages are used to mark
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 51214a46f558e..0832767505fe1 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -454,7 +454,7 @@ function (flang_module_target tgtname)
 
   # Let it find the other public module files
   target_compile_options(${tgtname} PRIVATE
-      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+      "$<$<COMPILE_LANGUAGE:Fortran>:-fintrinsic-modules-path=${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
     )
 
   if (ARG_PUBLIC)

>From 18422d4d6ea24aeb29a4fece11ba7b509fbd71d6 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 16:50:58 +0200
Subject: [PATCH 14/14] Rework intrinsic-module-path.f90 test

---
 flang/test/Driver/Inputs/ieee_arithmetic.mod |  1 +
 flang/test/Driver/Inputs/iso_fortran_env.mod |  1 +
 flang/test/Driver/intrinsic-module-path.f90  | 54 ++++++++++++++++----
 flang/test/lit.cfg.py                        |  2 +-
 4 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/flang/test/Driver/Inputs/ieee_arithmetic.mod b/flang/test/Driver/Inputs/ieee_arithmetic.mod
index 30fd57801970b..451d1af62a941 100644
--- a/flang/test/Driver/Inputs/ieee_arithmetic.mod
+++ b/flang/test/Driver/Inputs/ieee_arithmetic.mod
@@ -1,5 +1,6 @@
 ! DUMMY module
 ! Added for testing purposes. The contents of this file are currently not relevant.
+! Using this file file cause an error because of missing checksum
 module ieee_arithmetic
 type::ieee_round_type
 integer(1),private::mode=0_1
diff --git a/flang/test/Driver/Inputs/iso_fortran_env.mod b/flang/test/Driver/Inputs/iso_fortran_env.mod
index 689297d52027b..ad501c2d9c1b8 100644
--- a/flang/test/Driver/Inputs/iso_fortran_env.mod
+++ b/flang/test/Driver/Inputs/iso_fortran_env.mod
@@ -1,5 +1,6 @@
 ! DUMMY module
 ! Added for testing purposes. The contents of this file are currently not relevant.
+! Using this file file cause an error because of missing checksum
 module iso_fortran_env
 use __fortran_builtins,only:event_type=>__builtin_event_type
 use __fortran_builtins,only:lock_type=>__builtin_lock_type
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 3e696ce5d08e0..3317eb776f0a1 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -1,23 +1,55 @@
 ! Ensure argument -fintrinsic-modules-path works as expected.
-! WITHOUT the option, the default location for the module is checked and no error generated.
-! With the option GIVEN, the module with the same name is PREPENDED, and considered over the
-! default one, causing a CHECKSUM error.
 
 !-----------------------------------------
-! FRONTEND FLANG DRIVER (flang -fc1)
+! FLANG DRIVER
 !-----------------------------------------
-! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
-! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
-! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path=%S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
+! NOTE: Depending on how Flang is built, the default intrinsics may have higher
+!       or lower priority than -fintrinsic-modules-path added here. Using
+!       basictestmoduleone.mod from Inputs/module-dir/ will trigger an error.
 
-! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
-! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
+! RUN:     %flang -fsyntax-only -### %s 2>&1 | FileCheck %s --check-prefix=DEFAULTPATH
 
-! GIVEN: error: Cannot use module file for module 'ieee_arithmetic': File has invalid checksum
-! GIVEN: error: Cannot use module file for module 'iso_fortran_env': File has invalid checksum
+! RUN:     %flang -fsyntax-only -DINTRINSICS_DEFAULT %s
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTONE %s 2>&1 | FileCheck %s --check-prefix=NOINPUTONE
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTTWO %s 2>&1 | FileCheck %s --check-prefix=NOINPUTTWO
+! RUN:     %flang -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN:     %flang -fsyntax-only -DINTRINSICS_INPUTONE -fintrinsic-modules-path=%S/Inputs/ %s
+! RUN:     %flang -fsyntax-only -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/ -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir/ -fintrinsic-modules-path=%S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=WRONGINPUTONE
 
 
+!-----------------------------------------
+! FLANG FRONTEND (flang -fc1)
+!-----------------------------------------
+! NOTE: %flang_cc1 the default intrinsics path always has higher priority than
+!       -fintrinsic-modules-path added here. Accidentally using
+!       ieee_arithmetic/iso_fortran_env from the Inputs/ directory will trigger
+!       an error (e.g. when the default intrinsics dir is empty).
+
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT %s
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE %s 2>&1 | FileCheck %s --check-prefix=NOINPUTONE
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO %s 2>&1 | FileCheck %s --check-prefix=NOINPUTTWO
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir %s
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -fintrinsic-modules-path=%S/Inputs/ %s
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/ -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir -fintrinsic-modules-path=%S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=WRONGINPUTONE
+
+
+! DEFAULTPATH: flang{{.*}}-fc1{{.*}}-fintrinsic-modules-path
+
+! NOINPUTONE: Source file 'basictestmoduleone.mod' was not found
+! NOINPUTTWO: Source file 'basictestmoduletwo.mod' was not found
+! WRONGINPUTONE: 't1' not found in module 'basictestmoduleone'
+
 program test_intrinsic_module_path
+#ifdef INTRINSICS_DEFAULT
    use ieee_arithmetic, only: ieee_round_type
    use iso_fortran_env, only: team_type, event_type, lock_type
+#endif
+#ifdef INTRINSICS_INPUTONE
+   use basictestmoduleone, only: t1
+#endif
+#ifdef INTRINSICS_INPUTTWO
+   use basictestmoduletwo, only: t2
+#endif
 end program
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index bbf9a5a9f277f..ba1e5d55b503c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -180,7 +180,7 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-    # For not having intrinsic search paths to be added implicitly
+    # Do not implicitly add intrinsic search paths
     ToolSubst(
         "%bbc_bare",
         command=FindTool("bbc"),


From flang-commits at lists.llvm.org  Tue Jul 22 08:07:59 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 08:07:59 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687fa94f.630a0220.290657.911b@mx.google.com>


================
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
----------------
Meinersbur wrote:

@klausler 
If building using a bootstrapping configuration (`-DLLVM_ENABLE_RUNTIMES=flang-rt`, implicitly added when `-DLLVM_ENABLE_PROJECTS=flang`), the mod files are automatically put into the Clang/Flang resource directory as a dependency of `check-flang`. If explictly disabled using `-DFLANG_ENABLE_FLANG_RT=OFF` or in a Flang-standalone build (that currently does not support LLVM_ENABLE_RUNTIMES), Flang-RT can be compiled separately and `FLANG_INTRINSIC_MODULES_DIR` pointed to it. `lit.cfg.py` will add the path via `-fintrinsic-module-path` to each `flang`/`%flang_fc1` call. If FLANG_INTRINSIC_MODULES_DIR is not set then tests in `test/` will be skipped.

https://github.com/llvm/llvm-project/pull/137828

From flang-commits at lists.llvm.org  Tue Jul 22 08:09:17 2025
From: flang-commits at lists.llvm.org (Carlos Seo via flang-commits)
Date: Tue, 22 Jul 2025 08:09:17 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ASSIGN statement (PR #149941)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149941@github.com>
Message-ID: <687fa99d.170a0220.2ee6de.ab80@mx.google.com>


https://github.com/ceseo updated https://github.com/llvm/llvm-project/pull/149941

>From e468ec6f7e4034578f0fe6e6a6e72e6a182fc4fd Mon Sep 17 00:00:00 2001
From: Carlos Seo <carlos.seo at linaro.org>
Date: Mon, 21 Jul 2025 23:11:16 +0000
Subject: [PATCH 1/3] [Flang] Fix ASSIGN statement

Handle the case where the assigned variable also has a pointer attribute.

Fixes #121721
---
 flang/lib/Lower/Bridge.cpp | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 7ce397a11861b..243fda63d5e29 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5506,10 +5506,35 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   void genFIR(const Fortran::parser::AssignStmt &stmt) {
     const Fortran::semantics::Symbol &symbol =
         *std::get<Fortran::parser::Name>(stmt.t).symbol;
+
     mlir::Location loc = toLocation();
+    mlir::Type symbolType = genType(symbol);
+    mlir::Value addr = getSymbolAddress(symbol);
+
+    // Handle the case where the assigned variable is declared as a pointer
+    if (auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(symbolType)) {
+      if (auto ptrType = mlir::dyn_cast<fir::PointerType>(eleTy)) {
+        symbolType = ptrType.getEleTy();
+      } else {
+        symbolType = eleTy;
+      }
+    }
+    else if (auto ptrType = mlir::dyn_cast<fir::PointerType>(symbolType)) {
+      symbolType = ptrType.getEleTy();
+    }
+
     mlir::Value labelValue = builder->createIntegerConstant(
-        loc, genType(symbol), std::get<Fortran::parser::Label>(stmt.t));
-    builder->create<fir::StoreOp>(loc, labelValue, getSymbolAddress(symbol));
+        loc, symbolType, std::get<Fortran::parser::Label>(stmt.t));
+
+    // If the address points to a boxed pointer, we need to dereference it
+    if (auto refType = mlir::dyn_cast<fir::ReferenceType>(addr.getType())) {
+      if (auto boxType = mlir::dyn_cast<fir::BoxType>(refType.getEleTy())) {
+        mlir::Value boxValue = builder->create<fir::LoadOp>(loc, addr);
+        addr = builder->create<fir::BoxAddrOp>(loc, boxValue);
+      }
+    }
+
+    builder->create<fir::StoreOp>(loc, labelValue, addr);
   }
 
   void genFIR(const Fortran::parser::FormatStmt &) {

>From 04d5cd81990032aa1a21de253983363331f6df1a Mon Sep 17 00:00:00 2001
From: Carlos Seo <carlos.seo at linaro.org>
Date: Mon, 21 Jul 2025 23:35:54 +0000
Subject: [PATCH 2/3] Fix formatting

---
 flang/lib/Lower/Bridge.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 243fda63d5e29..9c0fcfb8c74ba 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5518,8 +5518,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       } else {
         symbolType = eleTy;
       }
-    }
-    else if (auto ptrType = mlir::dyn_cast<fir::PointerType>(symbolType)) {
+    } else if (auto ptrType = mlir::dyn_cast<fir::PointerType>(symbolType)) {
       symbolType = ptrType.getEleTy();
     }
 

>From 2de6a3ed9386199ca77cfa16c964f0c86c4fa278 Mon Sep 17 00:00:00 2001
From: Carlos Seo <carlos.seo at linaro.org>
Date: Tue, 22 Jul 2025 15:08:24 +0000
Subject: [PATCH 3/3] Added testcase

---
 flang/test/Lower/assign-statement.f90 | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 flang/test/Lower/assign-statement.f90

diff --git a/flang/test/Lower/assign-statement.f90 b/flang/test/Lower/assign-statement.f90
new file mode 100644
index 0000000000000..342355bf469c8
--- /dev/null
+++ b/flang/test/Lower/assign-statement.f90
@@ -0,0 +1,12 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+  ! CHECK-LABEL: func @_QQmain
+  program main
+  integer :: ip
+  pointer :: ip
+
+  allocate(ip)
+  assign 10 to ip
+  ! CHECK: fir.store %c10_i32 to %11 : !fir.ptr<i32>
+  10 return
+  end program main


From flang-commits at lists.llvm.org  Tue Jul 22 08:12:54 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 08:12:54 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687faa76.a70a0220.12dd05.b499@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 01/15] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 02/15] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 03/15] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 04/15] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 05/15] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 06/15] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 

>From 498d60cf39c237c654c0cfddbe444d63bc87bc6c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 09:57:18 +0200
Subject: [PATCH 07/15] CI test

---
 flang-rt/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 5ffa1bcc34a41..d987115c8e9e8 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,40 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
 enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
@@ -159,6 +193,7 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)

>From 1bb0144eb21c2b1c8e1e66029d5573d8f5619c4c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 14:24:44 +0200
Subject: [PATCH 08/15] Enforce REAL(16) support

---
 flang-rt/CMakeLists.txt      | 45 ++++-------------------
 llvm/runtimes/CMakeLists.txt |  8 ++++
 runtimes/CMakeLists.txt      | 71 ++++++++++++++++++------------------
 3 files changed, 51 insertions(+), 73 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d987115c8e9e8..bfbd0af0c31dc 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,41 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
@@ -79,6 +44,11 @@ include(CMakePushCheckState)
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
+# Fortran compiler not optional for building Flang-RT
+enable_language(Fortran)
+
+flang_module_fortran_enable()
+
 
 #################
 # Build Options #
@@ -193,7 +163,9 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+# Look for support of REAL(16), if not already defined via command line.
+# NOTE: Does not work with Flang and CMake < 3.24
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
@@ -206,7 +178,6 @@ check_fortran_source_compiles([[
 )
 cmake_pop_check_state()
 
-flang_module_fortran_enable()
 
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 2f2f6db16255e..2bf2b0ee8ed50 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -528,6 +528,14 @@ if(build_runtimes)
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
+  if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    # Ensure REAL(16) support in runtimes to be consistent with compiler
+    if (FLANG_RUNTIME_F128_MATH_LIB OR HAVE_LDBL_MANT_DIG_113)
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=TRUE")
+   else ()
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=FALSE")
+   endif ()
+  endif ()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 987d173a283be..51214a46f558e 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -94,6 +94,41 @@ include(CheckCXXCompilerFlag)
 include(ExtendPath)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang, if used.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
+
 # Determine whether we are in the runtimes/runtimes-bins directory of a
 # bootstrapping build.
 set(LLVM_TREE_AVAILABLE OFF)
@@ -334,42 +369,6 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
-      _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-
-    enable_language(Fortran)
-  endif ()
-endif ()
-
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 8ec6d03939c3a7e2f79bb3ea5911b7607028dd7c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 21:52:02 +0200
Subject: [PATCH 09/15] Also add dependency barrier for WIN32

---
 flang-rt/lib/runtime/CMakeLists.txt | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 1b8114c102205..20f5d84bb2b69 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -281,14 +281,18 @@ else()
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
 
-    add_flangrt_library(${name}.intrinsics OBJECT
+    add_flangrt_library(${name}.intrinsics.obj OBJECT
       ${intrinsics_sources}
     )
+    add_custom_target(${name}.intrinsics
+      COMMENT "Intrinsic module dependency barrier"
+    )
+    add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
 
     add_flangrt_library(${name} ${libtype}
-        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics.obj>
         ${ARGN}
-        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -311,8 +315,9 @@ else()
     endif ()
 
     get_target_property(compile_target ${name}.compile ALIASED_TARGET)
-    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
+    add_dependencies(${compile_target} ${name}.intrinsics)
     add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")

>From 580a0c56535a9cd1c65fd5a5e6309b73c36ae8b6 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:05:34 +0200
Subject: [PATCH 10/15] Dependency barrier test

---
 flang-rt/lib/runtime/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 20f5d84bb2b69..bdd4318832473 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,6 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
+      COMMAND echo "Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
@@ -295,6 +296,7 @@ else()
         LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
 
     if (msvc_lib)
       set_target_properties(${name}
@@ -311,14 +313,13 @@ else()
       set(is_public "")
     else ()
       set(is_public PUBLIC)
+      add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
       set(_has_public_intrinsics "YES" PARENT_SCOPE)
     endif ()
 
-    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
     flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
     add_dependencies(${compile_target} ${name}.intrinsics)
-    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")

>From 979691a5bba4888be8c7c82d1bed4e8cdc71fff9 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:06:40 +0200
Subject: [PATCH 11/15] Dependency barrier info

---
 flang-rt/lib/runtime/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index bdd4318832473..24b84b11f9513 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,7 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
-      COMMAND echo "Dependency barrier"
+      COMMAND echo "${name} Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)

>From b4adeab58e615b9059c0a6e5bbcb376d1fe21bb2 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Mon, 21 Jul 2025 21:56:26 +0200
Subject: [PATCH 12/15] Avoid unrelated changes

---
 clang/include/clang/Driver/Driver.h     | 1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 9343fed36b6ac..14e1e644e51aa 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,6 +403,7 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
+
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
   /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 688b5aacc4bcd..609ba27bc212b 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1376,10 +1376,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(
-        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
-                 "' was not generated and skipExternalRttiDefinition and "
-                 "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(loc,
+                        "runtime derived type info descriptor was not "
+                        "generated and skipExternalRttiDefinition and "
+                        "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition

>From c40f43c98ff7ddbcb52c1ac35210320926369b2c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 14:16:19 +0200
Subject: [PATCH 13/15] Use -fintrinsic-modules-path=

---
 flang/test/lit.cfg.py   | 20 +++-----------------
 runtimes/CMakeLists.txt |  2 +-
 2 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index b05eba8da0b0c..bbf9a5a9f277f 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -147,26 +147,18 @@ def get_resource_module_intrinsic_dir():
 
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
+    intrinsics_search_args += [f"-fintrinsic-modules-path={flang_intrinsics_dir}"]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
     extra_intrinsics_search_args += [
-        "-fintrinsic-modules-path",
-        config.flang_intrinsic_modules_dir,
+        f"-fintrinsic-modules-path={config.flang_intrinsic_modules_dir}",
     ]
     lit_config.note(
         f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
     )
 
-config.substitutions.append(
-    (
-        "%intrinsic_module_flags",
-        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
-    )
-)
-
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
@@ -193,13 +185,7 @@ def get_resource_module_intrinsic_dir():
         "%bbc_bare",
         command=FindTool("bbc"),
         unresolved="fatal",
-    ),
-    ToolSubst(
-        "%flang_bare",
-        command=FindTool("flang"),
-        extra_args=isysroot_flag,
-        unresolved="fatal",
-    ),
+    )
 ]
 
 # Flang has several unimplemented features. TODO messages are used to mark
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 51214a46f558e..0832767505fe1 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -454,7 +454,7 @@ function (flang_module_target tgtname)
 
   # Let it find the other public module files
   target_compile_options(${tgtname} PRIVATE
-      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+      "$<$<COMPILE_LANGUAGE:Fortran>:-fintrinsic-modules-path=${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
     )
 
   if (ARG_PUBLIC)

>From 18422d4d6ea24aeb29a4fece11ba7b509fbd71d6 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 16:50:58 +0200
Subject: [PATCH 14/15] Rework intrinsic-module-path.f90 test

---
 flang/test/Driver/Inputs/ieee_arithmetic.mod |  1 +
 flang/test/Driver/Inputs/iso_fortran_env.mod |  1 +
 flang/test/Driver/intrinsic-module-path.f90  | 54 ++++++++++++++++----
 flang/test/lit.cfg.py                        |  2 +-
 4 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/flang/test/Driver/Inputs/ieee_arithmetic.mod b/flang/test/Driver/Inputs/ieee_arithmetic.mod
index 30fd57801970b..451d1af62a941 100644
--- a/flang/test/Driver/Inputs/ieee_arithmetic.mod
+++ b/flang/test/Driver/Inputs/ieee_arithmetic.mod
@@ -1,5 +1,6 @@
 ! DUMMY module
 ! Added for testing purposes. The contents of this file are currently not relevant.
+! Using this file file cause an error because of missing checksum
 module ieee_arithmetic
 type::ieee_round_type
 integer(1),private::mode=0_1
diff --git a/flang/test/Driver/Inputs/iso_fortran_env.mod b/flang/test/Driver/Inputs/iso_fortran_env.mod
index 689297d52027b..ad501c2d9c1b8 100644
--- a/flang/test/Driver/Inputs/iso_fortran_env.mod
+++ b/flang/test/Driver/Inputs/iso_fortran_env.mod
@@ -1,5 +1,6 @@
 ! DUMMY module
 ! Added for testing purposes. The contents of this file are currently not relevant.
+! Using this file file cause an error because of missing checksum
 module iso_fortran_env
 use __fortran_builtins,only:event_type=>__builtin_event_type
 use __fortran_builtins,only:lock_type=>__builtin_lock_type
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 3e696ce5d08e0..3317eb776f0a1 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -1,23 +1,55 @@
 ! Ensure argument -fintrinsic-modules-path works as expected.
-! WITHOUT the option, the default location for the module is checked and no error generated.
-! With the option GIVEN, the module with the same name is PREPENDED, and considered over the
-! default one, causing a CHECKSUM error.
 
 !-----------------------------------------
-! FRONTEND FLANG DRIVER (flang -fc1)
+! FLANG DRIVER
 !-----------------------------------------
-! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
-! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
-! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path=%S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
+! NOTE: Depending on how Flang is built, the default intrinsics may have higher
+!       or lower priority than -fintrinsic-modules-path added here. Using
+!       basictestmoduleone.mod from Inputs/module-dir/ will trigger an error.
 
-! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
-! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
+! RUN:     %flang -fsyntax-only -### %s 2>&1 | FileCheck %s --check-prefix=DEFAULTPATH
 
-! GIVEN: error: Cannot use module file for module 'ieee_arithmetic': File has invalid checksum
-! GIVEN: error: Cannot use module file for module 'iso_fortran_env': File has invalid checksum
+! RUN:     %flang -fsyntax-only -DINTRINSICS_DEFAULT %s
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTONE %s 2>&1 | FileCheck %s --check-prefix=NOINPUTONE
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTTWO %s 2>&1 | FileCheck %s --check-prefix=NOINPUTTWO
+! RUN:     %flang -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN:     %flang -fsyntax-only -DINTRINSICS_INPUTONE -fintrinsic-modules-path=%S/Inputs/ %s
+! RUN:     %flang -fsyntax-only -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/ -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir/ -fintrinsic-modules-path=%S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=WRONGINPUTONE
 
 
+!-----------------------------------------
+! FLANG FRONTEND (flang -fc1)
+!-----------------------------------------
+! NOTE: %flang_cc1 the default intrinsics path always has higher priority than
+!       -fintrinsic-modules-path added here. Accidentally using
+!       ieee_arithmetic/iso_fortran_env from the Inputs/ directory will trigger
+!       an error (e.g. when the default intrinsics dir is empty).
+
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT %s
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE %s 2>&1 | FileCheck %s --check-prefix=NOINPUTONE
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO %s 2>&1 | FileCheck %s --check-prefix=NOINPUTTWO
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir %s
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -fintrinsic-modules-path=%S/Inputs/ %s
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/ -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir -fintrinsic-modules-path=%S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=WRONGINPUTONE
+
+
+! DEFAULTPATH: flang{{.*}}-fc1{{.*}}-fintrinsic-modules-path
+
+! NOINPUTONE: Source file 'basictestmoduleone.mod' was not found
+! NOINPUTTWO: Source file 'basictestmoduletwo.mod' was not found
+! WRONGINPUTONE: 't1' not found in module 'basictestmoduleone'
+
 program test_intrinsic_module_path
+#ifdef INTRINSICS_DEFAULT
    use ieee_arithmetic, only: ieee_round_type
    use iso_fortran_env, only: team_type, event_type, lock_type
+#endif
+#ifdef INTRINSICS_INPUTONE
+   use basictestmoduleone, only: t1
+#endif
+#ifdef INTRINSICS_INPUTTWO
+   use basictestmoduletwo, only: t2
+#endif
 end program
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index bbf9a5a9f277f..ba1e5d55b503c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -180,7 +180,7 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-    # For not having intrinsic search paths to be added implicitly
+    # Do not implicitly add intrinsic search paths
     ToolSubst(
         "%bbc_bare",
         command=FindTool("bbc"),

>From 9143388ef4c1c2bc9b09595c69ed85eac1daf2c8 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 17:12:30 +0200
Subject: [PATCH 15/15] Remove hint to reduce diff size

---
 clang/include/clang/Driver/Driver.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 451521eb70760..78a4c5738ae66 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -405,8 +405,6 @@ class Driver {
 
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
-  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
-  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,


From flang-commits at lists.llvm.org  Tue Jul 22 08:18:55 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 22 Jul 2025 08:18:55 -0700 (PDT)
Subject: [flang-commits] [flang] [not for review] Vol actual checks OLD (PR
	#150040)
Message-ID: <llvm.org/llvm/llvm-project/pull/150040@github.com>


https://github.com/eugeneepshteyn created https://github.com/llvm/llvm-project/pull/150040

This is not correct implementation, this PR is for easy reference not for review.

>From 50dd216ff0487709d52944006e35e6102015cef0 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Mon, 7 Jul 2025 10:20:38 -0400
Subject: [PATCH 1/4] [flang] Add -pedantic check for passing non-VOLATILE
 array section to VOLATILE dummy arg

Add the check, which partially addresses https://github.com/llvm/llvm-project/issues/137369

Implement HasTriplet().
---
 flang/include/flang/Evaluate/tools.h |  3 +++
 flang/lib/Evaluate/tools.cpp         | 19 +++++++++++++++++++
 flang/lib/Semantics/definable.cpp    |  3 +++
 3 files changed, 25 insertions(+)

diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index cad1b634f8924..0604049fd6a6d 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1123,6 +1123,9 @@ extern template semantics::UnorderedSymbolSet CollectCudaSymbols(
 // Predicate: does a variable contain a vector-valued subscript (not a triplet)?
 bool HasVectorSubscript(const Expr<SomeType> &);
 
+// Predicate: does a variable contain a triplet?
+bool HasTriplet(const Expr<SomeType> &);
+
 // Predicate: does an expression contain constant?
 bool HasConstant(const Expr<SomeType> &);
 
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index fcacdb93d662b..50a201dfcaced 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1173,6 +1173,25 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
   return HasVectorSubscriptHelper{}(expr);
 }
 
+// HasTriplet()
+struct HasTripletHelper
+    : public AnyTraverse<HasTripletHelper, bool,
+          /*TraverseAssocEntityDetails=*/false> {
+  using Base = AnyTraverse<HasTripletHelper, bool, false>;
+  HasTripletHelper() : Base{*this} {}
+  using Base::operator();
+  bool operator()(const Subscript &ss) const {
+    return std::holds_alternative<Triplet>(ss.u);
+  }
+  bool operator()(const ProcedureRef &) const {
+    return false; // don't descend into function call arguments
+  }
+};
+
+bool HasTriplet(const Expr<SomeType> &expr) {
+  return HasTripletHelper{}(expr);
+}
+
 // HasConstant()
 struct HasConstantHelper : public AnyTraverse<HasConstantHelper, bool,
                                /*TraverseAssocEntityDetails=*/false> {
diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp
index 08cb268b318ae..b2d2330887a53 100644
--- a/flang/lib/Semantics/definable.cpp
+++ b/flang/lib/Semantics/definable.cpp
@@ -371,6 +371,9 @@ std::optional<parser::Message> WhyNotDefinable(parser::CharBlock at,
         return parser::Message{at,
             "Variable '%s' has a vector subscript"_err_en_US, expr.AsFortran()};
       }
+    } else if (evaluate::HasTriplet(expr)) {
+      return parser::Message{at,
+          "Variable '%s' has array section"_err_en_US, expr.AsFortran()};
     }
     if (FindPureProcedureContaining(scope) &&
         evaluate::ExtractCoarrayRef(expr)) {

>From f5379722f62ea5066261055f3781c8788cf6f929 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 8 Jul 2025 22:24:06 -0400
Subject: [PATCH 2/4] Added DefinabilityFlag::VolatileNotDefinable

---
 flang/lib/Semantics/check-call.cpp | 4 ++--
 flang/lib/Semantics/definable.cpp  | 3 ++-
 flang/lib/Semantics/definable.h    | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 6f2503285013d..037d735b6d75f 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -734,8 +734,8 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
               characteristics::DummyDataObject::Attr::Asynchronous)) {
         undefinableMessage =
             "Actual argument associated with ASYNCHRONOUS %s is not definable"_warn_en_US;
-      } else if (dummy.attrs.test(
-                     characteristics::DummyDataObject::Attr::Volatile)) {
+      } else if (dummyIsVolatile) {
+        flags.set(DefinabilityFlag::VolatileNotDefinable);
         undefinableMessage =
             "Actual argument associated with VOLATILE %s is not definable"_warn_en_US;
       }
diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp
index b2d2330887a53..4a387303da8a2 100644
--- a/flang/lib/Semantics/definable.cpp
+++ b/flang/lib/Semantics/definable.cpp
@@ -371,7 +371,8 @@ std::optional<parser::Message> WhyNotDefinable(parser::CharBlock at,
         return parser::Message{at,
             "Variable '%s' has a vector subscript"_err_en_US, expr.AsFortran()};
       }
-    } else if (evaluate::HasTriplet(expr)) {
+    } else if (flags.test(DefinabilityFlag::VolatileNotDefinable) &&
+               evaluate::HasTriplet(expr)) {
       return parser::Message{at,
           "Variable '%s' has array section"_err_en_US, expr.AsFortran()};
     }
diff --git a/flang/lib/Semantics/definable.h b/flang/lib/Semantics/definable.h
index 0d027961417be..879d46a2f526c 100644
--- a/flang/lib/Semantics/definable.h
+++ b/flang/lib/Semantics/definable.h
@@ -33,7 +33,7 @@ ENUM_CLASS(DefinabilityFlag,
     SourcedAllocation, // ALLOCATE(a,SOURCE=)
     PolymorphicOkInPure, // don't check for polymorphic type in pure subprogram
     DoNotNoteDefinition, // context does not imply definition
-    AllowEventLockOrNotifyType, PotentialDeallocation)
+    AllowEventLockOrNotifyType, PotentialDeallocation, VolatileNotDefinable)
 
 using DefinabilityFlags =
     common::EnumSet<DefinabilityFlag, DefinabilityFlag_enumSize>;

>From 411eb310645b04dca904415a74fbef2243c1879c Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 8 Jul 2025 22:24:34 -0400
Subject: [PATCH 3/4] clang-format

---
 flang/lib/Semantics/definable.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp
index 4a387303da8a2..77bb7ceae65a6 100644
--- a/flang/lib/Semantics/definable.cpp
+++ b/flang/lib/Semantics/definable.cpp
@@ -372,7 +372,7 @@ std::optional<parser::Message> WhyNotDefinable(parser::CharBlock at,
             "Variable '%s' has a vector subscript"_err_en_US, expr.AsFortran()};
       }
     } else if (flags.test(DefinabilityFlag::VolatileNotDefinable) &&
-               evaluate::HasTriplet(expr)) {
+        evaluate::HasTriplet(expr)) {
       return parser::Message{at,
           "Variable '%s' has array section"_err_en_US, expr.AsFortran()};
     }

>From fcba7f4c8567aaad6928446a25d3cd66790297d9 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 8 Jul 2025 23:17:05 -0400
Subject: [PATCH 4/4] Updated flang/test/Semantics/call03.f90 with the new
 warning info

---
 flang/test/Semantics/call03.f90 | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/flang/test/Semantics/call03.f90 b/flang/test/Semantics/call03.f90
index 1721b59986862..0dd4f300c4f50 100644
--- a/flang/test/Semantics/call03.f90
+++ b/flang/test/Semantics/call03.f90
@@ -362,16 +362,28 @@ subroutine test15(assumedrank) ! C1539
     call valueassumedsize(c(::2)) ! ok
     call valueassumedsize(d(::2)) ! ok
     !ERROR: ASYNCHRONOUS or VOLATILE actual argument that is not simply contiguous may not be associated with a contiguous ASYNCHRONOUS or VOLATILE dummy argument 'x='
+    !ERROR: Actual argument associated with VOLATILE dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
+    !BECAUSE: Variable 'b(::2_8)' has array section
     call volatileassumedsize(b(::2))
     !ERROR: ASYNCHRONOUS or VOLATILE actual argument that is not simply contiguous may not be associated with a contiguous ASYNCHRONOUS or VOLATILE dummy argument 'x='
+    !ERROR: Actual argument associated with VOLATILE dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
+    !BECAUSE: Variable 'b(::2_8)' has array section
     call volatilecontiguous(b(::2))
     !ERROR: ASYNCHRONOUS or VOLATILE actual argument that is not simply contiguous may not be associated with a contiguous ASYNCHRONOUS or VOLATILE dummy argument 'x='
+    !ERROR: Actual argument associated with VOLATILE dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
+    !BECAUSE: Variable 'c(::2_8)' has array section
     call volatileassumedsize(c(::2))
     !ERROR: ASYNCHRONOUS or VOLATILE actual argument that is not simply contiguous may not be associated with a contiguous ASYNCHRONOUS or VOLATILE dummy argument 'x='
+    !ERROR: Actual argument associated with VOLATILE dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
+    !BECAUSE: Variable 'c(::2_8)' has array section
     call volatilecontiguous(c(::2))
     !ERROR: ASYNCHRONOUS or VOLATILE actual argument that is not simply contiguous may not be associated with a contiguous ASYNCHRONOUS or VOLATILE dummy argument 'x='
+    !ERROR: Actual argument associated with VOLATILE dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
+    !BECAUSE: Variable 'd(::2_8)' has array section
     call volatileassumedsize(d(::2))
     !ERROR: ASYNCHRONOUS or VOLATILE actual argument that is not simply contiguous may not be associated with a contiguous ASYNCHRONOUS or VOLATILE dummy argument 'x='
+    !ERROR: Actual argument associated with VOLATILE dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
+    !BECAUSE: Variable 'd(::2_8)' has array section
     call volatilecontiguous(d(::2))
     !ERROR: ASYNCHRONOUS or VOLATILE actual argument that is not simply contiguous may not be associated with a contiguous ASYNCHRONOUS or VOLATILE dummy argument 'x='
     call volatilecontiguous(assumedrank)


From flang-commits at lists.llvm.org  Tue Jul 22 08:21:21 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 22 Jul 2025 08:21:21 -0700 (PDT)
Subject: [flang-commits] [flang] [not for review] Vol actual checks OLD
 (implements HasTriplet()) (PR #150040)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150040@github.com>
Message-ID: <687fac71.170a0220.2392fc.9b81@mx.google.com>


https://github.com/eugeneepshteyn edited https://github.com/llvm/llvm-project/pull/150040

From flang-commits at lists.llvm.org  Tue Jul 22 08:21:28 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 08:21:28 -0700 (PDT)
Subject: [flang-commits] [flang] [not for review] Vol actual checks OLD
 (implements HasTriplet()) (PR #150040)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150040@github.com>
Message-ID: <687fac78.170a0220.292399.b19f@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions h,cpp -- flang/include/flang/Evaluate/tools.h flang/lib/Evaluate/tools.cpp flang/lib/Semantics/check-call.cpp flang/lib/Semantics/definable.cpp flang/lib/Semantics/definable.h
``````````

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 24aeabaa6..8657b4414 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1204,9 +1204,8 @@ bool HasVectorSubscript(const Expr<SomeType> &expr) {
 }
 
 // HasTriplet()
-struct HasTripletHelper
-    : public AnyTraverse<HasTripletHelper, bool,
-          /*TraverseAssocEntityDetails=*/false> {
+struct HasTripletHelper : public AnyTraverse<HasTripletHelper, bool,
+                              /*TraverseAssocEntityDetails=*/false> {
   using Base = AnyTraverse<HasTripletHelper, bool, false>;
   HasTripletHelper() : Base{*this} {}
   using Base::operator();
@@ -1218,9 +1217,7 @@ struct HasTripletHelper
   }
 };
 
-bool HasTriplet(const Expr<SomeType> &expr) {
-  return HasTripletHelper{}(expr);
-}
+bool HasTriplet(const Expr<SomeType> &expr) { return HasTripletHelper{}(expr); }
 
 // HasConstant()
 struct HasConstantHelper : public AnyTraverse<HasConstantHelper, bool,
diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp
index 77bb7ceae..d1fd8e005 100644
--- a/flang/lib/Semantics/definable.cpp
+++ b/flang/lib/Semantics/definable.cpp
@@ -373,8 +373,8 @@ std::optional<parser::Message> WhyNotDefinable(parser::CharBlock at,
       }
     } else if (flags.test(DefinabilityFlag::VolatileNotDefinable) &&
         evaluate::HasTriplet(expr)) {
-      return parser::Message{at,
-          "Variable '%s' has array section"_err_en_US, expr.AsFortran()};
+      return parser::Message{
+          at, "Variable '%s' has array section"_err_en_US, expr.AsFortran()};
     }
     if (FindPureProcedureContaining(scope) &&
         evaluate::ExtractCoarrayRef(expr)) {

``````````

</details>


https://github.com/llvm/llvm-project/pull/150040

From flang-commits at lists.llvm.org  Tue Jul 22 08:26:49 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 08:26:49 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Assign unittests to flang_compile
	pool (PR #150041)
Message-ID: <llvm.org/llvm/llvm-project/pull/150041@github.com>


https://github.com/Meinersbur created https://github.com/llvm/llvm-project/pull/150041

#127364 added the FLANG_PARALLEL_COMPILE_JOBS option to limit the number of parallel compilations of Flang sources which require an unusal amount of memory. This patch also adds Flang's unittests to this pool which use about the same amount of memory.

>From ed4290ec8fe7df65c5f215a2474514a12eb01d7e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 17:21:16 +0200
Subject: [PATCH] Assign unittests to flang_compile pool

---
 flang/unittests/CMakeLists.txt | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/flang/unittests/CMakeLists.txt b/flang/unittests/CMakeLists.txt
index b4e969c9426ee..db04923e2943a 100644
--- a/flang/unittests/CMakeLists.txt
+++ b/flang/unittests/CMakeLists.txt
@@ -12,6 +12,9 @@ set_target_properties(FlangUnitTests PROPERTIES FOLDER "Flang/Tests")
 
 function(add_flang_unittest test_dirname)
   add_unittest(FlangUnitTests ${test_dirname} ${ARGN})
+  if (FLANG_PARALLEL_COMPILE_JOBS)
+    set_property(TARGET ${test_dirname} PROPERTY JOB_POOL_COMPILE flang_compile_job_pool)
+  endif ()
 endfunction()
 
 if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
@@ -35,6 +38,9 @@ function(add_flang_nongtest_unittest test_name)
 
   add_executable(${test_name}${suffix} "${test_filepath}")
   set_target_properties(${test_name}${suffix} PROPERTIES FOLDER "Flang/Tests/Unit")
+  if (FLANG_PARALLEL_COMPILE_JOBS)
+    set_property(TARGET ${test_name}${suffix} PROPERTY JOB_POOL_COMPILE flang_compile_job_pool)
+  endif ()
 
   if (LLVM_LINK_LLVM_DYLIB AND NOT ARG_DISABLE_LLVM_LINK_LLVM_DYLIB)
     set(llvm_libs LLVM)


From flang-commits at lists.llvm.org  Tue Jul 22 09:04:16 2025
From: flang-commits at lists.llvm.org (Eugene Epshteyn via flang-commits)
Date: Tue, 22 Jul 2025 09:04:16 -0700 (PDT)
Subject: [flang-commits] [flang] [not for review] Vol actual checks OLD
 (implements HasTriplet()) (PR #150040)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150040@github.com>
Message-ID: <687fb680.050a0220.21ec03.b30e@mx.google.com>


https://github.com/eugeneepshteyn closed https://github.com/llvm/llvm-project/pull/150040

From flang-commits at lists.llvm.org  Tue Jul 22 09:43:50 2025
From: flang-commits at lists.llvm.org (Delaram Talaashrafi via flang-commits)
Date: Tue, 22 Jul 2025 09:43:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Extend symbol update to
 ArrayAttrext in `external-name-interop` (PR #150061)
Message-ID: <llvm.org/llvm/llvm-project/pull/150061@github.com>


https://github.com/delaram-talaashrafi created https://github.com/llvm/llvm-project/pull/150061

In the `external-name-interop` pass, when a symbol is changed, all the uses of the renamed symbols should also be updated. The update was only applied to the `SymbolRefAttr` type. With this change, the update will be applied to `ArrayAttr` containing elements of type `SymbolRefAttr`.

>From 6682a157bd206bb63e95cfa85874422ee2be1bba Mon Sep 17 00:00:00 2001
From: Delaram Talaashrafi <dtalaashrafi at rome5.pgi.net>
Date: Tue, 22 Jul 2025 09:38:27 -0700
Subject: [PATCH] [flang] Extend symbol update to ArrayAttrext in
 external-name-interop

In the `external-name-interop` pass, when a symbol is changed, all the
uses of the renamed symbols should also be updated. The update was only
applied to the `SymbolRefAttr` type. With this change, the update will
be applied to `ArrayAttr` containing elements of type `SymbolRefAttr`.
---
 .../Transforms/ExternalNameConversion.cpp     | 58 +++++++++++++++----
 1 file changed, 47 insertions(+), 11 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
index 3d84eaa4c1595..2fcff87fdc393 100644
--- a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
@@ -41,6 +41,23 @@ mangleExternalName(const std::pair<fir::NameUniquer::NameKind,
                                                   appendUnderscore);
 }
 
+/// Process a symbol reference and return the updated symbol reference if
+/// needed.
+std::optional<mlir::SymbolRefAttr>
+processSymbolRef(mlir::SymbolRefAttr symRef, mlir::Operation *nestedOp,
+                 const llvm::DenseMap<mlir::StringAttr, mlir::FlatSymbolRefAttr>
+                     &remappings) {
+  if (auto remap = remappings.find(symRef.getLeafReference());
+      remap != remappings.end()) {
+    mlir::SymbolRefAttr symAttr = mlir::FlatSymbolRefAttr(remap->second);
+    if (mlir::isa<mlir::gpu::LaunchFuncOp>(nestedOp))
+      symAttr = mlir::SymbolRefAttr::get(
+          symRef.getRootReference(), {mlir::FlatSymbolRefAttr(remap->second)});
+    return symAttr;
+  }
+  return std::nullopt;
+}
+
 namespace {
 
 class ExternalNameConversionPass
@@ -97,21 +114,40 @@ void ExternalNameConversionPass::runOnOperation() {
 
   // Update all uses of the functions and globals that have been renamed.
   op.walk([&remappings](mlir::Operation *nestedOp) {
-    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::SymbolRefAttr>> updates;
+    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::SymbolRefAttr>>
+        symRefUpdates;
+    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::ArrayAttr>>
+        arrayUpdates;
     for (const mlir::NamedAttribute &attr : nestedOp->getAttrDictionary())
       if (auto symRef = llvm::dyn_cast<mlir::SymbolRefAttr>(attr.getValue())) {
-        if (auto remap = remappings.find(symRef.getLeafReference());
-            remap != remappings.end()) {
-          mlir::SymbolRefAttr symAttr = mlir::FlatSymbolRefAttr(remap->second);
-          if (mlir::isa<mlir::gpu::LaunchFuncOp>(nestedOp))
-            symAttr = mlir::SymbolRefAttr::get(
-                symRef.getRootReference(),
-                {mlir::FlatSymbolRefAttr(remap->second)});
-          updates.emplace_back(std::pair<mlir::StringAttr, mlir::SymbolRefAttr>{
-              attr.getName(), symAttr});
+        if (auto newSymRef = processSymbolRef(symRef, nestedOp, remappings))
+          symRefUpdates.emplace_back(
+              std::pair<mlir::StringAttr, mlir::SymbolRefAttr>{attr.getName(),
+                                                               *newSymRef});
+      } else if (auto arrayAttr =
+                     llvm::dyn_cast<mlir::ArrayAttr>(attr.getValue())) {
+        llvm::SmallVector<mlir::Attribute> symbolRefs;
+        for (auto element : arrayAttr) {
+          if (!element) {
+            symbolRefs.push_back(element);
+            continue;
+          }
+          auto symRef = llvm::dyn_cast<mlir::SymbolRefAttr>(element);
+          std::optional<mlir::SymbolRefAttr> updatedSymRef;
+          if (symRef)
+            updatedSymRef = processSymbolRef(symRef, nestedOp, remappings);
+          if (!symRef || !updatedSymRef)
+            symbolRefs.push_back(element);
+          else
+            symbolRefs.push_back(*updatedSymRef);
         }
+        arrayUpdates.push_back(std::make_pair(
+            attr.getName(),
+            mlir::ArrayAttr::get(nestedOp->getContext(), symbolRefs)));
       }
-    for (auto update : updates)
+    for (auto update : symRefUpdates)
+      nestedOp->setAttr(update.first, update.second);
+    for (auto update : arrayUpdates)
       nestedOp->setAttr(update.first, update.second);
   });
 }


From flang-commits at lists.llvm.org  Tue Jul 22 09:44:22 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 09:44:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Extend symbol update to
 ArrayAttrext in `external-name-interop` (PR #150061)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150061@github.com>
Message-ID: <687fbfe6.050a0220.31b703.c392@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Delaram Talaashrafi (delaram-talaashrafi)

<details>
<summary>Changes</summary>

In the `external-name-interop` pass, when a symbol is changed, all the uses of the renamed symbols should also be updated. The update was only applied to the `SymbolRefAttr` type. With this change, the update will be applied to `ArrayAttr` containing elements of type `SymbolRefAttr`.

---
Full diff: https://github.com/llvm/llvm-project/pull/150061.diff


1 Files Affected:

- (modified) flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp (+47-11) 


``````````diff
diff --git a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
index 3d84eaa4c1595..2fcff87fdc393 100644
--- a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
@@ -41,6 +41,23 @@ mangleExternalName(const std::pair<fir::NameUniquer::NameKind,
                                                   appendUnderscore);
 }
 
+/// Process a symbol reference and return the updated symbol reference if
+/// needed.
+std::optional<mlir::SymbolRefAttr>
+processSymbolRef(mlir::SymbolRefAttr symRef, mlir::Operation *nestedOp,
+                 const llvm::DenseMap<mlir::StringAttr, mlir::FlatSymbolRefAttr>
+                     &remappings) {
+  if (auto remap = remappings.find(symRef.getLeafReference());
+      remap != remappings.end()) {
+    mlir::SymbolRefAttr symAttr = mlir::FlatSymbolRefAttr(remap->second);
+    if (mlir::isa<mlir::gpu::LaunchFuncOp>(nestedOp))
+      symAttr = mlir::SymbolRefAttr::get(
+          symRef.getRootReference(), {mlir::FlatSymbolRefAttr(remap->second)});
+    return symAttr;
+  }
+  return std::nullopt;
+}
+
 namespace {
 
 class ExternalNameConversionPass
@@ -97,21 +114,40 @@ void ExternalNameConversionPass::runOnOperation() {
 
   // Update all uses of the functions and globals that have been renamed.
   op.walk([&remappings](mlir::Operation *nestedOp) {
-    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::SymbolRefAttr>> updates;
+    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::SymbolRefAttr>>
+        symRefUpdates;
+    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::ArrayAttr>>
+        arrayUpdates;
     for (const mlir::NamedAttribute &attr : nestedOp->getAttrDictionary())
       if (auto symRef = llvm::dyn_cast<mlir::SymbolRefAttr>(attr.getValue())) {
-        if (auto remap = remappings.find(symRef.getLeafReference());
-            remap != remappings.end()) {
-          mlir::SymbolRefAttr symAttr = mlir::FlatSymbolRefAttr(remap->second);
-          if (mlir::isa<mlir::gpu::LaunchFuncOp>(nestedOp))
-            symAttr = mlir::SymbolRefAttr::get(
-                symRef.getRootReference(),
-                {mlir::FlatSymbolRefAttr(remap->second)});
-          updates.emplace_back(std::pair<mlir::StringAttr, mlir::SymbolRefAttr>{
-              attr.getName(), symAttr});
+        if (auto newSymRef = processSymbolRef(symRef, nestedOp, remappings))
+          symRefUpdates.emplace_back(
+              std::pair<mlir::StringAttr, mlir::SymbolRefAttr>{attr.getName(),
+                                                               *newSymRef});
+      } else if (auto arrayAttr =
+                     llvm::dyn_cast<mlir::ArrayAttr>(attr.getValue())) {
+        llvm::SmallVector<mlir::Attribute> symbolRefs;
+        for (auto element : arrayAttr) {
+          if (!element) {
+            symbolRefs.push_back(element);
+            continue;
+          }
+          auto symRef = llvm::dyn_cast<mlir::SymbolRefAttr>(element);
+          std::optional<mlir::SymbolRefAttr> updatedSymRef;
+          if (symRef)
+            updatedSymRef = processSymbolRef(symRef, nestedOp, remappings);
+          if (!symRef || !updatedSymRef)
+            symbolRefs.push_back(element);
+          else
+            symbolRefs.push_back(*updatedSymRef);
         }
+        arrayUpdates.push_back(std::make_pair(
+            attr.getName(),
+            mlir::ArrayAttr::get(nestedOp->getContext(), symbolRefs)));
       }
-    for (auto update : updates)
+    for (auto update : symRefUpdates)
+      nestedOp->setAttr(update.first, update.second);
+    for (auto update : arrayUpdates)
       nestedOp->setAttr(update.first, update.second);
   });
 }

``````````

</details>


https://github.com/llvm/llvm-project/pull/150061

From flang-commits at lists.llvm.org  Tue Jul 22 09:56:29 2025
From: flang-commits at lists.llvm.org (Delaram Talaashrafi via flang-commits)
Date: Tue, 22 Jul 2025 09:56:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Extend symbol update to
 ArrayAttrext in `external-name-interop` (PR #150061)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150061@github.com>
Message-ID: <687fc2bd.170a0220.292399.c845@mx.google.com>


https://github.com/delaram-talaashrafi converted_to_draft https://github.com/llvm/llvm-project/pull/150061

From flang-commits at lists.llvm.org  Tue Jul 22 09:56:50 2025
From: flang-commits at lists.llvm.org (Delaram Talaashrafi via flang-commits)
Date: Tue, 22 Jul 2025 09:56:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Extend symbol update to
 ArrayAttrext in `external-name-interop` (PR #150061)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150061@github.com>
Message-ID: <687fc2d2.050a0220.4898c.bb65@mx.google.com>


https://github.com/delaram-talaashrafi ready_for_review https://github.com/llvm/llvm-project/pull/150061

From flang-commits at lists.llvm.org  Tue Jul 22 10:02:06 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 22 Jul 2025 10:02:06 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687fc40e.050a0220.224104.b69a@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
tblah wrote:

As I understand it, flang-rt is set up to call either glibc or libquadmath depending what is available. If possible it would be good to re-use that mechanism instead of adding further logic for that here. So far as I can tell, the runtime call generator is calling the right runtime function for f128, at least for sind.

But yes I see what you mean about the 64-bit constants used with sind. I think merging this is okay so long as this does get fixed later.

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 10:02:22 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Tue, 22 Jul 2025 10:02:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687fc41e.170a0220.35ded4.d168@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks for the patch!

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 10:17:01 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Tue, 22 Jul 2025 10:17:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687fc78d.170a0220.13864f.d0d1@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
c8ef wrote:

Yes, handling these is somewhat tricky. I'm not exactly sure how to handle this correctly in flang. I am currently working on adding sinpi-related functions to libquadmath in the GCC source tree. The GCC maintainers are not satisfied with the precision of the generic implementation imported from glibc, so this patch has not been merged yet.

C23 has added support for sinpi-related functions, and eventually libc will also have support. This will allow us to achieve better precision. If neither glibc nor libquadmath have it, we can fallback to our current approach. 

However, glibc does not provide sind-related functions. We just need to fix the constant pi type later.

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 10:18:01 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Tue, 22 Jul 2025 10:18:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Extend symbol update to
 ArrayAttrext in `external-name-interop` (PR #150061)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150061@github.com>
Message-ID: <687fc7c9.050a0220.11b2d4.c561@mx.google.com>


vzakhari wrote:

Thank you, Delaram! The change looks good to me, but please try to create a LIT test for this.

https://github.com/llvm/llvm-project/pull/150061

From flang-commits at lists.llvm.org  Tue Jul 22 10:28:30 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Tue, 22 Jul 2025 10:28:30 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687fca3e.170a0220.2555ad.c274@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
vzakhari wrote:

I think you may follow the implementation of `SinF128` as Tom suggested.  As long as `libquadmath` does not support `sinpi` for `float128` you may implement `SinpiF128` in `flang-rt/lib/quadmath` same way you implement it here but using proper `PI` definition from https://gcc.gnu.org/onlinedocs/libquadmath/Typedef-and-constants.html

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 10:28:35 2025
From: flang-commits at lists.llvm.org (LLVM Continuous Integration via flang-commits)
Date: Tue, 22 Jul 2025 10:28:35 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <687fca43.170a0220.d296b.ef26@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `ppc64le-flang-rhel-clang` running on `ppc64le-flang-rhel-test` while building `flang,llvm` at step 5 "build-unified-tree".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/157/builds/34563

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 5 (build-unified-tree) failure: build (failure)
...
112.275 [53/87/6805] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Coarray.cpp.o
112.354 [53/86/6806] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/formatting.cpp.o
114.662 [53/85/6807] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/dump-expr.cpp.o
115.358 [53/84/6808] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-atomic.cpp.o
115.723 [53/83/6809] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/scope.cpp.o
116.078 [53/82/6810] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics-library.cpp.o
117.758 [53/81/6811] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-declarations.cpp.o
119.394 [53/80/6812] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-select-rank.cpp.o
119.684 [53/79/6813] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-io.cpp.o
120.483 [53/78/6814] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o
FAILED: tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o 
ccache /home/buildbots/llvm-external-buildbots/clang.19.1.7/bin/clang++ -DEXPERIMENTAL_KEY_INSTRUCTIONS -DFLANG_INCLUDE_TESTS=1 -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Semantics -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Semantics -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/include -I/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/llvm/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/../mlir/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/mlir/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/clang/include -isystem /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/llvm/../clang/include -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wc++98-compat-extra-semi -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Werror -Wno-deprecated-copy -Wno-string-conversion -Wno-ctad-maybe-unsupported -Wno-unused-command-line-argument -Wstring-conversion           -Wcovered-switch-default -Wno-nested-anon-types -Xclang -fno-pch-timestamp -O3 -DNDEBUG -std=c++17  -fno-exceptions -funwind-tables -fno-rtti -UNDEBUG -Winvalid-pch -Xclang -include-pch -Xclang /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/cmake_pch.hxx.pch -Xclang -include -Xclang /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/build/tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/cmake_pch.hxx -MD -MT tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o -MF tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o.d -o tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/resolve-directives.cpp.o -c /home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Semantics/resolve-directives.cpp
/home/buildbots/llvm-external-buildbots/workers/ppc64le-flang-rhel-test/ppc64le-flang-rhel-clang-build/llvm-project/flang/lib/Semantics/resolve-directives.cpp:753:7: error: default label in switch which covers all enumeration values [-Werror,-Wcovered-switch-default]
  753 |       default:
      |       ^
1 error generated.
120.806 [53/77/6815] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/initial-image.cpp.o
122.158 [53/76/6816] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/fold-designator.cpp.o
122.274 [53/75/6817] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/SymbolMap.cpp.o
123.309 [53/74/6818] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/expression.cpp.o
125.995 [53/73/6819] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/shape.cpp.o
126.232 [53/72/6820] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IterationSpace.cpp.o
126.341 [53/71/6821] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ComponentPath.cpp.o
128.756 [53/70/6822] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Runtime.cpp.o
129.661 [53/69/6823] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/ReductionProcessor.cpp.o
130.008 [53/68/6824] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-case.cpp.o
130.359 [53/67/6825] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Support/PrivateReductionUtils.cpp.o
130.828 [53/66/6826] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/canonicalize-do.cpp.o
130.850 [53/65/6827] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/definable.cpp.o
132.571 [53/64/6828] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/variable.cpp.o
134.826 [53/63/6829] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/intrinsics.cpp.o
135.233 [53/62/6830] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/tools.cpp.o
136.204 [53/61/6831] Building CXX object tools/flang/lib/Evaluate/CMakeFiles/FortranEvaluate.dir/characteristics.cpp.o
137.358 [53/60/6832] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CustomIntrinsicCall.cpp.o
139.989 [53/59/6833] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-omp-loop.cpp.o
140.476 [53/58/6834] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/check-call.cpp.o
143.450 [53/57/6835] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/symbol.cpp.o
144.291 [53/56/6836] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/IO.cpp.o
144.917 [53/55/6837] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertProcedureDesignator.cpp.o
146.198 [53/54/6838] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/openmp-utils.cpp.o
146.297 [53/53/6839] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/ConvertType.cpp.o
146.335 [53/52/6840] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/pointer-assignment.cpp.o
150.000 [53/51/6841] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/rewrite-parse-tree.cpp.o
150.292 [53/50/6842] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/canonicalize-omp.cpp.o
150.441 [53/49/6843] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/CallInterface.cpp.o
150.753 [53/48/6844] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/Mangler.cpp.o
151.014 [53/47/6845] Building CXX object tools/flang/tools/bbc/CMakeFiles/bbc.dir/bbc.cpp.o
151.918 [53/46/6846] Building CXX object tools/flang/lib/Semantics/CMakeFiles/FortranSemantics.dir/type.cpp.o
152.312 [53/45/6847] Building CXX object tools/flang/lib/Lower/CMakeFiles/FortranLower.dir/HlfirIntrinsics.cpp.o

```

</details>

https://github.com/llvm/llvm-project/pull/149137

From flang-commits at lists.llvm.org  Tue Jul 22 10:35:28 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Tue, 22 Jul 2025 10:35:28 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687fcbe0.170a0220.d296b.f000@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
c8ef wrote:

I think this is a good approach for handling the f128 case. However, if we implement this logic in libquadmath, will it diverge from how we handle f32/f64 here?

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 10:38:35 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 10:38:35 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Assign unittests to flang_compile
	pool (PR #150041)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150041@github.com>
Message-ID: <687fcc9b.050a0220.20336f.e883@mx.google.com>


https://github.com/Meinersbur ready_for_review https://github.com/llvm/llvm-project/pull/150041

From flang-commits at lists.llvm.org  Tue Jul 22 10:47:02 2025
From: flang-commits at lists.llvm.org (Slava Zakharin via flang-commits)
Date: Tue, 22 Jul 2025 10:47:02 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <687fce96.a70a0220.2047af.d431@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
vzakhari wrote:

Diverge in what sense?

If we "lower" the f32/f64 cases here as `sin(PI * x)`, and the f128 case in Flang's `quadmath` as `sinq(M_PIq * x)`, then they should be pretty consistent, right?

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 11:31:45 2025
From: flang-commits at lists.llvm.org (Carlos Seo via flang-commits)
Date: Tue, 22 Jul 2025 11:31:45 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix a crash when equivalence and
 namelist statements are used (PR #150081)
Message-ID: <llvm.org/llvm/llvm-project/pull/150081@github.com>


https://github.com/ceseo created https://github.com/llvm/llvm-project/pull/150081

Check for equivalence when generating namelist descriptors in IO.cpp.

>From ca88815b1dd40aa41ecdbd973d4eea5e8976ef4f Mon Sep 17 00:00:00 2001
From: Carlos Seo <carlos.seo at linaro.org>
Date: Tue, 22 Jul 2025 18:24:41 +0000
Subject: [PATCH] [Flang] Fix a crash when equivalence and namelist statements
 are used

Check for equivalence when generating namelist descriptors in IO.cpp.
---
 flang/lib/Lower/IO.cpp           |  4 +++-
 flang/test/Lower/equivalence-3.f | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Lower/equivalence-3.f

diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 53bf61922392d..087617ae86d3f 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -466,8 +466,10 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       fir::BoxType boxTy =
           fir::BoxType::get(fir::PointerType::get(converter.genType(s)));
       auto descFunc = [&](fir::FirOpBuilder &b) {
+        bool couldBeInEquivalence =
+            Fortran::semantics::FindEquivalenceSet(s) != nullptr;
         auto box = Fortran::lower::genInitialDataTarget(
-            converter, loc, boxTy, *expr, /*couldBeInEquivalence=*/true);
+            converter, loc, boxTy, *expr, couldBeInEquivalence);
         b.create<fir::HasValueOp>(loc, box);
       };
       builder.createGlobalConstant(loc, boxTy, mangleName, descFunc, linkOnce);
diff --git a/flang/test/Lower/equivalence-3.f b/flang/test/Lower/equivalence-3.f
new file mode 100644
index 0000000000000..19f8880189f0c
--- /dev/null
+++ b/flang/test/Lower/equivalence-3.f
@@ -0,0 +1,19 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+  ! CHECK-LABEL: func @_QQmain
+  program main
+  real a1,a2
+  equivalence (a1,a2)
+  ! A fir.alloca should never appear in a global constant initialization.
+  ! CHECK: fir.global linkonce @_QFEx1.desc constant : !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  ! CHECK: arith.constant 5 : index
+  ! CHECK:fir.address_of(@_QFEx1) : !fir.ref<!fir.array<5xf64>>
+  ! CHECK: fir.shape %c5 : (index) -> !fir.shape<1>
+  ! CHECK: fir.declare %0(%1) {uniq_name = "_QFEx1"} : (!fir.ref<!fir.array<5xf64>>, !fir.shape<1>) -> !fir.ref<!fir.array<5xf64>>
+  ! CHECK: fir.embox %2(%1) : (!fir.ref<!fir.array<5xf64>>, !fir.shape<1>) -> !fir.box<!fir.array<5xf64>>
+  ! CHECK: fir.rebox %3 : (!fir.box<!fir.array<5xf64>>) -> !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  ! CHECK: fir.has_value %4 : !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  real*8 x1(5)
+  namelist /y1/x1
+  read (5,y1)
+  end


From flang-commits at lists.llvm.org  Tue Jul 22 11:32:21 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Tue, 22 Jul 2025 11:32:21 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix a crash when equivalence and
 namelist statements are used (PR #150081)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150081@github.com>
Message-ID: <687fd935.170a0220.1e8534.aaf3@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-fir-hlfir

Author: Carlos Seo (ceseo)

<details>
<summary>Changes</summary>

Check for equivalence when generating namelist descriptors in IO.cpp.

---
Full diff: https://github.com/llvm/llvm-project/pull/150081.diff


2 Files Affected:

- (modified) flang/lib/Lower/IO.cpp (+3-1) 
- (added) flang/test/Lower/equivalence-3.f (+19) 


``````````diff
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 53bf61922392d..087617ae86d3f 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -466,8 +466,10 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       fir::BoxType boxTy =
           fir::BoxType::get(fir::PointerType::get(converter.genType(s)));
       auto descFunc = [&](fir::FirOpBuilder &b) {
+        bool couldBeInEquivalence =
+            Fortran::semantics::FindEquivalenceSet(s) != nullptr;
         auto box = Fortran::lower::genInitialDataTarget(
-            converter, loc, boxTy, *expr, /*couldBeInEquivalence=*/true);
+            converter, loc, boxTy, *expr, couldBeInEquivalence);
         b.create<fir::HasValueOp>(loc, box);
       };
       builder.createGlobalConstant(loc, boxTy, mangleName, descFunc, linkOnce);
diff --git a/flang/test/Lower/equivalence-3.f b/flang/test/Lower/equivalence-3.f
new file mode 100644
index 0000000000000..19f8880189f0c
--- /dev/null
+++ b/flang/test/Lower/equivalence-3.f
@@ -0,0 +1,19 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+  ! CHECK-LABEL: func @_QQmain
+  program main
+  real a1,a2
+  equivalence (a1,a2)
+  ! A fir.alloca should never appear in a global constant initialization.
+  ! CHECK: fir.global linkonce @_QFEx1.desc constant : !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  ! CHECK: arith.constant 5 : index
+  ! CHECK:fir.address_of(@_QFEx1) : !fir.ref<!fir.array<5xf64>>
+  ! CHECK: fir.shape %c5 : (index) -> !fir.shape<1>
+  ! CHECK: fir.declare %0(%1) {uniq_name = "_QFEx1"} : (!fir.ref<!fir.array<5xf64>>, !fir.shape<1>) -> !fir.ref<!fir.array<5xf64>>
+  ! CHECK: fir.embox %2(%1) : (!fir.ref<!fir.array<5xf64>>, !fir.shape<1>) -> !fir.box<!fir.array<5xf64>>
+  ! CHECK: fir.rebox %3 : (!fir.box<!fir.array<5xf64>>) -> !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  ! CHECK: fir.has_value %4 : !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  real*8 x1(5)
+  namelist /y1/x1
+  read (5,y1)
+  end

``````````

</details>


https://github.com/llvm/llvm-project/pull/150081

From flang-commits at lists.llvm.org  Tue Jul 22 14:51:36 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Tue, 22 Jul 2025 14:51:36 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
Message-ID: <llvm.org/llvm/llvm-project/pull/150121@github.com>


https://github.com/kparzysz created https://github.com/llvm/llvm-project/pull/150121

Fortran::parser::omp::GetOmpDirectiveName(t) will get the OmpDirectiveName object that corresponds to construct t. That object (an AST node) contains the enum id and the source information of the directive.

Replace uses of extractOmpDirective and getOpenMPDirectiveEnum with the new function.

>From 14072b0560cde74fda94a1f434b6cfd43b77939b Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 15:37:11 -0500
Subject: [PATCH] [flang][OpenMP] General utility to get directive id from AST
 node

Fortran::parser::omp::GetOmpDirectiveName(t) will get the OmpDirectiveName
object that corresponds to construct t. That object (an AST node) contains
the enum id and the source information of the directive.

Replace uses of extractOmpDirective and getOpenMPDirectiveEnum with the
new function.
---
 flang/include/flang/Parser/openmp-utils.h     | 175 ++++++++++++++++++
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp |   4 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  35 +---
 flang/lib/Lower/OpenMP/Utils.cpp              |  84 +--------
 flang/lib/Lower/OpenMP/Utils.h                |   2 -
 5 files changed, 187 insertions(+), 113 deletions(-)
 create mode 100644 flang/include/flang/Parser/openmp-utils.h

diff --git a/flang/include/flang/Parser/openmp-utils.h b/flang/include/flang/Parser/openmp-utils.h
new file mode 100644
index 0000000000000..363db852416f6
--- /dev/null
+++ b/flang/include/flang/Parser/openmp-utils.h
@@ -0,0 +1,175 @@
+//===-- flang/Parser/openmp-utils.h ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common OpenMP utilities.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_PARSER_OPENMP_UTILS_H
+#define FORTRAN_PARSER_OPENMP_UTILS_H
+
+#include "flang/Common/indirection.h"
+#include "flang/Parser/parse-tree.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cassert>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+namespace Fortran::parser::omp {
+
+namespace detail {
+using D = llvm::omp::Directive;
+
+template <typename Construct> //
+struct ConstructId {
+  static constexpr llvm::omp::Directive id{D::OMPD_unknown};
+};
+
+#define MAKE_CONSTR_ID(Construct, Id) \
+  template <> struct ConstructId<Construct> { \
+    static constexpr llvm::omp::Directive id{Id}; \
+  }
+
+MAKE_CONSTR_ID(OmpAssumeDirective, D::OMPD_assume);
+MAKE_CONSTR_ID(OmpCriticalDirective, D::OMPD_critical);
+MAKE_CONSTR_ID(OmpDeclareVariantDirective, D::OMPD_declare_variant);
+MAKE_CONSTR_ID(OmpErrorDirective, D::OMPD_error);
+MAKE_CONSTR_ID(OmpMetadirectiveDirective, D::OMPD_metadirective);
+MAKE_CONSTR_ID(OpenMPDeclarativeAllocate, D::OMPD_allocate);
+MAKE_CONSTR_ID(OpenMPDeclarativeAssumes, D::OMPD_assumes);
+MAKE_CONSTR_ID(OpenMPDeclareMapperConstruct, D::OMPD_declare_mapper);
+MAKE_CONSTR_ID(OpenMPDeclareReductionConstruct, D::OMPD_declare_reduction);
+MAKE_CONSTR_ID(OpenMPDeclareSimdConstruct, D::OMPD_declare_simd);
+MAKE_CONSTR_ID(OpenMPDeclareTargetConstruct, D::OMPD_declare_target);
+MAKE_CONSTR_ID(OpenMPExecutableAllocate, D::OMPD_allocate);
+MAKE_CONSTR_ID(OpenMPRequiresConstruct, D::OMPD_requires);
+MAKE_CONSTR_ID(OpenMPThreadprivate, D::OMPD_threadprivate);
+
+#undef MAKE_CONSTR_ID
+
+struct DirectiveNameScope {
+  // Helper types to make overloaded function signatures different.
+  struct TagA {};
+  struct TagB {};
+  struct TagC {};
+  struct TagD {};
+
+  static OmpDirectiveName MakeName(CharBlock source = {},
+      llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown) {
+    OmpDirectiveName name;
+    name.source = source;
+    name.v = id;
+    return name;
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpNothingDirective &x) {
+    return MakeName(x.source, llvm::omp::Directive::OMPD_nothing);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpBeginBlockDirective &x) {
+    auto &dir{std::get<OmpBlockDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpBeginLoopDirective &x) {
+    auto &dir{std::get<OmpLoopDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(
+      const OmpBeginSectionsDirective &x) {
+    auto &dir{std::get<OmpSectionsDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  template <typename T, typename = std::enable_if_t<WrapperTrait<T>>>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagA = {}) {
+    if constexpr (std::is_same_v<T, OpenMPCancelConstruct> ||
+        std::is_same_v<T, OpenMPCancellationPointConstruct> ||
+        std::is_same_v<T, OpenMPDepobjConstruct> ||
+        std::is_same_v<T, OpenMPFlushConstruct> ||
+        std::is_same_v<T, OpenMPInteropConstruct> ||
+        std::is_same_v<T, OpenMPSimpleStandaloneConstruct>) {
+      return x.v.DirName();
+    } else {
+      return GetOmpDirectiveName(x.v);
+    }
+  }
+
+  template <typename T, typename = std::enable_if_t<TupleTrait<T>>>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagB = {}) {
+    if constexpr (std::is_same_v<T, OpenMPAllocatorsConstruct> ||
+        std::is_same_v<T, OpenMPAtomicConstruct> ||
+        std::is_same_v<T, OpenMPDispatchConstruct>) {
+      return std::get<OmpDirectiveSpecification>(x.t).DirName();
+    } else if constexpr (std::is_same_v<T, OmpAssumeDirective> ||
+        std::is_same_v<T, OmpCriticalDirective> ||
+        std::is_same_v<T, OmpDeclareVariantDirective> ||
+        std::is_same_v<T, OmpErrorDirective> ||
+        std::is_same_v<T, OmpMetadirectiveDirective> ||
+        std::is_same_v<T, OpenMPDeclarativeAllocate> ||
+        std::is_same_v<T, OpenMPDeclarativeAssumes> ||
+        std::is_same_v<T, OpenMPDeclareMapperConstruct> ||
+        std::is_same_v<T, OpenMPDeclareReductionConstruct> ||
+        std::is_same_v<T, OpenMPDeclareSimdConstruct> ||
+        std::is_same_v<T, OpenMPDeclareTargetConstruct> ||
+        std::is_same_v<T, OpenMPExecutableAllocate> ||
+        std::is_same_v<T, OpenMPRequiresConstruct> ||
+        std::is_same_v<T, OpenMPThreadprivate>) {
+      return MakeName(std::get<Verbatim>(x.t).source, ConstructId<T>::id);
+    } else {
+      return GetFromTuple(
+          x.t, std::make_index_sequence<std::tuple_size_v<decltype(x.t)>>{});
+    }
+  }
+
+  template <typename T, typename = std::enable_if_t<UnionTrait<T>>>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagC = {}) {
+    return common::visit([](auto &&s) { return GetOmpDirectiveName(s); }, x.u);
+  }
+
+  template <typename... Ts, size_t... Is>
+  static OmpDirectiveName GetFromTuple(
+      const std::tuple<Ts...> &t, std::index_sequence<Is...>) {
+    OmpDirectiveName name = MakeName();
+    auto accumulate = [&](const OmpDirectiveName &n) {
+      if (name.v == llvm::omp::Directive::OMPD_unknown) {
+        name = n;
+      } else {
+        assert(
+            n.v == llvm::omp::Directive::OMPD_unknown && "Conflicting names");
+      }
+    };
+    (accumulate(GetOmpDirectiveName(std::get<Is>(t))), ...);
+    return name;
+  }
+
+  template <typename T>
+  static OmpDirectiveName GetOmpDirectiveName(const common::Indirection<T> &x) {
+    return GetOmpDirectiveName(x.value());
+  }
+
+  template <typename T,
+      typename = std::enable_if_t<!WrapperTrait<T> && !TupleTrait<T> &&
+          !UnionTrait<T>>>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagD = {}) {
+    return MakeName();
+  }
+};
+} // namespace detail
+
+template <typename T> OmpDirectiveName GetOmpDirectiveName(const T &x) {
+  return detail::DirectiveNameScope::GetOmpDirectiveName(x);
+}
+
+} // namespace Fortran::parser::omp
+
+#endif // FORTRAN_PARSER_OPENMP_UTILS_H
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 11e488371b886..2ac4d9548b65b 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -24,6 +24,7 @@
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Parser/openmp-utils.h"
 #include "flang/Semantics/attr.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/ADT/Sequence.h"
@@ -465,7 +466,8 @@ bool DataSharingProcessor::isOpenMPPrivatizingConstruct(
   // allow a privatizing clause) are: dispatch, distribute, do, for, loop,
   // parallel, scope, sections, simd, single, target, target_data, task,
   // taskgroup, taskloop, and teams.
-  return llvm::is_contained(privatizing, extractOmpDirective(omp));
+  return llvm::is_contained(privatizing,
+                            parser::omp::GetOmpDirectiveName(omp).v);
 }
 
 bool DataSharingProcessor::isOpenMPPrivatizingEvaluation(
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index fc5fef9b2c577..4c2d7badef382 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -31,6 +31,7 @@
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Parser/characters.h"
+#include "flang/Parser/openmp-utils.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/tools.h"
@@ -63,28 +64,6 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
                                    lower::pft::Evaluation &eval,
                                    mlir::Location loc);
 
-static llvm::omp::Directive
-getOpenMPDirectiveEnum(const parser::OmpLoopDirective &beginStatment) {
-  return beginStatment.v;
-}
-
-static llvm::omp::Directive getOpenMPDirectiveEnum(
-    const parser::OmpBeginLoopDirective &beginLoopDirective) {
-  return getOpenMPDirectiveEnum(
-      std::get<parser::OmpLoopDirective>(beginLoopDirective.t));
-}
-
-static llvm::omp::Directive
-getOpenMPDirectiveEnum(const parser::OpenMPLoopConstruct &ompLoopConstruct) {
-  return getOpenMPDirectiveEnum(
-      std::get<parser::OmpBeginLoopDirective>(ompLoopConstruct.t));
-}
-
-static llvm::omp::Directive getOpenMPDirectiveEnum(
-    const common::Indirection<parser::OpenMPLoopConstruct> &ompLoopConstruct) {
-  return getOpenMPDirectiveEnum(ompLoopConstruct.value());
-}
-
 namespace {
 /// Structure holding information that is needed to pass host-evaluated
 /// information to later lowering stages.
@@ -468,7 +447,7 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
     llvm::omp::Directive dir;
     auto &nested = parent.getFirstNestedEvaluation();
     if (const auto *ompEval = nested.getIf<parser::OpenMPConstruct>())
-      dir = extractOmpDirective(*ompEval);
+      dir = parser::omp::GetOmpDirectiveName(*ompEval).v;
     else
       return std::nullopt;
 
@@ -508,7 +487,7 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
     HostEvalInfo *hostInfo = getHostEvalInfoStackTop(converter);
     assert(hostInfo && "expected HOST_EVAL info structure");
 
-    switch (extractOmpDirective(*ompEval)) {
+    switch (parser::omp::GetOmpDirectiveName(*ompEval).v) {
     case OMPD_teams_distribute_parallel_do:
     case OMPD_teams_distribute_parallel_do_simd:
       cp.processThreadLimit(stmtCtx, hostInfo->ops);
@@ -569,7 +548,8 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
 
   const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
   assert(ompEval &&
-         llvm::omp::allTargetSet.test(extractOmpDirective(*ompEval)) &&
+         llvm::omp::allTargetSet.test(
+             parser::omp::GetOmpDirectiveName(*ompEval).v) &&
          "expected TARGET construct evaluation");
   (void)ompEval;
 
@@ -3872,7 +3852,7 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
             std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
                 &*optLoopCons)}) {
       llvm::omp::Directive nestedDirective =
-          getOpenMPDirectiveEnum(*ompNestedLoopCons);
+          parser::omp::GetOmpDirectiveName(*ompNestedLoopCons).v;
       switch (nestedDirective) {
       case llvm::omp::Directive::OMPD_tile:
         // Emit the omp.loop_nest with annotation for tiling
@@ -3889,7 +3869,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
     }
   }
 
-  llvm::omp::Directive directive = getOpenMPDirectiveEnum(beginLoopDirective);
+  llvm::omp::Directive directive =
+      parser::omp::GetOmpDirectiveName(beginLoopDirective).v;
   const parser::CharBlock &source =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
   ConstructQueue queue{
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index b1716d6afb200..13fda978c5369 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -20,6 +20,7 @@
 #include <flang/Lower/PFTBuilder.h>
 #include <flang/Optimizer/Builder/FIRBuilder.h>
 #include <flang/Optimizer/Builder/Todo.h>
+#include <flang/Parser/openmp-utils.h>
 #include <flang/Parser/parse-tree.h>
 #include <flang/Parser/tools.h>
 #include <flang/Semantics/tools.h>
@@ -663,89 +664,6 @@ bool collectLoopRelatedInfo(
   return found;
 }
 
-/// Get the directive enumeration value corresponding to the given OpenMP
-/// construct PFT node.
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
-  return common::visit(
-      common::visitors{
-          [](const parser::OpenMPAllocatorsConstruct &c) {
-            return llvm::omp::OMPD_allocators;
-          },
-          [](const parser::OpenMPAssumeConstruct &c) {
-            return llvm::omp::OMPD_assume;
-          },
-          [](const parser::OpenMPAtomicConstruct &c) {
-            return llvm::omp::OMPD_atomic;
-          },
-          [](const parser::OpenMPBlockConstruct &c) {
-            return std::get<parser::OmpBlockDirective>(
-                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPCriticalConstruct &c) {
-            return llvm::omp::OMPD_critical;
-          },
-          [](const parser::OpenMPDeclarativeAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPDispatchConstruct &c) {
-            return llvm::omp::OMPD_dispatch;
-          },
-          [](const parser::OpenMPExecutableAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPLoopConstruct &c) {
-            return std::get<parser::OmpLoopDirective>(
-                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPSectionConstruct &c) {
-            return llvm::omp::OMPD_section;
-          },
-          [](const parser::OpenMPSectionsConstruct &c) {
-            return std::get<parser::OmpSectionsDirective>(
-                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPStandaloneConstruct &c) {
-            return common::visit(
-                common::visitors{
-                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
-                      return c.v.DirId();
-                    },
-                    [](const parser::OpenMPFlushConstruct &c) {
-                      return llvm::omp::OMPD_flush;
-                    },
-                    [](const parser::OpenMPCancelConstruct &c) {
-                      return llvm::omp::OMPD_cancel;
-                    },
-                    [](const parser::OpenMPCancellationPointConstruct &c) {
-                      return llvm::omp::OMPD_cancellation_point;
-                    },
-                    [](const parser::OmpMetadirectiveDirective &c) {
-                      return llvm::omp::OMPD_metadirective;
-                    },
-                    [](const parser::OpenMPDepobjConstruct &c) {
-                      return llvm::omp::OMPD_depobj;
-                    },
-                    [](const parser::OpenMPInteropConstruct &c) {
-                      return llvm::omp::OMPD_interop;
-                    }},
-                c.u);
-          },
-          [](const parser::OpenMPUtilityConstruct &c) {
-            return common::visit(
-                common::visitors{[](const parser::OmpErrorDirective &c) {
-                                   return llvm::omp::OMPD_error;
-                                 },
-                                 [](const parser::OmpNothingDirective &c) {
-                                   return llvm::omp::OMPD_nothing;
-                                 }},
-                c.u);
-          }},
-      ompConstruct.u);
-}
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 8e3ad5c3452e2..11641ba5e8606 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -167,8 +167,6 @@ bool collectLoopRelatedInfo(
     mlir::omp::LoopRelatedClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &iv);
 
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct);
 } // namespace omp
 } // namespace lower
 } // namespace Fortran


From flang-commits at lists.llvm.org  Tue Jul 22 16:50:07 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Tue, 22 Jul 2025 16:50:07 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [acc][flang] Add recipes to data
	entry operations (PR #149210)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149210@github.com>
Message-ID: <688023af.170a0220.fa97.e623@mx.google.com>


https://github.com/razvanlupusoru edited https://github.com/llvm/llvm-project/pull/149210

From flang-commits at lists.llvm.org  Tue Jul 22 17:20:46 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Tue, 22 Jul 2025 17:20:46 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `sinpi` (PR #149525)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149525@github.com>
Message-ID: <68802ade.170a0220.30a0f5.e7c1@mx.google.com>


================
@@ -8060,6 +8061,21 @@ mlir::Value IntrinsicLibrary::genSind(mlir::Type resultType,
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
 
+// SINPI
+mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
----------------
c8ef wrote:

Ah, you're right. I get it. That seems like a good approach.

https://github.com/llvm/llvm-project/pull/149525

From flang-commits at lists.llvm.org  Tue Jul 22 18:50:33 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Tue, 22 Jul 2025 18:50:33 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <68803fe9.170a0220.15b9df.f0c3@mx.google.com>


https://github.com/terapines-osc-mlir updated https://github.com/llvm/llvm-project/pull/149959

>From 3f548d16dd3907ac00b7edb2cb41ce5fdde70597 Mon Sep 17 00:00:00 2001
From: Terapines MLIR <osc-mlir at terapines.com>
Date: Tue, 22 Jul 2025 11:09:20 +0800
Subject: [PATCH] [flang][fir]: Add conversion of `fir.if` to `scf.if`.

---
 flang/lib/Optimizer/Transforms/FIRToSCF.cpp | 43 +++++++++++++++-
 flang/test/Fir/FirToSCF/if.fir              | 57 +++++++++++++++++++++
 2 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Fir/FirToSCF/if.fir

diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index d7d1865bc56ba..f3198329028e9 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                  Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = cast<fir::ResultOp>(srcTerminator);
+
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    rewriter.create<scf::YieldOp>(resultOp.getLoc(), resultOp.getOperands());
+  }
+
+  rewriter.eraseOp(srcTerminator);
+}
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    mlir::Location loc = ifOp.getLoc();
+    bool hasElse = !ifOp.getElseRegion().empty();
+    auto scfIfOp = rewriter.create<scf::IfOp>(loc, ifOp->getResultTypes(),
+                                              ifOp.getCondition(), hasElse);
+
+    copyBlocksAndTransformResult(rewriter, ifOp.getThenRegion().front(),
+                                 scfIfOp.getThenRegion().front());
+
+    if (hasElse) {
+      copyBlocksAndTransformResult(rewriter, ifOp.getElseRegion().front(),
+                                   scfIfOp.getElseRegion().front());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/flang/test/Fir/FirToSCF/if.fir b/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 0000000000000..03be264c4cdf5
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,57 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_only(
+// CHECK-SAME:      %[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:           scf.if %[[ARG0]] {
+// CHECK:             %[[VAL_1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_else()   {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           scf.if %[[VAL_1]] {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           } else {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2]] -> (f32, f32) {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           } else {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}


From flang-commits at lists.llvm.org  Tue Jul 22 19:48:50 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Tue, 22 Jul 2025 19:48:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <68804d92.170a0220.c84d.1348@mx.google.com>


================
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlocksAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
----------------
terapines-osc-mlir wrote:

> I think this should be a lambda instead

Copying block and transforming terminator is a common operation which can be shared by other functions in this file.  Since all Fir to SCF transforms will be written in this file, I would like to keep this function this way for now.

https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Tue Jul 22 19:49:21 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Tue, 22 Jul 2025 19:49:21 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <68804db1.050a0220.f8142.e2e6@mx.google.com>


https://github.com/terapines-osc-mlir updated https://github.com/llvm/llvm-project/pull/149959

>From 3e73c009e2904f054710ad38403f52e53d1ec57b Mon Sep 17 00:00:00 2001
From: Terapines MLIR <osc-mlir at terapines.com>
Date: Tue, 22 Jul 2025 11:09:20 +0800
Subject: [PATCH] [flang][fir]: Add conversion of `fir.if` to `scf.if`.

---
 flang/lib/Optimizer/Transforms/FIRToSCF.cpp | 43 +++++++++++++++-
 flang/test/Fir/FirToSCF/if.fir              | 57 +++++++++++++++++++++
 2 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Fir/FirToSCF/if.fir

diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index d7d1865bc56ba..1902757e83bf3 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,52 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+void copyBlockAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
+                                 Block &dstBlock) {
+  Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = cast<fir::ResultOp>(srcTerminator);
+
+  dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
+                                  srcBlock.begin(), std::prev(srcBlock.end()));
+
+  if (!resultOp->getOperands().empty()) {
+    rewriter.setInsertionPointToEnd(&dstBlock);
+    scf::YieldOp::create(rewriter, resultOp->getLoc(), resultOp->getOperands());
+  }
+
+  rewriter.eraseOp(srcTerminator);
+}
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    bool hasElse = !ifOp.getElseRegion().empty();
+    auto scfIfOp =
+        scf::IfOp::create(rewriter, ifOp.getLoc(), ifOp.getResultTypes(),
+                          ifOp.getCondition(), hasElse);
+
+    copyBlockAndTransformResult(rewriter, ifOp.getThenRegion().front(),
+                                scfIfOp.getThenRegion().front());
+
+    if (hasElse) {
+      copyBlockAndTransformResult(rewriter, ifOp.getElseRegion().front(),
+                                  scfIfOp.getElseRegion().front());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/flang/test/Fir/FirToSCF/if.fir b/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 0000000000000..03be264c4cdf5
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,57 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_only(
+// CHECK-SAME:      %[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:           scf.if %[[ARG0]] {
+// CHECK:             %[[VAL_1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_else()   {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           scf.if %[[VAL_1]] {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           } else {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2]] -> (f32, f32) {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           } else {
+// CHECK:             scf.yield %[[VAL_1]], %[[VAL_1]] : f32, f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}


From flang-commits at lists.llvm.org  Tue Jul 22 19:51:01 2025
From: flang-commits at lists.llvm.org (Terapines MLIR via flang-commits)
Date: Tue, 22 Jul 2025 19:51:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][fir]: New optimizer transform of
 `fir.if` to `scf.if`. (PR #149959)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149959@github.com>
Message-ID: <68804e15.170a0220.3aadca.131d@mx.google.com>


terapines-osc-mlir wrote:




> The title of this MR confused me for a minute, can it just be "add conversion of `fir.if` to `scf.if`"? or "convert `fir.if` in FIRToSCF"

Thanks! Commit message is changed to what you had suggested.

https://github.com/llvm/llvm-project/pull/149959

From flang-commits at lists.llvm.org  Tue Jul 22 11:03:45 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Tue, 22 Jul 2025 11:03:45 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] [openmp] [Flang][OpenMP]
 Move builtin .mod generation into runtimes (PR #137828)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/137828@github.com>
Message-ID: <687fd281.630a0220.31eae0.c8c2@mx.google.com>


https://github.com/Meinersbur updated https://github.com/llvm/llvm-project/pull/137828

>From 839198d61f9937b5504d5e036c67266b4b84da8e Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 14:09:57 +0200
Subject: [PATCH 01/16] [Flang][Flang-RT][OpenMP] Move builtin .mod generation
 into runtimes

---
 clang/include/clang/Driver/Driver.h           |   3 +-
 clang/include/clang/Driver/Options.td         |   2 +-
 clang/include/clang/Driver/ToolChain.h        |   4 +
 clang/lib/Driver/Driver.cpp                   |  10 +
 clang/lib/Driver/ToolChain.cpp                |   6 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   7 +
 .../Modules}/GetToolchainDirs.cmake           |  11 ++
 flang-rt/CMakeLists.txt                       |  73 ++------
 flang-rt/cmake/modules/AddFlangRT.cmake       |  14 +-
 .../cmake/modules/AddFlangRTOffload.cmake     |  14 +-
 flang-rt/lib/runtime/CMakeLists.txt           | 109 ++++++++++-
 .../lib/runtime}/__cuda_builtins.f90          |   0
 .../lib/runtime}/__cuda_device.f90            |   0
 .../lib/runtime}/__fortran_builtins.f90       |   4 +-
 .../runtime}/__fortran_ieee_exceptions.f90    |   0
 .../lib/runtime}/__fortran_type_info.f90      |   7 +-
 .../lib/runtime}/__ppc_intrinsics.f90         |   0
 .../lib/runtime}/__ppc_types.f90              |   0
 .../lib/runtime}/cooperative_groups.f90       |   1 +
 .../lib/runtime}/cudadevice.f90               |   0
 .../lib/runtime}/ieee_arithmetic.f90          |  29 ++-
 .../lib/runtime}/ieee_exceptions.f90          |   0
 .../lib/runtime}/ieee_features.f90            |   0
 .../lib/runtime}/iso_c_binding.f90            |   0
 .../lib/runtime}/iso_fortran_env.f90          |   0
 .../lib/runtime}/iso_fortran_env_impl.f90     |   0
 .../module => flang-rt/lib/runtime}/mma.f90   |   0
 flang-rt/test/lit.site.cfg.py.in              |   2 +-
 flang-rt/unittests/CMakeLists.txt             |   5 +-
 flang/CMakeLists.txt                          |   2 +-
 .../flang/Frontend/CompilerInvocation.h       |   7 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  26 +--
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   8 +-
 flang/module/.clang-format                    |   1 -
 flang/test/CMakeLists.txt                     |   7 +-
 flang/test/Driver/intrinsic-module-path.f90   |   4 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   2 +-
 flang/test/Lower/OpenMP/simd_aarch64.f90      |   7 +-
 .../target-enter-data-default-openmp52.f90    |   4 +-
 flang/test/lit.cfg.py                         |  57 +++++-
 flang/test/lit.site.cfg.py.in                 |   1 +
 flang/tools/CMakeLists.txt                    |   1 -
 flang/tools/bbc/bbc.cpp                       |  16 +-
 flang/tools/f18/CMakeLists.txt                | 176 ------------------
 flang/tools/f18/dump.cpp                      |  42 -----
 llvm/runtimes/CMakeLists.txt                  |  21 +--
 openmp/CMakeLists.txt                         |   4 +
 openmp/runtime/src/CMakeLists.txt             |  62 ++----
 runtimes/CMakeLists.txt                       | 163 +++++++++++++++-
 49 files changed, 512 insertions(+), 400 deletions(-)
 rename {flang-rt/cmake/modules => cmake/Modules}/GetToolchainDirs.cmake (94%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_builtins.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__cuda_device.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_builtins.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__fortran_type_info.f90 (98%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_intrinsics.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/__ppc_types.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/cooperative_groups.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/cudadevice.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_arithmetic.f90 (95%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_exceptions.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/ieee_features.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_c_binding.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/iso_fortran_env_impl.f90 (100%)
 rename {flang/module => flang-rt/lib/runtime}/mma.f90 (100%)
 delete mode 100644 flang/module/.clang-format
 delete mode 100644 flang/tools/f18/CMakeLists.txt
 delete mode 100644 flang/tools/f18/dump.cpp

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index d9e328fe918bc..9343fed36b6ac 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,9 +403,10 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
+  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
+  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index a8c1b5dd8ab3b..ed0d0cda49ad7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5921,7 +5921,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index b8899e78176b4..5cac39a8ed4e9 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -542,6 +542,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index ec1135eecd401..28ae55b024c33 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6536,6 +6536,16 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..ba20cda5e339b 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -983,6 +983,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 1edb83f7255eb..dba2f6fae493b 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -957,6 +957,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (auto IntrModPath = TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang-rt/cmake/modules/GetToolchainDirs.cmake b/cmake/Modules/GetToolchainDirs.cmake
similarity index 94%
rename from flang-rt/cmake/modules/GetToolchainDirs.cmake
rename to cmake/Modules/GetToolchainDirs.cmake
index fba12502b5946..f32e1264cc373 100644
--- a/flang-rt/cmake/modules/GetToolchainDirs.cmake
+++ b/cmake/Modules/GetToolchainDirs.cmake
@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d048ac4b3e5a4..09f4f9e7213f1 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -67,68 +67,17 @@ include(GetToolchainDirs)
 include(FlangCommon)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
-
 
 #################
 # Build Options #
@@ -243,6 +192,22 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+check_fortran_source_compiles([[
+    subroutine test_quadmath
+      real(16) :: var1
+    end
+  ]]
+  FORTRAN_SUPPORTS_REAL16
+)
+cmake_pop_check_state()
+
+flang_module_fortran_enable()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -338,4 +303,4 @@ if (FLANG_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
 else ()
   add_custom_target(check-flang-rt)
-endif()
+endif()
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index e51590fdae3d3..febaac3058b3f 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -334,13 +340,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 
diff --git a/flang-rt/cmake/modules/AddFlangRTOffload.cmake b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
index cbc69f3a9656a..4a6f047a86af2 100644
--- a/flang-rt/cmake/modules/AddFlangRTOffload.cmake
+++ b/flang-rt/cmake/modules/AddFlangRTOffload.cmake
@@ -88,16 +88,16 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
@@ -105,6 +105,12 @@ macro(enable_omp_offload_compilation name files)
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..1b8114c102205 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -73,7 +73,16 @@ set(supported_sources
 
 # List of source not used for GPU offloading.
 set(host_sources
-  ${FLANG_SOURCE_DIR}/module/iso_fortran_env_impl.f90
+  __fortran_ieee_exceptions.f90
+  __fortran_type_info.f90
+  iso_fortran_env.f90
+  ieee_arithmetic.f90
+  ieee_exceptions.f90
+  ieee_features.f90
+  iso_c_binding.f90
+  iso_fortran_env_impl.f90
+  iso_fortran_env.f90
+
   command.cpp
   complex-powi.cpp
   complex-reduction.c
@@ -90,6 +99,35 @@ set(host_sources
   unit-map.cpp
 )
 
+# Module sources that are required by other modules
+set(intrinsics_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
+)
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc")
+  list(APPEND host_source
+    __ppc_types.f90
+    __ppc_intrinsics.f90
+    mma.f90
+  )
+endif ()
+
+list(APPEND supported_sources
+  __cuda_device.f90
+  cooperative_groups.f90
+  cudadevice.f90
+)
+
+# Compile as CUDA-Fortran, not directly supported by CMake
+set_property(SOURCE
+    __cuda_device.f90
+    cooperative_groups.f90
+    cudadevice.f90
+  APPEND PROPERTY
+    COMPILE_OPTIONS --offload-host-only -xcuda
+)
+
 # Sources that can be compiled directly for the GPU.
 set(gpu_sources
   ${FLANG_SOURCE_DIR}/lib/Decimal/binary-to-decimal.cpp
@@ -182,10 +220,42 @@ else ()
 endif ()
 
 
+if (FORTRAN_SUPPORTS_REAL16)
+  add_compile_definitions(FLANG_SUPPORT_R16=1)
+  add_compile_options("$<$<COMPILE_LANGUAGE:Fortran>:-cpp>")
+endif ()
+
+add_compile_options(
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+  # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+  "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+)
+
+
+# check-flang depends on this to build intrinsic modules
+if (NOT TARGET flang-rt-mod)
+  add_custom_target(flang-rt-mod)
+endif ()
+
 if (NOT WIN32)
+  # CMake ignores intrinsic USE dependencies
+  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES to disable this behavior, unfortunately it does not work with Ninja (https://gitlab.kitware.com/cmake/cmake/-/issues/26803) 
+  # As a workaround, we build those intrinsic modules first such that the main runtime can depend on it.
+  add_flangrt_library(flang_rt.intrinsics.obj OBJECT
+    ${intrinsics_sources}
+  )
+
+  # This barrier exists to force all of the intrinsic modules of flang_rt.intrinsics.obj to be built before anything that depends on it.
+  # Without it, CMake/Ninja seem to think that the modules of flang_rt.intrinsics.obj can be built concurrently to those in flang_rt.runtime.
+  add_custom_target(flang_rt.intrinsics
+    COMMENT "Intrinsic module dependency barrier"
+  )
+  add_dependencies(flang_rt.intrinsics flang_rt.intrinsics.obj)
+
   add_flangrt_library(flang_rt.runtime STATIC SHARED
-    ${sources}
-    LINK_LIBRARIES ${Backtrace_LIBRARY}
+    ${sources} $<TARGET_OBJECTS:flang_rt.intrinsics.obj>
+    LINK_LIBRARIES flang_rt.intrinsics.obj ${Backtrace_LIBRARY}
     INSTALL_WITH_TOOLCHAIN
     ADDITIONAL_HEADERS ${public_headers} ${private_headers}
   )
@@ -196,6 +266,13 @@ if (NOT WIN32)
   # Select a default runtime, which is used for unit and regression tests.
   get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
   add_library(flang_rt.runtime.unittest ALIAS "${default_target}")
+
+  # Select a target that compiles the sources to build the public module files.
+  get_target_property(compile_target flang_rt.runtime.compile ALIASED_TARGET)
+  flang_module_target(flang_rt.intrinsics.obj PUBLIC)
+  flang_module_target(${compile_target} PUBLIC)
+  add_dependencies(${compile_target} flang_rt.intrinsics)
+  add_dependencies(flang-rt-mod flang_rt.intrinsics ${compile_target})
 else()
   # Target for building all versions of the runtime
   add_custom_target(flang_rt.runtime)
@@ -203,10 +280,15 @@ else()
 
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
+
+    add_flangrt_library(${name}.intrinsics OBJECT
+      ${intrinsics_sources}
+    )
+
     add_flangrt_library(${name} ${libtype}
-        ${sources}
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
         ${ARGN}
-        LINK_LIBRARIES ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -219,10 +301,19 @@ else()
 
     # Setting an unique Fortran_MODULE_DIRECTORY is required for each variant to
     # write a different .mod file.
-    set_target_properties(${name}
-        PROPERTIES
-          Fortran_MODULE_DIRECTORY "module.${suffix}"
-      )
+    # One has to be selected to be the public module that is to be installed.
+    # We select the first.
+    if (_has_public_intrinsics)
+      set(is_public "")
+    else ()
+      set(is_public PUBLIC)
+      set(_has_public_intrinsics "YES" PARENT_SCOPE)
+    endif ()
+
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
+    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${compile_target} ${is_public})
+    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")
diff --git a/flang/module/__cuda_builtins.f90 b/flang-rt/lib/runtime/__cuda_builtins.f90
similarity index 100%
rename from flang/module/__cuda_builtins.f90
rename to flang-rt/lib/runtime/__cuda_builtins.f90
diff --git a/flang/module/__cuda_device.f90 b/flang-rt/lib/runtime/__cuda_device.f90
similarity index 100%
rename from flang/module/__cuda_device.f90
rename to flang-rt/lib/runtime/__cuda_device.f90
diff --git a/flang/module/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
similarity index 98%
rename from flang/module/__fortran_builtins.f90
rename to flang-rt/lib/runtime/__fortran_builtins.f90
index 4d134fa4b62b1..d5e55b5d95020 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -6,13 +6,13 @@
 !
 !===------------------------------------------------------------------------===!
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 ! These naming shenanigans prevent names from Fortran intrinsic modules
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module __fortran_builtins
+module    __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
similarity index 100%
rename from flang/module/__fortran_ieee_exceptions.f90
rename to flang-rt/lib/runtime/__fortran_ieee_exceptions.f90
diff --git a/flang/module/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
similarity index 98%
rename from flang/module/__fortran_type_info.f90
rename to flang-rt/lib/runtime/__fortran_type_info.f90
index 6af2a5a5e30ff..2f936c3787a61 100644
--- a/flang/module/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,8 +12,11 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-
-  use, intrinsic :: __fortran_builtins, &
+#if 0
+  use       __fortran_builtins, &
+#else
+  use, intrinsic ::    __fortran_builtins, &
+#endif
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang/module/__ppc_intrinsics.f90 b/flang-rt/lib/runtime/__ppc_intrinsics.f90
similarity index 100%
rename from flang/module/__ppc_intrinsics.f90
rename to flang-rt/lib/runtime/__ppc_intrinsics.f90
diff --git a/flang/module/__ppc_types.f90 b/flang-rt/lib/runtime/__ppc_types.f90
similarity index 100%
rename from flang/module/__ppc_types.f90
rename to flang-rt/lib/runtime/__ppc_types.f90
diff --git a/flang/module/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
similarity index 95%
rename from flang/module/cooperative_groups.f90
rename to flang-rt/lib/runtime/cooperative_groups.f90
index b8875f72f8079..82d1e0fe84042 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,6 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use :: cudadevice ! implicit dependency, make explicit for CMake
 
 implicit none
 
diff --git a/flang/module/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
similarity index 100%
rename from flang/module/cudadevice.f90
rename to flang-rt/lib/runtime/cudadevice.f90
diff --git a/flang/module/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
similarity index 95%
rename from flang/module/ieee_arithmetic.f90
rename to flang-rt/lib/runtime/ieee_arithmetic.f90
index 4e938a2daaa91..b3288a5cdd69f 100644
--- a/flang/module/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include '../include/flang/Runtime/magic-numbers.h'
+#include   '../include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,13 +336,28 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
-#define IEEE_IS_FINITE_R(XKIND) \
-  elemental logical function ieee_is_finite_a##XKIND(x); \
-    real(XKIND), intent(in) :: x; \
-    !dir$ ignore_tkr(d) x; \
-  end function ieee_is_finite_a##XKIND;
+! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
-    SPECIFICS_R(IEEE_IS_FINITE_R)
+elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a2;
+elemental logical function ieee_is_finite_a3(x); real(3), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a3;
+ elemental logical function ieee_is_finite_a4(x); real(4), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a4;
+elemental logical function ieee_is_finite_a8(x); real(8), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a8;
+elemental logical function ieee_is_finite_a10(x); real(10), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a10;
+#if FLANG_SUPPORT_R16
+elemental logical function ieee_is_finite_a16(x); real(16), intent(in) :: x;
+!dir$ ignore_tkr(d) x;
+end function ieee_is_finite_a16;
+#endif
   end interface ieee_is_finite
   public :: ieee_is_finite
 #undef IEEE_IS_FINITE_R
diff --git a/flang/module/ieee_exceptions.f90 b/flang-rt/lib/runtime/ieee_exceptions.f90
similarity index 100%
rename from flang/module/ieee_exceptions.f90
rename to flang-rt/lib/runtime/ieee_exceptions.f90
diff --git a/flang/module/ieee_features.f90 b/flang-rt/lib/runtime/ieee_features.f90
similarity index 100%
rename from flang/module/ieee_features.f90
rename to flang-rt/lib/runtime/ieee_features.f90
diff --git a/flang/module/iso_c_binding.f90 b/flang-rt/lib/runtime/iso_c_binding.f90
similarity index 100%
rename from flang/module/iso_c_binding.f90
rename to flang-rt/lib/runtime/iso_c_binding.f90
diff --git a/flang/module/iso_fortran_env.f90 b/flang-rt/lib/runtime/iso_fortran_env.f90
similarity index 100%
rename from flang/module/iso_fortran_env.f90
rename to flang-rt/lib/runtime/iso_fortran_env.f90
diff --git a/flang/module/iso_fortran_env_impl.f90 b/flang-rt/lib/runtime/iso_fortran_env_impl.f90
similarity index 100%
rename from flang/module/iso_fortran_env_impl.f90
rename to flang-rt/lib/runtime/iso_fortran_env_impl.f90
diff --git a/flang/module/mma.f90 b/flang-rt/lib/runtime/mma.f90
similarity index 100%
rename from flang/module/mma.f90
rename to flang-rt/lib/runtime/mma.f90
diff --git a/flang-rt/test/lit.site.cfg.py.in b/flang-rt/test/lit.site.cfg.py.in
index 662d076b1fe24..0e9dc08b59925 100644
--- a/flang-rt/test/lit.site.cfg.py.in
+++ b/flang-rt/test/lit.site.cfg.py.in
@@ -6,7 +6,7 @@ config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.flang_source_dir = "@FLANG_SOURCE_DIR@"
 config.flang_rt_source_dir = "@FLANG_RT_SOURCE_DIR@"
 config.flang_rt_binary_test_dir = os.path.dirname(__file__)
-config.flang_rt_output_resource_lib_dir = "@FLANG_RT_OUTPUT_RESOURCE_LIB_DIR@"
+config.flang_rt_output_resource_lib_dir = "@RUNTIMES_OUTPUT_RESOURCE_LIB_DIR@"
 config.flang_rt_experimental_offload_support = "@FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.flang = "@CMAKE_Fortran_COMPILER@"
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 5282196174134..82f32d027944e 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -53,9 +53,8 @@ function(add_flangrt_unittest_offload_properties target)
   # FIXME: replace 'native' in --offload-arch option with the list
   #        of targets that Fortran Runtime was built for.
   if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
-    set_target_properties(${target}
-      PROPERTIES LINK_OPTIONS
-      "-fopenmp;--offload-arch=native"
+    set_property(TARGET ${target} APPEND
+      PROPERTY LINK_OPTIONS -fopenmp --offload-arch=native
       )
   endif()
 endfunction()
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..5bdb43e5ab5e6 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -273,7 +273,7 @@ set(FLANG_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(FLANG_TOOLS_INSTALL_DIR)
 
-set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_BINARY_DIR}/include/flang)
+set(FLANG_INTRINSIC_MODULES_DIR "" CACHE PATH "Additional search path for modules; needed for running all tests if not building flang-rt in a bootstrapping build")
 set(FLANG_INCLUDE_DIR ${FLANG_BINARY_DIR}/include)
 
 # TODO: Remove when libclangDriver is lifted out of Clang
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d294955af780e..feaee28a53349 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -92,6 +92,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // intrinsic of iso_fortran_env.
   std::string allCompilerInvocOpts;
 
+  /// Location of the resource directory containing files specific to this
+  /// instance/version of Flang.
+  std::string resourceDir;
+
   /// Semantic options
   // TODO: Merge with or translate to frontendOpts. We shouldn't need two sets
   // of options.
@@ -177,6 +181,9 @@ class CompilerInvocation : public CompilerInvocationBase {
   getSemanticsCtx(Fortran::parser::AllCookedSources &allCookedSources,
                   const llvm::TargetMachine &);
 
+  std::string &getResourceDir() { return resourceDir; }
+  const std::string &getResourceDir() const { return resourceDir; }
+
   std::string &getModuleDir() { return moduleDir; }
   const std::string &getModuleDir() const { return moduleDir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..bd0f0a381bbd3 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -884,16 +884,6 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
   return diags.getNumErrors() == numErrorsBefore;
 }
 
-// Generate the path to look for intrinsic modules
-static std::string getIntrinsicDir(const char *argv) {
-  // TODO: Find a system independent API
-  llvm::SmallString<128> driverPath;
-  driverPath.assign(llvm::sys::fs::getMainExecutable(argv, nullptr));
-  llvm::sys::path::remove_filename(driverPath);
-  driverPath.append("/../include/flang/");
-  return std::string(driverPath);
-}
-
 // Generate the path to look for OpenMP headers
 static std::string getOpenMPHeadersDir(const char *argv) {
   llvm::SmallString<128> includePath;
@@ -1509,6 +1499,14 @@ bool CompilerInvocation::createFromArgs(
     success = false;
   }
 
+  // User-specified or default resource dir
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_resource_dir))
+    invoc.resourceDir = a->getValue();
+  else
+    invoc.resourceDir = clang::driver::Driver::GetResourcesPath(
+        llvm::sys::fs::getMainExecutable(argv0, nullptr));
+
   // -flang-experimental-hlfir
   if (args.hasArg(clang::driver::options::OPT_flang_experimental_hlfir) ||
       args.hasArg(clang::driver::options::OPT_emit_hlfir)) {
@@ -1759,9 +1757,11 @@ void CompilerInvocation::setFortranOpts() {
       preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
       preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
-  //  Add the default intrinsic module directory
-  fortranOptions.intrinsicModuleDirectories.emplace_back(
-      getIntrinsicDir(getArgv0()));
+  // Add the ordered list of -fintrinsic-modules-path
+  fortranOptions.intrinsicModuleDirectories.insert(
+      fortranOptions.intrinsicModuleDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
 
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5ca53ee48955e..14f422a6098b2 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1365,10 +1365,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(loc,
-                        "runtime derived type info descriptor was not "
-                        "generated and skipExternalRttiDefinition and "
-                        "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(
+        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
+                 "' was not generated and skipExternalRttiDefinition and "
+                 "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
deleted file mode 100644
index e3845288a2aec..0000000000000
--- a/flang/module/.clang-format
+++ /dev/null
@@ -1 +0,0 @@
-DisableFormat: true
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 8520bec646971..6cc7abd5fe7dc 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -59,7 +59,6 @@ set(FLANG_TEST_PARAMS
 
 set(FLANG_TEST_DEPENDS
   flang
-  module_files
   fir-opt
   tco
   bbc
@@ -97,8 +96,12 @@ if (LLVM_BUILD_EXAMPLES)
     )
 endif ()
 
+if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
+  list(APPEND FLANG_TEST_DEPENDS "flang-rt-mod") # For intrinsic module files (in flang-rt/)
+endif ()
+
 if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND NOT FLANG_STANDALONE_BUILD)
-  list(APPEND FLANG_TEST_DEPENDS "libomp-mod")
+  list(APPEND FLANG_TEST_DEPENDS "libomp-mod") # For omplib.mod and omplib_kinds.mod (in openmp/)
 endif ()
 
 add_custom_target(flang-test-depends DEPENDS ${FLANG_TEST_DEPENDS})
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 8fe486cf61c83..a6f0cb04453f4 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -6,8 +6,8 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang -fc1)
 !-----------------------------------------
-! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
-! RUN: not %flang_fc1 -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
+! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
 
 ! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
 ! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
index 2e0c72ccfe048..29a9784b984b6 100644
--- a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -1,6 +1,6 @@
 ! Test interface that lowering handles small interface mismatch with
 ! type bound procedures.
-! RUN: bbc -emit-hlfir %s -o - -I nw | FileCheck %s
+! RUN: %bbc_bare -emit-hlfir %s -o - -I nw | FileCheck %s
 
 module dispatch_mismatch
 type t
diff --git a/flang/test/Lower/OpenMP/simd_aarch64.f90 b/flang/test/Lower/OpenMP/simd_aarch64.f90
index 735237223bcb5..2e4136273c75b 100644
--- a/flang/test/Lower/OpenMP/simd_aarch64.f90
+++ b/flang/test/Lower/OpenMP/simd_aarch64.f90
@@ -1,6 +1,11 @@
-! Tests for 2.9.3.1 Simd and target dependent defult alignment for AArch64
+! Tests for 2.9.3.1 Simd and target dependent default alignment for AArch64
 ! The default alignment for AARCH64 is 0 so we do not emit aligned clause
 ! REQUIRES: aarch64-registered-target
+
+! Requires aarch64 iso_c_binding.mod which currently is only available if your host is also aarch64
+! FIXME: Make flang a cross-compiler
+! UNSUPPORTED: true
+
 ! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-hlfir -fopenmp %s -o - | FileCheck  %s
 subroutine simdloop_aligned_cptr(A)
     use iso_c_binding
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index 0d4fd964b71ec..72b5fea2c171e 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -1,7 +1,7 @@
 ! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
 
-!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
-!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
 
 module test
   real, allocatable :: A
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..8a375fdf49b8b 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -127,19 +127,64 @@
 if config.default_sysroot:
     config.available_features.add("default_sysroot")
 
+
+flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+if not flang_exe:
+    lit_config.fatal(f"Could not identify flang executable")
+
+def get_resource_module_intrinsic_dir():
+    # Determine the intrinsic module search path that is added by the driver. If
+    # skipping the driver using -fc1, we need to append the path manually.
+    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    if not flang_intrinsics_dir:
+        return None
+    flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
+    return flang_intrinsics_dir
+
+intrinsics_search_args = []
+if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
+    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
+
+extra_intrinsics_search_args = []
+if config.flang_intrinsic_modules_dir:
+    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
+    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+
+config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
+    ToolSubst(
+        "bbc",
+         command=FindTool("bbc"),
+         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+         unresolved="fatal",
+    ),
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
+        extra_args=isysroot_flag + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
     ToolSubst(
         "%flang_fc1",
         command=FindTool("flang"),
-        extra_args=["-fc1"],
+        extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
+    ),
+
+    # For not having intrinsic search paths to be added implicitly
+    ToolSubst(
+         "%bbc_bare",
+         command=FindTool("bbc"),
+         unresolved="fatal",
+    ),
+    ToolSubst(
+        "%flang_bare",
+        command=FindTool("flang"),
+        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
 ]
@@ -177,6 +222,14 @@
 if result:
     config.environment["LIBPGMATH"] = True
 
+# If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
+config.available_features.add("module-independent")
+if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
+   config.available_features.add("flangrt-modules")
+else:
+   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+   config.limit_to_features.add("module-independent")
+
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"
 if config.have_openmp_rtl:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..5d07af42c6487 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -23,6 +23,7 @@ config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
+config.have_flangrt_mod = ("flang-rt" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
 if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
     config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
diff --git a/flang/tools/CMakeLists.txt b/flang/tools/CMakeLists.txt
index 1d2d2c608faf9..1b297af74cae7 100644
--- a/flang/tools/CMakeLists.txt
+++ b/flang/tools/CMakeLists.txt
@@ -7,7 +7,6 @@
 #===------------------------------------------------------------------------===#
 
 add_subdirectory(bbc)
-add_subdirectory(f18)
 add_subdirectory(flang-driver)
 add_subdirectory(tco)
 add_subdirectory(f18-parse-demo)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..a98a94e32bee4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -98,6 +98,11 @@ static llvm::cl::alias
                           llvm::cl::desc("intrinsic module directory"),
                           llvm::cl::aliasopt(intrinsicIncludeDirs));
 
+static llvm::cl::alias
+    intrinsicModulePath("fintrinsic-modules-path",
+                        llvm::cl::desc("intrinsic module search paths"),
+                        llvm::cl::aliasopt(intrinsicIncludeDirs));
+
 static llvm::cl::opt<std::string>
     moduleDir("module", llvm::cl::desc("module output directory (default .)"),
               llvm::cl::init("."));
@@ -568,17 +573,8 @@ int main(int argc, char **argv) {
   ProgramName programPrefix;
   programPrefix = argv[0] + ": "s;
 
-  if (includeDirs.size() == 0) {
+  if (includeDirs.size() == 0)
     includeDirs.push_back(".");
-    // Default Fortran modules should be installed in include/flang (a sibling
-    // to the bin) directory.
-    intrinsicIncludeDirs.push_back(
-        llvm::sys::path::parent_path(
-            llvm::sys::path::parent_path(
-                llvm::sys::fs::getMainExecutable(argv[0], nullptr)))
-            .str() +
-        "/include/flang");
-  }
 
   Fortran::parser::Options options;
   options.predefinitions.emplace_back("__flang__"s, "1"s);
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
deleted file mode 100644
index 546b6acaaf91d..0000000000000
--- a/flang/tools/f18/CMakeLists.txt
+++ /dev/null
@@ -1,176 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenACC
-  FrontendOpenMP
-  Support
-  )
-
-# Define the list of Fortran module files for which it is
-# sufficient to generate the module file via -fsyntax-only.
-set(MODULES
-  "__fortran_builtins"
-  "__fortran_ieee_exceptions"
-  "__fortran_type_info"
-  "__ppc_types"
-  "__ppc_intrinsics"
-  "mma"
-  "__cuda_builtins"
-  "__cuda_device"
-  "cooperative_groups"
-  "cudadevice"
-  "ieee_arithmetic"
-  "ieee_exceptions"
-  "ieee_features"
-  "iso_c_binding"
-  "iso_fortran_env"
-  "iso_fortran_env_impl"
-)
-
-# Check if 128-bit float computations can be done via long double.
-check_cxx_source_compiles(
-  "#include <cfloat>
-   #if LDBL_MANT_DIG != 113
-   #error LDBL_MANT_DIG != 113
-   #endif
-   int main() { return 0; }
-  "
-  HAVE_LDBL_MANT_DIG_113)
-
-# Figure out whether we can support REAL(KIND=16)
-if (FLANG_RUNTIME_F128_MATH_LIB)
-  set(FLANG_SUPPORT_R16 "1")
-elseif (HAVE_LDBL_MANT_DIG_113)
-  set(FLANG_SUPPORT_R16 "1")
-else()
-  set(FLANG_SUPPORT_R16 "0")
-endif()
-
-# Init variable to hold extra object files coming from the Fortran modules;
-# these module files will be contributed from the CMakeLists in flang/tools/f18.
-set(module_objects "")
-
-# Create module files directly from the top-level module source directory.
-# If CMAKE_CROSSCOMPILING, then the newly built flang executable was
-# cross compiled, and thus can't be executed on the build system and thus
-# can't be used for generating module files.
-if (NOT CMAKE_CROSSCOMPILING)
-  foreach(filename ${MODULES})
-    set(depends "")
-    set(opts "")
-    if(${filename} STREQUAL "__fortran_builtins" OR
-       ${filename} STREQUAL "__ppc_types")
-    elseif(${filename} STREQUAL "__ppc_intrinsics" OR
-           ${filename} STREQUAL "mma")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device" OR
-           ${filename} STREQUAL "cudadevice" OR
-           ${filename} STREQUAL "cooperative_groups")
-      set(opts -fc1 -xcuda)
-      if(${filename} STREQUAL "__cuda_device")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-      elseif(${filename} STREQUAL "cudadevice")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
-      elseif(${filename} STREQUAL "cooperative_groups")
-        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
-      endif()
-    else()
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
-      if(${filename} STREQUAL "iso_fortran_env")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/iso_fortran_env_impl.mod)
-      endif()
-      if(${filename} STREQUAL "ieee_arithmetic" OR
-         ${filename} STREQUAL "ieee_exceptions")
-        set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_ieee_exceptions.mod)
-      endif()
-    endif()
-    if(NOT ${filename} STREQUAL "__fortran_type_info" AND NOT ${filename} STREQUAL "__fortran_builtins")
-      set(depends ${depends} ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_type_info.mod)
-    endif()
-
-    # The module contains PPC vector types that needs the PPC target.
-    if(${filename} STREQUAL "__ppc_intrinsics" OR
-       ${filename} STREQUAL "mma")
-      if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
-        set(opts "--target=ppc64le")
-      else()
-        # Do not compile PPC module if the target is not available.
-        continue()
-      endif()
-    endif()
-
-    set(decls "")
-    if (FLANG_SUPPORT_R16)
-      set(decls "-DFLANG_SUPPORT_R16")
-    endif()
-
-    # Some modules have an implementation part that needs to be added to the
-    # flang_rt.runtime library.
-    set(compile_with "-fsyntax-only")
-    set(object_output "")
-    set(include_in_link FALSE)
-
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/${filename})
-    # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
-    add_custom_command(OUTPUT ${base}.mod ${object_output}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang ${opts} ${decls} -cpp ${compile_with} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${FLANG_SOURCE_DIR}/module/${filename}.f90
-      DEPENDS flang ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
-    )
-    list(APPEND MODULE_FILES ${base}.mod)
-    install(FILES ${base}.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-
-    # If a module has been compiled into an object file, add the file to
-    # the link line for the flang_rt.runtime library.
-    if(include_in_link)
-      list(APPEND module_objects ${object_output})
-    endif()
-  endforeach()
-
-  # Set a CACHE variable that is visible to the CMakeLists.txt in runtime/, so that
-  # the compiled Fortran modules can be added to the link line of the flang_rt.runtime
-  # library.
-  set(FORTRAN_MODULE_OBJECTS ${module_objects} CACHE INTERNAL "" FORCE)
-
-  # Special case for omp_lib.mod, because its source comes from openmp/runtime/src/include.
-  # It also produces two module files: omp_lib.mod and omp_lib_kinds.mod.  Compile these
-  # files only if OpenMP support has been configured.
-  if (LLVM_TOOL_OPENMP_BUILD)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.mod")
-    set(base ${FLANG_INTRINSIC_MODULES_DIR}/omp_lib)
-    add_custom_command(OUTPUT ${base}.mod ${base}_kinds.mod
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
-        ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90
-      DEPENDS flang ${FLANG_INTRINSIC_MODULES_DIR}/iso_c_binding.mod ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.F90 ${depends}
-    )
-    add_custom_command(OUTPUT ${base}.f18.mod
-      DEPENDS ${base}.mod
-      COMMAND ${CMAKE_COMMAND} -E copy ${base}.mod ${base}.f18.mod)
-      add_custom_command(OUTPUT ${base}_kinds.f18.mod
-        DEPENDS ${base}.mod
-        COMMAND ${CMAKE_COMMAND} -E copy ${base}_kinds.mod ${base}_kinds.f18.mod)
-    list(APPEND MODULE_FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod)
-    install(FILES ${base}.mod ${base}.f18.mod ${base}_kinds.mod ${base}_kinds.f18.mod DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang" COMPONENT flang-module-interfaces)
-  elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-    message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.mod is built there")
-  else()
-    message(WARNING "Not building omp_lib.mod, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-  endif()
-  add_llvm_install_targets(install-flang-module-interfaces
-    COMPONENT flang-module-interfaces)
-endif()
-
-add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
-set_target_properties(module_files PROPERTIES FOLDER "Flang/Resources")
-
-# TODO Move this to a more suitable location
-# Copy the generated omp_lib.h header file, if OpenMP support has been configured.
-if (LLVM_TOOL_OPENMP_BUILD)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_PROJECTS, building omp_lib.h")
-  file(COPY ${CMAKE_BINARY_DIR}/projects/openmp/runtime/src/omp_lib.h DESTINATION "${CMAKE_BINARY_DIR}/include/flang/OpenMP/" FILE_PERMISSIONS OWNER_READ OWNER_WRITE)
-  install(FILES ${CMAKE_BINARY_DIR}/include/flang/OpenMP/omp_lib.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/flang/OpenMP")
-elseif ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES)
-  message(STATUS "OpenMP runtime support enabled via LLVM_ENABLE_RUNTIMES, assuming omp_lib.h is built there")
-else()
-  message(STATUS "Not copying omp_lib.h, no OpenMP runtime in either LLVM_ENABLE_PROJECTS or LLVM_ENABLE_RUNTIMES")
-endif()
diff --git a/flang/tools/f18/dump.cpp b/flang/tools/f18/dump.cpp
deleted file mode 100644
index f11b5aedf4c6a..0000000000000
--- a/flang/tools/f18/dump.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- tools/f18/dump.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This file defines Dump routines available for calling from the debugger.
-// Each is based on operator<< for that type. There are overloadings for
-// reference and pointer, and for dumping to a provided raw_ostream or errs().
-
-#ifdef DEBUGF18
-
-#include "llvm/Support/raw_ostream.h"
-
-#define DEFINE_DUMP(ns, name) \
-  namespace ns { \
-  class name; \
-  llvm::raw_ostream &operator<<(llvm::raw_ostream &, const name &); \
-  } \
-  void Dump(llvm::raw_ostream &os, const ns::name &x) { os << x << '\n'; } \
-  void Dump(llvm::raw_ostream &os, const ns::name *x) { \
-    if (x == nullptr) \
-      os << "null\n"; \
-    else \
-      Dump(os, *x); \
-  } \
-  void Dump(const ns::name &x) { Dump(llvm::errs(), x); } \
-  void Dump(const ns::name *x) { Dump(llvm::errs(), *x); }
-
-namespace Fortran {
-DEFINE_DUMP(parser, Name)
-DEFINE_DUMP(parser, CharBlock)
-DEFINE_DUMP(semantics, Symbol)
-DEFINE_DUMP(semantics, Scope)
-DEFINE_DUMP(semantics, IntrinsicTypeSpec)
-DEFINE_DUMP(semantics, DerivedTypeSpec)
-DEFINE_DUMP(semantics, DeclTypeSpec)
-} // namespace Fortran
-
-#endif
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..96391ed70133e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -252,6 +252,11 @@ function(runtime_default_target)
     # OpenMP tests
     list(APPEND extra_targets "libomp-mod")
   endif ()
+  if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # The target flang-rt-mod is a dependee of check-flang needed to run its
+    # tests.
+    list(APPEND extra_targets "flang-rt-mod")
+  endif ()
 
   if(LLVM_INCLUDE_TESTS)
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
@@ -519,18 +524,12 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
+
+  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
+  if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    list(APPEND extra_args ENABLE_FORTRAN)
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
-    if (${LLVM_TOOL_FLANG_BUILD})
-      message(STATUS "Configuring build of omp_lib.mod and omp_lib_kinds.mod via flang")
-      set(LIBOMP_FORTRAN_MODULES_COMPILER "${CMAKE_BINARY_DIR}/bin/flang")
-      set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
-      # TODO: This is a workaround until flang becomes a first-class project
-      # in llvm/CMakeList.txt.  Until then, this line ensures that flang is
-      # built before "openmp" is built as a runtime project.  Besides "flang"
-      # to build the compiler, we also need to add "module_files" to make sure
-      # that all .mod files are also properly build.
-      list(APPEND extra_deps "flang" "module_files")
-    endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
         list(APPEND extra_deps ${dep})
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c206386fa6b61..3b64db92dcdea 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -100,6 +100,10 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  flang_module_fortran_enable()
+endif ()
+
 set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 698e185d9c4dd..66acb3fd04136 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -368,45 +368,6 @@ endif()
 configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
 
-set(BUILD_FORTRAN_MODULES False)
-if (NOT ${LIBOMP_FORTRAN_MODULES_COMPILER} STREQUAL "")
-  # If libomp is built as an LLVM runtime and the flang compiler is available,
-  # compile the Fortran module files.
-  message(STATUS "configuring openmp to build Fortran module files using ${LIBOMP_FORTRAN_MODULES_COMPILER}")
-  set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${LIBOMP_FORTRAN_MODULES_COMPILER} -cpp -fsyntax-only ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set(BUILD_FORTRAN_MODULES True)
-elseif(${LIBOMP_FORTRAN_MODULES})
-  # The following requests explicit building of the Fortran module files
-  # Workaround for gfortran to build modules with the
-  # omp_sched_monotonic integer parameter
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    set(ADDITIONAL_Fortran_FLAGS "-fno-range-check")
-  endif()
-  add_custom_target(libomp-mod ALL DEPENDS omp_lib.mod omp_lib_kinds.mod)
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Misc")
-  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
-  if(CMAKE_Fortran_COMPILER_SUPPORTS_F90)
-    set(LIBOMP_FORTRAN_SOURCE_FILE omp_lib.F90)
-  else()
-    message(FATAL_ERROR "Fortran module build requires Fortran 90 compiler")
-  endif()
-  add_custom_command(
-    OUTPUT omp_lib.mod omp_lib_kinds.mod
-    COMMAND ${CMAKE_Fortran_COMPILER} -c ${ADDITIONAL_Fortran_FLAGS}
-            ${LIBOMP_CONFIGURED_FFLAGS} ${LIBOMP_FORTRAN_SOURCE_FILE}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_FORTRAN_SOURCE_FILE}
-      ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-  )
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES omp_lib${CMAKE_C_OUTPUT_EXTENSION})
-  set(BUILD_FORTRAN_MODULES True)
-endif()
 
 # Move files to exports/ directory if requested
 if(${LIBOMP_COPY_EXPORTS})
@@ -482,15 +443,32 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-if(${BUILD_FORTRAN_MODULES})
+
+
+# Build the modules files if a Fortran compiler is available
+# only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp has been deprecated.
+if(LLVM_RUNTIMES_BUILD AND RUNTIMES_FLANG_MODULES_ENABLED)
+  # TODO: Consider including in LIBOMP_SOURCE_FILES instead
+  add_library(libomp-mod OBJECT
+    omp_lib.F90
+  )
+  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+  libomp_get_fflags(LIBOMP_CONFIGURED_FFLAGS)
+  target_compile_options(libomp-mod PRIVATE ${LIBOMP_CONFIGURED_FFLAGS})
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+   target_compile_options(libomp-mod PRIVATE -fno-range-check)
+  endif()
+
+  flang_module_target(libomp-mod PUBLIC)
+  add_dependencies(libomp-mod flang-rt-mod)
+
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
   endif()
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.h
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib.mod
-    ${CMAKE_CURRENT_BINARY_DIR}/omp_lib_kinds.mod
     DESTINATION ${destination}
   )
 endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index e4dd4ebfc678d..2dcc68b80b07c 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -91,6 +91,16 @@ include(CheckLibraryExists)
 include(LLVMCheckCompilerLinkerFlag)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(ExtendPath)
+
+
+# Determine whether we are in the runtimes/runtimes-bins directory of a
+# bootstrapping build.
+set(LLVM_TREE_AVAILABLE OFF)
+if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
+  set(LLVM_TREE_AVAILABLE ON)
+endif()
+
 
 # CMake omits default compiler include paths, but in runtimes build, we use
 # -nostdinc and -nostdinc++ and control include paths manually so this behavior
@@ -98,6 +108,7 @@ include(CheckCXXCompilerFlag)
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
+# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -276,6 +287,156 @@ if(LLVM_INCLUDE_TESTS)
   umbrella_lit_testsuite_begin(check-runtimes)
 endif()
 
+# Determine paths for files that belong into the Clang/Flang resource dir.
+if (LLVM_TREE_AVAILABLE)
+  # In a bootstrap build emit the libraries into a default search path in the
+  # build directory of the just-built compiler. This allows using the
+  # just-built compiler without specifying paths to runtime libraries.
+  #
+  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
+  # being added to the build. Flang uses the same resource dir as clang.
+  include(GetClangResourceDir)
+  get_clang_resource_dir(RUNTIMES_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
+  get_clang_resource_dir(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT)
+else ()
+  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
+  # read-only and/or shared by multiple runtimes with different build
+  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
+  # non-toolchain library.
+  # For the install prefix, still use the resource dir assuming that Flang will
+  # be installed there using the same prefix. This is to not have a difference
+  # between bootstrap and standalone runtimes builds.
+  set(RUNTIMES_OUTPUT_RESOURCE_DIR "${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+  set(RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
+endif ()
+
+# Determine build and install paths.
+# The build path is absolute, but the install dir is relative, CMake's install
+# command has to apply CMAKE_INSTALL_PREFIX itself.
+# FIXME: For shared libraries, the toolchain resource lib dir is not a good
+#        destination because it is not a ld.so default search path.
+#        The machine where the executable is eventually executed may not be the
+#        machine where the Flang compiler and its resource dir is installed, so
+#        setting RPath by the driver is not an solution. It should belong into
+#        /usr/lib/<triple>/lib<name>.so, like e.g. libgcc_s.so.
+#        But the linker as invoked by the Flang driver also requires
+#        libflang_rt.so to be found when linking and the resource lib dir is
+#        the only reliable location.
+include(GetToolchainDirs)
+get_toolchain_library_subdir(toolchain_lib_subdir)
+extend_path(RUNTIMES_OUTPUT_RESOURCE_LIB_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
+
+set(RUNTIMES_INSTALL_RESOURCE_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH_DEFAULT}" CACHE PATH "Path to install headers, runtime libraries, and Fortran modules to (default: Clang resource dir)")
+extend_path(RUNTIMES_INSTALL_RESOURCE_LIB_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
+
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_PATH)
+cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
+cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
+
+
+# Enable building Fortran modules
+# * Set up the Fortran compiler
+# * Determine files location in the Clang/Flang resource dir
+# * Install module files
+macro (flang_module_fortran_enable)
+  include(CheckLanguage)
+  check_language(Fortran)
+  if(NOT CMAKE_Fortran_COMPILER)
+    message(STATUS "Not compiling Flang modules: Fortran not enabled")
+    return ()
+  endif ()
+  enable_language(Fortran)
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
+      message(STATUS "Compiling Flang modules")
+      set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+    else ()
+      # TODO: Introspection of whether intrinsic modules are already available, so the openmp modules can be built without flang-rt in LLVM_ENABLE_RUNTIMES
+      message(STATUS "Not compiling Flang modules: flang-rt not available")
+    endif ()
+  else ()
+    message(STATUS "Compiling modules for non-Flang compiler (${CMAKE_Fortran_COMPILER_ID})")
+    set(RUNTIMES_FLANG_MODULES_ENABLED ON)
+  endif ()
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+
+  if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+    get_toolchain_module_subdir(toolchain_mod_subdir)
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "${toolchain_mod_subdir}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "${toolchain_mod_subdir}")
+  else ()
+    # For non-Flang compilers, avoid the risk of Flang accidentally picking them up.
+    extend_path(RUNTIMES_OUTPUT_RESOURCE_MOD_DIR "${RUNTIMES_OUTPUT_RESOURCE_DIR}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+    extend_path(RUNTIMES_INSTALL_RESOURCE_MOD_PATH "${RUNTIMES_INSTALL_RESOURCE_PATH}" "finclude-${CMAKE_Fortran_COMPILER_ID}")
+  endif ()
+  cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_MOD_DIR)
+  cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_MOD_PATH)
+
+  # Avoid module files to be installed multiple times if this macro is called multiple times
+  get_property(is_installed GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED)
+  if (NOT is_installed)
+    # No way to find out which mod files built by target individually, so install the entire output directory
+    # https://stackoverflow.com/questions/52712416/cmake-fortran-module-directory-to-be-used-with-add-library
+    set(destination "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
+    cmake_path(NORMAL_PATH destination)
+    install(DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      DESTINATION "${destination}"
+    )
+    set_property(GLOBAL PROPERTY RUNTIMES_MODS_INSTALLED TRUE)
+  endif ()
+endmacro ()
+
+
+# Set options to compile Fortran module files.
+#
+# Usage:
+#
+# flang_module_target(name
+#   PUBLIC
+#     Modules files are to be used by other Fortran sources. If a library is
+#     compiled multiple times (e.g. static/shared, or msvcrt variants), only
+#     one of those can be public module files; non-public modules are still
+#     generated but to be forgotten deep inside the build directory to not
+#     conflict with each other.
+#     Also, installs the module with the toolchain.
+# )
+function (flang_module_target tgtname)
+  set(options PUBLIC)
+  cmake_parse_arguments(ARG
+    "${options}"
+    ""
+    ""
+    ${ARGN})
+
+  if (NOT RUNTIMES_FLANG_MODULES_ENABLED)
+    return ()
+  endif ()
+
+  # Let it find the other public module files
+  target_compile_options(${tgtname} PRIVATE
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+    )
+
+  if (ARG_PUBLIC)
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+      )
+  else ()
+    set_target_properties(${tgtname}
+      PROPERTIES
+        Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${tgtname}.mod"
+      )
+  endif ()
+endfunction ()
+
+
 # We do this in two loops so that HAVE_* is set for each runtime before the
 # other runtimes are added.
 foreach(entry ${runtimes})
@@ -363,4 +524,4 @@ if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE
                              -o ${LLVM_BINARY_DIR}/compile_commands.json
                      DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
   add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
-endif()
+endif()
\ No newline at end of file

>From a1533ccdd1e0ac3c153a3b8a6007004a0a6cac75 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 15:13:47 +0200
Subject: [PATCH 02/16] python format

---
 flang/test/lit.cfg.py | 45 ++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8a375fdf49b8b..b05eba8da0b0c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -128,39 +128,53 @@
     config.available_features.add("default_sysroot")
 
 
-flang_exe =  lit.util.which("flang", config.flang_llvm_tools_dir)
+flang_exe = lit.util.which("flang", config.flang_llvm_tools_dir)
 if not flang_exe:
     lit_config.fatal(f"Could not identify flang executable")
 
+
 def get_resource_module_intrinsic_dir():
     # Determine the intrinsic module search path that is added by the driver. If
     # skipping the driver using -fc1, we need to append the path manually.
-    flang_intrinsics_dir = subprocess.check_output([flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True).strip()
+    flang_intrinsics_dir = subprocess.check_output(
+        [flang_exe, "-print-file-name=__fortran_builtins.mod"], text=True
+    ).strip()
     if not flang_intrinsics_dir:
         return None
     flang_intrinsics_dir = os.path.dirname(flang_intrinsics_dir)
     return flang_intrinsics_dir
 
+
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path",flang_intrinsics_dir]
+    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
-    extra_intrinsics_search_args += ["-fintrinsic-modules-path", config.flang_intrinsic_modules_dir]
-    lit_config.note(f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}")
+    extra_intrinsics_search_args += [
+        "-fintrinsic-modules-path",
+        config.flang_intrinsic_modules_dir,
+    ]
+    lit_config.note(
+        f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
+    )
 
-config.substitutions.append(("%intrinsic_module_flags", ' '.join(intrinsics_search_args + extra_intrinsics_search_args)))
+config.substitutions.append(
+    (
+        "%intrinsic_module_flags",
+        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
+    )
+)
 
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
     ToolSubst(
         "bbc",
-         command=FindTool("bbc"),
-         extra_args=intrinsics_search_args + extra_intrinsics_search_args,
-         unresolved="fatal",
+        command=FindTool("bbc"),
+        extra_args=intrinsics_search_args + extra_intrinsics_search_args,
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang",
@@ -174,12 +188,11 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-
     # For not having intrinsic search paths to be added implicitly
     ToolSubst(
-         "%bbc_bare",
-         command=FindTool("bbc"),
-         unresolved="fatal",
+        "%bbc_bare",
+        command=FindTool("bbc"),
+        unresolved="fatal",
     ),
     ToolSubst(
         "%flang_bare",
@@ -225,10 +238,10 @@ def get_resource_module_intrinsic_dir():
 # If intrinsic modules are not available, disable tests unless they are marked as 'module-independent'.
 config.available_features.add("module-independent")
 if config.have_flangrt_mod or config.flang_intrinsic_modules_dir:
-   config.available_features.add("flangrt-modules")
+    config.available_features.add("flangrt-modules")
 else:
-   lit_config.warning(f"Intrinsic modules not available: disabling most tests")
-   config.limit_to_features.add("module-independent")
+    lit_config.warning(f"Intrinsic modules not available: disabling most tests")
+    config.limit_to_features.add("module-independent")
 
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 openmp_flags_substitution = "-fopenmp"

>From 60ad12fb5bf75fbb33348846bbbd1929eb15ea6d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 16:58:49 +0200
Subject: [PATCH 03/16] Minimize changes in .f90 files

---
 flang-rt/lib/runtime/__fortran_builtins.f90  | 2 +-
 flang-rt/lib/runtime/__fortran_type_info.f90 | 7 ++-----
 flang-rt/lib/runtime/cooperative_groups.f90  | 2 +-
 flang-rt/lib/runtime/ieee_arithmetic.f90     | 3 ++-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/__fortran_builtins.f90 b/flang-rt/lib/runtime/__fortran_builtins.f90
index d5e55b5d95020..7bb078c0b428e 100644
--- a/flang-rt/lib/runtime/__fortran_builtins.f90
+++ b/flang-rt/lib/runtime/__fortran_builtins.f90
@@ -12,7 +12,7 @@
 ! from being usable on INTRINSIC statements, and force the program
 ! to USE the standard intrinsic modules in order to access the
 ! standard names of the procedures.
-module    __fortran_builtins
+module __fortran_builtins
   implicit none
 
   ! Set PRIVATE by default to explicitly only export what is meant
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 2f936c3787a61..6af2a5a5e30ff 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -12,11 +12,8 @@
 ! in order to generate description tables for all other derived types.
 
 module __fortran_type_info
-#if 0
-  use       __fortran_builtins, &
-#else
-  use, intrinsic ::    __fortran_builtins, &
-#endif
+
+  use, intrinsic :: __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 82d1e0fe84042..fb6d24c8f7bc3 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -11,7 +11,7 @@
 module cooperative_groups
 
 use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, make explicit for CMake
+use :: cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/ieee_arithmetic.f90 b/flang-rt/lib/runtime/ieee_arithmetic.f90
index b3288a5cdd69f..bad290ab30097 100644
--- a/flang-rt/lib/runtime/ieee_arithmetic.f90
+++ b/flang-rt/lib/runtime/ieee_arithmetic.f90
@@ -8,7 +8,7 @@
 
 ! Fortran 2018 Clause 17
 
-#include   '../include/flang/Runtime/magic-numbers.h'
+#include '../../../flang/include/flang/Runtime/magic-numbers.h'
 
 module ieee_arithmetic
   ! F18 Clause 17.1p1:
@@ -336,6 +336,7 @@ end subroutine ieee_get_underflow_mode_l##GKIND;
   public ::  ieee_get_underflow_mode
 #undef IEEE_GET_UNDERFLOW_MODE_L
 
+! Workaround for https://github.com/llvm/llvm-project/issues/139297
 ! #define IEEE_IS_FINITE_R(XKIND) elemental logical function ieee_is_finite_a##XKIND(x);  real(XKIND), intent(in) :: x;  !dir$ ignore_tkr(d) x;   end function ieee_is_finite_a##XKIND;
   interface ieee_is_finite
 elemental logical function ieee_is_finite_a2(x); real(2), intent(in) :: x;

>From e58c524bd588484e99163899241d55be316b719a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 17:05:16 +0200
Subject: [PATCH 04/16] Move CMake 3.20 workaround to where it is needed

---
 flang-rt/CMakeLists.txt | 34 ----------------------------------
 runtimes/CMakeLists.txt | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 09f4f9e7213f1..61fb9e744bce9 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 2dcc68b80b07c..a49453a5fcf67 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -108,7 +108,6 @@ endif()
 # paths that are inside the build directory disables this behavior.
 #
 # See https://gitlab.kitware.com/cmake/cmake/-/issues/19227 for further details.
-# FIXME: This assumes compiling libc++, but creates problems for every library that uses the C++ standard library
 
 function(filter_prefixed list prefix outvar)
   foreach(str ${list})
@@ -335,6 +334,39 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 7d1a8bee595443112031604bf9e7bf43f3a2635a Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Thu, 17 Jul 2025 19:39:06 +0200
Subject: [PATCH 05/16] Don't forget to enable Fortran

---
 flang-rt/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 61fb9e744bce9..fd6ad5a170934 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,7 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"

>From adc01493eaa6785052dc794348149152c2c92bc3 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 00:57:01 +0200
Subject: [PATCH 06/16] enable_fortran test for CMake 3.20

---
 flang-rt/CMakeLists.txt      | 2 --
 llvm/runtimes/CMakeLists.txt | 1 -
 runtimes/CMakeLists.txt      | 5 ++++-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index fd6ad5a170934..5ffa1bcc34a41 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -159,8 +159,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-include(CheckFortranSourceCompiles)
-include(CMakePushCheckState)
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 96391ed70133e..2f2f6db16255e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -525,7 +525,6 @@ if(build_runtimes)
     endforeach()
   endif()
 
-  # TODO: Also enable Fortran for per-target runtimes needed for cross-compilation.
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a49453a5fcf67..987d173a283be 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -338,7 +338,8 @@ cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 # LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
 if (CMAKE_VERSION VERSION_LESS "3.24")
   cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
     include(CMakeForceCompiler)
     CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
 
@@ -364,6 +365,8 @@ if (CMAKE_VERSION VERSION_LESS "3.24")
 
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
     set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+
+    enable_language(Fortran)
   endif ()
 endif ()
 

>From 498d60cf39c237c654c0cfddbe444d63bc87bc6c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 09:57:18 +0200
Subject: [PATCH 07/16] CI test

---
 flang-rt/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 5ffa1bcc34a41..d987115c8e9e8 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,6 +23,40 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
+message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
 enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
@@ -159,6 +193,7 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)

>From 1bb0144eb21c2b1c8e1e66029d5573d8f5619c4c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 14:24:44 +0200
Subject: [PATCH 08/16] Enforce REAL(16) support

---
 flang-rt/CMakeLists.txt      | 45 ++++-------------------
 llvm/runtimes/CMakeLists.txt |  8 ++++
 runtimes/CMakeLists.txt      | 71 ++++++++++++++++++------------------
 3 files changed, 51 insertions(+), 73 deletions(-)

diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index d987115c8e9e8..bfbd0af0c31dc 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -23,41 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-message("CMAKE_Fortran_PREPROCESS_SOURCE1: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-    message("CMAKE_Fortran_PREPROCESS_SOURCE2: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
@@ -79,6 +44,11 @@ include(CMakePushCheckState)
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
+# Fortran compiler not optional for building Flang-RT
+enable_language(Fortran)
+
+flang_module_fortran_enable()
+
 
 #################
 # Build Options #
@@ -193,7 +163,9 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-message("CMAKE_Fortran_PREPROCESS_SOURCE3: ${CMAKE_Fortran_PREPROCESS_SOURCE}")
+
+# Look for support of REAL(16), if not already defined via command line.
+# NOTE: Does not work with Flang and CMake < 3.24
 cmake_push_check_state(RESET)
 set(CMAKE_REQUIRED_FLAGS "-ffree-form")
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
@@ -206,7 +178,6 @@ check_fortran_source_compiles([[
 )
 cmake_pop_check_state()
 
-flang_module_fortran_enable()
 
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 2f2f6db16255e..2bf2b0ee8ed50 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -528,6 +528,14 @@ if(build_runtimes)
   if ("openmp" IN_LIST LLVM_ENABLE_RUNTIMES AND "flang" IN_LIST LLVM_ENABLE_PROJECTS)
     list(APPEND extra_args ENABLE_FORTRAN)
   endif()
+  if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
+    # Ensure REAL(16) support in runtimes to be consistent with compiler
+    if (FLANG_RUNTIME_F128_MATH_LIB OR HAVE_LDBL_MANT_DIG_113)
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=TRUE")
+   else ()
+      list(APPEND extra_cmake_args "-DFORTRAN_SUPPORTS_REAL16=FALSE")
+   endif ()
+  endif ()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager clang-nvlink-wrapper)
       if(TARGET ${dep})
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 987d173a283be..51214a46f558e 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -94,6 +94,41 @@ include(CheckCXXCompilerFlag)
 include(ExtendPath)
 
 
+# CMake 3.24 is the first version of CMake that directly recognizes Flang.
+# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang, if used.
+if (CMAKE_VERSION VERSION_LESS "3.24")
+  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
+  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
+      _Fortran_COMPILER_STEM STREQUAL "flang")
+    include(CMakeForceCompiler)
+    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
+
+    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
+    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
+
+    set(CMAKE_Fortran_SUBMODULE_SEP "-")
+    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
+
+    set(CMAKE_Fortran_PREPROCESS_SOURCE
+      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+
+    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
+    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
+
+    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
+    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
+    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
+
+    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
+
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
+    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
+  endif ()
+endif ()
+
+
 # Determine whether we are in the runtimes/runtimes-bins directory of a
 # bootstrapping build.
 set(LLVM_TREE_AVAILABLE OFF)
@@ -334,42 +369,6 @@ cmake_path(NORMAL_PATH RUNTIMES_OUTPUT_RESOURCE_LIB_DIR)
 cmake_path(NORMAL_PATH RUNTIMES_INSTALL_RESOURCE_LIB_PATH)
 
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR
-      _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-
-    enable_language(Fortran)
-  endif ()
-endif ()
-
 # Enable building Fortran modules
 # * Set up the Fortran compiler
 # * Determine files location in the Clang/Flang resource dir

>From 8ec6d03939c3a7e2f79bb3ea5911b7607028dd7c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Fri, 18 Jul 2025 21:52:02 +0200
Subject: [PATCH 09/16] Also add dependency barrier for WIN32

---
 flang-rt/lib/runtime/CMakeLists.txt | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 1b8114c102205..20f5d84bb2b69 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -281,14 +281,18 @@ else()
   function (add_win_flangrt_runtime libtype suffix msvc_lib)
     set(name "flang_rt.runtime.${suffix}")
 
-    add_flangrt_library(${name}.intrinsics OBJECT
+    add_flangrt_library(${name}.intrinsics.obj OBJECT
       ${intrinsics_sources}
     )
+    add_custom_target(${name}.intrinsics
+      COMMENT "Intrinsic module dependency barrier"
+    )
+    add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
 
     add_flangrt_library(${name} ${libtype}
-        ${sources} $<TARGET_OBJECTS:${name}.intrinsics>
+        ${sources} $<TARGET_OBJECTS:${name}.intrinsics.obj>
         ${ARGN}
-        LINK_LIBRARIES ${name}.intrinsics ${Backtrace_LIBRARY}
+        LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
 
@@ -311,8 +315,9 @@ else()
     endif ()
 
     get_target_property(compile_target ${name}.compile ALIASED_TARGET)
-    flang_module_target(${name}.intrinsics ${is_public})
+    flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
+    add_dependencies(${compile_target} ${name}.intrinsics)
     add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")

>From 580a0c56535a9cd1c65fd5a5e6309b73c36ae8b6 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:05:34 +0200
Subject: [PATCH 10/16] Dependency barrier test

---
 flang-rt/lib/runtime/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 20f5d84bb2b69..bdd4318832473 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,6 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
+      COMMAND echo "Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)
@@ -295,6 +296,7 @@ else()
         LINK_LIBRARIES ${name}.intrinsics.obj ${Backtrace_LIBRARY}
         ADDITIONAL_HEADERS ${public_headers} ${private_headers}
       )
+    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
 
     if (msvc_lib)
       set_target_properties(${name}
@@ -311,14 +313,13 @@ else()
       set(is_public "")
     else ()
       set(is_public PUBLIC)
+      add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
       set(_has_public_intrinsics "YES" PARENT_SCOPE)
     endif ()
 
-    get_target_property(compile_target ${name}.compile ALIASED_TARGET)
     flang_module_target(${name}.intrinsics.obj ${is_public})
     flang_module_target(${compile_target} ${is_public})
     add_dependencies(${compile_target} ${name}.intrinsics)
-    add_dependencies(flang-rt-mod ${name}.intrinsics ${compile_target})
 
     enable_cuda_compilation(${name} "${supported_sources}")
     enable_omp_offload_compilation(${name} "${supported_sources}")

>From 979691a5bba4888be8c7c82d1bed4e8cdc71fff9 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Sat, 19 Jul 2025 01:06:40 +0200
Subject: [PATCH 11/16] Dependency barrier info

---
 flang-rt/lib/runtime/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index bdd4318832473..24b84b11f9513 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -285,7 +285,7 @@ else()
       ${intrinsics_sources}
     )
     add_custom_target(${name}.intrinsics
-      COMMAND echo "Dependency barrier"
+      COMMAND echo "${name} Dependency barrier"
       COMMENT "Intrinsic module dependency barrier"
     )
     add_dependencies(${name}.intrinsics ${name}.intrinsics.obj)

>From b4adeab58e615b9059c0a6e5bbcb376d1fe21bb2 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Mon, 21 Jul 2025 21:56:26 +0200
Subject: [PATCH 12/16] Avoid unrelated changes

---
 clang/include/clang/Driver/Driver.h     | 1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 9343fed36b6ac..14e1e644e51aa 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -403,6 +403,7 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
+
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
   /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 688b5aacc4bcd..609ba27bc212b 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1376,10 +1376,10 @@ getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
     return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
 
   if (!options.skipExternalRttiDefinition)
-    fir::emitFatalError(
-        loc, llvm::Twine("runtime derived type info descriptor of '") + name +
-                 "' was not generated and skipExternalRttiDefinition and "
-                 "ignoreMissingTypeDescriptors options are not set");
+    fir::emitFatalError(loc,
+                        "runtime derived type info descriptor was not "
+                        "generated and skipExternalRttiDefinition and "
+                        "ignoreMissingTypeDescriptors options are not set");
 
   // Rtti for a derived type defined in another compilation unit and for which
   // rtti was not defined in lowering because of the skipExternalRttiDefinition

>From c40f43c98ff7ddbcb52c1ac35210320926369b2c Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 14:16:19 +0200
Subject: [PATCH 13/16] Use -fintrinsic-modules-path=

---
 flang/test/lit.cfg.py   | 20 +++-----------------
 runtimes/CMakeLists.txt |  2 +-
 2 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index b05eba8da0b0c..bbf9a5a9f277f 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -147,26 +147,18 @@ def get_resource_module_intrinsic_dir():
 
 intrinsics_search_args = []
 if flang_intrinsics_dir := get_resource_module_intrinsic_dir():
-    intrinsics_search_args += ["-fintrinsic-modules-path", flang_intrinsics_dir]
+    intrinsics_search_args += [f"-fintrinsic-modules-path={flang_intrinsics_dir}"]
     lit_config.note(f"using default module intrinsics: {flang_intrinsics_dir}")
 
 extra_intrinsics_search_args = []
 if config.flang_intrinsic_modules_dir:
     extra_intrinsics_search_args += [
-        "-fintrinsic-modules-path",
-        config.flang_intrinsic_modules_dir,
+        f"-fintrinsic-modules-path={config.flang_intrinsic_modules_dir}",
     ]
     lit_config.note(
         f"using extra module intrinsics: {config.flang_intrinsic_modules_dir}"
     )
 
-config.substitutions.append(
-    (
-        "%intrinsic_module_flags",
-        " ".join(intrinsics_search_args + extra_intrinsics_search_args),
-    )
-)
-
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
@@ -193,13 +185,7 @@ def get_resource_module_intrinsic_dir():
         "%bbc_bare",
         command=FindTool("bbc"),
         unresolved="fatal",
-    ),
-    ToolSubst(
-        "%flang_bare",
-        command=FindTool("flang"),
-        extra_args=isysroot_flag,
-        unresolved="fatal",
-    ),
+    )
 ]
 
 # Flang has several unimplemented features. TODO messages are used to mark
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 51214a46f558e..0832767505fe1 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -454,7 +454,7 @@ function (flang_module_target tgtname)
 
   # Let it find the other public module files
   target_compile_options(${tgtname} PRIVATE
-      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-fintrinsic-modules-path;SHELL:${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
+      "$<$<COMPILE_LANGUAGE:Fortran>:-fintrinsic-modules-path=${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
     )
 
   if (ARG_PUBLIC)

>From 18422d4d6ea24aeb29a4fece11ba7b509fbd71d6 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 16:50:58 +0200
Subject: [PATCH 14/16] Rework intrinsic-module-path.f90 test

---
 flang/test/Driver/Inputs/ieee_arithmetic.mod |  1 +
 flang/test/Driver/Inputs/iso_fortran_env.mod |  1 +
 flang/test/Driver/intrinsic-module-path.f90  | 54 ++++++++++++++++----
 flang/test/lit.cfg.py                        |  2 +-
 4 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/flang/test/Driver/Inputs/ieee_arithmetic.mod b/flang/test/Driver/Inputs/ieee_arithmetic.mod
index 30fd57801970b..451d1af62a941 100644
--- a/flang/test/Driver/Inputs/ieee_arithmetic.mod
+++ b/flang/test/Driver/Inputs/ieee_arithmetic.mod
@@ -1,5 +1,6 @@
 ! DUMMY module
 ! Added for testing purposes. The contents of this file are currently not relevant.
+! Using this file file cause an error because of missing checksum
 module ieee_arithmetic
 type::ieee_round_type
 integer(1),private::mode=0_1
diff --git a/flang/test/Driver/Inputs/iso_fortran_env.mod b/flang/test/Driver/Inputs/iso_fortran_env.mod
index 689297d52027b..ad501c2d9c1b8 100644
--- a/flang/test/Driver/Inputs/iso_fortran_env.mod
+++ b/flang/test/Driver/Inputs/iso_fortran_env.mod
@@ -1,5 +1,6 @@
 ! DUMMY module
 ! Added for testing purposes. The contents of this file are currently not relevant.
+! Using this file file cause an error because of missing checksum
 module iso_fortran_env
 use __fortran_builtins,only:event_type=>__builtin_event_type
 use __fortran_builtins,only:lock_type=>__builtin_lock_type
diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.f90
index 3e696ce5d08e0..3317eb776f0a1 100644
--- a/flang/test/Driver/intrinsic-module-path.f90
+++ b/flang/test/Driver/intrinsic-module-path.f90
@@ -1,23 +1,55 @@
 ! Ensure argument -fintrinsic-modules-path works as expected.
-! WITHOUT the option, the default location for the module is checked and no error generated.
-! With the option GIVEN, the module with the same name is PREPENDED, and considered over the
-! default one, causing a CHECKSUM error.
 
 !-----------------------------------------
-! FRONTEND FLANG DRIVER (flang -fc1)
+! FLANG DRIVER
 !-----------------------------------------
-! RUN:     %flang_bare -fsyntax-only                                     %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=WITHOUT --allow-empty
-! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
-! RUN: not %flang_bare -fsyntax-only -fintrinsic-modules-path=%S/Inputs/ %s %intrinsic_module_flags 2>&1 | FileCheck %s --check-prefix=GIVEN
+! NOTE: Depending on how Flang is built, the default intrinsics may have higher
+!       or lower priority than -fintrinsic-modules-path added here. Using
+!       basictestmoduleone.mod from Inputs/module-dir/ will trigger an error.
 
-! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
-! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
+! RUN:     %flang -fsyntax-only -### %s 2>&1 | FileCheck %s --check-prefix=DEFAULTPATH
 
-! GIVEN: error: Cannot use module file for module 'ieee_arithmetic': File has invalid checksum
-! GIVEN: error: Cannot use module file for module 'iso_fortran_env': File has invalid checksum
+! RUN:     %flang -fsyntax-only -DINTRINSICS_DEFAULT %s
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTONE %s 2>&1 | FileCheck %s --check-prefix=NOINPUTONE
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTTWO %s 2>&1 | FileCheck %s --check-prefix=NOINPUTTWO
+! RUN:     %flang -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN:     %flang -fsyntax-only -DINTRINSICS_INPUTONE -fintrinsic-modules-path=%S/Inputs/ %s
+! RUN:     %flang -fsyntax-only -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/ -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN: not %flang -fsyntax-only -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir/ -fintrinsic-modules-path=%S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=WRONGINPUTONE
 
 
+!-----------------------------------------
+! FLANG FRONTEND (flang -fc1)
+!-----------------------------------------
+! NOTE: %flang_cc1 the default intrinsics path always has higher priority than
+!       -fintrinsic-modules-path added here. Accidentally using
+!       ieee_arithmetic/iso_fortran_env from the Inputs/ directory will trigger
+!       an error (e.g. when the default intrinsics dir is empty).
+
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT %s
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE %s 2>&1 | FileCheck %s --check-prefix=NOINPUTONE
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO %s 2>&1 | FileCheck %s --check-prefix=NOINPUTTWO
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir %s
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -fintrinsic-modules-path=%S/Inputs/ %s
+! RUN:     %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/ -fintrinsic-modules-path=%S/Inputs/module-dir/ %s
+! RUN: not %flang_fc1 -fsyntax-only -DINTRINSICS_DEFAULT -DINTRINSICS_INPUTONE -DINTRINSICS_INPUTTWO -fintrinsic-modules-path=%S/Inputs/module-dir -fintrinsic-modules-path=%S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=WRONGINPUTONE
+
+
+! DEFAULTPATH: flang{{.*}}-fc1{{.*}}-fintrinsic-modules-path
+
+! NOINPUTONE: Source file 'basictestmoduleone.mod' was not found
+! NOINPUTTWO: Source file 'basictestmoduletwo.mod' was not found
+! WRONGINPUTONE: 't1' not found in module 'basictestmoduleone'
+
 program test_intrinsic_module_path
+#ifdef INTRINSICS_DEFAULT
    use ieee_arithmetic, only: ieee_round_type
    use iso_fortran_env, only: team_type, event_type, lock_type
+#endif
+#ifdef INTRINSICS_INPUTONE
+   use basictestmoduleone, only: t1
+#endif
+#ifdef INTRINSICS_INPUTTWO
+   use basictestmoduletwo, only: t2
+#endif
 end program
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index bbf9a5a9f277f..ba1e5d55b503c 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -180,7 +180,7 @@ def get_resource_module_intrinsic_dir():
         extra_args=["-fc1"] + intrinsics_search_args + extra_intrinsics_search_args,
         unresolved="fatal",
     ),
-    # For not having intrinsic search paths to be added implicitly
+    # Do not implicitly add intrinsic search paths
     ToolSubst(
         "%bbc_bare",
         command=FindTool("bbc"),

>From 9143388ef4c1c2bc9b09595c69ed85eac1daf2c8 Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 17:12:30 +0200
Subject: [PATCH 15/16] Remove hint to reduce diff size

---
 clang/include/clang/Driver/Driver.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 451521eb70760..78a4c5738ae66 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -405,8 +405,6 @@ class Driver {
 
   /// Takes the path to a binary that's either in bin/ or lib/ and returns
   /// the path to clang's resource directory.
-  /// Do not pass argv[0] as argument, llvm-lit does not adjust argv[0] to the
-  /// changing cwd. Use llvm::sys::fs::getMainExecutable instead.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,

>From 68efc67e99218a9ccc01df0cfd7875ade07815ee Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project at meinersbur.de>
Date: Tue, 22 Jul 2025 19:49:07 +0200
Subject: [PATCH 16/16] Enable preprocessor for test

---
 .../{intrinsic-module-path.f90 => intrinsic-module-path.F90}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename flang/test/Driver/{intrinsic-module-path.f90 => intrinsic-module-path.F90} (100%)

diff --git a/flang/test/Driver/intrinsic-module-path.f90 b/flang/test/Driver/intrinsic-module-path.F90
similarity index 100%
rename from flang/test/Driver/intrinsic-module-path.f90
rename to flang/test/Driver/intrinsic-module-path.F90


From flang-commits at lists.llvm.org  Tue Jul 22 16:49:53 2025
From: flang-commits at lists.llvm.org (Razvan Lupusoru via flang-commits)
Date: Tue, 22 Jul 2025 16:49:53 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [draft][acc][flang] Add recipes to
 data entry operations (PR #149210)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149210@github.com>
Message-ID: <688023a1.a70a0220.31f104.d965@mx.google.com>


https://github.com/razvanlupusoru updated https://github.com/llvm/llvm-project/pull/149210

>From 4e24d9765993dcece195f59b0761a87fed540e71 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Wed, 16 Jul 2025 15:44:55 -0700
Subject: [PATCH 1/6] [draft][acc][flang] Add recipes to data entry operations

Currently recipes are held as part of the constructs. But
the recipes are needed specifically for materializing semantics
of the data clauses. There is no reason to not have this on the
data clauses themselves.

TODO: Fix the IR in flang tests which do not expect new operand
TODO: Deprecate the recipe list held on the constructs
---
 flang/lib/Lower/OpenACC.cpp                   |  8 +++++
 mlir/include/mlir/Dialect/OpenACC/OpenACC.h   |  3 ++
 .../mlir/Dialect/OpenACC/OpenACCOps.td        | 24 +++++++++++--
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       | 34 +++++++++++++++++++
 4 files changed, 66 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 39e4444cde4e3..c2a884d8e4c3a 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -1340,6 +1340,8 @@ static void genPrivatizationRecipes(
           builder, operandLocation, info.addr, asFortran, bounds, true,
           /*implicit=*/false, mlir::acc::DataClause::acc_private, retTy, async,
           asyncDeviceTypes, asyncOnlyDeviceTypes, /*unwrapBoxAddr=*/true);
+      op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                                recipe.getSymName().str()));
       dataOperands.push_back(op.getAccVar());
     } else {
       std::string suffix =
@@ -1353,6 +1355,8 @@ static void genPrivatizationRecipes(
           /*implicit=*/false, mlir::acc::DataClause::acc_firstprivate, retTy,
           async, asyncDeviceTypes, asyncOnlyDeviceTypes,
           /*unwrapBoxAddr=*/true);
+      op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                                recipe.getSymName().str()));
       dataOperands.push_back(op.getAccVar());
     }
     privatizationRecipes.push_back(mlir::SymbolRefAttr::get(
@@ -1787,6 +1791,8 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
     mlir::acc::ReductionRecipeOp recipe =
         Fortran::lower::createOrGetReductionRecipe(
             builder, recipeName, operandLocation, ty, mlirOp, bounds);
+    op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                              recipe.getSymName().str()));
     reductionRecipes.push_back(mlir::SymbolRefAttr::get(
         builder.getContext(), recipe.getSymName().str()));
     reductionOperands.push_back(op.getAccVar());
@@ -2038,6 +2044,8 @@ privatizeIv(Fortran::lower::AbstractConverter &converter,
         builder, loc, ivValue, asFortran, {}, true, /*implicit=*/true,
         mlir::acc::DataClause::acc_private, ivValue.getType(),
         /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
+    op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
+                                              recipe.getSymName().str()));
     privateOp = op.getOperation();
 
     privateOperands.push_back(op.getAccVar());
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index 4eb666239d4e4..b3a29d10e10e4 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -151,6 +151,9 @@ mlir::ValueRange getDataOperands(mlir::Operation *accOp);
 /// Used to get a mutable range iterating over the data operands.
 mlir::MutableOperandRange getMutableDataOperands(mlir::Operation *accOp);
 
+/// Used to get the recipe attribute from a data clause operation.
+mlir::SymbolRefAttr getRecipe(mlir::Operation *accOp);
+
 /// Used to obtain the enclosing compute construct operation that contains
 /// the provided `region`. Returns nullptr if no compute construct operation
 /// is found. The returns operation is one of types defined by
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 66378f116784e..dd395912f9e7b 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -514,7 +514,8 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           DefaultValuedAttr<BoolAttr, "false">:$implicit,
           DefaultValuedAttr<OpenACC_DataClauseModifierAttr,
             "mlir::acc::DataClauseModifier::none">:$modifiers,
-          OptionalAttr<StrAttr>:$name));
+          OptionalAttr<StrAttr>:$name,
+          OptionalAttr<SymbolRefAttr>:$recipe));
 
   let description = !strconcat(extraDescription, [{
     Description of arguments:
@@ -623,7 +624,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
-          /*name=*/nullptr);
+          /*name=*/nullptr, /*recipe=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Value":$var,
                    "bool":$structured, "bool":$implicit,
@@ -641,7 +642,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
-          /*name=*/$_builder.getStringAttr(name));
+          /*name=*/$_builder.getStringAttr(name), /*recipe=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Type":$accVarType, "::mlir::Value":$var,
                    "::mlir::Type":$varType, "::mlir::Value":$varPtrPtr,
@@ -652,10 +653,27 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
                    "::mlir::acc::DataClause":$dataClause, "bool":$structured,
                    "bool":$implicit, "::mlir::StringAttr":$name),
       [{
+        // Builder provided to ease transition for new data clause modifiers operand.
         build($_builder, $_state, accVarType, var, varType, varPtrPtr, bounds,
           asyncOperands, asyncOperandsDeviceType, asyncOnly, dataClause,
           structured, implicit, ::mlir::acc::DataClauseModifier::none, name);
       }]>,
+    OpBuilder<(ins "::mlir::Type":$accVarType, "::mlir::Value":$var,
+                   "::mlir::Type":$varType, "::mlir::Value":$varPtrPtr,
+                   "::mlir::ValueRange":$bounds,
+                   "::mlir::ValueRange":$asyncOperands,
+                   "::mlir::ArrayAttr":$asyncOperandsDeviceType,
+                   "::mlir::ArrayAttr":$asyncOnly,
+                   "::mlir::acc::DataClause":$dataClause, "bool":$structured,
+                   "bool":$implicit,
+                   "::mlir::acc::DataClauseModifier":$modifiers,
+                   "::mlir::StringAttr":$name),
+      [{
+        // Builder provided to simplify building after recipe operand was added.
+        build($_builder, $_state, accVarType, var, varType, varPtrPtr, bounds,
+              asyncOperands, asyncOperandsDeviceType, asyncOnly, dataClause,
+              structured, implicit, modifiers, name, /*recipe=*/nullptr);
+      }]>,
     ];
 }
 
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index f2eab62b286af..eb56d2d8933a0 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -332,6 +332,13 @@ checkValidModifier(Op op, acc::DataClauseModifier validModifiers) {
   return success();
 }
 
+template <typename Op>
+static LogicalResult checkNoRecipe(Op op) {
+  if (op.getRecipe().has_value())
+    return op.emitError("no recipes are allowed");
+  return success();
+}
+
 static ParseResult parseVar(mlir::OpAsmParser &parser,
                             OpAsmParser::UnresolvedOperand &var) {
   // Either `var` or `varPtr` keyword is required.
@@ -1063,6 +1070,24 @@ checkSymOperandList(Operation *op, std::optional<mlir::ArrayAttr> attributes,
       return op->emitOpError()
              << "expected as many " << symbolName << " symbol reference as "
              << operandName << " operands";
+    if (attributes) {
+      for (auto operandAndAttribute : llvm::zip(operands, *attributes)) {
+        mlir::Value operand = std::get<0>(operandAndAttribute);
+        if (auto *definingOp = operand.getDefiningOp()) {
+          mlir::SymbolRefAttr operandRecipe = getRecipe(definingOp);
+          // If the operand operation has a recipe - check that it is consistent
+          // with the one recorded in the construct.
+          if (operandRecipe) {
+            if (operandRecipe.getLeafReference().compare(
+                    llvm::cast<mlir::SymbolRefAttr>(
+                        std::get<1>(operandAndAttribute))
+                        .getLeafReference()) != 0)
+              return op->emitOpError() << "expected consistent recipe for "
+                                       << operandName << " operand";
+          }
+        }
+      }
+    }
   } else {
     if (attributes)
       return op->emitOpError()
@@ -4075,6 +4100,15 @@ mlir::acc::getMutableDataOperands(mlir::Operation *accOp) {
   return dataOperands;
 }
 
+mlir::SymbolRefAttr mlir::acc::getRecipe(mlir::Operation *accOp) {
+  auto recipe{
+      llvm::TypeSwitch<mlir::Operation *, mlir::SymbolRefAttr>(accOp)
+          .Case<ACC_DATA_ENTRY_OPS>(
+              [&](auto entry) { return entry.getRecipeAttr(); })
+          .Default([&](mlir::Operation *) { return mlir::SymbolRefAttr{}; })};
+  return recipe;
+}
+
 mlir::Operation *mlir::acc::getEnclosingComputeOp(mlir::Region &region) {
   mlir::Operation *parentOp = region.getParentOp();
   while (parentOp) {

>From ad2a3027a3507f6a7a9b6ca62ab8828200493433 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Tue, 22 Jul 2025 16:16:30 -0700
Subject: [PATCH 2/6] Remove construct recipes and fix all tests

---
 flang/lib/Lower/OpenACC.cpp                   |  46 +----
 flang/test/Lower/OpenACC/acc-kernels-loop.f90 |   4 +-
 flang/test/Lower/OpenACC/acc-loop.f90         |  73 ++++---
 .../test/Lower/OpenACC/acc-parallel-loop.f90  |  12 +-
 flang/test/Lower/OpenACC/acc-parallel.f90     |  12 +-
 .../acc-private-unwrap-defaultbounds.f90      |  63 +++---
 flang/test/Lower/OpenACC/acc-private.f90      |  69 ++++---
 .../acc-reduction-unwrap-defaultbounds.f90    | 144 ++++++-------
 flang/test/Lower/OpenACC/acc-reduction.f90    | 144 ++++++-------
 flang/test/Lower/OpenACC/acc-serial-loop.f90  |  12 +-
 flang/test/Lower/OpenACC/acc-serial.f90       |  12 +-
 .../mlir/Dialect/OpenACC/OpenACCOps.td        |  46 ++---
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       | 194 +++++++-----------
 mlir/test/Dialect/OpenACC/invalid.mlir        |  10 +-
 mlir/test/Dialect/OpenACC/legalize-data.mlir  |  24 +--
 mlir/test/Dialect/OpenACC/ops.mlir            | 107 ++++++----
 16 files changed, 476 insertions(+), 496 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index ab27678e9ab95..50571a70e3aaf 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -1323,7 +1323,6 @@ static void genPrivatizationRecipes(
     Fortran::semantics::SemanticsContext &semanticsContext,
     Fortran::lower::StatementContext &stmtCtx,
     llvm::SmallVectorImpl<mlir::Value> &dataOperands,
-    llvm::SmallVector<mlir::Attribute> &privatizationRecipes,
     llvm::ArrayRef<mlir::Value> async,
     llvm::ArrayRef<mlir::Attribute> asyncDeviceTypes,
     llvm::ArrayRef<mlir::Attribute> asyncOnlyDeviceTypes) {
@@ -1377,8 +1376,6 @@ static void genPrivatizationRecipes(
                                                 recipe.getSymName().str()));
       dataOperands.push_back(op.getAccVar());
     }
-    privatizationRecipes.push_back(mlir::SymbolRefAttr::get(
-        builder.getContext(), recipe.getSymName().str()));
   }
 }
 
@@ -1757,7 +1754,6 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
               Fortran::semantics::SemanticsContext &semanticsContext,
               Fortran::lower::StatementContext &stmtCtx,
               llvm::SmallVectorImpl<mlir::Value> &reductionOperands,
-              llvm::SmallVector<mlir::Attribute> &reductionRecipes,
               llvm::ArrayRef<mlir::Value> async,
               llvm::ArrayRef<mlir::Attribute> asyncDeviceTypes,
               llvm::ArrayRef<mlir::Attribute> asyncOnlyDeviceTypes) {
@@ -1811,8 +1807,6 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
             builder, recipeName, operandLocation, ty, mlirOp, bounds);
     op.setRecipeAttr(mlir::SymbolRefAttr::get(builder.getContext(),
                                               recipe.getSymName().str()));
-    reductionRecipes.push_back(mlir::SymbolRefAttr::get(
-        builder.getContext(), recipe.getSymName().str()));
     reductionOperands.push_back(op.getAccVar());
   }
 }
@@ -2024,7 +2018,6 @@ privatizeIv(Fortran::lower::AbstractConverter &converter,
             llvm::SmallVector<mlir::Location> &ivLocs,
             llvm::SmallVector<mlir::Value> &privateOperands,
             llvm::SmallVector<mlir::Value> &ivPrivate,
-            llvm::SmallVector<mlir::Attribute> &privatizationRecipes,
             bool isDoConcurrent = false) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
@@ -2067,8 +2060,6 @@ privatizeIv(Fortran::lower::AbstractConverter &converter,
     privateOp = op.getOperation();
 
     privateOperands.push_back(op.getAccVar());
-    privatizationRecipes.push_back(mlir::SymbolRefAttr::get(
-        builder.getContext(), recipe.getSymName().str()));
   }
 
   // Map the new private iv to its symbol for the scope of the loop. bindSymbol
@@ -2161,7 +2152,6 @@ static mlir::acc::LoopOp createLoopOp(
   llvm::SmallVector<mlir::Value> tileOperands, privateOperands, ivPrivate,
       reductionOperands, cacheOperands, vectorOperands, workerNumOperands,
       gangOperands, lowerbounds, upperbounds, steps;
-  llvm::SmallVector<mlir::Attribute> privatizationRecipes, reductionRecipes;
   llvm::SmallVector<int32_t> tileOperandsSegments, gangOperandsSegments;
   llvm::SmallVector<int64_t> collapseValues;
 
@@ -2284,13 +2274,13 @@ static mlir::acc::LoopOp createLoopOp(
                        &clause.u)) {
       genPrivatizationRecipes<mlir::acc::PrivateRecipeOp>(
           privateClause->v, converter, semanticsContext, stmtCtx,
-          privateOperands, privatizationRecipes, /*async=*/{},
+          privateOperands, /*async=*/{},
           /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
     } else if (const auto *reductionClause =
                    std::get_if<Fortran::parser::AccClause::Reduction>(
                        &clause.u)) {
       genReductions(reductionClause->v, converter, semanticsContext, stmtCtx,
-                    reductionOperands, reductionRecipes, /*async=*/{},
+                    reductionOperands, /*async=*/{},
                     /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
     } else if (std::get_if<Fortran::parser::AccClause::Seq>(&clause.u)) {
       for (auto crtDeviceTypeAttr : crtDeviceTypes)
@@ -2368,7 +2358,7 @@ static mlir::acc::LoopOp createLoopOp(
 
       const auto &name = std::get<Fortran::parser::Name>(control.t);
       privatizeIv(converter, *name.symbol, currentLocation, ivTypes, ivLocs,
-                  privateOperands, ivPrivate, privatizationRecipes,
+                  privateOperands, ivPrivate,
                   isDoConcurrent);
 
       inclusiveBounds.push_back(true);
@@ -2407,7 +2397,7 @@ static mlir::acc::LoopOp createLoopOp(
       Fortran::semantics::Symbol &ivSym =
           bounds->name.thing.symbol->GetUltimate();
       privatizeIv(converter, ivSym, currentLocation, ivTypes, ivLocs,
-                  privateOperands, ivPrivate, privatizationRecipes);
+                  privateOperands, ivPrivate);
 
       inclusiveBounds.push_back(true);
 
@@ -2489,14 +2479,6 @@ static mlir::acc::LoopOp createLoopOp(
   if (!autoDeviceTypes.empty())
     loopOp.setAuto_Attr(builder.getArrayAttr(autoDeviceTypes));
 
-  if (!privatizationRecipes.empty())
-    loopOp.setPrivatizationRecipesAttr(
-        mlir::ArrayAttr::get(builder.getContext(), privatizationRecipes));
-
-  if (!reductionRecipes.empty())
-    loopOp.setReductionRecipesAttr(
-        mlir::ArrayAttr::get(builder.getContext(), reductionRecipes));
-
   if (!collapseValues.empty())
     loopOp.setCollapseAttr(builder.getI64ArrayAttr(collapseValues));
   if (!collapseDeviceTypes.empty())
@@ -2618,8 +2600,6 @@ static Op createComputeOp(
 
   llvm::SmallVector<mlir::Value> reductionOperands, privateOperands,
       firstprivateOperands;
-  llvm::SmallVector<mlir::Attribute> privatizationRecipes,
-      firstPrivatizationRecipes, reductionRecipes;
 
   // Self clause has optional values but can be present with
   // no value as well. When there is no value, the op has an attribute to
@@ -2827,14 +2807,14 @@ static Op createComputeOp(
       if (!combinedConstructs)
         genPrivatizationRecipes<mlir::acc::PrivateRecipeOp>(
             privateClause->v, converter, semanticsContext, stmtCtx,
-            privateOperands, privatizationRecipes, async, asyncDeviceTypes,
+            privateOperands, async, asyncDeviceTypes,
             asyncOnlyDeviceTypes);
     } else if (const auto *firstprivateClause =
                    std::get_if<Fortran::parser::AccClause::Firstprivate>(
                        &clause.u)) {
       genPrivatizationRecipes<mlir::acc::FirstprivateRecipeOp>(
           firstprivateClause->v, converter, semanticsContext, stmtCtx,
-          firstprivateOperands, firstPrivatizationRecipes, async,
+          firstprivateOperands, async,
           asyncDeviceTypes, asyncOnlyDeviceTypes);
     } else if (const auto *reductionClause =
                    std::get_if<Fortran::parser::AccClause::Reduction>(
@@ -2846,7 +2826,7 @@ static Op createComputeOp(
       // instead.
       if (!combinedConstructs) {
         genReductions(reductionClause->v, converter, semanticsContext, stmtCtx,
-                      reductionOperands, reductionRecipes, async,
+                      reductionOperands, async,
                       asyncDeviceTypes, asyncOnlyDeviceTypes);
       } else {
         auto crtDataStart = dataClauseOperands.size();
@@ -2938,18 +2918,6 @@ static Op createComputeOp(
   if (!waitOnlyDeviceTypes.empty())
     computeOp.setWaitOnlyAttr(builder.getArrayAttr(waitOnlyDeviceTypes));
 
-  if constexpr (!std::is_same_v<Op, mlir::acc::KernelsOp>) {
-    if (!privatizationRecipes.empty())
-      computeOp.setPrivatizationRecipesAttr(
-          mlir::ArrayAttr::get(builder.getContext(), privatizationRecipes));
-    if (!reductionRecipes.empty())
-      computeOp.setReductionRecipesAttr(
-          mlir::ArrayAttr::get(builder.getContext(), reductionRecipes));
-    if (!firstPrivatizationRecipes.empty())
-      computeOp.setFirstprivatizationRecipesAttr(mlir::ArrayAttr::get(
-          builder.getContext(), firstPrivatizationRecipes));
-  }
-
   if (combinedConstructs)
     computeOp.setCombinedAttr(builder.getUnitAttr());
 
diff --git a/flang/test/Lower/OpenACC/acc-kernels-loop.f90 b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
index 8d95f35b186ee..fd6cb0bbda472 100644
--- a/flang/test/Lower/OpenACC/acc-kernels-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
@@ -704,7 +704,9 @@ subroutine acc_kernels_loop
 ! CHECK:      %[[COPYINREDR:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<f32>) -> !fir.ref<f32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_r"}
 ! CHECK:      %[[COPYINREDI:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_i"}
 ! CHECK:      acc.kernels {{.*}} dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
-! CHECK:        acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {{.*}} {
+! CHECK:        %[[REDUCTION_R:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<f32>) recipe(@reduction_add_ref_f32) -> !fir.ref<f32> {name = "reduction_r"}
+! CHECK:        %[[REDUCTION_I:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_mul_ref_i32) -> !fir.ref<i32> {name = "reduction_i"}
+! CHECK:        acc.loop {{.*}} reduction(%[[REDUCTION_R]], %[[REDUCTION_I]] : !fir.ref<f32>, !fir.ref<i32>) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
diff --git a/flang/test/Lower/OpenACC/acc-loop.f90 b/flang/test/Lower/OpenACC/acc-loop.f90
index f9f5e8c2165d5..0c394f849baf0 100644
--- a/flang/test/Lower/OpenACC/acc-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-loop.f90
@@ -27,7 +27,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK: acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}{{$}}
 
@@ -36,7 +37,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK: acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
 
@@ -45,7 +47,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK: acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 
@@ -54,7 +57,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -63,7 +67,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.loop gang private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop gang private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -73,7 +78,8 @@ program acc_loop
   END DO
 
 ! CHECK:      [[GANGNUM1:%.*]] = arith.constant 8 : i32
-! CHECK:      acc.loop gang({num=[[GANGNUM1]] : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop gang({num=[[GANGNUM1]] : i32}) private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -83,7 +89,8 @@ program acc_loop
   END DO
 
 ! CHECK:      [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.loop gang({num=[[GANGNUM2]] : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop gang({num=[[GANGNUM2]] : i32}) private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -92,7 +99,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK: acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -101,7 +109,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.loop vector private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop vector private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -110,8 +119,9 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK: [[CONSTANT128:%.*]] = arith.constant 128 : i32
-! CHECK:      acc.loop vector([[CONSTANT128]] : i32) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      [[CONSTANT128:%.*]] = arith.constant 128 : i32
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop vector([[CONSTANT128]] : i32) private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -121,7 +131,8 @@ program acc_loop
   END DO
 
 ! CHECK:      [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.loop vector([[VECTORLENGTH]] : i32) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop vector([[VECTORLENGTH]] : i32) private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -130,7 +141,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.loop worker private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop worker private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -139,8 +151,9 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK: [[WORKER128:%.*]] = arith.constant 128 : i32
-! CHECK:      acc.loop worker([[WORKER128]] : i32) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      [[WORKER128:%.*]] = arith.constant 128 : i32
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop worker([[WORKER128]] : i32) private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -149,7 +162,9 @@ program acc_loop
     c(:,i) = d(:,i)
   END DO
 
-! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_C:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_C]], %[[PRIVATE_I]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -159,7 +174,8 @@ program acc_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_I]] : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -168,7 +184,10 @@ program acc_loop
     c(:,i) = d(:,i)
   END DO
 
-! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_C:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      %[[PRIVATE_D:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "d"}
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_C]], %[[PRIVATE_D]], %[[PRIVATE_I]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -177,7 +196,10 @@ program acc_loop
     c(:,i) = d(:,i)
   END DO
 
-! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[PRIVATE_C:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      %[[PRIVATE_D:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<!fir.array<10x10xf32>>) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "d"}
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_C]], %[[PRIVATE_D]], %[[PRIVATE_I]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -312,7 +334,10 @@ program acc_loop
     reduction_i = 1
   end do
 
-! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      %[[REDUCTION_R:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<f32>) recipe(@reduction_add_ref_f32) -> !fir.ref<f32> {name = "reduction_r"}
+! CHECK:      %[[REDUCTION_I:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_mul_ref_i32) -> !fir.ref<i32> {name = "reduction_i"}
+! CHECK:      %[[PRIVATE_I:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK:      acc.loop private(%[[PRIVATE_I]] : !fir.ref<i32>) reduction(%[[REDUCTION_R]], %[[REDUCTION_I]] : !fir.ref<f32>, !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
@@ -379,8 +404,8 @@ subroutine sub1(i, j, k)
 ! CHECK: %[[DC_J:.*]]:2 = hlfir.declare %[[ARG_J]] dummy_scope %0 
 ! CHECK: %[[DC_K:.*]]:2 = hlfir.declare %[[ARG_K]] dummy_scope %0 
 ! CHECK: acc.parallel combined(loop)
-! CHECK: %[[P_I:.*]] = acc.private varPtr(%[[DC_I]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
-! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "j"}
-! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "k"}
-! CHECK: acc.loop combined(parallel) private(@privatization_ref_i32 -> %[[P_I]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_J]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_K]] : !fir.ref<i32>) control(%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%c10{{.*}}, %c100{{.*}}, %c200{{.*}} : i32, i32, i32)  step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32)
+! CHECK: %[[P_I:.*]] = acc.private varPtr(%[[DC_I]]#0 : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]]#0 : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "j"}
+! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]]#0 : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "k"}
+! CHECK: acc.loop combined(parallel) private(%[[P_I]], %[[P_J]], %[[P_K]] : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) control(%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%c10{{.*}}, %c100{{.*}}, %c200{{.*}} : i32, i32, i32)  step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32)
 ! CHECK: } attributes {inclusiveUpperbound = array<i1: true, true, true>, independent = [#acc.device_type<none>]}
diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
index 8086080bd3797..186a28fce7d5e 100644
--- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
@@ -451,10 +451,10 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[ACC_PRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.parallel {{.*}} firstprivate(@firstprivatization_ref_10xf32 -> %[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
-! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>{{.*}})
+! CHECK:      %[[ACC_PRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) recipe(@firstprivatization_ref_10xf32) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      acc.parallel {{.*}} firstprivate(%[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) recipe(@privatization_ref_10xf32) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:        acc.loop {{.*}} private(%[[ACC_PRIVATE_A]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -722,7 +722,9 @@ subroutine acc_parallel_loop
 ! CHECK:      %[[COPYINREDR:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<f32>) -> !fir.ref<f32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_r"}
 ! CHECK:      %[[COPYINREDI:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_i"}
 ! CHECK:      acc.parallel {{.*}} dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
-! CHECK:        acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {{.*}}
+! CHECK:        %[[REDUCTION_R:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<f32>) recipe(@reduction_add_ref_f32) -> !fir.ref<f32> {name = "reduction_r"}
+! CHECK:        %[[REDUCTION_I:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_mul_ref_i32) -> !fir.ref<i32> {name = "reduction_i"}
+! CHECK:        acc.loop {{.*}} reduction(%[[REDUCTION_R]], %[[REDUCTION_I]] : !fir.ref<f32>, !fir.ref<i32>) {{.*}}
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
diff --git a/flang/test/Lower/OpenACC/acc-parallel.f90 b/flang/test/Lower/OpenACC/acc-parallel.f90
index 1eae106ba61b2..24c18f99bb8a5 100644
--- a/flang/test/Lower/OpenACC/acc-parallel.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel.f90
@@ -330,17 +330,19 @@ subroutine acc_parallel
 !$acc parallel private(a) firstprivate(b) private(c) async(1)
 !$acc end parallel
 
-! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3:%.*]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK:      %[[ACC_PRIVATE_C:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
-! CHECK:      acc.parallel async([[ASYNC3]]) firstprivate(@firstprivatization_ref_10x10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10x10xf32>>) private(@privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3:%.*]]) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3]]) recipe(@firstprivatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:      %[[ACC_PRIVATE_C:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) async([[ASYNC3]]) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.parallel async([[ASYNC3]]) firstprivate(%[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10x10xf32>>) private(%[[ACC_PRIVATE_A]], %[[ACC_PRIVATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
 !$acc parallel reduction(+:reduction_r) reduction(*:reduction_i)
 !$acc end parallel
 
-! CHECK:      acc.parallel reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {
+! CHECK:      %[[REDUCTION_R:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<f32>) recipe(@reduction_add_ref_f32) -> !fir.ref<f32>
+! CHECK:      %[[REDUCTION_I:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_mul_ref_i32) -> !fir.ref<i32>
+! CHECK:      acc.parallel reduction(%[[REDUCTION_R]], %[[REDUCTION_I]] : !fir.ref<f32>, !fir.ref<i32>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
diff --git a/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
index b1dc4e79f9f73..c03c5d7d21781 100644
--- a/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
@@ -190,8 +190,8 @@ program acc_private
     a(i) = b(i) + c
   END DO
 
-! CHECK: %[[C_PRIVATE:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "c"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_i32 -> %[[C_PRIVATE]] : !fir.ref<i32>{{.*}})
+! CHECK: %[[C_PRIVATE:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {name = "c"}
+! CHECK: acc.loop private(%[[C_PRIVATE]]{{.*}} : !fir.ref<i32>{{.*}})
 ! CHECK: acc.yield
 
   !$acc loop private(b)
@@ -204,8 +204,8 @@ program acc_private
 ! CHECK: %[[LB:.*]] = arith.constant 0 : index
 ! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_100xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<100xf32>>{{.*}})
+! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) recipe(@privatization_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: acc.loop private(%[[B_PRIVATE]]{{.*}} : !fir.ref<!fir.array<100xf32>>{{.*}})
 ! CHECK: acc.yield
 
   !$acc loop private(b(1:50))
@@ -218,8 +218,8 @@ program acc_private
 ! CHECK: %[[LB:.*]] = arith.constant 0 : index
 ! CHECK: %[[UB:.*]] = arith.constant 49 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(1:50)"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_50xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<50xf32>>{{.*}})
+! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) recipe(@privatization_ref_50xf32) -> !fir.ref<!fir.array<50xf32>> {name = "b(1:50)"}
+! CHECK: acc.loop private(%[[B_PRIVATE]]{{.*}} : !fir.ref<!fir.array<50xf32>>{{.*}})
 
   !$acc parallel loop firstprivate(c)
   DO i = 1, n
@@ -227,8 +227,8 @@ program acc_private
     a(i) = b(i) + c
   END DO
 
-! CHECK: %[[FP_C:.*]] = acc.firstprivate varPtr(%[[DECLC]]#0 : !fir.ref<i32>)   -> !fir.ref<i32> {name = "c"}
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_ref_i32 -> %[[FP_C]] : !fir.ref<i32>)
+! CHECK: %[[FP_C:.*]] = acc.firstprivate varPtr(%[[DECLC]]#0 : !fir.ref<i32>) recipe(@firstprivatization_ref_i32) -> !fir.ref<i32> {name = "c"}
+! CHECK: acc.parallel {{.*}} firstprivate(%[[FP_C]] : !fir.ref<i32>)
 ! CHECK: acc.yield
 
   !$acc parallel loop firstprivate(b)
@@ -241,8 +241,8 @@ program acc_private
 ! CHECK: %[[LB:.*]] = arith.constant 0 : index
 ! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_section_ext100_ref_100xf32 -> %[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) recipe(@firstprivatization_section_ext100_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: acc.parallel {{.*}} firstprivate(%[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
 ! CHECK: acc.yield
 
   !$acc parallel loop firstprivate(b(51:100))
@@ -255,8 +255,8 @@ program acc_private
 ! CHECK: %[[LB:.*]] = arith.constant 50 : index
 ! CHECK: %[[UB:.*]] = arith.constant 99 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(51:100)"}
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_section_lb50.ub99_ref_50xf32 -> %[[FP_B]] : !fir.ref<!fir.array<50xf32>>)
+! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) recipe(@firstprivatization_section_lb50.ub99_ref_50xf32) -> !fir.ref<!fir.array<50xf32>> {name = "b(51:100)"}
+! CHECK: acc.parallel {{.*}} firstprivate(%[[FP_B]] : !fir.ref<!fir.array<50xf32>>)
 
 end program
 
@@ -274,8 +274,8 @@ subroutine acc_private_assumed_shape(a, n)
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>{{.*}})
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) recipe(@privatization_box_Uxi32) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private(%[[PRIVATE]]{{.*}} : !fir.ref<!fir.array<?xi32>>{{.*}})
 
 subroutine acc_private_allocatable_array(a, n)
   integer, allocatable :: a(:)
@@ -297,9 +297,10 @@ subroutine acc_private_allocatable_array(a, n)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.heap<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>{{.*}})
-! CHECK: acc.serial private(@privatization_box_heap_Uxi32 -> %{{.*}} : !fir.heap<!fir.array<?xi32>>)
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%{{.*}}) recipe(@privatization_box_heap_Uxi32) -> !fir.heap<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private(%[[PRIVATE]]{{.*}} : !fir.heap<!fir.array<?xi32>>{{.*}})
+! CHECK: %[[PRIVATE_SERIAL:.*]] = acc.private varPtr(%{{.*}} : !fir.heap<!fir.array<?xi32>>) bounds(%{{.*}}) recipe(@privatization_box_heap_Uxi32) -> !fir.heap<!fir.array<?xi32>>
+! CHECK: acc.serial private(%[[PRIVATE_SERIAL]] : !fir.heap<!fir.array<?xi32>>)
 
 subroutine acc_private_pointer_array(a, n)
   integer, pointer :: a(:)
@@ -317,8 +318,8 @@ subroutine acc_private_pointer_array(a, n)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ptr<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>{{.*}})
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xi32>>) bounds(%{{.*}}) recipe(@privatization_box_ptr_Uxi32) -> !fir.ptr<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private(%[[PRIVATE]]{{.*}} : !fir.ptr<!fir.array<?xi32>>{{.*}})
 
 subroutine acc_private_dynamic_extent(a, n)
   integer :: n, i
@@ -336,8 +337,8 @@ subroutine acc_private_dynamic_extent(a, n)
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>, !fir.dscope) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?x2xi32>>) -> !fir.ref<!fir.array<?x?x2xi32>>
-! CHECK: %[[PRIV:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>{{.*}})
+! CHECK: %[[PRIV:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) recipe(@privatization_box_UxUx2xi32) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private(%[[PRIV]]{{.*}} : !fir.ref<!fir.array<?x?x2xi32>>{{.*}})
 
 subroutine acc_firstprivate_assumed_shape(a, n)
   integer :: a(:), i, n
@@ -348,6 +349,10 @@ subroutine acc_firstprivate_assumed_shape(a, n)
   end do
 end subroutine
 
+! CHECK-LABEL: func.func @_QPacc_firstprivate_assumed_shape
+! CHECK: %[[FIRSTPRIVATE_A:.*]] = acc.firstprivate varPtr(%{{.*}} : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) recipe(@firstprivatization_box_Uxi32) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.parallel {{.*}}firstprivate(%[[FIRSTPRIVATE_A]] : !fir.ref<!fir.array<?xi32>>)
+
 subroutine acc_firstprivate_assumed_shape_with_section(a, n)
   integer :: a(:), i, n
 
@@ -357,6 +362,10 @@ subroutine acc_firstprivate_assumed_shape_with_section(a, n)
   end do
 end subroutine
 
+! CHECK-LABEL: func.func @_QPacc_firstprivate_assumed_shape_with_section
+! CHECK: %[[FIRSTPRIVATE_A:.*]] = acc.firstprivate varPtr(%{{.*}} : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) recipe(@firstprivatization_section_lb4.ub9_box_Uxi32) -> !fir.ref<!fir.array<?xi32>> {name = "a(5:10)"}
+! CHECK: acc.parallel {{.*}}firstprivate(%[[FIRSTPRIVATE_A]] : !fir.ref<!fir.array<?xi32>>)
+
 subroutine acc_firstprivate_dynamic_extent(a, n)
   integer :: n, i
   integer :: a(n, n, 2)
@@ -367,7 +376,9 @@ subroutine acc_firstprivate_dynamic_extent(a, n)
   end do
 end subroutine
 
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_box_UxUx2xi32 -> %{{.*}} : !fir.ref<!fir.array<?x?x2xi32>>)
+! CHECK-LABEL: func.func @_QPacc_firstprivate_dynamic_extent
+! CHECK: %[[FIRSTPRIVATE_A:.*]] = acc.firstprivate varPtr(%{{.*}} : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) recipe(@firstprivatization_box_UxUx2xi32) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
+! CHECK: acc.parallel {{.*}}firstprivate(%[[FIRSTPRIVATE_A]] : !fir.ref<!fir.array<?x?x2xi32>>)
 
 module acc_declare_equivalent
   integer, parameter :: n = 10
@@ -381,7 +392,9 @@ subroutine sub1()
   end subroutine
 end module
 
-! CHECK: acc.parallel private(@privatization_ref_10xf32 -> %{{.*}} : !fir.ref<!fir.array<10xf32>>)
+! CHECK-LABEL: func.func @_QMacc_declare_equivalentPsub1
+! CHECK: %[[PRIVATE_V2:.*]] = acc.private varPtr(%{{.*}} : !fir.ptr<!fir.array<10xf32>>) bounds(%{{.*}}) recipe(@privatization_ref_10xf32) -> !fir.ref<!fir.array<10xf32>> {name = "v2"}
+! CHECK: acc.parallel private(%[[PRIVATE_V2]] : !fir.ref<!fir.array<10xf32>>)
 
 subroutine acc_private_use()
   integer :: i, j
@@ -396,8 +409,8 @@ subroutine acc_private_use()
 ! CHECK: %[[I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFacc_private_useEi"}
 ! CHECK: %[[DECL_I:.*]]:2 = hlfir.declare %[[I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: acc.parallel
-! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#0 : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
 ! CHECK: %[[DECL_PRIV_I:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: acc.loop {{.*}} private(@privatization_ref_i32 -> %[[PRIV_I]] : !fir.ref<i32>) control(%[[IV0:.*]] : i32) = (%c1{{.*}} : i32) to (%c10{{.*}} : i32) step (%c1{{.*}} : i32)
+! CHECK: acc.loop {{.*}} private(%[[PRIV_I]] : !fir.ref<i32>) control(%[[IV0:.*]] : i32) = (%c1{{.*}} : i32) to (%c10{{.*}} : i32) step (%c1{{.*}} : i32)
 ! CHECK:   fir.store %[[IV0]] to %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
 ! CHECK:   %{{.*}} = fir.load %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index b1bfb02439f03..85075384c41b2 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -207,8 +207,8 @@ program acc_private
     a(i) = b(i) + c
   END DO
 
-! CHECK: %[[C_PRIVATE:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "c"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_i32 -> %[[C_PRIVATE]] : !fir.ref<i32>{{.*}})
+! CHECK: %[[C_PRIVATE:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {name = "c"}
+! CHECK: acc.loop private(%[[C_PRIVATE]]{{.*}} : !fir.ref<i32>{{.*}})
 ! CHECK: acc.yield
 
   !$acc loop private(b)
@@ -217,8 +217,8 @@ program acc_private
     a(i) = b(i) + c
   END DO
 
-! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_100xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<100xf32>>{{.*}})
+! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) recipe(@privatization_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: acc.loop private(%[[B_PRIVATE]]{{.*}} : !fir.ref<!fir.array<100xf32>>{{.*}})
 ! CHECK: acc.yield
 
   !$acc loop private(b(1:50))
@@ -231,8 +231,8 @@ program acc_private
 ! CHECK: %[[LB:.*]] = arith.constant 0 : index
 ! CHECK: %[[UB:.*]] = arith.constant 49 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(1:50)"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_50xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<50xf32>>{{.*}})
+! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) recipe(@privatization_ref_50xf32) -> !fir.ref<!fir.array<50xf32>> {name = "b(1:50)"}
+! CHECK: acc.loop private(%[[B_PRIVATE]]{{.*}} : !fir.ref<!fir.array<50xf32>>{{.*}})
 
   !$acc parallel loop firstprivate(c)
   DO i = 1, n
@@ -240,8 +240,8 @@ program acc_private
     a(i) = b(i) + c
   END DO
 
-! CHECK: %[[FP_C:.*]] = acc.firstprivate varPtr(%[[DECLC]]#0 : !fir.ref<i32>)   -> !fir.ref<i32> {name = "c"}
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_ref_i32 -> %[[FP_C]] : !fir.ref<i32>)
+! CHECK: %[[FP_C:.*]] = acc.firstprivate varPtr(%[[DECLC]]#0 : !fir.ref<i32>) recipe(@firstprivatization_ref_i32) -> !fir.ref<i32> {name = "c"}
+! CHECK: acc.parallel {{.*}} firstprivate(%[[FP_C]] : !fir.ref<i32>)
 ! CHECK: acc.yield
 
   !$acc parallel loop firstprivate(b)
@@ -250,8 +250,8 @@ program acc_private
     a(i) = b(i) + c
   END DO
 
-! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_ref_100xf32 -> %[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) recipe(@firstprivatization_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: acc.parallel {{.*}} firstprivate(%[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
 ! CHECK: acc.yield
 
   !$acc parallel loop firstprivate(b(51:100))
@@ -264,8 +264,8 @@ program acc_private
 ! CHECK: %[[LB:.*]] = arith.constant 50 : index
 ! CHECK: %[[UB:.*]] = arith.constant 99 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(51:100)"}
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_section_lb50.ub99_ref_50xf32 -> %[[FP_B]] : !fir.ref<!fir.array<50xf32>>)
+! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) recipe(@firstprivatization_section_lb50.ub99_ref_50xf32) -> !fir.ref<!fir.array<50xf32>> {name = "b(51:100)"}
+! CHECK: acc.parallel {{.*}} firstprivate(%[[FP_B]] : !fir.ref<!fir.array<50xf32>>)
 
 end program
 
@@ -282,8 +282,8 @@ subroutine acc_private_assumed_shape(a, n)
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[PRIVATE:.*]] = acc.private var(%[[DECL_A]]#0 : !fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.box<!fir.array<?xi32>>{{.*}})
+! CHECK: %[[PRIVATE:.*]] = acc.private var(%[[DECL_A]]#0 : !fir.box<!fir.array<?xi32>>) recipe(@privatization_box_Uxi32) -> !fir.box<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private(%[[PRIVATE]]{{.*}} : !fir.box<!fir.array<?xi32>>{{.*}})
 
 subroutine acc_private_allocatable_array(a, n)
   integer, allocatable :: a(:)
@@ -303,9 +303,10 @@ subroutine acc_private_allocatable_array(a, n)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}
 ! CHECK: %[[DECLA_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_arrayEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>{{.*}})
-! CHECK: acc.serial private(@privatization_ref_box_heap_Uxi32 -> %{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) recipe(@privatization_ref_box_heap_Uxi32) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "a"}
+! CHECK: acc.loop {{.*}} private(%[[PRIVATE]]{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>{{.*}})
+! CHECK: %[[PRIVATE_SERIAL:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) recipe(@privatization_ref_box_heap_Uxi32) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: acc.serial private(%[[PRIVATE_SERIAL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 
 subroutine acc_private_allocatable_scalar(b, a, n)
   integer :: a(n)
@@ -326,9 +327,10 @@ subroutine acc_private_allocatable_scalar(b, a, n)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<i32>>> {fir.bindc_name = "b"}
 ! CHECK: %[[DECLA_B:.*]]:2 = hlfir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_scalarEb"} : (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_B]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = "b"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_i32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<i32>>>{{.*}})
-! CHECK: acc.serial private(@privatization_ref_box_heap_i32 -> %{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>) {
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_B]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>) recipe(@privatization_ref_box_heap_i32) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = "b"}
+! CHECK: acc.loop {{.*}} private(%[[PRIVATE]]{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>{{.*}})
+! CHECK: %[[PRIVATE_SERIAL:.*]] = acc.private varPtr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>) recipe(@privatization_ref_box_heap_i32) -> !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK: acc.serial private(%[[PRIVATE_SERIAL]] : !fir.ref<!fir.box<!fir.heap<i32>>>) {
 
 subroutine acc_private_pointer_array(a, n)
   integer, pointer :: a(:)
@@ -344,8 +346,8 @@ subroutine acc_private_pointer_array(a, n)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_private_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>{{.*}})
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) recipe(@privatization_ref_box_ptr_Uxi32) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {name = "a"}
+! CHECK: acc.loop {{.*}} private(%[[PRIVATE]]{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>{{.*}})
 
 subroutine acc_private_dynamic_extent(a, n)
   integer :: n, i
@@ -362,8 +364,8 @@ subroutine acc_private_dynamic_extent(a, n)
 ! CHECK: %[[DECL_N:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>, !fir.dscope) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
 ! CHECK: acc.parallel {{.*}} {
-! CHECK: %[[PRIV:.*]] = acc.private var(%[[DECL_A]]#0 : !fir.box<!fir.array<?x?x2xi32>>) -> !fir.box<!fir.array<?x?x2xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.box<!fir.array<?x?x2xi32>>{{.*}})
+! CHECK: %[[PRIV:.*]] = acc.private var(%[[DECL_A]]#0 : !fir.box<!fir.array<?x?x2xi32>>) recipe(@privatization_box_UxUx2xi32) -> !fir.box<!fir.array<?x?x2xi32>> {name = "a"}
+! CHECK: acc.loop {{.*}} private(%[[PRIV]]{{.*}} : !fir.box<!fir.array<?x?x2xi32>>{{.*}})
 
 subroutine acc_firstprivate_assumed_shape(a, n)
   integer :: a(:), i, n
@@ -374,6 +376,10 @@ subroutine acc_firstprivate_assumed_shape(a, n)
   end do
 end subroutine
 
+! CHECK-LABEL: func.func @_QPacc_firstprivate_assumed_shape
+! CHECK: %[[FIRSTPRIVATE_A:.*]] = acc.firstprivate var(%{{.*}} : !fir.box<!fir.array<?xi32>>) recipe(@firstprivatization_box_Uxi32) -> !fir.box<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.parallel {{.*}}firstprivate(%[[FIRSTPRIVATE_A]] : !fir.box<!fir.array<?xi32>>) {
+
 subroutine acc_firstprivate_assumed_shape_with_section(a, n)
   integer :: a(:), i, n
 
@@ -383,6 +389,10 @@ subroutine acc_firstprivate_assumed_shape_with_section(a, n)
   end do
 end subroutine
 
+! CHECK-LABEL: func.func @_QPacc_firstprivate_assumed_shape_with_section
+! CHECK: %[[FIRSTPRIVATE_A:.*]] = acc.firstprivate var(%{{.*}} : !fir.box<!fir.array<?xi32>>) bounds(%{{.*}}) recipe(@firstprivatization_section_lb4.ub9_box_Uxi32) -> !fir.box<!fir.array<?xi32>> {name = "a(5:10)"}
+! CHECK: acc.parallel {{.*}}firstprivate(%[[FIRSTPRIVATE_A]] : !fir.box<!fir.array<?xi32>>)
+
 subroutine acc_firstprivate_dynamic_extent(a, n)
   integer :: n, i
   integer :: a(n, n, 2)
@@ -393,7 +403,9 @@ subroutine acc_firstprivate_dynamic_extent(a, n)
   end do
 end subroutine
 
-! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_box_UxUx2xi32 -> %{{.*}} : !fir.box<!fir.array<?x?x2xi32>>)
+! CHECK-LABEL: func.func @_QPacc_firstprivate_dynamic_extent
+! CHECK: %[[FIRSTPRIVATE_A:.*]] = acc.firstprivate var(%{{.*}} : !fir.box<!fir.array<?x?x2xi32>>) recipe(@firstprivatization_box_UxUx2xi32) -> !fir.box<!fir.array<?x?x2xi32>> {name = "a"}
+! CHECK: acc.parallel {{.*}}firstprivate(%[[FIRSTPRIVATE_A]] : !fir.box<!fir.array<?x?x2xi32>>)
 
 module acc_declare_equivalent
   integer, parameter :: n = 10
@@ -407,7 +419,8 @@ subroutine sub1()
   end subroutine
 end module
 
-! CHECK: acc.parallel private(@privatization_ptr_10xf32 -> %{{.*}} : !fir.ptr<!fir.array<10xf32>>)
+! CHECK: %[[PRIVATE_V2:.*]] = acc.private varPtr(%{{.*}} : !fir.ptr<!fir.array<10xf32>>) recipe(@privatization_ptr_10xf32) -> !fir.ptr<!fir.array<10xf32>>
+! CHECK: acc.parallel private(%[[PRIVATE_V2]] : !fir.ptr<!fir.array<10xf32>>)
 
 subroutine acc_private_use()
   integer :: i, j
@@ -422,8 +435,8 @@ subroutine acc_private_use()
 ! CHECK: %[[I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFacc_private_useEi"}
 ! CHECK: %[[DECL_I:.*]]:2 = hlfir.declare %[[I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: acc.parallel
-! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
+! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#0 : !fir.ref<i32>) recipe(@privatization_ref_i32) -> !fir.ref<i32> {implicit = true, name = "i"}
 ! CHECK: %[[DECL_PRIV_I:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: acc.loop {{.*}} private(@privatization_ref_i32 -> %[[PRIV_I]] : !fir.ref<i32>) control(%[[IV0:.*]] : i32) = (%c1{{.*}} : i32) to (%c10{{.*}} : i32) step (%c1{{.*}} : i32)
+! CHECK: acc.loop {{.*}} private(%[[PRIV_I]] : !fir.ref<i32>) control(%[[IV0:.*]] : i32) = (%c1{{.*}} : i32) to (%c10{{.*}} : i32) step (%c1{{.*}} : i32)
 ! CHECK:   fir.store %[[IV0]] to %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
 ! CHECK:   %{{.*}} = fir.load %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenACC/acc-reduction-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-reduction-unwrap-defaultbounds.f90
index 5bb751678ed53..8ec089ea265a7 100644
--- a/flang/test/Lower/OpenACC/acc-reduction-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction-unwrap-defaultbounds.f90
@@ -704,8 +704,8 @@ subroutine acc_reduction_add_int(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) recipe(@reduction_add_ref_i32) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<i32>)
 
 subroutine acc_reduction_add_int_array_1d(a, b)
   integer :: a(100)
@@ -720,8 +720,8 @@ subroutine acc_reduction_add_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) recipe(@reduction_add_section_ext100_ref_100xi32) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_add_int_array_2d(a, b)
   integer :: a(100, 10), b(100, 10)
@@ -738,8 +738,8 @@ subroutine acc_reduction_add_int_array_2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_2d(
 ! CHECK-SAME:  %[[ARG0:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "b"}) {
 ! CHECK:       %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
+! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) bounds(%{{.*}}, %{{.*}}) recipe(@reduction_add_section_ext100xext10_ref_100x10xi32) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
 ! CHECK: } attributes {collapse = [2]{{.*}}
 
 subroutine acc_reduction_add_int_array_3d(a, b)
@@ -759,8 +759,8 @@ subroutine acc_reduction_add_int_array_3d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_3d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10x2xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10x2xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10x2xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10x2xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) recipe(@reduction_add_section_ext100xext10xext2_ref_100x10x2xi32) -> !fir.ref<!fir.array<100x10x2xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100x10x2xi32>>)
 ! CHECK: } attributes {collapse = [3]{{.*}}
 
 subroutine acc_reduction_add_float(a, b)
@@ -776,8 +776,8 @@ subroutine acc_reduction_add_float(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_float(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) recipe(@reduction_add_ref_f32) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<f32>)
 
 subroutine acc_reduction_add_float_array_1d(a, b)
   real :: a(100), b(100)
@@ -792,8 +792,8 @@ subroutine acc_reduction_add_float_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_float_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_section_ext100_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) recipe(@reduction_add_section_ext100_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_mul_int(a, b)
   integer :: a(100)
@@ -808,8 +808,8 @@ subroutine acc_reduction_mul_int(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_int(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) recipe(@reduction_mul_ref_i32) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<i32>)
 
 subroutine acc_reduction_mul_int_array_1d(a, b)
   integer :: a(100)
@@ -824,8 +824,8 @@ subroutine acc_reduction_mul_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_int_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_section_ext100_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) recipe(@reduction_mul_section_ext100_ref_100xi32) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_mul_float(a, b)
   real :: a(100), b
@@ -840,8 +840,8 @@ subroutine acc_reduction_mul_float(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_float(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) recipe(@reduction_mul_ref_f32) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<f32>)
 
 subroutine acc_reduction_mul_float_array_1d(a, b)
   real :: a(100), b(100)
@@ -856,8 +856,8 @@ subroutine acc_reduction_mul_float_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_float_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_section_ext100_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) recipe(@reduction_mul_section_ext100_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_min_int(a, b)
   integer :: a(100)
@@ -872,8 +872,8 @@ subroutine acc_reduction_min_int(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_int(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_min_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) recipe(@reduction_min_ref_i32) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<i32>)
 
 subroutine acc_reduction_min_int_array_1d(a, b)
   integer :: a(100), b(100)
@@ -888,8 +888,8 @@ subroutine acc_reduction_min_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_int_array_1d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_min_section_ext100_ref_100xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) recipe(@reduction_min_section_ext100_ref_100xi32) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_min_float(a, b)
   real :: a(100), b
@@ -904,8 +904,8 @@ subroutine acc_reduction_min_float(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_float(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_min_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) recipe(@reduction_min_ref_f32) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<f32>)
 
 subroutine acc_reduction_min_float_array2d(a, b)
   real :: a(100, 10), b(100, 10)
@@ -922,8 +922,8 @@ subroutine acc_reduction_min_float_array2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_float_array2d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xf32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xf32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_min_section_ext100xext10_ref_100x10xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xf32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xf32>>) bounds(%{{.*}}, %{{.*}}) recipe(@reduction_min_section_ext100xext10_ref_100x10xf32) -> !fir.ref<!fir.array<100x10xf32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100x10xf32>>)
 ! CHECK: attributes {collapse = [2]{{.*}}
 
 subroutine acc_reduction_max_int(a, b)
@@ -939,8 +939,8 @@ subroutine acc_reduction_max_int(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_int(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_max_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) recipe(@reduction_max_ref_i32) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<i32>)
 
 subroutine acc_reduction_max_int_array2d(a, b)
   integer :: a(100, 10), b(100, 10)
@@ -957,8 +957,8 @@ subroutine acc_reduction_max_int_array2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_int_array2d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_max_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) bounds(%{{.*}}, %{{.*}}) recipe(@reduction_max_section_ext100xext10_ref_100x10xi32) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
 
 subroutine acc_reduction_max_float(a, b)
   real :: a(100), b
@@ -973,8 +973,8 @@ subroutine acc_reduction_max_float(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_float(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_max_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) recipe(@reduction_max_ref_f32) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<f32>)
 
 subroutine acc_reduction_max_float_array1d(a, b)
   real :: a(100), b(100)
@@ -989,8 +989,8 @@ subroutine acc_reduction_max_float_array1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_float_array1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_max_section_ext100_ref_100xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%{{.*}}) recipe(@reduction_max_section_ext100_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_iand()
   integer :: i
@@ -999,8 +999,8 @@ subroutine acc_reduction_iand()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_iand()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>)   -> !fir.ref<i32> {name = "i"}
-! CHECK: acc.parallel   reduction(@reduction_iand_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_iand_ref_i32) -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel   reduction(%[[RED]] : !fir.ref<i32>)
 
 subroutine acc_reduction_ior()
   integer :: i
@@ -1009,8 +1009,8 @@ subroutine acc_reduction_ior()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_ior()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>)   -> !fir.ref<i32> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_ior_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_ior_ref_i32) -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<i32>)
 
 subroutine acc_reduction_ieor()
   integer :: i
@@ -1019,8 +1019,8 @@ subroutine acc_reduction_ieor()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_ieor()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_xor_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_xor_ref_i32) -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<i32>)
 
 subroutine acc_reduction_and()
   logical :: l
@@ -1031,8 +1031,8 @@ subroutine acc_reduction_and()
 ! CHECK-LABEL: func.func @_QPacc_reduction_and()
 ! CHECK: %[[L:.*]] = fir.alloca !fir.logical<4> {bindc_name = "l", uniq_name = "_QFacc_reduction_andEl"}
 ! CHECK: %[[DECLL:.*]]:2 = hlfir.declare %[[L]]
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLL]]#0 : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
-! CHECK: acc.parallel reduction(@reduction_land_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLL]]#0 : !fir.ref<!fir.logical<4>>) recipe(@reduction_land_ref_l32) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.logical<4>>)
 
 subroutine acc_reduction_or()
   logical :: l
@@ -1041,8 +1041,8 @@ subroutine acc_reduction_or()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_or()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
-! CHECK: acc.parallel reduction(@reduction_lor_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) recipe(@reduction_lor_ref_l32) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.logical<4>>)
 
 subroutine acc_reduction_eqv()
   logical :: l
@@ -1051,8 +1051,8 @@ subroutine acc_reduction_eqv()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_eqv()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
-! CHECK: acc.parallel reduction(@reduction_eqv_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) recipe(@reduction_eqv_ref_l32) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.logical<4>>)
 
 subroutine acc_reduction_neqv()
   logical :: l
@@ -1061,8 +1061,8 @@ subroutine acc_reduction_neqv()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_neqv()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
-! CHECK: acc.parallel reduction(@reduction_neqv_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) recipe(@reduction_neqv_ref_l32) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.logical<4>>)
 
 subroutine acc_reduction_add_cmplx()
   complex :: c
@@ -1071,8 +1071,8 @@ subroutine acc_reduction_add_cmplx()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_cmplx()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) -> !fir.ref<complex<f32>> {name = "c"}
-! CHECK: acc.parallel reduction(@reduction_add_ref_z32 -> %[[RED]] : !fir.ref<complex<f32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) recipe(@reduction_add_ref_z32) -> !fir.ref<complex<f32>> {name = "c"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<complex<f32>>)
 
 subroutine acc_reduction_mul_cmplx()
   complex :: c
@@ -1081,8 +1081,8 @@ subroutine acc_reduction_mul_cmplx()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_cmplx()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) -> !fir.ref<complex<f32>> {name = "c"}
-! CHECK: acc.parallel reduction(@reduction_mul_ref_z32 -> %[[RED]] : !fir.ref<complex<f32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) recipe(@reduction_mul_ref_z32) -> !fir.ref<complex<f32>> {name = "c"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<complex<f32>>)
 
 subroutine acc_reduction_add_alloc()
   integer, allocatable :: i
@@ -1096,8 +1096,8 @@ subroutine acc_reduction_add_alloc()
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]]
 ! CHECK: %[[LOAD:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<i32>) -> !fir.heap<i32> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_add_heap_i32 -> %[[RED]] : !fir.heap<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<i32>) recipe(@reduction_add_heap_i32) -> !fir.heap<i32> {name = "i"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.heap<i32>)
 
 subroutine acc_reduction_add_pointer(i)
   integer, pointer :: i
@@ -1110,8 +1110,8 @@ subroutine acc_reduction_add_pointer(i)
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
 ! CHECK: %[[LOAD:.*]] = fir.load %[[DECLARG0]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<i32>) -> !fir.ptr<i32> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_add_ptr_i32 -> %[[RED]] : !fir.ptr<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<i32>) recipe(@reduction_add_ptr_i32) -> !fir.ptr<i32> {name = "i"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ptr<i32>)
 
 subroutine acc_reduction_add_static_slice(a)
   integer :: a(100)
@@ -1127,8 +1127,8 @@ subroutine acc_reduction_add_static_slice(a)
 ! CHECK: %[[LB:.*]] = arith.constant 10 : index
 ! CHECK: %[[UB:.*]] = arith.constant 19 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C100]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a(11:20)"}
-! CHECK: acc.parallel reduction(@reduction_add_section_lb10.ub19_ref_100xi32 -> %[[RED]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) recipe(@reduction_add_section_lb10.ub19_ref_100xi32) -> !fir.ref<!fir.array<100xi32>> {name = "a(11:20)"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_add_dynamic_extent_add(a)
   integer :: a(:)
@@ -1139,8 +1139,8 @@ subroutine acc_reduction_add_dynamic_extent_add(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_add(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_add_box_Uxi32 -> %[[RED:.*]] : !fir.ref<!fir.array<?xi32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) recipe(@reduction_add_box_Uxi32) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED:.*]] : !fir.ref<!fir.array<?xi32>>)
 
 subroutine acc_reduction_add_assumed_shape_max(a)
   real :: a(:)
@@ -1151,8 +1151,8 @@ subroutine acc_reduction_add_assumed_shape_max(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_assumed_shape_max(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_Uxf32 -> %[[RED]] : !fir.ref<!fir.array<?xf32>>) {
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xf32>>) bounds(%{{.*}}) recipe(@reduction_max_box_Uxf32) -> !fir.ref<!fir.array<?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.array<?xf32>>) {
 
 subroutine acc_reduction_add_dynamic_extent_add_with_section(a)
   integer :: a(:)
@@ -1165,8 +1165,8 @@ subroutine acc_reduction_add_dynamic_extent_add_with_section(a)
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_add_dynamic_extent_add_with_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c1{{.*}} : index) upperbound(%c3{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true}
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xi32>> {name = "a(2:4)"}
-! CHECK: acc.parallel reduction(@reduction_add_section_lb1.ub3_box_Uxi32 -> %[[RED]] : !fir.ref<!fir.array<?xi32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%[[BOUND]]) recipe(@reduction_add_section_lb1.ub3_box_Uxi32) -> !fir.ref<!fir.array<?xi32>> {name = "a(2:4)"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.array<?xi32>>)
 
 subroutine acc_reduction_add_allocatable(a)
   real, allocatable :: a(:)
@@ -1180,8 +1180,8 @@ subroutine acc_reduction_add_allocatable(a)
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true}
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>)   bounds(%{{[0-9]+}}) -> !fir.heap<!fir.array<?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_heap_Uxf32 -> %[[RED]] : !fir.heap<!fir.array<?xf32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>)   bounds(%{{[0-9]+}}) recipe(@reduction_max_box_heap_Uxf32) -> !fir.heap<!fir.array<?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.heap<!fir.array<?xf32>>)
 
 subroutine acc_reduction_add_pointer_array(a)
   real, pointer :: a(:)
@@ -1195,8 +1195,8 @@ subroutine acc_reduction_add_pointer_array(a)
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true}
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ptr<!fir.array<?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_ptr_Uxf32 -> %[[RED]] : !fir.ptr<!fir.array<?xf32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) recipe(@reduction_max_box_ptr_Uxf32) -> !fir.ptr<!fir.array<?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ptr<!fir.array<?xf32>>)
 
 subroutine acc_reduction_max_dynamic_extent_max(a, n)
   integer :: n
@@ -1209,5 +1209,5 @@ subroutine acc_reduction_max_dynamic_extent_max(a, n)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "a"}, %{{.*}}: !fir.ref<i32> {fir.bindc_name = "n"})
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
 ! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.array<?x?xf32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[ADDR]] : !fir.ref<!fir.array<?x?xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_UxUxf32 -> %[[RED]] : !fir.ref<!fir.array<?x?xf32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[ADDR]] : !fir.ref<!fir.array<?x?xf32>>) bounds(%{{.*}}, %{{.*}}) recipe(@reduction_max_box_UxUxf32) -> !fir.ref<!fir.array<?x?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.array<?x?xf32>>)
diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90
index 20b5ad28f78a1..2c7e64af848fe 100644
--- a/flang/test/Lower/OpenACC/acc-reduction.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction.f90
@@ -745,8 +745,8 @@ subroutine acc_reduction_add_int(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) recipe(@reduction_add_ref_i32) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<i32>)
 
 subroutine acc_reduction_add_int_array_1d(a, b)
   integer :: a(100)
@@ -761,8 +761,8 @@ subroutine acc_reduction_add_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) recipe(@reduction_add_ref_100xi32) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_add_int_array_2d(a, b)
   integer :: a(100, 10), b(100, 10)
@@ -779,8 +779,8 @@ subroutine acc_reduction_add_int_array_2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_2d(
 ! CHECK-SAME:  %[[ARG0:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "b"}) {
 ! CHECK:       %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
+! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) recipe(@reduction_add_ref_100x10xi32) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
 ! CHECK: } attributes {collapse = [2]{{.*}}
 
 subroutine acc_reduction_add_int_array_3d(a, b)
@@ -800,8 +800,8 @@ subroutine acc_reduction_add_int_array_3d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_3d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10x2xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10x2xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10x2xi32>>) -> !fir.ref<!fir.array<100x10x2xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_add_ref_100x10x2xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10x2xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10x2xi32>>) recipe(@reduction_add_ref_100x10x2xi32) -> !fir.ref<!fir.array<100x10x2xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100x10x2xi32>>)
 ! CHECK: } attributes {collapse = [3]{{.*}}
 
 subroutine acc_reduction_add_float(a, b)
@@ -817,8 +817,8 @@ subroutine acc_reduction_add_float(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_float(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) recipe(@reduction_add_ref_f32) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<f32>)
 
 subroutine acc_reduction_add_float_array_1d(a, b)
   real :: a(100), b(100)
@@ -833,8 +833,8 @@ subroutine acc_reduction_add_float_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_float_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_add_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) recipe(@reduction_add_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_mul_int(a, b)
   integer :: a(100)
@@ -849,8 +849,8 @@ subroutine acc_reduction_mul_int(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_int(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) recipe(@reduction_mul_ref_i32) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<i32>)
 
 subroutine acc_reduction_mul_int_array_1d(a, b)
   integer :: a(100)
@@ -865,8 +865,8 @@ subroutine acc_reduction_mul_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_int_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_100xi32 -> %[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xi32>>) recipe(@reduction_mul_ref_100xi32) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_mul_float(a, b)
   real :: a(100), b
@@ -881,8 +881,8 @@ subroutine acc_reduction_mul_float(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_float(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) recipe(@reduction_mul_ref_f32) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<f32>)
 
 subroutine acc_reduction_mul_float_array_1d(a, b)
   real :: a(100), b(100)
@@ -897,8 +897,8 @@ subroutine acc_reduction_mul_float_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_float_array_1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_mul_ref_100xf32 -> %[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) recipe(@reduction_mul_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_min_int(a, b)
   integer :: a(100)
@@ -913,8 +913,8 @@ subroutine acc_reduction_min_int(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_int(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_min_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) recipe(@reduction_min_ref_i32) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<i32>)
 
 subroutine acc_reduction_min_int_array_1d(a, b)
   integer :: a(100), b(100)
@@ -929,8 +929,8 @@ subroutine acc_reduction_min_int_array_1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_int_array_1d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xi32>>) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_min_ref_100xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xi32>>) recipe(@reduction_min_ref_100xi32) -> !fir.ref<!fir.array<100xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_min_float(a, b)
   real :: a(100), b
@@ -945,8 +945,8 @@ subroutine acc_reduction_min_float(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_float(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_min_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) recipe(@reduction_min_ref_f32) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<f32>)
 
 subroutine acc_reduction_min_float_array2d(a, b)
   real :: a(100, 10), b(100, 10)
@@ -963,8 +963,8 @@ subroutine acc_reduction_min_float_array2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_min_float_array2d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xf32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xf32>>) -> !fir.ref<!fir.array<100x10xf32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_min_ref_100x10xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xf32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xf32>>) recipe(@reduction_min_ref_100x10xf32) -> !fir.ref<!fir.array<100x10xf32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100x10xf32>>)
 ! CHECK: attributes {collapse = [2]{{.*}}
 
 subroutine acc_reduction_max_int(a, b)
@@ -980,8 +980,8 @@ subroutine acc_reduction_max_int(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_int(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_max_ref_i32 -> %[[RED_B]] : !fir.ref<i32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<i32>) recipe(@reduction_max_ref_i32) -> !fir.ref<i32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<i32>)
 
 subroutine acc_reduction_max_int_array2d(a, b)
   integer :: a(100, 10), b(100, 10)
@@ -998,8 +998,8 @@ subroutine acc_reduction_max_int_array2d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_int_array2d(
 ! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100x10xi32>> {fir.bindc_name = "b"})
 ! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
-! CHECK: acc.loop {{.*}} reduction(@reduction_max_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
+! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100x10xi32>>) recipe(@reduction_max_ref_100x10xi32) -> !fir.ref<!fir.array<100x10xi32>> {name = "b"}
+! CHECK: acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100x10xi32>>)
 
 subroutine acc_reduction_max_float(a, b)
   real :: a(100), b
@@ -1014,8 +1014,8 @@ subroutine acc_reduction_max_float(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_float(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<f32> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLB:.*]]:2 = hlfir.declare %[[B]]
-! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) -> !fir.ref<f32> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_max_ref_f32 -> %[[RED_B]] : !fir.ref<f32>)
+! CHECK:       %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#0 : !fir.ref<f32>) recipe(@reduction_max_ref_f32) -> !fir.ref<f32> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_B]] : !fir.ref<f32>)
 
 subroutine acc_reduction_max_float_array1d(a, b)
   real :: a(100), b(100)
@@ -1030,8 +1030,8 @@ subroutine acc_reduction_max_float_array1d(a, b)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_float_array1d(
 ! CHECK-SAME:  %{{.*}}: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xf32>> {fir.bindc_name = "b"})
 ! CHECK:       %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK:       acc.loop {{.*}} reduction(@reduction_max_ref_100xf32 -> %[[RED_ARG1]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK:       %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#0 : !fir.ref<!fir.array<100xf32>>) recipe(@reduction_max_ref_100xf32) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
+! CHECK:       acc.loop {{.*}} reduction(%[[RED_ARG1]] : !fir.ref<!fir.array<100xf32>>)
 
 subroutine acc_reduction_iand()
   integer :: i
@@ -1040,8 +1040,8 @@ subroutine acc_reduction_iand()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_iand()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>)   -> !fir.ref<i32> {name = "i"}
-! CHECK: acc.parallel   reduction(@reduction_iand_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_iand_ref_i32) -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel   reduction(%[[RED]] : !fir.ref<i32>)
 
 subroutine acc_reduction_ior()
   integer :: i
@@ -1050,8 +1050,8 @@ subroutine acc_reduction_ior()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_ior()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>)   -> !fir.ref<i32> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_ior_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_ior_ref_i32) -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<i32>)
 
 subroutine acc_reduction_ieor()
   integer :: i
@@ -1060,8 +1060,8 @@ subroutine acc_reduction_ieor()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_ieor()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_xor_ref_i32 -> %[[RED]] : !fir.ref<i32>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_xor_ref_i32) -> !fir.ref<i32> {name = "i"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<i32>)
 
 subroutine acc_reduction_and()
   logical :: l
@@ -1072,8 +1072,8 @@ subroutine acc_reduction_and()
 ! CHECK-LABEL: func.func @_QPacc_reduction_and()
 ! CHECK: %[[L:.*]] = fir.alloca !fir.logical<4> {bindc_name = "l", uniq_name = "_QFacc_reduction_andEl"}
 ! CHECK: %[[DECLL:.*]]:2 = hlfir.declare %[[L]]
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLL]]#0 : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
-! CHECK: acc.parallel reduction(@reduction_land_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLL]]#0 : !fir.ref<!fir.logical<4>>) recipe(@reduction_land_ref_l32) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.logical<4>>)
 
 subroutine acc_reduction_or()
   logical :: l
@@ -1082,8 +1082,8 @@ subroutine acc_reduction_or()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_or()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
-! CHECK: acc.parallel reduction(@reduction_lor_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) recipe(@reduction_lor_ref_l32) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.logical<4>>)
 
 subroutine acc_reduction_eqv()
   logical :: l
@@ -1092,8 +1092,8 @@ subroutine acc_reduction_eqv()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_eqv()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
-! CHECK: acc.parallel reduction(@reduction_eqv_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) recipe(@reduction_eqv_ref_l32) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.logical<4>>)
 
 subroutine acc_reduction_neqv()
   logical :: l
@@ -1102,8 +1102,8 @@ subroutine acc_reduction_neqv()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_neqv()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>> {name = "l"}
-! CHECK: acc.parallel reduction(@reduction_neqv_ref_l32 -> %[[RED]] : !fir.ref<!fir.logical<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.logical<4>>) recipe(@reduction_neqv_ref_l32) -> !fir.ref<!fir.logical<4>> {name = "l"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.logical<4>>)
 
 subroutine acc_reduction_add_cmplx()
   complex :: c
@@ -1112,8 +1112,8 @@ subroutine acc_reduction_add_cmplx()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_cmplx()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) -> !fir.ref<complex<f32>> {name = "c"}
-! CHECK: acc.parallel reduction(@reduction_add_ref_z32 -> %[[RED]] : !fir.ref<complex<f32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) recipe(@reduction_add_ref_z32) -> !fir.ref<complex<f32>> {name = "c"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<complex<f32>>)
 
 subroutine acc_reduction_mul_cmplx()
   complex :: c
@@ -1122,8 +1122,8 @@ subroutine acc_reduction_mul_cmplx()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_cmplx()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) -> !fir.ref<complex<f32>> {name = "c"}
-! CHECK: acc.parallel reduction(@reduction_mul_ref_z32 -> %[[RED]] : !fir.ref<complex<f32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) recipe(@reduction_mul_ref_z32) -> !fir.ref<complex<f32>> {name = "c"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<complex<f32>>)
 
 subroutine acc_reduction_add_alloc()
   integer, allocatable :: i
@@ -1135,8 +1135,8 @@ subroutine acc_reduction_add_alloc()
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_alloc()
 ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "i", uniq_name = "_QFacc_reduction_add_allocEi"}
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]]
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_add_ref_box_heap_i32 -> %[[RED]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>) recipe(@reduction_add_ref_box_heap_i32) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = "i"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
 
 subroutine acc_reduction_add_pointer(i)
   integer, pointer :: i
@@ -1147,8 +1147,8 @@ subroutine acc_reduction_add_pointer(i)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "i"})
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "i"}
-! CHECK: acc.parallel reduction(@reduction_add_ref_box_ptr_i32 -> %[[RED]] : !fir.ref<!fir.box<!fir.ptr<i32>>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>) recipe(@reduction_add_ref_box_ptr_i32) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "i"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.box<!fir.ptr<i32>>>)
 
 subroutine acc_reduction_add_static_slice(a)
   integer :: a(100)
@@ -1164,8 +1164,8 @@ subroutine acc_reduction_add_static_slice(a)
 ! CHECK: %[[LB:.*]] = arith.constant 10 : index
 ! CHECK: %[[UB:.*]] = arith.constant 19 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C100]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a(11:20)"}
-! CHECK: acc.parallel reduction(@reduction_add_section_lb10.ub19_ref_100xi32 -> %[[RED]] : !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) recipe(@reduction_add_section_lb10.ub19_ref_100xi32) -> !fir.ref<!fir.array<100xi32>> {name = "a(11:20)"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.array<100xi32>>)
 
 subroutine acc_reduction_add_dynamic_extent_add(a)
   integer :: a(:)
@@ -1176,8 +1176,8 @@ subroutine acc_reduction_add_dynamic_extent_add(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_add(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK: %[[RED:.*]] = acc.reduction var(%{{.*}} : !fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_add_box_Uxi32 -> %[[RED:.*]] : !fir.box<!fir.array<?xi32>>)
+! CHECK: %[[RED:.*]] = acc.reduction var(%{{.*}} : !fir.box<!fir.array<?xi32>>) recipe(@reduction_add_box_Uxi32) -> !fir.box<!fir.array<?xi32>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED:.*]] : !fir.box<!fir.array<?xi32>>)
 
 subroutine acc_reduction_add_assumed_shape_max(a)
   real :: a(:)
@@ -1188,8 +1188,8 @@ subroutine acc_reduction_add_assumed_shape_max(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_assumed_shape_max(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK: %[[RED:.*]] = acc.reduction var(%{{.*}} : !fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_Uxf32 -> %[[RED]] : !fir.box<!fir.array<?xf32>>) {
+! CHECK: %[[RED:.*]] = acc.reduction var(%{{.*}} : !fir.box<!fir.array<?xf32>>) recipe(@reduction_max_box_Uxf32) -> !fir.box<!fir.array<?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.box<!fir.array<?xf32>>) {
 
 subroutine acc_reduction_add_dynamic_extent_add_with_section(a)
   integer :: a(:)
@@ -1201,8 +1201,8 @@ subroutine acc_reduction_add_dynamic_extent_add_with_section(a)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_add_dynamic_extent_add_with_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c1{{.*}} : index) upperbound(%c3{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true}
-! CHECK: %[[RED:.*]] = acc.reduction var(%[[DECL]]#0 : !fir.box<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.box<!fir.array<?xi32>> {name = "a(2:4)"}
-! CHECK: acc.parallel reduction(@reduction_add_section_lb1.ub3_box_Uxi32 -> %[[RED]] : !fir.box<!fir.array<?xi32>>)
+! CHECK: %[[RED:.*]] = acc.reduction var(%[[DECL]]#0 : !fir.box<!fir.array<?xi32>>) bounds(%[[BOUND]]) recipe(@reduction_add_section_lb1.ub3_box_Uxi32) -> !fir.box<!fir.array<?xi32>> {name = "a(2:4)"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.box<!fir.array<?xi32>>)
 
 subroutine acc_reduction_add_allocatable(a)
   real, allocatable :: a(:)
@@ -1213,8 +1213,8 @@ subroutine acc_reduction_add_allocatable(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_allocatable(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_reduction_add_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_ref_box_heap_Uxf32 -> %[[RED]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) recipe(@reduction_max_ref_box_heap_Uxf32) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 
 subroutine acc_reduction_add_pointer_array(a)
   real, pointer :: a(:)
@@ -1225,8 +1225,8 @@ subroutine acc_reduction_add_pointer_array(a)
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {fir.bindc_name = "a"})
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_reduction_add_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_ref_box_ptr_Uxf32 -> %[[RED]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) recipe(@reduction_max_ref_box_ptr_Uxf32) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 
 subroutine acc_reduction_max_dynamic_extent_max(a, n)
   integer :: n
@@ -1238,5 +1238,5 @@ subroutine acc_reduction_max_dynamic_extent_max(a, n)
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_dynamic_extent_max(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "a"}, %{{.*}}: !fir.ref<i32> {fir.bindc_name = "n"})
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
-! CHECK: %[[RED:.*]] = acc.reduction var(%[[DECL_A]]#0 : !fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.array<?x?xf32>> {name = "a"}
-! CHECK: acc.parallel reduction(@reduction_max_box_UxUxf32 -> %[[RED]] : !fir.box<!fir.array<?x?xf32>>)
+! CHECK: %[[RED:.*]] = acc.reduction var(%[[DECL_A]]#0 : !fir.box<!fir.array<?x?xf32>>) recipe(@reduction_max_box_UxUxf32) -> !fir.box<!fir.array<?x?xf32>> {name = "a"}
+! CHECK: acc.parallel reduction(%[[RED]] : !fir.box<!fir.array<?x?xf32>>)
diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90
index cad0ee73f6cc5..9825142befeb5 100644
--- a/flang/test/Lower/OpenACC/acc-serial-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90
@@ -392,10 +392,10 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.serial {{.*}} firstprivate(@firstprivatization_ref_10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
-! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>{{.*}})
+! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) recipe(@firstprivatization_ref_10xf32) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+! CHECK:      acc.serial {{.*}} firstprivate(%[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) recipe(@privatization_ref_10xf32) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+! CHECK:        acc.loop {{.*}} private(%[[ACC_PRIVATE_A]]{{.*}} : !fir.ref<!fir.array<10xf32>>{{.*}})
 ! CHECK-NOT:          fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -661,7 +661,9 @@ subroutine acc_serial_loop
 ! CHECK:      %[[COPYINREDR:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<f32>) -> !fir.ref<f32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_r"}
 ! CHECK:      %[[COPYINREDI:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_i"}
 ! CHECK:      acc.serial {{.*}} dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
-! CHECK:        acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>)
+! CHECK:        %[[REDUCTION_R:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<f32>) recipe(@reduction_add_ref_f32) -> !fir.ref<f32> {name = "reduction_r"}
+! CHECK:        %[[REDUCTION_I:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_mul_ref_i32) -> !fir.ref<i32> {name = "reduction_i"}
+! CHECK:        acc.loop {{.*}} reduction(%[[REDUCTION_R]], %[[REDUCTION_I]] : !fir.ref<f32>, !fir.ref<i32>)
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
diff --git a/flang/test/Lower/OpenACC/acc-serial.f90 b/flang/test/Lower/OpenACC/acc-serial.f90
index 1e4f32fd209ef..ea66252287325 100644
--- a/flang/test/Lower/OpenACC/acc-serial.f90
+++ b/flang/test/Lower/OpenACC/acc-serial.f90
@@ -272,17 +272,19 @@ subroutine acc_serial
   !$acc serial private(a) firstprivate(b) private(c)
   !$acc end serial
 
-! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
-! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
-! CHECK:      %[[ACC_PRIVATE_C:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
-! CHECK:      acc.serial firstprivate(@firstprivatization_ref_10x10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10x10xf32>>) private(@privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %[[ACC_PRIVATE_C]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10x10xf32>>) recipe(@firstprivatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:      %[[ACC_PRIVATE_C:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<!fir.array<10x10xf32>>) recipe(@privatization_ref_10x10xf32) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.serial firstprivate(%[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10x10xf32>>) private(%[[ACC_PRIVATE_A]], %[[ACC_PRIVATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
 !$acc serial reduction(+:reduction_r) reduction(*:reduction_i)
 !$acc end serial
 
-! CHECK:      acc.serial reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {
+! CHECK:      %[[REDUCTION_R:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<f32>) recipe(@reduction_add_ref_f32) -> !fir.ref<f32> {name = "reduction_r"}
+! CHECK:      %[[REDUCTION_I:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<i32>) recipe(@reduction_mul_ref_i32) -> !fir.ref<i32> {name = "reduction_i"}
+! CHECK:      acc.serial reduction(%[[REDUCTION_R]], %[[REDUCTION_I]] : !fir.ref<f32>, !fir.ref<i32>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index c1a404b496f8d..e9d12827f6c27 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -603,6 +603,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
       | `bounds` `(` $bounds `)`
       | `async` `` custom<DeviceTypeOperandsWithKeywordOnly>($asyncOperands,
             type($asyncOperands), $asyncOperandsDeviceType, $asyncOnly)
+      | `recipe` `(` custom<RecipeSym>($recipe) `)`
     ) `->` type($accVar) attr-dict
   }];
 
@@ -1422,12 +1423,9 @@ def OpenACC_ParallelOp : OpenACC_Op<"parallel",
       Optional<I1>:$ifCond,
       Optional<I1>:$selfCond,
       UnitAttr:$selfAttr,
-      Variadic<AnyType>:$reductionOperands,
-      OptionalAttr<SymbolRefArrayAttr>:$reductionRecipes,
+      Variadic<OpenACC_AnyPointerOrMappableType>:$reductionOperands,
       Variadic<OpenACC_AnyPointerOrMappableType>:$privateOperands,
-      OptionalAttr<SymbolRefArrayAttr>:$privatizationRecipes,
       Variadic<OpenACC_AnyPointerOrMappableType>:$firstprivateOperands,
-      OptionalAttr<SymbolRefArrayAttr>:$firstprivatizationRecipes,
       Variadic<OpenACC_AnyPointerOrMappableType>:$dataClauseOperands,
       OptionalAttr<DefaultValueAttr>:$defaultAttr,
       UnitAttr:$combined);
@@ -1540,16 +1538,12 @@ def OpenACC_ParallelOp : OpenACC_Op<"parallel",
         `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)`
       | `async` `` custom<DeviceTypeOperandsWithKeywordOnly>($asyncOperands,
             type($asyncOperands), $asyncOperandsDeviceType, $asyncOnly)
-      | `firstprivate` `(` custom<SymOperandList>($firstprivateOperands,
-            type($firstprivateOperands), $firstprivatizationRecipes)
-        `)`
+      | `firstprivate` `(` $firstprivateOperands `:` type($firstprivateOperands) `)`
       | `num_gangs` `(` custom<NumGangs>($numGangs,
             type($numGangs), $numGangsDeviceType, $numGangsSegments) `)`
       | `num_workers` `(` custom<DeviceTypeOperands>($numWorkers,
             type($numWorkers), $numWorkersDeviceType) `)`
-      | `private` `(` custom<SymOperandList>(
-            $privateOperands, type($privateOperands), $privatizationRecipes)
-        `)`
+      | `private` `(` $privateOperands `:` type($privateOperands) `)`
       | `vector_length` `(` custom<DeviceTypeOperands>($vectorLength,
             type($vectorLength), $vectorLengthDeviceType) `)`
       | `wait` `` custom<WaitClause>($waitOperands, type($waitOperands),
@@ -1557,9 +1551,7 @@ def OpenACC_ParallelOp : OpenACC_Op<"parallel",
           $waitOnly)
       | `self` `(` $selfCond `)`
       | `if` `(` $ifCond `)`
-      | `reduction` `(` custom<SymOperandList>(
-            $reductionOperands, type($reductionOperands), $reductionRecipes)
-        `)`
+      | `reduction` `(` $reductionOperands `:` type($reductionOperands) `)`
     )
     $region attr-dict-with-keyword
   }];
@@ -1607,12 +1599,9 @@ def OpenACC_SerialOp : OpenACC_Op<"serial",
       Optional<I1>:$ifCond,
       Optional<I1>:$selfCond,
       UnitAttr:$selfAttr,
-      Variadic<AnyType>:$reductionOperands,
-      OptionalAttr<SymbolRefArrayAttr>:$reductionRecipes,
+      Variadic<OpenACC_AnyPointerOrMappableType>:$reductionOperands,
       Variadic<OpenACC_AnyPointerOrMappableType>:$privateOperands,
-      OptionalAttr<SymbolRefArrayAttr>:$privatizationRecipes,
       Variadic<OpenACC_AnyPointerOrMappableType>:$firstprivateOperands,
-      OptionalAttr<SymbolRefArrayAttr>:$firstprivatizationRecipes,
       Variadic<OpenACC_AnyPointerOrMappableType>:$dataClauseOperands,
       OptionalAttr<DefaultValueAttr>:$defaultAttr,
       UnitAttr:$combined);
@@ -1682,20 +1671,14 @@ def OpenACC_SerialOp : OpenACC_Op<"serial",
         `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)`
       | `async` `` custom<DeviceTypeOperandsWithKeywordOnly>($asyncOperands,
             type($asyncOperands), $asyncOperandsDeviceType, $asyncOnly)
-      | `firstprivate` `(` custom<SymOperandList>($firstprivateOperands,
-            type($firstprivateOperands), $firstprivatizationRecipes)
-        `)`
-      | `private` `(` custom<SymOperandList>(
-            $privateOperands, type($privateOperands), $privatizationRecipes)
-        `)`
+      | `firstprivate` `(` $firstprivateOperands `:` type($firstprivateOperands) `)`
+      | `private` `(` $privateOperands `:` type($privateOperands) `)`
       | `wait` `` custom<WaitClause>($waitOperands, type($waitOperands),
           $waitOperandsDeviceType, $waitOperandsSegments, $hasWaitDevnum,
           $waitOnly)
       | `self` `(` $selfCond `)`
       | `if` `(` $ifCond `)`
-      | `reduction` `(` custom<SymOperandList>(
-            $reductionOperands, type($reductionOperands), $reductionRecipes)
-        `)`
+      | `reduction` `(` $reductionOperands `:` type($reductionOperands) `)`
     )
     $region attr-dict-with-keyword
   }];
@@ -2243,9 +2226,7 @@ def OpenACC_LoopOp : OpenACC_Op<"loop",
       OptionalAttr<DeviceTypeArrayAttr>:$tileOperandsDeviceType,
       Variadic<OpenACC_AnyPointerOrMappableType>:$cacheOperands,
       Variadic<OpenACC_AnyPointerOrMappableType>:$privateOperands,
-      OptionalAttr<SymbolRefArrayAttr>:$privatizationRecipes,
-      Variadic<AnyType>:$reductionOperands,
-      OptionalAttr<SymbolRefArrayAttr>:$reductionRecipes,
+      Variadic<OpenACC_AnyPointerOrMappableType>:$reductionOperands,
       OptionalAttr<OpenACC_CombinedConstructsAttr>:$combined
   );
 
@@ -2405,14 +2386,11 @@ def OpenACC_LoopOp : OpenACC_Op<"loop",
             $workerNumOperandsDeviceType, $worker)
       | `vector` `` custom<DeviceTypeOperandsWithKeywordOnly>($vectorOperands,
             type($vectorOperands), $vectorOperandsDeviceType, $vector)
-      | `private` `(` custom<SymOperandList>(
-            $privateOperands, type($privateOperands), $privatizationRecipes) `)`
+      | `private` `(` $privateOperands  `:` type($privateOperands) `)`
       | `tile` `(` custom<DeviceTypeOperandsWithSegment>($tileOperands,
             type($tileOperands), $tileOperandsDeviceType, $tileOperandsSegments)
         `)`
-      | `reduction` `(` custom<SymOperandList>(
-            $reductionOperands, type($reductionOperands), $reductionRecipes)
-        `)`
+      | `reduction` `(` $reductionOperands  `:` type($reductionOperands) `)`
       | `cache` `(` $cacheOperands `:` type($cacheOperands) `)`
     )
     custom<LoopControl>($region, $lowerbound, type($lowerbound), $upperbound,
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 9e4982dc59d49..0039c6bde89e4 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -333,10 +333,18 @@ checkValidModifier(Op op, acc::DataClauseModifier validModifiers) {
   return success();
 }
 
-template <typename Op>
-static LogicalResult checkNoRecipe(Op op) {
-  if (op.getRecipe().has_value())
-    return op.emitError("no recipes are allowed");
+template <typename OpT, typename RecipeOpT>
+static LogicalResult checkRecipe(OpT op, llvm::StringRef operandName) {
+  mlir::SymbolRefAttr operandRecipe = op.getRecipeAttr();
+  if (!operandRecipe)
+    return op->emitOpError() << "recipe expected for " << operandName;
+
+  auto decl =
+      SymbolTable::lookupNearestSymbolFrom<RecipeOpT>(op, operandRecipe);
+  if (!decl)
+    return op->emitOpError()
+           << "expected symbol reference " << operandRecipe << " to point to a "
+           << operandName << " declaration";
   return success();
 }
 
@@ -446,6 +454,18 @@ static void printVarPtrType(mlir::OpAsmPrinter &p, mlir::Operation *op,
   }
 }
 
+static ParseResult parseRecipeSym(mlir::OpAsmParser &parser,
+                                  mlir::SymbolRefAttr &recipeAttr) {
+  if (failed(parser.parseAttribute(recipeAttr)))
+    return failure();
+  return success();
+}
+
+static void printRecipeSym(mlir::OpAsmPrinter &p, mlir::Operation *op,
+  mlir::SymbolRefAttr recipeAttr) {
+  p << recipeAttr;
+}
+
 //===----------------------------------------------------------------------===//
 // DataBoundsOp
 //===----------------------------------------------------------------------===//
@@ -468,6 +488,9 @@ LogicalResult acc::PrivateOp::verify() {
     return failure();
   if (failed(checkNoModifier(*this)))
     return failure();
+  if (failed(
+          checkRecipe<acc::PrivateOp, acc::PrivateRecipeOp>(*this, "private")))
+    return failure();
   return success();
 }
 
@@ -482,6 +505,9 @@ LogicalResult acc::FirstprivateOp::verify() {
     return failure();
   if (failed(checkNoModifier(*this)))
     return failure();
+  if (failed(checkRecipe<acc::FirstprivateOp, acc::FirstprivateRecipeOp>(
+          *this, "firstprivate")))
+    return failure();
   return success();
 }
 
@@ -496,6 +522,9 @@ LogicalResult acc::ReductionOp::verify() {
     return failure();
   if (failed(checkNoModifier(*this)))
     return failure();
+  if (failed(checkRecipe<acc::ReductionOp, acc::ReductionRecipeOp>(
+          *this, "reduction")))
+    return failure();
   return success();
 }
 
@@ -1008,40 +1037,6 @@ LogicalResult acc::ReductionRecipeOp::verifyRegions() {
   return success();
 }
 
-//===----------------------------------------------------------------------===//
-// Custom parser and printer verifier for private clause
-//===----------------------------------------------------------------------===//
-
-static ParseResult parseSymOperandList(
-    mlir::OpAsmParser &parser,
-    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
-    llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &symbols) {
-  llvm::SmallVector<SymbolRefAttr> attributes;
-  if (failed(parser.parseCommaSeparatedList([&]() {
-        if (parser.parseAttribute(attributes.emplace_back()) ||
-            parser.parseArrow() ||
-            parser.parseOperand(operands.emplace_back()) ||
-            parser.parseColonType(types.emplace_back()))
-          return failure();
-        return success();
-      })))
-    return failure();
-  llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
-                                               attributes.end());
-  symbols = ArrayAttr::get(parser.getContext(), arrayAttr);
-  return success();
-}
-
-static void printSymOperandList(mlir::OpAsmPrinter &p, mlir::Operation *op,
-                                mlir::OperandRange operands,
-                                mlir::TypeRange types,
-                                std::optional<mlir::ArrayAttr> attributes) {
-  llvm::interleaveComma(llvm::zip(*attributes, operands), p, [&](auto it) {
-    p << std::get<0>(it) << " -> " << std::get<1>(it) << " : "
-      << std::get<1>(it).getType();
-  });
-}
-
 //===----------------------------------------------------------------------===//
 // ParallelOp
 //===----------------------------------------------------------------------===//
@@ -1050,7 +1045,7 @@ static void printSymOperandList(mlir::OpAsmPrinter &p, mlir::Operation *op,
 template <typename Op>
 static LogicalResult checkDataOperands(Op op,
                                        const mlir::ValueRange &operands) {
-  for (mlir::Value operand : operands)
+  for (mlir::Value operand : operands) {
     if (!mlir::isa<acc::AttachOp, acc::CopyinOp, acc::CopyoutOp, acc::CreateOp,
                    acc::DeleteOp, acc::DetachOp, acc::DevicePtrOp,
                    acc::GetDevicePtrOp, acc::NoCreateOp, acc::PresentOp>(
@@ -1058,66 +1053,26 @@ static LogicalResult checkDataOperands(Op op,
       return op.emitError(
           "expect data entry/exit operation or acc.getdeviceptr "
           "as defining op");
+    mlir::SymbolRefAttr operandRecipe = getRecipe(operand.getDefiningOp());
+    if (operandRecipe)
+      return op.emitError("recipe not expected for data mapping operations");
+  }
   return success();
 }
 
-template <typename Op>
-static LogicalResult
-checkSymOperandList(Operation *op, std::optional<mlir::ArrayAttr> attributes,
-                    mlir::OperandRange operands, llvm::StringRef operandName,
-                    llvm::StringRef symbolName, bool checkOperandType = true) {
-  if (!operands.empty()) {
-    if (!attributes || attributes->size() != operands.size())
-      return op->emitOpError()
-             << "expected as many " << symbolName << " symbol reference as "
-             << operandName << " operands";
-    if (attributes) {
-      for (auto operandAndAttribute : llvm::zip(operands, *attributes)) {
-        mlir::Value operand = std::get<0>(operandAndAttribute);
-        if (auto *definingOp = operand.getDefiningOp()) {
-          mlir::SymbolRefAttr operandRecipe = getRecipe(definingOp);
-          // If the operand operation has a recipe - check that it is consistent
-          // with the one recorded in the construct.
-          if (operandRecipe) {
-            if (operandRecipe.getLeafReference().compare(
-                    llvm::cast<mlir::SymbolRefAttr>(
-                        std::get<1>(operandAndAttribute))
-                        .getLeafReference()) != 0)
-              return op->emitOpError() << "expected consistent recipe for "
-                                       << operandName << " operand";
-          }
-        }
-      }
-    }
-  } else {
-    if (attributes)
-      return op->emitOpError()
-             << "unexpected " << symbolName << " symbol reference";
-    return success();
-  }
-
+template <typename OpT, typename RecipeOpT>
+static LogicalResult checkPrivateOperands(mlir::Operation *accConstructOp,
+                                          const mlir::ValueRange &operands,
+                                          llvm::StringRef operandName) {
   llvm::DenseSet<Value> set;
-  for (auto args : llvm::zip(operands, *attributes)) {
-    mlir::Value operand = std::get<0>(args);
-
+  for (mlir::Value operand : operands) {
+    if (!mlir::isa<OpT>(operand.getDefiningOp()))
+      return accConstructOp->emitOpError()
+             << "expect " << operandName << " as defining op";
     if (!set.insert(operand).second)
-      return op->emitOpError()
+      return accConstructOp->emitOpError()
              << operandName << " operand appears more than once";
-
-    mlir::Type varType = operand.getType();
-    auto symbolRef = llvm::cast<SymbolRefAttr>(std::get<1>(args));
-    auto decl = SymbolTable::lookupNearestSymbolFrom<Op>(op, symbolRef);
-    if (!decl)
-      return op->emitOpError()
-             << "expected symbol reference " << symbolRef << " to point to a "
-             << operandName << " declaration";
-
-    if (checkOperandType && decl.getType() && decl.getType() != varType)
-      return op->emitOpError() << "expected " << operandName << " (" << varType
-                               << ") to be the same type as " << operandName
-                               << " declaration (" << decl.getType() << ")";
   }
-
   return success();
 }
 
@@ -1174,17 +1129,17 @@ static LogicalResult verifyDeviceTypeAndSegmentCountMatch(
 }
 
 LogicalResult acc::ParallelOp::verify() {
-  if (failed(checkSymOperandList<mlir::acc::PrivateRecipeOp>(
-          *this, getPrivatizationRecipes(), getPrivateOperands(), "private",
-          "privatizations", /*checkOperandType=*/false)))
+  if (failed(checkPrivateOperands<mlir::acc::PrivateOp,
+                                  mlir::acc::PrivateRecipeOp>(
+          *this, getPrivateOperands(), "private")))
     return failure();
-  if (failed(checkSymOperandList<mlir::acc::FirstprivateRecipeOp>(
-          *this, getFirstprivatizationRecipes(), getFirstprivateOperands(),
-          "firstprivate", "firstprivatizations", /*checkOperandType=*/false)))
+  if (failed(checkPrivateOperands<mlir::acc::FirstprivateOp,
+                                  mlir::acc::FirstprivateRecipeOp>(
+          *this, getFirstprivateOperands(), "firstprivate")))
     return failure();
-  if (failed(checkSymOperandList<mlir::acc::ReductionRecipeOp>(
-          *this, getReductionRecipes(), getReductionOperands(), "reduction",
-          "reductions", false)))
+  if (failed(checkPrivateOperands<mlir::acc::ReductionOp,
+                                  mlir::acc::ReductionRecipeOp>(
+          *this, getReductionOperands(), "reduction")))
     return failure();
 
   if (failed(verifyDeviceTypeAndSegmentCountMatch(
@@ -1315,7 +1270,6 @@ void ParallelOp::build(mlir::OpBuilder &odsBuilder,
                        mlir::ValueRange gangPrivateOperands,
                        mlir::ValueRange gangFirstPrivateOperands,
                        mlir::ValueRange dataClauseOperands) {
-
   ParallelOp::build(
       odsBuilder, odsState, asyncOperands, /*asyncOperandsDeviceType=*/nullptr,
       /*asyncOnly=*/nullptr, waitOperands, /*waitOperandsSegments=*/nullptr,
@@ -1324,9 +1278,8 @@ void ParallelOp::build(mlir::OpBuilder &odsBuilder,
       /*numGangsDeviceType=*/nullptr, numWorkers,
       /*numWorkersDeviceType=*/nullptr, vectorLength,
       /*vectorLengthDeviceType=*/nullptr, ifCond, selfCond,
-      /*selfAttr=*/nullptr, reductionOperands, /*reductionRecipes=*/nullptr,
-      gangPrivateOperands, /*privatizations=*/nullptr, gangFirstPrivateOperands,
-      /*firstprivatizations=*/nullptr, dataClauseOperands,
+      /*selfAttr=*/nullptr, reductionOperands, gangPrivateOperands,
+      gangFirstPrivateOperands, dataClauseOperands,
       /*defaultAttr=*/nullptr, /*combined=*/nullptr);
 }
 
@@ -1965,17 +1918,17 @@ mlir::Value SerialOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {
 }
 
 LogicalResult acc::SerialOp::verify() {
-  if (failed(checkSymOperandList<mlir::acc::PrivateRecipeOp>(
-          *this, getPrivatizationRecipes(), getPrivateOperands(), "private",
-          "privatizations", /*checkOperandType=*/false)))
+  if (failed(checkPrivateOperands<mlir::acc::PrivateOp,
+                                  mlir::acc::PrivateRecipeOp>(
+          *this, getPrivateOperands(), "private")))
     return failure();
-  if (failed(checkSymOperandList<mlir::acc::FirstprivateRecipeOp>(
-          *this, getFirstprivatizationRecipes(), getFirstprivateOperands(),
-          "firstprivate", "firstprivatizations", /*checkOperandType=*/false)))
+  if (failed(checkPrivateOperands<mlir::acc::FirstprivateOp,
+                                  mlir::acc::FirstprivateRecipeOp>(
+          *this, getFirstprivateOperands(), "firstprivate")))
     return failure();
-  if (failed(checkSymOperandList<mlir::acc::ReductionRecipeOp>(
-          *this, getReductionRecipes(), getReductionOperands(), "reduction",
-          "reductions", false)))
+  if (failed(checkPrivateOperands<mlir::acc::ReductionOp,
+                                  mlir::acc::ReductionRecipeOp>(
+          *this, getReductionOperands(), "reduction")))
     return failure();
 
   if (failed(verifyDeviceTypeAndSegmentCountMatch(
@@ -2606,14 +2559,13 @@ LogicalResult acc::LoopOp::verify() {
     }
   }
 
-  if (failed(checkSymOperandList<mlir::acc::PrivateRecipeOp>(
-          *this, getPrivatizationRecipes(), getPrivateOperands(), "private",
-          "privatizations", false)))
+  if (failed(checkPrivateOperands<mlir::acc::PrivateOp,
+                                  mlir::acc::PrivateRecipeOp>(
+          *this, getPrivateOperands(), "private")))
     return failure();
-
-  if (failed(checkSymOperandList<mlir::acc::ReductionRecipeOp>(
-          *this, getReductionRecipes(), getReductionOperands(), "reduction",
-          "reductions", false)))
+  if (failed(checkPrivateOperands<mlir::acc::ReductionOp,
+                                  mlir::acc::ReductionRecipeOp>(
+          *this, getReductionOperands(), "reduction")))
     return failure();
 
   if (getCombined().has_value() &&
diff --git a/mlir/test/Dialect/OpenACC/invalid.mlir b/mlir/test/Dialect/OpenACC/invalid.mlir
index 68afd9fccba79..1d9fe0a1a0c35 100644
--- a/mlir/test/Dialect/OpenACC/invalid.mlir
+++ b/mlir/test/Dialect/OpenACC/invalid.mlir
@@ -483,15 +483,6 @@ acc.loop gang({static=%i64Value: i64, ) control(%iv : i32) = (%1 : i32) to (%2 :
 
 // -----
 
-func.func @fct1(%0 : !llvm.ptr) -> () {
-  // expected-error at +1 {{expected symbol reference @privatization_i32 to point to a private declaration}}
-  acc.serial private(@privatization_i32 -> %0 : !llvm.ptr) {
-  }
-  return
-}
-
-// -----
-
 // expected-error at +1 {{expect at least one of num, dim or static values}}
 acc.loop gang({}) {
   "test.openacc_dummy_op"() : () -> ()
@@ -831,3 +822,4 @@ func.func @acc_loop_container() {
 %value = memref.alloc() : memref<f32>
 // expected-error @below {{invalid data clause modifiers: readonly}}
 %0 = acc.create varPtr(%value : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier readonly,zero,capture,always>}
+
diff --git a/mlir/test/Dialect/OpenACC/legalize-data.mlir b/mlir/test/Dialect/OpenACC/legalize-data.mlir
index 40604dcc736de..c7ef47ccc7e97 100644
--- a/mlir/test/Dialect/OpenACC/legalize-data.mlir
+++ b/mlir/test/Dialect/OpenACC/legalize-data.mlir
@@ -129,8 +129,8 @@ func.func @test(%a: memref<10xf32>) {
   %lb = arith.constant 0 : index
   %st = arith.constant 1 : index
   %c10 = arith.constant 10 : index
-  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
-  acc.parallel private(@privatization_memref_10_f32 -> %p1 : memref<10xf32>) {
+  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+  acc.parallel private(%p1 : memref<10xf32>) {
     acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
       acc.yield
@@ -142,8 +142,8 @@ func.func @test(%a: memref<10xf32>) {
 
 // CHECK-LABEL: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>)
-// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
-// CHECK: acc.parallel private(@privatization_memref_10_f32 -> %[[PRIVATE]] : memref<10xf32>) {
+// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+// CHECK: acc.parallel private(%[[PRIVATE]] : memref<10xf32>) {
 // CHECK:   acc.loop control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
 // CHECK:     acc.yield
@@ -167,9 +167,9 @@ func.func @test(%a: memref<10xf32>) {
   %lb = arith.constant 0 : index
   %st = arith.constant 1 : index
   %c10 = arith.constant 10 : index
-  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
+  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
   acc.parallel {
-    acc.loop private(@privatization_memref_10_f32 -> %p1 : memref<10xf32>) control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
+    acc.loop private(%p1 : memref<10xf32>) control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
       acc.yield
     } attributes {independent = [#acc.device_type<none>]}
@@ -180,9 +180,9 @@ func.func @test(%a: memref<10xf32>) {
 
 // CHECK-LABEL: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>)
-// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
+// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
 // CHECK: acc.parallel  {
-// CHECK:   acc.loop private(@privatization_memref_10_f32 -> %[[PRIVATE]] : memref<10xf32>) control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
+// CHECK:   acc.loop private(%[[PRIVATE]] : memref<10xf32>) control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
 // CHECK:     acc.yield
 // CHECK:   } attributes {independent = [#acc.device_type<none>]}
@@ -205,8 +205,8 @@ func.func @test(%a: memref<10xf32>) {
   %lb = arith.constant 0 : index
   %st = arith.constant 1 : index
   %c10 = arith.constant 10 : index
-  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
-  acc.serial private(@privatization_memref_10_f32 -> %p1 : memref<10xf32>) {
+  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+  acc.serial private(%p1 : memref<10xf32>) {
     acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
       acc.yield
@@ -218,8 +218,8 @@ func.func @test(%a: memref<10xf32>) {
 
 // CHECK-LABEL: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>)
-// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
-// CHECK: acc.serial private(@privatization_memref_10_f32 -> %[[PRIVATE]] : memref<10xf32>) {
+// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+// CHECK: acc.serial private(%[[PRIVATE]] : memref<10xf32>) {
 // CHECK:   acc.loop control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
 // CHECK:     acc.yield
diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir
index 7bb6cf43e49a7..ffafef294066d 100644
--- a/mlir/test/Dialect/OpenACC/ops.mlir
+++ b/mlir/test/Dialect/OpenACC/ops.mlir
@@ -120,8 +120,8 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x
   %pc = acc.present varPtr(%c : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   %pd = acc.present varPtr(%d : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.data dataOperands(%pa, %pb, %pc, %pd: memref<10x10xf32>, memref<10x10xf32>, memref<10xf32>, memref<10xf32>) {
-    %private = acc.private varPtr(%c : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
-    acc.parallel num_gangs({%numGangs: i64}) num_workers(%numWorkers: i64 [#acc.device_type<nvidia>]) private(@privatization_memref_10_f32 -> %private : memref<10xf32>) {
+    %private = acc.private varPtr(%c : memref<10xf32>) varType(tensor<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+    acc.parallel num_gangs({%numGangs: i64}) num_workers(%numWorkers: i64 [#acc.device_type<nvidia>]) private(%private : memref<10xf32>) {
       acc.loop gang control(%x : index) = (%lb : index) to (%c10 : index) step (%st : index) {
         acc.loop worker control(%y : index) = (%lb : index) to (%c10 : index) step (%st : index) {
           %axy = memref.load %a[%x, %y] : memref<10x10xf32>
@@ -157,8 +157,8 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x
 // CHECK-NEXT:   [[NUMGANG:%.*]] = arith.constant 10 : i64
 // CHECK-NEXT:   [[NUMWORKERS:%.*]] = arith.constant 10 : i64
 // CHECK:        acc.data dataOperands(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : memref<10x10xf32>, memref<10x10xf32>, memref<10xf32>, memref<10xf32>) {
-// CHECK-NEXT:     %[[P_ARG2:.*]] = acc.private varPtr([[ARG2]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> 
-// CHECK-NEXT:     acc.parallel num_gangs({[[NUMGANG]] : i64}) num_workers([[NUMWORKERS]] : i64 [#acc.device_type<nvidia>]) private(@privatization_memref_10_f32 -> %[[P_ARG2]] : memref<10xf32>) {
+// CHECK-NEXT:     %[[P_ARG2:.*]] = acc.private varPtr([[ARG2]] : memref<10xf32>) varType(tensor<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+// CHECK-NEXT:     acc.parallel num_gangs({[[NUMGANG]] : i64}) num_workers([[NUMWORKERS]] : i64 [#acc.device_type<nvidia>]) private(%[[P_ARG2]] : memref<10xf32>) {
 // CHECK-NEXT:       acc.loop gang control(%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
 // CHECK-NEXT:         acc.loop worker control(%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
 // CHECK-NEXT:           %{{.*}} = memref.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<10x10xf32>
@@ -429,7 +429,10 @@ func.func @testparallelop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x
   }
   acc.parallel vector_length(%idxValue: index) {
   }
-  acc.parallel private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(@privatization_memref_10xf32 -> %b: memref<10xf32>) {
+  %private_a = acc.private varPtr(%a : memref<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+  %private_c = acc.private varPtr(%c : memref<10x10xf32>) recipe(@privatization_memref_10_10_f32) -> memref<10x10xf32>
+  %firstprivate_b = acc.firstprivate varPtr(%b : memref<10xf32>) recipe(@privatization_memref_10xf32) -> memref<10xf32>
+  acc.parallel private(%private_a, %private_c : memref<10xf32>, memref<10x10xf32>) firstprivate(%firstprivate_b : memref<10xf32>) {
   }
   acc.parallel {
   } attributes {defaultAttr = #acc<defaultvalue none>}
@@ -482,7 +485,10 @@ func.func @testparallelop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x
 // CHECK-NEXT: }
 // CHECK:      acc.parallel vector_length([[IDXVALUE]] : index) {
 // CHECK-NEXT: }
-// CHECK:      acc.parallel firstprivate(@privatization_memref_10xf32 -> [[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) {
+// CHECK:      %[[PRIVATE_A:.*]] = acc.private varPtr([[ARGA]] : memref<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+// CHECK-NEXT: %[[PRIVATE_C:.*]] = acc.private varPtr([[ARGC]] : memref<10x10xf32>) recipe(@privatization_memref_10_10_f32) -> memref<10x10xf32>
+// CHECK-NEXT: %[[FIRSTPRIVATE_B:.*]] = acc.firstprivate varPtr([[ARGB]] : memref<10xf32>) recipe(@privatization_memref_10xf32) -> memref<10xf32>
+// CHECK-NEXT: acc.parallel firstprivate(%[[FIRSTPRIVATE_B]] : memref<10xf32>) private(%[[PRIVATE_A]], %[[PRIVATE_C]] : memref<10xf32>, memref<10x10xf32>) {
 // CHECK-NEXT: }
 // CHECK:      acc.parallel {
 // CHECK-NEXT: } attributes {defaultAttr = #acc<defaultvalue none>}
@@ -560,8 +566,10 @@ func.func @testserialop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x10
   }
   acc.serial wait({%i64value : i64, %i32value : i32, %idxValue : index}) {
   }
-  %firstprivate = acc.firstprivate varPtr(%b : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
-  acc.serial private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(@firstprivatization_memref_10xf32 -> %firstprivate : memref<10xf32>) {
+  %private_a = acc.private varPtr(%a : memref<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+  %private_c = acc.private varPtr(%c : memref<10x10xf32>) recipe(@privatization_memref_10_10_f32) -> memref<10x10xf32>
+  %firstprivate = acc.firstprivate varPtr(%b : memref<10xf32>) varType(tensor<10xf32>) recipe(@firstprivatization_memref_10xf32) -> memref<10xf32>
+  acc.serial private(%private_a, %private_c : memref<10xf32>, memref<10x10xf32>) firstprivate(%firstprivate : memref<10xf32>) {
   }
   acc.serial {
   } attributes {defaultAttr = #acc<defaultvalue none>}
@@ -597,8 +605,10 @@ func.func @testserialop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x10
 // CHECK-NEXT: }
 // CHECK:      acc.serial wait({[[I64VALUE]] : i64, [[I32VALUE]] : i32, [[IDXVALUE]] : index}) {
 // CHECK-NEXT: }
-// CHECK:      %[[FIRSTP:.*]] = acc.firstprivate varPtr([[ARGB]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
-// CHECK:      acc.serial firstprivate(@firstprivatization_memref_10xf32 -> %[[FIRSTP]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) {
+// CHECK:      %[[PRIVATE_A:.*]] = acc.private varPtr([[ARGA]] : memref<10xf32>) recipe(@privatization_memref_10_f32) -> memref<10xf32>
+// CHECK-NEXT: %[[PRIVATE_C:.*]] = acc.private varPtr([[ARGC]] : memref<10x10xf32>) recipe(@privatization_memref_10_10_f32) -> memref<10x10xf32>
+// CHECK-NEXT: %[[FIRSTP:.*]] = acc.firstprivate varPtr([[ARGB]] : memref<10xf32>) varType(tensor<10xf32>) recipe(@firstprivatization_memref_10xf32) -> memref<10xf32>
+// CHECK-NEXT: acc.serial firstprivate(%[[FIRSTP]] : memref<10xf32>) private(%[[PRIVATE_A]], %[[PRIVATE_C]] : memref<10xf32>, memref<10x10xf32>) {
 // CHECK-NEXT: }
 // CHECK:      acc.serial {
 // CHECK-NEXT: } attributes {defaultAttr = #acc<defaultvalue none>}
@@ -1475,32 +1485,43 @@ acc.private.recipe @privatization_struct_i32_i64 : !llvm.struct<(i32, i32)> init
 
 // -----
 
-acc.reduction.recipe @reduction_add_i64 : i64 reduction_operator<add> init {
-^bb0(%arg0: i64):
-  %0 = arith.constant 0 : i64
-  acc.yield %0 : i64
+acc.reduction.recipe @reduction_add_memref_i64 : memref<i64> reduction_operator <add> init {
+^bb0(%arg0: memref<i64>):
+  %c0_i64 = arith.constant 0 : i64
+  %alloca = memref.alloca() : memref<i64>
+  memref.store %c0_i64, %alloca[] : memref<i64>
+  acc.yield %alloca : memref<i64>
 } combiner {
-^bb0(%arg0: i64, %arg1: i64):
-  %0 = arith.addi %arg0, %arg1 : i64
-  acc.yield %0 : i64
+^bb0(%arg0: memref<i64>, %arg1: memref<i64>):
+  %0 = memref.load %arg0[] : memref<i64>
+  %1 = memref.load %arg1[] : memref<i64>
+  %2 = arith.addi %0, %1 : i64
+  memref.store %2, %arg0[] : memref<i64>
+  acc.yield %arg0 : memref<i64>
 }
 
-// CHECK-LABEL: acc.reduction.recipe @reduction_add_i64 : i64 reduction_operator <add> init {
-// CHECK:       ^bb0(%{{.*}}: i64):
+// CHECK-LABEL: acc.reduction.recipe @reduction_add_memref_i64 : memref<i64> reduction_operator <add> init {
+// CHECK:       ^bb0(%{{.*}}: memref<i64>):
 // CHECK:         %[[C0:.*]] = arith.constant 0 : i64
-// CHECK:         acc.yield %[[C0]] : i64
+// CHECK:         %[[ALLOCA:.*]] = memref.alloca() : memref<i64>
+// CHECK:         memref.store %[[C0]], %[[ALLOCA]][] : memref<i64>
+// CHECK:         acc.yield %[[ALLOCA]] : memref<i64>
 // CHECK:       } combiner {
-// CHECK:       ^bb0(%[[ARG0:.*]]: i64, %[[ARG1:.*]]: i64):
-// CHECK:         %[[RES:.*]] = arith.addi %[[ARG0]], %[[ARG1]] : i64
-// CHECK:         acc.yield %[[RES]] : i64
+// CHECK:       ^bb0(%[[ARG0:.*]]: memref<i64>, %[[ARG1:.*]]: memref<i64>):
+// CHECK:         %[[LOAD0:.*]] = memref.load %[[ARG0]][] : memref<i64>
+// CHECK:         %[[LOAD1:.*]] = memref.load %[[ARG1]][] : memref<i64>
+// CHECK:         %[[RES:.*]] = arith.addi %[[LOAD0]], %[[LOAD1]] : i64
+// CHECK:         memref.store %[[RES]], %[[ARG0]][] : memref<i64>
+// CHECK:         acc.yield %[[ARG0]] : memref<i64>
 // CHECK:       }
 
-func.func @acc_reduc_test(%a : i64) -> () {
+func.func @acc_reduc_test(%a : memref<i64>) -> () {
   %c0 = arith.constant 0 : index
   %c10 = arith.constant 10 : index
   %c1 = arith.constant 1 : index
-  acc.parallel reduction(@reduction_add_i64 -> %a : i64) {
-    acc.loop reduction(@reduction_add_i64 -> %a : i64) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
+  %reduction_a = acc.reduction varPtr(%a : memref<i64>) recipe(@reduction_add_memref_i64) ->  memref<i64>
+  acc.parallel reduction(%reduction_a :  memref<i64>) {
+    acc.loop reduction(%reduction_a :  memref<i64>) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
       acc.yield
     } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
     acc.yield
@@ -1509,31 +1530,39 @@ func.func @acc_reduc_test(%a : i64) -> () {
 }
 
 // CHECK-LABEL: func.func @acc_reduc_test(
-// CHECK-SAME:    %[[ARG0:.*]]: i64)
-// CHECK:         acc.parallel reduction(@reduction_add_i64 -> %[[ARG0]] : i64)
-// CHECK:           acc.loop reduction(@reduction_add_i64 -> %[[ARG0]] : i64)
+// CHECK-SAME:    %[[ARG0:.*]]: memref<i64>)
+// CHECK:         %[[REDUCTION_A:.*]] = acc.reduction varPtr(%[[ARG0]] : memref<i64>) recipe(@reduction_add_memref_i64) -> memref<i64>
+// CHECK-NEXT:    acc.parallel reduction(%[[REDUCTION_A]] : memref<i64>)
+// CHECK:           acc.loop reduction(%[[REDUCTION_A]] : memref<i64>)
 
 // -----
 
-acc.reduction.recipe @reduction_add_i64 : i64 reduction_operator<add> init {
-^bb0(%0: i64):
-  %1 = arith.constant 0 : i64
-  acc.yield %1 : i64
+acc.reduction.recipe @reduction_add_memref_i64 : memref<i64> reduction_operator <add> init {
+^bb0(%arg0: memref<i64>):
+  %c0_i64 = arith.constant 0 : i64
+  %alloca = memref.alloca() : memref<i64>
+  memref.store %c0_i64, %alloca[] : memref<i64>
+  acc.yield %alloca : memref<i64>
 } combiner {
-^bb0(%0: i64, %1: i64):
+^bb0(%arg0: memref<i64>, %arg1: memref<i64>):
+  %0 = memref.load %arg0[] : memref<i64>
+  %1 = memref.load %arg1[] : memref<i64>
   %2 = arith.addi %0, %1 : i64
-  acc.yield %2 : i64
+  memref.store %2, %arg0[] : memref<i64>
+  acc.yield %arg0 : memref<i64>
 }
 
-func.func @acc_reduc_test(%a : i64) -> () {
-  acc.serial reduction(@reduction_add_i64 -> %a : i64) {
+func.func @acc_reduc_test(%a : memref<i64>) -> () {
+  %reduction_a = acc.reduction varPtr(%a : memref<i64>) recipe(@reduction_add_memref_i64) -> memref<i64>
+  acc.serial reduction(%reduction_a : memref<i64>) {
   }
   return
 }
 
 // CHECK-LABEL: func.func @acc_reduc_test(
-// CHECK-SAME:    %[[ARG0:.*]]: i64)
-// CHECK:         acc.serial reduction(@reduction_add_i64 -> %[[ARG0]] : i64)
+// CHECK-SAME:    %[[ARG0:.*]]: memref<i64>)
+// CHECK:         %[[REDUCTION_A:.*]] = acc.reduction varPtr(%[[ARG0]] : memref<i64>) recipe(@reduction_add_memref_i64) -> memref<i64>
+// CHECK-NEXT:    acc.serial reduction(%[[REDUCTION_A]] : memref<i64>)
 
 // -----
 

>From fb1b8c86c55feb193063c3e7751f4d18e553f8da Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Tue, 22 Jul 2025 16:38:38 -0700
Subject: [PATCH 3/6] Update invalid test to include new messages

---
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp |  2 +-
 mlir/test/Dialect/OpenACC/invalid.mlir  | 45 +++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 0039c6bde89e4..122cadbb11922 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -1068,7 +1068,7 @@ static LogicalResult checkPrivateOperands(mlir::Operation *accConstructOp,
   for (mlir::Value operand : operands) {
     if (!mlir::isa<OpT>(operand.getDefiningOp()))
       return accConstructOp->emitOpError()
-             << "expect " << operandName << " as defining op";
+             << "expected " << operandName << " as defining op";
     if (!set.insert(operand).second)
       return accConstructOp->emitOpError()
              << operandName << " operand appears more than once";
diff --git a/mlir/test/Dialect/OpenACC/invalid.mlir b/mlir/test/Dialect/OpenACC/invalid.mlir
index 1d9fe0a1a0c35..bb3aef339639f 100644
--- a/mlir/test/Dialect/OpenACC/invalid.mlir
+++ b/mlir/test/Dialect/OpenACC/invalid.mlir
@@ -823,3 +823,48 @@ func.func @acc_loop_container() {
 // expected-error @below {{invalid data clause modifiers: readonly}}
 %0 = acc.create varPtr(%value : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier readonly,zero,capture,always>}
 
+// -----
+
+func.func @fct1(%0 : !llvm.ptr) -> () {
+  // expected-error @below {{expected symbol reference @privatization_i32 to point to a private declaration}}
+  %priv = acc.private varPtr(%0 : !llvm.ptr) varType(i32) recipe(@privatization_i32) -> !llvm.ptr
+  return
+}
+
+// -----
+
+acc.private.recipe @privatization_i32 : !llvm.ptr init {
+^bb0(%arg0: !llvm.ptr):
+  %c1 = arith.constant 1 : i32
+  %c0 = arith.constant 0 : i32
+  %0 = llvm.alloca %c1 x i32 : (i32) -> !llvm.ptr
+  llvm.store %c0, %0 : i32, !llvm.ptr
+  acc.yield %0 : !llvm.ptr
+}
+
+func.func @fct1(%0 : !llvm.ptr) -> () {
+  %priv = acc.private varPtr(%0 : !llvm.ptr) varType(i32) recipe(@privatization_i32) -> !llvm.ptr
+  // expected-error @below {{expected firstprivate as defining op}}
+  acc.serial firstprivate(%priv : !llvm.ptr) {
+  }
+  return
+}
+
+// -----
+
+acc.private.recipe @privatization_i32 : !llvm.ptr init {
+^bb0(%arg0: !llvm.ptr):
+  %c1 = arith.constant 1 : i32
+  %c0 = arith.constant 0 : i32
+  %0 = llvm.alloca %c1 x i32 : (i32) -> !llvm.ptr
+  llvm.store %c0, %0 : i32, !llvm.ptr
+  acc.yield %0 : !llvm.ptr
+}
+
+func.func @fct1(%0 : !llvm.ptr) -> () {
+  %priv = acc.private varPtr(%0 : !llvm.ptr) varType(i32) recipe(@privatization_i32) -> !llvm.ptr
+  // expected-error @below {{op private operand appears more than once}}
+  acc.serial private(%priv, %priv : !llvm.ptr, !llvm.ptr) {
+  }
+  return
+}

>From 3d04bd6fdf4b7e5d57ceffa617c803e9044bbbc4 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Tue, 22 Jul 2025 16:41:25 -0700
Subject: [PATCH 4/6] Fix format

---
 flang/lib/Lower/OpenACC.cpp             | 47 ++++++++++++-------------
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp |  2 +-
 2 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 50571a70e3aaf..366632e43db83 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -1317,15 +1317,15 @@ mlir::Type getTypeFromBounds(llvm::SmallVector<mlir::Value> &bounds,
 }
 
 template <typename RecipeOp>
-static void genPrivatizationRecipes(
-    const Fortran::parser::AccObjectList &objectList,
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semanticsContext,
-    Fortran::lower::StatementContext &stmtCtx,
-    llvm::SmallVectorImpl<mlir::Value> &dataOperands,
-    llvm::ArrayRef<mlir::Value> async,
-    llvm::ArrayRef<mlir::Attribute> asyncDeviceTypes,
-    llvm::ArrayRef<mlir::Attribute> asyncOnlyDeviceTypes) {
+static void
+genPrivatizationRecipes(const Fortran::parser::AccObjectList &objectList,
+                        Fortran::lower::AbstractConverter &converter,
+                        Fortran::semantics::SemanticsContext &semanticsContext,
+                        Fortran::lower::StatementContext &stmtCtx,
+                        llvm::SmallVectorImpl<mlir::Value> &dataOperands,
+                        llvm::ArrayRef<mlir::Value> async,
+                        llvm::ArrayRef<mlir::Attribute> asyncDeviceTypes,
+                        llvm::ArrayRef<mlir::Attribute> asyncOnlyDeviceTypes) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   Fortran::evaluate::ExpressionAnalyzer ea{semanticsContext};
   for (const auto &accObject : objectList.v) {
@@ -2011,14 +2011,14 @@ mlir::Type getTypeFromIvTypeSize(fir::FirOpBuilder &builder,
   return builder.getIntegerType(ivTypeSize * 8);
 }
 
-static void
-privatizeIv(Fortran::lower::AbstractConverter &converter,
-            const Fortran::semantics::Symbol &sym, mlir::Location loc,
-            llvm::SmallVector<mlir::Type> &ivTypes,
-            llvm::SmallVector<mlir::Location> &ivLocs,
-            llvm::SmallVector<mlir::Value> &privateOperands,
-            llvm::SmallVector<mlir::Value> &ivPrivate,
-            bool isDoConcurrent = false) {
+static void privatizeIv(Fortran::lower::AbstractConverter &converter,
+                        const Fortran::semantics::Symbol &sym,
+                        mlir::Location loc,
+                        llvm::SmallVector<mlir::Type> &ivTypes,
+                        llvm::SmallVector<mlir::Location> &ivLocs,
+                        llvm::SmallVector<mlir::Value> &privateOperands,
+                        llvm::SmallVector<mlir::Value> &ivPrivate,
+                        bool isDoConcurrent = false) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
   mlir::Type ivTy = getTypeFromIvTypeSize(builder, sym);
@@ -2358,8 +2358,7 @@ static mlir::acc::LoopOp createLoopOp(
 
       const auto &name = std::get<Fortran::parser::Name>(control.t);
       privatizeIv(converter, *name.symbol, currentLocation, ivTypes, ivLocs,
-                  privateOperands, ivPrivate,
-                  isDoConcurrent);
+                  privateOperands, ivPrivate, isDoConcurrent);
 
       inclusiveBounds.push_back(true);
     }
@@ -2807,15 +2806,13 @@ static Op createComputeOp(
       if (!combinedConstructs)
         genPrivatizationRecipes<mlir::acc::PrivateRecipeOp>(
             privateClause->v, converter, semanticsContext, stmtCtx,
-            privateOperands, async, asyncDeviceTypes,
-            asyncOnlyDeviceTypes);
+            privateOperands, async, asyncDeviceTypes, asyncOnlyDeviceTypes);
     } else if (const auto *firstprivateClause =
                    std::get_if<Fortran::parser::AccClause::Firstprivate>(
                        &clause.u)) {
       genPrivatizationRecipes<mlir::acc::FirstprivateRecipeOp>(
           firstprivateClause->v, converter, semanticsContext, stmtCtx,
-          firstprivateOperands, async,
-          asyncDeviceTypes, asyncOnlyDeviceTypes);
+          firstprivateOperands, async, asyncDeviceTypes, asyncOnlyDeviceTypes);
     } else if (const auto *reductionClause =
                    std::get_if<Fortran::parser::AccClause::Reduction>(
                        &clause.u)) {
@@ -2826,8 +2823,8 @@ static Op createComputeOp(
       // instead.
       if (!combinedConstructs) {
         genReductions(reductionClause->v, converter, semanticsContext, stmtCtx,
-                      reductionOperands, async,
-                      asyncDeviceTypes, asyncOnlyDeviceTypes);
+                      reductionOperands, async, asyncDeviceTypes,
+                      asyncOnlyDeviceTypes);
       } else {
         auto crtDataStart = dataClauseOperands.size();
         genDataOperandOperations<mlir::acc::CopyinOp>(
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 122cadbb11922..5a7c8a4f65e69 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -462,7 +462,7 @@ static ParseResult parseRecipeSym(mlir::OpAsmParser &parser,
 }
 
 static void printRecipeSym(mlir::OpAsmPrinter &p, mlir::Operation *op,
-  mlir::SymbolRefAttr recipeAttr) {
+                           mlir::SymbolRefAttr recipeAttr) {
   p << recipeAttr;
 }
 

>From a10166228354f73aef14a60624d750c5168b2f3a Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Tue, 22 Jul 2025 16:46:55 -0700
Subject: [PATCH 5/6] Restore the data operands verifier

---
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 5a7c8a4f65e69..7c601b706c064 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -1045,7 +1045,7 @@ LogicalResult acc::ReductionRecipeOp::verifyRegions() {
 template <typename Op>
 static LogicalResult checkDataOperands(Op op,
                                        const mlir::ValueRange &operands) {
-  for (mlir::Value operand : operands) {
+  for (mlir::Value operand : operands)
     if (!mlir::isa<acc::AttachOp, acc::CopyinOp, acc::CopyoutOp, acc::CreateOp,
                    acc::DeleteOp, acc::DetachOp, acc::DevicePtrOp,
                    acc::GetDevicePtrOp, acc::NoCreateOp, acc::PresentOp>(
@@ -1053,10 +1053,6 @@ static LogicalResult checkDataOperands(Op op,
       return op.emitError(
           "expect data entry/exit operation or acc.getdeviceptr "
           "as defining op");
-    mlir::SymbolRefAttr operandRecipe = getRecipe(operand.getDefiningOp());
-    if (operandRecipe)
-      return op.emitError("recipe not expected for data mapping operations");
-  }
   return success();
 }
 

>From 0b47fd46e78d9048a77487b5b9a8e6bd53b17037 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <rlupusoru at nvidia.com>
Date: Tue, 22 Jul 2025 16:49:39 -0700
Subject: [PATCH 6/6] Add missed invalid cases

---
 mlir/test/Dialect/OpenACC/invalid.mlir | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/mlir/test/Dialect/OpenACC/invalid.mlir b/mlir/test/Dialect/OpenACC/invalid.mlir
index bb3aef339639f..a19953e87bc91 100644
--- a/mlir/test/Dialect/OpenACC/invalid.mlir
+++ b/mlir/test/Dialect/OpenACC/invalid.mlir
@@ -868,3 +868,27 @@ func.func @fct1(%0 : !llvm.ptr) -> () {
   }
   return
 }
+
+// -----
+
+func.func @fct1(%0 : !llvm.ptr) -> () {
+  // expected-error @below {{op recipe expected for private}}
+  %priv = acc.private varPtr(%0 : !llvm.ptr) varType(i32) -> !llvm.ptr
+  return
+}
+
+// -----
+
+func.func @fct1(%0 : !llvm.ptr) -> () {
+  // expected-error @below {{op recipe expected for firstprivate}}
+  %priv = acc.firstprivate varPtr(%0 : !llvm.ptr) varType(i32) -> !llvm.ptr
+  return
+}
+
+// -----
+
+func.func @fct1(%0 : !llvm.ptr) -> () {
+  // expected-error @below {{op recipe expected for reduction}}
+  %priv = acc.reduction varPtr(%0 : !llvm.ptr) varType(i32) -> !llvm.ptr
+  return
+}


From flang-commits at lists.llvm.org  Wed Jul 23 00:03:39 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 23 Jul 2025 00:03:39 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Implement LOWER= argument for
 C_F_POINTER (Fortran 2023) (PR #149870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149870@github.com>
Message-ID: <6880894b.050a0220.35b21e.07e7@mx.google.com>


https://github.com/Meinersbur approved this pull request.

LGTM

Consider adding a line to https://github.com/llvm/llvm-project/blob/main/flang/docs/ReleaseNotes.md

https://github.com/llvm/llvm-project/pull/149870

From flang-commits at lists.llvm.org  Wed Jul 23 00:27:30 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 00:27:30 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
Message-ID: <llvm.org/llvm/llvm-project/pull/150178@github.com>


https://github.com/ergawy created https://github.com/llvm/llvm-project/pull/150178

Fixes #149089.

Before #145837, when processing a reduction symbol not yet supported by OpenMP lowering, the reduction processor would simply skip filling in the reduction symbols and variables. With #145837, this behvaior was slightly changed because the reduction symbols are populated before invoking the reduction processor (this is more convenient to shared the code with `do concurrent`).

This PR restores the previous behavior.

>From eb63f55d4092f033567029c7f3db6b9e18d817cd Mon Sep 17 00:00:00 2001
From: ergawy <kareem.ergawy at amd.com>
Date: Wed, 23 Jul 2025 02:22:59 -0500
Subject: [PATCH] [flang][OpenMP] Restore reduction processor behavior broken
 by #145837

Fixes #149089.

Before #145837, when processing a reduction symbol not yet supported by
OpenMP lowering, the reduction processor would simply skip filling in
the reduction symbols and variables. With #145837, this behvaior was
slightly changed because the reduction symbols are populated before
invoking the reduction processor (this is more convenient to shared the
code with `do concurrent`).

This PR restores the previous behavior.
---
 .../flang/Lower/Support/ReductionProcessor.h  |  2 +-
 flang/lib/Lower/Bridge.cpp                    |  3 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    | 32 +++++++++++--------
 .../lib/Lower/Support/ReductionProcessor.cpp  | 12 ++++---
 .../OpenMP/wsloop-reduction-non-intrinsic.f90 | 25 +++++++++++++++
 5 files changed, 53 insertions(+), 21 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90

diff --git a/flang/include/flang/Lower/Support/ReductionProcessor.h b/flang/include/flang/Lower/Support/ReductionProcessor.h
index 72d8a0096f511..0b49049d43ed2 100644
--- a/flang/include/flang/Lower/Support/ReductionProcessor.h
+++ b/flang/include/flang/Lower/Support/ReductionProcessor.h
@@ -124,7 +124,7 @@ class ReductionProcessor {
   /// Creates a reduction declaration and associates it with an OpenMP block
   /// directive.
   template <typename OpType, typename RedOperatorListTy>
-  static void processReductionArguments(
+  static bool processReductionArguments(
       mlir::Location currentLocation, lower::AbstractConverter &converter,
       const RedOperatorListTy &redOperatorList,
       llvm::SmallVectorImpl<mlir::Value> &reductionVars,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index b94833d852b2e..13420f365919d 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2125,9 +2125,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     llvm::SmallVector<mlir::Value> reduceVars;
     Fortran::lower::omp::ReductionProcessor rp;
-    rp.processReductionArguments<fir::DeclareReductionOp>(
+    bool result = rp.processReductionArguments<fir::DeclareReductionOp>(
         toLocation(), *this, info.reduceOperatorList, reduceVars,
         reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
+    assert(result && "Failed to process `do concurrent` reductions");
 
     doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
     doConcurrentLoopOp.setReduceSymsAttr(
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 8b3ad57c53810..594f95ecdda63 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1116,11 +1116,12 @@ bool ClauseProcessor::processInReduction(
         collectReductionSyms(clause, inReductionSyms);
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
-            inReductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
+                inReductionSyms))
+          inReductionSyms.clear();
 
         // Copy local lists into the output.
         llvm::copy(inReductionVars, std::back_inserter(result.inReductionVars));
@@ -1461,10 +1462,12 @@ bool ClauseProcessor::processReduction(
         }
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            reductionVars, reduceVarByRef, reductionDeclSymbols, reductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                reductionVars, reduceVarByRef, reductionDeclSymbols,
+                reductionSyms))
+          reductionSyms.clear();
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
@@ -1486,11 +1489,12 @@ bool ClauseProcessor::processTaskReduction(
         collectReductionSyms(clause, taskReductionSyms);
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
-            taskReductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
+                taskReductionSyms))
+          taskReductionSyms.clear();
         // Copy local lists into the output.
         llvm::copy(taskReductionVars,
                    std::back_inserter(result.taskReductionVars));
diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 80c32d066a38d..605a5b6b20b94 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -39,7 +39,7 @@ namespace lower {
 namespace omp {
 
 // explicit template declarations
-template void ReductionProcessor::processReductionArguments<
+template bool ReductionProcessor::processReductionArguments<
     mlir::omp::DeclareReductionOp, omp::clause::ReductionOperatorList>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const omp::clause::ReductionOperatorList &redOperatorList,
@@ -48,7 +48,7 @@ template void ReductionProcessor::processReductionArguments<
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
     const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void ReductionProcessor::processReductionArguments<
+template bool ReductionProcessor::processReductionArguments<
     fir::DeclareReductionOp, llvm::SmallVector<fir::ReduceOperationEnum>>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const llvm::SmallVector<fir::ReduceOperationEnum> &redOperatorList,
@@ -607,7 +607,7 @@ static bool doReductionByRef(mlir::Value reductionVar) {
 }
 
 template <typename OpType, typename RedOperatorListTy>
-void ReductionProcessor::processReductionArguments(
+bool ReductionProcessor::processReductionArguments(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const RedOperatorListTy &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
@@ -627,10 +627,10 @@ void ReductionProcessor::processReductionArguments(
               std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
         if (!ReductionProcessor::supportedIntrinsicProcReduction(
                 *reductionIntrinsic)) {
-          return;
+          return false;
         }
       } else {
-        return;
+        return false;
       }
     }
   }
@@ -765,6 +765,8 @@ void ReductionProcessor::processReductionArguments(
 
   if (isDoConcurrent)
     builder.restoreInsertionPoint(dcIP);
+
+  return true;
 }
 
 const semantics::SourceName
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
new file mode 100644
index 0000000000000..70b2ae1111a50
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
@@ -0,0 +1,25 @@
+! Tests reduction processor behavior when a reduction symbol is not supported.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+subroutine foo
+  implicit none
+  integer :: k, i
+
+  interface max
+    function max(m1,m2)
+      integer :: m1, m2
+    end function
+  end interface
+
+  !$omp do reduction (max: k)
+  do i=1,10
+  end do
+  !$omp end do
+end
+
+! Verify that unsupported reduction is ignored.
+! CHECK: omp.wsloop 
+! CHECK-SAME: private(@{{[^[:space:]]+}} %{{[^[:space:]]+}}
+! CHECK-SAME:         -> %{{[^[:space:]]+}} : !{{[^[:space:]]+}}) {
+! CHECK: }


From flang-commits at lists.llvm.org  Wed Jul 23 00:28:01 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 00:28:01 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <68808f01.170a0220.1a554b.fee1@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: Kareem Ergawy (ergawy)

<details>
<summary>Changes</summary>

Fixes #<!-- -->149089.

Before #<!-- -->145837, when processing a reduction symbol not yet supported by OpenMP lowering, the reduction processor would simply skip filling in the reduction symbols and variables. With #<!-- -->145837, this behvaior was slightly changed because the reduction symbols are populated before invoking the reduction processor (this is more convenient to shared the code with `do concurrent`).

This PR restores the previous behavior.

---
Full diff: https://github.com/llvm/llvm-project/pull/150178.diff


5 Files Affected:

- (modified) flang/include/flang/Lower/Support/ReductionProcessor.h (+1-1) 
- (modified) flang/lib/Lower/Bridge.cpp (+2-1) 
- (modified) flang/lib/Lower/OpenMP/ClauseProcessor.cpp (+18-14) 
- (modified) flang/lib/Lower/Support/ReductionProcessor.cpp (+7-5) 
- (added) flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90 (+25) 


``````````diff
diff --git a/flang/include/flang/Lower/Support/ReductionProcessor.h b/flang/include/flang/Lower/Support/ReductionProcessor.h
index 72d8a0096f511..0b49049d43ed2 100644
--- a/flang/include/flang/Lower/Support/ReductionProcessor.h
+++ b/flang/include/flang/Lower/Support/ReductionProcessor.h
@@ -124,7 +124,7 @@ class ReductionProcessor {
   /// Creates a reduction declaration and associates it with an OpenMP block
   /// directive.
   template <typename OpType, typename RedOperatorListTy>
-  static void processReductionArguments(
+  static bool processReductionArguments(
       mlir::Location currentLocation, lower::AbstractConverter &converter,
       const RedOperatorListTy &redOperatorList,
       llvm::SmallVectorImpl<mlir::Value> &reductionVars,
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index b94833d852b2e..13420f365919d 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2125,9 +2125,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     llvm::SmallVector<mlir::Value> reduceVars;
     Fortran::lower::omp::ReductionProcessor rp;
-    rp.processReductionArguments<fir::DeclareReductionOp>(
+    bool result = rp.processReductionArguments<fir::DeclareReductionOp>(
         toLocation(), *this, info.reduceOperatorList, reduceVars,
         reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
+    assert(result && "Failed to process `do concurrent` reductions");
 
     doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
     doConcurrentLoopOp.setReduceSymsAttr(
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 8b3ad57c53810..594f95ecdda63 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1116,11 +1116,12 @@ bool ClauseProcessor::processInReduction(
         collectReductionSyms(clause, inReductionSyms);
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
-            inReductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
+                inReductionSyms))
+          inReductionSyms.clear();
 
         // Copy local lists into the output.
         llvm::copy(inReductionVars, std::back_inserter(result.inReductionVars));
@@ -1461,10 +1462,12 @@ bool ClauseProcessor::processReduction(
         }
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            reductionVars, reduceVarByRef, reductionDeclSymbols, reductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                reductionVars, reduceVarByRef, reductionDeclSymbols,
+                reductionSyms))
+          reductionSyms.clear();
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
@@ -1486,11 +1489,12 @@ bool ClauseProcessor::processTaskReduction(
         collectReductionSyms(clause, taskReductionSyms);
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
-            taskReductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
+                taskReductionSyms))
+          taskReductionSyms.clear();
         // Copy local lists into the output.
         llvm::copy(taskReductionVars,
                    std::back_inserter(result.taskReductionVars));
diff --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 80c32d066a38d..605a5b6b20b94 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -39,7 +39,7 @@ namespace lower {
 namespace omp {
 
 // explicit template declarations
-template void ReductionProcessor::processReductionArguments<
+template bool ReductionProcessor::processReductionArguments<
     mlir::omp::DeclareReductionOp, omp::clause::ReductionOperatorList>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const omp::clause::ReductionOperatorList &redOperatorList,
@@ -48,7 +48,7 @@ template void ReductionProcessor::processReductionArguments<
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
     const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void ReductionProcessor::processReductionArguments<
+template bool ReductionProcessor::processReductionArguments<
     fir::DeclareReductionOp, llvm::SmallVector<fir::ReduceOperationEnum>>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const llvm::SmallVector<fir::ReduceOperationEnum> &redOperatorList,
@@ -607,7 +607,7 @@ static bool doReductionByRef(mlir::Value reductionVar) {
 }
 
 template <typename OpType, typename RedOperatorListTy>
-void ReductionProcessor::processReductionArguments(
+bool ReductionProcessor::processReductionArguments(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const RedOperatorListTy &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
@@ -627,10 +627,10 @@ void ReductionProcessor::processReductionArguments(
               std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
         if (!ReductionProcessor::supportedIntrinsicProcReduction(
                 *reductionIntrinsic)) {
-          return;
+          return false;
         }
       } else {
-        return;
+        return false;
       }
     }
   }
@@ -765,6 +765,8 @@ void ReductionProcessor::processReductionArguments(
 
   if (isDoConcurrent)
     builder.restoreInsertionPoint(dcIP);
+
+  return true;
 }
 
 const semantics::SourceName
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
new file mode 100644
index 0000000000000..70b2ae1111a50
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
@@ -0,0 +1,25 @@
+! Tests reduction processor behavior when a reduction symbol is not supported.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+subroutine foo
+  implicit none
+  integer :: k, i
+
+  interface max
+    function max(m1,m2)
+      integer :: m1, m2
+    end function
+  end interface
+
+  !$omp do reduction (max: k)
+  do i=1,10
+  end do
+  !$omp end do
+end
+
+! Verify that unsupported reduction is ignored.
+! CHECK: omp.wsloop 
+! CHECK-SAME: private(@{{[^[:space:]]+}} %{{[^[:space:]]+}}
+! CHECK-SAME:         -> %{{[^[:space:]]+}} : !{{[^[:space:]]+}}) {
+! CHECK: }

``````````

</details>


https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 01:13:25 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 01:13:25 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ASSIGN statement (PR #149941)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149941@github.com>
Message-ID: <688099a5.170a0220.1295a2.0604@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM. Thanks for the fix

https://github.com/llvm/llvm-project/pull/149941

From flang-commits at lists.llvm.org  Wed Jul 23 01:19:14 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 01:19:14 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ASSIGN statement (PR #149941)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149941@github.com>
Message-ID: <68809b02.050a0220.112965.07be@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/149941

From flang-commits at lists.llvm.org  Wed Jul 23 01:20:49 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 01:20:49 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ASSIGN statement (PR #149941)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149941@github.com>
Message-ID: <68809b61.170a0220.246fdf.294e@mx.google.com>


================
@@ -5506,10 +5506,34 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   void genFIR(const Fortran::parser::AssignStmt &stmt) {
     const Fortran::semantics::Symbol &symbol =
         *std::get<Fortran::parser::Name>(stmt.t).symbol;
+
     mlir::Location loc = toLocation();
+    mlir::Type symbolType = genType(symbol);
+    mlir::Value addr = getSymbolAddress(symbol);
+
+    // Handle the case where the assigned variable is declared as a pointer
+    if (auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(symbolType)) {
+      if (auto ptrType = mlir::dyn_cast<fir::PointerType>(eleTy)) {
+        symbolType = ptrType.getEleTy();
+      } else {
+        symbolType = eleTy;
+      }
+    } else if (auto ptrType = mlir::dyn_cast<fir::PointerType>(symbolType)) {
+      symbolType = ptrType.getEleTy();
+    }
+
     mlir::Value labelValue = builder->createIntegerConstant(
-        loc, genType(symbol), std::get<Fortran::parser::Label>(stmt.t));
-    builder->create<fir::StoreOp>(loc, labelValue, getSymbolAddress(symbol));
+        loc, symbolType, std::get<Fortran::parser::Label>(stmt.t));
+
+    // If the address points to a boxed pointer, we need to dereference it
+    if (auto refType = mlir::dyn_cast<fir::ReferenceType>(addr.getType())) {
+      if (auto boxType = mlir::dyn_cast<fir::BoxType>(refType.getEleTy())) {
+        mlir::Value boxValue = builder->create<fir::LoadOp>(loc, addr);
+        addr = builder->create<fir::BoxAddrOp>(loc, boxValue);
----------------
tblah wrote:

FYI: this syntax will be deprecated in LLVM 22. `fir::BoxAddrOp::create(bulider, loc, boxValue)` is the new style. But maybe don't do that here so that the patch is backportable to 21

https://github.com/llvm/llvm-project/pull/149941

From flang-commits at lists.llvm.org  Wed Jul 23 01:21:58 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 01:21:58 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix a crash when equivalence and
 namelist statements are used (PR #150081)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150081@github.com>
Message-ID: <68809ba6.170a0220.114fd6.0088@mx.google.com>


https://github.com/tblah approved this pull request.

LGTM, thanks. Please backport this to LLVM 21 if possible.

https://github.com/llvm/llvm-project/pull/150081

From flang-commits at lists.llvm.org  Wed Jul 23 01:22:15 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 23 Jul 2025 01:22:15 -0700 (PDT)
Subject: [flang-commits] [flang] 0586067 - [Flang] Build fix without
	precompiled headers
Message-ID: <68809bb7.170a0220.7752f.2a68@mx.google.com>


Author: Michael Kruse
Date: 2025-07-23T10:22:10+02:00
New Revision: 0586067cf07bef0f04fd1dc7135a9b773ebaa07a

URL: https://github.com/llvm/llvm-project/commit/0586067cf07bef0f04fd1dc7135a9b773ebaa07a
DIFF: https://github.com/llvm/llvm-project/commit/0586067cf07bef0f04fd1dc7135a9b773ebaa07a.diff

LOG: [Flang] Build fix without precompiled headers

The header semantics.h is added implitly in the precompiled headers, but
the build was failing when precompiled headers are disabled (e.g.
using CMAKE_DISABLE_PRECOMPILE_HEADERS=ON):

```
../_src/flang/lib/Semantics/canonicalize-omp.cpp: In constructor ‘Fortran::semantics::CanonicalizationOfOmp::CanonicalizationOfOmp(Fortran::semantics::SemanticsContext&)’:
../_src/flang/lib/Semantics/canonicalize-omp.cpp:31:38: error: invalid use of incomplete type ‘class Fortran::semantics::SemanticsContext’
   31 |       : context_{context}, messages_{context.messages()} {}
      |                                      ^~~~~~~
In file included from ../_src/flang/lib/Semantics/canonicalize-omp.cpp:9:
../_src/flang/lib/Semantics/canonicalize-omp.h:17:7: note: forward declaration of ‘class Fortran::semantics::SemanticsContext’
   17 | class SemanticsContext;
      |       ^~~~~~~~~~~~~~~~
compilation terminated due to -fmax-errors=1.
```

Added: 
    

Modified: 
    flang/lib/Semantics/canonicalize-omp.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 77e2fd6ca5097..9722eca19447d 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -9,6 +9,7 @@
 #include "canonicalize-omp.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/semantics.h"
 
 // After Loop Canonicalization, rewrite OpenMP parse tree to make OpenMP
 // Constructs more structured which provide explicit scopes for later


        

From flang-commits at lists.llvm.org  Wed Jul 23 01:24:41 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 23 Jul 2025 01:24:41 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <68809c49.170a0220.2b94bf.280f@mx.google.com>


================
@@ -27,7 +27,8 @@ class CanonicalizationOfOmp {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
-  CanonicalizationOfOmp(parser::Messages &messages) : messages_{messages} {}
+  CanonicalizationOfOmp(SemanticsContext &context)
+      : context_{context}, messages_{context.messages()} {}
----------------
Meinersbur wrote:

This adds a use of `SemanticContext` without including its header file. This means the build was failing when precompiled headers are disabled.

Fixed in 0586067cf07bef0f04fd1dc7135a9b773ebaa07a 

https://github.com/llvm/llvm-project/pull/149137

From flang-commits at lists.llvm.org  Wed Jul 23 01:26:30 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 01:26:30 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `tanpi` (PR #149527)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149527@github.com>
Message-ID: <68809cb6.170a0220.18ea6e.084a@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks!

https://github.com/llvm/llvm-project/pull/149527

From flang-commits at lists.llvm.org  Wed Jul 23 01:33:11 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 23 Jul 2025 01:33:11 -0700 (PDT)
Subject: [flang-commits] [flang] [llvm] [flang][OpenMP] Sema checks,
 lowering with new format of MAP modifiers (PR #149137)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149137@github.com>
Message-ID: <68809e47.170a0220.35ba72.01df@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/149137

From flang-commits at lists.llvm.org  Wed Jul 23 01:41:27 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 01:41:27 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880a037.170a0220.37897d.2a53@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 01:42:59 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 01:42:59 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880a093.170a0220.e2d29.0603@mx.google.com>


https://github.com/tblah commented:

Thanks for the quick patch. I wonder if for now this should be a NYI error rather than keeping the old behavior of silently ignoring it and producing the wrong result.

This also "fixes" https://github.com/llvm/llvm-project/issues/149700

Do you plan to implement these reductions or should I go ahead?

https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 01:46:18 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 01:46:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880a15a.170a0220.60494.070e@mx.google.com>


ergawy wrote:

> Thanks for the quick patch. I wonder if for now this should be a NYI error rather than keeping the old behavior of silently ignoring it and producing the wrong result.

I also think it should be handled somehow. But in this PR, I am restoring what was happening already for OpenMP. Maybe handling should happen in a separate PR?

> Do you plan to implement these reductions or should I go ahead?

I would suggest not in this PR.

https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 01:58:54 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 01:58:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880a44e.630a0220.290657.e233@mx.google.com>


https://github.com/tblah approved this pull request.

Okay I guess there is a danger the NYI could regress applications that use this feature but don't actually care about a correct result so maybe it should land separately so it can be more easily reverted.

https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 01:59:29 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 01:59:29 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880a471.170a0220.31f2d6.2c81@mx.google.com>


https://github.com/tblah edited https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 02:16:26 2025
From: flang-commits at lists.llvm.org (Michael Klemm via flang-commits)
Date: Wed, 23 Jul 2025 02:16:26 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Implement LOWER= argument for
 C_F_POINTER (Fortran 2023) (PR #149870)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149870@github.com>
Message-ID: <6880a86a.050a0220.11d4d4.0fe3@mx.google.com>


https://github.com/mjklemm updated https://github.com/llvm/llvm-project/pull/149870

>From aaa6fbb0429fd09b9fb250136bac832966bbcb08 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Mon, 21 Jul 2025 15:59:17 +0200
Subject: [PATCH 1/6] [Flang] Implement LOWER argument for C_F_POINTER

This PR adds support for the optional LOWER argument for C_F_POINTER
(Fortran 2023, 18.2.3.3)
---
 flang/docs/F202X.md                           |  1 -
 flang/lib/Evaluate/intrinsics.cpp             | 44 +++++++++++++++++--
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 26 +++++++++--
 3 files changed, 62 insertions(+), 9 deletions(-)

diff --git a/flang/docs/F202X.md b/flang/docs/F202X.md
index 67ea7fd97449a..7504561a64aa6 100644
--- a/flang/docs/F202X.md
+++ b/flang/docs/F202X.md
@@ -268,7 +268,6 @@ Addressing some issues and omissions in intrinsic modules:
  * LOGICAL8/16/32/64 and REAL16
  * IEEE module facilities upgraded to match latest IEEE FP standard
  * C_F_STRPOINTER, F_C_STRING for NUL-terminated strings
- * C_F_POINTER(LOWER=)
 
 #### Intrinsic Procedure Extensions
 
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9957010684d48..c48dea4b54b3a 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -3073,10 +3073,11 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
     ActualArguments &arguments, FoldingContext &context) const {
   characteristics::Procedure::Attrs attrs;
   attrs.set(characteristics::Procedure::Attr::Subroutine);
-  static const char *const keywords[]{"cptr", "fptr", "shape", nullptr};
+  static const char *const keywords[]{
+      "cptr", "fptr", "shape", "lower", nullptr};
   characteristics::DummyArguments dummies;
-  if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 1)) {
-    CHECK(arguments.size() == 3);
+  if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 2)) {
+    CHECK(arguments.size() == 4);
     if (const auto *expr{arguments[0].value().UnwrapExpr()}) {
       // General semantic checks will catch an actual argument that's not
       // scalar.
@@ -3169,11 +3170,30 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
           }
         }
       }
+      if (arguments[3] && fptrRank == 0) {
+        context.messages().Say(arguments[3]->sourceLocation(),
+            "LOWER= argument to C_F_POINTER() may not appear when FPTR= is scalar"_err_en_US);
+      } else if (arguments[3]) {
+        if (const auto *argExpr{arguments[3].value().UnwrapExpr()}) {
+          if (argExpr->Rank() > 1) {
+            context.messages().Say(arguments[3]->sourceLocation(),
+                "LOWER= argument to C_F_POINTER() must be a rank-one array."_err_en_US);
+          } else if (argExpr->Rank() == 1) {
+            if (auto constShape{GetConstantShape(context, *argExpr)}) {
+              if (constShape->At(ConstantSubscripts{1}).ToInt64() != fptrRank) {
+                context.messages().Say(arguments[3]->sourceLocation(),
+                    "LOWER= argument to C_F_POINTER() must have size equal to the rank of FPTR="_err_en_US);
+              }
+            }
+          }
+        }
+      }
     }
   }
   if (dummies.size() == 2) {
+    // Handle SHAPE
     DynamicType shapeType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
-    if (arguments[2]) {
+    if (arguments.size() >=3 && arguments[2]) {
       if (auto type{arguments[2]->GetType()}) {
         if (type->category() == TypeCategory::Integer) {
           shapeType = *type;
@@ -3185,6 +3205,22 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
     shape.intent = common::Intent::In;
     shape.attrs.set(characteristics::DummyDataObject::Attr::Optional);
     dummies.emplace_back("shape"s, std::move(shape));
+
+    // Handle LOWER
+    DynamicType lowerType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
+    if (arguments.size() >= 4 && arguments[3]) {
+      if (auto type{arguments[3]->GetType()}) {
+        if (type->category() == TypeCategory::Integer) {
+          lowerType = *type;
+        }
+      }
+    }
+    characteristics::DummyDataObject lower{
+        characteristics::TypeAndShape{lowerType, 1}};
+    lower.intent = common::Intent::In;
+    lower.attrs.set(characteristics::DummyDataObject::Attr::Optional);
+    dummies.emplace_back("lower"s, std::move(lower));
+
     return SpecificCall{
         SpecificIntrinsic{"__builtin_c_f_pointer"s,
             characteristics::Procedure{std::move(dummies), attrs}},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d77a656158a37..a99c4cb112c21 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -369,7 +369,8 @@ static constexpr IntrinsicHandler handlers[]{
      &I::genCFPointer,
      {{{"cptr", asValue},
        {"fptr", asInquired},
-       {"shape", asAddr, handleDynamicOptional}}},
+       {"shape", asAddr, handleDynamicOptional},
+       {"lower", asAddr, handleDynamicOptional}}},
      /*isElemental=*/false},
     {"c_f_procpointer",
      &I::genCFProcPointer,
@@ -3403,7 +3404,7 @@ IntrinsicLibrary::genCDevLoc(mlir::Type resultType,
 
 // C_F_POINTER
 void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
-  assert(args.size() == 3);
+  assert(args.size() == 4);
   // Handle CPTR argument
   // Get the value of the C address or the result of a reference to C_LOC.
   mlir::Value cPtr = fir::getBase(args[0]);
@@ -3418,9 +3419,12 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
     mlir::Value addr =
         builder.createConvert(loc, fPtr->getMemTy(), cPtrAddrVal);
     mlir::SmallVector<mlir::Value> extents;
+    mlir::SmallVector<mlir::Value> lbounds;
     if (box.hasRank()) {
       assert(isStaticallyPresent(args[2]) &&
              "FPTR argument must be an array if SHAPE argument exists");
+
+      // Handle and unpack SHAPE argument
       mlir::Value shape = fir::getBase(args[2]);
       int arrayRank = box.rank();
       mlir::Type shapeElementType =
@@ -3433,17 +3437,31 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
         mlir::Value load = builder.create<fir::LoadOp>(loc, var);
         extents.push_back(builder.createConvert(loc, idxType, load));
       }
+
+      // Handle and unpack LOWER argument if present
+      if (isStaticallyPresent(args[3])) {
+        mlir::Value lower = fir::getBase(args[3]);
+        mlir::Type lowerElementType =
+            fir::unwrapSequenceType(fir::unwrapPassByRefType(lower.getType()));
+        for (int i = 0; i < arrayRank; ++i) {
+          mlir::Value index = builder.createIntegerConstant(loc, idxType, i);
+          mlir::Value var = builder.create<fir::CoordinateOp>(
+              loc, builder.getRefType(lowerElementType), lower, index);
+          mlir::Value load = builder.create<fir::LoadOp>(loc, var);
+          lbounds.push_back(builder.createConvert(loc, idxType, load));
+        }
+      }
     }
     if (box.isCharacter()) {
       mlir::Value len = box.nonDeferredLenParams()[0];
       if (box.hasRank())
-        return fir::CharArrayBoxValue{addr, len, extents};
+        return fir::CharArrayBoxValue{addr, len, extents, lbounds};
       return fir::CharBoxValue{addr, len};
     }
     if (box.isDerivedWithLenParameters())
       TODO(loc, "get length parameters of derived type");
     if (box.hasRank())
-      return fir::ArrayBoxValue{addr, extents};
+      return fir::ArrayBoxValue{addr, extents, lbounds};
     return addr;
   };
 

>From 3e5f80c749997a49583c6fd40cd0d46fb3313b67 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Mon, 21 Jul 2025 19:01:17 +0200
Subject: [PATCH 2/6] Add LOWER= to semantics test

---
 flang/test/Semantics/c_f_pointer.f90 | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/flang/test/Semantics/c_f_pointer.f90 b/flang/test/Semantics/c_f_pointer.f90
index e2d00536cdacb..8a22175ffe19e 100644
--- a/flang/test/Semantics/c_f_pointer.f90
+++ b/flang/test/Semantics/c_f_pointer.f90
@@ -54,4 +54,12 @@ program test
   call c_f_pointer(scalarC, c2ptr)
   !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type or kind CHARACTER(KIND=4,LEN=1_8) [-Winteroperability]
   call c_f_pointer(scalarC, unicodePtr)
+
+  !ERROR: SHAPE= argument to C_F_POINTER() may not appear when FPTR= is scalar
+  !ERROR: LOWER= argument to C_F_POINTER() may not appear when FPTR= is scalar
+  call c_f_pointer(scalarC, scalarIntF, [1_8], [0_8])
+  !ERROR: LOWER= argument to C_F_POINTER() must be a rank-one array.
+  call c_f_pointer(scalarC, arrayIntF, shape=[1_8], lower=rankTwoArray)
+  !ERROR: SHAPE= argument to C_F_POINTER() must appear when FPTR= is an array
+  call c_f_pointer(scalarC, arrayIntF, lower=[0])
 end program

>From 117fe8e9e113e625d36f3fb6deca588e4ed23105 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Mon, 21 Jul 2025 20:30:49 +0200
Subject: [PATCH 3/6] Add LOWER= to lowering test

---
 flang/test/Lower/Intrinsics/c_f_pointer.f90 | 75 +++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/flang/test/Lower/Intrinsics/c_f_pointer.f90 b/flang/test/Lower/Intrinsics/c_f_pointer.f90
index 67817e39d5c2b..f41d5e471f540 100644
--- a/flang/test/Lower/Intrinsics/c_f_pointer.f90
+++ b/flang/test/Lower/Intrinsics/c_f_pointer.f90
@@ -140,3 +140,78 @@ subroutine dynamic_shape_size_2(cptr, fptr, shape, n)
 ! CHECK:         %[[VAL_16:.*]] = fir.shape %[[VAL_11]], %[[VAL_15]] : (index, index) -> !fir.shape<2>
   call c_f_pointer(cptr, fptr, shape)
 end subroutine
+
+! CHECK-LABEL: func.func @_QPdynamic_shape_lower(
+subroutine dynamic_shape_lower(cptr, fpr, shape, lower)
+  use iso_c_binding
+  type(c_ptr)  :: cptr
+  real, pointer :: fptr(:, :)
+  integer :: n
+  integer :: shape(:)
+  integer :: lower(:)
+! CHECK: %[[C_0:.*]] = arith.constant 0 : index
+! CHECK: %[[VAL_2:.*]] = fir.shape %[[C_0]], %[[C_0]] : (index, index) -> !fir.shape<2>
+! CHECK: %[[VAL_3:.*]] = fir.embox %[[VAL_1:.*]](%[[VAL_2]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK: fir.store %[[VAL_3]] to %[[VAL_0:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFdynamic_shape_lowerEn"}
+! CHECK: %[[VAL_5:.*]] = fir.coordinate_of %[[ARG_0:.*]], __address : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> !fir.ref<i64>
+! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<i64>
+! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (i64) -> !fir.ptr<!fir.array<?x?xf32>>
+! CHECK: %[[C_0:.*]]_0 = arith.constant 0 : index
+! CHECK: %[[VAL_8:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_0]]_0 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref<i32>
+! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+! CHECK: %[[C_1:.*]] = arith.constant 1 : index
+! CHECK: %[[VAL_11:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_1]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_11]] : !fir.ref<i32>
+! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i32) -> index
+! CHECK: %[[C_0:.*]]_1 = arith.constant 0 : index
+! CHECK: %[[VAL_14:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_0]]_1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
+! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> index
+! CHECK: %[[C_1:.*]]_2 = arith.constant 1 : index
+! CHECK: %[[VAL_17:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_1]]_2 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> index
+! CHECK: %[[VAL_20:.*]] = fir.shape_shift %[[VAL_16]], %[[VAL_10]], %[[VAL_19]], %[[VAL_13]] : (index, index, index, index) -> !fir.shapeshift<2>
+! CHECK: %[[VAL_21:.*]] = fir.embox %[[VAL_7]](%[[VAL_20]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+! CHECK: fir.store %[[VAL_21:.*]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  call c_f_pointer(cptr, fptr, shape, lower)
+end subroutine dynamic_shape_lower
+
+! CHECK-LABEL: func.func @_QPdynamic_shape_lower_2(
+subroutine dynamic_shape_lower_2(cptr, fpr, shape, lower, n)
+  use iso_c_binding
+  type(c_ptr)  :: cptr
+  real, pointer :: fptr(:, :)
+  integer :: n
+  integer :: shape(n)
+  integer :: lower(n)
+!CHECK: %[[C_0:.*]] = arith.constant 0 : index
+!CHECK: %[[VAL_2:.*]] = fir.shape %[[C_0]], %[[C_0]] : (index, index) -> !fir.shape<2>
+!CHECK: %[[VAL_3:.*]] = fir.embox %[[ARG1:.*]](%[[VAL_2]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
+!CHECK: fir.store %[[VAL_3]] to %[[VAL_0:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+!CHECK: %[[VAL_4:.*]] = fir.coordinate_of %[[ARG_0:.*]], __address : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> !fir.ref<i64>
+!CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<i64>
+!CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i64) -> !fir.ptr<!fir.array<?x?xf32>>
+!CHECK: %[[C_0:.*]]_0 = arith.constant 0 : index
+!CHECK: %[[VAL_7:.*]] = fir.coordinate_of %[[ARG_2:.*]], %[[C_0]]_0 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+!CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i32) -> index
+!CHECK: %[[C_1:.*]] = arith.constant 1 : index
+!CHECK: %[[VAL_10:.*]] = fir.coordinate_of %[[ARG_2]], %[[C_1]] : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+!CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+!CHECK: %[[C_0:.*]]_1 = arith.constant 0 : index
+!CHECK: %[[VAL_13:.*]] = fir.coordinate_of %[[ARG_3:.*]], %[[C_0]]_1 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<i32>
+!CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i32) -> index
+!CHECK: %[[C_1:.*]]_2 = arith.constant 1 : index
+!CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[ARG_3]], %[[C_1]]_2 : (!fir.ref<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+!CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
+!CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i32) -> index
+!CHECK: %[[VAL_19:.*]] = fir.shape_shift %[[VAL_15]], %[[VAL_9]], %[[VAL_18]], %[[VAL_12]] : (index, index, index, index) -> !fir.shapeshift<2>
+!CHECK: %[[VAL_20:.*]] = fir.embox %[[VAL_6]](%[[VAL_19]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>
+!CHECK: fir.store %[[VAL_20]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  call c_f_pointer(cptr, fptr, shape, lower)
+end subroutine dynamic_shape_lower_2

>From f9d3c1b1dc5ca7e0b5c525a0532059f73dd96941 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Mon, 21 Jul 2025 20:56:47 +0200
Subject: [PATCH 4/6] Fix code formatting

---
 flang/lib/Evaluate/intrinsics.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index c48dea4b54b3a..7b08889df72af 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -3193,7 +3193,7 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
   if (dummies.size() == 2) {
     // Handle SHAPE
     DynamicType shapeType{TypeCategory::Integer, defaults_.sizeIntegerKind()};
-    if (arguments.size() >=3 && arguments[2]) {
+    if (arguments.size() >= 3 && arguments[2]) {
       if (auto type{arguments[2]->GetType()}) {
         if (type->category() == TypeCategory::Integer) {
           shapeType = *type;

>From b7c520dfd2215821a831f7bb86f3e2e0e3743827 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Mon, 21 Jul 2025 21:04:17 +0200
Subject: [PATCH 5/6] Follow Copilot's recommendation

---
 flang/test/Lower/Intrinsics/c_f_pointer.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Lower/Intrinsics/c_f_pointer.f90 b/flang/test/Lower/Intrinsics/c_f_pointer.f90
index f41d5e471f540..c1f1d7972d4b1 100644
--- a/flang/test/Lower/Intrinsics/c_f_pointer.f90
+++ b/flang/test/Lower/Intrinsics/c_f_pointer.f90
@@ -211,7 +211,7 @@ subroutine dynamic_shape_lower_2(cptr, fpr, shape, lower, n)
 !CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
 !CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i32) -> index
 !CHECK: %[[VAL_19:.*]] = fir.shape_shift %[[VAL_15]], %[[VAL_9]], %[[VAL_18]], %[[VAL_12]] : (index, index, index, index) -> !fir.shapeshift<2>
-!CHECK: %[[VAL_20:.*]] = fir.embox %[[VAL_6]](%[[VAL_19]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>
+!CHECK: %[[VAL_20:.*]] = fir.embox %[[VAL_6]](%[[VAL_19]]) : (!fir.ptr<!fir.array<?x?xf32>>, !fir.shapeshift<2>)
 !CHECK: fir.store %[[VAL_20]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
   call c_f_pointer(cptr, fptr, shape, lower)
 end subroutine dynamic_shape_lower_2

>From 19746a303e299eaecc5f3a84a536bed3b0a560b6 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm at amd.com>
Date: Wed, 23 Jul 2025 11:15:58 +0200
Subject: [PATCH 6/6] Add entry in release notes

---
 flang/docs/ReleaseNotes.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index dfaa2eb6e03fe..26327ce7913f6 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -25,6 +25,7 @@ page](https://llvm.org/releases/).
 ## Major New Features
 
 * Initial support for VOLATILE variables and procedure interface arguments has been added.
+* Support for LOWER= argument for C_F_POINTER.
 * OpenMP support is stable and no longer considered experimental. All of OpenMP 3.1 is
   supported, along with much of OpenMP 4.0 and some parts of later standards.
 


From flang-commits at lists.llvm.org  Wed Jul 23 02:23:03 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 02:23:03 -0700 (PDT)
Subject: [flang-commits] [flang] 36c37b0 - [flang][OpenMP] Restore reduction
 processor behavior broken by #145837 (#150178)
Message-ID: <6880a9f7.170a0220.329945.2c7f@mx.google.com>


Author: Kareem Ergawy
Date: 2025-07-23T11:23:00+02:00
New Revision: 36c37b019b5daae79785e8558d693e6ec42b0ebd

URL: https://github.com/llvm/llvm-project/commit/36c37b019b5daae79785e8558d693e6ec42b0ebd
DIFF: https://github.com/llvm/llvm-project/commit/36c37b019b5daae79785e8558d693e6ec42b0ebd.diff

LOG: [flang][OpenMP] Restore reduction processor behavior broken by #145837 (#150178)

Fixes #149089 and #149700.

Before #145837, when processing a reduction symbol not yet supported by
OpenMP lowering, the reduction processor would simply skip filling in
the reduction symbols and variables. With #145837, this behvaior was
slightly changed because the reduction symbols are populated before
invoking the reduction processor (this is more convenient to shared the
code with `do concurrent`).

This PR restores the previous behavior.

Added: 
    flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90

Modified: 
    flang/include/flang/Lower/Support/ReductionProcessor.h
    flang/lib/Lower/Bridge.cpp
    flang/lib/Lower/OpenMP/ClauseProcessor.cpp
    flang/lib/Lower/Support/ReductionProcessor.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Lower/Support/ReductionProcessor.h b/flang/include/flang/Lower/Support/ReductionProcessor.h
index 72d8a0096f511..0b49049d43ed2 100644
--- a/flang/include/flang/Lower/Support/ReductionProcessor.h
+++ b/flang/include/flang/Lower/Support/ReductionProcessor.h
@@ -124,7 +124,7 @@ class ReductionProcessor {
   /// Creates a reduction declaration and associates it with an OpenMP block
   /// directive.
   template <typename OpType, typename RedOperatorListTy>
-  static void processReductionArguments(
+  static bool processReductionArguments(
       mlir::Location currentLocation, lower::AbstractConverter &converter,
       const RedOperatorListTy &redOperatorList,
       llvm::SmallVectorImpl<mlir::Value> &reductionVars,

diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index b94833d852b2e..13420f365919d 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2125,9 +2125,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     llvm::SmallVector<mlir::Value> reduceVars;
     Fortran::lower::omp::ReductionProcessor rp;
-    rp.processReductionArguments<fir::DeclareReductionOp>(
+    bool result = rp.processReductionArguments<fir::DeclareReductionOp>(
         toLocation(), *this, info.reduceOperatorList, reduceVars,
         reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
+    assert(result && "Failed to process `do concurrent` reductions");
 
     doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
     doConcurrentLoopOp.setReduceSymsAttr(

diff  --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 8b3ad57c53810..594f95ecdda63 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1116,11 +1116,12 @@ bool ClauseProcessor::processInReduction(
         collectReductionSyms(clause, inReductionSyms);
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
-            inReductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
+                inReductionSyms))
+          inReductionSyms.clear();
 
         // Copy local lists into the output.
         llvm::copy(inReductionVars, std::back_inserter(result.inReductionVars));
@@ -1461,10 +1462,12 @@ bool ClauseProcessor::processReduction(
         }
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            reductionVars, reduceVarByRef, reductionDeclSymbols, reductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                reductionVars, reduceVarByRef, reductionDeclSymbols,
+                reductionSyms))
+          reductionSyms.clear();
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
@@ -1486,11 +1489,12 @@ bool ClauseProcessor::processTaskReduction(
         collectReductionSyms(clause, taskReductionSyms);
 
         ReductionProcessor rp;
-        rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
-            currentLocation, converter,
-            std::get<typename omp::clause::ReductionOperatorList>(clause.t),
-            taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
-            taskReductionSyms);
+        if (!rp.processReductionArguments<mlir::omp::DeclareReductionOp>(
+                currentLocation, converter,
+                std::get<typename omp::clause::ReductionOperatorList>(clause.t),
+                taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
+                taskReductionSyms))
+          taskReductionSyms.clear();
         // Copy local lists into the output.
         llvm::copy(taskReductionVars,
                    std::back_inserter(result.taskReductionVars));

diff  --git a/flang/lib/Lower/Support/ReductionProcessor.cpp b/flang/lib/Lower/Support/ReductionProcessor.cpp
index 80c32d066a38d..605a5b6b20b94 100644
--- a/flang/lib/Lower/Support/ReductionProcessor.cpp
+++ b/flang/lib/Lower/Support/ReductionProcessor.cpp
@@ -39,7 +39,7 @@ namespace lower {
 namespace omp {
 
 // explicit template declarations
-template void ReductionProcessor::processReductionArguments<
+template bool ReductionProcessor::processReductionArguments<
     mlir::omp::DeclareReductionOp, omp::clause::ReductionOperatorList>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const omp::clause::ReductionOperatorList &redOperatorList,
@@ -48,7 +48,7 @@ template void ReductionProcessor::processReductionArguments<
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
     const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);
 
-template void ReductionProcessor::processReductionArguments<
+template bool ReductionProcessor::processReductionArguments<
     fir::DeclareReductionOp, llvm::SmallVector<fir::ReduceOperationEnum>>(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const llvm::SmallVector<fir::ReduceOperationEnum> &redOperatorList,
@@ -607,7 +607,7 @@ static bool doReductionByRef(mlir::Value reductionVar) {
 }
 
 template <typename OpType, typename RedOperatorListTy>
-void ReductionProcessor::processReductionArguments(
+bool ReductionProcessor::processReductionArguments(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const RedOperatorListTy &redOperatorList,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
@@ -627,10 +627,10 @@ void ReductionProcessor::processReductionArguments(
               std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {
         if (!ReductionProcessor::supportedIntrinsicProcReduction(
                 *reductionIntrinsic)) {
-          return;
+          return false;
         }
       } else {
-        return;
+        return false;
       }
     }
   }
@@ -765,6 +765,8 @@ void ReductionProcessor::processReductionArguments(
 
   if (isDoConcurrent)
     builder.restoreInsertionPoint(dcIP);
+
+  return true;
 }
 
 const semantics::SourceName

diff  --git a/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
new file mode 100644
index 0000000000000..70b2ae1111a50
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
@@ -0,0 +1,25 @@
+! Tests reduction processor behavior when a reduction symbol is not supported.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+subroutine foo
+  implicit none
+  integer :: k, i
+
+  interface max
+    function max(m1,m2)
+      integer :: m1, m2
+    end function
+  end interface
+
+  !$omp do reduction (max: k)
+  do i=1,10
+  end do
+  !$omp end do
+end
+
+! Verify that unsupported reduction is ignored.
+! CHECK: omp.wsloop 
+! CHECK-SAME: private(@{{[^[:space:]]+}} %{{[^[:space:]]+}}
+! CHECK-SAME:         -> %{{[^[:space:]]+}} : !{{[^[:space:]]+}}) {
+! CHECK: }


        

From flang-commits at lists.llvm.org  Wed Jul 23 02:23:07 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 02:23:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880a9fb.a70a0220.3519fd.0d3c@mx.google.com>


https://github.com/ergawy closed https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 02:23:09 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 02:23:09 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Skip processing reductions for
 unstructured `do concurrent` loops (PR #150188)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150188@github.com>
Message-ID: <6880a9fd.170a0220.36f34.0f34@mx.google.com>


https://github.com/ergawy edited https://github.com/llvm/llvm-project/pull/150188

From flang-commits at lists.llvm.org  Wed Jul 23 02:29:14 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 02:29:14 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880ab6a.170a0220.1f8ba9.084a@mx.google.com>


ergawy wrote:

Backport PR https://github.com/llvm/llvm-project/pull/150196.

https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 02:35:11 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 23 Jul 2025 02:35:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150121@github.com>
Message-ID: <6880accf.170a0220.7a643.08b9@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/150121

From flang-commits at lists.llvm.org  Wed Jul 23 02:35:11 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 23 Jul 2025 02:35:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150121@github.com>
Message-ID: <6880accf.a70a0220.1ac31.0faf@mx.google.com>


================

----------------
Meinersbur wrote:

Whu is this part of the parser if it is only used in FortranLower?

https://github.com/llvm/llvm-project/pull/150121

From flang-commits at lists.llvm.org  Wed Jul 23 02:35:12 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 23 Jul 2025 02:35:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150121@github.com>
Message-ID: <6880acd0.050a0220.370a4f.d323@mx.google.com>


https://github.com/Meinersbur commented:

Instead of using tag parameters, is it possible to instead 
```
template <typename T> OmpDirectiveName GetOmpDirectiveName(const T &x) {
  if constexpr (WrapperTrait<T>) {
    ... 
  } else 
    return MakeName();
```

This seems to better represent the logic of the tagged overloads and is less costly in terms of compile-time.


https://github.com/llvm/llvm-project/pull/150121

From flang-commits at lists.llvm.org  Wed Jul 23 02:38:49 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 02:38:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880ada9.170a0220.167ca.2f30@mx.google.com>


https://github.com/ergawy milestoned https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 02:43:20 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 02:43:20 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880aeb8.050a0220.1631d0.ff59@mx.google.com>


ergawy wrote:

/cherry-pick 36c37b019b5daae79785e8558d693e6ec42b0ebd

https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 02:48:18 2025
From: flang-commits at lists.llvm.org (Kareem Ergawy via flang-commits)
Date: Wed, 23 Jul 2025 02:48:18 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Skip processing reductions for
 unstructured `do concurrent` loops (PR #150188)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150188@github.com>
Message-ID: <6880afe2.170a0220.598f.06f3@mx.google.com>


ergawy wrote:

> This does seem like a reasonable approach.
> 
> What do you think about the following program:
> 
> ```
> program test
> integer i,r
> 
> do concurrent (i=1:10) reduce(+:r)
>   r = 1
> enddo
> 
> print *,r
> end
> ```
> 
> As I understand it, this should print the number of threads the loop actually used (because that will be the number of threadprivate values of `r` which were created). This would make something like what is done here user-visible. But I guess that is okay because the standard does not seem to require that do concurrent is actually run over multiple threads?

I probably missed what you are asking. This is a structured loop so reduction attributes will be emitted as usual. The changes in this PR only affect unstructured loops which are serialized so `r` will be `1` indeed.

https://github.com/llvm/llvm-project/pull/150188

From flang-commits at lists.llvm.org  Wed Jul 23 02:49:17 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 02:49:17 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] Restore reduction processor
 behavior broken by #145837 (PR #150178)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150178@github.com>
Message-ID: <6880b01d.a70a0220.162c4e.0f51@mx.google.com>


llvmbot wrote:

/pull-request llvm/llvm-project#150200

https://github.com/llvm/llvm-project/pull/150178

From flang-commits at lists.llvm.org  Wed Jul 23 02:54:07 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 02:54:07 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Skip processing reductions for
 unstructured `do concurrent` loops (PR #150188)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150188@github.com>
Message-ID: <6880b13f.170a0220.c84d.2f34@mx.google.com>


tblah wrote:

Sorry my example was a bit unclear. My point was that one can write programs that behave differently depending on the number of threads. I should have made my example an unstructured loop.

But I think this is okay if you agree with my understanding of the standard

https://github.com/llvm/llvm-project/pull/150188

From flang-commits at lists.llvm.org  Wed Jul 23 03:34:04 2025
From: flang-commits at lists.llvm.org (Sjoerd Meijer via flang-commits)
Date: Wed, 23 Jul 2025 03:34:04 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] add -floop-fuse to clang and
	flang (PR #142686)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/142686@github.com>
Message-ID: <6880ba9c.170a0220.4f8ee.33f2@mx.google.com>


https://github.com/sjoerdmeijer approved this pull request.

I can confirm that this is useful for testing purposes.
If folks are happy with the 'experimental' prefix, let's do that, LGTM. 

https://github.com/llvm/llvm-project/pull/142686

From flang-commits at lists.llvm.org  Wed Jul 23 05:38:41 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 05:38:41 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Add conflicting checks for EXTERNAL
	attribute (PR #149973)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149973@github.com>
Message-ID: <6880d7d1.170a0220.136d96.3cd6@mx.google.com>


https://github.com/foxtran updated https://github.com/llvm/llvm-project/pull/149973

>From 2dd5e5c20774aa824a828c163dc61bf65b963226 Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Tue, 22 Jul 2025 10:11:03 +0200
Subject: [PATCH 1/6] Add checks of EXTERNAL attribute

---
 flang/lib/Semantics/check-declarations.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index a2f2906af10b8..a99a65b0dc0b4 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -66,6 +66,7 @@ class CheckHelper {
   void CheckVolatile(const Symbol &, const DerivedTypeSpec *);
   void CheckContiguous(const Symbol &);
   void CheckPointer(const Symbol &);
+  void CheckExternal(const Symbol &);
   void CheckPassArg(
       const Symbol &proc, const Symbol *interface, const WithPassArg &);
   void CheckProcBinding(const Symbol &, const ProcBindingDetails &);
@@ -983,6 +984,7 @@ void CheckHelper::CheckObjectEntity(
     }
   }
   if (symbol.attrs().test(Attr::EXTERNAL)) {
+    CheckExternal(symbol);
     SayWithDeclaration(symbol,
         "'%s' is a data object and may not be EXTERNAL"_err_en_US,
         symbol.name());
@@ -2460,6 +2462,11 @@ void CheckHelper::CheckPointer(const Symbol &symbol) { // C852
   }
 }
 
+void CheckHelper::CheckExternal(const Symbol &symbol) {
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::INTRINSIC); // F'2023 C842
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::PARAMETER);
+}
+
 // C760 constraints on the passed-object dummy argument
 // C757 constraints on procedure pointer components
 void CheckHelper::CheckPassArg(

>From 50bf419ed31a9a467bc1353a8a531e5dca03a889 Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Tue, 22 Jul 2025 10:14:55 +0200
Subject: [PATCH 2/6] Add tests for EXTERNAL attribute

---
 flang/test/Semantics/declarations09.f90 | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 flang/test/Semantics/declarations09.f90

diff --git a/flang/test/Semantics/declarations09.f90 b/flang/test/Semantics/declarations09.f90
new file mode 100644
index 0000000000000..4bd08a1749721
--- /dev/null
+++ b/flang/test/Semantics/declarations09.f90
@@ -0,0 +1,11 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+
+subroutine bug149771
+
+  !ERROR: 'x1' may not have both the EXTERNAL and INTRINSIC attributes
+  integer, external, intrinsic :: x1
+
+  !ERROR: 'x2' may not have both the EXTERNAL and PARAMETER attributes
+  integer, external, parameter :: x2
+
+end subroutine bug149771

>From abf1dbf8004084b8974dc8ef35d3892e2a8611da Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Tue, 22 Jul 2025 13:56:47 +0200
Subject: [PATCH 3/6] Rename CheckExternal -> CheckExternalAttrConflicts

---
 flang/lib/Semantics/check-declarations.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index a99a65b0dc0b4..113543825558e 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -66,7 +66,7 @@ class CheckHelper {
   void CheckVolatile(const Symbol &, const DerivedTypeSpec *);
   void CheckContiguous(const Symbol &);
   void CheckPointer(const Symbol &);
-  void CheckExternal(const Symbol &);
+  void CheckExternalAttrConflicts(const Symbol &);
   void CheckPassArg(
       const Symbol &proc, const Symbol *interface, const WithPassArg &);
   void CheckProcBinding(const Symbol &, const ProcBindingDetails &);
@@ -984,7 +984,7 @@ void CheckHelper::CheckObjectEntity(
     }
   }
   if (symbol.attrs().test(Attr::EXTERNAL)) {
-    CheckExternal(symbol);
+    CheckExternalAttrConflicts(symbol);
     SayWithDeclaration(symbol,
         "'%s' is a data object and may not be EXTERNAL"_err_en_US,
         symbol.name());
@@ -2462,7 +2462,7 @@ void CheckHelper::CheckPointer(const Symbol &symbol) { // C852
   }
 }
 
-void CheckHelper::CheckExternal(const Symbol &symbol) {
+void CheckHelper::CheckExternalAttrConflicts(const Symbol &symbol) {
   CheckConflicting(symbol, Attr::EXTERNAL, Attr::INTRINSIC); // F'2023 C842
   CheckConflicting(symbol, Attr::EXTERNAL, Attr::PARAMETER);
 }

>From b22cf10bba7c5c425bda7b093256d284d0391c0a Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Wed, 23 Jul 2025 14:33:58 +0200
Subject: [PATCH 4/6] Conflicting attributes: PARAMETER & EXTERNAL, PARAMETER &
 INTRINSIC

---
 flang/lib/Semantics/check-declarations.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index 113543825558e..c726be82eb313 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -697,6 +697,8 @@ void CheckHelper::CheckObjectEntity(
   CheckConflicting(symbol, Attr::SAVE, Attr::PARAMETER);
   CheckConflicting(symbol, Attr::TARGET, Attr::PARAMETER);
   CheckConflicting(symbol, Attr::VOLATILE, Attr::PARAMETER);
+  CheckConflicting(symbol, Attr::EXTERNAL, Attr::PARAMETER);
+  CheckConflicting(symbol, Attr::INTRINSIC, Attr::PARAMETER);
   Check(details.shape());
   Check(details.coshape());
   if (details.shape().Rank() > common::maxRank) {
@@ -2464,7 +2466,6 @@ void CheckHelper::CheckPointer(const Symbol &symbol) { // C852
 
 void CheckHelper::CheckExternalAttrConflicts(const Symbol &symbol) {
   CheckConflicting(symbol, Attr::EXTERNAL, Attr::INTRINSIC); // F'2023 C842
-  CheckConflicting(symbol, Attr::EXTERNAL, Attr::PARAMETER);
 }
 
 // C760 constraints on the passed-object dummy argument

>From 92885b2dbef7eec4ee0e342db296bd06e0c71295 Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Wed, 23 Jul 2025 14:37:07 +0200
Subject: [PATCH 5/6] Add check for INTRINSIC & PARAMETER

---
 flang/test/Semantics/declarations02.f90 | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flang/test/Semantics/declarations02.f90 b/flang/test/Semantics/declarations02.f90
index 32c3517d13cd1..90f29f90d9d0e 100644
--- a/flang/test/Semantics/declarations02.f90
+++ b/flang/test/Semantics/declarations02.f90
@@ -24,6 +24,9 @@ module m
   target x4
   volatile x4
 
+  !ERROR: 'sin' may not have both the INTRINSIC and PARAMETER attributes
+  real, parameter, intrinsic :: sin
+
   type :: my_type1
     integer :: x4
   end type

>From fbca1e64d2756b2040294bbbae5170a3e274b1c3 Mon Sep 17 00:00:00 2001
From: "Igor S. Gerasimov" <foxtranigor at gmail.com>
Date: Wed, 23 Jul 2025 14:38:24 +0200
Subject: [PATCH 6/6] Use proper intrinsic name

---
 flang/test/Semantics/declarations09.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flang/test/Semantics/declarations09.f90 b/flang/test/Semantics/declarations09.f90
index 4bd08a1749721..8abe320f5d2f3 100644
--- a/flang/test/Semantics/declarations09.f90
+++ b/flang/test/Semantics/declarations09.f90
@@ -2,8 +2,8 @@
 
 subroutine bug149771
 
-  !ERROR: 'x1' may not have both the EXTERNAL and INTRINSIC attributes
-  integer, external, intrinsic :: x1
+  !ERROR: 'exp' may not have both the EXTERNAL and INTRINSIC attributes
+  real, external, intrinsic :: exp
 
   !ERROR: 'x2' may not have both the EXTERNAL and PARAMETER attributes
   integer, external, parameter :: x2


From flang-commits at lists.llvm.org  Wed Jul 23 05:46:11 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Wed, 23 Jul 2025 05:46:11 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150121@github.com>
Message-ID: <6880d993.170a0220.2a1902.1abf@mx.google.com>


================

----------------
kparzysz wrote:

Because it only needs definitions from parser.  Parser is almost at the bottom of library dependency list, so everything can use it, e.g. semantics.

https://github.com/llvm/llvm-project/pull/150121

From flang-commits at lists.llvm.org  Wed Jul 23 06:06:47 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Wed, 23 Jul 2025 06:06:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150121@github.com>
Message-ID: <6880de67.050a0220.3c2470.cdf4@mx.google.com>


https://github.com/kparzysz updated https://github.com/llvm/llvm-project/pull/150121

>From 14072b0560cde74fda94a1f434b6cfd43b77939b Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Mon, 21 Jul 2025 15:37:11 -0500
Subject: [PATCH 1/2] [flang][OpenMP] General utility to get directive id from
 AST node

Fortran::parser::omp::GetOmpDirectiveName(t) will get the OmpDirectiveName
object that corresponds to construct t. That object (an AST node) contains
the enum id and the source information of the directive.

Replace uses of extractOmpDirective and getOpenMPDirectiveEnum with the
new function.
---
 flang/include/flang/Parser/openmp-utils.h     | 175 ++++++++++++++++++
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp |   4 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  35 +---
 flang/lib/Lower/OpenMP/Utils.cpp              |  84 +--------
 flang/lib/Lower/OpenMP/Utils.h                |   2 -
 5 files changed, 187 insertions(+), 113 deletions(-)
 create mode 100644 flang/include/flang/Parser/openmp-utils.h

diff --git a/flang/include/flang/Parser/openmp-utils.h b/flang/include/flang/Parser/openmp-utils.h
new file mode 100644
index 0000000000000..363db852416f6
--- /dev/null
+++ b/flang/include/flang/Parser/openmp-utils.h
@@ -0,0 +1,175 @@
+//===-- flang/Parser/openmp-utils.h ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common OpenMP utilities.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_PARSER_OPENMP_UTILS_H
+#define FORTRAN_PARSER_OPENMP_UTILS_H
+
+#include "flang/Common/indirection.h"
+#include "flang/Parser/parse-tree.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cassert>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+namespace Fortran::parser::omp {
+
+namespace detail {
+using D = llvm::omp::Directive;
+
+template <typename Construct> //
+struct ConstructId {
+  static constexpr llvm::omp::Directive id{D::OMPD_unknown};
+};
+
+#define MAKE_CONSTR_ID(Construct, Id) \
+  template <> struct ConstructId<Construct> { \
+    static constexpr llvm::omp::Directive id{Id}; \
+  }
+
+MAKE_CONSTR_ID(OmpAssumeDirective, D::OMPD_assume);
+MAKE_CONSTR_ID(OmpCriticalDirective, D::OMPD_critical);
+MAKE_CONSTR_ID(OmpDeclareVariantDirective, D::OMPD_declare_variant);
+MAKE_CONSTR_ID(OmpErrorDirective, D::OMPD_error);
+MAKE_CONSTR_ID(OmpMetadirectiveDirective, D::OMPD_metadirective);
+MAKE_CONSTR_ID(OpenMPDeclarativeAllocate, D::OMPD_allocate);
+MAKE_CONSTR_ID(OpenMPDeclarativeAssumes, D::OMPD_assumes);
+MAKE_CONSTR_ID(OpenMPDeclareMapperConstruct, D::OMPD_declare_mapper);
+MAKE_CONSTR_ID(OpenMPDeclareReductionConstruct, D::OMPD_declare_reduction);
+MAKE_CONSTR_ID(OpenMPDeclareSimdConstruct, D::OMPD_declare_simd);
+MAKE_CONSTR_ID(OpenMPDeclareTargetConstruct, D::OMPD_declare_target);
+MAKE_CONSTR_ID(OpenMPExecutableAllocate, D::OMPD_allocate);
+MAKE_CONSTR_ID(OpenMPRequiresConstruct, D::OMPD_requires);
+MAKE_CONSTR_ID(OpenMPThreadprivate, D::OMPD_threadprivate);
+
+#undef MAKE_CONSTR_ID
+
+struct DirectiveNameScope {
+  // Helper types to make overloaded function signatures different.
+  struct TagA {};
+  struct TagB {};
+  struct TagC {};
+  struct TagD {};
+
+  static OmpDirectiveName MakeName(CharBlock source = {},
+      llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown) {
+    OmpDirectiveName name;
+    name.source = source;
+    name.v = id;
+    return name;
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpNothingDirective &x) {
+    return MakeName(x.source, llvm::omp::Directive::OMPD_nothing);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpBeginBlockDirective &x) {
+    auto &dir{std::get<OmpBlockDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpBeginLoopDirective &x) {
+    auto &dir{std::get<OmpLoopDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(
+      const OmpBeginSectionsDirective &x) {
+    auto &dir{std::get<OmpSectionsDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  template <typename T, typename = std::enable_if_t<WrapperTrait<T>>>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagA = {}) {
+    if constexpr (std::is_same_v<T, OpenMPCancelConstruct> ||
+        std::is_same_v<T, OpenMPCancellationPointConstruct> ||
+        std::is_same_v<T, OpenMPDepobjConstruct> ||
+        std::is_same_v<T, OpenMPFlushConstruct> ||
+        std::is_same_v<T, OpenMPInteropConstruct> ||
+        std::is_same_v<T, OpenMPSimpleStandaloneConstruct>) {
+      return x.v.DirName();
+    } else {
+      return GetOmpDirectiveName(x.v);
+    }
+  }
+
+  template <typename T, typename = std::enable_if_t<TupleTrait<T>>>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagB = {}) {
+    if constexpr (std::is_same_v<T, OpenMPAllocatorsConstruct> ||
+        std::is_same_v<T, OpenMPAtomicConstruct> ||
+        std::is_same_v<T, OpenMPDispatchConstruct>) {
+      return std::get<OmpDirectiveSpecification>(x.t).DirName();
+    } else if constexpr (std::is_same_v<T, OmpAssumeDirective> ||
+        std::is_same_v<T, OmpCriticalDirective> ||
+        std::is_same_v<T, OmpDeclareVariantDirective> ||
+        std::is_same_v<T, OmpErrorDirective> ||
+        std::is_same_v<T, OmpMetadirectiveDirective> ||
+        std::is_same_v<T, OpenMPDeclarativeAllocate> ||
+        std::is_same_v<T, OpenMPDeclarativeAssumes> ||
+        std::is_same_v<T, OpenMPDeclareMapperConstruct> ||
+        std::is_same_v<T, OpenMPDeclareReductionConstruct> ||
+        std::is_same_v<T, OpenMPDeclareSimdConstruct> ||
+        std::is_same_v<T, OpenMPDeclareTargetConstruct> ||
+        std::is_same_v<T, OpenMPExecutableAllocate> ||
+        std::is_same_v<T, OpenMPRequiresConstruct> ||
+        std::is_same_v<T, OpenMPThreadprivate>) {
+      return MakeName(std::get<Verbatim>(x.t).source, ConstructId<T>::id);
+    } else {
+      return GetFromTuple(
+          x.t, std::make_index_sequence<std::tuple_size_v<decltype(x.t)>>{});
+    }
+  }
+
+  template <typename T, typename = std::enable_if_t<UnionTrait<T>>>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagC = {}) {
+    return common::visit([](auto &&s) { return GetOmpDirectiveName(s); }, x.u);
+  }
+
+  template <typename... Ts, size_t... Is>
+  static OmpDirectiveName GetFromTuple(
+      const std::tuple<Ts...> &t, std::index_sequence<Is...>) {
+    OmpDirectiveName name = MakeName();
+    auto accumulate = [&](const OmpDirectiveName &n) {
+      if (name.v == llvm::omp::Directive::OMPD_unknown) {
+        name = n;
+      } else {
+        assert(
+            n.v == llvm::omp::Directive::OMPD_unknown && "Conflicting names");
+      }
+    };
+    (accumulate(GetOmpDirectiveName(std::get<Is>(t))), ...);
+    return name;
+  }
+
+  template <typename T>
+  static OmpDirectiveName GetOmpDirectiveName(const common::Indirection<T> &x) {
+    return GetOmpDirectiveName(x.value());
+  }
+
+  template <typename T,
+      typename = std::enable_if_t<!WrapperTrait<T> && !TupleTrait<T> &&
+          !UnionTrait<T>>>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagD = {}) {
+    return MakeName();
+  }
+};
+} // namespace detail
+
+template <typename T> OmpDirectiveName GetOmpDirectiveName(const T &x) {
+  return detail::DirectiveNameScope::GetOmpDirectiveName(x);
+}
+
+} // namespace Fortran::parser::omp
+
+#endif // FORTRAN_PARSER_OPENMP_UTILS_H
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 11e488371b886..2ac4d9548b65b 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -24,6 +24,7 @@
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Parser/openmp-utils.h"
 #include "flang/Semantics/attr.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/ADT/Sequence.h"
@@ -465,7 +466,8 @@ bool DataSharingProcessor::isOpenMPPrivatizingConstruct(
   // allow a privatizing clause) are: dispatch, distribute, do, for, loop,
   // parallel, scope, sections, simd, single, target, target_data, task,
   // taskgroup, taskloop, and teams.
-  return llvm::is_contained(privatizing, extractOmpDirective(omp));
+  return llvm::is_contained(privatizing,
+                            parser::omp::GetOmpDirectiveName(omp).v);
 }
 
 bool DataSharingProcessor::isOpenMPPrivatizingEvaluation(
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index fc5fef9b2c577..4c2d7badef382 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -31,6 +31,7 @@
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Parser/characters.h"
+#include "flang/Parser/openmp-utils.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/tools.h"
@@ -63,28 +64,6 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
                                    lower::pft::Evaluation &eval,
                                    mlir::Location loc);
 
-static llvm::omp::Directive
-getOpenMPDirectiveEnum(const parser::OmpLoopDirective &beginStatment) {
-  return beginStatment.v;
-}
-
-static llvm::omp::Directive getOpenMPDirectiveEnum(
-    const parser::OmpBeginLoopDirective &beginLoopDirective) {
-  return getOpenMPDirectiveEnum(
-      std::get<parser::OmpLoopDirective>(beginLoopDirective.t));
-}
-
-static llvm::omp::Directive
-getOpenMPDirectiveEnum(const parser::OpenMPLoopConstruct &ompLoopConstruct) {
-  return getOpenMPDirectiveEnum(
-      std::get<parser::OmpBeginLoopDirective>(ompLoopConstruct.t));
-}
-
-static llvm::omp::Directive getOpenMPDirectiveEnum(
-    const common::Indirection<parser::OpenMPLoopConstruct> &ompLoopConstruct) {
-  return getOpenMPDirectiveEnum(ompLoopConstruct.value());
-}
-
 namespace {
 /// Structure holding information that is needed to pass host-evaluated
 /// information to later lowering stages.
@@ -468,7 +447,7 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
     llvm::omp::Directive dir;
     auto &nested = parent.getFirstNestedEvaluation();
     if (const auto *ompEval = nested.getIf<parser::OpenMPConstruct>())
-      dir = extractOmpDirective(*ompEval);
+      dir = parser::omp::GetOmpDirectiveName(*ompEval).v;
     else
       return std::nullopt;
 
@@ -508,7 +487,7 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
     HostEvalInfo *hostInfo = getHostEvalInfoStackTop(converter);
     assert(hostInfo && "expected HOST_EVAL info structure");
 
-    switch (extractOmpDirective(*ompEval)) {
+    switch (parser::omp::GetOmpDirectiveName(*ompEval).v) {
     case OMPD_teams_distribute_parallel_do:
     case OMPD_teams_distribute_parallel_do_simd:
       cp.processThreadLimit(stmtCtx, hostInfo->ops);
@@ -569,7 +548,8 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
 
   const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
   assert(ompEval &&
-         llvm::omp::allTargetSet.test(extractOmpDirective(*ompEval)) &&
+         llvm::omp::allTargetSet.test(
+             parser::omp::GetOmpDirectiveName(*ompEval).v) &&
          "expected TARGET construct evaluation");
   (void)ompEval;
 
@@ -3872,7 +3852,7 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
             std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
                 &*optLoopCons)}) {
       llvm::omp::Directive nestedDirective =
-          getOpenMPDirectiveEnum(*ompNestedLoopCons);
+          parser::omp::GetOmpDirectiveName(*ompNestedLoopCons).v;
       switch (nestedDirective) {
       case llvm::omp::Directive::OMPD_tile:
         // Emit the omp.loop_nest with annotation for tiling
@@ -3889,7 +3869,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
     }
   }
 
-  llvm::omp::Directive directive = getOpenMPDirectiveEnum(beginLoopDirective);
+  llvm::omp::Directive directive =
+      parser::omp::GetOmpDirectiveName(beginLoopDirective).v;
   const parser::CharBlock &source =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
   ConstructQueue queue{
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index b1716d6afb200..13fda978c5369 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -20,6 +20,7 @@
 #include <flang/Lower/PFTBuilder.h>
 #include <flang/Optimizer/Builder/FIRBuilder.h>
 #include <flang/Optimizer/Builder/Todo.h>
+#include <flang/Parser/openmp-utils.h>
 #include <flang/Parser/parse-tree.h>
 #include <flang/Parser/tools.h>
 #include <flang/Semantics/tools.h>
@@ -663,89 +664,6 @@ bool collectLoopRelatedInfo(
   return found;
 }
 
-/// Get the directive enumeration value corresponding to the given OpenMP
-/// construct PFT node.
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
-  return common::visit(
-      common::visitors{
-          [](const parser::OpenMPAllocatorsConstruct &c) {
-            return llvm::omp::OMPD_allocators;
-          },
-          [](const parser::OpenMPAssumeConstruct &c) {
-            return llvm::omp::OMPD_assume;
-          },
-          [](const parser::OpenMPAtomicConstruct &c) {
-            return llvm::omp::OMPD_atomic;
-          },
-          [](const parser::OpenMPBlockConstruct &c) {
-            return std::get<parser::OmpBlockDirective>(
-                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPCriticalConstruct &c) {
-            return llvm::omp::OMPD_critical;
-          },
-          [](const parser::OpenMPDeclarativeAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPDispatchConstruct &c) {
-            return llvm::omp::OMPD_dispatch;
-          },
-          [](const parser::OpenMPExecutableAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPLoopConstruct &c) {
-            return std::get<parser::OmpLoopDirective>(
-                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPSectionConstruct &c) {
-            return llvm::omp::OMPD_section;
-          },
-          [](const parser::OpenMPSectionsConstruct &c) {
-            return std::get<parser::OmpSectionsDirective>(
-                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPStandaloneConstruct &c) {
-            return common::visit(
-                common::visitors{
-                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
-                      return c.v.DirId();
-                    },
-                    [](const parser::OpenMPFlushConstruct &c) {
-                      return llvm::omp::OMPD_flush;
-                    },
-                    [](const parser::OpenMPCancelConstruct &c) {
-                      return llvm::omp::OMPD_cancel;
-                    },
-                    [](const parser::OpenMPCancellationPointConstruct &c) {
-                      return llvm::omp::OMPD_cancellation_point;
-                    },
-                    [](const parser::OmpMetadirectiveDirective &c) {
-                      return llvm::omp::OMPD_metadirective;
-                    },
-                    [](const parser::OpenMPDepobjConstruct &c) {
-                      return llvm::omp::OMPD_depobj;
-                    },
-                    [](const parser::OpenMPInteropConstruct &c) {
-                      return llvm::omp::OMPD_interop;
-                    }},
-                c.u);
-          },
-          [](const parser::OpenMPUtilityConstruct &c) {
-            return common::visit(
-                common::visitors{[](const parser::OmpErrorDirective &c) {
-                                   return llvm::omp::OMPD_error;
-                                 },
-                                 [](const parser::OmpNothingDirective &c) {
-                                   return llvm::omp::OMPD_nothing;
-                                 }},
-                c.u);
-          }},
-      ompConstruct.u);
-}
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 8e3ad5c3452e2..11641ba5e8606 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -167,8 +167,6 @@ bool collectLoopRelatedInfo(
     mlir::omp::LoopRelatedClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &iv);
 
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct);
 } // namespace omp
 } // namespace lower
 } // namespace Fortran

>From b514947a2bc500bed56eb1785c829c8ea92cdb20 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek at amd.com>
Date: Wed, 23 Jul 2025 08:06:23 -0500
Subject: [PATCH 2/2] Put everything in a single function

---
 flang/include/flang/Parser/openmp-utils.h | 96 ++++++++++-------------
 1 file changed, 41 insertions(+), 55 deletions(-)

diff --git a/flang/include/flang/Parser/openmp-utils.h b/flang/include/flang/Parser/openmp-utils.h
index 363db852416f6..579ea7d74957f 100644
--- a/flang/include/flang/Parser/openmp-utils.h
+++ b/flang/include/flang/Parser/openmp-utils.h
@@ -56,12 +56,6 @@ MAKE_CONSTR_ID(OpenMPThreadprivate, D::OMPD_threadprivate);
 #undef MAKE_CONSTR_ID
 
 struct DirectiveNameScope {
-  // Helper types to make overloaded function signatures different.
-  struct TagA {};
-  struct TagB {};
-  struct TagC {};
-  struct TagD {};
-
   static OmpDirectiveName MakeName(CharBlock source = {},
       llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown) {
     OmpDirectiveName name;
@@ -90,52 +84,51 @@ struct DirectiveNameScope {
     return MakeName(dir.source, dir.v);
   }
 
-  template <typename T, typename = std::enable_if_t<WrapperTrait<T>>>
-  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagA = {}) {
-    if constexpr (std::is_same_v<T, OpenMPCancelConstruct> ||
-        std::is_same_v<T, OpenMPCancellationPointConstruct> ||
-        std::is_same_v<T, OpenMPDepobjConstruct> ||
-        std::is_same_v<T, OpenMPFlushConstruct> ||
-        std::is_same_v<T, OpenMPInteropConstruct> ||
-        std::is_same_v<T, OpenMPSimpleStandaloneConstruct>) {
-      return x.v.DirName();
-    } else {
-      return GetOmpDirectiveName(x.v);
-    }
-  }
-
-  template <typename T, typename = std::enable_if_t<TupleTrait<T>>>
-  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagB = {}) {
-    if constexpr (std::is_same_v<T, OpenMPAllocatorsConstruct> ||
-        std::is_same_v<T, OpenMPAtomicConstruct> ||
-        std::is_same_v<T, OpenMPDispatchConstruct>) {
-      return std::get<OmpDirectiveSpecification>(x.t).DirName();
-    } else if constexpr (std::is_same_v<T, OmpAssumeDirective> ||
-        std::is_same_v<T, OmpCriticalDirective> ||
-        std::is_same_v<T, OmpDeclareVariantDirective> ||
-        std::is_same_v<T, OmpErrorDirective> ||
-        std::is_same_v<T, OmpMetadirectiveDirective> ||
-        std::is_same_v<T, OpenMPDeclarativeAllocate> ||
-        std::is_same_v<T, OpenMPDeclarativeAssumes> ||
-        std::is_same_v<T, OpenMPDeclareMapperConstruct> ||
-        std::is_same_v<T, OpenMPDeclareReductionConstruct> ||
-        std::is_same_v<T, OpenMPDeclareSimdConstruct> ||
-        std::is_same_v<T, OpenMPDeclareTargetConstruct> ||
-        std::is_same_v<T, OpenMPExecutableAllocate> ||
-        std::is_same_v<T, OpenMPRequiresConstruct> ||
-        std::is_same_v<T, OpenMPThreadprivate>) {
-      return MakeName(std::get<Verbatim>(x.t).source, ConstructId<T>::id);
+  template <typename T>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x) {
+    if constexpr (WrapperTrait<T>) {
+      if constexpr (std::is_same_v<T, OpenMPCancelConstruct> ||
+          std::is_same_v<T, OpenMPCancellationPointConstruct> ||
+          std::is_same_v<T, OpenMPDepobjConstruct> ||
+          std::is_same_v<T, OpenMPFlushConstruct> ||
+          std::is_same_v<T, OpenMPInteropConstruct> ||
+          std::is_same_v<T, OpenMPSimpleStandaloneConstruct>) {
+        return x.v.DirName();
+      } else {
+        return GetOmpDirectiveName(x.v);
+      }
+    } else if constexpr (TupleTrait<T>) {
+      if constexpr (std::is_same_v<T, OpenMPAllocatorsConstruct> ||
+          std::is_same_v<T, OpenMPAtomicConstruct> ||
+          std::is_same_v<T, OpenMPDispatchConstruct>) {
+        return std::get<OmpDirectiveSpecification>(x.t).DirName();
+      } else if constexpr (std::is_same_v<T, OmpAssumeDirective> ||
+          std::is_same_v<T, OmpCriticalDirective> ||
+          std::is_same_v<T, OmpDeclareVariantDirective> ||
+          std::is_same_v<T, OmpErrorDirective> ||
+          std::is_same_v<T, OmpMetadirectiveDirective> ||
+          std::is_same_v<T, OpenMPDeclarativeAllocate> ||
+          std::is_same_v<T, OpenMPDeclarativeAssumes> ||
+          std::is_same_v<T, OpenMPDeclareMapperConstruct> ||
+          std::is_same_v<T, OpenMPDeclareReductionConstruct> ||
+          std::is_same_v<T, OpenMPDeclareSimdConstruct> ||
+          std::is_same_v<T, OpenMPDeclareTargetConstruct> ||
+          std::is_same_v<T, OpenMPExecutableAllocate> ||
+          std::is_same_v<T, OpenMPRequiresConstruct> ||
+          std::is_same_v<T, OpenMPThreadprivate>) {
+        return MakeName(std::get<Verbatim>(x.t).source, ConstructId<T>::id);
+      } else {
+        return GetFromTuple(
+            x.t, std::make_index_sequence<std::tuple_size_v<decltype(x.t)>>{});
+      }
+    } else if constexpr (UnionTrait<T>) {
+      return common::visit(
+          [](auto &&s) { return GetOmpDirectiveName(s); }, x.u);
     } else {
-      return GetFromTuple(
-          x.t, std::make_index_sequence<std::tuple_size_v<decltype(x.t)>>{});
+      return MakeName();
     }
   }
 
-  template <typename T, typename = std::enable_if_t<UnionTrait<T>>>
-  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagC = {}) {
-    return common::visit([](auto &&s) { return GetOmpDirectiveName(s); }, x.u);
-  }
-
   template <typename... Ts, size_t... Is>
   static OmpDirectiveName GetFromTuple(
       const std::tuple<Ts...> &t, std::index_sequence<Is...>) {
@@ -156,13 +149,6 @@ struct DirectiveNameScope {
   static OmpDirectiveName GetOmpDirectiveName(const common::Indirection<T> &x) {
     return GetOmpDirectiveName(x.value());
   }
-
-  template <typename T,
-      typename = std::enable_if_t<!WrapperTrait<T> && !TupleTrait<T> &&
-          !UnionTrait<T>>>
-  static OmpDirectiveName GetOmpDirectiveName(const T &x, TagD = {}) {
-    return MakeName();
-  }
 };
 } // namespace detail
 


From flang-commits at lists.llvm.org  Wed Jul 23 06:07:34 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Wed, 23 Jul 2025 06:07:34 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150121@github.com>
Message-ID: <6880de96.170a0220.283a77.2ad8@mx.google.com>


kparzysz wrote:

> Instead of using tag parameters, is it possible to instead
> [...]

Done

https://github.com/llvm/llvm-project/pull/150121

From flang-commits at lists.llvm.org  Wed Jul 23 06:13:12 2025
From: flang-commits at lists.llvm.org (Michael Kruse via flang-commits)
Date: Wed, 23 Jul 2025 06:13:12 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150121@github.com>
Message-ID: <6880dfe8.050a0220.2dd186.2fa3@mx.google.com>


https://github.com/Meinersbur approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/150121

From flang-commits at lists.llvm.org  Wed Jul 23 06:14:20 2025
From: flang-commits at lists.llvm.org (Erich Keane via flang-commits)
Date: Wed, 23 Jul 2025 06:14:20 -0700 (PDT)
Subject: [flang-commits] [flang] [mlir] [acc][flang] Add recipes to data
	entry operations (PR #149210)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149210@github.com>
Message-ID: <6880e02c.170a0220.fa8e6.4eb4@mx.google.com>


https://github.com/erichkeane commented:

I didn't quite get all the patch, but I was wondering if this is going to reject the old mechanism?  It would be great for me if it did, as that'll make sure I get it right in the clang implementation after rebase :)

https://github.com/llvm/llvm-project/pull/149210

From flang-commits at lists.llvm.org  Wed Jul 23 06:19:48 2025
From: flang-commits at lists.llvm.org (Ryotaro Kasuga via flang-commits)
Date: Wed, 23 Jul 2025 06:19:48 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] add -floop-fuse to clang and
	flang (PR #142686)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/142686@github.com>
Message-ID: <6880e174.050a0220.37c0b0.25e9@mx.google.com>


kasuga-fj wrote:

So, why wouldn't adding a `cl::opt` flag work in the end? In fact, you've added `enable-loopfuse` flag in this patch. If it's enough, I personally think it might be better to avoid introducing additional flags unnecessarily.

https://github.com/llvm/llvm-project/pull/142686

From flang-commits at lists.llvm.org  Wed Jul 23 06:25:49 2025
From: flang-commits at lists.llvm.org (Krzysztof Parzyszek via flang-commits)
Date: Wed, 23 Jul 2025 06:25:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][OpenMP] General utility to get
 directive id from AST node (PR #150121)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150121@github.com>
Message-ID: <6880e2dd.170a0220.a9a63.37d5@mx.google.com>


https://github.com/kparzysz closed https://github.com/llvm/llvm-project/pull/150121

From flang-commits at lists.llvm.org  Wed Jul 23 06:26:05 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 06:26:05 -0700 (PDT)
Subject: [flang-commits] [flang] 43db6c5 - [flang][OpenMP] General utility
 to get directive id from AST node (#150121)
Message-ID: <6880e2ed.170a0220.1c3ad4.56ed@mx.google.com>


Author: Krzysztof Parzyszek
Date: 2025-07-23T08:25:33-05:00
New Revision: 43db6c5cc1a81b540ddca49bee197895c420ec2d

URL: https://github.com/llvm/llvm-project/commit/43db6c5cc1a81b540ddca49bee197895c420ec2d
DIFF: https://github.com/llvm/llvm-project/commit/43db6c5cc1a81b540ddca49bee197895c420ec2d.diff

LOG: [flang][OpenMP] General utility to get directive id from AST node (#150121)

Fortran::parser::omp::GetOmpDirectiveName(t) will get the
OmpDirectiveName object that corresponds to construct t. That object (an
AST node) contains the enum id and the source information of the
directive.

Replace uses of extractOmpDirective and getOpenMPDirectiveEnum with the
new function.

Added: 
    flang/include/flang/Parser/openmp-utils.h

Modified: 
    flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
    flang/lib/Lower/OpenMP/OpenMP.cpp
    flang/lib/Lower/OpenMP/Utils.cpp
    flang/lib/Lower/OpenMP/Utils.h

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/openmp-utils.h b/flang/include/flang/Parser/openmp-utils.h
new file mode 100644
index 0000000000000..579ea7d74957f
--- /dev/null
+++ b/flang/include/flang/Parser/openmp-utils.h
@@ -0,0 +1,161 @@
+//===-- flang/Parser/openmp-utils.h ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common OpenMP utilities.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_PARSER_OPENMP_UTILS_H
+#define FORTRAN_PARSER_OPENMP_UTILS_H
+
+#include "flang/Common/indirection.h"
+#include "flang/Parser/parse-tree.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cassert>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+namespace Fortran::parser::omp {
+
+namespace detail {
+using D = llvm::omp::Directive;
+
+template <typename Construct> //
+struct ConstructId {
+  static constexpr llvm::omp::Directive id{D::OMPD_unknown};
+};
+
+#define MAKE_CONSTR_ID(Construct, Id) \
+  template <> struct ConstructId<Construct> { \
+    static constexpr llvm::omp::Directive id{Id}; \
+  }
+
+MAKE_CONSTR_ID(OmpAssumeDirective, D::OMPD_assume);
+MAKE_CONSTR_ID(OmpCriticalDirective, D::OMPD_critical);
+MAKE_CONSTR_ID(OmpDeclareVariantDirective, D::OMPD_declare_variant);
+MAKE_CONSTR_ID(OmpErrorDirective, D::OMPD_error);
+MAKE_CONSTR_ID(OmpMetadirectiveDirective, D::OMPD_metadirective);
+MAKE_CONSTR_ID(OpenMPDeclarativeAllocate, D::OMPD_allocate);
+MAKE_CONSTR_ID(OpenMPDeclarativeAssumes, D::OMPD_assumes);
+MAKE_CONSTR_ID(OpenMPDeclareMapperConstruct, D::OMPD_declare_mapper);
+MAKE_CONSTR_ID(OpenMPDeclareReductionConstruct, D::OMPD_declare_reduction);
+MAKE_CONSTR_ID(OpenMPDeclareSimdConstruct, D::OMPD_declare_simd);
+MAKE_CONSTR_ID(OpenMPDeclareTargetConstruct, D::OMPD_declare_target);
+MAKE_CONSTR_ID(OpenMPExecutableAllocate, D::OMPD_allocate);
+MAKE_CONSTR_ID(OpenMPRequiresConstruct, D::OMPD_requires);
+MAKE_CONSTR_ID(OpenMPThreadprivate, D::OMPD_threadprivate);
+
+#undef MAKE_CONSTR_ID
+
+struct DirectiveNameScope {
+  static OmpDirectiveName MakeName(CharBlock source = {},
+      llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown) {
+    OmpDirectiveName name;
+    name.source = source;
+    name.v = id;
+    return name;
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpNothingDirective &x) {
+    return MakeName(x.source, llvm::omp::Directive::OMPD_nothing);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpBeginBlockDirective &x) {
+    auto &dir{std::get<OmpBlockDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(const OmpBeginLoopDirective &x) {
+    auto &dir{std::get<OmpLoopDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  static OmpDirectiveName GetOmpDirectiveName(
+      const OmpBeginSectionsDirective &x) {
+    auto &dir{std::get<OmpSectionsDirective>(x.t)};
+    return MakeName(dir.source, dir.v);
+  }
+
+  template <typename T>
+  static OmpDirectiveName GetOmpDirectiveName(const T &x) {
+    if constexpr (WrapperTrait<T>) {
+      if constexpr (std::is_same_v<T, OpenMPCancelConstruct> ||
+          std::is_same_v<T, OpenMPCancellationPointConstruct> ||
+          std::is_same_v<T, OpenMPDepobjConstruct> ||
+          std::is_same_v<T, OpenMPFlushConstruct> ||
+          std::is_same_v<T, OpenMPInteropConstruct> ||
+          std::is_same_v<T, OpenMPSimpleStandaloneConstruct>) {
+        return x.v.DirName();
+      } else {
+        return GetOmpDirectiveName(x.v);
+      }
+    } else if constexpr (TupleTrait<T>) {
+      if constexpr (std::is_same_v<T, OpenMPAllocatorsConstruct> ||
+          std::is_same_v<T, OpenMPAtomicConstruct> ||
+          std::is_same_v<T, OpenMPDispatchConstruct>) {
+        return std::get<OmpDirectiveSpecification>(x.t).DirName();
+      } else if constexpr (std::is_same_v<T, OmpAssumeDirective> ||
+          std::is_same_v<T, OmpCriticalDirective> ||
+          std::is_same_v<T, OmpDeclareVariantDirective> ||
+          std::is_same_v<T, OmpErrorDirective> ||
+          std::is_same_v<T, OmpMetadirectiveDirective> ||
+          std::is_same_v<T, OpenMPDeclarativeAllocate> ||
+          std::is_same_v<T, OpenMPDeclarativeAssumes> ||
+          std::is_same_v<T, OpenMPDeclareMapperConstruct> ||
+          std::is_same_v<T, OpenMPDeclareReductionConstruct> ||
+          std::is_same_v<T, OpenMPDeclareSimdConstruct> ||
+          std::is_same_v<T, OpenMPDeclareTargetConstruct> ||
+          std::is_same_v<T, OpenMPExecutableAllocate> ||
+          std::is_same_v<T, OpenMPRequiresConstruct> ||
+          std::is_same_v<T, OpenMPThreadprivate>) {
+        return MakeName(std::get<Verbatim>(x.t).source, ConstructId<T>::id);
+      } else {
+        return GetFromTuple(
+            x.t, std::make_index_sequence<std::tuple_size_v<decltype(x.t)>>{});
+      }
+    } else if constexpr (UnionTrait<T>) {
+      return common::visit(
+          [](auto &&s) { return GetOmpDirectiveName(s); }, x.u);
+    } else {
+      return MakeName();
+    }
+  }
+
+  template <typename... Ts, size_t... Is>
+  static OmpDirectiveName GetFromTuple(
+      const std::tuple<Ts...> &t, std::index_sequence<Is...>) {
+    OmpDirectiveName name = MakeName();
+    auto accumulate = [&](const OmpDirectiveName &n) {
+      if (name.v == llvm::omp::Directive::OMPD_unknown) {
+        name = n;
+      } else {
+        assert(
+            n.v == llvm::omp::Directive::OMPD_unknown && "Conflicting names");
+      }
+    };
+    (accumulate(GetOmpDirectiveName(std::get<Is>(t))), ...);
+    return name;
+  }
+
+  template <typename T>
+  static OmpDirectiveName GetOmpDirectiveName(const common::Indirection<T> &x) {
+    return GetOmpDirectiveName(x.value());
+  }
+};
+} // namespace detail
+
+template <typename T> OmpDirectiveName GetOmpDirectiveName(const T &x) {
+  return detail::DirectiveNameScope::GetOmpDirectiveName(x);
+}
+
+} // namespace Fortran::parser::omp
+
+#endif // FORTRAN_PARSER_OPENMP_UTILS_H

diff  --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 11e488371b886..2ac4d9548b65b 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -24,6 +24,7 @@
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Parser/openmp-utils.h"
 #include "flang/Semantics/attr.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/ADT/Sequence.h"
@@ -465,7 +466,8 @@ bool DataSharingProcessor::isOpenMPPrivatizingConstruct(
   // allow a privatizing clause) are: dispatch, distribute, do, for, loop,
   // parallel, scope, sections, simd, single, target, target_data, task,
   // taskgroup, taskloop, and teams.
-  return llvm::is_contained(privatizing, extractOmpDirective(omp));
+  return llvm::is_contained(privatizing,
+                            parser::omp::GetOmpDirectiveName(omp).v);
 }
 
 bool DataSharingProcessor::isOpenMPPrivatizingEvaluation(

diff  --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index fc5fef9b2c577..4c2d7badef382 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -31,6 +31,7 @@
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Parser/characters.h"
+#include "flang/Parser/openmp-utils.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/tools.h"
@@ -63,28 +64,6 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
                                    lower::pft::Evaluation &eval,
                                    mlir::Location loc);
 
-static llvm::omp::Directive
-getOpenMPDirectiveEnum(const parser::OmpLoopDirective &beginStatment) {
-  return beginStatment.v;
-}
-
-static llvm::omp::Directive getOpenMPDirectiveEnum(
-    const parser::OmpBeginLoopDirective &beginLoopDirective) {
-  return getOpenMPDirectiveEnum(
-      std::get<parser::OmpLoopDirective>(beginLoopDirective.t));
-}
-
-static llvm::omp::Directive
-getOpenMPDirectiveEnum(const parser::OpenMPLoopConstruct &ompLoopConstruct) {
-  return getOpenMPDirectiveEnum(
-      std::get<parser::OmpBeginLoopDirective>(ompLoopConstruct.t));
-}
-
-static llvm::omp::Directive getOpenMPDirectiveEnum(
-    const common::Indirection<parser::OpenMPLoopConstruct> &ompLoopConstruct) {
-  return getOpenMPDirectiveEnum(ompLoopConstruct.value());
-}
-
 namespace {
 /// Structure holding information that is needed to pass host-evaluated
 /// information to later lowering stages.
@@ -468,7 +447,7 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
     llvm::omp::Directive dir;
     auto &nested = parent.getFirstNestedEvaluation();
     if (const auto *ompEval = nested.getIf<parser::OpenMPConstruct>())
-      dir = extractOmpDirective(*ompEval);
+      dir = parser::omp::GetOmpDirectiveName(*ompEval).v;
     else
       return std::nullopt;
 
@@ -508,7 +487,7 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
     HostEvalInfo *hostInfo = getHostEvalInfoStackTop(converter);
     assert(hostInfo && "expected HOST_EVAL info structure");
 
-    switch (extractOmpDirective(*ompEval)) {
+    switch (parser::omp::GetOmpDirectiveName(*ompEval).v) {
     case OMPD_teams_distribute_parallel_do:
     case OMPD_teams_distribute_parallel_do_simd:
       cp.processThreadLimit(stmtCtx, hostInfo->ops);
@@ -569,7 +548,8 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
 
   const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
   assert(ompEval &&
-         llvm::omp::allTargetSet.test(extractOmpDirective(*ompEval)) &&
+         llvm::omp::allTargetSet.test(
+             parser::omp::GetOmpDirectiveName(*ompEval).v) &&
          "expected TARGET construct evaluation");
   (void)ompEval;
 
@@ -3872,7 +3852,7 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
             std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
                 &*optLoopCons)}) {
       llvm::omp::Directive nestedDirective =
-          getOpenMPDirectiveEnum(*ompNestedLoopCons);
+          parser::omp::GetOmpDirectiveName(*ompNestedLoopCons).v;
       switch (nestedDirective) {
       case llvm::omp::Directive::OMPD_tile:
         // Emit the omp.loop_nest with annotation for tiling
@@ -3889,7 +3869,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
     }
   }
 
-  llvm::omp::Directive directive = getOpenMPDirectiveEnum(beginLoopDirective);
+  llvm::omp::Directive directive =
+      parser::omp::GetOmpDirectiveName(beginLoopDirective).v;
   const parser::CharBlock &source =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
   ConstructQueue queue{

diff  --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index b1716d6afb200..13fda978c5369 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -20,6 +20,7 @@
 #include <flang/Lower/PFTBuilder.h>
 #include <flang/Optimizer/Builder/FIRBuilder.h>
 #include <flang/Optimizer/Builder/Todo.h>
+#include <flang/Parser/openmp-utils.h>
 #include <flang/Parser/parse-tree.h>
 #include <flang/Parser/tools.h>
 #include <flang/Semantics/tools.h>
@@ -663,89 +664,6 @@ bool collectLoopRelatedInfo(
   return found;
 }
 
-/// Get the directive enumeration value corresponding to the given OpenMP
-/// construct PFT node.
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
-  return common::visit(
-      common::visitors{
-          [](const parser::OpenMPAllocatorsConstruct &c) {
-            return llvm::omp::OMPD_allocators;
-          },
-          [](const parser::OpenMPAssumeConstruct &c) {
-            return llvm::omp::OMPD_assume;
-          },
-          [](const parser::OpenMPAtomicConstruct &c) {
-            return llvm::omp::OMPD_atomic;
-          },
-          [](const parser::OpenMPBlockConstruct &c) {
-            return std::get<parser::OmpBlockDirective>(
-                       std::get<parser::OmpBeginBlockDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPCriticalConstruct &c) {
-            return llvm::omp::OMPD_critical;
-          },
-          [](const parser::OpenMPDeclarativeAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPDispatchConstruct &c) {
-            return llvm::omp::OMPD_dispatch;
-          },
-          [](const parser::OpenMPExecutableAllocate &c) {
-            return llvm::omp::OMPD_allocate;
-          },
-          [](const parser::OpenMPLoopConstruct &c) {
-            return std::get<parser::OmpLoopDirective>(
-                       std::get<parser::OmpBeginLoopDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPSectionConstruct &c) {
-            return llvm::omp::OMPD_section;
-          },
-          [](const parser::OpenMPSectionsConstruct &c) {
-            return std::get<parser::OmpSectionsDirective>(
-                       std::get<parser::OmpBeginSectionsDirective>(c.t).t)
-                .v;
-          },
-          [](const parser::OpenMPStandaloneConstruct &c) {
-            return common::visit(
-                common::visitors{
-                    [](const parser::OpenMPSimpleStandaloneConstruct &c) {
-                      return c.v.DirId();
-                    },
-                    [](const parser::OpenMPFlushConstruct &c) {
-                      return llvm::omp::OMPD_flush;
-                    },
-                    [](const parser::OpenMPCancelConstruct &c) {
-                      return llvm::omp::OMPD_cancel;
-                    },
-                    [](const parser::OpenMPCancellationPointConstruct &c) {
-                      return llvm::omp::OMPD_cancellation_point;
-                    },
-                    [](const parser::OmpMetadirectiveDirective &c) {
-                      return llvm::omp::OMPD_metadirective;
-                    },
-                    [](const parser::OpenMPDepobjConstruct &c) {
-                      return llvm::omp::OMPD_depobj;
-                    },
-                    [](const parser::OpenMPInteropConstruct &c) {
-                      return llvm::omp::OMPD_interop;
-                    }},
-                c.u);
-          },
-          [](const parser::OpenMPUtilityConstruct &c) {
-            return common::visit(
-                common::visitors{[](const parser::OmpErrorDirective &c) {
-                                   return llvm::omp::OMPD_error;
-                                 },
-                                 [](const parser::OmpNothingDirective &c) {
-                                   return llvm::omp::OMPD_nothing;
-                                 }},
-                c.u);
-          }},
-      ompConstruct.u);
-}
 } // namespace omp
 } // namespace lower
 } // namespace Fortran

diff  --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 8e3ad5c3452e2..11641ba5e8606 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -167,8 +167,6 @@ bool collectLoopRelatedInfo(
     mlir::omp::LoopRelatedClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &iv);
 
-llvm::omp::Directive
-extractOmpDirective(const parser::OpenMPConstruct &ompConstruct);
 } // namespace omp
 } // namespace lower
 } // namespace Fortran


        

From flang-commits at lists.llvm.org  Wed Jul 23 06:28:15 2025
From: flang-commits at lists.llvm.org (Ryotaro Kasuga via flang-commits)
Date: Wed, 23 Jul 2025 06:28:15 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] add -floop-fuse to clang and
	flang (PR #142686)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/142686@github.com>
Message-ID: <6880e36f.050a0220.37aa59.2ff2@mx.google.com>


================
@@ -2112,7 +2124,6 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
     LPM.addPass(LoopFlattenPass());
   LPM.addPass(IndVarSimplifyPass());
   LPM.addPass(LoopDeletionPass());
-  // FIXME: Add loop interchange.
----------------
kasuga-fj wrote:

I don't think this patch resolves this comment.

https://github.com/llvm/llvm-project/pull/142686

From flang-commits at lists.llvm.org  Wed Jul 23 06:28:15 2025
From: flang-commits at lists.llvm.org (Ryotaro Kasuga via flang-commits)
Date: Wed, 23 Jul 2025 06:28:15 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] add -floop-fuse to clang and
	flang (PR #142686)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/142686@github.com>
Message-ID: <6880e36f.050a0220.1d58bd.1873@mx.google.com>


================
@@ -518,6 +524,9 @@ PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
 
   invokeLoopOptimizerEndEPCallbacks(LPM2, Level);
 
+  if (PTO.LoopFuse)
+    FPM.addPass(LoopFusePass());
+
----------------
kasuga-fj wrote:

Is this an appropriate place? I think this is an optimization pass rather than simplification one.

https://github.com/llvm/llvm-project/pull/142686

From flang-commits at lists.llvm.org  Wed Jul 23 06:28:16 2025
From: flang-commits at lists.llvm.org (Ryotaro Kasuga via flang-commits)
Date: Wed, 23 Jul 2025 06:28:16 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [llvm] add -floop-fuse to clang and
	flang (PR #142686)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/142686@github.com>
Message-ID: <6880e370.170a0220.3ada64.58c6@mx.google.com>


================
@@ -3157,7 +3157,7 @@ void tools::handleVectorizeSLPArgs(const ArgList &Args,
 
 void tools::handleInterchangeLoopsArgs(const ArgList &Args,
                                        ArgStringList &CmdArgs) {
-  // FIXME: instead of relying on shouldEnableVectorizerAtOLevel, we may want to
+  // FIXME: Instead of relying on shouldEnableVectorizerAtOLevel, we may want to
----------------
kasuga-fj wrote:

nit: unrelated change.

https://github.com/llvm/llvm-project/pull/142686

From flang-commits at lists.llvm.org  Wed Jul 23 06:38:57 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 06:38:57 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `tanpi` (PR #149527)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149527@github.com>
Message-ID: <6880e5f1.170a0220.10a359.3736@mx.google.com>


c8ef wrote:

### Merge activity

* **Jul 23, 1:38 PM UTC**: A user started a stack merge that includes this pull request via [Graphite](https://app.graphite.dev/github/pr/llvm/llvm-project/149527).


https://github.com/llvm/llvm-project/pull/149527

From flang-commits at lists.llvm.org  Wed Jul 23 06:39:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 06:39:46 -0700 (PDT)
Subject: [flang-commits] [flang] f1bb5de - [flang] Implement `tanpi`
	(#149527)
Message-ID: <6880e622.050a0220.3bdc25.432d@mx.google.com>


Author: Connector Switch
Date: 2025-07-23T21:39:42+08:00
New Revision: f1bb5de611922bfd76846d0b36a9b92a1dfce80e

URL: https://github.com/llvm/llvm-project/commit/f1bb5de611922bfd76846d0b36a9b92a1dfce80e
DIFF: https://github.com/llvm/llvm-project/commit/f1bb5de611922bfd76846d0b36a9b92a1dfce80e.diff

LOG: [flang] Implement `tanpi` (#149527)

Added: 
    flang/test/Lower/Intrinsics/tanpi.f90

Modified: 
    flang/include/flang/Optimizer/Builder/IntrinsicCall.h
    flang/lib/Evaluate/intrinsics.cpp
    flang/lib/Optimizer/Builder/IntrinsicCall.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index d84d3593ebca6..ab08415fe32c8 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -439,6 +439,7 @@ struct IntrinsicLibrary {
                                mlir::ArrayRef<fir::ExtendedValue> args);
   void genSystemClock(llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genTand(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genTanpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genTime(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genTrailz(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genTransfer(mlir::Type,

diff  --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index d44239b41fa20..9c6d49dfda3e8 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -989,6 +989,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"tan", {{"x", SameFloating}}, SameFloating},
     {"tand", {{"x", SameFloating}}, SameFloating},
     {"tanh", {{"x", SameFloating}}, SameFloating},
+    {"tanpi", {{"x", SameFloating}}, SameFloating},
     {"team_number", {OptionalTEAM}, DefaultInt, Rank::scalar,
         IntrinsicClass::transformationalFunction},
     {"this_image",

diff  --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index b589a6c7c62fb..ba298298be675 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -942,6 +942,7 @@ static constexpr IntrinsicHandler handlers[]{
      {{{"count", asAddr}, {"count_rate", asAddr}, {"count_max", asAddr}}},
      /*isElemental=*/false},
     {"tand", &I::genTand},
+    {"tanpi", &I::genTanpi},
     {"this_grid", &I::genThisGrid, {}, /*isElemental=*/false},
     {"this_thread_block", &I::genThisThreadBlock, {}, /*isElemental=*/false},
     {"this_warp", &I::genThisWarp, {}, /*isElemental=*/false},
@@ -8177,6 +8178,21 @@ mlir::Value IntrinsicLibrary::genTand(mlir::Type resultType,
   return getRuntimeCallGenerator("tan", ftype)(builder, loc, {arg});
 }
 
+// TANPI
+mlir::Value IntrinsicLibrary::genTanpi(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
+  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
+  return getRuntimeCallGenerator("tan", ftype)(builder, loc, {arg});
+}
+
 // THIS_GRID
 mlir::Value IntrinsicLibrary::genThisGrid(mlir::Type resultType,
                                           llvm::ArrayRef<mlir::Value> args) {

diff  --git a/flang/test/Lower/Intrinsics/tanpi.f90 b/flang/test/Lower/Intrinsics/tanpi.f90
new file mode 100644
index 0000000000000..9cc3ae6ef1563
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/tanpi.f90
@@ -0,0 +1,22 @@
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = tanpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f32
+! CHECK: %[[tan:.*]] = math.tan %[[mul]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = tanpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[dfactor]] fastmath<contract> : f64
+! CHECK: %[[tan:.*]] = math.tan %[[mul]] fastmath<contract> : f64


        

From flang-commits at lists.llvm.org  Wed Jul 23 06:39:49 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 06:39:49 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `tanpi` (PR #149527)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149527@github.com>
Message-ID: <6880e625.170a0220.29e15a.317b@mx.google.com>


https://github.com/c8ef closed https://github.com/llvm/llvm-project/pull/149527

From flang-commits at lists.llvm.org  Wed Jul 23 07:14:58 2025
From: flang-commits at lists.llvm.org (Carlos Seo via flang-commits)
Date: Wed, 23 Jul 2025 07:14:58 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ASSIGN statement (PR #149941)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149941@github.com>
Message-ID: <6880ee62.170a0220.3250e.edbc@mx.google.com>


================
@@ -5506,10 +5506,34 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   void genFIR(const Fortran::parser::AssignStmt &stmt) {
     const Fortran::semantics::Symbol &symbol =
         *std::get<Fortran::parser::Name>(stmt.t).symbol;
+
     mlir::Location loc = toLocation();
+    mlir::Type symbolType = genType(symbol);
+    mlir::Value addr = getSymbolAddress(symbol);
+
+    // Handle the case where the assigned variable is declared as a pointer
+    if (auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(symbolType)) {
+      if (auto ptrType = mlir::dyn_cast<fir::PointerType>(eleTy)) {
+        symbolType = ptrType.getEleTy();
+      } else {
+        symbolType = eleTy;
+      }
+    } else if (auto ptrType = mlir::dyn_cast<fir::PointerType>(symbolType)) {
+      symbolType = ptrType.getEleTy();
+    }
+
     mlir::Value labelValue = builder->createIntegerConstant(
-        loc, genType(symbol), std::get<Fortran::parser::Label>(stmt.t));
-    builder->create<fir::StoreOp>(loc, labelValue, getSymbolAddress(symbol));
+        loc, symbolType, std::get<Fortran::parser::Label>(stmt.t));
+
+    // If the address points to a boxed pointer, we need to dereference it
+    if (auto refType = mlir::dyn_cast<fir::ReferenceType>(addr.getType())) {
+      if (auto boxType = mlir::dyn_cast<fir::BoxType>(refType.getEleTy())) {
+        mlir::Value boxValue = builder->create<fir::LoadOp>(loc, addr);
+        addr = builder->create<fir::BoxAddrOp>(loc, boxValue);
----------------
ceseo wrote:

OK, we can fix this after LLVM 21 is released.

https://github.com/llvm/llvm-project/pull/149941

From flang-commits at lists.llvm.org  Wed Jul 23 07:16:15 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 07:16:15 -0700 (PDT)
Subject: [flang-commits] [flang] fc0a978 - [Flang] Fix ASSIGN statement
	(#149941)
Message-ID: <6880eeaf.170a0220.1d6216.9947@mx.google.com>


Author: Carlos Seo
Date: 2025-07-23T11:16:11-03:00
New Revision: fc0a978327215aa8883ae6f18d1e316f3c04520a

URL: https://github.com/llvm/llvm-project/commit/fc0a978327215aa8883ae6f18d1e316f3c04520a
DIFF: https://github.com/llvm/llvm-project/commit/fc0a978327215aa8883ae6f18d1e316f3c04520a.diff

LOG: [Flang] Fix ASSIGN statement (#149941)

Handle the case where the assigned variable also has a pointer
attribute.

Fixes #121721

Added: 
    flang/test/Lower/assign-statement.f90

Modified: 
    flang/lib/Lower/Bridge.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 13420f365919d..37f2475763f08 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5509,10 +5509,34 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   void genFIR(const Fortran::parser::AssignStmt &stmt) {
     const Fortran::semantics::Symbol &symbol =
         *std::get<Fortran::parser::Name>(stmt.t).symbol;
+
     mlir::Location loc = toLocation();
+    mlir::Type symbolType = genType(symbol);
+    mlir::Value addr = getSymbolAddress(symbol);
+
+    // Handle the case where the assigned variable is declared as a pointer
+    if (auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(symbolType)) {
+      if (auto ptrType = mlir::dyn_cast<fir::PointerType>(eleTy)) {
+        symbolType = ptrType.getEleTy();
+      } else {
+        symbolType = eleTy;
+      }
+    } else if (auto ptrType = mlir::dyn_cast<fir::PointerType>(symbolType)) {
+      symbolType = ptrType.getEleTy();
+    }
+
     mlir::Value labelValue = builder->createIntegerConstant(
-        loc, genType(symbol), std::get<Fortran::parser::Label>(stmt.t));
-    builder->create<fir::StoreOp>(loc, labelValue, getSymbolAddress(symbol));
+        loc, symbolType, std::get<Fortran::parser::Label>(stmt.t));
+
+    // If the address points to a boxed pointer, we need to dereference it
+    if (auto refType = mlir::dyn_cast<fir::ReferenceType>(addr.getType())) {
+      if (auto boxType = mlir::dyn_cast<fir::BoxType>(refType.getEleTy())) {
+        mlir::Value boxValue = builder->create<fir::LoadOp>(loc, addr);
+        addr = builder->create<fir::BoxAddrOp>(loc, boxValue);
+      }
+    }
+
+    builder->create<fir::StoreOp>(loc, labelValue, addr);
   }
 
   void genFIR(const Fortran::parser::FormatStmt &) {

diff  --git a/flang/test/Lower/assign-statement.f90 b/flang/test/Lower/assign-statement.f90
new file mode 100644
index 0000000000000..342355bf469c8
--- /dev/null
+++ b/flang/test/Lower/assign-statement.f90
@@ -0,0 +1,12 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+  ! CHECK-LABEL: func @_QQmain
+  program main
+  integer :: ip
+  pointer :: ip
+
+  allocate(ip)
+  assign 10 to ip
+  ! CHECK: fir.store %c10_i32 to %11 : !fir.ptr<i32>
+  10 return
+  end program main


        

From flang-commits at lists.llvm.org  Wed Jul 23 07:16:18 2025
From: flang-commits at lists.llvm.org (Carlos Seo via flang-commits)
Date: Wed, 23 Jul 2025 07:16:18 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix ASSIGN statement (PR #149941)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/149941@github.com>
Message-ID: <6880eeb2.170a0220.a8279.700c@mx.google.com>


https://github.com/ceseo closed https://github.com/llvm/llvm-project/pull/149941

From flang-commits at lists.llvm.org  Wed Jul 23 07:17:33 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 07:17:33 -0700 (PDT)
Subject: [flang-commits] [flang] c113036 - [Flang] Fix a crash when
 equivalence and namelist statements are used (#150081)
Message-ID: <6880eefd.170a0220.150edf.9ea6@mx.google.com>


Author: Carlos Seo
Date: 2025-07-23T11:17:29-03:00
New Revision: c1130360902082e5d11fcf9a6a4ddd5dfc1a8ec9

URL: https://github.com/llvm/llvm-project/commit/c1130360902082e5d11fcf9a6a4ddd5dfc1a8ec9
DIFF: https://github.com/llvm/llvm-project/commit/c1130360902082e5d11fcf9a6a4ddd5dfc1a8ec9.diff

LOG: [Flang] Fix a crash when equivalence and namelist statements are used (#150081)

Check for equivalence when generating namelist descriptors in IO.cpp.

Fixes #124489

Added: 
    flang/test/Lower/equivalence-3.f

Modified: 
    flang/lib/Lower/IO.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index c95c3404a8e26..51f192ec08fe6 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -468,8 +468,10 @@ getNamelistGroup(Fortran::lower::AbstractConverter &converter,
       fir::BoxType boxTy =
           fir::BoxType::get(fir::PointerType::get(converter.genType(s)));
       auto descFunc = [&](fir::FirOpBuilder &b) {
+        bool couldBeInEquivalence =
+            Fortran::semantics::FindEquivalenceSet(s) != nullptr;
         auto box = Fortran::lower::genInitialDataTarget(
-            converter, loc, boxTy, *expr, /*couldBeInEquivalence=*/true);
+            converter, loc, boxTy, *expr, couldBeInEquivalence);
         fir::HasValueOp::create(b, loc, box);
       };
       builder.createGlobalConstant(loc, boxTy, mangleName, descFunc, linkOnce);

diff  --git a/flang/test/Lower/equivalence-3.f b/flang/test/Lower/equivalence-3.f
new file mode 100644
index 0000000000000..19f8880189f0c
--- /dev/null
+++ b/flang/test/Lower/equivalence-3.f
@@ -0,0 +1,19 @@
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
+
+  ! CHECK-LABEL: func @_QQmain
+  program main
+  real a1,a2
+  equivalence (a1,a2)
+  ! A fir.alloca should never appear in a global constant initialization.
+  ! CHECK: fir.global linkonce @_QFEx1.desc constant : !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  ! CHECK: arith.constant 5 : index
+  ! CHECK:fir.address_of(@_QFEx1) : !fir.ref<!fir.array<5xf64>>
+  ! CHECK: fir.shape %c5 : (index) -> !fir.shape<1>
+  ! CHECK: fir.declare %0(%1) {uniq_name = "_QFEx1"} : (!fir.ref<!fir.array<5xf64>>, !fir.shape<1>) -> !fir.ref<!fir.array<5xf64>>
+  ! CHECK: fir.embox %2(%1) : (!fir.ref<!fir.array<5xf64>>, !fir.shape<1>) -> !fir.box<!fir.array<5xf64>>
+  ! CHECK: fir.rebox %3 : (!fir.box<!fir.array<5xf64>>) -> !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  ! CHECK: fir.has_value %4 : !fir.box<!fir.ptr<!fir.array<5xf64>>>
+  real*8 x1(5)
+  namelist /y1/x1
+  read (5,y1)
+  end


        

From flang-commits at lists.llvm.org  Wed Jul 23 07:17:36 2025
From: flang-commits at lists.llvm.org (Carlos Seo via flang-commits)
Date: Wed, 23 Jul 2025 07:17:36 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Fix a crash when equivalence and
 namelist statements are used (PR #150081)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150081@github.com>
Message-ID: <6880ef00.050a0220.166ba4.5dcd@mx.google.com>


https://github.com/ceseo closed https://github.com/llvm/llvm-project/pull/150081

From flang-commits at lists.llvm.org  Wed Jul 23 08:06:27 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:06:27 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
Message-ID: <llvm.org/llvm/llvm-project/pull/150234@github.com>


https://github.com/c8ef created https://github.com/llvm/llvm-project/pull/150234

None

>From fc1e15dfe2dcab282bd6e65fabdbe750b7eaeb7a Mon Sep 17 00:00:00 2001
From: c8ef <c8ef at outlook.com>
Date: Wed, 23 Jul 2025 15:05:18 +0000
Subject: [PATCH] [flang] Implement `acospi`

---
 .../flang/Optimizer/Builder/IntrinsicCall.h   |  1 +
 flang/lib/Evaluate/intrinsics.cpp             |  1 +
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 45 ++++++++++++-------
 flang/test/Lower/Intrinsics/acospi.f90        | 26 +++++++++++
 4 files changed, 58 insertions(+), 15 deletions(-)
 create mode 100644 flang/test/Lower/Intrinsics/acospi.f90

diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index ab08415fe32c8..a93b397240576 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -174,6 +174,7 @@ struct IntrinsicLibrary {
   /// real and to the `hypot` math routine if the argument is of complex type.
   mlir::Value genAbs(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAcosd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAcospi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                                 mlir::Value, mlir::Value)>
   fir::ExtendedValue genAdjustRtCall(mlir::Type,
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9c6d49dfda3e8..97e87935c02b0 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -340,6 +340,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"acos", {{"x", SameFloating}}, SameFloating},
     {"acosd", {{"x", SameFloating}}, SameFloating},
     {"acosh", {{"x", SameFloating}}, SameFloating},
+    {"acospi", {{"x", SameFloating}}, SameFloating},
     {"adjustl", {{"string", SameChar}}, SameChar},
     {"adjustr", {{"string", SameChar}}, SameChar},
     {"aimag", {{"z", SameComplex}}, SameReal},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index ba298298be675..69e0959249acc 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -246,6 +246,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"abs", &I::genAbs},
     {"achar", &I::genChar},
     {"acosd", &I::genAcosd},
+    {"acospi", &I::genAcospi},
     {"adjustl",
      &I::genAdjustRtCall<fir::runtime::genAdjustL>,
      {{{"string", asAddr}}},
@@ -2676,6 +2677,21 @@ mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
+// ACOSPI
+mlir::Value IntrinsicLibrary::genAcospi(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value acos = getRuntimeCallGenerator("acos", ftype)(builder, loc, args);
+  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
+  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  return mlir::arith::MulFOp::create(builder, loc, acos, factor);
+}
+
 // ADJUSTL & ADJUSTR
 template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                               mlir::Value, mlir::Value)>
@@ -3981,21 +3997,20 @@ void IntrinsicLibrary::genExecuteCommandLine(
     mlir::Value waitAddr = fir::getBase(wait);
     mlir::Value waitIsPresentAtRuntime =
         builder.genIsNotNullAddr(loc, waitAddr);
-    waitBool = builder
-                   .genIfOp(loc, {i1Ty}, waitIsPresentAtRuntime,
-                            /*withElseRegion=*/true)
-                   .genThen([&]() {
-                     auto waitLoad =
-                         fir::LoadOp::create(builder, loc, waitAddr);
-                     mlir::Value cast =
-                         builder.createConvert(loc, i1Ty, waitLoad);
-                     fir::ResultOp::create(builder, loc, cast);
-                   })
-                   .genElse([&]() {
-                     mlir::Value trueVal = builder.createBool(loc, true);
-                     fir::ResultOp::create(builder, loc, trueVal);
-                   })
-                   .getResults()[0];
+    waitBool =
+        builder
+            .genIfOp(loc, {i1Ty}, waitIsPresentAtRuntime,
+                     /*withElseRegion=*/true)
+            .genThen([&]() {
+              auto waitLoad = fir::LoadOp::create(builder, loc, waitAddr);
+              mlir::Value cast = builder.createConvert(loc, i1Ty, waitLoad);
+              fir::ResultOp::create(builder, loc, cast);
+            })
+            .genElse([&]() {
+              mlir::Value trueVal = builder.createBool(loc, true);
+              fir::ResultOp::create(builder, loc, trueVal);
+            })
+            .getResults()[0];
   }
 
   mlir::Value exitstatBox =
diff --git a/flang/test/Lower/Intrinsics/acospi.f90 b/flang/test/Lower/Intrinsics/acospi.f90
new file mode 100644
index 0000000000000..dcacd25bca480
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/acospi.f90
@@ -0,0 +1,26 @@
+! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acosf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f32
+! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acos({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f64) -> f64
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f64
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f64


From flang-commits at lists.llvm.org  Wed Jul 23 08:06:43 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:06:43 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150234@github.com>
Message-ID: <6880fa83.170a0220.3844cc.978b@mx.google.com>


c8ef wrote:

* **#150234** <a href="https://app.graphite.dev/github/pr/llvm/llvm-project/150234?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/llvm/llvm-project/150234?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->

https://github.com/llvm/llvm-project/pull/150234

From flang-commits at lists.llvm.org  Wed Jul 23 08:08:09 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:08:09 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150234@github.com>
Message-ID: <6880fad9.630a0220.2d452b.6e7c@mx.google.com>


https://github.com/c8ef ready_for_review https://github.com/llvm/llvm-project/pull/150234

From flang-commits at lists.llvm.org  Wed Jul 23 08:08:42 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 08:08:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150234@github.com>
Message-ID: <6880fafa.170a0220.3d09fe.9ca0@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Connector Switch (c8ef)

<details>
<summary>Changes</summary>



---
Full diff: https://github.com/llvm/llvm-project/pull/150234.diff


4 Files Affected:

- (modified) flang/include/flang/Optimizer/Builder/IntrinsicCall.h (+1) 
- (modified) flang/lib/Evaluate/intrinsics.cpp (+1) 
- (modified) flang/lib/Optimizer/Builder/IntrinsicCall.cpp (+30-15) 
- (added) flang/test/Lower/Intrinsics/acospi.f90 (+26) 


``````````diff
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index ab08415fe32c8..a93b397240576 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -174,6 +174,7 @@ struct IntrinsicLibrary {
   /// real and to the `hypot` math routine if the argument is of complex type.
   mlir::Value genAbs(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAcosd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAcospi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                                 mlir::Value, mlir::Value)>
   fir::ExtendedValue genAdjustRtCall(mlir::Type,
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9c6d49dfda3e8..97e87935c02b0 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -340,6 +340,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"acos", {{"x", SameFloating}}, SameFloating},
     {"acosd", {{"x", SameFloating}}, SameFloating},
     {"acosh", {{"x", SameFloating}}, SameFloating},
+    {"acospi", {{"x", SameFloating}}, SameFloating},
     {"adjustl", {{"string", SameChar}}, SameChar},
     {"adjustr", {{"string", SameChar}}, SameChar},
     {"aimag", {{"z", SameComplex}}, SameReal},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index ba298298be675..69e0959249acc 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -246,6 +246,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"abs", &I::genAbs},
     {"achar", &I::genChar},
     {"acosd", &I::genAcosd},
+    {"acospi", &I::genAcospi},
     {"adjustl",
      &I::genAdjustRtCall<fir::runtime::genAdjustL>,
      {{{"string", asAddr}}},
@@ -2676,6 +2677,21 @@ mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
+// ACOSPI
+mlir::Value IntrinsicLibrary::genAcospi(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value acos = getRuntimeCallGenerator("acos", ftype)(builder, loc, args);
+  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
+  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  return mlir::arith::MulFOp::create(builder, loc, acos, factor);
+}
+
 // ADJUSTL & ADJUSTR
 template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                               mlir::Value, mlir::Value)>
@@ -3981,21 +3997,20 @@ void IntrinsicLibrary::genExecuteCommandLine(
     mlir::Value waitAddr = fir::getBase(wait);
     mlir::Value waitIsPresentAtRuntime =
         builder.genIsNotNullAddr(loc, waitAddr);
-    waitBool = builder
-                   .genIfOp(loc, {i1Ty}, waitIsPresentAtRuntime,
-                            /*withElseRegion=*/true)
-                   .genThen([&]() {
-                     auto waitLoad =
-                         fir::LoadOp::create(builder, loc, waitAddr);
-                     mlir::Value cast =
-                         builder.createConvert(loc, i1Ty, waitLoad);
-                     fir::ResultOp::create(builder, loc, cast);
-                   })
-                   .genElse([&]() {
-                     mlir::Value trueVal = builder.createBool(loc, true);
-                     fir::ResultOp::create(builder, loc, trueVal);
-                   })
-                   .getResults()[0];
+    waitBool =
+        builder
+            .genIfOp(loc, {i1Ty}, waitIsPresentAtRuntime,
+                     /*withElseRegion=*/true)
+            .genThen([&]() {
+              auto waitLoad = fir::LoadOp::create(builder, loc, waitAddr);
+              mlir::Value cast = builder.createConvert(loc, i1Ty, waitLoad);
+              fir::ResultOp::create(builder, loc, cast);
+            })
+            .genElse([&]() {
+              mlir::Value trueVal = builder.createBool(loc, true);
+              fir::ResultOp::create(builder, loc, trueVal);
+            })
+            .getResults()[0];
   }
 
   mlir::Value exitstatBox =
diff --git a/flang/test/Lower/Intrinsics/acospi.f90 b/flang/test/Lower/Intrinsics/acospi.f90
new file mode 100644
index 0000000000000..dcacd25bca480
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/acospi.f90
@@ -0,0 +1,26 @@
+! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acosf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f32
+! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acos({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f64) -> f64
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f64
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f64

``````````

</details>


https://github.com/llvm/llvm-project/pull/150234

From flang-commits at lists.llvm.org  Wed Jul 23 08:09:54 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:09:54 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150234@github.com>
Message-ID: <6880fb42.170a0220.333b71.b5fc@mx.google.com>


https://github.com/c8ef updated https://github.com/llvm/llvm-project/pull/150234

>From ee201af5d40892e122038a215f8bd72ef5d71304 Mon Sep 17 00:00:00 2001
From: c8ef <c8ef at outlook.com>
Date: Wed, 23 Jul 2025 15:05:18 +0000
Subject: [PATCH] [flang] Implement `acospi`

---
 .../flang/Optimizer/Builder/IntrinsicCall.h   |  1 +
 flang/lib/Evaluate/intrinsics.cpp             |  1 +
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 16 ++++++++++++
 flang/test/Lower/Intrinsics/acospi.f90        | 26 +++++++++++++++++++
 4 files changed, 44 insertions(+)
 create mode 100644 flang/test/Lower/Intrinsics/acospi.f90

diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index ab08415fe32c8..a93b397240576 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -174,6 +174,7 @@ struct IntrinsicLibrary {
   /// real and to the `hypot` math routine if the argument is of complex type.
   mlir::Value genAbs(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAcosd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAcospi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                                 mlir::Value, mlir::Value)>
   fir::ExtendedValue genAdjustRtCall(mlir::Type,
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9c6d49dfda3e8..97e87935c02b0 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -340,6 +340,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"acos", {{"x", SameFloating}}, SameFloating},
     {"acosd", {{"x", SameFloating}}, SameFloating},
     {"acosh", {{"x", SameFloating}}, SameFloating},
+    {"acospi", {{"x", SameFloating}}, SameFloating},
     {"adjustl", {{"string", SameChar}}, SameChar},
     {"adjustr", {{"string", SameChar}}, SameChar},
     {"aimag", {{"z", SameComplex}}, SameReal},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index ba298298be675..f3af423b24685 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -246,6 +246,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"abs", &I::genAbs},
     {"achar", &I::genChar},
     {"acosd", &I::genAcosd},
+    {"acospi", &I::genAcospi},
     {"adjustl",
      &I::genAdjustRtCall<fir::runtime::genAdjustL>,
      {{{"string", asAddr}}},
@@ -2676,6 +2677,21 @@ mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
+// ACOSPI
+mlir::Value IntrinsicLibrary::genAcospi(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value acos = getRuntimeCallGenerator("acos", ftype)(builder, loc, args);
+  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
+  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  return mlir::arith::MulFOp::create(builder, loc, acos, factor);
+}
+
 // ADJUSTL & ADJUSTR
 template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                               mlir::Value, mlir::Value)>
diff --git a/flang/test/Lower/Intrinsics/acospi.f90 b/flang/test/Lower/Intrinsics/acospi.f90
new file mode 100644
index 0000000000000..dcacd25bca480
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/acospi.f90
@@ -0,0 +1,26 @@
+! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acosf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f32
+! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acos({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f64) -> f64
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f64
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f64


From flang-commits at lists.llvm.org  Wed Jul 23 08:23:42 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 08:23:42 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150234@github.com>
Message-ID: <6880fe7e.050a0220.3c7e08.b237@mx.google.com>


https://github.com/tblah approved this pull request.

Thanks

https://github.com/llvm/llvm-project/pull/150234

From flang-commits at lists.llvm.org  Wed Jul 23 08:33:38 2025
From: flang-commits at lists.llvm.org (Kazu Hirata via flang-commits)
Date: Wed, 23 Jul 2025 08:33:38 -0700 (PDT)
Subject: [flang-commits] [flang] 97faab7 - [flang] Fix a warning
Message-ID: <688100d2.170a0220.3a4c7.a713@mx.google.com>


Author: Kazu Hirata
Date: 2025-07-23T08:33:32-07:00
New Revision: 97faab7bc279516a31001621203f4ff5a158ed13

URL: https://github.com/llvm/llvm-project/commit/97faab7bc279516a31001621203f4ff5a158ed13
DIFF: https://github.com/llvm/llvm-project/commit/97faab7bc279516a31001621203f4ff5a158ed13.diff

LOG: [flang] Fix a warning

This patch fixes:

  flang/lib/Lower/Bridge.cpp:2128:10: error: unused variable 'result'
  [-Werror,-Wunused-variable]

Added: 
    

Modified: 
    flang/lib/Lower/Bridge.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 37f2475763f08..92aae792248c5 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2129,6 +2129,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         toLocation(), *this, info.reduceOperatorList, reduceVars,
         reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
     assert(result && "Failed to process `do concurrent` reductions");
+    (void)result;
 
     doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
     doConcurrentLoopOp.setReduceSymsAttr(


        

From flang-commits at lists.llvm.org  Wed Jul 23 08:06:22 2025
From: flang-commits at lists.llvm.org (Tom Eccles via flang-commits)
Date: Wed, 23 Jul 2025 08:06:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Lower] Make reduction processing
 failure a hard error (PR #150233)
Message-ID: <llvm.org/llvm/llvm-project/pull/150233@github.com>


https://github.com/tblah created https://github.com/llvm/llvm-project/pull/150233

See #150178

This may regress some test cases which only ever passed by accident.

I've tested SPEC2017 and a sample of applications to check that this doesn't break anything too obvious. Presumably this was not a widely used feature or we would have noticed the bug sooner.

I'm unsure whether this should be backported to LLVM 21 or not: I think it is much better to refuse to compile than to silently produce the wrong result, but there is a chance this could regress something which previously worked by accident. Opinions welcome.

>From fc1b93bf0ad04ec4090019bca7ccdf04b5383b47 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles at arm.com>
Date: Wed, 23 Jul 2025 10:42:04 +0000
Subject: [PATCH] [flang][Lower] Make reduction processing failure a hard error

See #150178

This may regress some test cases which only ever passed by accident.

I've tested SPEC2017 and a sample of applications to check that this
doesn't break anything too obvious. Presumably this was not a widely
used feature or we would have noticed the bug sooner.

I'm unsure whether this should be backported to LLVM 21 or not: I think
it is much better to refuse to compile than to silently produce the
wrong result, but there is a chance this could regress something which
previously worked by accident. Opinions welcome.
---
 flang/lib/Lower/Bridge.cpp                               | 3 ++-
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp               | 6 +++---
 .../OpenMP/{ => Todo}/wsloop-reduction-non-intrinsic.f90 | 9 ++-------
 3 files changed, 7 insertions(+), 11 deletions(-)
 rename flang/test/Lower/OpenMP/{ => Todo}/wsloop-reduction-non-intrinsic.f90 (52%)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 13420f365919d..dc1e131b64126 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2128,7 +2128,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     bool result = rp.processReductionArguments<fir::DeclareReductionOp>(
         toLocation(), *this, info.reduceOperatorList, reduceVars,
         reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
-    assert(result && "Failed to process `do concurrent` reductions");
+    if (!result)
+      TODO(toLocation(), "Lowering unrecognised reduction type");
 
     doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
     doConcurrentLoopOp.setReduceSymsAttr(
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 594f95ecdda63..b83ea6f175031 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1121,7 +1121,7 @@ bool ClauseProcessor::processInReduction(
                 std::get<typename omp::clause::ReductionOperatorList>(clause.t),
                 inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
                 inReductionSyms))
-          inReductionSyms.clear();
+          TODO(currentLocation, "Lowering unrecognised reduction type");
 
         // Copy local lists into the output.
         llvm::copy(inReductionVars, std::back_inserter(result.inReductionVars));
@@ -1467,7 +1467,7 @@ bool ClauseProcessor::processReduction(
                 std::get<typename omp::clause::ReductionOperatorList>(clause.t),
                 reductionVars, reduceVarByRef, reductionDeclSymbols,
                 reductionSyms))
-          reductionSyms.clear();
+          TODO(currentLocation, "Lowering unrecognised reduction type");
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
@@ -1494,7 +1494,7 @@ bool ClauseProcessor::processTaskReduction(
                 std::get<typename omp::clause::ReductionOperatorList>(clause.t),
                 taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
                 taskReductionSyms))
-          taskReductionSyms.clear();
+          TODO(currentLocation, "Lowering unrecognised reduction type");
         // Copy local lists into the output.
         llvm::copy(taskReductionVars,
                    std::back_inserter(result.taskReductionVars));
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90 b/flang/test/Lower/OpenMP/Todo/wsloop-reduction-non-intrinsic.f90
similarity index 52%
rename from flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
rename to flang/test/Lower/OpenMP/Todo/wsloop-reduction-non-intrinsic.f90
index 70b2ae1111a50..84feb5aeeee7c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
+++ b/flang/test/Lower/OpenMP/Todo/wsloop-reduction-non-intrinsic.f90
@@ -1,6 +1,6 @@
 ! Tests reduction processor behavior when a reduction symbol is not supported.
 
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 
 subroutine foo
   implicit none
@@ -12,14 +12,9 @@ function max(m1,m2)
     end function
   end interface
 
+  !CHECK: not yet implemented: Lowering unrecognised reduction type
   !$omp do reduction (max: k)
   do i=1,10
   end do
   !$omp end do
 end
-
-! Verify that unsupported reduction is ignored.
-! CHECK: omp.wsloop 
-! CHECK-SAME: private(@{{[^[:space:]]+}} %{{[^[:space:]]+}}
-! CHECK-SAME:         -> %{{[^[:space:]]+}} : !{{[^[:space:]]+}}) {
-! CHECK: }


From flang-commits at lists.llvm.org  Wed Jul 23 08:07:00 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 08:07:00 -0700 (PDT)
Subject: [flang-commits] [flang] [flang][Lower] Make reduction processing
 failure a hard error (PR #150233)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150233@github.com>
Message-ID: <6880fa94.a70a0220.1ac31.a43a@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: Tom Eccles (tblah)

<details>
<summary>Changes</summary>

See #<!-- -->150178

This may regress some test cases which only ever passed by accident.

I've tested SPEC2017 and a sample of applications to check that this doesn't break anything too obvious. Presumably this was not a widely used feature or we would have noticed the bug sooner.

I'm unsure whether this should be backported to LLVM 21 or not: I think it is much better to refuse to compile than to silently produce the wrong result, but there is a chance this could regress something which previously worked by accident. Opinions welcome.

---
Full diff: https://github.com/llvm/llvm-project/pull/150233.diff


3 Files Affected:

- (modified) flang/lib/Lower/Bridge.cpp (+2-1) 
- (modified) flang/lib/Lower/OpenMP/ClauseProcessor.cpp (+3-3) 
- (renamed) flang/test/Lower/OpenMP/Todo/wsloop-reduction-non-intrinsic.f90 (+2-7) 


``````````diff
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 13420f365919d..dc1e131b64126 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2128,7 +2128,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     bool result = rp.processReductionArguments<fir::DeclareReductionOp>(
         toLocation(), *this, info.reduceOperatorList, reduceVars,
         reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
-    assert(result && "Failed to process `do concurrent` reductions");
+    if (!result)
+      TODO(toLocation(), "Lowering unrecognised reduction type");
 
     doConcurrentLoopOp.getReduceVarsMutable().assign(reduceVars);
     doConcurrentLoopOp.setReduceSymsAttr(
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 594f95ecdda63..b83ea6f175031 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1121,7 +1121,7 @@ bool ClauseProcessor::processInReduction(
                 std::get<typename omp::clause::ReductionOperatorList>(clause.t),
                 inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
                 inReductionSyms))
-          inReductionSyms.clear();
+          TODO(currentLocation, "Lowering unrecognised reduction type");
 
         // Copy local lists into the output.
         llvm::copy(inReductionVars, std::back_inserter(result.inReductionVars));
@@ -1467,7 +1467,7 @@ bool ClauseProcessor::processReduction(
                 std::get<typename omp::clause::ReductionOperatorList>(clause.t),
                 reductionVars, reduceVarByRef, reductionDeclSymbols,
                 reductionSyms))
-          reductionSyms.clear();
+          TODO(currentLocation, "Lowering unrecognised reduction type");
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reduceVarByRef, std::back_inserter(result.reductionByref));
@@ -1494,7 +1494,7 @@ bool ClauseProcessor::processTaskReduction(
                 std::get<typename omp::clause::ReductionOperatorList>(clause.t),
                 taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
                 taskReductionSyms))
-          taskReductionSyms.clear();
+          TODO(currentLocation, "Lowering unrecognised reduction type");
         // Copy local lists into the output.
         llvm::copy(taskReductionVars,
                    std::back_inserter(result.taskReductionVars));
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90 b/flang/test/Lower/OpenMP/Todo/wsloop-reduction-non-intrinsic.f90
similarity index 52%
rename from flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
rename to flang/test/Lower/OpenMP/Todo/wsloop-reduction-non-intrinsic.f90
index 70b2ae1111a50..84feb5aeeee7c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-non-intrinsic.f90
+++ b/flang/test/Lower/OpenMP/Todo/wsloop-reduction-non-intrinsic.f90
@@ -1,6 +1,6 @@
 ! Tests reduction processor behavior when a reduction symbol is not supported.
 
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 
 subroutine foo
   implicit none
@@ -12,14 +12,9 @@ function max(m1,m2)
     end function
   end interface
 
+  !CHECK: not yet implemented: Lowering unrecognised reduction type
   !$omp do reduction (max: k)
   do i=1,10
   end do
   !$omp end do
 end
-
-! Verify that unsupported reduction is ignored.
-! CHECK: omp.wsloop 
-! CHECK-SAME: private(@{{[^[:space:]]+}} %{{[^[:space:]]+}}
-! CHECK-SAME:         -> %{{[^[:space:]]+}} : !{{[^[:space:]]+}}) {
-! CHECK: }

``````````

</details>


https://github.com/llvm/llvm-project/pull/150233

From flang-commits at lists.llvm.org  Wed Jul 23 08:52:35 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:52:35 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150234@github.com>
Message-ID: <68810543.170a0220.35389b.a64e@mx.google.com>


c8ef wrote:

### Merge activity

* **Jul 23, 3:52 PM UTC**: A user started a stack merge that includes this pull request via [Graphite](https://app.graphite.dev/github/pr/llvm/llvm-project/150234).


https://github.com/llvm/llvm-project/pull/150234

From flang-commits at lists.llvm.org  Wed Jul 23 08:54:08 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:54:08 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150234@github.com>
Message-ID: <688105a0.170a0220.157c1.d91b@mx.google.com>


https://github.com/c8ef updated https://github.com/llvm/llvm-project/pull/150234

>From 9a58d28eb8c865ff722ea860170b54cc16d74f39 Mon Sep 17 00:00:00 2001
From: c8ef <c8ef at outlook.com>
Date: Wed, 23 Jul 2025 15:05:18 +0000
Subject: [PATCH] [flang] Implement `acospi`

---
 .../flang/Optimizer/Builder/IntrinsicCall.h   |  1 +
 flang/lib/Evaluate/intrinsics.cpp             |  1 +
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 16 ++++++++++++
 flang/test/Lower/Intrinsics/acospi.f90        | 26 +++++++++++++++++++
 4 files changed, 44 insertions(+)
 create mode 100644 flang/test/Lower/Intrinsics/acospi.f90

diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index ab08415fe32c8..a93b397240576 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -174,6 +174,7 @@ struct IntrinsicLibrary {
   /// real and to the `hypot` math routine if the argument is of complex type.
   mlir::Value genAbs(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAcosd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAcospi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                                 mlir::Value, mlir::Value)>
   fir::ExtendedValue genAdjustRtCall(mlir::Type,
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9c6d49dfda3e8..97e87935c02b0 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -340,6 +340,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"acos", {{"x", SameFloating}}, SameFloating},
     {"acosd", {{"x", SameFloating}}, SameFloating},
     {"acosh", {{"x", SameFloating}}, SameFloating},
+    {"acospi", {{"x", SameFloating}}, SameFloating},
     {"adjustl", {{"string", SameChar}}, SameChar},
     {"adjustr", {{"string", SameChar}}, SameChar},
     {"aimag", {{"z", SameComplex}}, SameReal},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index ba298298be675..f3af423b24685 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -246,6 +246,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"abs", &I::genAbs},
     {"achar", &I::genChar},
     {"acosd", &I::genAcosd},
+    {"acospi", &I::genAcospi},
     {"adjustl",
      &I::genAdjustRtCall<fir::runtime::genAdjustL>,
      {{{"string", asAddr}}},
@@ -2676,6 +2677,21 @@ mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
+// ACOSPI
+mlir::Value IntrinsicLibrary::genAcospi(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value acos = getRuntimeCallGenerator("acos", ftype)(builder, loc, args);
+  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
+  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  return mlir::arith::MulFOp::create(builder, loc, acos, factor);
+}
+
 // ADJUSTL & ADJUSTR
 template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                               mlir::Value, mlir::Value)>
diff --git a/flang/test/Lower/Intrinsics/acospi.f90 b/flang/test/Lower/Intrinsics/acospi.f90
new file mode 100644
index 0000000000000..dcacd25bca480
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/acospi.f90
@@ -0,0 +1,26 @@
+! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acosf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f32
+! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acos({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f64) -> f64
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f64
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f64


From flang-commits at lists.llvm.org  Wed Jul 23 08:56:13 2025
From: flang-commits at lists.llvm.org (Tom Stellard via flang-commits)
Date: Wed, 23 Jul 2025 08:56:13 -0700 (PDT)
Subject: [flang-commits] [flang] [Flang] Assign unittests to flang_compile
	pool (PR #150041)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150041@github.com>
Message-ID: <6881061d.170a0220.122c7f.ac4b@mx.google.com>


https://github.com/tstellar approved this pull request.


https://github.com/llvm/llvm-project/pull/150041

From flang-commits at lists.llvm.org  Wed Jul 23 08:57:10 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 08:57:10 -0700 (PDT)
Subject: [flang-commits] [flang] 4471d59 - [flang] Implement `acospi`
	(#150234)
Message-ID: <68810656.170a0220.1a3139.ad89@mx.google.com>


Author: Connector Switch
Date: 2025-07-23T23:57:06+08:00
New Revision: 4471d59a10f70b502b036f57f4728f2411442fc4

URL: https://github.com/llvm/llvm-project/commit/4471d59a10f70b502b036f57f4728f2411442fc4
DIFF: https://github.com/llvm/llvm-project/commit/4471d59a10f70b502b036f57f4728f2411442fc4.diff

LOG: [flang] Implement `acospi` (#150234)

Added: 
    flang/test/Lower/Intrinsics/acospi.f90

Modified: 
    flang/include/flang/Optimizer/Builder/IntrinsicCall.h
    flang/lib/Evaluate/intrinsics.cpp
    flang/lib/Optimizer/Builder/IntrinsicCall.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index ab08415fe32c8..a93b397240576 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -174,6 +174,7 @@ struct IntrinsicLibrary {
   /// real and to the `hypot` math routine if the argument is of complex type.
   mlir::Value genAbs(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAcosd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAcospi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                                 mlir::Value, mlir::Value)>
   fir::ExtendedValue genAdjustRtCall(mlir::Type,

diff  --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 9c6d49dfda3e8..97e87935c02b0 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -340,6 +340,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"acos", {{"x", SameFloating}}, SameFloating},
     {"acosd", {{"x", SameFloating}}, SameFloating},
     {"acosh", {{"x", SameFloating}}, SameFloating},
+    {"acospi", {{"x", SameFloating}}, SameFloating},
     {"adjustl", {{"string", SameChar}}, SameChar},
     {"adjustr", {{"string", SameChar}}, SameChar},
     {"aimag", {{"z", SameComplex}}, SameReal},

diff  --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index ba298298be675..f3af423b24685 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -246,6 +246,7 @@ static constexpr IntrinsicHandler handlers[]{
     {"abs", &I::genAbs},
     {"achar", &I::genChar},
     {"acosd", &I::genAcosd},
+    {"acospi", &I::genAcospi},
     {"adjustl",
      &I::genAdjustRtCall<fir::runtime::genAdjustL>,
      {{{"string", asAddr}}},
@@ -2676,6 +2677,21 @@ mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
+// ACOSPI
+mlir::Value IntrinsicLibrary::genAcospi(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value acos = getRuntimeCallGenerator("acos", ftype)(builder, loc, args);
+  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
+  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  return mlir::arith::MulFOp::create(builder, loc, acos, factor);
+}
+
 // ADJUSTL & ADJUSTR
 template <void (*CallRuntime)(fir::FirOpBuilder &, mlir::Location loc,
                               mlir::Value, mlir::Value)>

diff  --git a/flang/test/Lower/Intrinsics/acospi.f90 b/flang/test/Lower/Intrinsics/acospi.f90
new file mode 100644
index 0000000000000..dcacd25bca480
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/acospi.f90
@@ -0,0 +1,26 @@
+! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acosf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f32
+! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK-PRECISE: %[[acos:.*]] = fir.call @acos({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f64) -> f64
+! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f64
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f64


        

From flang-commits at lists.llvm.org  Wed Jul 23 08:57:13 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:57:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `acospi` (PR #150234)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150234@github.com>
Message-ID: <68810659.630a0220.81f5.99c7@mx.google.com>


https://github.com/c8ef closed https://github.com/llvm/llvm-project/pull/150234

From flang-commits at lists.llvm.org  Wed Jul 23 08:57:15 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:57:15 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `asinpi` (PR #150238)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150238@github.com>
Message-ID: <6881065b.050a0220.2be3ad.b82f@mx.google.com>


https://github.com/c8ef edited https://github.com/llvm/llvm-project/pull/150238

From flang-commits at lists.llvm.org  Wed Jul 23 08:57:24 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 08:57:24 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `asinpi` (PR #150238)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150238@github.com>
Message-ID: <68810664.630a0220.1a15fb.76cc@mx.google.com>


https://github.com/c8ef updated https://github.com/llvm/llvm-project/pull/150238

>From 748ced1628c646abc49593daaad650025d5de64a Mon Sep 17 00:00:00 2001
From: c8ef <c8ef at outlook.com>
Date: Wed, 23 Jul 2025 15:20:37 +0000
Subject: [PATCH] [flang] Implement `asinpi`

---
 .../flang/Optimizer/Builder/IntrinsicCall.h   |  1 +
 flang/lib/Evaluate/intrinsics.cpp             |  1 +
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 16 ++++++++++++
 flang/test/Lower/Intrinsics/asinpi.f90        | 26 +++++++++++++++++++
 4 files changed, 44 insertions(+)
 create mode 100644 flang/test/Lower/Intrinsics/asinpi.f90

diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index a93b397240576..2afd50410ae82 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -204,6 +204,7 @@ struct IntrinsicLibrary {
   fir::ExtendedValue
       genCommandArgumentCount(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAsind(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAsinpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genAssociated(mlir::Type,
                                    llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAtand(mlir::Type, llvm::ArrayRef<mlir::Value>);
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 97e87935c02b0..768e4bafa90ee 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -359,6 +359,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"asin", {{"x", SameFloating}}, SameFloating},
     {"asind", {{"x", SameFloating}}, SameFloating},
     {"asinh", {{"x", SameFloating}}, SameFloating},
+    {"asinpi", {{"x", SameFloating}}, SameFloating},
     {"associated",
         {{"pointer", AnyPointer, Rank::anyOrAssumedRank, Optionality::required,
              common::Intent::In, {ArgFlag::canBeNullPointer}},
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index f3af423b24685..4753d0add6787 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -279,6 +279,7 @@ static constexpr IntrinsicHandler handlers[]{
      {{{"mask", asValue}, {"pred", asValue}}},
      /*isElemental=*/false},
     {"asind", &I::genAsind},
+    {"asinpi", &I::genAsinpi},
     {"associated",
      &I::genAssociated,
      {{{"pointer", asInquired}, {"target", asInquired}}},
@@ -2845,6 +2846,21 @@ mlir::Value IntrinsicLibrary::genAsind(mlir::Type resultType,
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
+// ASINPI
+mlir::Value IntrinsicLibrary::genAsinpi(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value asin = getRuntimeCallGenerator("asin", ftype)(builder, loc, args);
+  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
+  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  return mlir::arith::MulFOp::create(builder, loc, asin, factor);
+}
+
 // ATAND, ATAN2D
 mlir::Value IntrinsicLibrary::genAtand(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {
diff --git a/flang/test/Lower/Intrinsics/asinpi.f90 b/flang/test/Lower/Intrinsics/asinpi.f90
new file mode 100644
index 0000000000000..1c1838c56ca27
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/asinpi.f90
@@ -0,0 +1,26 @@
+! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = asinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK-PRECISE: %[[asin:.*]] = fir.call @asinf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
+! CHECK-FAST: %[[asin:.*]] = math.asin %{{.*}} : f32
+! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %{{.*}} = arith.mulf %[[asin]], %[[inv_pi]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = asinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK-PRECISE: %[[asin:.*]] = fir.call @asin({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f64) -> f64
+! CHECK-FAST: %[[asin:.*]] = math.asin %{{.*}} : f64
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %{{.*}} = arith.mulf %[[asin]], %[[inv_pi]] fastmath<contract> : f64


From flang-commits at lists.llvm.org  Wed Jul 23 09:00:10 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 09:00:10 -0700 (PDT)
Subject: [flang-commits] [flang] 01b23c8 - [flang] Implement `asinpi`
	(#150238)
Message-ID: <6881070a.170a0220.e7685.da73@mx.google.com>


Author: Connector Switch
Date: 2025-07-24T00:00:06+08:00
New Revision: 01b23c8d81662ed8383df78c2de0ea100d92d503

URL: https://github.com/llvm/llvm-project/commit/01b23c8d81662ed8383df78c2de0ea100d92d503
DIFF: https://github.com/llvm/llvm-project/commit/01b23c8d81662ed8383df78c2de0ea100d92d503.diff

LOG: [flang] Implement `asinpi` (#150238)

Added: 
    flang/test/Lower/Intrinsics/asinpi.f90

Modified: 
    flang/include/flang/Optimizer/Builder/IntrinsicCall.h
    flang/lib/Evaluate/intrinsics.cpp
    flang/lib/Optimizer/Builder/IntrinsicCall.cpp

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index a93b397240576..2afd50410ae82 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -204,6 +204,7 @@ struct IntrinsicLibrary {
   fir::ExtendedValue
       genCommandArgumentCount(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAsind(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAsinpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genAssociated(mlir::Type,
                                    llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAtand(mlir::Type, llvm::ArrayRef<mlir::Value>);

diff  --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 97e87935c02b0..768e4bafa90ee 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -359,6 +359,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"asin", {{"x", SameFloating}}, SameFloating},
     {"asind", {{"x", SameFloating}}, SameFloating},
     {"asinh", {{"x", SameFloating}}, SameFloating},
+    {"asinpi", {{"x", SameFloating}}, SameFloating},
     {"associated",
         {{"pointer", AnyPointer, Rank::anyOrAssumedRank, Optionality::required,
              common::Intent::In, {ArgFlag::canBeNullPointer}},

diff  --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index f3af423b24685..4753d0add6787 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -279,6 +279,7 @@ static constexpr IntrinsicHandler handlers[]{
      {{{"mask", asValue}, {"pred", asValue}}},
      /*isElemental=*/false},
     {"asind", &I::genAsind},
+    {"asinpi", &I::genAsinpi},
     {"associated",
      &I::genAssociated,
      {{{"pointer", asInquired}, {"target", asInquired}}},
@@ -2845,6 +2846,21 @@ mlir::Value IntrinsicLibrary::genAsind(mlir::Type resultType,
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
+// ASINPI
+mlir::Value IntrinsicLibrary::genAsinpi(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  mlir::MLIRContext *context = builder.getContext();
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value asin = getRuntimeCallGenerator("asin", ftype)(builder, loc, args);
+  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
+  mlir::Value dfactor =
+      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
+  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  return mlir::arith::MulFOp::create(builder, loc, asin, factor);
+}
+
 // ATAND, ATAN2D
 mlir::Value IntrinsicLibrary::genAtand(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {

diff  --git a/flang/test/Lower/Intrinsics/asinpi.f90 b/flang/test/Lower/Intrinsics/asinpi.f90
new file mode 100644
index 0000000000000..1c1838c56ca27
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/asinpi.f90
@@ -0,0 +1,26 @@
+! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
+
+function test_real4(x)
+  real :: x, test_real4
+  test_real4 = asinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK-PRECISE: %[[asin:.*]] = fir.call @asinf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
+! CHECK-FAST: %[[asin:.*]] = math.asin %{{.*}} : f32
+! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %{{.*}} = arith.mulf %[[asin]], %[[inv_pi]] fastmath<contract> : f32
+
+function test_real8(x)
+  real(8) :: x, test_real8
+  test_real8 = asinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK-PRECISE: %[[asin:.*]] = fir.call @asin({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f64) -> f64
+! CHECK-FAST: %[[asin:.*]] = math.asin %{{.*}} : f64
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
+! CHECK: %{{.*}} = arith.mulf %[[asin]], %[[inv_pi]] fastmath<contract> : f64


        

From flang-commits at lists.llvm.org  Wed Jul 23 09:00:13 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 09:00:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Implement `asinpi` (PR #150238)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150238@github.com>
Message-ID: <6881070d.630a0220.3da9d1.ea79@mx.google.com>


https://github.com/c8ef closed https://github.com/llvm/llvm-project/pull/150238

From flang-commits at lists.llvm.org  Wed Jul 23 09:24:14 2025
From: flang-commits at lists.llvm.org (Peter Klausler via flang-commits)
Date: Wed, 23 Jul 2025 09:24:14 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch both EXTERNAL and INTRINSIC
 on type declaration stmt (PR #150254)
Message-ID: <llvm.org/llvm/llvm-project/pull/150254@github.com>


https://github.com/klausler created https://github.com/llvm/llvm-project/pull/150254

The EXTERNAL and INTRINSIC attributes can't both appear on the same type declarations statement.

Fixes https://github.com/llvm/llvm-project/issues/149771.

>From ad964cc6dad0b8e407014384db91134278413af0 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Wed, 23 Jul 2025 09:21:18 -0700
Subject: [PATCH] [flang] Catch both EXTERNAL and INTRINSIC on type declaration
 stmt

The EXTERNAL and INTRINSIC attributes can't both appear on the same
type declarations statement.

Fixes https://github.com/llvm/llvm-project/issues/149771.
---
 flang/lib/Semantics/resolve-names.cpp | 3 ++-
 flang/test/Semantics/resolve126.f90   | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Semantics/resolve126.f90

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index b3268605e7c0c..d08c669377cb2 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -2351,7 +2351,8 @@ bool AttrsVisitor::IsConflictingAttr(Attr attrName) {
       HaveAttrConflict(attrName, Attr::PASS, Attr::NOPASS) || // C781
       HaveAttrConflict(attrName, Attr::PURE, Attr::IMPURE) ||
       HaveAttrConflict(attrName, Attr::PUBLIC, Attr::PRIVATE) ||
-      HaveAttrConflict(attrName, Attr::RECURSIVE, Attr::NON_RECURSIVE);
+      HaveAttrConflict(attrName, Attr::RECURSIVE, Attr::NON_RECURSIVE) ||
+      HaveAttrConflict(attrName, Attr::INTRINSIC, Attr::EXTERNAL);
 }
 bool AttrsVisitor::CheckAndSet(Attr attrName) {
   if (IsConflictingAttr(attrName) || IsDuplicateAttr(attrName)) {
diff --git a/flang/test/Semantics/resolve126.f90 b/flang/test/Semantics/resolve126.f90
new file mode 100644
index 0000000000000..6313f8c539bf8
--- /dev/null
+++ b/flang/test/Semantics/resolve126.f90
@@ -0,0 +1,7 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+!ERROR: Attributes 'INTRINSIC' and 'EXTERNAL' conflict with each other
+real, external, intrinsic :: exp
+!ERROR: Symbol 'sin' cannot have both EXTERNAL and INTRINSIC attributes
+external sin
+intrinsic sin
+end


From flang-commits at lists.llvm.org  Wed Jul 23 09:24:47 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 09:24:47 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch both EXTERNAL and INTRINSIC
 on type declaration stmt (PR #150254)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150254@github.com>
Message-ID: <68810ccf.630a0220.397d09.7fcc@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-semantics

Author: Peter Klausler (klausler)

<details>
<summary>Changes</summary>

The EXTERNAL and INTRINSIC attributes can't both appear on the same type declarations statement.

Fixes https://github.com/llvm/llvm-project/issues/149771.

---
Full diff: https://github.com/llvm/llvm-project/pull/150254.diff


2 Files Affected:

- (modified) flang/lib/Semantics/resolve-names.cpp (+2-1) 
- (added) flang/test/Semantics/resolve126.f90 (+7) 


``````````diff
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index b3268605e7c0c..d08c669377cb2 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -2351,7 +2351,8 @@ bool AttrsVisitor::IsConflictingAttr(Attr attrName) {
       HaveAttrConflict(attrName, Attr::PASS, Attr::NOPASS) || // C781
       HaveAttrConflict(attrName, Attr::PURE, Attr::IMPURE) ||
       HaveAttrConflict(attrName, Attr::PUBLIC, Attr::PRIVATE) ||
-      HaveAttrConflict(attrName, Attr::RECURSIVE, Attr::NON_RECURSIVE);
+      HaveAttrConflict(attrName, Attr::RECURSIVE, Attr::NON_RECURSIVE) ||
+      HaveAttrConflict(attrName, Attr::INTRINSIC, Attr::EXTERNAL);
 }
 bool AttrsVisitor::CheckAndSet(Attr attrName) {
   if (IsConflictingAttr(attrName) || IsDuplicateAttr(attrName)) {
diff --git a/flang/test/Semantics/resolve126.f90 b/flang/test/Semantics/resolve126.f90
new file mode 100644
index 0000000000000..6313f8c539bf8
--- /dev/null
+++ b/flang/test/Semantics/resolve126.f90
@@ -0,0 +1,7 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+!ERROR: Attributes 'INTRINSIC' and 'EXTERNAL' conflict with each other
+real, external, intrinsic :: exp
+!ERROR: Symbol 'sin' cannot have both EXTERNAL and INTRINSIC attributes
+external sin
+intrinsic sin
+end

``````````

</details>


https://github.com/llvm/llvm-project/pull/150254

From flang-commits at lists.llvm.org  Wed Jul 23 09:31:13 2025
From: flang-commits at lists.llvm.org (Andre Kuhlenschmidt via flang-commits)
Date: Wed, 23 Jul 2025 09:31:13 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch both EXTERNAL and INTRINSIC
 on type declaration stmt (PR #150254)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150254@github.com>
Message-ID: <68810e51.170a0220.ca055.b44f@mx.google.com>


https://github.com/akuhlens approved this pull request.


https://github.com/llvm/llvm-project/pull/150254

From flang-commits at lists.llvm.org  Wed Jul 23 10:14:07 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 10:14:07 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150269@github.com>
Message-ID: <6881185f.050a0220.18a8fa.c05f@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang

@llvm/pr-subscribers-clang-driver

Author: Kajetan Puchalski (mrkajetanp)

<details>
<summary>Changes</summary>

Both clang and gfortran support the -fopenmp-simd flag, which enables OpenMP support only for simd constructs, while disabling the rest of OpenMP.

Add a new SimdOnly flang OpenMP IR pass which rewrites generated OpenMP FIR to remove all constructs except for omp.simd constructs, and constructs nested under them.
With this approach, the logic required to make the flag work can be self-contained within the pass, as opposed to being scattered all over the lowering code.

The flag is expected to have no effect if -fopenmp is passed explicitly, and is only expected to remove OpenMP constructs, not things like OpenMP library functions calls. This matches the behaviour of other compilers.

---

Patch is 70.32 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/150269.diff


18 Files Affected:

- (modified) clang/include/clang/Driver/Options.td (+10-5) 
- (modified) clang/lib/Driver/ToolChains/Flang.cpp (+5) 
- (modified) flang/include/flang/Optimizer/OpenMP/Passes.td (+5) 
- (modified) flang/include/flang/Optimizer/Passes/Pipelines.h (+4-1) 
- (modified) flang/include/flang/Optimizer/Transforms/Utils.h (+4) 
- (modified) flang/include/flang/Support/LangOptions.def (+2) 
- (modified) flang/include/flang/Tools/CrossToolHelpers.h (+1) 
- (modified) flang/lib/Frontend/CompilerInvocation.cpp (+9-2) 
- (modified) flang/lib/Frontend/FrontendActions.cpp (+16-9) 
- (modified) flang/lib/Lower/OpenMP/ClauseProcessor.h (+6-5) 
- (modified) flang/lib/Lower/OpenMP/OpenMP.cpp (+36-18) 
- (modified) flang/lib/Optimizer/OpenMP/CMakeLists.txt (+1) 
- (added) flang/lib/Optimizer/OpenMP/SimdOnly.cpp (+360) 
- (modified) flang/lib/Optimizer/Passes/Pipelines.cpp (+11-3) 
- (modified) flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp (+109-97) 
- (added) flang/test/Driver/fopenmp-simd.f90 (+59) 
- (added) flang/test/Transforms/OpenMP/simd-only.mlir (+622) 
- (modified) flang/tools/bbc/bbc.cpp (+3-1) 


``````````diff
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 916400efdb449..7a74dcffde4a9 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3706,14 +3706,19 @@ def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
 def fnoopenmp_relocatable_target : Flag<["-"], "fnoopenmp-relocatable-target">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>,
   Visibility<[ClangOption, CC1Option]>;
-def fopenmp_simd : Flag<["-"], "fopenmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
+def fopenmp_simd : Flag<["-"], "fopenmp-simd">,
+                   Group<f_Group>,
+                   Flags<[NoArgumentUnused]>,
+                   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+                   HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
 def fopenmp_enable_irbuilder : Flag<["-"], "fopenmp-enable-irbuilder">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Use the experimental OpenMP-IR-Builder codegen path.">;
-def fno_openmp_simd : Flag<["-"], "fno-openmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>;
+def fno_openmp_simd
+    : Flag<["-"], "fno-openmp-simd">,
+      Group<f_Group>,
+      Flags<[NoArgumentUnused]>,
+      Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
 def fopenmp_cuda_mode : Flag<["-"], "fopenmp-cuda-mode">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
 def fno_openmp_cuda_mode : Flag<["-"], "fno-openmp-cuda-mode">, Group<f_Group>,
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 7ab41e9b85a04..547e3156f519a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -937,6 +937,8 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
       if (Args.hasArg(options::OPT_fopenmp_force_usm))
         CmdArgs.push_back("-fopenmp-force-usm");
+      Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
+                      options::OPT_fno_openmp_simd);
 
       // FIXME: Clang supports a whole bunch more flags here.
       break;
@@ -952,6 +954,9 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
           << A->getSpelling() << A->getValue();
       break;
     }
+  } else {
+    Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
+                    options::OPT_fno_openmp_simd);
   }
 
   // Pass the path to compiler resource files.
diff --git a/flang/include/flang/Optimizer/OpenMP/Passes.td b/flang/include/flang/Optimizer/OpenMP/Passes.td
index 704faf0ccd856..79c1a5cfd9aca 100644
--- a/flang/include/flang/Optimizer/OpenMP/Passes.td
+++ b/flang/include/flang/Optimizer/OpenMP/Passes.td
@@ -112,4 +112,9 @@ def GenericLoopConversionPass
   ];
 }
 
+def SimdOnlyPass : Pass<"omp-simd-only", "mlir::func::FuncOp"> {
+  let summary = "Filters out non-simd OpenMP constructs";
+  let dependentDialects = ["mlir::omp::OpenMPDialect"];
+}
+
 #endif //FORTRAN_OPTIMIZER_OPENMP_PASSES
diff --git a/flang/include/flang/Optimizer/Passes/Pipelines.h b/flang/include/flang/Optimizer/Passes/Pipelines.h
index a3f59ee8dd013..fd8c43cc88a19 100644
--- a/flang/include/flang/Optimizer/Passes/Pipelines.h
+++ b/flang/include/flang/Optimizer/Passes/Pipelines.h
@@ -119,13 +119,16 @@ void registerDefaultInlinerPass(MLIRToLLVMPassPipelineConfig &config);
 void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
                                            MLIRToLLVMPassPipelineConfig &pc);
 
+/// Select which mode to enable OpenMP support in.
+enum class EnableOpenMP { None, Simd, Full };
+
 /// Create a pass pipeline for lowering from HLFIR to FIR
 ///
 /// \param pm - MLIR pass manager that will hold the pipeline definition
 /// \param optLevel - optimization level used for creating FIR optimization
 ///   passes pipeline
 void createHLFIRToFIRPassPipeline(
-    mlir::PassManager &pm, bool enableOpenMP,
+    mlir::PassManager &pm, EnableOpenMP enableOpenMP,
     llvm::OptimizationLevel optLevel = defaultOptLevel);
 
 struct OpenMPFIRPassPipelineOpts {
diff --git a/flang/include/flang/Optimizer/Transforms/Utils.h b/flang/include/flang/Optimizer/Transforms/Utils.h
index 49a616fb40fd5..307e6b59c57d4 100644
--- a/flang/include/flang/Optimizer/Transforms/Utils.h
+++ b/flang/include/flang/Optimizer/Transforms/Utils.h
@@ -33,6 +33,10 @@ void genMinMaxlocReductionLoop(fir::FirOpBuilder &builder, mlir::Value array,
                                mlir::Type maskElemType, mlir::Value resultArr,
                                bool maskMayBeLogicalScalar);
 
+std::pair<mlir::Block *, mlir::Block *>
+convertDoLoopToCFG(DoLoopOp loop, mlir::PatternRewriter &rewriter, bool setNSW,
+                   bool forceLoopToExecuteOnce);
+
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_TRANSFORMS_UTILS_H
diff --git a/flang/include/flang/Support/LangOptions.def b/flang/include/flang/Support/LangOptions.def
index d5bf7a2ecc036..ba72d7b4b7212 100644
--- a/flang/include/flang/Support/LangOptions.def
+++ b/flang/include/flang/Support/LangOptions.def
@@ -58,6 +58,8 @@ LANGOPT(OpenMPTeamSubscription, 1, 0)
 LANGOPT(OpenMPNoThreadState, 1, 0)
 /// Assume that no thread in a parallel region will encounter a parallel region
 LANGOPT(OpenMPNoNestedParallelism, 1, 0)
+/// Use SIMD only OpenMP support.
+LANGOPT(OpenMPSimd, 1, false)
 
 LANGOPT(VScaleMin, 32, 0)  ///< Minimum vscale range value
 LANGOPT(VScaleMax, 32, 0)  ///< Maximum vscale range value
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index df1da27058552..51958fa36c3ad 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -134,6 +134,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
                                       ///< functions.
   bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
   bool EnableOpenMP = false; ///< Enable OpenMP lowering.
+  bool EnableOpenMPSimd = false; ///< Enable OpenMP simd-only mode.
   std::string InstrumentFunctionEntry =
       ""; ///< Name of the instrument-function that is called on each
           ///< function-entry
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..80fd52b170f0c 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1152,8 +1152,15 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
                             clang::DiagnosticsEngine &diags) {
   llvm::opt::Arg *arg = args.getLastArg(clang::driver::options::OPT_fopenmp,
                                         clang::driver::options::OPT_fno_openmp);
-  if (!arg || arg->getOption().matches(clang::driver::options::OPT_fno_openmp))
-    return true;
+  if (!arg ||
+      arg->getOption().matches(clang::driver::options::OPT_fno_openmp)) {
+    bool isSimdSpecified = args.hasFlag(
+        clang::driver::options::OPT_fopenmp_simd,
+        clang::driver::options::OPT_fno_openmp_simd, /*Default=*/false);
+    if (!isSimdSpecified)
+      return true;
+    res.getLangOpts().OpenMPSimd = 1;
+  }
 
   unsigned numErrorsBefore = diags.getNumErrors();
   llvm::Triple t(res.getTargetOpts().triple);
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index b5f4f9421f633..0ac4c7094ec3b 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -297,6 +297,7 @@ bool CodeGenAction::beginSourceFileAction() {
   bool isOpenMPEnabled =
       ci.getInvocation().getFrontendOpts().features.IsEnabled(
           Fortran::common::LanguageFeature::OpenMP);
+  bool isOpenMPSimd = ci.getInvocation().getLangOpts().OpenMPSimd;
 
   fir::OpenMPFIRPassPipelineOpts opts;
 
@@ -328,12 +329,13 @@ bool CodeGenAction::beginSourceFileAction() {
     if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
             mlirModule->getOperation()))
       opts.isTargetDevice = offloadMod.getIsTargetDevice();
+  }
 
-    // WARNING: This pipeline must be run immediately after the lowering to
-    // ensure that the FIR is correct with respect to OpenMP operations/
-    // attributes.
+  // WARNING: This pipeline must be run immediately after the lowering to
+  // ensure that the FIR is correct with respect to OpenMP operations/
+  // attributes.
+  if (isOpenMPEnabled || isOpenMPSimd)
     fir::createOpenMPFIRPassPipeline(pm, opts);
-  }
 
   pm.enableVerifier(/*verifyPasses=*/true);
   pm.addPass(std::make_unique<Fortran::lower::VerifierPass>());
@@ -616,12 +618,14 @@ void CodeGenAction::lowerHLFIRToFIR() {
   pm.addPass(std::make_unique<Fortran::lower::VerifierPass>());
   pm.enableVerifier(/*verifyPasses=*/true);
 
+  fir::EnableOpenMP enableOpenMP = fir::EnableOpenMP::None;
+  if (ci.getInvocation().getFrontendOpts().features.IsEnabled(
+          Fortran::common::LanguageFeature::OpenMP))
+    enableOpenMP = fir::EnableOpenMP::Full;
+  if (ci.getInvocation().getLangOpts().OpenMPSimd)
+    enableOpenMP = fir::EnableOpenMP::Simd;
   // Create the pass pipeline
-  fir::createHLFIRToFIRPassPipeline(
-      pm,
-      ci.getInvocation().getFrontendOpts().features.IsEnabled(
-          Fortran::common::LanguageFeature::OpenMP),
-      level);
+  fir::createHLFIRToFIRPassPipeline(pm, enableOpenMP, level);
   (void)mlir::applyPassManagerCLOptions(pm);
 
   mlir::TimingScope timingScopeMLIRPasses = timingScopeRoot.nest(
@@ -747,6 +751,9 @@ void CodeGenAction::generateLLVMIR() {
           Fortran::common::LanguageFeature::OpenMP))
     config.EnableOpenMP = true;
 
+  if (ci.getInvocation().getLangOpts().OpenMPSimd)
+    config.EnableOpenMPSimd = true;
+
   if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
     config.NSWOnLoopVarInc = false;
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index f8a1f7983b79b..3e81f759ae69c 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -208,11 +208,12 @@ void ClauseProcessor::processTODO(mlir::Location currentLocation,
     if (!x)
       return;
     unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(currentLocation,
-         "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() +
-             " in " +
-             llvm::omp::getOpenMPDirectiveName(directive, version).upper() +
-             " construct");
+    if (!semaCtx.langOptions().OpenMPSimd)
+      TODO(currentLocation,
+           "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() +
+               " in " +
+               llvm::omp::getOpenMPDirectiveName(directive, version).upper() +
+               " construct");
   };
 
   for (ClauseIterator it = clauses.begin(); it != clauses.end(); ++it)
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4c2d7badef382..1647b8f516e46 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2259,7 +2259,8 @@ genOrderedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
              semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
              mlir::Location loc, const ConstructQueue &queue,
              ConstructQueue::const_iterator item) {
-  TODO(loc, "OMPD_ordered");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "OMPD_ordered");
   return nullptr;
 }
 
@@ -2446,7 +2447,8 @@ genScopeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
            mlir::Location loc, const ConstructQueue &queue,
            ConstructQueue::const_iterator item) {
-  TODO(loc, "Scope construct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "Scope construct");
   return nullptr;
 }
 
@@ -3238,7 +3240,8 @@ static mlir::omp::TaskloopOp genCompositeTaskloopSimd(
     lower::pft::Evaluation &eval, mlir::Location loc,
     const ConstructQueue &queue, ConstructQueue::const_iterator item) {
   assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
-  TODO(loc, "Composite TASKLOOP SIMD");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "Composite TASKLOOP SIMD");
   return nullptr;
 }
 
@@ -3410,8 +3413,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     break;
   case llvm::omp::Directive::OMPD_tile: {
     unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(loc, "Unhandled loop directive (" +
-                  llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+    if (!semaCtx.langOptions().OpenMPSimd)
+      TODO(loc, "Unhandled loop directive (" +
+                    llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+    break;
   }
   case llvm::omp::Directive::OMPD_unroll:
     genUnrollOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);
@@ -3446,35 +3451,40 @@ static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPDeclarativeAssumes &assumesConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
 }
 
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OmpDeclareVariantDirective &declareVariantDirective) {
-  TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
 }
 
 static void genOMP(
     lower::AbstractConverter &converter, lower::SymMap &symTable,
     semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
     const parser::OpenMPDeclareReductionConstruct &declareReductionConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
 }
 
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
 }
 
 static void
@@ -3670,14 +3680,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   (void)objects;
   (void)clauses;
 
-  TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPInteropConstruct &interopConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
 }
 
 static void
@@ -3693,7 +3705,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAllocatorsConstruct &allocsConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
 }
 
 //===----------------------------------------------------------------------===//
@@ -3765,7 +3778,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
         !std::holds_alternative<clause::Detach>(clause.u)) {
       std::string name =
           parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));
-      TODO(clauseLocation, name + " clause is not implemented yet");
+      if (!semaCtx.langOptions().OpenMPSimd)
+        TODO(clauseLocation, name + " clause is not implemented yet");
     }
   }
 
@@ -3785,7 +3799,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAssumeConstruct &assumeConstruct) {
   mlir::Location clauseLocation = converter.genLocation(assumeConstruct.source);
-  TODO(clauseLocation, "OpenMP ASSUME construct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(clauseLocation, "OpenMP ASSUME construct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
@@ -3810,21 +3825,24 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPUtilityConstruct &) {
-  TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPDispatchConstruct &) {
-  TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPExecutableAllocate &execAllocConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
diff --git a/flang/lib/Optimizer/OpenMP/CMakeLists.txt b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
index e31543328a9f9..3fb0bac05ce0d 100644
--- a/flang/lib/Optimizer/OpenMP/CMakeLists.txt
+++ b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
@@ -9,6 +9,7 @@ add_flang_library(FlangOpenMPTransforms
   MarkDeclareTarget.cpp
   LowerWorkshare.cpp
   LowerNontemporal.cpp
+  SimdOnly.cpp
 
   DEPENDS
   FIRDialect
diff --git a/flang/lib/Optimizer/OpenMP/SimdOnly.cpp b/flang/lib/Optimizer/OpenMP/SimdOnly.cpp
new file mode 100644
index 0000000000000..b4c97df767e65
--- /dev/null
+++ b/flang/lib/Optimizer/OpenMP/SimdOnly.cpp
@@ -0,0 +1,360 @@
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Transforms/Utils.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/IR/IRMapping.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include <llvm/Support/Debug.h>
+#include <mlir/IR/MLIRContext.h>
+#include <mlir/IR/Operation.h>
+#include <m...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/150269

From flang-commits at lists.llvm.org  Wed Jul 23 10:14:09 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 10:14:09 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150269@github.com>
Message-ID: <68811861.630a0220.2422b2.970c@mx.google.com>


llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: Kajetan Puchalski (mrkajetanp)

<details>
<summary>Changes</summary>

Both clang and gfortran support the -fopenmp-simd flag, which enables OpenMP support only for simd constructs, while disabling the rest of OpenMP.

Add a new SimdOnly flang OpenMP IR pass which rewrites generated OpenMP FIR to remove all constructs except for omp.simd constructs, and constructs nested under them.
With this approach, the logic required to make the flag work can be self-contained within the pass, as opposed to being scattered all over the lowering code.

The flag is expected to have no effect if -fopenmp is passed explicitly, and is only expected to remove OpenMP constructs, not things like OpenMP library functions calls. This matches the behaviour of other compilers.

---

Patch is 70.32 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/150269.diff


18 Files Affected:

- (modified) clang/include/clang/Driver/Options.td (+10-5) 
- (modified) clang/lib/Driver/ToolChains/Flang.cpp (+5) 
- (modified) flang/include/flang/Optimizer/OpenMP/Passes.td (+5) 
- (modified) flang/include/flang/Optimizer/Passes/Pipelines.h (+4-1) 
- (modified) flang/include/flang/Optimizer/Transforms/Utils.h (+4) 
- (modified) flang/include/flang/Support/LangOptions.def (+2) 
- (modified) flang/include/flang/Tools/CrossToolHelpers.h (+1) 
- (modified) flang/lib/Frontend/CompilerInvocation.cpp (+9-2) 
- (modified) flang/lib/Frontend/FrontendActions.cpp (+16-9) 
- (modified) flang/lib/Lower/OpenMP/ClauseProcessor.h (+6-5) 
- (modified) flang/lib/Lower/OpenMP/OpenMP.cpp (+36-18) 
- (modified) flang/lib/Optimizer/OpenMP/CMakeLists.txt (+1) 
- (added) flang/lib/Optimizer/OpenMP/SimdOnly.cpp (+360) 
- (modified) flang/lib/Optimizer/Passes/Pipelines.cpp (+11-3) 
- (modified) flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp (+109-97) 
- (added) flang/test/Driver/fopenmp-simd.f90 (+59) 
- (added) flang/test/Transforms/OpenMP/simd-only.mlir (+622) 
- (modified) flang/tools/bbc/bbc.cpp (+3-1) 


``````````diff
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 916400efdb449..7a74dcffde4a9 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3706,14 +3706,19 @@ def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
 def fnoopenmp_relocatable_target : Flag<["-"], "fnoopenmp-relocatable-target">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>,
   Visibility<[ClangOption, CC1Option]>;
-def fopenmp_simd : Flag<["-"], "fopenmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
+def fopenmp_simd : Flag<["-"], "fopenmp-simd">,
+                   Group<f_Group>,
+                   Flags<[NoArgumentUnused]>,
+                   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+                   HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
 def fopenmp_enable_irbuilder : Flag<["-"], "fopenmp-enable-irbuilder">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Use the experimental OpenMP-IR-Builder codegen path.">;
-def fno_openmp_simd : Flag<["-"], "fno-openmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>;
+def fno_openmp_simd
+    : Flag<["-"], "fno-openmp-simd">,
+      Group<f_Group>,
+      Flags<[NoArgumentUnused]>,
+      Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
 def fopenmp_cuda_mode : Flag<["-"], "fopenmp-cuda-mode">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
 def fno_openmp_cuda_mode : Flag<["-"], "fno-openmp-cuda-mode">, Group<f_Group>,
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 7ab41e9b85a04..547e3156f519a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -937,6 +937,8 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
       if (Args.hasArg(options::OPT_fopenmp_force_usm))
         CmdArgs.push_back("-fopenmp-force-usm");
+      Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
+                      options::OPT_fno_openmp_simd);
 
       // FIXME: Clang supports a whole bunch more flags here.
       break;
@@ -952,6 +954,9 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
           << A->getSpelling() << A->getValue();
       break;
     }
+  } else {
+    Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
+                    options::OPT_fno_openmp_simd);
   }
 
   // Pass the path to compiler resource files.
diff --git a/flang/include/flang/Optimizer/OpenMP/Passes.td b/flang/include/flang/Optimizer/OpenMP/Passes.td
index 704faf0ccd856..79c1a5cfd9aca 100644
--- a/flang/include/flang/Optimizer/OpenMP/Passes.td
+++ b/flang/include/flang/Optimizer/OpenMP/Passes.td
@@ -112,4 +112,9 @@ def GenericLoopConversionPass
   ];
 }
 
+def SimdOnlyPass : Pass<"omp-simd-only", "mlir::func::FuncOp"> {
+  let summary = "Filters out non-simd OpenMP constructs";
+  let dependentDialects = ["mlir::omp::OpenMPDialect"];
+}
+
 #endif //FORTRAN_OPTIMIZER_OPENMP_PASSES
diff --git a/flang/include/flang/Optimizer/Passes/Pipelines.h b/flang/include/flang/Optimizer/Passes/Pipelines.h
index a3f59ee8dd013..fd8c43cc88a19 100644
--- a/flang/include/flang/Optimizer/Passes/Pipelines.h
+++ b/flang/include/flang/Optimizer/Passes/Pipelines.h
@@ -119,13 +119,16 @@ void registerDefaultInlinerPass(MLIRToLLVMPassPipelineConfig &config);
 void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
                                            MLIRToLLVMPassPipelineConfig &pc);
 
+/// Select which mode to enable OpenMP support in.
+enum class EnableOpenMP { None, Simd, Full };
+
 /// Create a pass pipeline for lowering from HLFIR to FIR
 ///
 /// \param pm - MLIR pass manager that will hold the pipeline definition
 /// \param optLevel - optimization level used for creating FIR optimization
 ///   passes pipeline
 void createHLFIRToFIRPassPipeline(
-    mlir::PassManager &pm, bool enableOpenMP,
+    mlir::PassManager &pm, EnableOpenMP enableOpenMP,
     llvm::OptimizationLevel optLevel = defaultOptLevel);
 
 struct OpenMPFIRPassPipelineOpts {
diff --git a/flang/include/flang/Optimizer/Transforms/Utils.h b/flang/include/flang/Optimizer/Transforms/Utils.h
index 49a616fb40fd5..307e6b59c57d4 100644
--- a/flang/include/flang/Optimizer/Transforms/Utils.h
+++ b/flang/include/flang/Optimizer/Transforms/Utils.h
@@ -33,6 +33,10 @@ void genMinMaxlocReductionLoop(fir::FirOpBuilder &builder, mlir::Value array,
                                mlir::Type maskElemType, mlir::Value resultArr,
                                bool maskMayBeLogicalScalar);
 
+std::pair<mlir::Block *, mlir::Block *>
+convertDoLoopToCFG(DoLoopOp loop, mlir::PatternRewriter &rewriter, bool setNSW,
+                   bool forceLoopToExecuteOnce);
+
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_TRANSFORMS_UTILS_H
diff --git a/flang/include/flang/Support/LangOptions.def b/flang/include/flang/Support/LangOptions.def
index d5bf7a2ecc036..ba72d7b4b7212 100644
--- a/flang/include/flang/Support/LangOptions.def
+++ b/flang/include/flang/Support/LangOptions.def
@@ -58,6 +58,8 @@ LANGOPT(OpenMPTeamSubscription, 1, 0)
 LANGOPT(OpenMPNoThreadState, 1, 0)
 /// Assume that no thread in a parallel region will encounter a parallel region
 LANGOPT(OpenMPNoNestedParallelism, 1, 0)
+/// Use SIMD only OpenMP support.
+LANGOPT(OpenMPSimd, 1, false)
 
 LANGOPT(VScaleMin, 32, 0)  ///< Minimum vscale range value
 LANGOPT(VScaleMax, 32, 0)  ///< Maximum vscale range value
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index df1da27058552..51958fa36c3ad 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -134,6 +134,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
                                       ///< functions.
   bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
   bool EnableOpenMP = false; ///< Enable OpenMP lowering.
+  bool EnableOpenMPSimd = false; ///< Enable OpenMP simd-only mode.
   std::string InstrumentFunctionEntry =
       ""; ///< Name of the instrument-function that is called on each
           ///< function-entry
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..80fd52b170f0c 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1152,8 +1152,15 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
                             clang::DiagnosticsEngine &diags) {
   llvm::opt::Arg *arg = args.getLastArg(clang::driver::options::OPT_fopenmp,
                                         clang::driver::options::OPT_fno_openmp);
-  if (!arg || arg->getOption().matches(clang::driver::options::OPT_fno_openmp))
-    return true;
+  if (!arg ||
+      arg->getOption().matches(clang::driver::options::OPT_fno_openmp)) {
+    bool isSimdSpecified = args.hasFlag(
+        clang::driver::options::OPT_fopenmp_simd,
+        clang::driver::options::OPT_fno_openmp_simd, /*Default=*/false);
+    if (!isSimdSpecified)
+      return true;
+    res.getLangOpts().OpenMPSimd = 1;
+  }
 
   unsigned numErrorsBefore = diags.getNumErrors();
   llvm::Triple t(res.getTargetOpts().triple);
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index b5f4f9421f633..0ac4c7094ec3b 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -297,6 +297,7 @@ bool CodeGenAction::beginSourceFileAction() {
   bool isOpenMPEnabled =
       ci.getInvocation().getFrontendOpts().features.IsEnabled(
           Fortran::common::LanguageFeature::OpenMP);
+  bool isOpenMPSimd = ci.getInvocation().getLangOpts().OpenMPSimd;
 
   fir::OpenMPFIRPassPipelineOpts opts;
 
@@ -328,12 +329,13 @@ bool CodeGenAction::beginSourceFileAction() {
     if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
             mlirModule->getOperation()))
       opts.isTargetDevice = offloadMod.getIsTargetDevice();
+  }
 
-    // WARNING: This pipeline must be run immediately after the lowering to
-    // ensure that the FIR is correct with respect to OpenMP operations/
-    // attributes.
+  // WARNING: This pipeline must be run immediately after the lowering to
+  // ensure that the FIR is correct with respect to OpenMP operations/
+  // attributes.
+  if (isOpenMPEnabled || isOpenMPSimd)
     fir::createOpenMPFIRPassPipeline(pm, opts);
-  }
 
   pm.enableVerifier(/*verifyPasses=*/true);
   pm.addPass(std::make_unique<Fortran::lower::VerifierPass>());
@@ -616,12 +618,14 @@ void CodeGenAction::lowerHLFIRToFIR() {
   pm.addPass(std::make_unique<Fortran::lower::VerifierPass>());
   pm.enableVerifier(/*verifyPasses=*/true);
 
+  fir::EnableOpenMP enableOpenMP = fir::EnableOpenMP::None;
+  if (ci.getInvocation().getFrontendOpts().features.IsEnabled(
+          Fortran::common::LanguageFeature::OpenMP))
+    enableOpenMP = fir::EnableOpenMP::Full;
+  if (ci.getInvocation().getLangOpts().OpenMPSimd)
+    enableOpenMP = fir::EnableOpenMP::Simd;
   // Create the pass pipeline
-  fir::createHLFIRToFIRPassPipeline(
-      pm,
-      ci.getInvocation().getFrontendOpts().features.IsEnabled(
-          Fortran::common::LanguageFeature::OpenMP),
-      level);
+  fir::createHLFIRToFIRPassPipeline(pm, enableOpenMP, level);
   (void)mlir::applyPassManagerCLOptions(pm);
 
   mlir::TimingScope timingScopeMLIRPasses = timingScopeRoot.nest(
@@ -747,6 +751,9 @@ void CodeGenAction::generateLLVMIR() {
           Fortran::common::LanguageFeature::OpenMP))
     config.EnableOpenMP = true;
 
+  if (ci.getInvocation().getLangOpts().OpenMPSimd)
+    config.EnableOpenMPSimd = true;
+
   if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
     config.NSWOnLoopVarInc = false;
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index f8a1f7983b79b..3e81f759ae69c 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -208,11 +208,12 @@ void ClauseProcessor::processTODO(mlir::Location currentLocation,
     if (!x)
       return;
     unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(currentLocation,
-         "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() +
-             " in " +
-             llvm::omp::getOpenMPDirectiveName(directive, version).upper() +
-             " construct");
+    if (!semaCtx.langOptions().OpenMPSimd)
+      TODO(currentLocation,
+           "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() +
+               " in " +
+               llvm::omp::getOpenMPDirectiveName(directive, version).upper() +
+               " construct");
   };
 
   for (ClauseIterator it = clauses.begin(); it != clauses.end(); ++it)
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4c2d7badef382..1647b8f516e46 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2259,7 +2259,8 @@ genOrderedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
              semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
              mlir::Location loc, const ConstructQueue &queue,
              ConstructQueue::const_iterator item) {
-  TODO(loc, "OMPD_ordered");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "OMPD_ordered");
   return nullptr;
 }
 
@@ -2446,7 +2447,8 @@ genScopeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
            mlir::Location loc, const ConstructQueue &queue,
            ConstructQueue::const_iterator item) {
-  TODO(loc, "Scope construct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "Scope construct");
   return nullptr;
 }
 
@@ -3238,7 +3240,8 @@ static mlir::omp::TaskloopOp genCompositeTaskloopSimd(
     lower::pft::Evaluation &eval, mlir::Location loc,
     const ConstructQueue &queue, ConstructQueue::const_iterator item) {
   assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
-  TODO(loc, "Composite TASKLOOP SIMD");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "Composite TASKLOOP SIMD");
   return nullptr;
 }
 
@@ -3410,8 +3413,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     break;
   case llvm::omp::Directive::OMPD_tile: {
     unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(loc, "Unhandled loop directive (" +
-                  llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+    if (!semaCtx.langOptions().OpenMPSimd)
+      TODO(loc, "Unhandled loop directive (" +
+                    llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+    break;
   }
   case llvm::omp::Directive::OMPD_unroll:
     genUnrollOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);
@@ -3446,35 +3451,40 @@ static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPDeclarativeAssumes &assumesConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
 }
 
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OmpDeclareVariantDirective &declareVariantDirective) {
-  TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
 }
 
 static void genOMP(
     lower::AbstractConverter &converter, lower::SymMap &symTable,
     semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
     const parser::OpenMPDeclareReductionConstruct &declareReductionConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
 }
 
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
 }
 
 static void
@@ -3670,14 +3680,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   (void)objects;
   (void)clauses;
 
-  TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPInteropConstruct &interopConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
 }
 
 static void
@@ -3693,7 +3705,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAllocatorsConstruct &allocsConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
 }
 
 //===----------------------------------------------------------------------===//
@@ -3765,7 +3778,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
         !std::holds_alternative<clause::Detach>(clause.u)) {
       std::string name =
           parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));
-      TODO(clauseLocation, name + " clause is not implemented yet");
+      if (!semaCtx.langOptions().OpenMPSimd)
+        TODO(clauseLocation, name + " clause is not implemented yet");
     }
   }
 
@@ -3785,7 +3799,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAssumeConstruct &assumeConstruct) {
   mlir::Location clauseLocation = converter.genLocation(assumeConstruct.source);
-  TODO(clauseLocation, "OpenMP ASSUME construct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(clauseLocation, "OpenMP ASSUME construct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
@@ -3810,21 +3825,24 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPUtilityConstruct &) {
-  TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPDispatchConstruct &) {
-  TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPExecutableAllocate &execAllocConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
diff --git a/flang/lib/Optimizer/OpenMP/CMakeLists.txt b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
index e31543328a9f9..3fb0bac05ce0d 100644
--- a/flang/lib/Optimizer/OpenMP/CMakeLists.txt
+++ b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
@@ -9,6 +9,7 @@ add_flang_library(FlangOpenMPTransforms
   MarkDeclareTarget.cpp
   LowerWorkshare.cpp
   LowerNontemporal.cpp
+  SimdOnly.cpp
 
   DEPENDS
   FIRDialect
diff --git a/flang/lib/Optimizer/OpenMP/SimdOnly.cpp b/flang/lib/Optimizer/OpenMP/SimdOnly.cpp
new file mode 100644
index 0000000000000..b4c97df767e65
--- /dev/null
+++ b/flang/lib/Optimizer/OpenMP/SimdOnly.cpp
@@ -0,0 +1,360 @@
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Transforms/Utils.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/IR/IRMapping.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include <llvm/Support/Debug.h>
+#include <mlir/IR/MLIRContext.h>
+#include <mlir/IR/Operation.h>
+#include <m...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/150269

From flang-commits at lists.llvm.org  Wed Jul 23 10:13:34 2025
From: flang-commits at lists.llvm.org (Kajetan Puchalski via flang-commits)
Date: Wed, 23 Jul 2025 10:13:34 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
Message-ID: <llvm.org/llvm/llvm-project/pull/150269@github.com>


https://github.com/mrkajetanp created https://github.com/llvm/llvm-project/pull/150269

Both clang and gfortran support the -fopenmp-simd flag, which enables OpenMP support only for simd constructs, while disabling the rest of OpenMP.

Add a new SimdOnly flang OpenMP IR pass which rewrites generated OpenMP FIR to remove all constructs except for omp.simd constructs, and constructs nested under them.
With this approach, the logic required to make the flag work can be self-contained within the pass, as opposed to being scattered all over the lowering code.

The flag is expected to have no effect if -fopenmp is passed explicitly, and is only expected to remove OpenMP constructs, not things like OpenMP library functions calls. This matches the behaviour of other compilers.

>From 2e2b2cafe661604237f6a867459cfa1db8db5f6f Mon Sep 17 00:00:00 2001
From: Kajetan Puchalski <kajetan.puchalski at arm.com>
Date: Mon, 30 Jun 2025 16:00:08 +0000
Subject: [PATCH] [flang][OpenMP] Add -f[no]-openmp-simd

Both clang and gfortran support the -fopenmp-simd flag, which enables
OpenMP support only for simd constructs, while disabling the rest of
OpenMP.

Add a new SimdOnly flang OpenMP IR pass which rewrites generated
OpenMP FIR to remove all constructs except for omp.simd constructs,
and constructs nested under them.
With this approach, the logic required to make the flag work can
be self-contained within the pass, as opposed to being scattered
all over the lowering code.

The flag is expected to have no effect if -fopenmp is passed
explicitly, and is only expected to remove OpenMP constructs, not
things like OpenMP library functions calls. This matches the
behaviour of other compilers.

Signed-off-by: Kajetan Puchalski <kajetan.puchalski at arm.com>
---
 clang/include/clang/Driver/Options.td         |  15 +-
 clang/lib/Driver/ToolChains/Flang.cpp         |   5 +
 .../include/flang/Optimizer/OpenMP/Passes.td  |   5 +
 .../flang/Optimizer/Passes/Pipelines.h        |   5 +-
 .../flang/Optimizer/Transforms/Utils.h        |   4 +
 flang/include/flang/Support/LangOptions.def   |   2 +
 flang/include/flang/Tools/CrossToolHelpers.h  |   1 +
 flang/lib/Frontend/CompilerInvocation.cpp     |  11 +-
 flang/lib/Frontend/FrontendActions.cpp        |  25 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.h      |  11 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  54 +-
 flang/lib/Optimizer/OpenMP/CMakeLists.txt     |   1 +
 flang/lib/Optimizer/OpenMP/SimdOnly.cpp       | 360 ++++++++++
 flang/lib/Optimizer/Passes/Pipelines.cpp      |  14 +-
 .../Transforms/ControlFlowConverter.cpp       | 206 +++---
 flang/test/Driver/fopenmp-simd.f90            |  59 ++
 flang/test/Transforms/OpenMP/simd-only.mlir   | 622 ++++++++++++++++++
 flang/tools/bbc/bbc.cpp                       |   4 +-
 18 files changed, 1263 insertions(+), 141 deletions(-)
 create mode 100644 flang/lib/Optimizer/OpenMP/SimdOnly.cpp
 create mode 100644 flang/test/Driver/fopenmp-simd.f90
 create mode 100644 flang/test/Transforms/OpenMP/simd-only.mlir

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 916400efdb449..7a74dcffde4a9 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3706,14 +3706,19 @@ def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
 def fnoopenmp_relocatable_target : Flag<["-"], "fnoopenmp-relocatable-target">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>,
   Visibility<[ClangOption, CC1Option]>;
-def fopenmp_simd : Flag<["-"], "fopenmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
+def fopenmp_simd : Flag<["-"], "fopenmp-simd">,
+                   Group<f_Group>,
+                   Flags<[NoArgumentUnused]>,
+                   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+                   HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
 def fopenmp_enable_irbuilder : Flag<["-"], "fopenmp-enable-irbuilder">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Use the experimental OpenMP-IR-Builder codegen path.">;
-def fno_openmp_simd : Flag<["-"], "fno-openmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>;
+def fno_openmp_simd
+    : Flag<["-"], "fno-openmp-simd">,
+      Group<f_Group>,
+      Flags<[NoArgumentUnused]>,
+      Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
 def fopenmp_cuda_mode : Flag<["-"], "fopenmp-cuda-mode">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
 def fno_openmp_cuda_mode : Flag<["-"], "fno-openmp-cuda-mode">, Group<f_Group>,
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 7ab41e9b85a04..547e3156f519a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -937,6 +937,8 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
       if (Args.hasArg(options::OPT_fopenmp_force_usm))
         CmdArgs.push_back("-fopenmp-force-usm");
+      Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
+                      options::OPT_fno_openmp_simd);
 
       // FIXME: Clang supports a whole bunch more flags here.
       break;
@@ -952,6 +954,9 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
           << A->getSpelling() << A->getValue();
       break;
     }
+  } else {
+    Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
+                    options::OPT_fno_openmp_simd);
   }
 
   // Pass the path to compiler resource files.
diff --git a/flang/include/flang/Optimizer/OpenMP/Passes.td b/flang/include/flang/Optimizer/OpenMP/Passes.td
index 704faf0ccd856..79c1a5cfd9aca 100644
--- a/flang/include/flang/Optimizer/OpenMP/Passes.td
+++ b/flang/include/flang/Optimizer/OpenMP/Passes.td
@@ -112,4 +112,9 @@ def GenericLoopConversionPass
   ];
 }
 
+def SimdOnlyPass : Pass<"omp-simd-only", "mlir::func::FuncOp"> {
+  let summary = "Filters out non-simd OpenMP constructs";
+  let dependentDialects = ["mlir::omp::OpenMPDialect"];
+}
+
 #endif //FORTRAN_OPTIMIZER_OPENMP_PASSES
diff --git a/flang/include/flang/Optimizer/Passes/Pipelines.h b/flang/include/flang/Optimizer/Passes/Pipelines.h
index a3f59ee8dd013..fd8c43cc88a19 100644
--- a/flang/include/flang/Optimizer/Passes/Pipelines.h
+++ b/flang/include/flang/Optimizer/Passes/Pipelines.h
@@ -119,13 +119,16 @@ void registerDefaultInlinerPass(MLIRToLLVMPassPipelineConfig &config);
 void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
                                            MLIRToLLVMPassPipelineConfig &pc);
 
+/// Select which mode to enable OpenMP support in.
+enum class EnableOpenMP { None, Simd, Full };
+
 /// Create a pass pipeline for lowering from HLFIR to FIR
 ///
 /// \param pm - MLIR pass manager that will hold the pipeline definition
 /// \param optLevel - optimization level used for creating FIR optimization
 ///   passes pipeline
 void createHLFIRToFIRPassPipeline(
-    mlir::PassManager &pm, bool enableOpenMP,
+    mlir::PassManager &pm, EnableOpenMP enableOpenMP,
     llvm::OptimizationLevel optLevel = defaultOptLevel);
 
 struct OpenMPFIRPassPipelineOpts {
diff --git a/flang/include/flang/Optimizer/Transforms/Utils.h b/flang/include/flang/Optimizer/Transforms/Utils.h
index 49a616fb40fd5..307e6b59c57d4 100644
--- a/flang/include/flang/Optimizer/Transforms/Utils.h
+++ b/flang/include/flang/Optimizer/Transforms/Utils.h
@@ -33,6 +33,10 @@ void genMinMaxlocReductionLoop(fir::FirOpBuilder &builder, mlir::Value array,
                                mlir::Type maskElemType, mlir::Value resultArr,
                                bool maskMayBeLogicalScalar);
 
+std::pair<mlir::Block *, mlir::Block *>
+convertDoLoopToCFG(DoLoopOp loop, mlir::PatternRewriter &rewriter, bool setNSW,
+                   bool forceLoopToExecuteOnce);
+
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_TRANSFORMS_UTILS_H
diff --git a/flang/include/flang/Support/LangOptions.def b/flang/include/flang/Support/LangOptions.def
index d5bf7a2ecc036..ba72d7b4b7212 100644
--- a/flang/include/flang/Support/LangOptions.def
+++ b/flang/include/flang/Support/LangOptions.def
@@ -58,6 +58,8 @@ LANGOPT(OpenMPTeamSubscription, 1, 0)
 LANGOPT(OpenMPNoThreadState, 1, 0)
 /// Assume that no thread in a parallel region will encounter a parallel region
 LANGOPT(OpenMPNoNestedParallelism, 1, 0)
+/// Use SIMD only OpenMP support.
+LANGOPT(OpenMPSimd, 1, false)
 
 LANGOPT(VScaleMin, 32, 0)  ///< Minimum vscale range value
 LANGOPT(VScaleMax, 32, 0)  ///< Maximum vscale range value
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index df1da27058552..51958fa36c3ad 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -134,6 +134,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
                                       ///< functions.
   bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
   bool EnableOpenMP = false; ///< Enable OpenMP lowering.
+  bool EnableOpenMPSimd = false; ///< Enable OpenMP simd-only mode.
   std::string InstrumentFunctionEntry =
       ""; ///< Name of the instrument-function that is called on each
           ///< function-entry
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f55d866435997..80fd52b170f0c 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1152,8 +1152,15 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
                             clang::DiagnosticsEngine &diags) {
   llvm::opt::Arg *arg = args.getLastArg(clang::driver::options::OPT_fopenmp,
                                         clang::driver::options::OPT_fno_openmp);
-  if (!arg || arg->getOption().matches(clang::driver::options::OPT_fno_openmp))
-    return true;
+  if (!arg ||
+      arg->getOption().matches(clang::driver::options::OPT_fno_openmp)) {
+    bool isSimdSpecified = args.hasFlag(
+        clang::driver::options::OPT_fopenmp_simd,
+        clang::driver::options::OPT_fno_openmp_simd, /*Default=*/false);
+    if (!isSimdSpecified)
+      return true;
+    res.getLangOpts().OpenMPSimd = 1;
+  }
 
   unsigned numErrorsBefore = diags.getNumErrors();
   llvm::Triple t(res.getTargetOpts().triple);
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index b5f4f9421f633..0ac4c7094ec3b 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -297,6 +297,7 @@ bool CodeGenAction::beginSourceFileAction() {
   bool isOpenMPEnabled =
       ci.getInvocation().getFrontendOpts().features.IsEnabled(
           Fortran::common::LanguageFeature::OpenMP);
+  bool isOpenMPSimd = ci.getInvocation().getLangOpts().OpenMPSimd;
 
   fir::OpenMPFIRPassPipelineOpts opts;
 
@@ -328,12 +329,13 @@ bool CodeGenAction::beginSourceFileAction() {
     if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
             mlirModule->getOperation()))
       opts.isTargetDevice = offloadMod.getIsTargetDevice();
+  }
 
-    // WARNING: This pipeline must be run immediately after the lowering to
-    // ensure that the FIR is correct with respect to OpenMP operations/
-    // attributes.
+  // WARNING: This pipeline must be run immediately after the lowering to
+  // ensure that the FIR is correct with respect to OpenMP operations/
+  // attributes.
+  if (isOpenMPEnabled || isOpenMPSimd)
     fir::createOpenMPFIRPassPipeline(pm, opts);
-  }
 
   pm.enableVerifier(/*verifyPasses=*/true);
   pm.addPass(std::make_unique<Fortran::lower::VerifierPass>());
@@ -616,12 +618,14 @@ void CodeGenAction::lowerHLFIRToFIR() {
   pm.addPass(std::make_unique<Fortran::lower::VerifierPass>());
   pm.enableVerifier(/*verifyPasses=*/true);
 
+  fir::EnableOpenMP enableOpenMP = fir::EnableOpenMP::None;
+  if (ci.getInvocation().getFrontendOpts().features.IsEnabled(
+          Fortran::common::LanguageFeature::OpenMP))
+    enableOpenMP = fir::EnableOpenMP::Full;
+  if (ci.getInvocation().getLangOpts().OpenMPSimd)
+    enableOpenMP = fir::EnableOpenMP::Simd;
   // Create the pass pipeline
-  fir::createHLFIRToFIRPassPipeline(
-      pm,
-      ci.getInvocation().getFrontendOpts().features.IsEnabled(
-          Fortran::common::LanguageFeature::OpenMP),
-      level);
+  fir::createHLFIRToFIRPassPipeline(pm, enableOpenMP, level);
   (void)mlir::applyPassManagerCLOptions(pm);
 
   mlir::TimingScope timingScopeMLIRPasses = timingScopeRoot.nest(
@@ -747,6 +751,9 @@ void CodeGenAction::generateLLVMIR() {
           Fortran::common::LanguageFeature::OpenMP))
     config.EnableOpenMP = true;
 
+  if (ci.getInvocation().getLangOpts().OpenMPSimd)
+    config.EnableOpenMPSimd = true;
+
   if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
     config.NSWOnLoopVarInc = false;
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index f8a1f7983b79b..3e81f759ae69c 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -208,11 +208,12 @@ void ClauseProcessor::processTODO(mlir::Location currentLocation,
     if (!x)
       return;
     unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(currentLocation,
-         "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() +
-             " in " +
-             llvm::omp::getOpenMPDirectiveName(directive, version).upper() +
-             " construct");
+    if (!semaCtx.langOptions().OpenMPSimd)
+      TODO(currentLocation,
+           "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() +
+               " in " +
+               llvm::omp::getOpenMPDirectiveName(directive, version).upper() +
+               " construct");
   };
 
   for (ClauseIterator it = clauses.begin(); it != clauses.end(); ++it)
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4c2d7badef382..1647b8f516e46 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2259,7 +2259,8 @@ genOrderedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
              semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
              mlir::Location loc, const ConstructQueue &queue,
              ConstructQueue::const_iterator item) {
-  TODO(loc, "OMPD_ordered");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "OMPD_ordered");
   return nullptr;
 }
 
@@ -2446,7 +2447,8 @@ genScopeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
            mlir::Location loc, const ConstructQueue &queue,
            ConstructQueue::const_iterator item) {
-  TODO(loc, "Scope construct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "Scope construct");
   return nullptr;
 }
 
@@ -3238,7 +3240,8 @@ static mlir::omp::TaskloopOp genCompositeTaskloopSimd(
     lower::pft::Evaluation &eval, mlir::Location loc,
     const ConstructQueue &queue, ConstructQueue::const_iterator item) {
   assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
-  TODO(loc, "Composite TASKLOOP SIMD");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "Composite TASKLOOP SIMD");
   return nullptr;
 }
 
@@ -3410,8 +3413,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     break;
   case llvm::omp::Directive::OMPD_tile: {
     unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(loc, "Unhandled loop directive (" +
-                  llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+    if (!semaCtx.langOptions().OpenMPSimd)
+      TODO(loc, "Unhandled loop directive (" +
+                    llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+    break;
   }
   case llvm::omp::Directive::OMPD_unroll:
     genUnrollOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);
@@ -3446,35 +3451,40 @@ static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPDeclarativeAssumes &assumesConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
 }
 
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OmpDeclareVariantDirective &declareVariantDirective) {
-  TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
 }
 
 static void genOMP(
     lower::AbstractConverter &converter, lower::SymMap &symTable,
     semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
     const parser::OpenMPDeclareReductionConstruct &declareReductionConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
 }
 
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
 }
 
 static void
@@ -3670,14 +3680,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   (void)objects;
   (void)clauses;
 
-  TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPInteropConstruct &interopConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
 }
 
 static void
@@ -3693,7 +3705,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAllocatorsConstruct &allocsConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
 }
 
 //===----------------------------------------------------------------------===//
@@ -3765,7 +3778,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
         !std::holds_alternative<clause::Detach>(clause.u)) {
       std::string name =
           parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));
-      TODO(clauseLocation, name + " clause is not implemented yet");
+      if (!semaCtx.langOptions().OpenMPSimd)
+        TODO(clauseLocation, name + " clause is not implemented yet");
     }
   }
 
@@ -3785,7 +3799,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAssumeConstruct &assumeConstruct) {
   mlir::Location clauseLocation = converter.genLocation(assumeConstruct.source);
-  TODO(clauseLocation, "OpenMP ASSUME construct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(clauseLocation, "OpenMP ASSUME construct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
@@ -3810,21 +3825,24 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPUtilityConstruct &) {
-  TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPDispatchConstruct &) {
-  TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPExecutableAllocate &execAllocConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
diff --git a/flang/lib/Optimizer/OpenMP/CMakeLists.txt b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
index e31543328a9f9..3fb0bac05ce0d 100644
--- a/flang/lib/Optimizer/OpenMP/CMakeLists.txt
+++ b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
@@ -9,6 +9,7 @@ add_flang_library(FlangOpenMPTransforms
   MarkDeclareTarget.cpp
   LowerWorkshare.cpp
   LowerNontemporal.cpp
+  SimdOnly.cpp
 
   DEPENDS
   FIRDialect
diff --git a/flang/lib/Optimizer/OpenMP/SimdOnly.cpp b/flang/lib/Optimizer/OpenMP/SimdOnly.cpp
new file mode 100644
index 0000000000000..b4c97df767e65
--- /dev/null
+++ b/flang/lib/Optimizer/OpenMP/SimdOnly.cpp
@@ -0,0 +1,360 @@
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Transforms/Utils.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/IR/IRMapping.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include <llvm/Support/Debug.h>
+#include <mlir/IR/MLIRContext.h>
+#include <mlir/IR/Operation.h>
+#include <mlir/IR/PatternMatch.h>
+#include <mlir/Support/LLVM.h>
+
+namespace flangomp {
+#define GEN_PASS_DEF_SIMDONLYPASS
+#include "flang/Optimizer/OpenMP/Passes.h.inc"
+} // namespace flangomp
+
+namespace {
+
+#define DEBUG_TYPE "omp-simd-only-pass"
+
+class SimdOnlyConversionPattern : public mlir::RewritePattern {
+public:
+  SimdOnlyConversionPattern(mlir::MLIRContext *ctx)
+      : mlir::RewritePattern(MatchAnyOpTypeTag{}, 1, ctx) {}
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::Operation *op,
+                  mlir::PatternRewriter &rewriter) const override {
+    if (op->getDialect()->getNamespace() !=
+        mlir::omp::OpenMPDialect::getDialectNamespace())
+      return rewriter.notifyMatchFailure(op, "Not an OpenMP op");
+
+    if (auto simdOp = mlir::dyn_cast<mlir::omp::SimdOp>(op)) {
+      // Remove the composite attr given that the op will no longer be composite
+      if (simdOp.isComposite()) {
+        simdOp.setComposite(false);
+        return mlir::success();
+      }
+
+      return rewriter.notifyMatchFailure(op, "Op is a plain SimdOp");
+    }
+
+    if (op->getParentOfType<mlir::omp::SimdOp>())
+      return rewriter.notifyMatchFailure(op, "Op is nested under a SimdOp");
+
+    if (!mlir::isa<mlir::func::FuncOp>(op->getParentOp()) &&
+        (mlir::isa<mlir::omp::TerminatorOp>(op) ||
+         mlir::isa<mlir::omp::YieldOp>(op)))
+      return rewriter.notifyMatchFailure(op,
+                                         "Non top-level yield or terminator");
+
+    // SectionOp overrides its BlockArgInterface based on the parent SectionsOp.
+    // We need to make sure we only rewrite omp.sections once all omp.section
+    // ops inside it have been rewritten, otherwise the individual omp.section
+    // ops will not be able to access their argument values.
+    if (auto sectionsOp = mlir::dyn_cast<mlir::omp::SectionsOp>(op)) {
+      for (auto &opInSections : sectionsOp.getRegion().getOps())
+        if (mlir::isa<mlir::omp::SectionOp>(opInSections))
+          return rewriter.notifyMatchFailure(
+              op, "SectionsOp still contains individual sections");
+    }
+
+    LLVM_DEBUG(llvm::dbgs() << "SimdOnlyPass matched OpenMP op:\n");
+    LLVM_DEBUG(op->dump());
+
+    // Erase ops that don't need any special handling
+    if (mlir::isa<mlir::omp::BarrierOp>(op) ||
+        mlir::isa<mlir::omp::FlushOp>(op) ||
+        mlir::isa<mlir::omp::TaskyieldOp>(op) ||
+        mlir::isa<mlir::omp::MapBoundsOp>(op) ||
+        mlir::isa<mlir::omp::TargetEnterDataOp>(op) ||
+        mlir::isa<mlir::omp::TargetExitDataOp>(op) ||
+        mlir::isa<mlir::omp::TargetUpdateOp>(op) ||
+        mlir::isa<mlir::omp::OrderedOp>(op) ||
+        mlir::isa<mlir::omp::CancelOp>(op) ||
+        mlir::isa<mlir::omp::CancellationPointOp>(op) ||
+        mlir::isa<mlir::omp::ScanOp>(op) ||
+        mlir::isa<mlir::omp::TaskwaitOp>(op)) {
+      rewriter.eraseOp(op);
+      return mlir::success();
+    }
+
+    fir::FirOpBuilder builder(rewriter, op);
+    mlir::Location loc = op->getLoc();
+
+    auto inlineSimpleOp = [&](mlir::Operation *ompOp) -> bool {
+      if (!ompOp)
+        return false;
+
+      llvm::SmallVector<std::pair<mlir::Value, mlir::BlockArgument>>
+          blockArgsPairs;
+      if (auto iface =
+              mlir::dyn_cast<mlir::omp::BlockArgOpenMPOpInterface>(op)) {
+        iface.getBlockArgsPairs(blockArgsPairs);
+        for (auto [value, argument] : blockArgsPairs)
+          rewriter.replaceAllUsesWith(argument, value);
+      }
+
+      if (ompOp->getRegion(0).getBlocks().size() == 1) {
+        auto &block = *ompOp->getRegion(0).getBlocks().begin();
+        // This block is about to be removed so any arguments should have been
+        // replaced by now.
+        block.eraseArguments(0, block.getNumArguments());
+        if (auto terminatorOp =
+                mlir::dyn_cast<mlir::omp::TerminatorOp>(block.back())) {
+          rewriter.eraseOp(terminatorOp);
+        }
+        rewriter.inlineBlockBefore(&block, op, {});
+      } else {
+        // When dealing with multi-block regions we need to fix up the control
+        // flow
+        auto *origBlock = ompOp->getBlock();
+        auto *newBlock = rewriter.splitBlock(origBlock, ompOp->getIterator());
+        auto *innerFrontBlock = &ompOp->getRegion(0).getBlocks().front();
+        builder.setInsertionPointToEnd(origBlock);
+        builder.create<mlir::cf::BranchOp>(loc, innerFrontBlock);
+        // We are no longer passing any arguments to the first block in the
+        // region, so this should be safe to erase.
+        innerFrontBlock->eraseArguments(0, innerFrontBlock->getNumArguments());
+
+        for (auto &innerBlock : ompOp->getRegion(0).getBlocks()) {
+          // Remove now-unused block arguments
+          for (auto arg : innerBlock.getArguments()) {
+            if (arg.getUses().empty())
+              innerBlock.eraseArgument(arg.getArgNumber());
+          }
+          if (auto terminatorOp =
+                  mlir::dyn_cast<mlir::omp::TerminatorOp>(innerBlock.back())) {
+            builder.setInsertionPointToEnd(&innerBlock);
+            builder.create<mlir::cf::BranchOp>(loc, newBlock);
+            rewriter.eraseOp(terminatorOp);
+          }
+        }
+
+        rewriter.inlineRegionBefore(ompOp->getRegion(0), newBlock);
+      }
+
+      rewriter.eraseOp(op);
+      return true;
+    };
+
+    if (auto ompOp = mlir::dyn_cast<mlir::omp::LoopNestOp>(op)) {
+      mlir::Type indexType = builder.getIndexType();
+      mlir::Type oldIndexType = ompOp.getIVs().begin()->getType();
+      builder.setInsertionPoint(op);
+      auto one = builder.create<mlir::arith::ConstantIndexOp>(loc, 1);
+
+      // Generate the new loop nest
+      mlir::Block *nestBody = nullptr;
+      fir::DoLoopOp outerLoop = nullptr;
+      llvm::SmallVector<mlir::Value> loopIndArgs;
+      for (auto extent : ompOp.getLoopUpperBounds()) {
+        auto ub = builder.createConvert(loc, indexType, extent);
+        auto doLoop = builder.create<fir::DoLoopOp>(loc, one, ub, one, false);
+        nestBody = doLoop.getBody();
+        builder.setInsertionPointToStart(nestBody);
+        // Convert the indices to the type used inside the loop if needed
+        if (oldIndexType != indexType) {
+          auto convertedIndVar = builder.createConvert(
+              loc, oldIndexType, doLoop.getInductionVar());
+          loopIndArgs.push_back(convertedIndVar);
+        } else {
+          loopIndArgs.push_back(doLoop.getInductionVar());
+        }
+        if (!outerLoop)
+          outerLoop = doLoop;
+      }
+
+      // Move the omp loop body into the new loop body
+      if (ompOp->getRegion(0).getBlocks().size() == 1) {
+        auto &block = *ompOp->getRegion(0).getBlocks().begin();
+        rewriter.mergeBlocks(&block, nestBody, loopIndArgs);
+
+        // Find the new loop block terminator and move it before the end of the
+        // block
+        for (auto &loopBodyOp : nestBody->getOperations()) {
+          if (auto resultOp = mlir::dyn_cast<fir::ResultOp>(loopBodyOp)) {
+            rewriter.moveOpBefore(resultOp.getOperation(), &nestBody->back());
+            break;
+          }
+        }
+
+        // Remove omp.yield at the end of the loop body
+        if (auto yieldOp = mlir::dyn_cast<mlir::omp::YieldOp>(nestBody->back()))
+          rewriter.eraseOp(yieldOp);
+        // DoLoopOp does not support multi-block regions, thus if we're dealing
+        // with multiple blocks we need to convert it into basic control-flow
+        // operations.
+      } else {
+        rewriter.inlineRegionBefore(ompOp->getRegion(0), nestBody);
+        auto indVarArg = outerLoop->getRegion(0).front().getArgument(0);
+        // fir::convertDoLoopToCFG expects the induction variable to be of type
+        // index while the OpenMP LoopNestOp can have indices of different
+        // types. We need to work around it.
+        if (indVarArg.getType() != indexType)
+          indVarArg.setType(indexType);
+
+        auto loopBlocks =
+            fir::convertDoLoopToCFG(outerLoop, rewriter, false, false);
+        auto *conditionalBlock = loopBlocks.first;
+        auto *firstBlock =
+            conditionalBlock->getNextNode(); // Start of the loop body
+        auto *lastBlock = loopBlocks.second; // Incrementing induction variables
+
+        // If the induction variable is used within the loop and was originally
+        // not of type index, then we need to add a convert to the original type
+        // and replace its uses inside the loop body.
+        if (oldIndexType != indexType) {
+          indVarArg = conditionalBlock->getArgument(0);
+          builder.setInsertionPointToStart(firstBlock);
+          auto convertedIndVar =
+              builder.createConvert(loc, oldIndexType, indVarArg);
+          rewriter.replaceUsesWithIf(
+              indVarArg, convertedIndVar, [&](auto &use) -> bool {
+                return use.getOwner() != convertedIndVar.getDefiningOp() &&
+                       use.getOwner()->getBlock() != lastBlock;
+              });
+        }
+
+        // There might be an unused convert and an unused argument to the block.
+        // If so, remove them.
+        if (lastBlock->front().getUses().empty())
+          lastBlock->front().erase();
+        for (auto arg : lastBlock->getArguments()) {
+          if (arg.getUses().empty())
+            lastBlock->eraseArgument(arg.getArgNumber());
+        }
+
+        // Any loop blocks that end in omp.yield should just branch to
+        // lastBlock.
+        for (auto *loopBlock = conditionalBlock; loopBlock != lastBlock;
+             loopBlock = loopBlock->getNextNode()) {
+          if (auto yieldOp =
+                  mlir::dyn_cast<mlir::omp::YieldOp>(loopBlock->back())) {
+            builder.setInsertionPointToEnd(loopBlock);
+            builder.create<mlir::cf::BranchOp>(loc, lastBlock);
+            rewriter.eraseOp(yieldOp);
+          }
+        }
+      }
+
+      rewriter.eraseOp(ompOp);
+      return mlir::success();
+    }
+
+    if (auto mapInfoOp = mlir::dyn_cast<mlir::omp::MapInfoOp>(op)) {
+      mapInfoOp.getResult().replaceAllUsesWith(mapInfoOp.getVarPtr());
+      rewriter.eraseOp(mapInfoOp);
+      return mlir::success();
+    }
+
+    if (auto atomicReadOp = mlir::dyn_cast<mlir::omp::AtomicReadOp>(op)) {
+      builder.setInsertionPoint(op);
+      auto loadOp = builder.create<fir::LoadOp>(loc, atomicReadOp.getX());
+      auto storeOp = builder.create<fir::StoreOp>(loc, loadOp.getResult(),
+                                                  atomicReadOp.getV());
+      rewriter.replaceOp(op, storeOp);
+      return mlir::success();
+    }
+
+    if (auto atomicWriteOp = mlir::dyn_cast<mlir::omp::AtomicWriteOp>(op)) {
+      auto storeOp = builder.create<fir::StoreOp>(loc, atomicWriteOp.getExpr(),
+                                                  atomicWriteOp.getX());
+      rewriter.replaceOp(op, storeOp);
+      return mlir::success();
+    }
+
+    if (auto atomicUpdateOp = mlir::dyn_cast<mlir::omp::AtomicUpdateOp>(op)) {
+      assert("one block in region" &&
+             atomicUpdateOp.getRegion().getBlocks().size() == 1);
+      auto &block = *atomicUpdateOp.getRegion().getBlocks().begin();
+      builder.setInsertionPointToStart(&block);
+
+      // Load the update `x` operand and replace its uses within the block
+      auto loadOp = builder.create<fir::LoadOp>(loc, atomicUpdateOp.getX());
+      rewriter.replaceUsesWithIf(
+          block.getArgument(0), loadOp.getResult(),
+          [&](auto &op) { return op.get().getParentBlock() == &block; });
+
+      // Store the result back into `x` in line with omp.yield semantics for
+      // this op
+      auto yieldOp = mlir::cast<mlir::omp::YieldOp>(block.back());
+      assert("only one yield operand" && yieldOp->getNumOperands() == 1);
+      builder.setInsertionPointAfter(yieldOp);
+      builder.create<fir::StoreOp>(loc, yieldOp->getOperand(0),
+                                   atomicUpdateOp.getX());
+      rewriter.eraseOp(yieldOp);
+
+      // Inline the final block and remove the now-empty op
+      assert("only one block argument" && block.getNumArguments() == 1);
+      block.eraseArguments(0, block.getNumArguments());
+      rewriter.inlineBlockBefore(&block, op, {});
+      rewriter.eraseOp(op);
+      return mlir::success();
+    }
+
+    if (auto threadPrivateOp = mlir::dyn_cast<mlir::omp::ThreadprivateOp>(op)) {
+      threadPrivateOp.getTlsAddr().replaceAllUsesWith(
+          threadPrivateOp.getSymAddr());
+      rewriter.eraseOp(threadPrivateOp);
+      return mlir::success();
+    }
+
+    if (inlineSimpleOp(mlir::dyn_cast<mlir::omp::TeamsOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::ParallelOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::SingleOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::SectionOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::SectionsOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::WsloopOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::LoopOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::TargetOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::TargetDataOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::DistributeOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::TaskOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::TaskloopOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::MasterOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::CriticalOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::OrderedRegionOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::AtomicCaptureOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::MaskedOp>(op)))
+      return mlir::success();
+
+    op->emitOpError("OpenMP operation left unhandled after SimdOnly pass.");
+    return mlir::failure();
+  }
+};
+
+class SimdOnlyPass : public flangomp::impl::SimdOnlyPassBase<SimdOnlyPass> {
+
+public:
+  SimdOnlyPass() = default;
+
+  void runOnOperation() override {
+    mlir::func::FuncOp func = getOperation();
+
+    if (func.isDeclaration())
+      return;
+
+    mlir::MLIRContext *context = &getContext();
+    mlir::RewritePatternSet patterns(context);
+    patterns.insert<SimdOnlyConversionPattern>(context);
+
+    mlir::GreedyRewriteConfig config;
+    // Prevent the pattern driver from merging blocks.
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
+
+    if (mlir::failed(
+            mlir::applyPatternsGreedily(func, std::move(patterns), config))) {
+      mlir::emitError(func.getLoc(), "error in simd-only conversion pass");
+      signalPassFailure();
+    }
+  }
+};
+
+} // namespace
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index ca8e820608688..5a870928f8413 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -242,7 +242,8 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
 /// \param pm - MLIR pass manager that will hold the pipeline definition
 /// \param optLevel - optimization level used for creating FIR optimization
 ///   passes pipeline
-void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
+void createHLFIRToFIRPassPipeline(mlir::PassManager &pm,
+                                  EnableOpenMP enableOpenMP,
                                   llvm::OptimizationLevel optLevel) {
   if (optLevel.isOptimizingForSpeed()) {
     addCanonicalizerPassWithoutRegionSimplification(pm);
@@ -294,8 +295,10 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
     addNestedPassToAllTopLevelOperations<PassConstructor>(
         pm, hlfir::createInlineHLFIRAssign);
   pm.addPass(hlfir::createConvertHLFIRtoFIR());
-  if (enableOpenMP)
+  if (enableOpenMP != EnableOpenMP::None)
     pm.addPass(flangomp::createLowerWorkshare());
+  if (enableOpenMP == EnableOpenMP::Simd)
+    pm.addPass(flangomp::createSimdOnlyPass());
 }
 
 /// Create a pass pipeline for handling certain OpenMP transformations needed
@@ -396,7 +399,12 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
 void createMLIRToLLVMPassPipeline(mlir::PassManager &pm,
                                   MLIRToLLVMPassPipelineConfig &config,
                                   llvm::StringRef inputFilename) {
-  fir::createHLFIRToFIRPassPipeline(pm, config.EnableOpenMP, config.OptLevel);
+  fir::EnableOpenMP enableOpenMP = fir::EnableOpenMP::None;
+  if (config.EnableOpenMP)
+    enableOpenMP = fir::EnableOpenMP::Full;
+  if (config.EnableOpenMPSimd)
+    enableOpenMP = fir::EnableOpenMP::Simd;
+  fir::createHLFIRToFIRPassPipeline(pm, enableOpenMP, config.OptLevel);
 
   // Add default optimizer pass pipeline.
   fir::createDefaultFIROptimizerPassPipeline(pm, config);
diff --git a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
index e466aed753e63..4bcf7d857c7b0 100644
--- a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
+++ b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
@@ -14,6 +14,7 @@
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Support/TypeCode.h"
 #include "flang/Optimizer/Transforms/Passes.h"
+#include "flang/Optimizer/Transforms/Utils.h"
 #include "flang/Runtime/derived-api.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
@@ -31,6 +32,113 @@ namespace fir {
 using namespace fir;
 using namespace mlir;
 
+// Extracted here for use in other passes
+
+/// Convert fir::DoLoopOp to control-flow operations
+std::pair<mlir::Block *, mlir::Block *>
+fir::convertDoLoopToCFG(DoLoopOp loop, mlir::PatternRewriter &rewriter,
+                        bool setNSW, bool forceLoopToExecuteOnce) {
+  auto loc = loop.getLoc();
+  mlir::arith::IntegerOverflowFlags flags{};
+  if (setNSW)
+    flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
+  auto iofAttr =
+      mlir::arith::IntegerOverflowFlagsAttr::get(rewriter.getContext(), flags);
+
+  // Create the start and end blocks that will wrap the DoLoopOp with an
+  // initalizer and an end point
+  auto *initBlock = rewriter.getInsertionBlock();
+  auto initPos = rewriter.getInsertionPoint();
+  auto *endBlock = rewriter.splitBlock(initBlock, initPos);
+
+  // Split the first DoLoopOp block in two parts. The part before will be the
+  // conditional block since it already has the induction variable and
+  // loop-carried values as arguments.
+  auto *conditionalBlock = &loop.getRegion().front();
+  conditionalBlock->addArgument(rewriter.getIndexType(), loc);
+  auto *firstBlock =
+      rewriter.splitBlock(conditionalBlock, conditionalBlock->begin());
+  auto *lastBlock = &loop.getRegion().back();
+
+  // Move the blocks from the DoLoopOp between initBlock and endBlock
+  rewriter.inlineRegionBefore(loop.getRegion(), endBlock);
+
+  // Get loop values from the DoLoopOp
+  auto low = loop.getLowerBound();
+  auto high = loop.getUpperBound();
+  assert(low && high && "must be a Value");
+  auto step = loop.getStep();
+
+  // Initalization block
+  rewriter.setInsertionPointToEnd(initBlock);
+  auto diff = mlir::arith::SubIOp::create(rewriter, loc, high, low);
+  auto distance = mlir::arith::AddIOp::create(rewriter, loc, diff, step);
+  mlir::Value iters =
+      mlir::arith::DivSIOp::create(rewriter, loc, distance, step);
+
+  if (forceLoopToExecuteOnce) {
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto cond = mlir::arith::CmpIOp::create(
+        rewriter, loc, arith::CmpIPredicate::sle, iters, zero);
+    auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
+    iters = mlir::arith::SelectOp::create(rewriter, loc, cond, one, iters);
+  }
+
+  llvm::SmallVector<mlir::Value> loopOperands;
+  loopOperands.push_back(low);
+  auto operands = loop.getIterOperands();
+  loopOperands.append(operands.begin(), operands.end());
+  loopOperands.push_back(iters);
+
+  mlir::cf::BranchOp::create(rewriter, loc, conditionalBlock, loopOperands);
+
+  // Last loop block
+  auto *terminator = lastBlock->getTerminator();
+  rewriter.setInsertionPointToEnd(lastBlock);
+  auto iv = conditionalBlock->getArgument(0);
+  mlir::Value steppedIndex =
+      mlir::arith::AddIOp::create(rewriter, loc, iv, step, iofAttr);
+  assert(steppedIndex && "must be a Value");
+  auto lastArg = conditionalBlock->getNumArguments() - 1;
+  auto itersLeft = conditionalBlock->getArgument(lastArg);
+  auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
+  mlir::Value itersMinusOne =
+      mlir::arith::SubIOp::create(rewriter, loc, itersLeft, one);
+
+  llvm::SmallVector<mlir::Value> loopCarried;
+  loopCarried.push_back(steppedIndex);
+  auto begin = loop.getFinalValue() ? std::next(terminator->operand_begin())
+                                    : terminator->operand_begin();
+  loopCarried.append(begin, terminator->operand_end());
+  loopCarried.push_back(itersMinusOne);
+  auto backEdge =
+      mlir::cf::BranchOp::create(rewriter, loc, conditionalBlock, loopCarried);
+  rewriter.eraseOp(terminator);
+
+  // Copy loop annotations from the do loop to the loop back edge.
+  if (auto ann = loop.getLoopAnnotation())
+    backEdge->setAttr("loop_annotation", *ann);
+
+  // Conditional block
+  rewriter.setInsertionPointToEnd(conditionalBlock);
+  auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
+  auto comparison = mlir::arith::CmpIOp::create(
+      rewriter, loc, arith::CmpIPredicate::sgt, itersLeft, zero);
+
+  mlir::cf::CondBranchOp::create(rewriter, loc, comparison, firstBlock,
+                                 llvm::ArrayRef<mlir::Value>(), endBlock,
+                                 llvm::ArrayRef<mlir::Value>());
+
+  // The result of the loop operation is the values of the condition block
+  // arguments except the induction variable on the last iteration.
+  auto args = loop.getFinalValue()
+                  ? conditionalBlock->getArguments()
+                  : conditionalBlock->getArguments().drop_front();
+  rewriter.replaceOp(loop, args.drop_back());
+
+  return std::make_pair(conditionalBlock, lastBlock);
+}
+
 namespace {
 
 // Conversion of fir control ops to more primitive control-flow.
@@ -50,103 +158,7 @@ class CfgLoopConv : public mlir::OpRewritePattern<fir::DoLoopOp> {
   llvm::LogicalResult
   matchAndRewrite(DoLoopOp loop,
                   mlir::PatternRewriter &rewriter) const override {
-    auto loc = loop.getLoc();
-    mlir::arith::IntegerOverflowFlags flags{};
-    if (setNSW)
-      flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
-    auto iofAttr = mlir::arith::IntegerOverflowFlagsAttr::get(
-        rewriter.getContext(), flags);
-
-    // Create the start and end blocks that will wrap the DoLoopOp with an
-    // initalizer and an end point
-    auto *initBlock = rewriter.getInsertionBlock();
-    auto initPos = rewriter.getInsertionPoint();
-    auto *endBlock = rewriter.splitBlock(initBlock, initPos);
-
-    // Split the first DoLoopOp block in two parts. The part before will be the
-    // conditional block since it already has the induction variable and
-    // loop-carried values as arguments.
-    auto *conditionalBlock = &loop.getRegion().front();
-    conditionalBlock->addArgument(rewriter.getIndexType(), loc);
-    auto *firstBlock =
-        rewriter.splitBlock(conditionalBlock, conditionalBlock->begin());
-    auto *lastBlock = &loop.getRegion().back();
-
-    // Move the blocks from the DoLoopOp between initBlock and endBlock
-    rewriter.inlineRegionBefore(loop.getRegion(), endBlock);
-
-    // Get loop values from the DoLoopOp
-    auto low = loop.getLowerBound();
-    auto high = loop.getUpperBound();
-    assert(low && high && "must be a Value");
-    auto step = loop.getStep();
-
-    // Initalization block
-    rewriter.setInsertionPointToEnd(initBlock);
-    auto diff = mlir::arith::SubIOp::create(rewriter, loc, high, low);
-    auto distance = mlir::arith::AddIOp::create(rewriter, loc, diff, step);
-    mlir::Value iters =
-        mlir::arith::DivSIOp::create(rewriter, loc, distance, step);
-
-    if (forceLoopToExecuteOnce) {
-      auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
-      auto cond = mlir::arith::CmpIOp::create(
-          rewriter, loc, arith::CmpIPredicate::sle, iters, zero);
-      auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
-      iters = mlir::arith::SelectOp::create(rewriter, loc, cond, one, iters);
-    }
-
-    llvm::SmallVector<mlir::Value> loopOperands;
-    loopOperands.push_back(low);
-    auto operands = loop.getIterOperands();
-    loopOperands.append(operands.begin(), operands.end());
-    loopOperands.push_back(iters);
-
-    mlir::cf::BranchOp::create(rewriter, loc, conditionalBlock, loopOperands);
-
-    // Last loop block
-    auto *terminator = lastBlock->getTerminator();
-    rewriter.setInsertionPointToEnd(lastBlock);
-    auto iv = conditionalBlock->getArgument(0);
-    mlir::Value steppedIndex =
-        mlir::arith::AddIOp::create(rewriter, loc, iv, step, iofAttr);
-    assert(steppedIndex && "must be a Value");
-    auto lastArg = conditionalBlock->getNumArguments() - 1;
-    auto itersLeft = conditionalBlock->getArgument(lastArg);
-    auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
-    mlir::Value itersMinusOne =
-        mlir::arith::SubIOp::create(rewriter, loc, itersLeft, one);
-
-    llvm::SmallVector<mlir::Value> loopCarried;
-    loopCarried.push_back(steppedIndex);
-    auto begin = loop.getFinalValue() ? std::next(terminator->operand_begin())
-                                      : terminator->operand_begin();
-    loopCarried.append(begin, terminator->operand_end());
-    loopCarried.push_back(itersMinusOne);
-    auto backEdge = mlir::cf::BranchOp::create(rewriter, loc, conditionalBlock,
-                                               loopCarried);
-    rewriter.eraseOp(terminator);
-
-    // Copy loop annotations from the do loop to the loop back edge.
-    if (auto ann = loop.getLoopAnnotation())
-      backEdge->setAttr("loop_annotation", *ann);
-
-    // Conditional block
-    rewriter.setInsertionPointToEnd(conditionalBlock);
-    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
-    auto comparison = mlir::arith::CmpIOp::create(
-        rewriter, loc, arith::CmpIPredicate::sgt, itersLeft, zero);
-
-    mlir::cf::CondBranchOp::create(rewriter, loc, comparison, firstBlock,
-                                   llvm::ArrayRef<mlir::Value>(), endBlock,
-                                   llvm::ArrayRef<mlir::Value>());
-
-    // The result of the loop operation is the values of the condition block
-    // arguments except the induction variable on the last iteration.
-    auto args = loop.getFinalValue()
-                    ? conditionalBlock->getArguments()
-                    : conditionalBlock->getArguments().drop_front();
-    rewriter.replaceOp(loop, args.drop_back());
+    convertDoLoopToCFG(loop, rewriter, setNSW, forceLoopToExecuteOnce);
     return success();
   }
 
diff --git a/flang/test/Driver/fopenmp-simd.f90 b/flang/test/Driver/fopenmp-simd.f90
new file mode 100644
index 0000000000000..b25adee2779ee
--- /dev/null
+++ b/flang/test/Driver/fopenmp-simd.f90
@@ -0,0 +1,59 @@
+! RUN: %flang -target x86_64-linux-gnu -fopenmp-simd %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-OPENMP-SIMD-FLAG --check-prefix=CHECK-NO-LD-ANY
+! RUN: %flang -target x86_64-darwin -fopenmp-simd %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-OPENMP-SIMD-FLAG --check-prefix=CHECK-NO-LD-ANY
+! RUN: %flang -target x86_64-freebsd -fopenmp-simd %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-OPENMP-SIMD-FLAG --check-prefix=CHECK-NO-LD-ANY
+! RUN: %flang -target x86_64-windows-gnu -fopenmp-simd %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-OPENMP-SIMD-FLAG --check-prefix=CHECK-NO-LD-ANY
+
+! CHECK-OPENMP-SIMD-FLAG: "-fopenmp-simd"
+! CHECK-NO-LD-ANY-NOT: "-l{{(omp|gomp|iomp5)}}"
+
+! -fopenmp-simd enables openmp support only for simd constructs
+! RUN: %flang_fc1 -fopenmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP-SIMD %s
+! RUN: %flang_fc1 -fno-openmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-NO-OMP-SIMD %s
+! RUN: %flang_fc1 -fopenmp-simd -fno-openmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-NO-OMP-SIMD %s
+! RUN: %flang_fc1 -fno-openmp-simd -fopenmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP-SIMD %s
+! -fopenmp-simd should have no effect if -fopenmp is already set
+! RUN: %flang_fc1 -fopenmp %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP %s
+! RUN: %flang_fc1 -fopenmp -fno-openmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP %s
+
+subroutine main
+  ! CHECK-OMP-SIMD-NOT: omp.parallel
+  ! CHECK-OMP-SIMD-NOT: omp.wsloop
+  ! CHECK-OMP-SIMD-NOT: omp.loop_nest
+  ! CHECK-OMP-SIMD: fir.do_loop
+  ! CHECK-NO-OMP-SIMD-NOT: omp.parallel
+  ! CHECK-NO-OMP-SIMD-NOT: omp.wsloop
+  ! CHECK-NO-OMP-SIMD-NOT: omp.loop_nest
+  ! CHECK-NO-OMP-SIMD: fir.do_loop
+  ! CHECK-OMP: omp.parallel
+  ! CHECK-OMP: omp.wsloop
+  ! CHECK-OMP: omp.loop_nest
+  ! CHECK-OMP-NOT: fir.do_loop
+  !$omp parallel do
+  do i = 1, 10
+    print *, "test"
+  end do
+  ! CHECK-NO-OMP-SIMD-NOT: omp.yield
+  ! CHECK-NO-OMP-SIMD-NOT: omp.terminator
+  ! CHECK-OMP-SIMD-NOT: omp.yield
+  ! CHECK-OMP-SIMD-NOT: omp.terminator
+  ! CHECK-OMP: omp.yield
+  ! CHECK-OMP: omp.terminator
+  !$omp end parallel do
+
+  ! CHECK-OMP-SIMD: omp.simd
+  ! CHECK-NO-OMP-SIMD-NOT: omp.simd
+  ! CHECK-OMP: omp.simd
+  !$omp simd
+  ! CHECK-OMP-SIMD: omp.loop_nest
+  ! CHECK-NO-OMP-SIMD-NOT: omp.loop_nest
+  ! CHECK-NO-OMP-SIMD: fir.do_loop
+  ! CHECK-OMP: omp.loop_nest
+  ! CHECK-OMP-NOT: fir.do_loop
+  do i = 1, 10
+    print *, "test"
+  ! CHECK-OMP-SIMD: omp.yield
+  ! CHECK-NO-OMP-SIMD-NOT: omp.yield
+  ! CHECK-OMP: omp.yield
+  end do
+end subroutine
diff --git a/flang/test/Transforms/OpenMP/simd-only.mlir b/flang/test/Transforms/OpenMP/simd-only.mlir
new file mode 100644
index 0000000000000..c3efce13c4414
--- /dev/null
+++ b/flang/test/Transforms/OpenMP/simd-only.mlir
@@ -0,0 +1,622 @@
+// RUN: fir-opt --split-input-file --omp-simd-only %s | FileCheck %s
+
+// Check that simd operations are not removed and rewritten, but all the other OpenMP ops are.
+
+// CHECK-LABEL: func.func @simd
+omp.private {type = private} @_QFEi_private_i32 : i32
+func.func @simd(%arg0: i32, %arg1: !fir.ref<i32>, %arg2: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  %c100000_i32 = arith.constant 100000 : i32
+  // CHECK: omp.simd private
+  omp.simd private(@_QFEi_private_i32 %arg2 -> %arg3 : !fir.ref<i32>) {
+    // CHECK: omp.loop_nest
+    omp.loop_nest (%arg4) : i32 = (%c1_i32) to (%c100000_i32) inclusive step (%c1_i32) {
+      // CHECK: fir.store
+      fir.store %arg0 to %arg1 : !fir.ref<i32>
+      // CHECK: omp.yield
+      omp.yield
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @simd_composite
+func.func @simd_composite(%arg0: i32, %arg1: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  %c100000_i32 = arith.constant 100000 : i32
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK-NOT: omp.wsloop
+    omp.wsloop {
+      // CHECK: omp.simd
+      omp.simd {
+        // CHECK: omp.loop_nest
+        omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100000_i32) inclusive step (%c1_i32) {
+          // CHECK: fir.store
+          fir.store %arg0 to %arg1 : !fir.ref<i32>
+          // CHECK: omp.yield
+          omp.yield
+        }
+      // CHECK-NOT: {omp.composite}
+      } {omp.composite}
+    } {omp.composite}
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @parallel
+omp.private {type = private} @_QFEi_private_i32 : i32
+func.func @parallel(%arg0: i32, %arg1: !fir.ref<i32>) {
+  %c1 = arith.constant 1 : index
+  %c1_i32 = arith.constant 1 : i32
+  %c100000_i32 = arith.constant 100000 : i32
+  // CHECK-NOT: omp.parallel
+  omp.parallel private(@_QFEi_private_i32 %arg1 -> %arg3 : !fir.ref<i32>) {
+    // CHECK: fir.convert
+    %15 = fir.convert %c1_i32 : (i32) -> index
+    // CHECK: fir.convert
+    %16 = fir.convert %c100000_i32 : (i32) -> index
+    // CHECK: fir.do_loop
+    %18:2 = fir.do_loop %arg4 = %15 to %16 step %c1 iter_args(%arg2 = %arg0) -> (index, i32) {
+      // CHECK: fir.store
+      fir.store %arg0 to %arg1 : !fir.ref<i32>
+      // CHECK-NOT: omp.barrier
+      omp.barrier
+      fir.result %arg4, %arg2 : index, i32
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+    }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @do
+func.func @do(%arg5: i32, %arg6: !fir.ref<i32>) {
+  // CHECK: %[[C1:.*]] = arith.constant 1 : index
+  %c1_i32 = arith.constant 1 : i32
+  // CHECK: %[[C100:.*]] = fir.convert %c100_i32 : (i32) -> index
+  %c100_i32 = arith.constant 100 : i32
+  // CHECK-NOT: omp.wsloop
+  omp.wsloop {
+    // CHECK-NOT: omp.loop_nest
+    // CHECK: fir.do_loop %[[IVAR:.*]] = %[[C1]] to %[[C100]] step %[[C1]]
+    omp.loop_nest (%arg1) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
+      // CHECK: fir.store
+      fir.store %arg5 to %arg6 : !fir.ref<i32>
+      // CHECK-NOT: omp.yield
+      omp.yield
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @do_nested
+func.func @do_nested(%arg5: i32, %arg6: !fir.ref<i32>) {
+  // CHECK: %[[C1:.*]] = arith.constant 1 : index
+  %c1_i32 = arith.constant 1 : i32
+  %c100_i32 = arith.constant 100 : i32
+  %c200_i32 = arith.constant 200 : i32
+  // CHECK-NOT: omp.wsloop
+  omp.wsloop {
+    // CHECK: %[[C200:.*]] = fir.convert %c200_i32 : (i32) -> index
+    // CHECK-NOT: omp.loop_nest
+    // CHECK: fir.do_loop %[[IVAR_1:.*]] = %[[C1]] to %[[C200]] step %[[C1]]
+    // CHECK: %[[C100:.*]] = fir.convert %c100_i32 : (i32) -> index
+    // CHECK: fir.do_loop %[[IVAR_2:.*]] = %[[C1]] to %[[C100]] step %[[C1]]
+    omp.loop_nest (%arg2, %arg3) : i32 = (%c1_i32, %c1_i32) to (%c200_i32, %c100_i32) inclusive step (%c1_i32, %c1_i32) {
+      // CHECK: fir.store
+      fir.store %arg5 to %arg6 : !fir.ref<i32>
+      // CHECK-NOT: omp.yield
+      omp.yield
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @single
+func.func @single(%arg0: i32, %arg1: !fir.ref<i32>) {
+  // CHECK-NOT: omp.single
+  omp.single {
+    // CHECK: fir.store
+    fir.store %arg0 to %arg1 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @target_map(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @target_map(%arg5: i32, %arg6: !fir.ref<i32>) {
+  // CHECK-NOT: omp.map.info
+  %3 = omp.map.info var_ptr(%arg6 : !fir.ref<i32>, i32) map_clauses(implicit) capture(ByCopy) -> !fir.ref<i32>
+  // CHECK-NOT: omp.target
+  omp.target map_entries(%3 -> %arg0 : !fir.ref<i32>) {
+    // CHECK: arith.constant
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: fir.store %c1_i32 to %[[ARG_1]]
+    fir.store %c1_i32 to %arg0 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @task(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+omp.private {type = private} @_QFEi_private_i32 : i32
+func.func @task(%arg5: i32, %arg6: !fir.ref<i32>) {
+  // CHECK-NOT: omp.task
+  omp.task private(@_QFEi_private_i32 %arg6 -> %arg2 : !fir.ref<i32>) {
+    // CHECK: fir.store %[[ARG_0]] to %[[ARG_1]]
+    fir.store %arg5 to %arg2 : !fir.ref<i32>
+    // CHECK-NOT: omp.flush
+    omp.flush
+    // CHECK-NOT: omp.taskyield
+    omp.taskyield
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @teams
+func.func @teams(%arg0: i32, %arg1: !fir.ref<i32>) {
+  // CHECK-NOT: omp.teams
+  omp.teams {
+    // CHECK: fir.store
+    fir.store %arg0 to %arg1 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @distribute
+func.func @distribute(%arg0: i32, %arg1: i32, %arg2: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  // CHECK-NOT: omp.teams
+  omp.teams {
+    // CHECK-NOT: omp.distribute
+    omp.distribute {
+      // CHECK-NOT: omp.loop_nest
+      // CHECK: fir.do_loop
+      omp.loop_nest (%arg5) : i32 = (%arg0) to (%arg1) inclusive step (%c1_i32) {
+        // CHECK: fir.store
+        fir.store %arg0 to %arg2 : !fir.ref<i32>
+        // CHECK-NOT: omp.yield
+        omp.yield
+      }
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @threadprivate(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @threadprivate(%arg0: i32, %arg1: !fir.ref<i32>) {
+  // CHECK-NOT: omp.threadprivate
+  %1 = omp.threadprivate %arg1 : !fir.ref<i32> -> !fir.ref<i32>
+  // CHECK: fir.store %[[ARG_0]] to %[[ARG_1]]
+  fir.store %arg0 to %1 : !fir.ref<i32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @taskloop(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @taskloop(%funcArg0: i32, %funcArg1: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  %c2_i32 = arith.constant 2 : i32
+  %c10_i32 = arith.constant 10 : i32
+  // CHECK-NOT: omp.taskloop
+  omp.taskloop grainsize(%c2_i32: i32) {
+    // CHECK-NOT: omp.loop_nest
+    // CHECK: fir.do_loop
+    omp.loop_nest (%arg1) : i32 = (%c1_i32) to (%c10_i32) inclusive step (%c1_i32) {
+      // CHECK: fir.store %[[ARG_0]] to %[[ARG_1]]
+      fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+      // CHECK-NOT: omp.yield
+      omp.yield
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @target_update_enter_data_map_info(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @target_update_enter_data_map_info(%funcArg0: i32, %funcArg1: !fir.ref<i32>) {
+  %c1 = arith.constant 1 : index
+  // CHECK-NOT: omp.map.bounds
+  %1 = omp.map.bounds lower_bound(%c1 : index) upper_bound(%c1 : index) extent(%c1 : index) stride(%c1 : index) start_idx(%c1 : index)
+  // CHECK-NOT: omp.map.info
+  %13 = omp.map.info var_ptr(%funcArg1 : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) bounds(%1) -> !fir.ref<i32>
+  // CHECK-NOT: omp.target_enter_data
+  omp.target_enter_data map_entries(%13 : !fir.ref<i32>)
+  // CHECK-NOT: omp.target
+  omp.target map_entries(%13 -> %arg3 : !fir.ref<i32>) {
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: fir.store %c1_i32 to %[[ARG_1]]
+    fir.store %c1_i32 to %arg3 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  // CHECK-NOT: omp.map.info
+  %18 = omp.map.info var_ptr(%funcArg1 : !fir.ref<i32>, i32) map_clauses(from) capture(ByRef) bounds(%1) -> !fir.ref<i32>
+  // CHECK-NOT: omp.target_update
+  omp.target_update map_entries(%18 : !fir.ref<i32>)
+  // CHECK-NOT: omp.target_exit_data
+  omp.target_exit_data map_entries(%18 : !fir.ref<i32>)
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @target_data(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @target_data(%funcArg0: i32, %funcArg1: !fir.ref<i32>) {
+  %c1 = arith.constant 1 : index
+  // CHECK-NOT: omp.map.bounds
+  %3 = omp.map.bounds lower_bound(%c1 : index) upper_bound(%c1 : index) extent(%c1 : index) stride(%c1 : index) start_idx(%c1 : index)
+  // CHECK-NOT: omp.map.info
+  %4 = omp.map.info var_ptr(%funcArg1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) bounds(%3) -> !fir.ref<i32>
+  // CHECK-NOT: omp.target_data
+  omp.target_data map_entries(%4 : !fir.ref<i32>) {
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: fir.store %c1_i32 to %[[ARG_1]]
+    fir.store %c1_i32 to %4 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @sections(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_2:.*]]: !fir.ref<i32>
+func.func @sections(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %funcArg2: !fir.ref<i32>) {
+  // CHECK-NOT: omp.sections
+  omp.sections {
+    // CHECK-NOT: omp.section
+    omp.section {
+      // CHECK: fir.store
+      fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+      // CHECK-NOT: omp.terminator
+      omp.terminator
+    }
+    // CHECK-NOT: omp.section
+    omp.section {
+      // CHECK: fir.store
+      fir.store %funcArg0 to %funcArg2 : !fir.ref<i32>
+      // CHECK-NOT: omp.terminator
+      omp.terminator
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %c0_i32 = arith.constant 0 : i32
+  omp.yield(%c0_i32 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = arith.addi %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+// CHECK-LABEL: func.func @reduction_scan(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @reduction_scan(%funcArg0: i32, %funcArg1: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  %c8_i32 = arith.constant 8 : i32
+  // CHECK-NOT: omp.wsloop
+  omp.wsloop reduction(mod: inscan, @add_reduction_i32 %funcArg1 -> %arg3 : !fir.ref<i32>) {
+    // CHECK-NOT: omp.loop_nest
+    // CHECK: fir.do_loop
+    omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%c8_i32) inclusive step (%c1_i32) {
+      // CHECK: fir.declare %[[ARG_1]]
+      %1 = fir.declare %arg3 {uniq_name = "a"} : (!fir.ref<i32>) -> !fir.ref<i32>
+      // CHECK-NOT: omp.scan
+      omp.scan inclusive(%1 : !fir.ref<i32>)
+      // CHECK: fir.store
+      fir.store %funcArg0 to %1 : !fir.ref<i32>
+      // CHECK-NOT: omp.yield
+      omp.yield
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @ordered(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @ordered(%funcArg0: i32, %funcArg1: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  %c10_i32 = arith.constant 10 : i32
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK-NOT: omp.wsloop
+    omp.wsloop ordered(0) {
+      // CHECK-NOT: omp.loop_nest
+      // CHECK: fir.do_loop
+      omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%c10_i32) inclusive step (%c1_i32) {
+        // CHECK-NOT: omp.ordered.region
+        omp.ordered.region {
+          // CHECK: fir.store
+          fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+          // CHECK-NOT: omp.terminator
+          omp.terminator
+        }
+        // CHECK-NOT: omp.yield
+        omp.yield
+      }
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @master(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_2:.*]]: !fir.ref<i32>
+func.func @master(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %funcArg2: !fir.ref<i32>) {
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK: fir.store
+    fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+    // CHECK-NOT: omp.master
+    omp.master {
+      // CHECK: fir.store
+      fir.store %funcArg0 to %funcArg2 : !fir.ref<i32>
+      // CHECK-NOT: omp.terminator
+      omp.terminator
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @masked(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_2:.*]]: !fir.ref<i32>
+func.func @masked(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %funcArg2: !fir.ref<i32>) {
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK: fir.store
+    fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+    // CHECK-NOT: omp.masked
+    omp.masked {
+      // CHECK: fir.store
+      fir.store %funcArg0 to %funcArg2 : !fir.ref<i32>
+      // CHECK-NOT: omp.terminator
+      omp.terminator
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @critical(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_2:.*]]: !fir.ref<i32>
+omp.critical.declare @mylock
+func.func @critical(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %funcArg2: !fir.ref<i32>) {
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK: fir.store
+    fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+    // CHECK-NOT: omp.critical
+    omp.critical(@mylock) {
+      // CHECK: fir.store
+      fir.store %funcArg0 to %funcArg2 : !fir.ref<i32>
+      // CHECK-NOT: omp.terminator
+      omp.terminator
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @cancel(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_2:.*]]: !fir.ref<i32>, %[[ARG_3:.*]]: i1
+func.func @cancel(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %funcArg2: !fir.ref<i32>, %funcArg3: i1) {
+  %c1_i32 = arith.constant 1 : i32
+  %c10_i32 = arith.constant 10 : i32
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK-NOT: omp.wsloop
+    omp.wsloop {
+      // CHECK-NOT: omp.loop_nest
+      // CHECK: fir.do_loop
+      omp.loop_nest (%arg1) : i32 = (%c1_i32) to (%c10_i32) inclusive step (%c1_i32) {
+        // CHECK: fir.store
+        fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+        // CHECK-NOT: fir.if
+        fir.if %funcArg3 {
+          // CHECK-NOT: omp.cancel
+          omp.cancel cancellation_construct_type(loop)
+        }
+        // CHECK-NOT: omp.cancellation_point
+        omp.cancellation_point cancellation_construct_type(loop)
+        // CHECK: fir.store
+        fir.store %funcArg0 to %funcArg2 : !fir.ref<i32>
+        // CHECK-NOT: omp.yield
+        omp.yield
+      }
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @atomic(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_2:.*]]: !fir.ref<i32>, %[[ARG_3:.*]]: i32
+func.func @atomic(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %funcArg2: !fir.ref<i32>, %funcArg3: i32) {
+  %c1_i32 = arith.constant 1 : i32
+  %5 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
+  // CHECK: %[[VAL_0:.*]] = fir.declare
+  %6 = fir.declare %5 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK-NOT: omp.atomic.write
+    // CHECK: fir.store %[[ARG_0]] to %[[ARG_1]]
+    omp.atomic.write %funcArg1 = %funcArg0 : !fir.ref<i32>, i32
+    // CHECK-NOT: omp.atomic.read
+    // CHECK: %[[VAL_1:.*]] = fir.load %[[ARG_1]]
+    // CHECK-NEXT: fir.store %[[VAL_1]] to %[[ARG_2]]
+    omp.atomic.read %funcArg2 = %funcArg1 : !fir.ref<i32>, !fir.ref<i32>, i32
+    // CHECK-NOT: omp.atomic.update
+    // CHECK: fir.load %[[VAL_0]]
+    // CHECK-NEXT: %[[ADD_VAL:.*]] = arith.addi
+    // CHECK-NOT: omp.yield
+    // CHECK-NEXT: fir.store %[[ADD_VAL]] to %[[VAL_0]]
+    omp.atomic.update %6 : !fir.ref<i32> {
+    ^bb0(%arg3: i32):
+      %88 = arith.addi %arg3, %c1_i32 : i32
+      omp.yield(%88 : i32)
+    }
+    // CHECK-NOT: omp.atomic.read
+    // CHECK: %[[VAL_2:.*]] = fir.load %[[VAL_0]]
+    // CHECK-NEXT: fir.store %[[VAL_2]] to %[[ARG_1]]
+    omp.atomic.read %funcArg1 = %6 : !fir.ref<i32>, !fir.ref<i32>, i32
+    // CHECK-NOT: omp.atomic.capture
+    omp.atomic.capture {
+      // CHECK-NOT: omp.atomic.read
+      // CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_0]]
+      // CHECK-NEXT: fir.store %[[VAL_3]] to %[[ARG_2]]
+      omp.atomic.read %funcArg2 = %6 : !fir.ref<i32>, !fir.ref<i32>, i32
+      // CHECK-NOT: omp.atomic.update
+      // CHECK: fir.load %[[VAL_0]]
+      // CHECK-NEXT: %[[ADD_VAL_2:.*]] = arith.addi
+      // CHECK-NOT: omp.yield
+      // CHECK-NEXT: fir.store %[[ADD_VAL_2]] to %[[VAL_0]]
+      omp.atomic.update %6 : !fir.ref<i32> {
+      ^bb0(%arg3: i32):
+        %88 = arith.addi %arg3, %c1_i32 : i32
+        omp.yield(%88 : i32)
+      }
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @multi_block(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_3:.*]]: i1
+func.func @multi_block(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %6: i1) {
+  %false = arith.constant false
+  %c0_i32 = arith.constant 0 : i32
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK: cf.cond_br %[[ARG_3]], ^[[BB1:.*]], ^[[BB2:.*]]
+    cf.cond_br %6, ^bb1, ^bb2
+  // CHECK: ^[[BB1]]
+  ^bb1:  // pred: ^bb0
+    // CHECK: fir.call
+    fir.call @_FortranAStopStatement(%c0_i32, %false, %false) fastmath<contract> : (i32, i1, i1) -> ()
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  // CHECK: ^[[BB2]]
+  ^bb2:  // pred: ^bb0
+    // CHECK: fir.store
+    fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @do_multi_block(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_3:.*]]: i1
+func.func @do_multi_block(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %6: i1) {
+  %false = arith.constant false
+  %c1_i32 = arith.constant 1 : i32
+  %c100_i32 = arith.constant 100 : i32
+  // CHECK-NOT: omp.wsloop
+  omp.wsloop {
+    // CHECK-NOT: omp.loop_nest
+    // CHECK: cf.br ^[[CBB:.*]](
+    // CHECK: ^[[CBB]]
+    // CHECK: %[[CMP_VAL:.*]] = arith.cmpi
+    // CHECK: cf.cond_br %[[CMP_VAL]], ^[[FBB:.*]], ^[[LBB:.*]]
+    omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
+    // CHECK: ^[[FBB]]
+      // CHECK: fir.store
+      fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+      // CHECK: cf.br ^[[BBB:.*]]
+      cf.br ^bb1
+    // CHECK: ^[[BBB]]
+    ^bb1:  // pred: ^bb0
+      // CHECK: fir.store
+      fir.store %c1_i32 to %funcArg1 : !fir.ref<i32>
+      // CHECK: cf.cond_br
+      cf.cond_br %6, ^bb2, ^bb3
+    // CHECK: ^[[SBB:.*]]
+    ^bb2:  // pred: ^bb1
+      // CHECK: fir.call
+      fir.call @_FortranAStopStatement(%c1_i32, %false, %false) fastmath<contract> : (i32, i1, i1) -> ()
+      // CHECK-NOT: omp.yield
+      omp.yield
+      // CHECK: cf.br ^[[LBB:.*]]
+    // CHECK: ^[[OBB:.*]]
+      // CHECK: cf.br ^[[LBB]]
+    // CHECK: ^[[LBB]]
+      // CHECK: arith.subi
+      // CHECK: cf.br ^[[CBB]]
+    // CHECK: ^[[EBB:.*]]
+    ^bb3:  // pred: ^bb1
+      // CHECK-NOT: omp.yield
+      omp.yield
+    }
+  }
+  return
+}
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..82dff2653ad09 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -520,7 +520,9 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
 
     if (emitFIR && useHLFIR) {
       // lower HLFIR to FIR
-      fir::createHLFIRToFIRPassPipeline(pm, enableOpenMP,
+      fir::EnableOpenMP enableOmp =
+          enableOpenMP ? fir::EnableOpenMP::Full : fir::EnableOpenMP::None;
+      fir::createHLFIRToFIRPassPipeline(pm, enableOmp,
                                         llvm::OptimizationLevel::O2);
       if (mlir::failed(pm.run(mlirModule))) {
         llvm::errs() << "FATAL: lowering from HLFIR to FIR failed";


From flang-commits at lists.llvm.org  Wed Jul 23 10:21:25 2025
From: flang-commits at lists.llvm.org (Asher Mancinelli via flang-commits)
Date: Wed, 23 Jul 2025 10:21:25 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Extend symbol update to
 ArrayAttrext in `external-name-interop` (PR #150061)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150061@github.com>
Message-ID: <68811a15.050a0220.3ba056.bea9@mx.google.com>


ashermancinelli wrote:

Thank you Delaram! Merging at Slava's request, as he's on PTO.

https://github.com/llvm/llvm-project/pull/150061

From flang-commits at lists.llvm.org  Wed Jul 23 10:25:46 2025
From: flang-commits at lists.llvm.org (via flang-commits)
Date: Wed, 23 Jul 2025 10:25:46 -0700 (PDT)
Subject: [flang-commits] [flang] 32c9854 - [flang] Extend symbol update to
 ArrayAttrext in `external-name-interop` (#150061)
Message-ID: <68811b1a.050a0220.951e.c212@mx.google.com>


Author: Delaram Talaashrafi
Date: 2025-07-23T13:25:43-04:00
New Revision: 32c985485500b214d57cb25306734eb73833d59b

URL: https://github.com/llvm/llvm-project/commit/32c985485500b214d57cb25306734eb73833d59b
DIFF: https://github.com/llvm/llvm-project/commit/32c985485500b214d57cb25306734eb73833d59b.diff

LOG: [flang] Extend symbol update to ArrayAttrext in `external-name-interop` (#150061)

In the `external-name-interop` pass, when a symbol is changed, all the
uses of the renamed symbols should also be updated. The update was only
applied to the `SymbolRefAttr` type. With this change, the update will
be applied to `ArrayAttr` containing elements of type `SymbolRefAttr`.

---------

Co-authored-by: Delaram Talaashrafi <dtalaashrafi at nvidia.com>

Added: 
    flang/test/Transforms/external-name-interop-symref-array.fir

Modified: 
    flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp

Removed: 
    


################################################################################
diff  --git a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
index 3d84eaa4c1595..2fcff87fdc393 100644
--- a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
@@ -41,6 +41,23 @@ mangleExternalName(const std::pair<fir::NameUniquer::NameKind,
                                                   appendUnderscore);
 }
 
+/// Process a symbol reference and return the updated symbol reference if
+/// needed.
+std::optional<mlir::SymbolRefAttr>
+processSymbolRef(mlir::SymbolRefAttr symRef, mlir::Operation *nestedOp,
+                 const llvm::DenseMap<mlir::StringAttr, mlir::FlatSymbolRefAttr>
+                     &remappings) {
+  if (auto remap = remappings.find(symRef.getLeafReference());
+      remap != remappings.end()) {
+    mlir::SymbolRefAttr symAttr = mlir::FlatSymbolRefAttr(remap->second);
+    if (mlir::isa<mlir::gpu::LaunchFuncOp>(nestedOp))
+      symAttr = mlir::SymbolRefAttr::get(
+          symRef.getRootReference(), {mlir::FlatSymbolRefAttr(remap->second)});
+    return symAttr;
+  }
+  return std::nullopt;
+}
+
 namespace {
 
 class ExternalNameConversionPass
@@ -97,21 +114,40 @@ void ExternalNameConversionPass::runOnOperation() {
 
   // Update all uses of the functions and globals that have been renamed.
   op.walk([&remappings](mlir::Operation *nestedOp) {
-    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::SymbolRefAttr>> updates;
+    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::SymbolRefAttr>>
+        symRefUpdates;
+    llvm::SmallVector<std::pair<mlir::StringAttr, mlir::ArrayAttr>>
+        arrayUpdates;
     for (const mlir::NamedAttribute &attr : nestedOp->getAttrDictionary())
       if (auto symRef = llvm::dyn_cast<mlir::SymbolRefAttr>(attr.getValue())) {
-        if (auto remap = remappings.find(symRef.getLeafReference());
-            remap != remappings.end()) {
-          mlir::SymbolRefAttr symAttr = mlir::FlatSymbolRefAttr(remap->second);
-          if (mlir::isa<mlir::gpu::LaunchFuncOp>(nestedOp))
-            symAttr = mlir::SymbolRefAttr::get(
-                symRef.getRootReference(),
-                {mlir::FlatSymbolRefAttr(remap->second)});
-          updates.emplace_back(std::pair<mlir::StringAttr, mlir::SymbolRefAttr>{
-              attr.getName(), symAttr});
+        if (auto newSymRef = processSymbolRef(symRef, nestedOp, remappings))
+          symRefUpdates.emplace_back(
+              std::pair<mlir::StringAttr, mlir::SymbolRefAttr>{attr.getName(),
+                                                               *newSymRef});
+      } else if (auto arrayAttr =
+                     llvm::dyn_cast<mlir::ArrayAttr>(attr.getValue())) {
+        llvm::SmallVector<mlir::Attribute> symbolRefs;
+        for (auto element : arrayAttr) {
+          if (!element) {
+            symbolRefs.push_back(element);
+            continue;
+          }
+          auto symRef = llvm::dyn_cast<mlir::SymbolRefAttr>(element);
+          std::optional<mlir::SymbolRefAttr> updatedSymRef;
+          if (symRef)
+            updatedSymRef = processSymbolRef(symRef, nestedOp, remappings);
+          if (!symRef || !updatedSymRef)
+            symbolRefs.push_back(element);
+          else
+            symbolRefs.push_back(*updatedSymRef);
         }
+        arrayUpdates.push_back(std::make_pair(
+            attr.getName(),
+            mlir::ArrayAttr::get(nestedOp->getContext(), symbolRefs)));
       }
-    for (auto update : updates)
+    for (auto update : symRefUpdates)
+      nestedOp->setAttr(update.first, update.second);
+    for (auto update : arrayUpdates)
       nestedOp->setAttr(update.first, update.second);
   });
 }

diff  --git a/flang/test/Transforms/external-name-interop-symref-array.fir b/flang/test/Transforms/external-name-interop-symref-array.fir
new file mode 100644
index 0000000000000..ce14f17fbf30a
--- /dev/null
+++ b/flang/test/Transforms/external-name-interop-symref-array.fir
@@ -0,0 +1,42 @@
+// Test fir.do_concurrent.loop operation with array of symbol reference attributes
+// This test demonstrates operations that have ArrayAttr containing SymbolRefAttr elements
+
+// RUN: fir-opt %s --external-name-interop | fir-opt | FileCheck %s
+
+// Define reduction operations that will be referenced in the symbol array
+func.func @_QPadd_reduction_i32_init(%arg0: i32, %arg1: !fir.ref<i32>) {
+  %0 = arith.constant 0 : i32
+  fir.store %0 to %arg1 : !fir.ref<i32>
+  return
+}
+
+func.func @_QPadd_reduction_i32_combiner(%arg0: i32, %arg1: i32) -> i32 {
+  %0 = arith.addi %arg0, %arg1 : i32
+  return %0 : i32
+}
+
+// Define a local privatizer that will be referenced in local_syms
+func.func @_QPlocal_var_privatizer(%arg0: !fir.ref<i32>) -> !fir.ref<i32> {
+  return %arg0 : !fir.ref<i32>
+}
+
+// Test function demonstrating both local_syms and reduce_syms arrays
+func.func @_QPtest_symbol_arrays(%i_lb: index, %i_ub: index, %i_st: index) {
+  %local_var = fir.alloca i32
+  %sum = fir.alloca i32
+  
+  fir.do_concurrent {
+    %i = fir.alloca i32
+    fir.do_concurrent.loop (%i_iv) = (%i_lb) to (%i_ub) step (%i_st)
+      local(@_QPlocal_var_privatizer %local_var -> %local_arg : !fir.ref<i32>)
+      reduce(@_QPadd_reduction_i32_init #fir.reduce_attr<add> %sum -> %sum_arg : !fir.ref<i32>) {
+      %0 = fir.convert %i_iv : (index) -> i32
+      fir.store %0 to %i : !fir.ref<i32>
+    }
+  }
+  return
+}
+
+// CHECK: local(@local_var_privatizer_
+// CHECK: reduce(@add_reduction_i32_init_
+


        

From flang-commits at lists.llvm.org  Wed Jul 23 10:25:50 2025
From: flang-commits at lists.llvm.org (Asher Mancinelli via flang-commits)
Date: Wed, 23 Jul 2025 10:25:50 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Extend symbol update to
 ArrayAttrext in `external-name-interop` (PR #150061)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150061@github.com>
Message-ID: <68811b1e.170a0220.f8347.e5e9@mx.google.com>


https://github.com/ashermancinelli closed https://github.com/llvm/llvm-project/pull/150061

From flang-commits at lists.llvm.org  Wed Jul 23 10:26:22 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 10:26:22 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch both EXTERNAL and INTRINSIC
 on type declaration stmt (PR #150254)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150254@github.com>
Message-ID: <68811b3e.170a0220.282fba.e180@mx.google.com>


================
@@ -2351,7 +2351,8 @@ bool AttrsVisitor::IsConflictingAttr(Attr attrName) {
       HaveAttrConflict(attrName, Attr::PASS, Attr::NOPASS) || // C781
       HaveAttrConflict(attrName, Attr::PURE, Attr::IMPURE) ||
       HaveAttrConflict(attrName, Attr::PUBLIC, Attr::PRIVATE) ||
-      HaveAttrConflict(attrName, Attr::RECURSIVE, Attr::NON_RECURSIVE);
+      HaveAttrConflict(attrName, Attr::RECURSIVE, Attr::NON_RECURSIVE) ||
+      HaveAttrConflict(attrName, Attr::INTRINSIC, Attr::EXTERNAL);
----------------
c8ef wrote:

Nit: Add a comment stating it violates C842(F2023)?

https://github.com/llvm/llvm-project/pull/150254

From flang-commits at lists.llvm.org  Wed Jul 23 10:27:56 2025
From: flang-commits at lists.llvm.org (Connector Switch via flang-commits)
Date: Wed, 23 Jul 2025 10:27:56 -0700 (PDT)
Subject: [flang-commits] [flang] [flang] Catch both EXTERNAL and INTRINSIC
 on type declaration stmt (PR #150254)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150254@github.com>
Message-ID: <68811b9c.170a0220.3785a5.e5a3@mx.google.com>


https://github.com/c8ef edited https://github.com/llvm/llvm-project/pull/150254

From flang-commits at lists.llvm.org  Wed Jul 23 10:41:06 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Wed, 23 Jul 2025 10:41:06 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150269@github.com>
Message-ID: <68811eb2.170a0220.246fdf.eb7d@mx.google.com>


================
@@ -3706,14 +3706,19 @@ def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
 def fnoopenmp_relocatable_target : Flag<["-"], "fnoopenmp-relocatable-target">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>,
   Visibility<[ClangOption, CC1Option]>;
-def fopenmp_simd : Flag<["-"], "fopenmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
+def fopenmp_simd : Flag<["-"], "fopenmp-simd">,
+                   Group<f_Group>,
+                   Flags<[NoArgumentUnused]>,
+                   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+                   HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
 def fopenmp_enable_irbuilder : Flag<["-"], "fopenmp-enable-irbuilder">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Use the experimental OpenMP-IR-Builder codegen path.">;
-def fno_openmp_simd : Flag<["-"], "fno-openmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>;
+def fno_openmp_simd
+    : Flag<["-"], "fno-openmp-simd">,
+      Group<f_Group>,
+      Flags<[NoArgumentUnused]>,
+      Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
----------------
tarunprabhu wrote:

Could we add some help text here? Specifically, I am not sure if this option means

- Disable OpenMP simd constructs *only*, but allow all other OpenMP constructs

OR 

- Invert the -fopenmp-simd flag i.e. emit OpenMP code for *all* OpenMP constructs, *including* simd

https://github.com/llvm/llvm-project/pull/150269

From flang-commits at lists.llvm.org  Wed Jul 23 10:41:06 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Wed, 23 Jul 2025 10:41:06 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150269@github.com>
Message-ID: <68811eb2.170a0220.3a4137.e9e9@mx.google.com>


https://github.com/tarunprabhu commented:

Thanks. The frontend parts of it look fine. I haven't looked too closely at the core OpenMP changes.

https://github.com/llvm/llvm-project/pull/150269

From flang-commits at lists.llvm.org  Wed Jul 23 10:41:07 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Wed, 23 Jul 2025 10:41:07 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150269@github.com>
Message-ID: <68811eb3.050a0220.2cee37.1e2f@mx.google.com>


================
@@ -952,6 +954,9 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
           << A->getSpelling() << A->getValue();
       break;
     }
+  } else {
+    Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
----------------
tarunprabhu wrote:

If I understand this correctly, `-fopenmp-simd` is effectively another way to enable OpenMP, just that it only enables the `simd` constructs. Is that correct?

https://github.com/llvm/llvm-project/pull/150269

From flang-commits at lists.llvm.org  Wed Jul 23 10:41:06 2025
From: flang-commits at lists.llvm.org (Tarun Prabhu via flang-commits)
Date: Wed, 23 Jul 2025 10:41:06 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150269@github.com>
Message-ID: <68811eb2.170a0220.99ec4.b58d@mx.google.com>


https://github.com/tarunprabhu edited https://github.com/llvm/llvm-project/pull/150269

From flang-commits at lists.llvm.org  Wed Jul 23 10:50:00 2025
From: flang-commits at lists.llvm.org (Kajetan Puchalski via flang-commits)
Date: Wed, 23 Jul 2025 10:50:00 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150269@github.com>
Message-ID: <688120c8.170a0220.20a137.e806@mx.google.com>


================
@@ -3706,14 +3706,19 @@ def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
 def fnoopenmp_relocatable_target : Flag<["-"], "fnoopenmp-relocatable-target">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>,
   Visibility<[ClangOption, CC1Option]>;
-def fopenmp_simd : Flag<["-"], "fopenmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
+def fopenmp_simd : Flag<["-"], "fopenmp-simd">,
+                   Group<f_Group>,
+                   Flags<[NoArgumentUnused]>,
+                   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+                   HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
 def fopenmp_enable_irbuilder : Flag<["-"], "fopenmp-enable-irbuilder">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Use the experimental OpenMP-IR-Builder codegen path.">;
-def fno_openmp_simd : Flag<["-"], "fno-openmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>;
+def fno_openmp_simd
+    : Flag<["-"], "fno-openmp-simd">,
+      Group<f_Group>,
+      Flags<[NoArgumentUnused]>,
+      Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
----------------
mrkajetanp wrote:

The terminology does get a little confusing, I'll try to  think of a way to make it clearer. A good way to think about it is "openmp in simd-only mode", so -fno-openmp-simd just means "do not enable openmp in simd-only mode". The same as not passing any flags. That's how it's designed in other compilers at least based on my testing.

https://github.com/llvm/llvm-project/pull/150269

From flang-commits at lists.llvm.org  Wed Jul 23 10:55:16 2025
From: flang-commits at lists.llvm.org (Kajetan Puchalski via flang-commits)
Date: Wed, 23 Jul 2025 10:55:16 -0700 (PDT)
Subject: [flang-commits] [clang] [flang] [flang][OpenMP] Add
	-f[no]-openmp-simd (PR #150269)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/150269@github.com>
Message-ID: <68812204.170a0220.365237.e805@mx.google.com>


================
@@ -952,6 +954,9 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
           << A->getSpelling() << A->getValue();
       break;
     }
+  } else {
+    Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
----------------
mrkajetanp wrote:

Yes, that's correct. If openmp is already enabled with -fopenmp then it should have no effect.

https://github.com/llvm/llvm-project/pull/150269